From 99309092e508ed6b2f91a1bfe87d305ad98074fb Mon Sep 17 00:00:00 2001 From: Martin Gaievski Date: Tue, 12 Jul 2022 17:23:57 -0700 Subject: [PATCH 001/416] Refactor knn type and codecs (#439) (#443) * Refactor knn type and codecs Signed-off-by: Martin Gaievski --- .../org/opensearch/knn/index/KNNMethod.java | 33 ++----------- .../knn/index/KNNMethodContext.java | 48 ++---------------- .../knn/index/KNNVectorFieldMapper.java | 27 +++++----- .../opensearch/knn/index/MethodComponent.java | 21 ++------ .../knn/index/MethodComponentContext.java | 28 ++--------- .../knn/index/codec/KNNCodecFactory.java | 23 +++++---- .../knn/index/codec/KNNCodecService.java | 5 +- .../knn/index/codec/util/CodecBuilder.java | 49 +++++++++++++++++++ .../opensearch/knn/training/TrainingJob.java | 2 +- .../knn/index/KNNMethodContextTests.java | 4 +- .../knn/index/codec/KNNCodecFactoryTests.java | 11 ++++- .../index/codec/KNNFormatFactoryTests.java | 7 ++- .../opensearch/knn/jni/JNIServiceTests.java | 2 +- .../plugin/script/KNNScoringSpaceTests.java | 25 ++++++++-- .../knn/training/TrainingJobTests.java | 4 +- 15 files changed, 137 insertions(+), 152 deletions(-) create mode 100644 src/main/java/org/opensearch/knn/index/codec/util/CodecBuilder.java diff --git a/src/main/java/org/opensearch/knn/index/KNNMethod.java b/src/main/java/org/opensearch/knn/index/KNNMethod.java index da2d9c455..d46e8fccd 100644 --- a/src/main/java/org/opensearch/knn/index/KNNMethod.java +++ b/src/main/java/org/opensearch/knn/index/KNNMethod.java @@ -11,6 +11,8 @@ package org.opensearch.knn.index; +import lombok.AllArgsConstructor; +import lombok.Getter; import org.opensearch.common.ValidationException; import org.opensearch.knn.common.KNNConstants; @@ -26,31 +28,13 @@ * KNNMethod is used to define the structure of a method supported by a particular k-NN library. It is used to validate * the KNNMethodContext passed in by the user. It is also used to provide superficial string translations. */ +@AllArgsConstructor +@Getter public class KNNMethod { private final MethodComponent methodComponent; private final Set spaces; - /** - * KNNMethod Constructor - * - * @param methodComponent top level method component that is compatible with the underlying library - * @param spaces set of valid space types that the method supports - */ - public KNNMethod(MethodComponent methodComponent, Set spaces) { - this.methodComponent = methodComponent; - this.spaces = spaces; - } - - /** - * getMainMethodComponent - * - * @return mainMethodComponent - */ - public MethodComponent getMethodComponent() { - return methodComponent; - } - /** * Determines whether the provided space is supported for this method * @@ -61,15 +45,6 @@ public boolean containsSpace(SpaceType space) { return spaces.contains(space); } - /** - * Get all valid spaces for this method - * - * @return spaces that can be used with this method - */ - public Set getSpaces() { - return spaces; - } - /** * Validate that the configured KNNMethodContext is valid for this method * diff --git a/src/main/java/org/opensearch/knn/index/KNNMethodContext.java b/src/main/java/org/opensearch/knn/index/KNNMethodContext.java index cd7b657f0..2d6933877 100644 --- a/src/main/java/org/opensearch/knn/index/KNNMethodContext.java +++ b/src/main/java/org/opensearch/knn/index/KNNMethodContext.java @@ -11,8 +11,8 @@ package org.opensearch.knn.index; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; +import lombok.AllArgsConstructor; +import lombok.Getter; import org.opensearch.common.ValidationException; import org.opensearch.common.io.stream.StreamInput; import org.opensearch.common.io.stream.StreamOutput; @@ -40,10 +40,10 @@ * KNNMethodContext will contain the information necessary to produce a library index from an Opensearch mapping. * It will encompass all parameters necessary to build the index. */ +@AllArgsConstructor +@Getter public class KNNMethodContext implements ToXContentFragment, Writeable { - private static final Logger logger = LogManager.getLogger(KNNMethodContext.class); - private static KNNMethodContext defaultInstance = null; public static synchronized KNNMethodContext getDefault() { @@ -61,19 +61,6 @@ public static synchronized KNNMethodContext getDefault() { private final SpaceType spaceType; private final MethodComponentContext methodComponent; - /** - * Constructor - * - * @param knnEngine engine that this method uses - * @param spaceType space type that this method uses - * @param methodComponent MethodComponent describing the main index - */ - public KNNMethodContext(KNNEngine knnEngine, SpaceType spaceType, MethodComponentContext methodComponent) { - this.knnEngine = knnEngine; - this.spaceType = spaceType; - this.methodComponent = methodComponent; - } - /** * Constructor from stream. * @@ -86,33 +73,6 @@ public KNNMethodContext(StreamInput in) throws IOException { this.methodComponent = new MethodComponentContext(in); } - /** - * Gets the main method component - * - * @return methodComponent - */ - public MethodComponentContext getMethodComponent() { - return methodComponent; - } - - /** - * Gets the engine to be used for this context - * - * @return knnEngine - */ - public KNNEngine getEngine() { - return knnEngine; - } - - /** - * Gets the space type for this context - * - * @return spaceType - */ - public SpaceType getSpaceType() { - return spaceType; - } - /** * This method uses the knnEngine to validate that the method is compatible with the engine * diff --git a/src/main/java/org/opensearch/knn/index/KNNVectorFieldMapper.java b/src/main/java/org/opensearch/knn/index/KNNVectorFieldMapper.java index 020e5ae25..628995b5c 100644 --- a/src/main/java/org/opensearch/knn/index/KNNVectorFieldMapper.java +++ b/src/main/java/org/opensearch/knn/index/KNNVectorFieldMapper.java @@ -5,6 +5,7 @@ package org.opensearch.knn.index; +import lombok.Getter; import org.opensearch.common.Strings; import org.opensearch.common.ValidationException; import org.opensearch.common.xcontent.XContentFactory; @@ -206,7 +207,7 @@ public KNNVectorFieldMapper build(BuilderContext context) { if (knnMethodContext != null) { return new MethodFieldMapper( name, - new KNNVectorFieldType(buildFullName(context), meta.getValue(), dimension.getValue()), + new KNNVectorFieldType(buildFullName(context), meta.getValue(), dimension.getValue(), knnMethodContext), multiFieldsBuilder.build(this, context), copyTo.build(), ignoreMalformed(context), @@ -225,7 +226,7 @@ public KNNVectorFieldMapper build(BuilderContext context) { return new ModelFieldMapper( name, - new KNNVectorFieldType(buildFullName(context), meta.getValue(), -1, modelIdAsString), + new KNNVectorFieldType(buildFullName(context), meta.getValue(), -1, knnMethodContext, modelIdAsString), multiFieldsBuilder.build(this, context), copyTo.build(), ignoreMalformed(context), @@ -296,19 +297,25 @@ public Mapper.Builder parse(String name, Map node, ParserCont } } + @Getter public static class KNNVectorFieldType extends MappedFieldType { - int dimension; String modelId; + KNNMethodContext knnMethodContext; public KNNVectorFieldType(String name, Map meta, int dimension) { - this(name, meta, dimension, null); + this(name, meta, dimension, null, null); + } + + public KNNVectorFieldType(String name, Map meta, int dimension, KNNMethodContext knnMethodContext) { + this(name, meta, dimension, knnMethodContext, null); } - public KNNVectorFieldType(String name, Map meta, int dimension, String modelId) { + public KNNVectorFieldType(String name, Map meta, int dimension, KNNMethodContext knnMethodContext, String modelId) { super(name, false, false, true, TextSearchInfo.NONE, meta); this.dimension = dimension; this.modelId = modelId; + this.knnMethodContext = knnMethodContext; } @Override @@ -334,14 +341,6 @@ public Query termQuery(Object value, QueryShardContext context) { ); } - public int getDimension() { - return dimension; - } - - public String getModelId() { - return modelId; - } - @Override public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier searchLookup) { failIfNoDocValues(); @@ -623,7 +622,7 @@ private MethodFieldMapper( this.fieldType.putAttribute(DIMENSION, String.valueOf(dimension)); this.fieldType.putAttribute(SPACE_TYPE, knnMethodContext.getSpaceType().getValue()); - KNNEngine knnEngine = knnMethodContext.getEngine(); + KNNEngine knnEngine = knnMethodContext.getKnnEngine(); this.fieldType.putAttribute(KNN_ENGINE, knnEngine.getName()); try { diff --git a/src/main/java/org/opensearch/knn/index/MethodComponent.java b/src/main/java/org/opensearch/knn/index/MethodComponent.java index d7957d74f..c904f15c3 100644 --- a/src/main/java/org/opensearch/knn/index/MethodComponent.java +++ b/src/main/java/org/opensearch/knn/index/MethodComponent.java @@ -11,6 +11,7 @@ package org.opensearch.knn.index; +import lombok.Getter; import org.opensearch.common.TriFunction; import org.opensearch.common.ValidationException; import org.opensearch.knn.common.KNNConstants; @@ -26,7 +27,9 @@ */ public class MethodComponent { + @Getter private String name; + @Getter private Map> parameters; private BiFunction> mapGenerator; private TriFunction overheadInKBEstimator; @@ -45,24 +48,6 @@ private MethodComponent(Builder builder) { this.requiresTraining = builder.requiresTraining; } - /** - * Get the name of the component - * - * @return name - */ - public String getName() { - return name; - } - - /** - * Get the parameters for the component - * - * @return parameters - */ - public Map> getParameters() { - return parameters; - } - /** * Parse methodComponentContext into a map that the library can use to configure the method * diff --git a/src/main/java/org/opensearch/knn/index/MethodComponentContext.java b/src/main/java/org/opensearch/knn/index/MethodComponentContext.java index c43c9f77d..4a5e2377a 100644 --- a/src/main/java/org/opensearch/knn/index/MethodComponentContext.java +++ b/src/main/java/org/opensearch/knn/index/MethodComponentContext.java @@ -11,8 +11,8 @@ package org.opensearch.knn.index; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; +import lombok.AllArgsConstructor; +import lombok.Getter; import org.opensearch.common.io.stream.StreamInput; import org.opensearch.common.io.stream.StreamOutput; import org.opensearch.common.io.stream.Writeable; @@ -36,24 +36,13 @@ * * Each component is composed of a name and a map of parameters. */ +@AllArgsConstructor public class MethodComponentContext implements ToXContentFragment, Writeable { - private static final Logger logger = LogManager.getLogger(MethodComponentContext.class); - + @Getter private final String name; private final Map parameters; - /** - * Constructor - * - * @param name component name - * @param parameters component parameters - */ - public MethodComponentContext(String name, Map parameters) { - this.name = name; - this.parameters = parameters; - } - /** * Constructor from stream. * @@ -183,15 +172,6 @@ public int hashCode() { return new HashCodeBuilder().append(name).append(parameters).toHashCode(); } - /** - * Gets the name of the component - * - * @return name - */ - public String getName() { - return name; - } - /** * Gets the parameters of the component * diff --git a/src/main/java/org/opensearch/knn/index/codec/KNNCodecFactory.java b/src/main/java/org/opensearch/knn/index/codec/KNNCodecFactory.java index 28b0e528a..8af3f70d6 100644 --- a/src/main/java/org/opensearch/knn/index/codec/KNNCodecFactory.java +++ b/src/main/java/org/opensearch/knn/index/codec/KNNCodecFactory.java @@ -7,9 +7,9 @@ import com.google.common.collect.ImmutableMap; import org.apache.lucene.codecs.Codec; import org.apache.lucene.backward_codecs.lucene91.Lucene91Codec; -import org.opensearch.knn.index.codec.KNN910Codec.KNN910Codec; +import org.opensearch.index.mapper.MapperService; +import org.opensearch.knn.index.codec.util.CodecBuilder; -import java.lang.reflect.Constructor; import java.util.Map; /** @@ -17,23 +17,26 @@ */ public class KNNCodecFactory { - private static Map CODEC_BY_VERSION = ImmutableMap.of(KNNCodecVersion.KNN910, KNN910Codec.class); + private final Map codecByVersion; - private static KNNCodecVersion LATEST_KNN_CODEC_VERSION = KNNCodecVersion.KNN910; + private static final KNNCodecVersion LATEST_KNN_CODEC_VERSION = KNNCodecVersion.KNN910; - public static Codec createKNNCodec(final Codec userCodec) { + public KNNCodecFactory(MapperService mapperService) { + codecByVersion = ImmutableMap.of(KNNCodecVersion.KNN910, new CodecBuilder.KNN91CodecBuilder(mapperService)); + } + + public Codec createKNNCodec(final Codec userCodec) { return getCodec(LATEST_KNN_CODEC_VERSION, userCodec); } - public static Codec createKNNCodec(final KNNCodecVersion knnCodecVersion, final Codec userCodec) { + public Codec createKNNCodec(final KNNCodecVersion knnCodecVersion, final Codec userCodec) { return getCodec(knnCodecVersion, userCodec); } - private static Codec getCodec(final KNNCodecVersion knnCodecVersion, final Codec userCodec) { + private Codec getCodec(final KNNCodecVersion knnCodecVersion, final Codec userCodec) { try { - Constructor constructor = CODEC_BY_VERSION.getOrDefault(knnCodecVersion, CODEC_BY_VERSION.get(LATEST_KNN_CODEC_VERSION)) - .getConstructor(Codec.class); - return (Codec) constructor.newInstance(userCodec); + final CodecBuilder codecBuilder = codecByVersion.getOrDefault(knnCodecVersion, codecByVersion.get(LATEST_KNN_CODEC_VERSION)); + return codecBuilder.userCodec(userCodec).build(); } catch (Exception ex) { throw new RuntimeException("Cannot create instance of KNN codec", ex); } diff --git a/src/main/java/org/opensearch/knn/index/codec/KNNCodecService.java b/src/main/java/org/opensearch/knn/index/codec/KNNCodecService.java index cae6f7fb8..8ce5e6928 100644 --- a/src/main/java/org/opensearch/knn/index/codec/KNNCodecService.java +++ b/src/main/java/org/opensearch/knn/index/codec/KNNCodecService.java @@ -14,8 +14,11 @@ */ public class KNNCodecService extends CodecService { + private final KNNCodecFactory knnCodecFactory; + public KNNCodecService(CodecServiceConfig codecServiceConfig) { super(codecServiceConfig.getMapperService(), codecServiceConfig.getLogger()); + knnCodecFactory = new KNNCodecFactory(codecServiceConfig.getMapperService()); } /** @@ -26,6 +29,6 @@ public KNNCodecService(CodecServiceConfig codecServiceConfig) { */ @Override public Codec codec(String name) { - return KNNCodecFactory.createKNNCodec(super.codec(name)); + return knnCodecFactory.createKNNCodec(super.codec(name)); } } diff --git a/src/main/java/org/opensearch/knn/index/codec/util/CodecBuilder.java b/src/main/java/org/opensearch/knn/index/codec/util/CodecBuilder.java new file mode 100644 index 000000000..e909b07f2 --- /dev/null +++ b/src/main/java/org/opensearch/knn/index/codec/util/CodecBuilder.java @@ -0,0 +1,49 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.knn.index.codec.util; + +import lombok.AllArgsConstructor; +import org.apache.lucene.codecs.Codec; +import org.opensearch.index.mapper.MapperService; +import org.opensearch.knn.index.codec.KNN910Codec.KNN910Codec; + +/** + * Abstracts builder logic for plugin codecs. For each codec we need to set delegate that is typically + * Lucene codec implementation and this is made part of the base class. Exact builder implementation may add + * additional parameters required to build a codec. + */ +public abstract class CodecBuilder { + Codec userCodec; + + /** + * Set user defined codec for plugin + * @param userCodec + * @return + */ + public CodecBuilder userCodec(Codec userCodec) { + this.userCodec = userCodec; + return this; + } + + /** + * Builds instance of codec, implementation is specific for each codec version + * @return instance of codec + */ + public abstract Codec build(); + + /** + * Implements builder abstraction for KNN91Codec, adds MapperService that may be required to build + * per field format based on field mapper type + */ + @AllArgsConstructor + public static class KNN91CodecBuilder extends CodecBuilder { + private final MapperService mapperService; + + @Override + public Codec build() { + return new KNN910Codec(userCodec); + } + } +} diff --git a/src/main/java/org/opensearch/knn/training/TrainingJob.java b/src/main/java/org/opensearch/knn/training/TrainingJob.java index c83c69831..e01688ddc 100644 --- a/src/main/java/org/opensearch/knn/training/TrainingJob.java +++ b/src/main/java/org/opensearch/knn/training/TrainingJob.java @@ -75,7 +75,7 @@ public TrainingJob( this.modelAnonymousEntryContext = Objects.requireNonNull(modelAnonymousEntryContext, "AnonymousEntryContext cannot be null."); this.model = new Model( new ModelMetadata( - knnMethodContext.getEngine(), + knnMethodContext.getKnnEngine(), knnMethodContext.getSpaceType(), dimension, ModelState.TRAINING, diff --git a/src/test/java/org/opensearch/knn/index/KNNMethodContextTests.java b/src/test/java/org/opensearch/knn/index/KNNMethodContextTests.java index 176029ad6..81816daf1 100644 --- a/src/test/java/org/opensearch/knn/index/KNNMethodContextTests.java +++ b/src/test/java/org/opensearch/knn/index/KNNMethodContextTests.java @@ -74,7 +74,7 @@ public void testGetMethodComponent() { public void testGetEngine() { MethodComponentContext methodComponent = new MethodComponentContext("test-method", Collections.emptyMap()); KNNMethodContext knnMethodContext = new KNNMethodContext(KNNEngine.DEFAULT, SpaceType.DEFAULT, methodComponent); - assertEquals(KNNEngine.DEFAULT, knnMethodContext.getEngine()); + assertEquals(KNNEngine.DEFAULT, knnMethodContext.getKnnEngine()); } /** @@ -265,7 +265,7 @@ public void testParse_valid() throws IOException { Map in = xContentBuilderToMap(xContentBuilder); KNNMethodContext knnMethodContext = KNNMethodContext.parse(in); - assertEquals(KNNEngine.DEFAULT, knnMethodContext.getEngine()); + assertEquals(KNNEngine.DEFAULT, knnMethodContext.getKnnEngine()); assertEquals(SpaceType.DEFAULT, knnMethodContext.getSpaceType()); assertEquals(methodName, knnMethodContext.getMethodComponent().getName()); assertTrue(knnMethodContext.getMethodComponent().getParameters().isEmpty()); diff --git a/src/test/java/org/opensearch/knn/index/codec/KNNCodecFactoryTests.java b/src/test/java/org/opensearch/knn/index/codec/KNNCodecFactoryTests.java index c2ab1b5d9..f42405e5c 100644 --- a/src/test/java/org/opensearch/knn/index/codec/KNNCodecFactoryTests.java +++ b/src/test/java/org/opensearch/knn/index/codec/KNNCodecFactoryTests.java @@ -7,9 +7,12 @@ import org.apache.lucene.codecs.Codec; import org.apache.lucene.backward_codecs.lucene91.Lucene91Codec; +import org.opensearch.index.mapper.MapperService; import org.opensearch.knn.KNNTestCase; import org.opensearch.knn.index.codec.KNN910Codec.KNN910Codec; +import static org.mockito.Mockito.mock; + public class KNNCodecFactoryTests extends KNNTestCase { public void testKNN91DefaultDelegate() { @@ -20,14 +23,18 @@ public void testKNN91DefaultDelegate() { public void testKNN91DefaultCodec() { Lucene91Codec lucene91CodecDelegate = new Lucene91Codec(); - Codec knnCodec = KNNCodecFactory.createKNNCodec(lucene91CodecDelegate); + MapperService mapperService = mock(MapperService.class); + KNNCodecFactory knnCodecFactory = new KNNCodecFactory(mapperService); + Codec knnCodec = knnCodecFactory.createKNNCodec(lucene91CodecDelegate); assertNotNull(knnCodec); assertTrue(knnCodec instanceof KNN910Codec); } public void testKNN91CodecByVersion() { Lucene91Codec lucene91CodecDelegate = new Lucene91Codec(); - Codec knnCodec = KNNCodecFactory.createKNNCodec(KNNCodecFactory.KNNCodecVersion.KNN910, lucene91CodecDelegate); + MapperService mapperService = mock(MapperService.class); + KNNCodecFactory knnCodecFactory = new KNNCodecFactory(mapperService); + Codec knnCodec = knnCodecFactory.createKNNCodec(KNNCodecFactory.KNNCodecVersion.KNN910, lucene91CodecDelegate); assertNotNull(knnCodec); assertTrue(knnCodec instanceof KNN910Codec); } diff --git a/src/test/java/org/opensearch/knn/index/codec/KNNFormatFactoryTests.java b/src/test/java/org/opensearch/knn/index/codec/KNNFormatFactoryTests.java index 545e2a141..b50d03a8a 100644 --- a/src/test/java/org/opensearch/knn/index/codec/KNNFormatFactoryTests.java +++ b/src/test/java/org/opensearch/knn/index/codec/KNNFormatFactoryTests.java @@ -6,13 +6,18 @@ package org.opensearch.knn.index.codec; import org.apache.lucene.codecs.Codec; +import org.opensearch.index.mapper.MapperService; import org.opensearch.knn.KNNTestCase; +import static org.mockito.Mockito.mock; + public class KNNFormatFactoryTests extends KNNTestCase { public void testKNN91Format() { final Codec lucene91CodecDelegate = KNNCodecFactory.CodecDelegateFactory.createKNN91DefaultDelegate(); - final Codec knnCodec = KNNCodecFactory.createKNNCodec(lucene91CodecDelegate); + MapperService mapperService = mock(MapperService.class); + KNNCodecFactory knnCodecFactory = new KNNCodecFactory(mapperService); + final Codec knnCodec = knnCodecFactory.createKNNCodec(lucene91CodecDelegate); KNNFormatFacade knnFormatFacade = KNNFormatFactory.createKNN910Format(knnCodec); assertNotNull(knnFormatFacade); diff --git a/src/test/java/org/opensearch/knn/jni/JNIServiceTests.java b/src/test/java/org/opensearch/knn/jni/JNIServiceTests.java index d85c53e08..3464beff8 100644 --- a/src/test/java/org/opensearch/knn/jni/JNIServiceTests.java +++ b/src/test/java/org/opensearch/knn/jni/JNIServiceTests.java @@ -809,7 +809,7 @@ public void testCreateIndexFromTemplate() throws IOException { ) ); - String description = knnMethodContext.getEngine().getMethodAsMap(knnMethodContext).get(INDEX_DESCRIPTION_PARAMETER).toString(); + String description = knnMethodContext.getKnnEngine().getMethodAsMap(knnMethodContext).get(INDEX_DESCRIPTION_PARAMETER).toString(); assertEquals("IVF16,PQ16x8", description); Map parameters = ImmutableMap.of( diff --git a/src/test/java/org/opensearch/knn/plugin/script/KNNScoringSpaceTests.java b/src/test/java/org/opensearch/knn/plugin/script/KNNScoringSpaceTests.java index 090581970..b40cc9e86 100644 --- a/src/test/java/org/opensearch/knn/plugin/script/KNNScoringSpaceTests.java +++ b/src/test/java/org/opensearch/knn/plugin/script/KNNScoringSpaceTests.java @@ -6,6 +6,7 @@ package org.opensearch.knn.plugin.script; import org.opensearch.knn.KNNTestCase; +import org.opensearch.knn.index.KNNMethodContext; import org.opensearch.knn.index.KNNVectorFieldMapper; import org.opensearch.index.mapper.BinaryFieldMapper; import org.opensearch.index.mapper.NumberFieldMapper; @@ -25,7 +26,13 @@ public class KNNScoringSpaceTests extends KNNTestCase { public void testL2() { float[] arrayFloat = new float[] { 1.0f, 2.0f, 3.0f }; List arrayListQueryObject = new ArrayList<>(Arrays.asList(1.0, 2.0, 3.0)); - KNNVectorFieldMapper.KNNVectorFieldType fieldType = new KNNVectorFieldMapper.KNNVectorFieldType("test", Collections.emptyMap(), 3); + KNNMethodContext knnMethodContext = KNNMethodContext.getDefault(); + KNNVectorFieldMapper.KNNVectorFieldType fieldType = new KNNVectorFieldMapper.KNNVectorFieldType( + "test", + Collections.emptyMap(), + 3, + knnMethodContext + ); KNNScoringSpace.L2 l2 = new KNNScoringSpace.L2(arrayListQueryObject, fieldType); assertEquals(1F, l2.scoringMethod.apply(arrayFloat, arrayFloat), 0.1F); @@ -40,8 +47,14 @@ public void testCosineSimilarity() { float[] arrayFloat = new float[] { 1.0f, 2.0f, 3.0f }; List arrayListQueryObject = new ArrayList<>(Arrays.asList(1.0, 2.0, 3.0)); float[] arrayFloat2 = new float[] { 2.0f, 4.0f, 6.0f }; + KNNMethodContext knnMethodContext = KNNMethodContext.getDefault(); - KNNVectorFieldMapper.KNNVectorFieldType fieldType = new KNNVectorFieldMapper.KNNVectorFieldType("test", Collections.emptyMap(), 3); + KNNVectorFieldMapper.KNNVectorFieldType fieldType = new KNNVectorFieldMapper.KNNVectorFieldType( + "test", + Collections.emptyMap(), + 3, + knnMethodContext + ); KNNScoringSpace.CosineSimilarity cosineSimilarity = new KNNScoringSpace.CosineSimilarity(arrayListQueryObject, fieldType); assertEquals(3F, cosineSimilarity.scoringMethod.apply(arrayFloat2, arrayFloat), 0.1F); @@ -57,8 +70,14 @@ public void testInnerProdSimilarity() { float[] arrayFloat_case1 = new float[] { 1.0f, 2.0f, 3.0f }; List arrayListQueryObject_case1 = new ArrayList<>(Arrays.asList(1.0, 2.0, 3.0)); float[] arrayFloat2_case1 = new float[] { 1.0f, 1.0f, 1.0f }; + KNNMethodContext knnMethodContext = KNNMethodContext.getDefault(); - KNNVectorFieldMapper.KNNVectorFieldType fieldType = new KNNVectorFieldMapper.KNNVectorFieldType("test", Collections.emptyMap(), 3); + KNNVectorFieldMapper.KNNVectorFieldType fieldType = new KNNVectorFieldMapper.KNNVectorFieldType( + "test", + Collections.emptyMap(), + 3, + knnMethodContext + ); KNNScoringSpace.InnerProd innerProd = new KNNScoringSpace.InnerProd(arrayListQueryObject_case1, fieldType); assertEquals(7.0F, innerProd.scoringMethod.apply(arrayFloat_case1, arrayFloat2_case1), 0.001F); diff --git a/src/test/java/org/opensearch/knn/training/TrainingJobTests.java b/src/test/java/org/opensearch/knn/training/TrainingJobTests.java index a4a9bda98..b15bf8207 100644 --- a/src/test/java/org/opensearch/knn/training/TrainingJobTests.java +++ b/src/test/java/org/opensearch/knn/training/TrainingJobTests.java @@ -42,7 +42,7 @@ public class TrainingJobTests extends KNNTestCase { public void testGetModelId() { String modelId = "test-model-id"; KNNMethodContext knnMethodContext = mock(KNNMethodContext.class); - when(knnMethodContext.getEngine()).thenReturn(KNNEngine.DEFAULT); + when(knnMethodContext.getKnnEngine()).thenReturn(KNNEngine.DEFAULT); when(knnMethodContext.getSpaceType()).thenReturn(SpaceType.DEFAULT); TrainingJob trainingJob = new TrainingJob( @@ -66,7 +66,7 @@ public void testGetModel() { String error = ""; KNNMethodContext knnMethodContext = mock(KNNMethodContext.class); - when(knnMethodContext.getEngine()).thenReturn(knnEngine); + when(knnMethodContext.getKnnEngine()).thenReturn(knnEngine); when(knnMethodContext.getSpaceType()).thenReturn(spaceType); String modelID = "test-model-id"; From 32395a3a1d93b540c1f74ee33ef24ba1c2a2d130 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Wed, 13 Jul 2022 09:43:28 -0700 Subject: [PATCH 002/416] Move engine and lib components into separate files (#445) Refactors library and engine structure to be less coupled. Pulls classes and interfaces out into individual files. Cleans up implementation. Co-authored-by: John Mazanec --- .../KNN80Codec/KNN80DocValuesConsumer.java | 14 +- .../org/opensearch/knn/index/util/Faiss.java | 327 ++++++++++ .../opensearch/knn/index/util/KNNEngine.java | 23 +- .../opensearch/knn/index/util/KNNLibrary.java | 589 +----------------- .../knn/index/util/NativeLibrary.java | 123 ++++ .../org/opensearch/knn/index/util/Nmslib.java | 71 +++ .../KNN80DocValuesConsumerTests.java | 28 +- .../opensearch/knn/index/util/FaissTests.java | 68 ++ .../knn/index/util/KNNEngineTests.java | 16 +- ...raryTests.java => NativeLibraryTests.java} | 130 +--- .../LibraryInitializedSupplierTests.java | 9 +- 11 files changed, 642 insertions(+), 756 deletions(-) create mode 100644 src/main/java/org/opensearch/knn/index/util/Faiss.java create mode 100644 src/main/java/org/opensearch/knn/index/util/NativeLibrary.java create mode 100644 src/main/java/org/opensearch/knn/index/util/Nmslib.java create mode 100644 src/test/java/org/opensearch/knn/index/util/FaissTests.java rename src/test/java/org/opensearch/knn/index/util/{KNNLibraryTests.java => NativeLibraryTests.java} (57%) diff --git a/src/main/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumer.java b/src/main/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumer.java index 36ffe16de..05a0873f7 100644 --- a/src/main/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumer.java +++ b/src/main/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumer.java @@ -99,12 +99,7 @@ public void addKNNBinaryField(FieldInfo field, DocValuesProducer valuesProducer) KNNEngine knnEngine = model.getModelMetadata().getKnnEngine(); - engineFileName = buildEngineFileName( - state.segmentInfo.name, - knnEngine.getLatestBuildVersion(), - field.name, - knnEngine.getExtension() - ); + engineFileName = buildEngineFileName(state.segmentInfo.name, knnEngine.getVersion(), field.name, knnEngine.getExtension()); indexPath = Paths.get(((FSDirectory) (FilterDirectory.unwrap(state.directory))).getDirectory().toString(), engineFileName) .toString(); @@ -119,12 +114,7 @@ public void addKNNBinaryField(FieldInfo field, DocValuesProducer valuesProducer) String engineName = field.attributes().getOrDefault(KNNConstants.KNN_ENGINE, KNNEngine.DEFAULT.getName()); KNNEngine knnEngine = KNNEngine.getEngine(engineName); - engineFileName = buildEngineFileName( - state.segmentInfo.name, - knnEngine.getLatestBuildVersion(), - field.name, - knnEngine.getExtension() - ); + engineFileName = buildEngineFileName(state.segmentInfo.name, knnEngine.getVersion(), field.name, knnEngine.getExtension()); indexPath = Paths.get(((FSDirectory) (FilterDirectory.unwrap(state.directory))).getDirectory().toString(), engineFileName) .toString(); diff --git a/src/main/java/org/opensearch/knn/index/util/Faiss.java b/src/main/java/org/opensearch/knn/index/util/Faiss.java new file mode 100644 index 000000000..6c091da03 --- /dev/null +++ b/src/main/java/org/opensearch/knn/index/util/Faiss.java @@ -0,0 +1,327 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.util; + +import com.google.common.collect.ImmutableMap; +import lombok.AllArgsConstructor; +import org.opensearch.knn.common.KNNConstants; +import org.opensearch.knn.index.KNNMethod; +import org.opensearch.knn.index.KNNSettings; +import org.opensearch.knn.index.MethodComponent; +import org.opensearch.knn.index.MethodComponentContext; +import org.opensearch.knn.index.Parameter; +import org.opensearch.knn.index.SpaceType; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.function.Function; + +import static org.opensearch.knn.common.KNNConstants.BYTES_PER_KILOBYTES; +import static org.opensearch.knn.common.KNNConstants.ENCODER_PARAMETER_PQ_CODE_COUNT_DEFAULT; +import static org.opensearch.knn.common.KNNConstants.ENCODER_PARAMETER_PQ_CODE_COUNT_LIMIT; +import static org.opensearch.knn.common.KNNConstants.ENCODER_PARAMETER_PQ_CODE_SIZE; +import static org.opensearch.knn.common.KNNConstants.ENCODER_PARAMETER_PQ_CODE_SIZE_DEFAULT; +import static org.opensearch.knn.common.KNNConstants.ENCODER_PARAMETER_PQ_CODE_SIZE_LIMIT; +import static org.opensearch.knn.common.KNNConstants.ENCODER_PARAMETER_PQ_M; +import static org.opensearch.knn.common.KNNConstants.FAISS_HNSW_DESCRIPTION; +import static org.opensearch.knn.common.KNNConstants.FAISS_IVF_DESCRIPTION; +import static org.opensearch.knn.common.KNNConstants.FAISS_PQ_DESCRIPTION; +import static org.opensearch.knn.common.KNNConstants.METHOD_ENCODER_PARAMETER; +import static org.opensearch.knn.common.KNNConstants.METHOD_HNSW; +import static org.opensearch.knn.common.KNNConstants.METHOD_IVF; +import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_EF_CONSTRUCTION; +import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_EF_SEARCH; +import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_M; +import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_NLIST; +import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_NLIST_DEFAULT; +import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_NLIST_LIMIT; +import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_NPROBES; +import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_NPROBES_DEFAULT; +import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_NPROBES_LIMIT; +import static org.opensearch.knn.common.KNNConstants.NAME; +import static org.opensearch.knn.common.KNNConstants.PARAMETERS; + +/** + * Implements NativeLibrary for the faiss native library + */ +class Faiss extends NativeLibrary { + + private final static String CURRENT_VERSION = "165"; + + // Map that overrides OpenSearch score translation by space type of scores returned by faiss + private final static Map> SCORE_TRANSLATIONS = ImmutableMap.of( + SpaceType.INNER_PRODUCT, + rawScore -> SpaceType.INNER_PRODUCT.scoreTranslation(-1 * rawScore) + ); + + // Define encoders supported by faiss + private final static MethodComponentContext ENCODER_DEFAULT = new MethodComponentContext( + KNNConstants.ENCODER_FLAT, + Collections.emptyMap() + ); + + // TODO: To think about in future: for PQ, if dimension is not divisible by code count, PQ will fail. Right now, + // we do not have a way to base validation off of dimension. Failure will happen during training in JNI. + private final static Map encoderComponents = ImmutableMap.of( + KNNConstants.ENCODER_FLAT, + MethodComponent.Builder.builder(KNNConstants.ENCODER_FLAT) + .setMapGenerator( + ((methodComponent, methodComponentContext) -> MethodAsMapBuilder.builder( + KNNConstants.FAISS_FLAT_DESCRIPTION, + methodComponent, + methodComponentContext + ).build()) + ) + .build(), + KNNConstants.ENCODER_PQ, + MethodComponent.Builder.builder(KNNConstants.ENCODER_PQ) + .addParameter( + ENCODER_PARAMETER_PQ_M, + new Parameter.IntegerParameter( + ENCODER_PARAMETER_PQ_M, + ENCODER_PARAMETER_PQ_CODE_COUNT_DEFAULT, + v -> v > 0 && v < ENCODER_PARAMETER_PQ_CODE_COUNT_LIMIT + ) + ) + .addParameter( + ENCODER_PARAMETER_PQ_CODE_SIZE, + new Parameter.IntegerParameter( + ENCODER_PARAMETER_PQ_CODE_SIZE, + ENCODER_PARAMETER_PQ_CODE_SIZE_DEFAULT, + v -> v > 0 && v < ENCODER_PARAMETER_PQ_CODE_SIZE_LIMIT + ) + ) + .setRequiresTraining(true) + .setMapGenerator( + ((methodComponent, methodComponentContext) -> MethodAsMapBuilder.builder( + FAISS_PQ_DESCRIPTION, + methodComponent, + methodComponentContext + ).addParameter(ENCODER_PARAMETER_PQ_M, "", "").addParameter(ENCODER_PARAMETER_PQ_CODE_SIZE, "x", "").build()) + ) + .setOverheadInKBEstimator((methodComponent, methodComponentContext, dimension) -> { + // Size estimate formula: (4 * d * 2^code_size) / 1024 + 1 + + // Get value of code size passed in by user + Object codeSizeObject = methodComponentContext.getParameters().get(ENCODER_PARAMETER_PQ_CODE_SIZE); + + // If not specified, get default value of code size + if (codeSizeObject == null) { + Parameter codeSizeParameter = methodComponent.getParameters().get(ENCODER_PARAMETER_PQ_CODE_SIZE); + if (codeSizeParameter == null) { + throw new IllegalStateException( + String.format("%s is not a valid parameter. This is a bug.", ENCODER_PARAMETER_PQ_CODE_SIZE) + ); + } + + codeSizeObject = codeSizeParameter.getDefaultValue(); + } + + if (!(codeSizeObject instanceof Integer)) { + throw new IllegalStateException(String.format("%s must be an integer.", ENCODER_PARAMETER_PQ_CODE_SIZE)); + } + + int codeSize = (Integer) codeSizeObject; + return ((4L * (1L << codeSize) * dimension) / BYTES_PER_KILOBYTES) + 1; + }) + .build() + ); + + // Define methods supported by faiss + private final static Map METHODS = ImmutableMap.of( + METHOD_HNSW, + KNNMethod.Builder.builder( + MethodComponent.Builder.builder(METHOD_HNSW) + .addParameter( + METHOD_PARAMETER_M, + new Parameter.IntegerParameter(METHOD_PARAMETER_M, KNNSettings.INDEX_KNN_DEFAULT_ALGO_PARAM_M, v -> v > 0) + ) + .addParameter( + METHOD_PARAMETER_EF_CONSTRUCTION, + new Parameter.IntegerParameter( + METHOD_PARAMETER_EF_CONSTRUCTION, + KNNSettings.INDEX_KNN_DEFAULT_ALGO_PARAM_EF_CONSTRUCTION, + v -> v > 0 + ) + ) + .addParameter( + METHOD_PARAMETER_EF_SEARCH, + new Parameter.IntegerParameter( + METHOD_PARAMETER_EF_SEARCH, + KNNSettings.INDEX_KNN_DEFAULT_ALGO_PARAM_EF_SEARCH, + v -> v > 0 + ) + ) + .addParameter( + METHOD_ENCODER_PARAMETER, + new Parameter.MethodComponentContextParameter(METHOD_ENCODER_PARAMETER, ENCODER_DEFAULT, encoderComponents) + ) + .setMapGenerator( + ((methodComponent, methodComponentContext) -> MethodAsMapBuilder.builder( + FAISS_HNSW_DESCRIPTION, + methodComponent, + methodComponentContext + ).addParameter(METHOD_PARAMETER_M, "", "").addParameter(METHOD_ENCODER_PARAMETER, ",", "").build()) + ) + .build() + ).addSpaces(SpaceType.L2, SpaceType.INNER_PRODUCT).build(), + METHOD_IVF, + KNNMethod.Builder.builder( + MethodComponent.Builder.builder(METHOD_IVF) + .addParameter( + METHOD_PARAMETER_NPROBES, + new Parameter.IntegerParameter( + METHOD_PARAMETER_NPROBES, + METHOD_PARAMETER_NPROBES_DEFAULT, + v -> v > 0 && v < METHOD_PARAMETER_NPROBES_LIMIT + ) + ) + .addParameter( + METHOD_PARAMETER_NLIST, + new Parameter.IntegerParameter( + METHOD_PARAMETER_NLIST, + METHOD_PARAMETER_NLIST_DEFAULT, + v -> v > 0 && v < METHOD_PARAMETER_NLIST_LIMIT + ) + ) + .addParameter( + METHOD_ENCODER_PARAMETER, + new Parameter.MethodComponentContextParameter(METHOD_ENCODER_PARAMETER, ENCODER_DEFAULT, encoderComponents) + ) + .setRequiresTraining(true) + .setMapGenerator( + ((methodComponent, methodComponentContext) -> MethodAsMapBuilder.builder( + FAISS_IVF_DESCRIPTION, + methodComponent, + methodComponentContext + ).addParameter(METHOD_PARAMETER_NLIST, "", "").addParameter(METHOD_ENCODER_PARAMETER, ",", "").build()) + ) + .setOverheadInKBEstimator((methodComponent, methodComponentContext, dimension) -> { + // Size estimate formula: (4 * nlists * d) / 1024 + 1 + + // Get value of nlists passed in by user + Object nlistObject = methodComponentContext.getParameters().get(METHOD_PARAMETER_NLIST); + + // If not specified, get default value of nlist + if (nlistObject == null) { + Parameter nlistParameter = methodComponent.getParameters().get(METHOD_PARAMETER_NLIST); + if (nlistParameter == null) { + throw new IllegalStateException( + String.format("%s is not a valid parameter. This is a bug.", METHOD_PARAMETER_NLIST) + ); + } + + nlistObject = nlistParameter.getDefaultValue(); + } + + if (!(nlistObject instanceof Integer)) { + throw new IllegalStateException(String.format("%s must be an integer.", METHOD_PARAMETER_NLIST)); + } + + int centroids = (Integer) nlistObject; + return ((4L * centroids * dimension) / BYTES_PER_KILOBYTES) + 1; + }) + .build() + ).addSpaces(SpaceType.L2, SpaceType.INNER_PRODUCT).build() + ); + + final static Faiss INSTANCE = new Faiss(METHODS, SCORE_TRANSLATIONS, CURRENT_VERSION, KNNConstants.FAISS_EXTENSION); + + /** + * Constructor for Faiss + * + * @param methods map of methods the native library supports + * @param scoreTranslation Map of translation of space type to scores returned by the library + * @param currentVersion String representation of current version of the library + * @param extension String representing the extension that library files should use + */ + private Faiss( + Map methods, + Map> scoreTranslation, + String currentVersion, + String extension + ) { + super(methods, scoreTranslation, currentVersion, extension); + } + + /** + * MethodAsMap builder is used to create the map that will be passed to the jni to create the faiss index. + * Faiss's index factory takes an "index description" that it uses to build the index. In this description, + * some parameters of the index can be configured; others need to be manually set. MethodMap builder creates + * the index description from a set of parameters and removes them from the map. On build, it sets the index + * description in the map and returns the processed map + */ + @AllArgsConstructor + static class MethodAsMapBuilder { + String indexDescription; + MethodComponent methodComponent; + Map methodAsMap; + + /** + * Add a parameter that will be used in the index description for the given method component + * + * @param parameterName name of the parameter + * @param prefix to append to the index description before the parameter + * @param suffix to append to the index description after the parameter + * @return this builder + */ + @SuppressWarnings("unchecked") + MethodAsMapBuilder addParameter(String parameterName, String prefix, String suffix) { + indexDescription += prefix; + + // When we add a parameter, what we are doing is taking it from the methods parameter and building it + // into the index description string faiss uses to create the index. + Map methodParameters = (Map) methodAsMap.get(PARAMETERS); + Parameter parameter = methodComponent.getParameters().get(parameterName); + Object value = methodParameters.containsKey(parameterName) ? methodParameters.get(parameterName) : parameter.getDefaultValue(); + + // Recursion is needed if the parameter is a method component context itself. + if (parameter instanceof Parameter.MethodComponentContextParameter) { + MethodComponentContext subMethodComponentContext = (MethodComponentContext) value; + MethodComponent subMethodComponent = ((Parameter.MethodComponentContextParameter) parameter).getMethodComponent( + subMethodComponentContext.getName() + ); + + Map subMethodAsMap = subMethodComponent.getAsMap(subMethodComponentContext); + indexDescription += subMethodAsMap.get(KNNConstants.INDEX_DESCRIPTION_PARAMETER); + subMethodAsMap.remove(KNNConstants.INDEX_DESCRIPTION_PARAMETER); + + // We replace parameterName with the map that contains only parameters that are not included in + // the method description + methodParameters.put(parameterName, subMethodAsMap); + } else { + // Just add the value to the method description and remove from map + indexDescription += value; + methodParameters.remove(parameterName); + } + + indexDescription += suffix; + return this; + } + + /** + * Build + * + * @return Method as a map + */ + Map build() { + methodAsMap.put(KNNConstants.INDEX_DESCRIPTION_PARAMETER, indexDescription); + return methodAsMap; + } + + static MethodAsMapBuilder builder( + String baseDescription, + MethodComponent methodComponent, + MethodComponentContext methodComponentContext + ) { + Map initialMap = new HashMap<>(); + initialMap.put(NAME, methodComponent.getName()); + initialMap.put(PARAMETERS, MethodComponent.getParameterMapWithDefaultsAdded(methodComponentContext, methodComponent)); + return new MethodAsMapBuilder(baseDescription, methodComponent, initialMap); + } + } +} diff --git a/src/main/java/org/opensearch/knn/index/util/KNNEngine.java b/src/main/java/org/opensearch/knn/index/util/KNNEngine.java index 365226a01..d67c15f44 100644 --- a/src/main/java/org/opensearch/knn/index/util/KNNEngine.java +++ b/src/main/java/org/opensearch/knn/index/util/KNNEngine.java @@ -1,12 +1,6 @@ /* + * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - * - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. */ package org.opensearch.knn.index.util; @@ -42,8 +36,8 @@ public enum KNNEngine implements KNNLibrary { this.knnLibrary = knnLibrary; } - private String name; - private KNNLibrary knnLibrary; + private final String name; + private final KNNLibrary knnLibrary; /** * Get the engine @@ -60,7 +54,7 @@ public static KNNEngine getEngine(String name) { return FAISS; } - throw new IllegalArgumentException("Invalid engine type: " + name); + throw new IllegalArgumentException(String.format("Invalid engine type: %s", name)); } /** @@ -91,13 +85,8 @@ public String getName() { } @Override - public String getLatestBuildVersion() { - return knnLibrary.getLatestBuildVersion(); - } - - @Override - public String getLatestLibVersion() { - return knnLibrary.getLatestLibVersion(); + public String getVersion() { + return knnLibrary.getVersion(); } @Override diff --git a/src/main/java/org/opensearch/knn/index/util/KNNLibrary.java b/src/main/java/org/opensearch/knn/index/util/KNNLibrary.java index 767a9ad61..7cb14f7f0 100644 --- a/src/main/java/org/opensearch/knn/index/util/KNNLibrary.java +++ b/src/main/java/org/opensearch/knn/index/util/KNNLibrary.java @@ -12,46 +12,11 @@ package org.opensearch.knn.index.util; import org.opensearch.common.ValidationException; -import org.opensearch.knn.common.KNNConstants; import org.opensearch.knn.index.KNNMethod; import org.opensearch.knn.index.KNNMethodContext; -import org.opensearch.knn.index.KNNSettings; -import org.opensearch.knn.index.MethodComponent; -import org.opensearch.knn.index.MethodComponentContext; -import org.opensearch.knn.index.Parameter; import org.opensearch.knn.index.SpaceType; -import com.google.common.collect.ImmutableMap; -import java.util.Collections; -import java.util.HashMap; import java.util.Map; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.function.Function; - -import static org.opensearch.knn.common.KNNConstants.BYTES_PER_KILOBYTES; -import static org.opensearch.knn.common.KNNConstants.ENCODER_PARAMETER_PQ_M; -import static org.opensearch.knn.common.KNNConstants.ENCODER_PARAMETER_PQ_CODE_COUNT_DEFAULT; -import static org.opensearch.knn.common.KNNConstants.ENCODER_PARAMETER_PQ_CODE_COUNT_LIMIT; -import static org.opensearch.knn.common.KNNConstants.ENCODER_PARAMETER_PQ_CODE_SIZE; -import static org.opensearch.knn.common.KNNConstants.ENCODER_PARAMETER_PQ_CODE_SIZE_DEFAULT; -import static org.opensearch.knn.common.KNNConstants.ENCODER_PARAMETER_PQ_CODE_SIZE_LIMIT; -import static org.opensearch.knn.common.KNNConstants.FAISS_HNSW_DESCRIPTION; -import static org.opensearch.knn.common.KNNConstants.FAISS_IVF_DESCRIPTION; -import static org.opensearch.knn.common.KNNConstants.FAISS_PQ_DESCRIPTION; -import static org.opensearch.knn.common.KNNConstants.METHOD_ENCODER_PARAMETER; -import static org.opensearch.knn.common.KNNConstants.METHOD_HNSW; -import static org.opensearch.knn.common.KNNConstants.METHOD_IVF; -import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_EF_CONSTRUCTION; -import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_EF_SEARCH; -import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_M; -import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_NLIST; -import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_NLIST_DEFAULT; -import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_NLIST_LIMIT; -import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_NPROBES; -import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_NPROBES_DEFAULT; -import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_NPROBES_LIMIT; -import static org.opensearch.knn.common.KNNConstants.NAME; -import static org.opensearch.knn.common.KNNConstants.PARAMETERS; /** * KNNLibrary is an interface that helps the plugin communicate with k-NN libraries @@ -59,18 +24,13 @@ public interface KNNLibrary { /** - * Gets the library's latest build version - * - * @return the string representing the library's latest build version - */ - String getLatestBuildVersion(); - - /** - * Gets the library's latest version + * Gets the version of the library that is being used. In general, this can be used for ensuring compatibility of + * serialized artifacts. For instance, this can be used to check if a given file that was created on a different + * cluster is compatible with this instance of the library. * - * @return the string representing the library's latest version + * @return the string representing the library's version */ - String getLatestLibVersion(); + String getVersion(); /** * Gets the extension that files written with this library should have @@ -100,7 +60,7 @@ public interface KNNLibrary { * will return a score where the lower the score, the better the result. This is the opposite of how Lucene scores * documents. * - * @param rawScore returned by the library + * @param rawScore returned by the library * @param spaceType spaceType used to compute the score * @return Lucene score for the rawScore */ @@ -127,7 +87,7 @@ public interface KNNLibrary { * Estimate overhead of KNNMethodContext in Kilobytes. * * @param knnMethodContext to estimate size for - * @param dimension to estimate size for + * @param dimension to estimate size for * @return size overhead estimate in KB */ int estimateOverheadInKB(KNNMethodContext knnMethodContext, int dimension); @@ -153,539 +113,4 @@ public interface KNNLibrary { * @param isInitialized whether library has been initialized */ void setInitialized(Boolean isInitialized); - - /** - * Abstract implementation of KNNLibrary. It contains several default methods and fields that - * are common across different underlying libraries. - */ - abstract class NativeLibrary implements KNNLibrary { - protected Map methods; - private Map> scoreTranslation; - private String latestLibraryBuildVersion; - private String latestLibraryVersion; - private String extension; - private AtomicBoolean initialized; - - /** - * Constructor for NativeLibrary - * - * @param methods map of methods the native library supports - * @param scoreTranslation Map of translation of space type to scores returned by the library - * @param latestLibraryBuildVersion String representation of latest build version of the library - * @param latestLibraryVersion String representation of latest version of the library - * @param extension String representing the extension that library files should use - */ - public NativeLibrary( - Map methods, - Map> scoreTranslation, - String latestLibraryBuildVersion, - String latestLibraryVersion, - String extension - ) { - this.methods = methods; - this.scoreTranslation = scoreTranslation; - this.latestLibraryBuildVersion = latestLibraryBuildVersion; - this.latestLibraryVersion = latestLibraryVersion; - this.extension = extension; - this.initialized = new AtomicBoolean(false); - } - - @Override - public String getLatestBuildVersion() { - return this.latestLibraryBuildVersion; - } - - @Override - public String getLatestLibVersion() { - return this.latestLibraryVersion; - } - - @Override - public String getExtension() { - return this.extension; - } - - @Override - public String getCompoundExtension() { - return getExtension() + KNNConstants.COMPOUND_EXTENSION; - } - - @Override - public KNNMethod getMethod(String methodName) { - KNNMethod method = methods.get(methodName); - if (method != null) { - return method; - } - throw new IllegalArgumentException("Invalid method name: " + methodName); - } - - @Override - public float score(float rawScore, SpaceType spaceType) { - if (this.scoreTranslation.containsKey(spaceType)) { - return this.scoreTranslation.get(spaceType).apply(rawScore); - } - - return spaceType.scoreTranslation(rawScore); - } - - @Override - public ValidationException validateMethod(KNNMethodContext knnMethodContext) { - String methodName = knnMethodContext.getMethodComponent().getName(); - return getMethod(methodName).validate(knnMethodContext); - } - - @Override - public boolean isTrainingRequired(KNNMethodContext knnMethodContext) { - String methodName = knnMethodContext.getMethodComponent().getName(); - return getMethod(methodName).isTrainingRequired(knnMethodContext); - } - - @Override - public int estimateOverheadInKB(KNNMethodContext knnMethodContext, int dimension) { - String methodName = knnMethodContext.getMethodComponent().getName(); - return getMethod(methodName).estimateOverheadInKB(knnMethodContext, dimension); - } - - @Override - public Map getMethodAsMap(KNNMethodContext knnMethodContext) { - KNNMethod knnMethod = methods.get(knnMethodContext.getMethodComponent().getName()); - - if (knnMethod == null) { - throw new IllegalArgumentException("Invalid method name: " + knnMethodContext.getMethodComponent().getName()); - } - - return knnMethod.getAsMap(knnMethodContext); - } - - @Override - public Boolean isInitialized() { - return initialized.get(); - } - - @Override - public void setInitialized(Boolean isInitialized) { - initialized.set(isInitialized); - } - } - - /** - * Implements NativeLibrary for the nmslib native library - */ - class Nmslib extends NativeLibrary { - // ====================================== - // Constants pertaining to nmslib library - // ====================================== - public final static String HNSW_LIB_NAME = "hnsw"; - public final static String EXTENSION = ".hnsw"; - - public final static Map METHODS = ImmutableMap.of( - METHOD_HNSW, - KNNMethod.Builder.builder( - MethodComponent.Builder.builder(HNSW_LIB_NAME) - .addParameter( - METHOD_PARAMETER_M, - new Parameter.IntegerParameter(METHOD_PARAMETER_M, KNNSettings.INDEX_KNN_DEFAULT_ALGO_PARAM_M, v -> v > 0) - ) - .addParameter( - METHOD_PARAMETER_EF_CONSTRUCTION, - new Parameter.IntegerParameter( - METHOD_PARAMETER_EF_CONSTRUCTION, - KNNSettings.INDEX_KNN_DEFAULT_ALGO_PARAM_EF_CONSTRUCTION, - v -> v > 0 - ) - ) - .build() - ).addSpaces(SpaceType.L2, SpaceType.L1, SpaceType.LINF, SpaceType.COSINESIMIL, SpaceType.INNER_PRODUCT).build() - ); - - public final static Nmslib INSTANCE = new Nmslib( - METHODS, - Collections.emptyMap(), - Version.LATEST.getBuildVersion(), - Version.LATEST.indexLibraryVersion(), - EXTENSION - ); - - /** - * Constructor for Nmslib - * - * @param methods Set of methods the native library supports - * @param scoreTranslation Map of translation of space type to scores returned by the library - * @param latestLibraryBuildVersion String representation of latest build version of the library - * @param latestLibraryVersion String representation of latest version of the library - * @param extension String representing the extension that library files should use - */ - private Nmslib( - Map methods, - Map> scoreTranslation, - String latestLibraryBuildVersion, - String latestLibraryVersion, - String extension - ) { - super(methods, scoreTranslation, latestLibraryBuildVersion, latestLibraryVersion, extension); - } - - public enum Version { - /** - * Latest available nmslib version - */ - V2011("2011") { - @Override - public String indexLibraryVersion() { - return KNNConstants.NMSLIB_JNI_LIBRARY_NAME; - } - }; - - public static final Version LATEST = V2011; - - private String buildVersion; - - Version(String buildVersion) { - this.buildVersion = buildVersion; - } - - /** - * NMS library version used by the KNN codec - * @return nmslib name - */ - public abstract String indexLibraryVersion(); - - public String getBuildVersion() { - return buildVersion; - } - } - } - - /** - * Implements NativeLibrary for the faiss native library - */ - class Faiss extends NativeLibrary { - - // Map that overrides OpenSearch score translation by space type of scores returned by faiss - public final static Map> SCORE_TRANSLATIONS = ImmutableMap.of( - SpaceType.INNER_PRODUCT, - rawScore -> SpaceType.INNER_PRODUCT.scoreTranslation(-1 * rawScore) - ); - - // Define encoders supported by faiss - public final static MethodComponentContext ENCODER_DEFAULT = new MethodComponentContext( - KNNConstants.ENCODER_FLAT, - Collections.emptyMap() - ); - - // TODO: To think about in future: for PQ, if dimension is not divisible by code count, PQ will fail. Right now, - // we do not have a way to base validation off of dimension. Failure will happen during training in JNI. - public final static Map encoderComponents = ImmutableMap.of( - KNNConstants.ENCODER_FLAT, - MethodComponent.Builder.builder(KNNConstants.ENCODER_FLAT) - .setMapGenerator( - ((methodComponent, methodComponentContext) -> MethodAsMapBuilder.builder( - KNNConstants.FAISS_FLAT_DESCRIPTION, - methodComponent, - methodComponentContext - ).build()) - ) - .build(), - KNNConstants.ENCODER_PQ, - MethodComponent.Builder.builder(KNNConstants.ENCODER_PQ) - .addParameter( - ENCODER_PARAMETER_PQ_M, - new Parameter.IntegerParameter( - ENCODER_PARAMETER_PQ_M, - ENCODER_PARAMETER_PQ_CODE_COUNT_DEFAULT, - v -> v > 0 && v < ENCODER_PARAMETER_PQ_CODE_COUNT_LIMIT - ) - ) - .addParameter( - ENCODER_PARAMETER_PQ_CODE_SIZE, - new Parameter.IntegerParameter( - ENCODER_PARAMETER_PQ_CODE_SIZE, - ENCODER_PARAMETER_PQ_CODE_SIZE_DEFAULT, - v -> v > 0 && v < ENCODER_PARAMETER_PQ_CODE_SIZE_LIMIT - ) - ) - .setRequiresTraining(true) - .setMapGenerator( - ((methodComponent, methodComponentContext) -> MethodAsMapBuilder.builder( - FAISS_PQ_DESCRIPTION, - methodComponent, - methodComponentContext - ).addParameter(ENCODER_PARAMETER_PQ_M, "", "").addParameter(ENCODER_PARAMETER_PQ_CODE_SIZE, "x", "").build()) - ) - .setOverheadInKBEstimator((methodComponent, methodComponentContext, dimension) -> { - // Size estimate formula: (4 * d * 2^code_size) / 1024 + 1 - - // Get value of code size passed in by user - Object codeSizeObject = methodComponentContext.getParameters().get(ENCODER_PARAMETER_PQ_CODE_SIZE); - - // If not specified, get default value of code size - if (codeSizeObject == null) { - Object codeSizeParameter = methodComponent.getParameters().get(ENCODER_PARAMETER_PQ_CODE_SIZE); - if (codeSizeParameter == null) { - throw new IllegalStateException( - ENCODER_PARAMETER_PQ_CODE_SIZE + " is not a valid " + " parameter. This is a bug." - ); - } - - codeSizeObject = ((Parameter) codeSizeParameter).getDefaultValue(); - } - - if (!(codeSizeObject instanceof Integer)) { - throw new IllegalStateException(ENCODER_PARAMETER_PQ_CODE_SIZE + " must be " + "an integer."); - } - - int codeSize = (Integer) codeSizeObject; - return ((4L * (1 << codeSize) * dimension) / BYTES_PER_KILOBYTES) + 1; - }) - .build() - ); - - // Define methods supported by faiss - public final static Map METHODS = ImmutableMap.of( - METHOD_HNSW, - KNNMethod.Builder.builder( - MethodComponent.Builder.builder(METHOD_HNSW) - .addParameter( - METHOD_PARAMETER_M, - new Parameter.IntegerParameter(METHOD_PARAMETER_M, KNNSettings.INDEX_KNN_DEFAULT_ALGO_PARAM_M, v -> v > 0) - ) - .addParameter( - METHOD_PARAMETER_EF_CONSTRUCTION, - new Parameter.IntegerParameter( - METHOD_PARAMETER_EF_CONSTRUCTION, - KNNSettings.INDEX_KNN_DEFAULT_ALGO_PARAM_EF_CONSTRUCTION, - v -> v > 0 - ) - ) - .addParameter( - METHOD_PARAMETER_EF_SEARCH, - new Parameter.IntegerParameter( - METHOD_PARAMETER_EF_SEARCH, - KNNSettings.INDEX_KNN_DEFAULT_ALGO_PARAM_EF_SEARCH, - v -> v > 0 - ) - ) - .addParameter( - METHOD_ENCODER_PARAMETER, - new Parameter.MethodComponentContextParameter(METHOD_ENCODER_PARAMETER, ENCODER_DEFAULT, encoderComponents) - ) - .setMapGenerator( - ((methodComponent, methodComponentContext) -> MethodAsMapBuilder.builder( - FAISS_HNSW_DESCRIPTION, - methodComponent, - methodComponentContext - ).addParameter(METHOD_PARAMETER_M, "", "").addParameter(METHOD_ENCODER_PARAMETER, ",", "").build()) - ) - .build() - ).addSpaces(SpaceType.L2, SpaceType.INNER_PRODUCT).build(), - METHOD_IVF, - KNNMethod.Builder.builder( - MethodComponent.Builder.builder(METHOD_IVF) - .addParameter( - METHOD_PARAMETER_NPROBES, - new Parameter.IntegerParameter( - METHOD_PARAMETER_NPROBES, - METHOD_PARAMETER_NPROBES_DEFAULT, - v -> v > 0 && v < METHOD_PARAMETER_NPROBES_LIMIT - ) - ) - .addParameter( - METHOD_PARAMETER_NLIST, - new Parameter.IntegerParameter( - METHOD_PARAMETER_NLIST, - METHOD_PARAMETER_NLIST_DEFAULT, - v -> v > 0 && v < METHOD_PARAMETER_NLIST_LIMIT - ) - ) - .addParameter( - METHOD_ENCODER_PARAMETER, - new Parameter.MethodComponentContextParameter(METHOD_ENCODER_PARAMETER, ENCODER_DEFAULT, encoderComponents) - ) - .setRequiresTraining(true) - .setMapGenerator( - ((methodComponent, methodComponentContext) -> MethodAsMapBuilder.builder( - FAISS_IVF_DESCRIPTION, - methodComponent, - methodComponentContext - ).addParameter(METHOD_PARAMETER_NLIST, "", "").addParameter(METHOD_ENCODER_PARAMETER, ",", "").build()) - ) - .setOverheadInKBEstimator((methodComponent, methodComponentContext, dimension) -> { - // Size estimate formula: (4 * nlists * d) / 1024 + 1 - - // Get value of nlists passed in by user - Object nlistObject = methodComponentContext.getParameters().get(METHOD_PARAMETER_NLIST); - - // If not specified, get default value of nlist - if (nlistObject == null) { - Object nlistParameter = methodComponent.getParameters().get(METHOD_PARAMETER_NLIST); - if (nlistParameter == null) { - throw new IllegalStateException(METHOD_PARAMETER_NLIST + " is not a valid " + " parameter. This is a bug."); - } - - nlistObject = ((Parameter) nlistParameter).getDefaultValue(); - } - - if (!(nlistObject instanceof Integer)) { - throw new IllegalStateException(METHOD_PARAMETER_NLIST + " must be " + "an integer."); - } - - int centroids = (Integer) nlistObject; - return ((4L * centroids * dimension) / BYTES_PER_KILOBYTES) + 1; - }) - .build() - ).addSpaces(SpaceType.L2, SpaceType.INNER_PRODUCT).build() - ); - - public final static Faiss INSTANCE = new Faiss( - METHODS, - SCORE_TRANSLATIONS, - Version.LATEST.getBuildVersion(), - Version.LATEST.indexLibraryVersion(), - KNNConstants.FAISS_EXTENSION - ); - - /** - * Constructor for Faiss - * - * @param methods map of methods the native library supports - * @param scoreTranslation Map of translation of space type to scores returned by the library - * @param latestLibraryBuildVersion String representation of latest build version of the library - * @param latestLibraryVersion String representation of latest version of the library - * @param extension String representing the extension that library files should use - */ - private Faiss( - Map methods, - Map> scoreTranslation, - String latestLibraryBuildVersion, - String latestLibraryVersion, - String extension - ) { - super(methods, scoreTranslation, latestLibraryBuildVersion, latestLibraryVersion, extension); - } - - /** - * MethodAsMap builder is used to create the map that will be passed to the jni to create the faiss index. - * Faiss's index factory takes an "index description" that it uses to build the index. In this description, - * some parameters of the index can be configured; others need to be manually set. MethodMap builder creates - * the index description from a set of parameters and removes them from the map. On build, it sets the index - * description in the map and returns the processed map - */ - protected static class MethodAsMapBuilder { - String indexDescription; - MethodComponent methodComponent; - Map methodAsMap; - - /** - * Constructor - * - * @param baseDescription the basic description this component should start with - * @param methodComponent the method component that maps to this builder - * @param initialMap the initial parameter map that will be modified - */ - MethodAsMapBuilder(String baseDescription, MethodComponent methodComponent, Map initialMap) { - this.indexDescription = baseDescription; - this.methodComponent = methodComponent; - this.methodAsMap = initialMap; - } - - /** - * Add a parameter that will be used in the index description for the given method component - * - * @param parameterName name of the parameter - * @param prefix to append to the index description before the parameter - * @param suffix to append to the index description after the parameter - * @return this builder - */ - @SuppressWarnings("unchecked") - MethodAsMapBuilder addParameter(String parameterName, String prefix, String suffix) { - indexDescription += prefix; - - // When we add a parameter, what we are doing is taking it from the methods parameter and building it - // into the index description string faiss uses to create the index. - Map methodParameters = (Map) methodAsMap.get(PARAMETERS); - Parameter parameter = methodComponent.getParameters().get(parameterName); - Object value = methodParameters.containsKey(parameterName) - ? methodParameters.get(parameterName) - : parameter.getDefaultValue(); - - // Recursion is needed if the parameter is a method component context itself. - if (parameter instanceof Parameter.MethodComponentContextParameter) { - MethodComponentContext subMethodComponentContext = (MethodComponentContext) value; - MethodComponent subMethodComponent = ((Parameter.MethodComponentContextParameter) parameter).getMethodComponent( - subMethodComponentContext.getName() - ); - - Map subMethodAsMap = subMethodComponent.getAsMap(subMethodComponentContext); - indexDescription += subMethodAsMap.get(KNNConstants.INDEX_DESCRIPTION_PARAMETER); - subMethodAsMap.remove(KNNConstants.INDEX_DESCRIPTION_PARAMETER); - - // We replace parameterName with the map that contains only parameters that are not included in - // the method description - methodParameters.put(parameterName, subMethodAsMap); - } else { - // Just add the value to the method description and remove from map - indexDescription += value; - methodParameters.remove(parameterName); - } - - indexDescription += suffix; - return this; - } - - /** - * Build - * - * @return Method as a map - */ - Map build() { - methodAsMap.put(KNNConstants.INDEX_DESCRIPTION_PARAMETER, indexDescription); - return methodAsMap; - } - - static MethodAsMapBuilder builder( - String baseDescription, - MethodComponent methodComponent, - MethodComponentContext methodComponentContext - ) { - Map initialMap = new HashMap<>(); - initialMap.put(NAME, methodComponent.getName()); - initialMap.put(PARAMETERS, MethodComponent.getParameterMapWithDefaultsAdded(methodComponentContext, methodComponent)); - return new MethodAsMapBuilder(baseDescription, methodComponent, initialMap); - } - } - - /** - * Enum containing information about faiss versioning - */ - private enum Version { - - /** - * Latest available nmslib version - */ - V165("165") { - @Override - public String indexLibraryVersion() { - return KNNConstants.FAISS_JNI_LIBRARY_NAME; - } - }; - - static final Version LATEST = V165; - - String buildVersion; - - Version(String buildVersion) { - this.buildVersion = buildVersion; - } - - /** - * Faiss version used by the KNN codec - * @return library name - */ - abstract String indexLibraryVersion(); - - String getBuildVersion() { - return buildVersion; - } - } - } } diff --git a/src/main/java/org/opensearch/knn/index/util/NativeLibrary.java b/src/main/java/org/opensearch/knn/index/util/NativeLibrary.java new file mode 100644 index 000000000..2cbfcb12d --- /dev/null +++ b/src/main/java/org/opensearch/knn/index/util/NativeLibrary.java @@ -0,0 +1,123 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.util; + +import org.opensearch.common.ValidationException; +import org.opensearch.knn.common.KNNConstants; +import org.opensearch.knn.index.KNNMethod; +import org.opensearch.knn.index.KNNMethodContext; +import org.opensearch.knn.index.SpaceType; + +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.Function; + +/** + * Abstract implementation of KNNLibrary. It contains several default methods and fields that + * are common across different underlying libraries. + */ +abstract class NativeLibrary implements KNNLibrary { + protected Map methods; + private final Map> scoreTranslation; + private final String currentVersion; + private final String extension; + private final AtomicBoolean initialized; + + /** + * Constructor for NativeLibrary + * + * @param methods map of methods the native library supports + * @param scoreTranslation Map of translation of space type to scores returned by the library + * @param currentVersion String representation of current version of the library + * @param extension String representing the extension that library files should use + */ + NativeLibrary( + Map methods, + Map> scoreTranslation, + String currentVersion, + String extension + ) { + this.methods = methods; + this.scoreTranslation = scoreTranslation; + this.currentVersion = currentVersion; + this.extension = extension; + this.initialized = new AtomicBoolean(false); + } + + @Override + public String getVersion() { + return this.currentVersion; + } + + @Override + public String getExtension() { + return this.extension; + } + + @Override + public String getCompoundExtension() { + return getExtension() + KNNConstants.COMPOUND_EXTENSION; + } + + @Override + public KNNMethod getMethod(String methodName) { + KNNMethod method = methods.get(methodName); + if (method != null) { + return method; + } + throw new IllegalArgumentException(String.format("Invalid method name: %s", methodName)); + } + + @Override + public float score(float rawScore, SpaceType spaceType) { + if (this.scoreTranslation.containsKey(spaceType)) { + return this.scoreTranslation.get(spaceType).apply(rawScore); + } + + return spaceType.scoreTranslation(rawScore); + } + + @Override + public ValidationException validateMethod(KNNMethodContext knnMethodContext) { + String methodName = knnMethodContext.getMethodComponent().getName(); + return getMethod(methodName).validate(knnMethodContext); + } + + @Override + public boolean isTrainingRequired(KNNMethodContext knnMethodContext) { + String methodName = knnMethodContext.getMethodComponent().getName(); + return getMethod(methodName).isTrainingRequired(knnMethodContext); + } + + @Override + public int estimateOverheadInKB(KNNMethodContext knnMethodContext, int dimension) { + String methodName = knnMethodContext.getMethodComponent().getName(); + return getMethod(methodName).estimateOverheadInKB(knnMethodContext, dimension); + } + + @Override + public Map getMethodAsMap(KNNMethodContext knnMethodContext) { + KNNMethod knnMethod = methods.get(knnMethodContext.getMethodComponent().getName()); + + if (knnMethod == null) { + throw new IllegalArgumentException(String.format("Invalid method name: %s", knnMethodContext.getMethodComponent().getName())); + } + + return knnMethod.getAsMap(knnMethodContext); + } + + @Override + public Boolean isInitialized() { + return initialized.get(); + } + + @Override + public void setInitialized(Boolean isInitialized) { + Objects.requireNonNull(isInitialized, "isInitialized must not be null"); + initialized.set(isInitialized); + } +} diff --git a/src/main/java/org/opensearch/knn/index/util/Nmslib.java b/src/main/java/org/opensearch/knn/index/util/Nmslib.java new file mode 100644 index 000000000..617b311f4 --- /dev/null +++ b/src/main/java/org/opensearch/knn/index/util/Nmslib.java @@ -0,0 +1,71 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.util; + +import com.google.common.collect.ImmutableMap; +import org.opensearch.knn.index.KNNMethod; +import org.opensearch.knn.index.KNNSettings; +import org.opensearch.knn.index.MethodComponent; +import org.opensearch.knn.index.Parameter; +import org.opensearch.knn.index.SpaceType; + +import java.util.Collections; +import java.util.Map; +import java.util.function.Function; + +import static org.opensearch.knn.common.KNNConstants.METHOD_HNSW; +import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_EF_CONSTRUCTION; +import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_M; + +/** + * Implements NativeLibrary for the nmslib native library + */ +class Nmslib extends NativeLibrary { + + // Extension to be used for Nmslib files. It is ".hnsw" and not ".nmslib" for legacy purposes. + final static String EXTENSION = ".hnsw"; + + final static String CURRENT_VERSION = "2011"; + + final static Map METHODS = ImmutableMap.of( + METHOD_HNSW, + KNNMethod.Builder.builder( + MethodComponent.Builder.builder(METHOD_HNSW) + .addParameter( + METHOD_PARAMETER_M, + new Parameter.IntegerParameter(METHOD_PARAMETER_M, KNNSettings.INDEX_KNN_DEFAULT_ALGO_PARAM_M, v -> v > 0) + ) + .addParameter( + METHOD_PARAMETER_EF_CONSTRUCTION, + new Parameter.IntegerParameter( + METHOD_PARAMETER_EF_CONSTRUCTION, + KNNSettings.INDEX_KNN_DEFAULT_ALGO_PARAM_EF_CONSTRUCTION, + v -> v > 0 + ) + ) + .build() + ).addSpaces(SpaceType.L2, SpaceType.L1, SpaceType.LINF, SpaceType.COSINESIMIL, SpaceType.INNER_PRODUCT).build() + ); + + final static Nmslib INSTANCE = new Nmslib(METHODS, Collections.emptyMap(), CURRENT_VERSION, EXTENSION); + + /** + * Constructor for Nmslib + * + * @param methods Set of methods the native library supports + * @param scoreTranslation Map of translation of space type to scores returned by the library + * @param currentVersion String representation of current version of the library + * @param extension String representing the extension that library files should use + */ + private Nmslib( + Map methods, + Map> scoreTranslation, + String currentVersion, + String extension + ) { + super(methods, scoreTranslation, currentVersion, extension); + } +} diff --git a/src/test/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumerTests.java b/src/test/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumerTests.java index adbaf74d7..b86b4051a 100644 --- a/src/test/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumerTests.java +++ b/src/test/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumerTests.java @@ -178,12 +178,7 @@ public void testAddKNNBinaryField_fromScratch_nmslibCurrent() throws IOException knn80DocValuesConsumer.addKNNBinaryField(fieldInfoArray[0], randomVectorDocValuesProducer); // The document should be created in the correct location - String expectedFile = KNNCodecUtil.buildEngineFileName( - segmentName, - knnEngine.getLatestBuildVersion(), - fieldName, - knnEngine.getExtension() - ); + String expectedFile = KNNCodecUtil.buildEngineFileName(segmentName, knnEngine.getVersion(), fieldName, knnEngine.getExtension()); assertFileInCorrectLocation(state, expectedFile); // The footer should be valid @@ -227,12 +222,7 @@ public void testAddKNNBinaryField_fromScratch_nmslibLegacy() throws IOException knn80DocValuesConsumer.addKNNBinaryField(fieldInfoArray[0], randomVectorDocValuesProducer); // The document should be created in the correct location - String expectedFile = KNNCodecUtil.buildEngineFileName( - segmentName, - knnEngine.getLatestBuildVersion(), - fieldName, - knnEngine.getExtension() - ); + String expectedFile = KNNCodecUtil.buildEngineFileName(segmentName, knnEngine.getVersion(), fieldName, knnEngine.getExtension()); assertFileInCorrectLocation(state, expectedFile); // The footer should be valid @@ -283,12 +273,7 @@ public void testAddKNNBinaryField_fromScratch_faissCurrent() throws IOException knn80DocValuesConsumer.addKNNBinaryField(fieldInfoArray[0], randomVectorDocValuesProducer); // The document should be created in the correct location - String expectedFile = KNNCodecUtil.buildEngineFileName( - segmentName, - knnEngine.getLatestBuildVersion(), - fieldName, - knnEngine.getExtension() - ); + String expectedFile = KNNCodecUtil.buildEngineFileName(segmentName, knnEngine.getVersion(), fieldName, knnEngine.getExtension()); assertFileInCorrectLocation(state, expectedFile); // The footer should be valid @@ -364,12 +349,7 @@ public void testAddKNNBinaryField_fromModel_faiss() throws IOException, Executio knn80DocValuesConsumer.addKNNBinaryField(fieldInfoArray[0], randomVectorDocValuesProducer); // The document should be created in the correct location - String expectedFile = KNNCodecUtil.buildEngineFileName( - segmentName, - knnEngine.getLatestBuildVersion(), - fieldName, - knnEngine.getExtension() - ); + String expectedFile = KNNCodecUtil.buildEngineFileName(segmentName, knnEngine.getVersion(), fieldName, knnEngine.getExtension()); assertFileInCorrectLocation(state, expectedFile); // The footer should be valid diff --git a/src/test/java/org/opensearch/knn/index/util/FaissTests.java b/src/test/java/org/opensearch/knn/index/util/FaissTests.java new file mode 100644 index 000000000..abcb5e7f0 --- /dev/null +++ b/src/test/java/org/opensearch/knn/index/util/FaissTests.java @@ -0,0 +1,68 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.util; + +import org.opensearch.common.xcontent.XContentBuilder; +import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.knn.KNNTestCase; +import org.opensearch.knn.index.MethodComponent; +import org.opensearch.knn.index.MethodComponentContext; +import org.opensearch.knn.index.Parameter; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +import static org.opensearch.knn.common.KNNConstants.INDEX_DESCRIPTION_PARAMETER; +import static org.opensearch.knn.common.KNNConstants.NAME; +import static org.opensearch.knn.common.KNNConstants.PARAMETERS; + +public class FaissTests extends KNNTestCase { + + public void testMethodAsMapBuilder() throws IOException { + String methodName = "test-method"; + String methodDescription = "test-description"; + String parameter1 = "test-parameter-1"; + Integer value1 = 10; + Integer defaultValue1 = 1; + String parameter2 = "test-parameter-2"; + Integer value2 = 15; + Integer defaultValue2 = 2; + String parameter3 = "test-parameter-3"; + Integer defaultValue3 = 3; + MethodComponent methodComponent = MethodComponent.Builder.builder(methodName) + .addParameter(parameter1, new Parameter.IntegerParameter(parameter1, defaultValue1, value -> value > 0)) + .addParameter(parameter2, new Parameter.IntegerParameter(parameter2, defaultValue2, value -> value > 0)) + .addParameter(parameter3, new Parameter.IntegerParameter(parameter3, defaultValue3, value -> value > 0)) + .build(); + + XContentBuilder xContentBuilder = XContentFactory.jsonBuilder() + .startObject() + .field(NAME, methodName) + .startObject(PARAMETERS) + .field(parameter1, value1) + .field(parameter2, value2) + .endObject() + .endObject(); + Map in = xContentBuilderToMap(xContentBuilder); + MethodComponentContext methodComponentContext = MethodComponentContext.parse(in); + + Map expectedParametersMap = new HashMap<>(methodComponentContext.getParameters()); + expectedParametersMap.put(parameter3, defaultValue3); + expectedParametersMap.remove(parameter1); + Map expectedMap = new HashMap<>(); + expectedMap.put(PARAMETERS, expectedParametersMap); + expectedMap.put(NAME, methodName); + expectedMap.put(INDEX_DESCRIPTION_PARAMETER, methodDescription + value1); + + Map methodAsMap = Faiss.MethodAsMapBuilder.builder(methodDescription, methodComponent, methodComponentContext) + .addParameter(parameter1, "", "") + .build(); + + assertEquals(expectedMap, methodAsMap); + } + +} diff --git a/src/test/java/org/opensearch/knn/index/util/KNNEngineTests.java b/src/test/java/org/opensearch/knn/index/util/KNNEngineTests.java index a62a556e7..59467c36c 100644 --- a/src/test/java/org/opensearch/knn/index/util/KNNEngineTests.java +++ b/src/test/java/org/opensearch/knn/index/util/KNNEngineTests.java @@ -1,12 +1,6 @@ /* + * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - * - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. */ package org.opensearch.knn.index.util; @@ -16,10 +10,10 @@ public class KNNEngineTests extends KNNTestCase { /** - * Get latest build version from library + * Check that version from engine and library match */ public void testDelegateLibraryFunctions() { - assertEquals(KNNLibrary.Nmslib.INSTANCE.getLatestLibVersion(), KNNEngine.NMSLIB.getLatestLibVersion()); + assertEquals(Nmslib.INSTANCE.getVersion(), KNNEngine.NMSLIB.getVersion()); } /** @@ -38,9 +32,9 @@ public void testGetEngine() { } public void testGetEngineFromPath() { - String hnswPath1 = "test" + KNNLibrary.Nmslib.EXTENSION; + String hnswPath1 = "test" + Nmslib.EXTENSION; assertEquals(KNNEngine.NMSLIB, KNNEngine.getEngineNameFromPath(hnswPath1)); - String hnswPath2 = "test" + KNNLibrary.Nmslib.EXTENSION + KNNConstants.COMPOUND_EXTENSION; + String hnswPath2 = "test" + Nmslib.EXTENSION + KNNConstants.COMPOUND_EXTENSION; assertEquals(KNNEngine.NMSLIB, KNNEngine.getEngineNameFromPath(hnswPath2)); String faissPath1 = "test" + KNNConstants.FAISS_EXTENSION; diff --git a/src/test/java/org/opensearch/knn/index/util/KNNLibraryTests.java b/src/test/java/org/opensearch/knn/index/util/NativeLibraryTests.java similarity index 57% rename from src/test/java/org/opensearch/knn/index/util/KNNLibraryTests.java rename to src/test/java/org/opensearch/knn/index/util/NativeLibraryTests.java index 4f99c6833..bac3534d8 100644 --- a/src/test/java/org/opensearch/knn/index/util/KNNLibraryTests.java +++ b/src/test/java/org/opensearch/knn/index/util/NativeLibraryTests.java @@ -1,29 +1,21 @@ /* + * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - * - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. */ package org.opensearch.knn.index.util; +import com.google.common.collect.ImmutableMap; +import org.opensearch.common.ValidationException; +import org.opensearch.common.xcontent.XContentBuilder; +import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.knn.KNNTestCase; import org.opensearch.knn.common.KNNConstants; import org.opensearch.knn.index.KNNMethod; import org.opensearch.knn.index.KNNMethodContext; import org.opensearch.knn.index.MethodComponent; import org.opensearch.knn.index.MethodComponentContext; -import org.opensearch.knn.index.Parameter; import org.opensearch.knn.index.SpaceType; -import com.google.common.collect.ImmutableMap; -import org.opensearch.common.ValidationException; -import org.opensearch.common.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentFactory; -import org.opensearch.knn.index.util.KNNLibrary.Faiss.MethodAsMapBuilder; import java.io.IOException; import java.util.Collections; @@ -31,57 +23,40 @@ import java.util.Map; import java.util.function.Function; -import static org.opensearch.knn.common.KNNConstants.INDEX_DESCRIPTION_PARAMETER; import static org.opensearch.knn.common.KNNConstants.NAME; -import static org.opensearch.knn.common.KNNConstants.PARAMETERS; - -public class KNNLibraryTests extends KNNTestCase { - /** - * Test native library build version getter - */ - public void testNativeLibrary_getLatestBuildVersion() { - String latestBuildVersion = "test-build-version"; - TestNativeLibrary testNativeLibrary = new TestNativeLibrary( - Collections.emptyMap(), - Collections.emptyMap(), - latestBuildVersion, - "", - "" - ); - assertEquals(latestBuildVersion, testNativeLibrary.getLatestBuildVersion()); - } +public class NativeLibraryTests extends KNNTestCase { /** * Test native library version getter */ - public void testNativeLibrary_getLatestLibVersion() { - String latestVersion = "test-lib-version"; - TestNativeLibrary testNativeLibrary = new TestNativeLibrary(Collections.emptyMap(), Collections.emptyMap(), "", latestVersion, ""); - assertEquals(latestVersion, testNativeLibrary.getLatestLibVersion()); + public void testGetVersion() { + String latestBuildVersion = "test-build-version"; + TestNativeLibrary testNativeLibrary = new TestNativeLibrary(Collections.emptyMap(), Collections.emptyMap(), latestBuildVersion, ""); + assertEquals(latestBuildVersion, testNativeLibrary.getVersion()); } /** * Test native library extension getter */ - public void testNativeLibrary_getExtension() { + public void testGetExtension() { String extension = ".extension"; - TestNativeLibrary testNativeLibrary = new TestNativeLibrary(Collections.emptyMap(), Collections.emptyMap(), "", "", extension); + TestNativeLibrary testNativeLibrary = new TestNativeLibrary(Collections.emptyMap(), Collections.emptyMap(), "", extension); assertEquals(extension, testNativeLibrary.getExtension()); } /** * Test native library compound extension getter */ - public void testNativeLibrary_getCompoundExtension() { + public void testGetCompoundExtension() { String extension = ".extension"; - TestNativeLibrary testNativeLibrary = new TestNativeLibrary(Collections.emptyMap(), Collections.emptyMap(), "", "", extension); + TestNativeLibrary testNativeLibrary = new TestNativeLibrary(Collections.emptyMap(), Collections.emptyMap(), "", extension); assertEquals(extension + "c", testNativeLibrary.getCompoundExtension()); } /** * Test native library compound extension getter */ - public void testNativeLibrary_getMethod() { + public void testGetMethod() { String methodName1 = "test-method-1"; KNNMethod knnMethod1 = KNNMethod.Builder.builder(MethodComponent.Builder.builder(methodName1).build()).build(); @@ -90,7 +65,7 @@ public void testNativeLibrary_getMethod() { Map knnMethodMap = ImmutableMap.of(methodName1, knnMethod1, methodName2, knnMethod2); - TestNativeLibrary testNativeLibrary = new TestNativeLibrary(knnMethodMap, Collections.emptyMap(), "", "", ""); + TestNativeLibrary testNativeLibrary = new TestNativeLibrary(knnMethodMap, Collections.emptyMap(), "", ""); assertEquals(knnMethod1, testNativeLibrary.getMethod(methodName1)); assertEquals(knnMethod2, testNativeLibrary.getMethod(methodName2)); expectThrows(IllegalArgumentException.class, () -> testNativeLibrary.getMethod("invalid")); @@ -99,9 +74,9 @@ public void testNativeLibrary_getMethod() { /** * Test native library scoring override */ - public void testNativeLibrary_score() { + public void testScore() { Map> translationMap = ImmutableMap.of(SpaceType.L2, s -> s * 2); - TestNativeLibrary testNativeLibrary = new TestNativeLibrary(Collections.emptyMap(), translationMap, "", "", ""); + TestNativeLibrary testNativeLibrary = new TestNativeLibrary(Collections.emptyMap(), translationMap, "", ""); // Test override assertEquals(2f, testNativeLibrary.score(1f, SpaceType.L2), 0.0001); @@ -112,13 +87,13 @@ public void testNativeLibrary_score() { /** * Test native library method validation */ - public void testNativeLibrary_validateMethod() throws IOException { + public void testValidateMethod() throws IOException { // Invalid - method not supported String methodName1 = "test-method-1"; KNNMethod knnMethod1 = KNNMethod.Builder.builder(MethodComponent.Builder.builder(methodName1).build()).build(); Map methodMap = ImmutableMap.of(methodName1, knnMethod1); - TestNativeLibrary testNativeLibrary1 = new TestNativeLibrary(methodMap, Collections.emptyMap(), "", "", ""); + TestNativeLibrary testNativeLibrary1 = new TestNativeLibrary(methodMap, Collections.emptyMap(), "", ""); XContentBuilder xContentBuilder = XContentFactory.jsonBuilder().startObject().field(NAME, "invalid").endObject(); Map in = xContentBuilderToMap(xContentBuilder); @@ -135,14 +110,14 @@ public ValidationException validate(KNNMethodContext knnMethodContext) { }; methodMap = ImmutableMap.of(methodName2, knnMethod2); - TestNativeLibrary testNativeLibrary2 = new TestNativeLibrary(methodMap, Collections.emptyMap(), "", "", ""); + TestNativeLibrary testNativeLibrary2 = new TestNativeLibrary(methodMap, Collections.emptyMap(), "", ""); xContentBuilder = XContentFactory.jsonBuilder().startObject().field(NAME, methodName2).endObject(); in = xContentBuilderToMap(xContentBuilder); KNNMethodContext knnMethodContext2 = KNNMethodContext.parse(in); assertNotNull(testNativeLibrary2.validateMethod(knnMethodContext2)); } - public void testNativeLibrary_getMethodAsMap() { + public void testGetMethodAsMap() { String methodName = "test-method-1"; SpaceType spaceType = SpaceType.DEFAULT; Map generatedMap = ImmutableMap.of("test-key", "test-param"); @@ -151,13 +126,7 @@ public void testNativeLibrary_getMethodAsMap() { .build(); KNNMethod knnMethod = KNNMethod.Builder.builder(methodComponent).build(); - TestNativeLibrary testNativeLibrary = new TestNativeLibrary( - ImmutableMap.of(methodName, knnMethod), - Collections.emptyMap(), - "", - "", - "" - ); + TestNativeLibrary testNativeLibrary = new TestNativeLibrary(ImmutableMap.of(methodName, knnMethod), Collections.emptyMap(), "", ""); // Check that map is expected Map expectedMap = new HashMap<>(generatedMap); @@ -178,67 +147,22 @@ public void testNativeLibrary_getMethodAsMap() { expectThrows(IllegalArgumentException.class, () -> testNativeLibrary.getMethodAsMap(invalidKnnMethodContext)); } - public void testFaiss_methodAsMapBuilder() throws IOException { - String methodName = "test-method"; - String methodDescription = "test-description"; - String parameter1 = "test-parameter-1"; - Integer value1 = 10; - Integer defaultValue1 = 1; - String parameter2 = "test-parameter-2"; - Integer value2 = 15; - Integer defaultValue2 = 2; - String parameter3 = "test-parameter-3"; - Integer defaultValue3 = 3; - MethodComponent methodComponent = MethodComponent.Builder.builder(methodName) - .addParameter(parameter1, new Parameter.IntegerParameter(parameter1, defaultValue1, value -> value > 0)) - .addParameter(parameter2, new Parameter.IntegerParameter(parameter2, defaultValue2, value -> value > 0)) - .addParameter(parameter3, new Parameter.IntegerParameter(parameter3, defaultValue3, value -> value > 0)) - .build(); - - XContentBuilder xContentBuilder = XContentFactory.jsonBuilder() - .startObject() - .field(NAME, methodName) - .startObject(PARAMETERS) - .field(parameter1, value1) - .field(parameter2, value2) - .endObject() - .endObject(); - Map in = xContentBuilderToMap(xContentBuilder); - MethodComponentContext methodComponentContext = MethodComponentContext.parse(in); - - Map expectedParametersMap = new HashMap<>(methodComponentContext.getParameters()); - expectedParametersMap.put(parameter3, defaultValue3); - expectedParametersMap.remove(parameter1); - Map expectedMap = new HashMap<>(); - expectedMap.put(PARAMETERS, expectedParametersMap); - expectedMap.put(NAME, methodName); - expectedMap.put(INDEX_DESCRIPTION_PARAMETER, methodDescription + value1); - - Map methodAsMap = MethodAsMapBuilder.builder(methodDescription, methodComponent, methodComponentContext) - .addParameter(parameter1, "", "") - .build(); - - assertEquals(expectedMap, methodAsMap); - } - - static class TestNativeLibrary extends KNNLibrary.NativeLibrary { + static class TestNativeLibrary extends NativeLibrary { /** * Constructor for TestNativeLibrary * * @param methods map of methods the native library supports * @param scoreTranslation Map of translation of space type to scores returned by the library - * @param latestLibraryBuildVersion String representation of latest build version of the library - * @param latestLibraryVersion String representation of latest version of the library + * @param currentVersion String representation of current version of the library * @param extension String representing the extension that library files should use */ public TestNativeLibrary( Map methods, Map> scoreTranslation, - String latestLibraryBuildVersion, - String latestLibraryVersion, + String currentVersion, String extension ) { - super(methods, scoreTranslation, latestLibraryBuildVersion, latestLibraryVersion, extension); + super(methods, scoreTranslation, currentVersion, extension); } } } diff --git a/src/test/java/org/opensearch/knn/plugin/stats/suppliers/LibraryInitializedSupplierTests.java b/src/test/java/org/opensearch/knn/plugin/stats/suppliers/LibraryInitializedSupplierTests.java index 759f4dd5b..64cf86381 100644 --- a/src/test/java/org/opensearch/knn/plugin/stats/suppliers/LibraryInitializedSupplierTests.java +++ b/src/test/java/org/opensearch/knn/plugin/stats/suppliers/LibraryInitializedSupplierTests.java @@ -31,7 +31,7 @@ public void testEngineInitialized() { assertTrue(libraryInitializedSupplier.get()); } - private class TestLibrary implements KNNLibrary { + private static class TestLibrary implements KNNLibrary { private Boolean initialized; TestLibrary() { @@ -39,12 +39,7 @@ private class TestLibrary implements KNNLibrary { } @Override - public String getLatestBuildVersion() { - return null; - } - - @Override - public String getLatestLibVersion() { + public String getVersion() { return null; } From bb043e6ddbb6bb66fbc5904e073174540d0a156a Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Wed, 13 Jul 2022 17:40:29 -0700 Subject: [PATCH 003/416] Adding KNN codec that is based on Lucene92 codec (#444) (#446) * Adding KNN codec that is based on Lucene92 codec Signed-off-by: Martin Gaievski (cherry picked from commit 749937508d131c6094af7487ebbcbfb94d78c703) --- .../index/codec/KNN920Codec/KNN920Codec.java | 52 +++++++++++++++++++ .../knn/index/codec/KNNCodecFactory.java | 40 ++++---------- .../knn/index/codec/KNNFormatFactory.java | 18 +++++++ .../knn/index/codec/util/CodecBuilder.java | 49 ----------------- .../services/org.apache.lucene.codecs.Codec | 3 +- .../codec/KNN920Codec/KNN920CodecTests.java | 22 ++++++++ .../knn/index/codec/KNNCodecFactoryTests.java | 20 ++++--- .../knn/index/codec/KNNCodecTestCase.java | 4 +- .../index/codec/KNNFormatFactoryTests.java | 12 +++++ 9 files changed, 126 insertions(+), 94 deletions(-) create mode 100644 src/main/java/org/opensearch/knn/index/codec/KNN920Codec/KNN920Codec.java delete mode 100644 src/main/java/org/opensearch/knn/index/codec/util/CodecBuilder.java create mode 100644 src/test/java/org/opensearch/knn/index/codec/KNN920Codec/KNN920CodecTests.java diff --git a/src/main/java/org/opensearch/knn/index/codec/KNN920Codec/KNN920Codec.java b/src/main/java/org/opensearch/knn/index/codec/KNN920Codec/KNN920Codec.java new file mode 100644 index 000000000..41b01c772 --- /dev/null +++ b/src/main/java/org/opensearch/knn/index/codec/KNN920Codec/KNN920Codec.java @@ -0,0 +1,52 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.knn.index.codec.KNN920Codec; + +import lombok.Builder; +import org.apache.lucene.codecs.Codec; +import org.apache.lucene.codecs.CompoundFormat; +import org.apache.lucene.codecs.DocValuesFormat; +import org.apache.lucene.codecs.FilterCodec; +import org.opensearch.knn.index.codec.KNNFormatFacade; +import org.opensearch.knn.index.codec.KNNFormatFactory; + +import static org.opensearch.knn.index.codec.KNNCodecFactory.CodecDelegateFactory.createKNN92DefaultDelegate; + +/** + * KNN codec that is based on Lucene92 codec + */ +public final class KNN920Codec extends FilterCodec { + + private static final String KNN920 = "KNN920Codec"; + private final KNNFormatFacade knnFormatFacade; + + /** + * No arg constructor that uses Lucene91 as the delegate + */ + public KNN920Codec() { + this(createKNN92DefaultDelegate()); + } + + /** + * Constructor that takes a Codec delegate to delegate all methods this code does not implement to. + * + * @param delegate codec that will perform all operations this codec does not override + */ + @Builder + public KNN920Codec(Codec delegate) { + super(KNN920, delegate); + knnFormatFacade = KNNFormatFactory.createKNN920Format(delegate); + } + + @Override + public DocValuesFormat docValuesFormat() { + return knnFormatFacade.docValuesFormat(); + } + + @Override + public CompoundFormat compoundFormat() { + return knnFormatFacade.compoundFormat(); + } +} diff --git a/src/main/java/org/opensearch/knn/index/codec/KNNCodecFactory.java b/src/main/java/org/opensearch/knn/index/codec/KNNCodecFactory.java index 8af3f70d6..87de6e447 100644 --- a/src/main/java/org/opensearch/knn/index/codec/KNNCodecFactory.java +++ b/src/main/java/org/opensearch/knn/index/codec/KNNCodecFactory.java @@ -4,42 +4,23 @@ */ package org.opensearch.knn.index.codec; -import com.google.common.collect.ImmutableMap; +import lombok.AllArgsConstructor; import org.apache.lucene.codecs.Codec; import org.apache.lucene.backward_codecs.lucene91.Lucene91Codec; +import org.apache.lucene.codecs.lucene92.Lucene92Codec; import org.opensearch.index.mapper.MapperService; -import org.opensearch.knn.index.codec.util.CodecBuilder; - -import java.util.Map; +import org.opensearch.knn.index.codec.KNN920Codec.KNN920Codec; /** * Factory abstraction for KNN codec */ +@AllArgsConstructor public class KNNCodecFactory { - private final Map codecByVersion; - - private static final KNNCodecVersion LATEST_KNN_CODEC_VERSION = KNNCodecVersion.KNN910; - - public KNNCodecFactory(MapperService mapperService) { - codecByVersion = ImmutableMap.of(KNNCodecVersion.KNN910, new CodecBuilder.KNN91CodecBuilder(mapperService)); - } + private final MapperService mapperService; public Codec createKNNCodec(final Codec userCodec) { - return getCodec(LATEST_KNN_CODEC_VERSION, userCodec); - } - - public Codec createKNNCodec(final KNNCodecVersion knnCodecVersion, final Codec userCodec) { - return getCodec(knnCodecVersion, userCodec); - } - - private Codec getCodec(final KNNCodecVersion knnCodecVersion, final Codec userCodec) { - try { - final CodecBuilder codecBuilder = codecByVersion.getOrDefault(knnCodecVersion, codecByVersion.get(LATEST_KNN_CODEC_VERSION)); - return codecBuilder.userCodec(userCodec).build(); - } catch (Exception ex) { - throw new RuntimeException("Cannot create instance of KNN codec", ex); - } + return KNN920Codec.builder().delegate(userCodec).build(); } /** @@ -50,12 +31,9 @@ public static class CodecDelegateFactory { public static Codec createKNN91DefaultDelegate() { return new Lucene91Codec(); } - } - /** - * Collection of supported coded versions - */ - enum KNNCodecVersion { - KNN910 + public static Codec createKNN92DefaultDelegate() { + return new Lucene92Codec(); + } } } diff --git a/src/main/java/org/opensearch/knn/index/codec/KNNFormatFactory.java b/src/main/java/org/opensearch/knn/index/codec/KNNFormatFactory.java index 6742dc3f4..bd7e4c480 100644 --- a/src/main/java/org/opensearch/knn/index/codec/KNNFormatFactory.java +++ b/src/main/java/org/opensearch/knn/index/codec/KNNFormatFactory.java @@ -13,6 +13,11 @@ */ public class KNNFormatFactory { + /** + * Return facade class that abstracts format specific to KNN910 codec + * @param delegate delegate codec that is wrapped by KNN codec + * @return + */ public static KNNFormatFacade createKNN910Format(final Codec delegate) { final KNNFormatFacade knnFormatFacade = new KNNFormatFacade( new KNN80DocValuesFormat(delegate.docValuesFormat()), @@ -20,4 +25,17 @@ public static KNNFormatFacade createKNN910Format(final Codec delegate) { ); return knnFormatFacade; } + + /** + * Return facade class that abstracts format specific to KNN920 codec + * @param delegate delegate codec that is wrapped by KNN codec + * @return + */ + public static KNNFormatFacade createKNN920Format(final Codec delegate) { + final KNNFormatFacade knnFormatFacade = new KNNFormatFacade( + new KNN80DocValuesFormat(delegate.docValuesFormat()), + new KNN80CompoundFormat(delegate.compoundFormat()) + ); + return knnFormatFacade; + } } diff --git a/src/main/java/org/opensearch/knn/index/codec/util/CodecBuilder.java b/src/main/java/org/opensearch/knn/index/codec/util/CodecBuilder.java deleted file mode 100644 index e909b07f2..000000000 --- a/src/main/java/org/opensearch/knn/index/codec/util/CodecBuilder.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ -package org.opensearch.knn.index.codec.util; - -import lombok.AllArgsConstructor; -import org.apache.lucene.codecs.Codec; -import org.opensearch.index.mapper.MapperService; -import org.opensearch.knn.index.codec.KNN910Codec.KNN910Codec; - -/** - * Abstracts builder logic for plugin codecs. For each codec we need to set delegate that is typically - * Lucene codec implementation and this is made part of the base class. Exact builder implementation may add - * additional parameters required to build a codec. - */ -public abstract class CodecBuilder { - Codec userCodec; - - /** - * Set user defined codec for plugin - * @param userCodec - * @return - */ - public CodecBuilder userCodec(Codec userCodec) { - this.userCodec = userCodec; - return this; - } - - /** - * Builds instance of codec, implementation is specific for each codec version - * @return instance of codec - */ - public abstract Codec build(); - - /** - * Implements builder abstraction for KNN91Codec, adds MapperService that may be required to build - * per field format based on field mapper type - */ - @AllArgsConstructor - public static class KNN91CodecBuilder extends CodecBuilder { - private final MapperService mapperService; - - @Override - public Codec build() { - return new KNN910Codec(userCodec); - } - } -} diff --git a/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec b/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec index b897dc36a..ca1386288 100644 --- a/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec +++ b/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec @@ -2,4 +2,5 @@ org.opensearch.knn.index.codec.KNN80Codec.KNN80Codec org.opensearch.knn.index.codec.KNN84Codec.KNN84Codec org.opensearch.knn.index.codec.KNN86Codec.KNN86Codec org.opensearch.knn.index.codec.KNN87Codec.KNN87Codec -org.opensearch.knn.index.codec.KNN910Codec.KNN910Codec \ No newline at end of file +org.opensearch.knn.index.codec.KNN910Codec.KNN910Codec +org.opensearch.knn.index.codec.KNN920Codec.KNN920Codec \ No newline at end of file diff --git a/src/test/java/org/opensearch/knn/index/codec/KNN920Codec/KNN920CodecTests.java b/src/test/java/org/opensearch/knn/index/codec/KNN920Codec/KNN920CodecTests.java new file mode 100644 index 000000000..66010a58f --- /dev/null +++ b/src/test/java/org/opensearch/knn/index/codec/KNN920Codec/KNN920CodecTests.java @@ -0,0 +1,22 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.codec.KNN920Codec; + +import org.opensearch.knn.index.codec.KNNCodecTestCase; + +import java.io.IOException; +import java.util.concurrent.ExecutionException; + +public class KNN920CodecTests extends KNNCodecTestCase { + + public void testMultiFieldsKnnIndex() throws Exception { + testMultiFieldsKnnIndex(new KNN920Codec()); + } + + public void testBuildFromModelTemplate() throws InterruptedException, ExecutionException, IOException { + testBuildFromModelTemplate(new KNN920Codec()); + } +} diff --git a/src/test/java/org/opensearch/knn/index/codec/KNNCodecFactoryTests.java b/src/test/java/org/opensearch/knn/index/codec/KNNCodecFactoryTests.java index f42405e5c..7630cf6d8 100644 --- a/src/test/java/org/opensearch/knn/index/codec/KNNCodecFactoryTests.java +++ b/src/test/java/org/opensearch/knn/index/codec/KNNCodecFactoryTests.java @@ -7,9 +7,10 @@ import org.apache.lucene.codecs.Codec; import org.apache.lucene.backward_codecs.lucene91.Lucene91Codec; +import org.apache.lucene.codecs.lucene92.Lucene92Codec; import org.opensearch.index.mapper.MapperService; import org.opensearch.knn.KNNTestCase; -import org.opensearch.knn.index.codec.KNN910Codec.KNN910Codec; +import org.opensearch.knn.index.codec.KNN920Codec.KNN920Codec; import static org.mockito.Mockito.mock; @@ -21,21 +22,18 @@ public void testKNN91DefaultDelegate() { assertTrue(knn91DefaultDelegate instanceof Lucene91Codec); } - public void testKNN91DefaultCodec() { - Lucene91Codec lucene91CodecDelegate = new Lucene91Codec(); - MapperService mapperService = mock(MapperService.class); - KNNCodecFactory knnCodecFactory = new KNNCodecFactory(mapperService); - Codec knnCodec = knnCodecFactory.createKNNCodec(lucene91CodecDelegate); - assertNotNull(knnCodec); - assertTrue(knnCodec instanceof KNN910Codec); + public void testKNN92DefaultDelegate() { + Codec knn92DefaultDelegate = KNNCodecFactory.CodecDelegateFactory.createKNN92DefaultDelegate(); + assertNotNull(knn92DefaultDelegate); + assertTrue(knn92DefaultDelegate instanceof Lucene92Codec); } - public void testKNN91CodecByVersion() { + public void testKNNDefaultCodec() { Lucene91Codec lucene91CodecDelegate = new Lucene91Codec(); MapperService mapperService = mock(MapperService.class); KNNCodecFactory knnCodecFactory = new KNNCodecFactory(mapperService); - Codec knnCodec = knnCodecFactory.createKNNCodec(KNNCodecFactory.KNNCodecVersion.KNN910, lucene91CodecDelegate); + Codec knnCodec = knnCodecFactory.createKNNCodec(lucene91CodecDelegate); assertNotNull(knnCodec); - assertTrue(knnCodec instanceof KNN910Codec); + assertTrue(knnCodec instanceof KNN920Codec); } } diff --git a/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestCase.java b/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestCase.java index 2251308a0..b67bd1115 100644 --- a/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestCase.java +++ b/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestCase.java @@ -11,7 +11,7 @@ import org.opensearch.common.settings.ClusterSettings; import org.opensearch.knn.KNNTestCase; import org.opensearch.knn.common.KNNConstants; -import org.opensearch.knn.index.codec.KNN910Codec.KNN910Codec; +import org.opensearch.knn.index.codec.KNN920Codec.KNN920Codec; import org.opensearch.knn.jni.JNIService; import org.opensearch.knn.index.KNNQuery; import org.opensearch.knn.index.KNNSettings; @@ -61,7 +61,7 @@ */ public class KNNCodecTestCase extends KNNTestCase { - private static final KNN910Codec ACTUAL_CODEC = new KNN910Codec(); + private static final KNN920Codec ACTUAL_CODEC = new KNN920Codec(); private static FieldType sampleFieldType; static { sampleFieldType = new FieldType(KNNVectorFieldMapper.Defaults.FIELD_TYPE); diff --git a/src/test/java/org/opensearch/knn/index/codec/KNNFormatFactoryTests.java b/src/test/java/org/opensearch/knn/index/codec/KNNFormatFactoryTests.java index b50d03a8a..2cb366d0f 100644 --- a/src/test/java/org/opensearch/knn/index/codec/KNNFormatFactoryTests.java +++ b/src/test/java/org/opensearch/knn/index/codec/KNNFormatFactoryTests.java @@ -24,4 +24,16 @@ public void testKNN91Format() { assertNotNull(knnFormatFacade.compoundFormat()); assertNotNull(knnFormatFacade.docValuesFormat()); } + + public void testKNN92Format() { + final Codec lucene92CodecDelegate = KNNCodecFactory.CodecDelegateFactory.createKNN92DefaultDelegate(); + MapperService mapperService = mock(MapperService.class); + KNNCodecFactory knnCodecFactory = new KNNCodecFactory(mapperService); + final Codec knnCodec = knnCodecFactory.createKNNCodec(lucene92CodecDelegate); + KNNFormatFacade knnFormatFacade = KNNFormatFactory.createKNN920Format(knnCodec); + + assertNotNull(knnFormatFacade); + assertNotNull(knnFormatFacade.compoundFormat()); + assertNotNull(knnFormatFacade.docValuesFormat()); + } } From cd669d0548178b67d50a7a27cb79b2be5d99b08c Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Fri, 15 Jul 2022 11:02:55 -0500 Subject: [PATCH 004/416] Reject delete model request if model is in Training (#424) (#449) * Reject Delete Model Request if Model is in Training Signed-off-by: Naveen Tatikonda * Locking mechanism to block modelId if model is in the process of Deletion Signed-off-by: Naveen Tatikonda * Add Blocked modelIds Validation for new Train Model Request Signed-off-by: Naveen Tatikonda * spotless fix and other changes Signed-off-by: Naveen Tatikonda * Address Review Comments Signed-off-by: Naveen Tatikonda * Bug fix for copying ModelGraveyard reference Signed-off-by: Naveen Tatikonda * Add Integration Tests for ModelGraveyardDiff Signed-off-by: Naveen Tatikonda * Refactoring and Addressing other review comments Signed-off-by: Naveen Tatikonda * Remove Model from Model Graveyard even if model does not exist Signed-off-by: Naveen Tatikonda (cherry picked from commit 46f85a0dfbf8a31310a906ae92b93364f9d3141d) Signed-off-by: Naveen Tatikonda Co-authored-by: Naveen Tatikonda --- .../java/org/opensearch/knn/bwc/ModelIT.java | 41 +- .../org/opensearch/knn/indices/ModelDao.java | 211 ++++++++-- .../knn/indices/ModelGraveyard.java | 238 ++++++++++++ .../org/opensearch/knn/plugin/KNNPlugin.java | 30 +- .../transport/GetModelTransportAction.java | 10 +- .../transport/TrainingModelRequest.java | 16 +- .../transport/UpdateModelGraveyardAction.java | 29 ++ .../UpdateModelGraveyardRequest.java | 69 ++++ .../UpdateModelGraveyardTransportAction.java | 157 ++++++++ .../opensearch/knn/indices/ModelDaoTests.java | 363 +++++++++++++++++- .../knn/indices/ModelGraveyardTests.java | 135 +++++++ .../action/RestDeleteModelHandlerIT.java | 141 ++++++- .../transport/TrainingModelRequestTests.java | 44 +++ .../UpdateModelGraveyardRequestTests.java | 58 +++ ...ateModelGraveyardTransportActionTests.java | 168 ++++++++ 15 files changed, 1651 insertions(+), 59 deletions(-) create mode 100644 src/main/java/org/opensearch/knn/indices/ModelGraveyard.java create mode 100644 src/main/java/org/opensearch/knn/plugin/transport/UpdateModelGraveyardAction.java create mode 100644 src/main/java/org/opensearch/knn/plugin/transport/UpdateModelGraveyardRequest.java create mode 100644 src/main/java/org/opensearch/knn/plugin/transport/UpdateModelGraveyardTransportAction.java create mode 100644 src/test/java/org/opensearch/knn/indices/ModelGraveyardTests.java create mode 100644 src/test/java/org/opensearch/knn/plugin/transport/UpdateModelGraveyardRequestTests.java create mode 100644 src/test/java/org/opensearch/knn/plugin/transport/UpdateModelGraveyardTransportActionTests.java diff --git a/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/ModelIT.java b/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/ModelIT.java index 829382e13..f6ff2b1c2 100644 --- a/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/ModelIT.java +++ b/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/ModelIT.java @@ -16,7 +16,11 @@ import org.opensearch.common.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; import org.opensearch.knn.index.SpaceType; +import org.opensearch.knn.index.util.KNNEngine; +import org.opensearch.knn.indices.ModelMetadata; +import org.opensearch.knn.indices.ModelState; import org.opensearch.knn.plugin.KNNPlugin; +import org.opensearch.knn.plugin.transport.DeleteModelResponse; import org.opensearch.rest.RestStatus; import org.opensearch.search.SearchHit; @@ -52,7 +56,7 @@ public class ModelIT extends AbstractRestartUpgradeTestCase { private static int DOC_ID_TEST_MODEL_INDEX = 0; private static int DOC_ID_TEST_MODEL_INDEX_DEFAULT = 0; private static final int DELAY_MILLI_SEC = 1000; - private static final int EXP_NUM_OF_MODELS = 2; + private static final int EXP_NUM_OF_MODELS = 3; private static final int K = 5; private static final int NUM_DOCS = 10; private static final int NUM_DOCS_TEST_MODEL_INDEX = 100; @@ -63,6 +67,7 @@ public class ModelIT extends AbstractRestartUpgradeTestCase { private static int QUERY_COUNT_TEST_MODEL_INDEX_DEFAULT = 0; private static final String TEST_MODEL_ID = "test-model-id"; private static final String TEST_MODEL_ID_DEFAULT = "test-model-id-default"; + private static final String TEST_MODEL_ID_TRAINING = "test-model-id-training"; private static final String MODEL_DESCRIPTION = "Description for train model test"; // KNN model test @@ -135,12 +140,42 @@ public void testKNNModelDefault() throws Exception { } } + // KNN Delete Model test for model in Training State + public void testDeleteTrainingModel() throws IOException, InterruptedException { + byte[] testModelBlob = "hello".getBytes(); + ModelMetadata testModelMetadata = getModelMetadata(); + testModelMetadata.setState(ModelState.TRAINING); + if (isRunningAgainstOldCluster()) { + addModelToSystemIndex(TEST_MODEL_ID_TRAINING, testModelMetadata, testModelBlob); + } else { + String restURI = String.join("/", KNNPlugin.KNN_BASE_URI, MODELS, TEST_MODEL_ID_TRAINING); + Request request = new Request("DELETE", restURI); + + Response response = client().performRequest(request); + assertEquals(request.getEndpoint() + ": failed", RestStatus.OK, RestStatus.fromCode(response.getStatusLine().getStatusCode())); + + assertEquals(3, getDocCount(MODEL_INDEX_NAME)); + + String responseBody = EntityUtils.toString(response.getEntity()); + assertNotNull(responseBody); + + Map responseMap = createParser(XContentType.JSON.xContent(), responseBody).map(); + + assertEquals(TEST_MODEL_ID_TRAINING, responseMap.get(MODEL_ID)); + assertEquals("failed", responseMap.get(DeleteModelResponse.RESULT)); + + String errorMessage = String.format("Cannot delete model \"%s\". Model is still in training", TEST_MODEL_ID_TRAINING); + assertEquals(errorMessage, responseMap.get(DeleteModelResponse.ERROR_MSG)); + } + } + // Delete Models and ".opensearch-knn-models" index to clear cluster metadata @AfterClass public static void wipeAllModels() throws IOException { if (!isRunningAgainstOldCluster()) { deleteKNNModel(TEST_MODEL_ID); deleteKNNModel(TEST_MODEL_ID_DEFAULT); + deleteKNNModel(TEST_MODEL_ID_TRAINING); Request request = new Request("DELETE", "/" + MODEL_INDEX_NAME); @@ -241,4 +276,8 @@ public String modelIndexMapping(String fieldName, String modelId) throws IOExcep .endObject() ); } + + private ModelMetadata getModelMetadata() { + return new ModelMetadata(KNNEngine.DEFAULT, SpaceType.DEFAULT, 4, ModelState.CREATED, "2021-03-27", "test model", ""); + } } diff --git a/src/main/java/org/opensearch/knn/indices/ModelDao.java b/src/main/java/org/opensearch/knn/indices/ModelDao.java index 937e5f811..0d5d75d30 100644 --- a/src/main/java/org/opensearch/knn/indices/ModelDao.java +++ b/src/main/java/org/opensearch/knn/indices/ModelDao.java @@ -15,11 +15,13 @@ import com.google.common.io.Resources; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.opensearch.OpenSearchException; import org.opensearch.ResourceNotFoundException; import org.opensearch.action.ActionListener; import org.opensearch.action.DocWriteRequest; import org.opensearch.action.DocWriteResponse; import org.opensearch.action.FailedNodeException; +import org.opensearch.action.StepListener; import org.opensearch.action.admin.indices.create.CreateIndexRequest; import org.opensearch.action.admin.indices.create.CreateIndexResponse; import org.opensearch.action.delete.DeleteAction; @@ -49,14 +51,19 @@ import org.opensearch.knn.plugin.transport.RemoveModelFromCacheResponse; import org.opensearch.knn.plugin.transport.UpdateModelMetadataAction; import org.opensearch.knn.plugin.transport.UpdateModelMetadataRequest; +import org.opensearch.knn.plugin.transport.UpdateModelGraveyardAction; +import org.opensearch.knn.plugin.transport.UpdateModelGraveyardRequest; import java.io.IOException; import java.net.URL; import java.util.Base64; import java.util.HashMap; import java.util.Map; +import java.util.Objects; +import java.util.Optional; import java.util.concurrent.ExecutionException; +import static java.util.Objects.isNull; import static org.opensearch.knn.common.KNNConstants.MODEL_INDEX_MAPPING_PATH; import static org.opensearch.knn.common.KNNConstants.MODEL_INDEX_NAME; import static org.opensearch.knn.common.KNNConstants.MODEL_METADATA_FIELD; @@ -122,9 +129,8 @@ public interface ModelDao { * * @param modelId to retrieve * @param listener handles get model response - * @throws IOException thrown on search */ - void get(String modelId, ActionListener listener) throws IOException; + void get(String modelId, ActionListener listener); /** * searches model from the system index. Non-blocking. @@ -151,12 +157,24 @@ public interface ModelDao { */ void delete(String modelId, ActionListener listener); + /** + * Check if modelId is in model graveyard or not. Non-blocking. + * A modelId is added to model graveyard before deleting that + * model and removed from it after deleting the model + * + * @param modelId to retrieve + * @return true if modelId is in model graveyard, otherwise return false + */ + boolean isModelInGraveyard(String modelId); + /** * Implementation of ModelDao for k-NN model index */ final class OpenSearchKNNModelDao implements ModelDao { public static Logger logger = LogManager.getLogger(ModelDao.class); + private static final String DELETED = "deleted"; + private static final String FAILED = "failed"; private int numberOfShards; private int numberOfReplicas; @@ -356,10 +374,9 @@ public Model get(String modelId) throws ExecutionException, InterruptedException * * @param modelId to retrieve * @param actionListener handles get model response - * @throws IOException thrown on search */ @Override - public void get(String modelId, ActionListener actionListener) throws IOException { + public void get(String modelId, ActionListener actionListener) { /* GET //?_local */ @@ -427,61 +444,197 @@ private String getMapping() throws IOException { return Resources.toString(url, Charsets.UTF_8); } + @Override + public boolean isModelInGraveyard(String modelId) { + // Check if the objects are not null and throw a customized NullPointerException + Objects.requireNonNull(clusterService.state(), "Cluster state must not be null"); + Objects.requireNonNull(clusterService.state().metadata(), "Cluster metadata must not be null"); + ModelGraveyard modelGraveyard = clusterService.state().metadata().custom(ModelGraveyard.TYPE); + + if (isNull(modelGraveyard)) { + return false; + } + + return modelGraveyard.contains(modelId); + } + @Override public void delete(String modelId, ActionListener listener) { // If the index is not created, there is no need to delete the model if (!isCreated()) { logger.error("Cannot delete model \"" + modelId + "\". Model index " + MODEL_INDEX_NAME + "does not exist."); String errorMessage = String.format("Cannot delete model \"%s\". Model index does not exist", modelId); - listener.onResponse(new DeleteModelResponse(modelId, "failed", errorMessage)); + listener.onResponse(new DeleteModelResponse(modelId, FAILED, errorMessage)); return; } + StepListener getModelStep = new StepListener<>(); + StepListener blockModelIdStep = new StepListener<>(); + StepListener clearModelMetadataStep = new StepListener<>(); + StepListener deleteModelFromIndexStep = new StepListener<>(); + StepListener clearModelFromCacheStep = new StepListener<>(); + StepListener unblockModelIdStep = new StepListener<>(); + + // Get Model to check if model is in TRAINING + get(modelId, ActionListener.wrap(getModelStep::onResponse, exception -> { + if (exception instanceof ResourceNotFoundException) { + String errorMessage = String.format("Unable to delete model \"%s\". Model does not exist", modelId); + ResourceNotFoundException resourceNotFoundException = new ResourceNotFoundException(errorMessage); + removeModelIdFromGraveyardOnFailure(modelId, resourceNotFoundException, getModelStep); + } else { + removeModelIdFromGraveyardOnFailure(modelId, exception, getModelStep); + } + })); + + getModelStep.whenComplete(getModelResponse -> { + // If model is in Training state, fail delete model request + if (ModelState.TRAINING == getModelResponse.getModel().getModelMetadata().getState()) { + String errorMessage = String.format("Cannot delete model \"%s\". Model is still in training", modelId); + listener.onResponse(new DeleteModelResponse(modelId, FAILED, errorMessage)); + return; + } + + // Add modelId to model graveyard until delete model request is processed + updateModelGraveyardToDelete(modelId, false, blockModelIdStep, Optional.empty()); + }, listener::onFailure); + + // Remove the metadata asynchronously + blockModelIdStep.whenComplete( + acknowledgedResponse -> { clearModelMetadata(modelId, clearModelMetadataStep); }, + listener::onFailure + ); + // Setup delete model request DeleteRequestBuilder deleteRequestBuilder = new DeleteRequestBuilder(client, DeleteAction.INSTANCE, MODEL_INDEX_NAME); deleteRequestBuilder.setId(modelId); deleteRequestBuilder.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); - // On model deletion from the index, remove the model from all nodes' model cache - ActionListener onModelDeleteListener = ActionListener.wrap(deleteResponse -> { - // If model is not deleted, return with error message + // On model metadata removal, delete the model from the index + clearModelMetadataStep.whenComplete( + acknowledgedResponse -> deleteModelFromIndex(modelId, deleteModelFromIndexStep, deleteRequestBuilder), + listener::onFailure + ); + + deleteModelFromIndexStep.whenComplete(deleteResponse -> { + // If model is not deleted, remove modelId from model graveyard and return with error message if (deleteResponse.getResult() != DocWriteResponse.Result.DELETED) { + updateModelGraveyardToDelete(modelId, true, unblockModelIdStep, Optional.empty()); String errorMessage = String.format("Model \" %s \" does not exist", modelId); listener.onResponse(new DeleteModelResponse(modelId, deleteResponse.getResult().getLowercase(), errorMessage)); return; } - // After model is deleted from the index, make sure the model is evicted from every cache in the - // cluster - client.execute( - RemoveModelFromCacheAction.INSTANCE, - new RemoveModelFromCacheRequest(modelId), - ActionListener.wrap(removeModelFromCacheResponse -> { + // After model is deleted from the index, make sure the model is evicted from every cache in the cluster + removeModelFromCache(modelId, clearModelFromCacheStep); + }, e -> listener.onFailure(new OpenSearchException(e))); - if (!removeModelFromCacheResponse.hasFailures()) { - listener.onResponse(new DeleteModelResponse(modelId, deleteResponse.getResult().getLowercase(), null)); - return; - } + clearModelFromCacheStep.whenComplete(removeModelFromCacheResponse -> { - String failureMessage = buildRemoveModelErrorMessage(modelId, removeModelFromCacheResponse); + // If there are any failures while removing model from the cache build the error message + OpenSearchException exception = null; + if (removeModelFromCacheResponse.hasFailures()) { + String failureMessage = buildRemoveModelErrorMessage(modelId, removeModelFromCacheResponse); + exception = new OpenSearchException(failureMessage); + } - listener.onResponse(new DeleteModelResponse(modelId, "failed", failureMessage)); + // Remove modelId from model graveyard + updateModelGraveyardToDelete(modelId, true, unblockModelIdStep, Optional.ofNullable(exception)); - }, e -> listener.onResponse(new DeleteModelResponse(modelId, "failed", e.getMessage()))) - ); - }, e -> listener.onResponse(new DeleteModelResponse(modelId, "failed", e.getMessage()))); + }, e -> listener.onFailure(new OpenSearchException(e))); - // On model metadata removal, delete the model from the index - ActionListener onMetadataUpdateListener = ActionListener.wrap( - acknowledgedResponse -> deleteRequestBuilder.execute(onModelDeleteListener), - listener::onFailure + unblockModelIdStep.whenComplete(acknowledgedResponse -> { + // After clearing the cache, if there are no errors return the response + listener.onResponse(new DeleteModelResponse(modelId, DELETED, null)); + + }, listener::onFailure); + + } + + // Remove model from cache in the cluster + private void removeModelFromCache(String modelId, StepListener clearModelFromCacheStep) { + client.execute( + RemoveModelFromCacheAction.INSTANCE, + new RemoveModelFromCacheRequest(modelId), + ActionListener.wrap( + clearModelFromCacheStep::onResponse, + exception -> removeModelIdFromGraveyardOnFailure(modelId, exception, clearModelFromCacheStep) + ) ); + } + + // Delete model from the system index + private void deleteModelFromIndex( + String modelId, + StepListener deleteModelFromIndexStep, + DeleteRequestBuilder deleteRequestBuilder + ) { + deleteRequestBuilder.execute( + ActionListener.wrap( + deleteModelFromIndexStep::onResponse, + exception -> removeModelIdFromGraveyardOnFailure(modelId, exception, deleteModelFromIndexStep) + ) + ); + } + + // Update model graveyard to add/remove modelId + private void updateModelGraveyardToDelete( + String modelId, + boolean isRemoveRequest, + StepListener step, + Optional exception + ) { - // Remove the metadata asynchronously + client.execute( + UpdateModelGraveyardAction.INSTANCE, + new UpdateModelGraveyardRequest(modelId, isRemoveRequest), + ActionListener.wrap(acknowledgedResponse -> { + if (exception.isEmpty()) { + step.onResponse(acknowledgedResponse); + return; + } + throw exception.get(); + + }, e -> { + // If it fails to remove the modelId from Model Graveyard, then log the error message + String errorMessage = String.format("Failed to remove \" %s \" from Model Graveyard", modelId); + String failureMessage = String.format("%s%s%s", errorMessage, "\n", e.getMessage()); + logger.error(failureMessage); + + if (exception.isEmpty()) { + step.onFailure(e); + return; + } + step.onFailure(exception.get()); + }) + ); + } + + // Clear the metadata of the model for a given modelId + private void clearModelMetadata(String modelId, StepListener clearModelMetadataStep) { client.execute( UpdateModelMetadataAction.INSTANCE, new UpdateModelMetadataRequest(modelId, true, null), - onMetadataUpdateListener + ActionListener.wrap( + clearModelMetadataStep::onResponse, + exception -> removeModelIdFromGraveyardOnFailure(modelId, exception, clearModelMetadataStep) + ) + ); + } + + // This function helps to remove the model from model graveyard and return the exception from previous step + // when the delete request fails while executing after adding modelId to model graveyard + private void removeModelIdFromGraveyardOnFailure(String modelId, Exception exceptionFromPreviousStep, StepListener step) { + client.execute( + UpdateModelGraveyardAction.INSTANCE, + new UpdateModelGraveyardRequest(modelId, true), + ActionListener.wrap(acknowledgedResponse -> { throw exceptionFromPreviousStep; }, unblockingFailedException -> { + // If it fails to remove the modelId from Model Graveyard, then log the error message and + // throw the exception that was passed as a parameter from previous step + String errorMessage = String.format("Failed to remove \" %s \" from Model Graveyard", modelId); + String failureMessage = String.format("%s%s%s", errorMessage, "\n", unblockingFailedException.getMessage()); + logger.error(failureMessage); + step.onFailure(exceptionFromPreviousStep); + }) ); } diff --git a/src/main/java/org/opensearch/knn/indices/ModelGraveyard.java b/src/main/java/org/opensearch/knn/indices/ModelGraveyard.java new file mode 100644 index 000000000..78499c1f3 --- /dev/null +++ b/src/main/java/org/opensearch/knn/indices/ModelGraveyard.java @@ -0,0 +1,238 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.indices; + +import lombok.AllArgsConstructor; +import lombok.extern.log4j.Log4j2; +import org.opensearch.Version; +import org.opensearch.cluster.Diff; +import org.opensearch.cluster.NamedDiff; +import org.opensearch.cluster.metadata.Metadata; +import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.common.xcontent.XContentBuilder; +import org.opensearch.common.xcontent.XContentParser; + +import java.io.IOException; +import java.util.Collections; +import java.util.EnumSet; +import java.util.HashSet; +import java.util.Set; +import java.util.stream.Collectors; +import com.google.common.collect.Sets; + +/** + * This class implements Metadata.Custom Interface to store a set of modelIds in the cluster metadata. + * The modelIds of the models that are under deletion are added to this set and later removed from this set after deletion. + * Also, this class implements the methods to perform operations on this set (like add, remove, contains) + */ + +@AllArgsConstructor +@Log4j2 +public class ModelGraveyard implements Metadata.Custom { + public static final String TYPE = "opensearch-knn-blocked-models"; + private final Set modelIds; + + /** + * Default Constructor to initialize object when it is null + */ + public ModelGraveyard() { + this.modelIds = new HashSet<>(); + } + + /** + * @param in input stream + * @throws IOException if read from stream fails + */ + public ModelGraveyard(StreamInput in) throws IOException { + this.modelIds = new HashSet<>(in.readStringList()); + } + + @Override + public EnumSet context() { + return Metadata.ALL_CONTEXTS; + } + + /** + * @return WriteableName for ModelGraveyard + */ + @Override + public String getWriteableName() { + return TYPE; + } + + @Override + public Version getMinimalSupportedVersion() { + return Version.CURRENT.minimumCompatibilityVersion(); + } + + /** + * @param out output stream + * @throws IOException if write to stream fails + */ + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeStringCollection(modelIds); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + return builder; + } + + /** + * @param modelId id of the model that needs to be removed from modelIds set + */ + public void remove(String modelId) { + modelIds.remove(modelId); + } + + /** + * @param modelId id of the model that needs to be added to modelIds set + */ + public void add(String modelId) { + modelIds.add(modelId); + } + + /** + * @return Set of modelIds in modelGraveyard + */ + public Set getModelIds() { + return modelIds; + } + + /** + * @return number of modelIds in modelGraveyard + */ + public int size() { + return modelIds.size(); + } + + /** + * @param modelId to check if the id of given model is there in modelIds set + * @return true if the modelId is in the modelIds set, otherwise false + */ + public boolean contains(String modelId) { + return modelIds.contains(modelId); + } + + /** + * @param before The previous custom metadata object + * @return the diff between the current updated object and the previous object + */ + @Override + public Diff diff(Metadata.Custom before) { + return new ModelGraveyardDiff((ModelGraveyard) before, this); + } + + /** + * @param streamInput input stream + * @return ModelGraveyardDiff + * @throws IOException if read from stream fails + */ + public static NamedDiff readDiffFrom(StreamInput streamInput) throws IOException { + return new ModelGraveyardDiff(streamInput); + } + + /** + * @param xContentParser + * @return ModelGraveyard + * @throws IOException + */ + public static ModelGraveyard fromXContent(XContentParser xContentParser) throws IOException { + return new ModelGraveyard(xContentParser.list().stream().map(Object::toString).collect(Collectors.toSet())); + } + + /** + * The ModelGraveyardDiff class compares the previous modelGraveyard object with the current updated modelGraveyard object + * and returns only the diff of those 2 objects. So that, whenever there is a change in cluster state, master node only + * sends the diff to all the data nodes instead of the full cluster state + */ + public static class ModelGraveyardDiff implements NamedDiff { + private final Set added; + private final Set removed; + + /** + * @param inp input stream + * @throws IOException if read from stream fails + */ + public ModelGraveyardDiff(StreamInput inp) throws IOException { + added = Set.copyOf(inp.readStringList()); + removed = Set.copyOf(inp.readStringList()); + } + + /** + * @param previous previous ModelGraveyard object + * @param current current updated ModelGraveyard object + * + * Constructor which compares both the objects to find the entries that are newly added in current object, + * entries that are deleted from previous object and the deleted entries count + */ + public ModelGraveyardDiff(ModelGraveyard previous, ModelGraveyard current) { + final Set previousModelIdsSet = previous.modelIds; + final Set currentModelIdsSet = current.modelIds; + final Set added, removed; + if (previousModelIdsSet.isEmpty()) { + // nothing will have been removed in previous object, and all entries in current object are new + added = new HashSet<>(currentModelIdsSet); + removed = new HashSet<>(); + } else if (currentModelIdsSet.isEmpty()) { + // nothing will have been added to current object, and all entries in previous object are removed + added = new HashSet<>(); + removed = new HashSet<>(previousModelIdsSet); + } else { + // some entries in previous object are removed and few entries are added to current object + removed = Sets.difference(previousModelIdsSet, currentModelIdsSet); + added = Sets.difference(currentModelIdsSet, previousModelIdsSet); + } + this.added = Collections.unmodifiableSet(added); + this.removed = Collections.unmodifiableSet(removed); + } + + /** + * @param previous Previous custom metadata object + * @return ModelGraveyard object after calculating the diff + */ + @Override + public ModelGraveyard apply(Metadata.Custom previous) { + final ModelGraveyard old = (ModelGraveyard) previous; + int removedCount = removed.size(); + if (removedCount > old.size()) { + throw new IllegalStateException( + "ModelGraveyardDiff cannot remove [" + removedCount + "] entries from [" + old.size() + "] modelIds." + ); + } + Set updatedOldGraveyardSet = Sets.difference(old.modelIds, removed); + Set modelGraveyardDiffSet = new HashSet<>(); + modelGraveyardDiffSet.addAll(added); + modelGraveyardDiffSet.addAll(updatedOldGraveyardSet); + return new ModelGraveyard(modelGraveyardDiffSet); + } + + public Set getAdded() { + return added; + } + + public Set getRemoved() { + return removed; + } + + @Override + public String getWriteableName() { + return TYPE; + } + + /** + * @param out output stream + * @throws IOException if write to stream fails + */ + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeStringCollection(added); + out.writeStringCollection(removed); + } + } +} diff --git a/src/main/java/org/opensearch/knn/plugin/KNNPlugin.java b/src/main/java/org/opensearch/knn/plugin/KNNPlugin.java index 9cf0696f2..f3b7d8197 100644 --- a/src/main/java/org/opensearch/knn/plugin/KNNPlugin.java +++ b/src/main/java/org/opensearch/knn/plugin/KNNPlugin.java @@ -5,6 +5,9 @@ package org.opensearch.knn.plugin; +import org.opensearch.cluster.NamedDiff; +import org.opensearch.cluster.metadata.Metadata; +import org.opensearch.common.ParseField; import org.opensearch.index.codec.CodecServiceFactory; import org.opensearch.index.engine.EngineFactory; import org.opensearch.knn.index.KNNCircuitBreaker; @@ -15,6 +18,7 @@ import org.opensearch.knn.index.KNNWeight; import org.opensearch.knn.index.codec.KNNCodecService; import org.opensearch.knn.index.memory.NativeMemoryLoadStrategy; +import org.opensearch.knn.indices.ModelGraveyard; import org.opensearch.knn.indices.ModelCache; import org.opensearch.knn.indices.ModelDao; import org.opensearch.knn.plugin.rest.RestDeleteModelHandler; @@ -67,6 +71,8 @@ import org.opensearch.knn.plugin.transport.TrainingModelTransportAction; import org.opensearch.knn.plugin.transport.UpdateModelMetadataAction; import org.opensearch.knn.plugin.transport.UpdateModelMetadataTransportAction; +import org.opensearch.knn.plugin.transport.UpdateModelGraveyardAction; +import org.opensearch.knn.plugin.transport.UpdateModelGraveyardTransportAction; import org.opensearch.knn.training.TrainingJobRunner; import org.opensearch.knn.training.VectorReader; import org.opensearch.plugins.ActionPlugin; @@ -88,6 +94,7 @@ import org.opensearch.watcher.ResourceWatcherService; import java.util.Arrays; +import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.List; @@ -235,7 +242,8 @@ public List getRestHandlers( new ActionHandler<>(TrainingJobRouterAction.INSTANCE, TrainingJobRouterTransportAction.class), new ActionHandler<>(TrainingModelAction.INSTANCE, TrainingModelTransportAction.class), new ActionHandler<>(RemoveModelFromCacheAction.INSTANCE, RemoveModelFromCacheTransportAction.class), - new ActionHandler<>(SearchModelAction.INSTANCE, SearchModelTransportAction.class) + new ActionHandler<>(SearchModelAction.INSTANCE, SearchModelTransportAction.class), + new ActionHandler<>(UpdateModelGraveyardAction.INSTANCE, UpdateModelGraveyardTransportAction.class) ); } @@ -293,4 +301,24 @@ public ScriptEngine getScriptEngine(Settings settings, Collection> getExecutorBuilders(Settings settings) { return ImmutableList.of(new FixedExecutorBuilder(settings, TRAIN_THREAD_POOL, 1, 1, KNN_THREAD_POOL_PREFIX, false)); } + + @Override + public List getNamedWriteables() { + List entries = new ArrayList<>(); + + entries.add(new NamedWriteableRegistry.Entry(Metadata.Custom.class, ModelGraveyard.TYPE, ModelGraveyard::new)); + entries.add(new NamedWriteableRegistry.Entry(NamedDiff.class, ModelGraveyard.TYPE, ModelGraveyard::readDiffFrom)); + return entries; + } + + @Override + public List getNamedXContent() { + List entries = new ArrayList<>(); + + entries.add( + new NamedXContentRegistry.Entry(Metadata.Custom.class, new ParseField(ModelGraveyard.TYPE), ModelGraveyard::fromXContent) + ); + return entries; + } + } diff --git a/src/main/java/org/opensearch/knn/plugin/transport/GetModelTransportAction.java b/src/main/java/org/opensearch/knn/plugin/transport/GetModelTransportAction.java index 05cb12742..e47a42d8d 100644 --- a/src/main/java/org/opensearch/knn/plugin/transport/GetModelTransportAction.java +++ b/src/main/java/org/opensearch/knn/plugin/transport/GetModelTransportAction.java @@ -20,8 +20,6 @@ import org.opensearch.tasks.Task; import org.opensearch.transport.TransportService; -import java.io.IOException; - /** * Transport Action for {@link GetModelAction} */ @@ -38,10 +36,8 @@ public GetModelTransportAction(TransportService transportService, ActionFilters @Override protected void doExecute(Task task, GetModelRequest request, ActionListener actionListener) { String modelID = request.getModelID(); - try { - modelDao.get(modelID, actionListener); - } catch (IOException e) { - actionListener.onFailure(e); - } + + modelDao.get(modelID, actionListener); + } } diff --git a/src/main/java/org/opensearch/knn/plugin/transport/TrainingModelRequest.java b/src/main/java/org/opensearch/knn/plugin/transport/TrainingModelRequest.java index ce9397905..9b7066c81 100644 --- a/src/main/java/org/opensearch/knn/plugin/transport/TrainingModelRequest.java +++ b/src/main/java/org/opensearch/knn/plugin/transport/TrainingModelRequest.java @@ -254,12 +254,26 @@ public ActionRequestValidationException validate() { ActionRequestValidationException exception = null; // Check if model id exists via model metadata - if (modelDao.getMetadata(modelId) != null) { + // Also, check if model is not in model graveyard to make sure it is not being deleted + if (modelDao.getMetadata(modelId) != null && !modelDao.isModelInGraveyard(modelId)) { exception = new ActionRequestValidationException(); exception.addValidationError("Model with id=\"" + modelId + "\" already exists"); return exception; } + // Check if modelId is in model graveyard + // ModelId is added to model graveyard if that model is undergoing deletion + // and will be removed from it after model is deleted + if (modelDao.isModelInGraveyard(modelId)) { + exception = new ActionRequestValidationException(); + String errorMessage = String.format( + "Model with id = \"%s\" is being deleted. Cannot create a model with same modelID until that model is deleted", + modelId + ); + exception.addValidationError(errorMessage); + return exception; + } + // Confirm that the passed in knnMethodContext is valid and requires training ValidationException validationException = this.knnMethodContext.validate(); if (validationException != null) { diff --git a/src/main/java/org/opensearch/knn/plugin/transport/UpdateModelGraveyardAction.java b/src/main/java/org/opensearch/knn/plugin/transport/UpdateModelGraveyardAction.java new file mode 100644 index 000000000..a9897f711 --- /dev/null +++ b/src/main/java/org/opensearch/knn/plugin/transport/UpdateModelGraveyardAction.java @@ -0,0 +1,29 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.plugin.transport; + +import org.opensearch.action.ActionType; +import org.opensearch.action.support.master.AcknowledgedResponse; +import org.opensearch.common.io.stream.Writeable; + +/** + * Action to update model graveyard + */ +public class UpdateModelGraveyardAction extends ActionType { + + public static final String NAME = "cluster:admin/knn_update_model_graveyard_action"; + public static final UpdateModelGraveyardAction INSTANCE = new UpdateModelGraveyardAction(NAME, AcknowledgedResponse::new); + + /** + * Constructor. + * + * @param name name of action + * @param acknowledgedResponseReader reader for acknowledged response + */ + public UpdateModelGraveyardAction(String name, Writeable.Reader acknowledgedResponseReader) { + super(name, acknowledgedResponseReader); + } +} diff --git a/src/main/java/org/opensearch/knn/plugin/transport/UpdateModelGraveyardRequest.java b/src/main/java/org/opensearch/knn/plugin/transport/UpdateModelGraveyardRequest.java new file mode 100644 index 000000000..f8ca38507 --- /dev/null +++ b/src/main/java/org/opensearch/knn/plugin/transport/UpdateModelGraveyardRequest.java @@ -0,0 +1,69 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.plugin.transport; + +import lombok.Getter; +import org.opensearch.action.ActionRequestValidationException; +import org.opensearch.action.support.master.AcknowledgedRequest; +import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.common.io.stream.StreamOutput; + +import java.io.IOException; + +import static org.opensearch.action.ValidateActions.addValidationError; + +/** + * Request for updating model graveyard while processing delete model request + */ +public class UpdateModelGraveyardRequest extends AcknowledgedRequest { + + @Getter + private final String modelId; + @Getter + private final boolean isRemoveRequest; + + /** + * Constructor + * + * @param in input stream + * @throws IOException if read from stream fails + */ + public UpdateModelGraveyardRequest(StreamInput in) throws IOException { + super(in); + this.modelId = in.readString(); + this.isRemoveRequest = in.readBoolean(); + } + + /** + * Constructor + * + * @param modelId Id of model + * @param isRemoveRequest should this model id be removed + */ + public UpdateModelGraveyardRequest(String modelId, boolean isRemoveRequest) { + super(); + this.modelId = modelId; + this.isRemoveRequest = isRemoveRequest; + } + + @Override + public ActionRequestValidationException validate() { + ActionRequestValidationException validationException = null; + + if (modelId.isEmpty()) { + validationException = addValidationError("Missing model ID", validationException); + } + + return validationException; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeString(modelId); + out.writeBoolean(isRemoveRequest); + } +} diff --git a/src/main/java/org/opensearch/knn/plugin/transport/UpdateModelGraveyardTransportAction.java b/src/main/java/org/opensearch/knn/plugin/transport/UpdateModelGraveyardTransportAction.java new file mode 100644 index 000000000..68708a5e0 --- /dev/null +++ b/src/main/java/org/opensearch/knn/plugin/transport/UpdateModelGraveyardTransportAction.java @@ -0,0 +1,157 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.plugin.transport; + +import lombok.Value; +import lombok.extern.log4j.Log4j2; +import org.opensearch.action.ActionListener; +import org.opensearch.action.support.ActionFilters; +import org.opensearch.action.support.master.AcknowledgedResponse; +import org.opensearch.action.support.master.TransportMasterNodeAction; +import org.opensearch.cluster.ClusterState; +import org.opensearch.cluster.ClusterStateTaskConfig; +import org.opensearch.cluster.ClusterStateTaskExecutor; +import org.opensearch.cluster.ClusterStateTaskListener; +import org.opensearch.cluster.block.ClusterBlockException; +import org.opensearch.cluster.metadata.IndexNameExpressionResolver; +import org.opensearch.cluster.metadata.Metadata; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.Priority; +import org.opensearch.common.inject.Inject; +import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.knn.indices.ModelGraveyard; +import org.opensearch.threadpool.ThreadPool; +import org.opensearch.transport.TransportService; + +import java.io.IOException; +import java.util.HashSet; +import java.util.List; +import java.util.Objects; +import java.util.Set; + +import static org.opensearch.knn.common.KNNConstants.PLUGIN_NAME; + +/** + * Transport action used to update model graveyard on the cluster manager node. + */ +@Log4j2 +public class UpdateModelGraveyardTransportAction extends TransportMasterNodeAction { + private UpdateModelGraveyardExecutor updateModelGraveyardExecutor; + + @Inject + public UpdateModelGraveyardTransportAction( + TransportService transportService, + ClusterService clusterService, + ThreadPool threadPool, + ActionFilters actionFilters, + IndexNameExpressionResolver indexNameExpressionResolver + ) { + super( + UpdateModelGraveyardAction.NAME, + transportService, + clusterService, + threadPool, + actionFilters, + UpdateModelGraveyardRequest::new, + indexNameExpressionResolver + ); + this.updateModelGraveyardExecutor = new UpdateModelGraveyardExecutor(); + } + + @Override + protected String executor() { + return ThreadPool.Names.SAME; + } + + @Override + protected AcknowledgedResponse read(StreamInput streamInput) throws IOException { + return new AcknowledgedResponse(streamInput); + } + + @Override + protected void masterOperation( + UpdateModelGraveyardRequest request, + ClusterState clusterState, + ActionListener actionListener + ) { + // ClusterManager updates model graveyard based on request parameters + clusterService.submitStateUpdateTask( + PLUGIN_NAME, + new UpdateModelGraveyardTask(request.getModelId(), request.isRemoveRequest()), + ClusterStateTaskConfig.build(Priority.NORMAL), + updateModelGraveyardExecutor, + new ClusterStateTaskListener() { + @Override + public void onFailure(String s, Exception e) { + actionListener.onFailure(e); + } + + @Override + public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) { + actionListener.onResponse(new AcknowledgedResponse(true)); + } + } + ); + } + + @Override + protected ClusterBlockException checkBlock(UpdateModelGraveyardRequest request, ClusterState clusterState) { + return null; + } + + /** + * UpdateModelGraveyardTask is used to provide the executor with the information it needs to perform its task + */ + @Value + private static class UpdateModelGraveyardTask { + String modelId; + boolean isRemoveRequest; + } + + /** + * Updates the cluster state based on the UpdateModelGraveyardTask + */ + private static class UpdateModelGraveyardExecutor implements ClusterStateTaskExecutor { + /** + * @param clusterState ClusterState + * @param taskList contains the list of UpdateModelGraveyardTask request parameters (modelId and isRemoveRequest) + * @return Represents the result of a batched execution of cluster state update tasks (UpdateModelGraveyardTasks) + */ + @Override + public ClusterTasksResult execute(ClusterState clusterState, List taskList) { + + // Check if the objects are not null and throw a customized NullPointerException + Objects.requireNonNull(clusterState, "Cluster state must not be null"); + Objects.requireNonNull(clusterState.metadata(), "Cluster metadata must not be null"); + ModelGraveyard immutableModelGraveyard = clusterState.metadata().custom(ModelGraveyard.TYPE); + ModelGraveyard modelGraveyard; + Set copySet; + + if (immutableModelGraveyard == null) { + modelGraveyard = new ModelGraveyard(); + } else { + // Deep Copy to copy all the modelIds in ModelGraveyard to local object + // to avoid copying the reference + copySet = new HashSet<>(immutableModelGraveyard.getModelIds()); + modelGraveyard = new ModelGraveyard(copySet); + } + + for (UpdateModelGraveyardTask task : taskList) { + if (task.isRemoveRequest()) { + modelGraveyard.remove(task.getModelId()); + continue; + } + modelGraveyard.add(task.getModelId()); + } + + Metadata.Builder metaDataBuilder = Metadata.builder(clusterState.metadata()); + metaDataBuilder.putCustom(ModelGraveyard.TYPE, modelGraveyard); + + ClusterState updatedClusterState = ClusterState.builder(clusterState).metadata(metaDataBuilder).build(); + return new ClusterTasksResult.Builder().successes(taskList).build(updatedClusterState); + } + } +} diff --git a/src/test/java/org/opensearch/knn/indices/ModelDaoTests.java b/src/test/java/org/opensearch/knn/indices/ModelDaoTests.java index 51f8240f3..73d285886 100644 --- a/src/test/java/org/opensearch/knn/indices/ModelDaoTests.java +++ b/src/test/java/org/opensearch/knn/indices/ModelDaoTests.java @@ -12,15 +12,21 @@ package org.opensearch.knn.indices; import org.junit.AfterClass; +import org.junit.Assert; import org.junit.BeforeClass; import org.opensearch.ExceptionsHelper; import org.opensearch.ResourceAlreadyExistsException; import org.opensearch.action.ActionListener; import org.opensearch.action.DocWriteResponse; +import org.opensearch.action.StepListener; import org.opensearch.action.admin.indices.create.CreateIndexResponse; +import org.opensearch.action.delete.DeleteAction; +import org.opensearch.action.delete.DeleteRequestBuilder; +import org.opensearch.action.delete.DeleteResponse; import org.opensearch.action.index.IndexRequest; import org.opensearch.action.index.IndexResponse; import org.opensearch.action.support.WriteRequest; +import org.opensearch.action.support.master.AcknowledgedResponse; import org.opensearch.common.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.index.IndexNotFoundException; @@ -29,6 +35,14 @@ import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.util.KNNEngine; import org.opensearch.knn.plugin.transport.DeleteModelResponse; +import org.opensearch.knn.plugin.transport.GetModelResponse; +import org.opensearch.knn.plugin.transport.RemoveModelFromCacheAction; +import org.opensearch.knn.plugin.transport.RemoveModelFromCacheRequest; +import org.opensearch.knn.plugin.transport.RemoveModelFromCacheResponse; +import org.opensearch.knn.plugin.transport.UpdateModelMetadataAction; +import org.opensearch.knn.plugin.transport.UpdateModelMetadataRequest; +import org.opensearch.knn.plugin.transport.UpdateModelGraveyardAction; +import org.opensearch.knn.plugin.transport.UpdateModelGraveyardRequest; import org.opensearch.rest.RestStatus; import java.io.IOException; @@ -41,6 +55,7 @@ import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; +import static org.opensearch.cluster.metadata.Metadata.builder; import static org.opensearch.knn.common.KNNConstants.DIMENSION; import static org.opensearch.knn.common.KNNConstants.KNN_ENGINE; import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_SPACE_TYPE; @@ -55,6 +70,7 @@ public class ModelDaoTests extends KNNSingleNodeTestCase { private static ExecutorService modelGetterExecutor; + private static final String FAILED = "failed"; @BeforeClass public static void setup() { @@ -479,12 +495,13 @@ public void testGetMetadata() throws IOException, InterruptedException { public void testDelete() throws IOException, InterruptedException { ModelDao modelDao = ModelDao.OpenSearchKNNModelDao.getInstance(); String modelId = "testDeleteModelID"; + String modelId1 = "testDeleteModelID1"; byte[] modelBlob = "hello".getBytes(); int dimension = 2; final CountDownLatch inProgressLatch = new CountDownLatch(1); ActionListener deleteModelIndexDoesNotExistListener = ActionListener.wrap(response -> { - assertEquals("failed", response.getResult()); + assertEquals(FAILED, response.getResult()); inProgressLatch.countDown(); }, exception -> fail("Unable to delete the model: " + exception)); // model index doesnt exist @@ -493,30 +510,35 @@ public void testDelete() throws IOException, InterruptedException { createIndex(MODEL_INDEX_NAME); + // Model does not exist final CountDownLatch inProgressLatch1 = new CountDownLatch(1); - ActionListener deleteModelDoesNotExistListener = ActionListener.wrap(response -> { - assertEquals(DocWriteResponse.Result.NOT_FOUND.getLowercase(), response.getResult()); + ActionListener deleteModelDoesNotExistListener = ActionListener.wrap(Assert::assertNull, exception -> { + assertNotNull(exception); + assertTrue(exception.getMessage().contains(modelId)); + assertTrue(exception.getMessage().contains("Model does not exist")); + assertFalse(modelDao.isModelInGraveyard(modelId)); inProgressLatch1.countDown(); - }, exception -> fail("Unable to delete the model: " + exception)); + }); modelDao.delete(modelId, deleteModelDoesNotExistListener); - assertTrue(inProgressLatch1.await(100, TimeUnit.SECONDS)); + assertTrue(inProgressLatch1.await(60, TimeUnit.SECONDS)); final CountDownLatch inProgressLatch2 = new CountDownLatch(1); - ActionListener deleteModelExistsListener = ActionListener.wrap(response -> { + ActionListener deleteModelTrainingListener = ActionListener.wrap(response -> { assertEquals(modelId, response.getModelID()); - assertEquals(DocWriteResponse.Result.DELETED.getLowercase(), response.getResult()); - assertNull(response.getErrorMessage()); + assertEquals(FAILED, response.getResult()); + String errorMessage = String.format("Cannot delete model \"%s\". Model is still in training", modelId); + assertEquals(errorMessage, response.getErrorMessage()); inProgressLatch2.countDown(); }, exception -> fail("Unable to delete model: " + exception)); - // model id exists + // model id exists and model is still in Training Model model = new Model( new ModelMetadata( KNNEngine.DEFAULT, SpaceType.DEFAULT, dimension, - ModelState.CREATED, + ModelState.TRAINING, ZonedDateTime.now(ZoneOffset.UTC).toString(), "", "" @@ -527,13 +549,329 @@ public void testDelete() throws IOException, InterruptedException { ActionListener docCreationListener = ActionListener.wrap(response -> { assertEquals(modelId, response.getId()); - modelDao.delete(modelId, deleteModelExistsListener); + modelDao.delete(modelId, deleteModelTrainingListener); }, exception -> fail("Unable to put the model: " + exception)); - // We use put so that we can confirm cluster metadata gets added modelDao.put(model, docCreationListener); assertTrue(inProgressLatch2.await(100, TimeUnit.SECONDS)); + + final CountDownLatch inProgressLatch3 = new CountDownLatch(1); + ActionListener deleteModelExistsListener = ActionListener.wrap(response -> { + assertEquals(modelId1, response.getModelID()); + assertEquals(DocWriteResponse.Result.DELETED.getLowercase(), response.getResult()); + assertNull(response.getErrorMessage()); + inProgressLatch3.countDown(); + }, exception -> fail("Unable to delete model: " + exception)); + + // model id exists + Model model1 = new Model( + new ModelMetadata( + KNNEngine.DEFAULT, + SpaceType.DEFAULT, + dimension, + ModelState.CREATED, + ZonedDateTime.now(ZoneOffset.UTC).toString(), + "", + "" + ), + modelBlob, + modelId1 + ); + + ActionListener docCreationListener1 = ActionListener.wrap(response -> { + assertEquals(modelId1, response.getId()); + modelDao.delete(modelId1, deleteModelExistsListener); + }, exception -> fail("Unable to put the model: " + exception)); + + // We use put so that we can confirm cluster metadata gets added + modelDao.put(model1, docCreationListener1); + + assertTrue(inProgressLatch3.await(100, TimeUnit.SECONDS)); + } + + // Test Delete Model when modelId is in Model Graveyard (previous delete model request which failed to + // remove modelId from model graveyard). But, the model does not exist + public void testDeleteModelWithModelInGraveyardModelDoesNotExist() throws InterruptedException { + ModelDao modelDao = ModelDao.OpenSearchKNNModelDao.getInstance(); + String modelId = "test-model-in-graveyard"; + createIndex(MODEL_INDEX_NAME); + + // Model does not exist + final CountDownLatch inProgressLatch = new CountDownLatch(1); + StepListener blockModelIdStep = new StepListener<>(); + ActionListener deleteModelDoesNotExistListener1 = ActionListener.wrap(Assert::assertNull, exception -> { + assertNotNull(exception); + assertTrue(exception.getMessage().contains(modelId)); + assertTrue(exception.getMessage().contains("Model does not exist")); + // Assert that modelId is removed from graveyard even when the model does not exist + assertFalse(modelDao.isModelInGraveyard(modelId)); + inProgressLatch.countDown(); + }); + + // Adding the modelId to model graveyard + client().execute( + UpdateModelGraveyardAction.INSTANCE, + new UpdateModelGraveyardRequest(modelId, false), + ActionListener.wrap(blockModelIdStep::onResponse, blockModelIdStep::onFailure) + ); + + blockModelIdStep.whenComplete(acknowledgedResponse -> { + // Assert that model is in graveyard + assertTrue(modelDao.isModelInGraveyard(modelId)); + modelDao.delete(modelId, deleteModelDoesNotExistListener1); + }, exception -> fail(exception.getMessage())); + assertTrue(inProgressLatch.await(60, TimeUnit.SECONDS)); + } + + public void testDeleteModelInTrainingWithStepListeners() throws IOException, ExecutionException, InterruptedException { + String modelId = "test-model-id-training"; + ModelDao modelDao = ModelDao.OpenSearchKNNModelDao.getInstance(); + byte[] modelBlob = "deleteModel".getBytes(); + int dimension = 2; + createIndex(MODEL_INDEX_NAME); + + Model model = new Model( + new ModelMetadata( + KNNEngine.DEFAULT, + SpaceType.DEFAULT, + dimension, + ModelState.TRAINING, + ZonedDateTime.now(ZoneOffset.UTC).toString(), + "", + "" + ), + modelBlob, + modelId + ); + + // created model and added it to index + addDoc(model); + + final CountDownLatch inProgressLatch = new CountDownLatch(1); + + StepListener getModelStep = new StepListener<>(); + + modelDao.get(modelId, ActionListener.wrap(getModelStep::onResponse, getModelStep::onFailure)); + + // Asserting that model is in TRAINING state + getModelStep.whenComplete(getModelResponse -> { + assertEquals(model.getModelMetadata().getState(), getModelResponse.getModel().getModelMetadata().getState()); + assertEquals(ModelState.TRAINING, getModelResponse.getModel().getModelMetadata().getState()); + + inProgressLatch.countDown(); + }, exception -> fail(exception.getMessage())); + assertTrue(inProgressLatch.await(100, TimeUnit.SECONDS)); + } + + public void testDeleteWithStepListeners() throws IOException, InterruptedException, ExecutionException { + String modelId = "test-model-id-delete"; + ModelDao modelDao = ModelDao.OpenSearchKNNModelDao.getInstance(); + byte[] modelBlob = "deleteModel".getBytes(); + int dimension = 2; + createIndex(MODEL_INDEX_NAME); + + Model model = new Model( + new ModelMetadata( + KNNEngine.DEFAULT, + SpaceType.DEFAULT, + dimension, + ModelState.CREATED, + ZonedDateTime.now(ZoneOffset.UTC).toString(), + "", + "" + ), + modelBlob, + modelId + ); + + // created model and added it to index + addDoc(model); + + final CountDownLatch inProgressLatch = new CountDownLatch(1); + + StepListener getModelStep = new StepListener<>(); + StepListener blockModelIdStep = new StepListener<>(); + StepListener clearModelMetadataStep = new StepListener<>(); + StepListener deleteModelFromIndexStep = new StepListener<>(); + StepListener clearModelFromCacheStep = new StepListener<>(); + StepListener unblockModelIdStep = new StepListener<>(); + + modelDao.get(modelId, ActionListener.wrap(getModelStep::onResponse, getModelStep::onFailure)); + + // Asserting that model is in CREATED state + getModelStep.whenComplete(getModelResponse -> { + assertEquals(model.getModelMetadata().getState(), getModelResponse.getModel().getModelMetadata().getState()); + assertNotEquals(ModelState.TRAINING.getName(), getModelResponse.getModel().getModelMetadata().getState().toString()); + + client().execute( + UpdateModelGraveyardAction.INSTANCE, + new UpdateModelGraveyardRequest(modelId, false), + ActionListener.wrap(blockModelIdStep::onResponse, blockModelIdStep::onFailure) + ); + }, exception -> fail(exception.getMessage())); + + blockModelIdStep.whenComplete(acknowledgedResponse -> { + // Asserting that modelId is in blocked list + assertTrue(modelDao.isModelInGraveyard(modelId)); + + client().execute( + UpdateModelMetadataAction.INSTANCE, + new UpdateModelMetadataRequest(modelId, true, null), + ActionListener.wrap(clearModelMetadataStep::onResponse, clearModelMetadataStep::onFailure) + ); + + }, exception -> fail(exception.getMessage())); + + DeleteRequestBuilder deleteRequestBuilder = new DeleteRequestBuilder(client(), DeleteAction.INSTANCE, MODEL_INDEX_NAME); + deleteRequestBuilder.setId(modelId); + deleteRequestBuilder.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + + clearModelMetadataStep.whenComplete(acknowledgedResponse -> { + // Asserting that metadata is cleared + assertNull(modelDao.getMetadata(modelId)); + + deleteRequestBuilder.execute(ActionListener.wrap(deleteModelFromIndexStep::onResponse, deleteModelFromIndexStep::onFailure)); + + }, exception -> fail(exception.getMessage())); + + deleteModelFromIndexStep.whenComplete(deleteResponse -> { + // Asserting that model is deleted from index + assertEquals(DocWriteResponse.Result.DELETED, deleteResponse.getResult()); + client().execute( + RemoveModelFromCacheAction.INSTANCE, + new RemoveModelFromCacheRequest(modelId), + ActionListener.wrap(clearModelFromCacheStep::onResponse, clearModelFromCacheStep::onFailure) + ); + + }, exception -> fail(exception.getMessage())); + + clearModelFromCacheStep.whenComplete(removeModelFromCacheResponse -> { + assertFalse(removeModelFromCacheResponse.hasFailures()); + + client().execute( + UpdateModelGraveyardAction.INSTANCE, + new UpdateModelGraveyardRequest(modelId, true), + ActionListener.wrap(unblockModelIdStep::onResponse, unblockModelIdStep::onFailure) + ); + + unblockModelIdStep.whenComplete(acknowledgedResponse -> { + // Asserting that model is unblocked + assertFalse(modelDao.isModelInGraveyard(modelId)); + inProgressLatch.countDown(); + }, exception -> fail(exception.getMessage())); + }, exception -> fail(exception.getMessage())); + + assertTrue(inProgressLatch.await(100, TimeUnit.SECONDS)); + } + + public void testDeleteWithStepListenersOnFailure() throws IOException, InterruptedException, ExecutionException { + String modelId = "test-model-id-delete1"; + ModelDao modelDao = ModelDao.OpenSearchKNNModelDao.getInstance(); + byte[] modelBlob = "deleteModel".getBytes(); + int dimension = 2; + createIndex(MODEL_INDEX_NAME); + Model model = new Model( + new ModelMetadata( + KNNEngine.DEFAULT, + SpaceType.DEFAULT, + dimension, + ModelState.CREATED, + ZonedDateTime.now(ZoneOffset.UTC).toString(), + "", + "" + ), + modelBlob, + modelId + ); + + addDoc(model); + + // We will validate if the modelId gets unblocked when some exception occurs + // during the process of deletion after adding that modelId to blocked list + final CountDownLatch inProgressLatch = new CountDownLatch(1); + + StepListener blockModelIdStep = new StepListener<>(); + StepListener clearModelMetadataStep = new StepListener<>(); + + // Add modelId to blocked list + client().execute( + UpdateModelGraveyardAction.INSTANCE, + new UpdateModelGraveyardRequest(modelId, false), + ActionListener.wrap(blockModelIdStep::onResponse, blockModelIdStep::onFailure) + ); + + // Asserting that the modelId is blocked + blockModelIdStep.whenComplete(acknowledgedResponse -> { + assertTrue(modelDao.isModelInGraveyard(modelId)); + + // Sending empty string for modelId to fail the clear model metadata request + client().execute( + UpdateModelMetadataAction.INSTANCE, + new UpdateModelMetadataRequest("", true, null), + ActionListener.wrap(clearModelMetadataStep::onResponse, exp -> { + // Asserting that modelId is still blocked and clearModelMetadata throws an exception + assertNotNull(exp.getMessage()); + assertTrue(modelDao.isModelInGraveyard(modelId)); + client().execute( + // OnFailure sending request to unblock modelId + UpdateModelGraveyardAction.INSTANCE, + new UpdateModelGraveyardRequest(modelId, true), + ActionListener.wrap(ackResponse -> { + // Asserting that model is unblocked + assertFalse(modelDao.isModelInGraveyard(modelId)); + assertNotNull(exp.getMessage()); + }, exception -> fail(exception.getMessage())) + ); + }) + ); + inProgressLatch.countDown(); + }, exception -> fail(exception.getMessage())); + + assertTrue(inProgressLatch.await(100, TimeUnit.SECONDS)); + + // Some exception occurs during the process of deletion and unblocking model request also fails + final CountDownLatch inProgressLatch1 = new CountDownLatch(1); + + StepListener blockModelIdStep1 = new StepListener<>(); + StepListener clearModelMetadataStep1 = new StepListener<>(); + + // Add modelId to blocked list + client().execute( + UpdateModelGraveyardAction.INSTANCE, + new UpdateModelGraveyardRequest(modelId, false), + ActionListener.wrap(blockModelIdStep1::onResponse, blockModelIdStep1::onFailure) + ); + + // Asserting that the modelId is blocked + blockModelIdStep1.whenComplete(acknowledgedResponse -> { + assertTrue(modelDao.isModelInGraveyard(modelId)); + + // Sending empty string for modelId to fail the clear model metadata request + client().execute( + UpdateModelMetadataAction.INSTANCE, + new UpdateModelMetadataRequest("", true, null), + ActionListener.wrap(clearModelMetadataStep1::onResponse, exp -> { + assertNotNull(exp.getMessage()); + assertTrue(modelDao.isModelInGraveyard(modelId)); + + // Failing unblock modelId request by sending modelId as an empty string + client().execute( + UpdateModelGraveyardAction.INSTANCE, + new UpdateModelGraveyardRequest("", true), + ActionListener.wrap(ackResponse -> {}, unblockingFailedException -> { + // Asserting that model is still blocked and returns both exceptions in response + assertTrue(modelDao.isModelInGraveyard(modelId)); + assertNotNull(exp.getMessage()); + assertNotNull(unblockingFailedException.getMessage()); + }) + ); + }) + ); + inProgressLatch1.countDown(); + }, exception -> fail(exception.getMessage())); + + assertTrue(inProgressLatch1.await(100, TimeUnit.SECONDS)); } public void addDoc(Model model) throws IOException, ExecutionException, InterruptedException { @@ -564,4 +902,5 @@ public void addDoc(Model model) throws IOException, ExecutionException, Interrup IndexResponse response = client().index(indexRequest).get(); assertTrue(response.status() == RestStatus.CREATED || response.status() == RestStatus.OK); } + } diff --git a/src/test/java/org/opensearch/knn/indices/ModelGraveyardTests.java b/src/test/java/org/opensearch/knn/indices/ModelGraveyardTests.java new file mode 100644 index 000000000..28b3c7474 --- /dev/null +++ b/src/test/java/org/opensearch/knn/indices/ModelGraveyardTests.java @@ -0,0 +1,135 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.indices; + +import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.test.OpenSearchTestCase; + +import java.io.IOException; +import java.util.HashSet; +import java.util.Set; + +public class ModelGraveyardTests extends OpenSearchTestCase { + + public void testAdd() { + ModelGraveyard testModelGraveyard = new ModelGraveyard(); + String testModelId = "test-model-id"; + testModelGraveyard.add(testModelId); + assertTrue(testModelGraveyard.contains(testModelId)); + } + + public void testRemove() { + Set modelIds = new HashSet<>(); + String testModelId = "test-model-id"; + modelIds.add(testModelId); + ModelGraveyard testModelGraveyard = new ModelGraveyard(modelIds); + + assertTrue(testModelGraveyard.contains(testModelId)); + testModelGraveyard.remove(testModelId); + assertFalse(testModelGraveyard.contains(testModelId)); + } + + public void testContains() { + Set modelIds = new HashSet<>(); + String testModelId = "test-model-id"; + modelIds.add(testModelId); + + ModelGraveyard testModelGraveyard = new ModelGraveyard(modelIds); + assertTrue(testModelGraveyard.contains(testModelId)); + } + + public void testStreams() throws IOException { + Set modelIds = new HashSet<>(); + String testModelId = "test-model-id"; + modelIds.add(testModelId); + ModelGraveyard testModelGraveyard = new ModelGraveyard(modelIds); + + BytesStreamOutput streamOutput = new BytesStreamOutput(); + testModelGraveyard.writeTo(streamOutput); + + ModelGraveyard testModelGraveyardCopy = new ModelGraveyard(streamOutput.bytes().streamInput()); + + assertEquals(testModelGraveyard.size(), testModelGraveyardCopy.size()); + assertTrue(testModelGraveyard.contains(testModelId)); + assertTrue(testModelGraveyardCopy.contains(testModelId)); + } + + public void testDiffStreams() throws IOException { + Set added = new HashSet<>(); + Set removed = new HashSet<>(); + String testModelId = "test-model-id"; + String testModelId1 = "test-model-id-1"; + added.add(testModelId); + removed.add(testModelId1); + + ModelGraveyard modelGraveyardCurrent = new ModelGraveyard(added); + ModelGraveyard modelGraveyardPrevious = new ModelGraveyard(removed); + + ModelGraveyard.ModelGraveyardDiff modelGraveyardDiff = new ModelGraveyard.ModelGraveyardDiff( + modelGraveyardPrevious, + modelGraveyardCurrent + ); + assertEquals(added, modelGraveyardDiff.getAdded()); + assertEquals(removed, modelGraveyardDiff.getRemoved()); + + BytesStreamOutput streamOutput = new BytesStreamOutput(); + modelGraveyardDiff.writeTo(streamOutput); + + ModelGraveyard.ModelGraveyardDiff modelGraveyardDiffCopy = new ModelGraveyard.ModelGraveyardDiff( + streamOutput.bytes().streamInput() + ); + assertEquals(added, modelGraveyardDiffCopy.getAdded()); + assertEquals(removed, modelGraveyardDiffCopy.getRemoved()); + } + + public void testDiff() { + + // nothing will have been removed in previous object, and all entries in current object are new + ModelGraveyard modelGraveyard1 = new ModelGraveyard(); + + Set modelIds = new HashSet<>(); + modelIds.add("1"); + modelIds.add("2"); + ModelGraveyard modelGraveyard2 = new ModelGraveyard(modelIds); + + ModelGraveyard.ModelGraveyardDiff diff1 = new ModelGraveyard.ModelGraveyardDiff(modelGraveyard1, modelGraveyard2); + assertEquals(0, diff1.getRemoved().size()); + assertEquals(2, diff1.getAdded().size()); + + ModelGraveyard updatedGraveyard1 = diff1.apply(modelGraveyard1); + assertEquals(2, updatedGraveyard1.size()); + assertTrue(updatedGraveyard1.contains("1")); + assertTrue(updatedGraveyard1.contains("2")); + + // nothing will have been added to current object, and all entries in previous object are removed + ModelGraveyard modelGraveyard3 = new ModelGraveyard(); + ModelGraveyard.ModelGraveyardDiff diff2 = new ModelGraveyard.ModelGraveyardDiff(modelGraveyard2, modelGraveyard3); + assertEquals(2, diff2.getRemoved().size()); + assertEquals(0, diff2.getAdded().size()); + + ModelGraveyard updatedGraveyard2 = diff2.apply(modelGraveyard2); + assertEquals(0, updatedGraveyard2.size()); + + // some entries in previous object are removed and few entries are added to current object + modelIds = new HashSet<>(); + modelIds.add("1"); + modelIds.add("3"); + modelIds.add("4"); + ModelGraveyard modelGraveyard4 = new ModelGraveyard(modelIds); + + ModelGraveyard.ModelGraveyardDiff diff3 = new ModelGraveyard.ModelGraveyardDiff(modelGraveyard2, modelGraveyard4); + assertEquals(1, diff3.getRemoved().size()); + assertEquals(2, diff3.getAdded().size()); + + ModelGraveyard updatedGraveyard3 = diff3.apply(modelGraveyard2); + assertEquals(3, updatedGraveyard3.size()); + assertTrue(updatedGraveyard3.contains("1")); + assertTrue(updatedGraveyard3.contains("3")); + assertTrue(updatedGraveyard3.contains("4")); + assertFalse(updatedGraveyard3.contains("2")); + } + +} diff --git a/src/test/java/org/opensearch/knn/plugin/action/RestDeleteModelHandlerIT.java b/src/test/java/org/opensearch/knn/plugin/action/RestDeleteModelHandlerIT.java index edd8d2106..f2c31fa72 100644 --- a/src/test/java/org/opensearch/knn/plugin/action/RestDeleteModelHandlerIT.java +++ b/src/test/java/org/opensearch/knn/plugin/action/RestDeleteModelHandlerIT.java @@ -12,9 +12,11 @@ package org.opensearch.knn.plugin.action; import org.apache.http.util.EntityUtils; -import org.opensearch.action.DocWriteResponse; import org.opensearch.client.Request; import org.opensearch.client.Response; +import org.opensearch.client.ResponseException; +import org.opensearch.common.xcontent.XContentBuilder; +import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentType; import org.opensearch.knn.KNNRestTestCase; import org.opensearch.knn.index.SpaceType; @@ -28,9 +30,17 @@ import java.io.IOException; import java.util.Map; -import static org.opensearch.knn.common.KNNConstants.MODELS; +import static org.opensearch.knn.common.KNNConstants.ENCODER_PARAMETER_PQ_CODE_SIZE; +import static org.opensearch.knn.common.KNNConstants.ENCODER_PARAMETER_PQ_M; +import static org.opensearch.knn.common.KNNConstants.KNN_ENGINE; import static org.opensearch.knn.common.KNNConstants.MODEL_ID; +import static org.opensearch.knn.common.KNNConstants.MODELS; import static org.opensearch.knn.common.KNNConstants.MODEL_INDEX_NAME; +import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_SPACE_TYPE; +import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_NLIST; +import static org.opensearch.knn.common.KNNConstants.METHOD_ENCODER_PARAMETER; +import static org.opensearch.knn.common.KNNConstants.NAME; +import static org.opensearch.knn.common.KNNConstants.PARAMETERS; /** * Integration tests to check the correctness of {@link org.opensearch.knn.plugin.rest.RestDeleteModelHandler} @@ -57,23 +67,138 @@ public void testDeleteModelExists() throws IOException { Response response = client().performRequest(request); assertEquals(request.getEndpoint() + ": failed", RestStatus.OK, RestStatus.fromCode(response.getStatusLine().getStatusCode())); - assertEquals(getDocCount(MODEL_INDEX_NAME), 0); + assertEquals(0, getDocCount(MODEL_INDEX_NAME)); } - public void testDeleteModelFailsInvalid() throws IOException { + public void testDeleteTrainingModel() throws IOException { createModelSystemIndex(); - String restURI = String.join("/", KNNPlugin.KNN_BASE_URI, MODELS, "invalid-model-id"); + String testModelID = "test-model-id"; + byte[] testModelBlob = "hello".getBytes(); + ModelMetadata testModelMetadata = getModelMetadata(); + testModelMetadata.setState(ModelState.TRAINING); + + addModelToSystemIndex(testModelID, testModelMetadata, testModelBlob); + assertEquals(1, getDocCount(MODEL_INDEX_NAME)); + + String restURI = String.join("/", KNNPlugin.KNN_BASE_URI, MODELS, testModelID); Request request = new Request("DELETE", restURI); Response response = client().performRequest(request); assertEquals(request.getEndpoint() + ": failed", RestStatus.OK, RestStatus.fromCode(response.getStatusLine().getStatusCode())); + + assertEquals(1, getDocCount(MODEL_INDEX_NAME)); + String responseBody = EntityUtils.toString(response.getEntity()); assertNotNull(responseBody); Map responseMap = createParser(XContentType.JSON.xContent(), responseBody).map(); - assertEquals("invalid-model-id", responseMap.get(MODEL_ID)); - assertEquals(DocWriteResponse.Result.NOT_FOUND.getLowercase(), responseMap.get(DeleteModelResponse.RESULT)); - assertNotNull(responseMap.get(DeleteModelResponse.ERROR_MSG)); + assertEquals(testModelID, responseMap.get(MODEL_ID)); + assertEquals("failed", responseMap.get(DeleteModelResponse.RESULT)); + + String errorMessage = String.format("Cannot delete model \"%s\". Model is still in training", testModelID); + assertEquals(errorMessage, responseMap.get(DeleteModelResponse.ERROR_MSG)); + } + + public void testDeleteModelFailsInvalid() throws IOException { + String modelId = "invalid-model-id"; + createModelSystemIndex(); + String restURI = String.join("/", KNNPlugin.KNN_BASE_URI, MODELS, modelId); + Request request = new Request("DELETE", restURI); + + ResponseException ex = expectThrows(ResponseException.class, () -> client().performRequest(request)); + assertTrue(ex.getMessage().contains(modelId)); } + + // Test Train Model -> Delete Model -> Train Model with same modelId + public void testTrainingDeletedModel() throws IOException, InterruptedException { + String modelId = "test-model-id1"; + String trainingIndexName1 = "train-index-1"; + String trainingIndexName2 = "train-index-2"; + String trainingFieldName = "train-field"; + int dimension = 8; + + // Train Model + trainModel(modelId, trainingIndexName1, trainingFieldName, dimension); + + // Delete Model + String restURI = String.join("/", KNNPlugin.KNN_BASE_URI, MODELS, modelId); + Request request = new Request("DELETE", restURI); + + Response response = client().performRequest(request); + assertEquals(request.getEndpoint() + ": failed", RestStatus.OK, RestStatus.fromCode(response.getStatusLine().getStatusCode())); + + assertEquals(0, getDocCount(MODEL_INDEX_NAME)); + + // Train Model again with same ModelId + trainModel(modelId, trainingIndexName2, trainingFieldName, dimension); + } + + private void trainModel(String modelId, String trainingIndexName, String trainingFieldName, int dimension) throws IOException, + InterruptedException { + + // Create a training index and randomly ingest data into it + createBasicKnnIndex(trainingIndexName, trainingFieldName, dimension); + int trainingDataCount = 200; + bulkIngestRandomVectors(trainingIndexName, trainingFieldName, trainingDataCount, dimension); + + // Call the train API with this definition: + /* + { + "training_index": "train_index", + "training_field": "train_field", + "dimension": 8, + "description": "this should be allowed to be null", + "method": { + "name":"ivf", + "engine":"faiss", + "space_type": "l2", + "parameters":{ + "nlist":1, + "encoder":{ + "name":"pq", + "parameters":{ + "code_size":2, + "m": 2 + } + } + } + } + } + */ + XContentBuilder builder = XContentFactory.jsonBuilder() + .startObject() + .field(NAME, "ivf") + .field(KNN_ENGINE, "faiss") + .field(METHOD_PARAMETER_SPACE_TYPE, "l2") + .startObject(PARAMETERS) + .field(METHOD_PARAMETER_NLIST, 1) + .startObject(METHOD_ENCODER_PARAMETER) + .field(NAME, "pq") + .startObject(PARAMETERS) + .field(ENCODER_PARAMETER_PQ_CODE_SIZE, 2) + .field(ENCODER_PARAMETER_PQ_M, 2) + .endObject() + .endObject() + .endObject() + .endObject(); + Map method = xContentBuilderToMap(builder); + + Response trainResponse = trainModel(modelId, trainingIndexName, trainingFieldName, dimension, method, "dummy description"); + + assertEquals(RestStatus.OK, RestStatus.fromCode(trainResponse.getStatusLine().getStatusCode())); + + // Confirm that the model gets created + Response getResponse = getModel(modelId, null); + String responseBody = EntityUtils.toString(getResponse.getEntity()); + assertNotNull(responseBody); + + Map responseMap = createParser(XContentType.JSON.xContent(), responseBody).map(); + + assertEquals(modelId, responseMap.get(MODEL_ID)); + + // Make sure training succeeds after 30 seconds + assertTrainingSucceeds(modelId, 30, 1000); + } + } diff --git a/src/test/java/org/opensearch/knn/plugin/transport/TrainingModelRequestTests.java b/src/test/java/org/opensearch/knn/plugin/transport/TrainingModelRequestTests.java index 06b6f3d01..2acce6662 100644 --- a/src/test/java/org/opensearch/knn/plugin/transport/TrainingModelRequestTests.java +++ b/src/test/java/org/opensearch/knn/plugin/transport/TrainingModelRequestTests.java @@ -172,6 +172,9 @@ public void testValidation_invalid_modelIdAlreadyExists() { ); when(modelDao.getMetadata(modelId)).thenReturn(modelMetadata); + // ModelId is not added to model graveyard + when(modelDao.isModelInGraveyard(modelId)).thenReturn(false); + // This cluster service will result in no validation exceptions ClusterService clusterService = getClusterServiceForValidReturns(trainingIndex, trainingField, dimension); @@ -186,6 +189,47 @@ public void testValidation_invalid_modelIdAlreadyExists() { assertTrue(validationErrors.get(0).contains("already exists")); } + // Check that the validation produces an exception when we are + // training a model with modelId that is in model graveyard + public void testValidation_blocked_modelId() { + + // Setup the training request + String modelId = "test-model-id"; + KNNMethodContext knnMethodContext = mock(KNNMethodContext.class); + when(knnMethodContext.validate()).thenReturn(null); + when(knnMethodContext.isTrainingRequired()).thenReturn(true); + int dimension = 10; + String trainingIndex = "test-training-index"; + String trainingField = "test-training-field"; + + TrainingModelRequest trainingModelRequest = new TrainingModelRequest( + modelId, + knnMethodContext, + dimension, + trainingIndex, + trainingField, + null, + null + ); + + // Mock the model dao to return true to recognize that the modelId is in graveyard + ModelDao modelDao = mock(ModelDao.class); + when(modelDao.isModelInGraveyard(modelId)).thenReturn(true); + + // This cluster service will result in no validation exceptions + ClusterService clusterService = getClusterServiceForValidReturns(trainingIndex, trainingField, dimension); + + // Initialize static components with the mocks + TrainingModelRequest.initialize(modelDao, clusterService); + + // Test that validation produces an error message that modelId is being deleted + ActionRequestValidationException exception = trainingModelRequest.validate(); + assertNotNull(exception); + List validationErrors = exception.validationErrors(); + assertEquals(1, validationErrors.size()); + assertTrue(validationErrors.get(0).contains("is being deleted")); + } + public void testValidation_invalid_invalidMethodContext() { // Check that validation produces exception when the method is invalid and does not require training diff --git a/src/test/java/org/opensearch/knn/plugin/transport/UpdateModelGraveyardRequestTests.java b/src/test/java/org/opensearch/knn/plugin/transport/UpdateModelGraveyardRequestTests.java new file mode 100644 index 000000000..7c38adc36 --- /dev/null +++ b/src/test/java/org/opensearch/knn/plugin/transport/UpdateModelGraveyardRequestTests.java @@ -0,0 +1,58 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.plugin.transport; + +import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.knn.KNNTestCase; +import java.io.IOException; + +public class UpdateModelGraveyardRequestTests extends KNNTestCase { + + public void testStreams() throws IOException { + String modelId = "test-model-id"; + boolean isRemoveRequest = false; + + UpdateModelGraveyardRequest updateModelGraveyardRequest = new UpdateModelGraveyardRequest(modelId, isRemoveRequest); + + BytesStreamOutput streamOutput = new BytesStreamOutput(); + updateModelGraveyardRequest.writeTo(streamOutput); + + UpdateModelGraveyardRequest updateModelGraveyardRequest1 = new UpdateModelGraveyardRequest(streamOutput.bytes().streamInput()); + + assertEquals(updateModelGraveyardRequest.getModelId(), updateModelGraveyardRequest1.getModelId()); + assertEquals(updateModelGraveyardRequest.isRemoveRequest(), updateModelGraveyardRequest1.isRemoveRequest()); + } + + public void testValidate() { + String modelId = "test-model-id"; + UpdateModelGraveyardRequest updateModelGraveyardRequest1 = new UpdateModelGraveyardRequest(modelId, false); + assertNull(updateModelGraveyardRequest1.validate()); + + UpdateModelGraveyardRequest updateModelGraveyardRequest2 = new UpdateModelGraveyardRequest(modelId, true); + assertNull(updateModelGraveyardRequest2.validate()); + + UpdateModelGraveyardRequest updateModelGraveyardRequest3 = new UpdateModelGraveyardRequest("", false); + assertNotNull(updateModelGraveyardRequest3.validate()); + + UpdateModelGraveyardRequest updateModelGraveyardRequest4 = new UpdateModelGraveyardRequest("", true); + assertNotNull(updateModelGraveyardRequest4.validate()); + } + + public void testGetModelId() { + String modelId = "test-model-id"; + UpdateModelGraveyardRequest updateModelGraveyardRequest = new UpdateModelGraveyardRequest(modelId, false); + + assertEquals(modelId, updateModelGraveyardRequest.getModelId()); + } + + public void testIsRemoveRequest() { + String modelId = "test-model-id"; + boolean isRemoveRequest = false; + UpdateModelGraveyardRequest updateModelGraveyardRequest = new UpdateModelGraveyardRequest(modelId, isRemoveRequest); + + assertEquals(isRemoveRequest, updateModelGraveyardRequest.isRemoveRequest()); + } +} diff --git a/src/test/java/org/opensearch/knn/plugin/transport/UpdateModelGraveyardTransportActionTests.java b/src/test/java/org/opensearch/knn/plugin/transport/UpdateModelGraveyardTransportActionTests.java new file mode 100644 index 000000000..6216f985d --- /dev/null +++ b/src/test/java/org/opensearch/knn/plugin/transport/UpdateModelGraveyardTransportActionTests.java @@ -0,0 +1,168 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.plugin.transport; + +import org.opensearch.action.ActionListener; +import org.opensearch.action.support.master.AcknowledgedResponse; +import org.opensearch.cluster.ClusterState; +import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.knn.KNNSingleNodeTestCase; +import org.opensearch.knn.indices.ModelGraveyard; +import org.opensearch.threadpool.ThreadPool; + +import java.io.IOException; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +public class UpdateModelGraveyardTransportActionTests extends KNNSingleNodeTestCase { + + public void testExecutor() { + UpdateModelGraveyardTransportAction updateModelGraveyardTransportAction = node().injector() + .getInstance(UpdateModelGraveyardTransportAction.class); + assertEquals(ThreadPool.Names.SAME, updateModelGraveyardTransportAction.executor()); + } + + public void testRead() throws IOException { + UpdateModelGraveyardTransportAction updateModelGraveyardTransportAction = node().injector() + .getInstance(UpdateModelGraveyardTransportAction.class); + AcknowledgedResponse acknowledgedResponse = new AcknowledgedResponse(true); + BytesStreamOutput streamOutput = new BytesStreamOutput(); + acknowledgedResponse.writeTo(streamOutput); + AcknowledgedResponse acknowledgedResponse1 = updateModelGraveyardTransportAction.read(streamOutput.bytes().streamInput()); + + assertEquals(acknowledgedResponse, acknowledgedResponse1); + } + + public void testClusterManagerOperation() throws InterruptedException { + + String modelId = "test-model-id"; + + // Get update transport action + UpdateModelGraveyardTransportAction updateModelGraveyardTransportAction = node().injector() + .getInstance(UpdateModelGraveyardTransportAction.class); + + // Generate update request to add modelId to model graveyard + UpdateModelGraveyardRequest addModelGraveyardRequest = new UpdateModelGraveyardRequest(modelId, false); + + // Get cluster state, update metadata, check cluster state - all asynchronously + final CountDownLatch inProgressLatch1 = new CountDownLatch(1); + client().admin().cluster().prepareState().execute(ActionListener.wrap(stateResponse1 -> { + ClusterState clusterState1 = stateResponse1.getState(); + updateModelGraveyardTransportAction.masterOperation( + addModelGraveyardRequest, + clusterState1, + ActionListener.wrap(acknowledgedResponse -> { + assertTrue(acknowledgedResponse.isAcknowledged()); + + client().admin().cluster().prepareState().execute(ActionListener.wrap(stateResponse2 -> { + ClusterState updatedClusterState = stateResponse2.getState(); + ModelGraveyard modelGraveyard = updatedClusterState.metadata().custom(ModelGraveyard.TYPE); + + assertNotNull(modelGraveyard); + assertEquals(1, modelGraveyard.size()); + assertTrue(modelGraveyard.contains(modelId)); + + inProgressLatch1.countDown(); + + }, e -> fail("Update failed:" + e))); + }, e -> fail("Update failed: " + e)) + ); + }, e -> fail("Update failed: " + e))); + + assertTrue(inProgressLatch1.await(60, TimeUnit.SECONDS)); + + String modelId1 = "test-model-id-1"; + // Generate update request to add modelId1 to model graveyard + UpdateModelGraveyardRequest addModelGraveyardRequest1 = new UpdateModelGraveyardRequest(modelId1, false); + + final CountDownLatch inProgressLatch2 = new CountDownLatch(1); + client().admin().cluster().prepareState().execute(ActionListener.wrap(stateResponse1 -> { + ClusterState clusterState1 = stateResponse1.getState(); + updateModelGraveyardTransportAction.masterOperation( + addModelGraveyardRequest1, + clusterState1, + ActionListener.wrap(acknowledgedResponse -> { + assertTrue(acknowledgedResponse.isAcknowledged()); + + client().admin().cluster().prepareState().execute(ActionListener.wrap(stateResponse2 -> { + ClusterState updatedClusterState = stateResponse2.getState(); + ModelGraveyard modelGraveyard = updatedClusterState.metadata().custom(ModelGraveyard.TYPE); + + assertNotNull(modelGraveyard); + assertEquals(2, modelGraveyard.size()); + assertTrue(modelGraveyard.contains(modelId1)); + + ModelGraveyard modelGraveyardPrev = clusterState1.metadata().custom(ModelGraveyard.TYPE); + assertFalse(modelGraveyardPrev.contains(modelId1)); + + // Assertions to validate ModelGraveyard Diff + ModelGraveyard.ModelGraveyardDiff diff = new ModelGraveyard.ModelGraveyardDiff(modelGraveyardPrev, modelGraveyard); + assertEquals(0, diff.getRemoved().size()); + assertEquals(1, diff.getAdded().size()); + assertTrue(diff.getAdded().contains(modelId1)); + + ModelGraveyard updatedModelGraveyard = diff.apply(modelGraveyardPrev); + assertEquals(2, updatedModelGraveyard.size()); + assertTrue(updatedModelGraveyard.contains(modelId)); + assertTrue(updatedModelGraveyard.contains(modelId1)); + + inProgressLatch2.countDown(); + }, e -> fail("Update failed"))); + }, e -> fail("Update failed")) + ); + }, e -> fail("Update failed"))); + + assertTrue(inProgressLatch2.await(60, TimeUnit.SECONDS)); + + // Generate remove request to remove the modelId from model graveyard + UpdateModelGraveyardRequest removeModelGraveyardRequest = new UpdateModelGraveyardRequest(modelId, true); + + final CountDownLatch inProgressLatch3 = new CountDownLatch(1); + client().admin().cluster().prepareState().execute(ActionListener.wrap(stateResponse1 -> { + ClusterState clusterState1 = stateResponse1.getState(); + updateModelGraveyardTransportAction.masterOperation( + removeModelGraveyardRequest, + clusterState1, + ActionListener.wrap(acknowledgedResponse -> { + assertTrue(acknowledgedResponse.isAcknowledged()); + + client().admin().cluster().prepareState().execute(ActionListener.wrap(stateResponse2 -> { + ClusterState updatedClusterState = stateResponse2.getState(); + ModelGraveyard modelGraveyard = updatedClusterState.metadata().custom(ModelGraveyard.TYPE); + + assertNotNull(modelGraveyard); + assertEquals(1, modelGraveyard.size()); + assertFalse(modelGraveyard.contains(modelId)); + + ModelGraveyard modelGraveyardPrev = clusterState1.metadata().custom(ModelGraveyard.TYPE); + assertTrue(modelGraveyardPrev.contains(modelId)); + + // Assertions to validate ModelGraveyard Diff + ModelGraveyard.ModelGraveyardDiff diff = new ModelGraveyard.ModelGraveyardDiff(modelGraveyardPrev, modelGraveyard); + assertEquals(1, diff.getRemoved().size()); + assertEquals(0, diff.getAdded().size()); + assertTrue(diff.getRemoved().contains(modelId)); + + ModelGraveyard updatedModelGraveyard = diff.apply(modelGraveyardPrev); + assertEquals(1, updatedModelGraveyard.size()); + assertFalse(updatedModelGraveyard.contains(modelId)); + assertTrue(updatedModelGraveyard.contains(modelId1)); + + inProgressLatch3.countDown(); + }, e -> fail("Update failed"))); + }, e -> fail("Update failed")) + ); + }, e -> fail("Update failed"))); + + assertTrue(inProgressLatch3.await(60, TimeUnit.SECONDS)); + } + + public void testCheckBlock() { + UpdateModelGraveyardTransportAction updateModelGraveyardTransportAction = node().injector() + .getInstance(UpdateModelGraveyardTransportAction.class); + assertNull(updateModelGraveyardTransportAction.checkBlock(null, null)); + } +} From 9b06925092dfaf01d8780579bfc1570635d980dd Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Thu, 21 Jul 2022 14:27:43 -0700 Subject: [PATCH 005/416] Move mappers to separate files (#448) (#450) * Move mappers to separate files Signed-off-by: Martin Gaievski --- .../org/opensearch/knn/index/IndexUtil.java | 1 + .../opensearch/knn/index/KNNIndexShard.java | 1 + .../opensearch/knn/index/KNNQueryBuilder.java | 1 + .../KNN80Codec/KNN80DocValuesConsumer.java | 2 +- .../{ => mapper}/KNNVectorFieldMapper.java | 259 +++--------------- .../knn/index/mapper/LegacyFieldMapper.java | 120 ++++++++ .../knn/index/mapper/MethodFieldMapper.java | 61 +++++ .../knn/index/mapper/ModelFieldMapper.java | 64 +++++ .../opensearch/knn/indices/ModelMetadata.java | 2 +- .../org/opensearch/knn/plugin/KNNPlugin.java | 2 +- .../knn/plugin/script/KNNScoringSpace.java | 2 +- .../plugin/script/KNNScoringSpaceUtil.java | 2 +- .../knn/index/KNNQueryBuilderTests.java | 1 + .../opensearch/knn/index/OpenSearchIT.java | 1 + .../KNN80DocValuesConsumerTests.java | 2 +- .../knn/index/codec/KNNCodecTestCase.java | 2 +- .../KNNVectorFieldMapperTests.java | 12 +- .../opensearch/knn/indices/ModelTests.java | 2 +- .../script/KNNScoringSpaceFactoryTests.java | 2 +- .../plugin/script/KNNScoringSpaceTests.java | 2 +- .../script/KNNScoringSpaceUtilTests.java | 2 +- .../knn/plugin/script/PainlessScriptIT.java | 2 +- .../transport/TrainingModelRequestTests.java | 2 +- 23 files changed, 312 insertions(+), 235 deletions(-) rename src/main/java/org/opensearch/knn/index/{ => mapper}/KNNVectorFieldMapper.java (68%) create mode 100644 src/main/java/org/opensearch/knn/index/mapper/LegacyFieldMapper.java create mode 100644 src/main/java/org/opensearch/knn/index/mapper/MethodFieldMapper.java create mode 100644 src/main/java/org/opensearch/knn/index/mapper/ModelFieldMapper.java rename src/test/java/org/opensearch/knn/index/{ => mapper}/KNNVectorFieldMapperTests.java (97%) diff --git a/src/main/java/org/opensearch/knn/index/IndexUtil.java b/src/main/java/org/opensearch/knn/index/IndexUtil.java index f0d9ee944..60156b4a7 100644 --- a/src/main/java/org/opensearch/knn/index/IndexUtil.java +++ b/src/main/java/org/opensearch/knn/index/IndexUtil.java @@ -17,6 +17,7 @@ import org.opensearch.cluster.metadata.MappingMetadata; import org.opensearch.common.ValidationException; import org.opensearch.knn.common.KNNConstants; +import org.opensearch.knn.index.mapper.KNNVectorFieldMapper; import org.opensearch.knn.index.util.KNNEngine; import org.opensearch.knn.indices.ModelDao; import org.opensearch.knn.indices.ModelMetadata; diff --git a/src/main/java/org/opensearch/knn/index/KNNIndexShard.java b/src/main/java/org/opensearch/knn/index/KNNIndexShard.java index c00eda255..7293708be 100644 --- a/src/main/java/org/opensearch/knn/index/KNNIndexShard.java +++ b/src/main/java/org/opensearch/knn/index/KNNIndexShard.java @@ -16,6 +16,7 @@ import org.apache.lucene.store.FilterDirectory; import org.opensearch.index.engine.Engine; import org.opensearch.index.shard.IndexShard; +import org.opensearch.knn.index.mapper.KNNVectorFieldMapper; import org.opensearch.knn.index.memory.NativeMemoryCacheManager; import org.opensearch.knn.index.memory.NativeMemoryEntryContext; import org.opensearch.knn.index.memory.NativeMemoryLoadStrategy; diff --git a/src/main/java/org/opensearch/knn/index/KNNQueryBuilder.java b/src/main/java/org/opensearch/knn/index/KNNQueryBuilder.java index d573b9194..862d680ce 100644 --- a/src/main/java/org/opensearch/knn/index/KNNQueryBuilder.java +++ b/src/main/java/org/opensearch/knn/index/KNNQueryBuilder.java @@ -6,6 +6,7 @@ package org.opensearch.knn.index; import org.opensearch.index.mapper.NumberFieldMapper; +import org.opensearch.knn.index.mapper.KNNVectorFieldMapper; import org.opensearch.knn.indices.ModelDao; import org.opensearch.knn.indices.ModelMetadata; import org.opensearch.knn.plugin.stats.KNNCounter; diff --git a/src/main/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumer.java b/src/main/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumer.java index 05a0873f7..491f6e1cc 100644 --- a/src/main/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumer.java +++ b/src/main/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumer.java @@ -30,7 +30,7 @@ import org.apache.lucene.index.SegmentWriteState; import org.apache.lucene.store.FSDirectory; import org.apache.lucene.store.FilterDirectory; -import org.opensearch.knn.index.KNNVectorFieldMapper; +import org.opensearch.knn.index.mapper.KNNVectorFieldMapper; import org.opensearch.knn.common.KNNConstants; import java.io.Closeable; diff --git a/src/main/java/org/opensearch/knn/index/KNNVectorFieldMapper.java b/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapper.java similarity index 68% rename from src/main/java/org/opensearch/knn/index/KNNVectorFieldMapper.java rename to src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapper.java index 628995b5c..45d34dd68 100644 --- a/src/main/java/org/opensearch/knn/index/KNNVectorFieldMapper.java +++ b/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapper.java @@ -3,16 +3,12 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.knn.index; +package org.opensearch.knn.index.mapper; import lombok.Getter; -import org.opensearch.common.Strings; import org.opensearch.common.ValidationException; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.knn.common.KNNConstants; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; import org.apache.lucene.document.FieldType; import org.apache.lucene.document.StoredField; import org.apache.lucene.index.DocValuesType; @@ -20,7 +16,6 @@ import org.apache.lucene.search.DocValuesFieldExistsQuery; import org.apache.lucene.search.Query; import org.opensearch.common.Explicit; -import org.opensearch.common.settings.Settings; import org.opensearch.common.xcontent.ToXContent; import org.opensearch.common.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentParser; @@ -36,9 +31,11 @@ import org.opensearch.index.mapper.ValueFetcher; import org.opensearch.index.query.QueryShardContext; import org.opensearch.index.query.QueryShardException; -import org.opensearch.knn.index.util.KNNEngine; +import org.opensearch.knn.index.KNNMethodContext; +import org.opensearch.knn.index.KNNSettings; +import org.opensearch.knn.index.KNNVectorIndexFieldData; +import org.opensearch.knn.index.VectorField; import org.opensearch.knn.indices.ModelDao; -import org.opensearch.knn.indices.ModelMetadata; import org.opensearch.search.aggregations.support.CoreValuesSourceType; import org.opensearch.search.lookup.SearchLookup; @@ -47,17 +44,10 @@ import java.util.Arrays; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.function.Supplier; -import static org.opensearch.knn.common.KNNConstants.DIMENSION; -import static org.opensearch.knn.common.KNNConstants.HNSW_ALGO_EF_CONSTRUCTION; -import static org.opensearch.knn.common.KNNConstants.HNSW_ALGO_M; -import static org.opensearch.knn.common.KNNConstants.KNN_ENGINE; import static org.opensearch.knn.common.KNNConstants.KNN_METHOD; -import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_SPACE_TYPE; -import static org.opensearch.knn.common.KNNConstants.MODEL_ID; -import static org.opensearch.knn.common.KNNConstants.PARAMETERS; -import static org.opensearch.knn.common.KNNConstants.SPACE_TYPE; /** * Field Mapper for KNN vector type. @@ -69,8 +59,6 @@ */ public abstract class KNNVectorFieldMapper extends ParametrizedFieldMapper { - private static Logger logger = LogManager.getLogger(KNNVectorFieldMapper.class); - public static final String CONTENT_TYPE = "knn_vector"; public static final String KNN_FIELD = "knn_field"; @@ -99,11 +87,13 @@ public static class Builder extends ParametrizedFieldMapper.Builder { } int value = XContentMapValues.nodeIntegerValue(o); if (value > MAX_DIMENSION) { - throw new IllegalArgumentException("Dimension value cannot be greater than " + MAX_DIMENSION + " for vector: " + name); + throw new IllegalArgumentException( + String.format("Dimension value cannot be greater than %s for vector: %s", MAX_DIMENSION, name) + ); } if (value <= 0) { - throw new IllegalArgumentException("Dimension value must be greater than 0 " + "for vector: " + name); + throw new IllegalArgumentException(String.format("Dimension value must be greater than 0 for vector: %s", name)); } return value; }, m -> toType(m).dimension); @@ -285,12 +275,12 @@ public Mapper.Builder parse(String name, Map node, ParserCont // is done before any mappers are built. Therefore, validation should be done during parsing // so that it can fail early. if (builder.knnMethodContext.get() != null && builder.modelId.get() != null) { - throw new IllegalArgumentException("Method and model can not be both specified in the mapping: " + name); + throw new IllegalArgumentException(String.format("Method and model can not be both specified in the mapping: %s", name)); } // Dimension should not be null unless modelId is used if (builder.dimension.getValue() == -1 && builder.modelId.get() == null) { - throw new IllegalArgumentException("Dimension value missing for vector: " + name); + throw new IllegalArgumentException(String.format("Dimension value missing for vector: %s", name)); } return builder; @@ -337,7 +327,7 @@ public Query existsQuery(QueryShardContext context) { public Query termQuery(Object value, QueryShardContext context) { throw new QueryShardException( context, - "KNN vector do not support exact searching, use KNN queries " + "instead: [" + name() + "]" + String.format("KNN vector do not support exact searching, use KNN queries instead: [%s]", name()) ); } @@ -392,16 +382,39 @@ protected void parseCreateField(ParseContext context) throws IOException { protected void parseCreateField(ParseContext context, int dimension) throws IOException { - if (!KNNSettings.isKNNPluginEnabled()) { - throw new IllegalStateException("KNN plugin is disabled. To enable " + "update knn.plugin.enabled setting to true"); + validateIfKNNPluginEnabled(); + validateIfCircuitBreakerIsNotTriggered(); + + Optional arrayOptional = getFloatsFromContext(context, dimension); + + if (!arrayOptional.isPresent()) { + return; } + final float[] array = arrayOptional.get(); + VectorField point = new VectorField(name(), array, fieldType); + context.doc().add(point); + if (fieldType.stored()) { + context.doc().add(new StoredField(name(), point.toString())); + } + context.path().remove(); + } + + void validateIfCircuitBreakerIsNotTriggered() { if (KNNSettings.isCircuitBreakerTriggered()) { throw new IllegalStateException( - "Indexing knn vector fields is rejected as circuit breaker triggered." + " Check _opendistro/_knn/stats for detailed state" + "Indexing knn vector fields is rejected as circuit breaker triggered. Check _opendistro/_knn/stats for detailed state" ); } + } + void validateIfKNNPluginEnabled() { + if (!KNNSettings.isKNNPluginEnabled()) { + throw new IllegalStateException("KNN plugin is disabled. To enable update knn.plugin.enabled setting to true"); + } + } + + Optional getFloatsFromContext(ParseContext context, int dimension) throws IOException { context.path().add(simpleName()); ArrayList vector = new ArrayList<>(); @@ -438,7 +451,7 @@ protected void parseCreateField(ParseContext context, int dimension) throws IOEx context.parser().nextToken(); } else if (token == XContentParser.Token.VALUE_NULL) { context.path().remove(); - return; + return Optional.empty(); } if (dimension != vector.size()) { @@ -451,14 +464,7 @@ protected void parseCreateField(ParseContext context, int dimension) throws IOEx for (Float f : vector) { array[i++] = f; } - - VectorField point = new VectorField(name(), array, fieldType); - - context.doc().add(point); - if (fieldType.stored()) { - context.doc().add(new StoredField(name(), point.toString())); - } - context.path().remove(); + return Optional.of(array); } @Override @@ -505,187 +511,4 @@ public static class Defaults { FIELD_TYPE.freeze(); } } - - /** - * Field mapper for original implementation - */ - protected static class LegacyFieldMapper extends KNNVectorFieldMapper { - - protected String spaceType; - protected String m; - protected String efConstruction; - - private LegacyFieldMapper( - String simpleName, - KNNVectorFieldType mappedFieldType, - MultiFields multiFields, - CopyTo copyTo, - Explicit ignoreMalformed, - boolean stored, - boolean hasDocValues, - String spaceType, - String m, - String efConstruction - ) { - super(simpleName, mappedFieldType, multiFields, copyTo, ignoreMalformed, stored, hasDocValues); - - this.spaceType = spaceType; - this.m = m; - this.efConstruction = efConstruction; - - this.fieldType = new FieldType(KNNVectorFieldMapper.Defaults.FIELD_TYPE); - - this.fieldType.putAttribute(DIMENSION, String.valueOf(dimension)); - this.fieldType.putAttribute(SPACE_TYPE, spaceType); - this.fieldType.putAttribute(KNN_ENGINE, KNNEngine.NMSLIB.getName()); - - // These are extra just for legacy - this.fieldType.putAttribute(HNSW_ALGO_M, m); - this.fieldType.putAttribute(HNSW_ALGO_EF_CONSTRUCTION, efConstruction); - - this.fieldType.freeze(); - } - - @Override - public ParametrizedFieldMapper.Builder getMergeBuilder() { - return new KNNVectorFieldMapper.Builder(simpleName(), this.spaceType, this.m, this.efConstruction).init(this); - } - - static String getSpaceType(Settings indexSettings) { - String spaceType = indexSettings.get(KNNSettings.INDEX_KNN_SPACE_TYPE.getKey()); - if (spaceType == null) { - logger.info( - "[KNN] The setting \"" - + METHOD_PARAMETER_SPACE_TYPE - + "\" was not set for the index. " - + "Likely caused by recent version upgrade. Setting the setting to the default value=" - + KNNSettings.INDEX_KNN_DEFAULT_SPACE_TYPE - ); - return KNNSettings.INDEX_KNN_DEFAULT_SPACE_TYPE; - } - return spaceType; - } - - static String getM(Settings indexSettings) { - String m = indexSettings.get(KNNSettings.INDEX_KNN_ALGO_PARAM_M_SETTING.getKey()); - if (m == null) { - logger.info( - "[KNN] The setting \"" - + HNSW_ALGO_M - + "\" was not set for the index. " - + "Likely caused by recent version upgrade. Setting the setting to the default value=" - + KNNSettings.INDEX_KNN_DEFAULT_ALGO_PARAM_M - ); - return String.valueOf(KNNSettings.INDEX_KNN_DEFAULT_ALGO_PARAM_M); - } - return m; - } - - static String getEfConstruction(Settings indexSettings) { - String efConstruction = indexSettings.get(KNNSettings.INDEX_KNN_ALGO_PARAM_EF_CONSTRUCTION_SETTING.getKey()); - if (efConstruction == null) { - logger.info( - "[KNN] The setting \"" - + HNSW_ALGO_EF_CONSTRUCTION - + "\" was not set for" - + " the index. Likely caused by recent version upgrade. Setting the setting to the default value=" - + KNNSettings.INDEX_KNN_DEFAULT_ALGO_PARAM_EF_CONSTRUCTION - ); - return String.valueOf(KNNSettings.INDEX_KNN_DEFAULT_ALGO_PARAM_EF_CONSTRUCTION); - } - return efConstruction; - } - } - - /** - * Field mapper for method definition in mapping - */ - protected static class MethodFieldMapper extends KNNVectorFieldMapper { - - private MethodFieldMapper( - String simpleName, - KNNVectorFieldType mappedFieldType, - MultiFields multiFields, - CopyTo copyTo, - Explicit ignoreMalformed, - boolean stored, - boolean hasDocValues, - KNNMethodContext knnMethodContext - ) { - - super(simpleName, mappedFieldType, multiFields, copyTo, ignoreMalformed, stored, hasDocValues); - - this.knnMethod = knnMethodContext; - - this.fieldType = new FieldType(KNNVectorFieldMapper.Defaults.FIELD_TYPE); - - this.fieldType.putAttribute(DIMENSION, String.valueOf(dimension)); - this.fieldType.putAttribute(SPACE_TYPE, knnMethodContext.getSpaceType().getValue()); - - KNNEngine knnEngine = knnMethodContext.getKnnEngine(); - this.fieldType.putAttribute(KNN_ENGINE, knnEngine.getName()); - - try { - this.fieldType.putAttribute( - PARAMETERS, - Strings.toString(XContentFactory.jsonBuilder().map(knnEngine.getMethodAsMap(knnMethodContext))) - ); - } catch (IOException ioe) { - throw new RuntimeException("Unable to create KNNVectorFieldMapper: " + ioe); - } - - this.fieldType.freeze(); - } - } - - /** - * Field mapper for model in mapping - */ - protected static class ModelFieldMapper extends KNNVectorFieldMapper { - - private ModelFieldMapper( - String simpleName, - KNNVectorFieldType mappedFieldType, - MultiFields multiFields, - CopyTo copyTo, - Explicit ignoreMalformed, - boolean stored, - boolean hasDocValues, - ModelDao modelDao, - String modelId - ) { - super(simpleName, mappedFieldType, multiFields, copyTo, ignoreMalformed, stored, hasDocValues); - - this.modelId = modelId; - this.modelDao = modelDao; - - this.fieldType = new FieldType(KNNVectorFieldMapper.Defaults.FIELD_TYPE); - this.fieldType.putAttribute(MODEL_ID, modelId); - this.fieldType.freeze(); - } - - @Override - protected void parseCreateField(ParseContext context) throws IOException { - // For the model field mapper, we cannot validate the model during index creation due to - // an issue with reading cluster state during mapper creation. So, we need to validate the - // model when ingestion starts. - ModelMetadata modelMetadata = this.modelDao.getMetadata(modelId); - - if (modelMetadata == null) { - throw new IllegalStateException( - "Model \"" - + modelId - + "\" from " - + context.mapperService().index().getName() - + "'s mapping does not exist. Because the " - + "\"" - + MODEL_ID - + "\" parameter is not updateable, this index will need to " - + "be recreated with a valid model." - ); - } - - parseCreateField(context, modelMetadata.getDimension()); - } - } } diff --git a/src/main/java/org/opensearch/knn/index/mapper/LegacyFieldMapper.java b/src/main/java/org/opensearch/knn/index/mapper/LegacyFieldMapper.java new file mode 100644 index 000000000..868aec3e6 --- /dev/null +++ b/src/main/java/org/opensearch/knn/index/mapper/LegacyFieldMapper.java @@ -0,0 +1,120 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.mapper; + +import lombok.extern.log4j.Log4j2; +import org.apache.lucene.document.FieldType; +import org.opensearch.common.Explicit; +import org.opensearch.common.settings.Settings; +import org.opensearch.index.mapper.ParametrizedFieldMapper; +import org.opensearch.knn.index.KNNSettings; +import org.opensearch.knn.index.util.KNNEngine; + +import static org.opensearch.knn.common.KNNConstants.DIMENSION; +import static org.opensearch.knn.common.KNNConstants.HNSW_ALGO_EF_CONSTRUCTION; +import static org.opensearch.knn.common.KNNConstants.HNSW_ALGO_M; +import static org.opensearch.knn.common.KNNConstants.KNN_ENGINE; +import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_SPACE_TYPE; +import static org.opensearch.knn.common.KNNConstants.SPACE_TYPE; + +/** + * Field mapper for original implementation. It defaults to using nmslib as the engine and retrieves parameters from index settings. + * + * Example of this mapper output: + * + * { + * "type": "knn_vector", + * "dimension": 128 + * } + */ +@Log4j2 +public class LegacyFieldMapper extends KNNVectorFieldMapper { + + protected String spaceType; + protected String m; + protected String efConstruction; + + LegacyFieldMapper( + String simpleName, + KNNVectorFieldType mappedFieldType, + MultiFields multiFields, + CopyTo copyTo, + Explicit ignoreMalformed, + boolean stored, + boolean hasDocValues, + String spaceType, + String m, + String efConstruction + ) { + super(simpleName, mappedFieldType, multiFields, copyTo, ignoreMalformed, stored, hasDocValues); + + this.spaceType = spaceType; + this.m = m; + this.efConstruction = efConstruction; + + this.fieldType = new FieldType(KNNVectorFieldMapper.Defaults.FIELD_TYPE); + + this.fieldType.putAttribute(DIMENSION, String.valueOf(dimension)); + this.fieldType.putAttribute(SPACE_TYPE, spaceType); + this.fieldType.putAttribute(KNN_ENGINE, KNNEngine.NMSLIB.getName()); + + // These are extra just for legacy + this.fieldType.putAttribute(HNSW_ALGO_M, m); + this.fieldType.putAttribute(HNSW_ALGO_EF_CONSTRUCTION, efConstruction); + + this.fieldType.freeze(); + } + + @Override + public ParametrizedFieldMapper.Builder getMergeBuilder() { + return new KNNVectorFieldMapper.Builder(simpleName(), this.spaceType, this.m, this.efConstruction).init(this); + } + + static String getSpaceType(Settings indexSettings) { + String spaceType = indexSettings.get(KNNSettings.INDEX_KNN_SPACE_TYPE.getKey()); + if (spaceType == null) { + log.info( + String.format( + "[KNN] The setting \"%s\" was not set for the index. Likely caused by recent version upgrade. Setting the setting to the default value=%s", + METHOD_PARAMETER_SPACE_TYPE, + KNNSettings.INDEX_KNN_DEFAULT_SPACE_TYPE + ) + ); + return KNNSettings.INDEX_KNN_DEFAULT_SPACE_TYPE; + } + return spaceType; + } + + static String getM(Settings indexSettings) { + String m = indexSettings.get(KNNSettings.INDEX_KNN_ALGO_PARAM_M_SETTING.getKey()); + if (m == null) { + log.info( + String.format( + "[KNN] The setting \"%s\" was not set for the index. Likely caused by recent version upgrade. Setting the setting to the default value=%s", + HNSW_ALGO_M, + KNNSettings.INDEX_KNN_DEFAULT_ALGO_PARAM_M + ) + ); + return String.valueOf(KNNSettings.INDEX_KNN_DEFAULT_ALGO_PARAM_M); + } + return m; + } + + static String getEfConstruction(Settings indexSettings) { + String efConstruction = indexSettings.get(KNNSettings.INDEX_KNN_ALGO_PARAM_EF_CONSTRUCTION_SETTING.getKey()); + if (efConstruction == null) { + log.info( + String.format( + "[KNN] The setting \"%s\" was not set for the index. Likely caused by recent version upgrade. Setting the setting to the default value=%s", + HNSW_ALGO_EF_CONSTRUCTION, + KNNSettings.INDEX_KNN_DEFAULT_ALGO_PARAM_EF_CONSTRUCTION + ) + ); + return String.valueOf(KNNSettings.INDEX_KNN_DEFAULT_ALGO_PARAM_EF_CONSTRUCTION); + } + return efConstruction; + } +} diff --git a/src/main/java/org/opensearch/knn/index/mapper/MethodFieldMapper.java b/src/main/java/org/opensearch/knn/index/mapper/MethodFieldMapper.java new file mode 100644 index 000000000..42c12a5db --- /dev/null +++ b/src/main/java/org/opensearch/knn/index/mapper/MethodFieldMapper.java @@ -0,0 +1,61 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.mapper; + +import org.apache.lucene.document.FieldType; +import org.opensearch.common.Explicit; +import org.opensearch.common.Strings; +import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.knn.index.KNNMethodContext; +import org.opensearch.knn.index.util.KNNEngine; + +import java.io.IOException; + +import static org.opensearch.knn.common.KNNConstants.DIMENSION; +import static org.opensearch.knn.common.KNNConstants.KNN_ENGINE; +import static org.opensearch.knn.common.KNNConstants.PARAMETERS; +import static org.opensearch.knn.common.KNNConstants.SPACE_TYPE; + +/** + * Field mapper for method definition in mapping + */ +public class MethodFieldMapper extends KNNVectorFieldMapper { + + MethodFieldMapper( + String simpleName, + KNNVectorFieldType mappedFieldType, + MultiFields multiFields, + CopyTo copyTo, + Explicit ignoreMalformed, + boolean stored, + boolean hasDocValues, + KNNMethodContext knnMethodContext + ) { + + super(simpleName, mappedFieldType, multiFields, copyTo, ignoreMalformed, stored, hasDocValues); + + this.knnMethod = knnMethodContext; + + this.fieldType = new FieldType(KNNVectorFieldMapper.Defaults.FIELD_TYPE); + + this.fieldType.putAttribute(DIMENSION, String.valueOf(dimension)); + this.fieldType.putAttribute(SPACE_TYPE, knnMethodContext.getSpaceType().getValue()); + + KNNEngine knnEngine = knnMethodContext.getKnnEngine(); + this.fieldType.putAttribute(KNN_ENGINE, knnEngine.getName()); + + try { + this.fieldType.putAttribute( + PARAMETERS, + Strings.toString(XContentFactory.jsonBuilder().map(knnEngine.getMethodAsMap(knnMethodContext))) + ); + } catch (IOException ioe) { + throw new RuntimeException(String.format("Unable to create KNNVectorFieldMapper: %s", ioe)); + } + + this.fieldType.freeze(); + } +} diff --git a/src/main/java/org/opensearch/knn/index/mapper/ModelFieldMapper.java b/src/main/java/org/opensearch/knn/index/mapper/ModelFieldMapper.java new file mode 100644 index 000000000..0fa116937 --- /dev/null +++ b/src/main/java/org/opensearch/knn/index/mapper/ModelFieldMapper.java @@ -0,0 +1,64 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.mapper; + +import org.apache.lucene.document.FieldType; +import org.opensearch.common.Explicit; +import org.opensearch.index.mapper.ParseContext; +import org.opensearch.knn.indices.ModelDao; +import org.opensearch.knn.indices.ModelMetadata; + +import java.io.IOException; + +import static org.opensearch.knn.common.KNNConstants.MODEL_ID; + +/** + * Field mapper for model in mapping + */ +public class ModelFieldMapper extends KNNVectorFieldMapper { + + ModelFieldMapper( + String simpleName, + KNNVectorFieldType mappedFieldType, + MultiFields multiFields, + CopyTo copyTo, + Explicit ignoreMalformed, + boolean stored, + boolean hasDocValues, + ModelDao modelDao, + String modelId + ) { + super(simpleName, mappedFieldType, multiFields, copyTo, ignoreMalformed, stored, hasDocValues); + + this.modelId = modelId; + this.modelDao = modelDao; + + this.fieldType = new FieldType(KNNVectorFieldMapper.Defaults.FIELD_TYPE); + this.fieldType.putAttribute(MODEL_ID, modelId); + this.fieldType.freeze(); + } + + @Override + protected void parseCreateField(ParseContext context) throws IOException { + // For the model field mapper, we cannot validate the model during index creation due to + // an issue with reading cluster state during mapper creation. So, we need to validate the + // model when ingestion starts. + ModelMetadata modelMetadata = this.modelDao.getMetadata(modelId); + + if (modelMetadata == null) { + throw new IllegalStateException( + String.format( + "Model \"%s\" from %s's mapping does not exist. Because the \"%s\" parameter is not updatable, this index will need to be recreated with a valid model.", + modelId, + context.mapperService().index().getName(), + MODEL_ID + ) + ); + } + + parseCreateField(context, modelMetadata.getDimension()); + } +} diff --git a/src/main/java/org/opensearch/knn/indices/ModelMetadata.java b/src/main/java/org/opensearch/knn/indices/ModelMetadata.java index 9aa0d133b..1f0ea786f 100644 --- a/src/main/java/org/opensearch/knn/indices/ModelMetadata.java +++ b/src/main/java/org/opensearch/knn/indices/ModelMetadata.java @@ -34,7 +34,7 @@ import static org.opensearch.knn.common.KNNConstants.MODEL_ERROR; import static org.opensearch.knn.common.KNNConstants.MODEL_STATE; import static org.opensearch.knn.common.KNNConstants.MODEL_TIMESTAMP; -import static org.opensearch.knn.index.KNNVectorFieldMapper.MAX_DIMENSION; +import static org.opensearch.knn.index.mapper.KNNVectorFieldMapper.MAX_DIMENSION; public class ModelMetadata implements Writeable, ToXContentObject { diff --git a/src/main/java/org/opensearch/knn/plugin/KNNPlugin.java b/src/main/java/org/opensearch/knn/plugin/KNNPlugin.java index f3b7d8197..fd2ef0780 100644 --- a/src/main/java/org/opensearch/knn/plugin/KNNPlugin.java +++ b/src/main/java/org/opensearch/knn/plugin/KNNPlugin.java @@ -13,7 +13,7 @@ import org.opensearch.knn.index.KNNCircuitBreaker; import org.opensearch.knn.index.KNNQueryBuilder; import org.opensearch.knn.index.KNNSettings; -import org.opensearch.knn.index.KNNVectorFieldMapper; +import org.opensearch.knn.index.mapper.KNNVectorFieldMapper; import org.opensearch.knn.index.KNNWeight; import org.opensearch.knn.index.codec.KNNCodecService; diff --git a/src/main/java/org/opensearch/knn/plugin/script/KNNScoringSpace.java b/src/main/java/org/opensearch/knn/plugin/script/KNNScoringSpace.java index 3e066b7c9..4a419fd23 100644 --- a/src/main/java/org/opensearch/knn/plugin/script/KNNScoringSpace.java +++ b/src/main/java/org/opensearch/knn/plugin/script/KNNScoringSpace.java @@ -5,7 +5,7 @@ package org.opensearch.knn.plugin.script; -import org.opensearch.knn.index.KNNVectorFieldMapper; +import org.opensearch.knn.index.mapper.KNNVectorFieldMapper; import org.opensearch.knn.index.KNNWeight; import org.apache.lucene.index.LeafReaderContext; import org.opensearch.index.mapper.MappedFieldType; diff --git a/src/main/java/org/opensearch/knn/plugin/script/KNNScoringSpaceUtil.java b/src/main/java/org/opensearch/knn/plugin/script/KNNScoringSpaceUtil.java index 4d64b5b96..6f68d16b6 100644 --- a/src/main/java/org/opensearch/knn/plugin/script/KNNScoringSpaceUtil.java +++ b/src/main/java/org/opensearch/knn/plugin/script/KNNScoringSpaceUtil.java @@ -5,7 +5,7 @@ package org.opensearch.knn.plugin.script; -import org.opensearch.knn.index.KNNVectorFieldMapper; +import org.opensearch.knn.index.mapper.KNNVectorFieldMapper; import org.opensearch.knn.plugin.stats.KNNCounter; import org.opensearch.index.mapper.BinaryFieldMapper; import org.opensearch.index.mapper.MappedFieldType; diff --git a/src/test/java/org/opensearch/knn/index/KNNQueryBuilderTests.java b/src/test/java/org/opensearch/knn/index/KNNQueryBuilderTests.java index ccf6bcfca..8a252ca72 100644 --- a/src/test/java/org/opensearch/knn/index/KNNQueryBuilderTests.java +++ b/src/test/java/org/opensearch/knn/index/KNNQueryBuilderTests.java @@ -12,6 +12,7 @@ import org.opensearch.index.Index; import org.opensearch.index.mapper.NumberFieldMapper; import org.opensearch.index.query.QueryShardContext; +import org.opensearch.knn.index.mapper.KNNVectorFieldMapper; import org.opensearch.knn.indices.ModelDao; import org.opensearch.knn.indices.ModelMetadata; diff --git a/src/test/java/org/opensearch/knn/index/OpenSearchIT.java b/src/test/java/org/opensearch/knn/index/OpenSearchIT.java index 6a06de14a..3fd492782 100644 --- a/src/test/java/org/opensearch/knn/index/OpenSearchIT.java +++ b/src/test/java/org/opensearch/knn/index/OpenSearchIT.java @@ -27,6 +27,7 @@ import org.opensearch.index.query.ExistsQueryBuilder; import org.opensearch.knn.TestUtils; import org.opensearch.knn.common.KNNConstants; +import org.opensearch.knn.index.mapper.KNNVectorFieldMapper; import org.opensearch.knn.index.util.KNNEngine; import org.opensearch.knn.plugin.script.KNNScoringUtil; import org.opensearch.rest.RestStatus; diff --git a/src/test/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumerTests.java b/src/test/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumerTests.java index b86b4051a..19c57673d 100644 --- a/src/test/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumerTests.java +++ b/src/test/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumerTests.java @@ -26,7 +26,7 @@ import org.opensearch.knn.KNNTestCase; import org.opensearch.knn.common.KNNConstants; import org.opensearch.knn.index.KNNMethodContext; -import org.opensearch.knn.index.KNNVectorFieldMapper; +import org.opensearch.knn.index.mapper.KNNVectorFieldMapper; import org.opensearch.knn.index.MethodComponentContext; import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.codec.KNN87Codec.KNN87Codec; diff --git a/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestCase.java b/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestCase.java index b67bd1115..042dad331 100644 --- a/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestCase.java +++ b/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestCase.java @@ -15,7 +15,7 @@ import org.opensearch.knn.jni.JNIService; import org.opensearch.knn.index.KNNQuery; import org.opensearch.knn.index.KNNSettings; -import org.opensearch.knn.index.KNNVectorFieldMapper; +import org.opensearch.knn.index.mapper.KNNVectorFieldMapper; import org.opensearch.knn.index.KNNWeight; import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.VectorField; diff --git a/src/test/java/org/opensearch/knn/index/KNNVectorFieldMapperTests.java b/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java similarity index 97% rename from src/test/java/org/opensearch/knn/index/KNNVectorFieldMapperTests.java rename to src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java index b6a65cce9..6f489bd92 100644 --- a/src/test/java/org/opensearch/knn/index/KNNVectorFieldMapperTests.java +++ b/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.knn.index; +package org.opensearch.knn.index.mapper; import com.google.common.collect.ImmutableMap; import org.opensearch.knn.KNNTestCase; @@ -17,6 +17,10 @@ import org.opensearch.index.mapper.ContentPath; import org.opensearch.index.mapper.Mapper; import org.opensearch.index.mapper.MapperService; +import org.opensearch.knn.index.KNNMethodContext; +import org.opensearch.knn.index.KNNSettings; +import org.opensearch.knn.index.MethodComponentContext; +import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.util.KNNEngine; import org.opensearch.knn.indices.ModelDao; import org.opensearch.knn.indices.ModelMetadata; @@ -79,7 +83,7 @@ public void testBuilder_build_fromKnnMethodContext() { Mapper.BuilderContext builderContext = new Mapper.BuilderContext(settings, new ContentPath()); KNNVectorFieldMapper knnVectorFieldMapper = builder.build(builderContext); - assertTrue(knnVectorFieldMapper instanceof KNNVectorFieldMapper.MethodFieldMapper); + assertTrue(knnVectorFieldMapper instanceof MethodFieldMapper); assertNotNull(knnVectorFieldMapper.knnMethod); assertNull(knnVectorFieldMapper.modelId); } @@ -116,7 +120,7 @@ public void testBuilder_build_fromModel() { when(modelDao.getMetadata(modelId)).thenReturn(mockedModelMetadata); KNNVectorFieldMapper knnVectorFieldMapper = builder.build(builderContext); - assertTrue(knnVectorFieldMapper instanceof KNNVectorFieldMapper.ModelFieldMapper); + assertTrue(knnVectorFieldMapper instanceof ModelFieldMapper); assertNotNull(knnVectorFieldMapper.modelId); assertNull(knnVectorFieldMapper.knnMethod); } @@ -140,7 +144,7 @@ public void testBuilder_build_fromLegacy() { Mapper.BuilderContext builderContext = new Mapper.BuilderContext(settings, new ContentPath()); KNNVectorFieldMapper knnVectorFieldMapper = builder.build(builderContext); - assertTrue(knnVectorFieldMapper instanceof KNNVectorFieldMapper.LegacyFieldMapper); + assertTrue(knnVectorFieldMapper instanceof LegacyFieldMapper); assertNull(knnVectorFieldMapper.modelId); assertNull(knnVectorFieldMapper.knnMethod); diff --git a/src/test/java/org/opensearch/knn/indices/ModelTests.java b/src/test/java/org/opensearch/knn/indices/ModelTests.java index 7a01fd528..fb1e807fd 100644 --- a/src/test/java/org/opensearch/knn/indices/ModelTests.java +++ b/src/test/java/org/opensearch/knn/indices/ModelTests.java @@ -21,7 +21,7 @@ import java.util.HashMap; import java.util.Map; -import static org.opensearch.knn.index.KNNVectorFieldMapper.MAX_DIMENSION; +import static org.opensearch.knn.index.mapper.KNNVectorFieldMapper.MAX_DIMENSION; public class ModelTests extends KNNTestCase { diff --git a/src/test/java/org/opensearch/knn/plugin/script/KNNScoringSpaceFactoryTests.java b/src/test/java/org/opensearch/knn/plugin/script/KNNScoringSpaceFactoryTests.java index 70dbee248..24bc74ff4 100644 --- a/src/test/java/org/opensearch/knn/plugin/script/KNNScoringSpaceFactoryTests.java +++ b/src/test/java/org/opensearch/knn/plugin/script/KNNScoringSpaceFactoryTests.java @@ -6,7 +6,7 @@ package org.opensearch.knn.plugin.script; import org.opensearch.knn.KNNTestCase; -import org.opensearch.knn.index.KNNVectorFieldMapper; +import org.opensearch.knn.index.mapper.KNNVectorFieldMapper; import org.opensearch.knn.index.SpaceType; import org.opensearch.index.mapper.NumberFieldMapper; diff --git a/src/test/java/org/opensearch/knn/plugin/script/KNNScoringSpaceTests.java b/src/test/java/org/opensearch/knn/plugin/script/KNNScoringSpaceTests.java index b40cc9e86..c80949b43 100644 --- a/src/test/java/org/opensearch/knn/plugin/script/KNNScoringSpaceTests.java +++ b/src/test/java/org/opensearch/knn/plugin/script/KNNScoringSpaceTests.java @@ -7,7 +7,7 @@ import org.opensearch.knn.KNNTestCase; import org.opensearch.knn.index.KNNMethodContext; -import org.opensearch.knn.index.KNNVectorFieldMapper; +import org.opensearch.knn.index.mapper.KNNVectorFieldMapper; import org.opensearch.index.mapper.BinaryFieldMapper; import org.opensearch.index.mapper.NumberFieldMapper; diff --git a/src/test/java/org/opensearch/knn/plugin/script/KNNScoringSpaceUtilTests.java b/src/test/java/org/opensearch/knn/plugin/script/KNNScoringSpaceUtilTests.java index 789432ec8..92fd56e45 100644 --- a/src/test/java/org/opensearch/knn/plugin/script/KNNScoringSpaceUtilTests.java +++ b/src/test/java/org/opensearch/knn/plugin/script/KNNScoringSpaceUtilTests.java @@ -6,7 +6,7 @@ package org.opensearch.knn.plugin.script; import org.opensearch.knn.KNNTestCase; -import org.opensearch.knn.index.KNNVectorFieldMapper; +import org.opensearch.knn.index.mapper.KNNVectorFieldMapper; import org.opensearch.index.mapper.BinaryFieldMapper; import org.opensearch.index.mapper.NumberFieldMapper; diff --git a/src/test/java/org/opensearch/knn/plugin/script/PainlessScriptIT.java b/src/test/java/org/opensearch/knn/plugin/script/PainlessScriptIT.java index a8617896b..0f656e2d1 100644 --- a/src/test/java/org/opensearch/knn/plugin/script/PainlessScriptIT.java +++ b/src/test/java/org/opensearch/knn/plugin/script/PainlessScriptIT.java @@ -7,7 +7,7 @@ import org.opensearch.knn.KNNRestTestCase; import org.opensearch.knn.KNNResult; -import org.opensearch.knn.index.KNNVectorFieldMapper; +import org.opensearch.knn.index.mapper.KNNVectorFieldMapper; import org.apache.http.util.EntityUtils; import org.opensearch.client.Request; import org.opensearch.client.Response; diff --git a/src/test/java/org/opensearch/knn/plugin/transport/TrainingModelRequestTests.java b/src/test/java/org/opensearch/knn/plugin/transport/TrainingModelRequestTests.java index 2acce6662..0ed45dce3 100644 --- a/src/test/java/org/opensearch/knn/plugin/transport/TrainingModelRequestTests.java +++ b/src/test/java/org/opensearch/knn/plugin/transport/TrainingModelRequestTests.java @@ -25,7 +25,7 @@ import org.opensearch.knn.KNNTestCase; import org.opensearch.knn.common.KNNConstants; import org.opensearch.knn.index.KNNMethodContext; -import org.opensearch.knn.index.KNNVectorFieldMapper; +import org.opensearch.knn.index.mapper.KNNVectorFieldMapper; import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.util.KNNEngine; import org.opensearch.knn.indices.ModelDao; From f0ff6e1a2025aa455d9644956a2277efbd642005 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Wed, 27 Jul 2022 17:42:09 -0500 Subject: [PATCH 006/416] Add fix to flaky test in ModelDaoTests (#463) (#470) Signed-off-by: Naveen Tatikonda (cherry picked from commit 7501bfc6029ff6d5579e0654cdab376f9889b87b) Co-authored-by: Naveen Tatikonda --- .../opensearch/knn/indices/ModelDaoTests.java | 32 ++++++------------- 1 file changed, 10 insertions(+), 22 deletions(-) diff --git a/src/test/java/org/opensearch/knn/indices/ModelDaoTests.java b/src/test/java/org/opensearch/knn/indices/ModelDaoTests.java index 73d285886..678301424 100644 --- a/src/test/java/org/opensearch/knn/indices/ModelDaoTests.java +++ b/src/test/java/org/opensearch/knn/indices/ModelDaoTests.java @@ -765,27 +765,10 @@ public void testDeleteWithStepListeners() throws IOException, InterruptedExcepti assertTrue(inProgressLatch.await(100, TimeUnit.SECONDS)); } - public void testDeleteWithStepListenersOnFailure() throws IOException, InterruptedException, ExecutionException { + // Some exception occurs during the process of deletion and validate that the model is unblocked + public void testDeleteWithStepListenersOnFailureModelUnblocked() throws InterruptedException { String modelId = "test-model-id-delete1"; ModelDao modelDao = ModelDao.OpenSearchKNNModelDao.getInstance(); - byte[] modelBlob = "deleteModel".getBytes(); - int dimension = 2; - createIndex(MODEL_INDEX_NAME); - Model model = new Model( - new ModelMetadata( - KNNEngine.DEFAULT, - SpaceType.DEFAULT, - dimension, - ModelState.CREATED, - ZonedDateTime.now(ZoneOffset.UTC).toString(), - "", - "" - ), - modelBlob, - modelId - ); - - addDoc(model); // We will validate if the modelId gets unblocked when some exception occurs // during the process of deletion after adding that modelId to blocked list @@ -821,16 +804,21 @@ public void testDeleteWithStepListenersOnFailure() throws IOException, Interrupt // Asserting that model is unblocked assertFalse(modelDao.isModelInGraveyard(modelId)); assertNotNull(exp.getMessage()); + inProgressLatch.countDown(); }, exception -> fail(exception.getMessage())) ); }) ); - inProgressLatch.countDown(); }, exception -> fail(exception.getMessage())); assertTrue(inProgressLatch.await(100, TimeUnit.SECONDS)); + } + + // Some exception occurs during the process of deletion and unblocking model request also fails + public void testDeleteWithStepListenersOnFailureModelBlocked() throws InterruptedException { + String modelId = "test-model-id-delete2"; + ModelDao modelDao = ModelDao.OpenSearchKNNModelDao.getInstance(); - // Some exception occurs during the process of deletion and unblocking model request also fails final CountDownLatch inProgressLatch1 = new CountDownLatch(1); StepListener blockModelIdStep1 = new StepListener<>(); @@ -864,11 +852,11 @@ public void testDeleteWithStepListenersOnFailure() throws IOException, Interrupt assertTrue(modelDao.isModelInGraveyard(modelId)); assertNotNull(exp.getMessage()); assertNotNull(unblockingFailedException.getMessage()); + inProgressLatch1.countDown(); }) ); }) ); - inProgressLatch1.countDown(); }, exception -> fail(exception.getMessage())); assertTrue(inProgressLatch1.await(100, TimeUnit.SECONDS)); From 6ab1a99890a0bb3ed6aa511f0b25abf31c2547a1 Mon Sep 17 00:00:00 2001 From: John Mazanec Date: Thu, 28 Jul 2022 11:20:19 -0700 Subject: [PATCH 007/416] Staging for version increment automation (#475) Signed-off-by: John Mazanec --- build.gradle | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/build.gradle b/build.gradle index 63a299666..e6d82a7d0 100644 --- a/build.gradle +++ b/build.gradle @@ -270,3 +270,21 @@ run { } } } + +// updateVersion: Task to auto increment to the next development iteration +task updateVersion { + onlyIf { System.getProperty('newVersion') } + doLast { + ext.newVersion = System.getProperty('newVersion') + println "Setting version to ${newVersion}." + // String tokenization to support -SNAPSHOT + // Include the required files that needs to be updated with new Version + ant.replaceregexp(match: opensearch_version.tokenize('-')[0], replace: newVersion.tokenize('-')[0], flags:'g', byline:true) { + fileset(dir: projectDir) { + // Include the required files that needs to be updated with new Version + include(name: ".github/workflows/backwards_compatibility_tests_workflow.yml") + } + } + ant.replaceregexp(file:'build.gradle', match: '"opensearch.version", "\\d.*"', replace: '"opensearch.version", "' + newVersion.tokenize('-')[0] + '-SNAPSHOT"', flags:'g', byline:true) + } +} From 4289aeb9097835ffc4e2fdd0a66161c06c45e014 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Thu, 28 Jul 2022 16:41:13 -0700 Subject: [PATCH 008/416] Bump default bwc version to 1.3.4 (#477) (#478) Signed-off-by: Junqiu Lei --- gradle.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle.properties b/gradle.properties index 162d212e5..c36e5cca0 100644 --- a/gradle.properties +++ b/gradle.properties @@ -4,7 +4,7 @@ # version=1.0.0 -systemProp.bwc.version=1.3.2 +systemProp.bwc.version=1.3.4 org.gradle.jvmargs=--add-exports jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED \ --add-exports jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED \ From dee2fc2e71c336f23f13118ed17b2f0239060904 Mon Sep 17 00:00:00 2001 From: Naveen Tatikonda Date: Fri, 29 Jul 2022 09:48:48 -0500 Subject: [PATCH 009/416] Read BWC Version from Github workflow (#476) (#479) Signed-off-by: Naveen Tatikonda (cherry picked from commit 9f71ede43c7d6e7e90156fae65d7d45f676cc39e) --- .../workflows/backwards_compatibility_tests_workflow.yml | 4 ++-- qa/rolling-upgrade/build.gradle | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/backwards_compatibility_tests_workflow.yml b/.github/workflows/backwards_compatibility_tests_workflow.yml index ce654eee9..165f29369 100644 --- a/.github/workflows/backwards_compatibility_tests_workflow.yml +++ b/.github/workflows/backwards_compatibility_tests_workflow.yml @@ -52,7 +52,7 @@ jobs: name: k-NN Rolling-Upgrade BWC Tests runs-on: ubuntu-latest env: - OPENSEARCH_VERSION_ROLLING_UPGRADE: ${{ matrix.opensearch_version }} + BWC_VERSION_ROLLING_UPGRADE: ${{ matrix.bwc_version }} steps: - name: Checkout k-NN @@ -70,4 +70,4 @@ jobs: - name: Run k-NN Rolling-Upgrade BWC Tests from BWCVersion-${{ matrix.bwc_version }} to OpenSearch Version-${{ matrix.opensearch_version }} run: | echo "Running rolling-upgrade backwards compatibility tests ..." - ./gradlew :qa:rolling-upgrade:testRollingUpgrade -Drolling.bwctests.opensearch.version=$OPENSEARCH_VERSION_ROLLING_UPGRADE + ./gradlew :qa:rolling-upgrade:testRollingUpgrade -Dtests.bwc.version=$BWC_VERSION_ROLLING_UPGRADE diff --git a/qa/rolling-upgrade/build.gradle b/qa/rolling-upgrade/build.gradle index eb3ea7899..c884d5692 100644 --- a/qa/rolling-upgrade/build.gradle +++ b/qa/rolling-upgrade/build.gradle @@ -7,15 +7,15 @@ import org.opensearch.gradle.testclusters.StandaloneRestIntegTestTask apply from : "$rootDir/qa/build.gradle" -String knn_bwc_version = System.getProperty("bwc.version") +String default_bwc_version = System.getProperty("bwc.version") +String knn_bwc_version = System.getProperty("tests.bwc.version", default_bwc_version) String baseName = "knnBwcCluster-rolling" -String opensearch_version_upgraded = System.getProperty("rolling.bwctests.opensearch.version", opensearch_version) // Creates a test cluster of previous version and loads k-NN plugin of bwcVersion testClusters { "${baseName}" { testDistribution = "ARCHIVE" - versions = [knn_bwc_version, opensearch_version_upgraded] + versions = [knn_bwc_version, opensearch_version] numberOfNodes = 3 plugin(project.tasks.zipBwcPlugin.archiveFile) setting 'path.repo', "${buildDir}/cluster/shared/repo/${baseName}" From 88914dfb304ba2a58d26e149303d06280cce80c4 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Fri, 29 Jul 2022 16:08:07 -0700 Subject: [PATCH 010/416] Bump OS version to 2.2.0 (#471) (#481) Signed-off-by: Junqiu Lei --- .../workflows/backwards_compatibility_tests_workflow.yml | 8 ++++---- build.gradle | 2 +- qa/restart-upgrade/build.gradle | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/backwards_compatibility_tests_workflow.yml b/.github/workflows/backwards_compatibility_tests_workflow.yml index 165f29369..c76b22886 100644 --- a/.github/workflows/backwards_compatibility_tests_workflow.yml +++ b/.github/workflows/backwards_compatibility_tests_workflow.yml @@ -14,8 +14,8 @@ jobs: strategy: matrix: java: [ 11, 17 ] - bwc_version : [ "1.0.0", "1.1.0", "1.2.4", "1.3.2", "2.0.0" ] - opensearch_version : [ "2.1.0-SNAPSHOT" ] + bwc_version : [ "1.0.0", "1.1.0", "1.2.4", "1.3.2", "2.0.0", "2.1.0" ] + opensearch_version : [ "2.2.0-SNAPSHOT" ] name: k-NN Restart-Upgrade BWC Tests runs-on: ubuntu-latest @@ -46,8 +46,8 @@ jobs: strategy: matrix: java: [ 11, 17 ] - bwc_version: [ "1.3.2", "2.0.0" ] - opensearch_version: [ "2.1.0-SNAPSHOT" ] + bwc_version: [ "1.3.2", "2.0.0", "2.1.0" ] + opensearch_version: [ "2.2.0-SNAPSHOT" ] name: k-NN Rolling-Upgrade BWC Tests runs-on: ubuntu-latest diff --git a/build.gradle b/build.gradle index e6d82a7d0..8092e7a00 100644 --- a/build.gradle +++ b/build.gradle @@ -9,7 +9,7 @@ buildscript { ext { // build.version_qualifier parameter applies to knn plugin artifacts only. OpenSearch version must be set // explicitly as 'opensearch.version' property, for instance opensearch.version=2.0.0-rc1-SNAPSHOT - opensearch_version = System.getProperty("opensearch.version", "2.1.0-SNAPSHOT") + opensearch_version = System.getProperty("opensearch.version", "2.2.0-SNAPSHOT") version_qualifier = System.getProperty("build.version_qualifier", "") opensearch_group = "org.opensearch" } diff --git a/qa/restart-upgrade/build.gradle b/qa/restart-upgrade/build.gradle index 3dbea0f51..ea26da684 100644 --- a/qa/restart-upgrade/build.gradle +++ b/qa/restart-upgrade/build.gradle @@ -51,7 +51,7 @@ testClusters { } // Skip test if version is 1.0, 1.1, 1.2 or 1.3 as they are not supported in those versions - if (knn_bwc_version.startsWith("1.") || knn_bwc_version.startsWith("2.0")) { + if (knn_bwc_version.startsWith("1.") || knn_bwc_version.startsWith("2.")) { filter { excludeTestsMatching "org.opensearch.knn.bwc.IndexingIT.testEmptyParametersOnUpgrade" } @@ -94,7 +94,7 @@ testClusters { } // Skip test if version is 1.0, 1.1, 1.2 or 1.3 as they are not supported in those versions - if (knn_bwc_version.startsWith("1.") || knn_bwc_version.startsWith("2.0")) { + if (knn_bwc_version.startsWith("1.") || knn_bwc_version.startsWith("2.")) { filter { excludeTestsMatching "org.opensearch.knn.bwc.IndexingIT.testEmptyParametersOnUpgrade" } From b217105d1b0d083c20dc913fae237de143580cba Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Fri, 29 Jul 2022 17:14:38 -0700 Subject: [PATCH 011/416] Bump Gradle version to 7.5 (#472) (#482) Signed-off-by: Junqiu Lei (cherry picked from commit c40196bc7958c37fa4a20670d60083e3c99c03c4) Co-authored-by: Junqiu Lei --- gradle/wrapper/gradle-wrapper.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 97060749f..a89d7a90b 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -5,8 +5,8 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-7.4.2-all.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-7.5-all.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists -distributionSha256Sum=e6d864e3b5bc05cc62041842b306383fc1fefcec359e70cebb1d470a6094ca82 +distributionSha256Sum=97a52d145762adc241bad7fd18289bf7f6801e08ece6badf80402fe2b9f250b1 From 9ffcca3ecde6791bd700e0c3def3cf56ed388934 Mon Sep 17 00:00:00 2001 From: John Mazanec Date: Fri, 15 Jul 2022 15:58:32 -0700 Subject: [PATCH 012/416] Add Lucene as a KNNEngine (#452) Adds lucene as another k-NN engine that can be used. Adds one supported method to the engine, "hnsw", with parameters "ef_construction" and "m". In addition, adds some layers of abstraction to reduce code depuplication. Signed-off-by: John Mazanec (cherry picked from commit 7a76d11cd049ebe546989c78fc86e57da8d208f7) --- .../opensearch/knn/common/KNNConstants.java | 3 + .../opensearch/knn/index/KNNIndexShard.java | 2 +- .../codec/KNN80Codec/KNN80CompoundFormat.java | 2 +- .../knn/index/util/AbstractKNNLibrary.java | 58 +++++++ .../opensearch/knn/index/util/JVMLibrary.java | 32 ++++ .../opensearch/knn/index/util/KNNEngine.java | 22 ++- .../org/opensearch/knn/index/util/Lucene.java | 86 +++++++++++ .../knn/index/util/NativeLibrary.java | 56 +------ .../index/util/AbstractKNNLibraryTests.java | 145 ++++++++++++++++++ .../knn/index/util/LuceneTests.java | 110 +++++++++++++ .../knn/index/util/NativeLibraryTests.java | 100 ------------ 11 files changed, 462 insertions(+), 154 deletions(-) create mode 100644 src/main/java/org/opensearch/knn/index/util/AbstractKNNLibrary.java create mode 100644 src/main/java/org/opensearch/knn/index/util/JVMLibrary.java create mode 100644 src/main/java/org/opensearch/knn/index/util/Lucene.java create mode 100644 src/test/java/org/opensearch/knn/index/util/AbstractKNNLibraryTests.java create mode 100644 src/test/java/org/opensearch/knn/index/util/LuceneTests.java diff --git a/src/main/java/org/opensearch/knn/common/KNNConstants.java b/src/main/java/org/opensearch/knn/common/KNNConstants.java index 46d429fcc..87d7a1c21 100644 --- a/src/main/java/org/opensearch/knn/common/KNNConstants.java +++ b/src/main/java/org/opensearch/knn/common/KNNConstants.java @@ -50,6 +50,9 @@ public class KNNConstants { public static final String MAX_VECTOR_COUNT_PARAMETER = "max_training_vector_count"; public static final String SEARCH_SIZE_PARAMETER = "search_size"; + // Lucene specific constants + public static final String LUCENE_NAME = "lucene"; + // nmslib specific constants public static final String NMSLIB_NAME = "nmslib"; public static final String SPACE_TYPE = "spaceType"; // used as field info key diff --git a/src/main/java/org/opensearch/knn/index/KNNIndexShard.java b/src/main/java/org/opensearch/knn/index/KNNIndexShard.java index 7293708be..a12b0df0f 100644 --- a/src/main/java/org/opensearch/knn/index/KNNIndexShard.java +++ b/src/main/java/org/opensearch/knn/index/KNNIndexShard.java @@ -109,7 +109,7 @@ public void warmup() throws IOException { */ public Map getAllEnginePaths(IndexReader indexReader) throws IOException { Map engineFiles = new HashMap<>(); - for (KNNEngine knnEngine : KNNEngine.values()) { + for (KNNEngine knnEngine : KNNEngine.getEnginesThatCreateCustomSegmentFiles()) { engineFiles.putAll(getEnginePaths(indexReader, knnEngine)); } return engineFiles; diff --git a/src/main/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80CompoundFormat.java b/src/main/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80CompoundFormat.java index d001cd1ba..cbe4991f0 100644 --- a/src/main/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80CompoundFormat.java +++ b/src/main/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80CompoundFormat.java @@ -46,7 +46,7 @@ public CompoundDirectory getCompoundReader(Directory dir, SegmentInfo si, IOCont @Override public void write(Directory dir, SegmentInfo si, IOContext context) throws IOException { - for (KNNEngine knnEngine : KNNEngine.values()) { + for (KNNEngine knnEngine : KNNEngine.getEnginesThatCreateCustomSegmentFiles()) { writeEngineFiles(dir, si, context, knnEngine.getExtension()); } delegate.write(dir, si, context); diff --git a/src/main/java/org/opensearch/knn/index/util/AbstractKNNLibrary.java b/src/main/java/org/opensearch/knn/index/util/AbstractKNNLibrary.java new file mode 100644 index 000000000..c5a3d1d1e --- /dev/null +++ b/src/main/java/org/opensearch/knn/index/util/AbstractKNNLibrary.java @@ -0,0 +1,58 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.util; + +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.Getter; +import org.opensearch.common.ValidationException; +import org.opensearch.knn.index.KNNMethod; +import org.opensearch.knn.index.KNNMethodContext; + +import java.util.Map; + +/** + * AbstractKNNLibrary implements common functionality shared between libraries + */ +@AllArgsConstructor(access = AccessLevel.PACKAGE) +public abstract class AbstractKNNLibrary implements KNNLibrary { + + protected final Map methods; + @Getter + protected final String version; + + @Override + public KNNMethod getMethod(String methodName) { + KNNMethod method = methods.get(methodName); + if (method == null) { + throw new IllegalArgumentException(String.format("Invalid method name: %s", methodName)); + } + return method; + } + + @Override + public ValidationException validateMethod(KNNMethodContext knnMethodContext) { + String methodName = knnMethodContext.getMethodComponent().getName(); + return getMethod(methodName).validate(knnMethodContext); + } + + @Override + public boolean isTrainingRequired(KNNMethodContext knnMethodContext) { + String methodName = knnMethodContext.getMethodComponent().getName(); + return getMethod(methodName).isTrainingRequired(knnMethodContext); + } + + @Override + public Map getMethodAsMap(KNNMethodContext knnMethodContext) { + KNNMethod knnMethod = methods.get(knnMethodContext.getMethodComponent().getName()); + + if (knnMethod == null) { + throw new IllegalArgumentException(String.format("Invalid method name: %s", knnMethodContext.getMethodComponent().getName())); + } + + return knnMethod.getAsMap(knnMethodContext); + } +} diff --git a/src/main/java/org/opensearch/knn/index/util/JVMLibrary.java b/src/main/java/org/opensearch/knn/index/util/JVMLibrary.java new file mode 100644 index 000000000..f6d4dc283 --- /dev/null +++ b/src/main/java/org/opensearch/knn/index/util/JVMLibrary.java @@ -0,0 +1,32 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.util; + +import org.opensearch.knn.index.KNNMethod; +import org.opensearch.knn.index.KNNMethodContext; + +import java.util.Map; + +/** + * Abstract class for JVM based KNN libraries + */ +public abstract class JVMLibrary extends AbstractKNNLibrary { + + /** + * Constructor + * + * @param methods Map of k-NN methods that the library supports + * @param version String representing version of library + */ + JVMLibrary(Map methods, String version) { + super(methods, version); + } + + @Override + public int estimateOverheadInKB(KNNMethodContext knnMethodContext, int dimension) { + throw new UnsupportedOperationException("Estimating overhead is not supported for JVM based libraries."); + } +} diff --git a/src/main/java/org/opensearch/knn/index/util/KNNEngine.java b/src/main/java/org/opensearch/knn/index/util/KNNEngine.java index d67c15f44..160b1a9bb 100644 --- a/src/main/java/org/opensearch/knn/index/util/KNNEngine.java +++ b/src/main/java/org/opensearch/knn/index/util/KNNEngine.java @@ -10,9 +10,13 @@ import org.opensearch.knn.index.KNNMethodContext; import org.opensearch.knn.index.SpaceType; +import java.util.Arrays; +import java.util.List; import java.util.Map; +import java.util.stream.Collectors; import static org.opensearch.knn.common.KNNConstants.FAISS_NAME; +import static org.opensearch.knn.common.KNNConstants.LUCENE_NAME; import static org.opensearch.knn.common.KNNConstants.NMSLIB_NAME; /** @@ -21,7 +25,8 @@ */ public enum KNNEngine implements KNNLibrary { NMSLIB(NMSLIB_NAME, Nmslib.INSTANCE), - FAISS(FAISS_NAME, Faiss.INSTANCE); + FAISS(FAISS_NAME, Faiss.INSTANCE), + LUCENE(LUCENE_NAME, Lucene.INSTANCE); public static final KNNEngine DEFAULT = NMSLIB; @@ -50,10 +55,14 @@ public static KNNEngine getEngine(String name) { return NMSLIB; } - if (FAISS.getName().equals(name)) { + if (FAISS.getName().equalsIgnoreCase(name)) { return FAISS; } + if (LUCENE.getName().equalsIgnoreCase(name)) { + return LUCENE; + } + throw new IllegalArgumentException(String.format("Invalid engine type: %s", name)); } @@ -75,6 +84,15 @@ public static KNNEngine getEngineNameFromPath(String path) { throw new IllegalArgumentException("No engine matches the path's suffix"); } + /** + * Returns all engines that create custom segment files. This will be all engines except for Lucene + * + * @return List of all engines that create custom segment files. + */ + public static List getEnginesThatCreateCustomSegmentFiles() { + return Arrays.stream(KNNEngine.values()).filter(knnEngine -> knnEngine != KNNEngine.LUCENE).collect(Collectors.toList()); + } + /** * Get the name of the engine * diff --git a/src/main/java/org/opensearch/knn/index/util/Lucene.java b/src/main/java/org/opensearch/knn/index/util/Lucene.java new file mode 100644 index 000000000..a5f20195a --- /dev/null +++ b/src/main/java/org/opensearch/knn/index/util/Lucene.java @@ -0,0 +1,86 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.util; + +import com.google.common.collect.ImmutableMap; +import org.apache.lucene.util.Version; +import org.opensearch.knn.index.KNNMethod; +import org.opensearch.knn.index.KNNSettings; +import org.opensearch.knn.index.MethodComponent; +import org.opensearch.knn.index.Parameter; +import org.opensearch.knn.index.SpaceType; + +import java.util.Map; + +import static org.opensearch.knn.common.KNNConstants.METHOD_HNSW; +import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_EF_CONSTRUCTION; +import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_M; + +/** + * KNN Library for Lucene + */ +public class Lucene extends JVMLibrary { + + final static Map METHODS = ImmutableMap.of( + METHOD_HNSW, + KNNMethod.Builder.builder( + MethodComponent.Builder.builder(METHOD_HNSW) + .addParameter( + METHOD_PARAMETER_M, + new Parameter.IntegerParameter(METHOD_PARAMETER_M, KNNSettings.INDEX_KNN_DEFAULT_ALGO_PARAM_M, v -> v > 0) + ) + .addParameter( + METHOD_PARAMETER_EF_CONSTRUCTION, + new Parameter.IntegerParameter( + METHOD_PARAMETER_EF_CONSTRUCTION, + KNNSettings.INDEX_KNN_DEFAULT_ALGO_PARAM_EF_CONSTRUCTION, + v -> v > 0 + ) + ) + .build() + ).addSpaces(SpaceType.L2, SpaceType.COSINESIMIL, SpaceType.INNER_PRODUCT).build() + ); + + final static Lucene INSTANCE = new Lucene(METHODS, Version.LUCENE_9_2_0.toString()); + + /** + * Constructor + * + * @param methods Map of k-NN methods that the library supports + * @param version String representing version of library + */ + Lucene(Map methods, String version) { + super(methods, version); + } + + @Override + public String getExtension() { + throw new UnsupportedOperationException("Getting extension for Lucene is not supported"); + } + + @Override + public String getCompoundExtension() { + throw new UnsupportedOperationException("Getting compound extension for Lucene is not supported"); + } + + @Override + public float score(float rawScore, SpaceType spaceType) { + // The score returned by Lucene follows the higher the score, the better the result convention. It will + // actually invert the distance score so that a higher number is a better score. So, we can just return the + // score provided. + return rawScore; + } + + @Override + public Boolean isInitialized() { + return true; + } + + @Override + public void setInitialized(Boolean isInitialized) { + throw new UnsupportedOperationException("Setting Lucene as initialized is not supported"); + } +} diff --git a/src/main/java/org/opensearch/knn/index/util/NativeLibrary.java b/src/main/java/org/opensearch/knn/index/util/NativeLibrary.java index 2cbfcb12d..896c2b889 100644 --- a/src/main/java/org/opensearch/knn/index/util/NativeLibrary.java +++ b/src/main/java/org/opensearch/knn/index/util/NativeLibrary.java @@ -5,7 +5,7 @@ package org.opensearch.knn.index.util; -import org.opensearch.common.ValidationException; +import lombok.Getter; import org.opensearch.knn.common.KNNConstants; import org.opensearch.knn.index.KNNMethod; import org.opensearch.knn.index.KNNMethodContext; @@ -20,10 +20,9 @@ * Abstract implementation of KNNLibrary. It contains several default methods and fields that * are common across different underlying libraries. */ -abstract class NativeLibrary implements KNNLibrary { - protected Map methods; +abstract class NativeLibrary extends AbstractKNNLibrary { private final Map> scoreTranslation; - private final String currentVersion; + @Getter private final String extension; private final AtomicBoolean initialized; @@ -32,46 +31,26 @@ abstract class NativeLibrary implements KNNLibrary { * * @param methods map of methods the native library supports * @param scoreTranslation Map of translation of space type to scores returned by the library - * @param currentVersion String representation of current version of the library + * @param version String representation of version of the library * @param extension String representing the extension that library files should use */ NativeLibrary( Map methods, Map> scoreTranslation, - String currentVersion, + String version, String extension ) { - this.methods = methods; + super(methods, version); this.scoreTranslation = scoreTranslation; - this.currentVersion = currentVersion; this.extension = extension; this.initialized = new AtomicBoolean(false); } - @Override - public String getVersion() { - return this.currentVersion; - } - - @Override - public String getExtension() { - return this.extension; - } - @Override public String getCompoundExtension() { return getExtension() + KNNConstants.COMPOUND_EXTENSION; } - @Override - public KNNMethod getMethod(String methodName) { - KNNMethod method = methods.get(methodName); - if (method != null) { - return method; - } - throw new IllegalArgumentException(String.format("Invalid method name: %s", methodName)); - } - @Override public float score(float rawScore, SpaceType spaceType) { if (this.scoreTranslation.containsKey(spaceType)) { @@ -81,35 +60,12 @@ public float score(float rawScore, SpaceType spaceType) { return spaceType.scoreTranslation(rawScore); } - @Override - public ValidationException validateMethod(KNNMethodContext knnMethodContext) { - String methodName = knnMethodContext.getMethodComponent().getName(); - return getMethod(methodName).validate(knnMethodContext); - } - - @Override - public boolean isTrainingRequired(KNNMethodContext knnMethodContext) { - String methodName = knnMethodContext.getMethodComponent().getName(); - return getMethod(methodName).isTrainingRequired(knnMethodContext); - } - @Override public int estimateOverheadInKB(KNNMethodContext knnMethodContext, int dimension) { String methodName = knnMethodContext.getMethodComponent().getName(); return getMethod(methodName).estimateOverheadInKB(knnMethodContext, dimension); } - @Override - public Map getMethodAsMap(KNNMethodContext knnMethodContext) { - KNNMethod knnMethod = methods.get(knnMethodContext.getMethodComponent().getName()); - - if (knnMethod == null) { - throw new IllegalArgumentException(String.format("Invalid method name: %s", knnMethodContext.getMethodComponent().getName())); - } - - return knnMethod.getAsMap(knnMethodContext); - } - @Override public Boolean isInitialized() { return initialized.get(); diff --git a/src/test/java/org/opensearch/knn/index/util/AbstractKNNLibraryTests.java b/src/test/java/org/opensearch/knn/index/util/AbstractKNNLibraryTests.java new file mode 100644 index 000000000..66b6eee9f --- /dev/null +++ b/src/test/java/org/opensearch/knn/index/util/AbstractKNNLibraryTests.java @@ -0,0 +1,145 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.util; + +import com.google.common.collect.ImmutableMap; +import org.opensearch.common.ValidationException; +import org.opensearch.common.xcontent.XContentBuilder; +import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.knn.KNNTestCase; +import org.opensearch.knn.common.KNNConstants; +import org.opensearch.knn.index.KNNMethod; +import org.opensearch.knn.index.KNNMethodContext; +import org.opensearch.knn.index.MethodComponent; +import org.opensearch.knn.index.MethodComponentContext; +import org.opensearch.knn.index.SpaceType; + +import java.io.IOException; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +import static org.opensearch.knn.common.KNNConstants.NAME; + +public class AbstractKNNLibraryTests extends KNNTestCase { + + public void testGetVersion() { + String testVersion = "test-version"; + TestAbstractKNNLibrary testAbstractKNNLibrary = new TestAbstractKNNLibrary(Collections.emptyMap(), testVersion); + assertEquals(testVersion, testAbstractKNNLibrary.getVersion()); + } + + public void testGetMethod() { + String methodName1 = "test-method-1"; + KNNMethod knnMethod1 = KNNMethod.Builder.builder(MethodComponent.Builder.builder(methodName1).build()).build(); + + String methodName2 = "test-method-2"; + KNNMethod knnMethod2 = KNNMethod.Builder.builder(MethodComponent.Builder.builder(methodName2).build()).build(); + + Map knnMethodMap = ImmutableMap.of(methodName1, knnMethod1, methodName2, knnMethod2); + + TestAbstractKNNLibrary testAbstractKNNLibrary = new TestAbstractKNNLibrary(knnMethodMap, ""); + assertEquals(knnMethod1, testAbstractKNNLibrary.getMethod(methodName1)); + assertEquals(knnMethod2, testAbstractKNNLibrary.getMethod(methodName2)); + expectThrows(IllegalArgumentException.class, () -> testAbstractKNNLibrary.getMethod("invalid")); + } + + public void testValidateMethod() throws IOException { + // Invalid - method not supported + String methodName1 = "test-method-1"; + KNNMethod knnMethod1 = KNNMethod.Builder.builder(MethodComponent.Builder.builder(methodName1).build()).build(); + + Map methodMap = ImmutableMap.of(methodName1, knnMethod1); + TestAbstractKNNLibrary testAbstractKNNLibrary1 = new TestAbstractKNNLibrary(methodMap, ""); + + XContentBuilder xContentBuilder = XContentFactory.jsonBuilder().startObject().field(NAME, "invalid").endObject(); + Map in = xContentBuilderToMap(xContentBuilder); + KNNMethodContext knnMethodContext1 = KNNMethodContext.parse(in); + expectThrows(IllegalArgumentException.class, () -> testAbstractKNNLibrary1.validateMethod(knnMethodContext1)); + + // Invalid - method validation + String methodName2 = "test-method-2"; + KNNMethod knnMethod2 = new KNNMethod(MethodComponent.Builder.builder(methodName2).build(), Collections.emptySet()) { + @Override + public ValidationException validate(KNNMethodContext knnMethodContext) { + return new ValidationException(); + } + }; + + methodMap = ImmutableMap.of(methodName2, knnMethod2); + TestAbstractKNNLibrary testAbstractKNNLibrary2 = new TestAbstractKNNLibrary(methodMap, ""); + xContentBuilder = XContentFactory.jsonBuilder().startObject().field(NAME, methodName2).endObject(); + in = xContentBuilderToMap(xContentBuilder); + KNNMethodContext knnMethodContext2 = KNNMethodContext.parse(in); + assertNotNull(testAbstractKNNLibrary2.validateMethod(knnMethodContext2)); + } + + public void testGetMethodAsMap() { + String methodName = "test-method-1"; + SpaceType spaceType = SpaceType.DEFAULT; + Map generatedMap = ImmutableMap.of("test-key", "test-param"); + MethodComponent methodComponent = MethodComponent.Builder.builder(methodName) + .setMapGenerator(((methodComponent1, methodComponentContext) -> generatedMap)) + .build(); + KNNMethod knnMethod = KNNMethod.Builder.builder(methodComponent).build(); + + TestAbstractKNNLibrary testAbstractKNNLibrary = new TestAbstractKNNLibrary(ImmutableMap.of(methodName, knnMethod), ""); + + // Check that map is expected + Map expectedMap = new HashMap<>(generatedMap); + expectedMap.put(KNNConstants.SPACE_TYPE, spaceType.getValue()); + KNNMethodContext knnMethodContext = new KNNMethodContext( + KNNEngine.DEFAULT, + spaceType, + new MethodComponentContext(methodName, Collections.emptyMap()) + ); + assertEquals(expectedMap, testAbstractKNNLibrary.getMethodAsMap(knnMethodContext)); + + // Check when invalid method is passed in + KNNMethodContext invalidKnnMethodContext = new KNNMethodContext( + KNNEngine.DEFAULT, + spaceType, + new MethodComponentContext("invalid", Collections.emptyMap()) + ); + expectThrows(IllegalArgumentException.class, () -> testAbstractKNNLibrary.getMethodAsMap(invalidKnnMethodContext)); + } + + private static class TestAbstractKNNLibrary extends AbstractKNNLibrary { + public TestAbstractKNNLibrary(Map methods, String currentVersion) { + super(methods, currentVersion); + } + + @Override + public String getExtension() { + return null; + } + + @Override + public String getCompoundExtension() { + return null; + } + + @Override + public float score(float rawScore, SpaceType spaceType) { + return 0; + } + + @Override + public int estimateOverheadInKB(KNNMethodContext knnMethodContext, int dimension) { + return 0; + } + + @Override + public Boolean isInitialized() { + return null; + } + + @Override + public void setInitialized(Boolean isInitialized) { + + } + } +} diff --git a/src/test/java/org/opensearch/knn/index/util/LuceneTests.java b/src/test/java/org/opensearch/knn/index/util/LuceneTests.java new file mode 100644 index 000000000..d5fffc1bb --- /dev/null +++ b/src/test/java/org/opensearch/knn/index/util/LuceneTests.java @@ -0,0 +1,110 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.util; + +import org.opensearch.common.xcontent.XContentBuilder; +import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.knn.KNNTestCase; +import org.opensearch.knn.index.KNNMethod; +import org.opensearch.knn.index.KNNMethodContext; +import org.opensearch.knn.index.SpaceType; + +import java.io.IOException; +import java.util.Collections; +import java.util.Map; + +import static org.opensearch.knn.common.KNNConstants.METHOD_HNSW; +import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_EF_CONSTRUCTION; +import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_M; +import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_SPACE_TYPE; +import static org.opensearch.knn.common.KNNConstants.NAME; +import static org.opensearch.knn.common.KNNConstants.PARAMETERS; + +public class LuceneTests extends KNNTestCase { + + public void testLucenHNSWMethod() throws IOException { + KNNMethod luceneHNSW = Lucene.INSTANCE.getMethod(METHOD_HNSW); + assertNotNull(luceneHNSW); + + int efConstruction = 100; + int m = 17; + XContentBuilder xContentBuilder = XContentFactory.jsonBuilder() + .startObject() + .field(NAME, METHOD_HNSW) + .field(METHOD_PARAMETER_SPACE_TYPE, SpaceType.L2.getValue()) + .startObject(PARAMETERS) + .field(METHOD_PARAMETER_EF_CONSTRUCTION, efConstruction) + .field(METHOD_PARAMETER_M, m) + .endObject() + .endObject(); + Map in = xContentBuilderToMap(xContentBuilder); + KNNMethodContext knnMethodContext1 = KNNMethodContext.parse(in); + assertNull(luceneHNSW.validate(knnMethodContext1)); + + // Invalid parameter + String invalidParameter = "invalid"; + xContentBuilder = XContentFactory.jsonBuilder() + .startObject() + .field(NAME, METHOD_HNSW) + .startObject(PARAMETERS) + .field(invalidParameter, 10) + .endObject() + .endObject(); + in = xContentBuilderToMap(xContentBuilder); + KNNMethodContext knnMethodContext2 = KNNMethodContext.parse(in); + assertNotNull(luceneHNSW.validate(knnMethodContext2)); + + // Valid parameter, invalid value + int invalidEfConstruction = -1; + xContentBuilder = XContentFactory.jsonBuilder() + .startObject() + .field(NAME, METHOD_HNSW) + .startObject(PARAMETERS) + .field(METHOD_PARAMETER_EF_CONSTRUCTION, invalidEfConstruction) + .endObject() + .endObject(); + in = xContentBuilderToMap(xContentBuilder); + KNNMethodContext knnMethodContext3 = KNNMethodContext.parse(in); + assertNotNull(luceneHNSW.validate(knnMethodContext3)); + + // Unsupported space type + SpaceType invalidSpaceType = SpaceType.LINF; // Not currently supported + xContentBuilder = XContentFactory.jsonBuilder() + .startObject() + .field(NAME, METHOD_HNSW) + .field(METHOD_PARAMETER_SPACE_TYPE, invalidSpaceType.getValue()) + .endObject(); + in = xContentBuilderToMap(xContentBuilder); + KNNMethodContext knnMethodContext4 = KNNMethodContext.parse(in); + assertNotNull(luceneHNSW.validate(knnMethodContext4)); + } + + public void testGetExtension() { + Lucene luceneLibrary = new Lucene(Collections.emptyMap(), ""); + expectThrows(UnsupportedOperationException.class, luceneLibrary::getExtension); + } + + public void testGetCompundExtension() { + Lucene luceneLibrary = new Lucene(Collections.emptyMap(), ""); + expectThrows(UnsupportedOperationException.class, luceneLibrary::getCompoundExtension); + } + + public void testScore() { + Lucene luceneLibrary = new Lucene(Collections.emptyMap(), ""); + float rawScore = 10.0f; + assertEquals(rawScore, luceneLibrary.score(rawScore, SpaceType.DEFAULT), 0.001); + } + + public void testIsInitialized() { + Lucene luceneLibrary = new Lucene(Collections.emptyMap(), ""); + assertTrue(luceneLibrary.isInitialized()); + } + + public void testSetInitialized() { + Lucene luceneLibrary = new Lucene(Collections.emptyMap(), ""); + expectThrows(UnsupportedOperationException.class, () -> luceneLibrary.setInitialized(true)); + } +} diff --git a/src/test/java/org/opensearch/knn/index/util/NativeLibraryTests.java b/src/test/java/org/opensearch/knn/index/util/NativeLibraryTests.java index bac3534d8..00a628f1e 100644 --- a/src/test/java/org/opensearch/knn/index/util/NativeLibraryTests.java +++ b/src/test/java/org/opensearch/knn/index/util/NativeLibraryTests.java @@ -6,34 +6,15 @@ package org.opensearch.knn.index.util; import com.google.common.collect.ImmutableMap; -import org.opensearch.common.ValidationException; -import org.opensearch.common.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.knn.KNNTestCase; -import org.opensearch.knn.common.KNNConstants; import org.opensearch.knn.index.KNNMethod; -import org.opensearch.knn.index.KNNMethodContext; -import org.opensearch.knn.index.MethodComponent; -import org.opensearch.knn.index.MethodComponentContext; import org.opensearch.knn.index.SpaceType; -import java.io.IOException; import java.util.Collections; -import java.util.HashMap; import java.util.Map; import java.util.function.Function; -import static org.opensearch.knn.common.KNNConstants.NAME; - public class NativeLibraryTests extends KNNTestCase { - /** - * Test native library version getter - */ - public void testGetVersion() { - String latestBuildVersion = "test-build-version"; - TestNativeLibrary testNativeLibrary = new TestNativeLibrary(Collections.emptyMap(), Collections.emptyMap(), latestBuildVersion, ""); - assertEquals(latestBuildVersion, testNativeLibrary.getVersion()); - } /** * Test native library extension getter @@ -53,24 +34,6 @@ public void testGetCompoundExtension() { assertEquals(extension + "c", testNativeLibrary.getCompoundExtension()); } - /** - * Test native library compound extension getter - */ - public void testGetMethod() { - String methodName1 = "test-method-1"; - KNNMethod knnMethod1 = KNNMethod.Builder.builder(MethodComponent.Builder.builder(methodName1).build()).build(); - - String methodName2 = "test-method-2"; - KNNMethod knnMethod2 = KNNMethod.Builder.builder(MethodComponent.Builder.builder(methodName2).build()).build(); - - Map knnMethodMap = ImmutableMap.of(methodName1, knnMethod1, methodName2, knnMethod2); - - TestNativeLibrary testNativeLibrary = new TestNativeLibrary(knnMethodMap, Collections.emptyMap(), "", ""); - assertEquals(knnMethod1, testNativeLibrary.getMethod(methodName1)); - assertEquals(knnMethod2, testNativeLibrary.getMethod(methodName2)); - expectThrows(IllegalArgumentException.class, () -> testNativeLibrary.getMethod("invalid")); - } - /** * Test native library scoring override */ @@ -84,69 +47,6 @@ public void testScore() { assertEquals(SpaceType.L1.scoreTranslation(1f), testNativeLibrary.score(1f, SpaceType.L1), 0.0001); } - /** - * Test native library method validation - */ - public void testValidateMethod() throws IOException { - // Invalid - method not supported - String methodName1 = "test-method-1"; - KNNMethod knnMethod1 = KNNMethod.Builder.builder(MethodComponent.Builder.builder(methodName1).build()).build(); - - Map methodMap = ImmutableMap.of(methodName1, knnMethod1); - TestNativeLibrary testNativeLibrary1 = new TestNativeLibrary(methodMap, Collections.emptyMap(), "", ""); - - XContentBuilder xContentBuilder = XContentFactory.jsonBuilder().startObject().field(NAME, "invalid").endObject(); - Map in = xContentBuilderToMap(xContentBuilder); - KNNMethodContext knnMethodContext1 = KNNMethodContext.parse(in); - expectThrows(IllegalArgumentException.class, () -> testNativeLibrary1.validateMethod(knnMethodContext1)); - - // Invalid - method validation - String methodName2 = "test-method-2"; - KNNMethod knnMethod2 = new KNNMethod(MethodComponent.Builder.builder(methodName2).build(), Collections.emptySet()) { - @Override - public ValidationException validate(KNNMethodContext knnMethodContext) { - return new ValidationException(); - } - }; - - methodMap = ImmutableMap.of(methodName2, knnMethod2); - TestNativeLibrary testNativeLibrary2 = new TestNativeLibrary(methodMap, Collections.emptyMap(), "", ""); - xContentBuilder = XContentFactory.jsonBuilder().startObject().field(NAME, methodName2).endObject(); - in = xContentBuilderToMap(xContentBuilder); - KNNMethodContext knnMethodContext2 = KNNMethodContext.parse(in); - assertNotNull(testNativeLibrary2.validateMethod(knnMethodContext2)); - } - - public void testGetMethodAsMap() { - String methodName = "test-method-1"; - SpaceType spaceType = SpaceType.DEFAULT; - Map generatedMap = ImmutableMap.of("test-key", "test-param"); - MethodComponent methodComponent = MethodComponent.Builder.builder(methodName) - .setMapGenerator(((methodComponent1, methodComponentContext) -> generatedMap)) - .build(); - KNNMethod knnMethod = KNNMethod.Builder.builder(methodComponent).build(); - - TestNativeLibrary testNativeLibrary = new TestNativeLibrary(ImmutableMap.of(methodName, knnMethod), Collections.emptyMap(), "", ""); - - // Check that map is expected - Map expectedMap = new HashMap<>(generatedMap); - expectedMap.put(KNNConstants.SPACE_TYPE, spaceType.getValue()); - KNNMethodContext knnMethodContext = new KNNMethodContext( - KNNEngine.DEFAULT, - spaceType, - new MethodComponentContext(methodName, Collections.emptyMap()) - ); - assertEquals(expectedMap, testNativeLibrary.getMethodAsMap(knnMethodContext)); - - // Check when invalid method is passed in - KNNMethodContext invalidKnnMethodContext = new KNNMethodContext( - KNNEngine.DEFAULT, - spaceType, - new MethodComponentContext("invalid", Collections.emptyMap()) - ); - expectThrows(IllegalArgumentException.class, () -> testNativeLibrary.getMethodAsMap(invalidKnnMethodContext)); - } - static class TestNativeLibrary extends NativeLibrary { /** * Constructor for TestNativeLibrary From a531330d7ead21532b5d81226922ffa166b7e746 Mon Sep 17 00:00:00 2001 From: John Mazanec Date: Tue, 19 Jul 2022 19:16:48 -0700 Subject: [PATCH 013/416] Add support for Lucene KnnVectorQuery in KNNQueryBuilder (#454) Adds support for Lucene's KnnVectorQuery type in KNNQueryBuilder. KNNQueryBuilder logic was enhanced to detect when an engine does not need to use KNNQuery type and build the Lucene KnnVectorQuery instead. Signed-off-by: John Mazanec (cherry picked from commit 67a1bceebb217912c818d2120e326794c505c73f) --- .../org_opensearch_knn_jni_FaissService.h | 2 +- .../org_opensearch_knn_jni_NmslibService.h | 2 +- jni/src/faiss_wrapper.cpp | 4 +- jni/src/jni_util.cpp | 6 +- jni/src/nmslib_wrapper.cpp | 4 +- .../index/memory/NativeMemoryAllocation.java | 3 +- .../knn/index/{ => query}/KNNQuery.java | 6 +- .../index/{ => query}/KNNQueryBuilder.java | 70 +++++++++++-------- .../knn/index/query/KNNQueryFactory.java | 40 +++++++++++ .../knn/index/{ => query}/KNNQueryResult.java | 2 +- .../knn/index/{ => query}/KNNScorer.java | 2 +- .../knn/index/{ => query}/KNNWeight.java | 3 +- .../opensearch/knn/index/util/KNNEngine.java | 15 ++-- .../org/opensearch/knn/jni/FaissService.java | 2 +- .../org/opensearch/knn/jni/JNIService.java | 2 +- .../org/opensearch/knn/jni/NmslibService.java | 2 +- .../org/opensearch/knn/plugin/KNNPlugin.java | 4 +- .../knn/plugin/script/KNNScoringSpace.java | 2 +- .../opensearch/knn/KNNSingleNodeTestCase.java | 2 +- .../org/opensearch/knn/index/FaissIT.java | 1 + .../knn/index/KNNCircuitBreakerIT.java | 1 + .../knn/index/KNNESSettingsTestIT.java | 1 + .../knn/index/KNNMapperSearcherIT.java | 1 + .../org/opensearch/knn/index/NmslibIT.java | 1 + .../opensearch/knn/index/OpenSearchIT.java | 1 + .../knn/index/codec/KNNCodecTestCase.java | 4 +- .../knn/index/codec/KNNCodecTestUtil.java | 2 +- .../memory/NativeMemoryLoadStrategyTests.java | 2 +- .../{ => query}/KNNQueryBuilderTests.java | 7 +- .../knn/index/query/KNNQueryFactoryTests.java | 45 ++++++++++++ .../opensearch/knn/jni/JNIServiceTests.java | 2 +- .../plugin/action/RestKNNStatsHandlerIT.java | 2 +- .../action/RestLegacyKNNStatsHandlerIT.java | 2 +- .../org/opensearch/knn/KNNRestTestCase.java | 2 +- 34 files changed, 179 insertions(+), 68 deletions(-) rename src/main/java/org/opensearch/knn/index/{ => query}/KNNQuery.java (89%) rename src/main/java/org/opensearch/knn/index/{ => query}/KNNQueryBuilder.java (76%) create mode 100644 src/main/java/org/opensearch/knn/index/query/KNNQueryFactory.java rename src/main/java/org/opensearch/knn/index/{ => query}/KNNQueryResult.java (92%) rename src/main/java/org/opensearch/knn/index/{ => query}/KNNScorer.java (97%) rename src/main/java/org/opensearch/knn/index/{ => query}/KNNWeight.java (98%) rename src/test/java/org/opensearch/knn/index/{ => query}/KNNQueryBuilderTests.java (96%) create mode 100644 src/test/java/org/opensearch/knn/index/query/KNNQueryFactoryTests.java diff --git a/jni/include/org_opensearch_knn_jni_FaissService.h b/jni/include/org_opensearch_knn_jni_FaissService.h index 4af9a24bc..1ab6c5681 100644 --- a/jni/include/org_opensearch_knn_jni_FaissService.h +++ b/jni/include/org_opensearch_knn_jni_FaissService.h @@ -45,7 +45,7 @@ JNIEXPORT jlong JNICALL Java_org_opensearch_knn_jni_FaissService_loadIndex /* * Class: org_opensearch_knn_jni_FaissService * Method: queryIndex - * Signature: (J[FI)[Lorg/opensearch/knn/index/KNNQueryResult; + * Signature: (J[FI)[Lorg/opensearch/knn/index/query/KNNQueryResult; */ JNIEXPORT jobjectArray JNICALL Java_org_opensearch_knn_jni_FaissService_queryIndex (JNIEnv *, jclass, jlong, jfloatArray, jint); diff --git a/jni/include/org_opensearch_knn_jni_NmslibService.h b/jni/include/org_opensearch_knn_jni_NmslibService.h index dd907581d..02f58d20f 100644 --- a/jni/include/org_opensearch_knn_jni_NmslibService.h +++ b/jni/include/org_opensearch_knn_jni_NmslibService.h @@ -37,7 +37,7 @@ JNIEXPORT jlong JNICALL Java_org_opensearch_knn_jni_NmslibService_loadIndex /* * Class: org_opensearch_knn_jni_NmslibService * Method: queryIndex - * Signature: (J[FI)[Lorg/opensearch/knn/index/KNNQueryResult; + * Signature: (J[FI)[Lorg/opensearch/knn/index/query/KNNQueryResult; */ JNIEXPORT jobjectArray JNICALL Java_org_opensearch_knn_jni_NmslibService_queryIndex (JNIEnv *, jclass, jlong, jfloatArray, jint); diff --git a/jni/src/faiss_wrapper.cpp b/jni/src/faiss_wrapper.cpp index 2e0a4e7cb..d1342a2ba 100644 --- a/jni/src/faiss_wrapper.cpp +++ b/jni/src/faiss_wrapper.cpp @@ -213,8 +213,8 @@ jobjectArray knn_jni::faiss_wrapper::QueryIndex(knn_jni::JNIUtilInterface * jniU resultSize = it - ids.begin(); } - jclass resultClass = jniUtil->FindClass(env,"org/opensearch/knn/index/KNNQueryResult"); - jmethodID allArgs = jniUtil->FindMethod(env, "org/opensearch/knn/index/KNNQueryResult", ""); + jclass resultClass = jniUtil->FindClass(env,"org/opensearch/knn/index/query/KNNQueryResult"); + jmethodID allArgs = jniUtil->FindMethod(env, "org/opensearch/knn/index/query/KNNQueryResult", ""); jobjectArray results = jniUtil->NewObjectArray(env, resultSize, resultClass, nullptr); diff --git a/jni/src/jni_util.cpp b/jni/src/jni_util.cpp index de0ed7856..cb9270c22 100644 --- a/jni/src/jni_util.cpp +++ b/jni/src/jni_util.cpp @@ -61,9 +61,9 @@ void knn_jni::JNIUtil::Initialize(JNIEnv *env) { this->cachedMethods["java/lang/Integer:intValue"] = env->GetMethodID(tempLocalClassRef, "intValue", "()I"); env->DeleteLocalRef(tempLocalClassRef); - tempLocalClassRef = env->FindClass("org/opensearch/knn/index/KNNQueryResult"); - this->cachedClasses["org/opensearch/knn/index/KNNQueryResult"] = (jclass) env->NewGlobalRef(tempLocalClassRef); - this->cachedMethods["org/opensearch/knn/index/KNNQueryResult:"] = env->GetMethodID(tempLocalClassRef, "", "(IF)V"); + tempLocalClassRef = env->FindClass("org/opensearch/knn/index/query/KNNQueryResult"); + this->cachedClasses["org/opensearch/knn/index/query/KNNQueryResult"] = (jclass) env->NewGlobalRef(tempLocalClassRef); + this->cachedMethods["org/opensearch/knn/index/query/KNNQueryResult:"] = env->GetMethodID(tempLocalClassRef, "", "(IF)V"); env->DeleteLocalRef(tempLocalClassRef); } diff --git a/jni/src/nmslib_wrapper.cpp b/jni/src/nmslib_wrapper.cpp index 083850e88..88b29e0f3 100644 --- a/jni/src/nmslib_wrapper.cpp +++ b/jni/src/nmslib_wrapper.cpp @@ -209,8 +209,8 @@ jobjectArray knn_jni::nmslib_wrapper::QueryIndex(knn_jni::JNIUtilInterface * jni std::unique_ptr> neighbors(knnQuery.Result()->Clone()); int resultSize = neighbors->Size(); - jclass resultClass = jniUtil->FindClass(env,"org/opensearch/knn/index/KNNQueryResult"); - jmethodID allArgs = jniUtil->FindMethod(env, "org/opensearch/knn/index/KNNQueryResult", ""); + jclass resultClass = jniUtil->FindClass(env,"org/opensearch/knn/index/query/KNNQueryResult"); + jmethodID allArgs = jniUtil->FindMethod(env, "org/opensearch/knn/index/query/KNNQueryResult", ""); jobjectArray results = jniUtil->NewObjectArray(env, resultSize, resultClass, nullptr); diff --git a/src/main/java/org/opensearch/knn/index/memory/NativeMemoryAllocation.java b/src/main/java/org/opensearch/knn/index/memory/NativeMemoryAllocation.java index 9279c816f..53852f129 100644 --- a/src/main/java/org/opensearch/knn/index/memory/NativeMemoryAllocation.java +++ b/src/main/java/org/opensearch/knn/index/memory/NativeMemoryAllocation.java @@ -12,6 +12,7 @@ package org.opensearch.knn.index.memory; import org.apache.lucene.index.LeafReaderContext; +import org.opensearch.knn.index.query.KNNWeight; import org.opensearch.knn.jni.JNIService; import org.opensearch.knn.index.util.KNNEngine; import org.opensearch.watcher.FileWatcher; @@ -158,7 +159,7 @@ public long getMemoryAddress() { /** * The read lock will be obtained in the - * {@link org.opensearch.knn.index.KNNWeight#scorer(LeafReaderContext context) scorer} when a native index needs + * {@link KNNWeight#scorer(LeafReaderContext context) scorer} when a native index needs * to be queried. */ @Override diff --git a/src/main/java/org/opensearch/knn/index/KNNQuery.java b/src/main/java/org/opensearch/knn/index/query/KNNQuery.java similarity index 89% rename from src/main/java/org/opensearch/knn/index/KNNQuery.java rename to src/main/java/org/opensearch/knn/index/query/KNNQuery.java index 631b36d7a..9bf38008b 100644 --- a/src/main/java/org/opensearch/knn/index/KNNQuery.java +++ b/src/main/java/org/opensearch/knn/index/query/KNNQuery.java @@ -3,18 +3,20 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.knn.index; +package org.opensearch.knn.index.query; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; import org.apache.lucene.search.QueryVisitor; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Weight; +import org.opensearch.knn.index.KNNSettings; import java.io.IOException; /** - * Class for representing the KNN query + * Custom KNN query. Query is used for KNNEngine's that create their own custom segment files. These files need to be + * loaded and queried in a custom manner throughout the query path. */ public class KNNQuery extends Query { diff --git a/src/main/java/org/opensearch/knn/index/KNNQueryBuilder.java b/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java similarity index 76% rename from src/main/java/org/opensearch/knn/index/KNNQueryBuilder.java rename to src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java index 862d680ce..1defe45e8 100644 --- a/src/main/java/org/opensearch/knn/index/KNNQueryBuilder.java +++ b/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java @@ -3,15 +3,16 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.knn.index; +package org.opensearch.knn.index.query; +import lombok.extern.log4j.Log4j2; import org.opensearch.index.mapper.NumberFieldMapper; +import org.opensearch.knn.index.KNNMethodContext; import org.opensearch.knn.index.mapper.KNNVectorFieldMapper; +import org.opensearch.knn.index.util.KNNEngine; import org.opensearch.knn.indices.ModelDao; import org.opensearch.knn.indices.ModelMetadata; import org.opensearch.knn.plugin.stats.KNNCounter; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; import org.apache.lucene.search.Query; import org.opensearch.common.ParseField; import org.opensearch.common.ParsingException; @@ -31,8 +32,8 @@ /** * Helper class to build the KNN query */ +@Log4j2 public class KNNQueryBuilder extends AbstractQueryBuilder { - private static Logger logger = LogManager.getLogger(KNNQueryBuilder.class); private static ModelDao modelDao; public static final ParseField VECTOR_FIELD = new ParseField("vector"); @@ -152,10 +153,10 @@ public static KNNQueryBuilder fromXContent(XContentParser parser) throws IOExcep } } - KNNQueryBuilder knnQuery = new KNNQueryBuilder(fieldName, ObjectsToFloats(vector), k); - knnQuery.queryName(queryName); - knnQuery.boost(boost); - return knnQuery; + KNNQueryBuilder knnQueryBuilder = new KNNQueryBuilder(fieldName, ObjectsToFloats(vector), k); + knnQueryBuilder.queryName(queryName); + knnQueryBuilder.boost(boost); + return knnQueryBuilder; } @Override @@ -196,39 +197,50 @@ public void doXContent(XContentBuilder builder, Params params) throws IOExceptio } @Override - protected Query doToQuery(QueryShardContext context) throws IOException { - + protected Query doToQuery(QueryShardContext context) { MappedFieldType mappedFieldType = context.fieldMapper(this.fieldName); if (!(mappedFieldType instanceof KNNVectorFieldMapper.KNNVectorFieldType)) { - throw new IllegalArgumentException("Field '" + this.fieldName + "' is not knn_vector type."); + throw new IllegalArgumentException(String.format("Field '%s' is not knn_vector type.", this.fieldName)); } - int dimension = ((KNNVectorFieldMapper.KNNVectorFieldType) mappedFieldType).getDimension(); - - // If the dimension is not set, then the only valid route forward is if the field uses a model - if (dimension == -1) { - String modelId = ((KNNVectorFieldMapper.KNNVectorFieldType) mappedFieldType).getModelId(); - - if (modelId == null) { - throw new IllegalArgumentException("Field '" + this.fieldName + "' does not have dimension set."); - } - - ModelMetadata modelMetadata = modelDao.getMetadata(modelId); + KNNVectorFieldMapper.KNNVectorFieldType knnVectorFieldType = (KNNVectorFieldMapper.KNNVectorFieldType) mappedFieldType; + int fieldDimension = knnVectorFieldType.getDimension(); + KNNMethodContext knnMethodContext = knnVectorFieldType.getKnnMethodContext(); + KNNEngine knnEngine = KNNEngine.DEFAULT; - if (modelMetadata == null) { - throw new IllegalArgumentException("Model ID \"" + modelId + "\" does not exist."); - } - dimension = modelMetadata.getDimension(); + if (fieldDimension == -1) { + // If dimension is not set, the field uses a model and the information needs to be retrieved from there + ModelMetadata modelMetadata = getModelMetadataForField(knnVectorFieldType); + fieldDimension = modelMetadata.getDimension(); + knnEngine = modelMetadata.getKnnEngine(); + } else if (knnMethodContext != null) { + // If the dimension is set but the knnMethodContext is not then the field is using the legacy mapping + knnEngine = knnMethodContext.getKnnEngine(); } - if (dimension != vector.length) { + if (fieldDimension != vector.length) { throw new IllegalArgumentException( - "Query vector has invalid dimension: " + vector.length + ". Dimension should be: " + dimension + String.format("Query vector has invalid dimension: %d. Dimension should be: %d", vector.length, fieldDimension) ); } - return new KNNQuery(this.fieldName, vector, k, context.index().getName()); + String indexName = context.index().getName(); + return KNNQueryFactory.create(knnEngine, indexName, this.fieldName, this.vector, this.k); + } + + private ModelMetadata getModelMetadataForField(KNNVectorFieldMapper.KNNVectorFieldType knnVectorField) { + String modelId = knnVectorField.getModelId(); + + if (modelId == null) { + throw new IllegalArgumentException(String.format("Field '%s' does not have model.", this.fieldName)); + } + + ModelMetadata modelMetadata = modelDao.getMetadata(modelId); + if (modelMetadata == null) { + throw new IllegalArgumentException(String.format("Model ID '%s' does not exist.", modelId)); + } + return modelMetadata; } @Override diff --git a/src/main/java/org/opensearch/knn/index/query/KNNQueryFactory.java b/src/main/java/org/opensearch/knn/index/query/KNNQueryFactory.java new file mode 100644 index 000000000..cbdb03ea8 --- /dev/null +++ b/src/main/java/org/opensearch/knn/index/query/KNNQueryFactory.java @@ -0,0 +1,40 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.query; + +import lombok.extern.log4j.Log4j2; +import org.apache.lucene.search.KnnVectorQuery; +import org.apache.lucene.search.Query; +import org.opensearch.knn.index.util.KNNEngine; + +/** + * Creates the Lucene k-NN queries + */ +@Log4j2 +public class KNNQueryFactory { + + /** + * Creates a Lucene query for a particular engine. + * + * @param knnEngine Engine to create the query for + * @param indexName Name of the OpenSearch index that is being queried + * @param fieldName Name of the field in the OpenSearch index that will be queried + * @param vector The query vector to get the nearest neighbors for + * @param k the number of nearest neighbors to return + * @return Lucene Query + */ + public static Query create(KNNEngine knnEngine, String indexName, String fieldName, float[] vector, int k) { + // Engines that create their own custom segment files cannot use the Lucene's KnnVectorQuery. They need to + // use the custom query type created by the plugin + if (KNNEngine.getEnginesThatCreateCustomSegmentFiles().contains(knnEngine)) { + log.debug(String.format("Creating custom k-NN query for index: %s \"\", field: %s \"\", k: %d", indexName, fieldName, k)); + return new KNNQuery(fieldName, vector, k, indexName); + } + + log.debug(String.format("Creating Lucene k-NN query for index: %s \"\", field: %s \"\", k: %d", indexName, fieldName, k)); + return new KnnVectorQuery(fieldName, vector, k); + } +} diff --git a/src/main/java/org/opensearch/knn/index/KNNQueryResult.java b/src/main/java/org/opensearch/knn/index/query/KNNQueryResult.java similarity index 92% rename from src/main/java/org/opensearch/knn/index/KNNQueryResult.java rename to src/main/java/org/opensearch/knn/index/query/KNNQueryResult.java index a62ae3e4b..3ff74b2c6 100644 --- a/src/main/java/org/opensearch/knn/index/KNNQueryResult.java +++ b/src/main/java/org/opensearch/knn/index/query/KNNQueryResult.java @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.knn.index; +package org.opensearch.knn.index.query; /** * Place holder for the score of the document diff --git a/src/main/java/org/opensearch/knn/index/KNNScorer.java b/src/main/java/org/opensearch/knn/index/query/KNNScorer.java similarity index 97% rename from src/main/java/org/opensearch/knn/index/KNNScorer.java rename to src/main/java/org/opensearch/knn/index/query/KNNScorer.java index edef5fdd4..0005212bf 100644 --- a/src/main/java/org/opensearch/knn/index/KNNScorer.java +++ b/src/main/java/org/opensearch/knn/index/query/KNNScorer.java @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.knn.index; +package org.opensearch.knn.index.query; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.Scorer; diff --git a/src/main/java/org/opensearch/knn/index/KNNWeight.java b/src/main/java/org/opensearch/knn/index/query/KNNWeight.java similarity index 98% rename from src/main/java/org/opensearch/knn/index/KNNWeight.java rename to src/main/java/org/opensearch/knn/index/query/KNNWeight.java index 7defb1eeb..c7f2ddf47 100644 --- a/src/main/java/org/opensearch/knn/index/KNNWeight.java +++ b/src/main/java/org/opensearch/knn/index/query/KNNWeight.java @@ -3,9 +3,10 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.knn.index; +package org.opensearch.knn.index.query; import org.opensearch.knn.common.KNNConstants; +import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.jni.JNIService; import org.opensearch.knn.index.memory.NativeMemoryAllocation; import org.opensearch.knn.index.memory.NativeMemoryCacheManager; diff --git a/src/main/java/org/opensearch/knn/index/util/KNNEngine.java b/src/main/java/org/opensearch/knn/index/util/KNNEngine.java index 160b1a9bb..975da0b34 100644 --- a/src/main/java/org/opensearch/knn/index/util/KNNEngine.java +++ b/src/main/java/org/opensearch/knn/index/util/KNNEngine.java @@ -5,15 +5,14 @@ package org.opensearch.knn.index.util; +import com.google.common.collect.ImmutableSet; import org.opensearch.common.ValidationException; import org.opensearch.knn.index.KNNMethod; import org.opensearch.knn.index.KNNMethodContext; import org.opensearch.knn.index.SpaceType; -import java.util.Arrays; -import java.util.List; import java.util.Map; -import java.util.stream.Collectors; +import java.util.Set; import static org.opensearch.knn.common.KNNConstants.FAISS_NAME; import static org.opensearch.knn.common.KNNConstants.LUCENE_NAME; @@ -30,6 +29,8 @@ public enum KNNEngine implements KNNLibrary { public static final KNNEngine DEFAULT = NMSLIB; + private static final Set CUSTOM_SEGMENT_FILE_ENGINES = ImmutableSet.of(KNNEngine.NMSLIB, KNNEngine.FAISS); + /** * Constructor for KNNEngine * @@ -85,12 +86,12 @@ public static KNNEngine getEngineNameFromPath(String path) { } /** - * Returns all engines that create custom segment files. This will be all engines except for Lucene + * Returns all engines that create custom segment files. * - * @return List of all engines that create custom segment files. + * @return Set of all engines that create custom segment files. */ - public static List getEnginesThatCreateCustomSegmentFiles() { - return Arrays.stream(KNNEngine.values()).filter(knnEngine -> knnEngine != KNNEngine.LUCENE).collect(Collectors.toList()); + public static Set getEnginesThatCreateCustomSegmentFiles() { + return CUSTOM_SEGMENT_FILE_ENGINES; } /** diff --git a/src/main/java/org/opensearch/knn/jni/FaissService.java b/src/main/java/org/opensearch/knn/jni/FaissService.java index 3f90e33d1..f1d869bd2 100644 --- a/src/main/java/org/opensearch/knn/jni/FaissService.java +++ b/src/main/java/org/opensearch/knn/jni/FaissService.java @@ -12,7 +12,7 @@ package org.opensearch.knn.jni; import org.opensearch.knn.common.KNNConstants; -import org.opensearch.knn.index.KNNQueryResult; +import org.opensearch.knn.index.query.KNNQueryResult; import org.opensearch.knn.index.util.KNNEngine; import java.security.AccessController; diff --git a/src/main/java/org/opensearch/knn/jni/JNIService.java b/src/main/java/org/opensearch/knn/jni/JNIService.java index 7afc312ac..e32880fff 100644 --- a/src/main/java/org/opensearch/knn/jni/JNIService.java +++ b/src/main/java/org/opensearch/knn/jni/JNIService.java @@ -11,7 +11,7 @@ package org.opensearch.knn.jni; -import org.opensearch.knn.index.KNNQueryResult; +import org.opensearch.knn.index.query.KNNQueryResult; import org.opensearch.knn.index.util.KNNEngine; import java.util.Map; diff --git a/src/main/java/org/opensearch/knn/jni/NmslibService.java b/src/main/java/org/opensearch/knn/jni/NmslibService.java index b02cdca0f..77896822a 100644 --- a/src/main/java/org/opensearch/knn/jni/NmslibService.java +++ b/src/main/java/org/opensearch/knn/jni/NmslibService.java @@ -12,7 +12,7 @@ package org.opensearch.knn.jni; import org.opensearch.knn.common.KNNConstants; -import org.opensearch.knn.index.KNNQueryResult; +import org.opensearch.knn.index.query.KNNQueryResult; import org.opensearch.knn.index.util.KNNEngine; import java.security.AccessController; diff --git a/src/main/java/org/opensearch/knn/plugin/KNNPlugin.java b/src/main/java/org/opensearch/knn/plugin/KNNPlugin.java index fd2ef0780..c2564f179 100644 --- a/src/main/java/org/opensearch/knn/plugin/KNNPlugin.java +++ b/src/main/java/org/opensearch/knn/plugin/KNNPlugin.java @@ -11,11 +11,11 @@ import org.opensearch.index.codec.CodecServiceFactory; import org.opensearch.index.engine.EngineFactory; import org.opensearch.knn.index.KNNCircuitBreaker; -import org.opensearch.knn.index.KNNQueryBuilder; +import org.opensearch.knn.index.query.KNNQueryBuilder; import org.opensearch.knn.index.KNNSettings; import org.opensearch.knn.index.mapper.KNNVectorFieldMapper; -import org.opensearch.knn.index.KNNWeight; +import org.opensearch.knn.index.query.KNNWeight; import org.opensearch.knn.index.codec.KNNCodecService; import org.opensearch.knn.index.memory.NativeMemoryLoadStrategy; import org.opensearch.knn.indices.ModelGraveyard; diff --git a/src/main/java/org/opensearch/knn/plugin/script/KNNScoringSpace.java b/src/main/java/org/opensearch/knn/plugin/script/KNNScoringSpace.java index 4a419fd23..ca7526dcb 100644 --- a/src/main/java/org/opensearch/knn/plugin/script/KNNScoringSpace.java +++ b/src/main/java/org/opensearch/knn/plugin/script/KNNScoringSpace.java @@ -6,7 +6,7 @@ package org.opensearch.knn.plugin.script; import org.opensearch.knn.index.mapper.KNNVectorFieldMapper; -import org.opensearch.knn.index.KNNWeight; +import org.opensearch.knn.index.query.KNNWeight; import org.apache.lucene.index.LeafReaderContext; import org.opensearch.index.mapper.MappedFieldType; import org.opensearch.script.ScoreScript; diff --git a/src/test/java/org/opensearch/knn/KNNSingleNodeTestCase.java b/src/test/java/org/opensearch/knn/KNNSingleNodeTestCase.java index b4edcf304..c83433eca 100644 --- a/src/test/java/org/opensearch/knn/KNNSingleNodeTestCase.java +++ b/src/test/java/org/opensearch/knn/KNNSingleNodeTestCase.java @@ -7,7 +7,7 @@ import org.opensearch.common.bytes.BytesReference; import org.opensearch.common.xcontent.XContentHelper; -import org.opensearch.knn.index.KNNQueryBuilder; +import org.opensearch.knn.index.query.KNNQueryBuilder; import org.opensearch.knn.index.memory.NativeMemoryCacheManager; import org.opensearch.knn.index.memory.NativeMemoryLoadStrategy; import org.opensearch.knn.indices.ModelDao; diff --git a/src/test/java/org/opensearch/knn/index/FaissIT.java b/src/test/java/org/opensearch/knn/index/FaissIT.java index 157eb0ce8..979d1de73 100644 --- a/src/test/java/org/opensearch/knn/index/FaissIT.java +++ b/src/test/java/org/opensearch/knn/index/FaissIT.java @@ -23,6 +23,7 @@ import org.opensearch.knn.KNNResult; import org.opensearch.knn.TestUtils; import org.opensearch.knn.common.KNNConstants; +import org.opensearch.knn.index.query.KNNQueryBuilder; import org.opensearch.knn.index.util.KNNEngine; import org.opensearch.knn.plugin.script.KNNScoringUtil; diff --git a/src/test/java/org/opensearch/knn/index/KNNCircuitBreakerIT.java b/src/test/java/org/opensearch/knn/index/KNNCircuitBreakerIT.java index 86b728ab9..02ebdf2e0 100644 --- a/src/test/java/org/opensearch/knn/index/KNNCircuitBreakerIT.java +++ b/src/test/java/org/opensearch/knn/index/KNNCircuitBreakerIT.java @@ -9,6 +9,7 @@ import org.apache.http.util.EntityUtils; import org.opensearch.client.Response; import org.opensearch.common.settings.Settings; +import org.opensearch.knn.index.query.KNNQueryBuilder; import java.util.Collections; import java.util.Map; diff --git a/src/test/java/org/opensearch/knn/index/KNNESSettingsTestIT.java b/src/test/java/org/opensearch/knn/index/KNNESSettingsTestIT.java index 1319448c8..7f6afba05 100644 --- a/src/test/java/org/opensearch/knn/index/KNNESSettingsTestIT.java +++ b/src/test/java/org/opensearch/knn/index/KNNESSettingsTestIT.java @@ -6,6 +6,7 @@ package org.opensearch.knn.index; import org.opensearch.knn.KNNRestTestCase; +import org.opensearch.knn.index.query.KNNQueryBuilder; import org.opensearch.knn.plugin.stats.StatNames; import org.apache.http.util.EntityUtils; import org.opensearch.client.Response; diff --git a/src/test/java/org/opensearch/knn/index/KNNMapperSearcherIT.java b/src/test/java/org/opensearch/knn/index/KNNMapperSearcherIT.java index b98b081ab..b2477fa5d 100644 --- a/src/test/java/org/opensearch/knn/index/KNNMapperSearcherIT.java +++ b/src/test/java/org/opensearch/knn/index/KNNMapperSearcherIT.java @@ -11,6 +11,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.opensearch.client.Response; +import org.opensearch.knn.index.query.KNNQueryBuilder; import java.util.ArrayList; import java.util.Arrays; diff --git a/src/test/java/org/opensearch/knn/index/NmslibIT.java b/src/test/java/org/opensearch/knn/index/NmslibIT.java index 1e1b1e3b5..ac1e48826 100644 --- a/src/test/java/org/opensearch/knn/index/NmslibIT.java +++ b/src/test/java/org/opensearch/knn/index/NmslibIT.java @@ -25,6 +25,7 @@ import org.opensearch.knn.KNNResult; import org.opensearch.knn.TestUtils; import org.opensearch.knn.common.KNNConstants; +import org.opensearch.knn.index.query.KNNQueryBuilder; import org.opensearch.knn.index.util.KNNEngine; import org.opensearch.knn.plugin.script.KNNScoringUtil; diff --git a/src/test/java/org/opensearch/knn/index/OpenSearchIT.java b/src/test/java/org/opensearch/knn/index/OpenSearchIT.java index 3fd492782..646285cf3 100644 --- a/src/test/java/org/opensearch/knn/index/OpenSearchIT.java +++ b/src/test/java/org/opensearch/knn/index/OpenSearchIT.java @@ -28,6 +28,7 @@ import org.opensearch.knn.TestUtils; import org.opensearch.knn.common.KNNConstants; import org.opensearch.knn.index.mapper.KNNVectorFieldMapper; +import org.opensearch.knn.index.query.KNNQueryBuilder; import org.opensearch.knn.index.util.KNNEngine; import org.opensearch.knn.plugin.script.KNNScoringUtil; import org.opensearch.rest.RestStatus; diff --git a/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestCase.java b/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestCase.java index 042dad331..fce944ee6 100644 --- a/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestCase.java +++ b/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestCase.java @@ -13,10 +13,10 @@ import org.opensearch.knn.common.KNNConstants; import org.opensearch.knn.index.codec.KNN920Codec.KNN920Codec; import org.opensearch.knn.jni.JNIService; -import org.opensearch.knn.index.KNNQuery; +import org.opensearch.knn.index.query.KNNQuery; import org.opensearch.knn.index.KNNSettings; import org.opensearch.knn.index.mapper.KNNVectorFieldMapper; -import org.opensearch.knn.index.KNNWeight; +import org.opensearch.knn.index.query.KNNWeight; import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.VectorField; import org.apache.lucene.codecs.Codec; diff --git a/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestUtil.java b/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestUtil.java index 9b64815ac..d95e1dc3c 100644 --- a/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestUtil.java +++ b/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestUtil.java @@ -32,7 +32,7 @@ import org.apache.lucene.util.StringHelper; import org.apache.lucene.util.Version; import org.opensearch.common.collect.Set; -import org.opensearch.knn.index.KNNQueryResult; +import org.opensearch.knn.index.query.KNNQueryResult; import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.codec.util.KNNVectorSerializer; import org.opensearch.knn.index.codec.util.KNNVectorSerializerFactory; diff --git a/src/test/java/org/opensearch/knn/index/memory/NativeMemoryLoadStrategyTests.java b/src/test/java/org/opensearch/knn/index/memory/NativeMemoryLoadStrategyTests.java index 1b531e29e..8d94b1afb 100644 --- a/src/test/java/org/opensearch/knn/index/memory/NativeMemoryLoadStrategyTests.java +++ b/src/test/java/org/opensearch/knn/index/memory/NativeMemoryLoadStrategyTests.java @@ -17,7 +17,7 @@ import org.opensearch.knn.KNNTestCase; import org.opensearch.knn.common.KNNConstants; import org.opensearch.knn.jni.JNIService; -import org.opensearch.knn.index.KNNQueryResult; +import org.opensearch.knn.index.query.KNNQueryResult; import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.util.KNNEngine; import org.opensearch.knn.training.TrainingDataConsumer; diff --git a/src/test/java/org/opensearch/knn/index/KNNQueryBuilderTests.java b/src/test/java/org/opensearch/knn/index/query/KNNQueryBuilderTests.java similarity index 96% rename from src/test/java/org/opensearch/knn/index/KNNQueryBuilderTests.java rename to src/test/java/org/opensearch/knn/index/query/KNNQueryBuilderTests.java index 8a252ca72..c3d40cbc7 100644 --- a/src/test/java/org/opensearch/knn/index/KNNQueryBuilderTests.java +++ b/src/test/java/org/opensearch/knn/index/query/KNNQueryBuilderTests.java @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.knn.index; +package org.opensearch.knn.index.query; import org.opensearch.knn.KNNTestCase; import org.opensearch.common.xcontent.XContentBuilder; @@ -13,6 +13,7 @@ import org.opensearch.index.mapper.NumberFieldMapper; import org.opensearch.index.query.QueryShardContext; import org.opensearch.knn.index.mapper.KNNVectorFieldMapper; +import org.opensearch.knn.index.util.KNNEngine; import org.opensearch.knn.indices.ModelDao; import org.opensearch.knn.indices.ModelMetadata; @@ -88,7 +89,7 @@ public void testDoToQuery_Normal() throws Exception { assertEquals(knnQueryBuilder.vector(), query.getQueryVector()); } - public void testDoToQuery_FromModel() throws Exception { + public void testDoToQuery_FromModel() { float[] queryVector = { 1.0f, 2.0f, 3.0f, 4.0f }; KNNQueryBuilder knnQueryBuilder = new KNNQueryBuilder("myvector", queryVector, 1); Index dummyIndex = new Index("dummy", "dummy"); @@ -98,12 +99,14 @@ public void testDoToQuery_FromModel() throws Exception { // Dimension is -1. In this case, model metadata will need to provide dimension when(mockKNNVectorField.getDimension()).thenReturn(-1); + when(mockKNNVectorField.getKnnMethodContext()).thenReturn(null); String modelId = "test-model-id"; when(mockKNNVectorField.getModelId()).thenReturn(modelId); // Mock the modelDao to return mocked modelMetadata ModelMetadata modelMetadata = mock(ModelMetadata.class); when(modelMetadata.getDimension()).thenReturn(4); + when(modelMetadata.getKnnEngine()).thenReturn(KNNEngine.FAISS); ModelDao modelDao = mock(ModelDao.class); when(modelDao.getMetadata(modelId)).thenReturn(modelMetadata); KNNQueryBuilder.initialize(modelDao); diff --git a/src/test/java/org/opensearch/knn/index/query/KNNQueryFactoryTests.java b/src/test/java/org/opensearch/knn/index/query/KNNQueryFactoryTests.java new file mode 100644 index 000000000..06b0ce6ca --- /dev/null +++ b/src/test/java/org/opensearch/knn/index/query/KNNQueryFactoryTests.java @@ -0,0 +1,45 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.query; + +import org.apache.lucene.search.KnnVectorQuery; +import org.apache.lucene.search.Query; +import org.opensearch.knn.KNNTestCase; +import org.opensearch.knn.index.util.KNNEngine; + +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; + +public class KNNQueryFactoryTests extends KNNTestCase { + private final int testQueryDimension = 17; + private final float[] testQueryVector = new float[testQueryDimension]; + private final String testIndexName = "test-index"; + private final String testFieldName = "test-field"; + private final int testK = 10; + + public void testCreateCustomKNNQuery() { + for (KNNEngine knnEngine : KNNEngine.getEnginesThatCreateCustomSegmentFiles()) { + Query query = KNNQueryFactory.create(knnEngine, testIndexName, testFieldName, testQueryVector, testK); + assertTrue(query instanceof KNNQuery); + + assertEquals(testIndexName, ((KNNQuery) query).getIndexName()); + assertEquals(testFieldName, ((KNNQuery) query).getField()); + assertEquals(testQueryVector, ((KNNQuery) query).getQueryVector()); + assertEquals(testK, ((KNNQuery) query).getK()); + } + } + + public void testCreateLuceneDefaultQuery() { + List luceneDefaultQueryEngineList = Arrays.stream(KNNEngine.values()) + .filter(knnEngine -> !KNNEngine.getEnginesThatCreateCustomSegmentFiles().contains(knnEngine)) + .collect(Collectors.toList()); + for (KNNEngine knnEngine : luceneDefaultQueryEngineList) { + Query query = KNNQueryFactory.create(knnEngine, testIndexName, testFieldName, testQueryVector, testK); + assertTrue(query instanceof KnnVectorQuery); + } + } +} diff --git a/src/test/java/org/opensearch/knn/jni/JNIServiceTests.java b/src/test/java/org/opensearch/knn/jni/JNIServiceTests.java index 3464beff8..f4971e6fd 100644 --- a/src/test/java/org/opensearch/knn/jni/JNIServiceTests.java +++ b/src/test/java/org/opensearch/knn/jni/JNIServiceTests.java @@ -18,7 +18,7 @@ import org.opensearch.knn.TestUtils; import org.opensearch.knn.common.KNNConstants; import org.opensearch.knn.index.KNNMethodContext; -import org.opensearch.knn.index.KNNQueryResult; +import org.opensearch.knn.index.query.KNNQueryResult; import org.opensearch.knn.index.MethodComponentContext; import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.util.KNNEngine; diff --git a/src/test/java/org/opensearch/knn/plugin/action/RestKNNStatsHandlerIT.java b/src/test/java/org/opensearch/knn/plugin/action/RestKNNStatsHandlerIT.java index 1a879aa0c..10fa47c4c 100644 --- a/src/test/java/org/opensearch/knn/plugin/action/RestKNNStatsHandlerIT.java +++ b/src/test/java/org/opensearch/knn/plugin/action/RestKNNStatsHandlerIT.java @@ -20,7 +20,7 @@ import org.opensearch.index.query.MatchAllQueryBuilder; import org.opensearch.index.query.QueryBuilder; import org.opensearch.knn.KNNRestTestCase; -import org.opensearch.knn.index.KNNQueryBuilder; +import org.opensearch.knn.index.query.KNNQueryBuilder; import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.plugin.stats.KNNStats; import org.opensearch.knn.plugin.stats.StatNames; diff --git a/src/test/java/org/opensearch/knn/plugin/action/RestLegacyKNNStatsHandlerIT.java b/src/test/java/org/opensearch/knn/plugin/action/RestLegacyKNNStatsHandlerIT.java index e4fe1891f..f946b2473 100644 --- a/src/test/java/org/opensearch/knn/plugin/action/RestLegacyKNNStatsHandlerIT.java +++ b/src/test/java/org/opensearch/knn/plugin/action/RestLegacyKNNStatsHandlerIT.java @@ -13,7 +13,7 @@ import org.opensearch.knn.KNNRestTestCase; import org.opensearch.knn.index.SpaceType; -import org.opensearch.knn.index.KNNQueryBuilder; +import org.opensearch.knn.index.query.KNNQueryBuilder; import org.opensearch.knn.plugin.KNNPlugin; import org.opensearch.knn.plugin.stats.KNNStats; import org.opensearch.knn.plugin.stats.StatNames; diff --git a/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java b/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java index 932ae04b5..3edf508ab 100644 --- a/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java +++ b/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java @@ -12,7 +12,7 @@ import org.opensearch.common.bytes.BytesReference; import org.opensearch.common.xcontent.XContentHelper; import org.opensearch.index.query.MatchAllQueryBuilder; -import org.opensearch.knn.index.KNNQueryBuilder; +import org.opensearch.knn.index.query.KNNQueryBuilder; import org.opensearch.knn.index.KNNSettings; import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.indices.ModelDao; From 1ced35152921b9110d7f2f33c0edd1fdab557530 Mon Sep 17 00:00:00 2001 From: Martin Gaievski Date: Tue, 19 Jul 2022 20:11:33 -0700 Subject: [PATCH 014/416] [feature/lucene-knn] Add field mapper and per field format (#456) * Add field mapper and per field format Signed-off-by: Martin Gaievski (cherry picked from commit 1fb804740220aa1a0094d50a458c7198cba4982e) --- .../KNN80Codec/KNN80DocValuesConsumer.java | 31 +++-- .../index/codec/KNN920Codec/KNN920Codec.java | 82 ++++++++++- .../knn/index/codec/KNNCodecFactory.java | 5 +- .../index/mapper/KNNVectorFieldMapper.java | 54 ++++++-- .../knn/index/mapper/LuceneFieldMapper.java | 127 ++++++++++++++++++ .../org/opensearch/knn/index/LuceneIT.java | 118 ++++++++++++++++ .../codec/KNN920Codec/KNN920CodecTests.java | 106 ++++++++++++++- .../knn/index/codec/KNNCodecFactoryTests.java | 4 +- .../index/codec/KNNFormatFactoryTests.java | 2 +- .../mapper/KNNVectorFieldMapperTests.java | 91 +++++++++++++ 10 files changed, 590 insertions(+), 30 deletions(-) create mode 100644 src/main/java/org/opensearch/knn/index/mapper/LuceneFieldMapper.java create mode 100644 src/test/java/org/opensearch/knn/index/LuceneIT.java diff --git a/src/main/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumer.java b/src/main/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumer.java index 491f6e1cc..1368f63de 100644 --- a/src/main/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumer.java +++ b/src/main/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumer.java @@ -6,6 +6,8 @@ package org.opensearch.knn.index.codec.KNN80Codec; import com.google.common.collect.ImmutableMap; +import lombok.NonNull; +import lombok.extern.log4j.Log4j2; import org.apache.lucene.store.ChecksumIndexInput; import org.opensearch.common.xcontent.DeprecationHandler; import org.opensearch.common.xcontent.NamedXContentRegistry; @@ -54,6 +56,7 @@ /** * This class writes the KNN docvalues to the segments */ +@Log4j2 class KNN80DocValuesConsumer extends DocValuesConsumer implements Closeable { private final Logger logger = LogManager.getLogger(KNN80DocValuesConsumer.class); @@ -71,13 +74,29 @@ class KNN80DocValuesConsumer extends DocValuesConsumer implements Closeable { @Override public void addBinaryField(FieldInfo field, DocValuesProducer valuesProducer) throws IOException { delegatee.addBinaryField(field, valuesProducer); - if (field.attributes().containsKey(KNNVectorFieldMapper.KNN_FIELD)) { + if (isKNNBinaryFieldRequired(field)) { addKNNBinaryField(field, valuesProducer); } } - public void addKNNBinaryField(FieldInfo field, DocValuesProducer valuesProducer) throws IOException { + private boolean isKNNBinaryFieldRequired(FieldInfo field) { + final KNNEngine knnEngine = getKNNEngine(field); + log.debug(String.format("Read engine [%s] for field [%s]", knnEngine.getName(), field.getName())); + return field.attributes().containsKey(KNNVectorFieldMapper.KNN_FIELD) + && KNNEngine.getEnginesThatCreateCustomSegmentFiles().stream().anyMatch(engine -> engine == knnEngine); + } + + private KNNEngine getKNNEngine(@NonNull FieldInfo field) { + final String modelId = field.attributes().get(MODEL_ID); + if (modelId != null) { + var model = ModelCache.getInstance().get(modelId); + return model.getModelMetadata().getKnnEngine(); + } + final String engineName = field.attributes().getOrDefault(KNNConstants.KNN_ENGINE, KNNEngine.DEFAULT.getName()); + return KNNEngine.getEngine(engineName); + } + public void addKNNBinaryField(FieldInfo field, DocValuesProducer valuesProducer) throws IOException { // Get values to be indexed BinaryDocValues values = valuesProducer.getBinary(field); KNNCodecUtil.Pair pair = KNNCodecUtil.getFloats(values); @@ -85,20 +104,18 @@ public void addKNNBinaryField(FieldInfo field, DocValuesProducer valuesProducer) logger.info("Skipping engine index creation as there are no vectors or docs in the documents"); return; } - // Increment counter for number of graph index requests KNNCounter.GRAPH_INDEX_REQUESTS.increment(); // Create library index either from model or from scratch String engineFileName; String indexPath; NativeIndexCreator indexCreator; + final KNNEngine knnEngine = getKNNEngine(field); if (field.attributes().containsKey(MODEL_ID)) { String modelId = field.attributes().get(MODEL_ID); Model model = ModelCache.getInstance().get(modelId); - KNNEngine knnEngine = model.getModelMetadata().getKnnEngine(); - engineFileName = buildEngineFileName(state.segmentInfo.name, knnEngine.getVersion(), field.name, knnEngine.getExtension()); indexPath = Paths.get(((FSDirectory) (FilterDirectory.unwrap(state.directory))).getDirectory().toString(), engineFileName) .toString(); @@ -110,10 +127,6 @@ public void addKNNBinaryField(FieldInfo field, DocValuesProducer valuesProducer) indexCreator = () -> createKNNIndexFromTemplate(model.getModelBlob(), pair, knnEngine, indexPath); } else { - // Get engine to be used for indexing - String engineName = field.attributes().getOrDefault(KNNConstants.KNN_ENGINE, KNNEngine.DEFAULT.getName()); - KNNEngine knnEngine = KNNEngine.getEngine(engineName); - engineFileName = buildEngineFileName(state.segmentInfo.name, knnEngine.getVersion(), field.name, knnEngine.getExtension()); indexPath = Paths.get(((FSDirectory) (FilterDirectory.unwrap(state.directory))).getDirectory().toString(), engineFileName) .toString(); diff --git a/src/main/java/org/opensearch/knn/index/codec/KNN920Codec/KNN920Codec.java b/src/main/java/org/opensearch/knn/index/codec/KNN920Codec/KNN920Codec.java index 41b01c772..9ec032694 100644 --- a/src/main/java/org/opensearch/knn/index/codec/KNN920Codec/KNN920Codec.java +++ b/src/main/java/org/opensearch/knn/index/codec/KNN920Codec/KNN920Codec.java @@ -5,28 +5,42 @@ package org.opensearch.knn.index.codec.KNN920Codec; import lombok.Builder; +import lombok.extern.log4j.Log4j2; import org.apache.lucene.codecs.Codec; import org.apache.lucene.codecs.CompoundFormat; import org.apache.lucene.codecs.DocValuesFormat; import org.apache.lucene.codecs.FilterCodec; +import org.apache.lucene.codecs.KnnVectorsFormat; +import org.apache.lucene.codecs.lucene92.Lucene92HnswVectorsFormat; +import org.apache.lucene.codecs.perfield.PerFieldKnnVectorsFormat; +import org.opensearch.index.mapper.MapperService; +import org.opensearch.knn.common.KNNConstants; import org.opensearch.knn.index.codec.KNNFormatFacade; import org.opensearch.knn.index.codec.KNNFormatFactory; +import org.opensearch.knn.index.mapper.KNNVectorFieldMapper; + +import java.util.Map; +import java.util.Optional; import static org.opensearch.knn.index.codec.KNNCodecFactory.CodecDelegateFactory.createKNN92DefaultDelegate; /** * KNN codec that is based on Lucene92 codec */ +@Log4j2 public final class KNN920Codec extends FilterCodec { private static final String KNN920 = "KNN920Codec"; + private final KNNFormatFacade knnFormatFacade; + private final Optional mapperService; + private final KNN920PerFieldKnnVectorsFormat perFieldKnnVectorsFormat; /** * No arg constructor that uses Lucene91 as the delegate */ public KNN920Codec() { - this(createKNN92DefaultDelegate()); + this(createKNN92DefaultDelegate(), Optional.empty()); } /** @@ -35,9 +49,11 @@ public KNN920Codec() { * @param delegate codec that will perform all operations this codec does not override */ @Builder - public KNN920Codec(Codec delegate) { + public KNN920Codec(Codec delegate, Optional mapperService) { super(KNN920, delegate); + this.mapperService = mapperService; knnFormatFacade = KNNFormatFactory.createKNN920Format(delegate); + perFieldKnnVectorsFormat = new KNN920PerFieldKnnVectorsFormat(); } @Override @@ -49,4 +65,66 @@ public DocValuesFormat docValuesFormat() { public CompoundFormat compoundFormat() { return knnFormatFacade.compoundFormat(); } + + @Override + public KnnVectorsFormat knnVectorsFormat() { + return perFieldKnnVectorsFormat; + } + + /** + * Class provides per field format implementation for Lucene Knn vector type + */ + class KNN920PerFieldKnnVectorsFormat extends PerFieldKnnVectorsFormat { + + @Override + public KnnVectorsFormat getKnnVectorsFormatForField(final String field) { + if (isNotKnnVectorFieldType(field)) { + log.debug( + String.format( + "Initialize KNN vector format for field [%s] with default params [max_connections] = \"%d\" and [beam_width] = \"%d\"", + field, + Lucene92HnswVectorsFormat.DEFAULT_MAX_CONN, + Lucene92HnswVectorsFormat.DEFAULT_BEAM_WIDTH + ) + ); + return new Lucene92HnswVectorsFormat(); + } + final KNNVectorFieldMapper.KNNVectorFieldType type = (KNNVectorFieldMapper.KNNVectorFieldType) mapperService.orElseThrow( + () -> new IllegalStateException( + String.format("Cannot read field type for field [%s] because mapper service is not available", field) + ) + ).fieldType(field); + final Map params = type.getKnnMethodContext().getMethodComponent().getParameters(); + int maxConnections = getMaxConnections(params); + int beamWidth = getBeamWidth(params); + log.debug( + String.format( + "Initialize KNN vector format for field [%s] with params [max_connections] = \"%d\" and [beam_width] = \"%d\"", + field, + maxConnections, + beamWidth + ) + ); + var luceneHnswVectorsFormat = new Lucene92HnswVectorsFormat(maxConnections, beamWidth); + return luceneHnswVectorsFormat; + } + + private boolean isNotKnnVectorFieldType(final String field) { + return !mapperService.isPresent() || !(mapperService.get().fieldType(field) instanceof KNNVectorFieldMapper.KNNVectorFieldType); + } + + private int getMaxConnections(final Map params) { + if (params != null && params.containsKey(KNNConstants.METHOD_PARAMETER_M)) { + return (int) params.get(KNNConstants.METHOD_PARAMETER_M); + } + return Lucene92HnswVectorsFormat.DEFAULT_MAX_CONN; + } + + private int getBeamWidth(final Map params) { + if (params != null && params.containsKey(KNNConstants.METHOD_PARAMETER_EF_CONSTRUCTION)) { + return (int) params.get(KNNConstants.METHOD_PARAMETER_EF_CONSTRUCTION); + } + return Lucene92HnswVectorsFormat.DEFAULT_BEAM_WIDTH; + } + } } diff --git a/src/main/java/org/opensearch/knn/index/codec/KNNCodecFactory.java b/src/main/java/org/opensearch/knn/index/codec/KNNCodecFactory.java index 87de6e447..9c5a6efae 100644 --- a/src/main/java/org/opensearch/knn/index/codec/KNNCodecFactory.java +++ b/src/main/java/org/opensearch/knn/index/codec/KNNCodecFactory.java @@ -11,6 +11,8 @@ import org.opensearch.index.mapper.MapperService; import org.opensearch.knn.index.codec.KNN920Codec.KNN920Codec; +import java.util.Optional; + /** * Factory abstraction for KNN codec */ @@ -20,7 +22,8 @@ public class KNNCodecFactory { private final MapperService mapperService; public Codec createKNNCodec(final Codec userCodec) { - return KNN920Codec.builder().delegate(userCodec).build(); + var codec = KNN920Codec.builder().delegate(userCodec).mapperService(Optional.of(mapperService)).build(); + return codec; } /** diff --git a/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapper.java b/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapper.java index 45d34dd68..11cc03b26 100644 --- a/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapper.java +++ b/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapper.java @@ -6,6 +6,7 @@ package org.opensearch.knn.index.mapper; import lombok.Getter; +import lombok.extern.log4j.Log4j2; import org.opensearch.common.ValidationException; import org.opensearch.knn.common.KNNConstants; @@ -35,6 +36,7 @@ import org.opensearch.knn.index.KNNSettings; import org.opensearch.knn.index.KNNVectorIndexFieldData; import org.opensearch.knn.index.VectorField; +import org.opensearch.knn.index.util.KNNEngine; import org.opensearch.knn.indices.ModelDao; import org.opensearch.search.aggregations.support.CoreValuesSourceType; import org.opensearch.search.lookup.SearchLookup; @@ -57,6 +59,7 @@ * Implementations of this class define what needs to be stored in Lucene's fieldType. This allows us to have * alternative mappings for the same field type. */ +@Log4j2 public abstract class KNNVectorFieldMapper extends ParametrizedFieldMapper { public static final String CONTENT_TYPE = "knn_vector"; @@ -193,14 +196,39 @@ public KNNVectorFieldMapper build(BuilderContext context) { // set. If not, we fall back to the parameters set in the index settings. This means that if a user sets // the mappings, setting the index settings will have no impact. - KNNMethodContext knnMethodContext = this.knnMethodContext.getValue(); + final KNNMethodContext knnMethodContext = this.knnMethodContext.getValue(); + final MultiFields multiFieldsBuilder = this.multiFieldsBuilder.build(this, context); + final CopyTo copyToBuilder = copyTo.build(); + final Explicit ignoreMalformed = ignoreMalformed(context); + final Map metaValue = meta.getValue(); if (knnMethodContext != null) { + final KNNVectorFieldType mappedFieldType = new KNNVectorFieldType( + buildFullName(context), + metaValue, + dimension.getValue(), + knnMethodContext + ); + if (knnMethodContext.getKnnEngine() == KNNEngine.LUCENE) { + log.debug(String.format("Use [LuceneFieldMapper] mapper for field [%s]", name)); + LuceneFieldMapper.CreateLuceneFieldMapperInput createLuceneFieldMapperInput = + LuceneFieldMapper.CreateLuceneFieldMapperInput.builder() + .name(name) + .mappedFieldType(mappedFieldType) + .multiFields(multiFieldsBuilder) + .copyTo(copyToBuilder) + .ignoreMalformed(ignoreMalformed) + .stored(stored.get()) + .hasDocValues(hasDocValues.get()) + .knnMethodContext(knnMethodContext) + .build(); + return new LuceneFieldMapper(createLuceneFieldMapperInput); + } return new MethodFieldMapper( name, - new KNNVectorFieldType(buildFullName(context), meta.getValue(), dimension.getValue(), knnMethodContext), - multiFieldsBuilder.build(this, context), - copyTo.build(), - ignoreMalformed(context), + mappedFieldType, + multiFieldsBuilder, + copyToBuilder, + ignoreMalformed, stored.get(), hasDocValues.get(), knnMethodContext @@ -216,10 +244,10 @@ public KNNVectorFieldMapper build(BuilderContext context) { return new ModelFieldMapper( name, - new KNNVectorFieldType(buildFullName(context), meta.getValue(), -1, knnMethodContext, modelIdAsString), - multiFieldsBuilder.build(this, context), - copyTo.build(), - ignoreMalformed(context), + new KNNVectorFieldType(buildFullName(context), metaValue, -1, knnMethodContext, modelIdAsString), + multiFieldsBuilder, + copyToBuilder, + ignoreMalformed, stored.get(), hasDocValues.get(), modelDao, @@ -242,10 +270,10 @@ public KNNVectorFieldMapper build(BuilderContext context) { return new LegacyFieldMapper( name, - new KNNVectorFieldType(buildFullName(context), meta.getValue(), dimension.getValue()), - multiFieldsBuilder.build(this, context), - copyTo.build(), - ignoreMalformed(context), + new KNNVectorFieldType(buildFullName(context), metaValue, dimension.getValue()), + multiFieldsBuilder, + copyToBuilder, + ignoreMalformed, stored.get(), hasDocValues.get(), spaceType, diff --git a/src/main/java/org/opensearch/knn/index/mapper/LuceneFieldMapper.java b/src/main/java/org/opensearch/knn/index/mapper/LuceneFieldMapper.java new file mode 100644 index 000000000..2604a29a8 --- /dev/null +++ b/src/main/java/org/opensearch/knn/index/mapper/LuceneFieldMapper.java @@ -0,0 +1,127 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.mapper; + +import com.google.common.collect.ImmutableMap; +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.NonNull; +import org.apache.lucene.document.FieldType; +import org.apache.lucene.document.KnnVectorField; +import org.apache.lucene.document.StoredField; +import org.apache.lucene.index.IndexOptions; +import org.apache.lucene.index.VectorSimilarityFunction; +import org.apache.lucene.index.VectorValues; +import org.opensearch.common.Explicit; +import org.opensearch.index.mapper.ParseContext; +import org.opensearch.knn.index.KNNMethodContext; +import org.opensearch.knn.index.SpaceType; + +import java.io.IOException; +import java.util.Map; +import java.util.Optional; + +import static org.opensearch.knn.index.SpaceType.COSINESIMIL; +import static org.opensearch.knn.index.SpaceType.INNER_PRODUCT; +import static org.opensearch.knn.index.SpaceType.L2; + +/** + * Field mapper for case when Lucene has been set as an engine. + */ +public class LuceneFieldMapper extends KNNVectorFieldMapper { + + private static final int MAX_DIMENSION = VectorValues.MAX_DIMENSIONS; + + private static final Map SPACE_TYPE_TO_VECTOR_SIMILARITY_FUNCTION = ImmutableMap.of( + L2, + VectorSimilarityFunction.EUCLIDEAN, + COSINESIMIL, + VectorSimilarityFunction.COSINE, + INNER_PRODUCT, + VectorSimilarityFunction.DOT_PRODUCT + ); + + LuceneFieldMapper(final CreateLuceneFieldMapperInput input) { + super( + input.getName(), + input.getMappedFieldType(), + input.getMultiFields(), + input.getCopyTo(), + input.getIgnoreMalformed(), + input.isStored(), + input.isHasDocValues() + ); + + this.knnMethod = input.getKnnMethodContext(); + + this.fieldType = new FieldType(); + + this.fieldType.setTokenized(false); + this.fieldType.setIndexOptions(IndexOptions.NONE); + + final SpaceType spaceType = this.knnMethod.getSpaceType(); + final VectorSimilarityFunction vectorSimilarityFunction = Optional.ofNullable( + SPACE_TYPE_TO_VECTOR_SIMILARITY_FUNCTION.get(spaceType) + ).orElseThrow(() -> new IllegalArgumentException(String.format("Space type [%s] is not supported for Lucene engine", spaceType))); + + final int dimension = input.getMappedFieldType().getDimension(); + if (dimension > MAX_DIMENSION) { + throw new IllegalArgumentException( + String.format( + "Dimension value cannot be greater than [%s] but got [%s] for vector [%s]", + MAX_DIMENSION, + dimension, + input.getName() + ) + ); + } + + this.fieldType.setVectorDimensionsAndSimilarityFunction(dimension, vectorSimilarityFunction); + this.fieldType.freeze(); + } + + @Override + protected void parseCreateField(ParseContext context, int dimension) throws IOException { + + validateIfKNNPluginEnabled(); + validateIfCircuitBreakerIsNotTriggered(); + + Optional arrayOptional = getFloatsFromContext(context, dimension); + + if (!arrayOptional.isPresent()) { + return; + } + final float[] array = arrayOptional.get(); + + KnnVectorField point = new KnnVectorField(name(), array, fieldType); + + context.doc().add(point); + if (fieldType.stored()) { + context.doc().add(new StoredField(name(), point.toString())); + } + context.path().remove(); + } + + @AllArgsConstructor + @lombok.Builder + @Getter + static class CreateLuceneFieldMapperInput { + @NonNull + String name; + @NonNull + KNNVectorFieldType mappedFieldType; + @NonNull + MultiFields multiFields; + @NonNull + CopyTo copyTo; + @NonNull + Explicit ignoreMalformed; + boolean stored; + boolean hasDocValues; + @NonNull + KNNMethodContext knnMethodContext; + } +} diff --git a/src/test/java/org/opensearch/knn/index/LuceneIT.java b/src/test/java/org/opensearch/knn/index/LuceneIT.java new file mode 100644 index 000000000..2053bbb89 --- /dev/null +++ b/src/test/java/org/opensearch/knn/index/LuceneIT.java @@ -0,0 +1,118 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index; + +import com.google.common.collect.ImmutableList; +import org.opensearch.common.Strings; +import org.opensearch.common.xcontent.XContentBuilder; +import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.knn.KNNRestTestCase; +import org.opensearch.knn.common.KNNConstants; +import org.opensearch.knn.index.util.KNNEngine; + +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; + +public class LuceneIT extends KNNRestTestCase { + + private static final int DIMENSION = 3; + private static final String DOC_ID = "doc1"; + private static final int EF_CONSTRUCTION = 128; + private static final String INDEX_NAME = "test-index-1"; + private static final String FIELD_NAME = "test-field-1"; + private static final int M = 16; + + public void test_addDoc() throws IOException { + List mValues = ImmutableList.of(16, 32, 64, 128); + List efConstructionValues = ImmutableList.of(16, 32, 64, 128); + + XContentBuilder builder = XContentFactory.jsonBuilder() + .startObject() + .startObject("properties") + .startObject(FIELD_NAME) + .field("type", "knn_vector") + .field("dimension", DIMENSION) + .startObject(KNNConstants.KNN_METHOD) + .field(KNNConstants.NAME, KNNEngine.LUCENE.getMethod(KNNConstants.METHOD_HNSW).getMethodComponent().getName()) + .field(KNNConstants.METHOD_PARAMETER_SPACE_TYPE, SpaceType.L2.getValue()) + .field(KNNConstants.KNN_ENGINE, KNNEngine.LUCENE.getName()) + .startObject(KNNConstants.PARAMETERS) + .field(KNNConstants.METHOD_PARAMETER_M, mValues.get(random().nextInt(mValues.size()))) + .field(KNNConstants.METHOD_PARAMETER_EF_CONSTRUCTION, efConstructionValues.get(random().nextInt(efConstructionValues.size()))) + .endObject() + .endObject() + .endObject() + .endObject() + .endObject(); + + Map mappingMap = xContentBuilderToMap(builder); + String mapping = Strings.toString(builder); + + createKnnIndex(INDEX_NAME, mapping); + assertEquals(new TreeMap<>(mappingMap), new TreeMap<>(getIndexMappingAsMap(INDEX_NAME))); + + Float[] vector = new Float[] { 2.0f, 4.5f, 6.5f }; + addKnnDoc(INDEX_NAME, DOC_ID, FIELD_NAME, vector); + + refreshAllIndices(); + assertEquals(1, getDocCount(INDEX_NAME)); + + deleteKNNIndex(INDEX_NAME); + } + + public void test_updateDoc() throws Exception { + createKnnIndexMappingWithLuceneEngine(2); + Float[] vector = { 6.0f, 6.0f }; + addKnnDoc(INDEX_NAME, DOC_ID, FIELD_NAME, vector); + + Float[] updatedVector = { 8.0f, 8.0f }; + updateKnnDoc(INDEX_NAME, DOC_ID, FIELD_NAME, updatedVector); + + refreshAllIndices(); + assertEquals(1, getDocCount(INDEX_NAME)); + + deleteKNNIndex(INDEX_NAME); + } + + public void test_deleteDoc() throws Exception { + createKnnIndexMappingWithLuceneEngine(2); + Float[] vector = { 6.0f, 6.0f }; + addKnnDoc(INDEX_NAME, DOC_ID, FIELD_NAME, vector); + + deleteKnnDoc(INDEX_NAME, DOC_ID); + + refreshAllIndices(); + assertEquals(0, getDocCount(INDEX_NAME)); + + deleteKNNIndex(INDEX_NAME); + } + + private void createKnnIndexMappingWithLuceneEngine(int dimension) throws Exception { + XContentBuilder builder = XContentFactory.jsonBuilder() + .startObject() + .startObject("properties") + .startObject(FIELD_NAME) + .field("type", "knn_vector") + .field("dimension", dimension) + .startObject(KNNConstants.KNN_METHOD) + .field(KNNConstants.NAME, KNNEngine.LUCENE.getMethod(KNNConstants.METHOD_HNSW).getMethodComponent().getName()) + .field(KNNConstants.METHOD_PARAMETER_SPACE_TYPE, SpaceType.L2.getValue()) + .field(KNNConstants.KNN_ENGINE, KNNEngine.LUCENE.getName()) + .startObject(KNNConstants.PARAMETERS) + .field(KNNConstants.METHOD_PARAMETER_M, M) + .field(KNNConstants.METHOD_PARAMETER_EF_CONSTRUCTION, EF_CONSTRUCTION) + .endObject() + .endObject() + .endObject() + .endObject() + .endObject(); + + String mapping = Strings.toString(builder); + createKnnIndex(INDEX_NAME, mapping); + } +} diff --git a/src/test/java/org/opensearch/knn/index/codec/KNN920Codec/KNN920CodecTests.java b/src/test/java/org/opensearch/knn/index/codec/KNN920Codec/KNN920CodecTests.java index 66010a58f..ed39eca52 100644 --- a/src/test/java/org/opensearch/knn/index/codec/KNN920Codec/KNN920CodecTests.java +++ b/src/test/java/org/opensearch/knn/index/codec/KNN920Codec/KNN920CodecTests.java @@ -5,18 +5,120 @@ package org.opensearch.knn.index.codec.KNN920Codec; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.FieldType; +import org.apache.lucene.document.KnnVectorField; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.SerialMergeScheduler; +import org.apache.lucene.index.VectorSimilarityFunction; +import org.apache.lucene.store.Directory; +import org.apache.lucene.tests.index.RandomIndexWriter; +import org.opensearch.index.mapper.MapperService; +import org.opensearch.knn.index.KNNMethodContext; +import org.opensearch.knn.index.MethodComponentContext; +import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.codec.KNNCodecTestCase; +import org.opensearch.knn.index.mapper.KNNVectorFieldMapper; +import org.opensearch.knn.index.memory.NativeMemoryLoadStrategy; +import org.opensearch.knn.index.util.KNNEngine; +import org.opensearch.watcher.ResourceWatcherService; import java.io.IOException; +import java.util.Map; +import java.util.Optional; import java.util.concurrent.ExecutionException; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import static org.opensearch.knn.common.KNNConstants.HNSW_ALGO_EF_CONSTRUCTION; +import static org.opensearch.knn.common.KNNConstants.HNSW_ALGO_M; +import static org.opensearch.knn.common.KNNConstants.METHOD_HNSW; +import static org.opensearch.knn.index.codec.KNNCodecFactory.CodecDelegateFactory.createKNN92DefaultDelegate; + public class KNN920CodecTests extends KNNCodecTestCase { public void testMultiFieldsKnnIndex() throws Exception { - testMultiFieldsKnnIndex(new KNN920Codec()); + testMultiFieldsKnnIndex(KNN920Codec.builder().delegate(createKNN92DefaultDelegate()).build()); } public void testBuildFromModelTemplate() throws InterruptedException, ExecutionException, IOException { - testBuildFromModelTemplate(new KNN920Codec()); + testBuildFromModelTemplate((KNN920Codec.builder().delegate(createKNN92DefaultDelegate()).build())); + } + + public void testKnnVectorIndex() throws Exception { + final String fieldName = "test_vector"; + final String field1Name = "my_vector"; + final MapperService mapperService = mock(MapperService.class); + final KNNMethodContext knnMethodContext = new KNNMethodContext( + KNNEngine.LUCENE, + SpaceType.L2, + new MethodComponentContext(METHOD_HNSW, Map.of(HNSW_ALGO_M, 16, HNSW_ALGO_EF_CONSTRUCTION, 256)) + ); + final KNNVectorFieldMapper.KNNVectorFieldType mappedFieldType1 = new KNNVectorFieldMapper.KNNVectorFieldType( + fieldName, + Map.of(), + 3, + knnMethodContext + ); + final KNNVectorFieldMapper.KNNVectorFieldType mappedFieldType2 = new KNNVectorFieldMapper.KNNVectorFieldType( + field1Name, + Map.of(), + 2, + knnMethodContext + ); + when(mapperService.fieldType(eq(fieldName))).thenReturn(mappedFieldType1); + when(mapperService.fieldType(eq(field1Name))).thenReturn(mappedFieldType2); + + final KNN920Codec actualCodec = KNN920Codec.builder() + .delegate(createKNN92DefaultDelegate()) + .mapperService(Optional.of(mapperService)) + .build(); + final KNN920Codec codec = KNN920Codec.builder() + .delegate(createKNN92DefaultDelegate()) + .mapperService(Optional.of(mapperService)) + .build(); + setUpMockClusterService(); + Directory dir = newFSDirectory(createTempDir()); + IndexWriterConfig iwc = newIndexWriterConfig(); + iwc.setMergeScheduler(new SerialMergeScheduler()); + iwc.setCodec(codec); + + /** + * Add doc with field "test_vector" + */ + final FieldType luceneFieldType = KnnVectorField.createFieldType(3, VectorSimilarityFunction.EUCLIDEAN); + float[] array = { 1.0f, 3.0f, 4.0f }; + KnnVectorField vectorField = new KnnVectorField(fieldName, array, luceneFieldType); + RandomIndexWriter writer = new RandomIndexWriter(random(), dir, iwc); + Document doc = new Document(); + doc.add(vectorField); + writer.addDocument(doc); + writer.commit(); + writer.close(); + + /** + * Add doc with field "my_vector" + */ + IndexWriterConfig iwc1 = newIndexWriterConfig(); + iwc1.setMergeScheduler(new SerialMergeScheduler()); + iwc1.setCodec(actualCodec); + writer = new RandomIndexWriter(random(), dir, iwc1); + final FieldType luceneFieldType1 = KnnVectorField.createFieldType(2, VectorSimilarityFunction.EUCLIDEAN); + float[] array1 = { 6.0f, 14.0f }; + KnnVectorField vectorField1 = new KnnVectorField(field1Name, array1, luceneFieldType1); + Document doc1 = new Document(); + doc1.add(vectorField1); + writer.addDocument(doc1); + IndexReader reader = writer.getReader(); + writer.close(); + ResourceWatcherService resourceWatcherService = createDisabledResourceWatcherService(); + NativeMemoryLoadStrategy.IndexLoadStrategy.initialize(resourceWatcherService); + + reader.close(); + dir.close(); + resourceWatcherService.close(); + NativeMemoryLoadStrategy.IndexLoadStrategy.getInstance().close(); } } diff --git a/src/test/java/org/opensearch/knn/index/codec/KNNCodecFactoryTests.java b/src/test/java/org/opensearch/knn/index/codec/KNNCodecFactoryTests.java index 7630cf6d8..e339d0a3a 100644 --- a/src/test/java/org/opensearch/knn/index/codec/KNNCodecFactoryTests.java +++ b/src/test/java/org/opensearch/knn/index/codec/KNNCodecFactoryTests.java @@ -29,11 +29,11 @@ public void testKNN92DefaultDelegate() { } public void testKNNDefaultCodec() { - Lucene91Codec lucene91CodecDelegate = new Lucene91Codec(); MapperService mapperService = mock(MapperService.class); KNNCodecFactory knnCodecFactory = new KNNCodecFactory(mapperService); - Codec knnCodec = knnCodecFactory.createKNNCodec(lucene91CodecDelegate); + Codec knnCodec = knnCodecFactory.createKNNCodec(KNNCodecFactory.CodecDelegateFactory.createKNN92DefaultDelegate()); assertNotNull(knnCodec); assertTrue(knnCodec instanceof KNN920Codec); + assertEquals("KNN920Codec", knnCodec.getName()); } } diff --git a/src/test/java/org/opensearch/knn/index/codec/KNNFormatFactoryTests.java b/src/test/java/org/opensearch/knn/index/codec/KNNFormatFactoryTests.java index 2cb366d0f..1abf73661 100644 --- a/src/test/java/org/opensearch/knn/index/codec/KNNFormatFactoryTests.java +++ b/src/test/java/org/opensearch/knn/index/codec/KNNFormatFactoryTests.java @@ -26,8 +26,8 @@ public void testKNN91Format() { } public void testKNN92Format() { - final Codec lucene92CodecDelegate = KNNCodecFactory.CodecDelegateFactory.createKNN92DefaultDelegate(); MapperService mapperService = mock(MapperService.class); + final Codec lucene92CodecDelegate = KNNCodecFactory.CodecDelegateFactory.createKNN92DefaultDelegate(); KNNCodecFactory knnCodecFactory = new KNNCodecFactory(mapperService); final Codec knnCodec = knnCodecFactory.createKNNCodec(lucene92CodecDelegate); KNNFormatFacade knnFormatFacade = KNNFormatFactory.createKNN920Format(knnCodec); diff --git a/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java b/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java index 6f489bd92..3ddd1b73a 100644 --- a/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java +++ b/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java @@ -31,10 +31,13 @@ import java.time.ZonedDateTime; import java.util.HashSet; +import static org.opensearch.knn.common.KNNConstants.KNN_ENGINE; import static org.opensearch.knn.common.KNNConstants.KNN_METHOD; +import static org.opensearch.knn.common.KNNConstants.LUCENE_NAME; import static org.opensearch.knn.common.KNNConstants.METHOD_HNSW; import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_EF_CONSTRUCTION; import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_M; +import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_SPACE_TYPE; import static org.opensearch.knn.common.KNNConstants.MODEL_ID; import static org.opensearch.knn.common.KNNConstants.NAME; import static org.opensearch.knn.common.KNNConstants.PARAMETERS; @@ -150,6 +153,94 @@ public void testBuilder_build_fromLegacy() { assertNull(knnVectorFieldMapper.knnMethod); } + public void testBuilder_parse_fromKnnMethodContext_luceneEngine() throws IOException { + // Check that knnMethodContext is set + String fieldName = "test-field-name"; + String indexName = "test-index-name"; + + Settings settings = Settings.builder().put(settings(CURRENT).build()).build(); + + ModelDao modelDao = mock(ModelDao.class); + KNNVectorFieldMapper.TypeParser typeParser = new KNNVectorFieldMapper.TypeParser(() -> modelDao); + + int efConstruction = 321; + int m = 12; + int dimension = 133; + XContentBuilder xContentBuilder = XContentFactory.jsonBuilder() + .startObject() + .field("type", "knn_vector") + .field("dimension", dimension) + .startObject(KNN_METHOD) + .field(NAME, METHOD_HNSW) + .field(METHOD_PARAMETER_SPACE_TYPE, SpaceType.L2) + .field(KNN_ENGINE, LUCENE_NAME) + .startObject(PARAMETERS) + .field(METHOD_PARAMETER_EF_CONSTRUCTION, efConstruction) + .field(METHOD_PARAMETER_M, m) + .endObject() + .endObject() + .endObject(); + KNNVectorFieldMapper.Builder builder = (KNNVectorFieldMapper.Builder) typeParser.parse( + fieldName, + xContentBuilderToMap(xContentBuilder), + buildParserContext(indexName, settings) + ); + + assertEquals(METHOD_HNSW, builder.knnMethodContext.get().getMethodComponent().getName()); + assertEquals( + efConstruction, + builder.knnMethodContext.get().getMethodComponent().getParameters().get(METHOD_PARAMETER_EF_CONSTRUCTION) + ); + + XContentBuilder xContentBuilderInvalidSpaceType = XContentFactory.jsonBuilder() + .startObject() + .field("type", "knn_vector") + .field("dimension", dimension) + .startObject(KNN_METHOD) + .field(NAME, METHOD_HNSW) + .field(METHOD_PARAMETER_SPACE_TYPE, SpaceType.L1) + .field(KNN_ENGINE, LUCENE_NAME) + .startObject(PARAMETERS) + .field(METHOD_PARAMETER_EF_CONSTRUCTION, efConstruction) + .endObject() + .endObject() + .endObject(); + + expectThrows( + ValidationException.class, + () -> typeParser.parse( + fieldName, + xContentBuilderToMap(xContentBuilderInvalidSpaceType), + buildParserContext(indexName, settings) + ) + ); + + XContentBuilder xContentBuilderInvalidDimension = XContentFactory.jsonBuilder() + .startObject() + .field("type", "knn_vector") + .field("dimension", 2_000) + .startObject(KNN_METHOD) + .field(NAME, METHOD_HNSW) + .field(METHOD_PARAMETER_SPACE_TYPE, SpaceType.L2) + .field(KNN_ENGINE, LUCENE_NAME) + .startObject(PARAMETERS) + .field(METHOD_PARAMETER_EF_CONSTRUCTION, efConstruction) + .endObject() + .endObject() + .endObject(); + KNNVectorFieldMapper.Builder builderInvalidDimension = (KNNVectorFieldMapper.Builder) typeParser.parse( + fieldName, + xContentBuilderToMap(xContentBuilderInvalidDimension), + buildParserContext(indexName, settings) + ); + + IllegalArgumentException ex = expectThrows( + IllegalArgumentException.class, + () -> builderInvalidDimension.build(new Mapper.BuilderContext(settings, new ContentPath())) + ); + assertEquals("Dimension value cannot be greater than [1024] but got [2000] for vector [test-field-name]", ex.getMessage()); + } + public void testTypeParser_parse_fromKnnMethodContext() throws IOException { // Check that knnMethodContext is set String fieldName = "test-field-name"; From 647f859f1473ba70e9e073f6c8c164a91b6c1087 Mon Sep 17 00:00:00 2001 From: Martin Gaievski Date: Thu, 21 Jul 2022 11:43:40 -0700 Subject: [PATCH 015/416] Refactor per field format to a class, add more unit tests (#458) * Refactor per field format to a class, add tests Signed-off-by: Martin Gaievski (cherry picked from commit 77d3b2ef4860a68c0349b19f8822d4884555c051) --- .../index/codec/KNN920Codec/KNN920Codec.java | 72 +---------------- .../KNN920PerFieldKnnVectorsFormat.java | 78 +++++++++++++++++++ .../knn/index/codec/KNNCodecFactory.java | 6 +- .../KNN80DocValuesConsumerTests.java | 20 +++++ .../codec/KNN920Codec/KNN920CodecTests.java | 37 +++++++-- .../mapper/KNNVectorFieldMapperTests.java | 43 +++++++++- 6 files changed, 179 insertions(+), 77 deletions(-) create mode 100644 src/main/java/org/opensearch/knn/index/codec/KNN920Codec/KNN920PerFieldKnnVectorsFormat.java diff --git a/src/main/java/org/opensearch/knn/index/codec/KNN920Codec/KNN920Codec.java b/src/main/java/org/opensearch/knn/index/codec/KNN920Codec/KNN920Codec.java index 9ec032694..8d8e664ce 100644 --- a/src/main/java/org/opensearch/knn/index/codec/KNN920Codec/KNN920Codec.java +++ b/src/main/java/org/opensearch/knn/index/codec/KNN920Codec/KNN920Codec.java @@ -11,15 +11,9 @@ import org.apache.lucene.codecs.DocValuesFormat; import org.apache.lucene.codecs.FilterCodec; import org.apache.lucene.codecs.KnnVectorsFormat; -import org.apache.lucene.codecs.lucene92.Lucene92HnswVectorsFormat; -import org.apache.lucene.codecs.perfield.PerFieldKnnVectorsFormat; -import org.opensearch.index.mapper.MapperService; -import org.opensearch.knn.common.KNNConstants; import org.opensearch.knn.index.codec.KNNFormatFacade; import org.opensearch.knn.index.codec.KNNFormatFactory; -import org.opensearch.knn.index.mapper.KNNVectorFieldMapper; -import java.util.Map; import java.util.Optional; import static org.opensearch.knn.index.codec.KNNCodecFactory.CodecDelegateFactory.createKNN92DefaultDelegate; @@ -33,27 +27,26 @@ public final class KNN920Codec extends FilterCodec { private static final String KNN920 = "KNN920Codec"; private final KNNFormatFacade knnFormatFacade; - private final Optional mapperService; private final KNN920PerFieldKnnVectorsFormat perFieldKnnVectorsFormat; /** * No arg constructor that uses Lucene91 as the delegate */ public KNN920Codec() { - this(createKNN92DefaultDelegate(), Optional.empty()); + this(createKNN92DefaultDelegate(), new KNN920PerFieldKnnVectorsFormat(Optional.empty())); } /** * Constructor that takes a Codec delegate to delegate all methods this code does not implement to. * * @param delegate codec that will perform all operations this codec does not override + * @param knnVectorsFormat per field format for KnnVector */ @Builder - public KNN920Codec(Codec delegate, Optional mapperService) { + public KNN920Codec(Codec delegate, KNN920PerFieldKnnVectorsFormat knnVectorsFormat) { super(KNN920, delegate); - this.mapperService = mapperService; knnFormatFacade = KNNFormatFactory.createKNN920Format(delegate); - perFieldKnnVectorsFormat = new KNN920PerFieldKnnVectorsFormat(); + perFieldKnnVectorsFormat = knnVectorsFormat; } @Override @@ -70,61 +63,4 @@ public CompoundFormat compoundFormat() { public KnnVectorsFormat knnVectorsFormat() { return perFieldKnnVectorsFormat; } - - /** - * Class provides per field format implementation for Lucene Knn vector type - */ - class KNN920PerFieldKnnVectorsFormat extends PerFieldKnnVectorsFormat { - - @Override - public KnnVectorsFormat getKnnVectorsFormatForField(final String field) { - if (isNotKnnVectorFieldType(field)) { - log.debug( - String.format( - "Initialize KNN vector format for field [%s] with default params [max_connections] = \"%d\" and [beam_width] = \"%d\"", - field, - Lucene92HnswVectorsFormat.DEFAULT_MAX_CONN, - Lucene92HnswVectorsFormat.DEFAULT_BEAM_WIDTH - ) - ); - return new Lucene92HnswVectorsFormat(); - } - final KNNVectorFieldMapper.KNNVectorFieldType type = (KNNVectorFieldMapper.KNNVectorFieldType) mapperService.orElseThrow( - () -> new IllegalStateException( - String.format("Cannot read field type for field [%s] because mapper service is not available", field) - ) - ).fieldType(field); - final Map params = type.getKnnMethodContext().getMethodComponent().getParameters(); - int maxConnections = getMaxConnections(params); - int beamWidth = getBeamWidth(params); - log.debug( - String.format( - "Initialize KNN vector format for field [%s] with params [max_connections] = \"%d\" and [beam_width] = \"%d\"", - field, - maxConnections, - beamWidth - ) - ); - var luceneHnswVectorsFormat = new Lucene92HnswVectorsFormat(maxConnections, beamWidth); - return luceneHnswVectorsFormat; - } - - private boolean isNotKnnVectorFieldType(final String field) { - return !mapperService.isPresent() || !(mapperService.get().fieldType(field) instanceof KNNVectorFieldMapper.KNNVectorFieldType); - } - - private int getMaxConnections(final Map params) { - if (params != null && params.containsKey(KNNConstants.METHOD_PARAMETER_M)) { - return (int) params.get(KNNConstants.METHOD_PARAMETER_M); - } - return Lucene92HnswVectorsFormat.DEFAULT_MAX_CONN; - } - - private int getBeamWidth(final Map params) { - if (params != null && params.containsKey(KNNConstants.METHOD_PARAMETER_EF_CONSTRUCTION)) { - return (int) params.get(KNNConstants.METHOD_PARAMETER_EF_CONSTRUCTION); - } - return Lucene92HnswVectorsFormat.DEFAULT_BEAM_WIDTH; - } - } } diff --git a/src/main/java/org/opensearch/knn/index/codec/KNN920Codec/KNN920PerFieldKnnVectorsFormat.java b/src/main/java/org/opensearch/knn/index/codec/KNN920Codec/KNN920PerFieldKnnVectorsFormat.java new file mode 100644 index 000000000..5328e36a2 --- /dev/null +++ b/src/main/java/org/opensearch/knn/index/codec/KNN920Codec/KNN920PerFieldKnnVectorsFormat.java @@ -0,0 +1,78 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.codec.KNN920Codec; + +import lombok.AllArgsConstructor; +import lombok.extern.log4j.Log4j2; +import org.apache.lucene.codecs.KnnVectorsFormat; +import org.apache.lucene.codecs.lucene92.Lucene92HnswVectorsFormat; +import org.apache.lucene.codecs.perfield.PerFieldKnnVectorsFormat; +import org.opensearch.index.mapper.MapperService; +import org.opensearch.knn.common.KNNConstants; +import org.opensearch.knn.index.mapper.KNNVectorFieldMapper; + +import java.util.Map; +import java.util.Optional; + +/** + * Class provides per field format implementation for Lucene Knn vector type + */ +@AllArgsConstructor +@Log4j2 +public class KNN920PerFieldKnnVectorsFormat extends PerFieldKnnVectorsFormat { + + private final Optional mapperService; + + @Override + public KnnVectorsFormat getKnnVectorsFormatForField(final String field) { + if (isNotKnnVectorFieldType(field)) { + log.debug( + String.format( + "Initialize KNN vector format for field [%s] with default params [max_connections] = \"%d\" and [beam_width] = \"%d\"", + field, + Lucene92HnswVectorsFormat.DEFAULT_MAX_CONN, + Lucene92HnswVectorsFormat.DEFAULT_BEAM_WIDTH + ) + ); + return new Lucene92HnswVectorsFormat(); + } + var type = (KNNVectorFieldMapper.KNNVectorFieldType) mapperService.orElseThrow( + () -> new IllegalStateException( + String.format("Cannot read field type for field [%s] because mapper service is not available", field) + ) + ).fieldType(field); + var params = type.getKnnMethodContext().getMethodComponent().getParameters(); + int maxConnections = getMaxConnections(params); + int beamWidth = getBeamWidth(params); + log.debug( + String.format( + "Initialize KNN vector format for field [%s] with params [max_connections] = \"%d\" and [beam_width] = \"%d\"", + field, + maxConnections, + beamWidth + ) + ); + return new Lucene92HnswVectorsFormat(maxConnections, beamWidth); + } + + private boolean isNotKnnVectorFieldType(final String field) { + return !mapperService.isPresent() || !(mapperService.get().fieldType(field) instanceof KNNVectorFieldMapper.KNNVectorFieldType); + } + + private int getMaxConnections(final Map params) { + if (params != null && params.containsKey(KNNConstants.METHOD_PARAMETER_M)) { + return (int) params.get(KNNConstants.METHOD_PARAMETER_M); + } + return Lucene92HnswVectorsFormat.DEFAULT_MAX_CONN; + } + + private int getBeamWidth(final Map params) { + if (params != null && params.containsKey(KNNConstants.METHOD_PARAMETER_EF_CONSTRUCTION)) { + return (int) params.get(KNNConstants.METHOD_PARAMETER_EF_CONSTRUCTION); + } + return Lucene92HnswVectorsFormat.DEFAULT_BEAM_WIDTH; + } +} diff --git a/src/main/java/org/opensearch/knn/index/codec/KNNCodecFactory.java b/src/main/java/org/opensearch/knn/index/codec/KNNCodecFactory.java index 9c5a6efae..217bce3e8 100644 --- a/src/main/java/org/opensearch/knn/index/codec/KNNCodecFactory.java +++ b/src/main/java/org/opensearch/knn/index/codec/KNNCodecFactory.java @@ -10,6 +10,7 @@ import org.apache.lucene.codecs.lucene92.Lucene92Codec; import org.opensearch.index.mapper.MapperService; import org.opensearch.knn.index.codec.KNN920Codec.KNN920Codec; +import org.opensearch.knn.index.codec.KNN920Codec.KNN920PerFieldKnnVectorsFormat; import java.util.Optional; @@ -22,7 +23,10 @@ public class KNNCodecFactory { private final MapperService mapperService; public Codec createKNNCodec(final Codec userCodec) { - var codec = KNN920Codec.builder().delegate(userCodec).mapperService(Optional.of(mapperService)).build(); + var codec = KNN920Codec.builder() + .delegate(userCodec) + .knnVectorsFormat(new KNN920PerFieldKnnVectorsFormat(Optional.of(mapperService))) + .build(); return codec; } diff --git a/src/test/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumerTests.java b/src/test/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumerTests.java index 19c57673d..8fa63a2e3 100644 --- a/src/test/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumerTests.java +++ b/src/test/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumerTests.java @@ -45,8 +45,11 @@ import java.util.Map; import java.util.concurrent.ExecutionException; +import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doNothing; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.spy; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -409,4 +412,21 @@ public void testClose() throws IOException { verify(delegate, times(1)).close(); } + public void testAddBinaryField_luceneEngine_noInvocations_addKNNBinary() throws IOException { + var fieldInfo = KNNCodecTestUtil.FieldInfoBuilder.builder("test-field") + .addAttribute(KNNVectorFieldMapper.KNN_FIELD, "true") + .addAttribute(KNNConstants.KNN_ENGINE, KNNEngine.LUCENE.getName()) + .build(); + DocValuesProducer docValuesProducer = mock(DocValuesProducer.class); + + var delegate = mock(DocValuesConsumer.class); + doNothing().when(delegate).addBinaryField(fieldInfo, docValuesProducer); + + var knn80DocValuesConsumer = spy(new KNN80DocValuesConsumer(delegate, null)); + + knn80DocValuesConsumer.addBinaryField(fieldInfo, docValuesProducer); + + verify(delegate, times(1)).addBinaryField(fieldInfo, docValuesProducer); + verify(knn80DocValuesConsumer, never()).addKNNBinaryField(any(), any()); + } } diff --git a/src/test/java/org/opensearch/knn/index/codec/KNN920Codec/KNN920CodecTests.java b/src/test/java/org/opensearch/knn/index/codec/KNN920Codec/KNN920CodecTests.java index ed39eca52..8a3233fba 100644 --- a/src/test/java/org/opensearch/knn/index/codec/KNN920Codec/KNN920CodecTests.java +++ b/src/test/java/org/opensearch/knn/index/codec/KNN920Codec/KNN920CodecTests.java @@ -12,6 +12,8 @@ import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.SerialMergeScheduler; import org.apache.lucene.index.VectorSimilarityFunction; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Query; import org.apache.lucene.store.Directory; import org.apache.lucene.tests.index.RandomIndexWriter; import org.opensearch.index.mapper.MapperService; @@ -21,6 +23,7 @@ import org.opensearch.knn.index.codec.KNNCodecTestCase; import org.opensearch.knn.index.mapper.KNNVectorFieldMapper; import org.opensearch.knn.index.memory.NativeMemoryLoadStrategy; +import org.opensearch.knn.index.query.KNNQueryFactory; import org.opensearch.knn.index.util.KNNEngine; import org.opensearch.watcher.ResourceWatcherService; @@ -29,8 +32,12 @@ import java.util.Optional; import java.util.concurrent.ExecutionException; +import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import static org.opensearch.knn.common.KNNConstants.HNSW_ALGO_EF_CONSTRUCTION; import static org.opensearch.knn.common.KNNConstants.HNSW_ALGO_M; @@ -71,14 +78,13 @@ public void testKnnVectorIndex() throws Exception { when(mapperService.fieldType(eq(fieldName))).thenReturn(mappedFieldType1); when(mapperService.fieldType(eq(field1Name))).thenReturn(mappedFieldType2); + var knnVectorsFormat = spy(new KNN920PerFieldKnnVectorsFormat(Optional.of(mapperService))); + final KNN920Codec actualCodec = KNN920Codec.builder() .delegate(createKNN92DefaultDelegate()) - .mapperService(Optional.of(mapperService)) - .build(); - final KNN920Codec codec = KNN920Codec.builder() - .delegate(createKNN92DefaultDelegate()) - .mapperService(Optional.of(mapperService)) + .knnVectorsFormat(knnVectorsFormat) .build(); + final KNN920Codec codec = KNN920Codec.builder().delegate(createKNN92DefaultDelegate()).knnVectorsFormat(knnVectorsFormat).build(); setUpMockClusterService(); Directory dir = newFSDirectory(createTempDir()); IndexWriterConfig iwc = newIndexWriterConfig(); @@ -96,8 +102,18 @@ public void testKnnVectorIndex() throws Exception { doc.add(vectorField); writer.addDocument(doc); writer.commit(); + IndexReader reader = writer.getReader(); writer.close(); + verify(knnVectorsFormat).getKnnVectorsFormatForField(anyString()); + + IndexSearcher searcher = new IndexSearcher(reader); + Query query = KNNQueryFactory.create(KNNEngine.LUCENE, "dummy", fieldName, new float[] { 1.0f, 0.0f, 0.0f }, 1); + + assertEquals(1, searcher.count(query)); + + reader.close(); + /** * Add doc with field "my_vector" */ @@ -111,12 +127,19 @@ public void testKnnVectorIndex() throws Exception { Document doc1 = new Document(); doc1.add(vectorField1); writer.addDocument(doc1); - IndexReader reader = writer.getReader(); + IndexReader reader1 = writer.getReader(); writer.close(); ResourceWatcherService resourceWatcherService = createDisabledResourceWatcherService(); NativeMemoryLoadStrategy.IndexLoadStrategy.initialize(resourceWatcherService); - reader.close(); + verify(knnVectorsFormat, times(2)).getKnnVectorsFormatForField(anyString()); + + IndexSearcher searcher1 = new IndexSearcher(reader1); + Query query1 = KNNQueryFactory.create(KNNEngine.LUCENE, "dummy", field1Name, new float[] { 1.0f, 0.0f }, 1); + + assertEquals(1, searcher1.count(query1)); + + reader1.close(); dir.close(); resourceWatcherService.close(); NativeMemoryLoadStrategy.IndexLoadStrategy.getInstance().close(); diff --git a/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java b/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java index 3ddd1b73a..af4454e3f 100644 --- a/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java +++ b/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java @@ -154,7 +154,6 @@ public void testBuilder_build_fromLegacy() { } public void testBuilder_parse_fromKnnMethodContext_luceneEngine() throws IOException { - // Check that knnMethodContext is set String fieldName = "test-field-name"; String indexName = "test-index-name"; @@ -192,6 +191,25 @@ public void testBuilder_parse_fromKnnMethodContext_luceneEngine() throws IOExcep builder.knnMethodContext.get().getMethodComponent().getParameters().get(METHOD_PARAMETER_EF_CONSTRUCTION) ); + XContentBuilder xContentBuilderEmptyParams = XContentFactory.jsonBuilder() + .startObject() + .field("type", "knn_vector") + .field("dimension", dimension) + .startObject(KNN_METHOD) + .field(NAME, METHOD_HNSW) + .field(METHOD_PARAMETER_SPACE_TYPE, SpaceType.L2) + .field(KNN_ENGINE, LUCENE_NAME) + .endObject() + .endObject(); + KNNVectorFieldMapper.Builder builderEmptyParams = (KNNVectorFieldMapper.Builder) typeParser.parse( + fieldName, + xContentBuilderToMap(xContentBuilderEmptyParams), + buildParserContext(indexName, settings) + ); + + assertEquals(METHOD_HNSW, builder.knnMethodContext.get().getMethodComponent().getName()); + assertTrue(builderEmptyParams.knnMethodContext.get().getMethodComponent().getParameters().isEmpty()); + XContentBuilder xContentBuilderInvalidSpaceType = XContentFactory.jsonBuilder() .startObject() .field("type", "knn_vector") @@ -239,6 +257,29 @@ public void testBuilder_parse_fromKnnMethodContext_luceneEngine() throws IOExcep () -> builderInvalidDimension.build(new Mapper.BuilderContext(settings, new ContentPath())) ); assertEquals("Dimension value cannot be greater than [1024] but got [2000] for vector [test-field-name]", ex.getMessage()); + + XContentBuilder xContentBuilderUnsupportedParam = XContentFactory.jsonBuilder() + .startObject() + .field("type", "knn_vector") + .field("dimension", dimension) + .startObject(KNN_METHOD) + .field(NAME, METHOD_HNSW) + .field(METHOD_PARAMETER_SPACE_TYPE, SpaceType.L2) + .field(KNN_ENGINE, LUCENE_NAME) + .startObject(PARAMETERS) + .field("RANDOM_PARAM", 0) + .endObject() + .endObject() + .endObject(); + + expectThrows( + ValidationException.class, + () -> typeParser.parse( + fieldName, + xContentBuilderToMap(xContentBuilderUnsupportedParam), + buildParserContext(indexName, settings) + ) + ); } public void testTypeParser_parse_fromKnnMethodContext() throws IOException { From 822eec48380c82bf9ba1264750d86a52861a0d01 Mon Sep 17 00:00:00 2001 From: John Mazanec Date: Thu, 21 Jul 2022 14:23:56 -0700 Subject: [PATCH 016/416] Support doc values for fields using Lucene engine (#457) Builds VectorField for LuceneFieldMapper so that indices using Lucene can still use painless scripting. Signed-off-by: John Mazanec (cherry picked from commit 7a7be09922b362c5e92124de72474e4be01c540e) --- .../knn/index/mapper/LuceneFieldMapper.java | 46 ++++--- .../mapper/KNNVectorFieldMapperTests.java | 119 +++++++++++++++++- .../knn/plugin/script/PainlessScriptIT.java | 76 ++++++++++- 3 files changed, 222 insertions(+), 19 deletions(-) diff --git a/src/main/java/org/opensearch/knn/index/mapper/LuceneFieldMapper.java b/src/main/java/org/opensearch/knn/index/mapper/LuceneFieldMapper.java index 2604a29a8..755c12078 100644 --- a/src/main/java/org/opensearch/knn/index/mapper/LuceneFieldMapper.java +++ b/src/main/java/org/opensearch/knn/index/mapper/LuceneFieldMapper.java @@ -12,18 +12,21 @@ import org.apache.lucene.document.FieldType; import org.apache.lucene.document.KnnVectorField; import org.apache.lucene.document.StoredField; -import org.apache.lucene.index.IndexOptions; +import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.VectorSimilarityFunction; -import org.apache.lucene.index.VectorValues; import org.opensearch.common.Explicit; import org.opensearch.index.mapper.ParseContext; import org.opensearch.knn.index.KNNMethodContext; import org.opensearch.knn.index.SpaceType; +import org.opensearch.knn.index.VectorField; +import org.opensearch.knn.index.util.KNNEngine; import java.io.IOException; import java.util.Map; import java.util.Optional; +import static org.apache.lucene.index.VectorValues.MAX_DIMENSIONS; +import static org.opensearch.knn.common.KNNConstants.KNN_ENGINE; import static org.opensearch.knn.index.SpaceType.COSINESIMIL; import static org.opensearch.knn.index.SpaceType.INNER_PRODUCT; import static org.opensearch.knn.index.SpaceType.L2; @@ -33,7 +36,10 @@ */ public class LuceneFieldMapper extends KNNVectorFieldMapper { - private static final int MAX_DIMENSION = VectorValues.MAX_DIMENSIONS; + private static final int LUCENE_MAX_DIMENSION = MAX_DIMENSIONS; + + /** FieldType used for initializing VectorField, which is used for creating binary doc values. **/ + private final FieldType vectorFieldType; private static final Map SPACE_TYPE_TO_VECTOR_SIMILARITY_FUNCTION = ImmutableMap.of( L2, @@ -56,31 +62,38 @@ public class LuceneFieldMapper extends KNNVectorFieldMapper { ); this.knnMethod = input.getKnnMethodContext(); - - this.fieldType = new FieldType(); - - this.fieldType.setTokenized(false); - this.fieldType.setIndexOptions(IndexOptions.NONE); - final SpaceType spaceType = this.knnMethod.getSpaceType(); final VectorSimilarityFunction vectorSimilarityFunction = Optional.ofNullable( SPACE_TYPE_TO_VECTOR_SIMILARITY_FUNCTION.get(spaceType) ).orElseThrow(() -> new IllegalArgumentException(String.format("Space type [%s] is not supported for Lucene engine", spaceType))); final int dimension = input.getMappedFieldType().getDimension(); - if (dimension > MAX_DIMENSION) { + if (dimension > LUCENE_MAX_DIMENSION) { throw new IllegalArgumentException( String.format( "Dimension value cannot be greater than [%s] but got [%s] for vector [%s]", - MAX_DIMENSION, + LUCENE_MAX_DIMENSION, dimension, input.getName() ) ); } - this.fieldType.setVectorDimensionsAndSimilarityFunction(dimension, vectorSimilarityFunction); - this.fieldType.freeze(); + this.fieldType = KnnVectorField.createFieldType(dimension, vectorSimilarityFunction); + + if (this.hasDocValues) { + this.vectorFieldType = buildDocValuesFieldType(this.knnMethod.getKnnEngine()); + } else { + this.vectorFieldType = null; + } + } + + private static FieldType buildDocValuesFieldType(KNNEngine knnEngine) { + FieldType field = new FieldType(); + field.putAttribute(KNN_ENGINE, knnEngine.getName()); + field.setDocValuesType(DocValuesType.BINARY); + field.freeze(); + return field; } @Override @@ -91,7 +104,7 @@ protected void parseCreateField(ParseContext context, int dimension) throws IOEx Optional arrayOptional = getFloatsFromContext(context, dimension); - if (!arrayOptional.isPresent()) { + if (arrayOptional.isEmpty()) { return; } final float[] array = arrayOptional.get(); @@ -102,6 +115,11 @@ protected void parseCreateField(ParseContext context, int dimension) throws IOEx if (fieldType.stored()) { context.doc().add(new StoredField(name(), point.toString())); } + + if (hasDocValues && vectorFieldType != null) { + context.doc().add(new VectorField(name(), array, vectorFieldType)); + } + context.path().remove(); } diff --git a/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java b/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java index af4454e3f..ecafe2843 100644 --- a/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java +++ b/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java @@ -6,6 +6,13 @@ package org.opensearch.knn.index.mapper; import com.google.common.collect.ImmutableMap; +import org.apache.lucene.document.KnnVectorField; +import org.apache.lucene.index.IndexableField; +import org.apache.lucene.util.BytesRef; +import org.mockito.Mockito; +import org.opensearch.common.Explicit; +import org.opensearch.index.mapper.FieldMapper; +import org.opensearch.index.mapper.ParseContext; import org.opensearch.knn.KNNTestCase; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.ValidationException; @@ -21,6 +28,8 @@ import org.opensearch.knn.index.KNNSettings; import org.opensearch.knn.index.MethodComponentContext; import org.opensearch.knn.index.SpaceType; +import org.opensearch.knn.index.VectorField; +import org.opensearch.knn.index.codec.util.KNNVectorSerializerFactory; import org.opensearch.knn.index.util.KNNEngine; import org.opensearch.knn.indices.ModelDao; import org.opensearch.knn.indices.ModelMetadata; @@ -29,8 +38,14 @@ import java.io.IOException; import java.time.ZoneOffset; import java.time.ZonedDateTime; +import java.util.Arrays; +import java.util.Collections; import java.util.HashSet; +import java.util.List; +import java.util.Optional; +import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.doReturn; import static org.opensearch.knn.common.KNNConstants.KNN_ENGINE; import static org.opensearch.knn.common.KNNConstants.KNN_METHOD; import static org.opensearch.knn.common.KNNConstants.LUCENE_NAME; @@ -47,6 +62,18 @@ public class KNNVectorFieldMapperTests extends KNNTestCase { + private final static String TEST_FIELD_NAME = "test-field-name"; + + private final static int TEST_DIMENSION = 17; + + private final static float TEST_VECTOR_VALUE = 1.5f; + + private final static float[] TEST_VECTOR = createInitializedFloatArray(TEST_DIMENSION, TEST_VECTOR_VALUE); + + private final static BytesRef TEST_VECTOR_BYTES_REF = new BytesRef( + KNNVectorSerializerFactory.getDefaultSerializer().floatToByteArray(TEST_VECTOR) + ); + public void testBuilder_getParameters() { String fieldName = "test-field-name"; ModelDao modelDao = mock(ModelDao.class); @@ -236,7 +263,7 @@ public void testBuilder_parse_fromKnnMethodContext_luceneEngine() throws IOExcep XContentBuilder xContentBuilderInvalidDimension = XContentFactory.jsonBuilder() .startObject() .field("type", "knn_vector") - .field("dimension", 2_000) + .field("dimension", 2000) .startObject(KNN_METHOD) .field(NAME, METHOD_HNSW) .field(METHOD_PARAMETER_SPACE_TYPE, SpaceType.L2) @@ -567,6 +594,96 @@ public void testKNNVectorFieldMapper_merge_fromModel() throws IOException { expectThrows(IllegalArgumentException.class, () -> knnVectorFieldMapper1.merge(knnVectorFieldMapper3)); } + public void testLuceneFieldMapper_parseCreateField_docValues() throws IOException { + // Create a lucene field mapper that creates a binary doc values field as well as KnnVectorField + KNNMethodContext knnMethodContext = new KNNMethodContext( + KNNEngine.LUCENE, + SpaceType.DEFAULT, + new MethodComponentContext(METHOD_HNSW, Collections.emptyMap()) + ); + + KNNVectorFieldMapper.KNNVectorFieldType knnVectorFieldType = new KNNVectorFieldMapper.KNNVectorFieldType( + TEST_FIELD_NAME, + Collections.emptyMap(), + TEST_DIMENSION, + knnMethodContext + ); + + LuceneFieldMapper.CreateLuceneFieldMapperInput.CreateLuceneFieldMapperInputBuilder inputBuilder = + LuceneFieldMapper.CreateLuceneFieldMapperInput.builder() + .name(TEST_FIELD_NAME) + .mappedFieldType(knnVectorFieldType) + .multiFields(FieldMapper.MultiFields.empty()) + .copyTo(FieldMapper.CopyTo.empty()) + .hasDocValues(true) + .ignoreMalformed(new Explicit<>(true, true)) + .knnMethodContext(knnMethodContext); + + ParseContext.Document document = new ParseContext.Document(); + ContentPath contentPath = new ContentPath(); + ParseContext parseContext = mock(ParseContext.class); + when(parseContext.doc()).thenReturn(document); + when(parseContext.path()).thenReturn(contentPath); + + LuceneFieldMapper luceneFieldMapper = Mockito.spy(new LuceneFieldMapper(inputBuilder.build())); + doReturn(Optional.of(TEST_VECTOR)).when(luceneFieldMapper).getFloatsFromContext(parseContext, TEST_DIMENSION); + doNothing().when(luceneFieldMapper).validateIfCircuitBreakerIsNotTriggered(); + doNothing().when(luceneFieldMapper).validateIfKNNPluginEnabled(); + + luceneFieldMapper.parseCreateField(parseContext, TEST_DIMENSION); + + // Document should have 2 fields: one for VectorField (binary doc values) and one for KnnVectorField + List fields = document.getFields(); + assertEquals(2, fields.size()); + IndexableField field1 = fields.get(0); + IndexableField field2 = fields.get(1); + + VectorField vectorField; + KnnVectorField knnVectorField; + if (field1 instanceof VectorField) { + assertTrue(field2 instanceof KnnVectorField); + vectorField = (VectorField) field1; + knnVectorField = (KnnVectorField) field2; + } else { + assertTrue(field1 instanceof KnnVectorField); + assertTrue(field2 instanceof VectorField); + knnVectorField = (KnnVectorField) field1; + vectorField = (VectorField) field2; + } + + assertEquals(TEST_VECTOR_BYTES_REF, vectorField.binaryValue()); + assertArrayEquals(TEST_VECTOR, knnVectorField.vectorValue(), 0.001f); + + // Test when doc values are disabled + document = new ParseContext.Document(); + contentPath = new ContentPath(); + parseContext = mock(ParseContext.class); + when(parseContext.doc()).thenReturn(document); + when(parseContext.path()).thenReturn(contentPath); + + inputBuilder.hasDocValues(false); + luceneFieldMapper = Mockito.spy(new LuceneFieldMapper(inputBuilder.build())); + doReturn(Optional.of(TEST_VECTOR)).when(luceneFieldMapper).getFloatsFromContext(parseContext, TEST_DIMENSION); + doNothing().when(luceneFieldMapper).validateIfCircuitBreakerIsNotTriggered(); + doNothing().when(luceneFieldMapper).validateIfKNNPluginEnabled(); + + luceneFieldMapper.parseCreateField(parseContext, TEST_DIMENSION); + + // Document should have 1 field: one for KnnVectorField + fields = document.getFields(); + assertEquals(1, fields.size()); + IndexableField field = fields.get(0); + assertTrue(field instanceof KnnVectorField); + knnVectorField = (KnnVectorField) field; + assertArrayEquals(TEST_VECTOR, knnVectorField.vectorValue(), 0.001f); + } + + public static float[] createInitializedFloatArray(int dimension, float value) { + float[] array = new float[dimension]; + Arrays.fill(array, value); + return array; + } + public IndexMetadata buildIndexMetaData(String indexName, Settings settings) { return IndexMetadata.builder(indexName) .settings(settings) diff --git a/src/test/java/org/opensearch/knn/plugin/script/PainlessScriptIT.java b/src/test/java/org/opensearch/knn/plugin/script/PainlessScriptIT.java index 0f656e2d1..9dca4112c 100644 --- a/src/test/java/org/opensearch/knn/plugin/script/PainlessScriptIT.java +++ b/src/test/java/org/opensearch/knn/plugin/script/PainlessScriptIT.java @@ -5,8 +5,13 @@ package org.opensearch.knn.plugin.script; +import org.opensearch.common.xcontent.ToXContent; import org.opensearch.knn.KNNRestTestCase; import org.opensearch.knn.KNNResult; +import org.opensearch.knn.common.KNNConstants; +import org.opensearch.knn.index.KNNMethodContext; +import org.opensearch.knn.index.MethodComponentContext; +import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.mapper.KNNVectorFieldMapper; import org.apache.http.util.EntityUtils; import org.opensearch.client.Request; @@ -17,6 +22,7 @@ import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.index.query.MatchAllQueryBuilder; import org.opensearch.index.query.QueryBuilder; +import org.opensearch.knn.index.util.KNNEngine; import org.opensearch.rest.RestStatus; import org.opensearch.script.Script; @@ -28,11 +34,13 @@ import java.util.Map; import java.util.Objects; +import static org.opensearch.knn.common.KNNConstants.METHOD_HNSW; + public class PainlessScriptIT extends KNNRestTestCase { public static final int AGGREGATION_FIELD_NAME_MIN_LENGTH = 2; public static final int AGGREGATION_FIELD_NAME_MAX_LENGTH = 5; - private static String NUMERIC_INDEX_FIELD_NAME = "price"; + private static final String NUMERIC_INDEX_FIELD_NAME = "price"; /** * Utility to create a Index Mapping with multiple fields @@ -45,6 +53,13 @@ protected String createMapping(List properties) throws IOExcept if (property.getDimension() != null) { builder.field("dimension", property.getDimension()); } + + if (property.getKnnMethodContext() != null) { + builder.startObject(KNNConstants.KNN_METHOD); + property.getKnnMethodContext().toXContent(builder, ToXContent.EMPTY_PARAMS); + builder.endObject(); + } + builder.endObject(); } xContentBuilder.endObject().endObject(); @@ -57,6 +72,10 @@ protected String createMapping(List properties) throws IOExcept */ private void buildTestIndex(Map knnDocuments) throws Exception { List properties = buildMappingProperties(); + buildTestIndex(knnDocuments, properties); + } + + private void buildTestIndex(Map knnDocuments, List properties) throws Exception { createKnnIndex(INDEX_NAME, createMapping(properties)); for (Map.Entry data : knnDocuments.entrySet()) { addKnnDoc(INDEX_NAME, data.getKey(), FIELD_NAME, data.getValue()); @@ -140,6 +159,17 @@ private Request buildPainlessScoreScriptRequest(String source, int size, Map documents, + List properties + ) throws Exception { + buildTestIndex(documents, properties); + QueryBuilder qb = new MatchAllQueryBuilder(); + return constructScriptScoreContextSearchRequest(INDEX_NAME, qb, Collections.emptyMap(), Script.DEFAULT_SCRIPT_LANG, source, size); + } + private Request buildPainlessScriptedMetricRequest( String initScriptSource, String mapScriptSource, @@ -506,12 +536,41 @@ public void testScriptedMetricIsSupported() throws Exception { deleteKNNIndex(INDEX_NAME); } - class MappingProperty { + public void testL2ScriptingWithLuceneBackedIndex() throws Exception { + List properties = new ArrayList<>(); + KNNMethodContext knnMethodContext = new KNNMethodContext( + KNNEngine.NMSLIB, + SpaceType.DEFAULT, + new MethodComponentContext(METHOD_HNSW, Collections.emptyMap()) + ); + properties.add( + new MappingProperty(FIELD_NAME, KNNVectorFieldMapper.CONTENT_TYPE).dimension("2").knnMethodContext(knnMethodContext) + ); - private String name; - private String type; + String source = String.format("1/(1 + l2Squared([1.0f, 1.0f], doc['%s']))", FIELD_NAME); + Request request = buildPainlessScoreScriptRequest(source, 3, getL2TestData(), properties); + + Response response = client().performRequest(request); + assertEquals(request.getEndpoint() + ": failed", RestStatus.OK, RestStatus.fromCode(response.getStatusLine().getStatusCode())); + + List results = parseSearchResponse(EntityUtils.toString(response.getEntity()), FIELD_NAME); + assertEquals(3, results.size()); + + String[] expectedDocIDs = { "2", "4", "3", "1" }; + for (int i = 0; i < results.size(); i++) { + assertEquals(expectedDocIDs[i], results.get(i).getDocId()); + } + deleteKNNIndex(INDEX_NAME); + } + + static class MappingProperty { + + private final String name; + private final String type; private String dimension; + private KNNMethodContext knnMethodContext; + MappingProperty(String name, String type) { this.name = name; this.type = type; @@ -522,6 +581,15 @@ MappingProperty dimension(String dimension) { return this; } + MappingProperty knnMethodContext(KNNMethodContext knnMethodContext) { + this.knnMethodContext = knnMethodContext; + return this; + } + + KNNMethodContext getKnnMethodContext() { + return knnMethodContext; + } + String getDimension() { return dimension; } From 4974b1abc476a4762ded9c9f5e7b4e17859c1c66 Mon Sep 17 00:00:00 2001 From: Martin Gaievski Date: Thu, 21 Jul 2022 16:45:44 -0700 Subject: [PATCH 017/416] Refactor validation for max dimensions (#460) Signed-off-by: Martin Gaievski (cherry picked from commit 39a07dde7a25930a1e47694618d32889055d1cdb) --- .../index/mapper/KNNVectorFieldMapper.java | 31 ++++++++++++------- .../opensearch/knn/index/util/KNNEngine.java | 19 ++++++++++++ .../opensearch/knn/indices/ModelMetadata.java | 6 ++-- .../opensearch/knn/index/OpenSearchIT.java | 14 +++++++-- .../mapper/KNNVectorFieldMapperTests.java | 2 +- .../opensearch/knn/indices/ModelTests.java | 4 +-- 6 files changed, 55 insertions(+), 21 deletions(-) diff --git a/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapper.java b/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapper.java index 11cc03b26..ae4b08dd6 100644 --- a/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapper.java +++ b/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapper.java @@ -65,12 +65,6 @@ public abstract class KNNVectorFieldMapper extends ParametrizedFieldMapper { public static final String CONTENT_TYPE = "knn_vector"; public static final String KNN_FIELD = "knn_field"; - /** - * Define the max dimension a knn_vector mapping can have. This limit is somewhat arbitrary. In the future, we - * should make this configurable. - */ - public static final int MAX_DIMENSION = 10000; - private static KNNVectorFieldMapper toType(FieldMapper in) { return (KNNVectorFieldMapper) in; } @@ -89,11 +83,6 @@ public static class Builder extends ParametrizedFieldMapper.Builder { throw new IllegalArgumentException("Dimension cannot be null"); } int value = XContentMapValues.nodeIntegerValue(o); - if (value > MAX_DIMENSION) { - throw new IllegalArgumentException( - String.format("Dimension value cannot be greater than %s for vector: %s", MAX_DIMENSION, name) - ); - } if (value <= 0) { throw new IllegalArgumentException(String.format("Dimension value must be greater than 0 for vector: %s", name)); @@ -197,6 +186,7 @@ public KNNVectorFieldMapper build(BuilderContext context) { // the mappings, setting the index settings will have no impact. final KNNMethodContext knnMethodContext = this.knnMethodContext.getValue(); + validateMaxDimensions(knnMethodContext); final MultiFields multiFieldsBuilder = this.multiFieldsBuilder.build(this, context); final CopyTo copyToBuilder = copyTo.build(); final Explicit ignoreMalformed = ignoreMalformed(context); @@ -281,6 +271,25 @@ public KNNVectorFieldMapper build(BuilderContext context) { efConstruction ); } + + private KNNEngine validateMaxDimensions(final KNNMethodContext knnMethodContext) { + final KNNEngine knnEngine; + if (knnMethodContext != null) { + knnEngine = knnMethodContext.getKnnEngine(); + } else { + knnEngine = KNNEngine.DEFAULT; + } + if (dimension.getValue() > KNNEngine.getMaxDimensionByEngine(knnEngine)) { + throw new IllegalArgumentException( + String.format( + "Dimension value cannot be greater than %s for vector: %s", + KNNEngine.getMaxDimensionByEngine(knnEngine), + name + ) + ); + } + return knnEngine; + } } public static class TypeParser implements Mapper.TypeParser { diff --git a/src/main/java/org/opensearch/knn/index/util/KNNEngine.java b/src/main/java/org/opensearch/knn/index/util/KNNEngine.java index 975da0b34..d87d48c8f 100644 --- a/src/main/java/org/opensearch/knn/index/util/KNNEngine.java +++ b/src/main/java/org/opensearch/knn/index/util/KNNEngine.java @@ -6,6 +6,7 @@ package org.opensearch.knn.index.util; import com.google.common.collect.ImmutableSet; +import org.apache.lucene.index.VectorValues; import org.opensearch.common.ValidationException; import org.opensearch.knn.index.KNNMethod; import org.opensearch.knn.index.KNNMethodContext; @@ -31,6 +32,15 @@ public enum KNNEngine implements KNNLibrary { private static final Set CUSTOM_SEGMENT_FILE_ENGINES = ImmutableSet.of(KNNEngine.NMSLIB, KNNEngine.FAISS); + private static Map MAX_DIMENSIONS_BY_ENGINE = Map.of( + KNNEngine.NMSLIB, + 10_000, + KNNEngine.FAISS, + 10_000, + KNNEngine.LUCENE, + VectorValues.MAX_DIMENSIONS + ); + /** * Constructor for KNNEngine * @@ -94,6 +104,15 @@ public static Set getEnginesThatCreateCustomSegmentFiles() { return CUSTOM_SEGMENT_FILE_ENGINES; } + /** + * Return number of max allowed dimensions per single vector based on the knn engine + * @param knnEngine knn engine to check max dimensions value + * @return + */ + public static int getMaxDimensionByEngine(KNNEngine knnEngine) { + return MAX_DIMENSIONS_BY_ENGINE.getOrDefault(knnEngine, MAX_DIMENSIONS_BY_ENGINE.get(KNNEngine.DEFAULT)); + } + /** * Get the name of the engine * diff --git a/src/main/java/org/opensearch/knn/indices/ModelMetadata.java b/src/main/java/org/opensearch/knn/indices/ModelMetadata.java index 1f0ea786f..f1d1b7c40 100644 --- a/src/main/java/org/opensearch/knn/indices/ModelMetadata.java +++ b/src/main/java/org/opensearch/knn/indices/ModelMetadata.java @@ -34,7 +34,6 @@ import static org.opensearch.knn.common.KNNConstants.MODEL_ERROR; import static org.opensearch.knn.common.KNNConstants.MODEL_STATE; import static org.opensearch.knn.common.KNNConstants.MODEL_TIMESTAMP; -import static org.opensearch.knn.index.mapper.KNNVectorFieldMapper.MAX_DIMENSION; public class ModelMetadata implements Writeable, ToXContentObject { @@ -89,9 +88,10 @@ public ModelMetadata( ) { this.knnEngine = Objects.requireNonNull(knnEngine, "knnEngine must not be null"); this.spaceType = Objects.requireNonNull(spaceType, "spaceType must not be null"); - if (dimension <= 0 || dimension >= MAX_DIMENSION) { + int maxDimensions = KNNEngine.getMaxDimensionByEngine(this.knnEngine); + if (dimension <= 0 || dimension >= maxDimensions) { throw new IllegalArgumentException( - "Dimension \"" + dimension + "\" is invalid. Value must be greater " + "than 0 and less than " + MAX_DIMENSION + String.format("Dimension \"%s\" is invalid. Value must be greater than 0 and less than %d", dimension, maxDimensions) ); } this.dimension = dimension; diff --git a/src/test/java/org/opensearch/knn/index/OpenSearchIT.java b/src/test/java/org/opensearch/knn/index/OpenSearchIT.java index 646285cf3..a5a8437fa 100644 --- a/src/test/java/org/opensearch/knn/index/OpenSearchIT.java +++ b/src/test/java/org/opensearch/knn/index/OpenSearchIT.java @@ -27,7 +27,6 @@ import org.opensearch.index.query.ExistsQueryBuilder; import org.opensearch.knn.TestUtils; import org.opensearch.knn.common.KNNConstants; -import org.opensearch.knn.index.mapper.KNNVectorFieldMapper; import org.opensearch.knn.index.query.KNNQueryBuilder; import org.opensearch.knn.index.util.KNNEngine; import org.opensearch.knn.plugin.script.KNNScoringUtil; @@ -288,11 +287,20 @@ public void testVectorMappingValidation_invalidDimension() { Exception ex = expectThrows( ResponseException.class, - () -> createKnnIndex(INDEX_NAME, settings, createKnnIndexMapping(FIELD_NAME, KNNVectorFieldMapper.MAX_DIMENSION + 1)) + () -> createKnnIndex( + INDEX_NAME, + settings, + createKnnIndexMapping(FIELD_NAME, KNNEngine.getMaxDimensionByEngine(KNNEngine.DEFAULT) + 1) + ) ); assertThat( ex.getMessage(), - containsString("Dimension value cannot be greater than " + KNNVectorFieldMapper.MAX_DIMENSION + " for vector: " + FIELD_NAME) + containsString( + "Dimension value cannot be greater than " + + KNNEngine.getMaxDimensionByEngine(KNNEngine.DEFAULT) + + " for vector: " + + FIELD_NAME + ) ); } diff --git a/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java b/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java index ecafe2843..c5487a0f2 100644 --- a/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java +++ b/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java @@ -283,7 +283,7 @@ public void testBuilder_parse_fromKnnMethodContext_luceneEngine() throws IOExcep IllegalArgumentException.class, () -> builderInvalidDimension.build(new Mapper.BuilderContext(settings, new ContentPath())) ); - assertEquals("Dimension value cannot be greater than [1024] but got [2000] for vector [test-field-name]", ex.getMessage()); + assertEquals("Dimension value cannot be greater than 1024 for vector: test-field-name", ex.getMessage()); XContentBuilder xContentBuilderUnsupportedParam = XContentFactory.jsonBuilder() .startObject() diff --git a/src/test/java/org/opensearch/knn/indices/ModelTests.java b/src/test/java/org/opensearch/knn/indices/ModelTests.java index fb1e807fd..fd3173431 100644 --- a/src/test/java/org/opensearch/knn/indices/ModelTests.java +++ b/src/test/java/org/opensearch/knn/indices/ModelTests.java @@ -21,8 +21,6 @@ import java.util.HashMap; import java.util.Map; -import static org.opensearch.knn.index.mapper.KNNVectorFieldMapper.MAX_DIMENSION; - public class ModelTests extends KNNTestCase { public void testNullConstructor() { @@ -87,7 +85,7 @@ public void testInvalidDimension() { new ModelMetadata( KNNEngine.DEFAULT, SpaceType.DEFAULT, - MAX_DIMENSION + 1, + KNNEngine.getMaxDimensionByEngine(KNNEngine.DEFAULT) + 1, ModelState.CREATED, ZonedDateTime.now(ZoneOffset.UTC).toString(), "", From f27d9225c6ae466f16380f060161cdb04a739504 Mon Sep 17 00:00:00 2001 From: John Mazanec Date: Mon, 25 Jul 2022 16:30:10 -0700 Subject: [PATCH 018/416] Add query integration tests for Lucene engine (#464) Adds query integration tests for the lucene engine. Moves spacetype to lucene vectorsimilarityfunction translation to SpaceType enum for testability. Makes fields in KNNMethodContext non-null. Signed-off-by: John Mazanec (cherry picked from commit 37bdc7430e437332597159123c7b451fc9a4454d) --- .../knn/index/KNNMethodContext.java | 4 + .../org/opensearch/knn/index/SpaceType.java | 26 ++ .../knn/index/mapper/LuceneFieldMapper.java | 20 +- .../opensearch/knn/index/KNNMethodTests.java | 7 +- .../opensearch/knn/index/LuceneEngineIT.java | 258 ++++++++++++++++++ .../org/opensearch/knn/index/LuceneIT.java | 118 -------- .../opensearch/knn/index/SpaceTypeTests.java | 26 ++ .../java/org/opensearch/knn/TestUtils.java | 6 +- 8 files changed, 325 insertions(+), 140 deletions(-) create mode 100644 src/test/java/org/opensearch/knn/index/LuceneEngineIT.java delete mode 100644 src/test/java/org/opensearch/knn/index/LuceneIT.java create mode 100644 src/test/java/org/opensearch/knn/index/SpaceTypeTests.java diff --git a/src/main/java/org/opensearch/knn/index/KNNMethodContext.java b/src/main/java/org/opensearch/knn/index/KNNMethodContext.java index 2d6933877..3fd937155 100644 --- a/src/main/java/org/opensearch/knn/index/KNNMethodContext.java +++ b/src/main/java/org/opensearch/knn/index/KNNMethodContext.java @@ -13,6 +13,7 @@ import lombok.AllArgsConstructor; import lombok.Getter; +import lombok.NonNull; import org.opensearch.common.ValidationException; import org.opensearch.common.io.stream.StreamInput; import org.opensearch.common.io.stream.StreamOutput; @@ -57,8 +58,11 @@ public static synchronized KNNMethodContext getDefault() { return defaultInstance; } + @NonNull private final KNNEngine knnEngine; + @NonNull private final SpaceType spaceType; + @NonNull private final MethodComponentContext methodComponent; /** diff --git a/src/main/java/org/opensearch/knn/index/SpaceType.java b/src/main/java/org/opensearch/knn/index/SpaceType.java index ed96cce03..efa0f1be3 100644 --- a/src/main/java/org/opensearch/knn/index/SpaceType.java +++ b/src/main/java/org/opensearch/knn/index/SpaceType.java @@ -11,6 +11,8 @@ package org.opensearch.knn.index; +import org.apache.lucene.index.VectorSimilarityFunction; + import java.util.HashSet; import java.util.Set; @@ -26,12 +28,22 @@ public enum SpaceType { public float scoreTranslation(float rawScore) { return 1 / (1 + rawScore); } + + @Override + public VectorSimilarityFunction getVectorSimilarityFunction() { + return VectorSimilarityFunction.EUCLIDEAN; + } }, COSINESIMIL("cosinesimil") { @Override public float scoreTranslation(float rawScore) { return 1 / (1 + rawScore); } + + @Override + public VectorSimilarityFunction getVectorSimilarityFunction() { + return VectorSimilarityFunction.COSINE; + } }, L1("l1") { @Override @@ -61,6 +73,11 @@ public float scoreTranslation(float rawScore) { } return -rawScore + 1; } + + @Override + public VectorSimilarityFunction getVectorSimilarityFunction() { + return VectorSimilarityFunction.DOT_PRODUCT; + } }, HAMMING_BIT("hammingbit") { @Override @@ -79,6 +96,15 @@ public float scoreTranslation(float rawScore) { public abstract float scoreTranslation(float rawScore); + /** + * Get VectorSimilarityFunction that maps to this SpaceType + * + * @return VectorSimilarityFunction + */ + public VectorSimilarityFunction getVectorSimilarityFunction() { + throw new UnsupportedOperationException(String.format("Space [%s] does not have a vector similarity function", getValue())); + } + /** * Get space type name in engine * diff --git a/src/main/java/org/opensearch/knn/index/mapper/LuceneFieldMapper.java b/src/main/java/org/opensearch/knn/index/mapper/LuceneFieldMapper.java index 755c12078..5b6c65486 100644 --- a/src/main/java/org/opensearch/knn/index/mapper/LuceneFieldMapper.java +++ b/src/main/java/org/opensearch/knn/index/mapper/LuceneFieldMapper.java @@ -5,7 +5,6 @@ package org.opensearch.knn.index.mapper; -import com.google.common.collect.ImmutableMap; import lombok.AllArgsConstructor; import lombok.Getter; import lombok.NonNull; @@ -17,19 +16,14 @@ import org.opensearch.common.Explicit; import org.opensearch.index.mapper.ParseContext; import org.opensearch.knn.index.KNNMethodContext; -import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.VectorField; import org.opensearch.knn.index.util.KNNEngine; import java.io.IOException; -import java.util.Map; import java.util.Optional; import static org.apache.lucene.index.VectorValues.MAX_DIMENSIONS; import static org.opensearch.knn.common.KNNConstants.KNN_ENGINE; -import static org.opensearch.knn.index.SpaceType.COSINESIMIL; -import static org.opensearch.knn.index.SpaceType.INNER_PRODUCT; -import static org.opensearch.knn.index.SpaceType.L2; /** * Field mapper for case when Lucene has been set as an engine. @@ -41,15 +35,6 @@ public class LuceneFieldMapper extends KNNVectorFieldMapper { /** FieldType used for initializing VectorField, which is used for creating binary doc values. **/ private final FieldType vectorFieldType; - private static final Map SPACE_TYPE_TO_VECTOR_SIMILARITY_FUNCTION = ImmutableMap.of( - L2, - VectorSimilarityFunction.EUCLIDEAN, - COSINESIMIL, - VectorSimilarityFunction.COSINE, - INNER_PRODUCT, - VectorSimilarityFunction.DOT_PRODUCT - ); - LuceneFieldMapper(final CreateLuceneFieldMapperInput input) { super( input.getName(), @@ -62,10 +47,7 @@ public class LuceneFieldMapper extends KNNVectorFieldMapper { ); this.knnMethod = input.getKnnMethodContext(); - final SpaceType spaceType = this.knnMethod.getSpaceType(); - final VectorSimilarityFunction vectorSimilarityFunction = Optional.ofNullable( - SPACE_TYPE_TO_VECTOR_SIMILARITY_FUNCTION.get(spaceType) - ).orElseThrow(() -> new IllegalArgumentException(String.format("Space type [%s] is not supported for Lucene engine", spaceType))); + final VectorSimilarityFunction vectorSimilarityFunction = this.knnMethod.getSpaceType().getVectorSimilarityFunction(); final int dimension = input.getMappedFieldType().getDimension(); if (dimension > LUCENE_MAX_DIMENSION) { diff --git a/src/test/java/org/opensearch/knn/index/KNNMethodTests.java b/src/test/java/org/opensearch/knn/index/KNNMethodTests.java index 18aa90b46..b1dbaab32 100644 --- a/src/test/java/org/opensearch/knn/index/KNNMethodTests.java +++ b/src/test/java/org/opensearch/knn/index/KNNMethodTests.java @@ -98,7 +98,7 @@ public void testGetAsMap() { String methodName = "test-method"; Map generatedMap = ImmutableMap.of("test-key", "test-value"); MethodComponent methodComponent = MethodComponent.Builder.builder(methodName) - .setMapGenerator(((methodComponent1, methodComponentContext) -> generatedMap)) + .setMapGenerator(((methodComponent1, methodComponentContext) -> methodComponentContext.getParameters())) .build(); KNNMethod knnMethod = KNNMethod.Builder.builder(methodComponent).build(); @@ -106,7 +106,10 @@ public void testGetAsMap() { Map expectedMap = new HashMap<>(generatedMap); expectedMap.put(KNNConstants.SPACE_TYPE, spaceType.getValue()); - assertEquals(expectedMap, knnMethod.getAsMap(new KNNMethodContext(KNNEngine.DEFAULT, spaceType, null))); + assertEquals( + expectedMap, + knnMethod.getAsMap(new KNNMethodContext(KNNEngine.DEFAULT, spaceType, new MethodComponentContext(methodName, generatedMap))) + ); } public void testBuilder() { diff --git a/src/test/java/org/opensearch/knn/index/LuceneEngineIT.java b/src/test/java/org/opensearch/knn/index/LuceneEngineIT.java new file mode 100644 index 000000000..a3d619a69 --- /dev/null +++ b/src/test/java/org/opensearch/knn/index/LuceneEngineIT.java @@ -0,0 +1,258 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index; + +import com.google.common.collect.ImmutableList; +import com.google.common.primitives.Floats; +import org.apache.http.util.EntityUtils; +import org.junit.After; +import org.opensearch.client.Response; +import org.opensearch.client.ResponseException; +import org.opensearch.common.Strings; +import org.opensearch.common.xcontent.XContentBuilder; +import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.knn.KNNRestTestCase; +import org.opensearch.knn.KNNResult; +import org.opensearch.knn.TestUtils; +import org.opensearch.knn.common.KNNConstants; +import org.opensearch.knn.index.query.KNNQueryBuilder; +import org.opensearch.knn.index.util.KNNEngine; + +import java.io.IOException; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; +import java.util.stream.Collectors; + +import static org.opensearch.knn.common.KNNConstants.METHOD_HNSW; + +public class LuceneEngineIT extends KNNRestTestCase { + + private static final int DIMENSION = 3; + private static final String DOC_ID = "doc1"; + private static final int EF_CONSTRUCTION = 128; + private static final String INDEX_NAME = "test-index-1"; + private static final String FIELD_NAME = "test-field-1"; + private static final int M = 16; + + private static final Float[][] TEST_INDEX_VECTORS = { { 1.0f, 1.0f, 1.0f }, { 2.0f, 2.0f, 2.0f }, { 3.0f, 3.0f, 3.0f } }; + + private static final float[][] TEST_QUERY_VECTORS = { { 1.0f, 1.0f, 1.0f }, { 2.0f, 2.0f, 2.0f }, { 3.0f, 3.0f, 3.0f } }; + + @After + public final void cleanUp() throws IOException { + deleteKNNIndex(INDEX_NAME); + } + + public void testQuery_l2() throws Exception { + baseQueryTest(SpaceType.L2); + } + + public void testQuery_cosine() throws Exception { + baseQueryTest(SpaceType.COSINESIMIL); + } + + public void testQuery_innerProduct() throws Exception { + baseQueryTest(SpaceType.INNER_PRODUCT); + } + + public void testQuery_invalidVectorDimensionInQuery() throws Exception { + + createKnnIndexMappingWithLuceneEngine(DIMENSION, SpaceType.L2); + for (int j = 0; j < TEST_INDEX_VECTORS.length; j++) { + addKnnDoc(INDEX_NAME, Integer.toString(j + 1), FIELD_NAME, TEST_INDEX_VECTORS[j]); + } + + float[] invalidQuery = new float[DIMENSION - 1]; + int validK = 1; + expectThrows( + ResponseException.class, + () -> searchKNNIndex(INDEX_NAME, new KNNQueryBuilder(FIELD_NAME, invalidQuery, validK), validK) + ); + } + + public void testQuery_documentsMissingField() throws Exception { + + SpaceType spaceType = SpaceType.L2; + + createKnnIndexMappingWithLuceneEngine(DIMENSION, spaceType); + for (int j = 0; j < TEST_INDEX_VECTORS.length; j++) { + addKnnDoc(INDEX_NAME, Integer.toString(j + 1), FIELD_NAME, TEST_INDEX_VECTORS[j]); + } + + // Add a doc without the lucene field set + String secondField = "field-2"; + addDocWithNumericField(INDEX_NAME, Integer.toString(TEST_INDEX_VECTORS.length + 1), secondField, 0L); + + validateQueries(spaceType, FIELD_NAME); + } + + public void testQuery_multipleEngines() throws IOException { + String luceneField = "lucene-field"; + SpaceType luceneSpaceType = SpaceType.COSINESIMIL; + String nmslibField = "nmslib-field"; + SpaceType nmslibSpaceType = SpaceType.L2; + + XContentBuilder builder = XContentFactory.jsonBuilder() + .startObject() + .startObject("properties") + .startObject(luceneField) + .field("type", "knn_vector") + .field("dimension", DIMENSION) + .startObject(KNNConstants.KNN_METHOD) + .field(KNNConstants.NAME, METHOD_HNSW) + .field(KNNConstants.METHOD_PARAMETER_SPACE_TYPE, luceneSpaceType.getValue()) + .field(KNNConstants.KNN_ENGINE, KNNEngine.LUCENE.getName()) + .startObject(KNNConstants.PARAMETERS) + .field(KNNConstants.METHOD_PARAMETER_M, M) + .field(KNNConstants.METHOD_PARAMETER_EF_CONSTRUCTION, EF_CONSTRUCTION) + .endObject() + .endObject() + .endObject() + .startObject(nmslibField) + .field("type", "knn_vector") + .field("dimension", DIMENSION) + .startObject(KNNConstants.KNN_METHOD) + .field(KNNConstants.NAME, METHOD_HNSW) + .field(KNNConstants.METHOD_PARAMETER_SPACE_TYPE, nmslibSpaceType.getValue()) + .field(KNNConstants.KNN_ENGINE, KNNEngine.NMSLIB.getName()) + .startObject(KNNConstants.PARAMETERS) + .field(KNNConstants.METHOD_PARAMETER_M, M) + .field(KNNConstants.METHOD_PARAMETER_EF_CONSTRUCTION, EF_CONSTRUCTION) + .endObject() + .endObject() + .endObject() + .endObject() + .endObject(); + String mapping = Strings.toString(builder); + createKnnIndex(INDEX_NAME, mapping); + + for (int i = 0; i < TEST_INDEX_VECTORS.length; i++) { + addKnnDoc( + INDEX_NAME, + Integer.toString(i + 1), + ImmutableList.of(luceneField, nmslibField), + ImmutableList.of(TEST_INDEX_VECTORS[i], TEST_INDEX_VECTORS[i]) + ); + } + + validateQueries(luceneSpaceType, luceneField); + validateQueries(nmslibSpaceType, nmslibField); + } + + public void testAddDoc() throws IOException { + List mValues = ImmutableList.of(16, 32, 64, 128); + List efConstructionValues = ImmutableList.of(16, 32, 64, 128); + + XContentBuilder builder = XContentFactory.jsonBuilder() + .startObject() + .startObject("properties") + .startObject(FIELD_NAME) + .field("type", "knn_vector") + .field("dimension", DIMENSION) + .startObject(KNNConstants.KNN_METHOD) + .field(KNNConstants.NAME, KNNEngine.LUCENE.getMethod(METHOD_HNSW).getMethodComponent().getName()) + .field(KNNConstants.METHOD_PARAMETER_SPACE_TYPE, SpaceType.L2.getValue()) + .field(KNNConstants.KNN_ENGINE, KNNEngine.LUCENE.getName()) + .startObject(KNNConstants.PARAMETERS) + .field(KNNConstants.METHOD_PARAMETER_M, mValues.get(random().nextInt(mValues.size()))) + .field(KNNConstants.METHOD_PARAMETER_EF_CONSTRUCTION, efConstructionValues.get(random().nextInt(efConstructionValues.size()))) + .endObject() + .endObject() + .endObject() + .endObject() + .endObject(); + + Map mappingMap = xContentBuilderToMap(builder); + String mapping = Strings.toString(builder); + + createKnnIndex(INDEX_NAME, mapping); + assertEquals(new TreeMap<>(mappingMap), new TreeMap<>(getIndexMappingAsMap(INDEX_NAME))); + + Float[] vector = new Float[] { 2.0f, 4.5f, 6.5f }; + addKnnDoc(INDEX_NAME, DOC_ID, FIELD_NAME, vector); + + refreshAllIndices(); + assertEquals(1, getDocCount(INDEX_NAME)); + } + + public void testUpdateDoc() throws Exception { + createKnnIndexMappingWithLuceneEngine(2, SpaceType.L2); + Float[] vector = { 6.0f, 6.0f }; + addKnnDoc(INDEX_NAME, DOC_ID, FIELD_NAME, vector); + + Float[] updatedVector = { 8.0f, 8.0f }; + updateKnnDoc(INDEX_NAME, DOC_ID, FIELD_NAME, updatedVector); + + refreshAllIndices(); + assertEquals(1, getDocCount(INDEX_NAME)); + } + + public void testDeleteDoc() throws Exception { + createKnnIndexMappingWithLuceneEngine(2, SpaceType.L2); + Float[] vector = { 6.0f, 6.0f }; + addKnnDoc(INDEX_NAME, DOC_ID, FIELD_NAME, vector); + + deleteKnnDoc(INDEX_NAME, DOC_ID); + + refreshAllIndices(); + assertEquals(0, getDocCount(INDEX_NAME)); + } + + private void createKnnIndexMappingWithLuceneEngine(int dimension, SpaceType spaceType) throws Exception { + XContentBuilder builder = XContentFactory.jsonBuilder() + .startObject() + .startObject("properties") + .startObject(FIELD_NAME) + .field("type", "knn_vector") + .field("dimension", dimension) + .startObject(KNNConstants.KNN_METHOD) + .field(KNNConstants.NAME, KNNEngine.LUCENE.getMethod(METHOD_HNSW).getMethodComponent().getName()) + .field(KNNConstants.METHOD_PARAMETER_SPACE_TYPE, spaceType.getValue()) + .field(KNNConstants.KNN_ENGINE, KNNEngine.LUCENE.getName()) + .startObject(KNNConstants.PARAMETERS) + .field(KNNConstants.METHOD_PARAMETER_M, M) + .field(KNNConstants.METHOD_PARAMETER_EF_CONSTRUCTION, EF_CONSTRUCTION) + .endObject() + .endObject() + .endObject() + .endObject() + .endObject(); + + String mapping = Strings.toString(builder); + createKnnIndex(INDEX_NAME, mapping); + } + + private void baseQueryTest(SpaceType spaceType) throws Exception { + + createKnnIndexMappingWithLuceneEngine(DIMENSION, spaceType); + for (int j = 0; j < TEST_INDEX_VECTORS.length; j++) { + addKnnDoc(INDEX_NAME, Integer.toString(j + 1), FIELD_NAME, TEST_INDEX_VECTORS[j]); + } + + validateQueries(spaceType, FIELD_NAME); + } + + private void validateQueries(SpaceType spaceType, String fieldName) throws IOException { + + int k = LuceneEngineIT.TEST_INDEX_VECTORS.length; + for (float[] queryVector : TEST_QUERY_VECTORS) { + Response response = searchKNNIndex(INDEX_NAME, new KNNQueryBuilder(fieldName, queryVector, k), k); + String responseBody = EntityUtils.toString(response.getEntity()); + List knnResults = parseSearchResponse(responseBody, fieldName); + assertEquals(k, knnResults.size()); + + List actualScores = parseSearchResponseScore(responseBody, fieldName); + for (int j = 0; j < k; j++) { + float[] primitiveArray = Floats.toArray(Arrays.stream(knnResults.get(j).getVector()).collect(Collectors.toList())); + float distance = TestUtils.computeDistFromSpaceType(spaceType, primitiveArray, queryVector); + float rawScore = spaceType.getVectorSimilarityFunction().convertToScore(distance); + assertEquals(KNNEngine.LUCENE.score(rawScore, spaceType), actualScores.get(j), 0.0001); + } + } + } +} diff --git a/src/test/java/org/opensearch/knn/index/LuceneIT.java b/src/test/java/org/opensearch/knn/index/LuceneIT.java deleted file mode 100644 index 2053bbb89..000000000 --- a/src/test/java/org/opensearch/knn/index/LuceneIT.java +++ /dev/null @@ -1,118 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.knn.index; - -import com.google.common.collect.ImmutableList; -import org.opensearch.common.Strings; -import org.opensearch.common.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentFactory; -import org.opensearch.knn.KNNRestTestCase; -import org.opensearch.knn.common.KNNConstants; -import org.opensearch.knn.index.util.KNNEngine; - -import java.io.IOException; -import java.util.List; -import java.util.Map; -import java.util.TreeMap; - -public class LuceneIT extends KNNRestTestCase { - - private static final int DIMENSION = 3; - private static final String DOC_ID = "doc1"; - private static final int EF_CONSTRUCTION = 128; - private static final String INDEX_NAME = "test-index-1"; - private static final String FIELD_NAME = "test-field-1"; - private static final int M = 16; - - public void test_addDoc() throws IOException { - List mValues = ImmutableList.of(16, 32, 64, 128); - List efConstructionValues = ImmutableList.of(16, 32, 64, 128); - - XContentBuilder builder = XContentFactory.jsonBuilder() - .startObject() - .startObject("properties") - .startObject(FIELD_NAME) - .field("type", "knn_vector") - .field("dimension", DIMENSION) - .startObject(KNNConstants.KNN_METHOD) - .field(KNNConstants.NAME, KNNEngine.LUCENE.getMethod(KNNConstants.METHOD_HNSW).getMethodComponent().getName()) - .field(KNNConstants.METHOD_PARAMETER_SPACE_TYPE, SpaceType.L2.getValue()) - .field(KNNConstants.KNN_ENGINE, KNNEngine.LUCENE.getName()) - .startObject(KNNConstants.PARAMETERS) - .field(KNNConstants.METHOD_PARAMETER_M, mValues.get(random().nextInt(mValues.size()))) - .field(KNNConstants.METHOD_PARAMETER_EF_CONSTRUCTION, efConstructionValues.get(random().nextInt(efConstructionValues.size()))) - .endObject() - .endObject() - .endObject() - .endObject() - .endObject(); - - Map mappingMap = xContentBuilderToMap(builder); - String mapping = Strings.toString(builder); - - createKnnIndex(INDEX_NAME, mapping); - assertEquals(new TreeMap<>(mappingMap), new TreeMap<>(getIndexMappingAsMap(INDEX_NAME))); - - Float[] vector = new Float[] { 2.0f, 4.5f, 6.5f }; - addKnnDoc(INDEX_NAME, DOC_ID, FIELD_NAME, vector); - - refreshAllIndices(); - assertEquals(1, getDocCount(INDEX_NAME)); - - deleteKNNIndex(INDEX_NAME); - } - - public void test_updateDoc() throws Exception { - createKnnIndexMappingWithLuceneEngine(2); - Float[] vector = { 6.0f, 6.0f }; - addKnnDoc(INDEX_NAME, DOC_ID, FIELD_NAME, vector); - - Float[] updatedVector = { 8.0f, 8.0f }; - updateKnnDoc(INDEX_NAME, DOC_ID, FIELD_NAME, updatedVector); - - refreshAllIndices(); - assertEquals(1, getDocCount(INDEX_NAME)); - - deleteKNNIndex(INDEX_NAME); - } - - public void test_deleteDoc() throws Exception { - createKnnIndexMappingWithLuceneEngine(2); - Float[] vector = { 6.0f, 6.0f }; - addKnnDoc(INDEX_NAME, DOC_ID, FIELD_NAME, vector); - - deleteKnnDoc(INDEX_NAME, DOC_ID); - - refreshAllIndices(); - assertEquals(0, getDocCount(INDEX_NAME)); - - deleteKNNIndex(INDEX_NAME); - } - - private void createKnnIndexMappingWithLuceneEngine(int dimension) throws Exception { - XContentBuilder builder = XContentFactory.jsonBuilder() - .startObject() - .startObject("properties") - .startObject(FIELD_NAME) - .field("type", "knn_vector") - .field("dimension", dimension) - .startObject(KNNConstants.KNN_METHOD) - .field(KNNConstants.NAME, KNNEngine.LUCENE.getMethod(KNNConstants.METHOD_HNSW).getMethodComponent().getName()) - .field(KNNConstants.METHOD_PARAMETER_SPACE_TYPE, SpaceType.L2.getValue()) - .field(KNNConstants.KNN_ENGINE, KNNEngine.LUCENE.getName()) - .startObject(KNNConstants.PARAMETERS) - .field(KNNConstants.METHOD_PARAMETER_M, M) - .field(KNNConstants.METHOD_PARAMETER_EF_CONSTRUCTION, EF_CONSTRUCTION) - .endObject() - .endObject() - .endObject() - .endObject() - .endObject(); - - String mapping = Strings.toString(builder); - createKnnIndex(INDEX_NAME, mapping); - } -} diff --git a/src/test/java/org/opensearch/knn/index/SpaceTypeTests.java b/src/test/java/org/opensearch/knn/index/SpaceTypeTests.java new file mode 100644 index 000000000..5df9ec6f5 --- /dev/null +++ b/src/test/java/org/opensearch/knn/index/SpaceTypeTests.java @@ -0,0 +1,26 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.knn.index; + +import org.apache.lucene.index.VectorSimilarityFunction; +import org.opensearch.knn.KNNTestCase; + +public class SpaceTypeTests extends KNNTestCase { + + public void testGetVectorSimilarityFunction_l2() { + assertEquals(VectorSimilarityFunction.EUCLIDEAN, SpaceType.L2.getVectorSimilarityFunction()); + } + + public void testGetVectorSimilarityFunction_invalid() { + expectThrows(UnsupportedOperationException.class, SpaceType.L1::getVectorSimilarityFunction); + } +} diff --git a/src/testFixtures/java/org/opensearch/knn/TestUtils.java b/src/testFixtures/java/org/opensearch/knn/TestUtils.java index 56fdde720..6a6e28477 100644 --- a/src/testFixtures/java/org/opensearch/knn/TestUtils.java +++ b/src/testFixtures/java/org/opensearch/knn/TestUtils.java @@ -66,7 +66,11 @@ public class TestUtils { SpaceType.L2, KNNScoringUtil::l2Squared, SpaceType.LINF, - KNNScoringUtil::lInfNorm + KNNScoringUtil::lInfNorm, + SpaceType.COSINESIMIL, + KNNScoringUtil::cosinesimil, + SpaceType.INNER_PRODUCT, + KNNScoringUtil::innerProduct ); public static final String KNN_BWC_PREFIX = "knn-bwc-"; From 9f118f2836a74b036404fb49bcbef4e5923b3cc9 Mon Sep 17 00:00:00 2001 From: Martin Gaievski Date: Mon, 1 Aug 2022 13:36:19 -0700 Subject: [PATCH 019/416] Change validation error to invalid input for dimension value (#483) Signed-off-by: Martin Gaievski (cherry picked from commit 8624725bfee2d7b848643015570819d03bd5c38c) --- .../index/mapper/KNNVectorFieldMapper.java | 10 ++- .../mapper/KNNVectorFieldMapperTests.java | 62 +++++++++++++++---- 2 files changed, 59 insertions(+), 13 deletions(-) diff --git a/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapper.java b/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapper.java index ae4b08dd6..96fea1fbc 100644 --- a/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapper.java +++ b/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapper.java @@ -82,8 +82,14 @@ public static class Builder extends ParametrizedFieldMapper.Builder { if (o == null) { throw new IllegalArgumentException("Dimension cannot be null"); } - int value = XContentMapValues.nodeIntegerValue(o); - + int value; + try { + value = XContentMapValues.nodeIntegerValue(o); + } catch (Exception exception) { + throw new IllegalArgumentException( + String.format("Unable to parse [dimension] from provided value [%s] for vector [%s]", o, name) + ); + } if (value <= 0) { throw new IllegalArgumentException(String.format("Dimension value must be greater than 0 for vector: %s", name)); } diff --git a/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java b/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java index c5487a0f2..4f548e76c 100644 --- a/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java +++ b/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java @@ -55,6 +55,7 @@ import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_SPACE_TYPE; import static org.opensearch.knn.common.KNNConstants.MODEL_ID; import static org.opensearch.knn.common.KNNConstants.NAME; +import static org.opensearch.knn.common.KNNConstants.NMSLIB_NAME; import static org.opensearch.knn.common.KNNConstants.PARAMETERS; import static org.opensearch.Version.CURRENT; import static org.mockito.Mockito.mock; @@ -260,7 +261,42 @@ public void testBuilder_parse_fromKnnMethodContext_luceneEngine() throws IOExcep ) ); - XContentBuilder xContentBuilderInvalidDimension = XContentFactory.jsonBuilder() + XContentBuilder xContentBuilderUnsupportedParam = XContentFactory.jsonBuilder() + .startObject() + .field("type", "knn_vector") + .field("dimension", dimension) + .startObject(KNN_METHOD) + .field(NAME, METHOD_HNSW) + .field(METHOD_PARAMETER_SPACE_TYPE, SpaceType.L2) + .field(KNN_ENGINE, LUCENE_NAME) + .startObject(PARAMETERS) + .field("RANDOM_PARAM", 0) + .endObject() + .endObject() + .endObject(); + + expectThrows( + ValidationException.class, + () -> typeParser.parse( + fieldName, + xContentBuilderToMap(xContentBuilderUnsupportedParam), + buildParserContext(indexName, settings) + ) + ); + } + + public void testTypeParser_parse_fromKnnMethodContext_invalidDimension() throws IOException { + String fieldName = "test-field-name"; + String indexName = "test-index-name"; + + Settings settings = Settings.builder().put(settings(CURRENT).build()).build(); + + ModelDao modelDao = mock(ModelDao.class); + KNNVectorFieldMapper.TypeParser typeParser = new KNNVectorFieldMapper.TypeParser(() -> modelDao); + + int efConstruction = 321; + + XContentBuilder xContentBuilderOverMaxDimension = XContentFactory.jsonBuilder() .startObject() .field("type", "knn_vector") .field("dimension", 2000) @@ -273,40 +309,44 @@ public void testBuilder_parse_fromKnnMethodContext_luceneEngine() throws IOExcep .endObject() .endObject() .endObject(); - KNNVectorFieldMapper.Builder builderInvalidDimension = (KNNVectorFieldMapper.Builder) typeParser.parse( + KNNVectorFieldMapper.Builder builderOverMaxDimension = (KNNVectorFieldMapper.Builder) typeParser.parse( fieldName, - xContentBuilderToMap(xContentBuilderInvalidDimension), + xContentBuilderToMap(xContentBuilderOverMaxDimension), buildParserContext(indexName, settings) ); IllegalArgumentException ex = expectThrows( IllegalArgumentException.class, - () -> builderInvalidDimension.build(new Mapper.BuilderContext(settings, new ContentPath())) + () -> builderOverMaxDimension.build(new Mapper.BuilderContext(settings, new ContentPath())) ); assertEquals("Dimension value cannot be greater than 1024 for vector: test-field-name", ex.getMessage()); - XContentBuilder xContentBuilderUnsupportedParam = XContentFactory.jsonBuilder() + XContentBuilder xContentBuilderInvalidDimension = XContentFactory.jsonBuilder() .startObject() .field("type", "knn_vector") - .field("dimension", dimension) + .field("dimension", "2147483648") .startObject(KNN_METHOD) .field(NAME, METHOD_HNSW) .field(METHOD_PARAMETER_SPACE_TYPE, SpaceType.L2) - .field(KNN_ENGINE, LUCENE_NAME) + .field(KNN_ENGINE, NMSLIB_NAME) .startObject(PARAMETERS) - .field("RANDOM_PARAM", 0) + .field(METHOD_PARAMETER_EF_CONSTRUCTION, efConstruction) .endObject() .endObject() .endObject(); - expectThrows( - ValidationException.class, + IllegalArgumentException exInvalidDimension = expectThrows( + IllegalArgumentException.class, () -> typeParser.parse( fieldName, - xContentBuilderToMap(xContentBuilderUnsupportedParam), + xContentBuilderToMap(xContentBuilderInvalidDimension), buildParserContext(indexName, settings) ) ); + assertEquals( + "Unable to parse [dimension] from provided value [2147483648] for vector [test-field-name]", + exInvalidDimension.getMessage() + ); } public void testTypeParser_parse_fromKnnMethodContext() throws IOException { From aebff805b0d47c9297d59741422d82e7dc0c8087 Mon Sep 17 00:00:00 2001 From: Martin Gaievski Date: Tue, 2 Aug 2022 17:02:23 -0700 Subject: [PATCH 020/416] Fixing dependency on Lucene VectorSimilarity class Signed-off-by: Martin Gaievski --- .../org/opensearch/knn/index/LuceneEngineIT.java | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/test/java/org/opensearch/knn/index/LuceneEngineIT.java b/src/test/java/org/opensearch/knn/index/LuceneEngineIT.java index a3d619a69..94e126ad9 100644 --- a/src/test/java/org/opensearch/knn/index/LuceneEngineIT.java +++ b/src/test/java/org/opensearch/knn/index/LuceneEngineIT.java @@ -6,8 +6,10 @@ package org.opensearch.knn.index; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import com.google.common.primitives.Floats; import org.apache.http.util.EntityUtils; +import org.apache.lucene.index.VectorSimilarityFunction; import org.junit.After; import org.opensearch.client.Response; import org.opensearch.client.ResponseException; @@ -26,6 +28,7 @@ import java.util.List; import java.util.Map; import java.util.TreeMap; +import java.util.function.Function; import java.util.stream.Collectors; import static org.opensearch.knn.common.KNNConstants.METHOD_HNSW; @@ -43,6 +46,15 @@ public class LuceneEngineIT extends KNNRestTestCase { private static final float[][] TEST_QUERY_VECTORS = { { 1.0f, 1.0f, 1.0f }, { 2.0f, 2.0f, 2.0f }, { 3.0f, 3.0f, 3.0f } }; + private static final Map> VECTOR_SIMILARITY_TO_SCORE = ImmutableMap.of( + VectorSimilarityFunction.EUCLIDEAN, + (similarity) -> 1 / (1 + similarity), + VectorSimilarityFunction.DOT_PRODUCT, + (similarity) -> (1 + similarity) / 2, + VectorSimilarityFunction.COSINE, + (similarity) -> (1 + similarity) / 2 + ); + @After public final void cleanUp() throws IOException { deleteKNNIndex(INDEX_NAME); @@ -250,7 +262,7 @@ private void validateQueries(SpaceType spaceType, String fieldName) throws IOExc for (int j = 0; j < k; j++) { float[] primitiveArray = Floats.toArray(Arrays.stream(knnResults.get(j).getVector()).collect(Collectors.toList())); float distance = TestUtils.computeDistFromSpaceType(spaceType, primitiveArray, queryVector); - float rawScore = spaceType.getVectorSimilarityFunction().convertToScore(distance); + float rawScore = VECTOR_SIMILARITY_TO_SCORE.get(spaceType.getVectorSimilarityFunction()).apply(distance); assertEquals(KNNEngine.LUCENE.score(rawScore, spaceType), actualScores.get(j), 0.0001); } } From 2c07d981e002f2a0c200a28636809daaeeb52179 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Wed, 3 Aug 2022 15:12:36 -0700 Subject: [PATCH 021/416] Make innerproduct similarity not supported (#488) (#489) * Make innerproduct similarity as not supported Signed-off-by: Martin Gaievski (cherry picked from commit b89e6ed9791cab75afc64b61433fef5d2e436f74) Co-authored-by: Martin Gaievski --- .../org/opensearch/knn/index/util/Lucene.java | 2 +- .../opensearch/knn/index/LuceneEngineIT.java | 57 ++++++-- .../mapper/KNNVectorFieldMapperTests.java | 138 +++++++++++------- 3 files changed, 131 insertions(+), 66 deletions(-) diff --git a/src/main/java/org/opensearch/knn/index/util/Lucene.java b/src/main/java/org/opensearch/knn/index/util/Lucene.java index a5f20195a..df1353106 100644 --- a/src/main/java/org/opensearch/knn/index/util/Lucene.java +++ b/src/main/java/org/opensearch/knn/index/util/Lucene.java @@ -41,7 +41,7 @@ public class Lucene extends JVMLibrary { ) ) .build() - ).addSpaces(SpaceType.L2, SpaceType.COSINESIMIL, SpaceType.INNER_PRODUCT).build() + ).addSpaces(SpaceType.L2, SpaceType.COSINESIMIL).build() ); final static Lucene INSTANCE = new Lucene(METHODS, Version.LUCENE_9_2_0.toString()); diff --git a/src/test/java/org/opensearch/knn/index/LuceneEngineIT.java b/src/test/java/org/opensearch/knn/index/LuceneEngineIT.java index 94e126ad9..9fe42cb9f 100644 --- a/src/test/java/org/opensearch/knn/index/LuceneEngineIT.java +++ b/src/test/java/org/opensearch/knn/index/LuceneEngineIT.java @@ -11,6 +11,7 @@ import org.apache.http.util.EntityUtils; import org.apache.lucene.index.VectorSimilarityFunction; import org.junit.After; +import org.opensearch.client.Request; import org.opensearch.client.Response; import org.opensearch.client.ResponseException; import org.opensearch.common.Strings; @@ -54,6 +55,10 @@ public class LuceneEngineIT extends KNNRestTestCase { VectorSimilarityFunction.COSINE, (similarity) -> (1 + similarity) / 2 ); + private static final String DIMENSION_FIELD_NAME = "dimension"; + private static final String KNN_VECTOR_TYPE = "knn_vector"; + private static final String PROPERTIES_FIELD_NAME = "properties"; + private static final String TYPE_FIELD_NAME = "type"; @After public final void cleanUp() throws IOException { @@ -68,8 +73,34 @@ public void testQuery_cosine() throws Exception { baseQueryTest(SpaceType.COSINESIMIL); } - public void testQuery_innerProduct() throws Exception { - baseQueryTest(SpaceType.INNER_PRODUCT); + public void testQuery_innerProduct_notSupported() throws Exception { + XContentBuilder builder = XContentFactory.jsonBuilder() + .startObject() + .startObject(PROPERTIES_FIELD_NAME) + .startObject(FIELD_NAME) + .field(TYPE_FIELD_NAME, KNN_VECTOR_TYPE) + .field(DIMENSION_FIELD_NAME, DIMENSION) + .startObject(KNNConstants.KNN_METHOD) + .field(KNNConstants.NAME, KNNEngine.LUCENE.getMethod(METHOD_HNSW).getMethodComponent().getName()) + .field(KNNConstants.METHOD_PARAMETER_SPACE_TYPE, SpaceType.INNER_PRODUCT.getValue()) + .field(KNNConstants.KNN_ENGINE, KNNEngine.LUCENE.getName()) + .startObject(KNNConstants.PARAMETERS) + .field(KNNConstants.METHOD_PARAMETER_M, M) + .field(KNNConstants.METHOD_PARAMETER_EF_CONSTRUCTION, EF_CONSTRUCTION) + .endObject() + .endObject() + .endObject() + .endObject() + .endObject(); + + String mapping = Strings.toString(builder); + + createIndex(INDEX_NAME, getKNNDefaultIndexSettings()); + + Request request = new Request("PUT", "/" + INDEX_NAME + "/_mapping"); + request.setJsonEntity(mapping); + + expectThrows(ResponseException.class, () -> client().performRequest(request)); } public void testQuery_invalidVectorDimensionInQuery() throws Exception { @@ -111,10 +142,10 @@ public void testQuery_multipleEngines() throws IOException { XContentBuilder builder = XContentFactory.jsonBuilder() .startObject() - .startObject("properties") + .startObject(PROPERTIES_FIELD_NAME) .startObject(luceneField) - .field("type", "knn_vector") - .field("dimension", DIMENSION) + .field(TYPE_FIELD_NAME, KNN_VECTOR_TYPE) + .field(DIMENSION_FIELD_NAME, DIMENSION) .startObject(KNNConstants.KNN_METHOD) .field(KNNConstants.NAME, METHOD_HNSW) .field(KNNConstants.METHOD_PARAMETER_SPACE_TYPE, luceneSpaceType.getValue()) @@ -126,8 +157,8 @@ public void testQuery_multipleEngines() throws IOException { .endObject() .endObject() .startObject(nmslibField) - .field("type", "knn_vector") - .field("dimension", DIMENSION) + .field(TYPE_FIELD_NAME, KNN_VECTOR_TYPE) + .field(DIMENSION_FIELD_NAME, DIMENSION) .startObject(KNNConstants.KNN_METHOD) .field(KNNConstants.NAME, METHOD_HNSW) .field(KNNConstants.METHOD_PARAMETER_SPACE_TYPE, nmslibSpaceType.getValue()) @@ -162,10 +193,10 @@ public void testAddDoc() throws IOException { XContentBuilder builder = XContentFactory.jsonBuilder() .startObject() - .startObject("properties") + .startObject(PROPERTIES_FIELD_NAME) .startObject(FIELD_NAME) - .field("type", "knn_vector") - .field("dimension", DIMENSION) + .field(TYPE_FIELD_NAME, KNN_VECTOR_TYPE) + .field(DIMENSION_FIELD_NAME, DIMENSION) .startObject(KNNConstants.KNN_METHOD) .field(KNNConstants.NAME, KNNEngine.LUCENE.getMethod(METHOD_HNSW).getMethodComponent().getName()) .field(KNNConstants.METHOD_PARAMETER_SPACE_TYPE, SpaceType.L2.getValue()) @@ -218,10 +249,10 @@ public void testDeleteDoc() throws Exception { private void createKnnIndexMappingWithLuceneEngine(int dimension, SpaceType spaceType) throws Exception { XContentBuilder builder = XContentFactory.jsonBuilder() .startObject() - .startObject("properties") + .startObject(PROPERTIES_FIELD_NAME) .startObject(FIELD_NAME) - .field("type", "knn_vector") - .field("dimension", dimension) + .field(TYPE_FIELD_NAME, KNN_VECTOR_TYPE) + .field(DIMENSION_FIELD_NAME, dimension) .startObject(KNNConstants.KNN_METHOD) .field(KNNConstants.NAME, KNNEngine.LUCENE.getMethod(METHOD_HNSW).getMethodComponent().getName()) .field(KNNConstants.METHOD_PARAMETER_SPACE_TYPE, spaceType.getValue()) diff --git a/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java b/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java index 4f548e76c..13e2ab9a3 100644 --- a/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java +++ b/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java @@ -74,6 +74,9 @@ public class KNNVectorFieldMapperTests extends KNNTestCase { private final static BytesRef TEST_VECTOR_BYTES_REF = new BytesRef( KNNVectorSerializerFactory.getDefaultSerializer().floatToByteArray(TEST_VECTOR) ); + private static final String DIMENSION_FIELD_NAME = "dimension"; + private static final String KNN_VECTOR_TYPE = "knn_vector"; + private static final String TYPE_FIELD_NAME = "type"; public void testBuilder_getParameters() { String fieldName = "test-field-name"; @@ -195,8 +198,8 @@ public void testBuilder_parse_fromKnnMethodContext_luceneEngine() throws IOExcep int dimension = 133; XContentBuilder xContentBuilder = XContentFactory.jsonBuilder() .startObject() - .field("type", "knn_vector") - .field("dimension", dimension) + .field(TYPE_FIELD_NAME, KNN_VECTOR_TYPE) + .field(DIMENSION_FIELD_NAME, dimension) .startObject(KNN_METHOD) .field(NAME, METHOD_HNSW) .field(METHOD_PARAMETER_SPACE_TYPE, SpaceType.L2) @@ -221,8 +224,8 @@ public void testBuilder_parse_fromKnnMethodContext_luceneEngine() throws IOExcep XContentBuilder xContentBuilderEmptyParams = XContentFactory.jsonBuilder() .startObject() - .field("type", "knn_vector") - .field("dimension", dimension) + .field(TYPE_FIELD_NAME, KNN_VECTOR_TYPE) + .field(DIMENSION_FIELD_NAME, dimension) .startObject(KNN_METHOD) .field(NAME, METHOD_HNSW) .field(METHOD_PARAMETER_SPACE_TYPE, SpaceType.L2) @@ -238,33 +241,10 @@ public void testBuilder_parse_fromKnnMethodContext_luceneEngine() throws IOExcep assertEquals(METHOD_HNSW, builder.knnMethodContext.get().getMethodComponent().getName()); assertTrue(builderEmptyParams.knnMethodContext.get().getMethodComponent().getParameters().isEmpty()); - XContentBuilder xContentBuilderInvalidSpaceType = XContentFactory.jsonBuilder() - .startObject() - .field("type", "knn_vector") - .field("dimension", dimension) - .startObject(KNN_METHOD) - .field(NAME, METHOD_HNSW) - .field(METHOD_PARAMETER_SPACE_TYPE, SpaceType.L1) - .field(KNN_ENGINE, LUCENE_NAME) - .startObject(PARAMETERS) - .field(METHOD_PARAMETER_EF_CONSTRUCTION, efConstruction) - .endObject() - .endObject() - .endObject(); - - expectThrows( - ValidationException.class, - () -> typeParser.parse( - fieldName, - xContentBuilderToMap(xContentBuilderInvalidSpaceType), - buildParserContext(indexName, settings) - ) - ); - XContentBuilder xContentBuilderUnsupportedParam = XContentFactory.jsonBuilder() .startObject() - .field("type", "knn_vector") - .field("dimension", dimension) + .field(TYPE_FIELD_NAME, KNN_VECTOR_TYPE) + .field(DIMENSION_FIELD_NAME, dimension) .startObject(KNN_METHOD) .field(NAME, METHOD_HNSW) .field(METHOD_PARAMETER_SPACE_TYPE, SpaceType.L2) @@ -298,8 +278,8 @@ public void testTypeParser_parse_fromKnnMethodContext_invalidDimension() throws XContentBuilder xContentBuilderOverMaxDimension = XContentFactory.jsonBuilder() .startObject() - .field("type", "knn_vector") - .field("dimension", 2000) + .field(TYPE_FIELD_NAME, KNN_VECTOR_TYPE) + .field(DIMENSION_FIELD_NAME, 2000) .startObject(KNN_METHOD) .field(NAME, METHOD_HNSW) .field(METHOD_PARAMETER_SPACE_TYPE, SpaceType.L2) @@ -323,8 +303,8 @@ public void testTypeParser_parse_fromKnnMethodContext_invalidDimension() throws XContentBuilder xContentBuilderInvalidDimension = XContentFactory.jsonBuilder() .startObject() - .field("type", "knn_vector") - .field("dimension", "2147483648") + .field(TYPE_FIELD_NAME, KNN_VECTOR_TYPE) + .field(DIMENSION_FIELD_NAME, "2147483648") .startObject(KNN_METHOD) .field(NAME, METHOD_HNSW) .field(METHOD_PARAMETER_SPACE_TYPE, SpaceType.L2) @@ -349,6 +329,60 @@ public void testTypeParser_parse_fromKnnMethodContext_invalidDimension() throws ); } + public void testTypeParser_parse_fromKnnMethodContext_invalidSpaceType() throws IOException { + String fieldName = "test-field-name"; + String indexName = "test-index-name"; + + Settings settings = Settings.builder().put(settings(CURRENT).build()).build(); + + ModelDao modelDao = mock(ModelDao.class); + KNNVectorFieldMapper.TypeParser typeParser = new KNNVectorFieldMapper.TypeParser(() -> modelDao); + + int efConstruction = 321; + int dimension = 133; + XContentBuilder xContentBuilderL1SpaceType = XContentFactory.jsonBuilder() + .startObject() + .field(TYPE_FIELD_NAME, KNN_VECTOR_TYPE) + .field(DIMENSION_FIELD_NAME, dimension) + .startObject(KNN_METHOD) + .field(NAME, METHOD_HNSW) + .field(METHOD_PARAMETER_SPACE_TYPE, SpaceType.L1.getValue()) + .field(KNN_ENGINE, LUCENE_NAME) + .startObject(PARAMETERS) + .field(METHOD_PARAMETER_EF_CONSTRUCTION, efConstruction) + .endObject() + .endObject() + .endObject(); + + expectThrows( + ValidationException.class, + () -> typeParser.parse(fieldName, xContentBuilderToMap(xContentBuilderL1SpaceType), buildParserContext(indexName, settings)) + ); + + XContentBuilder xContentBuilderInnerproductSpaceType = XContentFactory.jsonBuilder() + .startObject() + .field(TYPE_FIELD_NAME, KNN_VECTOR_TYPE) + .field(DIMENSION_FIELD_NAME, dimension) + .startObject(KNN_METHOD) + .field(NAME, METHOD_HNSW) + .field(METHOD_PARAMETER_SPACE_TYPE, SpaceType.INNER_PRODUCT.getValue()) + .field(KNN_ENGINE, LUCENE_NAME) + .startObject(PARAMETERS) + .field(METHOD_PARAMETER_EF_CONSTRUCTION, efConstruction) + .endObject() + .endObject() + .endObject(); + + expectThrows( + ValidationException.class, + () -> typeParser.parse( + fieldName, + xContentBuilderToMap(xContentBuilderInnerproductSpaceType), + buildParserContext(indexName, settings) + ) + ); + } + public void testTypeParser_parse_fromKnnMethodContext() throws IOException { // Check that knnMethodContext is set String fieldName = "test-field-name"; @@ -363,8 +397,8 @@ public void testTypeParser_parse_fromKnnMethodContext() throws IOException { int dimension = 133; XContentBuilder xContentBuilder = XContentFactory.jsonBuilder() .startObject() - .field("type", "knn_vector") - .field("dimension", dimension) + .field(TYPE_FIELD_NAME, KNN_VECTOR_TYPE) + .field(DIMENSION_FIELD_NAME, dimension) .startObject(KNN_METHOD) .field(NAME, METHOD_HNSW) .startObject(PARAMETERS) @@ -388,8 +422,8 @@ public void testTypeParser_parse_fromKnnMethodContext() throws IOException { // Test invalid parameter XContentBuilder xContentBuilder2 = XContentFactory.jsonBuilder() .startObject() - .field("type", "knn_vector") - .field("dimension", dimension) + .field(TYPE_FIELD_NAME, KNN_VECTOR_TYPE) + .field(DIMENSION_FIELD_NAME, dimension) .startObject(KNN_METHOD) .field(NAME, METHOD_HNSW) .startObject(PARAMETERS) @@ -406,8 +440,8 @@ public void testTypeParser_parse_fromKnnMethodContext() throws IOException { // Test invalid method XContentBuilder xContentBuilder3 = XContentFactory.jsonBuilder() .startObject() - .field("type", "knn_vector") - .field("dimension", dimension) + .field(TYPE_FIELD_NAME, KNN_VECTOR_TYPE) + .field(DIMENSION_FIELD_NAME, dimension) .startObject(KNN_METHOD) .field(NAME, "invalid") .endObject() @@ -419,7 +453,7 @@ public void testTypeParser_parse_fromKnnMethodContext() throws IOException { ); // Test missing required parameter: dimension - XContentBuilder xContentBuilder4 = XContentFactory.jsonBuilder().startObject().field("type", "knn_vector").endObject(); + XContentBuilder xContentBuilder4 = XContentFactory.jsonBuilder().startObject().field(TYPE_FIELD_NAME, KNN_VECTOR_TYPE).endObject(); expectThrows( IllegalArgumentException.class, @@ -429,8 +463,8 @@ public void testTypeParser_parse_fromKnnMethodContext() throws IOException { // Check that this fails if model id is also set XContentBuilder xContentBuilder5 = XContentFactory.jsonBuilder() .startObject() - .field("type", "knn_vector") - .field("dimension", dimension) + .field(TYPE_FIELD_NAME, KNN_VECTOR_TYPE) + .field(DIMENSION_FIELD_NAME, dimension) .field(MODEL_ID, "test-id") .startObject(KNN_METHOD) .field(NAME, METHOD_HNSW) @@ -459,7 +493,7 @@ public void testTypeParser_parse_fromModel() throws IOException { String modelId = "test-id"; XContentBuilder xContentBuilder = XContentFactory.jsonBuilder() .startObject() - .field("type", "knn_vector") + .field(TYPE_FIELD_NAME, KNN_VECTOR_TYPE) .field(MODEL_ID, modelId) .endObject(); @@ -493,8 +527,8 @@ public void testTypeParser_parse_fromLegacy() throws IOException { int dimension = 122; XContentBuilder xContentBuilder = XContentFactory.jsonBuilder() .startObject() - .field("type", "knn_vector") - .field("dimension", dimension) + .field(TYPE_FIELD_NAME, KNN_VECTOR_TYPE) + .field(DIMENSION_FIELD_NAME, dimension) .endObject(); KNNVectorFieldMapper.Builder builder = (KNNVectorFieldMapper.Builder) typeParser.parse( @@ -520,8 +554,8 @@ public void testKNNVectorFieldMapper_merge_fromKnnMethodContext() throws IOExcep int efConstruction = 321; XContentBuilder xContentBuilder = XContentFactory.jsonBuilder() .startObject() - .field("type", "knn_vector") - .field("dimension", dimension) + .field(TYPE_FIELD_NAME, KNN_VECTOR_TYPE) + .field(DIMENSION_FIELD_NAME, dimension) .startObject(KNN_METHOD) .field(NAME, METHOD_HNSW) .startObject(PARAMETERS) @@ -551,8 +585,8 @@ public void testKNNVectorFieldMapper_merge_fromKnnMethodContext() throws IOExcep // merge with another mapper of the same field with different context xContentBuilder = XContentFactory.jsonBuilder() .startObject() - .field("type", "knn_vector") - .field("dimension", dimension) + .field(TYPE_FIELD_NAME, KNN_VECTOR_TYPE) + .field(DIMENSION_FIELD_NAME, dimension) .startObject(KNN_METHOD) .field(NAME, METHOD_HNSW) .endObject() @@ -592,7 +626,7 @@ public void testKNNVectorFieldMapper_merge_fromModel() throws IOException { XContentBuilder xContentBuilder = XContentFactory.jsonBuilder() .startObject() - .field("type", "knn_vector") + .field(TYPE_FIELD_NAME, KNN_VECTOR_TYPE) .field(MODEL_ID, modelId) .endObject(); @@ -617,8 +651,8 @@ public void testKNNVectorFieldMapper_merge_fromModel() throws IOException { // merge with another mapper of the same field with different context xContentBuilder = XContentFactory.jsonBuilder() .startObject() - .field("type", "knn_vector") - .field("dimension", dimension) + .field(TYPE_FIELD_NAME, KNN_VECTOR_TYPE) + .field(DIMENSION_FIELD_NAME, dimension) .startObject(KNN_METHOD) .field(NAME, METHOD_HNSW) .endObject() From 2f78a53fc22888c4515f05f56531c40374298b6d Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Wed, 3 Aug 2022 20:48:27 -0700 Subject: [PATCH 022/416] Increase max dimension to 16k for nmslib and faiss (#490) (#491) Increases max dimension to 16k for nmslib and faiss. Lucene has its own limit which we cannot override, so that will not change. Ran benchmarks for 16k dimension random data set. Refer to https://github.com/opensearch-project/k-NN/issues/455 for more details. Signed-off-by: John Mazanec (cherry picked from commit 206f7e18a4f79571f6efe367d7ece3a3117402e7) Co-authored-by: John Mazanec --- .../java/org/opensearch/knn/index/util/KNNEngine.java | 4 ++-- .../java/org/opensearch/knn/indices/ModelMetadata.java | 8 ++++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/opensearch/knn/index/util/KNNEngine.java b/src/main/java/org/opensearch/knn/index/util/KNNEngine.java index d87d48c8f..0751b332a 100644 --- a/src/main/java/org/opensearch/knn/index/util/KNNEngine.java +++ b/src/main/java/org/opensearch/knn/index/util/KNNEngine.java @@ -34,9 +34,9 @@ public enum KNNEngine implements KNNLibrary { private static Map MAX_DIMENSIONS_BY_ENGINE = Map.of( KNNEngine.NMSLIB, - 10_000, + 16_000, KNNEngine.FAISS, - 10_000, + 16_000, KNNEngine.LUCENE, VectorValues.MAX_DIMENSIONS ); diff --git a/src/main/java/org/opensearch/knn/indices/ModelMetadata.java b/src/main/java/org/opensearch/knn/indices/ModelMetadata.java index f1d1b7c40..679b8ea80 100644 --- a/src/main/java/org/opensearch/knn/indices/ModelMetadata.java +++ b/src/main/java/org/opensearch/knn/indices/ModelMetadata.java @@ -89,9 +89,13 @@ public ModelMetadata( this.knnEngine = Objects.requireNonNull(knnEngine, "knnEngine must not be null"); this.spaceType = Objects.requireNonNull(spaceType, "spaceType must not be null"); int maxDimensions = KNNEngine.getMaxDimensionByEngine(this.knnEngine); - if (dimension <= 0 || dimension >= maxDimensions) { + if (dimension <= 0 || dimension > maxDimensions) { throw new IllegalArgumentException( - String.format("Dimension \"%s\" is invalid. Value must be greater than 0 and less than %d", dimension, maxDimensions) + String.format( + "Dimension \"%s\" is invalid. Value must be greater than 0 and less than or equal to %d", + dimension, + maxDimensions + ) ); } this.dimension = dimension; From f88753480798806426838b5df5d470e0455e504e Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Thu, 4 Aug 2022 13:00:40 -0700 Subject: [PATCH 023/416] Remove 1.0.0 for BWC test (#492) (#494) Signed-off-by: Junqiu Lei (cherry picked from commit 57c54963e570fed7490d0e3d5a2e33bf73e061a1) Co-authored-by: Junqiu Lei --- .github/workflows/backwards_compatibility_tests_workflow.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/backwards_compatibility_tests_workflow.yml b/.github/workflows/backwards_compatibility_tests_workflow.yml index c76b22886..71fb3408a 100644 --- a/.github/workflows/backwards_compatibility_tests_workflow.yml +++ b/.github/workflows/backwards_compatibility_tests_workflow.yml @@ -14,7 +14,7 @@ jobs: strategy: matrix: java: [ 11, 17 ] - bwc_version : [ "1.0.0", "1.1.0", "1.2.4", "1.3.2", "2.0.0", "2.1.0" ] + bwc_version : [ "1.1.0", "1.2.4", "1.3.2", "2.0.0", "2.1.0" ] opensearch_version : [ "2.2.0-SNAPSHOT" ] name: k-NN Restart-Upgrade BWC Tests From 53185a0f165ebe1f8c3e01fd80a867727ba3674f Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Thu, 4 Aug 2022 16:51:30 -0700 Subject: [PATCH 024/416] Add 2.2.0.0 release notes (#484) (#495) Signed-off-by: Junqiu Lei (cherry picked from commit 641614d6264cba6b84833f857a366bcf757dca64) Co-authored-by: Junqiu Lei --- .../opensearch-knn.release-notes-2.2.0.0.md | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 release-notes/opensearch-knn.release-notes-2.2.0.0.md diff --git a/release-notes/opensearch-knn.release-notes-2.2.0.0.md b/release-notes/opensearch-knn.release-notes-2.2.0.0.md new file mode 100644 index 000000000..16aa9acef --- /dev/null +++ b/release-notes/opensearch-knn.release-notes-2.2.0.0.md @@ -0,0 +1,31 @@ +## Version 2.2.0.0 Release Notes + +Compatible with OpenSearch 2.2.0 + +### Features +* Lucene Based k-NN search support([#486](https://github.com/opensearch-project/k-NN/pull/486)) + +### Enhancements +* Add KNN codec that is based on Lucene92 codec([#444](https://github.com/opensearch-project/k-NN/pull/444)) +* Remove support for innerproduct for lucene engine([#488](https://github.com/opensearch-project/k-NN/pull/488)) +* Increase max dimension to 16k for nmslib and faiss([#490](https://github.com/opensearch-project/k-NN/pull/490)) + +### Bug Fixes +* Reject delete model request if model is in Training([#424](https://github.com/opensearch-project/k-NN/pull/424)) +* Change call to Lucene VectorSimilarityFunction.convertToScore([#487](https://github.com/opensearch-project/k-NN/pull/487)) + +### Infrastructure +* Add fix to flaky test in ModelDaoTests([#463](https://github.com/opensearch-project/k-NN/pull/463)) +* Read BWC Version from GitHub workflow([#476](https://github.com/opensearch-project/k-NN/pull/476)) +* Staging for version increment automation([#442](https://github.com/opensearch-project/k-NN/pull/442)) +* Remove 1.0.0 for BWC test([#492](https://github.com/opensearch-project/k-NN/pull/492)) + +### Maintenance +* Bump OpenSearch version to 2.2.0([#471](https://github.com/opensearch-project/k-NN/pull/471)) +* Bump Gradle version to 7.5([#472](https://github.com/opensearch-project/k-NN/pull/472)) +* Bump default bwc version to 1.3.4([#477](https://github.com/opensearch-project/k-NN/pull/477)) + +### Refactoring +* Move engine and lib components into separate files([#438](https://github.com/opensearch-project/k-NN/pull/438)) +* Refactor knn type and codecs([#439](https://github.com/opensearch-project/k-NN/pull/439)) +* Move mappers to separate files([#448](https://github.com/opensearch-project/k-NN/pull/448)) From 041c78bb8c1bebeb33eaec333434e0b45b0c39e4 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Tue, 9 Aug 2022 09:30:09 -0700 Subject: [PATCH 025/416] Remove overallocation in faiss query path (#503) Removes overallocation of 2 c++ vectors in faiss querying functionality. Performance results can be viewed in [497](https://github.com/opensearch-project/k-NN/issues/497#issuecomment-1208643311). In general, this change could provide a small improvement in memory footprint during search workloads. Signed-off-by: John Mazanec (cherry picked from commit 507bafe13b547099d3e2ee3355bc7086b6e18a76) Co-authored-by: John Mazanec --- jni/src/faiss_wrapper.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/jni/src/faiss_wrapper.cpp b/jni/src/faiss_wrapper.cpp index d1342a2ba..e0fcc822b 100644 --- a/jni/src/faiss_wrapper.cpp +++ b/jni/src/faiss_wrapper.cpp @@ -192,9 +192,10 @@ jobjectArray knn_jni::faiss_wrapper::QueryIndex(knn_jni::JNIUtilInterface * jniU throw std::runtime_error("Invalid pointer to index"); } - int dim = jniUtil->GetJavaFloatArrayLength(env, queryVectorJ); - std::vector dis(kJ * dim); - std::vector ids(kJ * dim); + // The ids vector will hold the top k ids from the search and the dis vector will hold the top k distances from + // the query point + std::vector dis(kJ); + std::vector ids(kJ); float* rawQueryvector = jniUtil->GetFloatArrayElements(env, queryVectorJ, nullptr); try { From bab079034d6a2b33a3fac9e22c43248151d59bd5 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Thu, 11 Aug 2022 14:59:04 -0700 Subject: [PATCH 026/416] Change initial size of DocIdSetBuilder (#511) Changes initial size of the docidsetbuilder used for iterating over results for k-NN queries. Originally, it was set to the maximum docid. This changes it to be the number of docs returned. Signed-off-by: John Mazanec (cherry picked from commit 586958e3ee2f99da85a6ec28ea4565c9ec572208) --- src/main/java/org/opensearch/knn/index/query/KNNWeight.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/opensearch/knn/index/query/KNNWeight.java b/src/main/java/org/opensearch/knn/index/query/KNNWeight.java index c7f2ddf47..716aed412 100644 --- a/src/main/java/org/opensearch/knn/index/query/KNNWeight.java +++ b/src/main/java/org/opensearch/knn/index/query/KNNWeight.java @@ -182,7 +182,10 @@ public Scorer scorer(LeafReaderContext context) throws IOException { .collect(Collectors.toMap(KNNQueryResult::getId, result -> knnEngine.score(result.getScore(), spaceType))); int maxDoc = Collections.max(scores.keySet()) + 1; DocIdSetBuilder docIdSetBuilder = new DocIdSetBuilder(maxDoc); - DocIdSetBuilder.BulkAdder setAdder = docIdSetBuilder.grow(maxDoc); + + // The docIdSetIterator will contain the docids of the returned results. So, before adding results to + // the builder, we can grow to results.length + DocIdSetBuilder.BulkAdder setAdder = docIdSetBuilder.grow(results.length); Arrays.stream(results).forEach(result -> setAdder.add(result.getId())); DocIdSetIterator docIdSetIter = docIdSetBuilder.build().iterator(); return new KNNScorer(this, docIdSetIter, scores, boost); From cc9ca0f46bbe24b49681287fbe14b97f0c584426 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Mon, 29 Aug 2022 19:09:45 -0500 Subject: [PATCH 027/416] Replace terminology 'master' with 'cluster manager' (#521) (#522) Signed-off-by: Naveen Tatikonda --- .../java/org/opensearch/knn/index/KNNCircuitBreaker.java | 2 +- .../java/org/opensearch/knn/indices/ModelGraveyard.java | 2 +- .../transport/UpdateModelGraveyardTransportAction.java | 8 +++++--- .../transport/UpdateModelMetadataTransportAction.java | 8 +++++--- .../UpdateModelGraveyardTransportActionTests.java | 6 +++--- .../UpdateModelMetadataTransportActionTests.java | 4 ++-- 6 files changed, 17 insertions(+), 13 deletions(-) diff --git a/src/main/java/org/opensearch/knn/index/KNNCircuitBreaker.java b/src/main/java/org/opensearch/knn/index/KNNCircuitBreaker.java index 5375beddd..4f361ae37 100644 --- a/src/main/java/org/opensearch/knn/index/KNNCircuitBreaker.java +++ b/src/main/java/org/opensearch/knn/index/KNNCircuitBreaker.java @@ -72,7 +72,7 @@ public void initialize(ThreadPool threadPool, ClusterService clusterService, Cli } // Leader node untriggers CB if all nodes have not reached their max capacity - if (KNNSettings.isCircuitBreakerTriggered() && clusterService.state().nodes().isLocalNodeElectedMaster()) { + if (KNNSettings.isCircuitBreakerTriggered() && clusterService.state().nodes().isLocalNodeElectedClusterManager()) { KNNStatsRequest knnStatsRequest = new KNNStatsRequest(KNNStatsConfig.KNN_STATS.keySet()); knnStatsRequest.addStat(StatNames.CACHE_CAPACITY_REACHED.getName()); knnStatsRequest.timeout(new TimeValue(1000 * 10)); // 10 second timeout diff --git a/src/main/java/org/opensearch/knn/indices/ModelGraveyard.java b/src/main/java/org/opensearch/knn/indices/ModelGraveyard.java index 78499c1f3..ff7232bdf 100644 --- a/src/main/java/org/opensearch/knn/indices/ModelGraveyard.java +++ b/src/main/java/org/opensearch/knn/indices/ModelGraveyard.java @@ -148,7 +148,7 @@ public static ModelGraveyard fromXContent(XContentParser xContentParser) throws /** * The ModelGraveyardDiff class compares the previous modelGraveyard object with the current updated modelGraveyard object - * and returns only the diff of those 2 objects. So that, whenever there is a change in cluster state, master node only + * and returns only the diff of those 2 objects. So that, whenever there is a change in cluster state, clusterManager node only * sends the diff to all the data nodes instead of the full cluster state */ public static class ModelGraveyardDiff implements NamedDiff { diff --git a/src/main/java/org/opensearch/knn/plugin/transport/UpdateModelGraveyardTransportAction.java b/src/main/java/org/opensearch/knn/plugin/transport/UpdateModelGraveyardTransportAction.java index 68708a5e0..a7b5dc876 100644 --- a/src/main/java/org/opensearch/knn/plugin/transport/UpdateModelGraveyardTransportAction.java +++ b/src/main/java/org/opensearch/knn/plugin/transport/UpdateModelGraveyardTransportAction.java @@ -10,7 +10,7 @@ import org.opensearch.action.ActionListener; import org.opensearch.action.support.ActionFilters; import org.opensearch.action.support.master.AcknowledgedResponse; -import org.opensearch.action.support.master.TransportMasterNodeAction; +import org.opensearch.action.support.clustermanager.TransportClusterManagerNodeAction; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.ClusterStateTaskConfig; import org.opensearch.cluster.ClusterStateTaskExecutor; @@ -38,7 +38,9 @@ * Transport action used to update model graveyard on the cluster manager node. */ @Log4j2 -public class UpdateModelGraveyardTransportAction extends TransportMasterNodeAction { +public class UpdateModelGraveyardTransportAction extends TransportClusterManagerNodeAction< + UpdateModelGraveyardRequest, + AcknowledgedResponse> { private UpdateModelGraveyardExecutor updateModelGraveyardExecutor; @Inject @@ -72,7 +74,7 @@ protected AcknowledgedResponse read(StreamInput streamInput) throws IOException } @Override - protected void masterOperation( + protected void clusterManagerOperation( UpdateModelGraveyardRequest request, ClusterState clusterState, ActionListener actionListener diff --git a/src/main/java/org/opensearch/knn/plugin/transport/UpdateModelMetadataTransportAction.java b/src/main/java/org/opensearch/knn/plugin/transport/UpdateModelMetadataTransportAction.java index c3ef866fd..2cfe04123 100644 --- a/src/main/java/org/opensearch/knn/plugin/transport/UpdateModelMetadataTransportAction.java +++ b/src/main/java/org/opensearch/knn/plugin/transport/UpdateModelMetadataTransportAction.java @@ -16,7 +16,7 @@ import org.opensearch.action.ActionListener; import org.opensearch.action.support.ActionFilters; import org.opensearch.action.support.master.AcknowledgedResponse; -import org.opensearch.action.support.master.TransportMasterNodeAction; +import org.opensearch.action.support.clustermanager.TransportClusterManagerNodeAction; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.ClusterStateTaskConfig; import org.opensearch.cluster.ClusterStateTaskExecutor; @@ -45,7 +45,9 @@ /** * Transport action used to update metadata of model's on the cluster manager node. */ -public class UpdateModelMetadataTransportAction extends TransportMasterNodeAction { +public class UpdateModelMetadataTransportAction extends TransportClusterManagerNodeAction< + UpdateModelMetadataRequest, + AcknowledgedResponse> { public static Logger logger = LogManager.getLogger(UpdateModelMetadataTransportAction.class); @@ -82,7 +84,7 @@ protected AcknowledgedResponse read(StreamInput streamInput) throws IOException } @Override - protected void masterOperation( + protected void clusterManagerOperation( UpdateModelMetadataRequest request, ClusterState clusterState, ActionListener actionListener diff --git a/src/test/java/org/opensearch/knn/plugin/transport/UpdateModelGraveyardTransportActionTests.java b/src/test/java/org/opensearch/knn/plugin/transport/UpdateModelGraveyardTransportActionTests.java index 6216f985d..29ef6fab4 100644 --- a/src/test/java/org/opensearch/knn/plugin/transport/UpdateModelGraveyardTransportActionTests.java +++ b/src/test/java/org/opensearch/knn/plugin/transport/UpdateModelGraveyardTransportActionTests.java @@ -51,7 +51,7 @@ public void testClusterManagerOperation() throws InterruptedException { final CountDownLatch inProgressLatch1 = new CountDownLatch(1); client().admin().cluster().prepareState().execute(ActionListener.wrap(stateResponse1 -> { ClusterState clusterState1 = stateResponse1.getState(); - updateModelGraveyardTransportAction.masterOperation( + updateModelGraveyardTransportAction.clusterManagerOperation( addModelGraveyardRequest, clusterState1, ActionListener.wrap(acknowledgedResponse -> { @@ -81,7 +81,7 @@ public void testClusterManagerOperation() throws InterruptedException { final CountDownLatch inProgressLatch2 = new CountDownLatch(1); client().admin().cluster().prepareState().execute(ActionListener.wrap(stateResponse1 -> { ClusterState clusterState1 = stateResponse1.getState(); - updateModelGraveyardTransportAction.masterOperation( + updateModelGraveyardTransportAction.clusterManagerOperation( addModelGraveyardRequest1, clusterState1, ActionListener.wrap(acknowledgedResponse -> { @@ -123,7 +123,7 @@ public void testClusterManagerOperation() throws InterruptedException { final CountDownLatch inProgressLatch3 = new CountDownLatch(1); client().admin().cluster().prepareState().execute(ActionListener.wrap(stateResponse1 -> { ClusterState clusterState1 = stateResponse1.getState(); - updateModelGraveyardTransportAction.masterOperation( + updateModelGraveyardTransportAction.clusterManagerOperation( removeModelGraveyardRequest, clusterState1, ActionListener.wrap(acknowledgedResponse -> { diff --git a/src/test/java/org/opensearch/knn/plugin/transport/UpdateModelMetadataTransportActionTests.java b/src/test/java/org/opensearch/knn/plugin/transport/UpdateModelMetadataTransportActionTests.java index a56c4d2bd..6fe9222c8 100644 --- a/src/test/java/org/opensearch/knn/plugin/transport/UpdateModelMetadataTransportActionTests.java +++ b/src/test/java/org/opensearch/knn/plugin/transport/UpdateModelMetadataTransportActionTests.java @@ -79,7 +79,7 @@ public void testClusterManagerOperation() throws InterruptedException { final CountDownLatch inProgressLatch1 = new CountDownLatch(1); client().admin().cluster().prepareState().execute(ActionListener.wrap(stateResponse1 -> { ClusterState clusterState1 = stateResponse1.getState(); - updateModelMetadataTransportAction.masterOperation( + updateModelMetadataTransportAction.clusterManagerOperation( updateModelMetadataRequest, clusterState1, ActionListener.wrap(acknowledgedResponse -> { @@ -114,7 +114,7 @@ public void testClusterManagerOperation() throws InterruptedException { final CountDownLatch inProgressLatch2 = new CountDownLatch(1); client().admin().cluster().prepareState().execute(ActionListener.wrap(stateResponse1 -> { ClusterState clusterState1 = stateResponse1.getState(); - updateModelMetadataTransportAction.masterOperation( + updateModelMetadataTransportAction.clusterManagerOperation( removeModelMetadataRequest, clusterState1, ActionListener.wrap(acknowledgedResponse -> { From 379bf9c3c41fac7000442d9c42e40b57f5660fe8 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Tue, 6 Sep 2022 12:38:55 -0700 Subject: [PATCH 028/416] Increment version to 2.3.0-SNAPSHOT (#526) Signed-off-by: opensearch-ci-bot Signed-off-by: opensearch-ci-bot Co-authored-by: opensearch-ci-bot --- .github/workflows/backwards_compatibility_tests_workflow.yml | 4 ++-- build.gradle | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/backwards_compatibility_tests_workflow.yml b/.github/workflows/backwards_compatibility_tests_workflow.yml index 71fb3408a..bac0dba44 100644 --- a/.github/workflows/backwards_compatibility_tests_workflow.yml +++ b/.github/workflows/backwards_compatibility_tests_workflow.yml @@ -15,7 +15,7 @@ jobs: matrix: java: [ 11, 17 ] bwc_version : [ "1.1.0", "1.2.4", "1.3.2", "2.0.0", "2.1.0" ] - opensearch_version : [ "2.2.0-SNAPSHOT" ] + opensearch_version : [ "2.3.0-SNAPSHOT" ] name: k-NN Restart-Upgrade BWC Tests runs-on: ubuntu-latest @@ -47,7 +47,7 @@ jobs: matrix: java: [ 11, 17 ] bwc_version: [ "1.3.2", "2.0.0", "2.1.0" ] - opensearch_version: [ "2.2.0-SNAPSHOT" ] + opensearch_version: [ "2.3.0-SNAPSHOT" ] name: k-NN Rolling-Upgrade BWC Tests runs-on: ubuntu-latest diff --git a/build.gradle b/build.gradle index 8092e7a00..e5dc1b662 100644 --- a/build.gradle +++ b/build.gradle @@ -9,7 +9,7 @@ buildscript { ext { // build.version_qualifier parameter applies to knn plugin artifacts only. OpenSearch version must be set // explicitly as 'opensearch.version' property, for instance opensearch.version=2.0.0-rc1-SNAPSHOT - opensearch_version = System.getProperty("opensearch.version", "2.2.0-SNAPSHOT") + opensearch_version = System.getProperty("opensearch.version", "2.3.0-SNAPSHOT") version_qualifier = System.getProperty("build.version_qualifier", "") opensearch_group = "org.opensearch" } From f20aaf17fd055d9c6cfbf7b11a79fb74def90fb4 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Tue, 6 Sep 2022 18:38:06 -0500 Subject: [PATCH 029/416] Nomenclature changes from Whitelist to Allowlist (#534) (#535) Signed-off-by: Naveen Tatikonda (cherry picked from commit c7379f8a43508a7ebb92a895581a292818b4d3aa) --- ...listExtension.java => KNNAllowlistExtension.java} | 4 ++-- .../opensearch/knn/plugin/script/KNNScoringUtil.java | 12 ++++++------ .../org.opensearch.painless.spi.PainlessExtension | 2 +- .../script/{knn_whitelist.txt => knn_allowlist.txt} | 0 .../knn/plugin/script/KNNScoringUtilTests.java | 2 +- 5 files changed, 10 insertions(+), 10 deletions(-) rename src/main/java/org/opensearch/knn/plugin/script/{KNNWhitelistExtension.java => KNNAllowlistExtension.java} (90%) rename src/main/resources/org/opensearch/knn/plugin/script/{knn_whitelist.txt => knn_allowlist.txt} (100%) diff --git a/src/main/java/org/opensearch/knn/plugin/script/KNNWhitelistExtension.java b/src/main/java/org/opensearch/knn/plugin/script/KNNAllowlistExtension.java similarity index 90% rename from src/main/java/org/opensearch/knn/plugin/script/KNNWhitelistExtension.java rename to src/main/java/org/opensearch/knn/plugin/script/KNNAllowlistExtension.java index 4c0fe01a7..959063d61 100644 --- a/src/main/java/org/opensearch/knn/plugin/script/KNNWhitelistExtension.java +++ b/src/main/java/org/opensearch/knn/plugin/script/KNNAllowlistExtension.java @@ -16,9 +16,9 @@ import java.util.List; import java.util.Map; -public class KNNWhitelistExtension implements PainlessExtension { +public class KNNAllowlistExtension implements PainlessExtension { - private static final Whitelist ALLOW_LIST = WhitelistLoader.loadFromResourceFiles(KNNWhitelistExtension.class, "knn_whitelist.txt"); + private static final Whitelist ALLOW_LIST = WhitelistLoader.loadFromResourceFiles(KNNAllowlistExtension.class, "knn_allowlist.txt"); @Override public Map, List> getContextWhitelists() { diff --git a/src/main/java/org/opensearch/knn/plugin/script/KNNScoringUtil.java b/src/main/java/org/opensearch/knn/plugin/script/KNNScoringUtil.java index 5ec462933..90468c2e7 100644 --- a/src/main/java/org/opensearch/knn/plugin/script/KNNScoringUtil.java +++ b/src/main/java/org/opensearch/knn/plugin/script/KNNScoringUtil.java @@ -65,7 +65,7 @@ private static float[] toFloat(List inputVector) { } /** - * Whitelisted l2Squared method for users to calculate L2 squared distance between query vector + * Allowlisted l2Squared method for users to calculate L2 squared distance between query vector * and document vectors * Example * "script": { @@ -110,7 +110,7 @@ public static float cosinesimilOptimized(float[] queryVector, float[] inputVecto } /** - * Whitelisted cosineSimilarity method that can be used in a script to avoid repeated + * Allowlisted cosineSimilarity method that can be used in a script to avoid repeated * calculation of normalization for the query vector. * Example: * "script": { @@ -156,7 +156,7 @@ public static float cosinesimil(float[] queryVector, float[] inputVector) { } /** - * Whitelisted cosineSimilarity method for users to calculate cosine similarity between query vectors and + * Allowlisted cosineSimilarity method for users to calculate cosine similarity between query vectors and * document vectors * Example: * "script": { @@ -216,7 +216,7 @@ public static float l1Norm(float[] queryVector, float[] inputVector) { } /** - * Whitelisted l1distance method for users to calculate L1 distance between query vector + * Allowlisted l1distance method for users to calculate L1 distance between query vector * and document vectors * Example * "script": { @@ -254,7 +254,7 @@ public static float lInfNorm(float[] queryVector, float[] inputVector) { } /** - * Whitelisted lInfNorm method for users to calculate L-inf distance between query vector + * Allowlisted lInfNorm method for users to calculate L-inf distance between query vector * and document vectors * Example * "script": { @@ -291,7 +291,7 @@ public static float innerProduct(float[] queryVector, float[] inputVector) { } /** - * Whitelisted innerProd method for users to calculate inner product distance between query vector + * Allowlisted innerProd method for users to calculate inner product distance between query vector * and document vectors * Example * "script": { diff --git a/src/main/resources/META-INF/services/org.opensearch.painless.spi.PainlessExtension b/src/main/resources/META-INF/services/org.opensearch.painless.spi.PainlessExtension index 5fdf5f649..530c96349 100644 --- a/src/main/resources/META-INF/services/org.opensearch.painless.spi.PainlessExtension +++ b/src/main/resources/META-INF/services/org.opensearch.painless.spi.PainlessExtension @@ -1,4 +1,4 @@ # Copyright OpenSearch Contributors # SPDX-License-Identifier: Apache-2.0 -org.opensearch.knn.plugin.script.KNNWhitelistExtension +org.opensearch.knn.plugin.script.KNNAllowlistExtension diff --git a/src/main/resources/org/opensearch/knn/plugin/script/knn_whitelist.txt b/src/main/resources/org/opensearch/knn/plugin/script/knn_allowlist.txt similarity index 100% rename from src/main/resources/org/opensearch/knn/plugin/script/knn_whitelist.txt rename to src/main/resources/org/opensearch/knn/plugin/script/knn_allowlist.txt diff --git a/src/test/java/org/opensearch/knn/plugin/script/KNNScoringUtilTests.java b/src/test/java/org/opensearch/knn/plugin/script/KNNScoringUtilTests.java index 7f66e909a..49add790e 100644 --- a/src/test/java/org/opensearch/knn/plugin/script/KNNScoringUtilTests.java +++ b/src/test/java/org/opensearch/knn/plugin/script/KNNScoringUtilTests.java @@ -169,7 +169,7 @@ public void testBitHammingDistance_Long() { assertEquals(0.0, KNNScoringUtil.calculateHammingBit(long3, long3), 0.1); } - public void testL2SquaredWhitelistedScoringFunction() throws IOException { + public void testL2SquaredAllowlistedScoringFunction() throws IOException { List queryVector = getTestQueryVector(); TestKNNScriptDocValues dataset = new TestKNNScriptDocValues(); dataset.createKNNVectorDocument(new float[] { 4.0f, 4.0f, 4.0f }, "test-index-field-name"); From d1b8998b3bb2e326645c289bcd3868f8063a976a Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Wed, 7 Sep 2022 10:49:09 -0700 Subject: [PATCH 030/416] Updated the BWC workflow to have 2.2.0 as the backward supported version in BWC tests (#536) (#539) Signed-off-by: Navneet Verma Signed-off-by: Navneet Verma (cherry picked from commit 7f90e2c4d3ed8f929724a6f5bca4122452457f78) Co-authored-by: Navneet Verma --- .github/workflows/backwards_compatibility_tests_workflow.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/backwards_compatibility_tests_workflow.yml b/.github/workflows/backwards_compatibility_tests_workflow.yml index bac0dba44..a1d15afaa 100644 --- a/.github/workflows/backwards_compatibility_tests_workflow.yml +++ b/.github/workflows/backwards_compatibility_tests_workflow.yml @@ -14,7 +14,7 @@ jobs: strategy: matrix: java: [ 11, 17 ] - bwc_version : [ "1.1.0", "1.2.4", "1.3.2", "2.0.0", "2.1.0" ] + bwc_version : [ "1.1.0", "1.2.4", "1.3.2", "2.0.0", "2.1.0", "2.2.0" ] opensearch_version : [ "2.3.0-SNAPSHOT" ] name: k-NN Restart-Upgrade BWC Tests @@ -46,7 +46,7 @@ jobs: strategy: matrix: java: [ 11, 17 ] - bwc_version: [ "1.3.2", "2.0.0", "2.1.0" ] + bwc_version: [ "1.3.2", "2.0.0", "2.1.0", "2.2.0" ] opensearch_version: [ "2.3.0-SNAPSHOT" ] name: k-NN Rolling-Upgrade BWC Tests From 8734b9da4726eb6dcb353d2aa4ff8497f859cee0 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Fri, 9 Sep 2022 15:10:16 -0700 Subject: [PATCH 031/416] Added release notes for version 2.3.0 (#546) (#547) Signed-off-by: Navneet Verma Signed-off-by: Navneet Verma (cherry picked from commit 89e7f4a00d48751dcdf9d9f09a7fc03e4aad8a7a) Co-authored-by: Navneet Verma --- .../opensearch-knn.release-notes-2.3.0.0.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 release-notes/opensearch-knn.release-notes-2.3.0.0.md diff --git a/release-notes/opensearch-knn.release-notes-2.3.0.0.md b/release-notes/opensearch-knn.release-notes-2.3.0.0.md new file mode 100644 index 000000000..57f6da308 --- /dev/null +++ b/release-notes/opensearch-knn.release-notes-2.3.0.0.md @@ -0,0 +1,13 @@ +## Version 2.3.0.0 Release Notes + +Compatible with OpenSearch 2.3.0 + +### Enhancements +* Change initial size of DocIdSetBuilder (#502) + +### Bug Fixes +* Remove overallocation in faiss query path (#501) + +### Refactoring +* Replace terminology 'master' with 'cluster manager' (#521) +* Nomenclature changes from Whitelist to Allowlist (#534) From 48d2303ad6964d386709ab5ae5fdbb0965420cb8 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Tue, 13 Sep 2022 10:22:32 -0700 Subject: [PATCH 032/416] Adding k-NN engine stat (#523) (#542) * Adding field by engine stat Signed-off-by: Martin Gaievski --- .../java/org/opensearch/knn/bwc/StatsIT.java | 8 +- .../index/mapper/KNNVectorFieldMapper.java | 7 + .../knn/index/mapper/LuceneFieldMapper.java | 5 + .../opensearch/knn/index/util/JVMLibrary.java | 12 ++ .../org/opensearch/knn/index/util/Lucene.java | 10 -- .../knn/plugin/stats/KNNStatsConfig.java | 1 + .../knn/plugin/stats/StatNames.java | 1 + .../mapper/KNNVectorFieldMapperTests.java | 5 + .../knn/index/util/LuceneTests.java | 5 +- .../plugin/action/RestKNNStatsHandlerIT.java | 121 ++++++++++++++++++ .../org/opensearch/knn/KNNRestTestCase.java | 21 +++ 11 files changed, 181 insertions(+), 15 deletions(-) diff --git a/qa/rolling-upgrade/src/test/java/org/opensearch/knn/bwc/StatsIT.java b/qa/rolling-upgrade/src/test/java/org/opensearch/knn/bwc/StatsIT.java index df9babd77..cca6a45c7 100644 --- a/qa/rolling-upgrade/src/test/java/org/opensearch/knn/bwc/StatsIT.java +++ b/qa/rolling-upgrade/src/test/java/org/opensearch/knn/bwc/StatsIT.java @@ -20,14 +20,16 @@ public class StatsIT extends AbstractRollingUpgradeTestCase { private KNNStats knnStats = new KNNStats(KNN_STATS); - // Validate if all the KNN Stats metrics are returned + // Validate if all the KNN Stats metrics from old version are present in new version public void testAllMetricStatsReturned() throws IOException { Response response = getKnnStats(Collections.emptyList(), Collections.emptyList()); String responseBody = EntityUtils.toString(response.getEntity()); Map clusterStats = parseClusterStatsResponse(responseBody); - assertEquals(knnStats.getClusterStats().keySet(), clusterStats.keySet()); + assertNotNull(clusterStats); + assertTrue(knnStats.getClusterStats().keySet().containsAll(clusterStats.keySet())); List> nodeStats = parseNodeStatsResponse(responseBody); - assertEquals(knnStats.getNodeStats().keySet(), nodeStats.get(0).keySet()); + assertNotNull(nodeStats.get(0)); + assertTrue(knnStats.getNodeStats().keySet().containsAll(nodeStats.get(0).keySet())); } // Verify if it returns failure for invalid metric diff --git a/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapper.java b/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapper.java index 96fea1fbc..3964dd98f 100644 --- a/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapper.java +++ b/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapper.java @@ -197,6 +197,7 @@ public KNNVectorFieldMapper build(BuilderContext context) { final CopyTo copyToBuilder = copyTo.build(); final Explicit ignoreMalformed = ignoreMalformed(context); final Map metaValue = meta.getValue(); + if (knnMethodContext != null) { final KNNVectorFieldType mappedFieldType = new KNNVectorFieldType( buildFullName(context), @@ -407,6 +408,7 @@ public KNNVectorFieldMapper( this.stored = stored; this.hasDocValues = hasDocValues; this.dimension = mappedFieldType.getDimension(); + updateEngineStats(); } public KNNVectorFieldMapper clone() { @@ -538,6 +540,11 @@ protected void doXContentBody(XContentBuilder builder, boolean includeDefaults, } } + /** + * Overwrite at child level in case specific stat needs to be updated + */ + void updateEngineStats() {} + public static class Names { public static final String IGNORE_MALFORMED = "ignore_malformed"; } diff --git a/src/main/java/org/opensearch/knn/index/mapper/LuceneFieldMapper.java b/src/main/java/org/opensearch/knn/index/mapper/LuceneFieldMapper.java index 5b6c65486..5dcb09318 100644 --- a/src/main/java/org/opensearch/knn/index/mapper/LuceneFieldMapper.java +++ b/src/main/java/org/opensearch/knn/index/mapper/LuceneFieldMapper.java @@ -105,6 +105,11 @@ protected void parseCreateField(ParseContext context, int dimension) throws IOEx context.path().remove(); } + @Override + void updateEngineStats() { + KNNEngine.LUCENE.setInitialized(true); + } + @AllArgsConstructor @lombok.Builder @Getter diff --git a/src/main/java/org/opensearch/knn/index/util/JVMLibrary.java b/src/main/java/org/opensearch/knn/index/util/JVMLibrary.java index f6d4dc283..e1d48cb0a 100644 --- a/src/main/java/org/opensearch/knn/index/util/JVMLibrary.java +++ b/src/main/java/org/opensearch/knn/index/util/JVMLibrary.java @@ -15,6 +15,8 @@ */ public abstract class JVMLibrary extends AbstractKNNLibrary { + boolean initialized; + /** * Constructor * @@ -29,4 +31,14 @@ public abstract class JVMLibrary extends AbstractKNNLibrary { public int estimateOverheadInKB(KNNMethodContext knnMethodContext, int dimension) { throw new UnsupportedOperationException("Estimating overhead is not supported for JVM based libraries."); } + + @Override + public Boolean isInitialized() { + return initialized; + } + + @Override + public void setInitialized(Boolean isInitialized) { + initialized = isInitialized; + } } diff --git a/src/main/java/org/opensearch/knn/index/util/Lucene.java b/src/main/java/org/opensearch/knn/index/util/Lucene.java index df1353106..e1996353f 100644 --- a/src/main/java/org/opensearch/knn/index/util/Lucene.java +++ b/src/main/java/org/opensearch/knn/index/util/Lucene.java @@ -73,14 +73,4 @@ public float score(float rawScore, SpaceType spaceType) { // score provided. return rawScore; } - - @Override - public Boolean isInitialized() { - return true; - } - - @Override - public void setInitialized(Boolean isInitialized) { - throw new UnsupportedOperationException("Setting Lucene as initialized is not supported"); - } } diff --git a/src/main/java/org/opensearch/knn/plugin/stats/KNNStatsConfig.java b/src/main/java/org/opensearch/knn/plugin/stats/KNNStatsConfig.java index 56089ed84..c41170b32 100644 --- a/src/main/java/org/opensearch/knn/plugin/stats/KNNStatsConfig.java +++ b/src/main/java/org/opensearch/knn/plugin/stats/KNNStatsConfig.java @@ -75,6 +75,7 @@ public class KNNStatsConfig { ) .put(StatNames.FAISS_LOADED.getName(), new KNNStat<>(false, new LibraryInitializedSupplier(KNNEngine.FAISS))) .put(StatNames.NMSLIB_LOADED.getName(), new KNNStat<>(false, new LibraryInitializedSupplier(KNNEngine.NMSLIB))) + .put(StatNames.LUCENE_LOADED.getName(), new KNNStat<>(false, new LibraryInitializedSupplier(KNNEngine.LUCENE))) .put(StatNames.TRAINING_REQUESTS.getName(), new KNNStat<>(false, new KNNCounterSupplier(KNNCounter.TRAINING_REQUESTS))) .put(StatNames.TRAINING_ERRORS.getName(), new KNNStat<>(false, new KNNCounterSupplier(KNNCounter.TRAINING_ERRORS))) .put( diff --git a/src/main/java/org/opensearch/knn/plugin/stats/StatNames.java b/src/main/java/org/opensearch/knn/plugin/stats/StatNames.java index f807c3eaa..ffe5882bb 100644 --- a/src/main/java/org/opensearch/knn/plugin/stats/StatNames.java +++ b/src/main/java/org/opensearch/knn/plugin/stats/StatNames.java @@ -26,6 +26,7 @@ public enum StatNames { MODEL_INDEX_STATUS("model_index_status"), FAISS_LOADED("faiss_initialized"), NMSLIB_LOADED("nmslib_initialized"), + LUCENE_LOADED("lucene_initialized"), INDEXING_FROM_MODEL_DEGRADED("indexing_from_model_degraded"), GRAPH_QUERY_ERRORS(KNNCounter.GRAPH_QUERY_ERRORS.getName()), GRAPH_QUERY_REQUESTS(KNNCounter.GRAPH_QUERY_REQUESTS.getName()), diff --git a/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java b/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java index 13e2ab9a3..3c311f86b 100644 --- a/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java +++ b/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java @@ -193,6 +193,8 @@ public void testBuilder_parse_fromKnnMethodContext_luceneEngine() throws IOExcep ModelDao modelDao = mock(ModelDao.class); KNNVectorFieldMapper.TypeParser typeParser = new KNNVectorFieldMapper.TypeParser(() -> modelDao); + KNNEngine.LUCENE.setInitialized(false); + int efConstruction = 321; int m = 12; int dimension = 133; @@ -215,12 +217,15 @@ public void testBuilder_parse_fromKnnMethodContext_luceneEngine() throws IOExcep xContentBuilderToMap(xContentBuilder), buildParserContext(indexName, settings) ); + Mapper.BuilderContext builderContext = new Mapper.BuilderContext(settings, new ContentPath()); + builder.build(builderContext); assertEquals(METHOD_HNSW, builder.knnMethodContext.get().getMethodComponent().getName()); assertEquals( efConstruction, builder.knnMethodContext.get().getMethodComponent().getParameters().get(METHOD_PARAMETER_EF_CONSTRUCTION) ); + assertTrue(KNNEngine.LUCENE.isInitialized()); XContentBuilder xContentBuilderEmptyParams = XContentFactory.jsonBuilder() .startObject() diff --git a/src/test/java/org/opensearch/knn/index/util/LuceneTests.java b/src/test/java/org/opensearch/knn/index/util/LuceneTests.java index d5fffc1bb..57e02e173 100644 --- a/src/test/java/org/opensearch/knn/index/util/LuceneTests.java +++ b/src/test/java/org/opensearch/knn/index/util/LuceneTests.java @@ -100,11 +100,12 @@ public void testScore() { public void testIsInitialized() { Lucene luceneLibrary = new Lucene(Collections.emptyMap(), ""); - assertTrue(luceneLibrary.isInitialized()); + assertFalse(luceneLibrary.isInitialized()); } public void testSetInitialized() { Lucene luceneLibrary = new Lucene(Collections.emptyMap(), ""); - expectThrows(UnsupportedOperationException.class, () -> luceneLibrary.setInitialized(true)); + luceneLibrary.setInitialized(true); + assertTrue(luceneLibrary.isInitialized()); } } diff --git a/src/test/java/org/opensearch/knn/plugin/action/RestKNNStatsHandlerIT.java b/src/test/java/org/opensearch/knn/plugin/action/RestKNNStatsHandlerIT.java index 10fa47c4c..2b6bb757e 100644 --- a/src/test/java/org/opensearch/knn/plugin/action/RestKNNStatsHandlerIT.java +++ b/src/test/java/org/opensearch/knn/plugin/action/RestKNNStatsHandlerIT.java @@ -14,8 +14,11 @@ import org.opensearch.client.Response; import org.opensearch.client.ResponseException; import org.opensearch.cluster.health.ClusterHealthStatus; +import org.opensearch.common.Strings; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; +import org.opensearch.common.xcontent.XContentBuilder; +import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentType; import org.opensearch.index.query.MatchAllQueryBuilder; import org.opensearch.index.query.QueryBuilder; @@ -33,6 +36,20 @@ import java.util.List; import java.util.Map; +import static org.opensearch.knn.TestUtils.KNN_VECTOR; +import static org.opensearch.knn.TestUtils.PROPERTIES; +import static org.opensearch.knn.TestUtils.VECTOR_TYPE; +import static org.opensearch.knn.common.KNNConstants.FAISS_NAME; +import static org.opensearch.knn.common.KNNConstants.KNN_ENGINE; +import static org.opensearch.knn.common.KNNConstants.LUCENE_NAME; +import static org.opensearch.knn.common.KNNConstants.METHOD_HNSW; +import static org.opensearch.knn.common.KNNConstants.METHOD_IVF; +import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_NLIST; +import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_SPACE_TYPE; +import static org.opensearch.knn.common.KNNConstants.MODEL_ID; +import static org.opensearch.knn.common.KNNConstants.NAME; +import static org.opensearch.knn.common.KNNConstants.NMSLIB_NAME; +import static org.opensearch.knn.common.KNNConstants.PARAMETERS; import static org.opensearch.knn.plugin.stats.KNNStatsConfig.KNN_STATS; /** @@ -41,8 +58,20 @@ public class RestKNNStatsHandlerIT extends KNNRestTestCase { private static final Logger logger = LogManager.getLogger(RestKNNStatsHandlerIT.class); + private static final String TRAINING_INDEX = "training-index"; + private static final String TRAINING_FIELD = "training-field"; + private static final String TEST_MODEL_ID = "model-id"; + private static final String TEST_INDEX = "test-index"; + private static final String MODEL_DESCRIPTION = "Description for train model test"; private boolean isDebuggingTest = new DisableOnDebug(null).isDebugging(); private boolean isDebuggingRemoteCluster = System.getProperty("cluster.debug", "false").equals("true"); + private static final String FIELD_NAME_2 = "test_field_two"; + private static final String FIELD_NAME_3 = "test_field_three"; + private static final int DIMENSION = 4; + private static int DOC_ID = 0; + private static final int NUM_DOCS = 10; + private static final int DELAY_MILLI_SEC = 1000; + private static final int NUM_OF_ATTEMPTS = 30; private KNNStats knnStats; @@ -333,6 +362,98 @@ public void testModelIndexingDegradedMetricsStats() throws IOException { assertEquals(false, nodeStats.get(statName)); } + /** + * Test checks that handler correctly returns value for field per engine stats + * + * @throws IOException throws IOException + */ + public void testFieldByEngineStats() throws Exception { + createKnnIndex(INDEX_NAME, createKnnIndexMapping(FIELD_NAME, 2, METHOD_HNSW, NMSLIB_NAME)); + putMappingRequest(INDEX_NAME, createKnnIndexMapping(FIELD_NAME_2, 3, METHOD_HNSW, LUCENE_NAME)); + putMappingRequest(INDEX_NAME, createKnnIndexMapping(FIELD_NAME_3, 3, METHOD_HNSW, FAISS_NAME)); + + Response response = getKnnStats(Collections.emptyList(), Collections.emptyList()); + + String responseBody = EntityUtils.toString(response.getEntity()); + + Map nodeStats0 = parseNodeStatsResponse(responseBody).get(0); + boolean faissField = (Boolean) nodeStats0.get(StatNames.FAISS_LOADED.getName()); + boolean luceneField = (Boolean) nodeStats0.get(StatNames.LUCENE_LOADED.getName()); + boolean nmslibField = (Boolean) nodeStats0.get(StatNames.NMSLIB_LOADED.getName()); + + assertTrue(faissField); + assertTrue(luceneField); + assertTrue(nmslibField); + } + + public void testFieldsByEngineModelTraining() throws Exception { + createBasicKnnIndex(TRAINING_INDEX, TRAINING_FIELD, DIMENSION); + bulkIngestRandomVectors(TRAINING_INDEX, TRAINING_FIELD, NUM_DOCS, DIMENSION); + trainKnnModel(TEST_MODEL_ID, TRAINING_INDEX, TRAINING_FIELD, DIMENSION, MODEL_DESCRIPTION); + + validateModelCreated(TEST_MODEL_ID); + + createKnnIndex(TEST_INDEX, modelIndexMapping(FIELD_NAME, TEST_MODEL_ID)); + + addKNNDocs(TEST_INDEX, FIELD_NAME, DIMENSION, DOC_ID, NUM_DOCS); + + final Response response = getKnnStats(Collections.emptyList(), Collections.emptyList()); + final String responseBody = EntityUtils.toString(response.getEntity()); + + final Map nodeStats0 = parseNodeStatsResponse(responseBody).get(0); + + boolean faissField = (Boolean) nodeStats0.get(StatNames.FAISS_LOADED.getName()); + + assertTrue(faissField); + } + + public void trainKnnModel(String modelId, String trainingIndexName, String trainingFieldName, int dimension, String description) + throws IOException { + XContentBuilder builder = XContentFactory.jsonBuilder() + .startObject() + .field(NAME, METHOD_IVF) + .field(KNN_ENGINE, FAISS_NAME) + .field(METHOD_PARAMETER_SPACE_TYPE, SpaceType.L2.getValue()) + .startObject(PARAMETERS) + .field(METHOD_PARAMETER_NLIST, 1) + .endObject() + .endObject(); + Map method = xContentBuilderToMap(builder); + + Response trainResponse = trainModel(modelId, trainingIndexName, trainingFieldName, dimension, method, description); + assertEquals(RestStatus.OK, RestStatus.fromCode(trainResponse.getStatusLine().getStatusCode())); + } + + public void validateModelCreated(String modelId) throws IOException, InterruptedException { + Response getResponse = getModel(modelId, null); + String responseBody = EntityUtils.toString(getResponse.getEntity()); + assertNotNull(responseBody); + + Map responseMap = createParser(XContentType.JSON.xContent(), responseBody).map(); + assertEquals(modelId, responseMap.get(MODEL_ID)); + assertTrainingSucceeds(modelId, NUM_OF_ATTEMPTS, DELAY_MILLI_SEC); + } + + // mapping to create index from model + public String modelIndexMapping(String fieldName, String modelId) throws IOException { + return Strings.toString( + XContentFactory.jsonBuilder() + .startObject() + .startObject(PROPERTIES) + .startObject(fieldName) + .field(VECTOR_TYPE, KNN_VECTOR) + .field(MODEL_ID, modelId) + .endObject() + .endObject() + .endObject() + ); + } + + @Override + protected boolean preserveClusterUponCompletion() { + return false; + } + // Useful settings when debugging to prevent timeouts @Override protected Settings restClientSettings() { diff --git a/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java b/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java index 3edf508ab..797b038d9 100644 --- a/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java +++ b/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java @@ -297,6 +297,27 @@ protected String createKnnIndexMapping(String fieldName, Integer dimensions) thr ); } + /** + * Utility to create a Knn Index Mapping with specific algorithm and engine + */ + protected String createKnnIndexMapping(String fieldName, Integer dimensions, String algoName, String knnEngine) throws IOException { + return Strings.toString( + XContentFactory.jsonBuilder() + .startObject() + .startObject("properties") + .startObject(fieldName) + .field("type", "knn_vector") + .field("dimension", dimensions.toString()) + .startObject("method") + .field("name", algoName) + .field("engine", knnEngine) + .endObject() + .endObject() + .endObject() + .endObject() + ); + } + /** * Utility to create a Knn Index Mapping with multiple k-NN fields */ From 3ae77b2e716aa8af69ee638dc67f924ad3046200 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Tue, 13 Sep 2022 17:33:40 -0700 Subject: [PATCH 033/416] Fixed the pull request links in the k-nn release notes 2.3.0. (#549) (#550) Signed-off-by: Navneet Verma Signed-off-by: Navneet Verma (cherry picked from commit 7fd6d784d25433a74d6764228b8d8f6ea1f17f89) Co-authored-by: Navneet Verma --- .../opensearch-knn.release-notes-2.3.0.0.md | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/release-notes/opensearch-knn.release-notes-2.3.0.0.md b/release-notes/opensearch-knn.release-notes-2.3.0.0.md index 57f6da308..eda123e9c 100644 --- a/release-notes/opensearch-knn.release-notes-2.3.0.0.md +++ b/release-notes/opensearch-knn.release-notes-2.3.0.0.md @@ -3,11 +3,15 @@ Compatible with OpenSearch 2.3.0 ### Enhancements -* Change initial size of DocIdSetBuilder (#502) +* Change initial size of DocIdSetBuilder ([#502](https://github.com/opensearch-project/k-NN/pull/502)) ### Bug Fixes -* Remove overallocation in faiss query path (#501) +* Remove overallocation in faiss query path ([#501](https://github.com/opensearch-project/k-NN/pull/501)) ### Refactoring -* Replace terminology 'master' with 'cluster manager' (#521) -* Nomenclature changes from Whitelist to Allowlist (#534) +* Replace terminology 'master' with 'cluster manager' ([#521](https://github.com/opensearch-project/k-NN/pull/521)) +* Nomenclature changes from Whitelist to Allowlist ([#534](https://github.com/opensearch-project/k-NN/pull/534)) + +### Maintenance +* Updated the BWC workflow to have 2.2.0 as the backward supported version in BWC tests ([#536](https://github.com/opensearch-project/k-NN/pull/536)) +* [AUTO] Increment version to 2.3.0-SNAPSHOT ([#526](https://github.com/opensearch-project/k-NN/pull/526)) From 60c5527923396cd356d75ae7c19df1a48b17ce56 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Thu, 15 Sep 2022 09:26:07 -0700 Subject: [PATCH 034/416] Add OSB index specification json for lucene hnsw (#552) (#553) Signed-off-by: Martin Gaievski (cherry picked from commit cfc6c4463849ddb216d1f4cdd991aa2dec849512) --- benchmarks/osb/indices/lucene-index.json | 26 ++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 benchmarks/osb/indices/lucene-index.json diff --git a/benchmarks/osb/indices/lucene-index.json b/benchmarks/osb/indices/lucene-index.json new file mode 100644 index 000000000..0a4ed868a --- /dev/null +++ b/benchmarks/osb/indices/lucene-index.json @@ -0,0 +1,26 @@ +{ + "settings": { + "index": { + "knn": true, + "number_of_shards": {{ target_index_primary_shards }}, + "number_of_replicas": {{ target_index_replica_shards }} + } + }, + "mappings": { + "properties": { + "target_field": { + "type": "knn_vector", + "dimension": {{ target_index_dimension }}, + "method": { + "name": "hnsw", + "space_type": "{{ target_index_space_type }}", + "engine": "lucene", + "parameters": { + "ef_construction": {{ hnsw_ef_construction }}, + "m": {{ hnsw_m }} + } + } + } + } + } +} From 9323679d3595bcbfc3ba7258e9bf541c2f77e60d Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Thu, 15 Sep 2022 20:50:04 -0700 Subject: [PATCH 035/416] Update dev guide with instructions for mac (#556) Signed-off-by: John Mazanec (cherry picked from commit 0598ca6be2afac439954bbd4cc3d0b9605fcf177) --- DEVELOPER_GUIDE.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index 291b902b8..1a47abef8 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -3,6 +3,9 @@ - [Fork OpenSearch k-NN Repo](#fork-opensearch-k-nn-repo) - [Install Prerequisites](#install-prerequisites) - [JDK 11](#jdk-11) + - [CMake](#cmake) + - [Faiss Dependencies](#Faiss-Dependencies) + - [Environment](#Environment) - [Use an Editor](#use-an-editor) - [IntelliJ IDEA](#intellij-idea) - [Build](#build) @@ -55,11 +58,26 @@ In addition to this, the plugin has been tested with JDK 17, and this JDK versio The plugin requires that cmake >= 3.17.2 is installed in order to build the JNI libraries. +One easy way to install on mac or linux is to use pip: +```bash +pip install cmake==3.17.2 +``` + #### Faiss Dependencies To build the *faiss* JNI library, you need to have openmp, lapack and blas installed. For more information on *faiss* dependencies, please refer to [their documentation](https://github.com/facebookresearch/faiss/blob/main/INSTALL.md). +[Openblas](https://www.openblas.net/) can be used for both lapack and blas. To install on Mac, run: +```bash +brew install openblas +``` + +Additionally, the `gcc` toolchain needs to be installed on Mac. To install, run: +```bash +brew install gcc +``` + #### Environment Currently, the plugin only supports Linux on x64 and arm platforms. From a58af66efa06e5fb0601edadd8f607166d02d759 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Fri, 16 Sep 2022 10:35:44 -0700 Subject: [PATCH 036/416] Increment version to 2.4.0-SNAPSHOT (#545) Signed-off-by: opensearch-ci-bot Signed-off-by: opensearch-ci-bot Co-authored-by: opensearch-ci-bot --- .github/workflows/backwards_compatibility_tests_workflow.yml | 4 ++-- build.gradle | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/backwards_compatibility_tests_workflow.yml b/.github/workflows/backwards_compatibility_tests_workflow.yml index a1d15afaa..47ef18a8e 100644 --- a/.github/workflows/backwards_compatibility_tests_workflow.yml +++ b/.github/workflows/backwards_compatibility_tests_workflow.yml @@ -15,7 +15,7 @@ jobs: matrix: java: [ 11, 17 ] bwc_version : [ "1.1.0", "1.2.4", "1.3.2", "2.0.0", "2.1.0", "2.2.0" ] - opensearch_version : [ "2.3.0-SNAPSHOT" ] + opensearch_version : [ "2.4.0-SNAPSHOT" ] name: k-NN Restart-Upgrade BWC Tests runs-on: ubuntu-latest @@ -47,7 +47,7 @@ jobs: matrix: java: [ 11, 17 ] bwc_version: [ "1.3.2", "2.0.0", "2.1.0", "2.2.0" ] - opensearch_version: [ "2.3.0-SNAPSHOT" ] + opensearch_version: [ "2.4.0-SNAPSHOT" ] name: k-NN Rolling-Upgrade BWC Tests runs-on: ubuntu-latest diff --git a/build.gradle b/build.gradle index e5dc1b662..dbba57602 100644 --- a/build.gradle +++ b/build.gradle @@ -9,7 +9,7 @@ buildscript { ext { // build.version_qualifier parameter applies to knn plugin artifacts only. OpenSearch version must be set // explicitly as 'opensearch.version' property, for instance opensearch.version=2.0.0-rc1-SNAPSHOT - opensearch_version = System.getProperty("opensearch.version", "2.3.0-SNAPSHOT") + opensearch_version = System.getProperty("opensearch.version", "2.4.0-SNAPSHOT") version_qualifier = System.getProperty("build.version_qualifier", "") opensearch_group = "org.opensearch" } From 7ff45e34de9633ae0fd0b59a235ef12f56f20eec Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Fri, 16 Sep 2022 14:02:41 -0700 Subject: [PATCH 037/416] Added sample perf-test configs for faiss-ivf, faiss-ivfpq, lucene-hnsw (#555) (#559) Signed-off-by: Navneet Verma Signed-off-by: Navneet Verma (cherry picked from commit a19149a7bcfee460c7338160f5cf84deee8bffc3) Co-authored-by: Navneet Verma --- .../release-configs/faiss-ivf/index.json | 17 ++++++ .../faiss-ivf/method-spec.json | 9 ++++ .../release-configs/faiss-ivf/test.yml | 53 +++++++++++++++++++ .../faiss-ivf/train-index-spec.json | 16 ++++++ .../release-configs/faiss-ivfpq/index.json | 17 ++++++ .../faiss-ivfpq/method-spec.json | 16 ++++++ .../release-configs/faiss-ivfpq/test.yml | 53 +++++++++++++++++++ .../faiss-ivfpq/train-index-spec.json | 16 ++++++ .../release-configs/lucene-hnsw/index.json | 26 +++++++++ .../release-configs/lucene-hnsw/test.yml | 29 ++++++++++ .../release-configs/nmslib-hnsw/index.json | 27 ++++++++++ .../release-configs/nmslib-hnsw/test.yml | 29 ++++++++++ 12 files changed, 308 insertions(+) create mode 100644 benchmarks/perf-tool/release-configs/faiss-ivf/index.json create mode 100644 benchmarks/perf-tool/release-configs/faiss-ivf/method-spec.json create mode 100644 benchmarks/perf-tool/release-configs/faiss-ivf/test.yml create mode 100644 benchmarks/perf-tool/release-configs/faiss-ivf/train-index-spec.json create mode 100644 benchmarks/perf-tool/release-configs/faiss-ivfpq/index.json create mode 100644 benchmarks/perf-tool/release-configs/faiss-ivfpq/method-spec.json create mode 100644 benchmarks/perf-tool/release-configs/faiss-ivfpq/test.yml create mode 100644 benchmarks/perf-tool/release-configs/faiss-ivfpq/train-index-spec.json create mode 100644 benchmarks/perf-tool/release-configs/lucene-hnsw/index.json create mode 100644 benchmarks/perf-tool/release-configs/lucene-hnsw/test.yml create mode 100644 benchmarks/perf-tool/release-configs/nmslib-hnsw/index.json create mode 100644 benchmarks/perf-tool/release-configs/nmslib-hnsw/test.yml diff --git a/benchmarks/perf-tool/release-configs/faiss-ivf/index.json b/benchmarks/perf-tool/release-configs/faiss-ivf/index.json new file mode 100644 index 000000000..479703412 --- /dev/null +++ b/benchmarks/perf-tool/release-configs/faiss-ivf/index.json @@ -0,0 +1,17 @@ +{ + "settings": { + "index": { + "knn": true, + "number_of_shards": 24, + "number_of_replicas": 1 + } + }, + "mappings": { + "properties": { + "target_field": { + "type": "knn_vector", + "model_id": "test-model" + } + } + } +} diff --git a/benchmarks/perf-tool/release-configs/faiss-ivf/method-spec.json b/benchmarks/perf-tool/release-configs/faiss-ivf/method-spec.json new file mode 100644 index 000000000..51ae89877 --- /dev/null +++ b/benchmarks/perf-tool/release-configs/faiss-ivf/method-spec.json @@ -0,0 +1,9 @@ +{ + "name":"ivf", + "engine":"faiss", + "space_type": "l2", + "parameters":{ + "nlist": 128, + "nprobes": 8 + } +} diff --git a/benchmarks/perf-tool/release-configs/faiss-ivf/test.yml b/benchmarks/perf-tool/release-configs/faiss-ivf/test.yml new file mode 100644 index 000000000..ce6b78867 --- /dev/null +++ b/benchmarks/perf-tool/release-configs/faiss-ivf/test.yml @@ -0,0 +1,53 @@ +endpoint: [END-POINT] +test_name: "index-workflow" +test_id: "index workflow" +num_runs: 10 +show_runs: false +setup: + - name: delete_index + index_name: train_index + - name: create_index + index_name: train_index + index_spec: /home/ec2-user/[PATH]/train-index-spec.json + - name: ingest + index_name: train_index + field_name: train_field + bulk_size: 500 + dataset_format: hdf5 + dataset_path: /home/ec2-user/[PATH]/sift-128-euclidean.hdf5 + doc_count: 50000 + - name: refresh_index + index_name: train_index +steps: + - name: delete_model + model_id: test-model + - name: delete_index + index_name: target_index + - name: train_model + model_id: test-model + train_index: train_index + train_field: train_field + dimension: 128 + method_spec: /home/ec2-user/[PATH]/method-spec.json + max_training_vector_count: 50000 + - name: create_index + index_name: target_index + index_spec: /home/ec2-user/[PATH]/index.json + - name: ingest + index_name: target_index + field_name: target_field + bulk_size: 500 + dataset_format: hdf5 + dataset_path: /home/ec2-user/data/sift-128-euclidean.hdf5 + - name: refresh_index + index_name: target_index + - name: query + k: 100 + r: 1 + calculate_recall: true + index_name: target_index + field_name: target_field + dataset_format: hdf5 + dataset_path: /home/ec2-user/data/sift-128-euclidean.hdf5 + neighbors_format: hdf5 + neighbors_path: /home/ec2-user/data/sift-128-euclidean.hdf5 diff --git a/benchmarks/perf-tool/release-configs/faiss-ivf/train-index-spec.json b/benchmarks/perf-tool/release-configs/faiss-ivf/train-index-spec.json new file mode 100644 index 000000000..804a5707e --- /dev/null +++ b/benchmarks/perf-tool/release-configs/faiss-ivf/train-index-spec.json @@ -0,0 +1,16 @@ +{ + "settings": { + "index": { + "number_of_shards": 24, + "number_of_replicas": 0 + } + }, + "mappings": { + "properties": { + "train_field": { + "type": "knn_vector", + "dimension": 128 + } + } + } +} diff --git a/benchmarks/perf-tool/release-configs/faiss-ivfpq/index.json b/benchmarks/perf-tool/release-configs/faiss-ivfpq/index.json new file mode 100644 index 000000000..479703412 --- /dev/null +++ b/benchmarks/perf-tool/release-configs/faiss-ivfpq/index.json @@ -0,0 +1,17 @@ +{ + "settings": { + "index": { + "knn": true, + "number_of_shards": 24, + "number_of_replicas": 1 + } + }, + "mappings": { + "properties": { + "target_field": { + "type": "knn_vector", + "model_id": "test-model" + } + } + } +} diff --git a/benchmarks/perf-tool/release-configs/faiss-ivfpq/method-spec.json b/benchmarks/perf-tool/release-configs/faiss-ivfpq/method-spec.json new file mode 100644 index 000000000..204b0a653 --- /dev/null +++ b/benchmarks/perf-tool/release-configs/faiss-ivfpq/method-spec.json @@ -0,0 +1,16 @@ +{ + "name":"ivf", + "engine":"faiss", + "space_type": "l2", + "parameters":{ + "nlist": 128, + "nprobes": 8, + "encoder": { + "name": "pq", + "parameters": { + "m": 16, + "code_size": 8 + } + } + } +} diff --git a/benchmarks/perf-tool/release-configs/faiss-ivfpq/test.yml b/benchmarks/perf-tool/release-configs/faiss-ivfpq/test.yml new file mode 100644 index 000000000..dd88affc3 --- /dev/null +++ b/benchmarks/perf-tool/release-configs/faiss-ivfpq/test.yml @@ -0,0 +1,53 @@ +endpoint: [ENDPOINT] +test_name: "index-workflow" +test_id: "index workflow" +num_runs: 10 +show_runs: false +setup: + - name: delete_index + index_name: train_index + - name: create_index + index_name: train_index + index_spec: /home/ec2-user/[PATH]/train-index-spec.json + - name: ingest + index_name: train_index + field_name: train_field + bulk_size: 500 + dataset_format: hdf5 + dataset_path: /home/ec2-user/data/sift-128-euclidean.hdf5 + doc_count: 50000 + - name: refresh_index + index_name: train_index +steps: + - name: delete_model + model_id: test-model + - name: delete_index + index_name: target_index + - name: train_model + model_id: test-model + train_index: train_index + train_field: train_field + dimension: 128 + method_spec: /home/ec2-user/[PATH]/method-spec.json + max_training_vector_count: 50000 + - name: create_index + index_name: target_index + index_spec: /home/ec2-user/[PATH]/index.json + - name: ingest + index_name: target_index + field_name: target_field + bulk_size: 500 + dataset_format: hdf5 + dataset_path: /home/ec2-user/data/sift-128-euclidean.hdf5 + - name: refresh_index + index_name: target_index + - name: query + k: 100 + r: 1 + calculate_recall: true + index_name: target_index + field_name: target_field + dataset_format: hdf5 + dataset_path: /home/ec2-user/data/sift-128-euclidean.hdf5 + neighbors_format: hdf5 + neighbors_path: /home/ec2-user/data/sift-128-euclidean.hdf5 diff --git a/benchmarks/perf-tool/release-configs/faiss-ivfpq/train-index-spec.json b/benchmarks/perf-tool/release-configs/faiss-ivfpq/train-index-spec.json new file mode 100644 index 000000000..804a5707e --- /dev/null +++ b/benchmarks/perf-tool/release-configs/faiss-ivfpq/train-index-spec.json @@ -0,0 +1,16 @@ +{ + "settings": { + "index": { + "number_of_shards": 24, + "number_of_replicas": 0 + } + }, + "mappings": { + "properties": { + "train_field": { + "type": "knn_vector", + "dimension": 128 + } + } + } +} diff --git a/benchmarks/perf-tool/release-configs/lucene-hnsw/index.json b/benchmarks/perf-tool/release-configs/lucene-hnsw/index.json new file mode 100644 index 000000000..7a9ff2890 --- /dev/null +++ b/benchmarks/perf-tool/release-configs/lucene-hnsw/index.json @@ -0,0 +1,26 @@ +{ + "settings": { + "index": { + "knn": true, + "number_of_shards": 24, + "number_of_replicas": 1 + } + }, + "mappings": { + "properties": { + "target_field": { + "type": "knn_vector", + "dimension": 128, + "method": { + "name": "hnsw", + "space_type": "l2", + "engine": "lucene", + "parameters": { + "ef_construction": 256, + "m": 16 + } + } + } + } + } +} diff --git a/benchmarks/perf-tool/release-configs/lucene-hnsw/test.yml b/benchmarks/perf-tool/release-configs/lucene-hnsw/test.yml new file mode 100644 index 000000000..96b991325 --- /dev/null +++ b/benchmarks/perf-tool/release-configs/lucene-hnsw/test.yml @@ -0,0 +1,29 @@ +endpoint: [ENDPOINT] +test_name: "index-workflow" +test_id: "Index workflow" +num_runs: 10 +show_runs: false +steps: + - name: delete_index + index_name: target_index + - name: create_index + index_name: target_index + index_spec: /home/ec2-user/[PATH]/index.json + - name: ingest + index_name: target_index + field_name: target_field + bulk_size: 500 + dataset_format: hdf5 + dataset_path: /home/ec2-user/data/sift-128-euclidean.hdf5 + - name: refresh_index + index_name: target_index + - name: query + k: 100 + r: 1 + calculate_recall: true + index_name: target_index + field_name: target_field + dataset_format: hdf5 + dataset_path: /home/ec2-user/data/sift-128-euclidean.hdf5 + neighbors_format: hdf5 + neighbors_path: /home/ec2-user/data/sift-128-euclidean.hdf5 diff --git a/benchmarks/perf-tool/release-configs/nmslib-hnsw/index.json b/benchmarks/perf-tool/release-configs/nmslib-hnsw/index.json new file mode 100644 index 000000000..fa88192f6 --- /dev/null +++ b/benchmarks/perf-tool/release-configs/nmslib-hnsw/index.json @@ -0,0 +1,27 @@ +{ + "settings": { + "index": { + "knn": true, + "number_of_shards": 24, + "number_of_replicas": 1, + "knn.algo_param.ef_search": 256 + } + }, + "mappings": { + "properties": { + "target_field": { + "type": "knn_vector", + "dimension": 128, + "method": { + "name": "hnsw", + "space_type": "l2", + "engine": "nmslib", + "parameters": { + "ef_construction": 256, + "m": 16 + } + } + } + } + } +} diff --git a/benchmarks/perf-tool/release-configs/nmslib-hnsw/test.yml b/benchmarks/perf-tool/release-configs/nmslib-hnsw/test.yml new file mode 100644 index 000000000..96b991325 --- /dev/null +++ b/benchmarks/perf-tool/release-configs/nmslib-hnsw/test.yml @@ -0,0 +1,29 @@ +endpoint: [ENDPOINT] +test_name: "index-workflow" +test_id: "Index workflow" +num_runs: 10 +show_runs: false +steps: + - name: delete_index + index_name: target_index + - name: create_index + index_name: target_index + index_spec: /home/ec2-user/[PATH]/index.json + - name: ingest + index_name: target_index + field_name: target_field + bulk_size: 500 + dataset_format: hdf5 + dataset_path: /home/ec2-user/data/sift-128-euclidean.hdf5 + - name: refresh_index + index_name: target_index + - name: query + k: 100 + r: 1 + calculate_recall: true + index_name: target_index + field_name: target_field + dataset_format: hdf5 + dataset_path: /home/ec2-user/data/sift-128-euclidean.hdf5 + neighbors_format: hdf5 + neighbors_path: /home/ec2-user/data/sift-128-euclidean.hdf5 From 52764cb756c9e15706762e0fcdbf179a48fc48ae Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Tue, 20 Sep 2022 12:16:57 -0700 Subject: [PATCH 038/416] Refactor unit tests for codec (#562) (#563) * Refactor unit test for codec for easier lucene version upgrades Signed-off-by: Martin Gaievski --- .../index/codec/KNN920Codec/KNN920Codec.java | 5 +- .../codec/KNN920Codec/KNN920CodecTests.java | 122 ++---------------- .../knn/index/codec/KNNCodecTestCase.java | 108 ++++++++++++++++ 3 files changed, 119 insertions(+), 116 deletions(-) diff --git a/src/main/java/org/opensearch/knn/index/codec/KNN920Codec/KNN920Codec.java b/src/main/java/org/opensearch/knn/index/codec/KNN920Codec/KNN920Codec.java index 8d8e664ce..26abcea60 100644 --- a/src/main/java/org/opensearch/knn/index/codec/KNN920Codec/KNN920Codec.java +++ b/src/main/java/org/opensearch/knn/index/codec/KNN920Codec/KNN920Codec.java @@ -11,6 +11,7 @@ import org.apache.lucene.codecs.DocValuesFormat; import org.apache.lucene.codecs.FilterCodec; import org.apache.lucene.codecs.KnnVectorsFormat; +import org.apache.lucene.codecs.perfield.PerFieldKnnVectorsFormat; import org.opensearch.knn.index.codec.KNNFormatFacade; import org.opensearch.knn.index.codec.KNNFormatFactory; @@ -27,7 +28,7 @@ public final class KNN920Codec extends FilterCodec { private static final String KNN920 = "KNN920Codec"; private final KNNFormatFacade knnFormatFacade; - private final KNN920PerFieldKnnVectorsFormat perFieldKnnVectorsFormat; + private final PerFieldKnnVectorsFormat perFieldKnnVectorsFormat; /** * No arg constructor that uses Lucene91 as the delegate @@ -43,7 +44,7 @@ public KNN920Codec() { * @param knnVectorsFormat per field format for KnnVector */ @Builder - public KNN920Codec(Codec delegate, KNN920PerFieldKnnVectorsFormat knnVectorsFormat) { + public KNN920Codec(Codec delegate, PerFieldKnnVectorsFormat knnVectorsFormat) { super(KNN920, delegate); knnFormatFacade = KNNFormatFactory.createKNN920Format(delegate); perFieldKnnVectorsFormat = knnVectorsFormat; diff --git a/src/test/java/org/opensearch/knn/index/codec/KNN920Codec/KNN920CodecTests.java b/src/test/java/org/opensearch/knn/index/codec/KNN920Codec/KNN920CodecTests.java index 8a3233fba..e4f848f6a 100644 --- a/src/test/java/org/opensearch/knn/index/codec/KNN920Codec/KNN920CodecTests.java +++ b/src/test/java/org/opensearch/knn/index/codec/KNN920Codec/KNN920CodecTests.java @@ -5,43 +5,16 @@ package org.opensearch.knn.index.codec.KNN920Codec; -import org.apache.lucene.document.Document; -import org.apache.lucene.document.FieldType; -import org.apache.lucene.document.KnnVectorField; -import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.IndexWriterConfig; -import org.apache.lucene.index.SerialMergeScheduler; -import org.apache.lucene.index.VectorSimilarityFunction; -import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.Query; -import org.apache.lucene.store.Directory; -import org.apache.lucene.tests.index.RandomIndexWriter; +import org.apache.lucene.codecs.Codec; +import org.apache.lucene.codecs.perfield.PerFieldKnnVectorsFormat; import org.opensearch.index.mapper.MapperService; -import org.opensearch.knn.index.KNNMethodContext; -import org.opensearch.knn.index.MethodComponentContext; -import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.codec.KNNCodecTestCase; -import org.opensearch.knn.index.mapper.KNNVectorFieldMapper; -import org.opensearch.knn.index.memory.NativeMemoryLoadStrategy; -import org.opensearch.knn.index.query.KNNQueryFactory; -import org.opensearch.knn.index.util.KNNEngine; -import org.opensearch.watcher.ResourceWatcherService; import java.io.IOException; -import java.util.Map; import java.util.Optional; import java.util.concurrent.ExecutionException; +import java.util.function.Function; -import static org.mockito.ArgumentMatchers.anyString; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.spy; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; -import static org.opensearch.knn.common.KNNConstants.HNSW_ALGO_EF_CONSTRUCTION; -import static org.opensearch.knn.common.KNNConstants.HNSW_ALGO_M; -import static org.opensearch.knn.common.KNNConstants.METHOD_HNSW; import static org.opensearch.knn.index.codec.KNNCodecFactory.CodecDelegateFactory.createKNN92DefaultDelegate; public class KNN920CodecTests extends KNNCodecTestCase { @@ -55,93 +28,14 @@ public void testBuildFromModelTemplate() throws InterruptedException, ExecutionE } public void testKnnVectorIndex() throws Exception { - final String fieldName = "test_vector"; - final String field1Name = "my_vector"; - final MapperService mapperService = mock(MapperService.class); - final KNNMethodContext knnMethodContext = new KNNMethodContext( - KNNEngine.LUCENE, - SpaceType.L2, - new MethodComponentContext(METHOD_HNSW, Map.of(HNSW_ALGO_M, 16, HNSW_ALGO_EF_CONSTRUCTION, 256)) - ); - final KNNVectorFieldMapper.KNNVectorFieldType mappedFieldType1 = new KNNVectorFieldMapper.KNNVectorFieldType( - fieldName, - Map.of(), - 3, - knnMethodContext - ); - final KNNVectorFieldMapper.KNNVectorFieldType mappedFieldType2 = new KNNVectorFieldMapper.KNNVectorFieldType( - field1Name, - Map.of(), - 2, - knnMethodContext - ); - when(mapperService.fieldType(eq(fieldName))).thenReturn(mappedFieldType1); - when(mapperService.fieldType(eq(field1Name))).thenReturn(mappedFieldType2); + Function perFieldKnnVectorsFormatProvider = ( + mapperService) -> new KNN920PerFieldKnnVectorsFormat(Optional.of(mapperService)); - var knnVectorsFormat = spy(new KNN920PerFieldKnnVectorsFormat(Optional.of(mapperService))); - - final KNN920Codec actualCodec = KNN920Codec.builder() + Function knnCodecProvider = (knnVectorFormat) -> KNN920Codec.builder() .delegate(createKNN92DefaultDelegate()) - .knnVectorsFormat(knnVectorsFormat) + .knnVectorsFormat(knnVectorFormat) .build(); - final KNN920Codec codec = KNN920Codec.builder().delegate(createKNN92DefaultDelegate()).knnVectorsFormat(knnVectorsFormat).build(); - setUpMockClusterService(); - Directory dir = newFSDirectory(createTempDir()); - IndexWriterConfig iwc = newIndexWriterConfig(); - iwc.setMergeScheduler(new SerialMergeScheduler()); - iwc.setCodec(codec); - - /** - * Add doc with field "test_vector" - */ - final FieldType luceneFieldType = KnnVectorField.createFieldType(3, VectorSimilarityFunction.EUCLIDEAN); - float[] array = { 1.0f, 3.0f, 4.0f }; - KnnVectorField vectorField = new KnnVectorField(fieldName, array, luceneFieldType); - RandomIndexWriter writer = new RandomIndexWriter(random(), dir, iwc); - Document doc = new Document(); - doc.add(vectorField); - writer.addDocument(doc); - writer.commit(); - IndexReader reader = writer.getReader(); - writer.close(); - - verify(knnVectorsFormat).getKnnVectorsFormatForField(anyString()); - - IndexSearcher searcher = new IndexSearcher(reader); - Query query = KNNQueryFactory.create(KNNEngine.LUCENE, "dummy", fieldName, new float[] { 1.0f, 0.0f, 0.0f }, 1); - - assertEquals(1, searcher.count(query)); - - reader.close(); - - /** - * Add doc with field "my_vector" - */ - IndexWriterConfig iwc1 = newIndexWriterConfig(); - iwc1.setMergeScheduler(new SerialMergeScheduler()); - iwc1.setCodec(actualCodec); - writer = new RandomIndexWriter(random(), dir, iwc1); - final FieldType luceneFieldType1 = KnnVectorField.createFieldType(2, VectorSimilarityFunction.EUCLIDEAN); - float[] array1 = { 6.0f, 14.0f }; - KnnVectorField vectorField1 = new KnnVectorField(field1Name, array1, luceneFieldType1); - Document doc1 = new Document(); - doc1.add(vectorField1); - writer.addDocument(doc1); - IndexReader reader1 = writer.getReader(); - writer.close(); - ResourceWatcherService resourceWatcherService = createDisabledResourceWatcherService(); - NativeMemoryLoadStrategy.IndexLoadStrategy.initialize(resourceWatcherService); - - verify(knnVectorsFormat, times(2)).getKnnVectorsFormatForField(anyString()); - - IndexSearcher searcher1 = new IndexSearcher(reader1); - Query query1 = KNNQueryFactory.create(KNNEngine.LUCENE, "dummy", field1Name, new float[] { 1.0f, 0.0f }, 1); - - assertEquals(1, searcher1.count(query1)); - reader1.close(); - dir.close(); - resourceWatcherService.close(); - NativeMemoryLoadStrategy.IndexLoadStrategy.getInstance().close(); + testKnnVectorIndex(knnCodecProvider, perFieldKnnVectorsFormatProvider); } } diff --git a/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestCase.java b/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestCase.java index fce944ee6..7991db135 100644 --- a/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestCase.java +++ b/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestCase.java @@ -7,11 +7,19 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; +import org.apache.lucene.codecs.perfield.PerFieldKnnVectorsFormat; +import org.apache.lucene.document.KnnVectorField; +import org.apache.lucene.index.VectorSimilarityFunction; +import org.apache.lucene.search.Query; import org.apache.lucene.search.TopDocs; import org.opensearch.common.settings.ClusterSettings; +import org.opensearch.index.mapper.MapperService; import org.opensearch.knn.KNNTestCase; import org.opensearch.knn.common.KNNConstants; +import org.opensearch.knn.index.KNNMethodContext; +import org.opensearch.knn.index.MethodComponentContext; import org.opensearch.knn.index.codec.KNN920Codec.KNN920Codec; +import org.opensearch.knn.index.query.KNNQueryFactory; import org.opensearch.knn.jni.JNIService; import org.opensearch.knn.index.query.KNNQuery; import org.opensearch.knn.index.KNNSettings; @@ -45,14 +53,24 @@ import java.time.ZonedDateTime; import java.util.Arrays; import java.util.List; +import java.util.Map; import java.util.concurrent.ExecutionException; +import java.util.function.Function; import java.util.stream.Collectors; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.RETURNS_DEEP_STUBS; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import static org.opensearch.Version.CURRENT; +import static org.opensearch.knn.common.KNNConstants.HNSW_ALGO_EF_CONSTRUCTION; +import static org.opensearch.knn.common.KNNConstants.HNSW_ALGO_M; import static org.opensearch.knn.common.KNNConstants.INDEX_DESCRIPTION_PARAMETER; +import static org.opensearch.knn.common.KNNConstants.METHOD_HNSW; import static org.opensearch.knn.common.KNNConstants.SPACE_TYPE; import static org.opensearch.knn.index.KNNSettings.MODEL_CACHE_SIZE_LIMIT_SETTING; @@ -72,6 +90,8 @@ public class KNNCodecTestCase extends KNNTestCase { sampleFieldType.putAttribute(KNNConstants.HNSW_ALGO_EF_CONSTRUCTION, "512"); sampleFieldType.freeze(); } + private static final String FIELD_NAME_ONE = "test_vector_one"; + private static final String FIELD_NAME_TWO = "test_vector_two"; protected void setUpMockClusterService() { ClusterService clusterService = mock(ClusterService.class, RETURNS_DEEP_STUBS); @@ -253,4 +273,92 @@ public void testWriteByOldCodec(Codec codec) throws IOException { dir.close(); NativeMemoryLoadStrategy.IndexLoadStrategy.getInstance().close(); } + + public void testKnnVectorIndex( + final Function codecProvider, + final Function perFieldKnnVectorsFormatProvider + ) throws Exception { + final MapperService mapperService = mock(MapperService.class); + final KNNMethodContext knnMethodContext = new KNNMethodContext( + KNNEngine.LUCENE, + SpaceType.L2, + new MethodComponentContext(METHOD_HNSW, Map.of(HNSW_ALGO_M, 16, HNSW_ALGO_EF_CONSTRUCTION, 256)) + ); + final KNNVectorFieldMapper.KNNVectorFieldType mappedFieldType1 = new KNNVectorFieldMapper.KNNVectorFieldType( + FIELD_NAME_ONE, + Map.of(), + 3, + knnMethodContext + ); + final KNNVectorFieldMapper.KNNVectorFieldType mappedFieldType2 = new KNNVectorFieldMapper.KNNVectorFieldType( + FIELD_NAME_TWO, + Map.of(), + 2, + knnMethodContext + ); + when(mapperService.fieldType(eq(FIELD_NAME_ONE))).thenReturn(mappedFieldType1); + when(mapperService.fieldType(eq(FIELD_NAME_TWO))).thenReturn(mappedFieldType2); + + var perFieldKnnVectorsFormatSpy = spy(perFieldKnnVectorsFormatProvider.apply(mapperService)); + final Codec codec = codecProvider.apply(perFieldKnnVectorsFormatSpy); + + setUpMockClusterService(); + Directory dir = newFSDirectory(createTempDir()); + IndexWriterConfig iwc = newIndexWriterConfig(); + iwc.setMergeScheduler(new SerialMergeScheduler()); + iwc.setCodec(codec); + + /** + * Add doc with field "test_vector_one" + */ + final FieldType luceneFieldType = KnnVectorField.createFieldType(3, VectorSimilarityFunction.EUCLIDEAN); + float[] array = { 1.0f, 3.0f, 4.0f }; + KnnVectorField vectorField = new KnnVectorField(FIELD_NAME_ONE, array, luceneFieldType); + RandomIndexWriter writer = new RandomIndexWriter(random(), dir, iwc); + Document doc = new Document(); + doc.add(vectorField); + writer.addDocument(doc); + writer.commit(); + IndexReader reader = writer.getReader(); + writer.close(); + + verify(perFieldKnnVectorsFormatSpy).getKnnVectorsFormatForField(anyString()); + + IndexSearcher searcher = new IndexSearcher(reader); + Query query = KNNQueryFactory.create(KNNEngine.LUCENE, "dummy", FIELD_NAME_ONE, new float[] { 1.0f, 0.0f, 0.0f }, 1); + + assertEquals(1, searcher.count(query)); + + reader.close(); + + /** + * Add doc with field "test_vector_two" + */ + IndexWriterConfig iwc1 = newIndexWriterConfig(); + iwc1.setMergeScheduler(new SerialMergeScheduler()); + iwc1.setCodec(codec); + writer = new RandomIndexWriter(random(), dir, iwc1); + final FieldType luceneFieldType1 = KnnVectorField.createFieldType(2, VectorSimilarityFunction.EUCLIDEAN); + float[] array1 = { 6.0f, 14.0f }; + KnnVectorField vectorField1 = new KnnVectorField(FIELD_NAME_TWO, array1, luceneFieldType1); + Document doc1 = new Document(); + doc1.add(vectorField1); + writer.addDocument(doc1); + IndexReader reader1 = writer.getReader(); + writer.close(); + ResourceWatcherService resourceWatcherService = createDisabledResourceWatcherService(); + NativeMemoryLoadStrategy.IndexLoadStrategy.initialize(resourceWatcherService); + + verify(perFieldKnnVectorsFormatSpy, times(2)).getKnnVectorsFormatForField(anyString()); + + IndexSearcher searcher1 = new IndexSearcher(reader1); + Query query1 = KNNQueryFactory.create(KNNEngine.LUCENE, "dummy", FIELD_NAME_TWO, new float[] { 1.0f, 0.0f }, 1); + + assertEquals(1, searcher1.count(query1)); + + reader1.close(); + dir.close(); + resourceWatcherService.close(); + NativeMemoryLoadStrategy.IndexLoadStrategy.getInstance().close(); + } } From 78ff5338a952393a7f2fe9a56b130c8694b821f9 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Fri, 7 Oct 2022 10:31:17 -0700 Subject: [PATCH 039/416] Fix NPE on null script context (#560) (#569) Fix NPE on null script context Signed-off-by: Daniel Widdis (cherry picked from commit 71048e2255f5bb87a283529a31082b97a5fa3a3b) --- .../plugin/script/KNNScoringScriptEngine.java | 3 ++- .../script/KNNScoringScriptEngineTests.java | 25 +++++++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) create mode 100644 src/test/java/org/opensearch/knn/plugin/script/KNNScoringScriptEngineTests.java diff --git a/src/main/java/org/opensearch/knn/plugin/script/KNNScoringScriptEngine.java b/src/main/java/org/opensearch/knn/plugin/script/KNNScoringScriptEngine.java index 2d6b5f69c..42e0e90ec 100644 --- a/src/main/java/org/opensearch/knn/plugin/script/KNNScoringScriptEngine.java +++ b/src/main/java/org/opensearch/knn/plugin/script/KNNScoringScriptEngine.java @@ -10,6 +10,7 @@ import org.opensearch.script.ScriptContext; import org.opensearch.script.ScriptEngine; +import java.util.Collections; import java.util.Map; import java.util.Set; @@ -44,6 +45,6 @@ public FactoryType compile(String name, String code, ScriptContext @Override public Set> getSupportedContexts() { - return null; + return Collections.singleton(ScoreScript.CONTEXT); } } diff --git a/src/test/java/org/opensearch/knn/plugin/script/KNNScoringScriptEngineTests.java b/src/test/java/org/opensearch/knn/plugin/script/KNNScoringScriptEngineTests.java new file mode 100644 index 000000000..cffd3d3cb --- /dev/null +++ b/src/test/java/org/opensearch/knn/plugin/script/KNNScoringScriptEngineTests.java @@ -0,0 +1,25 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.plugin.script; + +import java.io.IOException; +import java.util.Set; + +import org.opensearch.knn.KNNTestCase; +import org.opensearch.script.ScriptContext; +import org.opensearch.script.ScriptEngine; + +public class KNNScoringScriptEngineTests extends KNNTestCase { + + public void testGetSupportedContexts() throws IOException { + try (ScriptEngine engine = new KNNScoringScriptEngine()) { + Set> supportedContexts = engine.getSupportedContexts(); + assertNotNull(supportedContexts); + assertFalse(supportedContexts.isEmpty()); + } + } + +} From 050c3a3104e5690f2b86e5ad208e03789757fdfb Mon Sep 17 00:00:00 2001 From: John Mazanec Date: Mon, 10 Oct 2022 14:09:36 -0700 Subject: [PATCH 040/416] Replace Forum link in k-NN plugin README.md (#574) Signed-off-by: John Mazanec Co-authored-by: Chris Moore <107723039+cwillum@users.noreply.github.com> --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index bb8ca928b..bd4baac00 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ [![Build and Test k-NN](https://github.com/opensearch-project/k-NN/actions/workflows/CI.yml/badge.svg)](https://github.com/opensearch-project/k-NN/actions/workflows/CI.yml) [![codecov](https://codecov.io/gh/opensearch-project/k-NN/branch/main/graph/badge.svg?token=PYQO2GW39S)](https://codecov.io/gh/opensearch-project/k-NN) [![Documentation](https://img.shields.io/badge/doc-reference-blue)](https://opensearch.org/docs/search-plugins/knn/index/) -[![Chat](https://img.shields.io/badge/chat-on%20forums-blue)](https://discuss.opendistrocommunity.dev/c/k-NN/) +[![Chat](https://img.shields.io/badge/chat-on%20forums-blue)](https://forum.opensearch.org/c/plugins/k-nn/48) ![PRs welcome!](https://img.shields.io/badge/PRs-welcome!-success) # OpenSearch k-NN @@ -21,7 +21,7 @@ * [Project Website](https://opensearch.org/) * [Downloads](https://opensearch.org/downloads.html). * [Documentation](https://opensearch.org/docs/search-plugins/knn/index/) -* Need help? Try [Forums](https://discuss.opendistrocommunity.dev/c/k-nn/) +* Need help? Try the [Forum](https://forum.opensearch.org/c/plugins/k-nn/48) * [Project Principles](https://opensearch.org/#principles) * [Contributing to OpenSearch k-NN](CONTRIBUTING.md) * [Maintainer Responsibilities](MAINTAINERS.md) From efc69915c08e010cc3f3ee8bbe0da080b49adf65 Mon Sep 17 00:00:00 2001 From: Naveen Tatikonda Date: Tue, 11 Oct 2022 17:15:00 -0500 Subject: [PATCH 041/416] Backport lucene changes (#575) * Update lucene92 package Signed-off-by: Naveen Tatikonda * Fix KNNCodecUtil FieldInfo construction Signed-off-by: Naveen Tatikonda * Add KNN940Codec based on Lucene94 codec Signed-off-by: Naveen Tatikonda Signed-off-by: Naveen Tatikonda --- .../KNN920PerFieldKnnVectorsFormat.java | 2 +- .../index/codec/KNN940Codec/KNN940Codec.java | 62 +++++++++++++++ .../KNN940PerFieldKnnVectorsFormat.java | 78 +++++++++++++++++++ .../knn/index/codec/KNNCodecFactory.java | 15 ++-- .../knn/index/codec/KNNFormatFactory.java | 12 +++ .../services/org.apache.lucene.codecs.Codec | 3 +- .../codec/KNN920Codec/KNN920CodecTests.java | 18 ----- .../codec/KNN940Codec/KNN940CodecTests.java | 40 ++++++++++ .../knn/index/codec/KNNCodecFactoryTests.java | 17 ++-- .../knn/index/codec/KNNCodecTestCase.java | 4 +- .../knn/index/codec/KNNCodecTestUtil.java | 2 + .../index/codec/KNNFormatFactoryTests.java | 12 +++ 12 files changed, 233 insertions(+), 32 deletions(-) create mode 100644 src/main/java/org/opensearch/knn/index/codec/KNN940Codec/KNN940Codec.java create mode 100644 src/main/java/org/opensearch/knn/index/codec/KNN940Codec/KNN940PerFieldKnnVectorsFormat.java create mode 100644 src/test/java/org/opensearch/knn/index/codec/KNN940Codec/KNN940CodecTests.java diff --git a/src/main/java/org/opensearch/knn/index/codec/KNN920Codec/KNN920PerFieldKnnVectorsFormat.java b/src/main/java/org/opensearch/knn/index/codec/KNN920Codec/KNN920PerFieldKnnVectorsFormat.java index 5328e36a2..0286e829a 100644 --- a/src/main/java/org/opensearch/knn/index/codec/KNN920Codec/KNN920PerFieldKnnVectorsFormat.java +++ b/src/main/java/org/opensearch/knn/index/codec/KNN920Codec/KNN920PerFieldKnnVectorsFormat.java @@ -8,7 +8,7 @@ import lombok.AllArgsConstructor; import lombok.extern.log4j.Log4j2; import org.apache.lucene.codecs.KnnVectorsFormat; -import org.apache.lucene.codecs.lucene92.Lucene92HnswVectorsFormat; +import org.apache.lucene.backward_codecs.lucene92.Lucene92HnswVectorsFormat; import org.apache.lucene.codecs.perfield.PerFieldKnnVectorsFormat; import org.opensearch.index.mapper.MapperService; import org.opensearch.knn.common.KNNConstants; diff --git a/src/main/java/org/opensearch/knn/index/codec/KNN940Codec/KNN940Codec.java b/src/main/java/org/opensearch/knn/index/codec/KNN940Codec/KNN940Codec.java new file mode 100644 index 000000000..43a348cee --- /dev/null +++ b/src/main/java/org/opensearch/knn/index/codec/KNN940Codec/KNN940Codec.java @@ -0,0 +1,62 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.codec.KNN940Codec; + +import lombok.Builder; +import org.apache.lucene.codecs.CompoundFormat; +import org.apache.lucene.codecs.Codec; +import org.apache.lucene.codecs.DocValuesFormat; +import org.apache.lucene.codecs.FilterCodec; +import org.apache.lucene.codecs.KnnVectorsFormat; +import org.apache.lucene.codecs.perfield.PerFieldKnnVectorsFormat; +import org.opensearch.knn.index.codec.KNNFormatFacade; +import org.opensearch.knn.index.codec.KNNFormatFactory; + +import java.util.Optional; + +import static org.opensearch.knn.index.codec.KNNCodecFactory.CodecDelegateFactory.createKNN94DefaultDelegate; + +public class KNN940Codec extends FilterCodec { + private static final String KNN940 = "KNN940Codec"; + private final KNNFormatFacade knnFormatFacade; + private final PerFieldKnnVectorsFormat perFieldKnnVectorsFormat; + + /** + * No arg constructor that uses Lucene94 as the delegate + */ + public KNN940Codec() { + this(createKNN94DefaultDelegate(), new KNN940PerFieldKnnVectorsFormat(Optional.empty())); + } + + /** + * Sole constructor. When subclassing this codec, create a no-arg ctor and pass the delegate codec + * and a unique name to this ctor. + * + * @param delegate codec that will perform all operations this codec does not override + * @param knnVectorsFormat per field format for KnnVector + */ + @Builder + protected KNN940Codec(Codec delegate, PerFieldKnnVectorsFormat knnVectorsFormat) { + super(KNN940, delegate); + knnFormatFacade = KNNFormatFactory.createKNN940Format(delegate); + perFieldKnnVectorsFormat = knnVectorsFormat; + } + + @Override + public DocValuesFormat docValuesFormat() { + return knnFormatFacade.docValuesFormat(); + } + + @Override + public CompoundFormat compoundFormat() { + return knnFormatFacade.compoundFormat(); + } + + @Override + public KnnVectorsFormat knnVectorsFormat() { + return perFieldKnnVectorsFormat; + } +} diff --git a/src/main/java/org/opensearch/knn/index/codec/KNN940Codec/KNN940PerFieldKnnVectorsFormat.java b/src/main/java/org/opensearch/knn/index/codec/KNN940Codec/KNN940PerFieldKnnVectorsFormat.java new file mode 100644 index 000000000..5b717106f --- /dev/null +++ b/src/main/java/org/opensearch/knn/index/codec/KNN940Codec/KNN940PerFieldKnnVectorsFormat.java @@ -0,0 +1,78 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.codec.KNN940Codec; + +import lombok.AllArgsConstructor; +import lombok.extern.log4j.Log4j2; +import org.apache.lucene.codecs.lucene94.Lucene94HnswVectorsFormat; +import org.apache.lucene.codecs.KnnVectorsFormat; +import org.apache.lucene.codecs.perfield.PerFieldKnnVectorsFormat; +import org.opensearch.index.mapper.MapperService; +import org.opensearch.knn.common.KNNConstants; +import org.opensearch.knn.index.mapper.KNNVectorFieldMapper; + +import java.util.Map; +import java.util.Optional; + +/** + * Class provides per field format implementation for Lucene Knn vector type + */ +@AllArgsConstructor +@Log4j2 +public class KNN940PerFieldKnnVectorsFormat extends PerFieldKnnVectorsFormat { + + private final Optional mapperService; + + @Override + public KnnVectorsFormat getKnnVectorsFormatForField(final String field) { + if (isNotKnnVectorFieldType(field)) { + log.debug( + String.format( + "Initialize KNN vector format for field [%s] with default params [max_connections] = \"%d\" and [beam_width] = \"%d\"", + field, + Lucene94HnswVectorsFormat.DEFAULT_MAX_CONN, + Lucene94HnswVectorsFormat.DEFAULT_BEAM_WIDTH + ) + ); + return new Lucene94HnswVectorsFormat(); + } + var type = (KNNVectorFieldMapper.KNNVectorFieldType) mapperService.orElseThrow( + () -> new IllegalStateException( + String.format("Cannot read field type for field [%s] because mapper service is not available", field) + ) + ).fieldType(field); + var params = type.getKnnMethodContext().getMethodComponent().getParameters(); + int maxConnections = getMaxConnections(params); + int beamWidth = getBeamWidth(params); + log.debug( + String.format( + "Initialize KNN vector format for field [%s] with params [max_connections] = \"%d\" and [beam_width] = \"%d\"", + field, + maxConnections, + beamWidth + ) + ); + return new Lucene94HnswVectorsFormat(maxConnections, beamWidth); + } + + private boolean isNotKnnVectorFieldType(final String field) { + return !mapperService.isPresent() || !(mapperService.get().fieldType(field) instanceof KNNVectorFieldMapper.KNNVectorFieldType); + } + + private int getMaxConnections(final Map params) { + if (params != null && params.containsKey(KNNConstants.METHOD_PARAMETER_M)) { + return (int) params.get(KNNConstants.METHOD_PARAMETER_M); + } + return Lucene94HnswVectorsFormat.DEFAULT_MAX_CONN; + } + + private int getBeamWidth(final Map params) { + if (params != null && params.containsKey(KNNConstants.METHOD_PARAMETER_EF_CONSTRUCTION)) { + return (int) params.get(KNNConstants.METHOD_PARAMETER_EF_CONSTRUCTION); + } + return Lucene94HnswVectorsFormat.DEFAULT_BEAM_WIDTH; + } +} diff --git a/src/main/java/org/opensearch/knn/index/codec/KNNCodecFactory.java b/src/main/java/org/opensearch/knn/index/codec/KNNCodecFactory.java index 217bce3e8..e53e1dd2a 100644 --- a/src/main/java/org/opensearch/knn/index/codec/KNNCodecFactory.java +++ b/src/main/java/org/opensearch/knn/index/codec/KNNCodecFactory.java @@ -7,10 +7,11 @@ import lombok.AllArgsConstructor; import org.apache.lucene.codecs.Codec; import org.apache.lucene.backward_codecs.lucene91.Lucene91Codec; -import org.apache.lucene.codecs.lucene92.Lucene92Codec; +import org.apache.lucene.backward_codecs.lucene92.Lucene92Codec; +import org.apache.lucene.codecs.lucene94.Lucene94Codec; import org.opensearch.index.mapper.MapperService; -import org.opensearch.knn.index.codec.KNN920Codec.KNN920Codec; -import org.opensearch.knn.index.codec.KNN920Codec.KNN920PerFieldKnnVectorsFormat; +import org.opensearch.knn.index.codec.KNN940Codec.KNN940Codec; +import org.opensearch.knn.index.codec.KNN940Codec.KNN940PerFieldKnnVectorsFormat; import java.util.Optional; @@ -23,9 +24,9 @@ public class KNNCodecFactory { private final MapperService mapperService; public Codec createKNNCodec(final Codec userCodec) { - var codec = KNN920Codec.builder() + var codec = KNN940Codec.builder() .delegate(userCodec) - .knnVectorsFormat(new KNN920PerFieldKnnVectorsFormat(Optional.of(mapperService))) + .knnVectorsFormat(new KNN940PerFieldKnnVectorsFormat(Optional.of(mapperService))) .build(); return codec; } @@ -42,5 +43,9 @@ public static Codec createKNN91DefaultDelegate() { public static Codec createKNN92DefaultDelegate() { return new Lucene92Codec(); } + + public static Codec createKNN94DefaultDelegate() { + return new Lucene94Codec(); + } } } diff --git a/src/main/java/org/opensearch/knn/index/codec/KNNFormatFactory.java b/src/main/java/org/opensearch/knn/index/codec/KNNFormatFactory.java index bd7e4c480..ee17189e3 100644 --- a/src/main/java/org/opensearch/knn/index/codec/KNNFormatFactory.java +++ b/src/main/java/org/opensearch/knn/index/codec/KNNFormatFactory.java @@ -38,4 +38,16 @@ public static KNNFormatFacade createKNN920Format(final Codec delegate) { ); return knnFormatFacade; } + + /** + * Return facade class that abstracts format specific to KNN940 codec + * @param delegate delegate codec that is wrapped by KNN codec + */ + public static KNNFormatFacade createKNN940Format(final Codec delegate) { + final KNNFormatFacade knnFormatFacade = new KNNFormatFacade( + new KNN80DocValuesFormat(delegate.docValuesFormat()), + new KNN80CompoundFormat(delegate.compoundFormat()) + ); + return knnFormatFacade; + } } diff --git a/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec b/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec index ca1386288..8185e7858 100644 --- a/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec +++ b/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec @@ -3,4 +3,5 @@ org.opensearch.knn.index.codec.KNN84Codec.KNN84Codec org.opensearch.knn.index.codec.KNN86Codec.KNN86Codec org.opensearch.knn.index.codec.KNN87Codec.KNN87Codec org.opensearch.knn.index.codec.KNN910Codec.KNN910Codec -org.opensearch.knn.index.codec.KNN920Codec.KNN920Codec \ No newline at end of file +org.opensearch.knn.index.codec.KNN920Codec.KNN920Codec +org.opensearch.knn.index.codec.KNN940Codec.KNN940Codec \ No newline at end of file diff --git a/src/test/java/org/opensearch/knn/index/codec/KNN920Codec/KNN920CodecTests.java b/src/test/java/org/opensearch/knn/index/codec/KNN920Codec/KNN920CodecTests.java index e4f848f6a..06cc7fad8 100644 --- a/src/test/java/org/opensearch/knn/index/codec/KNN920Codec/KNN920CodecTests.java +++ b/src/test/java/org/opensearch/knn/index/codec/KNN920Codec/KNN920CodecTests.java @@ -5,15 +5,9 @@ package org.opensearch.knn.index.codec.KNN920Codec; -import org.apache.lucene.codecs.Codec; -import org.apache.lucene.codecs.perfield.PerFieldKnnVectorsFormat; -import org.opensearch.index.mapper.MapperService; import org.opensearch.knn.index.codec.KNNCodecTestCase; - import java.io.IOException; -import java.util.Optional; import java.util.concurrent.ExecutionException; -import java.util.function.Function; import static org.opensearch.knn.index.codec.KNNCodecFactory.CodecDelegateFactory.createKNN92DefaultDelegate; @@ -26,16 +20,4 @@ public void testMultiFieldsKnnIndex() throws Exception { public void testBuildFromModelTemplate() throws InterruptedException, ExecutionException, IOException { testBuildFromModelTemplate((KNN920Codec.builder().delegate(createKNN92DefaultDelegate()).build())); } - - public void testKnnVectorIndex() throws Exception { - Function perFieldKnnVectorsFormatProvider = ( - mapperService) -> new KNN920PerFieldKnnVectorsFormat(Optional.of(mapperService)); - - Function knnCodecProvider = (knnVectorFormat) -> KNN920Codec.builder() - .delegate(createKNN92DefaultDelegate()) - .knnVectorsFormat(knnVectorFormat) - .build(); - - testKnnVectorIndex(knnCodecProvider, perFieldKnnVectorsFormatProvider); - } } diff --git a/src/test/java/org/opensearch/knn/index/codec/KNN940Codec/KNN940CodecTests.java b/src/test/java/org/opensearch/knn/index/codec/KNN940Codec/KNN940CodecTests.java new file mode 100644 index 000000000..578f88f9f --- /dev/null +++ b/src/test/java/org/opensearch/knn/index/codec/KNN940Codec/KNN940CodecTests.java @@ -0,0 +1,40 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.codec.KNN940Codec; + +import org.apache.lucene.codecs.Codec; +import org.apache.lucene.codecs.perfield.PerFieldKnnVectorsFormat; +import org.opensearch.index.mapper.MapperService; +import org.opensearch.knn.index.codec.KNNCodecTestCase; +import java.io.IOException; +import java.util.Optional; +import java.util.concurrent.ExecutionException; +import java.util.function.Function; + +import static org.opensearch.knn.index.codec.KNNCodecFactory.CodecDelegateFactory.createKNN94DefaultDelegate; + +public class KNN940CodecTests extends KNNCodecTestCase { + + public void testMultiFieldsKnnIndex() throws Exception { + testMultiFieldsKnnIndex(KNN940Codec.builder().delegate(createKNN94DefaultDelegate()).build()); + } + + public void testBuildFromModelTemplate() throws InterruptedException, ExecutionException, IOException { + testBuildFromModelTemplate((KNN940Codec.builder().delegate(createKNN94DefaultDelegate()).build())); + } + + public void testKnnVectorIndex() throws Exception { + Function perFieldKnnVectorsFormatProvider = ( + mapperService) -> new KNN940PerFieldKnnVectorsFormat(Optional.of(mapperService)); + + Function knnCodecProvider = (knnVectorFormat) -> KNN940Codec.builder() + .delegate(createKNN94DefaultDelegate()) + .knnVectorsFormat(knnVectorFormat) + .build(); + + testKnnVectorIndex(knnCodecProvider, perFieldKnnVectorsFormatProvider); + } +} diff --git a/src/test/java/org/opensearch/knn/index/codec/KNNCodecFactoryTests.java b/src/test/java/org/opensearch/knn/index/codec/KNNCodecFactoryTests.java index e339d0a3a..6e1c96bcb 100644 --- a/src/test/java/org/opensearch/knn/index/codec/KNNCodecFactoryTests.java +++ b/src/test/java/org/opensearch/knn/index/codec/KNNCodecFactoryTests.java @@ -7,10 +7,11 @@ import org.apache.lucene.codecs.Codec; import org.apache.lucene.backward_codecs.lucene91.Lucene91Codec; -import org.apache.lucene.codecs.lucene92.Lucene92Codec; +import org.apache.lucene.backward_codecs.lucene92.Lucene92Codec; +import org.apache.lucene.codecs.lucene94.Lucene94Codec; import org.opensearch.index.mapper.MapperService; import org.opensearch.knn.KNNTestCase; -import org.opensearch.knn.index.codec.KNN920Codec.KNN920Codec; +import org.opensearch.knn.index.codec.KNN940Codec.KNN940Codec; import static org.mockito.Mockito.mock; @@ -28,12 +29,18 @@ public void testKNN92DefaultDelegate() { assertTrue(knn92DefaultDelegate instanceof Lucene92Codec); } + public void testKNN94DefaultDelegate() { + Codec knn94DefaultDelegate = KNNCodecFactory.CodecDelegateFactory.createKNN94DefaultDelegate(); + assertNotNull(knn94DefaultDelegate); + assertTrue(knn94DefaultDelegate instanceof Lucene94Codec); + } + public void testKNNDefaultCodec() { MapperService mapperService = mock(MapperService.class); KNNCodecFactory knnCodecFactory = new KNNCodecFactory(mapperService); - Codec knnCodec = knnCodecFactory.createKNNCodec(KNNCodecFactory.CodecDelegateFactory.createKNN92DefaultDelegate()); + Codec knnCodec = knnCodecFactory.createKNNCodec(KNNCodecFactory.CodecDelegateFactory.createKNN94DefaultDelegate()); assertNotNull(knnCodec); - assertTrue(knnCodec instanceof KNN920Codec); - assertEquals("KNN920Codec", knnCodec.getName()); + assertTrue(knnCodec instanceof KNN940Codec); + assertEquals("KNN940Codec", knnCodec.getName()); } } diff --git a/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestCase.java b/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestCase.java index 7991db135..623f2dc74 100644 --- a/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestCase.java +++ b/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestCase.java @@ -18,8 +18,8 @@ import org.opensearch.knn.common.KNNConstants; import org.opensearch.knn.index.KNNMethodContext; import org.opensearch.knn.index.MethodComponentContext; -import org.opensearch.knn.index.codec.KNN920Codec.KNN920Codec; import org.opensearch.knn.index.query.KNNQueryFactory; +import org.opensearch.knn.index.codec.KNN940Codec.KNN940Codec; import org.opensearch.knn.jni.JNIService; import org.opensearch.knn.index.query.KNNQuery; import org.opensearch.knn.index.KNNSettings; @@ -79,7 +79,7 @@ */ public class KNNCodecTestCase extends KNNTestCase { - private static final KNN920Codec ACTUAL_CODEC = new KNN920Codec(); + private static final KNN940Codec ACTUAL_CODEC = new KNN940Codec(); private static FieldType sampleFieldType; static { sampleFieldType = new FieldType(KNNVectorFieldMapper.Defaults.FIELD_TYPE); diff --git a/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestUtil.java b/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestUtil.java index d95e1dc3c..62236a153 100644 --- a/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestUtil.java +++ b/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestUtil.java @@ -22,6 +22,7 @@ import org.apache.lucene.index.SortedNumericDocValues; import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.index.VectorSimilarityFunction; +import org.apache.lucene.index.VectorEncoding; import org.apache.lucene.search.Sort; import org.apache.lucene.store.ChecksumIndexInput; import org.apache.lucene.store.Directory; @@ -178,6 +179,7 @@ public FieldInfo build() { pointIndexDimensionCount, pointNumBytes, vectorDimension, + VectorEncoding.FLOAT32, vectorSimilarityFunction, softDeletes ); diff --git a/src/test/java/org/opensearch/knn/index/codec/KNNFormatFactoryTests.java b/src/test/java/org/opensearch/knn/index/codec/KNNFormatFactoryTests.java index 1abf73661..1b076f10b 100644 --- a/src/test/java/org/opensearch/knn/index/codec/KNNFormatFactoryTests.java +++ b/src/test/java/org/opensearch/knn/index/codec/KNNFormatFactoryTests.java @@ -36,4 +36,16 @@ public void testKNN92Format() { assertNotNull(knnFormatFacade.compoundFormat()); assertNotNull(knnFormatFacade.docValuesFormat()); } + + public void testKNN94Format() { + MapperService mapperService = mock(MapperService.class); + final Codec lucene94CodecDelegate = KNNCodecFactory.CodecDelegateFactory.createKNN94DefaultDelegate(); + KNNCodecFactory knnCodecFactory = new KNNCodecFactory(mapperService); + final Codec knnCodec = knnCodecFactory.createKNNCodec(lucene94CodecDelegate); + KNNFormatFacade knnFormatFacade = KNNFormatFactory.createKNN940Format(knnCodec); + + assertNotNull(knnFormatFacade); + assertNotNull(knnFormatFacade.compoundFormat()); + assertNotNull(knnFormatFacade.docValuesFormat()); + } } From 9d2bc9d643aa498668a844a7ea81c1210156f03c Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Thu, 13 Oct 2022 11:52:48 -0700 Subject: [PATCH 042/416] add group = org.opensearch.plugin (#578) (#579) Signed-off-by: Prudhvi Godithi (cherry picked from commit 82173fb7cd64e9c9721b59ec5e63567ddaf2654c) Co-authored-by: Prudhvi Godithi --- build.gradle | 1 + 1 file changed, 1 insertion(+) diff --git a/build.gradle b/build.gradle index dbba57602..fe1762fb3 100644 --- a/build.gradle +++ b/build.gradle @@ -88,6 +88,7 @@ publishing { pom { name = "opensearch-knn" description = "OpenSearch k-NN plugin" + groupId = "org.opensearch.plugin" licenses { license { name = "The Apache License, Version 2.0" From ba023ec30e8af74be1ade41cfbd47ee431e2bbf1 Mon Sep 17 00:00:00 2001 From: Martin Gaievski Date: Wed, 19 Oct 2022 11:33:50 -0700 Subject: [PATCH 043/416] Refactor kNN codec related classes (#582) (#584) * Refactor codec related classes, create KNNCodecVersion abstraction Signed-off-by: Martin Gaievski (cherry picked from commit 3d0a9d7ed6b1c609a9ab1ed2eff897dbe05fca63) --- .../codec/BasePerFieldKnnVectorsFormat.java | 79 ++++++++++++++++ .../index/codec/KNN910Codec/KNN910Codec.java | 13 +-- .../index/codec/KNN920Codec/KNN920Codec.java | 16 +--- .../KNN920PerFieldKnnVectorsFormat.java | 70 ++------------ .../index/codec/KNN940Codec/KNN940Codec.java | 14 +-- .../KNN940PerFieldKnnVectorsFormat.java | 70 ++------------ .../knn/index/codec/KNNCodecFactory.java | 51 ----------- .../knn/index/codec/KNNCodecService.java | 7 +- .../knn/index/codec/KNNCodecVersion.java | 91 +++++++++++++++++++ .../knn/index/codec/KNNFormatFactory.java | 53 ----------- .../codec/KNN920Codec/KNN920CodecTests.java | 6 +- .../codec/KNN940Codec/KNN940CodecTests.java | 8 +- .../knn/index/codec/KNNCodecFactoryTests.java | 43 ++++----- .../knn/index/codec/KNNCodecTestCase.java | 3 +- .../index/codec/KNNFormatFactoryTests.java | 51 ----------- 15 files changed, 237 insertions(+), 338 deletions(-) create mode 100644 src/main/java/org/opensearch/knn/index/codec/BasePerFieldKnnVectorsFormat.java delete mode 100644 src/main/java/org/opensearch/knn/index/codec/KNNCodecFactory.java create mode 100644 src/main/java/org/opensearch/knn/index/codec/KNNCodecVersion.java delete mode 100644 src/main/java/org/opensearch/knn/index/codec/KNNFormatFactory.java delete mode 100644 src/test/java/org/opensearch/knn/index/codec/KNNFormatFactoryTests.java diff --git a/src/main/java/org/opensearch/knn/index/codec/BasePerFieldKnnVectorsFormat.java b/src/main/java/org/opensearch/knn/index/codec/BasePerFieldKnnVectorsFormat.java new file mode 100644 index 000000000..d10ad9821 --- /dev/null +++ b/src/main/java/org/opensearch/knn/index/codec/BasePerFieldKnnVectorsFormat.java @@ -0,0 +1,79 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.codec; + +import lombok.AllArgsConstructor; +import lombok.extern.log4j.Log4j2; +import org.apache.lucene.codecs.KnnVectorsFormat; +import org.apache.lucene.codecs.perfield.PerFieldKnnVectorsFormat; +import org.opensearch.index.mapper.MapperService; +import org.opensearch.knn.common.KNNConstants; +import org.opensearch.knn.index.mapper.KNNVectorFieldMapper; + +import java.util.Map; +import java.util.Optional; +import java.util.function.BiFunction; +import java.util.function.Supplier; + +/** + * Base class for PerFieldKnnVectorsFormat, builds KnnVectorsFormat based on specific Lucene version + */ +@AllArgsConstructor +@Log4j2 +public abstract class BasePerFieldKnnVectorsFormat extends PerFieldKnnVectorsFormat { + + private final Optional mapperService; + private final int defaultMaxConnections; + private final int defaultBeamWidth; + private final Supplier defaultFormatSupplier; + private final BiFunction formatSupplier; + + @Override + public KnnVectorsFormat getKnnVectorsFormatForField(final String field) { + if (isKnnVectorFieldType(field) == false) { + log.debug( + "Initialize KNN vector format for field [{}] with default params [max_connections] = \"{}\" and [beam_width] = \"{}\"", + field, + defaultMaxConnections, + defaultBeamWidth + ); + return defaultFormatSupplier.get(); + } + var type = (KNNVectorFieldMapper.KNNVectorFieldType) mapperService.orElseThrow( + () -> new IllegalStateException( + String.format("Cannot read field type for field [%s] because mapper service is not available", field) + ) + ).fieldType(field); + var params = type.getKnnMethodContext().getMethodComponent().getParameters(); + int maxConnections = getMaxConnections(params); + int beamWidth = getBeamWidth(params); + log.debug( + "Initialize KNN vector format for field [{}] with params [max_connections] = \"{}\" and [beam_width] = \"{}\"", + field, + maxConnections, + beamWidth + ); + return formatSupplier.apply(maxConnections, beamWidth); + } + + private boolean isKnnVectorFieldType(final String field) { + return mapperService.isPresent() && mapperService.get().fieldType(field) instanceof KNNVectorFieldMapper.KNNVectorFieldType; + } + + private int getMaxConnections(final Map params) { + if (params != null && params.containsKey(KNNConstants.METHOD_PARAMETER_M)) { + return (int) params.get(KNNConstants.METHOD_PARAMETER_M); + } + return defaultMaxConnections; + } + + private int getBeamWidth(final Map params) { + if (params != null && params.containsKey(KNNConstants.METHOD_PARAMETER_EF_CONSTRUCTION)) { + return (int) params.get(KNNConstants.METHOD_PARAMETER_EF_CONSTRUCTION); + } + return defaultBeamWidth; + } +} diff --git a/src/main/java/org/opensearch/knn/index/codec/KNN910Codec/KNN910Codec.java b/src/main/java/org/opensearch/knn/index/codec/KNN910Codec/KNN910Codec.java index 0acaccfbf..77783dc29 100644 --- a/src/main/java/org/opensearch/knn/index/codec/KNN910Codec/KNN910Codec.java +++ b/src/main/java/org/opensearch/knn/index/codec/KNN910Codec/KNN910Codec.java @@ -8,10 +8,8 @@ import org.apache.lucene.codecs.CompoundFormat; import org.apache.lucene.codecs.DocValuesFormat; import org.apache.lucene.codecs.FilterCodec; +import org.opensearch.knn.index.codec.KNNCodecVersion; import org.opensearch.knn.index.codec.KNNFormatFacade; -import org.opensearch.knn.index.codec.KNNFormatFactory; - -import static org.opensearch.knn.index.codec.KNNCodecFactory.CodecDelegateFactory.createKNN91DefaultDelegate; /** * Extends the Codec to support a new file format for KNN index @@ -19,15 +17,14 @@ * */ public final class KNN910Codec extends FilterCodec { - - private static final String KNN910 = "KNN910Codec"; + private static final KNNCodecVersion VERSION = KNNCodecVersion.V_9_1_0; private final KNNFormatFacade knnFormatFacade; /** * No arg constructor that uses Lucene91 as the delegate */ public KNN910Codec() { - this(createKNN91DefaultDelegate()); + this(VERSION.getDefaultCodecDelegate()); } /** @@ -36,8 +33,8 @@ public KNN910Codec() { * @param delegate codec that will perform all operations this codec does not override */ public KNN910Codec(Codec delegate) { - super(KNN910, delegate); - knnFormatFacade = KNNFormatFactory.createKNN910Format(delegate); + super(VERSION.getCodecName(), delegate); + knnFormatFacade = VERSION.getKnnFormatFacadeSupplier().apply(delegate); } @Override diff --git a/src/main/java/org/opensearch/knn/index/codec/KNN920Codec/KNN920Codec.java b/src/main/java/org/opensearch/knn/index/codec/KNN920Codec/KNN920Codec.java index 26abcea60..b79c1b4f2 100644 --- a/src/main/java/org/opensearch/knn/index/codec/KNN920Codec/KNN920Codec.java +++ b/src/main/java/org/opensearch/knn/index/codec/KNN920Codec/KNN920Codec.java @@ -12,21 +12,15 @@ import org.apache.lucene.codecs.FilterCodec; import org.apache.lucene.codecs.KnnVectorsFormat; import org.apache.lucene.codecs.perfield.PerFieldKnnVectorsFormat; +import org.opensearch.knn.index.codec.KNNCodecVersion; import org.opensearch.knn.index.codec.KNNFormatFacade; -import org.opensearch.knn.index.codec.KNNFormatFactory; - -import java.util.Optional; - -import static org.opensearch.knn.index.codec.KNNCodecFactory.CodecDelegateFactory.createKNN92DefaultDelegate; /** * KNN codec that is based on Lucene92 codec */ @Log4j2 public final class KNN920Codec extends FilterCodec { - - private static final String KNN920 = "KNN920Codec"; - + private static final KNNCodecVersion VERSION = KNNCodecVersion.V_9_2_0; private final KNNFormatFacade knnFormatFacade; private final PerFieldKnnVectorsFormat perFieldKnnVectorsFormat; @@ -34,7 +28,7 @@ public final class KNN920Codec extends FilterCodec { * No arg constructor that uses Lucene91 as the delegate */ public KNN920Codec() { - this(createKNN92DefaultDelegate(), new KNN920PerFieldKnnVectorsFormat(Optional.empty())); + this(VERSION.getDefaultCodecDelegate(), VERSION.getPerFieldKnnVectorsFormat()); } /** @@ -45,8 +39,8 @@ public KNN920Codec() { */ @Builder public KNN920Codec(Codec delegate, PerFieldKnnVectorsFormat knnVectorsFormat) { - super(KNN920, delegate); - knnFormatFacade = KNNFormatFactory.createKNN920Format(delegate); + super(VERSION.getCodecName(), delegate); + knnFormatFacade = VERSION.getKnnFormatFacadeSupplier().apply(delegate); perFieldKnnVectorsFormat = knnVectorsFormat; } diff --git a/src/main/java/org/opensearch/knn/index/codec/KNN920Codec/KNN920PerFieldKnnVectorsFormat.java b/src/main/java/org/opensearch/knn/index/codec/KNN920Codec/KNN920PerFieldKnnVectorsFormat.java index 0286e829a..ae1ef206c 100644 --- a/src/main/java/org/opensearch/knn/index/codec/KNN920Codec/KNN920PerFieldKnnVectorsFormat.java +++ b/src/main/java/org/opensearch/knn/index/codec/KNN920Codec/KNN920PerFieldKnnVectorsFormat.java @@ -5,74 +5,24 @@ package org.opensearch.knn.index.codec.KNN920Codec; -import lombok.AllArgsConstructor; -import lombok.extern.log4j.Log4j2; -import org.apache.lucene.codecs.KnnVectorsFormat; import org.apache.lucene.backward_codecs.lucene92.Lucene92HnswVectorsFormat; -import org.apache.lucene.codecs.perfield.PerFieldKnnVectorsFormat; import org.opensearch.index.mapper.MapperService; -import org.opensearch.knn.common.KNNConstants; -import org.opensearch.knn.index.mapper.KNNVectorFieldMapper; +import org.opensearch.knn.index.codec.BasePerFieldKnnVectorsFormat; -import java.util.Map; import java.util.Optional; /** * Class provides per field format implementation for Lucene Knn vector type */ -@AllArgsConstructor -@Log4j2 -public class KNN920PerFieldKnnVectorsFormat extends PerFieldKnnVectorsFormat { - - private final Optional mapperService; - - @Override - public KnnVectorsFormat getKnnVectorsFormatForField(final String field) { - if (isNotKnnVectorFieldType(field)) { - log.debug( - String.format( - "Initialize KNN vector format for field [%s] with default params [max_connections] = \"%d\" and [beam_width] = \"%d\"", - field, - Lucene92HnswVectorsFormat.DEFAULT_MAX_CONN, - Lucene92HnswVectorsFormat.DEFAULT_BEAM_WIDTH - ) - ); - return new Lucene92HnswVectorsFormat(); - } - var type = (KNNVectorFieldMapper.KNNVectorFieldType) mapperService.orElseThrow( - () -> new IllegalStateException( - String.format("Cannot read field type for field [%s] because mapper service is not available", field) - ) - ).fieldType(field); - var params = type.getKnnMethodContext().getMethodComponent().getParameters(); - int maxConnections = getMaxConnections(params); - int beamWidth = getBeamWidth(params); - log.debug( - String.format( - "Initialize KNN vector format for field [%s] with params [max_connections] = \"%d\" and [beam_width] = \"%d\"", - field, - maxConnections, - beamWidth - ) +public class KNN920PerFieldKnnVectorsFormat extends BasePerFieldKnnVectorsFormat { + + public KNN920PerFieldKnnVectorsFormat(final Optional mapperService) { + super( + mapperService, + Lucene92HnswVectorsFormat.DEFAULT_MAX_CONN, + Lucene92HnswVectorsFormat.DEFAULT_BEAM_WIDTH, + () -> new Lucene92HnswVectorsFormat(), + (maxConnm, beamWidth) -> new Lucene92HnswVectorsFormat(maxConnm, beamWidth) ); - return new Lucene92HnswVectorsFormat(maxConnections, beamWidth); - } - - private boolean isNotKnnVectorFieldType(final String field) { - return !mapperService.isPresent() || !(mapperService.get().fieldType(field) instanceof KNNVectorFieldMapper.KNNVectorFieldType); - } - - private int getMaxConnections(final Map params) { - if (params != null && params.containsKey(KNNConstants.METHOD_PARAMETER_M)) { - return (int) params.get(KNNConstants.METHOD_PARAMETER_M); - } - return Lucene92HnswVectorsFormat.DEFAULT_MAX_CONN; - } - - private int getBeamWidth(final Map params) { - if (params != null && params.containsKey(KNNConstants.METHOD_PARAMETER_EF_CONSTRUCTION)) { - return (int) params.get(KNNConstants.METHOD_PARAMETER_EF_CONSTRUCTION); - } - return Lucene92HnswVectorsFormat.DEFAULT_BEAM_WIDTH; } } diff --git a/src/main/java/org/opensearch/knn/index/codec/KNN940Codec/KNN940Codec.java b/src/main/java/org/opensearch/knn/index/codec/KNN940Codec/KNN940Codec.java index 43a348cee..a056581d6 100644 --- a/src/main/java/org/opensearch/knn/index/codec/KNN940Codec/KNN940Codec.java +++ b/src/main/java/org/opensearch/knn/index/codec/KNN940Codec/KNN940Codec.java @@ -12,15 +12,11 @@ import org.apache.lucene.codecs.FilterCodec; import org.apache.lucene.codecs.KnnVectorsFormat; import org.apache.lucene.codecs.perfield.PerFieldKnnVectorsFormat; +import org.opensearch.knn.index.codec.KNNCodecVersion; import org.opensearch.knn.index.codec.KNNFormatFacade; -import org.opensearch.knn.index.codec.KNNFormatFactory; - -import java.util.Optional; - -import static org.opensearch.knn.index.codec.KNNCodecFactory.CodecDelegateFactory.createKNN94DefaultDelegate; public class KNN940Codec extends FilterCodec { - private static final String KNN940 = "KNN940Codec"; + private static final KNNCodecVersion VERSION = KNNCodecVersion.V_9_4_0; private final KNNFormatFacade knnFormatFacade; private final PerFieldKnnVectorsFormat perFieldKnnVectorsFormat; @@ -28,7 +24,7 @@ public class KNN940Codec extends FilterCodec { * No arg constructor that uses Lucene94 as the delegate */ public KNN940Codec() { - this(createKNN94DefaultDelegate(), new KNN940PerFieldKnnVectorsFormat(Optional.empty())); + this(VERSION.getDefaultCodecDelegate(), VERSION.getPerFieldKnnVectorsFormat()); } /** @@ -40,8 +36,8 @@ public KNN940Codec() { */ @Builder protected KNN940Codec(Codec delegate, PerFieldKnnVectorsFormat knnVectorsFormat) { - super(KNN940, delegate); - knnFormatFacade = KNNFormatFactory.createKNN940Format(delegate); + super(VERSION.getCodecName(), delegate); + knnFormatFacade = VERSION.getKnnFormatFacadeSupplier().apply(delegate); perFieldKnnVectorsFormat = knnVectorsFormat; } diff --git a/src/main/java/org/opensearch/knn/index/codec/KNN940Codec/KNN940PerFieldKnnVectorsFormat.java b/src/main/java/org/opensearch/knn/index/codec/KNN940Codec/KNN940PerFieldKnnVectorsFormat.java index 5b717106f..d80c757c9 100644 --- a/src/main/java/org/opensearch/knn/index/codec/KNN940Codec/KNN940PerFieldKnnVectorsFormat.java +++ b/src/main/java/org/opensearch/knn/index/codec/KNN940Codec/KNN940PerFieldKnnVectorsFormat.java @@ -5,74 +5,24 @@ package org.opensearch.knn.index.codec.KNN940Codec; -import lombok.AllArgsConstructor; -import lombok.extern.log4j.Log4j2; import org.apache.lucene.codecs.lucene94.Lucene94HnswVectorsFormat; -import org.apache.lucene.codecs.KnnVectorsFormat; -import org.apache.lucene.codecs.perfield.PerFieldKnnVectorsFormat; import org.opensearch.index.mapper.MapperService; -import org.opensearch.knn.common.KNNConstants; -import org.opensearch.knn.index.mapper.KNNVectorFieldMapper; +import org.opensearch.knn.index.codec.BasePerFieldKnnVectorsFormat; -import java.util.Map; import java.util.Optional; /** * Class provides per field format implementation for Lucene Knn vector type */ -@AllArgsConstructor -@Log4j2 -public class KNN940PerFieldKnnVectorsFormat extends PerFieldKnnVectorsFormat { - - private final Optional mapperService; - - @Override - public KnnVectorsFormat getKnnVectorsFormatForField(final String field) { - if (isNotKnnVectorFieldType(field)) { - log.debug( - String.format( - "Initialize KNN vector format for field [%s] with default params [max_connections] = \"%d\" and [beam_width] = \"%d\"", - field, - Lucene94HnswVectorsFormat.DEFAULT_MAX_CONN, - Lucene94HnswVectorsFormat.DEFAULT_BEAM_WIDTH - ) - ); - return new Lucene94HnswVectorsFormat(); - } - var type = (KNNVectorFieldMapper.KNNVectorFieldType) mapperService.orElseThrow( - () -> new IllegalStateException( - String.format("Cannot read field type for field [%s] because mapper service is not available", field) - ) - ).fieldType(field); - var params = type.getKnnMethodContext().getMethodComponent().getParameters(); - int maxConnections = getMaxConnections(params); - int beamWidth = getBeamWidth(params); - log.debug( - String.format( - "Initialize KNN vector format for field [%s] with params [max_connections] = \"%d\" and [beam_width] = \"%d\"", - field, - maxConnections, - beamWidth - ) +public class KNN940PerFieldKnnVectorsFormat extends BasePerFieldKnnVectorsFormat { + + public KNN940PerFieldKnnVectorsFormat(final Optional mapperService) { + super( + mapperService, + Lucene94HnswVectorsFormat.DEFAULT_MAX_CONN, + Lucene94HnswVectorsFormat.DEFAULT_BEAM_WIDTH, + () -> new Lucene94HnswVectorsFormat(), + (maxConnm, beamWidth) -> new Lucene94HnswVectorsFormat(maxConnm, beamWidth) ); - return new Lucene94HnswVectorsFormat(maxConnections, beamWidth); - } - - private boolean isNotKnnVectorFieldType(final String field) { - return !mapperService.isPresent() || !(mapperService.get().fieldType(field) instanceof KNNVectorFieldMapper.KNNVectorFieldType); - } - - private int getMaxConnections(final Map params) { - if (params != null && params.containsKey(KNNConstants.METHOD_PARAMETER_M)) { - return (int) params.get(KNNConstants.METHOD_PARAMETER_M); - } - return Lucene94HnswVectorsFormat.DEFAULT_MAX_CONN; - } - - private int getBeamWidth(final Map params) { - if (params != null && params.containsKey(KNNConstants.METHOD_PARAMETER_EF_CONSTRUCTION)) { - return (int) params.get(KNNConstants.METHOD_PARAMETER_EF_CONSTRUCTION); - } - return Lucene94HnswVectorsFormat.DEFAULT_BEAM_WIDTH; } } diff --git a/src/main/java/org/opensearch/knn/index/codec/KNNCodecFactory.java b/src/main/java/org/opensearch/knn/index/codec/KNNCodecFactory.java deleted file mode 100644 index e53e1dd2a..000000000 --- a/src/main/java/org/opensearch/knn/index/codec/KNNCodecFactory.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ -package org.opensearch.knn.index.codec; - -import lombok.AllArgsConstructor; -import org.apache.lucene.codecs.Codec; -import org.apache.lucene.backward_codecs.lucene91.Lucene91Codec; -import org.apache.lucene.backward_codecs.lucene92.Lucene92Codec; -import org.apache.lucene.codecs.lucene94.Lucene94Codec; -import org.opensearch.index.mapper.MapperService; -import org.opensearch.knn.index.codec.KNN940Codec.KNN940Codec; -import org.opensearch.knn.index.codec.KNN940Codec.KNN940PerFieldKnnVectorsFormat; - -import java.util.Optional; - -/** - * Factory abstraction for KNN codec - */ -@AllArgsConstructor -public class KNNCodecFactory { - - private final MapperService mapperService; - - public Codec createKNNCodec(final Codec userCodec) { - var codec = KNN940Codec.builder() - .delegate(userCodec) - .knnVectorsFormat(new KNN940PerFieldKnnVectorsFormat(Optional.of(mapperService))) - .build(); - return codec; - } - - /** - * Factory abstraction for codec delegate - */ - public static class CodecDelegateFactory { - - public static Codec createKNN91DefaultDelegate() { - return new Lucene91Codec(); - } - - public static Codec createKNN92DefaultDelegate() { - return new Lucene92Codec(); - } - - public static Codec createKNN94DefaultDelegate() { - return new Lucene94Codec(); - } - } -} diff --git a/src/main/java/org/opensearch/knn/index/codec/KNNCodecService.java b/src/main/java/org/opensearch/knn/index/codec/KNNCodecService.java index 8ce5e6928..d56e09a3f 100644 --- a/src/main/java/org/opensearch/knn/index/codec/KNNCodecService.java +++ b/src/main/java/org/opensearch/knn/index/codec/KNNCodecService.java @@ -8,17 +8,18 @@ import org.opensearch.index.codec.CodecServiceConfig; import org.apache.lucene.codecs.Codec; import org.opensearch.index.codec.CodecService; +import org.opensearch.index.mapper.MapperService; /** * KNNCodecService to inject the right KNNCodec version */ public class KNNCodecService extends CodecService { - private final KNNCodecFactory knnCodecFactory; + private final MapperService mapperService; public KNNCodecService(CodecServiceConfig codecServiceConfig) { super(codecServiceConfig.getMapperService(), codecServiceConfig.getLogger()); - knnCodecFactory = new KNNCodecFactory(codecServiceConfig.getMapperService()); + mapperService = codecServiceConfig.getMapperService(); } /** @@ -29,6 +30,6 @@ public KNNCodecService(CodecServiceConfig codecServiceConfig) { */ @Override public Codec codec(String name) { - return knnCodecFactory.createKNNCodec(super.codec(name)); + return KNNCodecVersion.current().getKnnCodecSupplier().apply(super.codec(name), mapperService); } } diff --git a/src/main/java/org/opensearch/knn/index/codec/KNNCodecVersion.java b/src/main/java/org/opensearch/knn/index/codec/KNNCodecVersion.java new file mode 100644 index 000000000..adbbb01ca --- /dev/null +++ b/src/main/java/org/opensearch/knn/index/codec/KNNCodecVersion.java @@ -0,0 +1,91 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.codec; + +import lombok.AllArgsConstructor; +import lombok.Getter; +import org.apache.lucene.backward_codecs.lucene91.Lucene91Codec; +import org.apache.lucene.backward_codecs.lucene92.Lucene92Codec; +import org.apache.lucene.codecs.Codec; +import org.apache.lucene.codecs.lucene94.Lucene94Codec; +import org.apache.lucene.codecs.perfield.PerFieldKnnVectorsFormat; +import org.opensearch.index.mapper.MapperService; +import org.opensearch.knn.index.codec.KNN80Codec.KNN80CompoundFormat; +import org.opensearch.knn.index.codec.KNN80Codec.KNN80DocValuesFormat; +import org.opensearch.knn.index.codec.KNN910Codec.KNN910Codec; +import org.opensearch.knn.index.codec.KNN920Codec.KNN920Codec; +import org.opensearch.knn.index.codec.KNN920Codec.KNN920PerFieldKnnVectorsFormat; +import org.opensearch.knn.index.codec.KNN940Codec.KNN940Codec; +import org.opensearch.knn.index.codec.KNN940Codec.KNN940PerFieldKnnVectorsFormat; + +import java.util.Optional; +import java.util.function.BiFunction; +import java.util.function.Function; +import java.util.function.Supplier; + +/** + * Abstraction for k-NN codec version, aggregates all details for specific version such as codec name, corresponding + * Lucene codec, formats including one for k-NN vector etc. + */ +@AllArgsConstructor +@Getter +public enum KNNCodecVersion { + + V_9_1_0( + "KNN910Codec", + new Lucene91Codec(), + null, + (delegate) -> new KNNFormatFacade( + new KNN80DocValuesFormat(delegate.docValuesFormat()), + new KNN80CompoundFormat(delegate.compoundFormat()) + ), + (userCodec, mapperService) -> new KNN910Codec(userCodec), + KNN910Codec::new + ), + + V_9_2_0( + "KNN920Codec", + new Lucene92Codec(), + new KNN920PerFieldKnnVectorsFormat(Optional.empty()), + (delegate) -> new KNNFormatFacade( + new KNN80DocValuesFormat(delegate.docValuesFormat()), + new KNN80CompoundFormat(delegate.compoundFormat()) + ), + (userCodec, mapperService) -> KNN920Codec.builder() + .delegate(userCodec) + .knnVectorsFormat(new KNN920PerFieldKnnVectorsFormat(Optional.of(mapperService))) + .build(), + KNN920Codec::new + ), + + V_9_4_0( + "KNN940Codec", + new Lucene94Codec(), + new KNN940PerFieldKnnVectorsFormat(Optional.empty()), + (delegate) -> new KNNFormatFacade( + new KNN80DocValuesFormat(delegate.docValuesFormat()), + new KNN80CompoundFormat(delegate.compoundFormat()) + ), + (userCodec, mapperService) -> KNN940Codec.builder() + .delegate(userCodec) + .knnVectorsFormat(new KNN940PerFieldKnnVectorsFormat(Optional.of(mapperService))) + .build(), + KNN940Codec::new + ); + + private static final KNNCodecVersion CURRENT = V_9_4_0; + + private final String codecName; + private final Codec defaultCodecDelegate; + private final PerFieldKnnVectorsFormat perFieldKnnVectorsFormat; + private final Function knnFormatFacadeSupplier; + private final BiFunction knnCodecSupplier; + private final Supplier defaultKnnCodecSupplier; + + public static final KNNCodecVersion current() { + return CURRENT; + } +} diff --git a/src/main/java/org/opensearch/knn/index/codec/KNNFormatFactory.java b/src/main/java/org/opensearch/knn/index/codec/KNNFormatFactory.java deleted file mode 100644 index ee17189e3..000000000 --- a/src/main/java/org/opensearch/knn/index/codec/KNNFormatFactory.java +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ -package org.opensearch.knn.index.codec; - -import org.apache.lucene.codecs.Codec; -import org.opensearch.knn.index.codec.KNN80Codec.KNN80CompoundFormat; -import org.opensearch.knn.index.codec.KNN80Codec.KNN80DocValuesFormat; - -/** - * Factory abstraction for KNN format facade creation - */ -public class KNNFormatFactory { - - /** - * Return facade class that abstracts format specific to KNN910 codec - * @param delegate delegate codec that is wrapped by KNN codec - * @return - */ - public static KNNFormatFacade createKNN910Format(final Codec delegate) { - final KNNFormatFacade knnFormatFacade = new KNNFormatFacade( - new KNN80DocValuesFormat(delegate.docValuesFormat()), - new KNN80CompoundFormat(delegate.compoundFormat()) - ); - return knnFormatFacade; - } - - /** - * Return facade class that abstracts format specific to KNN920 codec - * @param delegate delegate codec that is wrapped by KNN codec - * @return - */ - public static KNNFormatFacade createKNN920Format(final Codec delegate) { - final KNNFormatFacade knnFormatFacade = new KNNFormatFacade( - new KNN80DocValuesFormat(delegate.docValuesFormat()), - new KNN80CompoundFormat(delegate.compoundFormat()) - ); - return knnFormatFacade; - } - - /** - * Return facade class that abstracts format specific to KNN940 codec - * @param delegate delegate codec that is wrapped by KNN codec - */ - public static KNNFormatFacade createKNN940Format(final Codec delegate) { - final KNNFormatFacade knnFormatFacade = new KNNFormatFacade( - new KNN80DocValuesFormat(delegate.docValuesFormat()), - new KNN80CompoundFormat(delegate.compoundFormat()) - ); - return knnFormatFacade; - } -} diff --git a/src/test/java/org/opensearch/knn/index/codec/KNN920Codec/KNN920CodecTests.java b/src/test/java/org/opensearch/knn/index/codec/KNN920Codec/KNN920CodecTests.java index 06cc7fad8..8cdfc2d69 100644 --- a/src/test/java/org/opensearch/knn/index/codec/KNN920Codec/KNN920CodecTests.java +++ b/src/test/java/org/opensearch/knn/index/codec/KNN920Codec/KNN920CodecTests.java @@ -9,15 +9,15 @@ import java.io.IOException; import java.util.concurrent.ExecutionException; -import static org.opensearch.knn.index.codec.KNNCodecFactory.CodecDelegateFactory.createKNN92DefaultDelegate; +import static org.opensearch.knn.index.codec.KNNCodecVersion.V_9_2_0; public class KNN920CodecTests extends KNNCodecTestCase { public void testMultiFieldsKnnIndex() throws Exception { - testMultiFieldsKnnIndex(KNN920Codec.builder().delegate(createKNN92DefaultDelegate()).build()); + testMultiFieldsKnnIndex(KNN920Codec.builder().delegate(V_9_2_0.getDefaultCodecDelegate()).build()); } public void testBuildFromModelTemplate() throws InterruptedException, ExecutionException, IOException { - testBuildFromModelTemplate((KNN920Codec.builder().delegate(createKNN92DefaultDelegate()).build())); + testBuildFromModelTemplate((KNN920Codec.builder().delegate(V_9_2_0.getDefaultCodecDelegate()).build())); } } diff --git a/src/test/java/org/opensearch/knn/index/codec/KNN940Codec/KNN940CodecTests.java b/src/test/java/org/opensearch/knn/index/codec/KNN940Codec/KNN940CodecTests.java index 578f88f9f..1101d93bb 100644 --- a/src/test/java/org/opensearch/knn/index/codec/KNN940Codec/KNN940CodecTests.java +++ b/src/test/java/org/opensearch/knn/index/codec/KNN940Codec/KNN940CodecTests.java @@ -14,16 +14,16 @@ import java.util.concurrent.ExecutionException; import java.util.function.Function; -import static org.opensearch.knn.index.codec.KNNCodecFactory.CodecDelegateFactory.createKNN94DefaultDelegate; +import static org.opensearch.knn.index.codec.KNNCodecVersion.V_9_4_0; public class KNN940CodecTests extends KNNCodecTestCase { public void testMultiFieldsKnnIndex() throws Exception { - testMultiFieldsKnnIndex(KNN940Codec.builder().delegate(createKNN94DefaultDelegate()).build()); + testMultiFieldsKnnIndex(KNN940Codec.builder().delegate(V_9_4_0.getDefaultCodecDelegate()).build()); } public void testBuildFromModelTemplate() throws InterruptedException, ExecutionException, IOException { - testBuildFromModelTemplate((KNN940Codec.builder().delegate(createKNN94DefaultDelegate()).build())); + testBuildFromModelTemplate((KNN940Codec.builder().delegate(V_9_4_0.getDefaultCodecDelegate()).build())); } public void testKnnVectorIndex() throws Exception { @@ -31,7 +31,7 @@ public void testKnnVectorIndex() throws Exception { mapperService) -> new KNN940PerFieldKnnVectorsFormat(Optional.of(mapperService)); Function knnCodecProvider = (knnVectorFormat) -> KNN940Codec.builder() - .delegate(createKNN94DefaultDelegate()) + .delegate(V_9_4_0.getDefaultCodecDelegate()) .knnVectorsFormat(knnVectorFormat) .build(); diff --git a/src/test/java/org/opensearch/knn/index/codec/KNNCodecFactoryTests.java b/src/test/java/org/opensearch/knn/index/codec/KNNCodecFactoryTests.java index 6e1c96bcb..d918f5439 100644 --- a/src/test/java/org/opensearch/knn/index/codec/KNNCodecFactoryTests.java +++ b/src/test/java/org/opensearch/knn/index/codec/KNNCodecFactoryTests.java @@ -5,42 +5,39 @@ package org.opensearch.knn.index.codec; +import org.apache.lucene.backward_codecs.lucene92.Lucene92Codec; import org.apache.lucene.codecs.Codec; import org.apache.lucene.backward_codecs.lucene91.Lucene91Codec; -import org.apache.lucene.backward_codecs.lucene92.Lucene92Codec; import org.apache.lucene.codecs.lucene94.Lucene94Codec; -import org.opensearch.index.mapper.MapperService; import org.opensearch.knn.KNNTestCase; -import org.opensearch.knn.index.codec.KNN940Codec.KNN940Codec; -import static org.mockito.Mockito.mock; +import static org.opensearch.knn.index.codec.KNNCodecVersion.V_9_1_0; +import static org.opensearch.knn.index.codec.KNNCodecVersion.V_9_2_0; +import static org.opensearch.knn.index.codec.KNNCodecVersion.V_9_4_0; public class KNNCodecFactoryTests extends KNNTestCase { - public void testKNN91DefaultDelegate() { - Codec knn91DefaultDelegate = KNNCodecFactory.CodecDelegateFactory.createKNN91DefaultDelegate(); - assertNotNull(knn91DefaultDelegate); - assertTrue(knn91DefaultDelegate instanceof Lucene91Codec); + public void testKNN910Codec() { + assertDelegateForVersion(V_9_1_0, Lucene91Codec.class); + assertNull(V_9_1_0.getPerFieldKnnVectorsFormat()); + assertNotNull(V_9_1_0.getKnnFormatFacadeSupplier().apply(V_9_1_0.getDefaultCodecDelegate())); } - public void testKNN92DefaultDelegate() { - Codec knn92DefaultDelegate = KNNCodecFactory.CodecDelegateFactory.createKNN92DefaultDelegate(); - assertNotNull(knn92DefaultDelegate); - assertTrue(knn92DefaultDelegate instanceof Lucene92Codec); + public void testKNN920Codec() { + assertDelegateForVersion(V_9_2_0, Lucene92Codec.class); + assertNotNull(V_9_2_0.getPerFieldKnnVectorsFormat()); + assertNotNull(V_9_2_0.getKnnFormatFacadeSupplier().apply(V_9_2_0.getDefaultCodecDelegate())); } - public void testKNN94DefaultDelegate() { - Codec knn94DefaultDelegate = KNNCodecFactory.CodecDelegateFactory.createKNN94DefaultDelegate(); - assertNotNull(knn94DefaultDelegate); - assertTrue(knn94DefaultDelegate instanceof Lucene94Codec); + public void testKNN940Codec() { + assertDelegateForVersion(V_9_4_0, Lucene94Codec.class); + assertNotNull(V_9_4_0.getPerFieldKnnVectorsFormat()); + assertNotNull(V_9_4_0.getKnnFormatFacadeSupplier().apply(V_9_4_0.getDefaultCodecDelegate())); } - public void testKNNDefaultCodec() { - MapperService mapperService = mock(MapperService.class); - KNNCodecFactory knnCodecFactory = new KNNCodecFactory(mapperService); - Codec knnCodec = knnCodecFactory.createKNNCodec(KNNCodecFactory.CodecDelegateFactory.createKNN94DefaultDelegate()); - assertNotNull(knnCodec); - assertTrue(knnCodec instanceof KNN940Codec); - assertEquals("KNN940Codec", knnCodec.getName()); + private void assertDelegateForVersion(final KNNCodecVersion codecVersion, final Class expectedCodecClass) { + final Codec defaultDelegate = codecVersion.getDefaultCodecDelegate(); + assertNotNull(defaultDelegate); + assertTrue(defaultDelegate.getClass().isAssignableFrom(expectedCodecClass)); } } diff --git a/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestCase.java b/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestCase.java index 623f2dc74..43ae19320 100644 --- a/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestCase.java +++ b/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestCase.java @@ -19,7 +19,6 @@ import org.opensearch.knn.index.KNNMethodContext; import org.opensearch.knn.index.MethodComponentContext; import org.opensearch.knn.index.query.KNNQueryFactory; -import org.opensearch.knn.index.codec.KNN940Codec.KNN940Codec; import org.opensearch.knn.jni.JNIService; import org.opensearch.knn.index.query.KNNQuery; import org.opensearch.knn.index.KNNSettings; @@ -79,7 +78,7 @@ */ public class KNNCodecTestCase extends KNNTestCase { - private static final KNN940Codec ACTUAL_CODEC = new KNN940Codec(); + private static final Codec ACTUAL_CODEC = KNNCodecVersion.current().getDefaultKnnCodecSupplier().get(); private static FieldType sampleFieldType; static { sampleFieldType = new FieldType(KNNVectorFieldMapper.Defaults.FIELD_TYPE); diff --git a/src/test/java/org/opensearch/knn/index/codec/KNNFormatFactoryTests.java b/src/test/java/org/opensearch/knn/index/codec/KNNFormatFactoryTests.java deleted file mode 100644 index 1b076f10b..000000000 --- a/src/test/java/org/opensearch/knn/index/codec/KNNFormatFactoryTests.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.knn.index.codec; - -import org.apache.lucene.codecs.Codec; -import org.opensearch.index.mapper.MapperService; -import org.opensearch.knn.KNNTestCase; - -import static org.mockito.Mockito.mock; - -public class KNNFormatFactoryTests extends KNNTestCase { - - public void testKNN91Format() { - final Codec lucene91CodecDelegate = KNNCodecFactory.CodecDelegateFactory.createKNN91DefaultDelegate(); - MapperService mapperService = mock(MapperService.class); - KNNCodecFactory knnCodecFactory = new KNNCodecFactory(mapperService); - final Codec knnCodec = knnCodecFactory.createKNNCodec(lucene91CodecDelegate); - KNNFormatFacade knnFormatFacade = KNNFormatFactory.createKNN910Format(knnCodec); - - assertNotNull(knnFormatFacade); - assertNotNull(knnFormatFacade.compoundFormat()); - assertNotNull(knnFormatFacade.docValuesFormat()); - } - - public void testKNN92Format() { - MapperService mapperService = mock(MapperService.class); - final Codec lucene92CodecDelegate = KNNCodecFactory.CodecDelegateFactory.createKNN92DefaultDelegate(); - KNNCodecFactory knnCodecFactory = new KNNCodecFactory(mapperService); - final Codec knnCodec = knnCodecFactory.createKNNCodec(lucene92CodecDelegate); - KNNFormatFacade knnFormatFacade = KNNFormatFactory.createKNN920Format(knnCodec); - - assertNotNull(knnFormatFacade); - assertNotNull(knnFormatFacade.compoundFormat()); - assertNotNull(knnFormatFacade.docValuesFormat()); - } - - public void testKNN94Format() { - MapperService mapperService = mock(MapperService.class); - final Codec lucene94CodecDelegate = KNNCodecFactory.CodecDelegateFactory.createKNN94DefaultDelegate(); - KNNCodecFactory knnCodecFactory = new KNNCodecFactory(mapperService); - final Codec knnCodec = knnCodecFactory.createKNNCodec(lucene94CodecDelegate); - KNNFormatFacade knnFormatFacade = KNNFormatFactory.createKNN940Format(knnCodec); - - assertNotNull(knnFormatFacade); - assertNotNull(knnFormatFacade.compoundFormat()); - assertNotNull(knnFormatFacade.docValuesFormat()); - } -} From b03cab93c3e51f35a9d9f523ad65b661e3b67197 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Tue, 25 Oct 2022 12:05:41 -0500 Subject: [PATCH 044/416] Add mac platform to CI (#590) (#591) Signed-off-by: Naveen Tatikonda Signed-off-by: Naveen Tatikonda (cherry picked from commit 5bb7a3fe6788fb744d5499feeac189d53afa4000) Co-authored-by: Naveen Tatikonda --- .github/workflows/CI.yml | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index f9e914099..20e403038 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -16,9 +16,10 @@ jobs: strategy: matrix: java: [11, 17] + os: [ubuntu-latest, macos-latest] name: Build and Test k-NN Plugin - runs-on: ubuntu-latest + runs-on: ${{ matrix.os }} steps: - name: Checkout k-NN @@ -29,10 +30,17 @@ jobs: with: java-version: ${{ matrix.java }} - - name: Install dependencies + - name: Install dependencies on ubuntu + if: startsWith(matrix.os,'ubuntu') run: | sudo apt-get install libopenblas-dev gfortran -y + - name: Install dependencies on macos + if: startsWith(matrix.os, 'macos') + run: | + brew reinstall gcc + export FC=/usr/local/Cellar/gcc/12.2.0/bin/gfortran + - name: Run build run: | ./gradlew build From 72f42e6447222291f11dc3ffb16dab3a7362a847 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Tue, 25 Oct 2022 10:10:53 -0700 Subject: [PATCH 045/416] [Backport 2.x] Merge efficient filtering from feature branch (#589) * Merge efficient filtering from feature branch (#588) * Adding efficient filtering Signed-off-by: Martin Gaievski (cherry picked from commit f332ccbde6152f823bb2803591aa983a8a33591d) --- qa/rolling-upgrade/build.gradle | 24 +++ .../opensearch/knn/bwc/LuceneFilteringIT.java | 86 +++++++++ .../opensearch/knn/index/KNNClusterUtil.java | 58 ++++++ .../knn/index/query/KNNQueryBuilder.java | 81 +++++++- .../knn/index/query/KNNQueryFactory.java | 79 +++++++- .../org/opensearch/knn/plugin/KNNPlugin.java | 2 + .../knn/plugin/stats/KNNCounter.java | 3 +- .../knn/plugin/stats/KNNStatsConfig.java | 4 + .../knn/plugin/stats/StatNames.java | 3 +- .../knn/index/KNNClusterTestUtils.java | 35 ++++ .../knn/index/KNNClusterUtilTests.java | 51 +++++ .../opensearch/knn/index/LuceneEngineIT.java | 108 +++++++++++ .../knn/index/query/KNNQueryBuilderTests.java | 175 ++++++++++++++++-- .../knn/index/query/KNNQueryFactoryTests.java | 31 ++++ .../plugin/action/RestKNNStatsHandlerIT.java | 20 ++ 15 files changed, 742 insertions(+), 18 deletions(-) create mode 100644 qa/rolling-upgrade/src/test/java/org/opensearch/knn/bwc/LuceneFilteringIT.java create mode 100644 src/main/java/org/opensearch/knn/index/KNNClusterUtil.java create mode 100644 src/test/java/org/opensearch/knn/index/KNNClusterTestUtils.java create mode 100644 src/test/java/org/opensearch/knn/index/KNNClusterUtilTests.java diff --git a/qa/rolling-upgrade/build.gradle b/qa/rolling-upgrade/build.gradle index c884d5692..158f7d4c7 100644 --- a/qa/rolling-upgrade/build.gradle +++ b/qa/rolling-upgrade/build.gradle @@ -33,6 +33,12 @@ task testAgainstOldCluster(type: StandaloneRestIntegTestTask) { systemProperty 'tests.rest.bwcsuite_cluster', 'old_cluster' systemProperty 'tests.plugin_bwc_version', knn_bwc_version systemProperty 'tests.skip_delete_model_index', 'true' + // Skip test if version is anything lower than 2.2 as they are not supported in those versions + if (knn_bwc_version.startsWith("1.") || knn_bwc_version.startsWith("2.0") || knn_bwc_version.startsWith("2.1")) { + filter { + excludeTestsMatching "org.opensearch.knn.bwc.LuceneFilteringIT.testLuceneFiltering" + } + } nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}".allHttpSocketURI.join(",")}") nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}".getName()}") systemProperty 'tests.security.manager', 'false' @@ -55,6 +61,12 @@ task testAgainstOneThirdUpgradedCluster(type: StandaloneRestIntegTestTask) { systemProperty 'tests.rest.first_round', 'true' systemProperty 'tests.skip_delete_model_index', 'true' systemProperty 'tests.plugin_bwc_version', knn_bwc_version + // Skip test if version is anything lower than 2.2 as they are not supported in those versions + if (knn_bwc_version.startsWith("1.") || knn_bwc_version.startsWith("2.0") || knn_bwc_version.startsWith("2.1")) { + filter { + excludeTestsMatching "org.opensearch.knn.bwc.LuceneFilteringIT.testLuceneFiltering" + } + } nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}".allHttpSocketURI.join(",")}") nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}".getName()}") systemProperty 'tests.security.manager', 'false' @@ -74,6 +86,12 @@ task testAgainstTwoThirdsUpgradedCluster(type: StandaloneRestIntegTestTask) { systemProperty 'tests.rest.first_round', 'false' systemProperty 'tests.skip_delete_model_index', 'true' systemProperty 'tests.plugin_bwc_version', knn_bwc_version + // Skip test if version is anything lower than 2.2 as they are not supported in those versions + if (knn_bwc_version.startsWith("1.") || knn_bwc_version.startsWith("2.0") || knn_bwc_version.startsWith("2.1")) { + filter { + excludeTestsMatching "org.opensearch.knn.bwc.LuceneFilteringIT.testLuceneFiltering" + } + } nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}".allHttpSocketURI.join(",")}") nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}".getName()}") systemProperty 'tests.security.manager', 'false' @@ -93,6 +111,12 @@ task testRollingUpgrade(type: StandaloneRestIntegTestTask) { systemProperty 'tests.rest.bwcsuite_cluster', 'upgraded_cluster' systemProperty 'tests.skip_delete_model_index', 'true' systemProperty 'tests.plugin_bwc_version', knn_bwc_version + // Skip test if version is anything lower than 2.2 as they are not supported in those versions + if (knn_bwc_version.startsWith("1.") || knn_bwc_version.startsWith("2.0") || knn_bwc_version.startsWith("2.1")) { + filter { + excludeTestsMatching "org.opensearch.knn.bwc.LuceneFilteringIT.testLuceneFiltering" + } + } nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}".allHttpSocketURI.join(",")}") nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}".getName()}") systemProperty 'tests.security.manager', 'false' diff --git a/qa/rolling-upgrade/src/test/java/org/opensearch/knn/bwc/LuceneFilteringIT.java b/qa/rolling-upgrade/src/test/java/org/opensearch/knn/bwc/LuceneFilteringIT.java new file mode 100644 index 000000000..3a7d0329d --- /dev/null +++ b/qa/rolling-upgrade/src/test/java/org/opensearch/knn/bwc/LuceneFilteringIT.java @@ -0,0 +1,86 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.bwc; + +import org.hamcrest.MatcherAssert; +import org.opensearch.knn.TestUtils; +import org.opensearch.knn.index.query.KNNQueryBuilder; +import org.opensearch.index.query.QueryBuilders; +import org.opensearch.index.query.TermQueryBuilder; + +import org.opensearch.client.Request; +import org.opensearch.client.ResponseException; +import org.opensearch.common.Strings; +import org.opensearch.common.xcontent.ToXContent; +import org.opensearch.common.xcontent.XContentBuilder; +import org.opensearch.common.xcontent.XContentFactory; + +import java.io.IOException; + +import static org.hamcrest.CoreMatchers.anyOf; +import static org.hamcrest.CoreMatchers.containsString; +import static org.opensearch.knn.TestUtils.NODES_BWC_CLUSTER; +import static org.opensearch.knn.common.KNNConstants.LUCENE_NAME; +import static org.opensearch.knn.common.KNNConstants.METHOD_HNSW; + +/** + * Tests scenarios specific to filtering functionality in k-NN in case Lucene is set as an engine + */ +public class LuceneFilteringIT extends AbstractRollingUpgradeTestCase { + private static final String TEST_FIELD = "test-field"; + private static final int DIMENSIONS = 50; + private static final int K = 10; + private static final int NUM_DOCS = 100; + private static final TermQueryBuilder TERM_QUERY = QueryBuilders.termQuery("_id", "100"); + + public void testLuceneFiltering() throws Exception { + waitForClusterHealthGreen(NODES_BWC_CLUSTER); + float[] queryVector = TestUtils.getQueryVectors(1, DIMENSIONS, NUM_DOCS, true)[0]; + switch (getClusterType()) { + case OLD: + createKnnIndex( + testIndex, + getKNNDefaultIndexSettings(), + createKnnIndexMapping(TEST_FIELD, DIMENSIONS, METHOD_HNSW, LUCENE_NAME) + ); + bulkAddKnnDocs(testIndex, TEST_FIELD, TestUtils.getIndexVectors(NUM_DOCS, DIMENSIONS, true), NUM_DOCS); + validateSearchKNNIndexFailed(testIndex, new KNNQueryBuilder(TEST_FIELD, queryVector, K, TERM_QUERY), K); + break; + case MIXED: + validateSearchKNNIndexFailed(testIndex, new KNNQueryBuilder(TEST_FIELD, queryVector, K, TERM_QUERY), K); + break; + case UPGRADED: + searchKNNIndex(testIndex, new KNNQueryBuilder(TEST_FIELD, queryVector, K, TERM_QUERY), K); + deleteKNNIndex(testIndex); + break; + } + } + + private void validateSearchKNNIndexFailed(String index, KNNQueryBuilder knnQueryBuilder, int resultSize) throws IOException { + XContentBuilder builder = XContentFactory.jsonBuilder().startObject().startObject("query"); + knnQueryBuilder.doXContent(builder, ToXContent.EMPTY_PARAMS); + builder.endObject().endObject(); + + Request request = new Request("POST", "/" + index + "/_search"); + + request.addParameter("size", Integer.toString(resultSize)); + request.addParameter("explain", Boolean.toString(true)); + request.addParameter("search_type", "query_then_fetch"); + request.setJsonEntity(Strings.toString(builder)); + + Exception exception = expectThrows(ResponseException.class, () -> client().performRequest(request)); + // assert for two possible exception messages, fist one can come from current version in case serialized request is coming from + // lower version, + // second exception is vise versa, when lower version node receives request with filter field from higher version + MatcherAssert.assertThat( + exception.getMessage(), + anyOf( + containsString("filter field is supported from version"), + containsString("[knn] unknown token [START_OBJECT] after [filter]") + ) + ); + } +} diff --git a/src/main/java/org/opensearch/knn/index/KNNClusterUtil.java b/src/main/java/org/opensearch/knn/index/KNNClusterUtil.java new file mode 100644 index 000000000..63a49f095 --- /dev/null +++ b/src/main/java/org/opensearch/knn/index/KNNClusterUtil.java @@ -0,0 +1,58 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index; + +import lombok.AccessLevel; +import lombok.NoArgsConstructor; +import lombok.extern.log4j.Log4j2; +import org.opensearch.Version; +import org.opensearch.cluster.service.ClusterService; + +/** + * Class abstracts information related to underlying OpenSearch cluster + */ +@NoArgsConstructor(access = AccessLevel.PRIVATE) +@Log4j2 +public class KNNClusterUtil { + + private ClusterService clusterService; + private static KNNClusterUtil instance; + + /** + * Return instance of the cluster context, must be initialized first for proper usage + * @return instance of cluster context + */ + public static synchronized KNNClusterUtil instance() { + if (instance == null) { + instance = new KNNClusterUtil(); + } + return instance; + } + + /** + * Initializes instance of cluster context by injecting dependencies + * @param clusterService + */ + public void initialize(final ClusterService clusterService) { + this.clusterService = clusterService; + } + + /** + * Return minimal OpenSearch version based on all nodes currently discoverable in the cluster + * @return minimal installed OpenSearch version, default to Version.CURRENT which is typically the latest version + */ + public Version getClusterMinVersion() { + try { + return this.clusterService.state().getNodes().getMinNodeVersion(); + } catch (Exception exception) { + log.error( + String.format("Failed to get cluster minimum node version, returning current node version %s instead.", Version.CURRENT), + exception + ); + return Version.CURRENT; + } + } +} diff --git a/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java b/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java index 1defe45e8..20cbd3a03 100644 --- a/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java +++ b/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java @@ -6,7 +6,10 @@ package org.opensearch.knn.index.query; import lombok.extern.log4j.Log4j2; +import org.opensearch.Version; import org.opensearch.index.mapper.NumberFieldMapper; +import org.opensearch.index.query.QueryBuilder; +import org.opensearch.knn.index.KNNClusterUtil; import org.opensearch.knn.index.KNNMethodContext; import org.opensearch.knn.index.mapper.KNNVectorFieldMapper; import org.opensearch.knn.index.util.KNNEngine; @@ -38,6 +41,7 @@ public class KNNQueryBuilder extends AbstractQueryBuilder { public static final ParseField VECTOR_FIELD = new ParseField("vector"); public static final ParseField K_FIELD = new ParseField("k"); + public static final ParseField FILTER_FIELD = new ParseField("filter"); public static int K_MAX = 10000; /** * The name for the knn query @@ -49,6 +53,8 @@ public class KNNQueryBuilder extends AbstractQueryBuilder { private final String fieldName; private final float[] vector; private int k = 0; + private QueryBuilder filter; + private static final Version MINIMAL_SUPPORTED_VERSION_FOR_LUCENE_HNSW_FILTER = Version.V_2_4_0; /** * Constructs a new knn query @@ -58,6 +64,10 @@ public class KNNQueryBuilder extends AbstractQueryBuilder { * @param k K nearest neighbours for the given vector */ public KNNQueryBuilder(String fieldName, float[] vector, int k) { + this(fieldName, vector, k, null); + } + + public KNNQueryBuilder(String fieldName, float[] vector, int k, QueryBuilder filter) { if (Strings.isNullOrEmpty(fieldName)) { throw new IllegalArgumentException("[" + NAME + "] requires fieldName"); } @@ -77,6 +87,7 @@ public KNNQueryBuilder(String fieldName, float[] vector, int k) { this.fieldName = fieldName; this.vector = vector; this.k = k; + this.filter = filter; } public static void initialize(ModelDao modelDao) { @@ -101,8 +112,13 @@ public KNNQueryBuilder(StreamInput in) throws IOException { fieldName = in.readString(); vector = in.readFloatArray(); k = in.readInt(); + // We're checking if all cluster nodes has at least that version or higher. This check is required + // to avoid issues with cluster upgrade + if (isClusterOnOrAfterMinRequiredVersion()) { + filter = in.readOptionalNamedWriteable(QueryBuilder.class); + } } catch (IOException ex) { - throw new RuntimeException("[KNN] Unable to create KNNQueryBuilder: " + ex); + throw new RuntimeException("[KNN] Unable to create KNNQueryBuilder", ex); } } @@ -111,6 +127,7 @@ public static KNNQueryBuilder fromXContent(XContentParser parser) throws IOExcep List vector = null; float boost = AbstractQueryBuilder.DEFAULT_BOOST; int k = 0; + QueryBuilder filter = null; String queryName = null; String currentFieldName = null; XContentParser.Token token; @@ -139,6 +156,35 @@ public static KNNQueryBuilder fromXContent(XContentParser parser) throws IOExcep "[" + NAME + "] query does not support [" + currentFieldName + "]" ); } + } else if (token == XContentParser.Token.START_OBJECT) { + String tokenName = parser.currentName(); + if (FILTER_FIELD.getPreferredName().equals(tokenName)) { + log.debug(String.format("Start parsing filter for field [%s]", fieldName)); + KNNCounter.KNN_QUERY_WITH_FILTER_REQUESTS.increment(); + // Query filters are supported starting from a certain k-NN version only, exact version is defined by + // MINIMAL_SUPPORTED_VERSION_FOR_LUCENE_HNSW_FILTER variable. + // Here we're checking if all cluster nodes has at least that version or higher. This check is required + // to avoid issues with rolling cluster upgrade + if (isClusterOnOrAfterMinRequiredVersion()) { + filter = parseInnerQueryBuilder(parser); + } else { + log.debug( + String.format( + "This version of k-NN doesn't support [filter] field, minimal required version is [%s]", + MINIMAL_SUPPORTED_VERSION_FOR_LUCENE_HNSW_FILTER + ) + ); + throw new IllegalArgumentException( + String.format( + "%s field is supported from version %s", + FILTER_FIELD.getPreferredName(), + MINIMAL_SUPPORTED_VERSION_FOR_LUCENE_HNSW_FILTER + ) + ); + } + } else { + throw new ParsingException(parser.getTokenLocation(), "[" + NAME + "] unknown token [" + token + "]"); + } } else { throw new ParsingException( parser.getTokenLocation(), @@ -153,7 +199,7 @@ public static KNNQueryBuilder fromXContent(XContentParser parser) throws IOExcep } } - KNNQueryBuilder knnQueryBuilder = new KNNQueryBuilder(fieldName, ObjectsToFloats(vector), k); + KNNQueryBuilder knnQueryBuilder = new KNNQueryBuilder(fieldName, ObjectsToFloats(vector), k, filter); knnQueryBuilder.queryName(queryName); knnQueryBuilder.boost(boost); return knnQueryBuilder; @@ -164,6 +210,11 @@ protected void doWriteTo(StreamOutput out) throws IOException { out.writeString(fieldName); out.writeFloatArray(vector); out.writeInt(k); + // We're checking if all cluster nodes has at least that version or higher. This check is required + // to avoid issues with cluster upgrade + if (isClusterOnOrAfterMinRequiredVersion()) { + out.writeOptionalNamedWriteable(filter); + } } /** @@ -184,6 +235,10 @@ public int getK() { return this.k; } + public QueryBuilder getFilter() { + return this.filter; + } + @Override public void doXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(NAME); @@ -191,6 +246,9 @@ public void doXContent(XContentBuilder builder, Params params) throws IOExceptio builder.field(VECTOR_FIELD.getPreferredName(), vector); builder.field(K_FIELD.getPreferredName(), k); + if (filter != null) { + builder.field(FILTER_FIELD.getPreferredName(), filter); + } printBoostAndQueryName(builder); builder.endObject(); builder.endObject(); @@ -225,8 +283,21 @@ protected Query doToQuery(QueryShardContext context) { ); } + if (KNNEngine.getEnginesThatCreateCustomSegmentFiles().contains(knnEngine) && filter != null) { + throw new IllegalArgumentException(String.format("Engine [%s] does not support filters", knnEngine)); + } + String indexName = context.index().getName(); - return KNNQueryFactory.create(knnEngine, indexName, this.fieldName, this.vector, this.k); + KNNQueryFactory.CreateQueryRequest createQueryRequest = KNNQueryFactory.CreateQueryRequest.builder() + .knnEngine(knnEngine) + .indexName(indexName) + .fieldName(this.fieldName) + .vector(this.vector) + .k(this.k) + .filter(this.filter) + .context(context) + .build(); + return KNNQueryFactory.create(createQueryRequest); } private ModelMetadata getModelMetadataForField(KNNVectorFieldMapper.KNNVectorFieldType knnVectorField) { @@ -257,4 +328,8 @@ protected int doHashCode() { public String getWriteableName() { return NAME; } + + private static boolean isClusterOnOrAfterMinRequiredVersion() { + return KNNClusterUtil.instance().getClusterMinVersion().onOrAfter(MINIMAL_SUPPORTED_VERSION_FOR_LUCENE_HNSW_FILTER); + } } diff --git a/src/main/java/org/opensearch/knn/index/query/KNNQueryFactory.java b/src/main/java/org/opensearch/knn/index/query/KNNQueryFactory.java index cbdb03ea8..c68ce9502 100644 --- a/src/main/java/org/opensearch/knn/index/query/KNNQueryFactory.java +++ b/src/main/java/org/opensearch/knn/index/query/KNNQueryFactory.java @@ -5,11 +5,21 @@ package org.opensearch.knn.index.query; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Getter; +import lombok.NonNull; +import lombok.Setter; import lombok.extern.log4j.Log4j2; import org.apache.lucene.search.KnnVectorQuery; import org.apache.lucene.search.Query; +import org.opensearch.index.query.QueryBuilder; +import org.opensearch.index.query.QueryShardContext; import org.opensearch.knn.index.util.KNNEngine; +import java.io.IOException; +import java.util.Optional; + /** * Creates the Lucene k-NN queries */ @@ -27,14 +37,81 @@ public class KNNQueryFactory { * @return Lucene Query */ public static Query create(KNNEngine knnEngine, String indexName, String fieldName, float[] vector, int k) { + final CreateQueryRequest createQueryRequest = CreateQueryRequest.builder() + .knnEngine(knnEngine) + .indexName(indexName) + .fieldName(fieldName) + .vector(vector) + .k(k) + .build(); + return create(createQueryRequest); + } + + /** + * Creates a Lucene query for a particular engine. + * @param createQueryRequest request object that has all required fields to construct the query + * @return Lucene Query + */ + public static Query create(CreateQueryRequest createQueryRequest) { // Engines that create their own custom segment files cannot use the Lucene's KnnVectorQuery. They need to // use the custom query type created by the plugin - if (KNNEngine.getEnginesThatCreateCustomSegmentFiles().contains(knnEngine)) { + final String indexName = createQueryRequest.getIndexName(); + final String fieldName = createQueryRequest.getFieldName(); + final int k = createQueryRequest.getK(); + final float[] vector = createQueryRequest.getVector(); + + if (KNNEngine.getEnginesThatCreateCustomSegmentFiles().contains(createQueryRequest.getKnnEngine())) { log.debug(String.format("Creating custom k-NN query for index: %s \"\", field: %s \"\", k: %d", indexName, fieldName, k)); return new KNNQuery(fieldName, vector, k, indexName); } + if (createQueryRequest.getFilter().isPresent()) { + final QueryShardContext queryShardContext = createQueryRequest.getContext() + .orElseThrow(() -> new RuntimeException("Shard context cannot be null")); + log.debug( + String.format("Creating Lucene k-NN query with filter for index [%s], field [%s] and k [%d]", indexName, fieldName, k) + ); + try { + final Query filterQuery = createQueryRequest.getFilter().get().toQuery(queryShardContext); + return new KnnVectorQuery(fieldName, vector, k, filterQuery); + } catch (IOException e) { + throw new RuntimeException("Cannot create knn query with filter", e); + } + } log.debug(String.format("Creating Lucene k-NN query for index: %s \"\", field: %s \"\", k: %d", indexName, fieldName, k)); return new KnnVectorQuery(fieldName, vector, k); } + + /** + * DTO object to hold data required to create a Query instance. + */ + @AllArgsConstructor + @Builder + @Setter + static class CreateQueryRequest { + @Getter + @NonNull + private KNNEngine knnEngine; + @Getter + @NonNull + private String indexName; + @Getter + private String fieldName; + @Getter + private float[] vector; + @Getter + private int k; + // can be null in cases filter not passed with the knn query + private QueryBuilder filter; + // can be null in cases filter not passed with the knn query + private QueryShardContext context; + + public Optional getFilter() { + return Optional.ofNullable(filter); + } + + public Optional getContext() { + return Optional.ofNullable(context); + } + } } diff --git a/src/main/java/org/opensearch/knn/plugin/KNNPlugin.java b/src/main/java/org/opensearch/knn/plugin/KNNPlugin.java index c2564f179..670294802 100644 --- a/src/main/java/org/opensearch/knn/plugin/KNNPlugin.java +++ b/src/main/java/org/opensearch/knn/plugin/KNNPlugin.java @@ -11,6 +11,7 @@ import org.opensearch.index.codec.CodecServiceFactory; import org.opensearch.index.engine.EngineFactory; import org.opensearch.knn.index.KNNCircuitBreaker; +import org.opensearch.knn.index.KNNClusterUtil; import org.opensearch.knn.index.query.KNNQueryBuilder; import org.opensearch.knn.index.KNNSettings; import org.opensearch.knn.index.mapper.KNNVectorFieldMapper; @@ -179,6 +180,7 @@ public Collection createComponents( NativeMemoryLoadStrategy.TrainingLoadStrategy.initialize(vectorReader); KNNSettings.state().initialize(client, clusterService); + KNNClusterUtil.instance().initialize(clusterService); ModelDao.OpenSearchKNNModelDao.initialize(client, clusterService, environment.settings()); ModelCache.initialize(ModelDao.OpenSearchKNNModelDao.getInstance(), clusterService); TrainingJobRunner.initialize(threadPool, ModelDao.OpenSearchKNNModelDao.getInstance()); diff --git a/src/main/java/org/opensearch/knn/plugin/stats/KNNCounter.java b/src/main/java/org/opensearch/knn/plugin/stats/KNNCounter.java index d933ce66d..ce04c9078 100644 --- a/src/main/java/org/opensearch/knn/plugin/stats/KNNCounter.java +++ b/src/main/java/org/opensearch/knn/plugin/stats/KNNCounter.java @@ -21,7 +21,8 @@ public enum KNNCounter { SCRIPT_QUERY_REQUESTS("script_query_requests"), SCRIPT_QUERY_ERRORS("script_query_errors"), TRAINING_REQUESTS("training_requests"), - TRAINING_ERRORS("training_errors"); + TRAINING_ERRORS("training_errors"), + KNN_QUERY_WITH_FILTER_REQUESTS("knn_query_with_filter_requests"); private String name; private AtomicLong count; diff --git a/src/main/java/org/opensearch/knn/plugin/stats/KNNStatsConfig.java b/src/main/java/org/opensearch/knn/plugin/stats/KNNStatsConfig.java index c41170b32..8769e0e46 100644 --- a/src/main/java/org/opensearch/knn/plugin/stats/KNNStatsConfig.java +++ b/src/main/java/org/opensearch/knn/plugin/stats/KNNStatsConfig.java @@ -55,6 +55,10 @@ public class KNNStatsConfig { .put(StatNames.CIRCUIT_BREAKER_TRIGGERED.getName(), new KNNStat<>(true, new KNNCircuitBreakerSupplier())) .put(StatNames.MODEL_INDEX_STATUS.getName(), new KNNStat<>(true, new ModelIndexStatusSupplier<>(ModelDao::getHealthStatus))) .put(StatNames.KNN_QUERY_REQUESTS.getName(), new KNNStat<>(false, new KNNCounterSupplier(KNNCounter.KNN_QUERY_REQUESTS))) + .put( + StatNames.KNN_QUERY_WITH_FILTER_REQUESTS.getName(), + new KNNStat<>(false, new KNNCounterSupplier(KNNCounter.KNN_QUERY_WITH_FILTER_REQUESTS)) + ) .put(StatNames.SCRIPT_COMPILATIONS.getName(), new KNNStat<>(false, new KNNCounterSupplier(KNNCounter.SCRIPT_COMPILATIONS))) .put( StatNames.SCRIPT_COMPILATION_ERRORS.getName(), diff --git a/src/main/java/org/opensearch/knn/plugin/stats/StatNames.java b/src/main/java/org/opensearch/knn/plugin/stats/StatNames.java index ffe5882bb..a098dd8b5 100644 --- a/src/main/java/org/opensearch/knn/plugin/stats/StatNames.java +++ b/src/main/java/org/opensearch/knn/plugin/stats/StatNames.java @@ -40,7 +40,8 @@ public enum StatNames { TRAINING_ERRORS(KNNCounter.TRAINING_ERRORS.getName()), TRAINING_MEMORY_USAGE("training_memory_usage"), TRAINING_MEMORY_USAGE_PERCENTAGE("training_memory_usage_percentage"), - SCRIPT_QUERY_ERRORS(KNNCounter.SCRIPT_QUERY_ERRORS.getName()); + SCRIPT_QUERY_ERRORS(KNNCounter.SCRIPT_QUERY_ERRORS.getName()), + KNN_QUERY_WITH_FILTER_REQUESTS(KNNCounter.KNN_QUERY_WITH_FILTER_REQUESTS.getName()); private String name; diff --git a/src/test/java/org/opensearch/knn/index/KNNClusterTestUtils.java b/src/test/java/org/opensearch/knn/index/KNNClusterTestUtils.java new file mode 100644 index 000000000..6ded05d17 --- /dev/null +++ b/src/test/java/org/opensearch/knn/index/KNNClusterTestUtils.java @@ -0,0 +1,35 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index; + +import org.opensearch.Version; +import org.opensearch.cluster.ClusterState; +import org.opensearch.cluster.node.DiscoveryNodes; +import org.opensearch.cluster.service.ClusterService; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * Collection of util methods required for testing and related to OpenSearch cluster setup and functionality + */ +public class KNNClusterTestUtils { + + /** + * Create new mock for ClusterService + * @param version min version for cluster nodes + * @return + */ + public static ClusterService mockClusterService(final Version version) { + ClusterService clusterService = mock(ClusterService.class); + ClusterState clusterState = mock(ClusterState.class); + when(clusterService.state()).thenReturn(clusterState); + DiscoveryNodes discoveryNodes = mock(DiscoveryNodes.class); + when(clusterState.getNodes()).thenReturn(discoveryNodes); + when(discoveryNodes.getMinNodeVersion()).thenReturn(version); + return clusterService; + } +} diff --git a/src/test/java/org/opensearch/knn/index/KNNClusterUtilTests.java b/src/test/java/org/opensearch/knn/index/KNNClusterUtilTests.java new file mode 100644 index 000000000..0e00a7f75 --- /dev/null +++ b/src/test/java/org/opensearch/knn/index/KNNClusterUtilTests.java @@ -0,0 +1,51 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index; + +import org.opensearch.Version; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.knn.KNNTestCase; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import static org.opensearch.knn.index.KNNClusterTestUtils.mockClusterService; + +public class KNNClusterUtilTests extends KNNTestCase { + + public void testSingleNodeCluster() { + ClusterService clusterService = mockClusterService(Version.V_2_4_0); + + final KNNClusterUtil knnClusterUtil = KNNClusterUtil.instance(); + knnClusterUtil.initialize(clusterService); + + final Version minVersion = knnClusterUtil.getClusterMinVersion(); + + assertTrue(Version.V_2_4_0.equals(minVersion)); + } + + public void testMultipleNodesCluster() { + ClusterService clusterService = mockClusterService(Version.V_2_3_0); + + final KNNClusterUtil knnClusterUtil = KNNClusterUtil.instance(); + knnClusterUtil.initialize(clusterService); + + final Version minVersion = knnClusterUtil.getClusterMinVersion(); + + assertTrue(Version.V_2_3_0.equals(minVersion)); + } + + public void testWhenErrorOnClusterStateDiscover() { + ClusterService clusterService = mock(ClusterService.class); + when(clusterService.state()).thenThrow(new RuntimeException("Cluster state is not ready")); + + final KNNClusterUtil knnClusterUtil = KNNClusterUtil.instance(); + knnClusterUtil.initialize(clusterService); + + final Version minVersion = knnClusterUtil.getClusterMinVersion(); + + assertTrue(Version.CURRENT.equals(minVersion)); + } +} diff --git a/src/test/java/org/opensearch/knn/index/LuceneEngineIT.java b/src/test/java/org/opensearch/knn/index/LuceneEngineIT.java index 9fe42cb9f..a7f04cef4 100644 --- a/src/test/java/org/opensearch/knn/index/LuceneEngineIT.java +++ b/src/test/java/org/opensearch/knn/index/LuceneEngineIT.java @@ -17,12 +17,14 @@ import org.opensearch.common.Strings; import org.opensearch.common.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.index.query.QueryBuilders; import org.opensearch.knn.KNNRestTestCase; import org.opensearch.knn.KNNResult; import org.opensearch.knn.TestUtils; import org.opensearch.knn.common.KNNConstants; import org.opensearch.knn.index.query.KNNQueryBuilder; import org.opensearch.knn.index.util.KNNEngine; +import org.opensearch.rest.RestStatus; import java.io.IOException; import java.util.Arrays; @@ -33,14 +35,19 @@ import java.util.stream.Collectors; import static org.opensearch.knn.common.KNNConstants.METHOD_HNSW; +import static org.opensearch.knn.common.KNNConstants.NMSLIB_NAME; public class LuceneEngineIT extends KNNRestTestCase { private static final int DIMENSION = 3; private static final String DOC_ID = "doc1"; + private static final String DOC_ID_2 = "doc2"; + private static final String DOC_ID_3 = "doc3"; private static final int EF_CONSTRUCTION = 128; private static final String INDEX_NAME = "test-index-1"; private static final String FIELD_NAME = "test-field-1"; + private static final String COLOR_FIELD_NAME = "color"; + private static final String TASTE_FIELD_NAME = "taste"; private static final int M = 16; private static final Float[][] TEST_INDEX_VECTORS = { { 1.0f, 1.0f, 1.0f }, { 2.0f, 2.0f, 2.0f }, { 3.0f, 3.0f, 3.0f } }; @@ -246,6 +253,107 @@ public void testDeleteDoc() throws Exception { assertEquals(0, getDocCount(INDEX_NAME)); } + public void testQueryWithFilter() throws Exception { + createKnnIndexMappingWithLuceneEngine(DIMENSION, SpaceType.L2); + + addKnnDocWithAttributes( + DOC_ID, + new float[] { 6.0f, 7.9f, 3.1f }, + ImmutableMap.of(COLOR_FIELD_NAME, "red", TASTE_FIELD_NAME, "sweet") + ); + addKnnDocWithAttributes(DOC_ID_2, new float[] { 3.2f, 2.1f, 4.8f }, ImmutableMap.of(COLOR_FIELD_NAME, "green")); + addKnnDocWithAttributes(DOC_ID_3, new float[] { 4.1f, 5.0f, 7.1f }, ImmutableMap.of(COLOR_FIELD_NAME, "red")); + + refreshAllIndices(); + + final float[] searchVector = { 6.0f, 6.0f, 4.1f }; + int kGreaterThanFilterResult = 5; + List expectedDocIds = Arrays.asList(DOC_ID, DOC_ID_3); + final Response response = searchKNNIndex( + INDEX_NAME, + new KNNQueryBuilder(FIELD_NAME, searchVector, kGreaterThanFilterResult, QueryBuilders.termQuery(COLOR_FIELD_NAME, "red")), + kGreaterThanFilterResult + ); + final String responseBody = EntityUtils.toString(response.getEntity()); + final List knnResults = parseSearchResponse(responseBody, FIELD_NAME); + + assertEquals(expectedDocIds.size(), knnResults.size()); + assertTrue(knnResults.stream().map(KNNResult::getDocId).collect(Collectors.toList()).containsAll(expectedDocIds)); + + int kLimitsFilterResult = 1; + List expectedDocIdsKLimitsFilterResult = Arrays.asList(DOC_ID); + final Response responseKLimitsFilterResult = searchKNNIndex( + INDEX_NAME, + new KNNQueryBuilder(FIELD_NAME, searchVector, kLimitsFilterResult, QueryBuilders.termQuery(COLOR_FIELD_NAME, "red")), + kLimitsFilterResult + ); + final String responseBodyKLimitsFilterResult = EntityUtils.toString(responseKLimitsFilterResult.getEntity()); + final List knnResultsKLimitsFilterResult = parseSearchResponse(responseBodyKLimitsFilterResult, FIELD_NAME); + + assertEquals(expectedDocIdsKLimitsFilterResult.size(), knnResultsKLimitsFilterResult.size()); + assertTrue( + knnResultsKLimitsFilterResult.stream() + .map(KNNResult::getDocId) + .collect(Collectors.toList()) + .containsAll(expectedDocIdsKLimitsFilterResult) + ); + } + + public void testQuery_filterWithNonLuceneEngine() throws Exception { + XContentBuilder builder = XContentFactory.jsonBuilder() + .startObject() + .startObject(PROPERTIES_FIELD_NAME) + .startObject(FIELD_NAME) + .field(TYPE_FIELD_NAME, KNN_VECTOR_TYPE) + .field(DIMENSION_FIELD_NAME, DIMENSION) + .startObject(KNNConstants.KNN_METHOD) + .field(KNNConstants.NAME, METHOD_HNSW) + .field(KNNConstants.METHOD_PARAMETER_SPACE_TYPE, SpaceType.L2.getValue()) + .field(KNNConstants.KNN_ENGINE, NMSLIB_NAME) + .endObject() + .endObject() + .endObject() + .endObject(); + + String mapping = Strings.toString(builder); + createKnnIndex(INDEX_NAME, mapping); + + addKnnDocWithAttributes( + DOC_ID, + new float[] { 6.0f, 7.9f, 3.1f }, + ImmutableMap.of(COLOR_FIELD_NAME, "red", TASTE_FIELD_NAME, "sweet") + ); + addKnnDocWithAttributes(DOC_ID_2, new float[] { 3.2f, 2.1f, 4.8f }, ImmutableMap.of(COLOR_FIELD_NAME, "green")); + addKnnDocWithAttributes(DOC_ID_3, new float[] { 4.1f, 5.0f, 7.1f }, ImmutableMap.of(COLOR_FIELD_NAME, "red")); + + final float[] searchVector = { 6.0f, 6.0f, 5.6f }; + int k = 5; + expectThrows( + ResponseException.class, + () -> searchKNNIndex( + INDEX_NAME, + new KNNQueryBuilder(FIELD_NAME, searchVector, k, QueryBuilders.termQuery(COLOR_FIELD_NAME, "red")), + k + ) + ); + } + + private void addKnnDocWithAttributes(String docId, float[] vector, Map fieldValues) throws IOException { + Request request = new Request("POST", "/" + INDEX_NAME + "/_doc/" + docId + "?refresh=true"); + + XContentBuilder builder = XContentFactory.jsonBuilder().startObject().field(FIELD_NAME, vector); + for (String fieldName : fieldValues.keySet()) { + builder.field(fieldName, fieldValues.get(fieldName)); + } + builder.endObject(); + request.setJsonEntity(Strings.toString(builder)); + client().performRequest(request); + + request = new Request("POST", "/" + INDEX_NAME + "/_refresh"); + Response response = client().performRequest(request); + assertEquals(request.getEndpoint() + ": failed", RestStatus.OK, RestStatus.fromCode(response.getStatusLine().getStatusCode())); + } + private void createKnnIndexMappingWithLuceneEngine(int dimension, SpaceType spaceType) throws Exception { XContentBuilder builder = XContentFactory.jsonBuilder() .startObject() diff --git a/src/test/java/org/opensearch/knn/index/query/KNNQueryBuilderTests.java b/src/test/java/org/opensearch/knn/index/query/KNNQueryBuilderTests.java index c3d40cbc7..e3376dda9 100644 --- a/src/test/java/org/opensearch/knn/index/query/KNNQueryBuilderTests.java +++ b/src/test/java/org/opensearch/knn/index/query/KNNQueryBuilderTests.java @@ -5,6 +5,20 @@ package org.opensearch.knn.index.query; +import com.google.common.collect.ImmutableMap; +import org.apache.lucene.search.KnnVectorQuery; +import org.apache.lucene.search.Query; +import org.opensearch.Version; +import org.opensearch.cluster.ClusterModule; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.common.io.stream.NamedWriteableAwareStreamInput; +import org.opensearch.common.io.stream.NamedWriteableRegistry; +import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.common.xcontent.NamedXContentRegistry; +import org.opensearch.index.query.QueryBuilder; +import org.opensearch.index.query.QueryBuilders; +import org.opensearch.index.query.TermQueryBuilder; import org.opensearch.knn.KNNTestCase; import org.opensearch.common.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; @@ -12,36 +26,50 @@ import org.opensearch.index.Index; import org.opensearch.index.mapper.NumberFieldMapper; import org.opensearch.index.query.QueryShardContext; +import org.opensearch.knn.index.KNNClusterUtil; +import org.opensearch.knn.index.KNNMethodContext; +import org.opensearch.knn.index.MethodComponentContext; +import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.mapper.KNNVectorFieldMapper; import org.opensearch.knn.index.util.KNNEngine; import org.opensearch.knn.indices.ModelDao; import org.opensearch.knn.indices.ModelMetadata; +import org.opensearch.plugins.SearchPlugin; import java.io.IOException; +import java.util.List; +import java.util.Optional; import static org.mockito.Mockito.anyString; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; +import static org.opensearch.knn.common.KNNConstants.METHOD_HNSW; +import static org.opensearch.knn.index.KNNClusterTestUtils.mockClusterService; public class KNNQueryBuilderTests extends KNNTestCase { + private static final String FIELD_NAME = "myvector"; + private static final int K = 1; + private static final TermQueryBuilder TERM_QUERY = QueryBuilders.termQuery("field", "value"); + private static final float[] QUERY_VECTOR = new float[] { 1.0f, 2.0f, 3.0f, 4.0f }; + public void testInvalidK() { float[] queryVector = { 1.0f, 1.0f }; /** * -ve k */ - expectThrows(IllegalArgumentException.class, () -> new KNNQueryBuilder("myvector", queryVector, -1)); + expectThrows(IllegalArgumentException.class, () -> new KNNQueryBuilder(FIELD_NAME, queryVector, -K)); /** * zero k */ - expectThrows(IllegalArgumentException.class, () -> new KNNQueryBuilder("myvector", queryVector, 0)); + expectThrows(IllegalArgumentException.class, () -> new KNNQueryBuilder(FIELD_NAME, queryVector, 0)); /** * k > KNNQueryBuilder.K_MAX */ - expectThrows(IllegalArgumentException.class, () -> new KNNQueryBuilder("myvector", queryVector, KNNQueryBuilder.K_MAX + 1)); + expectThrows(IllegalArgumentException.class, () -> new KNNQueryBuilder(FIELD_NAME, queryVector, KNNQueryBuilder.K_MAX + K)); } public void testEmptyVector() { @@ -49,18 +77,18 @@ public void testEmptyVector() { * null query vector */ float[] queryVector = null; - expectThrows(IllegalArgumentException.class, () -> new KNNQueryBuilder("myvector", queryVector, 1)); + expectThrows(IllegalArgumentException.class, () -> new KNNQueryBuilder(FIELD_NAME, queryVector, K)); /** * empty query vector */ float[] queryVector1 = {}; - expectThrows(IllegalArgumentException.class, () -> new KNNQueryBuilder("myvector", queryVector1, 1)); + expectThrows(IllegalArgumentException.class, () -> new KNNQueryBuilder(FIELD_NAME, queryVector1, K)); } public void testFromXcontent() throws Exception { float[] queryVector = { 1.0f, 2.0f, 3.0f, 4.0f }; - KNNQueryBuilder knnQueryBuilder = new KNNQueryBuilder("myvector", queryVector, 1); + KNNQueryBuilder knnQueryBuilder = new KNNQueryBuilder(FIELD_NAME, queryVector, K); XContentBuilder builder = XContentFactory.jsonBuilder(); builder.startObject(); builder.startObject(knnQueryBuilder.fieldName()); @@ -74,9 +102,74 @@ public void testFromXcontent() throws Exception { actualBuilder.equals(knnQueryBuilder); } + public void testFromXcontent_WithFilter() throws Exception { + final ClusterService clusterService = mockClusterService(Version.CURRENT); + + final KNNClusterUtil knnClusterUtil = KNNClusterUtil.instance(); + knnClusterUtil.initialize(clusterService); + + float[] queryVector = { 1.0f, 2.0f, 3.0f, 4.0f }; + KNNQueryBuilder knnQueryBuilder = new KNNQueryBuilder(FIELD_NAME, queryVector, K, TERM_QUERY); + XContentBuilder builder = XContentFactory.jsonBuilder(); + builder.startObject(); + builder.startObject(knnQueryBuilder.fieldName()); + builder.field(KNNQueryBuilder.VECTOR_FIELD.getPreferredName(), knnQueryBuilder.vector()); + builder.field(KNNQueryBuilder.K_FIELD.getPreferredName(), knnQueryBuilder.getK()); + builder.field(KNNQueryBuilder.FILTER_FIELD.getPreferredName(), knnQueryBuilder.getFilter()); + builder.endObject(); + builder.endObject(); + XContentParser contentParser = createParser(builder); + contentParser.nextToken(); + KNNQueryBuilder actualBuilder = KNNQueryBuilder.fromXContent(contentParser); + actualBuilder.equals(knnQueryBuilder); + } + + public void testFromXcontent_WithFilter_UnsupportedClusterVersion() throws Exception { + final ClusterService clusterService = mockClusterService(Version.V_2_3_0); + + final KNNClusterUtil knnClusterUtil = KNNClusterUtil.instance(); + knnClusterUtil.initialize(clusterService); + + float[] queryVector = { 1.0f, 2.0f, 3.0f, 4.0f }; + final KNNQueryBuilder knnQueryBuilder = new KNNQueryBuilder(FIELD_NAME, queryVector, K, TERM_QUERY); + final XContentBuilder builder = XContentFactory.jsonBuilder(); + builder.startObject(); + builder.startObject(knnQueryBuilder.fieldName()); + builder.field(KNNQueryBuilder.VECTOR_FIELD.getPreferredName(), knnQueryBuilder.vector()); + builder.field(KNNQueryBuilder.K_FIELD.getPreferredName(), knnQueryBuilder.getK()); + builder.field(KNNQueryBuilder.FILTER_FIELD.getPreferredName(), knnQueryBuilder.getFilter()); + builder.endObject(); + builder.endObject(); + final XContentParser contentParser = createParser(builder); + contentParser.nextToken(); + + expectThrows(IllegalArgumentException.class, () -> KNNQueryBuilder.fromXContent(contentParser)); + } + + @Override + protected NamedXContentRegistry xContentRegistry() { + List list = ClusterModule.getNamedXWriteables(); + SearchPlugin.QuerySpec spec = new SearchPlugin.QuerySpec<>( + TermQueryBuilder.NAME, + TermQueryBuilder::new, + TermQueryBuilder::fromXContent + ); + list.add(new NamedXContentRegistry.Entry(QueryBuilder.class, spec.getName(), (p, c) -> spec.getParser().fromXContent(p))); + NamedXContentRegistry registry = new NamedXContentRegistry(list); + return registry; + } + + @Override + protected NamedWriteableRegistry writableRegistry() { + final List entries = ClusterModule.getNamedWriteables(); + entries.add(new NamedWriteableRegistry.Entry(QueryBuilder.class, KNNQueryBuilder.NAME, KNNQueryBuilder::new)); + entries.add(new NamedWriteableRegistry.Entry(QueryBuilder.class, TermQueryBuilder.NAME, TermQueryBuilder::new)); + return new NamedWriteableRegistry(entries); + } + public void testDoToQuery_Normal() throws Exception { float[] queryVector = { 1.0f, 2.0f, 3.0f, 4.0f }; - KNNQueryBuilder knnQueryBuilder = new KNNQueryBuilder("myvector", queryVector, 1); + KNNQueryBuilder knnQueryBuilder = new KNNQueryBuilder(FIELD_NAME, queryVector, K); Index dummyIndex = new Index("dummy", "dummy"); QueryShardContext mockQueryShardContext = mock(QueryShardContext.class); KNNVectorFieldMapper.KNNVectorFieldType mockKNNVectorField = mock(KNNVectorFieldMapper.KNNVectorFieldType.class); @@ -89,16 +182,33 @@ public void testDoToQuery_Normal() throws Exception { assertEquals(knnQueryBuilder.vector(), query.getQueryVector()); } + public void testDoToQuery_KnnQueryWithFilter() throws Exception { + float[] queryVector = { 1.0f, 2.0f, 3.0f, 4.0f }; + KNNQueryBuilder knnQueryBuilder = new KNNQueryBuilder(FIELD_NAME, queryVector, K, TERM_QUERY); + Index dummyIndex = new Index("dummy", "dummy"); + QueryShardContext mockQueryShardContext = mock(QueryShardContext.class); + KNNVectorFieldMapper.KNNVectorFieldType mockKNNVectorField = mock(KNNVectorFieldMapper.KNNVectorFieldType.class); + when(mockQueryShardContext.index()).thenReturn(dummyIndex); + when(mockKNNVectorField.getDimension()).thenReturn(4); + MethodComponentContext methodComponentContext = new MethodComponentContext(METHOD_HNSW, ImmutableMap.of()); + KNNMethodContext knnMethodContext = new KNNMethodContext(KNNEngine.LUCENE, SpaceType.L2, methodComponentContext); + when(mockKNNVectorField.getKnnMethodContext()).thenReturn(knnMethodContext); + when(mockQueryShardContext.fieldMapper(anyString())).thenReturn(mockKNNVectorField); + Query query = knnQueryBuilder.doToQuery(mockQueryShardContext); + assertNotNull(query); + assertTrue(query instanceof KnnVectorQuery); + } + public void testDoToQuery_FromModel() { float[] queryVector = { 1.0f, 2.0f, 3.0f, 4.0f }; - KNNQueryBuilder knnQueryBuilder = new KNNQueryBuilder("myvector", queryVector, 1); + KNNQueryBuilder knnQueryBuilder = new KNNQueryBuilder(FIELD_NAME, queryVector, K); Index dummyIndex = new Index("dummy", "dummy"); QueryShardContext mockQueryShardContext = mock(QueryShardContext.class); KNNVectorFieldMapper.KNNVectorFieldType mockKNNVectorField = mock(KNNVectorFieldMapper.KNNVectorFieldType.class); when(mockQueryShardContext.index()).thenReturn(dummyIndex); // Dimension is -1. In this case, model metadata will need to provide dimension - when(mockKNNVectorField.getDimension()).thenReturn(-1); + when(mockKNNVectorField.getDimension()).thenReturn(-K); when(mockKNNVectorField.getKnnMethodContext()).thenReturn(null); String modelId = "test-model-id"; when(mockKNNVectorField.getModelId()).thenReturn(modelId); @@ -120,7 +230,7 @@ public void testDoToQuery_FromModel() { public void testDoToQuery_InvalidDimensions() { float[] queryVector = { 1.0f, 2.0f, 3.0f, 4.0f }; - KNNQueryBuilder knnQueryBuilder = new KNNQueryBuilder("myvector", queryVector, 1); + KNNQueryBuilder knnQueryBuilder = new KNNQueryBuilder(FIELD_NAME, queryVector, K); Index dummyIndex = new Index("dummy", "dummy"); QueryShardContext mockQueryShardContext = mock(QueryShardContext.class); KNNVectorFieldMapper.KNNVectorFieldType mockKNNVectorField = mock(KNNVectorFieldMapper.KNNVectorFieldType.class); @@ -128,13 +238,13 @@ public void testDoToQuery_InvalidDimensions() { when(mockKNNVectorField.getDimension()).thenReturn(400); when(mockQueryShardContext.fieldMapper(anyString())).thenReturn(mockKNNVectorField); expectThrows(IllegalArgumentException.class, () -> knnQueryBuilder.doToQuery(mockQueryShardContext)); - when(mockKNNVectorField.getDimension()).thenReturn(1); + when(mockKNNVectorField.getDimension()).thenReturn(K); expectThrows(IllegalArgumentException.class, () -> knnQueryBuilder.doToQuery(mockQueryShardContext)); } public void testDoToQuery_InvalidFieldType() throws IOException { float[] queryVector = { 1.0f, 2.0f, 3.0f, 4.0f }; - KNNQueryBuilder knnQueryBuilder = new KNNQueryBuilder("mynumber", queryVector, 1); + KNNQueryBuilder knnQueryBuilder = new KNNQueryBuilder("mynumber", queryVector, K); Index dummyIndex = new Index("dummy", "dummy"); QueryShardContext mockQueryShardContext = mock(QueryShardContext.class); NumberFieldMapper.NumberFieldType mockNumberField = mock(NumberFieldMapper.NumberFieldType.class); @@ -142,4 +252,45 @@ public void testDoToQuery_InvalidFieldType() throws IOException { when(mockQueryShardContext.fieldMapper(anyString())).thenReturn(mockNumberField); expectThrows(IllegalArgumentException.class, () -> knnQueryBuilder.doToQuery(mockQueryShardContext)); } + + public void testSerialization() throws Exception { + assertSerialization(Version.CURRENT, Optional.empty()); + + assertSerialization(Version.CURRENT, Optional.of(TERM_QUERY)); + + assertSerialization(Version.V_2_3_0, Optional.empty()); + } + + private void assertSerialization(final Version version, final Optional queryBuilderOptional) throws Exception { + final KNNQueryBuilder knnQueryBuilder = queryBuilderOptional.isPresent() + ? new KNNQueryBuilder(FIELD_NAME, QUERY_VECTOR, K, queryBuilderOptional.get()) + : new KNNQueryBuilder(FIELD_NAME, QUERY_VECTOR, K); + + final ClusterService clusterService = mockClusterService(version); + + final KNNClusterUtil knnClusterUtil = KNNClusterUtil.instance(); + knnClusterUtil.initialize(clusterService); + try (BytesStreamOutput output = new BytesStreamOutput()) { + output.setVersion(version); + output.writeNamedWriteable(knnQueryBuilder); + + try (StreamInput in = new NamedWriteableAwareStreamInput(output.bytes().streamInput(), writableRegistry())) { + in.setVersion(Version.CURRENT); + final QueryBuilder deserializedQuery = in.readNamedWriteable(QueryBuilder.class); + + assertNotNull(deserializedQuery); + assertTrue(deserializedQuery instanceof KNNQueryBuilder); + final KNNQueryBuilder deserializedKnnQueryBuilder = (KNNQueryBuilder) deserializedQuery; + assertEquals(FIELD_NAME, deserializedKnnQueryBuilder.fieldName()); + assertArrayEquals(QUERY_VECTOR, (float[]) deserializedKnnQueryBuilder.vector(), 0.0f); + assertEquals(K, deserializedKnnQueryBuilder.getK()); + if (queryBuilderOptional.isPresent()) { + assertNotNull(deserializedKnnQueryBuilder.getFilter()); + assertEquals(queryBuilderOptional.get(), deserializedKnnQueryBuilder.getFilter()); + } else { + assertNull(deserializedKnnQueryBuilder.getFilter()); + } + } + } + } } diff --git a/src/test/java/org/opensearch/knn/index/query/KNNQueryFactoryTests.java b/src/test/java/org/opensearch/knn/index/query/KNNQueryFactoryTests.java index 06b0ce6ca..908ea1021 100644 --- a/src/test/java/org/opensearch/knn/index/query/KNNQueryFactoryTests.java +++ b/src/test/java/org/opensearch/knn/index/query/KNNQueryFactoryTests.java @@ -7,6 +7,10 @@ import org.apache.lucene.search.KnnVectorQuery; import org.apache.lucene.search.Query; +import org.opensearch.index.mapper.MappedFieldType; +import org.opensearch.index.query.QueryBuilder; +import org.opensearch.index.query.QueryShardContext; +import org.opensearch.index.query.TermQueryBuilder; import org.opensearch.knn.KNNTestCase; import org.opensearch.knn.index.util.KNNEngine; @@ -14,6 +18,10 @@ import java.util.List; import java.util.stream.Collectors; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + public class KNNQueryFactoryTests extends KNNTestCase { private final int testQueryDimension = 17; private final float[] testQueryVector = new float[testQueryDimension]; @@ -42,4 +50,27 @@ public void testCreateLuceneDefaultQuery() { assertTrue(query instanceof KnnVectorQuery); } } + + public void testCreateLuceneQueryWithFilter() { + List luceneDefaultQueryEngineList = Arrays.stream(KNNEngine.values()) + .filter(knnEngine -> !KNNEngine.getEnginesThatCreateCustomSegmentFiles().contains(knnEngine)) + .collect(Collectors.toList()); + for (KNNEngine knnEngine : luceneDefaultQueryEngineList) { + QueryShardContext mockQueryShardContext = mock(QueryShardContext.class); + MappedFieldType testMapper = mock(MappedFieldType.class); + when(mockQueryShardContext.fieldMapper(any())).thenReturn(testMapper); + QueryBuilder filter = new TermQueryBuilder("foo", "fooval"); + final KNNQueryFactory.CreateQueryRequest createQueryRequest = KNNQueryFactory.CreateQueryRequest.builder() + .knnEngine(knnEngine) + .indexName(testIndexName) + .fieldName(testFieldName) + .vector(testQueryVector) + .k(testK) + .context(mockQueryShardContext) + .filter(filter) + .build(); + Query query = KNNQueryFactory.create(createQueryRequest); + assertTrue(query instanceof KnnVectorQuery); + } + } } diff --git a/src/test/java/org/opensearch/knn/plugin/action/RestKNNStatsHandlerIT.java b/src/test/java/org/opensearch/knn/plugin/action/RestKNNStatsHandlerIT.java index 2b6bb757e..0c51e77e1 100644 --- a/src/test/java/org/opensearch/knn/plugin/action/RestKNNStatsHandlerIT.java +++ b/src/test/java/org/opensearch/knn/plugin/action/RestKNNStatsHandlerIT.java @@ -22,6 +22,7 @@ import org.opensearch.common.xcontent.XContentType; import org.opensearch.index.query.MatchAllQueryBuilder; import org.opensearch.index.query.QueryBuilder; +import org.opensearch.index.query.QueryBuilders; import org.opensearch.knn.KNNRestTestCase; import org.opensearch.knn.index.query.KNNQueryBuilder; import org.opensearch.knn.index.SpaceType; @@ -67,6 +68,7 @@ public class RestKNNStatsHandlerIT extends KNNRestTestCase { private boolean isDebuggingRemoteCluster = System.getProperty("cluster.debug", "false").equals("true"); private static final String FIELD_NAME_2 = "test_field_two"; private static final String FIELD_NAME_3 = "test_field_three"; + private static final String FIELD_LUCENE_NAME = "lucene_test_field"; private static final int DIMENSION = 4; private static int DOC_ID = 0; private static final int NUM_DOCS = 10; @@ -106,6 +108,7 @@ public void testStatsValueCheck() throws IOException { Map nodeStats0 = parseNodeStatsResponse(responseBody).get(0); Integer hitCount0 = (Integer) nodeStats0.get(StatNames.HIT_COUNT.getName()); Integer missCount0 = (Integer) nodeStats0.get(StatNames.MISS_COUNT.getName()); + Integer knnQueryWithFilterCount0 = (Integer) nodeStats0.get(StatNames.KNN_QUERY_WITH_FILTER_REQUESTS.getName()); // Setup index createKnnIndex(INDEX_NAME, createKnnIndexMapping(FIELD_NAME, 2)); @@ -124,9 +127,11 @@ public void testStatsValueCheck() throws IOException { Map nodeStats1 = parseNodeStatsResponse(responseBody).get(0); Integer hitCount1 = (Integer) nodeStats1.get(StatNames.HIT_COUNT.getName()); Integer missCount1 = (Integer) nodeStats1.get(StatNames.MISS_COUNT.getName()); + Integer knnQueryWithFilterCount1 = (Integer) nodeStats1.get(StatNames.KNN_QUERY_WITH_FILTER_REQUESTS.getName()); assertEquals(hitCount0, hitCount1); assertEquals((Integer) (missCount0 + 1), missCount1); + assertEquals(knnQueryWithFilterCount0, knnQueryWithFilterCount1); // Second search: Ensure that hits=1 searchKNNIndex(INDEX_NAME, new KNNQueryBuilder(FIELD_NAME, qvector, 1), 1); @@ -137,9 +142,24 @@ public void testStatsValueCheck() throws IOException { Map nodeStats2 = parseNodeStatsResponse(responseBody).get(0); Integer hitCount2 = (Integer) nodeStats2.get(StatNames.HIT_COUNT.getName()); Integer missCount2 = (Integer) nodeStats2.get(StatNames.MISS_COUNT.getName()); + Integer knnQueryWithFilterCount2 = (Integer) nodeStats2.get(StatNames.KNN_QUERY_WITH_FILTER_REQUESTS.getName()); assertEquals(missCount1, missCount2); assertEquals((Integer) (hitCount1 + 1), hitCount2); + assertEquals(knnQueryWithFilterCount0, knnQueryWithFilterCount2); + + putMappingRequest(INDEX_NAME, createKnnIndexMapping(FIELD_LUCENE_NAME, 2, METHOD_HNSW, LUCENE_NAME)); + addKnnDoc(INDEX_NAME, "2", FIELD_LUCENE_NAME, vector); + + searchKNNIndex(INDEX_NAME, new KNNQueryBuilder(FIELD_LUCENE_NAME, qvector, 1, QueryBuilders.termQuery("_id", "1")), 1); + + response = getKnnStats(Collections.emptyList(), Collections.emptyList()); + responseBody = EntityUtils.toString(response.getEntity()); + + Map nodeStats3 = parseNodeStatsResponse(responseBody).get(0); + Integer knnQueryWithFilterCount3 = (Integer) nodeStats3.get(StatNames.KNN_QUERY_WITH_FILTER_REQUESTS.getName()); + + assertEquals((Integer) (knnQueryWithFilterCount0 + 1), knnQueryWithFilterCount3); } /** From 515dcdbf9f831cc30d4d0b77217dee6e817cfe7d Mon Sep 17 00:00:00 2001 From: Naveen Tatikonda Date: Wed, 26 Oct 2022 18:30:12 -0500 Subject: [PATCH 046/416] Add windows support (#583) * Add Windows Support Signed-off-by: Naveen * Add license for libwinpthread and copy all licenses to LICENSE.txt Signed-off-by: Naveen * Download OpenBLAS Signed-off-by: Naveen * Remove Git Patch and add branching logic for CMakeLists.txt Signed-off-by: Naveen Signed-off-by: Naveen --- .github/workflows/CI.yml | 46 ++++++ LICENSE.txt | 202 ++++++++++++++++++++++++++ THIRD-PARTY | 129 ---------------- build-tools/knnplugin-coverage.gradle | 10 +- build.gradle | 22 ++- jni/CMakeLists.txt | 128 ++++++++++------ scripts/windowsScript.ps1 | 19 +++ 7 files changed, 377 insertions(+), 179 deletions(-) delete mode 100644 THIRD-PARTY create mode 100644 scripts/windowsScript.ps1 diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 20e403038..e2ea9f21b 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -50,6 +50,52 @@ jobs: with: token: ${{ secrets.CODECOV_TOKEN }} + Build-k-NN-Windows: + strategy: + matrix: + java: [ 11, 17 ] + + name: Build and Test k-NN Plugin on Windows + runs-on: windows-latest + + steps: + - name: Checkout k-NN + uses: actions/checkout@v1 + + - name: Setup Java ${{ matrix.java }} + uses: actions/setup-java@v1 + with: + java-version: ${{ matrix.java }} + + - name: Install MinGW Using Scoop + run: | + iex "& {$(irm get.scoop.sh)} -RunAsAdmin" + scoop bucket add main + scoop install mingw + + - name: Add MinGW to PATH + run: | + echo "C:/Users/runneradmin/scoop/apps/mingw/current/bin" >> $env:GITHUB_PATH + refreshenv + + - name: Download OpenBLAS + run: | + curl -L -O https://github.com/xianyi/OpenBLAS/releases/download/v0.3.21/OpenBLAS-0.3.21-x64.zip + mkdir OpenBLAS + Expand-Archive -Path .\OpenBLAS-0.3.21-x64.zip -DestinationPath .\OpenBLAS\ + mkdir ./src/main/resources/windowsDependencies + cp ./OpenBLAS/bin/libopenblas.dll ./src/main/resources/windowsDependencies/ + rm .\OpenBLAS-0.3.21-x64.zip + rm -r .\OpenBLAS\ + + - name: Run build + run: | + ./gradlew.bat build + - name: Upload Coverage Report + uses: codecov/codecov-action@v1 + with: + token: ${{ secrets.CODECOV_TOKEN }} + # - name: Pull and Run Docker for security tests # run: | # plugin=`ls build/distributions/*.zip` diff --git a/LICENSE.txt b/LICENSE.txt index d64569567..62364a94c 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -200,3 +200,205 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. + +--------------- +OpenSearch k-NN plugin includes the following third-party software/licensing: + +** faiss; version 1.7.6 -- +MIT License + +Copyright (c) Facebook, Inc. and its affiliates. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +** OpenBLAS; version 0.3.3, 0.3.21 +Copyright (c) 2011-2014, The OpenBLAS Project +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + 1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + 3. Neither the name of the OpenBLAS project nor the names of + its contributors may be used to endorse or promote products + derived from this software without specific prior written + permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE +USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +Cerberus + +ISC License + +Copyright (c) 2012-2016 Nicola Iarocci. + +Permission to use, copy, modify, and/or distribute this software for any +purpose with or without fee is hereby granted, provided that the above +copyright notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH +REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND +FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, +INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR +OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +PERFORMANCE OF THIS SOFTWARE. + +------ +PyYAML + + +Copyright (c) 2017-2021 Ingy döt Net +Copyright (c) 2006-2016 Kirill Simonov + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +------ + +Copyright (c) 2005-2021, NumPy Developers. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + + * Neither the name of the NumPy Developers nor the names of any + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +------ + +** libwinpthread; version 12.2.0 -- https://www.mingw-w64.org/ +Copyright (c) 2011 mingw-w64 project + +Permission is hereby granted, free of charge, to any person obtaining a +copy of this software and associated documentation files (the "Software"), +to deal in the Software without restriction, including without limitation +the rights to use, copy, modify, merge, publish, distribute, sublicense, +and/or sell copies of the Software, and to permit persons to whom the +Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + + +/* + * Parts of this library are derived by: + * + * Posix Threads library for Microsoft Windows + * + * Use at own risk, there is no implied warranty to this code. + * It uses undocumented features of Microsoft Windows that can change + * at any time in the future. + * + * (C) 2010 Lockless Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without +modification, + * are permitted provided that the following conditions are met: + * + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * * Neither the name of Lockless Inc. nor the names of its contributors may be + * used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AN + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. + * IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, + * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE + * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF +ADVISED + * OF THE POSSIBILITY OF SUCH DAMAGE. + */ diff --git a/THIRD-PARTY b/THIRD-PARTY deleted file mode 100644 index 2add2c0c2..000000000 --- a/THIRD-PARTY +++ /dev/null @@ -1,129 +0,0 @@ -** faiss; version 1.7.6 -- -MIT License - -Copyright (c) Facebook, Inc. and its affiliates. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - -** OpenBLAS; version 0.3.3 -Copyright (c) 2011-2014, The OpenBLAS Project -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - 1. Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - - 2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in - the documentation and/or other materials provided with the - distribution. - 3. Neither the name of the OpenBLAS project nor the names of - its contributors may be used to endorse or promote products - derived from this software without specific prior written - permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE -LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE -USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -Cerberus - -ISC License - -Copyright (c) 2012-2016 Nicola Iarocci. - -Permission to use, copy, modify, and/or distribute this software for any -purpose with or without fee is hereby granted, provided that the above -copyright notice and this permission notice appear in all copies. - -THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH -REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND -FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, -INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM -LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR -OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR -PERFORMANCE OF THIS SOFTWARE. - ------- -PyYAML - - -Copyright (c) 2017-2021 Ingy döt Net -Copyright (c) 2006-2016 Kirill Simonov - -Permission is hereby granted, free of charge, to any person obtaining a copy of -this software and associated documentation files (the "Software"), to deal in -the Software without restriction, including without limitation the rights to -use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies -of the Software, and to permit persons to whom the Software is furnished to do -so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - ------- - -Copyright (c) 2005-2021, NumPy Developers. -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following - disclaimer in the documentation and/or other materials provided - with the distribution. - - * Neither the name of the NumPy Developers nor the names of any - contributors may be used to endorse or promote products derived - from this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/build-tools/knnplugin-coverage.gradle b/build-tools/knnplugin-coverage.gradle index 57a0eeeec..386cfcfda 100644 --- a/build-tools/knnplugin-coverage.gradle +++ b/build-tools/knnplugin-coverage.gradle @@ -16,6 +16,8 @@ * cluster is stopped and dump it to a file. Luckily our current security policy seems to allow this. This will also probably * break if there are multiple nodes in the integTestCluster. But for now... it sorta works. */ + +import org.apache.tools.ant.taskdefs.condition.Os apply plugin: 'jacoco' // Get gradle to generate the required jvm agent arg for us using a dummy tasks of type Test. Unfortunately Elastic's @@ -63,7 +65,13 @@ afterEvaluate { jacocoTestReport.dependsOn integTest testClusters.integTest { - jvmArgs " ${dummyIntegTest.jacoco.getAsJvmArg()}".replace('javaagent:','javaagent:/') + if (Os.isFamily(Os.FAMILY_WINDOWS)) { + // Replacing build with absolute path to fix the error "error opening zip file or JAR manifest missing : /build/tmp/expandedArchives/..../jacocoagent.jar" + jvmArgs " ${dummyIntegTest.jacoco.getAsJvmArg()}".replace('build',"${buildDir}") + } else { + jvmArgs " ${dummyIntegTest.jacoco.getAsJvmArg()}".replace('javaagent:','javaagent:/') + } + systemProperty 'com.sun.management.jmxremote', "true" systemProperty 'com.sun.management.jmxremote.authenticate', "false" systemProperty 'com.sun.management.jmxremote.port', "7777" diff --git a/build.gradle b/build.gradle index fe1762fb3..3def63c2d 100644 --- a/build.gradle +++ b/build.gradle @@ -4,6 +4,7 @@ */ import org.opensearch.gradle.test.RestIntegTestTask +import org.apache.tools.ant.taskdefs.condition.Os buildscript { ext { @@ -170,9 +171,19 @@ dependencies { def opensearch_tmp_dir = rootProject.file('build/private/opensearch_tmp').absoluteFile opensearch_tmp_dir.mkdirs() +task windowsPatches(type:Exec) { + commandLine 'cmd', '/c', "Powershell -File $rootDir\\scripts\\windowsScript.ps1" +} + task cmakeJniLib(type:Exec) { workingDir 'jni' - commandLine 'cmake', '.', "-DKNN_PLUGIN_VERSION=${opensearch_version}" + if (Os.isFamily(Os.FAMILY_WINDOWS)) { + dependsOn windowsPatches + commandLine 'cmake', '.', "-G", "Unix Makefiles", "-DKNN_PLUGIN_VERSION=${opensearch_version}", "-DBLAS_LIBRARIES=$rootDir\\src\\main\\resources\\windowsDependencies\\libopenblas.dll", "-DLAPACK_LIBRARIES=$rootDir\\src\\main\\resources\\windowsDependencies\\libopenblas.dll" + } + else { + commandLine 'cmake', '.', "-DKNN_PLUGIN_VERSION=${opensearch_version}" + } } task buildJniLib(type:Exec) { @@ -185,6 +196,10 @@ test { dependsOn buildJniLib systemProperty 'tests.security.manager', 'false' systemProperty "java.library.path", "$rootDir/jni/release" + if (Os.isFamily(Os.FAMILY_WINDOWS)) { + // Add the paths of built JNI libraries and its dependent libraries to PATH variable in System variables + environment('PATH', System.getenv('PATH') + ";$rootDir/jni/release" + ";$rootDir/src/main/resources/windowsDependencies") + } } def _numNodes = findProperty('numNodes') as Integer ?: 1 @@ -225,6 +240,11 @@ integTest { testClusters.integTest { testDistribution = "ARCHIVE" plugin(project.tasks.bundlePlugin.archiveFile) + if (Os.isFamily(Os.FAMILY_WINDOWS)) { + // Add the paths of built JNI libraries and its dependent libraries to PATH variable in System variables + environment('PATH', System.getenv('PATH') + ";$rootDir/jni/release" + ";$rootDir/src/main/resources/windowsDependencies") + } + // Cluster shrink exception thrown if we try to set numberOfNodes to 1, so only apply if > 1 if (_numNodes > 1) numberOfNodes = _numNodes // When running integration tests it doesn't forward the --debug-jvm to the cluster anymore diff --git a/jni/CMakeLists.txt b/jni/CMakeLists.txt index c6b2f5f22..75d4ba81b 100644 --- a/jni/CMakeLists.txt +++ b/jni/CMakeLists.txt @@ -35,6 +35,14 @@ if (${CMAKE_SYSTEM_NAME} STREQUAL Darwin) elseif(${CMAKE_SYSTEM_NAME} STREQUAL Linux) set(JVM_OS_TYPE linux) set(LIB_EXT .so) +elseif(${CMAKE_SYSTEM_NAME} STREQUAL Windows) +# Set the CXX_COMPILER_VERSION, CMAKE_CXX_FLAGS, JVM_OS_TYPE, prefix and extension for the target libraries that are built. + set(CXX_COMPILER_VERSION ${CMAKE_CXX_COMPILER_VERSION}) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fpermissive") + set(JVM_OS_TYPE win32) + set(LIB_EXT .dll) + set(CMAKE_SHARED_LIBRARY_PREFIX "") + set(CMAKE_STATIC_LIBRARY_PREFIX "") else() message(FATAL_ERROR "Unable to run on system: ${CMAKE_SYSTEM_NAME}") endif() @@ -57,7 +65,14 @@ add_library(${TARGET_LIB_COMMON} SHARED ${CMAKE_CURRENT_SOURCE_DIR}/src/jni_util target_include_directories(${TARGET_LIB_COMMON} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include $ENV{JAVA_HOME}/include $ENV{JAVA_HOME}/include/${JVM_OS_TYPE}) set_target_properties(${TARGET_LIB_COMMON} PROPERTIES SUFFIX ${LIB_EXT}) set_target_properties(${TARGET_LIB_COMMON} PROPERTIES POSITION_INDEPENDENT_CODE ON) -set_target_properties(${TARGET_LIB_COMMON} PROPERTIES LIBRARY_OUTPUT_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/release) + +if (WIN32) +# Use RUNTIME_OUTPUT_DIRECTORY, to build the target library (opensearchknn_common) in the specified directory at runtime. + set_target_properties(${TARGET_LIB_COMMON} PROPERTIES RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/release) +else() + set_target_properties(${TARGET_LIB_COMMON} PROPERTIES LIBRARY_OUTPUT_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/release) +endif() + list(APPEND TARGET_LIBS ${TARGET_LIB_COMMON}) # ---------------------------------------------------------------------------- @@ -79,7 +94,13 @@ if (${CONFIG_NMSLIB} STREQUAL ON OR ${CONFIG_ALL} STREQUAL ON OR ${CONFIG_TEST} target_include_directories(${TARGET_LIB_NMSLIB} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include $ENV{JAVA_HOME}/include $ENV{JAVA_HOME}/include/${JVM_OS_TYPE} ${CMAKE_CURRENT_SOURCE_DIR}/external/nmslib/similarity_search/include) set_target_properties(${TARGET_LIB_NMSLIB} PROPERTIES SUFFIX ${LIB_EXT}) set_target_properties(${TARGET_LIB_NMSLIB} PROPERTIES POSITION_INDEPENDENT_CODE ON) - set_target_properties(${TARGET_LIB_NMSLIB} PROPERTIES LIBRARY_OUTPUT_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/release) + + if (WIN32) + # Use RUNTIME_OUTPUT_DIRECTORY, to build the target library (opensearchknn_nmslib) in the specified directory at runtime. + set_target_properties(${TARGET_LIB_NMSLIB} PROPERTIES RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/release) + else() + set_target_properties(${TARGET_LIB_NMSLIB} PROPERTIES LIBRARY_OUTPUT_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/release) + endif() list(APPEND TARGET_LIBS ${TARGET_LIB_NMSLIB}) endif () @@ -130,7 +151,13 @@ if (${CONFIG_FAISS} STREQUAL ON OR ${CONFIG_ALL} STREQUAL ON OR ${CONFIG_TEST} S target_include_directories(${TARGET_LIB_FAISS} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include $ENV{JAVA_HOME}/include $ENV{JAVA_HOME}/include/${JVM_OS_TYPE} ${CMAKE_CURRENT_SOURCE_DIR}/external/faiss) set_target_properties(${TARGET_LIB_FAISS} PROPERTIES SUFFIX ${LIB_EXT}) set_target_properties(${TARGET_LIB_FAISS} PROPERTIES POSITION_INDEPENDENT_CODE ON) - set_target_properties(${TARGET_LIB_FAISS} PROPERTIES LIBRARY_OUTPUT_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/release) + + if (WIN32) + # Use RUNTIME_OUTPUT_DIRECTORY, to build the target library (opensearchknn_faiss) in the specified directory at runtime. + set_target_properties(${TARGET_LIB_FAISS} PROPERTIES RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/release) + else() + set_target_properties(${TARGET_LIB_FAISS} PROPERTIES LIBRARY_OUTPUT_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/release) + endif() list(APPEND TARGET_LIBS ${TARGET_LIB_FAISS}) endif () @@ -138,51 +165,56 @@ endif () # --------------------------------------------------------------------------- # --------------------------------- TESTS ----------------------------------- -if (${CONFIG_ALL} STREQUAL ON OR ${CONFIG_TEST} STREQUAL ON) - # Reference - https://crascit.com/2015/07/25/cmake-gtest/ - configure_file(CMakeLists.txt.in googletest-download/CMakeLists.txt) - execute_process(COMMAND "${CMAKE_COMMAND}" -G "${CMAKE_GENERATOR}" . - WORKING_DIRECTORY "${CMAKE_BINARY_DIR}/googletest-download" - ) - execute_process(COMMAND "${CMAKE_COMMAND}" --build . - WORKING_DIRECTORY "${CMAKE_BINARY_DIR}/googletest-download" - ) - set(gtest_force_shared_crt ON CACHE BOOL "" FORCE) - - add_subdirectory("${CMAKE_BINARY_DIR}/googletest-src" - "${CMAKE_BINARY_DIR}/googletest-build" EXCLUDE_FROM_ALL - ) - add_executable( - jni_test - tests/faiss_wrapper_test.cpp - tests/nmslib_wrapper_test.cpp - tests/test_util.cpp) - - target_link_libraries( - jni_test - gtest_main - gmock_main - faiss - NonMetricSpaceLib - OpenMP::OpenMP_CXX - ${TARGET_LIB_FAISS} - ${TARGET_LIB_NMSLIB} - ${TARGET_LIB_COMMON} - ) - - target_include_directories(jni_test PRIVATE - ${CMAKE_CURRENT_SOURCE_DIR}/tests - ${CMAKE_CURRENT_SOURCE_DIR}/include - $ENV{JAVA_HOME}/include - $ENV{JAVA_HOME}/include/${JVM_OS_TYPE} - ${CMAKE_CURRENT_SOURCE_DIR}/external/faiss - ${CMAKE_CURRENT_SOURCE_DIR}/external/nmslib/similarity_search/include - ${gtest_SOURCE_DIR}/include - ${gmock_SOURCE_DIR}/include) - - - set_target_properties(jni_test PROPERTIES RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/bin) -endif () +# Windows : Comment the TESTS for now because the tests are failing(failing to build jni_tests.exe) if we are building our target libraries as SHARED libraries. +# TODO: Fix the failing JNI TESTS on Windows +if (!WIN32) + if (${CONFIG_ALL} STREQUAL ON OR ${CONFIG_TEST} STREQUAL ON) + # Reference - https://crascit.com/2015/07/25/cmake-gtest/ + configure_file(CMakeLists.txt.in googletest-download/CMakeLists.txt) + execute_process(COMMAND "${CMAKE_COMMAND}" -G "${CMAKE_GENERATOR}" . + WORKING_DIRECTORY "${CMAKE_BINARY_DIR}/googletest-download" + ) + execute_process(COMMAND "${CMAKE_COMMAND}" --build . + WORKING_DIRECTORY "${CMAKE_BINARY_DIR}/googletest-download" + ) + set(gtest_force_shared_crt ON CACHE BOOL "" FORCE) + + add_subdirectory("${CMAKE_BINARY_DIR}/googletest-src" + "${CMAKE_BINARY_DIR}/googletest-build" EXCLUDE_FROM_ALL + ) + add_executable( + jni_test + tests/faiss_wrapper_test.cpp + tests/nmslib_wrapper_test.cpp + tests/test_util.cpp) + + target_link_libraries( + jni_test + gtest_main + gmock_main + faiss + NonMetricSpaceLib + OpenMP::OpenMP_CXX + ${TARGET_LIB_FAISS} + ${TARGET_LIB_NMSLIB} + ${TARGET_LIB_COMMON} + ) + + target_include_directories(jni_test PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR}/tests + ${CMAKE_CURRENT_SOURCE_DIR}/include + $ENV{JAVA_HOME}/include + $ENV{JAVA_HOME}/include/${JVM_OS_TYPE} + ${CMAKE_CURRENT_SOURCE_DIR}/external/faiss + ${CMAKE_CURRENT_SOURCE_DIR}/external/nmslib/similarity_search/include + ${gtest_SOURCE_DIR}/include + ${gmock_SOURCE_DIR}/include) + + + set_target_properties(jni_test PROPERTIES RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/bin) + endif () +endif() + # --------------------------------------------------------------------------- # -------------------------------- INSTALL ---------------------------------- diff --git a/scripts/windowsScript.ps1 b/scripts/windowsScript.ps1 new file mode 100644 index 000000000..eee64046a --- /dev/null +++ b/scripts/windowsScript.ps1 @@ -0,0 +1,19 @@ +# +# Copyright OpenSearch Contributors +# SPDX-License-Identifier: Apache-2.0 +# + +git submodule update --init -- jni/external/nmslib +git submodule update --init -- jni/external/faiss + +# _MSC_VER is a predefined macro which defines the version of Visual Studio Compiler +# As we are using x86_64-w64-mingw32-gcc compiler we need to replace this macro with __MINGW32__ +(Get-Content jni/external/faiss/faiss/impl/index_read.cpp).replace('_MSC_VER', '__MINGW32__') | Set-Content jni/external/faiss/faiss/impl/index_read.cpp +(Get-Content jni/external/faiss/faiss/impl/index_write.cpp).replace('_MSC_VER', '__MINGW32__') | Set-Content jni/external/faiss/faiss/impl/index_write.cpp + +# is a Unix header and is not available on Windows. So, adding condition to include it if not running on Windows +# Replace '#include ' with +# #ifndef __MINGW32__ +# #include +# #endif +(Get-Content jni/external/faiss/faiss/OnDiskInvertedLists.cpp).replace('#include ', "#ifndef __MINGW32__`n#include `n#endif") | Set-Content jni/external/faiss/faiss/OnDiskInvertedLists.cpp From 949ea01420b64a60fdbfac52dcd6e6f2858fa030 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Wed, 26 Oct 2022 18:11:37 -0700 Subject: [PATCH 047/416] Update build script to publish to maven local (#597) Updates build script to publish zip to local maven. This allows plugins like neural search to pick up the zip as a dependency. Signed-off-by: John Mazanec (cherry picked from commit 532369963938ecd42575d463b46ef8815b927fa7) --- scripts/build.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/build.sh b/scripts/build.sh index 2ff2bc2b6..b19950041 100644 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -101,6 +101,7 @@ make opensearchknn_faiss opensearchknn_nmslib cd $work_dir ./gradlew assemble --no-daemon --refresh-dependencies -DskipTests=true -Dopensearch.version=$VERSION -Dbuild.snapshot=$SNAPSHOT -Dbuild.version_qualifier=$QUALIFIER ./gradlew publishPluginZipPublicationToZipStagingRepository -Dopensearch.version=$VERSION -Dbuild.snapshot=$SNAPSHOT -Dbuild.version_qualifier=$QUALIFIER +./gradlew publishPluginZipPublicationToMavenLocal -Dbuild.snapshot=$SNAPSHOT -Dbuild.version_qualifier=$QUALIFIER -Dopensearch.version=$VERSION # Add lib to zip zipPath=$(find "$(pwd)" -path \*build/distributions/*.zip) From 0eece6083eda553213598019a5b527a5d9b4dca2 Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Thu, 27 Oct 2022 16:56:37 -0400 Subject: [PATCH 048/416] Add windows build script changes here (#595) Signed-off-by: Peter Zhu Signed-off-by: Peter Zhu Co-authored-by: Naveen --- scripts/build.sh | 43 ++++++++++++++++++++++++++++++++----------- 1 file changed, 32 insertions(+), 11 deletions(-) diff --git a/scripts/build.sh b/scripts/build.sh index b19950041..9790a4483 100644 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -75,7 +75,17 @@ work_dir=$PWD git submodule update --init -- jni/external/nmslib git submodule update --init -- jni/external/faiss -# Build knn libs +# Setup compile time dependency for Windows only +# As Linux version already have OpenBlas in the runner +if [ "$PLATFORM" = "windows" ]; then + openBlasVersion="0.3.21" + openBlasFile="openblas_${openBlasVersion}" + curl -SL https://github.com/xianyi/OpenBLAS/releases/download/v${openBlasVersion}/OpenBLAS-${openBlasVersion}-x64.zip -o ${openBlasFile}.zip + unzip -j -o ${openBlasFile}.zip bin/libopenblas.dll -d ./src/main/resources/windowsDependencies + rm -rf ${openBlasFile}.zip +fi + +# Setup knnlib build params for all platforms cd jni # For x64, generalize arch so library is compatible for processors without simd instruction extensions @@ -95,11 +105,10 @@ if [ "$JAVA_HOME" = "" ]; then echo "SET JAVA_HOME=$JAVA_HOME" fi -cmake . -make opensearchknn_faiss opensearchknn_nmslib - +# Build k-NN lib and plugin through gradle tasks cd $work_dir -./gradlew assemble --no-daemon --refresh-dependencies -DskipTests=true -Dopensearch.version=$VERSION -Dbuild.snapshot=$SNAPSHOT -Dbuild.version_qualifier=$QUALIFIER +# Gradle build is used here to replace gradle assemble due to build will also call cmake and make before generating jars +./gradlew build --no-daemon --refresh-dependencies -x integTest -DskipTests=true -Dopensearch.version=$VERSION -Dbuild.snapshot=$SNAPSHOT -Dbuild.version_qualifier=$QUALIFIER ./gradlew publishPluginZipPublicationToZipStagingRepository -Dopensearch.version=$VERSION -Dbuild.snapshot=$SNAPSHOT -Dbuild.version_qualifier=$QUALIFIER ./gradlew publishPluginZipPublicationToMavenLocal -Dbuild.snapshot=$SNAPSHOT -Dbuild.version_qualifier=$QUALIFIER -Dopensearch.version=$VERSION @@ -107,19 +116,31 @@ cd $work_dir zipPath=$(find "$(pwd)" -path \*build/distributions/*.zip) distributions="$(dirname "${zipPath}")" mkdir $distributions/lib -cp ./jni/release/libopensearchknn* $distributions/lib - -# Copy libomp to be packaged with the lib contents -ompPath=$(ldconfig -p | grep libgomp | cut -d ' ' -f 4) -cp $ompPath $distributions/lib +libPrefix="libopensearchknn" +if [ "$PLATFORM" = "windows" ]; then + libPrefix="opensearchknn" + cp -v ./src/main/resources/windowsDependencies/libopenblas.dll $distributions/lib + + # Have to define $MINGW_BIN either in ENV VAR or User Provided Var + cp -v "$MINGW_BIN/libgcc_s_seh-1.dll" $distributions/lib + cp -v "$MINGW_BIN/libwinpthread-1.dll" $distributions/lib + cp -v "$MINGW_BIN/libstdc++-6.dll" $distributions/lib + cp -v "$MINGW_BIN/libgomp-1.dll" $distributions/lib +else + ompPath=$(ldconfig -p | grep libgomp | cut -d ' ' -f 4) + cp -v $ompPath $distributions/lib +fi +cp -v ./jni/release/${libPrefix}* $distributions/lib +ls -l $distributions/lib +# Add lib directory to the k-NN plugin zip cd $distributions zip -ur $zipPath lib cd $work_dir echo "COPY ${distributions}/*.zip" mkdir -p $OUTPUT/plugins -cp ${distributions}/*.zip $OUTPUT/plugins +cp -v ${distributions}/*.zip $OUTPUT/plugins mkdir -p $OUTPUT/maven/org/opensearch cp -r ./build/local-staging-repo/org/opensearch/. $OUTPUT/maven/org/opensearch From 847b0743125915184e20c97307c2e418c03a113c Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Mon, 31 Oct 2022 13:16:25 -0500 Subject: [PATCH 049/416] Disable Code Coverage for Windows and Mac Platforms (#603) (#604) Signed-off-by: Naveen Tatikonda (cherry picked from commit d3f89590e6590823c9affa0e35313a3429e72cb8) Co-authored-by: Naveen Tatikonda --- .github/workflows/CI.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index e2ea9f21b..31d16f21f 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -46,6 +46,7 @@ jobs: ./gradlew build - name: Upload Coverage Report + if: startsWith(matrix.os,'ubuntu') uses: codecov/codecov-action@v1 with: token: ${{ secrets.CODECOV_TOKEN }} @@ -91,10 +92,6 @@ jobs: - name: Run build run: | ./gradlew.bat build - - name: Upload Coverage Report - uses: codecov/codecov-action@v1 - with: - token: ${{ secrets.CODECOV_TOKEN }} # - name: Pull and Run Docker for security tests # run: | From 40286a223ac4afceec2ab97c9c659a4b98cde344 Mon Sep 17 00:00:00 2001 From: Martin Gaievski Date: Thu, 3 Nov 2022 10:19:28 -0700 Subject: [PATCH 050/416] Fixing flaky test for knn codecs (#613) Signed-off-by: Martin Gaievski --- .../org/opensearch/knn/index/codec/KNNCodecTestCase.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestCase.java b/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestCase.java index 43ae19320..b570cac49 100644 --- a/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestCase.java +++ b/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestCase.java @@ -57,12 +57,11 @@ import java.util.function.Function; import java.util.stream.Collectors; -import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.RETURNS_DEEP_STUBS; +import static org.mockito.Mockito.atLeastOnce; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.spy; -import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import static org.opensearch.Version.CURRENT; @@ -321,7 +320,7 @@ public void testKnnVectorIndex( IndexReader reader = writer.getReader(); writer.close(); - verify(perFieldKnnVectorsFormatSpy).getKnnVectorsFormatForField(anyString()); + verify(perFieldKnnVectorsFormatSpy).getKnnVectorsFormatForField(eq(FIELD_NAME_ONE)); IndexSearcher searcher = new IndexSearcher(reader); Query query = KNNQueryFactory.create(KNNEngine.LUCENE, "dummy", FIELD_NAME_ONE, new float[] { 1.0f, 0.0f, 0.0f }, 1); @@ -348,7 +347,7 @@ public void testKnnVectorIndex( ResourceWatcherService resourceWatcherService = createDisabledResourceWatcherService(); NativeMemoryLoadStrategy.IndexLoadStrategy.initialize(resourceWatcherService); - verify(perFieldKnnVectorsFormatSpy, times(2)).getKnnVectorsFormatForField(anyString()); + verify(perFieldKnnVectorsFormatSpy, atLeastOnce()).getKnnVectorsFormatForField(eq(FIELD_NAME_TWO)); IndexSearcher searcher1 = new IndexSearcher(reader1); Query query1 = KNNQueryFactory.create(KNNEngine.LUCENE, "dummy", FIELD_NAME_TWO, new float[] { 1.0f, 0.0f }, 1); From 9d59ed428fb53511537b73e2109fd69a1ed786eb Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Thu, 3 Nov 2022 14:37:57 -0700 Subject: [PATCH 051/416] Add release note for 2.4.0 (#611) (#615) Signed-off-by: Heemin Kim Signed-off-by: Heemin Kim (cherry picked from commit 2e0de8ccde191f4e7211d8410be217ff08a3ef45) Co-authored-by: Heemin Kim --- .../opensearch-knn.release-notes-2.4.0.0.md | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 release-notes/opensearch-knn.release-notes-2.4.0.0.md diff --git a/release-notes/opensearch-knn.release-notes-2.4.0.0.md b/release-notes/opensearch-knn.release-notes-2.4.0.0.md new file mode 100644 index 000000000..9c0c45af5 --- /dev/null +++ b/release-notes/opensearch-knn.release-notes-2.4.0.0.md @@ -0,0 +1,33 @@ +## Version 2.4.0.0 Release Notes + +Compatible with OpenSearch 2.4.0 + +### Enhancements +* Merge efficient filtering from feature branch ([#588](https://github.com/opensearch-project/k-NN/pull/588)) +* add groupId to pluginzip publication ([#578](https://github.com/opensearch-project/k-NN/pull/578)) +* Added sample perf-test configs for faiss-ivf, faiss-ivfpq, lucene-hnsw ([#555](https://github.com/opensearch-project/k-NN/pull/555)) +* Adding OSB index specification json for lucene hnsw ([#552](https://github.com/opensearch-project/k-NN/pull/552)) +* Adding k-NN engine stat ([#523](https://github.com/opensearch-project/k-NN/pull/523)) + +### Infrastructure +* Fixed failing unit test ([#610](https://github.com/opensearch-project/k-NN/pull/610)) +* Disable Code Coverage for Windows and Mac Platforms ([#603](https://github.com/opensearch-project/k-NN/pull/603)) +* Update build script to publish to maven local ([#596](https://github.com/opensearch-project/k-NN/pull/596)) +* Add Windows Build.sh Related Changes in k-NN ([#595](https://github.com/opensearch-project/k-NN/pull/595)) +* Add mac platform to CI ([#590](https://github.com/opensearch-project/k-NN/pull/590)) +* Add windows support ([#583](https://github.com/opensearch-project/k-NN/pull/583)) + +### Documentation +* Replace Forum link in k-NN plugin README.md ([#540](https://github.com/opensearch-project/k-NN/pull/540)) +* Update dev guide with instructions for mac ([#518](https://github.com/opensearch-project/k-NN/pull/518)) + +### Bug Fixes +* Fix NPE on null script context ([#560](https://github.com/opensearch-project/k-NN/pull/560)) + +### Refactoring +* Refactor kNN codec related classes ([#582](https://github.com/opensearch-project/k-NN/pull/582)) +* Refactor unit tests for codec ([#562](https://github.com/opensearch-project/k-NN/pull/562)) + +### Maintenance +* Backport lucene changes ([#575](https://github.com/opensearch-project/k-NN/pull/575)) +* Increment version to 2.4.0-SNAPSHOT ([#545](https://github.com/opensearch-project/k-NN/pull/545)) \ No newline at end of file From 9d4cf3fbbc2fa591eec04b1e271b7de5625ed50e Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Thu, 10 Nov 2022 17:12:51 -0600 Subject: [PATCH 052/416] Add fix to fromXContent and toXContent in ModelGraveyard (#618) (#623) * Add fix to fromXContent and toXContent in ModelGraveyard Signed-off-by: Naveen Tatikonda * Address Review Comments Signed-off-by: Naveen Tatikonda Signed-off-by: Naveen Tatikonda (cherry picked from commit f92651bfcbcdf5b5f5443a5338ddc4ba15bb0638) Co-authored-by: Naveen Tatikonda --- .../knn/indices/ModelGraveyard.java | 33 +++++++++++++++++-- .../knn/indices/ModelGraveyardTests.java | 22 +++++++++++++ 2 files changed, 53 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/opensearch/knn/indices/ModelGraveyard.java b/src/main/java/org/opensearch/knn/indices/ModelGraveyard.java index ff7232bdf..21f13b6a8 100644 --- a/src/main/java/org/opensearch/knn/indices/ModelGraveyard.java +++ b/src/main/java/org/opensearch/knn/indices/ModelGraveyard.java @@ -21,7 +21,8 @@ import java.util.EnumSet; import java.util.HashSet; import java.util.Set; -import java.util.stream.Collectors; +import java.util.Iterator; + import com.google.common.collect.Sets; /** @@ -80,6 +81,13 @@ public void writeTo(StreamOutput out) throws IOException { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + Iterator model_ids = getModelIds().iterator(); + + builder.startArray("model_ids"); + while (model_ids.hasNext()) { + builder.value(model_ids.next()); + } + builder.endArray(); return builder; } @@ -143,7 +151,28 @@ public static NamedDiff readDiffFrom(StreamInput streamInput) throws IOException * @throws IOException */ public static ModelGraveyard fromXContent(XContentParser xContentParser) throws IOException { - return new ModelGraveyard(xContentParser.list().stream().map(Object::toString).collect(Collectors.toSet())); + ModelGraveyard modelGraveyard = new ModelGraveyard(); + + // If it is a fresh parser, move to the first token + if (xContentParser.currentToken() == null) { + xContentParser.nextToken(); + } + + // on a start object move to next token + if (xContentParser.currentToken() == XContentParser.Token.START_OBJECT) { + xContentParser.nextToken(); + } + + if (xContentParser.currentToken() != XContentParser.Token.FIELD_NAME) { + throw new IllegalArgumentException("expected field name but got a " + xContentParser.currentToken()); + } + + while (xContentParser.nextToken() != XContentParser.Token.END_OBJECT) { + if (xContentParser.currentToken() == XContentParser.Token.VALUE_STRING) { + modelGraveyard.add(xContentParser.text()); + } + } + return modelGraveyard; } /** diff --git a/src/test/java/org/opensearch/knn/indices/ModelGraveyardTests.java b/src/test/java/org/opensearch/knn/indices/ModelGraveyardTests.java index 28b3c7474..694d22ed2 100644 --- a/src/test/java/org/opensearch/knn/indices/ModelGraveyardTests.java +++ b/src/test/java/org/opensearch/knn/indices/ModelGraveyardTests.java @@ -6,6 +6,9 @@ package org.opensearch.knn.indices; import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.common.xcontent.ToXContent; +import org.opensearch.common.xcontent.XContentBuilder; +import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.test.OpenSearchTestCase; import java.io.IOException; @@ -57,6 +60,25 @@ public void testStreams() throws IOException { assertTrue(testModelGraveyardCopy.contains(testModelId)); } + public void testXContentBuilder() throws IOException { + Set modelIds = new HashSet<>(); + String testModelId1 = "test-model-id1"; + String testModelId2 = "test-model-id2"; + modelIds.add(testModelId1); + modelIds.add(testModelId2); + ModelGraveyard testModelGraveyard = new ModelGraveyard(modelIds); + + XContentBuilder xContentBuilder = XContentFactory.jsonBuilder(); + xContentBuilder.startObject(); + XContentBuilder builder = testModelGraveyard.toXContent(xContentBuilder, ToXContent.EMPTY_PARAMS); + builder.endObject(); + + ModelGraveyard testModelGraveyard2 = ModelGraveyard.fromXContent(createParser(builder)); + assertEquals(2, testModelGraveyard2.size()); + assertTrue(testModelGraveyard2.contains(testModelId1)); + assertTrue(testModelGraveyard2.contains(testModelId2)); + } + public void testDiffStreams() throws IOException { Set added = new HashSet<>(); Set removed = new HashSet<>(); From a5a4b6bc93bf8defac8b82a02100fa3b2ba873a2 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Mon, 14 Nov 2022 18:08:47 -0800 Subject: [PATCH 053/416] Adding benchmark workflow for queries with filters (#598) (#629) * Adding workflow for benchmarking queries with filters Signed-off-by: Martin Gaievski (cherry picked from commit 79ae6c24b989dd6c41fc17878ae85255080163fc) Co-authored-by: Martin Gaievski --- benchmarks/perf-tool/README.md | 76 ++++++ .../perf-tool/add-filters-to-dataset.py | 200 ++++++++++++++ .../dataset/data-with-attr-with-filters.hdf5 | Bin 0 -> 2404096 bytes .../perf-tool/dataset/data-with-attr.hdf5 | Bin 0 -> 306896 bytes .../perf-tool/okpt/io/config/parsers/util.py | 4 +- benchmarks/perf-tool/okpt/io/dataset.py | 10 +- .../perf-tool/okpt/test/steps/factory.py | 6 +- benchmarks/perf-tool/okpt/test/steps/steps.py | 250 +++++++++++++++--- .../filter-spec/filter-1-spec.json | 24 ++ .../filter-spec/filter-2-spec.json | 40 +++ .../filter-spec/filter-3-spec.json | 30 +++ .../filter-spec/filter-4-spec.json | 44 +++ .../filter-spec/filter-5-spec.json | 42 +++ .../lucene-sift-hnsw-filter/index-spec.json | 27 ++ .../lucene-sift-hnsw-filter/test.yml | 41 +++ 15 files changed, 752 insertions(+), 42 deletions(-) create mode 100644 benchmarks/perf-tool/add-filters-to-dataset.py create mode 100644 benchmarks/perf-tool/dataset/data-with-attr-with-filters.hdf5 create mode 100644 benchmarks/perf-tool/dataset/data-with-attr.hdf5 create mode 100644 benchmarks/perf-tool/sample-configs/filter-spec/filter-1-spec.json create mode 100644 benchmarks/perf-tool/sample-configs/filter-spec/filter-2-spec.json create mode 100644 benchmarks/perf-tool/sample-configs/filter-spec/filter-3-spec.json create mode 100644 benchmarks/perf-tool/sample-configs/filter-spec/filter-4-spec.json create mode 100644 benchmarks/perf-tool/sample-configs/filter-spec/filter-5-spec.json create mode 100644 benchmarks/perf-tool/sample-configs/lucene-sift-hnsw-filter/index-spec.json create mode 100644 benchmarks/perf-tool/sample-configs/lucene-sift-hnsw-filter/test.yml diff --git a/benchmarks/perf-tool/README.md b/benchmarks/perf-tool/README.md index eb4ac0dc1..d4404f551 100644 --- a/benchmarks/perf-tool/README.md +++ b/benchmarks/perf-tool/README.md @@ -229,6 +229,28 @@ Ingests a dataset of vectors into the cluster. | ----------- | ----------- | ----------- | | took | Total time to ingest the dataset into the index.| ms | +#### ingest_multi_field + +Ingests a dataset of multiple context types into the cluster. + +##### Parameters + +| Parameter Name | Description | Default | +| ----------- |-----------------------------------------------------------------------------------------------------------------------------------------------------------| ----------- | +| index_name | Name of index to ingest into | No default | +| field_name | Name of field to ingest into | No default | +| bulk_size | Documents per bulk request | 300 | +| dataset_path | Path to data-set | No default | +| doc_count | Number of documents to create from data-set | Size of the data-set | +| attributes_dataset_name | Name of dataset with additional attributes inside the main dataset | No default | +| attribute_spec | Definition of attributes, format is: [{ name: [name_val], type: [type_val]}] Order is important and must match order of attributes column in dataset file | No default | + +##### Metrics + +| Metric Name | Description | Unit | +| ----------- | ----------- | ----------- | +| took | Total time to ingest the dataset into the index.| ms | + #### query Runs a set of queries against an index. @@ -257,6 +279,60 @@ Runs a set of queries against an index. | recall@R | ratio of top R results from the ground truth neighbors that are in the K results returned by the plugin | float 0.0-1.0 | | recall@K | ratio of results returned that were ground truth nearest neighbors | float 0.0-1.0 | +#### query_with_filter + +Runs a set of queries with filter against an index. + +##### Parameters + +| Parameter Name | Description | Default | +| ----------- |-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------------------| +| k | Number of neighbors to return on search | 100 | +| r | r value in Recall@R | 1 | +| index_name | Name of index to search | No default | +| field_name | Name field to search | No default | +| calculate_recall | Whether to calculate recall values | False | +| dataset_format | Format the dataset is in. Currently hdf5 and bigann is supported. The hdf5 file must be organized in the same way that the ann-benchmarks organizes theirs. | 'hdf5' | +| dataset_path | Path to dataset | No default | +| neighbors_format | Format the neighbors dataset is in. Currently hdf5 and bigann is supported. The hdf5 file must be organized in the same way that the ann-benchmarks organizes theirs. | 'hdf5' | +| neighbors_path | Path to neighbors dataset | No default | +| neighbors_dataset | Name of filter dataset inside the neighbors dataset | No default | +| filter_spec | Path to filter specification | No default | +| filter_type | Type of filter format, we do support following types:
FILTER inner filter format for approximate k-NN search
SCRIPT score scripting with exact k-NN search and pre-filtering
BOOL_POST_FILTER Bool query with post-filtering | SCRIPT | +| score_script_similarity | Similarity function that has been used to index dataset. Used for SCRIPT filter type and ignored for others | l2 | +| query_count | Number of queries to create from data-set | Size of the data-set | + +##### Metrics + +| Metric Name | Description | Unit | +| ----------- | ----------- | ----------- | +| took | Took times returned per query aggregated as total, p50, p90 and p99 (when applicable) | ms | +| memory_kb | Native memory k-NN is using at the end of the query workload | KB | +| recall@R | ratio of top R results from the ground truth neighbors that are in the K results returned by the plugin | float 0.0-1.0 | +| recall@K | ratio of results returned that were ground truth nearest neighbors | float 0.0-1.0 | + +### Data sets + +This benchmark tool uses pre-generated data sets to run indexing and query workload. For some benchmark types existing dataset need to be +extended. Filtering is an example of use case where such dataset extension is needed. + +It's possible to use script provided with this repo to generate dataset and run benchmark for filtering queries. +You need to have existing dataset with vector data. This dataset will be used to generate additional attribute data and set of ground truth neighbours document ids. + +To generate dataset with attributes based on vectors only dataset use following command pattern: + +```commandline +python add-filters-to-dataset.py True False +``` + +To generate neighbours dataset for different filters based on dataset with attributes use following command pattern: + +```commandline +python add-filters-to-dataset.py False True +``` + +After that new dataset(s) can be referred from testcase definition in `ingest_extended` and `query_with_filter` steps. + ## Contributing ### Linting diff --git a/benchmarks/perf-tool/add-filters-to-dataset.py b/benchmarks/perf-tool/add-filters-to-dataset.py new file mode 100644 index 000000000..0624f7323 --- /dev/null +++ b/benchmarks/perf-tool/add-filters-to-dataset.py @@ -0,0 +1,200 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# The OpenSearch Contributors require contributions made to +# this file be licensed under the Apache-2.0 license or a +# compatible open source license. +""" +Script builds complex dataset with additional attributes from exiting dataset that has only vectors. +Additional attributes are predefined in the script: color, taste, age. Only HDF5 format of vector dataset is supported. + +Output dataset file will have additional dataset 'attributes' with multiple columns, each column corresponds to one attribute +from an attribute set, and value is generated at random, e.g.: + +0: green None 71 +1: green bitter 28 + +there is no explicit index reference in 'attributes' dataset, index of the row corresponds to a document id. +For instance, in example above two rows of fields mapped to documents with ids '0' and '1'. + +If 'generate_filters' flag is set script generates additional dataset of neighbours (ground truth) for each filter type. +Output is a new file with several datasets, each dataset corresponds to one filter. Datasets are named 'neighbour_filter_X' +where X is 1 based index of particular filter. +Each dataset has rows with array of integers, where integer corresponds to +a document id from original dataset with additional fields. Array ca have -1 values that are treated as null, this is because +subset of filtered documents is same of smaller than original set. + +For example, dataset file content may look like : + +neighbour_filter_1: [[ 2, 5, -1], + [ 3, 1, -1], + [ 2 5, 7]] +neighbour_filter_2: [[-1, -1, -1], + [ 5, 6, -1], + [ 4, 2, 1]] + +In this case we do have datasets for two filters, 3 query results for each. [2, 5, -1] indicates that for first query +if filter 1 is used most similar document is with id 2, next similar is 5, and the rest do not pass filter 1 criteria. + +Example of script usage: + + create new hdf5 file with attribute dataset + add-filters-to-dataset.py ~/dev/opensearch/k-NN/benchmarks/perf-tool/dataset/data.hdf5 ~/dev/opensearch/datasets/data-with-attr True False + + create new hdf5 file with filter datasets + add-filters-to-dataset.py ~/dev/opensearch/k-NN/benchmarks/perf-tool/dataset/data-with-attr.hdf5 ~/dev/opensearch/datasets/data-with-filters False True +""" + +import getopt +import os +import random +import sys + +import h5py + +from osb.extensions.data_set import HDF5DataSet + + +class _Dataset: + """Type of dataset container for data with additional attributes""" + DEFAULT_TYPE = HDF5DataSet.FORMAT_NAME + + def create_dataset(self, source_dataset_path, out_file_path, generate_attrs: bool, generate_filters: bool) -> None: + path_elements = os.path.split(os.path.abspath(source_dataset_path)) + data_set_dir = path_elements[0] + + # For HDF5, because multiple data sets can be grouped in the same file, + # we will build data sets in memory and not write to disk until + # _flush_data_sets_to_disk is called + # read existing dataset + data_hdf5 = os.path.join(os.path.dirname(os.path.realpath('/')), source_dataset_path) + + with h5py.File(data_hdf5, "r") as hf: + + if generate_attrs: + data_set_w_attr = self.create_dataset_file(out_file_path, self.DEFAULT_TYPE, data_set_dir) + + possible_colors = ['red', 'green', 'yellow', 'blue', None] + possible_tastes = ['sweet', 'salty', 'sour', 'bitter', None] + max_age = 100 + + for key in hf.keys(): + if key not in ['neighbors', 'test', 'train']: + continue + data_set_w_attr.create_dataset(key, data=hf[key][()]) + + attributes = [] + for i in range(len(hf['train'])): + attr = [random.choice(possible_colors), random.choice(possible_tastes), + random.randint(0, max_age + 1)] + attributes.append(attr) + + data_set_w_attr.create_dataset('attributes', (len(attributes), 3), 'S10', data=attributes) + + data_set_w_attr.flush() + data_set_w_attr.close() + + if generate_filters: + attributes = hf['attributes'][()] + expected_neighbors = hf['neighbors'][()] + + data_set_filters = self.create_dataset_file(out_file_path, self.DEFAULT_TYPE, data_set_dir) + + def filter1(attributes, vector_idx): + if attributes[vector_idx][0].decode() == 'red' and int(attributes[vector_idx][2].decode()) >= 20: + return True + else: + return False + + self.apply_filter(expected_neighbors, attributes, data_set_filters, 'neighbors_filter_1', filter1) + + # filter 2 - color = blue or None and taste = 'salty' + def filter2(attributes, vector_idx): + if (attributes[vector_idx][0].decode() == 'blue' or attributes[vector_idx][ + 0].decode() == 'None') and attributes[vector_idx][1].decode() == 'salty': + return True + else: + return False + + self.apply_filter(expected_neighbors, attributes, data_set_filters, 'neighbors_filter_2', filter2) + + # filter 3 - color and taste are not None and age is between 20 and 80 + def filter3(attributes, vector_idx): + if attributes[vector_idx][0].decode() != 'None' and attributes[vector_idx][ + 1].decode() != 'None' and 20 <= \ + int(attributes[vector_idx][2].decode()) <= 80: + return True + else: + return False + + self.apply_filter(expected_neighbors, attributes, data_set_filters, 'neighbors_filter_3', filter3) + + # filter 4 - color green or blue and taste is bitter and age is between (30, 60) + def filter4(attributes, vector_idx): + if (attributes[vector_idx][0].decode() == 'green' or attributes[vector_idx][0].decode() == 'blue') \ + and (attributes[vector_idx][1].decode() == 'bitter') \ + and 30 <= int(attributes[vector_idx][2].decode()) <= 60: + return True + else: + return False + + self.apply_filter(expected_neighbors, attributes, data_set_filters, 'neighbors_filter_4', filter4) + + # filter 5 color is (green or blue or yellow) or taste = sweet or age is between (30, 70) + def filter5(attributes, vector_idx): + if attributes[vector_idx][0].decode() == 'green' or attributes[vector_idx][0].decode() == 'blue' \ + or attributes[vector_idx][0].decode() == 'yellow' \ + or attributes[vector_idx][1].decode() == 'sweet' \ + or 30 <= int(attributes[vector_idx][2].decode()) <= 70: + return True + else: + return False + + self.apply_filter(expected_neighbors, attributes, data_set_filters, 'neighbors_filter_5', filter5) + + data_set_filters.flush() + data_set_filters.close() + + def apply_filter(self, expected_neighbors, attributes, data_set_w_filtering, filter_name, filter_func): + neighbors_filter = [] + filtered_count = 0 + for expected_neighbors_row in expected_neighbors: + neighbors_filter_row = [-1] * len(expected_neighbors_row) + idx = 0 + for vector_idx in expected_neighbors_row: + if filter_func(attributes, vector_idx): + neighbors_filter_row[idx] = vector_idx + idx += 1 + filtered_count += 1 + neighbors_filter.append(neighbors_filter_row) + overall_count = len(expected_neighbors) * len(expected_neighbors[0]) + perc = float(filtered_count / overall_count) * 100 + print('ground truth size for {} is {}, percentage {}'.format(filter_name, filtered_count, perc)) + data_set_w_filtering.create_dataset(filter_name, data=neighbors_filter) + return expected_neighbors + + def create_dataset_file(self, file_name, extension, data_set_dir) -> h5py.File: + data_set_file_name = "{}.{}".format(file_name, extension) + data_set_path = os.path.join(data_set_dir, data_set_file_name) + + data_set_w_filtering = h5py.File(data_set_path, 'a') + + return data_set_w_filtering + + +def main(argv): + opts, args = getopt.getopt(argv, "") + in_file_path = args[0] + out_file_path = args[1] + generate_attr = str2bool(args[2]) + generate_filters = str2bool(args[3]) + + worker = _Dataset() + worker.create_dataset(in_file_path, out_file_path, generate_attr, generate_filters) + + +def str2bool(v): + return v.lower() in ("yes", "true", "t", "1") + + +if __name__ == "__main__": + main(sys.argv[1:]) diff --git a/benchmarks/perf-tool/dataset/data-with-attr-with-filters.hdf5 b/benchmarks/perf-tool/dataset/data-with-attr-with-filters.hdf5 new file mode 100644 index 0000000000000000000000000000000000000000..01df75f835879aad0e4bac49b589a63276741997 GIT binary patch literal 2404096 zcmeF)U#O&cdeHHn*-c!6BkgiVMK4^O5Cj#(s2v#pl1v zs4NO`4vm9WYa8&YZ*&fr6$^V{E)-fYauoMw5Di`kiUtHh4)mRIX8QRYL9{kpmTSI_a+{qBC_YhSzd(VmWPJ5Rgy z+rRkYPv3G!+g%m-`qxhX=*K7gojL2jGTU4K@K^cMFUj_mTi^f5=l8|GKK_LM=1*Sd zl~-SU;{VjV>QBt)eZBpKk3R1o?|J)6A6;Ib?E9mi{QLVpy7lq>ufF=?Kwp|G}P7>+j5AAHDcmyY>CE{e{{7>3RG&&FzP?J)ir3 zXtv*+`~Ti-zdpzRm9xD!+kZRzpUmyQI{Tl_{eNS&zc5?hg|E&2AI$#$XdeH$x&7vB zfB$U1JlnI`@AlmO>fAog?bm1fjoJRtY+s$%`_Hrg~fh+3(NJ z?cX)~zc%;(p}GBwbNl_-@4w9L4`=)DXZx4t@o&%ef6o1XYqq~_j`thp@t>USubKO= zX1{+h_y5dn|MG1A`#k<^Za{wg z{r{V7D^P(7RGu6q@h`5%nfMl;;&uG);jEX7awty6qqrWI;%r=tv++Ux#?QFe_O8rURDlXqpaK=B zKm{sLfeKWh0u`u$0(^td@FPCN3%Cj=;XfRTzi<`)#FKagKjA~%fh+Mc?!bw7%Gvk} z2jgnyvUmoM;xW92vvDI{!k0MMhvO$4iT`jAF2l*V2#@1tT!#;GLEgo)IMV$&KiV#F4lYH{)x3ii2<@{=?Ha7f0en9EH>I zr1S9%-on3l9f#vKyo~2?5{|;F_!?K?NqmPN@*Ez-n>Zq`;yav)8*;P9a~}MUBl00G z#*H`;&*5MkiofwW-p9W<8Smmu+=+{EINrp?cqLclOMH|gp3Ql2JnqS<_#&s`jy#Ze z@j?!^y(_a7RiFYDs6YiOP=N|mpaK=BKm{tG059S}yohUEyUNV(a2?*nkGKy1;S}76 zfAK5s#dr7?2jf7Thj(!c&cWUI4d3EoJnLfIgIDn+zQsTI7T@4G+>VoRJr2fqPNowF z~ebWC4R?;RxjgQyo^(EGG4^T z_!@uXR6L4TaW7uSDft>d<9ytVhw(n%#>KeW>Q@|$bLCgQ3EEbm0u`u01u9U13RIv1 z6{tW3wp)PTa1JiSr?>>S;VqnriV=YGTz2u$Bco28vbNq%|aUt%s zIv8iI0C2O zHN0bWEH1yoFEjAil;qxD&VFE}V-0a4BxXe=g^Ixf!?Par}(m z@G)M->9`S>;!T{8GjTN@!>u?DU*a!ZksIk%pOF1f+>zkkg6{tW3 zDo}w6RG=ccR7rY@EdMp9*eJV6JEl*cn_c9M_i2? z@i0!svA7l&<6S(ATk#Se#I^VvH{m_Jj*syy&c%cH7x&?BoQkLMA^ykDI2f1Wd_0ER z@D^^zpST#W;x=56hjATF$&>gIpW;jA;rJ1!;!50zPjNoJ#pC!F$Kz|>;&*O0Bt~eRD;#=FjLR(n{Do}w6RG?|SaR z4R`?`;R8H~-*6a?!JYUPXW|x|gVEy8{a&c9=wjvzA?w+ znSB#fpaK=BKm{sLfeKWh0u`u01uC%J0$he~@DeV-Ww-**;3ZsvlkgjE!*@6p$Kp7= zjJt3Yo^?6Sz&$t!|KL{~g->uSuECo)8yDkRoQ_{z`^~E_7QmG_3s2*D9E98P5}w0% zcnlBXC>)SW9mZXF4Tmuw#eF!>!`Yw9aYBy6DLEv!;*`9Jk8wLr$oY5_U*mpUj2rS$ zzQ!dv9AA4f?=N@ai=2$7@lI~WF*)4%9EYEp-{OtDkaKcP4z}GZw3St$0u`u01u9U1 z3RIv16{tW3D!`360>9!!9EYE95{|%g-XA~UAiRpla4c@cFSrk1;yRp!U-62oaT+ee zLwFg7;%oecEAh9pSubzn7`%h;@FkwZy|^Cd<5XOSpYS0*$XPhsqd6Zw#)G(?xh;Og zzql30;#NG38*xAW!>{<@abEv0TMl(GeYhHr;(gqc({VO_#KCwQ@8eV4iRcAJcqQlLSiFpn^-WNL3RIv16{tW3Do}w6RGu&FxCx))Rjc=K5YEIo_z@@L zLcERN@E~r-0r?OQdN@ABYxo`a;zAsd%Uq5l@k7pYf9~gvJnMAs=TAI`L-9Xe#fNy^ zan{M9cp{JDihPduaVc)blei;dIfeKWh0u`u0 z1u9U13RIv18!o_E_y|YhH++M)a1EZsnO5)O2Rw>v@gvT_&G-i|;YGZP4{@*6Z@3#L z<58T3({U2+#eFyrkK#w1i;M9XUc$5Z9lzp7oQL1=KCZ^WI2G^We*BDkaU{;fWw;cl z<9D2q192rz#&P%+pW;WHi39OHj>(a@5O3t2yz*+?h*$9_9>=Bl)nQzTPw`Mb_iP^L zavY7HaWX#1J2@w}`e4?z;Y+loRiFYDs6YiOP=N|mpaK=BKm~SRfLrkle!@Mt4Ikk^ z+<;^79L~X)xCGzfQRcdM2cO|doQHRD3~s|6_z-{LP#lMc@iRWd;dl%m;bC6&6<^~!yo-9`>uL)!qe`LckmZ}#x1xOm*8T@aSqr{N16gs<=g9>fiJ56|FR$MFW9!%J50;!^yB zXK*xb!-4n2#xJ=g2jpKIk^6Bj-pI37SLB1djtla-lX-m($I7EJ3BTeI(ZKl zW6%d3-TiV$3?j!XXJn!j6+_|`nev@;!?creD>phT$Z!)wu{+s z!BVk z8Qv$HA;xxRBTX8%-#m%@6pW;27 zjmvNr{=~g_*l~P`Q*po7=YC$qiMSKD;x#;u=Wsk;$Ip1=ak}#^Uc~J<7LR&s*7L?Z zk7sc*F2#*bW`C~6=QtFH<9M8kWARXq$ESD{C**G2k3;e^uD1Ixf&x^a0u`u01u9U1 z3RIv16{tW3He7(0a3P+MR*3s;uZXe%kUmvaW?+Jg*X?N;beS> zi*PP(!B=<(58`edhSzbXt8o>M!^L3W{D~`u8xg77> z{TD$2Do}w6RG1MM z&chssm+>iX#lQF>kND5tub&pE&1TN}PaTUrGwP=N|mpaK=BKm{sL zfeKV$_XW5Kr|^58pZML+kGKdI;3AxffA9@{vw983;Sk(~Yw#Iv!Y#PV_i`mp!jt$3 zXX0Yqg`aRQ-ox9t8W-VOoQtFJ7jDMocotvcMLdx6aX=2kk9Zrm;$pmx)9^;V$B}p> zzvD;Tj$d*h-uGmDiJNgt&d1|8Ca2^=T#d)wo#XMwlW`;-#;N!vH+?YsaW^i;AGsNq zx}4Wp9gG8VM;^xQcK=0CfC^Ng0u`u01u9U13RIv16{x_53vd%o!!i7>=Pc%qxDF@a z51fJLa3X%eU3k{{I0M%)hs3Qo3TNOb{D^Px6kf(R_!zI@7QBY{a5Y}Ti#Qf%;W2#0 zJQg40fZU8f@gvT|tKJ-!;by#vd-0gZ)9Kl4`4A7{UA%^)aV_q~ulOR*;gNigCvr^Q z#)}TKZhpwYxFet9gZz#&@kP$ZBYEh>oF}j3T0GTU7SG~{oRzC_RW8e4Iow-w9{h5{ zmuO3?Km{sLfeKWh0u`u01u9U13hcfB-{2S=g+uWXUc-C%2!G&C+=1tCFRs8(xDR*W zW}J#!a2JlkMK}__;40jOKXEp0#g%v&e_I`flkhKY#{2jjXW~k{gvW3}UdC0p74PAC zJcUE?LjK0Rcp7)&Vw{Z6a2qbgjrbq;;)}fP;rPyj*>WeI#*?@jcjHPNkSB5@F3R~h z94F;UnCJdvPj0!+$sn_u*ok zjxTX7ZpT5m6>s8H9F5~~Fs^kR58-iqjJxqWF2!3o94F#oT#5tnF3!fAxDQw3ah!=m zalg~?8ji!Ocodi9V|e2KQS3RIv16{tW3Do}w6RGbxb}hZ51zv}xD#LC zCftY@aSZOjU$_~s;89$M7p)$~Z8#4%;&L2|Yj78i#NT)o&){0zj?eKW4#!)#9?#-3 z{EKUGA->1Wcos+ERosZja2-y>{kRj~;x*igdi{H*U$t4zo^P$F<&?;dIfeKWh z0u`u01u9U13RIv18!o_Q_!0NvPrQXwaU~wYS@;#V;bOdl3-K(j#BHvmglBOPesVcZ!i{(sPvUu;k5BO!&cg-8l}A;#%B@%W)&F!@+pZ`5cF@aVf6EwfH8d;%r=q>v1kV$>mPx zb+{dO<4-)0t8p`K_~CTnQCy7M@jSl8S2-8Y+VCaX(kf7a3RIv16{tW3Do}w6RGT6{tW3Do}w6RG^HaUc%Hi#Q=i<1svq({Vzs#Sb|S*W)<+iYFcC^|=)<;)#5SPo9nQ@Gkzw zhwjhgoQr#MM()J9I3};-lRS(wUd(Y1vt8Zm+1$?o`6B=1r(BWWaZb**;Y+loRiFYD zs6YiOP=N|mpaK=BKm~SRfDds79>kHj2shybJceKJ6>h|P_yCXLJ)DP|aUL$isdyMi z<7?c4PnqlCKYWa{aUCwlQMd@d;dz{ntMDo=!)bUKALDbpjeqewuJma9huHXg@)I1Ml2IJ}8d@jK4Ny*M1dGB3rWI2cd7n)BjZe2Rl{N{+|RI1{hq zdOVABo{v{?I3C5Tj`KK&O|azQ*jUO!o_$Lzu-LljDv6GvBABW*RJd3As9}dUsINHT{ z4~OG_T#!TYOK!;Bj^jbxk}L604#vs&5l`Zv{E=gEQtrjo_!bxCbDWF+aYnw%%eW)= z;&~gsL|a+~Do}w6RG7O1y#R@Tb?u zEjSoo;xgQY?{E~Z!q<2YSL11Xi`(!#4#shK54YniyovL1IF7Y?9Vg^Q9E{8HG2X=k zc^3aUj?-{P9%l}Vr}4;(*`JefG492kR!`zoJdYFdBM!yKxEUAZg?y9eaXdcBxA-dm z;->tPGjdP9#k+VWSLC$(cEguwORGQyDo}w6RG9gah##{>Qzz3y zQ#c?e<11W{OL0D~!v%Q|f8>UIi0|+(&cpM#9Jk_oyoi(WE1t=x_#)rrb-a-ap3nQn z^*9{2;+A}k_i;3C#=-a#|Kfm$aVoCH!+0CFy_ofE_!4bt6{tW3Do}w6RG%!7ca|U*jB{i_`EG ze#YOp2Cw63oQMZ;EY8NU_!{@(A^eM<@D>in0XYso;$D1(x9~-t!~ZxIpW#$e*|m zcjSQFjAwEzF2?aVCD-Dn=C*her{Z=Ob3R;-cX2^}$t$@S_vDbAk2~_b4PT-ytpXLO zKm{sLfeKWh0u`u01uC%n0z7~-aSOh~Q8*CC;8EO#M{o+B!xi`m594H9h(~ZOPQ;bC z85iSM{E2sPB~HP+I18WQMm&Yba2W2xbvPOq<2YQ619Byf!_D{_m*ZKhlW{MOXYPwr zaYdfRceoFqHSWUyxE){OZ2X5K@h=|6zqlG_;$b}IbUJZ7{>LXd9N*zu zT#)Z@DL%&+c^U`gVVCo|yzt4~=AitEdvZ#y#1}aw&*EiV^zj_`t?A0&xYdKXpW|^; zzR5i~Du?5i8@@zaS_LXlfeKWh0u`u01u9U13RGbC1F@a zx8hBlQ4#uN68OP#me3EZ+EY8Kv_#2nwc)R~1C_n`&P=N|mpaK=B zKm{sLfeKV$!v%N%$KV?rf;0J@&ms5>7vfC(hI4Qsj>03j6CYXqfn#tU?!ce;7N_77 zoQ><4E8-~Jg#U0TKE}Iv32)*5AjBGUc8bQaWmfaY}|=2aTsFD}Vhc^b#! zth@grC_n`&P=N|mpaK=BKm{sLfeKV$!v#2p-}^j=A8{6b#4~sbhvGFnhuiQmuEmEq z1yA5-oPnS42=hREhu3f!&c=^;3J2k1T!rIsD9*)McoLuCWL%Fg@gknb&3F}O;wyZR zL-8k`#^E>*U*R{rkl%4ZZp3?d8>ito{E0{ML(a&%?v5+*#)soyoRTA6&HWdPYjH8& z$-g)ucjBBpi%;=2e#tfYAqV7@obhDV!?(B`zvN`xjMr`W5^ZS}s6YiOP=N|mpaK=B zKm{sLf!!D2Kiq0{CcpPN50~N$yoD>7%i#r_h~un&!a4X0zv4xlgZJyjI^M?PI26z0MEs1u@DP5)-8c|e<31dP)0~a-@F*_C!#Egseu@IOw- zm$)0(;XwR|Q*kmr$gy}IkK>IT?7_S~cj9Eciqml>PQ)d79WQ)sj>jMQ((7|S&*E1+ zkpJ;Cj>#YS9_QkLeC~p~?f#pf02Qb}1u9U13RIv16{tW3Do}y#7T`d9$XpIr;WAu` zS8)n%!K*k6r{X-kg}-nrZpI%t6EEXpT<18RahR=nE z;x+t+-|;o>#k=?k|Kn;risSGw4#dfL9B<-1JcjS_GY-YUxFK)kb-a(`@ixxJ5&0j7 zyY{T<&XsrXqO<7AwQgYmJ|tN0oh+wK+G z$|_KS3RIv16{tW3Do}w6RGEn91Q+8le1&Im zFCM{V_!Q^jZ02{k760K+T!^P|BR;})_!f`jHk^i^@glCp0r?FV<0Kr4H}M@V#RIwB z>RkMeBXS%r!>hOrcj9W-jy3+ok$4-=<83^NJ8>p{#aV@ULf4CNh;XPc9XYnY`$cy+Hk2L?q zzploaI3@Stjhu=n@<#5(nK&W;;#fS1AMq~k#;dsKan{4DcqpghVZ4!Bt**r<`4^w# zX1r~?S7Ng`ALQaYpXNf4CCY;(mOM5AwOQaX%i!6?qXa z0IWqYl%XqjEJK$X|IGm*bNBt8an|RG(v)`@HQ^VnRpau+vKG#ff+zhdiCv=V`p|IPVL$PJx7tYAH_zwr;Ph63^UAyG0>vXm}lIL+h zPR8dr6c6G_9EfLeG#1KF)bQ=fxR09QWdTJdk&C zxxNW1P=N|mpaK=BKm{sLfeKWh0u|V90q(V%m6sO`{ypI#| z9S+3NI3c&f0B64&Ba-0Ez+ ziA(ZP{=}X56>mJu^LQJNzRcHD*!y*B5|m-r6<<0l-7e{nbd!ngPm2jXn}hWGG2&cm@d94F+H z9FXsELhi;7xg$s8T#x2FITw9?4Tr z=KizUkH7IxKFbaHYTpDEs6YiOP=N|mpaK=BKm{sLfeLK505{+Zyou{@iPP}_^H#io zS8x$t!F%`*_u))@hA;63j>eC;3P<4){D$lB6+Xo^j^i0TiGy(;-otZv318tQJdO8o zJubypcpl&3JzRxTaYC-gwKxmM<8YjaBl1HI#_c|w^W{){h0Pfe2U}nKie<2 zujF`Kl#lU4p2xMW=Jh!hms|agt8vDQIWLacH$ep|P=N|mpaK=BKm{sLfeKWh0^2RX zad;44TD^vw@GQQ;clZ(i;VpcLi*X^2a2S8!R{VvZaSZ;&$9NhK;xrtFV{tXk!fUu4 zH{wXs*e5@F4!gb9fA2;Wb>31M)LI$HRCbH{w8ih|lpf4#~CnA_wC& z{E4&it`Fz^;7Qz&Bc9IrayuTzhj`@49B1_;e#oVG8DHd6&*pX3IX>>iC%GNp;&hyh zQ}H+M#npJ*cCXM@R)GpspaK=BKm{sLfeKWh0u`tLpW#y6il1;C4#Y=z2k+rZtK)DY z4#TIo57*%doQOx6cj8Wbig)lZ-o?}S3Mb-d+=+wmA8x~6_zK_SO1zC9@gY9Oi5`yM z@HL*tXLuBc<49cZZ2X1`9_M&`hL7O$KrMzj|cWmP=N|mpaK=BKm{sL zfeKWh0u`vhb_;M7?!ke00dL|7e1!+`4)akwfJg8pPQqch4Ts@YJcB232yVqwI0@(B z9Q=y6a1)Ng+xQk=;v!syqi`DD#XYzP-{VhwhhG9 z4bNM>ilgx*esdTfI+-oUbv&2!G>d9F9A2BksfVI2E7ahg^`8UClbV5Le`P9EPv)MJ~r{cn_cCXdI3saWTHg zw>S?Md^qcRG+Q3U3lDQYzvG1*k#C(Yb7nm3Y;iko#RYjN7vzon?*H(zIUXlFpW9rl zZ-NR`paK=BKm{sLfeKWh0u`u01-4s&&u|D1!U1>*m%4V1S6=`CZ{b7yhLdm)KE)Y0 zlDQ(j!?QRN$KXCZi<59Be#J%j2FKt<{DKeh5A#EOjSq1!zQjE^3fJQu-IvdeKI zp2neg9RK8}JdiK)Gj4h{>*t|7?2UPyx8`xa$nP%ZabCyoxZQTI&{kG~3RIv16{tW3 zDo}w6RGg`5uf6Coa}rYiEDB@p2wYd7B}N!9FAY{ zDt^b)I2oViU%ZUVajm`yDo}w6RGY z^Md7$KgdhjPr3&PRPHw9cSZJeCh5Smxu8xp2eeh7`NkFypMZvE)KWdE3}nW zpaK=BKm{sLfeKWh0u`u01uDQRI2E_yK^%(T@F?EGvv>qw;ZuBx3$6aaceoB;;U)Zv zd+-^K#a;LZx8WuniNo+5{=`o>9OvOMoa}7;gq!dfZo`AP4@coHoQY3y6Q0NGI16Xv zEqsjM@UPVyxz@?}5=Y`pe37$pOMb|M_!+MI3{t z@HURbU3e4^;w)T>zwtAE!`ZIJQ}_$t<1D;}5Ah|A#(8)Wm*IUg?62CH^cYmJ8A2}Hp;(EN3w{bR(#Je~bALEN$lIL+XF2&!tAh+a< zoQwlX>XcjQN0 zkYk;Uqj4(k#Kkx$XXB^mvux@*`JR+p4)vBRG2Dr{bjCl$UZ$UdcJR zByZ(l{F7sGO`gYJKbZH6hjC4=y4@?Zl~teu6{tW3Do}w6RGLZ zfAA?D!AE!y-{C-9g3oX;p2e*=1-IZIyon$2E^fqgcmyxvVBCvCaUPDr>3AH+;$A$D zPjL?J#p5^-@8dSSh1+nc%W)ts#Jl(#*Wx>zinnkge#CD$62Iem{ESoaM)O%5kYDjH zPQ<-V#(((agSpL}xEBB8UA&G%@iE@Sp}5@L>BIXj<~FzFqFnNgdEDF=ui~9tkvsB0 zzQs%XCa6FKDo}w6RGKV7!Hwa32nG z9RJ`ee25qE8xF=bxCcMsO?-==a5KKaGq?(e<1-wMlW;RG#fLZ>$KgjDi=Xfy4#ZxwEzZYhcnm+}a$JT7aXrq)=XfLM;+0&CpYbOC#>2QC2jpv9j(c%YPRJK| z5l`b~JdJm8F0RGL4)eL;lDv>-axFf^4fz!x7@BRMg0^EY%a1Tzwp*RkQ;#T~L_wXYw!(I3e|Kd74hEMS{Uc^~A7U$w~ z9E9_5IxfR=cpX>bD13*na3!9^hd2oz;V0Z~bw8fOp*R}v;XxdVFYz<($GdnD7vg5z zhr@9n&c+kDB_HE`ypTt7L(atuxz_5BJdh)D!w(ma;&)dcPR76Z6Bpx;Jd9`Y zud8`IF3A!5Ca6FKDo}w6RGw&c$E2 z5KrP_JcP$^CCZa4gQnr?>`pS)Gjo@jFh$$M_GI;vSrcXYm?7!s&Ps zKjTx}h@Wx1%ki4SI19hyXIzY1@fTjjo4DP{9EU5JTjDyLhrd0X{rM2b<8V{tT|$yd1*U*vJTi>vZAUdFMwA{X24723)wP=N|mpaK=B zKm{sLfeKWh0u|sZ9EgkX9$v(EcoHAsUwncO@g~m2gSZY~;yql91Mv~Q!qqq%H{o2o zgk$jz4#%B%8Q0(=e2!OfFs{VcI1Yc~cf5qZaX)Uyjkp^(<5YZw7jYb3#J#ut|ZMlQy+coC1|MI4I%aW_uIGr1jC<4_!rE1BctRs4{<@iqR$ z_c$ES;*1>iVtkOJ@vOcHDo}w6RG4A953WEoQ&h~IS$5;co+BKm3)nB@g+{kp*SAr z;zV4IYjQTO$JO{8m*ZnxidS+iF2%!mD(~WnoQr?)Kt9FqxD{{XY1_R*TUiAvP=N|m zpaK=BKm{sLfeKWh0vw4?a19>7F?ayC;Vb-rCvhB3u(}aP;81*v7jYkRKD>r=@i898 zA$Sev;A9--iMgBk`*FO7FL4>3#P#?JPvd;thDUK3^G6(v2XYu5$Y=N)=ix$p>NxA= za-5D+@fwcB`?wXC;&pt8pK(b3#IHCScjJH@j8F1R&c*{d79ZkBJd6u+I}XUTI2HHe zi;t!Uzv6G_^Zs%^PRAXu=6M|U+Fj>)eG^om0u`u01u9U13RIv16{tW3DzM!GJb+_x zAHVbY7DwS!+=6596;8r|cnvS&Kir5DaV-AAeYgXM;TwF2=Wq)y#xpn*uP}ebjV{M8 z_zgeeO?-~SaUX8NN%$7u<5~QNw{aUz$Dz0v_v1*MkneFS^Ge+9IOoBa_!0NxIQ)v2 z@kH*&E%_0TyqM#2Ee>@uF2p}M7q{YC{E$O&(r0sAPQ@Yl9>3&-e2nk$Nsh_M_#J=b zW8906^1tm~p{=X}6{tW3Do}w6RGWq;8Gqwpyo$eZJ6_4X`X;DA1u9U13RIv16{tW3 zDo}w6RA9RWc*;62#i3T;;z`_wOK~qg#JRW=H{l(8jEC?mPR5t`6DQ+UyodjA9Uj6x zcnsg*CVY)A@f{w-b9ma-I1LZtZ`_W%a5qlF#dsT+;bi=U6LFPC^LiYMqwznk#i2M2 z7rPwK;!Zq@Gw~$O#N&7lN8@ljkXvygp2w;99na%-{Esi5%=vLEj>s9g8b{=De2iyt zJFdm`I2G6AS=@_f@xJX|p{=X}6{tW3Do}w6RGG z!K1hV7vekSrZ@$6;1FDjPjHj7aTxPqT!**u9FD_D_!DR1F`SKeag@t>9bUvecov7_ zb9|3$aVkE>r}zlJ;(Z*62k|T3!*RF_r+YH%oOJdbI2EVkpB#((@hbktnYbpu;$u9NOC6^RPvdf&lh<84=B$hR^-WNL z3RIv16{tW3Do}w6RGce#5(X z6CdGM{DV7i6+Xhpcn9y{Nt}uw9mX-Z2S?*yoQ9il8t%rMI1NwZe4L9H@h*#xiC*fcGg?sTM-oa^j6#wE}oP`TbT{sd);cI+~!|*YF#aH+t-{U*HjDK-1p2wZI5oh8;{D@O= zsl(!2{EyRdB0k8YI1>NkMVyf9@g?5IA-UjT-VfgRY;K>;md9}@?!-g68t=NA^WlkH zic|3`zQ_Z)THgc}s6YiOP=N|mpaK=BKm{sLfeLK506*antJiQIKEM|^7SG@Xe1%8x zF^<5Mj^hoSf=}=zzQs+r0~g^)Jce^{7mmVncm>blSzL^>aV9Ruvp5VF;v_tVCviRw z$9eb>XX7v2i05!B-onLBrWa@9J^b(SxDVIkYCMW>@j!0G`}iYg;%mH(!|^%J#($p7 z>zn)Hn3r=u@8plX?l_OXIp@dAxFSF0q1=#L@vOrn}SMfF;wmJ@n;aFUVhjA7@ z#8LPX&$>Ha!lyVCuj6w3hF9?;&c%uN8~@`#Jc+MyFuuoUxEF`wK75S}@-NQEo_Zq;$@tQ zTX8X7#kDS`!*;LGR#t%uRG>&zv6aWhkJ1-Zo`K-9G~Gh zyo(3%C!WLKxDtosL41YRa30>s(YPWPtyb z5Le@A{D;SJC2qxA-W)IEbzF+Ga2tNd#rO$_;zk^X)A2km#a}oZzvEB5kl*kip2qDs zBv<2ToQt#ZLk`HhxFzS}OPrBA@jE`p<#-o2;zJygZ}B(&#ND_VUwb_BQ9O&U@iRWT zdf#E*KTgH3_!uYUVB5VyTUiAvP=N|mpaK=BKm{sLfeKWh0$hjl@Frfvm$(MM;ud^? zhj1bO#B2B!x8Yx$jGyryuEZhu4X@x!9ECISr}J?SUc=vb6K~@`oQFU07!Ji#_!WoY zUR>@tZo=ER3U}i*rFukGt?B{>8iQ&+~W`x8Yixh%a&~-p6xzA`j$# zeC=w^lQ(i9{>k^a9w+0WT#R4wIR3>a`5k}bSe%Q~@iGp`xi}g3<6c~gEA~xLfeKWh z0u`u01u9U13RIv16{x^=3-A+;z>)X@FW>-v@ADd-wYmi+AxU-oR@(3g6)*JcZBjAg;v4c$7IR&cVly;~m_K%kUS@#+~@c>S&ya zhw&%A#IraFr{P1qjOTDN{>FXy5f9@n{Ebg>E>81g{Da4EjT zdCZmZDjvqC_!{TqXuOJd@;L6q4_}-2ix%q9mnH>yo*EeHcrYbITxSeuAGl= zT};=$2`W&53RIv16{tW3Do}w6RG28}Hy89Ej)eFmA%pxC?*cC|rm0a2npnwKxek z<669r8*v*R!|!+<|Km|ykQbhh$8bB&$d`B}XX9|(j7#w^KE>&{4=3bj{OgT*eQwDE zxgU??bNrHjayUNqc#g-}c-&zejC*l2zIi#%<9$4f7xFpYx7{nWl~teu6{tW3Do}w6 zRG!dn;9DGpYw-!*!h1Lf zkK;ldieK?G{>8_53eVzR9E{6wIex{VI0--FTpWsLaTV^w4S68v<4c@{-*G-}#HqL< z$KpmDh=cJvj>dJk5Z~fVJdXGAI*!Gy_!$4sK(5`W`*9E!*B zAAZM;xEsIXavYLJ@g!cy=XerF#U;(9@imUhy*StEeB5ojS7jCb%5p2Nww6#w8`e1#{OL*qOAgro5@9=7@rzv4B#hbwU^{>5{6 z8pq*y*Df=j#rgOnH{@uX&HNVE<4e5i^*J8r;#1s>`<=~P6DQ+}T$3Yl%9A-R2jieT zj0^HsUdmB96=&mz9E~6HJO0QC`zEMB1u9U13RIv16{tW3Do}w6RA9RWcmn_7Dm;ew za4X)#tM~%n;s$(#XK@(Lz`uA22jdj{<#HT^UvUv0WRA)_5BK0&yoM)nGmgY%I0{GO zL41TS@fnW7_jnBd<6)eKGw~TN#Oe4B$Kf;PinyM6DE`LN9*qC+HeSai`4r#bT>R>4 zp3mhtBM0PXylQo{C-Xk>EpEjR`4&IonOun@@+LlcF~{Y3e2=g3Cr-*A@6PkMBS++T z{E9bj_X=%g6{tW3Do}w6RG<-@_@hGmvoy3l6}cs!<3(JLkMT0z$;r;icfMa9=3WD_v4lPkDqco{`uj&&;0Ld zZgWY#dNJ$bf&7(6a#B9K-7B<}RiFYDs6YiOP=N|mpaK=BKm{tmPxuj^;whYi53LTx zwaoKyBCf;1xC4jcM0|)>aWr1TuXqrb;zay~PjC}1!H+l>f8r_JjpOk#ZpV{293SCx zyoM8TqmywJKEvs_3qRviT#3u^70$)IxDq#79m~8G2jW&-kjrsLuEyW^7%$>hJc$=^ zEWX5_xF+A@ejJR;@g=@!eu_i!ME=F+_|y6L6tCoIe2iQ1RKCXdcpbm&o1g*}s6YiO zP=N|mpaK=BKm{sLf$bLXJD&sa91g*o_zGv@8vKG!aUPDuPk0D_;t(8#x9}q##c_BA zFXAISh7<7#uEt9qjVJLg4#Pz*$3q^>e%yoG@S3yPkN5EPCp=i(0hgE#Rap2HQm24~?sT!TyTEB?Y~coEOwDZGZ$ zJRDcyaQug_@hN`CO?V4u<6WGEmvJ)Q#BKNw@8f?QinE=L$8a3J!xwoU-{UkKiL>!9 z9>$w^4(H*2yo$?lO@7Jy_#~I&cHEJF@v!qbA3n!9IUC>PgxrnOaZV2S!MuO`>11wm zEk4RKujX+s$qV^m-vkw?Km{sLfeKWh0u`u01u9U13T(FkN8n9dhvV=aZo>~a5SQRd zJb+hmARfh^coJ9P538SCj(czvUc@1I2`AwoyyGx#!8?v~n`dwre#1X_7^mYPJdPJ} zC|<<*xDtQkO}vV;albcbU7U#ja67KWcQ_yq;(k1cV{sr3$8GoyPvcU2?AjrxKVRc> z+>&2$GQN2_&*OT$j018~p2neg-{X1yThoPOaX{Y5CwUsTG|BxwRj>A^!OgfD58`Cp ziVty=)ou6_x8ri$ip%gHPQ?2-3jg6~yoaxFHr~UDI3FkEihPUi%y zNe;;Wc;Lf%-s*-tiJS2t-pA*7C{N^We2GtTHXg>K_!TGQb3E~C*2iJb#<}<(7v+?^ zj?eP0?Ovg+tO6CNKm{sLfeKWh0u`u01u9U1KQ_PjZ_oDcjtBhAY`-@5|HZle&fNZ~ zx&5uV{m5-{@1zxch2@JbDV!OkN>{8|BJKzA9Mc?&+Xqb`~TG3e(Lyh z`_A0{o3s4`vwd$K|IxYqgL9mB=l(u{6{tW3Do}w6RGfDf(i z!+ZD>KjK@wi*xZE-o>{#702O0yoe`pFP_G!I2JeKUEGPEt^UQKI1xAFaD0hVaWF2% z-#8Zs<8b_n+wm)o_VeRYJd0a#EDrYR?@o_apaK=BKm{sLfeKWh0u`u01u9U13h)-L z#DRDb@8Mh=i%apW=gZs?-{Czxiudp@9>vQz5>Mk`oQi|-A-=@Tc+uPAM|_MM@i<<_ z$M_Rh;%i)vtMM?-#qW3*C*x;)jJI(xp4BI?0u`u01u9U13RIv16{tW3Do}w6RN&JK za3@~Ft9Ta2;zj(58*wL2#f$hA592`Qp7;_!<5C=n197f<<751XQ}HOywYnOo;$ggr zD{(2_#MihMcjIi_ia&8R?#17D8aLx!oQr3D`n%Jk6{tW3Do}w6RG;#vHMQ*jx-!?*YqXW~%&iIZ_9KE|!M7?&B6iI1%=#*H`=pW|bz zXK^fZR=kXV-5CetaJ-9eaV_&!T#Hw6Hm>#A-lg`cKm{sLfeKWh0u`u01u9U13RIv1 z`z^plsSsaT?aUt%+!#LKxIW8CDU*^2H5trjf z+>F2RE*{05I1>lsa=eMNt?tGD_!Q6Las14@8IRlVC$BTFKm{sLfeKWh0u`u01u9U1 z3RIv1pRE9Q;yhf5OPNFBN*swt@h_glxi}R^;z3+$^)imctN0S%;!jVTw*&+?4q;?saGJu%~mIFYzdj#>sdZuj6t&kI!*1zQxPlo&SBr{eJQ~^9od;0u`u0 z1u9U13RIv16{tW3D)89~@EzX5oA?utG4I5gco*N{L41fO@hZN>g*X-O;zoRj_wY21 z#f!KXzv5YZiGT4fzQo<`j8pMB&cx@;qj5I=#rt?3SL0Fq?7QPr9FTu;FYfo*-lg`c zKm{sLfeKWh0u`u01u9U13RIv1`z^q2coh%gMSO?%@T_~|KfHz?@h$Gd!*~@h<2gKu zGw~;`#i_UvU*ck%i7)Z4r?Wne#@qN9&*E?Vj!SVe9>>Et9*5#=Jc?KGtkvy!7vI|N zC$BTFKm{sLfeKWh0u`u01u9U13RIv1pREAD;!b>s`*19-#i6(jFXB=>iidG1zQl*P zlzA-9#ih6pALC^lilgx^-gIZ&iZ5{`KKAy^MR7SEwK^7G;$XasV{tHU$E&y;$Kq}r zjJI*D&-O00R|P6ifeKWh0u`u01u9U13RIv171(b9F2t{R6Sv}3yoU$z9v;QJcnwE08{gwvT*~|w zx8hx#ig)q0J99q!{p5A#6{tW3Do}w6RGE!?+h$<88c%$8jg_#?QFa>SmmYdvPl6#nCt) zFXLF8j-T-?UdF39+0*8)_IuHF<`t+w1u9U13RIv16{tW3Do}w6yi@_s#C!M(&*C?H zhdc2aKE!pn3~%CByom2`Ebhg3co-MrWPFNOaW}5Uy?7UQ<5j$gPjR?=<4Ampr*SY2 z#lQF&Z{t(noxhLx&TM&|c`F{q_xRgOy;<#CfeKWh0u`u01u9U13RIv16{tW3c36P7 z@EqR6iFgZ-;Z9tL&u}BI!*h5L_u)`HhaYhw?!>!z84u%6T#ch~Ebhj$co^^EM!b%{ zaWziGuecol;%V1TH(qsT9E=BYt<~q=p8emQ?G8V3opJ>#P=N|mpaK=BKm{sLfeKWh z0u^|v0(^!KaTR{WU$_?k;X=HK1FfFKb2!xMK=;OdxED9#LmZ1g@iTtKop=#P<7T{w zx4koN#L2iFPvdL+jdO7=j>p^h9iQV^oXs2=2jptz&0gxwYUc`6paK=BKm{sLfeKWh z0u`u01uC$^0(^)gaUEX8o45|I;z>M+*E}DO;z!(w>+m4XWNwOwaWgK&&$tyQ;$o`@ z@ii{RulU-v*NtOwC396AjKgs~zQxNp*3-q`I2T{zU|f!Sz1#fN4zIROxdIiaKm{sL zfeKWh0u`u01u9U1n-t(p?~SMMBW}Z&cnpW)I~<2+@fps=lXw`1;#K^MQ*k8TbZ>l! zTk#@(#ou@kALCM-?Ap_o`755qrMQ>*DDK3;cpT5-dYp>8nY-d^T#WN^uA97PJ*NT{ zs6YiOP=N|mpaK=BKm{sLfeP%Y06*ebJccKk$Kpghi9@aK#j|)4Z{k-xjQ?;W{=>WY z83*G>oQwnUB0j~P_z-vFQoM?D@hHy4o%kB3Gl#~b_!zI_ZnwwBc-5WpHx9<#_}rd8 zYn^KaDo}w6RG8C)6-VQ2yojsuDDK9wI2pg=OShN#DNe?(xYzS>Htxr>xEz1u zSX_^>3M*Wz5OYw;oO#D_Q)pW;Ovi8paK-p1EfALCVg?D=>S*WyUriQ92Ce#YUr7-!>G z9F2?dDt^TQxfk!+(`T)7tw04TP=N|mpaK=BKm{sLfeKWh0yinZowya};YNIGbu1pm zgSZvn;!qrnXYr=n^Y`?45SQXbcgC4G6>s8fT#PsIAs)upxDqepS*v&PG5*A-I1`8B zSbU1RaVef>{)}&}PR7S>@}Bja3RIv16{tW3Do}w6RGh<90lXm+`On#_6~k=i*s=`mA-X6{tW3Do}w6RGR()p)4eke#jkkZ>Rp`cyR$#<<5%2@i{0cs>p2yuKm{sLfeKWh0u`u01u9U13RGZE z1-KFa;ZVGZGw~vx#JTtsN8(`o=g!PE@veJw+nf|X<5fI~mvJ6W#+f(~PvcBniW9Al z#jiLT593hWj+gN`{>82MoH;GN#m`o^;#qu-gYD_F*11-o0u`u01u9U13RIv16{tW3 zDo}x&6yQkQhFftSF2ixS4v*qM{D?DgBwodtcohfZUYv{na4N3E#dsO#dOjY;gUmUt z9>>%85ijFU=CrsMuX}HqqvBosjPLO*?#8+J8%N_{JnJU!SdW6vyE;oQg|vB<{n-I1(@8Qe2AznQP)~oQz-b zt$X8IT#l>pI-X|Uj5~2MKF0BQ6wl&d9Ez9mGrq;&cpv}bSbO@cb*>etKm{sLfeKWh z0u`u01u9U13RK`G1^5gv;z!(vYjGle#g8}<_jx*Q#Iv~9ow=W9aW5Xkv3MD0;!%8v z1DTKFOk9YI@iI=u$9Nsj;%wZGukkD%$DueGr{Y)~jIZ%4?#8Ql7Z1D1d)9L*P=N|m zpaK=BKm{sLfeKWh0u`vho(ga$-o$Nq71!cUJc@7eCCg?JVp z<3{|7e{nIM#jT!>m+>RM#Fe-c*Wy-Oi)V2rPR6_V8Xx0QJdjuMG`_~cIM<#&Yn^Ka zDo}w6RGSq+wiXE%iq)EGQ5l5@Gw5atvD9n z;%6L;Yw@Mkr#KWx;$HlSTX8zx#o@RVU*d3_if?f`ZpOj573bn*9FEuVH?GIWZt|Y> zoC;K+0u`u01u9U13RIv16{tW3DzK*l=BW4&KjJ<7iWBi6K4s2{fAJ-*!)-Vb|KeI) zieK>_j>N|}6er?KoQtP%DDL&n{P(Z(C+@_TI33sGZ@i61aV+k|vAEv7aXc=^&-fg# z;&&WtPoK5UwE`8WKm{sLfeKWh0u`u01u9U13f!as_u)i5h#zqxuEeLf6c^$*e205+ zF>b@HxEUwoO#F!-@iKnKnRpWS;%7XGi}52)#oag^Z{k+mjh8(gm*Q32iI4F#Ud88l z6}RJH+>E2$1>s%{PfeKWh0u`u01u9U13RIv16{x^X3UDA!#ILvu*WoyP zhI{cNb5)#*5AiMz#&tLpui{P|i;rp2yuKm{sLfeKWh0u`u01u9U13RGZE1$Yd{;yJvC zLvb1&#g%v#&*4;@iBsJh*WpCGhlBAiPQ9-q z8mHrU{EKgKG=9a;_}uDeoNP~@wa&Ey6{tW3Do}w6RGW3W1PyI6^G+e+>OU^s(0t_ z1#%{y#?3evC;MJr#kY7DU*lC=j<4P1J?l9Ys6YiOP=N|mpaK=BKm{sLfeKV$PX){i zaVgHiuXq;U;y`?fH*p~D!*}=(C*xpTix=@H9>uG86DQ+YJc*axTm0(U-NvQNm91{Z z*|^l|Pn?OH@h~pM?RXj2<5awkmvKB^wx`cp=URaZRG;Y3`CNAW4n#ml%5N8)L`izo3cF2uih8BgL> z{EV-yzQn>s%{PfeKWh0u`u01u9U1 z3RIv16{x^X3UD7D#br1X$Kp|(io@_C&ckhZ6}RCyJjQVajcuX zXFaC^6{tW3Do}w6RGV9Jk|C?=Ev# zJZw*&wa&Ey6{tW3Do}w6RGFKx=|KfK%i z#=ZCy@8Vh9jfd^&v(~v*paK=BKm{sLfeKWh0u`u01u9U1n-t(b9E5gKXDu$#GSay z>Nq@!L-8XH#&@_5_u^xGibwGyUdFR{5r^Vre2RzhF>c1iR*&Lt=C?Qx7I)%XoQH#Pq<7NDbyYa5ov-lV9<7C{9m-%~?d;H9G&K0OY1u9U1 z3RIv16{tW3Do}w6RN!R_@FD)gg?JT@;!_-m`|vIf#*KIr|KdV?iH~ue)rI)a^JNZ; z7x5&H#=AHecj8byjVEy?j&^5Whb!?d-o)v68yDkY+>O(5K7Pjs&24eFmwB(+w*nQY zKm{sLfeKWh0u`u01u9U13hc3fIU>%qIuyU+S{#cjajw;q_!HmZM&_q@4tL^6e2PQy zEiS~lxEUAYO5BP!@i|_^={OZ<<4oL%M{zG+$GP|y=i*s>Z*@8j#;>^A9zS!Pa|J3; zfeKWh0u`u01u9U13RIv16?mBfe1`L^?!sGm6360Dyyo7_FL4=O!G8zKfIbDvre6xD(GaU&ZZs98cqNeCzr66_?^< zT#kovwL7!_P2RJfQ-KOppaK=BKm{sLfeKWh0u`u01@=^c2XQR!#F6+DXW~BQl(-h3 z;!>Q7gYhPQ#Je~ZhvHORjE8X~PQ{T{cjH!EiA(V)?!>LQ6sO}{{E18Pr*~%VisSJ$ zF2&Ed6%XTDoG!oG(^b~FR-ghEs6YiOP=N|mpaK=BKm{uBc?xhN9>s^a6VKvUT#Eyl z*WyMzi}!FZPQ{D(4iDp1{EByRACAPWcosL~PODdOD2~L}xE;6RU>t9CDjvqYI24cL zR@{!eaVoyW?YP(HdGC5|1u9U13RIv16{tW3Do}w6RGc$F7ve^@XYPpG z@Tq(A_uzOBcj898hkJ1wuEl+L4_D${oQ$7wGVaE$_!NgSZ^gN|(du-(jeBt_uEn3Y z7hmI4{Eo-*tf$lE?dithI2O0t!)L5>tUv`SP=N|mpaK=BKm{sLfeKWh0-vVfZi7eC`yoQ{ieEc0M|i{EiA4#=;#*ynlgdTs?OP=N|mpaK=B zKm{sLfeKWh0u}gQ?EQbp=k=ZM_g|aS*6dN~#bhI8CqrgjjwH3)b*3KD?qulHP)ucJ z&-@@JX5t=fd$g{a(>XH|WCTY@IjmE|#bjn(8E08#afuMP9Gv4JgM`a0^?*tzI0*5d zj6&Uw2xl~)-RQL46Yn=E*I_9fb57gK=aN#$Kf=* ziTCg(F2tAk48P%EoQC7@9xk@pt+)|y;!ymHH}N$-#=H0#=i*)*iq~;BZpY>L7LVgn ztG$dv@h+an?RXmJTFTA%8o&EE{5LkgaXK&^m<~(_rUTP~>A-YgIxroW4onB81Ji-& zz(?r-AL3ISiw|)s-ou-?6{q4_oQjL_FuufsBDe2PPHJx<59xEhz^R6L8HaW9_rQU2Y{+3CP^U^*}zm<~(_ zrUTP~>A-YgIxroW4onB810RPD@FlLoVK@$#;zfLm8*w21!^8L%&*4YhiVN{5Uc{F; z5m)0_T#Kvu`=}@KC~n22xEn`$Hz(syyons6hGsB9FK2( z9R3@d-#8tZ4onB81Ji-&z;s|bFddigha*Q*kKH#k1aRA6Md1e2y>iyf3t$Tk$*o z#^dA-YgIxroW4onB81Ji-&z;s|bFdg_fbb#;h zBu>MRcoDDRU7Uwk`Fks_#l3hKPvT%aiwE%|-oROp&cS%xQhzV@arkd+e&cjtIxroW4onB81Ji-&z;s|b zFddi!B2@fwcBxA+u?;#>TPH*qjN#GSa$1Njd3;#fTF z-8_g>aX7xkp?Dgf<40VHn{h6##lv_Wf8uRCi*s=-e#N(V7a!wt<5wT$zuGxF9heSG z2c`qlf$6|>U^*}zm<~(_rUTP~>A=US1DuLaaTzYec{mMc;z!(y-|!%=#h*A8&*3;+ ziVtxgPQ=f+5+CAcoQg~FIDW>jxEV+KYW~H^_!S4^R=kd@@jc$h;ds;&tN!nBaWfvq z*SOxt>A$u4tf+?%{KMjqm+IzQw!#P5zzDnd!iEU^*}zm<~(_rUTP~>A-YgIxroW4onB8 z10Saja4Js3v3M1i;#|CmYxz5<)egj?I1tC-SUidM@Fq^huXqx7;$<9bwO{c!zQ)P8 z6Sv}OJdJblsMQX~?f4pp;#)k6pYb(r#j`lr_|?bhf6C^!P6ws~(}C&0bYMC#9heSG z2c`qlf$6|>U^?(oI>3>*6vyILe2ZssD9**Tcopa2P@Kr$M{y;t#f^9r7vo_(ia+r$ z4#mB=lE3>}?Oz=0-Tpp`EAcR1wc4Zj7;oZr9E!j3s>S}kif{2O{>HI5*+=U^*}zm=1g#I>1{v6))mCe2Z6cAA-YgIxroW4onB81Ji-&z(?r-|KU>{ia+rz z{=|j&5ijCYoQC`G9Ny*cq_`Bv;Z}T%oAEI2#ksfR{#{JWd8(}C&0bYMC#9heSG2c`qlf$6|>U^*}zm<~(_ zJ`Nq=LfmP!Q*k2h!(I3l|MK@s+=l0HAHKwUco%=-WL#;rr|~d8#hLgRPvT=dihuDd z-p02$8n5C^JdNW$z3TriJ742%oQtn5t@__1 zU^*}zm<~(_rUTP~>A-a0qjZ2@@f%LXk$4PW;ypZy4{AM+=>VBD}PtTqc|F;<4C-YhjA!Q#;N#~zdz$!JdEdkq2ICE^|&8L<69r) z-`$*@4onB81Ji-&z;s|bFddixE0=ixJa ziHmVCUcA-YgIxroW4onB810SUWoQMzc z9j?TQI1VRzH!tEi{EKUGF8;-J_!w8>Slo(xaVxIHz4#JG;#Yi%L-8qo#@U|kd-JFd zovV4y#+MfR-vi`Oyo=}YD6You_!m#(TpWw{eUyK9b9Oo~9heSG2c`qlf$6|>U^*}z zm<~(_rUTP~>A=UK1N@0&@gv^EiFgl3;#u5?Tk$5|!;kzu5|`p++=_p3EuQw(T!|;~ zDh|bi_!M{JQoQWl?%{Eqi%)T}2l6T&#hEx8m*Q7EiYI z4onB81Ji-&z;s|bFddiBQ?H=e}TKJ=;faXgO3#dsH|<8T~`>+vX_ z#lu#+7SG~)ALZZOoShC#2c`qlf$6|>U^*}zm<~(_rUTP~>A-YgI`DDm0GHuDoQUJ_ zAMV6?I2C8&I{yBMcX1$Y!?E}nf8t#HibwG=?!?#peHc&UVejTq{D`yhB7fJ#+jtm< z;%@wlU-|nlKE zU^*}zm<~(_K1v7p3g_WUoQTu#C|<;AxD5Z|QM`#~@g*L{zjzM6;!iw?cX1}}#Fuy* zf8udGji2#2j>g@17MJ2;yo#ssCZ5Le_!V#Ce7uZ*@w6}GYyST0qx`#@v(tg;z;s|b zFddiL^P8NcFN zT!@eHtksUR+KIRof8t$Silgx_-o)QH9S`GN{E4G+Io`&j_#UTvH=p8qe2`mlu<@&p z!~b;6Z=4QH2c`qlf$6|>U^*}zm<~(_rUTP~>A-a0SJ45U#D#bif8td9iSuwLuElHk z4bS36T#Fa+AWp`OI28BcN8D+(Pw_Uc#=ra>6bIu@9El(CIbOw=xEJ5!UH%@6S8+f7 z#_PD+SMw~+#pC$buj1d~+%X-P4onB81Ji-&z;s|bFddi# z)&9hXI1snuH~fj;@G#EAZ@3r_<4e4WgK;X}!_7Dr_u^2TjU(|T4(0E#co-++R@{vz zaXik%qxjX+c@)>;RGf}`aX3!K$2b~qTg(4#&2O6yOb4a|(}C&0bYMC#9heSG2c`ql zf$6|>U^?)t=m4+cSR9H6@h8s2cR0{u|M$vQdlF~jJiLfc@h>jMg?JK&;$D1>NAV=S z#?5#dXX9o3iktB=-o(qe8;{~(e2b^?sMSuzy?7h{TJ2Wc>sRsbaPF86Ob4a|(}C&0 zbYMC#9heSG2c`qlf$6|>U^=jtJHU7N5U1ij{E2UI81BPoI2X6#KRkyIz1!a}@f-fc zt9TZN;#L06id*q2f5*hnxD*HDah#1q@i{)m&$txF;%>ao-&1ib4#(xV6({?He2=H` zFD|y0|J$11HXWD_Ob4a|(}C&0bYMC#9heSG2c`qlf$6|>;8)QBe#M8l5Z~cL9ES(- zCvL^N_!Jl7U;Ktk@f|M2u{arT;%S`hLm$h*co+ZTN1Tgi@iXqmkvJ8<<4`<|!|^cw z#h>^Z$6D=RypLn8_OD;Xzr(p>IxroW4onB81Ji-&z;s|bFddiU^*}zm<~(_ zrUTP~>A-YgIxrphRdj$W@fgm+i}(vq;y1jB8}TAe#J6}Bm*Q6Zh+lCXZp5uP6<6bB zT#PS0(Epy^yEzhn<3yZ_Gw~>1#nt#2&*F02j*s!VFZ6kwj|cKAUdFX}8~6HE{5zaG zrUTP~>A-YgIxroW4onB81Ji-&z;s|bFddiA-YgIxroW4onB81Ji+DWe0c)ms#yVe97N4 z@fd!?z4#8_;yRp(A8{jI#JM;U-{M}}hX-*buEo80m%pFlR-BAm@i}hCt9TuM<8i!< zPw_gg_e3tmxwsy0<7-@tTk*hO<-g0hYdSC;m<~(_rUTP~>A-YgIxroW4onB81Ji-& zz*_78f8t2IhXe5>Uc>x6j>q3P6@TMt+=|a}w5RJ6@8V>9>sR^j za_*WAOb4a|(}C&0bYMC#9heSG2c`qlf$6|>U^=iCJHUDPkH5q6_eWfd|8Oj>#FID< z4_oay{!WWm@gSbVyZ9Ji;$nP@2k{}k#MyWihvHJ4jhk^bp2oE}8Q0=ryo|5$D30~j zRsP4-coz@ka^qKP@qe1;H%$ko1Ji-&z;s|bFddiPcnyc)IUI>c@h(2bb2t(&<2e5Ai#zcx9>t+}6IbFw+>F0*H@?Ql9_a6__!C#- zalDLUaVdVr-*^|#<6&ROvA7lY;&Ggh@BQ2OcRF`Y2c`qlf$6|>U^*}zm<~(_rUTP~ z>A-YgIxrnrOC5-F7QXq9Zoln+_;3I5KRNIZZ@vA_ga7EC{KL&3to&>9TmF{$fazb` zACUcP^MC(?d*YsjYkZOahx%&^|G~m9&vfwD7C!gO6TkL}4<`Qm&VTI_AN<4W@!wfk z{N?$@PcAI{&HwW^|G(|O{D1%Ze>3`5H+-4`mv;QA|?Y~=C_#gjotN-r)@&AQ>d}G50KkxsO{rGp+`T3;jz;s|b@c-`) z9PI1=qW|ydf71VV^P$Fztv^)t-syPj4>!N1@#h=ASG~1*W$T~o{x3FvY4guE{*9{l zQU6EdBh}6A|LyAS)vH>6t9o7Y|5cy2xA9HYlhvi_f8M_DbpFeY54P_8)>oVVjq0~s z|4jA9>fY90@4lO=r(6GA^$)9Gul|1ZyX}9h@eS42sy}P}nm)&~$%)omUN~c1eX05{ zoNs)-@t3NHs*kpQdE+~)%hfAdf4uR@>aREdM)fzFzohT`Th-mw51z#q7M^V1+3M-) zO7+jW@2?wgZM?mERvLe!`eyZ0)t`3m?&|L9_UgL!y;D8V{Qutg+3G*7o~zzcU97&_ zx&Nv0w(51wKh^kD^^NAex4qc>r>lFaKd!!9y{fvg^Pi{>KW}_>>;GHhyBq)A#(!LW zqWNn(zfkR2Z(&d4f3NyqRM&TIXZO5N{gdjZ_We%#{#o-os&`fYe*1pd*n9NnJNMtT z{-=%q&+0#K|EB6+HGjDAx2k{H{J!cBs;^Y{x9{ny_t)NA|9SiFsBWzORr~L6{M-7{ z{Qp|r-TFVQ?rZ*v_FriJNcDTI|8DEM8h^j_%EFggzpwRw*}^-GAE`dj{D0d1_Z#0_ z{bch$YX5&&{Yv$}ZvC0Yn;UO#e5`xku70igzt{Mi)&HgXYW4TqccFSi_08(_)t|Kg zSmXb=&;M-W_p86z{PD(H8sE`)TlL=RV(Y)x_)wqoeB*!6_>}o`h)6i z)&H{lcT_iY{+{Z8*8H`NcUCWUesA^m>Q(Lk_3DGoFL&;@8t-rXyN%`GrN;lLx|S?s ze#>-VIxroW4onB81Ji-&z}oKsXSlQH*|)0lvpmFSV>yw0&OPKFKE_4(4%g!dau8?a5vTin`H(|# zLO#K5uIu})@0?sEe=fJq53X#V+{%mOG){JF=Qtow;3Su{E(h-GUjD_+e$hPlINv!w zC2z{vHYYjjyQ6(d#?U75NhHTxmZKYq;=U=QzX0#vF`e{jhod zu(=C30k4+3-)uj3;$**kx%)S3-0Qja$;bSLH*+I?aYLWGt8xB*Y+ zSKHdhhfg-EO|+B(1IH#~?RaYv5Eg?J2?TIqZ7Og{Cc=C7*qKn}?N`OqWnA-YgIxroW4onBuQU~N_&&zU{oa;H9H^}315s&HddVy2O7xJ^bDxdJ8 z6Fq119eGjSmp{1*Z{kJrwfrle$&tK$ybYgubtJsos-|>VmbBF z)_IbADX;Slz9Zl9M$W{8_IDpolNaS&IbQCPx8*dskauz!{>bO#c<#ed`5%9igE=Vw z<3{{Jetx08eX+`I~&*f^ooEz{M6Y5h!-wIYlLzvl4XyLAO`YdCcQt;ZeM{AMo9BsdHRfSF_iXb# zpR;kLW8L?5_1^Y#EPl$By@%n*hui<7>O0-@`NltJ{O#(K&3g~SE4@$nRQuM_v&{UK z>A-YgIxroW4onB81Ji-E-vJK9UHFtd&VA%`dFDjVzw#QleQn?%Jndxj{N_UA_xj=w zH|89F+&nkCvUUD@sC8b=H6Co9bMjiQ#RV^GA0Oc$FZDUxg9CFQUdk~pwx92Fzir*W zqq?PieDl)A*HroF^Nla>9KX7+F~{I4T#DoI4?cCebNptdG55N)F^BOUgLD0U>sR!> zIP?9D`3jHWlbqfAgX8UA?!Mhs?(;_D!`0`i9Gcf&Xr3eTGET{#xa6n%-0M5Xi@5KT z&F^X7rp8B%^H+}iK=a(|(Z*lw9G~O12U}li{ZwN; zFddiORnZ%{DrG<6F$U?I0_fy zKirEG@uuy4PyWFp7W+P2?dHbvyIjjR0a$T zci?LBJ%9Lm`{X8mBLB;i^4|;fgEPL?`LoTx-1zavCmM4;9w0Bv-JFdpaX1ddL3kNY z;(a`A?#20j+J5<5uH&0ri6?P6KHA4F$l*^l!EN{=C%LwD z?#iR??t9$P`f_8gu&#N|_hJ5So@a1gxtQy{(mvkGAHLh?@HH+c*I(5-Z{jl?kFRhi zj&pD4sm*_cYH}o@t$z9B$01IN9;$xzhf| zJdwZi8a~KjFKa*dx!CvE(LAT;f$Lkp&^rI)QM_nh>r3swzsk3Ex6bDtZGC@rU-d|J zZ~G56=64sH=SF;wr}0_7#Lsvm=i1agd%B-5@-yyttabj!+c?)!_pGI7nfWc#f$6|> zU^*}zm<~(_rUPrg13P<;l{?QiUhOwrhnMgKj&-tgi#<2Xw{jOB;sqQ=UVOH5d`Vv6 zXL2{c*xG*XBS*<=T;@Xik z`|u4Ov(mi0DGwd!dvHWKQf`w+IgA|6hpy|qoF>QeH~CDi{uVc`mTMF~@kP^KvH_JlDKj&Es`5Sm<*%DM&l8s0 z&r8p@{&bZO@J5~~AM+b|`pwSC<9zc>pTp(&4BzJ8JcfhvJbu8bIMRvk2T!&9`4}QD8&*w)x zUz1aPH zlMC{k@3zi)8ZO-3dCq#ch272XYJ6{%t8&Sk+Q+S4Zy(p^qMvR4_p0BkE>$@>f90;6 znosU;Kd1A4(U<6qL4OY$qOwf3KI<`+%}rUTP~>A-Yg zIxroW4y>gP$T2+T`JRKHt#TZWaI|?&!i#u{yvi>=F5{6h}nJo3$j_RGEU9gp~7zmr>W9eJ_k1$j$;;)9%lm&vVH^ttzS!H(+X zRe7FAY;0cs=55@CugUw|?8(l3u_{N)_54Kcezot#B{p@Ahsg8tEFa@*Cpy2abGI~> zH}7dIFY+3m!x?zn9qs3$d~;{_^DI7dd-HsPqsiy|fIFUTKd0k0_xHJ%wayzk0*B#k z^7+Be$<-WFj^5Tf=f0-#!9HheW8S&1F$d>5zi6JL$n}5KJxi_2;hUT1FMOQ)a1<`} zQ2TinzvASaloPGA|Gnx{eLlD3i~NO~Ue*3L+Rqg@^`7Q0tu9wN+Kr9BUghgAbT2>I z(mHqkRO99Lahwf}ISz-~-TY$pudCdTQ}QZ)!}T8M^VYlP6CeBopX4)V+V|rs=jV=G zkXs#TA4lU?-mkpX=Xqc9cKffZ@>UMY%lOiU_VF;@$+vjo`99~J_IoeE+4vc^yx6`^ zS1SvLyN4t3rEA;3x41MnA-YgIxroW z4onB81Ji-E-vPeEr{u~-Jn!L6JU~8@f4IrU&ha9? z^nCZqqsJO^nio5_smj;n&Xdh^1>Ur`d*n>6B;WHRZXsi0A^Bjk-$k!a}T>JSKCzIcJ8+UrJd*x*=!MSeeKEA`_xEJ^2UR;P@@&+F9 zOrOUi`GY*jb@+f>f3okvZ{$GQY?x8XdGb&j8LMGhdB^E&>+)p)4< zd}p7xt$Vjt_chOB^6pUX$DY|L4>4#(suT!hQ-k%L;0N-6d?G)}xm-xjkt^lH1AU(SASZDpIgkU%o$`a+$Q$Hw zIYutKrO$b}Di87)d6>sM-S3v?j&)9cjPgdnh?#4CbAo-5R$mfrDo;&gvxsE$4`;aI#- zzU3;{^}DaBawUGk!#4GOd6FF6^n$$lOcPvGj^~UI_c{FQipK9$d4_!WUiWd3o7%_A z_BQ5hd{JIL+WPz5!%KMs=aScX0SA)P`71BE(7jwqp5`YU^IZGnZ#kJ;{ZM>^7*ZsrhFIIW;p2mFPvz^;t<&6AOb4a|(}C&0bYMEL z_B$ZY$^G)%>bxOO$$g%;<<=8DM|+<4{4eiwCvL@wJkNX1=Kyk(=WjWJL+}9khqHLD z=V)@CJns3QAH3D`yu2-6as;j<-<)qhCy-}(2mg^*txFUrUA7RQo*<#E0u4}YZ( zcfXGt@<%?#3pm&_otNLa4u|3~yn&nWiJhI7*X3k+oC99n_j;|my>maT%ISQM3%$@f zSKtG3Fn8V3y?l=Q@*nwB?)`4}a$^p{1vnW$;ji-Y*E-K7<$UgYxb>5LuVYm?nm7HT zd2S$Y|G0U+!C!eEf0DC#0>|K7ySs zU^=jtIw04`-8@C!mDk?vd6#3zm;6JHk~ifFekAX5C(qGcyCB!|3*IBQ$zk%c9K#p5 zg*?VDI#1=fawt#biktfP{+reP z-E+JuzssL|hktQ`E$!dYJ-pzB#@nkW+9$X2L$1J0IT`1WhxykRyKh7H$?<%M+iq+h z|J&AnUd;$=3FBYX1FP`!BZ65qZsG^E~hV#y3{?R5=f4J>R{Y@b1Q3>O$k? zDu?EjS2e%0$_KgmzCM@RbIoVk&!4#A_uGF%>wNq(&A-$9=RWblKRnc!tMR%go99kE zenay=tA46``RY<*4){p>?y4?TxhRMGLFYc-`q9R_8y{?Zw((kemYLr&9heSG2c`ql zf$6|>U^=k&JHVeV?YWj4a1c3&hsaYrh`UG?$l>cuLD<4jk#|Gn1bWL~kQb&kg)+P=Ww zIM!;9kmuw$xsfxT>;5y<*SnaP9cg~Cc}~bL<-|>`bDR^MlgBy2f%b8d^Nl&C+|M`U z-p{sAUYA2T# zf1vTt8}r%=&2tuB&igqlN8@9hhEs8O{&}*`;YfVMdj^io!=CSZJX+<@&vef#>>X z4LA&!Ho1m}=bfs`*?fd+@fcooWBd6HA9$>JUdt8u6HnkxJo4_& zZSQ-2x-sYEW}N8G_VHYKTn^`w+{51`eXDakVyVyBSLI(^j@QV~{w{@g$oo%rFW2Ll z{6Lyugetxmie%`@n_~Dz~%Sjg7&xiO3Z{k3_hFkM6UUas5H+0{T z>aOO`HRe%#>b}+wS2-9@yuSG_RXHMeKi0img+FmQ??;w8zpee8()$6P!@<_IZ$szq zX#94S%kd#@x~ctqmLG2I+zqWaT=2f*-sXSO0-rwKJV*5YWq12HE;o9={Tz#P@^8+{ ztvDXnyS95T>z-E|AFW<&o;!0n{^|Y8a{IaVT6&h5-!dJT4onB81Ji-&z;s|bu=YE^ zAvg|ikaKto7m^F)Hu;rr@D@2!PM6o@SNZJnp1U_zx!B&Gvw4Xe${n~1&yb%_bdI;l z7fa1^9r<6ry3jhG;9_#jf!2B2vyJ6O`Aa_9+3%6R)8y?Yf1h`Ug3hb8sjQbh3T&{%S{%x4VAf z7wwljKiA(QaFt72mlMx+kQ;K!N87i#%7>QQ&$T$)7n|ovoRy<;E`GYB{l~kXQ=M=A z=H}(rEsf=Fe!@wXI>#+}7`Nek+>|TwB;MffCHWaod9V9d=jPvQ{^h>ko~m5TZ+J0R zIVdOP&)kmJ@dX~X zt@B)vv#sy@-d*KLZ?>Oj@*D2`Q0ttPPcL^5ui+^?i*x+AeOs${b|0_c8@%$C_VHp) zbgp@>!M%7WANf}IbC0ig?wRJf4aefhJY=Q)yqEXzV2<^A_wh%5wxRR9WncTa3@`S+ z;mY>^pvu#qXw20vHr~{IZ&i66e_H2U>s*V6o@$*d?`j|4;%6MzdjJm3wJzx#Fa1$t zZg+2Eo^`CxU^*}zm<~(_)=~%L z6M61P&$)6Ox009n5BHG!<@gVCLC?p0Kd`a=H8{hspT|yN{1- zYb-a)7d%WJ<__G9-^lAvb&veRGvx0b{Z9FZ6Ueh1g^Qf-KDmtd$j7|l{`Ngql?(Zd zyv_%BnS8_}d5??8Z*nB3kQaHMT>oVE^Se*?xjaXn<}lod6Yx1cv)Fz7a8u)3x_4jo*{b}> zEiY}JQ*yPY7q06cPE>i?BYn=5&CB^*hCd%_-%IUVY5ZdKXH_1+<>Y>D$R{}oC;MsV z_jW&*;RhR9=QLd6aP#st2YH}-zT5iADhJ|P-1h0#xjP^Da^A`N_~rB6@9(epD8J+5 zueOgr>}kxu_#rRhE*ILzSvWJVzo&I>`G<|qRk;_h;%9HP&VxDg?&i70+4{i)In-6n za}wUMy?HLf>7Qx-X!n1!@hgqjH@>sVQ+VD|^PF;P=bmf+RO5H54>r%ScpAs$@fX_1 zw%6r_82gwx&`uxNFUXI44T3%S4U*%T*B#+C( zTu`2u_c#x4;6{9CN8f|9$Td&(MdUY*Ajk10Igek+hn$4hagulXJo!=1<1@U0AMheB zb!X=}9e3j~@){TA0X%7E=Qme>((mLQe5>UJ`SGUqan{G%AfL;ZkG1dm*5zDzoGbE* z_u9w#c%~eCuyy&KTm4n@{EVx}`|`3}%gMgdxp%AbJ+HXCc|OF0I0={C-#$*k0r|+G z`oOce&S#qEQT%`pu5|C7_HhdNp8ITU-?sK$tlnScmi&O@@DF~%?|;~RE1h3z%#(RK z7viS;_ju>t>KrfNKHHn;fSiMCaYL?pU;7ukm%DS1_04mqD;o1(zR!($&WZNjUFBL^ z8goSc$u$@IK0j#Rw#F|t=78Qa@PEE?vHkq>g&~wo!edISJyUP`%gIY3#S9qf$6|>U^*}zm<~(_)=~%LRk>T9 z^E~ajSMHIU``k^U+PyRCWe58i2>^B!*fPpaJXvyCsWdVj&o zI3utAQTx2l;J~Mw|8$>wePizTVB?k-PW2DoKP|PsmY!wiw@e471Ji-&z;s|bFddi< zto;snj+YPQJ5C{g%58G4oZ$Iip7mTT*Ld!KvFBya>v9SQkwbZsTr2SDUlMm(8^?k7XBLDAeo>%Y-c~hR1ueSC5_z#!7qkH5bF1Vxpa`2|cH?^NT@RZ}t z%f0-Dw;gF+E|){OPS-E+B!0r_IHkPL2jy6~pUZGgIg$@?#shsHzVXw(!1*d4`B~$~ z+Rvx>6))g^@-J`XKHTK8?&AggOWx)oe2zcxBDtB%$^9FbCCZS1&Zr zCwTdj%|BA*soNW$=$@78HPw5nuU7fSnZ|#&^IV8ma4ufK*>3C{r{l~#lGE{zPqlxs zx~0koFX{6?Uwx~}>3A|P_MYOZ_HnY^-TTSr&sI6g=RWbl1UKTse3290)w!=!&vozJ zjXCr8o9D`$mRl~ipObQazRbJ5H{qhI-HA_nKk$cruJC1CvIr{ z-p+CVx0~m~-rIbqeLrsf?&`X#_abjJ=AgHA?x8AoytaKmZ+`7R;mj|b4onB81Ji-& zz;s|bFdbM+9oXJ;tbET03Nr8y?D&Y_H*C+8!uP+)ANmapFGd~F7`e5wogvwA)Jf*@hQ2McRbTQE8W9I_$8l^ zqgQ(q_u$JUZW8J^6%7yqZKjY!Nf@AXop0%fYZmzEMd8ezK?(NpsRWEOyyB=$vlklxg z?c*67k|%mkz(qLKhR*RE-pRji=<|5|W$piFmHYC&7n}|}mjx=9cc%^^fa^8zv+x&G^UdxZqb}w&! zr+wU!V?NS8{>RZjnFHR@`cmhwZ~Q>xyBc3r<$g&&Be%=XH+4d*qoHdv4|oa!Zfh3-Z3_>O(Eah1^G; z;0SV*{3CD5`|{Sg?voF>gj~Q~xQx8T)#M1dj#s?ay{kP#o|NY~fn3M?cGMTXCD-z* zpSCWiJk{s%Jh@B$mTx+?Am{M|?y;#2>$^{$mGk6cp2O?*wV#v8kd z=H*W=@QdazR{0=b;y8SU6Y)QedA`r#43{+iN%!(Mj?0f8ZJob;y77jp{LdA3b}#Rf z^S{x4&MK$#g8N(lohk?Bhs(`BQRNFSH|CcI8*@{B{KtJS-oOjE>EYH-cK*sLpX7UQ zG{39;w={mQF*oJ*JcalDQTyJi@+v;?d(HDFPQw-by&13L~ zq569DXVu%Q{*LRe#y{vj9?G-${%!5!Y8;i@9_>7TtfkfXVmoWmpJOL2({m$(#f;C%e%MBnFp=j2_vm74JnZSloJ}5->o^<_;JEknz1CMb z#9wuvJk6ChH!pwk$cvrdSLMEZfaCBHuEELP?;LlKQ+Y8b`(mFjS92w9%NOKqUi^6H zcrRDx+q{d<^39!{6pUcgbm)aURh`TkJz{Pa|P ze5%SPxX+vIUupfx#ycDH22RV}u4|nSZEJi@b$6AkoNLUHZ*I(Sxe#yVFI@Lx-;2X> zC0@smIqe(mzpr|v&+{IFoA6EkeMS3Ts_y9g#_G?jw^n(-_a6L|V_n|9XR6%huEuZo zIo?BX%wx@SARhLo_IIA=ywW@myS;V(!|AxsqwVM6i|yaq{GRG9)ye|L;!NC*cYD8Z zS@&_J<;I+nv%lZ`U-SVNs+*eUq5O|iU*Ed-PTZ4EuC%_Eo@M5@Ob4a|(}C&0bYMC# z9heTR{SL^xT;kfEkLB@0jpc5>#h0#borlQtd}mkB$sCJQ$zdGd<$5S{TC*XwKk~^O7{*&Fa(pV1VtbF3HTj!vh;$-tY_hR?)!G;U_+Q$p|6F1-{ zT;UfEc20ih4Lo^&>+(2%*wj3~;&N{`&k6X*Yki(veyDv*)oraGY0P_g)XlALsq%2H z#Xoj*o0aLH(!6E{SWp%j#VG2avA=2uKnJl@MWIL zpE&xb+RxW`F+V)o`h`A^BXLo#yu0UC%%l16;pV;n;9Ivf|M@CkTl-Hq^9!c~(}C&0bYMC#9heSG z2i8&t1c?w5DDh`cFpaS6}C97T@Z({s6;At!pSm(QN-d73B4e_V?9a0UJ)PxCU* z>vEzzBLB&u@+aTnAMynkl5@F^+$=xxG0vF;D@)M7ctL0LD zzzO6;9?0#^*XO7D_j0T%ugTM#YJK|;w2#a15uU&?}il}+K%8eW92UnD{xyEaqm&@;M|F-JJs=Ud? z_|jvY=LdY1GjfAn-Lt96GnX3kKVHV4R$7<0U+wew3THUkJnwnA@n2LqEZ^W+9Q;n- z?-L*VgZ#}2xGf)eq5YS3KYuygJWuAEJeoUpY=PJORp&SW&*n2Xb>KvmTXWCF&hrZH z_NVRV6ugXwbLai-(w7JjYG#ulfH_zj?H|8?@iRWM6`V)OV596ZC&2tmZ z^LppFD3{@Joa(pQccgoG-&4D{Kd_;7o^`14_u9wBzt#Bh>ig~UK7oJor~U1FyM5lfaJajh=X)H@`;X^4 z&%1cta`T*y7jjO{#WQ#JIlP-E-`xD~cc1qIYw1~Le#>-VIxroW4onB81Ji-&z}oMC z=WMx;cX)1><2lK*JqOE0d}DR~kx%7c&++n_e8?mC0YB<7d%<(PoFNBt4t}t)1M-QS z&2cyZUy?uNFgb8Zss>UN1o?({DlkL(dWrO{6lV( z+qeub;U01*ALA=>E=PKy&*3Wa8DEooI0xV3k-UyW$c_9!-h8M2%6a@q{+Gk}lswAS zp6|RoD#!7a%ljS2+b36Z3Ep;D`{c)!#yhKWru?|A`_@-)sB#?MAfI!tL)|C8^O%#( zpXeUBlgIwF^{uV@`w6-Kt=730_mOYqb2*lSe7gI17?0s6T!j~LE>6ZXxDVIn4Oe$R z&%3kl$shR5H=Ey8{b7}h{-`mp;2ivgPw^~ypBKsJd}d4Ei$grw{al?te64x@_)udG zz}NW$xBFE0AFb}~+#^+f#@ViF{)^RbcaA%9l#9*tv)dbgrE{G3So8dIee2wcYy113 z^R4qA9{Wu5f8Kp(8*`c~n?G4ysa6&^^rKCDrn~75W*ZvdE{KDzLbYMC#9heSG z2c`qlfwj~D`AHs@>pVxxr<{q~a1^=O^S2x;C&>YFpS&X{%Qy0~=XR|py~Nx7o;N!upK-BETjwcq=!WL+>)fWQ{B^8#4kzE;(mdypzj?sn?vb11 z?suE#Wc=?;_wh=OC^yTWd{54n&pFKL?s=`c+~;wwla2W)pW{)ypFL!`_?t) z>D=!3T7R+1yEeDqdzgD$=RMxn>}uZo65hw0(=z;s|bFddi< zOb4a|(}A_$0eM%B+}3k1Uy|!N)S1>V>p7S|aTCrZ$8T!i&MJR+tMSF|<7m8vzwk4j z#eL)v9wdiz1%AP|_y@0&U*&Zk!;$3N!+lRVo}2Ky-)Nol^m7(g`-ptZ*SOys9gwf( zLHYdc*7?v%<43Ffi<9s-Ih+^B-TX^FmEU;5sqW=sa(6#-L5|~DPqx56_!Iw-U(aYw}C}#Qk^;*OYTP9N*+Wa-rPE8@bx?K9~Q}{UAa6&$Huyy&EtH{l~nL~02UT}ZsIn;^9oP`f8_x<@D_uwVm zkqdDa?srG`@+;0G$1k_eX?VbfK9{?3(RZ8Y2>gbxaupu;Soa^TURmWeE8WKtxgK9U z+xmM|-pAG6=|29rp)uFFuJP^Ff8ITOlYerayISW;w>G}C^E`uN@g$zbF?h?C&h4x6 zjLqG9vU+#bdxlrr&yy~+&gbrFokw%YFXv+W+t2MEYYDj(APKk8dqCexmw=Dxds$_wb|hjd>LB=16>OWBYj4 z$?mJ*9heSG2c`qlf$6|> zU@dh(o{|6MRr&10=lY&^qgFHk& zlgpm%bB|QdRpmyxQeKhw%H7-gV)CK<#{1+a`TWi9k?XhxAK+c`C4#D-`UvymF|(JZ*0t!_#JOqY+XL)PV%k1D#yvSD}CNW)vtGt+{fAY(fzGIUzM}@ z!uIar2|F9}4c^Mj`2fe3`{i`b&M~;&@h;-oa=e_suk~B2{CT<0 zl+`ddJn+cydU7;{F0|}R__V;oc98s?{|4W!n1h#lFx4+SLI)vk=uHoabNo{ zc79Lw&gM@w=6v3-cwey8KJP6KHRj2jj1zlr!nfAav&{UK>A-YgIxroW4onB81Ji-E z-vK#{PxQFFAYWhB^Dy_259P?aJHT7ywQbFF34ZZL^YWFPEQj6PItP-Uxe6zfqhILz zaW`JY0pv=#S&r-20`HN><&y(_53a}QIG8-i6CQ0p-{BsdimPnygE@$N&nv#!IzQwn zyoXo^_nk*DNizQLdPf&9)-p6Yk*?0d?Ia@;Rk-_bhH;Y4yFKam@` z5tlsHdA=#P$z?o>7i?-jkCRI|p1du;@fgneTHl**aYUXe?{gClz}w{g+xi~zH^<;h zKW&}k@k>s%q3^*1bq5b#sB!r$=0vwzB5&B z$-y3Lo`dib?!asIwU3)|3QqNOpZ8q#^(sf?8T|LX)?e;kuJV5KoQdynn=S3Alw()kOmAE@#!PQ}~rYJGj@HZze07_c!KGyW7tl?`VBn^OsaVTjgdvlwY22ogZ^tuEYU%cR#Qt_ zxGWDl*uGuuJKFeXjkz#4<6gXq&+@mm|AaHYa5^v@m<~(_rUTP~>A-YgEp>oP$!T(t ze8#EdX8Ge>&&`~KH}Dtvi&M0H!E-$K;UMy!T*}Ef0cVqoAMHXppGWWne!$^4k34u! z=eYyFd#ZW)oyW-$9E;CfR&F@eIgY@aQ zFLa*g@ihL+e{SkNPIIjB#_Dy}t32fS#(z=0tNJfZQmz9@xew8^WDyI|*D* z4u{*)KK{1cz6YA$*!VkDUbxsgkAA52ZB_obyD>*R-I&X+_9HIFyN>obS9Om6@u#KM zU#;HNKK^}e^St|P>z}W_Tji*0=~-re%XDBmFddiv!EAIpn!D;My*E|IqinNzkI!`3*<>oz^mjgd6I9f?;Z~HweH~?yhaY<0CM`> zotHCp1Rymbl$m1N47jvYG>w}T#pCw z-|M@FH*jg5!)y8TEqxAG<_7%nbn9T5* z#ednDXYhRf`18(jH4edZ`4+EO?mnLTgT~zSRAcWIc69$e&7Z6CjEjvq(^HK(>4nBe zt9<0X?!CRrjkmRLvHD2s4_5b9`4e9~*nS?vZTK%=(l#)q1Jr+rsc zx!{Jz-gofVCp-6NtuHk`TjidV-;twFHZO1T2f2c`$Q7RRO9OUo60K(Y2T^FoZ$Y(ype1CgXZ}jFW`s#>E`yy z`&{No^L&Ws@Z{sI^Epn-rMVyH<5nDnUvic=`+XeuK=;4jJU2Vv_>wB$<6}ID|L_<7 z)U_Y}-|ONNTw_!B@*KX(nK%PC=PulW3-JX|MTp_o#*-7 z?}pAlU%jh+JoAp``QXLI7u$EX`hN4gh?j99?si%Gw^hCG;QRcT=Wgv@?>CNgjz@hm ze?8Maj&`i^=JxZggXTNO$-H;qWE^^_^K0o@W`4_bU^*}zm<~(_rUTP~>A>3W02kmX zH}xDWr}UV+Aa~1Ayyxon^AheM*UE|V(8>18^E`w@@iF;DUXg2gg?z#vPIRw4B`3>? z@+?o{JRC*7l7m0Y9o;JrKGv9DaX)#A+kKed+Asg{Ek4B`xPx5JdH%Y80i1|u$Z0%_ zugR-&FIQRL_xN#@dvOEKBOl9caw-4en0NNQc-*<}HFfzES1e#DbG;PLix+uOV7)71xCzrQi};W1qLfz~;jCgYDyA-Yg zEp>n!$*KHE4wVDAjC{AQ=VopqZ*eVtB)@VF{=#E80_WP&y_|xF@Ep0Fv+)gHB!Ban ztGn-T&+T%lyeGf&0$%iZ`#B8Hk(;^8rQI)Qa=?x+$l>y%ydj_Q8ZL3ZgS>`&@hv{b z&Cd1tJnukbF2@IX)3>^xTk<>(!uz^rK^~RgI24yT*+zaR*YbiRt#b>xRerv*bvcb2 z%B8$b?&qK{^gH;2oXO!h5O?E7T!wqe<2;GC@j-c7?w9wk?R#^QryAed_u@73H$UP8 zkF}pO$djCi>&VB~wEzApH+ik`p(>x+*ZAe?&DEdw{dQGPG|wqH@%Gkv%E`um*nSRi zvGMMzoXuzXAMfP@`?`-?o^ns~T#zsFC9c3{IobZsy;Wv%mH&ck>3*B$NWy4;o1@)0ig+3w}lk2Ky`eW%Lpj&(nu+25GcthAp4zSR0x zn&*)`k00^qt?lPVyk~v$ypAvNFy6xjPxpBo)BBYdo99Q|=$Ym(uP#-Ws~4+$??T^) z2Xg6y&2wk(B{)5&eYA?vs6Axt#lP7CDi3a3r~NrSHdy`0TaK^ODt0w6}G+lk>=x@-GkJeH??k z@d^I(#eUEE>I+q_%7ysuqkWH+_HlzNn_sT-6y7TLZ)u(1$@zSdJ8?+9z&RHCT=|?o zbHYoz|C%b#yQwih*w&bHa}BQbtv=_L_HnWAHP81rFgN0bCp)*)es0H^dC{)cf6;yp z#8G$+7dzB`&cX9LzQFhHYaa*M(D+OnFKK+LF}LRP&$P~g-frKyDxc$wZ?*sR>YY`- z!|^ya&*yU2w|{H*zftAFdm3}F>l)u!J>Gdf?7as6;&}JAkK6NL{>`8G5VyLkb6oLk zWB&YbV~)(dy$|7UJoWv)Hy3}X^E{Ue^30|7AFHmb-d^RAT+e%ecUph4`@P5D<-42T zRQ*)@f8Ln0UDf#W)gQO-+g0z04!7=o)!Ki;nO`^^m<~(_rUTP~>A-YgIbBd5}x+H2Lw_?vYF6Wv<2}Swemkdd9Ba8xq5YVrOGc>`h2-qKIfHhwl0@*m9KXX zA9%ho|KpULR<7nT^6dkilSk!g4t!1PJeCvkDE=U?^FAKHmwD%M_sQEFm3Lm~_ws5v zp675>Ih>Pm0xol^&*5jBjl*yfp7WJHcTeZ|!c&d80Vn!y`?=jCjbE>xtlrW2-*5b4 z^+@v@s(hR~a|j-_y>s_fm5ZI{M?8~DawaZvfBU^Z;1`>^aAWJd>(R#VwZ5ymzIkr{ za^tzRgvR$!8vH{@p4M;>29*c=vM~ z-o}9sw$9tPH1=MEukjz=xuefLU%jOIy($;-UWPXOb4a|(}C&0bYMEL_B+69SQ;@G-8y@pu%^IMEm2cXBg7x~=dtz0~Z-RH}xD~%tj{z;WfbBfP4e^vFlD$jYQF{ik!F<<2iJcL8Z^IY*%-=7y; z*I4f6eEdwV<_R24Zs!9$=Ec4jf9HV@_c>gNmvNZeJI7&pA2&SPIv;wgG3VwSo0`A0 z&*Ku@eNXdOS9iCcFKua_3vX_GN0k>YHRj^{f`4#&-oaCN`!${CQd=AI6@Gea-;Z}+ zY!eg z6M2F4y8C^5UlY%kB2IpF40DE+Oa1E&N9QJ>PxXx<@|Z zE{`__@(yoJTAu#_>=t2z2)x9yXV#J zd9g7E;RQT`FI?UJ8>%O(+>e9sr4y|`Tjke0^ZGuQSMslW+Rx+oFwfvwe2N=$InKl( z`4o?PrTcl;V&l839E%TgHtz>G$vf@;v2*<%9_9VR+pXW!I`85n+neX1-)+pb4>$hp z?%_f|Y0QsLHRdPWg>Ukb7rJk6b-BvDmb!%h>w0(i&^J5;%_4w76KHqzeN4nrERZh*@{;>T=s}Htsf0b9> z)|e0S($6>Vea}*3ZoHPBW#+d`2c`qlf$6|>U^*}zm=3J{4y?{ce1j)&EYIz7x#w#6 zSB~xRc!9t0gg0A|(>TETp36Cp+{BSM1;5}8a^UgK%O&z9@8V=~fjswc=XUhHc!u2j z_0~BUCyQ9B#UiE{kSeIB3SpL~h0$i;G@oVKm=a=6?luk#8n z%1b#IU%0Y+_@F%fSo6GvmvFOFt#dQ3bFulwe)py-_v58+w2!OuhR?R2=kOM;z<+tJ z+z-@<=T)w;(wHOj zxsAzdzL<>p%(^OtXSFGux0V5NN=kn`W${x4PUZ{O>Uxy|{;ueHw0E^D4& zamX|6^L}A>`*u}1H%B|%{PUfwEZpAwauYoJWb+4`f4A{nRetv;jqhmR>Bf9)sqrU2 z@xed5+P<^R^IGp`j<(L-*ZvdE{KDzLbYMC#9heSG2c`qlfwj~D&+D8=F521iuROtT zIEGxr9poZTBuB`*7rRf6^l^T)_Y2ZZ5(%3G8DtTNkKHR>2-NS9Z(!4x>sqa-z<0@Zl zA5W8u`G{OD&+(}3os+-iaz3%Yb*>=yali*#KiT(^LoYOz$9bfj$pbfbo=0+Fp2GDw z8V}%w^6P3p=2YB@r(W6jI#GSG&*y|3kH>PE^X=PI-P3ul$}JBxe|`J7#iPyh%PoyL z&Prom%8{Pz9&Rqr^HzDCXK@4`PQ>H+eYSayyROgU zs63A2b2}c!SFi2dQuTrAceK(59MK{DaxTGPcmy|kqTj(w zpYEI-exb3v{%HFjt8$?)wXf*~zHwRe9F-4kZ=Pew?>vI<{dpHY*Z%t(f3q>q+TVCz z>z}S}Y@X-vE?)Ff`*yUCFYvh^HqR}%{jttnQ{_wikiY++buPZteS52WTIa$yH|8)L z^Un5jQO?YdUu^xA&hKjceD&4l@2GN;_gm+`7g|4BA-YgIxroW4onB818b=R^18o! zkfY^Zxm-@YwdZ8HhYQF(@*oeAYj_UllArjPTq%e1qcc5s%f)iF{CsumatHU}YjQki zlS4U{JTE_;>OOgwV{ow(t@9E-#uXOpPuu=4_U-W1>vG@o|6<>1pQFOUrf!TQsRv>r zHlyOLu?6D3(INDVODRsR`x+nOH+{# zwT~Cby(^o4qCW1&9pxB#mOst4Z)^K_7a!yYJmPBoaw3o8jXaQRawq=W#%}m1A)nemv8DuETMjZhoq|(D|G}KIRH9wtlS2Yj~b~ z%zrrm*OQaE;U9V~ui;oToiCTm(>#P*@*v*9#V&R(pW|LUi7)W6$@;k*pW-K+m2dLp z%k5jLa-=nl`Py9Lb^TrZfHS_@Ja6YQe2I7Pm7VqT75>c2IriH6PFA0+^2Pfab1ojq z$!@mpcZCldzft8t-#6af-+8LawKx-x;cK(?-K{=YJ<)l*ZBt_&^-*J9#8vmTe{+>j z9<6_|db)KUb*1rQmCte1`9RP9{h4C*SRI==J<~fkOz$duMp87ZlcevJjOt#KV^1FQcdgpR2`9?mH^W_$KNq&f3NqEL*>QU*0~s8`J#Sa zv$OGr`d@6!J@_IQk<&RO*W+69EVtr-T!;hpmSC3{y-=UL&jFvUj~{Xy4#EfJ{zIL^DfrCG%`aBZ zbS}r{7hGaZ>kHNW?fYYudvIs2cd&JyztnzCaiZ~C^(|ES(9?~%Cii)!eJ5N0AC39! zW6g6p{>NcB7{B6;oas*ccX#emV{Un>`RAISX?(1)-#um;bEDJsaeto1k3OiM`|!6j z&HKH=?;`x3zkl8N{Ox>WzU_Ame(QIEE1l10IV=x-sdGNBzFFmLes5XdIgeMLuO6xY zo5pt=zuS1e>i3tmt#ei$_pi-!=N;|)Q}Z{fT=e6{e(&JS-?q>1Ps{%cXZ(fZ2F4AH z8yGh*ZeZNNxPfs4%jpK>2oB`?o$vULma{mKoFhMTqHTR2%Za|n~Ih-9C9ozLOicz=ypjkK%P4>Y>)ZhiafcedH%>dxWmJ(Z|NMlS8kS9 zIY#Xh+)fVVnewk3Czr~}yp8*CR{q9a_=tQd&&vHrdtdG9EIbIHH2f3C^}FLeH`Dlg@q zYntcFTxhm=KF00-Pv`Tdef3Q@f4=d-DhJ}J9OJXrdCKX=JZXDl{<)^}`54#a7@X&) z`urZie~&dkRpl&vnu{;A&L#N>_jsW9TJN03+=Z9C-~42K9B;DuU)-lLpS<1pjrMU& zzaw04exW{Y$9wqoztp!>-=@Z&H0E`jjQ9UzeLU>fzxsJ%sd>Kx9B90?^Z3@)<}X$+ zwa$Ne`C{u6t@F;uo9ByHTK~7I-$@!yaPL!1@WR85S9acS8n0^XccJeZ&sLYyx6Jrk z#tn=c7&kC(VBEmCfpG)l2A2N~$Zy<)XUJ7tWo_TX@}ykKQTT-%EbrgxJK1+NkCIo9 zcdnfIL}Sjx5BLU8;WP4?yv(!adMa4A>ZOSJo1~)lcVKluEZDl+kEdYC-J$pog>%s zPq|UP-ccV<;~(5ozTH{h#h%BVH~}x>RPyla_1~y+gu9LTj9f0)bGmCims`Btn8)mI z{A}mRySth{R+UrbYu?MnII!Hvk9h3C&f!bEjt6il`Ch)h*}gNqA4j~PQi(J@(-QI9~w@4*g5=-FUbG@uf`2kezLkT zH`>ye4_xis@0#CT{iyk~jX4<4;H5l(->s>iPdwe2|8gvz$e;JrcVExtvb=?t^S!0| zIPiOoIT*j@qdb-Kak$sp$JIF@Z~as6^>%&y_2cG`HovhkXXZxt`U@xH+}GQ`qWxTn z4{O$vpagO>At#7Sz;KjxdwU3)|D1OL)IPtpr`O@c|!xN`l z=ZXB7mvguI`X;Nqo-^^SpISdq<;y(t;pX|=-rk4*amqvWZEu}FFaIx`@fVI87&kC( zVBEmCfpG)l2F496ryDrk_bgZ9P;&eFzI(Zp{L5vy2JiA6%^}|Gd_Kl0jH#_TfNWm&f$)Hi5JM>d~vFMyyV{-^DsH@i{|B5-oriQZhrEwy$3J7)R;T*T^=J( z@9#PCygbQO8ZVMpPxc(%DsS^F9gO@s+G9R!{X`RsjkzU%TGqg{zt34?Qmmmx~=~C=3i{gb0!;e(+BF~{oIO6oo$_y?rXfa%8A!??%L|D*1v4b z1vfSSyXL2>+=_4U_fz%ru+@#XRM%DKJBO=1?flkxA-8(J_2u*}Gyaxw1LFq94U8KY zH!yBs+`zbj<$nVlhxhOkUd0vU$zy%@av>hYzc>fq;UheUC&-z+NWPcDIZ*mQ} zL>}MOdwo=u19_O7CNIkk9E=CZ-}0W^&tLd}JjS0kloRBWo4v>D&GRWfD9`X3-nYJe z{D)Va>l_~VjlZ+XM6Wj^Efi2jN{!PjD%YC4bAyavL|3hk2Ad&2M;x+{umP zVLl`O@+oe1xfkOhJV@^4Opo<auYPXiIdV7m;0p3S zKj02LRc@EZIV>0AFk5@>vsEt2fw=F-t>0h0ROO{#^!LfZyp1pO#;@w*HyrMR=6MOX z;~D(0wuzVfkCjyp^q+b!xqD6HJ@u_<%;9(yXFO6LM|!O>Z`s-SO6PDpp2mSV=$ZOC zGRKkUXFG>8@(8}M*#5Ki9causdCi^XAF2N0h3(tX{O-n_m#c71e$UCiZ{K^>pQ`6O zhv)L|spdHiXXAfdmybSEKUY4{_*QkQ=RH~FBpk=@7#xht@Y=W9$N9L`Li5+Fe2T+! zO&;X;1Xp83h&_p$NCPI=j1#tAusV2ej+b(D89f|IG9`}|M4$Q!I?M%|GM3G zx_m8%$+Iu^-kj}V=gaS$ zgR9AHa`#Lx!kIW4uX(-m!x1-JTMoz@JZua+bRe4>$q*>}~z^&XHd?Ha}D4bv%@3U2Pvv`KmET+1{8x%H1nEm*4T^ zL-lhqUM=@uZXaKg>$%LMt#bl%=-JNU+gyzcahd&{%Z>Of?|HoSSF2oy|2@?F$tvgJ%e-)F z@AE-@e{RhG_BDU8`upm=zV<@^x3=s`XXP@2Yab2USG-NWlOs8eT=!wm{h}(5^C6DFyKeV)@UXj$ z<)b}~<@S!9;0c_7PjLx3hZn5w96rRkIE)<4|Mu0-CFIfV&2x(Nz0jE|_mJx!Y5v73 zci@3Xo976VjgM40Bp2gC{BmROBd2m$|NcO(=eWn~=V()nxz|k3!8a;Yz%QgK{dadAjHC zsB$%q#YK4Gsh-O-ITGjQ&HRf?@kd_CU%u`ms{sQa=Bb7FYjtUPvu$s zge!3>KE{QndJfm=xQUzn2hV!AvHZzr`KO#NcYoTs@*OXd9C?xsvL|5UFkX8k|%Km-XS;hS`Nfd)_1L zp2s)eZ_G`34p;hp`#I)RWA6Q6`=4x{8=Y@{zIhI|wlRdE$T z;A4$>llIZ|^qG$9dHU&GV+yJ(rW6X`Ro1*7|>`UT>Wr&Nlw4%GLNV$6RdP z?-hqTk2i8OzaQ{1ZuorrPqlA3eano$W!%8HfpG)l2F4AH8yGh*ZeaP}fE>&R@g6!L`pDSvYy4#p|?0iTky`H)<-+xZ6LJxEk(=c7U!0+R-*x^I zRlf3S>pYK3@e)oVcXB!Td`suasdD7A%^$7uG496mxD~hHIPwsO;w^F^4|%-zmn(T6 zzqvQ>$wNHvzV`D(ZX*x!77oTUc?!3g?eF~O&Xe;wA=lv-a^tG{*K{7g&*-pF2Q>r?Y(%*WaHIUp1G^@-)x>+OgFyTIw#?J9GNqGUjGAC zF7jpjIV=~IyE*4?>*Hf58}kJDUjFAJoR=GNuG_r_XXXIhj7xCCz4h~V9>}M-4WHvb z{OMTlyVN<~8&|m<=jIUH>8JW$Z~wZ+9EH>IUrxNEJ|4z7c`BDV+qspAi|yk(TU+4h zT$A7P#S8WM9pe4Y;ZPij5Ak_kbEtk^wW;$qHGiY=mFiscJawUYF6Z}&ZT0>L*pM#Pj@K!k?~pE-#*M%vn!$ z4%g&Z{Pu9`ypd-u|1X^J7mgbkH!yBs+`zbjaRcK9#tkf|8#vl`?Kgc7FIDAm`9vO) zSLH(aXnp(S7x{rRaSARZulXK-y7!Qm9e1acv9lpj>*7Q8NnYZy3 zzQnmWgPba_$U${bLFbDozJ=COnI9(%Gr;#pC|GeuEl9SsE^0UsVjP(+Th;uL#@v$&$<>Ei=Y+h7x7^n`@-DyPct5nCLvS*={8H=NtGt(= z@ostia(#Dt?y>5l&GSS)wXJoowX-ol_}`5=&z{CSg@1D;4#F$>6i?aRdvLzrbsk6M zjGUS;?y8TwO*ZCaM;b3y@Ae$7&sVwN*48;P$KfvTx6a4-F2~?+e1vcDL$1d^xC$TP ztvrOoFZSHIs^2+y3g=$i-+8J&zl%K3*zX-L)OUaLTOLEh7t#e1-?DvY-TIX6^&hJInTVL1n zS69Dmp6mX;@oejUx7gBnIep8Fzh&IOxPfs4;|9hJj2jp?Fm7P^-vEc;CUTX$cd+kY zzQ6(Y_WjFgI1KNS$K@W5!=>b0UUIK@aSC2?uIC-i;)ZZKazCzP^W9wyiGr#IWr;lAlPe5U0Id3K?74#3xV&{FH$s`52Iln*%! zZ;^N9()I0^m*nr)o9C3=M^5I1JcDm>8TopuKa7Lj?A$ZW^9w#9hsm3Avi!(TH+2rL z;xXKhiyrDZ^7`h+|D}4c%CC5mJh`!T?#6e{^nCfBW8LeX+=jPuHo2F}@iuv!H}Mo+ z!XLRDS9+%R$PQ%aM?e9}Q>wHeZmAMD6yj|ZLFWNA2o;co?_gOdOy0ah}D_ z<3KNT&f4ZrH$GYAgS?14?yP^RzT=Jke!;=`C+}OVpHFcu9>%fw+kEHlY2WF_+?`u- zwAuQW{};~q3&#zN8yGh*ZeZNNxPfs4;|7+~4ahIP&*eKgmmA1Ia_P3df8|1XV_ow+ zi|g3e^!{qt3h#c|{h z`Hd6Fd2+gZ#Es-S`Ag1}qvb<+MoyOB<$SrF({Mk&z%S$#-u8BXk9@=9c+4BEPj(My zt8&^0t#c*0niKL)Iau!JCmcZz<8g8t=ixJ)l8=1S`#f6ZgmN)Qn`wPj@5`6`J3uZf z_i?Tj?Yr4NF2I@Oz)h{os|OqN4>^<<$;EQ|BlRDye%f=n@W#fwtDjf-lstc;c@DAE zc^vOt>#sHcL*vQDa`n57xyzyY_%h$)6kLOga*M6)<5g=K^8vY^(=GJeXY1n}KQ_-H zc$B=)v3Lg$m-Bf7-;nb+_Z%M0y}22mU8s*^Kh*p1J?_i_YMHdRH@+`i4qy4|BaH=Uakop2IgsBm_#Ovb-?<;xzp2XiuC>n7xFe_B)%s8ETU~v+ z%Gdv3Tt7#hZT@uiiR%BV@5SnD)$am)oJaCxzd!gL=ve#qb}m2WfPRnQ;#|=0GW`GB z&f#C2Y&m_)jK5{vz_@{N1LFq94U8KYH!yBs`QHF9kTc|0uE1UR4Ns73f>|V zU|;Ke@r(MVn&;P?m=~UCozL=N?#P)ZJ8wso|L_1V!b7*!_g&}n%IU^Djr-kdo-^_J zfA1W=!)G|%8SB-_&f(WDHReV9^IY@qRJqpv&fz!QV{3iLTojh|?p z&s=ZJdpMil0gkrsQk8rE*YX~< zpRKN`@=2c0lRs#EPu1@?^NqP7SN*K#@TKqTTig8mRSvbYvEKzaBTt-a|CajhR{8t# z|H2u6;kbcu1LFq94U8KYH!yBs+`w|W0pG{m!gu(+c}T94Kd<#&?7RMAV}8VcE!CydOkPcHS&dgyQ;o-tMWJRc&~ZBCI84rv)$8&_3;|c$EWy=JjsnX7Uz=7_@+EB zAIXa!^*p(2p)pV6d`(Yqz1zL-LoIA<%wyzR&h>Qb{6=o(U0i{;@ecW)C-71ZJK6K) zWX{QxxG2Bi9`Z9k;~3nES8%G$y}!KvFFkit^Kvx*<93`wKIH?vM@~Q6IgfQNH|7ps zxBh081IW{SgU8O*$2nFs=D9rXrRL|WeCCDTgS)-jn5Xb2-pj`?bPgxuHtXBZA$X`< z&tdoluRGelt(|+OF{kFCkG8(Eb$-AdIO8|1^Sm4FJK6l<#+-!X?`?iTBU-#V9fbF&xgi3Tg?Y~t0 zOa1(eC;z(n+9voY*W$9gbajm!*Y7$@o#XcizZcxs{I^x!%&WQjvDOzlf2rzs0Y1K* zzGcSWGHzhpz_@{N1LFq94U8KYH?aI~fCq63c}tGx3cQI6`Tpl3@+d#zKl1y{zNa~a z94)WQ-SR6h<1O;6QzuSxR5c&Rc>pNQKjPe+7`>}PNDM!hja;`kY z9r>iZFW<}CTuiQ%$EJFoJj^lVVGhT^I08rHR{VlL$Yq?3pU8(tx(`mvXZYY;>qo0{ zqg=btd%scT0X&fN@FPwk@5;m6hsW_=uDY-1bIzSTkE`)6?#dbXpuEbz_I4iE;7~kz zRp;}E8}-Za9E=}qZ=KWdIR7q$n{pC9!4rAwgPp@8coTm(+&VAj$H#i#E3F@F%r&_l z@8cny^H%$~5BHSspX)i0{tmc#MY>DJd(uU02}PUV;XpW$~+@Lf*FX*kv=^>I5c z$O*Y2x8UvE=qK;-tDpaa=Wv&8t#cfn_j%9dveS*Hs*6?5%2UoZ|8HGYpnl%QA!eHAIr2C!<;`*{KjVq=IhWy!JeNE0QI58w_u@C}8}~XB z7y6GMs#~jkbyNNPi#u~KuErfs)_=3g2{(4`-71H8wlTl-?@eBBo)5m*dAq7>Tffto z19MS6!Q(jDWc?Sbe1P+8=-fS3p1M%~>s7A78F>VM<{*DpKW~}s+-=QsNUrx$^Vh3C zR{fs9zxWVmJX#-L;UT<(tGrwvU**~FH9yz;ab51l?|Bff+}}RV%8&TyZ(HYN+>JYO z=417(==q$D`*7;}Tj!~Kb#?O`mb0zvTrRk`zIU6Su0CAlmwf7c{qwEgXv`5bp_pN*<56a7Onj9iO%dx(zc@0;QFXVjR^S+WHbJtW4)+ZOrdGgnn zt;=?h`#{efu~8XW&B|j`Q*x`F~yeFL$0i|5@Wh)iw3~xiLT5)R=?uluw)I z*Sz`t=AZ05zO}3I>DD(j=5q&|=P`?|`<>vy*7?fz#$1bYa$fGlrFi62`?w_!;#Pd= zllrE5kEJUA;r#pR<1Bod*G{+Zwdx1e!|mfLZ#2*KUuoU%0lfF|`Zz4V`^VVG5<1hOg%g=Jm&gSJ@J|Q2-M@#+v{6~)F4}5{|@tO6VFURo}`AGhdD>)C( z;%M^Pir$0o@H}~&2h7y>A9_A#lcVHWF3A(+Gx<@j;u3Q6h0f;!axDkrBl4|$$E!HW z(azxqyo59HFz(2c`I(>H>>l_W2jsNdT9@bEYb+P@8?H3hIv4z*=YQBd zZ{uj3fHR$_pFeU|{={ea2_NQ;_j&~{y_VYyU{8fFNf;(_lKFCvdIJfh7!bxZY1_0`TfQsticRTB7BA$XQ>|~VpA#=Ozpnae>zs(2EjItV`hIcW#{BeTW4||SYRvI? zDd*zioNQJ7k5s3toRFW*Ht*lNa?e+KPmaeEm(#b*_*=#ej2jp?Fm7Pnz_@{N1LFpk z{|#^`xx#lpSCAL^0x!68&-XNM;9c^STq?i%p7$NkUF2Q4X=msAE|;%(4@ck<@|%3l zC*(VMOD>SZxSX8HdANr>CEv&!au6ruaB}mzy}vv(*;wA;Ta)FAyH(!7$K+ft;otAb zBS$+|{yx$}mg0m6vdbndTRIfLz51-y!*jg9#m*Wn7>fOq(J9;e#JkG6F_-{3y- zEC-dN`5yP>KXT~@oyUJJ^t=t#wbf^<{EClpF!@zJ=Ds&OpGV2@Jck$lSRaQz)br$d z{>k4C*3Tz6;&aXO4!*O$d9M9v=bx^wY<;0Im)X=j7vZkF`$qjMs{EXTaL1FK!)G|u z`sU}_&pYJ)zqEd@$KI}=m+|Op_46EF&!PDsU*PB4JCE1$nOn{ORh6s!y65a`o=e`> zJn!LHpVfD@_2;XcoO6BCJm2Ng+;?^7d|MxPoM?V)mGhpdkFRfOoi840{pBiG^!vjv zzSuf{=fT`(x^sEhZ2kWI+NIXH^TEbH)yG%)HHWNi;=TGf{?QixzImQE-~0>Bzti|w zV{YYl1-~EdtdE~^LcYvPcXiHEb#MFlmfwYVEVo|%UpV7095*m-VBEmCfpG)l2F4AH z8(2;^AWv}>j>Uuci~P2u?_H|7=V+9yvOZOmJ^ z&8p_*cKL_9$>}^{XZ>=p{LL%)h1|vuqci(BA5AY;@^jPb>i5D#OT&}`t{JRbLnQLzAoUf{Kz8udl=375fJ<)Tx z92eutE9>K1+=Q3N-^V+L3ve$Ed!}=)Re9=E`*;Np;F$dClluPH{3ikCgn{BP^$M5mhPBmC?{eZTlvebdczw|{M(FZn%$e{v>X z(EZZ`#*gsa(mKlG`xPfs4;|9hJj2jp?Fm7Pn z!1BKVJ|SnH>^qmIa1S|IevqT(^*wzL%g?Jihrh^`YwFw7IzQl7{6!w;W;^TWL;ObW z$lLsdC&}r&U|sumR29y^Z8)Y}D?iKE{83)!eH@K@?(Y5hj=aoIPPhI_ zmEZ6sd0j4**LjuPC#T7SypPW=^oPp3Jn5(Ac^JQukGb9U-jB!0*$3LsUAXFZ^)FWC zaNcvXee$z>%`G{dynTP?@D_QPFUkKrl}BuRlN7^ZquU`JwS!ox_LsHs#lXH(KYuT=Jjm<1f6B%eHNT z=Y8J9WR<%;**SifIM#S=eMhTY>_}r?!ku`^RQo5ZT#3(cH9p2yxg^iwd_0?jec!np zeoODe)p*=g^Dozbxbe2e{FY1dB0j%VANRW6n5S|oZufELpQ!S#|Jj&#-D=E}_$jyL zqCD|#{cC$ZPvgR;TAy!yU*l=>Rlc_Tzi`H1IBsCvz_@{N1LFq94U8KYH?W*;z<2Jp zzIWwZ9_2fpYh392SFYqp{DzD0AUXI@=W_!2-}k&+$*Fw5^DTKv4)k3v_i+gML@wq< za-19`U&@Q}5$EAhSNeP96)wahGD@^gQ{8Tgaz;MSkZfGwoZd-t2$BcdL9*9^TaaH&uC>d&-AAL2j0}pF)U$cOT;oX)ZM4IliV{qiIKu45+n(0^#+eEUCa%qe*iKY6irzQ?7w_My(<%{+$J@#_E6Ih>!< z@Tse<^B+F_Y3Fl_?Tz^?r~m8r^G5#1r`Na6_uuav-ttiE9GR=XS>L8AUp~>8f2?f( zy5{}fz;C%6uX&)py;a+|Bp>F~kJj*;Du?H--0`W_w^aEsuj5%9j3@1`pKm|h-}6NC zbJbPN@2cKzowX8g_jd+fezN|J^>bbB|DW?$-hZ+F+4_0j>Bb*d{eHtG zS2xe=c`=Wg?c8Hk9`;K6{GP%$m(#b*_*=#ej2jp?Fm7Pnz_@{N1LFpk{|(4D3w`hU z{^m~?`tIc@bM2D@cniniNH^Qh zm*kgg&GQWoCbx67Jv~R><|{w7pIqQ^Bit|sr?*IKIfXehjYn?zc@$xdBsxaa*YG6b5MEx za(&#BL&>*t_}2QkpB&BG_#_YbOV8sse1YFS(|KHhhw=cv!+ALlzvc_?_MCsJ%F*&Z z|Ku#3h9`5Cwoh=qeZAkh7N#1@^E`qV@ocW}VEsIYPs#uMlY{V~#rE-et})en@Gg$E zzj>~7sPS~?9IU?8{A6Q3a;hZkNq`k^Nm+mzpFmde%{Hw*0%mc>zs~P z@pgXf_lYg-z>wCc^UWMB=X+2`i@lPV6GvL$fF#AA8;Wa@OtmT1LS-;@ZLPd z+qjk7&NXH_kE0yyJ>|@X6MT%D$c^$YFO=(_>RkEkW(RRXz9vWWxcBPgmWMir+sUzV zp}hIq_VGua#Tj`lXXF69bZ^h$0XsU6w{S*Y!B2R^NA>^EIdbS%&GQqUGu=F&Ir1nh)?GUd>l{1b>nHx#2?3<$c_g^F7fz z_v93to980(^{37AVE*)W@6YMZ)yH{mH~)6 zJl&YH@-t5KX6xK>W8+V%cdLt49?d5>(ec)K4Yzo!_ve8;jVu1x`quXIKfb~1{Jy|1 z`Q-m>-~P_wDlat8Z8;S8;i~*-vi*nKx2iD@J<gwuTm4o|Tf&==Ucjj@MkRPtN37yi5Mz0P?T=&6juxC*ebKD&Kgh=gIYQsr=1>ZuIx^ z1-V6Dl|SV-ZX&13hw|Tl?76&$PjNf$A!qX&PR1qHcg~jHk2i5WKD4bq{=q#s+pDcF zbdFs3QS-b<9^YCYm*hus9G~E*kJkTF=gHG@A(xVy*VgafVel~CvZ-@;>(0iP+qb5% zoP4hz@Tj@^cmNOKrM#DeuB`v{o+Gy}H2+Uk9(bfa4k?G<>psua$0_9ZspjSTzwUf) zAh&0$D3=+4>>kB=U|8H`bvo~!<#uljv`oRdRwq*L}+XY1eG_H2h|C?-{D{{w)_AgdvtHGGYN zFAvLE@~6Ciw)5Yq%0Vyo{&F9u;c)UEkJ;FMIi9C*32t_?b9v5mV|ncR#&S6?m{w>*REv$1y%_Ki}KgemU@Se}D75g^Mnf1)~5*IeM zmbKrH?WXHnX#dOAmzw{idbE0`^|z}0 zg#U8h_v`0soN~T-uE+a0&hI;y@9|PTy1M;$tKU@@tGn9&MD_nQ&)NUI`4_6(hBx{> zU|s7x$nO&`cwXx~iR<~@U`u_k)_1<~n#TL9v(0lYj?L*g|Ks(qt)HLnZamvM_d3~_ zUvsvD_48mp$>(?~U*nR?>04&}E#n5p4U8KYH!yBs+`zbjaRbZ$2Dl67;Y0G894Rkw z6%NFexB^e$16+kq@hF}ncku!~CRfVYyh8qzWBD3~lIJ;*yvP^i6>h=ZI1xwU5_0g3 z@_`)AFL;Oiezv};s{F~r_ys5Xp+0$EzK~DlOSw}H=O}XJq0ZrAT#^%V5;^W_IY=&( zM|q$e%**6J`IC#ty}X7?$eEw@Uh**ahqoRO!K^k>)z`tSL)-4Je5Ck25xn`{&Q81v9{;VRX=T= z19Qc@&HvvjU*>VV>%XRgeA%5K6@ejFOe&c_Swr^eM@CNx?KD<~TKak@& znjFl{Pu_mBKHe+8 z@{ak|xzW1Do2$HmBQ5qm9FL3atDk?$-|Oq&+q%5XgZX1^6MTwu$@9F2bMvNe+Q^|k z?l~NfGx8`t#B2B}KY6e7-l)p|JnXsVc>y=!o>yDv75toMaVd^*qxa=-ypx0T!87$e zUFA4Di8t}c)AjKqo`0eFHP!vSFZbXn$D03t_3^>KYJ9r&qm7pu->KeLU22^p&Nt6V zHnh%LcC~(~dA{#=k@L;ptdH-{G|y%D_e1r))cW&{e_iG6T<`w+PF1-(C-gf3x0>yo zzo{Oo^5aJu``v^0ZEOD9Dwo>S_=)PjRC&@=i0n_w#X1c(T41t7od5jc*@p zex~}Tp2K_JX}p}iWyaq!ZeZNNxPfs4;|9hJj2jp?u>5a8&X$AZPI*PHmYd`+-^n-o zK9ZBz5|f1<^MCNX#(akp$>%&se&iIKiTBF&H#)atCvG?9GhCFv@~|iB zvzd9ypaR)0KUZUxRabMr?2mQ)>h}Le_y@Q`|tst z$r0sxzRAB%w{L5eFRtp`z0LDJuFWsyd(Od~9&i7zfA#Z!Om)uXDnI37Jev>jFRsfQ zj&&|S;GXBD|x$&0vP4*nF#7(v}&(*jIXIaxaSK=N|HqR^nThHN+ zCtK&{eD4qSU2UD?{HFQyRqpmdeV;YYgT8LOu62IOi9T=r;nqKH{QounsWGSGVm#|s z>-=iAF%R>*iQijz>D~I?=s8@Er@h;}-z6?Ko@^hd=8w~@f7kkJjpwVpZnp8M`mQ(r zzRHbxH;3bY%l`{!{DtEN#tn=c7&kC(VBEmCfpG)N=?3ofoh!%5=klvO?)#mSa3gt- zv&elb`kwY(&o$%^z9N@#gN^OuBXSbglE3A8xsS8)4373p=Uu4EpFBXm+1xq@k(>F` zLifbgb~fe@+)193Gvq~H#aHCrGd-8f$ZOnBPUdJli8FCJd5sUrNxbmso+Dp#mC61N zp20;H>X%E8*0-%HkMWVE`uWDq`qo!@$A&pB%||D+Zyxo}PUxGI->t3J-c4S1a#d#pabGTnJ!HGi+;@i1P@L-_!2mAg3`r#asF zyn#y|?0NDn&yu@&#H;mjY2NT$^PG%Ba-fzcI521WwgrB|ulUO^E?FOE;5nR--*8LL z_h^6bOI2R;Y2*1S4?5SF3$N_Cay?(->@)SBt*)(~+i<$~Tfbf9XB>EK^ZbRY@R0{v z=S94M*4eALrtyoQj+NzP>Fzcecv^_!9@`@SN+#_8)8?AH3H5-&8s3 zY~$thEi?X>aRcK9#tn=c7&kC(VBEmCf#rV#yoV3@UYFasfn3Ie_z=h96Wm3fmE+|m z-XXX1gdKfP|Iqh0*ICh+r|>qpk)O$JTuPpiPvzb*Ef=0>T@IF$AlE+IdvQ$;#*_G{TqzfGuch{LNx4rh z=LGwE{ufoQ#R)cd4tL>QawC_0us-g^-FOTqJzC$&p0}s^VRc)Tb8<{>$9?!E_c`9V zTy?537vKz>Oit$_@-%PZ8*`n*rMTI?<~a?Q<^gg(|2@>-C1;;*|N7=R5-;H>ypvnK z(>`v?1LS>9daQFFu5vwI&td*k=W!FxwyAymVzMzuJKsLO_um_H%=;U2w#CL=Wxn%x z7$4;-+>8%?*8ZQ`$G`aaT=RVMaQ$40H|}npxACFN&2t$J%SpDkUYYQF0uO(!=ku_8 zUFT98pRA9s@*W<>UB0hxU3IF;b2#CDZ2pbv2UUK=hqm_o1FiFS&c?qd>*Hg5m>=^v zPR5I_b}k>g<)W%wkBi>xRgbiv+i}V1<~i7v&f{yR8n3F4t6y(CTV4KNIO8uIH!yBs z+`zbjaRcK9#tn=cSWY(}pRDV9R*shsxeO1G=j9*yOJ0|o^{MKn{>M~ zk6a^X@eMiMzZ2mN@&;GnCi30gzU$>e`B|QkJNTWPC{M}3at>eMG+c^D$fa_myt2^W z$8+Qs{=ymL5}t6ReR8^7DL2WTa-7^K*UEp~LH_2Kyp8w0-1FovUbwe=l>g)=z9Qdp zBl(^?oohdrknhem&qdcZ=7fBYtH^2cHy`3ae0FDlhdjssc&(g#qvxHhpC`zn{9#Ay zJeJpd)I2}odEYcIKXcag&41E!`6I{WDrf5Bz`U$&6a4SNCZ?;rh$F0Mo#iJ>PTq_6v=B+yw98Ra|UW4X5jWrpi_Cw9YO5(wG}^wvFw3 zu62%murY_}#I0aan%8w)f<#{OYOp z9cX>A%4>McrTUInc|TV>(>#yjiA$aLr`Bg1e_OrXJRe@um=p6V?&NonFWdi4{hO;i z)9)#7*Ed~VsPAUurN+GK#peB9u%UIo+wjTz>ht^1uNxn$9&VrCBbL**%=lZz4U8KY zH!yBs+`zbjaRcK9mj4a-zW06ZdtMHe2jnxk{oe0qt|4d1V;o7Ye5vneIbAO1OmZ8) zkiWL|{mpM~HkP;LH~IQ(&ymw_H0EktMDCXR`G8#RJ74aVKX&$f`Ibk>$?}K1@K*hD zFSk0^d(BkkW6mId$TwVqyYMo(YoX`x5)SfH^GA9P2a@l(Cay<`| zKM%D}j^-3iPsn#)_dE_H*M8o-{Kye`ik!yX3h6xB|c94t$OC z@CBaBF(*5ZPjTQi&2u%rC)aY5XZw5QYfi@f_?8^Zv-pl&%K_zGp1~2h=H;Hpr~P}G zeZ4Q2JzgJw8ZWfIr^>Y+Zp^PZ_LI%?4}Qp{ zziyo;-E|Mue{7wX9&Mht?QYBq_BQ4`k2dCDe1~gr6aI9v=S{Vb&u~0$&yV;opZlQw zTiSoH`upZN$(82)PB7K_YgPVtzP{_tbG)t1|GCO>CL5oqKHomR#Gg3N>H2uq>-9fW z-BtBF$oGwJw~w3gHy-+3>)Wd<+voQYzYkn$o&T?@Z+(^HO*i&C#{=!-bgLWBRJ8$`a;f%j< z+`zbjaRcK9#tn=c7&kC(U^(5uj=p1g24DK7`3rpja}RDJ_i`qV!7I4JjrPlVJc?Iw z7j7Uga-=ow=QJFRZ_KoQrpldWdVk)u(3orR6u)0{x3itYU-%MNkxO~q>-FF4e7+=K zu5F!P@di#PPxB%^Bj0l_PA8{7)43O`T#pBFFwVl)dd-QW{m1_5t5x}$FRti(`JE4a zQ9noGPn?oN$ZsFk&qp}o51r3%HZ z-#oYD6+D_Vaq2 zIoo7?ocLJlT#fggs_#Ta zU*t1Bz^gdHh5il>vZMF>CC9hUKe&|@)I#sSqQ0Y5xp{44&h=CMJV}0)GkFj<*w}uaAn(hcM_T6ryh7gRNphily}tM0 zZJQd;cb=Rp_pfRG^Xkd^xeaIJUc2jCs`5xK!HM_`FW?hgl&i?YJh8`5@Hsh~uiogn za{HCW>)Ob9W}4?^T#}1%h%eeFkDh5CpX7}EhO5fwT;c7`;T~Ui4$tCb+(-WBTk=1L zdb<5Q?Pllm23~uvzC*2Z%<1Mi4bS3DoNJ*zzQ%tzv>eYBI2-?N`^1kupXYql_+M+_ zI&;l)K+efo=3D2^ON}|t{>Ge`Z-3HrxDDs|L-XgWx2l|hFHJW8PL+pl>vr3wg|Q^&PCdbf3(UG z|EB#Mm9uZCkIx*dkAvOoP4{~B`}K1=4tA<@CtK%2ymL$Id}~*ITbut=m6uMp&MA2m zPxL#%xApNhzR0_-w$9V|)TN&HV(a{Kb@M!smo5JBZ_ZRnEg3&+>^Q{RhvG z_oh0B8{KWpuQ-$3uDSNW5da1!~7cU|TpYx{fTt)rd8<9LJI$W?aM$0got%pK(3%bhQ0$$^}M zZ^&i*jyG`--XhQO)z>?pJMmb~$sIY%o!(Q9ztDbe$49Qz$9d&(xt1d>)VHQ{%TYu%F%N9yY=xHzQl#OlAOvLUTxpz-iH%%H~E>T$@$z-F6Nn>gOl+lE_%J^ zak&#cm+$h@!_7Zi{k!(@qM62Aj_2I#d^7Fm7Cilx=J$6FSKZqDW6hteeo)=mI)CA^ zJeG@oUq9dBmPgyqU3NF-y*%mu<~bR^<22lGy613~742JWp3BWO{&VY3Rc}|h-P`r; zseW1Iq@UH#k@?=u`p&n`+xYeN=6N1B`KWy>s}I!2A$i@t=C4#=uAifEE1tPf-%|Cy z`VUljD`(|$2konmK`P{4a;f$^3~g$T#wW?|9$i+=B=B{+D0)gWN1X?d(0|VflAk z^Z&cb#rOk<;tBGJ94AjM_Wds}$#L>8FOs|D5_yT+@gjLbe&7XsNsg1lvj&zQzst4*%jVoJEdY(>=y2zTP7e28Cs z*TKAXYhy0Ni}=Kr)_3$A9?VU70bk*(ypZQ{9Degs=W;#%%z-xbyd(83RIfG9AGj#T zIMzCMRCyX#y-^?6=iOX~`+Z&C&CYqFdZPK6 zDnCBgn6pjR|8Bh5- zA8!140EiApUBhl^3I+o?{gY?QNH0#JdD@K-@7_T zUg1g$&C73mjSF1qJ>_3caLl3zI-7rN7Xo$WdD=%bBaZk-cw zKK`|>^#|&g|K*>Rt_>deYU-P18>z}Xk1Rim;_u-`S zubeH1^C`KRk8m>1!u{lZj>_41lKjlixEODo>b>MbUdK-+`g_*bCs%V{`IQUTK5@DK z;F$d3h0ftG{NmI4`O9QuuJu#0L)yku?jypaF!g-xx?)Au*#$Zz$$?aj;k zyz$G{IXM5hRX>O3(>I#uV7!xCUGKcBRc^%}c^|)%-!FFVH|?8hEbkv`%pEp1=09AA z@9_?v#8r3$=jG}TcHUx@8*>Y8#x`<4Rw5K0ljn-+!%6SGgO1=1C`8=bv2n zSo3>3pQCXpuE@2xG{55G{ER<7-?_`_TW0(%;|9hJj2jp?Fm7Pnz_@{N1Izyg zFUa}wobO@Z&s;&?k~g`NTqj@1IdYSHCcnxfoIuWzOYii(%@cT$oFhNT1M(M#<18IB zA-Bm*a+Vw*|88poZ{i_5faCBYzICCz!6W3KU-C}p9;|*e^ zyV;mGY--FMpQ(>iOt#KFIHFw5iQcMDev{ATV!2oT0;UGDj3I)}ULY`nd7d3H@>IhmvJ*r+r{7UQGpU-ih z2U=g%{!@*44!3@x^%Jf0p(mT4Y<_i>mmF`**Jc|Zt@35RNAPv7&0+Z%C-%Dl@8$XY zly`BYtv&D8zxw%w_w?R;e0S>~H2?c5&*qc6nqOb#Qt!9FGQl4Y)aUn?TQzXpR~jFz zK3+fX<*rN3Z>f)S%{TTt&dU0D7(bnDerNs5{|jgQh2sXs4U8KYH!yBs+`zbjaRbZg z2DpM8C5OtHa;3cGJ6S&AAo8y7;<>($|MMhXAP32( za)8{!#rAcM+#s*Z6LLDI^4%|w@&FDXx5_i}F8`8$xDo%6!?>OtE)UD8kCiv%MGhd> zp6l;%=X2OKJ%@wwP7cU_dD5Qxx!&f+9F;HeD1Ob=cJ{v9fnz+~ zIoqq;g)d%eezE;L=cC3vf|KxFt|^c6+CO#P8|~+7T=BK$`O0&R_qLC_b1N=5SKrq9 zxa|4HXR17%zw@KdTj!!2=z8b!4$i=#cqO;sc6^S%Ug&)O$g?;kcl_I&hO^&o-`Bkd z|9HMJ@A;rHkKxd#+RvYO82{PW`jsjN{I2m-=W@@-8}ldb{%!Mqzj(LtPgTDMtZ%%d z%BfB^o~(Y|^G`J9k3Tl{I}9(o*gQvkv;DiPJoCZEr>ng5aRcK9#tn=c7&kC(VBEmCf#rV# za_7;$i~GEt;32o0=Q!WgAQ$r>-oy#yJb9iYaTxC8yPZF91@7gwxEOzskGPmz|7aVy zid-u1U-Jjn$Ei3HpWrwAh~x1PIgB@O6Mn;q=6eor;$U)y9M4bqk$ii+{k%!u;Y0jG z9-Zs&lec(*Jj+wK1Q(JUSM)r&noG&Q$2xy=eLQV_^Ku^#;}RT$ck&>PC^z#RIhlj- z9Im>v=Wz+n#yj|je9Ren10R#~p6T!5KX*Ex8_2m|HP2aZwU4{;0zM_T@*-|4hs(b_ zgv)O0IXq3SzT5M-m0Zg?_!2kaX54i{`+4(Rr(T)>vx^Y!+0Uro^1U>`<|+D zfUB+Zika3q9v}N(^>LV^jn`Ipw~y=anGag$Qy1$ySbec|{=y0QH@D$6+}!T~T#EPc zO?BS&0YKR-QE9}nYzJe7CyyRGf#Q>zv=lk8_+xoXO|NF+v{|jgQh2sXs4U8KYH!yBs+`zbjaRbZg2EOS#mLu^D{=u<0 z4&S-mckt=z`6@@?Eb_8kvDouCg}ljQI0-k9pX51tkiQ)3Jg&tB);G`37SE`G1?wI){fA95}Ck!QFMPvVDMhCA?sOFf^Ta4Zfd z$8tP5{95n9msT~F^W-q@#Mw9y*W()ekXOjFa=9EXcX0ywnV;~a)4h*;$$@q@&x!aP z=UePPV1E1E~GA~9%vs= z<8Pdm*YK9Z_4AFdJC7rAZ2q{j{%!T~7Y@fumRdhrKj-5?{EhGPMn1v^dFp)UoTxtC z`MiV&^45QE{apQDHs(H)&2z5l#_v=)&fUfwi7RpDnbzk!|7P`U^HYrvR(ap8*7r2e z<2X7unyqhBeSRn5f3G(GeU-yK-k2Az?VPpEU#)V;)t>XKpBMNVFS}kpcRJ9RbMeae zo1d+6F>W~B{Fa`#oW5no-!g7s+`zbjaRcK9#tn=c7&ox|Z$M6z>*Zcfz$fH{dvm({ zCLi7CyI78uKjolr`hMm+Jd6|gu9v&y8F}Pp`+xC=#@vn1$iZ@)JSd0A(|nAx$ieas zUy*M(&7Gbrr}8ZxCYQ)dm%11EbzA%8O7#+?K)-^}V@`0U zgSezT$|bl8|M;N&oP*EsK_2#ceR3wxU+P3xRQe&(D{ zw9db{248!#bvcvE^?DN=fzR+yJ|?%H@0^d?D5uKln_7Ri%73`!isrwr-s=f+Cs&b| zGM9!F%;_u9c0shkX84^L%r%@s-Y#tG{iYSFUd?@AE~u zo_qgP9}nW%+~RW2;X{0Ds(HS^V-o9mnK?Jp54S@xK4iIXuh1Z~39VebtBR z`)zfh`a<4%)Kj($q_w)L1SNRxc<-6B<-e>hqHRdAIjd!=cxyq|K z_aB<)>d&{2bMmOyn*VeC>l*Vmj>ca&4tJSr--+tS)$5((_XZxey>(uAvA)GBpPg;Y z16MWXWdGcFPnGL!?0g>27w4Pj^Stg<^AC59-v#&)XXRuZ^>q6Vw2zlAHqSHfHs*l0 z8gnndhj2G8`AU7u{|jgQh2sXs4U8KYH!yBs+`zbjaRbZg2IM!M!G(B%?|M0jW61v; zfxpNNa&>b>X(D~7boBdd;0tK*UwY<5Z5?cpIpk#O0l@t9}>Y%be_k`9q$RcX$&Q;4tz7$MAhFzsTWSL++L*pnRZU%A;l=liMgLi?Ypa#x=CN%L~AJS<l~AF@C!b{U1vI&d+cuibItR)*BbL9uE{}p?__=Q`i;)p(frZI zJmAsBe0NiQGtKi|PVi*w9Oh8tKiB_h^@-}ns+`Y*_&a~%Ag|QVb+|RR;mLfI%l`UT zKfee!;n8pOew^Z37P*58!hA=g0c4R1enwP~%*vuHuie~_dnP?hkK;)`Sz`E%LaZeZNNxPfs4;|9hJj2jp?u$*o{zL&@4LHUJ;$mMd+ zV&BJnVxjNi-OcZ8%o)}-<}RFpC#-4TiYf>5z0M=}k6a<2%H47X_mW%J_k4aMuk*Xf z);Smlkh|p|xl=BeGx!Q8;xKaA*8VPeUvB0cao^O4Thls^;wM~k zf9o8E!*NN@!bdnBC*ufmIzPJH-zT5#=sEmMzUD1*ITw^4xgO8sOdL>tm8a#%FM8fo z@4;s{%Y5tdq}=;h^E{SMa9O@6cXM9u$1%8v{L8gC2+xsMISa4if*fnEzk^4-+#wEwr&rYGcgPIG??E9>K3+?3~W=i~Lu-F%q4oo$^fUu-O2^Josj z7vAdmH|poJ9O+u?cdGI|=a%a^0q5q0-*xU2RgTFaw{`Bnw!W$HOk=KkviY~GJZ4v8 z($QPQ{{jh^9 z^2_$VkNK0FC};bwm*+VaXW~C{zr4+LI0~PV_j~LF=aJ9kZ*C*6ax%Ul7t8$|LOzm5 zb2-yPjpZJmAdhn_ z&c$`)XP$JZbAEBGU;R9>wfDJPl@sN&$@Xz4UdHJ->4x@itnv^p!5`&mIZY1cFmfG- z;v_s#zUL#Hk1MR}eK`fE;!<+z(fW8Lhv69?_jhof3yr_7@;DxIyLow8PUlsejW_U9 zJ|V9^;=MYLYrWZcclGlsN0FD0wSQH0qCRfH8Mzuy<|^CT$F+WOna-K2@@#p2p?wFc zJeG&?W6pNv|7YLr!*{Ol|NnPaqbMSF{77L_q;k=J#4MJJd<-A+ITB4J+V79g?baXXaqOS!-Eq86_x*F<;`KVQE|-@J%CY6G^5R+SlMl!rXqU(} zAdg)_Ef1_mJs6ey%5~(w@*g?0e02oxUw$VSlCO?oUEVRDS{}QY`VcBl{2uQ| z9@dQdL9{C>AC~jVh2$#o6uHn)US|*c<)d<4`Mlh30CV!#-PAtU=s+!Rl^Z=u-{%tx zseL{$koR$rzWhrb>GK8or(Et3^OaG#V{Oi>gUa0=rk0<{spOrs_MNPHQ$*_!pUb+r-QNLy znOAuqa+Rsn<|*@gDd))FdNL;;Gsl`A&6o0nVnSAhK zURN%*mfGAV&oK|1Yvm^LM!B2(OAa8vFmKB%<$`ih^Sil`w_PH?l#|JGSMFeDV&tq+F;T=a`4hxtG$P$o>V?aw9pMd|Eytza7H9N$fMne?%?Ul#9u2 z%-!Z@dD;Z_$wlN%au+#l8+o+sOo**ATz~|n= zTqpDp+Ku(@sN72)Fo3@NsgzpYRf$@T@-xnzh{^%)VNO0Gx0HX*XI+limRi0gH?70D z^4SlVlXv?3KrSN3TFl%!RL(PpT7J}wT3#$amebc}U5?e1x-lw$k_XH8l@qjE}l(W#s(=aMtYm*nDd(tXUI%XxCHm+9Y*%2VZs zbD7`H`cvq1`f~R>>C0j3v)+-uTu*K%kCMk#V_ptdfqn9;QPjnBm&s>|192b@#DO>v z2jV~+D1HazNnCeJGYeH6NOj2XoF_s5#P{DL;`X$b)7u zZ(cW#%WceSU6_-D$j{8r=3#kQA^+;ke)*5wt1W%GqFlqgE3cF9O!xD%??GyLklaUJ zC?_;8%4g*=@&Ngl{K~v9w~`;pA9wKj@=1AuIa?l7i8;B8c~rh=j_$zcn2XBgC5{DQp-6?spVmEt2VrjJZ&#?a*a~f@1ZZZlHD%$$dN`dCm$L{-4h+kez~~3Y76Uf zE_tO~Nlqv~{2B9dCHdL^(3j`Vqwa*t2jya`=r=+i=KaaRJX7k@v_L-N=yXJgzio8K?W8O38%TspqI`Ri|xVgYwZ;m$) zo4@6E<^}nQ+{D}=M=+PR0nqC?Ef!Q`6efBDu7KF1*D&GWUX z<(-AR#XK!1G4INqyRyFvx*au_n|qhgm;cDg}%}nU_b& zk>qXi*2>Ju%jE=B>C0t$^Ewx!atnFP8v1fc`Hwuk8RyA8<+E}Z`4eL$a+D7^S6;P; z0lBtZR$ll3^UY9s$0GL0d*xt5nUizPq279yR}) zx6Q}$6M2^T`OEt=_iFizInsPC50J~$W?sG`$C4k(6XXwtd`o^|elVBGBjhadEV+|8 z)O=|^mYbMcuI7H<0X5f|L%Q((%|+%{^NBfAo*<8Vi}U0al{iOUV_uUVnWxRcay7Y} z`Ac3Q7yF#ok)z2e&6je7Qs(55@+3KzyitB6=aQ$G`%C#8awPMpc}(spr!!xhCkL`$ z{%0QjGI#Sn&2j#oZXSKPgq(wRi9F9-EMJl5nOn`-@}`eCz`Q7Lk{imi}$*U@}*tW@|e2J$$RBQa;o0U%RNR=%cJFI@(VedywRL4?~*Udh4ymZExZr;K^tnh zlsruyBL9&~$XVvHPd+clkTXr^^~Rusn3qS$3FX*w=0nVFLFckx9yXO)?k*pa-!)}j zE+(IoM-5|Lp7S$aM-C+yk%QgCyd1Lv^V87>QF&7*=8+Qlrd;b>7Ua}-vR@82j`cqD zA@?%(nA6N{ z@*Z@9A7d2%xInEXP1Vvdy$$vxy$TX}tRlsv)QK8)9q!8eR>5{^yPC^sm;BIct7$i^YJM9a)I{j zlat5=8ZjpyGj9*3FNcAO(N|K*7CLHYeb=Hz zU%q=5{X?kSPM#;v@_ENY?3cgD3*{+t-~P6^>U_3|dUl=;eB@4el8CI>R# zm}}%*gLobDlKI-)D*rKmnJeUFatw37yu%zNUy&=xdCceLKy$SD*?c1JFo(*Mk z?ZiIwx_nJ;Aa8n`IeDRcrZRoGmHAp8W?rqsoc!Ss^#R_8JfbbNIbOaY_mtx>RwD14 zOhc|^o|M&b`Y9`bK<@;$t++@}V$xmlj}Ij`TG^$OH-tdZ36YWe4M=H=>*spSB2kxKOC zp*^VOX>x~;cpZ6TGwN#0%eCcTL+Q&C?qRk;BQ4v2jV~+hy!t;m>iJfm=nz}=05YGxlS&#nR~E&MouKRDrJ2p_h)&KIpRh7 z=6my;oMsUF&G$W-Z$rP3`^b6ZTJoYT%$xt^CFT+HkNH(zA#jxRC~^Tgg!ykT zbG3P0dD>L^=5_gmyhKi9E;UEV0nFo9bFTbAUSuwk)5sr6d0qLDd09^JWzON;38=YO z&M1#E&zhsmr{;I_lDtH|BFB(t$zSAV=5O=YSl)ME)O=>Hmk*eK<&1}zZ-p-5^T;ja zjBl}CkM*T!XZmtFdD+GE{nt8%V3Cts0E$@{Ca|90lOQOn!MP|LC8gmN}HuUxVk^Q$;d?k8{hfc2+Xmn+LZr?WmD zU5Cnv+fvJc<@0iTqf;HH`)NuY8MR zO6K!_nxk^XPVC!*%J<}Djp$E8eLm2PbL4mJsTZU2xs~kmIfy)0o^(9x^2Pv2a4YTbDjCvd$9Lz@9E}t z@9E|a^PTy{oFGq-*Hq=+>^?sW6m7%)Rm} zbB^4=T)vFgHTRg`W^ry8*3GSQl!^4^LFVDx*|&>z^N>06e^{4`m?PzYb6GbJ*W?^I zj-2Tu=FIK#x~a^YJLP-wExF5E%$YyU`ErxbIN#hS_cTA67e{c;OjHhH4wVOtXI(BJ zhmiNmgU#u3JNbxwuOshEekb=a*PqPmn0MtCa%4GQl(tIN4^b@_|@>NM83prbiYt|h;zO!`DWA~QV*b(+sd=ArZ4Z7%gO)cg!0-#ZaRYf@~A_+-U|A1Z8>Kt{XNXd z2k)Vl&&p%>)0g|oAN~F5Ue-HtuAFQFwOm&I+LQS`tjl5L(qmbd7y3NH=Q;A$am>kg z+jHK}&~sTIjMhPYZqu2%5A(yR<+Bgcmv>!3-31+o`h1}_^%&0axsm+O=Mu$#!^tO% z192b@#DO>v2jV~+C?*HYapr3Cid@4yW{x%o$`j;6-jB`y<}7oR`BE++Px1b5PBB-R zXXHBOC-3X#6uF8V%p5K!G6$K<%;DzwgWU7wF6J+DzTCh(Be&blz7NO;=0&%gw{jl1}{;|70lmqAM>VpTOJ{AkcY}Y z=ow&0u~jYA(H)dITz0Tt_WueT#ZJD*rQ2%XP-FF0YlZns?=K zr{a&iR$e7flK)KPJh`yEq%nQ-u)ItTVE#SCTw7E=Azzlq%2(tQa!Wb)diKjF%>BLS zPv`T%dCu{yk45FIa!Wa( z&p-Mzw;h!?`W%9nEg8@Mna11#RGuXFm5a(BtFh1L21BWvp;OVq=P2^3QM``4R=zfb zzI?1X=PX9$K=Y~PfDf`y{wjx*>&vO+XYwUEe{arPgI47{xs=Z#xnX}|yat(8-Ih12cSmzcZFsdCd%%*m6?$CuKdhstT?GH=tD2g|$U zn-}vw>!Wg5`GQm^J ze>t<SGt{#NXlPnh@RW%Bfe%*%=73|r{SJ3CU#>qbz^f8@XBb@__iOrE@#_w_&Q zm+!Tqp3b^_Ro?gk{e#TOIR{b8Bi1q}UphoBciccNXOqia%{g+wG0g2j>$1KItwdkW z;d6n`^yMlenU~Ya?R-9QEb}L$NQsTD&*gwpv?nT8k*AMiUcNMz zxjLx)R~|Q)c{#RxUB3DV>vC`TuY9?h=QAh2mgn8W`g~Ll--mg*`9RkD(f7H*c>3~v z`L5hb{w3EP%6uo(=O#YqnaKHjP`T+`<_FN12g+;Z&^~99M;8ALC!a75#DO>v2jV~+ zhy!t;m>lrFYyR%Y{a5Z_jx!gTKfEuScfG&MALJ_LAUT%Y!5p}S*OP;p-^>kimcHE2 z&4K1yb68{6krH#HoXead|C0a6W#m;KaDZIFJR(mpugNDq;r*LS zpFG2yEiaQ-ncL)l<{EQ;8~sB^p>i6z-3VUS{3BqCkN=wyd0t~`<9||xkJ?E{HpAeTgY){&|invW1rl2616-@z9XNK zx5<8m#z>_eP0j@OZA{fvGyR6f;(x(V~AQFlP)JbPJ}6UtfT^6OcbpUAhG)0e-< zd*sgY?bXb!W~f95B1ru?Q3^X4!4k32`NC0{ddn{&)r=J9)Yzve4BhCD#7B8M;!nn&e?%Q#1V zBWGAlUrr=HlM}UKU2gDY9;PqvGvCWc%zttL^Q(DR4j?x%N1Chd#7Fb_%hd9yiPY11 zA97RqlHA1n`!nXvp>k0<*;dY3#hkoPULaqp#+-b~d@d)Fm-J-L+$s-|Yqa5Y_2eq%YV&L**5w#-5xJjyPyW)D`L(D#Pkv!umqSe8eH~z4 zZa0K_0PFHh^R;}y{4Mu8mwnw)`HY-HPATt_S8m~TTzSc^>m0FehJ-i8ODaXtdF3UpU^Im6Lp{=Z<4RbAAOFomvf$Cej(~}gQl$eyrd3w z4fImxv z2jV~+hy!t;m>e*Fny0+)o3qSS-kZJeo6F@T@)NGhCFXncytzsqVjd_aFY{UAKpcnz zaUc%FfjAHc;y@gT192b@#DO>v2fld+3b~TJs*r2RrQ}=ksX~q=-;#gHspMPoEP2>B z|A}&99EbyPAP&TVI1mToKpcnzaUc%FfjAHc;y^JuAWxDr$)n^;@+`TOzti$}RPrym zmfTC;CC8Fq$-VsDRxx>*&k_gXKpcnzaUc%FfjAHc;y@gT192b@#DO^Q%{w5kl1Is@ z3OUn%;!<)eIhTA({v{WabIHr(S>OC8%879x4#a^t5C`Hw9EbyPAP&TVI1mToKpcnz z#pHloN^T{Ol55GO{JoUFtCCOo`zpDW+)GX+7c1mg;aA1v2jV~+Cv2jW24IUv81OUaw$R`M+Qlw3Q2a`|9wd7v%F!@$__Ac)| z4#a^t5C`Hw9EbyPAP&TVI1mToKpcnzaiHuS_)q*wP9?{ZL&>WOd6d83l4Hrc{JoVN zOs*AvRrddE^9pew4#a^t5C`Hw9EbyPAP&TVI1mToKpZFr2jo)z{z~p7xAON`axQ;A zCBKq$6>=##m)uIOCFhcB6@!QQ3~?Y1#DO>v2jV~+hy!sT4#a^t5C`Hw9EbyD=YV{w zkUzv2jV~+hy!sT4#a`7cR=n`$eH9)a;QRXC4cgFS8^_YcU8#4S1!n{Hp zhy!sT4#a^t5C`Hw9EbyPAP&TVI1mToKzVjRj^yvImC3%y4Nvv2jV~+hy!sT4#a^t5C`Hwd3Hb^ zC0CM5$)n^`a;`!iCGRTaT!p{Ol8?!)v2jV~+hy!sT4#a^t5C_V$19B*TS0#^vJr2ZyI1mToKpcnzaUc%FfjAHc;y@gT197129gs)KvE)v2DY=!rO8z9@D&$`B zF1eMQN`58Zl6RH;C(J9vfjAHc;y@gT192b@#DO>v2jV~+hy!sT4wPpH{JoW2NnRzF zl55GUkl3&TgCtZ%y8N9EbyPAP&TVI1mToKpcnzaUc%FfjAHc zzC8!zNpdQAl-x-UCBKq)$)n_5{=Q1?C7&wfU2-mY*|+ye^Gb0b4#a^t5C`Hw9EbyP zAP&TVI1mToKpcnz<v2jV~+hy!sT4#a^t@a;Jucam4hujEwnE%}w)N**QOl4HrMv2jV~+hy!sT4#a^t@a;Ju zuaYy#mkK$R{7EiV$h``Amt0HkRmiu(ufDy1#(AYU5C`Hw9EbyPAP&TVI1mToKpcnz zaUc$qX9wh0{%)#}Qx)v2jV~+hy!sT4wPpHv2jV~+D31=v ztK?VmCOMTHOHL(sl3&Tcv2jV~+ zhy!sT4#a^t@a;KJ$dd|rlpISACC`#y$+hHP@-R7;+)D1{@3iD(-`*$9E5(605C`Hw z9EbyPAP&TVI1mToKpcnzaUc$qM+f9jawj>Hd`Vs<$MSbmaw&O~JWKv1zmjYD`z$$_ z+^jr$miHV7;y@gT192b@#DO>v2jV~+hy!sT4#a^t@a;JuuaZy6vE);{ZN?s+G zl2ggOv2jV~+hy!sT4#a^t5C`Hwd3Hcmb^gv2jV~+hy!sT4#a^t5C`Hw9Ebzu*#UW!oJme4kCIOnaxVFk ze9PZq$*KJPmb^-?B_ET2m1pnr-s3V9EbyPAP&TVI1mToKpcnzaUc%F zfjAHc%CiH7oJ#H_Z}N9naw++i+)EB6-;!6!wd7U)E=$hk@4CvfcX{t|AP&TVI1mTo zKpcnzaUc%FfjAHc;y@gT17+`kyh%W-X({UbIGUV zQu3`ro+S^HYn5m3^4{Y>9EbyPAP&TVI1mToKpcnzaUc%FfjAHc%H9F_mE1|*B!7}m z6>=;&l-x_6V9EbyPAP&TVI1mToKpcnzaUc%FfjCeM z4is`Jd6xXDkUPn<>U&Y{IK0_Rc192b@#DO>v2jV~+hy!sT z4#a^t5C`Hw**PG;l2gg8{O<+jRq`phmA|8scgd&ZR`MzNm0U}HCHE@3PnMJ8Kpcnz zaUc%FfjAHc;y@gT192b@#DO>v2a3S~Ig%VpZY8ghQ^~dDT=FXUmb^-yCAX4i74j=N zSTT5*&kzUVKpcnzaUc%FfjAHc;y@gT192b@#DO?ab`Hp=)({7Nn+$C7uI-6zY*aUc%FfjAHc;y@gT z192b@#DO>v2jV~+hy%sofE=ojE6J(kTJkG-mpn>7CFd&SUvezDmYhrORSX{HGsJ;7 z5C`Hw9EbyPAP&TVI1mToKpcnzaUc$qoda?!e{UtPl55GSK* zIhOoN9#(drEGNf-I1mToKpcnzaUc%FfjAHc;y@gT192b@6oUhDD>;+gN{%I$l2gf_ zv2jV~+hy!sT4#a^t5C`Hw**PGu zk~_(%v2jV~+hy!sT4#a^tP<9T;qvTNjo~n>D$+P5C@+ov2jV~+hy!t;7#xrz$)V&}@+gh1^Q+B&U*F6>=*1)qmn+a;#$TFrOg~#DO>v2jV~+hy!sT4#a^t5C`Hw z9EbyPpzIv*_f+yKxl|#~l26H{{5_TbeV`mnUM2UEclrA+c~|&V+5I!k$#EbK#DO>v z2jV~+hy!sT4#a^t5C`Hw94ICSPm*WJndDUdcZKpO`IcO&kZ;Me zPF%HCmI1mToKpcnzaUc%FfjAHc;y@gT197039FQl;t>jAbCOMVd zN?s+`l5@$av2jajt?|{5YjwP>>H_5f+S@J6Rl)Or=CC8F?$*<&A zav2jV~+hy!sT4t#SC$dlw$ z@+tX~e5#N`$-CrQ@+^6n9ITLQ$*tsL-`pq3NpTv2jV~+hy!sT4#a^t5C`Hw z9EbzO=zttbP9?{ZOUa@9U6q_lE+vnWgUPStUve*bm3++KV-=&P`Al&j4#a^t5C`Hw z9EbyPAP&TVI1mToKpcnz-<$*TD7lr~Np2N0z z$w_e_4#a^t5C`Hw9EbyPAP&TVI1mToKpcnz#prv2jV~+hy!t;m>rNu$*JU5a;-xCRLG~~Sn@5omt3omSNZ!Zd08=g zo6i;p;y@gT192b@#DO>v2jV~+hy!sT4#a^t@bw*#Bgv`cO>(S4F6Hm2v2jV~+hy!sT4#a^t5C`Hw94KZ7 zE+yZRYZY=Rd6m3Nt|hv2jV~+hy!sT4#a^t z@bw*#JIS%+O>!)GmE1`VRmiX8QF1FemfWk5hsn#n{wK%@aUc%FfjAHc;y@gT192b@ z#DO>v2jV~+hy%s!fE=liR~7#5O8z95l6%R!m%K|(Chsa{Z}ZvWKpcnz zaUc%FfjAHc;y@gT192b@#DO>v2fn@oaw&P0d`b>g$hG8E@+Uc#zq680$;ISa@~c9w z_4PkNPKX0>AP&TVI1mToKpcnzaUc%FfjAHc;y@fIW(VX-@+$v(K{=NENlqo7l3)4z zDmj;YOP(bklUvEjv2jV~+hy!sT z4#a^tP|OaCOMUSN**Qel3&TEv2jV~+hy!sT4#a`4?|^(s{v@Z8PsyYF{gu2+ZdJ&+t z#DO>v2jV~+hy!sT4#a^t5C`Hw9EbyPAPy9>19ByKlUzwICEt>3$*1H{axS@+98B&d z=aPTPv;19GF?*ZO76;-$9EbyPAP&TVI1mToKpcnzaUc%FfjIE>9gs`Ot>ja3C;63p zOD-jcD&$u3EcuswOkO3&l7D^uPmmMhKpcnzaUc%FfjAHc;y@gT192b@#DO>v2a4GN zc~v2&k~7J*{9ToNO711cl2^&Mk1==Cj3tI1mToKpcnzaUc%FfjAHc z;y@gT192b@e0>MxS8^tKl>ACwC6AJ4$)V&}@~J`&Cig1jTj5t<|DRn>hy!sT4#a^t z5C`Hw9EbyPAP&TVI1mToz~SJ4yh$GAev2jV~+`05VGrwVzL9810>xAJ#b@-4ZRJj&l~ z$;0GTg&a%H_0`{bcE*7?5C`Hw9EbyPAP&TVI1mToKpcnzaUc#H4h|IZDSsy=zmh-s zJ1cpYyh@%WkCKDQvE*FxE4h~3>u~TmpD_-^fjAHc;y@gT192b@#DO>v2jV~+hy!up zt2-b^k~_(v{JoX@N=_x$l5@$wv z2jV~+hy!sT4#a`O!2$V`oJpP~r;=;Qv*cU;u1XFipORn6yX0T;F1ePx>u~TmpD_-^ zfjAHc;y@gT192b@#DO>v2jV~+hy!upt2-cqBk~_(_ zv2jV~+hy!sT4#a^t5C^`x19B*T zS0!JPTgjc|Q*tf2m;6fpC6|(0$-CrVaxFR6SAXZ(83*D(9EbyPAP&TVI1mToKpcnz zaUc%FfjDqDI3S0TU&)>1RdOu(l)Or=C7+Um$+zTKax1x*+)9phICz}T7zg4&9EbyP zAP&TVI1mToKpcnzaUc%FfjIEh9gr`{ujEd0DmjyUN}eUBl1s_A#W1v=?(# z(eI#5&`IcI<}aY0NIjW)I4bwLf%Qi8U!cB*dJXFD!KSc&8~uK0efmeC?dbPK8}hkY zQlCP7A@%8KCH60)J_)UX-p+b;_79l&%BdM35W9WZ@cBMa-x(anO^e*(b=#%JK z=m4}c`=3WoLq9@aWv(rHCH*JR|E2!}KF_W6kEGuP?T$Xl`ghR==ycYfN6$pZq4Sx$ zkk^rap2+&CzCLw3>K~y^(Dym#O>`*p%hB`DYnfk3eG>I0)c2w%qutP}nCpW6Kl=Yd zxBEG$w^4UTKcl}1U4Wj)dPC}3=#}WN&{NR|&>rZg?C(Q8oBB7@kD-Im39Q%PbF8PH z$$CY!4(siyx1vqyS76`q)KghMhMMJ)#q=++PhFFNHSRL(fFJp_S18=G>Xo zr?dZZ>Q$&*e=2o5_LWj^qkaPYjQ%F{X!<@c_$&R^^xN~grPRI9b5|yQo*7@1d8oeh6)Z)<7>~t~2#&^p~j5ef~e*e_iH}q~1r}nE8KDFQuM^4nrHV z?(?P_>CZvW^&bTKXXclqbWW%`dZ zcQXC+(T`cLhwfx<6!kgOw^Q$+?ukB!w&J`+)Em(E(828gC-pPb3(>pLdaU0^{Wf|Q zS`D4e+<#I3h5CN92mLRoC!hn-M!es;)YX~4hW<|Kdssh<{tWt$(yvYZ4!VN=0qQ?c zcS3)Hp22!g^f=DHi1nJ(SD+u$??!zDx`+N)bRX(--Hx2UivIc3&r+X_K7}GBQ<%Gy z*BMEDBK3z1?x(*2-Aw-%=rsCcsLy8qz4R;7e+Rt^J&5*3E3&T&`zp}yN_`vk7W8%W1GEow9bKn>pL!2< zdvrH?2J08|zOSIap8hN78LS^K`)Koc_P*-%bBDbRhjZsPCk%%D(5Qe}vwT zj$pnex)A+4>wlxZ2E73NHM)bj@zj;6pQfHeeGB??^ardjqP`ye4)3==b$#mk)UByI zpik4U!+FEeb67u@`fBuMbRN2q{fE#2sK1XmmUAAYKbL(!rv4td%wJu z^R_W>?rKN>N%|wGyHJ~p{ri0=>)unoLwz3V{cJMz8>sjDnbZ$)zWK%6a1HC`E$?^U zlf0K*z?}D~0i4&FzPa~I`ra?AQ#WC*4f-klchQNcyyqBdbGG-j-?4uis`x$PeW)7? zM{-VI>Ivvd)VyIXs>HnCr{*K?YxlD5{l5qGZ_!4aD_6XZ{v-6|N+aoef0#^tD*7B+ zm3`g^%{k@^zgMR)FURt}D@S;d^^-WSH97=ci<+CKGT#gJ{=S&G4ygClTGSsizXj{E_+y)ccxTPd+8j zxs3DWpK>GbqqnhcE_jLhKF)7}o`A{~{>)r$`rbc!)AxSqJ?DMq%xP7q=Q3~pt52Wn zN{Rf=d#oJA`!XcD*vp= zoV=(n^-A;|=FRusvwP8B!CYgsCVD#Sa){d0@1x$A*Dx<9mlJhou9Wpps9&WyFPkynp1#kY`{{2-PocjQ-HuL2UuSMAdL?QudXoBPbS?XiruO;F zZ2Beq^Bzt;4&7wUKDna#UQT)zbMnB)sO62nqMnD!dwxp22bFKmQVMW>^i(8j3GwLYb;h;~9}ab8>W2DB>r3$z+4moTrkray)A zhEU7phEn%N&EtEie~2z)|Kq6qZZ@^gFXW-0v9Av6ThW{752K!ejzaHdeIP0i@cFDf zXg%w4h+WhVagJPUKlKdOA3)_+=ThH+%0Ir#z7NqItaqjEkKTvghz>+c(R0}MGPS(t zQtH#t3z?HMuA+VxZOWgY0nATBJFz|iJxISEwa>XXQujmWqVmTRIDaAi`keD8^m()q z>++Uk=x?Q8om#HCfxaB3ChM1@x3IpB`k&N$sFzbej9!3_ML%Y4ANn+U06oZDW$HfY z+vs0dA5DE8_1~$VLEGcc4(jhSHw~40-b#Hfa|_Vt(ADU8<~~4IqjK2|%r&L|15{4^ zSJvCmmpk4|U;cIn^)<|mL0_R?UUGfjXB;>j9Ps|`-%rc=bGn^BxBk8C-=F?HZ{F}e z=RMk-?BA#6Oz+9&55LE~ANaj0&){v8m>q*{^ z-;45{*393=y7}M`{TlS!QG0)p`z~SK`>eTY0e!#U&D-8D>a%X%9Lo8-Q1guUE$?UM zd-L=n=G$`KGV}obJ=86zFF>o(Hy?OUG{^sib?;YI+1G`>_Z0JzoX-2DIm7$-RqQh_ zRHAOoxyMk?qV9@bhW?Rtd4ajgd*lk{Q(SIBh+(|;A+je1|6 zL47{w$dA15e@g#R=DxgFQG1`hjk#s0_qpeplY5V){s(h%j|tS>SvRjOre6iEhRSQr z&5yC~CDzR$H`ABvpF}Okl>^<%KJ)7zm>Wp{F6v9rmGtGREveU`*P}l|z3+Oz9>h5Z zQ181Pm_HVsj1ETsjQ$%vmi^{u%90-ZA9K%n%nzhL94$qUK)L>x%wgVq@EG+l=4PU! z(chpxZ+e~j5c(SX<(Eg&uZxzl{uAm?(RMqV*No>9`Zi>|BHTe^b6Fy zH=4Q)Y95@=e!1uSteb0UQNPFBi|7aF6KF@~o}!iy+)V#t`sUy%^jp!F7vE1kjdeNr zMbsCf70|yiFNbW$KVWm+rOcUg%~>1iU&;QFsGRnn)N(0vSZDetqbH(UIqzxe5!4l^ zpP;@SeFLqG?qu#>>XXo6sJU<-bKB^zq?Uthp)Y^f#`+-o{ZRAnOlo=LZ<%jKzd!XZ z^g~n*cm;E(qYYXA6*`yx(bVSt&D0}V?@PTHJq7&%T9G-Qd-+^Pjqwa=@2(QiqAEp-Fxsnqk) z*U|lGE#`($mr#F!p3k3$XIYoeFQIONjzf<@Z)bly^q1(#I5EVf1MFd#PWceuw%_>hGZY(AKQi=byVAz90QF z(9X=aM+eaNc}qv?QLOK$ZbZF^`d;*Y`gf!A&>AZ5$LIg@>FVr19GE~pV;m?i4!pvj zOLYwxHxjyRmw0Yq(`fsA<3-i6-qjI3TIoJI3WA^#I>iyHaypDPKg!dkEqkP2g zsjHaxKJNWR?j%R^{gu>+6{F-Z6x^-l$x} z{PH35@|`x+=B=UBelL!pmfLur_#J)kA6|+3HHVr<{=s=Cu|5-h5;Y&*K;0cRAINPQbB_Ge9J-W#6|^$@Uqj8IgIV`px{!Jq zdO!1i@6V%_J8q=zgjQmoIrv5T=J&0vo4XEB??%mY&rr)LB~Lav;R~2OQ`#x*V3Pk{uwpT z-^N@E`lZyDplea{x!j>8eRJ5W)U(kI=oZes8GRe|-fI4-!hCJi`@i?i(d;`LH3vP$ zyd19y^()Mo!_2>P=pSJ2OzK}!Z=`;d`rp((hcKVYOa8^&iRcCBHq?CZWj>=Xr!^0o zPv7A6s-iv6Yfw4$dDP|(^Vbmi=CucSz1`>z)|DCjAMN(U0lxK|iNInc5uxKK;|tpR(SUx&rk?YB`nstUl{^pl6{YnD;r;ALy?{ zuR$MVt_5{9>RG6~NuF7mek100pmWgmtPe-m(7%%UBXkgbpF7A)USi!Gdx$x6>ptps zsLvChpq4v7Mcsk(no~EY9!OmYZHi82eF}AN>Q&S?qnqfzMSVZDdHZFh7lTdFVUz<*%#Vk6yw0z0@_SzfbLRGN1d*V|^X^ zTjr`$pGjQ{twCSTbTs|WXg${dirz$jEcz?d=l=4bHO$|K`h0H|^L5eIXcOkfP|Le! zP`6~g4fTDf{C+6)IOe)i&qu#Qzaq6fsU!6rtUpXGFTRy}oPG3mv=RFn^6%4y^xr`z zp~oa55yVO6YHV+?5|1tXiO}&uX`_(S`@+G;%Hv00MW$crqm_y_*&6vLheT4Zg z)NQE8pyrhR)VHCN(7V}J6|F`83ADHC)UQxKiM~$X`>^+BbDcT)80LS0$`9+a-+Rke z>H+9c=nd>UNIeDJMBn^qo|?_N_p4W^TQPqY^%rPk`tn6NoLugC*3E5gspTZ*hP&u{ zUo|%^;&o1No!a}|2lVBm4Out;yg>abbB|Nkqc$)1rCx&G&78UP4EjCLnXJnblI)0eLt$@~-aFQe{@ z{)PS;^eR-Iyp!6Tvy**QoaY68M1Kgn6nz?%Bgp~esKYpiv6AEXKR2U2(9_Xt(IeP* z3AzCNg!Mht_oDK%Cs~&ln&)nzZywxC{W$w4Q+uELF72j`Gkjy}iw zY19p<~|dOPYJ)N-PW=+~$J5Gqgl4(oTK@1y2{^Qq;-a)k#t$DHc(h0XNu zN9Bzps9T~<(Noa3(DtZ2?_bn%*NVI!xuAKg33KMvj?9(PH-9&#Z+@36R%gzf`8u`S z><#wqMn7i#UTX7k9s2L1@1b&(LCl|ons4h+%ez~%Pu?VdGM8S@{0iodM#rPASw9bL zL4O+cLhA3M8|Ys|Ew_~inbXf^y&Ln>s4Gza8I^y0#(FjS&G>tI0)6xB7S`v`uR+}i zo$EaNOX!Z=y^!HG&ram35h^}G%pVV@iTd8}pK91VwVn3j+h5nnlJE-qL z<&$$*KNFRs{eiig(I29pqDM0?r|U#5e_7Am40JHs6CI1*gbqdJV>fc%edr-{5_4xz z%e$IWw@2lLvzXsc|3d0@)H_hQl6>$y=H$dia)O<{P{f*eUpAG>L;kZ-yEd&ep!V%bGLb4 zj$yvr$ef&OI_Hd}zn1!J)W83$vTnY4j&<+(-nYHi$lC@mcRBlCL>r>s8<$Z}M*aT0 zl>O$Z9ju#AhqK<7zPaDOuf3OePx>iy@`63o=83wTcO-r9!QJS0q5oTSAZosP#P@^l zV*LZuJk^VJ^TQ>B}|Vr+%7sx&0RB=r5$coBDb5UDW(`8FOdSH*elce;_3zC(kfP>}GyHIs{$J{04Ltx)+t39!vch`<|n|29;ZnrZ%5#qn6Xm zqn?V&ohDL0&-vG*2kHL=HHV%<{VsD8sD1v`fcm?v%heiCoAbsn_kFZ6dL?t8qUWQ% zSeIYkM1M7EUVMl-bM+$X0jRm{pVSM{b)54U^$7F=bQWqZJcId>=n1ULL5EXUWc?U) zB`Od919NiHBdFhCPVVsw`rYYQroIksPk%D{L;Ag`Yg5aAd`?x5{>{wEOU&8-VqLy7 zpZb^R3+$VPuA|?XdNC^3kpsxb<}mN?hF4R!K+Wk_Q4iugxz|`~dDt_|{gwVJ)IQJo z8}-jwUxv!7?x1eQ+UUK7a{TYn??ZnuwLI~U^heY8 zdBP&<-mF)lmizCZmdEy?K8`P*bA3JZhxDIj?nbm4>!+am=x;<@pcPPg=k3(zp*OOB z4fW~NU!Z@Ye-U+aYI&CYtGwj;yw5mrI5^<<>683jK$JMTH( ztIT_Hg||5G%kMdA?@fI6ZE zV*V&TPZ!khBlD2?ZzJo&*teY8+~a-scdW}3-{N6W*|CV!}M7_^7pA~1`nynh)lID5g3f2%@9#_KKaI*)CNekK{nW>y-uKOS=DJH+ z_ddRr{oXIU_Y9$bB&ifBv`z78t`ZMpn$oszc1Md~yXT3k~;aoYz zo1D`g^?qmmn@wMiuz>xq(4R*AD(ZdKd)4jqmoh(t+WRN%67N%kS(ootq&CNPrLKaG z;Q%>fGx{~rk*s$>L>Qb9;KB0b&^W+!ivGuIiMCFm*x6fmr{O@Dt=AmoR zmZ-VIJm$SnJ~M>52hk5W-#jZ9X~KFv)SPdg+QhzXtaqpW6)NvLopqmE+{OBCbPzff zox=QI(Z2Lop>jFz!Nci$58Xg5|Ju(v=F*+44@A#GXQRJB*P>6b{|ok;(8jF)29^KcLv8M>%zQomVyMsh zR{G|@*3^wq^UqM`%^UL8{`4P0PNd%um9uoDHs9aG zzDX+Uz0f7-4%GbgYijvSDf>T1Uu6Aq^bT|&>qnt;(4VqCgIbPz8MVCNO6q>-chS?( zU7Yg{`eXXt&=#oApFU=;Gg=9qYt6izZYlj<^v|Gv1%03XHE1{b70@2&$!I&~R-(=6 z`+TQ5^%<(B%AJE1qyuS)$H+JL^#bF0ypH_0m&F*gRi60Oa= zoa9*gXQGQ(Kb^Wa^PF1}o%%L(3%Z~6cd302<8z!9te?;NKhX2}dvzG=x1#doL)6>Q8fa(sAC1y3xt01D z8gkVO=*xdbQEx=$I2EXU?sf|2oQw8BpJi?ebpvX-tNd&qeYw`%)TQjZ3Vjc)i++k; zgPwrOzp7EM;XFC*0Q&c$FR(rn?MMG2^q1&%)(;0Jkk1$g%8LWb`SZDeKcC)jKBSgI z&0uZ<+7~tN%4_B-{%-1Xs8>_3 zM{jWsy`27&s5#C1h4+T#tedapCMR)DUDg|-=HR$!qkfKB&b5oa zeCIyux0pK;eI0$1^@pkVpwH01oce9*&!}gi_2{oay)T=yr_(^1s-^?A=>DNLxq2?3w-!b%G;5_rzE%cYspF!>Y zY7e!1r8{%x*2%2PdF9oYFn54;x#?Q^H>0n!{xbSFeea)gj#ugbnK|?M_o(Gmaxd?b z@)~oGx%&am^IqAWdNV4wIgaz?X!5xx%xz;`4kQnLiM|}b+FXuc@eFwEU@KO4e(1qx9bO7__i5uv9 zAN?Wq`{-2WpQQGA!aVvHq2A~J=6?F}XZhRjSzm`PK)W(8x7kO(AN^+N8u~R+^T!pe zZ>3LJaxMR7G};o~jQ%(Cz0fnzWvt6}%tM`7mxq~aA7TD1^nT`$lE(Ba)A$KGAFYL& zcbZW@!9Ka?2>SAfiPZA%Y0S$_K4$$7sQK$c=H8>Mzj0qi3QuIHx1^Fw{Ks zH0xE-;i!4f=LIX>&-z2C&xLMcT|RLo>x<~$h|2f6QU8m%`>Ewh=GFf6vm< zKEIKB$S0mnQ2Qh&xid5+IDenfvTI-dFFXk+vV^cmFWKx5b^ zSGj2&qwE?{g_)zU4dGD(24pJ*7u|G=MP!`0KE^DTePH}iprfPa1Q_4l(b;I z9sPHxPp58!cB3y(Xhgp!{gbFqryh*Tn~tWQ#GL%Tl>S8eH&Ks7tI=Ibr^)Jy?^jA>J%WkKC2D*s#an#kQ zH&UNMeKI-}ZO(dK>QB)Z=wC#AF4};89aOG$BkNDoKN>v`l^2$mT%Y$D2Mz}Z{QLhn z{+yo3pJVTp2dFoq%TVtxPg7rlZbe6No`3IoU-$cB8gu3u|9;pX+`| z-~0Kc)P67ig7dwvd4KZ0G=z1(*IsA8_y1R@{XV&YdHKj_*3FGusmkG&t6(--o2yvNL^ewh8< z!>*)Qt*#92d7Hx{kC(Nt=;QaSluSdNZHJAUC^@jA7?ff6_eM_icXZ}M} zE+cO{o4LnOIq37$li4TVGtYg@`U2K#qWjVQXnpil)ccuy<#WzC$a;756;$rig?co4 zCHpp@70|m_ACJn}z3(1m?kLt*qvpr1)YmX)p85%WbMjwVmz$b*%olevKa4qZLuG37 z=N{&sqd$n+++#kLW0*_5$D3E>q-)t<8NHKpt5GkdJ_=n*|0U|vspVBO=*tO?q27&F zVtyL+Otcw&`NwVa>!OoU`STOZ&qwcN-JG|R`UuvKrkYe!#> zax}I1{C?(;l4tlo=FRJ<<*JLQYj92vv>N&%>vBEw_FUHcq4SuVKz%DJzZyi%$1Ra> zO=RD5=uOPsLVZ8=Sn7XM%d^g)p33?ayg(=F6|BoQYSWi5R%iWobOY<3qVkQ#tjiJK zV!a~TjP)((KJ#47#z6yO5 zy%}wU%4IvS?46WSde%=+J{Z>4TP?Q_R9^nEVShxJA16{uW#HuK}???8R-=I=26M1Ldd^Ru(r ze>gCKe8xCXUL5f68Sj5R`E%<%%slNq-FuVYi+)e~eds;KzZbn9`uE!$&Nu&gul0Lx zH|xJc{XUw?-yeQI_J?QsYL-u)( zF;AJZHnZNH^SmFlr0?Iu53$|@^JO#k%*iwbb6X|48lM|K8udcYDwIm^ttL zU8w!O{)pG}9&28bw|H+g$8KhSd(?Zu7S1s@`F$~r{s`2(e=_^MFHdCsc~lAC6}=NA$|Fk_igWW-jls|n_EV+|8(|mLtmudmHHm)-qiKb z+PTiZ2;Peq)9=db{heBl^fZ0%uTQWpcap<;e{9VBR`dpRC3+3}+n}rHFQt|f$R+M# zeFf{Y&~xa^k5~)p%d%ymZPrk$8u zf_7v5d{kZ{e=v{MWd1$0KY9oH2K!r4H>2)>)<@4|{UYj`=qdEiLe1@Rk&l`CKI^|m ztE0a_w=!q`pHBVXtly4aPybzX7yT;eF|JcrrZ%75Pv7Sja)+DgS3;Me@|rt2XD5Am zK}Y&c`Fm*+wH#|U^OvJLP@f~#V*bbIchPCgEk^G{FF-$M?jh8i`W*EOs2t=%YM-z8 zd}aveorm^D-(lY8TJp<*te=Q(Lgms;sjH#p=9{SH0`GDDeEL2&y^X$nSI$0wIiCad zqLyFmrCx)6hjac$eJk}rbS{0L$6Y}GX0#Ob`K+AxH2QL|zfwPjRz;8G{Eg_j=oGXO zb8^e^^lw3ZzA0ajTYSPkxk+E<$I*YDdO13S{*~xV^l|hp=KhIRr(XlDjm||+VD5j> zs_4&Ie;Pd-y@>Ujs8^w_>9<9%LFHeRCHSL(?L47av0P1JaDd;xVYoI?x7qI?I>PG10 z^czrj;`93)^cB{(qb<;0=nvRem%1NydCB#8pK;)DaG(c&KFw)A;=6L@;^8VpHbrI(@M7?Kw-}T-q5Awd=oO$ne z-FbcQ1O7e6SjiIq-lFj#dMoPpl==HH=I=ti-?V1_Wz_qL_ak$Jx$boK`+Z%R+B|8V z{Rw^VN9GlCb!T4B@5xW8%~Re#2hsQM`Om1m4|=bi&g&&gH1w;p1B9{bJDXH$EBk>f39{zlaMwfALn+I6gd z%sFx#IimM`dB$$$%pvlbN9lWCHupYFUq0~)wfE%;y#BHDpG3`z-pl21-rv0kd!I9B z{E_`vq28lZDya8Y^Mm?<0M9o%>iHje6huf|{|CC#lz=n^{;! zEpOOF-}{REr3?M~%&(=MK|Ke3kA8P*^ON@_?^(lHH&1k}&IjfjoPHm3dN8g;(@djFh3Zb zh;BrWL=P~(6+MCeW7OW857L*jt!Lf5*qrqa^v#DWsoS74(bt%tiv9uh-t9ep2lH~$ zN2upAzaG5-{S)hQ%ZBvLndWu#vpHe|b1m85mHPi@?_R%tuG9CAUll_!5z$(tFo_UR z5=&82Le1C|WjkqX3adquK~W76O)06dt05s|n@l#@tfGw>+Zsg52ur?;NhzX!H@=VK zdt-ioK`XbOj^jLEuj_I(*XMnn$J<&{0h{x?;}EQf@94V(lgD09cf)USmvg)5qICA# ztM~)?CGko;58ub^TTSUvxWPTy-#+9o<)4MwW7eqmz~npo=-!x|ZH)6Z`Ge?w^iy>5 zr{VlPm|QM-L|Oi0*xUK+g^TIgI2E(6Xa7CG|I|IT{l1*w|C-M(m`86`zm@(cy#||L zRrTU{h+hsL;J-<~fXDdBL2C22VD{bT=v(m5&NZh?(R1*A{(o{mCPyAmKdJ9JdLUhz zeg%`S9-_a}_ZNB^{Sp@D*Q7tfC{_H4! z0sc{aI&S4p$K=qL(|PYPk)G+?3c4Ac_Y!%(*^$2y*W-oGCAa>J?xDVt?ujG#Z(u$C zudpb;1)ZE~FrD{u7wbFD?}XL-LO-GY62Cj0JTQ6WLiOS5XFFekUmHuS55}jliuzbO zIp{b1$=CqfVPE}Y>FQXBpSI{R6k&l}YP4*ceHlz1b%X_ zYw6@+C-t3+b=6nWnS(Q5SLNTV?;;$@&*y#et>kaXe;(4Ge4~_e9r)SfX7O*pBl?=CXTBQ3&-^lh{u=*=nHTmszkr`yE6?-fjM*oXqy0huc;`FNZ{y3@SN&1? zAf22h`&t9_@wixD_VwfuThx;){P(;)K)nlQeyl)eJ}Bb;%+4K+#YO!8!OGYklgniv$zEMTJ@Z@k#cKRxn7u#ydNY3JqK0%C zY~}k}ipB7BwE5#F`dR1N(>3TH=w*R?pm={4e|KWc3mH^FHE5dKV@? zX`}xY{%v&jlkDZ0lLzU$Ti*qk{bem(2TSPhj*a-qHx}^U}d!g(b+250&jMA4px1M`G<0rqW#ZOMXN&R0~7Mtly-gG$k zVfMAR=nwHG=ij5VXZ7P}FDb452qwo!{+Ycjc}3onBsVVVzU;qwuke`r8sa)^p??yc zocC5b`%pDHdvEsdfABwXPffZj4Fy~M;Tv49uOIHf`sy#>o&2+WpufpKg(2P)#@|p z{n&~BP&8hGyRf>xf78S1HP{QM<7N8#;|V;bp7#Pr`P=!)57zOM7tN>7$8!1?(sQvC ze?5JOK7ybE^FE|4|0DiX%zK!X>br4g&eK)s2kGPo zPw|&xdG+LvW9Uli$$y@pn_v%2?$eXL3a`Yu?yE;> zlUN)#;K%qW-h+qSvkLPbqcENP=1=q~=bomMwE0FeL`=T*bJ^?9e4l520>6ZxK<3HpqnRK5+7^uW`<%TYpEH>kv*%3E*B-OCWG+9+ z&s;pt7kG|;2Uf=9EwkwCIeGq;<4(D3Y=9uT=+v>@=GXEv-cv(HU(KB@B>&#)vIWos&Zl0+BEnheL zcM&>yLY@;f^)JTE)m!MyMagaM=Vy-UKxePa^Ci#2%%|t->*xEZj$JW(RG!btO*6M7 zH~B$-=BwH6J&jXvzP=mi_H_1&A#^SEM)Y#bo|xx(_Py+z7wXHLkUe89KYMwet80Co z?8BKqZrA^kdgh7D&mGnCoIXci^5W0cci=vJQeWoL%tHs&v)5!E8>T<|R!x1Gw;HM! z!`A98G5KjpdRETs|10k2H^zJUnPW=u2Vv&6%u(z4$#1thUm9Cra`B4#vrlL5PCk`= z<9&U}nP$?-H*PhD&ERLxOS!ylRs4CkHGJ-sC)X*nYX&p znbVHbP4%CHoA`z4Mf4ubK9fD`CVuk%H=N5pIe>l<+v)e?Dkw_N!}Iigh@JS;@L5d0 zkoO=1`P1<`=d+I{r+S9}s=fhu0VeNnqOT0U3)aS6>dWY>F*#-*^_y{&dI`*XlI%;_ zhYG1z!|eBa-IM$_xz|Acxpw;KA3$Vd;3Ct{W1GZ4fmDjXJ462UyI58Khu8;CU2-mchO&*-iyugHQbKbqe?q> z8Gkuu4_ct!ng0ph0J9%YqO_hoP2O&?MJ7zgo_msaK{ z&zVgp586amaBd8?;%}pS;7)#b`YX(SK36?^`uXZ7`9pCd|8?xfPu?|>9)^weJw_kH z*)@3Kl*CFKgms!FFl}s8!p5`&JU&M zU@5Gi-kZJx*YUrmdtq|PE0du zsyJSKCcT=@`~24Y>X_W;l)h`-n_TRaz9x78H|uYRJ#i%_*Sto5asC(d^LUz{{QXY4 zE{?*y=lPUw>HH-)oPR&o=dYlDM=!wc{5kj@{t_~QGk?c3KY^diPvHOdJ-?raFnd?_ zy5vsT_r~}=PY&>wbJ>q(sb^o$KApKUpX2qN`$&H+%p5pXJ@a~=gPC)itFOn*)9=#B z4Kgo|=4VgI{FD87rh4YzmA;=m|C8G!H^@Ac{rK0;SH$djnRD{oO}>};eVnh8edi83 zdwJ&658RvmKYL9x{=Lrq_r95Y?4-WTHOa-2KkQSl=$>};PMn4N)w5@8qOt}weOdrK(ozH&u5S?6g3B4ExIG6l5``}gT2h^uv=CjP_o%q>f zv**3+d~&Ka`Wj>A?d;J*_${2fnLb7*cWuePlz$&y%g-E}IWYTk_LVyNvPYMsGcPWu zU0d*`KeXllh%e$`oTBe>%-)xI^%H&PVCLHFAMApY&S#&@99qqMm;Jqr1I6^eLT66g zL|=}{ljrMqUBOhk4tCXFoBlJs0Fzf0rpxNPj!vGIJ+7g8_WDNps$duOCo%K*TzZ1O zhj0>RzfJD{t^N9&5NN3;OKxbc04tgVhx^v0#Gsh29FNzcO^~Yg&55A-? zx!y>6gnDPZn|~Xw!UpO;VD^^eGuc~esDF&V$GngF#C>frdrCQd7xR;I{e@o}U&Z7w z9q8m1HJndg*-yO^Kl^KYI`2dBek8eHaRQ-^9R?}l}1NOkYXQ@i}bZ!Q{f<6~}@sr0Mr+edXFz<~Xrr&XX8~r8j zW1 z9LT>Pw_peLysufwACJkYM(Asb-PPC7op2q$FSg@9NWV@O#{K+X;c%>?-i=PqTY;ZE zqNDns_)p+SesYh^^mO&@xCehyzn`v*%dj-&JPwzJT;DPNkMu;G!%u$QfZq+1^Ch=l&d>XT$DFUq|GDh-XTHxf zKY?GuPax0zJQp+n=ed2@f z&fKz}&U})*BXjn=^yA|=7^h+8v*Zrdd>@(5vR`C>Xz$!7_*cx_K7r2jHGBV`_<6o& z&c2cVxv$$Er(^QgmUQ;F?6JvNrmElVzU){1>5DP*Sw%YeU2>Ft{Bzxx{Gtw>y(76= zbAEEin)C)-<-U?wg`b=_duQgc%zc?Ni#nIRBXd_j_hcT*+&6=t`E3AQ8Yj9hd&fe4 z^1U+Zr!f0*=J4A3-^J|NmFfC83-{rp?)wA2!q2`?f?oua|7Tyy`-trKC7r)Yf9C5- z{8s#~*b%cYt)(+(CZE{ieDeJ2^sV@hu~KJ8u}mD4$oJgj*Bt#cyhYzHE*kD?@jKRe5S2?v*%{7 z8^%xGd!Kvu@b92c(%I)r^DFQdMB`)p6?Agrytm3;lRaRN{+*aT_CtEKdy8tg1q5n;Hr3cZ;ZJ(lNVBXhlqq9ffLT|@|xXZnp>EuEU=)B+iQr{u|D14FsTb$3o z4Ci8N9FCjxZ^Ml^5MR@GIh`D+7M*?kh`#L81L@`Zn&3K2-taelEBSrtW0)NCd3v6{ z1NZ>$R4+u2!tyv5@5DaQ{`Zx9poe?%zIm*^&6qrQBApzwl5^|%i|7vYar!TG6*@Uu zVg6ctK>aiNC^o=x>d6-`rSDcROwYt6{NxkE>7m#gdth?8vCh}yUqiR1%i((d`}8TS z%1?gPivMHo#kzd|ISP{dlu~~hKi1coKFfhA^o4jf_QT!IC7(J*w{!jpeP#H`&#vH4 z;#b5s_&exYbSK=-Z%lWm7vcSweCV*gPx)_Pb^b*qQB-}~-$&S$R5UYY$d&)JUp{*0MV+qyr`h3qAb^d$$&JoAG7z4)%a6 zE`8aL%hB0qH|R@lR7w4I{$zSBX3uOwzlq7YewD9_nG3VuUZ6j7X!7HF{FkvNW*<7` zzU1n^rZa~&rZX329=wpB`Su@l=C_*k5?^mRX8$>^J|2@_We!cgb*}mZ{Kh?XFnj*1 z^rct=YeYNW2a{9&Q$6$B0eUkgpUS>i$NA(JJM?8=&;FQyFO>Ywv4YI0nP=NOKLTfH zoJ((`Z=^HdPNQ#C&mORrP7Zuf-w*tk@JGyiU6_6dt2o!0Hr5vm<5$K(>IZQa|1-K7 zow@ibIyuBXeH-`>U_ZEtCB)4i|{uEciEU4hvXlQXU1x6z+HHG5O?{K5MEM_+Q3V{~Qp zhcWs8b?O!Q^Xcnx3;#6zIL_p+qo?2n{s(ipdiMY1HqG^AkI&wpJnIeT8tLnRtN3f^ zvGhp#Bqnc3{{9)iz5em^7W@xCds_03Cj3uuz5e8?!@O_H-hCIp43>AU5GEHYOOModqyJ9D=!WVoG4E57bKJ|HgxRkL(AC}3 zihdUJUN(9D5q>Sqd)T+>y4cBmqv;{^U+EKcNqm?;6qn&_^{w>p>6hr1^dop0lNTqy z_?&;gbIDnE@{@;5rEk)g{G}UzIUZF{PG6e;DsEIShF$oZa4>%+K7{9DZyc@vX8aw$ z3SF2UgJ0k{^<}u7{~&!4R>0Pn_krK*ugfojoA~e1TQTpK7SqYm56}~xTY=sA73i<9 zFTXo|K7P-?7W1CuPP&V}ymwf@pN7xi>-u)n-ROOoob?L26c*Co92f9c<8AmgZq!$V zE>GX&_crgn>+vtqH-^spxxW1U{Q6jt-vC?V9Q7V_L;4yz@4KF$e=d9dneX$=PvDpE z6Udx9&F^PE|MNN3fj`slbv}Rc`Ig`Bd>-VvoacX@=b1Y)4`lAhd~(v)$@4mMS3bY9 zx8=FLM1SUyJkM_6XWz@`c;>QvF6B9t{dbSQU*?`y=zP9s-}%J-`P|H0kmt_l&RwT3 z&%Nxq*^An%XD-h^^c4RD=6RL*F!RqK_3S74+%Dnk=eb&g&b*wxDDzO}x(V*fe3s|$ zVt!@J{yxjSFX2e_%*lCfWZ%txn)xaFQ0D2EozEPa=h|@hXJ4vHm()Ly&R&qYFLQSj z_3W{moZE$&*DC8bPyEbv$xkznC3nrf zpZUDDb8X#oHJ!b1zUjVw^mMF^nJg)lN)w73{bkAD;Etvc)^Rn@*;2HH}&Q-(g>sP6t!b0j}=!2NNsW{yn8|%;how+jm zRdVU%^riKGirG`Mr}yS3pZLnxd6(ajK7`pHl5^GJPtu<``5pcSOb+{;WU`X?WCq-{N6@@|El_*;^N@e}Kt}&UJnmKXZ5H_J-=& zCl1gTU@Khc{Ax^2pSq{Jhsm9xy`v67}R>f95~Q?}Y!x^6K3%?`3+> z*^3^gPdnELlOtt6&AvWW|8x4P;e;vF@vHGmV{)#%Z@Q8{ zSO0Zb4!7e7eaS!f^Rq8kr}Mrhxj_s5WzKKLmHZdzE3gNS!@S?g`-|T?m%a9WeWUnY z@g(Mb=2`l-;%+RiuMNJ-Uy0xFYthRvx%EDJJNDJTp6-fMv92_EKpLXsW z%zLz5>UrN;D6Zrk{L!5%iTZ-yWuDwV-IoWu+6(;vuK_~Ax z?4G-^A6C+r_goLq$zMwP`@GMeGmd<!9*zxghyM34 z`Fl_G$M`Q}E!?R7Yq}|x!e!W4Uvl9p^g#Sl-$eWue;$s)ydSxgz7dx=SBAcou1+UU z{gKZ5*}v%Pg~|ns{NWA$P#lZDgiPSf-|@^(;OFua$melB|1+Ou56Rq={WWt>o^P3R z^SsD&Kl6E4zxR1QtP4{QM$sW;|e-QJ0okb@n$vlvG(3dU9^KXm(j+lAoFr7UhbJ%GYWFAl6 znmOfW=NjRg`io=cvdjg)*SA%@JZ7FbL4Tm{FF1vt`J^v@HvWVi^(Du+h0b$3^Fa3K z?A6Piua3z{=h4r&C-ZFP_~gIIX_n~gjhP2#(8*7er!I1T=J;3W+c9%t=I2fBNe-I5 zppyRf>Zj<;$4mK@FnPl$edGC=-)GWA@ozW=lgnnGdYzxSEc5S+?hoG4mz=U0-4^HK z92|^pG@vhM}IlY9*{jX`%gLNi{UuTKAfDXIsc&hD$+;khw0VW z3-ey4hQ3|=tLcmJ9{xbAf}`;veQW6K)0gop^RJ-W(R=B9V-Z{Rc2PXl3=zn4IJxeIxiS=m)V4KA@iWKR@u3%k-nK*H;ne@;6{|{_LYK>AMdn z=}RuUke-ZP^wq;A{7Lj0yb6;i45DAv{{`K`gZ*JTd+&Aha!hV>p7X2uPh%BKK9)VT zhyHfz$%m3R&Q$M$$t}v$oiY1xY4?=GLh6O-iF951Iy$*&5&8=JPx}2F<~w&MKkrZ8 z=Kr04lD?cSgTKQ2)mvlUOBGeG9s0UM=}oxXxpj2*@~Zr5&efzFhWgjhf5J|fT(zyf zyRZ``2iT^60RI#{oPLA8)A!Me{$9N*e#Jk6JNfl6d0gHLyvHA{|3++zi`75HiTurU zN!-rgNKdEp9;QA$U;QH7$sbH7M=!*GgFlSU`=)99;rxH%WBl&87H?C33h%|~>MQ8H zZ_Ru43hE26gTCi6dEfc|yR7Foz~qo4o&S-a-1lRC-Vaq!?~9M9ccX9db-t$aKIMFR zxBldqW9X{RJwtENzZnN$DZD^`4?6Fw57GPG(-SY@zeKmBufeVSC+HRQ96W|Um%aYX z_j%?g@Jsj!H1T_xoF;qCQvQpWd7&HK0W$~Yc|P9n|02xi*$n-)`FTEM?#rH#yheLL z=IQL=H!I{hH^hCJt9Gg9b3F6z1bvxL8q%L&a=Av%XHLqV-Ai9`u;d3L^k)w#rtf9` zxpZ=)%)y!Wv+qsOKOT2D*OD$z=Q-JspZwr?^*l$eRnNXNT|N6(=APt#$$yf+)zx1K zU&rhh$upD3WuEBc>vqLW&Yi@UaUb@^y81KcCI`#jTwGt~ugoWzAMWesYd!boSS~=n?ue$0k?$jDJ*L=8^0XxAQZ{ z{+XWR+;f$NL>e(0i@b_b7 z^@EuGC;Q+QecRPDrzSVZe3iW~`A+td?0YlZlf0>i`{wcgLC?Z<{AY0%PQ>j0*{d?w zw^cul$zP_@rSkWnhtWmpu9$r11ND;FPd)SFRyuiLEqx#21?t(ilfM*EPrh_QU-F~F z^rZNhylkNJ*?&r?|Hyxm{*InP_ryQoe9V6HI9=Ph%NUbru^*N*Cu zZF6o9Kl{{reixjk{vMt84Dawe;$HRAblx-ENVib0MOVY@qgB=K!MpJ#9HhSucE&a8 zOE9_K*XpJ4*Xqf~U*I?3SEm=z+3T}U@8l=Pszh(WfzDro*-M+y<8UN)$C~(-dp6S9 z-?KMoPfi|HA>R)s*GpcJ_bZRO?;w`MzhVo_`-!|aOHNKA~@3e0n{u;J2a=(aBXh(aD8g(^r;X6R+ea@2kO2{+wKRuD&0! zj=tnG<@mSZaGa}eDvsep!A^fjPI-*ps(&Qb#{+nczMJp>CYRZ*ZxnwMT@J6{AIC}j z^7u5rwh!LP{CfPxn7sA4dQts(|5Zd^Z}q(I9m~I!Uj|R&C>*VCFx`Xxlx{@##mo7p z=&x`rzcnUD&HK||LMCwL?|9}X@N@YI&9$=h*%@VhcoYg zuRn8B_S0?r%*C0L_VY8RCZAi$Z}00R&uK{yRnK!T`9q#d2lXZ2$eh%{xj(BP!B_d2 zqw3R@Fweb%boS_P={(Q(((N(x>|#3m#R*?G^Ya8cdt@_u1J1zPae(_eV+nrdo;)}A z@H3C)IlP#koGyEMU493=3*Yp0G8Z@EUx_Q!@1dWevqxnvP%g;4JB0Co{yT6TzY=Eu zD6gLRKXctAeX}ro$_P4pdUBM^Mae|@yz`>0RA>~Gnd zm-#xyFndIDzByYJI`dKHxXJ2m)UzkO!_V9~ zlisW^d-@IhZkYToIsI_`nTt=-d4JK{x#ShuKlVT&sU7ojw0``Y=w_SBBnBCts|<&)%5)Xpp|Xm_4?i z{!Z9W{YN_c(L(;?{DwFKtE$(gr(<&3!Rib6voU+;C-hfX0(&_35S{$`B|3Xj-qXCQ z|2g%gn0&7%ot&wT{%iQj*YD(Kf9|FJ7=H@Ah^|Z*#a5Uc`XYVT;dyvSU-Gh6{Jc-f z9@|yl59-6{b=Zxc++ijCBK`-HgJqwq&)@By-_yypYVnhIB=1e`@V@@#nEiPgy$|y~ z?QQoZU(DWq3BMU8SNhKRkNB%F?>mxTt`eo<(WAed|>DsshYhm6$FQEs!ubH36Bm7;M_Y~df_4=>ELYQ2&m%ij6t?351 zPk(ZiDf}jQA-2(%yyHQB0e=DA6sz<9LjRL4Oplla1 z{dC@^b*GPF^5wB~^2ANfAH{m=m*W21kB#~5=;V*TqR&!ahAsFja3@Ywe;V_?v8VcQ zerc?N8*x6C(f>T1oVh%|Q@$SN-(M8g-;iGslP`WoUx82P?}0y;z5dMidFChZOZW*4 z^n04m$;^k{{Jv(6$n$En{%_T@x7VSS|L@;Tb>L@C&-_|XW1jo{=q38IALhB0&-?62 zbM@`TGQ=!!P`Wtw$03+KV-7tHlMiH0%Kn;tFY`$DwugLQ|Gl3-<-RKF{pe@08b3L1Nk8{e z>gT9eqVv3emCpW^JZiT7=&Gf1ooTWiFh>zsr4X@o|1~sLZ9=OS6w9r)lNfTbMa!I-R|CDgB)LKcw%*ke$ocY^z48Nn_woiISTpug@MgPT!-r&b`?Os`0n+vrmkmlUps+H-lfq z@7YW`?@u=A%lo9?s87IIn4Gwz{^S9*)yLxq_3Zb_2ivRnR_{hHN=*zygi@r`hdF?X(OZ>U?YxKkP zDEcN`&%d6&3R`3L^2_z@=5NDi_?_wG{ZskLMXsg$U|CE~u!VloJ)P-VbYZ*zo8oDG zZSV)YTK!9UA0}VvOHac?`n%G}Ut00A_byj|AbslJ=N=q}W7H4ed;IM6+xac=0i28{ z^&h~p{Ny&D(aBY|(k=C`#)16X>56y^FT%mt1KT=x5JzGeT%zySxDscmKaF2w-n%|Q zzksEkOKy^!YdgQFzVq>K*c`vYa{7~Z?BoyS7oji2t`1wZDW<#T&8 z|5JW>EWsa)d9T=nZmMr8{RusjE=g~s%g}A;uJ{d~EI5}Q%NUOR`Ss~Rblx-dcNCLXS5m)%pZ9;q=09Yq{(B_X9<6>O zHpb+%Yn-dc|0QGsXa0_7egZ$2pFrmMetti%!R+JNWAgc*JwE%%C;Id6cEd{#$hB+ZO!HOL>0R;%82%?p*fMh)eFL1!-kRt3Nq+W*2kFcmHR+pi7}jxro}Ux=+1nbbSHMwtj=mC@ z`6oH%E8`a^8X zUq)yCEXGeB(49_xvxKgM3vjx7ZlPz=**i1$W$vD%Z#8BQ&i-*Re<{v&U-G$y{NyRg zfwD&>NA9n$k@K0;Gr#?rzg}PF(#HJkb;%Rf=$jwvtAN=TldBcuCzs6r(#iRO>Y3}u z^S{T;r^z>x-;Po5hS_^-)8D!GbDV_9SF;a{(?3o9UU~|y;Xg{}{l-u_bMf!>WiNhB zeINe>UWeHuUZk_PWdEI#KKGo)x0m_4o+{j$ETxCi&*V13C8 z4)cfb$6|B5UOjo(UHmclta|d9H|crmh3U?C5x*{d9=(pPirFKxw>RS7uD=<5ix2KC z{0aO6^ieuF!VmoYcn|&-vp4UglYcIEPd)xj`a(K;?qq&@esatI(8=vu_`ZAVe@T4= zuESmG*6Q75^!-R5z;bxM`V!20ylQmb6Ktcie-Cti6#oSMBR!k`3;i5DnI1-;qFZ1Y zd`JCzdK-NfCKpN$l=tL=^moFgSk1ZJ^hSCEZo!GT6Q9D;*b;Z(OZYqz(*5afn7pYD{RXziea>Bt?f9E8c}-dM3-EIFygy3b@{sy7SV-R==_>Tgc!*z$ zZc699^5$r~4-4z>i^)?LslS0$)j!6f{Jd9P!B5_}Tm51Fv7Ez&nD<9L=mj|4`MiI5 zjDH>fY`QN^P~-3-RDD|KY3o{`Iz}Fb4Na}@;RLOAahdY zkw)&#{Phq$*4NFRnCIadeph{K>HV1f^l3Vuhwtmp=V$ije2#vYx^vwz&#~;SneQ@x zW=^Q@eCE&W&+FY&fP*o4$a&6vfOFN859Ray2L60Zu93MTbNy8JWX{aIl$cE9uOMN9ngP^Ktfw?0K1+lD}kL{U2ZFMdvd| zcnlU~pUm8QQeRi+vX6aBXK%^=eUP7A@liT+@(%YUKh9kHIsZ-ldoZ~`=GPbanTyIf zS04MTCm+bZn0-0(?qYqzFuCnvdYt=eVdkRO)TiP~{2rHLa+cN3XC6tuk$EBW{t11x zov(pkVdla$!XT8XU_k^xtW;# zAbVXs{ZrJ>!pi)*^e6Z(CKvov-vHdLei>$;Evvo@vp-Fu@73QBPhuBLe!X3PAI$!7 z7rjA$Q9Aq9gLL+o>{T21#hgnXFq;1uCU-cZFFEvb^-=t_bl$gQuO6*_uKE$2#ZN9m z7L-;$jt6lj{)E||ve%Dv!6Eg$XKT-|#-Bvz{Y`Iv_QaBO_RHjOpYZ>duY<|EUZwLs zC3$Ea51Kma$!lu5uQB#hpGE(U&c3&ppPaj#`T?w?{yJR^vv(G!|E_NfZsa$mlh-w& zi>Utzd+}@1&*MTIuf7-mz+a6c`7`KS>Es5p`N_>bqrcPF1_$E;_3Y=1=%SdMW553M z{P*}hoqTTzUCX(7n7#Wq>MQV79HZ}9yajJpKa5rRy>T5s?+<>UOQ{d1gNONV<5n!| z?=y&Qg~xCO&T&!R4_2jnVk3RK@EQKKxQ&0Beu}=6z8RA@p5^=6$bT4<%XV|VJAV^> zAzp)(a0ez&`iFBv`NQZ6bZ5F3-JiaKz8YV~+i(arz`QT|*!h7t14rR0{XgP8{Jj5c z!e58yV{)T@`fucajSt{b^__SLe>2?|cVpf=^wswb4!|GveUIHR@0Sid9~0(;YDH-H*`?u`afDPc8T6JxhIlXPm3Q3ogT7 zLMCwL?|9}X@N@YIWKYU`n9sk=f!VY3d6LiJd;C6U@5<*=KCkk*lsTrW&xOowYw64x z{prjJH_(6fb^c9fp3Z()Sv`A0KDRTUWv=Yt{7n5N>38YO2g!Zd1^?8S=Xjo1nG5ez z?}S%6c$8j&*(WNh=ee2ZfA)avH zIG1@p&*7EonddVPHPJsoJ@ZTU!Q?<^t0&*c^C5G0@{!DAnGdG9|54{(q%$A4=kLMf z8_7SCr)3|@T$TJH`)BgZ?IV}5I=A*s-KFzR} z{yf(w@;hMm{^VM<_}SlnODCTzMklw(9G-nBbAN4LcZ2>KcsD=XJ1XelKC&W z(q#R~Z!;&q;(R0Z?3oSu$-i3Cna_vn&mNe0yPUp{)HBy-p3l6Qc{}skyZV3a{CGP1 zPWI|K{LF#Lt;RW*d9OH~d2|iE+xhJM*^4sI*4LNZBy;E}=Ptl;cs-WKNzU)X7X0Kn zx6#?7vwuz1ze@dloWLJ~TlsV8BlHw{HD-Uw-abj+Me3P*->3Ir_PIg&vrlwY{|{!q zPrlcfpZq_0P<8iQih0kFdA+}LnZFzAOO8;6PCi(Peg+R=_Wu)fHO!uKfd0(aTZ>=t zOVF3m+2^xYPFJ6+p7%r*=@w`IZ$tK{5&Ug9%J~{}_KW0Nd-?BRVeI2v_T_K*6VaC~ z=td_$c)+=~{8scTyqBN-?J#{(y#&3H&Yr%A{~0DnOK!K5Kh(J%xD5AWU45r8dv#Ow zC;1EM=9u?>{piK%!)Nhs^}Nq|ou9n>CH3r=+54W;m-mC+^mXBv#^g97)Su@MqUX~^ zu>n8%-efv?V>|jdHg~QQeK|J553!KG;W(fF1}5h|t^PN>AFJvcj%_jf{|tR!VJY>R z_yYebEYJTHF5qv#Z?Tg9;r66I*Pr(oE9vAOGxcra=e@)ybaIf&`ljF-gn}RaVJcpX zd2dqKeaY>h&3n6%{OXwZ@l}1D zr=5QSXX`7AMX?PgR~}0bcJBY)kI)5J6;JEmL0^e=`N>@e^4noA^}@d2Kk|>^SZszZ zu_Hc?N8NKPF2eEnn!d(Zg?}AgkFJE>`P=bh{`d4TI{z->A^uC4d~PZIss1+fGQ5zV z_W@h!QtH3MdN^3U0&eD)q5IP7@iBfiKi}4L3w#d?;YR0vE_?l%@AJ%0;Fs_d$mdj^ z^O?6Z2jz1ob6uVTnU_ZRz3zY$FmugxI-kGEUoxj;ZqMgjp3j*xwmF}NT;YdY<1i=m9uVU*_jm={&E_)0a6k^I7(}a{4o8<++%7_Xp?i z!aV;nPfm0Gzn|~PRR^eNpH1%gHUF@CGrv4U7gHZXH>5L9R-lKeSHki5g8DT2Ma(?( zsCqkoS32|9EBtQ!%%9068mni{PtKElD0^>msK@kY9%$;`%=63Dvj?oB_hNFQ59#E1 z-_gm@l9MiX|18WLmc2LmYx0^q^d+y){QeIAYRq1rIW)QEO5euf|1G^GiAR0lv(yMjyn?vB?)_ z@w3kqcP?}CRdn`^{_W^B(6X`d#PGp}(Q;r%T{v`8qyVv&U^wFNu5f9ig-L zFQzxCucmLI4`W|^9?RIZ-d`@Gv+vHO`^R@Kx#3QJdt9OZ5nY--g*|btdL8;Gy&n(p&+#(>#=LjkM}Mp@IaI%VJ=~37=+FC?TlvYc zlb;>s-=_Z~ z{++mjpZ6`v7m|-XqVG@oCelyP$w3~VTd8-#ytgh-PtiAq9*LjuFU90bpQ`Yb=L(FL{!l?|gH5F8-VUPwdVA3+8=bC-ph}c(>AH0B{-0d{oNIiMN4*s9{Bk0dB9izhs_D4yL`}zxVoi3gxh?`xesK z5As~wrrr^glV(plM6d+E$s+3yzdv;Sp}uFB8;;oB<6eDxyZGJV5vA^$GSd$ehENzB}KE4|YBZJ7Ca zk9svsZjc-%xpOt=GT$$uw>mci@8g%Ezo)z4JGcsm=qrjX`I*Ps(u>tIKW5&_{GU7~ zIm$QsUvs`RHpJv)$@|9h4?34U;1l{8tc~6DXa8+Qk5bPZd^&b&3;CH}{xY&KYu?$X7{}B7|JK+MXsood6 zV{(D_>Fi%C=t0;HD_}89j^4!A`7QrP{0@_UXWwkD|05i)FS%|dehGf^)Ajsm_&ko* zSDCI#CkI(WpR4{Z&g1`$&i?%sy-EECoR9b8Ont|(7C*W1Zn}bca*n0@v-_l&{1>aWrr z>AaUou9khbj=o-4PyZBp08YdfzMs55twi^YtgEFHEmhuZI`mF5IW@ zDy++YfF6rC^JmjNFnL_^$GQ5JsV}GZ<1xHKeH`|~&(*8rCH&-0x6l=_Gro-FovVn+ zwWg>)iwD$q;&!}7-LFeQDf%<@IoKboU@IJmOK_v}nV-Nf;V1Bc-_ty|v)|?YROX_+`Z9k_rzc|O$$X9< z;Xi}Wup9j@2&zF3DXP&R@Tyf0vH_x-=QhVLgAG2@fd7tNP@__7B`JCMCTu;oL zx`EE~IQwhn<`urLC-pVO5&Z1AZ}PLJ6{KI^RJsP{dH11u=A=A_M)Q+jWzNjLl-wZC ztIYkG+Xgs)udkQD#aVBAxuRcfK!v_Uh*R#{A-R=K8nzmH6xE!TfOOq?t2{D@lVjn(LbWIzf`BU;&Yh&qltU6XJ#*7p|7+0O>}a* z8|ds)=hF}CPfj_N9*kY}WzYLFKYMKU`f2=c^ymG_J9P4mZuJ6~_2)|X|MrTjn!+)2b{Q7MECVuw0*8V%+t=<@yU~-ug&Rxm>BR+)#)obEo z*hqafCf~{XlvngUs(t}xzfV5s+bnohy$ohAxyL=9V$e(9YDRlHd*?`g_Py=&LwGG_ zPfPCD+C9k^^WLnaZ}f8KvJYLN{~;``J|45zj#PhypFON2-2Hy+o&9_P z|6weop4=mQWAd5``pW66OwYtm_<28Aoi2k7u%G^~a1LIEoiMpna^7yvUx_31CHHF0 z--5}Z3+o%hzm8svo%qR#4$vjBEB4j@2K_bt4E;8peRu&sd+vpFa>F8YY3FXGTVYN9 zA29j)YxM8+Rl=40{&e1ZT+Hu-hw-$&MOc>qANnXggRY7{@^8b6I39ncFS+Iv{xx_Z zF304cwdlOhZSI~T{2rLRwjP~)>m|B_bC=S2PcetTouBt~#pnms%hN6C7wEjtIYzg_ zRF$veKM&&7fG8|NnA6Z~gzIOcukD1BF8A@zf}47=c59E-{2XFFdXXRE(Q zFUQ)L+_yVDL;vsaEbNZS(}w9k&2K}uq5II~=x#U>7hz$2EimuLuTal>(y{8hG4Fpz z>D$U5f*bjL>AWA=NY};WaPQMM`dn||d~)u3>aX*wVnhBfArm{k_D+8G;&bTCb5FV;^WgycpuWsKGwJKqkI>0&7V_uv z^ErBe{}w-U_aXiV{LEd+eI8ZMJd%ANb4zFSefiR9&(dw;7xxl`uApZJ;EGUu(}CnxLV{^U@Zk1Ofhf$Q`&rGJC9`Pr9N z(3uBM=v&LrKD3u!t6qX`Nw>phI2(`YOP(-@{=NF;I195U?xR=Z1m`Zs>_I2hckz4D z*~6;vlTYnYPhNVQP99c4|BL+0*-QCb_<0XeiY|#4=*#|=T&S!15%tXd!|CL5%k-_m zF_?*67=Ug>aj`?29+?|H1Er z-(e3thV}G+PbZg{LSLo+8J@rn>doooFAezFf0I8X&;C?>93~GeMo)9!_gI?$0=){e zr?#P!kIbhp!H#&Pd$Najq-)?X9D~a-d-P6v5x(r+Yv~qe|L-5F(o1nYZqUD<9*zIx z&%oryr|F{l7Sr45AMg|YFih^ak3OI;xyC8FmU?oVdJtj{$i=Kk-J9iD8{Np3~Tue^zH@duY_v2}-r2aCUTw(|R zJ^oSb%kNK*r;B0oxL$N}-c57^ERR=X1@~Que*FL6zbi@(^tf}^;T6uc#(w;~_v}p< zQGcC&36rNkqCS&z0*wW6P=C*nqY0+T-$rr*HX*up*C=|XfvO#Z)2{pYgR zpZPw|`~-dpKY_!3Pha!Z~t&{&V`0`(@sq%Fo`C zJ+2|YDNeu-+?zeA9Y1qX_MPm%gVdkI>|rhFbKI9)>`i|5i{ycM9wb-F{ys;4@`B>- zTg1=abO)WiGtc|o`m@((F3R)#R`q-E2)^UKM=`m^ZFJ_cFZHj&vY5STkN(=2`82uR zO#OLYXP&9Ae~5baj?6{L)yC-Cj$QS4#OwJ3a0`DNo&6*8_!9nO`pVF+&_ijmpb`BO zqXu1x&OS7RpM5BMMNNGj)cauO)MNCwnE7H7oq2P)d$I>)uiV4WUYh(aIo}2DNnZB{ zI&`!cmT8Ko}j;R z??Ac8R&*{6dvGb$oKV$N_%*F5W-@;MOWgni*FUL>Lei#2-%${6A zU-FciboP+!8Rt1SLj4rIm(Ct}C7nH>Ib8v7z-7*7j(?iJ4tL{0%>J7_X$8NQd)i?! ze)i3k{K0rLPQ^ty!@0coxs2X`E%a5!EsR%>-!h~1+0kK=SS+Rfj8wI%zLKn=^ycv zgLk9f#j|`M^mO07{NK{aeGbr9sSm@V&L=N;i9b&Ne0&wxU~7Fh;&D8QZT02-)k%6Y zmen_w-im+aH=wicColMcf0n+pu{0+4`<1@6nD?tq>B2Y>XJPW09dr|X!2N46dDd(? z?^*st7r_UeAArBcE7XtEU(?O#ygw_&KZ;e=ucnjZ59g1@ypJtOC!eWL-|u`~?2B8l z0nW#I&b^NdusV*`SCM`O|IPmdljqe|FUhZh$)ECGV}-t#a6dN3NxkEPWw$MRTM{VQz9e}W#1Utk&aiF9|&`=ig)ui@uC(I);C_#KYH zf9l^!C->?^_fY>OWCCaYj%R)XKbN0Cowsg!9Qw@_CtgGjrQH`tn@K9GN+$xO+3dC-=#`kX$9t?N#o}=hnT>XPzlR zXCE%2e=7fDx-y+PEYH=>c0It=v3Ah}ucs?0l?KT9~DdEf^3bmf<)|Av|0 zUZyjT-K{_SboRu|?U@U^sRLK2*>C+>K5?o_%gUKXYmJg2nvot=s9$o4b7<$t4e}Z|5gx zYfiVtHTrtdZ7{h*a_}nrGML<^rt=f{$shXB$yc(^WS&0k+){j8e>=?kf~9mV%s#r; zx$N0H)Td#6%)XZ#a+!0=WnHBQg8;b?$kNUx_||?1JQ; z$=Ca;XK&1YpS}4p_2e1J%NDzEmilQdi4$=;W-mxia)_UN>U;Nn&d(l}J#q$rp}wZr zg+GHXO8D!;nF z=W_Zr_2i@@a!&nUbaKLm{JcMTgD!{Jn-@6u2V929_ZQK5FMGT5gZRIqx6#>~eftHc z`N@giq<_TooF7Q%JwO%yFn)5|+5DsY`gC%EvHYr-_cX)w-H-pB+x?z@y>lgTGXF35 zSN<*>z<-F&d*$os_tcY5Ji&j2pZ6tyrh8*@#@qDw#lq@e)5&d;FCbWJ+%O`oQd3(ciB>OYr03%B3_Ob$Co|3Q9o zqP+j9&c9M$JG_`*$^SqL@oQrpT!5G8zk>b-^WI{%dh)JTbn>oy^tZsFI0WnIFNepm z5thYu`isz|u>wv~Z%l8ahtmt`>GW876L!Z!>OJY?>Z9o7tGnr+%U*xx`#kd#_$B-V zGS_9#`poZh_V87HZ?jk4NjKEDhi-z|cjl}2;^*_TEuB267M*!&fOGlWu0dy>Dn(~r z%$zjNJtfqWJJqHus<)++FKnjo#mqg~cQdc|QP1b~$L`B>V+x&lGy7tmUwNKvaL-!I zb1a|tH@l}LW={K(KIz<2%;$gR-OLTiJ$CBLbLj*3wBcvpZp6?0oac7tw(Mh1I=5E; zE^N(TkC_9qPh6`nb9UyeTl6ne&-^i+&b*s__*{MmT#lL3`nfNAZRX4_`Z8Y~q%Xp^ zohw7n#J^+a|Lh&v6EasF(bvp9o$2H=>+*eJ_OU5+_T%KOnSTenCv$H0w66T*U@Pev z*bo0Zr#S5EWp62~Z!16hZ*rf^b%*p7(^nVA;vn^_=%#cVyqmugN8$wBsjm#3eKB*! zRQ^Nyo}=5-nTs>uOynoO$UacTx$H-E=RJr>q~HJ$k^dGDosZU6WGgI1iMxwkSsTH`15 z%b30IsCxF4?7ai{_4Pl2*;lidci?Bf{?NGv*aJ7?^ZJ+4c`uM0DD&(H^`rXR;TnGS z&PDvj{Jj6jUiG+o_L^RFa?JCbtHqy(7vgI5mh^-4VmkZIP&#|pTz$zckEs{NTh%+$ zUtx0bbJV9{_NDCkbM#kMUx#D(oiOh;M$i-Vji<9uenH=@-U;)b<{3JB%_sCLn1Apehkb*?L2j!sUJyr!o*6kOmB?J@gj_S{PDNiH#*-l0GHadUoho4glm zt}lCG_PQ?o($00nA(;KLslG=sxkpvHD2{aQQ!L4EKp&)&cTeTl;}4{>m$sslyDiYy z18b}2z0|Mh$#^+#z`PIW?tC}?F8T_bjB7AC+-Q0W{y+Ba{p;sCegFSe5u(VhwMc^! zA)*wPN=-@XolQ|TDSd28R*P(kifS05DJ7L{L&BhJN=-J|EYWVVt0uB3Z(+%MQ6z-E zKYVY;{4oDPD?i;G$9X=U*X1(T>v^8XqqQbES#p_o_{k-b8&vmoO5#GSt^aMhDEQ~X1((`d4&QiaKZcXQ&W*5IA)>O|u)Y)`$gXBXy^zT-0 zfHU|v`F^X=xqm4_msig{(38&Bz})-YqrU}yiuda4O;4sDq(8?+cmiLI)<1w=N1vo0 z#pC#k`kgp}KMjxKrRsmd|Mv`%-{X#k^1wV%=<3;OXltDsd>LW z<9zap4xTfCpM5g(LgxJS`tttFJd*h@^K@6w>8yV`4#&*L+1ryBW?snNv`>HXi@e9~ z^Ze|4rRlryGv}t#J?Xr^7SVa1E~7Ii+(2i4y3upy@;}Bgm^rAmzSsGA&zI#F<$sBN z_~q&3MVbFIM`VA^yp|m4J?Aoy9ieAn_TaI;PUeut^b$;-($cx?S&h_Z@{i&*{B1ZK zJ7DITGw9cx%bc~FP9D>oep`R$y5wKU53^ro?my4Dna*X8{D989xt`7(n*AfWS@xUE z0fY5te_cvvF8kHj%^a2eBJ)q?xyI0=(uWKN&$+*#^>#((f%z%6(VUa0S{^fbC7U7T)0AEx(W zS^QkRG@bjfhw0=^-_kYoSEB!d?_%Woeyk+Fl>V*s<+vDY;{r^6@(BHk^H2+`6Is) zzc76OlNWcVE8uc;tl$EFIL6g*{}Y{@WG!78YwAmGIFi4H-`u~?)%?f!d+6`*A^uMK9c+Vj)qla{>KD_c^dN^w5;7jTchWxEqjb9Yc$HwYQG5JRBKfmW+g~^w@I6t0W4_Dw+JfZJ?Y{B1* zCGfENJnY0DPgka=(U;Qa(qGX1=mO0BPx8%6^i}ciIiJ2x|02wNK{vXQzR~nHdImj@ z9)acgpW+h!aZHX=Q@tI(N5~&c7olIohFC~_G#=m&!{kYW=yLj!XFkhMF7<$V^4&7( zhxp0y3h^uQpQamNH-7G?y7QCwuTvk&zZQQld;O{J^VCn^kMI*X=J&EcW{=wC_cwcQ z_T_wzk5SLOQQG5^7@5{cLeXpN-=Jo7J*$XnSE%m(QH0|kYu!-j+_gTo_fSKQS=xfYRE>MepGT)E) zS?0(}>X~mcFJw>1zI4?2>G&LGUOw(QnQyaq7w4D5e>yjrCJP$TcVXu2n{+UY8^qlNlnNPA`-LC!_&c*C=-+In|e)gab=%VV$r?Rh|c_=wda?I>A z+1DrOPcC_}=f8%TYaXYQQ;%})2)`wrdGu;}hWf`?m7lq037y<^FuexX<9#^Lb4KC{ zeo6W>%s!qxJ9BVyxa{4T9~V3KD<+pnzIc#7(bp?aC*SaETTqRkeLeYFC+D&+4A+-j zq$i!dI5}GKjeDHmjSKW=f4PFc7vI6V^>xSV`Pq*$cW1B4J}^Sx8(7x)-|!IsBFsLV zJ?MS@9?V=km7d@^$&a#^WR84YUk!ab=}+i)F}XnI`3?Lb`Y*$$FnPrQeLMJ>ryrwB zs}IBEJhSK@^nHc*<4H_z`YL@lZg;*FPT(iE&3(#tep7v0@I0J^gYX&sr{gAm9jwMr z-Zz^*fytFx)7j&*UzFgl#^j*m=~2E;a+62-Q}~OpENo&9sqe$=M|aW#^<}@lfF6rQ@DfZ8 znmzbE&-sUX?ki@|HGJ^ggva!E!8w@xxwmsq<5f5X>*pN(GM#&~ZFF+5?B~gwYO7~o zecN+B;@84bSP$>O&H5YAy)ilLUb+G%FWTnZdw7%jBDyYS&%R6jJN^jzODxKthkwUg z)#uYaunNB+-Iq?Tn4EKrdMRv!r}_6Q;au(^E>~ZNo7Jb$Tj@3UAV2p357Esqxqa@9 zvd8aJ@8CJPx0y&^r`{B2@+;$7{$V=zyJP8>)TiUK*az$21N!g96`1_-puR%*jrwwW zBev!5qTiyI(aBdQ^FQa8$4%Hn{WQ8FU6I~`H{yrtTj_`BET*!MTpJOBae&@X~gU;t;=9|1H@}6$&b0DAdjujO5hsV^jU*>ZvpG(Q5 zGSB7nCv$Z6hJ5bkJ)QSc-iLd9z09e3pX75pds61n%$<3EXTQjNmAUh7&z6FrC~u^Jq)`JJmCPWFF5RcZ>QNc$@uT4xK#d?{spE%&&EP zoy1z!hZNRPSL-D&U`V8}WAL=`ee-G~DUq^pJpNHAEhN++BfgROn;rZ&xryKIKPp+X$ zVfL3J&i%qKPG5$}FOw_m(D%0b&Db8ZmmbkKo1gvfYI-_gl%!vX`$^_n3mu z{NWq^2Xqrmo;pOmHvdmJ5!8_^5YXVWLK27f7i80+HG z>OWyw`~qiTHT==JUDyjZsSl@OS&zc`_7N}H{o9O+{d(`lcQZvx5IOB ztn;~VSxZ+?-xlqEM-}J^_&7GyUzF~SE%_H=5B`YsWAd5*p_8X3M_uRK*XpfsH;z*; zi;eiXUpb4Pyl+r6*3_Te{WzWbu`B)kU(tV1eGP8FlUP_^Z@M~t2>W7z`gnW}d#YEb zN8?La8Xv$#_@;BO(E0Bga*uYH-$~zm%)Rmwx`Dp#xRE~?TVoX*fo1d`q_3qP!87?S z@M63K*Xvu0Lvexn<^COVf7gcpB$mV9%U*x#`#kj%_#^xTvOjk6`@9D;x8;45`7rx@ zK9BN#&wCDq-;ve^Y-A^ay%jan3r0ge|_cB+mbT0FB-fMZEW*?a1>tsHAnaBXjv~{r{bpWX|dCIg|9=OfSP3{A2WXWEZsXhs^(_)XQTz_05<$We$C( zzEzkxF7sIOfjiW9IhQ#%bILXR>H0F~BoBFypZzuaTk`DeRgLsb^Srz8Y<~8ojr`g8 zGhTt&!?Uks->G9hc~;*{dB2*Z9~!K1+_)grA(ZtZy*;(th?Hr{;fEYUyIIuk$aHK)Xz}Qyj+*xn4kSD_Xfk%vtO2_k7M%C zR{mYH4s0r{&a0T!2bumht3|9y<{Q3wZ1Yqi$5Gk;#XKjUt_Gn zzlW}aZ87&+*$*$^CqGy~SM;3M>2CN>{y{oeq(AgW`q35W-Kqntbjx$F6 zuj-TO`Se737p~z?q93Fm#fI2fy)L%H^*CPN4tfy%FM1TcnXZnN_;2HF{9>5<&!g(^ z^0U8QOD7*{N}q{CF!;CUoWSHUdo?V=3hK#mvUk43zYe$KQ#jT6?BU7ba^KKGy|Dhz zaU;J9R_0H}SMj8Jb$SX8;Qxw)uoYI+m;1l3=&SHKOrAWDei4(?Or_gRs34Oou2hdgV2tKJ^58Lxg`rJCmuZ@4hU+@9_ zh3MR)yiH%O{u#E!v6vjDx&HP1oAD<8YPugjkM*&F~4$vNLwAL}_+;uii*&RxRa zjXTx`o_-6zHIBwte4RJ(27bqo{}?vH+((_zmmIk_T@u&n zUyhsbB=*75`q$FQ>AUiK@{8gjY>Hj6n*K+yKIUGqxxSkGS@ey#2oJ0G!9PMKaO&@P z>L>7f`3WQk$sBUn?`?9H%%hn{GCyqAmp$+P{NDRLpGjxW$vl#IIr&oNnw`#7#$}j% zBzao)Er>Kt1`DW^1OvOT>mOMd;ETWQ-0>DF8sbYMLlzDFZv1fx9I6OmtUT) zNw>g}{Nz+?>1yhK!e=qL-(Y%&{=u02HTzN%eMi+>(%H-IqTj*nDXZuX&K<{I{N!a_ z`1A1*^?vja?1q{9voAizpXl6!bW6-VKyuE^?<3U5;Q-9O`UQOzF7*85m(TIP;O9P~ zD4l#Gdtqb!->GL0n@(q6{951h{LIgF=-fw4)z_GxJv4h%fA#EhyXowME9f7bpGz;t zFYy(ejLr1#!Q=zk&$1U^tp7B9FJN{4*K|vJ)ew4(`WVc;#Zo#sz*PMu_**ge1nt$6 zAFWry~WFLCZP&%cbWg6HW=?lPJ#to{+5 zdx+%j-TBw*YmPJUg!(f2D87wX;2?cX@N@no+<_bLbbpVvbS39!}b)$1{bR~FZ(_r?eCH}wWs92cv9fb;Mb+@fz9eSmI( zxxY!CI9A^*%)Li_{mI8S(g$&v{;pV)-;RC`H}I23CcjILwp?HCF_JHq=0AnW+j770 zt>?a^K9qh0tKpqkQD0a3Lp;LI{mltFd0A0?k7M$@r}cfqFNwL&e3CALt*{V|cfKE1 z#TA(U?qU(0oP4(P4f)ma_p;ZY`aVzn1pWv=fqd>|4$1tI-|xJqvu_OddwksQcjoyO zbmpLebmsZYZ^@xDm*suY!*k!k?2D`Dl9)L)du_m%f}a;)SI zng46(>w=kIGk5IcAHmF#c^~FIlKK2fec8jZ_hnzqoS(gas(ydHg6s)X`1w4~TvCDm zAm;P^Qx9m3$v^VG$();gr?0*nJSX#5a-htc*(=}Em$~83bO#*ldBy0Z^mfeLb)I_W zxy)h7q4M6%`zU!{=A^tgv&Vno`PrK@Z+!3TB&W;!KlAEc&SgH%o<4(r3s&)*lx`N#R0)03a=RNs$-^kq(;PbbgaOD7K+MQ1+F{Fgl?^Ijc&GqEjZ z9^FqTXZ!EHZZdvv-f+SLQ#A=kPD4 zH`3kcg>?3`iOFe_J2d0}!}$rg67Rv5cnEKDZVArE66(qMrqdm;kiLO*1ALiZ z2CHHA+~iTojjF1D?tJ#x()`?0B)^%!Z;WSQa=hdfO+0V5`f}XNA4zw_=2%L-Ki0!X z@iu+iFnQc(>dAF;pP9XTiTYtoPTQPbkGnm$Jl4aHa4){9e+&ME?bVZ~&Y@Rg?k%&A zH|5`U`v3ojZUqJW4y4Xp*1tzzwPq)KH_zMog8$IX0bM@p%$;H>|&wcH?bTHQ+a^F{% zeg%_j9;W|0UwnjL-}h4i&*c9FC*qrUn!eh&2H(e;`U+!jep`AD{WB)tdxq|cFXC3` zlD8*U`y0PHw$k5+PHxOR5`O-Bfd=$w%>Bn7ArmTfOKR&gb(vb9vtXnWNv( zpU>&9oC|JOzYDV`W-i&MF}cz#I-k3lukv}Fy{)3>b3pu|bM_d&m-9>M%=b^w`CQNE{7im(+>EaMzKz+l zv)?37$onpHW#)|H&Sn1?NN3L2#kC-O=nKcyzvFUEoOfB&AH6CHS%@wJ#360IG=ekd(>6x znS+yOZ0AqFdP>5>2q}Eo3ot1oBs+XhfluLng1+i zzRaGITr2Zc_N=S*XAjD}H{93Vk9#qD^MB{_T|6f_{1JLBo{v>AxnOeE%&EyG+xU71 z^-smOu%vqO(BuT$)sNx=EQWQQyBOQy3H4I+-B<Ys?oX>zZWy(u|T_PFfhk2yEY^Rq|%z|THEjjoSP^(SX5L+9S$C4JeK zs?lro9mM1=%jq)uR?y|~J^ng;2rH{s#k+8o`UQ9y>dmmH zzTC6)=8xc?=6}bz_nE7ndz35aqWT-r$pdmvV*KP**@M^fH|g6f9rAPx=NrxkUcE#N-fD)t}aXfIf%LJwS4SEL-}ATwhv0QsNq-4!&aZ-{`3=&DJcI6mt@X9Sm%rTBkqrU-pE2uDqu|pT~J`9dhnJ?;(A8ALny8d-GBCyx+24 z}d-;U=(J4%ATpcV4^?deS8~T(YfTx%jxWq$$K*|WDd%llsqJJRrbT|J$b*^_w{b{ z{Okiy^0O!0qdpneVq1OfFnLt=jO=+I=+9hVoX)(EJ$r%YWUtAbyi|V^_2gYU>D}tt zdoyoke$RYAQh!U#K49!A$i9<#r;z8iz^2Y6Kgzs*lK&yzjLA8&Klb7G#_VUApNjL7 zPd4>+vu{tLGZ!A9XF8X8e;|J}Kl}2d{Kotvn0%@-{eSw_;C{?JpSiOS{|QXqm)tA) z<7&@6jLFBU`oSl6%zef}&r80Od^WkyKJ}`&1SjCXe4Qcmbj)7%Go5`ld1UhH<>Tlv;Sls$=s6nOXlEw z4mEQw`#|=~7X0kxnL{#f=RKUfBlB({=ZgA!WZupkzeD{;^>gXuL?`HcK33Ov6!RX) zdoXiJ-b;C}RP~&3xD_)eW{rIc+nNLUNxxQ}p&dlvI)Qe;GgC2AZ9N_u2Fmvd9^+Wg< zT&XX)Rra*J-^-}4(|;eGJor+6a*52*WA$Y&olS4Y>`~c&uI4AdsqMK%unTt8mt5*k zbSL%9sU_(G_0@E8i_96B_bygHL%)B9g5(0r_~mi3{*L$+zddHIUZ9?Q{~h(@A=w|^ z)>l_O`_}^c2xgwozLkBtp7TBQCFi=9eoQ^{ZgPd0>Y2|oAAiBmoZf`aJfD4LH-8dV z#KXQ`W4bDR68rNz<6ZpZ0FThS)W^{^=nlnzO%6seuBwM_R+~>GH;LZoVT&1 zz7q66`ddt%K8wyAd%gbTBunVb%lFWkt3RXv<~hlaCefL@r_wWVKUQ(R9i2SreSY?p z?ALGbzs51nW&cZFIGq1W>i8}uC*DPG$J{48Nar4Dsjv4XKl@zv#^mhDmEP08AE!Il z+%M>G{$WghG>d*%|9kj2CV$$Z?*Kph(G>nvd<&EN{8|6g{JZJo80>sSdFshE2k{@^|3HtRlOKJ` zZ^3U#UxM540=yB|<5=gCqmAMx|LI0|)wd1%;tFh~uNeIko%_w{^m*!a=stKk-k|0UvyaU(iJBVfYxzE~7U#Fg2?nC~+`B&1(!@AM+ zaS?9D*3PZPrr21$C;fZb>rZ{3r+xx|gr7h@FTe2nnS3hqT=wwq{T}bbY1quU%+=Xz zGA}mNmpMJ};mon!)j!6(&;Lp1b29Jk(|x_{VVVDu*JOU*t1ok0Cpz=PYR}L7lzpKe zKXYdG;g|V~J?CCJ^LOUk%>O6#?Z>>2Yth;Bljr3<*u-;}U%w3r;$~(V6U-E#=$(eh8)we-k7hKNIoVk+D9JN(n=Df@q6ZqHQBUl$_C8tf_}QO((a8my zI-k8T^L+_^75y#nuXq$^Vh>CX`y8FzIrG9No}2kQ^ZC#ElVeTNm;bILdDdC{?2p+? zKjUZq%-(Xl=Vm`jUOby$72Dxt&sjq6q>s_b8#4Fx<-e`31_sGXdoZ%6zTw;xxIsO8 z_dERLYL)0$^kvSxmac;1Z~<;}t~7ptnd86Fmz?KEI=S!t`gdUV$K*6)`Ae~=^T`=c z(j(N9yB+1v=VxBtOixu`gj;c!dS`qO?@~XF&ir^jT>}SV=Kqp(F`Vl;edx^N$)D@0 z-={tX=ktrx*^B%0xA05SnX8jeO;x`XJK_NS!|(`Rh{yG{$E$ILdiKKS`7dDR{6lp1 zm+XuG%OCGKkKp_KC+XiXImSoy0DYO;hwy9TYV|wm-E`*q&)~VZ5i|cMZ|dlI zJ23Z1$z83l1t-*JVr$P^OmD*MmD#ty(wBRW?7`*umpV6@&YqooY#~4U{O9yvoa;(w z@7Yc#FG=p0JTLjxqB4UeD>Cs^uy}8e<;OIemH~9o|JrGgmVSz$-xKHx$kMD zZzVtb>j^qJMDm@A{Oo@Z(35aFR`%TNnaOjj^2cEI&x6h-Un#4;5R>z~MX%AH+@>qP z8b5hL?r}=14^yv1e@6d}zK&joZTTBSyB*{N>mXkEze2Yj}R{UB2TdzuK(-Y5s2dVjRT39*bZ%te`(^0_?H?|DA&^0}0~C-Y}L&&%gb=9$b(Q#>c{jqIJ-5BsTSu4qN) z{hxh1^VL#cFLP_=$IQu@PxJYi`6GEm_HiHU1$iH2Z~NHS&-|J9SLTEr>X|e0-p`y} z#rK!b%k0tlygWla`}jwmm-q1L>e-`?(Zw;J+m)Srg5L=d!v9fX;rF_jl&1%!4)bUFLbs=*&yc^T+cC;3qglJ@d#S{y*_P z^@lKf;kWA9Q}@uBgCEp?4`v?ufzF)1%ekhQJh8OC_^EZvxg1U-ybtqXAe)VHBCKvO>!4@LGq#GlUse=%)ey8Sq^MqBv;J5T}pj|`dgU$ zm%Hi3xCFD$?{$6#|9d+7^F)4f&g^Z;{gMMD@0;y>JLmS`%lvoftMMOr0FyJd(SIR6 zfd9Q073cSHem~uXev>|*{tKPGtS$YvdiLJrpl7OIf_<@!{_L~Y@?YRjq)(&g&_nP! z{!yHb2h^XY$71f2hO1ZR&&L`3>`PPW#(2NJFX-e@7x7=k^VC<+N9bNylm8r@eR(~7 zQ2jyrH!P34)q7$e{si2|??9iRlT&4{9mSunZv`e#X-_BDxG~Sg68JZL!|9^*H#mrY z4mRN*p?A{BJxB6~;bQe>bbWd-4&@i3AEE2g&(L$}7oy-0BFnwcjH}2P0 z#QQAwaaHNB^lj0X`?TB#cjfQJ+{<1|Cm%dSZ^GnJ3+P*LIyT0pn7ng|?`H(w==(fG zcg7Of(D@3uk)Qj6H~1}aAa2IwpHt`-&R4+!{F~^LbPu{V-5Cu2Te^WI9%k@;RCJME1g!`tv@y-ShK4%KLB}Kl5GY zrpz;w)bn|kJtTW{_L}U2c`rQX>t{cE!gJbV-m`gMC!fjumG?&WtCpU(43m=;^SqUK zgL?9v%sH9MlB3PkpSdG*hL8J#$-YqE^Rl1Ur88e87x=U1Wxs8%FY`s-xBuoRPiaDDUfAq8$?>x9WdFQaJ@dd7 z&S&44sh)W;^Y>@^vL9p)&VE_axe2&IKUt96_&)wgh0L-2=;Vdn>C8_TId=y?bM8nw z`E%y}2lUrbzY>$zFICSxIg6g9uM57-Prk5+&Rm%}`Z4|o*cfl}i}e+qJuLH5=BFCI zUgp@$UF)2C9y6bPtA8T@2)@S8eq4)R2g|Bg3VnacN&D(M7q7;-n0t&}&iBD`*baZi z9E%P0W#8GyuYnI?a+lNm zoKELw9iB*~@yOO(HrN1tg!FMrx^WDzR#!GO7zGvwJSQCFyPtKJ5Df>?LndGKpoU4ZW zoxckwVe-#+^nHi-VDjnY(Bt_P@e|Ma7bZ{HqCOg*#`E!J{d@5!|08+|uHo;dFU2AJ zhBylE!kzfF{@?IHertLG-3Pzqm#2?o5B?_lPC9vH_PCeSx2XRwJq)kM+y_+DR}7Pv zB|k~NJk7Zs`jUfWk37yl6CcLhLo{-JDE~=(8waS@rf;FU;X$0HUJF0Q?E&4)w89j<#hb#CS=-PDh%zOFCSvspv z=9kA4{AzSPY>waHMtwtZ1YU(p^^M2eD?UUo$K+pIol8zPSUq`qQ9rk9^_@^ZhPi)C zezaU)a<-QAXZjCeVb5JiH`7-I7h`GlU3Bir4)6>357IZ`NPc1Lf!E=um^?A}+cWr$ zJSX>(pYx$0_g616%KQ8Dqfg=>Ea7?Q(3j%b{AQT{9^o)O8k4*K5i)^Of5%flf#1td zAahvW%T4^g*7f_HdE^?p16IP!m6>lE^V?!_rp&j=F)}yr@%1`l=K0K_+3#z4UMtM! z^N-Hu^XV))`^=5{GZ#NaXI_|1XFte1oA+$q%l$ofIaYQq^KSO&ymyl~ysSU_Yv#HB z&QDU$T=fD!d-!zqt$>W+j*VVbqubJ;N zFJ*5TqkpFU%p=KBM)9*Z?9g9wNAb zX3@zFuB45<1(}bhdtUa)mHLv0j8V^in0Yz*+|}xfo%@a+N+%y!!cR`!PyHYKc61d? zj`NCo_P;miVt9l8;q);qgq;xzmg!%Q$xBwz4`BAM>{r=e_IuD1SQWQ9*A+|ik7IJL zE9mS!pXoo$Z-Z<26X*|c3;!scoGSa^EdEm5tiL*)d#FM5Nt~)Ld(L+L8Mp?MqYT%- zil4lqB7GV@irE8m-;q7Jj_0)3H;7Qzy?_u`8e>?XTKe@&o^xxG7KP3Jzf3O!lr}SDZrf&&8!(T^F!Q%Xln0zI95Poh4bC9wV%&9>TmKFU~-@@>D;$>(qEl_KfM)m4{}I-1pfg33G3j6`nJ){=!JMK zzao7$HsbHY2K;0A4Q{~7*g*db+>FVA%IQn~+nRnD4`On?dGS4`6kV21jWk^|&JD%8aiIDrOg>+M?u1A1D%_2I@oUfPOxrt z$(f#;xivXNafFb5_=B9UORu4q<42e|_epw{{_GKD`I#${6RhBWq5m&*=A{+<%%9o2 zJMib~KaAOTGQTGu&c1z@bMvt%&T@V#JqByyH1+I#ZRmeua|A^LXlcjFwK zpng7`IbEDCdCz7LO9^0;- zd~h+H`SwqqH=LjSICFM#n|syw>tBt>a2lS5{q-jwP2QCIhvXVN_2oXH7oGWcndfD{ z$^7}TzRZ_Z^<}TPna+MzME@p!O?oN46|+ZYZ-16Q8Cy7)oP0EY3jb>QZ*+3|iFEeP zUZvbVI-w*)iKPNK6PmGE_1@@v!C>$B%2FK@2C12eZLUwg%KGWX}6qaA-dc62^D zN%p_w@X3#JPqfXsZ}Dg6n_%vHcBub~xes|u-wggtI=N5wmFz!<)XUpb);pK|>xg>x z{jcc%&QoeTcMOYT4gJmO?1%ryABoxfhw95-d=u^4EBJ+8@7(wF)!44dL6=ey(o@UO!G*fm<;5PAXj#Lv|KL4S@9@V8(w z%)Qz`eaWj{Rv(Lh#`0K6e_Q%J?8JW$lUE<4yXl*X*Yj_qyV1+BtbfQe)F0ztNGI>u zOE<)xxDelWE_wfE{uoRy)L-9RR4!OUuVkF0tJCNDK9d*JSMQDo^{u7L(39xofe+F5 z;Y0eKp$pMh(Yxud@n4u+buGP4Kl~@B9>qAIZxG#yejAJNPtv*P>%;HE|2w^o{t|oO zXuKOs>92*!?|Q3O#eV9^?XTh&;Wx(QqD|G0;5_w5=sI-r(j|2Ak-_wp`Uheu?5v)9 z!gBNt>bZxkNEcS$j1#bfdRKgqpL@u=>6TboUnTsGzl$D;_4v2odHhT0&h*XL8lP6L zM!$*4r>dwwkIB1oulFMVVdu8dTj|pD96EVo8aHBN^>gS&^dBJ;IQ4ft^%MBL`~;G3eB$@>1HZ2|=qKnF*d7lLHsL2vsmaftGeUhOKl|N1{7L-GKNs( zH@)S2RetuUyr;|ZSLyo}$MKUdWPW{{-%;NzdKjHKy$W3cKf*cs|AZBAzj{Ngj#Je$ zFHNC);9Iy){~>&b{}N`uov%I(k7D-X%k)p+KSC#0BMZ*;hvZJly-w#3a6Y+j_L%Gq znXgvq&)ohLeKVfvIZNr+>C8!)i}vyh^koju9-KLMuzK>l!OmsxIHsO?{%`8bF!@yW zxxea9Ubm9Yo}T?FdCe@(*@T(j+tA~2INs%XncI?&WdBY6l)bWvb6+_3Ee=J7Z_ZZFK9(Hj8h-Z3 z?5o51*^@?lZaI7mi|V@;lk+4edR*UMu#3J2Fu6h_^{xD`=-279>9%xoh~z-I|M*(p zOdNshohyTjF?oFUf$jRU50uo`i@yxTV z;Mbo2DelEz)o;Vk`D5v7boQ|gbW0LEOy2g>J8~VI1qnPzns1b-{T+0 zFERV~0)55#KhVADZJ2yM_Yj5n$;<0HcQw{kFGnZe`<|bDdcS(^MP8+ouXmu6W3+UB zH`Z1kh1;;5`ginl`Uoz;omd#x>TgTe$M)D#eHcEJ=i?=~4wGL@r?2vys@Qwg)OcdeisIk%2Zo|XH*k?OnDo8ff^>Lu!%G5K#dda?f8ANA)a7aOJC&huu_ zSKEi&p(fzK_?ep%ioNJ)RUio$luAo4hLZotb`TyUqQFV z+*9R#=Ysh7G$s$*>NyAa$3r~ zOb(HKw}$>F)su^4&fmdLj*$H?b64iGyl)13-ek=Fe2o6jeBnKl_j%@%eZJm&%$$GR z`6ihC@Tk7Le;2Ahhnb(U$8P6mzB}xE=D*c+_R@*;eCM)1-cDzJ+(Gxo>=PB}4w$+6 z1Nsrqe}NuGpM{w-vv>62H`m`D^FA-Fp1mvcPCI=i@Cj^zjumA88o)TJp8a_^ow;`+ zo!o6RUBvm!>&Zhm@=IcU{n=x-(%Ey9vmMg^sQN`XoS!);dqqe7Y<&ypb#(TfKl7{d zYt!e_Gw8q4*=v{3M{pcwzsWw>#`(;**@s@?FT;k;&!jidZ{VH$ukd4Bu6{kv!k;lY z(p36&=d!;(A0M+fXCE8L-{9Ptm^~%)Lgt*h`jaPJtUr6$emeP8_K?5uGw=5Goa`Od z)C>5@`=;@;pO;Zj?wfr(`)cx;cl5pHeD>tB^keEP>EtoV!M^4H6Zh-yLHEPVhpp)* z*dOa*_LH4-cg$RzdG!Q8b7c0X?0e08AJgz2%>6?Z=acWxQP2LC{r4b0c}Y>9*OU0u zJ?9!aIZN`<%-`p$zl!hU-8jy3vd^~W58{76q<956Yi(&SsJLu-P((~q_ zf3AWS>b3aUE3@|{H)y9X`&{^q7R>FGtmcHag1L^zKzrv?5c|`69UgA%7 zZVC+r|KjJqtcpTKJWstUKEh9)vW#A@{wp5lPoa}bz0H3Q7vo)+J^n^I`}(fQ(6F7AUfyzU<>;=zn8bEU$kTcHt-AI>ygE)l~X4eRHuS?!gE2 zHK0q;f1-QQt^7Qa|19Qr!rUtirfP(3-(-{_6{_u&@)H`pjtuS=g! zFQr$}o$*2by?8CZ9Gx6)A^kWemr5R3lRq1i4?f^|7xD+uxo_FVzkq)Q?!x47Gw5e= z5tecO0DXWSkLU1nPxK;vwt6ePj{iMfj_!lW1X&zcO8sPOjRM{}I0$-Hra7&b>z+x|w=mx-mVH-b5$Q9L-PO_eaPCPW>HE{RDn5 zKY?R@KPO}MgzTZO@H01OPtKg4TqgI{**~&>C%?!(c#Y>x!t6825i&<+PtKl@edROf zvafXUd6W6=a`pCDADik+zBH1)0T*EAJw^L5_4 z$rCac)b+gV=gGC!@qhH3%vDF|%&C3#WlmbAo;~#i_3UZe=;oOHGW%)vqs^X^xoan# z+$H(USNtNl!*es2Wo}Jwnfc*4eaYi~qMyLb3E5+B=VuS|7F>(xf*mIOpca4C;L}&xHI$>bAC2158xKuiXUL|_rKDg;sDQE zKqu!c$Nvj9QqP{;oIix0J$MJd9Y6cTGxSmQ0(uIaJfs5uO#VRZ!vB(9Lf58~>yGBP z!JDxZexZL6om~18{ulht^Z=~=Wfr}!`;|kU-tQp zba&jR?;*U3zbkdjew=;mi2fh&Nqui)a^K^05q-%iN72RAv&Ze^Pv-B&Z*dwXr&>X; z!@W2N4|!g4g16}3)VtFs=&HDkKLr=@=i?RpKG+!F!{eC!x`J~Xu#@^`OdkI=y%Dp| zKj7Rj{`puGi}>7Vpl=S2z}%-!pmPs9$N5|N$qyUR$-%Op-^PE@xl6Gy-mQKNm-BPa zlKd;VP)&XF_4UR|xCTGRH}$_sCr8*zkHFkd6w*J2f0%Ac7s9Kswffz3Kdg`gTz=`^%(JL@HeR8Xt^d-;kp>HH!ho}4BcWM2l=V-C*5>{HnX zZg4K|fs*>#@H00(#_x**)H8Qxj+(;XrZ1n*z364?*|)~fn=o^5-m|Nm%UoPRUvj(N zbR9g3c@Jk!&ilNVdUB@B_nGH!_w|0&_W+%JeivN>^S;S@F!N{T!sJ?Y^q2D7gY?bx zbC`UuJ)QS-_P-(g*3KtSol7TA8l~^gn7O+KJqPPKcapx3ElnFm+W#nfNL%r(iMGIwP^%Dg^7fA*mIbT!N#mprHzKl|r&Uwed1 zPfzDM;a9j5Z^KrYJpXIYdyGE=OYpPbCl}hzUxJrmU*}fSSK~6=qP~YF3l{ssw^$x8 z#`7@q?(5Ef!q0w`x&0dT>_5rHv)5!_o~Z9DOb(U3HT&I1zRqy0jZ>W)i^+pCZ)RUf z{&%jv&dzVgcd;@)uP=L0Px@^vu5SUZ!gcEZq%#Nqi>|G{k4}#E8lAl5Q+?xcrh4+< zGW1=TxqY<$#K4%e99#PfF3wdiSh1%C`y;b#vi;D3gtFndvQr|dh) z(OT-u9#+wFKj7a=SE2ub?eIPIDfD-^mY;p;75-mwl6vxiZVe*5| z^>5=>r59sz;68Nr&*UbD_{n4L^qhOKrg{l_BM!qev4Xw>_%lvW&pl>UdN=-$z8~p6 z_%L3l-VAH2dlj zEU#}U-4+kxY+Rsk9{mL-k89!QQ;C0({tK`hChxkOuB$&eRattFdT)FdTVZm)N_2Hx z?fg-?C~m>zDrM-u=}#{6GQAw1#N@>Pou78~oR0d6V(!7`sISH3mh<)H{v!8cgZLA1 zDJG}7)^lFMZt7#P5nh6u^yNOcE`1}e(^nWr<5oOPUspQ!8inZO(g*0=_cn0u5dTB$ z$4?&Ike-H1^tHg`Q_1ZL=}T_gL*I-1KSCyO>hE~!C-8gu31r?L?)UU{{K4;Q_OZ-$ z!}(w7%jet_I-md9W3ta@&&+!y`%vb~%%yoRzw3F)HTKYXZ>{uw^uqn>$sMxKE#qe| zNbZq&IiKg512aEmE@t!CULa)}B_xeIQdrNYa>>ts zm^>tTR`${C%ZK&1!0EpJ2zm_t1U-%(f{)>T^~{ak=~wap@UK_`k6}B!0uNvpEP~0S z9;1_AWuM>d`^o;694FlS=J$C_SA5Nb4vwr1*p^$o*q=6e6XYvPP=H~1fJM|?G-9%^qSV0ePz9){s-MAjtV)ob{ zozK3My|^F0kN%2u9XfmSN`7O0P3+I_fEBQodUBWK3#D>iUuin~bFrMqht#{`8f>LL zgFkr)=dzy#S01-u?iZ4`*5+r=9O=1hF#FySdWZhn^mzJo9L3N6 z_6olfKYM8M)#N63>#Jk`zf}J%{7dm|{=Ik_{{}kw#Cvq|k=z$N%zqM3IG^0?m-v|c zv7x@~o7?Dr=j;2vZ>G=3t32m8w!?AigE09>b@j?P5%=nQ8$00y^?mfM^qKex_QB-T zWA!ILm_WmS{(qmxJGTRG)Uc4AirLE_poiiYm>mCm`XOxYdAY}%!}sGXs7oj3eMf)p zRkO!0(w7|RWjZ;=2D%&;#oWUcr?-0k61p$;;3t==N7qyTg8mdI@sppg;19(b*i2tj z{1ubGeoeQ=j?PuZqnO+z`SIQQlUsgE7t{Y7Zoos>RbMGQi3RG9&`;A1a28&ozJNZD z9*bA;PhuhdK&-+~j@FD`j1S@&m>g@X^Beg+=!4iE8{hyetp6PB&2Nawi`LP-^_8Z# zVDhEG^da1-e;!?w9)gqkFVe}OpQoo`cYVnn`|)RBa;$3lelL6dsqgdDPvDR66UcwB zl-%Vpzqgr7_xZh@&o6=5*E7GJ(3kyWyT0U8`5e#ta=-q}1&8U(f0?^8A7sDCe!tH1 zt2mcAEAQXD$C~O}jUDx8{zz`~mU=!v^Zq)>&s>|mpglkHe)jk?JU{cu3;OcD%D!}^ zzI)UsV?KwoKW1;sJYL(m%q7Ve^WJ#JbMn6EqCYv)?dmu2GZ(%~XaD_A4xoRg`d7FY z_uzVLu74^fZ(2(4*7qA`Zp<9hLSOdLEu|+M_2Q+k7S-J={eaeTIypLWFEeTu?jQyP1m2jtSz0~b3T199>s;8 zlbmlP|3Us^^nQ8;ogC|*boPTW`jTTjO5cZ>=aUOv&2Ndx^O7SZ|7@n7ef=(9_gj2l zJ@fx^{!{!eSf8JL|7^Oy`V2aI^4Ihc_2KkC=%I99dI?^J$xFJ@$z29IcYzQ(^0Yx8qIl|5u7 z|3`ht@JXztekWa@J{OZCHC6u`R>$k~W&cTTGfh4FVHx@kY~|cue3^e1{(`rtufXI! z_2}dh$>TrZx4}l9lRfMqe(p2s(4F<&hE4cu@frLXb1&3Se=E#h)==MKetAsZ-Ag^W zUpw_unB1&Cy&1C?=KkpbKe+FpJVQma}U&wKMgn`Sx)SsMi48H;YLwY8*>g{!5%nNUrk)gzkn`;<@j519VXXmKv&kkgf30*qW?ek?e**D zI(`56RT0HRcCDopN`w+6u}svIq`uiSlx=B5VOb;^lr}>|Q%Wk^iG)G6$s`I(G^G%k z$fk_2wR6W=K8$P^LUxuL*#z&G5#ohx$nJ}?ut+2 zEd9TSOyKn2@$^sNxAGH6F0;e$Y4WVRSF;~x&pyZR@%{R<7iW%Xq@Fpi51sjNgL6yx zM=|^PZu$j%$rZ}v`%}+c-=FTN{tKO)AbVeOv#t6vA8eveIaeODSKp>y4KttCqgP}0 z)(y_L#VP82>5iCv<*@oP%zl*pA-Td`>eo46oX#GUxis@c=HG+*viHw({sjL5dM%yY z=T?69z-{W8i`vqen~KtzACtS?;W>lUn_~&gJeAzB<w3ZaQ;+_P6W{UG-JUIjqFboboJxEWak5c{Dk7_KNJ+ zE%jwT%D$X@B6(JC&&_-@-?@MBufietNA*{+3jb>y!_Pi_51oBE`%dP!cFtx0dQg9I z&YkqTn7J}@)h_41#ar}sptG-JKg!;fc{aJ&Uj1)6UyeSK=inkt&c2DBfnVWZ&zVH8 zr%Ta&>FjHD=vvqZyJ2#JRrHshGl{N@U+`CB=Hsi?lQSmQNnSHX|MU8i3*>$`xyf{W z%Q3lnGv{u?`s$g>b5D@GF?&KW{mC7(H?83J!m-hw`xe~}-{H^2$T^(#0@2arVLNrx)x0o4!Bbqx_$+GcHyyfyMDN z_2gGi@IT?N!^`>Oa1sAI+{4ek+ZOsbUZXF$>V3OS$7bq(rE@PZkxuTIe=nCDw2JzTI1AT!-ba`mq^J6o{NxWa`Q`8m9D%F! zC%;Xuw4MKmzH{l)^sjXCj+6Z4I@Q&a19hXP>RW(4a0t%SR|u=}zrnirp8DO`0{5w3 zix2Z_VDi`{>Z9-wCWm=Qe+7P5oXkIi{yTjMogB6y{ZD*MU-IJr;U^!gqh5sn6n5b^ zz&kNH*@tv;($aM9$&(8uS30DgoHO}sMPKg$_2hBu=<8F*%WyUh#d%l=U&h?u752R3 zS;_TA@qa6O{ps)X^iSaT@Ds>fmOV9}i}p zYkh9Kj@hSf)?dKiO=oXyL+3q|_ed{(_Ol(%=W{>rm-YPYTY2AQ58CNDnalpDzcv3H zOx~9HqJ+M@pECzNz<&a>H|29Q@5g_8ZsxSON@90Z@lK0jCeVN~~-{t-Lwt6?rdp-MK_U!HI<9wY7 z_yNBpj>jqL$t^M$ChuIPo}BqxI`8Am@drI`KW0wJT$_347WEFElRddToq1s&y&5xT z9HpmX_NkjZH~V{E_3V4uZ|m_l>Q5e&y}Jp&M(ViVxy=7(@^@n9;pBKN`R%c@^V#Q; zcVv!A4z*1GYD`W&!1=@c|Gl^6SI5jvKhmXehI7~8$NXaSmslOYR?oh5grB*qwt91% zj@GAwvUKLx?6VhpP9OC~bmqr(^g;E^r)&7xd-|zo?rub9zqo>)5g%9KX3twfXCA$p z-wJP2-%2MBYe(OJCvhZZ-#h928#oiE=zAAmr*++}>4`3B+;(Yd*ZFKh6rZZY zIo%Y4mj003HTlvQ&)J08C$FRz;acb4q?4~F2f9?f9VQp9q`xS?IQGEgDgE?y!QSev z>3x`edosOL-$V2U`bPRzdKmo}-4*BJ5$uom>aR*y!6BHOp|ZZ@rSH?pnLeYFyYF>= z0Y7Yw6=_z{-CWBR|tlK!5xu!X)k>UHVliOGNJtB=MLn4F{(-4+{S?yWX^?(_U-u`s^~ z*20n45ohbakG_Yli$CEgd>NZza-HqY{|g_*+oO8bLnANhTj{<@`uwa=}uUW z|2+K#=6H<0c~|4Qe6^bF6hs(u-s%O63{cCHhh`=y%v za+tjH3Vm03PHjA_ue^G4v5ovw{N!)HhfLt~-|_TM;J5M<80+^lpZ9scW-rKlGIMqI z)-`^=FL6HmT;8L}QL?9IA5X54Tp)8<=7r=!ojpH$WZsAQJk5K%n7-^m)##giU-{ha z#lMSx6lkAbJAH#FyGu1bEEJx{ryzdvMd$=sUv?Q!+Yjg@@A$rrNsBp=C~mAOB8 zVh!ic#N;j6HQ(5zm>gxkdiLwi^nWq?O6K3> z0Qaje^}O68WWRbreHk9b8!)+2_L&*{37EV&ds6nqyfB3Wh+?TYW_h9y)tG!UO&phHeH|r~lyK$}hS9EP0#lHrV^Yx&U zFK53`9@y0R&-C3#H^L5h0q(><>VFO|!sNdr>84oUxuW#bnERl9>fNv;-hsIf+UMM1 z{!V%ze#D=NL-~8@##jh*kC7auF8_#gtLfZFUO~51Pad>}uBASXUPo7sh)f$ z`+4^M)%xzm@AW@{<@m`b*3qr7nZAv9Bma4NJ-*COu9SPz;_AsM-c27?P~S}_zZpa) zpM8LCjI;1Pd=1-sUJ<$heTqJe!?BNgCwecg<(H-(!pHbk={|Tn{;J-UzK%XYx2CsZ za=PktO?`i(lP6cA=c&JhqxctM^6G=?Q}|VV?hT`ttIx_gJdb~p-bOE@m(s21f70J! zcg#I{{=Gqe{ygWF<8Ztco9o+x$&Y?kufSh|3;D^P{)c}tKA?UCTkx;NE?6IPAMyj8 zd%xexUVr-gJpB{+J^Tdn`8mVyb>`4~PGoP)yqeF&yiaEM{r&!*ji!*2b?;hcE zbdvL*`uchQBwxzsbMmf5`c^qNm;MuGj?CwN=9bKN6Pz2NKY32}yUc6ZBRN5FL{4wZpdD6gY%hd8ato;EOXy-e&)Nr&SmbcsGhku^I+aX zng22mOxHgfZ}Gh2n0ddl`u+TTzGuIjrk=SW`)}rrvz*JEnB3tQKl{QN^nTCFoWFyg zIqWhz@AvG{nR_xKd&!2;Ld0zJK?3LLA)~olx%&QZe--5|6O6bd6nmPOr z`sU*vec4a8)15K<(_;N+@^{fEFmplXjD`9};WPMw{;hORe1+c$FTis#^J-`M7aWJ< z@p5eDx&My#d!K!3DnIjZ@|6djyG=cF*;sxHOpe!0U*?m{pKJKZJGVHuf}gxDc`3W# zS$(^38D5RA<4_06)1xtSeCE02T5Z*nUnJM~D?j^0HM)uCzerc5Gv^Pa=cw<)amuH`7r`||CxpwxPH~0%M_XvZX%icAS zE~{@QT?})tQJl_wLH3~JYsnpld(Hs;$)kLH6eRbUr#>H#JD2=oHvb`h^25<|@_^(D z$(zQi-yQnGmw4XWSYEvhy#Wv7$LiVda<5Q{|0CY&`D2``fM2Nhqu0{OCEue5VRe0D z=t6WEx(1zn@F>3?KY4EM#SW-{u09a2<6nw9`J3?X{OjpQupu^APd@)x`u!X_>Z`y{ zE}UHCTJ@Id&(X=H>hTBQQ1wIf`SfBO#?Sp$_UxzBv%lU%pRK<+oqQ|#W$qzH>8p+n za0w1`z6Sk1{XN|Rd*YoqUEfsf&#y>7N*B=E=tgu|yp`XLZbo;ZufVR@2RrI3?7vrX zjNa<)F*!n4{mIqNQEz~)utR*$T|y_9c%E*83$U2yFQ+fWZ}|u4+~>8Se^pP8c@e(^ zKlgKA)9ch9!rT*Arr*PHc+9!s^at3R--&*U&V5EvTDjmne>jMR@gAI_|3^A`P91)7 z`sCK7^i@$Gj59I0=U?>I<-bXPPVd2s`T2JMx6>!pPtm35E?5K)<3?Pge-@p4ltOgw zYrdkBlWnJmIR7O5C0!L);7QDV;?vQ16hHKw-$N#F`tNx9C-7VO2{iMb$ljHFC3|B2 z{Zi(u$NfGp!n}7U(L?>7XaC5)m;E67#|`?EAN8g4-q__i*`r@n&wi0PGr7(L^?lAI zkNBF-zLov)ef^pD8`GIrvoE~O@8>z^V&>oE0Uz@xV)ljXOPM=XssGh;GtXsC%6yym zYTh5M_2)f(8T}bH!_1AV=udsU|#^i(t z>Eu(L^(R+M-tjkmnJ3HZ%iPpnJ@4;n^!57IWA>D$>M!$G(lv1ke?DEC9+Q5|K9~Le z4*he~XW$n8BCL*?)9%vu3;$b0F>`Pgx+9L#pPV8&PA&C6sMo?K*dH6=CjF~1|1Ryn^iBAf{v9|8 zi>qh<$UgWk|2v$m{}s$0y;gl0W)Dp+x={Zp^>UbfehHoZG5crogBi{b!f&xNKH_}# z=I(UvEs|^8%RdXVPe187pYYG7lb=kb+hARNeds4}H5?=QBu5*^ID##G&TOM!z(;WdcEHu1UlpI@Z^Y!9CFtAqt;5_a z&Qs4`+=woO*ZRKhr?ZED;W-!3mwO7Iv&F07M17)&Luy-f`6Dlp5C8+tfc-qeFc3Hojj!w|4jZ5cnY^;7uy3dx9T$?<#AUDcn)3;Ap4x9Gl@ylf2J0oUV8&TXfM)8*-t*a|OJ{}LbISIs%U zC$soV@Su9~&WHKy_#e{$qzmJ8tgqe;lhc)?>tb@s+nsC1KaW0~=ixl{2KZaq>ra25 zr+)&!ho3<9?&5x5_h2*3epuP>?`Hlxbmo-o_1W8xsIS5im|P)wK<2;f=gC`k`8s)T zXV1(YmG@HSo?)K99BX26yutI5tF@!^-uSn^HvE6kHR$AI*(b7BuTo@HwA|(+>Imo* zWX|cUo;)&pTK4Mfe+~3a^_-A-B%bb_FyuH4em^|TM z^exWU#LREq>EzLe^k=X7MEy3Lqn^Amdtc`FW$MXElB2YBKJ(jTI`d`r+Bf(+akJ;W zf|;wbFYVQr{3i2v=H%QnW^SG6Ihn6Mrbm0uS{%)KbL=k&K^>MpLw(zU0z>VypCUlK8o2V+tW)hdqno`H=NIY zpZq&>cEs#X_?7T_ z^}Y1X$S!E&4|DNtoQ9X-Rn8r!-%lUEHYRU4m(E^!4xPMYwe!zn_Q1dETf+6K}^l`jU&R zq?1q8*H?gds%KySoF1j#n65%6pWMoC$Ztol!Q_zZ>Fn3P(4Fuq=byvm>B&iQKk}IV zL74shpmW)K%d02U>g&KSN?%MT?|79y zj6L<8q6_2M{N#I6`N_Zjq`n0&QE!PS_z%*#A9#Tthh;FiP8;W{=DB_l+vqBIj&n0{ z2med@NxB$)1#ZO$)IX=Y(|fQ#&cf3As^fe7@t9nH0G<1&Npw-?O5tw&LOu5i7ts6F zE7I$*6?RlV1LyO*(k#NMBX; zCG=7J4}UoQBQC%V>d)f=Jc4gwH~qQCUP?E>W;hsoJGTxS;SemVFZWV!(NC+_#pIe@ z)sqVzru*XWArm@Wb6@8F%w5@Q@;=Qzkk8l5#rZtC z+2_C*pBoM7?16btWe)A9FY|mpU-NmL&zH;#zj)p=&gFgkH9vDl5jyih=7@a0X79>8 zo;@X>!=s(g9+LM~-p|=vG8g6Z^)BDnddz&8`7!Ukw(1RVEoOfk>g(q{m3!pL`tmuP z{cf}V?ENL^8}!$~_WWZwjDIyw;}4)Sr(}-I+<2Y77Wx{~c@MnJZ->b%E~GP`XAhpn z@8dZO>CEq$4>QM=R?pm;d8i9N@2@?cmwBxVoqam{N#@k-X_*u99=^wO2RT0wlNTi4 zx?W%Amnr(*<>$Sb{VDrQ=9?JzS`+|$?iR0cjHQZ|HSUN1rO@$OJ7K@qHE)a{4cRM|20g0R+!$5Yp}a>t1l@Fnj^+51 z=oxqu|33OrIyu>3ep&uVIyqGCW0$Mf!`m@=)|qr;JcNJ5<~Z5&lk+@ACpYMzZzC?o zar$ngPtwU@AE2wN=RUCqy+r*ox+eV--4H+L=YD4n|0{k|Y>BVpE?kC>I5&rGfaUP8 zdL6nZy@8&9xmV1+#t416&+JO?!&{yE8}{Ol!eQ}o9)CN|$1Bxm(?jU;xQqWO-2`)= z)k*y}erfy}XQ>aT-=oX<{5b3a&`n`OJreI9>g}_!8#+ zxd{C~nES}}&KLFfJVIB}SD((k;5Yo-%jI4+_g%S{yHa0C9OXH`mA(G-_j&p!@O$_P z`1LHve7Su&spbmra6Pv7&8>dQPbhM$~an0htL9`!n%c_ed8 z=F{wdd-dh>>PgRUjtABAKFa)1gP%Dgdqn2B3eIOQKS1}y%<)w{CpklMi@fhX*8iry zr!b$(c|V`zH`0HAE`{0uk|$>0%AQxux$I+4(3LQAOWxmw_}P~_(yw6Nr#*fB%uSi= z&f#Zo&A!);pZzIw{{Q1I@cet|u5_<_U4Aj#k44q97qsV>;b*SNK5)Bw_SUuZ|L#-H zCBM0j&igy_?9Kelb7MWHA7;8fc%bwPQ&itLdZJYD+agM(1JK4`OKW9!lSAPTjrRZgJ_Qm7_b<}TA z&mKOMKa}4TAIEdlYthf(U6{OQm%i+O*^4vRX8z4yo}BUp&l!)?G5bL^&)v!2j@c7F zr}ydm8k?c9t6-_V()cA_jM=x&axV7{jp@vZnY)r_FILYSpLs3$##GPG+&oc#^5b^& zaeW`qNAZ3BSo&Ysfj@*kLXX6jnB2cJo%@eL&Lw}!e%hU%J#;#~4o~1Tyx()PCp|=8 zpk9H_-rt@-nV)=Q2fal7dipIodrW11_VVTGL-^TKvxg4g->t6`{UiM&eG?YN4cHK~ zuVg>TebKL;^Qyl5`HZ+rB0=O<^#KDmwGMql>B?01XQlV7&gm;B}q`fs=y z`#N_v{Q><2o&B&3Kl|Qf^$Yn$@IC%b{yR+MkHIT2IZtc-mTE)3KxL^Hg`Vf61{Q$1Vdg{q>E7IlFtJB}nclz%&o8FC0 z^)*v(gQwKr#nHGKldl|cKDpZ=^-lb{bPc*IU5oxJeL0=H`of&YRd|j5zte>=_qQGB z+L)X&`Q#bSCx5?G-?RKCbaKK)c@AEIEwKS6$4ajDn&*vD--@5{lW&dWkKiX)OkP!5 zy|?;Rm>jmM|6Y6b{T?!b(|^a)KY`!MPayAseBM3d_qUDT*ZlrAr1M_P=XDGHtub@o zD0(Pno=v`zy*i)2**{;@pZ(xfU-x^==UevD2Kq0;ye~5kWe(5$n>jw8-}zk5dvv<5 zlX)$9$o-z5y&#|KnPal==DnBC*~~vBoNtFWVD{+abmo}srF(U zeK|;9_N1P^PILZRx*z5}Je$^5kohWmeDaju`m%Q<*WAd@JefHv`9tQ#yY&4L*ZG3k zbKCP*@c)3xH#3*5;b#vz;#~H>zbn z|5J4Kk_!Bxn7OJmJyricI{W&1I(cz1I=NWpqU1njJty-$OT|GuNKQ zpT^IAw4Ohn{{)>mwK0DcKYQa4I`j89eN!;|&bjna{WoHNybZ6xBe+%I{EAh{zv@Un7kU+_*S&*DK`|4-v*EzSH-i-%vrut3vbWE=D4V_%&9C|V4 zKJJL;w8B4QQS6~Vxpxu%&A1w$*Y_5kT--z^ zE6i! zK<4$#0f+ov@Amte{qi09IrZdMFVfjR-qW`qGv`dx*M@%wKEuBYliMU$$h^{CfA+2m z=^dCkZw0*-*WfSJhw1D&d>axxi53tBK3PPdBO-fxz{+)Ykyd5FQk)Ky~58t{knQD zeowp^E8|YgzTbroGJg*ByzGJ5i!$G4?@PW=On-T7?txS2qBsY$H?5_MW9G+a=_@_w z7u>+FPfx()OP8s)$K>5-(#b)Se?7#{-doG__Vd5PG<~|^~+`H=8TfX<) z^4J7tV)pjrJ;@#WdQL-q$%RVLC)KOdWAO^iK9D@RF@F;#XUP7MeX6kMcGuU3UW(aA z=c)I@2h>Mm_Oj&lP5IyIFG63BO>nvTar$h!IM%~`>J2b^_ZB+)$Nls%Z0Gz_boRJH z^b#D0#q>|XS^R$375|CJk#3{Ua_%^e!KRq~x-9)E{sWW8E}++8bGx`1L z-E{8vl6P<7Z^ZZUdFLw7$)&f`$?vk?j^ZcJDdl`}wlC;9`o?2&*pJkc&*z?Dw!WM2 zRs0NdpOXFhOz*SB>SeKm=O>RWL+4)VX?@%HUt{*-=hffhuf=Noe_Y41`f5t~WCplS7x*}%(E~&pVzZ~t$6%?V{ zVncno-{?=b!2&!7o8h0FZ%My|OZd0bt?1-%Bk7u02s`4%&UK?p(kM9n_O=4CYtG7Wjg`N9d(^H-3dj_4URT_!m5^uM1rRpTPY8T1M&n zhQ9_&`w#jPy+_{_^!4;SdKcY;hJyP3keuk0z7KId?$x)EPJYvzpWN(9`Z0Z1`#w+c zSMux8L+IM{=ky6WxzR5AJe-c5@LGHXU-g_y^ygTIUjZM*nG_Jkq)op=|X#4mmQEjSL7r~H9#t3UJMh4h11TVFMNk$;TtOrMF#CC;Fm z>dPK8oS(d?3Z4A#a{XiZ*_X11ovWVxt}~tdCHHLE)61*peUf=G`$u_S=WRTI=i^7t zC-+%H--jP#^2IXFWxwm8o;_$Xoq6&({n_s`FC}kzQ$4xDWpr}D_MSHuv)^{q_Z~m_ zOLcms`XM^=TXKxd`}Nc_H&1ph^LX~0M*PgF**h}7W#1m}dD#;V&^I{$EM_mxzFkUR za)>edvM*+j&3;l$fA+$PboRn4>Er|T>1CdqdDA#jkh$p&eL?opcNm$MGuLOYoa4C@ zG4sNg^gidi(V6Fyhd$3w9=KnBEB*vJ`Elm?tN5A!7V59ePhK#e|1yri>{SQ#e}&l- z7STiXKTKyXOMdi>`U>o?FMCcex}kdJyUbac1Fu$JjeDJ&L9d`QZ$HE@fhW{6SHD76 zSAT=YQLFKJGf)1SS+slWemesRowOapqF^Vws2(lfA=z5(=n z9E#VdXOGU_G=-m>ezpE>{N$%K>Evk1iI(!S$7fHS#b1E$V(tl&`yb`kz~#RFh1iw9 z3$ve2QBTfti+UISE_{mr1wDc8K;Mc_@N41o{OjrO>23H6R>ka(XX)?5--)}in0j)% zf&APr-ll##R#G2<|KTsAH_?UZPFNPNz|yz__uxS1lhZEdSK{}k$%5QdOk-5VbM$Sb zw_-_59+>>Ob-s@O{V%1T#0t(OryfMVg&Xwsqi55t>EwFJ0g@vXRex3gU^@347t_fB zO6og|H>xKezMo!?$u*MSC1-oc`TF{L;|@&zxlmsb{&n)k^!-oH z(_845a2~%0eF5Dvb!?8mmA(G-_j&p!@O$_PWdF;Ym_0YY*O^E1`Ixz)vETE|*V(7; z@_U}WBcHpO<1&A|tuJ$E-Wy~2gFHX`bMlhR<9*d%!Ku#Wb24*bKJW7WI;KCL%XyDy zuF71H&$G-;C!EV(o%c?1gUsQXuak%Lb}o6!V!9;O@_iS>%)QCG@?M>!zde4UzbLNf zXAU02uf|_U*P}0@^WMo^l>OlmeRynK^g>zmoa! z5S=+Hb6xh`>>Z!$>+k#cH#Xt_H+9Uu?#mRM&(D1Ip7WV++N)>2KS_VBZ$AD5%j0z1 zsJ}LSCS8lpd~|@$oR}OsIYQ>wp_4;pu1;>!Q$2g@0MA*;Z%AivPaav3pZ(@z zx(}Y=eCD^z%`fsRVDjWurTEJ*bKx3#pT3TCa)xpI%;h)G zJ@gHu_hRP#Z>2N0CkM%1nY}yvQ)m4h z@J&pv^R4G@=QqNK`N<8kzu(DUj$8C6KdVgdSI>Svk3W;2Jz+Wj3Vw2ucj(Ob$qOd% z2jfI6?m2CI;N(8)QuQU+R^JfJexE%rIc0J_#|n}kJmuU6SO&A_oaI1y{$DWrby@W? zSW|rg<{sh$_3U%mZ&&N~rLHw0J7WZO5^&xcf%IEmW5k68cd7tY(sOWw{@lAHuUgCR zfUR+}bMIsF#pHg^@V~(t&fSeS@rPmwe)iadbPM&f>2t9Ue+%7!9*)oA9QAwXzhWhR z^2?$0W-Q|8wnTqJoQ=8X$~|T7YeqXiSzm3;{mn=^`*?M_xN{Zh+{^Z+SK+Pt)?rzG zM}&eS>X$Gc!rc4Rh{oCalaGz0zf`YIPsUFCI&>TQTD&cNc!GbJew3bo>#!9jhs{0F z<<8B;X8I=LANVbB7eD#VXgc?cxvy`jzX;aRcMn|{b8mNu{tD;nUrGNRkK;JJTHj;z zPxLJM3Oe_PZ}F@7-(hm3fAN=MW9JrQ?mwESFTyqIqwq6KzWAZOm-!p;4Sp56E7r%t zzQ6AJ_FyUXX6gg^x!23R$wq#2{g+`m{wex9yb(vKKZ(i3zMx0w8%F1TJh}AmArmEW#tqGm?;{nghewz1B z=ATdWCkM%#Glif1U_G6^JnxU>5WRezc{|&gus+dtTqTc@8p7X&|G>;2mFVnsnSY=1oK09l-x2JA7vOSzGq5n$R!Z19KdX(iPM@H0neZq5EzLp^zG z=E$q~nS&S6+weFh|9ONy>FZ9Sm(%alFJSh9^6GE$Z>4L|-_n_XXV87HxxVZxWBE_; z`_ZlOBtQFkdw%xQ`{+jcM$*^Q$*EV;xevHf-&THUI(tO}ei_XEl^m^#{_I8H>3bE= z!{qE&(r4l^=euB8{xz6=JbT%H`Pmne`{mxHoqG1!+#4h(&7N7**ISJJoJ)>0o$iE# zalZZ^={EFx_zKol&puOxpL{?0!BBmFQ}2q&3CMy!`9lx=73n|H*=x4)+u_UVm($7H z#_^x$_r}-wZ(%+DJy?$46O(6Ep_2<=sQ-EXH0;FB-gt=Lg1-=xGgYUPI~>*j93IB{ z`g&r2{KEZeO#YL5lMedtRZpIs+@mIc6z;>Nc((IH=y~)KT#5(PSJT7k=N_j#e++*no!sI-^lp4K)PE^H&F_uX@uNH!3t>g|u9%#4HoXS#)StW~dD1EM zTlt-`jXpi=hwly*Z{lX zD4d1Kt8)Llj{jTP>ra25r+)&!ho3-xe=}d!_Io-Gv!7<3%3hW|DsyG_gUp?Izvq4a zt>@k4^Cj~^=GMITGIwXc&3us0r>UOvw&&z?ekz@ODf4wcU-O>Id#04<OxhWLAA9;!w^h`W5fO?2kbIs8TZ ziF8Hmh5yFvgLzM6@4U$Q#`^kU_Ke-??_=iO%y--P?VZc}D0x&he)gWshuM1yIiGzh z`&jmn>_geVk}D+V*zNh5yRy&h^xPLP@3*mZG3UBs_Lt0E$>}mDyy)CM{h6OK&oolM zPu=UbAoFVGhU7Ne^>@RMoy-1MnNFUWxiRy}R_CtL*9S9Ku2nxDl?#$XC9lXnoOv(v zRp#dz&NszpFuCfp^eOzaFEp3Vyj+xSgPDK6*1r=Qs3&huj`@Ll_JDWj>|5C<7xOcZ zPN5HAU(bIWljCJh&Ky@of9BTAo7tZ-=VY$?lm7jfyrqV(bAq3HfX@7t{N&?1=;_!| zU-qaw_>=gf>9cS({~&!OX5YL=J$v;_^csEh>EoFBbTpkkI`DTd+L;Q9Omo6_cxFPd(ur6kM*qJ3fioUruT)iI1p{z#3RyJ$w2>e)hLoezBMF zt78ew{GYu(dsjKny#aI2lRbMcKl@@C&$}6uBW8|I9{m^R8tGd>51^aVy>Je;R_~9w z_sAa9Utcr4ACKy zH=#%3C;SrF7{_5@eaA8R<~;R=DqW8I`2uZse*p|%v}}eW1fEy-^R>ucj#-47psq7?_&`j)StcX2%Y!PK014M-j4(Mv#~nv#>`pE>FoED zeBDj_cj@Ffi|NeIx6|3fXVBTRG8boG$zJu2=Wo)VJvr~m%)veNWj`LRKl9Nb^&*%# zFLPRQit_5oL-M}Q9GJbdO`h+6V97sf@w3lnKF)kF-19R}XO5cbT=tKn`quCtqYGj3 z%JFpe-sE>@IhXx5dqn2A%x9IHUxn8?x1Y{@x`Y25_EpcEUX0Ejo7|)^e>FD6UY^qo zx8fZ2$~YQVsAoP*ewcl6jlSeI4fSU)szxUt%AR)IxdZCS@sqocP_LnWg8mXm@;hPj zku7uu?1)pH%eSjS^Xj`rQR8f;vDtll*u)&;U_mq&R4~`8`O)_W9jTiJ?ZQ}3-x7h zTS32qxA^_-=Uhkrvvl^A%5?UI1^U|Yvkzxqn#KPK|A-$sHyzK#Pt@yS_MOA($zQ)w zFONkr`|K|L?QnzoFuFC}kk0;>oGg1za<5|gC+p8XI-h<<{T{51`!M-Sa)9m5&BEN{ z+(+mBZ!ulMf1m7^t@u}ae(ueZr@qghh;8++rO(0KTV(Glt?y1u4)=)u-0QSauaDUu zAEvwM{}T6O56nH=1NzV7XV3kf-yO5xW?wv0|1MmqZ!evF^9g!~dIS2en0)tcb#_5r zx-9lpNDlW4e-hrLehUB2Pi~WZtQ`MkeaUUv1^@RxW2-{;_bqgC_#f%(G54t>JU@BT zM!Jmu9cBOTz@LC6@lT%j16>^-;^!Xm5ItADK9L1HV18#hxzQE$XV_m~S4<9AQN1L;34K0Z zk5}RVeO2fJJcOe#`P7s8YvEM&-gF~+9i98RadZQ`OyAR(e6p#Z({}zw{jXyF{X$vw zdHiwoAL!lmxAc2-a`@a=KgzF(P4o|?3t>h6CG>f|&j;zxu`Z6({}vX;FK{V#!`w6G zUiL=M>7c#`yYd&{75pLe+4OH^uRr~Lp8g5^9)1F+{9f+Ds(8%rZRUz2{CwVJp2+86 z=JL$*)%0in%G{9mblzk4`#PD|vJXvgKJ#BSeR;o}pfl(ETYvWbyhpQ7=5sK6eDay> zMTd8 zd{h06aFY7n^iDc+Yxb1n0&Uf^cV$jUUh%kk^0~~-*}t-HRnnLDdghANzK^`$v)|ec^NCo}JFz?IEZU1%dN%fUjo}YQQJpV<0=GSd>W!!{$uO^pk$4|cb z6P@?m5YK;`zlzR0u#n%G-xbT?T1@Vfye9K*=A*^>GA}0&%iNc}Eqg%r%=*qv^?j7Y z>}&n$N^WhVD^v9)1UG)@2&Jaf9-P~IF`c9S0B@}v4eB-aS(qzW{=&h-VlqZA50&=3ucbXzO$9zLw`~FZ}eFF7iJD$ zp|1}=xzYFZ9`&~{`^^pXu$-gY)3?!^=@OWIAba>T`jVq0pLvj*5=}-cn>2iURdr|hobJUYVeW0%we=nW=x+A}Z4}y{OVQi11 zv90s}L$AhtSOagzSM=}4m-tYSJvDp!>k7N|m%}^oOZDVrU&P0+@D1FCbDe8Ux6aqW zr*JsFi6`|Jq5EO(FHh3l^(E)IimvYG+J^3=KYRaTdb#>m?8-lYg|Q`;!4>*1q3huj z`~mbv+`&(NHi&-)j!{qEGL1f^o}Az)og8%y?=^M1hd@K@)O7tP@x!GU-P|Ezx&=02eX-BVvP`c_~6JAM`VAl*b? z?)j28{!@LQ`a`$}?@>=)RF!^M{Zg!mL)Dki9q}Rl7Wxn-hiOMQ*7q$vf_@s4w_i(- z*0+GZkG>H9!%zO&lRt`|ym}qK6ING$4s#E=UHw+9pnfNP9k$}Pz)}3NbOZc5zYA93 zzeit1--z|G4_5ZyIrnqx_}wx0By~J5`SHc-74ZS}2AEvY&)lCmG;{1WUpMY0mM@%!>C<0<}?xQKri zom}7={#<+kGxui>TkG5;^~>qxXI<%sF>~w&{h7bVs%OufOlLpM+_}ZM%;ithkLb_d zkv+V)dUBZLC&{xiZzh+h=s6$bv(9I)KFptk`!Rd=6uLdu#+9CvJvejWFX|`NllOF^ zOXCE6uVCipLiE-8?x1JU$tlZ>5J%TcodtepFyvnlOtr`s--?( zeFJ{ZZ-d!)vX5pzJ)xc)Ec;aU?d;X%^d%qf=Iiw4zl-f~Bj#Qp`~3`l2j{czf5QI@ zCYPN+Cw~~}+(LeG-&^@L_{qf@(z~!HPQeA3eYb+=+>QTLpGapvFT~%Bxo;RvPsi+) zx6wb~UYv@_18ev?3o-Yd$<2~SB|mNMd|Ca;>!#5+s2`zo-}WnATYU$;5wkD9s(vxf z!7H(@{$=a!$xI#Vo>7)9pU@!ISu}SLwo=@@H^6SyLpDIr`RKJW~g~=Iat3Qv)qqfk^ ze7)=GBF@+5kEDBG1FVj-aUL%9oaEf4_{l3y(Pw!6%k)oJ7hl5`o;Mde;-{Ew#y zsXmJ?jo0#T#P4vR`e$@!gBoNO;`K*{mdVsFL_*Ys^s3u zw<_uz6*~W0+3Qb#pQnEUzlWbd_RE|6o|eG8*YZC9BR_k?WWUe%^Rs^p;P>U{^D6mI z_R@ULWPWe%Ts{Z?O@DydBQhuC{g*k&mn+D7Ebpa!p67j+_e0*pnJY4v9`~RxeBJCF z+3$v_U!|V+N8Y=cqj%`b=Vd+p*?04KojEphZ|1*c&SwsJoX(t@IVpQq_J=!tUib3# z&UY^R(Mo>yoXiK=ck|xKoRRswq;tuqG8etU&*%IcI&*dMo9(`jpD=Sl@|nDEC#qNW zoV&3RZdJb;Gq)zkdXRqzS7K-9v#)Qa$71%d%jp@I95;Kze$UIEnmsD}arTyC`jQ)F zuG#6i*{_mUX1+T|y*y@5UhH`_u%i0ybmooZ3}>m&RnLC(0iAh$oxVl95<9n$v-PQxDhlXEwuvp;mBlaFVg>CIn@ zl{_ywOh^7qe)g1q@@IwWOE9@IS+Lw6j_Th{Cx6SHRYrXZCYMO=bildfQQ5~g=-;Yd zA3x_mhuJqDQeVj5L|3QVVfM}!>52MYrIYVBrjs*duV1Wxjrs&S`&jPB`m0}$Rj_ON zu_|W&t43d?{}=iK?2l`)AC~m_n)?c}Ap7+=ed|5_wS#5Aic6B%eIr`Qn&)`>6ipd?VFA;AanBL4T>9x$<$kB4*ERNhg;WN|(j!G5dED zx*cY(xrxr)wTRBVlf7o2?{AQL=G)|snfLyouMxJ@pB(ZRe&&Hac`j!DZ{^(kn0z7c z@ys*H`?4pD)1SR%1ii-dGv8#c>&(yGFY200rjCQvGiPU>&Yqv# zXoUXT^|i#0KG`6JFJU(Vi^ysozTD(5oyW_~Qr&pe;~^AY|??B_XG z;9Z!xak0M4pKs8gVsf=9&Q;;}p`XF&n7MKy{e=FFbn@ohFJwN>UfNXuE4ahC%5+aW z&cA?84pN7o{bQPX=EU7}?kBQO&U0=RX1<(5Xa4N({3bqGkU4oi|13PHVHcgeA-Q() z`V;z^U^D$t(8C`dR!>gBU$Izr(KjI?&12k{3+i{}V64`*9y8 zuT9RGy()Q4^2ERDPp;FLu8)uUex}jq&|7dRzXHC1mDDTXL@bBdL+_zKaqc|KeZfHW zpZM8tE~aPWFn!19+$$wloW!q&YjF+sa6bE0FMjf}8FVFl8CyBm7XN^ou%o`@m@D}O z{QKy+n0!C`RQBTRVY#n3!w1$Z=dQvbo}2r;&*&lQw_;J}*QqCG`6Hcsv6b|G=c>?$ zFneC|u>AX&srn1)%l`Q;o&Em_`bquQ`Mz$Wug5a_lD9oTXFq?B&V5_<*INAYp7RcF z#L{?eo=g25wf+To zn4jFZIh`D}9G!e%opT@Iv+8T<|bFV5iq z4eMfc_2ewI_~Y>h^$GM~Om29Do{x(#`QO=eNzd6q|CgRk@20!ch4E4T8JIk8xO#He z-2W!eU9J8f=UUP==_6Q`|0=$P^KdKH$K->f=$Aa_6t3t0j2$rdBunY5eePUAcf;Sx zUVr-gJpB{+J^TbRx1I0zdy3!F?{OSI^VM-W`9k)O>{ZDTGG}LR$>)Cd@b#W^v#*o< z<{5tGw1?=-VK35|Yl_jCFET$i_uPEG_oqM5pZPv{RX)c{sAn!uo^YM>dEf4#GnZse zN*-{7=VUL;du{^%YCMRUFNV>b@m1W3*)NkLCU41JJXT*j%>LVr&Ro=l9_sJ$Hl2C( z2|9b-({$#Axpd~I?CaSxvd7oacNbQ}p}4}=|2O`~KSmG0?8W=&%wc(tHFfSLtfp@< z?%-$MUPh0>9u_U+f{&H9RBa?Q?k_QJ{fFXw-Y+pq|>)t9}eHJ!QV z5S@H8dB8&FOJW(!K9hVibKVK{6`q%Qb`ZZYPRGna**CMVcTmqfG0^i)V)o~+^;N>; zYT2tR>u-Y(>B~Gbnf?GXXKvE}JZ68$eBGB{%(>(ZpY!kF-$*BK%HEayV}!m|`ZC{K z$Uh5{R~DiVVo&D|(2vs<>C9>G(SOEzc%lBKnB3wvdaS-_Sf8J~pcb8+GJEMc{LI%+ zI-j{SdwffMcdKVEPhNP5dh){L1P%CIu&47!>Ex^<>DKB8=)rgoW4_o6opXd4dlM5W@*W@RcXvc4c4KVjMx%VsUTymAa(Ao2H zU%21-#_HLt7t(9hbMG^UUZb8otRFwQWOsU>z6JORW}h!Yzlq7=O3__B=L2yX;V1tn%uimO`$%CrFQyw}a-_HD<<2D^e~i9X z{R;ewKZ^c{P7d`GKY35`sN6%1R=*0TWAgTvp7$vBQD2PZ_?77K^ejxC`YC-W&c@{D zb?KU^d){_BdESTgDfK$^DEvKS0;m6ur+)&!m7hRAzo)}6pTpUoiut|GUY5ORI{zqc za=rq#!?BorYP0_A_t{%}>T9Te7#r}L)7hW0e`X%Z9-V#Yfc_htzk|;Hlf31)dgiM! z^ac8h(V0gxU#{SPuP<|V=FsGCnX8fm<^9~+`OL$a*E0uXPt87`eeqT27W=xtVDiJE z^ju6nmt5r;=aYXG(f2iG-fp399e*EY-`-7E)pv}3o6eq=c{BUk^XdmN@AE0ncj33i z?4_&eoAtHBJ$Q|J=JU+q*%ub6XI`94XV1LObAI4wzRx^4iJv_r^WI4Qea>e-|B;`3 zDSJTX)@u4Qw`C8^UaVYjz2|Jet1Mnlw@qQS$fi(Kqll(aQbU*_gHV)YvdLzNrcAaq$fn%FlJBCFBJ};` zam)|*f6&TLPseeduh(_Crt9-Q&*SaBG5d4&j1A6D!t8l>(wXZs55CB6;=bgw!}t$j z=78;V=HA)P-OW#qdx%cHkoh?C)!WW3*Y|hq$j_Xyl%G8^dtT=6?BiYaWnN5P+?IbE zW{+<`zwYauM<@5mTzHV5`84zY`OYP8NUodQ{giXrQ=g%$Wqx=5Y5pbHou6E*Fx?uP z=}XR#y|6EThrT;W4A=S?1jd z{4X%`?A4(Uemt#{LfMYN@&e!y6=bysY_$}xg zFnf0PzoPu?|681UjDHUPji0^sbnf@RV=cOi{^Y|~((5oeNA{Jz&K+0Zjo0(D|0d^n zP(6G7<#bj3+0T~oCu4H7x9FR&pL4zGLUdEQKKg4Gl%?~2s4ksct0(=jd$J!+!&8^s@e5*j0TnR>oTD73sXE*-mFa>ZI=^9>d|-R{wmu0X+(5^QY3; zN3%!fJ;0OdBk>Bnzz?>Bd-mfV^}LUqPG79P8OQVIVqMJsUXuP?fAYQ&{CD`fa6OK| z(l{4?z-rEK#3TF*unhKAPj0)HKc7Df*W&N-ZGFvg2=2k}^*v7~x7$XaRR4xPNLQsN z)1~k%{s7#@FG5$L-^14YKhe$cQGRlw^Z2Xy$sgY2zk|t3@;;)W{(0CBZ$>C6?fgDQ z@`oC9ZS^vk+&=lqtNh#X&v*ovxaVBD3B8(bgJ=8ixEcMazWZUe z)|cFJBEKD8hNJW)4_wFZ!GDDAg_Zd==&$Gk?8#qD7slj9AF0>JcB#3)z42d7u4tg}zz(w$piDXAarR&wiA>By(2wpUeYw+}9Nk zV4e^8T+8QuANB0pnRoKM$v)E5_mSsh=B&&SnbTW%&Sc-tUX%PL&$~YE>*ed`IgmLc z&!x;&nfG6DF8f0p=kMp=O`oK{qce|X9?!gy=VP8*Gn~sjmc1p<|9`1xo=R?(JvIB< zL;5n0Wxl+J-_rM${ADSf=S7~Y+1t80mpm@d^X$vnGn1bjc3<|<%&(ajJGd{;)s^(? z&dsB5qz_^8sLbh^=kxqMs4x4^4RoHX1KpcBD*0WWuZz_)Ph^hCT>OpuuG2RYGp8np zZN$&Kdx>+I4-cwmUTaHdzbr|QbZ#&X=hvpQhh$#K+?@P2`}j5bcVK7Ce33c&CVq0r z0lr@5-Ph^N(aDn=IJZRodQ4t*KApMjS^e3kl4nihZ_(cZJMlBOW&h7y|0jLP(UR9! zbS`u3dVOp7*Wy0Re3hIlb5!<}{`wxk_c3$eBD%P*Gmy@_Hjdu~Gyi9v&t9{|xt;g~ z{@JJ3k(iGbMk?dzK06jupI3CtoV3&8B z@gn!;z1c8+QEaT9ed;!T_NnA_dHYKBdLGeKI#!VUVhH~y zg@4e=yOPUa%E!nv=w~pw#<%*X;SOA{FMI8A{$&1sdL8{2tdEWH z9{)j@9(njchP^P{^SkGy`SNa#N<7BPg}_O-gp|f;ap5EoPYoQz5BMRuf;w5 zW|;ST73miGl2f&%kEqwfhWz9k$tOPGCl6bo|0(`-Y=+D6W-RT$mvf!_5XY-0AFaW! ziPhATKaHSksh`5U?|M^xEPpm0#Xag{=n8b+f91W(Ab#?y#p!eIRr&?o$Dd6;aUo4F|SM?QZu&%B~PpQC%}%!`BFlYJv|c%D0Vt7i_%{CEvNduCJj4#Y2U z5oVsrzK}UT`|)e;%jaiutXug*-IKg0d-40~nOCznZ{gR$8?hp0kI23=&;5B0WncQ4 zpZPSO$9e9Y<-R<3^ITfUPcD^xFmrYt_in{Jk1nFG!sG7Ab3b!so)?+7X6SFEKlAn> zdKz}b%;lNmvcKi|l>M=RdoxGpd762%mHV>KXWrPsPkx)6i zJiyOfk~w)hKXc*pbaI{S`{C8+~o*f6`~um(dT>^Y9=)`&IIg@6-=qFZ@FP!#ExD zeqcSlUjI-`4p^MtsILhg#?shYU-rVzbn=np_BHusoqK~$zLs1ixyU|!^>G{SbnX=W zA%4nFUVa1JOg--ll22qW9;}``a|Hb!*2cWASmxf1xJ><8x*tBy|Aa12e}?Dct2jwt z4SFK3z|rdO;H9`h{Ve~TO{J@==e^Eq`Ul*H7hv*>lJpqt>AvJSi}`c-FVp{v4f)&X zcj(*jJe+`K^xZ+{yEpC0MlF}#nz zkWOy@B;8Q`cDffPXC0+}5?f&M@juXck3tsYy-{+%vF`n~?Dc29&oe)P-@;GeF29%A zcL)0g{)vAG-{tS6&%ymTP(69p5`GuV9`n7v>ih$kyeoOqU-e}V?x$}ge+%BizaKZm z_qmz*>l*$J%wG7pb9eKP(wXlv2PO}ypzkmy$IIT^k$)>D{}@mI?Ea3J9BCz;y|peq z)w!kEjh}t@IQ^P>f4Ue>@AD$MgZ_osf}c5WAbn6hxkn@VIeZwC8;+y1kH0{>u3)G?%;5K-U!>dM3T&(1 z0motXh~&Y|^bf=L@d+%3*=ru8vo|Hz%09D8{c7Cp>+ZlJc%ypuv%k>UH(t=U73bnS z{1=vY?qfQ+cXENu&DnD^FJ^wryp}v{xqB{gZ}y4o`#aT-t0!0LM%Td;cpLV?u9&&A zmiyLV74_sN$#33P&t6ttU-se5cbS8~)!!5!)1Nth44t_+xzJ_$lOMcCZ^xpz()syx z_WB3-{qT8AzPeF=a*O2Wuj-qveiD0N_T1U}*71{1{)e9&sT|!{U)~4&#IMQE{__w& zdwePN>|>v+55WrRU*Kq5r+xu$;aA2<_;1YnoUU|F=N_fI<0Sr{=r)*qC^>TW%%l2G z;Vb%YqMycQ{QSG4?Ell%%d3y38)I?)YnXj?vwAfgul_y0g)Mxpo~N%b_Q%yYOaEki z7)Rj|ec5009;F_CJ#Nx}l3q(EcPmMkP)|P4f?lnDK4!1|8(jzU{%A0LFDB0&?!GJd zRWR>M#?YhmRi#hU*}Ic}%;PuEcNTpWU5UP){t&nEr_inGTIheCf_>`Ca4Htow+fRd zRZ>rmyp7(4H|U>E55wfT$LKv+UwFPjLkQ06h^m z^ONJ&a$z?{O3V9ePnT*1#+AclsX3{qg;rle5g> zUykSMuS{QxrLdLyV)|m7iEHr@eLv6xFz>nc(8(i{KNaw=biN(V=a{V0g4eHr{vZrKU@1y>?{v-5W zI=Rs`{E?VFB=dIm(3#F1!sHa$M@#cFKP6AQ$-QmWZ=`2n_P7ajSIl!V^L}!@Chp0c zmANMQ!V}JCekelEb}oBYa)OTh#`v!O?3>wZ?^4e`lYDAB|8D29*Zz^8Id3hU{j8_{ z%s<`LlQ$&S*{yFXUZ-yj{X9N}L)5!q=8rn+2l?5LX3*7fu)cp__Ml$${rY~v>`U3( z&*Nv`S?JtKemlAt4#ng%nb%wB&%RSp-y;4=%p8^I`IrpVX6E z9Hxt5W9N3@)p(A2_P)RJGiR=&2V&;v?8(_XGWRamcfNZwZ*Ao_!puii>FiOrJGY;o zeW5Zxc~}Say8QWc=D2zM>`fKax8tAH*WfK!0gq$m=vmI4z}C1$-zmBqeFYBVPr;vX z5Wb?XKHZhhzZ1lH+gT zkHAlGJisKG{*3_WDZEnbW_d|Ah@O`{73CreS~e8JHg#+o=?eG0yYw_!JZm(lOj zS7Ld7-n&)dZ{R)(29vsF$n{I%$_`~Uc(hIO3{|EdFR#RV$4Kev&a-fC$9hiUT z{gQi1@_S+OmU{FUyjK79^#9Ud(3^24zZ0F@wI#h2-_%#ee_*ZnUHGrko#?!;Jj(x+ z|2Vx0=i)~E2-o2wxXJmdbaJJ&{3ZCPdP6#S-w6H%{K8llOGW$p&C#D+t+IN3{%QKx zve%#aKF|CFehWW=?S3yaugvuOn&;dD>Y20Wt7l%!b3XG+_S}5#CMQWgmCuu(+_M+6 zKV?tMoS7UX`)KC19lo!@n7z9kz2Cjb1@bwZJ?Us?>lo%UAm?3 zBXf41^X2(H_2qfng#HZkylC9yZoS(o?u92Lm9Y1?X-V>~MF8fS(dI@fHE_3n!qrX*OjoCXh?_b4llzZ?n zzU!XkEra+M^UGl7?rZ32`f6e3tp8q%=3%&W&g=s|2F?;ya<1APjaGv&_mS!gs<}#VBWhdRd3JF z-Z7Z}J-->996s}R_Stdj>+~nLyOFMkwK4f)7drdg0lKvNR$%stF^x1q;i9~`MZm9CD-&F)nnz%NX9 zrT<7br;pZesadk>Expm^}WSU?(+owDt5%#`ftHe{3@8eE$mR5foi}G)xC*h~q0_WhPn7m}F^ELUur~6{|=$iB!`tlxNE!|4}pID54 zg#H4X<76z0tM%u-Qj>fgY>RvF1-uj&p*6l>j(hiEa{s({*rq@4g_7%jssA1IuP`}H zRr+S!qyIj-20p`o6i4Cp>TBs9blw{l=a=PIrJK?OkWAfMD>UZELI9&fRoPzt* zC(tYDQ}`jj2sY%8r%T~j{xG^qtcukU5d z`_8B6e|{i*YQ!CH@vNfir){Ge3b}%THjA-_y(k*&`qE`J{|^nCIJWU-w4L=VCsuGACwlSmgZIn9tQb zf4cLt7o6wbJZH22WxwmHo;kFs^LdWdQO~}1n9g(Ir2c&F<~g%NUp{BEf8=?Qd7-xR zHT7q{JVNKWna}0S1DPi>Ul!MY$bES}=lPQ7XXfYRf5|;oJGaz5Yv_FL=W{-}JoloJR37lyyu=0{LJl{Cs(Myub#am`)lU9HR=!RPwtm}D0B2z`m*09 zC&^rzd32|~%zew3G-vri;f%Dj_(@kRG0M^4U{Ir}B` z>_08szk%O|zL?%hXV1Ey&VIj}&K!8FbKhZK^(J^XKe8qO(6`PwUUm+?gEYVSe(PNx2V~`1*(F;kcfE1Y7gV;X3|M+=%b0=RFBoFu@=4 zej+*SHT-*>pGXg&vo|N_U96sbqq4qP*j)W4%pTKP{TzNbOun>7eI$P?CRfOwH(K8& zJf<(XcmsM0W}nSo@gRS%^MAwr{NxkKm!IHguRBU#i}Ua<_w>T){PJ}6qU^^r`Tx=P zJkG%6nAvBu-<@!-y}s;~ZRwHf)|i69{Bis@aUs7sT^9%PlRvMd|E*qz9z|!bOMabP zAba7H`W9pMtGxgFnLpmWwQ&Si!p{1t)4MS5PcEnH;s@9m`#JwC{SMt5%lLs6p_4DQ z(%+x|0{sK7#qH|JcW$Q(sW+pOf1Kd&;BUj}I5+383MSVqsxLXjYW24KJL#GDSDcJJ zaD)ENbUhrwA4liC$3N2N=T$*p^2!U;Z^o&Z9AUox&HPPxJtl7&r0+F;EiA-uNe`s& z!0T}(=Dl(9t2WMc!Q1q0p~uk4&+g#oJ#ljFtN4#%a?uv_LHyEvlkhM6@98mgN4$uC z8kgcU^+)N!SP~E7aed!lCw_ANCj7VgSJOqYF#k$?HClZq{QzBuUQd_DypOA>{sQJb z%1M2n^BdrT-0Q!uQuI&idGC9*`qS!N)bpNt6}=zp;}_TitKn()bJr=qr`P%YeTq&lmAUCl_3TU2=wjFyOE|v|v!7)j%6^%-F8gJk zPnrLBxG(#5KDV>i^-+J<*Xf0sD?e7xp0R*lrZ4+ja=Ba7PhsZEJjZ|GKjVDzlI%S# z)$<&^icU^6#JQFH?9n&!dt&CP1NySJ)u1z{X0Q0fxy_Ib`_UiFwQ$ezAe|F`&>{v~wgo0|OWd&AW; zf8{wmQs39=UGV_FDP5LME|GmJ&*AKY)AcvgpShwqKl6D$GS{E2o?P)8IyqPc=T`7*;97p>yySo{s%I}w zPBC785qt&j)}Q??^YcLU+3Lybk`v#_&s=s~e{p{D@W=TD{K|A~I&<=^^jrA9_yp!X zLf%tkE>DhT3{$or4p>dC{W(x2z8XkMm3EzVv%I6Q5Vl{`o0g7oX9W{UGyw z2mWgO6DCh6?R*n{a>n=hQ}{FRW`0A={+K=OM}4=c@1u{>kI-{4`OH`9-TAe#G`7P_ z^licBn7p`@zUll8^g2wAwVeJZj@Ca97vhz8Sl=Nm!+(v=o_zy9`B(O%{`_wE0A|0u zn$CXJ*1d!HC$J$u`9t#3NArDo@YSQoVBQDJa8LHcVf1lKuH1?4flaY0CTCnrm%}2s z0mor=-$!No2+rm=p!2@xC;mGAD9oOmJgT|A5$acAa=rfQ$^GxAU)I-#?uLKF-{A-P z{)$KVqi{WcKfZ)j)NjK`k?+v^c`59e-qB-@5BfBk73@Yokfq; zw;vl}D{QQ<1+M4!!lU?S+^KIfJrcj?C!f!IkSEnUt6xcPq>~FLH`~rXjfM3u#pn3d z=)Uyj_$EK^+miEr#?QZh?MeR@GJ!LH$1^{HU&~LRhu_NyexLK)$nW{x|L6BV^H_3% z7u56loaa|QUoy|+b0nWfRh-M`Smu_@3z^5K>r1|r`7Luzo;$iDXAf2Q+%&Rms!EYGbxKk{74b0GU<=9uf< zH{ShK>CCa2dy+dS7i2Cf#>iZi`66>lo(E63e*rGUJf|~<%;Gn7?|JlK`W39sZ$M`c z$ef$J8;9U0H?6oEJWgd7?UmYBvK7!6ZntapPSn$63TKzLH z^U~|;+4HjJ4CI&9|18~t&RljO|5bkG)yxb3Pd)kaE&7tfWj@VbnD_nJHy_o%**)1W zvJdP~&)n0BZlgc@@>YK4(#%_#UouBs?OYlC$;syNvp<%m$6@w|PVt>To8E<)<1>$D zjyj~CIeLQgefT?Y0IpHbe$=O^T`GN%D)lkV)D}DiH-Ol zJ71e_MrU74j!KTJ9N-=iv*uOy3>!&2;v$Lj278m#eqK z3Yh)l8U4vAv-c$DzF7ZieH(BeevKt?BDQiadv@Mq>{kC@_2%>qboRRJZOPB~>&w1z zlKvdGI)51dj@i4G(w+7HME9W=;-9b==DpKy{ataIdQHsUHd6gZemPvvznAWSN3f#$ zYFx?hhF9T3>V>f--mjkbQN8*9@!_@O$WeD9rLB727fhlT>syKka1##4hWI2Vw?5^bfAee7D{&A$kICmB)_({8 zPP~qv{H+qdrJw76^E&=I{mJw4p6?U&XRtDUhl{W*{?0vb(Jx{0k-Udm%YR6JX&l4f zhHdz>@gBUu-=n0yQn*q5FlNuMrv5trDSA}yqxWOp#}uc>;#t0K^46pLp_um@Q{7um ze-*lcz9N|SWcSfE^qr*h{^o7^Q~XHZ3VI0k=D$H7z(@GUun~^HqQ1`>`tv^RN_v65 z*7PzuIn(2GRrQwiD7rG;f?kaK5DI$u!yJ4WZ^AB^_gi@{)t&zpCSSjVKH=-P!(XtM z`WN(Qx+(U<+ts_{hgjFoZ3TS_8#*_Vo`d!9TJ_{(^XdG1gW~#yC} z&&@oiGxz3mBlBmT>yw?&eAArH=h`awW`57-Se~buOY-@fediqKvcF`W$=sLc>}$SW zKIhM)M?1e9^SmuaXO7Hsp^tOP3-USsHh(VWx%)Tw9LL`3d0u4h$R3gB%x?X!U_PIR z)7fJ#c38O)^hZ@qL}fnV7jS&-u*z znRhckHrD?a_x}T@VDir~^#9=BF*#D^)@l4j?(Kk?o9Cz}Ke|Od$o^i0k@;*7-2#&f z5*&~|pvj?rFtK$j%H)8Vt zTj>Ls9B2T&1^YwFc-mA=gB?dUD)nZx_@C-4to_L3**GI%{^56#|q z+&$T!lFMg*N{(1WUj@wGHPpSOu(=1;xAbBCcVlwJ1#~@JhJSKyI!?uRun#_94d=??x9JfeOoKD`#R z_hoPH$4?HDJozX7EF6j1C+5+!eVydjZ_=&RlM_y&*QqB*8O3jo$yHa-uVORIKHr1x z?7r;nkJ3Zbr_&eUCftP$aX22u_wx1V7wE~D+`k6>j=u3&9$&@0hg+xrbIkk268Z}B zlV|khcjYJl`ktRXGkMH*ej&{J%3jWQ#0Gd7zKI)fFt%_{ak?e_7$ztFhTg2NHYWe= zPXCA-^smNk{LgR{{}4{X8u%{m)BhsD%b@=$-UQ`ZRsEzh7nkUDycoe(lfnR_9962k9~N&v-vpc7I>` zHcY;goNOU~8b0sd>k zaFF_Fesb1b{JOXlCt_JV;#~6Zw*0RAI`nqDhJOW}_a+tSr`3N)&&Tg^B~H}Wi7rf^ zOE;nGV^zFT{XzOB?13Zjw~z^(`8%HZ3H(}q0-1lauV;=;zEa%pZE~1(boM8|_62Y8 z=inlRJ@g>VJdyk)&)@75$!!ifpS>dUQ*ZZVPpD33uB+}`9sYed2A8Y<-{)ZFpzN`m z^*6^3&XEQC{2}?;IqJ9Izik?%y-G3lf$i5&;DK4`5W;BCa=Af{(pGP`3LBa={9sT?2j`r z^L*y=+xR!*XYNU^mAzmPKRL@!`jZQ_Q7?-9@FBbhGtV!kvyUd1%e>aX{TJe6*d4Q1 z_i|5izs$eo_}}7g=d#~s&&%9?xq9;aujql8yea$I1ozHTzZ8q&8oV3xej)o;_L1Zg z7r8HU{78Dd^XFkx{vR>9%j-AM}e=AImxrn|V*XW;2AEyW4gZ$lea=GM+$tSYEWq!}zUCp_Y?zxJ7hW<0X0Q=xv zoT)GS(@1&?UV~TT037Rl_V8wOPyCC%&*|FswOVu~9IkIWZpB9G$vZCQC&%hazlNRh z9p`?*M=*J2L;4^3PvQIcADpgl5_Z8II9p$G{1W_2osx24~tuf*hw>*$U8 zI^qicG;GhGj>#+kss15u$Gm@gn;weEI~u#EBEL1Aydt^6GWC|~3oyCg3-mZ_tA8QA zh3-c`N+(zNo<1Kd>ARkui0k=(#e;Yfmtq&}hRJ`<_uu6_{{8MvE?S76eg8219Ny#n zDCZ9GtKr}HEirlGH1&o!7|Ubw#pEQNoPQPT;#vAvV%{qxpYNeB?<vrOg<6nl|v7-7gdJ~EBMLBchZ-r z*Q9&nSpI8tUH`lE;2-6Gfo1Vp_2h$(^XK5x>c5t~{>=Az<|pu5_zCRvd)d|RY4*14 zN6+v-!$Wwva}#hXE>eGy&K}X2e;AXaWKO%6zsI?rxEv?p%lHmvFHBBwBR_jU6*|wU zBfd_5es#JXo!lpL?z8;tt;gvZ&Sj5Z&ClMNIVy8W@|2m*WuEImPj^1^a&m_!)w|(7 zeJAP6-IeLWSVv#xxa2)8)VHYT`SJ=sdr;=6>~YycvNt9V%5yHy(d3b{+?P2hd-V)o zceQ%-VXonqaXz_T=Htw7CG=&FO?z>FsH~p3K6}>> z{6e@;f1dMe={f3|1CkFVzr9pFdB$YAnDYzhf6>X6Gyi_Cp1pGkojf7ASn|p?>J4y^ zdo#a3#?PKqPW|7QeLeZoIs87(ZKZ#QQ~1e8GXG_+TBn|Q_a?e74tCGSnEdh__2gE? z={5RZrIW*c&9BL?O^>9rFPzJ7f^VrO-(N{*FB?IZ(4Sl}d&~fS2Ys1ycknas-bQEM zzFYqUelJW;xm!JZ`XV|x>s9)X@V8?2yv(uLN0T33<Ayi=@{H_L?fKd3r|KVx_0+T14WhHxC)eoA-;cNA{qD&gy@#%Zr7(G8a+jx^PmZ^P z&VHQzrz-z*Y>odn*O~3>Y{3;cSO0K)1IuIH4-KS~Umkb9HD(V_{{1R{Jl4V9?x{vE z$J00*@6-1#R>y~N7hbPFd)-g`q5P(F_Tb(8?7i8KHtS2CxL03EyaKfqjG=2{5qwEQ z^3dcZ+tmMz*=v*6_Ti7l?d~ayz3^gOfyoR1lzXua_Q&Qpz`0tOz4-lTJgC1GW*<$i z(?Z{R^*_*e&`t3K4pUEl(1oA9xQ2Rp{wZw6ABxMcG$s#xPybB5kHLZu`PsXZi#Fn) zr++@?{a0!AG5lThYuFwOtCzvv{QYfZHe}R9*?&`Df0Dl_2k3LEdq)XDtv0Bly zaXmhSb8|0#$A6RVj@$8OKgi4IPMEwcd28N_jB?*7eR)qfpB{$Y^nFRcM(6!NKmMcG z7Vp7&_?2@lF}c?rbYmQh$DDhfE=1>j?pXe5T#J9jzF0Tkho4Uu{y6@RcpWC6`is8X z`6uZGbVa%W-7G%dslJX*E?buFr(OzsU~-}4c*)SD=&ECeLlfzgFLGArmt#;=@1Dn>;C!BYf8+mypSdY> zb>_t#>e&l2cOB+uzRrG`=iYet_te)8oA5LD9O7pW&+{d7c%Jjw$Mc*{UiG#6zQxX% z`TSz{FTm{SJ@ggB>G-0)%!kEuUi}r!b2Rfu8GV@x@_byv@9$im^LcJRq25$I^KCQ! zH2x-R&+mYv@P75%=*&skcc$~9Ap30g`r`_jC(m@9;8c4GF-t$3Es#rLT9fy$7Vcuec3~f@@wM=^$W2w zKl^0%w&bq6^u3R3@g3(X(-r9^blz{(rcbJurSl$Y6+KZsdCtT98?XzWt8Y1O=C}3p z$oqyb)PGP<9+f@6kossWq3?4%fDht+?4>{XQ#bl7^^J6Yx(xj&ojo-B-Zu3`>L;)_ zzNP+qT+Yva+=0ImAHuo%l3zT@pNaEv1m5H4cM+ZUF$3IF4xfqE_dMN>PClCV2|Lxd z;~snrn>$wx@4;0#3-cb~-}E-;lVc~JE2}<4eI7Q%t@yaUiF9)8OX&x&mA=+=-V1%m zugA}O#8>IOH@S*#idCF@j?R0vp8Q9!mHMmnYC1X320HI6D(UNjC-545$qADaRN|My z3i>D0$+yPP8*ziaoBa1ulD~_8Gu?(>h@ayd>V+^l$EWnY`d-KD`A6ug^nQ9QuIDF* zok9Ojy*qBkr`3~-UQHLlyf14(&vR}n&gbu=U!t#|oBR3PM&~`^Bz@Q73+iL(G4#vy z5PA;Q=O;fuLa$fob+<~0^jFX^ius-(#gq2@^9jora!~&nD@!w=u5tJ zhkE|+TpH2c^%bT^(>?Ilve%#aKF|CFehWW=u6|E5=VV?w%wK_-|1(!D@cTR-liy?? zJe!~Aedgvo7xFotc|7}k=8WvUQ{9vO;a|R=?3L}+vj^lk^*%p)LY}uPo&Ot7)7Khz z^Ro}v;b#xY9GK@va)RuOC7sV)lf5j@&93e%uP@KFRs787$wQLUWDb5&-?Nzcv9kLz zH$R|$7ynjz9i4n(GM)Xf8J)c&`(JXs>FUW5=ef5y&cZw=cjzC-KMQwYWlZjvJ#HC4 z^JezGj_%9cka_8l{xz8W@ou`0bIA`f4`q(Y+?<^58t0N5w4t+i?R8(~&ARFz^OJLC zj_$|L^E>lE_NL@TE%epG%$u2Kvwuub&)ihS*Pnp9G5b)S!^uUmmuBurE|mQxb4cdE z^L)L`Grj5Ce4Xs=*{9y(AJW$tn_zN-=T3B7gX|xDVTg~toyPDXa9MKpE>XYI=RLx?#bT%oO);eG0dJ+j?Vsbq5cE-SIoSe z`Fc42eCM+#WM4eUFRCwlY)SeT^~|?Z_-pyu>kjkp!*?;c&1t&0bLHv&boRozbWQc_ z_nCvQSI<6?+#qvy_KSngCr8~tpX+=}9ETm$7vrV4T|ImM5`OZM>_^GdYv}K!Zxa0k z{TiKpatOZ|cE(lulB=HLkHq`c^L}eGzW~?a?fQ~8WpB&A@r=H&^=+bC(^ujP{Ny0n zW0tEY=g8iaeD6o~70%bfya$;>Z^SkFx6w=KK6HJYi<|KRd>aQlw*<2{7op$OcY?kJ zkMfsdS$;>l34X!vi`QXt>FodC^UuL8&Nsuh{02Cfe=%nNoTgrbe-lpS*Tt>;?KqVG zXIzwid+iYVdi{-XKED_?;Qs@=;b!%f^gzr$e_Fj7|66(y-4wHT|CD|_gA`hHJmUr$bx_XF?f>!Gg^X5YV3{cV0j z`l+15`l*>n+Thi^-^PcAh{++l<{V3)=-FQ0hjrP#rJ6FTk>BcX_ zUqB~+tgY`W_3!XMSW5pe%zMFB>d66<+wA2Z!|u*Kipgg;(AD+jea1cfllWlfU@F)J0bVYg? zzR7PvSEVnZlZRKL`(izP$*+@>uIJC!x05b}V{r~P*7qU(TgU{?{2kBy1b!_)fy@h; zw=&meKFK_l&;6NxkMnty=WOP`d_H6k%RG@iX{67C?CIHOl80oT%ID(U`ltH3ncp*q z&Qi~Qme0S;bJ-V<=z9Y59Le04=i@5%>{ZDF^7)^=GoP1v4kbs)evo}M&z;OO#oV9g zM4rcg^Y_d1H1lAdn*-G|Pdx9QWq1My=(~~5^D}d0K94)7Czr}Tm*?JY_askDUXyvQ zr1RNBE9lSj^c6bK^E@|8IG24QbNz+?zrA-}QBB>U)FEUXuCebM@qWTj|W#4>&gllhgK~U(>&xevf_& zGnZz5S;eoQzcHOXD*NmV^~{x%={5Sl#%cV_QxEeqH$SJontzJUJbp9%0sf*d`&8zi zY5e4D<>{`@9l|cy4wI9v)t`K3tol|wUwt{;q>Ff!a*V{XHzrIzSI z{T;A3e+%Bl&%QX6zaMvDO>C#XF7C(VIC)Qyee8PY2J35s-T2wlvp@A$zeBw*=6ygN z_2K;0cn$8u4LDu@WAsG)9J8M{r?1BBW6hmk!O#1xkNC-BvImdW_XC!}{CoPp(0Lz` zoOcd?4YqfG8GM#sf-dU6uXpIYr>IXSS3K!l6WoiVaI^mN@OrF;3vd!v!^QY-tmnSu z!bj;czR%>NwfJlCJ}i!Za$j=C?B$*KZ|kc;=l$93zE1L$?7^M&kJR59f5f-c>tcP( z-v0@`1M|Ko?=KI#r;7SN>EGcBtn2%`N?&rh`Rb4IFQPxjSNY}W;>a$j<_~}7cg6wy z*_iy~XZ3abe%O+KJD$dg*i7Fky0E`b-a{osDdVo^MTJ+Yy4arJZQs#usm6`$e1NzbL1;UK(Ky&gRlKf^uhed(R_ZS+z6o}c%bOZmev z?~l*+^ZlOR50lqjNsqvH+*_G0g*Wj(#a{d=n4Gn#`bONLUO+Fwa=x#Z=*qZJe+#-O zZo`G@$M8wad%FhuPUEj-uRrsBp7{y<7JdR<{l1p)`?(ymcb!zfke@j)&%x~Jd7kET zIQ#Aq=dypca9{TCmGq_hvbSY!Y0b}^nRzGAiOgHsH!Ha>pVP@9%JVb#W-oux{cF{; zKNsWQjrqLnN+*BWM`v%$zBAQ*nR_#zX0OO|>P6?5C8JN={(o+yi4wtTrT_DUj4g$-RvcqTaT#EP|y68x%rTKo>!mK+c3|^ zS@bFQ^vC2M$qgFlOHPrTBy&;n(+>K6!h@K7a)JBXVfKaz`quD!V4i2-8lOs@0_{ekBU=uG}@yg|Jfom^uR-ABC# zoqas>@qB*f`OKY9JC}KXn!d)EJScnm0sW(KJ?1?_CpzyplA9(^O)hq>`^#eXuI$gv z`G3RA|0~^pBW6E;n|=e6n`Cdv+}y@}*(2-d-;S--+tZUVdsJt7xV|d%3wS3M!V`E) z`Z2j%_R{17H|bCQ@Sgs%{Ny$r`71E_SoRL>1;zd0b$#71b9{1*()#XK&z@SDU!4CC zJ(12Ha}WKD`WJNG2PL1JrT!F_($@^D@h`?Un0)dzx~BfT7n)5kRL?$gfS>((JUvI> za7^C*i~0<#gIn|!#fLF@Q+0X~PIvBJOg{aMdUDw0D#Q1kF&86HpOxJH_=P61V1@?KYrf#C0D=M|3Jwj=juD{{D0`=h5h(z`R`*h zd{sU9UoHA8^+8yTpZCYf0~@ICQ@;&;QDw zt8Wz!!6$K`zP_0EHS5*);AX6!bC|sIL%M}~TG0!z3s%IccvAn5^iE7}R!RLd<~?yY zx-pi*OEB*#A9r8!<>X-X_{sm1(~jj|>z;}@hyN(%J@IcL6FBpCJo6LywfqFKm*?|t zo8Q-b4rkwZ!tZtFox%Dt@8|R69e(!Q?CZ%HJbns#>09RB%o%yU-Oc+?V_& zbIBBb=DN)DKQOU`U`+J;rt}&fGBl%H(emhJa zmH9DqZ|0hB+_y{rDmwFIa?2O_nWswX&)%IpJGs+7{Uh;f{mJXA@RJj4Qoov?`7iU< z#r*RybLW5aGk$XD%)Qx5_U3*cpy$%r8oC*yPNoavd1SE z+RQ(SnO8FpFXj)!O76W4pW`RDYDW)JUr7(7v!@QFAI9u|6X{o-+la|a>Znh}pYUCM z*?+S46;p45$Mqe=+xf{M_VBCo+u~8ozL$Nhvi^pc{q`;Wv+y%4tnUC0$0_RBpD*NR z?@GS$F#kRMQ|ZgGHD*s-PjAQM1=+J6ET zVcr)ccWBBlg~?Gaa8L63A?hz+N%iFAH_$~f@4LF_Z-^_gzP<`{a<AkN$yt*7?dLbwKbp?| zpMCdr{yUiWWyx)qIzI|4`&?eCKlxh`dN*#xQFhs13ngC|t8Wc{ zAL>6w51{it;0XU2ejRLykEk!FlaD^c{}ax}j(7<3-g-Oz4E}<5VGH-)hWGOG9(WO* z_np1<&B2rEm(a;ilCvcTuBP4;*I@F;Q|>8_-PDuwzCl;OYw$75dzFfG755}(+)w9y z#moA3^5@g{)5)ne@TcR4>eth~un_iCKSB?|`TXzb+5hjRv%hBUe2|~q=?*&2<=VauyI`|F9K-A@CFwSpIXH93bobn# z-i*%tGnf8E{b4%u?ru8IljJDbpEG}#)VB`b#-r{ni^;?8qkCdLxAXa4&iQlHx8rsE z!f8b5PFiD>ogDTC?c3EAIQ^2_5k?C756xPqTKI{Q=Rr{s!x zj%V)M?|gY|iDmG5_a?u}zLGuuPJP+?vX8yyT;}3k`eyPo?@i-p|Ic2Mc`LczpvJ#$PGe)g`1>E-$s(*MBZhne$}M`hlB#koqD`MDNd z!#$brlgl(#UyG~sB}Y!amc98e>Y1yu*KXl=bx-E8%*zMVAHZY!GG}L>&%QKK{iOc> z^lW#1i{;aPyJ%O%7XRj@ruY=k9TIT&ulklgj>*fCLrrpi zhI;mt?0eaFPV1YZ?*xv+yq`)wko~EudTHnXNRP!+{Oqv<=+StOzNVP{CAr06eM8jO z(dS{_rxjCAPM`g;x4zHSufTufPW9xo$;~SAf6+IG{)j$7Uy5JyXV9(aQ`n6ECf$}! zj+=a;J%0oi!n_x`!};uY$-DC2>LLBT^lhM%Q)QoA$v?|}SwnwSelcu;Kj2)Phf{FB z^W$*=?!aOCe7SW}fdC!pidE!T@T=h> zd|%%-+=I#CR_R;8--?T|9=5{z`YY3)(d+3Ebn?EJ`J4I4hu+{nhIzl0Tz#beqj;^p zKj1(4kK$VXy>tmIgw1ijz8<)b-;JJ(d9RlDHx2m{@o9YA`Jz~b-;#a=lPh+k>tXW2 z59w0QCkHITZ-h@`a`0;U=kR|EnZTL9|FNNed?Y0$-k=5!C`;M{`9tU$%&E^)Yf+Zo9oMS^>%(EemOdG zLVLQ4diL?;2Vbk_d2k+`z3q4OTujbU!hM}_u=;h_0%zi#`d-6z{LGEXy^=pA2g!V% z{2|Z7?05IM?>+Y=PkDvTo|Sp#sQ%UJnU4qZ$KV`1s4sch4g6mGT6E_5 z6sI#6X8!M|{}aqyk~#8z=l-FdIkXx7Km6o}&+u>J-$G}9%p8_^HFIh9{KL*AkC{rh zzzOcVo32G??_SN%o{;&o2tV`u!}JR0Ge13>d+jN2sqe+?#q;zfuSu@fn%_l#_Osb^ zW4u}4G}9s0BPwW0T7_L}4>hbh1?kS+RVcs{KrnB!|rT<2L_T=IGWQvk;#$!>QCNTnNALqT(gGr$M7pWseckC$6iY>!|YcL=u^(Wi+OMNf_nD&uIiJp zH?Gn*gIACMSDd{Ui4zuc%8;Q%|mve6=Ni0ltqFolAZ*oZp>49+%)) ztdDc`52Y`{jr^;z6kdq)@dHfm^Q!YN@uy%XesZdL^l|m%3(19&vmeu!ysn)7o_RX+c%ILf_!1CtNjsc$7edv4~D4eDLg>(H4mGe2b>%)FfWJ$ul+ zKl8*t_*M81V>|w}IG*qEU+@K8LOr>0=GN>rKdG<6>_^!HlBZ@r%l@`q|17)#GnZw* znCJUAqMmszxk2)d?7v<0z2)2vI`dlg)5GeSTjuNQgqd@H(U<)(dvb4na?TIwN;n-0 zxi9%~@{G)-1N8OBXZ26P`}vRHLVhn?fRoiTk7ll$sh-^QV|@qsr|7?7=DVKi7vnSP zBQW`I_M92|D&ba~ifx^1Lnl94NB2?BUQ>bYuU?Z*j=7W0-aeZil&^>9^IxPhfA6Mi zV|#3_KksL<&poETKs`Cm6S>DPP8;>)l;5lG<7W>_E|)#!diB!!vu`ZppXL7Ky6gFk zaH4Y?>FmYXOZW4)WA>7+bW7(y$9~uV$6)re8uVW=|L(Ck-Os(Xu`quvy$Ty(6TCp* zd^&l0b^bU!8?)~f*Pk4zh@aO~e)g^GcW*dfQhgr15j$gYv=8XP`X9yQ3FpzT;b#4l zFu88>knF4N_2>OgUAjCjbN)Ccw;!V(WWWDPU-G0Q8j@e+{m^QCi||2w4(np}yX?1T z@%Q3l_ZOvC(0T7ul|KN>sAsPnL?2V%hsniu(#c0#(EoPsG?w$fcS-fhSXF%@uH{dn zm*O_8r#=E(<9f_qT!j7<|AWbo%hHW;mitd&^4-zu+c0@&-lMJ7-xHha%lm*~{I~g~ z>6__x^c=bep2REh3Vq30l5<_g@1n0SU55SyU*+e0RULW{zJ~`f`PL}tBpN zAQY_Aw}jCU-{j|g#5??v{O)ve#A^IzSPskUOYWQZ;?4P&V=w)iG4IjuQ@@@6B;J9^ z!yD2c;aKNtVg5b9x$3{e`goPTQJ6e2Ie*?8J*uAkY@zdc@70R#gjZtn&tvpy_l%`~ z3z@)~zvG#oz^~;eaFyT7%&}elz8=8jHrWHF^EWwnA)V*wjr0V}KG>hW4WGb#z9hd| z$j@Fl!`I0^l$>przWdel`I|i^dtdgDhxKO3g;~VJghnWw1 zIG@}h``1eS$sdR7%if;(ICD;N?`HazIo}Se@CVV2=@;pL)0t0u@>lSeU~hi%tpohb z&Dk3=cmH=U(w8|bbJr{UxAFJ*IsV^0nq4r?_i+tgmUHyibO&t0A5CZO&ip(_J-Nyh zI{VZ#=d#aa&&j-*eCu0%nIp5`{6F_yjMcCXmcU1xpGjXqXCD3k?cMv=&Sm<>@uy-? zCL&skQ7G9)l$6y*O$jyGG?Z;=Y$D4dRJ3nMQBqRb&X7UKHkm{=t7uBu)L=XKrpbNAez>pYL^S*!TQ@f|j)i|=+d9ltd5b65MU#|zYD ze$D)Gn|$WP5_J0TT67!h&%js3H)H(A$?~}$=}%|wjqecuCI0;_>MG+dENcJwvz6%h zg2thO^y^ctOTYa)_3{7WvsF-cE)G$bxi9nDB>A`H-=W{Y7UECSnU~^2)E2MpjmH1a zytc#ors^_h%n&asegjs+`SN?|_$L*`-xU9dZcWDz%KTnjyeF zPkj$8t-cfffCJ?_VtmGb(wV20sLx#a7`;+mKYUjFd+dUl4|}W2oSONwk$8Nqd#sDk zSX(~6LVTe3Cz%J=s5@@`TzWq}mfnIB#E0P_jIZ2FUFOHkx2x3U9_fU-uW+vX9{Ov# z6+MpbLjQyAP8X(M#Y=G`9#Pko-bIhc8sY87UL_mq@kdp`fdW7@p3$ZL)ATkxlf6oI8t5w;!f)RB;J-@i)AssKr#C7 z>MMCZ#46U!#F02$z9!x%z7CZheC}7#xi?-< zFIJ!Xqt)W^!Aj9B)qQ|##eXex{i*kP>J#`ad;;n3^W0Crn7%#F^ZR^%(_iI0$UHVt zKF_OSbmnM}m4ZB%c6u(Hr7q9!oR67LhN(NI?kjo>o%1@+`}8w&I{x;`yo`<)q8-eNX9;NfVOkY*iemS4gALRLaR6cXS zB6L^&v{T+Jbik6i_D`xs!!iKhTeqNSl^9KKlBm(9low^GNw=bmwfuq z|H!9bi=PyqB>i3Hp!5xUtlMC}rF7=q^f~k8(@$ky%sibrGXB}s>NA&CqvHeZ^g7MO z(=X&aUo9Sgwm|&=@rjted${~cY$2aMBlBwLS6)}$PuNeq2)zq4C#Ap1e3X9FvV!=$uUZ#hpbz~Drf+`B!iU8(FQhNd z9I`-NYn-P(zFsBqDdL;xe)J{S1B=L~kBq81qZ8ZM`x~DLB|)$eP!m2((>ovi`c<_-(XXWU)Gr}ss14T zfa%lY!=}$ZB!9yC_)76PGY90JrG)w?)NiJn;ZeLpK67*p`mb0YAHnK)8!oZWUb-*l zp662eYccceGMv{^RHL>crLO9w+w)CB)~c8&79Gj2{`lJ9A!#)MI>^%r(2k z=io2>g7^xT%V*x{GDJ9+}E_DbKg}%qf&xLj@ol&l%gT4eL)ce-4LvLMco}BC-%5XvHLa_QwZ(hl zf5k7MPvCxx|8xue4UVubK6GWeJQh+HfAS%EDz3--v(E45a`E_+-Q{bD=U%xZ9l!ip zdK3=9!Z^}C`|v99S~v*L#&fY8es0}$bR9bWTYQ$*;!E*XeA>FI_!Q=zX}`LQ#N)FM z7tj6YY4TTzm#1&?`JGRXm0yo%%f}ySq3%ohpRp1)#sfGAJ6qQnH;PZ9SI{3|?!mgy z@nfD*Umx$p!MF@_UsBxq)BPMb)2q~di}&IecmS8H|2;0j+&_J)?m5i;+GpzGqi>@d z;>+se%MTUbgZJU{*h+m@I`=3S(pBW!_&M(uKPi4a4#W8CzXea=)X#Y86Zo}!0(m}_ z@IB4>z1mkMd$o# zDxNta&-Xm%az3Vy`tSA1Igo?VIPu+5f%sD%YP9L{W{d9aE+v2V2AI7UO{!RL&oZFd;(hsD6oM*qx z8TtE#%oRBwGoSBMKO9GS-S~SsUo#h`-`lNz4#rPfOQ#=5pOwCKn*HNjW$qa#9-pr^ zojG%_eLoUUpFCSU{aO0Q^d0Hf(m$u)jBilKewpji2Tm8SW&g|xKhT+n(obdX&z#&> zUHYYK=*+)$=tmRxdi$`4c>K86=(q4yJc_^cYtv8GmLF)}jr3uR|2j!Nb6QI}^H_Y# z0pjs>2H5AAcrUs)y@-xa5?|*N`S?)B)y0=RgT54tt8Yke!pwi)(D4gPt8axJ@CvMe z?J@oD3Hq4*GJjl4r$0^KdxQ86jE{MO&YU>IzUeE6(1)=tW`2l2Q$ze+v~9sk|Cohi za0SMPNS_=3Ha<)G-G27{3QJnwEU%|8jz1b-K6CGEb(!-s|2DBt=KOy2JoQI0bJk7r zE5(1t_*wCfwu(Qj{s=C{P4Y$Q@py%wqdGl7-9c<3{{?+OU1#iy?_o#$7~@C8FNz;f zO+NQd!|b2A<4O5y;_(O9iQg@rd$2O%@&9x0k-6kb^)=M}8{>hYm|vc8`D z3s^+F1jZkGobIXaE}Ve5_p3>tp?)x)g_#S_QWyXKM)`K)m(UCG0`dCv8v3tv5ju15 zcj8;{b@}+(@vZ90my%yhPsaFrh2{PF6!fQG#@<*Lufo!3yMp*al?CEU#K$bF{xZy5 zehwWU_Z92+Bu>XyXh$E%%Icr`>ys>!v_V}mQi^un{Nxw;V;-||iQ zH^d9!`FOKmnD{MS#Gkh=zFhqCLgKl97^{9Ko+tkZRuUhK9k7&q6&#HP@(u7byc#!S zBlT;sD}E~fI6f%84!;)Pfbsp~SLR;f1ND!oTR`XU0XEQo#H#9+(05=3@!TI36+a}t zlWu{#@kh)(UVOgXBfez)LwK9|V)RIQ3f+gkAD_UP_^!J77{7G}JyBg3+$CO*UW7gH zKKZ%WP`o?Fr~kFg^{3wFsZZdy@Cl^9-{pInz9hay`mD?^IWN;MWj@RNmA)%|O>_Ik zXV^ifzj~OCf6~8~*JOy=y&6**UOZl<4XZ(aO`6825Mx``g3F7r$J z@62WChrU-|Uj4One1P=j@xT5dKMt2-=Dwfoleum?oq2by`pn(&G2+)uR-e8x{bBma zoZp%IGJig4-}EnA?34K>{$l3QPV$-m>R5jh#?Q!k`;GV@Jb)`Oena}RLH5sFdW*Uq zSQOK5w^9Fwc;>|PmpN~j$hT6T{x0+6LHW}1bFmDL!p``d`qgwrTH9E#olc*Sel&ej zZ}~aaryt0?ocTNb(WC0)KgHMGD&EmPncw55Z54k}UFL`)bo!Fvbo#o^bo!U{`xC{} z*S%u@^#32xrPXB)Z6F@M_JsU3c(?pg+=H1b-&glBK8=;sEyK(Ona5kH8z;Xi6#qYr zPxcOdJ|4s6)^(xVV*2bY^kJNYC9SJQPp4~R{ND=nX?}s~tIr%8|0=#{eCNzFnKR>a z#82sKpZE~hd7bzjGw6n?!|u3QK67M$dN2;bI`}1~FVB1$-?poL&&6u$-^RUokNoAB zxh{Um&FaQr=GpicW34NfJT4Y*OLxKR#N&fE6dx&`IbuBBTfPt-pKK-lH~GvdQ^fxz zejhFrZ-j-#kJ6W7e~kY=OkL)=%rOVWx ze@j=SPh#f8zVrlq1P9_W>uX?qu21AMPo6Gc9pgW3Q&(JkJ!bwaB40*4zQkhjm&I$+ zb^P3W#eWo^kGV&yLTAp4-}hhf+&h-D&&A^Ly=tXSem}jN-i&*26n4Urcp5%veH%J{ zN?AY8a`9n!rux@$3RaPizxJVcDe(pL7xX&%E_xo;5WfT0aSqY(MbA|?TYNTM zm_8fhPsIN?rf!4$`}9BYF`OX(0G&DiCGmUlb3BeWU=!;mVjuDNZ8hm`SXA9#=>Ong zjBl}9-Gky6(*5XJbneyer#s3&i`U~Eya3~`uC%T)28-0q#5*NA(BtXcGZctJ!F&Eu z3$Ic?i=L15@EMGM)>8dd;s@yAn0xk8bai|cb5B_|+CEk3o%oP=b8IRepKrBz2k~a~ zyBPm(jr*aI`;$de||3>-*zKC78l|V_Nh-F#?SCg9E#mB ze)Q?ycYNQS_KVMT2_1hqe>d=ieMaI*b@6w93!cEKpYhZu@N4-5(wC)w&-Xn2W1eq0 zKl6Ob^DOgJ&doe`kNMu$#rP9>-sb#DKbGfl&d>C*eLP2Uu4F#Ryf|GxK1oSB{dD@} zJU?^JrcX?Nv&g#4Gb6n3^dI-o>7UbYR~FB?kbdTL@$>APId7tPo}ZoNbDmVAH(>gi zAL;bR=^t`#u9Q!IlD?~zc>0j}^ce5s9n85gM!p;79LjnBg8J#0^SzP!oIB}Hn~2}1 zK7C?ax*eW_$_vt;jup>*afgD<;xjOFRs5a7>e45qe@q{CmUU04%RH9zJ9ANd#PkPe zs864fJ~I7z=9%W|ZtyypQ`4WPufJA4^LH^ibNVX#ATC?C%()3^mqCBFn(y}n)to1 z$Zx`;){n>H*j7G$;>Y6Ydq0$~h5hBH(dkEz(&^)_rqfTR4}94A`20KQ5;z+tV_*BE z-#tWUu6s${ix@xaFLe6HHuMhbAER?Gl76JJ#TZkR; z4|q1F|NhAOCgQn&(KZ*HCLSMgBOO0)fb}D=BgP*~-`(4~%>@fih~58t3OA7uXOB;L!qd3aJh^Ya|KB`#IhnyyX%osJ)p`6WI?{NzgN-&8-E zZj23Zfc$v+Tsre}=93ow!!(tjtbRFG74MFvF#g@W^jrA2byw2Ma0zDaY)%Wm-ffr!>y|dJ{$BMX5-9g+f{wF&381aS6%3q9`7oVZO!l$htikD$^j1Rh4{b2F( zI8^)~&J#aDXYSlZ_mV$GucxoV*5dJpCyBp?cjD{nX3;m{I?R3Dt?K5Aufpx(HL;NR zee@js28&~9b*r(i_J1Wjgno@n6eWw?)1WPR01@Z>xJ4 z;}e}wS5Le!{?G5AH{%iYTj=ZPt1s|UO`J3t78&>ssj1;e^ZW~=5hl#Jn^RWeXRW}IZhvwesIdvb% z|61nyQ}6TCC-7VN1g`b{OdpZCDScJu&CCy(XQuf+=X}jPoIWA_PiOm|gXz2DL!__E zxpSs{$Ebf4GhgO;lldp}*Iae!QzzLczC`+sJQsi2e`Y>--#+OB;tS+_%{h_&BIjVv z;q)mvx6>D-4~;L9ek49Y`pul9*LWX`{2co+^UPWDnUB&BWbVk^GQhg@otaj@$|R*#J?6#A2LKdbLAX5ePH_QoQEZ>|D(Fy^d8(Ho_VpIc+Q{r z3CG0K2c^GF|MjB$boT%mu9}5 zt}cB_3p(>s`qF0NU9pC~B=gek;_=PmJFgI5=6$9=T~5bW987z0iZ6Jb_!xEZqvGd$ zB;HuvQOrHXX!=KW!!UDqP5B2fefuBi%vBBP^xv8L&a>Z_I7r&xP&@>}s-@ywOY==c!T)a}L0eev^^|L^}d5TCq=#H*Nl$;$RwgQMj) z(B0@JbRoJNZV``vxK4Zz<{qg6T?bF#f2^N@4e&GhUf2M0e>+%Re50}Qld&n*!t>R~ z=lH#N7xCeA{G@5(@%ulJ?}*RJpQO9vZY+k6tBbGu1$_xFP)8QL z#rQ05%CEMzme{Px&P`T|Csn(EQN1k{H$H- z`-)e`HMkh_cMMhN!q%OQEyb^*>(eLc--0J_>SsLl3H(|+Hnf2Nx6{Z8DE=~MDNOTU!9A^pHY`!>V)2I%%N(7)_o%v@r|BQ=5zqXR zez1*r`rdYQ`q<1xnfubGr5{P(__TGI58`jemrehdz9&Az%hv724H*9{{ZQtDpM75G zr{l|HPDx+)mVMHv+-cna%$%}B-74|S1(%6`D1HpnpN^1E9~*yXrub~U2s>M!zNa<) z%f2xE%RKAi-^QPfAD_9avVAfKq~9+pp8jedeU5#b(tpt54rtQhc)bEV?|Fz~b^ZViWPJ@dAuLHcDOQl=brQ;}+0O zajE(_I7Ykqo~w{)%3PEiisj1Nsd6wZrGon)`$o#N%`3 zelvdV0r`Vi(f;f3EOGzY3l5485id)(qMOrG>7LjZAD6E|-%rP1J6k+HM(%}+iN~+_ z0}Tc7$GY1u_f47aYb(53zC5pDkE7h;2Z=o;86<7p&VS8L`-Ff(i_((eb)Nt`Z z;$7+7OH`mglb=GLpj*&$>3wwW>z<=K%D;l~Oa3AMC;U>rDjnZ=B%S-4Qgl1@edr2w zZ@f-Ce(00*BAkzf)#u)?6P^3D_{sOF?E%Tj}rfoJqf&=W6=R z_$fK3GY3BEIdY@-k$FCI(M0i_=Q+1Czvp?JIWK+1nbu9f^l@L&@y-5AU+aBje$Bb^ zmUw4OKQ`F9^vhf6^dmLZ$LApn(if(0O+S2G!3d1sH<8ZynEv~pUN8OYPjvi)^a=5= zzP8UE%z2l&F6VOQjr29g?Az43%vqU3f0R#un)xjLPx_8l>T>R+FaMYLG5hz!_$+tG zPsO3~>HklPr=Ol7pFSt&Y37?I@{O#EkCA!g3GwvptLPrqjiDc=U#D{}$Ct{yo^w9^ z!DRI-u^RTr%#Y*kpMHOeeCCS|@+&cORQmJu(|5^dZj2vyr~Tqrr_Y-$p7}C!*n{Ff z<8f?f|0VQ%yi`1W(0lYw9H8!Wx(b~>FMfIEoXqPb)E&j!txw+;Um|nE2Kmg}JLvch z*U+VLI%Zxf;&rNuXKqZN7hiF#y7ZSfs2?Z3A2(zA=DOEF^X$A7FXpFThHQ|8g4UN3!Y=9BoIqpUxK`!GK7i`Lf^&%6@9JAGqib!+fHxD+!7 z&7=olTO5n=F*AW#X^WOX!|-e3QHBws?`c z_&a|Ue;B*T-%sa0D!$Qu;_=V6s~;ggl71Cyh-dz)$Gkj0K zf$o~aXA*ptMD{zV}tm04d^*o62Ddd44&?N<(_De zy0hiS(+|*HG50O~=|Wgb{b9`gchrvcUaKyC@?7zTSRH?l7h-vAWqoVaO!6~^$Gl1K7pLq=__-txAFZ=pPT1I`q>il z{e7=@V9xE%be>b0gY#V5Z+*_WbLh+$>Ce)yZ?a#`xtw3i#WQzgZppdRTz(2x^*X0x z=IBf0Gk3;cxLW)>{KmTU)idezkNfEK4e6)e6Hh<0g3h`2iv81v<(%jxp67G=%giS) z+NYAb%nR{*>d0rlxt31~l6 zeJ}YU^y75;m7{d#`a9`5>eELr6;HpPzW4(1GV0?qr*HkQ_;FmTKK<@adYya^I`iRb z@lxWMm*N{`F50Iq{dWy|7sj8-d{o6Rz+n0G^YIPZd%Xkl2kB*WP5fLu^U&wwjc|l~ ze2>gWW8~A%ucW8p)7BNGD`QWLFP8rE3GvKBtLRGhi4WG0&K%Q1-Fhr0KaS4)I7PfO zmc-n*9H28drO!S~{3(nN5dSs)-6Z)Na2{^L_^6xd%rR@c&&(;K==jaY)W_HCN9X=w zJDs_+hxK>hUGnAWpK&}+!psGas?WSpm;OJz9NS|2ji&TP*bUUYq}bpIq}c*T5O^&^Y;?*7UEUu6_|UlVe}|<3+UVjloLNBUY&jk z-@>x;@d5UTZxE0FHC=qN_+}h}$FRP-`1hHE>xsXQQ`L9Jvf?-4mEr?2_jHfZpWtEj z+vxbZ^XLb$rQd`2z;9X?pMorS!9QwY?h}gHryrKW_@aI30oGNdi6J{*izjV{7L*Et`|Q{=RSXicuVo)bbOL7;-8Aw#>=p0wBL{TlGVibT3;IP z6>m)6N5_ZnO`j(pA2s)5MdhEDuSlPy57F_-hSLpjx4Mb+T)Gcd!Q1d|T&I2{{QzDq zeg?f07h>*Ta)1A$_-oen!GB{%j9*?I1cM^}kxJ&-mGS{Da MpQk>7-@+&G|0YWOpa1{> literal 0 HcmV?d00001 diff --git a/benchmarks/perf-tool/dataset/data-with-attr.hdf5 b/benchmarks/perf-tool/dataset/data-with-attr.hdf5 new file mode 100644 index 0000000000000000000000000000000000000000..22873b06c220b5cf9feb60c69295374c50b70ff7 GIT binary patch literal 306896 zcmeF%2mIIb|Nr}Fmr5xqG_|x9p|rPlO4)_Tipmykdt{_Rl2As&9z{b`HX$;iWMxET z=3MW6`CopQ|G1oUxtz;6=W;IRe12}<+x_u)z21$_=kwv0^V=7w_HX0=_y4E916@0{>yy9mpnv|>hyLT$|2${^ zlm*T?b@*wg4;?Xb)IW{Ur;Zw(IC|ue;pgy>(W6HWA9}v_e{T5qxvln}@_+c>-3k=h z_5A+lfAWOZ-F44yyZ`5tr~LDYLI3Pgy|8n*JzSzH4?~c3qoBZdyP@urr zlm1zl|BFAuKYy75)&A>W=6`bUzx%^Qh5siP|4XadKl!R6|E2l=!uWsmx%ziE{`a~1 zw=U>;$+J+lTGjvdTvaSqpyCz(d@ugj&)3++|NJN_{nu~X|M+~J{$DTuU!MQ>z90Xt zuYdpk|9$;`;`_1opYO-i|N58wfBO6JU;ixsSLSSvzk86r;|BJyx8&P#oI2m@EVIVQ z!*V;OyVinlzl}dILwx}cLk-zJSD>pr1aIyWM!70qjbreu`UJk8Z{%4x3%95%_`Tlef;>mB81|RD^EP~juUxN! zCi+FW6wlNz%=vm&Zk^0~)7kuXuH$K}*834nV9 za^B2y@tgcHTFQmc)@R>Qw~@Qc%lSCGtGAV3#cX*mevUuE>GEg%IiJss;G+e)sQYqT zl#_qK@p8tRU*#3J66-JoBXFc^`}**n@>}u_OqD0$4jh6b&|ZHF55r?PO#KLd%`ag+ zI;%_IFVw>~dKGvhI?Da{O^nC8sO0yp&68cLhTYXaaeckd`FXw%=cBGVFC?eP)A#{y z?*9D6*U}5vA7@}F`st5#Pal3vJxE@OqvhRr9T#%#Fnp(O!6nfX`>K1oeiy&Ohx4O6 z3kS&6-P4$#Ro9Zs%Mal+JfYVEXX9Yk%HdS~Qg~jkG}q^xFGtH$`DGr#-P}7{epoIo zx5wY=>(o!-9(g~#L$F<49Z%?eiAor&ZpIh#7kn%h;2j*{Gtck`Sda7E^9?`5PvUY^ zS5M+s(GcZuzutJBfF1H{xC+Opck(Sb6ZQQ18@L+wa;>-eOCG0gDBmqlllSG<@sj*I z-^+b*A`Vm^ixTcXOuaXE!FG8d?}i`bYw;82yZ3Ot=G+Hw=pDijbE8~`|8`B$Gq{6$ zPv$F7R{q|_U*r#PGxkF@{ZI5S!X4^sxRKsOc@Oz@9Em@1Jc_xt(zP-usvf`>aBIE? zom_vKFTqkArC!EOxhcPiLRf-ZeeO#Bk-z7CxDJ}hgHTJZ?{nwyi}(~L==b4{>B%4R zYy1PcV=E5SYw2@cZTj5E(tloh4T^&UBe6=a z9iEXV@JU?6uW=gJ;2PY9d*DHNfA?IR`_N3UIrqfRuARY)@sa!&`eQh1yRSH|moIa@ zqP&QYQrARVxh4*f&x>6@T<#^G!Uv&@-c5Wej?%A!RdUXpr8rC8hU}M_0kTf>bIO^Y zpZ(d$GrAwL=CW7iXOKNJ&-rT3{&KFoS2H`fh=&-SIpH!{vUQb956oQ0IG+{bet1%x%#V zzpCHDZOA-Sn7=|tIeWle?%9ldIyk)V>+thQ~d?JCuiTdORk9WxCeROs_4&xU_ddas z_;9qBOLJy|wb)nAny7^1)!AdRzCKjvTq(ng_&n5*UEVb(W`A0t&U(*U&Dz_8FGOAW zK0GS7#xS`qXK$;9X6o!shoZ511d7UKIs0)1zsa|JvtHJD5BXTMmb>wK%tcA{8oY)^ z$PDT8_ws!ddqnSm*D5%zg_QKWw?>zeXfR~ zcu)NZKg2I^=8Kc%ujRL}yLznpc{%40>SH+XJul1C zkaOvEz7Dq|GhPMv%$AS9>2l73tFQ%C^qxn~zAbzwGEcs!o{5&|hV1Y2_zbLbPu}$k z$$59XU!4cL30*ATgpXr&A^jKS%)bq}7SH0$pObkpGW(p)KOi&gVxMaz&%qn=G(3rd zcp42*9W`CgEZY`K)yve&_*&eFk?N26ModP1tkoOAdH4L4`=c28Vm|i4D)-!n_85vi zQC0sUE{Ct=tI<);3~{R5MxMc=_$g#29L)v!c+Oeefq%s3=!6ySKO6TTbL0J7hcoy7 z=KB8X{qPx1RiDpyV}d-EFXuDS6Zv<8L3;1YMdYvf26g7SU-)kI)yPcNoG-yS`W5jq zPQ>^4LVqd-$fscvPQjU2j%Kbs#h0Ord>J~(*K=(?lryKk!4IQ}JY4?{WUi=&4f=g? zfZi?qCs*hEJIF^o42zJtHSY#b${SI`eXFn#*P)(X=9a?pXYyX0+4%$cD0y#m#yRSD z`5XR$U*KEO0@q@@-cNWCzvEB6J@^E?ir3Uv^1-};PsT&&?)UnF5755@nRCzOqxBxg z-|_-}6F1BE@~8YJGD~NU`athnG|_8^-Spm3x0N%8PLwm>UBbQf#_%V7Ub#3*^)K!vYcn=TkNOKvz}+GH0S4AhOgy3k6BlFCXV9~+z8*vS$j3*9-NtP zho4mvXK~KWXY||i3j8kTx&A>uoLgW| zIeTT+`1|VDIBV`mIs4|G=!yP%SsU5&Guve!$bLGAk8~}2;~;!^KJIbSvX&B4z9$F$a~;W z{TJoGuuRSjy1QHr%hf0HugG3H4i)w8LFR~@c_s92P*;<$=gbiwVuE}Sc48r}L|d%F zBd%vQ?IeGQ?9G*VJ?FfBjPvgKDbK}MnC*HW%$H9^=AiBTFq+FVaD==Mj>T2DLoe^Q zYvugg@L}rO@}XQ5@5z}ryCP>u2fdtUZRL;Ti_lHJ0(;{dWNylN&`;hEW$~H&9^^~8 zD9_~!u^M~eOTCF)9_M3%I%msIyjlGUXP(P!uu+}!d$ZnI@-W;b-+?)D=F7~7Q`9f3 zo8mP2Rvy7$<9p<+&jZkfTB_^tI9w*z#bhi}7vfcDflqLS-VlBkRnQK_^zts*4#VYj zdi(M~9>b5K7J91lE_^QMEWVBp!~0l-*6x|g&9T4SfS=O?sxyyObbTbg zQrA}xkk?>~`X#Q)`8SN?xRrVx*W%84Ps=m$2wuc5sO;J(F2Fl+i2EK^cH*Vn7&oCX zE_A&s#^XWtfj9!q)W`BE+zYSEx8qdYhDmxiV4z%%Z^u)3TU{3Y@d5tSdkg#H3w*3s zhpY3!JOhWKH_q0Zf(PWAuwA|cJJ1)~&{4k_@}5vzT>@_)?+0~I6Q`-)bk8QfmH$8? z`A1&Duk&SG1V5pTx`xl?9b%Y#Ao}Td$8dQps=M|%_erh462Hjr@dmEP&+tTCC0~xk zs1STM|JIT@>_l~D^!B{cy?bH2yo7fk^F?NopZ7N%Gn3Amh#MHR?57Vz2F(w^GxN8xKurt ztMf&8Q+^cnQne(w+9NCXQ*DEj2!92W(><2^n zMjp)BS6`D~GY|2%+}1tK<)%0kJ&`s27cb-&_(|S^>@$a`pO=U8 z4CMUD{FwbGd)$?-Wt~>TmvRR#qJKZ1jO;1j@n@Vhej%zL=V^HqmAfMQ(;PmGZ{cfk zqWleKZ`=oM&_QoMWQORa9*SkiJId|+7Um-}YkHP#{L?@nAVx57X?hx_pXCSarMXW&(2->IaReY`5l$onGu=H==u@huvmfZi-* zE^N$`wU%Hdaz^Cbd_ex)wGrF^Td8gIkja^81x)~#3nj9w^<+Gv5z$iDh8I?0)v z-;9@yD zoTXmCgOC|D`~JRqnVWvo`vo^+vtHh}hR6@0E}H1=z-%ZBoP;sL1$+w^=UZ_$4nzyg zL(cGH@r&Gu_rWUMpgx4Z!JXI*BQZh07`~H_!glTbLZ zXUTaF$}IAP`X~IQHw-^vBf28*(sTI;*E45kp8rZcA3gLA;)D21E{P_XuD%yDLXtw-^%UOck#KnDK##{=cwnNx~?B8zmD5cSnpsy4x{j`dUw78)#S`uqqze< zM?w9}Pv3Ipu$;G9TP4&h@DVy=2Yz$^@4SZJ;*p%QvA8^oH*pm%!cTHDe2R;4Bfi$p z?DP)zKxT!_oH=KO>+6v{xgBSIX)2$Csj0bbuHzcytb9W6VLlBP;C5WDwtb^TACfsf?^Sc9Am z_banbm*5D!hP)H?<(#9LIoGK(|1H$(Ew|^k>g=OqFD1RsIy2Z|kbF zk3Wjqde7k=xg}Re&dnj|>n-KW`B^>?$6z(Gp62M^i7x8Q0dJwMynweLd&G_E4R}R; zCf>(q>dE{DX36j3DEThjhnec^WtoHTluPTK&6!L5w`-s6$m1{suetU%GJiHvA1P-> z|B^3KXAl39%W&qaUh*iMq<)?sMCPTO-d#TJXgdZjMqP$b6&5L&y^p+RQWp$LthNldkW9X{@bsW-UxwA>uu<9?j4HzPH_!Y}ZdToj$<-?0b>sWb0=EEh*PbyLonR$hKn z&ffikJQ&;5HSwTa0k6qda&!KG|HKxQQs08ijQgn{lC$^C#9%DYtIf5zAvVi*W1D;; zy2@X0=9|OhlTkzc6>h*F6w$j3A0m6?A>7IRIlHe`?~T06ROC-^o$FU|p3{x1=8wJf-%wx0k8| z&n8TgGv8jvnL9H7FVo)x)%5;C54jAM$z9O_8!;JA=wFF_<;-S3$VcE!bsZGL+v;yP zGs?MqtokM{qHf2P@wZ+tz634hah&<{0l5J(qur}_20Gza6w)7q%yoUaFy7a{8#l}6 z@haSfV{t7`MSs^ub7rwqIaBgHeuT_jcdD=n(B zIVI1?q4F!7=VvyGAam;-`kUk(ybljWdyK=!$ljARpY>gZ+xkq-+O_h5dY|$AcvZd| zMPqgLfb5Ssw=;j<#dG!c^_SrV zWR|Li7V>iUEaJ{mH^E6OGGcFh<3EmrGg&uJ}Ritg&0FbvnIKjFMH zRN$OPy^;MT=f`Jw8;_y`K165NpX2BGQ=EzH&%f*KFVEri+!oo7ZdBLhuADQip8Sja zH`<|+`b(^npGV%K+Nm>t9m|6-L;oml%V(jGocDmE`3DR@ZT;Wyh+LAhPyU5-uv4!( za!%ZaMS90z4ce%8aAt)M<>q)$eLXtrZB!SQpX46=9AC~0@fLEXbVLLBDxQd+WFOsi zeqO+jt6xKA^RDWdxJUgnXAeCC<>c(KTlp7^!n5iR&=GH-6t2_%k$3ZJ73a(Xzv+#~ z0`X8)z-#h}ob#<2zoou~JE&`M zMRk3QkZWGYA$D^*_hj-@Qd4@_ZGx zqnCO<>dAlexqK5pg#mu8M(U^KF?blSW1HUV$ooi5^$GHMoEh_J`AWIH{0twjF2i{@ z>5MPr%)}?~0k|0@)Xnp2pp*Qd-YF=ju7aQBk8n7OBJ=Bo{JCof^C|o`*WsO5C1-X! zOg>1?td&_T&&xbC*8dGJ$Q`&Q@_go*{$4L@I_o9tubO-^=iI5Jms#Lt-u1KRtg|7m zFT-Kjq|QE&XL|*v>OGE|<*)c&yoBscS;v_j@~jNPqv{WlpI5$LdvTuEEqYJLS%(vF zC$b*0k9;D(h(^dh^d~Ps=8gfJJ!&y>7A!>#%u#0#&)Qw6z8aaq@~k$L3%b@H`B~SrxJg|%|d*NyU4Ip@n- z`D0#>wwQ$M=kM#!lD|e~u2a?LAp6rJdU+08%MT*+$ZhKB@;N+#OK|qWeRw8rm8WqB zWFPxX{S;>{jl%B8+?2I{hWr6%AKFu%%vo11?>ejV+@B_Ajb-o4Go0r<`%3nvA2@Ts zKDPm&+uw~;-2D3`~(a#dczKk;0=g5v5@{03i& z`tps)UYYYW?=a&yXG#y=L%+KG8rM+Y%2n_g=BwLqJ!D@Wtv-P>6K6lpJbs$q4XCf5 zv$!xWSLckbtk)T*IPaRL^U=5pP4q`|OTHKVTtAlc?)jNefST&h*QvpxCCda3*rHsuD+WyM|b4E&<(>-5w-EPdvXRIB@e{m$a~~=y?!W(MX`QS{paQ5 zlI!G7qf{|e^gAoW-@bl+Zlpxz?1lKbLYc?$N&W$2HE`h)Ph z+!v?Fd-89nCFkD?%E_65p5tD4M*nHPhPz>3IrBtuxdEP0*Wu&&S$-3f@VEL;?!wDB zvrPUCpq+dzDk8I3N1lpX+;=5TmJ8rSs(@`6p)Ta!Z}~@Dg+!vuxhea>-RfH~1UU;Exi$o2kbNR+ zK6~{7jFSK4<@iXx1MB5HPY0lsyaY|;t=x>?`B{kCqBT{n1SMGoR5+7 z{eJZxn5ACG_u&9!UdTEgCD%sQXIo@1dyQM_eSs&D88PeYQ)JIAi>$d0*bP1Ovu>;K zBUmgC;#nw&y-*n^>3_^;;ww4lNi`IaGjkQze^kz%e~!F@YvOJ>>+%P*bzf$N!TLGN zrt4*9&g|Y=Z#90`%S`*Xybj-~&)^sNsUY8jX>#_Iv+y0B!&<%UMLBD0%9*DwRCmEW z>Q#I_XKz1&uU6MXcYK1(E+zDz$J+GOjd>$7w{6zXyi-v9gxm%>^L|%nzIua~scRtf z-%@Ud%K>7i(m0Mis@%A8Y)+pzr<2JhKYJTcp;v^CF:`IP!@g(lV zaJ}|C6xl~Q@rU?XedY?L-+Mhs3_mYJCIrbRc@|#2di`Vpf^ZOVn z*We;N0hc2)$K`tapaIU+%WPAIbM9ZFS4pmbCO8p4>SadAyL?A=F+8ld6MgWQ`gYFT z^)&yc&RkO)@5#Mznf`hF6>gRfLVe7|L&!VbjjmmT%#&y99g3X)`=UJ#!6FRPU&kxC zC}yB9cIf?rAF=D-%Jja+r-LfXLCia!)-B89*%l)DW1>A za(S-9H}iR%*{QT#NPdoQN8USnsh8t6^{IRWU%}^`Xk>}%9c_T72X5PyTR91b0 zYniPslP{3}!ZNg0_vE*E8#W>Da;tcM{o?2=FXHj|Mt%b?$(h?Oln=x+>YKP6f5cUI zKa7`0^FdsVf5oHnVcZlF2aav>ay%#Zormq1z93gCBqgcWG6m35Q#^dMRz z&r$ZE&-f3FmGk$?zL=lS9mqN;!TGagKhO6(-|IXFqmg|jYxR41E;=C3&PB+Y&VG_L znYB2I^9;NwXCGa{`FZ8%GZdLauGGtZlr!oxIp=Hkvplz7yC-XK5WbKPzzpQ)caPqW z@=DHGn!WK`InPYi!4za3$j>nQ>k#$3I0%h{YuT&%xt8ZRXYZ4o_45ed%Gs;4c898u z#&hyWWM+Fx-Gz6@Wb9DqneQ!UJ=c-f;466qvd854%ii%GXU6?^eGgE7>z+^09mCZ{ zIOoH>d`7;87jkCjoI7Q(LNCwtB+he_^Y$yAsF!`Yty}|H{B4nEb1|}aRpE|$nQ!it zhhRVCIjhIDF#|b|vUlfs%^GZl`;oo$KCY#ob2RI$4c1>NkNoDBp{_k#oE-Z$(Mh?%+9SDzC*2xIkSE(~+5duHGY@hfPLmgWpyz$RUe6s_z9i#KF4)fu5Qbt_#Bk?xl7b7#490otO}qi;%C$J}+i%NxUpYbj47%W9z58*K`~~tZI}oenkC6A&My}t5 z{JHb{X3ZaqtkYAFJuqu7KmR-j`Pt|B&)+HQKY!*2)E98}ocxTps_#O6PBW43aegNE z@Z%`v+CXlMd_O92Wwep=z3;&v@)XoSp07O5`TOMOS=aT&a`yV10Xb)~)<4nDzBX9S zbJtZ`+Z|u)`zP2Iz zN%s287}-B|ps4y2?v6a08;d}SSEU3SvTp74O&oS#6R zy^}FnuRLFYJoBy9-^-bq%5m1Ixa%A!$=QpqM9!Kz>dkmeJ%GQ$IQe+4juM!P?7KO` zdhxqxsh4y5EY3Q5n={KU=0nwmxF)~A^KrgBB|Y~nY}k$rL~D!Sg4vp;7KdKLrpHgM*m2j%Q3bJh2vpxjo!mRw){3`6k> zdh4~t8}jq~D{|(qQjft)>Ys5RnyGu@D7>O>h-c)>F&~9}Zkk@^^~O9AC+O!rZps+t~KTCNsl1!C_B}y@U?m@-@u)njp|d^F$6Iahz;?3rKU8~m=$Ui%&Aytz{Di?h^k;#y=+%{$r_-i9l1m3wl| zKQFJ~Ke;%%$T^c9;+%U6^vWae0MqsQA#-hCy$Q%UyOam&zlTGR*yp1cP);;WE%o=WPr@)3LncR@+4Krg+^@TGh`kK{hw#OH43%ux61 zb;cLSdvj&|);J0kafkl1n1mMiL$4}sKudMbwEg6c@&bMoXQH$Ip`7=vypN4?{a_rA z*Y)%M_=j9P_j6g4mOtjq@lCiK?n2JYQ#oh#diOjcmy|D%Ge76uVYpu2<&NW=|3C8E zDB_;?_&OegbMYq@=@oEa3pxYKQAnNHxwf2nYCOM#S-25G?Dk@ zk^CSU;V|{7oOjtbxQ+T#ZqL1uxr%oE`^hyzW|n$rr}r^R%ekQg{ldTVO}vTwxpts@ zxx9g|8Pw~F|0gB3NFcl?GO0O_~&SztyJe(iK3~W|cKr{Jtew@d1J#?0P za4q!p=NPBXd-@XIg41z0jzc^5RO1o+Ef3>kxFGWU-N%>mlbo3$>m}dwoICf(d44i0 z^w!&oEAEr z)$8cKujCVvHJ3g6MY*5;&B(L#E9Xq;#or-6gR?k4gWouNLiVCNaXG%ne7x#fQ_gx^ zjXdKg=vC*DJR6zkG6U4tDqs)al zN1w$h$o}>+=ef;Zw#7Yzcssv~%$nIVGe2zQ?9G{7rg8SIN}RK?z0W)-XFgcM-ywTg zQM@A;$2>fyex9=jPUoDrH^|wqvi85k(>M~1^k?(&Tn5>La)xBT&6)P2oZ0zkUW11) z8Kv@h&YC$vp2Rsvvxax_)6m{EFTKm#6rW&)oO$45IcunsT#);thP;mN<_D0qT7}p1 zV9vRab(wX0F~-U{A8I0N^*(hw_w36jr6*^99wB$sD}v0n-8p-4X5Y*Y*;`BE4ZRn5 z1^DDQO@}Zd;=~(dwh;oxW#?h7p~z) z&{-~rHE4#%@hpzP+c+J0=gGTP_PbxzndA3DS-HC2)u@bH)aT(ge1zTcyZ&b=EzjcY zX@AN+@xGjM>LBz;AFrS-j?$aM%lKZN86$h%X8uAi^X^n$q<$Ymu}Yo!v~DtTj^+H# z**V;`5A<5`0Nx+nC2DyZ#qNAM~586Ub`-LP3M$Y*jd*Y`o@x@)KhRvh znm1vh{0TDuWiA*e?~SoI)Agw+ibrvs-UEC$7NR%4zy|%CAB}zfH+3ml+%@M^;y3lW z@N2vrZBa+P2fmgo;~e=`JSN|YpXAIKjpesc7*o&*6I|=Y<@iAim$&79bdWQb-zHC% z59JGaCGUm1kQw-EJ{jZjjqBI*9KIceZ>c@2ff#@2q){G$aDE^K0O)Vpebsju+Q}5 z8MxnlOSmhajl3V_J^pX`SiFvxP~JTa`C}f&OZhZRk~iXE9IP&aVsdxR%ykL(;Ih1* z{x5tS>gtVG-!7lZnI#Khv3jCD?sfLETPw;4qwxfjHN- z4Y)}@5QXH-sfTjj3-;A}Ms9W&EcBM=bIyhK za-P4AcwOFxvoIc+v+m=ZyRA6;YhQdPPe*(GL--!O-%%U~sh9Fte2!D_6|$$*K}Xjs z;z~K+R^)d3(n1yrIV|X@yiOeQF)%lrbt~*S=4Uf8( z^K~b`r7oBr9>85VA19!b>)9Vl;W2rC&NEw7&i;2NvL9qWu86F|%zCfz4V?Y$OZfrL z^Vym&L>qKdAJ5Y;1dph{MGXu_=DJVxb4Hd?XAS54sVo14M_kK!+Ew0xtf9lX0{X~L zVw+r&|3Z;u_x~+V!F=4PH<;`5A3P3j*|pc@tT{seAml9Upw1rq9v4K;lFZes<$PuS ztEbl=N2)L2Vd-iPfNnDj@aQ55@Xn+=a**EUtCAnxHi{uyN$I)Njm!ILh-<06&U(e|MjHl%DakSinZ^x_h%Ule3$0@JQ``vTuBTx$$ z;ZQu`TIP$ZI5Tn+y@SylAM17FMtB%+sNcih=&Ejrt*EX3n2X>EtVMghm1rlgLiYdj z)b-@Oa2Xy%=F*%ClU*w!XRajLkiG5#9A>@=8>dyKwgK5nLB1BInx`uC2nW z>UX&TAD}-(_TR3Vc)ReD+>6iSO?V$I)VHFid=z%`xyRIh;!}KrLRgQVUC$ZVNWOsc zuD83~68+^tdZ%L`4%E9CrR6nzG=Gk}(MkOo@5bL?Pu!^f0B_1`csQPu=i)OgQ{T^9 zaGX4v@8v^MqY6I6fqI*GA}_}osDimTKz|f|!%ym~k@t`v)a&J(+j;MNQGL03A(uvF zc`-kP%qiv6XUi|)Yx#RDlQRc&;e69O;1n@){95@=`AVEC*XB2Hjy#kvM_UX-=9kmB2}+bo7=rBedB!r!W^H7q_(QJ}vi=@$|6jNh3)QW8FlS%A z6phra_*vxK$@5lA&e>F+^NennS8&eJJTrMVR_NvI%YK$;y9eLSd2aJ;ZNjm7d6o*w znWZim@H?X$eQ_5o&7j#?+RqU&wkbbS!;*ul|=T!zUYn2u6yz(%tJe5&A)m%8GV;a9y3C%QHT9CZt^BsU7rG;JeKq~Pg)MjynL%&jV=>9~e*6uO=ggnK$`w#cuB0~vcdLIv zd3kUC7|Z0m+vNPnewXv}X8phA#hmkT6z3ew-dmsdRcCL#1BW2*79Z<(!V&6~$l3iJ z4o1$J6ZH0wKSMX%i}lzCoiW+Nj1EV3uJ#I1N$y@-$Jclj_phMa*jc|Ni~AJ22~ zDr%yZ`%B>wc`QH4CEV8u_o<&!zrf@8QoaC(VVb%${zm4{6ZL+;_xM8Z9-NIckXa&U z=yHA(c}MDuCGLGsJ&sqQpPX6XN8ZX;;&pi{Uxrr5`%o`_33s3muETPCjicTF0iVPb z(H}Y2-ojSo?B3J$hVti}S>-g&8J#(3sD3?O#{+O39(Mf`evfm;?k9gFf6wpWVtEZR z+t$c6Jb(#$nQ@xQZ(x7*wdfzC!?I^8&j(^AH_!h70)x=ti zP|xR6cqJFWc)AEz;tI6UE6Zc?6EdfF;iCE-IP>|J^3U?Ed<{m*m*5HcO|+M*^CdVy zz6mGe2z6^d2ItAm{92hiGB2H@J`#&j$n_?;Tz(Mcu)q3Du7?-pANdSqCVi5t<=4=k zjT&+zwd>h}-%eaSYedPo3 zvifSAi9d0OYnjvjbgeDEz?=Gq<1{QqX4mHWd&r;i&G<{+P5&r92v;HRcN;kGDo44e zH>zNw-f(o3TVRRYlNaGs`9ZA0uCs~1M$VGwIWtouzSg~i`3dAqn5WM9{JxyMC*R8! zcv)_Q>zJNaVVkL+Db_43?)%LS2j zc0F>2meQ|=oI_bhd5$voWxu>d&R(DAsXw>X%bv4Fo&7p{`ZQ#(@2Vauujlrh8SHm? z0WyDO9@va(t_|c5Fb`YQkMnAN12Zul`PtlvkCA7g0pG-5^0}O!clP2<$k~&%ah+>F z@El~mTd1Bbk3ycitFZtpFb1Evo^|pK@~mdP55T>0Ph`F1c|Hqyc5czj`mBK;aHHNG zJeo7dKF*Kge7(%nIggsCYvK#NJF!j9`pf=yoVpW^Rp+dkg-Xa=wOqdqvUjc0`wsi5 zbGBZ}-PIj=AGE|z>dd>n<*M@OI7H4qlJn_PWKI9d*_WGg=B1pmd&*hs&oceD|39zn zJAZ5JcdEzXQe@9~S8o~$BKt|sgzUTd`kn8?k@5u2K2aA9H_aI|~^CgW9P4|@pHT|0?K z^H<1TKa+C~=6&O5*Vd|kL(aIIYdz%6T*vjbd>i(_8EEHPA0EdibI!!%~xLvQdYd6bR%9Hqb43dw;*?1m@=~coX$o`wNG;_=s za^|l3>UYo!c`tldzaO&yR@SS{`(d$sJsM-I`W`-!kKxP-_2rG}<2-eB&KdTjJXp^A zR6As*9Fgm}rvJTsAgZY=t9Qri>H~OdG76!dUS@%|Tn;(M+UV~gKZ&p9Wqd7OK^OHf zF6_QKe46?`Uc$w3hMbu>bL=njnW%{)koUd|_&N6-f_3ue$c*`!`a|rCvU;ugd^E$+ z>dZh}`CxR^tEv8y-&1F?+)hQ*d6&*P`ztQeyMQ0#^7tsI7vVfSjX`=3qd%@t=R94* zZ(=t*pr7}`4>|8n!}MC>EOiwW!U1?q?=Q}I`=b1`JcL(sMSg_a<7%u^&*9lvCXeAw z=q;B;Yx!Z`z=!g7z6$TiUtuI>rN_;2p*$RK;TUxne}<*W$?2To_yABAAF;>QgxL_x^HE9IL*F$Ky&l^TV4wSG^ve;BWPCu7PK;7xKRL z0B5$_&VS)ReCyuo*oM5P70~+wnSaXh;aVfOJ3oa-%V>gCD2jvdy=yCZ2{J3@z3Nf9h5qF{9+%-X^uvjG%e7mv0lT|DvtTj( zyi@ejn~0y(J9#M2;0HOg{$X-wK82st%M6&=s{>Z*J&U~SjCHM!ob{4tInPV>*8H5? zxRy0G0j1QB@wGgOyW$9ZgY3`QACBg0)Kic%Hv4U6lPA=ZUCY|ZzL>pqj@~wIfXp&? z@Rxd7msxY4p}X9Qa|YylRS<8;C6MQKvN}J{eB~V9TQAS=Wyt>l}l>P8t-lEPr+9~hL3vwOfa0%uh>-i7lTrKKzbvf($TAZdo z7zfBbk>~zh^)zHJoz8jwCLrhiMV#j@`(Qh4bkAVSMb5f!^?Gy8pPZ2m)px50acj;w zGD_YLJJ4TmBxerDy00c@rhJtfV>61o=WJx=sHAQx-;T_O-8ge>13t>NHF!mSp7%s8 zxfI@!XLHWMx4D2i`%W=r|K7k^k45G4kTsgUV?VB`e;;RmI3IgqhTe-z73KrZf0KyoPU(*?f_{xa-}dBF|Jz=Q~hCuH&Ab zJPSFyOX`o67vp*PQoJoM=T|WhrPW{YTz(YU4}ViXz)vIRW#-*`^adbj<9z)q^R z{0Xn({`g&9$>n)3-k%r67%MN~T7HeZS6-+788_%e2Ob-cg#%PofUKMhSG)|Bf?LW}a;7-umiO`8T|bq3UVKTz?~hFOy=K_C+=KJj6%gtXQ4%db~W$y_qeGaQ;o=CcW?EH#xKP z49?uzQm;B*QlE@Rf?Dg7w+Wp*d6ELBiAxl{lo+LY96m&mY>5@@dVkWeFl$075OyYjlboQcv7y(1%2i{ z`9V1|T6bQ8RX7|~QOWi1`7~b3Q}}LV-{{VH9*4+T`+5GdM?8+n>O4OSxF)jxx9Vqo z=lNcPJkN`f=ORD%{F!H<3UWr}S)403mUB+#`Iv;9KL;S|Kl56imuLttof{+e1BiSGC9x6Q=Dggi(a0)?3H(M zXFim(9`k+8xs$Ud&qAK9V>v&w37kD;fczx>K=$Dg+{(4gUjG{p_5r@-{QiIZON)vM z*|an?iHs=ilq6{=MWqr#Nk)`5*=43siAoxj(4e9rqf`jdQZll)@PE8Ij^E$mc|2d| zdEF(S`?}8ed9{u696f~XcvGG8ej@*hi}bROo+r#|=A zmfuD8+1q$Laz_nMb3ByaTzn3vlkW0i5$_ zsGR5K5EN18xmx3zfA`JGZbE(ZP=C$SIWwjk7w62T#`sFkGn8F9_jwaE)nBBS^Ct7U z3V*GaIg)$%0N$*ZbM$Dw5AVqNkC#I$aUw5A?(@u`oM{cz*)!*JcCW(vXJIokH!>JX zxi(VFex8}}B&y?7y_^SoIqww*ac0gr$lS=xI85%1cd=8Q=e{~#lBePs{HA^wwUGIn zxsx61K=pY1qCOe#$#d`!o>rGZ2~>;q`r>V5C;f`w)33l^@^}0`UxFi$^Xq)QlaW2* zW4)ZqIm>d+XWz>|S7No^Cho+e@T~kiUyqG)&V+`@%$mZ@^)h2mlb7>{yc>VZkK$is z?iJ!2P*Lv6`A^<1m&YygZcLXy=X3cZyegl>pCWs~7wUK9YtbK>fv0eGy=KX1k7xCM z=bRgNVw`#)*Fbij>^UR#DsX>w&b+C}+PomajrzEI|j?p23O8IaH1R z<7M2IbH2^uJ?gw;4B|cNk2t%>&nPe7h@0gzP*$G9^{`(4hlgRByplWO36%2tRnp73 zoxP=nUL*ByoHIMS)hzYd>UQXi*VW5-0UnU=;J->?a6Bqsz?Jwm9>MeZA7uB=j<8j} zQa`(I-jP00XU~|-<8Y8`tRvcbZh~*nPkmDQ@+3Z(|KQV*_o;S#sa`?(CwVTW;68kV8LnN#pP{Gx z1+pWb$cLbyehbW%m*PD9groG>h(Zk?%IMAkWn;a^~(D{tEdSmQa6( z+*8#!Gb=MZ&sU!9@jjDhw}^Tkm&d#EbUcbX)R|BD9&+F0EZfKzAote-*D}X5zux61 zQ4smgN9tv^S4K(oO`Ll-vop_m=2`>(7wcT#j6AR9)kEdnXRqO9e1!pe&ti~#0EWn! zGda^|s_V%eFcsgZ^ZYHx&FVbI*>P9P*C5YmcHx}4ljK@xFQ39W^J?RHWR~Zd%3L`b zm-WTt8Lb%LVyv&bhc7*=ve(3!LLx=G4m6a%O$@^7G}qLmbMVB4<|4(yP%| z{swQ#m+;r9D4)gIk51+Z>KXhR-a&TN-}UCo1(35jXL4q5=G`EE#Py=cKiMBA@iP2| z%)2{y3Vx6?V}9j!_*Tvy@w=S4TTh*HYaZXLu7tPb>@Ww)x67GRnPolXx|})rC6_`Q zyr5o+yf0**`d%KaKMdnAL%oGtqO9vjsy~)9R~z%KI80uP67p_77XP5Vx+}kq_vFmb z?bs=o!x6FHvl_40e}bDKXWm}*A#!J4fFI@0k$1AYc$D7pyayxX(cA?)aE!Vq_8{+9 zWBF74-dvWOV6=QAn&LhDqjxqs$vLNThEB#bs&Ym3`EpD71b!b& zs?S?T?gak0a%0VrcZG8j00TDzW$=RH6BJAWS_c>7b82sw?20<`lFoQiF(7i53WV_ zi;{T7_0{U{cqXzlcH`TyN`Dc*fI@gkeHHiRu3Qvn%9rtpTmfz6y8Jd5@_X%&zs4ze zO79kKz#s4f{4$#3K=on#1HZ*(`AQ7N!5D^0e!s%p$MwVce|$V$fz##M{4-zZbKB$! z@^2W4Gx4$B3gjK)3iVL=NPd!QVy)bW%b=Bf5I4giScr4}N&miAbzT+yKpZnnS zF%0?d=f2GMkZ1Q@Wg>*kLP?RPa{8v2a#trXHjOzKggVTPJgmI z0y!%)(~IdfRp)!le7zMpd*aQn8SaBMa(3-{@^bklT#cL6FY>d<-kbgRMZL_;r??d|A2R1Mf7amwb>?h2?!dy|@V#^tRylMwkDLXkyZ#PElj zD13>Za5-{LWM?k#GsCzmXMedt9woQr>(Lh1s)wSX>wl}4pg-#AmBsV&JIH&?|J08n zyGtLQz}c5y;2oHVOZDrZC>p3I@G|VgNyzSS3qS4JHg2iDk$=M(ScET7)b-1-RK5ol z@d#>Qr+#mAkbmLqu+up2_?L3_jY_T^BX>n(^`YvLD5x&Pk8^E4gWu)s%|+x>aUy<1 zcF;MlwUb-Pugb5;e{mC3l1rm7u2f%$yjvcit`+1VybhnawvuxWSC*@Ab*>xqSMts1 zg6zxJ=(R^jbU{`9jTk8J<&iv#3!|OSx8XnZD&rgZdwe3-!QXOr!My9NP=AE_xJ$nw znqi)LI6lSW>i4-MHpy#wDqqc~^Bg_`Tjfviwmb->ko__58wK@esc+}JS5A>f%I{&Q z+zZQbo_Y-4Ko3mSTg;{Nd*C?rTy)kOhwPEp@%1=ZUd5AKKa&S@_SHN1U-ka-2l8vk zJL7SB{cxMQCpXpmiSwRzFn^|>efDmy%=hu9`s?vN2I3Hua;-P#?=imST|RdOj+e8m zoh09nM{$APsn{S7=fyk|d+;+F-~;`qk(vFn`Vn+g=f3SJcSfG&%*fn>52K;pi=6+5 zXXRRQ&f3+S?U5__@#_nk~P})3H*%A34LO^W!)YIhQlnGXvY>9}LsWE;3m@ zgy-??d;oXH0KBCxiT-k){|(4KSW|BeN~y2Gck&EmFKNp;OR|3yk@vX%C$a;5%dIgX zHU8AQ37Oj^@UJ|Yf9EIo3S5Rc$T>Zf|KMR#yF zAlEVWF=bnPiVmT%$Gd;{9aJGmnKvVD8$!SX

|mC?y}pziw2{B&>>yn@d(Sg^hs)JbL(V=}P2P{Q zKd;bh!O!!*IN7!AD@V&`;XGWVHM zrSU5M!kxHFe-QWPPx)+q1lP#@xDxN>Z}5P8JzvP#b@OhYpJC=n?zM4pe&(4u-=er) zzK;TO&Zc~qzpMY@%)ZR2{OoU(^9CY`FuB3^z;9ido|zByU6#N|F`@<-;Ugm`95>c<}Aw3Q`~pH#%u1il!#kBaN(UeAnrO8$et<2;Lh${Ue+w;g#7cOi2od)^Yc zlYVxY%$W=1A)NbTDf0d9*1G|jrFr&p4|e9}7@_w)o|f}Gp3Xbb80CE?{|j+2HX}Pg&WD^mYq&2CmXGA|$Qd#f3sDT$ zqJdwN^Z#UI&Ygw<@*VsNaz-uSMrewoaTf~s%&}N1Kgox1p0_;H6VXx*J?J8ZxK>K;4^f5?ltJ>Eq%WM_F< z|1J4pG{7tP1=aL##0WWaD(BZmeo%cs%F8)ND$8H;Gx$sX2jAmwG}e0^Z_3^IaBjkW zt9@rmLm~V1CF-rn`EVQ;!N>9p&b-=zJt&N3I1tNSo6I+12EIi{Ox7>WV|f!s%f}(- zPxhZ%&<_=Y-uc|0b6#}e{_1}@^Zf;R7!O5VJd4VDwRkFu`!yxigRojX7tbN*)rWjP zf5RovTfTv_+ds;g?|Bb+ag$d^@j1Px)so!6Wz)*~1>- z)A5Q=c`b7G_Tc07&PDd4-_>u+ zuOshF*{_!AeWZSv58-WiRQ``2!Yq7_rYMD0xEXVN=1{DVPvJT^3XiD!;6ZsEe}~ro zJLU0?ygydCc095_l~vz`P3VWcdf89jmzU#ib$etdJ5#+=ev#k68Tdt=caz`byf@V5 zC-nv(d*cVZ1AR~nAEPFUx%Ltd^7(V++vL2LZB=iU^Ip`8SE-xHCvia>q<0mvPn3yq zAv^$kCD6JaWqyJ(yuF*MRv^3 z`9uuY?~R+}{QqVi^hECSe9w33|H>u!WX?aC$(@n;mpM}hzpHc4_LuV=za;19^9u4k z=jWR_|GfToJS8tjc{$JAlgNFLJ#2KY$(g4!VuGbzZU1L zxxuw-P2{09>JM|ndSA>7s+|{kLK)MmvNrAd-bMs zM{bW{as$^hM{<_soXFnNh8G~u%2=L?oOdtt{TM5+;fs+yvIXa?_)|a6=Y80WJm=SQ zp0)A(G;iT|Q2^WUz1|(jd&+m5U1|>J+0Bf)1;=2u`YrU3cXQ@ncK74eUm@=nAM?rj zSKv%}3+HSo#{a0l;ewcjOVoLmBTv-K!>l)Q&u zz_W4@{b`){sC9aI_p6}ZhVht=?Aqt^+bHQXIkRiY4`Y-%=W=$&PU@@Wo;-`Q^WDpH zahYDuoL%x&at-c>%=+s53NDo!Bkvs9L$W_jP-pilsQyfT5g*|+^?!Ui=A(>yHjm@? z`EkCLm+<~*Cs*RW$Qgez*U)>B=i>-miJW;u`ApZ2PsX5(AE*LfL6$}b>i(fR5V z{JMGE3oZ55@Xg!>WnJ6MXXsVNLitgCo}b|x`A%+xRq_dVL#~ID@RNEcUx=KwmGGIo z5%?0@vvM#Ci9B zOdc*jghS=*97l2XuuXh`{=t}p7Wf87>;KK!A3otR>M|IDBI?7CeYUfD2R^}-_!#H8 zRuLb_d$6C+y|4aMJ_aS^LCE`a18$}_nLp%d+yo`%vv@5U%B9g(?v7t%+V^hsfG`pR z)TeVFz6E(tY^A;eSF4NTD!DGt#oKakESCFlH@*|q{kpyUrrr;jBxjc|$0y-Ly}T1Y z#XlqO2fg&am78--^)Sx9f0tfEoP+Gj%k=BW2O;yZKTbu?-OTX(+;7De`9sdm@Swa) z&KdbG=UG^aYI0^m&Z(Sh9nI+j7vX3m*%Z$Gbx$oOxx?c96`uvJ|InU$b+!x2k z>YV%U@(y+Gx!>_Bo<_dE3CO*YpXpmT1i9BQ_G>QV*~mSUd+G-M9+}Cj)wzG)QRhs` zynRakQO+*%7>~erdYR=p<7%TlwxK*SzjH1<&1HS|d(4zyM@e}BU&t+ynUi}W&sgri z`lzX&`?so`vn_k(xpHRfUC7+XZnDj1_Mj7Tz6{jQS(bb5FFCXGdgT7D&P|Xr^mabk zXL7HMmvdg!m5;-nD4mYl-@)Cu8t>0VI5Tk#x95eNc~A@$E5rl3WgRF;(4^r*Y1r zdvUd#Sv6B$iW+j}=uz^qcp1;@HPCxO&Mq-deIzbXKf{ZV{d<5qv*B!Y5zccuLSDti z`5feY$oaiYZ%$C(g9Y+&+!ph(LY-Z$n4H}*Gk7QJsk`t^oSFBqob&$=&iu%ZkRADM zbx-7MtHo<@kvs`U%D?kgOhESC%*bQ$plegO2S0&j@?g$;(GboYYOS{urP*YGFK zz=3!K@1cR~*YXoMQ2qyn{QBqAP2}v+H?n{CokazNj<`g95l`pyk@My^w81_23qRv| zba(w=9>@L^Hc`qBy-|21UL7aW*RJlLiR_6@+T+TaLIX+Dg*)*dA7O`cf}EMZ7#|y zxiU^i&epTJmHu@&S8m8n_1E(o7_avw|G)?F-`It2XpEePpCJ3{BA=-vzroY+A7-e( z!f+ghoOKiR+o6-X2LH&fb4%B<3%(*>jYsuIVI?-JbGEJHn^9LUyU#UpYg9m4z3c!D z@tb@m8tJd%hqyOJW4ro13`2GG!#tV4=9Bnwu7xskWxrQrl$T%B`!0yjeh2IEI6lD`oZ$0^@&))9k768t@M|{l2^fxs z*r0a=Ux5K~Puz^`Mq~M8%)<4qU&vqZDd-@tL-v(d`Eb2bs3hk-F#E?1>ibYo?{col zmH0{inh)W@I3CT^&HS3|xBo*AdSCBKo;sa}9g&0`lJ1n@eFXnqsQY=6!6ae1^Qge3@KOE+wDCh1I|Cr^w!Qh5AH{RQE$2 z{GvV|xo3}6XO?AFbl`on!nHgznQ!@d<(~LPy^!>Rd=DSR`Tt#w z{4A<*eimy|b1(dd+ygc6BQjTVFQ1NsaF#mf>MG=W9E9u}MU`*id3AQ5Q|0`O>#OG= zdqnR2%*M&c%ou}>$X<9c=ibVFQ;K>R&{95EFZb*cWDfj*1;~8L zT+JDi=j#}s?TtNnM4kKDrG2~a8oiuRZ=jL<7kB5J5#Pzba^~|xa?aS@a(2qxFM}}} zAG?-)J9B3%=Id=iF*!5kAvw>~SL&m23+4dk8PDU4GUXJzI z3y@jS7kR!X;v#fbXUEGN_yE}f%Jb`7L_c$I8|vwm#=iFoo|&w_h)W`8b9SVoR89zc7&B3#k$mEG(& z^)mI7*d|Xy=IOoaI(S0f59Q_B{2@lmZ}JHK6a(>@`UK?6J6ydP4Y2_e(8IO5_zRb! zF|zA#a;+~iZ#(ik`kAA{aUrfp=Ji|pna_vg9lgxWVsg&xyX2u5h41y=!y@@kK8Y9b za&G0bIXmydf%z<^%M+3FsUudyzx&=He&gMkhU@SjGV?RbEBH{(d*O3jB_yXyw|ss4QQNm2%!|{^mo}Bl%v=KR5EL*nlQD+~;oLHuxEPd}cAvHmiktC@dLd`G&ApbnpZSpg*UW*Ov-xl4zRR4*&oAdgH_n-n zKWqN%9XV$}{=9j%x+Bl`a^x8qz}vVe=Q+szUmqjn{24Oqa<=Z^%)6XZr?^%gAFA{7 z&NH5w(pjDR>^=1~l$77(JOkN-^3QvE`T6Ev&b^iUqk-IqyP$^N54;#(VUxNna<6Sh z=4)pB2XdaP-Fyu)KXdLKD9=Woi_EW`d^Z{*&t_)GMVOD5kmsp_e&%WJ&n0@9o6jK6 zW&k>iWq2n)@ORrgJ>4l1NJvcw+$DHTc(`w2`;!re3_QYbm7nzkg12SuC;T^qdxLD4cjXcpE;XPMqRm*{so+QcoGlMU(cuV(fl@NUQR;h<&o;llFYt5({HK| z;)gkNzN9>pGXt_?J*dtzUz`i;XTB_x_uw;i&g&cbeD!yD760ONWEaV7>4Q6crW7*M zALM#C1{2g@pru>^*>y&%&y&aUE68~=nP1mC7~|zzu|a;Gzvm}-DY~M#dIFA+GoM!T z7MvyLyt#!lb5E7K<43%!&fJ`WbJQD=J*cX>4D!Bnxn5@BQ|bY7=3ZSn=YDCvQm-k> z$(g73@z=Oc&e{H_Tm_luU*dE9<=7^d;1f9KbbNEH| z=g8ic{qIxx6AW?fEcC`g>_S=BYrD1=C!(3&XnuiP;3E|Dnf^R4ecX?~aj{KV9AeiS*Ya^{TH%e!)8?x}YvABx5DD2$VD;W>N&E|S0FGr1jSKb)A)AvT`l8*5NdZzP7xb1)0p_sj4<`mgaMz7mz?k=QA>=bYKk%HN=e`VhP!SLcWLq+FAa zL1D~Qzk^?KJxb+sDChUNB7J!)8X@mYm+=xbaBUja$$1z0kY}lzqdcC#GWd7je$v8c zCM%9b-n$CvXHWcv`|IU>c(YstU*T@ORlL9cU2+@Tr0&fpawDF}*^#T`4!J7wcPW)| zkL$znoZjI$0Bw=|{3w0^gIwSDF2#e@`NLMmPvSrLSiTL{$The&XJ@`i9wv9-tGJL} z0bH*B84uuGT!8E&m+&zB;Iqv!125uXJflCK&*6%E2G`+oXd?H*CfuX`k3Zl)_-C%+ z_r5`X6xsb==11_UYi)TU4wJWX-d{Gzhsfun4hE@@L;n8geRXNMAFsktxr+XU+!8BL z5Nln_pE399k9Yz3@8{Xh&wi{r&)q0=#(3n}nTq^;vUBA5$jr;1{R$LBQ@?H_=l&Xn z+<*CV)z{0>g5@^ zfs662{1NKP!#Q){INUCm!+bo9hxFdX6!{V4nHsFlbJ+piP~n94b*ubSRl8Q z*K*FrmvM}`EGEgBd%NW$u}prPGr#g~kl9&T{S()~9y!nSL-Hz|ib3kl$XPOp55)b* zu5mgy@R`i4=g>qx9xuxE_&KaY_URIO*->(4W>>gaT@UZ@QLqZx{;T7j@p#u9QpQ1obnV-63-^=f-+< zNo00zaXm9S?|k2@Z^1mAg>|ki<)-{C`e7Un*ZY-cpf)mp^UrnYi|nL%Cn>GpNPP=` zhMjUlo{5%n&ZwLCcl8in%F{3=*Z5NY91qCd*vI$n=cn4EUt#i^4WYL zCdxM=zqmSGgaa`WOI-gR*>5l7yi1+WZQiM8?& z6p?o!yZ7mw_nf`@O}U?XtNaiD#dcWc^{HWf;*^ja-yn+6B9iQPnyx}w3Fk5bp8OUB=9zV!= z4_o2-dH5Ok;7t8Xxh79WBY6Pdk3-~@Xy(_oS64-Ayo*=#AH-eQA7y>!IzAbnBYzk2 zzR&f+CiQ-JRUV8t(GJb^79;OuE7aM~viIL6->m-@9+!u6c8hDdjk*i+cNaJFF?d4% zZoUKE*_j&ZeTs|mvR-ZehOfuL^1-|ctL2YT9v9+hA-+dIxd#`|O1k)y>r9_z@hBRyY;!=%2`6@=G`cbubGLAn&XF zUB5~$gFEHMdjE1w9EOjOzY{3LdEe^FXJVnxUyll?i@)_|;znd9W`3NcUtF$-oPA$% zOH@Y(oQ6dx?%J{ZEqCT-$Qh6wB)eD6r*i6?eOI`4zMQivGjX)~Gfb6#L@7Bt-9LP` z`Wl|ZH{fG=1drlc7%d;g^Z9aql!O3A<23uPsK(Yp@KHQB*HyclOS<>YeI$I5RrWan8g4nUc>9SRX=C%b@K6?8F2tEQ0F;r&-1uE z^1Nn_o{sD(zo@^#GwMT87at%yT=vF~TzgnO0=wnR)DLkVYN4lo&doMl0e9++#ldnb z9>fdKO>WM4&aaT)#9!+5oO3F(YdadtcVjE|M?Jm1Xef8XYjS3CHF-Ja<4637@hITh zvpg8*U;r|c-p52#RX>i!xCq$`?&Iv*!M}MFvhja&FAGmc?dFVA60)L z-;P`5oLz^@=VH2g81~=;bvxI-l_y{?Uevpd59IH7GOy&D@eAgv58#($zxGDni@g8k zog;hsMe3iBbMjj5jJ25O*IkX$$XS>(YP|j$^wsNvxu~x0hwsn}*#VB?)yVm=7`Gtj zS_eK3#ZW-q3JuU$y#uS|fBARh-Qhg`P_G5v#>48zkiEQ|`d*Y!XLnvDPr*3crB|LS z^B7*n2lAVkEl=W{yWj9S_1~C-D{(od={MxxxF%leW@-3YA zzDDw4^11TMoPD;xULiD)@8yvwf|FgZ!0%$6d_RBA+jtN@MN9mF|MaszO^_R4KlOW9 zCC|dE^0{d0Gk>Wo;Q?Ha>=>Q6AhKHz@wu<%YM6m$I7KhJ^j(~Hi*|Y|@QC_YJ``V} zHRkB$9i$L1SJ%N2*o-~+LjNSxmapXp@SfZR&&j=!{VqG`^Lh)_i})Vw!p}HS??Lp( zR&`YzFAqibrlwpLUGXGd!7)Bl7K4#}>j}O4@um6`Y{IE(m-g+kpYu!%Kt;5{O?U}w zeeOZNkY^#g`EBab$a_Rx{upI&De`XF2*Z8$DJ<0M#~1QPToZ4}xwmp|WZvYt*{;qr zHx-YgsA~<7pWpZD{OqsAMC94a{hj;zYg9qLr%yRQ$IQCDC@&XB?xlR!*`?O-b__ty zrUrVse{xK%y7klDXnuOaTnU+RsVS@H_j z$z!=Ds^BB!9?bKfXY@+-yL=Qj%Y!)kY3{u#@{vsYc8&J@H;zN*ZstxO9E9u;pXg;i zP=ec@EegWOozwtHxA( zcGuPFtC4f?0R21zIU`%>oue)*7nd{h7O2bdN}j;`W`cZ?JR6&Fi26$&kFxR~XfI#I z%Q&;O17E>K@iI!P^PFd1{>6E&ipiO8PxEqQ&g9t}f?x5S&)kJ&@}u|$oz;2X=ivxk zikv0c53h6W19j&6P`Q+RHFBQRRS!c4^`D$~gSUB=x+=fQ8+a@-XB((<{^aa@28-pc zd_EeZ?m&y4;EnLOtXxIvXG8|B8)r&cQA6n{syh{pEFXH4MXr zdKc@RgO`xK;~M_cwFO)QmtzcW(yNZ4I9t6GtC07sr}PHPrI7RXV0Csu|L)rpui>Zl zOJf$gsXs;jxk|m1J8{maJLFyX4juJ2@eMc)IWNw|h5FeY^KM!amGnNw@%lI5Q2Alp zjGS#Na0hNv=ZrW7m!LPM>gQ~mp1wSRbFR*nZ$v?Lc7j9X>>tPR(R$^18a|YZ=vU>^ zI1~N#zrm5Hgqb)Ywa<0o!l;Ps#yx$$21co~PhBXl#|U*Lbs@e^o%ix1Gd?^2j=5iZUmoG!!1D@g5dgZW0 zK92{WplipfFOvK6xA;dMhp*%p`EdS`f8gxg&&rkYulhWckxxQd`C;Tew6MB~JfE|V zJ|VZlsW=Raa3yLYe`nCb=ZeU^kv+H$XRm68N9E40m*L!yGij5YciUooz4{owfFI&F zIPYYC%Ri$HvM2th|Gm5(*&Y7iK6(f8aK4Ev@OTWD8}fZT5{=}%kKf5BqM+Vuyqg!| zQaq0baFqT-cno=Gc#coUbFTfwE3gQ)&{{8hbT9c?`F!k$N$Or)-nHB1f8@5@mizM2 z+!K%EMZAGR`Yo|juHe_5$a!~vK<_i;?-K6R+aTYHwQ@hs`_WpijqGqw>93Zvvp2+< z>g@45^y*-Sx-#m@d2Xu9v(X1FaTE@B?PB)tzWpjQx0{^%GjsABxj4Q>0p!^pjm(09 zyjw5#dY-Roay|Kb&Q9}!{0N>`=jZSx?!^Li=GxbIOV0Oy0!GVwcrfy8{GL8%cIMgX zitHnw@T>ZH-ipgFVW*s**8q z&eoiP+c90;9+Q#hB6BBa#@DXpOnw0+^=9+Qm?>Y)zjDrl%-MU@nJ3rs>71Rq2sg*C zdYw7*CFk5l^1;YXHBp^szb-l;vobRw^W-zG$wP3OoE@ktH^3FBqJKG0;2Syj^Mm{* za@OWNe@gFlJdJPkuSHQg&(}}#L2~x0TR7)Y&Vb3t9{H8)Zz1RS<9hSuzBpgbY&!!@ zQ68`9zrmU3dG7zhD0L6Mid&(vJQf=;0(apWG;(bVzsc`%&ijgfp98tRdJtDf-Vw%g z_NVXgzH4(hXVorwf8;$Ob0#yXF|t49x&KdIi#+GI`P^d8zSc#~{`5ST)B6}J+h<$dsq+?p$4 zI&waJ!AmjTwZVKWZ^B>lSd{gd&FU-UoEH`46}VWv1ry}oIQ!X0e4P3b&YXK$Ue9?i zI)qE2lwQu0>^|Mq@8Ns(iCB%1$iBCR7rM3#JLG|U4(GheS(tg*Nj*hffnVVt_(slL z&d&aeoIPNwx*8hD^Y!kL|K;rWFY*3d0Ke$pjBk;>@fp_&%O9ZtI;uzU5R8+Dq84sd zui^H{8GN!j=jTqIrroIJ_0Hm6+=$=hzW5!z@iXe^ zx5FRuk9YznsXxb`=&WwRjr_OIlye>~QfKe~RQ({bGZy11`ls?Ad?-J}ucMH>2W{oc zxGc}XJ@WB<32Mlv;}TTE?YJ8UV-b$@nYR21UPKf1lNckPhL-*emHB0?!aJ_*;=Ayk z{4CzUP-GAPM*koA0G`L2_-}kD59ixCJLnhur8>LAUaqOWi2tX~PSQne!?pFxaVhoF z@}U@n|Diu+=i9;C)FpT+&*l60N~}W8_fL60Jb~4&pUT-G-sS^wieAq0N%ArBmG}qc z)K7ADfsf@C@(<`I58#Kn7-q=X3$oK)rOsZxj?cugt~EptT!UluPQh{VVerxa|93#G z_0PvJoPiTv8-@$fU!C{%tGS-K2xiD-_)*S#!)@|3xd~?{-^wHLpI&cNlt18pTpBka zyHI7l*W~-Tpn9gf85gRX;#Kri_vD}W2s|a9i#l?7UdOeO{dBInfqXq4MpZn98}!@o zQ5Yvbjy z@2$9e2cUR82JQj_Sy)g4@GBSsAmeiG-Aw&S9*?)=-#N3rshoYWgL=N4XY6G; zXG+e+TX-~bMzll$*Rz++mosOxGv>RfqHe`Axv*Yw`9-+`GV3ow&aT|2pK)e=&f+{@ zxi6b@H++xR)Ps@zp%R~moacE~ce?&My2*$0RNjJGSf$R6H;r@e<+(YXZ&hzZp0Vs{ zck69Jcl9MWO|FN5Sg(GNGhZ{;UXq7$X?}{&!w@9-pi%*nj`z(Wt`cwTmNf} zK_|R`FEGgU)|?rgb7D63=zYkaqCPTTGAmZfnO}K6HgINkf6nvxHL|xI!C#`CYcuet z{0ra2H**8tg)8O4ycM4yd&N8yL|3%KSf6deNV#o3s>Y_Y~v&Zh0KajU0=Ti3X8ggc9c8Pmj z&kSv$cN|K|^?4zm%lBXuwMYG2pCjYEg zNWMy5EB8m$1-t`XVBI?IbL0yr{ z@&wNLc>|Z>=eaFb$p6FDRE~4bOvK3;8S5`ac9D~Kuig{58rd)E@DxnPzqkSK=>LnF z@{>42?uG0hNAo;8(|Kf#*cc-_(E=o-!NRg z4Xc8dvcz+y+_8e9R4St?SE?ed{Rob8_CpZssDmUT*|Hg-kOwuPon(9rB0xNX}kZ zi%Y62Vy2w;qs9Cv>R}gJAbanHoPqNa<-H^O%ro+#7p{@>zV|tws?Ph}8@x_^BKlwte#hN-&$UlD`&Sq4tC2`EGLmZ{|ER z#pTSt>@&HKvv=g4dmZ_6_CTJM64-+x`jhzs&NEhC&a;{OCHHqDoQ7MG=Q;CfKJqik zJ)c>X`#v+ZV`|Q<&i9>rb1R?2XCe1`XTAj&>mSZJ%f@qlj`^OyQ)dVKLB3SJAjm5@ z|FlO#{DcO2vvGu+`!3&aU1ZmrtoI>uz8r}1Xn})V?~2?znGMVJTB_g2uV|zG4w)6h zxESY*$&Ad7mYwPlIrE|fa9EAq@fsm^nGvATgeXG~^Z?$-_YL(cuU z6b~Zzb635l@@Vvz|HlK7=d~keCY>f{rdPodbwk{Zb~syaIZxwu$b5biYmhUrCx7L7 z_WYa=bCB7PnK)XWht=wwSx3s7FcP2XWzXE5o}At25iWohXpf>;=K59K7Uv?nenl>g zZ(Pf6Qk}=E$8k44jQewTj?ClCq;K_($IBR|*8$t)cRBATFLO=gtlF-hGxB5g6r7Hn zBNKR`Yfp1#Rd$e0>e9!K`0seB(+`OMc`ga5!*`8NKTbG|&wFQ}X0Qu$dv zl@Gvw*rq;(d*C(sZG0tXf1bjF)lKwVR$@HTSxexP4m-ie$?FLGw-4*lP_7hjIv_yvFJt>ByZ65hboP)g3bUlDYc z&qp8q0_ra~JIMk(C09ii`A0ODPvH}gv+G74qc;}c%C$K&{UNy)+WO4(+#QwV!koF@ zhxbQ4IeY3S@=M71{57A;Gx$l@8{iWhhl}*OU=gy{WyiZ;UW>B06H9&W5loi{@oKz^ zCAeB|3--$2^GrM{FXo>y0+rQO{l1I%8Flv4QMs=Eh%2a%;=1bnfgk9@2Du>F5l1DfuG|a z)Ti?}egKt`_pjUacFI?BS@r+zbFJ0&^h;u*T!GKyd-0fj5pU+97%0DnR`Mji4lVp% zH>mS>4`=WSy<;$3zKw52L)S)dW4)8OnfhwZZh4+wKfHnS^_p-Y{)w}%S zetrCi`_!l6cllF30GG(6F#r!^G0OY3b+`h0W17#M$!BqOE{>NU;uDc~@n3kB-g9^bZ>Y0# zwUP7XccYdVf?IKoejOZ*-;np6v$-`6$4Z|m!iVBGRKf(kb9f_<&~AcXD>#&-gq2%z~U3pUac*x_S=Up{{xhH$dk69Q9RlW=MV>?d9x7H95~` z6AZ+=$j`MJuSTBLL-=#fGx;E&iW2f3)IwQxd(L_PKKDSLyJ09G=ibb$?92ICejsn< zd{0lvnJ3@z1a-dmd~fgYa^$e@p2M@&WLQqIT&(HW1Z7jd5F z{&MDN&cDo&OZC3OVB~o`9=mY_j>W4!Q<|sY72Jal^fE(NaOQF5Yo5XHT|1Y%a^`dP zn3d}KxCFVEpWvLGxhFFxo|ZGW265(8&X?|T175<oa%HnXn4P1;jah~I^ zQB1Cl2XKq}ByPqt&=wb~mmts39XuYN$fY@FQ||4_@(nzeJ8@=Go|DJrlF0d#8Cp^A z8Jvuqsh1ORiN9;zm4%-^2p>QLf94 z@vFRt3;T1NDc3_G^+U+>dOuFl%QKpN`Y|q}&Od*mxSSc3`C6Xu()*B~#C6CS^n>1Q zn6CaE1Lf0sIC9p_;LgYlTZ6gijrTE5KYP^6@)+bDB{MsF&qf}FAAF_*mdV=t&b)KD zi6ZlUm}`sGS8;aJqH;sjRL{m&I1EemF5shZy*!UItA}vr{_}d-MSqqHp}D#%e}hK) z1=Le;pn8AxOfIipiM!=4$h*%d9Z3V3Mb7B#D!;0)(D(1Yy|0sSqFm6mkGLG}#Yl|B$FAqh zYk~S`g}hg`(0@VB`LIU56h(2U-gUSHw<2dxGYm%dgUWof{sDXke~OZF_Wd*E-E#K) z$#Na}CN9WrIJ?_ixi@Ob=lNV=9-*GdmGBPU#(Bv5NH5oV%A5Jbd=5vcmvMHvyLdfv z{$?M`zHyf8^VK=q{~t~E0siIu_J90GJCf0`wWkV|hG-9?(h#y*lD)E{p(T>Y9xZ86 z8b&Hj*)x=kCfOsQCHLdiaompI^Z$IE=k=+(zMtzl-{;l;1+J{AK*3o zi^pR9oB1Q&%eP_=rl_m%KYT7m%DeF_cA}Qv-Kc~sk=^q^J{9@*gS@le?{imUrQV@< zLf*v<`FL*6c~3e)K1F^Wb>;u~UVfWT#!zfh7xmdD@_Kxro{xjD9qrIVe<3Q%H)E?j z4kyax{Mr?A861iJ_)q^t{qb`4wCmJ0afA92?#-u>IJCpT3L05;wN-C z{>B5yS@S9vK%U9mTlp{ZJb(8E{5dj1M{}Nwob@>`mdlwZ&&YWuav#*h^?EroGZQm6 zvVY}1ejfSGpGM|?1>_vv#hHgUah`!u_!YVDaz8bdGYdJ<^MF<{ZlB;XT(T z@fDnV<2BCDHTT>S&i&tta~`zT%l$VSZ{s24S?q+#a_;91D21Zx%+Sw}@9uiNGJFwM zBhOwXzh|C_Vd}SdJ67Qp^&H;8Q<1ZA8$Ya$`Q)d=r_bP&%8}I68rk3T$kaKRD&*dC@osUA!r;m{t zlv&-(XENt=58s9Je0DMC+1`yjqi16i^89CRzQO~L8ThCEIk*bn<30U6AD`j~bk=*0 zGou^uN7#Y;_4B-C7C*{4M|1WZAkW6*n5WKjm}j<&x&r^gwedc(`(#e_L=)VJq4>?U zZdfm0$PKv%f5q8PkK;bbY#oH0@0sK8$eGpca25VjUyIH1I?Rw?!d%q#`RvDI^rs+m z@HYJA+812UwKusc&cXf2xs>^}hBMRZ-~?n|tmM^r8Do4V^F8x2=Vi`^<@&w#n?bqn ztRKja=wE|+@&vw#3v+hH=9nnor*|?wL09z)yo4*b-bP-Fjd&W_9Ww7`x?Y@HV!AvQ zU*RBB)ccGx4`0P_JgavBAHc2AME;w9L@9irK9r9|-Ysw74%mQduC3u?kh$FgcjIMb zmpIw=r*M|~5o8zY%5Nci#l5b5$G0PA!G~OqTj+m+j#!J9$WB|+_1mzY`VG#x(VWlZ zycgwtv!r~m-XQ)OWpOv^Vj+&lT;x3|JIO8bqg;>6bI#b>ybw>KmbzGww_q$TKn=hC zbp0RXygywl|A>$T?zyb6EH+5fVez9IjicMKQe-;o{s zKXqSu9FD|N90Bcp@3yVLp@;Cp!^u93I!d~`+~^*^`= zXQ{77-Xq@TR`^A~3J#XDBNgIp=%H5_hhd!h${>G@%5qU0jmGN5*ed6}{U`Zb^up_S zN527QpYJI@A)kn4n1p}z`k@NWP_M(`az)gY55(W{i?~o8&!_Sed>HhoUKVp@rT}{0qL5b2jGO%+K~A^&dD?9+J#CKXaeXQ!mGz z^5c9JzrdL%Iomp;yI%II+?ylhoCA~PA}D}7!zbylmb>u|Px&di3}4G9==tBi zGiCzkZ0@apA^wp^pdJR|3gq0(E>{7U_{>(!)z6Gvfjdx7??=qXMe4q2A!qKFwjW%ENGhJcEDY&(RNikzMyOz6k4GKO0ZV*_ZQPHCVk~ z-Hg9NS>&vIns3KFuKkMr@rAlJD$AMkYcLc4;%@y;sW~(CKzS9i&pyHT>KDVU^4*vy zzlLj32V?YdUcAZ|s6R*c?HXKIFT2L+a!GkE3dx<2Go+OIH+dzm;_Slb%Go`ttMeX` zUF>@KO#Sy!6*uETz1Q(G`l`?6+c@XtvHUQK;AB)scG*u|dj-?gcXDgJ8S(%ep?(n6 z^x7il)GqYI^VsY&*WoXD2sgk)&d6jovHBj^9@Onp_73<@2yn9?74fsQe-( z;4JkV?t#m2H8$&&M}5CnL-mbvKh%=5&pyOc)F0qrxij|DKTw@_qPpr+<-_?StdpPR zulY?}Dd(ML17CwS$oom&2XhwR>N7vQ+ zaq8!AAFfAjT&~{(%a9%PPrWZuFxFejt8tF}9cK@nCBGvtK_$62+RM-I2V4eU;1pb@ zw;vzHy>OKL6|J1=PU6su29!Ub9pI#ms|S1E6V+oQ64v_ zAH$_`AHJO%aXFve$`#ew8DHQ%>fiCMJQa=Qy2!5fpt^$GgEM3DjO3Zm_57LhUF7%A z_nE)XDCD~-fvIwS26@i%J>{Nk%pa)pGtYmycMsq@)H%cQ=gyzC0A~i|&vl?WbK!I3 z=azdhJ4C+g+%MnjWd^+^=X-k)`58}E-;B(f{48>RKB;al=XuS{xrnE6=5)TZ^W>U1 z5f{7GopWC-!avB*r-y#-sd9J>`FZDi$upMcC^NDH@-xnNI*T*M9^#WZJ4>FcTh&La z58~VxH*(JZHr!r)H!>S?4`f#6jLfsP1V7_#oR8)DS7VmE6S*fcFU~~1zuX(m{Jxvz ze5e2aEdHhcxL)qJjmY=kiu2t6g40s_e4eYE!7m5-Ue0~LK+Zfn1`G8v$KS_I@5i75$o=XFM~h8S>0M$0bl&?u5*Z>{!+GzQx7rmyk1cJ|EAye{YgoVzm4a@~q^3 z{{Q*)uDTsEXD4x-~kL@&LY;dm}sj+v@DZ+37N) zpU}?^a3x=lN%9exgUpx)_*Gtp!MGbm{MzUB>&ZEr50%GYv3g*x;{s&1^yV(eJgKM7 z?s<>;1&l(@%IvtOyO#Hjrrbe4XWVXN@7$=)ykEs*&_wRVc}F@;-hivr$MSn9gI}>z zFFR{VzF)lyuc1HYVWR#r&a8X}Psv{(?++E!FJc{r>%EGTklBBdo{#LC>-))c)L*4~{;7?rsnpdJ9X5$vUo}9gHy1Ygn$5VJTYRJ#xH#z6~be^qV zhspRHKcE`|{KgX+h0=CN!aCX+b=ba~CrdJ(j%RljL+!KA|_4rEO#kFuJ zE=55G?DIpBeeMp^FYl*o^!`>~ z$a7FpZxMg1w}R_)U#ynzLf+{+ae0iyVXpnjP5BlSmLK6WxC*M{5`3)pA%@=7&9$!QJI~K8^Wqoe&zyTX-&>x^sp^CIM)a5S z^UZUYd!rok9lWfrCAY`7_+6cuTavFr?#s-a(|8s#t8*Xb49Y#zQLhc>%*#ER8B>IF zujQF9h&^&Y&VHDmQJ&2&^u{1FA#-FoXU^r`$bELX-bCd2euVSv=4{LTQ5)CjFXueJ znWN*8`{NoO;o4|qE@rN*(p!srklpGwo`8Jkd4_t*nUT$rGco7J4RW5vWAQdt`ds#v zJgXJZO1%+}BhO8~%LX`Fo&Bc^=lRS0xE}`~b1=`}PP`)L-pFp5dn@zwCGLew^>ZH$ zMO``1!5zGn|9{8GGu2n`X}!~V38u=sxHzxp+=KqNZ@0)CJy32zO#Fp< z$eEfM{iW+0QCoeVdK&J*67<$@$rG>``{5J4%$Ch^6Xf~qt~VRq|!;s3_1U1^PW{wZ^uTI$ zd2Eo&;Tb%Jy?Rx7B5sy<<93{lxAcxe-h1Zq8iIe43=#O{RWB3w2mrp_&zpgiD zf1R(tKz=yC4qwS_^{dNUu|<71x6(Tk56TPh1J1$*6u~e|NA{C7T!p{lu6}(degKck zFQAotL6Gl2BNX%bfAz*9`}uV|UcVwr%HQ(g{1Gq37wCY!xE_bP_95qeqmO)rTvjg1 z+tCbtP}#Lxa1u^M-ih1k*N|s$YrKX^`jzzl#`PGWSDVk`JR5&=c96`NJR7;+a^J7x zoIU-JXJ8s<&adHF_y^gC@*HhJIV?f;k?a%M*T{WA-yq+?^Ln}0iXh+fuiOQ1 z%egm~%ej{_YjPiF$LhxU`L0HGv3!5|E{m)4Jy*j;*oMrK%!AEx?&Z;33i)~FyI&+9 zfgkZI@=P6!+{a^(`S&||%b9_h&AFeZA1ZXD*?a8!{u@{O2^gK-TC;2Y$5UBS2E zTV%i5U#}!8sxz-D@P&9%Zp}v`JMx+8Hge{2Y56kFyv{S7{V~s6p6NW#lac2)&(AeH z6Q|4T@d&b~+@tq3GM~TDd!I9RZ;O$8GIKBUDfjfxa%TP$Xr`ALll%2u*Pi9u9hg^;;>m7&tkX`XE-sIX{$UJ!(nfsri33jSGa?XyNE$7Me zk$K%-y)v1*W4Am2zsh@XmE4efVh(Om_vZqB?Lf}lxRh_u&um?fJJe6{ARHoJ%~x_q z|3mibkHqXZ8`Ou( zd2jfGf5K&YyZC2b$aU~ArmM^I0ek_Uht6_Myn;pQPRKcVC(7xagz<9DuD|5sa%+Bv z3+bIGUx`BMxwu$vga>dmu1DTm55g4JFHq0mgp9Lh}089j*$=l9{vippi|MNXhK<@J+(Hd7G-@&_n zQ72r3e4qJEGQTt1Z$vAtJhKIG9da(cg51k3xjXLixw>2onQe>J>v6Q4=dmpEJU7KP za_+T!zu67*Y~>!>ssAbG9LQOjd7AtFTNF^|-py<|k290{W2$^6%FEeNtI1uEGpD`Y zGw6(*r+EgmJLmrT3A2!0Av0~EyaA`mS6~p1K8jX6?qRn!{x{hw?+RqY{cz)nSFDRJ^Dw! zU;kcy8<|~@*KZDrcGcVvIsgv`ou`jv4U2I3<9%+xi! z7ainVIrI7-`6JvbKg4D5lDvvra1nlxGh>(VN>q?D&sO0A`B^T{*;n4;g~)mIv^x7( zFO)!bkY{)}E=C{ME9#BsN$SO%b9JO#j&Hz0@>!@N--p%mrTh{oy`6m=bGn|2^ksUSrXhWY_ zp?;Nn<6#ugZ>d*WZiDIalUR;R&=K3w*0n{LgICoDB75O;+#Ln=-{9L&AJf&3;4oCi zpZEjWfhzC^ocE>Pe5g9_Lx*tom~-)@+!)1?-7ot<6UmF*ai^ z`lAv)bNzk31>Yij!2rF(SK8fU(a=Lp?oLKK=#j*xEmT_2!6mnxE-y1{{PJczHg@_tYiqvfhNTmO2lq?f(>GI@aB z2CSB|%XZ?DyjL&puhrH0x10L>0P=2>`#1M(p1C{pa+WUSoV(9)?xT)8!u9;z@_#pf z|KE^5<5ifdpPygmS?*&m-qgo`av|!aNx>Kg*qw`*R-GMoXN9oZp$xInQ$sWIp7+KR_?D z^)}9Z|2-Z-?%$DmnG1PNaxeakJfpX|)&nKwQ+PP%nW}@#1XMO~J!;i>W*v$3n+!J|^*ufv-XjH&tpSu>> zdox$M;AQ{v7AagZO0bi_EXQYpm9L3a_g3u2%)wZQfA##+mX7e232r-v@`GfO;H`mG4Gn`Az;4ucNnmDIdo7@G=~MGI&MrH2#b~!Zm22E{6&@ z4=eRDpN~Zc6xJJ!8!!qV>&@ZpR^{Yd@vgcJU%`2=x=tP`?4pxaz5rfd`m2X`M1TK>nrddCg}~~-}xZk%bR%?e}iXn8%{(G{hZe`_#JgQ`2*gF7J4V(6S=G{e?U9AJPwr`q74?RXYze0D^JB` z=%hXy2Vsx8D+8& z{*mY530#d`7>d_idxo#%yZ9w+!ryoRMg2Y-UHbqZso&!H#dJtctrO#cWmFFXW zulyXcJLK;$9qlm#nHTx~mLO+Z?%ir~W^B%w+@Ir+`#9fQ?#DYY6aVUGKINW%gmX{i zS=%2u7sl!zf&s-_sIx?!BIJ?#T%#hULgi{uS@w4%ah-|B<_J?$24A=jmNg zSw0t!sWT5=#&I|r59%Gk^EvZ2dsFV!wR&gpTRf0=@D%(gKh8Oua_`rabMKc#59GP> zzkPez1^fqc?w-tuivyY6Cdn0E=c81*FcjF}2@=Ryex8@S+Tk)Sfh%ewv@C-i0IeOiY-79lF=ho}U z96o^aOyr!Nt8S>yTwKgecs;gYt2*cFb~&^53VD>=5t+?e`}W+N4|)Cyx;~iC;LL?) z$SyTQy;pt%rz5+^e7)=C>yYy&&*}kk_SHPI%`jS>cax{(%!%JIO)qn{Ic`%wz&S5+ z=H}huR=vi0JCOaW9Ny3?j&gdx@CtsDf5Q+tyJ}`_=FatccOrXbcDT&{?5nHf68f1v zvpBnb1!P~Xh?n%TD`n5_CjX2IScE6>zF&752FlN2GycL@y)|4G_aW~Ehv{X{T+2oD z&gID%j|EtzmvgihH^C=*Pw@smk1yd~T%M<*v)rApz^};8_dQodc7vU+*O2e$8~H{a zj<#}R&KI5ipny10eHETZZ*_5;BoD(fY*Sa^Z}}UXi8=TXndfDYxt%j{gM5qrLe4wx z34F79HWxz9y%*HkZL-Teq*oUw>RpBIawj~C!dR@gg-^rhxLW-p*2$k@q+A$H<)e5% z&Y4kB{*fDSJ3Nm5I1rt%%eBfpf-B((`C%T3B67~t6XlLLP9Db#xhPlX<8dvfsjH&1 zd=GM-HTU;=md`@o?@xFALiEGgdgo&+M&NzD&$$mD!Px=Nl(R<;<*)P}EFXe_-}p$ zry#rY6P$OlJ^U&z^y^;7rO1w1PH#6JQjgAOxDr;%7x6{>J&r{UoP*u^rTyBExhDQW zBmKOGtl*{UyYK91ZoZ<2(6Ld@NVtb*L;KgR=4o_!`+? z59a)P$xU3}^`ZPOj+c9IYuBc5-nqx{#aM%bT>p|wqoiv;sEzx-KZfV!s($SRK2-e)s-vI!b_|g(N8U}oQ)lPOyKvr72B;Sy``MHHfzQq4d-0Q8 z53A+u$94Dt^(fpg=igg?;XkogZ!>?(#q?k2%&7cXa^~dcc(r^qGAFxno~y!mNv?(N zcosRQG8_LwW^tZ@JQw-SGHdeZX~UU&`R;P&oWVIObMI$H#pLtRcMYWGbW>jX{ zCj22E!lgLRWbVa()%kl?MSJYTa^#-KGm-Bz-%kZ(&ODB>v{HzBeNsV$spdv zW07Yf|J^Uw1hb8spSK+zR^JR{DJMPAr*oft> z__H z=2j8zi9T|6q~rKPb!JXUJ_P^MTSYnZ^Gsw0R6zlK<-RjG=kgMD8LW_>!3FqS{Uy#v zX4_o7ypPP6pG0QvJk*hMKFsCp8#_4jF=xYUoS0w7m*I4|Dc`}FOA9buJq>TknOUF8 znS)LEF3$Ykil=cD{?=>E@1uhJGVkS^_+p$VkLDG4LB0jw$&Yh29*L#cj;4C8@tItL zkHI_09`Q40?>K^I;~}4!hmP`rd>dzP+rYCh1DQQ7T+9BO{b7-uxpfIL%iiPBoZY5} z{Is0?IWumOJPmd68_K)hjSFCboH?B{bc#GdZ!JH>H8^|V;qnOlgY57%_+?~={*Kq< zZJ*71%awB8^%ko?lON%nD?{X*PaQbB&ntSr;W>3X^$7VVIXmp%a(12>e1qOA7=S|R zukgEkIZwnS%tA%ImKY*m#MuiT*4&+F7@^Sjt6Ux}Q%r>H06G4&qq%jNZd=ezMB zu0cIi)-Q~gd~Q05=&i($@(w(Ya_XTxoo_=aEL4BNjqtggJ)%6HiqrMJMP;nQi+U}P z{q%PAHF0Jq8`k)7yW^%?mbzlx5ij&E=*TH|Qf2jgS;VQ#?pB0J8d+=stGMfpZv zhO^~6xfr&|l{oL)&&g%wL0lWxqb^$MJ%;x3Q9O#f^TqfVcc}9&@?t)V3231n=vrHJ z#YcL7;xM^_Yg@UwI{V!l@*4SM&JOm5e5u@(PvY!Zukku`(mRgF^M`1L#_B6LfA>$3 zUFssef8_JU%h|_z_;H?!nh`X5K;As9uk~ z@^8pJH~}@~oXgn_e^uw6X~K)p-L+Qg%JO zGqqdq6S*gr%9D9H=lSc+xhKCyp7lHfqwyq8!!4+Tsrft}mm4E9?pywhb2c4_p7Jj^ z7|MP3$M?ce`7Gp~JX1Xo$78(uNKBM-UglX_Cg+UI9`GyYIl79oH#U>6L}uZE$h;}< zT6W7bcsO=q8ZJY2O2{uD&uR9UQ}|)^Vic9PAv@<@9;uhzZl_!cbJUq*>v$5E<=6OYd?#OwS8$$s zKhEqpnycY9WR~{ek|>6;>e;wn&iu<>Ux$0Cv#ZpVACl+r0i1L4CV2(EQkUdAxSam| z@+7$yF2K#|Vfakm!Ka}j_NdRtsThQ1dO4TI$YXJ_yq#Mk=hJ3>LhlwVkOy&F{*SXK ze1Oc=xA8=JtKSPaJ_TY z*$ZdN1JDZBqmq7gbi@tnnm)6g4_0TdoXS@q=lDqd|H(UW7xI4BNNc&AGwyBv7enO7 zI6KWzoU`Y0ej0zd_8gbvoLSkSEAT_;ia%Xz!=>;Nnz=rU-_*~p(nr1omDTMzXMEn* z4&`BbWB5)Sjbreh-e4Y$=JFDpBd^9&^5@)}JMfo08%5+txscy?l$>*XGZ*4U_)c!Z zA9CKsuH%(B2pdq*wL|m^%R}W`u@+0zPa%8Y7wX#hM?D!^8SGaB4@8UgwNn3cs>dtJIT8~Q(C@(e@8p{KK_Ph@L^bi`_$P*o{;kn zTaHiQ`ud~fdodW()p^fv%;VK1_$ThiOSvI-$uDv>pUeB#M!7vYV4Z8(orlO<v_KK`$A{>Og05faa}7`(ckBJcA941uJjfI2Zxl=)r{gGO&#S@L`RrkwJ^WOD zP5mFb$eqwvet@sWrE*oD+nnp1KhurK-z(2pp11rQM32pUr@8{kfEoY9ENB-_PJM;H0fSj>!`CRV7{gK(e3kM)S%ep9s zJJlB=&*wwx<;c&knYyNaEBP7Z-YTiivyii66Q&{G&24&_o4HSF%9&?RbH2l3a?a}E z@;7n?&OO~l-YMst$}|1GoSB;EzBvxo+sosT?>zTF?d408M#;A;kS4oPC@SdefP5dtLUY74F<@W3mZ^c{XJ(MT`6CJ z9>{a>6X#jV%=^w~n&B$>Q2Zrl$IJ7d^YRLQ41RrRu)T#W1F zC43tGK%VQ<^!9K^z6KA;mGKBVs82@bZ7ILkZuwD+R8K zz(>fO&)(UNhq`tXGP}BRExi`}EdG|WYvjM|GubyYQ@_^B49cbA_(V9CvoC(YnP)jy z55|+O%|hO}F5-n)gq*)WVygTtFTksE6Fi0Np?BzI@5{Md5;KwWA#?c>xtrXDf5BPu z_jp9k88crl;B%+)bo`F&Mx9-&h|AGlFLV9?TrdB|J^2Mbn(yYZSSp{x<^1}2@`-Zx z)xumIHzMyhhq;z>u%5apepPSgv3xR~lK121`CJ~z_w!E7#P8}M{4nw!IA49WJc7&d zJv@h}a_0Ud{y{y28>656DHrCWPzkNok8xRE#yfZ$u0jp<{hV|3WByZpCI5#q*s8t) z*UL|$7k*ZE=WFq=JP6sd`p38z&$+e?*-6gNXZ2R_t;lZCTHPH7Vzk}}F3Rulm;658 zj)Txdy@R)K_Sg>cC=A0wy*8X3^h-WdeGPxmJhs?+PEv*;f{G_KZjMcjG$b z-$RD!Ka8#TO|OXF{-}V@^}f=}J8S+up^y9{@^1l6IXmNa{sM)3wm0U=XW}_D#s)M) zZ!B`XBDX@`wMVKapryJBTIcsuUyK9sE*k2O<0E-BH}{z-a!1^Tb;wRTkR2@8<>E#KWkXolRYnG2b< z`TOR($howf^Zj&@^WEnDc%1V*=Q-QLPpk81&rEKCoTY!OUyMe+P5zrD!+uoP*pEy zXYP}`)S1Q8IcG|~pQ-A($UJ=&i(Jchml;_^{TUa=|>}K~Ipr74if_gpYzRu2&ohq|Cv+`~}3Df1AyLFJ~F!x&* zy#i>7oPq81a&PCJS|g9)>>o|!+`sG8xv#PdW?r{O&aEmK?fR?8eOnKiwbPJUklFPf zGAlEGa!!@RllVkG|DA?^&s&)CtZz$08rFq9#te07EpWY0f!96igeiGj!Ga)l;h5i)Wp?)2W(N_O+&dkoU-a36b&tA2B zUVSNl#AA?Wb%XjT-huIQIj)J?D5X9o_&r+Tb@@j9LvSi)Bkw`O^}oaQ*oM;RhTX`1 ze-dBlvv2ZooVj`|Uc?DFTmK7`!#L!;&wMJ#tJMW@o%$@4#KUN)mwEj&#>&fthm7%~#~-k^OlvXHItG*Z3B0$~hPQk~2SV<(|ksbPCSF6)$Exi_!nVVs%yja&-Zxw3a;?l(Xk9EO*1EY9_roP!r(CaPnke$MOr@H=YYAHC|x zj+7alS)3jBAymd6_|5h6@HDcs&ez+BOOTyCJNgsyQFzB^-bZ$auhb7<2wuj~INh}$ zu~UAFC*nlI{(zrA4#8_^DSGRFjs5lRQs+IPD4Od1 ztKNmYYhTU%kez3#Yt3*9I-&=@#Cxu9$KfdCb1!il{cBMk3sDh=>VL;KAn&QatN$lo zNo`S1ZDm%n5 z&fooez5w0i%$l6lHRb%@KAW524>^C{;ux-Ojm)lZc`$O%pQ)b7nX#APALJg){dp6X zqMUj-m*C8;oTa%Rvjf!Ae*{0^RJ@Cvftd+6`AqhJr}gq2k44Vk=hW-*D)Jn*)6e}o z8996I;LN>}a?YXr_bjr5)zDuo=jWUKVhA@xX6Oa3&F0TJbM6P^+`Eb=VL9fg$0E2`+tKxYM((`@ z$Q;kC&K%C3^ssBa)tSK;qXS-4=N>-{$EY*&rgHY~4>`|Op7G3y_s|e^)CG{4eiq-x zPhpjO4nC8!!*AvG>X-O2tdO(6Wd7vbzLGNwv(J=LKQFgLaeR*^$eer}nM&j zg+hAYBIm{+y|2(k{XQyS6z)SM{ReQZJP?l}GvF%j$q(|e{1gAr@3n>wz-GN3d^V29 zAY|4o(|-o#efCoI7BrLd+@FtETx-GC<3#yMzMM11-oR^e_VH?*eL3eP|AZJAzzh`FmZ?4^m3dsKajb0xds@{XcDA)(D1bLnSnp2c9dQJ&)SJae@qV}!Idk$3)lR=Q&exmI z)6f#1qKw`(_*K3iKgpk<3yP@o&XF^&1x{4wUGqNIYRdnjGwxRxT@xT)`cYGj_D!W3t>GFUZq5=l&aV8+@m(s@^Wo#!Piv{ur(0{rL_)fxqCg zyb5{0FVD~D_2N-@7ayXS-Xi=b--D0jiCBOG)T=Q;-iGWW*;jYUPwQu|%)cw=rac5( z@FO~5nRZ`va=jNHr1t{PP1gI2AJHojnA*af5`Wm`TDb*XE*mm&c6Tfo1An19qypcGmxL>A9!3n0_Pz2UVg@zJ0<0L zTo>)-`FsX4TNbPH%;w&k&t3GhH{2tSMPX##)WKxeb|QPvCF-N(JR|kwvrtvNit~Kl zBA>@Obh6uAuFkz!RI7;kYB{sHGG=0m{>%7Q&MY|v&&jVL_jhK<6wY4txcn)uQs2&X z_)YGDrFaXOQG<9TXU>+AbHC?&UM7Dk-_3ajmtlW2Rd?sv{5LYcYjI`HGnzB=Jf4H1 znBm$aeh4-2tUBjOXU-Y*Di_wzF7_2KM`rp!{qN+hcpguxzvQEld;ej+1kcHXc_rt3 zX(MNboXw5YnR$hH8s0^Ap3J|y^$TJKD&j_rbNy5lm*2-4Iqw_&`Fmvcwnm=W_2}(0 z3pumsIE;`F=bVk1bJmT+c!&c-)5bLDM0^P_@#v-}qS52qvNOA+N4at&V0UnA$~tvF5ZK7Iz- zrO!}j#^gQW0lkUxshn9g8hQ6Pj?ctX$T|17>vzeS>1XpItkoONPvbA#i68Y2L`V5} zK9O_YJ;QlNXeV#NmFjcxC^AFy{?tP5h9_`=&(uQBro2o1pjUy)T{Y({{LO%c=awEfNJ{fcnE%%f59kx zp#Fy^^H5}m&Dq&cZVPeWd*#PI^Q}Ap+vGzz?`%0kYRen+cJi&Lgz9)#?=gHS_uxIq z`IPye_k-5z_s|Sq==H_Rauu}4PwM7;Gy25pfjk#i;63$p{s5QDU!fSX*X8~7HvQ~e zkLW#&mMDXb_{_BdXp3WTj@~P{P<{p3;s4~fajAZ9OhOBF2fmb-qLj}SP~Ra}9zvEGuj=DHe?^m3Ovg+!5F`tf3xDQ8Th<;^$0okp}tH&U_Sl)FXly7kD8~&OP z<>xu?3nkG=T@Pz8M!y^%#Mg0GK9=9$laP15nfxN!V5)0p;e5G1w#c2Z8jaNl@oStN z;vGI5uj}P`s3zaZXW)FyRWHEf7^}{%)s1srt>-z|;Nr z=F)f+MeeDbzxN|&(ldOh{@cj8-9V~L_N$QQ9XKJF{TE2{D;b*x8wqw3} z9CEJBQ9p~F$hmid-jj09+*{;QO#628>`d9c_Sd*sFFSNg&YaBiwi20dgK;_vySA9u z@MPo}%Cqw$=Qqv#T7l1znQ|8n#|C8QyUWLJ#ccI1Waf>=5)@bW=U&KpF+E+L;I zzlZEGc~8o&n!RnUYtt|if4KfJ?vxMXg(%`PnUU-Djz;EWN%d9oY7CXP;$!&-+=zL| zUYT8Vv7C9^8x`@V&(`DYr}xO;%lYprxh1ZZvlD;H12Ijn6(5Mc=%;=aMs`*6P2DynFu0yY)Kcn)-Df zhY$2S;9O?zQBGsM%{*UK33)5)feJP z`D(6@(@@3ru6niPuQ=~A#rO|(bu5q{ZnU|_Q7xEoT0zs zH{_k`F)qg4xe(srfpZA-h2vJSR8sxeBN+_rZNQ7uiR%Z$IbS8|p{+5L_!C#1nWKH{y@@1upONqxd^@ z8#GrhQm?}{_04=K7Rq@iyh;8N*~RYF>&8?0WL%E^u4PC1MIMYQ>g=h{au0Paxfy2{ zxtI%bbG`vr`pgx$NPY};aTu}-jp4?wmE#xrGVaUMald>pcjdD=`(Sr@X8Jf&Z#DYJ zlTb!3%dI%?0aH1<_J?}wP*9zBgE!%GJ;mdyzA7dU~$qoER<-LThBtcu#+m+?#V2<@wAr zoA2!|JcG8#dAf`D-RE-d$2$t_Wv0N6z3 z`|t=Hsy7xnBL*PP&u(LF)l=1yk$qv|DSC{N>;`4ddRx$4ic25)1jUJJ~?Unqb> z^hfg~WKK?2=WNNjcP7T@WzOfUoGcHKbFSvh$g_SM3gZ*~!pNN7hCEw!ID1qT`B&uZ z_>KRM2KxXXseb!7{x4e;p`yr^kx(Sc9%UpnwUDHNR4NUj&{9e%B(rFsK}jkzQfbnZ z8SR}kJ+F`Jdb)Z)@6Uaoqvzl6ch3F2Pua%`@&@&-oSAgBoH<-Wo%#MhWQQ!yvz0lI z%gEWi*KuFHoV_{ovg<6=nYnjiP!3W@I?8Vi3 zg|HcCqZo1?oQ{*R!aZI28P06=|7PDV*@^4wZNWU`96CVnOyrz7gC}AM&T##ER769} z*Q@KEQ+XJ&Zw}Dki0o4%IeS6Aahk{v;8FL@V1MuXM#%ST6~*B=RIe(olaI%1cu)N| zcFTkLB+Nkes^|4?mS5zTaW)I>wSKJTFlx_lYFlqcd3xh?06 z%+7UTa@^Kf*;1iXq;{#?%YnmAIu8rg%!@dvJbhwK^y)q~~F`7@jl)Xi~> zyg$FftM~$3F84-Dxg2M|Tg26I6S7B*cCC*56b8wcqMCdI=Ns({xsrSwc4DOZVxEJ$ zu>@uGe&U~aG7gmUU3fp2RG)^{@|V0GUF32Yg6wp)dAa__cvBwFJvjUFGP%F}Ev}MB zaUE2aEA!#p8q?&hoL%=ad97T7FXr`}Z>9R&L45}b$<6pq{)CT3_R;a`?0^H*+wqtB zQDi>moO*~ei{`qPd6pez7C(U=df5wp;G8Ed^rj+nIA?m^`TYKGLFVHmyyg1W$gJ(7 zzJia%IcS7ddaeA~%+EW~4$mQTXB>LT|KUYs?yW^3xi9BDYKtG_TD$=H-Q^wqgfrvv zEayzgIaff=Gm>ZUM!6rdC%&!Dj#Cv!$T|O}%Ny{RdJ;dunNc%wqTC#RV+H2wy~CML zBamk*=XRdcW3X7A-RfQWOZ=?PJD0iIPF)zcVvqiVoSo%r&U-cnui+Hjr?;8g;|uv@ zpcSwBcFUh$7hLzVkRUC41>8&hvebes;WH_-5pp&byv7tfxA& zKYyK#oH0-09y!lp=2Ui$(VR18vTK=P9dNLmv-y2)?OJB-mE00t_2%$HxKPgdkeT}< z*W%)QIcLXuhX2I%dLJRPrHZ-@Iwj*l?5Dn(|G;&*=Fesh{4T$PzmeJ5o@?hG7#JzG+-k(RJ3_e$%#6MyuGLO&I+sm1I!}(V%MrU<0w?rpNL~z>!i*%zELKjCw1!mG$P!)hLb`|@Y_T7HfXK{>f6%FDO%37m89Wce03 z=iD~=P(Bk^$#?Ld{6C&SXYtWofeZ66^pSH8ZIGABIlJm1JMnn+O1TH0f>QWUy@a!i zuaiHNFW_tWNq(8LJ51we)F1IO{sbT6LR3am>~?K4F2F9FqjwiRK!1FX7xV}4cRUEk z%h}m>riYtQLw^sx#R&ED$X@&pchM_?)8)JP0M41Q1CPqPum!X6EwTqK=bv4Fm%qUc z_)VR2cs}pt{`@aj#cT2mERmnZaqY9l?#qp_Mek|89r<>DfTy6Xe!lPTke`-s=IZJ$oL#UIpMboSIa^zZ zr(&IZ@5j+-jzYLdKi_r-@?q-p`7bP!^Ier)D?3gdz3F&X|4RHWSL8K#NnX$Mcpg{A zH@F@7wkfDTLEg=+xFC*4clG&v4eBBPEhD?@V{$oUFE51b(ucTrnBJM(0FC9rd@3K# z=W=$Bv$+~N=yk-?f78j|1 z;;r~hzJzOFjywPZQ3wCx5&hY`9QmHep7n;@Q9pb9CGwqUjo#Q_?_k$T@t53L?*?>0 zzJE&V4MH`PK=!2Hcrx6wZ^z4ClzsFw^(f8kh1rp6$_Jt)R=ai^Z|6#UIXA>`j6-ky zr$3(8qqIN2iC@v1#NTrhK7ccGhau;7H~t}6e-@YJ=6o|3;IW)Jp5JfIp>cBNT;@cc zy(P%AlXo=lV`l6*oV|wjowJ!&c}A{LXK%}M^cUtL=RoFXJJ<5_-O2~4+i(x${k&0q zHLjI2Px6k=;4k#@v$}#`Q|DZmz@H$qX(aLtt#UmxssdWM=Sr@RKQR*}UC+GA^LT}N z5Ar+DjGT_l#>}Ih{5N09IfHldQONHozxRKTUEvSTY*@>gnIHKx9XZc*_O3jy^-)Uy zS3ZRo;vN)GpTl|Hat@W}ThLuUyYB+dPIQ-ETV(zng#kDMnNMH4=V`ei|Hpl~2p`Hd zT+exMB(kf0z?pw{q7Npi??QIBJkJNq?K$VjbonE6LNC4Hyqrs7syqOhU-R%1hN>?_ z=1TU6Cdho)!Fg{7;R*LG;v#qy+tir{D^N`RF!Ii{R%egOJDGDJ^Kc5X!}jM!sDYd{ zBX~RNqN#fy$9MAGyob*~Z)AqA;6;2MUlwB{GIw_1RGfx>sfi zOVI#VyY@Ptgx7GN`g43G=dYKs1&1Je*BGveLy@^t9uLd2cpYcXX9l)FW_rF4GSi#O zm2tZK2yaA9WUu;HFXzx&9*cUI>Dq(Ho}IaodA(YFq@2C6w0wcQ886@$RK~sfkKr;r zj|22J@N>Kh@5*Okv7EW|HtMMl!*_CaxoMm|pItuZoWJ*- z$@l1G<`zbFzGi%)YrD7s*G1<3Y(9=xVgjnD2jPG6TPTIh^Xs+lM-TPETn^pj8vHjO z&Xw@0+=XxDDtH2C;zhm1JOf+tnED-L-an)sgQ@DA@A>w~oW4(QFpib4zko^;Qn_yhCNO0PPv z=P$S_zl?+AAzY2S;%NCdWN$f3-5Yh(CHMo3MqyOY`w72ein_J?tMV@Vpw|^|%0FWW zu0m_Q4yYqP$+bA&WXt4Ra0!Ota%3O6gS)%uZO*smMf|BcXaAS-t#Tvml}|=z`9I#y zXJR+Hx%UXYR`MwPgEi{gFj#(^7w~l$iidC^vWpkMwXQc-KgG@X8ScqTcrw4p@9{v+ zH^@5PggN+Le*>~F-N83uB~C(d9O9na_ze7t^>_-;=|96CBD-BDTqIXSb=NB4Vq^z9 zmXFb|EcfN?R%Q7H{VQ=dPD6IXid+k?yZ$E^#plSrQHDG4IM=e1Oyp&_86%K={8B#E zeXsIp?uY#Mf^*eHk^lDbwB9?&KH8S^jZoUPXHiS8f*a&V@Cvd!wRGPDI2{-3J&5CQ z0&3$E{c`R*hTG#^y_@&}&OV**yF&7KY{J*r&-H%1g{vXoQ3chX$@vEONp8=L)xXKB zagCh8)0Agpsj?HYPi5CDs5cLn=-tnm4ViCa)wA%eI%h?mkJWPC@3GjZ&cELb&hH}g zI%jcnbqzVcv&{F*i+A+$OuoR`BfjGN&R5_H(_=gA(`}f8k6Vho6x5yAQHo72#9z&*AJ- zoB31q8073|h3thb)NOE=>v=~?B4^Y%&a*NQdA8@^FIN7Ae_@QA&E@%QycE=# zu{-7LT5HwW5yq<@mKSio16IigA-mi^dNc8@dMjrRltj+y%(}jMqtO7F135!7A0ES0 zoUY!CQ_%;d^)iDpJF+)^pqH63U%w*S$dB@T{bhI#IR~@%{jQ(Ce!za}U)7maqts8} z0Qqn{gSXXfIOpJHXe57`J|0jvK`(hEGM{orpQ!hVx+kW|nTM~*IpcB$3_xeSE7Wu4 zVaUA7yxpOf-SrcG9m{d8e$I-^Faaa53-93_+<~9n*AQY>X59V9B80un>x-I%)4Km|T)1M(P!O8Ngdu zxA-W$A@AnOoZaD1l*MQGRPS~!$IW;r{*ZfPI=)dKf(__|dDx@>7TU`9qMJO9Cvwh_ zoV&y2`#ERQ8|bLc?wfg^^PmAL=^cX4k#Cvo!1d&uS9iL;2034*^B!z-?H8_x$@2cV zOU`-mkbIt;v+HtPk489OzrO27%h_Mc^9H@nI2zBZdt*Pj9cL$ak+W|cf$jQN@d*4Y zAI9ssGyjnuvST*Y%XePRsW0S@kX@x6+H2pWegUW82pq5X56;9S^$mDRu7%3T`ME{! z5&0W_9_L|-`aABzg|QApkUjoE{ha4h*x&nh@5x+Ev5njFBA$uWxD?IxZsQM;?~N~c z5Kh+5x9-z&Yl!>yne5M7<-)EnM85kA@)Eu5L=(A*dL*BK_fc8D92Z4B`2b$*o=x~# zUW%)6iMk>;9wQJ?N``pMOI?xfK3Geu&r6D87#y^I$xQ+Un=HCN7k}$Due7*-szj<6Zj?1?6F= zB4>x|z%S!+Ohf7X8E%P9@_KwASK@(u3IB@E%P-->hO8OYh3XSje~o|C*c_aV>3 zwVdZUXL}R%h3ejT6sO}&y|Xw!*ZkbyR_FZ5ysRhZoyxP6ee_hl{2mu_eusUK=Vy%i z3GT@qIWyx0&Y5}y?>p!9=E@uK4ZgyAI9Y!tN}`qfU*TQ)d+|FmNAjL^!ZFy1j(E|v zWqdqV$$7?)Mc$)}uuU)TVxH?Kk=^nUb>5%6f7wOL${%8f`fJYl+?F$!@>ib!emtE& z;u3fZkE`?ijpdX0VGNTC;4$P~?}a_Nrq1l0%bD$~aWp z|IDN8_$PB$y-7TYZ|A+-motmT$(bpce69-Q}bikv-fAU9P1 zgS<=G-ye}r#76aN*o6`J2k&7TG7n#I&k&x<2XN+P1!9dZJa%~xO_F1q5-mx^y9X!55vE5RW5>`<@x*^ zDq)_wJ9=R|zSYZ_`Ir1P=bNhnf1tjMKjK>W7z6M!X6esB8~IF3ldGYgT*RNxJYFIH z%!T&aN;{u7%d>%=WJG^O%jCQ)PH1PRD8LZ_rboh~oGI zo%NRE0Xcin&Dlef0(MsW?%72Y29ic7l~Am0a9sIxZ`^=-`)9k94t3*?JK^NPr-h28JvK2n2aN^-nAn+yZAlw zrSe1^BQM~5dpykr)kSy;*G9g7vj^6c@6`X2C!hryt5@@66he25*4v6wsEAMWTJqf} z;Gc82`fiLzGnCWo;@Y>^jWsB!ca>`cajZO;^KEveYsbs^<=jOFab+Hb^IV(9EAYO2 zG1uk~at)>N5H7)O`gQP*T!{;!oO~}f;az0E8o-TF+4ZG-Fz@7i^VGsk@~arHpZ|8S zUH%@&srzFTo>ISoFXZf5d2Xg4=VUSTm2=jW!bfO{yvsv4@A`Z465fNHpF8+o&Usyy z+o?0_uEqK4zj-?6?8sjBshnr?L;jB|Am_(sK824$=0x^{KkzzoM!o0yIhd)wn$Jh( z>ss}%$UJ)mnc10-IhV)DJ<&ui!I{e?kr{Cj@+@b@{J=l*ecS`j;2(8m|NKMcx8!rt zQO|-iEf|>XcXW)PO-_TZ^ zjOW$!(Ow?Ie_<@Tsb}z;{2iu3xo;<2AY3nBj^E^yu?3rOJC^Hz!0Y%c)Waj_p!Y9l zFKNZkqlsSTbN1gm)j4JHHB^!F-2Wiofk|>b z&V0yDo;@PF@BR9jl}~Wa!=KR|FCeo!=U&eF*W{PbQGF$k;JiD}@?vhz^|?IX#z){J z`61Mhw_rLRROgI5n=7gF?&VzEfPu)o&imX8IS&TtWhdRtFClXx@8dStzeIcaJkI?6 zUe3(VtjV6WiyzZFgm>p2EJ97aoV9~F-v}S-&BbRJuJCHha z+<+7H#-gx$-p_Sphsl1j)Ah3Im-%hp#nt#+OvDOxZ$5%6^8r|i?9;8ej(!*946B4& z^-AJ8`FE_u!|Fc#A|HejSg+3hbc_6^{0?f$lW`#yslUZe`D;{j?Xa_ApXD`?sIwHIbKybO?`?wyT~-T7+RnuCZIxa ztvBC~>}->{v_CV7U&Iaa9a+P(p3h%$eRP#?#0Wg1E{A0psP2U8@|)W)4yrJo%*`%epb6Ry!O&c~sQ zyqDj_zjF4L$K+~q9ljgcZ!Y24de7kwIlI;@v{1NoQ1(ykJIp# zYscX#xh}H%?o}_rcx3aD)&hC36KY^R{PUq~|T{*kr z0KJpY2u1WpqMzIhry_gVtGpW7Wy^3!eu}dro$CIdf){I0)^F$g1(Gvo{R<>#N@ z{pa030-e-(E;7GzPGz=c9+#CLMP^T5y*v}yC06qcy;0a9FThdAIr4(ucJ9l0pYr=2 z#(B4AaXUQWTAqzOa|_j%AkXM!_yKM8H*wD1$?_w3Oujs#y?Ek}D`&Hie{U_JKX!UP=Fy_j~Vx4>de~iqYI=n?MXU1+h z?`!5*&YXtmhQj)D*rk2DV`fm!!%F%?xGlHDN;$LQ9r;K3RIHIpBJ+NzI&-fjzrfd^ zwmcWtU==p%jldE)^Q8fL;0>Ij_bGqOz4$(E!T;heWM1Vg`WLhCBx>r<#7nqH{Sb1N zW#0b9rPN*dS^f)?Yc$~A?NC1o`LoH zOL3|E9=GPL+z|sXR(%uq#{%rewYX5f7T3US$XQ=i?@c*7?yGWrtjGO&`Nr7A&#ANb z_mICwS$V47BKc;{9+sVPkNgl;Av;WVxxwhT&yc1RMzH9HHAs$!1 zjn!C-gYc*RG<=Iv{%l7+hEH>CzPykdBfDi?K0)s>p1}X$DfwVN2PdM4`!?#`Bj15S zauFPZyKp|9*6+<{b7TCDLopsV>rdnd&W6tek40Je zN=%YF<92xkUx4G~d-!YK!1=B}1Q)66;WNDvcpp=6o!$-nH^0Q^a~s}^!I+MGKnbx zv~|2w-Ijkx-n|)|=dmzyj^>@bf^%L>%YB@4C3{5Po1D$9(A2d_xEn3?^W5);$?BrW zJkNVqPA|`HW=%n5-OXQg?uh*CJ9Bz}3;H~I}x>%t1GUu;rk#{7sB+t}e z>b(C|VOYtbXQbcJr?CIe1=uFkgf><>B}Nd9U_&-xpjQ zmmu%xdiTxdx#2(2w)36+`xb%=$*ioZQRh^>RjM zHuP5KJeY&e)erJKp3mDj^YT<)fb1Q4crs)E7LRgI&WAhYhmrUGDlUucb>FB*%ZpJJ z3)R^pC&+on3#tbp`|8npId7EvcAU(MPZhJU969?(qLzMURo?w4@R43={sqPHIBrDd zZDwfpliRRGotbej|A*`KYVg%K4A~zZ<7f4ACicPg>Mpz;nZ-}5a~_-}zben;zE~<( z(Lavcst@9Oc_hEWd(Z-7aF^a({sWI7`*(KB%=y;1&-Hu{6yqN;&^^;pSZ>Nk@>-t7 zMX(VAkh5k9zw26dh!5p6)|Qa}KsgHMs(oBj-fU zf!SP$|HWaLf)RSpqC2j{F?wThx}5K)GkGTt)yoWTpqDv)joePZ9CqL>^?`gpK0`y) z)cYFaaV)a$UC!CNvy;9rKaP{#yPkVAsSGl{fPZSSvq?(ef^Ch2|)T!tT#*(ZsdJ>fQW0=R3MKuTytN z_L((&jNW9ftbRwXA#X(mT&~VHTOB#y*k7q1lHWsBb!9#stMwnm`MHi#dM|L!+hNH5 zvw<(v8;*r|554qm;}Sd*H^`^(lNgGex%vLex8@bDwZOr8Ib)l1&ga+pLcHYK&ln-! z!iR8nf_JzFI_Z6Y*7804E*_Ax59J%ag!)^w)5|XKQm&z*-VWZxH*)shi@6n^!pnFV z=ek~moAH1AD^|+4^Jf?$U%U=6M!Six`;$#eg@<4w(I#l*XR69vv;lK{0tW9t&$gTTV&Q{ZshlocWEQ?K0ksx^|IsS z{K>nWS^K7(=Rd#GJd1@mGxR5{k}twkWahMTJ=KYSyv=P!_VwL0>g<~+?im3J}ca%NGUqoq6_mGye6b7p;noK2mPnUFd9 zuHNOy%y|T#;D4^=897Klvn4aHB%Z}iy|yTc%;%i5o45}$$8r|DsF!)Ozr0QU2YH6` z>_4wJ0^>0Tujzftc@D4Ob(kn$fJ0FnnKfVQ_u;=W8BK~AIBJ<=3UW2B{d;STQ%jY0F?s1&I?$F=>&WtM9jTP#4$Tv(W zJ{OrA)%j+=4%zd*S1*$PMiuof`~foO4sopuPL|K)yLc;S#%5mT8)28c7}+D&axYws z0_yJWnI>naY9tp&d-ci4?9W+nlis6T0N=~MqapsmUd+(H7Z2kp^+xW^HP9FrtIy(` zLDTq7e5_ZW%U}cUMLoUKF-1O+FT)_*hu8JK!3?Za_r?PGD9*ggob9Dmy`WrHuFG9H=g}VSiR1O=A>XOr zso$4xKohwdHp?GkoZOVJ!HMXm&Ns+S>GSz|73I5-UE?@)cljmUEnl8KhGDe2x_ee~ zJN%?qnqS0NIp3c-Q%53aTpKQ|zaHJ>>}1(huTqak9W=udjCH*(zsgtPXZc&4hKcGc z_*Bf1J7PAjRafTg_-_19egU70Ex8;z%1_`DxfA!}+USqL>LL6)pN~TRw`F&@Qm+Zl z#Xxme?$6olPC+qsGxWfD$$CpTJHb8N0ngz8e`Xnae z{D?~G@+c&qjhXUq{3<_>M##6xFP!gygI%j3566>Oi0rx-={J@iK-XJ^~Qx4AxoKjvCkj(?GF?NKrIV6}U4X62k{ z#gq6ky~B`aKJVnsdW(=}?ol4inaeqI3dotuE#&M%S95+R+mL6uxB3F^jQiy8kvWpR z<`MZT{qj7YGsCxV&Wv;PUPb2RUbI5yPI1mW&3s)b4^0o*-6rrDy>&blnXNek@-Ez@ zzlyKtoB2e}yO&v%XDYvsGmz)7ItMIm*y>c;UXKbX`P`-k5K4eDaz0bb+ zFz>`&n5=&YW*~p9!5sYM+AJ)PbB4aaH>8iu*Nv{-8PsQU=GWI;AMKI-A?IdO*Ymvg zl1uOyY()0OA9>&VF86W0FpuMHxYf1FTmgB%&Q*`#JZFz%o!k`fA~X18y$9qKTn{tl zXSpFih_$(foFCKlyI{V&mFMy&ya;trRh?PVlmF&F&`d7F)j01#W=dvSp52^v?d8nP zL2BB!_kYc^kh3Q$WBEAF8GNK%U%m$i z$b&e0__zG5I`ix<`A=kqPT}nI!}PP~XMfDO(H0*g-xV$Ra@R}aO}Q(w>n%`MlKWF9V~$aml#jsxbilR98PyUWx}G`zI&!XT=Qg;>wJ-1} zN~j-0&YOY!628ULu1&7@ei3usb0faQsp_2f zedKH8^SL}$%aB<0EbK;ifBtj67uVrE zoQLi1dmXiKxB3ITfHvw&agclpzk_^Bw^cuboP#Y<5Xb5FI^XA4$o=(4;c$5rD(kON&y%l4cCwLN8V}+;G{C#K-+i0-cCNtx zq6aFeKSpz0tge;o{yEu8pHjC|FX11#E3T5?;nn;EPv@0f2y<|P`VzGD=Sy*8WEbu1 zS|_;+ABbObFP_nBhMc)2)FrVKov;j5UE9bn@U{FSXOGDqbgR4@tI))?J2~fn&h)(Z z->R$QWW0@S?#cU@=k#~<#*H}1wfw#hL*BQsybSeF0xk5i%k|>?zTVKgmCwSNn5M4F z+1FaiW6)V%q4$;iE$4k%&m}qU`QgZWzYsOuyO;CV&-(K*6OXA!<1FM%{GIR8-;P!| z26OZ-;_O_f^G$dP&GqvxFOtj1fAD6mgJA zeIVz5U%4wzbnP!>&&l(fbLwHdCZCV&=mR+CbX|VLwV`|`XHWk;_u^l?fg#9ww#_|_ z_z&L8J$Wv2uAi;WOd7+PMTh8bk*`7r`8&KT=N(^)W7V07=c2rNEuNHf{x9bC>Uzj~ zo|$%;TtogzJ{oVTPr-aS3*2#(^Cxri75Qtvi?ffMBj>%If#T?aA^Nj;Gd9W-k#}-E zavrqBE@VDu&$&mx1v39$*2}w}IW!d=)p=Lf$rJc3Opq&cS^fyi<+a?=pF2c;07LO7 z2I^nVIn(dsoWI!}a`qhTTE08J*UKFHMZEz3s56_l$`z3ra}-zLa=1fof+F%B93kfn z8;5hT1ew1%e`mRtGdtf2r|M<)wURT#o|ngB5ONM&f(P`w@z;De&X7;%Rh+Y_n_L9L z> zN&ip24#(p(RMh*KZ$Q3x&f)2LZIJz~lzI{7sB`W&m8;4PxF_F?qtIJD9m_EZgOT0g zG1qdwWtTopUWrHXpWX*(jN$70IA_!qa%VJBe})yveszN0MRI=dr&7-0_j$a!71!V| z`BR)I_u@4uApebb<(!{?;{w#zyBqm#$X=0etV7k?UAqZ4BIoaqdO5Fi{#DRxjn{Dt zcBIzNcUx)q?T>s1`}===@gfT9O~s|SSKY?NkNG@xQLe<<8y@0cafRN0TnL5bS8<&@ z8^>d%x+b^fPjI9>7~OH9`X78L4?wc`ab>}WAyTDZISc*7vVh1 zU+d>N&(Cau`gP9C&ChXyIx{xE^E`LmkasITgPwR7nG1RU^Guw>&F~iXV6Mmgiq6N#rGr6VSdgF z^?D)Cd3zqohjV6I&br4@UT+A_lAp(8$h)6+xs06Il6R^f^1I!sznb&T_LFlSPM0%B zat3^@&hI_vd7kZG@C5SwosApi8;~=)9h&OpotcU3N-wGNtYo%lK0m9UIhg0YZ!$K> zc_(vb=lRTBs)q06IhgEP&YV1hdEOV`we(!eewKHw50>kd#xcnI`xP=vW+F3k9g3r- z`d;L(&Nu>j@0WAl*-6NI))(19TdMz+Kj5-hgY%K|INt*k*|_$a+QFjRh$8)KpTFg`?i z^(-ERU*ycXUUE%TmYecHJR8O2dyw5Bv%d%5hGX=U`_AlqpPZtXd9xXPaHMOO@ZGow zUn1vN=G8E~BG<)mWT(E7^ZiuIwf*E8JeRZAo-XISug`blCcXFZ2;Rm9y)Q9Aev)tD z+I%oFqdr&ng?yO`5Np+ z1^0Z)6_MGRS)TppPyM@4MgMx#s`^!~ zjI*#p{SiOGm!XaP3IE7f^WnUTv#&Jd)#{H?LjDZnaFP09&hE8RE+lV6c8TJ=2AAo- zj(n#bq0XFN!iS+H&QO=aK=}szfz#A2@PIs<%W^~h1o?*hmao+-iRb0cdgFMA`X4@l z>+(-{0Xg?7VTzn@(mLEkeIXyhkMdl!k$dxX{67q*a(owOPdZe-LoO^YlN-p#b7y3~ z+Mk>2wc?*}61riAUR&IRztx-gJpKSD$uII`p2n^5ubiFtVtzqg2kYg08(%Ijk*jk( z{+&-mTfDE%H`;u8p!^w5Mt3}fZFmLOxIUko;Sijzeu;nOpV3IJ$M5kYm?-BvIp08A z)px4%tvyB_EO$gT^v74J^>4)>c`?Q#|NUcZdU!^?3BSwF^3nV>-^LZNP411ie+PJr8{=toas59O(r=3cH^4) zkY`{6y6U&VIy|NxnI7`o9g2U^(zWk6bMs)%88uzbUYT~0P_vC%d?=J6tF)oapkC}aWcXDp^l5>V+F0IzfO!{8^9RGl` z<^G)ari6R|zClrS=5Ef6bJbZZb z-{KZI&wu7y-sjW#2%IA48J;QMjqBt-ocC{yoZa*uz7N@@&)`}33^@mPxUVTnqnzGC zUdTCjF2~z=5jlfC$2$30p3m1JXL;sZ-oN7d&G=w0$(hUd;T8D_+=xc%_GlsJ9LqUA zT0NR`_MObX@XcI|kKr-siR?<-^jgZ(I5VjUGOKciXNSF4Jq7pc&Ed@3qp(@dUb~4q zBD1$RzmKovEf^tpVz=$vRm!M8MkVANDWIS4pH`fik=bxG@?G$vI&(7b_kTQFKl5y= zoSpp%?uEbfUgJG@Ri4C)QNlgvsc(~a^IhnUe7E%BH}zMdmHI{=sCN-^241GlSvQB@ zK^Ogh_;K`=+oGd^u1msE+L5E7dEo z)V0j_f^xnCa^{rR>%dcZt$xn1%=A_AdwLz^)8x10oN-m;a`M%fE5C#5<%WDD=i7F% zyi>l6=X1`iukpG1Os?#n%lTCGyL=NbLQVa1)UD*f=px^WjrdZ1H44kG@diEtW#ybZ zGvvd$2ws-AaL%MQ{+Uzdad;55ksas^*Q&{rF;?D+yX8-Dt-KZ=$bTX`_j2_|a=r~u zmD}?pTv_h{JdSH{B#y(InBe+QT!~HSiAVJZAm5Hxsk?DMZlG6{v*W%lKP+$O-uy9M zz!G)-s)q&g>9|vVlJDU`Tpi!b?fHD1gQwMn_$3~Q-g0Yx2HE*<;4d&4_q+B1&&51> zJeJ`w^$`4lTh$BrEZ)wapcW2M*T6|~BmRa*@lV{Ef5ydFjdyXpes=82TtQujzva$+ zs{YgR4&19g8J+a<4fLPhOPGL-sP0}6= zN^Z=naI8Fz3nBkq;XH1L?5WvL_j5hp9o6*ikUQ`I+>BGO2qjS)-O&u$Q?q|%mwVWs z8L#&#*319XyNm9WvtMSnDx*wI0JdFj>dJ!JDd0E z201_X3G$ho*)oaq442{AoZs2=a=G-7_ao0_et&uA@~mc0x<)@|KxSD5Y*rtFoG&xg z&+-q*vs?-}<1#1njOE?Sdzg9hJ5S6#{0nmSnR+Gc`NJmB>4KE3%7zpw4@r z_vtmvMNhqx@Q$4G>s)!coM-ra*!~3A4dkU+wuhv2dz3gkPkeO6k zomn?So*=iymvYXa4>3x;g@^G;oP8sEY%#epF2dg!>-x=TkE^i}LP>*afBB@aUb48d!zEkP%_8js=r=!rkoP0$}Z)j4Ob;OXjoQ`Y5C>Ye-} zKfuj-6N<~NaI;(jd$3%+mK)+T`Exuh=lkPF`9o~S2poVfk#D-}@N48l`C~o`=OE|O zB#gtG>Kgni#-gFRDtF*2T!*Kk9qOv*^Ak8rp2bJ==g7{N^P&dUse7Xm=Ho!Ug}fE_ z$~jL4^PlRj@>2Q#zh$<`7wi9qN994h3vc6f^)C1$`@Rudb9U!0dO5pVsMpI)coTom zJ9sePf$!uec_O#L9L&R3Y``)ULuvQC$j|X^^uXyTis`uAwe0gnz>@=tH zC+fHO5@e6rWq4Ztl?(7hX2%mV3E9;yJHaAx&K1F zYw*}!Ioz`E+b`e2H!2QAW%+FV_fS^-Jm1bA zp*DJ>pWc4%JA$88|AuGqnfg*ZDObYjI29xGs^K;id@&cy(| z!}$x0L(Zq={3+I8o$Ft5p1JHCGdO2yea@^q4SkSjq6cy&mT`a1<&B)5cV*7+{zhaT zf3BCA_8PJStx)$#&Dlpv%9$qz;W)XKUY^@y)a`jRW?+JP7;^sajj>8U=SWx1Jk6Z! zjg`prcDQ?Lx}NuKK5kGi#I1M$XXx$6?a@;%!>#d{ya0L6vPagGbEcMe?IbzB<25L;eN3ak08Jev}uWF)l%AJccT+6~fJOX6P*WW8~Rfq<1z};Z9^P zo2=gqwd6iRXz+q^u|C8%q%U>Yx?gcXow?gr{u90A zk(_g-Ag|{oyakhSfO;mLl5=hh<^FgAwf!@<>YXAN;J@%7PDakxZ@C=a#x&fI0`7f| z-$5PuB%C6@iLrQ6eJ$_ceE0qjAIcx#Rs4;6v@3FU{Y{+x@NB*Da?a(K>+o`M{&NX zrpWmYdcn2D@*lWP{u%R-|He>{E8}0+N};NJFaLse@*(WDedlWtJ`m^NMAs%F-&)u5 znfM>FgBIgXXsgwo@5d7PBXpJrqlcWm?=G%~>~2QCn`x&+syX6m%U8b8l z``podlU@V)KY4|GJm1C_qM`gg|HIjbw()_Og9~sRuE9g@*^WcyDwr<6&yBbuu0wXQ z!8`)l!Jg;Fc-eh5_+WmD>tceu2${dHa^}Q1tU}(S>?76WyyrP<^Bx_Et?HY(KF&sF z=xdzcRp#q$*o>a&jeqg6Kl2GDV?Ija9Q{0?hw}h5R1V~;aFl!>|H*xLf8^}1q~1?H znKQRkeOQn3-KLFW3B7i zVa}4*$i1-+*(1y9^Yj>zzKLoU61`E_q|tL_*A|8 zm2<2newSZB=2PZj&Zqg9iafu^>3=LA%U|IVc`;{?y+_V|_>H=U{0L|Ft}54;^FBW> zk3&^;&hTgD2XH4|ME3f%{G;oiAv1FkM(bV0FCp*d3+mr^3@^vexK^FHJ%~?KHw@|( z>IQQ5<%#?{cInOKbyzNEeifFt;AZ(!J_vbt|3PNgM6ScLco1jyUBS=sC|-p#<=?ms z#$q-G>Mh~dI5R!-Xt(+>{DdR$fL>eP!sW3Mv(%Y~9q@*nIoXlB^8&qx<$SZ`T$-xR z3@w8-n4|uK$D@GU6yM61apqsXo3axo}3(EI#=G8n5QRh43Q7(o? zXpGfp@A@zv%1yY2`*M!Hi6`+5#=4e0q$S?Qxq8cRqMUi0Gb}SXyTT^@oI5%1vzwPw zmsOX*Lj0}X%{g!S<3e>S_uVUBjSrDsqZ>EJA+BGCll5At|B_3fu(}yPu6HDIwq3z< zae@9y43nS5Xk;HfRxfAe`#clXknfMzU9XPM)!BpVarU<dvU5z7rqGukeRlh%Z1n)J7Rh(|?%1!2Q^X z8}za-cfg_Yi+WY~N~V4L)CM8nNsn_a6x08J$MC0I3J1#BAG61wAy?t0JOUM6&u)Js z&%=9qL-}bwgR=+bjLz=UfQ#xq#KXC&UL`(BT?3`?D_ZMyK^xcmashsqvnN#M{Exqs zPWV3>>;wGE`R&8_pQdO?Mnrp4X-J{Hl(dbE29;Gdp=hb>A|sTBN+DaB4YZ6xh@zq> zGqWOjUhj_MaX63jeO>pr==Z(v>vP@l75EC-Vc*qz0Y%g&aB<$n=V6ar16$Bhy`Jyn zOZYkrlE-5w2B4wdNN$a@(Lvo6r{EU#Fy4w1xCRUHGkUw$0^2b{J(uV3?`SIji&nTt zJ%%sjX*_`Y@EWW}Pj&XFC*^Wzq3+2A(FU*L3w)@52lmRJVYQt1#qRPnIe&LCm)}u; z$M0b?N@J8>-Y@Rw2i1*{{kRH#laJQ#%=P(mz8BBRm-6AM ze4dN^d2`>lP-m{?T7Xz~gfM zyqN=?)%i2!9@vfiymD{md3>33ALq}IXKxQy$vIc^z2~uXl$^y9* z>Z%XKGC8|gp1<5*Q`P4n&)d5wiZ$xj`F_-pU*MdRIfomo%c=AIf7Z`@&Ha`$X9jYA|KeJ{w}a)O=!Y}a`9Aw2 zyIr35-g=q41<_KS`IVV{2ChZs#75ULr!sGHULGZv#ct$2%G|m^&b-*cc}DZx=Ugm^ zkL2roW+mFm-=s#)VE^vhMe}^VCg=G%2bq!o@+Q6KkhA_VJ`!{Av1_eyyu3fx<;>X& zi9_VfvCN#ln5y26%)tg2qL+D)`FIRp zjO-_usdFCQpk6AU%zd~JD#)4ftN2Rnr}rQ_$pbJ^J{ozC_(=UF9>);9@A0;rvu_Ye z%WojFVH(O}IkvfW9%mnTST2EPxKwW(e}~M{I#`0?>J!jR&OFY!+#%QW*5Gfvgj4js zLQ}k^euTf^QJi_0U1PTTMEno^aGQ88&qDT{HoR5uOKy&<G%MP@ww|K zawYzra~5aD&6B%hv0Rh0bM%#8mM0`2{ zl>bBaj>-HT za)w{)+88-|>78=U_Z9L=xr}@aXHRIv^U=q(50Uq?yZJqwi(YsOKl{w5coW$%vmgE{ zkHu!x_L*XQCb!~SQBrQqzi?MB%$K7Wu2g@{O?f=B6W_-3_0Hmn#-oxj*Rvy*l`*^x*$78HK3we*bL){s*aFAXF&hUGYu8{x1U3eSq^;Y9h z`8@b{-+o?RxEROzb+uzWxN-^MYukn#Al%p{!?#7 z-h+3j*W(%WQoaj+<6-pFtA}CopZp}(ZjE;IJ?lD@uxhfAt6SPMmpV`Ehxb`jbzE)b@k#AJzeK_w}AIn|ztKcwsFJFZp z&=M2$Mj$)bC!D?O0sSiS?L3Z$;{g5a@hj!rLwWv|$ocaR!l}qU^dg^te_YEwnEUhp z|DI}wG#wuhsJQ2C?^8JmwFmzsaNp( z=#K)(cbQ%HK<=Ubfs1noKAVeTo7|T3oa7lgPrVtLMSGCvdpR=Se#Rv}b36CtEnE)8 zy7)>c|US zdx;C9g5C!H8`)tRxOR=4=lx$yl`|_B<6gX}z6Y71N2+JZnW63Fy4(|6<;J6JvkIoJAfbNxH`ME-!Yw=b3V%IoQ+X%K zAm>9*{gb#VZ%5|*4|otIki9+c49Cg)W4F&-h(qND7>b-3-|#!gT>T1p2kWiQd6##U z?5um_qOL8#wQ_c$Ieb2DMj!p>@P~W`zlI0UNS#?-M9w=$&hcCIMym(nKIET^&>EMz zR)J69PxuS;#w2w$&Ym_xei(PFufRj{(`YQ8g~Q~_FjvklRaMT;e3bfbc{a|MvwO_s z9qMZ6D!+hw$UanybH1O#yIdd3ck=HjAveKgcuajA9>8t*6Ibik#hthW-Sn=+YYWVS-gtZak}0wSSF9>ydOL$pDTCgoaL{{1MsUl z?_JaQa&>m#Bl&*yJg$Pzum#tnnf~>tj8W<(1_*&nhK?Nw(-yh?8bdf-XDe9;H+Ay|!;>TcTO(OJC^Yvdw)s%u^OBRq@W zkX>y*J{bkkR{cD`h3qsR`?aOz>iicU%%^cpehb4;2uJ5L$lr~$;R})X@s6&wLKAiN zs$+P7dM;1m{9QvOydyu2hWgq2zLe{u5t?EUCZL^bPjYsu{C&b|c@*;hmCs~;9M2CU zXIq}roEtd@GB$9G&r^ z&*kU;INBkzEkCP#|M?C}Bll(Q=lq#7hx48OD(7DQgY(ShKF`l_sycIWBIoCj*_ZPv zb38xuvYdN1-+P|je0O7z=dcD}gs**WEoa`$mtT@U!4x?^!%>|3t^yCiO4M~d-%rl8 z+@Iy-6=)$}i_W_p=h7jS062KgL0-*W-EhMXCnnNbu4^>QX`mUC{5=4*K< za^^pVEy(lM7kQ2|*Eh=dAos;8o{R%s8;H#F+~awUf8xxU-0u_R-0$1uJg0f4nxn5; zd*58jeV;w%OU}$5#H(D(IaP_zQD<)CdHDi$RCo|#^IeWt}&KX%* zKhNoz$jp0!GjlueShVn&+juW}>SxwJgFHvaa#5UtJa3z@5lz)?T+bexvv`qSX6?(! z*^#qs0cYmSz>nDE*IbRv#NqruWR~T5FD#EhRrOQ3mS4|#zTW3N`>pXkvZK`2Yb)=; zPsj|*J4`?QrE=aOzT!FPgUsFYkTc^n^;G#jE`Vk7FWeu;;cA?v_cSv5@5OSx=lE}A z-rUDIZ}XnCU7hncXGhNeZt~eEFJHZ;%1%+Oj~5?8tQJnG8x_%Jj>-t%(q zeutb_Wl#cb&;tvR89CGS3(#D>ge&OXAa{|6U_N@OuS52Oyl-XR|Ct_lM1Sm3zlr1J z?4&vK4p6t}#+);uzWlR%DAvh+uv(sn>=2)-^G@?JchD=r=kta9HqTAZ^(yM&@=P2m z&p;#jHQXo{^qJ4)r?6Z741di7_+(U)PvITth=cGhTKjcBy7rfR0_R-3i5ue;T!weB z8E4}|6hT#d>$A^ex|}nrfqa~N5Dt@nz#H-v$Qk_=*TXvfkNG!jkPGo5jF)%tXZ!^| zkjEis+cv&UZzNYh1010q&Bc-PZnpYTzLg*1D{(!t*PfvF55}v{UssRYsMlbyJQANEJ5ncP z&$v)s#`Ww;U6K8yl-@OZd4DV}zk~ceL3Z^gd*0!-eKxT<;C(#cvWu4cidCwUr(y{zyH-(uAX=*TQ~xeclk?tl3xBH~!~da<-ht{_@;FS8yYdw{ z9NpF5#dup@jYH%rxJ1ql*p0LEROh@q{Krl3ug_eBNpe%3%k}wvRK#p`Z~hby%ID(; zJcjJ*TlqKFUc^Z#i`(@EpfhUXA-x%V1M=U?{h4R&9BfqQdwNcO1Z{DzdKKsXE-AO> zoSC`jW+La->%2-Yf9`yrv()(x@(kv_$TM<1@*TC|b8xe~6Px7Bp`4-lbLRQJ2Q}oJ zLzxd7)Hx%6lsC$`|H|_F$T`~_`L0(W_ihi)eV=)jb0FW{qsVueIlGc`&xreWp?Bmw zn-}0{>{kDR7m;W9G`(IpM!ugH;xQbdeuEbw&(43`lV9b0$Gf?O|48nqd?)WAds=2v zzNgHUJTpV^nfwxl;TLu8iJT>W$(g_T&bQ%d{oF@4%1@ydwjj@9zVpRcCQrp$`4z6r z3wS7Jc6Gv~a(O%k;scR9{;T#?^KL-~7d!P&dM;yZD*Ugq3^$eEVCqJ_K+9WX?% zBfh|VbtT+~CF-1|nFZMy2di`LELUegIa$s;KM6IE-R4$2gugLb??>c)BkvBk>(y58 z;dQt}J|Ec~wyQsvk4E<5^Z0SS%$l6rqtR79p8w#I$U8~qL}$IsuBGzbI7)sE7s&lF z5yO#L`4n>QjOUy;PxxG4-oPIqv*%w-kuOBvPo}HOp|rXmiekPxyJq%*W@w-^LFPtoI;+!qnw-(C%<%!rX z&qdDP>{u)LKumWn=ht&QR-N7QEIuCFP#BAl^Y|^F`4Uee?`C~D@52kYB}T4 zzb5aY*T`?mQ*gHY9v^^Cavcni@8nXMqA_@%=xrGe~qPj-|>SOEZ@TI@s-?^C-G1&!)M}W*}waC%|rQW zMRmX5Kk`(xQt!oGa`uzid>AI_Whc%*+3E8RR9*f8Ifp;zyfgg4CnE3hxBE5Ocgm_i zmJjAK>YUf-svkuuwATNPYvE$~P2~M}2S20N1SjAS^wR%{iy%8{W1p#pzttCGr`(0J z8x`Wa)%WrYev7MNf?S`kz#924-rwi<G@IXUw_#n0~&u%=nj(@3Iw+&|ibG$a$0b_yp(AobUc_ zWJa#l%lG{ddBNQ3jhva2 zxt(+TB>8OQ%*g$o-C#Ip@5>yU=yO$(v*KsY`CU<7kKDuk_3~^iozKZFW;wLz}#Qxk^{Rh`W z&Vh2+jI;DN;dpdIp0U5N6M4R#@|o;)nco|c9iss6L1yq=9*XVAY}@Yh@AGTC6h5+V zR+QzuZ!FgPQ67)dsEJj03g^1^9Bz^yLC&{N)hDBfx;X!mK6mB|c?>ekzu-&tit=dA zjGc^yxB^S`ujDEC72m3#M&@@j9v5SZd=4LsQ{@xzsaz6uF-TpG$MTz;9i+1V{z-h1 zUe1i>JPwuga#k$Dt#WppgXEH!gAb7z*avlyz5Z?f!DrvaV06bkT#ox)JCw7tej>ku z1Jo7ML*DNOan7+#=pw&>>`aIAGpHuF;YOG#zmA2-KSy!q@f4Ijbje=KmW!&9Cc)HF81DEdN!` zxpM$Nfb2_)TpJ+|!FO`@g`B77sk`zW{5#IUeK^}^I&(uF&C9S2wb2O==zouUW&iGb zZ+@0@-rmX6G`Db1K8#PpI(Yy-Nyd82#0EU0{~LEg3#?H0!WN9gDSG2M@0YWF@J%9@{cHlx76LZ43@|ZP+o3~TjU$~ zV|*oNN4rVh5BIBY#%%0Ww|4CVd6%3WcQ!wyew8Qj3S?h+R(%9&sq-#&Bezwbi9&K; z9)?l!A((+y*s7Nuwx#^DoLys>e7gK6S5#lbd7r3*()z!l1pd>n$@B3SZp04s)XzK0 zlbm;*u6&vPH{22J^?p!~l&i|W$nE636K4;dtFDY~dKcqz`CC4Vvm;gKtJJrlGAiI! zy;u1>Tr7XiJMfPDG`Hd2+=93A?K}#t{C?%t4dwgzF&@s(b6-9RpUK&mFOw(8+4EP+ z=cAVTPMoIqrMf+iR-eu%pu1d!yYd7+l$+u%`9f5cU*kG_8d{K?oi{pCE92V)VMtFOb= zXp4L|IiHV_vvYi}&dySpU&bE&p~wuF%ReIXdo9|zzLG!T1>A~n#{bZdj>MPp6`beh zCN9JWaCW-PluI1d! zv;PnNRcGfJ#E0Nfy|?%Qu7`_JFj?<3ev)%WW-nc?z6ZTg3b)`a{DO+8?z1I1XKri0 zO8qJShKVSHjVR=LS=XMGr*IQ&LFQUFJ_5V(gn9&a$akWId>YT;({Y#Ff-gkQz0BQH z+(zYJBaQ z*1jEhJ^!bdvms~Ag?u~r<$*X#F2}R63mw$^@e*#$3-B50VyxbC_z)kd&%kz6Mb5OG zZF}S&Tz?bU<8n6STrI(eA@2)=T)RSk1-&pJdd1xNBhi~_(na2KSR#gXpx{EBEB0e(%2WQhck<`{ZW%C0wlD!N;Pgd>@bIhxtnWl3zw2 z`CHCj^&oFkui?B?t>&8Q?7r8_c}J_tL-bC>N%H&LjuC z7vjaZRlXO8$=Ok}C)LAF_3irEzrx%F7xxu zeUR@U_gC)kv7DKad**Ncgfm-PW1PAU=ljV0m9rqT=_}5ky|LZ{ob#?YPgCE6{5&&X zGIze?H<0IGIA@-`j67$V^#kSH^QX&0_+WKr_)=s(=RVGvm+vq4eCApy`3Gc|%CmL3 zJQMj@&*#jo>@HvOpU8dNp35Wm(iG&Z=y(TaQ0Q zGfYNV+E_LQ)=J|_qOWY{8@Y$Rd zc~8juMh(4RTpPl9Kgj%hLSCYG9Nv}x!g@J#>3hBe?c{ab3%|)9ad*zSlpScS{0L^r zy^%Afhq@Uu>ki^PESW7giASTU+zpq?-(oq=MrQhMzS*^$Q<*0@H*+?2mY3i_bq9Qn zZK$ZXhs&X#d>=;QT9n0exWKhjIqyzu`^Z`FA0Mheo;Py^ewwpi6heJ<&a_|oDBghwFdLWZ?d6j>b8`@P=gi<^ zaE1CkG>|Le0{J}5lQVbc;>n;sfnVj;oLxKrEXFSN7S2wV`Cdjn17GRQMfRN^d6V8@ z$Qk+}f1%d|AIWzjXV7i97%!;TaxLzSN92h-g&*XrxCQq?&esL%&#+v*6Wj5q`c}@n z<#l`_s$!u2JA5lwN6wHMT!(Y6|05rc>rg@c9$$-6a^7S7yYEbVnAi9<3;9!C!LRe- zC@de0+E}Ze!FBm9ydYp=aTkrw8B0I@cKEbuu zuu|@c>_+|Z26~_kM(OXyBwUQVr{rBbXZ)?` zC<@{&{DnhYyBFCfKH==*qxCEDxgudrbrNDe@M+8;au>$gUAbxHM=T%Yg4^|&8J^@ib9T&Uj5z5JTI>s_h7Q2hrgsoU~7dQWrq zxVAW5o`)qk3J1FW8+OTyP*%?UklFg8oO?FULl-&!y|d&^oO^eqoO^CEXAk&6&i{9R zoQwnYbALUA+&?)h^W0<>oyob6^0OGnIVYNO?$6A$S@KW#7N6?nnfoYK--z6I7a;f4 zshI9sZOp)Wv_WUI@O$O?Dj?^%&-}?;d^x`cXCUW7?!}+=|Ksd~wK>meJW?#nO5`k$L;DerEXXyj(BOO`h?7 z$i4n1_vJ@0SH7C_{5D2r!yCxF+m6ZU;bu9rE;B9Xb#|WHF&Lfn%b~ZN9WpcHI&4&L z#j(ijyGAejxPSNUcgL&WmNSbo7Y;&O6jJ9ryGFhP*Q#rx2lDJM(91l{+?Xl9kL+YK z)K%pxc`W82b6^T?(a-b$u{@vi-jbQL5P3$A;0|~IU#Tl`JzR#r)rHYRJ{(8L=W~5t zhS~Bt*n+(0^XNIb0Dh4V=dY1j^B`xIX2&mr zJ#ssA)c5bc{pTvqd&S+Hv+o%fzLQ_&e#kjeMm-C?(ASeoj7~y!TajN zIA_^e{I0qfN+9nlH|qT*jg-57QkSFn}K|Ybo zxR$+c0ROIjjk~FHe$|z?^O1NNAL9@V({G4Ua%CQfujPk$7@vZZva!#M$3i0P3 z|8epk@?tzM58yX&h5R{=m%rx<{42l82csj_tG_@s`9lPXR2V;i(D;}2fz2=$y8*Rm1*n*rF>+~`! zGw(9T^Zoqe+8S=dE%e9AdA9qjC*orDCm4f?$n&;>4@91k%#V8Lf_x{}apqT9Uc{N9 zKVh&s-$h9|bFn#2K<>|PInPg?(KZ+%H$%>>+&7QP_3#VwY>eZ6cn}+LgT59>xo3OxOdiDBk>{`--^wrY2`DFL9u>wd$hk9$^E^C*Jfk_EbB1Il7vjwM`rHFG za4VMRtwg^2Gm-oL58ln4QBA&q563oiQ9sL>Egj@ms3HG=1^7x`6>lK-)&00e|3&2a zKABf>?(Jic`*|sHCQswJdY_^wW~%$~P@axz@>Xm>o|htgw*Fe~#d#)oAUo|jdM}_9 zMxhKc>$AsZR_u|NaAws_atq8sGvs;9*_LzqT;%@0(dYg}&ecuoipWlvGxlJ)l51z+ zbX=^?vo)O;p$9T&*6Q7j%-}!LlMm$V1Ru$#$u0Q~ejEklHn?5x&Nca{^l_8AApe5P zdM(mvzzCvJ3!r9e*`jDe&*KP zP5(@JJ2DfqQdKYI0zJq(If8?8S7KW=coA#op`WBwV<9IQT#8mlf zbd|Gf9nPiIS7Lv88D};%moLS2n2pTtoM-pBmUp(*JP<|X%U!!!F2}|B63#!b@PEku zocDou^>W4y*881b#+~x}ct(DW^Y@^~%h_oPs>jM7;|ci-+%G?XdvK6C=ltpNo5XWx@BvEWLcLG$7nJ+X&CNpI_ZF%vA!qjhy_4_*@*dny z|6jDnVxk!Y zJkHlUpD*Cb{1SJ?5cwhG?*;O%n?1_E`}Wpv^k0?V#$CvcGKMSg4(^17KGO$zUpR(O zMt*~*d9j#xhGE=LT@y#h>oGxI!P%|;lpjO(o6k7!llO5QY{eYp9iyAie}L=JTJK6M zz+vht_y!f#kMNVoj{H7X##6{UMI$~owaDZ9J_310@~rm8WVG;meyTT9&Y5-@pRCTzYbtL?7r9&duIHJ!TfIyDBRAvI_$tZAGv`5G!2cjS z;;sA)va9@rqVm{$*0mz?CS;Cv)%#gqjcv&HG#p#x>b#p@v=f&Op3v%Dh z(7%w+#x3$PZo|FtEppE_(93MQfHON^LZ0{Oe4o#pfWq<$9$17HxCV1^2X^{Q_LZ&j+ww>}B6s0a@QIvx zewTbX=R7Hj!hX$K^-#G09+Ru-?d3O-eehU5-?i+~HTgZX(ksS2k=>yoU!k{~H{wKj zD1MdqoqIlWk^Bk%)vv-;`F+m$@CHvocC`825?8u@DHr4x+z2l3+dcCBHcWjyvSSUx zD%WzxJS=CH&*i$jf%~Gf{3O=lCmgPKH6Mn|@=5AT&`-Vr6>+TkXkLjgP#@>&?ZHU- zFTMq5$fxtOXe^J%JUQ<_)8(8~pCRYZVEvZz#TbECFb>r)3MG7|q|bbZuhj+B**PAU z%jtcL`{k$5T0Wk;;s=z%r+UZn7(6e3h5xV`3o!=UT|0y)@l*UKMx(m=LyW}^bqSQg zrRwa>TjXcuC$JnptN%f3`8n>08}T*%(7Ti8^XI&W```!pd#=h;xHR^YufgYX5w67L z@iP8IcfBL{F&@nwalJf*55Q76=lyzqTAlsxdCrb;gkIizn(#BY5}Wng^Cq-{fA{To zdDs6|aj<#;*Fqh+3{I18=Ik90a(mo`9eB;P!rYwS=3dy}=Q{8QdVTODvL|L&SSkPQ zT4mnJ74Q`n;}jf<&s=+*^FB96ZYr1OBl$~yl=tU1I6Gzbq~FyOa1sXVSHb0SL)Ws4 zc9xgpDE(4=Dz`)S#J6}V_UdOReONvVl`&o~JL7xuK=~?ufWP5OP*JYMSMg?)kn=+w zM6YroWFK0geoJn|7a@CDO+HTVFRq~O#(B^iP3c0w(v6K zXPFtE=QMLab1e6BUGzhqvu((ClmGYptS>{(+B{GBS?7CvTW*aOa_*TttGVYh4>NnG zah|cV$ZXG!a~p3)?v2ZEiEH@|@*L!QeFF3K9^i|3Cwif;x(Dy!-}o-HmNNqnLGGK% zJR5m#FW}7X+`B8~BAh)gXJ5{ii;?Fx^P(rZ%A+wGuOr__zT=$5xt9v)_2Zv7_v1U9 zGj)+(_K;t26EbTu7kVT2&l1eke};3WG?RZs_NdIj$FTz6>*cx6d_P6bebY+57J2qE ziyz|LL$B)P-p=zfpMT&Jcn*djJ8@I4gkxO$ncHxl`R|cu_6%hI%b8V59_RYId;{OY zb@({W^Kd;X`drT22J!}sa_v0MUYX~$F!D_0K7Sf5GE z>Q&*KDKGGWxJmAa_wgA%&|AQn&!^)>^*Ov0v+xh5qhNVpV z&>aVcgb6M7P6b=on(?;IrTtv zl|R7kn56!c3vu38dh@Nw&hV@L3~az^z2mT3p2FYpdwdTX;bipmYmQ9K^?0Dq9Ed;V z3dr7krMd>HsgK8-@_Md_w(@SCgJ*H7`W()gTbl2{dcFVfk^CjM%72UZBD>HU{StCx zl)+GS-j8!GpD)+c%MG4)gImRM$bRz!=kE&&x^^jgVlO_&>6qyHQ{01}MBc;x;B|V- z@wMCmc_%)ayXa+~`bIuO9?UyXA5GM`39}2lDZhpnv_|sd=zv9Ng+qMiPS*A!XM*B>{1uy#XN$q;VV#8&fhy+ARj2_=axTfeioDQnqGd^d4_UU zXC@cs{P%KB=VzGzZqC3toM&(a3L?)}o`d|%^1M7C=jXA7Gb3)0^E1c{eo@YuoA0v+ z@=Rq`G>}_xo`V*8gZWzCgayd6liAvzGdFWqKBCUgC_jVjTC2D%Zqm!1(}iDAx5HX_ zE!Rae9F5G{!}uB3@?B=$|08F9js7bMJ4EvoB}A_(jefo5-0bnWJ^&%&E+|+1!b9zvNk1g4~;Da?aQ@@VfeMJb^vx zKloa-k(Y61{6-XzcVL#BIW_~+)HkA_{48?j=GnhmOG-goHJ<%>Z!AfpTM2er8qlL=HCGMC|oUH z!?&U;a=u@ILi#@;bE~I%IOpu`$766L4#6X?eacVp$Gim_@sm0;F6U1r+>f)7IaP>n zcI_&@9ytrYQm;qez1DFB6m#tuZpvl3D~e+&dSRP>8LUF~lX?7}e&+e5sE(5AMLd$T zvrdthVm|KG+sa>|xtw>toM%(jIkU3==G=W>o%hBm@-6aZ$S#pHa1bAz8rciW@rmex zFYqXaVkD{spIOO8_*B2PmwW~<uNfxcV`NA8O@A9M!6vNN zAA-m60}A`hYF_QyaNfx`A-iB@J_sM`H^(nH1l{#=hMdQHIqwj)<;QTSdM;-N%RW(4 z{Vf{dbA0LAzx)nf#0To^ICsgX$Q}4{j7A>PkLU(u{~W_FqYkp4_pB zl3Y{%PQD!D)PJFm{1RWqdA}GTuaujhy?hC>|F&0;#~!?eF33ATXKv*)dFOgS_V2#; zz$%<3#d*F9D=S=27=0x_MRrwr#L3Y5Ljpv{(vP<00 z|M>hlD2XLFOYeA|&ab1bd?CilyZI5mnup>*%uxTwIeWLu8!-!+GubP%vrSgt>N9n? zEY6cRVGgFMyWvCRIl2Th^q=6P(JA(O%+{NTv(Z&Ad(hK-fjYDBI_`wb+P>;(D2~(B zFYqGdImn)Rs$3MexR%+O9pFdxjdFI-1w0&?^@nqB*Iq&wInVyJ_+C8>*(r8%Q{0Br zF$CEo@8g`ocXR%^3A5$Pcp~@0V&oY)1HEw{{zY5AzJqHw=Q@VtKxC$0#5o7DV`Y|q zrLKpKe%%HXK{IvEkb5|@_-?(zypEs3Ggyfu^;U6q`gL+{jQ@_zl;|7oslyMmt=F_t6jYajefRLnV0&vbVJ6=Qw9a4f!{;S7-M8 z%7eKD59OJhv-S?WjLf2+xU&8y`~m0Of0@5PQ7qAamLI}G^jBYrBjlc3j(_F3d?y#c zhjMm_ycayEo~Pb}fylWyPwzdxfRDgu$ewfc=kx0Fcuc++ zIkWqtH11O84EdCA#8GwiAp3XrpgQWuU27*dl#j&i>XUJh{0OSbAEP?%SO07*`}V+Z`3+1*A^lT$DQB0eAZK6yoGb8Ge2Qy7@MwOVZ^qAZ_V2&s zuhB@o2~FkfUTgdKRTisFq1gnv= zxgA$W8`lb|7t1+6_g7~JKbNo3%lmZ``A_+2d@CQuOVLLz$dmX7WG_CEAJ!Ym`>R*- z!Rqx~iyLwYz61?iZ^akmZ;Wy6G)$5Y<(=G-vlG;p=gL!%9qnDd6g%{fM)rg37KP-l zxYe~Uk$3qb?B9L+;9zdAcnXKg&AAl{;w5!!d?Y{53;3U8)Iv=Z*B^$T@SXaB^!zun zhde1)!AZCZg?*+XUykx9jc$5l@v2-7b8x46IrrhK&;Wm`uR-2LzvfYTry{%5<(xlI zO?ne&;aPPlyoK!0W%Nd11+p*StA9OSSHH%CxCt7evw9$&lpn+dIeXJZa^7<{tG|)Y zO)cj^$nH5nUZb3VcjOj&T{(NhSpE&Kx%MdEfm`Km$nG*xeUsb+IWwknXSBr{*E0Jv zw{s3(re1|l_3{kuz!o_>Q3H7#Rv_~_^R2Ah0&{%kN8Ex%I9)H#OrFQ=ZY}u?bu*rg zoH=LmWb{Msq0Gw4JQL035u9f;&rvTPj@N(ehF7YU;Y0uA2V?PGQacOzr@9`M1B!x$p_*lWPi!*>!M$hkH!Ld5AsZ9 zFUhWv8S{p^r~Y%CXFj`1=Hf2Y(EAU+$iw(RKAAV59-65OUZ5oaZ<@=p^+s z$jm=V?+e_eUWPU}3cHb+^#q2?e{*-NmH*`nxf*Vkr(zeb!vlJaIOjk=Is5Z??u6q| z4oABF2QqU$#|*uLcsqW<+sHmKSbv(Ff9}S;n6ECL&&Zc@=4)%tJI7icgLknS*%6-b zxeB;lo%84%{si|S?=j8!A=gjB?{X#n1KBsSfBmTUo%%r@#c!d5+{U$$@+wE56-~% z$hnd8XR+Li+wsN7F7mVb0t`jYi64-6hU{Lg_3p>ndarP443IbRN;Jj}e5f}ASIgPW zyU97ja;|4jxLSQIy5In`#b9KA%=^qSKC?tUjwd7g#cf;?uj_ZlwjeP_sGxlW^6%AbQRPnhB^()JV^DC%{Bh_tPt0b?H+w%xM8rR5Q<7Rn(Zj58)TX-SAfGg!caSy7i zUqut_@ArI@HzRw`OT5WvF6Rw+K+f)UZ+a-DcL|P@>+>CGD4)m;@Qu8lKgJyST(02P zXGiWTpQQIXKh0(Iy2ijo%$y<=~AZJtNMfQqk)Ol_<;08Q|!|QL)S8MTFO81ulzP<;VE@yUgkqvJ`PRsA%4M&t`Eg^a(2;j zav5Z9HQ*V1EZ#-t&KtN9EqwMHba#rnC??E*@ zqu&Ubad~IROnX)TH)I}Xu4Pw!PW=xK$L09b=d!=d;v3bYF;jj4d8c|^eYU&`nVmP_ z54kKl$a!zbyvq4>AO4ZwL-x65{0Zt~l52H1d*%83kUD3=Hu(@f28&Tlow>P~H*?O< zA=s-vfU^S~#`AH3-e|lpU(VUxipvMc{dg+hjQdc~wZd4V|D!tRV_~_MdV7y~-I0ICAM#iDOnwt5$mie(IeTIi&W_qZ?_tbWXEzzn z58|zSo)==C+y>+2O32wz*=KL(&-E`uQ8~Ns34AbecDL7GFQ0^oI9Yu%-^FjBo}9g4 zqC6ABF%}#3DupAMkgD7Y<@CSC zFu4zE;a2taI0@sC_sA3Vm&*gWGFs^!q&`%x${qM&l$4)G{s%|VrFy3#JJ-8>i{4+D zif`4ExhF5dzj#Mo1cT8;eLt7R1o>vZGN0oy$o{^8t8(^@)vo1z;Xl3ZD6PJj3u2Ib z39_G_&h7D%{tFQI{T<0JVTC*lRZtP-ak%~wI7>bScj7Je#dr-bsUs*oq^R z)%h_Dz$*1O_z3gVEAb!xR#)y6~>!}awW@CfdM&ib!ng4`9E zSx+MOSN@#2?{kLr<*W4a{O12N|DC0J`SWD9&Xn`B%Fif!N9Iveb>?yI>q>HFQ2w7Y zH`>WLKQbH7)Z2-M@&nk0{2B5y%RRLb`Ce9VGvsHMKU?mr%$dzLuQlD03@w=^*}*Z$UHUKFB>fAK5jtoAlRz8?Wmv<9uiN zuCmi^lV{6$Hgd1`z$jdZ%E`|cFZ zo|R`V=h{R$JN!bHY zQ}O?3v=8tv=l1{OH%&!~24%D(4I!bTT^dT1ifAbrk;CKeGtL zeYUH|1NnGYZK4aHzZkW92PqBa40*4ra@=bUd{c`WaS%lqC@r|=V4=2}}m9`!Ixon3!5 zZ%|K0Y55`^hk@!Obum%k-bbyYeQSg2&X?;C{Il3d)^$7@vt#up0S$h%b2_y14#5U&yWa zAufl_n4r$?ccz@bi^%&yZTVVcAKuFy@in&l>;t%4{(yhQ1o?cFlW)dT!{QV#~R*+`uGt2Q3L<^Y#G;g@)NjU z?>%(K#QZs!ipA>e-?g|pw&|7S8hS(JyXEYP)8z4ZTzxyggEDe8+<hw%}(TwcnTB6Hz&b!O_Vcp5A9J8;g6IdTj6Ut~7q9^5OB=Imn|IJ?6P z?xdglrZ#7Xxj|knzsIlRHTiTNfINS>r@unZi<41PK8SmA=IR@A=G^5dgjISgaig4b zKhIlsuzV+ZCOTsw{`Q%{d>^LBeUKeAXJnq0?1ZoCRdoFTOqbh0d*7M3g}=a0xZm~X zInPXH&T*V`+;3xfCueq_j^6l6uL+mo@6cY(J-3PH;zYDj=bk?h z$05%^X4Uz~?wB(nvoFudXMCB@m*Wr6LmtJ;Ip@T1&V86?w;*0qKgb8*2l--jmS-R{ z<23arcpiB!o8U;-GNTXX%%#k>7W%#UNi@~ljqQVKj*zT3pXS0Gu2U4KEn0v8ktdf zU;32Kga7S2?=r(OS88aiK?h{kwZ=d^qR#oA_omC#+3o*O&z9fk^Z0r$iCS{@qEC2` z`X4?F6Xd!$2^ZlE+@XIAABl72-|&Kb6)(pE+@a1n^%m-3v|bC&?Ci@uxH-Sfw_%KY zD9(_xOP1gj>MEEi&qp))ROB4WdGUu{EA^NB3t!ETVFQlBXL=WN9UO{oI7)9b+9Lb( zA$oJ=xA{$Ee|m|ZMRoo0$h+ubUWQ}Q3qQF2ElSJxbI!`f@`uQtm3cly?uS0O%V&<~ z%>1tMHS!*uDmTL>xrl2!`C4`MniF^e@~$yo{|VfJzfeH`0`89{sD|u{>$xn7;B8F7 zF+N*~f5fA51J1kGpYogXIJA^=)@2`RE05BvjqGtl)OnZa#rNoS;fpX2{csiz!i%m= z$B*(jY?OClGAgTE;Tq)qbC}*w7^!{-JLE;YmW$ydbXPa^*`s+H%HnAKr+5i3<}dgJ zJb-i57111d2U^G{^DD0XCl|wS^0la}zeD|xd=fv#4`Pk{4F8TL^4A!QD=|~=K7JCf zprE>?KPP+cGwQtaWRI95Ux50^yV(s~55J?m-}5+fcGu!gxD#9O0gC#}&%6$K537jV zR zH9p3>>aqA$einIW$h*)8y`Jjt_*lFnXLp{;*}?AC+a_O$|4>MM9rFJ70^hDzgIC}x z`D$cGyjfjUeuDFUnjP>xxfN#@A0YR~44=s}lJ7Ua{~YAc%QIXU1#lPg^Ui&ef2TZO z`RC@}w=z%S{CDM<`d-e@JkLYs-CX_bM|sBctmQt)3>&V_z5503mh(((;yja^Irq_D z@{(M~B)#g$ci$DcC+qT0I0mn}HVuEvxo7jt=3K~qknbQ}X=ftjqnMXYF{DmQUfuC?FroQ;=DDK{6lC`=hD+31=Q=Zj_Vr zJblZVUo-SBmp{T#tW#&6pU-c{=!ZJ$lkf>0@nP?_|fXuyu>deHdayQ(nuD~04G_pgk;LPsq=tsErvivr##FyBKgK;0u zcKtJ6&N)x>p7W{tbM;Ti?7Rpc%5AwZ=X}Y{ekAQeX6iruKDN0217;!T!3!9v-xHbt zf8ug^8E(U+$a~+X`n~b6I`jTGz7u2Rhwz6y20x=L?!iX=^SB3o#F6Tkv0Cnj+8C%V zgih$C&O6{`^0V>|{tvD7Zc*nQ=>~pXFYitR_%JNe>x}p1AJIxK&DqVD%jI#fdI2`z zf2cq?d)9LHkgj^w_yK-S?*)8?O&E@y`ls>;43huF2KhLy==ZJWJJdJvi##38ggmUya|f6sMt~{wOZ$&pC%{s`HL|EI+O8%(wG5Sc&XYH*sTB zbL|rDgmH2u?#R#KOZeZuU8}$F7;eKoXo@RbyO0a}^YX6voBCq)x%douw>XOT=y$_j zT&n(%*YbEY!Cg;V#a(g~dFT30~>b!eYRZm0* zbv~c-Dreep>f9T*AouiSWHx3#<)4{9|43w3{DC}+nGty&Xx~3Cvmp0PJ=BtOU*+CB z8~N|gKeG`k;X<%RQd|jy1>(&A{FVz59`$ z_i`M7pIsl1mymlq_hE54GbZ23-hp&H3Iw<*CTe zayW0q5&D@Qc@FZO=6=Zxd_m5=*$+9N7N7u@A@^7hzL0aDSLXk)0GR{1H#0lCq9Pip z@8)@!nc8P2a9_+qc7fK&j+8m`u>1`7z$QGV9?F?HdA3?0zjAID;XJcDxFf#z+4`LO ztUKqPZ6fEao}zwFo!v0cz`g1`OU;9v=cp*oSO4GhI85DGo#*`?c|P(?uF@Ne^|%0) zu^QQ(>3`di9V^eoLX8(uR(>2S}I34hLQa2q~W=WK5$pC#Ae%+539%80^tI9eK}a z$s_f?;-8S+XDDyRbnJEQCa#M-zd5Jw*QOnsUzAS@IlY?<|4}$lmmW-i>%xeF(SJ`-#h` zGfOw{4>$+eL!agou-mnjd>hZjY&?a_(M!J%vbUV1zFp32%ud!oUMF|J8#ohH^|qor z?p8PB+Bi-whH|)A-OBH;!=u%A@fOZ|-L1GgzW5pVj;#Nvve^( z>i1;de}X%!=ks;^0N#|dhm4coLR(}M)+)h?#VfCujLzfE|s_&?s?pHoS0k4tbG@}7GqZ*jdOf6WtkE&fI?RPvcu^skazA^S{rwPo_dxWl#V zd$YMSj@KK&WAHze#TTgIGpB0h-SHmvK)i#`_1@wkd<)je75PxU8by(J>1*^l^LTak z;ZM}{@B;2b8T{zlt(^CUuKXQ(>(%9L_**^^)#aOU1@cZ-fk*0JiG$>|{3QR1)woZ6 z0T<%Jd(fqnF_}`4Jo==iYmYkKsI{b&-4i5AKDF zFaa;={olQl=P*C>oSE;s))qbVXy5tYK`0=%z-!one#kwY=kZtkt3I4BMb4W_d>kG@ z?uEv@+-GwBY~cLi*>9$b{~_n*dwR{~%4mcF>g=BjSd44Y+i|( zxD}b(c@7S7tqHoxkNQ0ekb5nAZtj^pSM8AdyeDTa&c|W+3Ykw2;(ByM&b0=9-%)Zy z%$A==U)+Msx7qq@9_iJ3Jy&qm--oS>iCI16X0 zD*d<6)x+u7gKl~)xhWU$nVjq2sekA8+y}4AUAPT~U_K@zv#}ML;XR+p zo>oiF%)17q@uxcbQwO;VisDYa!8{P>;trJ4%iMep)8zM&XS)ZN!gaV9*WyzB>^Rvi zGqcL7f5HlUs?PcNGG{hsmQ=-VxdrF_|r@yX1KOceKd+<9lp^!hcnaj&hv%( zA7qaxhs?{dd@mn>*U?0M7P7na;W@|*zTCB+u}FP0|Ap;#h(kk^Q2P z>qp8rqqF=!y`1+e#ZKzHH$Tnu)Ft>XOu=4s)mz4!c`#qgA0X%cuUrfj^-tmZuu-0l zpXIN4E&szW;6n66_Ml=|EcbFf`)@({6Zr=|oeT5rIM=mt>Tl(TP$O1%!U#Eg=Tv?e zr{a2)M|RNc9T)iAG}Oa>`m2$5qr7_@t=AvNBYSUqJTI3*K@9TQYCHtZPy~x{wfk*nZh9I3vAH*wxehRHRtR(&@&zyxHMU9VRkPoO69?$Q$F zUB6G=5I4x<`4H5VvlpEqXLqfyu7K=*Z}C9=tMD9NP=CXB;9dC&WIsEb|I-_V>T=!@ zKIZRml-_ZC879gv^8KjiGyT=S$RDB%PEu!g{9L{Xk71bZv;- zl8-?*IXg_|%NXSR`itkg_BZE@eVKE9xVD( z{^7$pGpiB`A@^imz1GNcJBK^+W}eRZuBM`bx(0GiWVYt4$^Me>Cf|GRmF$-b)mP#q znv4Vfi-@V9(3XRpgV&Uw5QZISb0rrwEMn*Yb!ky*4K4o1$g{5lYo z)R}pi^}oplu}q!&aXmk#&i$D^Cj0Qw{5=M`_F($_G{43tb9V2{nqTG7cpf>U$K><) z4b9O-zX7h1Gv9JnUB}r^Gr!BKH((Ryp|)NDF2i!e={nbQtSkv(`PAE#f3$8zRpVVr?Q>b!$2 z=heut+RLMGu(B^_J`a^YlzSlaCNuLR+=I-XR=hu7$LDYdo`FkI#pgfO%lY~YS3wiG zwrfAiTafc4vu252cE)m?vv!GIS4I6uEwL-j8%ACe>fgNc~tl3y`jHT9)|3nf2lho=jk7MvoKA4FY4kl zbq#(A|6w>T(p!pSklkVse}Yq78_$2@M!5+ei&ycfdM)n4t;mdjS3kSs7jj8dht|Gx z>o_ig$r>9_Le9S0LSBqV)LU^Ua>l)?zE;;l&a)rX9kEf}fzRY7m@40dJ@PG_GyFoli<;`Q&>rR0 z|Duh2BUYk14#lzh%dtQ{4E}Nd|Ns9aWOw~ceHspM?MdVv^I1MrZvub9_oF)ssqf&@ zd>QXT3)E5Hh6m&iaTYFCFT_{!T7DQShs}T#HiZigWRb zYX{*3`7`c>>|E!l&y*YCdi<`wg9~#0o?xQT+h{yC#<>xp%RDG_A>~YzvC(1{;UV*b4XKy}RK1naLAopFK z^PC@oTRF2LXKcQ+*Eut9 zFmlf3p39E4Q~n7(kv%E1?+DkP#z%606vtdl(({phJ3#KKsmOgZnRDNlmfLae+uS41 zV~9HUWA3}N)HfsdMiX`BO&j@SJS=ZQOLSL%&pr4&bdbk!FK)%lk!S8t%!U8$JNuRi ze<3rXk6z|!Eo5H*$Jbzg`8qCu%$Ga|4{}Q$%6)kW=Nwqay)hiS^&23&^S!)7uLLsF z8X>zwVg1avH!uh}KS$_okyqjgxi%NkuPf)7se-Q9tDk4(7R->VpeUNFZ{)v`=cb{$ zCqBSfRK}h9IlD85^E_UqzJPB+=3@o!s+T!5jx+OT;YIXSKg)TBEBfpM>db`w(N^ys zb@r&tm9;odFY~t;&&D;#y!t`!QaLjxGqWS-og-&g_Q#o=U*$P-`#1S|e2ko7!}N-A z6Q8||pHgQwtjCdZBm624!|llYt%uHXcCFs>7@V%oUhynHisE`-<2d;-o`U7_H~3QC zgayd^PG-(g^5d>OkJIHMoEiI%{5wX=dB-XwpN7oxSM}=RL}YKtyHOX{)}fQ$srVha zQE$cNn2s9w4llbjkT2qAkUi@_uEs}j-pfjH_Qykgu9-X^Z_2B;D8p@yJN>suL_#@fx{lxVZSdXXD<0JVD&O2ru{zqMgYpF-dIVabsTjM@;_LxC( zV_b-(dWDhq-Iw@Xy{>X$e5n3So&D(vxgf^lV7>S86)wRpT&ll{`|*DK4iDj3{4;iA z3`*k>TQXRnbJ#qxdxw$HCTwe%WU9Q zyw&x_a`ye~dj;^lUKza`cri}J7TlzFA7|%3n)~2vy%*70&U;{XNc7S{}&Q?$3mpQx4@$%c8ccO~&h1i2v^bY6td?JpO|K^yk$>cByo)E|4s29+Kz8c+ ze30J7n2xK}=kU82DHrsaB$@+0(|#P31%R zJ#0g5{crTLZ=I>m?sytstXGJ~bAR-f^K9k$+khYBRrpdq7*FF~>_&E&oHzT+xks}% zyv+G7@-xl6%Q>8TeWg5G&MbXR&RNtQb>)1oE9CqPGDDjp_tomu{189Po$;ysH|omy zz6x=k&CG&)FGG=YrX+7b&V<6Q<-Y07<#|4D#~NhjeU8$|OuF6mN%9zE9#m3iUiFdN z%Psj$-hv15EZX2RWaj2fnZt9rqTiR@aj`o4-GRIixj#O_6nQ^rIfpNIy)EbY$+MBUbRx3Xov5GlxC?SGHAJ4roC%pjZ{Q1L zM%2UW^8MV62P5}g?$^xw-0RsRvio$@uZ7HrhRA%$?45xt)Op@2^9+27%=P9mUPpF? zoM*Y;tGKomx$pC=AAme3m!XwjX7p&}3|OT8mK)&=?!#>u?D_&eoM+)+c^vP+e7QDu z<12L4%bvH42jB$cY|U)Q%>O{n9K8nlRS`3hooS-$Yvr7QnbQL~&-9C&-C!+GRp*?| zEW4e1s=w#oIWxR0XI_`m`wG*MS(G_7TrQUDoH>~@=4&phJ_uz|5_|PBQ)bFluwKqN z@q>I3mf{a&Px+4zbiF(e;hZIp^Krb0GjC4e$=nOq;xKjQRCV;l*LvAu-{!*VPMCro z>gRAYwyDeEc{%$-=5+qvtES#7dKcgU`DVVKE8qsXINrli>YluYJK#m^o1y-UfAzzm&hjPI)J{#3s2jAB(%>>ht*w+$w*Ft8g-!_~+*g{#7o3%|3G-x8cd$6+g)h_z{_4zsIVV$l2Sb$|dAe`Boeu_vhcZe)>LBlt<|;z)kW-uB6|IyQ}kF z)>s}Rx8qIx3;N3Ua(m9Z+z$Cyxu)Dto`r(y5qMh8K3NAhsXxQd_yC8vb_=dVW!KNv z%PyF`Y(9+}VL z5&1|QAV10jaSl4Dci}9V_U(}qgfHdn$OAb0_(^(~A@6kK^s*lnQ9ma)!`b*#J%W$q zTX4Gk1Ukyq_$bahKt;X^C+KCDzk{>Kw$R%skHnMm$6OGlFcFWTk^U>3dp^%?p8L$J z{Il}>fHBvZtqa%=d}`hFjVh!&b^e`v0Tp2st4xa2jpJ()aSBCe!;J* z^PC^VC6VWII#yF5laHXze>^>*(h@ zyq~vn=1@ichI23HIqrn)n=k6;KFO?EjLh!bGu7om`ag5_yWw0`z3<+ahsd)~7?stT zeaHG-?wQQCI-Gkf^JKi7=RUJ}gljnmGh=ceE|rhtM>uo&JIWI!yN}i21@<>#W8*$EpoO|=tnG+9Uo!(A7EazDpBxhD-Uf-_w zG9QkA{-{L}aL-yMB`q@_|s;|Hn6w`Z&oAA{b zEzjfa*k9hmm*F3*R_E-xUGBKr*f;zJ|JMlJoHKyn{#D03c)idSn5-sGf zIWxVfyg+`9^Xp!@CN4&1;fKhXR8`#t&Eyq)1`p)VQAJ+NbFm*L;ZB^Ue+`67T#|d>GI=xF z$?cJI=tk_qCpZq>^y^_4&R6Fhx-B1%oHaQM+UoViD!sz$&*UTUDh|`jj&cl-$8RW&ob7jW zD`cNO%4Z*w*P**y8f&o>WAu9AV7aKzKE-+Wsn2)m|ARB|1;(MF{$75U55Os?g{ScV zYT!9+zzIH=eft+K%^83X$+fT^-)UXM2jVt4d&}osLY?#U0(psClOI9eOIxF_-Ye>! zJeE7-4>ZR2dW%pG^>Mb|8~hPk`n`GQ{z0#@`dRLSgRlzO6>ICCC}+QWlCM>ld`hM@D+yQ^oqL{N88PEx0nS zmNz1M&JNeFMN#~S0XWsQwYXP)jxR+&`5yio=VAaJ)LW6y<4ydfSAw(87UpUwq&F3_ z@gY9Y%lqGGenR~?uf#QIsXl=x;ysj8AI2y14qPRF%&qtr{3_?2`~~?I~uZ5-?4HPc@#gzmG}@emEXdp@=R3p+3dqN%T4rK;~jjY&KL7M%DTRtC!-SHaj7F0 z#a1j+Uxb_F-}y=&!Oi`?269#YFV^dgrZ`yr5^`?*qb|jn4}Ij^M>&h9^R38zm!Iht z`A5##)={p5!?7Ai=Q?uF73Mrso%wn`gY(_BLI-s-uE?2#`PpVaC@*JjDdp7d@58=$9QJ9TMSc%&DGx!E%hUGb~r1vsr$;a?%ST1L# zW$(=WmHXpHF54`+aOh?w9Ye z5dZqj5d17>rzjvdkOyHA^6X`=?ZJPdq3d6B&Zwv5Zpf^ipC#yI&Ip2KVKH}d?<<4g3HqCGa_V!g7+d67AmIhq-mXLT3n`CTAq zM*qN5kmu+g&e?pXoZU7%VfNaS)m>3Vp2{!!OmiHI%(V|WGb6iw=INct`$V4IL$RN_ zfI72e0rD(gq4%lWn2Ym!T!lB`a=AbMhEwF-$ozUzoq3r#GE)7o`e8Ie_MM!?Utv7X z!bUXFzkoBNvnS=A-OKL;X1v}=q^8qwRjUfaX5y%b_6QRcOmCZ(NA$o1l5H-|C;CpmI_R{Jz6_qE;LvayqL-v>p`2p8f z@u?^x*W=?b0Z-sE{EShqwoaa_({(0R+0Zh{tlua zH^Lz}3OAv>{?25-C;NN$vlHaJ$7G+LD4*o|k$6nL7Wq4f>{;K-Q}uHeufbO2=Xa-G zGcJRh<%zgn&J4{z`zd)SzLN{UAGPnCSt;lKpMuYj|JK}Zxkq1<_m>OF%{brDwY(U) zH_pND*n`ad{OmH{ORHa3=U#eIE-$~#Im>cJWzOYU7$RrBWai~NdlhG@oAWZ{`U`3_ri7hEFuM`mIf zJSgXR&9l$|o$-k}Gv*`YyU%>hb8(Bh7Sh0U>V+Y?Erj@ z?a0}h9eIoVDKc*|kEh8Uk^7)KPSDT2RtI@zGf%(8BJ@PgiXuL@7$2ZCj>b6sa>)HY z34P?;gE>>~lIJ0_b%)*`oOyjXXXpA$??1T_XJ%%<86tN>W@C1uqvTtWc~b*dBJ(HD z!8iPgx*@MYp8M>aljOq4b5#-_BKvvmd`dMG8ue=a9+^KQku&N>9)tq857S)#jvv80 za(0Hg@@)KxrRvP>%6{)_TwE{b{~7$3`fF^*2DH}OfF1IyI7aS@gXCYhFGgXox)RpP zr*h7>7v-zvMZ6I2p_sZ5ev`lA%+1~MaGuGB@e(e@IUA0ZkH8>#InU;*=!~328+jYv zN6vuLedZEgf{GZSF3&IXLHM71GChFI%hT1_!-vbyqa6O$`jVqAdxW4NZK~AIh&y<`Ue1m`xE)`K{CXc7 zF+iR7k^8uVdMYZaS8{gJN<0{O_qm-%`do87ik|8PydTbzFGEvYhwO{9IPVKLx}Lox zJK!$4t^RnPj+}LKcrBj7QrGfMT8hubJiLH$xWe^HuD!s|qX<657W|9sEcg0c6%<6? ztqjpYPP~f&=BNcr3n<-{FVR)#qBO z_vaJTHMxiSX1Ru(b9|z_4!`20*!ACeDChm-M|red7oBjm`X&CKYXj*vJc{%2Eo$p; z2fD`pkj&U8jxKdb{Mj@7$yQSbmawarULR<+*r8 zeGX2MFGVZY-cX+>7eonp0FFoYuOIbpz?15q`9oy4DuJ?c6*R?1XzVlB@K&6Qi`A2P z6;6}S;(ol4mvb#{&0k@qya->*hjZ?!1LS;1IXk=Z7g*}rYupR@esYg>$4~mXmybct z_;dI$5$?--XQcgSaH}-?sr>Fa~)(=DOaP`*IuP z8K16xoO7nXfjlRJ)R|e6@G0^gXK%^bn^{;={U(+m=UN$jj*Bn=`T6GOUKR!9Z8%xZ z9?+9>|1?K__L*A~_!9kF@Q}Qd@8{hA2gmBn@9UAhpqO6H^VRC|*oyP@+G3TQ`Ihq_ z&t>k-?3zuurG93>E;+NPJa{QMf-cIiWc{U%!_aigw6ZHZ)&;QAI7&+q` z>h+fM{C>dWa5J(CHE=Cw!8e?H_d7g|8?jO^v$C7KKicD364%6nP%L<%#???!@=%mpSL#uM0`7geUZ|4@g0?n`(gV7(iVue;YSFZ^!#?|UM_yISl-{b>O&7W6X zeLV7xvQF$M{v=!$)JgJfF|zf6+#Mg1hpc+>*Ov z6yC$_dIdS}U}wtnkTWH_SkC6WXCB1^^)BVNxv5^xrz_Q~)%(dcoYW zSGYctck#V=8Xu~&<9;OHB8&UZ#9QS8ic;#cB9zk{dnKa@f<+=c9U)3~WiN8&hK zgw802UtRkQJ26sy6?S72>iNB!^xu&OAiG@;eqJv-_4}M%WIsL;Z{gA4Gv$ySG5_NA z^!~()I6^D$XBWs7kR5Ajdh$^?4%stWa3MT{i%}IXA@8DjCm-nd6hhv!HgaXo?(&pe z8c+D_9PWnJ@{h6WYvp5c3O>|(0awXoT$>=@ff{%QEA&s|t$Z$aAiLihy;bsFY{7Fl z0cWGWYnAvtJ{%k6QG5=vJ08u`agYAHoc-t#xsQAu=Up}L6=T&u;}@KyUsbOIKdZhL z>*e$KW*m%yXp8Ugq-*{0rThr`$qyp$zIn$xRlW}+ag*!W!-{fcbtk?W*+={F3CMo= z3LoNncFH31i#QDHjc#wVk!oO`)DE>!2d z{f>8Xe*On=&X9xnab#~ekq`Enp158<86B|z?ey-)n{wtt_N?qm*&)u=tIg*j_eRc@ zi+GIwPrRJ7b4=x&@7L(%nfV1T>t$9AlXD+Wz+?DYuK+j3rSg?r3x{KaI=kLyoLO@x z`kkE&sMHwKYf);su%K^JR421L7j6o_xMn3le=*Czy|VI zWVg#K&P*<;9)j^cvz#-t^DJ*sUx|<9Px%QqFGv<8n07+kwp6oL8UAmm<$n zUA~QbBWM2_b#r8PU0U3xcThx`o}^0`&~r}{FqM9!}> zk@MyXbio%m8*?zlXR-qpmoJg`aB)5jIR|FoKDjSG*k(64?j?eIq`T=B@FRWfK-;R~Y{BOYj>Mup!aqm)JEpO!B z{02Y3yU_xdVU6B=UXAQ;E7V=(leh`S%XcH^{m-1Uu&e%Rl#@TzTPNSj)o>kdQFq2n zypKtG>rp{&$rJb^>_T_-@jMWDcWb169dDtuUf%PwpWUH;Rh@G%JHg-Tdfbp7Mpb0z z?Zi)G1&(w*yUTESt$Z==#q*em`gq*68@U|bk&nkAD1~S7Bwj@JnQ1;VS)PdtkGkcexopKz6*cu1}J)<75ZjEDyj>$oo@vxufMixEYmw{$PFz{qc}` z2%p5G(O53e7vo6y?fZ7l!rTeRYwV|A6o05E@?0D)&%?W@fg-pTBQOftLE8A-{qiRm zhV!uvL-q5HcouI&cAo4O+4-^;jq{o9I0pq?AIY0IJIz3@fogikp(y&Pi}5tRf@`8K zZomt84Bugd>)ChC;JqlMw;Pr8K2$G}&&FNyPW&q82l5NOjkS6gaew}tPvjT*SllDG zxxV@;j6y;66MQIcl1HZJb7ka5<@<07odM2BXD@RgGb=MN z^ZY2i>39u!rmO4kln>%{yodALPLVh8C+IJ4K`G?S%DvY^K97&bM{=I8JeSAdCESY4 zuguR^eC9^(i5ulU$n%+*cpS#-=RTh&SLV#qLFlGF3HQqWvHRxc!Ri)@%$pD8%t#d9n=)(HP(8<+;tQUCck~_2quN3TMf~(OX``A92o`JjXfHe&9SK zPwHKO%$~i-S(Nj!DgKZz2#?__FcZDhS7VucDW1k| z^@GR^xrrC(Wq*BL{!0FrA3~nx0XPDq)uWJ|IrDnHUPWxdY;<+4Id?%zIs49=^5ybq zuE0b1Ej|u&a5p|i&e})$8rO5)7LzA&=GUjle654-XzNf0`r+!+U{p99+hP$ZG zL3YYt^}dv|8-K!?o!Q?qdry(?z@c*H>q_kB+9a;V8~HTc=lVOGb1-MtE_oZ4pf@_{ zJ;LwvU^Ku4E?P3ngHBCeOOMb7Y9+=~z3`|+;)0Gi9$8Ls1; zTirSDY4x}~R_cx9zqq>o(eec3S0{X?Ka)?ykH{{N_qLbypTQ-_zIi*>!xcWW6gfN2 zQ16E?aI@ZXC@QbwyrbR2->a*kfO;sG)_at%;otd7>_v8|&U#nkWA*>g5gWw+P}cRT zD5>`#s_ET__3{J!0=D7=^(b`4XX@K{BXa%~;kVFOewIH#Uz9;3EY$x1dyt)Zyxvwc zm-FuVn%vH{m+>10pow1IOKy>u$cJ(>?#5O4G1PaxKmUVcQQfud9#w*Tq25(on%}@Q zIq!ybc?Xu`PW`X>DU8E=*rAtw?hyHHe50@d;&M+5H?9$oU8>$aO{{Ew_epOtmei!+Bk&D!=<)3&2pNZ`Kr}C?q znCn;}*W&zL%v!z@h4l`?C8&?udgJgnvRCH4q!6a6uf^v+lf!8l)Mq)xGf*vcqJi&XqU0R*G{TcFrkgqc4%xYu z$$ORak-3rQcsb_`n8ORvOE2>$Khw0QuQxc4a@7!=PYC{p3J$wPC@Rw+>@DKFS=F?xi9nVj>A~}^YA*}Q0ELTE@wZ<^Yt{c7yiq| zkZ1jTpF16!k$HW&UOWDSU&gn{Oniw8@GyKN=h^Jezw!aR3V+M{adyt^n%}FNaPI9Q z`k7NL<(zX3cs2gQTdp0>*_nUfHR}6$4`*j@%sJCCNBZlJ=Fd21Oy<&RWH+sd;d(j0 zGuvL3^N!GlSNcphRF<#dMm&i(@C*!)SMl?>OkTuaAu}$scZZxapd{xR{*DjAR4hSr zUwm=fkj5?twGp@w|vL>mK9l)Y)BT%6;*$I_E$) z&a6K*xSsj;6EZ(C;tbBaN;|#EsDdw0R6o0QC2p=B#maqibG|zBurRV~W+yySuQW#My^Z?vY%G^o z@FVCX=PVi{=bWg9IzewQ=RKy7&wPqA)dwN7`B8OIZiZv3ya_2tN!l{4}jUZV7b zIp^x1u3yYw@B^qR55r>l5Hygl;5jHG=S=>cZ^b~|qJJ_zL=ilQJJHm&Z;<`zZ1%r> z?*RRUtvCtU8UEz4uHA&<>e4(>ubF(I+(GWn8LFx9@SiX3==UyeJ+l}(driX2vhKi-f7r^k;p!gJ^EeO zYVg-Q60J}`-Pvcplw0z(+!b%i&tZi8A6LXna!uTd`e>o|9-q&5pt3*fUiByPFWiJX zp^IG2wHtV`x;2-b86&2Fo9ypnNHwz$3UxuOp@+ zJJqS2zjt_tpL0F$iRKDd*q$ z05tJ?{cqpDSN=VBpt%0`oS$Xxlg!)ka{e9nxVD|kaqfi~ock|7@66KoIM2%nJc&m! zQa@)`?&&6S?z7CMx8*-L^CUm_X2{QE5{By!=ggCZ@=(sanI-4H{UZ4sz6*KQ^K9RY z{7j~C4bJzS?>P5&etv(UB{D0Y;ir-NFXz_fyhfexr5-ZpvIl&}f9Pk%%;3#vEcfR; zTd(s^m?J-oJnyd|v*L4g&ec4#*?qD@@_>|I?4xe?(gNuJ-Z&+mvh$s55s6CGSent4LYgw z4CK5kB{x82b!KDspX{A|)Q9N5%10p2$0p8vIROQb=ivXI>5E)zqrQ`G<-ajRF2Y|R zGd|B#=6dFdxNmQMO_(O<9KMRDA~S0#Zj>`KGn+DFpGFrf(LbEC=arBv$#Z!#58>Jv zh$8BW$S(SjdJy*DUA;HZ9WP>`-Zj{Oe(G`j2p7c)yp1#TY9Y^mX6Z8Si&1h9&P-p8 z>@0buS4Fe z4%QnXZ{@c53+JhS!{w-n+fW}j;aHr77kut(9*n%3_2tPpUjJ@BojdRxEI?-Z{b-2B z*n-UB$GI40;~DkKm@hwzL-8ou;Y$3B=Ux8^b-XTiN7lrV`WR>|-c#LYw&7v?j^}ZkYx8hD zrlXMF(Wr=F>I-o$vdf*Xw^Lq(>GFp>2ph2*=jeTgx$=EjE?%)Me2W$J?|-X zVY#6EGC!uy+1s8c;xxRfpZDt*xq z>xo(T6&v&i<4yT-E{hY<1RvpN9Oqg+)R8a6PWfc6jel{E`UjrJ*^}DIU*RrwVRdD$ zfP#A2H?QOQsDgKJb*}5LlCx_LP+x+Y>Zbe#n#^HE(}r!aMJ&WX>=wo3eP$4B3~a#`EbIWgySod!3-j3B-QC^Y@n7?;^}Eis z?t5lGAm@Gdo|%mY&?D#(G%;TjfffauTSk#D0S#do?1Ro=ccR%7s2uoQ`2J{p z;^ydI;&LzsauVAe!G5k(#P;>JKketcM{Y4{=gHok?eS&AbHRFoeLg3^J_p+y{`k$T z+52k=*6$vZ+k-EN+B;5xuL3XeZ^1rO`~R>#VrPCKxsj-y8QZ^^pdPXPtarga5BvPH zlegYs=hgOUBS=CnIeGwH3f8x+r`x->J#Oc>KI~_0Il2$6gW8$0yY2?bukd!x?A=X9ZBO+X&*3#b936{J zMQ4NU9jlSndyZQkp27ym4Clc1rqx(Ge^yskpf)pA2dnLKQJen?qt>rp!b!Z|F*a7Tw0B?x?r=i_4Oje@rdnQuf!L`CxRyUFR0b>5~%fT>y2IUN5PZW>ajoG zfsVw-Q0wPK(Ee}?{~Fri)1tG`Eie;)zz6vMS=i2gVdCECCG;zN$1j4zPz#QO&6rVW z6^P=QpJ)fvYH=snj^7P7%bXz&$Nz>{e0;PhB*&LUFN5`w=U}s-2h;}Z8^6dk#+L-E zQCXl7SY7#pK7gNi>xmsvtE~c7E83!ga1ZVh|3L?%0qAPfYQS*xD};j8Mw?0U!!BU? zfAjKM%xpLS707Kvt>0T+sS7rDSY7FkzYnQduZr5NVl(O^;)%r8*Vd!fa|@tWcPf*! zzS|w$MEnD+p1&oo00)Wx!49}ed>ieHeutf4HQMe8mr*z3c~Fu#A3C4hC#Xz(9eqzO z25o`1g1yiLPJ#7un{PhiC!=}MU~*&dx$uJ_o8@5%oF{IMmgkvtc)Ls3UB~KZY4StK z<%er<2)>ZBS@S=97qEFh8vRAS82SVzLSo`nVDr{MG?H8(`VDOc^C5t>0%&~l!yqFh z1M3&3%(Lc)FOLty7eco}Hhc;6G&&Sbj9Sn7fWHY#h_e$X!uvuR;+<%J@Ph`#i&2~N z65^5n-Oa2I*?q4c+6gkkBi0t8CBYZ0KbAo~!2Ub|>)q9eeel-TZ5~aJ76 za-cRN&%i&!|A!_8>(OB_h}>_~2VDW>p*4&mHyr){p80~fBC*{aY`!grw>e-0`8@C! zGLWj$fOsR=oNcqS-ECG8FJR5)jGE{jNJwrC`WX#| zA^2!?4%q#?9NHTE$WH*9JJ%6UgH^-{AU?UQXjXE$(MrTNSF|LqL>vKv9}oNSc1K8n z&j)jeAHrU|%{dv+evqBqFzAeLjP^p)z*)!z(;*J|QV zd}fRBHs{qK?@oLI{r}(T4IzF5y~x!-i^DB^5}1TvigrOequtO!um)_7-9;`IY_qRM z7hogcF}@hO9a2IwXa_sUw}*{TmbeWB@wFB0d*L60owGp@19rx(w%b0lb8CIY`p9tz zz}tDO$-2z~_nbH zO9fV!E~8d2qVRoDtBd8p&fZ62dk0>4!B0VLugrppkO(}<*?hJPKNkiQ??LOLzrgy? zU9es`8qTp6A6|gX5i7_ohTmZI!s?0b)ynV+Z+$KWelS!ZP5`IiCRh)sjwS)?e@)SU zP{f{vASevB&z?gc@Pzkp1`dGD2X@~3@O9SjtrrFpw*xn@dRU*eRjBnbt1q^{c0+CQ zRs*)-uj4gNKyo%0#QrP0agN$|j1&ntzu z+Ex(%2X&x1xjpadZ58{mLHRnHF;=5Ppd}QAvbGLwU<>gLSO(=F2RZ9sJ@9p47z~FZa0{#k4(A!0qg$bA zp#VNFYumu;dq;9F(Uzzm#Ne$5UBg$x`@?MbM{NBuK7I~96PgNk;zLmfB0!1lUQiZ4 z310)9hju~}pf=APL>CfUzqeYw6g@#a6m5@gMFSuq{s&qVdVt+4#*iC`--}wE{)YdA zo-9 zCZ5ZU9|{F6NBjsvk+-;e#AZD3;qO}1MQBkMeW|&4b2Em$axbtN2d`Nhtc?QumQ4x&CmzQ`+)uV zKyEv%friA5(RrxN0CpdWL(G5wyMG+Q6ofLw&0sA)7K*_ruo=$gK zC$W7l@!=t{z5gK8J}28#_V-xr`G$5u?WI}0w%)Xo(B7BL7`CTVqlr+fX;#1Nv#@<< zb;90>oxAN|wR-}wotu`Z)i8S>R_A>1_W9dci-*sETHmprX`e+4VtY4v(HH1N)b^&G z&$oCxXWvk(U$*yc4;@79+}N3}jkhym`^G-ID-esHjs6Gro@T=}XbtvWZSPKml`xsu z&f;DOz*hi!ug{<|zCCK^!)l53?P6k#Z20=NnKGKn#?NRFiGx2t2GlT8#_^6!$I}bbHDc)*)O$fwW&)bGt z{j}ODXiHQf3Eq13QM~QhrNl$gGjI^!3%w0iXYB0S-YrEw7M2rlM(aQXSe*|eXSL1h zyv+mC@eZ6P-xI7iSRGgb<%#V)*jYOSFX1-ygu7t-y+3*j_Q3zw89Q@D!0O{+^aW%B zt2vj+S+6Ms8^LO`)r@G=YQS&QYTpA$kGI+7F#3Vm`bP{DA-1|d7px|oB3Be)G+^@YB)eXhpOXMB(S5_tCUqebs7o2n-=#h1_CvKjgu; zfiw_EZ1Z4S{1zBUoEt3-`(QU%&#H;qJ#aAVb72dlg6HH9z(;&G)aJxw_>5>H)B$gJ zP5cQ2?*~s|3REX&cP5+9(-Y^08RV=_je`4N_oWf!M#3N{4*4Jv`JJo}!+XI<{3qCi z|AVfB$IyZJCwc)b1NHEapcQ@*%z}n+h};rbi?0CZ!S27+$Yq0##M98D=v}k`B*3>v z+rkET4#~lT{4wyw=Y)6oK)zQyd@#Nwx)1CQSsiUhZaq2;JqZcO+nk(^+)d&xupFv^ z-6h76-v*hp*i3xe!Jy)rXT(~J_!DS2XQ3&5ffX-z)=nd`_^wZ|B?IS5lZwu0IsQ+uv`s+|I7`hK=N{ zt}Mb^&$s<<|9@7)t$vh7t!^GhZO>YNwmog{)%K8m_O{pU{SPN+XTtVdLNpoJ>|p!f z`n1&s`>bq_IItc64y-mO1FPea1py$Z4Pa;5>XW_qR1ie`6nzG^&+ZXhp9#a;d$RiW0&ji$C)jzh{cg3z z&Ysn<$*>Y%1oc5}26>CNC$|1*Gr|bs1o%~8`}#2P0(2YL+*%ZkPtIyz3p6!qd;JvN z&dvzp1E}?{S@<)k?ax~HH822v5-$X+u~rW*Wg#m_(+ft{fO=qqy8?`@C9fc3ML zsO|L-)au44u)0tcY!+#S`lELKt!CE&o4Kq8*erek-N(AshW)SuhQSB0a}b6WhU{Ry zVhfrAwKIGVwZ5|rTEc5$n+vR_+KkYK*v|M`G%s;3)M~2rq}(u`*!uci{BE$heG9oB zc$;@Sp;k9+X1<8O%vvfm4{H6@1MN)ghX%lLu)6#N-3)Erl_eITkzjWW ztI4lmAMsqY7n&J;h&BYP$5w0n@%bT^wZ3RMSPDtuD%gDX7@Fa8z$usyFUT!JH$e@2 z1-K5Gh|fb5SYMiiwgsE953(MLw>mZwpAEGdJ_P>*6rT;Xx<4DWndu6-J@}JQ5&r{H zkPjrDj{gPU@MEAFG$$?q;rM9O8#2Qgu$o^EzTwM))w?ys8Q>Q5A^rm=@E5@5qCLbV zU_S&xMRMO!H`ZsO?d>`AAsQF`4b>oqxFF9?z<7V(H&s5y(4jB^cKv=zX9v# ziY7thf!!B+qAkJd^c2>9!5!iTFb7&g6-W-jta*d=fG@-W_|H%emckS056z%FM8JLc zk7w7z35X9R$?Zmuqeszh=rpK+x8833sU29qtPSbmEb(nfi?`lX73~Gq>obzi3opU$ zQW?n~!du_5JIEno`(wS%?g(e`Ss^jZ1iO#>@VwnePNUPwT|`@>mC*!fM>G?96Sdj$ zJpLKj{Vxkz1Z;-?gFfP!)ldqq5O;u5P@eT+U{G=dc5 zBG9p@%|$El;V_Z-2sDAw#Mf*ca^wG^lc5bfg(NVLyv^3v(L%799P+<=&pfm=6oai$ z7FI!19;gYykO9&_dcH>w)EB(*bJ4*(y8!GPR)U?d5I(^m(l(pe{p2pO&3yIAb%&SG z8;X(N1rP9EkR2Zj>F~ZV9@Y_GfV9w)cmw($>V|fOGWZ-&5TBI4BO&A`?ngWceFR0x zH6gx@zl7SXbq!5Pd=C8pLo8362U0^N=tj=&8(;CEuofyqJaVI0yMw=vMxvk5iBJ)r z2pU5)e1c=-FQHw)?w&J`~`o5jxgD@Vf#@KsH3M0WjqiDQ^ZSQG2UwOOuX&;SzxoC&5wQYR*y!o_5*LV!}gZd zE33n{=NsT#g4OV$V6~t=Sd9t;t3y`z_v7sx+ZnO?FaovuYBR0P&UW_geAzrO4Sfn1 z!R9(UYgRk|p;pIirm;G0=Xwj+xv(12kTt85QK;4C-QvwE2Xb%QNn=kYbTldl0) z@O63C&Uaqo;l%$?JKsMbJ>CPXF3%^nbDABsGg%sK4;{hk?Mc*nr5_rI7KL_Df%qWU znLkOq57y$(pcUZ?R3mPTT2HV(SP1_Lg7H=ptxncK2f=!XB3=Tq_-C*KzY~J+o8U11 z1{?vKbIy<(X&!8@x=4Hz?A~BCJDS`gcn1D(3#^Cwpf;;?LWjXuSj5*|g);c5@CH5; zPe%)*R<~B6sfo{`{b4bzCH8_;@Rm3^l!wa1*5h1gHmJbYPbJQczYkWMXP_6r=D*Eo z71%?51I)%3f!1I>Z#lUJ@B!iyhr?WW2Q^_o`G#mBc!^KI_pAiRh-(n{L+_ynz`+kg zo1%A7n~B?^R>xaHPZ-3S&5F+MjNQxL5ZnE?7J3`j!Bc1sHal+P*(cBd-v<2%FIWqN z0GLnQ5B(1H;S})$$cq0Bm%;86qtI~Z0CicfgqDQeP>%Qp8h~B~o0WPK+nv55`iER^ zxCtJ_o^TzWfX$<;(dsahb(=ZIpw{P-;V0sY<9niZZ?}2+J^mATlRpm`@yVbZp5_1D zF+QQg$)!hYqczdxXe4xnfy7@RKYkk84NBot!Zmmc<@jbvP@4%dqx|>3d$-MJZSc9# zTxekyT&Nd>g3X@k(9-0Op^qRKAA=5oDd0zZ01`t7XaqaSXMk1sS?DCNIVy;FEc796 zg$9zdnI;SV3c01|A2bBrh`vVcUOXB<4&MZw0XAzaBd!j&i1VO6( z)b48E@#FBF;Szow41sje2YNs)ILCSd^doA&tC^0z2fIsWB_AKHN?Zb84K_kM;Z01Qqhu>+Icaffit&v7OOjs3&Um>NePUC_o$^wa|1@Y z8uc3{KzZT?=oOgEbGC1+&h5ZkjroPzH?X~Id+<3tAbtk6Cpw|F@2u|Hx%mlI$Yn+C zEL$%)4Bv>Y7g;^AelZYld%rf`-lf&K6Y!IKci06Dh?m1iyw%6Q_?BR`YA>uNZ|BB( zkju5Yh5&LSz~%+3$AeI-dq<%i@ocm&x&*a-XY$2IElCZV11w` z{Da>RMt(OM3q|pEzV^aHh$0sc&4pG%?VM)=yBAo^wmzGR*lNrR^oOnS+!xf&^H;pp z(~NM@Jlq32yFsiwcn??!*U2S@JoqPIeaO!IQ~Vs%>O~-aF8&?dfEUD8uY6IPeb=K_ zlkbvyWqEQHi4}$u+j*`4R&00y+8yewDI5%1x-Hc{~CirvE1aCd%0)7+T6ZHnWque3hhku8*h3xQ$ zI3;w(+su#{oeC=;HGBr^IbC?BDYSu`(3AWKC<}|A0$hdctogxlC{AqsZYJLP_-^79 z5RQ+8PtY7fAt!m8XNKbY!Xe_;&<&r1yv;!E@$29&egZlPoe5j;`JtmhY`wZ1-WR$O z*FlS*xzY4c4RR3MO!OaqD83w6-~UMb5x)wSf%W4J=nt@-=)-ziI0J{^2081c<)9co znA}=ehhK&cK@~lN+ANS3{RhWjHTgv3_M?f>3*-{Qe^3~Xkhk947CjBq$^C+Ua2~Fb zvt~96{{|WT1I3@8sG#YIMb|>qBCV+zEzrsW)X5SBr;NP=mb7vm>Hkbr`?0L8ZP}@0MA*sI<^M2`tTaHJ#1%gFL;u(^YIoA5Zn8n zfwyyM@1g_VK0`Z$RxfVBWb)ZjtEDH>TWb!C-sY2dxe^6I?+{g9kKX&GuOW z)Xr8`bUK>XPj%8yTqKZtGvn`eIz+x_SwvDIINT@VH%;XBmknJwVpw?bE_1Gb0a zpb5d|umL|GtVh|bJ{z@qWoK>;bigM@2cmW^exTE! zA^d_OtXVDEgSVPzb*3Y6HZ&bt4s51~Pi*z$3Gr_@hqv1D5Ufs}C)W<&9z6zW@K(QV z=CNA+2>%+aUgbw0!$15K)aDVZ-MP?V#8%&45`RG(q3_X8=svUnT*X_BzKRck;$U;@ z1oA1s<_4?VRs&|ER__kN3h*R10^IOE5D7MiS=|_fw>o_kvh&O%7>qBATHOu7Uw|aU zb5ZLvOYtorH4GuP`_+4V8L+yQ%<@nNZ~eFv`U2d+=DkI%1;Qq{3_;}DvUUS+HTX1G z&7BH1hg%=|2{w;*N9{gy7Y&CPo@oaSpc%0j`VZ#8BB%oW$y-hJMQxURgJuLj2nCx9 zsz6R?4cB?rHgq=R8*wW%Et(Z><86Mox#1S^X{Z5}$yY>_Rgqw$r{5pW!z3+Ce=pm|a2Ep^dzP??nU{DczZ z?CxOwtQh_n zsLk68&g*YAbf*|5c z=l~c44sl}Ce%G)L{|w5*WT-{{Cj7#Gf(!VsXaLWy#@E89gdF(aV6#I6@e}++^ek$# zT4A&}M4A#;1e+!E62IiR+^7fI7R`)Sg!Xv5Cv-wH5Zk<(6mR!TH*`EXZ}czv8MPT} zEZU6t1X>E7;uoMB(Jl}Sxu6%+faY+X^-1UlG#YiG8PWLA9N!rp;s>Ey(O~`_`(4U6 z{48=g(f-f{e+`Q@4S-dkvCi!8!Cu z(P~;Pu4tu$x%JxG@iJ<`5sCjO)QML%P|qb#qPg9AY1GHZ;?=l-|GA1UT~C6mYIuXT z+P2^ZSvx1Qj#(M*3P}{AH7b0RjwOAiRgO2Z_h@D*@w2k-uM?%c$GwsH3EIoF|Gf3u zx=>AZ(py`U&Z@OzJL{AU{<=MDS4}cui!<|kGNk&mX@Z@qi z7dTl?`91UR)F`{2ys}jmM0S+=OY3TyH?8zmz)fj4d#jYq99MgI=F(DS$IHs5ZDi8c zo09K#ejRpnpbj}4=(s+n(I30xs#EBYj9I^5;$7G$$ZzOSkB>#P z;IoBNy4WO%EFYzAMUOan3uMut$L%z>v1*ZFpPiFGddb@Fs%brRSpx`e#)Q4eLKeDn@#0 z)rrMi{i5n?#@Y|%N0CDMxXNWo^tOgJ>D*Hnc)XJRNhipe?w(ry%n;c;aE{|Uc8^oL zU0*FUG^^aWm`U=rDJYdHJ&^)~lBnO&8aiv!0L@arw9Xm6NwPH0BLP){wN;ZZPSw`c zB>Vm+uD+jq_4C8_dUxYPnNsV#q?@r^manR<{RaJ%e#ctN(ugWL^Fw#ZHa%J{y{)DX zTNTqzK7}+_{I=Tsa$0@YFia;+=%o2uW!H-PZo59VFQt+siCiGRVq_|9dMc&XdgiO$ z4;0YFZeg0RV5GLL(^{7n3)Y7Jy6H!Mk#Dn(i2vNvax~dv85)#9eRjN%nB=MhJ>RZg5A_0{FU)imz@ zlA8OOr`EjE!RguNqtmF7OX~!c%Kp zc-Gyycp{UW+%sMN>`Sc2SA23#dk@efKG$TY_X+38_0CShS3@M(&6HYW<{{a8wuk;q zxlnRkye8xSHPuvJJ!N0F=i*tix7LmBs!t=@>#K5ZI&@EOb+6;0KMICucvv%a+wee= zcIv4q?yeP&A4_HG!e6rT(;C;eEL&vNsqxOFcwyQs&nFr4y1cv&S>qH4{VuH&RMfcZ zW;pjxIr{9$GAG%}^4g=?R*C+XRO61#FJ)$=(re3qNRMi7of^*`%ZkboTIuZx349o= zgHkru+#%cLbcRy;EUUZb%Nn7D&Te)(e~T+)vN`OP+|J)`Dmk|P5O>(|5=CVw>WJ1L{PKs~$dfgf4RGr;S8y7317ZR?N z8Wk?c!k_sxQL@!i_f~G%y?B|-^_(Q*Gv1O$!^>!kl{uZei$iom`Jb+9uPf_qr;r|A z?(5Wdb&+F((x`XNAI^FYA3cyFu>^WW>hbg)oo;EvWKZr6+N^n~PP>#upVaQF^J51~ z5WQpXn5;7My0_$=pIG-!ORd3Snp zaGus(>Uj2!bybhaFALrk)vMiVtMBq#k}G#vO%m2#N3HYKnA@JZba@47_aw1yT#;Yv z>}jMyuWIP8cX?&qf7P_`Es%y4( z=OyX%%C6^us#Uj@*Zun@OS{ie`Z@6n$+Ne;Mr1rC`zz0rEOq|Kzlat(Z-0I5d@8l$ zULitbXZGR!7uD|b1NCaYC?`qjfjaDKEAf4CN*b*5(;QU?$e)Q5}IN)GiYm>-rO-LkjPd+>eTC=ePazh-U?-se63Ad+oZ6?O#QQ*WMu4e`L}+&Ex3U zQ)8t5k{r@6QG88MC|a)^z9~nZCD!m~FJ#G?%sRJ?YWx!;C8F~lSC6n3`s&m`c~&!4 z`glEbd;*8M9-WS-lco;PB{ga8tG%?Z6RlYbMQXUOzj_Bg zb6W0CsMpudbfOxyRnOKTy2+=w({4#2O_sQ^jw{JQwx=V`mh^Jq2E_A%s)e+yzWwmC_ zo*KR@KzpVvqzQL6k**snOS4paU5n~pk=DDW%aZN`^+V7o$+k91?)Z4=;7ZY&#Ph9a zp1zvk&^k#``-7BwT0thf8Y}aXhHF5BLvnLkPAAote!6nuC0X5nymb6`R7P(tsoB0Y z(UC9X>!s9x;a{c07@vibgLM~*HPO}o}ivqJ9x1P`C;Fyv+D_&J6aml&Pj-q>< z+)uk{9ap3-tLmxk%7y8!bxoz_SC?ilc1x<>O0N66=GL2!;_AfS)%4w-3G$#}Ux{w) z;dsYha*8Ktr!Us0&?dpDG}GCHT47sC89Te0&T!l4+VXO&{H)hoS9JI!fgT?Ec>HIH z7@J+I&+eylvGKG|f*}%l=&Ng)WY$V${PgYDRC;qy zGF{#3kW@&x%#~}>B#F3NTp!Ger-yFel6@E4HO>B`((sq+{kI=o9`pO^i>d>(Pw6YN zeN~u#O<6(99B8czw!z>r9F?IzgPjANZ%VuwZ>4<7h7#{x zI{m_o()?F8z1%*N_NfvnQLQG*Q12B|d4x+g_r5EG*45PD(8*5itqpX|i&1jmZW?`X zW2vlqT}LNJ4swq9=g{tZCdtG!&GbY_GRJM#Kppxsm3~VWBYrFLI0;q`&_89zO1H=; ztu!rMQxvPNquP|zMgvRf=H&lL`vR)5m)=VMfGjfn%L)0ntg2Kk<0Av}CeVU)#>%## z)igRyCaE;1t8Vg+)QuH=G;Q1d8n4xMkPxr+j13gCg*t}vB}%((7LO|BjpfzI{Bp>dsAJ#6Hk@&?hmEe{YKj0 zYFWJ=n@-mR2FidjzFMO8cqjSWDN?s-77YmBDY-ZojhjA_VZP<{c+(cL*X_2v{k}my zO>U=~UzO9_$Fl0kCGqu6h9a7#R!RLjq_$?-mqcscnJSyUpK!Wtj@0X!!gYPCc)Bl5 zM=j@9NxgCx(ys+8YD>4?daLMAY1FN>o>&~Ll8`!{t7@jLzA1PI9eOOv3gvbY@r00iCsX@r_bp#vs|dvz?aP-obGS)RI>T zF3ZV`v*p_F5*jD#P&t~TnKsy+U*~kmB85Ud)HNqq{-pF%zj^62;jwJm^hyRj*!P!w zPqJI4ME2H=Qyc5NO5OGLnwj!3p_d+78lff9wpO?K6}4APdHs=OfIfe4&A)$6)u!nW z%8$Ai<$+r|{{`7R^`GAvIlrsFy6;J%2_{w6M|BT7X?C2H5%ZdA+YIv@uk3fkGixhd zasGjPpYmIFX89=@_Gi+(PqsM==11s)jehL+5dC-HnuLX%lg#doG(m;_I@0I7OgSq$ zLZ&&JJ@aYfHa}(R>ICZkvxx5Q+((OqN6FA;L7Hr226^1rOWQ5&BinoR)VM+2^!}9O zx_WRHU9jS!Q{JP8R;e*Tw(p-KIWwG-C#~bCtJDIyk$1mL{eD`uF{=&kQe4*ib#xNt znFEB&r^*E897Ir+Dban>fNuD*RT>tHP^Q5(NV_2Al?!gsf1s9j3$ z<=rHe(#4AJ#6o>dZ8&Rafp!_i=omabKFyP2EmEk&$UOO1a-XbkJij@qOA;PaoRn ztjyu38!B~lW{fB#KJE+U&6+ye+=-+6`=!&rBuzfLz10cL;JCeUXYXaxh&aFH^&&@q zhg^_|BP}&P^WWuO8Kh~s92&n!dtJUNSRU`pqz?~umkNzSo#tb>PbSZ$i~knVR_)xh z=yp#Tx+vP^a`n}5M+$3=2sbCu%dYyS?;BaN@TPQ6epiO)57N)+8cUiHZrZ>Y&&-GATBhB@dTx1 z^*)!DJ$%M_bgjL{QpIxsl9bUu4by!Y3Y$<+($qb60wr(AL!JS{@q{*>1?=~HSekE(jB>SQUoZMKuI z=W%(PsF5a2-CJv&imR!*CeRAcg4Dn1H7999Z=KS=qHb7oPZG~9C}X|iXp=G@<-(pF zaw^U==k8BG$e&7%YT zyUQo;)W@e)*OiZUOPZh>PTCP4T?0xK)RmRrNsacYq{N#ldVX`LHfR&Bvi5=O;rumy zQAn2ldn_LgiC(K*%{A|Opd24QP<;+2))||!=n&7sveSKz6FoSS{<$zZ2eUd3LAEe&V3G#GWLP`F>UuQ4RA;}i?m1`{v%9qh$ zn)_lpJw#0ji#aI;HWt$gD>CW$g@vWg-p)FDAHQ?)t*!Uscxd}Oza;;wg!-=3UD>&9 zhBS!qmnr&Jb|w|Q*KV{kZRIP8Zr)zquO^f^Db728{EkV|BROQ)I(I#Oj?XL_dSy(SW zuk3uy(oC|Ztf|-LMrq{33_7#xFUis(t5i+7z*&BaJ4V+nQt@D0{k$cWzzH#->2Opi=-Osh|*f6JMfju(&dw>iI>ZiwNx-?mu#`-3EEgk)=g;TEQ z6sdHmxlUVMOy`&UD4TL-m#za|I>U=)*ZzOv>FBYUb@Yb3@{xXis;rc$Qbr3EarD-X^jiB*Lf=HtbEM1m7?@lI!AxJt*d97RMCf7+iT{4tgb-sY8P}OugJYu~oNvRMW_~@?Wn>bzu zPns+vPR7WKkI{OZyH=8oIqCb!^hApBvbBGVjJc9jyWi?6tLJ3XPrE*ee_S^`zO}9< z$>gOig7-P~%fFIojce<$NMAjBxV8GV>8WE!uage9mdccmQyq_>*D^nDOWAyTwj?V% zQhH=5D_J(bloUBLY0j-(rG5J3nrhfNxzRD&shzpG3^?*l#+>rfA?VMwa%V~+I zukx$(csbwcraX8bM|vG9>+C3YR?3YYBrO-GcCPO$rB6b8Xq!GgG}q(b60b*Q4L;vn zs}I;MtvhDYaVsCl;VJLr>$$o*K3QGu)3$`pqz6>GcTmE1)Yp^q+_lK9Z!*g>zD^$@ zF4w5b@@;w*JrUhR4@@i|5shl;+q`dO%B~uEcaVoZPrcREuzE>xeL3U&*RQDhRIR7U z_Fj=qg+x29zv-m$80%`j_>vrHpItr-@Y26w4<+KxTj?`!k_=n0$u;+HuqIjFQx>%J z*Ham0IA_zhb7lSMuQ}7Dr*?mn;ZIv@#pX@*#qeIvy**cC+lAB`G%&Z6jDJnaUQVYA znm&~LJM!t${LAE2bg<4IkXnltF0VbK(rP20Kn?k<^802SsWrBTeBJV$-?7KlVyj=t zf|+;aTGCD$zkFql?ekT3HtDNhiVt=TKQh+Iex-q~Kaf~b%v>v(-uP>u+Q~KOVIj@& z;+2$bbyP|`d+($f|H-*A{f^97sZ+&ov^<%&RSqBRt67$E zPr6sqai%5E;-ix};{wU01JuOQ(~%#v^shEQ*kAB(>Svu zwD*U8dirGsZSrEJ#HpD=S04ZA9E>_3wIUu!O!acoppTC(FXiZP`eBDHKjdM~;ncLA zTKwA&$&}w+67{Je*W6 zC+Me#9;xH5*9Mhh9_05=N4n|lr2W;a$R9Z#7+=QT?xLRab7_trO*J%02kG;VN6(%yrMpxEJ8(VAU33c`6iYam~ zs-VUZM1`FjWZXtk7bcNxeMQ>0f(?PQTNPJdK3^>7sykADZ5E9=!N21*&Zo%+)z^nV7CSqdwotz=UG(mlVmhhl5tjt_ zR$Vkrs@C}?hw`V?V<$4`=&8e`NW&MBb^l9=-`HKZt|=xXzWs9w&8efS|4XC!kA>Stq|wHPpvftLYT~XuXv-vEC?ELqDHL zBQw0Vxti5D=A7?ZM6Y)EBg0O|lecqwYp0Scq;}i~^7ndP?xqE_#)0!PW_~tjeujy% z_C^DZ9++C+bo?VB-VOEi>)yJ}l~|g$tfc4tYsh7+&x8?|(C?VU{%gZ)1<#_E= zPK$sbz4@n=9_ig$es@|fy;28iuC>W!WzG0faKavE*XsVx!OxL8yILzbT{znHVr(a! z;n_f@BwIz@TrY3#*3g{XF{6hi(H>X4HQD$~8kiWF^Cc{Q0bR3mgrxk^OalhB&{Ju`c~8-L??+q5t6nAb`LC(Inh>nZrzdh! zly9I5((IIUi+bzBuSs>x>tGEGd?9gr*U{vzIQlhxX3hO_nIzs>S+6XwuEmCxbUnOv zP}T>Ya-ut?(U0>Zb?%tX+HT_~+4bIC&qrm}u5Pbo`jrtfWWaKfOnvlrwxLq6@*(-z zw4SD{Jx8nKhzcHLbCvt5bTyD5quDx>{_lyOz%DrwevgcfB0xuAbX! zYdyCp2{@Ea@AL0=&Z}@e^RuW{%;Tw_cSLCAseU@V%us2bqqA0CI9RUVd@p~iw$^#e zL-oz33fl8S8;u^&L4RJDE79?a>GdW-x_oCteRw08{8uKf7L5PR<-g{D^U!0iG?=%{ zdEpjM^EK?LOE$&V6;-x7@uK|nRj;E?P`C8@a8_dV+A>(OJT2^W4DP2zN8XjYuLn!U z*=2RO`xDtxYMYE5*IpO+g=@=Z@pR?4Mp|!Oek~YVLvsuZ(6;k#NWrrOB&2l>oiOpL zQ)*A77RfP98hgi6--Cs(J!*3Zu}=#h`jb<2xWlKGjpylVc;6>#v9)2>JcIoG$aHZHy2x%`KJ zJ5AuJBX1PYdcDeMwVGj4^-oo)+^oBvj4h=5!#ri=(NK+hI;R#4UN2$Idpa3zR@bEb z8%xi(g>}T=MUrfB3LTxRmH(a9zFNP3T79vlf|hWuOWEV!q|n_8dOqTpe16_rr@4Z3 zK>mK}IVXd*&3{J@ObS(xscteNvX>5CP+oEj?WSX16xA*%SIhgHS0r}L5oi5>MP&6; zAFVttzMjw8S$)0F$ak-^@?&0cy;^6Vyg9i*mb(?#C7)Ni+I;krhHtuPu?tl+dHxsD ztmry{yp^B)J9q;V=uity1Ldn>Cy)!gEelPL^^f+J!$71rg?u& zac-xoDNmng)^r8p%g?Zcl5$_ivfXh7Qgp z<2N-CCupFo?=oEq_bDb@GsKtQDGJM*?z?3Dgb=MXwFjT^LZ{n}5Pkctx30fiO-H}) zq5Zn^Z?XHDY8rlzIY;Vi{6HT)%^j#oWBz@(ZanF`*-c0Piqv^CKT53=4Q1caK=sR8 z)0sRcp61+;UZ=-2*Pj0FTFl=|&kWw@RP;_QMP6snL(4mB=Ai$azegWCJ1Vr+d+SnZ zn+GH0dd2}-vT-79*65Sm?ByY`M@#Fvf4Ma5>nJ%E;il_PU69?|AIq<+Ep$hzS(3ih zDT(*x0>94+*TI=%WMAg`+Ab!CY>o2K*i-p+;LFt#R%EF(Jl z!(+7Z9$IdAdr(#@<>u z-WK`wG^uVM9;`$9wUUsvjWwZE(Nukk$kWu*<>I*d5?ykDCSOrW7v^fuFkV#4EpDMH z+hviEGOaaL!@OE^ZEhJ|^8YwG@3@}7KZ++ZB57Hry@V)=>T}PLQqe@EC=FUF6{(C0 z$;yoET_U?-W<+G~z4ArbWXt~D$M662&inIz-+N!@JkR;4(){vABxrW#)Wu_XaL^T$ z7eB$3vR3qNn=4fN`0&ln2wrM2ftfFRuu1k3?0DqJ0-YmpR}zTN?#Ag=zKpe725s9x zeAgJmAprphYw5}dS>&px<7q!c@*<%T{N>P;-}Q17cGH!3>(W95?c0tiqmE#}Y(MD( z45d%322HsDCC@w`LZPW0#e?alEFAR+{hMb{`;sbG zx<5x*KOcInH$p{zJ4U+&aNS!l&Xp|Azk=&#a^`N`nb}=WK)qlXdxi=|6spts#srN1 zdJ9=r_MGsqUeWBRzC7gjg|90tMCuAJX7=5JHqqA1_2~?|86U*S;ivI?>Ui;VMgr!vy^cADS0mCkQ&Aps z0&BIhIVWr+E^Ypfhf`PM(%pd=ePliktXqz^AA2LVM>4dXGP!ooOhg*_v$8RT14~Y! zTieIDP?HG<)gky^QYWIu#)`W)qG?#~#KoJ(QvXi}_S`s@D%n~5J>LyqE~PN|#dkR0 zb{D75#0%}4U776E1E)`W@dKL zJ?gvT!lfvBuK_~`sf)E$<*;3T5Uv$c|ILZx-*HKtpRgO2)-Q1J={D&8d@7o5G3T6v z`NHMt4t!L3f>!-*-R-uTosk~{tlL*>Bj(8#EPN#tVm-)Tb4 zDESN)jb_V(o-mWzp6h?hd#eXnq_$usng{eh!hDZQ~=yqP6L zRBIifwsZ#EW`9Ogw-&4mv17|CWyCdOE7|4tb8eu(K^I&=H82F3Sn z-kkR|lozV~sNJz137hq))GM5Zwkph(?`l%ZXx`BN1{Kkis@{Xyp;(iPubdVOb}vA+ zPCvAg>qX|FjhInu$|{?7yx0E_jB_S&d|EN8Xhr9CgIKHd7S?{wc%|^>_6`f6*wtR@ z#fRX!WC9;;Sq``H9@J0VC(PZYKb7Rf6SuF!+W9`r>pl40{)b}Rs+OF#J55-J3}Er@ z2`H9~Ld&-Uu*9xGq-w1f>uaau&l0JJCp^T!$K{G)n;+sx?O5h!P2{a+gK%P-0SDL} z#FB$HFg>it`Il>utr(5KgRimwrar@~Z^GzHbN1NUmlI|vnEu$Doqtth^ThW!`%a#} zu8tKJbz1Z)vtsWXf!yL9BAKqABFJ^QxIXbMoWmslQ@2Qw?`npgJO9J;U4_D1YFTOV zPEZb5jmz(K_;C3Q1fLmBrC;|I4jn$>!(B6a?lNck%rWfG2O`BDe52GED#rpi($||8 ze&}+Lz8k`Zzd?Sf^bZ@Z<)0)v7s5FJY4To{V;f~x|u}sVy{RpoN<-TehjuG4CbL6)N zn{GQ|bF+$#p2)7}$v)PKL4JioQ zIE*@3TXD|)B1{Tg=n~eBiw=Cl=I^ucsG$zm{3kJT-UneVxvrk4@8Q*ol_;q5WMP~o z!V|tgHF`1!DKEvG3lZG&vj_cD{=(~l1Mh8`gxfuzioFZk((-Ev7gg%hrDg(;jf!JM zyCi0{9mAjoU+7C;{Z&IUj}5h;;;$Qr^nHw#mQ&>K9L5kwV-#t2qPy*Bachtk55!KT z!32LsScJjXNEL5GXJ90C8Nb__-Il5IZ&5C?5A@@neNT`+_B}q|p2+IC-7&PZGp;TO zWtqWL_HGQPS(Ymc9~jWDWGUw#26q|T(?u1v&U8NrddZfx7*3KnOlbN@se54H@KFL*VHu(Szll|B@>m{ad>m@w_saa3@j!kN9(X{F$*dUoy z>oA`6E5YWFa1IEF;Lo4O;D5Fso@u7zMoDK{=iEe2&Q}Dq8^}QqyU^0efYrm=)5qf< z>Wizzw$}4;t)Dg5>0HHOts#7PP>G}Uh#xQG=%ixA&(~rwGR&3MPT^d0sF#>DFPLpx zCUSLQGpAP?y?Nzm63ph^gMY7RzE={Q6W^XAjJt8?pgoxLp$1=dn{w^@LcG{_1`3lO zIJ190iod0!q~4y!TD~k;(oh4~>}r5!jy5!%y8ucz zEji0VlO-Ga(Pz3E8;;pZ-)@zn-M^94llrrthO5}HPcWux2%SP)*nIR0vF>&TBT`56 z*NJp4oB0{j-(>Jxw@AjP1n|W3E%^L!6mte$MM;1mU576~kJC;JJuLO&Ely0`*aL@y z=VIcrK!%tv5KFqP6z?K>2@i`0h-_W}jY+1w+-A9AmQEU{={R%6+V|)sb&iAPsT?*c z6tmjgMy~{Q@!;iLVG|-(tE;N~;B^R1OY7lw<|cA;rlEBCcQJ5=Bllm}CSJAJ0ZY?J zy07>xM6@NnzW1Q(h&`h6>rWWW_&oN zxSP_AS8kohkQXHw&7TdvVGtTikl`OW369uzs;4i#l4e<)leC z?i?pFvh^k3>?Y^9{yb@y$VSQc=yZ$Y+Q4o+q;AF&sljaC_!9a7%1mCeA8}#j!ZBwc zM}8{D^W$T=$7lvN7ka~LViayoxg?$^n6dCju_%6b7%!WTViS25_m`aU%J!+$nH|KN zTU3~>*om21&H1DtgiCZU2%q?tEHu*=#^<|o`nS<^Q2BwxPh;>iFPMY(j^LyhJ8{J4 z7e2jp!4z{_)bJKscGJe+_m0qcoypM0!_Zyl13qedvv2iBRIIv=p-oStOXCK_?pq{0 zF6QF;;=9-zJ{4mQX>m`p74U24LDP9zwDdj(r`NX36!9E-T9bFL>*MzEceowyML)Ie zLOAS)?z&hm*_276Kg8|E_H3v~;kl2waB?!`6`KhN@OPBl>>qJ~f zQ+QQ%j>k@p5Y_)J$0XIW;%h@+ z4lqwdn>oL5y2Vh=jTpxr1y$HS!XEFUGU>7P8J1mpjw_YMoL)9cln(uYU*lY;x@a(` z1g%5l$pwxXpIacnGlf0|zI^feKQwVx=f5pIsXaMQbXho@!RdFLTFLXDD11nDrhTg1B9GQ_V_9;ttb74F9{S6e&UM2JIqwDB+WhhsA+(gD} z$p^mfi3Gy{?o+-DrQPdL|4vt&UvGo2DU*tEVQ|m?l@6o7#y9Z}iS4wYCizWtID4E)xh8EV?k?~0EJai6zHf2zG zzX_l8r||iWwHOtb!37=9W0%x@9~o$JkJNm=V zek>f1TgGLI-!>{(aAY`P5XM1iJE8qMSM;9I0;=znc-6Qs$2^SXio3q7?dZd4hxK`( zei5S2PvI4}fjpU>LCv`$^KMMGT?8sb*&SmRNq;wunQ`%4WZMQ04=fagb9+D` z^Bn!UDCi>J`Hki-O#9FkM`vylS>Y|Yw^bQVwja)n`~A?vP;#HwM=~ZlT{O89%#JIp z8FX|2|Mb>m(v`t9%C}6Vs0JTr8DOc)8nG_Skd{{SF<@IU&fZMs*u_dT8`_bD zI(A$-JB$Bq&m?MYV^+-n9=f>=nxHuO}+*DKu%M@yN0Lha{%GypQSk z&!VT9^u{+C(lDVPt~~eUqHTK2doU2&`inSLNRdUA`ZAR8;M*z@3x6-W9wSdN*E(B{5BCePfAO+PDi(riQRYBa*jn??j%1I@4x1XMt}DoWhgDxU_nV z-J=1Mx^&LV`Hro*>1@$yHngvP5V~Kpcy{|^OzLnHJ9!G{0$!s*`slVRH_ClSvVX&! z8DYN$4=pOtEl8Pa4_Apr`2+a!)J#R{{QwTWb4Fyu1~D(Q3MVu#iFnBhUVqV!J;IgQ zpwXN*f7LMeZXlm@PQ)a6PuC{?#O7~JX>xQF&nwNtUbB5znPW>mLmv*?n1{DzlW}~Q zpq9lGJk4GKcjr-@Gq+0kmn_7x`^TJ$Hf_cqojzE&#EHSCW%yV36_smF;o;@e_>fzV z&*v^7XT=z9Uel2mtlb!@84sOK{tS8J1GQY(%G%y zCh7{5_}gJDC%@f{-ysGNpM1rMyEovxh(p))c4xmvMwLn#|B0G@9XQ3Z zEu%fWc`q-GX%`*2#O4QTKNTx%quto@WD6*YN?~~S3Dm5VxYtIXueR3+6MY}PlIM@W zr=t<9J`+<8+*O3i=SnR+gqmfAIDS=n9B-4vA)_k1z#r+wM#H-?h^}>GIpsqPi=VjA zrpTG^YSXwjv^OU;jeyFXX~HhBJElI7{cloSVn7MutTRx2D8}sh;&uGdKZZ=eT zy$xON2yVHZNv%(k-O3LTQ^)^PI39QAwHe1mz@~>NXon1 zqi9_-iaYZpho&O^(M)x0yRs6#buH*$1orD2!)Y(*f zY)OSpMKBld--duAllkL(9ex}?gfBN9Ve!>`tj)DU@aO*gcXkV21{H`sYEg7w+nKS={_a^4sYh;;Hll_gWg_^=4pM#w4157=sqAUSL&1Tdpc=$1cl8 z@q?usw;goi{^^Foe8WTRC{Co(C&>hfb|S{AJ-3yrv3-p*jcS`9err8eyAFcG@nDoI zNzTzClb>c9^4aL`=-RyudK@oxOX7%Q8az{2fuv)j*y4KuUQSbCt>IQt;|K*_>OP7WX9HSe4`0 z*<7A8RsWzdeFn-}lf6cd;1Jyae$TUp`$CzM8$3&_KU@WsO(85bi9&<_K6G&pq^a3> zK9~?qP31)PQeKRwTl>??XCj~F^9l3U z%<`Hi^2ZLN;bm|3Uz^Dp+R6O!Y7%c8nuC#vncP2k5}$1yPuqN3W)*he(GCSTW#`Bj zM$Ki`)SUj8nsI{6D=DQFBGdT0=wzjXQPyhw*x8tx!JEX4Ge$g=8c2E0|#x$$2TdSeG;pT6dqL`;SO= za0|fFfNJb^|A;GVW7zPv6R%a?#?(&P7<&FA9)4Q`-FZH&cyG@|A&YQLbm#jHBcYel zgX(uCu;F7j$@z9;uSc2mS!j$|XDJmIktO{_a$VT&(V zm8iw1rY$(_=4Wg<+YPqL-I1_wBKyu)#pbhDkU4v;VuR#W!!K*oeM%{;%-mV&X)ctd z-r+pB5RG1&kZBghj8$cdz2-Ynd}KRvdb{wyx(Ga**bJGDDtJ8LH^imS|Ihs8m;Qn3 z&b^9^D{u*(p6^BW9CNCf22fpDdX;XQ#jkrK*x#T%gYSFtVXz8jMm<3P5!+xW z*@N1wZ004H(lkuYnJUhViEl)gnswrH#TQh+$%oOR4&3=>6ceYcRTLGbGwGoY|J;kD zXOrcawtXn~e|?VFK3=q(KaSljJXonFb*JI3G&tu#)9_p9GxQCf#qL3F|Mt9|eHxF8 zy_nd0Ae$Kb@yy~PQL)%Cq8Q&CUqI8u3iH1Xp+-Xs_FEr8 zqdmJ&k`>O~WB$ReYXz2fb>xk2x-8eWVR8K_M65e1Uj5vF9ao(&rPhuMrDoTlcLB$9 zvN%HHBF>bJrj43CN1yD?Ar1{^X422B)2ezM)%d0~~H*O8qPY zzPLV-EA!?2Y-uJGdnR*Kw-<7*Q{v}oMlhC|(BXP59$i*~*C#u2kZX4iPEzA5M?t@H zCDJ#~$5vB0b6wP+!ME0&|8@yxm&=Su$w>C!Xvy^OIY3K|+ z-QS4vjo=O!eU^UcAnK(4shykzn{_8}6#KB>H4lAK>HZkenlIW zNUrzVvS>DRzK0p#-oj#>CeO9$#_boB6qEe2c=1LvwrgU}FSFO<;9(7n`u!M9x0-TQ zmjwPYw8ic3BL8bgWyR4`ZgEsy{YEw^6?3aXWa}&k~{l@%(V>x8naj18ig{@y7 zi5p{!u;rfvCO((5%l>X``Pdkp!>jP_@>d*?vy4yVIG%HE3%{OPyz=iYIvlg2_qYZO zn{I*WIfi88d6+Ki%88y!5qZ^~A1<_~NSTAf9vzrDkMw(G%lL9zoZQ%z^Q_yk!yF~L z7JKrgj^x}P%y!r?Vl+N@`0~Tf4dR*1_!zyj7GoQ&G5wP#_im6pNnyFdLSE-bqaiJp zXHjdH5(BT#ksSh7yjzpOrMgYnZ{2C!$~uO-i#ovM{&XZ9Yl~(x99S@WG*{jYH< z@Ov;1v8SeT`i5~#*-?d;^8y)vv^C5=B%$9{GscExIz4j;;g9PUe7dzO?>-sGgYS+CuKI4ulTF4e`o;{VyWuFw^lgC@YWZW)R65OW&y<5h*{7oq#x8M#UScvs z$F;?ZU@tljP~iiqRrOJ?kXh6hnDb1ROAhVAi3yf`(Au9%8_hU3xK6nBRB+ySH>S>t zV8^k?VH_}-%`|#&eAO)^wRWUU)miMGVjwe=1>)yH9nLLj#TozoMYgvco`=4M(b?7< zy2lG|PG|Fp-bk9IjVJtfh|QjnLch~{+?{O((XS1cv{)s6^_iw?$77Pnlj5(nOVo2Bxm8uXB&b!A7kPA(TlmR#%!53 znRANI;c<0;u2?6tELM49@x|+6{YV9)YWnk&??4poipR7)L-;k@T9LS+0NV#X6c&#R zcz;098 zML1jE>BO8dS?nqK>p#DZ#kz<(G2Z4AMu&S-!~GBNAsBsaoKXA5jvp1_y!|1RF(rK^ zpKQXH6N?%nFYG-N(XMP5$W;!F0`Td_43k+-KC`$bVHR-!C=H zuWy_x*O_9>twU(+Xw8cqWZvm=K1MiP2UmL0TQb=GZ<@36$X-z~$OZ>H>9d(q4T>BV z;>Wk8C~DuGjoOlDRP)DIPgk~UdRaUfw;c+%P7JK*$@im2vY=tRm~r(pu7p|8sf`6P zbbIlUlLK3RFN4|le%veh{?A!@>?gT?$D@Zth4xJBObO*RO**JNjlSQOwfPV83bneVA0p9T&dTEnLXp>-sFbR{>Py*+@Ho92eQ)oHhg|+GcUF^ zFJ7Gut<$r`x-GBJuFR8tyT3pm*&ktw)fl^xYxo{Ae#Wv?7MA zPvznddMvH~hL8#23~F@^13Y9`M5#5;t^0)i)zYg_pUSkTFLGYD=K7+?$bXuOtGSan zOx>D4Bs1VK=`5DHRbZAVK6BggZjyv;P`kS4h;|Bh40;I zHpv1f?DYBQ>@4h#?8P4EmqB^RBiwA-3Vv?;vEW)XO%pBXSsF?EdwOiur8!$)ux4Jq z9o#D?(C~Qz?;Cta-KH^AH)zRDd%~#xOnSK?i($0VgCE~zqs5baF=Og+sLJ!~{AT6*5$~lVE#7F4h62(W?sraCI3{~f*(x$r#?##E~s>o?jQf;EwM4Egr(3S@ptU}w)R9NEW= z4-_(k^X4EHJfFf}N1O42%u@JH?98>#&SR-k6jL%hI6f|rx~H4cYg9O&s0?7kLYb*N zk;JhEW;9A2$KovT;Q0{P%&E=OJ%%=0bH%S+GjQj&8>@yovacg6*A*( zbx7(Y&v!t_q8RhH{=>8%BXP3Wf|o{BLbr`6x8#TOsB?e5`R@b1TK8oAxlzb&*$mM? z;o`tq^5&1#eta@H51s{mc*?p~%noV55%0rr+;vu@MWrF)q8p1;LYRLaI{!=Kh=3?QKd8*U zxxHv%(SfBsDo`osri-gu!DabD@oQ!Tt<|nVLoLiH?%pPNsyFAeJq>8>Ta6YUI%DR$ z9*n&0!qoMJcva`cMwhC1+#c>QR7hD!CyC%UU zN}iLp_{ljv9pOFmpy(j`NZt)*y;ziZIpt_5bklCwotG?we_gPqh+udFDCy3DX87HJ9XsJ`5>cOh2Z{3$LMEyl2K zCe-=UiTB?pbIYPPibjnXZjvl|{pzmV6xEBpoSa0ci8hW(zoSUYM`j0A`RmkaG?$;- z4h^K%jZLke@aOaLY#* zv_2!V4Z3|ee{BRSEwc}>9n{HgCw{5UT-xlcgXC@ z-~?8ue8%Kr*;n=;jJFqz;x~=XjPUvlGZSateH6}KRSsO%eG;49PiE(ohr%~3jLo<0 z72^0DC@<@e)!$o6J);V%I@vKPEf!Hzq@Ulx9ZiS!;7B_Y-ibC8;|!go_Su`mj-_z& z@<`0-<%0?PJF|c2c;49b0>9pkX3p<;DDRLAV%-4jcT467b!|9#WXKLXspVhl$XeH? zyqJ*0-S5m0sdWpHJ!-_46Bi{PVvN&KiQE}3-`6#*aDVt@4v)QvOATR+Z_|;wvYTeN z;uH+)vf%S^sgt>SsW7_V7K_>>v%XS8_E>eqr#79DJa{FxpO56Dz~|!afMjm-+a+ve zcgee;aolG)QnDGbGz?k@E7J)qs91o~+`D4Yb5jNxmgD&R+tAGF#IWG)2s0tQ?7~Gi zlfjJs>dzFJ`|S5JAK%{h;Px-4&_4b>;%Dfgq=yQVep~X_+dN>MCI_Eei1s%_@g~og z9p%hrxZsW8{BYWAdWPd(S-8_*lZ9idVf8T@-wu1jT&D%blp8QcuI)@8w_}LSvhVsJMFaddpCFw z9MAbxj*PqDNIU)ET-L{$cAJd3x|7tpK8}Uj&NX=9(g3Zw12|@JJewBC?4(sI#Fr&- zi*66DjvLRJUn4}zZ)?$G$|$4z=5c4CQfI+77bqB(t|YSr5!rdz1fsLdCuHwoQ_;|cfLwbqu zHIZ4S24eUNGTc3qYJN|!^!X-a44jUD&#Fx9YDWBNz|0K;aIm*6Kla><*dje9y7Z%d zkTH|)JXF+s7h#Oe2274?$7SW?gs+ApU&>tih1J$5uF1gZKN?I9vgL-{A1Dj^BM$DK z%v_$l$8eZRGip9C)fDOG=`D$`|-p#y&cLSui{%!b z(P?A+Yq9A$akY(30^ji-Jp7W}RgN>=}{M)Hj@vS)D3LLcUj-iy8eY4g(6 z>3I3`EJCG+(ROPuM&&t5|2TqaYwqLOKDoE+8u4`YOS~&CLVcT&96v?oxnzH3?7e|> zKIty?T{G^t>cs zTiY->e;;xx#<4{417okqJiqs4bS-e@fbd?>w(9`D4Izs6yM~B%dpt41Xgu9&sj%8s zieC@T3t{NY`v)6^nVJTDV}|oWS)O>*wYyv^yYN)qI9>`}jF?q(@b`-zo}P4rLTWF= zKjn(^MzX`s^tqTi#0z?_eVClvMe2)cY@_u|xCDJe?vOwDWIKx1y@IIUd^FE#WpKm& z7w8!=f@lB6v3LL09Gzp$mIVPE^`tFhYiFYVYcqOWTCPayo{$8@B z`D3N8o4_hv?8lKz{M`r9bO_mmisTHs z=WG=x%tm9hA9=n+_S}?CgxUBhSQXes9QyhIjbmlcgd!9BlPZuKb{G18{6(uy-^GG= ze)KDm{`%D;_-&F%)%`>0rn(QBb4oGSqZTpxF&JfX6jnZx&wTF0sF8I}ZzSJTZ!wnM zhF$3}HHa&ZU&H3Z4M6*Q;z!r6_)o`>+mbu5@5Q;|)Wp^Jx8D{EPyQ8lF&R85{fR=! zua~9_Q(Tw1h=!|Gu$Ej}?Xo<~8vYWeWmZi|*GgsuyqNmpIQ$xAN61EFHv80>eb$Y_ zoQqX>Qku!btpmJg}=2lMNc-_ADMn=j>UwJ{PO~0wH8R=HloG)~sw#>mEBr@cq>|aV(U-{%CXi zRAnxk7%w%l?dauSg(i~=@o3OYyx%I%j%hi_v4~7wU6V|{r?vNrY>X9g&6vWXbTM_y`oB3ZhpWz;gS5Nx)ZG=RZZnall0m$r>PNi`>0*zbFUO8uCH5F$CY7HbR99zI4aSkRgm#RxrfK3%MSVkepS~p1x+Ml+`a@B zy-U!|YA`dBmSO+j2DE(XLzOLo*kVwMz!uM-c4H4#s<+~>Z%>@guJq>k73a{h_aKU| zd5Sq4Bxn1<9IWa{Ey?DqjbDMDQNdIk>?}UbJAml-!!RNn#^t{#8i<>9Tk`o-IcJ|(m>o%Tqiu4%84$&@LbHG6a$o%rM;1RsF<_!)R^kB^2HTX9x zn}&~@(CNWRq2Vk$6dHBeu2rm9-=ZfzZa)B@$1kDPd7Jpv&4Sg~h7C1|d^{;h?g^7= zxz~p)G`|X^HFn(i<{@%l1XAx&pk#dvsj0sfYfVSctAh${W_M)Ad#A;h3KjZpTZfPp zzU=fb3!lTR*fBur$juz#mzT((2$`F|uT7<8ZRvetGH>hlWTn2$x|NgDwMNp)xid}# zo6u%^TNW0}UKcMjbZy?8&&Nq0<*C#X)Q*a0->rGZ>ywz490C`qx1Brfj5{vpoQzs_ zWkXRN?sUB^v!q0nsKVjIe%yVq7cPFe2}jk(&>nadGu9<>@U#q8D_?Lj`!$*NkpsE6 zaWFKZIxzW(0&hmje!TF$cp`P4y+;RO=gEV3+NU`e$ehk2gAC>lkHkXhtNKTk;f$*% zGt?w^_;0A}q8!BQ#+n$fDRX)~vaqImGFNLq#dtvC+DnbfU7BpSxE zu(}A7f*tW#KaDkg=VI8+2H9Dyjn7r<;njmI`BRM-iyvTUa1)l{i5MK2%()s<>F}o! z?HU?Ty!46PF&{ z4n_HzD`HjXN#Xf?4^EAr%B*@jar?VC!&UGqey+h@!=Xv>p( zLl|)HIusRFJlrXQz4j*&*(yA6RrWq?)L`J4b6EA+m(v|$sp9kw+iUK@WZ(!MJJXp) zfgUvL62LpRro-l&>@Zuk14*;26*oVRX3#oE&TKlAKUU2G_KjxMhraOb8OUM1L%CHi zl9A6=>t)F8vs%ON**NyKZ$j7HO>lYh9EI&7s4u%PW=Y-a)}sh6 z*!LWder6!&-2|3Y7K?rMZ(ylzD7uH5@OGI7d_GTP>v!u=wLcJ@?&vaTvoEu+?m{=| zeGhbN%>!?g88M+Jb>p0PDJ+|2p5{1rD2v8Nyu?n~Z{O}!7pk3|%xx`7U=`slE>E&# z%}4{D&1uQ1nxVY5I2YkVeP~%Je_!oo4E@uRsrv2s*-@Wi8>UkEnJ0VwaG^z5q|D-u zrO48yd+}ucd*saW%?4t#JQH~-569iV?RfTgSN_mRW-I4(mdFm%9m?G#6J|@b-kUJ? zpBtaglbJG&4T^IagZbk`FZ^?wfxb_!AflCmkK}hE>>GI(Gq!26EVlq!b}MncOBw_NY$Xg&U=-VAkZ72=A#0- z95cs>tP|tuEYDMEiyavHvL%aTkK@>bdtq=aRAH~-hK)-#rdYa(Dad~%+{1c z1YU;Ot1WOosGwO#Eo@onj=Z&Z@cE@SOAlu7{NP16Xpz8~vWqv>Mwxd{=<)QX>%y#P zKbqf^OxTw!Y}Osan(l+qVa8@`JsZi_A?a*dKAy{d$oVHVhT&D+`1Ni+UcPU|LW618 zx!r~1TK7bUdByVkBZB$!dW6gX>QeE$QcOFu660#qdHVQx*lNO)vb)4d_N^>0ROU;Yw$$1%g*J(w z#nInc4A?t}>4$G%xwj_O?%8qh6?fh-c#262^f>-wbN+H#Cwo^$^V_4%PHiGOaPt)d zQTJ;n))!=Ri0lxnTm2W^;{`tT&=YIDThn;#S}b0ug{Ph|bj28k=&5sDvx&6uS&l+| zYrfxA0_5+;KqN5TA(gwfwBqj_Kg2e~03jggoq!oxNu>r!x4<5cz!`SK+(#Bf{D} z5mp5@oSKp%+Lp*=&3FqEI%i{WMOPN~v|{Fw6n6S+%5jDXto`Cbht{8Q;d3YxUv+2c z?jU|uzc1>`l^A|UgAb)1^!3L}@$1GtVbdd;Drx7CmZUB_xYpy;Ie*TzAIfm){XdQF z!YvzZIAl?*%o~MpZ-E`|^t9wH(|-KkV}bNHZP0ye6-LCH@?pOd7_QtKJ^#C?$eP<) zJenfAWal^Gy_vFe-fa)=7{9{wl3CcCkR>eFWC^wM9WX0MX3maerr$GCq)qdrxyl?d zy+Zai%TDYmsjn59Yuz|Cp$Z9g@f>C?yJU-0xJ7>=_g?FZxb10lc;SYF4`p}a#|iAE zsfp=o!?>w=K8lAv7uNqQx#>&<>t2U5Ep!*gs7BzCeJqDNY{i7)<}|VN=1iA6;?#pW znJHa~z|_;2Z+!$easoK~M0f7Ekjdl2&4qg97SX~k9~IM@aj3myRKq>6wV+yDue*nb z-`|U86aP~zZlBFhb%Hfd*W-zv0~-=b@FYVMkH5ANv(xhM%S5tQYIE_@@+B18lBrgG z3%%C`GvGfR#L8#(*kfZBPWmGL*|$g4@B*B5-6+D}C9-g4TPE+4>sWmZFDsXeuXjgt zspQ>HE}bgRRLKbbKAGxrk7+fqN;I2#6)%6C7Y*`RQfV8`%ST3_yrKnE9M0nCN_)mU za}k|Nzac62EaL9%Lh=77IuE~``!9^QHAyAy(x#F&b$`xPN>&=0q@kh|?Wtw&k)0$X zGZL90du9~bo0L7`vDfeX{s6Dn<8j~L`+U#2uJ?O#3|hJ#q*>kPQxD533ik0t{q>n* zHb0-z|4oMWEDg-a*g|=($*8H=BKmE$!@}eVoVSdjK`A|G-2yG>eIAY9qfN1JK^qUdp@jh3~|5{`{3)nNOuTH>hEuJEkU= z;FH9XxjX_+b<9#rdnQimN8@{1CAnJIQm9K`6r8D{mSeq1Lxvf2my2*?x05imVivBQ zIv%gHLiF9Y^d)*9RmC{tVD&57V1Jx07wVGofD?2oXt}s`N)|(#b+CA%6|OUT$5}xY zcV?JDj9*9&SAuZ=D?5N9-SMk!jbuD$z|0@n)3UAI5NmXcKFVuhd&C90oH!6Z9b=Gc zl!vn^)5WbC4V*4?!T?Vv3X+rtLNSreAL3+$U%<=sp9Cmex zG@iHgTl!%h^IGl~l)$~zhb}Wi+Gbgb@T*Cq0O@A({Lb&8!j@j!w^Qq;?)XnW2J7~F zira?w#d$poXeOA@AyAt5JH?lhbW&Rzz>J#N;;E4-=IkuR*4~=%JUJGjDaObj z&?;>=%~$!p;DB3AFGZ{3ciL3x&OV8MBp-2>R-G%Ozn^+h$aCguY6IT=vN7qL z2N@O|rDyVkG1zZ16yS?)vJudE$63o$(U|7lL3cJBqw&)>lg{0hEv?5I7;^5zp z_ZQ!b$non}$;&D=49s`Lr!q4bZj8tAVP_?7|JBf_e+3xzPz@iqcwmv)7fPG4ggW>3 zC&OFa;m>`8@j0CJ;C)>N`wAj1uO)rnd9~l@ikT7%)Rc}y+K1sV%#olexd-_!mcg^K zt8{U+3Tp3o(c}BFI67jzXuPys#O(LNsjvR%-8`MfKWm`HBU&UOoNo!yTTZX|cjrsm zVE1I+OV)L$5tDFG+f9+=R98|Y@6BbKR#2B&p*S(T7&d((gk}FFWV?p*rX{2Bn?H14 zGI!ASLM5_mlfvt6Q)t;91L4rNn1)|GM60X=u^ojk9`9hOY!!8rDW#HZq%b8 zQQV3BN>!ZOGaspm>uctV%2n1FlbVHp30KMNpQQ+?c;FE~D-XA$1MuK;Pcd{?5*|O9 zP2~zP_*VLuu6(>nHf_7;y!t?6H>u!AcST{|pSch_z3}Fb1Oa}lg_-jm z(LXsy%v<(RY~=fy$_aD!JQ>6KT0Rb$8DN0YQ^~jumr2;XrC#0F(E4fmsEa8PdZuwO zxykQ=DxtW=@B3W`2I5j4=VCU+LCbSCseD{bhx@mQnX49wo6Lzb;N6?r;2Y$SZh<#_ ztuf@4HZ&9aVaKW&q~axq7m*53Y+XWuowm3#q=qhk)yBQyf@Gd#B2luR{ypj>DQ^XQ z)V6@%ehd7y%Mxw-4Kd8SQ*w+MIiE|rL+)QU9NspEq?7puI6V=oKE|O?{l27HX%3B& z3PD4MA?$J*=v=XkWPe!}nqTqRq6erSQC13n4GK`t=NOQ0-MEo^oj0o?&ouz8k2 z-~7hFr!1bF@3qs`EmiDnbHo^{Ad!~xnRcH{LU$Vv&Q~ZPpV=8cv*dA^yAOTly{4GI zp`_45um{E%o+q3z^V!mxzCG~WT(Q$$iCxB$%5i4i-gbb)(qj}Hl#90+OGxRtKHBvQq3rg9 ziuMGMJMRbYjxoj*8!0>sxl99?fxL2vBJ4B`De1lk_RR(IE(If|)d+5Xlu0f>6Kxrv zm@9LN0xpci`r{f{%D<_97q^jB{$6HOJK(U!H?m)=hKee7!c3S+*HXGsLnE`shKG^S z^!)Bl-pv2r~Sie<@99enm2j;_+CaEj3QzGFd8T`!S2zTcuMspwOQ7(Pu9~j&TQ>?&>u}}dC$)rWrO7JBz4ORQgv}? z{hJDzWM|qkVY9H5-zq*R?4rYoQ;2GuVLayrnZ}HuvSm2q;S5M_>XrB&6YFi!QU%1;RAc+a<+>}^@{NAJpezq9U{|14fOB0 z8Pyi^yr%V&-X{`0PP|K#zuKbbP%GR@^`bVDH0Db^6?4yWPJOE$DM&x3;Gc5%>7tCA zrm66DOcS01$Kq_q2rQhD?QtgA2|jN6$WARm@mEu5r#Fz<_5p}X*i2-rPj8|sh3ctZ zLe41!S+lCh^-(MZy&eJQhauIT6PINH-1SU znSHcKvj{E)UfA9o$?t*O56R4j=Bf8&9>87VGfDIyC>ztrOLTBf)Y`F%A42(WU-)m_}X3|c=RooN{J*ves`R9@`xBgJYj#hE^t`4#$wvujB z7;2q@pvm{oS5KeOodsiDl@%T{uU9%;n=-8A5Nqo+?R~w`Fs)C|Hm`nVpSYp zUWkSX?&we~q`&+5UAAsNJso32O+829jhPghI3v`X^JBQVea&!^y_s zSXLiJD}))!o83|1uZ{=oD!6xveP*8A9bu-N^7=;-Sk7%sPf-uZ_M{gT6Ce6=>sE&D_I`?IL=UluB)Wa-08-nDqz zz&*o89DSq&rGW)dN@^ijo*~aZE5f)#I+*Oo-SKX|6m8rEXQot8{UdK`e0Yttuj~+a zb57I9#x=A-Q3d+3%GBc}^Gviz+Bca1qT}4+ms$SJ5w^VxDF-#p)Xd<5#Jw( zdfr7_Y=}XO!edHp?uF?*ms^k)28@`9l6_hD>0Tx-PT3$_RIbo=o;6>csiN@VEo9Qv zN$c5taXu;+-2Yyh*`pHL-is2Fc)VnsEM4P=aYBHDphr6J^=B&{0W~Gkd~Y zRuc_V>!@n>DQfIkL-kde=c4SDz_i?zA^5GZ_cE-wd7 zTi9dONi^Q>Hktgj!|I#1G*e+ZwK}uU@9JZESH_IykN9Wzn z`rXpFKI)XP3^tHxEJ~t3FPI6n?VltuDi(L{Sz-xij3@T;mS{WY(D3U+;V@Q~-$ABO z=e{C@Jk3Ummk!ESHcBdfTR>s0HGbMM8zRyiML#>)&GU|4smZ`@uOn>YCSd{R^Bn?9J|_lzi4SVC2Ym;wA=8WV;sq=>o%@pT91xc=xN z?oI~&^yvdrt*#hK)2J)Io0mHA{!HZ+$=sbo!~2~USufORgm^{gTyp8tiZDuS>yW%U zXGMRf@_XL0F;r+VhAvL%kL1-3So_k0TGjeslwWV|X-XmRIxu@2b538`(auY2=zZ8s zkG(#!BJbu;@*ZUf^*4by8*qf3Fv*;8SWD68i)j8keQa3SgB*8j!Y%ELczXIJ?cceX z+++ff@%#gwigSV9(h11>5QL}gp~y(-hQ5x*B4S!7Y$tWWd(UyuIs1?H{qjVx-f;N2 z-V+9#p`I(Xlr*A#2*({#w9z~P502lb+eWuV#Nw0U+kOe%p5-p;H3p;7v5>iFDR}UJ zI|$vrk;T!0m^-JZkYT>$#g#TFnq-e%)@D#-*8k>zU1(~FKSutlq`60z&@!hE3Xhvh zrCqPmx~Hp!g`P4BCS9g$y^7IL?h8%-Yow*Ph5GfhM$BPZROG%8=eO>WWS`(W3n_{T z1GDI)Bo8gCH`3+R<8kcuNwVznndCP0$EisJVW!qdw*R8h%GsC7MFViPYm|6>&;Xm4 zRY-of=wVZS99>^uh{FyApH9Hqw;=H-NPI`=4g-|cYW6Lo#irTpI{99s~Qj872COQkc|mD4fxKI zgL4ahlNIwk?22ady*VHK+j~L9p*LvKc(Hq<4}Cbbk|fM8i%8f)<{OL2uR|02UYOy- z7j?)q2TGEO^T4~l)1MVPNb)fnfr>E{Gb0cB4SR%rejaif&5(aM83!U$@YSvuV@w8M zX@(A!+7+PjK`o`MGez7W4dz(rLw8D*c)VjS?YNMJ!-EA5yh_I6zJuu4c5|2no~MwK zDO6^XjZ+b|61Sp>NI3J1)VBqo>0UIhT$jh8#pU$S#U347i?EMB*W)CU>BFpE^qB8@ z>aTdGXSj)KxT_I(u8R7qYvIVa$+)p{jpSacHyPecVwdbX%D-4is+xmQH%tvTk450_ zDLV@7T$0KUK8jk){gf}~? z($xlGjBP2+UVehU?)pLRJLF)OSucicabW%tXMpG3qbt89sJs-9n{VR9^GidR|64~r z?aRdbHg&%7ts--cL=@_5pm(~pw6Bk)D7~mpGlvhQ$R}GRJHJw@EgVdQHoQ5x8&$6r<7EU=rt7^Z| zngQ163t$scB#MA(=b71vs=~faI#30nRgzK=XSVS+0{ovvDrg`fA{+e<%Gt zY=NoooDhBJKU#Q*e=~U=G4DMT|9-k*4DU9^Z;8g?^#c$;A{{6FL$US8Wm?sjISelT z_^6tQeIpW($+usfsMB=Wif>ZV>ev>?|7Ko`gvZRDWIWc{!%JEfuIx1UGxiA`?>z** zt()jVR!>ZrwbsL$-BRn7#>47MPm=OeK-xFGWG{~HD6PkHEQ&ZdgYR_gTY#%#lH)a_eO1TIYzYbM8Gl@HHpZu2Eqme|tN8{v37 zue+#c%OG_0@uj;jRS~@` z1scVDD6#K7iG{)ha-71R;w7o##=v~6ZZd*l$S?A0`9wVjt)-d=6Un!~3hh71o&Cf= zqWs(%seAWuNo?WnBpIvd!N5jdcLC8)_hN|=vs^Y!P;Sc7B8)A;Qdz`e>vrzLb6vH`l?v!kf ztdVELf19q*-_L^mNM9tja?@#{yDwgDW@ipZ1z!HAhlBuT;v1i(te4t27pH>X%#yjh zCK5ISbD-X6Pxt#=_h@Q7Kn|}15Y#0e&zlWVu`CNL#f9q^vJuf?VPW8k{E zg=}tI;+$MJ*u6hQdJcPdr|^~e8s&UTMuh{<*`1f@N00$_UUsC`ltIFZncyBuYS_gfHh3ug5h z4-41vIH$INUJjUuaRoEPT+QzIvHUO{lpYI{BT=-z>lfN8m56Ho&9a|sj&(gpQ=bec zoO?YK3%8b{dgcRC$!I6NdBhIkMzV3yC$n=`NY(R+2u$b{MeXdc3%gBEOjnAnhT7s9 zQpkZh>q)*r;>{iIAKn^-6F#e{&vs^NxE~@9tx*_JKLTo1OK1b%#M2iRqm*~aGnm^P zsp<*WxenC3pjr%697BDRbnxQZBl=2K2>rH}b7tS^`sJ}G{N#aEI$NmSpo3bJ6tQO3 zZWOE@4r}>ILE=TeK1xnIY}97RWZ0uk6J^F+2`viyykxs z6C38z#DpcxDI7xwpYv|%nNsC7I)?2%Mh<3A-7Gjrm1W>aR=U8#CP-n)i{(s8(9mwO&k% zh!W*HGBNn8G?|_LOP9Xvpp|;FX=e}aszxwdCnp5?XRKhp!x?=uhG6K78oKvF6^5L5{5r5|xJIbmV>t;>X^h9V3TBBb+&n z6M%)YLi+Cv?S<+EN2ss;SlaleJES+WOGwTaS3@GO-$;U(aomkaaKjO|`yNUe@-(t^ zyNJnm!S}Egq_IX9Wsa#BG9eWq2i@>;Yyx6OpB%8Nx;w^KxstK80_3Nc(XwE75(oJ2 zIg|G}b!TXost-QJ4aMV+5{ww*0$ZC3S}=_rS3Ta+n*E+=-=+lL>QGo6)1iEod9-Zl z3_34k0f%mTB$1sxST^0s43;$f=gVy8tQ*u49!TpQyCbt_1RQch@aS|PW=HoI9h>Da z{b-=%nqC1^3wZC|dj!vOU2)dLlnglIDLZf|<}*)mwuK^EcV}YL!Sy1cPziks1x%KC zliANU`m>R_aym|s8dgbO?W5TZ@QI8Z*HQpG`PRraQf6-%{CvKFLLX~lUf67LFsv6; z)3=aWs}`z{<->tHbVvFBgG>D}nw`m}p9VRd zA*Wv<$gRFFwtwI|N7XZ$Yh{n5HVJ12bf~yvJ}p1Qy_Z*B)YNf+o-gW+*hixh<}*>2j~Z#1$4R4B?(4$@B5lp8KUxpzdfOCRd!`iB(K|KpzPC=|2jHS;;|qmq(w zt9UWJD@;K;yTktOmBXY)9rO(F$Iqj%zNeD+9BxS49cG@q0`$k=xEJr z$-6Em*~xjAUM3HK8fTlTxrg8R^C3k(kQE7!bE)on2)(*cN2Bd5@sasR^8SYOVTr8R z)R2vsGv0JGD~vME=_4SeA6@*Wf+^Z^q&VCLPDc;Wu~6pbh6Ui$bp>R8E})Qg#n^54 zgPdd7!E|&aI*Q`R;bbXlwWHCp;|O_qoDfehrHKjS@}QJ*f;#?;f}fl%WXD9HZ+NkI zeyUk)Pb}a$^qv^g$^3#%GSIlXh3fs-rQh;=9NCWRHos`m=brZycaZS;uFC*fG*K=GxQX}nJ~ebcbRE6863~z|sL{>F0s{G-LZPoOzf6 z@3m5x$un9dcZhy-9)4Z4KV6PLMj8(s!z0TWCC9tYDYwQ#t(jvY3KO9?~$? zT3Q=6iw@m5Br2IezZFKb1b4l3I7}4A?kM3PB#Lq}?QqN;w-6V4; zKD5QFFD8(_>VfZO{&XZ#nJzDRK$gRu5EB}J>fUc??+<63Fn>*(dznLreV7+k`oeaf zH9A9+DR`?j-c@pL{Jyn#)@>XXy?;eMWtuSXFf3TlwJf(($0dV zh9?Z_L(p8OPbuA;;C!qA*vT)+M8F%+!#K z!Vtw=sQl7`=AR`rvcMh|+IQ$zU@zqRWW#iKFcMs3&`(j5{R&aE4X?OB)+_!o{51ZeT z>kU7;^)duGuJ$;iDUVjAZWudf2=XV|(tFo3vdf<-)*m=aXNE>m_M@NTiq1`8ZspI8 zwD07zYa?BF$~>b{eW^!e1vzcE#y@63eE)AS-xd0Z7P~}z^>3&37aFM6elFGCpFuC3 z{Lr;FO8mY%`p76)uJwd}V7&-T-$r*rjgVcyJI7z|>E|9DI=^}W?bUB4 zm9K|MZLvB2Y;B`ggE@o0qKPusKBdpq$Hhu^U%nY&%x?7g6tXv#EHBB^;HJ^Y=X`0M zXN{;f7>9nwx{zDvi0^6^h(9+7&vq=51Zl3PrgM5?$Kf1QXH}BaH!~>bBvE63<{P(n z(xtL#ykk=5oA5`G_x=z>=nWwdYHT^fAA3$OF8(Gj=4*cHzC z(C}b1{B=XL+z!fDxE3#1`>mJUdD{&GDrbw*3*~g{ zNhutZ`r*>@ouatOmP&iPrl2u~DAC|7sb@YJhh;M7{Wj%!^};Wg)wERmB-vHAP+!h+ zymK{^WBcK-{adm5m{7%UHl%L*|$?zXADPEiac`# zo5-S_^JOxn=+zKIaaZIam6=a7hCGv`?Cy$9-77^(M>;d@gUR@(3SM{Zik@*WHT%-#nn)6GSkj>M znH*C*F=j>)!uGzW`Tq>W+J~9AbxjRhr+CAN^BS_jJ1O#HC96Y+iNwZ!8B(?2{XuiNLQGB_r>?YoV7S9u>yFXN)30e!i#pWiZlY-? zmkX`O2T3n=6zx`!r}Al~(9ln$b9G&?_vbv?T78-ZeKw+h+xS`NNLZ>glxBFCQ)hoE zO6pTaolE-T_7Qn{d2)5Ts7EE>YQVk`|v&) zbeN&@R41*L$sU^tK$0#KjV+|EbTN6Xk05cf6s>69$7M&AB1Uv;RyT_1D~p3m|y9Q&%8(4 ze@hvv*NySSE)YF)%Ej}yzRYxtLr`H7db)H_M<{o4SCvw@q=lUFBP2_QY$0uPSF?)HxwiuT+q zjI6z3^InrO-sLj)&5DY>){2Ks37EJ0Deb-zM4}WhyV`|P??|DXx!w=fB?z@Q%%B@S zmCia2hFyIo4L2Es=+@I>W`+cNZ%!vJT8L*UHT0&TFE)xyEDLNBj@B)bb#F$}rM?y9 zYg`cmbbb7Uw$EEjj!k?!bl*ik9^McywC1xL`MxkHX13*uSX6!M zgB7L6Xv7%<(PfV(y`QfLp9vax%>3!;>#x$sqo!my={%*@+JnyC6bb2Rc*dNukDM2* zXRo_@!d`MnSVxPQ7cs=RkK}D%!e;r0G_K|{m1Xg);ckHqin8cFF98E}Pm6!6`jOwr z2}s?bfdD5(thSCreDe~TXJLgHuRj#E=q%}uEEL~f_CP6_(9St8>C9v;jQEe8!OJUX zqvRF!9LOx*HIBHH%05b-4-B8kVCUp`NuLL5qOTg~G}b+)MO*feHZy)+KN^jI1M#%# zKi)aWCBn1uz3}gPn!@V~sWmDN`9(d+`_}@B&?kJEmX8Bz-)O#dDV&41(dI|4n0Wj! z_i8(6XyR!6Irq?GaAJVua@i^|W`Q^ET3e#<+i+Y?VWb6j*0@>U(jSM8{n zpK0fI7NDW_|IFSaG$!E*8TE;WwEO{TQwhhl_Dq_%u1@H8$kTRqxOE1U3$2!NIyd43 zRWx$$amsg^UYCwI_HMkIaEv}4D!^Z6km2EUx>D31v#Qe(bEp>rDy~WD0`Ji2)e?Lj z`k!c=lM8pBWcXETp!0qi<>>QF^Kg~uyV^p;>kp&(GyIXVqKw+ETVv|`aj4I@L<4TH z8*exBm%121r^*tqJasT`Ul1zvxx2PF6kAUJBS%+vjMn$Ypq&b2cKfne^Q(#Wb^N1K zOCsTw)<>L^*unMp2@kn1Mkzr$J(OxKJvc9SMbw>>6TW;W4|~Jk zolF;ice6VlrBOI@Q7)qYn=;bax% zOV}%QnLo2Oo9QdN<@LJ{MIp`!x8GmHlgs^Sj&HL>e(?w9HMvo>ygg3p4xl8{o*3lh zM_t*OXY}D7z4#c5;UyDrJu(t6wm5?ROQ9?6zSzBEFcwy1!?7-$;>zTa`?F14%2C6( zo6%Ugl=}>(p}03)mQI}Lh3C2KQ#Xl1@%*_Yb>$@ux$6o`&Z-~(HBNG9Pa=+n&!JOG zO_1ldh)#ZGM$+HE6ccHR@wI&sXEqx9T4vI(^uf6JTo;Ra_M+vIThw(jclVR65w-az zIm*Z31alugYQ3eXT|@Evvma(nc~0(z-SKKZ_v$L%iNX3)$@oAZWFPzEP9pagKUGu3 z@U6mWlM2qYrx=tZL~~J97>Te|q>GUPAg|M(jh4hI-W}%HZ2u_|joG zQNxY^g)A()rvo+XO4`5esKh{8A7Oej_%-dXDDL)KOnuyspJfBkKR`{W{tU$=TSKfF zf0s7=Q=@wCZ{#|m2d0Eq(U6r%I3u&j!)U51Ml4N++K@=h^3B7XysgycrARLv+9=?` zc&M(s!5vZyx)MDayJV%wG1nN1-%n9}k3MLee_L#@d`oRp^2KQL>2z<)3R*BrpDwO{ zDQ*s#Ayi#BGqg_+x!Nrh&FBJ;Da==z>;d%=Jm)SbqQT71nX=4Gq&o3jtTPHxrTb~Y z=dY3z$;`UFr^cSU$29*e`)lS+qdK|I6ga5_O(%beUNH&uQb&Wfn+KyWzb|Nf><@)- z=Kov)U}1-6cBYb7qwTQ5p^RpYPp3j>P3S(=#Ntb_=zEWFeg%6qt=V6BOcr*QgYh@l z8k?izkp9ja8<$KG5!{Q@KFRy}jRm;h>Vw0kcPVWS&!icubli9t`7xJd|9v_1l`^D# z!3uctD+)`e{Gnl452?;|B^^xgr{CPe-E;AQ=o6xd)1r#Z^tI_=R~6pN&8FW2KTxmE z3$%FE8#1bJ#P&W+T)hO$Fj2ywkv(aLPY_vXpQd3s$7r%hlWeK`Mar}G)3}jckUTB~ z7qo(~h%>KK*M;I>zj>sh97=~{`k_3XZ?|)LLbkqBNKy(>J};PdSh*sh;4X!yE+;i9 z;PenZj2_tF(P!jW3LlmOt(l>8q{@$Xq#tPG>uJ<*c?%gOWgz0UEEa9f#OoK2#pF^g zsMX0}WymLH$LrB{#r|~WYCdG&cflC_Zrt(xLgABM(z$sz=#CFRM`YONd07Tesywlb zca^K>7Q>x$C~~WNa|YU)cYvG3f2#v9_~KFOpE-d>xU-k+cROuVt&uFtP^5ruW9dO< zEzNvwK(#-a|2fkL-A1v?)`P^^AxlJ&lN8$2+0W<^h}Z=;X-kDUW^2fx;Z`s+lr~77 zaaO->^d=FWYDDKxb9b@O1+I6tP!E3g7jbu_{9r7cdor^irHK8}^^%F_Ysl%NGXD5? zgQ{^9!gVU>!{$`OdAr4u{iM(*_Drn!vF=yvAZ@V)F08x+L&mkfxpjVap}thWLa@W z{=FsL?zfhCaVBWDNP^dce0X`Bp~s4fBo(2L{ff1;a{EBYXT-xViT9^k%Sm}rDc(HX zDM@1HJFZIJ%#5MpLT41#0=?Bj@VwU| z%KR-$Gww!s7_&hI=F8xGd@Rsi2tc9@ua*1!W#rsM%T#gC9!c zOSvBg8>jR0k2wbH3mp=`YnVyZ;>!vd;o(>zvM0z=<6{@`c(MgiBXp8exsyZdEj{mmNCZMI*p+GIJgG(egW2sUP!j_C*YWyXg)|Q?)bBr@Lb7hnb&g|9;Y$S-82Ai3_YmKEE}^&FfZnSA~o&ohN~GdqOk|F zdunbkg3whdwe_z|Zsn{ny}&Gu)$Fm&*QFcBUNlWH0VF zE9gZpqFc|Iak2Y^sLZINF#Q&0&h&@>CNt5b?tqUy$BXi;Cc3>*3Z*}`)7$;eL~mUs zYF(WS#nmo|@LWn8K6qp53+^8O8wgou=X!kThFebpB}UH%P*qYC>bMhk`u0o;|7$L3 z7;lWDb4Oxf%Siemm5P9~kICu40PGquOPG%OLi?w7((-C`kz>4owzCVf^DJ{ex9yO) zNk-yWtQBgV?|ZyEs)Uj|&Ej6h2fA^~gjq{VX#XNv_d}mDP-V4J42e>~yu%X_dAl2J zU3Psi;-H|hu3uIPtb-8B)$xy+PVsw9!{ zJ8ktj8rWS8^?Tpa)G~qe*)J&WgEw{47(gw*>V?>^iJq%XsPsRcQ%9sid&C0r8S9R# zcdT*7>pwD$P{h-v(a`SK7cW0Pp`oS&;V_7w)lS@@oT7>wpkw@1!5pHi$b3J%?iIEzUPQ;XUNkOh_&Q$Hw*Mj=&vK)0!h$(jUG30C#_w**Cz|@<{PR*uQGcMCp-J@i|xp&*dY)l=hO1Xz4 zvFLX+dMQ`Yl4*m;eDFbu3h|w0tq+p()7djW77h#3v9E*oh0&Z(DeR5!@m-+v2nfs8 zz>mXDWN>HG)D1TouO32$T@MN5D*PW)BE ze2qUOTa}EB@q6i{cQ%CvjzJS=d+T&l$fJIiWZz9@oX&YiYu}W>%i0Tut<3nfw@0e> zWZbp5KsxMwty39=&Nbl(F7kllj#m_Q@`E_AVm$qe;wsAKS@1cK3#8fK==D0^kOPAy0=B*V{tBQFK?!Li6su| zE~V?wGhq8wf{`;eQr*6?w}i(gn`lJ{Q|1%8udqrt_QL z1#k{`g$2}y-X)o9<>dB`iFCE7H&r^@LDuLb9ndI6 zf|@ouUl0ucR8iH}25QV44x8I^Y1^7?cuh7XHHv^3;e({F7pa0j^EEZ=L|}s=#oc`( zNjp%0t6CFr)X4*DE*~R}5uEe89*io^t$uhfgOqZGSPK3QDm-2W~Llbb8CsTwPgmM%sx8^rf!IF7r1l2rF) zAJ<@J5apkcxSrOLw76yA3wI|zHxA}op%NsDb_n~^3&%@UVVCHDat}+qjn2iZDqyX} z91(EDAFpMZ4?AZTIXB&B+3yVYayJqDG++#@Jss5)aR2A~n|x@|F+i)X%wO z7oCH_i;;aWtYABp{oY9Cj!$XY z;7QQ+%@a#{w}_3W|9M0z&ZZuG+ZfNhzhBP9Sk&o@Chx&$-PbJoY}SOMdT)ql{MjmH zCu@QemDzLOSZ*vHmX?bpmO5zX9=v+?WYkP`LD1|{gjts0*~YuHvC>T1K7^C(`rDtEl4SPP#jiC}C>6h?e5p zr+Xvmtl3FPr(U19W|`|PC!66A#crIwC8r1?8VWVaCp^>JsP zdVh?Zx=#3AVixj5Cur~5OgC5c#)@AplprOC(*emi+P+J&cq4lsm25HVg*-J(ctdV? z-C-3JOV_WirhiXHk!vGR+b0txmOU^gD-s?tC78uIweOq<|JUOwjlZ#1jEa?_6|cXE zDYN7t=ffV)2VQ8~90BjP1iEs<3)`-)5c{qV7tN6+h!6ZqtyO#J=^y4^-g`oa7TV&{ zCf-+eE5PzY2J|{`5%q4IBmVmSrP|$tQ6!y&q1kR2a3_l-!-!%_){)2f6s*)S#FNVl zgtglRYX0cK{ zZ<&m?wFBX%+e{ZYqm%LJ5j7N~BjCN2aQGC9E81JBzON%P#+hNgEZ(Ev2`LC%l87_^K9OGENjRtNBuR@`gmDK!$!UX_wOmyUzCMv%)!}q{%XV@d zvWqHCD`R^>DL&lUNf{G;Fd?avA}zGpQ^{_`sg{_r@;}NOHH9vodP%hfL7b&xALv)A_m?&&x@f0{${ zRvS^)&mDAg=Qr^+#08qEX85|+lFnYKpaW|SP&{BE#c%jU_q3;qkhu={E0ryd?O|SH z-#W=(`*K?RR*zyg`=ae=5ze3Liz4PTW1t-Aaz9~NMGm%>l%R$)Lys1!kcW3Nx(`x; z&Gk`G-l~Tsn+)+0CN%$)3?9AckDZ6vl~cDs*dJ0w!LZ?&+2a-|h34z^d**|8~NZ*DIWC40UJ{%L>?G$Y31vg7ixG6rSmK+6K`u+div?GlSU_PnGKn$j8 z@nF*+Dl(`b%?rF+*pV!DE`2E6SDzP)ZYZLM)KK^*kH*1|i%90N6xjyk^9;C*Bztzz zyPxfJVY(hls+6ETQXwt`jbWsjDT~*g zV=%0+jYie3rFE}!;e0!lqM2Q)+|eCfrfcKMh9d0O4FLdSqErm{Tc6&r(LYZ52&ZYNkCi_LA4+`vmr4wj6VUR|orDD;APY z6Z>g+w{fTa2s?+n;a$HNjQiIe@i|Jks4_}8KmIKiN!_ICo64wAxk~f)E|K{0jZcDUN6E&Irz- zRTW`me`$Jm>nwdPJ5OV6rqlb$JRA2@#Z&VfG2qQi>i&m&=a$vC*9kmiMo*Nv7#+2UQ6m_)X>82urFr~sM|r_f%Cpl$(r3c zJWq8WPzdLte7Ep;NTm}ml790}$;3QgGR}3t{k)x$^q_;(e&L0Ii|LkV?CwTB?^B@^C5P0S>*D>nWWMz% zV(T03wex%N9Tim=@C~Z}^j@4nPsIANITUwa8hb8VU`3zda0%>(3sYW`7WY9u4frPH zLQHYn{Rdq)lEUZ{DksxOpq1!G9|S_6R|=Nip8#DsUd@B+Yh= z!w}wqX0Mqc{G^z*JuCCmx&5ku+6apsRH}(+xFXrwo5T2Qj;? zF&vJ8>#3L0U~K$ifi=hCNYcX$^ZHAph-WX|`bXp)&kR=YBz&G0LWlc~aj!bt8=s!J z)94o^bV@xKmC17`cT)_i4i>|pLJ_48{@32Ug=$(>aRC2jrc9;?(jsBR%7QF0=G*7Z zIPVJ@btB>-$^#J|wvI45p<|O!X+&rm(T$;W5JiZb3`LU;P>@8>MMV&FQ&CxnAfi}y zy6Brdv;V(EExxY0+jBAVe((C$>A%*q)_R`zoxNwjBlf+I3}5@mfqh@Tac%tM?59us z=JDA(8)rW~JiL7Gu=Jkq4tM|R!u`)Z_2O{&JA1}GUzz=Xf}^wFJ^Q7bhWpPuYkcL; zKOR2!nHPqmv!B7j;qi?)&ylC-FP8%-x=fe2HnX{ia_`sXTNB_7u-u|=6{=Yqa-~Kxv zzkc}oA0C|b|I^369(a1V|E}ry*=z3`e)NSahDTod@cundoj-o-^0$oVeRJm5!bi`& z`%`xf&)qlsIYqB|X86lRZyUaP`ZN2FTzATN!N1-)9KGwnc>n9)HD3C<+4Gj?FCJg{ z;;F-1KlFy-nq5c67ytgi@YucoSpV&_`^K}+oni8gmxtSbu{=Eb?N85s$HLX)>T53@ zAA0!A@%ktKG(0i;t+dyD@CU;=PnVR*_Rm?zK23J)JbvS{otqyg zyN>@e`OnY0CMR#Qo;v&b?%8)Z13Z5ViTwDYNe% z`*Td&4TmnfW-|Hs=I8Cdwh`DyU>kvL1hx^_M&SR4z|{wD-8e4&>KhIp-njYL_3Cs> z-}N_dY}~Z@0d;XfSAfZrzkSnJlj~)T<62zHRgN(sDyRvaxY^ z^Y-%67PZqmUEiYKym9S`OE#}WGTvfUcsqaBL&Zxnk5G9&jb4(A-r$V4qw)UA5~FD(7poHe)rLy%T5>_@$Do}V{}0>SgkgaNs&24FC?S4Lu0jzV0cBJ zzw|C9+l$`VywXu4nZL!VU=^b#o#cg}0V^?DOXjdyjHZboUetxyjF;qTqZe;Sg_X5X z^ahQ!=*2v!!c_iDd{St+_R^5^qttGGMMO_V*WFD=j=Ae?IUa=V! zFCsQ9|SDh?YK{9Bdx>`)ec6xQIPOpB1%4S%# zl9N}kV%2iu>t2a|hh!?lTKp|~L*-GYcQICpz6fT4*1C}hE~mCbgZ1*cSQXx-CcgY# z2pS#lN*BTS8;9C%wV14fhP+yBkVKs}{<2vw>(Edc=*7DhqfrZzTjgrf#Yn9ySY<7U ze&KDRPi_im*Qk9<#UM`g2aRXhqB({7c8 z8e)k;=aC3PqpN16ZZy%yt9tVib$ttrVikYk-0?0p*-o-)L~ihRwTob~uIpB;BDq)< zoFjNrkW6l@Ck(8DC&7b@Pws7Q`gOQ9BzLR|>XBtvo`5iB_wtJ>)c zHI;#65$tvZ^r}}KZ#M;zY`?%}o-bu8K&5vn6}XFFm+g4X8=d5}xFn*RHKdjz@L`9U9#pht1~6B3bm)%R0$UPCDvpa#Ggu$b^iS=c+eeHtS`@ z=Ac1x$+~znzal}BE%bJis_(&>UR0+LY(}+?pix#zs%SwY8Ee`syg?&NuyjdLSWdkv zlEb@}=(E(-{9318@z+{t=Sd!=5keAg8eKc(svGpOd7<$L&Pmv||E|O;I73p`U^Bhl zJc+s##U!IzM?9j}#7HlXmXbLgua!b?kvz9MM{va|tIX_O@+wr*CMti`SRS#e>ylPz z!rSHste`=!YKB^nOz$q#tQ1|NL$a$3JaXp779QEP!HQK@Xt63dt97XKcKry|PE!=v zr-$;_3Z)k>o4ZPgx0@$fMQ?}3ax#{`B&*(qs^)~ZcoeECN-wL{!rMu_ z=QMy-B?>W%WYm=`QB-=(Dm;Q!>rwK`l5BMs ztAb>k8;QQjoHm)$)~oQgdz^TbqEg~ZN4FNlSFM92-uYcp*0sD_g|Mv~Y8_r^K(eJK zb7Y+u+0zwXNM;@3rB|243k})so}4tPcGR}R#;PpA+EYSRRb(9~gR@@NT9qNGc$6s2e@^FFIjiJ` z)y+#+&Nt}At4IcU0aDhz1c9ZS$v%%6)Gf~N?3S1YIx-p z$vn~ryC~SF6n`U`&9GvXT^qe{79&`Z;$1boY@Qz@c||h4^Lgnl{w9LD6|DGc8Y#TV z4O0QDc$CyA&&yD)P-y7vx-+XFNir&%N$#w&J1)G5ezzZlvoqBWjc#IOmAyvCYwZ`S zg0me^!3s4MxRb2%qe9ZYn_e-phs7aAtb#MF?D+5&$*~!)@9FHUDv~n=;f1rlPadju zL^7*VCfj$JRE83TtRv^Ka32faF7@hDN~-dGR?IY}~qbsQeyHMbYuM6i1YgXEOwHeBws-n{K>_N1o8IRShw6FC)3#FGRA_0P;kE1^ViH!Y>vOsV2k*g8Bks6?C!bp#H91rgv=vY$(&+yR@wZ$q|RP= z<8S+hMyQbNqR>r!{7r~z^ds5oF8;0VS8Oi4$w~7SkMKJEU=^w}D3axs zxxL^VkMwTx#Md)ER*|fl;dN!or5CSvnQ+KwJ(Ndq*2@Yt5wucZh1axMsNr>fRV2eI zsly|ZGcwdYYQ@t3$%6p8hRcT0tnlw%`SR*WS3faBbm)SVzatYc$1T9zcsHyb;{I9 zcKU%T`qPx8&LewvRu!8wtP~^@Ba-pb+o1vHEV5dkBROMEosohwtVD)h&wyCPqppXd zS}CwXZC7ZlvKH{N&jbZYlEoJXtdh=#O2MY1kgsNwad z1b@{W)w{#WbGnY#dz(Chl@puJs&2mzNtLkZjYsy~MRL$^E{YeXIvpy#U3_^YCp&+2 z$xgCoPc6#Uw1BIsQ;daXh0mt_8mK3-H4GG2P^C3!@zX|wRgBbA>=u3*(! zg^I(KUPdpfeQ1|Crt6L>+sP|f1!!pao}D$T@Os9F!Ww3kea}9$#3*^?4Ef(ZK?SV< literal 0 HcmV?d00001 diff --git a/benchmarks/perf-tool/okpt/io/config/parsers/util.py b/benchmarks/perf-tool/okpt/io/config/parsers/util.py index cecb9f2d0..454fec5a0 100644 --- a/benchmarks/perf-tool/okpt/io/config/parsers/util.py +++ b/benchmarks/perf-tool/okpt/io/config/parsers/util.py @@ -13,9 +13,9 @@ def parse_dataset(dataset_format: str, dataset_path: str, - context: Context) -> DataSet: + context: Context, custom_context=None) -> DataSet: if dataset_format == 'hdf5': - return HDF5DataSet(dataset_path, context) + return HDF5DataSet(dataset_path, context, custom_context) if dataset_format == 'bigann' and context == Context.NEIGHBORS: return BigANNNeighborDataSet(dataset_path) diff --git a/benchmarks/perf-tool/okpt/io/dataset.py b/benchmarks/perf-tool/okpt/io/dataset.py index 4f8bc22a2..001563bab 100644 --- a/benchmarks/perf-tool/okpt/io/dataset.py +++ b/benchmarks/perf-tool/okpt/io/dataset.py @@ -34,6 +34,7 @@ class Context(Enum): INDEX = 1 QUERY = 2 NEIGHBORS = 3 + CUSTOM = 4 class DataSet(ABC): @@ -64,9 +65,9 @@ class HDF5DataSet(DataSet): `_ """ - def __init__(self, dataset_path: str, context: Context): + def __init__(self, dataset_path: str, context: Context, custom_context=None): file = h5py.File(dataset_path) - self.data = cast(h5py.Dataset, file[self._parse_context(context)]) + self.data = cast(h5py.Dataset, file[self._parse_context(context, custom_context)]) self.current = 0 def read(self, chunk_size: int): @@ -88,7 +89,7 @@ def reset(self): self.current = 0 @staticmethod - def _parse_context(context: Context) -> str: + def _parse_context(context: Context, custom_context=None) -> str: if context == Context.NEIGHBORS: return "neighbors" @@ -98,6 +99,9 @@ def _parse_context(context: Context) -> str: if context == Context.QUERY: return "test" + if context == Context.CUSTOM: + return custom_context + raise Exception("Unsupported context") diff --git a/benchmarks/perf-tool/okpt/test/steps/factory.py b/benchmarks/perf-tool/okpt/test/steps/factory.py index 2b7bcc68d..2e53b4d4d 100644 --- a/benchmarks/perf-tool/okpt/test/steps/factory.py +++ b/benchmarks/perf-tool/okpt/test/steps/factory.py @@ -9,7 +9,7 @@ from okpt.test.steps.base import Step, StepConfig from okpt.test.steps.steps import CreateIndexStep, DisableRefreshStep, RefreshIndexStep, DeleteIndexStep, \ - TrainModelStep, DeleteModelStep, ForceMergeStep, ClearCacheStep, IngestStep, QueryStep + TrainModelStep, DeleteModelStep, ForceMergeStep, ClearCacheStep, IngestStep, IngestMultiFieldStep, QueryStep, QueryWithFilterStep def create_step(step_config: StepConfig) -> Step: @@ -27,8 +27,12 @@ def create_step(step_config: StepConfig) -> Step: return DeleteIndexStep(step_config) elif step_config.step_name == IngestStep.label: return IngestStep(step_config) + elif step_config.step_name == IngestMultiFieldStep.label: + return IngestMultiFieldStep(step_config) elif step_config.step_name == QueryStep.label: return QueryStep(step_config) + elif step_config.step_name == QueryWithFilterStep.label: + return QueryWithFilterStep(step_config) elif step_config.step_name == ForceMergeStep.label: return ForceMergeStep(step_config) elif step_config.step_name == ClearCacheStep.label: diff --git a/benchmarks/perf-tool/okpt/test/steps/steps.py b/benchmarks/perf-tool/okpt/test/steps/steps.py index b61781a6e..bc43bf195 100644 --- a/benchmarks/perf-tool/okpt/test/steps/steps.py +++ b/benchmarks/perf-tool/okpt/test/steps/steps.py @@ -9,6 +9,7 @@ so the profiling decorators aren't needed for some functions. """ import json +from abc import abstractmethod from typing import Any, Dict, List import numpy as np @@ -18,7 +19,8 @@ from opensearchpy import OpenSearch, RequestsHttpConnection from okpt.io.config.parsers.base import ConfigurationError -from okpt.io.config.parsers.util import parse_string_param, parse_int_param, parse_dataset, parse_bool_param +from okpt.io.config.parsers.util import parse_string_param, parse_int_param, parse_dataset, parse_bool_param, \ + parse_list_param from okpt.io.dataset import Context from okpt.io.utils.reader import parse_json_from_path from okpt.test.steps import base @@ -279,11 +281,8 @@ def _get_measures(self) -> List[str]: return ['took'] -class IngestStep(OpenSearchStep): +class BaseIngestStep(OpenSearchStep): """See base class.""" - - label = 'ingest' - def __init__(self, step_config: StepConfig): super().__init__(step_config) self.index_name = parse_string_param('index_name', step_config.config, @@ -300,9 +299,9 @@ def __init__(self, step_config: StepConfig): self.dataset = parse_dataset(dataset_format, dataset_path, Context.INDEX) - input_doc_count = parse_int_param('doc_count', step_config.config, {}, + self.input_doc_count = parse_int_param('doc_count', step_config.config, {}, self.dataset.size()) - self.doc_count = min(input_doc_count, self.dataset.size()) + self.doc_count = min(self.input_doc_count, self.dataset.size()) def _action(self): @@ -313,10 +312,7 @@ def action(doc_id): # much state may cause out of memory failure for i in range(0, self.doc_count, self.bulk_size): partition = self.dataset.read(self.bulk_size) - if partition is None: - break - body = bulk_transform(partition, self.field_name, action, i) - bulk_index(self.opensearch, self.index_name, body) + self._handle_data_bulk(partition, action, i) self.dataset.reset() @@ -325,11 +321,96 @@ def action(doc_id): def _get_measures(self) -> List[str]: return ['took'] + @abstractmethod + def _handle_data_bulk(self, partition, action, i): + pass + -class QueryStep(OpenSearchStep): +class IngestStep(BaseIngestStep): """See base class.""" - label = 'query' + label = 'ingest' + + def _handle_data_bulk(self, partition, action, i): + if partition is None: + return + body = bulk_transform(partition, self.field_name, action, i) + bulk_index(self.opensearch, self.index_name, body) + + +class IngestMultiFieldStep(BaseIngestStep): + """See base class.""" + + label = 'ingest_multi_field' + + def __init__(self, step_config: StepConfig): + super().__init__(step_config) + + dataset_path = parse_string_param('dataset_path', step_config.config, + {}, None) + + self.attributes_dataset_name = parse_string_param('attributes_dataset_name', + step_config.config, {}, None) + + self.attributes_dataset = parse_dataset('hdf5', dataset_path, + Context.CUSTOM, self.attributes_dataset_name) + + self.attribute_spec = parse_list_param('attribute_spec', + step_config.config, {}, []) + + self.partition_attr = self.attributes_dataset.read(self.doc_count) + + def _handle_data_bulk(self, partition, action, i): + if partition is None: + return + body = self.bulk_transform_with_attributes(partition, self.partition_attr, self.field_name, + action, i, self.attribute_spec) + bulk_index(self.opensearch, self.index_name, body) + + def bulk_transform_with_attributes(self, partition: np.ndarray, partition_attr, field_name: str, + action, offset: int, attributes_def) -> List[Dict[str, Any]]: + """Partitions and transforms a list of vectors into OpenSearch's bulk + injection format. + Args: + partition: An array of vectors to transform. + partition_attr: dictionary of additional data to transform + field_name: field name for action + action: Bulk API action. + offset: to start counting from + attributes_def: definition of additional doc fields + Returns: + An array of transformed vectors in bulk format. + """ + actions = [] + _ = [ + actions.extend([action(i + offset), None]) + for i in range(len(partition)) + ] + idx = 1 + part_list = partition.tolist() + for i in range(len(partition)): + actions[idx] = {field_name: part_list[i]} + attr_idx = i + offset + attr_def_idx = 0 + for attribute in attributes_def: + attr_def_name = attribute['name'] + attr_def_type = attribute['type'] + + if attr_def_type == 'str': + val = partition_attr[attr_idx][attr_def_idx].decode() + if val != 'None': + actions[idx][attr_def_name] = val + elif attr_def_type == 'int': + val = int(partition_attr[attr_idx][attr_def_idx].decode()) + actions[idx][attr_def_name] = val + attr_def_idx += 1 + idx += 2 + + return actions + + +class BaseQueryStep(OpenSearchStep): + """See base class.""" def __init__(self, step_config: StepConfig): super().__init__(step_config) @@ -353,29 +434,13 @@ def __init__(self, step_config: StepConfig): self.dataset.size()) self.query_count = min(input_query_count, self.dataset.size()) - neighbors_format = parse_string_param('neighbors_format', - step_config.config, {}, 'hdf5') - neighbors_path = parse_string_param('neighbors_path', - step_config.config, {}, None) - self.neighbors = parse_dataset(neighbors_format, neighbors_path, - Context.NEIGHBORS) - self.implicit_config = step_config.implicit_config + self.neighbors_format = parse_string_param('neighbors_format', + step_config.config, {}, 'hdf5') + self.neighbors_path = parse_string_param('neighbors_path', + step_config.config, {}, None) def _action(self): - def get_body(vec): - return { - 'size': self.k, - 'query': { - 'knn': { - self.field_name: { - 'vector': vec, - 'k': self.k - } - } - } - } - results = {} query_responses = [] for _ in range(self.query_count): @@ -384,7 +449,7 @@ def get_body(vec): break query_responses.append( query_index(self.opensearch, self.index_name, - get_body(query[0]), [self.field_name])) + self.get_body(query[0]) , [self.field_name])) results['took'] = [ float(query_response['took']) for query_response in query_responses @@ -414,6 +479,115 @@ def _get_measures(self) -> List[str]: return measures + @abstractmethod + def get_body(self, vec): + pass + + +class QueryStep(BaseQueryStep): + """See base class.""" + + label = 'query' + + def __init__(self, step_config: StepConfig): + super().__init__(step_config) + self.neighbors = parse_dataset(self.neighbors_format, self.neighbors_path, + Context.NEIGHBORS) + self.implicit_config = step_config.implicit_config + + def get_body(self, vec): + return { + 'size': self.k, + 'query': { + 'knn': { + self.field_name: { + 'vector': vec, + 'k': self.k + } + } + } + } + + +class QueryWithFilterStep(BaseQueryStep): + """See base class.""" + + label = 'query_with_filter' + + def __init__(self, step_config: StepConfig): + super().__init__(step_config) + + neighbors_dataset = parse_string_param('neighbors_dataset', + step_config.config, {}, None) + + self.neighbors = parse_dataset(self.neighbors_format, self.neighbors_path, + Context.CUSTOM, neighbors_dataset) + + self.filter_type = parse_string_param('filter_type', step_config.config, {}, 'SCRIPT') + self.filter_spec = parse_string_param('filter_spec', step_config.config, {}, None) + self.score_script_similarity = parse_string_param('score_script_similarity', step_config.config, {}, 'l2') + + self.implicit_config = step_config.implicit_config + + def get_body(self, vec): + filter_json = json.load(open(self.filter_spec)) + if self.filter_type == 'FILTER': + return { + 'size': self.k, + 'query': { + 'knn': { + self.field_name: { + 'vector': vec, + 'k': self.k, + 'filter': filter_json + } + } + } + } + elif self.filter_type == 'SCRIPT': + return { + 'size': self.k, + 'query': { + 'script_score': { + 'query': { + 'bool': { + 'filter': filter_json + } + }, + 'script': { + 'source': 'knn_score', + 'lang': 'knn', + 'params': { + 'field': self.field_name, + 'query_value': vec, + 'space_type': self.score_script_similarity + } + } + } + } + } + elif self.filter_type == 'BOOL_POST_FILTER': + return { + 'size': self.k, + 'query': { + 'bool': { + 'filter': filter_json, + 'must': [ + { + 'knn': { + self.field_name: { + 'vector': vec, + 'k': self.k + } + } + } + ] + } + } + } + else: + raise ConfigurationError('Not supported filter type {}'.format(self.filter_type)) + # Helper functions - (AKA not steps) def bulk_transform(partition: np.ndarray, field_name: str, action, @@ -520,16 +694,20 @@ def recall_at_r(results, neighbor_dataset, r, k, query_count): Recall at R """ correct = 0.0 + total_num_of_results = 0 for query in range(query_count): true_neighbors = neighbor_dataset.read(1) if true_neighbors is None: break true_neighbors_set = set(true_neighbors[0][:k]) - for j in range(r): + true_neighbors_set.discard(-1) + min_r = min(r, len(true_neighbors_set)) + total_num_of_results += min_r + for j in range(min_r): if results[query][j] in true_neighbors_set: correct += 1.0 - return correct / (r * query_count) + return correct / total_num_of_results def get_index_size_in_kb(opensearch, index_name): diff --git a/benchmarks/perf-tool/sample-configs/filter-spec/filter-1-spec.json b/benchmarks/perf-tool/sample-configs/filter-spec/filter-1-spec.json new file mode 100644 index 000000000..f529de4fe --- /dev/null +++ b/benchmarks/perf-tool/sample-configs/filter-spec/filter-1-spec.json @@ -0,0 +1,24 @@ +{ + "bool": + { + "must": + [ + { + "range": + { + "age": + { + "gte": 20, + "lte": 100 + } + } + }, + { + "term": + { + "color": "red" + } + } + ] + } +} \ No newline at end of file diff --git a/benchmarks/perf-tool/sample-configs/filter-spec/filter-2-spec.json b/benchmarks/perf-tool/sample-configs/filter-spec/filter-2-spec.json new file mode 100644 index 000000000..9d4514e62 --- /dev/null +++ b/benchmarks/perf-tool/sample-configs/filter-spec/filter-2-spec.json @@ -0,0 +1,40 @@ +{ + "bool": + { + "must": + [ + { + "term": + { + "taste": "salty" + } + }, + { + "bool": + { + "should": + [ + { + "bool": + { + "must_not": + { + "exists": + { + "field": "color" + } + } + } + }, + { + "term": + { + "color": "blue" + } + } + ] + } + } + ] + } +} \ No newline at end of file diff --git a/benchmarks/perf-tool/sample-configs/filter-spec/filter-3-spec.json b/benchmarks/perf-tool/sample-configs/filter-spec/filter-3-spec.json new file mode 100644 index 000000000..d69f8768e --- /dev/null +++ b/benchmarks/perf-tool/sample-configs/filter-spec/filter-3-spec.json @@ -0,0 +1,30 @@ +{ + "bool": + { + "must": + [ + { + "range": + { + "age": + { + "gte": 20, + "lte": 80 + } + } + }, + { + "exists": + { + "field": "color" + } + }, + { + "exists": + { + "field": "taste" + } + } + ] + } +} \ No newline at end of file diff --git a/benchmarks/perf-tool/sample-configs/filter-spec/filter-4-spec.json b/benchmarks/perf-tool/sample-configs/filter-spec/filter-4-spec.json new file mode 100644 index 000000000..9e6356f1c --- /dev/null +++ b/benchmarks/perf-tool/sample-configs/filter-spec/filter-4-spec.json @@ -0,0 +1,44 @@ +{ + "bool": + { + "must": + [ + { + "range": + { + "age": + { + "gte": 30, + "lte": 60 + } + } + }, + { + "term": + { + "taste": "bitter" + } + }, + { + "bool": + { + "should": + [ + { + "term": + { + "color": "blue" + } + }, + { + "term": + { + "color": "green" + } + } + ] + } + } + ] + } +} \ No newline at end of file diff --git a/benchmarks/perf-tool/sample-configs/filter-spec/filter-5-spec.json b/benchmarks/perf-tool/sample-configs/filter-spec/filter-5-spec.json new file mode 100644 index 000000000..fecde0392 --- /dev/null +++ b/benchmarks/perf-tool/sample-configs/filter-spec/filter-5-spec.json @@ -0,0 +1,42 @@ +{ + "bool": + { + "should": + [ + { + "range": + { + "age": + { + "gte": 30, + "lte": 70 + } + } + }, + { + "term": + { + "color": "green" + } + }, + { + "term": + { + "color": "blue" + } + }, + { + "term": + { + "color": "yellow" + } + }, + { + "term": + { + "color": "sweet" + } + } + ] + } +} \ No newline at end of file diff --git a/benchmarks/perf-tool/sample-configs/lucene-sift-hnsw-filter/index-spec.json b/benchmarks/perf-tool/sample-configs/lucene-sift-hnsw-filter/index-spec.json new file mode 100644 index 000000000..83ea79b15 --- /dev/null +++ b/benchmarks/perf-tool/sample-configs/lucene-sift-hnsw-filter/index-spec.json @@ -0,0 +1,27 @@ +{ + "settings": { + "index": { + "knn": true, + "refresh_interval": "10s", + "number_of_shards": 30, + "number_of_replicas": 0 + } + }, + "mappings": { + "properties": { + "target_field": { + "type": "knn_vector", + "dimension": 128, + "method": { + "name": "hnsw", + "space_type": "l2", + "engine": "lucene", + "parameters": { + "ef_construction": 100, + "m": 16 + } + } + } + } + } +} diff --git a/benchmarks/perf-tool/sample-configs/lucene-sift-hnsw-filter/test.yml b/benchmarks/perf-tool/sample-configs/lucene-sift-hnsw-filter/test.yml new file mode 100644 index 000000000..aa2ee6389 --- /dev/null +++ b/benchmarks/perf-tool/sample-configs/lucene-sift-hnsw-filter/test.yml @@ -0,0 +1,41 @@ +endpoint: localhost +test_name: lucene_sift_hnsw +test_id: "Test workflow for lucene hnsw" +num_runs: 1 +show_runs: false +setup: + - name: delete_index + index_name: target_index +steps: + - name: create_index + index_name: target_index + index_spec: sample-configs/lucene-sift-hnsw-filter/index-spec.json + - name: ingest_multi_field + index_name: target_index + field_name: target_field + bulk_size: 500 + dataset_format: hdf5 + dataset_path: ../dataset/sift-128-euclidean-with-attr.hdf5 + attributes_dataset_name: attributes + attribute_spec: [ { name: 'color', type: 'str' }, { name: 'taste', type: 'str' }, { name: 'age', type: 'int' } ] + - name: refresh_index + index_name: target_index + - name: force_merge + index_name: target_index + max_num_segments: 10 + - name: query_with_filter + k: 10 + r: 1 + calculate_recall: true + index_name: target_index + field_name: target_field + dataset_format: hdf5 + dataset_path: ../dataset/sift-128-euclidean-with-attr.hdf5 + neighbors_format: hdf5 + neighbors_path: ../dataset/sift-128-euclidean-with-attr-with-filters.hdf5 + neighbors_dataset: neighbors_filter_1 + filter_spec: sample-configs/filter-spec/filter-1-spec.json + query_count: 100 +cleanup: + - name: delete_index + index_name: target_index \ No newline at end of file From 536e8fdfede2f8d88355596eb8b376e9661d4373 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Wed, 30 Nov 2022 18:50:32 -0600 Subject: [PATCH 054/416] Update 2.4.0 Release Notes (#620) (#626) Signed-off-by: Naveen Tatikonda Signed-off-by: Naveen Tatikonda (cherry picked from commit 120ef2af88b4f09537ec8596483a5a0f79756bb8) Co-authored-by: Naveen Tatikonda --- release-notes/opensearch-knn.release-notes-2.4.0.0.md | 1 + 1 file changed, 1 insertion(+) diff --git a/release-notes/opensearch-knn.release-notes-2.4.0.0.md b/release-notes/opensearch-knn.release-notes-2.4.0.0.md index 9c0c45af5..5b5cbcc7e 100644 --- a/release-notes/opensearch-knn.release-notes-2.4.0.0.md +++ b/release-notes/opensearch-knn.release-notes-2.4.0.0.md @@ -23,6 +23,7 @@ Compatible with OpenSearch 2.4.0 ### Bug Fixes * Fix NPE on null script context ([#560](https://github.com/opensearch-project/k-NN/pull/560)) +* Add fix to fromXContent and toXContent in ModelGraveyard ([#618](https://github.com/opensearch-project/k-NN/pull/618)) ### Refactoring * Refactor kNN codec related classes ([#582](https://github.com/opensearch-project/k-NN/pull/582)) From d2e5fc4e0ea1343041ad6a05c5c24984aec6ea59 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Tue, 6 Dec 2022 12:22:40 -0800 Subject: [PATCH 055/416] Increment version to 2.5.0-SNAPSHOT (#632) Signed-off-by: opensearch-ci-bot Signed-off-by: opensearch-ci-bot Co-authored-by: opensearch-ci-bot --- .github/workflows/backwards_compatibility_tests_workflow.yml | 4 ++-- build.gradle | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/backwards_compatibility_tests_workflow.yml b/.github/workflows/backwards_compatibility_tests_workflow.yml index 47ef18a8e..d3675cd45 100644 --- a/.github/workflows/backwards_compatibility_tests_workflow.yml +++ b/.github/workflows/backwards_compatibility_tests_workflow.yml @@ -15,7 +15,7 @@ jobs: matrix: java: [ 11, 17 ] bwc_version : [ "1.1.0", "1.2.4", "1.3.2", "2.0.0", "2.1.0", "2.2.0" ] - opensearch_version : [ "2.4.0-SNAPSHOT" ] + opensearch_version : [ "2.5.0-SNAPSHOT" ] name: k-NN Restart-Upgrade BWC Tests runs-on: ubuntu-latest @@ -47,7 +47,7 @@ jobs: matrix: java: [ 11, 17 ] bwc_version: [ "1.3.2", "2.0.0", "2.1.0", "2.2.0" ] - opensearch_version: [ "2.4.0-SNAPSHOT" ] + opensearch_version: [ "2.5.0-SNAPSHOT" ] name: k-NN Rolling-Upgrade BWC Tests runs-on: ubuntu-latest diff --git a/build.gradle b/build.gradle index 3def63c2d..16c57f21c 100644 --- a/build.gradle +++ b/build.gradle @@ -10,7 +10,7 @@ buildscript { ext { // build.version_qualifier parameter applies to knn plugin artifacts only. OpenSearch version must be set // explicitly as 'opensearch.version' property, for instance opensearch.version=2.0.0-rc1-SNAPSHOT - opensearch_version = System.getProperty("opensearch.version", "2.4.0-SNAPSHOT") + opensearch_version = System.getProperty("opensearch.version", "2.5.0-SNAPSHOT") version_qualifier = System.getProperty("build.version_qualifier", "") opensearch_group = "org.opensearch" } From fa9ec8c5033addec6c3fa8636d3e84cd29b6cad0 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Tue, 13 Dec 2022 13:28:28 -0800 Subject: [PATCH 056/416] Add model index to system index plugin (#676) Extends SystemIndexPlugin. Adds k-NN model system index plugin to SystemIndexPlugin list of system index descriptors. Signed-off-by: John Mazanec (cherry picked from commit 6ba2b5d579ee229664f430a9efdff0bd35dab8dd) --- .../org/opensearch/knn/plugin/KNNPlugin.java | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/opensearch/knn/plugin/KNNPlugin.java b/src/main/java/org/opensearch/knn/plugin/KNNPlugin.java index 670294802..4836c6c47 100644 --- a/src/main/java/org/opensearch/knn/plugin/KNNPlugin.java +++ b/src/main/java/org/opensearch/knn/plugin/KNNPlugin.java @@ -10,6 +10,7 @@ import org.opensearch.common.ParseField; import org.opensearch.index.codec.CodecServiceFactory; import org.opensearch.index.engine.EngineFactory; +import org.opensearch.indices.SystemIndexDescriptor; import org.opensearch.knn.index.KNNCircuitBreaker; import org.opensearch.knn.index.KNNClusterUtil; import org.opensearch.knn.index.query.KNNQueryBuilder; @@ -83,6 +84,7 @@ import org.opensearch.plugins.Plugin; import org.opensearch.plugins.ScriptPlugin; import org.opensearch.plugins.SearchPlugin; +import org.opensearch.plugins.SystemIndexPlugin; import org.opensearch.repositories.RepositoriesService; import org.opensearch.rest.RestController; import org.opensearch.rest.RestHandler; @@ -105,6 +107,7 @@ import static java.util.Collections.singletonList; import static org.opensearch.knn.common.KNNConstants.KNN_THREAD_POOL_PREFIX; +import static org.opensearch.knn.common.KNNConstants.MODEL_INDEX_NAME; import static org.opensearch.knn.common.KNNConstants.TRAIN_THREAD_POOL; /** @@ -137,7 +140,15 @@ * } * */ -public class KNNPlugin extends Plugin implements MapperPlugin, SearchPlugin, ActionPlugin, EnginePlugin, ScriptPlugin, ExtensiblePlugin { +public class KNNPlugin extends Plugin + implements + MapperPlugin, + SearchPlugin, + ActionPlugin, + EnginePlugin, + ScriptPlugin, + ExtensiblePlugin, + SystemIndexPlugin { public static final String LEGACY_KNN_BASE_URI = "/_opendistro/_knn"; public static final String KNN_BASE_URI = "/_plugins/_knn"; @@ -323,4 +334,8 @@ public List getNamedXContent() { return entries; } + @Override + public Collection getSystemIndexDescriptors(Settings settings) { + return ImmutableList.of(new SystemIndexDescriptor(MODEL_INDEX_NAME, "Index for storing models used for k-NN indices")); + } } From 144a5e8a9513999bbb989bdec42f1c9182bedbf4 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Wed, 14 Dec 2022 09:53:52 -0800 Subject: [PATCH 057/416] Adding release configs for lucene filtering (#663) (#679) * Adding release configs for lucene filtering Signed-off-by: Martin Gaievski (cherry picked from commit 99ade4318ca330ad170870560cc631a59fb5f7da) Co-authored-by: Martin Gaievski --- .../filtering/relaxed-filter/index.json | 26 +++++++++++ .../relaxed-filter/relaxed-filter-spec.json | 42 ++++++++++++++++++ .../relaxed-filter/relaxed-filter-test.yml | 33 ++++++++++++++ .../filtering/restrictive-filter/index.json | 26 +++++++++++ .../restrictive-filter-spec.json | 44 +++++++++++++++++++ .../restrictive-filter-test.yml | 33 ++++++++++++++ 6 files changed, 204 insertions(+) create mode 100644 benchmarks/perf-tool/release-configs/lucene-hnsw/filtering/relaxed-filter/index.json create mode 100644 benchmarks/perf-tool/release-configs/lucene-hnsw/filtering/relaxed-filter/relaxed-filter-spec.json create mode 100644 benchmarks/perf-tool/release-configs/lucene-hnsw/filtering/relaxed-filter/relaxed-filter-test.yml create mode 100644 benchmarks/perf-tool/release-configs/lucene-hnsw/filtering/restrictive-filter/index.json create mode 100644 benchmarks/perf-tool/release-configs/lucene-hnsw/filtering/restrictive-filter/restrictive-filter-spec.json create mode 100644 benchmarks/perf-tool/release-configs/lucene-hnsw/filtering/restrictive-filter/restrictive-filter-test.yml diff --git a/benchmarks/perf-tool/release-configs/lucene-hnsw/filtering/relaxed-filter/index.json b/benchmarks/perf-tool/release-configs/lucene-hnsw/filtering/relaxed-filter/index.json new file mode 100644 index 000000000..7a9ff2890 --- /dev/null +++ b/benchmarks/perf-tool/release-configs/lucene-hnsw/filtering/relaxed-filter/index.json @@ -0,0 +1,26 @@ +{ + "settings": { + "index": { + "knn": true, + "number_of_shards": 24, + "number_of_replicas": 1 + } + }, + "mappings": { + "properties": { + "target_field": { + "type": "knn_vector", + "dimension": 128, + "method": { + "name": "hnsw", + "space_type": "l2", + "engine": "lucene", + "parameters": { + "ef_construction": 256, + "m": 16 + } + } + } + } + } +} diff --git a/benchmarks/perf-tool/release-configs/lucene-hnsw/filtering/relaxed-filter/relaxed-filter-spec.json b/benchmarks/perf-tool/release-configs/lucene-hnsw/filtering/relaxed-filter/relaxed-filter-spec.json new file mode 100644 index 000000000..fecde0392 --- /dev/null +++ b/benchmarks/perf-tool/release-configs/lucene-hnsw/filtering/relaxed-filter/relaxed-filter-spec.json @@ -0,0 +1,42 @@ +{ + "bool": + { + "should": + [ + { + "range": + { + "age": + { + "gte": 30, + "lte": 70 + } + } + }, + { + "term": + { + "color": "green" + } + }, + { + "term": + { + "color": "blue" + } + }, + { + "term": + { + "color": "yellow" + } + }, + { + "term": + { + "color": "sweet" + } + } + ] + } +} \ No newline at end of file diff --git a/benchmarks/perf-tool/release-configs/lucene-hnsw/filtering/relaxed-filter/relaxed-filter-test.yml b/benchmarks/perf-tool/release-configs/lucene-hnsw/filtering/relaxed-filter/relaxed-filter-test.yml new file mode 100644 index 000000000..a47782649 --- /dev/null +++ b/benchmarks/perf-tool/release-configs/lucene-hnsw/filtering/relaxed-filter/relaxed-filter-test.yml @@ -0,0 +1,33 @@ +endpoint: [ENDPOINT] +test_name: "index-workflow" +test_id: "Index workflow" +num_runs: 10 +show_runs: false +steps: + - name: delete_index + index_name: target_index + - name: create_index + index_name: target_index + index_spec: [INDEX_SPEC_PATH]/index.json + - name: ingest_multi_field + index_name: target_index + field_name: target_field + bulk_size: 500 + dataset_format: hdf5 + dataset_path: [DATASET_PATH]/sift-128-euclidean-with-attr.hdf5 + attributes_dataset_name: attributes + attribute_spec: [ { name: 'color', type: 'str' }, { name: 'taste', type: 'str' }, { name: 'age', type: 'int' } ] + - name: refresh_index + index_name: target_index + - name: query_with_filter + k: 100 + r: 1 + calculate_recall: true + index_name: target_index + field_name: target_field + dataset_format: hdf5 + dataset_path: [DATASET_PATH]/sift-128-euclidean-with-attr.hdf5 + neighbors_format: hdf5 + neighbors_path: [DATASET_PATH]/sift-128-euclidean-with-filters.hdf5 + neighbors_dataset: neighbors_filter_5 + filter_spec: [INDEX_SPEC_PATH]/relaxed-filter-spec.json diff --git a/benchmarks/perf-tool/release-configs/lucene-hnsw/filtering/restrictive-filter/index.json b/benchmarks/perf-tool/release-configs/lucene-hnsw/filtering/restrictive-filter/index.json new file mode 100644 index 000000000..7a9ff2890 --- /dev/null +++ b/benchmarks/perf-tool/release-configs/lucene-hnsw/filtering/restrictive-filter/index.json @@ -0,0 +1,26 @@ +{ + "settings": { + "index": { + "knn": true, + "number_of_shards": 24, + "number_of_replicas": 1 + } + }, + "mappings": { + "properties": { + "target_field": { + "type": "knn_vector", + "dimension": 128, + "method": { + "name": "hnsw", + "space_type": "l2", + "engine": "lucene", + "parameters": { + "ef_construction": 256, + "m": 16 + } + } + } + } + } +} diff --git a/benchmarks/perf-tool/release-configs/lucene-hnsw/filtering/restrictive-filter/restrictive-filter-spec.json b/benchmarks/perf-tool/release-configs/lucene-hnsw/filtering/restrictive-filter/restrictive-filter-spec.json new file mode 100644 index 000000000..9e6356f1c --- /dev/null +++ b/benchmarks/perf-tool/release-configs/lucene-hnsw/filtering/restrictive-filter/restrictive-filter-spec.json @@ -0,0 +1,44 @@ +{ + "bool": + { + "must": + [ + { + "range": + { + "age": + { + "gte": 30, + "lte": 60 + } + } + }, + { + "term": + { + "taste": "bitter" + } + }, + { + "bool": + { + "should": + [ + { + "term": + { + "color": "blue" + } + }, + { + "term": + { + "color": "green" + } + } + ] + } + } + ] + } +} \ No newline at end of file diff --git a/benchmarks/perf-tool/release-configs/lucene-hnsw/filtering/restrictive-filter/restrictive-filter-test.yml b/benchmarks/perf-tool/release-configs/lucene-hnsw/filtering/restrictive-filter/restrictive-filter-test.yml new file mode 100644 index 000000000..61e55f113 --- /dev/null +++ b/benchmarks/perf-tool/release-configs/lucene-hnsw/filtering/restrictive-filter/restrictive-filter-test.yml @@ -0,0 +1,33 @@ +endpoint: [ENDPOINT] +test_name: "index-workflow" +test_id: "Index workflow" +num_runs: 10 +show_runs: false +steps: + - name: delete_index + index_name: target_index + - name: create_index + index_name: target_index + index_spec: [INDEX_SPEC_PATH]/index.json + - name: ingest_multi_field + index_name: target_index + field_name: target_field + bulk_size: 500 + dataset_format: hdf5 + dataset_path: [DATASET_PATH]/sift-128-euclidean-with-attr.hdf5 + attributes_dataset_name: attributes + attribute_spec: [ { name: 'color', type: 'str' }, { name: 'taste', type: 'str' }, { name: 'age', type: 'int' } ] + - name: refresh_index + index_name: target_index + - name: query_with_filter + k: 100 + r: 1 + calculate_recall: true + index_name: target_index + field_name: target_field + dataset_format: hdf5 + dataset_path: [DATASET_PATH]/sift-128-euclidean-with-attr.hdf5 + neighbors_format: hdf5 + neighbors_path: [DATASET_PATH]/sift-128-euclidean-with-filters.hdf5 + neighbors_dataset: neighbors_filter_4 + filter_spec: [INDEX_SPEC_PATH]/restrictive-filter-test.yml From 04f677ef457317832144b71d155276e4af7360ec Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Wed, 21 Dec 2022 10:12:02 -0800 Subject: [PATCH 058/416] Allow mapping service to be null for scenarious of shard recovery from translog (#685) (#687) Signed-off-by: Martin Gaievski (cherry picked from commit c412c8a403cf2c4b6008df7b36847f2ee329e887) Co-authored-by: Martin Gaievski --- .../knn/index/codec/KNNCodecVersion.java | 4 +- .../knn/index/codec/KNNCodecServiceTests.java | 67 +++++++++++++++++++ 2 files changed, 69 insertions(+), 2 deletions(-) create mode 100644 src/test/java/org/opensearch/knn/index/codec/KNNCodecServiceTests.java diff --git a/src/main/java/org/opensearch/knn/index/codec/KNNCodecVersion.java b/src/main/java/org/opensearch/knn/index/codec/KNNCodecVersion.java index adbbb01ca..a6d699484 100644 --- a/src/main/java/org/opensearch/knn/index/codec/KNNCodecVersion.java +++ b/src/main/java/org/opensearch/knn/index/codec/KNNCodecVersion.java @@ -56,7 +56,7 @@ public enum KNNCodecVersion { ), (userCodec, mapperService) -> KNN920Codec.builder() .delegate(userCodec) - .knnVectorsFormat(new KNN920PerFieldKnnVectorsFormat(Optional.of(mapperService))) + .knnVectorsFormat(new KNN920PerFieldKnnVectorsFormat(Optional.ofNullable(mapperService))) .build(), KNN920Codec::new ), @@ -71,7 +71,7 @@ public enum KNNCodecVersion { ), (userCodec, mapperService) -> KNN940Codec.builder() .delegate(userCodec) - .knnVectorsFormat(new KNN940PerFieldKnnVectorsFormat(Optional.of(mapperService))) + .knnVectorsFormat(new KNN940PerFieldKnnVectorsFormat(Optional.ofNullable(mapperService))) .build(), KNN940Codec::new ); diff --git a/src/test/java/org/opensearch/knn/index/codec/KNNCodecServiceTests.java b/src/test/java/org/opensearch/knn/index/codec/KNNCodecServiceTests.java new file mode 100644 index 000000000..2599200e6 --- /dev/null +++ b/src/test/java/org/opensearch/knn/index/codec/KNNCodecServiceTests.java @@ -0,0 +1,67 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.codec; + +import org.apache.lucene.codecs.Codec; +import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.common.settings.Settings; +import org.opensearch.index.Index; +import org.opensearch.index.IndexSettings; +import org.opensearch.index.codec.CodecServiceConfig; +import org.opensearch.index.mapper.MapperService; +import org.opensearch.knn.KNNTestCase; + +import org.apache.logging.log4j.Logger; + +import java.util.UUID; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * Test for KNNCodecService class with focus on codec by name lookup + */ +public class KNNCodecServiceTests extends KNNTestCase { + private static final String TEST_INDEX = "test-index"; + private static final int NUM_OF_SHARDS = 1; + private static final UUID INDEX_UUID = UUID.randomUUID(); + + private IndexSettings indexSettings; + + @Override + public void setUp() throws Exception { + super.setUp(); + IndexMetadata indexMetadata = mock(IndexMetadata.class); + when(indexMetadata.getIndex()).thenReturn(new Index(TEST_INDEX, INDEX_UUID.toString())); + when(indexMetadata.getSettings()).thenReturn(Settings.EMPTY); + Settings settings = Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, Integer.toString(NUM_OF_SHARDS)).build(); + indexSettings = new IndexSettings(indexMetadata, settings); + } + + public void testGetCodecByName() { + MapperService mapperService = mock(MapperService.class); + Logger loggerMock = mock(Logger.class); + CodecServiceConfig codecServiceConfig = new CodecServiceConfig(indexSettings, mapperService, loggerMock); + KNNCodecService knnCodecService = new KNNCodecService(codecServiceConfig); + Codec codec = knnCodecService.codec(KNNCodecVersion.current().getCodecName()); + assertNotNull(codec); + } + + /** + * This test case covers scenarios when MapperService is null, for instance this may happen during index.close operation. + * In such scenario codec is not really required but is created as part of engine initialization, please check code references below: + * @see EngineConfig.java + * @see IndexShard.java + * @see Engine.java + */ + public void testGetCodecByNameWithNoMapperService() { + Logger loggerMock = mock(Logger.class); + CodecServiceConfig codecServiceConfig = new CodecServiceConfig(indexSettings, null, loggerMock); + KNNCodecService knnCodecService = new KNNCodecService(codecServiceConfig); + Codec codec = knnCodecService.codec(KNNCodecVersion.current().getCodecName()); + assertNotNull(codec); + } +} From 2b12ea505d7feb6f2c4443ef577289c3426e811d Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Thu, 22 Dec 2022 21:39:15 -0600 Subject: [PATCH 059/416] Add Backward Compatibility and Validation checks to ModelGraveyard XContent Bugfix (#692) (#694) * Add Backward Compatibility and Validation checks to ModelGraveyard XContent Bugfix Signed-off-by: Naveen Tatikonda * Address Review Comments Signed-off-by: Naveen Tatikonda Signed-off-by: Naveen Tatikonda (cherry picked from commit fcd2d551c57aab35958516d3198265fe95c906a2) Co-authored-by: Naveen Tatikonda --- .../knn/indices/ModelGraveyard.java | 70 ++++++++++++-- .../knn/indices/ModelGraveyardTests.java | 93 ++++++++++++++++++- 2 files changed, 154 insertions(+), 9 deletions(-) diff --git a/src/main/java/org/opensearch/knn/indices/ModelGraveyard.java b/src/main/java/org/opensearch/knn/indices/ModelGraveyard.java index 21f13b6a8..5db6d77fd 100644 --- a/src/main/java/org/opensearch/knn/indices/ModelGraveyard.java +++ b/src/main/java/org/opensearch/knn/indices/ModelGraveyard.java @@ -7,6 +7,7 @@ import lombok.AllArgsConstructor; import lombok.extern.log4j.Log4j2; +import org.opensearch.OpenSearchParseException; import org.opensearch.Version; import org.opensearch.cluster.Diff; import org.opensearch.cluster.NamedDiff; @@ -35,6 +36,7 @@ @Log4j2 public class ModelGraveyard implements Metadata.Custom { public static final String TYPE = "opensearch-knn-blocked-models"; + private static final String MODEL_IDS = "model_ids"; private final Set modelIds; /** @@ -83,7 +85,7 @@ public void writeTo(StreamOutput out) throws IOException { public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { Iterator model_ids = getModelIds().iterator(); - builder.startArray("model_ids"); + builder.startArray(MODEL_IDS); while (model_ids.hasNext()) { builder.value(model_ids.next()); } @@ -151,6 +153,17 @@ public static NamedDiff readDiffFrom(StreamInput streamInput) throws IOException * @throws IOException */ public static ModelGraveyard fromXContent(XContentParser xContentParser) throws IOException { + // Added validation checks to validate all the different possible scenarios + // model_ids:"abcd" - Throws exception as the START_OBJECT token is missing + // {} - Returns an empty ModelGraveyard object (BackwardCompatibility) + // {["abcd", "1234"]} - Throws exception as the FIELD_NAME token is missing + // {"dummy_field_name":} - Throws exception as the FIELD_NAME is not matching with model_ids + // {model_ids:"abcd"} - Throws exception as the START_ARRAY token is missing after field name + // {model_ids:null} - Throws exception as the START_ARRAY token is missing + // {model_ids:[]} - Parses and returns an empty ModelGraveyard object as there are no model ids + // {model_ids: ["abcd", "1234"]} - Parses and returns a ModelGraveyard object which contains the model ids "abcd" and "1234" + // {model_ids:[],dummy_field:[]} - Throws exception as we have FIELD_NAME(dummy_field) instead of END_OBJECT token + ModelGraveyard modelGraveyard = new ModelGraveyard(); // If it is a fresh parser, move to the first token @@ -158,19 +171,60 @@ public static ModelGraveyard fromXContent(XContentParser xContentParser) throws xContentParser.nextToken(); } - // on a start object move to next token - if (xContentParser.currentToken() == XContentParser.Token.START_OBJECT) { - xContentParser.nextToken(); + // Validate if the first token is START_OBJECT + if (xContentParser.currentToken() != XContentParser.Token.START_OBJECT) { + throw new OpenSearchParseException( + "Unable to parse ModelGraveyard. Expecting token start of an object but got {}", + xContentParser.currentToken() + ); + } + + // Adding Backward Compatibility for the domains that have already parsed the old toXContent logic which has XContent as {} + if (xContentParser.nextToken() == XContentParser.Token.END_OBJECT) { + return modelGraveyard; } + // Validate it starts with FIELD_NAME token after START_OBJECT if (xContentParser.currentToken() != XContentParser.Token.FIELD_NAME) { - throw new IllegalArgumentException("expected field name but got a " + xContentParser.currentToken()); + throw new OpenSearchParseException( + "Unable to parse ModelGraveyard. Expecting token field name but got {}", + xContentParser.currentToken() + ); + } + + // Validating that FIELD_NAME matches with "model_ids" + if (!MODEL_IDS.equals(xContentParser.currentName())) { + throw new OpenSearchParseException( + "Unable to parse ModelGraveyard. Expecting field {} but got {}", + MODEL_IDS, + xContentParser.currentName() + ); } - while (xContentParser.nextToken() != XContentParser.Token.END_OBJECT) { - if (xContentParser.currentToken() == XContentParser.Token.VALUE_STRING) { - modelGraveyard.add(xContentParser.text()); + // Validate it starts with START_ARRAY token after FIELD_NAME + if (xContentParser.nextToken() != XContentParser.Token.START_ARRAY) { + throw new OpenSearchParseException( + "Unable to parse ModelGraveyard. Expecting token start of an array but got {}", + xContentParser.currentToken() + ); + } + + while (xContentParser.nextToken() != XContentParser.Token.END_ARRAY) { + if (xContentParser.currentToken() != XContentParser.Token.VALUE_STRING) { + throw new OpenSearchParseException( + "Unable to parse ModelGraveyard. Expecting token value string but got {}", + xContentParser.currentToken() + ); } + modelGraveyard.add(xContentParser.text()); + } + + // Validate if the last token is END_OBJECT + if (xContentParser.nextToken() != XContentParser.Token.END_OBJECT) { + throw new OpenSearchParseException( + "Unable to parse ModelGraveyard. Expecting token end of an object but got {}", + xContentParser.currentToken() + ); } return modelGraveyard; } diff --git a/src/test/java/org/opensearch/knn/indices/ModelGraveyardTests.java b/src/test/java/org/opensearch/knn/indices/ModelGraveyardTests.java index 694d22ed2..98dcf0cea 100644 --- a/src/test/java/org/opensearch/knn/indices/ModelGraveyardTests.java +++ b/src/test/java/org/opensearch/knn/indices/ModelGraveyardTests.java @@ -5,6 +5,8 @@ package org.opensearch.knn.indices; +import lombok.SneakyThrows; +import org.opensearch.OpenSearchParseException; import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.common.xcontent.ToXContent; import org.opensearch.common.xcontent.XContentBuilder; @@ -60,7 +62,9 @@ public void testStreams() throws IOException { assertTrue(testModelGraveyardCopy.contains(testModelId)); } - public void testXContentBuilder() throws IOException { + // Validating {model_ids: ["test-model-id1", "test-model-id2"]} + @SneakyThrows + public void testXContentBuilder_withModelIds_returnsModelGraveyardWithModelIds() { Set modelIds = new HashSet<>(); String testModelId1 = "test-model-id1"; String testModelId2 = "test-model-id2"; @@ -79,6 +83,93 @@ public void testXContentBuilder() throws IOException { assertTrue(testModelGraveyard2.contains(testModelId2)); } + // Validating {model_ids:[]} + @SneakyThrows + public void testXContentBuilder_withoutModelIds_returnsModelGraveyardWithoutModelIds() { + ModelGraveyard testModelGraveyard = new ModelGraveyard(); + XContentBuilder xContentBuilder = XContentFactory.jsonBuilder(); + xContentBuilder.startObject(); + XContentBuilder builder = testModelGraveyard.toXContent(xContentBuilder, ToXContent.EMPTY_PARAMS); + builder.endObject(); + + ModelGraveyard testModelGraveyard2 = ModelGraveyard.fromXContent(createParser(builder)); + assertEquals(0, testModelGraveyard2.size()); + } + + // Validating {test-model:"abcd"} + @SneakyThrows + public void testXContentBuilder_withWrongFieldName_throwsException() { + XContentBuilder xContentBuilder = XContentFactory.jsonBuilder(); + xContentBuilder.startObject(); + xContentBuilder.field("test-model"); + xContentBuilder.value("abcd"); + xContentBuilder.endObject(); + + OpenSearchParseException ex = expectThrows( + OpenSearchParseException.class, + () -> ModelGraveyard.fromXContent(createParser(xContentBuilder)) + ); + assertTrue(ex.getMessage().contains("Expecting field model_ids but got test-model")); + } + + // Validating {} + @SneakyThrows + public void testXContentBuilder_validateBackwardCompatibility_returnsEmptyModelGraveyardObject() { + XContentBuilder xContentBuilder = XContentFactory.jsonBuilder(); + xContentBuilder.startObject(); + xContentBuilder.endObject(); + + ModelGraveyard testModelGraveyard = ModelGraveyard.fromXContent(createParser(xContentBuilder)); + assertEquals(0, testModelGraveyard.size()); + } + + // Validating null + @SneakyThrows + public void testXContentBuilder_withNull_throwsExceptionExpectingStartObject() { + XContentBuilder xContentBuilder = XContentFactory.jsonBuilder(); + + OpenSearchParseException ex = expectThrows( + OpenSearchParseException.class, + () -> ModelGraveyard.fromXContent(createParser(xContentBuilder)) + ); + assertTrue(ex.getMessage().contains("Expecting token start of an object but got null")); + } + + // Validating {model_ids:"abcd"} + @SneakyThrows + public void testXContentBuilder_withMissingStartArray_throwsExceptionExpectingStartArray() { + XContentBuilder xContentBuilder = XContentFactory.jsonBuilder(); + xContentBuilder.startObject(); + xContentBuilder.field("model_ids"); + xContentBuilder.value("abcd"); + xContentBuilder.endObject(); + + OpenSearchParseException ex = expectThrows( + OpenSearchParseException.class, + () -> ModelGraveyard.fromXContent(createParser(xContentBuilder)) + ); + assertTrue(ex.getMessage().contains("Expecting token start of an array but got VALUE_STRING")); + } + + // Validating {model_ids:["abcd"],model_ids_2:[]} + @SneakyThrows + public void testXContentBuilder_validateEndObject_throwsExceptionGotFieldName() { + XContentBuilder xContentBuilder = XContentFactory.jsonBuilder(); + xContentBuilder.startObject(); + xContentBuilder.startArray("model_ids"); + xContentBuilder.value("abcd"); + xContentBuilder.endArray(); + xContentBuilder.startArray("model_ids_2"); + xContentBuilder.endArray(); + xContentBuilder.endObject(); + + OpenSearchParseException ex = expectThrows( + OpenSearchParseException.class, + () -> ModelGraveyard.fromXContent(createParser(xContentBuilder)) + ); + assertTrue(ex.getMessage().contains("Expecting token end of an object but got FIELD_NAME")); + } + public void testDiffStreams() throws IOException { Set added = new HashSet<>(); Set removed = new HashSet<>(); From 1e8b8c9e88eb55c6e55e571ff40176faa4a7d203 Mon Sep 17 00:00:00 2001 From: Martin Gaievski Date: Tue, 3 Jan 2023 12:55:32 -0800 Subject: [PATCH 060/416] Adding integ test for index close/open scenario (#699) Signed-off-by: Martin Gaievski Signed-off-by: Martin Gaievski --- .../opensearch/knn/index/LuceneEngineIT.java | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/src/test/java/org/opensearch/knn/index/LuceneEngineIT.java b/src/test/java/org/opensearch/knn/index/LuceneEngineIT.java index a7f04cef4..2b6ee25cc 100644 --- a/src/test/java/org/opensearch/knn/index/LuceneEngineIT.java +++ b/src/test/java/org/opensearch/knn/index/LuceneEngineIT.java @@ -9,6 +9,7 @@ import com.google.common.collect.ImmutableMap; import com.google.common.primitives.Floats; import org.apache.http.util.EntityUtils; +import org.apache.commons.lang.math.RandomUtils; import org.apache.lucene.index.VectorSimilarityFunction; import org.junit.After; import org.opensearch.client.Request; @@ -338,6 +339,28 @@ public void testQuery_filterWithNonLuceneEngine() throws Exception { ); } + public void testIndexReopening() throws Exception { + createKnnIndexMappingWithLuceneEngine(DIMENSION, SpaceType.L2); + + for (int j = 0; j < TEST_INDEX_VECTORS.length; j++) { + addKnnDoc(INDEX_NAME, Integer.toString(j + 1), FIELD_NAME, TEST_INDEX_VECTORS[j]); + } + + final float[] searchVector = TEST_QUERY_VECTORS[0]; + final int k = 1 + RandomUtils.nextInt(TEST_INDEX_VECTORS.length); + + final List knnResultsBeforeIndexClosure = queryResults(searchVector, k); + + closeIndex(INDEX_NAME); + openIndex(INDEX_NAME); + + ensureGreen(INDEX_NAME); + + final List knnResultsAfterIndexClosure = queryResults(searchVector, k); + + assertArrayEquals(knnResultsBeforeIndexClosure.toArray(), knnResultsAfterIndexClosure.toArray()); + } + private void addKnnDocWithAttributes(String docId, float[] vector, Map fieldValues) throws IOException { Request request = new Request("POST", "/" + INDEX_NAME + "/_doc/" + docId + "?refresh=true"); @@ -406,4 +429,13 @@ private void validateQueries(SpaceType spaceType, String fieldName) throws IOExc } } } + + private List queryResults(final float[] searchVector, final int k) throws Exception { + final String responseBody = EntityUtils.toString( + searchKNNIndex(INDEX_NAME, new KNNQueryBuilder(FIELD_NAME, searchVector, k), k).getEntity() + ); + final List knnResults = parseSearchResponse(responseBody, FIELD_NAME); + assertNotNull(knnResults); + return knnResults.stream().map(KNNResult::getVector).collect(Collectors.toUnmodifiableList()); + } } From efa2dd4f79e48a1ea94335cacbae12460e8915d4 Mon Sep 17 00:00:00 2001 From: Martin Gaievski Date: Wed, 4 Jan 2023 12:24:17 -0800 Subject: [PATCH 061/416] Fix flaky test, closing gap in previous backport (#707) Signed-off-by: Martin Gaievski Signed-off-by: Martin Gaievski --- .../java/org/opensearch/knn/index/codec/KNNCodecTestCase.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestCase.java b/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestCase.java index b570cac49..edb3ed75d 100644 --- a/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestCase.java +++ b/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestCase.java @@ -320,7 +320,7 @@ public void testKnnVectorIndex( IndexReader reader = writer.getReader(); writer.close(); - verify(perFieldKnnVectorsFormatSpy).getKnnVectorsFormatForField(eq(FIELD_NAME_ONE)); + verify(perFieldKnnVectorsFormatSpy, atLeastOnce()).getKnnVectorsFormatForField(eq(FIELD_NAME_ONE)); IndexSearcher searcher = new IndexSearcher(reader); Query query = KNNQueryFactory.create(KNNEngine.LUCENE, "dummy", FIELD_NAME_ONE, new float[] { 1.0f, 0.0f, 0.0f }, 1); From 084f94f6a84c85b43f6f3bc50ed3acb6cff86b97 Mon Sep 17 00:00:00 2001 From: John Mazanec Date: Wed, 4 Jan 2023 12:58:00 -0800 Subject: [PATCH 062/416] Update backwards compatibility versions (#702) Signed-off-by: John Mazanec --- ...backwards_compatibility_tests_workflow.yml | 4 +- .../opensearch/knn/bwc/LuceneFilteringIT.java | 86 ------------------- 2 files changed, 2 insertions(+), 88 deletions(-) delete mode 100644 qa/rolling-upgrade/src/test/java/org/opensearch/knn/bwc/LuceneFilteringIT.java diff --git a/.github/workflows/backwards_compatibility_tests_workflow.yml b/.github/workflows/backwards_compatibility_tests_workflow.yml index d3675cd45..dc164d4d7 100644 --- a/.github/workflows/backwards_compatibility_tests_workflow.yml +++ b/.github/workflows/backwards_compatibility_tests_workflow.yml @@ -14,7 +14,7 @@ jobs: strategy: matrix: java: [ 11, 17 ] - bwc_version : [ "1.1.0", "1.2.4", "1.3.2", "2.0.0", "2.1.0", "2.2.0" ] + bwc_version : [ "1.1.0", "1.2.4", "1.3.7", "2.0.1", "2.1.0", "2.2.1", "2.3.0", "2.4.1" ] opensearch_version : [ "2.5.0-SNAPSHOT" ] name: k-NN Restart-Upgrade BWC Tests @@ -46,7 +46,7 @@ jobs: strategy: matrix: java: [ 11, 17 ] - bwc_version: [ "1.3.2", "2.0.0", "2.1.0", "2.2.0" ] + bwc_version: [ "1.3.7", "2.0.1", "2.1.0", "2.2.1", "2.3.0", "2.4.1" ] opensearch_version: [ "2.5.0-SNAPSHOT" ] name: k-NN Rolling-Upgrade BWC Tests diff --git a/qa/rolling-upgrade/src/test/java/org/opensearch/knn/bwc/LuceneFilteringIT.java b/qa/rolling-upgrade/src/test/java/org/opensearch/knn/bwc/LuceneFilteringIT.java deleted file mode 100644 index 3a7d0329d..000000000 --- a/qa/rolling-upgrade/src/test/java/org/opensearch/knn/bwc/LuceneFilteringIT.java +++ /dev/null @@ -1,86 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.knn.bwc; - -import org.hamcrest.MatcherAssert; -import org.opensearch.knn.TestUtils; -import org.opensearch.knn.index.query.KNNQueryBuilder; -import org.opensearch.index.query.QueryBuilders; -import org.opensearch.index.query.TermQueryBuilder; - -import org.opensearch.client.Request; -import org.opensearch.client.ResponseException; -import org.opensearch.common.Strings; -import org.opensearch.common.xcontent.ToXContent; -import org.opensearch.common.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentFactory; - -import java.io.IOException; - -import static org.hamcrest.CoreMatchers.anyOf; -import static org.hamcrest.CoreMatchers.containsString; -import static org.opensearch.knn.TestUtils.NODES_BWC_CLUSTER; -import static org.opensearch.knn.common.KNNConstants.LUCENE_NAME; -import static org.opensearch.knn.common.KNNConstants.METHOD_HNSW; - -/** - * Tests scenarios specific to filtering functionality in k-NN in case Lucene is set as an engine - */ -public class LuceneFilteringIT extends AbstractRollingUpgradeTestCase { - private static final String TEST_FIELD = "test-field"; - private static final int DIMENSIONS = 50; - private static final int K = 10; - private static final int NUM_DOCS = 100; - private static final TermQueryBuilder TERM_QUERY = QueryBuilders.termQuery("_id", "100"); - - public void testLuceneFiltering() throws Exception { - waitForClusterHealthGreen(NODES_BWC_CLUSTER); - float[] queryVector = TestUtils.getQueryVectors(1, DIMENSIONS, NUM_DOCS, true)[0]; - switch (getClusterType()) { - case OLD: - createKnnIndex( - testIndex, - getKNNDefaultIndexSettings(), - createKnnIndexMapping(TEST_FIELD, DIMENSIONS, METHOD_HNSW, LUCENE_NAME) - ); - bulkAddKnnDocs(testIndex, TEST_FIELD, TestUtils.getIndexVectors(NUM_DOCS, DIMENSIONS, true), NUM_DOCS); - validateSearchKNNIndexFailed(testIndex, new KNNQueryBuilder(TEST_FIELD, queryVector, K, TERM_QUERY), K); - break; - case MIXED: - validateSearchKNNIndexFailed(testIndex, new KNNQueryBuilder(TEST_FIELD, queryVector, K, TERM_QUERY), K); - break; - case UPGRADED: - searchKNNIndex(testIndex, new KNNQueryBuilder(TEST_FIELD, queryVector, K, TERM_QUERY), K); - deleteKNNIndex(testIndex); - break; - } - } - - private void validateSearchKNNIndexFailed(String index, KNNQueryBuilder knnQueryBuilder, int resultSize) throws IOException { - XContentBuilder builder = XContentFactory.jsonBuilder().startObject().startObject("query"); - knnQueryBuilder.doXContent(builder, ToXContent.EMPTY_PARAMS); - builder.endObject().endObject(); - - Request request = new Request("POST", "/" + index + "/_search"); - - request.addParameter("size", Integer.toString(resultSize)); - request.addParameter("explain", Boolean.toString(true)); - request.addParameter("search_type", "query_then_fetch"); - request.setJsonEntity(Strings.toString(builder)); - - Exception exception = expectThrows(ResponseException.class, () -> client().performRequest(request)); - // assert for two possible exception messages, fist one can come from current version in case serialized request is coming from - // lower version, - // second exception is vise versa, when lower version node receives request with filter field from higher version - MatcherAssert.assertThat( - exception.getMessage(), - anyOf( - containsString("filter field is supported from version"), - containsString("[knn] unknown token [START_OBJECT] after [filter]") - ) - ); - } -} From dd7d0c56a99a9e82ab8095b03a9a885a190b8670 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Fri, 6 Jan 2023 13:30:55 -0800 Subject: [PATCH 063/416] Making version of lucene k-nn engine match lucene current version (#691) (#712) * Making version of lucene k-nn engine match lucene current version Signed-off-by: Martin Gaievski (cherry picked from commit 577205eabca73284db9aa273a9bd2b6511d3ca7f) Co-authored-by: Martin Gaievski --- src/main/java/org/opensearch/knn/index/util/Lucene.java | 2 +- .../java/org/opensearch/knn/index/util/KNNEngineTests.java | 2 ++ .../java/org/opensearch/knn/index/util/LuceneTests.java | 6 ++++++ 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/opensearch/knn/index/util/Lucene.java b/src/main/java/org/opensearch/knn/index/util/Lucene.java index e1996353f..83f123969 100644 --- a/src/main/java/org/opensearch/knn/index/util/Lucene.java +++ b/src/main/java/org/opensearch/knn/index/util/Lucene.java @@ -44,7 +44,7 @@ public class Lucene extends JVMLibrary { ).addSpaces(SpaceType.L2, SpaceType.COSINESIMIL).build() ); - final static Lucene INSTANCE = new Lucene(METHODS, Version.LUCENE_9_2_0.toString()); + final static Lucene INSTANCE = new Lucene(METHODS, Version.LATEST.toString()); /** * Constructor diff --git a/src/test/java/org/opensearch/knn/index/util/KNNEngineTests.java b/src/test/java/org/opensearch/knn/index/util/KNNEngineTests.java index 59467c36c..af9feeb9e 100644 --- a/src/test/java/org/opensearch/knn/index/util/KNNEngineTests.java +++ b/src/test/java/org/opensearch/knn/index/util/KNNEngineTests.java @@ -14,6 +14,8 @@ public class KNNEngineTests extends KNNTestCase { */ public void testDelegateLibraryFunctions() { assertEquals(Nmslib.INSTANCE.getVersion(), KNNEngine.NMSLIB.getVersion()); + assertEquals(Faiss.INSTANCE.getVersion(), KNNEngine.FAISS.getVersion()); + assertEquals(Lucene.INSTANCE.getVersion(), KNNEngine.LUCENE.getVersion()); } /** diff --git a/src/test/java/org/opensearch/knn/index/util/LuceneTests.java b/src/test/java/org/opensearch/knn/index/util/LuceneTests.java index 57e02e173..bbf43a30a 100644 --- a/src/test/java/org/opensearch/knn/index/util/LuceneTests.java +++ b/src/test/java/org/opensearch/knn/index/util/LuceneTests.java @@ -5,6 +5,7 @@ package org.opensearch.knn.index.util; +import org.apache.lucene.util.Version; import org.opensearch.common.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.knn.KNNTestCase; @@ -108,4 +109,9 @@ public void testSetInitialized() { luceneLibrary.setInitialized(true); assertTrue(luceneLibrary.isInitialized()); } + + public void testVersion() { + Lucene luceneInstance = Lucene.INSTANCE; + assertEquals(Version.LATEST.toString(), luceneInstance.getVersion()); + } } From d56ce9d11fd58a5b0df3758197ae2312c19a43a6 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Tue, 10 Jan 2023 16:01:10 -0800 Subject: [PATCH 064/416] Add 2.5 release notes (#715) Signed-off-by: John Mazanec (cherry picked from commit 5f4bef9d8cb18b361f726d7e75609a4d97799025) --- .../opensearch-knn.release-notes-2.5.0.0.md | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 release-notes/opensearch-knn.release-notes-2.5.0.0.md diff --git a/release-notes/opensearch-knn.release-notes-2.5.0.0.md b/release-notes/opensearch-knn.release-notes-2.5.0.0.md new file mode 100644 index 000000000..e0bbd0513 --- /dev/null +++ b/release-notes/opensearch-knn.release-notes-2.5.0.0.md @@ -0,0 +1,34 @@ +## Version 2.5.0.0 Release Notes + +Compatible with OpenSearch 2.5.0 + +### Enhancements + +* Extend SystemIndexPlugin for k-NN model system index ([#630](https://github.com/opensearch-project/k-NN/pull/630)) + +### Bug Fixes + +* Add fix to fromXContent and toXContent in ModelGraveyard ([#624](https://github.com/opensearch-project/k-NN/pull/624)) +* Allow mapping service to be null for scenarios of shard recovery from translog ([#685](https://github.com/opensearch-project/k-NN/pull/685)) +* Add backward compatibility and validation checks to ModelGraveyard XContent bug fix ([#692](https://github.com/opensearch-project/k-NN/pull/692)) + +### Infrastructure + +* Add benchmark workflow for queries with filters ([#598](https://github.com/opensearch-project/k-NN/pull/598)) +* Fix failing codec unit test ([#610](https://github.com/opensearch-project/k-NN/pull/610)) +* Update bwc tests for 2.5.0 ([#661](https://github.com/opensearch-project/k-NN/pull/661)) +* Add release configs for lucene filtering ([#663](https://github.com/opensearch-project/k-NN/pull/663)) +* Update backwards compatibility versions ([#701](https://github.com/opensearch-project/k-NN/pull/701)) +* Update tests for backwards codecs ([#710](https://github.com/opensearch-project/k-NN/pull/710)) + +### Documentation + +* Updat MAINTAINERS.md format ([#709](https://github.com/opensearch-project/k-NN/pull/709)) + +### Maintenance + +* Fix the codec94 version import statements ([#684](https://github.com/opensearch-project/k-NN/pull/684)) +* Add integ test for index close/open scenario ([#693](https://github.com/opensearch-project/k-NN/pull/693)) +* Add Lucene 9.5 codec and make it new default ([#700](https://github.com/opensearch-project/k-NN/pull/700)) +* Make version of lucene k-nn engine match lucene current version ([#691](https://github.com/opensearch-project/k-NN/pull/691)) +* Increment version to 2.5.0-SNAPSHOT ([#632](https://github.com/opensearch-project/k-NN/pull/632)) From 360cdfd588b0ff5d4e7886e919595897629f7ec3 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Wed, 11 Jan 2023 11:51:09 -0800 Subject: [PATCH 065/416] Fix typo in release notes (#719) Signed-off-by: John Mazanec (cherry picked from commit 4229216bf10947ad47036e0246ca2fea8f379dbb) --- release-notes/opensearch-knn.release-notes-2.5.0.0.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release-notes/opensearch-knn.release-notes-2.5.0.0.md b/release-notes/opensearch-knn.release-notes-2.5.0.0.md index e0bbd0513..397e860c3 100644 --- a/release-notes/opensearch-knn.release-notes-2.5.0.0.md +++ b/release-notes/opensearch-knn.release-notes-2.5.0.0.md @@ -23,7 +23,7 @@ Compatible with OpenSearch 2.5.0 ### Documentation -* Updat MAINTAINERS.md format ([#709](https://github.com/opensearch-project/k-NN/pull/709)) +* Update MAINTAINERS.md format ([#709](https://github.com/opensearch-project/k-NN/pull/709)) ### Maintenance From 5fbdad2cef23f7caf336c812453427197cda2f57 Mon Sep 17 00:00:00 2001 From: John Mazanec Date: Wed, 11 Jan 2023 17:02:32 -0800 Subject: [PATCH 066/416] Upgrade to 2.6.0-SNAPSHOT (#722) Signed-off-by: John Mazanec --- ...backwards_compatibility_tests_workflow.yml | 8 ++++---- build.gradle | 2 +- qa/build.gradle | 19 ++++++++++++++----- qa/restart-upgrade/build.gradle | 6 ++++-- qa/rolling-upgrade/build.gradle | 6 ++++-- 5 files changed, 27 insertions(+), 14 deletions(-) diff --git a/.github/workflows/backwards_compatibility_tests_workflow.yml b/.github/workflows/backwards_compatibility_tests_workflow.yml index dc164d4d7..4a2e50594 100644 --- a/.github/workflows/backwards_compatibility_tests_workflow.yml +++ b/.github/workflows/backwards_compatibility_tests_workflow.yml @@ -14,8 +14,8 @@ jobs: strategy: matrix: java: [ 11, 17 ] - bwc_version : [ "1.1.0", "1.2.4", "1.3.7", "2.0.1", "2.1.0", "2.2.1", "2.3.0", "2.4.1" ] - opensearch_version : [ "2.5.0-SNAPSHOT" ] + bwc_version : [ "1.1.0", "1.2.4", "1.3.7", "2.0.1", "2.1.0", "2.2.1", "2.3.0", "2.4.1", "2.5.0-SNAPSHOT" ] + opensearch_version : [ "2.6.0-SNAPSHOT" ] name: k-NN Restart-Upgrade BWC Tests runs-on: ubuntu-latest @@ -46,8 +46,8 @@ jobs: strategy: matrix: java: [ 11, 17 ] - bwc_version: [ "1.3.7", "2.0.1", "2.1.0", "2.2.1", "2.3.0", "2.4.1" ] - opensearch_version: [ "2.5.0-SNAPSHOT" ] + bwc_version: [ "1.3.7", "2.0.1", "2.1.0", "2.2.1", "2.3.0", "2.4.1", "2.5.0-SNAPSHOT" ] + opensearch_version: [ "2.6.0-SNAPSHOT" ] name: k-NN Rolling-Upgrade BWC Tests runs-on: ubuntu-latest diff --git a/build.gradle b/build.gradle index 16c57f21c..7bc3fefd9 100644 --- a/build.gradle +++ b/build.gradle @@ -10,7 +10,7 @@ buildscript { ext { // build.version_qualifier parameter applies to knn plugin artifacts only. OpenSearch version must be set // explicitly as 'opensearch.version' property, for instance opensearch.version=2.0.0-rc1-SNAPSHOT - opensearch_version = System.getProperty("opensearch.version", "2.5.0-SNAPSHOT") + opensearch_version = System.getProperty("opensearch.version", "2.6.0-SNAPSHOT") version_qualifier = System.getProperty("build.version_qualifier", "") opensearch_group = "org.opensearch" } diff --git a/qa/build.gradle b/qa/build.gradle index b292d413c..fccbf3cb1 100644 --- a/qa/build.gradle +++ b/qa/build.gradle @@ -31,6 +31,8 @@ def tmp_dir = project.file('build/private/artifact_tmp').absoluteFile tmp_dir.mkdirs() String default_bwc_version = System.getProperty("bwc.version") String knn_bwc_version = System.getProperty("tests.bwc.version", default_bwc_version) +boolean isSnapshot = knn_bwc_version.contains("-SNAPSHOT") +String knn_bwc_version_no_qualifier = isSnapshot ? knn_bwc_version - "-SNAPSHOT" : knn_bwc_version // Task to pull k-NN plugin from archive task pullBwcPlugin { @@ -39,20 +41,27 @@ task pullBwcPlugin { } doLast { + ext { + if (isSnapshot) { + srcUrl = "https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/${knn_bwc_version_no_qualifier}/latest/linux/x64/tar/dist/opensearch/opensearch-${knn_bwc_version_no_qualifier}-linux-x64.tar.gz" + } else { + srcUrl = "https://artifacts.opensearch.org/releases/bundle/opensearch/${knn_bwc_version}/opensearch-${knn_bwc_version}-linux-x64.tar.gz" + } + } ant.get( - src: "https://artifacts.opensearch.org/releases/bundle/opensearch/${knn_bwc_version}/opensearch-${knn_bwc_version}-linux-x64.tar.gz", + src: srcUrl, dest: tmp_dir.absolutePath, httpusecaches: false ) copy { - from tarTree(java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-${knn_bwc_version}-linux-x64.tar.gz")) + from tarTree(java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-${knn_bwc_version_no_qualifier}-linux-x64.tar.gz")) into tmp_dir.absolutePath } copy { - from(java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-${knn_bwc_version}", "plugins", "opensearch-knn")) + from(java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-${knn_bwc_version_no_qualifier}", "plugins", "opensearch-knn")) into java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-knn") } - delete java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-${knn_bwc_version}"), java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-${knn_bwc_version}-linux-x64.tar.gz") + delete java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-${knn_bwc_version_no_qualifier}"), java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-${knn_bwc_version_no_qualifier}-linux-x64.tar.gz") } } @@ -61,7 +70,7 @@ task zipBwcPlugin(type: Zip) { dependsOn "pullBwcPlugin" from(java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-knn")) destinationDirectory = tmp_dir - archiveFileName = "opensearch-knn-${knn_bwc_version}.zip" + archiveFileName = "opensearch-knn-${knn_bwc_version_no_qualifier}.zip" doLast { delete java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-knn") } diff --git a/qa/restart-upgrade/build.gradle b/qa/restart-upgrade/build.gradle index ea26da684..6bae754a2 100644 --- a/qa/restart-upgrade/build.gradle +++ b/qa/restart-upgrade/build.gradle @@ -9,6 +9,8 @@ apply from : "$rootDir/qa/build.gradle" String default_bwc_version = System.getProperty("bwc.version") String knn_bwc_version = System.getProperty("tests.bwc.version", default_bwc_version) +boolean isSnapshot = knn_bwc_version.contains("-SNAPSHOT") +String knn_bwc_version_no_qualifier = isSnapshot ? knn_bwc_version - "-SNAPSHOT" : knn_bwc_version String baseName = "knnBwcCluster-restart" // Creates a test cluster of previous version and loads k-NN plugin of bwcVersion @@ -20,8 +22,8 @@ testClusters { plugin(project.tasks.zipBwcPlugin.archiveFile) setting 'path.repo', "${buildDir}/cluster/shared/repo/${baseName}" setting 'http.content_type.required', 'true' - environment "LD_LIBRARY_PATH", "${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version}-ARCHIVE/plugins/opensearch-knn/knnlib;${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version}-ARCHIVE/plugins/opensearch-knn/lib" - systemProperty "java.library.path", "${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version}-ARCHIVE/plugins/opensearch-knn/knnlib:${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version}-ARCHIVE/plugins/opensearch-knn/lib" + environment "LD_LIBRARY_PATH", "${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/knnlib;${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/lib" + systemProperty "java.library.path", "${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/knnlib:${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/lib" } } diff --git a/qa/rolling-upgrade/build.gradle b/qa/rolling-upgrade/build.gradle index 158f7d4c7..3309566c0 100644 --- a/qa/rolling-upgrade/build.gradle +++ b/qa/rolling-upgrade/build.gradle @@ -9,6 +9,8 @@ apply from : "$rootDir/qa/build.gradle" String default_bwc_version = System.getProperty("bwc.version") String knn_bwc_version = System.getProperty("tests.bwc.version", default_bwc_version) +boolean isSnapshot = knn_bwc_version.contains("-SNAPSHOT") +String knn_bwc_version_no_qualifier = isSnapshot ? knn_bwc_version - "-SNAPSHOT" : knn_bwc_version String baseName = "knnBwcCluster-rolling" // Creates a test cluster of previous version and loads k-NN plugin of bwcVersion @@ -20,8 +22,8 @@ testClusters { plugin(project.tasks.zipBwcPlugin.archiveFile) setting 'path.repo', "${buildDir}/cluster/shared/repo/${baseName}" setting 'http.content_type.required', 'true' - environment "LD_LIBRARY_PATH", "${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version}-ARCHIVE/plugins/opensearch-knn/knnlib;${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version}-ARCHIVE/plugins/opensearch-knn/lib" - systemProperty "java.library.path", "${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version}-ARCHIVE/plugins/opensearch-knn/knnlib:${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version}-ARCHIVE/plugins/opensearch-knn/lib" + environment "LD_LIBRARY_PATH", "${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/knnlib;${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/lib" + systemProperty "java.library.path", "${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/knnlib:${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/lib" } } From d3d1983be0b2df60d2d1d4a3c141932484ed1919 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Wed, 11 Jan 2023 17:16:25 -0800 Subject: [PATCH 067/416] Add Lucene specific file extensions to core HybridFS (#721) (#724) * Add lucene vector specific file extensions for io with mmap Signed-off-by: Martin Gaievski (cherry picked from commit 8a2aa04d7b03edef40a7c43f00215e4bc906b082) Co-authored-by: Martin Gaievski --- .../opensearch-knn.release-notes-2.5.0.0.md | 1 + .../opensearch/knn/index/util/KNNEngine.java | 6 +++++ .../opensearch/knn/index/util/KNNLibrary.java | 11 +++++++++ .../org/opensearch/knn/index/util/Lucene.java | 6 +++++ .../org/opensearch/knn/plugin/KNNPlugin.java | 24 +++++++++++++++++++ .../knn/index/util/KNNEngineTests.java | 14 +++++++++++ .../knn/index/util/LuceneTests.java | 9 +++++++ 7 files changed, 71 insertions(+) diff --git a/release-notes/opensearch-knn.release-notes-2.5.0.0.md b/release-notes/opensearch-knn.release-notes-2.5.0.0.md index 397e860c3..25f7d3aab 100644 --- a/release-notes/opensearch-knn.release-notes-2.5.0.0.md +++ b/release-notes/opensearch-knn.release-notes-2.5.0.0.md @@ -5,6 +5,7 @@ Compatible with OpenSearch 2.5.0 ### Enhancements * Extend SystemIndexPlugin for k-NN model system index ([#630](https://github.com/opensearch-project/k-NN/pull/630)) +* Add Lucene specific file extensions to core HybridFS ([#721](https://github.com/opensearch-project/k-NN/pull/721)) ### Bug Fixes diff --git a/src/main/java/org/opensearch/knn/index/util/KNNEngine.java b/src/main/java/org/opensearch/knn/index/util/KNNEngine.java index 0751b332a..fe28de43e 100644 --- a/src/main/java/org/opensearch/knn/index/util/KNNEngine.java +++ b/src/main/java/org/opensearch/knn/index/util/KNNEngine.java @@ -12,6 +12,7 @@ import org.opensearch.knn.index.KNNMethodContext; import org.opensearch.knn.index.SpaceType; +import java.util.List; import java.util.Map; import java.util.Set; @@ -176,4 +177,9 @@ public Boolean isInitialized() { public void setInitialized(Boolean isInitialized) { knnLibrary.setInitialized(isInitialized); } + + @Override + public List mmapFileExtensions() { + return knnLibrary.mmapFileExtensions(); + } } diff --git a/src/main/java/org/opensearch/knn/index/util/KNNLibrary.java b/src/main/java/org/opensearch/knn/index/util/KNNLibrary.java index 7cb14f7f0..b990ce33b 100644 --- a/src/main/java/org/opensearch/knn/index/util/KNNLibrary.java +++ b/src/main/java/org/opensearch/knn/index/util/KNNLibrary.java @@ -16,6 +16,8 @@ import org.opensearch.knn.index.KNNMethodContext; import org.opensearch.knn.index.SpaceType; +import java.util.Collections; +import java.util.List; import java.util.Map; /** @@ -113,4 +115,13 @@ public interface KNNLibrary { * @param isInitialized whether library has been initialized */ void setInitialized(Boolean isInitialized); + + /** + * Getter for mmap file extensions + * + * @return list of file extensions that will be read/write with mmap + */ + default List mmapFileExtensions() { + return Collections.EMPTY_LIST; + } } diff --git a/src/main/java/org/opensearch/knn/index/util/Lucene.java b/src/main/java/org/opensearch/knn/index/util/Lucene.java index 83f123969..bfa6cb040 100644 --- a/src/main/java/org/opensearch/knn/index/util/Lucene.java +++ b/src/main/java/org/opensearch/knn/index/util/Lucene.java @@ -13,6 +13,7 @@ import org.opensearch.knn.index.Parameter; import org.opensearch.knn.index.SpaceType; +import java.util.List; import java.util.Map; import static org.opensearch.knn.common.KNNConstants.METHOD_HNSW; @@ -73,4 +74,9 @@ public float score(float rawScore, SpaceType spaceType) { // score provided. return rawScore; } + + @Override + public List mmapFileExtensions() { + return List.of("vec", "vex"); + } } diff --git a/src/main/java/org/opensearch/knn/plugin/KNNPlugin.java b/src/main/java/org/opensearch/knn/plugin/KNNPlugin.java index 4836c6c47..efa9065de 100644 --- a/src/main/java/org/opensearch/knn/plugin/KNNPlugin.java +++ b/src/main/java/org/opensearch/knn/plugin/KNNPlugin.java @@ -20,6 +20,7 @@ import org.opensearch.knn.index.query.KNNWeight; import org.opensearch.knn.index.codec.KNNCodecService; import org.opensearch.knn.index.memory.NativeMemoryLoadStrategy; +import org.opensearch.knn.index.util.KNNEngine; import org.opensearch.knn.indices.ModelGraveyard; import org.opensearch.knn.indices.ModelCache; import org.opensearch.knn.indices.ModelDao; @@ -104,6 +105,8 @@ import java.util.Map; import java.util.Optional; import java.util.function.Supplier; +import java.util.stream.Collectors; +import java.util.stream.Stream; import static java.util.Collections.singletonList; import static org.opensearch.knn.common.KNNConstants.KNN_THREAD_POOL_PREFIX; @@ -338,4 +341,25 @@ public List getNamedXContent() { public Collection getSystemIndexDescriptors(Settings settings) { return ImmutableList.of(new SystemIndexDescriptor(MODEL_INDEX_NAME, "Index for storing models used for k-NN indices")); } + + /** + * Plugin can provide additional node settings, that includes new settings or overrides for existing one from core. + * + * @return settings that are set by plugin + */ + @Override + public Settings additionalSettings() { + // We add engine specific extensions to the core list for HybridFS store type. We read existing values + // and append ours because in core setting will be replaced by override. + // Values are set as cluster defaults and are used at index creation time. Index specific overrides will take priority over values + // that are set here. + final List engineSettings = Arrays.stream(KNNEngine.values()) + .flatMap(engine -> engine.mmapFileExtensions().stream()) + .collect(Collectors.toList()); + final List combinedSettings = Stream.concat( + IndexModule.INDEX_STORE_HYBRID_MMAP_EXTENSIONS.getDefault(Settings.EMPTY).stream(), + engineSettings.stream() + ).collect(Collectors.toList()); + return Settings.builder().putList(IndexModule.INDEX_STORE_HYBRID_MMAP_EXTENSIONS.getKey(), combinedSettings).build(); + } } diff --git a/src/test/java/org/opensearch/knn/index/util/KNNEngineTests.java b/src/test/java/org/opensearch/knn/index/util/KNNEngineTests.java index af9feeb9e..bed0b7908 100644 --- a/src/test/java/org/opensearch/knn/index/util/KNNEngineTests.java +++ b/src/test/java/org/opensearch/knn/index/util/KNNEngineTests.java @@ -8,6 +8,10 @@ import org.opensearch.knn.KNNTestCase; import org.opensearch.knn.common.KNNConstants; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; + public class KNNEngineTests extends KNNTestCase { /** * Check that version from engine and library match @@ -47,4 +51,14 @@ public void testGetEngineFromPath() { String invalidPath = "test.invalid"; expectThrows(IllegalArgumentException.class, () -> KNNEngine.getEngineNameFromPath(invalidPath)); } + + public void testMmapFileExtensions() { + final List mmapExtensions = Arrays.stream(KNNEngine.values()) + .flatMap(engine -> engine.mmapFileExtensions().stream()) + .collect(Collectors.toList()); + assertNotNull(mmapExtensions); + final List expectedSettings = List.of("vex", "vec"); + assertTrue(expectedSettings.containsAll(mmapExtensions)); + assertTrue(mmapExtensions.containsAll(expectedSettings)); + } } diff --git a/src/test/java/org/opensearch/knn/index/util/LuceneTests.java b/src/test/java/org/opensearch/knn/index/util/LuceneTests.java index bbf43a30a..f5c8e45b9 100644 --- a/src/test/java/org/opensearch/knn/index/util/LuceneTests.java +++ b/src/test/java/org/opensearch/knn/index/util/LuceneTests.java @@ -15,6 +15,7 @@ import java.io.IOException; import java.util.Collections; +import java.util.List; import java.util.Map; import static org.opensearch.knn.common.KNNConstants.METHOD_HNSW; @@ -114,4 +115,12 @@ public void testVersion() { Lucene luceneInstance = Lucene.INSTANCE; assertEquals(Version.LATEST.toString(), luceneInstance.getVersion()); } + + public void testMmapFileExtensions() { + final List luceneMmapExtensions = Lucene.INSTANCE.mmapFileExtensions(); + assertNotNull(luceneMmapExtensions); + final List expectedSettings = List.of("vex", "vec"); + assertTrue(expectedSettings.containsAll(luceneMmapExtensions)); + assertTrue(luceneMmapExtensions.containsAll(expectedSettings)); + } } From a7766c9d543d07b070f4dbb9421845d07b7400b2 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Wed, 18 Jan 2023 09:09:04 -0800 Subject: [PATCH 068/416] Remove lucene 95 upgrade from release notes (#730) Signed-off-by: John Mazanec (cherry picked from commit 308e7f873b91e66390c2cbc37bf573e566598a16) --- release-notes/opensearch-knn.release-notes-2.5.0.0.md | 1 - 1 file changed, 1 deletion(-) diff --git a/release-notes/opensearch-knn.release-notes-2.5.0.0.md b/release-notes/opensearch-knn.release-notes-2.5.0.0.md index 25f7d3aab..138d03149 100644 --- a/release-notes/opensearch-knn.release-notes-2.5.0.0.md +++ b/release-notes/opensearch-knn.release-notes-2.5.0.0.md @@ -30,6 +30,5 @@ Compatible with OpenSearch 2.5.0 * Fix the codec94 version import statements ([#684](https://github.com/opensearch-project/k-NN/pull/684)) * Add integ test for index close/open scenario ([#693](https://github.com/opensearch-project/k-NN/pull/693)) -* Add Lucene 9.5 codec and make it new default ([#700](https://github.com/opensearch-project/k-NN/pull/700)) * Make version of lucene k-nn engine match lucene current version ([#691](https://github.com/opensearch-project/k-NN/pull/691)) * Increment version to 2.5.0-SNAPSHOT ([#632](https://github.com/opensearch-project/k-NN/pull/632)) From ed147f058454bf85e9d3c00412750c4b6e74cc48 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Thu, 19 Jan 2023 09:52:06 -0800 Subject: [PATCH 069/416] Remove latestSettings cache from KNNSettings (#733) Removes the latestSettings cache from the KNNSettings class. latestSettings cache gets updated in a consumer when the particular settings are updated. KNNSettings.getSettingValue would pull from this cache and then fallback to the default if it is not present. However, in the case when the settings are set via opensearch.yml config file, the settings update consumers never get called, so the config values never get put in the cache. This leads to getSettingValue to always return the default instead of the value specified in the config file. To fix this, this change refactors getSettingValue to pull from the cluster settings and removes the latestSetting cache. However, because the dynamicCacheSettings have consumers that rebuild the NativeMemoryCacheManager cache when they are changed, the logic for passing the parameters to rebuild this cache had to change as well. Also, switches configuration of cache manager to use DTO. Signed-off-by: John Mazanec (cherry picked from commit 8e2ad4595bd4f7c64d0b312cac30de8a42e4bf0b) --- .../org/opensearch/knn/index/KNNSettings.java | 102 +++++++----------- .../memory/NativeMemoryCacheManager.java | 54 +++++++--- .../memory/NativeMemoryCacheManagerDto.java | 18 ++++ .../java/org/opensearch/knn/KNNTestCase.java | 37 ++++++- .../knn/index/KNNSettingsTests.java | 96 +++++++++++++++++ .../knn/index/codec/KNNCodecTestCase.java | 12 +++ 6 files changed, 242 insertions(+), 77 deletions(-) create mode 100644 src/main/java/org/opensearch/knn/index/memory/NativeMemoryCacheManagerDto.java create mode 100644 src/test/java/org/opensearch/knn/index/KNNSettingsTests.java diff --git a/src/main/java/org/opensearch/knn/index/KNNSettings.java b/src/main/java/org/opensearch/knn/index/KNNSettings.java index e0e70310f..526a529f1 100644 --- a/src/main/java/org/opensearch/knn/index/KNNSettings.java +++ b/src/main/java/org/opensearch/knn/index/KNNSettings.java @@ -20,16 +20,17 @@ import org.opensearch.common.unit.TimeValue; import org.opensearch.index.IndexModule; import org.opensearch.knn.index.memory.NativeMemoryCacheManager; +import org.opensearch.knn.index.memory.NativeMemoryCacheManagerDto; import org.opensearch.monitor.jvm.JvmInfo; import org.opensearch.monitor.os.OsProbe; import java.security.InvalidParameterException; +import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.concurrent.ConcurrentHashMap; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -41,16 +42,15 @@ /** * This class defines - * 1. KNN settings to hold the HNSW algorithm parameters. - * https://github.com/nmslib/hnswlib/blob/master/ALGO_PARAMS.md + * 1. KNN settings to hold the HNSW algorithm parameters. * 2. KNN settings to enable/disable plugin, circuit breaker settings * 3. KNN settings to manage graphs loaded in native memory */ public class KNNSettings { - private static Logger logger = LogManager.getLogger(KNNSettings.class); + private static final Logger logger = LogManager.getLogger(KNNSettings.class); private static KNNSettings INSTANCE; - private static OsProbe osProbe = OsProbe.getInstance(); + private static final OsProbe osProbe = OsProbe.getInstance(); private static final int INDEX_THREAD_QTY_MAX = 32; @@ -85,6 +85,7 @@ public class KNNSettings { public static final Integer KNN_DEFAULT_CIRCUIT_BREAKER_UNSET_PERCENTAGE = 75; public static final Integer KNN_DEFAULT_MODEL_CACHE_SIZE_LIMIT_PERCENTAGE = 10; // By default, set aside 10% of the JVM for the limit public static final Integer KNN_MAX_MODEL_CACHE_SIZE_LIMIT_PERCENTAGE = 25; // Model cache limit cannot exceed 25% of the JVM heap + public static final String KNN_DEFAULT_MEMORY_CIRCUIT_BREAKER_LIMIT = "50%"; /** * Settings Definition @@ -233,7 +234,13 @@ public class KNNSettings { put(KNN_MEMORY_CIRCUIT_BREAKER_ENABLED, Setting.boolSetting(KNN_MEMORY_CIRCUIT_BREAKER_ENABLED, true, NodeScope, Dynamic)); put( KNN_MEMORY_CIRCUIT_BREAKER_LIMIT, - knnMemoryCircuitBreakerSetting(KNN_MEMORY_CIRCUIT_BREAKER_LIMIT, "50%", NodeScope, Dynamic) + new Setting<>( + KNNSettings.KNN_MEMORY_CIRCUIT_BREAKER_LIMIT, + KNNSettings.KNN_DEFAULT_MEMORY_CIRCUIT_BREAKER_LIMIT, + (s) -> parseknnMemoryCircuitBreakerValue(s, KNNSettings.KNN_MEMORY_CIRCUIT_BREAKER_LIMIT), + NodeScope, + Dynamic + ) ); /** @@ -247,9 +254,6 @@ public class KNNSettings { } }; - /** Latest setting value for each registered key. Thread-safe is required. */ - private final Map latestSettings = new ConcurrentHashMap<>(); - private ClusterService clusterService; private Client client; @@ -262,35 +266,32 @@ public static synchronized KNNSettings state() { return INSTANCE; } - public void setSettingsUpdateConsumers() { - for (Setting setting : dynamicCacheSettings.values()) { - clusterService.getClusterSettings().addSettingsUpdateConsumer(setting, newVal -> { - logger.debug("The value of setting [{}] changed to [{}]", setting.getKey(), newVal); - latestSettings.put(setting.getKey(), newVal); - - // Rebuild the cache with updated limit - NativeMemoryCacheManager.getInstance().rebuildCache(); - }); - } + private void setSettingsUpdateConsumers() { + clusterService.getClusterSettings().addSettingsUpdateConsumer(updatedSettings -> { + // When any of the dynamic settings are updated, rebuild the cache with the updated values. Use the current + // cluster settings values as defaults. + NativeMemoryCacheManagerDto.NativeMemoryCacheManagerDtoBuilder builder = NativeMemoryCacheManagerDto.builder(); - /** - * We do not have to rebuild the cache for below settings - */ - clusterService.getClusterSettings() - .addSettingsUpdateConsumer( - KNN_CIRCUIT_BREAKER_TRIGGERED_SETTING, - newVal -> { latestSettings.put(KNN_CIRCUIT_BREAKER_TRIGGERED, newVal); } + builder.isWeightLimited( + updatedSettings.getAsBoolean(KNN_MEMORY_CIRCUIT_BREAKER_ENABLED, getSettingValue(KNN_MEMORY_CIRCUIT_BREAKER_ENABLED)) ); - clusterService.getClusterSettings() - .addSettingsUpdateConsumer( - KNN_CIRCUIT_BREAKER_UNSET_PERCENTAGE_SETTING, - newVal -> { latestSettings.put(KNN_CIRCUIT_BREAKER_UNSET_PERCENTAGE, newVal); } + + builder.maxWeight(((ByteSizeValue) getSettingValue(KNN_MEMORY_CIRCUIT_BREAKER_LIMIT)).getKb()); + if (updatedSettings.hasValue(KNN_MEMORY_CIRCUIT_BREAKER_LIMIT)) { + builder.maxWeight(((ByteSizeValue) getSetting(KNN_MEMORY_CIRCUIT_BREAKER_LIMIT).get(updatedSettings)).getKb()); + } + + builder.isExpirationLimited( + updatedSettings.getAsBoolean(KNN_CACHE_ITEM_EXPIRY_ENABLED, getSettingValue(KNN_CACHE_ITEM_EXPIRY_ENABLED)) ); - clusterService.getClusterSettings() - .addSettingsUpdateConsumer( - KNN_ALGO_PARAM_INDEX_THREAD_QTY_SETTING, - newVal -> { latestSettings.put(KNN_ALGO_PARAM_INDEX_THREAD_QTY, newVal); } + + builder.expiryTimeInMin( + updatedSettings.getAsTime(KNN_CACHE_ITEM_EXPIRY_TIME_MINUTES, getSettingValue(KNN_CACHE_ITEM_EXPIRY_TIME_MINUTES)) + .getMinutes() ); + + NativeMemoryCacheManager.getInstance().rebuildCache(builder.build()); + }, new ArrayList<>(dynamicCacheSettings.values())); } /** @@ -302,10 +303,10 @@ public void setSettingsUpdateConsumers() { */ @SuppressWarnings("unchecked") public T getSettingValue(String key) { - return (T) latestSettings.getOrDefault(key, getSetting(key).getDefault(Settings.EMPTY)); + return (T) clusterService.getClusterSettings().get(getSetting(key)); } - public Setting getSetting(String key) { + private Setting getSetting(String key) { if (dynamicCacheSettings.containsKey(key)) { return dynamicCacheSettings.get(key); } @@ -364,19 +365,6 @@ public void initialize(Client client, ClusterService clusterService) { setSettingsUpdateConsumers(); } - /** - * Creates a setting which specifies a circuit breaker memory limit. This can either be - * specified as an absolute bytes value or as a percentage. - * - * @param key the key for the setting - * @param defaultValue the default value for this setting - * @param properties properties properties for this setting like scope, filtering... - * @return the setting object - */ - public static Setting knnMemoryCircuitBreakerSetting(String key, String defaultValue, Setting.Property... properties) { - return new Setting<>(key, defaultValue, (s) -> parseknnMemoryCircuitBreakerValue(s, key), properties); - } - public static ByteSizeValue parseknnMemoryCircuitBreakerValue(String sValue, String settingName) { settingName = Objects.requireNonNull(settingName); if (sValue != null && sValue.endsWith("%")) { @@ -436,24 +424,11 @@ public void onFailure(Exception e) { * @return efSearch value */ public static int getEfSearchParam(String index) { - return getIndexSettingValue(index, KNN_ALGO_PARAM_EF_SEARCH, 512); - } - - /** - * - * @param index Name of the index - * @return spaceType name in KNN plugin - */ - public static String getSpaceType(String index) { return KNNSettings.state().clusterService.state() .getMetadata() .index(index) .getSettings() - .get(KNN_SPACE_TYPE, SpaceType.DEFAULT.getValue()); - } - - public static int getIndexSettingValue(String index, String settingName, int defaultValue) { - return KNNSettings.state().clusterService.state().getMetadata().index(index).getSettings().getAsInt(settingName, defaultValue); + .getAsInt(KNNSettings.KNN_ALGO_PARAM_EF_SEARCH, 512); } public void setClusterService(ClusterService clusterService) { @@ -475,7 +450,6 @@ public void validate(String value) { public void onIndexModule(IndexModule module) { module.addSettingsUpdateConsumer(INDEX_KNN_ALGO_PARAM_EF_SEARCH_SETTING, newVal -> { logger.debug("The value of [KNN] setting [{}] changed to [{}]", KNN_ALGO_PARAM_EF_SEARCH, newVal); - latestSettings.put(KNN_ALGO_PARAM_EF_SEARCH, newVal); // TODO: replace cache-rebuild with index reload into the cache NativeMemoryCacheManager.getInstance().rebuildCache(); }); diff --git a/src/main/java/org/opensearch/knn/index/memory/NativeMemoryCacheManager.java b/src/main/java/org/opensearch/knn/index/memory/NativeMemoryCacheManager.java index efdc4fd31..8b3a3bce1 100644 --- a/src/main/java/org/opensearch/knn/index/memory/NativeMemoryCacheManager.java +++ b/src/main/java/org/opensearch/knn/index/memory/NativeMemoryCacheManager.java @@ -40,11 +40,11 @@ public class NativeMemoryCacheManager implements Closeable { public static String GRAPH_COUNT = "graph_count"; - private static Logger logger = LogManager.getLogger(NativeMemoryCacheManager.class); + private static final Logger logger = LogManager.getLogger(NativeMemoryCacheManager.class); private static NativeMemoryCacheManager INSTANCE; private Cache cache; - private ExecutorService executor; + private final ExecutorService executor; private AtomicBoolean cacheCapacityReached; private long maxWeight; @@ -68,20 +68,31 @@ public static synchronized NativeMemoryCacheManager getInstance() { } private void initialize() { + initialize( + NativeMemoryCacheManagerDto.builder() + .isWeightLimited(KNNSettings.state().getSettingValue(KNNSettings.KNN_MEMORY_CIRCUIT_BREAKER_ENABLED)) + .maxWeight(KNNSettings.getCircuitBreakerLimit().getKb()) + .isExpirationLimited(KNNSettings.state().getSettingValue(KNNSettings.KNN_CACHE_ITEM_EXPIRY_ENABLED)) + .expiryTimeInMin( + ((TimeValue) KNNSettings.state().getSettingValue(KNNSettings.KNN_CACHE_ITEM_EXPIRY_TIME_MINUTES)).getMinutes() + ) + .build() + ); + } + + private void initialize(NativeMemoryCacheManagerDto nativeMemoryCacheDTO) { CacheBuilder cacheBuilder = CacheBuilder.newBuilder() .recordStats() .concurrencyLevel(1) .removalListener(this::onRemoval); - if (KNNSettings.state().getSettingValue(KNNSettings.KNN_MEMORY_CIRCUIT_BREAKER_ENABLED)) { - maxWeight = KNNSettings.getCircuitBreakerLimit().getKb(); - cacheBuilder.maximumWeight(maxWeight).weigher((k, v) -> v.getSizeInKB()); + if (nativeMemoryCacheDTO.isWeightLimited()) { + this.maxWeight = nativeMemoryCacheDTO.getMaxWeight(); + cacheBuilder.maximumWeight(this.maxWeight).weigher((k, v) -> v.getSizeInKB()); } - if (KNNSettings.state().getSettingValue(KNNSettings.KNN_CACHE_ITEM_EXPIRY_ENABLED)) { - long expiryTime = ((TimeValue) KNNSettings.state().getSettingValue(KNNSettings.KNN_CACHE_ITEM_EXPIRY_TIME_MINUTES)) - .getMinutes(); - cacheBuilder.expireAfterAccess(expiryTime, TimeUnit.MINUTES); + if (nativeMemoryCacheDTO.isExpirationLimited()) { + cacheBuilder.expireAfterAccess(nativeMemoryCacheDTO.getExpiryTimeInMin(), TimeUnit.MINUTES); } cacheCapacityReached = new AtomicBoolean(false); @@ -93,13 +104,32 @@ private void initialize() { * Evicts all entries from the cache and rebuilds. */ public synchronized void rebuildCache() { + rebuildCache( + NativeMemoryCacheManagerDto.builder() + .isWeightLimited(KNNSettings.state().getSettingValue(KNNSettings.KNN_MEMORY_CIRCUIT_BREAKER_ENABLED)) + .maxWeight(KNNSettings.getCircuitBreakerLimit().getKb()) + .isExpirationLimited(KNNSettings.state().getSettingValue(KNNSettings.KNN_CACHE_ITEM_EXPIRY_ENABLED)) + .expiryTimeInMin( + ((TimeValue) KNNSettings.state().getSettingValue(KNNSettings.KNN_CACHE_ITEM_EXPIRY_TIME_MINUTES)).getMinutes() + ) + .build() + ); + } + + /** + * Evict all entries from the cache and rebuilds + * + * @param nativeMemoryCacheDTO DTO for cache configuration + */ + public synchronized void rebuildCache(NativeMemoryCacheManagerDto nativeMemoryCacheDTO) { logger.info("KNN Cache rebuilding."); - // TODO: Does this really need to be executed with an executor? Also, does invalidateAll really need to be - // called? + // TODO: Does this really need to be executed with an executor? executor.execute(() -> { + // Explicitly invalidate all so that we do not have to wait for garbage collection to be invoked to + // free up native memory cache.invalidateAll(); - initialize(); + initialize(nativeMemoryCacheDTO); }); } diff --git a/src/main/java/org/opensearch/knn/index/memory/NativeMemoryCacheManagerDto.java b/src/main/java/org/opensearch/knn/index/memory/NativeMemoryCacheManagerDto.java new file mode 100644 index 000000000..e5c1484ed --- /dev/null +++ b/src/main/java/org/opensearch/knn/index/memory/NativeMemoryCacheManagerDto.java @@ -0,0 +1,18 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.memory; + +import lombok.Builder; +import lombok.Value; + +@Value +@Builder +public class NativeMemoryCacheManagerDto { + boolean isWeightLimited; + long maxWeight; + boolean isExpirationLimited; + long expiryTimeInMin; +} diff --git a/src/test/java/org/opensearch/knn/KNNTestCase.java b/src/test/java/org/opensearch/knn/KNNTestCase.java index d5ac75287..f5955f959 100644 --- a/src/test/java/org/opensearch/knn/KNNTestCase.java +++ b/src/test/java/org/opensearch/knn/KNNTestCase.java @@ -5,6 +5,13 @@ package org.opensearch.knn; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.settings.ClusterSettings; +import org.opensearch.common.settings.Setting; +import org.opensearch.common.settings.Settings; +import org.opensearch.knn.index.KNNSettings; import org.opensearch.knn.index.memory.NativeMemoryCacheManager; import org.opensearch.knn.plugin.stats.KNNCounter; import org.opensearch.common.bytes.BytesReference; @@ -12,24 +19,52 @@ import org.opensearch.common.xcontent.XContentHelper; import org.opensearch.test.OpenSearchTestCase; +import java.util.HashSet; import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +import static org.mockito.Mockito.when; /** * Base class for integration tests for KNN plugin. Contains several methods for testing KNN ES functionality. */ public class KNNTestCase extends OpenSearchTestCase { + + @Mock + protected ClusterService clusterService; + private AutoCloseable openMocks; + + @Override + public void setUp() throws Exception { + super.setUp(); + openMocks = MockitoAnnotations.openMocks(this); + } + @Override public void tearDown() throws Exception { super.tearDown(); resetState(); + openMocks.close(); } - public static void resetState() { + public void resetState() { // Reset all of the counters for (KNNCounter knnCounter : KNNCounter.values()) { knnCounter.set(0L); } + Set> defaultClusterSettings = new HashSet<>(ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); + defaultClusterSettings.addAll( + KNNSettings.state() + .getSettings() + .stream() + .filter(s -> s.getProperties().contains(Setting.Property.NodeScope)) + .collect(Collectors.toList()) + ); + when(clusterService.getClusterSettings()).thenReturn(new ClusterSettings(Settings.EMPTY, defaultClusterSettings)); + KNNSettings.state().setClusterService(clusterService); + // Clean up the cache NativeMemoryCacheManager.getInstance().invalidateAll(); NativeMemoryCacheManager.getInstance().close(); diff --git a/src/test/java/org/opensearch/knn/index/KNNSettingsTests.java b/src/test/java/org/opensearch/knn/index/KNNSettingsTests.java new file mode 100644 index 000000000..724c9ad6a --- /dev/null +++ b/src/test/java/org/opensearch/knn/index/KNNSettingsTests.java @@ -0,0 +1,96 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index; + +import lombok.SneakyThrows; +import org.opensearch.cluster.ClusterName; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.network.NetworkModule; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.env.Environment; +import org.opensearch.knn.KNNTestCase; +import org.opensearch.knn.plugin.KNNPlugin; +import org.opensearch.node.MockNode; +import org.opensearch.node.Node; +import org.opensearch.plugins.Plugin; +import org.opensearch.test.InternalTestCluster; +import org.opensearch.test.MockHttpTransport; + +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +import static org.opensearch.test.NodeRoles.dataNode; + +public class KNNSettingsTests extends KNNTestCase { + + @SneakyThrows + public void testGetSettingValueFromConfig() { + long expectedKNNCircuitBreakerLimit = 13; + Node mockNode = createMockNode( + Map.of(KNNSettings.KNN_MEMORY_CIRCUIT_BREAKER_LIMIT, "\"" + expectedKNNCircuitBreakerLimit + "kb\"") + ); + mockNode.start(); + ClusterService clusterService = mockNode.injector().getInstance(ClusterService.class); + KNNSettings.state().setClusterService(clusterService); + long actualKNNCircuitBreakerLimit = ((ByteSizeValue) KNNSettings.state() + .getSettingValue(KNNSettings.KNN_MEMORY_CIRCUIT_BREAKER_LIMIT)).getKb(); + mockNode.close(); + assertEquals(expectedKNNCircuitBreakerLimit, actualKNNCircuitBreakerLimit); + } + + @SneakyThrows + public void testGetSettingValueDefault() { + Node mockNode = createMockNode(Collections.emptyMap()); + mockNode.start(); + ClusterService clusterService = mockNode.injector().getInstance(ClusterService.class); + KNNSettings.state().setClusterService(clusterService); + long actualKNNCircuitBreakerLimit = ((ByteSizeValue) KNNSettings.state() + .getSettingValue(KNNSettings.KNN_MEMORY_CIRCUIT_BREAKER_LIMIT)).getKb(); + mockNode.close(); + assertEquals( + ((ByteSizeValue) KNNSettings.dynamicCacheSettings.get(KNNSettings.KNN_MEMORY_CIRCUIT_BREAKER_LIMIT).getDefault(Settings.EMPTY)) + .getKb(), + actualKNNCircuitBreakerLimit + + ); + } + + private Node createMockNode(Map configSettings) throws IOException { + Path configDir = createTempDir(); + File configFile = configDir.resolve("opensearch.yml").toFile(); + FileWriter configFileWriter = new FileWriter(configFile); + + for (Map.Entry setting : configSettings.entrySet()) { + configFileWriter.write("\"" + setting.getKey() + "\": " + setting.getValue()); + } + configFileWriter.close(); + return new MockNode(baseSettings().build(), basePlugins(), configDir, true); + } + + private List> basePlugins() { + List> plugins = new ArrayList<>(); + plugins.add(getTestTransportPlugin()); + plugins.add(MockHttpTransport.TestPlugin.class); + plugins.add(KNNPlugin.class); + return plugins; + } + + private static Settings.Builder baseSettings() { + final Path tempDir = createTempDir(); + return Settings.builder() + .put(ClusterName.CLUSTER_NAME_SETTING.getKey(), InternalTestCluster.clusterName("single-node-cluster", randomLong())) + .put(Environment.PATH_HOME_SETTING.getKey(), tempDir) + .put(NetworkModule.TRANSPORT_TYPE_KEY, getTestTransportType()) + .put(dataNode()); + } +} diff --git a/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestCase.java b/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestCase.java index edb3ed75d..b5ef7f180 100644 --- a/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestCase.java +++ b/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestCase.java @@ -13,6 +13,7 @@ import org.apache.lucene.search.Query; import org.apache.lucene.search.TopDocs; import org.opensearch.common.settings.ClusterSettings; +import org.opensearch.common.settings.Setting; import org.opensearch.index.mapper.MapperService; import org.opensearch.knn.KNNTestCase; import org.opensearch.knn.common.KNNConstants; @@ -51,8 +52,10 @@ import java.time.ZoneOffset; import java.time.ZonedDateTime; import java.util.Arrays; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.concurrent.ExecutionException; import java.util.function.Function; import java.util.stream.Collectors; @@ -95,6 +98,15 @@ protected void setUpMockClusterService() { ClusterService clusterService = mock(ClusterService.class, RETURNS_DEEP_STUBS); Settings settings = Settings.Builder.EMPTY_SETTINGS; when(clusterService.state().getMetadata().index(Mockito.anyString()).getSettings()).thenReturn(settings); + Set> defaultClusterSettings = new HashSet<>(ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); + defaultClusterSettings.addAll( + KNNSettings.state() + .getSettings() + .stream() + .filter(s -> s.getProperties().contains(Setting.Property.NodeScope)) + .collect(Collectors.toList()) + ); + when(clusterService.getClusterSettings()).thenReturn(new ClusterSettings(Settings.EMPTY, defaultClusterSettings)); KNNSettings.state().setClusterService(clusterService); } From bc0e5d4d097d1166139f189560c639966eccb1b2 Mon Sep 17 00:00:00 2001 From: John Mazanec Date: Mon, 23 Jan 2023 12:05:28 -0800 Subject: [PATCH 070/416] Refactor structure of stats module (#737) Removes valid stats arg from KNNStatsRequest. Valid stats is a global constant so there is not a good reason to make it configurable. It is still written in and out of the stream. This is to prevent breakage in any kind of backwards compatibility. Removes KNNStatsConfig class and moves logic directly into the KNNStats class. KNNStatsConfig just hosted a single map of the stats, so it felt unneccessary to have the whole class. Signed-off-by: John Mazanec (cherry picked from commit 185b54daee1f9009c7a67dfa24adb153cfdcd160) --- .../java/org/opensearch/knn/bwc/StatsIT.java | 11 +- .../knn/index/KNNCircuitBreaker.java | 3 +- .../org/opensearch/knn/plugin/KNNPlugin.java | 5 +- .../knn/plugin/rest/RestKNNStatsHandler.java | 23 +-- .../opensearch/knn/plugin/stats/KNNStats.java | 132 +++++++++++++++--- .../knn/plugin/stats/KNNStatsConfig.java | 94 ------------- .../knn/plugin/transport/KNNStatsRequest.java | 13 +- .../plugin/action/RestKNNStatsHandlerIT.java | 3 +- .../action/RestLegacyKNNStatsHandlerIT.java | 4 +- 9 files changed, 136 insertions(+), 152 deletions(-) delete mode 100644 src/main/java/org/opensearch/knn/plugin/stats/KNNStatsConfig.java diff --git a/qa/rolling-upgrade/src/test/java/org/opensearch/knn/bwc/StatsIT.java b/qa/rolling-upgrade/src/test/java/org/opensearch/knn/bwc/StatsIT.java index cca6a45c7..25e1f030f 100644 --- a/qa/rolling-upgrade/src/test/java/org/opensearch/knn/bwc/StatsIT.java +++ b/qa/rolling-upgrade/src/test/java/org/opensearch/knn/bwc/StatsIT.java @@ -6,6 +6,7 @@ package org.opensearch.knn.bwc; import org.apache.http.util.EntityUtils; +import org.junit.Before; import org.opensearch.client.Response; import org.opensearch.client.ResponseException; import org.opensearch.knn.plugin.stats.KNNStats; @@ -15,10 +16,14 @@ import java.util.List; import java.util.Map; -import static org.opensearch.knn.plugin.stats.KNNStatsConfig.KNN_STATS; - public class StatsIT extends AbstractRollingUpgradeTestCase { - private KNNStats knnStats = new KNNStats(KNN_STATS); + private KNNStats knnStats; + + @Before + public void setUp() throws Exception { + super.setUp(); + this.knnStats = new KNNStats(); + } // Validate if all the KNN Stats metrics from old version are present in new version public void testAllMetricStatsReturned() throws IOException { diff --git a/src/main/java/org/opensearch/knn/index/KNNCircuitBreaker.java b/src/main/java/org/opensearch/knn/index/KNNCircuitBreaker.java index 4f361ae37..f5d49e1a3 100644 --- a/src/main/java/org/opensearch/knn/index/KNNCircuitBreaker.java +++ b/src/main/java/org/opensearch/knn/index/KNNCircuitBreaker.java @@ -6,7 +6,6 @@ package org.opensearch.knn.index; import org.opensearch.knn.index.memory.NativeMemoryCacheManager; -import org.opensearch.knn.plugin.stats.KNNStatsConfig; import org.opensearch.knn.plugin.stats.StatNames; import org.opensearch.knn.plugin.transport.KNNStatsAction; import org.opensearch.knn.plugin.transport.KNNStatsNodeResponse; @@ -73,7 +72,7 @@ public void initialize(ThreadPool threadPool, ClusterService clusterService, Cli // Leader node untriggers CB if all nodes have not reached their max capacity if (KNNSettings.isCircuitBreakerTriggered() && clusterService.state().nodes().isLocalNodeElectedClusterManager()) { - KNNStatsRequest knnStatsRequest = new KNNStatsRequest(KNNStatsConfig.KNN_STATS.keySet()); + KNNStatsRequest knnStatsRequest = new KNNStatsRequest(); knnStatsRequest.addStat(StatNames.CACHE_CAPACITY_REACHED.getName()); knnStatsRequest.timeout(new TimeValue(1000 * 10)); // 10 second timeout diff --git a/src/main/java/org/opensearch/knn/plugin/KNNPlugin.java b/src/main/java/org/opensearch/knn/plugin/KNNPlugin.java index efa9065de..de4932c47 100644 --- a/src/main/java/org/opensearch/knn/plugin/KNNPlugin.java +++ b/src/main/java/org/opensearch/knn/plugin/KNNPlugin.java @@ -60,7 +60,6 @@ import org.opensearch.index.IndexModule; import org.opensearch.index.IndexSettings; import org.opensearch.index.mapper.Mapper; -import org.opensearch.knn.plugin.stats.KNNStatsConfig; import org.opensearch.knn.plugin.transport.RemoveModelFromCacheAction; import org.opensearch.knn.plugin.transport.RemoveModelFromCacheTransportAction; import org.opensearch.knn.plugin.transport.SearchModelAction; @@ -202,7 +201,7 @@ public Collection createComponents( KNNQueryBuilder.initialize(ModelDao.OpenSearchKNNModelDao.getInstance()); KNNWeight.initialize(ModelDao.OpenSearchKNNModelDao.getInstance()); TrainingModelRequest.initialize(ModelDao.OpenSearchKNNModelDao.getInstance(), clusterService); - knnStats = new KNNStats(KNNStatsConfig.KNN_STATS); + knnStats = new KNNStats(); return ImmutableList.of(knnStats); } @@ -221,7 +220,7 @@ public List getRestHandlers( Supplier nodesInCluster ) { - RestKNNStatsHandler restKNNStatsHandler = new RestKNNStatsHandler(settings, restController, knnStats); + RestKNNStatsHandler restKNNStatsHandler = new RestKNNStatsHandler(); RestKNNWarmupHandler restKNNWarmupHandler = new RestKNNWarmupHandler( settings, restController, diff --git a/src/main/java/org/opensearch/knn/plugin/rest/RestKNNStatsHandler.java b/src/main/java/org/opensearch/knn/plugin/rest/RestKNNStatsHandler.java index d5e991858..3536b40fe 100644 --- a/src/main/java/org/opensearch/knn/plugin/rest/RestKNNStatsHandler.java +++ b/src/main/java/org/opensearch/knn/plugin/rest/RestKNNStatsHandler.java @@ -5,18 +5,14 @@ package org.opensearch.knn.plugin.rest; +import lombok.AllArgsConstructor; import org.opensearch.knn.plugin.KNNPlugin; -import org.opensearch.knn.plugin.stats.KNNStats; import org.opensearch.knn.plugin.transport.KNNStatsAction; import org.opensearch.knn.plugin.transport.KNNStatsRequest; import com.google.common.collect.ImmutableList; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; import org.opensearch.client.node.NodeClient; import org.opensearch.common.Strings; -import org.opensearch.common.settings.Settings; import org.opensearch.rest.BaseRestHandler; -import org.opensearch.rest.RestController; import org.opensearch.rest.RestRequest; import org.opensearch.rest.action.RestActions; @@ -33,22 +29,9 @@ * Resthandler for stats api endpoint. The user has the ability to get all stats from * all nodes or select stats from specific nodes. */ +@AllArgsConstructor public class RestKNNStatsHandler extends BaseRestHandler { - - private static final Logger LOG = LogManager.getLogger(RestKNNStatsHandler.class); private static final String NAME = "knn_stats_action"; - private KNNStats knnStats; - - /** - * Constructor - * - * @param settings Settings - * @param controller Rest Controller - * @param knnStats KNNStats - */ - public RestKNNStatsHandler(Settings settings, RestController controller, KNNStats knnStats) { - this.knnStats = knnStats; - } @Override public String getName() { @@ -104,7 +87,7 @@ private KNNStatsRequest getRequest(RestRequest request) { nodeIdsArr = nodesIdsStr.split(","); } - KNNStatsRequest knnStatsRequest = new KNNStatsRequest(knnStats.getStats().keySet(), nodeIdsArr); + KNNStatsRequest knnStatsRequest = new KNNStatsRequest(nodeIdsArr); knnStatsRequest.timeout(request.param("timeout")); // parse the stats the customer wants to see diff --git a/src/main/java/org/opensearch/knn/plugin/stats/KNNStats.java b/src/main/java/org/opensearch/knn/plugin/stats/KNNStats.java index a2b35083a..66b3f215b 100644 --- a/src/main/java/org/opensearch/knn/plugin/stats/KNNStats.java +++ b/src/main/java/org/opensearch/knn/plugin/stats/KNNStats.java @@ -5,6 +5,23 @@ package org.opensearch.knn.plugin.stats; +import com.google.common.cache.CacheStats; +import com.google.common.collect.ImmutableMap; +import org.opensearch.knn.common.KNNConstants; +import org.opensearch.knn.index.memory.NativeMemoryCacheManager; +import org.opensearch.knn.index.util.KNNEngine; +import org.opensearch.knn.indices.ModelCache; +import org.opensearch.knn.indices.ModelDao; +import org.opensearch.knn.plugin.stats.suppliers.EventOccurredWithinThresholdSupplier; +import org.opensearch.knn.plugin.stats.suppliers.KNNCircuitBreakerSupplier; +import org.opensearch.knn.plugin.stats.suppliers.KNNCounterSupplier; +import org.opensearch.knn.plugin.stats.suppliers.KNNInnerCacheStatsSupplier; +import org.opensearch.knn.plugin.stats.suppliers.LibraryInitializedSupplier; +import org.opensearch.knn.plugin.stats.suppliers.ModelIndexStatusSupplier; +import org.opensearch.knn.plugin.stats.suppliers.ModelIndexingDegradingSupplier; +import org.opensearch.knn.plugin.stats.suppliers.NativeMemoryCacheManagerSupplier; + +import java.time.temporal.ChronoUnit; import java.util.HashMap; import java.util.Map; @@ -13,15 +30,13 @@ */ public class KNNStats { - private Map> knnStats; + private final Map> knnStats; /** * Constructor - * - * @param knnStats Map that maps name of stat to KNNStat object */ - public KNNStats(Map> knnStats) { - this.knnStats = knnStats; + public KNNStats() { + this.knnStats = buildStatsMap(); } /** @@ -33,20 +48,6 @@ public Map> getStats() { return knnStats; } - /** - * Get individual stat by stat name - * - * @param key Name of stat - * @return ADStat - * @throws IllegalArgumentException thrown on illegal statName - */ - public KNNStat getStat(String key) throws IllegalArgumentException { - if (!knnStats.keySet().contains(key)) { - throw new IllegalArgumentException("Stat=\"" + key + "\" does not exist"); - } - return knnStats.get(key); - } - /** * Get a map of the stats that are kept at the node level * @@ -75,4 +76,97 @@ private Map> getClusterOrNodeStats(Boolean getClusterStats) { } return statsMap; } + + private Map> buildStatsMap() { + ImmutableMap.Builder> builder = ImmutableMap.>builder(); + addQueryStats(builder); + addNativeMemoryStats(builder); + addEngineStats(builder); + addScriptStats(builder); + addModelStats(builder); + return builder.build(); + } + + private void addQueryStats(ImmutableMap.Builder> builder) { + builder.put(StatNames.KNN_QUERY_REQUESTS.getName(), new KNNStat<>(false, new KNNCounterSupplier(KNNCounter.KNN_QUERY_REQUESTS))) + .put( + StatNames.KNN_QUERY_WITH_FILTER_REQUESTS.getName(), + new KNNStat<>(false, new KNNCounterSupplier(KNNCounter.KNN_QUERY_WITH_FILTER_REQUESTS)) + ); + + } + + private void addNativeMemoryStats(ImmutableMap.Builder> builder) { + builder.put(StatNames.HIT_COUNT.getName(), new KNNStat<>(false, new KNNInnerCacheStatsSupplier(CacheStats::hitCount))) + .put(StatNames.MISS_COUNT.getName(), new KNNStat<>(false, new KNNInnerCacheStatsSupplier(CacheStats::missCount))) + .put(StatNames.LOAD_SUCCESS_COUNT.getName(), new KNNStat<>(false, new KNNInnerCacheStatsSupplier(CacheStats::loadSuccessCount))) + .put( + StatNames.LOAD_EXCEPTION_COUNT.getName(), + new KNNStat<>(false, new KNNInnerCacheStatsSupplier(CacheStats::loadExceptionCount)) + ) + .put(StatNames.TOTAL_LOAD_TIME.getName(), new KNNStat<>(false, new KNNInnerCacheStatsSupplier(CacheStats::totalLoadTime))) + .put(StatNames.EVICTION_COUNT.getName(), new KNNStat<>(false, new KNNInnerCacheStatsSupplier(CacheStats::evictionCount))) + .put( + StatNames.GRAPH_MEMORY_USAGE.getName(), + new KNNStat<>(false, new NativeMemoryCacheManagerSupplier<>(NativeMemoryCacheManager::getIndicesSizeInKilobytes)) + ) + .put( + StatNames.GRAPH_MEMORY_USAGE_PERCENTAGE.getName(), + new KNNStat<>(false, new NativeMemoryCacheManagerSupplier<>(NativeMemoryCacheManager::getIndicesSizeAsPercentage)) + ) + .put( + StatNames.INDICES_IN_CACHE.getName(), + new KNNStat<>(false, new NativeMemoryCacheManagerSupplier<>(NativeMemoryCacheManager::getIndicesCacheStats)) + ) + .put( + StatNames.CACHE_CAPACITY_REACHED.getName(), + new KNNStat<>(false, new NativeMemoryCacheManagerSupplier<>(NativeMemoryCacheManager::isCacheCapacityReached)) + ) + .put(StatNames.GRAPH_QUERY_ERRORS.getName(), new KNNStat<>(false, new KNNCounterSupplier(KNNCounter.GRAPH_QUERY_ERRORS))) + .put(StatNames.GRAPH_QUERY_REQUESTS.getName(), new KNNStat<>(false, new KNNCounterSupplier(KNNCounter.GRAPH_QUERY_REQUESTS))) + .put(StatNames.GRAPH_INDEX_ERRORS.getName(), new KNNStat<>(false, new KNNCounterSupplier(KNNCounter.GRAPH_INDEX_ERRORS))) + .put(StatNames.GRAPH_INDEX_REQUESTS.getName(), new KNNStat<>(false, new KNNCounterSupplier(KNNCounter.GRAPH_INDEX_REQUESTS))) + .put(StatNames.CIRCUIT_BREAKER_TRIGGERED.getName(), new KNNStat<>(true, new KNNCircuitBreakerSupplier())); + } + + private void addEngineStats(ImmutableMap.Builder> builder) { + builder.put(StatNames.FAISS_LOADED.getName(), new KNNStat<>(false, new LibraryInitializedSupplier(KNNEngine.FAISS))) + .put(StatNames.NMSLIB_LOADED.getName(), new KNNStat<>(false, new LibraryInitializedSupplier(KNNEngine.NMSLIB))) + .put(StatNames.LUCENE_LOADED.getName(), new KNNStat<>(false, new LibraryInitializedSupplier(KNNEngine.LUCENE))); + } + + private void addScriptStats(ImmutableMap.Builder> builder) { + builder.put(StatNames.SCRIPT_COMPILATIONS.getName(), new KNNStat<>(false, new KNNCounterSupplier(KNNCounter.SCRIPT_COMPILATIONS))) + .put( + StatNames.SCRIPT_COMPILATION_ERRORS.getName(), + new KNNStat<>(false, new KNNCounterSupplier(KNNCounter.SCRIPT_COMPILATION_ERRORS)) + ) + .put(StatNames.SCRIPT_QUERY_REQUESTS.getName(), new KNNStat<>(false, new KNNCounterSupplier(KNNCounter.SCRIPT_QUERY_REQUESTS))) + .put(StatNames.SCRIPT_QUERY_ERRORS.getName(), new KNNStat<>(false, new KNNCounterSupplier(KNNCounter.SCRIPT_QUERY_ERRORS))); + } + + private void addModelStats(ImmutableMap.Builder> builder) { + builder.put( + StatNames.INDEXING_FROM_MODEL_DEGRADED.getName(), + new KNNStat<>( + false, + new EventOccurredWithinThresholdSupplier( + new ModelIndexingDegradingSupplier(ModelCache::getEvictedDueToSizeAt), + KNNConstants.MODEL_CACHE_CAPACITY_ATROPHY_THRESHOLD_IN_MINUTES, + ChronoUnit.MINUTES + ) + ) + ) + .put(StatNames.MODEL_INDEX_STATUS.getName(), new KNNStat<>(true, new ModelIndexStatusSupplier<>(ModelDao::getHealthStatus))) + .put(StatNames.TRAINING_REQUESTS.getName(), new KNNStat<>(false, new KNNCounterSupplier(KNNCounter.TRAINING_REQUESTS))) + .put(StatNames.TRAINING_ERRORS.getName(), new KNNStat<>(false, new KNNCounterSupplier(KNNCounter.TRAINING_ERRORS))) + .put( + StatNames.TRAINING_MEMORY_USAGE.getName(), + new KNNStat<>(false, new NativeMemoryCacheManagerSupplier<>(NativeMemoryCacheManager::getTrainingSizeInKilobytes)) + ) + .put( + StatNames.TRAINING_MEMORY_USAGE_PERCENTAGE.getName(), + new KNNStat<>(false, new NativeMemoryCacheManagerSupplier<>(NativeMemoryCacheManager::getTrainingSizeAsPercentage)) + ); + } } diff --git a/src/main/java/org/opensearch/knn/plugin/stats/KNNStatsConfig.java b/src/main/java/org/opensearch/knn/plugin/stats/KNNStatsConfig.java deleted file mode 100644 index 8769e0e46..000000000 --- a/src/main/java/org/opensearch/knn/plugin/stats/KNNStatsConfig.java +++ /dev/null @@ -1,94 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.knn.plugin.stats; - -import com.google.common.cache.CacheStats; -import com.google.common.collect.ImmutableMap; -import org.opensearch.knn.common.KNNConstants; -import org.opensearch.knn.index.memory.NativeMemoryCacheManager; -import org.opensearch.knn.index.util.KNNEngine; -import org.opensearch.knn.indices.ModelCache; -import org.opensearch.knn.indices.ModelDao; -import org.opensearch.knn.plugin.stats.suppliers.LibraryInitializedSupplier; -import org.opensearch.knn.plugin.stats.suppliers.EventOccurredWithinThresholdSupplier; -import org.opensearch.knn.plugin.stats.suppliers.KNNCircuitBreakerSupplier; -import org.opensearch.knn.plugin.stats.suppliers.KNNCounterSupplier; -import org.opensearch.knn.plugin.stats.suppliers.KNNInnerCacheStatsSupplier; -import org.opensearch.knn.plugin.stats.suppliers.ModelIndexStatusSupplier; -import org.opensearch.knn.plugin.stats.suppliers.ModelIndexingDegradingSupplier; -import org.opensearch.knn.plugin.stats.suppliers.NativeMemoryCacheManagerSupplier; - -import java.time.temporal.ChronoUnit; -import java.util.Map; - -public class KNNStatsConfig { - public static Map> KNN_STATS = ImmutableMap.>builder() - .put(StatNames.HIT_COUNT.getName(), new KNNStat<>(false, new KNNInnerCacheStatsSupplier(CacheStats::hitCount))) - .put(StatNames.MISS_COUNT.getName(), new KNNStat<>(false, new KNNInnerCacheStatsSupplier(CacheStats::missCount))) - .put(StatNames.LOAD_SUCCESS_COUNT.getName(), new KNNStat<>(false, new KNNInnerCacheStatsSupplier(CacheStats::loadSuccessCount))) - .put(StatNames.LOAD_EXCEPTION_COUNT.getName(), new KNNStat<>(false, new KNNInnerCacheStatsSupplier(CacheStats::loadExceptionCount))) - .put(StatNames.TOTAL_LOAD_TIME.getName(), new KNNStat<>(false, new KNNInnerCacheStatsSupplier(CacheStats::totalLoadTime))) - .put(StatNames.EVICTION_COUNT.getName(), new KNNStat<>(false, new KNNInnerCacheStatsSupplier(CacheStats::evictionCount))) - .put( - StatNames.GRAPH_MEMORY_USAGE.getName(), - new KNNStat<>(false, new NativeMemoryCacheManagerSupplier<>(NativeMemoryCacheManager::getIndicesSizeInKilobytes)) - ) - .put( - StatNames.GRAPH_MEMORY_USAGE_PERCENTAGE.getName(), - new KNNStat<>(false, new NativeMemoryCacheManagerSupplier<>(NativeMemoryCacheManager::getIndicesSizeAsPercentage)) - ) - .put( - StatNames.INDICES_IN_CACHE.getName(), - new KNNStat<>(false, new NativeMemoryCacheManagerSupplier<>(NativeMemoryCacheManager::getIndicesCacheStats)) - ) - .put( - StatNames.CACHE_CAPACITY_REACHED.getName(), - new KNNStat<>(false, new NativeMemoryCacheManagerSupplier<>(NativeMemoryCacheManager::isCacheCapacityReached)) - ) - .put(StatNames.GRAPH_QUERY_ERRORS.getName(), new KNNStat<>(false, new KNNCounterSupplier(KNNCounter.GRAPH_QUERY_ERRORS))) - .put(StatNames.GRAPH_QUERY_REQUESTS.getName(), new KNNStat<>(false, new KNNCounterSupplier(KNNCounter.GRAPH_QUERY_REQUESTS))) - .put(StatNames.GRAPH_INDEX_ERRORS.getName(), new KNNStat<>(false, new KNNCounterSupplier(KNNCounter.GRAPH_INDEX_ERRORS))) - .put(StatNames.GRAPH_INDEX_REQUESTS.getName(), new KNNStat<>(false, new KNNCounterSupplier(KNNCounter.GRAPH_INDEX_REQUESTS))) - .put(StatNames.CIRCUIT_BREAKER_TRIGGERED.getName(), new KNNStat<>(true, new KNNCircuitBreakerSupplier())) - .put(StatNames.MODEL_INDEX_STATUS.getName(), new KNNStat<>(true, new ModelIndexStatusSupplier<>(ModelDao::getHealthStatus))) - .put(StatNames.KNN_QUERY_REQUESTS.getName(), new KNNStat<>(false, new KNNCounterSupplier(KNNCounter.KNN_QUERY_REQUESTS))) - .put( - StatNames.KNN_QUERY_WITH_FILTER_REQUESTS.getName(), - new KNNStat<>(false, new KNNCounterSupplier(KNNCounter.KNN_QUERY_WITH_FILTER_REQUESTS)) - ) - .put(StatNames.SCRIPT_COMPILATIONS.getName(), new KNNStat<>(false, new KNNCounterSupplier(KNNCounter.SCRIPT_COMPILATIONS))) - .put( - StatNames.SCRIPT_COMPILATION_ERRORS.getName(), - new KNNStat<>(false, new KNNCounterSupplier(KNNCounter.SCRIPT_COMPILATION_ERRORS)) - ) - .put(StatNames.SCRIPT_QUERY_REQUESTS.getName(), new KNNStat<>(false, new KNNCounterSupplier(KNNCounter.SCRIPT_QUERY_REQUESTS))) - .put(StatNames.SCRIPT_QUERY_ERRORS.getName(), new KNNStat<>(false, new KNNCounterSupplier(KNNCounter.SCRIPT_QUERY_ERRORS))) - .put( - StatNames.INDEXING_FROM_MODEL_DEGRADED.getName(), - new KNNStat<>( - false, - new EventOccurredWithinThresholdSupplier( - new ModelIndexingDegradingSupplier(ModelCache::getEvictedDueToSizeAt), - KNNConstants.MODEL_CACHE_CAPACITY_ATROPHY_THRESHOLD_IN_MINUTES, - ChronoUnit.MINUTES - ) - ) - ) - .put(StatNames.FAISS_LOADED.getName(), new KNNStat<>(false, new LibraryInitializedSupplier(KNNEngine.FAISS))) - .put(StatNames.NMSLIB_LOADED.getName(), new KNNStat<>(false, new LibraryInitializedSupplier(KNNEngine.NMSLIB))) - .put(StatNames.LUCENE_LOADED.getName(), new KNNStat<>(false, new LibraryInitializedSupplier(KNNEngine.LUCENE))) - .put(StatNames.TRAINING_REQUESTS.getName(), new KNNStat<>(false, new KNNCounterSupplier(KNNCounter.TRAINING_REQUESTS))) - .put(StatNames.TRAINING_ERRORS.getName(), new KNNStat<>(false, new KNNCounterSupplier(KNNCounter.TRAINING_ERRORS))) - .put( - StatNames.TRAINING_MEMORY_USAGE.getName(), - new KNNStat<>(false, new NativeMemoryCacheManagerSupplier<>(NativeMemoryCacheManager::getTrainingSizeInKilobytes)) - ) - .put( - StatNames.TRAINING_MEMORY_USAGE_PERCENTAGE.getName(), - new KNNStat<>(false, new NativeMemoryCacheManagerSupplier<>(NativeMemoryCacheManager::getTrainingSizeAsPercentage)) - ) - .build(); -} diff --git a/src/main/java/org/opensearch/knn/plugin/transport/KNNStatsRequest.java b/src/main/java/org/opensearch/knn/plugin/transport/KNNStatsRequest.java index 500c36203..26d8a9c5b 100644 --- a/src/main/java/org/opensearch/knn/plugin/transport/KNNStatsRequest.java +++ b/src/main/java/org/opensearch/knn/plugin/transport/KNNStatsRequest.java @@ -8,6 +8,7 @@ import org.opensearch.action.support.nodes.BaseNodesRequest; import org.opensearch.common.io.stream.StreamInput; import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.knn.plugin.stats.StatNames; import java.io.IOException; import java.util.HashSet; @@ -23,15 +24,16 @@ public class KNNStatsRequest extends BaseNodesRequest { * Key indicating all stats should be retrieved */ public static final String ALL_STATS_KEY = "_all"; - - private Set validStats; - private Set statsToBeRetrieved; + private final Set validStats; + private final Set statsToBeRetrieved; /** * Empty constructor needed for KNNStatsTransportAction */ public KNNStatsRequest() { super((String[]) null); + validStats = StatNames.getNames(); + statsToBeRetrieved = new HashSet<>(); } /** @@ -49,12 +51,11 @@ public KNNStatsRequest(StreamInput in) throws IOException { /** * Constructor * - * @param validStats set of stat names that are valid for KNN plugin * @param nodeIds NodeIDs from which to retrieve stats */ - public KNNStatsRequest(Set validStats, String... nodeIds) { + public KNNStatsRequest(String... nodeIds) { super(nodeIds); - this.validStats = validStats; + validStats = StatNames.getNames(); statsToBeRetrieved = new HashSet<>(); } diff --git a/src/test/java/org/opensearch/knn/plugin/action/RestKNNStatsHandlerIT.java b/src/test/java/org/opensearch/knn/plugin/action/RestKNNStatsHandlerIT.java index 0c51e77e1..d89b78d24 100644 --- a/src/test/java/org/opensearch/knn/plugin/action/RestKNNStatsHandlerIT.java +++ b/src/test/java/org/opensearch/knn/plugin/action/RestKNNStatsHandlerIT.java @@ -51,7 +51,6 @@ import static org.opensearch.knn.common.KNNConstants.NAME; import static org.opensearch.knn.common.KNNConstants.NMSLIB_NAME; import static org.opensearch.knn.common.KNNConstants.PARAMETERS; -import static org.opensearch.knn.plugin.stats.KNNStatsConfig.KNN_STATS; /** * Integration tests to check the correctness of RestKNNStatsHandler @@ -79,7 +78,7 @@ public class RestKNNStatsHandlerIT extends KNNRestTestCase { @Before public void setup() { - knnStats = new KNNStats(KNN_STATS); + knnStats = new KNNStats(); } /** diff --git a/src/test/java/org/opensearch/knn/plugin/action/RestLegacyKNNStatsHandlerIT.java b/src/test/java/org/opensearch/knn/plugin/action/RestLegacyKNNStatsHandlerIT.java index f946b2473..a4243537d 100644 --- a/src/test/java/org/opensearch/knn/plugin/action/RestLegacyKNNStatsHandlerIT.java +++ b/src/test/java/org/opensearch/knn/plugin/action/RestLegacyKNNStatsHandlerIT.java @@ -38,8 +38,6 @@ import java.util.List; import java.util.Map; -import static org.opensearch.knn.plugin.stats.KNNStatsConfig.KNN_STATS; - /** * Integration tests to check the correctness of Legacy stats api */ @@ -53,7 +51,7 @@ public class RestLegacyKNNStatsHandlerIT extends KNNRestTestCase { @Before public void setup() { - knnStats = new KNNStats(KNN_STATS); + knnStats = new KNNStats(); } /** From 9aa95694ac0bb2949d17f2e5e1a5345a06a602b6 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Tue, 24 Jan 2023 21:59:10 -0600 Subject: [PATCH 071/416] Update certifi dependency version (#742) (#743) Signed-off-by: Naveen Tatikonda Signed-off-by: Naveen Tatikonda (cherry picked from commit 46694f5a435dffa6a55a1b6c21dd9fe87a0c2bd3) Co-authored-by: Naveen Tatikonda --- benchmarks/osb/requirements.txt | 2 +- benchmarks/perf-tool/requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/benchmarks/osb/requirements.txt b/benchmarks/osb/requirements.txt index 271e8ab07..c3f29af48 100644 --- a/benchmarks/osb/requirements.txt +++ b/benchmarks/osb/requirements.txt @@ -16,7 +16,7 @@ attrs==21.4.0 # jsonschema cachetools==4.2.4 # via google-auth -certifi==2021.10.8 +certifi==2022.12.7 # via # opensearch-benchmark # opensearch-py diff --git a/benchmarks/perf-tool/requirements.txt b/benchmarks/perf-tool/requirements.txt index 2886795dc..c5bb0f7d3 100644 --- a/benchmarks/perf-tool/requirements.txt +++ b/benchmarks/perf-tool/requirements.txt @@ -8,7 +8,7 @@ cached-property==1.5.2 # via h5py cerberus==1.3.4 # via -r requirements.in -certifi==2021.5.30 +certifi==2022.12.7 # via # opensearch-py # requests From cedd5ecf387d390ced6084919fbdd235a782f3f9 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Wed, 25 Jan 2023 11:32:08 -0800 Subject: [PATCH 072/416] Adding p99.9, p100 and num_of_segments metrics to perf-tool (#739) (#744) * Adding p99.9, p100 and num_of_segments metrics to perf-tool Signed-off-by: Martin Gaievski (cherry picked from commit 4de9f83cbc35ca9a21bc7326c3254a4b40b233ec) Co-authored-by: Martin Gaievski --- benchmarks/perf-tool/README.md | 27 +++++++++--- .../perf-tool/okpt/test/steps/factory.py | 5 ++- benchmarks/perf-tool/okpt/test/steps/steps.py | 41 ++++++++++++++++++- benchmarks/perf-tool/okpt/test/test.py | 10 ++++- 4 files changed, 75 insertions(+), 8 deletions(-) diff --git a/benchmarks/perf-tool/README.md b/benchmarks/perf-tool/README.md index d4404f551..9c1c18918 100644 --- a/benchmarks/perf-tool/README.md +++ b/benchmarks/perf-tool/README.md @@ -272,12 +272,12 @@ Runs a set of queries against an index. ##### Metrics -| Metric Name | Description | Unit | -| ----------- | ----------- | ----------- | -| took | Took times returned per query aggregated as total, p50, p90 and p99 (when applicable) | ms | -| memory_kb | Native memory k-NN is using at the end of the query workload | KB | +| Metric Name | Description | Unit | +| ----------- |---------------------------------------------------------------------------------------------------------| ----------- | +| took | Took times returned per query aggregated as total, p50, p90, p99, p99.9 and p100 (when applicable) | ms | +| memory_kb | Native memory k-NN is using at the end of the query workload | KB | | recall@R | ratio of top R results from the ground truth neighbors that are in the K results returned by the plugin | float 0.0-1.0 | -| recall@K | ratio of results returned that were ground truth nearest neighbors | float 0.0-1.0 | +| recall@K | ratio of results returned that were ground truth nearest neighbors | float 0.0-1.0 | #### query_with_filter @@ -311,6 +311,23 @@ Runs a set of queries with filter against an index. | recall@R | ratio of top R results from the ground truth neighbors that are in the K results returned by the plugin | float 0.0-1.0 | | recall@K | ratio of results returned that were ground truth nearest neighbors | float 0.0-1.0 | +#### get_stats + +Gets the index stats. + +##### Parameters + +| Parameter Name | Description | Default | +| ----------- |-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------------------| +| index_name | Name of index to search | No default | + +##### Metrics + +| Metric Name | Description | Unit | +| ----------- |-------------------------------------------------|------------| +| num_of_committed_segments | Total number of commited segments in the index | integer >= 0 | +| num_of_search_segments | Total number of search segments in the index | integer >= 0 | + ### Data sets This benchmark tool uses pre-generated data sets to run indexing and query workload. For some benchmark types existing dataset need to be diff --git a/benchmarks/perf-tool/okpt/test/steps/factory.py b/benchmarks/perf-tool/okpt/test/steps/factory.py index 2e53b4d4d..ba0fc5b60 100644 --- a/benchmarks/perf-tool/okpt/test/steps/factory.py +++ b/benchmarks/perf-tool/okpt/test/steps/factory.py @@ -9,7 +9,8 @@ from okpt.test.steps.base import Step, StepConfig from okpt.test.steps.steps import CreateIndexStep, DisableRefreshStep, RefreshIndexStep, DeleteIndexStep, \ - TrainModelStep, DeleteModelStep, ForceMergeStep, ClearCacheStep, IngestStep, IngestMultiFieldStep, QueryStep, QueryWithFilterStep + TrainModelStep, DeleteModelStep, ForceMergeStep, ClearCacheStep, IngestStep, IngestMultiFieldStep, \ + QueryStep, QueryWithFilterStep, GetStatsStep def create_step(step_config: StepConfig) -> Step: @@ -37,5 +38,7 @@ def create_step(step_config: StepConfig) -> Step: return ForceMergeStep(step_config) elif step_config.step_name == ClearCacheStep.label: return ClearCacheStep(step_config) + elif step_config.step_name == GetStatsStep.label: + return GetStatsStep(step_config) raise ConfigurationError(f'Invalid step {step_config.step_name}') diff --git a/benchmarks/perf-tool/okpt/test/steps/steps.py b/benchmarks/perf-tool/okpt/test/steps/steps.py index bc43bf195..0de61078f 100644 --- a/benchmarks/perf-tool/okpt/test/steps/steps.py +++ b/benchmarks/perf-tool/okpt/test/steps/steps.py @@ -454,7 +454,8 @@ def _action(self): results['took'] = [ float(query_response['took']) for query_response in query_responses ] - results['memory_kb'] = get_cache_size_in_kb(self.endpoint, 80) + port = 9200 if self.endpoint == 'localhost' else 80 + results['memory_kb'] = get_cache_size_in_kb(self.endpoint, port) if self.calculate_recall: ids = [[int(hit['_id']) @@ -588,6 +589,41 @@ def get_body(self, vec): else: raise ConfigurationError('Not supported filter type {}'.format(self.filter_type)) +class GetStatsStep(OpenSearchStep): + """See base class.""" + + label = 'get_stats' + + def __init__(self, step_config: StepConfig): + super().__init__(step_config) + + self.index_name = parse_string_param('index_name', step_config.config, + {}, None) + + def _action(self): + """Get stats for cluster/index etc. + + Returns: + Stats with following info: + - number of committed and search segments in the index + """ + results = {} + segment_stats = get_segment_stats(self.opensearch, self.index_name) + shards = segment_stats["indices"][self.index_name]["shards"] + num_of_committed_segments = 0 + num_of_search_segments = 0; + for shard_key in shards.keys(): + for segment in shards[shard_key]: + + num_of_committed_segments += segment["num_committed_segments"] + num_of_search_segments += segment["num_search_segments"] + + results['committed_segments'] = num_of_committed_segments + results['search_segments'] = num_of_search_segments + return results + + def _get_measures(self) -> List[str]: + return ['committed_segments', 'search_segments'] # Helper functions - (AKA not steps) def bulk_transform(partition: np.ndarray, field_name: str, action, @@ -755,3 +791,6 @@ def query_index(opensearch: OpenSearch, index_name: str, body: dict, def bulk_index(opensearch: OpenSearch, index_name: str, body: List): return opensearch.bulk(index=index_name, body=body, timeout='5m') + +def get_segment_stats(opensearch: OpenSearch, index_name: str): + return opensearch.indices.segments(index=index_name) diff --git a/benchmarks/perf-tool/okpt/test/test.py b/benchmarks/perf-tool/okpt/test/test.py index dbd65d053..c947545ad 100644 --- a/benchmarks/perf-tool/okpt/test/test.py +++ b/benchmarks/perf-tool/okpt/test/test.py @@ -53,6 +53,8 @@ def _pxx(values: List[Any], p: float): if p < 0 or p > 1: return -1.0 elif p < lowest_percentile or p > highest_percentile: + if p == 1.0 and len(values) > 1: + return float(values[len(values) - 1]) return -1.0 else: return float(values[floor(len(values) * p)]) @@ -105,7 +107,7 @@ def _aggregate_steps(step_results: List[Dict[str, Any]], for measure_label in step_measure_labels: step_measure = step[measure_label] - step_measure_label = f'{step_label}_{measure_label}' + step_measure_label = f'{measure_label}' if step_label == 'get_stats' else f'{step_label}_{measure_label}' # Add cumulative test measures from steps to test measures if measure_label in measure_labels: @@ -137,6 +139,12 @@ def _aggregate_steps(step_results: List[Dict[str, Any]], p99 = _pxx(step_measure, 0.99) if p99 != -1: aggregate[step_measure_label + '_p99'] = p99 + p99_9 = _pxx(step_measure, 0.999) + if p99_9 != -1: + aggregate[step_measure_label + '_p99.9'] = p99_9 + p100 = _pxx(step_measure, 1.00) + if p100 != -1: + aggregate[step_measure_label + '_p100'] = p100 return aggregate From 2a40da4d5a386e44014901e93de28d0f83e8595e Mon Sep 17 00:00:00 2001 From: Naveen Tatikonda Date: Thu, 2 Feb 2023 19:09:07 -0600 Subject: [PATCH 073/416] Update codec version to 9.5 (#747) * Update lucene94 package Signed-off-by: Naveen Tatikonda * Add Lucene 9.5 codec and make it new default (#700) * Add Lucene 9.5 codec and make it new default Signed-off-by: Martin Gaievski * Update tests for backwards codecs (#710) Updates tests for backwards codecs to prevent backwards codecs from writing with a read only codec. Signed-off-by: John Mazanec --------- Signed-off-by: Naveen Tatikonda Signed-off-by: Martin Gaievski Signed-off-by: John Mazanec Co-authored-by: Martin Gaievski Co-authored-by: John Mazanec --- DEVELOPER_GUIDE.md | 4 +- .../KNN940PerFieldKnnVectorsFormat.java | 2 +- .../index/codec/KNN950Codec/KNN950Codec.java | 58 +++++++++++++++++++ .../KNN950PerFieldKnnVectorsFormat.java | 28 +++++++++ .../knn/index/codec/KNNCodecVersion.java | 22 ++++++- .../services/org.apache.lucene.codecs.Codec | 3 +- .../codec/KNN940Codec/KNN940CodecTests.java | 18 ++---- .../codec/KNN950Codec/KNN950CodecTests.java | 51 ++++++++++++++++ .../knn/index/codec/KNNCodecFactoryTests.java | 10 +++- 9 files changed, 176 insertions(+), 20 deletions(-) create mode 100644 src/main/java/org/opensearch/knn/index/codec/KNN950Codec/KNN950Codec.java create mode 100644 src/main/java/org/opensearch/knn/index/codec/KNN950Codec/KNN950PerFieldKnnVectorsFormat.java create mode 100644 src/test/java/org/opensearch/knn/index/codec/KNN950Codec/KNN950CodecTests.java diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index 1a47abef8..4a6f360b5 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -286,10 +286,12 @@ Before adding any new tests to Backward Compatibility Tests, we should be aware Starting from 2.0 release the new versioning for codec has been introduced. Two positions will be used to define the version, in format 'X.Y', where 'X' corresponds to underlying version of Lucene and 'Y' is the version of the format. +Please note that Lucene version along with corresponding Lucene codec is part of the core OpenSearch. KNN codec should be in sync with Lucene codec version from core OpenSearch. Codec version is used in following classes and methods: - org.opensearch.knn.index.codec.KNNXYCodec.KNNXYCodec -- org.opensearch.knn.index.codec.KNNFormatFactory.createKNNXYFormat +- org.opensearch.knn.index.codec.KNNXYCodec.KNNXYPerFieldKnnVectorsFormat +- org.opensearch.knn.index.codec.KNNCodecVersion These classes and methods are tied directly to Lucene version represented by 'X' part. Other classes use the delegate pattern so no direct tie to Lucene version are related to format and represented by 'Y' diff --git a/src/main/java/org/opensearch/knn/index/codec/KNN940Codec/KNN940PerFieldKnnVectorsFormat.java b/src/main/java/org/opensearch/knn/index/codec/KNN940Codec/KNN940PerFieldKnnVectorsFormat.java index d80c757c9..d9a1a9251 100644 --- a/src/main/java/org/opensearch/knn/index/codec/KNN940Codec/KNN940PerFieldKnnVectorsFormat.java +++ b/src/main/java/org/opensearch/knn/index/codec/KNN940Codec/KNN940PerFieldKnnVectorsFormat.java @@ -5,7 +5,7 @@ package org.opensearch.knn.index.codec.KNN940Codec; -import org.apache.lucene.codecs.lucene94.Lucene94HnswVectorsFormat; +import org.apache.lucene.backward_codecs.lucene94.Lucene94HnswVectorsFormat; import org.opensearch.index.mapper.MapperService; import org.opensearch.knn.index.codec.BasePerFieldKnnVectorsFormat; diff --git a/src/main/java/org/opensearch/knn/index/codec/KNN950Codec/KNN950Codec.java b/src/main/java/org/opensearch/knn/index/codec/KNN950Codec/KNN950Codec.java new file mode 100644 index 000000000..338e54451 --- /dev/null +++ b/src/main/java/org/opensearch/knn/index/codec/KNN950Codec/KNN950Codec.java @@ -0,0 +1,58 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.codec.KNN950Codec; + +import lombok.Builder; +import org.apache.lucene.codecs.Codec; +import org.apache.lucene.codecs.CompoundFormat; +import org.apache.lucene.codecs.DocValuesFormat; +import org.apache.lucene.codecs.FilterCodec; +import org.apache.lucene.codecs.KnnVectorsFormat; +import org.apache.lucene.codecs.perfield.PerFieldKnnVectorsFormat; +import org.opensearch.knn.index.codec.KNNCodecVersion; +import org.opensearch.knn.index.codec.KNNFormatFacade; + +public class KNN950Codec extends FilterCodec { + private static final KNNCodecVersion VERSION = KNNCodecVersion.V_9_5_0; + private final KNNFormatFacade knnFormatFacade; + private final PerFieldKnnVectorsFormat perFieldKnnVectorsFormat; + + /** + * No arg constructor that uses Lucene95 as the delegate + */ + public KNN950Codec() { + this(VERSION.getDefaultCodecDelegate(), VERSION.getPerFieldKnnVectorsFormat()); + } + + /** + * Sole constructor. When subclassing this codec, create a no-arg ctor and pass the delegate codec + * and a unique name to this ctor. + * + * @param delegate codec that will perform all operations this codec does not override + * @param knnVectorsFormat per field format for KnnVector + */ + @Builder + protected KNN950Codec(Codec delegate, PerFieldKnnVectorsFormat knnVectorsFormat) { + super(VERSION.getCodecName(), delegate); + knnFormatFacade = VERSION.getKnnFormatFacadeSupplier().apply(delegate); + perFieldKnnVectorsFormat = knnVectorsFormat; + } + + @Override + public DocValuesFormat docValuesFormat() { + return knnFormatFacade.docValuesFormat(); + } + + @Override + public CompoundFormat compoundFormat() { + return knnFormatFacade.compoundFormat(); + } + + @Override + public KnnVectorsFormat knnVectorsFormat() { + return perFieldKnnVectorsFormat; + } +} diff --git a/src/main/java/org/opensearch/knn/index/codec/KNN950Codec/KNN950PerFieldKnnVectorsFormat.java b/src/main/java/org/opensearch/knn/index/codec/KNN950Codec/KNN950PerFieldKnnVectorsFormat.java new file mode 100644 index 000000000..66dfcd46e --- /dev/null +++ b/src/main/java/org/opensearch/knn/index/codec/KNN950Codec/KNN950PerFieldKnnVectorsFormat.java @@ -0,0 +1,28 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.codec.KNN950Codec; + +import org.apache.lucene.codecs.lucene95.Lucene95HnswVectorsFormat; +import org.opensearch.index.mapper.MapperService; +import org.opensearch.knn.index.codec.BasePerFieldKnnVectorsFormat; + +import java.util.Optional; + +/** + * Class provides per field format implementation for Lucene Knn vector type + */ +public class KNN950PerFieldKnnVectorsFormat extends BasePerFieldKnnVectorsFormat { + + public KNN950PerFieldKnnVectorsFormat(final Optional mapperService) { + super( + mapperService, + Lucene95HnswVectorsFormat.DEFAULT_MAX_CONN, + Lucene95HnswVectorsFormat.DEFAULT_BEAM_WIDTH, + () -> new Lucene95HnswVectorsFormat(), + (maxConnm, beamWidth) -> new Lucene95HnswVectorsFormat(maxConnm, beamWidth) + ); + } +} diff --git a/src/main/java/org/opensearch/knn/index/codec/KNNCodecVersion.java b/src/main/java/org/opensearch/knn/index/codec/KNNCodecVersion.java index a6d699484..cbf6680f7 100644 --- a/src/main/java/org/opensearch/knn/index/codec/KNNCodecVersion.java +++ b/src/main/java/org/opensearch/knn/index/codec/KNNCodecVersion.java @@ -10,7 +10,8 @@ import org.apache.lucene.backward_codecs.lucene91.Lucene91Codec; import org.apache.lucene.backward_codecs.lucene92.Lucene92Codec; import org.apache.lucene.codecs.Codec; -import org.apache.lucene.codecs.lucene94.Lucene94Codec; +import org.apache.lucene.backward_codecs.lucene94.Lucene94Codec; +import org.apache.lucene.codecs.lucene95.Lucene95Codec; import org.apache.lucene.codecs.perfield.PerFieldKnnVectorsFormat; import org.opensearch.index.mapper.MapperService; import org.opensearch.knn.index.codec.KNN80Codec.KNN80CompoundFormat; @@ -20,6 +21,8 @@ import org.opensearch.knn.index.codec.KNN920Codec.KNN920PerFieldKnnVectorsFormat; import org.opensearch.knn.index.codec.KNN940Codec.KNN940Codec; import org.opensearch.knn.index.codec.KNN940Codec.KNN940PerFieldKnnVectorsFormat; +import org.opensearch.knn.index.codec.KNN950Codec.KNN950Codec; +import org.opensearch.knn.index.codec.KNN950Codec.KNN950PerFieldKnnVectorsFormat; import java.util.Optional; import java.util.function.BiFunction; @@ -74,9 +77,24 @@ public enum KNNCodecVersion { .knnVectorsFormat(new KNN940PerFieldKnnVectorsFormat(Optional.ofNullable(mapperService))) .build(), KNN940Codec::new + ), + + V_9_5_0( + "KNN950Codec", + new Lucene95Codec(), + new KNN950PerFieldKnnVectorsFormat(Optional.empty()), + (delegate) -> new KNNFormatFacade( + new KNN80DocValuesFormat(delegate.docValuesFormat()), + new KNN80CompoundFormat(delegate.compoundFormat()) + ), + (userCodec, mapperService) -> KNN950Codec.builder() + .delegate(userCodec) + .knnVectorsFormat(new KNN950PerFieldKnnVectorsFormat(Optional.ofNullable(mapperService))) + .build(), + KNN950Codec::new ); - private static final KNNCodecVersion CURRENT = V_9_4_0; + private static final KNNCodecVersion CURRENT = V_9_5_0; private final String codecName; private final Codec defaultCodecDelegate; diff --git a/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec b/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec index 8185e7858..5c44d5756 100644 --- a/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec +++ b/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec @@ -4,4 +4,5 @@ org.opensearch.knn.index.codec.KNN86Codec.KNN86Codec org.opensearch.knn.index.codec.KNN87Codec.KNN87Codec org.opensearch.knn.index.codec.KNN910Codec.KNN910Codec org.opensearch.knn.index.codec.KNN920Codec.KNN920Codec -org.opensearch.knn.index.codec.KNN940Codec.KNN940Codec \ No newline at end of file +org.opensearch.knn.index.codec.KNN940Codec.KNN940Codec +org.opensearch.knn.index.codec.KNN950Codec.KNN950Codec \ No newline at end of file diff --git a/src/test/java/org/opensearch/knn/index/codec/KNN940Codec/KNN940CodecTests.java b/src/test/java/org/opensearch/knn/index/codec/KNN940Codec/KNN940CodecTests.java index 1101d93bb..805edac9d 100644 --- a/src/test/java/org/opensearch/knn/index/codec/KNN940Codec/KNN940CodecTests.java +++ b/src/test/java/org/opensearch/knn/index/codec/KNN940Codec/KNN940CodecTests.java @@ -6,13 +6,9 @@ package org.opensearch.knn.index.codec.KNN940Codec; import org.apache.lucene.codecs.Codec; -import org.apache.lucene.codecs.perfield.PerFieldKnnVectorsFormat; -import org.opensearch.index.mapper.MapperService; import org.opensearch.knn.index.codec.KNNCodecTestCase; import java.io.IOException; -import java.util.Optional; import java.util.concurrent.ExecutionException; -import java.util.function.Function; import static org.opensearch.knn.index.codec.KNNCodecVersion.V_9_4_0; @@ -26,15 +22,9 @@ public void testBuildFromModelTemplate() throws InterruptedException, ExecutionE testBuildFromModelTemplate((KNN940Codec.builder().delegate(V_9_4_0.getDefaultCodecDelegate()).build())); } - public void testKnnVectorIndex() throws Exception { - Function perFieldKnnVectorsFormatProvider = ( - mapperService) -> new KNN940PerFieldKnnVectorsFormat(Optional.of(mapperService)); - - Function knnCodecProvider = (knnVectorFormat) -> KNN940Codec.builder() - .delegate(V_9_4_0.getDefaultCodecDelegate()) - .knnVectorsFormat(knnVectorFormat) - .build(); - - testKnnVectorIndex(knnCodecProvider, perFieldKnnVectorsFormatProvider); + // Ensure that the codec is able to return the correct per field knn vectors format for codec + public void testCodecSetsCustomPerFieldKnnVectorsFormat() { + final Codec codec = new KNN940Codec(); + assertTrue(codec.knnVectorsFormat() instanceof KNN940PerFieldKnnVectorsFormat); } } diff --git a/src/test/java/org/opensearch/knn/index/codec/KNN950Codec/KNN950CodecTests.java b/src/test/java/org/opensearch/knn/index/codec/KNN950Codec/KNN950CodecTests.java new file mode 100644 index 000000000..8eafb6a4a --- /dev/null +++ b/src/test/java/org/opensearch/knn/index/codec/KNN950Codec/KNN950CodecTests.java @@ -0,0 +1,51 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.codec.KNN950Codec; + +import lombok.SneakyThrows; +import org.apache.lucene.codecs.Codec; +import org.apache.lucene.codecs.perfield.PerFieldKnnVectorsFormat; +import org.opensearch.index.mapper.MapperService; +import org.opensearch.knn.index.codec.KNNCodecTestCase; + +import java.util.Optional; +import java.util.function.Function; + +import static org.opensearch.knn.index.codec.KNNCodecVersion.V_9_5_0; + +public class KNN950CodecTests extends KNNCodecTestCase { + + @SneakyThrows + public void testMultiFieldsKnnIndex() { + testMultiFieldsKnnIndex(KNN950Codec.builder().delegate(V_9_5_0.getDefaultCodecDelegate()).build()); + } + + @SneakyThrows + public void testBuildFromModelTemplate() { + testBuildFromModelTemplate((KNN950Codec.builder().delegate(V_9_5_0.getDefaultCodecDelegate()).build())); + } + + // Ensure that the codec is able to return the correct per field knn vectors format for codec + public void testCodecSetsCustomPerFieldKnnVectorsFormat() { + final Codec codec = new KNN950Codec(); + assertTrue(codec.knnVectorsFormat() instanceof KNN950PerFieldKnnVectorsFormat); + } + + // IMPORTANT: When this Codec is moved to a backwards Codec, this test needs to be removed, because it attempts to + // write with a read only codec, which will fail + @SneakyThrows + public void testKnnVectorIndex() { + Function perFieldKnnVectorsFormatProvider = ( + mapperService) -> new KNN950PerFieldKnnVectorsFormat(Optional.of(mapperService)); + + Function knnCodecProvider = (knnVectorFormat) -> KNN950Codec.builder() + .delegate(V_9_5_0.getDefaultCodecDelegate()) + .knnVectorsFormat(knnVectorFormat) + .build(); + + testKnnVectorIndex(knnCodecProvider, perFieldKnnVectorsFormatProvider); + } +} diff --git a/src/test/java/org/opensearch/knn/index/codec/KNNCodecFactoryTests.java b/src/test/java/org/opensearch/knn/index/codec/KNNCodecFactoryTests.java index d918f5439..2ec953b18 100644 --- a/src/test/java/org/opensearch/knn/index/codec/KNNCodecFactoryTests.java +++ b/src/test/java/org/opensearch/knn/index/codec/KNNCodecFactoryTests.java @@ -8,12 +8,14 @@ import org.apache.lucene.backward_codecs.lucene92.Lucene92Codec; import org.apache.lucene.codecs.Codec; import org.apache.lucene.backward_codecs.lucene91.Lucene91Codec; -import org.apache.lucene.codecs.lucene94.Lucene94Codec; +import org.apache.lucene.backward_codecs.lucene94.Lucene94Codec; +import org.apache.lucene.codecs.lucene95.Lucene95Codec; import org.opensearch.knn.KNNTestCase; import static org.opensearch.knn.index.codec.KNNCodecVersion.V_9_1_0; import static org.opensearch.knn.index.codec.KNNCodecVersion.V_9_2_0; import static org.opensearch.knn.index.codec.KNNCodecVersion.V_9_4_0; +import static org.opensearch.knn.index.codec.KNNCodecVersion.V_9_5_0; public class KNNCodecFactoryTests extends KNNTestCase { @@ -35,6 +37,12 @@ public void testKNN940Codec() { assertNotNull(V_9_4_0.getKnnFormatFacadeSupplier().apply(V_9_4_0.getDefaultCodecDelegate())); } + public void testKNN950Codec() { + assertDelegateForVersion(V_9_5_0, Lucene95Codec.class); + assertNotNull(V_9_5_0.getPerFieldKnnVectorsFormat()); + assertNotNull(V_9_5_0.getKnnFormatFacadeSupplier().apply(V_9_5_0.getDefaultCodecDelegate())); + } + private void assertDelegateForVersion(final KNNCodecVersion codecVersion, final Class expectedCodecClass) { final Codec defaultDelegate = codecVersion.getDefaultCodecDelegate(); assertNotNull(defaultDelegate); From 7ba9088c5c29e7b6c376a0cdfdd71613298abf0c Mon Sep 17 00:00:00 2001 From: John Mazanec Date: Wed, 8 Feb 2023 15:40:50 -0800 Subject: [PATCH 074/416] Update 2.5 bwc on 2.x branch (#757) Updates 2.5 to 2.5.0 for bwc testing. Also updates 1.3.7 to 1.3.8. Signed-off-by: John Mazanec --- .github/workflows/backwards_compatibility_tests_workflow.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/backwards_compatibility_tests_workflow.yml b/.github/workflows/backwards_compatibility_tests_workflow.yml index 4a2e50594..e194d47b0 100644 --- a/.github/workflows/backwards_compatibility_tests_workflow.yml +++ b/.github/workflows/backwards_compatibility_tests_workflow.yml @@ -14,7 +14,7 @@ jobs: strategy: matrix: java: [ 11, 17 ] - bwc_version : [ "1.1.0", "1.2.4", "1.3.7", "2.0.1", "2.1.0", "2.2.1", "2.3.0", "2.4.1", "2.5.0-SNAPSHOT" ] + bwc_version : [ "1.1.0", "1.2.4", "1.3.8", "2.0.1", "2.1.0", "2.2.1", "2.3.0", "2.4.1", "2.5.0" ] opensearch_version : [ "2.6.0-SNAPSHOT" ] name: k-NN Restart-Upgrade BWC Tests @@ -46,7 +46,7 @@ jobs: strategy: matrix: java: [ 11, 17 ] - bwc_version: [ "1.3.7", "2.0.1", "2.1.0", "2.2.1", "2.3.0", "2.4.1", "2.5.0-SNAPSHOT" ] + bwc_version: [ "1.3.8", "2.0.1", "2.1.0", "2.2.1", "2.3.0", "2.4.1", "2.5.0" ] opensearch_version: [ "2.6.0-SNAPSHOT" ] name: k-NN Rolling-Upgrade BWC Tests From 7da4e43d20ff0bb5ba53f0c033f3bd1139db1439 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Tue, 14 Feb 2023 15:30:26 -0800 Subject: [PATCH 075/416] Adding test for KNNWeight (#759) (#763) * Adding test for KNNWeight Signed-off-by: Martin Gaievski (cherry picked from commit 5b06cfe8902920161f6515347b49fcb2eae3e4e7) Co-authored-by: Martin Gaievski --- build.gradle | 5 + .../knn/index/query/KNNWeightTests.java | 384 ++++++++++++++++++ .../org.mockito.plugins.MockMaker | 1 + 3 files changed, 390 insertions(+) create mode 100644 src/test/java/org/opensearch/knn/index/query/KNNWeightTests.java create mode 100644 src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker diff --git a/build.gradle b/build.gradle index 7bc3fefd9..3e1f97eb4 100644 --- a/build.gradle +++ b/build.gradle @@ -165,6 +165,9 @@ dependencies { api group: 'com.google.guava', name: 'guava', version:'30.0-jre' api group: 'commons-lang', name: 'commons-lang', version: '2.6' testFixturesImplementation "org.opensearch.test:framework:${opensearch_version}" + testImplementation group: 'net.bytebuddy', name: 'byte-buddy', version: '1.12.22' + testImplementation group: 'org.objenesis', name: 'objenesis', version: '3.2' + testImplementation group: 'net.bytebuddy', name: 'byte-buddy-agent', version: '1.12.22' } @@ -196,6 +199,8 @@ test { dependsOn buildJniLib systemProperty 'tests.security.manager', 'false' systemProperty "java.library.path", "$rootDir/jni/release" + //this change enables mockito-inline that supports mocking of static classes/calls + systemProperty "jdk.attach.allowAttachSelf", true if (Os.isFamily(Os.FAMILY_WINDOWS)) { // Add the paths of built JNI libraries and its dependent libraries to PATH variable in System variables environment('PATH', System.getenv('PATH') + ";$rootDir/jni/release" + ";$rootDir/src/main/resources/windowsDependencies") diff --git a/src/test/java/org/opensearch/knn/index/query/KNNWeightTests.java b/src/test/java/org/opensearch/knn/index/query/KNNWeightTests.java new file mode 100644 index 000000000..444f763a6 --- /dev/null +++ b/src/test/java/org/opensearch/knn/index/query/KNNWeightTests.java @@ -0,0 +1,384 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.query; + +import com.google.common.collect.Comparators; +import lombok.SneakyThrows; +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.FieldInfos; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.SegmentCommitInfo; +import org.apache.lucene.index.SegmentInfo; +import org.apache.lucene.index.SegmentReader; +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.search.Scorer; +import org.apache.lucene.search.Sort; +import org.apache.lucene.store.FSDirectory; +import org.apache.lucene.util.StringHelper; +import org.apache.lucene.util.Version; +import org.junit.BeforeClass; +import org.mockito.MockedStatic; +import org.opensearch.common.io.PathUtils; +import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.common.unit.TimeValue; +import org.opensearch.knn.KNNTestCase; +import org.opensearch.knn.index.KNNSettings; +import org.opensearch.knn.index.SpaceType; +import org.opensearch.knn.index.codec.KNNCodecVersion; +import org.opensearch.knn.index.memory.NativeMemoryAllocation; +import org.opensearch.knn.index.memory.NativeMemoryCacheManager; +import org.opensearch.knn.index.util.KNNEngine; +import org.opensearch.knn.indices.ModelDao; +import org.opensearch.knn.indices.ModelMetadata; +import org.opensearch.knn.jni.JNIService; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.Function; +import java.util.stream.Collectors; + +import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.mockStatic; +import static org.mockito.Mockito.when; +import static org.opensearch.knn.KNNRestTestCase.INDEX_NAME; +import static org.opensearch.knn.common.KNNConstants.KNN_ENGINE; +import static org.opensearch.knn.common.KNNConstants.MODEL_ID; +import static org.opensearch.knn.common.KNNConstants.SPACE_TYPE; + +public class KNNWeightTests extends KNNTestCase { + private static final String FIELD_NAME = "target_field"; + private static final float[] QUERY_VECTOR = new float[] { 1.8f, 2.4f }; + private static final String SEGMENT_NAME = "0"; + private static final int K = 5; + private static final Set SEGMENT_FILES_NMSLIB = Set.of("_0.cfe", "_0_2011_target_field.hnswc"); + private static final Set SEGMENT_FILES_FAISS = Set.of("_0.cfe", "_0_2011_target_field.faissc"); + private static final String CIRCUIT_BREAKER_LIMIT_100KB = "100Kb"; + + private static final Map DOC_ID_TO_SCORES = Map.of(10, 0.4f, 101, 0.05f, 100, 0.8f, 50, 0.52f); + + private static MockedStatic nativeMemoryCacheManagerMockedStatic; + private static MockedStatic jniServiceMockedStatic; + + @BeforeClass + public static void setUpClass() throws Exception { + final KNNSettings knnSettings = mock(KNNSettings.class); + final MockedStatic knnSettingsMockedStatic = mockStatic(KNNSettings.class); + when(knnSettings.getSettingValue(eq(KNNSettings.KNN_MEMORY_CIRCUIT_BREAKER_ENABLED))).thenReturn(true); + when(knnSettings.getSettingValue(eq(KNNSettings.KNN_MEMORY_CIRCUIT_BREAKER_LIMIT))).thenReturn(CIRCUIT_BREAKER_LIMIT_100KB); + when(knnSettings.getSettingValue(eq(KNNSettings.KNN_CACHE_ITEM_EXPIRY_ENABLED))).thenReturn(false); + when(knnSettings.getSettingValue(eq(KNNSettings.KNN_CACHE_ITEM_EXPIRY_TIME_MINUTES))).thenReturn(TimeValue.timeValueMinutes(10)); + + final ByteSizeValue v = ByteSizeValue.parseBytesSizeValue( + CIRCUIT_BREAKER_LIMIT_100KB, + KNNSettings.KNN_MEMORY_CIRCUIT_BREAKER_LIMIT + ); + knnSettingsMockedStatic.when(KNNSettings::getCircuitBreakerLimit).thenReturn(v); + knnSettingsMockedStatic.when(KNNSettings::state).thenReturn(knnSettings); + knnSettingsMockedStatic.when(KNNSettings::isKNNPluginEnabled).thenReturn(true); + + jniServiceMockedStatic = mockStatic(JNIService.class); + nativeMemoryCacheManagerMockedStatic = mockStatic(NativeMemoryCacheManager.class); + + final NativeMemoryCacheManager nativeMemoryCacheManager = mock(NativeMemoryCacheManager.class); + final NativeMemoryAllocation nativeMemoryAllocation = mock(NativeMemoryAllocation.class); + when(nativeMemoryCacheManager.get(any(), anyBoolean())).thenReturn(nativeMemoryAllocation); + + nativeMemoryCacheManagerMockedStatic.when(NativeMemoryCacheManager::getInstance).thenReturn(nativeMemoryCacheManager); + + final MockedStatic pathUtilsMockedStatic = mockStatic(PathUtils.class); + final Path indexPath = mock(Path.class); + when(indexPath.toString()).thenReturn("/mydrive/myfolder"); + pathUtilsMockedStatic.when(() -> PathUtils.get(anyString(), anyString())).thenReturn(indexPath); + } + + @SneakyThrows + public void testQueryResultScoreNmslib() { + for (SpaceType space : List.of(SpaceType.L2, SpaceType.L1, SpaceType.COSINESIMIL, SpaceType.INNER_PRODUCT, SpaceType.LINF)) { + testQueryScore(space::scoreTranslation, SEGMENT_FILES_NMSLIB, Map.of(SPACE_TYPE, space.getValue())); + } + } + + @SneakyThrows + public void testQueryResultScoreFaiss() { + testQueryScore( + SpaceType.L2::scoreTranslation, + SEGMENT_FILES_FAISS, + Map.of(SPACE_TYPE, SpaceType.L2.getValue(), KNN_ENGINE, KNNEngine.FAISS.getName()) + ); + // score translation for Faiss and inner product is different from default defined in Space enum + testQueryScore( + rawScore -> SpaceType.INNER_PRODUCT.scoreTranslation(-1 * rawScore), + SEGMENT_FILES_FAISS, + Map.of(SPACE_TYPE, SpaceType.INNER_PRODUCT.getValue(), KNN_ENGINE, KNNEngine.FAISS.getName()) + ); + } + + @SneakyThrows + public void testQueryScoreForFaissWithModel() throws IOException { + SpaceType spaceType = SpaceType.L2; + final Function scoreTranslator = spaceType::scoreTranslation; + final String modelId = "modelId"; + jniServiceMockedStatic.when(() -> JNIService.queryIndex(anyLong(), any(), anyInt(), anyString())).thenReturn(getKNNQueryResults()); + + final KNNQuery query = new KNNQuery(FIELD_NAME, QUERY_VECTOR, K, INDEX_NAME); + + ModelDao modelDao = mock(ModelDao.class); + ModelMetadata modelMetadata = mock(ModelMetadata.class); + when(modelMetadata.getKnnEngine()).thenReturn(KNNEngine.FAISS); + when(modelMetadata.getSpaceType()).thenReturn(spaceType); + when(modelDao.getMetadata(eq("modelId"))).thenReturn(modelMetadata); + + KNNWeight.initialize(modelDao); + final KNNWeight knnWeight = new KNNWeight(query, 0.0f); + + final LeafReaderContext leafReaderContext = mock(LeafReaderContext.class); + final SegmentReader reader = mock(SegmentReader.class); + when(leafReaderContext.reader()).thenReturn(reader); + + final FSDirectory directory = mock(FSDirectory.class); + when(reader.directory()).thenReturn(directory); + final SegmentInfo segmentInfo = new SegmentInfo( + directory, + Version.LATEST, + Version.LATEST, + SEGMENT_NAME, + 100, + true, + KNNCodecVersion.current().getDefaultCodecDelegate(), + Map.of(), + new byte[StringHelper.ID_LENGTH], + Map.of(), + Sort.RELEVANCE + ); + segmentInfo.setFiles(SEGMENT_FILES_FAISS); + final SegmentCommitInfo segmentCommitInfo = new SegmentCommitInfo(segmentInfo, 0, 0, 0, 0, 0, new byte[StringHelper.ID_LENGTH]); + when(reader.getSegmentInfo()).thenReturn(segmentCommitInfo); + + final Path path = mock(Path.class); + when(directory.getDirectory()).thenReturn(path); + final FieldInfos fieldInfos = mock(FieldInfos.class); + final FieldInfo fieldInfo = mock(FieldInfo.class); + when(reader.getFieldInfos()).thenReturn(fieldInfos); + when(fieldInfos.fieldInfo(any())).thenReturn(fieldInfo); + when(fieldInfo.attributes()).thenReturn(Map.of()); + when(fieldInfo.getAttribute(eq(MODEL_ID))).thenReturn(modelId); + + final KNNScorer knnScorer = (KNNScorer) knnWeight.scorer(leafReaderContext); + assertNotNull(knnScorer); + final DocIdSetIterator docIdSetIterator = knnScorer.iterator(); + assertNotNull(docIdSetIterator); + assertEquals(DOC_ID_TO_SCORES.size(), docIdSetIterator.cost()); + + final List actualDocIds = new ArrayList(); + final Map translatedScores = getTranslatedScores(scoreTranslator); + for (int docId = docIdSetIterator.nextDoc(); docId != NO_MORE_DOCS; docId = docIdSetIterator.nextDoc()) { + actualDocIds.add(docId); + assertEquals(translatedScores.get(docId), knnScorer.score(), 0.01f); + } + assertEquals(docIdSetIterator.cost(), actualDocIds.size()); + assertTrue(Comparators.isInOrder(actualDocIds, Comparator.naturalOrder())); + } + + @SneakyThrows + public void testQueryScoreForFaissWithNonExistingModel() throws IOException { + SpaceType spaceType = SpaceType.L2; + final String modelId = "modelId"; + + final KNNQuery query = new KNNQuery(FIELD_NAME, QUERY_VECTOR, K, INDEX_NAME); + + ModelDao modelDao = mock(ModelDao.class); + ModelMetadata modelMetadata = mock(ModelMetadata.class); + when(modelMetadata.getKnnEngine()).thenReturn(KNNEngine.FAISS); + when(modelMetadata.getSpaceType()).thenReturn(spaceType); + + KNNWeight.initialize(modelDao); + final KNNWeight knnWeight = new KNNWeight(query, 0.0f); + + final LeafReaderContext leafReaderContext = mock(LeafReaderContext.class); + final SegmentReader reader = mock(SegmentReader.class); + when(leafReaderContext.reader()).thenReturn(reader); + + final FSDirectory directory = mock(FSDirectory.class); + when(reader.directory()).thenReturn(directory); + + final Path path = mock(Path.class); + when(directory.getDirectory()).thenReturn(path); + final FieldInfos fieldInfos = mock(FieldInfos.class); + final FieldInfo fieldInfo = mock(FieldInfo.class); + when(reader.getFieldInfos()).thenReturn(fieldInfos); + when(fieldInfos.fieldInfo(any())).thenReturn(fieldInfo); + when(fieldInfo.attributes()).thenReturn(Map.of()); + when(fieldInfo.getAttribute(eq(MODEL_ID))).thenReturn(modelId); + + RuntimeException ex = expectThrows(RuntimeException.class, () -> knnWeight.scorer(leafReaderContext)); + assertEquals(String.format("Model \"%s\" does not exist.", modelId), ex.getMessage()); + } + + @SneakyThrows + public void testShardWithoutFiles() { + final KNNQuery query = new KNNQuery(FIELD_NAME, QUERY_VECTOR, K, INDEX_NAME); + final KNNWeight knnWeight = new KNNWeight(query, 0.0f); + + final LeafReaderContext leafReaderContext = mock(LeafReaderContext.class); + final SegmentReader reader = mock(SegmentReader.class); + when(leafReaderContext.reader()).thenReturn(reader); + + final FSDirectory directory = mock(FSDirectory.class); + when(reader.directory()).thenReturn(directory); + + final SegmentInfo segmentInfo = new SegmentInfo( + directory, + Version.LATEST, + Version.LATEST, + SEGMENT_NAME, + 100, + false, + KNNCodecVersion.current().getDefaultCodecDelegate(), + Map.of(), + new byte[StringHelper.ID_LENGTH], + Map.of(), + Sort.RELEVANCE + ); + segmentInfo.setFiles(Set.of()); + final SegmentCommitInfo segmentCommitInfo = new SegmentCommitInfo(segmentInfo, 0, 0, 0, 0, 0, new byte[StringHelper.ID_LENGTH]); + when(reader.getSegmentInfo()).thenReturn(segmentCommitInfo); + + final Path path = mock(Path.class); + when(directory.getDirectory()).thenReturn(path); + final FieldInfos fieldInfos = mock(FieldInfos.class); + final FieldInfo fieldInfo = mock(FieldInfo.class); + when(reader.getFieldInfos()).thenReturn(fieldInfos); + when(fieldInfos.fieldInfo(any())).thenReturn(fieldInfo); + + final Scorer knnScorer = knnWeight.scorer(leafReaderContext); + assertNull(knnScorer); + } + + @SneakyThrows + public void testEmptyQueryResults() { + final KNNQueryResult[] knnQueryResults = new KNNQueryResult[] {}; + jniServiceMockedStatic.when(() -> JNIService.queryIndex(anyLong(), any(), anyInt(), anyString())).thenReturn(knnQueryResults); + + final KNNQuery query = new KNNQuery(FIELD_NAME, QUERY_VECTOR, K, INDEX_NAME); + final KNNWeight knnWeight = new KNNWeight(query, 0.0f); + + final LeafReaderContext leafReaderContext = mock(LeafReaderContext.class); + final SegmentReader reader = mock(SegmentReader.class); + when(leafReaderContext.reader()).thenReturn(reader); + + final FSDirectory directory = mock(FSDirectory.class); + when(reader.directory()).thenReturn(directory); + final SegmentInfo segmentInfo = new SegmentInfo( + directory, + Version.LATEST, + Version.LATEST, + SEGMENT_NAME, + 100, + true, + KNNCodecVersion.current().getDefaultCodecDelegate(), + Map.of(), + new byte[StringHelper.ID_LENGTH], + Map.of(), + Sort.RELEVANCE + ); + segmentInfo.setFiles(SEGMENT_FILES_NMSLIB); + final SegmentCommitInfo segmentCommitInfo = new SegmentCommitInfo(segmentInfo, 0, 0, 0, 0, 0, new byte[StringHelper.ID_LENGTH]); + when(reader.getSegmentInfo()).thenReturn(segmentCommitInfo); + + final Path path = mock(Path.class); + when(directory.getDirectory()).thenReturn(path); + final FieldInfos fieldInfos = mock(FieldInfos.class); + final FieldInfo fieldInfo = mock(FieldInfo.class); + when(reader.getFieldInfos()).thenReturn(fieldInfos); + when(fieldInfos.fieldInfo(any())).thenReturn(fieldInfo); + + final Scorer knnScorer = knnWeight.scorer(leafReaderContext); + assertNull(knnScorer); + } + + private void testQueryScore( + final Function scoreTranslator, + final Set segmentFiles, + final Map fileAttributes + ) throws IOException { + jniServiceMockedStatic.when(() -> JNIService.queryIndex(anyLong(), any(), anyInt(), anyString())).thenReturn(getKNNQueryResults()); + + final KNNQuery query = new KNNQuery(FIELD_NAME, QUERY_VECTOR, K, INDEX_NAME); + final KNNWeight knnWeight = new KNNWeight(query, 0.0f); + + final LeafReaderContext leafReaderContext = mock(LeafReaderContext.class); + final SegmentReader reader = mock(SegmentReader.class); + when(leafReaderContext.reader()).thenReturn(reader); + + final FSDirectory directory = mock(FSDirectory.class); + when(reader.directory()).thenReturn(directory); + final SegmentInfo segmentInfo = new SegmentInfo( + directory, + Version.LATEST, + Version.LATEST, + SEGMENT_NAME, + 100, + true, + KNNCodecVersion.current().getDefaultCodecDelegate(), + Map.of(), + new byte[StringHelper.ID_LENGTH], + Map.of(), + Sort.RELEVANCE + ); + segmentInfo.setFiles(segmentFiles); + final SegmentCommitInfo segmentCommitInfo = new SegmentCommitInfo(segmentInfo, 0, 0, 0, 0, 0, new byte[StringHelper.ID_LENGTH]); + when(reader.getSegmentInfo()).thenReturn(segmentCommitInfo); + + final Path path = mock(Path.class); + when(directory.getDirectory()).thenReturn(path); + final FieldInfos fieldInfos = mock(FieldInfos.class); + final FieldInfo fieldInfo = mock(FieldInfo.class); + when(reader.getFieldInfos()).thenReturn(fieldInfos); + when(fieldInfos.fieldInfo(any())).thenReturn(fieldInfo); + when(fieldInfo.attributes()).thenReturn(fileAttributes); + + final KNNScorer knnScorer = (KNNScorer) knnWeight.scorer(leafReaderContext); + assertNotNull(knnScorer); + final DocIdSetIterator docIdSetIterator = knnScorer.iterator(); + assertNotNull(docIdSetIterator); + assertEquals(DOC_ID_TO_SCORES.size(), docIdSetIterator.cost()); + + final List actualDocIds = new ArrayList(); + final Map translatedScores = getTranslatedScores(scoreTranslator); + for (int docId = docIdSetIterator.nextDoc(); docId != NO_MORE_DOCS; docId = docIdSetIterator.nextDoc()) { + actualDocIds.add(docId); + assertEquals(translatedScores.get(docId), knnScorer.score(), 0.01f); + } + assertEquals(docIdSetIterator.cost(), actualDocIds.size()); + assertTrue(Comparators.isInOrder(actualDocIds, Comparator.naturalOrder())); + } + + private Map getTranslatedScores(Function scoreTranslator) { + return DOC_ID_TO_SCORES.entrySet() + .stream() + .collect(Collectors.toMap(Map.Entry::getKey, entry -> scoreTranslator.apply(entry.getValue()))); + } + + private KNNQueryResult[] getKNNQueryResults() { + return DOC_ID_TO_SCORES.entrySet() + .stream() + .map(entry -> new KNNQueryResult(entry.getKey(), entry.getValue())) + .collect(Collectors.toList()) + .toArray(new KNNQueryResult[0]); + } +} diff --git a/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker b/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker new file mode 100644 index 000000000..ca6ee9cea --- /dev/null +++ b/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker @@ -0,0 +1 @@ +mock-maker-inline \ No newline at end of file From 26c2f3c4d46f37ceb681f555920e1adc5c58eaad Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Tue, 14 Feb 2023 15:33:28 -0800 Subject: [PATCH 076/416] Set NoMergePolicy for codec test (#764) Sets NoMerge as merge policy for codec test. This allows us to reliably predict how many segments will be created. Signed-off-by: John Mazanec (cherry picked from commit b8f2deb3d07fd6e95369d0c6b1df477b8e053f66) --- .../java/org/opensearch/knn/index/codec/KNNCodecTestCase.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestCase.java b/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestCase.java index b5ef7f180..6bfde31bb 100644 --- a/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestCase.java +++ b/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestCase.java @@ -9,6 +9,7 @@ import com.google.common.collect.ImmutableSet; import org.apache.lucene.codecs.perfield.PerFieldKnnVectorsFormat; import org.apache.lucene.document.KnnVectorField; +import org.apache.lucene.index.NoMergePolicy; import org.apache.lucene.index.VectorSimilarityFunction; import org.apache.lucene.search.Query; import org.apache.lucene.search.TopDocs; @@ -121,6 +122,8 @@ public void testMultiFieldsKnnIndex(Codec codec) throws Exception { IndexWriterConfig iwc = newIndexWriterConfig(); iwc.setMergeScheduler(new SerialMergeScheduler()); iwc.setCodec(codec); + // Set merge policy to no merges so that we create a predictable number of segments. + iwc.setMergePolicy(NoMergePolicy.INSTANCE); /** * Add doc with field "test_vector" From eb88235ce3f90b8bb1a8eac83de87c10463b242f Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Fri, 17 Feb 2023 10:29:36 -0800 Subject: [PATCH 077/416] Replace KnnQueryVector by KnnFloatVectorQuery (#767) (#768) Signed-off-by: Martin Gaievski (cherry picked from commit 71d7232097cdd78c3b67cee47a5b5da41102388a) Co-authored-by: Martin Gaievski --- .../org/opensearch/knn/index/query/KNNQueryFactory.java | 6 +++--- .../opensearch/knn/index/query/KNNQueryBuilderTests.java | 4 ++-- .../opensearch/knn/index/query/KNNQueryFactoryTests.java | 6 +++--- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/main/java/org/opensearch/knn/index/query/KNNQueryFactory.java b/src/main/java/org/opensearch/knn/index/query/KNNQueryFactory.java index c68ce9502..188bbc150 100644 --- a/src/main/java/org/opensearch/knn/index/query/KNNQueryFactory.java +++ b/src/main/java/org/opensearch/knn/index/query/KNNQueryFactory.java @@ -11,7 +11,7 @@ import lombok.NonNull; import lombok.Setter; import lombok.extern.log4j.Log4j2; -import org.apache.lucene.search.KnnVectorQuery; +import org.apache.lucene.search.KnnFloatVectorQuery; import org.apache.lucene.search.Query; import org.opensearch.index.query.QueryBuilder; import org.opensearch.index.query.QueryShardContext; @@ -73,13 +73,13 @@ public static Query create(CreateQueryRequest createQueryRequest) { ); try { final Query filterQuery = createQueryRequest.getFilter().get().toQuery(queryShardContext); - return new KnnVectorQuery(fieldName, vector, k, filterQuery); + return new KnnFloatVectorQuery(fieldName, vector, k, filterQuery); } catch (IOException e) { throw new RuntimeException("Cannot create knn query with filter", e); } } log.debug(String.format("Creating Lucene k-NN query for index: %s \"\", field: %s \"\", k: %d", indexName, fieldName, k)); - return new KnnVectorQuery(fieldName, vector, k); + return new KnnFloatVectorQuery(fieldName, vector, k); } /** diff --git a/src/test/java/org/opensearch/knn/index/query/KNNQueryBuilderTests.java b/src/test/java/org/opensearch/knn/index/query/KNNQueryBuilderTests.java index e3376dda9..982c1a0c0 100644 --- a/src/test/java/org/opensearch/knn/index/query/KNNQueryBuilderTests.java +++ b/src/test/java/org/opensearch/knn/index/query/KNNQueryBuilderTests.java @@ -6,7 +6,7 @@ package org.opensearch.knn.index.query; import com.google.common.collect.ImmutableMap; -import org.apache.lucene.search.KnnVectorQuery; +import org.apache.lucene.search.KnnFloatVectorQuery; import org.apache.lucene.search.Query; import org.opensearch.Version; import org.opensearch.cluster.ClusterModule; @@ -196,7 +196,7 @@ public void testDoToQuery_KnnQueryWithFilter() throws Exception { when(mockQueryShardContext.fieldMapper(anyString())).thenReturn(mockKNNVectorField); Query query = knnQueryBuilder.doToQuery(mockQueryShardContext); assertNotNull(query); - assertTrue(query instanceof KnnVectorQuery); + assertTrue(query.getClass().isAssignableFrom(KnnFloatVectorQuery.class)); } public void testDoToQuery_FromModel() { diff --git a/src/test/java/org/opensearch/knn/index/query/KNNQueryFactoryTests.java b/src/test/java/org/opensearch/knn/index/query/KNNQueryFactoryTests.java index 908ea1021..0f8f43bf2 100644 --- a/src/test/java/org/opensearch/knn/index/query/KNNQueryFactoryTests.java +++ b/src/test/java/org/opensearch/knn/index/query/KNNQueryFactoryTests.java @@ -5,7 +5,7 @@ package org.opensearch.knn.index.query; -import org.apache.lucene.search.KnnVectorQuery; +import org.apache.lucene.search.KnnFloatVectorQuery; import org.apache.lucene.search.Query; import org.opensearch.index.mapper.MappedFieldType; import org.opensearch.index.query.QueryBuilder; @@ -47,7 +47,7 @@ public void testCreateLuceneDefaultQuery() { .collect(Collectors.toList()); for (KNNEngine knnEngine : luceneDefaultQueryEngineList) { Query query = KNNQueryFactory.create(knnEngine, testIndexName, testFieldName, testQueryVector, testK); - assertTrue(query instanceof KnnVectorQuery); + assertTrue(query.getClass().isAssignableFrom(KnnFloatVectorQuery.class)); } } @@ -70,7 +70,7 @@ public void testCreateLuceneQueryWithFilter() { .filter(filter) .build(); Query query = KNNQueryFactory.create(createQueryRequest); - assertTrue(query instanceof KnnVectorQuery); + assertTrue(query.getClass().isAssignableFrom(KnnFloatVectorQuery.class)); } } } From 38e72c65f867876220335dc44f222192fc27ebe4 Mon Sep 17 00:00:00 2001 From: Vijayan Balasubramanian Date: Wed, 22 Feb 2023 11:19:14 -0800 Subject: [PATCH 078/416] Add release note to 2.6 Signed-off-by: Vijayan Balasubramanian (cherry picked from commit 15e5c4fa7cd9185641769b1ef66274d66f1faeec) --- .../opensearch-knn.release-notes-2.6.0.0.md | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 release-notes/opensearch-knn.release-notes-2.6.0.0.md diff --git a/release-notes/opensearch-knn.release-notes-2.6.0.0.md b/release-notes/opensearch-knn.release-notes-2.6.0.0.md new file mode 100644 index 000000000..639e6b955 --- /dev/null +++ b/release-notes/opensearch-knn.release-notes-2.6.0.0.md @@ -0,0 +1,24 @@ +## Version 2.6.0.0 Release Notes + +Compatible with OpenSearch 2.6.0 + + +### Bug Fixes + +* Remove latestSettings cache from KNNSettings ([#727](https://github.com/opensearch-project/k-NN/pull/727)) + +### Infrastructure + +* Add p99.9, p100 and num_of_segments metrics to perf-tool ([#739](https://github.com/opensearch-project/k-NN/pull/739)) +* Update bwc to 2.6.0-SNAPSHOT ([#723](https://github.com/opensearch-project/k-NN/pull/723)) +* Add Windows Support to BWC Tests ([#726](https://github.com/opensearch-project/k-NN/pull/726)) +* Add test for KNNWeight ([#759](https://github.com/opensearch-project/k-NN/pull/759)) +* Set NoMergePolicy for codec tests ([#754](https://github.com/opensearch-project/k-NN/pull/754)) + +### Maintenance + +* Replace KnnQueryVector by KnnFloatVectorQuery for Lucene knn ([#767](https://github.com/opensearch-project/k-NN/pull/767)) + +### Refactoring + +* Refactor structure of stats module ([#736](https://github.com/opensearch-project/k-NN/pull/736)) From 68ff671ab6bc838f68bd4201452b6ba4eb5e3d46 Mon Sep 17 00:00:00 2001 From: Vijayan Balasubramanian Date: Tue, 28 Feb 2023 16:26:35 -0800 Subject: [PATCH 079/416] Backport #782 to 2.x (#783) * Add GHA for publishing artifacts * Add new repository to publish Signed-off-by: Vijayan Balasubramanian --- .github/workflows/maven-publish.yml | 35 +++++++++++++++++++++++++++++ build.gradle | 10 +++++++++ 2 files changed, 45 insertions(+) create mode 100644 .github/workflows/maven-publish.yml diff --git a/.github/workflows/maven-publish.yml b/.github/workflows/maven-publish.yml new file mode 100644 index 000000000..724e3a213 --- /dev/null +++ b/.github/workflows/maven-publish.yml @@ -0,0 +1,35 @@ +name: Publish snapshots to maven + +on: + workflow_dispatch: + push: + branches: + - 'main' + - '1.*' + - '2.*' + +jobs: + build-and-publish-snapshots: + runs-on: ubuntu-latest + + permissions: + id-token: write + contents: write + + steps: + - uses: actions/setup-java@v3 + with: + distribution: temurin # Temurin is a distribution of adoptium + java-version: 11 + - uses: actions/checkout@v3 + - uses: aws-actions/configure-aws-credentials@v1 + with: + role-to-assume: ${{ secrets.PUBLISH_SNAPSHOTS_ROLE }} + aws-region: us-east-1 + - name: publish snapshots to maven + run: | + export SONATYPE_USERNAME=$(aws secretsmanager get-secret-value --secret-id maven-snapshots-username --query SecretString --output text) + export SONATYPE_PASSWORD=$(aws secretsmanager get-secret-value --secret-id maven-snapshots-password --query SecretString --output text) + echo "::add-mask::$SONATYPE_USERNAME" + echo "::add-mask::$SONATYPE_PASSWORD" + ./gradlew publishPluginZipPublicationToSnapshotsRepository diff --git a/build.gradle b/build.gradle index 3e1f97eb4..3013eb90d 100644 --- a/build.gradle +++ b/build.gradle @@ -84,6 +84,16 @@ allprojects { } publishing { + repositories { + maven { + name = "Snapshots" + url = "https://aws.oss.sonatype.org/content/repositories/snapshots" + credentials { + username "$System.env.SONATYPE_USERNAME" + password "$System.env.SONATYPE_PASSWORD" + } + } + } publications { pluginZip(MavenPublication) { publication -> pom { From 582369f81346bab48e121096b9d1b0bdebad9fa7 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Tue, 28 Feb 2023 16:42:22 -0800 Subject: [PATCH 080/416] Increment version to 2.7.0-SNAPSHOT (#780) Signed-off-by: balasvij@amazon.com --- .github/workflows/backwards_compatibility_tests_workflow.yml | 4 ++-- build.gradle | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/backwards_compatibility_tests_workflow.yml b/.github/workflows/backwards_compatibility_tests_workflow.yml index e194d47b0..193fe88f9 100644 --- a/.github/workflows/backwards_compatibility_tests_workflow.yml +++ b/.github/workflows/backwards_compatibility_tests_workflow.yml @@ -15,7 +15,7 @@ jobs: matrix: java: [ 11, 17 ] bwc_version : [ "1.1.0", "1.2.4", "1.3.8", "2.0.1", "2.1.0", "2.2.1", "2.3.0", "2.4.1", "2.5.0" ] - opensearch_version : [ "2.6.0-SNAPSHOT" ] + opensearch_version : [ "2.7.0-SNAPSHOT" ] name: k-NN Restart-Upgrade BWC Tests runs-on: ubuntu-latest @@ -47,7 +47,7 @@ jobs: matrix: java: [ 11, 17 ] bwc_version: [ "1.3.8", "2.0.1", "2.1.0", "2.2.1", "2.3.0", "2.4.1", "2.5.0" ] - opensearch_version: [ "2.6.0-SNAPSHOT" ] + opensearch_version: [ "2.7.0-SNAPSHOT" ] name: k-NN Rolling-Upgrade BWC Tests runs-on: ubuntu-latest diff --git a/build.gradle b/build.gradle index 3013eb90d..2945fa7b5 100644 --- a/build.gradle +++ b/build.gradle @@ -10,7 +10,7 @@ buildscript { ext { // build.version_qualifier parameter applies to knn plugin artifacts only. OpenSearch version must be set // explicitly as 'opensearch.version' property, for instance opensearch.version=2.0.0-rc1-SNAPSHOT - opensearch_version = System.getProperty("opensearch.version", "2.6.0-SNAPSHOT") + opensearch_version = System.getProperty("opensearch.version", "2.7.0-SNAPSHOT") version_qualifier = System.getProperty("build.version_qualifier", "") opensearch_group = "org.opensearch" } From 62991a39234d13331c9cc21855f04be71f98b7cf Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Mon, 13 Mar 2023 14:54:49 -0700 Subject: [PATCH 081/416] Adding filter type to filtering release configs (#792) (#799) Signed-off-by: Martin Gaievski (cherry picked from commit e4cf3d40dbb620087588ee379627a4d11bc0a48a) Co-authored-by: Martin Gaievski --- .../lucene-hnsw/filtering/relaxed-filter/relaxed-filter-test.yml | 1 + .../filtering/restrictive-filter/restrictive-filter-test.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/benchmarks/perf-tool/release-configs/lucene-hnsw/filtering/relaxed-filter/relaxed-filter-test.yml b/benchmarks/perf-tool/release-configs/lucene-hnsw/filtering/relaxed-filter/relaxed-filter-test.yml index a47782649..f20fba203 100644 --- a/benchmarks/perf-tool/release-configs/lucene-hnsw/filtering/relaxed-filter/relaxed-filter-test.yml +++ b/benchmarks/perf-tool/release-configs/lucene-hnsw/filtering/relaxed-filter/relaxed-filter-test.yml @@ -31,3 +31,4 @@ steps: neighbors_path: [DATASET_PATH]/sift-128-euclidean-with-filters.hdf5 neighbors_dataset: neighbors_filter_5 filter_spec: [INDEX_SPEC_PATH]/relaxed-filter-spec.json + filter_type: FILTER diff --git a/benchmarks/perf-tool/release-configs/lucene-hnsw/filtering/restrictive-filter/restrictive-filter-test.yml b/benchmarks/perf-tool/release-configs/lucene-hnsw/filtering/restrictive-filter/restrictive-filter-test.yml index 61e55f113..af5155893 100644 --- a/benchmarks/perf-tool/release-configs/lucene-hnsw/filtering/restrictive-filter/restrictive-filter-test.yml +++ b/benchmarks/perf-tool/release-configs/lucene-hnsw/filtering/restrictive-filter/restrictive-filter-test.yml @@ -31,3 +31,4 @@ steps: neighbors_path: [DATASET_PATH]/sift-128-euclidean-with-filters.hdf5 neighbors_dataset: neighbors_filter_4 filter_spec: [INDEX_SPEC_PATH]/restrictive-filter-test.yml + filter_type: FILTER From 8409ba00722e11205a87a1f88b15ffbc1584ec74 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Mon, 13 Mar 2023 16:39:26 -0700 Subject: [PATCH 082/416] Add CHANGELOG (#800) (#801) Signed-off-by: Heemin Kim (cherry picked from commit 8c616fcce562b23656d8d6c07b101cc646d2b0ef) Co-authored-by: Heemin Kim --- .github/workflows/changelog_verifier.yml | 18 ++++++++++++ CHANGELOG.md | 24 +++++++++++++++ CONTRIBUTING.md | 37 ++++++++++++++++++++++++ 3 files changed, 79 insertions(+) create mode 100644 .github/workflows/changelog_verifier.yml create mode 100644 CHANGELOG.md diff --git a/.github/workflows/changelog_verifier.yml b/.github/workflows/changelog_verifier.yml new file mode 100644 index 000000000..992a38b62 --- /dev/null +++ b/.github/workflows/changelog_verifier.yml @@ -0,0 +1,18 @@ +name: "Changelog Verifier" +on: + pull_request: + types: [opened, edited, review_requested, synchronize, reopened, ready_for_review, labeled, unlabeled] + +jobs: + # Enforces the update of a changelog file on every pull request + verify-changelog: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + token: ${{ secrets.GITHUB_TOKEN }} + ref: ${{ github.event.pull_request.head.sha }} + + - uses: dangoslen/changelog-enforcer@v3 + with: + skipLabels: "autocut, skip-changelog" diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 000000000..5577efe02 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,24 @@ +# CHANGELOG +All notable changes to this project are documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). See the [CONTRIBUTING guide](./CONTRIBUTING.md#Changelog) for instructions on how to add changelog entries. + +## [Unreleased 3.0](https://github.com/opensearch-project/k-NN/compare/2.x...HEAD) +### Features +### Enhancements +### Bug Fixes +### Infrastructure +### Documentation +### Maintenance +### Refactoring + +## [Unreleased 2.x](https://github.com/opensearch-project/k-NN/compare/2.6...2.x) +### Features +### Enhancements +### Bug Fixes +### Infrastructure +* Adding filter type to filtering release configs ([#792](https://github.com/opensearch-project/k-NN/pull/792)) +* Add CHANGELOG ([#800](https://github.com/opensearch-project/k-NN/pull/800)) +### Documentation +### Maintenance +### Refactoring \ No newline at end of file diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 958c2986d..49c6b1d9b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -8,6 +8,7 @@ OpenSearch k-NN is a community project that is built and maintained by people ju - [Ways to Contribute](#ways-to-contribute) - [Developer Certificate of Origin](#developer-certificate-of-origin) - [License Headers](#license-headers) +- [Changelog](#changelog) - [Review Process](#review-process) @@ -129,6 +130,42 @@ New files in your code contributions should contain the following license header # SPDX-License-Identifier: Apache-2.0 ``` +## Changelog + +OpenSearch maintains version specific changelog by enforcing a change to the ongoing [CHANGELOG](CHANGELOG.md) file adhering to the [Keep A Changelog](https://keepachangelog.com/en/1.0.0/) format. The purpose of the changelog is for the contributors and maintainers to incrementally build the release notes throughout the development process to avoid a painful and error-prone process of attempting to compile the release notes at release time. On each release the "unreleased" entries of the changelog are moved to the appropriate release notes document in the `./release-notes` folder. Also, incrementally building the changelog provides a concise, human-readable list of significant features that have been added to the unreleased version under development. + +### Which changes require a CHANGELOG entry? +Changelogs are intended for operators/administrators, developers integrating with libraries and APIs, and end-users interacting with OpenSearch Dashboards and/or the REST API (collectively referred to as "user"). In short, any change that a user of OpenSearch might want to be aware of should be included in the changelog. The changelog is _not_ intended to replace the git commit log that developers of OpenSearch itself rely upon. The following are some examples of changes that should be in the changelog: + +- A newly added feature +- A fix for a user-facing bug +- Dependency updates +- Fixes for security issues + +The following are some examples where a changelog entry is not necessary: + +- Adding, modifying, or fixing tests +- An incremental PR for a larger feature (such features should include _one_ changelog entry for the feature) +- Documentation changes or code refactoring +- Build-related changes + +Any PR that does not include a changelog entry will result in a failure of the validation workflow in GitHub. If the contributor and maintainers agree that no changelog entry is required, then the `skip-changelog` label can be applied to the PR which will result in the workflow passing. + +### How to add my changes to [CHANGELOG](CHANGELOG.md)? + +Adding in the change is two step process: +1. Add your changes to the corresponding section within the CHANGELOG file with dummy pull request information, publish the PR +2. Update the entry for your change in [`CHANGELOG.md`](CHANGELOG.md) and make sure that you reference the pull request there. + +### Where should I put my CHANGELOG entry? +Please review the [branching strategy](https://github.com/opensearch-project/.github/blob/main/RELEASING.md#opensearch-branching) document. The changelog on the `main` branch will contain sections for the _next major_ and _next minor_ releases. Your entry should go into the section it is intended to be released in. In practice, most changes to `main` will be backported to the next minor release so most entries will likely be in that section. + +The following examples assume the _next major_ release on main is 3.0, then _next minor_ release is 2.5, and the _current_ release is 2.4. + +- **Add a new feature to release in next minor:** Add a changelog entry to `[Unreleased 2.x]` on main, then backport to 2.x (including the changelog entry). +- **Introduce a breaking API change to release in next major:** Add a changelog entry to `[Unreleased 3.0]` on main, do not backport. +- **Upgrade a dependency to fix a CVE:** Add a changelog entry to `[Unreleased 2.x]` on main, then backport to 2.x (including the changelog entry), then backport to 2.4 and ensure the changelog entry is added to `[Unreleased 2.4.1]`. + ## Review Process We deeply appreciate everyone who takes the time to make a contribution. We will review all contributions as quickly as possible. As a reminder, [opening an issue](https://github.com/opensearch-project/k-NN/issues/new/choose) discussing your change before you make it is the best way to smooth the PR process. This will prevent a rejection because someone else is already working on the problem, or because the solution is incompatible with the architectural direction. From 6ff359cd1bc1bf5b43af4d17161b9a861f06a8b9 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Thu, 16 Mar 2023 13:12:23 -0500 Subject: [PATCH 083/416] Bump byte-buddy version from 1.12.22 to 1.14.2 (#804) (#807) Signed-off-by: Naveen Tatikonda (cherry picked from commit e1d2e45ff04dab2624ddd0f300b407f26fdb8f61) Co-authored-by: Naveen Tatikonda --- CHANGELOG.md | 1 + build.gradle | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5577efe02..2f9bf5da2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Infrastructure * Adding filter type to filtering release configs ([#792](https://github.com/opensearch-project/k-NN/pull/792)) * Add CHANGELOG ([#800](https://github.com/opensearch-project/k-NN/pull/800)) +* Bump byte-buddy version from 1.12.22 to 1.14.2 ([#804](https://github.com/opensearch-project/k-NN/pull/804)) ### Documentation ### Maintenance ### Refactoring \ No newline at end of file diff --git a/build.gradle b/build.gradle index 2945fa7b5..5b59f13bf 100644 --- a/build.gradle +++ b/build.gradle @@ -175,9 +175,9 @@ dependencies { api group: 'com.google.guava', name: 'guava', version:'30.0-jre' api group: 'commons-lang', name: 'commons-lang', version: '2.6' testFixturesImplementation "org.opensearch.test:framework:${opensearch_version}" - testImplementation group: 'net.bytebuddy', name: 'byte-buddy', version: '1.12.22' + testImplementation group: 'net.bytebuddy', name: 'byte-buddy', version: '1.14.2' testImplementation group: 'org.objenesis', name: 'objenesis', version: '3.2' - testImplementation group: 'net.bytebuddy', name: 'byte-buddy-agent', version: '1.12.22' + testImplementation group: 'net.bytebuddy', name: 'byte-buddy-agent', version: '1.14.2' } From 4dc7359ab1b6c6b679ddcb0b6db066e608790d6f Mon Sep 17 00:00:00 2001 From: Naveen Tatikonda Date: Mon, 20 Mar 2023 15:39:32 -0500 Subject: [PATCH 084/416] Add 2.6.0 to BWC Version Matrix (#810) Signed-off-by: Naveen Tatikonda --- .github/workflows/backwards_compatibility_tests_workflow.yml | 4 ++-- CHANGELOG.md | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/backwards_compatibility_tests_workflow.yml b/.github/workflows/backwards_compatibility_tests_workflow.yml index 193fe88f9..a09d6ec34 100644 --- a/.github/workflows/backwards_compatibility_tests_workflow.yml +++ b/.github/workflows/backwards_compatibility_tests_workflow.yml @@ -14,7 +14,7 @@ jobs: strategy: matrix: java: [ 11, 17 ] - bwc_version : [ "1.1.0", "1.2.4", "1.3.8", "2.0.1", "2.1.0", "2.2.1", "2.3.0", "2.4.1", "2.5.0" ] + bwc_version : [ "1.1.0", "1.2.4", "1.3.8", "2.0.1", "2.1.0", "2.2.1", "2.3.0", "2.4.1", "2.5.0", "2.6.0" ] opensearch_version : [ "2.7.0-SNAPSHOT" ] name: k-NN Restart-Upgrade BWC Tests @@ -46,7 +46,7 @@ jobs: strategy: matrix: java: [ 11, 17 ] - bwc_version: [ "1.3.8", "2.0.1", "2.1.0", "2.2.1", "2.3.0", "2.4.1", "2.5.0" ] + bwc_version: [ "1.3.8", "2.0.1", "2.1.0", "2.2.1", "2.3.0", "2.4.1", "2.5.0", "2.6.0" ] opensearch_version: [ "2.7.0-SNAPSHOT" ] name: k-NN Rolling-Upgrade BWC Tests diff --git a/CHANGELOG.md b/CHANGELOG.md index 2f9bf5da2..893ffecce 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), * Adding filter type to filtering release configs ([#792](https://github.com/opensearch-project/k-NN/pull/792)) * Add CHANGELOG ([#800](https://github.com/opensearch-project/k-NN/pull/800)) * Bump byte-buddy version from 1.12.22 to 1.14.2 ([#804](https://github.com/opensearch-project/k-NN/pull/804)) +* Add 2.6.0 to BWC Version Matrix ([#810](https://github.com/opensearch-project/k-NN/pull/810)) ### Documentation ### Maintenance ### Refactoring \ No newline at end of file From fd1402876cdcfb6eb7d5152ac3ec542a035d359b Mon Sep 17 00:00:00 2001 From: Naveen Tatikonda Date: Mon, 27 Mar 2023 12:05:51 -0500 Subject: [PATCH 085/416] Update BWC Version with OpenSearch Version Bump (#813) Signed-off-by: Naveen Tatikonda --- CHANGELOG.md | 1 + build.gradle | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 893ffecce..4bf24601e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), * Add CHANGELOG ([#800](https://github.com/opensearch-project/k-NN/pull/800)) * Bump byte-buddy version from 1.12.22 to 1.14.2 ([#804](https://github.com/opensearch-project/k-NN/pull/804)) * Add 2.6.0 to BWC Version Matrix ([#810](https://github.com/opensearch-project/k-NN/pull/810)) +* Update BWC Version with OpenSearch Version Bump ([#813](https://github.com/opensearch-project/k-NN/pull/813)) ### Documentation ### Maintenance ### Refactoring \ No newline at end of file diff --git a/build.gradle b/build.gradle index 5b59f13bf..8368a06ff 100644 --- a/build.gradle +++ b/build.gradle @@ -322,5 +322,10 @@ task updateVersion { } } ant.replaceregexp(file:'build.gradle', match: '"opensearch.version", "\\d.*"', replace: '"opensearch.version", "' + newVersion.tokenize('-')[0] + '-SNAPSHOT"', flags:'g', byline:true) + // Extract the oldBWCVersion from the existing OpenSearch Version (oldBWCVersion = major + (minor-1) + patch) + ext.os_version_without_snapshot = opensearch_version.tokenize('-')[0] + ext.oldBWCVersion = os_version_without_snapshot.tokenize('.')[0] + '.' + Integer.toString(Integer.valueOf(os_version_without_snapshot.tokenize('.')[1]) - 1) + '.' + os_version_without_snapshot.tokenize('.')[2] + // Include the current OpenSearch Version before version bump to the bwc_version matrix + ant.replaceregexp(file:".github/workflows/backwards_compatibility_tests_workflow.yml", match: oldBWCVersion, replace: oldBWCVersion + '", "' + opensearch_version.tokenize('-')[0], flags:'g', byline:true) } } From 6eba6da6e3ef5f18eded1d809fc8200dbe4433dc Mon Sep 17 00:00:00 2001 From: Naveen Tatikonda Date: Wed, 29 Mar 2023 12:12:51 -0500 Subject: [PATCH 086/416] Fixing the imports failures happening due to changes in OpenSearch Core. (#781) (#824) Signed-off-by: Navneet Verma (cherry picked from commit 9ff933184c6796edfd05b8eb21294191a3a061cd) Co-authored-by: Navneet Verma --- .../java/org/opensearch/knn/bwc/ModelIT.java | 16 +++++++++++----- .../opensearch/knn/index/KNNMethodContext.java | 4 ++-- .../knn/index/MethodComponentContext.java | 4 ++-- .../codec/KNN80Codec/KNN80DocValuesConsumer.java | 4 ++-- .../knn/index/mapper/KNNVectorFieldMapper.java | 6 +++--- .../knn/index/query/KNNQueryBuilder.java | 6 +++--- .../java/org/opensearch/knn/indices/Model.java | 4 ++-- .../opensearch/knn/indices/ModelGraveyard.java | 4 ++-- .../opensearch/knn/indices/ModelMetadata.java | 4 ++-- .../org/opensearch/knn/plugin/KNNPlugin.java | 4 ++-- .../knn/plugin/rest/RestTrainModelHandler.java | 2 +- .../plugin/transport/DeleteModelResponse.java | 4 ++-- .../knn/plugin/transport/GetModelResponse.java | 4 ++-- .../plugin/transport/KNNStatsNodeResponse.java | 4 ++-- .../knn/plugin/transport/KNNStatsResponse.java | 4 ++-- .../knn/plugin/transport/KNNWarmupResponse.java | 2 +- ...TrainingJobRouteDecisionInfoNodeResponse.java | 6 +++--- .../TrainingJobRouteDecisionInfoResponse.java | 4 ++-- .../plugin/transport/TrainingModelResponse.java | 4 ++-- .../opensearch/knn/KNNSingleNodeTestCase.java | 2 +- .../java/org/opensearch/knn/KNNTestCase.java | 2 +- .../java/org/opensearch/knn/index/FaissIT.java | 2 +- .../knn/index/KNNMethodContextTests.java | 4 ++-- .../org/opensearch/knn/index/KNNMethodTests.java | 2 +- .../org/opensearch/knn/index/LuceneEngineIT.java | 2 +- .../knn/index/MethodComponentContextTests.java | 4 ++-- .../knn/index/MethodComponentTests.java | 2 +- .../java/org/opensearch/knn/index/NmslibIT.java | 2 +- .../org/opensearch/knn/index/OpenSearchIT.java | 2 +- .../index/mapper/KNNVectorFieldMapperTests.java | 2 +- .../knn/index/query/KNNQueryBuilderTests.java | 6 +++--- .../knn/index/util/AbstractKNNLibraryTests.java | 2 +- .../opensearch/knn/index/util/FaissTests.java | 2 +- .../opensearch/knn/index/util/LuceneTests.java | 2 +- .../opensearch/knn/indices/ModelDaoTests.java | 2 +- .../knn/indices/ModelGraveyardTests.java | 4 ++-- .../plugin/action/RestDeleteModelHandlerIT.java | 2 +- .../knn/plugin/action/RestKNNStatsHandlerIT.java | 2 +- .../plugin/action/RestSearchModelHandlerIT.java | 2 +- .../plugin/action/RestTrainModelHandlerIT.java | 2 +- .../knn/plugin/script/KNNScriptScoringIT.java | 4 ++-- .../knn/plugin/script/PainlessScriptIT.java | 4 ++-- .../transport/DeleteModelResponseTests.java | 2 +- .../plugin/transport/GetModelResponseTests.java | 2 +- ...ingJobRouteDecisionInfoNodeResponseTests.java | 4 ++-- ...rainingJobRouteDecisionInfoResponseTests.java | 4 ++-- .../transport/TrainingModelResponseTests.java | 4 ++-- .../TrainingModelTransportActionTests.java | 2 +- .../java/org/opensearch/knn/KNNRestTestCase.java | 4 ++-- .../org/opensearch/knn/ODFERestTestCase.java | 6 +++--- .../java/org/opensearch/knn/TestUtils.java | 4 ++-- 51 files changed, 94 insertions(+), 88 deletions(-) diff --git a/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/ModelIT.java b/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/ModelIT.java index f6ff2b1c2..f8e832c50 100644 --- a/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/ModelIT.java +++ b/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/ModelIT.java @@ -11,9 +11,9 @@ import org.opensearch.client.Request; import org.opensearch.client.Response; import org.opensearch.common.Strings; -import org.opensearch.common.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; -import org.opensearch.common.xcontent.XContentParser; +import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.util.KNNEngine; @@ -25,7 +25,9 @@ import org.opensearch.search.SearchHit; import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.util.Arrays; +import java.util.Locale; import java.util.Map; import static org.opensearch.knn.TestUtils.KNN_BWC_PREFIX; @@ -141,8 +143,8 @@ public void testKNNModelDefault() throws Exception { } // KNN Delete Model test for model in Training State - public void testDeleteTrainingModel() throws IOException, InterruptedException { - byte[] testModelBlob = "hello".getBytes(); + public void testDeleteTrainingModel() throws Exception { + byte[] testModelBlob = "hello".getBytes(StandardCharsets.UTF_8); ModelMetadata testModelMetadata = getModelMetadata(); testModelMetadata.setState(ModelState.TRAINING); if (isRunningAgainstOldCluster()) { @@ -164,7 +166,11 @@ public void testDeleteTrainingModel() throws IOException, InterruptedException { assertEquals(TEST_MODEL_ID_TRAINING, responseMap.get(MODEL_ID)); assertEquals("failed", responseMap.get(DeleteModelResponse.RESULT)); - String errorMessage = String.format("Cannot delete model \"%s\". Model is still in training", TEST_MODEL_ID_TRAINING); + String errorMessage = String.format( + Locale.ROOT, + "Cannot delete model \"%s\". Model is still in " + "training", + TEST_MODEL_ID_TRAINING + ); assertEquals(errorMessage, responseMap.get(DeleteModelResponse.ERROR_MSG)); } } diff --git a/src/main/java/org/opensearch/knn/index/KNNMethodContext.java b/src/main/java/org/opensearch/knn/index/KNNMethodContext.java index 3fd937155..db97df6aa 100644 --- a/src/main/java/org/opensearch/knn/index/KNNMethodContext.java +++ b/src/main/java/org/opensearch/knn/index/KNNMethodContext.java @@ -19,8 +19,8 @@ import org.opensearch.common.io.stream.StreamOutput; import org.opensearch.common.io.stream.Writeable; import org.opensearch.knn.index.util.KNNEngine; -import org.opensearch.common.xcontent.ToXContentFragment; -import org.opensearch.common.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.ToXContentFragment; +import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.index.mapper.MapperParsingException; import java.io.IOException; diff --git a/src/main/java/org/opensearch/knn/index/MethodComponentContext.java b/src/main/java/org/opensearch/knn/index/MethodComponentContext.java index 4a5e2377a..756d463d9 100644 --- a/src/main/java/org/opensearch/knn/index/MethodComponentContext.java +++ b/src/main/java/org/opensearch/knn/index/MethodComponentContext.java @@ -16,8 +16,8 @@ import org.opensearch.common.io.stream.StreamInput; import org.opensearch.common.io.stream.StreamOutput; import org.opensearch.common.io.stream.Writeable; -import org.opensearch.common.xcontent.ToXContentFragment; -import org.opensearch.common.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.ToXContentFragment; +import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.index.mapper.MapperParsingException; import java.io.IOException; diff --git a/src/main/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumer.java b/src/main/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumer.java index 1368f63de..691f789e0 100644 --- a/src/main/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumer.java +++ b/src/main/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumer.java @@ -9,10 +9,10 @@ import lombok.NonNull; import lombok.extern.log4j.Log4j2; import org.apache.lucene.store.ChecksumIndexInput; -import org.opensearch.common.xcontent.DeprecationHandler; -import org.opensearch.common.xcontent.NamedXContentRegistry; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentType; +import org.opensearch.core.xcontent.DeprecationHandler; +import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.knn.index.KNNSettings; import org.opensearch.knn.jni.JNIService; import org.opensearch.knn.index.SpaceType; diff --git a/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapper.java b/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapper.java index 3964dd98f..ab45c384f 100644 --- a/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapper.java +++ b/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapper.java @@ -17,9 +17,9 @@ import org.apache.lucene.search.DocValuesFieldExistsQuery; import org.apache.lucene.search.Query; import org.opensearch.common.Explicit; -import org.opensearch.common.xcontent.ToXContent; -import org.opensearch.common.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentParser; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.support.XContentMapValues; import org.opensearch.index.fielddata.IndexFieldData; import org.opensearch.index.mapper.FieldMapper; diff --git a/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java b/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java index 20cbd3a03..3de0e69d4 100644 --- a/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java +++ b/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java @@ -17,13 +17,13 @@ import org.opensearch.knn.indices.ModelMetadata; import org.opensearch.knn.plugin.stats.KNNCounter; import org.apache.lucene.search.Query; -import org.opensearch.common.ParseField; +import org.opensearch.core.ParseField; import org.opensearch.common.ParsingException; import org.opensearch.common.Strings; import org.opensearch.common.io.stream.StreamInput; import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentParser; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; import org.opensearch.index.mapper.MappedFieldType; import org.opensearch.index.query.AbstractQueryBuilder; import org.opensearch.index.query.QueryShardContext; diff --git a/src/main/java/org/opensearch/knn/indices/Model.java b/src/main/java/org/opensearch/knn/indices/Model.java index 18e085f5a..c0071ad94 100644 --- a/src/main/java/org/opensearch/knn/indices/Model.java +++ b/src/main/java/org/opensearch/knn/indices/Model.java @@ -15,8 +15,8 @@ import org.opensearch.common.io.stream.StreamInput; import org.opensearch.common.io.stream.StreamOutput; import org.opensearch.common.io.stream.Writeable; -import org.opensearch.common.xcontent.ToXContentObject; -import org.opensearch.common.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.ToXContentObject; +import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.knn.common.KNNConstants; import java.io.IOException; diff --git a/src/main/java/org/opensearch/knn/indices/ModelGraveyard.java b/src/main/java/org/opensearch/knn/indices/ModelGraveyard.java index 5db6d77fd..e7e11e66f 100644 --- a/src/main/java/org/opensearch/knn/indices/ModelGraveyard.java +++ b/src/main/java/org/opensearch/knn/indices/ModelGraveyard.java @@ -14,8 +14,8 @@ import org.opensearch.cluster.metadata.Metadata; import org.opensearch.common.io.stream.StreamInput; import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentParser; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; import java.io.IOException; import java.util.Collections; diff --git a/src/main/java/org/opensearch/knn/indices/ModelMetadata.java b/src/main/java/org/opensearch/knn/indices/ModelMetadata.java index 679b8ea80..88994cf52 100644 --- a/src/main/java/org/opensearch/knn/indices/ModelMetadata.java +++ b/src/main/java/org/opensearch/knn/indices/ModelMetadata.java @@ -16,8 +16,8 @@ import org.opensearch.common.io.stream.StreamInput; import org.opensearch.common.io.stream.StreamOutput; import org.opensearch.common.io.stream.Writeable; -import org.opensearch.common.xcontent.ToXContentObject; -import org.opensearch.common.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.ToXContentObject; +import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.knn.common.KNNConstants; import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.util.KNNEngine; diff --git a/src/main/java/org/opensearch/knn/plugin/KNNPlugin.java b/src/main/java/org/opensearch/knn/plugin/KNNPlugin.java index de4932c47..bf89644cb 100644 --- a/src/main/java/org/opensearch/knn/plugin/KNNPlugin.java +++ b/src/main/java/org/opensearch/knn/plugin/KNNPlugin.java @@ -7,7 +7,7 @@ import org.opensearch.cluster.NamedDiff; import org.opensearch.cluster.metadata.Metadata; -import org.opensearch.common.ParseField; +import org.opensearch.core.ParseField; import org.opensearch.index.codec.CodecServiceFactory; import org.opensearch.index.engine.EngineFactory; import org.opensearch.indices.SystemIndexDescriptor; @@ -54,7 +54,7 @@ import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; import org.opensearch.common.settings.SettingsFilter; -import org.opensearch.common.xcontent.NamedXContentRegistry; +import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.env.Environment; import org.opensearch.env.NodeEnvironment; import org.opensearch.index.IndexModule; diff --git a/src/main/java/org/opensearch/knn/plugin/rest/RestTrainModelHandler.java b/src/main/java/org/opensearch/knn/plugin/rest/RestTrainModelHandler.java index 9ddf7410b..67ce80959 100644 --- a/src/main/java/org/opensearch/knn/plugin/rest/RestTrainModelHandler.java +++ b/src/main/java/org/opensearch/knn/plugin/rest/RestTrainModelHandler.java @@ -13,7 +13,7 @@ import com.google.common.collect.ImmutableList; import org.opensearch.client.node.NodeClient; -import org.opensearch.common.xcontent.XContentParser; +import org.opensearch.core.xcontent.XContentParser; import org.opensearch.index.mapper.NumberFieldMapper; import org.opensearch.knn.index.KNNMethodContext; import org.opensearch.knn.plugin.KNNPlugin; diff --git a/src/main/java/org/opensearch/knn/plugin/transport/DeleteModelResponse.java b/src/main/java/org/opensearch/knn/plugin/transport/DeleteModelResponse.java index af63fd1e7..1b0d8e3c8 100644 --- a/src/main/java/org/opensearch/knn/plugin/transport/DeleteModelResponse.java +++ b/src/main/java/org/opensearch/knn/plugin/transport/DeleteModelResponse.java @@ -15,8 +15,8 @@ import org.opensearch.common.Strings; import org.opensearch.common.io.stream.StreamInput; import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.xcontent.ToXContentObject; -import org.opensearch.common.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.ToXContentObject; +import org.opensearch.core.xcontent.XContentBuilder; import java.io.IOException; diff --git a/src/main/java/org/opensearch/knn/plugin/transport/GetModelResponse.java b/src/main/java/org/opensearch/knn/plugin/transport/GetModelResponse.java index 6c61befef..c94c6769e 100644 --- a/src/main/java/org/opensearch/knn/plugin/transport/GetModelResponse.java +++ b/src/main/java/org/opensearch/knn/plugin/transport/GetModelResponse.java @@ -13,8 +13,8 @@ import org.opensearch.action.ActionResponse; import org.opensearch.common.io.stream.StreamInput; import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.xcontent.ToXContentObject; -import org.opensearch.common.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.ToXContentObject; +import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.knn.indices.Model; import java.io.IOException; diff --git a/src/main/java/org/opensearch/knn/plugin/transport/KNNStatsNodeResponse.java b/src/main/java/org/opensearch/knn/plugin/transport/KNNStatsNodeResponse.java index 5bdf03ea7..1f1164bd8 100644 --- a/src/main/java/org/opensearch/knn/plugin/transport/KNNStatsNodeResponse.java +++ b/src/main/java/org/opensearch/knn/plugin/transport/KNNStatsNodeResponse.java @@ -9,8 +9,8 @@ import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.common.io.stream.StreamInput; import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.xcontent.ToXContentFragment; -import org.opensearch.common.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.ToXContentFragment; +import org.opensearch.core.xcontent.XContentBuilder; import java.io.IOException; import java.util.Map; diff --git a/src/main/java/org/opensearch/knn/plugin/transport/KNNStatsResponse.java b/src/main/java/org/opensearch/knn/plugin/transport/KNNStatsResponse.java index 0679eefaa..f108e536e 100644 --- a/src/main/java/org/opensearch/knn/plugin/transport/KNNStatsResponse.java +++ b/src/main/java/org/opensearch/knn/plugin/transport/KNNStatsResponse.java @@ -11,8 +11,8 @@ import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.common.io.stream.StreamInput; import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.xcontent.ToXContentObject; -import org.opensearch.common.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.ToXContentObject; +import org.opensearch.core.xcontent.XContentBuilder; import java.io.IOException; import java.util.List; diff --git a/src/main/java/org/opensearch/knn/plugin/transport/KNNWarmupResponse.java b/src/main/java/org/opensearch/knn/plugin/transport/KNNWarmupResponse.java index 3409bb395..842fdf630 100644 --- a/src/main/java/org/opensearch/knn/plugin/transport/KNNWarmupResponse.java +++ b/src/main/java/org/opensearch/knn/plugin/transport/KNNWarmupResponse.java @@ -8,7 +8,7 @@ import org.opensearch.action.support.DefaultShardOperationFailedException; import org.opensearch.action.support.broadcast.BroadcastResponse; import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.xcontent.ToXContentObject; +import org.opensearch.core.xcontent.ToXContentObject; import java.io.IOException; import java.util.List; diff --git a/src/main/java/org/opensearch/knn/plugin/transport/TrainingJobRouteDecisionInfoNodeResponse.java b/src/main/java/org/opensearch/knn/plugin/transport/TrainingJobRouteDecisionInfoNodeResponse.java index 2782a8374..0947f7431 100644 --- a/src/main/java/org/opensearch/knn/plugin/transport/TrainingJobRouteDecisionInfoNodeResponse.java +++ b/src/main/java/org/opensearch/knn/plugin/transport/TrainingJobRouteDecisionInfoNodeResponse.java @@ -15,9 +15,9 @@ import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.common.io.stream.StreamInput; import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.xcontent.ToXContent; -import org.opensearch.common.xcontent.ToXContentFragment; -import org.opensearch.common.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.core.xcontent.ToXContentFragment; +import org.opensearch.core.xcontent.XContentBuilder; import java.io.IOException; diff --git a/src/main/java/org/opensearch/knn/plugin/transport/TrainingJobRouteDecisionInfoResponse.java b/src/main/java/org/opensearch/knn/plugin/transport/TrainingJobRouteDecisionInfoResponse.java index 4fe50410c..906ffcab8 100644 --- a/src/main/java/org/opensearch/knn/plugin/transport/TrainingJobRouteDecisionInfoResponse.java +++ b/src/main/java/org/opensearch/knn/plugin/transport/TrainingJobRouteDecisionInfoResponse.java @@ -17,8 +17,8 @@ import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.common.io.stream.StreamInput; import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.xcontent.ToXContentObject; -import org.opensearch.common.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.ToXContentObject; +import org.opensearch.core.xcontent.XContentBuilder; import java.io.IOException; import java.util.List; diff --git a/src/main/java/org/opensearch/knn/plugin/transport/TrainingModelResponse.java b/src/main/java/org/opensearch/knn/plugin/transport/TrainingModelResponse.java index ac38bca8e..ccbd718b3 100644 --- a/src/main/java/org/opensearch/knn/plugin/transport/TrainingModelResponse.java +++ b/src/main/java/org/opensearch/knn/plugin/transport/TrainingModelResponse.java @@ -14,8 +14,8 @@ import org.opensearch.action.ActionResponse; import org.opensearch.common.io.stream.StreamInput; import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.xcontent.ToXContentObject; -import org.opensearch.common.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.ToXContentObject; +import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.knn.common.KNNConstants; import java.io.IOException; diff --git a/src/test/java/org/opensearch/knn/KNNSingleNodeTestCase.java b/src/test/java/org/opensearch/knn/KNNSingleNodeTestCase.java index c83433eca..48effbb39 100644 --- a/src/test/java/org/opensearch/knn/KNNSingleNodeTestCase.java +++ b/src/test/java/org/opensearch/knn/KNNSingleNodeTestCase.java @@ -21,7 +21,7 @@ import org.opensearch.action.search.SearchResponse; import org.opensearch.action.support.WriteRequest; import org.opensearch.common.settings.Settings; -import org.opensearch.common.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.index.IndexService; import org.opensearch.plugins.Plugin; diff --git a/src/test/java/org/opensearch/knn/KNNTestCase.java b/src/test/java/org/opensearch/knn/KNNTestCase.java index f5955f959..ed1dd0e4e 100644 --- a/src/test/java/org/opensearch/knn/KNNTestCase.java +++ b/src/test/java/org/opensearch/knn/KNNTestCase.java @@ -15,7 +15,7 @@ import org.opensearch.knn.index.memory.NativeMemoryCacheManager; import org.opensearch.knn.plugin.stats.KNNCounter; import org.opensearch.common.bytes.BytesReference; -import org.opensearch.common.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentHelper; import org.opensearch.test.OpenSearchTestCase; diff --git a/src/test/java/org/opensearch/knn/index/FaissIT.java b/src/test/java/org/opensearch/knn/index/FaissIT.java index 979d1de73..d4a66a800 100644 --- a/src/test/java/org/opensearch/knn/index/FaissIT.java +++ b/src/test/java/org/opensearch/knn/index/FaissIT.java @@ -16,7 +16,7 @@ import org.apache.http.util.EntityUtils; import org.junit.BeforeClass; import org.opensearch.client.Response; -import org.opensearch.common.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.knn.KNNRestTestCase; import org.opensearch.common.Strings; import org.opensearch.common.xcontent.XContentFactory; diff --git a/src/test/java/org/opensearch/knn/index/KNNMethodContextTests.java b/src/test/java/org/opensearch/knn/index/KNNMethodContextTests.java index 81816daf1..a3011cef5 100644 --- a/src/test/java/org/opensearch/knn/index/KNNMethodContextTests.java +++ b/src/test/java/org/opensearch/knn/index/KNNMethodContextTests.java @@ -15,8 +15,8 @@ import org.opensearch.knn.KNNTestCase; import org.opensearch.knn.index.util.KNNEngine; import com.google.common.collect.ImmutableMap; -import org.opensearch.common.xcontent.ToXContent; -import org.opensearch.common.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.index.mapper.MapperParsingException; diff --git a/src/test/java/org/opensearch/knn/index/KNNMethodTests.java b/src/test/java/org/opensearch/knn/index/KNNMethodTests.java index b1dbaab32..d4dd989f7 100644 --- a/src/test/java/org/opensearch/knn/index/KNNMethodTests.java +++ b/src/test/java/org/opensearch/knn/index/KNNMethodTests.java @@ -13,7 +13,7 @@ import com.google.common.collect.ImmutableMap; import org.opensearch.knn.KNNTestCase; -import org.opensearch.common.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.knn.common.KNNConstants; import org.opensearch.knn.index.util.KNNEngine; diff --git a/src/test/java/org/opensearch/knn/index/LuceneEngineIT.java b/src/test/java/org/opensearch/knn/index/LuceneEngineIT.java index 2b6ee25cc..2266d755d 100644 --- a/src/test/java/org/opensearch/knn/index/LuceneEngineIT.java +++ b/src/test/java/org/opensearch/knn/index/LuceneEngineIT.java @@ -16,7 +16,7 @@ import org.opensearch.client.Response; import org.opensearch.client.ResponseException; import org.opensearch.common.Strings; -import org.opensearch.common.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.index.query.QueryBuilders; import org.opensearch.knn.KNNRestTestCase; diff --git a/src/test/java/org/opensearch/knn/index/MethodComponentContextTests.java b/src/test/java/org/opensearch/knn/index/MethodComponentContextTests.java index 00c87503c..cbbb872cf 100644 --- a/src/test/java/org/opensearch/knn/index/MethodComponentContextTests.java +++ b/src/test/java/org/opensearch/knn/index/MethodComponentContextTests.java @@ -14,8 +14,8 @@ import com.google.common.collect.ImmutableMap; import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.knn.KNNTestCase; -import org.opensearch.common.xcontent.ToXContent; -import org.opensearch.common.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.index.mapper.MapperParsingException; diff --git a/src/test/java/org/opensearch/knn/index/MethodComponentTests.java b/src/test/java/org/opensearch/knn/index/MethodComponentTests.java index b752764c3..55ff963a2 100644 --- a/src/test/java/org/opensearch/knn/index/MethodComponentTests.java +++ b/src/test/java/org/opensearch/knn/index/MethodComponentTests.java @@ -13,7 +13,7 @@ import com.google.common.collect.ImmutableMap; import org.opensearch.knn.KNNTestCase; -import org.opensearch.common.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; import java.io.IOException; diff --git a/src/test/java/org/opensearch/knn/index/NmslibIT.java b/src/test/java/org/opensearch/knn/index/NmslibIT.java index ac1e48826..5937c0242 100644 --- a/src/test/java/org/opensearch/knn/index/NmslibIT.java +++ b/src/test/java/org/opensearch/knn/index/NmslibIT.java @@ -16,7 +16,7 @@ import org.apache.http.util.EntityUtils; import org.junit.BeforeClass; import org.opensearch.client.Response; -import org.opensearch.common.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.knn.KNNRestTestCase; import org.opensearch.client.ResponseException; import org.opensearch.common.Strings; diff --git a/src/test/java/org/opensearch/knn/index/OpenSearchIT.java b/src/test/java/org/opensearch/knn/index/OpenSearchIT.java index a5a8437fa..6862a01ac 100644 --- a/src/test/java/org/opensearch/knn/index/OpenSearchIT.java +++ b/src/test/java/org/opensearch/knn/index/OpenSearchIT.java @@ -22,7 +22,7 @@ import org.opensearch.client.ResponseException; import org.opensearch.common.Strings; import org.opensearch.common.settings.Settings; -import org.opensearch.common.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.index.query.ExistsQueryBuilder; import org.opensearch.knn.TestUtils; diff --git a/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java b/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java index 3c311f86b..d4a5b5aea 100644 --- a/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java +++ b/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java @@ -18,7 +18,7 @@ import org.opensearch.common.ValidationException; import org.opensearch.common.settings.IndexScopedSettings; import org.opensearch.common.settings.Settings; -import org.opensearch.common.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.index.IndexSettings; import org.opensearch.index.mapper.ContentPath; diff --git a/src/test/java/org/opensearch/knn/index/query/KNNQueryBuilderTests.java b/src/test/java/org/opensearch/knn/index/query/KNNQueryBuilderTests.java index 982c1a0c0..7eb089b42 100644 --- a/src/test/java/org/opensearch/knn/index/query/KNNQueryBuilderTests.java +++ b/src/test/java/org/opensearch/knn/index/query/KNNQueryBuilderTests.java @@ -15,14 +15,14 @@ import org.opensearch.common.io.stream.NamedWriteableAwareStreamInput; import org.opensearch.common.io.stream.NamedWriteableRegistry; import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.xcontent.NamedXContentRegistry; +import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.index.query.QueryBuilder; import org.opensearch.index.query.QueryBuilders; import org.opensearch.index.query.TermQueryBuilder; import org.opensearch.knn.KNNTestCase; -import org.opensearch.common.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; -import org.opensearch.common.xcontent.XContentParser; +import org.opensearch.core.xcontent.XContentParser; import org.opensearch.index.Index; import org.opensearch.index.mapper.NumberFieldMapper; import org.opensearch.index.query.QueryShardContext; diff --git a/src/test/java/org/opensearch/knn/index/util/AbstractKNNLibraryTests.java b/src/test/java/org/opensearch/knn/index/util/AbstractKNNLibraryTests.java index 66b6eee9f..916e87414 100644 --- a/src/test/java/org/opensearch/knn/index/util/AbstractKNNLibraryTests.java +++ b/src/test/java/org/opensearch/knn/index/util/AbstractKNNLibraryTests.java @@ -7,7 +7,7 @@ import com.google.common.collect.ImmutableMap; import org.opensearch.common.ValidationException; -import org.opensearch.common.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.knn.KNNTestCase; import org.opensearch.knn.common.KNNConstants; diff --git a/src/test/java/org/opensearch/knn/index/util/FaissTests.java b/src/test/java/org/opensearch/knn/index/util/FaissTests.java index abcb5e7f0..01841363d 100644 --- a/src/test/java/org/opensearch/knn/index/util/FaissTests.java +++ b/src/test/java/org/opensearch/knn/index/util/FaissTests.java @@ -5,7 +5,7 @@ package org.opensearch.knn.index.util; -import org.opensearch.common.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.knn.KNNTestCase; import org.opensearch.knn.index.MethodComponent; diff --git a/src/test/java/org/opensearch/knn/index/util/LuceneTests.java b/src/test/java/org/opensearch/knn/index/util/LuceneTests.java index f5c8e45b9..38cacffa4 100644 --- a/src/test/java/org/opensearch/knn/index/util/LuceneTests.java +++ b/src/test/java/org/opensearch/knn/index/util/LuceneTests.java @@ -6,7 +6,7 @@ package org.opensearch.knn.index.util; import org.apache.lucene.util.Version; -import org.opensearch.common.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.knn.KNNTestCase; import org.opensearch.knn.index.KNNMethod; diff --git a/src/test/java/org/opensearch/knn/indices/ModelDaoTests.java b/src/test/java/org/opensearch/knn/indices/ModelDaoTests.java index 678301424..b2ff95b13 100644 --- a/src/test/java/org/opensearch/knn/indices/ModelDaoTests.java +++ b/src/test/java/org/opensearch/knn/indices/ModelDaoTests.java @@ -27,7 +27,7 @@ import org.opensearch.action.index.IndexResponse; import org.opensearch.action.support.WriteRequest; import org.opensearch.action.support.master.AcknowledgedResponse; -import org.opensearch.common.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.index.IndexNotFoundException; import org.opensearch.index.engine.VersionConflictEngineException; diff --git a/src/test/java/org/opensearch/knn/indices/ModelGraveyardTests.java b/src/test/java/org/opensearch/knn/indices/ModelGraveyardTests.java index 98dcf0cea..aeb7f6f10 100644 --- a/src/test/java/org/opensearch/knn/indices/ModelGraveyardTests.java +++ b/src/test/java/org/opensearch/knn/indices/ModelGraveyardTests.java @@ -8,8 +8,8 @@ import lombok.SneakyThrows; import org.opensearch.OpenSearchParseException; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.xcontent.ToXContent; -import org.opensearch.common.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.test.OpenSearchTestCase; diff --git a/src/test/java/org/opensearch/knn/plugin/action/RestDeleteModelHandlerIT.java b/src/test/java/org/opensearch/knn/plugin/action/RestDeleteModelHandlerIT.java index f2c31fa72..aaa64625e 100644 --- a/src/test/java/org/opensearch/knn/plugin/action/RestDeleteModelHandlerIT.java +++ b/src/test/java/org/opensearch/knn/plugin/action/RestDeleteModelHandlerIT.java @@ -15,7 +15,7 @@ import org.opensearch.client.Request; import org.opensearch.client.Response; import org.opensearch.client.ResponseException; -import org.opensearch.common.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentType; import org.opensearch.knn.KNNRestTestCase; diff --git a/src/test/java/org/opensearch/knn/plugin/action/RestKNNStatsHandlerIT.java b/src/test/java/org/opensearch/knn/plugin/action/RestKNNStatsHandlerIT.java index d89b78d24..a1756cbf1 100644 --- a/src/test/java/org/opensearch/knn/plugin/action/RestKNNStatsHandlerIT.java +++ b/src/test/java/org/opensearch/knn/plugin/action/RestKNNStatsHandlerIT.java @@ -17,7 +17,7 @@ import org.opensearch.common.Strings; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; -import org.opensearch.common.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentType; import org.opensearch.index.query.MatchAllQueryBuilder; diff --git a/src/test/java/org/opensearch/knn/plugin/action/RestSearchModelHandlerIT.java b/src/test/java/org/opensearch/knn/plugin/action/RestSearchModelHandlerIT.java index d65f434f6..609fe7f09 100644 --- a/src/test/java/org/opensearch/knn/plugin/action/RestSearchModelHandlerIT.java +++ b/src/test/java/org/opensearch/knn/plugin/action/RestSearchModelHandlerIT.java @@ -17,7 +17,7 @@ import org.opensearch.client.Response; import org.opensearch.client.ResponseException; import org.opensearch.common.settings.Settings; -import org.opensearch.common.xcontent.XContentParser; +import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; import org.opensearch.knn.KNNRestTestCase; import org.opensearch.knn.index.SpaceType; diff --git a/src/test/java/org/opensearch/knn/plugin/action/RestTrainModelHandlerIT.java b/src/test/java/org/opensearch/knn/plugin/action/RestTrainModelHandlerIT.java index 4b72e22fa..7e82c7ce5 100644 --- a/src/test/java/org/opensearch/knn/plugin/action/RestTrainModelHandlerIT.java +++ b/src/test/java/org/opensearch/knn/plugin/action/RestTrainModelHandlerIT.java @@ -13,7 +13,7 @@ import org.apache.http.util.EntityUtils; import org.opensearch.client.Response; -import org.opensearch.common.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentType; import org.opensearch.knn.KNNRestTestCase; diff --git a/src/test/java/org/opensearch/knn/plugin/script/KNNScriptScoringIT.java b/src/test/java/org/opensearch/knn/plugin/script/KNNScriptScoringIT.java index 16df35921..b2bd160f5 100644 --- a/src/test/java/org/opensearch/knn/plugin/script/KNNScriptScoringIT.java +++ b/src/test/java/org/opensearch/knn/plugin/script/KNNScriptScoringIT.java @@ -14,8 +14,8 @@ import org.opensearch.client.ResponseException; import org.opensearch.common.Strings; import org.opensearch.common.settings.Settings; -import org.opensearch.common.xcontent.ToXContent; -import org.opensearch.common.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentType; import org.opensearch.index.query.MatchAllQueryBuilder; diff --git a/src/test/java/org/opensearch/knn/plugin/script/PainlessScriptIT.java b/src/test/java/org/opensearch/knn/plugin/script/PainlessScriptIT.java index 9dca4112c..b68d8211c 100644 --- a/src/test/java/org/opensearch/knn/plugin/script/PainlessScriptIT.java +++ b/src/test/java/org/opensearch/knn/plugin/script/PainlessScriptIT.java @@ -5,7 +5,7 @@ package org.opensearch.knn.plugin.script; -import org.opensearch.common.xcontent.ToXContent; +import org.opensearch.core.xcontent.ToXContent; import org.opensearch.knn.KNNRestTestCase; import org.opensearch.knn.KNNResult; import org.opensearch.knn.common.KNNConstants; @@ -18,7 +18,7 @@ import org.opensearch.client.Response; import org.opensearch.client.ResponseException; import org.opensearch.common.Strings; -import org.opensearch.common.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.index.query.MatchAllQueryBuilder; import org.opensearch.index.query.QueryBuilder; diff --git a/src/test/java/org/opensearch/knn/plugin/transport/DeleteModelResponseTests.java b/src/test/java/org/opensearch/knn/plugin/transport/DeleteModelResponseTests.java index d25c0929a..f72cabe8a 100644 --- a/src/test/java/org/opensearch/knn/plugin/transport/DeleteModelResponseTests.java +++ b/src/test/java/org/opensearch/knn/plugin/transport/DeleteModelResponseTests.java @@ -13,7 +13,7 @@ import org.opensearch.common.Strings; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentType; import org.opensearch.knn.KNNTestCase; diff --git a/src/test/java/org/opensearch/knn/plugin/transport/GetModelResponseTests.java b/src/test/java/org/opensearch/knn/plugin/transport/GetModelResponseTests.java index 04d3b419e..ff69d4b4e 100644 --- a/src/test/java/org/opensearch/knn/plugin/transport/GetModelResponseTests.java +++ b/src/test/java/org/opensearch/knn/plugin/transport/GetModelResponseTests.java @@ -13,7 +13,7 @@ import org.opensearch.common.Strings; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentType; import org.opensearch.knn.KNNTestCase; diff --git a/src/test/java/org/opensearch/knn/plugin/transport/TrainingJobRouteDecisionInfoNodeResponseTests.java b/src/test/java/org/opensearch/knn/plugin/transport/TrainingJobRouteDecisionInfoNodeResponseTests.java index 8085b0170..75deb4639 100644 --- a/src/test/java/org/opensearch/knn/plugin/transport/TrainingJobRouteDecisionInfoNodeResponseTests.java +++ b/src/test/java/org/opensearch/knn/plugin/transport/TrainingJobRouteDecisionInfoNodeResponseTests.java @@ -17,8 +17,8 @@ import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.common.transport.TransportAddress; -import org.opensearch.common.xcontent.ToXContent; -import org.opensearch.common.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.knn.KNNTestCase; diff --git a/src/test/java/org/opensearch/knn/plugin/transport/TrainingJobRouteDecisionInfoResponseTests.java b/src/test/java/org/opensearch/knn/plugin/transport/TrainingJobRouteDecisionInfoResponseTests.java index 87d0b7ee5..372f86ff8 100644 --- a/src/test/java/org/opensearch/knn/plugin/transport/TrainingJobRouteDecisionInfoResponseTests.java +++ b/src/test/java/org/opensearch/knn/plugin/transport/TrainingJobRouteDecisionInfoResponseTests.java @@ -20,8 +20,8 @@ import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.common.transport.TransportAddress; -import org.opensearch.common.xcontent.ToXContent; -import org.opensearch.common.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.knn.KNNTestCase; diff --git a/src/test/java/org/opensearch/knn/plugin/transport/TrainingModelResponseTests.java b/src/test/java/org/opensearch/knn/plugin/transport/TrainingModelResponseTests.java index 2fef2635a..3459a11c3 100644 --- a/src/test/java/org/opensearch/knn/plugin/transport/TrainingModelResponseTests.java +++ b/src/test/java/org/opensearch/knn/plugin/transport/TrainingModelResponseTests.java @@ -13,8 +13,8 @@ import com.google.common.collect.Maps; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.xcontent.ToXContent; -import org.opensearch.common.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.knn.KNNTestCase; import org.opensearch.knn.common.KNNConstants; diff --git a/src/test/java/org/opensearch/knn/plugin/transport/TrainingModelTransportActionTests.java b/src/test/java/org/opensearch/knn/plugin/transport/TrainingModelTransportActionTests.java index 90f26aa59..6013913dd 100644 --- a/src/test/java/org/opensearch/knn/plugin/transport/TrainingModelTransportActionTests.java +++ b/src/test/java/org/opensearch/knn/plugin/transport/TrainingModelTransportActionTests.java @@ -12,7 +12,7 @@ package org.opensearch.knn.plugin.transport; import org.opensearch.action.ActionListener; -import org.opensearch.common.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.knn.KNNSingleNodeTestCase; import org.opensearch.knn.index.KNNMethodContext; diff --git a/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java b/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java index 797b038d9..6ac7e63a1 100644 --- a/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java +++ b/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java @@ -27,8 +27,8 @@ import org.opensearch.client.Response; import org.opensearch.common.Strings; import org.opensearch.common.settings.Settings; -import org.opensearch.common.xcontent.ToXContent; -import org.opensearch.common.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentType; import org.opensearch.index.query.ExistsQueryBuilder; diff --git a/src/testFixtures/java/org/opensearch/knn/ODFERestTestCase.java b/src/testFixtures/java/org/opensearch/knn/ODFERestTestCase.java index d98faa265..5f174b964 100644 --- a/src/testFixtures/java/org/opensearch/knn/ODFERestTestCase.java +++ b/src/testFixtures/java/org/opensearch/knn/ODFERestTestCase.java @@ -28,9 +28,9 @@ import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.concurrent.ThreadContext; -import org.opensearch.common.xcontent.DeprecationHandler; -import org.opensearch.common.xcontent.NamedXContentRegistry; -import org.opensearch.common.xcontent.XContentParser; +import org.opensearch.core.xcontent.DeprecationHandler; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; import org.opensearch.test.rest.OpenSearchRestTestCase; import org.junit.After; diff --git a/src/testFixtures/java/org/opensearch/knn/TestUtils.java b/src/testFixtures/java/org/opensearch/knn/TestUtils.java index 6a6e28477..f179eef36 100644 --- a/src/testFixtures/java/org/opensearch/knn/TestUtils.java +++ b/src/testFixtures/java/org/opensearch/knn/TestUtils.java @@ -6,8 +6,8 @@ package org.opensearch.knn; import com.google.common.collect.ImmutableMap; -import org.opensearch.common.xcontent.DeprecationHandler; -import org.opensearch.common.xcontent.NamedXContentRegistry; +import org.opensearch.core.xcontent.DeprecationHandler; +import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentType; import org.opensearch.knn.index.codec.util.KNNCodecUtil; From d670d9c9f6b6c5d5b649bbe71543e0deb7e09824 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Wed, 29 Mar 2023 13:03:28 -0500 Subject: [PATCH 087/416] Replace Map, List, and Set in org.opensearch.common.collect with java.util references (#816) (#825) Signed-off-by: Naveen Tatikonda (cherry picked from commit f6d3d40f5a29a4c54672e7fe7b76def71760c4de) Co-authored-by: Naveen Tatikonda --- CHANGELOG.md | 3 ++- .../java/org/opensearch/knn/index/codec/KNNCodecTestUtil.java | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4bf24601e..cc8c3285a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,4 +24,5 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), * Update BWC Version with OpenSearch Version Bump ([#813](https://github.com/opensearch-project/k-NN/pull/813)) ### Documentation ### Maintenance -### Refactoring \ No newline at end of file +### Refactoring +* Replace Map, List, and Set in org.opensearch.common.collect with java.util references ([#816](https://github.com/opensearch-project/k-NN/pull/816)) \ No newline at end of file diff --git a/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestUtil.java b/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestUtil.java index 62236a153..1e8c255f2 100644 --- a/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestUtil.java +++ b/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestUtil.java @@ -32,7 +32,7 @@ import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.StringHelper; import org.apache.lucene.util.Version; -import org.opensearch.common.collect.Set; +import java.util.Set; import org.opensearch.knn.index.query.KNNQueryResult; import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.codec.util.KNNVectorSerializer; From bc4fcedae3fb5de67bb8a0ecc6f22f20b868f93a Mon Sep 17 00:00:00 2001 From: Martin Gaievski Date: Fri, 31 Mar 2023 14:28:24 -0700 Subject: [PATCH 088/416] Bump numpy from 1.22.x to 1.24.2 (#811) (#828) Signed-off-by: Martin Gaievski (cherry picked from commit f7f12022fdecad7f8fa0dbf1f9727480b91b392c) --- CHANGELOG.md | 1 + benchmarks/osb/requirements.txt | 2 +- benchmarks/perf-tool/requirements.txt | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cc8c3285a..154dbf406 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), * Bump byte-buddy version from 1.12.22 to 1.14.2 ([#804](https://github.com/opensearch-project/k-NN/pull/804)) * Add 2.6.0 to BWC Version Matrix ([#810](https://github.com/opensearch-project/k-NN/pull/810)) * Update BWC Version with OpenSearch Version Bump ([#813](https://github.com/opensearch-project/k-NN/pull/813)) +* Bump numpy version from 1.22.x to 1.24.2 ([#811](https://github.com/opensearch-project/k-NN/pull/811)) ### Documentation ### Maintenance ### Refactoring diff --git a/benchmarks/osb/requirements.txt b/benchmarks/osb/requirements.txt index c3f29af48..2c35c8497 100644 --- a/benchmarks/osb/requirements.txt +++ b/benchmarks/osb/requirements.txt @@ -52,7 +52,7 @@ multidict==6.0.2 # via # aiohttp # yarl -numpy==1.22.3 +numpy==1.24.2 # via # -r requirements.in # h5py diff --git a/benchmarks/perf-tool/requirements.txt b/benchmarks/perf-tool/requirements.txt index c5bb0f7d3..b1e95c7d3 100644 --- a/benchmarks/perf-tool/requirements.txt +++ b/benchmarks/perf-tool/requirements.txt @@ -18,7 +18,7 @@ h5py==3.3.0 # via -r requirements.in idna==3.2 # via requests -numpy==1.22.1 +numpy==1.24.2 # via # -r requirements.in # h5py From dd8fd3b0dc54e1c45fcdbe88debf501550f3a003 Mon Sep 17 00:00:00 2001 From: Naveen Tatikonda Date: Mon, 3 Apr 2023 16:57:45 -0500 Subject: [PATCH 089/416] Fix the BWC Version Bump Issues (#833) Signed-off-by: Naveen Tatikonda --- build.gradle | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/build.gradle b/build.gradle index 8368a06ff..47d70b807 100644 --- a/build.gradle +++ b/build.gradle @@ -322,10 +322,17 @@ task updateVersion { } } ant.replaceregexp(file:'build.gradle', match: '"opensearch.version", "\\d.*"', replace: '"opensearch.version", "' + newVersion.tokenize('-')[0] + '-SNAPSHOT"', flags:'g', byline:true) - // Extract the oldBWCVersion from the existing OpenSearch Version (oldBWCVersion = major + (minor-1) + patch) + ext.os_version_without_snapshot = opensearch_version.tokenize('-')[0] - ext.oldBWCVersion = os_version_without_snapshot.tokenize('.')[0] + '.' + Integer.toString(Integer.valueOf(os_version_without_snapshot.tokenize('.')[1]) - 1) + '.' + os_version_without_snapshot.tokenize('.')[2] - // Include the current OpenSearch Version before version bump to the bwc_version matrix - ant.replaceregexp(file:".github/workflows/backwards_compatibility_tests_workflow.yml", match: oldBWCVersion, replace: oldBWCVersion + '", "' + opensearch_version.tokenize('-')[0], flags:'g', byline:true) + ext.os_version_major = os_version_without_snapshot.tokenize('.')[0] + ext.os_version_minor = os_version_without_snapshot.tokenize('.')[1] + ext.os_version_patch = os_version_without_snapshot.tokenize('.')[2] + // This condition will check if the BWC workflow is already updated or not and will run next steps if not updated + if (!fileTree(".github/workflows/backwards_compatibility_tests_workflow.yml").getSingleFile().text.contains(os_version_without_snapshot)) { + // Extract the oldBWCVersion from the existing OpenSearch Version (oldBWCVersion = major . (minor-1) . patch) + ext.oldBWCVersion = os_version_major + '.' + Integer.toString(Integer.valueOf(os_version_minor) - 1) + '.' + os_version_patch + // Include the current OpenSearch Version before version bump to the bwc_version matrix + ant.replaceregexp(file:".github/workflows/backwards_compatibility_tests_workflow.yml", match: oldBWCVersion, replace: oldBWCVersion + '", "' + opensearch_version.tokenize('-')[0], flags:'g', byline:true) + } } } From 66664f94a337cdfd92645da288343a305a240203 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Wed, 5 Apr 2023 13:34:53 -0500 Subject: [PATCH 090/416] Bump byte-buddy version to 1.14.3 (#839) (#840) Signed-off-by: Martin Gaievski (cherry picked from commit 59cc7333c74f42021da6a4aae9cbcbce1fcf628e) Co-authored-by: Martin Gaievski --- CHANGELOG.md | 1 + build.gradle | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 154dbf406..91744f50b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Enhancements ### Bug Fixes ### Infrastructure +* Bump byte-buddy version to 1.14.3 ([#839](https://github.com/opensearch-project/k-NN/pull/839)) ### Documentation ### Maintenance ### Refactoring diff --git a/build.gradle b/build.gradle index 47d70b807..f8a87db4e 100644 --- a/build.gradle +++ b/build.gradle @@ -175,9 +175,9 @@ dependencies { api group: 'com.google.guava', name: 'guava', version:'30.0-jre' api group: 'commons-lang', name: 'commons-lang', version: '2.6' testFixturesImplementation "org.opensearch.test:framework:${opensearch_version}" - testImplementation group: 'net.bytebuddy', name: 'byte-buddy', version: '1.14.2' + testImplementation group: 'net.bytebuddy', name: 'byte-buddy', version: '1.14.3' testImplementation group: 'org.objenesis', name: 'objenesis', version: '3.2' - testImplementation group: 'net.bytebuddy', name: 'byte-buddy-agent', version: '1.14.2' + testImplementation group: 'net.bytebuddy', name: 'byte-buddy-agent', version: '1.14.3' } From 7562510af9ea97bbd7dc7890c8fdca18904ca5bb Mon Sep 17 00:00:00 2001 From: Martin Gaievski Date: Wed, 5 Apr 2023 17:36:56 -0700 Subject: [PATCH 091/416] Support .opensearch-knn-model index as system index with security enabled (#827) (#842) * Add support for integ tests on secured cluster (cherry picked from commit b94b030afc74efc1fae5c5c7528edc25974a3fec) Signed-off-by: Martin Gaievski --- CHANGELOG.md | 1 + build.gradle | 1 + .../java/org/opensearch/knn/bwc/ModelIT.java | 5 - .../org/opensearch/knn/indices/ModelDao.java | 147 ++++++++++++------ .../action/RestDeleteModelHandlerIT.java | 108 ++++++++----- .../plugin/action/RestGetModelHandlerIT.java | 84 +++++----- .../plugin/action/RestKNNStatsHandlerIT.java | 22 +-- .../action/RestLegacyKNNStatsHandlerIT.java | 9 +- .../action/RestSearchModelHandlerIT.java | 101 +++++++----- src/test/resources/security/sample.pem | 28 ++++ src/test/resources/security/test-kirk.jks | Bin 0 -> 3874 bytes .../org/opensearch/knn/KNNRestTestCase.java | 89 ++++++++++- .../org/opensearch/knn/ODFERestTestCase.java | 136 ++++++++++++++-- .../java/org/opensearch/knn/TestUtils.java | 2 + 14 files changed, 536 insertions(+), 197 deletions(-) create mode 100644 src/test/resources/security/sample.pem create mode 100644 src/test/resources/security/test-kirk.jks diff --git a/CHANGELOG.md b/CHANGELOG.md index 91744f50b..6d954ace3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), * Add 2.6.0 to BWC Version Matrix ([#810](https://github.com/opensearch-project/k-NN/pull/810)) * Update BWC Version with OpenSearch Version Bump ([#813](https://github.com/opensearch-project/k-NN/pull/813)) * Bump numpy version from 1.22.x to 1.24.2 ([#811](https://github.com/opensearch-project/k-NN/pull/811)) +* Support .opensearch-knn-model index as system index with security enabled ([#827](https://github.com/opensearch-project/k-NN/pull/827)) ### Documentation ### Maintenance ### Refactoring diff --git a/build.gradle b/build.gradle index f8a87db4e..7b7c54809 100644 --- a/build.gradle +++ b/build.gradle @@ -178,6 +178,7 @@ dependencies { testImplementation group: 'net.bytebuddy', name: 'byte-buddy', version: '1.14.3' testImplementation group: 'org.objenesis', name: 'objenesis', version: '3.2' testImplementation group: 'net.bytebuddy', name: 'byte-buddy-agent', version: '1.14.3' + api "org.opensearch:common-utils:${version}" } diff --git a/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/ModelIT.java b/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/ModelIT.java index f8e832c50..e052f6dcf 100644 --- a/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/ModelIT.java +++ b/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/ModelIT.java @@ -182,11 +182,6 @@ public static void wipeAllModels() throws IOException { deleteKNNModel(TEST_MODEL_ID); deleteKNNModel(TEST_MODEL_ID_DEFAULT); deleteKNNModel(TEST_MODEL_ID_TRAINING); - - Request request = new Request("DELETE", "/" + MODEL_INDEX_NAME); - - Response response = client().performRequest(request); - assertEquals(request.getEndpoint() + ": failed", RestStatus.OK, RestStatus.fromCode(response.getStatusLine().getStatusCode())); } } diff --git a/src/main/java/org/opensearch/knn/indices/ModelDao.java b/src/main/java/org/opensearch/knn/indices/ModelDao.java index 0d5d75d30..cf0dd1890 100644 --- a/src/main/java/org/opensearch/knn/indices/ModelDao.java +++ b/src/main/java/org/opensearch/knn/indices/ModelDao.java @@ -13,6 +13,7 @@ import com.google.common.base.Charsets; import com.google.common.io.Resources; +import lombok.SneakyThrows; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.opensearch.OpenSearchException; @@ -42,6 +43,7 @@ import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.concurrent.ThreadContext; import org.opensearch.index.IndexNotFoundException; import org.opensearch.knn.common.KNNConstants; import org.opensearch.knn.plugin.transport.DeleteModelResponse; @@ -49,10 +51,10 @@ import org.opensearch.knn.plugin.transport.RemoveModelFromCacheAction; import org.opensearch.knn.plugin.transport.RemoveModelFromCacheRequest; import org.opensearch.knn.plugin.transport.RemoveModelFromCacheResponse; -import org.opensearch.knn.plugin.transport.UpdateModelMetadataAction; -import org.opensearch.knn.plugin.transport.UpdateModelMetadataRequest; import org.opensearch.knn.plugin.transport.UpdateModelGraveyardAction; import org.opensearch.knn.plugin.transport.UpdateModelGraveyardRequest; +import org.opensearch.knn.plugin.transport.UpdateModelMetadataAction; +import org.opensearch.knn.plugin.transport.UpdateModelMetadataRequest; import java.io.IOException; import java.net.URL; @@ -62,6 +64,7 @@ import java.util.Objects; import java.util.Optional; import java.util.concurrent.ExecutionException; +import java.util.function.Supplier; import static java.util.Objects.isNull; import static org.opensearch.knn.common.KNNConstants.MODEL_INDEX_MAPPING_PATH; @@ -216,14 +219,21 @@ public void create(ActionListener actionListener) throws IO if (isCreated()) { return; } - CreateIndexRequest request = new CreateIndexRequest(MODEL_INDEX_NAME).mapping(getMapping()) - .settings( - Settings.builder() - .put("index.hidden", true) - .put("index.number_of_shards", this.numberOfShards) - .put("index.number_of_replicas", this.numberOfReplicas) - ); - client.admin().indices().create(request, actionListener); + runWithStashedThreadContext(() -> { + CreateIndexRequest request; + try { + request = new CreateIndexRequest(MODEL_INDEX_NAME).mapping(getMapping()) + .settings( + Settings.builder() + .put("index.hidden", true) + .put("index.number_of_shards", this.numberOfShards) + .put("index.number_of_replicas", this.numberOfReplicas) + ); + } catch (IOException e) { + throw new RuntimeException(e); + } + client.admin().indices().create(request, actionListener); + }); } @Override @@ -293,8 +303,9 @@ private void putInternal(Model model, ActionListener listener, Do parameters.put(KNNConstants.MODEL_BLOB_PARAMETER, base64Model); } - IndexRequestBuilder indexRequestBuilder = client.prepareIndex(MODEL_INDEX_NAME); - + final IndexRequestBuilder indexRequestBuilder = ModelDao.runWithStashedThreadContext( + () -> client.prepareIndex(MODEL_INDEX_NAME) + ); indexRequestBuilder.setId(model.getModelID()); indexRequestBuilder.setSource(parameters); @@ -304,8 +315,8 @@ private void putInternal(Model model, ActionListener listener, Do // After metadata update finishes, remove item from every node's cache if necessary. If no model id is // passed then nothing needs to be removed from the cache ActionListener onMetaListener; - onMetaListener = ActionListener.wrap( - indexResponse -> client.execute( + onMetaListener = ActionListener.wrap(indexResponse -> { + client.execute( RemoveModelFromCacheAction.INSTANCE, new RemoveModelFromCacheRequest(model.getModelID()), ActionListener.wrap(removeModelFromCacheResponse -> { @@ -318,9 +329,8 @@ private void putInternal(Model model, ActionListener listener, Do listener.onFailure(new RuntimeException(failureMessage)); }, listener::onFailure) - ), - listener::onFailure - ); + ); + }, listener::onFailure); // After the model is indexed, update metadata only if the model is in CREATED state ActionListener onIndexListener; @@ -357,16 +367,30 @@ private ActionListener getUpdateModelMetadataListener( ); } + @SneakyThrows @Override - public Model get(String modelId) throws ExecutionException, InterruptedException { + public Model get(String modelId) { /* GET //?_local */ - GetRequestBuilder getRequestBuilder = new GetRequestBuilder(client, GetAction.INSTANCE, MODEL_INDEX_NAME).setId(modelId) - .setPreference("_local"); - GetResponse getResponse = getRequestBuilder.execute().get(); - Map responseMap = getResponse.getSourceAsMap(); - return Model.getModelFromSourceMap(responseMap); + try { + return ModelDao.runWithStashedThreadContext(() -> { + GetRequestBuilder getRequestBuilder = new GetRequestBuilder(client, GetAction.INSTANCE, MODEL_INDEX_NAME).setId(modelId) + .setPreference("_local"); + GetResponse getResponse; + try { + getResponse = getRequestBuilder.execute().get(); + } catch (InterruptedException | ExecutionException e) { + throw new RuntimeException(e); + } + Map responseMap = getResponse.getSourceAsMap(); + return Model.getModelFromSourceMap(responseMap); + }); + } catch (RuntimeException runtimeException) { + // we need to use RuntimeException as container for real exception to keep signature + // of runWithStashedThreadContext generic + throw runtimeException.getCause(); + } } /** @@ -380,20 +404,22 @@ public void get(String modelId, ActionListener actionListener) /* GET //?_local */ - GetRequestBuilder getRequestBuilder = new GetRequestBuilder(client, GetAction.INSTANCE, MODEL_INDEX_NAME).setId(modelId) - .setPreference("_local"); - - getRequestBuilder.execute(ActionListener.wrap(response -> { - if (response.isSourceEmpty()) { - String errorMessage = String.format("Model \" %s \" does not exist", modelId); - actionListener.onFailure(new ResourceNotFoundException(modelId, errorMessage)); - return; - } - final Map responseMap = response.getSourceAsMap(); - Model model = Model.getModelFromSourceMap(responseMap); - actionListener.onResponse(new GetModelResponse(model)); + ModelDao.runWithStashedThreadContext(() -> { + GetRequestBuilder getRequestBuilder = new GetRequestBuilder(client, GetAction.INSTANCE, MODEL_INDEX_NAME).setId(modelId) + .setPreference("_local"); + + getRequestBuilder.execute(ActionListener.wrap(response -> { + if (response.isSourceEmpty()) { + String errorMessage = String.format("Model \" %s \" does not exist", modelId); + actionListener.onFailure(new ResourceNotFoundException(modelId, errorMessage)); + return; + } + final Map responseMap = response.getSourceAsMap(); + Model model = Model.getModelFromSourceMap(responseMap); + actionListener.onResponse(new GetModelResponse(model)); - }, actionListener::onFailure)); + }, actionListener::onFailure)); + }); } /** @@ -404,8 +430,10 @@ public void get(String modelId, ActionListener actionListener) */ @Override public void search(SearchRequest request, ActionListener actionListener) { - request.indices(MODEL_INDEX_NAME); - client.search(request, actionListener); + ModelDao.runWithStashedThreadContext(() -> { + request.indices(MODEL_INDEX_NAME); + client.search(request, actionListener); + }); } @Override @@ -505,16 +533,17 @@ public void delete(String modelId, ActionListener listener) ); // Setup delete model request - DeleteRequestBuilder deleteRequestBuilder = new DeleteRequestBuilder(client, DeleteAction.INSTANCE, MODEL_INDEX_NAME); - deleteRequestBuilder.setId(modelId); - deleteRequestBuilder.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); - - // On model metadata removal, delete the model from the index - clearModelMetadataStep.whenComplete( - acknowledgedResponse -> deleteModelFromIndex(modelId, deleteModelFromIndexStep, deleteRequestBuilder), - listener::onFailure - ); - + ModelDao.runWithStashedThreadContext(() -> { + DeleteRequestBuilder deleteRequestBuilder = new DeleteRequestBuilder(client, DeleteAction.INSTANCE, MODEL_INDEX_NAME); + deleteRequestBuilder.setId(modelId); + deleteRequestBuilder.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + + // On model metadata removal, delete the model from the index + clearModelMetadataStep.whenComplete( + acknowledgedResponse -> deleteModelFromIndex(modelId, deleteModelFromIndexStep, deleteRequestBuilder), + listener::onFailure + ); + }); deleteModelFromIndexStep.whenComplete(deleteResponse -> { // If model is not deleted, remove modelId from model graveyard and return with error message if (deleteResponse.getResult() != DocWriteResponse.Result.DELETED) { @@ -653,4 +682,26 @@ private String buildRemoveModelErrorMessage(String modelId, RemoveModelFromCache return stringBuilder.toString(); } } + + /** + * Set the thread context to default, this is needed to allow actions on model system index + * when security plugin is enabled + * @param function runnable that needs to be executed after thread context has been stashed, accepts and returns nothing + */ + private static void runWithStashedThreadContext(Runnable function) { + try (ThreadContext.StoredContext context = OpenSearchKNNModelDao.client.threadPool().getThreadContext().stashContext()) { + function.run(); + } + } + + /** + * Set the thread context to default, this is needed to allow actions on model system index + * when security plugin is enabled + * @param function supplier function that needs to be executed after thread context has been stashed, return object + */ + private static T runWithStashedThreadContext(Supplier function) { + try (ThreadContext.StoredContext context = OpenSearchKNNModelDao.client.threadPool().getThreadContext().stashContext()) { + return function.get(); + } + } } diff --git a/src/test/java/org/opensearch/knn/plugin/action/RestDeleteModelHandlerIT.java b/src/test/java/org/opensearch/knn/plugin/action/RestDeleteModelHandlerIT.java index aaa64625e..12d45d8a3 100644 --- a/src/test/java/org/opensearch/knn/plugin/action/RestDeleteModelHandlerIT.java +++ b/src/test/java/org/opensearch/knn/plugin/action/RestDeleteModelHandlerIT.java @@ -19,15 +19,11 @@ import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentType; import org.opensearch.knn.KNNRestTestCase; -import org.opensearch.knn.index.SpaceType; -import org.opensearch.knn.index.util.KNNEngine; -import org.opensearch.knn.indices.ModelMetadata; -import org.opensearch.knn.indices.ModelState; import org.opensearch.knn.plugin.KNNPlugin; import org.opensearch.knn.plugin.transport.DeleteModelResponse; import org.opensearch.rest.RestStatus; -import java.io.IOException; +import java.util.List; import java.util.Map; import static org.opensearch.knn.common.KNNConstants.ENCODER_PARAMETER_PQ_CODE_SIZE; @@ -48,59 +44,92 @@ public class RestDeleteModelHandlerIT extends KNNRestTestCase { - private ModelMetadata getModelMetadata() { - return new ModelMetadata(KNNEngine.DEFAULT, SpaceType.DEFAULT, 4, ModelState.CREATED, "2021-03-27", "test model", ""); - } - - public void testDeleteModelExists() throws IOException { + public void testDeleteModelExists() throws Exception { createModelSystemIndex(); - String testModelID = "test-model-id"; - byte[] testModelBlob = "hello".getBytes(); - ModelMetadata testModelMetadata = getModelMetadata(); - addModelToSystemIndex(testModelID, testModelMetadata, testModelBlob); - assertEquals(getDocCount(MODEL_INDEX_NAME), 1); + String modelId = "test-model-id"; + String trainingIndexName = "train-index"; + String trainingFieldName = "train-field"; + int dimension = 8; + String modelDescription = "dummy description"; - String restURI = String.join("/", KNNPlugin.KNN_BASE_URI, MODELS, testModelID); - Request request = new Request("DELETE", restURI); + createBasicKnnIndex(trainingIndexName, trainingFieldName, dimension); + ingestDataAndTrainModel(modelId, trainingIndexName, trainingFieldName, dimension, modelDescription); + assertTrainingSucceeds(modelId, NUM_OF_ATTEMPTS, DELAY_MILLI_SEC); - Response response = client().performRequest(request); - assertEquals(request.getEndpoint() + ": failed", RestStatus.OK, RestStatus.fromCode(response.getStatusLine().getStatusCode())); + Response getModelResponse = getModel(modelId, List.of()); + assertEquals(RestStatus.OK, RestStatus.fromCode(getModelResponse.getStatusLine().getStatusCode())); - assertEquals(0, getDocCount(MODEL_INDEX_NAME)); + String responseBody = EntityUtils.toString(getModelResponse.getEntity()); + assertNotNull(responseBody); + + Map responseMap = createParser(XContentType.JSON.xContent(), responseBody).map(); + + assertEquals(modelId, responseMap.get(MODEL_ID)); + + String deleteModelRestURI = String.join("/", KNNPlugin.KNN_BASE_URI, MODELS, modelId); + Request deleteModelRequest = new Request("DELETE", deleteModelRestURI); + + Response deleteModelResponse = client().performRequest(deleteModelRequest); + assertEquals( + deleteModelRequest.getEndpoint() + ": failed", + RestStatus.OK, + RestStatus.fromCode(deleteModelResponse.getStatusLine().getStatusCode()) + ); + + ResponseException ex = expectThrows(ResponseException.class, () -> getModel(modelId, List.of())); + assertTrue(ex.getMessage().contains(modelId)); } - public void testDeleteTrainingModel() throws IOException { + public void testDeleteTrainingModel() throws Exception { createModelSystemIndex(); - String testModelID = "test-model-id"; - byte[] testModelBlob = "hello".getBytes(); - ModelMetadata testModelMetadata = getModelMetadata(); - testModelMetadata.setState(ModelState.TRAINING); - - addModelToSystemIndex(testModelID, testModelMetadata, testModelBlob); - assertEquals(1, getDocCount(MODEL_INDEX_NAME)); - String restURI = String.join("/", KNNPlugin.KNN_BASE_URI, MODELS, testModelID); - Request request = new Request("DELETE", restURI); + String modelId = "test-model-id"; + String trainingIndexName = "train-index"; + String trainingFieldName = "train-field"; + int dimension = 8; + String modelDescription = "dummy description"; - Response response = client().performRequest(request); - assertEquals(request.getEndpoint() + ": failed", RestStatus.OK, RestStatus.fromCode(response.getStatusLine().getStatusCode())); + createBasicKnnIndex(trainingIndexName, trainingFieldName, dimension); + // we do not wait for training to be completed + ingestDataAndTrainModel(modelId, trainingIndexName, trainingFieldName, dimension, modelDescription); - assertEquals(1, getDocCount(MODEL_INDEX_NAME)); + Response getModelResponse = getModel(modelId, List.of()); + assertEquals(RestStatus.OK, RestStatus.fromCode(getModelResponse.getStatusLine().getStatusCode())); - String responseBody = EntityUtils.toString(response.getEntity()); + String responseBody = EntityUtils.toString(getModelResponse.getEntity()); assertNotNull(responseBody); Map responseMap = createParser(XContentType.JSON.xContent(), responseBody).map(); - assertEquals(testModelID, responseMap.get(MODEL_ID)); + assertEquals(modelId, responseMap.get(MODEL_ID)); + + String deleteModelRestURI = String.join("/", KNNPlugin.KNN_BASE_URI, MODELS, modelId); + Request deleteModelRequest = new Request("DELETE", deleteModelRestURI); + + Response deleteModelResponse = client().performRequest(deleteModelRequest); + assertEquals( + deleteModelRequest.getEndpoint() + ": failed", + RestStatus.OK, + RestStatus.fromCode(deleteModelResponse.getStatusLine().getStatusCode()) + ); + + responseBody = EntityUtils.toString(deleteModelResponse.getEntity()); + assertNotNull(responseBody); + + responseMap = createParser(XContentType.JSON.xContent(), responseBody).map(); + + assertEquals(modelId, responseMap.get(MODEL_ID)); assertEquals("failed", responseMap.get(DeleteModelResponse.RESULT)); - String errorMessage = String.format("Cannot delete model \"%s\". Model is still in training", testModelID); + String errorMessage = String.format("Cannot delete model \"%s\". Model is still in training", modelId); assertEquals(errorMessage, responseMap.get(DeleteModelResponse.ERROR_MSG)); + + // need to wait for training operation as it's required for after test cleanup + assertTrainingSucceeds(modelId, NUM_OF_ATTEMPTS, DELAY_MILLI_SEC); } - public void testDeleteModelFailsInvalid() throws IOException { + public void testDeleteModelFailsInvalid() throws Exception { String modelId = "invalid-model-id"; createModelSystemIndex(); String restURI = String.join("/", KNNPlugin.KNN_BASE_URI, MODELS, modelId); @@ -111,7 +140,7 @@ public void testDeleteModelFailsInvalid() throws IOException { } // Test Train Model -> Delete Model -> Train Model with same modelId - public void testTrainingDeletedModel() throws IOException, InterruptedException { + public void testTrainingDeletedModel() throws Exception { String modelId = "test-model-id1"; String trainingIndexName1 = "train-index-1"; String trainingIndexName2 = "train-index-2"; @@ -134,8 +163,7 @@ public void testTrainingDeletedModel() throws IOException, InterruptedException trainModel(modelId, trainingIndexName2, trainingFieldName, dimension); } - private void trainModel(String modelId, String trainingIndexName, String trainingFieldName, int dimension) throws IOException, - InterruptedException { + private void trainModel(String modelId, String trainingIndexName, String trainingFieldName, int dimension) throws Exception { // Create a training index and randomly ingest data into it createBasicKnnIndex(trainingIndexName, trainingFieldName, dimension); diff --git a/src/test/java/org/opensearch/knn/plugin/action/RestGetModelHandlerIT.java b/src/test/java/org/opensearch/knn/plugin/action/RestGetModelHandlerIT.java index b6853e8bb..092ca31e3 100644 --- a/src/test/java/org/opensearch/knn/plugin/action/RestGetModelHandlerIT.java +++ b/src/test/java/org/opensearch/knn/plugin/action/RestGetModelHandlerIT.java @@ -18,10 +18,6 @@ import org.opensearch.client.ResponseException; import org.opensearch.common.xcontent.XContentType; import org.opensearch.knn.KNNRestTestCase; -import org.opensearch.knn.index.SpaceType; -import org.opensearch.knn.index.util.KNNEngine; -import org.opensearch.knn.indices.ModelMetadata; -import org.opensearch.knn.indices.ModelState; import org.opensearch.knn.plugin.KNNPlugin; import org.opensearch.rest.RestStatus; @@ -39,6 +35,8 @@ import static org.opensearch.knn.common.KNNConstants.MODEL_ID; import static org.opensearch.knn.common.KNNConstants.MODEL_STATE; import static org.opensearch.knn.common.KNNConstants.MODEL_TIMESTAMP; +import static org.opensearch.knn.index.SpaceType.L2; +import static org.opensearch.knn.index.util.KNNEngine.FAISS; /** * Integration tests to check the correctness of {@link org.opensearch.knn.plugin.rest.RestGetModelHandler} @@ -46,19 +44,28 @@ public class RestGetModelHandlerIT extends KNNRestTestCase { - private ModelMetadata getModelMetadata() { - return new ModelMetadata(KNNEngine.DEFAULT, SpaceType.DEFAULT, 4, ModelState.CREATED, "2021-03-27", "test model", ""); - } - - public void testGetModelExists() throws IOException { + public void testGetModelExists() throws Exception { createModelSystemIndex(); - String testModelID = "test-model-id"; - byte[] testModelBlob = "hello".getBytes(); - ModelMetadata testModelMetadata = getModelMetadata(); - - addModelToSystemIndex(testModelID, testModelMetadata, testModelBlob); - String restURI = String.join("/", KNNPlugin.KNN_BASE_URI, MODELS, testModelID); + String modelId = "test-model-id"; + String trainingIndexName = "train-index"; + String trainingFieldName = "train-field"; + int dimension = 8; + String modelDescription = "dummy description"; + + createBasicKnnIndex(trainingIndexName, trainingFieldName, dimension); + + ingestDataAndTrainModel( + modelId, + trainingIndexName, + trainingFieldName, + dimension, + modelDescription, + xContentBuilderToMap(getModelMethodBuilder()) + ); + assertTrainingSucceeds(modelId, NUM_OF_ATTEMPTS, DELAY_MILLI_SEC); + + String restURI = String.join("/", KNNPlugin.KNN_BASE_URI, MODELS, modelId); Request request = new Request("GET", restURI); Response response = client().performRequest(request); @@ -68,30 +75,30 @@ public void testGetModelExists() throws IOException { assertNotNull(responseBody); Map responseMap = createParser(XContentType.JSON.xContent(), responseBody).map(); - - assertEquals(testModelID, responseMap.get(MODEL_ID)); - assertEquals(testModelMetadata.getDescription(), responseMap.get(MODEL_DESCRIPTION)); - assertEquals(testModelMetadata.getDimension(), responseMap.get(DIMENSION)); - assertEquals(testModelMetadata.getError(), responseMap.get(MODEL_ERROR)); - assertEquals(testModelMetadata.getKnnEngine().getName(), responseMap.get(KNN_ENGINE)); - assertEquals(testModelMetadata.getSpaceType().getValue(), responseMap.get(METHOD_PARAMETER_SPACE_TYPE)); - assertEquals(testModelMetadata.getState().getName(), responseMap.get(MODEL_STATE)); - assertEquals(testModelMetadata.getTimestamp(), responseMap.get(MODEL_TIMESTAMP)); + assertEquals(modelId, responseMap.get(MODEL_ID)); + assertEquals(modelDescription, responseMap.get(MODEL_DESCRIPTION)); + assertEquals(FAISS.getName(), responseMap.get(KNN_ENGINE)); + assertEquals(L2.getValue(), responseMap.get(METHOD_PARAMETER_SPACE_TYPE)); } - public void testGetModelExistsWithFilter() throws IOException { + public void testGetModelExistsWithFilter() throws Exception { createModelSystemIndex(); - String testModelID = "test-model-id"; - byte[] testModelBlob = "hello".getBytes(); - ModelMetadata testModelMetadata = getModelMetadata(); - - addModelToSystemIndex(testModelID, testModelMetadata, testModelBlob); - - String restURI = String.join("/", KNNPlugin.KNN_BASE_URI, MODELS, testModelID); + String modelId = "test-model-id"; + String trainingIndexName = "train-index"; + String trainingFieldName = "train-field"; + int dimension = 8; + String modelDescription = "dummy description"; + + createBasicKnnIndex(trainingIndexName, trainingFieldName, dimension); + Map method = xContentBuilderToMap(getModelMethodBuilder()); + ingestDataAndTrainModel(modelId, trainingIndexName, trainingFieldName, dimension, modelDescription, method); + assertTrainingSucceeds(modelId, NUM_OF_ATTEMPTS, DELAY_MILLI_SEC); + + String restURI = String.join("/", KNNPlugin.KNN_BASE_URI, MODELS, modelId); Request request = new Request("GET", restURI); - List filterdPath = Arrays.asList(MODEL_ID, MODEL_DESCRIPTION, MODEL_TIMESTAMP, KNN_ENGINE); - request.addParameter("filter_path", Strings.join(filterdPath, ",")); + List filteredPath = Arrays.asList(MODEL_ID, MODEL_DESCRIPTION, MODEL_TIMESTAMP, KNN_ENGINE); + request.addParameter("filter_path", Strings.join(filteredPath, ",")); Response response = client().performRequest(request); assertEquals(RestStatus.OK, RestStatus.fromCode(response.getStatusLine().getStatusCode())); @@ -101,11 +108,10 @@ public void testGetModelExistsWithFilter() throws IOException { Map responseMap = createParser(XContentType.JSON.xContent(), responseBody).map(); - assertTrue(responseMap.size() == filterdPath.size()); - assertEquals(testModelID, responseMap.get(MODEL_ID)); - assertEquals(testModelMetadata.getDescription(), responseMap.get(MODEL_DESCRIPTION)); - assertEquals(testModelMetadata.getTimestamp(), responseMap.get(MODEL_TIMESTAMP)); - assertEquals(testModelMetadata.getKnnEngine().getName(), responseMap.get(KNN_ENGINE)); + assertTrue(responseMap.size() == filteredPath.size()); + assertEquals(modelId, responseMap.get(MODEL_ID)); + assertEquals(modelDescription, responseMap.get(MODEL_DESCRIPTION)); + assertEquals(FAISS.getName(), responseMap.get(KNN_ENGINE)); assertFalse(responseMap.containsKey(DIMENSION)); assertFalse(responseMap.containsKey(MODEL_ERROR)); assertFalse(responseMap.containsKey(METHOD_PARAMETER_SPACE_TYPE)); diff --git a/src/test/java/org/opensearch/knn/plugin/action/RestKNNStatsHandlerIT.java b/src/test/java/org/opensearch/knn/plugin/action/RestKNNStatsHandlerIT.java index a1756cbf1..6ec699d87 100644 --- a/src/test/java/org/opensearch/knn/plugin/action/RestKNNStatsHandlerIT.java +++ b/src/test/java/org/opensearch/knn/plugin/action/RestKNNStatsHandlerIT.java @@ -48,6 +48,7 @@ import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_NLIST; import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_SPACE_TYPE; import static org.opensearch.knn.common.KNNConstants.MODEL_ID; +import static org.opensearch.knn.common.KNNConstants.MODEL_INDEX_NAME; import static org.opensearch.knn.common.KNNConstants.NAME; import static org.opensearch.knn.common.KNNConstants.NMSLIB_NAME; import static org.opensearch.knn.common.KNNConstants.PARAMETERS; @@ -341,20 +342,23 @@ public void testScriptStats_multipleShards() throws Exception { public void testModelIndexHealthMetricsStats() throws IOException { // Create request that filters only model index String modelIndexStatusName = StatNames.MODEL_INDEX_STATUS.getName(); + // index can be created in one of previous tests, and as we do not delete it each test the check below became optional + if (!systemIndexExists(MODEL_INDEX_NAME)) { - Response response = getKnnStats(Collections.emptyList(), Arrays.asList(modelIndexStatusName)); - String responseBody = EntityUtils.toString(response.getEntity()); - Map statsMap = createParser(XContentType.JSON.xContent(), responseBody).map(); + final Response response = getKnnStats(Collections.emptyList(), Arrays.asList(modelIndexStatusName)); + final String responseBody = EntityUtils.toString(response.getEntity()); + final Map statsMap = createParser(XContentType.JSON.xContent(), responseBody).map(); - // Check that model health status is null since model index is not created to system yet - assertNull(statsMap.get(StatNames.MODEL_INDEX_STATUS.getName())); + // Check that model health status is null since model index is not created to system yet + assertNull(statsMap.get(StatNames.MODEL_INDEX_STATUS.getName())); - createModelSystemIndex(); + createModelSystemIndex(); + } - response = getKnnStats(Collections.emptyList(), Arrays.asList(modelIndexStatusName)); + Response response = getKnnStats(Collections.emptyList(), Arrays.asList(modelIndexStatusName)); - responseBody = EntityUtils.toString(response.getEntity()); - statsMap = createParser(XContentType.JSON.xContent(), responseBody).map(); + final String responseBody = EntityUtils.toString(response.getEntity()); + final Map statsMap = createParser(XContentType.JSON.xContent(), responseBody).map(); // Check that model health status is not null assertNotNull(statsMap.get(modelIndexStatusName)); diff --git a/src/test/java/org/opensearch/knn/plugin/action/RestLegacyKNNStatsHandlerIT.java b/src/test/java/org/opensearch/knn/plugin/action/RestLegacyKNNStatsHandlerIT.java index a4243537d..0d900cfbe 100644 --- a/src/test/java/org/opensearch/knn/plugin/action/RestLegacyKNNStatsHandlerIT.java +++ b/src/test/java/org/opensearch/knn/plugin/action/RestLegacyKNNStatsHandlerIT.java @@ -319,10 +319,15 @@ public void testScriptStats_multipleShards() throws Exception { // Useful settings when debugging to prevent timeouts @Override protected Settings restClientSettings() { + final Settings.Builder builder = Settings.builder(); if (isDebuggingTest || isDebuggingRemoteCluster) { - return Settings.builder().put(CLIENT_SOCKET_TIMEOUT, TimeValue.timeValueMinutes(10)).build(); + builder.put(CLIENT_SOCKET_TIMEOUT, TimeValue.timeValueMinutes(10)); } else { - return super.restClientSettings(); + if (System.getProperty("tests.rest.client_path_prefix") != null) { + builder.put(CLIENT_PATH_PREFIX, System.getProperty("tests.rest.client_path_prefix")); + } } + builder.put("strictDeprecationMode", false); + return builder.build(); } } diff --git a/src/test/java/org/opensearch/knn/plugin/action/RestSearchModelHandlerIT.java b/src/test/java/org/opensearch/knn/plugin/action/RestSearchModelHandlerIT.java index 609fe7f09..92834217e 100644 --- a/src/test/java/org/opensearch/knn/plugin/action/RestSearchModelHandlerIT.java +++ b/src/test/java/org/opensearch/knn/plugin/action/RestSearchModelHandlerIT.java @@ -16,7 +16,6 @@ import org.opensearch.client.Request; import org.opensearch.client.Response; import org.opensearch.client.ResponseException; -import org.opensearch.common.settings.Settings; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; import org.opensearch.knn.KNNRestTestCase; @@ -39,6 +38,8 @@ import static org.opensearch.knn.common.KNNConstants.PARAM_SIZE; import static org.opensearch.knn.common.KNNConstants.SEARCH_MODEL_MAX_SIZE; import static org.opensearch.knn.common.KNNConstants.SEARCH_MODEL_MIN_SIZE; +import static org.opensearch.knn.index.SpaceType.L2; +import static org.opensearch.knn.index.util.KNNEngine.FAISS; /** * Integration tests to check the correctness of {@link org.opensearch.knn.plugin.rest.RestSearchModelHandler} @@ -96,15 +97,25 @@ public void testSizeValidationFailsInvalidSize() throws IOException { } - public void testSearchModelExists() throws IOException { + public void testSearchModelExists() throws Exception { createModelSystemIndex(); - createIndex("irrelevant-index", Settings.EMPTY); - addDocWithBinaryField("irrelevant-index", "id1", "field-name", "value"); + String trainingIndex = "irrelevant-index"; + String trainingFieldName = "train-field"; + int dimension = 8; + String modelDescription = "dummy description"; + createBasicKnnIndex(trainingIndex, trainingFieldName, dimension); + List testModelID = Arrays.asList("test-modelid1", "test-modelid2"); - byte[] testModelBlob = "hello".getBytes(); - ModelMetadata testModelMetadata = getModelMetadata(); - for (String modelID : testModelID) { - addModelToSystemIndex(modelID, testModelMetadata, testModelBlob); + for (String modelId : testModelID) { + ingestDataAndTrainModel( + modelId, + trainingIndex, + trainingFieldName, + dimension, + modelDescription, + xContentBuilderToMap(getModelMethodBuilder()) + ); + assertTrainingSucceeds(modelId, NUM_OF_ATTEMPTS, DELAY_MILLI_SEC); } String restURI = String.join("/", KNNPlugin.KNN_BASE_URI, MODELS, "_search"); @@ -128,21 +139,25 @@ public void testSearchModelExists() throws IOException { for (SearchHit hit : searchResponse.getHits().getHits()) { assertTrue(testModelID.contains(hit.getId())); Model model = Model.getModelFromSourceMap(hit.getSourceAsMap()); - assertEquals(getModelMetadata(), model.getModelMetadata()); - assertArrayEquals(testModelBlob, model.getModelBlob()); + assertEquals(modelDescription, model.getModelMetadata().getDescription()); + assertEquals(FAISS, model.getModelMetadata().getKnnEngine()); + assertEquals(L2, model.getModelMetadata().getSpaceType()); } } } - public void testSearchModelWithoutSource() throws IOException { + public void testSearchModelWithoutSource() throws Exception { createModelSystemIndex(); - createIndex("irrelevant-index", Settings.EMPTY); - addDocWithBinaryField("irrelevant-index", "id1", "field-name", "value"); - List testModelID = Arrays.asList("test-modelid1", "test-modelid2"); - byte[] testModelBlob = "hello".getBytes(); - ModelMetadata testModelMetadata = getModelMetadata(); - for (String modelID : testModelID) { - addModelToSystemIndex(modelID, testModelMetadata, testModelBlob); + String trainingIndex = "irrelevant-index"; + String trainingFieldName = "train-field"; + int dimension = 8; + createBasicKnnIndex(trainingIndex, trainingFieldName, dimension); + + List testModelIds = Arrays.asList("test-modelid1", "test-modelid2"); + for (String modelId : testModelIds) { + String modelDescription = "dummy description"; + ingestDataAndTrainModel(modelId, trainingIndex, trainingFieldName, dimension, modelDescription); + assertTrainingSucceeds(modelId, NUM_OF_ATTEMPTS, DELAY_MILLI_SEC); } String restURI = String.join("/", KNNPlugin.KNN_BASE_URI, MODELS, "_search"); @@ -163,24 +178,27 @@ public void testSearchModelWithoutSource() throws IOException { assertNotNull(searchResponse); // returns only model from ModelIndex - assertEquals(searchResponse.getHits().getHits().length, testModelID.size()); + assertEquals(searchResponse.getHits().getHits().length, testModelIds.size()); for (SearchHit hit : searchResponse.getHits().getHits()) { - assertTrue(testModelID.contains(hit.getId())); + assertTrue(testModelIds.contains(hit.getId())); assertNull(hit.getSourceAsMap()); } } } - public void testSearchModelWithSourceFilteringIncludes() throws IOException { + public void testSearchModelWithSourceFilteringIncludes() throws Exception { createModelSystemIndex(); - createIndex("irrelevant-index", Settings.EMPTY); - addDocWithBinaryField("irrelevant-index", "id1", "field-name", "value"); - List testModelID = Arrays.asList("test-modelid1", "test-modelid2"); - byte[] testModelBlob = "hello".getBytes(); - ModelMetadata testModelMetadata = getModelMetadata(); - for (String modelID : testModelID) { - addModelToSystemIndex(modelID, testModelMetadata, testModelBlob); + String trainingIndex = "irrelevant-index"; + String trainingFieldName = "train-field"; + int dimension = 8; + createBasicKnnIndex(trainingIndex, trainingFieldName, dimension); + + List testModelIds = Arrays.asList("test-modelid1", "test-modelid2"); + for (String modelId : testModelIds) { + String modelDescription = "dummy description"; + ingestDataAndTrainModel(modelId, trainingIndex, trainingFieldName, dimension, modelDescription); + assertTrainingSucceeds(modelId, NUM_OF_ATTEMPTS, DELAY_MILLI_SEC); } String restURI = String.join("/", KNNPlugin.KNN_BASE_URI, MODELS, "_search"); @@ -208,10 +226,10 @@ public void testSearchModelWithSourceFilteringIncludes() throws IOException { assertNotNull(searchResponse); // returns only model from ModelIndex - assertEquals(searchResponse.getHits().getHits().length, testModelID.size()); + assertEquals(searchResponse.getHits().getHits().length, testModelIds.size()); for (SearchHit hit : searchResponse.getHits().getHits()) { - assertTrue(testModelID.contains(hit.getId())); + assertTrue(testModelIds.contains(hit.getId())); Map sourceAsMap = hit.getSourceAsMap(); assertFalse(sourceAsMap.containsKey("model_blob")); assertTrue(sourceAsMap.containsKey("state")); @@ -221,15 +239,18 @@ public void testSearchModelWithSourceFilteringIncludes() throws IOException { } } - public void testSearchModelWithSourceFilteringExcludes() throws IOException { + public void testSearchModelWithSourceFilteringExcludes() throws Exception { createModelSystemIndex(); - createIndex("irrelevant-index", Settings.EMPTY); - addDocWithBinaryField("irrelevant-index", "id1", "field-name", "value"); - List testModelID = Arrays.asList("test-modelid1", "test-modelid2"); - byte[] testModelBlob = "hello".getBytes(); - ModelMetadata testModelMetadata = getModelMetadata(); - for (String modelID : testModelID) { - addModelToSystemIndex(modelID, testModelMetadata, testModelBlob); + String trainingIndex = "irrelevant-index"; + String trainingFieldName = "train-field"; + int dimension = 8; + createBasicKnnIndex(trainingIndex, trainingFieldName, dimension); + + List testModelIds = Arrays.asList("test-modelid1", "test-modelid2"); + for (String modelId : testModelIds) { + String modelDescription = "dummy description"; + ingestDataAndTrainModel(modelId, trainingIndex, trainingFieldName, dimension, modelDescription); + assertTrainingSucceeds(modelId, NUM_OF_ATTEMPTS, DELAY_MILLI_SEC); } String restURI = String.join("/", KNNPlugin.KNN_BASE_URI, MODELS, "_search"); @@ -257,10 +278,10 @@ public void testSearchModelWithSourceFilteringExcludes() throws IOException { assertNotNull(searchResponse); // returns only model from ModelIndex - assertEquals(searchResponse.getHits().getHits().length, testModelID.size()); + assertEquals(searchResponse.getHits().getHits().length, testModelIds.size()); for (SearchHit hit : searchResponse.getHits().getHits()) { - assertTrue(testModelID.contains(hit.getId())); + assertTrue(testModelIds.contains(hit.getId())); Map sourceAsMap = hit.getSourceAsMap(); assertFalse(sourceAsMap.containsKey("model_blob")); assertTrue(sourceAsMap.containsKey("state")); diff --git a/src/test/resources/security/sample.pem b/src/test/resources/security/sample.pem new file mode 100644 index 000000000..fa785ca10 --- /dev/null +++ b/src/test/resources/security/sample.pem @@ -0,0 +1,28 @@ +-----BEGIN CERTIFICATE----- +MIIEyTCCA7GgAwIBAgIGAWLrc1O2MA0GCSqGSIb3DQEBCwUAMIGPMRMwEQYKCZIm +iZPyLGQBGRYDY29tMRcwFQYKCZImiZPyLGQBGRYHZXhhbXBsZTEZMBcGA1UECgwQ +RXhhbXBsZSBDb20gSW5jLjEhMB8GA1UECwwYRXhhbXBsZSBDb20gSW5jLiBSb290 +IENBMSEwHwYDVQQDDBhFeGFtcGxlIENvbSBJbmMuIFJvb3QgQ0EwHhcNMTgwNDIy +MDM0MzQ3WhcNMjgwNDE5MDM0MzQ3WjBeMRIwEAYKCZImiZPyLGQBGRYCZGUxDTAL +BgNVBAcMBHRlc3QxDTALBgNVBAoMBG5vZGUxDTALBgNVBAsMBG5vZGUxGzAZBgNV +BAMMEm5vZGUtMC5leGFtcGxlLmNvbTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCC +AQoCggEBAJa+f476vLB+AwK53biYByUwN+40D8jMIovGXm6wgT8+9Sbs899dDXgt +9CE1Beo65oP1+JUz4c7UHMrCY3ePiDt4cidHVzEQ2g0YoVrQWv0RedS/yx/DKhs8 +Pw1O715oftP53p/2ijD5DifFv1eKfkhFH+lwny/vMSNxellpl6NxJTiJVnQ9HYOL +gf2t971ITJHnAuuxUF48HcuNovW4rhtkXef8kaAN7cE3LU+A9T474ULNCKkEFPIl +ZAKN3iJNFdVsxrTU+CUBHzk73Do1cCkEvJZ0ZFjp0Z3y8wLY/gqWGfGVyA9l2CUq +eIZNf55PNPtGzOrvvONiui48vBKH1LsCAwEAAaOCAVkwggFVMIG8BgNVHSMEgbQw +gbGAFJI1DOAPHitF9k0583tfouYSl0BzoYGVpIGSMIGPMRMwEQYKCZImiZPyLGQB +GRYDY29tMRcwFQYKCZImiZPyLGQBGRYHZXhhbXBsZTEZMBcGA1UECgwQRXhhbXBs +ZSBDb20gSW5jLjEhMB8GA1UECwwYRXhhbXBsZSBDb20gSW5jLiBSb290IENBMSEw +HwYDVQQDDBhFeGFtcGxlIENvbSBJbmMuIFJvb3QgQ0GCAQEwHQYDVR0OBBYEFKyv +78ZmFjVKM9g7pMConYH7FVBHMAwGA1UdEwEB/wQCMAAwDgYDVR0PAQH/BAQDAgXg +MCAGA1UdJQEB/wQWMBQGCCsGAQUFBwMBBggrBgEFBQcDAjA1BgNVHREELjAsiAUq +AwQFBYISbm9kZS0wLmV4YW1wbGUuY29tgglsb2NhbGhvc3SHBH8AAAEwDQYJKoZI +hvcNAQELBQADggEBAIOKuyXsFfGv1hI/Lkpd/73QNqjqJdxQclX57GOMWNbOM5H0 +5/9AOIZ5JQsWULNKN77aHjLRr4owq2jGbpc/Z6kAd+eiatkcpnbtbGrhKpOtoEZy +8KuslwkeixpzLDNISSbkeLpXz4xJI1ETMN/VG8ZZP1bjzlHziHHDu0JNZ6TnNzKr +XzCGMCohFfem8vnKNnKUneMQMvXd3rzUaAgvtf7Hc2LTBlf4fZzZF1EkwdSXhaMA +1lkfHiqOBxtgeDLxCHESZ2fqgVqsWX+t3qHQfivcPW6txtDyrFPRdJOGhiMGzT/t +e/9kkAtQRgpTb3skYdIOOUOV0WGQ60kJlFhAzIs= +-----END CERTIFICATE----- \ No newline at end of file diff --git a/src/test/resources/security/test-kirk.jks b/src/test/resources/security/test-kirk.jks new file mode 100644 index 0000000000000000000000000000000000000000..174dbda656f41b10341adb78ab91a46afaae8a1c GIT binary patch literal 3874 zcmY+GcQhM}zs8e@RFtSn>{Y9_XzfvZl*TSQG6F9FNd(wu zFab99cRY+FKt2CO5E21u`*&mo0s{VisDB9%$qk|Z?*;}S1PKGv1*1XCugHHEKp;B6 z69Saqd|bch;ZdXcj@o48Or^T{VjiQWQ)um?koax&EW2Jd6%cmO+99&?<0M#TkhMY0 z>TOc9NNj$5o%GwnI2>ZpA<-syd;YVlrkqVstJxqe_w8#F0dlKW!#D3WVDWfwaN@uX z{)l!>hgv`=r)M_tPedAH8wS zrMCsCM3^vbf3iWkdUoK)O(h9`bxp3s^zq4CU5%IJN;Y04OLiLfXPS%;Duo}L?EKtE z$4DyO?uRf+Ovm@OBmMKYjcI;;3k(jA`wJ`_W&){Es6Nv(A-s;NYZhfPTZJ%tBZ{1@ zc|_(P(o|Du6c{sJ4@Q6w- zF)*aVb&dDqmGoH8(8Y;T2S?DR9+P|nUT>q8177|so}DjY7IWc!jB(9r?rJ%YyVvh5 z4`BJLeFX6F2g1N^WT?dWin3^|1>$*MQP~CSqFMgQ4m&bJp``1>I(!5Pe9&NB7{wXc z+p)Bs6Durb104tWmIOYRkBU~Waz;l#k`+@Fye00vbTIQq3dY*R{KBH-UF3%r{=+v` zqu(DD1~xv;*N0vqhN9l+bCm(5u37KF+&JF&or0qB&J%}ZmdviHekDmr#GlPK60J4Q zJ#vSZYt1pSxEPM~S27`bL-X}ig&?t1ubwy1&P?lEwQUs|t?a7>dqM7^&@^5tSL9pMp+&5H?jk>BGMj!JcQ+3*rxFcY4MY2z z4C?1*^xq&(g`+u7JnXS-Yuq8?$%DG-Zs#VDo=cTmcJRfEFTG1T4~(u1j$Snc+7Cs; zyB9?mE4rqbq_*xqj?#OlN%@YGt*PgH+-~Fy+blur5jn zu_S?>vGKl_57zp6>#CW5Q&HHKl|qVToNrM`8!zz5n*{CQ+r2#n4{2tk@;0m{ zM8pbY25rVQv1<0iw2CPT?uG+>NVZVLalVoRSZQdC(&M@`0$mC@6l?zxF&LAM8XHR1Ah3S zb?4&7@N$w<+PVC^0ws=h2pqrozQ!=b!?Zy2@uQjFh1)BEPT$JlDa9Q8(%YHT_r)w# z<4bW`j)gX^ktonho#Uf=U=ZH5QT!;ug%qe!Fi?N(OjphEVY3YTU5B*j^ZMOg+XmnL zPpT%`zoHjGCw~=w|5zC`KWOFwsF`=Jjwez^hwA2rgTt^ z^10Gp<3*%@mI37QZ>P3$*PX4;4LpFQqK9AnvMxAg!|B)unEQ{13w`0LO;;mgV22L5 z=Y8bwo8Fch2UFgZEqeTdMGZMKmz)4Uzb#-R)&H4zUC45?<4&g?`6XX-=`F2|(~Esf z4P+-+Y;J{*hV8L55?o`K^wL+ zE>e|WH7ZW48)vi%Zq4nbkLikeTd&2pCr5A#jJC9jypS>*@uF<#i}Xp$3X7~b0>bXQ zd@CV7FY-$A{IR_m5uZie z+ckdOpNC4bjck=wZ@3lTl5+`W3~_4oPuGx4#mk-f?CsbGulgu|BAb)LTI|hBYM==Q zPLdu6@x)I_O{qq^{%cI*Q`-C+WZjpp^GjGiWv(#7Vr(pZ@A532u&Rn|3@4+xgKqNc zMhtgDOn)7lv}KZc^U}jD!KU{3;=7as(>uBwDx5}ii8iIz!F(WDlbe(V`WH5PS-XhZ zPJFI;eV}4{aJ?&?Sv%?zMZJ9SRFL%?ZZ0C(FdozY2R@i=1>&&E< z<(hauSRE!6;QE6ujbYrYrWNm9;!ixJV`}*=J$7wZ^0l>rTb7|)`olK^*^m3Ex%nq2 zL({r^1)T=Q7qM>-F~1lC817t!PNhq1c&?{#kiAuiMtlDELuI?Ut6LMQ6()675@U5L z_g(P7&7MR-N3z!C5a+qZ$!xmrg0qbsQn*7vqc!v-^yqc6`tlc%aQl-Fe+IYP5Pe^K z^%zx2w*a+^&+F*;<~HZ&=XwRTB6z)Uec2XkH=^cl)cHs|VxGqSQStks&td*NQbTPW z@??ewN#dRVCH?t{p-$)JDIxkVF$#9Q?iS!Qqby9p zttQuw3k2_4Hs9`5TG}3Jwk97Nste6#I!jG)f$b(~xI#)Bs7nQ7es#6RzYPh=8vCY$@K;aE z0JYYxSm&6)?GS&eI-ibs8vhi$EXK)Yhv7%bHy2C$czjfz?F4J+b%lJkXj+1&h?Ti_R;#D>}h%qh-ltN3^kJE=J$q9lGN z97&*c`aeQNBG8(G3ADz4#|D3&4&?Ix=oLK>L?VDUkp%Gi|FbTdf2<2!*X4kUenR(; zb%6=se)ca%eZ zOyn3`1eb66NoONNlb!Qgq|BuMxwULjnW>4u2iuhj(ZUV8fC!eY=nsZF*}w6V0(LxJ zVJ|ew^cV0%UizR_Y1yOEtM1}iw*f#fPAX(#E)%*G)QD7W7O$XT5e!*pv0krMED!yw zv)_h?54B@8<=GZ6ukEmkmrx<@jaUud2Y%EQU-vBcCChZ&9Xf`1Rw3w4G=@{y>I<<5 zr)BfiiXe`(Z@ksE4@BqB5d!$>pA(N&9b7XX5GBfr?j{H(J6=OSr*~9Ff8Zh0^d;HS3|V9O<+-Py zxI&YAI-gM^t2+X1O6JyQ*^8SfuZ5{?m1F14fGg;0aeF|P)4c8tw{C;?*J)`bjV2~qOsSjk^$@gQ1{3jw}OGfYhan!3#Y zHIQX-5|4fmT69zTvDd3aW(AkQqj4t}?Md}bd>>Q>N!29V@klLOr#L%^gPrlgw8ASS>!fstf*6i;ka?xLu@MUq>?r_mf*HCZ0jHy2N^B`x>Y90Tt5-jn7*G)Ai~?r^6!i zChFK}Z-Np|s#K(ct1NYcNSoxM%p~ng6bf7}uXm#_v&(wHHp4Tljgd6EW$Kg0xZkkr zi&o;({o`MC#=#JXFx-Py14vyFMbGypX`-a>1F9n21b`MXKk|zU$zEO&>l1Rjkx$4Vg-UeUetqM3xCVt2 z#4}QY$t__sQxkuq9U8E_JbjM8#9JvlSK48A@`?q^I*~JnT-!@f$l49YlT>fpGqYJ9 zr+k*tw-oT8l~Dr<$GT8lt$6D+{n7Af1%CX7h0*}>N)s;I);DZqq{57a method = xContentBuilderToMap(builder); + ingestDataAndTrainModel(modelId, trainingIndexName, trainingFieldName, dimension, modelDescription, method); + } + + protected void ingestDataAndTrainModel( + String modelId, + String trainingIndexName, + String trainingFieldName, + int dimension, + String modelDescription, + Map method + ) throws Exception { + int trainingDataCount = 40; + bulkIngestRandomVectors(trainingIndexName, trainingFieldName, trainingDataCount, dimension); + + Response trainResponse = trainModel(modelId, trainingIndexName, trainingFieldName, dimension, method, modelDescription); + + assertEquals(RestStatus.OK, RestStatus.fromCode(trainResponse.getStatusLine().getStatusCode())); + } + + protected XContentBuilder getModelMethodBuilder() throws IOException { + XContentBuilder modelMethodBuilder = XContentFactory.jsonBuilder() + .startObject() + .field(NAME, "ivf") + .field(KNN_ENGINE, FAISS.getName()) + .field(METHOD_PARAMETER_SPACE_TYPE, L2.getValue()) + .startObject(PARAMETERS) + .field(METHOD_PARAMETER_NLIST, 1) + .startObject(METHOD_ENCODER_PARAMETER) + .field(NAME, "pq") + .startObject(PARAMETERS) + .field(ENCODER_PARAMETER_PQ_CODE_SIZE, 2) + .field(ENCODER_PARAMETER_PQ_M, 2) + .endObject() + .endObject() + .endObject() + .endObject(); + return modelMethodBuilder; + } + /** * We need to be able to dump the jacoco coverage before cluster is shut down. * The new internal testing framework removed some of the gradle tasks we were listening to diff --git a/src/testFixtures/java/org/opensearch/knn/ODFERestTestCase.java b/src/testFixtures/java/org/opensearch/knn/ODFERestTestCase.java index 5f174b964..097fe014d 100644 --- a/src/testFixtures/java/org/opensearch/knn/ODFERestTestCase.java +++ b/src/testFixtures/java/org/opensearch/knn/ODFERestTestCase.java @@ -5,13 +5,6 @@ package org.opensearch.knn; -import java.io.IOException; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.stream.Collectors; - import org.apache.http.Header; import org.apache.http.HttpHost; import org.apache.http.auth.AuthScope; @@ -21,23 +14,54 @@ import org.apache.http.impl.client.BasicCredentialsProvider; import org.apache.http.message.BasicHeader; import org.apache.http.ssl.SSLContextBuilder; +import org.apache.http.util.EntityUtils; +import org.junit.After; +import org.opensearch.action.search.SearchResponse; import org.opensearch.client.Request; import org.opensearch.client.Response; import org.opensearch.client.RestClient; import org.opensearch.client.RestClientBuilder; +import org.opensearch.common.Strings; +import org.opensearch.common.io.PathUtils; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.concurrent.ThreadContext; +import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.DeprecationHandler; import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; +import org.opensearch.commons.rest.SecureRestClientBuilder; +import org.opensearch.knn.plugin.KNNPlugin; +import org.opensearch.rest.RestStatus; +import org.opensearch.search.SearchHit; import org.opensearch.test.rest.OpenSearchRestTestCase; -import org.junit.After; +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.nio.file.Path; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; + +import static org.opensearch.commons.ConfigConstants.OPENSEARCH_SECURITY_SSL_HTTP_ENABLED; +import static org.opensearch.commons.ConfigConstants.OPENSEARCH_SECURITY_SSL_HTTP_KEYSTORE_FILEPATH; +import static org.opensearch.commons.ConfigConstants.OPENSEARCH_SECURITY_SSL_HTTP_KEYSTORE_KEYPASSWORD; +import static org.opensearch.commons.ConfigConstants.OPENSEARCH_SECURITY_SSL_HTTP_KEYSTORE_PASSWORD; +import static org.opensearch.commons.ConfigConstants.OPENSEARCH_SECURITY_SSL_HTTP_PEMCERT_FILEPATH; import static org.opensearch.knn.TestUtils.KNN_BWC_PREFIX; import static org.opensearch.knn.TestUtils.OPENDISTRO_SECURITY; +import static org.opensearch.knn.TestUtils.OPENSEARCH_SYSTEM_INDEX_PREFIX; +import static org.opensearch.knn.TestUtils.SECURITY_AUDITLOG_PREFIX; import static org.opensearch.knn.TestUtils.SKIP_DELETE_MODEL_INDEX; +import static org.opensearch.knn.common.KNNConstants.MODELS; import static org.opensearch.knn.common.KNNConstants.MODEL_INDEX_NAME; /** @@ -45,6 +69,8 @@ */ public abstract class ODFERestTestCase extends OpenSearchRestTestCase { + private final Set IMMUTABLE_INDEX_PREFIXES = Set.of(KNN_BWC_PREFIX, SECURITY_AUDITLOG_PREFIX, OPENSEARCH_SYSTEM_INDEX_PREFIX); + protected boolean isHttps() { boolean isHttps = Optional.ofNullable(System.getProperty("https")).map("true"::equalsIgnoreCase).orElse(false); if (isHttps) { @@ -66,7 +92,22 @@ protected String getProtocol() { protected RestClient buildClient(Settings settings, HttpHost[] hosts) throws IOException { RestClientBuilder builder = RestClient.builder(hosts); if (isHttps()) { - configureHttpsClient(builder, settings); + String keystore = settings.get(OPENSEARCH_SECURITY_SSL_HTTP_KEYSTORE_FILEPATH); + if (Objects.nonNull(keystore)) { + URI uri; + try { + uri = this.getClass().getClassLoader().getResource("security/sample.pem").toURI(); + } catch (URISyntaxException e) { + throw new RuntimeException(e); + } + Path configPath = PathUtils.get(uri).getParent().toAbsolutePath(); + return new SecureRestClientBuilder(settings, configPath).build(); + } else { + configureHttpsClient(builder, settings); + boolean strictDeprecationMode = settings.getAsBoolean("strictDeprecationMode", true); + builder.setStrictDeprecationMode(strictDeprecationMode); + return builder.build(); + } } else { configureClient(builder, settings); } @@ -120,8 +161,8 @@ protected boolean preserveIndicesUponCompletion() { @SuppressWarnings("unchecked") @After - protected void wipeAllODFEIndices() throws IOException { - Response response = client().performRequest(new Request("GET", "/_cat/indices?format=json&expand_wildcards=all")); + protected void wipeAllODFEIndices() throws Exception { + Response response = adminClient().performRequest(new Request("GET", "/_cat/indices?format=json&expand_wildcards=all")); XContentType xContentType = XContentType.fromMediaType(response.getEntity().getContentType().getValue()); try ( XContentParser parser = xContentType.xContent() @@ -140,7 +181,11 @@ protected void wipeAllODFEIndices() throws IOException { } for (Map index : parserList) { - String indexName = (String) index.get("index"); + final String indexName = (String) index.get("index"); + if (isIndexCleanupRequired(indexName)) { + wipeIndexContent(indexName); + continue; + } if (!skipDeleteIndex(indexName)) { adminClient().performRequest(new Request("DELETE", "/" + indexName)); } @@ -148,6 +193,57 @@ protected void wipeAllODFEIndices() throws IOException { } } + private boolean isIndexCleanupRequired(final String index) { + return MODEL_INDEX_NAME.equals(index) && !getSkipDeleteModelIndexFlag(); + } + + private void wipeIndexContent(String indexName) throws IOException { + deleteModels(getModelIds()); + deleteAllDocs(indexName); + } + + private List getModelIds() throws IOException { + final String restURIGetModels = String.join("/", KNNPlugin.KNN_BASE_URI, MODELS, "_search"); + final Response response = adminClient().performRequest(new Request("GET", restURIGetModels)); + + assertEquals(RestStatus.OK, RestStatus.fromCode(response.getStatusLine().getStatusCode())); + + final String responseBody = EntityUtils.toString(response.getEntity()); + assertNotNull(responseBody); + + final XContentParser parser = createParser(XContentType.JSON.xContent(), responseBody); + final SearchResponse searchResponse = SearchResponse.fromXContent(parser); + + return Arrays.stream(searchResponse.getHits().getHits()).map(SearchHit::getId).collect(Collectors.toList()); + } + + private void deleteModels(final List modelIds) throws IOException { + for (final String testModelID : modelIds) { + final String restURIGetModel = String.join("/", KNNPlugin.KNN_BASE_URI, MODELS, testModelID); + final Response getModelResponse = adminClient().performRequest(new Request("GET", restURIGetModel)); + if (RestStatus.OK != RestStatus.fromCode(getModelResponse.getStatusLine().getStatusCode())) { + continue; + } + final String restURIDeleteModel = String.join("/", KNNPlugin.KNN_BASE_URI, MODELS, testModelID); + adminClient().performRequest(new Request("DELETE", restURIDeleteModel)); + } + } + + private void deleteAllDocs(final String indexName) throws IOException { + final String restURIDeleteByQuery = String.join("/", indexName, "_delete_by_query"); + final Request request = new Request("POST", restURIDeleteByQuery); + final XContentBuilder matchAllDocsQuery = XContentFactory.jsonBuilder() + .startObject() + .startObject("query") + .startObject("match_all") + .endObject() + .endObject() + .endObject(); + + request.setJsonEntity(Strings.toString(matchAllDocsQuery)); + adminClient().performRequest(request); + } + private boolean getSkipDeleteModelIndexFlag() { return Boolean.parseBoolean(System.getProperty(SKIP_DELETE_MODEL_INDEX, "false")); } @@ -159,11 +255,25 @@ private boolean skipDeleteModelIndex(String indexName) { private boolean skipDeleteIndex(String indexName) { if (indexName != null && !OPENDISTRO_SECURITY.equals(indexName) - && !indexName.startsWith(KNN_BWC_PREFIX) + && IMMUTABLE_INDEX_PREFIXES.stream().noneMatch(indexName::startsWith) && !skipDeleteModelIndex(indexName)) { return false; } return true; } + + @Override + protected Settings restAdminSettings() { + return Settings.builder() + // disable the warning exception for admin client since it's only used for cleanup. + .put("strictDeprecationMode", false) + .put("http.port", 9200) + .put(OPENSEARCH_SECURITY_SSL_HTTP_ENABLED, isHttps()) + .put(OPENSEARCH_SECURITY_SSL_HTTP_PEMCERT_FILEPATH, "sample.pem") + .put(OPENSEARCH_SECURITY_SSL_HTTP_KEYSTORE_FILEPATH, "test-kirk.jks") + .put(OPENSEARCH_SECURITY_SSL_HTTP_KEYSTORE_PASSWORD, "changeit") + .put(OPENSEARCH_SECURITY_SSL_HTTP_KEYSTORE_KEYPASSWORD, "changeit") + .build(); + } } diff --git a/src/testFixtures/java/org/opensearch/knn/TestUtils.java b/src/testFixtures/java/org/opensearch/knn/TestUtils.java index f179eef36..0843176e7 100644 --- a/src/testFixtures/java/org/opensearch/knn/TestUtils.java +++ b/src/testFixtures/java/org/opensearch/knn/TestUtils.java @@ -95,6 +95,8 @@ public class TestUtils { public static final String ROLLING_UPGRADE_FIRST_ROUND = "tests.rest.first_round"; public static final String SKIP_DELETE_MODEL_INDEX = "tests.skip_delete_model_index"; public static final String UPGRADED_CLUSTER = "upgraded_cluster"; + public static final String SECURITY_AUDITLOG_PREFIX = "security-auditlog"; + public static final String OPENSEARCH_SYSTEM_INDEX_PREFIX = ".opensearch"; // Generating vectors using random function with a seed which makes these vectors standard and generate same vectors for each run. public static float[][] randomlyGenerateStandardVectors(int numVectors, int dimensions, int seed) { From fb8a356f101c714b06c8ef92ac8ca70ebc71fe81 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Thu, 6 Apr 2023 12:52:06 -0700 Subject: [PATCH 092/416] Set gradle dependency scope for common-utils to testFixturesImplementation (#844) (#845) Signed-off-by: Martin Gaievski (cherry picked from commit 427cd326bf027067e75d4507f5019be41069646a) Co-authored-by: Martin Gaievski --- CHANGELOG.md | 1 + build.gradle | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6d954ace3..3eae413d0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), * Update BWC Version with OpenSearch Version Bump ([#813](https://github.com/opensearch-project/k-NN/pull/813)) * Bump numpy version from 1.22.x to 1.24.2 ([#811](https://github.com/opensearch-project/k-NN/pull/811)) * Support .opensearch-knn-model index as system index with security enabled ([#827](https://github.com/opensearch-project/k-NN/pull/827)) +* Set gradle dependency scope for common-utils to testFixturesImplementation ([#844](https://github.com/opensearch-project/k-NN/pull/844)) ### Documentation ### Maintenance ### Refactoring diff --git a/build.gradle b/build.gradle index 7b7c54809..229c2d11c 100644 --- a/build.gradle +++ b/build.gradle @@ -178,7 +178,7 @@ dependencies { testImplementation group: 'net.bytebuddy', name: 'byte-buddy', version: '1.14.3' testImplementation group: 'org.objenesis', name: 'objenesis', version: '3.2' testImplementation group: 'net.bytebuddy', name: 'byte-buddy-agent', version: '1.14.3' - api "org.opensearch:common-utils:${version}" + testFixturesImplementation "org.opensearch:common-utils:${version}" } From e17d64e5e3440bc4cb115c7c8fea8883b46518a2 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Fri, 7 Apr 2023 12:30:31 -0700 Subject: [PATCH 093/416] Add support of .opensearch-knn-model as system index to transport actions (#847) (#848) * Add thread context stashing to transport actions Signed-off-by: Martin Gaievski (cherry picked from commit 1b6fd481a6b4bbbdea0c93f652654a2ff2f7758d) Co-authored-by: Martin Gaievski --- CHANGELOG.md | 1 + .../knn/common/ThreadContextHelper.java | 39 +++++ .../org/opensearch/knn/indices/ModelDao.java | 134 +++++++----------- .../transport/DeleteModelTransportAction.java | 12 +- .../transport/GetModelTransportAction.java | 15 +- .../transport/SearchModelTransportAction.java | 19 ++- .../TrainingJobRouterTransportAction.java | 11 +- .../TrainingModelTransportAction.java | 39 +++-- .../knn/common/ThreadContextHelperTests.java | 52 +++++++ 9 files changed, 210 insertions(+), 112 deletions(-) create mode 100644 src/main/java/org/opensearch/knn/common/ThreadContextHelper.java create mode 100644 src/test/java/org/opensearch/knn/common/ThreadContextHelperTests.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 3eae413d0..ca4f5be93 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), * Bump numpy version from 1.22.x to 1.24.2 ([#811](https://github.com/opensearch-project/k-NN/pull/811)) * Support .opensearch-knn-model index as system index with security enabled ([#827](https://github.com/opensearch-project/k-NN/pull/827)) * Set gradle dependency scope for common-utils to testFixturesImplementation ([#844](https://github.com/opensearch-project/k-NN/pull/844)) +* Add support of .opensearch-knn-model as system index to transport actions ([#847](https://github.com/opensearch-project/k-NN/pull/847)) ### Documentation ### Maintenance ### Refactoring diff --git a/src/main/java/org/opensearch/knn/common/ThreadContextHelper.java b/src/main/java/org/opensearch/knn/common/ThreadContextHelper.java new file mode 100644 index 000000000..e0c5ad8f9 --- /dev/null +++ b/src/main/java/org/opensearch/knn/common/ThreadContextHelper.java @@ -0,0 +1,39 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.common; + +import org.opensearch.client.Client; +import org.opensearch.common.util.concurrent.ThreadContext; + +import java.util.function.Supplier; + +/** + * Class abstracts execution of runnable or function in specific context + */ +public class ThreadContextHelper { + + /** + * Sets the thread context to default and execute function, this needed to allow actions on model system index + * when security plugin is enabled + * @param function runnable that needs to be executed after thread context has been stashed, accepts and returns nothing + */ + public static void runWithStashedThreadContext(Client client, Runnable function) { + try (ThreadContext.StoredContext context = client.threadPool().getThreadContext().stashContext()) { + function.run(); + } + } + + /** + * Sets the thread context to default and execute function, this needed to allow actions on model system index + * when security plugin is enabled + * @param function supplier function that needs to be executed after thread context has been stashed, return object + */ + public static T runWithStashedThreadContext(Client client, Supplier function) { + try (ThreadContext.StoredContext context = client.threadPool().getThreadContext().stashContext()) { + return function.get(); + } + } +} diff --git a/src/main/java/org/opensearch/knn/indices/ModelDao.java b/src/main/java/org/opensearch/knn/indices/ModelDao.java index cf0dd1890..a5b478213 100644 --- a/src/main/java/org/opensearch/knn/indices/ModelDao.java +++ b/src/main/java/org/opensearch/knn/indices/ModelDao.java @@ -13,7 +13,6 @@ import com.google.common.base.Charsets; import com.google.common.io.Resources; -import lombok.SneakyThrows; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.opensearch.OpenSearchException; @@ -43,18 +42,18 @@ import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.settings.Settings; -import org.opensearch.common.util.concurrent.ThreadContext; import org.opensearch.index.IndexNotFoundException; import org.opensearch.knn.common.KNNConstants; +import org.opensearch.knn.common.ThreadContextHelper; import org.opensearch.knn.plugin.transport.DeleteModelResponse; import org.opensearch.knn.plugin.transport.GetModelResponse; import org.opensearch.knn.plugin.transport.RemoveModelFromCacheAction; import org.opensearch.knn.plugin.transport.RemoveModelFromCacheRequest; import org.opensearch.knn.plugin.transport.RemoveModelFromCacheResponse; -import org.opensearch.knn.plugin.transport.UpdateModelGraveyardAction; -import org.opensearch.knn.plugin.transport.UpdateModelGraveyardRequest; import org.opensearch.knn.plugin.transport.UpdateModelMetadataAction; import org.opensearch.knn.plugin.transport.UpdateModelMetadataRequest; +import org.opensearch.knn.plugin.transport.UpdateModelGraveyardAction; +import org.opensearch.knn.plugin.transport.UpdateModelGraveyardRequest; import java.io.IOException; import java.net.URL; @@ -64,7 +63,6 @@ import java.util.Objects; import java.util.Optional; import java.util.concurrent.ExecutionException; -import java.util.function.Supplier; import static java.util.Objects.isNull; import static org.opensearch.knn.common.KNNConstants.MODEL_INDEX_MAPPING_PATH; @@ -219,21 +217,14 @@ public void create(ActionListener actionListener) throws IO if (isCreated()) { return; } - runWithStashedThreadContext(() -> { - CreateIndexRequest request; - try { - request = new CreateIndexRequest(MODEL_INDEX_NAME).mapping(getMapping()) - .settings( - Settings.builder() - .put("index.hidden", true) - .put("index.number_of_shards", this.numberOfShards) - .put("index.number_of_replicas", this.numberOfReplicas) - ); - } catch (IOException e) { - throw new RuntimeException(e); - } - client.admin().indices().create(request, actionListener); - }); + CreateIndexRequest request = new CreateIndexRequest(MODEL_INDEX_NAME).mapping(getMapping()) + .settings( + Settings.builder() + .put("index.hidden", true) + .put("index.number_of_shards", this.numberOfShards) + .put("index.number_of_replicas", this.numberOfReplicas) + ); + client.admin().indices().create(request, actionListener); } @Override @@ -303,9 +294,8 @@ private void putInternal(Model model, ActionListener listener, Do parameters.put(KNNConstants.MODEL_BLOB_PARAMETER, base64Model); } - final IndexRequestBuilder indexRequestBuilder = ModelDao.runWithStashedThreadContext( - () -> client.prepareIndex(MODEL_INDEX_NAME) - ); + IndexRequestBuilder indexRequestBuilder = client.prepareIndex(MODEL_INDEX_NAME); + indexRequestBuilder.setId(model.getModelID()); indexRequestBuilder.setSource(parameters); @@ -315,8 +305,8 @@ private void putInternal(Model model, ActionListener listener, Do // After metadata update finishes, remove item from every node's cache if necessary. If no model id is // passed then nothing needs to be removed from the cache ActionListener onMetaListener; - onMetaListener = ActionListener.wrap(indexResponse -> { - client.execute( + onMetaListener = ActionListener.wrap( + indexResponse -> client.execute( RemoveModelFromCacheAction.INSTANCE, new RemoveModelFromCacheRequest(model.getModelID()), ActionListener.wrap(removeModelFromCacheResponse -> { @@ -329,8 +319,9 @@ private void putInternal(Model model, ActionListener listener, Do listener.onFailure(new RuntimeException(failureMessage)); }, listener::onFailure) - ); - }, listener::onFailure); + ), + listener::onFailure + ); // After the model is indexed, update metadata only if the model is in CREATED state ActionListener onIndexListener; @@ -367,14 +358,13 @@ private ActionListener getUpdateModelMetadataListener( ); } - @SneakyThrows @Override - public Model get(String modelId) { + public Model get(String modelId) throws ExecutionException, InterruptedException { /* GET //?_local */ try { - return ModelDao.runWithStashedThreadContext(() -> { + return ThreadContextHelper.runWithStashedThreadContext(client, () -> { GetRequestBuilder getRequestBuilder = new GetRequestBuilder(client, GetAction.INSTANCE, MODEL_INDEX_NAME).setId(modelId) .setPreference("_local"); GetResponse getResponse; @@ -389,7 +379,16 @@ public Model get(String modelId) { } catch (RuntimeException runtimeException) { // we need to use RuntimeException as container for real exception to keep signature // of runWithStashedThreadContext generic - throw runtimeException.getCause(); + Throwable throwable = runtimeException.getCause(); + if (throwable != null) { + if (throwable instanceof InterruptedException) { + throw (InterruptedException) throwable; + } + if (throwable instanceof ExecutionException) { + throw (ExecutionException) throwable; + } + } + throw runtimeException; } } @@ -404,22 +403,20 @@ public void get(String modelId, ActionListener actionListener) /* GET //?_local */ - ModelDao.runWithStashedThreadContext(() -> { - GetRequestBuilder getRequestBuilder = new GetRequestBuilder(client, GetAction.INSTANCE, MODEL_INDEX_NAME).setId(modelId) - .setPreference("_local"); - - getRequestBuilder.execute(ActionListener.wrap(response -> { - if (response.isSourceEmpty()) { - String errorMessage = String.format("Model \" %s \" does not exist", modelId); - actionListener.onFailure(new ResourceNotFoundException(modelId, errorMessage)); - return; - } - final Map responseMap = response.getSourceAsMap(); - Model model = Model.getModelFromSourceMap(responseMap); - actionListener.onResponse(new GetModelResponse(model)); + GetRequestBuilder getRequestBuilder = new GetRequestBuilder(client, GetAction.INSTANCE, MODEL_INDEX_NAME).setId(modelId) + .setPreference("_local"); - }, actionListener::onFailure)); - }); + getRequestBuilder.execute(ActionListener.wrap(response -> { + if (response.isSourceEmpty()) { + String errorMessage = String.format("Model \" %s \" does not exist", modelId); + actionListener.onFailure(new ResourceNotFoundException(modelId, errorMessage)); + return; + } + final Map responseMap = response.getSourceAsMap(); + Model model = Model.getModelFromSourceMap(responseMap); + actionListener.onResponse(new GetModelResponse(model)); + + }, actionListener::onFailure)); } /** @@ -430,7 +427,7 @@ public void get(String modelId, ActionListener actionListener) */ @Override public void search(SearchRequest request, ActionListener actionListener) { - ModelDao.runWithStashedThreadContext(() -> { + ThreadContextHelper.runWithStashedThreadContext(client, () -> { request.indices(MODEL_INDEX_NAME); client.search(request, actionListener); }); @@ -533,17 +530,16 @@ public void delete(String modelId, ActionListener listener) ); // Setup delete model request - ModelDao.runWithStashedThreadContext(() -> { - DeleteRequestBuilder deleteRequestBuilder = new DeleteRequestBuilder(client, DeleteAction.INSTANCE, MODEL_INDEX_NAME); - deleteRequestBuilder.setId(modelId); - deleteRequestBuilder.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); - - // On model metadata removal, delete the model from the index - clearModelMetadataStep.whenComplete( - acknowledgedResponse -> deleteModelFromIndex(modelId, deleteModelFromIndexStep, deleteRequestBuilder), - listener::onFailure - ); - }); + DeleteRequestBuilder deleteRequestBuilder = new DeleteRequestBuilder(client, DeleteAction.INSTANCE, MODEL_INDEX_NAME); + deleteRequestBuilder.setId(modelId); + deleteRequestBuilder.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + + // On model metadata removal, delete the model from the index + clearModelMetadataStep.whenComplete( + acknowledgedResponse -> deleteModelFromIndex(modelId, deleteModelFromIndexStep, deleteRequestBuilder), + listener::onFailure + ); + deleteModelFromIndexStep.whenComplete(deleteResponse -> { // If model is not deleted, remove modelId from model graveyard and return with error message if (deleteResponse.getResult() != DocWriteResponse.Result.DELETED) { @@ -682,26 +678,4 @@ private String buildRemoveModelErrorMessage(String modelId, RemoveModelFromCache return stringBuilder.toString(); } } - - /** - * Set the thread context to default, this is needed to allow actions on model system index - * when security plugin is enabled - * @param function runnable that needs to be executed after thread context has been stashed, accepts and returns nothing - */ - private static void runWithStashedThreadContext(Runnable function) { - try (ThreadContext.StoredContext context = OpenSearchKNNModelDao.client.threadPool().getThreadContext().stashContext()) { - function.run(); - } - } - - /** - * Set the thread context to default, this is needed to allow actions on model system index - * when security plugin is enabled - * @param function supplier function that needs to be executed after thread context has been stashed, return object - */ - private static T runWithStashedThreadContext(Supplier function) { - try (ThreadContext.StoredContext context = OpenSearchKNNModelDao.client.threadPool().getThreadContext().stashContext()) { - return function.get(); - } - } } diff --git a/src/main/java/org/opensearch/knn/plugin/transport/DeleteModelTransportAction.java b/src/main/java/org/opensearch/knn/plugin/transport/DeleteModelTransportAction.java index ee7f9e939..f535f37dc 100644 --- a/src/main/java/org/opensearch/knn/plugin/transport/DeleteModelTransportAction.java +++ b/src/main/java/org/opensearch/knn/plugin/transport/DeleteModelTransportAction.java @@ -14,7 +14,9 @@ import org.opensearch.action.ActionListener; import org.opensearch.action.support.ActionFilters; import org.opensearch.action.support.HandledTransportAction; +import org.opensearch.client.Client; import org.opensearch.common.inject.Inject; +import org.opensearch.knn.common.ThreadContextHelper; import org.opensearch.knn.indices.ModelDao; import org.opensearch.tasks.Task; import org.opensearch.transport.TransportService; @@ -22,16 +24,20 @@ public class DeleteModelTransportAction extends HandledTransportAction { private final ModelDao modelDao; + private final Client client; @Inject - public DeleteModelTransportAction(TransportService transportService, ActionFilters filters) { + public DeleteModelTransportAction(TransportService transportService, ActionFilters filters, Client client) { super(DeleteModelAction.NAME, transportService, filters, DeleteModelRequest::new); this.modelDao = ModelDao.OpenSearchKNNModelDao.getInstance(); + this.client = client; } @Override protected void doExecute(Task task, DeleteModelRequest request, ActionListener listener) { - String modelID = request.getModelID(); - modelDao.delete(modelID, listener); + ThreadContextHelper.runWithStashedThreadContext(client, () -> { + String modelID = request.getModelID(); + modelDao.delete(modelID, listener); + }); } } diff --git a/src/main/java/org/opensearch/knn/plugin/transport/GetModelTransportAction.java b/src/main/java/org/opensearch/knn/plugin/transport/GetModelTransportAction.java index e47a42d8d..23fa431d3 100644 --- a/src/main/java/org/opensearch/knn/plugin/transport/GetModelTransportAction.java +++ b/src/main/java/org/opensearch/knn/plugin/transport/GetModelTransportAction.java @@ -15,7 +15,9 @@ import org.opensearch.action.ActionListener; import org.opensearch.action.support.ActionFilters; import org.opensearch.action.support.HandledTransportAction; +import org.opensearch.client.Client; import org.opensearch.common.inject.Inject; +import org.opensearch.knn.common.ThreadContextHelper; import org.opensearch.knn.indices.ModelDao; import org.opensearch.tasks.Task; import org.opensearch.transport.TransportService; @@ -27,17 +29,20 @@ public class GetModelTransportAction extends HandledTransportAction actionListener) { - String modelID = request.getModelID(); - - modelDao.get(modelID, actionListener); - + ThreadContextHelper.runWithStashedThreadContext(client, () -> { + String modelID = request.getModelID(); + modelDao.get(modelID, actionListener); + }); } } diff --git a/src/main/java/org/opensearch/knn/plugin/transport/SearchModelTransportAction.java b/src/main/java/org/opensearch/knn/plugin/transport/SearchModelTransportAction.java index 4d9f67059..53a08d80e 100644 --- a/src/main/java/org/opensearch/knn/plugin/transport/SearchModelTransportAction.java +++ b/src/main/java/org/opensearch/knn/plugin/transport/SearchModelTransportAction.java @@ -16,7 +16,9 @@ import org.opensearch.action.search.SearchResponse; import org.opensearch.action.support.ActionFilters; import org.opensearch.action.support.HandledTransportAction; +import org.opensearch.client.Client; import org.opensearch.common.inject.Inject; +import org.opensearch.knn.common.ThreadContextHelper; import org.opensearch.knn.indices.ModelDao; import org.opensearch.tasks.Task; import org.opensearch.transport.TransportService; @@ -26,18 +28,23 @@ public class SearchModelTransportAction extends HandledTransportAction { private ModelDao modelDao; + private final Client client; + @Inject - public SearchModelTransportAction(TransportService transportService, ActionFilters actionFilters) { + public SearchModelTransportAction(TransportService transportService, ActionFilters actionFilters, Client client) { super(SearchModelAction.NAME, transportService, actionFilters, SearchRequest::new); this.modelDao = ModelDao.OpenSearchKNNModelDao.getInstance(); + this.client = client; } @Override protected void doExecute(Task task, SearchRequest request, ActionListener listener) { - try { - this.modelDao.search(request, listener); - } catch (IOException e) { - listener.onFailure(e); - } + ThreadContextHelper.runWithStashedThreadContext(client, () -> { + try { + this.modelDao.search(request, listener); + } catch (IOException e) { + listener.onFailure(e); + } + }); } } diff --git a/src/main/java/org/opensearch/knn/plugin/transport/TrainingJobRouterTransportAction.java b/src/main/java/org/opensearch/knn/plugin/transport/TrainingJobRouterTransportAction.java index 774029c58..b37c03a59 100644 --- a/src/main/java/org/opensearch/knn/plugin/transport/TrainingJobRouterTransportAction.java +++ b/src/main/java/org/opensearch/knn/plugin/transport/TrainingJobRouterTransportAction.java @@ -23,6 +23,7 @@ import org.opensearch.common.ValidationException; import org.opensearch.common.collect.ImmutableOpenMap; import org.opensearch.common.inject.Inject; +import org.opensearch.knn.common.ThreadContextHelper; import org.opensearch.search.builder.SearchSourceBuilder; import org.opensearch.tasks.Task; import org.opensearch.transport.TransportRequestOptions; @@ -58,10 +59,12 @@ protected void doExecute(Task task, TrainingModelRequest request, ActionListener // Get the size of the training request and then route the request. We get/set this here, as opposed to in // TrainingModelTransportAction, because in the future, we may want to use size to factor into our routing // decision. - getTrainingIndexSizeInKB(request, ActionListener.wrap(size -> { - request.setTrainingDataSizeInKB(size); - routeRequest(request, listener); - }, listener::onFailure)); + ThreadContextHelper.runWithStashedThreadContext(client, () -> { + getTrainingIndexSizeInKB(request, ActionListener.wrap(size -> { + request.setTrainingDataSizeInKB(size); + routeRequest(request, listener); + }, listener::onFailure)); + }); } protected void routeRequest(TrainingModelRequest request, ActionListener listener) { diff --git a/src/main/java/org/opensearch/knn/plugin/transport/TrainingModelTransportAction.java b/src/main/java/org/opensearch/knn/plugin/transport/TrainingModelTransportAction.java index a3c4be16e..1f5b85afd 100644 --- a/src/main/java/org/opensearch/knn/plugin/transport/TrainingModelTransportAction.java +++ b/src/main/java/org/opensearch/knn/plugin/transport/TrainingModelTransportAction.java @@ -14,8 +14,10 @@ import org.opensearch.action.ActionListener; import org.opensearch.action.support.ActionFilters; import org.opensearch.action.support.HandledTransportAction; +import org.opensearch.client.Client; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Inject; +import org.opensearch.knn.common.ThreadContextHelper; import org.opensearch.knn.index.memory.NativeMemoryCacheManager; import org.opensearch.knn.index.memory.NativeMemoryEntryContext; import org.opensearch.knn.index.memory.NativeMemoryLoadStrategy; @@ -34,10 +36,18 @@ public class TrainingModelTransportAction extends HandledTransportAction wrappedListener.onResponse(new TrainingModelResponse(indexResponse.getId())), - wrappedListener::onFailure - ) - ); - } catch (IOException e) { - wrappedListener.onFailure(e); - } + ThreadContextHelper.runWithStashedThreadContext(client, () -> { + try { + TrainingJobRunner.getInstance() + .execute( + trainingJob, + ActionListener.wrap( + indexResponse -> wrappedListener.onResponse(new TrainingModelResponse(indexResponse.getId())), + wrappedListener::onFailure + ) + ); + } catch (IOException e) { + wrappedListener.onFailure(e); + } + }); } } diff --git a/src/test/java/org/opensearch/knn/common/ThreadContextHelperTests.java b/src/test/java/org/opensearch/knn/common/ThreadContextHelperTests.java new file mode 100644 index 000000000..cb1e2ef02 --- /dev/null +++ b/src/test/java/org/opensearch/knn/common/ThreadContextHelperTests.java @@ -0,0 +1,52 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.common; + +import org.opensearch.client.node.NodeClient; +import org.opensearch.common.settings.Settings; +import org.opensearch.knn.KNNTestCase; +import org.opensearch.threadpool.TestThreadPool; +import org.opensearch.threadpool.ThreadPool; + +import java.util.function.Supplier; + +public class ThreadContextHelperTests extends KNNTestCase { + + public void testRunWithStashedContextRunnable() { + ThreadPool threadPool = new TestThreadPool(this.getClass().getSimpleName() + "ThreadPool"); + threadPool.getThreadContext().putHeader("key", "value"); + NodeClient client = new NodeClient(Settings.EMPTY, threadPool); + + assertTrue(client.threadPool().getThreadContext().getHeaders().containsKey("key")); + + Runnable runnable = () -> { assertFalse(client.threadPool().getThreadContext().getHeaders().containsKey("key")); }; + ThreadContextHelper.runWithStashedThreadContext(client, () -> runnable); + + assertTrue(client.threadPool().getThreadContext().getHeaders().containsKey("key")); + + threadPool.shutdownNow(); + client.close(); + } + + public void testRunWithStashedContextSupplier() { + ThreadPool threadPool = new TestThreadPool(this.getClass().getSimpleName() + "ThreadPool"); + threadPool.getThreadContext().putHeader("key", "value"); + NodeClient client = new NodeClient(Settings.EMPTY, threadPool); + + assertTrue(client.threadPool().getThreadContext().getHeaders().containsKey("key")); + + Supplier supplier = () -> { + assertFalse(client.threadPool().getThreadContext().getHeaders().containsKey("key")); + return this.getClass().getName(); + }; + ThreadContextHelper.runWithStashedThreadContext(client, () -> supplier); + + assertTrue(client.threadPool().getThreadContext().getHeaders().containsKey("key")); + + threadPool.shutdownNow(); + client.close(); + } +} From e00407921728bcc3409744b366b458d3e0a23e47 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Mon, 10 Apr 2023 11:07:15 -0700 Subject: [PATCH 094/416] Add client setting to ignore warning exceptions (#850) (#851) Signed-off-by: Martin Gaievski (cherry picked from commit d63f5b00f55c88875e3a6cb7cf774e646e421101) Co-authored-by: Martin Gaievski --- CHANGELOG.md | 1 + src/test/java/org/opensearch/knn/index/NmslibIT.java | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ca4f5be93..8ef7dc39f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), * Support .opensearch-knn-model index as system index with security enabled ([#827](https://github.com/opensearch-project/k-NN/pull/827)) * Set gradle dependency scope for common-utils to testFixturesImplementation ([#844](https://github.com/opensearch-project/k-NN/pull/844)) * Add support of .opensearch-knn-model as system index to transport actions ([#847](https://github.com/opensearch-project/k-NN/pull/847)) +* Add client setting to ignore warning exceptions ([#850](https://github.com/opensearch-project/k-NN/pull/850)) ### Documentation ### Maintenance ### Refactoring diff --git a/src/test/java/org/opensearch/knn/index/NmslibIT.java b/src/test/java/org/opensearch/knn/index/NmslibIT.java index 5937c0242..75a5378d3 100644 --- a/src/test/java/org/opensearch/knn/index/NmslibIT.java +++ b/src/test/java/org/opensearch/knn/index/NmslibIT.java @@ -383,4 +383,9 @@ public void testInvalidQueryHnswAlgoParams() { ); assertThat(ex.getMessage(), containsString("Failed to parse value [-1] for setting [index.knn.algo_param.ef_search]")); } + + @Override + protected Settings restClientSettings() { + return noStrictDeprecationModeSettingsBuilder().build(); + } } From c039b7351539b390482f352affa2326890085337 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Wed, 12 Apr 2023 16:00:27 -0500 Subject: [PATCH 095/416] Add github action for secure integ tests (#836) (#853) Signed-off-by: Martin Gaievski (cherry picked from commit 303d811f141508f137fb185e64709389c306e394) Co-authored-by: Martin Gaievski --- .github/workflows/test_security.yml | 98 +++++++++++++++++++++++++++++ CHANGELOG.md | 1 + 2 files changed, 99 insertions(+) create mode 100644 .github/workflows/test_security.yml diff --git a/.github/workflows/test_security.yml b/.github/workflows/test_security.yml new file mode 100644 index 000000000..3813ff675 --- /dev/null +++ b/.github/workflows/test_security.yml @@ -0,0 +1,98 @@ +name: Test k-NN on Secure Cluster +on: + schedule: + - cron: '0 0 * * *' # every night + push: + branches: + - "*" + - "feature/**" + pull_request: + branches: + - "*" + - "feature/**" + +jobs: + Build-ad: + strategy: + matrix: + java: [ 11,17 ] + os: [ubuntu-latest] + fail-fast: true + + name: Test k-NN on Secure Cluster + runs-on: ${{ matrix.os }} + + steps: + - name: Checkout k-NN + uses: actions/checkout@v1 + + - name: Setup Java ${{ matrix.java }} + uses: actions/setup-java@v1 + with: + java-version: ${{ matrix.java }} + + - name: Install dependencies on ubuntu + if: startsWith(matrix.os,'ubuntu') + run: | + sudo apt-get install libopenblas-dev gfortran -y + + - name: Assemble k-NN + run: | + ./gradlew assemble + # example of variables: + # plugin = opensearch-knn-2.7.0.0-SNAPSHOT.zip + # version = 2.7.0 + # plugin_version = 2.7.0.0 + # qualifier = `SNAPSHOT` + - name: Pull and Run Docker + run: | + plugin=`basename $(ls build/distributions/*.zip)` + version=`echo $plugin|awk -F- '{print $3}'| cut -d. -f 1-3` + plugin_version=`echo $plugin|awk -F- '{print $3}'| cut -d. -f 1-4` + qualifier=`echo $plugin|awk -F- '{print $4}'| cut -d. -f 1-1` + if [ $qualifier != `SNAPSHOT` ]; + then + docker_version=$version-$qualifier + else + docker_version=$version + fi + echo plugin version plugin_version qualifier docker_version + echo "($plugin) ($version) ($plugin_version) ($qualifier) ($docker_version)" + + cd .. + if docker pull opensearchstaging/opensearch:$docker_version + then + echo "FROM opensearchstaging/opensearch:$docker_version" >> Dockerfile + # knn plugin cannot be deleted until there are plugin that has dependency on it + echo "RUN if [ -d /usr/share/opensearch/plugins/opensearch-neural-search ]; then /usr/share/opensearch/bin/opensearch-plugin remove opensearch-neural-search; fi" >> Dockerfile + echo "RUN if [ -d /usr/share/opensearch/plugins/opensearch-performance-analyzer ]; then /usr/share/opensearch/bin/opensearch-plugin remove opensearch-performance-analyzer; fi" >> Dockerfile + # saving pre-built artifacts of native libraries as we can't build it with gradle assemle + echo "RUN if [ -d /usr/share/opensearch/plugins/opensearch-knn ]; then cp -r /usr/share/opensearch/plugins/opensearch-knn/lib /usr/share/opensearch/knn-libs; fi" >> Dockerfile + echo "RUN if [ -d /usr/share/opensearch/plugins/opensearch-knn ]; then /usr/share/opensearch/bin/opensearch-plugin remove opensearch-knn; fi" >> Dockerfile + echo "ADD k-NN/build/distributions/$plugin /tmp/" >> Dockerfile + echo "RUN /usr/share/opensearch/bin/opensearch-plugin install --batch file:/tmp/$plugin" >> Dockerfile + # moving pre-built artifacts of native libraries back to plugin folder + echo "RUN if [ -d /usr/share/opensearch/knn-libs ]; then mv /usr/share/opensearch/knn-libs /usr/share/opensearch/plugins/opensearch-knn/lib; fi" >> Dockerfile + docker build -t opensearch-knn:test . + echo "imagePresent=true" >> $GITHUB_ENV + else + echo "imagePresent=false" >> $GITHUB_ENV + fi + + - name: Run Docker Image + if: env.imagePresent == 'true' + run: | + cd .. + docker run -p 9200:9200 -d -p 9600:9600 -e "discovery.type=single-node" opensearch-knn:test + sleep 90 + - name: Run k-NN Integ Test + if: env.imagePresent == 'true' + run: | + security=`curl -XGET https://localhost:9200/_cat/plugins?v -u admin:admin --insecure |grep opensearch-security|wc -l` + if [ $security -gt 0 ] + then + echo "Security plugin is available" + ./gradlew integTest -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername="docker-cluster" -Dhttps=true -Duser=admin -Dpassword=admin + else + echo "Security plugin is NOT available, skipping integration tests" + fi diff --git a/CHANGELOG.md b/CHANGELOG.md index 8ef7dc39f..571bc94a5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), * Support .opensearch-knn-model index as system index with security enabled ([#827](https://github.com/opensearch-project/k-NN/pull/827)) * Set gradle dependency scope for common-utils to testFixturesImplementation ([#844](https://github.com/opensearch-project/k-NN/pull/844)) * Add support of .opensearch-knn-model as system index to transport actions ([#847](https://github.com/opensearch-project/k-NN/pull/847)) +* Add github action for secure integ tests ([#836](https://github.com/opensearch-project/k-NN/pull/836)) * Add client setting to ignore warning exceptions ([#850](https://github.com/opensearch-project/k-NN/pull/850)) ### Documentation ### Maintenance From bfb90785f6bef1bbd51e91da80a7bd8050977f9c Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Thu, 13 Apr 2023 13:53:15 -0500 Subject: [PATCH 096/416] Throw errors on model deletion failures (#834) (#855) Throws errors on model deletion. Previously we were indicating failure, but returning 200 response codes. This changes that to throw exceptions with the correct response code. Signed-off-by: John Mazanec (cherry picked from commit 5c3bf53838080b45152fe3774bef15d6c2186250) Co-authored-by: John Mazanec --- CHANGELOG.md | 1 + .../java/org/opensearch/knn/bwc/ModelIT.java | 26 ++--------- .../DeleteModelWhenInTrainStateException.java | 32 ++++++++++++++ .../org/opensearch/knn/indices/ModelDao.java | 20 ++++----- .../plugin/transport/DeleteModelResponse.java | 31 ++++++++++--- .../transport/DeleteModelTransportAction.java | 7 ++- .../opensearch/knn/indices/ModelDaoTests.java | 43 +++++++++++-------- .../action/RestDeleteModelHandlerIT.java | 22 ++-------- .../action/RestSearchModelHandlerIT.java | 12 ++++-- .../transport/DeleteModelResponseTests.java | 15 +------ 10 files changed, 114 insertions(+), 95 deletions(-) create mode 100644 src/main/java/org/opensearch/knn/common/exception/DeleteModelWhenInTrainStateException.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 571bc94a5..69fa335bc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Features ### Enhancements ### Bug Fixes +* Throw errors on model deletion failures ([#834](https://github.com/opensearch-project/k-NN/pull/834)) ### Infrastructure * Adding filter type to filtering release configs ([#792](https://github.com/opensearch-project/k-NN/pull/792)) * Add CHANGELOG ([#800](https://github.com/opensearch-project/k-NN/pull/800)) diff --git a/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/ModelIT.java b/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/ModelIT.java index e052f6dcf..0157ca45e 100644 --- a/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/ModelIT.java +++ b/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/ModelIT.java @@ -10,6 +10,7 @@ import org.opensearch.action.search.SearchResponse; import org.opensearch.client.Request; import org.opensearch.client.Response; +import org.opensearch.client.ResponseException; import org.opensearch.common.Strings; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; @@ -20,14 +21,12 @@ import org.opensearch.knn.indices.ModelMetadata; import org.opensearch.knn.indices.ModelState; import org.opensearch.knn.plugin.KNNPlugin; -import org.opensearch.knn.plugin.transport.DeleteModelResponse; import org.opensearch.rest.RestStatus; import org.opensearch.search.SearchHit; import java.io.IOException; import java.nio.charset.StandardCharsets; import java.util.Arrays; -import java.util.Locale; import java.util.Map; import static org.opensearch.knn.TestUtils.KNN_BWC_PREFIX; @@ -44,7 +43,6 @@ import static org.opensearch.knn.common.KNNConstants.PARAMETERS; import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_NLIST; import static org.opensearch.knn.common.KNNConstants.NMSLIB_NAME; -import static org.opensearch.knn.common.KNNConstants.MODEL_INDEX_NAME; public class ModelIT extends AbstractRestartUpgradeTestCase { private static final String TEST_MODEL_INDEX = KNN_BWC_PREFIX + "test-model-index"; @@ -153,25 +151,8 @@ public void testDeleteTrainingModel() throws Exception { String restURI = String.join("/", KNNPlugin.KNN_BASE_URI, MODELS, TEST_MODEL_ID_TRAINING); Request request = new Request("DELETE", restURI); - Response response = client().performRequest(request); - assertEquals(request.getEndpoint() + ": failed", RestStatus.OK, RestStatus.fromCode(response.getStatusLine().getStatusCode())); - - assertEquals(3, getDocCount(MODEL_INDEX_NAME)); - - String responseBody = EntityUtils.toString(response.getEntity()); - assertNotNull(responseBody); - - Map responseMap = createParser(XContentType.JSON.xContent(), responseBody).map(); - - assertEquals(TEST_MODEL_ID_TRAINING, responseMap.get(MODEL_ID)); - assertEquals("failed", responseMap.get(DeleteModelResponse.RESULT)); - - String errorMessage = String.format( - Locale.ROOT, - "Cannot delete model \"%s\". Model is still in " + "training", - TEST_MODEL_ID_TRAINING - ); - assertEquals(errorMessage, responseMap.get(DeleteModelResponse.ERROR_MSG)); + ResponseException ex = expectThrows(ResponseException.class, () -> client().performRequest(request)); + assertEquals(RestStatus.CONFLICT.getStatus(), ex.getResponse().getStatusLine().getStatusCode()); } } @@ -181,7 +162,6 @@ public static void wipeAllModels() throws IOException { if (!isRunningAgainstOldCluster()) { deleteKNNModel(TEST_MODEL_ID); deleteKNNModel(TEST_MODEL_ID_DEFAULT); - deleteKNNModel(TEST_MODEL_ID_TRAINING); } } diff --git a/src/main/java/org/opensearch/knn/common/exception/DeleteModelWhenInTrainStateException.java b/src/main/java/org/opensearch/knn/common/exception/DeleteModelWhenInTrainStateException.java new file mode 100644 index 000000000..38854aa59 --- /dev/null +++ b/src/main/java/org/opensearch/knn/common/exception/DeleteModelWhenInTrainStateException.java @@ -0,0 +1,32 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.common.exception; + +import org.opensearch.OpenSearchException; +import org.opensearch.common.logging.LoggerMessageFormat; +import org.opensearch.rest.RestStatus; + +/** + * Exception thrown when a model is deleted while it is in the training state. The RestStatus associated with this + * exception should be a {@link RestStatus#CONFLICT} because the request cannot be deleted due to the model being in + * the training state. + */ +public class DeleteModelWhenInTrainStateException extends OpenSearchException { + /** + * Constructor + * + * @param msg detailed exception message + * @param args arguments of the message + */ + public DeleteModelWhenInTrainStateException(String msg, Object... args) { + super(LoggerMessageFormat.format(msg, args)); + } + + @Override + public RestStatus status() { + return RestStatus.CONFLICT; + } +} diff --git a/src/main/java/org/opensearch/knn/indices/ModelDao.java b/src/main/java/org/opensearch/knn/indices/ModelDao.java index a5b478213..90868f0b7 100644 --- a/src/main/java/org/opensearch/knn/indices/ModelDao.java +++ b/src/main/java/org/opensearch/knn/indices/ModelDao.java @@ -45,6 +45,7 @@ import org.opensearch.index.IndexNotFoundException; import org.opensearch.knn.common.KNNConstants; import org.opensearch.knn.common.ThreadContextHelper; +import org.opensearch.knn.common.exception.DeleteModelWhenInTrainStateException; import org.opensearch.knn.plugin.transport.DeleteModelResponse; import org.opensearch.knn.plugin.transport.GetModelResponse; import org.opensearch.knn.plugin.transport.RemoveModelFromCacheAction; @@ -174,8 +175,6 @@ public interface ModelDao { final class OpenSearchKNNModelDao implements ModelDao { public static Logger logger = LogManager.getLogger(ModelDao.class); - private static final String DELETED = "deleted"; - private static final String FAILED = "failed"; private int numberOfShards; private int numberOfReplicas; @@ -487,9 +486,8 @@ public boolean isModelInGraveyard(String modelId) { public void delete(String modelId, ActionListener listener) { // If the index is not created, there is no need to delete the model if (!isCreated()) { - logger.error("Cannot delete model \"" + modelId + "\". Model index " + MODEL_INDEX_NAME + "does not exist."); - String errorMessage = String.format("Cannot delete model \"%s\". Model index does not exist", modelId); - listener.onResponse(new DeleteModelResponse(modelId, FAILED, errorMessage)); + String errorMessage = String.format("Cannot delete model [%s]. Model index [%s] does not exist", modelId, MODEL_INDEX_NAME); + listener.onFailure(new ResourceNotFoundException(errorMessage)); return; } @@ -503,7 +501,7 @@ public void delete(String modelId, ActionListener listener) // Get Model to check if model is in TRAINING get(modelId, ActionListener.wrap(getModelStep::onResponse, exception -> { if (exception instanceof ResourceNotFoundException) { - String errorMessage = String.format("Unable to delete model \"%s\". Model does not exist", modelId); + String errorMessage = String.format("Unable to delete model [%s]. Model does not exist", modelId); ResourceNotFoundException resourceNotFoundException = new ResourceNotFoundException(errorMessage); removeModelIdFromGraveyardOnFailure(modelId, resourceNotFoundException, getModelStep); } else { @@ -514,8 +512,8 @@ public void delete(String modelId, ActionListener listener) getModelStep.whenComplete(getModelResponse -> { // If model is in Training state, fail delete model request if (ModelState.TRAINING == getModelResponse.getModel().getModelMetadata().getState()) { - String errorMessage = String.format("Cannot delete model \"%s\". Model is still in training", modelId); - listener.onResponse(new DeleteModelResponse(modelId, FAILED, errorMessage)); + String errorMessage = String.format("Cannot delete model [%s]. Model is still in training", modelId); + listener.onFailure(new DeleteModelWhenInTrainStateException(errorMessage)); return; } @@ -544,8 +542,8 @@ public void delete(String modelId, ActionListener listener) // If model is not deleted, remove modelId from model graveyard and return with error message if (deleteResponse.getResult() != DocWriteResponse.Result.DELETED) { updateModelGraveyardToDelete(modelId, true, unblockModelIdStep, Optional.empty()); - String errorMessage = String.format("Model \" %s \" does not exist", modelId); - listener.onResponse(new DeleteModelResponse(modelId, deleteResponse.getResult().getLowercase(), errorMessage)); + String errorMessage = String.format("Model [%s] does not exist", modelId); + listener.onFailure(new ResourceNotFoundException(errorMessage)); return; } @@ -569,7 +567,7 @@ public void delete(String modelId, ActionListener listener) unblockModelIdStep.whenComplete(acknowledgedResponse -> { // After clearing the cache, if there are no errors return the response - listener.onResponse(new DeleteModelResponse(modelId, DELETED, null)); + listener.onResponse(new DeleteModelResponse(modelId)); }, listener::onFailure); diff --git a/src/main/java/org/opensearch/knn/plugin/transport/DeleteModelResponse.java b/src/main/java/org/opensearch/knn/plugin/transport/DeleteModelResponse.java index 1b0d8e3c8..3f2cb90de 100644 --- a/src/main/java/org/opensearch/knn/plugin/transport/DeleteModelResponse.java +++ b/src/main/java/org/opensearch/knn/plugin/transport/DeleteModelResponse.java @@ -12,7 +12,6 @@ import org.opensearch.action.ActionResponse; import org.opensearch.common.Nullable; -import org.opensearch.common.Strings; import org.opensearch.common.io.stream.StreamInput; import org.opensearch.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.ToXContentObject; @@ -29,16 +28,40 @@ public class DeleteModelResponse extends ActionResponse implements ToXContentObj public static final String RESULT = "result"; public static final String ERROR_MSG = "error"; + private static final String DELETED = "deleted"; private final String modelID; private final String result; private final String errorMessage; + /** + * Ctor to build delete model response. + * @deprecated + * Returning errors through {@link DeleteModelResponse} should not be done. Instead, if there is an + * error, throw/return a suitable exception. Use {@link DeleteModelResponse#DeleteModelResponse(String)} to + * construct valid responses instead. + * + * @param modelID ID of the model that is deleted + * @param result Resulting action of the deletion. + * @param errorMessage Error message to be returned to the user + */ + @Deprecated public DeleteModelResponse(String modelID, String result, @Nullable String errorMessage) { this.modelID = modelID; this.result = result; this.errorMessage = errorMessage; } + /** + * Ctor to build delete model response + * + * @param modelID ID of the model that is deleted + */ + public DeleteModelResponse(String modelID) { + this.modelID = modelID; + this.result = DELETED; + this.errorMessage = null; + } + public DeleteModelResponse(StreamInput in) throws IOException { super(in); this.modelID = in.readString(); @@ -63,16 +86,12 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws /* Response should look like below: { "model_id": "my_model_id" - "result": "not_found", - "error": "Model my_model_id doesn't exist" + "result": "deleted" } */ builder.startObject(); builder.field(MODEL_ID, getModelID()); builder.field(RESULT, getResult()); - if (Strings.hasText(errorMessage)) { - builder.field(ERROR_MSG, getErrorMessage()); - } builder.endObject(); return builder; diff --git a/src/main/java/org/opensearch/knn/plugin/transport/DeleteModelTransportAction.java b/src/main/java/org/opensearch/knn/plugin/transport/DeleteModelTransportAction.java index f535f37dc..1a8d43552 100644 --- a/src/main/java/org/opensearch/knn/plugin/transport/DeleteModelTransportAction.java +++ b/src/main/java/org/opensearch/knn/plugin/transport/DeleteModelTransportAction.java @@ -11,6 +11,7 @@ package org.opensearch.knn.plugin.transport; +import lombok.extern.log4j.Log4j2; import org.opensearch.action.ActionListener; import org.opensearch.action.support.ActionFilters; import org.opensearch.action.support.HandledTransportAction; @@ -21,6 +22,7 @@ import org.opensearch.tasks.Task; import org.opensearch.transport.TransportService; +@Log4j2 public class DeleteModelTransportAction extends HandledTransportAction { private final ModelDao modelDao; @@ -37,7 +39,10 @@ public DeleteModelTransportAction(TransportService transportService, ActionFilte protected void doExecute(Task task, DeleteModelRequest request, ActionListener listener) { ThreadContextHelper.runWithStashedThreadContext(client, () -> { String modelID = request.getModelID(); - modelDao.delete(modelID, listener); + modelDao.delete(modelID, ActionListener.wrap(listener::onResponse, e -> { + log.error(e); + listener.onFailure(e); + })); }); } } diff --git a/src/test/java/org/opensearch/knn/indices/ModelDaoTests.java b/src/test/java/org/opensearch/knn/indices/ModelDaoTests.java index b2ff95b13..5122d6998 100644 --- a/src/test/java/org/opensearch/knn/indices/ModelDaoTests.java +++ b/src/test/java/org/opensearch/knn/indices/ModelDaoTests.java @@ -16,6 +16,7 @@ import org.junit.BeforeClass; import org.opensearch.ExceptionsHelper; import org.opensearch.ResourceAlreadyExistsException; +import org.opensearch.ResourceNotFoundException; import org.opensearch.action.ActionListener; import org.opensearch.action.DocWriteResponse; import org.opensearch.action.StepListener; @@ -32,6 +33,7 @@ import org.opensearch.index.IndexNotFoundException; import org.opensearch.index.engine.VersionConflictEngineException; import org.opensearch.knn.KNNSingleNodeTestCase; +import org.opensearch.knn.common.exception.DeleteModelWhenInTrainStateException; import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.util.KNNEngine; import org.opensearch.knn.plugin.transport.DeleteModelResponse; @@ -500,10 +502,13 @@ public void testDelete() throws IOException, InterruptedException { int dimension = 2; final CountDownLatch inProgressLatch = new CountDownLatch(1); - ActionListener deleteModelIndexDoesNotExistListener = ActionListener.wrap(response -> { - assertEquals(FAILED, response.getResult()); - inProgressLatch.countDown(); - }, exception -> fail("Unable to delete the model: " + exception)); + ActionListener deleteModelIndexDoesNotExistListener = ActionListener.wrap( + response -> fail("Deleting model when model index does not exist should throw ResourceNotFoundException"), + exception -> { + assertTrue(exception instanceof ResourceNotFoundException); + inProgressLatch.countDown(); + } + ); // model index doesnt exist modelDao.delete(modelId, deleteModelIndexDoesNotExistListener); assertTrue(inProgressLatch.await(100, TimeUnit.SECONDS)); @@ -512,25 +517,27 @@ public void testDelete() throws IOException, InterruptedException { // Model does not exist final CountDownLatch inProgressLatch1 = new CountDownLatch(1); - ActionListener deleteModelDoesNotExistListener = ActionListener.wrap(Assert::assertNull, exception -> { - assertNotNull(exception); - assertTrue(exception.getMessage().contains(modelId)); - assertTrue(exception.getMessage().contains("Model does not exist")); - assertFalse(modelDao.isModelInGraveyard(modelId)); - inProgressLatch1.countDown(); - }); + ActionListener deleteModelDoesNotExistListener = ActionListener.wrap( + response -> fail("Deleting model when model does not exist should throw ResourceNotFoundException"), + exception -> { + assertTrue(exception instanceof ResourceNotFoundException); + assertFalse(modelDao.isModelInGraveyard(modelId)); + inProgressLatch1.countDown(); + } + ); modelDao.delete(modelId, deleteModelDoesNotExistListener); assertTrue(inProgressLatch1.await(60, TimeUnit.SECONDS)); final CountDownLatch inProgressLatch2 = new CountDownLatch(1); - ActionListener deleteModelTrainingListener = ActionListener.wrap(response -> { - assertEquals(modelId, response.getModelID()); - assertEquals(FAILED, response.getResult()); - String errorMessage = String.format("Cannot delete model \"%s\". Model is still in training", modelId); - assertEquals(errorMessage, response.getErrorMessage()); - inProgressLatch2.countDown(); - }, exception -> fail("Unable to delete model: " + exception)); + ActionListener deleteModelTrainingListener = ActionListener.wrap( + response -> fail("Deleting model when model does not exist should throw ResourceNotFoundException"), + exception -> { + assertTrue(exception instanceof DeleteModelWhenInTrainStateException); + assertFalse(modelDao.isModelInGraveyard(modelId)); + inProgressLatch2.countDown(); + } + ); // model id exists and model is still in Training Model model = new Model( diff --git a/src/test/java/org/opensearch/knn/plugin/action/RestDeleteModelHandlerIT.java b/src/test/java/org/opensearch/knn/plugin/action/RestDeleteModelHandlerIT.java index 12d45d8a3..edc13c83a 100644 --- a/src/test/java/org/opensearch/knn/plugin/action/RestDeleteModelHandlerIT.java +++ b/src/test/java/org/opensearch/knn/plugin/action/RestDeleteModelHandlerIT.java @@ -20,7 +20,6 @@ import org.opensearch.common.xcontent.XContentType; import org.opensearch.knn.KNNRestTestCase; import org.opensearch.knn.plugin.KNNPlugin; -import org.opensearch.knn.plugin.transport.DeleteModelResponse; import org.opensearch.rest.RestStatus; import java.util.List; @@ -107,23 +106,8 @@ public void testDeleteTrainingModel() throws Exception { String deleteModelRestURI = String.join("/", KNNPlugin.KNN_BASE_URI, MODELS, modelId); Request deleteModelRequest = new Request("DELETE", deleteModelRestURI); - Response deleteModelResponse = client().performRequest(deleteModelRequest); - assertEquals( - deleteModelRequest.getEndpoint() + ": failed", - RestStatus.OK, - RestStatus.fromCode(deleteModelResponse.getStatusLine().getStatusCode()) - ); - - responseBody = EntityUtils.toString(deleteModelResponse.getEntity()); - assertNotNull(responseBody); - - responseMap = createParser(XContentType.JSON.xContent(), responseBody).map(); - - assertEquals(modelId, responseMap.get(MODEL_ID)); - assertEquals("failed", responseMap.get(DeleteModelResponse.RESULT)); - - String errorMessage = String.format("Cannot delete model \"%s\". Model is still in training", modelId); - assertEquals(errorMessage, responseMap.get(DeleteModelResponse.ERROR_MSG)); + ResponseException ex = expectThrows(ResponseException.class, () -> client().performRequest(deleteModelRequest)); + assertEquals(RestStatus.CONFLICT.getStatus(), ex.getResponse().getStatusLine().getStatusCode()); // need to wait for training operation as it's required for after test cleanup assertTrainingSucceeds(modelId, NUM_OF_ATTEMPTS, DELAY_MILLI_SEC); @@ -136,7 +120,7 @@ public void testDeleteModelFailsInvalid() throws Exception { Request request = new Request("DELETE", restURI); ResponseException ex = expectThrows(ResponseException.class, () -> client().performRequest(request)); - assertTrue(ex.getMessage().contains(modelId)); + assertEquals(RestStatus.NOT_FOUND.getStatus(), ex.getResponse().getStatusLine().getStatusCode()); } // Test Train Model -> Delete Model -> Train Model with same modelId diff --git a/src/test/java/org/opensearch/knn/plugin/action/RestSearchModelHandlerIT.java b/src/test/java/org/opensearch/knn/plugin/action/RestSearchModelHandlerIT.java index 92834217e..f5a62f838 100644 --- a/src/test/java/org/opensearch/knn/plugin/action/RestSearchModelHandlerIT.java +++ b/src/test/java/org/opensearch/knn/plugin/action/RestSearchModelHandlerIT.java @@ -87,11 +87,15 @@ public void testSizeValidationFailsInvalidSize() throws IOException { Request request = new Request("GET", restURI); ResponseException ex = expectThrows(ResponseException.class, () -> client().performRequest(request)); + String messageExpected = String.format( + "%s must be between %s and %s inclusive", + PARAM_SIZE, + SEARCH_MODEL_MIN_SIZE, + SEARCH_MODEL_MAX_SIZE + ); assertTrue( - ex.getMessage() - .contains( - String.format("%s must be between %d and %d inclusive", PARAM_SIZE, SEARCH_MODEL_MIN_SIZE, SEARCH_MODEL_MAX_SIZE) - ) + String.format("FAILED - Expected \"%s\" to have \"%s\"", ex.getMessage(), messageExpected), + ex.getMessage().contains(messageExpected) ); } diff --git a/src/test/java/org/opensearch/knn/plugin/transport/DeleteModelResponseTests.java b/src/test/java/org/opensearch/knn/plugin/transport/DeleteModelResponseTests.java index f72cabe8a..ed0245c8f 100644 --- a/src/test/java/org/opensearch/knn/plugin/transport/DeleteModelResponseTests.java +++ b/src/test/java/org/opensearch/knn/plugin/transport/DeleteModelResponseTests.java @@ -24,7 +24,7 @@ public class DeleteModelResponseTests extends KNNTestCase { public void testStreams() throws IOException { String modelId = "test-model"; - DeleteModelResponse deleteModelResponse = new DeleteModelResponse(modelId, "delete action failed", "error message"); + DeleteModelResponse deleteModelResponse = new DeleteModelResponse(modelId); BytesStreamOutput streamOutput = new BytesStreamOutput(); deleteModelResponse.writeTo(streamOutput); DeleteModelResponse deleteModelResponseCopy = new DeleteModelResponse(streamOutput.bytes().streamInput()); @@ -33,20 +33,9 @@ public void testStreams() throws IOException { assertEquals(deleteModelResponse.getErrorMessage(), deleteModelResponseCopy.getErrorMessage()); } - public void testXContentWithError() throws IOException { - String modelId = "test-model"; - DeleteModelResponse deleteModelResponse = new DeleteModelResponse(modelId, "not_found", "model id not found"); - BytesStreamOutput streamOutput = new BytesStreamOutput(); - deleteModelResponse.writeTo(streamOutput); - String expectedResponseString = "{\"model_id\":\"test-model\",\"result\":\"not_found\",\"error\":\"model id not found\"}"; - XContentBuilder xContentBuilder = XContentFactory.contentBuilder(XContentType.JSON); - deleteModelResponse.toXContent(xContentBuilder, null); - assertEquals(expectedResponseString, Strings.toString(xContentBuilder)); - } - public void testXContentWithoutError() throws IOException { String modelId = "test-model"; - DeleteModelResponse deleteModelResponse = new DeleteModelResponse(modelId, "deleted", null); + DeleteModelResponse deleteModelResponse = new DeleteModelResponse(modelId); BytesStreamOutput streamOutput = new BytesStreamOutput(); deleteModelResponse.writeTo(streamOutput); String expectedResponseString = "{\"model_id\":\"test-model\",\"result\":\"deleted\"}"; From 648c550d66b4d6003bf22f1f5a1ea27a0910af69 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Fri, 14 Apr 2023 13:16:43 -0700 Subject: [PATCH 097/416] Switch stashing thread context back to only around system index calls (#862) Revert "Add support of .opensearch-knn-model as system index to transport actions (#847)" Makes minor fixes to stashed thread context for calls to the model system index. In general, only direct calls to the system index should be wrapped in the stashed context. Signed-off-by: John Mazanec (cherry picked from commit 11c6616456e74d5c72ca5704e4a4ec867acf8d60) --- CHANGELOG.md | 1 - .../knn/common/ThreadContextHelper.java | 39 ----- .../org/opensearch/knn/indices/ModelDao.java | 144 +++++++++++------- .../transport/DeleteModelTransportAction.java | 18 +-- .../transport/GetModelTransportAction.java | 15 +- .../transport/SearchModelTransportAction.java | 19 +-- .../TrainingJobRouterTransportAction.java | 11 +- .../TrainingModelTransportAction.java | 39 ++--- .../knn/common/ThreadContextHelperTests.java | 52 ------- 9 files changed, 120 insertions(+), 218 deletions(-) delete mode 100644 src/main/java/org/opensearch/knn/common/ThreadContextHelper.java delete mode 100644 src/test/java/org/opensearch/knn/common/ThreadContextHelperTests.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 69fa335bc..5db16394a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,7 +27,6 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), * Bump numpy version from 1.22.x to 1.24.2 ([#811](https://github.com/opensearch-project/k-NN/pull/811)) * Support .opensearch-knn-model index as system index with security enabled ([#827](https://github.com/opensearch-project/k-NN/pull/827)) * Set gradle dependency scope for common-utils to testFixturesImplementation ([#844](https://github.com/opensearch-project/k-NN/pull/844)) -* Add support of .opensearch-knn-model as system index to transport actions ([#847](https://github.com/opensearch-project/k-NN/pull/847)) * Add github action for secure integ tests ([#836](https://github.com/opensearch-project/k-NN/pull/836)) * Add client setting to ignore warning exceptions ([#850](https://github.com/opensearch-project/k-NN/pull/850)) ### Documentation diff --git a/src/main/java/org/opensearch/knn/common/ThreadContextHelper.java b/src/main/java/org/opensearch/knn/common/ThreadContextHelper.java deleted file mode 100644 index e0c5ad8f9..000000000 --- a/src/main/java/org/opensearch/knn/common/ThreadContextHelper.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.knn.common; - -import org.opensearch.client.Client; -import org.opensearch.common.util.concurrent.ThreadContext; - -import java.util.function.Supplier; - -/** - * Class abstracts execution of runnable or function in specific context - */ -public class ThreadContextHelper { - - /** - * Sets the thread context to default and execute function, this needed to allow actions on model system index - * when security plugin is enabled - * @param function runnable that needs to be executed after thread context has been stashed, accepts and returns nothing - */ - public static void runWithStashedThreadContext(Client client, Runnable function) { - try (ThreadContext.StoredContext context = client.threadPool().getThreadContext().stashContext()) { - function.run(); - } - } - - /** - * Sets the thread context to default and execute function, this needed to allow actions on model system index - * when security plugin is enabled - * @param function supplier function that needs to be executed after thread context has been stashed, return object - */ - public static T runWithStashedThreadContext(Client client, Supplier function) { - try (ThreadContext.StoredContext context = client.threadPool().getThreadContext().stashContext()) { - return function.get(); - } - } -} diff --git a/src/main/java/org/opensearch/knn/indices/ModelDao.java b/src/main/java/org/opensearch/knn/indices/ModelDao.java index 90868f0b7..0573b87c1 100644 --- a/src/main/java/org/opensearch/knn/indices/ModelDao.java +++ b/src/main/java/org/opensearch/knn/indices/ModelDao.java @@ -13,6 +13,7 @@ import com.google.common.base.Charsets; import com.google.common.io.Resources; +import lombok.SneakyThrows; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.opensearch.OpenSearchException; @@ -42,19 +43,19 @@ import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.concurrent.ThreadContext; import org.opensearch.index.IndexNotFoundException; import org.opensearch.knn.common.KNNConstants; -import org.opensearch.knn.common.ThreadContextHelper; import org.opensearch.knn.common.exception.DeleteModelWhenInTrainStateException; import org.opensearch.knn.plugin.transport.DeleteModelResponse; import org.opensearch.knn.plugin.transport.GetModelResponse; import org.opensearch.knn.plugin.transport.RemoveModelFromCacheAction; import org.opensearch.knn.plugin.transport.RemoveModelFromCacheRequest; import org.opensearch.knn.plugin.transport.RemoveModelFromCacheResponse; -import org.opensearch.knn.plugin.transport.UpdateModelMetadataAction; -import org.opensearch.knn.plugin.transport.UpdateModelMetadataRequest; import org.opensearch.knn.plugin.transport.UpdateModelGraveyardAction; import org.opensearch.knn.plugin.transport.UpdateModelGraveyardRequest; +import org.opensearch.knn.plugin.transport.UpdateModelMetadataAction; +import org.opensearch.knn.plugin.transport.UpdateModelMetadataRequest; import java.io.IOException; import java.net.URL; @@ -64,6 +65,7 @@ import java.util.Objects; import java.util.Optional; import java.util.concurrent.ExecutionException; +import java.util.function.Supplier; import static java.util.Objects.isNull; import static org.opensearch.knn.common.KNNConstants.MODEL_INDEX_MAPPING_PATH; @@ -216,14 +218,21 @@ public void create(ActionListener actionListener) throws IO if (isCreated()) { return; } - CreateIndexRequest request = new CreateIndexRequest(MODEL_INDEX_NAME).mapping(getMapping()) - .settings( - Settings.builder() - .put("index.hidden", true) - .put("index.number_of_shards", this.numberOfShards) - .put("index.number_of_replicas", this.numberOfReplicas) - ); - client.admin().indices().create(request, actionListener); + runWithStashedThreadContext(() -> { + CreateIndexRequest request; + try { + request = new CreateIndexRequest(MODEL_INDEX_NAME).mapping(getMapping()) + .settings( + Settings.builder() + .put("index.hidden", true) + .put("index.number_of_shards", this.numberOfShards) + .put("index.number_of_replicas", this.numberOfReplicas) + ); + } catch (IOException e) { + throw new RuntimeException(e); + } + client.admin().indices().create(request, actionListener); + }); } @Override @@ -293,8 +302,7 @@ private void putInternal(Model model, ActionListener listener, Do parameters.put(KNNConstants.MODEL_BLOB_PARAMETER, base64Model); } - IndexRequestBuilder indexRequestBuilder = client.prepareIndex(MODEL_INDEX_NAME); - + final IndexRequestBuilder indexRequestBuilder = client.prepareIndex(MODEL_INDEX_NAME); indexRequestBuilder.setId(model.getModelID()); indexRequestBuilder.setSource(parameters); @@ -304,8 +312,8 @@ private void putInternal(Model model, ActionListener listener, Do // After metadata update finishes, remove item from every node's cache if necessary. If no model id is // passed then nothing needs to be removed from the cache ActionListener onMetaListener; - onMetaListener = ActionListener.wrap( - indexResponse -> client.execute( + onMetaListener = ActionListener.wrap(indexResponse -> { + client.execute( RemoveModelFromCacheAction.INSTANCE, new RemoveModelFromCacheRequest(model.getModelID()), ActionListener.wrap(removeModelFromCacheResponse -> { @@ -318,9 +326,8 @@ private void putInternal(Model model, ActionListener listener, Do listener.onFailure(new RuntimeException(failureMessage)); }, listener::onFailure) - ), - listener::onFailure - ); + ); + }, listener::onFailure); // After the model is indexed, update metadata only if the model is in CREATED state ActionListener onIndexListener; @@ -331,14 +338,18 @@ private void putInternal(Model model, ActionListener listener, Do } // Create the model index if it does not already exist + Runnable indexModelRunnable = () -> indexRequestBuilder.execute(onIndexListener); if (!isCreated()) { create( - ActionListener.wrap(createIndexResponse -> indexRequestBuilder.execute(onIndexListener), onIndexListener::onFailure) + ActionListener.wrap( + createIndexResponse -> ModelDao.runWithStashedThreadContext(indexModelRunnable), + onIndexListener::onFailure + ) ); return; } - indexRequestBuilder.execute(onIndexListener); + ModelDao.runWithStashedThreadContext(indexModelRunnable); } private ActionListener getUpdateModelMetadataListener( @@ -357,13 +368,14 @@ private ActionListener getUpdateModelMetadataListener( ); } + @SneakyThrows @Override - public Model get(String modelId) throws ExecutionException, InterruptedException { + public Model get(String modelId) { /* GET //?_local */ try { - return ThreadContextHelper.runWithStashedThreadContext(client, () -> { + return ModelDao.runWithStashedThreadContext(() -> { GetRequestBuilder getRequestBuilder = new GetRequestBuilder(client, GetAction.INSTANCE, MODEL_INDEX_NAME).setId(modelId) .setPreference("_local"); GetResponse getResponse; @@ -378,16 +390,7 @@ public Model get(String modelId) throws ExecutionException, InterruptedException } catch (RuntimeException runtimeException) { // we need to use RuntimeException as container for real exception to keep signature // of runWithStashedThreadContext generic - Throwable throwable = runtimeException.getCause(); - if (throwable != null) { - if (throwable instanceof InterruptedException) { - throw (InterruptedException) throwable; - } - if (throwable instanceof ExecutionException) { - throw (ExecutionException) throwable; - } - } - throw runtimeException; + throw runtimeException.getCause(); } } @@ -402,20 +405,22 @@ public void get(String modelId, ActionListener actionListener) /* GET //?_local */ - GetRequestBuilder getRequestBuilder = new GetRequestBuilder(client, GetAction.INSTANCE, MODEL_INDEX_NAME).setId(modelId) - .setPreference("_local"); - - getRequestBuilder.execute(ActionListener.wrap(response -> { - if (response.isSourceEmpty()) { - String errorMessage = String.format("Model \" %s \" does not exist", modelId); - actionListener.onFailure(new ResourceNotFoundException(modelId, errorMessage)); - return; - } - final Map responseMap = response.getSourceAsMap(); - Model model = Model.getModelFromSourceMap(responseMap); - actionListener.onResponse(new GetModelResponse(model)); + ModelDao.runWithStashedThreadContext(() -> { + GetRequestBuilder getRequestBuilder = new GetRequestBuilder(client, GetAction.INSTANCE, MODEL_INDEX_NAME).setId(modelId) + .setPreference("_local"); + + getRequestBuilder.execute(ActionListener.wrap(response -> { + if (response.isSourceEmpty()) { + String errorMessage = String.format("Model \" %s \" does not exist", modelId); + actionListener.onFailure(new ResourceNotFoundException(modelId, errorMessage)); + return; + } + final Map responseMap = response.getSourceAsMap(); + Model model = Model.getModelFromSourceMap(responseMap); + actionListener.onResponse(new GetModelResponse(model)); - }, actionListener::onFailure)); + }, actionListener::onFailure)); + }); } /** @@ -426,7 +431,7 @@ public void get(String modelId, ActionListener actionListener) */ @Override public void search(SearchRequest request, ActionListener actionListener) { - ThreadContextHelper.runWithStashedThreadContext(client, () -> { + ModelDao.runWithStashedThreadContext(() -> { request.indices(MODEL_INDEX_NAME); client.search(request, actionListener); }); @@ -528,15 +533,12 @@ public void delete(String modelId, ActionListener listener) ); // Setup delete model request - DeleteRequestBuilder deleteRequestBuilder = new DeleteRequestBuilder(client, DeleteAction.INSTANCE, MODEL_INDEX_NAME); - deleteRequestBuilder.setId(modelId); - deleteRequestBuilder.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); - - // On model metadata removal, delete the model from the index - clearModelMetadataStep.whenComplete( - acknowledgedResponse -> deleteModelFromIndex(modelId, deleteModelFromIndexStep, deleteRequestBuilder), - listener::onFailure - ); + clearModelMetadataStep.whenComplete(acknowledgedResponse -> { + DeleteRequestBuilder deleteRequestBuilder = new DeleteRequestBuilder(client, DeleteAction.INSTANCE, MODEL_INDEX_NAME); + deleteRequestBuilder.setId(modelId); + deleteRequestBuilder.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + deleteModelFromIndex(modelId, deleteModelFromIndexStep, deleteRequestBuilder); + }, listener::onFailure); deleteModelFromIndexStep.whenComplete(deleteResponse -> { // If model is not deleted, remove modelId from model graveyard and return with error message @@ -591,10 +593,12 @@ private void deleteModelFromIndex( StepListener deleteModelFromIndexStep, DeleteRequestBuilder deleteRequestBuilder ) { - deleteRequestBuilder.execute( - ActionListener.wrap( - deleteModelFromIndexStep::onResponse, - exception -> removeModelIdFromGraveyardOnFailure(modelId, exception, deleteModelFromIndexStep) + ModelDao.runWithStashedThreadContext( + () -> deleteRequestBuilder.execute( + ActionListener.wrap( + deleteModelFromIndexStep::onResponse, + exception -> removeModelIdFromGraveyardOnFailure(modelId, exception, deleteModelFromIndexStep) + ) ) ); } @@ -676,4 +680,26 @@ private String buildRemoveModelErrorMessage(String modelId, RemoveModelFromCache return stringBuilder.toString(); } } + + /** + * Set the thread context to default, this is needed to allow actions on model system index + * when security plugin is enabled + * @param function runnable that needs to be executed after thread context has been stashed, accepts and returns nothing + */ + private static void runWithStashedThreadContext(Runnable function) { + try (ThreadContext.StoredContext context = OpenSearchKNNModelDao.client.threadPool().getThreadContext().stashContext()) { + function.run(); + } + } + + /** + * Set the thread context to default, this is needed to allow actions on model system index + * when security plugin is enabled + * @param function supplier function that needs to be executed after thread context has been stashed, return object + */ + private static T runWithStashedThreadContext(Supplier function) { + try (ThreadContext.StoredContext context = OpenSearchKNNModelDao.client.threadPool().getThreadContext().stashContext()) { + return function.get(); + } + } } diff --git a/src/main/java/org/opensearch/knn/plugin/transport/DeleteModelTransportAction.java b/src/main/java/org/opensearch/knn/plugin/transport/DeleteModelTransportAction.java index 1a8d43552..a325d8b38 100644 --- a/src/main/java/org/opensearch/knn/plugin/transport/DeleteModelTransportAction.java +++ b/src/main/java/org/opensearch/knn/plugin/transport/DeleteModelTransportAction.java @@ -15,9 +15,7 @@ import org.opensearch.action.ActionListener; import org.opensearch.action.support.ActionFilters; import org.opensearch.action.support.HandledTransportAction; -import org.opensearch.client.Client; import org.opensearch.common.inject.Inject; -import org.opensearch.knn.common.ThreadContextHelper; import org.opensearch.knn.indices.ModelDao; import org.opensearch.tasks.Task; import org.opensearch.transport.TransportService; @@ -26,23 +24,19 @@ public class DeleteModelTransportAction extends HandledTransportAction { private final ModelDao modelDao; - private final Client client; @Inject - public DeleteModelTransportAction(TransportService transportService, ActionFilters filters, Client client) { + public DeleteModelTransportAction(TransportService transportService, ActionFilters filters) { super(DeleteModelAction.NAME, transportService, filters, DeleteModelRequest::new); this.modelDao = ModelDao.OpenSearchKNNModelDao.getInstance(); - this.client = client; } @Override protected void doExecute(Task task, DeleteModelRequest request, ActionListener listener) { - ThreadContextHelper.runWithStashedThreadContext(client, () -> { - String modelID = request.getModelID(); - modelDao.delete(modelID, ActionListener.wrap(listener::onResponse, e -> { - log.error(e); - listener.onFailure(e); - })); - }); + String modelID = request.getModelID(); + modelDao.delete(modelID, ActionListener.wrap(listener::onResponse, e -> { + log.error(e); + listener.onFailure(e); + })); } } diff --git a/src/main/java/org/opensearch/knn/plugin/transport/GetModelTransportAction.java b/src/main/java/org/opensearch/knn/plugin/transport/GetModelTransportAction.java index 23fa431d3..e47a42d8d 100644 --- a/src/main/java/org/opensearch/knn/plugin/transport/GetModelTransportAction.java +++ b/src/main/java/org/opensearch/knn/plugin/transport/GetModelTransportAction.java @@ -15,9 +15,7 @@ import org.opensearch.action.ActionListener; import org.opensearch.action.support.ActionFilters; import org.opensearch.action.support.HandledTransportAction; -import org.opensearch.client.Client; import org.opensearch.common.inject.Inject; -import org.opensearch.knn.common.ThreadContextHelper; import org.opensearch.knn.indices.ModelDao; import org.opensearch.tasks.Task; import org.opensearch.transport.TransportService; @@ -29,20 +27,17 @@ public class GetModelTransportAction extends HandledTransportAction actionListener) { - ThreadContextHelper.runWithStashedThreadContext(client, () -> { - String modelID = request.getModelID(); - modelDao.get(modelID, actionListener); - }); + String modelID = request.getModelID(); + + modelDao.get(modelID, actionListener); + } } diff --git a/src/main/java/org/opensearch/knn/plugin/transport/SearchModelTransportAction.java b/src/main/java/org/opensearch/knn/plugin/transport/SearchModelTransportAction.java index 53a08d80e..4d9f67059 100644 --- a/src/main/java/org/opensearch/knn/plugin/transport/SearchModelTransportAction.java +++ b/src/main/java/org/opensearch/knn/plugin/transport/SearchModelTransportAction.java @@ -16,9 +16,7 @@ import org.opensearch.action.search.SearchResponse; import org.opensearch.action.support.ActionFilters; import org.opensearch.action.support.HandledTransportAction; -import org.opensearch.client.Client; import org.opensearch.common.inject.Inject; -import org.opensearch.knn.common.ThreadContextHelper; import org.opensearch.knn.indices.ModelDao; import org.opensearch.tasks.Task; import org.opensearch.transport.TransportService; @@ -28,23 +26,18 @@ public class SearchModelTransportAction extends HandledTransportAction { private ModelDao modelDao; - private final Client client; - @Inject - public SearchModelTransportAction(TransportService transportService, ActionFilters actionFilters, Client client) { + public SearchModelTransportAction(TransportService transportService, ActionFilters actionFilters) { super(SearchModelAction.NAME, transportService, actionFilters, SearchRequest::new); this.modelDao = ModelDao.OpenSearchKNNModelDao.getInstance(); - this.client = client; } @Override protected void doExecute(Task task, SearchRequest request, ActionListener listener) { - ThreadContextHelper.runWithStashedThreadContext(client, () -> { - try { - this.modelDao.search(request, listener); - } catch (IOException e) { - listener.onFailure(e); - } - }); + try { + this.modelDao.search(request, listener); + } catch (IOException e) { + listener.onFailure(e); + } } } diff --git a/src/main/java/org/opensearch/knn/plugin/transport/TrainingJobRouterTransportAction.java b/src/main/java/org/opensearch/knn/plugin/transport/TrainingJobRouterTransportAction.java index b37c03a59..774029c58 100644 --- a/src/main/java/org/opensearch/knn/plugin/transport/TrainingJobRouterTransportAction.java +++ b/src/main/java/org/opensearch/knn/plugin/transport/TrainingJobRouterTransportAction.java @@ -23,7 +23,6 @@ import org.opensearch.common.ValidationException; import org.opensearch.common.collect.ImmutableOpenMap; import org.opensearch.common.inject.Inject; -import org.opensearch.knn.common.ThreadContextHelper; import org.opensearch.search.builder.SearchSourceBuilder; import org.opensearch.tasks.Task; import org.opensearch.transport.TransportRequestOptions; @@ -59,12 +58,10 @@ protected void doExecute(Task task, TrainingModelRequest request, ActionListener // Get the size of the training request and then route the request. We get/set this here, as opposed to in // TrainingModelTransportAction, because in the future, we may want to use size to factor into our routing // decision. - ThreadContextHelper.runWithStashedThreadContext(client, () -> { - getTrainingIndexSizeInKB(request, ActionListener.wrap(size -> { - request.setTrainingDataSizeInKB(size); - routeRequest(request, listener); - }, listener::onFailure)); - }); + getTrainingIndexSizeInKB(request, ActionListener.wrap(size -> { + request.setTrainingDataSizeInKB(size); + routeRequest(request, listener); + }, listener::onFailure)); } protected void routeRequest(TrainingModelRequest request, ActionListener listener) { diff --git a/src/main/java/org/opensearch/knn/plugin/transport/TrainingModelTransportAction.java b/src/main/java/org/opensearch/knn/plugin/transport/TrainingModelTransportAction.java index 1f5b85afd..a3c4be16e 100644 --- a/src/main/java/org/opensearch/knn/plugin/transport/TrainingModelTransportAction.java +++ b/src/main/java/org/opensearch/knn/plugin/transport/TrainingModelTransportAction.java @@ -14,10 +14,8 @@ import org.opensearch.action.ActionListener; import org.opensearch.action.support.ActionFilters; import org.opensearch.action.support.HandledTransportAction; -import org.opensearch.client.Client; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Inject; -import org.opensearch.knn.common.ThreadContextHelper; import org.opensearch.knn.index.memory.NativeMemoryCacheManager; import org.opensearch.knn.index.memory.NativeMemoryEntryContext; import org.opensearch.knn.index.memory.NativeMemoryLoadStrategy; @@ -36,18 +34,10 @@ public class TrainingModelTransportAction extends HandledTransportAction { - try { - TrainingJobRunner.getInstance() - .execute( - trainingJob, - ActionListener.wrap( - indexResponse -> wrappedListener.onResponse(new TrainingModelResponse(indexResponse.getId())), - wrappedListener::onFailure - ) - ); - } catch (IOException e) { - wrappedListener.onFailure(e); - } - }); + + try { + TrainingJobRunner.getInstance() + .execute( + trainingJob, + ActionListener.wrap( + indexResponse -> wrappedListener.onResponse(new TrainingModelResponse(indexResponse.getId())), + wrappedListener::onFailure + ) + ); + } catch (IOException e) { + wrappedListener.onFailure(e); + } } } diff --git a/src/test/java/org/opensearch/knn/common/ThreadContextHelperTests.java b/src/test/java/org/opensearch/knn/common/ThreadContextHelperTests.java deleted file mode 100644 index cb1e2ef02..000000000 --- a/src/test/java/org/opensearch/knn/common/ThreadContextHelperTests.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.knn.common; - -import org.opensearch.client.node.NodeClient; -import org.opensearch.common.settings.Settings; -import org.opensearch.knn.KNNTestCase; -import org.opensearch.threadpool.TestThreadPool; -import org.opensearch.threadpool.ThreadPool; - -import java.util.function.Supplier; - -public class ThreadContextHelperTests extends KNNTestCase { - - public void testRunWithStashedContextRunnable() { - ThreadPool threadPool = new TestThreadPool(this.getClass().getSimpleName() + "ThreadPool"); - threadPool.getThreadContext().putHeader("key", "value"); - NodeClient client = new NodeClient(Settings.EMPTY, threadPool); - - assertTrue(client.threadPool().getThreadContext().getHeaders().containsKey("key")); - - Runnable runnable = () -> { assertFalse(client.threadPool().getThreadContext().getHeaders().containsKey("key")); }; - ThreadContextHelper.runWithStashedThreadContext(client, () -> runnable); - - assertTrue(client.threadPool().getThreadContext().getHeaders().containsKey("key")); - - threadPool.shutdownNow(); - client.close(); - } - - public void testRunWithStashedContextSupplier() { - ThreadPool threadPool = new TestThreadPool(this.getClass().getSimpleName() + "ThreadPool"); - threadPool.getThreadContext().putHeader("key", "value"); - NodeClient client = new NodeClient(Settings.EMPTY, threadPool); - - assertTrue(client.threadPool().getThreadContext().getHeaders().containsKey("key")); - - Supplier supplier = () -> { - assertFalse(client.threadPool().getThreadContext().getHeaders().containsKey("key")); - return this.getClass().getName(); - }; - ThreadContextHelper.runWithStashedThreadContext(client, () -> supplier); - - assertTrue(client.threadPool().getThreadContext().getHeaders().containsKey("key")); - - threadPool.shutdownNow(); - client.close(); - } -} From 4c3aec9a9fda85ec5d82df26912303df48b9dfd1 Mon Sep 17 00:00:00 2001 From: Naveen Tatikonda Date: Mon, 17 Apr 2023 18:01:39 -0500 Subject: [PATCH 098/416] Add 2.7.0 Release Notes (#857) (#868) Signed-off-by: Naveen Tatikonda --- CHANGELOG.md | 17 ++--------- .../opensearch-knn.release-notes-2.7.0.0.md | 28 +++++++++++++++++++ 2 files changed, 30 insertions(+), 15 deletions(-) create mode 100644 release-notes/opensearch-knn.release-notes-2.7.0.0.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 5db16394a..d8500233b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,28 +8,15 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Enhancements ### Bug Fixes ### Infrastructure -* Bump byte-buddy version to 1.14.3 ([#839](https://github.com/opensearch-project/k-NN/pull/839)) ### Documentation ### Maintenance ### Refactoring -## [Unreleased 2.x](https://github.com/opensearch-project/k-NN/compare/2.6...2.x) +## [Unreleased 2.x](https://github.com/opensearch-project/k-NN/compare/2.7...2.x) ### Features ### Enhancements ### Bug Fixes -* Throw errors on model deletion failures ([#834](https://github.com/opensearch-project/k-NN/pull/834)) ### Infrastructure -* Adding filter type to filtering release configs ([#792](https://github.com/opensearch-project/k-NN/pull/792)) -* Add CHANGELOG ([#800](https://github.com/opensearch-project/k-NN/pull/800)) -* Bump byte-buddy version from 1.12.22 to 1.14.2 ([#804](https://github.com/opensearch-project/k-NN/pull/804)) -* Add 2.6.0 to BWC Version Matrix ([#810](https://github.com/opensearch-project/k-NN/pull/810)) -* Update BWC Version with OpenSearch Version Bump ([#813](https://github.com/opensearch-project/k-NN/pull/813)) -* Bump numpy version from 1.22.x to 1.24.2 ([#811](https://github.com/opensearch-project/k-NN/pull/811)) -* Support .opensearch-knn-model index as system index with security enabled ([#827](https://github.com/opensearch-project/k-NN/pull/827)) -* Set gradle dependency scope for common-utils to testFixturesImplementation ([#844](https://github.com/opensearch-project/k-NN/pull/844)) -* Add github action for secure integ tests ([#836](https://github.com/opensearch-project/k-NN/pull/836)) -* Add client setting to ignore warning exceptions ([#850](https://github.com/opensearch-project/k-NN/pull/850)) ### Documentation ### Maintenance -### Refactoring -* Replace Map, List, and Set in org.opensearch.common.collect with java.util references ([#816](https://github.com/opensearch-project/k-NN/pull/816)) \ No newline at end of file +### Refactoring \ No newline at end of file diff --git a/release-notes/opensearch-knn.release-notes-2.7.0.0.md b/release-notes/opensearch-knn.release-notes-2.7.0.0.md new file mode 100644 index 000000000..93e505e03 --- /dev/null +++ b/release-notes/opensearch-knn.release-notes-2.7.0.0.md @@ -0,0 +1,28 @@ +## Version 2.7.0.0 Release Notes + +Compatible with OpenSearch 2.7.0 + +### Enhancements + +* Support .opensearch-knn-model index as system index with security enabled ([#827](https://github.com/opensearch-project/k-NN/pull/827)) + +### Bug Fixes + +* Throw errors on model deletion failures ([#834](https://github.com/opensearch-project/k-NN/pull/834)) + +### Infrastructure + +* Add filter type to filtering release configs ([#792](https://github.com/opensearch-project/k-NN/pull/792)) +* Add CHANGELOG ([#800](https://github.com/opensearch-project/k-NN/pull/800)) +* Bump byte-buddy version from 1.12.22 to 1.14.2 ([#804](https://github.com/opensearch-project/k-NN/pull/804)) +* Add 2.6.0 to BWC Version Matrix (([#810](https://github.com/opensearch-project/k-NN/pull/810))) +* Bump numpy version from 1.22.x to 1.24.2 ([#811](https://github.com/opensearch-project/k-NN/pull/811)) +* Update BWC Version with OpenSearch Version Bump (([#813](https://github.com/opensearch-project/k-NN/pull/813))) +* Add GitHub action for secure integ tests ([#836](https://github.com/opensearch-project/k-NN/pull/836)) +* Bump byte-buddy version to 1.14.3 ([#839](https://github.com/opensearch-project/k-NN/pull/839)) +* Set gradle dependency scope for common-utils to testFixturesImplementation ([#844](https://github.com/opensearch-project/k-NN/pull/844)) +* Add client setting to ignore warning exceptions ([#850](https://github.com/opensearch-project/k-NN/pull/850)) + +### Refactoring + +* Replace Map, List, and Set in org.opensearch.common.collect with java.util references ([#816](https://github.com/opensearch-project/k-NN/pull/816)) From 50da0b5d64f8a00e68ea72b5dbf0d3f96b7e3492 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Wed, 26 Apr 2023 16:32:32 -0500 Subject: [PATCH 099/416] Fix the filter_spec file name in restrictive-filter-test (#876) (#877) Signed-off-by: Naveen Tatikonda (cherry picked from commit 50a4b1f933e41558e25a3fd2c791da571c047bf6) Co-authored-by: Naveen Tatikonda --- .../filtering/restrictive-filter/restrictive-filter-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmarks/perf-tool/release-configs/lucene-hnsw/filtering/restrictive-filter/restrictive-filter-test.yml b/benchmarks/perf-tool/release-configs/lucene-hnsw/filtering/restrictive-filter/restrictive-filter-test.yml index af5155893..b1d7b60d7 100644 --- a/benchmarks/perf-tool/release-configs/lucene-hnsw/filtering/restrictive-filter/restrictive-filter-test.yml +++ b/benchmarks/perf-tool/release-configs/lucene-hnsw/filtering/restrictive-filter/restrictive-filter-test.yml @@ -30,5 +30,5 @@ steps: neighbors_format: hdf5 neighbors_path: [DATASET_PATH]/sift-128-euclidean-with-filters.hdf5 neighbors_dataset: neighbors_filter_4 - filter_spec: [INDEX_SPEC_PATH]/restrictive-filter-test.yml + filter_spec: [INDEX_SPEC_PATH]/restrictive-filter-spec.json filter_type: FILTER From 41a16bc3edd8bcff9b0a3c19b7a4f8381565776d Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Wed, 26 Apr 2023 16:08:05 -0700 Subject: [PATCH 100/416] Bulk allocate objects for nmslib index creation (#880) For nmslib, we need to allocate objects that wrap vectors with metadata. This commit updates the objects to be allocated in one buffer as opposed to allocating objects individually. This was shown to prevent memory fragmentation. Signed-off-by: John Mazanec (cherry picked from commit ef44fca43ab73b9eef3d866357db5e1853018f8c) --- CHANGELOG.md | 1 + jni/CMakeLists.txt | 6 +++--- jni/src/nmslib_wrapper.cpp | 35 ++++++++++++++++++++++++++++++++--- 3 files changed, 36 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d8500233b..88a06ea6f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ## [Unreleased 2.x](https://github.com/opensearch-project/k-NN/compare/2.7...2.x) ### Features ### Enhancements +* Bulk allocate objects for nmslib index creation to avoid malloc fragmentation ([#773](https://github.com/opensearch-project/k-NN/pull/773)) ### Bug Fixes ### Infrastructure ### Documentation diff --git a/jni/CMakeLists.txt b/jni/CMakeLists.txt index 75d4ba81b..668ce684d 100644 --- a/jni/CMakeLists.txt +++ b/jni/CMakeLists.txt @@ -66,7 +66,7 @@ target_include_directories(${TARGET_LIB_COMMON} PRIVATE ${CMAKE_CURRENT_SOURCE_D set_target_properties(${TARGET_LIB_COMMON} PROPERTIES SUFFIX ${LIB_EXT}) set_target_properties(${TARGET_LIB_COMMON} PROPERTIES POSITION_INDEPENDENT_CODE ON) -if (WIN32) +if (NOT "${WIN32}" STREQUAL "") # Use RUNTIME_OUTPUT_DIRECTORY, to build the target library (opensearchknn_common) in the specified directory at runtime. set_target_properties(${TARGET_LIB_COMMON} PROPERTIES RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/release) else() @@ -152,7 +152,7 @@ if (${CONFIG_FAISS} STREQUAL ON OR ${CONFIG_ALL} STREQUAL ON OR ${CONFIG_TEST} S set_target_properties(${TARGET_LIB_FAISS} PROPERTIES SUFFIX ${LIB_EXT}) set_target_properties(${TARGET_LIB_FAISS} PROPERTIES POSITION_INDEPENDENT_CODE ON) - if (WIN32) + if (NOT "${WIN32}" STREQUAL "") # Use RUNTIME_OUTPUT_DIRECTORY, to build the target library (opensearchknn_faiss) in the specified directory at runtime. set_target_properties(${TARGET_LIB_FAISS} PROPERTIES RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/release) else() @@ -167,7 +167,7 @@ endif () # --------------------------------- TESTS ----------------------------------- # Windows : Comment the TESTS for now because the tests are failing(failing to build jni_tests.exe) if we are building our target libraries as SHARED libraries. # TODO: Fix the failing JNI TESTS on Windows -if (!WIN32) +if ("${WIN32}" STREQUAL "") if (${CONFIG_ALL} STREQUAL ON OR ${CONFIG_TEST} STREQUAL ON) # Reference - https://crascit.com/2015/07/25/cmake-gtest/ configure_file(CMakeLists.txt.in googletest-download/CMakeLists.txt) diff --git a/jni/src/nmslib_wrapper.cpp b/jni/src/nmslib_wrapper.cpp index 88b29e0f3..f63fd2b01 100644 --- a/jni/src/nmslib_wrapper.cpp +++ b/jni/src/nmslib_wrapper.cpp @@ -27,6 +27,10 @@ std::string TranslateSpaceType(const std::string& spaceType); +// We do not use label functionality of nmslib so we pass default label. Setting as a const allows us to avoid a few +// allocations +const similarity::LabelType DEFAULT_LABEL = -1; + void knn_jni::nmslib_wrapper::CreateIndex(knn_jni::JNIUtilInterface * jniUtil, JNIEnv * env, jintArray idsJ, jobjectArray vectorsJ, jstring indexPathJ, jobject parametersJ) { @@ -96,6 +100,7 @@ void knn_jni::nmslib_wrapper::CreateIndex(knn_jni::JNIUtilInterface * jniUtil, J // Read dataset similarity::ObjectVector dataset; + dataset.reserve(numVectors); int* idsCpp; try { // Read in data set @@ -103,17 +108,41 @@ void knn_jni::nmslib_wrapper::CreateIndex(knn_jni::JNIUtilInterface * jniUtil, J float* floatArrayCpp; jfloatArray floatArrayJ; + size_t vectorSizeInBytes = dim*sizeof(float); + + // Allocate a large buffer that will contain all the vectors. Allocating the objects in one large buffer as + // opposed to individually will prevent heap fragmentation. We have observed that allocating individual + // objects causes RSS to rise throughout the lifetime of a process + // (see https://github.com/opensearch-project/k-NN/issues/772 and + // https://github.com/opensearch-project/k-NN/issues/72). This is because, in typical systems, small + // allocations will reside on some kind of heap managed by an allocator. Once freed, the allocator does not + // always return the memory to the OS. If the heap gets fragmented, this will cause the allocator + // to ask for more memory, causing RSS to grow. On large allocations (> 128 kb), most allocators will + // internally use mmap. Once freed, unmap will be called, which will immediately return memory to the OS + // which in turn prevents RSS from growing out of control. Wrap with a smart pointer so that buffer will be + // freed once variable goes out of scope. For reference, the code that specifies the layout of the buffer can be + // found: https://github.com/nmslib/nmslib/blob/v2.1.1/similarity_search/include/object.h#L61-L75 + std::unique_ptr objectBuffer(new char[(similarity::ID_SIZE + similarity::LABEL_SIZE + similarity::DATALENGTH_SIZE + vectorSizeInBytes) * numVectors]); + char* ptr = objectBuffer.get(); for (int i = 0; i < numVectors; i++) { - floatArrayJ = (jfloatArray)jniUtil->GetObjectArrayElement(env, vectorsJ, i); + dataset.push_back(new similarity::Object(ptr)); + + memcpy(ptr, &idsCpp[i], similarity::ID_SIZE); + ptr += similarity::ID_SIZE; + memcpy(ptr, &DEFAULT_LABEL, similarity::LABEL_SIZE); + ptr += similarity::LABEL_SIZE; + memcpy(ptr, &vectorSizeInBytes, similarity::DATALENGTH_SIZE); + ptr += similarity::DATALENGTH_SIZE; + floatArrayJ = (jfloatArray)jniUtil->GetObjectArrayElement(env, vectorsJ, i); if (dim != jniUtil->GetJavaFloatArrayLength(env, floatArrayJ)) { throw std::runtime_error("Dimension of vectors is inconsistent"); } floatArrayCpp = jniUtil->GetFloatArrayElements(env, floatArrayJ, nullptr); - - dataset.push_back(new similarity::Object(idsCpp[i], -1, dim*sizeof(float), floatArrayCpp)); + memcpy(ptr, floatArrayCpp, vectorSizeInBytes); jniUtil->ReleaseFloatArrayElements(env, floatArrayJ, floatArrayCpp, JNI_ABORT); + ptr += vectorSizeInBytes; } jniUtil->ReleaseIntArrayElements(env, idsJ, idsCpp, JNI_ABORT); From a58de15d8154260f18e7617e8ea7da0f6f7cd8e4 Mon Sep 17 00:00:00 2001 From: John Mazanec Date: Wed, 3 May 2023 12:57:36 -0700 Subject: [PATCH 101/416] Upgrade to 2.x to 2.8 (#886) Upgrade gradle to 7.6.1. Bump OpenSearch to 2.8.0 Signed-off-by: John Mazanec --- ...backwards_compatibility_tests_workflow.yml | 8 +- build.gradle | 2 +- gradle/wrapper/gradle-wrapper.jar | Bin 59536 -> 61574 bytes gradle/wrapper/gradle-wrapper.properties | 10 +- gradlew | 279 +++++++++++------- gradlew.bat | 35 +-- 6 files changed, 184 insertions(+), 150 deletions(-) diff --git a/.github/workflows/backwards_compatibility_tests_workflow.yml b/.github/workflows/backwards_compatibility_tests_workflow.yml index a09d6ec34..592941053 100644 --- a/.github/workflows/backwards_compatibility_tests_workflow.yml +++ b/.github/workflows/backwards_compatibility_tests_workflow.yml @@ -14,8 +14,8 @@ jobs: strategy: matrix: java: [ 11, 17 ] - bwc_version : [ "1.1.0", "1.2.4", "1.3.8", "2.0.1", "2.1.0", "2.2.1", "2.3.0", "2.4.1", "2.5.0", "2.6.0" ] - opensearch_version : [ "2.7.0-SNAPSHOT" ] + bwc_version : [ "1.1.0", "1.2.4", "1.3.8", "2.0.1", "2.1.0", "2.2.1", "2.3.0", "2.4.1", "2.5.0", "2.6.0", "2.7.0" ] + opensearch_version : [ "2.8.0-SNAPSHOT" ] name: k-NN Restart-Upgrade BWC Tests runs-on: ubuntu-latest @@ -46,8 +46,8 @@ jobs: strategy: matrix: java: [ 11, 17 ] - bwc_version: [ "1.3.8", "2.0.1", "2.1.0", "2.2.1", "2.3.0", "2.4.1", "2.5.0", "2.6.0" ] - opensearch_version: [ "2.7.0-SNAPSHOT" ] + bwc_version: [ "1.3.8", "2.0.1", "2.1.0", "2.2.1", "2.3.0", "2.4.1", "2.5.0", "2.6.0", "2.7.0"] + opensearch_version: [ "2.8.0-SNAPSHOT" ] name: k-NN Rolling-Upgrade BWC Tests runs-on: ubuntu-latest diff --git a/build.gradle b/build.gradle index 229c2d11c..82c6f155a 100644 --- a/build.gradle +++ b/build.gradle @@ -10,7 +10,7 @@ buildscript { ext { // build.version_qualifier parameter applies to knn plugin artifacts only. OpenSearch version must be set // explicitly as 'opensearch.version' property, for instance opensearch.version=2.0.0-rc1-SNAPSHOT - opensearch_version = System.getProperty("opensearch.version", "2.7.0-SNAPSHOT") + opensearch_version = System.getProperty("opensearch.version", "2.8.0-SNAPSHOT") version_qualifier = System.getProperty("build.version_qualifier", "") opensearch_group = "org.opensearch" } diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar index 7454180f2ae8848c63b8b4dea2cb829da983f2fa..943f0cbfa754578e88a3dae77fce6e3dea56edbf 100644 GIT binary patch delta 36900 zcmaI7V{m3&)UKP3ZQHh;j&0kvlMbHPwrx94Y}@X*V>{_2yT4s~SDp9Nsq=5uTw|_Z z*SyDA;~q0%0W54Etby(aY}o0VClxFRhyhkI3lkf_7jK2&%Ygpl=wU>3Rs~ZgXSj(C z9wu-Y1}5%m9g+euEqOU4N$)b6f%GhAiAKT7S{5tUZQ+O8qA*vXC@1j8=Hd@~>p~x- z&X>HDXCKd|8s~KfK;O~X@9)nS-#H{9?;Af5&gdstgNg%}?GllZ=%ag+j&895S#>oj zCkO*T+1@d%!}B4Af42&#LFvJYS1eKc>zxiny{a-5%Ej$3?^j5S_5)6c_G+!8pxufC zd9P-(56q5kbw)>3XQ7K853PQh24-~p}L;HQuyEO+s)M^Gk)Y#4fr1I*ySS6Z>g^ z3j2|yAwKXw?b#D4wNzK4zxeH;LuAJJct5s&k>(Qc2tH}2R3kpSJ)aaz!4*)5Vepww zWc0`u&~Lj*^{+V~D(lFTr?Eemqm3a{8wwF}l_dQsAQURmW$Bm$^?R10r)Xd_(HUYG zN)trq(ix@qb6alE>CCw@_H0*-r?5@|Fbx<6itm$^Qt~aj+h+Vd7l?ycraz%`lP%aB ziO6K|F?9|uUnx$T5aqKdAs74ED7SPSfzocG)~*66q;Yb=gB{=6k{ub6ho3Y`=;SnB z;W96mM@c5#(3(N~i_;u05{yUL8-BBVd|Z@8@(TO#gk&+1Ek#oDaZ?RNw{yG|z+^vm zz_8?GT|RX|oO;EH*3wMsfQTe(p6)G9a)6&yM+tYvZwg;#pZsdueT#%;G9gwXq%a(| zl*TBJYLyjOBS4he@nGA-CofFCVpGz!${(Qa{d?g*Yt zftsoLCHu-*AoZMC;gVx%qEKPVg@Ca2X(0LIQMr5^-B;1b)$5s^R@wa}C&FS9hr_0< zR(PnkT$}=;M;g}bw|7HERCSm?{<0JLnk{!U8*bbod@i#tj?Jr}|IcqMfaed&D?MHW zQQ>7BEPK-|c&@kx4femtLMpewFrq`MVIB%4e_8@IyFi9-$z0o48vnBWlh@E7Lz`C& z{~7u$g;@syjzMCZR|Nm+Jx^T!cp)q9$P*jxSQZ3le#HSIj=wN~)myB;srp0eMln_T z6?=}jUvU5_s4rEcO3k}*z#DQrR;TOvZGc03OR0)P5RI8M<#*B)8fYxxxX(I`Dks;X z_q5?sAs zMlaiDTP-1_XRMwL(q5h(W2yvr9HmtlnR);!9>U%TyViU)t#_5B#W0DnP!P#s!my-T zqbgQRIf%MWo*YUK2vXE8RIy;gJ8p^LU$c6POWt88``5^mIqohk~I!a zv-T{zI?eSLajm^r3>inooK|w$a_2H9J=;|sziKGRQ&FC5CWUF*#N6?n4rD-}S>Eg!tFkOpE7otS)$s3hyim=Ldy&-I$%Yra=M3xIOG{Jc zr8d_wbB301%Zy*8ILfeRiGfeQUIh2N3|41xAR|uvQ%?AIGUkdX*Ymgh z54d1)Igp9~)o7-h8AAH#6DzJ}UPh+srx=B^tGe~_(uwPoOov8sptn}$Rx@&$Ox^8H z!MND`vATA1%mR>+iCrV=b!*TSrj2TDv?Fnmj$=uw{JX1c$tt@zIC9gt)3Inpb+Q~= zh0Y@1o@R7|g+n0^b;v#5cc24{OYlnusF0tun^X?qHRYl#m%6UY?tK9vA zvtPnt7tgpi=qBIQ{v=D|p=4@{^E7)c3MLDCNMKPYec~o)VJ6zmZRE?UqXgYj7O~uG z^YQwQfQr>T!u&NaBfm|PW%g%cDoE8%t<-Ma$wIkMS{3sTS+aWpx=g7(+XtaLt9nqB zrLi<%uH29tuKZ6?`Ka5N0@G{F134GZ+6+RnA|Y+wCs~N*%N4CxyoB6?*{>AMy4w}` z@CMj>CaC}<;Y&#-a6~6AB=v2>)b=&t&D7SK6Vc4p+Tfg{AO(<+v?R1IsPA~@FvGJw z*d@a@6bydfT8{(k2N*D`FO@sUHbUIw4kQ(jrMPa2Mjc&~AK*xoe*c+VfsGx$cnzHQb4bSL2wJvVg>oYR*?s}CgoHMPLwA`Km%5LJm4a&OZ3QL*-+4G0t%;_ zS|DOILXL@I?hGl*3JvMq)Uq;%_B{$ipS*Qkn~F!-P^6Afg;Qf!n-zi$tpUjh9TEgk z$Em>`JJ(>S;8ZLM+$-RWUzFrR!@<;W=Y3ASjLR1`U zRnQ{ZU%JK?(2oo+c(5g;5Ez&I&5{C8{!I?aB34uFL`IQg#2z;=$Si?P0|qnfM1VdS zb6@5YL(+>w;EPEyeuX)yIA~VlFjk5^LQ^)aZ$<1LmDozK0cxH1z>q2*h5eR(*B8Pj6nS=K`)S3FLEV-S*4c;F0<9nRRu$YqiDCFaTc zU2LxT3wJJWeBb8}%B59!#)-W}_%?lSsy~vH3%oytE`j-^9*~SvMr-z3q=A7uy$?X& zf*Ky)z&7X0jy`YDtCs@NJw0+j_3CeDw_I25HR6CPV2t!asKPJV^R_r+u&LUxP)wtR zmFA-~HswLN)Ts=7{YPysG?DY))3+-L*En93o=+v+Kjw;_cUsONDZ!zzk{1O05Wm+3 z*2;}O&??lNOe-V{mDB}Gn<0_7H$ZCa5dWoq#}QCT(~h%=J=n@;@VXR52l^?vcj%GP zh7{kjosPu`1x+iQVU?(TJ^?xlT@AS>a?&FMQRTyRO?(2jczyS@T%&!d8mzxqO0r&;UjTNkbB)J1%*iB$McM0+stU%2(C}f0}_{G?dWaCGjmX7PnOq1 zdRr-MGfS#yqMH&mW5BiJE3#|^%`(niIKQ_BQ7xk`QFp50^I!yunb~0m24`10O=`w3 zc#^=Ae(B8CPKMDwLljERn*+I@7u8~-_2TPH`L# z=1~{&_1Fg{r>4*vu5rRTtDZ3}td&uZ)(p*OD4xfn01zzS+v3c_N~GkBgN$cm$Y%H} z1sPjxf=IxdrC~^)&Pvq1^e`~xXM2! zYU)LU02y$#S?v+CQ~GP{$|nR0d%`>hOlNwPU0Rr{E9ss;_>+ymGd10ASM{eJn+1RF zT}SD!JV-q&r|%0BQcGcRzR&sW)3v$3{tIN=O!JC~9!o8rOP6q=LW3BvlF$48 ziauC6R(9yToYA82viRfL#)tA@_TW;@)DcknleX^H4y+0kpRm zT&&(g50ZC+K(O0ZX6thiJEA8asDxF-J$*PytBYttTHI&)rXY!*0gdA9%@i#Sme5TY z(K6#6E@I~B?eoIu!{?l}dgxBz!rLS{3Q4PhpCSpxt4z#Yux6?y7~I=Yc?6P%bOq~j zI*D}tM^VMu{h6(>+IP|F8QYN`u{ziSK)DC*4*L>I4LoUwdEX_n{knkLwS`D-NRr>0 z&g8^|y3R$61{TgSK6)9&JZFhtApbp$KzF13WaC(QKwAZ|peA@Aol`&*>8RK(2|0%R zyo9nL{gtv}osWeNwLf@YG!wb9H2WRcYhg_DT60dzQGW(y7h7|4U*<;c*4N*sE2sdR zZRP^g;h(t0JLIuv)VNY6gZ)yUD)2d)p?eFznY8$~EZMYTiu%DF*7UeVQPV}h zF*|ls`|a+{u;cd>D@%~dRZBn~-Ac+m&Vg>P=3VY8+$<7Zi7p<~Nq zR^M^jl=zI!T`8H(gK0H945KY=N1J#Up`sWvfY$>1SGEfqEyKIokPVbexYnI`OXJF$ zkMS3dBE8RnB1dK)tJbNSu5Y&$IYBy38luzK-TGMpQcEojhte7Xff-zI50I2qM(i2F2)9DdagoKYlK zz%x8sxFf>5@1bI$-n*}N>o3o#^zP{$d7pf& zf*4SNbn9QDXDCVn;wo6|E0$(wBv*pgxHCA(S3lXJ4HMQW)rU}U7?F zxI}V}W~d>wx97Ozh+^glLBo{*j$o`=hK;idHhi4CG!_fG89V-Ew-^^hhMOWUdu-2< zd(t0O>8BgZ1N<2Xi1G3>r1@d)nBD*K3PsmP{s{&G;tmG_!k=7FNuKO+fCm`SxKP>B zK>mtj;Etn5J%mKvT;yE_zl8vk?q3f9hwea!Dt8yLUCgFO*BnS=YuY}-c!&0jb}J)D zV(s~BTYfVyXK<9y&hpVuS= zc!!wNsFjPgspRhCIw6}w^RvLX#?KnhpM(hB`U3x zg*!~MI$JfAFWhsN7xRdV^%0aygs+rZ;dpWzncKOTAa`0Xq7m(z zS_LwFYW$1KXsfgpFzlw7r#2KOQn(%ww?YQ$bT(GWx*gx2Bsny3J z!6UUPr8>TIGiK`%2m`PSS3Pd36m#OIl#SN?$h?mU25XXidM(*ZGBAelMO)H+;9Uw= z8`vjt5)+09c$b2FAWm3{jId9*ui3~Ihbw`9e-2;@?!T%Dqin&WFbQJt4_m@V=j9P* zbXi|lvH3x49-&)RB5c* zheg*i@5p((w*%DOB8-%Yv2P#-IHB%v>`Y&_9BR4)7ngJze2&>4c~NOkQnJ)jt+X$L z9`^6#2vV*K89hV$gu10|zu~;nKfa?ohox&sMS7NyTlMJCQAe^h{9nZwpoX?uy5xO? zW@PBU$b1{UOpv~AtZ#<+*z+(g?Fjwseh8lsxs5iozi*#gI!;qXBt)G~j z9v5n^MQKOT?2!Dj8;SOO0>6f3orwHJiOFK6`b<|b^4}5n{l-VQ?SoksHS=yv3$O(l zK4aL#0Zq4{g#z$jo$*dAJfuB~zb-n^5(3@{JHT~GGc;Ky(^y99NCxW2rZg%U^gIg; zJ%kBn@NxZn`e|BO6V4* z39i>kJU<7SyAHVHI%uKdcv|~U@W=4e@t=p!S?jnBEq^yQ2E14shzIlXKC?om(H84vN=o^2NtMBm7J~D=rmbm*NWjSVJeDEz-N5UmBk5`GjywWp zZ6s1IpXkUutr~lnCT>!2PPR9DIkuVbt|MCCR|#D(rD%~B zubEU^cc78hxs+x%Vg6$X@16i4ob@ek?PQijQzieZfi>E5NEg`76N6^2(v~ar1-yk2 z{{lAO$SjM{aof;NApyxnbEZnRO}8?!fT!U_<`21g+Y&qC_&99r6|*kDkDETgh-Blb z?9T7UIB}thISUzkw0O~5y~+>wtL{7Fc;gSldH8639yf31)qi4|Wq~g>_I0dfs^OGe z!K&|A^L|jeya>y7<>8(f3SXza9%^rl#3_31Neefn#Uk7*_^}IkM)e_&Fg~Ughu3}B zG0}?Kod{eb?94;$6dD4YV>n9mC5+Hy8M_h+bQmvUNvJ>0P#9a~pPDU9l#NrDP39Z> z7R3hA*IMVAod6Yl=s=BNyrblFv9ahxsA&Gst+0`2T@WSesGH1hRhw z#t7Smp){oxPiCm!XedMT9Xls`K+YKLV>+PC>98;G(5Lw*eBS5`f9B8Y2br|#y@jcz z`ddmVevy*mwN3@%YsE|Fsj!mu|5S)>5)wx;dbtMZ6Z1juCz$0kMS5-C{B5qnD{7ViiFNTv<&?w+5J7 zOvuImg^_o-ySHEQGAp-85!m8;Kjq_i-SzRFWcdAdj|VdIswTnUkggogN4`x{jEyG? zQ*_r9na<4wW8fySLr;PuoDVKKN@|y=99HWqBR+2kiH1prFkUgL{}*5_>twEG!W=|` z!(x}*NZ|P}Bf#p=-xK3y2>!x$6v(pYq)(6dQWk)$ZWSp%-^30dq``oVSfEWcTXE)1aMtpTQ;FW3e5ffMASm16(q#bJ}PAM2+l8m-{ z*nkDPH}ha-U3r{s>8XetSzpDN&nlc>|Er_gOMq?H8gtx5_)=$=rKn8D)UFKeitTF< zrA6>w`_sOEN&t!qEx|Pjw>cpv6y3zP58py3u%=88_f1w?Dh6qHi_=ps1{zKT3c+AJ z-CHtS&YwELV7i&XOXFt+doDFc=HdO@cjpeR_V#?~+=e|BdnS5C#8DCu@>*3!I9V9< zW8$!NLpp)$6Dt$s16B6U0ukr;dz~cWFIBq~D_Il@v4E@wH%Sf#P50K?&Z#GHc^JwQ5QyPaJatDTEbA97~OHLu)q6tU>srf)aJKx!w!`g-`+$hp=yl`47e};Vme|`Otn|zcuTh4TQZ6IKVT7?o{08_qzzuC#0N+` zUL{|(2B|=83J;W>uqDA61!wZ8=lN%B^2FGwkZO!2?1c;bDLELF1bQ^Y?Y+7uH}!W` z^`^=K4S@v^Hf0N&e`kde(pQ;BIt`1ze5~`Nn*fETHo^-|6KuqPj||YZ}sKX zV?ZxRbyMRcdpZnDH1-C5U5;4JguMyzlQm)=l~l=@z2)laaTx@kKq5APotoUE)xH#J z6)(ramD2fUHPdL793*l5S06`4Z3{&?tnR3xfYKS3B*A9}jW9$!H?R6_%7X{4+i!*D z*)40tp!3LCaUi_0jXN?z7Y6AEkZ^eIVyo1w;KO5iZg~7 zHCM5Jk&G}NQwK`~bXb=f#j!xIJJ#ETt7@1qhw9lR(hEuxbrv?Ct!{87z|%xN)YC*i zx*N?__cB*&7kQ_BKkH|g0C{L*XHjv2;aHF<^+m0ch@q*5qw}L{NLOF~Wij{R7GRxv zl5Ne^rT$D06;D(gWfiTsBRtZy(NY}48_YzA+&O?{^mT^%=g%f;Ze*H{?}d8=k;bAO*Q1?nvfP#$3|aI1lz{jcLWDIa9v7R}*UUhVLB> z?TDq)NCcJE9S%g0rVmhrf>=Nw6kt8m!lpu=;6aU-%{(-cj)pA`DiK5kE7&tX-cAxk zV7ZG}Y!Ot|OEx!qA%%(cHP{?eqT&8(26rmJ5#`!FG&0ynY|*(Kz?poEylYbT zipX*&ApQikP2)eD@Cw5>GKY=XH&1uQkIwKs&xAMXwn91ntk9#gnYz6e93PIWrmt>FDJ!k43qNZXPf6WzmzXnJHc=iBBr{8^QV3P3jBjzp1TS;KxA;CN~^( z+=W87)Xjkhvi+QF4Lx^aaWOqm(0Y9CO0GFZR8z&yMefP`|0m~2!!3xZ8Lm2Rvv@2r^&{YhR@ zw^UuX9c)b@B%u83iCNC~IC#%5yDEAF)=sG2Ixi3%m!~JwM$*P5x2h-9J*IpQSa~@J zrrr`+ovQAga*z#m7tsT{r|u?Zhxkhp{;cu*=@#(3`WZu}iQhp)>uS`C#CQB#V0r*V zTe2;aKaHbKz)(xpB<;4XJks+e6S0l-xv_|GDdg@Di2SHte&&#+NZ(2^BxzTs#s&{h zT+P^yaLR3Ngh&SYr_pGSlo1CA2wot^gmLX*Kry~2|D>4C=?)BOyuKoq!#CwNE>=xz z@B8_S`HEpn&6xHL%`uv=rD%h>RB_zhRU&TJz}mn5F1e&^ASo;(3ppRY={cnp``a?A zC0wiV5$%pZ!_*FuGrqYzT=2e770vS1j+=c~|zjkE7i4Y4E(NTKXd-je8>=6q<+#B7yc*NLp6Yi7`s>jG~xBpI-ljN3WLT@-~ z1>TEAk)dHU%i@jw-oY^D2AAb|%)}JjA7Bt{nKOF_Hp_!A9$XYm%X^ ztmK?aV&I-7@30n?X3rXfNuWHp0#VN~t=DRNoaeHi)w&{-K@k@5vgoq(MtF*-_fe2= zYChH0%?FP}6|_HapKK0kzEY{&1ar1-#X(o*HA;tY509Qp>zLBfP;v#}!^mV5J)dZ^ z>BgG%+gA^6~) zZIvs|p~pM!mkV)(Wj^@{;btztU>>X7r>wpDwmCLZ-ovAvPh4@D&-`&>!9aQ4ozB$& zp5iU5W6N}(oJL1>m258VY_?OHJtQ4roUQ9xnhBhaxRO?2T*pfCJ;?Y5nAyb%ZmWeQdtfRjFHZ{sZX3=>dcPZA7K6U&rrSMJ3 z23`Lst@rcgM;A*bOBZ7^yX5>5bBMmNiu{;nn9^8K@J#x?!{n@TH!x&BoMx1Y zpdS!C^i-FX$r+VWfUDF)D_ay~adG-ZLIz0`K#)}p3kzvR0rp=Om7M8tl78YAV0KgX{bGW4+cEG<+t|p2oXOxm#xNQfN z8f%1y6(O6G{7C}RnVfKJuiXZaj0W?HdU$68{-jOybhcswAmTI)jig>@#_t4FFbU=& z)3D3#bDeYZ26=;Z?rb?le{I}drsj^85p*AB*D=t(sbAMU^rLueRZ8e8j2qQV1~Fi> z8hYmusOb@gaqj3$`75=b|ETY1Q+Fq*KH$RLu8u@?^hVwkzBUu&NT}LcfTObO{CffG zsFXYPCekhefLbLr_#$o*i+-Y*PU)i`#x}$R}_=G*KKA8Od zg?&d1E5yBkIi!?6gDJR}d@@sZwG!db9)PIXWr=&{#YBo-o^KfC-w7L=Y$2_q5tA_s zd_)K$q}9eV8#$HB4v)xO`cRrV5M0lbBS^BQ?N_Uyj}uJ$8D))4`RzrAKn8@Bl20*K zK?_9(EL!7Tu@<%jia$Ut+x-QJbj1FEus=kWHhxabUvLKbdZYo9sf_2ZyUzTtQ`H9634fzfh{>IZs*n7#nJFjd~cRk}k{P;z%|sOnYp)rqs0 zMntK7EEh?ZW;Dj{ezME8Ko#w`;YZB7WQfu8Cl3?Ixic3l%&`v9SfHWm2pdd-N*w#6 z>pThQ1uF0rDpJ1vzbcK8Z)NAyf7p9L{2y_q0+dc+(u%0J1ZfqPj;s8HrXflA*Q%+? zSWY;#r_OEyUMB4@+!+QYb20UJ1&W~+YkpIj`Znt-)9V}-KKM^_-T2*HO#8n*e~|@< z*PKcjON29GAwVEB^Quix92bUpcgU|UHxv~9a~In6`L>OeU`GfbThFhw;fLI}TJzeF z0G!n|WK%ep~kHJws&s(en>DFZ0)ld zbX&L4=&DqT55oSDXVOUIOCNtJ?&o_+z|RdgGV~cu#bIU7P1)FXPox?Pt^Wzf#Uyju zHJ-wt;Q{pYCwybEi&h!8>!GxjB3=MYmJsd7{?h#Zb#sZQCgbR3-)Ak*c5Jng=kai# z@B_>mOjhgPQ7~?18moe?$->ieFbaQeT=5~Jd?z*=lLj*#XEpObnQ3^>$2tY5G-}a@ zEmSX?WSoC1&Qmzkw_{vO&V@N_n)R`16?m2h8z&f4!ZL=IT1Aj1)01Uq2tWZO5y$=s zaORP;**KR8NS$#Cee%5<5+F>(+o;+NQrr(r-VaWFBjbZZN76SSb_b1o zc^0aIX`Kg^LWGJ>O)L_3w-hi3`3e%|1sEYkdcfy++pC_P2+`cQV&+tAkLXej;;z$0P<*&mKBafg$S*@#Iivr!)FZxfykAAa& zl+J;luT&!5ym{m^r_*pS9j1jMnop!C&aB@CGMetbC}E6!cJ5#tE)p{Eerq_dc}p;( zrX=B=qAHr%w2o-7rgx<`E+s|9@rhVcgE~DvjDj#@ST0A8q{kD=UCuJ&zxFA}DVC+G za|Tc}KzT+i3WcdDzc_ZvU9+aGyS#D$I1Z}`a7V_(Oe4LSTyu*)ut(@ewfH*g6qn0b z5B!c7#hijdWXoSr@(n%%p}4>se!uezwv4nqN+dY#Aawu%=d-Rn+zkJ-QcHv4x~>H$ z;nl83-22HjF)2QMpNEM1ozq$th2#KRj5s^@lA)tHO0f36Asv{XHuEFwPv8h3aVTxQ z%oEW6IvV#QJ0B;vgw^Hp1Px?Mz2A(2dQ^;}4MsY<8eV>fzO;Af@2_ABvNCN&Vi@_$ zRA;E+5L+M~+U^kL3Cv6VGRI-YP4;A4S&FiV_IwHwRVdRsZgQhV)RgM4Ma^G}ULm!> z8q`CgL(VPvlGhnd4Y_Q(w#EU{=fE(mCcuyXqOz6x9k}xk63wR%n2?k=jbfx8KC{_QVW? z2ys94)HvxzFg3~`E+&TzC@%OAsX|h=**G(r1*OP#MUZ>t$ZBnnJ56m_n+*g-@o>wMN)L+r|C7%OU{k&i7w!T&(lEg>(Lm5?YI)Z zMu*56HN&c15ADmoxo6=V1AoJDxTx;8r_dWba= z34d+4zF0+J$*d`EgH=4aGD~iWMN?r-nPLgUypU3y7jqF-rKVVCMolJ?vXnQCHq3E? zygp@tR;A8@wwqP-$|X$GqUu>re>O?GO0#leqeF|PxrbFUnRX?&+9UTQ^-bmx!a%#? zHr;DWVKXE_Vk>kZU zv>7s5$dTD>2U*zg;YNegvp*xjy`Rq?-EF}S83Bmx;bgi)&qtF#*)1e44g-Oe6BOHb zLCMn`&=S1x^%&^OkftmS_H!DNy0tXtDm$oL#m`o9$?ic5tK&QaR`dqD8&VydP=hmO z4eNH1Vl)1SSv86{1;1>GZ7eRkgcGt^oM^b@+S81dqf)DFG?wjas_XRIoXwxA)TbD$ z&;YM#{~CaV6{j&!q8Q4}E87~4tjOhR`yD|jD7xz-`qG4CixswD1SJ!dNNr(YceB(S zdTBg-bN&brgS8l(!5vd%3#(D9Rs}p}8tkD#7%)3&P(x)5m)j6WJgmsD;%%#t?U^$$ zt}rR)lG=wjUkB3_m9)G?t6Pgk^z+!P)&Q}&ZX<4NL*j8pdJ{Kbnpl=Rg^*{}#rC$9 zgeHxM@YlVRDsc-hGD6kMZ~@(KO!AY7e3CkQJJ^eBC4qsB&hMFE~sc=K_u%p7dodffBw1U*#b6=_ylpuw)MUa&2g24IPnQkKD+p8Kjt| zBrA0e{WbCdZ9sUUwkn@$zfRSJdC;+_fgm}R!nrJph!|;r$;y6jNTv>VK%(mFIc71& zbYEKGXaibyqWmY@Tk{fC;#Flu0igd4Olz3+NBQp<*MZDTvWGBG8rigCLOH%o>>M6OIYwohsAYg2z8B&M~f7N=iLOPie+-I#!D&YrLJ#*|r zk`%QWr}mFM^d&^%W6EKt!Jense)RQoMqrAg_=q!e_ky9mt-vXrEWn`?scHMlBa@%fis_I33 zTO#Cq>!AB*P3)GH3GO0kE#&p6ALzGH1785t(r5xFj0@C83E@@HBtSSGZ|q#57SXzC zBcVYI{w#qZOiY|a25^Fdny!G``ENdD%DlS3Zk}KXPO%lG*^rJ-*YoTz0!5gcbUBIU zcxsp)g(jX$tR0mbI%5n51@)hFEWCS&4h~-C>z+e9XP2#9L=w6n0&{JJOi_tKFjBOmkydTxF?{=r~Z0SZ zQ!+?)lb|XW*a39dgeKjifBjqg6C6^fO>>mhlO5^a!?k@%Fm%OcR)0o}*qm6=$;a85F~$*LPd>M4+h=KK^p< zUTLr~iZCJ`#!sTSSP?A25d9$@jEe9}IiHO>I(cU!JV|?&>({{a8~_Oyc02#bw!fyZ z@HrqJOcWp<_mvL~UYdVG%AR6M@$eurF>ywq!qkU^T{D$%{9=rQK{Mr0e$Ev<4Z5_S zNnwMk`o5QFbqF(j*?kTXXP`Tk>0tE2420%Wbv=sgM}= zFD&odG<``_Nk$!;UUlNa@pUE;@K9l8cg(6Zp^76 zHSY4thE?HEz;V#!D}=e137fguh3sSu$@cn(U(I~bzJ+UcXJ=Q1O00`zY_m-#grEj4 zEGB@jzU304JM9hH$ewewKoi}a*G)7>aprL9L{@#&E63^!f5;GKKdIcz3u zIX?;8Hm+myU<%}TY{&)aehJtE{bUL5REqCLEv$}$XOuvB|LmWM={@UM30}Tc@D;(g zGwu3b=?d;_K`#|5(k3D+azz2#*`b*#(L%u7Pt3A#1qc<-_e7jCTL6jjvyRPZR?)zb zWgFrXi*Z})op{VWcX)K(M?p| z^}a9&&u8|iSNZT&G=-;Z1>0&GKleLMJk=huD4Vlz{zHe^OpLbVZE?7JHGRxRVhX@R zX#DjtFQ~S{-S678C8X4#M?IY@6Nj@YeQh)P53f_5{5@XcsQhQG$hZ}!=|IIsPG@-~ z_{~ws>hNg`<7R&15+VS9kG-XsFaWQ-qAIYaR{NtS)$_Kp8Ny;9bOV?yFjO|C|BAb1>)p63 z4?AKjs4JeWs^@~NgVY^gp5av^K1B~{YF7jfwz3uM!~O04tZ#R7eB-b!IWW%tVX4NF zZl~8XZhad1Tj?)(6C#PG6UgWf`0A^X+pq%_o&XegitvOnypX9A-jKwgoqIsk`7vDH zPz9}L=G;#3Lf5f!K3`t}l&J?TXKzH~Uzk?{5_k9H9xWw9crd@!v&1VY zsOuRn#7S^4j73)ETazCqI7bwNo$t{cZ&ry=x*Xgs76A|6USJp|n$Y_yB zDC2KGY3x!h=P8)>V7&ntYvVVK`hxw4Z_sN~Bp#BR6^2R37pGT z1Dj`(PM$x)t^Bc$%_kZgDbs?_&wIue+uUzpy}>uET;=1A)F*)A>Ata~GY4hAc!A?U z?{U63R0JMe536-g^k(*$`+N?+OJ(#XPk0Vrn^Rty$T*_`6p2GBZiWkJ{>w7+4g|H2 z4M328#NL_h?{$DR4^iA=7M|n{ahQctX<$tp*M$UZN+xz_oI{cx8*`dJ7 zuF=LPSVu%73wwaH{>HwHrblU4zy99llp3ScT+Mw7rR)7PJ^rA!wpR1f3=q)%h-?9K zK52(MxZVT~sZMJ~do{4JL-m{KI{J9x5!DKd$(}V4$Q5i);pa(WYKq|3lh&(wpC>*+ zMJlvE1NX)k5PT%eqpH=J7er0}#EOfJJqW;C+V(XcP_4kkIdOF!3{~9L+ z48Ix^+H}>9X`82&#cyS?k1$qbwT4ZbD>dvelVc$YL!v08DPS3-|GFX_@L!9d*r0D=CD`8m24nd4 zMFjft2!0|nj%z%!`PTgn`g{CLS1g*#*(w8|sFV~Bqc{^=k(H{#0Ah@*tQgwCd0N@ON!OYy9LF`#s=)zI0>F&P85;TXwk#VAWS+GnLle5w zSz<>g3hqrf#qGfiyY=*_G1~|k*h-g(AA+NbC~N@AVhf6A6qXmVY2Temx2|X$S0UFw z%*D3^qpS5e`ZtH#e-p_hv3bYtz!vUA56&MBhN4*snI=g8YNZ{TYX{~dPZ=Z_gk$3Z?0ZR{D-aliB#|SEnR`T;N3$!}02ZQ(F`K#y94FLke@r>i04JrfBacpWL!tC&p$j#%e~c zG0Oa(wM# zM(Mn!CQ&`w@usAmfZg29h)&o{r_NeX64w5N5WxG6q(-s6n3+LYQoV!fQdogT)Mf~f zrQ*(MSoLcIu2Zpl1bcHm-1-=no;nuG(Rr?&=9Dia+wfu8KmGNY@a~FBD`eM%#b5IC zn=aI`v<7i^08qgeb@EmZ1l73Fe^)VHH>vwnl#LfZYM}d!X*vZ=X-Kmm)|p~g8rR~7 zTHpjqRDXxKte4N;M7->5uZ?~X`;`Oeoq;87kGDaWGMa(5g9dgC3{EpOF1o}w3Ms0+ z270RrL{cUBU0=kwNClDNSwY!Lm!3n$dY&svjk#S0d>tPZn?&G%Bdtl_HV)BD3T&C$JTZ)yChEr+){ zP!q~(%s;6J22$ep1;aq;vT%}A@4H_e%j*18G#k|8R4HfuOLp~*H8ydsM!zd^J6-{I z0L19#cSH6Ztna?VS=NwT9B)9MqJAc(Hd_EwUk?-sA$*+!uqnSkia#g=*o}g> z+r%Me7rkks(=8I_1ku94GwiBA%18pKMzhP#Af0}Seaw|!n{!*P9TQbotzCQLm5EQN z>{zN@{lSM;n`U!Q*p-J1;p{VH`75=x^d=n#jJ1K1%%tgPj|GD0Xz zq9fV3Ma?HtM@!DivcDoBi|RXcCu&(8=pz_F%Qq#Kd@NT0|MtB&yqr?e&x3@7k^qX=q=oz=wvkChK5$_^jhq9 zhI+$s(bJ#2(25kdPfP>T<$A@3xOU9Xu;*O>W zPlGz<+y;?kBjzc;6Cx`rv_6DV)$7dgS>VSX3u8DBYT4@c~$tokVRZKT>AAJcn zM`3)eO!3jw64$ia2bI*ky%;JvZAew%gfzr@2z=cx-FW{@F2|Z2yJ)(40FvA_tyb$4 zHp-iN;@m7h0Wd7=&Re6T*H*wT&g*@8FgUyIHK5&0SUQ1)UCLemXi3}48~TLSgCCyk zrp@aYZmn?H^Jl<7jH)47mR8%{zw5cawx$r(oP>dTGqsxPPP=R8-^vbHS!I{bImH+d8&wJ9%Q;wmq?JKe27wwv&l7u{E(hv31^a>U`O|>aMzfL3gd{Uh8TtBa3!a zM{Iu}AI>-WSaizNSJ-FtewydP57^1>j^mNBnaaxoQn&p9y9&-_w4i7^xOT?7NKl?lKxm79T1T;#zGve! z^z&y}PFN96@n!`suxGzHHb%{=V`PLBTAb6YsDu-M5z|b*X1U-HtKvIeCp^%4PTA_v zr^@B{_qoGaW6!xov5Prol9ez6kdqH&(Vd~>o$?gruojX(F}osv#OuA9XCm{BA{HQ6 z7I#HXLktMs2!{a#?(wMAlBNdNxg}5ft0q4}Erg)PFo+~m7-_8kEk4%&n`n!qprR3_ zRKcyO67pN^HTAedB<#V{RM6J$?2A+0nwfZkx z)#H~>#TqYNMDy~b^!AI9>aavY_!YH!u%px+~ zAR_r);-C5#UfvaZNPmjHSuC39+iWbb>#uq)ntooMYNm#v%L5gx`qHNM^>O%V(&=$_ z)SkW9)C`tI#lQ5oYR4|5rnABn0GHiGa>kIEA)V)lr~lGU5$|u7S!kwV34&t z#Znst?`+H+{F>XL5Ihe`v2bcY2LZjt7?Bt^Q*1(5Xcp&jtGCX0X8@7GN*e>1pKz{? zTsY$-TL0JWaic5zP>F zBpD0yg8$LFD8iM^) zk-SPvJ|)^m$UbXDe<1>130Xcxq=9HeXVixa5li>o3bOiCmS8->t{1==s+|s)1#Fxf z`>r33c=P^?sE%sIN{nLrVKP2=8#A#L4aVF0&5hX+277!PfIi#w^-B=A(-v7xyZMmjc^*yX$#oLqK zZ9ANck>T6&l`fxVTgmj2FMyTGi}%N@9p_{)5@W~|eKY+}O(1Eb@~8MeO%U*3OJV&~O!Y|BfsbcWre3Qam04<^Ox8b7rmU*W?BC?5tQ&Maqv&(zE=o#*zFyM3A~aLQx(BIxtIGzX$s zVzx&kS;C&nIUnJf=0g?za@(IQ$b3sWi-$AZ35<7zDuzQDl|s$cdI)pS9|?_@L&YG= zTz1|NMy|(^-ZMSEMkmyA*Ec=8U#qiWonuyZ>vO5Uib@8!;^$YYmuBR+aS?1{mN|pv zw-8JT%`sus&h{q!ics^;33&wOgzyRooPenPBHseN0(uMGO0M=K4B# zfGQ7bWrup@w+0D8zuXDVG3`|9WQUIU2=lfs0}uW&$pO=+x%3;BTP?egh9}g!y|nxQ zF7c19A0dClYKuSr+0{^h;p=f9Z}r~jC}s(xg1yzB|3z2;`K_IX0kqq}KEYNiMmwrL zR11gCd%Misw-RpfU}^|g2}g%6#Etdt0G?#sN0(*BU)z~$KoK{Kq`9iHM72 zx#?+K`4Y8`;N;NJ+f!qAkK#UXrFMqzBWj;wJTv=9yxWXYj<=2W?S}YbPJurHi zQ($FF9S}jGm#Ch5G_{9=G&4K1rES6e)EtmgOi_(}8r`}~fLVtU&2@>eeNlYH>3oCK z-!_xrX%uzAB(J7fGqJ$WVfFlaX$_^-S(u6ywL|Ek8l5*sT z8D9aA(LyK~&|Ms@$?%C~OSUB8zJuyoz!y2nEHMk4VjBmJdxc06{ee>417r_Zx8M_f zQv&2&0cujOd<5@MSTY9gXQR_E^F$=~C=15`95Ht{YHmdLk$@3n#NUOMK$};s*lX~Z zj-hg?05PqDKaXM*=@C*FUgq$9FSP4gH_)(EMoJ6Vkgs{7exk&Q6_1EM;VrM=HLvKN zx7hNZad6+T$rH*0HD{xnW|(A;fL<{)@*L+A~DI2+a&j9;VV7>2~< zOwYgnm%NW?RDa+8Z;c&Dn}UQ!4V=-1_4~gI?EYyNM=CB-ToUF;W;(fN7&0R;6*M#$ zvq5<4o!#$u zL;H83)18fEmc^I%kG9Y0u2a8LzSGT&l-IvE1-?m<>GyN@RiOc=MG0pwK%(g}7UrlR z%-M&;96}o7L1r8apQ&v zS?_M`X_R4kkwW!jor7h&G=I3cyLo=WiDB0_Gi1V3Z<9=>`A-w>Q89bJ>Y)nS-T|=~ z@1h8-J2K?H;h0g6ESyOVVEyg9o<40j9gBKQkt9MJkx!1&%PpEAT{s(tVflR)k?!o2 z0mU~aI_52$;dv3)8$;S9zy4g!NYM&dv+h1r*xa)+IiI?ql;2upk;*aEok5LD%PUqS zz8;1l^|}F5xF(Ao%CIC$YgCZ|0wJ6yU9ZfstHAOwKs1ms4V(xMc;b-etG-ivj|D2A zWYxMR_SLI#Y)|w~S9~nxto669sc=HX zbX$_ZzOwkuE=C*zP%=)t7J$QsNW$t3`nShXVT*uu$f8k+iyTDp@_c=Lp{vaFBc^0&k4p3rk*Y7Zi_uzwrjSgca zMtjp&+ZrhxKyKW{K)&dq@Gfe!?G-`-PBLfo;s&_z5DRcM(+!N~fXTq|3O~PQbs=qA-pTg2l^u+d z%ds=eY1sNyehE&1F?Kp*1nt?h_p`OIU`aFI@{{AP0W(he39BQ}N&Fxr(_Nn9C@|Fv zF2CjVJpZj*KW06pkPfYefvVkXhPmEzhB0ZpvW78P+6b`(DXmx4XD$i@yG6uVoa7U_hH3k2Py`({xw)s6nAe(f(@W-J| zz@YAV6gVhtFUM>qy-n`}{EY%a%Z!g{Uc4KbHQ4Cysq(A?;rg&6Xew@Z;N+ZaVY|*= zY%CB8ewT@Az-G0c2It&IF33z$Exgk%iGnm9(StB(7KF?4q@06F#2&%w!1|s-vJ<$R z#XzNy)JYP=0BaD~u#sigQN$gNdTInmz#5sK4BSByfA_#G&)Zj<2A?Bk3$T_QnC;|2 z<0|qNBOdcGWX_efUbjcIbf9DLA2^E&r#fq>Gu)@g=vUoWqV-D~(xUfMfaCeY?ig%5 zNlo{2#2{?+Ykm2};*J1&Ep^Bz&WB;0YXN=I6)&JUITYUOUDcL5p;6b?izK++B7%r5 z9mr&h^fGbKR>>e`KebYXfs9w~PV?6xQw%lJOA*R&83!gvx2_G^Zzl1NjQ*&uWXlIJ zA5d%t%)`R6RVN`l7|hlJO0zti;vgD9yyKBh-oiXL(LgU}D{!LToK9roJSM_z=}gA@ zV0mkG5=+m9kztd>9U`MRFOYqw_R@@-88|~TY&n;wx0Y%6<;}H~Vhw9l)<<3|O$g znOS~HbBeb++hP5w^R9fzH*%%;O@OyRJ2HQ!`5r6TvCxLMt;lTth4BYout)}a_|rR1 zP|nlJjcdDbp~VeGki#sSoP(U~1 zzvfGSEi^1h$ayZla(pu`eFFiu-MqSdt8cz0qRmg++c}@ChaW9!{X)T1I}H&3h$C+b&J+B z&WGhay#y)vpbmts^9+1um2a^f=rUg9gc(vaIvdu9{ z=g~Ari+YZ*_9#%du+x0Tj|uG&ivk6<0W0(z->5&_@J!xrKJh+-N7(ay9KI1^9DKq1 z-`Q>5RXJWR>^gJg=ceSH1FhP&;-(b&yx3;%21tElpT5B-^B5lRW1stx=Lw@yl4K-H zH_&#(_w~Tx6OXfPTcCLo9$$?1c^Nx?=R`f{P#LiJu7|AN{H=1s9vgkea6`f*yNy6m zELFO8tlEHRx_O|Rftnf+yTTazHib2IaSS}hRg2p_EFj}MmiDQ$RqH#OP&*!>JX=+E zhHHTXEmdmJGX}fFret#wSWMoxwfs%78tQ;lJ+%#EPSxrJ1@y5{w3>3s`&VRTmheQ7 zm(`N@=UL#bJ3J63M84cI!+dq8*0Pa~cm)*vOH>96OZZ8rI+@#sxvX%J;j#2UyoI-P zoHw?w+>h2y0-i8E=E{R&#ky4YXy`dpzp?LN@i=(bZ>Ps)txu1NjX9j_ZqK;J7FkwVRy|k|*99~?Y z`*dy80oA`CJ_$tFQGtxLJfj|?%k{~!rK(wP%(jJ&e^AP#2mSmhEOc8GXcC^~u~)IG z&bB&9qn$v@0V@7Z+WqyCihnp!(NDz!v+(tZ6+efxni(EuvIZgq!%Q;IG-q zqF8&i9!)wS_%M!tY{yK|t}-+MVeB2X)^xwo4U+^n6ZT(3n^9s0^N~ZpVA-p-|=@^inh<~GA#G0Fb6cqg`G}K)*o{T5?_kIK6JI}m$v_ol&8oO4P_zX{TbEI^ zP4gy_X(a!@XOe=(Mp}U0!7ra+gbWnl2qGN(SI*+{5}&-NnMCpgbIjJJMM#>k=g30^ zDbJL&s-oi`3YUeZ9y-BZu65hbFPz;5@(6>;XEhacr$vW+pjdI#rGBriL|0cF)|$5S?ZhrZRY7Vy{kdqRI7&X0dtGtm6}Z)oRm-4;l8Ds`lB z1{;=7P~qZ2_n6wIDqX_QLr64UbcGnv7W5MkBQOQpPgUnUuZmy*Y1;{C(bD+H71WwI zFxkY4N6=#*ys|B0K*aJKZ-tf_Feu|x0wGE^{ za6HB=IjXDV7hj^UMqY@8D*!&A%+%g?A)#u;s#rUkuh7i!inq{PbR#Dr|8ZT+Wh(ZI z1r+upwLB#jrdiBGjm$~v%G;|eT(?4SqN&z(RF;+MW+&TN%T|}sR;8Dh>e|RrS`1xo z;obvgl5Z|wz0;94M2z-Y2WT6-(${?#QL}TPndp;hQjRZh6!1&D`+%7IvJc29LIBMq zvwi(+IZ(P1qKSTq#x08<=kru=S9oc!%gVY%A{T9{D%p8jSYCIzFy$TV^U4-RLFD+w zn77r`QwzNhX2Pbr7lOF`qlaW1HJk_R3Xg`iqZN?BZle86?}o%OyRW zEc|gt<9{tSk0Td&`c-N?)$%jzYaJhoOAjaF;6Z6r1}Rm!15{WMTw!4o5~)Fo-HoU_ z-&ujRx$TNix^SgDySgxKt>YCrB`EyID}h2#B6*Zab@La310Ghd_ma8AO#8-ulwSnj zZ<5BIUzZE;5*FP#&vkvaG!H~2tU$Jkd%gFw`T!S{2mp9?Vh1R?kv;~X`YAwb63>)? znkAD~i^l250{N2CJV<@SZeNTq!pqthV6F>e_QO<+Mykoxd5^JzHJaZeQZ zhJkUxQe7WRdWlz!MRJxF0W`KL@`p~)x5J(z5M;XocV_|rgnnd1%sW+|yq!Q`G&7GP zY07mPEwX@!LGr!_kNsDN#hMPL7#l zlc=pE5aWH28%^Dr5#obbnK@SMPeMr&YC`p^e?y)lV?@3LQVmf_yWw)b$Jl&Of#Rp# z&|KH+IbPYoU^~mj`IAFEK^Z{Gyzpb8*3I%bzXzl%M=>mC%Q2%)jr6JJ(KPB8q85*d zB`H_bk5V~4&VPE&gUAO>5~Zr82#kI9vNGHonE(8&8C(Hj-eU@GWQ@M~+4I^wF?8-BT6Km@x@%lir9`u3T}u<#oKmr!E| z2--yCX0m;Giv$T$>#E8290L1S=M=3CD`(J9s?1X>SX6lZ4GocaWFnHAC)t1T^hkf* zUD3KeM&diP@80N9p%T&fLe$oqvOhhZt`JxBO+^LSf?Q@z_`9Vr$Q6~<0L2-m>O(g4 zOan%-sNta~Xk*}&{@r#)usawmHs1u<1GjQ|b56{BDO&snX)z?_ zAankXRi*W~FHQC%{R2T17EVv=NN_~B7>6qS8-oRfDB^`%jRb@OLn=Vxce}tFY;7n@ zj#*voq%N#N>y$Y|*HtC2U!S=)^IxgQ0-7$v2yiqNXRM zwteC_-%jMY93pATf5JRZt)5Ay&cMar+UEM%P_tH6YH%!8xM83G_bjXj(q~&xt5EB% z3%t+9ys%^4AWWnRiJ*K6xjY*LNS|#O;pS)*K=AB^uJVW_JHF`#iYDK!(>=WUhh6%c zX>sTwaqCCJrW6nIY`0WWbIIb}bAzF+1oH!VTEEkh=Zo6npGn$x%=adz9iX3#tW4ZG zd<(6Uxn#z9!I5&G|DBlUn~4sC6q09u=rux4?hdLGj!_7Cw~W?;w)!zdM>lGL9?iJ}t$XPovsz-)cS-!LHv0ZC zb4AsYLrHn^FyZ^K^RfN==H_K5|Kmms8C*LII4c6rK%~mwn+cs0!Hx`!kJU7zAV@+T zY78x5H8b;aj{WU`xKGLdJJr*0Ydv@5KHQ6gH)}c2!V)JwlsWfdsGezcK zvNM+<{?KLS;}dCbka?fVSkA4*j<+1;zd^mMTl-!=UrG}%Dar#cYGiWKt*OnI2`}s& zKuJNJ^nn0>uh!6qs230jLkzPYLh2_ii7q$|O>AsUP2s0Lrn|+I5<#4D>kLax=_gwF z9%;kCQJZOVwWh{(5l+S2;i@c9Ea^@^d5H*?CXc?hq}byCKRwrA*C%v%mfkhaNtGo( z6ZP->A4&OCCWA#*#FO}#W|pFnPK7yjF|1x3zOLK4rW)-`{Id_xRgaYRE<$eQ5uvhX zwf1^~0@8-xJluw=SU}u}Dw6aJ;q1JO9ug~KY0 zc4j+Rx)`6g89&yl&N%L(+7`jSN#4N90mygg2v-%B)UllG#o_hk%4qb{}DFugg+wjSK#BF}Y6uqK(T} z?kzHTS{^k4!@fD4XcX#W(^8wah zxhMD99Ne&1gVtZZcgbC`hyPk0Duv+(pFsD@Nk!o&HRyRK5G1T7+eQevJC6LPk{?9c zQ-J=nD3qA?mBsZ7LMZK)4N_>F2_tu$3G)*!f%X;15m2(%QTyX5jbibaL(DZZ?^X)6 z6IQe1C)xidS(*m&S%Nxg6*Wvr#c_5a;M1(O#!UP zK|w*!f?nnepYPN2Q*1CL6QwdI+R$^%?Xi@THq}&u@#=_#DZffv#+TLtqCOXu9c<0O zBsjTGdF-y+Z@mK*MKeXymw+sY=m5iC_W;0f&xoJ>Z_(Nj$u*A&fs%=i& zXib;4XQuQ`Jk*=)+;=g|>19uWnY|Fm@!=U93(mB|GesI4Wr=-T+cXbcT)0}e zk9@N7!pP7X;)b3=9w&;zB8_zwDYIgysR+6MlJV2JZgTIABOgT$H7|24>D8+#;3xzh zyKY%iqA_a64CM6~S%7)I77x*&ho@z-+9T$)J3p7ZAAvXTlleQ)85O-Aovu)#(nBFp zlZv+~J@s!EXPC?AV2Qe2x8xWM@qgW+EK=kDvM;^m-$jX%#8X}}_^WbZAFz~n4^?Xl zj%R5)@O^*Xqwo3nF0=1jxhKO#Xm|5ZH%Ot*~o~Quw z_cI`0zS0)qV;eDMqE&yp@f(f!aI}g#JA3@l8p?CR&@Kv6EZIB?Qasr@Gt@Z{w77Nv z-U{;yNYdDIL049ee>V>Tr3Z~994}6y+LfVe( zL~*qRBcjeUeu*d3^?P%t9mHjZr3zcH#b1=(bHZuj@nb&CSkplmQTCO5-ncOKUr7>~ zXO}(#MI0}p_XUBw9Z{>_&I}hoUH;%ATm@}@Ytb5^tGOt&!%kKyT~|z0b_-_?RCARZ zLcxg9h%d{=k%-3K6b}W*odahEdv~P*`guGU=-EBpAXK}9hD!(mCb7CfG)h!eG^FI5 zd=4Io{XOpVr+hC9GHRYg2{EiG9pbO0{pc-`u!{CO2&6VBS#c?uQcF@Ge1pz8z`x7f zHE9T}UBeEQwl^S|gy7HSeu)=DMQEd|gKT=|>Z0d0x2Brl>e0Q*+NDE2Z%mv2r~4?* zs)BH22pO&FW692q$)y8BkuyA5=q{G1BlUhq1an)0@}`oN?EEaV#~%0orHAOc%vR{q z*;tAA6OP9cdMCD$ae+24Qm~2WV^os>Wz#8!J5r1cHjce&Nb+|lF^e;j^Bs&p-JGc~ zKav4|l*k}_e7EyWNLxyMK5|AW7)i^q2!*m2O?(+3 zqby+A^sT-jtH~dn3!P$OMc{Pqj?n#pg7Crsn{p4bJZ}i!``h8~b}(@ZpyEJ+ZW^DyE{7Z#gl4O)5m zjbk$DMFbl+chBv*PFd^V$J6J}hZ+3qBvi5k!tI_S>L$TzcJ^*G+St!ob6TYl)tfN? z;`rk9+C7v-`K&b^3?Dx02XH;WA*noz_@;rr@7b?!{e&;*zzHX(n!PtW~ul z&|=dUNrRvwc>mRXpQk5&-8k|D{su?2jk5!p^G#(vbx?!4tIQ>Il)tb9 znC3VL0&yIpl}_;L7*w91$b^Glb%SBKJYJjTcuN?=rjSt#n#loPeNN^GB|4QV6#|9A z))*lnJ%TH?o7n-B!{luw>GsRBh3~I*pndrHkLfbiN>UjYod}a51nzmD1+I0(7{u`r zlA9>4UXUc)z-!bi7JWd-w@wwKTI>{`9hR1r15}NZ1`EQ*5she490`UZDi{~)hLQAo zF@x+OMp^;QY=JO+x+2Qg;;>mIgf=Xmo^UY0Bv}V83(+id3?Mv1kz18z$0;fV^tm_A z!e*cJtvb-M`dwsOP$-dbF6uU5Yd&C02k~DDA0g?;H9dbopc?PCHW8bAv+1xXzXd!O z=bs!>6tU4sZ00nAP~*Y@frV6L2{yXW)wS2JPr{^!5n9UpOZ(@-%sgtOXPyQVQ0umj z#|bhR`~OAdK?1RqGv8gu00994KtM=RP(+H`^)6R6>^1s-x*RQ7 zWr)DO1*QM_-!NK!6}Zmzcz=fY-cT3weAX9u+-qCImEls)cv({&mB31~sTfkfRfSU9 z@{dXYKVzUjk4~#tJ(Jl*gbJoBq+P2EDx8xF>QB!Xr{_D@l}x+DS2Jw%PYzv#wr4Q$ z<{p>C>mQc{_~j%mrj`i2vup17g&@6~3r-)vgjQ}vy$vX4OsqwR&q%c1yrRY`CLUFV z{F5^#_Qw760bedcYqxO3Ym?KmN#AZdos&wy!>-x!nld4=Lmwf)5eFXEt2N8Iu~QxU zWhsx^S#3sLoZt=#IX=fu>74~JaBEzFwQ*Ew%DaZW;C2b#FMZ6?)-Rqv|FVK@{dUR5 zVYPEq$u{iW#^I@nmdSoGl-=QFN%G%3_toixR}MR>kbQbmWkLJB8S!{&f*kt2D|G?z z<}kD%#qQWOx+6xG&u@#;zXQfCXpHY`nN;(7PYJ1{<4tW*zw)l)3*&h1^^I(YQps}i zB8H=1{BZ7_mKGn)uj;B>p1prd=_Znix70hLVg6M%uEAvS(nMw|Qrw1jI^F()!-C3& zOp?`_DhrI>MoZJNcGqb(x_b=q@-iLhxTW0DzMt#9g0IPfxm;jr$3;gjS=-mVARB6W ztsy^bdmzeWVb4lNyELxF=1qS0?7=q3UL}}s)nKQDQ-|8(A~ke&#g3l#WP`@%Uw22? zB)w&2o_*2U=pf-^*y)C+Da9ck%PAFlPpgQ(dR#wP9%Z2=N0El$$fXrdZs87;i^-C& zXE6y+u3L-}y;k80%=MJv#%fPz%`^BU_3`hd8prA}Lr>|U+Oc7ct3@844p(p8khf!I zrX`B(z)4b&BxATa7wK3*4L_ygb7}WSJpTf~E;UYL?w5|XuB(L1cpyi#hi$6C4#SO` zYEZT>4d2N&MRgWadgfOhb;v4S%whUtMwPiTS75Z!$IWInA)SZHK%ixRWree_0x^?4tck^;}2eX5ll} zQ$3s;24vdFNEq!91S!!HNtcb#`rsV65H_yl+SsCNpV%AB9$hf^FcSg89XBzCduf8r zq7_K2+e^`mYkFJ|=V7htVLEbT;9K?W!9s=@*1EMVC&8$fB4t}SJcmER&6$rwdI6wI zp`@w+t>nlOd_al$CSHl!zWkvr`**OUFZ(yyQs=b=+16^F?cmcLccS|kNnHfpbz}y+ zV#VD(^0}rdw)0xQx65Nxyo*)MydMApuvD4itFO5-(yK$pMmDYQ5qC z>YI+^l$RA5o+1+kGO}l6qs*?<$W6-U5He|J;D}e}!K$EJcbA$rT4U13njeXmUWV04 zE*(&~v=J+wZ#wNB)meIcT;()U9*UkehG0O#b`t2MofG%By7p%!z8goIN;Qw!=U?(Z zXQIu)LM5u$=Q&UtL#ebx@zBKd?u#VPLds9n#p!FWEHr*k{0WtXAA}6?Sr9T{ntB zlb-DYLh__hEgQ+wY$KAZh& zt&aS4yp;Kg{@0JZhqpmXX%=86H-Ppe3S$=9LlRDkaf6p$%&H$n*X1D8<+2f>4syKQ zecCRqs12xWrI8C$2l&dto;YDkFnx%!xah6#`qIaO&!|S16m{T6l1s@JxC~txbpV#| zk}fu78*-_opFd&<)Ghrw*T^F(gm!-i?<-v*^%1X_TP))>kk2?ud zS>ABr25C^WWbW2A_G`(T>sQ0W+8b1yW9omVy?$VpN{_*i_DXgI#L9*`=02#eRg;M=HgS}J9^gh_9dw?cM2yCSonba zrkM9~Z@{}d^CI1%bV}4Oa%$+4biTEe);qYRO3qzE!$ZD~$CWauy#-f%&=%{&U^UX+ z!~hIB60(p$6*T*D_k~Bi{0173X#Ld0fwhJUOPakRaMlQ)3YkVBx# zg5knbl=(sY@Tiu8tx-ohlpN;g$h{F79#p!7C8)Le%inWP^DOB~p4DHV-J z%iRm{p|f<1+6U9e;@N};bY3A^C8fb2H*J%lU4r)6`S8^JoA7txgYiV(VZ=#hE3B;TL6vk(G(qY_W z!POO0YKZ-vI1SC)sYD#G;emLBMVFt4Ej(J~FvIPe{CDkLfm=Y>Pwm66S71Ztj`3Os z@9#@NqkqMB9WAzSs(>z(#CrZ*|UuT27M@1;t zZUYh8EeBojHewBZ)>j|%p+X5BY%J3l!Ume)@n*gy9%`4o$E1H2a8OZo{WZ-OPrsI5 zn;3l+TqmR$*P(Q;JJVe2Df%Se2%sR- zpqj9(xHtFlijQ#C#2pH2HE!G7y`#4H%Xsw=0o=d(?;->v=_AAEo%HI?v2MZNOLFm)M@RZds19xmfL+ z*|#nYtu=Hgcjw7Gy&}%1%S2>>v$8wAJ2R~+M-kNn21-)ocgfmrC-ArQ-Xh%l!S}+Nf=QLbte! zep3kGSahTxx~WCY-IbL{MyGt_qY%(_XX3GeEA)%;x8`3hU0@05AgN7g3Oy?a+V;Hg`*-ss>O+;-AIeMN=up-v9_UVbSd##|#j*F#DP!Td`gd@>xDb?WLvhVQ0Fq+?C?warby;8PufI~? z<-x`!=fDNS#g~QK#b*D~wDcQtN9$2Rye2K@SN^|IM-qJaeDu}~GeHQh)^sx^YSw}V zA^$P=sr-ZbrAzb0sWg?yH1d7Wy7Y0r&gI)2GCJvUs`81g$EIuze3XV*Y#w3&Y`S0VSRR_xr|q6*|QwRQZgI{ z9k@Jpq6J>dJD&D?SWbqg-67GR)r=H~73}CP%VZGiA^$CuoJsX3R?O#lvMJQVc==e} zg8@B@KFY}*)1dk5MQM1<=aMq$eXK5s7R3y`VZ4yjU*=^)`#4Wc#G3axQ-1-lGwk7V)I^lqBYBxsT0Kx2?zkRV8*_ar!tkJt z=|F*IsI*-eOxopCqFj4awt>@kgXY2S9RTy((EO7v<|`_58AtjJm`_I6+hS}M8iGyn z_x{c}*|HIA!gjiYJ7I&`Xc=AMJrz_UQUMCj9}(ZFV$nfn92bZ(o6+ZX!;3inf}!|B zw;Xg|HrIE>_rr^k*9sr|x^slE$-fv|GTpFfHzJBNIzcBecC?-;DJCA5;0Tmo0D zDkKj%y8mPQYnS+kI@VXwb6ni{3zyv0t0eB0oa3$Z$_+zzHe)BYf*-?J`G|k3dd)8> zI|o`Y-!iusuKN?Gv3E`4zo?xD(Dk6R9skkdGOaebO}zw}nI;!jpYJW8BOWZ)3Bj5e zx#CMhIEXnU~ZtFn%w%zMBj{~So6hLKHD34vBImBB6|rr=k_Ov9TDKb zjHv8x?aep|-NHo6bZw~E7&z;lfqdX7)6_9d!3T%O%i+h2Qy8eO#Jzu97y_0DR%Boi zZskbi)tz4_p5?G3RN}xVz)_VC7q~7k757;4Jkcm*1b>l{oR8B5A(n(aqU2MYFPpVB z6h&y5q*B8!@;^PIV@`WkEl>P_59)go7fUVT5s5G*^>im-k*|s-$5wkRp}EQ76+Ugj zIq!eLU!gEOZb?$hz0Nd=-2hv+OEaKb!CToAt`hn51=q`0DETbq)jvAF-4q1sk#2!_$hgUltLx=?;T2fk9Gvi^`h@3j zR&uPc^HEtoq0tCt$W$3NxBs3N*XP!q*QZ75Oa8EYU7qIO+Fg|}YnA-+Zm7E?he&Gn z(AN0GyFR}uX2}`m7h&ZmOt0-I_21pyb+NddB+Stfe7xs*vz#j`{sX^tCE}YRD%^E4 zBDjOl`FAUNnt63d#O!&I>x*cPXld<~b;(78#6_cVXV_SgKgMbR!m}^f z>2Zqo9XrXZ8r%X~!OMUxcEMkb4&r zAnz}M7jly&d4ZP}*|0Wqm5KCVeU^iDA?5RPpo+xYb z6%IN{rz>_6!{12CoCs)<+eX?XBJ8i zR`WZ_Fx(qnx%dyy(NMo?28O; z-Z+y)dMKc{Y(WBe0QS2<<+6vl>x$12LGh3Av;PrYZn-p;M6MM4hQ!pmLfci5##IU6 zs)BR1Xu&DENU7-N0JSwmYN5iL{aO^r^Ip>_oaH0nWGEizG-=y7Cz?v!P{V5jfANQF z4-avR%xP{HbGBg?@5|<0>Rq}g`@701KjGl;*CWuelQ!k)D(`1d(OH4R8inw#Y+>_e zi7c*o;0cv^4iPe|)so#OLYe%rSM2Slj9-JoEFm(^=!Nl%%U^sek|oG`!HP?^E1Y%R z!(|EVWzAaLJB)6RaozREJGc*39Tlm~n943AQZ} zxZ&%U!!a$wR#p0hG)dkF;NeG9AwCww8KmbS#%b09Y%L|}A!8ti-} zaK3ggH3Jg7HK+O&nyt|aYOmF+`N0s&Y~xbzzzLFjnPtxjQ=jm(yg5^D=vb+kTl=j>XHlhNK5n z2XGxTQ^(Nk(5Yn1$99jxX4jp^;DLcclXrG#h1(96y*!pJr@c3V8%vLKyT5*e8bLmb zqJ&d}@gokjki-s!gXDm&7f+qCn^~`8?Lp4)v0p7FqLVNQ2L);`F>Edas{wj!ZeS&4 zuE#B8m(>8`w3r+Svb-mQQB~NHt^DxfwPU!|N8ZgB#iltJ3ce0H%gM>VK4mKuBz_Bw z`qbSnzEXE1a>Ji)l^hx+=IA66VBY|RwJV08LAR64Kqkv&Wei5^?(SV1O^pZTDoz5D zLv?Ec`f|yFK7|7RavcaDE9G$Ql)G9Lhx*&1IwPaHTENXoZV_<#0-#nD_=>dOZFAaF zPo6y6h>h01UT)Rh6VW_|OaJ1JuH~`qiQVBfGvVgQH21epcy)N2(9(ymoY~oca|Kpis{4TTYxkX}3){rPMoy_j)Au0Fk}LiD`tK{%8G41l z!}o9ErvR}jd*hiP#QCVAKQO!%PM&!FmW^cH`A+y2Ea;{A53?yOOMep|!ABg|!UHT_ z%fq>&Z6dvcusl7km06wysty^a|6TcdtUeojF$w}dFcrb-B#B8p z33}B=f#s0%7e1>!8^mRd90+D`6`>IP@2@SiXhW7B0@pbRj%_5l)KC2IOGL#o1Lw%` z7fvSn1I{QN2sz;*lKw^lie-k)(IrSii!6Q;455=K!1zZ@P&yIPJ1(2cUwDi^QHp!O zFmb;D;SZM}wizbTOQ5{F{|KWrE=QUm$s=+IQSXV>>i?`G5s(h;T<=X-5Rh6-5D=RG zUq8?(3Jxg$aaA#nF@F@Ab2boCj5sM!V7g6G%{@t@RZvilVaz$ST433YauhjJ%*P9tfk zK~UTVHD+vRo2UoD@7{c&h}XTZPj7IwU7VpDFF&@M-Y`o?#C>~y!GVH~h+8D0-H9V; zZx8NJ&%0L?;11!CuNVLSY3t16q3RkqJ|?nOV;e?SmN7JzELqA{$U2m*tn(=QzLYGX zX+(N5QC-=xuaPZ-NGODalET;-G+EL-l~Ufk*F0@{-}Cv*=PdVowtLV0W9~io_iN3L z(+iVNTydGm*NiyQ@m23L>`pLAEm6ic7JK4cx`$NQ>LbJ+w~GY#)M-7XJ=CB}PgvbF zD^Bh>sGV?l%+8YiP)aY%Qupb+t9QNieMc<@i@oj9wD<2>^#MyorDx1al}A;YbeWKy5iM_g|DkJ`>%5{()W ztgM<67>~4rMx0%{Y9QGQh0$;`K*ejnhC2xoxOTIr zE>n|L)B8t1+1e-c)dqxim_-+#^r}1M{>Ge|>UBNi*2kJA0;P)PWB*km_{h^o**ou^ zsm$8btMa+AGb)RuvQw2QRW-Ue!jRmkq)wiTSytqmv0H;@Dp=vGF**qW8i#mqK`+t< zWTVK}i!*j(6$o89ZbtQ@_j|any;@#<^i6_QA^=$yjJ3vGv9uPIr&_t@75e1EUjQ{q z!J;nS`B7OlY$&_#Ap9-a5gh|5azpg8Z{^q*B{tYRd zD?aRkDFrotu<`BswHuCcX(V~Se6Nv$?BvD4;eEZ;&?}C1Y>pk()h|Dh%d$046jP&} zd6@mZLFBt<7RcsO^9w*-`Md;0Gj8nl_KV)sYMSp{^4gm__xT$u4PBC6X}|6h@Uj*e z;7B8zl~Y);4YI~wM_YXQa6LPn4vOJg3J>E?Cgp?}vAuNWhjkA^E}B6^A@yk{->SjMlvizuS|jYZcY{TyXS6c6|_`N|D0iu4K=6SU=P*Pu6_!MAp?HR-mCpfA#Z$F(s+k zHk&Fb0-?e=BZ|(6T*s}OJgy91-Ayu2*)6yD5QQY%y3!alN^w0sDmUIeG4_wL8Itb6 z-_o{ne4V%-6VHtzSktA}?K+&S*ZB!nbZE~}$D!lvoE{RsG(~itw0Hzpgm^V>@^yis zc5(4lMLm(Lf_6@geUdzGed3iNB~f+`ql-ZV%lu=Z@@HrdW8B^b`M2@}RI*M-cXuZT z{=H&mHyC>R>j}d(2egu=eDX_XZ<=$~OW%!-ndO0_{GZjTBwHZ6t@(MG%F;`oYxpOQ zSNR2mim^8%U)or^Oe8k&MDw0gtt2<*MBlSLaHKmMEO=fbY|zJDJln(>H*=wp&!hiv z5+SSFgy*l~B)_g_Ma+4|s|HJNc1J2|#VmRo>q=|ozGt!S9D;n`tLp|_;^mWH@K%>} zWu4|xH)Ayley*yIQL%33T+mmE40HHqorHuW$KX>UCLS@#B=-!bIe*OiO^)b>u;A5FUzxo?HC!@vPnv0m4=6-T>(jY$TEZ?c- zaL+ySPYp@I!u__#2rHI?qJ28{e!4q)FC?Rk^!DEtx)OV*m^)P`&{Ifd;94R_z2Aqk z1i=(%ji}?V5m}fVA4O|sAWqiv?_oaOPcDzRyyIF;rWAWnr3r;c4`&*TL*E6-q*%zg zz8qj{XGarHl)dXRsdryOJg}765&TI*w-69!d)`+vth~S;wvWjv5ZH0IJt)S7PW2># zs&Vg5Y6ijIJ9l1Ix>|%)j`s@F-eqO0K)9NWl?`4+9*ih=4!BDW%_WC&hwoL2jnC}G z^vz?U@Ags}Us4)Pm*mc_=JicfdtLLGiMv~6Snu9IO+V1+zNUO4BQnPK%9I!&1_~GZ z>THXu6y+SH?fPia({^+A%g&km=`+n7DK08=gDQL^mDG0orA~FAy*4IDE4Qq(jZmNP z?P365ABnrW&9j3{2c{RS1Ut?!DY~%YoIBF2FplG-(qguP^l0gPlcJVYWl7Hz5v31v z*BoN(^j&rztZjV1__D*^b_Z;J076Jr z!?xlt9mg1D17rC?N#-|P$z87Gql7!K9J6xnI_-s?*3yZB_q* zj}SE3mH1TO+{gHYmBriGr0N_yx!Ce7*BET(El)=y7a1aX4|ndUv)cRc4kF=HLAXL7 zS?!1!AfAv&!UK7xW)|bdU;3$?<WNZas@@+6uTG=e2qc>=e`PYj*jdmEs9{p4>F}mh@nn}D?EB(S+oig zq?=b0d#zNsAV%bc|1pFIn!dEAe1|7Bv_4ghNA3O4FAZwAx1JBPzyi zjK2(1(HMVfA^*#iRe2uHpW{CM^xlVNb4yy5(Jxju3WFBTTWryoaeWNpB~+zEhe zI*4KdF42ZUr8r=)zXV_~X-ItRM<^f)Gl4;}yTPduF<`V~UywX>WIyyn{~(~afJov5 zBPWi**Ezx7iQ{m6E>L1p10Ku;o|?qNH+Di13ZzUPg;(){xg`MjfFJ-mPD#TJ_!(Ir z8aKExxf8q`jo|vxY5}nb$vF6RN)^5YKuI*XahVmwPa~LVpS@bZplKw0NSIMxHZ2Wo zy0qs(ZUT~!P|D`;euM&Igct)#xXJ^@jUj+7_SiotC@vuSOEAEY85w|KjSIE50;xF} zY=Iu{Wk6FiDgeXabW^L18wS(b0tL%}iqvDk7Mr*&K%Nq#l@_WD^QQe4_?C)<=cqts zSjc-z68O{X=ttcGV&MTWXx8{&lcVNYB)nFGQE6jV3}DzCL1V6C`ST1^YeA3-WA?xN zWd0m;*o}mX7qQS~aZZMFFVBWNB0L|x-aJoLDJbr#3@XMXy zU)8!_W0f(6AaU^1yaK$>0VF;X2XU_z;G-^3avya05n$tMA^3(nIP}^bKHv!+qG>T! z!QnwJ@l8R!e**%xtW)Iuo8QxSdA-e*%aGUmg$@26?5EhCIgSa=w+&k0Y|sM(m=5eu zvAyrzLCav5&;R!JvzaZ@dz)tzlwtaP(f0d;#32XxP#_dxLDpdfxK0Rk`|yK-6gKe0 zupqESBkV_~P+UNi2>l6`uuFoy!w6uD`p*`)HsU9&xf2D-QxL!}eGwQ;YztgM_zoX{ zKfdv^UIRN464;i8*Mf{90!9?n9+8GWNQbiWVA==*`ZDA9sa?oqa9RgCQWg0XFHff%59CjAh5zR|&066m+{l``Lbm0wQbicUTBq8bttGcD?h``a_(MU|_#sz`#V)mi$T5NH3^>3e7!r0!_>>r|)?YmKbU>w3vD# z+xXyAnhfx^_WGpw_;OU35_JnyJxJTkechWP|00E6er64vrLE!^^HGR-RtB!-d{KP) zE#nm|yGjW@qX&7w^AM#?_i#V&xDVX)onHQ?0f0}~A%>SJ323qi_ zUW`-V&I%*7n^c=Qw>x~9I^J|gWMN33y3~i?&6N0$Ie8MCEi*wjr_1;druf($Jr;<= z16yD)wdSS&GJ39dF)J&gh>q4ev!sNPP!$wn!qc%a!REZ?DPT14#~;gBqYkPMA67ep z*yw3I_G+zm+dteG-Dzm(J{(y0y4n{QJ^l%NgDga7b&Q1?>_7`p0TwOdTad> zD$c+J)ihS1d%b-R1hNq_ZfQndv$=+CHwdaxP-5bc^V}|R)VV?sQ zG`MpON9^Y5sB&G@uWp8}YHprga>ERzXU9BnKh^Ve94m5f(oQ#Xr}q_owr7v3CY-az z+)VtLTWqS*nAQmYq*{+?7}0yH??dfumg4P|baz-_|G*zVa+qfC&9GJh*E<{0L~!JB zC?O)kPApy>p+iKk6NR|Z$(C9kfy)Ql&w6~(s^>nu&_xXUom17|NQJ zC!W#J`GShp z{)gR21Y#3FrI5xcJFz4~Y=Mo`#nr7e&&QLS!6V0^xW_}UrI5erSoP7xqV8g1sghvh zN-O20s{OXLL^}_k7@xYAN6%4T*3|WEN+;B5BHDZl~&} z^&cC!{>r83p4b2)mRfEWLm}E^u?J%nc?d{&FfdqHu>Up+SYc?xc1hZlzbNqAU0o9M z-<9H-q7yggm|Trc4LY0bHl^f8v1D<1vB{h1U~xP6c3#2b!QWjUck^@MBM!dY(m5WX zb3~Lmo?t$q7wwmQjM2^Q_O$W>O#bt0-o8Qir~EzMzUSqKq9AA&d@2ZOHv9@udx%hf z-A@kH{;21S$B+;d*YzRX2~QxO164DaRw#DAKbOVhkeu4XAhsBFxIA$d+RtTN1e}Dy zx#+CB_7Gn@YtTtE%{MZn^diIEQaRlrXZu#7g8au$c^~LkBW(i4ZT_*&mv7{-hO~uW z44Hw8d}>LR4X<18({b)2_E@eWLrkeXyuYkZ<_bZaDHizEyx;YY`4}K~keO(YJ>td> z@uT)orpYAEP7|Ga@BHk@2nN#|(0yyO7y$WIR0_^|;wn|HjQ1Vbr?{6FZIeh4n_(S$ zTkBJy{rWXRcX|@I=r#ixi#p}4xM39y{W4x#{$lLWwoi|@P{UI!37}Y22a*ZO}b((VF*`8paErO^WCTp%N z<>FN$pHBV+K8IX9p2Is6LJ}3&!_{Kncsy70KWeG#EZUoORe|!(^O}=NJ6_7o(DDOH zW9Ug28!xAm3HH&NtiRisRH{FCw96|_s%;`v`gN_(v~VoDV*I^t8ytiBA>=gx)7(}) z#l({u(KeWVjO}at0n5{~plTc`GD0_w)GhzVT^sy{s_Vj=YfjDjaXQU}RPuvdqJ{e3 z8I^kn%`FmyFMyM&p$|qO&G&Otxe9IgpO5e1ZE7+srpdb?A-_6Zfkr1ZSu&eHYN|AY zN?Uj%RL;~%!Irg)-2wts;VR0l=}%^XN{`mw$X-V^kqOIMPR zw+INRO)}`8{ZJkr@DrAif%1aH-(HSr54jVK%aMrk0PF9En zH%MNT!mPugh>L{*x{ijH)TKet#zMAshp#goVhm!_p0~i|d=b zKX7*^*a-1xuCQu`L9M{HiekBiSQ0yn`J$*EPfRJ5xty~Qm)yRw2Dbcz`oGhg0uX|1lABxTc^AgGQH#C~UWis6c^j@uoY% z5%W9q98fvVAT}DuiIJ>>vg{baVd$R_*It34ZyL{HL7T6j=ZXD zKGVCZcj{bZlHWA0wSDWvXs~uqKy|(%$5&z#$PrDdK2o&w5ts!UVaKN#7Ztt9Z`11g}{ zcd{hS(ApwuI{YHb3KQC~^mFnZ@0!Up62{`MAJ3d9HmhzD@kf^LL)2q)w%}XS*^~qS%%ns#qGIN=NbuLV#TR|pEGSRY(K;zUkUVM%e zd!=*>X#socMI;hG0N&8IDlSeAmvLz`KGE`M(?pj3nCq&ZQ1SginfsILm|eS zH@kIU+X7XJ-5G53@UV6*F_ZZ1hYCDC`*%TSH$F^~9sBIS6jh4C@9r~Uiy^MeGcH4g z?Kv`etoI%EL8;x-skig=DTOOurPqz}J`I$goshX~=SFDnq6`?7Z3u|C3if z-*`tqVlp!`ZkoQHn$!ajh*^DsADebD$yGPh2$f#y#BXWtF865&F`QwbsdD4=7O=$n zT=AhV>SpHUA$I}?!opy)s2EuKlWR(B{ASlW&pm68z_fhD?mXOEG`|*EE z8mqiOCkRh)+dW$P$&~q@%j&Djt3?&!hj6mpwNG&0&BO1N-jNMx9wt3F;sc>59P`X- zMVw!hBqY&r#{O5n=Rzd$eb<>an8LGvr?NvZ^y% z6U#A93?#Ue|GpZ|F98zK1+GjremNb1@6@cz z7V_ywkBWBAo1>I1)h&AV6h5MC_rVk-cUbkht>BYOwEBVkIp>4fUpez)BPtm14(Z#fEq|jjBK#7&zc4OF1<&#B8gHm3f~};t!6o*nbFq z3B@xY|0V_RD$!hrO8|zNzpW823?jnPp~tz8_>(T?O9T2ahz_ zec%rwzyE!9tR9p&hZzsOlF1 z1;Kz9-<+FbPv@}5xU;}3FJtCpVG#x&Lh&khYWz)?k-B@_E&+TC4M`La=?JOu`Rm%N zWamCs)eN`k)X;cwYcN9j3Anl}F&B`^p`!WCf8FIki?6h*HvytD0Nr8Ike3=J;yH0A zV+P5P8*ixF?qoy>YJQ-LAN{~DK=$ur#VVcTvGbd-zd_7Jt+|elsV|mkHc`5t%(NembP<$4=Gb1pKp5sg^O!rh**7qbcT&jeu;haDMQQE7iCS#+w6MCo znvrj`4uwQG2YaQluyN&~X;}bvxNl1qvXbgMzX+CEYX(pFTdGn=f=F(%kpGOi*`XBK zc873Gx75)Ar>HH*zo-dBMAQTdDZ{X3A31^gaSO!Ki^V@NR(plHRkt{Br8OU19Oh(M zbQK+PpsuC;XfnHm&>(36OT8cS)qs~W&NXI_mHZZ}=6c+9WVw(4{T?72(>Ai}A$JRO zDcD>=fBm(wgNJSH+;pO2NE^Jh7-*qv*$nj(^}JQKZX?NOO$Cc)aypmxVd)EDb$DtC zuuS3NuWXpkV!wJ7{5N`H5-;Om9KiD7ZHs1pnT^Na1IdWE?zfaaIK}8Cb~jrrx#q|L zQYtpP=ej12rIGe@j|H?Ok^hxMJ5@eZCnB2lh6o&0>7Sv#b)l=m1?FQfIX=ehys%Cb z%@F|bhsvi3!eMvT2opkg8j^c7Ms@f8eV^lD>Ops2(Eom?{v%#l8q6Aqev&V~B<1G4 zV`{27?tR11a0?|gKMIgy--}ugV_BBujMG~EJX_Pbd;}Au{Ril2Fn3vRV!)?Q6{-w} zbokVSg(mz8Y0>HN%{PEBKf11;PIgPxsBG*_)0jaWfF?p&l|Q;_Y!H^kKLqJTE-+Sd z_)HK{&Ep6ArOptwU!9HRY?&vYr{`*=yu7dJshy+i$z`oj+m$-mW$M8+zpLp<8J9Gb z!Z4lLKY9je{sD@eWgY~`snUNL>_KL6d83>Vj~fv10*XQriS&=ZAR9=l#FF$WBKkGR z`%>T->GNH5Fkb%2&*=*Ji23cy&a(0(APAAx*5Q@K=58Ho=&A$x0bD_+uDOPX-b6Hw zcvZX*9iHZ#&petTj)g8s;>2$OGE{aUaE--kz35JQ(tvw47OidBaeJX%jUj&V_!h-! zXK()YA4(-Ti<@YVyfZi$K1=1|Nvip>%@6NkTIP4gy^%%r$Mytj2z$uI*j($Fzz5~j zLCD6s^fD+nkKCC_TaXA+;c%SN5^owz4i)!xv1EHnZH+p;qht4o)|=}2d8(w5%An$; z!^7V+aiEd0X?E!Vv7oO(3YVT0&P3h?<+2^`lZlrHGxP=TEfMM9W~EKX*T89_9p+QP zi(`^lNA;t{5zE^>t?mi3AgkmdZ|Bfsc!-AyZ)ie((nhyyub||=OOdNL=pJ7SYQ|EG z-Gj@b#{+M0^OcPJbLAYims2u9t!>FA*z~=|4DbNqE1&B*pKq}b&Nf-u91rELq(<4E z!s%s{#9ddly6Oq;_xZ%H=hxmZFbUQ-{ng5tcGlJ0B-G>A^IH@zH=S{RDTJ{JDaW&) z-4CzTTdM7+IalL;(k613=lJR2aUiOo`IgJ!k+bKSt1-wRp0!a_S@?$7L0FMUE$P6c z1Za~xY`p4m{G?v!+TBPriv0eP!PfgnL*3VvEEe^EMffiwqfp##<#UL7Ko9y;V3GA~ z6I3t^s?SIPRXfsIFTTOHE!&lZ$Tj#$W0__-MYcD@Mi}fB>tAq32+sH%G!=4ANaLLL zET>Z1Rx844r6FtCF@yzNC4)x33V)^-;^poN@n4;5>qz6Wk zH1`8L-x!w%1NV|+Kl-MY$%&AOITrdB?mFEsUPT(%SA;$T`Nfbb%-k^>LP3H z@V%U>P^u|el)68Y zHRfPclv6g}53DhQBoxm_l%H|`5&{>5RZI{AyIXAV1*s)OB6zz7$&OAi$H?VN{1su6 zPr@WsK{-K`uNUXf`=|^z-7%g}b@F330#|bnnE9k?7V=0>XBUmaVXfyEO%Y0XTW?^t z?4+G!q<;dmt;?*z*wod9rM4S>iSlL71;;^=s^IR>E)ZYtM`%5OC4q@}^8$a)EdDx9 zQ#EE99N3izLyE{XzoEZT_LePFIFo^G)rUQO+(X&&3Xp*n~#pW5rDe*%X$V{*^!4s3IYyJvIFM!qv zl}{<`8bba7n}-Iuz{K;XL1t^jXk!TcVfb$HktTU5c<5dIF~4|D8vVuH#|83xr%hMs z?g!K-mER8;P9UOiXeuSYAxWn1ATmaNOZlv+q^#M6DMP`;KPsFJ{0yifhkjB36I>vK zgOnXlEh0PBk-^ST=V?>an#`_GY?jC(oM;=p?p^g@zCRNq5UqA|#8SkQ`>7Ah2iv!F1;=MSG_PjzE9Z@Ihk0{-CiM3(Nu|DR6MCsw1By)R$53g5 z#m^3N8fF;Z*7_=Hr-Ay~0=H~>f#@9mXu`@iaSds<-7JE>BOk!&@`3ImsZR_dc8>^O#aza>KF7OPJNFbBpU5oQa=xTw~Kg5qa`qDG5KVr;V zvd%Jb9y*iFOlpZgKfPB*<5G718R?Z1^ZpIAO_{Z2_zdgE^i*AjF25CL9Z}K~{}*1^ zCsqMe0xd+_(M{1ZzNNAeJE`5AH)e;WKn6k9(%|&do@&8Z!h$Rb##hJ^Z*>6ow|j)U zA9#dDd~zs#@&LmBlBTqe3;edj)H--16}R4;Iyf*eCTuV;`u}_=>@=ls_<#@QB-R&9 zL3`C&sat6bd66W447mcE&Il?Q9AyBh2)e{RSX_H5^0m|WE-{tTfk#!UR4h>y4vj0k zQhr)9_?VKn-_6?jkF*1xSLhm(1RfBp}!&W62uV{8+sIp^h(gXNbNw;NmE8IFLE*VeMV&tjeq3Dx7ySe(L!VuACxIEUqWVk3Eo5-ULbj0C!@Z#i2M1Uf$(|=WR$t2vLIm$kD|q+s&H&prb@UFUX*7CDW3j4iT&QwM;?T)`FVr zAoBOGzNR$$P+F!LGOwb9?YEqG^CLJb%N?gSu38#&M_^*#ivy3uri&3KI_G!iE?|}= zbU-;6+JsP#q)4<2uHL0&zxvm##w$;@ZqMZ*KxtT1p9zbdL_nfFr|M8uon)yQto?rO22a!{f)QsCJr5#CP%*YhG?2B^GG|4jGNjDN`v7jb<+0c*G1csqlK zwUNL+{l(bT9D;p}i0(oraA54VH;5(B2om-Y8wR-eC^6Z@F(gN-qRkZ3U1Fg&cts`b z*lC`q4!tO?EU@W}U$|818*Y(Sd=#ro6-?yoh?DZXT!xC%*dkefu`K?Ey@N;2)nZKm zWRszUd2Di8OoaVc*#u1?vse@vjSJGE3?~x_K0B#7+0<(pv?U^_=_NDB!E>vj)oY&K zU<@$YTr|;9pg8fll%FS* z$9!@7sPV^BRX#m>)njt7dzagyjHD$1?aH5uljSyD(qHcS2YT=QyB^FtnBIS z+4=Gab_OLJtsgl24Zgj*K2Hnvj!Ld3CB*EPmtJhnrG}VZ>Quikp*j`I=&fZMh8%)GX+z@gc?v?uzt*1tXSgn`q$APMC@hR2J&L~=;A9-S{ zu^m}+$E(|N8uZjPO2?jtRjc2DxbJn+dFMiif2iY?SD)JZ_Vr=umGD0aP)kBD-rW3f^0sdjmVw3&&0ZM#eGu|RmLzDDl6TbtXzLw3HSusL zciNsdFQ=E1jh=(|Ff00G&nqm4h|wo>&OesTO>4-`+=xM~Wp+0sD0)yT$H7fnvAm^c z2&}ecDki1fAmA4U#rPX;dmRbPj8yuP^N!3aotbk*sipoyd_rVJ1_S7Ch zq&?lb`Bkcx<$~;yrMIzcFJ7*+yMl?S1FE!&1Ng@9Ul3da2lBL64Djim&#&Nm-tZji zv_+KKGHw-=B)HO8-q5+R_OZvifAEdP;oEZMCRqDqYgA>J@Fod?);UE}BX}+@gPgsi z(^y~)7klb_q;e(0T<2%`dNtBv^;I1mQPe(eHyJA7c*0@z1;qm`c9PjNPo~;>D`uv$ z-vGw9#926x=z;YzLIzeGh8EbmX5zZ#5H83^YO|Kan*tk+Gb^Xvt4 z24bnYu-)i5RAdm~MH7(qYQ(1?A@7PN{lXQ7Ph4I;N?Tg^UUG=r^K?M@#wPMJ$<4_m z8I7&m9d=Zux-P?edKB@Pcgus2hW1LpF^+s9dW=XAoOP`aBHxf}FL#{9C0}ZVCoTd@Qscs~AwyA% zj&Wsh+!?kwBXwGNf{ttoeNW{X*X8mqw2FmmwEy6nZHiFf@%~%$Q5Wi56q=A!rZG%3 ztP~-q`HHQ`zjJB<1wmjj4Q z3n`=rbbJFay|Mm%wN5goeOplx!?DTJb8u$?(T9(UiLp7Nlahr)mKR(i=aIE>TwF4S z_^CKHNdLIV@GH`htoY?1wmk7JV*kT=S*t->@Pgz?T{6(wihJ`nBOP1O;@5)r=kEK! z^Sk20=V?jQxB3y`6H^FAr_`PPWP-drOzy;Z0K1%uFa>QSI=qbCqTJUlUb-vlmi*dy zj)4VqQn5pLdV-7x*RLSOZL~07@Zf@DG+fqa*^l02ma0ALgLDlC>QH#=MKxM%-6cIt z@WE*6?;(6XU{ZL|DjaAaRPFyk$krd0w~TsycKg7+8uxi5b#w7y zv!6u5nO68I0n|(mb!Aol_utq$>3N%PCR@u)Z5!V!vlZrJ9=*CSRxK5QljrMW@Ww{TK8JD2=pW2QKzZJL;Ipv&^+&dW*v}{*1 zSUzz-yK%XYM+8n8D!*HqqTM4Lc_-gI;eE7Rm!`_Tsd3LA9k5(^){8_@3QECWKC&h zCr@|mbxH@a?XoFck%y&nlL4g-@8)YcrGgjwG#%lq86u8o*|@sgwzrco{#xoL?kwCI z@w!7&z(9>{i$)%o8Ga@{#l*J}JvqVh4lHv;*LsU6F9{CVB##$(Wxgwd6y#E>Va-_arru~T^%DM0)SC}t=>%lJyH+;qKTSZHpLz?X%Wvr?H)0zy>%QPY(d&NOjBWY* z!SAuVhR-(dr(=O^vNf2cG^gWs?zx2CbWD9?xS(57MrT>>X}N(zZg#v#+wXXMt=Qt9 zHN4_l3L{lm0?}+x+pcM$iofbj5V#jd6W}||@3)SEPS0ppm=N{>keQg`9{PIR zX1NU};MSM|;cb{3)b={V);NP^*yVIJKQcQEp4>zcN3-h5moc59y zDtyQyVE~>TUaiI8I997TTcecMbun!xS8O*~s>BHw-pj>hnZrc+w<%zM5Of1yI8r{e zVteCRr6{dzqb|0o?GavZd34-H#bC=a5kHjC7Am#>CazJJfzyI7G`A{8PJt{x3jN3JZT(?OwH)DNXS<$3g9xJJe}mS&YG!ux)&++&B|Sh zZF711Zn8<8kus5sZs|RthJ7-I>&ECTyT6sIW;xg$lyy@+(I@lrbzH;*JYR>8NWmfpc zndd}Z7MjyZm(}f5ZF+q{wZti%EWL7arC9&9TkrQ>$VDJ)sSZaLQ%kjm2Kly>;%o5!S(7tXZ-*hlmEM zS!2UZ$Ey_eXDc0Z`)sdxqa6BW3i7;kXuosy_fDBd41q|)X`ku#o^>8u8RcdJq8t6a z+TyaUg^0!8G(dH=(|e0p5~V4TKQ*$v((Us0Jo@s#aW{WUaAz|q_IPF1B>Lg^A8DTP zUzrcz@B=z6pQ(POCcVhh`SL;$=nPN%d&j$qErsw*W#m$V(-JZ)Klvj$K+(@oB~JjN z(pb$>LYNYQWT1bcgH#!$+FlKtx;j@pdU|AZ^Y`Ok<}OVN;=c_zaH?7cn;}&N3=KbV zB@9P#Xa3+%?$;r_PwqD%z)YZ4Bfw0e))PcMf&r?TAS=7DF_ii-rk`5N__87}yg?IZJ;Aw%*omusSz3X32H#`< z{>9TsEX~1&Wbq@2qjvGN9)-kCB9|~+t69|%`^3Tvj|s9ZqG`VulKH~8egD3?BOGFB zI15O#3Dm*ORw>xrMSbe3nt^Lu$ucyNhfW|iQkNpu{+PGd3HSv-FW!+|K9?JAXSMl& zGwAL7K80_G90}p*Rx-iN^Y!>qd}>)urBhxWnI0bIp|F@+U+Url-VsRi#h;TwI91FX z=C>{_yyYNqPwc@N|ypzNQ7+oK4-KMcR&hx<(fw^s%CI|+S&gknxmwmJy^$_&m4`vP!{ z`xS}YLS%SA>JT^Ls_>R& z%Kd~Is;s8;H`Pmcx^dD7A4+y5=rP6do0KQ^JJ*5h<7(qjba$4Uz3?3|&htK)?&aue zDLTuLXsR1AQsWVrEd*xi^OF;Way8Jtg7^ylBnvBh76grOvM1xkD>kwZ#h8hjf$9(4 z5JkoLi2(DJ0IMoW@m&~>PopJch55RIh};Q3)QuBoRXRgnAgz$`ymDjs0l4EXRP8~V4a&p%-U<(H-UIN=o?l>H4#tha`*Nd``l?S%`?`+yAIv< zaD+y^u1o!Dbe?OqOh(@J?^e}8x@1(_ie-FTNO9jAbD3+d?!f+8<Idi}L_YObnei1w_ z%6Vp(8SI*>cT2f*=tNw^nod!}pxrxwnN~)jcE?OXi;oCds^ZgBf9M3g66ysV6E3qj zD&)!q&x@J6%QPdZIT(>~gdnbFfBUI0l9M}aMezuf(U4^NDwXwT%>fZl1iepidXMqU z5`Fzvef`wpw~U|W(ec9OY3A8wwci%uec4)x_%AMae~-tQ8o9{?;2_|PSycWDLBh6n zbq?m?%YO;-pX5Kdi8i2CqQ5iqZ|fVsWOr>|I}$|{%&36z zumlqfOq>Y}jP(D3&aWB*fSe35j{<#4?pKybi!3ZUVhDOBwBBDTUs)-uhk1guB}sj( ztj_iIl~_ZEhK$ZqtPDs+$%Zw(u5~A`wXMKaCu1Cay*J_Kc?Ife@u9s*mYw(AAE$-> zng4j7`}vhWpNGvQ+Oz-Rm;W%JoY!4ZNU7Axt%PT zu12AZaBQ105f_GeaxQ8#A|Lj1X!gjnhm)aPmp3u-t`=;=u3xWm1M-~cgBs6(VE>^U za8JJI78*igZ&NCF1~5ndiqeA~Ao@k$s1vxMZJ~^dUEPzlO!*O=QY$5M=SQsL7z5>l zyJlqSCbl_uiT8=V?b1OwBdG~?$+j`b2%r4MA5=W-nmvpV?G0vuUy&NnF{hBpi+GoE zLUD=e_mFE-Gv|=m?vX#dCVh61$dwOmSC@K%wB=StanX3o1~?hQ2u~$~(?kc-8^n}a znCL4Y0&*UIkgF6;e2V@-t9!cLb$#RxisHQa`C=#oFn@|WNO1ig7~28fVv91F90U3i)`7JUGYECJD=%M|GT{tFB=nuk}v)Yc{Fy)-)hPJ zSz^B@r;(q3Ao6h-d6v_`-H_6fqrq*>q-u4v#4zQ$-SSt8M1W_{;iF8clmmI=*;J7= zy|AO!5>Sn?t)KGL-tXL1s(?ZGH~sn0`}B2$;x{UTC+ zt$l}NA}#3lr>v1uHcMNV@!n}(#r|&W1Hc=Z*MBQ6SLka&`PDWatgpa;En7hejv7|h zBf1Pee9*qr4ME@LUT5pUH_d73O}*lU++=t07mmT|S10+cRLaK?&1RxRq4gY-me`70 zARoFXk8A3AeG4SJc_M7od{4Du!NZ{5GUjBa79U*MXd!F^JL;c=^XKhSIfI_>k1{fDe49P5NnAuUZ98$_|~)A3~OZ$+4;WtuH=92N+& z=4k85L+euotP<`#=H@EAlF(`5!D^_f`%#skcLZU;$U1R^h_c2dF=x8)39~_Wa?SSNfH~sIe?@qW#m*(1apk%K zjN@u4BcJIDa-d%M#_kz*J?j6AdET;*1BO}q*Bajfc1cU$22`Up>k<2nTi_t0^@XXb z!ZK z9IYToj^*N!N3dj7)1yP_rh>r}zgV=O@f5}Ukb~aSa#@kjP=4dQJ*jc|g@W(qH0jR= z+koyN#JyYG0?DcJ*@x^GBmlp-A^J{k`b1aYe5@=U5rC9JsmJ|OvrKR0l_P+FUGmGp z2sI4C<9PA@iVsM~RtXs~-viWKR2DoC*fVo@Ly1PW@l43U119 za+rmTrwJCCSVkV?)gML+;5e`nX)al347Q`kMy2{mEU*`j!jFca0MNwTH=<4q5Oevz z=FO-!fh`iF^s)=%;1vsrJu_wQ_OGJD1W~ zN89e%V0ZpSx`eC=U>nRyJ2!ioV(;tx_ z0k81pZJ1R!za3r2<~gcFdhqgCq@53987jvYmy^*_ohLPPD^mxB`6ivpbTrf^M*!BN z=8AoG)KH5Y`u&#{A620XeK%C84$mMxa#?j9QdXth;bu5KkojM1Cm)p0!p}Z#*>Dg4 zEBrzug2zhibn?XtQ*!iWD>rdFB|C?~i1KV8R?Up(eO)(mnT1a0bn;xXplHA8{G(hT zkO;ZFNJas2o8nG^5FxBeg)hJU5 zEU4C>cM8)D;O#HqEf}0$L@0BXeYirCJD!m&7^J|yixs4r8OWm|(0w}p5G2d{e9I`B zU^)8;{0dnRPT$dG|2}Dq%oU`2T6DMQ`2|%rvFcY)s&;A&+%k?P$0fU+p6|E5MhrnkB+8-t^Z@8R=|5C?~e)EG#;i8W+j@g8fF(0~euF=cv=^V^W&#KQG0XSUR+2V`9#FIs=@+d$Q)hv!-E&TO=#7`J6Ht%F(OG+}j$F`W7qLATqzZ7@_2+NT$sK#QX;( zEre^&v(sKXE#Q4BeXBZ-|1i>=hG&LJGNX2NodosFbjTW*#1ub$ofrDG~tPY zgl6;Pc+Ce_nfG(ea%MRB!qBLiaZjJZd71hNw?+|e)*(KZtsAO^mD%ZOGiPJ@Ynlob z>BQ}t=(9y|Vcy3ESJ#|*(C*$7Aab4bVuyYAbM4ReK)$MQBfnRT-c`)PSjF;TD1KH+ z+2P&qkzpp)7))wZ{p|1{dTSH$7yN;8^?v6C#pAQQ*nnF;5=#c(iItG2pp2Xv6h5J? zK}^Hm^fH{{U|4Yf< z;)h-X|1)jsc=#;pY!nyGHc>5^^UiJNoFvpUU}2G+fA zY{^l57)_9>phz1^s?kMORPsMi?Ki%@b$$s@rzl_5`l;?U%TrW8FzHklk#;UIrGIIB ze_h5|rG;P%;nDcK%E^3`*X|O0a*gw|<(I_1 zjZ81K4b{;riuTQeIVA3RX%n;J6*G+NP{(>1U(Pf`GU1F{C0DOH%S(-zJf0BYpA4GvS;qPdnqm+)!s=OYv@ zzG*}X%SwUVQ=mumb?6+EhtO{%W~0l2%mIn#;G$qpI$N5d^`>Q`1Ub%L?Xq{BviBIH zvds%FKJ*tB#fd&CQz4}XPCK83i6oa}FeIyDUvPmyasWyIIJ2(_3O?Z=DyEaP+>NU4 zpI2Y=OQ%m%I~L5Y5j*L@QeP{p55nqkht*P@_W*T zFw_Yik*HK3(=M~v7;f$-1O<0>^4~*2nIth`l4|WGK>L>Ryo$^^3ffPhLdG}Mg-J!( zSkp96hf4K}8~4Qig-0;OJs>0&lpx*?ud2;pYy0<`UYL_2Lc5U~(}Fk6rBV zhA}gqs#G-b&-zUF^jGk=Pr1iQ7l(ZB;Qpwn>hgxxv-vQMt{DBu>Vf%xs9f#7vFpPZ zk_orG27?2h$qU~1FVIJ>N5z#8?LpDsJCT;50LS}X0hv7LnhI>+Kn{l=P~RU>mh`vm zAe2>PWf->pjLFe1@rg9>r;v<~ZR;VgC`4T$3mla5$T<`J4_Dt5omtc^n~rVUwr$(C z)3Kc|wr$(CZL_0}(XpMIbH*L#-v7L>v7hE%HCN4=Rr%~#>ty)Q2i5bTmK>bDHK&&# zE(QIF+dz7(f*1s$>?4r%)>d8T_QJ@HhV4IeYM zOVDU~aP_BtoV2C2hOex@53IlsSTBcJf1hamKX7Mb?EmU|;P-!`tNTfKvO=|A4O>0n z9+SRE3w`st{VUMQ@5J?{FQ|F2RrGGy1$)qY!}oFKvoy%RHn9=leFy#&4ESuo1;S1C!d=IqLgWna1UnCfn3qH zeN$qFRONo5TnwPuRk2hEtJ5Gy3@N}gPJWs~eae1_V53PV0<1zs2KUu#{l$WQ43o)_ zVGSLki!mb0BqKt_U=p8Xz$X9*%eZVtB+p1@2Mp&xazB4*(JpFFDZ##9(!}Vw1cfq4 zlIok`9YWG@i7`%6DVS&RfOz_(^m9JRgPhZII4cAKUPlzS%Oq(MLWBaK#)dTd;SPHt z_9&Ybj6st3`D>8j=c7bTn0)aEYV+@4(kBel^S(h@fJnuoyXgrazY*|)!HEY^_pJ<+oq#-vC;*ov@jjQC3BDw zoOHe^=N&fMR}{4BOgw;xqSd4bFfYJz5{z2{JhnK&sSHAwQhzYrdbAU_6kPdRZSIkP z_ZHfp181Ym{iRxkjN0wSIiCEUGjjq(F-EqygO}=BmSN^hJMzyFeTg;I#akrzQV#Yc zh-B(~pPHVlrj?$9?(e+!I29%Y7(OZ>gAWQ47ZUXeq(U{-{R;p*tj4Tg%Lpu)@H$bz zCN2^y=NwZTIsI_t)&v(-Kdc7#&vm0;?vn`E*7^q@FoYe&cj2maA<#3z|73x_W{#X_ zfM$JFl@ok0XLaP>3``IMV&~HxHXE-%q%V?(yUH>jbYmFb(f7O&2Ecu6zCnrg9)la6X06HGjjM zAcmlx2l-`NmGM`1|C9Vinvegc+>;Eiu#=X&QIfK*V4Dd0IuM~N`6>|Vf2el>h@@)= zti&5^KunUY0*Vmgm_@25>Otp zd%PK7%nIYYWKHD*iQsdXm=Li99`Z#foVIBL0L9C2z;UWI#Ol*3_$tfxBiq#`Y@?Dw zRF_;;EL$7ZbI-{DQIN2ErQbNsJ^t0Xd{VM!3u6C3uEvJhQ_>uOewYFRwL9@-js4)e3o4G$RA5pFE zfC(!%UU}N^EW1AgZzV|<(q^w0Rt9$1^mt@QoT)~i!{ZvD4X)3cUk52yk+HB28!7w+79`(@vPSv<@9kn##{YP9ap zn*p3bB#9GWM5Xfmszx|ALSn-nd+`ZGep8n?_^pBaW=SmW8;t%|eZ#ePKZqfm2P}Rf z!4p`eH_h_EF_YInZSzevJZZ{HxhB+^F~<{^w1|7%Cu`4{$)# z4Z}Ib5^ozONB63POBWFQcH^g|2gTSAaK5$0#Mno>xGJ)9enWkLLFJp4&p(#uEWmV) zfI?m9nIA=2cSIv450a%8x*Fs|lavLgDjL1`C5#|~qd+ahie)Me%KUhx1l z0Ub|8Hl7d5Tn9>3Ap~v~FSbnks0cIx72k+VN)*Ja5t#lvJ{Yz!GP4Dr(DN5_4XD&4 zp&HpZ2%Drb_=ez27Cs@^FJ_eA=HI{mfA(GoNaCX$0qsYnjQd02Q~noupLhe2WV(b1 zcm|-HV14J(y&fKDGK1T|B8~dT+rWZC(iE?!@2`rq*n|_+aLHJ_3$9X?q5MV7Tv&7| zrm@Y8zjB$+NJqE9<|sh<<8s~eZgIHuS3;r0VH&nI0&A?yZr?!?oBJvi>>Lx~&^twDgWhr$a;3{wcX z!JW%H-eY0r#~D1)41k&b@&t1~fT`Zc@O&iG_vH$%tACqg8G>Oh_4Lb~P#A9qlpFH& zP9D}#Ngf~v>8mpaX@P0nJR<5R&)4_yaB99MV zYP%_sDAI$RigzX-O$zZ2(MgR2;7f+)B(uoi+HQp7V=$^H@)}@gzKq!Cs_4rfcI_XJ z|AN7lAF?^&b6hT-zDQ@HHxh}nifN0}(dI5{%WG`L-L@9En9d0-Gqh?oGCxz^PPa

yHlr~Qj z%`kgh<2P>C>fTYE?E#Zh!{+2Qw=75K)1B;8ZJ3zCdDjI$qG`W%*$ojvA?sB=lZvgK zCFeTxA=XpCI{8fHWVEwdoN>)8KI3>wS1$ku!D@vDi!H##`d8bvA;7sf3*MOzNT&#^ z6;g_U-7z1Ji^{Am0x$ju^_X3VOn#pQQ_u;Ery^^ukw>}3FKln<4!Fg-PrZajr)_E1<>}I=v!q+(^ic#+0V+3yx3Z0nrya_ z9ic5(Ikj|7NP?0XaV4ST+E6HsCdv`M=q3j>e)^RmxA|<+tdj)5`<9`iZFSU6^%l5* zuUeaN*&D0)#-8)Fe8S>ey88ImsV>hoi8l7tzto01!b%xWUi?smIhTFWrN(* z72BPsG2KQLsTev>OM7u4F?%B<)XaC6+c>m+gLJt14bLXKdsoBql`8Ch7U`e5&WtBI z{7_XNoZW&^y+%(!etb)eRFCFwWNp11VzQfYOez$uKK4HTM0Tqzw##t8%t{NA6gj9W zKr&BClpUjOKiNRO!TZ#1dGtT= zB`TCkrZO!<(Z~t%LVQWIwqm8~$~fG4edEMFghmK%DbN7NvY2B^SOBG4jSsoeU9}I8 z@8tTrx#)0!Xk0e)MZ`Fi?_`7re_2^HlZb*ubafpShf`3ZQHVytq3Y_Yy!VIl$x_mk z4=1NlMp^cA)$r!Ekfy3uHS+39uf5rJpqII8@)&kPvu8s|XKlfWi*nPacSu_ocf{qc z+xaIq-h_5~osS{9#FPQ&ab=Z9DCd27WKnP7`JEqNIt4Mih~u8SY>LJssztE)gH8&1 zo7?yh*HL<>%aIbkUB;2UVY6-5xHtskHxzkB=KL#I`rI|7FOR8h83?)nmh`T}qu5h% zQWjOGpb_k!((<5@6aw=PODD3#6s27RkYmVFX7bHtkAD_PHnK>4bo@4=f40un2ISaZ zT*dnU7O4-Dn}eO`yK#}wA`O{eMAJn8;TFq&{Vj>EwfS1;EX%&RCIj(z_&GnYOCG*= zwdURH4UVPWsV0Lc#x`s1unv=`3@^@^dnq>ruZX5Nx190n~xHjIs1bmta%p3XQ;HW;dWus-?1PTxQh) zTo&#LVZXaVb-7~QO>QaTsjo9s|JE5c@9J1V{ndcBAc|v8VreFNW38yh^~0^ z0b;Cn#MZ0x-y<`c!rvJ&GLS)L$Mi~j!FC?X^IYlY~!7^!u=K`S0asx?9WJ`VOnME#>b-Xb@JrQG- zr5(}9i1&C=%^H_Ir3HO~9k{JaV}g?f_~p{Avg8mkb53wO!3WfW>>Wz1=%~{p^gcbW zKS!c|wH)MPm1XM06~_X-U>V7%5x}_>GOUo5M0~&DJ&YVY1tkdWOzZo_G^87HWV^JUE$HO3acF-XQ z+MH^-f^k$^xO}KuQ=&*qC}otWrr=C6BX_8~NKU4eX}OjoV4!&HCUn?2Bv4W`bMK@xJVgK%Up<|o zBI0#8S^-@%7*f5za7q*^w2;)zZmZru;SI7)F(0tJL5+UVAZg=|vfGSk$631oW1Ut^ z1_L6E*=(dzpt-5w0=T$QdW{hNfA|H7-D2&%m-u0XU)OVLJ&a5?T|?A!4O2Ucm%5Q9Qea6=O|vm?(voLlGudNwwm}k{+C`LbTmF=T z5rS3bW*+k13AaxniDC5b;o$6Rk=33KK+@qxqhe|?zt%m1$`}STyM7B z21-TZyt3Ga)$UF!(yzp{>Eps~TVLqdG1#n=M6lV0(P~-8o`^^y@=&2rLAn#nVm05f zaY~j-$-G$RtY3~A{LO&9Km@;LC*E5l@FrYm{^ zKJAg#f$PL%jYUBr)Hir5sGn@)={bU`+9f(d)>5!kp?iSJ25sX;KKaYZP$%Zn-;o1N z7;s0u&geOrpsh$p8QBw*A;N~N(pucAB1R7zW}POLuaIgf<@Ep*VCs`>W9Elsw`f%_ zk%{y$3mGxospU5L;HOsQI<7D$T3hZG^lM=`-#YbXg4t(pVt@h&J$w7NE7M+6eqof~ zDc!?A3%@=~jpoWA85f3mg#AW=s7u-qAf1MCP+JNKRdNTIZBe0WyQN97 zUtvi7c!Os|Rv_yPpq#vZ0UJ7`S;RH{d+HAtoL+JM#w^-owJ!-YvHZXmtJIbw4C+Kq z6jyD#gP8qhnPn5UEPPGeQcgj~S$0tFV8ML>^23b4x4n@>@VD!cNUpccQAU3*2Z3j# z+8+KxiX;S7f+bp%6hkBjXf7w@*8mNmaqy2M9u>VIB1Myn7xyq~Y_{O)xyraKctQH0 z?~NBFTNp<88^%1VKj*ZV2x5|XF*`l`Wp3_n_kO?DMgU~)xal9O1Y#BKn#5XLWJwqy z1)@^#BKt4hXk4}1D<|sr1QPp@;zSZ#6}jh1OHJfIO@$7d^_3D|Kpt4=GM)tImtJT> zgU9nNvxw6~6*6xbEY0SloDTm%7QL2yayPX5lwXp9tK%8JqSy63_6^)TkzL%3o} zc-?8@C?-^{(v{JP)I2^IH}&v*o5VO0I(I^@-Yw_!g*V8!%n(y&3r z_V%_g!9~|ZlYbCz%)}y)f8MQhMNp5!Cz%d*w6cwk=1D~2aYQg{F1eC13byfgd#)G< zEZz@&Y;tD3-*U4P0k6T~v7Q*oRCZvF-o`k`=vfVJn$9^3*kGB)?_)c?j}cG{U1-JO zyXb{>^n)efW_trzrdtwxS$Enxp4}g3lKV;0=o9npPXnMaaz zS3vrg8MfvefljB-XdU2Mwob`m%S_oOr_#1o`Mak!=}#fUxQB)as+A^>;-#>>1uZN{ zs+NoDCKaz6?9|~)u+hAZckk&uk&aH%tHgQR@6yW56xoFaxTeH^$+E8^*Y$Fkft7kl z%dYE1_7)v)qKR!c@RmB3o914w-S!^!A(g^QV@ex`XOM%CEv*1&3EvAp-B{wGS)2)) zZ$$I$Eg0S$q@ileW6b@YEtB{t^`TWt3sGTs_fuJzE41v9@Ia&Nz4ozqe)O{aJ72J@ zm*fK$Fftpa;g1*98=yQE+E=em`>XU-lqMPTT)qp*0j_8$RRbnc1owJl4Q#e;ms)|9 z2Xp*v>&$32XHtM3SxouMyghcezJH^W zIFx)fU|kyWBy}VOPVyC6DiNtA^qd5^Gs}Kw_~%XPBTWhcgNxh|b%gvDyoL;<3B$x=6@kASCN-9KVH$I;`3F?2+8j2rri z(6i_VCTT$HUTt}5V)PzJw!QWz46ZM0m3O@K1nQ>PuK2zLXl{|fBZ~(R1Ja~4$>MeT z<1j_9gbRWbmDHv~;6sXqHzuW+f^^@$Dpfi?zl1495W^E9U5P}ohPFMQGYGQcE=ii9 z3@A&KQtA+QYNI!E`@msN(Ts%37irtKZTr zcJTpy2?z06PMxVAXO3&Mf1AB7r-nWAqw+m_f4q$87#k) z6Tfl)mrG?cb(OZ<57m7A<6|wJWQ2y7gn$o`q&}>ndr&jcYTajGI zj0#HtKCeFWyGdRW7oOQvZGo{jZXxQ&+2l}zNDl}h z=t}ue@=MPpb{@pAWEi|wV4WvV&8J?AmmZU5HU=+xOOGY<1pbx} z<^0(d?6zBR10*GO%Q5$>S+2rI2J^wUt>>@A*qFCEfJ}2ls=3dj_0{^nwx!g~K>=6e zWs{OwSijrMBXLn3CI+x|A^tf)mF!mF${J6CzrURVzBimNA_xbU#eUqPinfVmORr4< z6qZjPf-*~ajJ^X|Obn(UuyUH1Vsm!uA0dut0B0@DQ3`%8A15y4G2KhPYWMC2#X~mx z#0Ri6&uda3+5G8*=n$(0bC*;TPqRnRjLVL;@fo}<->3AZjPwc{#0NA_Zn1#gfdT?1 zYq|6&GN6#^?(de2X<@tA7p;Uq8)zO)QmpB(~UT3Tfd@q&lr&dVTkzz z{ZB;lxlo>+|5+^{M*;%k`=7#_J-|(xqrn4IH;dJv)6m0C#KRY}xSB5p;#_rwM@lL= zh&W>KDp&vY+CumaJ$d2q;5_ePNh-Dlwt78Gd*0b{e|{tbeB3{_0cqccM0;(K75#FT zX_pYEVoyd9Juo9-aMVZcK8@~_5@rtk1r-`CwoY3Ftn-o_X;=?TPAiU`s1)V>x|9m| zJ6S&J07}AayiRR`b9IpQZnhN-fq6RsiEljq1icj)=IJRqSmg7GX&|5y}w+=U&V@wtyFqN1aaCU{7LusiK zW&i=rjQYp@D^Cq?RoSYwvC+DTy}G4Xk7Q-hjFWylUpaoSYI z&>g2q$0|K^liVTSFI1oAs$xGjBjXm%7q|ePMrbu>gp%)UAg0r|s+CDBzLFk5Q(N-J zy7~7S2-67y)=BLVdkLG#w}#yF`)(f^m7HvDB6Y)#VkxNe3|dzw?|LURBb2?+>{ack z2_;=D{FZL}kD}qWO>BsH7vGzDnktf}wtz`SQ&OjQ(D5NHRgHc75KAm&m@>C_#k369 zr0x{n{AG(!1*M2SCrh5^SrP`|l8}b9o6smM7z51j{rg1M@xn}BKh;KWa*A1B+f!?H z3c7a4%7HNKS=)-I*1+DuudI|%wbe1=enkeFe#8vA&{BOq zumn1_KyAQDxA3ocHBxwvc8)A^^&jlDpmKVI+AL+4x;H)L8lC;+3Md(XyXumYn#N{f zRc3{GVq1o`3ccr=-B$IOR8!h5bXA+oK-D^3edD(3;{cJnPO2>40T8N<7LCF zs1n%wZE0{DYIlq~YIhW18yfyEAK0}s>7ULesZzTTQ zL)SiCRG&fkZ`3@g7hOR*bzW%rz54zVi**z*?J}*Ir0`=@f3}%&I!M;p;!?2RWown? za3_`3ODncBEjHLMBQVXxSlInzu|fR_mI&{&##0LDGGk*r#K%Sd|{b3l))N z*=_TwbRdE(IpOQ@+~lpdpG>Wq<*VPp65tkF~I&r-rK2T ze5ag!qh}8VOin*$e^_&;jf^U(1-cGfUJ>nUo@*(I?D%_NBytL7_Qh#CBHHeYxJ1VB z!c_X6X~B5aL$4*-Rh{7qPk_Ok`G9bP*m8LM0g;i+WeshTV9FzlOLAt6)EZOVp3~<) znKvafZ+hK#R*e!-9Kpyn9I-%!)W6(=PVs+mfhukREY3zkiSP#aM4|Iwq{zWo? z0G6k3dANxSFaY?z+n~iS%bwiJ$r`A-Gzx)ix%%4&SZv@u zSypcZ;O=uCN7^Hz?5d~&`uX-HqQmp*Wj>;nZee;7{e~QGdHj$8e>EHj?=_Nr8l&!7 zv-Wi(4-Pxp`p?RpP;55My%=Db{8vl<4f3S}05C@QxVym#Eh&uM|jG8R1P&8hDniW$T*;Zu{xc3 zg>KJNcpGE?u=FB~95RgI2PBYuyVW}VO9p%@@hW@M+3%#`GOw@C4$Sy#66>)wuJNE8PNQ{8S^7ddoadRBf)RbmxSCU3#$; zL%W1hV++9DCkw-t9(zPhA#qdLE{AB+OytP@kbEeg1fFoUi?CDh{h!|?5>4znLJBwI zF2uIeHQuqIe=`ZUEPe#{O72X}2-Db2XmcNX2v)s5HwoM_HY^SD?19gsGd7>pZ){Sl@N%ey z2}Uag$*6e%_1qKU1co1Rr^xT%X`y4KyRAVWZ-gAF?1H9+eq0NwKn5z>qFt`&koghB zACn50u5e%Ld)7{b*6o3XKe%uwjsqw2slnM6sCmr&hF=hcU6_=z*TV09kk1oiX23)2 zc8tSRQWR9ecV^LHf4z+YrNByY55fxac${Qg3ntuRv2@{-&X)UuTqL20#s4a*|;( zJ%Z5~fu6ss4Wcblpc3Z1{4f4X6;y`5@~5JQe=7R_b#J?DWQ4_z`|YI3?7EX=#Z+?J zGJgcAdK{?G#Lx-|!NjQTamJEJ+35hoJ)Fqn74wYL?rW-E(G}w+x*@SpU`f=dvNV+C z;U?-rN&~K;!F#M(TeT^)o2KKbxJnGmV0CQMfeZD}3LOqJf6fV}kwuohtvWg~@K51& z-}B>7&8Awrd0-Ll2W|{sZ=pp@S1ObmrOwtZ*{VuCMyufNV3To!IH+|s7oPw*NE!4Z zZxgK+Tu+nm7`@sX2lyi`uAA&5zk|AJrP@RKX`OpAPW4pezFL1Ll6CvS4k`9NMD`tr zfVce%X{4a->Sg`PCYl!0Bi}+RPUUS!v~mm5J%!8!+IRCnLVHkd=L(X>_i zr5n|!=~Ql;r*q?<`1OsIi)Z$ayB#HT){Ow~FoI+rWG1hRdy-MQ9u2Op9jyUPJ0)&TwKk0O zi3M{d;slF`;72|n70KBicfm*nMA$$>SdG%bkV~116mA19PiREGP8fR%Ut058kxjI! z?17|HM&UkIkqcPbb0C*F%aBMXV6gAgQKmAgs(CMg<6$Dblp_Ooc)SZDxs>$#$Rk+v zBnS5w`E@bW=XprvmHYth4Gz&=q8VnWjIkY(j) z5s~e}I`5PxXyKwbRBC<54Yx%SPKhdcE7DU>cI3kJSQ@0)?*%5YaLyVQQl}!lsP+Fv zdZm;7o$mT6(#oGA<@lMF*gIJ;SU4G(+9cVcA^rC|cb5%3>6}vn?0dA_Af}0(D+U=zJF5eN_v=l|T*|8?+ZR8$Ems##)6X*iD%+gdgnlAIF!TchtaXlfs{i_e@McHfOjwmNinCu7t7Z0Gk%BiJKKQgc61+ zZP0d)r*5w{)EgEGe-*QFYV(7njrVG;x&^@L^7#i?L}5OByT5Fv@L$(0@{nrpcHOqJ zriCJn(25bJrkk&YSy}H{u>DKvNw{plOphymr?5TNipNw8X0%#HJ(S2f%&z-jR3q_sNTq1s%7&0Gt$P|xgVrQ~g9SOUti{HV&WvrH5L=c3Rtfw~*+qmFb27ivH= zfbRGyOrx9V%(8thJ~HUIAru0ZVNTWE-Op?T=V+-K(TwOA)5#*jN|Aa8wXINSK$E(I1wHAqAG!Fu~{$uvNxWtKljP z5?62fmwOZwlgnTrJ#-AV#QD~I`~xs#u)XDW@sfNtZe8e&a8`RF_WnqDY=qn6d_Wgk z0G~wHT}Cs912@ym)IT$|yg_Ag7>F;HJ!Am4-%F%0^`ylpiJi2iyuu z8)907bo$J<+}x4CMj;e_f)UN|!7DvbKUFZZ0+amRg9VnP9dh zQ4CL;xtnjE1abNr*g!DP4xfPhn_&Zs4r0E~_~A7FdU=3;go3mTKVXD)V#sp8)kC+W z58UjoMx210{7Nj!U#!YOHWPx;Ew0L%7>go4QLZ?;{6n0^Bjv6Vcq5x0UwDHDFLsxC z%cc{TLv%>AiU`|oGBjKdK8Z`xRJlE*g56y8%ueEz#2f`#TS$KrSp3Kb75foSH&C9X zz<~S_<3Ae}3n9nG~F~j_GCFNUAKv= z)R(&ciL5mJZo$Hcg(^T2Q}0GCC3?;6yr;l%)^qQ(t9hS~_cu~MvAWBHiFg=22AtQ1ul!T8?^=_u=ziBoscx#)IMjB~#4BzI$`c&p8+uK#8UVZD_*3W#jboPlb6h zN7^2BPwblV4VBZPb1dZU9KNJ0D&*hqAj=pRz!Ag+ zNw(C5qA_D)rklIcI_7xQNQG=P+^??H*L`iuCq74zV7ca{6U&+O_iDwMCjti*v~zTjmCt7 z;=T8z7`&v$Su@8#n{c9a2Y=5cUG2S^{;fnX{_9){ScC~36hNO`x@ENzFVmN#?8cyW zQ4>H$qKLXKc2QfyFgm@Pa$`_5v8Wy%ch4!f=Gr!7Msh0VA$5IJ^$b(Y3}*mIBSFLS zjqVmiUd8EQxs~GVjW;PHpi+qCnL!cWfngxTDj3y1f{m?59!JdzAuq^&(QwI|wqh>3 z+;=nwv}=hF#fJrSBffj>@XB0M#Z!&ra5dJ;tXt6@d#)}>*!uWMmwzK<8a@X(v$^bg zy)AQ?GuraWA)()aR^3wDT(#+-Yl~eJ*cj#2w@usd{^`5Kg`3?n66MtNyA1xbzgNpD z6B}re9&YJT*|&2}4Bj-^rw;$tXn2a|?+`=+2%~G5x%%?Ijllz97jWj5B12tgAO~u# z@}H1ajE$hSK}m$yz{>1YoA3#HeZ-#8mTgK9M9y6A3SmP;sXdUF^})!>rr7FIU5hm7 zt)tnLrYZ_a!xO;h%2O!I2=@DFp;VjC40lxxizzsa(#PG{G!Ibh!; zqJv{N`rq0JhZ#+{?H^>e{z+vN_#b3u6xV=C!7+g0u-iIiXo?rF0ER;>;)6i{323sR z`e7me??G??y@`#HvvZD?m7(rP!k2Vr28WkdtJy{)pP|hj$iGyk*7_qAejqFv_SA+1 zglSE$L~;DN@C>9@PT}@Jq*%mQLlocu!!Xdm4pW$b4Y~F~=&&MRx^vHCHv)m9-UxIy~ONLQl-w}Z^G5B}mm}VmcJ(Ck040Km z^ais%LteX4umg2>GT{YD6=L+rW`?M%Q|Qsa2us-{*T9LXK*uJ2WDb&BMPiqT3^`H& zWqrre>nw&Wr$8eg@-|ij#u})JBg<+sB)P2Is`Hq$LVc?c;~%p(U?C+DO8k@6r{8+j z+uDV6uC`Dt=5wQLR_M_!=CjZv`w^vAw#(KMjEmC0WM*0|r>8U5Oid<#x$*=tv6$@2 z1%5jW}YtyNbUY`3>G)EbTas9|0It=4F6QbJar!|EefU&#j#t}r!iZ>jZ= zr{}9Dyap;M>1>qnNnsT&mg5BK6;D`0w@3s=Tw&7bCUkW6e__Fk|EaS5b*~|2a=CKZ zU}(KwZ3h)riMOd9LR?yN@gbJX#f=Fs;m#iHmQfSi1v>f0wCXeJ>1a01iiXDo__uba z$lFe5vl!6}Rv<~)AQ`WtJn8&E8`YXA4Y*of?=i{3(kX)k3#lrk8@PEhq%HR2Ny-(K z2v02Y3F&NYs;F+0i2=1pwZXQrw`v8As$r9ZCp&C|{V3+5Hx8GgacfDRnBO2y*GUvt zo4Z$zM6l->QeMBUHhhW~m&ZW`oFwnFkkmxm;>+>{5oSiS9w}lxl9A5a6fRBRxIWFo zQA3$*%Nn7&n9*E25!->EqZcK)s)=N!S*^EE`=6dkgNI~|=?UwC-9SQHZ_J|BYqE7H z*8g6=7~&qD0HG2NcL1i;$H0P3Wcx;LM@guRi?26LU(rqi&WfNkVplloB-B;0}m<}+~i=cE-p+n|TXh3#Mm%z&Ug}vODE}%L+ zHA%v#J6ch<%NeHE11u3)70N?xHC;7wc(cJmICL%Q%Wk&kfpgt}00>ZeN|ju#3%dku z+)^b2o)VRe3J4wTX%C-2*%>TgOERJ20m}LdTwUhy4zp_67O-K?idqS%ObQV<41`&} zS^wk~t~6n+NkYaCz@;jconW^jbzryrap1P9#dilTMau)|W}!xT+GEJ+LYpJ4{(847 zDDt9Sz$XqgGZo7L{&WPnl!vzI&cv_9Si6?B^RR8$Nou-bA}5p+={YeWk-gu*MnDZQ zmNhQM2fM&fhix(S+^FK{39r{wZ@KIZ(jA3fB)1cF6_3Ts95IW~r_n&-kwqPpz>f@8 zGK=&QX;2s1V>_kj%6T-et~6?o*tUnLMYCvhlvGAL=7H-1CeCfdXwhS^oMM!{KK?dC zhUln`LSA;N*RmYyIQ0;5P)cl3YG67g`E15#9sL%u8@LSJqHe>w!y}`9-vS?LBx;*- z*V63hFOH1CV4ii=n`ZT_4O|M-LWkp}NVdLKoXH8@B6FvRaj9o%+_rHAj??0j-P?%6 z6zQdSHceLsU_|{y%rLW%Qb)pd2LTvO+jJTHiM$W>MS2;YEuHcLIF2AfxAI1EfvrXG z759!a@bmB|!ntvN!M*-$(TxY)AwFl=;Vr~rirwxTj~I>*QICvvnB3Uu zz$*=u8cEZ}iVyOQ&@D(3V@4`2)W#YH9}f%DjnLuoHlT-UX5UskHFnmpRQ56(UJk7t zI{qZ#(uk3#+UWbd9@kEt4<>t$lrEP${Y!0B7RimLI9nz%i6DDUB#H?2;h)1%9*)po z9Exy%c5gLYT?6F6LIf+^i085J(&9as64>!u2yB6&8Ju`B6UF6Bo&wGF_-Ana67(axgbJ{ET9OESa1Ez60$&?0iMij*+#C10&6I)I}3q1;r1d zu9|;A)$%Lm^!lu$UD#FRTYK%NaYuQ$|Dgo_ zfLdnPa?l@SBPjqI8Khh;GnwiLc$fLI2rNys8Yo1V~= zm0iOL`g%uq1{UvSgQfdgX#AftM!tV5X~1X}ETQthDTtc{Nj(2)S@YYeW55Hz8X5Uq zu;aa~;$|fc-n&BX)|^;&kYUIK{9G$2zH~8?!p=Z<-I~UP4--J5;DnA~>moS-o!j=l zw)K`DTYf#CaD!t%AVJ?XZclSMwbJeQZ3qMk?OJ$-H!bwMKH{+IQOc@4jdEq;cEfi$IlJ9ddzYtFQGcWZ83btpIhaB}+pK_;p}IEa8uR zIf`GqJJk^O`TRP@!HZTjzr|r`%s=Asmaw*k(9>~Yb@)JJ-~crGE86mOZ2Y(pn#*4) z=E#@wFU%my&4W?1VOw{tct~L1V7j)wS^s8KL)TG*e_MSy#(`T=KEXj2+P~mYUnhbx zkRDDe4tZj;ewqCwZ>EM-0LIPZJ}R=Ve4rG%kXpY^eLY5!wGX=)5>+Hx4f;Ir$5F@l zK3|HgMUqwIh)bo|zgzBNRGgbPWtXJ9;blHb;zw5HYau^@(tApI?*LlT%15dukY4`j z@q(^VDlL8s2^pU5qw(4mTIrdB?#f02GE`M<&DAI;G2NXg=oN)(z$3&*Px)5Npud0> zz1o1>@6O5vog|IqGF|mg!sA8iFJ(8hwet*OSBc_WWUUns+uRGDuYG>nQu@T&+NNHF zrLaXAq_fq88JjJ48*?)T`MPy`vGB+;3Z;Q3URgtASuvFJdUzT~{>?{7W02MZ;D>xH z4P%leLlhHR7W`3k0B;P;?b>>z!2xl%%;a-DTwW2_*a9_);iO0N1eIl)v5O=X_mQkk z8hNl8ikl=w;bI7V2QbEzT=<0k@R8D&A2`nu*TeW!yXwv`$DxQW6`-H(4y!gv;J}M3 z6vx>qJ(c>2V8rtLXb8bUV6%%6>qi!f%NMP*nk_y9>z&dGSa-p8&kBUNMRbWUVe%7= z<^A0dpR1H;fQib!W)>! z$Wb=={zAnzGh#B~(pK&_x^R%KtOAcavllH4T{C?T>ooObQ7~Vl`qj#cx`@jX zOjAp28XwL>xi61_q`}0V+aMO6_TwY9S$%U1WX_h%p^jg9d${Tm)h(6_kufQ@qt((I zX)2$a5X3({I}mE!6aBuc_Fxp7->?Wy6kX@SST0TkP!VI8-E#j3Y7EfK9aI7S+@m;_ z+pm~0H5h8=j63NLIO$EWD1FG0o1rL}=bE{HS(AZ%pyX50?8JhgqkUvSdAp&dlg};S zTbjdi4OQ9WnpJ$TI$gfW4n5g`-o6DZ#Zzi}M=&AIfZqe#B`lL%j&V}@{7?#esBh~7b9gkx}G zi}TJ2Orz~&E8dvGy>TQM5|)hV(hW}oLRW()lAf>WPZ>w&Ft)5b6QND{-3VSJsPS!4&eILoa8y> zF^rq?+#14qbZA2ADAAf^IW3_{LsA(@Lzd}wiX4wxztrw}ZSCx8dXP{#r@BOmN>tl( zjWJ9zCMIpt1N)mB+Pn9k-}n2Q&-Z)popbN~4c*<4qQA*Qwdpx=`=ar`MyjA)=TPVj(d-n08Z;$`OZaF0^yEZ&JDd+g%Zn=l$&+uh@K{Pw$6<)HL^Gt>_MJCo8fd|H80eCo5~iE+~0ScyWCJ* z!+v&WM_=34an9!x+DU;UjWraLi%E)4b$r$(3B9xtb^*Gg1;hEmqH>TE>f%mBYQN8g`;?eizdzJqapW8M zn0Iws_;WqzB4Jj?b(+qAo&8K$EMY)B#cE(R6LzE-A<+;D6;2>e6ILnQu+*CHdRJ6^ z`4q*gd{CBZ>JZ`lIfyrh3kTe=(gWvToJ1L^3-n+?Av^HRxS#0CfiG z7-h-VX;gjV!M>BQE({xF0p~DMEgD=3B%4UFzQG3S4za+E$VpWfh7UObtr${Ow$6vd z5FPuv)&klHyc#S}u`o*OI)yRX^@W)|+c$+5oxCRj@}&%Hx;+cARurBufTy)> zpjj6Svp-T84nJaaovD+G@cP5(M=RLg&A`+>VFBnNB2X7Tdx}7# z2tS)mLPumYXeYD5)ZHzoPzco)J#8)&kdrqFT4H2N0rHltjfz?*(8{AEq>|au$ns*i zu*V4ed<;$cL17Oaqm+J9EZ3eOE!%qRX=Kd|oIsX)O36u&UOS9Zc0jRAItd%x7ejHc zE%yJk?-VD(Q$z^zAg_Uv=A9zYD8dhy!w&W`Nc7TaWRe$_$&J7vG3j2N+m*|WX=I+P z;H443&rQzTVq{hV{b^UwyX;Ky$gd=C;Ki!BYOfe2KurOgsz}gjwK)k=0@M_6yas`m zFtN`GY;1;#@I~-W9}DpABheC?zFG>hAHbkjF(Bd*L>*Sf>jP*g1+M;bxN7*L*VE~- GTKgBj+ffbx diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index a89d7a90b..508322917 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,12 +1,6 @@ -# -# Copyright OpenSearch Contributors -# SPDX-License-Identifier: Apache-2.0 -# - distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-7.5-all.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-7.6.1-bin.zip +networkTimeout=10000 zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists -distributionSha256Sum=97a52d145762adc241bad7fd18289bf7f6801e08ece6badf80402fe2b9f250b1 - diff --git a/gradlew b/gradlew index 8250acdd7..65dcd68d6 100755 --- a/gradlew +++ b/gradlew @@ -1,17 +1,7 @@ -#!/usr/bin/env sh +#!/bin/sh # -# SPDX-License-Identifier: Apache-2.0 -# -# The OpenSearch Contributors require contributions made to -# this file be licensed under the Apache-2.0 license or a -# compatible open source license. -# -# Modifications Copyright OpenSearch Contributors. See -# GitHub history for details. -# -# -# Copyright 2015 the original author or authors. +# Copyright © 2015-2021 the original authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -27,78 +17,113 @@ # ############################################################################## -## -## Gradle start up script for UN*X -## +# +# Gradle start up script for POSIX generated by Gradle. +# +# Important for running: +# +# (1) You need a POSIX-compliant shell to run this script. If your /bin/sh is +# noncompliant, but you have some other compliant shell such as ksh or +# bash, then to run this script, type that shell name before the whole +# command line, like: +# +# ksh Gradle +# +# Busybox and similar reduced shells will NOT work, because this script +# requires all of these POSIX shell features: +# * functions; +# * expansions «$var», «${var}», «${var:-default}», «${var+SET}», +# «${var#prefix}», «${var%suffix}», and «$( cmd )»; +# * compound commands having a testable exit status, especially «case»; +# * various built-in commands including «command», «set», and «ulimit». +# +# Important for patching: +# +# (2) This script targets any POSIX shell, so it avoids extensions provided +# by Bash, Ksh, etc; in particular arrays are avoided. +# +# The "traditional" practice of packing multiple parameters into a +# space-separated string is a well documented source of bugs and security +# problems, so this is (mostly) avoided, by progressively accumulating +# options in "$@", and eventually passing that to Java. +# +# Where the inherited environment variables (DEFAULT_JVM_OPTS, JAVA_OPTS, +# and GRADLE_OPTS) rely on word-splitting, this is performed explicitly; +# see the in-line comments for details. +# +# There are tweaks for specific operating systems such as AIX, CygWin, +# Darwin, MinGW, and NonStop. +# +# (3) This script is generated from the Groovy template +# https://github.com/gradle/gradle/blob/HEAD/subprojects/plugins/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt +# within the Gradle project. +# +# You can find Gradle at https://github.com/gradle/gradle/. +# ############################################################################## # Attempt to set APP_HOME + # Resolve links: $0 may be a link -PRG="$0" -# Need this for relative symlinks. -while [ -h "$PRG" ] ; do - ls=`ls -ld "$PRG"` - link=`expr "$ls" : '.*-> \(.*\)$'` - if expr "$link" : '/.*' > /dev/null; then - PRG="$link" - else - PRG=`dirname "$PRG"`"/$link" - fi +app_path=$0 + +# Need this for daisy-chained symlinks. +while + APP_HOME=${app_path%"${app_path##*/}"} # leaves a trailing /; empty if no leading path + [ -h "$app_path" ] +do + ls=$( ls -ld "$app_path" ) + link=${ls#*' -> '} + case $link in #( + /*) app_path=$link ;; #( + *) app_path=$APP_HOME$link ;; + esac done -SAVED="`pwd`" -cd "`dirname \"$PRG\"`/" >/dev/null -APP_HOME="`pwd -P`" -cd "$SAVED" >/dev/null -APP_NAME="Gradle" -APP_BASE_NAME=`basename "$0"` +# This is normally unused +# shellcheck disable=SC2034 +APP_BASE_NAME=${0##*/} +APP_HOME=$( cd "${APP_HOME:-./}" && pwd -P ) || exit # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' # Use the maximum available, or set MAX_FD != -1 to use that value. -MAX_FD="maximum" +MAX_FD=maximum warn () { echo "$*" -} +} >&2 die () { echo echo "$*" echo exit 1 -} +} >&2 # OS specific support (must be 'true' or 'false'). cygwin=false msys=false darwin=false nonstop=false -case "`uname`" in - CYGWIN* ) - cygwin=true - ;; - Darwin* ) - darwin=true - ;; - MINGW* ) - msys=true - ;; - NONSTOP* ) - nonstop=true - ;; +case "$( uname )" in #( + CYGWIN* ) cygwin=true ;; #( + Darwin* ) darwin=true ;; #( + MSYS* | MINGW* ) msys=true ;; #( + NONSTOP* ) nonstop=true ;; esac CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar + # Determine the Java command to use to start the JVM. if [ -n "$JAVA_HOME" ] ; then if [ -x "$JAVA_HOME/jre/sh/java" ] ; then # IBM's JDK on AIX uses strange locations for the executables - JAVACMD="$JAVA_HOME/jre/sh/java" + JAVACMD=$JAVA_HOME/jre/sh/java else - JAVACMD="$JAVA_HOME/bin/java" + JAVACMD=$JAVA_HOME/bin/java fi if [ ! -x "$JAVACMD" ] ; then die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME @@ -107,7 +132,7 @@ Please set the JAVA_HOME variable in your environment to match the location of your Java installation." fi else - JAVACMD="java" + JAVACMD=java which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. Please set the JAVA_HOME variable in your environment to match the @@ -115,79 +140,105 @@ location of your Java installation." fi # Increase the maximum file descriptors if we can. -if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then - MAX_FD_LIMIT=`ulimit -H -n` - if [ $? -eq 0 ] ; then - if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then - MAX_FD="$MAX_FD_LIMIT" - fi - ulimit -n $MAX_FD - if [ $? -ne 0 ] ; then - warn "Could not set maximum file descriptor limit: $MAX_FD" - fi - else - warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" - fi +if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then + case $MAX_FD in #( + max*) + # In POSIX sh, ulimit -H is undefined. That's why the result is checked to see if it worked. + # shellcheck disable=SC3045 + MAX_FD=$( ulimit -H -n ) || + warn "Could not query maximum file descriptor limit" + esac + case $MAX_FD in #( + '' | soft) :;; #( + *) + # In POSIX sh, ulimit -n is undefined. That's why the result is checked to see if it worked. + # shellcheck disable=SC3045 + ulimit -n "$MAX_FD" || + warn "Could not set maximum file descriptor limit to $MAX_FD" + esac fi -# For Darwin, add options to specify how the application appears in the dock -if $darwin; then - GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" -fi +# Collect all arguments for the java command, stacking in reverse order: +# * args from the command line +# * the main class name +# * -classpath +# * -D...appname settings +# * --module-path (only if needed) +# * DEFAULT_JVM_OPTS, JAVA_OPTS, and GRADLE_OPTS environment variables. # For Cygwin or MSYS, switch paths to Windows format before running java -if [ "$cygwin" = "true" -o "$msys" = "true" ] ; then - APP_HOME=`cygpath --path --mixed "$APP_HOME"` - CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` - JAVACMD=`cygpath --unix "$JAVACMD"` - - # We build the pattern for arguments to be converted via cygpath - ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` - SEP="" - for dir in $ROOTDIRSRAW ; do - ROOTDIRS="$ROOTDIRS$SEP$dir" - SEP="|" - done - OURCYGPATTERN="(^($ROOTDIRS))" - # Add a user-defined pattern to the cygpath arguments - if [ "$GRADLE_CYGPATTERN" != "" ] ; then - OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" - fi +if "$cygwin" || "$msys" ; then + APP_HOME=$( cygpath --path --mixed "$APP_HOME" ) + CLASSPATH=$( cygpath --path --mixed "$CLASSPATH" ) + + JAVACMD=$( cygpath --unix "$JAVACMD" ) + # Now convert the arguments - kludge to limit ourselves to /bin/sh - i=0 - for arg in "$@" ; do - CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` - CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option - - if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition - eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` - else - eval `echo args$i`="\"$arg\"" + for arg do + if + case $arg in #( + -*) false ;; # don't mess with options #( + /?*) t=${arg#/} t=/${t%%/*} # looks like a POSIX filepath + [ -e "$t" ] ;; #( + *) false ;; + esac + then + arg=$( cygpath --path --ignore --mixed "$arg" ) fi - i=`expr $i + 1` + # Roll the args list around exactly as many times as the number of + # args, so each arg winds up back in the position where it started, but + # possibly modified. + # + # NB: a `for` loop captures its iteration list before it begins, so + # changing the positional parameters here affects neither the number of + # iterations, nor the values presented in `arg`. + shift # remove old arg + set -- "$@" "$arg" # push replacement arg done - case $i in - 0) set -- ;; - 1) set -- "$args0" ;; - 2) set -- "$args0" "$args1" ;; - 3) set -- "$args0" "$args1" "$args2" ;; - 4) set -- "$args0" "$args1" "$args2" "$args3" ;; - 5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; - 6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; - 7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; - 8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; - 9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; - esac fi -# Escape application args -save () { - for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done - echo " " -} -APP_ARGS=`save "$@"` +# Collect all arguments for the java command; +# * $DEFAULT_JVM_OPTS, $JAVA_OPTS, and $GRADLE_OPTS can contain fragments of +# shell script including quotes and variable substitutions, so put them in +# double quotes to make sure that they get re-expanded; and +# * put everything else in single quotes, so that it's not re-expanded. + +set -- \ + "-Dorg.gradle.appname=$APP_BASE_NAME" \ + -classpath "$CLASSPATH" \ + org.gradle.wrapper.GradleWrapperMain \ + "$@" + +# Stop when "xargs" is not available. +if ! command -v xargs >/dev/null 2>&1 +then + die "xargs is not available" +fi + +# Use "xargs" to parse quoted args. +# +# With -n1 it outputs one arg per line, with the quotes and backslashes removed. +# +# In Bash we could simply go: +# +# readarray ARGS < <( xargs -n1 <<<"$var" ) && +# set -- "${ARGS[@]}" "$@" +# +# but POSIX shell has neither arrays nor command substitution, so instead we +# post-process each arg (as a line of input to sed) to backslash-escape any +# character that might be a shell metacharacter, then use eval to reverse +# that process (while maintaining the separation between arguments), and wrap +# the whole thing up as a single "set" statement. +# +# This will of course break if any of these variables contains a newline or +# an unmatched quote. +# -# Collect all arguments for the java command, following the shell quoting and substitution rules -eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS" +eval "set -- $( + printf '%s\n' "$DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS" | + xargs -n1 | + sed ' s~[^-[:alnum:]+,./:=@_]~\\&~g; ' | + tr '\n' ' ' + )" '"$@"' exec "$JAVACMD" "$@" diff --git a/gradlew.bat b/gradlew.bat index 9109989e3..6689b85be 100644 --- a/gradlew.bat +++ b/gradlew.bat @@ -14,7 +14,7 @@ @rem limitations under the License. @rem -@if "%DEBUG%" == "" @echo off +@if "%DEBUG%"=="" @echo off @rem ########################################################################## @rem @rem Gradle startup script for Windows @@ -25,7 +25,8 @@ if "%OS%"=="Windows_NT" setlocal set DIRNAME=%~dp0 -if "%DIRNAME%" == "" set DIRNAME=. +if "%DIRNAME%"=="" set DIRNAME=. +@rem This is normally unused set APP_BASE_NAME=%~n0 set APP_HOME=%DIRNAME% @@ -40,7 +41,7 @@ if defined JAVA_HOME goto findJavaFromJavaHome set JAVA_EXE=java.exe %JAVA_EXE% -version >NUL 2>&1 -if "%ERRORLEVEL%" == "0" goto init +if %ERRORLEVEL% equ 0 goto execute echo. echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. @@ -54,7 +55,7 @@ goto fail set JAVA_HOME=%JAVA_HOME:"=% set JAVA_EXE=%JAVA_HOME%/bin/java.exe -if exist "%JAVA_EXE%" goto init +if exist "%JAVA_EXE%" goto execute echo. echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% @@ -64,38 +65,26 @@ echo location of your Java installation. goto fail -:init -@rem Get command-line arguments, handling Windows variants - -if not "%OS%" == "Windows_NT" goto win9xME_args - -:win9xME_args -@rem Slurp the command line arguments. -set CMD_LINE_ARGS= -set _SKIP=2 - -:win9xME_args_slurp -if "x%~1" == "x" goto execute - -set CMD_LINE_ARGS=%* - :execute @rem Setup the command line set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar + @rem Execute Gradle -"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS% +"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %* :end @rem End local scope for the variables with windows NT shell -if "%ERRORLEVEL%"=="0" goto mainEnd +if %ERRORLEVEL% equ 0 goto mainEnd :fail rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of rem the _cmd.exe /c_ return code! -if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 -exit /b 1 +set EXIT_CODE=%ERRORLEVEL% +if %EXIT_CODE% equ 0 set EXIT_CODE=1 +if not ""=="%GRADLE_EXIT_CONSOLE%" exit %EXIT_CODE% +exit /b %EXIT_CODE% :mainEnd if "%OS%"=="Windows_NT" endlocal From 220963238ad9256b91832cf71e01b4d33bc4ac51 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Wed, 3 May 2023 16:34:38 -0500 Subject: [PATCH 102/416] Baseline Codeowners (#889) (#896) Signed-off-by: Naveen Tatikonda (cherry picked from commit 36fc01cc6799b8c00b2189706a1308d9e300289d) Co-authored-by: Naveen Tatikonda --- .github/CODEOWNERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 2c0e88993..2ca3ba012 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,2 +1,2 @@ # This should match the owning team set up in https://github.com/orgs/opensearch-project/teams -* @opensearch-project/k-nn \ No newline at end of file +* @heemin32 @navneet1v @VijayanB @vamshin @jmazanec15 @naveentatikonda @junqiu-lei @martin-gaievski \ No newline at end of file From 485a748c83cf69dd69d1a199ff07352ddf053393 Mon Sep 17 00:00:00 2001 From: Naveen Tatikonda Date: Wed, 3 May 2023 19:25:10 -0500 Subject: [PATCH 103/416] Baseline Maintainers (#883) (#890) Signed-off-by: Naveen Tatikonda --- MAINTAINERS.md | 78 +++++++------------------------------------------- 1 file changed, 11 insertions(+), 67 deletions(-) diff --git a/MAINTAINERS.md b/MAINTAINERS.md index 6c8ef5ec2..c285c166c 100644 --- a/MAINTAINERS.md +++ b/MAINTAINERS.md @@ -1,71 +1,15 @@ -- [Overview](#overview) -- [Current Maintainers](#current-maintainers) -- [Maintainer Responsibilities](#maintainer-responsibilities) - - [Uphold Code of Conduct](#uphold-code-of-conduct) - - [Prioritize Security](#prioritize-security) - - [Review Pull Requests](#review-pull-requests) - - [Triage Open Issues](#triage-open-issues) - - [Be Responsive](#be-responsive) - - [Maintain Overall Health of the Repo](#maintain-overall-health-of-the-repo) - - [Use Semver](#use-semver) - - [Release Frequently](#release-frequently) - - [Promote Other Maintainers](#promote-other-maintainers) - ## Overview -This document explains who the maintainers are (see below), what they do in this repo, and how they should be doing it. If you're interested in contributing, see [CONTRIBUTING](CONTRIBUTING.md). - +This document contains a list of maintainers in this repo. See [opensearch-project/.github/RESPONSIBILITIES.md](https://github.com/opensearch-project/.github/blob/main/RESPONSIBILITIES.md#maintainer-responsibilities) that explains what the role of maintainer means, what maintainers do in this and other repos, and how they should be doing it. If you're interested in contributing, and becoming a maintainer, see [CONTRIBUTING](CONTRIBUTING.md). ## Current Maintainers -| Maintainer | GitHub ID | Affiliation | -| ------------------------| --------------------------------------------| ----------| -| Jack Mazanec | [jmazanec15](https://github.com/jmazanec15) | Amazon | -| Vamshi Vijay Nakkirtha | [vamshin](https://github.com/vamshin) | Amazon | -| Vijayan Balasubramanian | [VijayanB](https://github.com/VijayanB) | Amazon | - -## Maintainer Responsibilities - -Maintainers are active and visible members of the community, and have [maintain-level permissions on a repository](https://docs.github.com/en/organizations/managing-access-to-your-organizations-repositories/repository-permission-levels-for-an-organization). Use those privileges to serve the community and evolve code as follows. - -### Uphold Code of Conduct - -Model the behavior set forward by the [Code of Conduct](CODE_OF_CONDUCT.md) and raise any violations to other maintainers and admins. - -### Prioritize Security - -Security is your number one priority. Maintainer's Github keys must be password protected securely and any reported security vulnerabilities are addressed before features or bugs. - -Note that this repository is monitored and supported 24/7 by Amazon Security, see [Reporting a Vulnerability](SECURITY.md) for details. - -### Review Pull Requests - -Review pull requests regularly, comment, suggest, reject, merge and close. Accept only high quality pull-requests. Provide code reviews and guidance on incomming pull requests. Don't let PRs be stale and do your best to be helpful to contributors. - -### Triage Open Issues - -Manage labels, review issues regularly, and triage by labelling them. - -All repositories in this organization have a standard set of labels, including `bug`, `documentation`, `duplicate`, `enhancement`, `good first issue`, `help wanted`, `blocker`, `invalid`, `question`, `wontfix`, and `untriaged`, along with release labels, such as `v1.0.0`, `v1.1.0` and `v2.0.0`, and `backport`. - -Use labels to target an issue or a PR for a given release, add `help wanted` to good issues for new community members, and `blocker` for issues that scare you or need immediate attention. Request for more information from a submitter if an issue is not clear. Create new labels as needed by the project. - -### Be Responsive - -Respond to enhancement requests, and forum posts. Allocate time to reviewing and commenting on issues and conversations as they come in. - -### Maintain Overall Health of the Repo - -Keep the `main` branch at production quality at all times. Backport features as needed. Cut release branches and tags to enable future patches. - -### Use Semver - -Use and enforce [semantic versioning](https://semver.org/) and do not let breaking changes be made outside of major releases. - -### Release Frequently - -Make frequent project releases to the community. - -### Promote Other Maintainers - -Assist, add, and remove [MAINTAINERS](MAINTAINERS.md). Exercise good judgement, and propose high quality contributors to become co-maintainers. - +| Maintainer | GitHub ID | Affiliation | +|-------------------------|-------------------------------------------------------|-------------| +| Heemin Kim | [heemin32](https://github.com/heemin32) | Amazon | +| Jack Mazanec | [jmazanec15](https://github.com/jmazanec15) | Amazon | +| Junqiu Lei | [junqiu-lei](https://github.com/junqiu-lei) | Amazon | +| Martin Gaievski | [martin-gaievski](https://github.com/martin-gaievski) | Amazon | +| Naveen Tatikonda | [naveentatikonda](https://github.com/naveentatikonda) | Amazon | +| Navneet Verma | [navneet1v](https://github.com/navneet1v) | Amazon | +| Vamshi Vijay Nakkirtha | [vamshin](https://github.com/vamshin) | Amazon | +| Vijayan Balasubramanian | [VijayanB](https://github.com/VijayanB) | Amazon | From 7b182b8d7490ceaa677048772f6c80d71c6560dc Mon Sep 17 00:00:00 2001 From: John Mazanec Date: Wed, 10 May 2023 16:02:35 -0700 Subject: [PATCH 104/416] Fix checksum for gradle (#902) Signed-off-by: John Mazanec --- gradle/wrapper/gradle-wrapper.properties | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 508322917..58e3982e3 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,6 +1,7 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-7.6.1-bin.zip +distributionSha256Sum=518a863631feb7452b8f1b3dc2aaee5f388355cc3421bbd0275fbeadd77e84b2 +distributionUrl=https\://services.gradle.org/distributions/gradle-7.6.1-all.zip networkTimeout=10000 zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists From 9a1606fcb6ca87b5565c663827ba087d8eff8ae8 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Wed, 24 May 2023 14:23:56 -0700 Subject: [PATCH 105/416] Add Auto Release Workflow (#898) (#912) Signed-off-by: Naveen Tatikonda (cherry picked from commit 7a5c101ceb7825e795dbb1f84381248daf87106b) Co-authored-by: Naveen Tatikonda --- .github/workflows/auto-release.yml | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 .github/workflows/auto-release.yml diff --git a/.github/workflows/auto-release.yml b/.github/workflows/auto-release.yml new file mode 100644 index 000000000..c67e12bad --- /dev/null +++ b/.github/workflows/auto-release.yml @@ -0,0 +1,28 @@ +name: Releases + +on: + push: + tags: + - '*' + +jobs: + build: + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - name: GitHub App token + id: github_app_token + uses: tibdex/github-app-token@v1.5.0 + with: + app_id: ${{ secrets.APP_ID }} + private_key: ${{ secrets.APP_PRIVATE_KEY }} + installation_id: 22958780 + - name: Get tag + id: tag + uses: dawidd6/action-get-tag@v1 + - uses: actions/checkout@v2 + - uses: ncipollo/release-action@v1 + with: + github_token: ${{ steps.github_app_token.outputs.token }} + bodyFile: release-notes/opensearch-knn.release-notes-${{steps.tag.outputs.tag}}.md From 591c3f95b6ff9b27c229c0a0cb38b46969776008 Mon Sep 17 00:00:00 2001 From: Martin Gaievski Date: Thu, 25 May 2023 16:28:28 -0700 Subject: [PATCH 106/416] Do refresh only for non system indices (#915) (#916) Signed-off-by: Martin Gaievski (cherry picked from commit a5214a3dfe86d5790fe204385bc92af147f98434) --- CHANGELOG.md | 1 + .../org/opensearch/knn/index/FaissIT.java | 4 +- .../org/opensearch/knn/KNNRestTestCase.java | 42 ++++++++++++++++++- 3 files changed, 44 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 88a06ea6f..cde84481f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), * Bulk allocate objects for nmslib index creation to avoid malloc fragmentation ([#773](https://github.com/opensearch-project/k-NN/pull/773)) ### Bug Fixes ### Infrastructure +* Disable index refresh for system indices ([#773](https://github.com/opensearch-project/k-NN/pull/915)) ### Documentation ### Maintenance ### Refactoring \ No newline at end of file diff --git a/src/test/java/org/opensearch/knn/index/FaissIT.java b/src/test/java/org/opensearch/knn/index/FaissIT.java index d4a66a800..8eb99b625 100644 --- a/src/test/java/org/opensearch/knn/index/FaissIT.java +++ b/src/test/java/org/opensearch/knn/index/FaissIT.java @@ -55,7 +55,7 @@ public static void setUpClass() throws IOException { testData = new TestUtils.TestData(testIndexVectors.getPath(), testQueries.getPath()); } - public void testEndToEnd_fromMethod() throws IOException, InterruptedException { + public void testEndToEnd_fromMethod() throws Exception { String indexName = "test-index-1"; String fieldName = "test-field-1"; @@ -106,7 +106,7 @@ public void testEndToEnd_fromMethod() throws IOException, InterruptedException { } // Assert we have the right number of documents in the index - refreshAllIndices(); + refreshAllNonSystemIndices(); assertEquals(testData.indexData.docs.length, getDocCount(indexName)); int k = 10; diff --git a/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java b/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java index 89bc030eb..c4e3cbbc7 100644 --- a/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java +++ b/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java @@ -9,8 +9,12 @@ import com.google.common.io.Resources; import com.google.common.primitives.Floats; import org.apache.commons.lang.StringUtils; +import org.apache.http.util.EntityUtils; import org.opensearch.common.bytes.BytesReference; import org.opensearch.common.xcontent.XContentHelper; +import org.opensearch.core.xcontent.DeprecationHandler; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.core.xcontent.XContentParser; import org.opensearch.index.query.MatchAllQueryBuilder; import org.opensearch.knn.index.query.KNNQueryBuilder; import org.opensearch.knn.index.KNNSettings; @@ -20,7 +24,6 @@ import org.opensearch.knn.indices.ModelState; import org.opensearch.knn.plugin.KNNPlugin; import org.opensearch.knn.plugin.script.KNNScoringScriptEngine; -import org.apache.http.util.EntityUtils; import org.junit.AfterClass; import org.junit.Before; import org.opensearch.client.Request; @@ -59,6 +62,7 @@ import java.util.Locale; import java.util.Map; import java.util.PriorityQueue; +import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; @@ -111,6 +115,7 @@ public class KNNRestTestCase extends ODFERestTestCase { private static final String DOCUMENT_FIELD_FOUND = "found"; protected static final int DELAY_MILLI_SEC = 1000; protected static final int NUM_OF_ATTEMPTS = 30; + private static final String SYSTEM_INDEX_PREFIX = ".opendistro"; @AfterClass public static void dumpCoverage() throws IOException, MalformedObjectNameException { @@ -1266,4 +1271,39 @@ public interface IProxy { void reset(); } + + protected void refreshAllNonSystemIndices() throws Exception { + Response response = adminClient().performRequest(new Request("GET", "/_cat/indices?format=json&expand_wildcards=all")); + XContentType xContentType = XContentType.fromMediaType(response.getEntity().getContentType().getValue()); + try ( + XContentParser parser = xContentType.xContent() + .createParser( + NamedXContentRegistry.EMPTY, + DeprecationHandler.THROW_UNSUPPORTED_OPERATION, + response.getEntity().getContent() + ) + ) { + XContentParser.Token token = parser.nextToken(); + List> parserList; + if (token == XContentParser.Token.START_ARRAY) { + parserList = parser.listOrderedMap().stream().map(obj -> (Map) obj).collect(Collectors.toList()); + } else { + parserList = Collections.singletonList(parser.mapOrdered()); + } + Set indices = parserList.stream() + .map(index -> (String) index.get("index")) + .filter(index -> !index.startsWith(SYSTEM_INDEX_PREFIX)) + .collect(Collectors.toSet()); + for (String index : indices) { + refreshIndex(index); + } + } + } + + protected void refreshIndex(final String index) throws IOException { + Request request = new Request("POST", "/" + index + "/_refresh"); + + Response response = client().performRequest(request); + assertEquals(request.getEndpoint() + ": failed", RestStatus.OK, RestStatus.fromCode(response.getStatusLine().getStatusCode())); + } } From d0517be2eb1080c499774a55b1290f5074a25d4d Mon Sep 17 00:00:00 2001 From: Martin Gaievski Date: Thu, 25 May 2023 17:21:16 -0700 Subject: [PATCH 107/416] Bump requests version from 2.26.0 to 2.31.0 (#913) (#917) (cherry picked from commit 15319e2e57aace412809abed0ac4e29e6d18a4ad) Signed-off-by: Martin Gaievski Co-authored-by: Naveen Tatikonda --- CHANGELOG.md | 1 + benchmarks/perf-tool/requirements.txt | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cde84481f..29ac80dc7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), * Bulk allocate objects for nmslib index creation to avoid malloc fragmentation ([#773](https://github.com/opensearch-project/k-NN/pull/773)) ### Bug Fixes ### Infrastructure +* Bump requests version from 2.26.0 to 2.31.0 ([#913](https://github.com/opensearch-project/k-NN/pull/913)) * Disable index refresh for system indices ([#773](https://github.com/opensearch-project/k-NN/pull/915)) ### Documentation ### Maintenance diff --git a/benchmarks/perf-tool/requirements.txt b/benchmarks/perf-tool/requirements.txt index b1e95c7d3..b1776be83 100644 --- a/benchmarks/perf-tool/requirements.txt +++ b/benchmarks/perf-tool/requirements.txt @@ -28,7 +28,7 @@ psutil==5.8.0 # via -r requirements.in pyyaml==5.4.1 # via -r requirements.in -requests==2.26.0 +requests==2.31.0 # via -r requirements.in urllib3==1.26.6 # via From f11f1f1d4ad0de76b05517b57bcc87e0a6788031 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Fri, 26 May 2023 09:35:01 -0700 Subject: [PATCH 108/416] Adding release notes for 2.8 (#918) (#919) Signed-off-by: Martin Gaievski (cherry picked from commit 556bb1a554bff6a1282be617e13e45f451843738) Co-authored-by: Martin Gaievski --- CHANGELOG.md | 5 +---- .../opensearch-knn.release-notes-2.8.0.0.md | 12 ++++++++++++ 2 files changed, 13 insertions(+), 4 deletions(-) create mode 100644 release-notes/opensearch-knn.release-notes-2.8.0.0.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 29ac80dc7..1cd03958b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,14 +12,11 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Maintenance ### Refactoring -## [Unreleased 2.x](https://github.com/opensearch-project/k-NN/compare/2.7...2.x) +## [Unreleased 2.x](https://github.com/opensearch-project/k-NN/compare/2.8...2.x) ### Features ### Enhancements -* Bulk allocate objects for nmslib index creation to avoid malloc fragmentation ([#773](https://github.com/opensearch-project/k-NN/pull/773)) ### Bug Fixes ### Infrastructure -* Bump requests version from 2.26.0 to 2.31.0 ([#913](https://github.com/opensearch-project/k-NN/pull/913)) -* Disable index refresh for system indices ([#773](https://github.com/opensearch-project/k-NN/pull/915)) ### Documentation ### Maintenance ### Refactoring \ No newline at end of file diff --git a/release-notes/opensearch-knn.release-notes-2.8.0.0.md b/release-notes/opensearch-knn.release-notes-2.8.0.0.md new file mode 100644 index 000000000..8639aba3f --- /dev/null +++ b/release-notes/opensearch-knn.release-notes-2.8.0.0.md @@ -0,0 +1,12 @@ +## Version 2.8.0.0 Release Notes + +Compatible with OpenSearch 2.8.0 + +### Enhancements + +* Bulk allocate objects for nmslib index creation to avoid malloc fragmentation ([#773](https://github.com/opensearch-project/k-NN/pull/773)) + +### Infrastructure + +* Bump requests version from 2.26.0 to 2.31.0 ([#913](https://github.com/opensearch-project/k-NN/pull/913)) +* Disable index refresh for system indices ([#773](https://github.com/opensearch-project/k-NN/pull/915)) From d90964db883e96f93ec216a69a9ffb3c6287839a Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Tue, 6 Jun 2023 10:30:10 -0700 Subject: [PATCH 109/416] Increment version to 2.9.0-SNAPSHOT (#923) Signed-off-by: opensearch-ci-bot Co-authored-by: opensearch-ci-bot --- .../workflows/backwards_compatibility_tests_workflow.yml | 8 ++++---- build.gradle | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/backwards_compatibility_tests_workflow.yml b/.github/workflows/backwards_compatibility_tests_workflow.yml index 592941053..3005c6b6b 100644 --- a/.github/workflows/backwards_compatibility_tests_workflow.yml +++ b/.github/workflows/backwards_compatibility_tests_workflow.yml @@ -14,8 +14,8 @@ jobs: strategy: matrix: java: [ 11, 17 ] - bwc_version : [ "1.1.0", "1.2.4", "1.3.8", "2.0.1", "2.1.0", "2.2.1", "2.3.0", "2.4.1", "2.5.0", "2.6.0", "2.7.0" ] - opensearch_version : [ "2.8.0-SNAPSHOT" ] + bwc_version : [ "1.1.0", "1.2.4", "1.3.8", "2.0.1", "2.1.0", "2.2.1", "2.3.0", "2.4.1", "2.5.0", "2.6.0", "2.7.0", "2.8.0" ] + opensearch_version : [ "2.9.0-SNAPSHOT" ] name: k-NN Restart-Upgrade BWC Tests runs-on: ubuntu-latest @@ -46,8 +46,8 @@ jobs: strategy: matrix: java: [ 11, 17 ] - bwc_version: [ "1.3.8", "2.0.1", "2.1.0", "2.2.1", "2.3.0", "2.4.1", "2.5.0", "2.6.0", "2.7.0"] - opensearch_version: [ "2.8.0-SNAPSHOT" ] + bwc_version: [ "1.3.8", "2.0.1", "2.1.0", "2.2.1", "2.3.0", "2.4.1", "2.5.0", "2.6.0", "2.7.0", "2.8.0"] + opensearch_version: [ "2.9.0-SNAPSHOT" ] name: k-NN Rolling-Upgrade BWC Tests runs-on: ubuntu-latest diff --git a/build.gradle b/build.gradle index 82c6f155a..f35644f4c 100644 --- a/build.gradle +++ b/build.gradle @@ -10,7 +10,7 @@ buildscript { ext { // build.version_qualifier parameter applies to knn plugin artifacts only. OpenSearch version must be set // explicitly as 'opensearch.version' property, for instance opensearch.version=2.0.0-rc1-SNAPSHOT - opensearch_version = System.getProperty("opensearch.version", "2.8.0-SNAPSHOT") + opensearch_version = System.getProperty("opensearch.version", "2.9.0-SNAPSHOT") version_qualifier = System.getProperty("build.version_qualifier", "") opensearch_group = "org.opensearch" } From ab5f3cc7536978205b05bffda4d448867f52fd9b Mon Sep 17 00:00:00 2001 From: Junqiu Lei Date: Mon, 10 Jul 2023 10:53:50 -0500 Subject: [PATCH 110/416] Move ImmutableOpenMap usage to j.u.Map (#955) --- .../DeleteModelWhenInTrainStateException.java | 2 +- .../knn/index/query/KNNQueryBuilder.java | 4 +-- .../plugin/rest/RestDeleteModelHandler.java | 4 +-- .../knn/plugin/rest/RestGetModelHandler.java | 4 +-- .../knn/plugin/rest/RestKNNStatsHandler.java | 6 ++--- .../knn/plugin/rest/RestKNNWarmupHandler.java | 4 +-- .../plugin/transport/DeleteModelRequest.java | 4 +-- .../TrainingJobRouterTransportAction.java | 9 ++++--- .../opensearch/knn/training/TrainingJob.java | 4 +-- ...TrainingJobRouterTransportActionTests.java | 25 ++++++++++--------- .../transport/TrainingModelRequestTests.java | 5 ++-- .../org/opensearch/knn/KNNRestTestCase.java | 4 +-- 12 files changed, 38 insertions(+), 37 deletions(-) diff --git a/src/main/java/org/opensearch/knn/common/exception/DeleteModelWhenInTrainStateException.java b/src/main/java/org/opensearch/knn/common/exception/DeleteModelWhenInTrainStateException.java index 38854aa59..fba20bd17 100644 --- a/src/main/java/org/opensearch/knn/common/exception/DeleteModelWhenInTrainStateException.java +++ b/src/main/java/org/opensearch/knn/common/exception/DeleteModelWhenInTrainStateException.java @@ -6,7 +6,7 @@ package org.opensearch.knn.common.exception; import org.opensearch.OpenSearchException; -import org.opensearch.common.logging.LoggerMessageFormat; +import org.opensearch.core.common.logging.LoggerMessageFormat; import org.opensearch.rest.RestStatus; /** diff --git a/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java b/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java index 3de0e69d4..c9f6ebe6d 100644 --- a/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java +++ b/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java @@ -6,6 +6,7 @@ package org.opensearch.knn.index.query; import lombok.extern.log4j.Log4j2; +import org.apache.commons.lang.StringUtils; import org.opensearch.Version; import org.opensearch.index.mapper.NumberFieldMapper; import org.opensearch.index.query.QueryBuilder; @@ -19,7 +20,6 @@ import org.apache.lucene.search.Query; import org.opensearch.core.ParseField; import org.opensearch.common.ParsingException; -import org.opensearch.common.Strings; import org.opensearch.common.io.stream.StreamInput; import org.opensearch.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.XContentBuilder; @@ -68,7 +68,7 @@ public KNNQueryBuilder(String fieldName, float[] vector, int k) { } public KNNQueryBuilder(String fieldName, float[] vector, int k, QueryBuilder filter) { - if (Strings.isNullOrEmpty(fieldName)) { + if (StringUtils.isBlank(fieldName)) { throw new IllegalArgumentException("[" + NAME + "] requires fieldName"); } if (vector == null) { diff --git a/src/main/java/org/opensearch/knn/plugin/rest/RestDeleteModelHandler.java b/src/main/java/org/opensearch/knn/plugin/rest/RestDeleteModelHandler.java index 37074d128..31b589ae1 100644 --- a/src/main/java/org/opensearch/knn/plugin/rest/RestDeleteModelHandler.java +++ b/src/main/java/org/opensearch/knn/plugin/rest/RestDeleteModelHandler.java @@ -11,9 +11,9 @@ package org.opensearch.knn.plugin.rest; +import org.apache.commons.lang.StringUtils; import com.google.common.collect.ImmutableList; import org.opensearch.client.node.NodeClient; -import org.opensearch.common.Strings; import org.opensearch.knn.plugin.KNNPlugin; import org.opensearch.knn.plugin.transport.DeleteModelAction; import org.opensearch.knn.plugin.transport.DeleteModelRequest; @@ -58,7 +58,7 @@ public List routes() { @Override protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) { String modelID = request.param(MODEL_ID); - if (!Strings.hasText(modelID)) { + if (StringUtils.isBlank(modelID)) { throw new IllegalArgumentException("model ID cannot be empty"); } DeleteModelRequest deleteModelRequest = new DeleteModelRequest(modelID); diff --git a/src/main/java/org/opensearch/knn/plugin/rest/RestGetModelHandler.java b/src/main/java/org/opensearch/knn/plugin/rest/RestGetModelHandler.java index 09f2daab2..8b1f0676b 100644 --- a/src/main/java/org/opensearch/knn/plugin/rest/RestGetModelHandler.java +++ b/src/main/java/org/opensearch/knn/plugin/rest/RestGetModelHandler.java @@ -12,8 +12,8 @@ package org.opensearch.knn.plugin.rest; import com.google.common.collect.ImmutableList; +import org.apache.commons.lang.StringUtils; import org.opensearch.client.node.NodeClient; -import org.opensearch.common.Strings; import org.opensearch.knn.plugin.KNNPlugin; import org.opensearch.knn.plugin.transport.GetModelAction; import org.opensearch.knn.plugin.transport.GetModelRequest; @@ -50,7 +50,7 @@ public List routes() { @Override protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) throws IOException { String modelID = restRequest.param(MODEL_ID); - if (!Strings.hasText(modelID)) { + if (StringUtils.isBlank(modelID)) { throw new IllegalArgumentException("model ID cannot be empty"); } diff --git a/src/main/java/org/opensearch/knn/plugin/rest/RestKNNStatsHandler.java b/src/main/java/org/opensearch/knn/plugin/rest/RestKNNStatsHandler.java index 3536b40fe..9049a83db 100644 --- a/src/main/java/org/opensearch/knn/plugin/rest/RestKNNStatsHandler.java +++ b/src/main/java/org/opensearch/knn/plugin/rest/RestKNNStatsHandler.java @@ -6,12 +6,12 @@ package org.opensearch.knn.plugin.rest; import lombok.AllArgsConstructor; +import org.apache.commons.lang.StringUtils; import org.opensearch.knn.plugin.KNNPlugin; import org.opensearch.knn.plugin.transport.KNNStatsAction; import org.opensearch.knn.plugin.transport.KNNStatsRequest; import com.google.common.collect.ImmutableList; import org.opensearch.client.node.NodeClient; -import org.opensearch.common.Strings; import org.opensearch.rest.BaseRestHandler; import org.opensearch.rest.RestRequest; import org.opensearch.rest.action.RestActions; @@ -83,7 +83,7 @@ private KNNStatsRequest getRequest(RestRequest request) { // parse the nodes the user wants to query String[] nodeIdsArr = null; String nodesIdsStr = request.param("nodeId"); - if (!Strings.isEmpty(nodesIdsStr)) { + if (StringUtils.isNotEmpty(nodesIdsStr)) { nodeIdsArr = nodesIdsStr.split(","); } @@ -93,7 +93,7 @@ private KNNStatsRequest getRequest(RestRequest request) { // parse the stats the customer wants to see Set statsSet = null; String statsStr = request.param("stat"); - if (!Strings.isEmpty(statsStr)) { + if (StringUtils.isNotEmpty(statsStr)) { statsSet = new HashSet<>(Arrays.asList(statsStr.split(","))); } diff --git a/src/main/java/org/opensearch/knn/plugin/rest/RestKNNWarmupHandler.java b/src/main/java/org/opensearch/knn/plugin/rest/RestKNNWarmupHandler.java index f457d6782..a31c2f297 100644 --- a/src/main/java/org/opensearch/knn/plugin/rest/RestKNNWarmupHandler.java +++ b/src/main/java/org/opensearch/knn/plugin/rest/RestKNNWarmupHandler.java @@ -5,6 +5,7 @@ package org.opensearch.knn.plugin.rest; +import org.apache.commons.lang.StringUtils; import org.opensearch.knn.common.exception.KNNInvalidIndicesException; import org.opensearch.knn.plugin.KNNPlugin; import org.opensearch.knn.plugin.transport.KNNWarmupAction; @@ -15,7 +16,6 @@ import org.opensearch.client.node.NodeClient; import org.opensearch.cluster.metadata.IndexNameExpressionResolver; import org.opensearch.cluster.service.ClusterService; -import org.opensearch.common.Strings; import org.opensearch.common.settings.Settings; import org.opensearch.index.Index; import org.opensearch.rest.BaseRestHandler; @@ -81,7 +81,7 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient cli } private KNNWarmupRequest createKNNWarmupRequest(RestRequest request) { - String[] indexNames = Strings.splitStringByCommaToArray(request.param("index")); + String[] indexNames = StringUtils.split(request.param("index"), ","); Index[] indices = indexNameExpressionResolver.concreteIndices(clusterService.state(), strictExpandOpen(), indexNames); List invalidIndexNames = new ArrayList<>(); diff --git a/src/main/java/org/opensearch/knn/plugin/transport/DeleteModelRequest.java b/src/main/java/org/opensearch/knn/plugin/transport/DeleteModelRequest.java index 792ccc543..fee82adb5 100644 --- a/src/main/java/org/opensearch/knn/plugin/transport/DeleteModelRequest.java +++ b/src/main/java/org/opensearch/knn/plugin/transport/DeleteModelRequest.java @@ -11,9 +11,9 @@ package org.opensearch.knn.plugin.transport; +import org.apache.commons.lang.StringUtils; import org.opensearch.action.ActionRequest; import org.opensearch.action.ActionRequestValidationException; -import org.opensearch.common.Strings; import org.opensearch.common.io.stream.StreamInput; import org.opensearch.common.io.stream.StreamOutput; @@ -43,7 +43,7 @@ public void writeTo(StreamOutput out) throws IOException { @Override public ActionRequestValidationException validate() { - if (Strings.hasText(modelID)) { + if (StringUtils.isNotBlank(modelID)) { return null; } return addValidationError("Model id cannot be empty ", null); diff --git a/src/main/java/org/opensearch/knn/plugin/transport/TrainingJobRouterTransportAction.java b/src/main/java/org/opensearch/knn/plugin/transport/TrainingJobRouterTransportAction.java index 774029c58..941d95212 100644 --- a/src/main/java/org/opensearch/knn/plugin/transport/TrainingJobRouterTransportAction.java +++ b/src/main/java/org/opensearch/knn/plugin/transport/TrainingJobRouterTransportAction.java @@ -11,6 +11,7 @@ package org.opensearch.knn.plugin.transport; +import org.apache.commons.lang.StringUtils; import org.opensearch.action.ActionListener; import org.opensearch.action.ActionListenerResponseHandler; import org.opensearch.action.search.SearchRequest; @@ -19,15 +20,15 @@ import org.opensearch.client.Client; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.service.ClusterService; -import org.opensearch.common.Strings; import org.opensearch.common.ValidationException; -import org.opensearch.common.collect.ImmutableOpenMap; import org.opensearch.common.inject.Inject; import org.opensearch.search.builder.SearchSourceBuilder; import org.opensearch.tasks.Task; import org.opensearch.transport.TransportRequestOptions; import org.opensearch.transport.TransportService; +import java.util.Map; + import static org.opensearch.knn.common.KNNConstants.BYTES_PER_KILOBYTES; import static org.opensearch.search.internal.SearchContext.DEFAULT_TERMINATE_AFTER; @@ -94,7 +95,7 @@ protected DiscoveryNode selectNode(String preferredNode, TrainingJobRouteDecisio DiscoveryNode selectedNode = null; - ImmutableOpenMap eligibleNodes = clusterService.state().nodes().getDataNodes(); + Map eligibleNodes = clusterService.state().nodes().getDataNodes(); DiscoveryNode currentNode; for (TrainingJobRouteDecisionInfoNodeResponse response : jobInfo.getNodes()) { @@ -107,7 +108,7 @@ protected DiscoveryNode selectNode(String preferredNode, TrainingJobRouteDecisio if (response.getTrainingJobCount() < 1) { selectedNode = currentNode; // Return right away if the user didnt pass a preferred node or this is the preferred node - if (Strings.isEmpty(preferredNode) || selectedNode.getId().equals(preferredNode)) { + if (StringUtils.isEmpty(preferredNode) || selectedNode.getId().equals(preferredNode)) { return selectedNode; } } diff --git a/src/main/java/org/opensearch/knn/training/TrainingJob.java b/src/main/java/org/opensearch/knn/training/TrainingJob.java index e01688ddc..27a1c6025 100644 --- a/src/main/java/org/opensearch/knn/training/TrainingJob.java +++ b/src/main/java/org/opensearch/knn/training/TrainingJob.java @@ -11,9 +11,9 @@ package org.opensearch.knn.training; +import org.apache.commons.lang.StringUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.opensearch.common.Strings; import org.opensearch.common.UUIDs; import org.opensearch.knn.common.KNNConstants; import org.opensearch.knn.index.KNNSettings; @@ -68,7 +68,7 @@ public TrainingJob( String description ) { // Generate random base64 string if one is not provided - this.modelId = Strings.hasText(modelId) ? modelId : UUIDs.randomBase64UUID(); + this.modelId = StringUtils.isNotBlank(modelId) ? modelId : UUIDs.randomBase64UUID(); this.knnMethodContext = Objects.requireNonNull(knnMethodContext, "MethodContext cannot be null."); this.nativeMemoryCacheManager = Objects.requireNonNull(nativeMemoryCacheManager, "NativeMemoryCacheManager cannot be null."); this.trainingDataEntryContext = Objects.requireNonNull(trainingDataEntryContext, "TrainingDataEntryContext cannot be null."); diff --git a/src/test/java/org/opensearch/knn/plugin/transport/TrainingJobRouterTransportActionTests.java b/src/test/java/org/opensearch/knn/plugin/transport/TrainingJobRouterTransportActionTests.java index 5f907b6c4..4fc4a1ccd 100644 --- a/src/test/java/org/opensearch/knn/plugin/transport/TrainingJobRouterTransportActionTests.java +++ b/src/test/java/org/opensearch/knn/plugin/transport/TrainingJobRouterTransportActionTests.java @@ -22,7 +22,6 @@ import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.node.DiscoveryNodes; import org.opensearch.cluster.service.ClusterService; -import org.opensearch.common.collect.ImmutableOpenMap; import org.opensearch.knn.KNNTestCase; import org.opensearch.knn.index.KNNMethodContext; import org.opensearch.search.SearchHit; @@ -31,7 +30,9 @@ import java.util.ArrayList; import java.util.Collections; +import java.util.HashMap; import java.util.List; +import java.util.Map; import static org.mockito.Mockito.any; import static org.mockito.Mockito.doAnswer; @@ -45,7 +46,7 @@ public void testSingleNode_withCapacity() { // Mock datanodes in the cluster through mocking the cluster service List nodeIds = ImmutableList.of("node-1"); - ImmutableOpenMap discoveryNodesMap = generateDiscoveryNodes(nodeIds); + Map discoveryNodesMap = generateDiscoveryNodes(nodeIds); ClusterService clusterService = generateMockedClusterService(discoveryNodesMap); // Create a response to be returned with job route decision info @@ -85,7 +86,7 @@ public void testSingleNode_withoutCapacity() { // Mock datanodes in the cluster through mocking the cluster service List nodeIds = ImmutableList.of("node-1"); - ImmutableOpenMap discoveryNodesMap = generateDiscoveryNodes(nodeIds); + Map discoveryNodesMap = generateDiscoveryNodes(nodeIds); ClusterService clusterService = generateMockedClusterService(discoveryNodesMap); // Create a response to be returned with job route decision info @@ -125,7 +126,7 @@ public void testMultiNode_withCapacity() { // Mock datanodes in the cluster through mocking the cluster service List nodeIds = ImmutableList.of("node-1", "node-2", "node-3"); - ImmutableOpenMap discoveryNodesMap = generateDiscoveryNodes(nodeIds); + Map discoveryNodesMap = generateDiscoveryNodes(nodeIds); ClusterService clusterService = generateMockedClusterService(discoveryNodesMap); // Create a response to be returned with job route decision info @@ -168,7 +169,7 @@ public void testMultiNode_withCapacity_withPreferredAvailable() { String preferredNode = nodeIds.get(2); - ImmutableOpenMap discoveryNodesMap = generateDiscoveryNodes(nodeIds); + Map discoveryNodesMap = generateDiscoveryNodes(nodeIds); ClusterService clusterService = generateMockedClusterService(discoveryNodesMap); // Create a response to be returned with job route decision info @@ -211,7 +212,7 @@ public void testMultiNode_withCapacity_withoutPreferredAvailable() { String preferredNode = nodeIds.get(2); - ImmutableOpenMap discoveryNodesMap = generateDiscoveryNodes(nodeIds); + Map discoveryNodesMap = generateDiscoveryNodes(nodeIds); ClusterService clusterService = generateMockedClusterService(discoveryNodesMap); // Create a response to be returned with job route decision info @@ -253,7 +254,7 @@ public void testMultiNode_withoutCapacity() { // Mock datanodes in the cluster through mocking the cluster service List nodeIds = ImmutableList.of("node-1", "node-2", "node-3"); - ImmutableOpenMap discoveryNodesMap = generateDiscoveryNodes(nodeIds); + Map discoveryNodesMap = generateDiscoveryNodes(nodeIds); ClusterService clusterService = generateMockedClusterService(discoveryNodesMap); // Create a response to be returned with job route decision info @@ -338,19 +339,19 @@ public void testTrainingIndexSize() { transportAction.getTrainingIndexSizeInKB(trainingModelRequest, listener); } - private ImmutableOpenMap generateDiscoveryNodes(List dataNodeIds) { - ImmutableOpenMap.Builder builder = ImmutableOpenMap.builder(); + private Map generateDiscoveryNodes(List dataNodeIds) { + Map nodes = new HashMap<>(); for (String nodeId : dataNodeIds) { DiscoveryNode discoveryNode = mock(DiscoveryNode.class); when(discoveryNode.getId()).thenReturn(nodeId); - builder.put(nodeId, discoveryNode); + nodes.put(nodeId, discoveryNode); } - return builder.build(); + return nodes; } - private ClusterService generateMockedClusterService(ImmutableOpenMap discoveryNodeMap) { + private ClusterService generateMockedClusterService(Map discoveryNodeMap) { DiscoveryNodes discoveryNodes = mock(DiscoveryNodes.class); when(discoveryNodes.getDataNodes()).thenReturn(discoveryNodeMap); ClusterState clusterState = mock(ClusterState.class); diff --git a/src/test/java/org/opensearch/knn/plugin/transport/TrainingModelRequestTests.java b/src/test/java/org/opensearch/knn/plugin/transport/TrainingModelRequestTests.java index 0ed45dce3..09ca6c73b 100644 --- a/src/test/java/org/opensearch/knn/plugin/transport/TrainingModelRequestTests.java +++ b/src/test/java/org/opensearch/knn/plugin/transport/TrainingModelRequestTests.java @@ -20,7 +20,6 @@ import org.opensearch.cluster.node.DiscoveryNodes; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.ValidationException; -import org.opensearch.common.collect.ImmutableOpenMap; import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.knn.KNNTestCase; import org.opensearch.knn.common.KNNConstants; @@ -529,7 +528,7 @@ public void testValidation_invalid_preferredNodeDoesNotExist() { // Empty set of data nodes to produce exception DiscoveryNodes discoveryNodes = mock(DiscoveryNodes.class); - when(discoveryNodes.getDataNodes()).thenReturn(ImmutableOpenMap.of()); + when(discoveryNodes.getDataNodes()).thenReturn(Map.of()); ClusterState clusterState = mock(ClusterState.class); when(clusterState.metadata()).thenReturn(metadata); @@ -679,7 +678,7 @@ public void testValidation_valid_trainingIndexBuiltFromModel() { Metadata metadata = mock(Metadata.class); when(metadata.index(trainingIndex)).thenReturn(indexMetadata); DiscoveryNodes discoveryNodes = mock(DiscoveryNodes.class); - when(discoveryNodes.getDataNodes()).thenReturn(ImmutableOpenMap.of()); + when(discoveryNodes.getDataNodes()).thenReturn(Map.of()); ClusterState clusterState = mock(ClusterState.class); when(clusterState.metadata()).thenReturn(metadata); diff --git a/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java b/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java index c4e3cbbc7..6c07783ec 100644 --- a/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java +++ b/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java @@ -122,7 +122,7 @@ public static void dumpCoverage() throws IOException, MalformedObjectNameExcepti // jacoco.dir is set in esplugin-coverage.gradle, if it doesn't exist we don't // want to collect coverage so we can return early String jacocoBuildPath = System.getProperty("jacoco.dir"); - if (Strings.isNullOrEmpty(jacocoBuildPath)) { + if (StringUtils.isBlank(jacocoBuildPath)) { return; } @@ -657,7 +657,7 @@ protected void createModelSystemIndex() throws IOException { } protected void addModelToSystemIndex(String modelId, ModelMetadata modelMetadata, byte[] model) throws IOException { - assertFalse(Strings.isNullOrEmpty(modelId)); + assertFalse(StringUtils.isBlank(modelId)); String modelBase64 = Base64.getEncoder().encodeToString(model); Request request = new Request("POST", "/" + MODEL_INDEX_NAME + "/_doc/" + modelId + "?refresh=true"); From 42110134fbe87c813c10988bb192dbfe094e5dca Mon Sep 17 00:00:00 2001 From: Navneet Verma Date: Wed, 14 Jun 2023 10:38:40 -0700 Subject: [PATCH 111/416] Added support for Efficient Pre-filtering for Faiss Engine. The changes include (#936) * Enabled the efficient filtering support for Faiss Engine (#907) * Fixed the ef_search default value for faiss HNSW with filters and updated the perf-tool to include Faiss HNSW tests (#926) * Added exact search for cases when filteredIds < k to improve the recall for exact search (#928) * Improved Exact Search to return only K results and added client side latency metric for query Benchmarks (#933) * Added Integration Tests and Unit test for Efficient Filtering for Faiss Engine (#934) Signed-off-by: Navneet Verma --- CHANGELOG.md | 3 +- DEVELOPER_GUIDE.md | 4 +- benchmarks/perf-tool/README.md | 47 +++-- .../perf-tool/okpt/io/config/parsers/test.py | 5 + .../perf-tool/okpt/io/config/schemas/test.yml | 3 + benchmarks/perf-tool/okpt/test/steps/steps.py | 22 ++- .../filtering/relaxed-filter/index.json | 26 +++ .../relaxed-filter/relaxed-filter-spec.json | 42 ++++ .../relaxed-filter/relaxed-filter-test.yml | 34 ++++ .../filtering/restrictive-filter/index.json | 26 +++ .../restrictive-filter-spec.json | 44 +++++ .../restrictive-filter-test.yml | 37 ++++ .../release-configs/faiss-hnsw/index.json | 26 +++ .../release-configs/faiss-hnsw/test.yml | 32 +++ .../relaxed-filter/relaxed-filter-test.yml | 4 +- .../restrictive-filter-test.yml | 17 +- jni/CMakeLists.txt | 4 +- jni/external/faiss | 2 +- jni/include/faiss_wrapper.h | 6 + .../org_opensearch_knn_jni_FaissService.h | 8 + jni/src/faiss_wrapper.cpp | 165 +++++++++++++++- .../org_opensearch_knn_jni_FaissService.cpp | 12 ++ .../opensearch/knn/index/query/KNNQuery.java | 33 ++++ .../knn/index/query/KNNQueryBuilder.java | 2 +- .../knn/index/query/KNNQueryFactory.java | 36 +++- .../opensearch/knn/index/query/KNNScorer.java | 33 ++++ .../opensearch/knn/index/query/KNNWeight.java | 185 ++++++++++++++++-- .../opensearch/knn/index/util/KNNEngine.java | 5 + .../opensearch/knn/index/util/KNNLibrary.java | 2 +- .../org/opensearch/knn/jni/FaissService.java | 4 +- .../org/opensearch/knn/jni/JNIService.java | 18 +- .../org/opensearch/knn/index/FaissIT.java | 94 +++++++++ .../opensearch/knn/index/LuceneEngineIT.java | 19 -- .../knn/index/codec/KNNCodecTestUtil.java | 2 +- .../memory/NativeMemoryLoadStrategyTests.java | 2 +- .../knn/index/query/KNNQueryFactoryTests.java | 35 +++- .../knn/index/query/KNNWeightTests.java | 168 +++++++++++++++- .../opensearch/knn/jni/JNIServiceTests.java | 20 +- .../org/opensearch/knn/KNNRestTestCase.java | 16 ++ 39 files changed, 1133 insertions(+), 110 deletions(-) create mode 100644 benchmarks/perf-tool/release-configs/faiss-hnsw/filtering/relaxed-filter/index.json create mode 100644 benchmarks/perf-tool/release-configs/faiss-hnsw/filtering/relaxed-filter/relaxed-filter-spec.json create mode 100644 benchmarks/perf-tool/release-configs/faiss-hnsw/filtering/relaxed-filter/relaxed-filter-test.yml create mode 100644 benchmarks/perf-tool/release-configs/faiss-hnsw/filtering/restrictive-filter/index.json create mode 100644 benchmarks/perf-tool/release-configs/faiss-hnsw/filtering/restrictive-filter/restrictive-filter-spec.json create mode 100644 benchmarks/perf-tool/release-configs/faiss-hnsw/filtering/restrictive-filter/restrictive-filter-test.yml create mode 100644 benchmarks/perf-tool/release-configs/faiss-hnsw/index.json create mode 100644 benchmarks/perf-tool/release-configs/faiss-hnsw/test.yml diff --git a/CHANGELOG.md b/CHANGELOG.md index 1cd03958b..78c2b5c91 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,9 +14,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ## [Unreleased 2.x](https://github.com/opensearch-project/k-NN/compare/2.8...2.x) ### Features +* Added efficient filtering support for Faiss Engine ([#936](https://github.com/opensearch-project/k-NN/pull/936)) ### Enhancements ### Bug Fixes ### Infrastructure ### Documentation ### Maintenance -### Refactoring \ No newline at end of file +### Refactoring diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index 4a6f360b5..d8ef9e413 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -56,11 +56,11 @@ In addition to this, the plugin has been tested with JDK 17, and this JDK versio #### CMake -The plugin requires that cmake >= 3.17.2 is installed in order to build the JNI libraries. +The plugin requires that cmake >= 3.23.1 is installed in order to build the JNI libraries. One easy way to install on mac or linux is to use pip: ```bash -pip install cmake==3.17.2 +pip install cmake==3.23.1 ``` #### Faiss Dependencies diff --git a/benchmarks/perf-tool/README.md b/benchmarks/perf-tool/README.md index 9c1c18918..f98227e27 100644 --- a/benchmarks/perf-tool/README.md +++ b/benchmarks/perf-tool/README.md @@ -13,18 +13,36 @@ file. ## Install Prerequisites -### Python +### Setup -Python 3.7 or above is required. +K-NN perf requires Python 3.8 or greater to be installed. One of +the easier ways to do this is through Conda, a package and environment +management system for Python. -### Pip +First, follow the +[installation instructions](https://docs.conda.io/projects/conda/en/latest/user-guide/install/index.html) +to install Conda on your system. -Use pip to install the necessary requirements: +Next, create a Python 3.8 environment: +``` +conda create -n knn-perf python=3.8 +``` + +After the environment is created, activate it: +``` +source activate knn-perf +``` +Lastly, clone the k-NN repo and install all required python packages: ``` +git clone https://github.com/opensearch-project/k-NN.git +cd k-NN/benchmarks/perf-tool pip install -r requirements.txt ``` +After all of this completes, you should be ready to run your first performance benchmarks! + + ## Usage ### Quick Start @@ -72,16 +90,17 @@ The output will be the delta between the two metrics. ### Test Parameters -| Parameter Name | Description | Default | -| ----------- | ----------- | ----------- | -| endpoint | Endpoint OpenSearch cluster is running on | localhost | -| test_name | Name of test | No default | -| test_id | String ID of test | No default | -| num_runs | Number of runs to execute steps | 1 | -| show_runs | Whether to output each run in addition to the total summary | false | -| setup | List of steps to run once before metric collection starts | [] | -| steps | List of steps that make up one test run. Metrics will be collected on these steps. | No default | -| cleanup | List of steps to run after each test run | [] | +| Parameter Name | Description | Default | +|----------------|------------------------------------------------------------------------------------|------------| +| endpoint | Endpoint OpenSearch cluster is running on | localhost | +| port | Port on which OpenSearch Cluster is running on | 9200 | +| test_name | Name of test | No default | +| test_id | String ID of test | No default | +| num_runs | Number of runs to execute steps | 1 | +| show_runs | Whether to output each run in addition to the total summary | false | +| setup | List of steps to run once before metric collection starts | [] | +| steps | List of steps that make up one test run. Metrics will be collected on these steps. | No default | +| cleanup | List of steps to run after each test run | [] | ### Steps diff --git a/benchmarks/perf-tool/okpt/io/config/parsers/test.py b/benchmarks/perf-tool/okpt/io/config/parsers/test.py index 34b1752c7..d0ef4c02f 100644 --- a/benchmarks/perf-tool/okpt/io/config/parsers/test.py +++ b/benchmarks/perf-tool/okpt/io/config/parsers/test.py @@ -23,6 +23,7 @@ class TestConfig: test_name: str test_id: str endpoint: str + port: int num_runs: int show_runs: bool setup: List[Step] @@ -48,6 +49,9 @@ def parse(self, file_obj: TextIOWrapper) -> TestConfig: if 'endpoint' in config_obj: implicit_step_config['endpoint'] = config_obj['endpoint'] + if 'port' in config_obj: + implicit_step_config['port'] = config_obj['port'] + # Each step should have its own parse - take the config object and check if its valid setup = [] if 'setup' in config_obj: @@ -62,6 +66,7 @@ def parse(self, file_obj: TextIOWrapper) -> TestConfig: test_config = TestConfig( endpoint=config_obj['endpoint'], + port=config_obj['port'], test_name=config_obj['test_name'], test_id=config_obj['test_id'], num_runs=config_obj['num_runs'], diff --git a/benchmarks/perf-tool/okpt/io/config/schemas/test.yml b/benchmarks/perf-tool/okpt/io/config/schemas/test.yml index 1939a8a31..06b880cc7 100644 --- a/benchmarks/perf-tool/okpt/io/config/schemas/test.yml +++ b/benchmarks/perf-tool/okpt/io/config/schemas/test.yml @@ -9,6 +9,9 @@ endpoint: type: string default: "localhost" +port: + type: integer + default: 9200 test_name: type: string test_id: diff --git a/benchmarks/perf-tool/okpt/test/steps/steps.py b/benchmarks/perf-tool/okpt/test/steps/steps.py index 0de61078f..cc1773330 100644 --- a/benchmarks/perf-tool/okpt/test/steps/steps.py +++ b/benchmarks/perf-tool/okpt/test/steps/steps.py @@ -5,7 +5,7 @@ # compatible open source license. """Provides steps for OpenSearch tests. -Some of the OpenSearch operations return a `took` field in the response body, +Some OpenSearch operations return a `took` field in the response body, so the profiling decorators aren't needed for some functions. """ import json @@ -454,8 +454,10 @@ def _action(self): results['took'] = [ float(query_response['took']) for query_response in query_responses ] - port = 9200 if self.endpoint == 'localhost' else 80 - results['memory_kb'] = get_cache_size_in_kb(self.endpoint, port) + results['client_time'] = [ + float(query_response['client_time']) for query_response in query_responses + ] + results['memory_kb'] = get_cache_size_in_kb(self.endpoint, self.port) if self.calculate_recall: ids = [[int(hit['_id']) @@ -473,7 +475,7 @@ def _action(self): return results def _get_measures(self) -> List[str]: - measures = ['took', 'memory_kb'] + measures = ['took', 'memory_kb', 'client_time'] if self.calculate_recall: measures.extend(['recall@K', f'recall@{str(self.r)}']) @@ -614,7 +616,6 @@ def _action(self): num_of_search_segments = 0; for shard_key in shards.keys(): for segment in shards[shard_key]: - num_of_committed_segments += segment["num_committed_segments"] num_of_search_segments += segment["num_search_segments"] @@ -689,12 +690,13 @@ def delete_model(endpoint, port, model_id): return response.json() -def get_opensearch_client(endpoint: str, port: int): +def get_opensearch_client(endpoint: str, port: int, timeout=60): """ Get an opensearch client from an endpoint and port Args: endpoint: Endpoint OpenSearch is running on port: Port OpenSearch is running on + timeout: timeout for OpenSearch client, default value 60 Returns: OpenSearch client @@ -708,7 +710,7 @@ def get_opensearch_client(endpoint: str, port: int): use_ssl=False, verify_certs=False, connection_class=RequestsHttpConnection, - timeout=60, + timeout=timeout, ) @@ -784,9 +786,13 @@ def get_cache_size_in_kb(endpoint, port): def query_index(opensearch: OpenSearch, index_name: str, body: dict, excluded_fields: list): - return opensearch.search(index=index_name, + start_time = round(time.time()*1000) + queryResponse = opensearch.search(index=index_name, body=body, _source_excludes=excluded_fields) + end_time = round(time.time() * 1000) + queryResponse['client_time'] = end_time - start_time + return queryResponse def bulk_index(opensearch: OpenSearch, index_name: str, body: List): diff --git a/benchmarks/perf-tool/release-configs/faiss-hnsw/filtering/relaxed-filter/index.json b/benchmarks/perf-tool/release-configs/faiss-hnsw/filtering/relaxed-filter/index.json new file mode 100644 index 000000000..b8f591176 --- /dev/null +++ b/benchmarks/perf-tool/release-configs/faiss-hnsw/filtering/relaxed-filter/index.json @@ -0,0 +1,26 @@ +{ + "settings": { + "index": { + "knn": true, + "number_of_shards": 24, + "number_of_replicas": 1 + } + }, + "mappings": { + "properties": { + "target_field": { + "type": "knn_vector", + "dimension": 128, + "method": { + "name": "hnsw", + "space_type": "l2", + "engine": "faiss", + "parameters": { + "ef_construction": 256, + "m": 16 + } + } + } + } + } +} diff --git a/benchmarks/perf-tool/release-configs/faiss-hnsw/filtering/relaxed-filter/relaxed-filter-spec.json b/benchmarks/perf-tool/release-configs/faiss-hnsw/filtering/relaxed-filter/relaxed-filter-spec.json new file mode 100644 index 000000000..fecde0392 --- /dev/null +++ b/benchmarks/perf-tool/release-configs/faiss-hnsw/filtering/relaxed-filter/relaxed-filter-spec.json @@ -0,0 +1,42 @@ +{ + "bool": + { + "should": + [ + { + "range": + { + "age": + { + "gte": 30, + "lte": 70 + } + } + }, + { + "term": + { + "color": "green" + } + }, + { + "term": + { + "color": "blue" + } + }, + { + "term": + { + "color": "yellow" + } + }, + { + "term": + { + "color": "sweet" + } + } + ] + } +} \ No newline at end of file diff --git a/benchmarks/perf-tool/release-configs/faiss-hnsw/filtering/relaxed-filter/relaxed-filter-test.yml b/benchmarks/perf-tool/release-configs/faiss-hnsw/filtering/relaxed-filter/relaxed-filter-test.yml new file mode 100644 index 000000000..61486b3b6 --- /dev/null +++ b/benchmarks/perf-tool/release-configs/faiss-hnsw/filtering/relaxed-filter/relaxed-filter-test.yml @@ -0,0 +1,34 @@ +endpoint: [ENDPOINT] +test_name: "Faiss HNSW Relaxed Filter Test" +test_id: "Faiss HNSW Relaxed Filter Test" +num_runs: 10 +show_runs: false +steps: + - name: delete_index + index_name: target_index + - name: create_index + index_name: target_index + index_spec: [INDEX_SPEC_PATH]/relaxed-filter/index.json + - name: ingest_multi_field + index_name: target_index + field_name: target_field + bulk_size: 500 + dataset_format: hdf5 + dataset_path: [DATASET_PATH]/sift-128-euclidean-with-attr.hdf5 + attributes_dataset_name: attributes + attribute_spec: [ { name: 'color', type: 'str' }, { name: 'taste', type: 'str' }, { name: 'age', type: 'int' } ] + - name: refresh_index + index_name: target_index + - name: query_with_filter + k: 100 + r: 1 + calculate_recall: true + index_name: target_index + field_name: target_field + dataset_format: hdf5 + dataset_path: [DATASET_PATH]/sift-128-euclidean-with-attr.hdf5 + neighbors_format: hdf5 + neighbors_path: [DATASET_PATH]/sift-128-euclidean-with-filters.hdf5 + neighbors_dataset: neighbors_filter_5 + filter_spec: [INDEX_SPEC_PATH]/relaxed-filter-spec.json + filter_type: FILTER diff --git a/benchmarks/perf-tool/release-configs/faiss-hnsw/filtering/restrictive-filter/index.json b/benchmarks/perf-tool/release-configs/faiss-hnsw/filtering/restrictive-filter/index.json new file mode 100644 index 000000000..b8f591176 --- /dev/null +++ b/benchmarks/perf-tool/release-configs/faiss-hnsw/filtering/restrictive-filter/index.json @@ -0,0 +1,26 @@ +{ + "settings": { + "index": { + "knn": true, + "number_of_shards": 24, + "number_of_replicas": 1 + } + }, + "mappings": { + "properties": { + "target_field": { + "type": "knn_vector", + "dimension": 128, + "method": { + "name": "hnsw", + "space_type": "l2", + "engine": "faiss", + "parameters": { + "ef_construction": 256, + "m": 16 + } + } + } + } + } +} diff --git a/benchmarks/perf-tool/release-configs/faiss-hnsw/filtering/restrictive-filter/restrictive-filter-spec.json b/benchmarks/perf-tool/release-configs/faiss-hnsw/filtering/restrictive-filter/restrictive-filter-spec.json new file mode 100644 index 000000000..9e6356f1c --- /dev/null +++ b/benchmarks/perf-tool/release-configs/faiss-hnsw/filtering/restrictive-filter/restrictive-filter-spec.json @@ -0,0 +1,44 @@ +{ + "bool": + { + "must": + [ + { + "range": + { + "age": + { + "gte": 30, + "lte": 60 + } + } + }, + { + "term": + { + "taste": "bitter" + } + }, + { + "bool": + { + "should": + [ + { + "term": + { + "color": "blue" + } + }, + { + "term": + { + "color": "green" + } + } + ] + } + } + ] + } +} \ No newline at end of file diff --git a/benchmarks/perf-tool/release-configs/faiss-hnsw/filtering/restrictive-filter/restrictive-filter-test.yml b/benchmarks/perf-tool/release-configs/faiss-hnsw/filtering/restrictive-filter/restrictive-filter-test.yml new file mode 100644 index 000000000..bf02144ac --- /dev/null +++ b/benchmarks/perf-tool/release-configs/faiss-hnsw/filtering/restrictive-filter/restrictive-filter-test.yml @@ -0,0 +1,37 @@ +endpoint: [ENDPOINT] +test_name: "Faiss HNSW Restrictive Filter Test" +test_id: "Faiss HNSW Restrictive Filter Test" +num_runs: 10 +show_runs: false +steps: + - name: delete_index + index_name: target_index + - name: create_index + index_name: target_index + index_spec: [INDEX_SPEC_PATH]/index.json + - name: ingest_multi_field + index_name: target_index + field_name: target_field + bulk_size: 500 + dataset_format: hdf5 + dataset_path: [DATASET_PATH]/sift-128-euclidean-with-attr.hdf5 + attributes_dataset_name: attributes + attribute_spec: [ { name: 'color', type: 'str' }, { name: 'taste', type: 'str' }, { name: 'age', type: 'int' } ] + - name: refresh_index + index_name: target_index + - name: force_merge + index_name: target_index + max_num_segments: 1 + - name: query_with_filter + k: 100 + r: 1 + calculate_recall: true + index_name: target_index + field_name: target_field + dataset_format: hdf5 + dataset_path: [DATASET_PATH]/sift-128-euclidean-with-attr.hdf5 + neighbors_format: hdf5 + neighbors_path: [DATASET_PATH]/sift-128-euclidean-with-filters.hdf5 + neighbors_dataset: neighbors_filter_4 + filter_spec: [INDEX_SPEC_PATH]/restrictive-filter-spec.json + filter_type: FILTER diff --git a/benchmarks/perf-tool/release-configs/faiss-hnsw/index.json b/benchmarks/perf-tool/release-configs/faiss-hnsw/index.json new file mode 100644 index 000000000..b8f591176 --- /dev/null +++ b/benchmarks/perf-tool/release-configs/faiss-hnsw/index.json @@ -0,0 +1,26 @@ +{ + "settings": { + "index": { + "knn": true, + "number_of_shards": 24, + "number_of_replicas": 1 + } + }, + "mappings": { + "properties": { + "target_field": { + "type": "knn_vector", + "dimension": 128, + "method": { + "name": "hnsw", + "space_type": "l2", + "engine": "faiss", + "parameters": { + "ef_construction": 256, + "m": 16 + } + } + } + } + } +} diff --git a/benchmarks/perf-tool/release-configs/faiss-hnsw/test.yml b/benchmarks/perf-tool/release-configs/faiss-hnsw/test.yml new file mode 100644 index 000000000..f3e976cf3 --- /dev/null +++ b/benchmarks/perf-tool/release-configs/faiss-hnsw/test.yml @@ -0,0 +1,32 @@ +endpoint: localhost +test_name: "Faiss HNSW Test" +test_id: "Faiss HNSW Test" +num_runs: 10 +show_runs: false +steps: + - name: delete_index + index_name: target_index + - name: create_index + index_name: target_index + index_spec: /home/ec2-user/[PATH]/index.json + - name: ingest + index_name: target_index + field_name: target_field + bulk_size: 500 + dataset_format: hdf5 + dataset_path: [DATASET_PATH]/sift-128-euclidean.hdf5 + - name: refresh_index + index_name: target_index + - name: force_merge + index_name: target_index + max_num_segments: 1 + - name: query + k: 100 + r: 1 + calculate_recall: true + index_name: target_index + field_name: target_field + dataset_format: hdf5 + dataset_path: [DATASET_PATH]/sift-128-euclidean.hdf5 + neighbors_format: hdf5 + neighbors_path: [DATASET_PATH]/sift-128-euclidean.hdf5 diff --git a/benchmarks/perf-tool/release-configs/lucene-hnsw/filtering/relaxed-filter/relaxed-filter-test.yml b/benchmarks/perf-tool/release-configs/lucene-hnsw/filtering/relaxed-filter/relaxed-filter-test.yml index f20fba203..44ed8e66e 100644 --- a/benchmarks/perf-tool/release-configs/lucene-hnsw/filtering/relaxed-filter/relaxed-filter-test.yml +++ b/benchmarks/perf-tool/release-configs/lucene-hnsw/filtering/relaxed-filter/relaxed-filter-test.yml @@ -1,6 +1,6 @@ endpoint: [ENDPOINT] -test_name: "index-workflow" -test_id: "Index workflow" +test_name: "Lucene HNSW Relaxed Filter Test" +test_id: "Lucene HNSW Relaxed Filter Test" num_runs: 10 show_runs: false steps: diff --git a/benchmarks/perf-tool/release-configs/lucene-hnsw/filtering/restrictive-filter/restrictive-filter-test.yml b/benchmarks/perf-tool/release-configs/lucene-hnsw/filtering/restrictive-filter/restrictive-filter-test.yml index b1d7b60d7..d7f451a48 100644 --- a/benchmarks/perf-tool/release-configs/lucene-hnsw/filtering/restrictive-filter/restrictive-filter-test.yml +++ b/benchmarks/perf-tool/release-configs/lucene-hnsw/filtering/restrictive-filter/restrictive-filter-test.yml @@ -1,6 +1,6 @@ endpoint: [ENDPOINT] -test_name: "index-workflow" -test_id: "Index workflow" +test_name: "Lucene HNSW Restrictive Filter Test" +test_id: "Lucene HNSW Restrictive Filter Test" num_runs: 10 show_runs: false steps: @@ -8,17 +8,20 @@ steps: index_name: target_index - name: create_index index_name: target_index - index_spec: [INDEX_SPEC_PATH]/index.json + index_spec: /home/ec2-user/k-NN/benchmarks/perf-tool/release-configs/lucene-hnsw/filtering/restrictive-filter/index.json - name: ingest_multi_field index_name: target_index field_name: target_field bulk_size: 500 dataset_format: hdf5 - dataset_path: [DATASET_PATH]/sift-128-euclidean-with-attr.hdf5 + dataset_path: /home/ec2-user/k-NN/benchmarks/perf-tool/dataset/sift-128-euclidean-with-attr.hdf5 attributes_dataset_name: attributes attribute_spec: [ { name: 'color', type: 'str' }, { name: 'taste', type: 'str' }, { name: 'age', type: 'int' } ] - name: refresh_index index_name: target_index + - name: force_merge + index_name: target_index + max_num_segments: 1 - name: query_with_filter k: 100 r: 1 @@ -26,9 +29,9 @@ steps: index_name: target_index field_name: target_field dataset_format: hdf5 - dataset_path: [DATASET_PATH]/sift-128-euclidean-with-attr.hdf5 + dataset_path: /home/ec2-user/k-NN/benchmarks/perf-tool/dataset/sift-128-euclidean-with-attr.hdf5 neighbors_format: hdf5 - neighbors_path: [DATASET_PATH]/sift-128-euclidean-with-filters.hdf5 + neighbors_path: /home/ec2-user/k-NN/benchmarks/perf-tool/dataset/sift-128-euclidean-with-filters.hdf5 neighbors_dataset: neighbors_filter_4 - filter_spec: [INDEX_SPEC_PATH]/restrictive-filter-spec.json + filter_spec: /home/ec2-user/k-NN/benchmarks/perf-tool/release-configs/lucene-hnsw/filtering/restrictive-filter/restrictive-filter-spec.json filter_type: FILTER diff --git a/jni/CMakeLists.txt b/jni/CMakeLists.txt index 668ce684d..29a844ee0 100644 --- a/jni/CMakeLists.txt +++ b/jni/CMakeLists.txt @@ -3,7 +3,7 @@ # SPDX-License-Identifier: Apache-2.0 # -cmake_minimum_required(VERSION 3.17) +cmake_minimum_required(VERSION 3.23.1) project(KNNPlugin_JNI) @@ -95,7 +95,7 @@ if (${CONFIG_NMSLIB} STREQUAL ON OR ${CONFIG_ALL} STREQUAL ON OR ${CONFIG_TEST} set_target_properties(${TARGET_LIB_NMSLIB} PROPERTIES SUFFIX ${LIB_EXT}) set_target_properties(${TARGET_LIB_NMSLIB} PROPERTIES POSITION_INDEPENDENT_CODE ON) - if (WIN32) + if (NOT "${WIN32}" STREQUAL "") # Use RUNTIME_OUTPUT_DIRECTORY, to build the target library (opensearchknn_nmslib) in the specified directory at runtime. set_target_properties(${TARGET_LIB_NMSLIB} PROPERTIES RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/release) else() diff --git a/jni/external/faiss b/jni/external/faiss index 88eabe97f..3219e3d12 160000 --- a/jni/external/faiss +++ b/jni/external/faiss @@ -1 +1 @@ -Subproject commit 88eabe97f96d0c0964dfa075f74373c64d46da80 +Subproject commit 3219e3d12e6fc36dfdfe17d4cf238ef70bf89568 diff --git a/jni/include/faiss_wrapper.h b/jni/include/faiss_wrapper.h index 6c8a86143..284214631 100644 --- a/jni/include/faiss_wrapper.h +++ b/jni/include/faiss_wrapper.h @@ -40,6 +40,12 @@ namespace knn_jni { jobjectArray QueryIndex(knn_jni::JNIUtilInterface * jniUtil, JNIEnv * env, jlong indexPointerJ, jfloatArray queryVectorJ, jint kJ); + // Execute a query against the index located in memory at indexPointerJ along with Filters + // + // Return an array of KNNQueryResults + jobjectArray QueryIndex_WithFilter(knn_jni::JNIUtilInterface * jniUtil, JNIEnv * env, jlong indexPointerJ, + jfloatArray queryVectorJ, jint kJ, jintArray filterIdsJ); + // Free the index located in memory at indexPointerJ void Free(jlong indexPointer); diff --git a/jni/include/org_opensearch_knn_jni_FaissService.h b/jni/include/org_opensearch_knn_jni_FaissService.h index 1ab6c5681..a25264335 100644 --- a/jni/include/org_opensearch_knn_jni_FaissService.h +++ b/jni/include/org_opensearch_knn_jni_FaissService.h @@ -50,6 +50,14 @@ JNIEXPORT jlong JNICALL Java_org_opensearch_knn_jni_FaissService_loadIndex JNIEXPORT jobjectArray JNICALL Java_org_opensearch_knn_jni_FaissService_queryIndex (JNIEnv *, jclass, jlong, jfloatArray, jint); +/* + * Class: org_opensearch_knn_jni_FaissService + * Method: queryIndex_WithFilter + * Signature: (J[FI[J)[Lorg/opensearch/knn/index/query/KNNQueryResult; + */ +JNIEXPORT jobjectArray JNICALL Java_org_opensearch_knn_jni_FaissService_queryIndexWithFilter + (JNIEnv *, jclass, jlong, jfloatArray, jint, jintArray); + /* * Class: org_opensearch_knn_jni_FaissService * Method: free diff --git a/jni/src/faiss_wrapper.cpp b/jni/src/faiss_wrapper.cpp index e0fcc822b..a1bbb9635 100644 --- a/jni/src/faiss_wrapper.cpp +++ b/jni/src/faiss_wrapper.cpp @@ -18,12 +18,18 @@ #include "faiss/IndexHNSW.h" #include "faiss/IndexIVFFlat.h" #include "faiss/MetaIndexes.h" +#include "faiss/Index.h" +#include "faiss/impl/IDSelector.h" #include #include #include #include +// Defines type of IDSelector +enum FilterIdsSelectorType{ + BITMAP, BATCH +}; // Translate space type to faiss metric faiss::MetricType TranslateSpaceToMetric(const std::string& spaceType); @@ -33,7 +39,19 @@ void SetExtraParameters(knn_jni::JNIUtilInterface * jniUtil, JNIEnv *env, const std::unordered_map& parametersCpp, faiss::Index * index); // Train an index with data provided -void InternalTrainIndex(faiss::Index * index, faiss::Index::idx_t n, const float* x); +void InternalTrainIndex(faiss::Index * index, faiss::idx_t n, const float* x); + +// Create the SearchParams based on the Index Type +std::unique_ptr buildSearchParams(const faiss::IndexIDMap *indexReader, faiss::IDSelector* idSelector); + +// Helps to choose the right FilterIdsSelectorType for Faiss +FilterIdsSelectorType getIdSelectorType(const int* filterIds, int filterIdsLength); + +// Converts the int FilterIds to Faiss ids type array. +void convertFilterIdsToFaissIdType(const int* filterIds, int filterIdsLength, faiss::idx_t* convertedFilterIds); + +// Concerts the FilterIds to BitMap +void buildFilterIdsBitMap(const int* filterIds, int filterIdsLength, uint8_t* bitsetVector); void knn_jni::faiss_wrapper::CreateIndex(knn_jni::JNIUtilInterface * jniUtil, JNIEnv * env, jintArray idsJ, jobjectArray vectorsJ, jstring indexPathJ, jobject parametersJ) { @@ -181,12 +199,17 @@ jlong knn_jni::faiss_wrapper::LoadIndex(knn_jni::JNIUtilInterface * jniUtil, JNI jobjectArray knn_jni::faiss_wrapper::QueryIndex(knn_jni::JNIUtilInterface * jniUtil, JNIEnv * env, jlong indexPointerJ, jfloatArray queryVectorJ, jint kJ) { + return knn_jni::faiss_wrapper::QueryIndex_WithFilter(jniUtil, env, indexPointerJ, queryVectorJ, kJ, nullptr); +} + +jobjectArray knn_jni::faiss_wrapper::QueryIndex_WithFilter(knn_jni::JNIUtilInterface * jniUtil, JNIEnv * env, jlong indexPointerJ, + jfloatArray queryVectorJ, jint kJ, jintArray filterIdsJ) { if (queryVectorJ == nullptr) { throw std::runtime_error("Query Vector cannot be null"); } - auto *indexReader = reinterpret_cast(indexPointerJ); + auto *indexReader = reinterpret_cast(indexPointerJ); if (indexReader == nullptr) { throw std::runtime_error("Invalid pointer to index"); @@ -195,14 +218,49 @@ jobjectArray knn_jni::faiss_wrapper::QueryIndex(knn_jni::JNIUtilInterface * jniU // The ids vector will hold the top k ids from the search and the dis vector will hold the top k distances from // the query point std::vector dis(kJ); - std::vector ids(kJ); + std::vector ids(kJ); float* rawQueryvector = jniUtil->GetFloatArrayElements(env, queryVectorJ, nullptr); - - try { - indexReader->search(1, rawQueryvector, kJ, dis.data(), ids.data()); - } catch (...) { - jniUtil->ReleaseFloatArrayElements(env, queryVectorJ, rawQueryvector, JNI_ABORT); - throw; + // create the filterSearch params if the filterIdsJ is not a null pointer + if(filterIdsJ != nullptr) { + int *filteredIdsArray = jniUtil->GetIntArrayElements(env, filterIdsJ, nullptr); + int filterIdsLength = env->GetArrayLength(filterIdsJ); + std::unique_ptr idSelector; + FilterIdsSelectorType idSelectorType = getIdSelectorType(filteredIdsArray, filterIdsLength); + // start with empty vectors for 2 different types of empty Selectors. We need define them here to avoid copying of data + // during the returns. We could have used pass by reference, but we choose pointers. Returning reference to local + // vector is also an option which can be efficient than copying during returns but it requires upto date C++ compilers. + // To avoid all those confusions, its better to work with pointers here. Ref: https://cplusplus.com/forum/general/56177/ + std::vector convertedIds; + std::vector bitmap; + // Choose a selector which suits best + if(idSelectorType == BATCH) { + convertedIds.resize(filterIdsLength); + convertFilterIdsToFaissIdType(filteredIdsArray, filterIdsLength, convertedIds.data()); + idSelector.reset(new faiss::IDSelectorBatch(convertedIds.size(), convertedIds.data())); + } else { + int maxIdValue = filteredIdsArray[filterIdsLength - 1]; + // >> 3 is equivalent to value / 8 + const int bitsetArraySize = (maxIdValue >> 3) + 1; + bitmap.resize(bitsetArraySize, 0); + buildFilterIdsBitMap(filteredIdsArray, filterIdsLength, bitmap.data()); + idSelector.reset(new faiss::IDSelectorBitmap(filterIdsLength, bitmap.data())); + } + std::unique_ptr searchParameters = buildSearchParams(indexReader, idSelector.get()); + try { + indexReader->search(1, rawQueryvector, kJ, dis.data(), ids.data(), searchParameters.get()); + } catch (...) { + jniUtil->ReleaseFloatArrayElements(env, queryVectorJ, rawQueryvector, JNI_ABORT); + jniUtil->ReleaseIntArrayElements(env, filterIdsJ, filteredIdsArray, JNI_ABORT); + throw; + } + jniUtil->ReleaseIntArrayElements(env, filterIdsJ, filteredIdsArray, JNI_ABORT); + } else { + try { + indexReader->search(1, rawQueryvector, kJ, dis.data(), ids.data()); + } catch (...) { + jniUtil->ReleaseFloatArrayElements(env, queryVectorJ, rawQueryvector, JNI_ABORT); + throw; + } } jniUtil->ReleaseFloatArrayElements(env, queryVectorJ, rawQueryvector, JNI_ABORT); @@ -344,7 +402,7 @@ void SetExtraParameters(knn_jni::JNIUtilInterface * jniUtil, JNIEnv *env, } } -void InternalTrainIndex(faiss::Index * index, faiss::Index::idx_t n, const float* x) { +void InternalTrainIndex(faiss::Index * index, faiss::idx_t n, const float* x) { if (auto * indexIvf = dynamic_cast(index)) { if (indexIvf->quantizer_trains_alone == 2) { InternalTrainIndex(indexIvf->quantizer, n, x); @@ -356,3 +414,90 @@ void InternalTrainIndex(faiss::Index * index, faiss::Index::idx_t n, const float index->train(n, x); } } + +/** + * This function takes a call on what ID Selector to use: + * https://github.com/facebookresearch/faiss/wiki/Setting-search-parameters-for-one-query#idselectorarray-idselectorbatch-and-idselectorbitmap + * + * class storage lookup construction(Opensearch + Faiss) + * IDSelectorArray O(k) O(k) O(2k) + * IDSelectorBatch O(k) O(1) O(2k) + * IDSelectorBitmap O(n/8) O(1) O(k) -> n is the max value of id in the index + * + * TODO: We need to ideally decide when we can take another hit of K iterations in latency. Some facts: + * an OpenSearch Index can have max segment size as 5GB which, which on a vector with dimension of 128 boils down to + * 7.5M vectors. + * Ref: https://opensearch.org/docs/latest/search-plugins/knn/knn-index/#hnsw-memory-estimation + * M = 16 + * Dimension = 128 + * (1.1 * ( 4 * 128 + 8 * 16) * 7500000)/(1024*1024*1024) ~ 4.9GB + * Ids are sequential in a Segment which means for IDSelectorBitmap total size if the max ID has value of 7.5M will be + * 7500000/(8*1024) = 915KBs in worst case. But with larger dimensions this worst case value will decrease. + * + * With 915KB how many ids can be represented as an array of 64-bit longs : 117,120 ids + * So iterating on 117k ids for 1 single pass is also time consuming. So, we are currently concluding to consider only size + * as factor. We need to improve on this. + * + * TODO: Best way is to implement a SparseBitSet in C++. This can be done by extending the IDSelector Interface of Faiss. + * + * @param filterIds + * @param filterIdsLength + * @return std::string + */ +FilterIdsSelectorType getIdSelectorType(const int* filterIds, int filterIdsLength) { + int maxIdValue = filterIds[filterIdsLength - 1]; + if(filterIdsLength * sizeof(faiss::idx_t) * 8 <= maxIdValue ) { + return BATCH; + } + return BITMAP; +} + +void convertFilterIdsToFaissIdType(const int* filterIds, int filterIdsLength, faiss::idx_t* convertedFilterIds) { + for (int i = 0; i < filterIdsLength; i++) { + convertedFilterIds[i] = filterIds[i]; + } +} + +void buildFilterIdsBitMap(const int* filterIds, int filterIdsLength, uint8_t* bitsetVector) { + /** + * Coming from Faiss IDSelectorBitmap::is_member function bitmap id will be selected + * iff id / 8 < n and bit number (i%8) of bitmap[floor(i / 8)] is 1. + */ + for(int i = 0 ; i < filterIdsLength ; i ++) { + int value = filterIds[i]; + // / , % are expensive operation. Hence, using BitShift operation as they are fast. + int bitsetArrayIndex = value >> 3 ; // is equivalent to value / 8 + // (value & 7) equivalent to value % 8 + bitsetVector[bitsetArrayIndex] = bitsetVector[bitsetArrayIndex] | (1 << (value & 7)); + } +} + +/** + * Based on the type of the index reader we need to return the SearchParameters. The way we do this by dynamically + * casting the IndexReader. + * @param indexReader + * @param idSelector + * @return SearchParameters + */ +std::unique_ptr buildSearchParams(const faiss::IndexIDMap *indexReader, faiss::IDSelector* idSelector) { + auto hnswReader = dynamic_cast(indexReader->index); + if(hnswReader) { + // we need to make this variable unique_ptr so that the scope can be shared with caller function. + std::unique_ptr hnswParams(new faiss::SearchParametersHNSW); + // Setting the ef_search value equal to what was provided during index creation. SearchParametersHNSW has a default + // value of ef_search = 16 which will then be used. + hnswParams->efSearch = hnswReader->hnsw.efSearch; + hnswParams->sel = idSelector; + return hnswParams; + } + + auto ivfReader = dynamic_cast(indexReader->index); + auto ivfFlatReader = dynamic_cast(indexReader->index); + if(ivfReader || ivfFlatReader) { + // we need to make this variable unique_ptr so that the scope can be shared with caller function. + std::unique_ptr ivfParams(new faiss::SearchParametersIVF); + ivfParams->sel = idSelector; + return ivfParams; + } + throw std::runtime_error("Invalid Index Type supported for Filtered Search on Faiss"); +} diff --git a/jni/src/org_opensearch_knn_jni_FaissService.cpp b/jni/src/org_opensearch_knn_jni_FaissService.cpp index 543ce8ec4..1b79d9114 100644 --- a/jni/src/org_opensearch_knn_jni_FaissService.cpp +++ b/jni/src/org_opensearch_knn_jni_FaissService.cpp @@ -88,6 +88,18 @@ JNIEXPORT jobjectArray JNICALL Java_org_opensearch_knn_jni_FaissService_queryInd return nullptr; } +JNIEXPORT jobjectArray JNICALL Java_org_opensearch_knn_jni_FaissService_queryIndexWithFilter + (JNIEnv * env, jclass cls, jlong indexPointerJ, jfloatArray queryVectorJ, jint kJ, jintArray filteredIdsJ) { + + try { + return knn_jni::faiss_wrapper::QueryIndex_WithFilter(&jniUtil, env, indexPointerJ, queryVectorJ, kJ, filteredIdsJ); + } catch (...) { + jniUtil.CatchCppExceptionAndThrowJava(env); + } + return nullptr; + +} + JNIEXPORT void JNICALL Java_org_opensearch_knn_jni_FaissService_free(JNIEnv * env, jclass cls, jlong indexPointerJ) { try { diff --git a/src/main/java/org/opensearch/knn/index/query/KNNQuery.java b/src/main/java/org/opensearch/knn/index/query/KNNQuery.java index 9bf38008b..5ac207c43 100644 --- a/src/main/java/org/opensearch/knn/index/query/KNNQuery.java +++ b/src/main/java/org/opensearch/knn/index/query/KNNQuery.java @@ -5,6 +5,11 @@ package org.opensearch.knn.index.query; +import lombok.Getter; +import lombok.Setter; +import org.apache.lucene.search.BooleanClause; +import org.apache.lucene.search.BooleanQuery; +import org.apache.lucene.search.FieldExistsQuery; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; import org.apache.lucene.search.QueryVisitor; @@ -25,6 +30,10 @@ public class KNNQuery extends Query { private final int k; private final String indexName; + @Getter + @Setter + private Query filterQuery; + public KNNQuery(String field, float[] queryVector, int k, String indexName) { this.field = field; this.queryVector = queryVector; @@ -32,6 +41,14 @@ public KNNQuery(String field, float[] queryVector, int k, String indexName) { this.indexName = indexName; } + public KNNQuery(String field, float[] queryVector, int k, String indexName, Query filterQuery) { + this.field = field; + this.queryVector = queryVector; + this.k = k; + this.indexName = indexName; + this.filterQuery = filterQuery; + } + public String getField() { return this.field; } @@ -61,9 +78,25 @@ public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float bo if (!KNNSettings.isKNNPluginEnabled()) { throw new IllegalStateException("KNN plugin is disabled. To enable update knn.plugin.enabled to true"); } + final Weight filterWeight = getFilterWeight(searcher); + if (filterWeight != null) { + return new KNNWeight(this, boost, filterWeight); + } return new KNNWeight(this, boost); } + private Weight getFilterWeight(IndexSearcher searcher) throws IOException { + if (this.getFilterQuery() != null) { + // Run the filter query + final BooleanQuery booleanQuery = new BooleanQuery.Builder().add(this.getFilterQuery(), BooleanClause.Occur.FILTER) + .add(new FieldExistsQuery(this.getField()), BooleanClause.Occur.FILTER) + .build(); + final Query rewritten = searcher.rewrite(booleanQuery); + return searcher.createWeight(rewritten, ScoreMode.COMPLETE_NO_SCORES, 1f); + } + return null; + } + @Override public void visit(QueryVisitor visitor) { diff --git a/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java b/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java index c9f6ebe6d..f215335a0 100644 --- a/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java +++ b/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java @@ -283,7 +283,7 @@ protected Query doToQuery(QueryShardContext context) { ); } - if (KNNEngine.getEnginesThatCreateCustomSegmentFiles().contains(knnEngine) && filter != null) { + if (KNNEngine.getEnginesThatCreateCustomSegmentFiles().contains(knnEngine) && filter != null && knnEngine != KNNEngine.FAISS) { throw new IllegalArgumentException(String.format("Engine [%s] does not support filters", knnEngine)); } diff --git a/src/main/java/org/opensearch/knn/index/query/KNNQueryFactory.java b/src/main/java/org/opensearch/knn/index/query/KNNQueryFactory.java index 188bbc150..20c456c4a 100644 --- a/src/main/java/org/opensearch/knn/index/query/KNNQueryFactory.java +++ b/src/main/java/org/opensearch/knn/index/query/KNNQueryFactory.java @@ -59,27 +59,53 @@ public static Query create(CreateQueryRequest createQueryRequest) { final String fieldName = createQueryRequest.getFieldName(); final int k = createQueryRequest.getK(); final float[] vector = createQueryRequest.getVector(); + final Query filterQuery = getFilterQuery(createQueryRequest); if (KNNEngine.getEnginesThatCreateCustomSegmentFiles().contains(createQueryRequest.getKnnEngine())) { + if (filterQuery != null && KNNEngine.getEnginesThatSupportsFilters().contains(createQueryRequest.getKnnEngine())) { + log.debug( + String.format( + "Creating custom k-NN query with filters for index: %s \"\", field: %s \"\", " + "k: %d", + indexName, + fieldName, + k + ) + ); + return new KNNQuery(fieldName, vector, k, indexName, filterQuery); + } log.debug(String.format("Creating custom k-NN query for index: %s \"\", field: %s \"\", k: %d", indexName, fieldName, k)); return new KNNQuery(fieldName, vector, k, indexName); } + if (filterQuery != null) { + log.debug( + String.format("Creating Lucene k-NN query with filters for index: %s \"\", field: %s \"\", k: %d", indexName, fieldName, k) + ); + return new KnnFloatVectorQuery(fieldName, vector, k, filterQuery); + } + log.debug(String.format("Creating Lucene k-NN query for index: %s \"\", field: %s \"\", k: %d", indexName, fieldName, k)); + return new KnnFloatVectorQuery(fieldName, vector, k); + } + + private static Query getFilterQuery(CreateQueryRequest createQueryRequest) { if (createQueryRequest.getFilter().isPresent()) { final QueryShardContext queryShardContext = createQueryRequest.getContext() .orElseThrow(() -> new RuntimeException("Shard context cannot be null")); log.debug( - String.format("Creating Lucene k-NN query with filter for index [%s], field [%s] and k [%d]", indexName, fieldName, k) + String.format( + "Creating k-NN query with filter for index [%s], field [%s] and k [%d]", + createQueryRequest.getIndexName(), + createQueryRequest.fieldName, + createQueryRequest.k + ) ); try { - final Query filterQuery = createQueryRequest.getFilter().get().toQuery(queryShardContext); - return new KnnFloatVectorQuery(fieldName, vector, k, filterQuery); + return createQueryRequest.getFilter().get().toQuery(queryShardContext); } catch (IOException e) { throw new RuntimeException("Cannot create knn query with filter", e); } } - log.debug(String.format("Creating Lucene k-NN query for index: %s \"\", field: %s \"\", k: %d", indexName, fieldName, k)); - return new KnnFloatVectorQuery(fieldName, vector, k); + return null; } /** diff --git a/src/main/java/org/opensearch/knn/index/query/KNNScorer.java b/src/main/java/org/opensearch/knn/index/query/KNNScorer.java index 0005212bf..3e5c8fff6 100644 --- a/src/main/java/org/opensearch/knn/index/query/KNNScorer.java +++ b/src/main/java/org/opensearch/knn/index/query/KNNScorer.java @@ -56,4 +56,37 @@ public float score() { public int docID() { return docIdsIter.docID(); } + + /** + * Returns the Empty Scorer implementation. We use this scorer to short circuit the actual search when it is not + * required. + * @param knnWeight {@link KNNWeight} + * @return {@link KNNScorer} + */ + public static Scorer emptyScorer(KNNWeight knnWeight) { + return new Scorer(knnWeight) { + private final DocIdSetIterator docIdsIter = DocIdSetIterator.empty(); + + @Override + public DocIdSetIterator iterator() { + return docIdsIter; + } + + @Override + public float getMaxScore(int upTo) throws IOException { + return 0; + } + + @Override + public float score() throws IOException { + assert docID() != DocIdSetIterator.NO_MORE_DOCS; + return 0; + } + + @Override + public int docID() { + return docIdsIter.docID(); + } + }; + } } diff --git a/src/main/java/org/opensearch/knn/index/query/KNNWeight.java b/src/main/java/org/opensearch/knn/index/query/KNNWeight.java index 716aed412..b8b88b4fe 100644 --- a/src/main/java/org/opensearch/knn/index/query/KNNWeight.java +++ b/src/main/java/org/opensearch/knn/index/query/KNNWeight.java @@ -5,16 +5,27 @@ package org.opensearch.knn.index.query; +import lombok.extern.log4j.Log4j2; +import org.apache.lucene.index.BinaryDocValues; +import org.apache.lucene.index.DocValues; +import org.apache.lucene.search.FilteredDocIdSetIterator; +import org.apache.lucene.search.HitQueue; +import org.apache.lucene.search.ScoreDoc; +import org.apache.lucene.util.BitSet; +import org.apache.lucene.util.BitSetIterator; +import org.apache.lucene.util.Bits; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.FixedBitSet; import org.opensearch.knn.common.KNNConstants; import org.opensearch.knn.index.SpaceType; +import org.opensearch.knn.index.codec.util.KNNVectorSerializer; +import org.opensearch.knn.index.codec.util.KNNVectorSerializerFactory; import org.opensearch.knn.jni.JNIService; import org.opensearch.knn.index.memory.NativeMemoryAllocation; import org.opensearch.knn.index.memory.NativeMemoryCacheManager; import org.opensearch.knn.index.memory.NativeMemoryEntryContext; import org.opensearch.knn.index.memory.NativeMemoryLoadStrategy; import org.opensearch.knn.index.util.KNNEngine; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.FilterLeafReader; import org.apache.lucene.index.LeafReaderContext; @@ -31,10 +42,12 @@ import org.opensearch.knn.indices.ModelMetadata; import org.opensearch.knn.plugin.stats.KNNCounter; +import java.io.ByteArrayInputStream; import java.io.IOException; import java.nio.file.Path; import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.concurrent.ExecutionException; @@ -49,20 +62,30 @@ /** * Calculate query weights and build query scorers. */ +@Log4j2 public class KNNWeight extends Weight { - private static Logger logger = LogManager.getLogger(KNNWeight.class); private static ModelDao modelDao; private final KNNQuery knnQuery; private final float boost; - private NativeMemoryCacheManager nativeMemoryCacheManager; + private final NativeMemoryCacheManager nativeMemoryCacheManager; + private final Weight filterWeight; public KNNWeight(KNNQuery query, float boost) { super(query); this.knnQuery = query; this.boost = boost; this.nativeMemoryCacheManager = NativeMemoryCacheManager.getInstance(); + this.filterWeight = null; + } + + public KNNWeight(KNNQuery query, float boost, Weight filterWeight) { + super(query); + this.knnQuery = query; + this.boost = boost; + this.nativeMemoryCacheManager = NativeMemoryCacheManager.getInstance(); + this.filterWeight = filterWeight; } public static void initialize(ModelDao modelDao) { @@ -76,13 +99,91 @@ public Explanation explain(LeafReaderContext context, int doc) { @Override public Scorer scorer(LeafReaderContext context) throws IOException { + final int[] filterIdsArray = getFilterIdsArray(context); + // We don't need to go to JNI layer if no documents are found which satisfy the filters + // We should give this condition a deeper look that where it should be placed. For now I feel this is a good + // place, + if (filterWeight != null && filterIdsArray.length == 0) { + return KNNScorer.emptyScorer(this); + } + final Map docIdsToScoreMap = new HashMap<>(); + + /* + * The idea for this optimization is to get K results, we need to atleast look at K vectors in the HNSW graph + * . Hence, if filtered results are less than K and filter query is present we should shift to exact search. + * This improves the recall. + */ + if (filterWeight != null && filterIdsArray.length <= knnQuery.getK()) { + docIdsToScoreMap.putAll(doExactSearch(context, filterIdsArray)); + } else { + final Map annResults = doANNSearch(context, filterIdsArray); + if (annResults == null) { + return null; + } + docIdsToScoreMap.putAll(annResults); + } + if (docIdsToScoreMap.isEmpty()) { + return KNNScorer.emptyScorer(this); + } + return convertSearchResponseToScorer(docIdsToScoreMap); + } + + private BitSet getFilteredDocsBitSet(final LeafReaderContext ctx, final Weight filterWeight) throws IOException { + final Bits liveDocs = ctx.reader().getLiveDocs(); + final int maxDoc = ctx.reader().maxDoc(); + + final Scorer scorer = filterWeight.scorer(ctx); + if (scorer == null) { + return new FixedBitSet(0); + } + + return createBitSet(scorer.iterator(), liveDocs, maxDoc); + } + + private BitSet createBitSet(final DocIdSetIterator filteredDocIdsIterator, final Bits liveDocs, int maxDoc) throws IOException { + if (liveDocs == null && filteredDocIdsIterator instanceof BitSetIterator) { + // If we already have a BitSet and no deletions, reuse the BitSet + return ((BitSetIterator) filteredDocIdsIterator).getBitSet(); + } + // Create a new BitSet from matching and live docs + FilteredDocIdSetIterator filterIterator = new FilteredDocIdSetIterator(filteredDocIdsIterator) { + @Override + protected boolean match(int doc) { + return liveDocs == null || liveDocs.get(doc); + } + }; + return BitSet.of(filterIterator, maxDoc); + } + + private int[] getFilterIdsArray(final LeafReaderContext context) throws IOException { + if (filterWeight == null) { + return new int[0]; + } + final BitSet filteredDocsBitSet = getFilteredDocsBitSet(context, this.filterWeight); + final int[] filteredIds = new int[filteredDocsBitSet.cardinality()]; + int filteredIdsIndex = 0; + int docId = 0; + while (docId < filteredDocsBitSet.length()) { + docId = filteredDocsBitSet.nextSetBit(docId); + if (docId == DocIdSetIterator.NO_MORE_DOCS || docId + 1 == DocIdSetIterator.NO_MORE_DOCS) { + break; + } + log.debug("Docs in filtered docs id set is : {}", docId); + filteredIds[filteredIdsIndex] = docId; + filteredIdsIndex++; + docId++; + } + return filteredIds; + } + + private Map doANNSearch(final LeafReaderContext context, final int[] filterIdsArray) throws IOException { SegmentReader reader = (SegmentReader) FilterLeafReader.unwrap(context.reader()); String directory = ((FSDirectory) FilterDirectory.unwrap(reader.directory())).getDirectory().toString(); FieldInfo fieldInfo = reader.getFieldInfos().fieldInfo(knnQuery.getField()); if (fieldInfo == null) { - logger.debug("[KNN] Field info not found for {}:{}", knnQuery.getField(), reader.getSegmentName()); + log.debug("[KNN] Field info not found for {}:{}", knnQuery.getField(), reader.getSegmentName()); return null; } @@ -121,7 +222,7 @@ public Scorer scorer(LeafReaderContext context) throws IOException { .collect(Collectors.toList()); if (engineFiles.isEmpty()) { - logger.debug("[KNN] No engine index found for field {} for segment {}", knnQuery.getField(), reader.getSegmentName()); + log.debug("[KNN] No engine index found for field {} for segment {}", knnQuery.getField(), reader.getSegmentName()); return null; } @@ -148,7 +249,6 @@ public Scorer scorer(LeafReaderContext context) throws IOException { // Now that we have the allocation, we need to readLock it indexAllocation.readLock(); - try { if (indexAllocation.isClosed()) { throw new RuntimeException("Index has already been closed"); @@ -158,8 +258,10 @@ public Scorer scorer(LeafReaderContext context) throws IOException { indexAllocation.getMemoryAddress(), knnQuery.getQueryVector(), knnQuery.getK(), - knnEngine.getName() + knnEngine.getName(), + filterIdsArray ); + } catch (Exception e) { GRAPH_QUERY_ERRORS.increment(); throw new RuntimeException(e); @@ -174,21 +276,70 @@ public Scorer scorer(LeafReaderContext context) throws IOException { * neighbors we are inverting the scores. */ if (results.length == 0) { - logger.debug("[KNN] Query yielded 0 results"); + log.debug("[KNN] Query yielded 0 results"); return null; } - Map scores = Arrays.stream(results) + return Arrays.stream(results) .collect(Collectors.toMap(KNNQueryResult::getId, result -> knnEngine.score(result.getScore(), spaceType))); - int maxDoc = Collections.max(scores.keySet()) + 1; - DocIdSetBuilder docIdSetBuilder = new DocIdSetBuilder(maxDoc); + } + + private Map doExactSearch(final LeafReaderContext leafReaderContext, final int[] filterIdsArray) { + final SegmentReader reader = (SegmentReader) FilterLeafReader.unwrap(leafReaderContext.reader()); + final FieldInfo fieldInfo = reader.getFieldInfos().fieldInfo(knnQuery.getField()); + float[] queryVector = this.knnQuery.getQueryVector(); + try { + final BinaryDocValues values = DocValues.getBinary(leafReaderContext.reader(), fieldInfo.getName()); + final SpaceType spaceType = SpaceType.getSpace(fieldInfo.getAttribute(SPACE_TYPE)); + // Creating min heap and init with MAX DocID and Score as -INF. + final HitQueue queue = new HitQueue(this.knnQuery.getK(), true); + ScoreDoc topDoc = queue.top(); + final Map docToScore = new HashMap<>(); + for (int filterId : filterIdsArray) { + int docId = values.advance(filterId); + final BytesRef value = values.binaryValue(); + final ByteArrayInputStream byteStream = new ByteArrayInputStream(value.bytes, value.offset, value.length); + final KNNVectorSerializer vectorSerializer = KNNVectorSerializerFactory.getSerializerByStreamContent(byteStream); + final float[] vector = vectorSerializer.byteToFloatArray(byteStream); + // Calculates a similarity score between the two vectors with a specified function. Higher similarity + // scores correspond to closer vectors. + float score = spaceType.getVectorSimilarityFunction().compare(queryVector, vector); + if (score > topDoc.score) { + topDoc.score = score; + topDoc.doc = docId; + // As the HitQueue is min heap, updating top will bring the doc with -INF score or worst score we + // have seen till now on top. + topDoc = queue.updateTop(); + } + } + // If scores are negative we will remove them. + // This is done, because there can be negative values in the Heap as we init the heap with Score as -INF. + // If filterIds < k, the some values in heap can have a negative score. + while (queue.size() > 0 && queue.top().score < 0) { + queue.pop(); + } + + while (queue.size() > 0) { + final ScoreDoc doc = queue.pop(); + docToScore.put(doc.doc, doc.score); + } + + return docToScore; + } catch (Exception e) { + log.error("Error while getting the doc values to do the k-NN Search for query : {}", this.knnQuery, e); + } + return Collections.emptyMap(); + } + private Scorer convertSearchResponseToScorer(final Map docsToScore) throws IOException { + final int maxDoc = Collections.max(docsToScore.keySet()) + 1; + final DocIdSetBuilder docIdSetBuilder = new DocIdSetBuilder(maxDoc); // The docIdSetIterator will contain the docids of the returned results. So, before adding results to - // the builder, we can grow to results.length - DocIdSetBuilder.BulkAdder setAdder = docIdSetBuilder.grow(results.length); - Arrays.stream(results).forEach(result -> setAdder.add(result.getId())); - DocIdSetIterator docIdSetIter = docIdSetBuilder.build().iterator(); - return new KNNScorer(this, docIdSetIter, scores, boost); + // the builder, we can grow to docsToScore.size() + final DocIdSetBuilder.BulkAdder setAdder = docIdSetBuilder.grow(docsToScore.size()); + docsToScore.keySet().forEach(setAdder::add); + final DocIdSetIterator docIdSetIter = docIdSetBuilder.build().iterator(); + return new KNNScorer(this, docIdSetIter, docsToScore, boost); } @Override diff --git a/src/main/java/org/opensearch/knn/index/util/KNNEngine.java b/src/main/java/org/opensearch/knn/index/util/KNNEngine.java index fe28de43e..776ea5366 100644 --- a/src/main/java/org/opensearch/knn/index/util/KNNEngine.java +++ b/src/main/java/org/opensearch/knn/index/util/KNNEngine.java @@ -32,6 +32,7 @@ public enum KNNEngine implements KNNLibrary { public static final KNNEngine DEFAULT = NMSLIB; private static final Set CUSTOM_SEGMENT_FILE_ENGINES = ImmutableSet.of(KNNEngine.NMSLIB, KNNEngine.FAISS); + private static final Set ENGINES_SUPPORTING_FILTERS = ImmutableSet.of(KNNEngine.LUCENE, KNNEngine.FAISS); private static Map MAX_DIMENSIONS_BY_ENGINE = Map.of( KNNEngine.NMSLIB, @@ -105,6 +106,10 @@ public static Set getEnginesThatCreateCustomSegmentFiles() { return CUSTOM_SEGMENT_FILE_ENGINES; } + public static Set getEnginesThatSupportsFilters() { + return ENGINES_SUPPORTING_FILTERS; + } + /** * Return number of max allowed dimensions per single vector based on the knn engine * @param knnEngine knn engine to check max dimensions value diff --git a/src/main/java/org/opensearch/knn/index/util/KNNLibrary.java b/src/main/java/org/opensearch/knn/index/util/KNNLibrary.java index b990ce33b..ba1d3ac84 100644 --- a/src/main/java/org/opensearch/knn/index/util/KNNLibrary.java +++ b/src/main/java/org/opensearch/knn/index/util/KNNLibrary.java @@ -122,6 +122,6 @@ public interface KNNLibrary { * @return list of file extensions that will be read/write with mmap */ default List mmapFileExtensions() { - return Collections.EMPTY_LIST; + return Collections.emptyList(); } } diff --git a/src/main/java/org/opensearch/knn/jni/FaissService.java b/src/main/java/org/opensearch/knn/jni/FaissService.java index f1d869bd2..5dce15d6e 100644 --- a/src/main/java/org/opensearch/knn/jni/FaissService.java +++ b/src/main/java/org/opensearch/knn/jni/FaissService.java @@ -24,7 +24,7 @@ * * In order to compile C++ header file, run: * javac -h jni/include src/main/java/org/opensearch/knn/jni/FaissService.java - * src/main/java/org/opensearch/knn/index/KNNQueryResult.java + * src/main/java/org/opensearch/knn/index/query/KNNQueryResult.java * src/main/java/org/opensearch/knn/common/KNNConstants.java */ class FaissService { @@ -83,6 +83,8 @@ public static native void createIndexFromTemplate( */ public static native KNNQueryResult[] queryIndex(long indexPointer, float[] queryVector, int k); + public static native KNNQueryResult[] queryIndexWithFilter(long indexPointer, float[] queryVector, int k, int[] filterIds); + /** * Free native memory pointer */ diff --git a/src/main/java/org/opensearch/knn/jni/JNIService.java b/src/main/java/org/opensearch/knn/jni/JNIService.java index e32880fff..f45fb0c73 100644 --- a/src/main/java/org/opensearch/knn/jni/JNIService.java +++ b/src/main/java/org/opensearch/knn/jni/JNIService.java @@ -11,6 +11,7 @@ package org.opensearch.knn.jni; +import org.apache.commons.lang.ArrayUtils; import org.opensearch.knn.index.query.KNNQueryResult; import org.opensearch.knn.index.util.KNNEngine; @@ -94,20 +95,27 @@ public static long loadIndex(String indexPath, Map parameters, S * Query an index * * @param indexPointer pointer to index in memory - * @param queryVector vector to be used for query - * @param k neighbors to be returned - * @param engineName name of engine to query index + * @param queryVector vector to be used for query + * @param k neighbors to be returned + * @param engineName name of engine to query index + * @param filteredIds array of ints on which should be used for search. * @return KNNQueryResult array of k neighbors */ - public static KNNQueryResult[] queryIndex(long indexPointer, float[] queryVector, int k, String engineName) { + public static KNNQueryResult[] queryIndex(long indexPointer, float[] queryVector, int k, String engineName, int[] filteredIds) { if (KNNEngine.NMSLIB.getName().equals(engineName)) { return NmslibService.queryIndex(indexPointer, queryVector, k); } if (KNNEngine.FAISS.getName().equals(engineName)) { + // This code assumes that if filteredIds == null / filteredIds.length == 0 if filter is specified then empty + // k-NN results are already returned. Otherwise, it's a filter case and we need to run search with + // filterIds. FilterIds is coming as empty then its the case where we need to do search with Faiss engine + // normally. + if (ArrayUtils.isNotEmpty(filteredIds)) { + return FaissService.queryIndexWithFilter(indexPointer, queryVector, k, filteredIds); + } return FaissService.queryIndex(indexPointer, queryVector, k); } - throw new IllegalArgumentException("QueryIndex not supported for provided engine"); } diff --git a/src/test/java/org/opensearch/knn/index/FaissIT.java b/src/test/java/org/opensearch/knn/index/FaissIT.java index 8eb99b625..8eff19da5 100644 --- a/src/test/java/org/opensearch/knn/index/FaissIT.java +++ b/src/test/java/org/opensearch/knn/index/FaissIT.java @@ -12,11 +12,14 @@ package org.opensearch.knn.index; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import com.google.common.primitives.Floats; import org.apache.http.util.EntityUtils; +import lombok.SneakyThrows; import org.junit.BeforeClass; import org.opensearch.client.Response; import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.index.query.QueryBuilders; import org.opensearch.knn.KNNRestTestCase; import org.opensearch.common.Strings; import org.opensearch.common.xcontent.XContentFactory; @@ -43,6 +46,11 @@ import static org.opensearch.knn.common.KNNConstants.PARAMETERS; public class FaissIT extends KNNRestTestCase { + private static final String DOC_ID_1 = "doc1"; + private static final String DOC_ID_2 = "doc2"; + private static final String DOC_ID_3 = "doc3"; + private static final String COLOR_FIELD_NAME = "color"; + private static final String TASTE_FIELD_NAME = "taste"; static TestUtils.TestData testData; @@ -280,4 +288,90 @@ public void testEndToEnd_fromModel() throws IOException, InterruptedException { assertEquals(numDocs - i - 1, Integer.parseInt(results.get(i).getDocId())); } } + + @SneakyThrows + public void testQueryWithFilter_withDifferentCombination_thenSuccess() { + setupKNNIndexForFilterQuery(); + final float[] searchVector = { 6.0f, 6.0f, 4.1f }; + // K > filteredResults + int kGreaterThanFilterResult = 5; + List expectedDocIds = Arrays.asList(DOC_ID_1, DOC_ID_3); + final Response response = searchKNNIndex( + INDEX_NAME, + new KNNQueryBuilder(FIELD_NAME, searchVector, kGreaterThanFilterResult, QueryBuilders.termQuery(COLOR_FIELD_NAME, "red")), + kGreaterThanFilterResult + ); + final String responseBody = EntityUtils.toString(response.getEntity()); + final List knnResults = parseSearchResponse(responseBody, FIELD_NAME); + + assertEquals(expectedDocIds.size(), knnResults.size()); + assertTrue(knnResults.stream().map(KNNResult::getDocId).collect(Collectors.toList()).containsAll(expectedDocIds)); + + // K Limits Filter results + int kLimitsFilterResult = 1; + List expectedDocIdsKLimitsFilterResult = List.of(DOC_ID_1); + final Response responseKLimitsFilterResult = searchKNNIndex( + INDEX_NAME, + new KNNQueryBuilder(FIELD_NAME, searchVector, kLimitsFilterResult, QueryBuilders.termQuery(COLOR_FIELD_NAME, "red")), + kLimitsFilterResult + ); + final String responseBodyKLimitsFilterResult = EntityUtils.toString(responseKLimitsFilterResult.getEntity()); + final List knnResultsKLimitsFilterResult = parseSearchResponse(responseBodyKLimitsFilterResult, FIELD_NAME); + + assertEquals(expectedDocIdsKLimitsFilterResult.size(), knnResultsKLimitsFilterResult.size()); + assertTrue( + knnResultsKLimitsFilterResult.stream() + .map(KNNResult::getDocId) + .collect(Collectors.toList()) + .containsAll(expectedDocIdsKLimitsFilterResult) + ); + + // Empty filter docIds + int k = 10; + final Response emptyFilterResponse = searchKNNIndex( + INDEX_NAME, + new KNNQueryBuilder( + FIELD_NAME, + searchVector, + kLimitsFilterResult, + QueryBuilders.termQuery(COLOR_FIELD_NAME, "color_not_present") + ), + k + ); + final String responseBodyForEmptyDocIds = EntityUtils.toString(emptyFilterResponse.getEntity()); + final List emptyKNNFilteredResultsFromResponse = parseSearchResponse(responseBodyForEmptyDocIds, FIELD_NAME); + + assertEquals(0, emptyKNNFilteredResultsFromResponse.size()); + } + + protected void setupKNNIndexForFilterQuery() throws Exception { + // Create Mappings + XContentBuilder builder = XContentFactory.jsonBuilder() + .startObject() + .startObject("properties") + .startObject(FIELD_NAME) + .field("type", "knn_vector") + .field("dimension", 3) + .startObject(KNNConstants.KNN_METHOD) + .field(KNNConstants.NAME, KNNEngine.FAISS.getMethod(KNNConstants.METHOD_HNSW).getMethodComponent().getName()) + .field(KNNConstants.METHOD_PARAMETER_SPACE_TYPE, SpaceType.L2) + .field(KNNConstants.KNN_ENGINE, KNNEngine.FAISS.getName()) + .endObject() + .endObject() + .endObject() + .endObject(); + final String mapping = Strings.toString(builder); + + createKnnIndex(INDEX_NAME, mapping); + + addKnnDocWithAttributes( + DOC_ID_1, + new float[] { 6.0f, 7.9f, 3.1f }, + ImmutableMap.of(COLOR_FIELD_NAME, "red", TASTE_FIELD_NAME, "sweet") + ); + addKnnDocWithAttributes(DOC_ID_2, new float[] { 3.2f, 2.1f, 4.8f }, ImmutableMap.of(COLOR_FIELD_NAME, "green")); + addKnnDocWithAttributes(DOC_ID_3, new float[] { 4.1f, 5.0f, 7.1f }, ImmutableMap.of(COLOR_FIELD_NAME, "red")); + + refreshIndex(INDEX_NAME); + } } diff --git a/src/test/java/org/opensearch/knn/index/LuceneEngineIT.java b/src/test/java/org/opensearch/knn/index/LuceneEngineIT.java index 2266d755d..b05211b25 100644 --- a/src/test/java/org/opensearch/knn/index/LuceneEngineIT.java +++ b/src/test/java/org/opensearch/knn/index/LuceneEngineIT.java @@ -25,7 +25,6 @@ import org.opensearch.knn.common.KNNConstants; import org.opensearch.knn.index.query.KNNQueryBuilder; import org.opensearch.knn.index.util.KNNEngine; -import org.opensearch.rest.RestStatus; import java.io.IOException; import java.util.Arrays; @@ -45,8 +44,6 @@ public class LuceneEngineIT extends KNNRestTestCase { private static final String DOC_ID_2 = "doc2"; private static final String DOC_ID_3 = "doc3"; private static final int EF_CONSTRUCTION = 128; - private static final String INDEX_NAME = "test-index-1"; - private static final String FIELD_NAME = "test-field-1"; private static final String COLOR_FIELD_NAME = "color"; private static final String TASTE_FIELD_NAME = "taste"; private static final int M = 16; @@ -361,22 +358,6 @@ public void testIndexReopening() throws Exception { assertArrayEquals(knnResultsBeforeIndexClosure.toArray(), knnResultsAfterIndexClosure.toArray()); } - private void addKnnDocWithAttributes(String docId, float[] vector, Map fieldValues) throws IOException { - Request request = new Request("POST", "/" + INDEX_NAME + "/_doc/" + docId + "?refresh=true"); - - XContentBuilder builder = XContentFactory.jsonBuilder().startObject().field(FIELD_NAME, vector); - for (String fieldName : fieldValues.keySet()) { - builder.field(fieldName, fieldValues.get(fieldName)); - } - builder.endObject(); - request.setJsonEntity(Strings.toString(builder)); - client().performRequest(request); - - request = new Request("POST", "/" + INDEX_NAME + "/_refresh"); - Response response = client().performRequest(request); - assertEquals(request.getEndpoint() + ": failed", RestStatus.OK, RestStatus.fromCode(response.getStatusLine().getStatusCode())); - } - private void createKnnIndexMappingWithLuceneEngine(int dimension, SpaceType spaceType) throws Exception { XContentBuilder builder = XContentFactory.jsonBuilder() .startObject() diff --git a/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestUtil.java b/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestUtil.java index 1e8c255f2..ad0cd37a0 100644 --- a/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestUtil.java +++ b/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestUtil.java @@ -333,7 +333,7 @@ public static void assertLoadableByEngine( ); int k = 2; float[] queryVector = new float[dimension]; - KNNQueryResult[] results = JNIService.queryIndex(indexPtr, queryVector, k, knnEngine.getName()); + KNNQueryResult[] results = JNIService.queryIndex(indexPtr, queryVector, k, knnEngine.getName(), null); assertTrue(results.length > 0); JNIService.free(indexPtr, knnEngine.getName()); } diff --git a/src/test/java/org/opensearch/knn/index/memory/NativeMemoryLoadStrategyTests.java b/src/test/java/org/opensearch/knn/index/memory/NativeMemoryLoadStrategyTests.java index 8d94b1afb..ce08e0350 100644 --- a/src/test/java/org/opensearch/knn/index/memory/NativeMemoryLoadStrategyTests.java +++ b/src/test/java/org/opensearch/knn/index/memory/NativeMemoryLoadStrategyTests.java @@ -74,7 +74,7 @@ public void testIndexLoadStrategy_load() throws IOException { // Confirm that the file was loaded by querying float[] query = new float[dimension]; Arrays.fill(query, numVectors + 1); - KNNQueryResult[] results = JNIService.queryIndex(indexAllocation.getMemoryAddress(), query, 2, knnEngine.getName()); + KNNQueryResult[] results = JNIService.queryIndex(indexAllocation.getMemoryAddress(), query, 2, knnEngine.getName(), null); assertTrue(results.length > 0); } diff --git a/src/test/java/org/opensearch/knn/index/query/KNNQueryFactoryTests.java b/src/test/java/org/opensearch/knn/index/query/KNNQueryFactoryTests.java index 0f8f43bf2..674d1be39 100644 --- a/src/test/java/org/opensearch/knn/index/query/KNNQueryFactoryTests.java +++ b/src/test/java/org/opensearch/knn/index/query/KNNQueryFactoryTests.java @@ -5,8 +5,11 @@ package org.opensearch.knn.index.query; +import org.apache.lucene.index.Term; import org.apache.lucene.search.KnnFloatVectorQuery; import org.apache.lucene.search.Query; +import org.apache.lucene.search.TermQuery; +import org.mockito.Mockito; import org.opensearch.index.mapper.MappedFieldType; import org.opensearch.index.query.QueryBuilder; import org.opensearch.index.query.QueryShardContext; @@ -23,6 +26,10 @@ import static org.mockito.Mockito.when; public class KNNQueryFactoryTests extends KNNTestCase { + private static final String FILTER_FILED_NAME = "foo"; + private static final String FILTER_FILED_VALUE = "fooval"; + private static final QueryBuilder FILTER_QUERY_BUILDER = new TermQueryBuilder(FILTER_FILED_NAME, FILTER_FILED_VALUE); + private static final Query FILTER_QUERY = new TermQuery(new Term(FILTER_FILED_NAME, FILTER_FILED_VALUE)); private final int testQueryDimension = 17; private final float[] testQueryVector = new float[testQueryDimension]; private final String testIndexName = "test-index"; @@ -59,7 +66,6 @@ public void testCreateLuceneQueryWithFilter() { QueryShardContext mockQueryShardContext = mock(QueryShardContext.class); MappedFieldType testMapper = mock(MappedFieldType.class); when(mockQueryShardContext.fieldMapper(any())).thenReturn(testMapper); - QueryBuilder filter = new TermQueryBuilder("foo", "fooval"); final KNNQueryFactory.CreateQueryRequest createQueryRequest = KNNQueryFactory.CreateQueryRequest.builder() .knnEngine(knnEngine) .indexName(testIndexName) @@ -67,10 +73,35 @@ public void testCreateLuceneQueryWithFilter() { .vector(testQueryVector) .k(testK) .context(mockQueryShardContext) - .filter(filter) + .filter(FILTER_QUERY_BUILDER) .build(); Query query = KNNQueryFactory.create(createQueryRequest); assertTrue(query.getClass().isAssignableFrom(KnnFloatVectorQuery.class)); } } + + public void testCreateFaissQueryWithFilter_withValidValues_thenSuccess() { + final KNNEngine knnEngine = KNNEngine.FAISS; + final QueryShardContext mockQueryShardContext = mock(QueryShardContext.class); + MappedFieldType testMapper = mock(MappedFieldType.class); + when(mockQueryShardContext.fieldMapper(any())).thenReturn(testMapper); + when(testMapper.termQuery(Mockito.any(), Mockito.eq(mockQueryShardContext))).thenReturn(FILTER_QUERY); + final KNNQueryFactory.CreateQueryRequest createQueryRequest = KNNQueryFactory.CreateQueryRequest.builder() + .knnEngine(knnEngine) + .indexName(testIndexName) + .fieldName(testFieldName) + .vector(testQueryVector) + .k(testK) + .context(mockQueryShardContext) + .filter(FILTER_QUERY_BUILDER) + .build(); + final Query query = KNNQueryFactory.create(createQueryRequest); + assertTrue(query instanceof KNNQuery); + + assertEquals(testIndexName, ((KNNQuery) query).getIndexName()); + assertEquals(testFieldName, ((KNNQuery) query).getField()); + assertEquals(testQueryVector, ((KNNQuery) query).getQueryVector()); + assertEquals(testK, ((KNNQuery) query).getK()); + assertEquals(FILTER_QUERY, ((KNNQuery) query).getFilterQuery()); + } } diff --git a/src/test/java/org/opensearch/knn/index/query/KNNWeightTests.java b/src/test/java/org/opensearch/knn/index/query/KNNWeightTests.java index 444f763a6..53d0330f0 100644 --- a/src/test/java/org/opensearch/knn/index/query/KNNWeightTests.java +++ b/src/test/java/org/opensearch/knn/index/query/KNNWeightTests.java @@ -6,17 +6,24 @@ package org.opensearch.knn.index.query; import com.google.common.collect.Comparators; +import com.google.common.collect.ImmutableMap; import lombok.SneakyThrows; +import org.apache.lucene.index.BinaryDocValues; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.FieldInfos; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.SegmentCommitInfo; import org.apache.lucene.index.SegmentInfo; import org.apache.lucene.index.SegmentReader; +import org.apache.lucene.index.Term; import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.search.Query; import org.apache.lucene.search.Scorer; import org.apache.lucene.search.Sort; +import org.apache.lucene.search.TermQuery; +import org.apache.lucene.search.Weight; import org.apache.lucene.store.FSDirectory; +import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.StringHelper; import org.apache.lucene.util.Version; import org.junit.BeforeClass; @@ -28,6 +35,7 @@ import org.opensearch.knn.index.KNNSettings; import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.codec.KNNCodecVersion; +import org.opensearch.knn.index.codec.util.KNNVectorAsArraySerializer; import org.opensearch.knn.index.memory.NativeMemoryAllocation; import org.opensearch.knn.index.memory.NativeMemoryCacheManager; import org.opensearch.knn.index.util.KNNEngine; @@ -70,6 +78,10 @@ public class KNNWeightTests extends KNNTestCase { private static final String CIRCUIT_BREAKER_LIMIT_100KB = "100Kb"; private static final Map DOC_ID_TO_SCORES = Map.of(10, 0.4f, 101, 0.05f, 100, 0.8f, 50, 0.52f); + private static final Map FILTERED_DOC_ID_TO_SCORES = Map.of(101, 0.05f, 100, 0.8f, 50, 0.52f); + private static final Map EXACT_SEARCH_DOC_ID_TO_SCORES = Map.of(0, 0.12048191f); + + private static final Query FILTER_QUERY = new TermQuery(new Term("foo", "fooValue")); private static MockedStatic nativeMemoryCacheManagerMockedStatic; private static MockedStatic jniServiceMockedStatic; @@ -133,7 +145,8 @@ public void testQueryScoreForFaissWithModel() throws IOException { SpaceType spaceType = SpaceType.L2; final Function scoreTranslator = spaceType::scoreTranslation; final String modelId = "modelId"; - jniServiceMockedStatic.when(() -> JNIService.queryIndex(anyLong(), any(), anyInt(), anyString())).thenReturn(getKNNQueryResults()); + jniServiceMockedStatic.when(() -> JNIService.queryIndex(anyLong(), any(), anyInt(), anyString(), any())) + .thenReturn(getKNNQueryResults()); final KNNQuery query = new KNNQuery(FIELD_NAME, QUERY_VECTOR, K, INDEX_NAME); @@ -272,7 +285,8 @@ public void testShardWithoutFiles() { @SneakyThrows public void testEmptyQueryResults() { final KNNQueryResult[] knnQueryResults = new KNNQueryResult[] {}; - jniServiceMockedStatic.when(() -> JNIService.queryIndex(anyLong(), any(), anyInt(), anyString())).thenReturn(knnQueryResults); + jniServiceMockedStatic.when(() -> JNIService.queryIndex(anyLong(), any(), anyInt(), anyString(), any())) + .thenReturn(knnQueryResults); final KNNQuery query = new KNNQuery(FIELD_NAME, QUERY_VECTOR, K, INDEX_NAME); final KNNWeight knnWeight = new KNNWeight(query, 0.0f); @@ -311,12 +325,152 @@ public void testEmptyQueryResults() { assertNull(knnScorer); } + @SneakyThrows + public void testANNWithFilterQuery_whenDoingANN_thenSuccess() { + final int[] filterDocIds = new int[] { 0, 1, 2, 3, 4, 5 }; + jniServiceMockedStatic.when(() -> JNIService.queryIndex(anyLong(), any(), anyInt(), anyString(), eq(filterDocIds))) + .thenReturn(getFilteredKNNQueryResults()); + final LeafReaderContext leafReaderContext = mock(LeafReaderContext.class); + final SegmentReader reader = mock(SegmentReader.class); + when(reader.maxDoc()).thenReturn(K + 1); + when(leafReaderContext.reader()).thenReturn(reader); + + final KNNQuery query = new KNNQuery(FIELD_NAME, QUERY_VECTOR, K, INDEX_NAME, FILTER_QUERY); + final Weight filterQueryWeight = mock(Weight.class); + final Scorer filterScorer = mock(Scorer.class); + when(filterQueryWeight.scorer(leafReaderContext)).thenReturn(filterScorer); + // Just to make sure that we are not hitting the exact search condition + when(filterScorer.iterator()).thenReturn(DocIdSetIterator.all(K + 1)); + + final KNNWeight knnWeight = new KNNWeight(query, 0.0f, filterQueryWeight); + + final FSDirectory directory = mock(FSDirectory.class); + when(reader.directory()).thenReturn(directory); + final SegmentInfo segmentInfo = new SegmentInfo( + directory, + Version.LATEST, + Version.LATEST, + SEGMENT_NAME, + 100, + true, + KNNCodecVersion.current().getDefaultCodecDelegate(), + Map.of(), + new byte[StringHelper.ID_LENGTH], + Map.of(), + Sort.RELEVANCE + ); + segmentInfo.setFiles(SEGMENT_FILES_FAISS); + final SegmentCommitInfo segmentCommitInfo = new SegmentCommitInfo(segmentInfo, 0, 0, 0, 0, 0, new byte[StringHelper.ID_LENGTH]); + when(reader.getSegmentInfo()).thenReturn(segmentCommitInfo); + + final Path path = mock(Path.class); + when(directory.getDirectory()).thenReturn(path); + final FieldInfos fieldInfos = mock(FieldInfos.class); + final FieldInfo fieldInfo = mock(FieldInfo.class); + final Map attributesMap = ImmutableMap.of(KNN_ENGINE, KNNEngine.FAISS.getName()); + + when(reader.getFieldInfos()).thenReturn(fieldInfos); + when(fieldInfos.fieldInfo(any())).thenReturn(fieldInfo); + when(fieldInfo.attributes()).thenReturn(attributesMap); + + final KNNScorer knnScorer = (KNNScorer) knnWeight.scorer(leafReaderContext); + assertNotNull(knnScorer); + + final DocIdSetIterator docIdSetIterator = knnScorer.iterator(); + assertNotNull(docIdSetIterator); + assertEquals(FILTERED_DOC_ID_TO_SCORES.size(), docIdSetIterator.cost()); + jniServiceMockedStatic.verify(() -> JNIService.queryIndex(anyLong(), any(), anyInt(), anyString(), eq(filterDocIds))); + + final List actualDocIds = new ArrayList<>(); + final Map translatedScores = getTranslatedScores(SpaceType.L2::scoreTranslation); + for (int docId = docIdSetIterator.nextDoc(); docId != NO_MORE_DOCS; docId = docIdSetIterator.nextDoc()) { + actualDocIds.add(docId); + assertEquals(translatedScores.get(docId), knnScorer.score(), 0.01f); + } + assertEquals(docIdSetIterator.cost(), actualDocIds.size()); + assertTrue(Comparators.isInOrder(actualDocIds, Comparator.naturalOrder())); + } + + @SneakyThrows + public void testANNWithFilterQuery_whenExactSearch_thenSuccess() { + float[] vector = new float[] { 0.1f, 0.3f }; + int filterDocId = 0; + final LeafReaderContext leafReaderContext = mock(LeafReaderContext.class); + final SegmentReader reader = mock(SegmentReader.class); + when(leafReaderContext.reader()).thenReturn(reader); + + final KNNQuery query = new KNNQuery(FIELD_NAME, QUERY_VECTOR, K, INDEX_NAME, FILTER_QUERY); + final Weight filterQueryWeight = mock(Weight.class); + final Scorer filterScorer = mock(Scorer.class); + when(filterQueryWeight.scorer(leafReaderContext)).thenReturn(filterScorer); + // scorer will return 2 documents + when(filterScorer.iterator()).thenReturn(DocIdSetIterator.all(1)); + when(reader.maxDoc()).thenReturn(1); + + final KNNWeight knnWeight = new KNNWeight(query, 0.0f, filterQueryWeight); + final Map attributesMap = ImmutableMap.of(KNN_ENGINE, KNNEngine.FAISS.getName(), SPACE_TYPE, SpaceType.L2.name()); + final FieldInfos fieldInfos = mock(FieldInfos.class); + final FieldInfo fieldInfo = mock(FieldInfo.class); + final BinaryDocValues binaryDocValues = mock(BinaryDocValues.class); + when(reader.getFieldInfos()).thenReturn(fieldInfos); + when(fieldInfos.fieldInfo(any())).thenReturn(fieldInfo); + when(fieldInfo.attributes()).thenReturn(attributesMap); + when(fieldInfo.getAttribute(SPACE_TYPE)).thenReturn(SpaceType.L2.name()); + when(fieldInfo.getName()).thenReturn(FIELD_NAME); + when(reader.getBinaryDocValues(FIELD_NAME)).thenReturn(binaryDocValues); + when(binaryDocValues.advance(filterDocId)).thenReturn(filterDocId); + BytesRef vectorByteRef = new BytesRef(new KNNVectorAsArraySerializer().floatToByteArray(vector)); + when(binaryDocValues.binaryValue()).thenReturn(vectorByteRef); + + final KNNScorer knnScorer = (KNNScorer) knnWeight.scorer(leafReaderContext); + assertNotNull(knnScorer); + final DocIdSetIterator docIdSetIterator = knnScorer.iterator(); + assertNotNull(docIdSetIterator); + assertEquals(EXACT_SEARCH_DOC_ID_TO_SCORES.size(), docIdSetIterator.cost()); + + final List actualDocIds = new ArrayList<>(); + for (int docId = docIdSetIterator.nextDoc(); docId != NO_MORE_DOCS; docId = docIdSetIterator.nextDoc()) { + actualDocIds.add(docId); + assertEquals(EXACT_SEARCH_DOC_ID_TO_SCORES.get(docId), knnScorer.score(), 0.01f); + } + assertEquals(docIdSetIterator.cost(), actualDocIds.size()); + assertTrue(Comparators.isInOrder(actualDocIds, Comparator.naturalOrder())); + } + + @SneakyThrows + public void testANNWithFilterQuery_whenEmptyFilterIds_thenReturnEarly() { + final LeafReaderContext leafReaderContext = mock(LeafReaderContext.class); + final SegmentReader reader = mock(SegmentReader.class); + when(leafReaderContext.reader()).thenReturn(reader); + + final Weight filterQueryWeight = mock(Weight.class); + final Scorer filterScorer = mock(Scorer.class); + when(filterQueryWeight.scorer(leafReaderContext)).thenReturn(filterScorer); + when(filterScorer.iterator()).thenReturn(DocIdSetIterator.empty()); + + final KNNQuery query = new KNNQuery(FIELD_NAME, QUERY_VECTOR, K, INDEX_NAME, FILTER_QUERY); + final KNNWeight knnWeight = new KNNWeight(query, 0.0f, filterQueryWeight); + + final FieldInfos fieldInfos = mock(FieldInfos.class); + final FieldInfo fieldInfo = mock(FieldInfo.class); + when(reader.getFieldInfos()).thenReturn(fieldInfos); + when(fieldInfos.fieldInfo(any())).thenReturn(fieldInfo); + + final Scorer knnScorer = knnWeight.scorer(leafReaderContext); + assertNotNull(knnScorer); + final DocIdSetIterator docIdSetIterator = knnScorer.iterator(); + assertNotNull(docIdSetIterator); + assertEquals(0, docIdSetIterator.cost()); + assertEquals(0, docIdSetIterator.cost()); + } + private void testQueryScore( final Function scoreTranslator, final Set segmentFiles, final Map fileAttributes ) throws IOException { - jniServiceMockedStatic.when(() -> JNIService.queryIndex(anyLong(), any(), anyInt(), anyString())).thenReturn(getKNNQueryResults()); + jniServiceMockedStatic.when(() -> JNIService.queryIndex(anyLong(), any(), anyInt(), anyString(), any())) + .thenReturn(getKNNQueryResults()); final KNNQuery query = new KNNQuery(FIELD_NAME, QUERY_VECTOR, K, INDEX_NAME); final KNNWeight knnWeight = new KNNWeight(query, 0.0f); @@ -381,4 +535,12 @@ private KNNQueryResult[] getKNNQueryResults() { .collect(Collectors.toList()) .toArray(new KNNQueryResult[0]); } + + private KNNQueryResult[] getFilteredKNNQueryResults() { + return FILTERED_DOC_ID_TO_SCORES.entrySet() + .stream() + .map(entry -> new KNNQueryResult(entry.getKey(), entry.getValue())) + .collect(Collectors.toList()) + .toArray(new KNNQueryResult[0]); + } } diff --git a/src/test/java/org/opensearch/knn/jni/JNIServiceTests.java b/src/test/java/org/opensearch/knn/jni/JNIServiceTests.java index f4971e6fd..6d52e5544 100644 --- a/src/test/java/org/opensearch/knn/jni/JNIServiceTests.java +++ b/src/test/java/org/opensearch/knn/jni/JNIServiceTests.java @@ -583,12 +583,12 @@ public void testLoadIndex_faiss_valid() throws IOException { } public void testQueryIndex_invalidEngine() { - expectThrows(IllegalArgumentException.class, () -> JNIService.queryIndex(0L, new float[] {}, 0, "invalid-engine")); + expectThrows(IllegalArgumentException.class, () -> JNIService.queryIndex(0L, new float[] {}, 0, "invalid" + "-engine", null)); } public void testQueryIndex_nmslib_invalid_badPointer() { - expectThrows(Exception.class, () -> JNIService.queryIndex(0L, new float[] {}, 0, KNNEngine.NMSLIB.getName())); + expectThrows(Exception.class, () -> JNIService.queryIndex(0L, new float[] {}, 0, KNNEngine.NMSLIB.getName(), null)); } public void testQueryIndex_nmslib_invalid_nullQueryVector() throws IOException { @@ -611,7 +611,7 @@ public void testQueryIndex_nmslib_invalid_nullQueryVector() throws IOException { ); assertNotEquals(0, pointer); - expectThrows(Exception.class, () -> JNIService.queryIndex(pointer, null, 10, KNNEngine.NMSLIB.getName())); + expectThrows(Exception.class, () -> JNIService.queryIndex(pointer, null, 10, KNNEngine.NMSLIB.getName(), null)); } public void testQueryIndex_nmslib_valid() throws IOException { @@ -637,7 +637,7 @@ public void testQueryIndex_nmslib_valid() throws IOException { assertNotEquals(0, pointer); for (float[] query : testData.queries) { - KNNQueryResult[] results = JNIService.queryIndex(pointer, query, k, KNNEngine.NMSLIB.getName()); + KNNQueryResult[] results = JNIService.queryIndex(pointer, query, k, KNNEngine.NMSLIB.getName(), null); assertEquals(k, results.length); } } @@ -645,7 +645,7 @@ public void testQueryIndex_nmslib_valid() throws IOException { public void testQueryIndex_faiss_invalid_badPointer() { - expectThrows(Exception.class, () -> JNIService.queryIndex(0L, new float[] {}, 0, FAISS_NAME)); + expectThrows(Exception.class, () -> JNIService.queryIndex(0L, new float[] {}, 0, FAISS_NAME, null)); } public void testQueryIndex_faiss_invalid_nullQueryVector() throws IOException { @@ -664,7 +664,7 @@ public void testQueryIndex_faiss_invalid_nullQueryVector() throws IOException { long pointer = JNIService.loadIndex(tmpFile.toAbsolutePath().toString(), Collections.emptyMap(), FAISS_NAME); assertNotEquals(0, pointer); - expectThrows(Exception.class, () -> JNIService.queryIndex(pointer, null, 10, FAISS_NAME)); + expectThrows(Exception.class, () -> JNIService.queryIndex(pointer, null, 10, FAISS_NAME, null)); } public void testQueryIndex_faiss_valid() throws IOException { @@ -693,9 +693,15 @@ public void testQueryIndex_faiss_valid() throws IOException { assertNotEquals(0, pointer); for (float[] query : testData.queries) { - KNNQueryResult[] results = JNIService.queryIndex(pointer, query, k, FAISS_NAME); + KNNQueryResult[] results = JNIService.queryIndex(pointer, query, k, FAISS_NAME, null); assertEquals(k, results.length); } + + // Filter will result in no ids + for (float[] query : testData.queries) { + KNNQueryResult[] results = JNIService.queryIndex(pointer, query, k, FAISS_NAME, new int[] { 0 }); + assertEquals(0, results.length); + } } } } diff --git a/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java b/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java index 6c07783ec..244667a77 100644 --- a/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java +++ b/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java @@ -1306,4 +1306,20 @@ protected void refreshIndex(final String index) throws IOException { Response response = client().performRequest(request); assertEquals(request.getEndpoint() + ": failed", RestStatus.OK, RestStatus.fromCode(response.getStatusLine().getStatusCode())); } + + protected void addKnnDocWithAttributes(String docId, float[] vector, Map fieldValues) throws IOException { + Request request = new Request("POST", "/" + INDEX_NAME + "/_doc/" + docId + "?refresh=true"); + + final XContentBuilder builder = XContentFactory.jsonBuilder().startObject().field(FIELD_NAME, vector); + for (String fieldName : fieldValues.keySet()) { + builder.field(fieldName, fieldValues.get(fieldName)); + } + builder.endObject(); + request.setJsonEntity(Strings.toString(builder)); + client().performRequest(request); + + request = new Request("POST", "/" + INDEX_NAME + "/_refresh"); + Response response = client().performRequest(request); + assertEquals(request.getEndpoint() + ": failed", RestStatus.OK, RestStatus.fromCode(response.getStatusLine().getStatusCode())); + } } From 6738952e1a0f7a28cfb7c6308fd8738381960055 Mon Sep 17 00:00:00 2001 From: Navneet Verma Date: Fri, 30 Jun 2023 16:44:49 -0700 Subject: [PATCH 112/416] Added Tests for KNNQueryBuilder for Faiss Filtering (#942) Signed-off-by: Navneet Verma --- .../knn/index/query/KNNQueryBuilder.java | 4 ++- .../knn/index/query/KNNQueryBuilderTests.java | 32 +++++++++++++++++++ 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java b/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java index f215335a0..cb02aadd1 100644 --- a/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java +++ b/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java @@ -283,7 +283,9 @@ protected Query doToQuery(QueryShardContext context) { ); } - if (KNNEngine.getEnginesThatCreateCustomSegmentFiles().contains(knnEngine) && filter != null && knnEngine != KNNEngine.FAISS) { + if (KNNEngine.getEnginesThatCreateCustomSegmentFiles().contains(knnEngine) + && filter != null + && !KNNEngine.getEnginesThatSupportsFilters().contains(knnEngine)) { throw new IllegalArgumentException(String.format("Engine [%s] does not support filters", knnEngine)); } diff --git a/src/test/java/org/opensearch/knn/index/query/KNNQueryBuilderTests.java b/src/test/java/org/opensearch/knn/index/query/KNNQueryBuilderTests.java index 7eb089b42..74d99a805 100644 --- a/src/test/java/org/opensearch/knn/index/query/KNNQueryBuilderTests.java +++ b/src/test/java/org/opensearch/knn/index/query/KNNQueryBuilderTests.java @@ -199,6 +199,38 @@ public void testDoToQuery_KnnQueryWithFilter() throws Exception { assertTrue(query.getClass().isAssignableFrom(KnnFloatVectorQuery.class)); } + public void testDoToQuery_WhenknnQueryWithFilterAndFaissEngine_thenSuccess() { + float[] queryVector = { 1.0f, 2.0f, 3.0f, 4.0f }; + KNNQueryBuilder knnQueryBuilder = new KNNQueryBuilder(FIELD_NAME, queryVector, K, TERM_QUERY); + Index dummyIndex = new Index("dummy", "dummy"); + QueryShardContext mockQueryShardContext = mock(QueryShardContext.class); + KNNVectorFieldMapper.KNNVectorFieldType mockKNNVectorField = mock(KNNVectorFieldMapper.KNNVectorFieldType.class); + when(mockQueryShardContext.index()).thenReturn(dummyIndex); + when(mockKNNVectorField.getDimension()).thenReturn(4); + MethodComponentContext methodComponentContext = new MethodComponentContext(METHOD_HNSW, ImmutableMap.of()); + KNNMethodContext knnMethodContext = new KNNMethodContext(KNNEngine.FAISS, SpaceType.L2, methodComponentContext); + when(mockKNNVectorField.getKnnMethodContext()).thenReturn(knnMethodContext); + when(mockQueryShardContext.fieldMapper(anyString())).thenReturn(mockKNNVectorField); + Query query = knnQueryBuilder.doToQuery(mockQueryShardContext); + assertNotNull(query); + assertTrue(query.getClass().isAssignableFrom(KNNQuery.class)); + } + + public void testDoToQuery_whenknnQueryWithFilterAndNmsLibEngine_thenException() { + float[] queryVector = { 1.0f, 2.0f, 3.0f, 4.0f }; + KNNQueryBuilder knnQueryBuilder = new KNNQueryBuilder(FIELD_NAME, queryVector, K, TERM_QUERY); + Index dummyIndex = new Index("dummy", "dummy"); + QueryShardContext mockQueryShardContext = mock(QueryShardContext.class); + KNNVectorFieldMapper.KNNVectorFieldType mockKNNVectorField = mock(KNNVectorFieldMapper.KNNVectorFieldType.class); + when(mockQueryShardContext.index()).thenReturn(dummyIndex); + when(mockKNNVectorField.getDimension()).thenReturn(4); + MethodComponentContext methodComponentContext = new MethodComponentContext(METHOD_HNSW, ImmutableMap.of()); + KNNMethodContext knnMethodContext = new KNNMethodContext(KNNEngine.NMSLIB, SpaceType.L2, methodComponentContext); + when(mockKNNVectorField.getKnnMethodContext()).thenReturn(knnMethodContext); + when(mockQueryShardContext.fieldMapper(anyString())).thenReturn(mockKNNVectorField); + expectThrows(IllegalArgumentException.class, () -> knnQueryBuilder.doToQuery(mockQueryShardContext)); + } + public void testDoToQuery_FromModel() { float[] queryVector = { 1.0f, 2.0f, 3.0f, 4.0f }; KNNQueryBuilder knnQueryBuilder = new KNNQueryBuilder(FIELD_NAME, queryVector, K); From 50a30c6e1e5e715480dcce3bb179e7222897b6bd Mon Sep 17 00:00:00 2001 From: Navneet Verma Date: Mon, 3 Jul 2023 18:49:01 -0700 Subject: [PATCH 113/416] Fixing unit test for Faiss due to faiss upgrade. (#951) Signed-off-by: Navneet Verma --- DEVELOPER_GUIDE.md | 4 ++-- jni/tests/faiss_wrapper_test.cpp | 16 ++++++++-------- jni/tests/test_util.cpp | 6 +++--- jni/tests/test_util.h | 6 +++--- 4 files changed, 16 insertions(+), 16 deletions(-) diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index d8ef9e413..4fe31487a 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -56,11 +56,11 @@ In addition to this, the plugin has been tested with JDK 17, and this JDK versio #### CMake -The plugin requires that cmake >= 3.23.1 is installed in order to build the JNI libraries. +The plugin requires that cmake >= 3.23.3 is installed in order to build the JNI libraries. One easy way to install on mac or linux is to use pip: ```bash -pip install cmake==3.23.1 +pip install cmake==3.23.3 ``` #### Faiss Dependencies diff --git a/jni/tests/faiss_wrapper_test.cpp b/jni/tests/faiss_wrapper_test.cpp index 440061b0e..abe4ecb20 100644 --- a/jni/tests/faiss_wrapper_test.cpp +++ b/jni/tests/faiss_wrapper_test.cpp @@ -23,8 +23,8 @@ using ::testing::Return; TEST(FaissCreateIndexTest, BasicAssertions) { // Define the data - faiss::Index::idx_t numIds = 200; - std::vector ids; + faiss::idx_t numIds = 200; + std::vector ids; std::vector> vectors; int dim = 2; for (int64_t i = 0; i < numIds; ++i) { @@ -70,8 +70,8 @@ TEST(FaissCreateIndexTest, BasicAssertions) { TEST(FaissCreateIndexFromTemplateTest, BasicAssertions) { // Define the data - faiss::Index::idx_t numIds = 100; - std::vector ids; + faiss::idx_t numIds = 100; + std::vector ids; std::vector> vectors; int dim = 2; for (int64_t i = 0; i < numIds; ++i) { @@ -122,8 +122,8 @@ TEST(FaissCreateIndexFromTemplateTest, BasicAssertions) { TEST(FaissLoadIndexTest, BasicAssertions) { // Define the data - faiss::Index::idx_t numIds = 100; - std::vector ids; + faiss::idx_t numIds = 100; + std::vector ids; std::vector vectors; int dim = 2; for (int64_t i = 0; i < numIds; i++) { @@ -174,8 +174,8 @@ TEST(FaissLoadIndexTest, BasicAssertions) { TEST(FaissQueryIndexTest, BasicAssertions) { // Define the index data - faiss::Index::idx_t numIds = 100; - std::vector ids; + faiss::idx_t numIds = 100; + std::vector ids; std::vector vectors; int dim = 16; for (int64_t i = 0; i < numIds; i++) { diff --git a/jni/tests/test_util.cpp b/jni/tests/test_util.cpp index 096f2f415..a75886c51 100644 --- a/jni/tests/test_util.cpp +++ b/jni/tests/test_util.cpp @@ -234,7 +234,7 @@ faiss::Index *test_util::FaissLoadFromSerializedIndex( } faiss::IndexIDMap test_util::FaissAddData(faiss::Index *index, - std::vector ids, + std::vector ids, std::vector dataset) { faiss::IndexIDMap idMap = faiss::IndexIDMap(index); idMap.add_with_ids(ids.size(), dataset.data(), ids.data()); @@ -251,11 +251,11 @@ faiss::Index *test_util::FaissLoadIndex(const std::string &indexPath) { } void test_util::FaissQueryIndex(faiss::Index *index, float *query, int k, - float *distances, faiss::Index::idx_t *ids) { + float *distances, faiss::idx_t *ids) { index->search(1, query, k, distances, ids); } -void test_util::FaissTrainIndex(faiss::Index *index, faiss::Index::idx_t n, +void test_util::FaissTrainIndex(faiss::Index *index, faiss::idx_t n, const float *x) { if (auto *indexIvf = dynamic_cast(index)) { if (indexIvf->quantizer_trains_alone == 2) { diff --git a/jni/tests/test_util.h b/jni/tests/test_util.h index cd47be476..6eac70fcf 100644 --- a/jni/tests/test_util.h +++ b/jni/tests/test_util.h @@ -108,7 +108,7 @@ namespace test_util { faiss::Index* FaissLoadFromSerializedIndex(std::vector* indexSerial); faiss::IndexIDMap FaissAddData(faiss::Index* index, - std::vector ids, + std::vector ids, std::vector dataset); void FaissWriteIndex(faiss::Index* index, const std::string& indexPath); @@ -116,9 +116,9 @@ namespace test_util { faiss::Index* FaissLoadIndex(const std::string& indexPath); void FaissQueryIndex(faiss::Index* index, float* query, int k, float* distances, - faiss::Index::idx_t* ids); + faiss::idx_t* ids); - void FaissTrainIndex(faiss::Index* index, faiss::Index::idx_t n, + void FaissTrainIndex(faiss::Index* index, faiss::idx_t n, const float* x); // ------------------------------------------------------------------------------- From 8e4c728b4f14f5ad355007fee3bd4956b2e56c20 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Mon, 10 Jul 2023 18:40:44 -0700 Subject: [PATCH 114/416] Add index settings to super constructor call for codec service (#962) (#963) Signed-off-by: Martin Gaievski (cherry picked from commit a249f210305eb6b6764ba16a39344fd50fcdbbc3) Co-authored-by: Martin Gaievski --- .../java/org/opensearch/knn/index/codec/KNNCodecService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/opensearch/knn/index/codec/KNNCodecService.java b/src/main/java/org/opensearch/knn/index/codec/KNNCodecService.java index d56e09a3f..9e210fcd9 100644 --- a/src/main/java/org/opensearch/knn/index/codec/KNNCodecService.java +++ b/src/main/java/org/opensearch/knn/index/codec/KNNCodecService.java @@ -18,7 +18,7 @@ public class KNNCodecService extends CodecService { private final MapperService mapperService; public KNNCodecService(CodecServiceConfig codecServiceConfig) { - super(codecServiceConfig.getMapperService(), codecServiceConfig.getLogger()); + super(codecServiceConfig.getMapperService(), codecServiceConfig.getIndexSettings(), codecServiceConfig.getLogger()); mapperService = codecServiceConfig.getMapperService(); } From 19965f59fdf8b4cb08703bb1a0a2909e539eff34 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Tue, 11 Jul 2023 10:09:52 -0700 Subject: [PATCH 115/416] Add 2.9.0 release notes (#961) (#964) Signed-off-by: Junqiu Lei (cherry picked from commit 3e2a4065ccf09ecbe861e2805c41c09e42454f7c) Co-authored-by: Junqiu Lei --- CHANGELOG.md | 3 +-- release-notes/opensearch-knn.release-notes-2.9.0.0.md | 6 ++++++ 2 files changed, 7 insertions(+), 2 deletions(-) create mode 100644 release-notes/opensearch-knn.release-notes-2.9.0.0.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 78c2b5c91..eb2b04796 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,9 +12,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Maintenance ### Refactoring -## [Unreleased 2.x](https://github.com/opensearch-project/k-NN/compare/2.8...2.x) +## [Unreleased 2.x](https://github.com/opensearch-project/k-NN/compare/2.9...2.x) ### Features -* Added efficient filtering support for Faiss Engine ([#936](https://github.com/opensearch-project/k-NN/pull/936)) ### Enhancements ### Bug Fixes ### Infrastructure diff --git a/release-notes/opensearch-knn.release-notes-2.9.0.0.md b/release-notes/opensearch-knn.release-notes-2.9.0.0.md new file mode 100644 index 000000000..10e9cbdda --- /dev/null +++ b/release-notes/opensearch-knn.release-notes-2.9.0.0.md @@ -0,0 +1,6 @@ +## Version 2.9.0.0 Release Notes + +Compatible with OpenSearch 2.9.0 + +### Features +Added support for Efficient Pre-filtering for Faiss Engine ([#936](https://github.com/opensearch-project/k-NN/pull/936)) From 912dc4c546353a09b63783b467214c9357aa0e53 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Tue, 11 Jul 2023 14:17:13 -0700 Subject: [PATCH 116/416] Setting the OMP thread to 1 while doing the query in Faiss to reduce thread creation overhead. (#965) (#966) Signed-off-by: Navneet Verma (cherry picked from commit 3b318cec8a3c3954af2716ef4b801a7ab8097474) Co-authored-by: Navneet Verma --- jni/src/faiss_wrapper.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/jni/src/faiss_wrapper.cpp b/jni/src/faiss_wrapper.cpp index a1bbb9635..3d396610e 100644 --- a/jni/src/faiss_wrapper.cpp +++ b/jni/src/faiss_wrapper.cpp @@ -220,6 +220,10 @@ jobjectArray knn_jni::faiss_wrapper::QueryIndex_WithFilter(knn_jni::JNIUtilInter std::vector dis(kJ); std::vector ids(kJ); float* rawQueryvector = jniUtil->GetFloatArrayElements(env, queryVectorJ, nullptr); + /* + Setting the omp_set_num_threads to 1 to make sure that no new OMP threads are getting created. + */ + omp_set_num_threads(1); // create the filterSearch params if the filterIdsJ is not a null pointer if(filterIdsJ != nullptr) { int *filteredIdsArray = jniUtil->GetIntArrayElements(env, filterIdsJ, nullptr); From 0578f0b3e74c11ec5861d071eeb75686d255c24a Mon Sep 17 00:00:00 2001 From: Naveen Tatikonda Date: Tue, 11 Jul 2023 23:29:44 -0500 Subject: [PATCH 117/416] Add Support for Lucene Byte Sized Vector (#971) (#973) --- .../opensearch-knn.release-notes-2.9.0.0.md | 3 +- .../opensearch/knn/common/KNNConstants.java | 5 + .../knn/index/KNNVectorDVLeafFieldData.java | 6 +- .../knn/index/KNNVectorIndexFieldData.java | 12 +- .../knn/index/KNNVectorScriptDocValues.java | 22 +- .../opensearch/knn/index/VectorDataType.java | 120 ++++ .../org/opensearch/knn/index/VectorField.java | 15 + .../index/mapper/KNNVectorFieldMapper.java | 171 ++++-- .../mapper/KNNVectorFieldMapperUtil.java | 164 ++++++ .../knn/index/mapper/LuceneFieldMapper.java | 65 ++- .../knn/index/query/KNNQueryBuilder.java | 17 +- .../knn/index/query/KNNQueryFactory.java | 68 ++- .../knn/plugin/script/KNNScoringSpace.java | 30 +- .../plugin/script/KNNScoringSpaceUtil.java | 16 +- .../knn/plugin/script/KNNScoringUtil.java | 27 +- .../index/KNNVectorDVLeafFieldDataTests.java | 18 +- .../index/KNNVectorIndexFieldDataTests.java | 2 +- .../index/KNNVectorScriptDocValuesTests.java | 3 +- .../opensearch/knn/index/LuceneEngineIT.java | 111 ++-- .../knn/index/VectorDataTypeIT.java | 545 ++++++++++++++++++ .../knn/index/VectorDataTypeTests.java | 109 ++++ .../knn/index/codec/KNNCodecTestCase.java | 19 +- .../mapper/KNNVectorFieldMapperTests.java | 196 ++++++- .../knn/index/query/KNNQueryBuilderTests.java | 4 + .../knn/index/query/KNNQueryFactoryTests.java | 20 +- .../script/KNNScoringSpaceUtilTests.java | 10 +- .../plugin/script/KNNScoringUtilTests.java | 9 +- 27 files changed, 1598 insertions(+), 189 deletions(-) create mode 100644 src/main/java/org/opensearch/knn/index/VectorDataType.java create mode 100644 src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperUtil.java create mode 100644 src/test/java/org/opensearch/knn/index/VectorDataTypeIT.java create mode 100644 src/test/java/org/opensearch/knn/index/VectorDataTypeTests.java diff --git a/release-notes/opensearch-knn.release-notes-2.9.0.0.md b/release-notes/opensearch-knn.release-notes-2.9.0.0.md index 10e9cbdda..0ea90d037 100644 --- a/release-notes/opensearch-knn.release-notes-2.9.0.0.md +++ b/release-notes/opensearch-knn.release-notes-2.9.0.0.md @@ -3,4 +3,5 @@ Compatible with OpenSearch 2.9.0 ### Features -Added support for Efficient Pre-filtering for Faiss Engine ([#936](https://github.com/opensearch-project/k-NN/pull/936)) +* Added support for Efficient Pre-filtering for Faiss Engine ([#936](https://github.com/opensearch-project/k-NN/pull/936)) +* Add Support for Lucene Byte Sized Vector ([#971](https://github.com/opensearch-project/k-NN/pull/971)) diff --git a/src/main/java/org/opensearch/knn/common/KNNConstants.java b/src/main/java/org/opensearch/knn/common/KNNConstants.java index 87d7a1c21..6d387eec4 100644 --- a/src/main/java/org/opensearch/knn/common/KNNConstants.java +++ b/src/main/java/org/opensearch/knn/common/KNNConstants.java @@ -5,6 +5,8 @@ package org.opensearch.knn.common; +import org.opensearch.knn.index.VectorDataType; + public class KNNConstants { // shared across library constants public static final String DIMENSION = "dimension"; @@ -50,6 +52,9 @@ public class KNNConstants { public static final String MAX_VECTOR_COUNT_PARAMETER = "max_training_vector_count"; public static final String SEARCH_SIZE_PARAMETER = "search_size"; + public static final String VECTOR_DATA_TYPE_FIELD = "data_type"; + public static final VectorDataType DEFAULT_VECTOR_DATA_TYPE_FIELD = VectorDataType.FLOAT; + // Lucene specific constants public static final String LUCENE_NAME = "lucene"; diff --git a/src/main/java/org/opensearch/knn/index/KNNVectorDVLeafFieldData.java b/src/main/java/org/opensearch/knn/index/KNNVectorDVLeafFieldData.java index 5f522e3de..f4caa4f20 100644 --- a/src/main/java/org/opensearch/knn/index/KNNVectorDVLeafFieldData.java +++ b/src/main/java/org/opensearch/knn/index/KNNVectorDVLeafFieldData.java @@ -18,10 +18,12 @@ public class KNNVectorDVLeafFieldData implements LeafFieldData { private final LeafReader reader; private final String fieldName; + private final VectorDataType vectorDataType; - public KNNVectorDVLeafFieldData(LeafReader reader, String fieldName) { + public KNNVectorDVLeafFieldData(LeafReader reader, String fieldName, VectorDataType vectorDataType) { this.reader = reader; this.fieldName = fieldName; + this.vectorDataType = vectorDataType; } @Override @@ -38,7 +40,7 @@ public long ramBytesUsed() { public ScriptDocValues getScriptValues() { try { BinaryDocValues values = DocValues.getBinary(reader, fieldName); - return new KNNVectorScriptDocValues(values, fieldName); + return new KNNVectorScriptDocValues(values, fieldName, vectorDataType); } catch (IOException e) { throw new IllegalStateException("Cannot load doc values for knn vector field: " + fieldName, e); } diff --git a/src/main/java/org/opensearch/knn/index/KNNVectorIndexFieldData.java b/src/main/java/org/opensearch/knn/index/KNNVectorIndexFieldData.java index 367cfae53..deef8bae1 100644 --- a/src/main/java/org/opensearch/knn/index/KNNVectorIndexFieldData.java +++ b/src/main/java/org/opensearch/knn/index/KNNVectorIndexFieldData.java @@ -21,10 +21,12 @@ public class KNNVectorIndexFieldData implements IndexFieldData build(IndexFieldDataCache cache, CircuitBreakerService breakerService) { - return new KNNVectorIndexFieldData(name, valuesSourceType); + return new KNNVectorIndexFieldData(name, valuesSourceType, vectorDataType); } } } diff --git a/src/main/java/org/opensearch/knn/index/KNNVectorScriptDocValues.java b/src/main/java/org/opensearch/knn/index/KNNVectorScriptDocValues.java index 0c8240dd4..9f7d52205 100644 --- a/src/main/java/org/opensearch/knn/index/KNNVectorScriptDocValues.java +++ b/src/main/java/org/opensearch/knn/index/KNNVectorScriptDocValues.java @@ -5,26 +5,22 @@ package org.opensearch.knn.index; +import lombok.Getter; +import lombok.RequiredArgsConstructor; import org.apache.lucene.index.BinaryDocValues; -import org.apache.lucene.util.BytesRef; import org.opensearch.ExceptionsHelper; import org.opensearch.index.fielddata.ScriptDocValues; -import org.opensearch.knn.index.codec.util.KNNVectorSerializer; -import org.opensearch.knn.index.codec.util.KNNVectorSerializerFactory; -import java.io.ByteArrayInputStream; import java.io.IOException; +@RequiredArgsConstructor public final class KNNVectorScriptDocValues extends ScriptDocValues { private final BinaryDocValues binaryDocValues; private final String fieldName; - private boolean docExists; - - public KNNVectorScriptDocValues(BinaryDocValues binaryDocValues, String fieldName) { - this.binaryDocValues = binaryDocValues; - this.fieldName = fieldName; - } + @Getter + private final VectorDataType vectorDataType; + private boolean docExists = false; @Override public void setNextDocId(int docId) throws IOException { @@ -47,11 +43,7 @@ public float[] getValue() { throw new IllegalStateException(errorMessage); } try { - BytesRef value = binaryDocValues.binaryValue(); - ByteArrayInputStream byteStream = new ByteArrayInputStream(value.bytes, value.offset, value.length); - final KNNVectorSerializer vectorSerializer = KNNVectorSerializerFactory.getSerializerByStreamContent(byteStream); - final float[] vector = vectorSerializer.byteToFloatArray(byteStream); - return vector; + return vectorDataType.getVectorFromDocValues(binaryDocValues.binaryValue()); } catch (IOException e) { throw ExceptionsHelper.convertToOpenSearchException(e); } diff --git a/src/main/java/org/opensearch/knn/index/VectorDataType.java b/src/main/java/org/opensearch/knn/index/VectorDataType.java new file mode 100644 index 000000000..23b374e9d --- /dev/null +++ b/src/main/java/org/opensearch/knn/index/VectorDataType.java @@ -0,0 +1,120 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index; + +import lombok.AllArgsConstructor; +import lombok.Getter; +import org.apache.lucene.document.FieldType; +import org.apache.lucene.document.KnnByteVectorField; +import org.apache.lucene.document.KnnVectorField; +import org.apache.lucene.index.VectorSimilarityFunction; +import org.apache.lucene.util.BytesRef; +import org.opensearch.knn.index.codec.util.KNNVectorSerializer; +import org.opensearch.knn.index.codec.util.KNNVectorSerializerFactory; + +import java.io.ByteArrayInputStream; +import java.util.Arrays; +import java.util.Locale; +import java.util.Objects; +import java.util.stream.Collectors; + +import static org.opensearch.knn.common.KNNConstants.VECTOR_DATA_TYPE_FIELD; + +/** + * Enum contains data_type of vectors and right now only supported for lucene engine in k-NN plugin. + * We have two vector data_types, one is float (default) and the other one is byte. + */ +@AllArgsConstructor +public enum VectorDataType { + BYTE("byte") { + + @Override + public FieldType createKnnVectorFieldType(int dimension, VectorSimilarityFunction vectorSimilarityFunction) { + return KnnByteVectorField.createFieldType(dimension, vectorSimilarityFunction); + } + + @Override + public float[] getVectorFromDocValues(BytesRef binaryValue) { + float[] vector = new float[binaryValue.length]; + int i = 0; + int j = binaryValue.offset; + + while (i < binaryValue.length) { + vector[i++] = binaryValue.bytes[j++]; + } + return vector; + } + }, + FLOAT("float") { + + @Override + public FieldType createKnnVectorFieldType(int dimension, VectorSimilarityFunction vectorSimilarityFunction) { + return KnnVectorField.createFieldType(dimension, vectorSimilarityFunction); + } + + @Override + public float[] getVectorFromDocValues(BytesRef binaryValue) { + ByteArrayInputStream byteStream = new ByteArrayInputStream(binaryValue.bytes, binaryValue.offset, binaryValue.length); + final KNNVectorSerializer vectorSerializer = KNNVectorSerializerFactory.getSerializerByStreamContent(byteStream); + return vectorSerializer.byteToFloatArray(byteStream); + } + + }; + + public static final String SUPPORTED_VECTOR_DATA_TYPES = Arrays.stream(VectorDataType.values()) + .map(VectorDataType::getValue) + .collect(Collectors.joining(",")); + @Getter + private final String value; + + /** + * Creates a KnnVectorFieldType based on the VectorDataType using the provided dimension and + * VectorSimilarityFunction. + * + * @param dimension Dimension of the vector + * @param vectorSimilarityFunction VectorSimilarityFunction for a given spaceType + * @return FieldType + */ + public abstract FieldType createKnnVectorFieldType(int dimension, VectorSimilarityFunction vectorSimilarityFunction); + + /** + * Deserializes float vector from doc values binary value. + * + * @param binaryValue Binary Value of DocValues + * @return float vector deserialized from binary value + */ + public abstract float[] getVectorFromDocValues(BytesRef binaryValue); + + /** + * Validates if given VectorDataType is in the list of supported data types. + * @param vectorDataType VectorDataType + * @return the same VectorDataType if it is in the supported values + * throws Exception if an invalid value is provided. + */ + public static VectorDataType get(String vectorDataType) { + Objects.requireNonNull( + vectorDataType, + String.format( + Locale.ROOT, + "[%s] should not be null. Supported types are [%s]", + VECTOR_DATA_TYPE_FIELD, + SUPPORTED_VECTOR_DATA_TYPES + ) + ); + try { + return VectorDataType.valueOf(vectorDataType.toUpperCase(Locale.ROOT)); + } catch (Exception e) { + throw new IllegalArgumentException( + String.format( + Locale.ROOT, + "Invalid value provided for [%s] field. Supported values are [%s]", + VECTOR_DATA_TYPE_FIELD, + SUPPORTED_VECTOR_DATA_TYPES + ) + ); + } + } +} diff --git a/src/main/java/org/opensearch/knn/index/VectorField.java b/src/main/java/org/opensearch/knn/index/VectorField.java index c88630f6c..f28ef6238 100644 --- a/src/main/java/org/opensearch/knn/index/VectorField.java +++ b/src/main/java/org/opensearch/knn/index/VectorField.java @@ -23,4 +23,19 @@ public VectorField(String name, float[] value, IndexableFieldType type) { throw new RuntimeException(e); } } + + /** + * @param name FieldType name + * @param value an array of byte vector values + * @param type FieldType to build DocValues + */ + public VectorField(String name, byte[] value, IndexableFieldType type) { + super(name, new BytesRef(), type); + try { + this.setBytesValue(value); + } catch (Exception e) { + throw new RuntimeException(e); + } + + } } diff --git a/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapper.java b/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapper.java index ab45c384f..346d4c238 100644 --- a/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapper.java +++ b/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapper.java @@ -11,7 +11,6 @@ import org.opensearch.knn.common.KNNConstants; import org.apache.lucene.document.FieldType; -import org.apache.lucene.document.StoredField; import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.IndexOptions; import org.apache.lucene.search.DocValuesFieldExistsQuery; @@ -35,6 +34,7 @@ import org.opensearch.knn.index.KNNMethodContext; import org.opensearch.knn.index.KNNSettings; import org.opensearch.knn.index.KNNVectorIndexFieldData; +import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.VectorField; import org.opensearch.knn.index.util.KNNEngine; import org.opensearch.knn.indices.ModelDao; @@ -45,11 +45,21 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Optional; import java.util.function.Supplier; +import static org.opensearch.knn.common.KNNConstants.DEFAULT_VECTOR_DATA_TYPE_FIELD; import static org.opensearch.knn.common.KNNConstants.KNN_METHOD; +import static org.opensearch.knn.common.KNNConstants.VECTOR_DATA_TYPE_FIELD; +import static org.opensearch.knn.index.KNNSettings.KNN_INDEX; +import static org.opensearch.knn.index.mapper.KNNVectorFieldMapperUtil.validateVectorDataTypeWithEngine; +import static org.opensearch.knn.index.mapper.KNNVectorFieldMapperUtil.validateVectorDataTypeWithKnnIndexSetting; +import static org.opensearch.knn.index.mapper.KNNVectorFieldMapperUtil.addStoredFieldForVectorField; +import static org.opensearch.knn.index.mapper.KNNVectorFieldMapperUtil.validateByteVectorValue; +import static org.opensearch.knn.index.mapper.KNNVectorFieldMapperUtil.validateFloatVectorValue; +import static org.opensearch.knn.index.mapper.KNNVectorFieldMapperUtil.validateVectorDimension; /** * Field Mapper for KNN vector type. @@ -96,6 +106,18 @@ public static class Builder extends ParametrizedFieldMapper.Builder { return value; }, m -> toType(m).dimension); + /** + * data_type which defines the datatype of the vector values. This is an optional parameter and + * this is right now only relevant for lucene engine. The default value is float. + */ + private final Parameter vectorDataType = new Parameter<>( + VECTOR_DATA_TYPE_FIELD, + false, + () -> DEFAULT_VECTOR_DATA_TYPE_FIELD, + (n, c, o) -> VectorDataType.get((String) o), + m -> toType(m).vectorDataType + ); + /** * modelId provides a way for a user to generate the underlying library indices from an already serialized * model template index. If this parameter is set, it will take precedence. This parameter is only relevant for @@ -168,7 +190,7 @@ public Builder(String name, String spaceType, String m, String efConstruction) { @Override protected List> getParameters() { - return Arrays.asList(stored, hasDocValues, dimension, meta, knnMethodContext, modelId); + return Arrays.asList(stored, hasDocValues, dimension, vectorDataType, meta, knnMethodContext, modelId); } protected Explicit ignoreMalformed(BuilderContext context) { @@ -203,7 +225,8 @@ public KNNVectorFieldMapper build(BuilderContext context) { buildFullName(context), metaValue, dimension.getValue(), - knnMethodContext + knnMethodContext, + vectorDataType.getValue() ); if (knnMethodContext.getKnnEngine() == KNNEngine.LUCENE) { log.debug(String.format("Use [LuceneFieldMapper] mapper for field [%s]", name)); @@ -216,10 +239,17 @@ public KNNVectorFieldMapper build(BuilderContext context) { .ignoreMalformed(ignoreMalformed) .stored(stored.get()) .hasDocValues(hasDocValues.get()) + .vectorDataType(vectorDataType.getValue()) .knnMethodContext(knnMethodContext) .build(); return new LuceneFieldMapper(createLuceneFieldMapperInput); } + + // Validates and throws exception if data_type field is set in the index mapping + // using any VectorDataType (other than float, which is default) because other + // VectorDataTypes are only supported for lucene engine. + validateVectorDataTypeWithEngine(vectorDataType); + return new MethodFieldMapper( name, mappedFieldType, @@ -265,9 +295,14 @@ public KNNVectorFieldMapper build(BuilderContext context) { this.efConstruction = LegacyFieldMapper.getEfConstruction(context.indexSettings()); } + // Validates and throws exception if index.knn is set to true in the index settings + // using any VectorDataType (other than float, which is default) because we are using NMSLIB engine for LegacyFieldMapper + // and it only supports float VectorDataType + validateVectorDataTypeWithKnnIndexSetting(context.indexSettings().getAsBoolean(KNN_INDEX, false), vectorDataType); + return new LegacyFieldMapper( name, - new KNNVectorFieldType(buildFullName(context), metaValue, dimension.getValue()), + new KNNVectorFieldType(buildFullName(context), metaValue, dimension.getValue(), vectorDataType.getValue()), multiFieldsBuilder, copyToBuilder, ignoreMalformed, @@ -336,20 +371,43 @@ public static class KNNVectorFieldType extends MappedFieldType { int dimension; String modelId; KNNMethodContext knnMethodContext; + VectorDataType vectorDataType; - public KNNVectorFieldType(String name, Map meta, int dimension) { - this(name, meta, dimension, null, null); + public KNNVectorFieldType(String name, Map meta, int dimension, VectorDataType vectorDataType) { + this(name, meta, dimension, null, null, vectorDataType); } public KNNVectorFieldType(String name, Map meta, int dimension, KNNMethodContext knnMethodContext) { - this(name, meta, dimension, knnMethodContext, null); + this(name, meta, dimension, knnMethodContext, null, DEFAULT_VECTOR_DATA_TYPE_FIELD); } public KNNVectorFieldType(String name, Map meta, int dimension, KNNMethodContext knnMethodContext, String modelId) { + this(name, meta, dimension, knnMethodContext, modelId, DEFAULT_VECTOR_DATA_TYPE_FIELD); + } + + public KNNVectorFieldType( + String name, + Map meta, + int dimension, + KNNMethodContext knnMethodContext, + VectorDataType vectorDataType + ) { + this(name, meta, dimension, knnMethodContext, null, vectorDataType); + } + + public KNNVectorFieldType( + String name, + Map meta, + int dimension, + KNNMethodContext knnMethodContext, + String modelId, + VectorDataType vectorDataType + ) { super(name, false, false, true, TextSearchInfo.NONE, meta); this.dimension = dimension; this.modelId = modelId; this.knnMethodContext = knnMethodContext; + this.vectorDataType = vectorDataType; } @Override @@ -378,7 +436,7 @@ public Query termQuery(Object value, QueryShardContext context) { @Override public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier searchLookup) { failIfNoDocValues(); - return new KNNVectorIndexFieldData.Builder(name(), CoreValuesSourceType.BYTES); + return new KNNVectorIndexFieldData.Builder(name(), CoreValuesSourceType.BYTES, this.vectorDataType); } } @@ -386,6 +444,7 @@ public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, S protected boolean stored; protected boolean hasDocValues; protected Integer dimension; + protected VectorDataType vectorDataType; protected ModelDao modelDao; // These members map to parameters in the builder. They need to be declared in the abstract class due to the @@ -408,6 +467,7 @@ public KNNVectorFieldMapper( this.stored = stored; this.hasDocValues = hasDocValues; this.dimension = mappedFieldType.getDimension(); + this.vectorDataType = mappedFieldType.getVectorDataType(); updateEngineStats(); } @@ -430,18 +490,34 @@ protected void parseCreateField(ParseContext context, int dimension) throws IOEx validateIfKNNPluginEnabled(); validateIfCircuitBreakerIsNotTriggered(); - Optional arrayOptional = getFloatsFromContext(context, dimension); + if (VectorDataType.BYTE == vectorDataType) { + Optional bytesArrayOptional = getBytesFromContext(context, dimension); - if (!arrayOptional.isPresent()) { - return; - } - final float[] array = arrayOptional.get(); - VectorField point = new VectorField(name(), array, fieldType); + if (!bytesArrayOptional.isPresent()) { + return; + } + final byte[] array = bytesArrayOptional.get(); + VectorField point = new VectorField(name(), array, fieldType); - context.doc().add(point); - if (fieldType.stored()) { - context.doc().add(new StoredField(name(), point.toString())); + context.doc().add(point); + addStoredFieldForVectorField(context, fieldType, name(), point.toString()); + } else if (VectorDataType.FLOAT == vectorDataType) { + Optional floatsArrayOptional = getFloatsFromContext(context, dimension); + + if (!floatsArrayOptional.isPresent()) { + return; + } + final float[] array = floatsArrayOptional.get(); + VectorField point = new VectorField(name(), array, fieldType); + + context.doc().add(point); + addStoredFieldForVectorField(context, fieldType, name(), point.toString()); + } else { + throw new IllegalArgumentException( + String.format(Locale.ROOT, "Cannot parse context for unsupported values provided for field [%s]", VECTOR_DATA_TYPE_FIELD) + ); } + context.path().remove(); } @@ -459,50 +535,65 @@ void validateIfKNNPluginEnabled() { } } - Optional getFloatsFromContext(ParseContext context, int dimension) throws IOException { + // Returns an optional array of byte values where each value in the vector is parsed as a float and validated + // if it is a finite number without any decimals and within the byte range of [-128 to 127]. + Optional getBytesFromContext(ParseContext context, int dimension) throws IOException { context.path().add(simpleName()); - ArrayList vector = new ArrayList<>(); + ArrayList vector = new ArrayList<>(); XContentParser.Token token = context.parser().currentToken(); float value; + if (token == XContentParser.Token.START_ARRAY) { token = context.parser().nextToken(); while (token != XContentParser.Token.END_ARRAY) { value = context.parser().floatValue(); - - if (Float.isNaN(value)) { - throw new IllegalArgumentException("KNN vector values cannot be NaN"); - } - - if (Float.isInfinite(value)) { - throw new IllegalArgumentException("KNN vector values cannot be infinity"); - } - - vector.add(value); + validateByteVectorValue(value); + vector.add((byte) value); token = context.parser().nextToken(); } } else if (token == XContentParser.Token.VALUE_NUMBER) { value = context.parser().floatValue(); + validateByteVectorValue(value); + vector.add((byte) value); + context.parser().nextToken(); + } else if (token == XContentParser.Token.VALUE_NULL) { + context.path().remove(); + return Optional.empty(); + } + validateVectorDimension(dimension, vector.size()); + byte[] array = new byte[vector.size()]; + int i = 0; + for (Byte f : vector) { + array[i++] = f; + } + return Optional.of(array); + } - if (Float.isNaN(value)) { - throw new IllegalArgumentException("KNN vector values cannot be NaN"); - } + Optional getFloatsFromContext(ParseContext context, int dimension) throws IOException { + context.path().add(simpleName()); - if (Float.isInfinite(value)) { - throw new IllegalArgumentException("KNN vector values cannot be infinity"); + ArrayList vector = new ArrayList<>(); + XContentParser.Token token = context.parser().currentToken(); + float value; + if (token == XContentParser.Token.START_ARRAY) { + token = context.parser().nextToken(); + while (token != XContentParser.Token.END_ARRAY) { + value = context.parser().floatValue(); + validateFloatVectorValue(value); + vector.add(value); + token = context.parser().nextToken(); } - + } else if (token == XContentParser.Token.VALUE_NUMBER) { + value = context.parser().floatValue(); + validateFloatVectorValue(value); vector.add(value); context.parser().nextToken(); } else if (token == XContentParser.Token.VALUE_NULL) { context.path().remove(); return Optional.empty(); } - - if (dimension != vector.size()) { - String errorMessage = String.format("Vector dimension mismatch. Expected: %d, Given: %d", dimension, vector.size()); - throw new IllegalArgumentException(errorMessage); - } + validateVectorDimension(dimension, vector.size()); float[] array = new float[vector.size()]; int i = 0; diff --git a/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperUtil.java b/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperUtil.java new file mode 100644 index 000000000..bf331eeb3 --- /dev/null +++ b/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperUtil.java @@ -0,0 +1,164 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.knn.index.mapper; + +import org.apache.lucene.document.FieldType; +import org.apache.lucene.document.StoredField; +import org.apache.lucene.index.DocValuesType; +import org.opensearch.index.mapper.ParametrizedFieldMapper; +import org.opensearch.index.mapper.ParseContext; +import org.opensearch.knn.index.VectorDataType; +import org.opensearch.knn.index.util.KNNEngine; + +import java.util.Locale; + +import static org.opensearch.knn.common.KNNConstants.KNN_ENGINE; +import static org.opensearch.knn.common.KNNConstants.LUCENE_NAME; +import static org.opensearch.knn.common.KNNConstants.VECTOR_DATA_TYPE_FIELD; + +public class KNNVectorFieldMapperUtil { + /** + * Validate the float vector value and throw exception if it is not a number or not in the finite range. + * + * @param value float vector value + */ + public static void validateFloatVectorValue(float value) { + if (Float.isNaN(value)) { + throw new IllegalArgumentException("KNN vector values cannot be NaN"); + } + + if (Float.isInfinite(value)) { + throw new IllegalArgumentException("KNN vector values cannot be infinity"); + } + } + + /** + * Validate the float vector value in the byte range if it is a finite number, + * with no decimal values and in the byte range of [-128 to 127]. If not throw IllegalArgumentException. + * + * @param value float value in byte range + */ + public static void validateByteVectorValue(float value) { + validateFloatVectorValue(value); + if (value % 1 != 0) { + throw new IllegalArgumentException( + String.format( + Locale.ROOT, + "[%s] field was set as [%s] in index mapping. But, KNN vector values are floats instead of byte integers", + VECTOR_DATA_TYPE_FIELD, + VectorDataType.BYTE.getValue() + ) + + ); + } + if ((int) value < Byte.MIN_VALUE || (int) value > Byte.MAX_VALUE) { + throw new IllegalArgumentException( + String.format( + Locale.ROOT, + "[%s] field was set as [%s] in index mapping. But, KNN vector values are not within in the byte range [%d, %d]", + VECTOR_DATA_TYPE_FIELD, + VectorDataType.BYTE.getValue(), + Byte.MIN_VALUE, + Byte.MAX_VALUE + ) + ); + } + } + + /** + * Validate if the given vector size matches with the dimension provided in mapping. + * + * @param dimension dimension of vector + * @param vectorSize size of the vector + */ + public static void validateVectorDimension(int dimension, int vectorSize) { + if (dimension != vectorSize) { + String errorMessage = String.format(Locale.ROOT, "Vector dimension mismatch. Expected: %d, Given: %d", dimension, vectorSize); + throw new IllegalArgumentException(errorMessage); + } + + } + + /** + * Validates and throws exception if data_type field is set in the index mapping + * using any VectorDataType (other than float, which is default) because other + * VectorDataTypes are only supported for lucene engine. + * + * @param vectorDataType VectorDataType Parameter + */ + public static void validateVectorDataTypeWithEngine(ParametrizedFieldMapper.Parameter vectorDataType) { + if (VectorDataType.FLOAT == vectorDataType.getValue()) { + return; + } + throw new IllegalArgumentException( + String.format( + Locale.ROOT, + "[%s] field with value [%s] is only supported for [%s] engine", + VECTOR_DATA_TYPE_FIELD, + vectorDataType.getValue().getValue(), + LUCENE_NAME + ) + ); + } + + /** + * Validates and throws exception if index.knn is set to true in the index settings + * using any VectorDataType (other than float, which is default) because we are using NMSLIB engine + * for LegacyFieldMapper, and it only supports float VectorDataType + * + * @param knnIndexSetting index.knn setting in the index settings + * @param vectorDataType VectorDataType Parameter + */ + public static void validateVectorDataTypeWithKnnIndexSetting( + boolean knnIndexSetting, + ParametrizedFieldMapper.Parameter vectorDataType + ) { + + if (VectorDataType.FLOAT == vectorDataType.getValue()) { + return; + } + if (knnIndexSetting) { + throw new IllegalArgumentException( + String.format( + Locale.ROOT, + "[%s] field with value [%s] is only supported for [%s] engine", + VECTOR_DATA_TYPE_FIELD, + vectorDataType.getValue().getValue(), + LUCENE_NAME + ) + ); + } + } + + /** + * @param knnEngine KNNEngine + * @return DocValues FieldType of type Binary + */ + public static FieldType buildDocValuesFieldType(KNNEngine knnEngine) { + FieldType field = new FieldType(); + field.putAttribute(KNN_ENGINE, knnEngine.getName()); + field.setDocValuesType(DocValuesType.BINARY); + field.freeze(); + return field; + } + + public static void addStoredFieldForVectorField( + ParseContext context, + FieldType fieldType, + String mapperName, + String vectorFieldAsString + ) { + if (fieldType.stored()) { + context.doc().add(new StoredField(mapperName, vectorFieldAsString)); + } + } +} diff --git a/src/main/java/org/opensearch/knn/index/mapper/LuceneFieldMapper.java b/src/main/java/org/opensearch/knn/index/mapper/LuceneFieldMapper.java index 5dcb09318..94e42ee7c 100644 --- a/src/main/java/org/opensearch/knn/index/mapper/LuceneFieldMapper.java +++ b/src/main/java/org/opensearch/knn/index/mapper/LuceneFieldMapper.java @@ -9,21 +9,24 @@ import lombok.Getter; import lombok.NonNull; import org.apache.lucene.document.FieldType; +import org.apache.lucene.document.KnnByteVectorField; import org.apache.lucene.document.KnnVectorField; -import org.apache.lucene.document.StoredField; -import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.VectorSimilarityFunction; import org.opensearch.common.Explicit; import org.opensearch.index.mapper.ParseContext; import org.opensearch.knn.index.KNNMethodContext; +import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.VectorField; import org.opensearch.knn.index.util.KNNEngine; import java.io.IOException; +import java.util.Locale; import java.util.Optional; import static org.apache.lucene.index.VectorValues.MAX_DIMENSIONS; -import static org.opensearch.knn.common.KNNConstants.KNN_ENGINE; +import static org.opensearch.knn.common.KNNConstants.VECTOR_DATA_TYPE_FIELD; +import static org.opensearch.knn.index.mapper.KNNVectorFieldMapperUtil.addStoredFieldForVectorField; +import static org.opensearch.knn.index.mapper.KNNVectorFieldMapperUtil.buildDocValuesFieldType; /** * Field mapper for case when Lucene has been set as an engine. @@ -34,6 +37,7 @@ public class LuceneFieldMapper extends KNNVectorFieldMapper { /** FieldType used for initializing VectorField, which is used for creating binary doc values. **/ private final FieldType vectorFieldType; + private final VectorDataType vectorDataType; LuceneFieldMapper(final CreateLuceneFieldMapperInput input) { super( @@ -46,6 +50,7 @@ public class LuceneFieldMapper extends KNNVectorFieldMapper { input.isHasDocValues() ); + vectorDataType = input.getVectorDataType(); this.knnMethod = input.getKnnMethodContext(); final VectorSimilarityFunction vectorSimilarityFunction = this.knnMethod.getSpaceType().getVectorSimilarityFunction(); @@ -53,6 +58,7 @@ public class LuceneFieldMapper extends KNNVectorFieldMapper { if (dimension > LUCENE_MAX_DIMENSION) { throw new IllegalArgumentException( String.format( + Locale.ROOT, "Dimension value cannot be greater than [%s] but got [%s] for vector [%s]", LUCENE_MAX_DIMENSION, dimension, @@ -61,7 +67,7 @@ public class LuceneFieldMapper extends KNNVectorFieldMapper { ); } - this.fieldType = KnnVectorField.createFieldType(dimension, vectorSimilarityFunction); + this.fieldType = vectorDataType.createKnnVectorFieldType(dimension, vectorSimilarityFunction); if (this.hasDocValues) { this.vectorFieldType = buildDocValuesFieldType(this.knnMethod.getKnnEngine()); @@ -70,36 +76,46 @@ public class LuceneFieldMapper extends KNNVectorFieldMapper { } } - private static FieldType buildDocValuesFieldType(KNNEngine knnEngine) { - FieldType field = new FieldType(); - field.putAttribute(KNN_ENGINE, knnEngine.getName()); - field.setDocValuesType(DocValuesType.BINARY); - field.freeze(); - return field; - } - @Override protected void parseCreateField(ParseContext context, int dimension) throws IOException { validateIfKNNPluginEnabled(); validateIfCircuitBreakerIsNotTriggered(); - Optional arrayOptional = getFloatsFromContext(context, dimension); + if (VectorDataType.BYTE == vectorDataType) { + Optional bytesArrayOptional = getBytesFromContext(context, dimension); + if (bytesArrayOptional.isEmpty()) { + return; + } + final byte[] array = bytesArrayOptional.get(); + KnnByteVectorField point = new KnnByteVectorField(name(), array, fieldType); - if (arrayOptional.isEmpty()) { - return; - } - final float[] array = arrayOptional.get(); + context.doc().add(point); + addStoredFieldForVectorField(context, fieldType, name(), point.toString()); - KnnVectorField point = new KnnVectorField(name(), array, fieldType); + if (hasDocValues && vectorFieldType != null) { + context.doc().add(new VectorField(name(), array, vectorFieldType)); + } + } else if (VectorDataType.FLOAT == vectorDataType) { + Optional floatsArrayOptional = getFloatsFromContext(context, dimension); - context.doc().add(point); - if (fieldType.stored()) { - context.doc().add(new StoredField(name(), point.toString())); - } + if (floatsArrayOptional.isEmpty()) { + return; + } + final float[] array = floatsArrayOptional.get(); + + KnnVectorField point = new KnnVectorField(name(), array, fieldType); - if (hasDocValues && vectorFieldType != null) { - context.doc().add(new VectorField(name(), array, vectorFieldType)); + context.doc().add(point); + addStoredFieldForVectorField(context, fieldType, name(), point.toString()); + + if (hasDocValues && vectorFieldType != null) { + context.doc().add(new VectorField(name(), array, vectorFieldType)); + } + } else { + throw new IllegalArgumentException( + String.format(Locale.ROOT, "Cannot parse context for unsupported values provided for field [%s]", VECTOR_DATA_TYPE_FIELD) + ); } context.path().remove(); @@ -126,6 +142,7 @@ static class CreateLuceneFieldMapperInput { Explicit ignoreMalformed; boolean stored; boolean hasDocValues; + VectorDataType vectorDataType; @NonNull KNNMethodContext knnMethodContext; } diff --git a/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java b/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java index cb02aadd1..2b1950017 100644 --- a/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java +++ b/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java @@ -12,6 +12,7 @@ import org.opensearch.index.query.QueryBuilder; import org.opensearch.knn.index.KNNClusterUtil; import org.opensearch.knn.index.KNNMethodContext; +import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.mapper.KNNVectorFieldMapper; import org.opensearch.knn.index.util.KNNEngine; import org.opensearch.knn.indices.ModelDao; @@ -32,6 +33,8 @@ import java.util.List; import java.util.Objects; +import static org.opensearch.knn.index.mapper.KNNVectorFieldMapperUtil.validateByteVectorValue; + /** * Helper class to build the KNN query */ @@ -266,6 +269,7 @@ protected Query doToQuery(QueryShardContext context) { int fieldDimension = knnVectorFieldType.getDimension(); KNNMethodContext knnMethodContext = knnVectorFieldType.getKnnMethodContext(); KNNEngine knnEngine = KNNEngine.DEFAULT; + VectorDataType vectorDataType = knnVectorFieldType.getVectorDataType(); if (fieldDimension == -1) { // If dimension is not set, the field uses a model and the information needs to be retrieved from there @@ -283,6 +287,15 @@ protected Query doToQuery(QueryShardContext context) { ); } + byte[] byteVector = new byte[0]; + if (VectorDataType.BYTE == vectorDataType) { + byteVector = new byte[vector.length]; + for (int i = 0; i < vector.length; i++) { + validateByteVectorValue(vector[i]); + byteVector[i] = (byte) vector[i]; + } + } + if (KNNEngine.getEnginesThatCreateCustomSegmentFiles().contains(knnEngine) && filter != null && !KNNEngine.getEnginesThatSupportsFilters().contains(knnEngine)) { @@ -294,7 +307,9 @@ protected Query doToQuery(QueryShardContext context) { .knnEngine(knnEngine) .indexName(indexName) .fieldName(this.fieldName) - .vector(this.vector) + .vector(VectorDataType.FLOAT == vectorDataType ? this.vector : null) + .byteVector(VectorDataType.BYTE == vectorDataType ? byteVector : null) + .vectorDataType(vectorDataType) .k(this.k) .filter(this.filter) .context(context) diff --git a/src/main/java/org/opensearch/knn/index/query/KNNQueryFactory.java b/src/main/java/org/opensearch/knn/index/query/KNNQueryFactory.java index 20c456c4a..65c15499d 100644 --- a/src/main/java/org/opensearch/knn/index/query/KNNQueryFactory.java +++ b/src/main/java/org/opensearch/knn/index/query/KNNQueryFactory.java @@ -11,15 +11,21 @@ import lombok.NonNull; import lombok.Setter; import lombok.extern.log4j.Log4j2; +import org.apache.lucene.search.KnnByteVectorQuery; import org.apache.lucene.search.KnnFloatVectorQuery; import org.apache.lucene.search.Query; import org.opensearch.index.query.QueryBuilder; import org.opensearch.index.query.QueryShardContext; +import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.util.KNNEngine; import java.io.IOException; +import java.util.Locale; import java.util.Optional; +import static org.opensearch.knn.common.KNNConstants.VECTOR_DATA_TYPE_FIELD; +import static org.opensearch.knn.index.VectorDataType.SUPPORTED_VECTOR_DATA_TYPES; + /** * Creates the Lucene k-NN queries */ @@ -36,12 +42,20 @@ public class KNNQueryFactory { * @param k the number of nearest neighbors to return * @return Lucene Query */ - public static Query create(KNNEngine knnEngine, String indexName, String fieldName, float[] vector, int k) { + public static Query create( + KNNEngine knnEngine, + String indexName, + String fieldName, + float[] vector, + int k, + VectorDataType vectorDataType + ) { final CreateQueryRequest createQueryRequest = CreateQueryRequest.builder() .knnEngine(knnEngine) .indexName(indexName) .fieldName(fieldName) .vector(vector) + .vectorDataType(vectorDataType) .k(k) .build(); return create(createQueryRequest); @@ -59,6 +73,8 @@ public static Query create(CreateQueryRequest createQueryRequest) { final String fieldName = createQueryRequest.getFieldName(); final int k = createQueryRequest.getK(); final float[] vector = createQueryRequest.getVector(); + final byte[] byteVector = createQueryRequest.getByteVector(); + final VectorDataType vectorDataType = createQueryRequest.getVectorDataType(); final Query filterQuery = getFilterQuery(createQueryRequest); if (KNNEngine.getEnginesThatCreateCustomSegmentFiles().contains(createQueryRequest.getKnnEngine())) { @@ -77,14 +93,54 @@ public static Query create(CreateQueryRequest createQueryRequest) { return new KNNQuery(fieldName, vector, k, indexName); } + if (VectorDataType.BYTE == vectorDataType) { + return getKnnByteVectorQuery(indexName, fieldName, byteVector, k, filterQuery); + } else if (VectorDataType.FLOAT == vectorDataType) { + return getKnnFloatVectorQuery(indexName, fieldName, vector, k, filterQuery); + } else { + throw new IllegalArgumentException( + String.format( + Locale.ROOT, + "Invalid value provided for [%s] field. Supported values are [%s]", + VECTOR_DATA_TYPE_FIELD, + SUPPORTED_VECTOR_DATA_TYPES + ) + ); + } + } + + private static Query getKnnByteVectorQuery(String indexName, String fieldName, byte[] byteVector, int k, Query filterQuery) { + if (filterQuery != null) { + log.debug( + String.format( + Locale.ROOT, + "Creating Lucene k-NN query with filters for index: %s \"\", field: %s \"\", k: %d", + indexName, + fieldName, + k + ) + ); + return new KnnByteVectorQuery(fieldName, byteVector, k, filterQuery); + } + log.debug(String.format("Creating Lucene k-NN query for index: %s \"\", field: %s \"\", k: %d", indexName, fieldName, k)); + return new KnnByteVectorQuery(fieldName, byteVector, k); + } + + private static Query getKnnFloatVectorQuery(String indexName, String fieldName, float[] floatVector, int k, Query filterQuery) { if (filterQuery != null) { log.debug( - String.format("Creating Lucene k-NN query with filters for index: %s \"\", field: %s \"\", k: %d", indexName, fieldName, k) + String.format( + Locale.ROOT, + "Creating Lucene k-NN query with filters for index: %s \"\", field: %s \"\", k: %d", + indexName, + fieldName, + k + ) ); - return new KnnFloatVectorQuery(fieldName, vector, k, filterQuery); + return new KnnFloatVectorQuery(fieldName, floatVector, k, filterQuery); } log.debug(String.format("Creating Lucene k-NN query for index: %s \"\", field: %s \"\", k: %d", indexName, fieldName, k)); - return new KnnFloatVectorQuery(fieldName, vector, k); + return new KnnFloatVectorQuery(fieldName, floatVector, k); } private static Query getFilterQuery(CreateQueryRequest createQueryRequest) { @@ -126,6 +182,10 @@ static class CreateQueryRequest { @Getter private float[] vector; @Getter + private byte[] byteVector; + @Getter + private VectorDataType vectorDataType; + @Getter private int k; // can be null in cases filter not passed with the knn query private QueryBuilder filter; diff --git a/src/main/java/org/opensearch/knn/plugin/script/KNNScoringSpace.java b/src/main/java/org/opensearch/knn/plugin/script/KNNScoringSpace.java index ca7526dcb..16bf6e204 100644 --- a/src/main/java/org/opensearch/knn/plugin/script/KNNScoringSpace.java +++ b/src/main/java/org/opensearch/knn/plugin/script/KNNScoringSpace.java @@ -54,7 +54,11 @@ public L2(Object query, MappedFieldType fieldType) { throw new IllegalArgumentException("Incompatible field_type for l2 space. The field type must " + "be knn_vector."); } - this.processedQuery = parseToFloatArray(query, ((KNNVectorFieldMapper.KNNVectorFieldType) fieldType).getDimension()); + this.processedQuery = parseToFloatArray( + query, + ((KNNVectorFieldMapper.KNNVectorFieldType) fieldType).getDimension(), + ((KNNVectorFieldMapper.KNNVectorFieldType) fieldType).getVectorDataType() + ); this.scoringMethod = (float[] q, float[] v) -> 1 / (1 + KNNScoringUtil.l2Squared(q, v)); } @@ -81,7 +85,11 @@ public CosineSimilarity(Object query, MappedFieldType fieldType) { throw new IllegalArgumentException("Incompatible field_type for cosine space. The field type must " + "be knn_vector."); } - this.processedQuery = parseToFloatArray(query, ((KNNVectorFieldMapper.KNNVectorFieldType) fieldType).getDimension()); + this.processedQuery = parseToFloatArray( + query, + ((KNNVectorFieldMapper.KNNVectorFieldType) fieldType).getDimension(), + ((KNNVectorFieldMapper.KNNVectorFieldType) fieldType).getVectorDataType() + ); float qVectorSquaredMagnitude = getVectorMagnitudeSquared(this.processedQuery); this.scoringMethod = (float[] q, float[] v) -> 1 + KNNScoringUtil.cosinesimilOptimized(q, v, qVectorSquaredMagnitude); } @@ -159,7 +167,11 @@ public L1(Object query, MappedFieldType fieldType) { throw new IllegalArgumentException("Incompatible field_type for l1 space. The field type must " + "be knn_vector."); } - this.processedQuery = parseToFloatArray(query, ((KNNVectorFieldMapper.KNNVectorFieldType) fieldType).getDimension()); + this.processedQuery = parseToFloatArray( + query, + ((KNNVectorFieldMapper.KNNVectorFieldType) fieldType).getDimension(), + ((KNNVectorFieldMapper.KNNVectorFieldType) fieldType).getVectorDataType() + ); this.scoringMethod = (float[] q, float[] v) -> 1 / (1 + KNNScoringUtil.l1Norm(q, v)); } @@ -185,7 +197,11 @@ public LInf(Object query, MappedFieldType fieldType) { throw new IllegalArgumentException("Incompatible field_type for l-inf space. The field type must " + "be knn_vector."); } - this.processedQuery = parseToFloatArray(query, ((KNNVectorFieldMapper.KNNVectorFieldType) fieldType).getDimension()); + this.processedQuery = parseToFloatArray( + query, + ((KNNVectorFieldMapper.KNNVectorFieldType) fieldType).getDimension(), + ((KNNVectorFieldMapper.KNNVectorFieldType) fieldType).getVectorDataType() + ); this.scoringMethod = (float[] q, float[] v) -> 1 / (1 + KNNScoringUtil.lInfNorm(q, v)); } @@ -213,7 +229,11 @@ public InnerProd(Object query, MappedFieldType fieldType) { ); } - this.processedQuery = parseToFloatArray(query, ((KNNVectorFieldMapper.KNNVectorFieldType) fieldType).getDimension()); + this.processedQuery = parseToFloatArray( + query, + ((KNNVectorFieldMapper.KNNVectorFieldType) fieldType).getDimension(), + ((KNNVectorFieldMapper.KNNVectorFieldType) fieldType).getVectorDataType() + ); this.scoringMethod = (float[] q, float[] v) -> KNNWeight.normalizeScore(-KNNScoringUtil.innerProduct(q, v)); } diff --git a/src/main/java/org/opensearch/knn/plugin/script/KNNScoringSpaceUtil.java b/src/main/java/org/opensearch/knn/plugin/script/KNNScoringSpaceUtil.java index 6f68d16b6..3ec1a9941 100644 --- a/src/main/java/org/opensearch/knn/plugin/script/KNNScoringSpaceUtil.java +++ b/src/main/java/org/opensearch/knn/plugin/script/KNNScoringSpaceUtil.java @@ -5,6 +5,7 @@ package org.opensearch.knn.plugin.script; +import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.mapper.KNNVectorFieldMapper; import org.opensearch.knn.plugin.stats.KNNCounter; import org.opensearch.index.mapper.BinaryFieldMapper; @@ -16,6 +17,7 @@ import java.util.Base64; import static org.opensearch.index.mapper.NumberFieldMapper.NumberType.LONG; +import static org.opensearch.knn.index.mapper.KNNVectorFieldMapperUtil.validateByteVectorValue; public class KNNScoringSpaceUtil { @@ -85,8 +87,8 @@ public static BigInteger parseToBigInteger(Object object) { * @param expectedDimensions int representing the expected dimension of this array. * @return float[] of the object */ - public static float[] parseToFloatArray(Object object, int expectedDimensions) { - float[] floatArray = convertVectorToPrimitive(object); + public static float[] parseToFloatArray(Object object, int expectedDimensions, VectorDataType vectorDataType) { + float[] floatArray = convertVectorToPrimitive(object, vectorDataType); if (expectedDimensions != floatArray.length) { KNNCounter.SCRIPT_QUERY_ERRORS.increment(); throw new IllegalStateException( @@ -103,13 +105,17 @@ public static float[] parseToFloatArray(Object object, int expectedDimensions) { * @return Float array representing the vector */ @SuppressWarnings("unchecked") - public static float[] convertVectorToPrimitive(Object vector) { + public static float[] convertVectorToPrimitive(Object vector, VectorDataType vectorDataType) { float[] primitiveVector = null; if (vector != null) { - final ArrayList tmp = (ArrayList) vector; + final ArrayList tmp = (ArrayList) vector; primitiveVector = new float[tmp.size()]; for (int i = 0; i < primitiveVector.length; i++) { - primitiveVector[i] = tmp.get(i).floatValue(); + float value = tmp.get(i).floatValue(); + if (VectorDataType.BYTE == vectorDataType) { + validateByteVectorValue(value); + } + primitiveVector[i] = value; } } return primitiveVector; diff --git a/src/main/java/org/opensearch/knn/plugin/script/KNNScoringUtil.java b/src/main/java/org/opensearch/knn/plugin/script/KNNScoringUtil.java index 90468c2e7..130c4d8e0 100644 --- a/src/main/java/org/opensearch/knn/plugin/script/KNNScoringUtil.java +++ b/src/main/java/org/opensearch/knn/plugin/script/KNNScoringUtil.java @@ -8,11 +8,14 @@ import org.opensearch.knn.index.KNNVectorScriptDocValues; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.opensearch.knn.index.VectorDataType; import java.math.BigInteger; import java.util.List; import java.util.Objects; +import static org.opensearch.knn.index.mapper.KNNVectorFieldMapperUtil.validateByteVectorValue; + public class KNNScoringUtil { private static Logger logger = LogManager.getLogger(KNNScoringUtil.class); @@ -54,12 +57,16 @@ public static float l2Squared(float[] queryVector, float[] inputVector) { return squaredDistance; } - private static float[] toFloat(List inputVector) { + private static float[] toFloat(List inputVector, VectorDataType vectorDataType) { Objects.requireNonNull(inputVector); float[] value = new float[inputVector.size()]; int index = 0; for (final Number val : inputVector) { - value[index++] = val.floatValue(); + float floatValue = val.floatValue(); + if (VectorDataType.BYTE == vectorDataType) { + validateByteVectorValue(floatValue); + } + value[index++] = floatValue; } return value; } @@ -81,7 +88,7 @@ private static float[] toFloat(List inputVector) { * @return L2 score */ public static float l2Squared(List queryVector, KNNVectorScriptDocValues docValues) { - return l2Squared(toFloat(queryVector), docValues.getValue()); + return l2Squared(toFloat(queryVector, docValues.getVectorDataType()), docValues.getValue()); } /** @@ -127,7 +134,11 @@ public static float cosinesimilOptimized(float[] queryVector, float[] inputVecto * @return cosine score */ public static float cosineSimilarity(List queryVector, KNNVectorScriptDocValues docValues, Number queryVectorMagnitude) { - return cosinesimilOptimized(toFloat(queryVector), docValues.getValue(), queryVectorMagnitude.floatValue()); + return cosinesimilOptimized( + toFloat(queryVector, docValues.getVectorDataType()), + docValues.getValue(), + queryVectorMagnitude.floatValue() + ); } /** @@ -172,7 +183,7 @@ public static float cosinesimil(float[] queryVector, float[] inputVector) { * @return cosine score */ public static float cosineSimilarity(List queryVector, KNNVectorScriptDocValues docValues) { - return cosinesimil(toFloat(queryVector), docValues.getValue()); + return cosinesimil(toFloat(queryVector, docValues.getVectorDataType()), docValues.getValue()); } /** @@ -232,7 +243,7 @@ public static float l1Norm(float[] queryVector, float[] inputVector) { * @return L1 score */ public static float l1Norm(List queryVector, KNNVectorScriptDocValues docValues) { - return l1Norm(toFloat(queryVector), docValues.getValue()); + return l1Norm(toFloat(queryVector, docValues.getVectorDataType()), docValues.getValue()); } /** @@ -270,7 +281,7 @@ public static float lInfNorm(float[] queryVector, float[] inputVector) { * @return L-inf score */ public static float lInfNorm(List queryVector, KNNVectorScriptDocValues docValues) { - return lInfNorm(toFloat(queryVector), docValues.getValue()); + return lInfNorm(toFloat(queryVector, docValues.getVectorDataType()), docValues.getValue()); } /** @@ -307,6 +318,6 @@ public static float innerProduct(float[] queryVector, float[] inputVector) { * @return inner product score */ public static float innerProduct(List queryVector, KNNVectorScriptDocValues docValues) { - return innerProduct(toFloat(queryVector), docValues.getValue()); + return innerProduct(toFloat(queryVector, docValues.getVectorDataType()), docValues.getValue()); } } diff --git a/src/test/java/org/opensearch/knn/index/KNNVectorDVLeafFieldDataTests.java b/src/test/java/org/opensearch/knn/index/KNNVectorDVLeafFieldDataTests.java index 8bda1aefc..cbe11dd6b 100644 --- a/src/test/java/org/opensearch/knn/index/KNNVectorDVLeafFieldDataTests.java +++ b/src/test/java/org/opensearch/knn/index/KNNVectorDVLeafFieldDataTests.java @@ -62,30 +62,38 @@ public void tearDown() throws Exception { } public void testGetScriptValues() { - KNNVectorDVLeafFieldData leafFieldData = new KNNVectorDVLeafFieldData(leafReaderContext.reader(), MOCK_INDEX_FIELD_NAME); + KNNVectorDVLeafFieldData leafFieldData = new KNNVectorDVLeafFieldData( + leafReaderContext.reader(), + MOCK_INDEX_FIELD_NAME, + VectorDataType.FLOAT + ); ScriptDocValues scriptValues = leafFieldData.getScriptValues(); assertNotNull(scriptValues); assertTrue(scriptValues instanceof KNNVectorScriptDocValues); } public void testGetScriptValuesWrongFieldName() { - KNNVectorDVLeafFieldData leafFieldData = new KNNVectorDVLeafFieldData(leafReaderContext.reader(), "invalid"); + KNNVectorDVLeafFieldData leafFieldData = new KNNVectorDVLeafFieldData(leafReaderContext.reader(), "invalid", VectorDataType.FLOAT); ScriptDocValues scriptValues = leafFieldData.getScriptValues(); assertNotNull(scriptValues); } public void testGetScriptValuesWrongFieldType() { - KNNVectorDVLeafFieldData leafFieldData = new KNNVectorDVLeafFieldData(leafReaderContext.reader(), MOCK_NUMERIC_INDEX_FIELD_NAME); + KNNVectorDVLeafFieldData leafFieldData = new KNNVectorDVLeafFieldData( + leafReaderContext.reader(), + MOCK_NUMERIC_INDEX_FIELD_NAME, + VectorDataType.FLOAT + ); expectThrows(IllegalStateException.class, () -> leafFieldData.getScriptValues()); } public void testRamBytesUsed() { - KNNVectorDVLeafFieldData leafFieldData = new KNNVectorDVLeafFieldData(leafReaderContext.reader(), ""); + KNNVectorDVLeafFieldData leafFieldData = new KNNVectorDVLeafFieldData(leafReaderContext.reader(), "", VectorDataType.FLOAT); assertEquals(0, leafFieldData.ramBytesUsed()); } public void testGetBytesValues() { - KNNVectorDVLeafFieldData leafFieldData = new KNNVectorDVLeafFieldData(leafReaderContext.reader(), ""); + KNNVectorDVLeafFieldData leafFieldData = new KNNVectorDVLeafFieldData(leafReaderContext.reader(), "", VectorDataType.FLOAT); expectThrows(UnsupportedOperationException.class, () -> leafFieldData.getBytesValues()); } } diff --git a/src/test/java/org/opensearch/knn/index/KNNVectorIndexFieldDataTests.java b/src/test/java/org/opensearch/knn/index/KNNVectorIndexFieldDataTests.java index 8523c4146..ee57cb190 100644 --- a/src/test/java/org/opensearch/knn/index/KNNVectorIndexFieldDataTests.java +++ b/src/test/java/org/opensearch/knn/index/KNNVectorIndexFieldDataTests.java @@ -27,7 +27,7 @@ public class KNNVectorIndexFieldDataTests extends KNNTestCase { @Before public void setUp() throws Exception { super.setUp(); - indexFieldData = new KNNVectorIndexFieldData(MOCK_INDEX_FIELD_NAME, CoreValuesSourceType.BYTES); + indexFieldData = new KNNVectorIndexFieldData(MOCK_INDEX_FIELD_NAME, CoreValuesSourceType.BYTES, VectorDataType.FLOAT); directory = newDirectory(); createEmptyDocument(directory); } diff --git a/src/test/java/org/opensearch/knn/index/KNNVectorScriptDocValuesTests.java b/src/test/java/org/opensearch/knn/index/KNNVectorScriptDocValuesTests.java index 876117940..a0df3ce64 100644 --- a/src/test/java/org/opensearch/knn/index/KNNVectorScriptDocValuesTests.java +++ b/src/test/java/org/opensearch/knn/index/KNNVectorScriptDocValuesTests.java @@ -37,7 +37,8 @@ public void setUp() throws Exception { LeafReaderContext leafReaderContext = reader.getContext().leaves().get(0); scriptDocValues = new KNNVectorScriptDocValues( leafReaderContext.reader().getBinaryDocValues(MOCK_INDEX_FIELD_NAME), - MOCK_INDEX_FIELD_NAME + MOCK_INDEX_FIELD_NAME, + VectorDataType.FLOAT ); } diff --git a/src/test/java/org/opensearch/knn/index/LuceneEngineIT.java b/src/test/java/org/opensearch/knn/index/LuceneEngineIT.java index b05211b25..0c1f0a451 100644 --- a/src/test/java/org/opensearch/knn/index/LuceneEngineIT.java +++ b/src/test/java/org/opensearch/knn/index/LuceneEngineIT.java @@ -9,6 +9,7 @@ import com.google.common.collect.ImmutableMap; import com.google.common.primitives.Floats; import org.apache.http.util.EntityUtils; +import lombok.SneakyThrows; import org.apache.commons.lang.math.RandomUtils; import org.apache.lucene.index.VectorSimilarityFunction; import org.junit.After; @@ -34,8 +35,10 @@ import java.util.function.Function; import java.util.stream.Collectors; +import static org.opensearch.knn.common.KNNConstants.DIMENSION; import static org.opensearch.knn.common.KNNConstants.METHOD_HNSW; import static org.opensearch.knn.common.KNNConstants.NMSLIB_NAME; +import static org.opensearch.knn.common.KNNConstants.VECTOR_DATA_TYPE_FIELD; public class LuceneEngineIT extends KNNRestTestCase { @@ -110,7 +113,7 @@ public void testQuery_innerProduct_notSupported() throws Exception { public void testQuery_invalidVectorDimensionInQuery() throws Exception { - createKnnIndexMappingWithLuceneEngine(DIMENSION, SpaceType.L2); + createKnnIndexMappingWithLuceneEngine(DIMENSION, SpaceType.L2, VectorDataType.FLOAT); for (int j = 0; j < TEST_INDEX_VECTORS.length; j++) { addKnnDoc(INDEX_NAME, Integer.toString(j + 1), FIELD_NAME, TEST_INDEX_VECTORS[j]); } @@ -127,7 +130,7 @@ public void testQuery_documentsMissingField() throws Exception { SpaceType spaceType = SpaceType.L2; - createKnnIndexMappingWithLuceneEngine(DIMENSION, spaceType); + createKnnIndexMappingWithLuceneEngine(DIMENSION, spaceType, VectorDataType.FLOAT); for (int j = 0; j < TEST_INDEX_VECTORS.length; j++) { addKnnDoc(INDEX_NAME, Integer.toString(j + 1), FIELD_NAME, TEST_INDEX_VECTORS[j]); } @@ -224,35 +227,35 @@ public void testAddDoc() throws IOException { Float[] vector = new Float[] { 2.0f, 4.5f, 6.5f }; addKnnDoc(INDEX_NAME, DOC_ID, FIELD_NAME, vector); - refreshAllIndices(); + refreshIndex(INDEX_NAME); assertEquals(1, getDocCount(INDEX_NAME)); } public void testUpdateDoc() throws Exception { - createKnnIndexMappingWithLuceneEngine(2, SpaceType.L2); + createKnnIndexMappingWithLuceneEngine(2, SpaceType.L2, VectorDataType.FLOAT); Float[] vector = { 6.0f, 6.0f }; addKnnDoc(INDEX_NAME, DOC_ID, FIELD_NAME, vector); Float[] updatedVector = { 8.0f, 8.0f }; updateKnnDoc(INDEX_NAME, DOC_ID, FIELD_NAME, updatedVector); - refreshAllIndices(); + refreshIndex(INDEX_NAME); assertEquals(1, getDocCount(INDEX_NAME)); } public void testDeleteDoc() throws Exception { - createKnnIndexMappingWithLuceneEngine(2, SpaceType.L2); + createKnnIndexMappingWithLuceneEngine(2, SpaceType.L2, VectorDataType.FLOAT); Float[] vector = { 6.0f, 6.0f }; addKnnDoc(INDEX_NAME, DOC_ID, FIELD_NAME, vector); deleteKnnDoc(INDEX_NAME, DOC_ID); - refreshAllIndices(); + refreshIndex(INDEX_NAME); assertEquals(0, getDocCount(INDEX_NAME)); } - public void testQueryWithFilter() throws Exception { - createKnnIndexMappingWithLuceneEngine(DIMENSION, SpaceType.L2); + public void testQueryWithFilterUsingFloatVectorDataType() throws Exception { + createKnnIndexMappingWithLuceneEngine(DIMENSION, SpaceType.L2, VectorDataType.FLOAT); addKnnDocWithAttributes( DOC_ID, @@ -262,39 +265,28 @@ public void testQueryWithFilter() throws Exception { addKnnDocWithAttributes(DOC_ID_2, new float[] { 3.2f, 2.1f, 4.8f }, ImmutableMap.of(COLOR_FIELD_NAME, "green")); addKnnDocWithAttributes(DOC_ID_3, new float[] { 4.1f, 5.0f, 7.1f }, ImmutableMap.of(COLOR_FIELD_NAME, "red")); - refreshAllIndices(); + refreshIndex(INDEX_NAME); final float[] searchVector = { 6.0f, 6.0f, 4.1f }; - int kGreaterThanFilterResult = 5; - List expectedDocIds = Arrays.asList(DOC_ID, DOC_ID_3); - final Response response = searchKNNIndex( - INDEX_NAME, - new KNNQueryBuilder(FIELD_NAME, searchVector, kGreaterThanFilterResult, QueryBuilders.termQuery(COLOR_FIELD_NAME, "red")), - kGreaterThanFilterResult - ); - final String responseBody = EntityUtils.toString(response.getEntity()); - final List knnResults = parseSearchResponse(responseBody, FIELD_NAME); + List expectedDocIdsKGreaterThanFilterResult = Arrays.asList(DOC_ID, DOC_ID_3); + List expectedDocIdsKLimitsFilterResult = Arrays.asList(DOC_ID); + validateQueryResultsWithFilters(searchVector, 5, 1, expectedDocIdsKGreaterThanFilterResult, expectedDocIdsKLimitsFilterResult); + } - assertEquals(expectedDocIds.size(), knnResults.size()); - assertTrue(knnResults.stream().map(KNNResult::getDocId).collect(Collectors.toList()).containsAll(expectedDocIds)); + @SneakyThrows + public void testQueryWithFilterUsingByteVectorDataType() { + createKnnIndexMappingWithLuceneEngine(3, SpaceType.L2, VectorDataType.BYTE); - int kLimitsFilterResult = 1; - List expectedDocIdsKLimitsFilterResult = Arrays.asList(DOC_ID); - final Response responseKLimitsFilterResult = searchKNNIndex( - INDEX_NAME, - new KNNQueryBuilder(FIELD_NAME, searchVector, kLimitsFilterResult, QueryBuilders.termQuery(COLOR_FIELD_NAME, "red")), - kLimitsFilterResult - ); - final String responseBodyKLimitsFilterResult = EntityUtils.toString(responseKLimitsFilterResult.getEntity()); - final List knnResultsKLimitsFilterResult = parseSearchResponse(responseBodyKLimitsFilterResult, FIELD_NAME); + addKnnDocWithAttributes(DOC_ID, new float[] { 6.0f, 7.0f, 3.0f }, ImmutableMap.of(COLOR_FIELD_NAME, "red")); + addKnnDocWithAttributes(DOC_ID_2, new float[] { 3.0f, 2.0f, 4.0f }, ImmutableMap.of(COLOR_FIELD_NAME, "green")); + addKnnDocWithAttributes(DOC_ID_3, new float[] { 4.0f, 5.0f, 7.0f }, ImmutableMap.of(COLOR_FIELD_NAME, "red")); - assertEquals(expectedDocIdsKLimitsFilterResult.size(), knnResultsKLimitsFilterResult.size()); - assertTrue( - knnResultsKLimitsFilterResult.stream() - .map(KNNResult::getDocId) - .collect(Collectors.toList()) - .containsAll(expectedDocIdsKLimitsFilterResult) - ); + refreshIndex(INDEX_NAME); + + final float[] searchVector = { 6.0f, 6.0f, 4.0f }; + List expectedDocIdsKGreaterThanFilterResult = Arrays.asList(DOC_ID, DOC_ID_3); + List expectedDocIdsKLimitsFilterResult = Arrays.asList(DOC_ID); + validateQueryResultsWithFilters(searchVector, 5, 1, expectedDocIdsKGreaterThanFilterResult, expectedDocIdsKLimitsFilterResult); } public void testQuery_filterWithNonLuceneEngine() throws Exception { @@ -337,7 +329,7 @@ public void testQuery_filterWithNonLuceneEngine() throws Exception { } public void testIndexReopening() throws Exception { - createKnnIndexMappingWithLuceneEngine(DIMENSION, SpaceType.L2); + createKnnIndexMappingWithLuceneEngine(DIMENSION, SpaceType.L2, VectorDataType.FLOAT); for (int j = 0; j < TEST_INDEX_VECTORS.length; j++) { addKnnDoc(INDEX_NAME, Integer.toString(j + 1), FIELD_NAME, TEST_INDEX_VECTORS[j]); @@ -358,13 +350,14 @@ public void testIndexReopening() throws Exception { assertArrayEquals(knnResultsBeforeIndexClosure.toArray(), knnResultsAfterIndexClosure.toArray()); } - private void createKnnIndexMappingWithLuceneEngine(int dimension, SpaceType spaceType) throws Exception { + private void createKnnIndexMappingWithLuceneEngine(int dimension, SpaceType spaceType, VectorDataType vectorDataType) throws Exception { XContentBuilder builder = XContentFactory.jsonBuilder() .startObject() .startObject(PROPERTIES_FIELD_NAME) .startObject(FIELD_NAME) .field(TYPE_FIELD_NAME, KNN_VECTOR_TYPE) .field(DIMENSION_FIELD_NAME, dimension) + .field(VECTOR_DATA_TYPE_FIELD, vectorDataType) .startObject(KNNConstants.KNN_METHOD) .field(KNNConstants.NAME, KNNEngine.LUCENE.getMethod(METHOD_HNSW).getMethodComponent().getName()) .field(KNNConstants.METHOD_PARAMETER_SPACE_TYPE, spaceType.getValue()) @@ -384,7 +377,7 @@ private void createKnnIndexMappingWithLuceneEngine(int dimension, SpaceType spac private void baseQueryTest(SpaceType spaceType) throws Exception { - createKnnIndexMappingWithLuceneEngine(DIMENSION, spaceType); + createKnnIndexMappingWithLuceneEngine(DIMENSION, spaceType, VectorDataType.FLOAT); for (int j = 0; j < TEST_INDEX_VECTORS.length; j++) { addKnnDoc(INDEX_NAME, Integer.toString(j + 1), FIELD_NAME, TEST_INDEX_VECTORS[j]); } @@ -419,4 +412,42 @@ private List queryResults(final float[] searchVector, final int k) thro assertNotNull(knnResults); return knnResults.stream().map(KNNResult::getVector).collect(Collectors.toUnmodifiableList()); } + + @SneakyThrows + private void validateQueryResultsWithFilters( + float[] searchVector, + int kGreaterThanFilterResult, + int kLimitsFilterResult, + List expectedDocIdsKGreaterThanFilterResult, + List expectedDocIdsKLimitsFilterResult + ) { + final Response response = searchKNNIndex( + INDEX_NAME, + new KNNQueryBuilder(FIELD_NAME, searchVector, kGreaterThanFilterResult, QueryBuilders.termQuery(COLOR_FIELD_NAME, "red")), + kGreaterThanFilterResult + ); + final String responseBody = EntityUtils.toString(response.getEntity()); + final List knnResults = parseSearchResponse(responseBody, FIELD_NAME); + + assertEquals(expectedDocIdsKGreaterThanFilterResult.size(), knnResults.size()); + assertTrue( + knnResults.stream().map(KNNResult::getDocId).collect(Collectors.toList()).containsAll(expectedDocIdsKGreaterThanFilterResult) + ); + + final Response responseKLimitsFilterResult = searchKNNIndex( + INDEX_NAME, + new KNNQueryBuilder(FIELD_NAME, searchVector, kLimitsFilterResult, QueryBuilders.termQuery(COLOR_FIELD_NAME, "red")), + kLimitsFilterResult + ); + final String responseBodyKLimitsFilterResult = EntityUtils.toString(responseKLimitsFilterResult.getEntity()); + final List knnResultsKLimitsFilterResult = parseSearchResponse(responseBodyKLimitsFilterResult, FIELD_NAME); + + assertEquals(expectedDocIdsKLimitsFilterResult.size(), knnResultsKLimitsFilterResult.size()); + assertTrue( + knnResultsKLimitsFilterResult.stream() + .map(KNNResult::getDocId) + .collect(Collectors.toList()) + .containsAll(expectedDocIdsKLimitsFilterResult) + ); + } } diff --git a/src/test/java/org/opensearch/knn/index/VectorDataTypeIT.java b/src/test/java/org/opensearch/knn/index/VectorDataTypeIT.java new file mode 100644 index 000000000..14ce819f8 --- /dev/null +++ b/src/test/java/org/opensearch/knn/index/VectorDataTypeIT.java @@ -0,0 +1,545 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index; + +import lombok.SneakyThrows; +import org.apache.http.util.EntityUtils; +import org.junit.After; +import org.opensearch.client.Request; +import org.opensearch.client.Response; +import org.opensearch.client.ResponseException; +import org.opensearch.common.Strings; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.index.query.MatchAllQueryBuilder; +import org.opensearch.index.query.QueryBuilder; +import org.opensearch.knn.KNNRestTestCase; +import org.opensearch.knn.KNNResult; +import org.opensearch.knn.common.KNNConstants; +import org.opensearch.knn.index.query.KNNQueryBuilder; +import org.opensearch.knn.index.util.KNNEngine; +import org.opensearch.rest.RestStatus; +import org.opensearch.script.Script; + +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; + +import static org.opensearch.knn.common.KNNConstants.DIMENSION; +import static org.opensearch.knn.common.KNNConstants.LUCENE_NAME; +import static org.opensearch.knn.common.KNNConstants.METHOD_HNSW; +import static org.opensearch.knn.common.KNNConstants.VECTOR_DATA_TYPE_FIELD; +import static org.opensearch.knn.index.VectorDataType.SUPPORTED_VECTOR_DATA_TYPES; + +public class VectorDataTypeIT extends KNNRestTestCase { + private static final String INDEX_NAME = "test-index-vec-dt"; + private static final String FIELD_NAME = "test-field-vec-dt"; + private static final String PROPERTIES_FIELD = "properties"; + private static final String DOC_ID = "doc1"; + private static final String TYPE_FIELD_NAME = "type"; + private static final String KNN_VECTOR_TYPE = "knn_vector"; + private static final int EF_CONSTRUCTION = 128; + private static final int M = 16; + private static final QueryBuilder MATCH_ALL_QUERY_BUILDER = new MatchAllQueryBuilder(); + + @After + @SneakyThrows + public final void cleanUp() { + deleteKNNIndex(INDEX_NAME); + } + + // Validate if we are able to create an index by setting data_type field as byte and add a doc to it + @SneakyThrows + public void testAddDocWithByteVector() { + createKnnIndexMappingWithLuceneEngine(2, SpaceType.L2, VectorDataType.BYTE.getValue()); + Byte[] vector = { 6, 6 }; + addKnnDoc(INDEX_NAME, DOC_ID, FIELD_NAME, vector); + + refreshAllIndices(); + assertEquals(1, getDocCount(INDEX_NAME)); + } + + // Validate by creating an index by setting data_type field as byte, add a doc to it and update it later. + @SneakyThrows + public void testUpdateDocWithByteVector() { + createKnnIndexMappingWithLuceneEngine(2, SpaceType.L2, VectorDataType.BYTE.getValue()); + Byte[] vector = { -36, 78 }; + addKnnDoc(INDEX_NAME, DOC_ID, FIELD_NAME, vector); + + Byte[] updatedVector = { 89, -8 }; + updateKnnDoc(INDEX_NAME, DOC_ID, FIELD_NAME, updatedVector); + + refreshAllIndices(); + assertEquals(1, getDocCount(INDEX_NAME)); + } + + // Validate by creating an index by setting data_type field as byte, add a doc to it and delete it later. + @SneakyThrows + public void testDeleteDocWithByteVector() { + createKnnIndexMappingWithLuceneEngine(2, SpaceType.L2, VectorDataType.BYTE.getValue()); + Byte[] vector = { 35, -46 }; + addKnnDoc(INDEX_NAME, DOC_ID, FIELD_NAME, vector); + + deleteKnnDoc(INDEX_NAME, DOC_ID); + refreshAllIndices(); + + assertEquals(0, getDocCount(INDEX_NAME)); + } + + @SneakyThrows + public void testSearchWithByteVector() { + createKnnIndexMappingWithLuceneEngine(2, SpaceType.L2, VectorDataType.BYTE.getValue()); + ingestL2ByteTestData(); + + Byte[] queryVector = { 1, 1 }; + Response response = searchKNNIndex(INDEX_NAME, new KNNQueryBuilder(FIELD_NAME, convertByteToFloatArray(queryVector), 4), 4); + + validateL2SearchResults(response); + } + + @SneakyThrows + public void testSearchWithInvalidByteVector() { + createKnnIndexMappingWithLuceneEngine(2, SpaceType.L2, VectorDataType.BYTE.getValue()); + ingestL2ByteTestData(); + + // Validate search with floats instead of byte vectors + float[] queryVector = { -10.76f, 15.89f }; + ResponseException ex = expectThrows( + ResponseException.class, + () -> searchKNNIndex(INDEX_NAME, new KNNQueryBuilder(FIELD_NAME, queryVector, 4), 4) + ); + assertTrue( + ex.getMessage() + .contains( + String.format( + Locale.ROOT, + "[%s] field was set as [%s] in index mapping. But, KNN vector values are floats instead of byte integers", + VECTOR_DATA_TYPE_FIELD, + VectorDataType.BYTE.getValue() + ) + ) + ); + + // validate search with search vectors outside of byte range + float[] queryVector1 = { -1000.0f, 200.0f }; + ResponseException ex1 = expectThrows( + ResponseException.class, + () -> searchKNNIndex(INDEX_NAME, new KNNQueryBuilder(FIELD_NAME, queryVector1, 4), 4) + ); + + assertTrue( + ex1.getMessage() + .contains( + String.format( + Locale.ROOT, + "[%s] field was set as [%s] in index mapping. But, KNN vector values are not within in the byte range [%d, %d]", + VECTOR_DATA_TYPE_FIELD, + VectorDataType.BYTE.getValue(), + Byte.MIN_VALUE, + Byte.MAX_VALUE + ) + ) + ); + } + + @SneakyThrows + public void testSearchWithFloatVectorDataType() { + createKnnIndexMappingWithLuceneEngine(2, SpaceType.L2, VectorDataType.FLOAT.getValue()); + ingestL2FloatTestData(); + + float[] queryVector = { 1.0f, 1.0f }; + Response response = searchKNNIndex(INDEX_NAME, new KNNQueryBuilder(FIELD_NAME, queryVector, 4), 4); + + validateL2SearchResults(response); + } + + // Set an invalid value for data_type field while creating the index which should throw an exception + public void testInvalidVectorDataType() { + String vectorDataType = "invalidVectorType"; + ResponseException ex = expectThrows( + ResponseException.class, + () -> createKnnIndexMappingWithLuceneEngine(2, SpaceType.L2, vectorDataType) + ); + assertTrue( + ex.getMessage() + .contains( + String.format( + Locale.ROOT, + "Invalid value provided for [%s] field. Supported values are [%s]", + VECTOR_DATA_TYPE_FIELD, + SUPPORTED_VECTOR_DATA_TYPES + ) + ) + ); + } + + // Set null value for data_type field while creating the index which should throw an exception + public void testVectorDataTypeAsNull() { + ResponseException ex = expectThrows(ResponseException.class, () -> createKnnIndexMappingWithLuceneEngine(2, SpaceType.L2, null)); + assertTrue( + ex.getMessage() + .contains( + String.format( + Locale.ROOT, + "[%s] on mapper [%s] of type [%s] must not have a [null] value", + VECTOR_DATA_TYPE_FIELD, + FIELD_NAME, + KNN_VECTOR_TYPE + ) + ) + ); + } + + // Create an index with byte vector data_type and add a doc with decimal values which should throw exception + @SneakyThrows + public void testInvalidVectorData() { + createKnnIndexMappingWithLuceneEngine(2, SpaceType.L2, VectorDataType.BYTE.getValue()); + Float[] vector = { -10.76f, 15.89f }; + + ResponseException ex = expectThrows(ResponseException.class, () -> addKnnDoc(INDEX_NAME, DOC_ID, FIELD_NAME, vector)); + assertTrue( + ex.getMessage() + .contains( + String.format( + Locale.ROOT, + "[%s] field was set as [%s] in index mapping. But, KNN vector values are floats instead of byte integers", + VECTOR_DATA_TYPE_FIELD, + VectorDataType.BYTE.getValue() + ) + ) + ); + } + + // Create an index with byte vector data_type and add a doc with values out of byte range which should throw exception + @SneakyThrows + public void testInvalidByteVectorRange() { + createKnnIndexMappingWithLuceneEngine(2, SpaceType.L2, VectorDataType.BYTE.getValue()); + Float[] vector = { -1000f, 155f }; + + ResponseException ex = expectThrows(ResponseException.class, () -> addKnnDoc(INDEX_NAME, DOC_ID, FIELD_NAME, vector)); + assertTrue( + ex.getMessage() + .contains( + String.format( + Locale.ROOT, + "[%s] field was set as [%s] in index mapping. But, KNN vector values are not within in the byte range [%d, %d]", + VECTOR_DATA_TYPE_FIELD, + VectorDataType.BYTE.getValue(), + Byte.MIN_VALUE, + Byte.MAX_VALUE + ) + ) + ); + } + + // Create an index with byte vector data_type using nmslib engine which should throw an exception + public void testByteVectorDataTypeWithNmslibEngine() { + ResponseException ex = expectThrows( + ResponseException.class, + () -> createKnnIndexMappingWithNmslibEngine(2, SpaceType.L2, VectorDataType.BYTE.getValue()) + ); + assertTrue( + ex.getMessage() + .contains( + String.format( + Locale.ROOT, + "[%s] field with value [%s] is only supported for [%s] engine", + VECTOR_DATA_TYPE_FIELD, + VectorDataType.BYTE.getValue(), + LUCENE_NAME + ) + ) + ); + } + + @SneakyThrows + public void testByteVectorDataTypeWithLegacyFieldMapperKnnIndexSetting() { + // Create an index with byte vector data_type and index.knn as true without setting KnnMethodContext, + // which should throw an exception because the LegacyFieldMapper will use NMSLIB engine and byte data_type + // is not supported for NMSLIB engine. + XContentBuilder builder = XContentFactory.jsonBuilder() + .startObject() + .startObject(PROPERTIES_FIELD) + .startObject(FIELD_NAME) + .field(TYPE_FIELD_NAME, KNN_VECTOR_TYPE) + .field(DIMENSION, 2) + .field(VECTOR_DATA_TYPE_FIELD, VectorDataType.BYTE.getValue()) + .endObject() + .endObject() + .endObject(); + + String mapping = Strings.toString(builder); + + ResponseException ex = expectThrows(ResponseException.class, () -> createKnnIndex(INDEX_NAME, mapping)); + assertTrue( + ex.getMessage() + .contains( + String.format( + Locale.ROOT, + "[%s] field with value [%s] is only supported for [%s] engine", + VECTOR_DATA_TYPE_FIELD, + VectorDataType.BYTE.getValue(), + LUCENE_NAME + ) + ) + ); + + } + + public void testDocValuesWithByteVectorDataTypeLuceneEngine() throws Exception { + createKnnIndexMappingWithLuceneEngine(2, SpaceType.L2, VectorDataType.BYTE.getValue()); + ingestL2ByteTestData(); + + Byte[] queryVector = { 1, 1 }; + Request request = createScriptQueryRequest(queryVector, SpaceType.L2.getValue(), MATCH_ALL_QUERY_BUILDER); + Response response = client().performRequest(request); + assertEquals(request.getEndpoint() + ": failed", RestStatus.OK, RestStatus.fromCode(response.getStatusLine().getStatusCode())); + + validateL2SearchResults(response); + } + + public void testDocValuesWithFloatVectorDataTypeLuceneEngine() throws Exception { + createKnnIndexMappingWithLuceneEngine(2, SpaceType.L2, VectorDataType.FLOAT.getValue()); + ingestL2FloatTestData(); + + Byte[] queryVector = { 1, 1 }; + Request request = createScriptQueryRequest(queryVector, SpaceType.L2.getValue(), MATCH_ALL_QUERY_BUILDER); + Response response = client().performRequest(request); + assertEquals(request.getEndpoint() + ": failed", RestStatus.OK, RestStatus.fromCode(response.getStatusLine().getStatusCode())); + + validateL2SearchResults(response); + } + + public void testL2ScriptScoreWithByteVectorDataType() throws Exception { + createKnnIndexMappingForScripting(2, VectorDataType.BYTE.getValue()); + ingestL2ByteTestData(); + + Byte[] queryVector = { 1, 1 }; + Request request = createScriptQueryRequest(queryVector, SpaceType.L2.getValue(), MATCH_ALL_QUERY_BUILDER); + Response response = client().performRequest(request); + assertEquals(request.getEndpoint() + ": failed", RestStatus.OK, RestStatus.fromCode(response.getStatusLine().getStatusCode())); + + validateL2SearchResults(response); + } + + public void testL2ScriptScoreWithFloatVectorDataType() throws Exception { + createKnnIndexMappingForScripting(2, VectorDataType.FLOAT.getValue()); + ingestL2FloatTestData(); + + Float[] queryVector = { 1.0f, 1.0f }; + Request request = createScriptQueryRequest(queryVector, SpaceType.L2.getValue(), MATCH_ALL_QUERY_BUILDER); + Response response = client().performRequest(request); + assertEquals(request.getEndpoint() + ": failed", RestStatus.OK, RestStatus.fromCode(response.getStatusLine().getStatusCode())); + + validateL2SearchResults(response); + } + + public void testL2PainlessScriptingWithByteVectorDataType() throws Exception { + createKnnIndexMappingForScripting(2, VectorDataType.BYTE.getValue()); + ingestL2ByteTestData(); + + String source = String.format("1/(1 + l2Squared([1, 1], doc['%s']))", FIELD_NAME); + Request request = constructScriptScoreContextSearchRequest( + INDEX_NAME, + MATCH_ALL_QUERY_BUILDER, + Collections.emptyMap(), + Script.DEFAULT_SCRIPT_LANG, + source, + 4 + ); + + Response response = client().performRequest(request); + assertEquals(request.getEndpoint() + ": failed", RestStatus.OK, RestStatus.fromCode(response.getStatusLine().getStatusCode())); + + validateL2SearchResults(response); + } + + public void testL2PainlessScriptingWithFloatVectorDataType() throws Exception { + createKnnIndexMappingForScripting(2, VectorDataType.FLOAT.getValue()); + ingestL2FloatTestData(); + + String source = String.format("1/(1 + l2Squared([1.0f, 1.0f], doc['%s']))", FIELD_NAME); + Request request = constructScriptScoreContextSearchRequest( + INDEX_NAME, + MATCH_ALL_QUERY_BUILDER, + Collections.emptyMap(), + Script.DEFAULT_SCRIPT_LANG, + source, + 4 + ); + + Response response = client().performRequest(request); + assertEquals(request.getEndpoint() + ": failed", RestStatus.OK, RestStatus.fromCode(response.getStatusLine().getStatusCode())); + + validateL2SearchResults(response); + } + + public void testKNNScriptScoreWithInvalidVectorDataType() { + // Set an invalid value for data_type field while creating the index for script scoring which should throw an exception + ResponseException ex = expectThrows(ResponseException.class, () -> createKnnIndexMappingForScripting(2, "invalid_data_type")); + assertTrue( + ex.getMessage() + .contains( + String.format( + Locale.ROOT, + "Invalid value provided for [%s] field. Supported values are [%s]", + VECTOR_DATA_TYPE_FIELD, + SUPPORTED_VECTOR_DATA_TYPES + ) + ) + ); + } + + public void testKNNScriptScoreWithInvalidByteQueryVector() throws Exception { + // Create an index with byte vector data_type, add docs and run a scoring script query with decimal values + // which should throw exception + createKnnIndexMappingForScripting(2, VectorDataType.BYTE.getValue()); + + Byte[] f1 = { 6, 6 }; + addKnnDoc(INDEX_NAME, "1", FIELD_NAME, f1); + + Byte[] f2 = { 2, 2 }; + addKnnDoc(INDEX_NAME, "2", FIELD_NAME, f2); + + // Construct Search Request with query vector having decimal values + Float[] queryVector = { 10.67f, 19.78f }; + Request request = createScriptQueryRequest(queryVector, SpaceType.L2.getValue(), MATCH_ALL_QUERY_BUILDER); + ResponseException ex = expectThrows(ResponseException.class, () -> client().performRequest(request)); + assertTrue( + ex.getMessage() + .contains( + String.format( + Locale.ROOT, + "[%s] field was set as [%s] in index mapping. But, KNN vector values are floats instead of byte integers", + VECTOR_DATA_TYPE_FIELD, + VectorDataType.BYTE.getValue() + ) + ) + ); + } + + @SneakyThrows + private void ingestL2ByteTestData() { + Byte[] b1 = { 6, 6 }; + addKnnDoc(INDEX_NAME, "1", FIELD_NAME, b1); + + Byte[] b2 = { 2, 2 }; + addKnnDoc(INDEX_NAME, "2", FIELD_NAME, b2); + + Byte[] b3 = { 4, 4 }; + addKnnDoc(INDEX_NAME, "3", FIELD_NAME, b3); + + Byte[] b4 = { 3, 3 }; + addKnnDoc(INDEX_NAME, "4", FIELD_NAME, b4); + } + + @SneakyThrows + private void ingestL2FloatTestData() { + Float[] f1 = { 6.0f, 6.0f }; + addKnnDoc(INDEX_NAME, "1", FIELD_NAME, f1); + + Float[] f2 = { 2.0f, 2.0f }; + addKnnDoc(INDEX_NAME, "2", FIELD_NAME, f2); + + Float[] f3 = { 4.0f, 4.0f }; + addKnnDoc(INDEX_NAME, "3", FIELD_NAME, f3); + + Float[] f4 = { 3.0f, 3.0f }; + addKnnDoc(INDEX_NAME, "4", FIELD_NAME, f4); + } + + private void createKnnIndexMappingWithNmslibEngine(int dimension, SpaceType spaceType, String vectorDataType) throws Exception { + createKnnIndexMappingWithCustomEngine(dimension, spaceType, vectorDataType, KNNEngine.NMSLIB.getName()); + } + + private void createKnnIndexMappingWithLuceneEngine(int dimension, SpaceType spaceType, String vectorDataType) throws Exception { + createKnnIndexMappingWithCustomEngine(dimension, spaceType, vectorDataType, KNNEngine.LUCENE.getName()); + } + + private void createKnnIndexMappingWithCustomEngine(int dimension, SpaceType spaceType, String vectorDataType, String engine) + throws Exception { + XContentBuilder builder = XContentFactory.jsonBuilder() + .startObject() + .startObject(PROPERTIES_FIELD) + .startObject(FIELD_NAME) + .field(TYPE_FIELD_NAME, KNN_VECTOR_TYPE) + .field(DIMENSION, dimension) + .field(VECTOR_DATA_TYPE_FIELD, vectorDataType) + .startObject(KNNConstants.KNN_METHOD) + .field(KNNConstants.NAME, METHOD_HNSW) + .field(KNNConstants.METHOD_PARAMETER_SPACE_TYPE, spaceType.getValue()) + .field(KNNConstants.KNN_ENGINE, engine) + .startObject(KNNConstants.PARAMETERS) + .field(KNNConstants.METHOD_PARAMETER_M, M) + .field(KNNConstants.METHOD_PARAMETER_EF_CONSTRUCTION, EF_CONSTRUCTION) + .endObject() + .endObject() + .endObject() + .endObject() + .endObject(); + + String mapping = Strings.toString(builder); + createKnnIndex(INDEX_NAME, mapping); + } + + private void createKnnIndexMappingForScripting(int dimension, String vectorDataType) throws Exception { + XContentBuilder builder = XContentFactory.jsonBuilder() + .startObject() + .startObject(PROPERTIES_FIELD) + .startObject(FIELD_NAME) + .field(TYPE_FIELD_NAME, KNN_VECTOR_TYPE) + .field(DIMENSION, dimension) + .field(VECTOR_DATA_TYPE_FIELD, vectorDataType) + .endObject() + .endObject() + .endObject(); + + String mapping = Strings.toString(builder); + createKnnIndex(INDEX_NAME, Settings.EMPTY, mapping); + } + + @SneakyThrows + private Request createScriptQueryRequest(Byte[] queryVector, String spaceType, QueryBuilder qb) { + Map params = new HashMap<>(); + params.put("field", FIELD_NAME); + params.put("query_value", queryVector); + params.put("space_type", spaceType); + return constructKNNScriptQueryRequest(INDEX_NAME, qb, params); + } + + @SneakyThrows + private Request createScriptQueryRequest(Float[] queryVector, String spaceType, QueryBuilder qb) { + Map params = new HashMap<>(); + params.put("field", FIELD_NAME); + params.put("query_value", queryVector); + params.put("space_type", spaceType); + return constructKNNScriptQueryRequest(INDEX_NAME, qb, params); + } + + @SneakyThrows + private void validateL2SearchResults(Response response) { + + List results = parseSearchResponse(EntityUtils.toString(response.getEntity()), FIELD_NAME); + + assertEquals(4, results.size()); + + String[] expectedDocIDs = { "2", "4", "3", "1" }; + for (int i = 0; i < results.size(); i++) { + assertEquals(expectedDocIDs[i], results.get(i).getDocId()); + } + } + + private float[] convertByteToFloatArray(Byte[] arr) { + float[] floatArray = new float[arr.length]; + for (int i = 0; i < arr.length; i++) { + floatArray[i] = arr[i]; + } + return floatArray; + } +} diff --git a/src/test/java/org/opensearch/knn/index/VectorDataTypeTests.java b/src/test/java/org/opensearch/knn/index/VectorDataTypeTests.java new file mode 100644 index 000000000..4423c85d8 --- /dev/null +++ b/src/test/java/org/opensearch/knn/index/VectorDataTypeTests.java @@ -0,0 +1,109 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index; + +import lombok.SneakyThrows; +import org.apache.lucene.document.BinaryDocValuesField; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.FieldType; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.store.Directory; +import org.apache.lucene.tests.analysis.MockAnalyzer; +import org.junit.Assert; +import org.opensearch.knn.KNNTestCase; + +import java.io.IOException; + +public class VectorDataTypeTests extends KNNTestCase { + + private static final String MOCK_FLOAT_INDEX_FIELD_NAME = "test-float-index-field-name"; + private static final String MOCK_BYTE_INDEX_FIELD_NAME = "test-byte-index-field-name"; + private static final float[] SAMPLE_FLOAT_VECTOR_DATA = new float[] { 10.0f, 25.0f }; + private static final byte[] SAMPLE_BYTE_VECTOR_DATA = new byte[] { 10, 25 }; + private Directory directory; + private DirectoryReader reader; + + @SneakyThrows + public void testGetDocValuesWithFloatVectorDataType() { + KNNVectorScriptDocValues scriptDocValues = getKNNFloatVectorScriptDocValues(); + + scriptDocValues.setNextDocId(0); + Assert.assertArrayEquals(SAMPLE_FLOAT_VECTOR_DATA, scriptDocValues.getValue(), 0.1f); + + reader.close(); + directory.close(); + } + + @SneakyThrows + public void testGetDocValuesWithByteVectorDataType() { + KNNVectorScriptDocValues scriptDocValues = getKNNByteVectorScriptDocValues(); + + scriptDocValues.setNextDocId(0); + Assert.assertArrayEquals(SAMPLE_FLOAT_VECTOR_DATA, scriptDocValues.getValue(), 0.1f); + + reader.close(); + directory.close(); + } + + @SneakyThrows + private KNNVectorScriptDocValues getKNNFloatVectorScriptDocValues() { + directory = newDirectory(); + createKNNFloatVectorDocument(directory); + reader = DirectoryReader.open(directory); + LeafReaderContext leafReaderContext = reader.getContext().leaves().get(0); + return new KNNVectorScriptDocValues( + leafReaderContext.reader().getBinaryDocValues(VectorDataTypeTests.MOCK_FLOAT_INDEX_FIELD_NAME), + VectorDataTypeTests.MOCK_FLOAT_INDEX_FIELD_NAME, + VectorDataType.FLOAT + ); + } + + @SneakyThrows + private KNNVectorScriptDocValues getKNNByteVectorScriptDocValues() { + directory = newDirectory(); + createKNNByteVectorDocument(directory); + reader = DirectoryReader.open(directory); + LeafReaderContext leafReaderContext = reader.getContext().leaves().get(0); + return new KNNVectorScriptDocValues( + leafReaderContext.reader().getBinaryDocValues(VectorDataTypeTests.MOCK_BYTE_INDEX_FIELD_NAME), + VectorDataTypeTests.MOCK_BYTE_INDEX_FIELD_NAME, + VectorDataType.BYTE + ); + } + + private void createKNNFloatVectorDocument(Directory directory) throws IOException { + IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random())); + IndexWriter writer = new IndexWriter(directory, conf); + Document knnDocument = new Document(); + knnDocument.add( + new BinaryDocValuesField( + MOCK_FLOAT_INDEX_FIELD_NAME, + new VectorField(MOCK_FLOAT_INDEX_FIELD_NAME, SAMPLE_FLOAT_VECTOR_DATA, new FieldType()).binaryValue() + ) + ); + writer.addDocument(knnDocument); + writer.commit(); + writer.close(); + } + + private void createKNNByteVectorDocument(Directory directory) throws IOException { + IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random())); + IndexWriter writer = new IndexWriter(directory, conf); + Document knnDocument = new Document(); + knnDocument.add( + new BinaryDocValuesField( + MOCK_BYTE_INDEX_FIELD_NAME, + new VectorField(MOCK_BYTE_INDEX_FIELD_NAME, SAMPLE_BYTE_VECTOR_DATA, new FieldType()).binaryValue() + ) + ); + writer.addDocument(knnDocument); + writer.commit(); + writer.close(); + } +} diff --git a/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestCase.java b/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestCase.java index 6bfde31bb..6c7631216 100644 --- a/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestCase.java +++ b/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestCase.java @@ -69,6 +69,7 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import static org.opensearch.Version.CURRENT; +import static org.opensearch.knn.common.KNNConstants.DEFAULT_VECTOR_DATA_TYPE_FIELD; import static org.opensearch.knn.common.KNNConstants.HNSW_ALGO_EF_CONSTRUCTION; import static org.opensearch.knn.common.KNNConstants.HNSW_ALGO_M; import static org.opensearch.knn.common.KNNConstants.INDEX_DESCRIPTION_PARAMETER; @@ -338,7 +339,14 @@ public void testKnnVectorIndex( verify(perFieldKnnVectorsFormatSpy, atLeastOnce()).getKnnVectorsFormatForField(eq(FIELD_NAME_ONE)); IndexSearcher searcher = new IndexSearcher(reader); - Query query = KNNQueryFactory.create(KNNEngine.LUCENE, "dummy", FIELD_NAME_ONE, new float[] { 1.0f, 0.0f, 0.0f }, 1); + Query query = KNNQueryFactory.create( + KNNEngine.LUCENE, + "dummy", + FIELD_NAME_ONE, + new float[] { 1.0f, 0.0f, 0.0f }, + 1, + DEFAULT_VECTOR_DATA_TYPE_FIELD + ); assertEquals(1, searcher.count(query)); @@ -365,7 +373,14 @@ public void testKnnVectorIndex( verify(perFieldKnnVectorsFormatSpy, atLeastOnce()).getKnnVectorsFormatForField(eq(FIELD_NAME_TWO)); IndexSearcher searcher1 = new IndexSearcher(reader1); - Query query1 = KNNQueryFactory.create(KNNEngine.LUCENE, "dummy", FIELD_NAME_TWO, new float[] { 1.0f, 0.0f }, 1); + Query query1 = KNNQueryFactory.create( + KNNEngine.LUCENE, + "dummy", + FIELD_NAME_TWO, + new float[] { 1.0f, 0.0f }, + 1, + DEFAULT_VECTOR_DATA_TYPE_FIELD + ); assertEquals(1, searcher1.count(query1)); diff --git a/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java b/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java index d4a5b5aea..1f3598781 100644 --- a/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java +++ b/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java @@ -6,8 +6,11 @@ package org.opensearch.knn.index.mapper; import com.google.common.collect.ImmutableMap; +import lombok.SneakyThrows; +import org.apache.lucene.document.KnnByteVectorField; import org.apache.lucene.document.KnnVectorField; import org.apache.lucene.index.IndexableField; +import org.apache.lucene.index.VectorEncoding; import org.apache.lucene.util.BytesRef; import org.mockito.Mockito; import org.opensearch.common.Explicit; @@ -28,6 +31,7 @@ import org.opensearch.knn.index.KNNSettings; import org.opensearch.knn.index.MethodComponentContext; import org.opensearch.knn.index.SpaceType; +import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.VectorField; import org.opensearch.knn.index.codec.util.KNNVectorSerializerFactory; import org.opensearch.knn.index.util.KNNEngine; @@ -42,10 +46,13 @@ import java.util.Collections; import java.util.HashSet; import java.util.List; +import java.util.Locale; import java.util.Optional; +import java.util.stream.Collectors; import static org.mockito.Mockito.doNothing; import static org.mockito.Mockito.doReturn; +import static org.opensearch.knn.common.KNNConstants.DIMENSION; import static org.opensearch.knn.common.KNNConstants.KNN_ENGINE; import static org.opensearch.knn.common.KNNConstants.KNN_METHOD; import static org.opensearch.knn.common.KNNConstants.LUCENE_NAME; @@ -60,6 +67,7 @@ import static org.opensearch.Version.CURRENT; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; +import static org.opensearch.knn.common.KNNConstants.VECTOR_DATA_TYPE_FIELD; public class KNNVectorFieldMapperTests extends KNNTestCase { @@ -71,9 +79,13 @@ public class KNNVectorFieldMapperTests extends KNNTestCase { private final static float[] TEST_VECTOR = createInitializedFloatArray(TEST_DIMENSION, TEST_VECTOR_VALUE); + private final static byte TEST_BYTE_VECTOR_VALUE = 10; + private final static byte[] TEST_BYTE_VECTOR = createInitializedByteArray(TEST_DIMENSION, TEST_BYTE_VECTOR_VALUE); + private final static BytesRef TEST_VECTOR_BYTES_REF = new BytesRef( KNNVectorSerializerFactory.getDefaultSerializer().floatToByteArray(TEST_VECTOR) ); + private final static BytesRef TEST_BYTE_VECTOR_BYTES_REF = new BytesRef(TEST_BYTE_VECTOR); private static final String DIMENSION_FIELD_NAME = "dimension"; private static final String KNN_VECTOR_TYPE = "knn_vector"; private static final String TYPE_FIELD_NAME = "type"; @@ -82,7 +94,11 @@ public void testBuilder_getParameters() { String fieldName = "test-field-name"; ModelDao modelDao = mock(ModelDao.class); KNNVectorFieldMapper.Builder builder = new KNNVectorFieldMapper.Builder(fieldName, modelDao); - assertEquals(6, builder.getParameters().size()); + + assertEquals(7, builder.getParameters().size()); + List actualParams = builder.getParameters().stream().map(a -> a.name).collect(Collectors.toList()); + List expectedParams = Arrays.asList("store", "doc_values", DIMENSION, VECTOR_DATA_TYPE_FIELD, "meta", KNN_METHOD, MODEL_ID); + assertEquals(expectedParams, actualParams); } public void testBuilder_build_fromKnnMethodContext() { @@ -334,6 +350,56 @@ public void testTypeParser_parse_fromKnnMethodContext_invalidDimension() throws ); } + // Validate TypeParser parsing invalid vector data_type which throws exception + @SneakyThrows + public void testTypeParser_parse_invalidVectorDataType() { + String fieldName = "test-field-name-vec"; + String indexName = "test-index-name-vec"; + String vectorDataType = "invalid"; + String supportedTypes = String.join( + ",", + Arrays.stream((VectorDataType.values())).map(VectorDataType::getValue).collect(Collectors.toCollection(HashSet::new)) + ); + + Settings settings = Settings.builder().put(settings(CURRENT).build()).build(); + + ModelDao modelDao = mock(ModelDao.class); + KNNVectorFieldMapper.TypeParser typeParser = new KNNVectorFieldMapper.TypeParser(() -> modelDao); + + XContentBuilder xContentBuilderOverInvalidVectorType = XContentFactory.jsonBuilder() + .startObject() + .field(TYPE_FIELD_NAME, KNN_VECTOR_TYPE) + .field(DIMENSION, 10) + .field(VECTOR_DATA_TYPE_FIELD, vectorDataType) + .startObject(KNN_METHOD) + .field(NAME, METHOD_HNSW) + .field(METHOD_PARAMETER_SPACE_TYPE, SpaceType.L2) + .field(KNN_ENGINE, LUCENE_NAME) + .startObject(PARAMETERS) + .field(METHOD_PARAMETER_EF_CONSTRUCTION, 128) + .endObject() + .endObject() + .endObject(); + + IllegalArgumentException ex = expectThrows( + IllegalArgumentException.class, + () -> typeParser.parse( + fieldName, + xContentBuilderToMap(xContentBuilderOverInvalidVectorType), + buildParserContext(indexName, settings) + ) + ); + assertEquals( + String.format( + Locale.ROOT, + "Invalid value provided for [%s] field. Supported values are [%s]", + VECTOR_DATA_TYPE_FIELD, + supportedTypes + ), + ex.getMessage() + ); + } + public void testTypeParser_parse_fromKnnMethodContext_invalidSpaceType() throws IOException { String fieldName = "test-field-name"; String indexName = "test-index-name"; @@ -673,30 +739,11 @@ public void testKNNVectorFieldMapper_merge_fromModel() throws IOException { expectThrows(IllegalArgumentException.class, () -> knnVectorFieldMapper1.merge(knnVectorFieldMapper3)); } - public void testLuceneFieldMapper_parseCreateField_docValues() throws IOException { + @SneakyThrows + public void testLuceneFieldMapper_parseCreateField_docValues_withFloats() { // Create a lucene field mapper that creates a binary doc values field as well as KnnVectorField - KNNMethodContext knnMethodContext = new KNNMethodContext( - KNNEngine.LUCENE, - SpaceType.DEFAULT, - new MethodComponentContext(METHOD_HNSW, Collections.emptyMap()) - ); - - KNNVectorFieldMapper.KNNVectorFieldType knnVectorFieldType = new KNNVectorFieldMapper.KNNVectorFieldType( - TEST_FIELD_NAME, - Collections.emptyMap(), - TEST_DIMENSION, - knnMethodContext - ); - LuceneFieldMapper.CreateLuceneFieldMapperInput.CreateLuceneFieldMapperInputBuilder inputBuilder = - LuceneFieldMapper.CreateLuceneFieldMapperInput.builder() - .name(TEST_FIELD_NAME) - .mappedFieldType(knnVectorFieldType) - .multiFields(FieldMapper.MultiFields.empty()) - .copyTo(FieldMapper.CopyTo.empty()) - .hasDocValues(true) - .ignoreMalformed(new Explicit<>(true, true)) - .knnMethodContext(knnMethodContext); + createLuceneFieldMapperInputBuilder(VectorDataType.FLOAT); ParseContext.Document document = new ParseContext.Document(); ContentPath contentPath = new ContentPath(); @@ -731,6 +778,7 @@ public void testLuceneFieldMapper_parseCreateField_docValues() throws IOExceptio } assertEquals(TEST_VECTOR_BYTES_REF, vectorField.binaryValue()); + assertEquals(VectorEncoding.FLOAT32, vectorField.fieldType().vectorEncoding()); assertArrayEquals(TEST_VECTOR, knnVectorField.vectorValue(), 0.001f); // Test when doc values are disabled @@ -757,12 +805,112 @@ public void testLuceneFieldMapper_parseCreateField_docValues() throws IOExceptio assertArrayEquals(TEST_VECTOR, knnVectorField.vectorValue(), 0.001f); } - public static float[] createInitializedFloatArray(int dimension, float value) { + @SneakyThrows + public void testLuceneFieldMapper_parseCreateField_docValues_withBytes() { + // Create a lucene field mapper that creates a binary doc values field as well as KnnByteVectorField + + LuceneFieldMapper.CreateLuceneFieldMapperInput.CreateLuceneFieldMapperInputBuilder inputBuilder = + createLuceneFieldMapperInputBuilder(VectorDataType.BYTE); + + ParseContext.Document document = new ParseContext.Document(); + ContentPath contentPath = new ContentPath(); + ParseContext parseContext = mock(ParseContext.class); + when(parseContext.doc()).thenReturn(document); + when(parseContext.path()).thenReturn(contentPath); + + LuceneFieldMapper luceneFieldMapper = Mockito.spy(new LuceneFieldMapper(inputBuilder.build())); + doReturn(Optional.of(TEST_BYTE_VECTOR)).when(luceneFieldMapper).getBytesFromContext(parseContext, TEST_DIMENSION); + doNothing().when(luceneFieldMapper).validateIfCircuitBreakerIsNotTriggered(); + doNothing().when(luceneFieldMapper).validateIfKNNPluginEnabled(); + + luceneFieldMapper.parseCreateField(parseContext, TEST_DIMENSION); + + // Document should have 2 fields: one for VectorField (binary doc values) and one for KnnByteVectorField + List fields = document.getFields(); + assertEquals(2, fields.size()); + IndexableField field1 = fields.get(0); + IndexableField field2 = fields.get(1); + + VectorField vectorField; + KnnByteVectorField knnByteVectorField; + if (field1 instanceof VectorField) { + assertTrue(field2 instanceof KnnByteVectorField); + vectorField = (VectorField) field1; + knnByteVectorField = (KnnByteVectorField) field2; + } else { + assertTrue(field1 instanceof KnnByteVectorField); + assertTrue(field2 instanceof VectorField); + knnByteVectorField = (KnnByteVectorField) field1; + vectorField = (VectorField) field2; + } + + assertEquals(TEST_BYTE_VECTOR_BYTES_REF, vectorField.binaryValue()); + assertArrayEquals(TEST_BYTE_VECTOR, knnByteVectorField.vectorValue()); + + // Test when doc values are disabled + document = new ParseContext.Document(); + contentPath = new ContentPath(); + parseContext = mock(ParseContext.class); + when(parseContext.doc()).thenReturn(document); + when(parseContext.path()).thenReturn(contentPath); + + inputBuilder.hasDocValues(false); + luceneFieldMapper = Mockito.spy(new LuceneFieldMapper(inputBuilder.build())); + doReturn(Optional.of(TEST_BYTE_VECTOR)).when(luceneFieldMapper).getBytesFromContext(parseContext, TEST_DIMENSION); + doNothing().when(luceneFieldMapper).validateIfCircuitBreakerIsNotTriggered(); + doNothing().when(luceneFieldMapper).validateIfKNNPluginEnabled(); + + luceneFieldMapper.parseCreateField(parseContext, TEST_DIMENSION); + + // Document should have 1 field: one for KnnByteVectorField + fields = document.getFields(); + assertEquals(1, fields.size()); + IndexableField field = fields.get(0); + assertTrue(field instanceof KnnByteVectorField); + knnByteVectorField = (KnnByteVectorField) field; + assertArrayEquals(TEST_BYTE_VECTOR, knnByteVectorField.vectorValue()); + } + + private LuceneFieldMapper.CreateLuceneFieldMapperInput.CreateLuceneFieldMapperInputBuilder createLuceneFieldMapperInputBuilder( + VectorDataType vectorDataType + ) { + KNNMethodContext knnMethodContext = new KNNMethodContext( + KNNEngine.LUCENE, + SpaceType.DEFAULT, + new MethodComponentContext(METHOD_HNSW, Collections.emptyMap()) + ); + + KNNVectorFieldMapper.KNNVectorFieldType knnVectorFieldType = new KNNVectorFieldMapper.KNNVectorFieldType( + TEST_FIELD_NAME, + Collections.emptyMap(), + TEST_DIMENSION, + knnMethodContext, + vectorDataType + ); + + return LuceneFieldMapper.CreateLuceneFieldMapperInput.builder() + .name(TEST_FIELD_NAME) + .mappedFieldType(knnVectorFieldType) + .multiFields(FieldMapper.MultiFields.empty()) + .copyTo(FieldMapper.CopyTo.empty()) + .hasDocValues(true) + .vectorDataType(vectorDataType) + .ignoreMalformed(new Explicit<>(true, true)) + .knnMethodContext(knnMethodContext); + } + + private static float[] createInitializedFloatArray(int dimension, float value) { float[] array = new float[dimension]; Arrays.fill(array, value); return array; } + private static byte[] createInitializedByteArray(int dimension, byte value) { + byte[] array = new byte[dimension]; + Arrays.fill(array, value); + return array; + } + public IndexMetadata buildIndexMetaData(String indexName, Settings settings) { return IndexMetadata.builder(indexName) .settings(settings) diff --git a/src/test/java/org/opensearch/knn/index/query/KNNQueryBuilderTests.java b/src/test/java/org/opensearch/knn/index/query/KNNQueryBuilderTests.java index 74d99a805..e97cee611 100644 --- a/src/test/java/org/opensearch/knn/index/query/KNNQueryBuilderTests.java +++ b/src/test/java/org/opensearch/knn/index/query/KNNQueryBuilderTests.java @@ -30,6 +30,7 @@ import org.opensearch.knn.index.KNNMethodContext; import org.opensearch.knn.index.MethodComponentContext; import org.opensearch.knn.index.SpaceType; +import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.mapper.KNNVectorFieldMapper; import org.opensearch.knn.index.util.KNNEngine; import org.opensearch.knn.indices.ModelDao; @@ -175,6 +176,7 @@ public void testDoToQuery_Normal() throws Exception { KNNVectorFieldMapper.KNNVectorFieldType mockKNNVectorField = mock(KNNVectorFieldMapper.KNNVectorFieldType.class); when(mockQueryShardContext.index()).thenReturn(dummyIndex); when(mockKNNVectorField.getDimension()).thenReturn(4); + when(mockKNNVectorField.getVectorDataType()).thenReturn(VectorDataType.FLOAT); when(mockQueryShardContext.fieldMapper(anyString())).thenReturn(mockKNNVectorField); KNNQuery query = (KNNQuery) knnQueryBuilder.doToQuery(mockQueryShardContext); assertEquals(knnQueryBuilder.getK(), query.getK()); @@ -190,6 +192,7 @@ public void testDoToQuery_KnnQueryWithFilter() throws Exception { KNNVectorFieldMapper.KNNVectorFieldType mockKNNVectorField = mock(KNNVectorFieldMapper.KNNVectorFieldType.class); when(mockQueryShardContext.index()).thenReturn(dummyIndex); when(mockKNNVectorField.getDimension()).thenReturn(4); + when(mockKNNVectorField.getVectorDataType()).thenReturn(VectorDataType.FLOAT); MethodComponentContext methodComponentContext = new MethodComponentContext(METHOD_HNSW, ImmutableMap.of()); KNNMethodContext knnMethodContext = new KNNMethodContext(KNNEngine.LUCENE, SpaceType.L2, methodComponentContext); when(mockKNNVectorField.getKnnMethodContext()).thenReturn(knnMethodContext); @@ -241,6 +244,7 @@ public void testDoToQuery_FromModel() { // Dimension is -1. In this case, model metadata will need to provide dimension when(mockKNNVectorField.getDimension()).thenReturn(-K); + when(mockKNNVectorField.getVectorDataType()).thenReturn(VectorDataType.FLOAT); when(mockKNNVectorField.getKnnMethodContext()).thenReturn(null); String modelId = "test-model-id"; when(mockKNNVectorField.getModelId()).thenReturn(modelId); diff --git a/src/test/java/org/opensearch/knn/index/query/KNNQueryFactoryTests.java b/src/test/java/org/opensearch/knn/index/query/KNNQueryFactoryTests.java index 674d1be39..4dccfd087 100644 --- a/src/test/java/org/opensearch/knn/index/query/KNNQueryFactoryTests.java +++ b/src/test/java/org/opensearch/knn/index/query/KNNQueryFactoryTests.java @@ -24,6 +24,7 @@ import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; +import static org.opensearch.knn.common.KNNConstants.DEFAULT_VECTOR_DATA_TYPE_FIELD; public class KNNQueryFactoryTests extends KNNTestCase { private static final String FILTER_FILED_NAME = "foo"; @@ -38,7 +39,14 @@ public class KNNQueryFactoryTests extends KNNTestCase { public void testCreateCustomKNNQuery() { for (KNNEngine knnEngine : KNNEngine.getEnginesThatCreateCustomSegmentFiles()) { - Query query = KNNQueryFactory.create(knnEngine, testIndexName, testFieldName, testQueryVector, testK); + Query query = KNNQueryFactory.create( + knnEngine, + testIndexName, + testFieldName, + testQueryVector, + testK, + DEFAULT_VECTOR_DATA_TYPE_FIELD + ); assertTrue(query instanceof KNNQuery); assertEquals(testIndexName, ((KNNQuery) query).getIndexName()); @@ -53,7 +61,14 @@ public void testCreateLuceneDefaultQuery() { .filter(knnEngine -> !KNNEngine.getEnginesThatCreateCustomSegmentFiles().contains(knnEngine)) .collect(Collectors.toList()); for (KNNEngine knnEngine : luceneDefaultQueryEngineList) { - Query query = KNNQueryFactory.create(knnEngine, testIndexName, testFieldName, testQueryVector, testK); + Query query = KNNQueryFactory.create( + knnEngine, + testIndexName, + testFieldName, + testQueryVector, + testK, + DEFAULT_VECTOR_DATA_TYPE_FIELD + ); assertTrue(query.getClass().isAssignableFrom(KnnFloatVectorQuery.class)); } } @@ -71,6 +86,7 @@ public void testCreateLuceneQueryWithFilter() { .indexName(testIndexName) .fieldName(testFieldName) .vector(testQueryVector) + .vectorDataType(DEFAULT_VECTOR_DATA_TYPE_FIELD) .k(testK) .context(mockQueryShardContext) .filter(FILTER_QUERY_BUILDER) diff --git a/src/test/java/org/opensearch/knn/plugin/script/KNNScoringSpaceUtilTests.java b/src/test/java/org/opensearch/knn/plugin/script/KNNScoringSpaceUtilTests.java index 92fd56e45..b5bc4b95f 100644 --- a/src/test/java/org/opensearch/knn/plugin/script/KNNScoringSpaceUtilTests.java +++ b/src/test/java/org/opensearch/knn/plugin/script/KNNScoringSpaceUtilTests.java @@ -6,6 +6,7 @@ package org.opensearch.knn.plugin.script; import org.opensearch.knn.KNNTestCase; +import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.mapper.KNNVectorFieldMapper; import org.opensearch.index.mapper.BinaryFieldMapper; import org.opensearch.index.mapper.NumberFieldMapper; @@ -64,11 +65,14 @@ public void testParseKNNVectorQuery() { KNNVectorFieldMapper.KNNVectorFieldType fieldType = mock(KNNVectorFieldMapper.KNNVectorFieldType.class); when(fieldType.getDimension()).thenReturn(3); - assertArrayEquals(arrayFloat, KNNScoringSpaceUtil.parseToFloatArray(arrayListQueryObject, 3), 0.1f); + assertArrayEquals(arrayFloat, KNNScoringSpaceUtil.parseToFloatArray(arrayListQueryObject, 3, VectorDataType.FLOAT), 0.1f); - expectThrows(IllegalStateException.class, () -> KNNScoringSpaceUtil.parseToFloatArray(arrayListQueryObject, 4)); + expectThrows( + IllegalStateException.class, + () -> KNNScoringSpaceUtil.parseToFloatArray(arrayListQueryObject, 4, VectorDataType.FLOAT) + ); String invalidObject = "invalidObject"; - expectThrows(ClassCastException.class, () -> KNNScoringSpaceUtil.parseToFloatArray(invalidObject, 3)); + expectThrows(ClassCastException.class, () -> KNNScoringSpaceUtil.parseToFloatArray(invalidObject, 3, VectorDataType.FLOAT)); } } diff --git a/src/test/java/org/opensearch/knn/plugin/script/KNNScoringUtilTests.java b/src/test/java/org/opensearch/knn/plugin/script/KNNScoringUtilTests.java index 49add790e..4a2bb7254 100644 --- a/src/test/java/org/opensearch/knn/plugin/script/KNNScoringUtilTests.java +++ b/src/test/java/org/opensearch/knn/plugin/script/KNNScoringUtilTests.java @@ -7,6 +7,7 @@ import org.opensearch.knn.KNNTestCase; import org.opensearch.knn.index.KNNVectorScriptDocValues; +import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.VectorField; import org.apache.lucene.tests.analysis.MockAnalyzer; import org.apache.lucene.document.BinaryDocValuesField; @@ -81,7 +82,7 @@ public void testGetInvalidVectorMagnitudeSquared() { public void testConvertInvalidVectorToPrimitive() { float[] primitiveVector = null; - assertEquals(primitiveVector, KNNScoringSpaceUtil.convertVectorToPrimitive(primitiveVector)); + assertEquals(primitiveVector, KNNScoringSpaceUtil.convertVectorToPrimitive(primitiveVector, VectorDataType.FLOAT)); } public void testCosineSimilQueryVectorZeroMagnitude() { @@ -243,7 +244,11 @@ public KNNVectorScriptDocValues getScriptDocValues(String fieldName) throws IOEx if (scriptDocValues == null) { reader = DirectoryReader.open(directory); LeafReaderContext leafReaderContext = reader.getContext().leaves().get(0); - scriptDocValues = new KNNVectorScriptDocValues(leafReaderContext.reader().getBinaryDocValues(fieldName), fieldName); + scriptDocValues = new KNNVectorScriptDocValues( + leafReaderContext.reader().getBinaryDocValues(fieldName), + fieldName, + VectorDataType.FLOAT + ); } return scriptDocValues; } From cc12e5d8328bfd75ee608b25a2b85f581d728589 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Wed, 12 Jul 2023 12:55:41 -0700 Subject: [PATCH 118/416] Fixing the opensearch-build issue happening due to lower version of gcc. (#976) (#977) Signed-off-by: Navneet Verma (cherry picked from commit 54a75c1ea76b2e48fd90e3f1551965729a8db72b) Co-authored-by: Navneet Verma --- jni/src/faiss_wrapper.cpp | 54 ++++++++++++++------------------------- 1 file changed, 19 insertions(+), 35 deletions(-) diff --git a/jni/src/faiss_wrapper.cpp b/jni/src/faiss_wrapper.cpp index 3d396610e..e8fb4de20 100644 --- a/jni/src/faiss_wrapper.cpp +++ b/jni/src/faiss_wrapper.cpp @@ -41,9 +41,6 @@ void SetExtraParameters(knn_jni::JNIUtilInterface * jniUtil, JNIEnv *env, // Train an index with data provided void InternalTrainIndex(faiss::Index * index, faiss::idx_t n, const float* x); -// Create the SearchParams based on the Index Type -std::unique_ptr buildSearchParams(const faiss::IndexIDMap *indexReader, faiss::IDSelector* idSelector); - // Helps to choose the right FilterIdsSelectorType for Faiss FilterIdsSelectorType getIdSelectorType(const int* filterIds, int filterIdsLength); @@ -249,9 +246,26 @@ jobjectArray knn_jni::faiss_wrapper::QueryIndex_WithFilter(knn_jni::JNIUtilInter buildFilterIdsBitMap(filteredIdsArray, filterIdsLength, bitmap.data()); idSelector.reset(new faiss::IDSelectorBitmap(filterIdsLength, bitmap.data())); } - std::unique_ptr searchParameters = buildSearchParams(indexReader, idSelector.get()); + faiss::SearchParameters *searchParameters; + faiss::SearchParametersHNSW hnswParams; + faiss::SearchParametersIVF ivfParams; + auto hnswReader = dynamic_cast(indexReader->index); + if(hnswReader) { + // Setting the ef_search value equal to what was provided during index creation. SearchParametersHNSW has a default + // value of ef_search = 16 which will then be used. + hnswParams.efSearch = hnswReader->hnsw.efSearch; + hnswParams.sel = idSelector.get(); + searchParameters = &hnswParams; + } else { + auto ivfReader = dynamic_cast(indexReader->index); + auto ivfFlatReader = dynamic_cast(indexReader->index); + if(ivfReader || ivfFlatReader) { + ivfParams.sel = idSelector.get(); + searchParameters = &ivfParams; + } + } try { - indexReader->search(1, rawQueryvector, kJ, dis.data(), ids.data(), searchParameters.get()); + indexReader->search(1, rawQueryvector, kJ, dis.data(), ids.data(), searchParameters); } catch (...) { jniUtil->ReleaseFloatArrayElements(env, queryVectorJ, rawQueryvector, JNI_ABORT); jniUtil->ReleaseIntArrayElements(env, filterIdsJ, filteredIdsArray, JNI_ABORT); @@ -475,33 +489,3 @@ void buildFilterIdsBitMap(const int* filterIds, int filterIdsLength, uint8_t* bi bitsetVector[bitsetArrayIndex] = bitsetVector[bitsetArrayIndex] | (1 << (value & 7)); } } - -/** - * Based on the type of the index reader we need to return the SearchParameters. The way we do this by dynamically - * casting the IndexReader. - * @param indexReader - * @param idSelector - * @return SearchParameters - */ -std::unique_ptr buildSearchParams(const faiss::IndexIDMap *indexReader, faiss::IDSelector* idSelector) { - auto hnswReader = dynamic_cast(indexReader->index); - if(hnswReader) { - // we need to make this variable unique_ptr so that the scope can be shared with caller function. - std::unique_ptr hnswParams(new faiss::SearchParametersHNSW); - // Setting the ef_search value equal to what was provided during index creation. SearchParametersHNSW has a default - // value of ef_search = 16 which will then be used. - hnswParams->efSearch = hnswReader->hnsw.efSearch; - hnswParams->sel = idSelector; - return hnswParams; - } - - auto ivfReader = dynamic_cast(indexReader->index); - auto ivfFlatReader = dynamic_cast(indexReader->index); - if(ivfReader || ivfFlatReader) { - // we need to make this variable unique_ptr so that the scope can be shared with caller function. - std::unique_ptr ivfParams(new faiss::SearchParametersIVF); - ivfParams->sel = idSelector; - return ivfParams; - } - throw std::runtime_error("Invalid Index Type supported for Filtered Search on Faiss"); -} From c40d142339623e4e9a84771d632fea0586060781 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Thu, 13 Jul 2023 10:15:53 -0700 Subject: [PATCH 119/416] Fixing the k-nn build for windows platform. (#980) (#982) Signed-off-by: Navneet Verma (cherry picked from commit 4c945753754c25e106c65227d22aed1e37fa0c8f) Co-authored-by: Navneet Verma --- scripts/windowsScript.ps1 | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/scripts/windowsScript.ps1 b/scripts/windowsScript.ps1 index eee64046a..93945b29f 100644 --- a/scripts/windowsScript.ps1 +++ b/scripts/windowsScript.ps1 @@ -10,6 +10,16 @@ git submodule update --init -- jni/external/faiss # As we are using x86_64-w64-mingw32-gcc compiler we need to replace this macro with __MINGW32__ (Get-Content jni/external/faiss/faiss/impl/index_read.cpp).replace('_MSC_VER', '__MINGW32__') | Set-Content jni/external/faiss/faiss/impl/index_read.cpp (Get-Content jni/external/faiss/faiss/impl/index_write.cpp).replace('_MSC_VER', '__MINGW32__') | Set-Content jni/external/faiss/faiss/impl/index_write.cpp +(Get-Content jni/external/faiss/faiss/impl/platform_macros.h).replace('_MSC_VER', '__MINGW32__') | Set-Content jni/external/faiss/faiss/impl/platform_macros.h +(Get-Content jni/external/faiss/faiss/impl/platform_macros.h).replace('#define __PRETTY_FUNCTION__ __FUNCSIG__', ' ') | Set-Content jni/external/faiss/faiss/impl/platform_macros.h +(Get-Content jni/external/faiss/faiss/utils/utils.cpp).replace('_MSC_VER', '__MINGW32__') | Set-Content jni/external/faiss/faiss/utils/utils.cpp +(Get-Content jni/external/faiss/faiss/utils/prefetch.h).replace('_MSC_VER', '__MINGW32__') | Set-Content jni/external/faiss/faiss/utils/prefetch.h +(Get-Content jni/external/faiss/faiss/invlists/InvertedListsIOHook.cpp).replace('_MSC_VER', '__MINGW32__') | Set-Content jni/external/faiss/faiss/invlists/InvertedListsIOHook.cpp +(Get-Content jni/external/faiss/faiss/AutoTune.cpp).replace('__PRETTY_FUNCTION__', 'NULL') | Set-Content jni/external/faiss/faiss/AutoTune.cpp +(Get-Content jni/external/faiss/faiss/utils/distances_simd.cpp).replace('FAISS_PRAGMA_IMPRECISE_FUNCTION_BEGIN', ' ') | Set-Content jni/external/faiss/faiss/utils/distances_simd.cpp +(Get-Content jni/external/faiss/faiss/utils/distances_simd.cpp).replace('FAISS_PRAGMA_IMPRECISE_FUNCTION_END', ' ') | Set-Content jni/external/faiss/faiss/utils/distances_simd.cpp + + # is a Unix header and is not available on Windows. So, adding condition to include it if not running on Windows # Replace '#include ' with @@ -17,3 +27,13 @@ git submodule update --init -- jni/external/faiss # #include # #endif (Get-Content jni/external/faiss/faiss/OnDiskInvertedLists.cpp).replace('#include ', "#ifndef __MINGW32__`n#include `n#endif") | Set-Content jni/external/faiss/faiss/OnDiskInvertedLists.cpp +# intrin.h function like __builtin_ctz, __builtin_clzll is not available in MINGW32. So, adding condition to include it if not running on Windows +# Replace '#include ' with +# #ifndef __MINGW32__ +# include +# and +# Closing the above #ifndef with +# #define __builtin_popcountl __popcnt64 +# #endif +(Get-Content jni/external/faiss/faiss/impl/platform_macros.h).replace('#include ', "#ifndef __MINGW32__`n#include `n") | Set-Content jni/external/faiss/faiss/impl/platform_macros.h +(Get-Content jni/external/faiss/faiss/impl/platform_macros.h).replace('#define __builtin_popcountl __popcnt64', "#define __builtin_popcountl __popcnt64`n#endif`n") | Set-Content jni/external/faiss/faiss/impl/platform_macros.h From 3f4c5a24fababcb8b31d3fd3dd60c9e4bbff3a2c Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Thu, 13 Jul 2023 10:22:36 -0700 Subject: [PATCH 120/416] Add gcc version verification before building k-NN (#984) (#985) Signed-off-by: Peter Zhu (cherry picked from commit fe25c6dd15ca04c0f70ce4bf68d9e548575c22de) Co-authored-by: Peter Zhu --- scripts/build.sh | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/scripts/build.sh b/scripts/build.sh index 9790a4483..683ed47dc 100644 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -105,6 +105,15 @@ if [ "$JAVA_HOME" = "" ]; then echo "SET JAVA_HOME=$JAVA_HOME" fi +# Ensure gcc version is above 4.9.0 for faiss 1.7.4+ compilation +GCC_VERSION=`gcc --version | head -n 1 | cut -d ' ' -f3` +GCC_REQUIRED_VERSION=4.9.0 +COMPARE_VERSION=`echo $GCC_REQUIRED_VERSION $GCC_VERSION | tr ' ' '\n' | sort -V | uniq | head -n 1` +if [ ! "$COMPARE_VERSION" = "$GCC_REQUIRED_VERSION" ]; then + echo "gcc version on this env is older than $GCC_REQUIRED_VERSION, exit 1" + exit 1 +fi + # Build k-NN lib and plugin through gradle tasks cd $work_dir # Gradle build is used here to replace gradle assemble due to build will also call cmake and make before generating jars From 535d9afdc7f5843334c6448a20698a8d11778304 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Thu, 13 Jul 2023 22:29:45 -0400 Subject: [PATCH 121/416] Add gcc version ceiling verification before building k-NN on arm64 (#987) (#989) * Add gcc version ceiling verification before building k-NN on arm64 Signed-off-by: Peter Zhu * Add more comments Signed-off-by: Peter Zhu --------- Signed-off-by: Peter Zhu (cherry picked from commit afdf12511bc9a171ca364965b7107d6cf669f691) Co-authored-by: Peter Zhu --- scripts/build.sh | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/scripts/build.sh b/scripts/build.sh index 683ed47dc..a7f8e356a 100644 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -106,6 +106,7 @@ if [ "$JAVA_HOME" = "" ]; then fi # Ensure gcc version is above 4.9.0 for faiss 1.7.4+ compilation +# https://github.com/opensearch-project/k-NN/issues/975 GCC_VERSION=`gcc --version | head -n 1 | cut -d ' ' -f3` GCC_REQUIRED_VERSION=4.9.0 COMPARE_VERSION=`echo $GCC_REQUIRED_VERSION $GCC_VERSION | tr ' ' '\n' | sort -V | uniq | head -n 1` @@ -114,6 +115,16 @@ if [ ! "$COMPARE_VERSION" = "$GCC_REQUIRED_VERSION" ]; then exit 1 fi +# Ensure gcc version is below 8.0.0 for faiss 1.7.4+ compilation so it will not crash on arm64 CentOS7 +# https://github.com/opensearch-project/k-NN/issues/975 +GCC_REQUIRED_VERSION_CEILING=8.0.0 +COMPARE_VERSION_CEILING=`echo $GCC_REQUIRED_VERSION_CEILING $GCC_VERSION | tr ' ' '\n' | sort -V | uniq | tail -n 1` +if [ ! "$COMPARE_VERSION_CEILING" = "$GCC_REQUIRED_VERSION_CEILING" ]; then + echo "gcc version on this env is newer than $GCC_REQUIRED_VERSION_CEILING, exit 1" + exit 1 +fi + + # Build k-NN lib and plugin through gradle tasks cd $work_dir # Gradle build is used here to replace gradle assemble due to build will also call cmake and make before generating jars From 5757050da2b5cac1ced214543ae18fb0544ee0ad Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Thu, 13 Jul 2023 22:57:41 -0400 Subject: [PATCH 122/416] Only check the gcc version ceiling when it is on linux (#991) (#992) Signed-off-by: Peter Zhu (cherry picked from commit 08014080f62f06c2662e8353cd778d0d369c81d7) Co-authored-by: Peter Zhu --- scripts/build.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/build.sh b/scripts/build.sh index a7f8e356a..930f0b602 100644 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -110,7 +110,7 @@ fi GCC_VERSION=`gcc --version | head -n 1 | cut -d ' ' -f3` GCC_REQUIRED_VERSION=4.9.0 COMPARE_VERSION=`echo $GCC_REQUIRED_VERSION $GCC_VERSION | tr ' ' '\n' | sort -V | uniq | head -n 1` -if [ ! "$COMPARE_VERSION" = "$GCC_REQUIRED_VERSION" ]; then +if [ "$COMPARE_VERSION" != "$GCC_REQUIRED_VERSION" ]; then echo "gcc version on this env is older than $GCC_REQUIRED_VERSION, exit 1" exit 1 fi @@ -119,7 +119,7 @@ fi # https://github.com/opensearch-project/k-NN/issues/975 GCC_REQUIRED_VERSION_CEILING=8.0.0 COMPARE_VERSION_CEILING=`echo $GCC_REQUIRED_VERSION_CEILING $GCC_VERSION | tr ' ' '\n' | sort -V | uniq | tail -n 1` -if [ ! "$COMPARE_VERSION_CEILING" = "$GCC_REQUIRED_VERSION_CEILING" ]; then +if [ "$COMPARE_VERSION_CEILING" != "$GCC_REQUIRED_VERSION_CEILING" ] && (echo "$OSTYPE" | grep -i linux); then echo "gcc version on this env is newer than $GCC_REQUIRED_VERSION_CEILING, exit 1" exit 1 fi From 74f2da376b0ace9fc2ca8c2ec0cdbfea7494b09d Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Thu, 20 Jul 2023 12:37:03 -0700 Subject: [PATCH 123/416] Fixing the filter-spec files and .yml file with latest dataset. (#1000) (#1001) Signed-off-by: Navneet Verma (cherry picked from commit 5b26be318e71fed55efb7f1038bbfcb44fc06029) Co-authored-by: Navneet Verma --- .../filtering/relaxed-filter/relaxed-filter-spec.json | 4 ++-- .../filtering/relaxed-filter/relaxed-filter-test.yml | 2 +- .../filtering/relaxed-filter/relaxed-filter-test.yml | 2 +- .../perf-tool/sample-configs/filter-spec/filter-4-spec.json | 2 +- .../perf-tool/sample-configs/filter-spec/filter-5-spec.json | 4 ++-- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/benchmarks/perf-tool/release-configs/faiss-hnsw/filtering/relaxed-filter/relaxed-filter-spec.json b/benchmarks/perf-tool/release-configs/faiss-hnsw/filtering/relaxed-filter/relaxed-filter-spec.json index fecde0392..3e04d12c4 100644 --- a/benchmarks/perf-tool/release-configs/faiss-hnsw/filtering/relaxed-filter/relaxed-filter-spec.json +++ b/benchmarks/perf-tool/release-configs/faiss-hnsw/filtering/relaxed-filter/relaxed-filter-spec.json @@ -34,9 +34,9 @@ { "term": { - "color": "sweet" + "taste": "sweet" } } ] } -} \ No newline at end of file +} diff --git a/benchmarks/perf-tool/release-configs/faiss-hnsw/filtering/relaxed-filter/relaxed-filter-test.yml b/benchmarks/perf-tool/release-configs/faiss-hnsw/filtering/relaxed-filter/relaxed-filter-test.yml index 61486b3b6..3445634b2 100644 --- a/benchmarks/perf-tool/release-configs/faiss-hnsw/filtering/relaxed-filter/relaxed-filter-test.yml +++ b/benchmarks/perf-tool/release-configs/faiss-hnsw/filtering/relaxed-filter/relaxed-filter-test.yml @@ -28,7 +28,7 @@ steps: dataset_format: hdf5 dataset_path: [DATASET_PATH]/sift-128-euclidean-with-attr.hdf5 neighbors_format: hdf5 - neighbors_path: [DATASET_PATH]/sift-128-euclidean-with-filters.hdf5 + neighbors_path: [DATASET_PATH]/sift-128-euclidean-with-filters-updated.hdf5 neighbors_dataset: neighbors_filter_5 filter_spec: [INDEX_SPEC_PATH]/relaxed-filter-spec.json filter_type: FILTER diff --git a/benchmarks/perf-tool/release-configs/lucene-hnsw/filtering/relaxed-filter/relaxed-filter-test.yml b/benchmarks/perf-tool/release-configs/lucene-hnsw/filtering/relaxed-filter/relaxed-filter-test.yml index 44ed8e66e..62d032b00 100644 --- a/benchmarks/perf-tool/release-configs/lucene-hnsw/filtering/relaxed-filter/relaxed-filter-test.yml +++ b/benchmarks/perf-tool/release-configs/lucene-hnsw/filtering/relaxed-filter/relaxed-filter-test.yml @@ -28,7 +28,7 @@ steps: dataset_format: hdf5 dataset_path: [DATASET_PATH]/sift-128-euclidean-with-attr.hdf5 neighbors_format: hdf5 - neighbors_path: [DATASET_PATH]/sift-128-euclidean-with-filters.hdf5 + neighbors_path: [DATASET_PATH]/sift-128-euclidean-with-filters-updated.hdf5 neighbors_dataset: neighbors_filter_5 filter_spec: [INDEX_SPEC_PATH]/relaxed-filter-spec.json filter_type: FILTER diff --git a/benchmarks/perf-tool/sample-configs/filter-spec/filter-4-spec.json b/benchmarks/perf-tool/sample-configs/filter-spec/filter-4-spec.json index 9e6356f1c..822d63b37 100644 --- a/benchmarks/perf-tool/sample-configs/filter-spec/filter-4-spec.json +++ b/benchmarks/perf-tool/sample-configs/filter-spec/filter-4-spec.json @@ -41,4 +41,4 @@ } ] } -} \ No newline at end of file +} diff --git a/benchmarks/perf-tool/sample-configs/filter-spec/filter-5-spec.json b/benchmarks/perf-tool/sample-configs/filter-spec/filter-5-spec.json index fecde0392..3e04d12c4 100644 --- a/benchmarks/perf-tool/sample-configs/filter-spec/filter-5-spec.json +++ b/benchmarks/perf-tool/sample-configs/filter-spec/filter-5-spec.json @@ -34,9 +34,9 @@ { "term": { - "color": "sweet" + "taste": "sweet" } } ] } -} \ No newline at end of file +} From 24e9fb3386dd9bdc34acbe326be928c1baac8cb2 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Thu, 27 Jul 2023 15:10:33 -0700 Subject: [PATCH 124/416] Bump certifi and aiohttp to fix vulnerabilities (#1010) (#1011) Signed-off-by: Junqiu Lei (cherry picked from commit 84509fbf4ef9326129e73807aae914448cb68602) Co-authored-by: Junqiu Lei --- benchmarks/osb/requirements.txt | 6 +++--- benchmarks/perf-tool/requirements.txt | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/benchmarks/osb/requirements.txt b/benchmarks/osb/requirements.txt index 2c35c8497..423253c71 100644 --- a/benchmarks/osb/requirements.txt +++ b/benchmarks/osb/requirements.txt @@ -4,7 +4,7 @@ # # pip-compile # -aiohttp==3.8.1 +aiohttp==3.8.5 # via opensearch-py aiosignal==1.2.0 # via aiohttp @@ -16,7 +16,7 @@ attrs==21.4.0 # jsonschema cachetools==4.2.4 # via google-auth -certifi==2022.12.7 +certifi==2023.7.22 # via # opensearch-benchmark # opensearch-py @@ -95,4 +95,4 @@ zipp==3.7.0 # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: -# setuptools \ No newline at end of file +# setuptools diff --git a/benchmarks/perf-tool/requirements.txt b/benchmarks/perf-tool/requirements.txt index b1776be83..5489e63dd 100644 --- a/benchmarks/perf-tool/requirements.txt +++ b/benchmarks/perf-tool/requirements.txt @@ -8,7 +8,7 @@ cached-property==1.5.2 # via h5py cerberus==1.3.4 # via -r requirements.in -certifi==2022.12.7 +certifi==2023.7.22 # via # opensearch-py # requests From ae9c9f4d08b8948897801959e5f01ed614b38251 Mon Sep 17 00:00:00 2001 From: Junqiu Lei Date: Fri, 28 Jul 2023 12:59:53 -0500 Subject: [PATCH 125/416] Bump version and fix core refactor (#1006) Signed-off-by: Junqiu Lei --- .../workflows/backwards_compatibility_tests_workflow.yml | 8 ++++---- build.gradle | 2 +- .../src/test/java/org/opensearch/knn/bwc/ModelIT.java | 2 +- .../test/java/org/opensearch/knn/bwc/ScriptScoringIT.java | 2 +- .../exception/DeleteModelWhenInTrainStateException.java | 2 +- .../java/org/opensearch/knn/index/KNNMethodContext.java | 6 +++--- .../org/opensearch/knn/index/MethodComponentContext.java | 6 +++--- .../org/opensearch/knn/index/query/KNNQueryBuilder.java | 6 +++--- src/main/java/org/opensearch/knn/indices/Model.java | 6 +++--- .../java/org/opensearch/knn/indices/ModelGraveyard.java | 4 ++-- .../java/org/opensearch/knn/indices/ModelMetadata.java | 6 +++--- src/main/java/org/opensearch/knn/indices/ModelState.java | 6 +++--- src/main/java/org/opensearch/knn/plugin/KNNPlugin.java | 2 +- .../opensearch/knn/plugin/rest/RestKNNWarmupHandler.java | 2 +- .../opensearch/knn/plugin/rest/RestTrainModelHandler.java | 2 +- .../knn/plugin/transport/DeleteModelAction.java | 2 +- .../knn/plugin/transport/DeleteModelRequest.java | 4 ++-- .../knn/plugin/transport/DeleteModelResponse.java | 4 ++-- .../opensearch/knn/plugin/transport/GetModelAction.java | 2 +- .../opensearch/knn/plugin/transport/GetModelRequest.java | 4 ++-- .../opensearch/knn/plugin/transport/GetModelResponse.java | 4 ++-- .../opensearch/knn/plugin/transport/KNNStatsAction.java | 2 +- .../knn/plugin/transport/KNNStatsNodeRequest.java | 4 ++-- .../knn/plugin/transport/KNNStatsNodeResponse.java | 4 ++-- .../opensearch/knn/plugin/transport/KNNStatsRequest.java | 4 ++-- .../opensearch/knn/plugin/transport/KNNStatsResponse.java | 4 ++-- .../knn/plugin/transport/KNNStatsTransportAction.java | 2 +- .../opensearch/knn/plugin/transport/KNNWarmupAction.java | 2 +- .../opensearch/knn/plugin/transport/KNNWarmupRequest.java | 2 +- .../knn/plugin/transport/KNNWarmupResponse.java | 4 ++-- .../knn/plugin/transport/KNNWarmupTransportAction.java | 4 ++-- .../knn/plugin/transport/RemoveModelFromCacheAction.java | 2 +- .../plugin/transport/RemoveModelFromCacheNodeRequest.java | 4 ++-- .../transport/RemoveModelFromCacheNodeResponse.java | 2 +- .../knn/plugin/transport/RemoveModelFromCacheRequest.java | 4 ++-- .../plugin/transport/RemoveModelFromCacheResponse.java | 4 ++-- .../transport/RemoveModelFromCacheTransportAction.java | 2 +- .../knn/plugin/transport/SearchModelAction.java | 2 +- .../transport/TrainingJobRouteDecisionInfoAction.java | 2 +- .../TrainingJobRouteDecisionInfoNodeRequest.java | 2 +- .../TrainingJobRouteDecisionInfoNodeResponse.java | 4 ++-- .../transport/TrainingJobRouteDecisionInfoRequest.java | 2 +- .../transport/TrainingJobRouteDecisionInfoResponse.java | 4 ++-- .../TrainingJobRouteDecisionInfoTransportAction.java | 2 +- .../knn/plugin/transport/TrainingJobRouterAction.java | 2 +- .../knn/plugin/transport/TrainingModelAction.java | 2 +- .../knn/plugin/transport/TrainingModelRequest.java | 4 ++-- .../knn/plugin/transport/TrainingModelResponse.java | 4 ++-- .../knn/plugin/transport/UpdateModelGraveyardAction.java | 2 +- .../knn/plugin/transport/UpdateModelGraveyardRequest.java | 4 ++-- .../transport/UpdateModelGraveyardTransportAction.java | 2 +- .../knn/plugin/transport/UpdateModelMetadataAction.java | 2 +- .../knn/plugin/transport/UpdateModelMetadataRequest.java | 4 ++-- .../transport/UpdateModelMetadataTransportAction.java | 2 +- .../java/org/opensearch/knn/KNNSingleNodeTestCase.java | 4 ++-- src/test/java/org/opensearch/knn/KNNTestCase.java | 2 +- .../org/opensearch/knn/index/KNNESSettingsTestIT.java | 2 +- src/test/java/org/opensearch/knn/index/OpenSearchIT.java | 2 +- .../java/org/opensearch/knn/index/VectorDataTypeIT.java | 2 +- .../opensearch/knn/index/codec/KNNCodecServiceTests.java | 2 +- .../opensearch/knn/index/query/KNNQueryBuilderTests.java | 8 ++++---- .../java/org/opensearch/knn/indices/ModelDaoTests.java | 2 +- .../knn/plugin/action/RestDeleteModelHandlerIT.java | 2 +- .../knn/plugin/action/RestGetModelHandlerIT.java | 2 +- .../knn/plugin/action/RestKNNStatsHandlerIT.java | 2 +- .../knn/plugin/action/RestLegacyKNNStatsHandlerIT.java | 2 +- .../knn/plugin/action/RestSearchModelHandlerIT.java | 2 +- .../knn/plugin/action/RestTrainModelHandlerIT.java | 2 +- .../opensearch/knn/plugin/script/KNNScriptScoringIT.java | 2 +- .../opensearch/knn/plugin/script/PainlessScriptIT.java | 2 +- .../plugin/transport/KNNWarmupTransportActionTests.java | 2 +- .../TrainingJobRouteDecisionInfoTransportActionTests.java | 2 +- .../opensearch/knn/training/TrainingJobRunnerTests.java | 2 +- .../java/org/opensearch/knn/KNNRestTestCase.java | 4 ++-- .../java/org/opensearch/knn/ODFERestTestCase.java | 2 +- 75 files changed, 115 insertions(+), 115 deletions(-) diff --git a/.github/workflows/backwards_compatibility_tests_workflow.yml b/.github/workflows/backwards_compatibility_tests_workflow.yml index 3005c6b6b..b5c25fd6c 100644 --- a/.github/workflows/backwards_compatibility_tests_workflow.yml +++ b/.github/workflows/backwards_compatibility_tests_workflow.yml @@ -14,8 +14,8 @@ jobs: strategy: matrix: java: [ 11, 17 ] - bwc_version : [ "1.1.0", "1.2.4", "1.3.8", "2.0.1", "2.1.0", "2.2.1", "2.3.0", "2.4.1", "2.5.0", "2.6.0", "2.7.0", "2.8.0" ] - opensearch_version : [ "2.9.0-SNAPSHOT" ] + bwc_version : [ "1.1.0", "1.2.4", "1.3.8", "2.0.1", "2.1.0", "2.2.1", "2.3.0", "2.4.1", "2.5.0", "2.6.0", "2.7.0", "2.8.0", "2.9.0" ] + opensearch_version : [ "2.10.0-SNAPSHOT" ] name: k-NN Restart-Upgrade BWC Tests runs-on: ubuntu-latest @@ -46,8 +46,8 @@ jobs: strategy: matrix: java: [ 11, 17 ] - bwc_version: [ "1.3.8", "2.0.1", "2.1.0", "2.2.1", "2.3.0", "2.4.1", "2.5.0", "2.6.0", "2.7.0", "2.8.0"] - opensearch_version: [ "2.9.0-SNAPSHOT" ] + bwc_version: [ "1.3.8", "2.0.1", "2.1.0", "2.2.1", "2.3.0", "2.4.1", "2.5.0", "2.6.0", "2.7.0", "2.8.0", "2.9.0"] + opensearch_version: [ "2.10.0-SNAPSHOT" ] name: k-NN Rolling-Upgrade BWC Tests runs-on: ubuntu-latest diff --git a/build.gradle b/build.gradle index f35644f4c..3303cbde2 100644 --- a/build.gradle +++ b/build.gradle @@ -10,7 +10,7 @@ buildscript { ext { // build.version_qualifier parameter applies to knn plugin artifacts only. OpenSearch version must be set // explicitly as 'opensearch.version' property, for instance opensearch.version=2.0.0-rc1-SNAPSHOT - opensearch_version = System.getProperty("opensearch.version", "2.9.0-SNAPSHOT") + opensearch_version = System.getProperty("opensearch.version", "2.10.0-SNAPSHOT") version_qualifier = System.getProperty("build.version_qualifier", "") opensearch_group = "org.opensearch" } diff --git a/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/ModelIT.java b/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/ModelIT.java index 0157ca45e..41f26c57e 100644 --- a/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/ModelIT.java +++ b/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/ModelIT.java @@ -21,7 +21,7 @@ import org.opensearch.knn.indices.ModelMetadata; import org.opensearch.knn.indices.ModelState; import org.opensearch.knn.plugin.KNNPlugin; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.search.SearchHit; import java.io.IOException; diff --git a/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/ScriptScoringIT.java b/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/ScriptScoringIT.java index 409a07507..ccb30fa1a 100644 --- a/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/ScriptScoringIT.java +++ b/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/ScriptScoringIT.java @@ -13,7 +13,7 @@ import org.opensearch.knn.IDVectorProducer; import org.opensearch.knn.KNNResult; import org.opensearch.knn.index.SpaceType; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import java.util.HashMap; import java.util.List; diff --git a/src/main/java/org/opensearch/knn/common/exception/DeleteModelWhenInTrainStateException.java b/src/main/java/org/opensearch/knn/common/exception/DeleteModelWhenInTrainStateException.java index fba20bd17..00f6e6e80 100644 --- a/src/main/java/org/opensearch/knn/common/exception/DeleteModelWhenInTrainStateException.java +++ b/src/main/java/org/opensearch/knn/common/exception/DeleteModelWhenInTrainStateException.java @@ -7,7 +7,7 @@ import org.opensearch.OpenSearchException; import org.opensearch.core.common.logging.LoggerMessageFormat; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; /** * Exception thrown when a model is deleted while it is in the training state. The RestStatus associated with this diff --git a/src/main/java/org/opensearch/knn/index/KNNMethodContext.java b/src/main/java/org/opensearch/knn/index/KNNMethodContext.java index db97df6aa..5f5a0f232 100644 --- a/src/main/java/org/opensearch/knn/index/KNNMethodContext.java +++ b/src/main/java/org/opensearch/knn/index/KNNMethodContext.java @@ -15,9 +15,9 @@ import lombok.Getter; import lombok.NonNull; import org.opensearch.common.ValidationException; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.knn.index.util.KNNEngine; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/src/main/java/org/opensearch/knn/index/MethodComponentContext.java b/src/main/java/org/opensearch/knn/index/MethodComponentContext.java index 756d463d9..cf10e9a64 100644 --- a/src/main/java/org/opensearch/knn/index/MethodComponentContext.java +++ b/src/main/java/org/opensearch/knn/index/MethodComponentContext.java @@ -13,9 +13,9 @@ import lombok.AllArgsConstructor; import lombok.Getter; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.index.mapper.MapperParsingException; diff --git a/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java b/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java index 2b1950017..b69b3dbfb 100644 --- a/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java +++ b/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java @@ -20,9 +20,9 @@ import org.opensearch.knn.plugin.stats.KNNCounter; import org.apache.lucene.search.Query; import org.opensearch.core.ParseField; -import org.opensearch.common.ParsingException; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.ParsingException; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.index.mapper.MappedFieldType; diff --git a/src/main/java/org/opensearch/knn/indices/Model.java b/src/main/java/org/opensearch/knn/indices/Model.java index c0071ad94..5007650dd 100644 --- a/src/main/java/org/opensearch/knn/indices/Model.java +++ b/src/main/java/org/opensearch/knn/indices/Model.java @@ -12,9 +12,9 @@ package org.opensearch.knn.indices; import org.opensearch.common.Nullable; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.knn.common.KNNConstants; diff --git a/src/main/java/org/opensearch/knn/indices/ModelGraveyard.java b/src/main/java/org/opensearch/knn/indices/ModelGraveyard.java index e7e11e66f..ead95cf9a 100644 --- a/src/main/java/org/opensearch/knn/indices/ModelGraveyard.java +++ b/src/main/java/org/opensearch/knn/indices/ModelGraveyard.java @@ -12,8 +12,8 @@ import org.opensearch.cluster.Diff; import org.opensearch.cluster.NamedDiff; import org.opensearch.cluster.metadata.Metadata; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; diff --git a/src/main/java/org/opensearch/knn/indices/ModelMetadata.java b/src/main/java/org/opensearch/knn/indices/ModelMetadata.java index 88994cf52..0d0c79bc3 100644 --- a/src/main/java/org/opensearch/knn/indices/ModelMetadata.java +++ b/src/main/java/org/opensearch/knn/indices/ModelMetadata.java @@ -13,9 +13,9 @@ import org.apache.commons.lang.builder.EqualsBuilder; import org.apache.commons.lang.builder.HashCodeBuilder; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.knn.common.KNNConstants; diff --git a/src/main/java/org/opensearch/knn/indices/ModelState.java b/src/main/java/org/opensearch/knn/indices/ModelState.java index d78ae06bc..35def5335 100644 --- a/src/main/java/org/opensearch/knn/indices/ModelState.java +++ b/src/main/java/org/opensearch/knn/indices/ModelState.java @@ -11,9 +11,9 @@ package org.opensearch.knn.indices; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import java.io.IOException; diff --git a/src/main/java/org/opensearch/knn/plugin/KNNPlugin.java b/src/main/java/org/opensearch/knn/plugin/KNNPlugin.java index bf89644cb..54011c259 100644 --- a/src/main/java/org/opensearch/knn/plugin/KNNPlugin.java +++ b/src/main/java/org/opensearch/knn/plugin/KNNPlugin.java @@ -48,7 +48,7 @@ import org.opensearch.cluster.metadata.IndexNameExpressionResolver; import org.opensearch.cluster.node.DiscoveryNodes; import org.opensearch.cluster.service.ClusterService; -import org.opensearch.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.IndexScopedSettings; import org.opensearch.common.settings.Setting; diff --git a/src/main/java/org/opensearch/knn/plugin/rest/RestKNNWarmupHandler.java b/src/main/java/org/opensearch/knn/plugin/rest/RestKNNWarmupHandler.java index a31c2f297..42991af13 100644 --- a/src/main/java/org/opensearch/knn/plugin/rest/RestKNNWarmupHandler.java +++ b/src/main/java/org/opensearch/knn/plugin/rest/RestKNNWarmupHandler.java @@ -17,7 +17,7 @@ import org.opensearch.cluster.metadata.IndexNameExpressionResolver; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.settings.Settings; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.rest.BaseRestHandler; import org.opensearch.rest.RestController; import org.opensearch.rest.RestRequest; diff --git a/src/main/java/org/opensearch/knn/plugin/rest/RestTrainModelHandler.java b/src/main/java/org/opensearch/knn/plugin/rest/RestTrainModelHandler.java index 67ce80959..a4a0de5de 100644 --- a/src/main/java/org/opensearch/knn/plugin/rest/RestTrainModelHandler.java +++ b/src/main/java/org/opensearch/knn/plugin/rest/RestTrainModelHandler.java @@ -27,7 +27,7 @@ import java.util.List; import java.util.Locale; -import static org.opensearch.common.xcontent.XContentParserUtils.ensureExpectedToken; +import static org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken; import static org.opensearch.knn.common.KNNConstants.DIMENSION; import static org.opensearch.knn.common.KNNConstants.KNN_METHOD; import static org.opensearch.knn.common.KNNConstants.MAX_VECTOR_COUNT_PARAMETER; diff --git a/src/main/java/org/opensearch/knn/plugin/transport/DeleteModelAction.java b/src/main/java/org/opensearch/knn/plugin/transport/DeleteModelAction.java index f58728368..33be3fde3 100644 --- a/src/main/java/org/opensearch/knn/plugin/transport/DeleteModelAction.java +++ b/src/main/java/org/opensearch/knn/plugin/transport/DeleteModelAction.java @@ -12,7 +12,7 @@ package org.opensearch.knn.plugin.transport; import org.opensearch.action.ActionType; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.Writeable; public class DeleteModelAction extends ActionType { diff --git a/src/main/java/org/opensearch/knn/plugin/transport/DeleteModelRequest.java b/src/main/java/org/opensearch/knn/plugin/transport/DeleteModelRequest.java index fee82adb5..dbc274160 100644 --- a/src/main/java/org/opensearch/knn/plugin/transport/DeleteModelRequest.java +++ b/src/main/java/org/opensearch/knn/plugin/transport/DeleteModelRequest.java @@ -14,8 +14,8 @@ import org.apache.commons.lang.StringUtils; import org.opensearch.action.ActionRequest; import org.opensearch.action.ActionRequestValidationException; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import java.io.IOException; diff --git a/src/main/java/org/opensearch/knn/plugin/transport/DeleteModelResponse.java b/src/main/java/org/opensearch/knn/plugin/transport/DeleteModelResponse.java index 3f2cb90de..b4879d881 100644 --- a/src/main/java/org/opensearch/knn/plugin/transport/DeleteModelResponse.java +++ b/src/main/java/org/opensearch/knn/plugin/transport/DeleteModelResponse.java @@ -12,8 +12,8 @@ import org.opensearch.action.ActionResponse; import org.opensearch.common.Nullable; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/src/main/java/org/opensearch/knn/plugin/transport/GetModelAction.java b/src/main/java/org/opensearch/knn/plugin/transport/GetModelAction.java index 74b115830..18c792ec7 100644 --- a/src/main/java/org/opensearch/knn/plugin/transport/GetModelAction.java +++ b/src/main/java/org/opensearch/knn/plugin/transport/GetModelAction.java @@ -12,7 +12,7 @@ package org.opensearch.knn.plugin.transport; import org.opensearch.action.ActionType; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.Writeable; /** * GetModelAction class diff --git a/src/main/java/org/opensearch/knn/plugin/transport/GetModelRequest.java b/src/main/java/org/opensearch/knn/plugin/transport/GetModelRequest.java index 08ba03e07..4c3d4de97 100644 --- a/src/main/java/org/opensearch/knn/plugin/transport/GetModelRequest.java +++ b/src/main/java/org/opensearch/knn/plugin/transport/GetModelRequest.java @@ -13,8 +13,8 @@ import org.opensearch.action.ActionRequest; import org.opensearch.action.ActionRequestValidationException; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import java.io.IOException; diff --git a/src/main/java/org/opensearch/knn/plugin/transport/GetModelResponse.java b/src/main/java/org/opensearch/knn/plugin/transport/GetModelResponse.java index c94c6769e..4df2b7f03 100644 --- a/src/main/java/org/opensearch/knn/plugin/transport/GetModelResponse.java +++ b/src/main/java/org/opensearch/knn/plugin/transport/GetModelResponse.java @@ -11,8 +11,8 @@ package org.opensearch.knn.plugin.transport; import org.opensearch.action.ActionResponse; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.knn.indices.Model; diff --git a/src/main/java/org/opensearch/knn/plugin/transport/KNNStatsAction.java b/src/main/java/org/opensearch/knn/plugin/transport/KNNStatsAction.java index ccafb00d5..df7799d1f 100644 --- a/src/main/java/org/opensearch/knn/plugin/transport/KNNStatsAction.java +++ b/src/main/java/org/opensearch/knn/plugin/transport/KNNStatsAction.java @@ -6,7 +6,7 @@ package org.opensearch.knn.plugin.transport; import org.opensearch.action.ActionType; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.Writeable; /** * KNNStatsAction class diff --git a/src/main/java/org/opensearch/knn/plugin/transport/KNNStatsNodeRequest.java b/src/main/java/org/opensearch/knn/plugin/transport/KNNStatsNodeRequest.java index ba66784f8..930039f06 100644 --- a/src/main/java/org/opensearch/knn/plugin/transport/KNNStatsNodeRequest.java +++ b/src/main/java/org/opensearch/knn/plugin/transport/KNNStatsNodeRequest.java @@ -6,8 +6,8 @@ package org.opensearch.knn.plugin.transport; import org.opensearch.action.support.nodes.BaseNodeRequest; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import java.io.IOException; diff --git a/src/main/java/org/opensearch/knn/plugin/transport/KNNStatsNodeResponse.java b/src/main/java/org/opensearch/knn/plugin/transport/KNNStatsNodeResponse.java index 1f1164bd8..14b78fed4 100644 --- a/src/main/java/org/opensearch/knn/plugin/transport/KNNStatsNodeResponse.java +++ b/src/main/java/org/opensearch/knn/plugin/transport/KNNStatsNodeResponse.java @@ -7,8 +7,8 @@ import org.opensearch.action.support.nodes.BaseNodeResponse; import org.opensearch.cluster.node.DiscoveryNode; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/src/main/java/org/opensearch/knn/plugin/transport/KNNStatsRequest.java b/src/main/java/org/opensearch/knn/plugin/transport/KNNStatsRequest.java index 26d8a9c5b..2e245e5a3 100644 --- a/src/main/java/org/opensearch/knn/plugin/transport/KNNStatsRequest.java +++ b/src/main/java/org/opensearch/knn/plugin/transport/KNNStatsRequest.java @@ -6,8 +6,8 @@ package org.opensearch.knn.plugin.transport; import org.opensearch.action.support.nodes.BaseNodesRequest; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.knn.plugin.stats.StatNames; import java.io.IOException; diff --git a/src/main/java/org/opensearch/knn/plugin/transport/KNNStatsResponse.java b/src/main/java/org/opensearch/knn/plugin/transport/KNNStatsResponse.java index f108e536e..ec8aa6f6e 100644 --- a/src/main/java/org/opensearch/knn/plugin/transport/KNNStatsResponse.java +++ b/src/main/java/org/opensearch/knn/plugin/transport/KNNStatsResponse.java @@ -9,8 +9,8 @@ import org.opensearch.action.support.nodes.BaseNodesResponse; import org.opensearch.cluster.ClusterName; import org.opensearch.cluster.node.DiscoveryNode; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/src/main/java/org/opensearch/knn/plugin/transport/KNNStatsTransportAction.java b/src/main/java/org/opensearch/knn/plugin/transport/KNNStatsTransportAction.java index 0189bf88d..7edc44894 100644 --- a/src/main/java/org/opensearch/knn/plugin/transport/KNNStatsTransportAction.java +++ b/src/main/java/org/opensearch/knn/plugin/transport/KNNStatsTransportAction.java @@ -12,7 +12,7 @@ import org.opensearch.action.support.nodes.TransportNodesAction; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.transport.TransportService; import org.opensearch.threadpool.ThreadPool; diff --git a/src/main/java/org/opensearch/knn/plugin/transport/KNNWarmupAction.java b/src/main/java/org/opensearch/knn/plugin/transport/KNNWarmupAction.java index 32fabe0c1..16b5c3a4d 100644 --- a/src/main/java/org/opensearch/knn/plugin/transport/KNNWarmupAction.java +++ b/src/main/java/org/opensearch/knn/plugin/transport/KNNWarmupAction.java @@ -6,7 +6,7 @@ package org.opensearch.knn.plugin.transport; import org.opensearch.action.ActionType; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.Writeable; /** * Action associated with k-NN warmup diff --git a/src/main/java/org/opensearch/knn/plugin/transport/KNNWarmupRequest.java b/src/main/java/org/opensearch/knn/plugin/transport/KNNWarmupRequest.java index 545f6fc47..30c7b52a0 100644 --- a/src/main/java/org/opensearch/knn/plugin/transport/KNNWarmupRequest.java +++ b/src/main/java/org/opensearch/knn/plugin/transport/KNNWarmupRequest.java @@ -6,7 +6,7 @@ package org.opensearch.knn.plugin.transport; import org.opensearch.action.support.broadcast.BroadcastRequest; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import java.io.IOException; diff --git a/src/main/java/org/opensearch/knn/plugin/transport/KNNWarmupResponse.java b/src/main/java/org/opensearch/knn/plugin/transport/KNNWarmupResponse.java index 842fdf630..f13fbd3db 100644 --- a/src/main/java/org/opensearch/knn/plugin/transport/KNNWarmupResponse.java +++ b/src/main/java/org/opensearch/knn/plugin/transport/KNNWarmupResponse.java @@ -5,9 +5,9 @@ package org.opensearch.knn.plugin.transport; -import org.opensearch.action.support.DefaultShardOperationFailedException; +import org.opensearch.core.action.support.DefaultShardOperationFailedException;; import org.opensearch.action.support.broadcast.BroadcastResponse; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.xcontent.ToXContentObject; import java.io.IOException; diff --git a/src/main/java/org/opensearch/knn/plugin/transport/KNNWarmupTransportAction.java b/src/main/java/org/opensearch/knn/plugin/transport/KNNWarmupTransportAction.java index 79c0315fd..c55c9b619 100644 --- a/src/main/java/org/opensearch/knn/plugin/transport/KNNWarmupTransportAction.java +++ b/src/main/java/org/opensearch/knn/plugin/transport/KNNWarmupTransportAction.java @@ -9,7 +9,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.opensearch.action.support.ActionFilters; -import org.opensearch.action.support.DefaultShardOperationFailedException; +import org.opensearch.core.action.support.DefaultShardOperationFailedException;; import org.opensearch.action.support.broadcast.node.TransportBroadcastByNodeAction; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.block.ClusterBlockException; @@ -19,7 +19,7 @@ import org.opensearch.cluster.routing.ShardsIterator; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.indices.IndicesService; import org.opensearch.transport.TransportService; import org.opensearch.threadpool.ThreadPool; diff --git a/src/main/java/org/opensearch/knn/plugin/transport/RemoveModelFromCacheAction.java b/src/main/java/org/opensearch/knn/plugin/transport/RemoveModelFromCacheAction.java index c2ce43069..6a0e61684 100644 --- a/src/main/java/org/opensearch/knn/plugin/transport/RemoveModelFromCacheAction.java +++ b/src/main/java/org/opensearch/knn/plugin/transport/RemoveModelFromCacheAction.java @@ -12,7 +12,7 @@ package org.opensearch.knn.plugin.transport; import org.opensearch.action.ActionType; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.Writeable; /** * Action used to remove a model from the cache on some or all nodes diff --git a/src/main/java/org/opensearch/knn/plugin/transport/RemoveModelFromCacheNodeRequest.java b/src/main/java/org/opensearch/knn/plugin/transport/RemoveModelFromCacheNodeRequest.java index 09e7e1148..c13a26707 100644 --- a/src/main/java/org/opensearch/knn/plugin/transport/RemoveModelFromCacheNodeRequest.java +++ b/src/main/java/org/opensearch/knn/plugin/transport/RemoveModelFromCacheNodeRequest.java @@ -12,8 +12,8 @@ package org.opensearch.knn.plugin.transport; import org.opensearch.action.support.nodes.BaseNodeRequest; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import java.io.IOException; diff --git a/src/main/java/org/opensearch/knn/plugin/transport/RemoveModelFromCacheNodeResponse.java b/src/main/java/org/opensearch/knn/plugin/transport/RemoveModelFromCacheNodeResponse.java index 7e1be85d0..d8bf28d40 100644 --- a/src/main/java/org/opensearch/knn/plugin/transport/RemoveModelFromCacheNodeResponse.java +++ b/src/main/java/org/opensearch/knn/plugin/transport/RemoveModelFromCacheNodeResponse.java @@ -13,7 +13,7 @@ import org.opensearch.action.support.nodes.BaseNodeResponse; import org.opensearch.cluster.node.DiscoveryNode; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import java.io.IOException; diff --git a/src/main/java/org/opensearch/knn/plugin/transport/RemoveModelFromCacheRequest.java b/src/main/java/org/opensearch/knn/plugin/transport/RemoveModelFromCacheRequest.java index 5da60c809..3a31f7ada 100644 --- a/src/main/java/org/opensearch/knn/plugin/transport/RemoveModelFromCacheRequest.java +++ b/src/main/java/org/opensearch/knn/plugin/transport/RemoveModelFromCacheRequest.java @@ -12,8 +12,8 @@ package org.opensearch.knn.plugin.transport; import org.opensearch.action.support.nodes.BaseNodesRequest; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import java.io.IOException; diff --git a/src/main/java/org/opensearch/knn/plugin/transport/RemoveModelFromCacheResponse.java b/src/main/java/org/opensearch/knn/plugin/transport/RemoveModelFromCacheResponse.java index 3293573ce..74a95bf69 100644 --- a/src/main/java/org/opensearch/knn/plugin/transport/RemoveModelFromCacheResponse.java +++ b/src/main/java/org/opensearch/knn/plugin/transport/RemoveModelFromCacheResponse.java @@ -14,8 +14,8 @@ import org.opensearch.action.FailedNodeException; import org.opensearch.action.support.nodes.BaseNodesResponse; import org.opensearch.cluster.ClusterName; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import java.io.IOException; import java.util.List; diff --git a/src/main/java/org/opensearch/knn/plugin/transport/RemoveModelFromCacheTransportAction.java b/src/main/java/org/opensearch/knn/plugin/transport/RemoveModelFromCacheTransportAction.java index 92938ed3c..042b870ea 100644 --- a/src/main/java/org/opensearch/knn/plugin/transport/RemoveModelFromCacheTransportAction.java +++ b/src/main/java/org/opensearch/knn/plugin/transport/RemoveModelFromCacheTransportAction.java @@ -18,7 +18,7 @@ import org.opensearch.action.support.nodes.TransportNodesAction; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.knn.indices.ModelCache; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportService; diff --git a/src/main/java/org/opensearch/knn/plugin/transport/SearchModelAction.java b/src/main/java/org/opensearch/knn/plugin/transport/SearchModelAction.java index 219e6d1e3..ff217fdbd 100644 --- a/src/main/java/org/opensearch/knn/plugin/transport/SearchModelAction.java +++ b/src/main/java/org/opensearch/knn/plugin/transport/SearchModelAction.java @@ -13,7 +13,7 @@ import org.opensearch.action.ActionType; import org.opensearch.action.search.SearchResponse; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.Writeable; /** * GetModelAction class diff --git a/src/main/java/org/opensearch/knn/plugin/transport/TrainingJobRouteDecisionInfoAction.java b/src/main/java/org/opensearch/knn/plugin/transport/TrainingJobRouteDecisionInfoAction.java index 4336cbf62..03d62bfb7 100644 --- a/src/main/java/org/opensearch/knn/plugin/transport/TrainingJobRouteDecisionInfoAction.java +++ b/src/main/java/org/opensearch/knn/plugin/transport/TrainingJobRouteDecisionInfoAction.java @@ -12,7 +12,7 @@ package org.opensearch.knn.plugin.transport; import org.opensearch.action.ActionType; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.Writeable; /** * Action used to collect information from each node to determine which node would be best to route a particular diff --git a/src/main/java/org/opensearch/knn/plugin/transport/TrainingJobRouteDecisionInfoNodeRequest.java b/src/main/java/org/opensearch/knn/plugin/transport/TrainingJobRouteDecisionInfoNodeRequest.java index 7bca7e563..051ec0278 100644 --- a/src/main/java/org/opensearch/knn/plugin/transport/TrainingJobRouteDecisionInfoNodeRequest.java +++ b/src/main/java/org/opensearch/knn/plugin/transport/TrainingJobRouteDecisionInfoNodeRequest.java @@ -12,7 +12,7 @@ package org.opensearch.knn.plugin.transport; import org.opensearch.action.support.nodes.BaseNodeRequest; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import java.io.IOException; diff --git a/src/main/java/org/opensearch/knn/plugin/transport/TrainingJobRouteDecisionInfoNodeResponse.java b/src/main/java/org/opensearch/knn/plugin/transport/TrainingJobRouteDecisionInfoNodeResponse.java index 0947f7431..2469528f2 100644 --- a/src/main/java/org/opensearch/knn/plugin/transport/TrainingJobRouteDecisionInfoNodeResponse.java +++ b/src/main/java/org/opensearch/knn/plugin/transport/TrainingJobRouteDecisionInfoNodeResponse.java @@ -13,8 +13,8 @@ import org.opensearch.action.support.nodes.BaseNodeResponse; import org.opensearch.cluster.node.DiscoveryNode; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/src/main/java/org/opensearch/knn/plugin/transport/TrainingJobRouteDecisionInfoRequest.java b/src/main/java/org/opensearch/knn/plugin/transport/TrainingJobRouteDecisionInfoRequest.java index 23a0691b8..316168a1c 100644 --- a/src/main/java/org/opensearch/knn/plugin/transport/TrainingJobRouteDecisionInfoRequest.java +++ b/src/main/java/org/opensearch/knn/plugin/transport/TrainingJobRouteDecisionInfoRequest.java @@ -12,7 +12,7 @@ package org.opensearch.knn.plugin.transport; import org.opensearch.action.support.nodes.BaseNodesRequest; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import java.io.IOException; diff --git a/src/main/java/org/opensearch/knn/plugin/transport/TrainingJobRouteDecisionInfoResponse.java b/src/main/java/org/opensearch/knn/plugin/transport/TrainingJobRouteDecisionInfoResponse.java index 906ffcab8..71812409c 100644 --- a/src/main/java/org/opensearch/knn/plugin/transport/TrainingJobRouteDecisionInfoResponse.java +++ b/src/main/java/org/opensearch/knn/plugin/transport/TrainingJobRouteDecisionInfoResponse.java @@ -15,8 +15,8 @@ import org.opensearch.action.support.nodes.BaseNodesResponse; import org.opensearch.cluster.ClusterName; import org.opensearch.cluster.node.DiscoveryNode; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/src/main/java/org/opensearch/knn/plugin/transport/TrainingJobRouteDecisionInfoTransportAction.java b/src/main/java/org/opensearch/knn/plugin/transport/TrainingJobRouteDecisionInfoTransportAction.java index 407036c86..cdee47927 100644 --- a/src/main/java/org/opensearch/knn/plugin/transport/TrainingJobRouteDecisionInfoTransportAction.java +++ b/src/main/java/org/opensearch/knn/plugin/transport/TrainingJobRouteDecisionInfoTransportAction.java @@ -16,7 +16,7 @@ import org.opensearch.action.support.nodes.TransportNodesAction; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.knn.training.TrainingJobRunner; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportService; diff --git a/src/main/java/org/opensearch/knn/plugin/transport/TrainingJobRouterAction.java b/src/main/java/org/opensearch/knn/plugin/transport/TrainingJobRouterAction.java index d6a263a4b..ad340a582 100644 --- a/src/main/java/org/opensearch/knn/plugin/transport/TrainingJobRouterAction.java +++ b/src/main/java/org/opensearch/knn/plugin/transport/TrainingJobRouterAction.java @@ -12,7 +12,7 @@ package org.opensearch.knn.plugin.transport; import org.opensearch.action.ActionType; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.Writeable; /** * Action to route training request to a particular node in the cluster diff --git a/src/main/java/org/opensearch/knn/plugin/transport/TrainingModelAction.java b/src/main/java/org/opensearch/knn/plugin/transport/TrainingModelAction.java index b82107407..19f1fa366 100644 --- a/src/main/java/org/opensearch/knn/plugin/transport/TrainingModelAction.java +++ b/src/main/java/org/opensearch/knn/plugin/transport/TrainingModelAction.java @@ -12,7 +12,7 @@ package org.opensearch.knn.plugin.transport; import org.opensearch.action.ActionType; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.Writeable; public class TrainingModelAction extends ActionType { diff --git a/src/main/java/org/opensearch/knn/plugin/transport/TrainingModelRequest.java b/src/main/java/org/opensearch/knn/plugin/transport/TrainingModelRequest.java index 9b7066c81..9035a8e84 100644 --- a/src/main/java/org/opensearch/knn/plugin/transport/TrainingModelRequest.java +++ b/src/main/java/org/opensearch/knn/plugin/transport/TrainingModelRequest.java @@ -16,8 +16,8 @@ import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.ValidationException; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.knn.common.KNNConstants; import org.opensearch.knn.index.IndexUtil; import org.opensearch.knn.index.KNNMethodContext; diff --git a/src/main/java/org/opensearch/knn/plugin/transport/TrainingModelResponse.java b/src/main/java/org/opensearch/knn/plugin/transport/TrainingModelResponse.java index ccbd718b3..2886c30ae 100644 --- a/src/main/java/org/opensearch/knn/plugin/transport/TrainingModelResponse.java +++ b/src/main/java/org/opensearch/knn/plugin/transport/TrainingModelResponse.java @@ -12,8 +12,8 @@ package org.opensearch.knn.plugin.transport; import org.opensearch.action.ActionResponse; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.knn.common.KNNConstants; diff --git a/src/main/java/org/opensearch/knn/plugin/transport/UpdateModelGraveyardAction.java b/src/main/java/org/opensearch/knn/plugin/transport/UpdateModelGraveyardAction.java index a9897f711..216efa78e 100644 --- a/src/main/java/org/opensearch/knn/plugin/transport/UpdateModelGraveyardAction.java +++ b/src/main/java/org/opensearch/knn/plugin/transport/UpdateModelGraveyardAction.java @@ -7,7 +7,7 @@ import org.opensearch.action.ActionType; import org.opensearch.action.support.master.AcknowledgedResponse; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.Writeable; /** * Action to update model graveyard diff --git a/src/main/java/org/opensearch/knn/plugin/transport/UpdateModelGraveyardRequest.java b/src/main/java/org/opensearch/knn/plugin/transport/UpdateModelGraveyardRequest.java index f8ca38507..887f5d7a2 100644 --- a/src/main/java/org/opensearch/knn/plugin/transport/UpdateModelGraveyardRequest.java +++ b/src/main/java/org/opensearch/knn/plugin/transport/UpdateModelGraveyardRequest.java @@ -8,8 +8,8 @@ import lombok.Getter; import org.opensearch.action.ActionRequestValidationException; import org.opensearch.action.support.master.AcknowledgedRequest; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import java.io.IOException; diff --git a/src/main/java/org/opensearch/knn/plugin/transport/UpdateModelGraveyardTransportAction.java b/src/main/java/org/opensearch/knn/plugin/transport/UpdateModelGraveyardTransportAction.java index a7b5dc876..99aa1b23a 100644 --- a/src/main/java/org/opensearch/knn/plugin/transport/UpdateModelGraveyardTransportAction.java +++ b/src/main/java/org/opensearch/knn/plugin/transport/UpdateModelGraveyardTransportAction.java @@ -21,7 +21,7 @@ import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.Priority; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.knn.indices.ModelGraveyard; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportService; diff --git a/src/main/java/org/opensearch/knn/plugin/transport/UpdateModelMetadataAction.java b/src/main/java/org/opensearch/knn/plugin/transport/UpdateModelMetadataAction.java index 905e05aa0..756a32575 100644 --- a/src/main/java/org/opensearch/knn/plugin/transport/UpdateModelMetadataAction.java +++ b/src/main/java/org/opensearch/knn/plugin/transport/UpdateModelMetadataAction.java @@ -13,7 +13,7 @@ import org.opensearch.action.ActionType; import org.opensearch.action.support.master.AcknowledgedResponse; -import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.common.io.stream.Writeable; /** * Action to update model metadata. diff --git a/src/main/java/org/opensearch/knn/plugin/transport/UpdateModelMetadataRequest.java b/src/main/java/org/opensearch/knn/plugin/transport/UpdateModelMetadataRequest.java index d0628a519..af063ad27 100644 --- a/src/main/java/org/opensearch/knn/plugin/transport/UpdateModelMetadataRequest.java +++ b/src/main/java/org/opensearch/knn/plugin/transport/UpdateModelMetadataRequest.java @@ -13,8 +13,8 @@ import org.opensearch.action.ActionRequestValidationException; import org.opensearch.action.support.master.AcknowledgedRequest; -import org.opensearch.common.io.stream.StreamInput; -import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.knn.indices.ModelMetadata; import java.io.IOException; diff --git a/src/main/java/org/opensearch/knn/plugin/transport/UpdateModelMetadataTransportAction.java b/src/main/java/org/opensearch/knn/plugin/transport/UpdateModelMetadataTransportAction.java index 2cfe04123..8c95c2db8 100644 --- a/src/main/java/org/opensearch/knn/plugin/transport/UpdateModelMetadataTransportAction.java +++ b/src/main/java/org/opensearch/knn/plugin/transport/UpdateModelMetadataTransportAction.java @@ -28,7 +28,7 @@ import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.Priority; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.knn.indices.ModelMetadata; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportService; diff --git a/src/test/java/org/opensearch/knn/KNNSingleNodeTestCase.java b/src/test/java/org/opensearch/knn/KNNSingleNodeTestCase.java index 48effbb39..792ebde69 100644 --- a/src/test/java/org/opensearch/knn/KNNSingleNodeTestCase.java +++ b/src/test/java/org/opensearch/knn/KNNSingleNodeTestCase.java @@ -5,7 +5,7 @@ package org.opensearch.knn; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.xcontent.XContentHelper; import org.opensearch.knn.index.query.KNNQueryBuilder; import org.opensearch.knn.index.memory.NativeMemoryCacheManager; @@ -25,7 +25,7 @@ import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.index.IndexService; import org.opensearch.plugins.Plugin; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.test.OpenSearchSingleNodeTestCase; import org.opensearch.test.hamcrest.OpenSearchAssertions; diff --git a/src/test/java/org/opensearch/knn/KNNTestCase.java b/src/test/java/org/opensearch/knn/KNNTestCase.java index ed1dd0e4e..1b0dbefef 100644 --- a/src/test/java/org/opensearch/knn/KNNTestCase.java +++ b/src/test/java/org/opensearch/knn/KNNTestCase.java @@ -14,7 +14,7 @@ import org.opensearch.knn.index.KNNSettings; import org.opensearch.knn.index.memory.NativeMemoryCacheManager; import org.opensearch.knn.plugin.stats.KNNCounter; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentHelper; import org.opensearch.test.OpenSearchTestCase; diff --git a/src/test/java/org/opensearch/knn/index/KNNESSettingsTestIT.java b/src/test/java/org/opensearch/knn/index/KNNESSettingsTestIT.java index 7f6afba05..09acc2599 100644 --- a/src/test/java/org/opensearch/knn/index/KNNESSettingsTestIT.java +++ b/src/test/java/org/opensearch/knn/index/KNNESSettingsTestIT.java @@ -12,7 +12,7 @@ import org.opensearch.client.Response; import org.opensearch.client.ResponseException; import org.opensearch.common.settings.Settings; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import java.io.IOException; import java.util.Collections; diff --git a/src/test/java/org/opensearch/knn/index/OpenSearchIT.java b/src/test/java/org/opensearch/knn/index/OpenSearchIT.java index 6862a01ac..ea4e4eb5f 100644 --- a/src/test/java/org/opensearch/knn/index/OpenSearchIT.java +++ b/src/test/java/org/opensearch/knn/index/OpenSearchIT.java @@ -30,7 +30,7 @@ import org.opensearch.knn.index.query.KNNQueryBuilder; import org.opensearch.knn.index.util.KNNEngine; import org.opensearch.knn.plugin.script.KNNScoringUtil; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import java.io.IOException; import java.net.URL; diff --git a/src/test/java/org/opensearch/knn/index/VectorDataTypeIT.java b/src/test/java/org/opensearch/knn/index/VectorDataTypeIT.java index 14ce819f8..5f4c2070c 100644 --- a/src/test/java/org/opensearch/knn/index/VectorDataTypeIT.java +++ b/src/test/java/org/opensearch/knn/index/VectorDataTypeIT.java @@ -22,7 +22,7 @@ import org.opensearch.knn.common.KNNConstants; import org.opensearch.knn.index.query.KNNQueryBuilder; import org.opensearch.knn.index.util.KNNEngine; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.script.Script; import java.util.Collections; diff --git a/src/test/java/org/opensearch/knn/index/codec/KNNCodecServiceTests.java b/src/test/java/org/opensearch/knn/index/codec/KNNCodecServiceTests.java index 2599200e6..233b9adf7 100644 --- a/src/test/java/org/opensearch/knn/index/codec/KNNCodecServiceTests.java +++ b/src/test/java/org/opensearch/knn/index/codec/KNNCodecServiceTests.java @@ -8,7 +8,7 @@ import org.apache.lucene.codecs.Codec; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.settings.Settings; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.index.IndexSettings; import org.opensearch.index.codec.CodecServiceConfig; import org.opensearch.index.mapper.MapperService; diff --git a/src/test/java/org/opensearch/knn/index/query/KNNQueryBuilderTests.java b/src/test/java/org/opensearch/knn/index/query/KNNQueryBuilderTests.java index e97cee611..236cbd644 100644 --- a/src/test/java/org/opensearch/knn/index/query/KNNQueryBuilderTests.java +++ b/src/test/java/org/opensearch/knn/index/query/KNNQueryBuilderTests.java @@ -12,9 +12,9 @@ import org.opensearch.cluster.ClusterModule; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.io.stream.NamedWriteableAwareStreamInput; -import org.opensearch.common.io.stream.NamedWriteableRegistry; -import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.NamedWriteableAwareStreamInput; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.index.query.QueryBuilder; import org.opensearch.index.query.QueryBuilders; @@ -23,7 +23,7 @@ import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.XContentParser; -import org.opensearch.index.Index; +import org.opensearch.core.index.Index; import org.opensearch.index.mapper.NumberFieldMapper; import org.opensearch.index.query.QueryShardContext; import org.opensearch.knn.index.KNNClusterUtil; diff --git a/src/test/java/org/opensearch/knn/indices/ModelDaoTests.java b/src/test/java/org/opensearch/knn/indices/ModelDaoTests.java index 5122d6998..082757eb5 100644 --- a/src/test/java/org/opensearch/knn/indices/ModelDaoTests.java +++ b/src/test/java/org/opensearch/knn/indices/ModelDaoTests.java @@ -45,7 +45,7 @@ import org.opensearch.knn.plugin.transport.UpdateModelMetadataRequest; import org.opensearch.knn.plugin.transport.UpdateModelGraveyardAction; import org.opensearch.knn.plugin.transport.UpdateModelGraveyardRequest; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import java.io.IOException; import java.time.ZoneOffset; diff --git a/src/test/java/org/opensearch/knn/plugin/action/RestDeleteModelHandlerIT.java b/src/test/java/org/opensearch/knn/plugin/action/RestDeleteModelHandlerIT.java index edc13c83a..21459612c 100644 --- a/src/test/java/org/opensearch/knn/plugin/action/RestDeleteModelHandlerIT.java +++ b/src/test/java/org/opensearch/knn/plugin/action/RestDeleteModelHandlerIT.java @@ -20,7 +20,7 @@ import org.opensearch.common.xcontent.XContentType; import org.opensearch.knn.KNNRestTestCase; import org.opensearch.knn.plugin.KNNPlugin; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import java.util.List; import java.util.Map; diff --git a/src/test/java/org/opensearch/knn/plugin/action/RestGetModelHandlerIT.java b/src/test/java/org/opensearch/knn/plugin/action/RestGetModelHandlerIT.java index 092ca31e3..ea893021a 100644 --- a/src/test/java/org/opensearch/knn/plugin/action/RestGetModelHandlerIT.java +++ b/src/test/java/org/opensearch/knn/plugin/action/RestGetModelHandlerIT.java @@ -19,7 +19,7 @@ import org.opensearch.common.xcontent.XContentType; import org.opensearch.knn.KNNRestTestCase; import org.opensearch.knn.plugin.KNNPlugin; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import java.io.IOException; import java.util.Arrays; diff --git a/src/test/java/org/opensearch/knn/plugin/action/RestKNNStatsHandlerIT.java b/src/test/java/org/opensearch/knn/plugin/action/RestKNNStatsHandlerIT.java index 6ec699d87..f44aaa78b 100644 --- a/src/test/java/org/opensearch/knn/plugin/action/RestKNNStatsHandlerIT.java +++ b/src/test/java/org/opensearch/knn/plugin/action/RestKNNStatsHandlerIT.java @@ -28,7 +28,7 @@ import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.plugin.stats.KNNStats; import org.opensearch.knn.plugin.stats.StatNames; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import java.io.IOException; import java.util.Arrays; diff --git a/src/test/java/org/opensearch/knn/plugin/action/RestLegacyKNNStatsHandlerIT.java b/src/test/java/org/opensearch/knn/plugin/action/RestLegacyKNNStatsHandlerIT.java index 0d900cfbe..90dd12947 100644 --- a/src/test/java/org/opensearch/knn/plugin/action/RestLegacyKNNStatsHandlerIT.java +++ b/src/test/java/org/opensearch/knn/plugin/action/RestLegacyKNNStatsHandlerIT.java @@ -29,7 +29,7 @@ import org.opensearch.common.unit.TimeValue; import org.opensearch.index.query.MatchAllQueryBuilder; import org.opensearch.index.query.QueryBuilder; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import java.io.IOException; import java.util.Arrays; diff --git a/src/test/java/org/opensearch/knn/plugin/action/RestSearchModelHandlerIT.java b/src/test/java/org/opensearch/knn/plugin/action/RestSearchModelHandlerIT.java index f5a62f838..64960cedc 100644 --- a/src/test/java/org/opensearch/knn/plugin/action/RestSearchModelHandlerIT.java +++ b/src/test/java/org/opensearch/knn/plugin/action/RestSearchModelHandlerIT.java @@ -25,7 +25,7 @@ import org.opensearch.knn.indices.ModelMetadata; import org.opensearch.knn.indices.ModelState; import org.opensearch.knn.plugin.KNNPlugin; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.search.SearchHit; import java.io.IOException; diff --git a/src/test/java/org/opensearch/knn/plugin/action/RestTrainModelHandlerIT.java b/src/test/java/org/opensearch/knn/plugin/action/RestTrainModelHandlerIT.java index 7e82c7ce5..4b996be28 100644 --- a/src/test/java/org/opensearch/knn/plugin/action/RestTrainModelHandlerIT.java +++ b/src/test/java/org/opensearch/knn/plugin/action/RestTrainModelHandlerIT.java @@ -17,7 +17,7 @@ import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentType; import org.opensearch.knn.KNNRestTestCase; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import java.io.IOException; import java.util.Map; diff --git a/src/test/java/org/opensearch/knn/plugin/script/KNNScriptScoringIT.java b/src/test/java/org/opensearch/knn/plugin/script/KNNScriptScoringIT.java index b2bd160f5..bfafd889d 100644 --- a/src/test/java/org/opensearch/knn/plugin/script/KNNScriptScoringIT.java +++ b/src/test/java/org/opensearch/knn/plugin/script/KNNScriptScoringIT.java @@ -21,7 +21,7 @@ import org.opensearch.index.query.MatchAllQueryBuilder; import org.opensearch.index.query.QueryBuilder; import org.opensearch.index.query.functionscore.ScriptScoreQueryBuilder; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.script.Script; import java.util.ArrayList; diff --git a/src/test/java/org/opensearch/knn/plugin/script/PainlessScriptIT.java b/src/test/java/org/opensearch/knn/plugin/script/PainlessScriptIT.java index b68d8211c..def52c478 100644 --- a/src/test/java/org/opensearch/knn/plugin/script/PainlessScriptIT.java +++ b/src/test/java/org/opensearch/knn/plugin/script/PainlessScriptIT.java @@ -23,7 +23,7 @@ import org.opensearch.index.query.MatchAllQueryBuilder; import org.opensearch.index.query.QueryBuilder; import org.opensearch.knn.index.util.KNNEngine; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.script.Script; import java.io.IOException; diff --git a/src/test/java/org/opensearch/knn/plugin/transport/KNNWarmupTransportActionTests.java b/src/test/java/org/opensearch/knn/plugin/transport/KNNWarmupTransportActionTests.java index da4c8d834..1f72f78a7 100644 --- a/src/test/java/org/opensearch/knn/plugin/transport/KNNWarmupTransportActionTests.java +++ b/src/test/java/org/opensearch/knn/plugin/transport/KNNWarmupTransportActionTests.java @@ -16,7 +16,7 @@ import org.opensearch.cluster.service.ClusterService; import org.opensearch.index.IndexService; import org.opensearch.knn.index.memory.NativeMemoryCacheManager; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import java.io.IOException; import java.util.EnumSet; diff --git a/src/test/java/org/opensearch/knn/plugin/transport/TrainingJobRouteDecisionInfoTransportActionTests.java b/src/test/java/org/opensearch/knn/plugin/transport/TrainingJobRouteDecisionInfoTransportActionTests.java index 7049bad2d..2eeaab418 100644 --- a/src/test/java/org/opensearch/knn/plugin/transport/TrainingJobRouteDecisionInfoTransportActionTests.java +++ b/src/test/java/org/opensearch/knn/plugin/transport/TrainingJobRouteDecisionInfoTransportActionTests.java @@ -15,7 +15,7 @@ import org.junit.Before; import org.opensearch.action.ActionListener; import org.opensearch.action.index.IndexResponse; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.knn.KNNSingleNodeTestCase; import org.opensearch.knn.indices.Model; import org.opensearch.knn.indices.ModelDao; diff --git a/src/test/java/org/opensearch/knn/training/TrainingJobRunnerTests.java b/src/test/java/org/opensearch/knn/training/TrainingJobRunnerTests.java index 8b56518a8..3edff82c6 100644 --- a/src/test/java/org/opensearch/knn/training/TrainingJobRunnerTests.java +++ b/src/test/java/org/opensearch/knn/training/TrainingJobRunnerTests.java @@ -13,7 +13,7 @@ import org.opensearch.action.ActionListener; import org.opensearch.action.index.IndexResponse; -import org.opensearch.index.shard.ShardId; +import org.opensearch.core.index.shard.ShardId; import org.opensearch.knn.KNNTestCase; import org.opensearch.knn.indices.Model; import org.opensearch.knn.indices.ModelDao; diff --git a/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java b/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java index 244667a77..b6634a612 100644 --- a/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java +++ b/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java @@ -10,7 +10,7 @@ import com.google.common.primitives.Floats; import org.apache.commons.lang.StringUtils; import org.apache.http.util.EntityUtils; -import org.opensearch.common.bytes.BytesReference; +import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.xcontent.XContentHelper; import org.opensearch.core.xcontent.DeprecationHandler; import org.opensearch.core.xcontent.NamedXContentRegistry; @@ -37,7 +37,7 @@ import org.opensearch.index.query.ExistsQueryBuilder; import org.opensearch.index.query.QueryBuilder; import org.opensearch.index.query.functionscore.ScriptScoreQueryBuilder; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.script.Script; import org.opensearch.search.aggregations.metrics.ScriptedMetricAggregationBuilder; diff --git a/src/testFixtures/java/org/opensearch/knn/ODFERestTestCase.java b/src/testFixtures/java/org/opensearch/knn/ODFERestTestCase.java index 097fe014d..9e35644fa 100644 --- a/src/testFixtures/java/org/opensearch/knn/ODFERestTestCase.java +++ b/src/testFixtures/java/org/opensearch/knn/ODFERestTestCase.java @@ -34,7 +34,7 @@ import org.opensearch.common.xcontent.XContentType; import org.opensearch.commons.rest.SecureRestClientBuilder; import org.opensearch.knn.plugin.KNNPlugin; -import org.opensearch.rest.RestStatus; +import org.opensearch.core.rest.RestStatus; import org.opensearch.search.SearchHit; import org.opensearch.test.rest.OpenSearchRestTestCase; From 85c5a3a670c89144a1e0148c9670a78933587e6c Mon Sep 17 00:00:00 2001 From: Navneet Verma Date: Mon, 31 Jul 2023 16:31:18 -0700 Subject: [PATCH 126/416] Enabled the IVF algorithm to work with Filters of K-NN Query. (#1013) Signed-off-by: Navneet Verma --- CHANGELOG.md | 1 + .../opensearch/knn/index/query/KNNWeight.java | 20 ++++++++++- .../org/opensearch/knn/index/FaissIT.java | 35 ++++++++++++++++--- .../org/opensearch/knn/KNNRestTestCase.java | 22 ++++++++++++ 4 files changed, 73 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index eb2b04796..ecc1736df 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ## [Unreleased 2.x](https://github.com/opensearch-project/k-NN/compare/2.9...2.x) ### Features ### Enhancements +* Enabled the IVF algorithm to work with Filters of K-NN Query. [#1013](https://github.com/opensearch-project/k-NN/pull/1013) ### Bug Fixes ### Infrastructure ### Documentation diff --git a/src/main/java/org/opensearch/knn/index/query/KNNWeight.java b/src/main/java/org/opensearch/knn/index/query/KNNWeight.java index b8b88b4fe..4bbf61f25 100644 --- a/src/main/java/org/opensearch/knn/index/query/KNNWeight.java +++ b/src/main/java/org/opensearch/knn/index/query/KNNWeight.java @@ -6,6 +6,7 @@ package org.opensearch.knn.index.query; import lombok.extern.log4j.Log4j2; +import org.apache.commons.lang.StringUtils; import org.apache.lucene.index.BinaryDocValues; import org.apache.lucene.index.DocValues; import org.apache.lucene.search.FilteredDocIdSetIterator; @@ -49,6 +50,7 @@ import java.util.Collections; import java.util.HashMap; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.concurrent.ExecutionException; import java.util.stream.Collectors; @@ -290,7 +292,7 @@ private Map doExactSearch(final LeafReaderContext leafReaderCont float[] queryVector = this.knnQuery.getQueryVector(); try { final BinaryDocValues values = DocValues.getBinary(leafReaderContext.reader(), fieldInfo.getName()); - final SpaceType spaceType = SpaceType.getSpace(fieldInfo.getAttribute(SPACE_TYPE)); + final SpaceType spaceType = getSpaceType(fieldInfo); // Creating min heap and init with MAX DocID and Score as -INF. final HitQueue queue = new HitQueue(this.knnQuery.getK(), true); ScoreDoc topDoc = queue.top(); @@ -351,4 +353,20 @@ public static float normalizeScore(float score) { if (score >= 0) return 1 / (1 + score); return -score + 1; } + + private SpaceType getSpaceType(final FieldInfo fieldInfo) { + final String spaceTypeString = fieldInfo.getAttribute(SPACE_TYPE); + if (StringUtils.isNotEmpty(spaceTypeString)) { + return SpaceType.getSpace(spaceTypeString); + } + + final String modelId = fieldInfo.getAttribute(MODEL_ID); + if (StringUtils.isNotEmpty(modelId)) { + ModelMetadata modelMetadata = modelDao.getMetadata(modelId); + return modelMetadata.getSpaceType(); + } + throw new IllegalArgumentException( + String.format(Locale.ROOT, "Unable to find the Space Type from Field Info attribute for field %s", fieldInfo.getName()) + ); + } } diff --git a/src/test/java/org/opensearch/knn/index/FaissIT.java b/src/test/java/org/opensearch/knn/index/FaissIT.java index 8eff19da5..a579fb3fd 100644 --- a/src/test/java/org/opensearch/knn/index/FaissIT.java +++ b/src/test/java/org/opensearch/knn/index/FaissIT.java @@ -221,7 +221,7 @@ public void testDocDeletion() throws IOException { deleteKnnDoc(INDEX_NAME, "1"); } - public void testEndToEnd_fromModel() throws IOException, InterruptedException { + public void testKNNQuery_withModelDifferentCombination_thenSuccess() throws IOException, InterruptedException { String modelId = "test-model"; int dimension = 128; @@ -270,10 +270,9 @@ public void testEndToEnd_fromModel() throws IOException, InterruptedException { // Index some documents int numDocs = 100; for (int i = 0; i < numDocs; i++) { - Float[] indexVector = new Float[dimension]; + float[] indexVector = new float[dimension]; Arrays.fill(indexVector, (float) i); - - addKnnDoc(indexName, Integer.toString(i), fieldName, indexVector); + addKnnDocWithAttributes(indexName, Integer.toString(i), fieldName, indexVector, ImmutableMap.of("rating", String.valueOf(i))); } // Run search and ensure that the values returned are expected @@ -287,6 +286,34 @@ public void testEndToEnd_fromModel() throws IOException, InterruptedException { for (int i = 0; i < k; i++) { assertEquals(numDocs - i - 1, Integer.parseInt(results.get(i).getDocId())); } + + // doing exact search with filters + Response exactSearchFilteredResponse = searchKNNIndex( + indexName, + new KNNQueryBuilder(fieldName, queryVector, k, QueryBuilders.rangeQuery("rating").gte("90").lte("99")), + k + ); + List exactSearchFilteredResults = parseSearchResponse( + EntityUtils.toString(exactSearchFilteredResponse.getEntity()), + fieldName + ); + for (int i = 0; i < k; i++) { + assertEquals(numDocs - i - 1, Integer.parseInt(exactSearchFilteredResults.get(i).getDocId())); + } + + // doing exact search with filters + Response aNNSearchFilteredResponse = searchKNNIndex( + indexName, + new KNNQueryBuilder(fieldName, queryVector, k, QueryBuilders.rangeQuery("rating").gte("80").lte("99")), + k + ); + List aNNSearchFilteredResults = parseSearchResponse( + EntityUtils.toString(aNNSearchFilteredResponse.getEntity()), + fieldName + ); + for (int i = 0; i < k; i++) { + assertEquals(numDocs - i - 1, Integer.parseInt(aNNSearchFilteredResults.get(i).getDocId())); + } } @SneakyThrows diff --git a/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java b/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java index b6634a612..1efde29c1 100644 --- a/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java +++ b/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java @@ -1322,4 +1322,26 @@ protected void addKnnDocWithAttributes(String docId, float[] vector, Map fieldValues + ) throws IOException { + Request request = new Request("POST", "/" + indexName + "/_doc/" + docId + "?refresh=true"); + + final XContentBuilder builder = XContentFactory.jsonBuilder().startObject().field(vectorFieldName, vector); + for (String fieldName : fieldValues.keySet()) { + builder.field(fieldName, fieldValues.get(fieldName)); + } + builder.endObject(); + request.setJsonEntity(Strings.toString(builder)); + client().performRequest(request); + + request = new Request("POST", "/" + indexName + "/_refresh"); + Response response = client().performRequest(request); + assertEquals(request.getEndpoint() + ": failed", RestStatus.OK, RestStatus.fromCode(response.getStatusLine().getStatusCode())); + } } From decc8e2e134b3c1d3b48de2ef0246866ccba1217 Mon Sep 17 00:00:00 2001 From: Martin Gaievski Date: Fri, 4 Aug 2023 00:34:56 +0200 Subject: [PATCH 127/416] Fixing compilation errors from mediatype refactoring changes (#1029) Signed-off-by: Martin Gaievski --- src/main/java/org/opensearch/knn/index/KNNSettings.java | 6 +++--- .../org/opensearch/knn/index/KNNVectorIndexFieldData.java | 2 +- .../knn/index/codec/KNN80Codec/KNN80DocValuesConsumer.java | 3 +-- .../java/org/opensearch/knn/index/KNNSettingsTests.java | 2 +- .../java/org/opensearch/knn/index/query/KNNWeightTests.java | 2 +- .../knn/plugin/transport/DeleteModelResponseTests.java | 4 ++-- .../knn/plugin/transport/GetModelResponseTests.java | 6 +++--- .../TrainingJobRouteDecisionInfoNodeResponseTests.java | 2 +- .../TrainingJobRouteDecisionInfoResponseTests.java | 2 +- .../java/org/opensearch/knn/KNNRestTestCase.java | 3 ++- .../java/org/opensearch/knn/ODFERestTestCase.java | 3 ++- src/testFixtures/java/org/opensearch/knn/TestUtils.java | 3 +-- 12 files changed, 19 insertions(+), 19 deletions(-) diff --git a/src/main/java/org/opensearch/knn/index/KNNSettings.java b/src/main/java/org/opensearch/knn/index/KNNSettings.java index 526a529f1..c47a2b197 100644 --- a/src/main/java/org/opensearch/knn/index/KNNSettings.java +++ b/src/main/java/org/opensearch/knn/index/KNNSettings.java @@ -15,9 +15,9 @@ import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; -import org.opensearch.common.unit.ByteSizeUnit; -import org.opensearch.common.unit.ByteSizeValue; import org.opensearch.common.unit.TimeValue; +import org.opensearch.core.common.unit.ByteSizeUnit; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.index.IndexModule; import org.opensearch.knn.index.memory.NativeMemoryCacheManager; import org.opensearch.knn.index.memory.NativeMemoryCacheManagerDto; @@ -37,8 +37,8 @@ import static org.opensearch.common.settings.Setting.Property.Dynamic; import static org.opensearch.common.settings.Setting.Property.IndexScope; import static org.opensearch.common.settings.Setting.Property.NodeScope; -import static org.opensearch.common.unit.ByteSizeValue.parseBytesSizeValue; import static org.opensearch.common.unit.MemorySizeValue.parseBytesSizeValueOrHeapRatio; +import static org.opensearch.core.common.unit.ByteSizeValue.parseBytesSizeValue; /** * This class defines diff --git a/src/main/java/org/opensearch/knn/index/KNNVectorIndexFieldData.java b/src/main/java/org/opensearch/knn/index/KNNVectorIndexFieldData.java index deef8bae1..745aee977 100644 --- a/src/main/java/org/opensearch/knn/index/KNNVectorIndexFieldData.java +++ b/src/main/java/org/opensearch/knn/index/KNNVectorIndexFieldData.java @@ -8,9 +8,9 @@ import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.search.SortField; import org.opensearch.common.util.BigArrays; +import org.opensearch.core.indices.breaker.CircuitBreakerService; import org.opensearch.index.fielddata.IndexFieldData; import org.opensearch.index.fielddata.IndexFieldDataCache; -import org.opensearch.indices.breaker.CircuitBreakerService; import org.opensearch.search.DocValueFormat; import org.opensearch.search.MultiValueMode; import org.opensearch.search.aggregations.support.ValuesSourceType; diff --git a/src/main/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumer.java b/src/main/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumer.java index 691f789e0..ed423c5c8 100644 --- a/src/main/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumer.java +++ b/src/main/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumer.java @@ -9,7 +9,6 @@ import lombok.NonNull; import lombok.extern.log4j.Log4j2; import org.apache.lucene.store.ChecksumIndexInput; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.xcontent.DeprecationHandler; import org.opensearch.core.xcontent.NamedXContentRegistry; @@ -176,7 +175,7 @@ private void createKNNIndexFromScratch(FieldInfo fieldInfo, KNNCodecUtil.Pair pa parameters.put(PARAMETERS, algoParams); } else { parameters.putAll( - XContentFactory.xContent(XContentType.JSON) + XContentType.JSON.xContent() .createParser(NamedXContentRegistry.EMPTY, DeprecationHandler.THROW_UNSUPPORTED_OPERATION, parametersString) .map() ); diff --git a/src/test/java/org/opensearch/knn/index/KNNSettingsTests.java b/src/test/java/org/opensearch/knn/index/KNNSettingsTests.java index 724c9ad6a..f6a10616b 100644 --- a/src/test/java/org/opensearch/knn/index/KNNSettingsTests.java +++ b/src/test/java/org/opensearch/knn/index/KNNSettingsTests.java @@ -10,7 +10,7 @@ import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.network.NetworkModule; import org.opensearch.common.settings.Settings; -import org.opensearch.common.unit.ByteSizeValue; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.env.Environment; import org.opensearch.knn.KNNTestCase; import org.opensearch.knn.plugin.KNNPlugin; diff --git a/src/test/java/org/opensearch/knn/index/query/KNNWeightTests.java b/src/test/java/org/opensearch/knn/index/query/KNNWeightTests.java index 53d0330f0..20ee69744 100644 --- a/src/test/java/org/opensearch/knn/index/query/KNNWeightTests.java +++ b/src/test/java/org/opensearch/knn/index/query/KNNWeightTests.java @@ -29,8 +29,8 @@ import org.junit.BeforeClass; import org.mockito.MockedStatic; import org.opensearch.common.io.PathUtils; -import org.opensearch.common.unit.ByteSizeValue; import org.opensearch.common.unit.TimeValue; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.knn.KNNTestCase; import org.opensearch.knn.index.KNNSettings; import org.opensearch.knn.index.SpaceType; diff --git a/src/test/java/org/opensearch/knn/plugin/transport/DeleteModelResponseTests.java b/src/test/java/org/opensearch/knn/plugin/transport/DeleteModelResponseTests.java index ed0245c8f..a7eac4637 100644 --- a/src/test/java/org/opensearch/knn/plugin/transport/DeleteModelResponseTests.java +++ b/src/test/java/org/opensearch/knn/plugin/transport/DeleteModelResponseTests.java @@ -13,8 +13,8 @@ import org.opensearch.common.Strings; import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentType; import org.opensearch.knn.KNNTestCase; @@ -39,7 +39,7 @@ public void testXContentWithoutError() throws IOException { BytesStreamOutput streamOutput = new BytesStreamOutput(); deleteModelResponse.writeTo(streamOutput); String expectedResponseString = "{\"model_id\":\"test-model\",\"result\":\"deleted\"}"; - XContentBuilder xContentBuilder = XContentFactory.contentBuilder(XContentType.JSON); + XContentBuilder xContentBuilder = MediaTypeRegistry.contentBuilder(XContentType.JSON); deleteModelResponse.toXContent(xContentBuilder, null); assertEquals(expectedResponseString, Strings.toString(xContentBuilder)); } diff --git a/src/test/java/org/opensearch/knn/plugin/transport/GetModelResponseTests.java b/src/test/java/org/opensearch/knn/plugin/transport/GetModelResponseTests.java index ff69d4b4e..616dfbfd2 100644 --- a/src/test/java/org/opensearch/knn/plugin/transport/GetModelResponseTests.java +++ b/src/test/java/org/opensearch/knn/plugin/transport/GetModelResponseTests.java @@ -13,8 +13,8 @@ import org.opensearch.common.Strings; import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentType; import org.opensearch.knn.KNNTestCase; import org.opensearch.knn.index.SpaceType; @@ -49,7 +49,7 @@ public void testXContent() throws IOException { GetModelResponse getModelResponse = new GetModelResponse(model); String expectedResponseString = "{\"model_id\":\"test-model\",\"model_blob\":\"aGVsbG8=\",\"state\":\"created\",\"timestamp\":\"2021-03-27 10:15:30 AM +05:30\",\"description\":\"test model\",\"error\":\"\",\"space_type\":\"l2\",\"dimension\":4,\"engine\":\"nmslib\"}"; - XContentBuilder xContentBuilder = XContentFactory.contentBuilder(XContentType.JSON); + XContentBuilder xContentBuilder = MediaTypeRegistry.contentBuilder(XContentType.JSON); getModelResponse.toXContent(xContentBuilder, null); assertEquals(expectedResponseString, Strings.toString(xContentBuilder)); } @@ -60,7 +60,7 @@ public void testXContentWithNoModelBlob() throws IOException { GetModelResponse getModelResponse = new GetModelResponse(model); String expectedResponseString = "{\"model_id\":\"test-model\",\"model_blob\":\"\",\"state\":\"failed\",\"timestamp\":\"2021-03-27 10:15:30 AM +05:30\",\"description\":\"test model\",\"error\":\"\",\"space_type\":\"l2\",\"dimension\":4,\"engine\":\"nmslib\"}"; - XContentBuilder xContentBuilder = XContentFactory.contentBuilder(XContentType.JSON); + XContentBuilder xContentBuilder = MediaTypeRegistry.contentBuilder(XContentType.JSON); getModelResponse.toXContent(xContentBuilder, null); assertEquals(expectedResponseString, Strings.toString(xContentBuilder)); } diff --git a/src/test/java/org/opensearch/knn/plugin/transport/TrainingJobRouteDecisionInfoNodeResponseTests.java b/src/test/java/org/opensearch/knn/plugin/transport/TrainingJobRouteDecisionInfoNodeResponseTests.java index 75deb4639..04f990dc9 100644 --- a/src/test/java/org/opensearch/knn/plugin/transport/TrainingJobRouteDecisionInfoNodeResponseTests.java +++ b/src/test/java/org/opensearch/knn/plugin/transport/TrainingJobRouteDecisionInfoNodeResponseTests.java @@ -16,7 +16,7 @@ import org.opensearch.Version; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.transport.TransportAddress; +import org.opensearch.core.common.transport.TransportAddress; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; diff --git a/src/test/java/org/opensearch/knn/plugin/transport/TrainingJobRouteDecisionInfoResponseTests.java b/src/test/java/org/opensearch/knn/plugin/transport/TrainingJobRouteDecisionInfoResponseTests.java index 372f86ff8..d828e0604 100644 --- a/src/test/java/org/opensearch/knn/plugin/transport/TrainingJobRouteDecisionInfoResponseTests.java +++ b/src/test/java/org/opensearch/knn/plugin/transport/TrainingJobRouteDecisionInfoResponseTests.java @@ -19,7 +19,7 @@ import org.opensearch.cluster.ClusterName; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.transport.TransportAddress; +import org.opensearch.core.common.transport.TransportAddress; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; diff --git a/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java b/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java index 1efde29c1..b7c742a99 100644 --- a/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java +++ b/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java @@ -13,6 +13,7 @@ import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.xcontent.XContentHelper; import org.opensearch.core.xcontent.DeprecationHandler; +import org.opensearch.core.xcontent.MediaType; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.index.query.MatchAllQueryBuilder; @@ -1274,7 +1275,7 @@ public interface IProxy { protected void refreshAllNonSystemIndices() throws Exception { Response response = adminClient().performRequest(new Request("GET", "/_cat/indices?format=json&expand_wildcards=all")); - XContentType xContentType = XContentType.fromMediaType(response.getEntity().getContentType().getValue()); + MediaType xContentType = MediaType.fromMediaType(response.getEntity().getContentType().getValue()); try ( XContentParser parser = xContentType.xContent() .createParser( diff --git a/src/testFixtures/java/org/opensearch/knn/ODFERestTestCase.java b/src/testFixtures/java/org/opensearch/knn/ODFERestTestCase.java index 9e35644fa..1af14b50e 100644 --- a/src/testFixtures/java/org/opensearch/knn/ODFERestTestCase.java +++ b/src/testFixtures/java/org/opensearch/knn/ODFERestTestCase.java @@ -28,6 +28,7 @@ import org.opensearch.common.util.concurrent.ThreadContext; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.DeprecationHandler; +import org.opensearch.core.xcontent.MediaType; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; @@ -163,7 +164,7 @@ protected boolean preserveIndicesUponCompletion() { @After protected void wipeAllODFEIndices() throws Exception { Response response = adminClient().performRequest(new Request("GET", "/_cat/indices?format=json&expand_wildcards=all")); - XContentType xContentType = XContentType.fromMediaType(response.getEntity().getContentType().getValue()); + MediaType xContentType = MediaType.fromMediaType(response.getEntity().getContentType().getValue()); try ( XContentParser parser = xContentType.xContent() .createParser( diff --git a/src/testFixtures/java/org/opensearch/knn/TestUtils.java b/src/testFixtures/java/org/opensearch/knn/TestUtils.java index 0843176e7..06d38cdf9 100644 --- a/src/testFixtures/java/org/opensearch/knn/TestUtils.java +++ b/src/testFixtures/java/org/opensearch/knn/TestUtils.java @@ -8,7 +8,6 @@ import com.google.common.collect.ImmutableMap; import org.opensearch.core.xcontent.DeprecationHandler; import org.opensearch.core.xcontent.NamedXContentRegistry; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentType; import org.opensearch.knn.index.codec.util.KNNCodecUtil; import java.io.BufferedReader; @@ -252,7 +251,7 @@ private KNNCodecUtil.Pair readIndexData(String path) throws IOException { BufferedReader reader = new BufferedReader(new FileReader(path)); String line = reader.readLine(); while (line != null) { - Map doc = XContentFactory.xContent(XContentType.JSON) + Map doc = XContentType.JSON.xContent() .createParser(NamedXContentRegistry.EMPTY, DeprecationHandler.THROW_UNSUPPORTED_OPERATION, line) .map(); idsList.add((Integer) doc.get("id")); From 038db05e9b93b8312ef30588dd2b3d7ef0ddd4a4 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Mon, 7 Aug 2023 13:26:55 -0400 Subject: [PATCH 128/416] Remove k-NN gcc requirements ceiling and not equals 8.3.1 version (#1027) (#1028) Signed-off-by: Peter Zhu (cherry picked from commit 0d97aba804067d06f4e48ba16fa439bc07345206) Co-authored-by: Peter Zhu --- scripts/build.sh | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/scripts/build.sh b/scripts/build.sh index 930f0b602..6a9adc256 100644 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -105,26 +105,16 @@ if [ "$JAVA_HOME" = "" ]; then echo "SET JAVA_HOME=$JAVA_HOME" fi -# Ensure gcc version is above 4.9.0 for faiss 1.7.4+ compilation +# Ensure gcc version is above 4.9.0 and is not 8.3.1 for faiss 1.7.4+ compilation # https://github.com/opensearch-project/k-NN/issues/975 GCC_VERSION=`gcc --version | head -n 1 | cut -d ' ' -f3` GCC_REQUIRED_VERSION=4.9.0 COMPARE_VERSION=`echo $GCC_REQUIRED_VERSION $GCC_VERSION | tr ' ' '\n' | sort -V | uniq | head -n 1` -if [ "$COMPARE_VERSION" != "$GCC_REQUIRED_VERSION" ]; then - echo "gcc version on this env is older than $GCC_REQUIRED_VERSION, exit 1" +if [ "$COMPARE_VERSION" != "$GCC_REQUIRED_VERSION" ] || [ "$GCC_VERSION" = "8.3.1" ]; then + echo "gcc version on this env is either older than $GCC_REQUIRED_VERSION, or equals 8.3.1, exit 1" exit 1 fi -# Ensure gcc version is below 8.0.0 for faiss 1.7.4+ compilation so it will not crash on arm64 CentOS7 -# https://github.com/opensearch-project/k-NN/issues/975 -GCC_REQUIRED_VERSION_CEILING=8.0.0 -COMPARE_VERSION_CEILING=`echo $GCC_REQUIRED_VERSION_CEILING $GCC_VERSION | tr ' ' '\n' | sort -V | uniq | tail -n 1` -if [ "$COMPARE_VERSION_CEILING" != "$GCC_REQUIRED_VERSION_CEILING" ] && (echo "$OSTYPE" | grep -i linux); then - echo "gcc version on this env is newer than $GCC_REQUIRED_VERSION_CEILING, exit 1" - exit 1 -fi - - # Build k-NN lib and plugin through gradle tasks cd $work_dir # Gradle build is used here to replace gradle assemble due to build will also call cmake and make before generating jars From fe6fdf01a47b94312833e4bfa0b54de5ea1d94cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Romain=20Tarti=C3=A8re?= Date: Wed, 16 Aug 2023 06:43:10 -1000 Subject: [PATCH 129/416] Fixed compilation errors after recent changes in core (#1031) (#1033) Signed-off-by: Martin Gaievski (cherry picked from commit 0a7fc4e3d382c17bcd3537a6d625eec07f23085d) Co-authored-by: Martin Gaievski --- .../org/opensearch/knn/bwc/IndexingIT.java | 86 ++++---- .../java/org/opensearch/knn/bwc/ModelIT.java | 22 +- .../knn/index/mapper/MethodFieldMapper.java | 3 +- .../org/opensearch/knn/index/FaissIT.java | 30 ++- .../index/KNNCreateIndexFromModelTests.java | 22 +- .../opensearch/knn/index/LuceneEngineIT.java | 11 +- .../org/opensearch/knn/index/NmslibIT.java | 204 +++++++++--------- .../opensearch/knn/index/OpenSearchIT.java | 53 +++-- .../knn/index/VectorDataTypeIT.java | 7 +- .../KNN80DocValuesConsumerTests.java | 5 +- .../plugin/action/RestKNNStatsHandlerIT.java | 22 +- .../knn/plugin/script/KNNScriptScoringIT.java | 43 ++-- .../knn/plugin/script/PainlessScriptIT.java | 3 +- .../transport/DeleteModelResponseTests.java | 3 +- .../transport/GetModelResponseTests.java | 5 +- .../lang-painless-1.3.0-SNAPSHOT.zip | Bin 0 -> 1291433 bytes .../org/opensearch/knn/KNNRestTestCase.java | 172 +++++++-------- .../org/opensearch/knn/ODFERestTestCase.java | 3 +- 18 files changed, 328 insertions(+), 366 deletions(-) create mode 100644 src/test/resources/lang-painless/lang-painless-1.3.0-SNAPSHOT.zip diff --git a/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/IndexingIT.java b/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/IndexingIT.java index 7437cf0c1..c8d0cac93 100644 --- a/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/IndexingIT.java +++ b/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/IndexingIT.java @@ -5,7 +5,6 @@ package org.opensearch.knn.bwc; -import org.opensearch.common.Strings; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.knn.index.SpaceType; @@ -96,21 +95,20 @@ public void testKNNIndexCustomMethodFieldMapping() throws Exception { // test null parameters public void testNullParametersOnUpgrade() throws Exception { if (isRunningAgainstOldCluster()) { - String mapping = Strings.toString( - XContentFactory.jsonBuilder() - .startObject() - .startObject(PROPERTIES) - .startObject(TEST_FIELD) - .field(VECTOR_TYPE, KNN_VECTOR) - .field(DIMENSION, String.valueOf(DIMENSIONS)) - .startObject(KNN_METHOD) - .field(NAME, METHOD_HNSW) - .field(PARAMETERS, (String) null) - .endObject() - .endObject() - .endObject() - .endObject() - ); + String mapping = XContentFactory.jsonBuilder() + .startObject() + .startObject(PROPERTIES) + .startObject(TEST_FIELD) + .field(VECTOR_TYPE, KNN_VECTOR) + .field(DIMENSION, String.valueOf(DIMENSIONS)) + .startObject(KNN_METHOD) + .field(NAME, METHOD_HNSW) + .field(PARAMETERS, (String) null) + .endObject() + .endObject() + .endObject() + .endObject() + .toString(); createKnnIndex(testIndex, getKNNDefaultIndexSettings(), mapping); } else { @@ -121,21 +119,20 @@ public void testNullParametersOnUpgrade() throws Exception { // test empty parameters public void testEmptyParametersOnUpgrade() throws Exception { if (isRunningAgainstOldCluster()) { - String mapping = Strings.toString( - XContentFactory.jsonBuilder() - .startObject() - .startObject(PROPERTIES) - .startObject(TEST_FIELD) - .field(VECTOR_TYPE, KNN_VECTOR) - .field(DIMENSION, String.valueOf(DIMENSIONS)) - .startObject(KNN_METHOD) - .field(NAME, METHOD_HNSW) - .field(PARAMETERS, "") - .endObject() - .endObject() - .endObject() - .endObject() - ); + String mapping = XContentFactory.jsonBuilder() + .startObject() + .startObject(PROPERTIES) + .startObject(TEST_FIELD) + .field(VECTOR_TYPE, KNN_VECTOR) + .field(DIMENSION, String.valueOf(DIMENSIONS)) + .startObject(KNN_METHOD) + .field(NAME, METHOD_HNSW) + .field(PARAMETERS, "") + .endObject() + .endObject() + .endObject() + .endObject() + .toString(); createKnnIndex(testIndex, getKNNDefaultIndexSettings(), mapping); } else { @@ -146,20 +143,19 @@ public void testEmptyParametersOnUpgrade() throws Exception { // test no parameters public void testNoParametersOnUpgrade() throws Exception { if (isRunningAgainstOldCluster()) { - String mapping = Strings.toString( - XContentFactory.jsonBuilder() - .startObject() - .startObject(PROPERTIES) - .startObject(TEST_FIELD) - .field(VECTOR_TYPE, KNN_VECTOR) - .field(DIMENSION, String.valueOf(DIMENSIONS)) - .startObject(KNN_METHOD) - .field(NAME, METHOD_HNSW) - .endObject() - .endObject() - .endObject() - .endObject() - ); + String mapping = XContentFactory.jsonBuilder() + .startObject() + .startObject(PROPERTIES) + .startObject(TEST_FIELD) + .field(VECTOR_TYPE, KNN_VECTOR) + .field(DIMENSION, String.valueOf(DIMENSIONS)) + .startObject(KNN_METHOD) + .field(NAME, METHOD_HNSW) + .endObject() + .endObject() + .endObject() + .endObject() + .toString(); createKnnIndex(testIndex, getKNNDefaultIndexSettings(), mapping); } else { diff --git a/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/ModelIT.java b/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/ModelIT.java index 41f26c57e..abf44e016 100644 --- a/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/ModelIT.java +++ b/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/ModelIT.java @@ -11,7 +11,6 @@ import org.opensearch.client.Request; import org.opensearch.client.Response; import org.opensearch.client.ResponseException; -import org.opensearch.common.Strings; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.XContentParser; @@ -245,17 +244,16 @@ public void trainKNNModelDefault(String modelId, String trainingIndexName, Strin // mapping to create index from model public String modelIndexMapping(String fieldName, String modelId) throws IOException { - return Strings.toString( - XContentFactory.jsonBuilder() - .startObject() - .startObject(PROPERTIES) - .startObject(fieldName) - .field(VECTOR_TYPE, KNN_VECTOR) - .field(MODEL_ID, modelId) - .endObject() - .endObject() - .endObject() - ); + return XContentFactory.jsonBuilder() + .startObject() + .startObject(PROPERTIES) + .startObject(fieldName) + .field(VECTOR_TYPE, KNN_VECTOR) + .field(MODEL_ID, modelId) + .endObject() + .endObject() + .endObject() + .toString(); } private ModelMetadata getModelMetadata() { diff --git a/src/main/java/org/opensearch/knn/index/mapper/MethodFieldMapper.java b/src/main/java/org/opensearch/knn/index/mapper/MethodFieldMapper.java index 42c12a5db..1db710bad 100644 --- a/src/main/java/org/opensearch/knn/index/mapper/MethodFieldMapper.java +++ b/src/main/java/org/opensearch/knn/index/mapper/MethodFieldMapper.java @@ -7,7 +7,6 @@ import org.apache.lucene.document.FieldType; import org.opensearch.common.Explicit; -import org.opensearch.common.Strings; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.knn.index.KNNMethodContext; import org.opensearch.knn.index.util.KNNEngine; @@ -50,7 +49,7 @@ public class MethodFieldMapper extends KNNVectorFieldMapper { try { this.fieldType.putAttribute( PARAMETERS, - Strings.toString(XContentFactory.jsonBuilder().map(knnEngine.getMethodAsMap(knnMethodContext))) + XContentFactory.jsonBuilder().map(knnEngine.getMethodAsMap(knnMethodContext)).toString() ); } catch (IOException ioe) { throw new RuntimeException(String.format("Unable to create KNNVectorFieldMapper: %s", ioe)); diff --git a/src/test/java/org/opensearch/knn/index/FaissIT.java b/src/test/java/org/opensearch/knn/index/FaissIT.java index a579fb3fd..cfda93113 100644 --- a/src/test/java/org/opensearch/knn/index/FaissIT.java +++ b/src/test/java/org/opensearch/knn/index/FaissIT.java @@ -21,7 +21,6 @@ import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.index.query.QueryBuilders; import org.opensearch.knn.KNNRestTestCase; -import org.opensearch.common.Strings; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.knn.KNNResult; import org.opensearch.knn.TestUtils; @@ -98,7 +97,7 @@ public void testEndToEnd_fromMethod() throws Exception { .endObject(); Map mappingMap = xContentBuilderToMap(builder); - String mapping = Strings.toString(builder); + String mapping = builder.toString(); createKnnIndex(indexName, mapping); assertEquals(new TreeMap<>(mappingMap), new TreeMap<>(getIndexMappingAsMap(indexName))); @@ -175,7 +174,7 @@ public void testDocUpdate() throws IOException { .endObject() .endObject(); - String mapping = Strings.toString(builder); + String mapping = builder.toString(); createKnnIndex(indexName, mapping); Float[] vector = { 6.0f, 6.0f }; @@ -211,7 +210,7 @@ public void testDocDeletion() throws IOException { .endObject() .endObject(); - String mapping = Strings.toString(builder); + String mapping = builder.toString(); createKnnIndex(indexName, mapping); Float[] vector = { 6.0f, 6.0f }; @@ -253,17 +252,16 @@ public void testKNNQuery_withModelDifferentCombination_thenSuccess() throws IOEx // Create knn index from model String fieldName = "test-field-name"; String indexName = "test-index-name"; - String indexMapping = Strings.toString( - XContentFactory.jsonBuilder() - .startObject() - .startObject("properties") - .startObject(fieldName) - .field("type", "knn_vector") - .field(MODEL_ID, modelId) - .endObject() - .endObject() - .endObject() - ); + String indexMapping = XContentFactory.jsonBuilder() + .startObject() + .startObject("properties") + .startObject(fieldName) + .field("type", "knn_vector") + .field(MODEL_ID, modelId) + .endObject() + .endObject() + .endObject() + .toString(); createKnnIndex(indexName, getKNNDefaultIndexSettings(), indexMapping); @@ -387,7 +385,7 @@ protected void setupKNNIndexForFilterQuery() throws Exception { .endObject() .endObject() .endObject(); - final String mapping = Strings.toString(builder); + final String mapping = builder.toString(); createKnnIndex(INDEX_NAME, mapping); diff --git a/src/test/java/org/opensearch/knn/index/KNNCreateIndexFromModelTests.java b/src/test/java/org/opensearch/knn/index/KNNCreateIndexFromModelTests.java index e0b2c05cf..3a4746158 100644 --- a/src/test/java/org/opensearch/knn/index/KNNCreateIndexFromModelTests.java +++ b/src/test/java/org/opensearch/knn/index/KNNCreateIndexFromModelTests.java @@ -14,7 +14,6 @@ import com.google.common.collect.ImmutableMap; import org.opensearch.action.ActionListener; import org.opensearch.action.admin.indices.create.CreateIndexRequestBuilder; -import org.opensearch.common.Strings; import org.opensearch.common.settings.Settings; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.knn.KNNSingleNodeTestCase; @@ -75,17 +74,16 @@ public void testCreateIndexFromModel() throws IOException, InterruptedException String indexName = "test-index"; String fieldName = "test-field"; - final String mapping = Strings.toString( - XContentFactory.jsonBuilder() - .startObject() - .startObject("properties") - .startObject(fieldName) - .field("type", "knn_vector") - .field("model_id", modelId) - .endObject() - .endObject() - .endObject() - ); + final String mapping = XContentFactory.jsonBuilder() + .startObject() + .startObject("properties") + .startObject(fieldName) + .field("type", "knn_vector") + .field("model_id", modelId) + .endObject() + .endObject() + .endObject() + .toString(); modelDao.put(model, ActionListener.wrap(indexResponse -> { CreateIndexRequestBuilder createIndexRequestBuilder = client().admin() diff --git a/src/test/java/org/opensearch/knn/index/LuceneEngineIT.java b/src/test/java/org/opensearch/knn/index/LuceneEngineIT.java index 0c1f0a451..30ac6922a 100644 --- a/src/test/java/org/opensearch/knn/index/LuceneEngineIT.java +++ b/src/test/java/org/opensearch/knn/index/LuceneEngineIT.java @@ -16,7 +16,6 @@ import org.opensearch.client.Request; import org.opensearch.client.Response; import org.opensearch.client.ResponseException; -import org.opensearch.common.Strings; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.index.query.QueryBuilders; @@ -101,7 +100,7 @@ public void testQuery_innerProduct_notSupported() throws Exception { .endObject() .endObject(); - String mapping = Strings.toString(builder); + String mapping = builder.toString(); createIndex(INDEX_NAME, getKNNDefaultIndexSettings()); @@ -179,7 +178,7 @@ public void testQuery_multipleEngines() throws IOException { .endObject() .endObject() .endObject(); - String mapping = Strings.toString(builder); + String mapping = builder.toString(); createKnnIndex(INDEX_NAME, mapping); for (int i = 0; i < TEST_INDEX_VECTORS.length; i++) { @@ -219,7 +218,7 @@ public void testAddDoc() throws IOException { .endObject(); Map mappingMap = xContentBuilderToMap(builder); - String mapping = Strings.toString(builder); + String mapping = builder.toString(); createKnnIndex(INDEX_NAME, mapping); assertEquals(new TreeMap<>(mappingMap), new TreeMap<>(getIndexMappingAsMap(INDEX_NAME))); @@ -305,7 +304,7 @@ public void testQuery_filterWithNonLuceneEngine() throws Exception { .endObject() .endObject(); - String mapping = Strings.toString(builder); + String mapping = builder.toString(); createKnnIndex(INDEX_NAME, mapping); addKnnDocWithAttributes( @@ -371,7 +370,7 @@ private void createKnnIndexMappingWithLuceneEngine(int dimension, SpaceType spac .endObject() .endObject(); - String mapping = Strings.toString(builder); + String mapping = builder.toString(); createKnnIndex(INDEX_NAME, mapping); } diff --git a/src/test/java/org/opensearch/knn/index/NmslibIT.java b/src/test/java/org/opensearch/knn/index/NmslibIT.java index 75a5378d3..9aae61298 100644 --- a/src/test/java/org/opensearch/knn/index/NmslibIT.java +++ b/src/test/java/org/opensearch/knn/index/NmslibIT.java @@ -19,7 +19,6 @@ import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.knn.KNNRestTestCase; import org.opensearch.client.ResponseException; -import org.opensearch.common.Strings; import org.opensearch.common.settings.Settings; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.knn.KNNResult; @@ -85,7 +84,7 @@ public void testEndToEnd() throws IOException, InterruptedException { .endObject(); Map mappingMap = xContentBuilderToMap(builder); - String mapping = Strings.toString(builder); + String mapping = builder.toString(); createKnnIndex(indexName, mapping); assertEquals(new TreeMap<>(mappingMap), new TreeMap<>(getIndexMappingAsMap(indexName))); @@ -187,25 +186,24 @@ public void testCreateIndexWithValidAlgoParams_mapping() { int efConstruction = 14; int m = 13; - String mapping = Strings.toString( - XContentFactory.jsonBuilder() - .startObject() - .startObject("properties") - .startObject(FIELD_NAME) - .field("type", "knn_vector") - .field("dimension", 2) - .startObject(KNNConstants.KNN_METHOD) - .field(KNNConstants.METHOD_PARAMETER_SPACE_TYPE, spaceType) - .field(KNNConstants.NAME, KNNConstants.METHOD_HNSW) - .startObject(KNNConstants.PARAMETERS) - .field(KNNConstants.METHOD_PARAMETER_EF_CONSTRUCTION, efConstruction) - .field(KNNConstants.METHOD_PARAMETER_M, m) - .endObject() - .endObject() - .endObject() - .endObject() - .endObject() - ); + String mapping = XContentFactory.jsonBuilder() + .startObject() + .startObject("properties") + .startObject(FIELD_NAME) + .field("type", "knn_vector") + .field("dimension", 2) + .startObject(KNNConstants.KNN_METHOD) + .field(KNNConstants.METHOD_PARAMETER_SPACE_TYPE, spaceType) + .field(KNNConstants.NAME, KNNConstants.METHOD_HNSW) + .startObject(KNNConstants.PARAMETERS) + .field(KNNConstants.METHOD_PARAMETER_EF_CONSTRUCTION, efConstruction) + .field(KNNConstants.METHOD_PARAMETER_M, m) + .endObject() + .endObject() + .endObject() + .endObject() + .endObject() + .toString(); createKnnIndex(INDEX_NAME, settings, mapping); @@ -243,25 +241,24 @@ public void testCreateIndexWithValidAlgoParams_mappingAndSettings() { .put("index.knn.algo_param.ef_construction", efConstruction1) .build(); - String mapping = Strings.toString( - XContentFactory.jsonBuilder() - .startObject() - .startObject("properties") - .startObject(FIELD_NAME) - .field("type", "knn_vector") - .field("dimension", 2) - .startObject(KNNConstants.KNN_METHOD) - .field(KNNConstants.METHOD_PARAMETER_SPACE_TYPE, spaceType1) - .field(KNNConstants.NAME, KNNConstants.METHOD_HNSW) - .startObject(KNNConstants.PARAMETERS) - .field(KNNConstants.METHOD_PARAMETER_EF_CONSTRUCTION, efConstruction1) - .field(KNNConstants.METHOD_PARAMETER_M, m1) - .endObject() - .endObject() - .endObject() - .endObject() - .endObject() - ); + String mapping = XContentFactory.jsonBuilder() + .startObject() + .startObject("properties") + .startObject(FIELD_NAME) + .field("type", "knn_vector") + .field("dimension", 2) + .startObject(KNNConstants.KNN_METHOD) + .field(KNNConstants.METHOD_PARAMETER_SPACE_TYPE, spaceType1) + .field(KNNConstants.NAME, KNNConstants.METHOD_HNSW) + .startObject(KNNConstants.PARAMETERS) + .field(KNNConstants.METHOD_PARAMETER_EF_CONSTRUCTION, efConstruction1) + .field(KNNConstants.METHOD_PARAMETER_M, m1) + .endObject() + .endObject() + .endObject() + .endObject() + .endObject() + .toString(); createKnnIndex(INDEX_NAME + "1", settings, mapping); Float[] vector = { 6.0f, 6.0f }; @@ -271,37 +268,36 @@ public void testCreateIndexWithValidAlgoParams_mappingAndSettings() { int efConstruction2 = 114; int m2 = 113; - mapping = Strings.toString( - XContentFactory.jsonBuilder() - .startObject() - .startObject("properties") - .startObject(FIELD_NAME + "1") - .field("type", "knn_vector") - .field("dimension", 2) - .startObject(KNNConstants.KNN_METHOD) - .field(KNNConstants.METHOD_PARAMETER_SPACE_TYPE, spaceType1) - .field(KNNConstants.NAME, KNNConstants.METHOD_HNSW) - .startObject(KNNConstants.PARAMETERS) - .field(KNNConstants.METHOD_PARAMETER_EF_CONSTRUCTION, efConstruction1) - .field(KNNConstants.METHOD_PARAMETER_M, m1) - .endObject() - .endObject() - .endObject() - .startObject(FIELD_NAME + "2") - .field("type", "knn_vector") - .field("dimension", 2) - .startObject(KNNConstants.KNN_METHOD) - .field(KNNConstants.METHOD_PARAMETER_SPACE_TYPE, spaceType2) - .field(KNNConstants.NAME, KNNConstants.METHOD_HNSW) - .startObject(KNNConstants.PARAMETERS) - .field(KNNConstants.METHOD_PARAMETER_EF_CONSTRUCTION, efConstruction2) - .field(KNNConstants.METHOD_PARAMETER_M, m2) - .endObject() - .endObject() - .endObject() - .endObject() - .endObject() - ); + mapping = XContentFactory.jsonBuilder() + .startObject() + .startObject("properties") + .startObject(FIELD_NAME + "1") + .field("type", "knn_vector") + .field("dimension", 2) + .startObject(KNNConstants.KNN_METHOD) + .field(KNNConstants.METHOD_PARAMETER_SPACE_TYPE, spaceType1) + .field(KNNConstants.NAME, KNNConstants.METHOD_HNSW) + .startObject(KNNConstants.PARAMETERS) + .field(KNNConstants.METHOD_PARAMETER_EF_CONSTRUCTION, efConstruction1) + .field(KNNConstants.METHOD_PARAMETER_M, m1) + .endObject() + .endObject() + .endObject() + .startObject(FIELD_NAME + "2") + .field("type", "knn_vector") + .field("dimension", 2) + .startObject(KNNConstants.KNN_METHOD) + .field(KNNConstants.METHOD_PARAMETER_SPACE_TYPE, spaceType2) + .field(KNNConstants.NAME, KNNConstants.METHOD_HNSW) + .startObject(KNNConstants.PARAMETERS) + .field(KNNConstants.METHOD_PARAMETER_EF_CONSTRUCTION, efConstruction2) + .field(KNNConstants.METHOD_PARAMETER_M, m2) + .endObject() + .endObject() + .endObject() + .endObject() + .endObject() + .toString(); createKnnIndex(INDEX_NAME + "2", settings, mapping); addKnnDoc(INDEX_NAME + "2", "1", FIELD_NAME, vector); @@ -330,23 +326,22 @@ public void testInvalidIndexHnswAlgoParams_settings() { public void testInvalidIndexHnswAlgoParams_mapping() throws IOException { Settings settings = Settings.builder().put(getKNNDefaultIndexSettings()).build(); - String mapping = Strings.toString( - XContentFactory.jsonBuilder() - .startObject() - .startObject("properties") - .startObject(FIELD_NAME) - .field("type", "knn_vector") - .field("dimension", 2) - .startObject(KNNConstants.KNN_METHOD) - .field(KNNConstants.NAME, KNNConstants.METHOD_HNSW) - .startObject(KNNConstants.PARAMETERS) - .field(KNNConstants.METHOD_PARAMETER_EF_CONSTRUCTION, "-1") - .endObject() - .endObject() - .endObject() - .endObject() - .endObject() - ); + String mapping = XContentFactory.jsonBuilder() + .startObject() + .startObject("properties") + .startObject(FIELD_NAME) + .field("type", "knn_vector") + .field("dimension", 2) + .startObject(KNNConstants.KNN_METHOD) + .field(KNNConstants.NAME, KNNConstants.METHOD_HNSW) + .startObject(KNNConstants.PARAMETERS) + .field(KNNConstants.METHOD_PARAMETER_EF_CONSTRUCTION, "-1") + .endObject() + .endObject() + .endObject() + .endObject() + .endObject() + .toString(); expectThrows(ResponseException.class, () -> createKnnIndex(INDEX_NAME, settings, mapping)); } @@ -354,23 +349,22 @@ public void testInvalidIndexHnswAlgoParams_mapping() throws IOException { public void testInvalidIndexHnswAlgoParams_mappingAndSettings() throws IOException { Settings settings = Settings.builder().put(getKNNDefaultIndexSettings()).put("index.knn.algo_param.m", "-1").build(); - String mapping = Strings.toString( - XContentFactory.jsonBuilder() - .startObject() - .startObject("properties") - .startObject(FIELD_NAME) - .field("type", "knn_vector") - .field("dimension", 2) - .startObject(KNNConstants.KNN_METHOD) - .field(KNNConstants.NAME, KNNConstants.METHOD_HNSW) - .startObject(KNNConstants.PARAMETERS) - .field(KNNConstants.METHOD_PARAMETER_M, "-1") - .endObject() - .endObject() - .endObject() - .endObject() - .endObject() - ); + String mapping = XContentFactory.jsonBuilder() + .startObject() + .startObject("properties") + .startObject(FIELD_NAME) + .field("type", "knn_vector") + .field("dimension", 2) + .startObject(KNNConstants.KNN_METHOD) + .field(KNNConstants.NAME, KNNConstants.METHOD_HNSW) + .startObject(KNNConstants.PARAMETERS) + .field(KNNConstants.METHOD_PARAMETER_M, "-1") + .endObject() + .endObject() + .endObject() + .endObject() + .endObject() + .toString(); expectThrows(ResponseException.class, () -> createKnnIndex(INDEX_NAME, settings, mapping)); } diff --git a/src/test/java/org/opensearch/knn/index/OpenSearchIT.java b/src/test/java/org/opensearch/knn/index/OpenSearchIT.java index ea4e4eb5f..35d59cbd9 100644 --- a/src/test/java/org/opensearch/knn/index/OpenSearchIT.java +++ b/src/test/java/org/opensearch/knn/index/OpenSearchIT.java @@ -20,7 +20,6 @@ import org.opensearch.client.Request; import org.opensearch.client.Response; import org.opensearch.client.ResponseException; -import org.opensearch.common.Strings; import org.opensearch.common.settings.Settings; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; @@ -108,7 +107,7 @@ public void testEndToEnd() throws IOException, InterruptedException { .endObject(); Map mappingMap = xContentBuilderToMap(builder); - String mapping = Strings.toString(builder); + String mapping = builder.toString(); createKnnIndex(indexName, mapping); assertEquals(new TreeMap<>(mappingMap), new TreeMap<>(getIndexMappingAsMap(indexName))); @@ -267,16 +266,15 @@ public void testIndexingVectorValidation_differentSizes() throws Exception { public void testVectorMappingValidation_noDimension() throws Exception { Settings settings = Settings.builder().put(getKNNDefaultIndexSettings()).build(); - String mapping = Strings.toString( - XContentFactory.jsonBuilder() - .startObject() - .startObject("properties") - .startObject(FIELD_NAME) - .field("type", "knn_vector") - .endObject() - .endObject() - .endObject() - ); + String mapping = XContentFactory.jsonBuilder() + .startObject() + .startObject("properties") + .startObject(FIELD_NAME) + .field("type", "knn_vector") + .endObject() + .endObject() + .endObject() + .toString(); Exception ex = expectThrows(ResponseException.class, () -> createKnnIndex(INDEX_NAME, settings, mapping)); assertThat(ex.getMessage(), containsString("Dimension value missing for vector: " + FIELD_NAME)); @@ -338,21 +336,20 @@ public void testVectorMappingValidation_multiFieldsDifferentDimension() throws E String f4 = FIELD_NAME + "-4"; String f5 = FIELD_NAME + "-5"; - String mapping = Strings.toString( - XContentFactory.jsonBuilder() - .startObject() - .startObject("properties") - .startObject(f4) - .field("type", "knn_vector") - .field("dimension", "4") - .endObject() - .startObject(f5) - .field("type", "knn_vector") - .field("dimension", "5") - .endObject() - .endObject() - .endObject() - ); + String mapping = XContentFactory.jsonBuilder() + .startObject() + .startObject("properties") + .startObject(f4) + .field("type", "knn_vector") + .field("dimension", "4") + .endObject() + .startObject(f5) + .field("type", "knn_vector") + .field("dimension", "5") + .endObject() + .endObject() + .endObject() + .toString(); createKnnIndex(INDEX_NAME, settings, mapping); @@ -384,7 +381,7 @@ public void testExistsQuery() throws IOException { Request request = new Request("POST", "/" + INDEX_NAME + "/_doc/7?refresh=true"); XContentBuilder builder = XContentFactory.jsonBuilder().startObject().field("non-knn-field", "test").endObject(); - request.setJsonEntity(Strings.toString(builder)); + request.setJsonEntity(builder.toString()); Response response = client().performRequest(request); assertEquals(request.getEndpoint() + ": failed", RestStatus.CREATED, RestStatus.fromCode(response.getStatusLine().getStatusCode())); diff --git a/src/test/java/org/opensearch/knn/index/VectorDataTypeIT.java b/src/test/java/org/opensearch/knn/index/VectorDataTypeIT.java index 5f4c2070c..adb222caf 100644 --- a/src/test/java/org/opensearch/knn/index/VectorDataTypeIT.java +++ b/src/test/java/org/opensearch/knn/index/VectorDataTypeIT.java @@ -11,7 +11,6 @@ import org.opensearch.client.Request; import org.opensearch.client.Response; import org.opensearch.client.ResponseException; -import org.opensearch.common.Strings; import org.opensearch.common.settings.Settings; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.XContentBuilder; @@ -274,7 +273,7 @@ public void testByteVectorDataTypeWithLegacyFieldMapperKnnIndexSetting() { .endObject() .endObject(); - String mapping = Strings.toString(builder); + String mapping = builder.toString(); ResponseException ex = expectThrows(ResponseException.class, () -> createKnnIndex(INDEX_NAME, mapping)); assertTrue( @@ -484,7 +483,7 @@ private void createKnnIndexMappingWithCustomEngine(int dimension, SpaceType spac .endObject() .endObject(); - String mapping = Strings.toString(builder); + String mapping = builder.toString(); createKnnIndex(INDEX_NAME, mapping); } @@ -500,7 +499,7 @@ private void createKnnIndexMappingForScripting(int dimension, String vectorDataT .endObject() .endObject(); - String mapping = Strings.toString(builder); + String mapping = builder.toString(); createKnnIndex(INDEX_NAME, Settings.EMPTY, mapping); } diff --git a/src/test/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumerTests.java b/src/test/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumerTests.java index 8fa63a2e3..58f4b6e39 100644 --- a/src/test/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumerTests.java +++ b/src/test/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumerTests.java @@ -19,7 +19,6 @@ import org.junit.AfterClass; import org.junit.BeforeClass; import org.opensearch.cluster.service.ClusterService; -import org.opensearch.common.Strings; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; import org.opensearch.common.xcontent.XContentFactory; @@ -162,7 +161,7 @@ public void testAddKNNBinaryField_fromScratch_nmslibCurrent() throws IOException new MethodComponentContext(METHOD_HNSW, ImmutableMap.of(METHOD_PARAMETER_M, 16, METHOD_PARAMETER_EF_CONSTRUCTION, 512)) ); - String parameterString = Strings.toString(XContentFactory.jsonBuilder().map(knnEngine.getMethodAsMap(knnMethodContext))); + String parameterString = XContentFactory.jsonBuilder().map(knnEngine.getMethodAsMap(knnMethodContext)).toString(); FieldInfo[] fieldInfoArray = new FieldInfo[] { KNNCodecTestUtil.FieldInfoBuilder.builder(fieldName) @@ -257,7 +256,7 @@ public void testAddKNNBinaryField_fromScratch_faissCurrent() throws IOException new MethodComponentContext(METHOD_HNSW, ImmutableMap.of(METHOD_PARAMETER_M, 16, METHOD_PARAMETER_EF_CONSTRUCTION, 512)) ); - String parameterString = Strings.toString(XContentFactory.jsonBuilder().map(knnEngine.getMethodAsMap(knnMethodContext))); + String parameterString = XContentFactory.jsonBuilder().map(knnEngine.getMethodAsMap(knnMethodContext)).toString(); FieldInfo[] fieldInfoArray = new FieldInfo[] { KNNCodecTestUtil.FieldInfoBuilder.builder(fieldName) diff --git a/src/test/java/org/opensearch/knn/plugin/action/RestKNNStatsHandlerIT.java b/src/test/java/org/opensearch/knn/plugin/action/RestKNNStatsHandlerIT.java index f44aaa78b..43f07dc0f 100644 --- a/src/test/java/org/opensearch/knn/plugin/action/RestKNNStatsHandlerIT.java +++ b/src/test/java/org/opensearch/knn/plugin/action/RestKNNStatsHandlerIT.java @@ -14,7 +14,6 @@ import org.opensearch.client.Response; import org.opensearch.client.ResponseException; import org.opensearch.cluster.health.ClusterHealthStatus; -import org.opensearch.common.Strings; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; import org.opensearch.core.xcontent.XContentBuilder; @@ -459,17 +458,16 @@ public void validateModelCreated(String modelId) throws IOException, Interrupted // mapping to create index from model public String modelIndexMapping(String fieldName, String modelId) throws IOException { - return Strings.toString( - XContentFactory.jsonBuilder() - .startObject() - .startObject(PROPERTIES) - .startObject(fieldName) - .field(VECTOR_TYPE, KNN_VECTOR) - .field(MODEL_ID, modelId) - .endObject() - .endObject() - .endObject() - ); + return XContentFactory.jsonBuilder() + .startObject() + .startObject(PROPERTIES) + .startObject(fieldName) + .field(VECTOR_TYPE, KNN_VECTOR) + .field(MODEL_ID, modelId) + .endObject() + .endObject() + .endObject() + .toString(); } @Override diff --git a/src/test/java/org/opensearch/knn/plugin/script/KNNScriptScoringIT.java b/src/test/java/org/opensearch/knn/plugin/script/KNNScriptScoringIT.java index bfafd889d..54a80926a 100644 --- a/src/test/java/org/opensearch/knn/plugin/script/KNNScriptScoringIT.java +++ b/src/test/java/org/opensearch/knn/plugin/script/KNNScriptScoringIT.java @@ -12,7 +12,6 @@ import org.opensearch.client.Request; import org.opensearch.client.Response; import org.opensearch.client.ResponseException; -import org.opensearch.common.Strings; import org.opensearch.common.settings.Settings; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; @@ -282,7 +281,7 @@ public void testKNNInvalidSourceScript() throws Exception { builder.endObject(); Request request = new Request("POST", "/" + INDEX_NAME + "/_search"); - request.setJsonEntity(Strings.toString(builder)); + request.setJsonEntity(builder.toString()); ResponseException ex = expectThrows(ResponseException.class, () -> client().performRequest(request)); assertThat(EntityUtils.toString(ex.getResponse().getEntity()), containsString("Unknown script name Dummy_source")); } @@ -413,16 +412,15 @@ public void testKNNScoreforNonVectorDocument() throws Exception { @SuppressWarnings("unchecked") public void testHammingScriptScore_Long() throws Exception { createIndex(INDEX_NAME, Settings.EMPTY); - String longMapping = Strings.toString( - XContentFactory.jsonBuilder() - .startObject() - .startObject("properties") - .startObject(FIELD_NAME) - .field("type", "long") - .endObject() - .endObject() - .endObject() - ); + String longMapping = XContentFactory.jsonBuilder() + .startObject() + .startObject("properties") + .startObject(FIELD_NAME) + .field("type", "long") + .endObject() + .endObject() + .endObject() + .toString(); putMappingRequest(INDEX_NAME, longMapping); addDocWithNumericField(INDEX_NAME, "0", FIELD_NAME, 8L); @@ -519,17 +517,16 @@ public void testHammingScriptScore_Long() throws Exception { @SuppressWarnings("unchecked") public void testHammingScriptScore_Base64() throws Exception { createIndex(INDEX_NAME, Settings.EMPTY); - String longMapping = Strings.toString( - XContentFactory.jsonBuilder() - .startObject() - .startObject("properties") - .startObject(FIELD_NAME) - .field("type", "binary") - .field("doc_values", true) - .endObject() - .endObject() - .endObject() - ); + String longMapping = XContentFactory.jsonBuilder() + .startObject() + .startObject("properties") + .startObject(FIELD_NAME) + .field("type", "binary") + .field("doc_values", true) + .endObject() + .endObject() + .endObject() + .toString(); putMappingRequest(INDEX_NAME, longMapping); addDocWithBinaryField(INDEX_NAME, "0", FIELD_NAME, "AAAAAAAAAAk="); diff --git a/src/test/java/org/opensearch/knn/plugin/script/PainlessScriptIT.java b/src/test/java/org/opensearch/knn/plugin/script/PainlessScriptIT.java index def52c478..da10578b9 100644 --- a/src/test/java/org/opensearch/knn/plugin/script/PainlessScriptIT.java +++ b/src/test/java/org/opensearch/knn/plugin/script/PainlessScriptIT.java @@ -17,7 +17,6 @@ import org.opensearch.client.Request; import org.opensearch.client.Response; import org.opensearch.client.ResponseException; -import org.opensearch.common.Strings; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.index.query.MatchAllQueryBuilder; @@ -63,7 +62,7 @@ protected String createMapping(List properties) throws IOExcept builder.endObject(); } xContentBuilder.endObject().endObject(); - return Strings.toString(xContentBuilder); + return xContentBuilder.toString(); } /* diff --git a/src/test/java/org/opensearch/knn/plugin/transport/DeleteModelResponseTests.java b/src/test/java/org/opensearch/knn/plugin/transport/DeleteModelResponseTests.java index a7eac4637..973ff00db 100644 --- a/src/test/java/org/opensearch/knn/plugin/transport/DeleteModelResponseTests.java +++ b/src/test/java/org/opensearch/knn/plugin/transport/DeleteModelResponseTests.java @@ -11,7 +11,6 @@ package org.opensearch.knn.plugin.transport; -import org.opensearch.common.Strings; import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.XContentBuilder; @@ -41,6 +40,6 @@ public void testXContentWithoutError() throws IOException { String expectedResponseString = "{\"model_id\":\"test-model\",\"result\":\"deleted\"}"; XContentBuilder xContentBuilder = MediaTypeRegistry.contentBuilder(XContentType.JSON); deleteModelResponse.toXContent(xContentBuilder, null); - assertEquals(expectedResponseString, Strings.toString(xContentBuilder)); + assertEquals(expectedResponseString, xContentBuilder.toString()); } } diff --git a/src/test/java/org/opensearch/knn/plugin/transport/GetModelResponseTests.java b/src/test/java/org/opensearch/knn/plugin/transport/GetModelResponseTests.java index 616dfbfd2..42adfaf66 100644 --- a/src/test/java/org/opensearch/knn/plugin/transport/GetModelResponseTests.java +++ b/src/test/java/org/opensearch/knn/plugin/transport/GetModelResponseTests.java @@ -11,7 +11,6 @@ package org.opensearch.knn.plugin.transport; -import org.opensearch.common.Strings; import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.XContentBuilder; @@ -51,7 +50,7 @@ public void testXContent() throws IOException { "{\"model_id\":\"test-model\",\"model_blob\":\"aGVsbG8=\",\"state\":\"created\",\"timestamp\":\"2021-03-27 10:15:30 AM +05:30\",\"description\":\"test model\",\"error\":\"\",\"space_type\":\"l2\",\"dimension\":4,\"engine\":\"nmslib\"}"; XContentBuilder xContentBuilder = MediaTypeRegistry.contentBuilder(XContentType.JSON); getModelResponse.toXContent(xContentBuilder, null); - assertEquals(expectedResponseString, Strings.toString(xContentBuilder)); + assertEquals(expectedResponseString, xContentBuilder.toString()); } public void testXContentWithNoModelBlob() throws IOException { @@ -62,6 +61,6 @@ public void testXContentWithNoModelBlob() throws IOException { "{\"model_id\":\"test-model\",\"model_blob\":\"\",\"state\":\"failed\",\"timestamp\":\"2021-03-27 10:15:30 AM +05:30\",\"description\":\"test model\",\"error\":\"\",\"space_type\":\"l2\",\"dimension\":4,\"engine\":\"nmslib\"}"; XContentBuilder xContentBuilder = MediaTypeRegistry.contentBuilder(XContentType.JSON); getModelResponse.toXContent(xContentBuilder, null); - assertEquals(expectedResponseString, Strings.toString(xContentBuilder)); + assertEquals(expectedResponseString, xContentBuilder.toString()); } } diff --git a/src/test/resources/lang-painless/lang-painless-1.3.0-SNAPSHOT.zip b/src/test/resources/lang-painless/lang-painless-1.3.0-SNAPSHOT.zip new file mode 100644 index 0000000000000000000000000000000000000000..1ea8b8117fba8682ca19e4d7bd09439408f0f632 GIT binary patch literal 1291433 zcmV(`K-0faO9KQ7000OG02)_;Q}hq>KSl!p067N$02}}S0B~$|XK8LNWMy+>a%pgM zZ*neha&K^Da&&2BbA44!Z`?Kzz3W#DyobGbg`@?F!iWR7DbNIH5(AqV-azVvfSD?&*y+!KrEr)_HRBby7~KUJjKto(67&f(178%2TVfP$i_ZRzs6% z#33ddsU=F*5d&GwIIM_rbf~}u zC4NuJc!lQhOq}Z2?EvRt1lKGVQ@U!cy-H=`qkK{$y*eBy{!gm$O6*`pT+^cpeQvi8 zZmPrN_$?YaxyYxXmY0Dq{J`gnk`1ETX+Hq`@XVT%U@pI4L8cyb+Bd;oUB z@`!PN@<9fgJ$B0T^`q41U9kwP9*K`8_B_4iD!xyT&4ve%$~D#)=>UBYIIdO$Ydqv2 zog|-o8QF2bVyjMcRbF8hahwC6sAoX5a1+U]cFXE)ma-<_A$uko}Qu>qQM>CZSj z%|6-KCbuEs3m0EHxJK1>C|j$smp9|i=ND7YcThb&_Nwl;PQplHI(Q&y{NHolCN0{_ zw-9$rQ_eX+%Yx;cQ(j^-XTBWIoSnKJzgh2!?+G&TzGq*WG@~;88d>DwK>pZEqDR%n zhHb(?V9*&uE2ems>9%FB^J1PMaT)GwCdotY1_h6>Zdl zS7cEkOeR%9lbeH)Md=u;JB7xUvI$->J`%Dc>qyl>x4o8r?qJ-YdENsKcTFTZ zL#C@@#%QZ^*9miZS?8<`3`zL7fJ_sL;nPI|Q8I-OizG^B@EGmxHV->^Tx_?Cco(e_ z*lc0BiEpD_w25i;5fFuQ? zPZg~w5DNW_MsZ{sG_|zWzbc_J3P&xS&y{89U)nBS#TT7%-@csQ?J<4-ddxoejQb?r zUAqSg2yGuc)ePyzD|>8T7@-{engE0$D7uUOHVULj3x{A@`kK`A)F{|=&s2BT0pG>5 z2v};c?aS`ZXVcUmHCQrkt?4Ubrt`_`F}WloShH_m`7++;f6$fEVMWZT_BbALqwD|R zP@uac@OwV_15ir?1PTBE2nYa2Sb1n0lEK&M*FXzoRGYfxR{D6y_~q4;*j+YBU&%cH)629a&&9*MpNHa)t2C& zRvgIQ(8cYT?{)P=GLo6F*Xx?y#wh3$ZU^(d*_**DQ28MK!h+V8WT^2K)AK)Dgmry) zuA`qw<5KnNc?dt>qp3LR_M$vQo@*;hJr=4)Y7&)PLbNC=rrRO6iYSy`N@4s}l0-0Nh0gC?(l9?o6ydAMt%|rX^X&CUxD(y+Pf{5Kj@C_Q%^+h73Sn{E%nWx9} zZAmeeZH<3sYHR%~rs5*jY164Rop8;snXI>0NRn9Qn;Djf7$(JTg|JC=&L!SZuKs@P z(tQVsRrz+^xR&bP`@CVkO#TPo5G%)=1a^(Kty@ z82$6Ym(dqgYU=T+E?3Sbb+grS{*-s0I|lpMH6X??lZSHeR>fVR&-?d8Al2=U>(TYc zweYu`dr2$g|eXcpfg8ZFd_AhP#gUz0`9Qib(_{>1|pfnH02k zAohV4!}uVmzEB;m13Uy?trZMW9Si|LxqEQspuw z_2r(yiU;(}XarK|_yD4j`x9u#Gz}I-=tSJjFjP5&hGLn8FJ&fx{a99aQ2RS0Pwz}W z;@=`UVmCl39o@^<6Zi_bbqxU>xKHO67o2wtn*zA&lyr5yQ3sj|>T(TVLfb*Dt~TuS z*+vfIU#L!N`LZLclomQ}ZBH}y6k1|vF-wT0YmJk~LkY`EYEJb?>r5|Bk;0|n<H2^;Ki*X!nu77HQkuw z#b1*E*4Gti%9}%(!LkLgKVH_P8{B=`xE;vn{Nns%!h9eM!LVUCBk@-fXLcHNz%%hZ;evQVzB%~@P zuNhM?8q-&hjYlCUmPi~t@jyGrz3|_ceWdL|ioPUx+zlnFwT_6f{*n0O)H~5TVO@>; zm(CeMz??RL0H7?ZwGeU*W3&(}-+U-=40d*35ahi%FLo~@&C}EnNuq0+%~{q|v_@G6 zH^`pc&u_E@Q2Di#XNJ+HkKPx??7n-+-UfhroASiBYF{fFl_e&(culP}LL%fY1@jI> zG5B$5N?kXc{05L{dpa3<6md<`QYD3Sky~n2%Emr%>K7nj{dj^4Nrioz0kBA+TUn-r zh{pa1J?40tC7Ect$=(X%A0YifEE5f^v_H5m7ih;}55&Xk3AQJ+B$l1x&5v$LGW>|D zjod4Hdbeergo-X|_6PsiuVgTjE%V6g>B($OfA%QLTpwfO3u>lOk}WZC9S95aA}r39 zGX51~!%dWbd|t=E5aUE#V@lDoVyMiN=&R>wOBt<+W*_udh8vjp3IdZukV;5fy_p!@ z8Iq%N{S*m-%|lgFSh_mt7usDwo!^`$$>vm?ff*}HRLI<}6~}SM*;Vz_Y9jq%d1W#o z)U%4UYQiw|QcHw)SV^~GW+@gSxg+frRk%RzhEkFK^tqknCPGn^AjxP!oH%&R7W_4x zcwz11hBYkV-Hq3Ma&SR8e`v<(EA1hhr%Us&e2m#2fTn4sRvSpII%$0 zW&ht{c+qw+dZG#6_#^Z(8Rz{3d@VSQaN%oQs9?f=;JmIip5Jkl*j~petAH_6(rF6b zLaNI!hWjD`3*@de*zP~jd)-0iHN;HAMQ6Am;-KJooMM@!zgjH!zY>`~>C@Rh1zv>_5@AfZxy=@J)P?NHeq{;Iao%E78;LvG z$Gb928F|I6#7`8yv3VK9NZ#yMLGjy^bm&10n+ko8z$7Z)aR-&;49D^u+EQ|vxz3T| z(Z82aRKS^J$I6qiC{--Q{mMhdNlr5vyn@u6+y>Fa?!z>b=#koC`r(E!>tL4~6>70y z`modQVc~k*su9G1rd$L3gSTBcZgpUgCC%e)Diu zmHjyHNc$_80k1ozK2HbaN>(=|jREIH@ncBAuR&|q$FsVC8Dw%0h|QF z`3o1gs`rdcoDz#fp=q^nC^}VaUh?b|mU)CK$44o7nU__|HL_s!=K+WB#kH2Txu2)o z7r5kmCqf^SKOcOF&K6%MTWLarS+ZPBv>!C6=-LX<{Hf?KnN%06S}nwsQn`Q4jd?4P znID8MpvE#b*-%~fhAnQThai{%WO(f=AE|o`Qd=;iQ0#uFtEoa0rq`xLQmS1RZtwh% zf29&nz4-71q4g+ohmyPG&>vL$o5rfr%nrP_QfJ;EENWE=%wUkAE`YYQ2p82E9up%j zauCj478yyJ6j8DlzwkVaA`L3c=fCAnf^;r9PeSmC*%~Gl%dg&dh)20)&HpypPF4>G z=-hZbMb4Ijt@VZVlcK`3_k&8DSQPuiOyH=Mbz9ql+U1GL0%6?!3+72+ii5)q$1)6W znG#*0zR1di+Ps$Zv=QirH!14v$ z=5;X}Gi~W5RV`Sml(tq?Y4P$dMW3v6WlEDGcFdQ1Nd={BS4*MC*Wy@AiIy?Nlbd91 zR^hsVpho!0%*78tALz3+336Z(a90+`=a%Cel_;Bh$)L2rCUfp{;Ts+ zl_&7vBRXeZ**#tU4Oq$c$`-RW2Oc5L>{dT~P1KjJX;8~{^kGPR59TwGEsoyqTz7ol zdjbW3(I7yMkT6*A^{|EKgQz^*NkYTlFs)?`hS|bxo$N8N;pO{M{RXr8|3PtoV4X#X zEJDPIl$NruRqu=6IwUj}@;pGNyE;>$`L7(kuv&wvL+mLH1NYGwuLPz1N%^pyEI;5d zfS*qVYF{llKa%FL&U;vH#~=s_2-ZVjf-bJk0p)01^)iV`GYssNoRi58MxQd`f74Tb z0dX5E8gh!s7Kl5ZiE?3Zss~RL(5W4B=0g>rU7vAdyqh@vN#G#38=Dy0-8FU_=+%$O z>G{_FJsIL#av+M`#&)~{lB3Ux1`zweG#So* zo&J1XgVgqWcj@-^=+ysu|E#=R^&<)B?f3m4x9Wa3`VRE|%M!lWee!qfJKKQ7&>z1& zLco1J{5g`}@$c;E(=u>rJKY|JaKI4!htIy|dM(JovV$%^0?un8=fJk?!tDsZ6qca) zO*T4rR>o`Riol7C&oS8M9(psM%H2K%clDc}R^Id>c5THn)AW|X<%>W*CU)Ppa}iFz zkfG@XzF1&S$fpV6ySCVf>KZD~gVPq8Lkm?%M1bvTy=`e|P}WU*Waq(zeL6n1U~&8t z90fA}jMGOXEHQ28gpYhrH=GXIA9Qfx{_X3}=q*1DiC`8y1CYb{iUR6VZ+GU&J3PL6 zkQVgj8hnAdUvJsAkfbDR_e~1~3%BU@Bsvd+2p!MDxrjTqp7A)-95bmK*Ev2G`R zkH!AL+4Zt8@61+!qtEX{MbtiQ!WjYXVJrW0Ov`Ii{|ik6E1F0or-<>kHkVe7hmPincR&%Azc+gi*M zL{S1TU9ajJnLkm8|AT<$QGNTcmduJ|(!ws}VZMG}sgTmD@fKfzL{|^wvIz)LXBTKPcOe(67QcMf? zEsNztuE)Y}GVIcyVs~YkBfYj*{{Uk+zLvGr)k;UCM7XgOO_e8MvtpN3fD1}4q7gqw zji?l}()D$bSRzX-y<8^|G}iE{I)?`u@Wv^gJ>QX#^DdG5>Mj((5@+W27v)1SxS4bK_qNaZcdY?Z%eWN6PCV>Le?f-t~ zUG&`jU{5V%5?3wdBzF5T7g=HE!rlrKOc#NATj+EC#P9sF_WMCmEhjS=UMx7m|IK4T zONm>R-*+6A3Hc7HzwmP|2^a_n^}p)5yn^a~1pWWD^Rxse#9&6Gu(#i69J5Bkly2;j z5ZI94pk;M!5pOx4w1<0eGGy^`nY})r{k~}%j}2l;P#lqyW2ObC#UqVltZZHri)X*a zIHJnmR~BxVcKJ{u=2u4cV8qH~(nk4qsY@v2g4L)(q00{Y2wiZu1^T*3Q?pznNb3zZ z-GA)5#^V;N&6Eh+&6~GIPTkQkh$V+7VYuUd%#AeU6<}JfF&v{CuQUrAZaqaP$A`rC zY*8kEApiFx#s5c`1ooeCaJKkwInw`R5dS~M!O_g##mvar#Pa`$o&GD1^?%}yMppK= zW-c!O#~xhsqq{#G5KuTS5D@+UWlz-1T-d?E)y37>$dOb|N`&6T*2u-BObgmmbq)Pj z!Q6`_MS>7C3M@$!Dp3{%1d19OL<)*Dxsfc8R?UE%Aqz4k>v0Z;T}+H7daYZD&U!2w zMyxK9IjXpLt-C?**4RU5`_EjxHYML%-;`-GwA3QQ{%qfUzso$|b--O-@9RKCJ~(5D zGnpw8%!s#-7iV#Jadq(*=qa)oEclvbt8f1wbvy(dB@qo`{LRC}gH)m+F-pw_Th-|h z>B>G`SUWfe#Tguv@MRQLn^eO_DHhdWmv)ThST0$e)kOI^K#-%&-}Fdrt=?sLL}s!{ z=5V4IMOHLF7@TVxsMo~hAssIjc1vbgx>bED)A+FX+0(R!iwr4T<1 zX)cusFFDdYR$}&=&6054oYAmWRlN3cv;GTC+E~a9nGvzk$sbp7MDh|z;U+i&X{ZWB zBoJYB(WR}F@am0F6QU498tQsX^VF%#VCUCZlsah05L(Umri@}XAu>Q+ErF~OkzID| zC1(tsdhvR{3xrghkeN55%J>#%#6f(ucuR3dP3_NnO0*m!M#Hlq1~sYbbY|RND3Z_5 z1km|n+3n2%u`w$MA4bf-q^eVu5EjP2KvLV4SitT^jJSJ9F||-spBTg6u9)Q%kBT=9 zX-z<|U1)GCB_CoGX3Toj%f#x9u#`Q*t?>tXmPdO z#o;1tJ;ZDL+F?LYxXqlk7S-xkFyq}Y{|nQJ=KR(rWp07+kbCYrN97t5-Db5j^@Zl~ zu(-V_n=~aFJB%nedh6w3d^E_Tb4x7UCbc8OFw{_8nQbE2CE4DyV_jm4++^C!bg9-M zrAda>#zRieZmu{6aFrdJxox%+)X*IM3zSkL;{%(ct}5^h!px+4<605hqwKD&zd@=# zKz@whbk(F@lfk+XRobsHz5#yGwcX>B3hCoC5DIC^8|fG71X|8gvDf&TS=hqWxE8>}p$mKuw0IBTE;ygY0*;VAsfB+j9!@>=}{!knSPY%YcftjEBkQz%; zY@>jSqVd#gM142RqWkz8c(uPs$oxW?pYqTf>u?Az?4BUVuMepfrf1@y-`ARM@Zxfl z73(nL3C-Y=?oo)`Ga zYqK5J(d9i?an*?Ic&O*UHLj6|F*YF7dD1gXwojr8KLi`H{SAsOfW~eClcQnXjjR%k z;xC%HI?|3Qe_7S6_8v8OSMWM?cc(ujQJM8)_!$_zwy^=r?<3cl zHp15{1X~qjf=35e0@5%>Mth?XOoOh6x)3GSgS^<;>IW~;4 z3#p#doi*U1>?$X1j2D$9y?o2L1suw?Qyz@by$$d3pxLc$BOjo+)$L~eW(xQc()gjC zFzA3P4b1BD<#WlL3r^hTA?`IIrY`wBJF(rzRX=<_^?h66Uj1AYr>kZ%OXHl)tQyOw zzQYj-=qw?@rFMuGBj%297SeQB($-NkidE4_tK9l68G0xWj`RY~W;L{PtvCa$@3$u2 zB39dpzq!rk$j=<*w7w<|XJDxTSqG@U42b+x>0zPd{>(JP(rH=g*&se-TE*{bGhoYB z0i)dp@NC!yFg}h*SKkC31mdwZle9^=c0^8z+z8@EmpjV7)xfeQ@o!a$@rXqpnARdX z8D*|V{*9jd_8PZ(G@%|{kO+juK8nD|_Zm~+JIkCSU^>e@#&Ys!ndm_|=8JNwO19T5V5Sj@r_I!=;7WekT&Qb=J+iB^*0{|>gguJwDA z`G#S%7{+I|fPJuOSDE!6Vc(j~YFxIT56wfEHU0J2BBtO!Si@a}=0D~k^CbONA2uZ> zq88B&d(V+8YonEfx_IUbI)t3%drZ89gKFF28w59|ZIVkk)-8ATFp?)pASs*i7@2up zNGPdqg^dBc9kMpv9lX%S?C8?-O(HT3=@}RNGV|1<${=Vvco9)GTR9mdna%ca>Xmls zx<5dv*WHa?l}IsYn1JQ5po6o5s#ucG7qv76&JUq*W7<@LTwAh(xZ>{vpBwYcjl8Ia z?W)FoRYD{v(h=ZZ&qBR#;;Fpfi@X{`wEw+%<3-)OlNB#R4Hy)SpN2Gt=z@(mDh~Zl zDhOu2olpXKe1IPKM>5dy_Z?YAJNz{2raQer6t_Y!U@A35<6l=i=d_?~)1dD@zwv;VO`kIkM!T%*XK~%hMd_lr`mnw3H9~fg-L+ z5IhgW+SA0%nLg@For^qXzDNJow4OWO+>5fI#`TKl^}*e!n|8rPm*N<1gKXw<+K;i| zoMaFw_RFO=@1(&s|D#D`-ml6jll3#Rr-f*EM6{EAegIRO`1GFO(U$h-T@Tj)In8oE z;^nNRknbAFJ*Typ`^%^?N7;w70)Ke7LRcWmtw|N*WLQ!no8Rk|+5I4&en;c5QFZ`x zMKtvm_mueXDRjW@)uJ(=ZO*5#3Kmg=^DpimsMkC_t|@yE4?mHQcosG?f(_F&6CRN~ zTN17Z^FbiJX@(wL7FujMMl1Ob`>#uPZb6UCsSjLnVe`h%7%fA($x(wxj_`Wy%&>$*`i4|KPB_0KEEE|59J|8lZ_o?4tuY>7 ztDzF}&v(veS!(mB7vo@)7%Oe&Q=67}*{I4OaP$Arl^to7jMwWYQhM+e2Jfg%<9LSD z|H$poneLoAgGxcEnfBO_-3ZKEKMt(?K`Q)C zw8-Y?0g-s83LKHA8yRuaO9RQOM+I&`5;=MQu|v$6B^JpoE?Ue-tDGdp`GPx}l}GSr zuuYvCC)vY0SJFHa4ChVY1ltka#COVaB4ghjFR>)mG6UICXus?y9fujZJ1NJ{S911| zDU;3LN}g}ONT9e(pm?awkY*l@tJW#AL7_fP*Y?NaFSJf5uT0NTrQ@j5wpVW3VO?`h zsXl#Gw^OaQW`(n5h-dm@tQB%qg?J?(@QZb8HGyJuH%ngR#AIwAHkr$$$TJ1eU)X!kY@YRVC_ZE*PO{fG%{64>TFT1+XzN)8f0_S@? zT`5y@>{5YJn!!f!GLA&j!G3x7=0~Q%g`I^5(XiCSg`H{2*BFW- zx{eL&W!fIw+6Hxh`0Lbfi`S!jUHW}2&gMy5Li_s-0u%!FPIVUk1`+c-IB#*FSxdWAt{@8_uUQBf_V*(npGKwsiNBg zN`t}`5B_#!ymRaA(PymeH+Pq=T)uki>gkcW=-NS%oN3V3WHwbBPbEi>tGTVgp0m&D zv>EPIXGd3ui^%|x#n-|tTzob@*E^*#Izeju>i*{LsVCIzKwu|U38zDB8CDz~wTnxx z$kc3bY2BERXNosGarSgiQ;=sWgz1LnI_8A0`R{MdnOT;T&DTHpb(tG`>UHTpFu#hjZL3(4Fjg@k3KG1+0K99;j?YWh%jEmC=?Qvt(#XXiY_mR1>iF<>)2koB0j}OY(}4sx?6wIw}|K|YZQSq z=nSrrUp2&-EoVEuA?<>iPKEv~W6WdF@(I%yf!1+#K4Lbpb440Xzz?Yu$?SE!7m>qEy_Cf3us zcmapmo{ZG)IyZ)w4^Z8q|g`^D{L5&glF z0a}5iSeWZ8h_RV>_1&q3u1k}{p`mwlZh=G9{lcLBBFPgTx6GnVbS zkeUrcT#ocSm<}~IyT$B%((ieHSvd*l1to8&W`h&|jlMN?Aw}HwnxO=5RXpFF( zh$qaTv9B7;YgMet<#Iw=CEo6bzjcr zz^$&CJN59=!K$9QcT0xgm;00H#-WVg;>ge-}z%(}?LWyuOD^EB3w}(0TSrQ!N17CdFMwT`{3@kI;w^0OL{P zm;muM><1N}NoRG7h1{vGx=2mHN7NL?DQK?Is}S<@0F)(Fv%g{G-24*a+&op5$Ga~A zcpe6Rf~4b@=Fy(@@1r_Qg}y>v5P-*DKS1_O{T7vRRbM<TnE0_ZR1hOAEWBPx9o1 z-4

>kbia-0;&6DR2=~>E1c-+_V~~9Flop{`d-9Q81E1$eP{cID5At56N=7w zK%gxfyLW!leqR~b(ipYYJGXIiZXtQTSc+1a^^&cWBDf!PVDlvSjd6D&LcjLC@~ZbM zr2IkfEgZT(6j5esLwx_fp8G05cG|^y`_&8)aJ;{u>H^;Q*8x$`D7jTfzo!YN8w`mN zY4fji$l`%33Zr(dFP)!j-8qGU^*tF9iVa%i=xV_7K_g$Fnw#@ypY0vcZ}ixG7Fx~- zR)(GKk`rM0pr@J>edqovQm9Gwi_Omz}=loi!N!lkB# z8F!nRZ;32G_5dP*6@*L&+ba3rJP-$h3M=uP^`2{tUH>NFC4qr`QZOgns`7EXS%xD^ zi|G=@m<$<}QDj7RmiQdI3422UjaWhdjI@PFf%lZX)h5#fmmF$T-BBexPJ|UVdIT!( zy4Jp;f@ZvH%30nMR*kl>5$}-0^8^_XKdOXgXmwIg@6nU-qaL@vF;x1Rz}bph%c(t5 z;z?6QU#Zz^ZSMw}5x3+>o1t}WeT~zZ=#I&$KLX8}1gLlO?$+y-!$!%H2jIjxqf;Ok z96p8Z;?2CJ49KRPWidY30ZCDowEH2|wqM<4(j-dPrZpgJBE0Lps zzZGU9emH6Z;BOdsz5Jw3WJ>vX?L3>q-EdP| zI4>Yqh+do3btP?_&5BaGePG4*t5SOpjo}rHp52vHZ=ISZR;${L`G!M7i9^jDhRXo- z*BqHnu@MzYT~pPSv2&SGn)Qb;MGb$#ugK8hW70Pr$?}VisNr)wJCjGV50@x!z}=G1 z-16{fTZ~!gxQRnlC_5-!(LdLl!8MbT(R8bfbw<~-c{?UfpZ_d|)v&q#End%lX*RPR zp{^Ig!f(5)wluNRhG=DP(Rt?5sVr|?TN}rfx>#vMT_A2eY^Ajr7ToTDxmjim++|Jl zzi3 z$)zt;RqhCL7<8>tmE5g|?i8-qmMts^Yc(6L^8kQv7^@>iDVPZLQ7=>8^>#Pqul-eNie04&AwP?$$O>WUwmOG#Bo0 z!~^J$8Ccfy5G4~9S`>1`rH!gA#STE_UA{M%k^krb^e3cx+vJVL^pL%085aKshGR9(>r^G?Ebh}$dcWA zWjIYz|3?#sJnUSn*x_T> zMlXh2OY&D4D>@h@d0p1vnetinLwl-_<$6ayGg<3$(H%06^K6pK>nPQ81p*-%?~d$lqe@2jD9y3WAVD_k1SgB~`YJ zU+zBWTI$Zy;*I$%N9~}qp{(J^mbH%**yXjRa$vxT#>UjYX0{;-kT^j_}W#Z#M?_b>XB%HBaS=vDEjIAclM9(wK1 z*B2;Y_xcJ3ETs$WCk)pwca9DA;7E>*3uL}9cwyp!8+nA%H1-60MI?f=XkkCepfZJg zOp$C3m)Db&Qw7q3FSki)AW#Jt%0e6)bs(rHRKWE7lEz2sG*HTvx_IF7Kyznfo zpL%&jWcdWIpVqpDpUb34i*b1M@Sz22jYCMa>@Y?z#@w1^s8Oqb8S+F@tp5-;hHict zyb@+B5#yi=z9W3re{~5ip)tCm61N%C<;_=(7<)wjJb{tdW*2E*m@M zJ;#;C`T2%6u~oMCCDz0u_+>o*2VUdE$s6pjUlb}KAT4|WV3qoZ13OCWKPmJ$i!Mur zES}>H?@2g~E!IQy`X@l`hNMk8q~BX4K9&BMNP{6^_Q49W8Z#!P&3l?rmuRE%pYg&u zsX?OAGg~`3t5i)*-H=6Y3bKyPiwJ^GKn%)$QYYQ*h)tYGDc1Q>+EEC68#<&oG++Y~ z@+{9gxLwVcaMo^Wip)|Q2RA(-8l^RlAXNi5g49x1WGlh4wg86Al5av*=cU_F2bJ$| zm=XT48ENOj9FE4Dm&m_!k=DFZb;Hw4@%R`r`H$~SF$Y}L0JHFq3CfE;gR`Mt@28{< zR!(7q|Apum+Y*;+FDkyFaWL>Og>b9oDB-w9(gRY|BBVeG*wTg=mJ0Fnkt%IuKlySy zO3{YGY_N1Ls+r9($}@ZX%B*w@d-|v3af~sSF7l9WEBZRTVHg{z)?qFs&8m!>K_)Nl zh=;g)LcYKimc8^16C`IQtiwH|^aBJE)d`+Uaj|RDrr>NQqo7kV$yxSS^#0Xav(8Ku z^_mB@NC}Qhlchu6))Dk?;od!o_-yDX4>UERKl#c+5XNlHQb3s?m04fI_Wn_|)8F)M z?uj+_Gzrpu%FwW9otV67u=A?mWmNMuZUyiY0RfOSuRe4i(JtK>B+%PZ2i zm2hAYYbq_b0z0?Nn1bkw@5HJ~vg(v{hpM-FC8fDqMR!i`|i z@m(^KDHLKp)RhPVmQ{QCz$}@NkCZq~5?eJ{YJpl3R&7#>qjVOeCY5f>ZItR$E(F~yTg1ngqm*YE zEn*$ArjobrT*;B`Y4sNJV3s*YH#n0hH)76cPUbo+UZ2xO1#rjrEkRVhtest+ zYAtQ$t?#XWg-04h$B<`%a1&@8kzHZ8tVlTHxHzDd;$xJzyb6o+c`&>@+uPbJdo@6n z7w#h-kv%TBf_J$f;?*p;QfyStG7Bfs9hh}FiN?r+CP6+jn<&!~I=JPvMaqbgG`Q70 zm}!TxdDmzmcN-=n?Q#)xNOxVVFuNA5-tdTdfD-i=%rPM zIMRl~3%xy!i_jyxW45*rAreC&mw)8Q`GhO-6#hE&+HB-AWTU#!1+RVr&nJoZbh;ee z3nKA*>>=1sQjm`ZXr^iiEZ_?kTj_g0D2QR#3EOzv5RZ&i_Csbg?BMMG2;n;8459bPUL8f%A7jVSueNU z`5N(wkHy!wGPe2#Y5`_MS6+h9svKgJjd03@PGN+VAH{V%bbL5~f;yn&gp?Q|#|6ze z0^1BnJw%`l|GM}kOdk}=g-|$JdLYIJbvSN%Ai{?l_-fl&5WWjrk@{n;>5Xb zgt>M?s2>El9`WjiP?TgyFARfw($f#iJfigiXB^P2UQKKU@^zvi9x?kuD~upEA`6VH zkPWg^!7rrVbczFl@U5tK6GR$Zi9bunI_0ekeMs_2*bcjtVeJ#w55SipdKJ6TIHWa> z#g^H132%pGJU@=j81cu0RHl=hham7N6Q2VA6;cc4TS76F4#_d<0$>VMwjvT4;7Kr( zfek{axhiQ<^Mg`E?YQxY84?w4BbJ~_NraOG@wQG5mK;>f%4VyioVjCKpyv{a_3*(BaZHg_~HX zpq`2VnfaL@U|fdMX>$kCxRi0w$O`CXQFp6nRa!aY1ZjnoGO8234=d`aipUBSz!-6U z`I0Tw(WRxaNSLZsjw@W67LiQ1Hh21&WSE_DwQDYG-aAhufn-^54aKA1rKk7NKAj#( zp5RS+ins#Cjd|j04OmZB>6g_qoOK2nu04}h`Mef(7hpFu_I`Z|7bFLCk2_z32pcph#kkbcAv&y5m}-Y|j_9~!*xC|Zh9pgQ z#!8v32gm3sY}_*3>#wEl-Y3>`X>>0M;>o=cHTKXOqZpPG==DIsw2dYFU0`1KG+7#u zQ&)p03O8C2LR#^~w%IyWZROkh6HmYIXokSksY?HM7K6d#sRNIo+G{ZJ-EG-GsyTcL3^*f)7yil^1l*|KHqI^u8c%qDeEt}nE(d%nwK;|JgVF@S_(DbnRQveqyTkj@`wrt6h6gdSG*EAP$ zDL@lkPSX}km#XYHFaQF5Ae8kxT6bvQh}2(1*B4ghp&)h57m*D`7!TN?7=O+*JcqP~ zaa>DO1n)!|b}AOlGQ6ZzY}wez{!195TY=UMO`&&{jJdNwxm$*O%E= zAY8O~kz$AiWjR7PiBLw9PXucTkt$Fl)4+_;gDi=Wj{Ry5JwqZBpx7Kc;zh$!fq}3{Lre z81w0kLE#T>%OnRz;A>W38DKINl8_JkG+l^x1$bLb3BbgekDFsQuzat>(mpoc01NM2 zAjMV3^+m_qQM^w|h|kQRqgXr1J!xmpWTrj2NjEsm=`N#wKYIC8$I#=`tz)t;6hz3#2>zo(BTZTX zkc#O5C_vZ0gkkCr!dx-|=!{8)ocGSX(FknS@V<{iS5c}ZdVK7nMUM|`>jYdmq=}*XN_XuN0DtyCCT5!w|xcbJ0au(Zv|iMH$h>8PP?C z$U=?(bX1`MickZxPy?!vgJ8-xdFzZrkb*xX`}9CK&gq-1=<pHM*)$s^QgaVnV>U zp|u?;s?HX$?I!ebW@8>>wAU8&NM?Lq;vDS_m=4xAU!SKzsY+)+qLGkC*8s%>0m^$q zq3kaj_31*P&@af&lLon>ja(t+tA*UjaXniR5HjfEO9(_OgOs3?$s`BZg$~FwB8V#% zHF_w77;r~&GE^c3ny9}C&>vZ1%uD{xO2oIfw?(Pp6<1Hc4YDW(aV&rY#@IhpS~aN% zSu)UY$}C3ZsbH^ZBu+7G2s&xtS{3n3NBj8KJ=8j|RuZ(VeD*+h0xD(^CqF5W807rW zqf}xL_VB>CFzjj4O2u-)K4y4znu$yxHwkKb70P*>XmKW5CD9BaVDI<66QD6DLGD6S zs71I*<>^9Mt|;23WZI@|S|@vAnP>13Sz1A2TR50SlBA@QC6Y(K#xtHOcW(2H4MlOln>U>CKbf%D1V5QW!p zf^1_8A>cmGeW7V@D11|)<+yt;qZp2Sl+#>^#ksJT(-4g`=Ob8VBy|Zfav2^?3_CX6zlz+#p2Lyb?V zIJLb1=F|&=baPavLY8M1~8akA4Y%`Gk1_g%{nB#>!vy&LeRGC5xlRG^OL zjvB^jc1tm$m5VW0&gPB|#;t$quibr;yLa&^C^Ym*0QwYwJwm`9DT!}P?1NoqZ7=8q z_5HBzsfJ?7Hz1vIAV~a_k5j~pfIAJ*u;r;JD*l?- z&tS^WWXjKI%1>COXN}?uhXdtb1pHGQPMHr3di7su4yp)Q)!V4qX>hgR8rV`;`1)zM z`cH_X==AZwO1j`Nvbxw}=JmmKvbuUkXt(Pzqvhk2e@a$5D5O4L;bixPCy5-ya8zSJ zs4!qlb(9BN+Vylx!3!-<&E= z6vts5(v^k1c#C+06z2(3%|f3@g|(z8YoJeeLMa&##yS?@Q#Bx!p6n?{HsN0m8HkDX zmjhWsdWi6exJH?dN;%h?X$ePRp&yBbd$Gc`&J0GlG#b!UR2-U3B_QA$LlX$h!P z6KExUay|RP9qYncb{uF|qrk=gG4US3Wl!K99Hf-T%5NSnVMG?TkJ#Ri+CFmY<@oJaql^ z2OM-PtBrFEd98-tFPA`ch(CnFOH%EHiFY2>lmh2>AoUytgX!EIv@x zt8YgZ#7-Yg-Zhr|1^2LSl*p$0kiIo_bop>2H; zA{muce&mNYJFd}QGOOwK0hX_a2aq~APo<0&8XeJ!KS~@JOX&u?rI7-=k${R%M0kUQ z2h>3Pz2ctwK~^KGi~ZV{5hfr=xoV9L5PG{1us-sm6P6&z_R^n}57Ki4t#u(FR)iPksaN`mG1ku=wfzO5Q83=r~eyGHDRpxl>J;c*x2P-d?Q{uJ7)|)+G-B1e+jn8$W4f^aXBdxTvIj^2K+_1F+Jo{$ z^(TcjfXfN9Z(r_~_R=5jxIS%?fPhdqfPj$y-#sz%#%@;s?TcyCgE2xg ze-3c8SPu2DDxxP+D#R19)=|1YYLwyDl?lRJB*9qbum`hNNlwK*Q9%jP*Jl(;)D_

!idBN z4{{F6pCSyh1d|MpC?#^iG{L&hx_Jw;ijc3yKB0>*EQNCj_euf*Z8AT28E^O|NJ&O& zHdaq?`Vu-)!gn~C8Md}J9?6L16coGYm`aHZEz!&$7xyK;b#w7>nzof>b=4t&c>{GHfl}}kmJDV6`K*p+SN-5atDhLOIPgohCRli2 z1Fh0cgfethD}a1RNT0<_1b{ssSP%W{!z}anJ*QXN5}?I(*L*UV0DAsyR-g)#n6v_D z6ty?yMENLkfyIgmL9)l=NQLb!;)wlM$Phe(5jpCO!K7l|tpAb-aEwvKA&+2+L>72Kq7kcSA-H?^ulCT$2eudaD)d3wiu3qMI`8ybn^ni3ro!; z%Q(Z3ZiJjEIZ#s58;l+L%_1|B^&T;TRV*Rdm@T%5yZ(;^x>+-wbqRXq4g8<0wTaw%Eh z$nzJAq2j8NODD*KC3VCNn;~olSw*dNqM0c>FK_71m78?Q%0AJ?HCVCCA=VW8E!mi2 zHrbd12rU4MZz?d+oU%4+iGcf}Ri{|S9M{9a&~zs1D(tsI*7-9R955U8$9``Z;)`p) zivtaBQQpZ~RTN{y229Qpsby0X@VxjF0;S|3^Dklx(^(3gnuQN2`_0&JXC~y&SWjm? zMGN}W{Pb~d!Ikm12d;Aq`pGUdjf*o02xie|8j0C8NP+4ZF>=veP?11eb1kTiBqiWEyVr;D$k7tcCOp6{GN%rOqvffmlWSIoir7#Q&7|{@1ot+8 zsb1+!9{q)LY|Oyu_%pq8b54h9{(mgMJ0>WUqY6I!>`OSw?2?Agk5q_tw8`hDQ<{@| zOIq3!BEU|3V$E&YfHkEJ`#~svcUgeiU$TjI@@4|;}Bto+GNu;EebwNskXKEh+M93`)(NCYoaD{X5u z<%F8=Hmu6Rhp<58mT));$Ry0Y#^+nYtwLItE0bsqP}h zTu3S9fWQppiyKYGdzWCGzC*2PPZr&iQl%Qascrt-1K+^ZUj90qBH9mWILf*IGyZB) z)BKgPH=IbmE>o_KXFO~xUS?bKr*fDK693kS$FtHrfkktK+;bz+Udo~0YoH9yndN%v zR0~CXE?-xr8=kgWhCE6^z@3<4$?!*l*(;WSuoD|Zb#N=ki>;5h4&12UT@`#mQ~3P2 zQX?_eefc2Zt_-a|`Ox+ivZ$|~7L}9+TYAqvUCO~_*b|zc?#z=91J%C)uAkC=El46v zmT5Hf1uA=A#-92S4OFAKGr-vSJKD+MLkJD43ogm6U8Upbx2UoN41Vp09{L^IYxPz* z(qS-%Pa^^UpNrI`uwPcH!Ykz(#?L7g2AsKTmR+r%!dY&FKU+}S^fbd8=Mr7k-dg>J z5O1pE?MRg>b@8oOMFFR#kh}#^P7?D}yKb!xS(^?&cxN`l%=N3wlX?=LBY&-#GwLEl zJ4Nl9&x&jld*J9R^SGESRcf1?O2koT3zXZfS0s1o|v0?&QXUu|VxtOe5G=@CZ?3@$}0c18WnFds5 z;fP_Gj1box#-}zVB6OJ6EhrMvs-o*to9c08^El9dLVs;B;K?>IzqS}n%xI0KS0b)G ztp;__wPPm!gkec1`jy1%1>pGWx7H7>ia2Mdk^kF{B9xv|!G@k1qln-ZW+O$5(v&)q z?vZ6Y%_TIcf-3DksgX=R#hJ%Irdyz^*6QzMm8pe19;^FD=~Byh0v*0SOS!McRH-yt zxx56{qQ~l?Kl#wX4uwu%Bd8_Uy$iNwv-e7$V z!F}(6-EkbhzOCL>-%hPG2gwbj6Ex}&wUvVPO%l(}UisKQyaef#9p9a;5~Yqxc4pi= z9OX6dg?qX5K0=9u{i_WdOuURENmi+5lX2R*C7VMj4~hRvBy07Pg+OTAoTo z(Nu%rP`Gy@D#EN6U|q3Ey`IfkB4ov7Eguwa6xXnRUc?{H(r9O|So?)<1^gv46&6v2 zeq^qf)v+ym0bg~h*W+TSKPY-7ALE_Ze+pe96+20f<=__;XXj*0RY+)ZUuhJVk@mYk zP_wSAy!i)lX$E-BmI6;Hd?Id{RA0jQUeBln+5(xENzv*ukWW)t#JvM9t4Y0D=)9A>1cB-K)nlcIf^=Weab zsU%j7%kd;uoy(CVR?$nL2OnGOc6?$(eZNr7f}xzbgE@;ki$^{p<|eZwC=ACjI;;nk zSzgAoJb+FDTXR(XGq5NY(@W!r!Pqt=gh0 z%GL9E-uG%fV)+Ja*8Ym3>Twl@T2+4Ud0~qa4;D&pb7pJgx_VoPvWKQF?u|hl$Nfw- z26%_wgsQGIj>6Y9b;}8Qm|xFrDKr%*YyMxDei9+=leW7sWEoI7B5Qk10_BPjxr!3hMDc_r%_*YjAgrlLDhjay~thz+>T6c3w(%G>u=Jt@!oUw1*|HSdQ#z9O!J^K}b;@GP9d-oHx{x8U=(LR*D8 zLd<@3!SE!uS+bhJ)#wxwwje0kgJl~Vt)@3@KjauYt#XL9Ay|k1QNxY-BaEfvj1^vy zU=<#`V?kvX-sF~QAc8cfG(AB?V~Pb%{m2EA%Y@A{f`0MCJQ42Hi9;?7G!xR)h)0_t z47L;A=FnwMYvWkjH`zm8aj6LnFOT(t5TRE9HrF?vr%87-5;tKy0c%X&51sQZl|7xa z$ub3n%|LX8wDDOZjG%4yFPOtR2@&s8c!+- zD}I?Po`e~z%o$5Omg-@MIZ39<_;FLZA`~AjobX7h)eZouDRcrl{7j5_Ey~oU zBf3g0z6z^Ug}c-wQsy)=`Ug!K7)^MET6_got_pXqiIvQ$6-)3`232M|YG%7I&kmx# zG4Ept{y})2lfX@Ij1gyS1V5RMk1^!V4}wm#4<|sVb($U`6q{?aE>ihp{Ip;;iJ-t7 z<(o)8L6z={H)-x~bv<{Dg=1FO+|8vHx6tT}^I`+CzeTS6Eq)SK_#8lP)F+ovyV;rj z*rvxz-U;XyCFoqXtVB+h2^lUC@c8x)rwyHQq~m&YYBj@ z8o)Mx=rpUwI2mieq}XSauQ=4|f~O*1?`&Q;P!t$P`975|s=r3^2qXRBlx?e2xT?#BVo#Jrg4 zk_%Jqxe*$F=g;yJmkaTPiKt{nA)4GkIxcDP(-g6>ib@9{PJN)I5dj~R?SWS(qCp_; zfjG&ztP_Yq_{72O>J^XzcA}OfbOQn~mduR3bb_!@!u|=fZUpFz@ALul20E*#Q$?WZ8uT(wXWTA9Th6cPGKZ2t9Dfy192L_B0MM{aBoX z&XLXZjSp#{v*{6SMwF#mvy-bVgkaSyo^Vw%a|Kl?ADoHw>QAjyejXX6UeQ+^(Q!{yEGkJ3} zc`F-zn29`P{U=eUX`>Ick*6*HBtp5+v#MbT~`Y*!O_U}WO z5yhnA++sMA|CbGM?Jp|Jw+ZlfT25ijc#h$tG@3yT)M_k|Hp;0h4MrnMw2cTGUcbub z6)44Cf!X%82pe0!JFRO`^?#Ov8ew(a(~xN$lKTB$rk^T=7PwEwbJW)(5*M;r`(l3^ zvAva^c99%Ij5{$TjySLqI!4@f5CwLAz&TOYj_5nl{toYPX5@?Y99Hgy%OCbr+uBDU zJn(--K`IOk7@Z%z3j#9chrKNEd|T2DTF{M*Xa&)iV!oJy6Are$B43V-bfOU!(Z(Mz z|Mh|?SiBgUQ0{xxDgk6A?vd8`^qPpxqL++UVl8Y^NsP)d_ezWb_avA}ETWZd)nYU3 z(Me6pvE2WWYyU{hXk(WZ)cr=~*h$AKxT_nWq3wi^Bj2s9otL5IoKFM$t?jXwVG7ml zb*sq3OxzT7@>CSH3qO;wsp)iUF8#75q}g@EUj(jPy@MUIi;wGQ6f< z1uy(lxmUrA|5Wl-aKk^9eHHBZZ%F=C@VAQo9oQO#;6H`-PpRtP!L30Im{bK`porB1 z$n;1$o%#}kbwi*3O+`1B!JQq$-J;U%zK^JIiI_P0xim!Wgvn!lkc&$06D{LWh6V-w zi$QqocE4*MCfC+w;Rcsem+N=6Go9ocW6g)7d+sd$0IGW}B;CuUPfFS`0px^aNqoXM ziG%wGlIiHDt3#8^&d-Ba;OhPTYowu+Z}bA2+8tk*4~<5uZ9=XjIokDCi5b zapv@nNSgZYo4xIP`DP+I1@!!;b(mZCpIKRlRk(~F9_}|ZVMDK(K`q^XSlyc%a}IM$ zI4@0y$eUJR)7;N3aZHVmhU@#MkY~Ig$rtk$R>y`N_FSpElnLQ{33%(M;(z;=Gh>zq z-CW^5H4W-R4lUXHuBKu7n8;mklc!hdVVX zVVfo}HK{{fQk4gr8cPv0RX}|{%inb&eR`ZP+-xKjFFJN*x%_S|bay|R?%Gk~c5HFd z%jk4r;8dN@sl!{4*W=E}p+2wEh!2UPJ$H#wvGwzqtJKrOq{4u~L6g$Fi5DkSW5-muvqbnp>^DcBoh3H7HXu}@^6--u-A>@zFribt6KOrB!}#2AU8m?~ zDE+VoAF&2_7sia^}Db%71K4b_Q-JIQNDf`HkwqacF>|Eq1sAv+qC$QvxH^0xWV zC@&hI45Pfbb#3_mYgF7itcpsTTOhNOs*VYRoOa@coI!?`PVDl9qIlJWREE|+h00!D zDz%-0Oh$K3+*;G*--8mV?c0&b(ziFG^{S|4bsYbtvSXI(ImE3qvZ!{QvWp*)qm~V& zQVA%Oy+g$=l~Kzkq*A|RirxMhR8h;$sAZH=sS1fyd1T7o$zqq(sAUz>srlr}-uYse z5U6DU=~M#}W$!Gp%Ud#;hD55*5V6aGe<+2jPq_GzBWl@)Oos5oQQSH$i|SXq(NRTI z%rcFt^0vj{>&(BH(k=;BrtdrcL$2_bryYM&Fv!&W3rYe_`R|rQs>Ua#ax_9Yy>hp; zu1Im~oL;I2C7DdO)U24b?$2Vkl>bEbTNR~D3mLPxwa#978KMZKY}Y<|*@0EL`|VLo zTFfaLv^1K;EDAN3mZwljE*QHQ?MfU_i=1+ZsT^)^;_GTc?{9r!{5+)jOjJu#-sL_Hf4sbnIPqoK98=Ma5!MTA0Rd{wnu{9g=WSyM0$Pe zD(je1$nef)QRS!n%MUX~89iqFf}&pPdS-NT+OhG!_;6H}n_cu0|9P8`%XE*$rRiEx z%Gd{_Q`^b@B@7j{jQ*^+m6Ta|`(G|~?Zl@3^AS%_M=fLG{+F5+x0cK)Zaw~1O&w3B ziX&CVQ7l#yD^`;$W(yLl3Kg>^iCQK@Eu$urF(;EbC6iH=PAyNOlBZPeP8F*P7PF>^ zQX!98#vqevkxI2lq4JVSt^Oy{sl8+}?qo7gWHSD!WkgZSz28+-O@hH=*sh5IkDW4O z(y#;AUV(noKvp+1LNbcb>H-XfLn59+3PT6^^B=$D9M=~f?I~1$qUE}EW|n)vlt#%G z6?DQsjGi)y^`c5d(y|l6*AE9s2x8@etLT)o#uX1hA>Lm&9xi3Hp*TdEvWG?Ve505A zk`n=Q*FwsI6*yP-i8hx>4&cMuoq#pEvLqtu#xjUIxwoH#?0LreH5ny7a@`U2M(tL z3lIn}bq@{=Qt)sb>9Ot`H$jms`U#qcMod`wM(37FIDlHdHb5=&xQI0!zFL-{k>k*a zlF$g#(1?EkE$Y~~2%A6yYr^KGDI&}QsAWAGvO7V5sdI3+K*8<11&IVle`j=dt&Yn_ z$2KCmgV$#q5)oDr9)YnA)T$aAfkor%l?MVN?(P^45BY~jprVcsZG7V-xl_;N%__Q| zbb1!KHW%0l@RfzQQq_u&&blgbF_7c)w`>8}DEDncb0~4R!p?uXDPm06Vj|?5UkPQ! zTEid1sY!lw-anO)ljt1(&9K%(%6|R3*1T%z+(v5X=wfIP9sM?~5mOl73w_n*p=>)$ zjn?f}(R@vwBqI;QBQ2es-&|rnU(QB&XpOm*(VOgCXPpedO{Oc`B=h-EYZ<)rOd+5lDd!6b$3rI^f8HTj(^`^3Fm z+Q3Hc0VId4fmQy2#WZ=BL)ySH>4cObg?p1?>Y6UucXf0F`JcN*>cJV!5b$rxJip|O zvE<%WvZ)d3DN?E_2pEd2Pkv?CBzJDa)rQ>i#22)=J3H21BAX)Px7i zT6g%(l>wv&`YvDEt8X?+1FHp+IjI?(qAJ}^OfSeErbxNMiVe3#KL~<3pBLlLkw0u1 zxi0UG7_RlB>1CG1aCE83WK|0X)F8W-)#TZ{oR?1StzM5(cl1{wo@tS8{Q8MBn+G|T zWUAtB<*QS6r_sb8DO!5PnUhcH#~v_Sdh2xdH0T^zXLzem@y8zKg*nnMq*5COWHUso z+0b}oT8c7Gg8GvI^XWE+d{u}N(oakZ4?}F$h-y;)@|d*u*c>9K7urU%zPdi~aq9e% zk~ZP0h5FTtO?{L8ubca%iG2x#cTL|}A$G&Wz>MTCW4MoZ|5v?0$wN3!!B-EHJ?F`l z-)!F{doEN`-woZTdRbqlb}d-|Tx*`=G&fo^=^P^6ZFuoz`)RA@HHLdpfozyp6n{DF znc$&UnCHSp6YkQQVs5t|U*O^jujxpq`)sL-_pAb`8(pol(FkL80&Rl}bs^KX_-fW) z&RqLkm3)8agsku}2HJ{o4f>sdywK6SFws4QD%&OSjG72wg6AVSVTO%n)RK0EppOwn z(?a6%pqJkHLfrz{LgtfcO~e7xqF=BBXQ<4{s?~4^-1=7drViIwNI0#Q1>yAisY;b# ze0X)#6)q&DT?s+6zo}2#3k#E#XqjGu$#$WBm02LLt=!=Faw~~+rSu65OT`0MR z)&=}FB7Jhge+N-NG;r!fIxBxMW3wG9wScB3be3O;#aVTnFU^odxbnPQwv<4Ko1V~E zfojEJ>Vi<6WFoVXvL9k&Ik_!ss_NTW48DY-2EB1x#Fw?EHd{O-WgCw3H6TxrD)@&LfJ?4`sQO?v(R}tuCt=qT~ldxzyI86eT^RAc6rTgP zS6h?j{;QBxh&9e>;}*@_9`evo5_0|_$QH(6H<9ATKA<=%D0^<=87|k4>gaE2We%Bd z>~1LKoi_SZz;?4iBH;01GA=1MJ7+s23*YELCLBEifgSIG z6L&S;VoQ{;mD_sNP8e*J_QOj7yxCN+7IsoIdY@TUd*OhwL{h8$NtM2u;p#%~oKm7!Cq zkokD{i(iwW{qd{^ppu%E9*>Ka8eG*n#^x7NS$Q!7M0IIw*J)&&11@VS5d<3QvP0rN z$N}N~3S83PtUFhB(3`|rpH;+PRX@#CNMIoXypd&T5sy!j^<@60Ms;=JA;A-su{r&^ zHdeW|){(&oIYw5nAD;;)?_zCCj_;vZay4L`?)K(>;G4=WM-ketWb|p643WLBFtt;b>i)=Ni>Pe_4GwJHz=8OW&J} zL%)in!PFe>)7jciua0AXbWX$7)nazT;hF8@neD%q*1Z``g>)7T`(XvVK!YLrOV4b& z8is~`_5%3O;DvSWg|hzcV|RAwamG!MfPm}iN7e2-t1$ASJsHgp|n$NfvNfirb0?4 zO^ihrR^=TxXvM>jz89f#DVj;=CsuPrsU$n$f2j&BRWKxT38y7Gab4QZ0t96IA4Tzhw8qFeni{)VJ39QgHzrO)R})PW{Ry}Ui|?Y|)g^u5|+4=R6P2_g8hkU_Lnnp$;N?P>TR ztj#{f&+naN&nRs)Hb&s*x7qEk?u0mmwoHK$oZpyc7ySO7%ljx{R;*1w`9MyHPI@IHkaQPPG zI|z(hXD*cP!i8M!)TJowcTb-a&3Wo=qQk+%RYFWnSF(m48wq`|&E}>7ZXr;}!#Tyb zdGLNqaYBX@$>^lqfgvo#uqKhLt#yC(XtL|`&qqAj4X9qqi6?f~hAZ=yHrG}$*(pJ} zLLIRAVZHCy|K8K_Ff6dH*0k3#*k~9h5%*7(rsQPOR3=aXaV25WS_szmF4hZ8&`iQ7 zFnU8gu|3?X<45e33tc5VSG7YdtqV2H9+__p=2$zA*zJC%)0Y$!wRIO%#B?W}dVs*Z zegG4QZFDSN@R^m9IFF#|>IYG?T5g~XFYOG)Lg8t+;)qSsbchU)BY(@6@hGxMk&oAWqJSwyFHI_v*Q@zRGPZOxg?`H) z6c=86!PepU8CJd-$P97(8faCQqwVkM7B3u4VLAmu;jBHL8cf?MQ{9M@r?Km$lG3XX zS{p~fzYnRW=>)d2$`_Kuy+BMOKvo#dV~HJKC(gvyux(n(y0hAOutIzN6mmu+?(OXj znklL<-V2J)iH%d^J>5txJ%^6<2_d6A_#Kd;F2 zT2-;1VnFrY{?32JX2>=pR`G%XwOPmQZCe2kSO=DgT)Xn8+ed7NAiEB*A> zYv(~#{qCTAsKdDe^#jSQ84PX#CEGige<(7Lh((%Jb~EEQjd4J^k9PJOevv@cPe5|% z?^DYQSm}OI(?`0)J2bqW1kqtwZtj`kJ z)$>0@ze~Pd1OKchlIgz#8VO0&^of7yuLiz$3NXd>_|BCVpOaRiBzbOcrdt)$atAMJ7p|W6yP*!oA z?28X(?_jGDKMRaa4p2VsQR@*xbFfE!GoU~AZdGpPrbm!^Vc8A ztjemz;GCu!t-kD?6MljH@194!0Avt0<3>?(*NYy-(v( z6IUJmo4;Lpco9;jiEdXn2i&T)t`1d4o6cURkO~^&h9wK$ScWBK9-gQUu5PW`y+)nZ z-EjUshVX_Pl;0(u@bh5*METYyWf7$40Dwd}>*u-m+2<4Bne}qMKk$iofb5^DG^k{+ z6V-^_Q(84&iU~*SLE}qNm+d_FTm(IaDq|*f$F!=aLXqmE!i;Jxiz~hS<&=TFA?Y?} zlIYeW%9%lI?d+Eoi=cPA^vhvvi86GVu_TwauY1?@ z5-pmvHOEZNg2gRKilORwkA>(U=b ztltrurWVC-lM@{{BDqa&;&nACD8p*6#~wCoh@I%#tN8h8k-V64Vgw=#LyY}oWppQ4 zdLC<(QdFwV>2{7KYD~0wJLVrs&PRMKjc2q;Z>4A(UPqnIgQGy(owRDlyOXbG=QFsB zuTW#iu4>IQbG3#i$%)D#TIs4jW9~!xF)7NDKWyx>L!r=$9wv%~kvF{M3`@FG&TEF8 zbjLQbp4OIl9FHWqI9d|dsf|gDpXLm|fNE@0zyFh$Fbe&?2*VzA+ng`r7(VDJoD8;3 zQgfmlyh;>RfoZkDVz;JZS-Q?oO?BdCy)zk;*^Ox0luMi%y7tU`z~MUHT0Hm@m8a!5 z{T4-K39NFP9vGoQNlr2*3sK<1im#p*IOjwh*R74J1Wra(6n=~S^pXy}Y}OG!0bV&pSvl>3hKbK|sjG>!$p9Ni zJ>_v-wM+!cTF`|OlMKhN$J4@4kE`%?{`_cbje?%O;uWZP0VCu9UAaxg;)G=2PQm1XRao}j0RJ{(J1b^`^JbJ~L5GGTD85+3eePxo(^ zaiS)y4U=P-=8aUfuP^;0Y-My+JI^yE9SyGRyb*j%_oO_1^OO?|+KwR_jc!G2G+fMq zb+cw_#mv=vv@OgUooo%88@X_rjQ4ds9*N`glA8z?NsK@9F}8O8J(dJ8F@&<7@T_diCo?FY#AbngGFbP~J;fOpp+9K&^~F zeX*@I`9OUnr9SOwih$cAT@V0q-1-H4&_*dxWO=VDFQ{WDp+HFu*n0K@*7Bh0&wEAJ zO_`?qDFiZ{HeF`VCKp&$y;dZKUTSD(R*bmF4v7(;NIgb6!G=o?ut{{1cMMK0W{lvz z*!-{6?!t)%F~#M#aq7}Hv>DzQCafhRgc}HbtIe2ntFqRwP9Z=cS^TkicSdg%T4x;9 zRN;FSG?Hp3DD)+bLD7Nv&S*_3?VxSxZYYZ(+&dggih03;Cd~?=ef7k3YtSDyAx%$0 zdo|HNn}F&9annj;Oej$}E|$(NAi=_HG4~^Fc24h;t5w!eID{L#hR?Zk^VH8*s1GQG-Gd!9yl4p(JQ&*b1z_h3jZ6yR|qS2g`es*Lhn&} zld(vBI0Dr{wsRyiRFQlqjSBhop8FTwAiVbe2*5#^L{2eYW0v}_xLqcC+Duvs3ENBO zk(?F0;ZaSjGa>iXNR`guq>(*XMB%>hU-o%ZH}37VT#_msvTB)BNVMgR*Kj(lesR&W z&=KqxK0F^p(-#1SQwQ)FKnl^z`~tvjCNCPP`^zqDNyr(+yWC*&-s#^!lPZ zoR521S?Uj3b9wAjlwQ~0X=YRDA@bi@f^QdJzi*xQ&VA-@{U1VJ2>2n5FznX~kwK%( z0#r>F1Xz>%_jXp$6%g3$31Z2-=TdPIQ?01hiiV``Zr}_74mdKBrt&#;zo+z z*sL97MmZ72OhU)p@z>k6xb_OR)|=fPQgIzptry)>OM)AxlZ_W+D`uTonmeqS@Zu(B z)vBq}rEUmu?6#A=ZW|3e8=;pTe~8Hc9W3nPU&4&1JX(_CYR@y|41KHaS$Q z7K)yCW5Kz#q^>?#I#^RiAS`m0P(e=L{DT{Ql;p;Sf;6#xY|Xj_h|k>0!<~EF6JtC~ zHaU_@$&oJ+4=G>@$d&P-vv`g$y-xlV1+(Dm`zk?kRVrOnnhDRUydoD}(rUD z7EpyoYJROXFx@r#sleeW5spcTiJGc388sH_#Xsamf`!-&h5POj4lmJ?4*%PuI9`Z< zxYJ;Qg$+6PGE_eKpoaEJ6KqC&jv`oDuAKp(VN@3W$E|kwZRN#xy}*Gu%P%W!B9I>A z_|;*WYc6iBQ)mPSpE#+KHZ7Fk{ovVt$&1 zwHxc2O|Q5FAp_N-z?^aDXO0}a0}N#rzOMEeSIYTXTL&XQEfR`-Nc@Zw_j#Y0K|DBN z!xfffM7$Nvu1YwqQ&~77K5kPK5;_AKUeyIcIfm~q1-rwS$2}_2c@8(5&3aR)?#v6Z z1?O4jy-?FC82>p?`Ht`!Xje*eX-l454p-_UF-|zMu*}iyEIA4;Ru7XdT>jU_a9cOM z7^WiQ>Pjhmtj&~2c5}1d7AY7BM9j^D&bog#L$(bzFGNZCqYVk(Z#4A^Wzj#tkd)RczQsnUcyuIZXAP~de&)ag;{p`#~E6;3k7$UTNMZ+&hokjF>vP7o&lS_|9 z)atl)PwktnG;jOX#oO`wM!NcN0rp{h9=_;R8^2)W+&=o}?4uHkgK8~9p%s8~>l*^! z^I)Ot=FitJwC3F$e%Ab31>dL^ZU+NK@u$N0h6BdVj>dadxPo-Q6wxHN;E>`zppG0` z)E_D4>z+BVOq5mfd1$P>(}mh#{+@k=`2J)!*Nh$p&#nuUilfhne;qu(+iMIe@A$}AQKv?Ve9hg znR#Qddf#8}g$wQ4%GOP?!JJ8dz)<6RXJUn#$j<2NA~z@ zpk8%2!&1RT+_O0M&CV-9f5V}pJ6}`M>!rXa9`I7F=3OEa`o-y4`j~uhZ?<_$XUbuV(eAo&Y^N#t2LEHb- zhJ75kent8S^+W%C3Ua^E2B@miZ@#bK$)8vNlJbh#IQlH*e?feRjKuKxM`gTHFr`Uv zI4|)#ftA>ErC}OIrSO)@%zztP*;qn5=v*nNSS*WqzC0F%ZZ-E9jML&6tljS)HAOpn z>qk-j9B=a5NPjXp-f;Wawy|>Ia5!#GPcQpU>i@+V8kf@Z*<$kiPW;!gi@iW5!)|xe z;8#KX&e0YqInl!<3f*6DjLlCXt?JEv zxz%}dR?`)?i;R!u_y_ie$9gI5C(}&W3SNNNGOpVBZ@l(M4^IDVwLHA#I7K}GhFPB) zrTn~)xBMmhO8M#ijdg~C15L_Gl`QME*QV5NbA?_-G>p$W!jmk(^m*DlO@ZJ+W-U ziuF_q|NAe??PklP?9*Sgy$lQm+LbNTrrhYc-(NwDhSDwtBP@ofHyJeV?X6u6a~=pi zI~dplZ)ytA{`phan|7#Q!@%?gr*@V8sJC^UfpluRd?#Hi`gVDae8b7!r`oOTAEexx z^ScISf(4Byix)2WG=-%w{arjIpZARyawqTK;c_OyNFWL;Y)fv$c1vt8UA$xN7&4XG3XEY8~-} z%_lLO@&+L!jc<6Ov-KlT?($iKyM91@0|AqjViGZCC#g%QU)V#-BUXF_PW-`0afRF` z>B=$FmJG0IQ@~G@R<9)!xeW8<(DvTj3>fiY}iJ7Ox>WJhE$qV6*!8qTM0>kaku!oT9K+<4%)dqszB=1Y~*GG*%xK7|+R|6&S zhpI?lIWZy*+S5h)!z&tN?5$rBj%}^lO?K}HhXQDPodXgP&W{tH4RdlT?Tvo7(VXus zccJXA_lour;qDcr(#f|>u= zrQ)QCofc9bL_1*OiskG_b7_(L{ci)fjfdKq5Y8<5()0TPiPs8J=yzcecp(PU&U$wv zgwcI4p`IC1(*!~&fnP6-^jt#7TRW^HRG%Fn`$Dn;>CD78qa3z2 zNk~ik)`e4%s=-&`i`hWsd7ysk&|A(IXh>_vdrIbC@&R6vW_ZvsJp&Z=!gr*pxNmoX zysbp+tw}%TUz_M#WFc`!puWO5QR+xm-4$kd!f~kyQ7hk*>-NSS*jGrq(pM`J5E6JY zeyAz=@Qt-uLt~Tm3&N~nTt))n+fgg5!IY^oR#m0&UAQLvzV1 z90Bu&tJZv+%m_o6MTk|Y8RZg06GV3AG@K`k)TIcZT8%*%a@|3T|6>iO4*6mnx~^`t zQPu-UL0hS!IxMWmHg#rraO{EM^;_Y)ZJO$>5$nsT>wr(i;@Num7+&U9A3f-a7h|TzP-^#k{0dCZ~pKm)mvRm9g z+t(+XNgFI~CQEd5CAp2Ssow1_feLeXhDdQ7PJ9F-xvUpUjq)SW%zEGh;WY9Td&_r) z?0s=;-Lrc}p9Zlvgtmg?cS^h|r0ehWN_kj^0XR0}nX^zu2b0PNFdMpD4bS>Ft9NT1 zOGmEyvz?7b7G_z$VYzD2euhq9<=C8o%Z$S;@^*Hcz{jl=k*Gqr5#hX6_@u(dr^h+7 zDZG%K7T}rQ6+92kW^Q}!a|JD5iY=Of;?x^S?}=0Q#j&@UY9VZ-nvsFWvL=298ZTlj z?=7d*-rMy~nK*^S?BD0KPS{=}9ryh;AHk||#G)wccg~c#LYQFUzK%;aUcQc7HmgeF= zX6)eh|DNuu&MDwBW8qQCiVTCo*uPbjp_6HC|1MJrKy?=WMZGAh#Kjwv!?g`0ZRn`< zi;V^xA`?4bfISvf`*RejhoO338E>>a-vH14oD&K*41fW{@>pCZ0f%pyB0)vV#|2(k zu4^!Y?LH(d@L}C*%XmyDo-aD&awTeFTNCiU;d5f1!)8QkG@bF?hk>#fF>DA(XlXe0 z7cZAwQJqOqCYv^~C7cqu*=q%Vm=j<_Du9MebZlQh8ytG|i6%^6zClrwEJiRB`h6?j zh4W(k|A(-53KoWkqQ#DF+qP}nwr$(CZQHhO+s1cn^}p?Hr*k`PUy|%h)=VB&CTlV) zdq1||=Pg7X(|u|TF%8Ga5_!_}L(Mig66auIhUYV(Y68!Btf6;}DOgR#9g!`Q{P`iZ z^RcZbGVGq*s7MP(ftO(3?dbtsHS&vxoK_+HRz#xt*`2yXo#%13{@3RdcJrhHHwXX#d_({Mn*R@qDE!ZXvayq;!~Y~osrrvM$||a# zy<0PLH}?lY!v$EYHEe^V1QFUYn`AK|n5HBUDPdwoh#@lx@bnB9GxmU=1@w}dcI^+! zr|2bkZE5Y2n`McBwl#l?{@U+e`Cq{NvAwT{$mibXZU_yaakej$_v;+*dEQgrTaNb~ zwSB(_-T>FPvS8@I67OfrUfR=)jCTC-1^Acjdb9vRMIjPlphCK3re3=M!6pY%U{pg; zL>NU&lATyP*>+etj(2|_wK0T>EY@B^dHEY0w^vp_^5hu!=egNj@6|NM7tHiEOfeQK zloczDVD5m`cBV~pbJ!_XS8|@!^i^3ZqfWc+3M23EWh?FaGVMHS#PjPL3~Vg?E6g+G z%B=Pgh4L1!4YL-9b=H3wnA;_Y6{zpmg0p8T4RQG`}D(ifN~WUh@Gv~)zk@1 zZ(8sE)QPP)^DVWZy_^VEwsprKMK1M`Q<~0h7#f2!g zK-G02S7}9z$698GuGUe;GFk3(B(i!4d4^0Ynb1ZwcM6YuvCkdK1f87Ou+|bO;6&DM zg&l7hbtY3Gff!1U{()F$iI)_!7tXj_k%hPdm=a;(@|dEuIan<49?kKSX?V+vv=3P6 zqfJh7+X^r$fqEor4C1DS0?M4xaau{9v=hS9fKN)ZOw|I$o&jOu zj|kU7xJ}QQkg;cC$T71d^xET~pYfC$lnqXcz7-myT@OxZ=6B3VU^1OLzeB);!^$RX znz2wB6s^(3+xV=LSWAaUm`W)TV~p!#n27IJfCS#?7sjK4n?+YWbz7Gvx`q3di);tg z5?1dSA{+a5BoTsVkbaukvmslJqOH zsp6Igh~FcE;;r5>_~zieUQo^qq})wMKYlD_ta!=}k>5$UgHGY+?NdhO$`9^)YC{KE zo+lNGI6{cA_()+xGWKM{_@6OhLE(vKPIxauXvq<0jJz%+FvkNNIo1uG{JZrTVodg8o1=sE=L9X3Cg0;iOo%5LrRk zuW4Z}m_RI_UpSA{Q4Z?`9c@48n)*#^&Lz!kJ)gh2Z@YD*Y0rhPM$bJzen9eO(2}|u z5N>qsHEN+}EY8Lc^giAG;HP1sF(+a^B%Uex2|($gcAS2St1G!qLwP1qv=A+RO=GR>Y|1*~5cwrDh;O z=MH2)Ol3HUhkIeEIdXB^&1)=OO~Z1n;Y72GXBj18 zwx|;DWp1+%ihsfZKZwl-pIoPeZ>R9`21l!(QvLnNo~N35^Io-ls0knZ7%uO+Ru z-8MF~yoC|t4kAyqh^l7Bs2*VSqC4JD%<#}CjJ#^b*w-s(Z(oRH**;R?@c{GYqFSFZ z$RB`#@D-=P%L2q+P-Kz5R1DNVRtQI`kD+Gv(D|)OUn(zcp^7J$*Yl&*``|f*KjP$z zzc&Ibf%x<7DX<-~dOnteDT2;!#M7p*J@vGY(7t9^?trBeV~G8Fz-Uu;hnYVh#2JSp zbHpicL_RM#HudX|V$mB1`4z2YXNY7sJfgipq+*M-^6E!Q+knu%n23VJGw~Cl&l81Z zj6X#2ld;4W{)8m$tmTi*SBi2{g`QVl>Nq3N6Tq$NutNXyAA>eT_Y2H_vdvKZ&UV71 zT8}t}@q-jiicaXI;FP?i@e>7-y2P1F4ynQz2NgmQP~nN#KWIO>lp?xE@*5XEm{yS@ zP-uDmDr$9pAFesWulyjN+k(DjBCmwXLj7n#sZnZ>9i$GL)*a#kogN|8+Y}kqu0Xdh z78L9af;}Qi(;buYRPEDB*@4uO5?fX`p5-&^$7#yISgL^lrAf_w{rOJal<`~F6YRS(y1&t(ja4^RspPI$(9MV~>>8vW^MD>3;mvol&=PPJ{#1TCRWfG?KQvi%W2>(0l z|K`>^cIO3Omm1YdUBMaWmuMC)IleBu_XqrcujY*R|Dj(TEDW7Z{~tA-5jU=NglPXm z^#TA$|No*zOw9~kZCw6OwJc_DV`6D%F7M!CX=~~AzgyoF{>QriLmb`uf_tclzP)iW zySq0}hlVH+i%=Gl3IHIMLI6P`afJ|39U-A7&m(vL*UQ}h#BFXimqW?!d3Z1!4=iAB=>dUp z4zu9#M;eW;Hbz-#dWRn*mVg;RgNY@~CUnd^1U(eWy1Mu{)W|#+k)o!eNDjJMpiY$=^g~U4n69+^t6PY$XCTf(*HLn+Bz8Fyo$FmH@xm1gy z;x&*M*nvi<57YZ4?>fsFtLwF-qW=b}O?NkpABEV9hIC^|c=gLBjn$lPxP>z<(scJ@Hqm=7*fOv~ z7mT5y*|-}%B{lr`aAad~BWVKy8mK`djI;=|6P9VX2=&l|DCrZ`^s3rQfYxHg1Wm0txIra> z8`}_sxKSkq-bOC*u#=!{OdoPl82DsBDIo{E8rXpkrdlM79PlCup^Hn0SDcit0W7IK zpYkZw5NOa>sNJy=R21|JfKJH5B%sVm4V=14b=?zBhe(+8Ayex1##F@CXpV1xVr@S_ zhfo;xA(iu8UPna|iro??rxUgoUR~7{en|CU7Jv@*FjvRi;2?S8VQF$0LCc!}1qAy* zEMSLZSoR?r?}184x{8XpnK%VS8krofx=j5Q0`wr(2ElNxL-y8$i!>q2tqo#8n1o6# z4E7;hNQ{N(%ZY=htO#VQ^hmY;gai6cEP@6>UVLuoLXWc~Q;e7dqIW&a+u#pL8AR~v=nCIj4%pmH!qa{ zzVL)h52hL(w0ii(EHaZ#I60zF^GGtt4a38uCE!;K!%~BZMk&MLKZFi$$`t7ys;Je4 z2;?I5`RRM09tp2`6+F^}=#lMo&`0VaIMK-oE638?+oVHQ58WroMyA@%6V4W0UUb(K zcEmWTs<_9;i5rK`Bl&3bs7sJjsDW(=TBJ;=>f^gt9`S$GGS$s& z%|@rpl;YVEJb2M>*2>C=ni`U;XqlU3SzIF7$|9NmllOo=6lS!kA`WF4%CSt*5}6Mr z5~RKSVY`6uB4P4}%prSjS!v@&&el}W zgdbs)Kya=wHF%#qnmAn%32&9%WGbT(**Cn!|420~M(`nOIhug2fG*N0${hzOI#qlK zmCz<2)?{Xc1`ENgVjd~?(d25s8GW;bPpC}}dVMnpn3!`gvIi`QI_glo%8g^glK zc}#B#5ptAx^w%B}Uhz&m6)xSs_|ylEmtL=Z`LW-rACi|}4@__?ScCfvqK6X@OmJ#W zIV^6$9r)Bni5+R-vY1?iAwv%%fbeOJAmW`Nl|}4{q?IvZR7t|#i~{($`8q_z3J)Pf zQ@8UID-6`ys(nx z9GWo4N~p4a&4qw@Bp`C485;C~q;p8j31{XjBju$qQB)Y9z#O_`%ZxI+%~f^+t1CmC z@@P~klN`z$Yxwaq(Xy^%t<*wZbYEa<}`<{+M&g5O}6-Y0Nw{3k1Dif=yDphp_QYCbV{3z-T8e^nY=UB=l zc>j>#=hGZBJtoHFCI(I+IJ3+ZN=duJ5b6-Ra?fvW!p6>2!;TlLG79%TgnCu&=IB~; zs>7&NVCT^(mmEO#NRCIn3i1Y!Zlx?dD`V0rwb0_x+tnvLN{u3q9!@CMj#dGC#Kx>0 zb)||`uT!*iP5fj#dZx%scr^j^=#OEcelv_5y^~|utBI8wk22S$I|My_k3Fz!R%5V;6P7DBB8GMO?spSFk1ZN+b23u+Y^yv)GibIeAoFjQMN_-?-eoehsP}=5L407 zRPq(k6Askr7^ z4YH-q6iBE&Jzq9om|RCR5^>f9zM z)xgkVmggy?9xlRI_Y)JkJL#Gv!u8Fd-U;+<&m(9(ko+rCs$1!*?84Q*z_kX85}&h+ zi5|==x2l3w;f1Tq$h689>e^z^+C4m-fdr2ZnxvKUPYAhjFDJlBQC-Lv)R0fka2rUK z+*yFZGY@wCT&=m(DFQ6k4am7R%1RA{Y-~&{I(l}SI?HetlE4v>VhqJPEWjRq_; zw0WzBs$8-g4ua()v%F<#&gV=@4w$Dq1;`nbuw9hc#2B%z*MDlIyPmP=uogf;hOS)(sA3qWQ&gQL)VWN8=43+L<+49nQn{lvgGOva3g^lIta)lao zGH5KuOw}e&h*;DclVWa96RuPv!7Wsos?As-X<;nN!p_F%|$W`gfj|8rnGjoR1h)l9d%P!k^B(({fo;IXHZIVc> zQP1Ht@Q-SfHmuzO>okmD-cjqw#jW6y)+TO<-5TpOq^}OkHFgC@f_tS*-7&g#?I_OQ zr<6Uc$e zLT=KQ8|f?*WnwzIam!^r) zBflwkPjD%k**l^q22rI&iA_6a{VhPZu+jqS7`C(X6PHY6?}7N^2+-KSQtp8%}emlO@;6hqZ1sGfAYrsHTvIMQruAY z2k%p*Cw!A%^5)$Qe2Cf=PwiYd&=EA;Iw{QcJT#BQK7BLr4ZU$#)+cNz)t2YO%2hKo zyc7H+4#!oW^q0cutUIay{1IKSbEfo;`HIm|H{QAyi!acLpqjuLCQpTT0*T}t9*Hd{ ztjSVU;oj>ieDBVd-_p4O9*M~mPc%E)d4(#ZowNvP-k2_kWkr*4HYb%9BBnC&nJw}v z2A6h@3B~%b$!(zuulNWPd6&<|xg%|5KL;b;dVf<3|9YMR_@KHf8uGY^6E1791#@a2 z$7yvH!%*cx0E=5Ccv6air3+hZ9;+iz-M9QTG4El~qlqDJFDjwSfTAGG13;gjp-vOPv@xzFpx zW-m9Muaq%`M7|SqXp^Um@I~T3iin2ihPa%|>Ln)4)RPH-!watvWQcQ|e-tHtxI^E$ z#m?ajp(b^>M6n2UdT}g>QC%EMB^j%;iC!@F%jen1vD+e75WUyP`@}T-H%~#&t5yiH z*vq&faeHN=5USyZK&gD2H5NPHvSxx<&7vWp;nq%>d~afl9*^J&5OL#ANzG9O{!srdg7#tt?z!J zDU)$yqwbxRueL&PVm`dtpDPwZklmgZLXu%mOCi6~YY>%IUIx13t1JsKg=whz+6IwJ zWXb%6i#a*7;OumAb6$?oCnCGJx}VU-FHi$XYTr6Q4JWOF?Eo{^Tt>Y+x>s5i(zV%W za=TkrSl2Z#R@I+cj6&e@(o+x*e?GUJdTMgIobHWHmYKV=Q(pQ@whSF30cW=m z?HCNXx6xjp_+xZ2()WoQwT{qyJ-62@)^LWs{HILe4Vx=Y*}^Nv^fbP?S|Z16_?XX9 z{=o=_n}&O8HR>s;J(^cYrlmL4L^K>c*$ppRC6b$xZjq--?zy{1%y-@^FOU3Xvu@~r z#(P=rPyClQr{(hfkr3`Oo^4a62n~Gkr)&@< z{!!Gvp;^{YnR6AMm3U8$Z1StgbDABW*2Y}t=c3q8ivsdnCC+&bzjW*$1tsX|*-i~h z$1(Dn@RGt-E*5t zaU9+An>_gub?W|YT!*(y`Lyt%-7n`Y@=5E~1U24bc!dj&-X;GEy+WGd&N;umXz#Xt zj1HO}P2UWIwyV^A`79*#Yi(3(Pv%(>x;h`EKDvMn9}d58NPJ;~P%Kw1vdcWEEnj7Y&93ZMoxgS$*}<_w&f{ZjzKSr%PTiTexyhLqATPbG znM0-*akC$&c!|=E-MmC@h|-^Bt`WvpfhV|nSb;h7dz_%Yt_3H(giOG&GXC8xz%X>+ zJso+iw2vMMc*)eLOU_^fFX@p<%@?3Sn_RLYm4gZ~Cp~>gjE^catNgX3B`;TFUcWrK z{A7>91z{HHb7_-bPPXrmGYRA5pW$hG4W~b$8G2o5*e#Z=tJk#kTni7mtPu2uS@M}A z)`{lIrDrsPmAIdg_3|0{&QbKj-FX~M!JK|aM*mp0Ja75z276j&jXv@idBLti2RZMq zab{T2`Q^MZcbpZ8r-h%-W?;@We@k)I(+^!@h}=d&JW&p?<0`$#e~h z>Dhxg^co=z&m3KZuuXDafi-;eBAls^-$rVwUL|qfqMDs80H`i0qY#ofMZ62PI)^Qd#S9 z_uMT?PAUR{aeS9+@>%*yh?e?j@{9VH^^Ys6TaIaex}OA5SLhw-FBuO#VqDj>FWg_y zeRov1aEAXpe^FQ7ab5Z>pUS;=beEB7f0nk?AkOU&j4sMAtvle}KQF`*?I$ zAhUn|Pt=_}qI;0ZKfGTWN1BR?$niABwuFiRH4%{4wIYBe#)$m92Evb(xy4k`fZ$ga z9L0YQJ|UWaT}8rOLS6qCZ9q1~pa=5jnHs+Qq3l^%jrrcJhf`P42UfgOH`W@3MGRhlD`LpA)8~CP)_jz zz?|ga%HsZNBL2u01r7K5{wiarn)YGpKrnV)D^?wF@?RftR&=yreYsry8(hNZC;>U; zNtUuGpo##1kpsv`A+3ra-wFvL4Fd$)y<1POPJ(v)%~>e(Kz6? z^)nYTFTr2=(pS2eJ=Y>;K@*li0~V16>T>bnRrcXnHq!1G@d`U}414hlyRpi3Jfmhm z@g)hC8h~2D%RKpYebAl*;d3}E0~lSPqyuFRSUR~S_wIrl*WJ()>TcQkqMKJzn_oiQ z#N^_2F@S=Pugzlj7*-AWMGd*;YhCL~QhiyPPps_S)l|3VmF{lj)PJfAnkpHVO4*gF zTuZ4luVP0)#jd4LX~Ns1Xeqhkd&Y0EBBxZ*Qq-xuQk84gfMs}vLzU>d5{g{`buUe% zBvr5k^=>p?ZKSPAxJ1=zm`BKXB^$$c-mk#`MG;dJM$DN8b5}6tABmXzCIeU4n`HcA z%*3$e^_7&6dVkeo9)vOJ% z5WMobKS`ttd9VsOUjn=mg-exil`>Q%a>R<=Yot_Eeq~I@fK$XERPdlx?n)H7;Vc&q zh0?a+EJl~!j(kym`rFf~1pL63mIvGT8-R{JG;O!*Hq3SVFl_eW>s778xmkxf@eYZe zom7vH0W*(AGinjzHHA^gA$pDBlD2W<2{4fDZ`#f%W8_70i(zXGl0>GALsd z%h-4gDVs{yn1J;H?j0F_CR~A>a=_PzLPy-Up8Vgbs6%vwjz-*`BZcZrc%+?BrO3FY z9G}4Cy!^@|d5$)s32w+Fpayb4HL#Dz^;+%Q@Pa{j@yI*_FCU56JU}$qK@R{9(<5Lp ziCKhBzc6tvb6mmPJK&eXZ47Kap(ZB4U_TU_F?28U5u%p)h|ck1;%6~hkw2I|Dt+oo z^HJuOGGlsOQS*`FkI!?nEPua+jIA}I^S6|+eg4XVGYK)Lxcy^&lutI&he-1JW2A|S z{U^3)pZI^F_}XLS38q)R9@!&r_5}H7OPV~%`PHsBRO4**a>0*Puw4*t%2{pO1=$AW zVoA0EdTAp!+kFh|5EN)@#;HxHQ@mQn`63i5UM<2`y_O-d$tyI(L5@I^A`m)s@v{S~ zRH=CJ%$r3@hj_8d&ptzxe<6dE^7^k+mw0gqFP3STcyY=P?awGWI`F=hI`z!6MQDUL z^~|G1=%#-(YKhY#T3lVI?3|iLg|=a4FRDnNXcy{lbWk)o!H>2^Gh)ySfcl!bw*hU^^e@_)N2&pD1==1l8_|MIK8i(^P0C59S3~EbX16o1 zRr`_sB3#NVe+k+@S2N!@@wMqcxV7v6!w^~1awGS)4E6hu4PDrQP1zDe18IvO&ILqm zL4F%pVFq+DB_0mE_#jFYG5K*f2JG!&s+nNvM&O-rTtjHLAom7l9~r_|LYe`yjo{z6 z!9Wk3xY0TdsY*)@2x1P5^FxFljPN5ZnxXIp4E$lkcaXe@yM-5Fnn7L;ptmCls2*_! z{5ZqfhLo4V${GlH!(<1V>?r1lkINetm}X9n74m$2UPti^hec> zu>ZOT>wBXG>nxCA=Q>t?;)GtfQYkvyMr3nBQe~r1npV`1%jdj(-u`!_kjAVh~8_fd8HJ%2VYeWs;tuZ&ootUti)G%)~vtt5ljK?U} zIE}Haa}mvUna`SrVyM=LjUlfy8&h7UG^E$BHSc1yHyJVIoB1&5n~a(J41I$qz`(Sd ziwyX%(Qf8HLBSgp2a(vgIPzj40Rt8R~H-srsLwaqYPu9jxt-ja?f&bA#$f^Qd?`VS0o z>Sa#f2Xr3Hu;oM&%xi957=NHoKv>XMLFCcu$P2piaPWIolTQLB%1=SE4l1)BScDf~njXbq5NX8L1}R|M zBq%k&(R2w}1vbhaaD)SZJThSACjo@f;~1omJ)*!gKOt%|rTEay?FTv2RBQ>Hfwmp! z)N{O<`u=$0u;J$Ko~Z*OciCesGlZ*bltX~`Fy8yU3&X*WGN0%}Ky z-1y?mt0#u*#yGkG=uOd&SZ!eU#^{D*Z;X5rbc3`vOy3#0f!v#@A8gyu_>I($(QR^< zZyP<}@&}|VA>qd{@+BzoV>wTKkn&;te6X9| zQBG-D?R)Q(u1NS)pLL}uebJrnQT>=w4_QDrqukRM2J6R)T3gWQnh#uYS?c)f zoqwALMD3rWE13d!A!I2`E+^~){cLz)46!Q)_o0qAZ5cvWhg{ka&6>F@9`+%vFT6Or zvx6>V?@_jax;NO~Lw13^H<0gfx8eF5$qx~{p%QsxZ!fn(ZwCZ#M0qlM#SBNWJ0fmJ zN^f|1;(ZHP{v8?(3=)0@1pB@Kg2%6c&Yn#skmr%oQvTCOm;4;ndt?3{>I>A*uzyJZhW%#1Z`|$%K4;{2#L|Yp&IJ4( z=@or%0`3U+2EpzqV4L$OI_DT$XQ;j-pAfzgW-q|hIkWW`R+GXf&n*hAhDK{xtiamiv&8*4f~ z(a5Fta9q;*{XvNqpNe-Py42iL|I(9BX=_wm^6qJIsoW>#nHB%qdS?8?>D^=BfNxNJ zNZOb9Lu6m4YwkSRH~JlAAIUH2&a12Ly!^fGe%ZJFNwaU)C+-g1uj&q6?~r@I*pI*d znf>VLXV!DapQrv2{mJT&NAI})5c=Evk@;tR;YF-(!k=6JW`836-F@lB)UWN2xo^ln zK_4OEBTIlzAI)N#9^U0l9mMN_8u;g3HIUH9mSCzLZsvR)WbMHk`1+%3p!U0KVA;ht zZy&)Ub|1qdcpv3Gvd;hh1v9gUjW~Zl*Ab_my*R{v0%I0`CSwABiX+sX`RFAxaKFXF zSJI#zt)3pfr+)Oh8p!_Y!E3D2t4FT#^4kTmUYUwjPyMm_q3cmDdYv@O;>~)=0b_It zya4eBr7}3dI%JbsIp`jBO|YX}Cb6GgxjQCj#}SX|vx(`?f9(N<{-bBJ@a#_z*3v6o z^|!z0m;9a!1h&=kGwQ_BE2Hd^zA?R(IPfkZ896+Ke8IvupOM^OAU}D(yoL%-x-z`& z=*)FNmIs;HF403HX2ZqTgqn|#43#K-^q>Y0?JQmp`4`0HXuB|hACmV&nIWQ1_??*{ zh#&g+Lv9rD964Laq8vAuNN?<82+{|PURe%VuEM#`L1-x`0^WgtuySgS#5CV{ex27h zavp1n&)oGhsb9iHzlqZq^RSsGw=YL_Dc^A3YN>v$ADTjWNU=rwOva$5Rh3FIwbmxP3aY3~sb*GYA-yE#%&$G%y2WTNjy!0inj}oz}B)Tvovbvso-mZE9_sE)^LB8 z)jNDR)^B3VZ)D4Frq#Q6)uH}!#i5m+az2)xK-JDXn`X5Wfodu+nJOG-#UpuTQ@#qP zK-ITo6^@K>D)YzCXGK$lN@w=2^13Zk*_+Qz z7-4T<=`(=jd_!ViO8%|nCsUFicK5E37qPIHvGgUAq`d|44^H1s^0Tw#C#_9y=yO@v zGg{gbcJi|g_9gYZXI@D?$t8OOHf;&Mp~QS^%3e-MdqbihpZ=-jr&W?4t+qK~Z!BSN zZ|SodNpEZHGxU4te-Oe;^~P4TC2p*xZVf4W;ic^@iT()v8_7>nk{>3w&d}$&uxBCZ zOI!&raqP?V$7vqP{+ts15xTa7z3BwMn9^sTBs~M-A1J?4@=Nu^=hYh^=*zKk*P8y) zXQ8CMVX;pwzgF_g^@QiA4JqmNS+KGWTQ-JO+vdSnJMAmj%q7MI+Lu5eskf9k>0w*_ z>GjodTYd4WoZ&R>YNeDasYJp7-H2ZiPS^O3Mb_LPQI!gp(+}aXqv1yj$+SrEw($wl^*eJ zmb(|9;PNwHYRP&tRC0ouPY~?O_J?g=$$HYuRGYrgXJy#SW!h4CvP)%~ z68jT(k0d{GNqX}qnv!{j5^?({H?6r>`4?qdf$)$FExiG!J2pX!|B%jT_iQoZkT4q0K#n zx#PbUVUBg|A*1iB;mO+uXu*h8kiO}IlOdkFOhS~m!F58;k9`&QwOE$ks! zpKRPA*F6NgL-=jNofFtYw%-WcAuxLg_j~bM1gl=comu>5;rDR%kgtzpH;6YcVUGlU z!|?k6_K;eiZrmX=dx-c)@>>MF*RY0Me$(*#4)zf9_hmN-c~4=FIDX^shGFa>>2FQ$ z5cD3xnHd{~@oNFhkR2VVE?ulF+6f@J zl|H%vQc~e_ECu~X`qm45g=Svwjcn7@kZ1Hazz+P0tOKR z1qcMK$dj=#GG;`&-g3PApK|Yf_W!=}?{eOEnd$QZ8025CF~W&p;ebaSpn47` z#=0E(F{85jBK(zg`{@y&vKB3L|H$-LYOLA3eIE>Jc5j|SjR5_PC(w5=VMWv{;(^SC z2~Z+*2`Q>_mJfrZ0oYw))(tQ+0wRMx7gy^g+{72`pT~rMH3)F5o?aTYx4JglXVuZ4 zL)q3^$B6fnh4zZbu6-j*w0jn1& zG6d?YXEu6}l$cyx(Q7O=`%tTk*5Akoa;RjXK$Icgv=wU^gO_HF5A&U0c))ZrV+ z>#dtSZ6Y`e6Y}%FP+QO9%2m!F#@$4bz*L#{F4+^6USD= z(kRssKOTt^LBXtLYXHM#D^t0+sZfnc(xs#XGsER#b-~n#w|*KnL1|~u^qScdn7FwY z1!5bs$KsT*1P%)^vmkOo#i&gIMAP+I8*rrpM3G9*V%#oNd>J(_;69FqH5@A!SFoEk z7j^93)IE=WUHusL^*vN*nNuqmj!%usX2;qH2^heQbr(;L$q5cSp<=N?l$rPKrND!D z%|OJ>RxEiAg$Y@dS9`O66vaY4RM1S0gRsz?5y08bFvR35&X}K1F?lvVT)TH;UNW&Y zj+-K5wPSX}*of3!xwtL|?(BGaH;wR=-T#(?fyzYHxAWC-IC2lJ`Du38{Qj+bKJA19q>ln8g{!$Zp&+?#rlL5}QR= zmGn*z>~@5ACP&cSS;ybDeDFvQqlYwup{pjvU}66Zsw~m&8Mc3=uK4eW85k0@%%a0Q zr<7NPI6*%Hu0E%|g&h~>Jt}nYWMy$rxhD!9zR>YVyxCCf<3-p~>DT+bHaxx(Q_&0; zke%wF;}rw5r@Wan-1lyjUsckd-+WgJxKix%8y-mdgl^o6%V@Q5AVDHi?#X5qpin7XMdw|V5Q7SU8p}@U>*i?17 zm=rY(ZB+ zFgwm}#L$>XiAfH#gQlY#C}anqiU0KZO_zql^jLJd7>7x(hNr{ehz-&8tclF#q0lzn zXXdEBw`4O`D{)&LVCbC%TqFRNrgA<)=^fvngY|_XyY!wG4F(z7;;ENLAASFu+8ABU zk=3{u(?6hR_1PY9zqhIqX;xT7giF}RspeLvIl~I46@t6;0QN6E><0VAvWKD)Z+Cu( z`-SjcE0S{J40V;^gR{m_8Cdzi7Sshy)HD6`N|}|XX8FwxygU4YY4%UP+#fpFDA37= zPr(F3ft$sLI72mX2&czjS4FI-`Sk|=3-E7gUl$}4yoaB8@mV83hY+U%GdG^tALPe= zPpnc_4Jx5sxu*|(*s#(k(FDEb2=!N<`7(A5z_zLgUCPIJcmD=g-^ZWV`tkKgLBe#a z_0I@r`Brgo1dOwQbo@F@)U16x2m3|-8^w{qM@in{c8A%?PTYh#__IAoe@`7)HkLQ5 zJ4Kt+B5NbTX|!e@9OMVvvYm7xL(f1n=KG?;xp>D1?pL(bPU@zD?Wc7hh7JF+U0c8J z4?Q(CX2{s0_ly5qz7Gt9NBvNZk@KUhFGm(PR2~+uQFRiiM@vy8&pi!0dj$4m5{aSksg<7=y-rkzh!m@QMt((wlXH+tb?TCXoOY!J=B+en3}}cq;!uXS zONU4g^~=jORy_ffhUVmP=oM!R0p=Z;k*SJ?;_dvM300FWr%?IJ-+iUrP0JX@80BtQj#8ZI-6W3W~CF zB4U$5(YqivrLvP`Nx>EL)cl;mcG$s4$^pol{4eda?-vDU6${*XTF-QK!#NSFcoV(P z8Cn`m$%7=V$JY(4=#^0czRVua14D)E3}^3uW8|%Z;$Wg~(Ihwlg1c*QcXuba4emBL z1b26L8{Bmm+=J`jPH=Y#LBjd|y7%GMdAR4ibak()uCCsDb+78)RqIZ7HOq1rn{!LC zkOEdZFT+`L;*zP2dzO@liaI!hr4F&^A4)OyBJQu6FKvO1w9Xt(9g9#*<~H=B079i5 z(%G>-ewRUDXz!f)X-WhOl9WaskTslD9RZ={C7npUt+r&UXk5AEEtziAPq~-8lBM6Y z1eaNoHx3jCI9L6tWKR8IDtpc&lnJYeJB^>7earv{E2sGVSBQjVNoPiO(tTtOQU~wd zTA=9MLH(O`!tV8abiAfUs_*>=3A_29BLM`~&)glNA08`mE3S)aYwQ`;&^pPjO!U-H zYd>%IDNC7UkZx*EN!n8fjIGuYlCg1qGD1D}Njla%0v}+_+V~Hm-u2>QO@ZAn zmASk@U$q0TtQ5_=@KvIiOs_cta70FRNP;!f?IL$YeqHhDO&I&yP(|`gYB>-x_Ye&U(T6<_33$Wakvr+DUz+|?JZf&>Gw=j zk@Ue@&F_yl5*98I;oFz=dxUE^f?_`Jv2W6^#!glz?r+bOLgruj);%Y#<`JG#4J^KV zOBbdx)pM*E;d6<5{As9D!wk&m zPlj>U(OMxMxE){rAG(i2P|JREDY%T(>J>s)bG>$GD**eCM-R9sw z65UvT-N*(I7Hi4Z1~cL9;a0E0YtdgGlsDs2?uyJ`TxklY5fzDE%CG=s2x>^=v=|PV zTdNejW)&9$kFIo%jV53+nmqdm1LZoF89sUaXeHj9Cfy8;*b%)3l>mQ&N7wk;-zz#U z$ut%Fjx&~%-lYKT2?lJCihh~l4233M$jDCs9(lSUqZuV)k?F5h^+S{ZH?vZ?|ql@T=i+^d@WHzcT^g}nHANl zfg02L2oKG_pK}l@Z(zdUozo)#2>1^zN*ToupXRjHMzIC650sy_7X4EA?-c--v3!4t zxkf+p#o+SURpW3N0aTU1Jy#wXIM>4m9HS!mVdAF%q;BOYKK?5wt&e=z058n9ce;jh z>I#N&dD3{oLSv{O|$&+SZ zeN)o&lfKA~qR5WUtT?P^QCwe}o8eCr)insu2qeB@1s`B;de z3n^#(Re9K7(DZLmW@!!Icwl}^SglFOIM7&yrY+cv`kZQfZ*pd1E(?XSHkYl=*Pdh! zaTkTG`jq7jC}Aioz^j@+BFvG`oW{{729J+HG!jEg&^Ubh>}&iqwq*^l`+>Sa62)1;eyjkyUz>*|O)O-8rOQ>to#Umgfq@Uy1m zjN<`HK^CoUGF38pMzz<;;V9aagG@4^RrJT)SZ1;r%p1Y7I7i zj{G>lu25)Mj=}8Pejiz~C3m(PZtUX6bp;k{&SoV@9>ZNhCUsUL%nvnhxHS1NDGd`9 zrY~|I<;bm6PM*s9^~i#mJMpWQ;!+1{p_GP1Jw8)eUWPd#=fw9dPhxJaNoFcrT`qXk zx8Bj0_u@^v%coS^W1)`tdO##?NB%62<^+ke8-im>*IJ;$RWd98mMhuox5f|DCo=cW z(`DZS3Xz@Nd(#O~%EWPzryorpmJ>ab$NLw3qsLf&y|c%ui*781#-KMEKapW;m_lQZ z&X51idZ`;R*zpU>r%#L$pFZ*aAHOTK+$>!+T`eso{u^l9y4kur{l8B8>AAip2GTeq zv!KK|#g`)HvhLS_Hc(l2j;c)xGps}~E%Wv0^byZm^BQVbE@byzRU8gS^Z#i+!(snV z4_ov9gWXI`_2lOIT3_Y;rF(rjdYhTb<9GA4x<3K@+4}aW=9c4|X9zng8w6uyz$=U% z@D2>nfU-VvldQCGkFB7Ov|%fRuj_}g$KpCYnOJAdeKMlD_!5ukHksaAerKXu4IeQq_08*6OXmd-rvoAN|ALQf@nhz4sNJVtT9 zdE(7yBi=jA8#EDmHC7~DTR!ef5wfXYQ!T#gzuU0-vMI`0Q%ynQj;yn7S~dmZh(gy@!w-}tdPN5*6~B3+ zXrH80*J3K8HKM8hv|J5^34bE@jeF&0mxx4s1ZjKx3@q#33bg_`HbpoBIy4#6BEBOo z$G}H!NVZ`QJUJj+;)F@J|DlyVk+ci0Y3e zHo-~Ow;Hg(2UOJwv7HPw6;Hzh{EpI|y(mNmcnbKD!_6p+i>^GQ;kTAC&c18e2hfwH z*(N*TDyx7!Lyl_O$oJM*eVm(Z++sX5-J~ayNb_hHD*(Pwy~G# zS)9H5TJa|Kldqm$&cUk2HjkwXWGAPO!tJ_atW7euV0uTC@6D`O!0q716g=FL^XBx_3-3*=t-WYBGz5?ThNHEMqO3NnZCr z$s|emTaqBSy3PQ71cHHU>S_(bR^@t{8x8o8yq(RU({I(Mj@~fDJG%VxRg;aRzc$fq zhMQ^%AL!!6*1W#wSYOt~_yaX(CB^VZx>I!qz`TCTjJptGRwCgXC8ypfo-pB;!5rqI zwHt@ulBO3HF{c%VWC+gxt4W*$jy|?t)fS-ZB7ox^H0Al4yf~-l=@zEt0Gu-BW^Udz zQ;J=*RF1{IDVJU|i)+Fh@pDWP`8k*1+E@o1#SLvMxPh3*x_TsRArErEGaenVfKrQf zWo;n5xDuf)sc9_|AY!BbmWW2t)Z7@vzpcR-80wig80!78h?cP!7z$Z2QBNFHlD7<0M?eIbRfP@EBB&SHJ0MDHiov~bCQD4R=TOo9)*_n9w^Z}T6Wcxnv_EE zQuvN7OoJNP&?@;m>=dQi`ktnrde)sD={TETMkiBYSX1 zU~s8Rf)pcNxMT%!{s`D>{hgxyF>4z?@|Qq5XVTnsMu7e)R-+VVtIW^yT;%>5dG zO|MZVeqpmhERKixXETVi)vl-iGV-O%Ayd%Mf-_B?Y8;H6a&@kuf-}j z3_)6Jrl>3dd+}cu-EbIfle*|G-VB8kl72EG>7BKH3~=3k)eh`;RhZ@4((dW~x;V*P z_F7Ve+trl-i`tk@p5<_vweTBP7|^g4l_sG8o1Qy)+V>AUj!TN#gJKOV^!zGLTJjVt zWwFv-Ww9+v^Dk%Xiv_hTf%v}7m{ynL3IiLdlJ;@J{F>znBxftCPv()yj5W39k)2T< z7w9dvqMlLF=I>b|oecioTRL@~dHF7wG_c5Bb(Q2g%hSgz7I=yk25@*c{i1feH(I;?A!c+$}9V%n3 zWgNbnYCC*~W95;sk6T4duR`YXjGwL)Ge4_E=Azx#Dst(HL*`;he6%^LG&9qjP1w7- zcp|;RVP#hr<~G*PTqrv6SprKGApprkrMNGDEjd1lkrW6(l&8B-`} zL2@}iT3cdhI~o9Hw9L5|5S?w;))XBb6KSCO%*sm-h}$d;JLN6Mso>KJbVr6JD6wDm^wzv&yQ6>xBnDx@VMArw9P}BwGG~K1^gO>R&C@i(Bc3&}Oyt5BcI*>IXge2jC5y>;s0+PQ4D1$l5wbS>xpp62_|R~ z{=xsoL-v7J?nAlVICfhwx*oIBN=)?Aj;mB)?!!(nLhH$U$_YsrDNhku$hAOmpX|%^ z(2)K^9`(x?31b^E(JzSt#erfkWP|4V4{y{j>JrBPO-B=TmHNnf5DlgSpGeC;9ESA{ z1_fXSoJIX3I1NPmVGC7EJz+%hPyhVR^)`s{t~mdJTmUWBp@!;Uy%55@=smYmz3`TP zFbh>kHz*Z1&^&m+KiuC4%067ne;{~?#~S5J8UHvTLOH?PO-wiuP5w7~lY;B-jOkAW za^fg5j_Bhn{!n=78(`Fb$f16*lGrg5>q8FwDfJQh@Da8GJ{d|m0U-J3ets8to5Xl` zn*Tu9o-0n2D|VuLAc23#y)l-3_>lX+?XHtd?ckpLnykq|2V=SM~EQ`fIl#2@~9$MfZC=D*6j3iFV z+LdBOsGnNzG1!)*V;h;K5KGf=D+{#S;1*`!q4aZ>64Y%=TBn) zw}w^!KSj`5VFW*Ht(P5N&U-6QPNXDslPdA#oFI2ZWC={_w4yMhUo>lC)E4H*(=9(Q z4EY!tR=?V*szC+2w$~mPHD5J?x-wp~*lSYz#v!J#D=o&e!%spPh>c{lvkY-RhrR3fqV9S`-~93_L3Hx6P78 zt0BiHliR(FaS#Kc;g%}r1>$i*r*!ubd)6ioHedIV6CGj)TU^$M2|PlZP_h&ixz#ly zl>0v8!NIc^aLs6;oE+|a`v`Pb~ z!|$V7&H{Xg?S$CPo7NDPqXUV1ZHG>l$l2OV(__7)?u`cHw zpkm2Ak>~8KA`J$03RQpvntS1^=+z4}615PK6e%2rkaK*0h85Y>6MRw@+kCK%v6keAu`8nqG_(#;aYWr+_LBhEO^lCx0AEIY@5 zl?}CoJS~`(iiA?2kuY;$B^*;S#s}f8L39cq({v(>OKWWY@u~z&Ew^bt>bWV*6SNx< z2J3G*b#t##4ohs+hzrjfT!M9x&gAH=#j$fU5*?(NY0+Yf0DjkkZ#YZ?$cL&TJ7H%S zgDt}_h^qBi-%9bQVraRA@qpQ3fK1h93I8=kPS*)tzRbZ^4``{}!{tbAp7+Um-5D3E zi69CoU0j1b&T_6tJs$Xr0$;K1EOc}np`$4+w@;mZ=)z4qx5@(Er7UjSwh-kibH}E5 zj>WI1!tDI@KL%V<)`o{8QNLVW(CdR3l~)~xaG&j6CmY6t7W~{yoGrc#sBc0D9;vWE z6GNKl^%3?;?aLA8V`p~BMk%~yN)Z0!mxy&c%PB)t0I)rnd}W+{->jb!=$eYUVIsI! zCD1$r92VblVNTWmf@#+UBCk^PJ*VoH0}tk=a4zAkmJyfdeVDy!eG!M zzeS|n?vFA5!3a8esoZoAX?wdfUfmn6Q2F$wN*e~XYre**WU;hfi2x;nrH;C#-IjIT zT^d}WSba0Sng6avtqE=K0*7=w{D-)T0? zGF}EcmE=lZnFFglV&>YV0b1bh%6cXPs$FPi&g$b)2_zzl-38P;eAQE&BTct9*J7m!(yB~({soGM%_HxTJvr-gOtmAb!T4A`pNpl zotz53GAUsYhf@y5f{u+^S6w~1E?ud(8FFd5Q&PM>Lf0rP?OOv7BaS{fJ$;F@M2#z< z*NMg;b?#hiDZnAOMJY`QM_#hqZjcp7bahb8fOko<%S#jxDLRK+6R*nkN&L6lN9I3% zb!A-2)eV_{!vrRH=Em}~5=aug)r`Wm8@uC;1PdV$zr$rmyX{2i)~F%=OOg%$|z6%#LV6b%3Ivd=902jG&(-b~>R#Xx>i? zvtE|tBptop5yP%*(aKH!#$tDqR;OqnET`A>kim8fZp{zUZ}AlfiHZS}x-c?J1XmNQ zoEDYXcx>pJwUiyid8!=TP`EXgI6(49>hRfT4vBL@NDkc$=DI8y9kfR#44uT&^XW?p zD;)v0_3~kwTG{9pI1Xjw+H18bBnvT`a=L)w?Y#;?-fD?=1R5kT{}FnPMCb}NXi%@t<*q<2<g|S$l*VxG&lqbM&N1>0qeG5mu%u$YDSS$Y-8J__ z~QIX ziXYmVte;rX#`$|vHJEJ1v1zUDpvbz4X0p1Gu))1o*;6o7eB2{OgKo^xJf-ofd2Q>0 z6dp9u`In3B8w}tpB8*#741u;Jn!!$8fh$sww5EW%@D7Spj`9dvN*Dfq` z6wevY%l@T5CMXZFh(9r8OE%DO1CEv*8&`>LpH%%XBeG^CnRmyGES=UafW z+Wgp0*9YfcGzG@CO zMRD-o0yVz?*I3tdXkQzqduhGsvWGXdTA>~jc2U7b-Y`3qe}*fEWvO@A@%K=-U0}_N zw31^c&IFKue0MuXXwC>!>U{n!kdFIGpv*^hmh;oB$v8Pp=sH7aLR5@hEnDbFM}&8oyPUo%NAyi_aI?UxG?>}`lXcJwOm3ed6LK(u1MMgEWL!V&II zqpAAncE)9dR1D3kh>AY_1hu$CkSd6K74=Ln>#Ie7oY-V>by3+FSrYEHKm9L zwN1a?5W|aGqohtkP4M$2KBBus`;QPNjuelTL9d|>+s50oO!+cfsHODZ6e`;QS##*Q z$JIz_DkZNV9x~eN5jiPLUCxxw^d z7i2+^JY{2~YUNrB8b1azg8{$$C39xpt#dS&I># z$R+31xozMeef@a|KaLE8RY{alsESX=aOPiF3(C?-*=<)pfph!=S+|9QT?_SQ8zT?= zM@HG@8O3e!)o3NECiGxw2P1LV@>CBCmxsK|yh=t{XYx0j;?1Q>Mhl_N%E+RNA9fjLi}>tC^o*GCXnLJjk24Ow~EG9t}6A> zC5|WWjlL|25tx3G$Tg;xURm5rk??Vjo@Py7DCG8b9N`Z<%cvhJ(uzto*e$D9`Pl?bZWnTwH(~McSR!%@X`)Z-gQ1+9B(x z<8N8I{T|Aw%M^1!nmOR1PtuN(@@5K?m{#f8wvvZac_%Mpeep&wWi^841&kFgcb!}AJCj>wD&7CV>INxMG=a2_R9=%_eMmRUYdg@W@o8JDx zS8`gIV(ghfI(|c&{tE*SS2f+Sw9J$+lW9mvbhj>>7#+CI13aJ6Q5&PHEg9}JarVkB zH$1Ah4zGEmv{(PvO#Xpvi2HA+^h+`Ma@cU6sk5#fxm~L@?M-B$_5HPcU5XB8+cJ{K z?|U-fztkGXiin!C#jj~d|4~LK1t~j0DT-+QdMtR_UH4usiE>DsD|s1rXUN1gSB$7i zAE3A5VE2kx`I@TFC#|_Qk01Jm-NB^hOarNhjM zUqLlFgpgcMD?8y$-DoZ~3vdA;S~GH8X5HiR3mf8aWXmBYLmd!`ya7H!vpDNs}k~Su(`(T{|<CCq-$)`b|l_gtznQtKQ~IgDhJTP_o$+{w`@X-KB0z6W;RVsKcR zr=>8lCu}?&$4v9$cL)o7R3fUtS=FadEGU+YdN1XCNQp%p*?7#YnOEUXE5V;B2P-d(W*YZ{K3OSraFRiBgGciPx?W7HlH&X8WQ}l#fbhA#*4Xe+c1HNapOp+&8 zXwc|n+L91uQM&CIG6tY#PMZTC>Nz8sHknM%BA&aQnVIm5b6}AHu-}q1VcK*yq)@yR zo)**&AE`^1IQZBBa2rTk! zVX(7ImtOArxV15wR1r1Rsud#?Y-jL#OqWvGC8vL@pJ5H39Ve5NaQ-eH9@W}7^I=Y_ zUus@*X$WpD&lH7cE~kEvL2k6qj;T7x!~}9cKjrG#*jl$~MS0tXanbV=PMmb;3$85B zP{AW(!tfK#t4C5B$w+SaF*69UAW9rX$>Wm$b zx;Amt3ouy6KNm$E^MFwk5qiIK;2QtbiMr6rYXcbA)8-LqF0gR#4o7q2EUEEs$#_Eh zFY2>sUz^=84JB@U!&ciJ^BRxB$P?{8np(ff4Zx`h@MCx@z9Iq(gcM&(>r(5qQJ?81 zOSmFuv8E|ZZ7&r8uQ8s+df-n)EDZ)uw#NI;6QRzDPjz+giLP?Bavt6!G)wyHf1c}; zw%={+Mb(7WaH@>-`eI1m&FuT^`asj#D~-h}A=>j^A^F5|`++vTO2CG?%Y~;=;5B_5 zPo=T}_wIUUly4@RMj1K)N-$_}tXVv_lUN-UbMftYbuqp@J{|E7PO`o`+)9}LDS2mf zNd2m^Uh>lW$ubPzJ+k7DAyJJ`pUwi)GZKfMl5KiN$%VygoAv@+Hrt08<-vhTo@Sey ze0oWT>cE@k?XaK%G%|pI&*Rpnsn%RuT(I2!6=d-X9e zFD4R0N$ssO^h9S=;{J^VvgY^-=2@7d+|Puj&TLx5q?!3@jmW45TnzTyLGQ1jF9R>% z?aa8o*qMlLs)GUO8w84%aj`>MrwrirkQZIy!s;L7Txlavh8|hAqRh2nI7v$np*QFf zag&3Mii`A2gpK5kfDfVLml_{iriMXWD*p6G4%4xDx&^}iW{`M5t&z!jc)gsn_BNBh z9l%n&qnaSH*jx?wWH{PFJY-%!M`UQ(+9$e{!KA$8n?RE?j9MPq4oCdH+yhZge2Jc^ zBCBQ6jtspnPOe;%sAN9KNsv7F(*w=}Y+D(tRAzU7CIamw6j_?7^q83>@?jX#`)e9i zBNE89GhW9Un`*1zg7A3w0xc(ncywS1$!mI|)xFUdJ7;`U26WWP9^m>Q%j$&dEbF~Y zKxDoh=9m$WLHDPfJ$JQKOZe3M`GYauYkHBspx-`1H&<$a!qqF4C^Gs#s%v8wGi1)r zF*-czoD6OCeI!&Hmd=%WwwH+>TI-m8ab1^dwDV97;8wXoD%tUOBI;kqCsees3CXb!=?qDg}D_zf2qiBTY}~*t}-4aeYvd^X?`D{@U4~EsyD`(~%T{ z9Ni>od?I8tJ<?vof~535gK-~H zf$!Mwnq1_nT+1r;6(%ybMPC#}w=(%-dU!R%?dfBFTbx57EQbmhWeVpiqF8l;?@Y~C z$ofrJ&z3K=08v>{Jmnh>IM;RGq`4J))6W4RRfUPo#o=vbMAhd*=jL=jPw*hksl!_m z#EYzRtdK{=s4Dt2&{N5TDtvFp)G{j_RV?h?MNCX5B69Nm10&GC#l9IwCsTG=1KL-- zRBybL0$<>Cy7CO`P1&9yO;kNmrs-ak8U&>~Us2;-p^CwD-}j~DWZrrUeE*U{!%O+C z6&TD{!O(3b?af!|6e?ZbD{R~DTF&HndSYMzd&BS zV`(4SYLldO@@Mr0?cHdRdP2)3f8RMB1x_n$GUp(fGb`nLMNfRy-G`_Oz$`=7dH?VLHj(@oeay&YU4TMmdBMx-bG3&@%D%ee7X4H{PYkGQsIS# zhtVjWo#w~3$Db$P3-9j6wY2J_tZ)G@T?ctpy{eLZPUjy`QPtO$6F!_2?HWoxKr2pB zpn~-mWyC?87g@-^rhiDo(8)=21ii&p7WB8O$|e(c`(xWO^s?rMS?7eZ#~Oz72>lnl z6PUxC7aicF1CxXbC`j@`PxS*+)hHm8BUqo-_aG4RFF&Ea0i!;3_tgs<+AOa6q4qJp z7zlYZelP~GUu`zcMr$UWrP2|yKQkPv#O@s6vCnD`xxJCFe~k1y<*GKs@0I-nH&V3BCz1aZ{2B0D_zj$_xX-^aV_SxD z#X-t!xcL!A=1{S$opcYlcn|6GDlr@Y_qPc3u<=2rIWj&Q&tvJvBB}%sQ~3AqgnYe`S04PrIw6)V`vLIa}Z?ks5^tJr03jx$0* zv$JBSu3tu|N!OP#e&bx^K(pv~`v!>%#h zbWv;1UkBBflfOH40;h^}UFg=GD!+qN3o11o30m+UY?i0=E7H!Gg7CQ$8sn+*h5I!a z>62%^OE0+h>dbK*jh9|4G}jCsdPC2uQ6*3P2LEO>B@I+&xYEo$Gmq*tm3D!8+k2Yu zd%l{>Hv>JT!=XM*_DodA_B@8nZSf`a+)0}dD)hqi^FveA5oB;I30nhlL;OK)6lDlb zk?nP`nj;ZM1ePnOQ_eLrmv;5zfZYy8h)fxfc)#H8OfXsXb(~|FO*gIcS#2ycnH(XC zA158d{a;3EblWQ+pLH8m4_+sUwas>r$0Lw&sGnp~?_N{@8edjmn?e z)AYd|Tm~5I@vV+#f+Qn+4ldaACFmI3Z7Dnn+W6-{^Ux*?9R0Vm+KY^V&9o=+iVOR3 zZ8}|NdsWd@zHJ^AHWjICJ+gcq4XPhW@kye(Y?VO|v*VDb8W)8YnCZ=T@e$kaNepVt z`2%8-2wNO8W^<#{WR&IgC2Xz)(kj)4e1H+!=->FOO#24Lt(_Uw`HAJ0Rj-4FBA%GC zD_rE1%X9w}q}IZtXc}6Nn6pQQfi2-^iK=lW@+{dd7#ntknXPqY^I^<}3e2q;PD#$0 z!8w_h4&sgNEoIbAe2uTcep;y^9?@UnAIoGQmImjN(M((2rM3n;ko&MXRnJk91DX+3 zh)92XQ#g0A+2m>jY3}5j3ayp5g|PCWmFs=}z&J0n%FA*&4Ir!4a&v~9uGgZNU}p`o z>7u^LuDKQ8{EO+=+?4@nf$I~-C({i6NtBp9vR|1Q*T;TRK)$()t@XKdIOA3du%eiQWJ+9} zkT0^;8E;TEVoQ1S6 z-h%!^EaAP~gtWH6#OejuKt{?toa`?dw+x{eb(OC;t-Bn??^(@(`DyP&@;b)ls{Tws zri#Kghkf8rtRR$j2I&tcaDsuO+L2*R zmCkK-+<dWjyEb8XafYNWFXk&oxh#>ZT}@(MXYXRo!isL zN@ka0m0o*BCKIqK(jivi>4n5Q#W^L@98$D%RUA|hO;oiVud?n7Zai;4f38DW%l(xl z#6|-*1*z#PblA7v#{*AU2HK!0v`|+;SL=HNTH<%&MdxUf{nG_rQ&UN3FLXaH&s-XS z7o1NerwqPIzuDQp|5)}9(Z3{!HG%w=FrQ<;VzcCru%JW7eP8QE1t6MsCh3U(#wKq- zhSZD(F^5o2mzi5{@zRq_x?Y~KZs8IXhfbDR6jk(>Y1SI{cr;6wkKKJ=lmDuqGOjY$ zL?UdpQ5<^`Gp~PJHQ{d3$VT5#opO1`X%Tx>9Jx{PlTd12B>z94C6n1FR>X#k;V)jG zE=6cbs>0wi;U%G@F@TsH%_3=6-j9yi*M-5%I0LBDhp<-MhyPnWeZ3;-i&j<0e;Z0- zhL5GV`g~nqD%mHf1zN2)x_4mLL~%QvSmQ|jV#rUkv2^SVh(o8^%I*Y=sAp9Hz=k~u zkYRlG<@6BOAroYE)eL)`z{`TEYvE9C?yA*r-TwlCvtibmwQIKgvs?^0 z>oMI?XQ#zMjnb;iRi30IDa1e>ZewXb8Inb%VAz{oA)qwDd%wQ5BLtYj`)aWMNQRi6 z=4l@O-%6AKCR|0crvU11fxk*i^g5SkPT9-}n3IM^F0x6|ku((P=jEAwJae`5Vwu*5 zt?#%^>%%OGJaa;;bX}#}VUYemhsI(3rkQk~UMz}KMd6RX$(Luoq0Brn z`v#12g1dYjIQW979ft3uPO(_OqU(*+tg+}881E4GJs&-rnN=_zGFY(zdwd(GC;EKm#q z^X-UEqy?mRIh+Xzl@6n&1TG^_A52Ab$^qX%*(mT80rjGFe?|if-E20j;{0?D{kYM@t;3Wl7Uf(;an!FhAA}>=v;qzPsM2#f#-WGxe(Hh%cy#L|J{nFaj$!iS9xyt;B zkN9V>+O;=GWe6rgGHi(jj`4mj{2LmILxO7h?o;WRGbS>R`=gzrYA72?*ZIePiRR$X+P+WBg z<7p0$70SeaTW-92qVyv-pzjMt&xkxf+46{{Z>c*{d7}WLVAZB(c?s3R+fJVl~ zZ=;6H^%4E^!`;?ygv!S_fr)C@&ig<){bI_0?OvLMz{W=sf!ny7G~H$E7D28<60!r; z+B-A#65Y{#c{|0lku^*!pnvO^kjh#OW1`8JOUv5*~VUl}&>|oR!dn8@ze=8`*`iLXyd`l`)RQooTgycUm{?n0d z50Wu+{Pej^l7^&KvJkPx^P*WS-}k0MX)kp&h%$lj;B;UN`Ecg8FTpPnE=;|d?)@`$ zq4CEMz#@0!PKOH1|z z#!Nfe^Pdo)V0jkAblFG-V6(aSmM5F~m{0_@Q5WIsrf+UaBjydKeF+^-H9 zr0pI+xLm|p9bXx$2BrW3YNLw7Xx)VA&F6c9MKj3(Gix?B@7e;mF)iTI0yH@nl8N9{ z-}xNL$%$72C8te_`|7~nQ(I1y&JlTm*1WnPXtR8l7b#y$XCl*SazCg2d z!GR|%$>1*dyp*GKxjy$#P+eNrI3~fwdogj`V&%^GN>L&`A3Z4~$3#oZu2X_Gi#*Q; zz^jn^MPRT=#nr`bqs}m&;oeL=wUr?}0z1SGV}L$URJEcaY^7a0p{sTIFgC%COr=x^ zGF0y7T{^f?apXu(eV^)@p-&nqsyF_*Q-N9m^+aVJBb0P{^VeIJEyj3 zr(B)A9RMRqp_i`7{nq|B3QgmE_n&Q^iDXJ|lL2*Mgv7{Hrp9^A^)_1##So^SUlRC# zOUm0Fi3y>3C>%?NJuj)X(8`uqX7ABe+dHGgS>>$ry*4#;owhYhdj zl)njwn8a@`?O|*1BE{(0oJ`UtZPzf4(E;c+-sExl@MOXj>K*xya@pY4n#7Sy9_hVJ zI5y^ke>g}c2tf)R{ht11K~`UlggLM^HOfSuTgQJ!Y2_7(h%8^^IwZ72?9p;ll;7#) z`xv=43!zBsA@$mB)OREm|0T_!^!BqO=ti2UvVUwz{7;cyYvehsfjF@^UfI5~{iaT#t($g%qiO_>Aub_YG7?3NU=^f`frWi!EMNPsN5O9KC`eyvYKMm$@ z^(OQQGd8tqLBy{^kN1+u-9^!ro!Od`V95h>D5orV5 z@RM%suF1%(m{_drt)rPhDZ?nqH#9K3JYbeOQ@Zzt57qdC;T%yg?wIqTqAQ`!${$JkCVQ{is)zHiN4S7>cWU6tsRJtug!|tkr;p|xq?my6Vm2fV1 zif0A78mJ6duvUl_Qb~R$fZM+7u&XK&>|e$1qW;ImFEH~pZPXF(7xSyAbojII~J~F;w;v<^u zDRdJ>{6zU64E_0~3xTzLHLg!Wq+7#bz!-ot*M|`}+!yCzzE;ga6pjk_zW^RU;lJ?N z!Xb@dmgEk}OmBn^j{*K>=j(VIlDGKVBj3Q=-i?qmM&Lz1x-j!XddSEY+2#Y#Ki>*j zPeEqxkX?|wC}iz~p^Y$n0gXqOe!^u&qL|6+QzUTO2KJz$*jW%@sF_n-B)0 z;XttsuT1PCTWl9MgC%akcDi62vZlDzt%+BLxDDUs0Nam437BxQ36=>RP5lBBZpVL& znc{YWAdH819Pls3F=+TVZOBs)7&17w5k}LbH96@F`RYmR;wcEiGjeh{s<=0~G;yc6 zOHM9LO->F%0~SO~b@nioiF2TeS71Mv$$lmv9u8@QiJM_00)6tLT~M%SClqB)ZG>s+ z6`~G*XX1V45%|*xb9TbKMp(ETUoTBi^C(qwsjP{=M<=X_#4<bi$fti!viSVI@6} z&3O!@qT@f;Oi+dbAoR3uv{5uFKdDVk}+*y+MARs$!hHiMVk-=kdLcTsY zq$B7~+P?pj*oZT6tN`CbZ}8QuDpE!Zp=5eF;=!EAIrj9VZ6M_wjP9=InLlY2vfP*7`x z`xcyLvF-aG#YYbjhc?2a55nIGC}ZL<{}duQkFfm={)B`xvgb1Y;Y0q5yF+Hk%6zF2 zURf|WWQEK|c)bzck`Mpg2=9%dp}apPm>t*x1=+zZFfnB8fDdnK($(iFb(J013^q=M zT7JbRMyjC){_a75?nRL9gKn@N`XD0uGX}$%1_exqxy*oaCZG=QH?RPl$kO0kX2Z=a z3wGh(-7EypvhMI2>jCexp706l4ZpBHEQ|HW|NF4PESC*qd2AdTCh6dO{Wwe|%1$t0lL_~jaJ7_XQ4{XRQT`5vd!WdK-Fx8}#?p~uFBexh zbo&hsGy^9x4iWn(v6G~nP7g%0uFCjMstmmN zfi5gmM1wi-wxRhuii^nl3BmbPtP`y2y#Q|RQIO^q4bcx4f9oQ)`P8PIjo8eVo^4f)v&RwR+9B; zD5%%L_2LQfBwjrV%fwTX&=B7qF^`jE$mErydD%$uG?J@^;}3{u1Sz+1Xr4Zcbq$yd zzli6g=A91Ti07r|Ern0S3wUM0eE3lOgGd+_!TaK$c$FrJnd}qUu6PkeG7)Y)r0s#8 zNTmBwXQWH=-A$U9sP=1y|DkR~K}(Y0Kz!2CO5YC?AR|FRCJW*X%K7~$&h5mF0i+Y{ zUo0+h^lg8f4e66jLmHW0lAD*sf`roCyj>7lw1e3@SqCz32Iu8AvP}8g1xtqV^}NSf zPo&V0{xoDtO3GrriIi9lD$kHkG>Q~RSKoP8KhTv_O{}jZU0o3rk*cn!>DL7@g*6d6 z;0QM&%#K59`U~`C$3s7M0t{v+!NKelIE0-FGuRn0kDUq2*;!D@&PHLl2~J?=z}f6v ziTzP(cAiSxBwi9POO>|Cv$0MWub}$X;9wL&uOc1j5;r#Ry>N&$jdo`sw9Xe7IP#)Y znx(+zAS4b*8Dk;=`5%)JB?r%axxE=9-Gn~h6gDcqW za6P*NZew?%4!9eo$UX2p+X1h#`{5(jh*D%He8(O_&e+9F_Au+fcC)_hF*blb&JJcz zuxac`b|iaR($PePaZh5*5wD5YrNo%yNsL)89kBxx9l-{8OT2+s%Hr+;!5BsDfywyi z4=C!@tE<)3LeknK&T*KsQ4tutPbiy(D9=rq%7*3Z+8BcfZ8IF2_ZaKh$PON3WE)#N zdKoF{W%OVg_rGcP##*Vo1w$tyvM*I@NSndEJGL;lZ(h^5MpNL>-ak> z-_XVg;-reUqG_CmCP8+fi>4BZrWw0Q{AWAQj&PpoV+8g&C*KU2=!6drnbc~?kZf$Y zY;3q}Y$!H1Vr*M%ZuQ6emI=Yr@;k$I$X+Uzzw_v4er^n6V>iR zd_EG#LU@udf*1Hwc$qJQxA{@<5nm2p@-Y0tS1`uQSVvyL`tp@*5MRZH^VO1w<|yoX z0(^w{w|G|y@DVQQuz}(|aTk)#RP@s&>A+0*RFabcv*9C2fC6pyTJb)* zVN7PmK3L(1gDmurJK?n>6;2Q*Dk{^6fNkvaO~lo_m{3k(kP$MmrTTm;cgQZ*Kp`u(j_3|c zY|IaY0x_9%oZCz&;L=OLrI&!Bmq3hO0!`_~tDfhNt_4HEMt1yW$it!F)k)}^vr~1Q zB57%%v{)o9&55LKhet!fEOypSkdehUQI&Hy!^zI0xh`yiS{A!dYh)KuBTqwKJb*4E z00kdlS4bFL9SLyPmoW32ZoIv^t2NYX6VBvk!?}DD{FR>vH}do04t@dL z$1g&XzZjn9mms-c4qxyq;XA$=e&TV zkl)OT`K_#k-^Nz(J6JvbUC(#0Gx`1OGQN{-;SaD|_(SY=zKcD^ce7XU?;HFHHz2@t z=q%%jY3Qj<6#o$);MHaBGa_VoAbrB|en_ zf#qzyjBN_k;plc+Y0*-JBIKnB$N7E6A zz~4YVdK1$4ThNui4HNnMP{{v->f=M?qmN)C{~S)@U!vOh3f0Dc;adJRY~$Y`?|cXM z^6yb)`~WZVA0_B36udluza){sA*Q~|2JFhLRI9vFedi5RUFbq-$pVNXOD z7a^z~CA+Z@;r6(UGOj{M{Vj%;Cn%QP$exl|eHMSQkdjRvp4jt^?1kM;90+R9K@JM+ zelaDaJPDtU-;-F^Pl1j=WNGky5a9bEO9SYy=`dO|VJcov*V3U>%YbsN15{}pp;qez z>$T2sl9ml;XZ?H&I9Dxq*y4 zH;$3#gfa5mC`O))Tp)6luQbNMi=R~x*72Ffn64Ky#=@@trDCvX@;K7_c)F6}`i=G6%=tJgb1YD;b3Abu<;680GJg&`ye`@pL9c=-8p)G`8 zv_#UWrkXFw6YAacPZ511)t!AUNDt4$=?TOfPVYT>K{6fN( zP4rBJiQ-rBn=}z7dM3hn$6YIe-{cB_4khqE@w=qS1@M^+)(Qy2yE3F|!fJR`hEy#{ zcSDHIx}hfDkN%!g_OD81zuOIoQt1Cw+qPxgvqG$Ni10#Zt!lcfIp1wfbH0~YbN+|4 z-v`Mw=Z6V3r>y%i+32RH1{2umpU}z|`y?dL5Z(g$&4qtRxTYvc|4<--@RwbEPNw)% zurviiCrdx*tsw-RH3WI}vwP*mZG--}trBZ%(MkbrEfVB9NYgezS8XFq)J}v#?If73 zodR>UQ&E(ig{t6eI8WOIS83;<7&{N{)Xs;$X&1pu+NJOTUVox(fv>cy;D6fHNQBog zUAvZLXxFi9Z7b`e-M|KEH?oo1O>CmJjTLJ*OX90hSoKJ5x+Fs;uUwKVa`ii#6;1S0v-3m90KjiY*SFl_XZpbs3@{4?_v@TZDqXt8w9n#Lr8lH4%D8;5kH3`ejW<67kq2# zh2Dq@#XhlLj=0cgN$bEv-Egdn#S+iFS0f(TY{b9Ff&Jzim-P-=~M&w?Qv@_yF%kEAg54cy?G|J3K-q`ZlR{Gd=9L z4MHddAK(M9mMQ-R^)YGNVH3R_+|t`BdYjkM+Y)+v@HVvGNP&e{fDo%eX-6_^nU?%>n6L5}lo2cP9xp9bYE(eiG8x8a~yFUx&o zc^;XeTrqwGElcCXElr~!NIKqFAzha@8K{u>kyMM9%5X`#3Xt%*i$du;`F!~-!+Ew~ zQ7B_4U!*!bhiG)MWb_p+P@OcXw>1f`@TCcT8018R8p*$Of)_tZ_H5=OW%YHZrpHY! z)0rWKbJ7Pxi7 z|BKS*8^?_A4B({RXFNzoT{j2U_QQU_1JIcWL|J z0c}4#t5fKj!FxJ~PjwBx(RKJ)H(-x0n68`5)-Bdq53uffkR70>vBA2{M(F8moSwlB z(>t)4dPg=_@5GMQGubLVi`DDd>{PuQyI2pgm-PYc4SgW{LLbDw(Fe1i^;{m*^LVB{ zlpm-M+1ue_fx!-_>XG@AVQ*(`Ra?UaED`=W5;c`C2c1ftIH))JExxvMIcl7{ta<1O-o_zxu>WbOk)nM|GVRkqgx=2{vNv-j`l53QPM%Ek=LoK-d-X? zKPOKki?8*Tf>Ef;m7F!bizN~ zZ-Ss2`D3wG+!4fgh-yXEQ&tBV1JWs*8n3ZW@Wbt-7~Hb>DN8$%3TD;2t>qBC4YCma=f2KCL)iVumIJy)@8>p}LZkGDc!B;fpQb;; z7wEga#H{^jKvPn5bR|h4;1xln`ybUHfY@K;eERZYhZhl!jpF43~5>Em8 zLFec{*hEmtsLrZE=@c1`cY`j^-Fea{>OjxPC&EWp=Q&Q0uh`l6tetsY=StGO*=@ zpBz^wf!w54Q7{JN5^kDY%EhVqgr;0~4TjQ4Sfb~3u7X%F6v({iCOE+rx&a3ni2yYr z<`#*jZjtc0MWVS|B;p#6$Fw7cEk_KK1pX_PNIJE7ref^YkCsQvF9(ssF@I(0^uU=)bV@ z_21Yf`tNMBzK3ns_pKmM3;5P!iKz~46p^KXqj{xP0Rn*wKS656HeYL0hR;a!~pSY%|~9%@vbVoD>)x|b&vP#3HM5kNoo)|5~*M0 zNMs*!B(j_wiMPys5riE5vLB9uG%3$F#mjR`igfuip|hQt)wz4Rok`Na<Isn?;k){p570Q@AToQG^UFg&Pv?1PU72-D2}iHld9v!KdCxyB-L?K8`9*k_%?s+ z%nh^niZ>4~(A!urL5nTz8Ha&kOo5I@0Tdb2;Rs^}oNmm4bBt2B%$N(?jCpW}F(2+T z7Q#cuB6z}B46hqY;NQkl_{ca4elV8%vK4>yqK7}4-OP}T9{%VBW50C?9x5Gze+crG z0~%X4l|+MRnko)=@{yi)yp+i}@$G*SS-1X4WZg>0x-B(5e|rjg9#z8~H$gZ2bGOnL z_bPqS=;_7>N!`rFM-QuVyH&Zrsd7*HRkW3|61{bI$ol_RqPOmDWjY6t8~4m+=pGXC zuM@%M&pB{abuy_iX<$og;^D4hwc^blUx=`|E{u^nDZUq^Zv1Dw;yVNSBIU0{%CAJq zKL#oPSfu<~r2Gv?`5Te)kAtg>zrgLr39!RB5gs;9g1;Ll!>h*W@U3wM{9&BQx*BJ( ze#Ry?(>RYEX`IhaG_GLh8C%#jc)iuQj{VcPfxT;NV?P+%xn|tVvy5AKcjH#x*SLcZ zH}2$z8FzWBLBFE7?Jh-oU-ya#-g6x$n?1~)(sh{ZxekYTu0x}%KGVs7~ z;ij4JlWJM23fglZdzE6muY0B1P~x6y%8-n#NpB^Wjsx2h?UjxzaC0Kjp4I~k4v>z; zydJ%?DHmdjC}@&;{1GzBi$10v7|)cixI5JRO!%6QiJO#vy@m2BCohkdKXu9_Z&K~e z#MN+YZzYhlUKH1hXD5=lSgJ|%R^_7@JCW=jM6!DbMjMaAVaDGeVmu4A#`Cbwcmc`o zMc8V*grxT}lHDutr12`eV7!K;_Xd1uyy+9RA9=#|BeR#;TMFBcJYoBuTrAX}7LIi` z7yNLk#e#`j! zPYdOn;wyf+nt$k2i_2}qdN{a`{RnfT`U&L1ClW!Q#Um)6_ShNkBZ59a1bv7I`T`O3 zB_ilMM9>e2pdaCC;}^KW_!YI{Z>SajK#jNuk+au_nAbeSyk_<>b0lJ3^APi*hnN*+ zUx}C-V7OeS^dkn}B}B|vg_wLX!O@Pxog{%xF`T%f^>Yc8FMZH=-+b-7L3n+e>b4F3 zzc?GE)ev0EO7RE3Qp5_vYeaaB2)B`hDl;yUg<34C^yJUV^Ni0v;J>A&xxVzmX2B+k z#=Q>($wv*UXjvek3l?a`?^|IYfI$2R0o_F!93*TQB|5=)kqJ{o7R(WyVWG%|qeNFI z7u}#*gkZhs4yTA7aIWYHmxx|)gXjadh#a_A^o2*n0r0dq5MCDj;9ue(_*@J?^J*~s zCWbIB@|Y=xvL0dt8ze@ve7v40Ca@B52s=s~%2tSpEFvbcDlvuCi2}A!6tT0>$|ADe z2sYzzw0Qt|5^ROz5z+QTZ&MoEWFBbtlSvBhrnK?wNN4*=IqPp8BvsH2@R_56Zh;TX z{!#^f4qKfSiXY)}6(M6)&|CH4DkIfD`46gp=~Kwoq%x9iNM$5zdWj1o!RQ{^&jOAw z@Rb|>_WJ@M@!A;b=ckYUcj~^JvzeW*)}~a|al31}L|5Y=It5$URGj+*RmZON$jxcZ(Zw)9m0r zjNGX%4Q|VX0wsa<|3QTLpEColf+_} zC6>TaaTKf&%b`jf4QoXhP7*6%lPH6WL^)h9Dqy=<3HOLquuD|Jlj0bZ(5vAMQ3aoh zYWPM(VXrur^%4y%SFB^B#CldBHnJJwIJQ`vgsgQkTPaRu>%?j7OmR9pN1VaV7iY3d z#U}PwaSq!m&SUq8^Ic1it%QD(%>t-VoxlT8pT}l9%mL;=$!0qoHk*X(N=&1{3U-8J z>0Qq{Ic&C_r8$<~r|^Pf>0RnDmuKmzUZrDyjvF3OZAj*lH6?Rh>M)nG^hQCjdq|pa zq5W{3W9sdbEEbA4_4;IV=`wN;EjAH%jSwT947SnEhNZF1xJ%0->d>+ia920yO_tWZ z#5Dx8My;o^1ajjn0WF6t0Y|=T2gHm4+43%}A8C25zoay`sVr@fPTMl%zLh4;4e>XZ z*Q~i=wBDf&S9x#R2$iiV=;Q>MtyxG{a>u+SljhF%aO#dKoY+SNZO7P*TFbM^i**p%n&!B7TE?h__rPrc&fM^E)#dc7I7EcA?}9V z`1kMPK6phm!Vh96lJo;CLp;d3iicQlv5Orf9%ci@qil>&>mIb)@B@r+lE)E&9)XTE z2bqH9xYJCl<^>w7y*N|G@LTY^nsr6l?*7uNF-$!bF52^Jdq}ET+&iNF5na|-{@dfM? z|7GdoYqW8`L5lqrsr7$slK75I72mU2;zzbb{Dd~n&up#ug`FsVWoL-r*m+pz60wI} zFZOyuxChjlc}Up-w#1?FAY*W`In+5pWUEK*Tg_qS!IIjyy422&ajBgxb*UXkC8-_$ z1<#qo&pzk%M_C+KOpzT3qxS&{>Jai4l# z+^6O!bF}p0K6OoBBva+ZO@-@}d+iB2`NZ?xDiul(aktUX4ypYxS}k#EGo&(=JHF^( z3`?bTfMvn8_|ID;ZC=jsm&3=u(NS5VKl%eoYL3Av&>qg`5bASF@Mi&GS4s~$3dSXt zk@2Eyv5RbtGTgEaIt$3>m}^H#$D9AF1qis zjcZ{>{0ROMHv$TX#){KOw3A|kil3Y|CYT+}Hnzfyke(g%L~B}ZD9vMoG-c^#2S<36 zpXO42nxgzPDO`gnT<>#;-)u(wGI(hYLK+y13@`*bn|aX39163|e3)lWKtJkGh?o=M z9CI>kHVfbyvk-QeQ{hGY`>HtuJ~AnwSHg62HkyD(vK(^`8)eR8Mdm_Qj@PTqqgkyP zW+$0t==GJetIY`8W>&B}%$4jueD{!f413mG?RkzTA?uAnrb%O~aSS9w$)S4AlcCR< zW6g0=hCb(|w?FByPa4~8j@M|nO=s3<=3`w0=CKBu{4U^VN7kCKnGKh+)?zw5Wlk^; zaXiNyw+)n`x+*UXSyk##$4eXtVjm8T7%0kQe)}fLjE74pd#PzoRKm3cx#s|JpyDCp zLhP6Jk3ER&a|-d$nW^y5SxIqmRlhxx1bNmxCgj|pz~8Yzm6V5{2Y+yofg-0+?}U`(bPYd6OMgQhx|?4nzV)V14* z)OTs!7txVs*YDKs%1@`OtMcvb@C@ZE9pczfmSaOH=HKCCepwrz-LGonRU_5jL^CX* z+7FVqsqJzCf!d>svV#w3kL}c+aN5IpKi#xFf`w`iub!dy@+l!ej{I5Lh^;;EjXdo> z?VqYFUcF3ZGh((J3W|*E3=$OcP*7xef}%risDmdcI!Hm0j^%|ygc_LTWNd)|D3y>0%3{nvbv?K5BEw)ryeV!pzAn6L7|=IeZ%`39e2 z{)^8t-{lL;_jtto509E3@N>-%`DNxu{3`QfegnR{#r%vvV16z|)%Vbio$mPDHah`= z@~F3T_IK~u!|pSl%fnIMdlaer;s2}dJHV`} zwSQN#OLlg0rZT`mP>?DN(nlD2FCtC42&gm>MZqo?xG>VJAcCL*;>8gZ6@+2DUcg4N zVFR%sD$NEquu$i%WS2Q-W{#fs|L%L=d*8>fa*~~2de+KHvO)*5E_$LJMF+61tQ(?T z1{3`Jq07sko+6dNd$+&p_sea+9+X@A{U`AA%B}poa%;b~pM8efh!jt`MbOAQC~M3N1eu9MPr{F*gv*S@gc6?(teHIr1bMak{SwGV_T44OR6e8&Q1zk&ESeBXQmR!~z*;?PI zUtk>q7aFGuO~|#^n}#}(dn%X3C287aij@eFX*xk-rkiYHrYtlwRAT!0?|mf` zE1x*AieX}vbYk6NV6)6ZvvQ$X6@h6F1>CeGaPt0bOl{unm+6M7#CY=p=3EOq&Wg=Iw{c5NreGUIT-0=!7QgD-0M_^6;2h%cg}|b z=R$bXsSdlGTJV8W13q$U!bzt#i8ysgIp-o$-KkGnB3_! zB@Z~w$U>(%S?uJHCC(*etJ9La?X)6aIIYQHr!6_{w4;X8o(iV}t?G27b(~JLq0^am zbGpzzPFFhE=}t#DJ?JQB2p#M6q}MvV>5Wbwy22SiS2}~~PG=b1>s(2XIK%brl}b<< z>moR7{3bZ8C%a5-`a><&i}gl1xMg@wIVT7u?ZVK7Vdw*4=#n6mI^k(&BM7DSL%%wq z-+<7sPl)n1EYV79M!taAtdBn0a2TeszR0vV9TmnH5u#iRBfNCgWEL{eZ&*Kexn`i> zgbefm>#rH;y~qv+Af8Eg!UaAax;#A1`jowaUE%rYmZ585cs|+*UG+oPfbct=ccl*Z za;!sjTA_jJ$DzU*sHwJQ8=iqE$r{Q`vv#IWhN=J0L3uTGM>`=aI|~ie3F#WOi$M5? zoF{q=k!YhY!@PshR$K@WgeijNzWzP*2K4sBY9|0g9#P(|;OmOs@pbXT>J^wB{>Plc z>|6$?FuNw?6lS-wI0Z&1W7_UgdV;pp1a0QSJ-K!g6SRU=+KJNaCp~HxW2;YmZeuBK zgZTB5;x@XRaVjZ)da}aliQ>{}J#573$RHUcE1ZV39Xe+WPDA5x8oCyzp$Rw*U60ey zB%FpO<1};!EOqXJwaye+?@WV@&Ocz2GZnsY{s~_>GvGI8CNZ2@I1SAv8O}VC<;)=! zoVld6b06vH+)u7>9tbZitoMhP32Xh~Wx*zYSlc8c2qlpqlvD@?J$waMvVl0flqJdV zIvL9CgVijQT_9s! zTo zsk$ycC8C_ymAPW2Dy4DyZSE-a-AHoc7ZFRcnT2ZSN98`*8cjqLcX4ResL5pzhX zanXOP-mt}iu?yXho;U(XzLV0N9QHD%=tGChnHaz z(p#OaFwJ=t@|-u|A!l2tq@5CWZ!B$CZ423zY`Ctrg~7%t4DdHj^B^roNvjMYObNQ9 z7Z!HMbakFUn>~#>XOuQp?Pt2S@YSk)y`!k)EcUeFz3{(lR9VZ`1es$(NoZVwZ<-^n z>y3|Tkf!lbuPYrjz88-7j!2o;7nnD#H*b3?kaf*}C|MVSF>lkfZAT-AWidoBj<1CG zqh=N>FsG}%Bvo2aRf@e72H@Wm{JRE}H7ljH_WUN`-*Ei99{uognzf--)#K5 z9slNM+37HSdXAkBAfTPi(X*X)U7Tez?Wh@7 z6YBn1mQ!HPimAG-d5q*2W!KNL3(dKM@}YMwd)a&dXW;^KzWQ&WITGb&bMeSc=F&K@ z#}l_mlx;GX7nm#k(XH;)#13yvDpV1z148o1J>-7#K7Xh$g<4pT@8VG3g+2RyDDQj# zU7Sy#uk$I)aQ4Fj=Sz6P`4;8N?_rH|6esp$@QU*jkJKy%dTOg^kjcF zob3N(qkVntuWXDqN0Ltl>RZVctRa`Po`~n&EV~%e&q5bHRj+G#{y7lLe1MQMFr2s! z=iKGaDX4Y|tnSRb?=+|b`8Lk*dU!eNSVAUF6%9+zVNdFAzp9Km?CBsx8Oe!RMwvJS z)~JUZcEE#~WKOQFm2;NX-)=IW!DW_J(%RDKz}k|2yusapVgtjdFPYC#z<$WGnI--G@|Dmy%Ccnm z5=SKJhQHS3rbIb?@s_<*_fW z>wVyV>ZyGT%#XC%SO1E{Zs(6w)tD0N?-u)~DnHc^)1qm9Kg?DxZS^ZJH$9pXO)nrb zz3Paj;tz9g(4F4LK=zXPi5KE}E|jPD6`1>T({QSr8%>MaSa2-)0c}LPUD#2MsJc(n zT|~VD;^(XY-oj?c&B$1nn~qbT`TdY+#wPQqcNMK&eZB98s9bm8O!FAxdlBnvX7p2` zc|12m2YT^O>iDPqnN$e{>BSSPd2UEFZIfEj>z@`;3w_H7ewxZyhh+yp`H*PpCQAfA zr84hfrNrCE7zzVXLb?zHuMU-XEvU(BLsMQCTJU<%nP$Sd3R#+OG$a8m3SZ0iT5M@cz-gA4<-}& z5ON#8f=uPZarPZS9_J&;GyE#@0>7HP#Yd4nd<@RUV=3_Q)a4^+8lOnZ^BZYpeiN<1 zC(|bUX4;ZQX8gAFhXGU%Y!}BA1f*9l(T;-2dG01!r zY84ma@B^vI#wi6iZ4c*Z!&VX68zTB;mlRqXTI!cwq#>aayVm{|(v2CA$;M-7hA|5= zv}BiVY=!ddIyOO>;xxm1cV1HySP1f52d&m{+W^|>{ zW27Uy1zo8|7xG9<_lkU61zbK|LxzuO8KRUQMAZ z;?)4U(!El!TZ2;Q7`hY+#mhT}x?Vn6m`@D#^NFFUUhWj0wy;qVqLrdW(89A@XFjD@ zVe|C)zlol19{jgI>L9<@>yl?r0zU~9|DA(YxRf(qR2R#Out>a24E2T5dt@X|1HT|D zjpp%bSVbtyM-pdD8PT1{%y;zDI=N?{8iTEMmBrokzmKEeC7Brf(-!&~pMe{623mQ` z0%{w+9gXi$ZixWUMZTdvD^*D!R)#vXYh~uzfzXn}immWDF`^1GSDZ=FAS^-1-_lWOe?CKb_w2TowqfM;aW33R^>8c#jPhWn%d88kF&lT#yY<^R@E?; z3ucILKRnV+ttt!x{ZN^lr^lMbcGg`OQ>U$S7LGJi6SbQ^b$Ssi0w?YN}3RL5- zK{kIKn)B_@4GVuc--(>@J>-n2@55LeKVZiJ3@Q|JUYolZjk+uTe#%dJQs zcB8=^Za505njZ*w19J7;>XP$)ejrFqpGUal92ojLNwa+pB5A`Ql;SL{V@R?AK4iBe zr%~2;7KOU#MeGiCr`AO;3U$%*12c{E9?c_6NT+vegN+tcpwskLuMO4cE$l9B88n-Y z*Bs7)=5)B`KAd)k%e)+f7xQ!}^j(OW{_O60`BW|sLu;jV%<#_jH6V(hn>UMEY5wHq z84~w(!Vm>$nR6OCG;f)679}M{^3FhA3_S%|MbOc^M3E?}KIjd^I)~G)r&0bLi=mGC z+=#PWvxIs#t9coEH!C-x-py)RmflT8Q0Hc1Uc%Im^abb8eDRU_;$yPM(hxRkW?HS4 zysRJho2y>aTTd^;*}xN;eKFPX#Z-0R_mX%S+KHPNIgV>PYI*~ktATA?g#){-&Z|xV zYpboAMYQY?G+lfTn&LrLKLaMo4Ne#CfspWPj z_1qq$fqN-w>h>gU-9Dr<`uA}Ak-qNbWTZQgOmqi@$G;*_P7ftc(jilKWSjKx2cOcF z-54qsZe&x~KeS@u#!#_voqrH)0x|VaGU0akQx6P_^z}8%Hq3-$Y^pv8wgXmcG0F*a z%wDQm*y-zQmi_NT!_U3I@-?PK*y;5)ij@=SljnsOgwDVi^ihwO@zS3JCI!~%F>uN{ z9U6-E?hkbgx^s9GAxDV-?Q*T)J&3^e?sm9InO2Thtyuk2f4_^$fXM2`J2eFVFa|=9G@gb~Ehm zR_`SxbJWip2ha{1mX7Om9gi8Zi{+YatX;O0XV?fQ-Yx_7~|?iBdQod%z})8U{y z1H1Z6Qpuf#-F!A_=*}Tc+_|KKdrznz`xFNGrgRVDtg21vl4_6|-h>P7%%Y?`jZN3x zIIuIzNv@vX4ag)7**~>~=qRb{^MTr=GMf?FaVihg!__z3bXaJWsV=Z080u~2S?#g= zDeKVdbmy#}-dw5Gv_0OFP%#BN;qTTTUMEc1hh07sIs1+>Iw3ZC1=Z-j&cYg`TV;qh zn=1Kqn7V>s4UfIBqVJ<9g$D~RkEy2+SnwLHI#IhFXEc-+a^vg`SXX)7z`90v1Em00 zGE4m1wPSVah~_<*`>n#N7u%ck+qVHq+*la0Ri#a?`%+z-n(X$cA%`4EZW+%hV+Lq(;1J1^6^Wt9#)U&{{4 z0+TxTqJnaT%APVOudKBGHYneIM!{Fm@Q9(ZkQ*^_Qu3j$s@)2@A7Fs0-Q0+&0%G4r z_ItgmV40Pos^GTV)M)AstH`T$^bCu8moBbRU`{WzrWBa7P~tAIrh7%zkNTuV(@+v9 z9dYL0M=TZR?!SwZt`c*i>D$TJXxetTESl!&ZdXB?a&Gj`?3P#u%`R@Txmv?lR(qvy zhhEY2Koy*>oSuH!TwW6Io@DSadvz6PO^xKAA6D626?U}y{BNr$OSkxaebvmWbWmOW zS1u;zEAXu94JtnN$&wj*Aju3VPGim2y2gjJ5SQ-zEmZ1AYmp~oWm=1MFEj4QN?B^$ zoM~L=kC}Uykbn4%W?;`SV_Px23YM-C>R8+`>3+j_HCck?F8QnGXkOd~R;iee{#7}9 zs_0#H!eRBrRAzzo_!5ZfENWjf@Ll@9*ED8XO;aMD%l;efu|l;+#zdS9R2v+KTK-|M zLTjbJ9j9gM`nehLiLWZDD^NP<HH;oAJV!W?X^gFRojhCRdd#b^c$q zYxkITT@lyI%Ys_{|4BnX^WSLbwfc=uWWAHr1RsAb$a+3bs8|<{`Pgegm|hu?LN1#y zQa%h;1#cZq-C(^~V7*jmZCL^h;-T~gnisyUOpa@9*E$i{0LNbmq0!DF%*U$ zhfCb$(Ar%AUEL?3m%9?KaG!*6?kW_DpMqKL8d&JAg;nTY?To!uP#i$FHA--Y!QI_m zgS)%CySqz}!8O6%-95MyY;Y&|;5Gq*<$mY>_g0Q&d=DbQy+;e9%A$iiKSOqW1O2G(C-xiFkDkPF6)@wOe1CI~RB%>!q;`X1*Y_ zk|DKp)864_{_(P_;P68X71dy&0=llgzV@F{8{hGQ*flooUxe9<^Uuj(W zQpcmM!)V8{0MBnrQn`f~7`D-4S-WfUzD!1EyP+jr_)(tiB*uEWnKu^l6gKwpv~4-i zZ^Y96oFMar)H3!d#nP!3FQ?CY&)~#DW<@&A#iw6yZZ(eWoUT9p1Gwtkray`v=W`Y1 z52|6{6IdOWt!iQL7D~v<2OZj+ncl~9oBh0~$8-~@8{@P4wsG!^cPr2o=T|^#nf2J? zzyBBeoUkRYDlR1;q&#Wt>}So!WM4e{i~Buo5zxC;V9X9U8r=AjEJhXj``5U}C%Z@D zAVcU0d%{t0-;0(T#%eed4T9_iZ@-d|cv$}(;Fvenxl(fxw7RI=bVDlw5BO1a%bX9D zx0_S~cts#^#$zkeL={HlrIM&l!;9LYFG0_n3zUlC9NwBZ-7u5Ny|3ZTo|Dn*)tDnY zo77=rZ}be9Qn`w7ChJ4cbO?BgG~Gjb$kx(3ThU3VHZPgBZSM|cFo<}Kj8;ID6mU$3U@&4YYO zTs8bp*4RvtL)s+Iv_j5KxV2wMV@&Nz$S7yvd#nBsUo;V(o!gX+uG~yYf~q8Zqjy+GauBl#&cmO6-A8pS=3R_GsCD)J20_2Lka7R5d^3a%E4?|mqHKu~gkKUMynx znDOgdMItFOw`*b+`*-C2n8e<3&6+oUO-A`8abw=XVN8*X; z>w)Pb@iV1QDh%~kPf@ti#y3dJh;Q!a&m__5rqb`oR3f5(T8X(53Fx1ygHMDaNKI&G zyCXQLe_jF=vjGDfTAd`ux^qzvE}}jfV}G`uj>@;z@O6B)pV(cuQLMIfr?YPM6Kdm# z`ht|Lnt*llQKc9E`qzqxIzQ*8Q%zihxOJ%`GnL%z= z)Qz&U!9ljc-C$xK6_y{=zGz)hY;IcTp}t*%#aHb^L+-l9=jqtplh)UaKq3vuOn zr~R4!)5@8{mX{j5neUw5bL_R5PLkFz<<`pTV=bIZxxHL{CBIF1O(VRlWP`t@=39ns zgAv&FGjl$(A55~D$M*dLT<3`yqJ&-{831g3~kGg-Zwx77z=KkERr@Q62oD!ac_l)1v zQssS{aX6J193^43%k7#rIOPgR$zi|cU7vcJWqtVa7*gJMBKcSxtbZB#EEV9-!84}~ zPGBY(i1J=YQH=^5AhlKWZHD8Qk7H)7P%?KAP=V_x`fq0oB~W{dex8pP52ix1 z$5^D(_Y<79Qp6m(9}xbKa;6N@FR~EC0aBixMG#nqV*Upt7#=ql7xjdT6nCk>=RLQ| zD57%uz0nu3i$(<@aty(5l9!{7%`bEQPv zof$y-lP?mN_>0tKV9z|0a49`0+@@x%#wXt`l9vgV(#xARZ3~CJHAW?Kf$=uADg>WtX4j(XHQaaCqT z7=FAYHVMTIn&O!qb;F0gy!l+dyy&hv_Ir)8JAyw@JIu~vac?Cvry3OBReW}rwesCG zc#Bn|doSv%MRA2pdx9T@?tNtr6M@zHoSqGdiVd5rEz_ah#r3;)-;5p;@XY%|vNdcadoJqa)9cRi9#&z#=)`!Np(9u6-KQ-s5^G=W7*;Bg?UD=mb_Sav z3HUi$kuNfiFz6%vzl#%%x+6SLSc}}7BxcIWH8iy{rydwuN<MX*IVW^L}Ib=mqGENM$*@|KV9^mxp_=~-pk@(8qbb1IYzOlNC55Uu>6@GV1 z0|w_=|MHVh4$f---ltCFpMsi6o_g>gl};~4*LFg0owFmSJH0R+SiXIXZ(a2s|C4!; zj(iH{*eW2EBsF%L{{c~ub+8J5hPwU7SBcB%eS4U{!Y)kz#A^;{u36MTln*2Y$vo^M z=aIm2H5pAXN}6^rZMv$TEEo7dp6-KFGx-y5y3~$}5k5(qXNg*)%(A>yLs)a%o}|XH zJ;gt!z%)Xc3zN+@p(8EUnc&K(&H%oH*UfLbbLfy!5UQY^mOZNz#l$F>8Y8{&Dd{U> zFO+RSiQ^u!Spay5%t8smj6pi0M`nm{N|vYu?S(NhUeXh5m20=nkuy-W3X58~yxyKW z#K|PNOF&%XGYnBIQQ>n&aVd8+wuv8p6KzivLgGq5lbn7eHdJ*hJ zZN)zW0rq1M{qR>by~Dva0me((umGDN2@inUWV6^>l+=A7DMjwu-fJ4`ANCX8ajrT9 zncdz$t!5GJ(3kf2%uSsGRe$nbNo#f+^_jxjNZF4TDn%_a2}Yq`{gp<%PL~3Yn94^j zG73hKIh#;LxlU6;EHAMsAAZjN>ovd)0DI3M%a|fYUmbtL9_r#Q%}Uljk-9@Q<_+Ju^?8!NdGh? zi+K~hq(6sXl+0C6Ewh$u=h#Z&&Ab^rx^{8UU)Cpd_b0ZXln`?-;NdNX zMx_|}x!g-1{L-#`TiT7pKV5k-Ss~PO)AE#2%O|9zB!;xG_`0mGcCJ<%m zmZSefeh@(wX^Grf^^Hm-e(;T2B*q(ai98Km<`p|;KOSON-j@0q`I$oXfqTwXKAAS~ z4FbC~Djx>pTK(`Zx&6zMSZbrhEB#VdYN2#5+&N>p8Se8-vQpa7fb-n5l`RX(*jT$A3L!jb&TIdh-@}7=t{Czkf%Pf^nA` zpI`R%@9(~pPioQd{ens!1$@JTSYf}vH-3XB&#Ym4|Nihp3454G5C)Tk;+|BSZ{)R1 zQ0LThxRe;g#mVa~t4MA5(g>GqQhDl^^Q^lcQZ?ix7YZ=7mp8{81EgfSTrcLoz?*sf za)A4y?a9DYTg=77I=(hbBSW)yn~Nm3kd4}LG?HV4=ICH)w7~tG4*dr68FW zK$2SOQgdW$`Mf0&5J%4>%e@7E% zTKvK2GE4Efon`Sos|pN%k79*AX^X37fil{OJ2Tw;gX}i1!#;)grZW0Gbv)AR)Q@x* zhzjmp)`_NbKC;&ZMe4@n9$LWBwUnls1>**c8I%cXkhhm6H^f@tdl*GN4kMu{Pg0B);kt56hU?lQb}sd4 z+`J zA84`4LqQViJbD(1hd&0o6P7@{r6O;}u<~)KJ{+X>-|CjH-SHCn=Jxrq=N<>j*6H-o zM~0_04unZg3F*-YuAXy`-W(W9WWSawX*F?Chv|od6~|EoR2(mon|b7rx7A_a^h1k^ z!{Dh&%9!R4WssAjX)=E>kfO&kP9}tF1RT-gvLAhgi#d0}w`&i^J0!5$=pQlP<*NHj z(u-p+UyHf)^$QmU$Yg*8mspRJQma%=iOz<@Slt5vFBr4-IBd>-I0dK6q5@r3_K(@r zdXrPfI@PASaK)vcLLfFT>)6_okEhN(2UY-)1)+(XWJGHA_p=W5K7%utK@p|d!;De@2 zqav;ycz$)2PSUiNBMku`Iz^%d89Hzdnj&40M2KjK#Sw!6Jk{${@w=RL9dB(^cI0s* zUPDH6E1Gs7gqOogT}$ReNs5z29`f`0LP_V+RYZI}Pcie{G*KB#caoIV+Euk0@@+w>{79e8v7@%w*+vl z{jIMNdN7CnZkp6J+WTVRXm54V1RCfeIaMq+$*=kQ0e-s-v*bxhJW1gv%$$o~i_mR~B<=-nJbSgbrO8S+brAb6BtV{>aw^jrJN%B@2nFwoLm_`C~-cje2f z)`9DoYr7@8&&L~wq{v7BNmMeKG3CW2c>EbWngX6w6Noed2IwkElsq~rA92lHc%U^M z@&=Xx^zMcBC_Wo_2I9J=Y|lJ!+>S`y4+Al84@Df|rs4xoer>+J47I(w1Orp1=uch< z$(K#YMN4+!w{lKAPAF5<;wJ69*SS@Q_NSnc=n3Wu%)ftY zz)(jCqhu89ZHzU3mmwn;K35?Bz-`?Y;W>795KLtVz@53P;17*%IwvpRq`} z`hx!bAN2o}hPoQ7<4XV()P@5T6z~5k4GNmto;Eg;|AD#=UJl-Fp8rP>3i1QONn~&* zE>)7}LhLe9p^#`OpW)>x@FT1DvmNGp1*kxbrZ@znl!H!QrI7VXBKG zc`{ZPhtE(>F;4vl?8JsQRR-DC)?Kb%9+La+{=EIT05|G2U36fCc%1c10u6HhhDghL zJ7Mm=#gUcL10`Z@_-*qZHb;6ZL)J)6Pi80r+3rw611N#68FWoI)PZq(Xr*dmAUy~S z=Kw*y6O(UXbb1z=C#gz-!C>mMVg;R;z2m%{QU&g~FNXBLq|j-0e!rK$PeaV%`d@4Q z%cs^_$yDptj84Q^!;e}^jzJx_mEg_PFRhd?5?AI9qpgjSjaT?6t%M1IH@;y6t;Jz zosGN+kraJ`II0;V=M=umw9bq{622;;L?d)H1xTgSd}O9=gv~wVfEt*$>F)vUlb+T_DEiNj}mp1=oBuIJ4o+gqASpW{XregOD_(IwVQ>Phm;-Pos!NJrKT4H@Vddvj45_)KXYxr726QG z!m~5=F&;G2DeLI@lm!>GfAg!*uI%r9xqZV2KtFeq0xQ=~-fMpVwf(ii6J>%`Eblo* zJ0_rip2P>TM*L7R3Jw>;MSWaR5{`&nN54YhPgN#-u)fpUdC<-&&cn6vXhiiWsidpgC8cs&U$`>+V@=CgZirFSOsJSYq2* z`_@LqvEx;E^ya$^pe}kpPb+F@EklmCFgoHy$VByfjlL@p&4HW zM6P5bJCiNd=Y3>GP%7@Ig20)Qf9J=)i)GG`-^&Ld9gZkf+;N%eZ#J?%^VQ!+s{8lg zm=3GG96}Y%ZRwlR>ga8ot>iLg8Z*c_YDjp4Uw<+03|nSn$LD`domZf34ed;P1oSuTWP*`e^g`~Dw@~Sd{XJCZTU4Q z^)d-ceS4|1;aJJ{;QS;yNkTg$RLVe&qXXW|eGm3AuQR*T^`6!+p*1tXlrl`{aUOVO zk_)>sy}48B{#~VL`#ok30YvZ|`-S~MaA9>bQpZJ*r)Utf{|V{QImiE7wi^AWqp$01 zqXgs+3~fYF_B<|-jCPYi&=jeOf~d4I61;obG&;vV;b&~G*~yLdYa;p#Y4w zzLX~MJ((fak^;{_#No_N+7A9L!WI~Wh~Jg%D#$B!t~u!b9Vz8X{K_`$VkHX1^5-tFYFuYR$(@qyMijb8_15ZNXcq^^*bhizvEIc?qO#rpZ~pc zrv)WvNU*dGPR1PBl};vgR}$2_be^69>wg`*7VrusmzH2 z5=M^#*zndd?$7m&yDThQ4igi`;~gx0IE`__?OS@OZ}5PkNe?iR;ls~$=G`J+E3|s| z>X!cK4BKLQH4K#IVU{}ee^78iaf&7pV{&UAU~PkEXi@U2blfai0t{E9el0sTZj{&A z=Gc3m)W$k_UyA}=z2~AFymxWe!M!$ATJ@#^9%?y(oHBOtwYV9K6C|EFma_sm-qXw+ zeCM@$83yT%wMbdstHvz~dUKi%kfCN!M}h!m&_=;n;}!4}W8`%5N?h5;9oe!|i5QDf z^*TOP$-~kw&O08P+e$OPTZwHv&Io%M>xpKrox`L?MX8^iu~Oz-IoE;nPbJYcmR@ae zJNMqmR%wC8kgPCwOp33q2~|r1TT~BPou`{Jy>C1qW25G#drY-M81yiV)n3H zM&4y5YvwhRG+Qn&GrR!fdPh<=C;Pd+-1!Q2S3H7nlT3amJDrRi$wp+i%oaD zZsI{X=vu=QC|v=X9zkJS<r8mlixv50Ad>DtmPvK+$DOJPaSuKP)jIg%SFn1q`)>P=jX<(`Wa`| ztb;*Sl|IC$E7~3t1-VR>ioXzIh4CxUQ-fF_xe}%LQtn-)29en}#{^@IuB-}IiQg4v zT8~4I2KM+Wk!w?-X}~ znn4R=6a21YbPBzGT(GU4!xQJE+~1znH^K8AdU-vY&gqnB8S(dnb=j0anWB$lF5zmt z!MSk2H;;J6)!nKbO%toDN=?PQ$~?o9hW@-=!G(K&2m2&~1boYkf^Y4iC+iau~`cN{TGsx@=nY?RdzesbhF%^W03%AwcA{7~X_ zh2sn{kZq!%bFziwgzm;2W6*pR;;H9L4%vpcjO$P5zo#WBNTqj?;1Lu$Q^ZC~w?6lO zVE)>;6aC_z!sbT_tK5SUZ6UTukM*>b{?$SW&O|-WyMycgoq_mr_?Ld zLr?vRna*aIgqmHtT1(t$-{Dxr<1-gIbNiUis-OLieZGCSX>)}#(P^odI+_%zj6jS{ zE7{KllNH)^*sM41>~2tmIqV8^;VA4a z8vYROp8x%?off{BQK6_cv`MgvJ|NAC~&rJKajMQ$)dzcCqRZ23yc78J2997jr zm)=+%ZWr1i@4b+rlDd`5DrUq;SP;y=CyaLy>OJYVt}(GB@TbtLyEFpqtJ&dxGPn!$ zNqSFC9hjQD2L!9Yz78Im$U%ba{|&qh0m*91QY0ZD*jK6hH0oEY_J7jBj?(-WU@Jff zNyMk}`vTxYa4U>*fTvXM9E^<_Qc9SlCfr@}#f$_DzncW+6nfQ`E|M({5?7P+hA9n_ zaH9@1A`QVSU7%YGDQ==B+V=osz$AjNy&->r1L5KrQ4-1EVIUc_R1z3T=#@&^Ou0CA z*j>(>x>RmRo;ol+c>o8L1psR_f2R)QDV-czlZ9xLhR~KaQ7vu{btZ#bfMf{UgdlEj zh!1d}P}~jUh!lj4DFX22DD_(b@l(FqLW>~L371w;1Traem)f0!jQ}CU5ryd7Bp?-F zA(Ai=sP6*I3LN+*ehVusTxv%d$aiHay|4`OqkbiW?q{C$kY131e9d-~zO5{~m4bZ0 z27b9Z1)2iEyA~MKff%G8P^IKFiwWi})V+itCG#4?-jD=fXq-4XEP`+;h6JSK4vjjnA~_Tn)C2&#jdYQK45XVP@&WyW$)VUF zHvri8js)Zc=tmby1jG11wt=Cs;&1NR)PW+UkIlnCQGL0=SnxBF_m}u&u&OjUdIaDD zjr4=FRE%AB~Twao? zUkRkiQ6>pNm)`$HH2{1&@=OAnq<)1J00Bjn<({Ry!=OX3@~?oRaGIHzP5g~%^jnuDSrHe4|2|%=@B19l7M9A|2LNXWy5CSlE zmUdeKSph_n#s6W39s>tpL0KaPMe1MzY12ZSUhyGlZop+jKa#{ePJrEZaVN#NTHumP}8r8o>(%u9Md5^Qw9_n*6y zir|AD5g`i)_{rdHKtJ3K$A9u82nEnD9PvrJa}FFp0r`!J$U+>1{)=uzNdFzP9Ha>l zLcZfBT~t%{Ea{CB@rm}n2KW$`7K6ed0-b@uCqPq);xLRbcWJRGaO?qq`Zbue9}DCH z`d+s0{#$?#)Y78p zvqZ3xw>KWh2@&#m7fJo<1`Gw5a+QYFm#tF;l16+A|GWZ*!hxDboMa*2h5n0k*jFF_ zQNRa=^dOQOA!r@^U!f`t7ZOa}V_KFEH$N$*#e zrAR|GU|(Tg76BiCQn47dL~zl6QUPWNOHX<51@IFU^zYk364=NGat{=Z5D)ue)?31S zj`|w|`xRg46-T*8bbxI6bEu`QFvBf72|h5yL5~>@j1g6B(c11>mz()y6(OvPRf6nChU8adN;n7yiH{E zPi{g#+yyKKR9Fty<}H0)$LDf+e00Zb>giRsw$%snFXk7f?h3Q-HCxc1M@uo!u#XHiyG% zzWX7)Ra${wy)?a-Bzw&IB>wS|S}e3Xda3hU@4VXa-dCcyTuIIA*t)17>^9T7F{%6+ z!M|DyK#S4Zd4z|s!z?|sr0BiO9pw1We=mwCY_%vJu@dzt94AP5l7(t4o#8&0YD-h5 zCuCU|kjuIFy~s6-dqw^JEw>Xv@2(+22i15QueE9M5w=RpZ+PfqjKx|Hlu=Il>-Q(t zLztcY`R*fHdB#szyT;!<2S~_UOj?my%*QjMzXS#$H*?F1F(*kam9a zqP|DxCFTt`U9nOOqJLZCw6HWsi)lgsHI~Dh53alSbTz!tKwr3X{vI2n$Jv?)+P!e8 z_(iO2lH#{0e>(mAKSSAJdpXekKVj_23n*LA4A}Z+*rRAS|0Twhg*_LV300$1VEQiUd`JY{*{FZU1v&Y7|5^@XhamEB)s5sr{kx=cx$UXNjAw zm~P!BD4zNssSqmEuiyI?9An`Djq4d63A2ox%fIlsWR^Xhb(w79;o8QyHlNT_T1<4O zA57c`PBxEqNOBa7w(rWYoig5c^MB95-r~kGzyqA>tJyC^vO@>AGU|(Sp3ZJ zncp6{g#z)6*&kX>2MVck#wOEO?nm-}kK1ya$#c`^YX_+w3JQoV2Z;4}p4l#NzfqaZ zIe6+y(d@0E3SgYM>CT$1*%fXYn5b@RRyN8i7&GZfX)?`&Gwoa2e!2wOs>dF?@a(ZwX95l{fI4D^_Z(0anT{31!d(qEk(%tAKRuMB(_tlJ$54voWK`HaY z`RX_52vIK=31l?{?-H3zcI9(v?T*!|_W9K*QTVEnaj))bQ7qjSGUO7681wX=WSg3G zcG(j*D-=d$Vv+wsNSCyC^o7qdQW-3Ebd;~&U0JaM+Zj?ZJ5f-l z^^xaifr>t?e``M(7UfZ`m*7wyQ+*T%&X04p8**cFHG{}38KXL3*HyD{>xq|jJ81K@ z`&i@7Fcn?flESJLT^W|xdOGY~Ar1#%R$U}F3?d$vOY;+lAN;+RQNCr{YE1LTD~r~3 z28lsCOe!p}@+>pk!^g@aPrDOOv2HlE4?3$3Zu>q+Q6eZ!x)@c*L3l_V&-JgU3LnaM z1h0&*IqLk`p_MZf&F)YakxN2M*t!?sJZ{! zjOtqWef?{uZbqeiDfI1lL=XnMRPi>~q_0^5aUNT{D`H)7ww^>E%Yqr5tT2Cb&htcd zRd-gQqwCNg{ot71edqWU)Pmy`SW2efLrEul?gE^yiHfN$9X&JZkaSYKj$!&C32tXz z*ZSOU7j_XXeB{V^PG#;cEMkAFeH_a+e*4PFnBuOC4~$f;vQ-UL#$R_LU#6u~%on=y zgy-_f4_4R0U&sKnzOOCM1PbS5O%T(QXLk7)jsvJ{x#^Wn6D;X}Yyq*vH zl3l8@ONCmVaN-QGlK<3V{b@1P5d>QG_u`CDHJPY&O&tYZ%M@k<+xQt@tDTL;n;RB{ zOs>Mes64O_`8dY${?+S^;yGORU*YNPoE!NE((<4eTTnv;i7wo-XDzzIjof{u=3#K_ z@^A3%qon~1Wbl1Ys1**!y1p%6)py~F5P?Q*qoUa&6^l;7*>(BN;$dTeN`T(sjpD$qwn-G^p z!5(ch_A5Ct=SQqPTB=onu(XjMBO!e!M6jUoT4T z#P7fh0~aJkiMQgsrbwGJ{P7=L1!7-}j~{;rAEKnPMb$W`T^*_irw)qj>k^F4G%9< zn_`K9Zqfrtu5<%%e|5W)UVM**f2$t$rQXH_0dHjnFmC8Wq5ewv179qT<$h}%PNtH_ z(gTa6Ly-&VLgD`!_$R*v9OM6%Im}8mi_rs`$%LYr(uczQRq;=FaXI$;ZF2aWx*v-H z6qEi$6s7xw{fil__)hq6=AQk6a?F0feAtvW7~vK-DD_DeO7RK!OD`b#&i%lC$9(TL zH8@H>7It_Gt{VJ(d$$nwiSR$=`^)s{fD}9wlso_mitB%`#NDi99c-Mfz5ZLVU7%;A zhpmV6iB6-8SSAVOv=td|4+E=d)S-x(V4R6Z#f$jE@H!QhgUI%9b}X`gBcXr)c6HC$ zEcgb+*=N#c0$b4L*(?p6{FHnvm)Gg6;M;bV+ilz9+2G*6FXrT-NTtZDSixNH z)fO90*^ivmbr(NsvumRDbwQ9sM}wBDNXwcQtsjo6xwV0rWR*NhZD<)Km1@hpeMKcg zOPsjUP#sv2To)AF?O7RB#16cC`NpRCyj-XqkF^El{4zol^UPV9`m&%iD~0V9$X|b` zJcZCLc{q&}cqB@~Bzwzhv;OoI4ffN_#PLNE#&j>sr=Pj4N2J_Xom^M!RX=DmfoL_1 zquS1Miw#d2PE+zHcsjf2mc%1iWSRGvWVYaX3Yr|Qj`{*+5ElHV=Cy>Bnv*fZ=G_m08jK>1EP{6?@906{_$$#&{~_OPkt6s4AJOq=X#XF%ls{ zzy$0==lyCw*CE)dT5HL{@&H(AyHp3frbF~o7h69@l-hpHMPOcI8hR!%20Jop?lGvtj(2J8b8WrK15O+jCcCljz|k@Ml6N zXjq;;Q-(JjmbHYD%|&v8TR;KNsZ()!LTP#pYe&Pcu$kUNaU?U*A)oI0#$<#7kb!lP z&Z?b#AZ%;>HI}y0QvQ)Esfm>gCbBJUxpjFz4o??iTqNCu#H@ws zNF*xS8RMU|g{a`v2Bc4UPif?PASVs;ojz>m*Oc1^#Yi5nsVRNf69ski9$KqbtZeyJ;GK|*KCYSY@kuS9#$Ue*rA)lm|h*|%(C3o0NDq--2rZJk8eX2Nfl zX0$H92`ccKFxHli-Mv%y9|Yb(y!YCJ6s%hf85?H))H!kC&Muj=+dY1{HR|YG=%f6I z?nbcX{A$A_D+Hxm`NDrj;V36;NI}YRkbX4 zK-_B`xqPt_*SROA*IutW01iXh%VJe>!?wj<3`r=Q&att3k#t^GkAb{DW*ttQdv7SO zd_t$Ch!;dt5YJ`_FWSL7;8vUsoGd%>`^Ws z-5H7O;*a8ng1cpmOCUyRa$MJ*`^$kgN)CG$-IpIB#F(Hz5{6m8dmY?x%xRw5DD`_z zgpK3^?g(Z9FMo41T_JLkmUScbM3tJl=iBI~kvx~NRCY?CO?T4MUaor*rpM#p zpsS_bhF%EDf?gc|r@i0CZ8!fFyTCgA>=orvxQC{IL>Ur~Iy2$00*O?*E#0}WOvpA5 z+i#v8Puc9S{1s8zL5!&(qQLEN42Q8@8(d?{1L|NFV@q7)(mpnaAr{f$s`6nwHj#py z1&7nxqt`G@NtyrRA#Mi|J0-P|6|ehu)CdOHbuf*#DFyLh=VJ72^MF zC7_@r{;z+X>Ui0B{#S3Q?e>3q%>SlPbA|!I89HmB^D24#3i~(CU!<);l16pmK-SXC zIY*;ZXhZd!MR>$=1r<#^zj4xMQhsCooe38nxXZ}+vc&D*-AVD=iN9vM6O)e;;~xsV z_IXs6_*frV2b;oZ0&YETdYv~J!n0xC~WR>uJvoF8Yj;oeu_Db`ne}KT)X)oq9la$6gLpg&J}0Q zOvJkYhma3%OxJ3T^>nN1PO$-It;^fPvIo9Iev!Pvax4F+Sz8{ne8WkqH9a*rV0;XM zR+_n>qBJvCh=vh_9qpUhUc|cZO^pqPfulnP#inKn4ON$5V8!3;Sl2>@|FeSe*JM${v2aO!m4jodIWc%Ex z%Fp~pHVAHx3*m^;QQP<+S)da8CB~3coh_=st-q&!WoMopcR(N^P@ENri5zh6uokC^ z&g$Us08bTc33olYk9Gg|S8d3FybRvc{#<)qT5>o9LhYOg6pLA zC}=O;1T!4Eo(9oVWEgD3*r(Z76UTDVBLRIGF zL|aC}JqK9pUbPS-P8%Pfyls3l4D2e|n{<$Q&byDEskLs({i;fmWQD4o zS1sWZ1Pjh6z=!a9!p_zd@g}A2pHG=&M_ZxI@&G?Tz`tq43MTq8eq22a+|{Qx#i8|$ zAea1wKdp$fT#F~AB)NyBD$OQ<_+$Ucc9TgFs5zmgUfO62!KvfNr(;y^iIqdbRF|Q1s+#8HQDz+OzGh0)NSTPZg%A2zq{OOg1 z46KtCa}=xGgg&;ko~)TKsw(+TXumOg@|)d8WZy$?u~G)v4&(f*8XWKV5{v}gX>T{| ze9)5I{V~wwBw5i=lYfrcbQ^xX%EcibhuXD7U(e|#<8M|WmNZQxL5ADpIf*;6>w={k zs7#h&maVwtai_NM&!wn!xqoYbGK|GG?uJ7_+fgL=ZmSqu_e&b4%VV~j(3KX!Dd|t^ zMBt8tB@ACBd~)<4f-iY!!P;6CA55OgPITX9W8s9X3qaxENBdIyQaXGa2h4LPLZs?t zh7{$3fv}rN5`oh?;a(C4c)p5EK zgOT%60;hu5D}j14)>pdNDJ4l+$nH{1*okl*5%h>hK?C~w69rYp-0X-M!tRezQU#)Eii*euV{B>(Hx7`0IC^zY8h{_YU zZEphBED!AzzzINY0l$5p9Ot)9J5dowWpTc1a~KjV?%qmwC_VsxANTji1$oYkzL3d| z-~#p`%(~a7^C7mw533p@QXtEp2OdhkqMR3!uoeMurny$z!Y$fr05u>6RyCg1!Bg+T zejX`o6d!(o=l*;1a_OFG`;)n3cb0u?PIGHTdeWXS&bmFch|nh#hR(%CqOOYBCE~_7JNk{e`avuX5O8 z7W|WW9o`M=+XS7^5m~I4p_{I7pO%&w5sPA-4r;|78tytIE*;}bE_9sMu&(kIl*uNG zmI+Nh?cG3mMg5s$Megev&Pw`iuw0A+x~C@#Z%hzUg3Z-)mr8 zq4h(7&Zez3m&$L2bn;KJ^UuybrAbbPERJ@1cyMtr-!5WNl+@4>xy5w#lI!qp?plgG zF;S*ev}LboI&Ux2oqk4WqP8J6+F$1Wgqg1gm905*Xg`OWC194?eEZ8~?JgF;${!A? z3g2kU;bI&klLp2`8x+EbESCl%8<$F|kWz*0#Tu1mS5zBw=&(#ZOVeeudD5Tod&!oj z$0MIVmU~QtjiVFd?K36HRG2E$A-(j7{DkdCM(2#*#}Ha+Nxdn~QUN~PXY++zbH(4b z;)>uu6wRW(EJ|4Uvr}K3Di?A$*cx8v3n6sI{fN!xF@+Yxh&Ir*8n}d2 z^+wPS(;rh!d(4=A_i|(=bJZwfF}S4{+jEeNjjGnyxGb?(wC(OAbAY<)_r(cv}L?`_{i_MRMOKN_cJi# zbQxOp6wSC&Iaxo-IUa?6IgYMC8fCLM&D_Vmf}clbeT`5?t2^&BeX?9;g@UCxXKqwx z!~EvFT*kQvIp^RzVFGW$mGZ{&ys=}t+QxTktac$dzpT9DD!R)~C)bVbDcE6pJ*M=? z3?OxotqdMq&1P|i3NjMyFVzd1VYFfKC=;b^Zc9z2NDsB*FZ<2+r}<(fS!*=?Qu9h~8-Sj`+b@@R{V7kNk>=xFD{JFx(hcvq=>79olZ**p z`wEHq?2uS~Kki$J46<}fnJk#C)OzlZiE=rx26Mo8$0cLHID(^>TV|4*!r;B2lGO0` zwFx7eBbNfD`{qu&#A9z`9ZbCx7y>C>f2lOV;Y`zSg^InHOTU16L=bN*oK^o^69|h7 zKUgp$RB3^}Lf=Fr7S3&S?c-&t6wV}E!jWZK_lD>HVdNcyGhw1Gz&MkMJ;4*(w(aDJ zZA@(2wx1*u+qP}n#>Cbn`+i^TR_*@T-KtaPUsreE+gRCb-}Lu*%LN@0 zRlDtq?r0N;?+krISdbRPE>@s@bYH7wy7W!-?s~B?QVq+Q=nUuf_&sd!qz>p%z$UaV zSl}cmdGMX8Hv9qk@GIAwvEVMikr1RA4kYRY><7(YXTi}E-xv$8{em+z z@e}gLH2sISKT>c#>Ka^UNyKRsph##hw**8`_Oox@Mw^dpSPOP->XtVEJ!1OWLOXqY zjk*xl>VZU}H>7tn9;>+`>ju|*$Yq<;xz2d^K`N#4wH4JM74}ATEZ^lk|KfIm=-|}6 zSYbhwFfu(D$qXe=kcqu)6|*2J3ytm%odLVo|;h35C{lsB5~g{7uIvan&jU>ggE zv7++jsBoaKSYmm~9PYG)FFuieJC$JYqM4#U`foK69w=3ca4>&57b@&zoSevY zj4+nuCB^PpmvXw)!Rw+n3A}_e1;>yD@*3s05*npY{ErT+%Qkkhr59CBH-w!JTG-`P z)qkq~`l}NeDs%x)VLwJ+RCPzSmh>4bQ#Ms0mxU8SyVVL6g=JZV;9BRy29T9g&A ztrk*arQ(csE*YxXRG_Kh`p2(d<0U`e>$#V%w*OVjEJsp?)p7SM*SIJ?Z%uV7Ww6U; zB((d=qhrV``Ayxv+!N5+BEIHv$?h7cUdRtWipxqO>znlG#VJTZwi_+%-gFWB!o#-TO@Xu7B(G{C#btixSj_&LkmNY(kJ)jU%C}1X z%eQVn_=#eTAjOwEU(B00)jptiJ>}N#PriRo4~k6NjiNX467c7(Cp+bKZR57eLsyIG z0YDBpO|HYl%#;4oaD36**O7{j@yPfScj)r_Z#k{`&HNX$jT3#nssknCUAM08U3; z{x=alU0+lr8Opq996Fj^Y?<~JDU0-(bbP0i9wm)ccwNm>=x>G+)uDLiX>rZauF*vT zApwDYA<3(1$C6--rvO#}uA6G{d9D4ScsY#OzYLUf zI%@rpXCi8W8WH{R5M^4f85RC^(E|tBoFnDxaQsBRHM9g>&=V1nXI)1B=8W9>cx5Ni zG-eo2tovrW`GJIR$r3(i$IDSlAdrmgr2INwF!II`V|*$_)pMm&twJ^I$!#hVdios44 z4zt_rLT^T(>!o&}Bm1UdhgI$c4ul{(IfaA)9;2fc7_jOENhZz98%h7`V&l(uI$M7N zT@6K~;?Yu3LRpup3f_gpz}k1fJKn_!TkVJn<^4RFS1Idt_Zo7lQi32l*|#Wya`>$Z z&QUXJ0(S@IBg(gu6`%ix>mi-%ZyN|?CG;huPx*VdMNoD%X(0v%G@(sPbetnBhlq~sQ77o8QX?9xpMna$7+dJFiK zi};Txm#2cXRDEK)sN->GwrAXnmUm8ypTFx*pBxk!D`E<_9kq7HC?(8=F845g`UK??n9F5opK zlMaY_(bnhYx{@#ZUK>$(XVg`Nrw*ju5m{&STOv0H>hHh)p0jRA7#tvN#PV7qbCk*1 zETB3P-ZLX}sMXLAeV2PwgUXP?qg^AL>p2FEdy$0VKt*wTJ7#0*k`?1)B82a{Y8BLD zLa5y}B^8!F%4UxZpg8TgF1RMWX(h@2K9{Hp8do;_u4Kt2<_gfUUI3O_q1Ap;Tl8?z zC#c+9I{UuXWZ4GkNlv^%VcID6h|BSg zimQ)X4B3IByzwtVgr;!>KbuRUKwN};Fb61QLN3Lki3KYi78i^FfW^T(w_Xz)S1H^K3l=qqJI%$;X`g`XbTJ%jR*MtlWMEAqek5*^m6~XV??UZY-(N+!YB< z>_4VkjHzM3OPtEQ*bTm*#mp+8FQAZK!JBk)$`pbStUdh zQ$O_tnZ+;EA3t+|barJ%+pn_e;&w$S#m+VCc=5j0sMsYe;gSQX=$$uPm4zH@tQ$f? zH$#oo+vg={FJyHr)!Kv6DdF{|4J1+&sN)vSIa*CAeI?Lo%go{hD0)BNkkj@GOju zxX;FlfxYP0mc;)EgByp)hW64(X=4wl%ZjLUrC_d|Q|v(9|+Vz!sB7x?E23S3GX zc^$Pb#`#y36(sma09%?1eG|~80m4|WD-ciFx z9Jcr32n>0_>qjO)nM0;HtwAx|cyvpMv^gWL%!zss>btO zVotl6skf8660JUPHA_y4xGK0ExSOzTYN( z$rDETJErtEUbD=DI!6k}JO$h+8k{83x`>xt&^&R!rd0fc$q9Ofnmuh1<=+eLO`FP1 zk=Y>=6*?Klrg@~y?*dKLS%2vst;F=|J>TM3w}}A3AE#!hKrg*yt8=MrPmKdP;@w9H zyl#5P9V42YVv8QyUA|j>cXd49KAIghDMGU$)lU_@6oGvT<4;2yyW(E4bNI9r%;H{I zbK1ly^A33e>de^QZ{p19rA+!sTZ!}BxQw!!C%bPwjakWe6TR;41h>^?XPVFA8ote( ze@CzQo3C3v?Y_>}dC=FFJ?$nF4_ltV(ECr#1WX#F1_^gRk+NgT5N*GLJCBmJnVj$M zL6-u_*))~Rt^AmK2zO!P>R1{(H*6Q{`eryR9i6gljup90D_e~g9?o=*?}q$;2v)Mk zP%e%gEWJK0@zc<7E>XjksAx3}Y3~|+?ydCuu*{#Rp0_F4- z(4&FEvcU1r9#EPI2j!R7&w@aO5tg5%17v$_R{JUSL6wnMHDv0ncG*#q8Nc`xN=b;d>ij!U)z6Rz_xn*s@nyx)FY7rvbPy=N2T)wUmddu8~QNTvwKSJ+so`TnlN z6|uD<ZiH2>7{0V!JZm~MnQgf?mCkYH1`6u3d17zFJs{c)kc;yOREB&!h_A?OgLp<)PW zu@?JdGX1eD)7s=Huhy+RHZ<_mwz!2FsC6N9rXxFJLco!tBJOh_cs^Mk{%M<5hcNpOfxMp z_1Bfwe~lDHV>N%PTu<3*GG;r;`#P$|qGcG{DdDOWLI+#|U5N(M)RZ(G9%UpO7*F{F zWIa^Y*HvM7GvGy4u_P*NQYv&+%OKwTe@v&nT~-3-`$WwDVpsm;YWjO5 zdL_}55Az)Zgh|X_IgvNVZeuggcVvcdU*p)EW2_h{>a2CvifRZ6ylA*4i@8dLV&G%A zQ5a~`m)iTHeLPLQp!p&gBptmq(w&C&D(jHYvv7Ah=TOx{rbHFo?E|(dY!}(p^DZ0u z^BT||RWS2GT1o4Aqn?-X6QM%T^Xn>*NVuy3kd_HEW-Azrsq&v;nE zSHI4SWp%aL>)r3qH*Bz&enQ{6*}z09yX67~Wb_&1TD`w4f(RZ-OC7dpw;Rd>hr<~U zW`cK%RM)}(kPvDb5!K-}*DAI(wNCXf`toBhzn=ogBOwndan-`Z=4oo#L+uLcUos+- zCy^dG|EZ5H=uZL4G679%k4hw4kRC8HkM{3bl<*O`P!9AE| zES^m(*+%87%(1);)6f-(V}xl^kyJ2#bv#*x#tLY$I>UEm=I0Z~HX zr>!J|d-?!j@^;;FERfvFQT(GLsf{GIo|j7)Cw$mr(-2?Dj29_WZJpaS&HN#+_nIZ= zY9`T~{Mnc{jswvcKF8L^QoUx?3!}tu?GR?wnz*@Jf4o+|mf1pL)w-myM?X~S{FB;d zP+%ga6nY5*fpq>y}?|S+P(@ioVowv{r6NH=~H| z(WA+n36sAHvy9U$700O-#0e!S+eslmGmQiPTcTh+ z>d=->7I#|YIae&G*s)0okr%M5q?LQ{4i_Qn`21~N9Rwz~;h4LH)VQeZCU5r}W|`cX z7aqQ6{$PRW)D~auwN!*#t=${b+60bZUCS(CBMnef=7`@K%Wq}sCfkLrYTf*YZXzZR zYSzpvOq#BVB>g@59<#cl&>1)B{|?3j%-^aimSB)QS+dRL)LTV1<9<--3zz>6*H) z(GU6)9G||E{|pl&)%b2R9fKuG(dgG=-ibYUTlDIy%YWVQPFV$Hy%;9Q#UvC#x_X=Q zVPnO2=nL#5`oLYk>DzVyYe%*u#232`7aa;ujPC=x10I<$N%92u~yol#C=4^~3)|Z!&b6U=a2&gq^@Rz;y?8Po_)K6XOG2gQiHVD4OE|vmUB-ppj79kAGJ=Rx2F* z%5%@RMdSx~L)0X)BVimsyJNYR_eg%FKWg0v9JnX$2ERlK3Voscy8GDxTRDX9iKsnf zYf88hX-^D0M~sw6dWPm7f-fqPP4Z$G?1AnH-x4V(@dYJ7{)O=7+$R4CeV=}ddAB;Y z+t)g@OYDyleB^G`sr^{r{|M$ka;u4WSPP77harCiMUJ)64%$ zSwmAdtdFYNGnb#8=M6RDyW3&oHJoU?nAJ0K+ zf1zTbHR@ZJmzz6RT(4|3#8bYxUwSwn%vmy}??1i-Z!^0*TXUSR&s(S9{rT^?!0`Vx zP=VoxK%56aa|FkrN3CETr(mm9jb%1ohAbRt!#tlgL^W*GBZ%$#!DALNaWyq~i7S|? zS&GYB0$uMWvMUN2qV5+!plq$wVew|O|Lzou;astXmbk5@t2mHbKuH6-SyPj*xvHtF zhyv^vfdisC!WC@Elo1-j=T1u^?GE7oeXyve$R<*}&A^KuGy5tyI~ zV0rI)jgguXVi7CcJ90W9*zL(iD2jt%Q{^-rwmeBQ>0DA%PF3=jW6)I9#RiRJfpW&~ zD*Q!8Viy>Mim<>=y}Fc^dOF2WArlBh*mwl7Sv+syG?q-!X0Z5w#e7;qQzuO3Nz&Pq zB2ucWudFPoW2;RwRO@MJ6&SJw3q+zg>@(AWV6}dZ5D!Y0G*%hg9#<@6ON$>LLz^{s z%s|0XpurZCLQc_E>#67o)1^Qp->KD_5rF+bP`ef;T*S{nVns>BEN)hHw*#`XC{&*u zPxbsW7gzeDrmm7zx+euwZL$XJw@Jx{3Bchi#a2rplIcdsqRl`r7DJGw;ffMImEa*N zQ_crFq1D{$6VVLmlc<;x;;E&omgvti;BxcPHzZmgLj`e_H2sj}ZUC3164WFH2CovtSZ;Meba`h|65R1ISER46Jf&J% zdO8-~bVEpqX(%LZH#M9hY^1aN&+LG*s^`V!1a{~;4Xr>|0b3ru#lBW>7~elM8vF-{4@WC|S-5Cbv?j{_PeOr3)f z@CmAQni?rvrdC=9)26IUPb+G=%rF_UG}+un6jcA4Uqq9=+zD|UGV)(0QZ}>&DS9H) zhbBHWTRq>$g;k;eW&KWuroL9!#Mq4aSV8;)psf$`SE-SVydy9*4MdNm3x%B(LS#$-^}EKwkXAR+CftQ6DZPfFx{Mvtf)byrqJiL(CWIZe$P80_ z8PB|`hK3W&h7_!&9|_hisV2`7)X-CbYn}YOsZ?y#wG#Mu?|n@fayKvJR(y2@Y$qdw zC+rZxfivMxCwc$UJ3F=7u5jDdNuOO12V#DXT8D13L_Z+%oJG_cM>*I0Y%`Lfbr1>q zg>Bml50DD+l7CAV-N%kj;IJp`@ng3DBTEKhWVC)nl9TPahjY1jQkuK10!3|QTS>BJ zf&fP2#kNa7SOCuxbzZL#t>Q)^I#Tb`H(zh*YyRps0`lp$3`kAzC5mT}-Dc|A>9rL| z8KGjEG&}Lcwy(%@%L#HoTlX}hRqtT^Vq>ug!pT+BGR2QrFRo0_S6E*B!jS(^8In{? zl=k_#vXH^ZbxVRMWc8DM2Ul?qqif1gD2Z7~HZ2j`2*xAwe*x!390R z7y)mQVYYq35UU5G3Y_S#_GQ35Aa@GhzA1y_zTR?s8#W5tnT)_<-Q`(Y!+nY1;iUL*QL$e839 z=xeH0;ymPQrJUpKDKM9pVD?O2#rjK;d|+}}&vspk=43@IV`KXAx-4C+sRrZ+3mQ5! zOxaSdk@t6zeBi!?v5i6vZr;bQFU8NH3oC-Qk@6Bgq zzjhPe-}(ay%ab3?QvaTY%R=0;eirQYB7GwJnK^hgisV9fcP%0M zdJV(*;&u_&`MDbk1}=Nai(?r6L}EJ2r0NnPi_Pdk_UCp;0M|yfxfJ(?59cq(0V=x&$S}7hq`vW8|{UPCK&r>3f&f{C4)tY6@kxCFtx#*Cyh5ho5)CX86{W()&7 z^9aYRH=CC$XJsi8O+G+dhDg0fQ9r0tZrM6>sZHqzI;uoXUvy)ie>vS?8uuopQo98W zswbM;u@$Fv(H@3~HE>2%n=>`wObk1eVej}&(E!wk6~tIYl9jc9%H|R{%Ob1haPls( zgI4Rfi=~XF*U5%aFX$Kucy$ zaEFyG@C~%UJt@6UfA`R?gB>WCnR8nAuntYJ_c#(W-V#fW%c6Z?ej}oCCFbX zG-~?c5bJc-Y?s_PQ7s#KtIHG82bi?&4a|f7h+5C@5ky_M$uqM+v)tibGDLZKq3&)S z)_LQEZCVa57h~Y5X(We^KPCI^fTA`UmJ{o!CoFB0%GMwweZ`!V5k$8Y6?Q(3a;ics zhYVhsnXvIpK%}q)Io--zUdM?*SFX6(k%oIuPn%Qbl+~v5# z`qD(j673;6RL|NFb0!aZQQd!tx*`e@1|tQAnR>4Oj43g4I)7r#@8`m)*CPPT>5%I< zrJB2jcU;%JFMUlK2Y#+hT2cTb&`G|$3EBaKJeQ=;j0l}En??oUtjXn829)0x%sz4i z$n|EqT$d_8rC6+tQOIe0uIm`b$eKvLT|LSEMX7EStofsrf?3ezD(fHe+!-uF`6Tz9 zzEA{!pw`ma;0zkw?;?Fg6mu6l=^q~(2mOc>ejArHAeS+&f8#&Xn)$+ImWd4eDNZFu zGhxw^uQTiY6TYSR$%@$3p%(B@V(&|M@9RJAw>sO#8io!g1@*5<`D2{0r&?1*)eH!< z9Xn**Z=RdGZ7C0{poJSI(-z+CgAaw5l4Aypr-Ij@;1CFzwuzYDmWS=w_9NR-|4Hrh zE{*Xsb2XiFWBdI9g?7>Lk-7~2N4l`(P&nNEAcy&YD2#Oy|4c@`6FyNIde(t`_9&dI zi}vj<<-+a5v9lleqg8I88gR3J`-kkLsGX8dGAGh*1()+l>L32e?G710i~Vk56V;X_ zk|R?r3w#>>7rcx{VK`9z(hLi-OK{fhmFSPjUL?JwzRWefcWe{O*)h8~`y(^q!EOE# zYl+F;3B1bNzE9f$DZW6`k3_CMh74uZt_3k<9u${{{3ybCKE$4cE#@54gF=r#gObVq z@_wbYdk$m_Q%8sR0)K33MADPeZZUE*xQJse#gwqBv`YSFW%A&6O9%vd*g(jCLV+63 zeZme{0e)nN0$t)R;&oj%IJtcmynMM%8`aYVpccm0hUd$EyNI@{QTMUWr7>sTnaBrk3V;PM;+ykNj#?m+qrZcpwx)HX~C^IC%z&ge9 zYPQf2z4k#3^mn66z7AEMt_e|Z332IjukO8R%lt+Q9N$uzW+a2d=9N?zYtsx@Bm+fO z&_0hzD(0Dmhf34@J|%Fqx=WpcR$82@uasiXpR-s)xkL9(USDSD^|Xd;ZqI`2Q_c?y zDQ+vu%`Ftkw?ti|Ca(9`B=R5&2f-4J^8=cpZaN1 znV2s>B)H!>MzE7M70Zw}GD3RFiJsE=3P9>j-nyr2Oxky~#dr2|4|>XwinqfXzL7uR z5qpVey8Qrrf%&6&_LLQom2epm)EL>PupGK4GW3Kqbw=Mfz7CqY4Rq|rcp~dY6Mh22 z1tsLNwa`Ob6W6@+Bm-{9lLRuCYS|)xRL#6Vy)OFX9?WumYT>p4bLX1fjl$QdQp)lKgRHi_|NX&JRnb%)Hnz?$YbwEfw=Wa(=W`FE$vaW zy5u#xrUhMJErxz<)V>7y48qtyFyepND;s+iMyt!OWpsFE>E70&57uI9A%5mlk`}Z zi^XeT-jmL4aLpj4~vZSO0dr>!Y+bp(^4R_6lUE?xPTJARbU&P=*u6qa(H*Ng0zU-xA1EL-`_pD5aQ+GqP%4=Fh0AQ^M}Yz>+Sb9Ek~lEf!= znkIYIB<fh(N21{3GNimQ zcoL#fYmx3JOeN*+AlSQwDhE-SWBVgcOmCv1>xZf+qsxuMNX?Z^g{e!*3NV-_Q5&fr z;Z@aEmS!{+D#(dWBHCr}LxEWt=|WdHobyaSgTe1w@YzgZy6>ooPNJb@=v;L1@2{V=ZA&u~gZ!Lqr^7NYfU`ng)9gm(Rc%QL^}@FSSEt64sjAJQH7n zeuy%l7sqK{%a_#}7E`GlK&c`W1)*g{QFmmb9m5r5a48+7*9eK{iHRQ+Z3X=X>PSx* zvyX3Dp*1fTjG#8_=lz2A=ShuIHuimBqo`=UZ#Zbsw(6co^$NG)MO(ROz@!>KKF0H5 z*OU>;rddekqR;%`9xel~QuvWg!!rlFQQrvSbcLp!Ov@5d&!*Pm-(3Ok)FxDI(9t%sh* z5Avw(@K|EN>yv+3QZg^N4l(i$Q+Zto-PjeDs(N`TM{c5TLS9srbySTs|2|NFIIb~` zY@&t<+lCsnTgxBvou0<mg);2eCu>E^G5NEV8i&@&ixRMK%}K z*eJE=4YlZ5J>xsWl+SPikuQL|z*C}`TDL*iDmOgLuSrxwZ{epU5_?1#*~BbCS$tYK zY6Bg0lL~*~zAxT#@Hu*j0~Lse{{U}Xpa_@57!fH@4D40RJ}TIr*thW?;>8d|`yohv z%wOv%@`a7xi;><; zoRH-TVdY_$DWO1j+zq01eGW=O3hsw%Gj{{3;g=*;Pg_2DUv)oz%&kW-WCgi;p z#m_61tV3R(NxXtXM5`l}<@E4AGnJHtA%`)B*j?-i$8mga1A60mCxvs}7Kj0q*RlsO zr+LGPk>#@u#u)XmwO|tYBRKezf2OQb5p|+EcAz?XjS0kE%3gbTX~o)GB1N0|OCsH> zQjg+i#rE*6C`0?lFW&*fF`tU(Z)hyKzPAK5q5drvQl3Y$?fdx%BLEypoVvRgv zCp;{(n>gpYp9Z4xr~~zptR0ELx>|c6t}hIBUG{C9)f8=puL8!^_;gF|~%R zHjQ~O^0mQ?-xN~V>TJ&7J7#!{$ruThW~ro#Mf_7*0;xhvxYM-7?!5g19kB79fcWQ+ol?imV2bEb#yz(xL9wy8_XaTS0kXe%V<(BwJ8 zs;RkV{F7i2OX~|kd)H;xOE;HhHpzmveLeAX%o&|51kF7TibYbT>ji=35O9Bd?<1)% zw3Xg4K+X;Hj3#AF!p?dsfZ;!7�OUis)cE{^Mx(~tYSB)F!djk7_!LYK@tw8ayT z?!WFw&Xi5tgMJsPY|g0~TZWsV91*J}XuU3STP1sPxu0KAOUHyI6Cb~SFM3McolsA% z?nu1YO<6+dqf)hp&X=ha&vA449+@(De> z2W%aVg%6e^GMdN~^Pic}`_-jjym58V4x0;7qKRC)9wj$vk>o+I5s}{*zEHinJfJc4 zkgD8gLa|6s8fun@aNue%GSV>QXmoTnbzqS98H#mE9Y09F8LH1;n30Qvx5w*dr_VuK zt~4a_7TB94`2c$k#x#ztukT|u$nVlzW+(eDewROwuL9n}KEu5$*cQZhN16kBVH;@d z$)y(0$*+%NQLp6-hIpqZ5fsa*YRIY&DTg-i`6YW4`Gof)G$23H33={G8)yNKKi}?e zUlVy|nGt#fEO$r(9S6sR-tu2H{Z7Va;i4XI-z-dqzP9;#VzPsOj4^6vCK<3ZzG3>7 ziVJ+!;B|qDqEb!ZQdeLC=B3k(|CgzfvqNq2Hzz@k$!ZSMYxPac5v5`J z=XY|;t6&AL0O>{-|BbqiOMjIg+vLAw+u}@?_6DrK+K>QRof`w)eLK9ObnZCzjD@Np zA!p`CYMolOAa*wC26RgB&gBB8$|mE`&JHug{Bryt+%eoluIj405dx*%thB-MV+Z4) z{UNBqYQ9lid6H!1!;`%fb`D4N9cY@Z<6jp^*(%mCsijrY^s<^BNX^fQ5G!KVQ6}=u zCCq2PDC5g$$puh<9hUyZ`mr4|M9mbh57T%avhFfq!-Hzwk$R))T-AOJ>IgKFN*0BN zgz$n}=^C0;V+ceB+Y+Go6^w!6AwvJWNk1L4{xHmjO0j_Z8pR&m#8R(Vs$)g+UNWQxJ znmXOXY>l(Jv~RX>1FjqPy87zH>^^PMLM43$;D2QxUj7GZ1y$B)m1gpMi1ET{D^>K` zjV*3Tmpd72YUDw0xVu8J_C)pV?02r5P+f6Wd~?-0W#{y4*Yr?N0Qp;$$37!(kr$?Y zST_e=`IjsF_UoRb^lLG9Cq~!*MKdz_jyB#>E8a=_ zrvS&$)bM50U0&6S|2KDK^Y>r8%3scS1fIL>?*u_%s}@G-PTc+Xe@wa+-3N*?+48;L zhG$p83~1B;I_DjI-+2cgua#sm zf)9oshhfFkVcJoQdun`wzfT}wQ~(<{)N1HIY_b9;`*yA}sNQAzxtt&-Tt|ij^=(gj z<6b3L1xYY1pHvOU#`!-(4dPtqVB+0f2b$r!D*v+kvt{wN)D=^vnpS2JrK(m#o2^*vI=x#bkQ?>eJ#Tq;;54`TMgO|8ETFbd& zAYjbQ`60hYu2K(u=<0fD_neQgP{>9cEH1C+Pyeei+~lRn+jHy4F2fRnvz=Xs$=H7M z+T`V_?(x}`9bC573V!7BW)1Q(@K4X;<~k_v=6|fnF;?$<+5ZP{clBr;1h`wa>mL&Z zcxU)>@x*x@jrmQ4Y$h9X8nhU0p~mqq5FAgJw<mQt{-;eJlQha4aB@#wHh8p#_^B;xjZLoEboBh8B?|C_fP?NgZQ_TQ=rY; zijo@gLJOx|E=jMn2zuXh%upO26|QSib0fpDzn|Z5_vT59psucWcSYGdx|Ro?*qf z2bt71O!EIb;SI~t?R`W3^WR>6j@C3)^35{%&m;ZSLHm}bi3d_mpg z$!@;zK=dNLRqOuW7F^Gn&su-zGByiAMDQiX@FSa-Yo}|cUaHykU2v+|^Jux}KBGN- zJxn&i2&!t_V4(fA+6A84~M;%;vg&BFVD?rFjYmeHQA9ugZ&s`yomO2g}R`<34l zJuJdC0@tB^2uEby8dXnK4N=Mnap^|&?KhMvurd3azEiSVok>Aq^RWC%cJ!kQ=-(6T9d>7tD1YaSLS_xVc_r7p^%sl{pYkW!zYF!LOZt68)yOJ{j>LS5 zsfb)heTtx0pl5d-s5M4c@Xc!l#}IyWlKw-N?fS+NoA#)`LT?b7T0w@nlErKH^H*k> zp!kl2xZEj^x@d9?i*Og@qgPk=)>vibqC+*-XBVN?CZ4|!CwAXlQ%jluH=2Y(AC0P~ zW_%rNF-JR}a9w@3CYsf7kwLqI?r1&;zaLp0O+xu^{(~C%fl^1L*IpH{u29&{*yT}b zDOP~N@Y5bbW*{SRT7-*$*_b5-@xv20UR#Xzlwwl?1Ip^^W4dvqbjAL-N%(fxH+zWa zwerN0X6C`t-NJLpb~h98rIIvTKL>94y%ld^vQY6Ko82zR;f&J8cIpXk#%ecq#Q;h` zwZDtPSEp$crQg_3h|{vt?jw1cmZSJ!hHMP`2(mo3U|K zKBnx{aIrLotemYfowgEfWwN|9SxUh}%yN&lEV{JYBXsFVafI`>DOx&-0=(!|hIbgc zGVi+#4-DX6!fWkMQ>H4*zE?cdPD^XOqwW1vF(3#m17iJlI)gNshxV?PX4#7oH<-ih zdiC}x52R(x@FY7x$K3poA|u`9OJUB_z$Q4V@4wpiJ?`nCn@R0kCQAl>8%2CI2EbH&KoZx=ORwqNbG;_Pgy2H9DBg)*u@{c~t;eGd4 z`r@H(+L*xZk_kQ{szm5ti&Ei!^^D0svi&i`1I4V;^3-Ua^7=R z3u(P4Ms}{<{t)lDY^(#BXIN`|Jjb+vh5FNw&KYXn^&hGh$D8tn`FZ}&i_RTt(M!5^fDoJ z{sxmo>Y5p^Vw#gJ6R#3%r-OPayMiE45hZjHouQjKw_Oo?>j8fABKCy*&-WDBK z&tGl!Bevs$y#3jUQ_j*Bu*jC*5?2vDy&0zB3{00tuy=}@@n3~)E#9QHXFil276y)<-`Wr~25Nl{{u!mT1fR&|Pgm^Z}LJ>2Xn?{EFx%4>vuLwwGVFoRG>2Dc7w9Wj1?g{3QF2 z<>flc`dTNF>`0n1^$FNBK^z~RiD#9rr4+TbncdE|hwjj1#;-%I2*2Rs&*}0*Mmdnv z*xXQZ`y+;oY)P-~&-7uOX*Q1kCxBYwAKCRu5UyVp!lc??D!Q;{pgz!8CN(EE!5u#b zH^%~w6hwj~kORE^!tCL_7e5ic|GZEf1sxdZ2*;^A2BSY9jIft>hDl!6Z_fD9u7S0W z*!njK$4D&bExY7cX`~gV|9~aOJj!4fu?FN|QxZ5G61)$)ysU-JTfWPpyMnmGjJE!? zl_n&&^E=|<2(p~0I-v=&l(gK17a|i={>QX`zf! zgM+cNj$4g`sLh1VGtJaif+fpUu=-`tn zi4*a%{mBKbw2EbOmRN4aD%5j&26K5jPmvinKxE#Yo^nusk?p2vy_>xYnjHNi{h>HM z3$sst*j83#ES6;=&K8hVVpnSk@1>+=z<+h&s{>Vnp?oVaT#bWgEThAe4dxjCu@^^s zXiS~wPTNiljo4>jVi!&K&$r2=>(*f(`w!XkcZ;8L9lEjiXu-1Rghjh|HJGvP{Yi`2 zt*qJ&_b~I_~1cL zYW(V;HyIX>pJI)3+Htz0P2moNT|00~+%q;>4E?6ko}@zIR$pPcV-0WCmIx?&(5F8X zxOr5cEU_038j-fV5ma9;*zZt6DM#Csa2qa>fFuE!R;Ys^ z$X`tnCWVT(O;Y>s$cIf*2U~;e^rYPa^ysaYLAh!U`!zPyjcub^+8hQu`zdMQ{DsWj z8M++kazN$vm~1G^e2>gvfFoWh*4)h*h|b1rkS6vS|K+#j{bPkaJpZvOBy_5EMYL?` zMX=m9I$-~oSe_9bQ(?it_O*_7p>hZF8E)WuwxI)KAG4r9I53P~7M7BxSTZQ<8@B}R zS5tg1rd~1r$`S1`O{KSZhbpx%+GS+4ZN$)>g&FIV8}xVOW5zC`gSq6@yL^t4{xyGV z)MppJ?fl@ivH!PjF@i$EN<=!qA_48K_*&I~rNq~ySB&m7)0QH+pe3_N?x%&p=E#R5 zAB-jXC*B4t-j@en9N||eOSZ!U#%>q8w8$St54*C~~ z{KsqW>i@ymJH_b2L|wvd+c<68IBnauZQJfXZQHhO+qP{!?f?Dq&&ABmWG<@K-lr%!ADXxrAkHiDlqVC_;L@|w?Q|5X)Wd6ky8we_&X zgnnIhg9lJ=)U*k+wAq0GmViW}xe`P-D~;j;BPVgS;Wa@+s1Bai!A=ej$=6jdgAks_ z{&j&BeEUHhG!(=R0(Pu~XIcp(t6<+&!Nt~v7vvNQRzUPzHj=4DZ)P3~GvN8jKntQ0 zqybZAxgIkxD;UpKRiI>ff`zG>?f?&c{bsDoSGbr zY=X|F86(R~(;#{pxtu?`2NbAT#Lsa)mlT`|HDHM&cYPi0wJy`2lVuHQOUuV#0e*v| zqk)(LT#I*(9^q@#zpbw2XCoooO@-M)4;^lKXeTs-!+bX@!Ab!Eyk z$br8Ve`|160(n#eFGT2`Q79yyFDxEHsJ;z9mssNMq!fH9f+=@^e;jKLVksVf4jgo# z8HtBL)iR=jmDaH2pmcFOEJd$Ss_vLekiuCFN^gYUl*j}AU(WiWk8*cq!eK|LhwGQo z@J_NiITe3H!YyXjVWz|by+w}o$`Ed-U=_ly&mmv(@V{XrN7gSqunDTFFk=OB{}5IH zwAKK_Jn|2$?t#Ap))x%ufjbt0e?KQDkn+BKI=H_eaMUAH^pF#Iezs(}A%=|!%&ZA; zdql~QYBjpfnAT7YEvTNc?LJ;=WK;SGNHyJ*!HtIZ*IKBn0k1|pyg|-}!&mB9XJh2G z!ZIwc2JqELYU7Xw^wp4BgP#rj)tsCe=thPO($=8UI=T&`cM|-GrVY>CAjFfM4gKBB z>4Ob>UYrRmK%+HY-tZqFp*nufSp3O&ovAyW?u4S=*b|*Qx6XJ9a8aF}HzfrasZO>x zr2)vPjoO=1+;^>0{}5`=TxpW+ja34quSI?!st)cN*}hm>Q~hUFAM(~Aez>_Mq1+lJ zjLRZq+%lmWmqg{=!m#>h(S$cP4x!v)(}!o#SvEWmDcu4!rs+|{n_x$jT1Bf3Fs*Yu zkd;2Ed)Ia?^+7RkXT3MqHrE1Y%pAvhcYA@+lnxL-#XiX;1aW+<_H6b*DkK6iZ?tc8 zkpy(Qw%Rtjh+_6&-nQDd+=0I7t^0eIdSO#otW?yu7MFHjc{%SajVgTWd@X~_z-~==wf`SnHV|z;q*8amkrp@d$!>Lpn{=&b& zpu+dBdLI1zpPIvmyMdt0{BQ$(uY=*m)(@($J;hpmguEYwKA||UzdpZkCg*+L`(--A zWJ$eWuW(?`dpceCKa|r519`5|1@ywU7)F17HHgTAsJVylcX~|3nlmzBY?1P(2>WXT z)NE)>>Z|f$yKp+2@aB_rVZS$A8AEn}#OrYlsk@*bH)iSsZh}m&*(r^?3ZixptT%GJ zq)xGmc?J)Zb9%(HYT+|O{=}%3eAjA_6?F1G%0$@u1|XDi?i;sL^j+2seyfZ_W>^`Zo54NR*MUQoLTo9P7{6%)B3 zHRlB;6DCf=P%oc8Vta5~@Pe`o;qbl0-^`PwM_uNCODvQn7Oa=(_wO25*-g@OlBRp{ zTbsg^P&LaG6yJt}t*|iG=e7(WXkuhQRL*QdD07EvbQx4aHNdu5+CyX*IPrT;Ut8+^1b&bdWU)c64=XOuU1 zam~UR?i&kp1=}9&3(mc=(wd&Tx~6~rJtl8`gr{b zlbNdh&nR9BApLjLA$&6utYdTjX(wz#Yp)g z>yQtU!j1V+za76|g|MTcay83VAOx!5VY@J1DxD5hbx<4uv5N2DsOf)C>ntKj|9?PRx0D-{oRn4_BV zA7^FvfpCppH*PM9a!d}J@ST@y6XLxJ*K1ZfSAZ8P8|Sy01s+@k?{zg2w>zdyu{-8& zkR8KM4*(;X<;#16E*^t)ZmOzFb#11svFf>Nnb#)VFHQQccI9sIwI27mYxnuq?7Maa zzg4<#>W{##7)Do5><+e(J3lfU(){=3#y5R3qE$|y0^f2y^Lu?KMrT6^Y-hsYyS4kXvPbG|eYg5UmaSB1{y* zSNN(bFFI5ie(Ye2hWPK;Z~7>Ah^`?&#FPw?C-0o^-fv)Le|fi*?*jISjcejh5*2Hh z$93{g{qu|B`l5ZXoA2VVnBrOj_Qa0;KOsI6WcDGls|b>O$~I)1k}i_$MX#aA=Y`qZ zbaXT{RV#M}ZTvtr3Kuux{9y|#thJ@@We94(FjNkMRQ?#o6p5pl&6z~vBdV<7i%F}j z3};#!-@&P=La)?Ld$qxL=1CDhc%o>+^h0DD@G3IuB1j&{m!n1|(I_U-N-8Wg2ILIE zBVS;=MByZy8xp&5H}cIOv~X2vLZ8Z?EH!#Cx{q9p+NjbNLCqjP6fOeGEww}p2Y)_w zFejwz8I;tDE;n#Pjg{0oTi(Zf9QzwJMzYwqTn=bQ?KKPzK>t@A%; z1>Smo9)+~arj3dAD$V9mu`O7^(`aH z!in&0{cZh=i1=o9kA-}CU>}aXUU3Dx`(nQ>34f*peByf@?g@{h0m06Pi1et!-dsZ7 z5JKMMLf#l~XIX|Q{9}+e$BfNMH}*Gotj$S7^RbP6ZT|D~xh8qrKw7m(TDe+dc<^-c zYx9q4AJSKh&DeYvTclyHAR(_pA#Sij-pE4U&_dppB=bKOzso?Ic{lp3%~#hi^v$0- z7XL0i$pyDsW8E3&-AI9iyZ`+x&II_C|19pnvkG;m{p?aiOX(9(f8z*`tpC(ROr}Ik zvY*e?1tI%z%bchW@${n346*SgM-5Veo?Zn+Anqb)Cd@+}73gBg6|e+*z&Srmwm>Yl(eg=g;G-XQbg-*p9d&@9 zK|M5V8H)4f38-YLj8RxiTZD3A3!on=2Ulzm@zD*NUxfc9;*;mz@Cjs(_c3H_$;HH2 zeJE-IFcJCDzMvdV33am$tA_!oT|Nx;BGdtuIqUUWF&8$r5%z?_lk)7qqd9`y|MHw* znJ=7lT@t`JF}=6WEk9f9RM5TaMEMF6yZPB+1p0dh@r+|HGTQ+i zx^IRxZXc)<-DUoy>%iIv8ugx|Fme~<`enzNvIly0?|p#cM__aGHq`P3>=Df$@%lpq zhz{T9)`RT3=e@`EPj5MltDo)8ec6vI%<+$gKZ$GN`2ouse{0P6l2jZ09aVcM(}4HS ztU3Bz-vE5;{Lpqq@Q=|S%{A40fxEHzXY7pW8+rq*uQh+(9PA2UW?TH*-il_#^;&-G z5%WiG;I6LuZ~EqY^-p_9a^bh$&aQ)DntlCePhMBA4M)(#?Fh!*E*aH@=({AK@nWaV zD5xrKJ{N3S*i~SLuJmp#-U@0OlW&^tI{nsr^T4Sf^Yadu#*FX1k@SMm8lS%$8lQbS z5h9oZGAd+!PX#rg1I4D$`Eik^n-Gouq_$&>A%eUABGgQ1D!`nyE!H}rFE!PcF?Qwd z$^3D$#@ma@1?lTiln>lHc|rs568RB$!Nhn%iski2N}cDO<8{aE4sibmkWGM3zd3=D z%79lmv+$L9o0L)+6kj(Dcv*#uO4VlbetQIOh^lKR%!+tiiPYDH_jp{W-%B;AqLn2&CwWEhG*pVYtmq`B5pDz}yQU36SuRRCSU*>4cSb|Zr z)Y140clxXWthAZVeuEV)Q^G8!PQm4o1CuqQXEtCEe-KbGW2T={!7ytaZ`}9HGW>}f1-gSCMWqnQhf3(E|d4&VcC74~d;letA z+#(!dWjxkVSl#F{ zLF+bAoIe$yz!#@z4)p_^3wH#hcM4N$OQibOsy^aT!L?1UWtY5Z0@7FSQz7?Nu3xT+ z1sphO1#Mr;?TTyhQTCCgxbYIxlsKY2Lino;A0W1icTwv;B4@z zVV3nOu1Y!o6PbTgu888Hkf<|+pIlmeVqq?X>vm)=3DoYAcmJx6@sr!ckoq1%iX~pS=UT)d{##{SQFUD^ zY#GrkRW(f`t}*MPXMfR=$mZA)C4iJ5cm>RQixGLlZk%PBEv<_{wtUD zMxA)gx3Q9NnUS?EYaKfGS}2O|eXB$bzW}fQia*DbnSuvX&I_^KDmeNsoNJ25mf3;b z5+e3yuqF|%KBDI=)I0Wo4_6p6u#$iq;b$fd%is@(T!Lzi$HJCr5+aWCA`2N6bHJ@~ z#KlE9VzXPHX_^;2iyA3L@|Gy`)p*loOkV}11TV$}%g#rY@kZ(`WCQm3Z-i4*SKk?Y z3%3rf(7!6b04WO_eLz9({pr9PRrbqD6!KS!<>pO~ol7pXyW+b+mIwrybT4ajraY+8 zcSk5yhOthxt8Q?^#g*8~EE{peL9MCqm14vbBq@d^0bp9)!w4T*#PAcp!kJj>DC_E2 z!i|lv<5Q+jVz_}qnyhh>E6CY}P86V~8&L`}idEA9(zKww^JJskq$b}G4UJ7{4 z*h(;zIJW2t5iKe7%Arp3h&sqxmO-h1;~38Vs zju^4)kUSp9IihWooDLWl0_9>n9f&pn>0!bi@HSxaVT9gya6JqOu*c$P6so5B` zMI>#29-_UbbEfpr${GJ6Wvz!Fy1aI9W_=T~JCn~IC}qznkAQavx5v9hjj2&hFoIrz}yrVdb<0D#Q}(M%=~PN+0!Cki8~~8T_?RjChFO#nx>; zgw3B#Q^cjo4=6>}1}TR*IkXGDfjOfM;h3Kov|h6%{boJfkWz(kJh?!M65}#HbE#H^ z$578-uFIN#9el239;SfLL-Ifo`%zj6Mmy>tj?B|qAr}ag;5$J$p~gnL-#TxTqng-f za&JGfrFGLTsg-c2h;d9ZFxE|p6oqg|^M*Y{Q#DF>7uZQpX(Xnd7s&spGN=`qqNcbO zMs>?OX%a_<#d9)Qn1+kE&JS`~3mMha60IuAiP}JE+3BCdMVs6c1kYvl|i!hBWCAg~Kn*-BvTxo*A3lJdAWo z!3OHxz?#%Nrq!uGDV@&j)5zKd2l;|Y3rx02*cLA{_l4S-m>refOaz(GW<%FocG(j7 zCd-D05Ysf))qBEQYyY=Q@4DEtr&Vq{J`az^pTo>}L6*pooB}3PA+7`}} zxNP!0%|}sRLHjpC&xl`hs(-)71BDp0|NED+e=?&o9yU4|C!^$yQd5h6rg@xQd^vg- zG=TeZcn3bPFMKz`n~h`_7v(K{SU6-hH{L_T_&#DFU(erj*~gRCuuzRyKuJKI%CNrK zAi3EvN`+Crbnt;0iC?kmaV2|Qq$aZ}c9AlsS-gW1ow&oIr!%-qA)1g^j*tzARN&u3 zBYJaMjqlVp-XN%(!jnoMetj@EVNqVKQ+Njn^Nl>pNp2N3LgA3PZ^7 zRHj5-sO#t+>glN|#$pBCVg=q}1#fg39+5r-ZpP!gm0&QHemREa$MJfwZa;wwaGpMc z9!3AJBE`=J(tR6Sx|ykDcU51vd9P1-pN~nePhz~~2@#a9dPpxWTzvQ;t^1hfEi`yz z8e;IfA>3)zNTq@?BY*UOON$6z4HK3Dq6<^|#E5}b4`;8)Y< zW8CYLm457teC`M3;t%Ep1d;=g0pXF81S^*aLEbpK+siZy}Q+Q z>Z{xXhz0s7ko($3+Sg~T`%-_J^cOx{iw6kz{Vm`711E`*=p-Pa)!>;b+|Bb(=HZbx zq7?3$Yu!T06zpaqMHqQ$2zlvXUOG0w=8Ib+HVF9<3M<^7Vsb%&c1$4j4=lyxgF|LE z(yRnp(UnVgk2vnFtf7~UlBiWi=~yWvua#;8^#g^r8O=kETwBXu_j~z;KBSP)U^*AaX%at`(%Lzc|%%+%1M5;pBE%POqG> zohD??!B2Re3z<)Z5_P1r6Pc$!5GG}t{$oGf5`53+VJDTg;O>>hMVWE)88#AN=W1w_ z32xGQYvUrNc@q+pPzyiCaZ)?^&~@NS$tw=mGsLgBlqB+u!vA>o5H>Fk8{=akowK&N zhMmCzrP8dk)-I>u<%(}Qc9l-`%~ zq*^}O8#_6&K^y|w2T$x82*<-pf^gO zH%#CMFVF`x@JpOTAh%%F22H|H8gqO}m#}H(R>IR6&%3@LbTh>PSgbdOuqT+XC%p5( zmwx`Q{4OGghVhD zrww1V2Ie8n!o34qoj`99G;>M%QIV_1Wbzr9T=$fJ;J<)~T-Pn_^NqXVb;ho1>XyMg-E>7igz{>Vz zSH5j6x~cllD_D*+bNY3WPtNE*w$$Q4(}&EuGd%M-Tv!8)TcLX`B)(uC=y4yRs2|A9 zaoN16Ul{(y*}3NzZrDT}qsAA6*|@!7?H7Je96`Ry4^-8OyaI{87~*kXz2py!_OX4v z>TP(=BhYoK|Df+9;dKMnbrWZ|4YbEn;(f4b6BN9NxH@_MjV_A^@wg+PfXPeGAk%)qgMGX?$lwt zNbEF-<*>`4o7Ft~FrqMB!nOMSlO&N*Q_|7IWnmpSFKCuNajZSO=U21Rt5b%WpnKbabX#lyF%4#A4G-rQ4!pXi@jjn5ZMQ*8@LKi5eJUI9 zZX5Yv+A1kvA#*T9S)NVkw)Gz|OS z9ldfI$q-zqificM<_N;wo?+99nrn#}yZ!I14`9bu8*J~~c0SQmuy(Vw)(JJq!fuCS zOZW8(%4r<~!yo|eXw{F$mZv838@&15!$V=p{%!Pa_3rxzGU;4Tn z+m@dRM^02&tjOl2!DZG0?0ujQ3Kj0@f{0JnzWYjz;n&5RQ-T)`N)xFu-EM8m^!0oA z+U{;)+Kz+OWpBvn}9uK?iIF8puRJ?-xp5rQc9_|fafgf;Xfr+x@FUw=W=n~gP%^yK4@gIX-P%-G~kByHJRqnU;Erm07 z1@2UvsLcYcn93XrJR$RbJ!aW`bQG&MhsqSw3Z!ncN4RBuh8u`{*Rgr25EK3z)Nz|8 zItQQ_5szE zl*jI7*rp}7kAW_Ho)-7M|Ee5dsWVJ-wS{)46M=2{h1O!Hzf+T=ad&mb!p`0lwR&s) zkF`5ML`z6kR0m~M-gyQ7QmiKLH0)&wxJ?xLl7efP=s*M@LBQHF&-p7$VQAA1X*d5l zgT@1{Jw1Q`0J1Vda69K4l&-Dl*X0eH{_J@PzVcIQrNk3Fk4iFZS)9(Y#dp-ZUz#M$ zCApiMB)*&5(RZ)f)qe}j5VU$uPMgOTw6SR^qv*-SE~ZdY*6e`c=RPeRuD&a2+y^j6xMRYZvpM+}>j#`R~ zFVz?bKht1i)S&Ll<^f+-K%&7=iV?+C#r>e>kE{~ZV1V1*ORd_E!xLuI07go=5cqn_y99U2QP{ZN0+x*_*f+;P8O1B0FkEp7WI z;c9o*tWZX%3pl)G*-XI?A%Q&QRs#!<89SP1C5|fXczC@)8D}|cvq4a=5k2Cp8@0B> zq|nSSt`^ul8Kc0Yqtsq7@I|JSde;Xxdm_7#r?T^i*a7V z(%VY6sx3BUcQ%K3VOY*y-bfN7I7{U zA|3CH9o^3{DVUD3N<glpZG}9R*+Sk$WZU6S6QDSlOk`(g?m-!K-&k1MWb0+& zfW#rT6y}=l<;BJ`@oZ4x`xQG4f!T_!o@86VHDp2i{R$K@FUdZZhCpL$+Ayx%qz?^z zngvhN4ItQbD}nWJ8e?HKrwZ!{H0)C@<sVzcU6LB(Tc}M`J(3No%FjjM$6L^P zjHIJ=8%iSWBkizi&5pVJD6OA?tPkT&YX;nP>t$(^uZko>6zt}6vX2waO_qicPpCxb zVGBZhRa`2=9%Csfcc8e73`6ihAIW()u^k2YmZJUDgyZL8!9AO9k981_pKyYF{IHHK zbo~ZU#a`O1Z!Uh^YJy2kF$-VagKB~lH~n>k$g($YG`^u#Gix)Y*tl88cJp@2-3hd9;BK&WGj}W63B74WG<@6C-3WGb zS2KSz_`J@&R_(&{UgL}QHGVVW-|}7e>teukBe2{0_UxZMwY?GS&dM3`-U{|?1TaL3g(j@`4pYV=Ovk1;fjJ>9&5`StK7+>Q3R zZzjF>yt4j8@aEh#nct^fXMDkbO8oHse2?eO@VA^jGF%q{7+!h&=IN0pa~ZfcU`5ou z6V9y}Fov#$nEoT7*@pN(kWVZ}xF6nGJ~gy0QKOa~nd-KGBbIKF>o#do{~qp4Utt_e ze1q2Q>QK0EoE*b-sqh=BhV{C{x+DDcTjWs))+w%dz0L~z^44Q_A!}CfF6G7p{`;Z3 zjkkm`EVS;H+=jue;xVuAUh4Va9HcZd)Aq>kAlk0SXnX{CA?2M4H+VteUBanh*1plh zskvGqn%vvXtMXRk50pLVQSF74yj&_;480j%7+*U2P>ZO?IniK$!Pezsg}vk|co)QI z#J%LI_z!tO9#|>M+aM0{$mtwz{$Lw5S%1O~Pt@Kppjqo~`Dd3a$y;veQ^mU5{#qK< zdTHGVZ}o}FET@qa50DvU_x>3OoEovX!7d(3IfS~wZboqzU~t!IPCDQXA*eK8Zt_Ar z2p?rZsw2OLdj{m1_t%BNm58;tDGB)&x4|cRN*EJMuT>o1mqe!5Z)b+Gr0Av#^NlVqJ=_zv>em;^L~?j$a&2 z-u`_XHf}>h0oqVF!Ae0)AqptMz`*bkSo#4o7Y7Cf*_RCqPSsx3n<~HwMZ_x7zWKJH z>BgChUaY|I27^@w$z?%=VDXPAw8$caz^#25T||MLT;Jw>Z@W348C}jv@4jahwrFXK zp&7ixBchw4qMKdi5?Rr6*qEXJB{G|>yx7;}#5Z`VAp}rrZJQO7ICYgRJ{URv#{a%K zUU$LmZQ>sk%Y9Y4IM`SGNtTQjUfC;eq^CL9VOnCMGMh1Zo@9PsVd+x*#xUP2-cbD} zrPnKlv-udpfO}@*H3<(*n^}~O3}}Jmj$nMbTt>?X;mrjg_~1J@QDGjiv4ZCY!F&+N z4{6h*n+m|r9F(8p81eK2%pXM4W1Roie`8OktIZsEQ}u$l0&td(uIj;mLAW34yevRs z_kALcKoWvXZSr@~EH!4yFXV&-=a32c#sPlxW z9wN3&Ob?D6lKIF_kG*Yhu0-JI5T_m~Db95w@FQh8bgT!`3!MCx{oZo&Bq^d%`i#Rkz|WevDQ>4cbAT?t^QCwtL5P(vo~ zoej7Xqde!4g{N=zrV#*EKB;-Y`1KNLEC|Pg{z8mbZI}7g`)}lj|MI&l5FH+WSkcuY;j4M~2%-QBaI1xFWCpy;=OwPlvGvb+U z!aZVHL07lP*@Y5i4e~afYu0#X2Vyi=K3jK;_IlK|g>Xjo9`L2sw~XdY6F*%yi{rEd!0E% zX{bWd{pAU>OBK?b{?#p2u(>7mx1T`v?5m^ZZp_y1QvAiKz04Tvh(ec}j&|}0y=DY8 zHWYPaNtl2visElrMuUle2VnEjL-?9_sHhA^=lS!wzAs~-o;6H#dheTAr@+1a$(`c5 zzDuKC{h|W))Y?PSzlKgpc8Vs@i6uxsOQJ4j(w!mr3G(g%%cb9b--!oU z(*q^y9xZxo$pMpn2-E`tf2c_z>TF*Ug)x|Px`2>&pMShfQV=iHfR}859Y&Z$G(V$I zNGVF9G?rK_M?_u<{qM5qv9dZ-2t9fz8gfxZ7G=ThA+nTlmuH4Y?&1ipOQ^>`W)U)7 zvfTo`!c;$6HD&vTT;>t-NtCX0oCS%EL+wkN!)19fsx^^72)AL;uem~VXJnIso93M7 z!TZIhnGH>(d^|7Yuy_aF8@NVA^a%w4cygFy0qUE>Fk`e>;f4$-Tnw^!nXZg9IM|TK zyAv@-mK#BK0tLwM3)s!6j+*eCX9CJoEpJ?#Qs3r0>l-l*A;|PZQmcZbjRg5S$K*v6 zIujVYVqq?=d$zu+b`)~5+-A^U=ACo6DBbxr8P5AL^A}p&Cz@v`E@F)_C0MeqkKJ?)hg)4A~WuBMMtcQJEMh6nKdRId3sU+_JU!+ z)v%@?;m^6Wmwc~cS-$#Ib6c+C_-h$u`j%@7=@W46HRJM@bY!pbdEWXv169-9N=tp& z=GK!hHwVQ#_(%xob4pD8T#Q5dk2T33Cz3xUMeKv>b|FX079>75Vw{rtTTm3+P%|#m zVw^W(5lfD^2i2&+pwBU(0ef9Z?lghtAu;LJ&C($csBrJ?GUi4kfq+h&fHjUR}tIRCXjL9mr1T z>rj8~LU_n*Lh{;?A*d}$bgsm(3PIQh2{{HG(3_FNI+4XF=fv6vVjPl$a1X>d2I-<~ zLb7(C+$l{-w*RZ59BLENU>A~CMT%Rn9>XXYY8&!k7a~c$AqEJ=Jy54MB9U<;*{&*N zALO+Q`BaUHQ8*Oy5OA$WE^SA~SHrUnf!u+LtCV9OB(V!oTG@g(WnVLuYj5&go>M=<8aofl``6Fhfk zh&RL)>PoBs%X`HM_l@{}`pdunfgG8D1Ol2v0RrOtfAyDFHnlahbFnm*Ff}wWb^71j zkUw1M~ts`uovtOIbzWF~epVM-4%fGib3=Q2qTmPhYIN!Z}_kJDSZruTK zMsHD9D%9kQ3DaGI;ZJccJ;k7LkvOHAVZKfV{K3qrN;yjq^3}HLM9dYuD$*w&! z#|r4CS!U_A3Ak3fs)$0pz^K3*T7bk=i_FayM*|${O1f(MPc4VIQ<{=;Ga4X0`fE-# ztbf736Y=LfOVRH*bn956Ay}!`s|%oKK;TQpK!}!D{_J2{T0^_(*UQ7utrr?pg?&B% zd8!0KQL4D~w_m?s%1A`p;?<*`k|GhhEL<=vrD++@8@60A?6?CH#Dm?^f4 znKp%wSxR2@ELoi;-sy&PO!~0#a`-U zyS~VEd~{kImwHFzA#(8Vuo-hM6~(Re__IR9L*+2u%y5yK5mgG>Q%FTZXPh}D!rDbB zw_J2?r!tAe%h15h=5m0ADloU0NEH~XQ9?v*i+X>TOu9=V*{!}mps_a@K=cPQLRL(c z!^*t7{6R;qtx(S_(xkOng-{cUB3&_Q!DUoaC8DbgY`FN?Q%gAS*<^eQ4N%S*!qN!z z3~Qq)2hI-!!O9aO80fIbRGfe(UMg)w^`#I?#MNc-i*iDPpbBTGF%S`E&PYAq`V&uE zgGKU@JW>M+H^e59Y!LtZH<~7W-T|Kg$zl(R8J>--XDQE6f+0Jb=@blKcQjxMuzL0* z2tRiy%zA-h3kH!EQy3%B)+IuVQAQOQPUBQEmN7en)2jX~0Tq2fQbjJzmDHrXph9oW zl*U^`+P?^*n)Io9&Pk$cZPgy|50*+dzLh_zBYmoIruvX(Dm*f+5b$E67EA}~j!T7r ztvB01QyCRSo2$HU;V_`J(?CRCjXvQNM1tLcvfdpL!Wqj)(UlYRccWF`fIRsh`k;6P076RZxx2c7VC^2-Cviko zzGQMzVE#ypq8L?g`QCDaM|J6@Npf(NuH#!1J}+sDXeotlU1k#S6BNd9C>?FTs1t4P zj;*Yw(b)YiFY~2OMd*E-#?~EOr{L6Q4N){Uu0_X!WG?)jm{XZHZj<_5Y72L4R>sMW zJ}K73uvUJX{>Z&v{a_|DnW&L4rBx(AaYQ(_VRL~rlG?Cq8ITk*{HjWqf2*0Mdamhi zscN?ukujc}kl5{^HV0u={&v(DZ!e3EEa-t<05w3$zcX?nu-Ql)3)bv1;n3JaT!aaU zh*QHe&8cQ4JsMYn+cT|zfxaNFxLC4d@H?u72%o06$%@lx8cNwa@HoDBLaUFAeHWS( zq>C$MBk|Ij3PCClnxZwyJHN6j%hxp}v#NRl2Hon8UQgU&*ILTYC_Nbq7F**VUqh>0 z^rO;PTdjxD;kTaM>Y5oXHoqn@VM;Heiz^9YN$tv+7>2{ZwDA+QW zkIm+T-!Hcd^tw-Il&wnE#D04h5@dgXPEg2ZK%-uFxYbKp13>7N7VH~y#ZtW!Qk{P! zV5cFn*8|_EbXN$UL3597yusctVpAM4_by~tVu@?-M|IV{ByRb6gyihFBS(nY8$LSX za43Gaz>fr?j=Md@f7jIhx;`<7{LPeqOc+zI1c2qgtK4jgn5FeF!$C#ng-sfa80G(kC#0N8VC1mz~3SBm^PB+O5s2yrlp zkJCWQ!U)%qvWv2V`(dZi>@lgM!ka7-X0VFkOlA0pB-&}jNQ(Spojo#E#p-gqdx1hn z6lb0Mq5AP9kDH5=)L#SiwtCJ%#4@VHF6zraX6E&?@5a3zV)c1B;84ULdO>`L@cyIe zj_-~T{0hR1*fUHPBN(EKK4M7k?~5PpN=M!@jN17ltZf+SoZVW=NCBsVoaVR^SEN2a z{$%HqH)~7!tPCL1JK`J;(1@U{HA#l{SB)swIyevysS?vz_rT*Xg^)FZ;6JFu*cxsE zP}MTk+aQn5-8+w7pl_FC*AaJJbs+X5_`;XRjlqp@dJ>W_ql}VZ#R{ipCzl}A6RmUoH@(7+YJxzXUL+AH$|`<9OoywI|bW_7dG z;$Px+yMxII@=xS|zE1lJY;O%KkAqqHWPGdM)wW(+_C}GEU0a0mF|Ng77>-7z(Dbw* z!Hv34glCl1-+LqZKgvM-592*yU;g(dH2>4E3k6_MB^3KZl(K|{Fq!=sA8EmrgX0>a$J%ELqI6| z#c9i7D!F0n*2d=->OXMDCkYl!{sZKn{4m=bG;b8OVRAY<-Eo%Pu(U6a^*@wZepky2H(OhnUX z4Qfd&Eefg=d-F_Y>iHD=10|zRDQ$Awb!kEei}zf0o?`wRF2Q&>ta)f(n$#Hv0vkdO zmO*XCX~AI-OTZq93FSMQ0Qp^d|To;MdXaXlm?h|W&Pa`R^+)_5x|Qfsw(AltTD8dykpj9j zgb9MpSTl?S*18=gmpQuSJYciqR&anN%w)=Rgtdo=_j&pq37r8}RPkYQ;xFkMu+TiJ z@DXdxF4CfSQ!O5GrSDk(ldw)S0-gbX>T=1UEt{0GnwZwGNSe_SFLPGqpIE<`4RRYVGUw zPfer$Xk-Zn1jO-wq0xUTn^e%r$2`7Si#>h=R$~JrK+})0B0^KMy$rzHnLZ&btPcaXp8c5tdUTX_)-G<&LF{}>IRJ+ zGpri6tC}N(Ch)`93#>7);LAwfN$rZ9!?FWr;lE5{2}fGLmn3X0F=joo3$Q;JWV9`R zP z{A>TV%XnK95wE&h~U)&u zMD)tB$AUxv;dsn_Oh*Na*)ld*nrBBe{QSlI_soH^8?(iNf$qLpw+lhpSJ zHFPW^F7G%;Nab@7*rmGSAk&;;2AaD`bLaV7-^WU}guDWQHBSvAd<{2cbg*>XlUch? z_nwXbBCc#}l|>$#3d*j?@gZ+S)eMDZ%dMO%UaN$dg|z34i-tAtX|TK=OUs~yh7CDWPk;gwAMnlT!u^zfGH{n zAoR)ngdCfP)xr0%OrDm6s9jy#JXu@`xL-zevm_B@7#_)E7#`e7)ah@J%y-&5&Ld5M zEGUqB+x2+UHL3mWGShQgxk~7b*2m&h6pR-X8n&+f_ebrivO1;Y;U*G2RP4>7-J@}< zXNskb_)pOT}L;$mfgu`kKoor75y@P54Qa&tlMahsR(+zYTb^m`R|d z8NZuQ>ns<3*!oBFYXyedHmHJ~q?m7XcBj_(RYkF+z@j0D6cS%QUxY>0n}8N~0+t}o zJ7LgMD^MIg$_|KWO@@*|qA5#~w@@#cik2@IZFJXuB+0cz;m{GY5EmH%oTA63>d~dN z6Rq5eEt+`chgotMm}ogb0HQECJlrJ7u#!;zWl>G_ISJAd6c$aboH1_sdaWwr^rEbs z)s)rt1lpwsc-0t=HFY*tTB?|9Wc5U{WS;d9z~DCO)q~za3(}gXNxUTYj?u)9I@)DQ z>qQFv&LbX5!xFYh3JQaqMkNPnfyLE)V-IXqYccd7Z#pbSV#Q)vpblt~xSRd9c$piU zYPRC6k+tO4OxmgBP)!T22YTo=T=7haOEnk|NeT=3Y{ctN5^*;w&6R6m*GHrqWGzIH z>E>bdE*(O{p;nZ5S}Hf-=BQB=;4ea=;lT+}OjcgwWb5ieEr%mJ3?V8U(?HnC0Rq*m6eliJF znuqZ#TV`$KL5RY_(oMsqRywk%lVuvsLj_Nx`S2)~sa+k6i}1!LVj@PisoyTi^AnJm^}Y%sv_>kRe`kI(DBs(2Y+W_)9~k#oHW`mJTbb3>?y&@U z4T@bPNpG8}n}I*?sdR@&pP;jFx6*jl1qM`=mxapmwWHjgei99G3 zQYs9RR?XDV*biQ(Y+i$(idWwtXXWUmk#ojUJrb%qQR31HD*a-pujHRoTHTsT^4A-t z)Kc+SHG((U+^#tw*ph9%lmS>|thb&?S1F`-OKQhnpmMTx;HA z@03!g;Ll5q<=yWAd86}H91yK8sFsPv$ytRL#8_Y=z

ods~(JdK42%qQO>&4oTo`X0tZAjPyNMQr@{<3kJ^q zdj;kP1ZWr7!$etVUeeO^ zPFNo|>C_-E$^ewX4a-A#!Zdzj$e%O#COeaZ|A5`pO1rJ6Q-ocTD<&(`z8!o~^Cc7{ zT6LEwQoEG^t52iP3Etdw9jfjJukMdm`;);=1>?4;Y6O4K>(jh?tGkL)X=b$!_1)CB{`G8#1QVCRGX*RUsD#$(rZHKqgTgS+naPaG z{U|tTiMTF1>%i@~aQQMmG1XhSLHfBrN2O5;`J1SySaj_%ZDHTmye~jL%LDj0d`^KS z(O2`%GN=mhCf;udPE&q}3(knTtNeuw;y{j92DQ+J#THF5o%92?ZJu8cZTb18RFC(S z+u}*`=abAtwCVZsJNLr`>+9X?Yc-!}Bxxno9JPg~x@;TZs#RD`FxA0nicGh*_y_}6 zzzUqLY9eO){(-lK-(@RugN|q14QV#$McnCs%`X z9I8?qGaffB?aNn90K=b{6od6(dAd?;fDEmcb1F`Xt9mpO8=ojyB1aHKEOxs$WCZwp zUW8but-h=*jKm&C^5LOCNI`L$DU@8!c-fW8?A-cUUf;#gd6&Ri2t_ zR~zry($a_5`mFo9l%zt1{|h^cio{>Rf%_FaHmxw{-Qa75DxY0T3& z&hZ*Y zC0r&5vgtLS8HR2n1z)~|C{)ejW6zWLIm)v=X*_e_PWuTO|Ielc z6&}hC%RnZ7UJ$;Tyjh{$5d)_t|eR_@iVuLjlNaQ$M zxYk*vSE~u;FdC=rJh43Xl>w~3YR-^^Zjx)!8%L4G5_>cO>82t0UPmp>Hjn$Y^ln|b z3a7r52I;|2-Nu`#2sqrnb>+zJCk$DpIn$Q9YzIEs$8gq7vz70jy0GKu5q(uiO2gfG0%lrSDu5V9ptw1xT`V#Ii|HNd5FEmVqQ zMPgE`DVZ>c43AW#j^`$p7eKk2dqv{7tyZN;;=L6%c+(FoTi{lni3MAeizX$FcFXQ@ z<~^meG^cQ%QypbFVfG8t^3kjlWzixAnis2xOJ-fxi9XDm;UISysB7V)^q7BSE}oDu zyW~vY>`>c1M`@q0Igs3A=nuNTolm~rv{gLnc5%@AE{NxqSL)bsh5#HA91r+~u=o0Q2&i_H~Q0XxU zly^3&8t&*G`eL)=FgK>`Q_`>?{jt)_O=ESX%lB@FLyod*S4{@=IoDN2IEz$(uz%sB z{P#UMrjPLkNNEyGIvJ|jGSPz6Dx^CW>1nTf^r#z6Wf}h`6+X}CEc$U=!Xw_58Hfx82pkPsLPUaregl%o-le$$-|+vb zH^}RHNaOy>WCw5{Am)G28>#?1M_Y4ID|1sD7hyAF$G>9ve+06cog%6*l7A~bzqcB- zcw*E8iJA^f4pncbQIyxQbB&}s(jbl4tv|`MZiNe1x@6?oF*stXrx7u#&kUk zOUsi?OAGga&#y1kK8h#|61YkKImbV7cPV>8o31>1-t>hjk30#5P!ql7A~&HlQOJE| zhFig@LsTem+=sfnNcOJ=OL(;#3Y;nQdO{=3^)BG(YS| z+G$eZJ`0{G4CRgBq+!#O+GCu;1_07G3;a68Z%L)Xkx`Qav}AOP58X}x~+u=S>C z>=AFB4RqthI@adM5#U>=^5LGaCQOJ0o0KofM|Y)E!X#_LQ-@lNTIOo~0w`Hr3>G+4 zh{>lTZp~@^od)Ju?CoA$xOu%^Yg3peZ^6+%msD7xl(66TyUnzecVPMpcKUtvu+h?V z#FGhvHvaF0aoN_t#R^Jpnd|YxhPS+~r!}!n%zrUpjnFQjGIm~PK^ffZM6;bo-9Bif zLaN-xr!>@4qo=-!h!>2X)XJ70-QKrG^3NIZ{DeJbAxp9Kx4Z`%Zxg_C~?h%5v^9Nq<&lX@#l0LZe99Q9)730s?AL1!56;+qHkDO5pMd< zT!0Q-Jl`SzZN6bgEP+GRkQ9ee6dvKnLBV|%PHM7~qr4@VS5nzatf9mdgz}!JidDvV zsN5c3KStoF(c71&tv@-$Lb{tkGZJ8AolnFYWaweD8y4V>o zIZnUmkB^8uHhl@S>37iov{K1mh?FFOfq<5P|5;2c{m+f6sWZUQm5lknBRf_}UUpCj z$#1hv9!E91Rp%!qqWF^u5?>URx^O*>8vj$Pe3-k<}e;UVE2T`(l#e?@gWH2VF? zfbTWjZBOQG-3ubn$6CFz$(qa(nUqRWRy$0&@I+{g^yQv+BAh7s!c=HzRBRO_E`q4( z2%#5}3peyT^J-QhY9dBdk!g*zIZ5JRjHSyWmBVd}wPSTA?)n|}A47D_L+nuOA1+8EB2icMM7^f<@6uutccBTc+yPk0o#Vn-7ZOIE zf_vvh?&0Hlp{^7UMW34E_6lY~3g%dpI%DI3jR2Dn&r_8o8U>+^T{atzDZ9+wY0PO> zE~_GQldg7d(XTA}=dcG<+e>VXH}n?FN9}>A`(=|OjGrT#N)K^U1ZQa1?5^_7Q9*8| zwc}fOGakLb|A{vT;=jYyp@4u;F#g%y{{OM%{-^St;iZQ=iar`Z3rsu26KIPHxf)c) zh5U;28j&0f2ni&iizAXF*-B$D2^#*X@7CfhvdJc&)hxfbL!#6V61apAE=;?{wFahSqJeQ>&=6E`S(MgbCNk1KiszX98UIoy(2us8PEgnD4@y$S0d zxFy@*PHd}58lhG1`CvP^=3LR-Zy_@UJ~9Y2eu`59@a;vTQ5XpQFkHDV)5cc+S( z$`CZ>)PufB>g4JEYX^c;C-!h!XBlJhL<=$UirEs>BPiB=LE|3g{1hD2 z^OSimYW67W;xQqEtC7h})$5Y|*+**Yvqm0K0d*5b9dD@gz|GSW{&NR83x9v@xTb#&eM(FQoK~)5M-tr*%zqG)Z!+OOk_q_tI3z5 z`k(xT^O169pniBbB;>K1JrP7SMra?ERrRSvr~ysM^HkIc4UQXkcUYQnD#4Ar031|7T!em?iqQMQ1T#S<7~7AWDKg8O3JZ|vt1kU zNR!U)6CsKVfzX6cGfqUQ$TJgG^S8OL^O#f2gQ`4^X_P${u)Q2VxB(#R(b>RCoFaa3 zCbsDa2LL+_U$TW~gHIj)4#`VfuB;%&anAXng<^Mo@omHH)t&gPu3#yarD27BEV`hS zG%OEJsq5w09XS}5W6zYD)a$=60xgX;Pm~b-yVKX`p3K^0#_!r{s={+zo;ZS&5)Q?Ke zT#ZB9FwmYAP}wDF9t1N}fx0uaRlAq@+;b;jLgP7S=1jG-kXI~$nL|_%fk;Jq-0=sh z5^CE|_S}9{n7 z(P=BYHU$xTqtzKH&^x+7~a5EZr9@<169P+Qtv2L zyWFis`fn8x65z`RA$VNC;kD*(OtO|9TVveImwd|NT@S`RQ4|)GRhz)g%Z0RrOEsmG zHDtEETA>r7w|)(s_@UOe)immz0O`=dxB=E{lW;UUQPySFQ#-pLe*OORTf~K?=dn(c z?39~g*MSkl1&Y=scTBF~Cx#o%OlDeD03f>hjzqp}IVvL~2PIqOM0B8XJ-;giY@ z1GR4vS+kl`+d)nC(y3C@qQTQ@>N-ZGjdkN0B2|ew^PX2-KWNwUUiVMj=q>mk-~hen z3FBcbW_(guCml2PbQw8ylJ8B{I8CWdr|yBti7K9&7Oiu7Dl!_GlpEh`_BF|-2TaCo zt#6vW@^+;kuy`T$RWhBK&?-@*f@jrq4MVP$RPU{7iJ6ez$Xl|j7Jaea?v!Qa-p?Qq z2$pB-p-oS@JK|?v@Mj!RE=$ka@2$ynTuG^?eZx8mgMI^Q0Z{_9TH%+<_21D@*yK5* z`edwQv)KiVXu+NkRn0&0>Vs}(f_trkd)1&&u+C6;&${7V6)~R7QJ+#pl;}ZgT7KlU z1R+=o8?p-<(t<5B&q45>dEuT(VLZ8^KE;a6NkD9vLG~DdZl{Cu+XeS(Ls^>@&|VdM zZwS3u3eRQ~IG_h}u+FJ-By?>+@T7tJZG!RZLRlLZ)VM4CfPYfLc)|zq!^EVUb4tnN z$)$&d;)`KFt>-=2birB86O4>YV?A?DtYm!L8ckw);9AKYBKl-78j{yAh4n zF>;K13uV(8r!9{Y__96^ZmXkRPf}^W6EnR%t|%>UwTnO~_cDXq4Bzy7byjJL~{yf&L zj|bby{Wfmwvf69h0_Ckwpa6d_WO1~oCvtr5aCnfo?-N{}(Iu+RP^Icw0s13W<+b7~>sQz>)k&_Zio2e` z0F+SClkm0@qj4SC6>aBYa$Ne3^(6YVoZ|Gcfc?hr|HD)MQWGE$p#Q#b|AD6foEbrZ z{&Q)M*~Y~HC=gHx^gm0gze^=-0giu}jI?{nj8GOA?Plo(%7Y=Vs;z=Lnb2cgEKgY&#

@wSyL^}zDI6Pn8w6o{Y_clD!Ew<)#zLh+L zX%1c;)W$iOIaBOY37>4B7^C{bSJ;}*_P*35$5~Nc-`*}?{BE{#l)euE=o0+D* zHypGC|F9TtkuTM#N|N9(aar#}r&Cf_0e`YD_uWHoYEpyfk)vhN&J1b}5~pLfGD8Je zEPzJk!ldVqUBLXg$tk|I(3!ScTu}lMh=QBAE~# zI#%XEcvIgJrdxH|vdD1O`vq7(K&<7F7`%wcF7MrN+-8FEn_P5EH$JhdnSiUs9y7^p zhqavbn;fTdSmziV{NhU!G35Gfi`RCoIk*u>oT-`0uaAbX3t*?+V$?;?cDFDm-t0%I!XeWbt^?=5vaTO*FGQY;Uzp3e(+=x!OK6!iUFVVqI@oRvSa z*kAz$|8P<`-sIR_*#Nt2WH4^`rrt|_B%jBtqq=&8KWBtj z&deYv&Gg1ohs>0&qy=E3ZCUSKMyO2RLLOslJ9j6S(!`;ye0P!qYcxjhq*sd5nG86L#*1uXdfi z1O3+sId_gPp%|H0j9;0nGvV8}{vuNx?i>FVbcE)7#JVi5P#7_KL`uEOfp17C~ZkuV6c@*S`*n#n9P6$p6N=KB2f)zW1d76j%;NGN2^`VV0aPzGVeUd6OP}rX0 zZSs5a=4_?wB?kZNnZ+Tn{b7`Kg0Wh3wxZLY&94VP!lW=5YInv%fB%CuMo8`HZ@lLHa-+ju}(AUIO zN83*Kg@azipjIl3vMXiZFD6ZCO;c(PZ5@J$2vM>7%#yQDDjb)~$rO5{{sJ*{qmP9p z)D!qPNDde|R+-(B4hu1}HEzJ*cDd>KxN4g1usnGQ@P9+^12Gtx3r`5CkAW(JgDq26 znZn_;t<}-duB_}--fyirOKY5n1`O_NC@;TaFgi!$;_NcP<`}p4MfY9?6=s5OQ!&Gt zyD#&rS5`cAcP)GjAO4jzh~ zupOz|;c-58>A9$k=W@*)cN*KGqW$x%Zq|t4A#dW z$FRVqv@hzIWXO>j!Xu4lVHg_xcjUhw#OopZ6!8Dpoy|b$$b=LaneR9v6jyxFy7)XFD(0p=B;?2qiIbs zw2xz;Fn30qLlaKCLcL&}C?fPDQCh!TT|5U(x^yqAksSKvX{vQqwcAib)54i0u#u1l9|E-XcQdK!mvUg;}X^)U0JSA?zAKzF4Zt~rqt!?7yxa99Oh>C&_C$x zIT{pjIoJcp5)~*tuo%|1dT#h7WIe@0xYFn0dzI@>I$Exed5;X@WN5$3Sg%sttjBcA zvyq01(T0kflCy@(z{CXLUVfAd#Y3;UU@+&**TgY%`H;wwnEL4REWs!PZ!E!a_n1}Ouo-!_l66OK3v*QNZH3Azw9EjgH- zr>)B?DoMzQfP?H;)1axc^Fd`pZ?E*5fw;ZstR)Hm;{Tpu$g$8$xXa@71`q%wZO_%8 zbfo9WtKGXPYI_W_*i4=`CH9}77s>{X@zi}9YFD!4F=sQ~WDgPDUW`fZWDGpABcRIR zf9R4xzxXZgE`X$;=eni|IAVP+c))8bH@1zxCBX$p^$??_Nm8JGf?k=<6Y0>70(cA536@bv-!g&zoDz_-Nx$y5UdsCyM_3r zHxx`MR1y38vo&xEaqLfEH7(I2Xk%^Q6ZA6u$1InN-m_%MoKE1CxJuLr*Th`5Rsafb zq<<5uDtwN^-T>SOyaPLvaJ(jhQ>-yFABMmX++bUVTa6dQMgvwRel(k-4KoibhMf_> zu_Q=+^}RhP_V(-ZP%0WaD8CTIj9F@$RnqBpP9}`Y9;JK>|81JuDT&;sgIqbR zHS`X$0U@2HU9J?=m3vV9@oN7**~pha46ufa#qiwfH78@uhZCo*S>;K&GpwX&fu0U^ zz^9Q428JveAN*6zb`ObvI;q3QE-oJ;Y^5g$tASm2_N%C35@SNGlNpKnBtgxxMHL@Y!!DY-Jxsy>nE8OGPQ3w?7aJp4 z50IMDOMXRe)3zF1?k?!g`Wl_8+3)sfb|T?dG~0O9ox9k`mtCjn1j;sGnZde}qA>ZX z!!tdHTRTI<_<6s5gH9G!66KW8606eb8mQ)2=@Hd#2&QKOlMDlg?!zRGGQ*#b9gG&f z<0}EN-+jl2Jk1nRg&E!?UB&A~pd%}eL1+e*AjK`uX~VTcaLV;aV=Fi5izBV8mSQvD zOCHukV?QL|Z~pioJ9c1^ccG}yMR0olK^%6xYz^7M%#T0hzcuyqkX|I~?k-&27pyJm zASR78Kj$`0!-VyXF>v!<8Jtkn3G&8F>+o36>*fq)ax)8XO3i%`8FbwfxqrP%>28l} zq_uy{Ub}CP{<3$7_P9_s-9xGi=}v~F(QzfKlL-fKj_aBx zse9mKw@jz|{GAY`xM$qS8561hiERzS05v*$&?u?yn}o}D32qOM>8>7*_Sn0wJG!q;1*F z;uwx?={!q$b65n@|F3TZo=x&dN(11YG&AT)T8=LM^j&qK$vf=|Hs#jOc5d{c9|HCg z9!5@Fpr$oWbdiinsa#*=Zv55KNcQ4t8=+h@=f}fSu50f`y;DrqXqdAQCfqsNs$egI*cW)CL0noz}7915dWici+*-MWGJ zKwQ5>Z%m^mJ+s0sg0E3!NTx!;QD>GEHM2<5aoi4u!RW&x1%%FO&WEDH9iIbz6x~)q z*Cp<~J%G%;9JuS*!R-KkyPBXoD+kWY*wt`yF&V!f<$}DDoUjdxSa>i8*X0eitWWPh z$ldbNw17HH5gCJ-uJ~P&NHBZu`tiD0D&e`jt|*YZDW}b-nxAuhUV&_=bSe2_XrYkY z?QJ|wRJo(O0^^Y<>LR5vlR|U;-loD%eP-%CK}5NHMp3$La&exlDyp->z0r?s!;`+m ziS1jz(8E@Jyiay{SSIa3_Y$GNK4Q2e96u)hgaG3ER8zbQCkbeV8a)%_{|zo5?|E-m zrRCtZay~4-E1~0qJPPm)me@S&ikQ4U;{zUfSt=0v^~1!S^c<&8x+qPWA!k4~uz)OT zQS9eaD(k3>w;{y^E!Uw{s>eT3}X zf?xDk_t-T-vRr?!Uy|*Bk1fa$0XCBFfd=9cHUIGY?*3~=Ta(BJ$$cxU$Y^s=7QaY)&)k#q$Ce#IjB3v%8tc)>lxx;y&J35p=LyCsn3M(wB*fU$`mq5- zcY^;(v7C&DcIH5Vfc`-LbH(#7#Zq;3a5fina8frjoVnuFJUMQBz@5|5eNji?uoM5N&lT8uegOzrGp2>)P1Bx{9Dd?yc%ac~XTG=8qK}+6!tKk&K zV#cTwJf&;S5XfNZAd@JyOq%JNdD!GVowfbu6aA{PaM@jNoT`@D9Pd`0E4>e^kj{h| zBG2gdu6M9purj68p=i#Eqr|A>YX-5;Q7+sf@4y3KE7nr4L4MYUWLu1Rg?jAV*kI>I zW3mPQSi>P%vSL6RKmP!45U=u1gGl8Q2$=+Xu{|&vhvB^4*QhKEnq}&zY2CfXFI6{x zH}Ww{9+okdocbKQ7jDEqv}Jaz)kD#&U)Gk{)JMv5^(BWia1*CuAJ_}QAAb96?mj8m z$)cQU; z_E`f-kB-de9`y0XUI!bc1-U&1laP8D_mM>)O0glv>;j{vICE=d{f<*8ADv4JVi{s1 z$>PM~I6>uj*P3w)s8t@ONmQj0N}lE-OCs57)w=kjCA5-`g6T^aHOdIbk!%~$OP;!y zB?(EoZan>!BbL_{Ta0)HS4+ejKHCxRSBY5|ZmTtZ%K^t$b}%UtUE4J4FUt?xU7=Rw1f!gf@*C>c9Cet>-j zstIU%g3<$zlQR#Mirv21fjcue#a4{c_XsDT%O~+jvpF zE2qH#0Y(w6)(i(}q#}w2X-o{HgBg@xIH3g1VH}f#a}j2`6G7th4tY|`?8$unf2_(EnJr@nFVaosdKPoKx+bCq}FGO7Yg$Ty~mqBQ3_7@?< z%q{-CUU03lse#15N(hAu4n@XxPDxD0&Jk%Pnv9{!*{^`ZiNMxyDL2Wu-n%AaHcmRO zWKcA_Fw@L-E%y%ORiW@=P}vxZ9CRPk-05+f*Cg3R`|I=NHUOylP#lJN2%SugjJdzw zgx7BR#%c1nbMeZGf4g4prZa2b3SHMB)wXWvrS%%aQtIZ?BLBuEHT}J}HnVeir7F)< z>-rBGMWRZA1~YgnF(!J;>b!n!YW^zT%w?F3XX_wS6Js?n&furq@l^MNsI5B=qu1hO z-O^&lNKCLcIwrG+cN${A4YwIi>#-$w13)WG)2JO;zQ}~rEnY#Y1dE}vJqA3ilH=uT z*F%BN7GTg|AR&VNi4k>f$!L0)V{|NE$7eDJ}ElkA!=Ck zka2>_b-g6@G-`5?l(?#gKD4j=FiI^emFvjMl=%qW53-U*$nL^(drj0CFK|vQ422!F zJL*9dHI_2FUl%XU7`w8j*t9f_W6XL)82(uT>6DpNN81&#s_i>G;(Apj^qbFmv-3VM zY+>t%J+XqI~c+yZ+)Ax9Dz*W}XSyx8YO-$J@zOsm2S(2HUr_lawK-43R~2 zvzbxG_t*Yy{kd5?Oy;~Q%$aW6%tJ2q8FxHkvbT?TeqwYR#L=>(>Ft} zzwu#S7R&Uuu}+n^(j`Dk`VwHIJEHLiZMkzKCQ7gs(n~#NF;t{xpu+~OXd1^qJCfpE zaq_)(D|T6kIR?DJxAMBjIweg0I|sg;%?t)CzuQXicAd0e!?)W8KWN z(`(+&WxDxFuLR$Q*3|<`(NdE#A2nbPEgZ2XOno6d+hAFXFzh57tL+j)4c@ho5h1i8 zmlGyTm}`CqqlE9F5FV%$+`e@XCDW2w`hmI zBI84*nBmJ|*qM!|jDegE?-6G53M(I_9|m(YjWt^2J|uR>7vX;bLtq{1NBCc0$oUHl zT>mfbVC-scXKwE*?&;|KcMjlS|8EAhx{u3(5YjjPTv^GzdQ0zaMfb*Qg<`$NPjxh1 z^lH{G%rj`GKqhJHntT~cG{S&a;6F-nKIvF=fO@wKXdj=hU!T`E*W0s(y8%EM{n2D5 z5wucPSp8Ik$A1U%t9(<#`q_=yYgvj426~ zjwVEnIJ_}On8Qjy)CalPoMRuZIN-xCOHb9aAI%q;^(QU3UNxk$kxZ#d!xJOMN|pRg zyV;-8<^gB2Eu7sq7%Y=C?k4HM+c?ig!Ng6+TUZigf?-|rW*pwrtUj>YRE-^q|9zT7$@V=2#AnFS8h;G`Vk_fl-HCg?lAos=<~t z6Z_B}*((#Wo37mK&bwJiAP2H_hP+$op@PN4p-pPifWCTy+(F+-M?A!OB*2r$niZ{O zkA9O*C)>}Z@c}`IQ}&WX<26yY^Ow%f9dZkwN}kVNX_iJTZg?*_W~!q@?UMy!(O>nA z>o(nGxC*+MBOwKc!NGqi$3Dy8(TN3t_pq;`3STaDWV23A~^^tdi%aj#y6+X^Wr%e*#CAr-GIqKzgT zJYikds9QgamUiG=GPYOX7vt4FUAikp9vZV8vTD$?7u(}og2x3vZ#&8-i*^WwI&E9o z%Hec;2E`riZIUMoYQYD&ulk`CSvoYT)w^ae<>m{Ekw@)yQZjDpv9`N_iUYifq;bgK zcTA- z@!#a$f&v1f{paLXF?V%yw*PlIpsR?gjP#|TOGOV`8lm2;CKh54_}ky61`~`On7O$! z(m|HH^`~sxy1p6;>UR=F)I21i-<>4(X^ws?y&G+bw1@l56zA^N%-7fBA!VSACyFT4 zJ{1WJ1k;$~eDg)s`hk45jYdsjD_al1aZ|%hc6)CoZrRtu-YHD^tJgfyfRkU`za{?E zIo-(^OEf|pS(6DaRPP6$F8uV##bRvKu$Ow9+h{XZ*e=ZY@qp6QQ}m{Vo|a!t&$61R z&C2rc>DBHNw8mvFiNpvKIboDTxOGv>J|-eZ%6 z%V4~f_C=;P1Fg=%N`2JZkUdnxfZr1*aa4c6!CBj-Qd1z{y85~4u(RyC0z-w0XX^Vk zo^PJZ02K?>dZe)0>1-!xW(lR8TFz!wSq-8&1DUp|#L_W{&jAg$Tlg6sz$n^D{*mfE z~Ko3F~K>ivPz>p6tSIfI;WXD#`Y=VejCKW`gLx|CK-#hb#mD4OBKuK zHj4|3OYp1CBwQ%Cw6)}4#1@NDYiyOJLNQ;A!!Vp#n33%OzEJkmU;67Y1TRFU4eS~vp|uiCoix@k`<+on z`_{M&*D7)uA=2dO$uClzusxKUJ(U7RRpns0jxd3AtXdSl}@8Qu03z>{0X{ds&J)Vld2u>DXEJ+}-M3HP=lG>Ep4*zNrm1ci2us6jqo{8iD;-y|-K{q7t{B4VX;>B@g+IeW&*%)y# z<-)dWhgt-3rDEq&YLA;jc|=>ytUD+AXjE2JnqaJX!Bff{TKXJ04SG`6F)2>d3?%*= zzeXk^jw91#=HY6KiwRf7kAsx~gcJNMCVua9uH+Y$LQq!cCkx zLqs^jO7~$w%Y9ME#TM?|U2s=3wiXt`&huN0g8ha@>NJSbsrGD$m3*g`B{TuVh?Qz` z`AviMlPu9_>b#cTogcoRZuvKZgx{ave|-uE@oH=VNccr*C$Ob^V<2UP?|5|KuttkR29wT_lL5frAmxKX@58lx5?fr$hMZ z3C}N9LOslz?zcKpyhkNk4NZ)Bn{KOvlST~^vjX3 zjE=KoW#BUD%DbFpgUcp2h;PD$5@Rwoalrt-M}FHL)ix)2EHkeAH8dxBReebm?4P6N zC^NhXVyT$W&*foRlbPKrMMv(dOvi91$1P}mLKkE#G)y$ZWJ?K03s;^lgSHUY$rp>} zI+}=!_>(KzCB>?-(wd?OKFRz&Qz=4j_1i{k+2_sOWjd%7;+A~6St*Nqd_X$bBj3ykbL}G$~6~gr7Fcc zywReO#zN?A+dcz!g|80wLeA*8`j(eN9CC2qK6fy=Uwp~T3QDsOc^#oO4x2(TH5%K# z{I^^XS?<9dMp=Z%{SCC}O|@_j&(}k8VBd*m(6z$?b5=nqsGPCl*g8g(0_@#^m?n5O ze3!lVZ~Kat1%T^NA(Aqut9DhBHw~2{0sbfMx;c|Rzo#e(ii3P9C*Vkan||}i-djmV zk<$cgW3EDLow~<&C6buiO~N8}AZEJK;ivcXpVG87y3*fGF-i`3uD`GTv>QA`STVjn z?%lmF3T=66=-|c{q+bMrb!c%C#P#YannlpP+tq0o#PTP}QPwo(9;d^3kzL4qWQ?I8 z%qz((fIFgo&i5$Ro?%GLx-%Uml<#VL<0jz<>GIv9U0GII&N+z5e2hJVomjQ}tP1r) z1L{CAIOhl_k(*7Y*bQ2FMXk^G)wEObl<8Fd36qBl0b2?|^pPi`4R4lUI|BxUY#bRY zDE5LQltQQp%-uzS)CruJ#xzqL`W5I%lII&2hXF-L&#uLQ8md{6d{b2vPY#D0=ZY8h z|1tJX(YbA1*I>AJY}>YN+fH_D+s+f4JGO1xwr$(lN$R}sSFP1|POGZ(->lizyjmA) z_BnbRz4sBxWI_KLHq;{ez2TEUE2)nke`)0RfIn(P&1-ZA)yzLa4O6y#vO01L(P7~b zw&#Y!iMx(0MMCZVc^BCO)*1)4A}K!$bJNce{eY6sA_hLnG1ig0i|kFsYRI>l2G70> z#C|*w?O~5U+YXxXB`VVQ81e~ zlR}_%OnWGp*mLqG9Gl#e_9iu&*|YK%=H`nVTAgp2tSH^HBemQ4X9VN*-M}EzUqx5s zUqu(y|38$gm^<3J{qLH~N_jy5g?9s_h}53|q#GFcBQTeUFQQb%9tV1IRW55z-a>O- z-<(U*{gXON5$AIk{#KlGdn;)cNlBk8aeH&q>w2x><@)RMh$9dVJnPgj zjioBWa@k|*I9;{6+(otqbF*1>O9@(aV)GX}JEt%{f2sE2@>KeT#L+rQbt9OW=WMh& z7$Z2Z0tG8YgWi&Lb@zutEeHL1^K}}Uu(iBK0+cY>c6H?ld*a6OGt1JfC8|q7Avsnk0Iv38;A{M3zZ#!-8h_8-u5;(k1NLndj&l}++EjX;zSPEXhrVR1vkPl zlhARWz~Caq8!)&{XaJaQ*nZ0BD=<&&QheB#7-N;gOr%l@jdMhJL6I?vUY&$h>2gv$ zv{x8&WK%G~%wowS=h$ozm2fHhI~$wm`N`GXJ#A$nwLoc&5zOSSA{CTsw!7PLaA%g= zGRyp`gbz0yl6NSU#eU_S*BeFD*MQSo$1HFJ*O~_FV;#3PrJ41I+bm7Fof1-NhkZPU&uqf~5bntq`}+G3QwM|b8qITybHc}wSgl42 zL0JfBWg&%g!8gFtsCI*9jos+~;w_~+%S>g6z@GItNMaIKBx(y}`1CiY`u!Cmfe|R( z8ZHf}&Jl%VI{x9QlRtQu2NtuBhzhk=H=^OKecR1TxT7q2iTRf8HN-!mWe*O)Ugx`| z66XQ_TRrX#im8U+0ksf&g8oDPF+0U48O0NuzzZOf`yj0Y4qHk8xGeZhe0GRfjb|V- znjqXFp_~WT$$4|M`5v?j=%73^O1I zXpInrVKlZbsa~VCZPRfphpj5YP8rixXU4G^aMin55*usQ)^?3%j#9I!Fm(smyj{qY zg}!)}*cgBfNRZ{0Vg!uGMLSxvI;UFM@vGoWx4^F4T89!g(HFvT!B{zU8M-mjC%HcP z=rX%hRCn#H9jVsDZL8S>8$lMT=nQfUYIODP-wU+Ip*)kUv=xh!g$Jkin;|?4gQE4*3 zX#dHjJwBO_?-R-fkKdz2bz_v}&xvbh#&}CP}+wZ4Mo4 zDw5u^ZJgUfz(u@u+oO|VBJHkN-CC9B++^mXB6Tu0+Ks|ov@u_>026T+C92`3ydumi z#5aW5EF}OZOW%8+CrZ?brwLv>TTQ^&0ynQU?u~0%4#iN!uT)eanMM#J zW=Z}cT_|VeBdZ&EJQaBr1C_XYmq8k9sP^K4sh0{y@eCe0ApTskVICs8q6Bl%RQ!c$ zGP`C1Gbt#>Ve)zI=}@crC6j80K_rB%J>5pHFy)w8tOTL=RN4(hQkZ0Cz{hiD)-vRg zF>lJaC&-7C5}@fxt@IJ+jdtr_D7gdWBmt^~l_cRTPp~mOe~5)K=?(gUBmgUUlpn0< z6T}9CT*9mgdBXMM6gXi`=9(M{6hY?)S1ZbC0;N{ z!81=}#UN51(x_IBkPVF-3)B)jD!*VW?Jrk}PNzUm!FxaPQqm{`Wg;{093ajRGj}?V zl3JPHv65wB=!X+wHNBp1|G3lSVs41Oz<_|vVg9v!AZhDrXJsO6U~T<(ezKv4@CDZ# z&j>{#ASnStilVd=hLtdnrpz87AZm!0cmbrquG7w{XJu zOzvFlm%m`qej)*ZO$>2>>D9D8@p5&oIp+80<-!i=y8lc7TL`J2iOAecB27(_u|jKT zS*4=2NJX{F(EPvhRfRQHw?@q+hLlO!=)A5u}J`nDscl(vF->K)W_ok=f90%O z9Xee76ZZ6YVRa#z#lVw}2HMnIzXv^|7+ywCMezrQ5{@@pr2eXZ9a@8?W|8Gv^;pH( zlQM0&2XpaLNp<2G{VQN}r&=)c+Y2etkU=7UQX6?A z&?Cs%T1Ud7S{TL=sv%r<3@*j^kZiNJFWS1wMe}XdIQlS#5mMs?5(tSbCm4`*=8!#+ z6c$*9NjDSnc5wl(l<6AWlkL}qBJp%Cs<8F3Cj-7vYzJVIMA#LBqL zeuNny-AKCs8vcoMVjdU7y)93(tzr4fOvf_|)if`3RE8=1V*!aTDgJ2cl8@~cOr}Mh zrmK?sy?U5=JJA83dC>r*v-ub4a+&)CtKEe!+zu9ps4u5ro<~5zu z(U{s9*Cl(=AH7za8>4<5Xv=D|edk0Aj>VbbmW|$G!p|U+O}_e=&MktId^iS!y&#ls z37f;a^2HU=?aEOhqF33!M!U7z$f8A7zkaL0VMLLRPE%vNtr*E2X}NTvx_FzqIJ!n1 zu`GUfYCf;$U4<+YHzCV?aeR?w0a#`JUd zoY}n2wVAGw{{ff$wNFhiO)PT8F?b1yuto4IF(!LgFAhl%dd^Oq>NlT&8N|rkv|{{; zXsyA@toTNN-EEb`EHEl~{q`;AotNnMHrd(cA=X*;`&ds&7~;Du$wjNPKYQ58D{wU< z&j>1#EyJIfy0TjVj^yftEA*PL#Z~a}T>Mtm{S^4q2R2s{{PmInfDlPkt&agE%w_IS zQnaQs#ZRT3sqU=pCKD;>K}i$)W5AGz;YG4}t>5XYs2EGDqs&$<=OH~&R&5_+sEuNp z8e;$)aEP6&=69)Kd~Uk2^|2ecy_|(JFs1(kyrG!0QOFc(FCoy4VVYs{aC(j!FeY+; z1>_*JD}|zt+DRQI_k)(gvBYFKN27f}vSNl6h5;KTv!2D0)%Ko^#WqOHNc+ecPw$Z3W^0d zrVO*rm9AH#zkx)HG@OO9Q5ebnm`%L8sy%cI<}ie3$O_EV{Kk}ta0BEwQJHrXdDh?) zooT_VNFUBMW0W-S;oRzhs%nGV;=OOE6fP;xiWIr0K|=rfq<(5*lplUgZLurJB5B~h5X-FmKsgBE#h z@h)@#pUA1+6u^3?6W9CvWBbiG1geR~Zn`aS`xv`}Th%tXsa!A{dOs5zwLCS=YdUOU zzRbR)dk3av+dlC#c19Bi}m&K=DVMkA>XrT(Z%JN+%O=P(GH!h_(JWtkiB9>wn zPuB5tf=QkZ5@$W90|jpeoPwlXGwXkd%@z1r*3D5IccR$HZysSPk|RCaGqJ7x#SZ{X z5A3$P7cxnRv9Qr9d@<2Y*alETfZ&!56UX@WPFXsDY9Qu?L5F)uT}{U@m&lUo(muZ5vQ@0F$Bh(*IL@1a;lH^5@@?=S>b zahEQK7Jm{IHo63&TpCWCkm40>J|epy4T{Xl#jO_z6{)3HyT!uJrR zkT?KLc*@R0W;ENpcRN0^-F75(HutA zJx#uGiN8~QvQWleB$mx5=M?dTC-pIq<`SJ5P=#Y=>m zkqDuh;=DuukZOzi^kKaPWhv7N+P8FEfnB8Jlaa5xndR3=FAOVQEPwh(7nt$+gTnhS zPzrTiny7%axKPjUmLQaawZU5?tQj5-L7}Lx zT9q=EmT(mIh3fIIVORk4{`3Xv3BxqSxUM!HDkCPxoRM%_ahvJ#y51`I{OIBT2C>I? z#X%Gyg&D;dL>Qo5sHQG1YQm9is(>9*dP>RTjLF}Snq#%G*>+Z+N*`uuEwU1eHn8Nh z4mU{WwvpC+8HwEFXOF_zM?zu(5HcJ0qiMq~%_a4VjV(9UU!S66>}^Why4>rJk_!j* z#M{|=X#12=0oKc@t&(l3&X1Ljz{KqJ`n8$F90CmXNoed2gd04i-5y}r`wQqZRF{^f zQXr7}NHOh)T!qaU0A#lY;9a)9MUReLG*fT-acg1n4UHGF*P?eD0WyD$TpTf*t# zYBfcblCq>{nsco1TOR7fOW?c6l?E9GMsg!OXuSUNb&_Cd@q3DPo*^9iNpz;{TmkXT zqM?g&+3kEl!AK!*438hpghn5g>ab-8E)!aU!P_GM_Tg}gE5-#=p3^gG_A`A5r^;v> zl3#LUT(4XvBpo^Q?oa*pyxPW8glXzxgdj7zdk(fqIXeDFBB-r!N&k~-7%`Oi+fvXU z+ZRsN5i$uP!olP{hu7s8#3Q4S9$iJpqHZxyK}+W^umj6j@W}(-D?$qEaYR&ALlyE_@hQwtJM* z%+(7ehQV~?QQOYsgjQjfuqbDm`HI=c#NKRK9_5d-dN@S8IS5_~Tlzj3>VzTR%x$S5{$oHA7igruXsh<8 zffx<+uOugc2C2{XcV;{<`gF_mN_-|CR3Hp~=E4eQ?5&~YTjpoM%56#B9vPBxb;M6? zFz)V&z34=!v3J5>1Ks?@{B6x}HwJ7P)MqrK#nU^_*Zl{4Y~i&E*O8T2%Q~d~R{<^` zY1dA!wjd%rMUVI)qx=EsdfyM$32007EC_aUYoti;R15lV{Zup$YiKyy%Q!O;4cSJD zY)Xx=KaoX{_qAY7{Qp$B8C(21Oali3@`m|W7?U$``)~Nq-#zijswlp;9cRsFXy$pH zLEz9Nvg3h7P*qC_fne+R2^hIV^^z&Z1`XJ5?2SWiLoN8dKZhNL9Wu|N3h*=MEI)I7 zorZ6oe{vk;=wsY%c8nz;DCsjLcD-Ekp75P`Ot-mxzu&Cv0O9nW=$Z?6z(@*0;KF!W zm*lRSY&p72jiD~pn`#tax0mmgF1%m19+*EYxayffV5jV1rdw?0LS~$ak+&NMqdE&4 zgf&Gd(^*zmlg$+4>ds(P@2$ZJ&$#fF51C;^%J7lJ#E(_1yU4P#R^(=n>n%BLORyjp zXR1+ZM6<$JLAA4Aqfn7<-M0o!%%X?C>!HptEJ+SLiM7v#HZd(zo{o3|RHJJ!xnSkp zSOQ#pg52sYOCYf8B!^AGwQG}~&Gq^Tl&=@wafYm7Ma)`9T*2iH`~~ZSJ#=YZefXGM znMBlg4^DN~^70I{dHNV!^%eT)E=?gI^)O=^j~@s<8+Ta(OQQ(E83!?msCUceuHHlD z8ONzjq72lnP!e6`9w-9J-MvstS4U=L_m}h-VUW0EVGxFy-PMMH!Dk#YgQ{?foomcb zW;k^;ki1ji$sVnX;u@=0J8`nrZ$aSVS*lCzm1xF{Vkt0&B8})0r-g#TJlRqW0=$!ITzHi4xu;d*cp|O2p)ypjGL6<| zABB$EIlTaH#q)xq%6<3#gKlCL#W=p;%a;S)Swit0xjn-vs{ZtN*i_ad#sC!zzCDnd zg25VLi(QqG*QnFgl$3U5-%O%`dh2g{j9a%#%~aPRocr7ToC`ITyc#b*j_!9jhvo$qq#NXcY+m zD@jLGv5)QuMZOL$vvW~YV37xEX#Z3c)Wx1D4D}{G^>tEpveMPK6bdZ!%_&J){*o@! zVhVNqj=R3oyK;6)2U}5cg@|){UtY_@oH*W`jfRI$O!Xc~*6+3tT@kZuBUbD_de+R) z4*Z(W(pb|VIBm+aV^@-uBwYu`cg7&IP3>QCUTyl$&W&?fpbVb#Es)V%<~qTG-|7}@ zV$W>0xa;@GtMDwA1FY}x&=-97Yw(F;7oVyWeISDrBt(p>t6I@T5u6BE?}(Qj1S+8n zcnAh4euAtp2k10a`_tf!3-G5AutTW?EkQ;WH3LjSJ7NVyqy9Y%z4IYV+b97YyS+tA zdj4rW6<{~ji8g9L8N?gjwQnsbj3P6qvqYXLvl9%wox!j$Ep_ofHZZ<=V+Q8QDA9fm zE5Pi6gwg$8eMwkP!WA#R}^+o@%FHmi@TWH0$ zaYIb0yMt?9+AFaOBSSUfH<}5e!YkQKpjGkyd7}gqzC-6r^9(-ao<%bf9nq!G93>wj@%)Lh>CPS!>1DtmS zVYev*NxSvs1LPj~JM*#}B;&1=O0e{}$NnZqU-%5$8u?D7R@#N*0s7P?*t4fxF3vgE zsuQ{7l=y{3^$rt>R@5bBSkidZGsbrR!FfS9_lXkejL3HGQd*~9y*274Z%_X4mVHaP zSYMN@2bjU$uu%DaR!doLheFgkfLvuU5}@?IvNW6zCTV-{kdNWqrnoOl&FdsrgAxLwVm2GpIfi7aH}dZnumDh znRTma0?w{NvrjfKFY*S1nfk%IVvFM=mE=^@CY}AT?K58E!XJA(P#mN<#2or2LL8Yc(s6fFN zMq?}vGC-W<<1(t9etl&{q(QW)>Hwd^R2U7%{#@QJ4(rW{of>bYR5wlm((NwEauqBW#ez<$Fe&rI2$u$%i$yiC z7b8*6n^;GUEQtcaqS^oGn=5b^)?GNK;xVbQC|7p)6eI{aaWG=`%SN~10Fs<+=I)1s z;vV83v?KGztNQQ_!?_@`^KL}_x!3p^jxF2KSW|HWrChrEkfjN$F2C5Hn;&ggDSo#v zIKg(?Mx;g_&@I|H9JBNyJ%$5g@SY7wFSD%W>_L6{$=OvL%0t+7-~L>^F($Ro`C^{$ zkm01JNNhq7$qW4H-&jn7L>x>nL?hb`s{EF!-y^E9Fz=z>Un(d7F>p)Tolglzs1xpIqN6I-^B$RXa6Y0F!1aD+iZvf4pur`{UaG4$@ zG2g5#?dCuS{{}te%g37lkKTdRee<07oBB91?}6TXek7~U6}^-M!W*!A0^x}$e|G3b z67CIzA5Ss2-r^WZ8c`}8LCSKAYrslMq@8zU=BU2&&sY>n++3aEA3#9c2>;p;6fyY= z85|9q|1N6&&-_CZ4BsDUp&^JmBvIp?d9<~y7R96@gc8+xO4eybYqe1w#0|_FlR){d z4VS)6ADI257pz}JR zKLO_)TP_+QaZzO86>~HO;-q{46ImJ4{DHQoIF0SAbHMeqgvJ4q_vX2}?8G8+d0!1K z3JQCubhb6UiEb=fWxPXCpqh@TY1trAq7p7GRtL4M)q3gmX|;*82J27*kCF5#G`-0x zQIGS<9Vd&cl633VRe?+E4=WR$3VqyVY4aXye#ufqZ<5P0nfxd`R1NDvGE5sNHlxVk zTO=AWou#Gu^fONqwa3%q>u0E+D($Ssq``=T!m1F|=Hj8j?}f*7IFjbjE#K?{xJ0K3 zb*h3%9?h!l_5x1I%dZC7Zmf(za?We#Z>g8 zNoZoO0S;Heww49uRTV}*Y;Y)HyXRLcD)q*YZJ=ihZ|OEk4oty2=4cH*?g>dojm_w3 z)k8O=!+tnGdzx&P@3KuF0frtFrSVIU5cruo*~55g7Uh3;xp$A$!^pA*txlK)fxEp- zwrU}Kyb_3k6U|^yPPhiO2FU}$6!Ef9j-&!{&;|whcvWsSCY_rKO`U)?%XaCa5E>>6 z6Y~jitngv?ZM&JZ1Pn)+6215Ey6uBWUm6a5XgnJImvRlt=pwgg@<%kC<3v@_a0?cH zq2Sh%jg`}IPuF8jz%|B>OGufcUL#7fRz-`dlH>9wNh)M>(9}?KjCBSP*%=iZIo@MK z&F8+>APQ0$6g%QUhI$BY=_*q2KT=C44@Qno>5fvkBP21opfLN$twU#2peZ#7F!L3> zIx|ulN%gVFNt#pAi*X@mn?cPOF#2Bpr{cW;KMgZbq^y8@kOYILb_a`al9eExei(Ht)IEV2+$ z5C)~??wCn7xmOFS=_KV+Q(rcmhdZc-g><5~B`Jk?#^VC2Xrl2Wxm|ZQqa$SrgfaoA zCW$@SRTz>x6?7T-2Jez5HIxU^KbCD}BGWO|!DX~wR z`ho=hB@6T|c_YRdF(wjTfd}ezUiNY90pz<+wyUQ_O~f!NZWg~#JBe|2;f{;p%a8u` zfD-fNeaSrw9?&q%9-_IK26wtpR8*_n;)9p8ruB)!Si#Fb$N)rV91$7u^9PlN}>dAbMP9zD1TCCB1(=ns; zt@&6(>v+XhFA#B0*|{|~2zL9lz&L{&?xBVBuWWk_Zl4V=<}<|Kd{;iZ{Jv~2?qLN- zZWKJuV)d=>2@qOja|B>9Lc@&j4_EGZ@p1Abhsco02#{|gLYzeSP~b(X9#NEZ4j{|Z>B3ZQYl zuC}Q9Ej~lT<#CxYM27d%)PXg&WyX*-_QOnZ<{%)lG*-ookta069rwhAA#%V8;^AFS zZ^g{P8czNG9axXuI6h6-COBJq%Am;XyqboLXtGzUi5WW7j9HVX(fmu02|HIAMka3B zqLTD#Jx&lu`7p_%ev}d;El<97;c$vsMR{6Dh+pRSBZcU@(cuL-MgBB&#!~|8I>Z68 zqb2OdP|G$C-;i0q<;lO{on6Iu)S8y0uUgejvxOjPZM%9F|hhu)%-sb z|D>g*fklVV`2>3BK_cN_dlM`P7#YjbR<)2nS+vQnc3oS3>k|B-K`NMo|G($`*_N5a zH;)}(Z!w=_dYxQf*6i^6g1E;_IYbe}i!p^N7GgwnTWa`fy9ex+Z#S;%cveGzRBRmv z)avzAP!U=(#pTst)QfYiD+cB=Xgs-r?(pu7Z9Q_VG3auE6X9)4wbG0t zOcY}abvVShM@LWWtqa z>uNPCY*C7PO6dZ+^$pqZyF>QuntG2hs%;C>B<=efML9I)>5?IHPLasWt+bKtPnRDZFt+Nb-$7WahjqNQ3hf5?HPI8rQYrV=d^Q z4-bO~=J+Jy@0jDeuvuu7nKae+HN!{iaK~6mJi% zRM+`Py#{x=F3O{J_BZ2d5%alnHp7)~$MuRZQ(-8GO+H?G{l|blg(nx43NRobbm)Ko z)vfJ}{`RY1D4~d>d|^HJU&9fUigbcUrFHomp|BdTNBXDdLX!yzun(_tZ-5gIOfF9n z>?z)ZsTJK5mqBTesFwYiuIUufBTA0o+~S_Rc0`UaEctlkEa~~0zMA>={lx915CO3< zk0s70jKCN$8PT23O0VVZ(>2npaad>$J~=C{kk^W-W4cfQ7fwBK`c(Pj z=R!_#nPGO1zBV~R@pNE{QO1%hh$VDG*j{IP3FiBB-$pg;dz9I6z)Yw)l4 zNw1`EoJ4w*OHkFKQ4WmeVeXs@<%i2qHe4rIM6NY0McAKs3=v7zz~nXc>MCD@8Vvqo zD?`@d)KXQ%=u&fDD+nS6lkSH%(VGqS)E<{_A_VN??JV?iheK&Dfu9eFn7m%$0JSQ;Ji`M66mWL*k{x(-ui>8Sqw$ zrPGX1=rUzFE$awxvL8`*^t1F(Ne&aZ-KFyw3=v@%s$g6 zIAnhRL{Gn6u!l>_A^bDnNuJ>qr1Q*vD!;Q^AX_UY(1BtR`8a8X(n>BXcm)~%v}?sf zoQlZF1G=K!7&)c_Q5Mewb_g?}+AG)iYuLvb8NXtPJcHs<+-ZPV`}Of@qX%n_r(%=0 z0OTDY;VCEiO*?gmo)oSVfJGV>R7e^gS&p+Az%GY2gUWee{6I9S5Ci1_XCXzIUY(ep zx2^a2k4CQF^vjQQP#~aM2p}Nle@_PfpZN#>U)jKDii1`J7FtRmuu%u-i-jz^gRTCA zyM{V$@8?(;fPh-!WZ>({Y)Crr#@3$=7a?D;_)xSP+ z_`g8xap_C!VIV;zp^+sRjeXYYKLy%Bbthx93nDB!%7JaR> zz@a$&3F4q7B*nI?)~PSH96he6g-^GD!lDDO^w{u34wK!+5N3{vT4!Tv0Av;-pyQ77 zHAiOHco%0HJFSi(k{$yM-r#QXpniiqj*2TAIT@pRkh%0777x(`))DEK)+-cx*hjWA zk>gx+3rzh001}P1hGHOOpm$v*$qKD^4T-W&vX!cPRcX~BJI$s@ zfPs>_{17y+V`BB=XSFuPJ{lyK&R4%Zp$`Q7Jkbl0UK-e2-FTKGQaQ1Dr5II?!7;&-?QQ(^IS0j%R!^nW;T%RK z<`+1t*un~AVB$2o?Xxb%5C7l=-N zo*&$KBAD_(s-7I_dQ)63J;U#*L*Jj$ySz3?uL0L&Wk3_gt$`bjTBp?eaMzaNi4+HT zoXQm+s{~g+8?)!s&esO^8lc8}3!fNyh%{Xww(*%MBjH6W>drI|b)3 z^GZg-vt&wAzLZXerF}#2w3TJFXBf4j z%5|DAu^`T!G?(uFuw{}#ZaJXs=M2<}6BwqYk#_$jD6|W6)I6}2qUvXPL04ki=}Zw- zU(wb%CF|BQXWvRqIAp&jy$k6#T2X%7xRkGU1c)MG5{g0vNC@?22bAxEZssLE*E8aT zAWvhbAG|I#SGc{i?>&SWtNXGY*wtYz%+D;^=rf-~r5p3OfDEHaHYAl6dDxL;W?2YJj*%4uTXzkH$sB;& zA}65LHH9hz^;aj3Nzyx;aAhUC?9|nIlC9r@_7=u)9QVQ?wn{lp3p3>*?2NZG`#D>Q zCW8r>(cW-ID2Qyr`fmRm3D$+)ytPeEI?}|!mdg440J2XypRkqE4hu`ryBM9eZpUzo zi-W>~Z^0&7zhS)X0%}+fMVdSGoOY{(ysXG6=-_!~&w`Ys%}Zy1v1 zijVl^zBIie%&$F2Qe-tu)cddWOAmUf%PVj%d5bIT`k6ha z3*S_nHT5}8Or<>Gc6Yc(VG)?yGf(Keva@!S!p_w%wU0j&X7q@=5T87%AMLrBBY&ag z^HxF)m;I{yg39oD_@eeczYvJYdAg+_#^J};*CL}8K(>d1KP727)4?}@1TSStWxGww zc0EYF;m3l?w_pp|lRUj2L3paz+tPwufWBvLE^#`s!e)IVYF+Bd= znI>xOYT@)Z&`?JbMDZmPy=bq%wT5W1`~V{kLM?Q#9EnHrGO)BOo#4~k99;V*%s2rC@MXlzrGw_#)3 zwrOuE+hqeWaadi^k^z`_N^Xi6cmvpnm~q35!Owf@&j;l(D8q5n;OKHzt-I(^^f2>b zv|DJvetJCN)_8-l4dVQo@B$kw!^XN+A+?HX+mDzBy(#*09O z?sHTcpon8IdGU=g6sj=O0HD9bn}uBPm+WOCmL;ar%~_H&Nn#E}a1OBK_E82F=^oKS zOgJ`pE$i}YgVpmv_cq5}r*0*p@V}w}4QaQDXFd5)mkE9Q3rv5$OL2arEG^9 zin=*z_%uGKq0i#hQnL=v9 za84>eLI!Ga>0HKGbXw3`5Y8XjJM8J*I#iovb=#^|Mum%Rc${I?Q93+JXjfKPbVgtS z26b&xunUZ`fCat*K_9-+PJlL4p$&B$H`YB8$vRW!%cF+U5U&av+OMm|^35^~)kN2r zQKd9&0P*4aPL!uCXUu|Eh8Q5Ie~od`>Eqn8JcxRMa14fpV_;PduyMstbMk^*q;Xc% zDH7(eQ;bp*05(9$zX>pL#|_=xMq9G=jT5%`rMw=i@}_Ju9*6Z>RLpPzSF$IO)|V;s zuG#yR^Qh)&R?~KopGUOFRTg&jqVC45)PC z6;HBP2ZlYsyGBlHM5`0oo-mFrQ@-B{+T9Xf_Y!#q<|*zL1u zdS_7@KOzI3TIkdEQecdbS0c$Sp&h6aoe-m3s2_+TgZK?obk8-^BMgJKDE|YMbvH#Eb0+DYN9|6B zapnMAPw83g>=A2Om0d-pQN8CMCO6f`-}@+FKtKk6ZL$1&16<6)#M)Ta#KzFX@o&fZ zzgd4|k{M*W0=ZS-6-$jONI@b9>|<#(WF4Fx+LnG1_@9U9{1oEn z6aUTo8^~IlBAYS|$;@o{oz>=)bHe9zllA%1)AJ2{%hxSQ2yjP&1XoTo)u_8nv3zPh zv){69T*|gh!@cGxH>e<$wzI|PT-e;-;%aHBA~Tan^GyAcOUgc#cWElfpl!Nb2dvEp zpJ0Mh7Gj#hlw6)y&*|$oX;N?`3@U<;++qpyLDjD;<5t^lSFC2L!;aX~W*A(TVMHq! zXv`#$8CAm~c5D0fV^^nC6+3{CC#ESg#DW7Zzgu;_^UM}zYh~m^&r}CkHdz7Cb23tD zK`ga0;V1+7fyx|H3>*8le&Gh3i9T&XEGf5QNOn~2(g?aS>$O8$%}bqj*wqMUr`Z^r zdrw=-#cx^3{hXQd`!?Vct-bb?yTX^_1cSQ}#GQI22@Pm@^9newry__$5y5{pUXV+;i*q;?&ovg;|z?mpdNaeo3-B*t~Ev9pS&WPh7~ z9@~E%RRy3Vm`vB=%Zj4KtFLGk&e=bNH9J0cxKtdlh6}<>6Vl0Hkq*j= ze>3a9BCwy9&23<54xajPrTW1=XKdC<%`|3qiFwr=5M$B0s~P(t#}V%(+e&Zt+w7!L z?F5u<-st!Is4hM#B-^6M4!8kOp$#F#Gtb21kUAzlZ$=Oh^&|CCZej|PyiGvF?`sfU ztP)VywS{o}HzV*By1@a0?v9WyCq6hwn%s?;d0JL|9}HXkgI%xj<7%`^c11-~q%iUD$vG z3}`Kv`rkC54IK{fAXTc`irRJ;ii$i^qKoCK(9mH-YLP8fRZGjw%}q^JUGnfEXD?lA zTbFXCLj87rJz4B_-3Q-UH$L><_p7!*Ifw^76oO?Su~78mV1aicBp5dfMv}VH!RkLg zScpx}$Bbtsz=Z2q@0L6xP17Tv%v-X}V_!}|l1+Ike^;GpaF`4lfjF&iijb^`b{v>8 zh_9EqWV|mn3tg{873+k+qk>p!MiiyBcc`3RJ5xIqg-DK0M^ zGzVt2wj4VIAzn5bBm!vwP?s0^%R=E47}ER!`4S3{_Vh~hloAL|GwzJAltCzEG8{`n zgTC-+B8OGCDNR$h29^ z3h-pl@}X`j#Dk%|kU8~Cjb~B13l@07(pwXzoz*&t*dk>$MJLdn?~;NqG5y(2H5yl- z>J6m~J}mOM_(y~Ucgcv-R3d@Z1vrvAi)m6nBVHz+b!1lq+=U7xO-@tsk7n+s#2KWl zVUEHe?)Y_L@Q+Ca`^*ReK3T3Gi4YWrk*+?pW7Jj9mHIIUs+j`dAo4P{u`^?IjpBj@ z0{Rd=G+G;}xuivG*ct(v9Fjw0=18|v{LnP^FBQVxI^JAAODN3M7||cV&H=rNPHyw% zTlJh`lM<}ZGEU-s;8a!%P}?A=v>Fnc5`7asNNG~roTdIyjr%I#IjV?;naW!jmG`{N zeE~)OFAs)!+gUNehFy11nK5kOk*bYO3>k1;Yxd_P+)BZ;lcOo+uQ(}l#h5FeTq)yx>omus{8pNqXX`0A*qlC+Mu)^PJxh99sB!z*WX!VK7K2jjG^6N1H(d}jw)8vd zINgq~UsPw$okdEj9&HT5tpXwlLq+%*Y@!dOWU-Bsw^buXh}u(CM8m#_v~@zFa6bJk zC_oiTt%3m&HH|T!IB=e$h;&o|lP#bQ{D`z=G@s)gC9{SIm?&-68uSSs$*}7D;s>O| zeI|>ppjPg){&Pe4C)Khc?ohnRNv}NBY|^ZkYyty9IQV6?(XhxjUC+G-Q8&%4(@Pp*T$X3WoN#<5qD;G zH)d}{Mg4(_tjfxK^L?Q0~Q`xt{2bN+aw}*-yzL9mK7YXGRyssewg-Z2IddYfUkO&8}lCt zT+oPUSKtD*-QV$`?=QIo?`Q@ZH&IccjDCWCZZ%DMk$+Z~-z) z9R2t#^87`aa)pdK_{(q47Yw8Li}hpkS=JAWWu{wdi2SW?9jd*3fPB~9zH#r}#UJ0b z7dpEz^q~H95}yKbbhRqV#SJQ})?+fZ^Zv*-l$a?p+heQq^jUK9Q+oq5xc6mS-R;{7 z9f|f|*%6*onhz5b>^FI~wz)d=a)JXqw7Hw{N=Q=*elFzWVEK@A83Bt9s>sVhGX zDYIeDep8ZAWc&!p6Hptoq>z|A)_Oi_#zQ}BhE6`=h6eBJXFUDVd{~xyqoA|&`QV6< zB;V3~FeLHJmVVTG&(c7rqj`t!gF1|Ag?sIZ-7zR>#kt={aYwXnLfA% zCurw@kn%?qp%m?H&KY>KQZAiDvuxK+~6oyYgl0(c zT=joB{b}3}+}lAC8d>9Cxcq)7=mmlu6XQ)V*pKW0+gr38{u&lX68gCwRO15a!jcsc zl}ScbO1xX0zukbIEIXhj1K*Gn?~=8pEmbR?F}e@@y2-=10KRzzRS1re{c$Az(|1|i z6Hngo(emVM@bW=EFFV`+fDpGHX$^kK>QQ811c6jMEEmL|v2<8@yU1zgw4VlX&BPTC zd0bA}04-TS46e-35LvLv4}8GK#sa! z5c?il(wWy{qu7gN%_GqjBMgOt#l}+UQC7Q@c;U$fjj28;hgmR0eADM~a90w#dQQD{J=rVuZ)*F9?hX@KNE|)P_Kb@||1&_+x$N&_7(xsEer*w=f3#QT#-u?G zJK8PcR5K>6=Z`k{%A@6tEQDUGT80^;4-Y^6vuJ0~YV^}fBc`VL4Wfz7-Jzl5p3ar^ z!`0x((T206DsFbxFjwP9@_;@fyE*PAef!IYfewGaTwWJlUkV4!l1!^7dRYVP(u9Iv zdnl@|)e3F!x0OS-tNtgJRfCFM=E~hjVV_dHXj?^SMSGE2ioSKr$y9tW+jOY zs)lgZ_#hRqO^3Oj|8ok7=Euiy0~8n-5b3`XkNzo7mv*prvo^M~_W4iosiw9oNCV@O zp|N-s^&7Z3H7~`YuF{&wxWs^vb2g+*u(c}3nlb^ZHcL_tO6Q#aj=>vz#;Gc=sQ!s) z`K!uXHtg_tud znLiVuwbt5O^H7)eu6~O+q`+aVAMrxU&=OY_u2m)#7Q(+%x{8msg1k6nggqu#W9Sj^ z;j?^47*e9MPF-Na3wg2XeNy$eP&zQ?HSCfb^hc2RCg;?@vI<2#*>y4B+n2p9yS+tL z!?w^K$5x~LTO^8ML&SVyte?Ku(_`fy62X}9g@+)JsLDBN57os=4w3-8;LT@tXxG>t zFd~a!m~oxSaRxur)-<{-Z2obGt4^KXSOy5| znCXWpiYP*M1Tbk39i9Tt+EqVq0dU4;vIk-@7{rxfdr_lLZiE(P{7!Pwou;Z*Ug2aK zl#u3(bWH7{nms$$rasMAco75j;?(nc(-adurlc*ODqGaigxVd23R}!F;uN`evK|iy zK+N&pn%d4Xva?_R!69>biZ*Q9L29fw0-5gMDSKYka@^8gxZbIavM-i6Yz@0&mCcUK z6~)O!y(f>m;v8aZSggKHgJq4%IDmkvW9FO_SBoOlytxPwMYomYj86kqknOr$qk+As zWK^dy@@MTr*kA;M>fBg41p$8GO(H^rem_X4cwGg#%>^rpdr~QM&1?J4w)%%DG!J3P zl2)~m^%7G>Rd6J`BZ2ymHZGuIIZ1qJ{m9B8_ZOvRtWNrGy4-){5W99b^YZpJdQ;Yq zp;@d}W!c2Y5@}~Kg1*Tlo)YB_P8We}geLl0Bc1ibdyBywOXbaK`H~vQ`_vN+2JPps zw%1v4K2YedcRCd^5MgF4vXp43VHtE!hZ<<>*kdE@7g)BcIA z$O~-je)G~*ylVeg-iCnfRpaNX5HaUFNB2i4)wPCb*NmJ?&g;~t|`{= zk3}w3w;g>+;?6%&s9Wr(?GOm!84vdJpoRvpb@cK4fdDiQir2WPHT1)c@GiD>oz`GP z7!<_=ioRHzcxq{!kqHA^@ zaqm5^pRcz|f5QZ?`m#Os_B|tq;*{-p6$Q&H-XH+AuF)^do{2WB8}j!&A0EOzLX#@t z@&cb97!?@E$J{}TOi}!9^CtroLr&TWa&zxQQqN$-aM2x#oL8V&&bin(w*#WBc&p7w z_pcu`#w$Ck`oMgKqFC*BsPQB1Zka}Nh^A*|0g@OL#}0M6Wd+45I2S5I(uua6#$Vjm>2H#p$;SDX{elCE&vffS}3Xfrol z@k>v}Q?T>IH)+8?_j7QwpJTJ+3!~l2aH}lAai8iAhjMTw;*MTU^$3GWmVdpZsF(KB z7I6BHr(I^^zC9JNuzQI9xe6g9i z>3zjIo9lbL^Y!@&)6f2%G>CzQm&8-X3^d|$7yntoXdC9AMX6JMB22H( z?D1J~{_xl_^yJ_M#*F&sceS))wXeA39Dn3BqRy;z?6`A8yJm>Xi0X_IxB>h&uMAcm zI+Xo4owyz06j9&-sv8SF>8QXDIlAZ!bPU_sDSJ?ZT$d@I4gZ0CUxs}dbNm6G!UDQC}_zeN?W7$fqg1U|&k`r&yV;A+x=?(?KL+i}h@4j-5p9X+d$v;Z#E?c-!> zdo^*2&T+xuLC&pOo^@}5>yXgrqsRAK!|133L-)Z!zwg$SB|1TyB%46Go z4$LPes0l}7e;7BgwRsxjj><}F!5I_!$hGJL!8-lTf5qYAk&2Tt+@V4S!1K@)hv_#^ zkZ2KPD8Rap16NV^7s(N~tleD7QlujeAFp@u7Sn-+F3uBv?tK6M!DM;(e_^sTy!ZrG z4{rh1nymiGWUF885d_23vzYR?3W>o6y!-=ZL@PUx5!cU6 ze|;NkQaRFt@uCFM*nZWxM_S2K*rhIJ=@b<*s%{n%n?iY-W${st(+SkgS&CL4+Z0i6 z(%45*_U&aJ>XJi^+?gY_gJuo2D{Lf2k293Z9tSc1M=AN}&hzEovp$y=4hi!Pbaxt> zdf0V5`&wTA_UUW*%tFus!?k)Pi07fizpZskG+w3QFN72a2*hknbOb17QZBIlr2Kw^bhv%e}Tv;Gq)Pze=!But;B{b2f84HsQOik;|VP|HseAcE;m(=dX0bZv=zw z4r+xu!J+_kc7$naek`fdrIJD#A95loS{;@`JP9EXgz4+M5=|9L2k;-?iwV4Ic=gr! z^i+5kRt8)LFC0G9;h3S9mH{Q=&ax?*AY4^hNRcdKXj_tEg+XXseC64OLa)q1ok$7X zwicZ?G#M5w93(?kGK+>w6MDQx=|Kxz-MOJ%Mi_cRO$2r(CYG}5vknU@3mHqN={r>J zai{LqBux(F0d{k9>(+F<7v^W*5T(fDsi`=tw%`$`$UvwK@A1EfZED@dJq%pzP*O_(UnI z`ip=-4p_b$;JGkZybKprF?u|6i^eLK4NVzO?N9mWt}F-o%1nL)-^pZb8RZG zJtkmTcYaT2-PHZmof132(Vuu&N$Mo^lO$9zPEmU~_GX?j+PkNqG`^DoSXjlelOv>5 za}y=@gUXZ88BuYynOor*;NG~jC_BXis~sjBWV@>kR{;(vC;?3ao#k5e-GMIBmI#>6q^89YNL|d98QRNLcfN42<0g}BF-mkPQGfI z!I%&G0Xxh@*A`+P24|{bP=yh#{y96|F%@0Ok##0du7Hb5QF{lbpIR6JHmqVw%}L=e zCXoU=vTUXY0R=L);#5magV3|nkw}XEy>yPsJ`IXpnS;v0 zSu&9x&Y72fUX2=A*RL!+ieg?xH%p*Ch3iz+9d^jLzO_FHZ85Z4!&yi=o11deS{hAp z3bw%WHYXp!)IZpHECI&YxA8lHatceI|5v-#q>XfPNjMK3;vr&@Xc-D{9 z!*&dCLN3RPJA8x6WwN!QO>qE;%lG)LW3c0$mAgU_&iBQQknPa|(%65I;TKop@)zYT za}hS_>%n-@HDv4f{&EOYf0#2*kcfRBga))^_!BYUoim--FUZ9wK-&0yLN(-vP9gO0 zn>;d|as_MSv1_Yj0NsURIy@w1`5aa|!T0f8+(0YT+dcq z7cyI?dr4Q)4U0^5B9f&4HUms_bu#M?O%Hp~yrHSB4JE4$1$Ds@L>I1D#a;QgCE-+^ ze%lN%d_%*l8emq(HsTQtMfz^uMK?h3lha@nfAAN+<34>-0B?9seNVEkZ~7m)gX{Ut zH|~Szl6mRA+#IDqUufOmw6d$?OpA@7TL_$uvCwb00U>>pd+tt!ak`U%t|>@CP1064 z8q}8eSPoyR<(B# zwjeLTkds*bd_!jYA|QHsx!jCVC9!@S6?Y3WaS2E8D#zbLB|40e*!G7%N2u{UyAP;L ze-<-x)^6E9nLF`?j9wSSr{rI_n8D0k6qIe9kL_YmrhLL+F!(f{U!@ZM3}jc zM_}-W@$72<8A5w$P~JKZCzO7edV%tApGXLizH;PF_d`x_2CBj!y_H1aS6}#~t}8rJ z+po}-)YjzA)^4-x&d660>&dH1zrk1|)*eRXLH(b2bw<2(MNr0_qmrlF)%%b?QAKhX z9Pp$mtSF>~<&;HAZ*kQ>k=35sEAOzSH}b`<48=}s=FUlfhma!b<%|%ldEPP$pceK< zD@seI;`qlgepiMFIMyUC3gErEp+yd>g%Z5iMO?yBjFgsoa+Vg!|LckTH-%`D3$8ts z2G&sKB#|jgw_H53PAJA;vYUwpc0MY<_fyzEIr+w)UgaeFKP`5XYx1j^p}@d&5dJHt zK-0?F&ivmEc60yN%N0kCiv}&;U{!?`VgvzMu3nY)gD$px6>Ld~Zc&zUoCG&>s)Q9i z(cto^RZ-tRA#g9mxmCY9Uf_YVRe$SP4$Y`kx-eAkk-K?TVCVDE^Kpte@avWL7q#C` z2*imBiV~_=gz=z9RF?i;+N3+B;Q~#kaptCbtBkbg@3h{oL#JN54uB;u0!;q*bAxdv z?~yiI1lL=+E-*kbJ0bQ3xS|EoxuVItZa7;69 z@t?uvt&g@@1VvTij93GRWU8H`e|U^l;r<9#&QZQ;i|Y8b%2Dfmm3lGAKYGJec|QhV zYRbF1WohpoZe>sRMU62P5f&LHpYfn-?$VxFEBk3`et$SHXMY>pxhAbcPn!(4ZOldU zVV=wBr2yJ$N}G4xj1dBI4}5q7>N*N+Fs>^*enle$HC@%P=2WpIYdGT*Ga)35EMN^ZySZw6BJf+P zY`5?8#_27)B2;H=N)k%qB}&ELYbiBU?2B$1bG3h=AixPvWwlXGreI}Ss?+U}*_b!U z?bnoER8f@rl?@0Vn_`tE2PzJaY8jtQzsc+-QJeJ8N17~A(p*KWC*}bE{_SLn-)rjN zQ@O2F(o`JiHOX4p>NeN|cT^BJsA{odTzg1P%H2~MX?PZC%{iD!%D3(lTBQ^9XW30f zNIH4m&c{wk+Pm<1m%FOoO8V6QdU$O2FaQi}QE}2DmS8_NPfjH<{?(4EShNQ?EmP@q zYPwTxp2g(kIt(*))sK!oD#*ry#1Y0Rm7~09xpF#f8QuHdU%L$|j~DDY@UL$I;iF{o zcCKI+&z}}KDiBjW|L(E++LDv^#u9FMm=MQ?$z&ghhcun-YXljeeG%BK3)*Vt^7R=@ zEnTVXXe-lC(AjHmCa$@tIC`2gn=FRKi@rnk58*@hXkc2YiDCQyx*77?$8KQgE)d4% z(kD9v&nXT&Zj@N2-bqYLF)Ls~;43p4)Crf~HgAGCqH&(J*>1M36M6jRC*i8#dY^xu zJvw5mWFU$uh#raOGmdjJ3Sr{+5H>Vz{`^CqQDlJPfh4jxgT$zr7i=U8%vAnYBi66R zpZ!{d1+-6GbDmZ(YCd}f)xUUTSxxk<{Pr=o{Yd!Rq-9X6Y|%o9`4@*!g2QQszeiB! z5Jw+bmCW_jLQ;5D0TRdYMz8`x+V0@46=3irm{1WHnZyOs+n7#VMe|rAfnG$2%TDf1 zUlNGzWuo(Qwht6Vg?KCh{apfqqWhG1UXkY3kcK8pmMKMiH~QotcD_v!t*bIhvBOGx zR63(yb?s7+?eozHN1Tc|bSj=bA&^%KwnAxlIwSHnXSg+U#{%pJFhf_OG!JBckpuxH z9Pj9j_c>bH$@6B9@^Ni9koZklx?WN8{=qy%C8E?H8d_5tktGBND;z>X5IG>EOqo4y zVbt7wH6vmo-fhEATWSLp5}Y+uO{APg73HT+FCtcP z7C?V&XZ^y9RP!nLAuG`qOb}z4k*pxp&$KKlx_xx(-qXrAya>^`HmziY_V%+_0ufJm zs~ozc1-dnvT!B#mO*`qAH5<@p&M-4l?{J(;OjzJ2OG+7tbg@u^&2ej@tf7qk9CeFr z5P=(?i|v9F05xn)n0mN30coGsS_ks8Fqdu)&+)1V79ki(I9-|YsoiP;GFNw)(QUWt z7cvF;StsS34X3k_O{)PP%UHn{f>1r8e81M;3F+c#@3R?&>=N&szV%Mob=(1=qRupfdk zF7cKSs`+;kXn|0xDw9xRqpHEdCIw>+pSwZIHum*A_!1gTcZ`fBu6&XY(2ide1oqS#1J|jyEJ2%r->OlMclxkg zU8~a{g3c5l?jR7NrMGY~^3)(SA_BG4JtoI*RBMD{qj{qmjj?z~W^x&o>gQh+ocyd~ zRdF;)0jJP5hS-YbT^S4=t+z9jK8ab=F!on|`B4H&9_vLcdb7qTRa~I4GHtX;e88s0 zIDs1TqDn2RN`f4Jnp4$VHyUJ?Q@4vyJ%~tb$ML`cU0G?)Z9PB1`64k_-6cPb7>mAM zX+^$RfJA23IX^(nBWe)LKxm`&SJ}(F#xjFNt^yvCtq;$NO+mtJbm~$j_YaPm3X)7k;$}wsA6n5>NFA$Ej;5@$&Kox4=eM6o;eJF#M zt}(7{w@b$n$HT3Lv5@B$boRTctzzpt_HEV766y^El%**;&V!8kt%_d0T&V zs(;y~bVk)l!19mS#a5(l|hI1P*7XkSh)|`iRS^0LWhr-06h{8 z%@Jsp>qrN13rrE0Xkiyh_u&O5!dSJ~>vX~JTO~Gz7@OGmdMS4a4=7xamP$g7^lEyO znJ_P2`#L2DXU|V2h+AHeT3icq*B&gFkd^e5^D%wEISkaM0DX8Ftq_t`U~%An!MCeU z&&{OV-ZNLdlh)>nI7&=4;3YY)e1BU1eOV~9U2FvQq!PPU09~}s(s_*@eU6rkid<9r z#{r7@g2eM=0CB|U4EK_qX2rue%1$m@+g63sfbmN62Fx=!J4dJ|_r1Rnna1n;8{JAx zThe31eq$0S?Bg)|*XdxfniNFYpTUFB8}xnb(3|^-M~u9*+n8X>TLR$x>APinH69rb z^HKqBVRdhfkF>qK>cMHZ6AM9B?~OnhHpV*|W72Xft%A1Jj+Aze3FlIaf(fx>d|J4n z$_6O~R{$NMx=BH=pA!imsoR{tZ zFIJH%qAIl~=h?zGEUD0S!lH?KN$ApvdHlnx0Zy9C~`eN1|{jQ9!hM- znVyU?rfr+bIqKR7!f`cbn+u}M9@Syk>ycd@@$itwz21aD!id~8%D?o#S(*sk_207~ zuGM{Y@%i${O0Zpku+IGwIqwWo@1;y= zrg=c8(+qRrd1qK+iq=f;%fUOPn852B-`7tFSDEu$wb-|b?@_^{d^M5)hY z!swU!&*+~BS}4Z~>j(bt6f={ywL#9T*AQJB#;L|6)pQ(DJtQ-Lz4~YT%l+_lN*YpE z@gYd@s#7_6JaM~w0jCbvsjj#IVzdWTkKt1CDyn!5XJ+6_qcBR85W7{WM=Oco*jM0twUn8R} zZxExDIggQ|Z^(OgM}icafrNYW9XH*X%or(Y!%ke^ROyYhV(n~YS($ah^m4KaWb%st zxeStMyyIxRrK{#-`MIASjn+>#b!C~k=;g-*fyYCl`?JQ`>3rFG97G9MRAjCveVZJP z8VEZ_kNm+0ZDf;eDF=`Hp=taTH3n5^ob?^{_zDU z@$OgDzGRA4Dw=Jo3r%-{{VW3Gj*9T15qMvpa91J0=>V(HNoi=RzUI5iHP zn(+>_+Yn`NvA0)d({4mH_Ay73tC5o@T^wnraIcey26O3)*dn^{O*<5Gsvnwx=!C$g z4pSWoX2Oe6N+i&om(HWHj!c=kvxVU3GN+S z=?+AF5FB|!6OLgH+CNI=U`~`?^p8uIg@x;Iu6%&o0@a?P3o4;7{jrP?$R-LLiY z`nTN6e{}!aoA%^-p}@cd5&x^nzlfuwow@P9l2&j=le{&@alZ-M)kG0WD4cF-a&GQN%&yk> ziOpD%Rp`1AXRG1Y&*ol-XZi=h@_TY(DNmL)6M>vpE}kLF+1f9kP|*mgRj{x^(76~% z`8uBfOJWmO{-WJ9Gd=WCTf)RGu7VgpL|y1wa!28|#zCiz*}=gnKHtGZi=qV8fjk_C zDLfEB=ZWUG?Cg3L)mHz<%d6KYQEE+U2bHaWIn*Am%@juTElZIF(o6sWVmp$r94T+D zyQDzctXPQyU5A4b6lsL*5#As|3|P!OV~|Ra0S@fzPnKhFSqNT;8O0Kbz@Kt3c#x~gOJ%`nu%u) zS42)Y-D>ZH(M|%|@M~4+yE7nnK?}<@uKb`XH=c5lZH4S)ghaU$XhwkP&9Pe%yy6tI zn!JaL)lJJ*QQdYqczMgFlb6z-+LNG|?W31P!D=R~j=YBmDAPGP2t!D;imoEkoC`77 zSIy4Fo5IF<6q7_F8kZr`oiE=d-!{3yj`E26EK;ot5E}RakV<4%{RzA}fq&0YxoK;e ztzMQFo{0!%4bJSmjYL61c&qZ~D>~<^K9iPof4=bpU_%O$#GtaO7nhh?OL!DioU(34 z8DWH9CAaFGuVF|7(Pq`;!NXr57z8My~K!dns$k9JDO}JL~4m)S-SjX=nF06J6 zR?x(>e?XfumDagzh$eA=rI!|l{IyR#U9zSd{$O6#)tYzcXlfIv8zJDc+YZYX+3I(t zjI3gT$eSwmQjJF^79%oTd`6eiOF54e?1;S)Qq6tbf40^X@`y)o%Kw$QywySQKlkBO zBP&fygH)I5wSabAOu{OkSUyj^^xGZo%B;vycQ0xhcipz;oT$9x)<}E|EbIImF~UwS zX(HHX(%xQcug@a(hXy}UfO5G}ppv1*frV3YNhA{_wLqtX8z6(ushh@b*=_{&X6@oR zzk){3bekxql5j}p;g@4W@wMj?*U>km$1D!zQ1Rd-EW@4|mLCLR(!%oiwDD!S1b_1mm-gg{JM*b11 z7eYs`@=#dRzR+=itO&y%%Llt&u`KMF=oDXN(u)8l0Zw@ykEEHxTnzbq@cqLPn{=(u zw~r2~;XKml$&wzC6>Ggw^{IWlc^|@PC1-5wO&~%q_-OmXA&kvFjM}p##vW|q18?Fz z<@`PN^wkbnM=IxnVcpn;BRuNgx-`SzSMgzFr}AVRKkq#?qCTn8Qg! zFvkn=W8aw`e_vd8kuePR$D3+OwbO?mhziQ>Z&~fK(%s2vm`o|16Wl0n5>k>n@scWZ zLroRy1sj`u`zdO^FqLo|@(q}jIs(N<(CO$46Cikd5f4pMk*36|EEzJ;3wHkZII*NS zVlbt;gF+pSKBzZVirB>=E4D16DO$ErclQty|9}SK%zL3 zRK+K_a=AREqZ+Z@HmX$mXTy>q!Pu4K$_*||)vNNLiV*d+>j0R?xN~bDlm)P1JaM5; z$)ScH?=8@DVW45T9fY*LgB@I^06P^#_$c9j5ywmO|4e4;&QLLUWH=7F;Kp74`Z`1y zFYKM)*nFER+o5-cM@Ed-4oXQMK2}T z4g}?KTC=xIm+pLhZ$D|tdEMu@0()%%xK9i@VLy$vRrRDIOjX7hDym?%-Zd(imLX|p zVc3PZu~FeJBNT1^22C1c9*Vr4hBX9^WO!f|2Hz|e|%l!%pEM$s~%SdAi zlSvPYNy)PXthSl*nyuMev>OFt88W^8Q%S_Kotai!QmeFK_B6BQJIn2PwY6s07x)f& zgEMP_Bl080hyeJV*^EG&J%^`V-(kXKx*~h9t>YZ;Z`P@c>Dx!%VKvKRkbWy*YWb(aLNtI>I z*9KcH-8p|I3AMry1)<$JTw%SdrHWK2@JEugWnT!ClUy{|)jYcl$uD8+=nPWZDoiOQ z5FS-9Oa=*)`Lsz~5pfcKVe?{h$rqh?FWZ{vSOVjDa^Qj<*m(lLZDGPNbc)hUfaIL? zf3_0Wthy*BMIuZ>8OvT0zZx>0(zs-v zeg6muG?@Qd2p6$-Fm~~lws-nh9HF|tBHlj&f$V#5bz7ZPpRIi>`2M=aWq4>y76)@7 zhq6g>FFjWn^CZKXe#<+Zq1}4;=daI-GEa-sy3itU*v(A-r%l&u0>DmSz$bV^+{yut zFkNH^K$#rpE>4#-2dkamp|ce*&k|8e zXbNg!rZuNadwlS&OCcqCGYqJEO@%<5PiJGPwFo3$D0(H>=` zR_AkWVS1jN2LYm9hr1*QV@xEQM3UG59#Onx$|V_b|Iu-#4ccbVE2X<@2FPkGT> z)VW4`3K((47lln7D~EWejCALZgNG6XLM*-NhAeM_0Zc_jhh^nt}h(k>?an zvlaM9ia`wiUsDWKH)A*Rf94qfI~2l{ zls;YH*`0Avuc-y5!=Um#L%b{Rn{7mkr1qOxusCjaj~}nR!Te%n^T%k=2*x%d6bGbn zAQ$GrrouWTIddJTl#))0SP!0W1odJg47*mA^cZp3NLEjfv(Hb+uw$dfi%-ny%?+a`u3Dmo^JN2GRCk>1pewk+1v!ER8A!Ix zw=1!vDWLDI>81$dRVUasLr){{Pxs9inoTpA-83g&`Azc+a~Mil2tNCe@Ej|Sf1+x# z;xe8tBz&t&7aNH>0v zJIQ5m__d77)k(4E^sXVf0H)w(lP&5eM<#4VhH_ zrnR;zla30s13 zmw-Gb85U!9kd{XAV9v%$kn3tulD$!F3r2j?1OSi1^pa0evRGC2*{`mYirwXj`k+p? zCXKl*eH8TPS1}z>l2;pItyBSM8WRcV83prHRj4aD76ttTzA_6Q1q^~Bzs%j}ied^k z6Dy}W5r&tq5(=47wL8?jKQhpw3TZEJHr=dvzdb|&($PZG**WNV{L zv|LQg8KS3;Mk^@cn{4iu4noO;(xG&)k_uvcoz|%t))aic(D7?(0&Bh|lnxkEF-;?O z{j=>lyY-agKlA%O{j1lRu%GCiJD3NO4bpv)jC^j-Cs~@)lG}UoVL^2~$C_80HeYew zfaNI1Sy30lf}?9T3&({uO=ym}{KX2F&peMgSsRoohlI_yk$GV)goBExOo%uKwO+k8 zEuZO4o5{nouf1H6+YVnR>1$ciVka0B^t`Ln|CesS&FVcRmEnP0qL|Mq`vif7L#ZTvl9X8#pS_LPaZZ1t6 zXix$NkK@#f`09`-l?EKIhsi9rxJQW3S+8!j=-^|OhQ9{j$hcRaq@o7NvL~Wn~4Lk3$ObLgb_HJobMK^|2pic!1qh zCmb%w=78rDV)p8CBX^D$f(X&fLbEFM+y)6@bUD`p_<01VoZB>Xo#trderGZ(LZ^Kb zqz&_5ex5R$$aG@gq;wjUX}ivCY{s38E`mpv;&iL#inBSv)at{z4wz-Nq_8hK8Gyhe zB8S!FQM;=(95q0g17wbWAKI_>`si5Ps6Q~~6Nx*#ET?ZZmq99byAvqvL_^BW`{;qc}2G62o`z z0hE^(IP;w*(VDtGMeAa*)15alaq-eTVeoa=cA4R`<7}TD7x0x!XJr7z#R?Jf zBZC$CL-@SR};(34&Gz}h&pD}Gig@J1PEAVb{^aMw&#}u z<$eUQZS5C#bHB^tM()y|&?`#s`eU|SrXJ)_Jy&!*P6=;m_46SOQlLnIk0quPHb0!Y zHDf(AO-^)G!wojAOVspENtP~P%Cr)-4k6uK$6LcQ)hC}O>ruqExb&Dn<;YpTi^vAo zT=2I)x8*a~880>ar%e1;@4qm95*3?`9|t8?YTVe0S8EjS5*b%-qCoOCHN2#gTG~;X z=9dMo>zd+H_M*+|E*B>(2}4(zz!GTDxSn*spKB#+UF-0@^@ahxm(G%hT#lJC0Z2+wfT&uQa1drCwcYwA`y z!DX*$q(Nj`I{Na%#qG86QFwLm7ad{!)*Ui2z^j<_Hlj8PM@}`2GJO|#Xs#o!wDRQ% z(Qw(4BlhyRc5|Vd@oRb+k_qe^(yzccM}LC)VTAh@(x;3D=#&)k(P3cCN?EyFY@( z!#)D&_O4brlFj`}$9z{PYTHuA&HadW=SX-1k8#`=9W6J=unZObtv9fI!Sr1Hppugd zNz5^hIL9Es^=_98nKB@o3MjAEFQ2maGs>w?^7l9QF9|i^NcL(sq+nkFO|ci0XMM~* zN#3PXDpdfnU_op_clui!_<2Q7T%&hI$jnI&5T>s1d=+zZhp-z!1V?;U#|T>}lx`8K z^ZMsjiEsp5e@|=Gb5|fJ+Dm+*@rDm+k@`tIM#>ik5ebikj3pfSNkevrC1D^j6S3JZ z<+)hFS>%fI_t<1F!{e^B4w?+dHUoIunDLV`U+GcNBh>(>)?sTd`nFS7hQ`Rk)3kh^ zE8NW!-P0~BT=~MWj&*)$59;d+#2V z!7CglYcCtMH;R={!N33e%-*RtAHGodVK*u)&fj}410$q-gOV{^2}B&Szq4PjV*ZVmz5+Xgi{qm>q}tws(n2#cnhWXzSnkH7CTz z$XbM)P!Zknwig?8MI}Vj0a%}K``QK7c-2!jt7XQa@v$oP&4ASGiZA3Wws+x8HK+%G zOhx;RWU5NA@x)we;(sc0TXB$gkRidq*x>#v>qgY^|6=T&zcYc7HQb=%FSebIZQHhO z+r}509jl{`ZGEwgj%~A(PI71Fth?5nS$EwzzwG)4YQ4Lv_I~PhG`IODY;5xHO|j;e zD!vBBC!-83oSa_0s*Rch-O^w4Wz8yz_zdB6PN)bDhF3GoRQ;5ioUJKG>iasUJ-ySo zGPS%raDRw`k$?IfLPzDKm{{`sqdD$<$aA~y;0pSBA2kBI-haoTLWv=(CJB`RHea!; zcTF>7UMSX7wt9eAJ1(k#B^ma=9NhsM3)2i;Gq{$v0r71!X;R)!f7d*vFL6)GSt7JC zT%Z<}$2i8Cww4TP9j~pCPB1|nysaSTvxp?cTNCdj0?faFC;vYkpQ2g{TecM&%O~9{ zK&0DHt+%o)bwPP4UCTYeWcqP3M zCk#*s5K^SMgeCK7!Y=w^WHA@;+ar7cIdYO!fSgxC&uJ6@P8QksOxF~$;}V)&u~b#nk|TRvpMjilIOeqMVqJPm`FXO3-u zMATN*S}EJKg{PzZr??$d&nzF%Wl-YSUd&HSqdg#C8|PAQWZQyU`<-hNe0%~(zlRR> zYm6-%ztxhBBBidOuda~#O0V+)QXy{=hV3LT<|tXs>sY#s_M1jZAoe71xD829 zXxsOrdm(8R8eJrFr~^9Ev23Lhyf>g%#Wafe&m!yjIxZ0duNq9JjFm0v?d4)Xe+}>O zS_^U>UXXQ^rlFwQRE7kybE;#9-D3jlo6_uenN*6PnAy|!U`>eyC0GaE2fnpB3I*f% zUYE*`B>(F|@r7=y2oHNO)fj0CU}W=&Nu?{I_(I=5U7!_Yk7B}A_``?*=r=eza3|2B zk1#{5XP0#1Vkr8e-`h9KPui(P9KC^pnK*)@*l6591%9%NQ9-i?t>>5gBu6>44XUlo zz}o`f0rXHGs*Pd!&RDLuK};IJ>lCT{4u>dxa<}5s7WV+%-h=FGH=0;Cnqb`!3X|^> zuO;CYY*2@!+JbVne^5P#)m{AhdoXyN^+OxNf<*#g84c2~)*f0N+oQSO!}V}U#JN!$ z*)hi6k$sj(%$zkWAyinAB%h6$QXv1bQs2?JUnBowCapso5nK*NO^Ev13i{vTvNRCQ znf>j5zWx7~&7uEVIQZ|fVxqUGi4o|e2g{%WO*XZctL@^T(!^svqe&Gyfd+*7no6`MFP|z2oF_I-7 z90g^mIlwq+uj+~&qpL=*ww6~{&E^--;=J}3PC}#U>G4W-&rh~ez2}Suhtq_~{s{sz zX-Pa>-^S?@TNsBJyqe33I*noV8epS&RjuBX@CYuPl||exIj|$pzRcm~x+VSlK!C)i z;1gi+772$+Ahd2>uF66WSeBjZpx}UdAkQfIBhkAm-%LB6Y?Y7dfM5hyHA9;lD-mzx z8x^HL{Q-oaN!C?)%a*!<0ZrX1fx!;5KB$QhIA!M;9m9g*mY$l{f-`^H-dkd4$kLZQ zAmLBDml8MY{KMo8r|`bBk0*>0#M%v|5+AsSsi!6YYW)VnG^f;0X9=?dtOv@8^Azv^ z%hdwO5x3A2m=EYWx@W1QAs&(PL2s$Sbf&@*s84bP$M8cr*RYW00a_q zs~Ij9wyY|8f<-l<9?-wbYVx{SbdT1iHb$2~yvE&1$dBy%g!;pk$AT5`EkFyh?v3;W zhA?&;ZJD9TP^`Djob2<>bogMuqe7ax;SfCv)d>LQXI(^U9Jr;Z=Ht~&QgF1IHVn{N zf~~TI5Vb2ckCY`0X=&4@zB0)6CEZUd&%~gI1eivdY*CM|hb7EjG4wlL3bylm+yh{BR3^p2$J+ZfxY0=9Lv5fxg{h5AKa&ZxhkKgpjd+waUz%4d#(a4@y=yX{)Wgc%Y{EJ2lUy+i znbE-*fCpfF20!{`uDyr$!{EZg`X>>Ow<${8E%smr16-24aIQym5_SPl+yIRw)1T$) zQeARiQSFlfsz>dr!9Nt8lZKQZ9N@0& z$bgp8JM?^nU3Sj9&|)pI;lJy%drd}TCt7!ue+(lInZ9BF-IkM77fM8FOiGFfr9n*$W>NTQRq6Gb>lCtscKDIf zAhszuw1qUErCqF|(JGyxNNlqaTE8n65w7D~8|&VPF^}TfXgye%!h8waz4f2v(AUgPVWx`l7BTW)}ZzaFecSr?w%9ArR{j9xsX(DSVo`K=FjIRGOq}ZyiQj z69!4)DNPeF-dYdUc$oYxJZRZnSC%hz=Fu)lz1&M{kaO+$ZrIPm<7M-u;q&9Bt`A(k z1XYq40xgtP^wdgP$5W#LFZ;~8%Al48QG*L{+67}v6Ta5MF1A&7C8POD<5w|DxJOfQ z4{&v4)swPDdyn%@TfSEF~DL^J>ul=Ej#LqPd7vUUUv^MFJxRv zgHR@2o#kfvo@%_!jVj&mi7%#vj0X`LR90F0K`kBsRBMRcUDLuv3OX^Z;u1qnl zW&hAr#dTKkmkxJYH=sRzP&(_+4O3mZD{Bib!ks8%oHok1F`QvSU>7#H@6+|{Y1uVn z)RN!Un4ZP*GJ_iT8HkT{d)N=)s^Qkeu~p$Mo!ao~BSYWkxJH2%H%1bQd|MRmf=g@? zN*Z{U11N&j^E%_dBy6zWqI0w?_rnud+|1VK7@Od!&ZHMZwTxiZdDUpAJ0QAd1E^vu z!i#GNmBUW6N}`DAi;7C#Zz{IHj4@Lz7%s^AE$4PPM6g8Dz{SKV{mDjbp)v(6&;bVa zkKb5d9+gL|vMiIRm7VHRw#xLsky@S!)6rC^X|=oP4w3_tubC*PSmpKW!!R9kGVK)q zaho_<554*nZN?0D(f%NP**R}^jveZ$bEYJ^GBe%tsjned%W{veOjE4wIEg}9VnsvIZ{DEi zbdk0)K7g8*88u$^qH(UPK>vI~h4(E#vmk-PUZOivWajZ&v;j6eae{}5JsF}%7a+Dr zrUb#ji&A&n;2on}$q+6z)P+0w*4L|_jlUCKyX@2jtuE$p60lvVr&NMI^O7`n9(I-o0g8 zlHbeMfo>hMTA-_1VE&JDqI>lqh7-icW2X(h6cG=&g|cHFL1r@Sd~ z=2QIMtDcJjN*YEy+%x-nW|v2XJRj4r5Da!)j3I@7@yzC4L5NA_xA4eI@f>;^9AzNn zy97>kES)|N9f3Cq=}rjDeT@5#X=)(Pm>98na2SCYZ)~a!2=~^-nzP{uM&;)cggY=b zda0lttAE|T$=yLFIL;3^Z4Y`S5`x`j4jfW4H?Us`?|bj~qez*D?H}!aRQABIhj2yt zTs+glW3CWceU{|G{m%#@4T$-(tZU8jjJ)Z)Se>4EPOITtBV|T62TMLCl;&R@G{!sB zmBKx!&C1~u{-Q<^=pU{WC~=O1{I+9i$%gWgJ2YrHcY!QJ%K8>6H=0#|P*$8c8mhX> zkdb)nL09-K;+rc`(U0hbsm-$1M70g{rB|4xUY&($wF#w4lG%?RhmfvlxA3i%E%MSF zr(}7Um&Td0yh=th(C2A@FdBZ{um7;hdSS0LUxEY!qk{jx$)Bc)tBr}Ny~V$?omZ|K z;ur!KiPWCzy7G9^e`qNkj_P0+P)HOFVr-)O6$&{SoBQJ~FW1h;QlwPT?jiV*r-E@` z2OuAdaIS0Zhece_*zy}RyS!!vUM_pQp7sm+KB4Oqh}on`DoK8l#q$7DE<8NE`Uwki z49m-r1!j_5rvwFDpH7ZLDmTg&Dd}ciJJX@2$V*6m znS#@bx!_$FTD|FW5pfC3=w&!1BMId~KxwIW z#}R-ge1c8V-1IWcpfrFvi!@7H3Aoy|=ka^R=AwJ~Eb$TK2QKW$ayG$A1i~BXoOqpXdk)ZB6}*e2 znQSU>*<;FhhrGR9&()J{rieFHax_2tAcZQTbhEL@C->fBo2W$qh=0;1S#>$%vRLHl zTeK}M^IIsnsf}@^ra~L}d$z0#@LD2-P@J;wGsqW_$Ep3@FjA=b+lVTCht}|&vNiC) zSqV5K@6NVpoiKuFKmTRm`)k2pbP6k`FYH6McM z%c6zKxrR;}yrEUdCS4Sn3{i$fzUo6*)gy*g_eJ5vwlI~n@J(zR@|Y)!VYx{YBe<-8_5b{u=v9EnRWtvBH$s<`)juMyE;wUQP^w`b zq8nEAiPD@ULq)`+7wr$sV2FtA~4 zFffMy(gb=qn*HZCI~y;2E!@D741pB>=*%^nLE zXiU6yCg-j?nRQf^tuwUWoeWdRxUY`5a*$KK@B+K@FCGLe?d^{S3LZVl=TW2)vgj-x z{2ybM05?rl8@vqT%1~p;*%_;;@d$RC9KvQ$be4t80u|!4wMsp^Tyb6lVA%30?q}gR zRlzJ?^jMUM#XJ673@7F!hkgj%xN>JrAtxtWNg>0fa+8^FAJOcy*ClFp?Lxh%tt@tP z+}$jio|auZ^eXKw{)#wf2FE@;#(0yiSl1ft%y4z3wxye;=0e?ay&^qVQsBVU%$Ygr z1wMQDya?v_+Fr6ctw3%zS7qcvG2*FcTMQB6Drs@yk z$epHhVd#J}M)vR*fD5gCbi85q3hM1USr8JV%f^=4j&iddE^+T0g8NuB9jLX_a`!sER3{2))|p z2cD-HvVsM!l)KGMMa+heyBE=bYdH{kH>~uhstRf`VLEuJsji3_E*d5qWq#uo#os0+ zP_^@n>c%dVT?I2nb(J`p5@jk)lW{o7v9s`yoan%c^H1XLguYcfG4a0Ww3ucfa>%j4 z!4Z^CbtpQX!iRFDUTb6tBd_ed794sxGcJw};sY5S1nX=h?gu!*pa~`dUP4sDT(`G3 zYI74rxRdM4%*(3_*A5zew&+X`o?u*wzVxh~+usrdpTl?Oz8DyGfAt~IQzmhjCypY8 zs!q{?8J=K*Qayxhb!l`qmYtmeH*I?SE-7Nc5a<$~T~Bbww<>KEzNb+#P!?2vhZ`e6 zv4TnOokD8LU_+2-^IH9w=s2^M@+xRDmCf>OQ*6gDI~_7J5XeH3#<>B` z#VXR?3U4B~2KB=-ojot1>dL<1#Jzs|6-X&}0NdnjFUV74Mdh&C8Z}{19sFmwYPz~q ziIo%9R$h_Afsu#SJ@&MLMi)ewPI0-2!E%k`qibMGctp<>A-ll8Nk@AcVFERovaUKl zDGRn%Ow;bV>}iZ{RU&f|ED^oFaC7V*;W=Z>0Ob>_mL))@Z^{kZ!k4#*wOl6G7x5bn zt;)biBaA!&qS?#D_#xs2R;)p&p8H!za>44QHC=+uCc2p3Uda!?5(`jvuzGLn&{&$h zxj-gIAM*!8S8%hsCmB>vCb_-wf}*ucAjem&@ocdF#V-sa3(acUfJS95_=A(%+~lnB zQ~Kmj@jz+SxzFdYv@BQisTJJgxIpcJkuR&u-FiZ==SGiAb;+~8<}#p%-t#mJP04Z6 z(X3?;+=vV_OM0RcqN@?hbNSzNJ#Rc_2{*&+D0b$^KF>kt(#OBN-Zx&HAWTw*kmE)c zQlIDUp=Y_0vuB5%YJe4d(=)2xd9b8-jJDanl6Y#tci8+P%iRKx56?7={N3*N)8DVd zQG~YN3U}T7e+{?@`n1pm2IbFUCfqP3CAKk+@8o|OWqzG7#C%Q4_(2#QKFIyWqfkQp zy5Q@Y5%w{h@Xnnr)bB~w=G=oDr>v|;RFai{LZiGf-XBcO#7y21HaLk@&Zmp1%8dO` zjk82Q9H#b$SLy;+nGCTfpY=p08I%{j%XC?Qiqb_gpTZo|y46sHJI+%iL4=U5fgI!Z zq2do-3-m3cHmD|G@;$cm=ZmO2Y2ha2OcTp>4Y5Y}1HIa-6V96+S*0Rvw#q~QYoOM( z-)}Cs2dQ5dwI1k{4vt0lGq_;5%>axm1G0A!{no4fd8;_JjTe<}R=R68{etR| zr0gbHTW8@QF}m1x&3&m?7LL~;jhPkorV#z&6NAU6H}TKh{uEB5)GzD08;s`sjF(n% zx3ylmZ;x>7&40389xEvH&4M@wKJC5XZZ1+I&Ivqe_}904-+mjmC?GByaYyS7OXm0@ zc`e2*b?C3&jOJj`l{p1 ze~)h`*1G%i4e<*D@t$3GeWSMQV5O1Ps|jJVpK)%MtUUtCGuPq{322t0aYdxrVm7ru z+Toe<$3MIB5ngRqLojIM1Rqe^R-!#{n%PR0q zpo*$z@FHTyp5w$V!V5;8cQUJCD!4mp-2wK(F!r6?Us>qOzFWKdx2Svez_6W#jst$s z4fU^lI=x%Jzac3IAEPTlxp<${Ah7`SnwX%og*Ft`2Cdg=uM+r zE!=K+xdB)utp6kjF0q1DIibM7-Vpw4u_fl@=;m(n@4x!eHFezZB~U&`8X5Gqx$kgO z!CXR9Uo}NTL&ms=8VPg9W=Kazq=M7glR0&6Vz$zeYn{X2ku=ngqNM~>pfu`fQfL#n zV!Q&ugMy8cimkkA(&_2eRMVF>wjQ2V1q6B?R>S=IAh@HQ&Q&6WQI^qcl_#CmxfV9c?`SoJ!cDe76BT)=1?-I`HA|`a+EfJgk0oYLXU{MwD-4n zoK9MB1sA%K@?*g8zIyn%{mc||v&Sgs0Z#}K0~~wV2^uZ3i4akyGguw~kw%Jx?ibGl z`P>#8l{EnxK|aatH4C_FmPs2jYteZlrYLrT?5QNO*lG6MD+)S9v6Ap3LKGZ0%h=qh z*pPejHCTcScfxnva}Cv3u#<27GT^sqVBFuq@-40J!KpZ6%R5j9QS#HB;ZbebCFu5< zX-M+XQOF@oRfYLy=k~?K*&0c|Gl09|m7o>lkdw(Qyiw1SP|3O`TDgb05|1LFQMn|z zaz%_7p(2xMrktO4NJITuz4S4}K7;9I7MF?RX52QOerV;Q zGm9N8X8e2@P|L%I>2OnNFQahQFImZKS}LIdhSc!Z%O63-!bZgfD|6OtawlMF zt?WTVcF}@mP+HH~Trg}snOI*j8p#Y`-BwA*CuoF?$4W|WXcko-E~}jA-z3oB70fVv zZ)YK2WI<(%OK9Mw!;kTPPFvQLWU(=|_xpfv}cmrpeL zjO7MdQz$vd?Dg`pE~!GVR)ncL@v;%TeZeUXYhxZ{n)I-ySJ38^@%<#n%!{$|v0!ow z;3@iLGCDn5M0zGw$Tiy1 zd1rOw3n3I7mNA5k5w&w0z(8vvQQ%_?3OL&ry{R!dDt9i|O@*w2bP7q8@eqdkxs6D~d0rBe6QBJ2lLBo8`XhE$_ zKNsQ}>;`x8=u5557#HHY_sEDdQJcY{&5rx;5!}({^X!p;#rF871eEDm@|OnYAY!=(uRGNydvcmf&V}h!nbx9hiD#}LA(%5wKqsF&2@pMN zkUapiQ*v`q!e0tI+^;>Z4|TM!eNdPAM7?jd=0fsG=IvWSOb_K5n?K_-u$1YRa`<)P z>{@wL@Ww6bZ?EU`)AN5kd48*YRQw461~v`-e-U7J8%Ga|e{LZ5*=oXjk5!V&-SB({DV)1ixpE=`#{ zV~mC?`ixc^^PPV^BcpsAHrl{~rcPWtA)(NoE4ct%lc#T=$HcpfFZF!8%=1>jAF>8A~U-(bzXs-pkb(jKzBN;~PZW}F!L4VE^ zE-mu7cZ=`c35w65SDp2T^c^QwoCca1n>Vc&3zadQJDD;%D?!m zzFY%Fon>Z?sxWYT5kk$DRF6Z*83w~~n`uq%zOJS)c~`E)xTBFQW-)3vE*ecKtn0B$ zRIX*Z9CLuk7bnc?r*yY01T-k3WdXGxT9OFyNrFhre`Y#GE4rIaz4CzmCGMG1fai1I zwjTZM7Bl$A&;Sg<%WG)&YzdX=Ux-T4z$+MPpc>H-^Xv}07+6@Q!#=oPqMtDT5P1Rh za6S~(;tE+lvv2ejPf&Wk9@2^CTHC}kPUGcd#J-U~L-1QMT+sYVFiAux2K++OKLR_& z*eYXCi*d97UfvjXFS*Y-`CH}n?QHQW-3vE*kt>7`MizMjM>UK-F)sQlT7jQ>Zz$NwYWY5xE7 zUD3n--#-qi8~hJemdU^%fq~hF`3FtcSOpHb*+#t>PA1|TEY(}H0o7H~l_Ogd)E}L0 z>&O2WD|2X|U*KHd?C`thvAW*O`yZMH*rEa;O$VlfnC*e0)OLz%havm5kLQYum?5+2 zuGt=1E>?D4My`P*X;Gr%Njxj_o|7-QoLICnEuy@5O@u`ErblA}o`gGywlj)PQGO~e zcO<_lxT%rOL@eN6tFK2FLAG5|Y%m;qWy)C9n7{62lojl7EFC%YpvKeI4-;NY)> zUwAW$i{!lN!wQ?SW@M2Y?I7lyq1R-or`y|94AjJ~dd!O80o}C)+Ef7B>M1m+zS6=w z)BY=E#^|>M+75f}0VeA0m1fDrQLo+;4O@}kE^)i+smFI436G)*msMU)APXr|?#I0_ z(58HXy#9INB+ul&cwq6kWBf@(UH08iw?vz2#Ql5O!8&?Nf4pwWsbHqOXUUG3Pa)(f ztp~)`CyQG~j>jIctYQ=Mz87?&$=er5hLCq2a@aZB-eS=!-FDW11$BupZ=BlWcLlK5 z%y>&6&etH~mu1N+tXQqfURYLNJ_qnprdm;J7EH6!oylG%F$l$1c4X$nU7~<4|A29e z`pqh2kLY@l@FiH2qx2Nn9RANwVq==zd9sMe$0@XL+CLjJy~XF%@uNn)O`}-7W30r> zN@&n<2HA&G%Vhg+PbBdpcM6eJVwUeIU{&g%M$|^Bv zm&HkL{2BBrodpzkEt74%P?l^SH=9}DBWP0~A^2eM>UV`VJa$oD;E#MA`HJZ~A!_x; zpYQ1oHoDlFK`Bbvq-W`@d@9wj5aG}ZR19;%gc~B1%7rebZ7fI?Sy`s&(r=KB^!6e2 zJ`|LsHw&4;KeCJR&LAI)c(?d)BbPLz>=KW;ay)DvWY@%vx7DY`NMmzASsHy(nn-X+O!kf$nSipKxZbZs)TdS`ca-awLsA z5{{RqwE((}80i!DOW>pzj$2b1hNpa;wik+G46BB7ww{4vBRFVi=1>R9I9QQNlZEV4 zg>%i9zT5`VGsGjcuT@v`Nvg1_tcu2>gECO%&e0uKQ5CF5G)l-e!6N6rvSMN$#*y;0R@$ERg)xvht2s;n7R(Y^-|3kk_^NS zeU(C2GM6ANCpVA=RO}|RXp&MbmGwga`5rJdr+xTV17j=6dcfnauopyuNwG3xdQ3!> zreToomcMeT0e7&BMzPBmk3hGy)5Pu7@+?^;gH zEo#PXt8h?TMiM-T->qW4Dip$zjbey%o?wTxCq=ity=)ihI;H5R9}})6bWsm>ZZ~}K z_IKF~vs$M%+cK?XJ5C+Nhx=A8_u2i2S{5R9hK2jLuQL_ED8r2~zCMnvn;O(>cRx={ zZ@>(*M-tQc`jNjGPTG-ed$ens%6Wp!NzAhIqwz+GyWZqr=h7@GmwMB0sjA+cw$dc; z!r}v8DuE@It_?F^3omVAQyi{mK7^-Qat@O7`x;2U_xpKWw6N}t3to2!r1|0sQf|gN z^mF3mWtl&8*R57rc344iY^WX{5kfHfg`jJ|>vyD)S$y7nj%*H;5~d{Sa@L!EQ*VV- zoWR0g@{ENmtU$YVsgj2xa@lq!G|Kt}R?34CnAhZP;stM6@w*mTryAK-DRx01Vxz!y z2v^sxYZW9--r4SJ(7coC-x}n(BcjetvCQ{>#Yn|S0s|i31o}-s_VmEm@zp^yVxNd@ zJy!^xDW2bAM8oLzstMdeGo2lQur?*Jo)>r`n}MC0_@HkN3zc|&rzYhxs@WQs9~`~AVgs1<>` z5z1XZiifp9CDBm+z!9c>tov6?C7BT!^IJe?MR>L_`zO@@R+OkBUTpb)C`u3Pf4wEe zOx*sLmBWSVqVhi^gkcE?ri+>G3y%4dYyU%3P6M7+QACP%=9P(|0TEc5T1xBmlTBAD z=oL&b?I1su9b*F{Yl^~;v+F&;&+{c$C@AOy+8KsvA4ik|SP^Fu$E@FQ*=fqbu{f{Z zwW6F|<%pMkK;2oBrao%W*2)gFju!{&^GyA&uSTvZyPq&}oBd0jKuQ!wVTQpfMm5#8 zu1u()6JXH5;h^T_zS@YpPu!u7R*&FT)NVlkaOCh*QiW5cqh+Ys){>p0bjJDbm$AGrg*bHoQ{FELQkG z3bqr|td!&-AK))Y?QRwgzg2Q()j5SN2G%Jc8?_$CGUWsWMMx|wqe*sqP@_kt=q1oE z1F-C}?aH=KVA)*OfoOL@LTBYwZi^ekyit)HVKgwwTDwaNqSmMvOKT)n?k6FHTXhxy z^59m6@U18$ndq1HRBPxxX*~1{hnbt3Czp(1^mUJPqxRB$K$EqMqqwp6{VJhU$&GGw zw7sawtFqKusuIcm_T9<8s&!_ym-U>}J(AYDLWU&E7e*4dUhYGhHSX-WR|Qb;7U&=+ zaB#bD$M4IVW#Ss-+BK~h`jQZ;TH$@Aa#;Dsnr-3vN=;+~cuAto zODHA0NBSO^v4-cd&q8iJF$vQ~@#Ny*WC^|8?ALty_xJj0$tn_;Jz$~SIR6wwaPT(y z0=V30_ggJb%es2e=)qr!MJ&uG;~GQKs8d!NkJN{V$Cs{|HYG61Zt=&Bv5^ekAfm6F zs9&nT<2-)WW3{yhKFUIU)X-^b=f$Y%5FU*hOm9D?&3PEJ|yAsQyIFS z-|F=zTFleR?v!TA)W##|pjrLSn!ayrPTq!;hFxR@uSxDspy|dCwqX(nODkQHBa5fV zXdttuxJ#>$kbc)6();)i8-<>GqQw_*FtA?8{|dmWT3A}RS~!~hTc;M0vhKJji~;TS9X_{+H|M`=lHI^A|Q@Bc&x?h7DO+>_*%LUqt)=bI_W{tNIM5UBUG#0U@ps^h{uk5k@t4Z#&6%~a`AU!Ds`a@l`PV9FAM5a0U*P#KI&>>AO51KS^;+F z{35rhXK9*G708vjfvVb|5M>6xF=aw&p+j9p&~fhXEp%Gw=#-+JtqayAX)`9f`P`*V znG=R5)!U<9!0_kemUKm?KSt8e8lEsxE!7=2Ga8_`<%rqtF(n z?a>OpdU8EagxU+ZLer}0Y6m^+@jrMDZCL9W8t0dewP#_~GHrSIq_|K13ZFcoYW9W_ zHgSwWc?;V(mPAK>f7NB{#cFk-J8D?qe*X{0HqIJpMdW|fh-MH=i zz4=uKxfbIktMV-qtS!nkjro;b0Bz3Hm7P(}*P9eQk9?RQB=Gm z_?NhTyahln&-+(4XL1yj@8XTYv0YzLMNG1J1xr*w!3vc`O9fhnI%ADqHbwMB02JAU z8#SqzRxv8N)6b|Q?SWl*IcP}y{bpNKj$F;kGg7)Gotw?!aGVue5gNKte&OU&nsrS&)5JhrtA#p-+&85Pm&T zyw#Q{G(b%&1N*qW zLPY_*Z|pI*X5^sGVS6)hY8%bsn0Zs)tuygVtX9#bMjhcHzgt5fYopFF`nX=dxem0U z{eUL;hPb^E$^p?j&Z^|gSq!P$Eb)GFKg@q;Ya^8bRFC6iM^tSj)@ZjILzP_x60ddD>#ct3z@%ezAphQOgu<=dgqv z*L2ryOeMtxYuo-!EcivF#?D|Q;BV1Wfd*zCmYEvgrYOZrsvuf3F^5zZpj|+o0n$Wv z;ASWzs^8r7ZUW_gnxf-QlEe+>oyaAnsQASqj^-b%y~XdvJDWl@XXp^m47;nIKGhnF zaqUHNEJU(lWI$$ljs#H~Mvm&vwoNT3wLh)KeNlRMsL&xZ+I;v|=NhDzz3WrUB19tO zWP|+vF?ug5&d$t@Q7lNn)zDA*4@}`QohIoB!t~K^1jZGu*yj-t8bcJ+Xp_+$n6OyS zdY9F9WQ3k?yA?SwhhAFhY~Peu=qKw?4RNGlv(Q{%Bgc>3cKAww1baWIr(XsBx>NZXDj z0en{X>In`gMf*S03GVmWs4B?EI*&wSz>;uY%9VusI93iE`;U|TXDyZJ}6pV9(pf?&8og$*WoWd@@{YO~;FcST4S3z1siohA}Ff=`9H3W!T4^88fq9kz%?YcI&oPWK&;1vtEg0Bg-Gf zo__1!c&6PLG?#b=o1JPRGu-`g2LIINdy7f5;2Iv}Ei=UP0GfIBL^eE5ms1$B(S@f; zt}v%?uN4cVv39hEOHtWxOkPChv^utYTA3g_7`u5jJ$suGxM01(3P+O`` zY5k=xRrPe5M9vruDZ7LJH~Q#O>?QF?YU=iNgc0FlsmrsY7(u7wjrP*Ys2HDIFZt`d z&}x&lUJT3W0L_P^Y=kookaAOpEv@-ZV=1^}f}c^@XzKkjy|rRdr+m?_N)u=~dsvDt zrk6$L=jksG`_)6tCITNDOV|t9DfT-K#c^q{iaL)9)$puFMP90i`0yi-i4Nvbo0`c% zw*zE}TTBHRjDge#U^k)`Cl_j**!p~w?iboIbt5z8$L$aiO~wp_leFBaNrxBUB^v?LIa;b1v5^&6^T7iJO6$;N;l`)YQTa-Q$YrBbjdoJ9|Ei z-}vSW?T4>|E&3f`pRA^;#^4pt5ONALQ_^zu!kc`W&bSwH7?CNgC(A;^6irW_v$%zH z8n}1y3VpR7>E{*#OAti|BTbiQ;|6A0=|?k}r>3d{>7S~}&i?b~p(w^Zsppz-bC!Rs z_0CADjR05q4{z>NowTEe^*qpMs>T!FDP#SgiHC}@HXp9>YZv8ozm&;O#$GG4hnOZG z;pgAK|Gqy9SKWsI=6(bq3EM6RZ#P4!rX!n=VZm-mEILLfH5ok8JJ{d$h1%(lz4VC# z30QtC>EJhM%icl$oio>uA>JEpepkg}`Fc*hY%4&Uc0HMmdVZ+By=!&=RY0o0b`pbM z<#~BJ{sS=(hNsTN{kU zq?J^CNIGfwoJyOcx;BM=N8p$>qJTZm?+wnNa6MJLw%}<7&-a1Jt#P5aN7Y9cq4&3C znC9`7hxE|P3bqK>)Bif$t}CbO4~Fco%|RAlnLV+l*ZN`hh>h3IAv8vX0}2%>I`Tyl z#p~5VK&P#oXj7Io+TV2DntFMX{Q(iJbU&;+W%zMi&1t1v5aA;A1NKj6duT$eBeO;7 zm%9G%s_j0~kcdXlmR#J~4G_uLJ_L1;qsz`MtqbJ0t=(`<+u$ zyWLRNa*ORQ;Lb*^kddW9KawrcTh_X{;~j80K@s-7wv(2XrzfHJ1QE#2N&19Rlm^n8 zWFa2z7Qe$O91C-l*b$#{1X{GmR%|li2RRNm z$rr)#CE$8~vs#1WpH*Ds{~(xYl=BYhkvN+Yd-bdWqqC9ijWbg1)0Lt;&}yco>R5i| z*aiv{u>C^+fjM|d^3V6{XbPEk!ugmp&tR6A^Uox#`uU$X~^(h^nGAD}}sXIF?)uCLIG@W&+fZEcI__ znvy+L(aP3ss#GyncUE*OD&@eKzJ4u($&mzU+^!%i(1{-~9SFI`c#EM8dw#MYa zOhVa%LNAEW@yvgk(G?iJ=x^1Yj3A~{tJ7(6>1LvwQ<~9DIS0P zRmyI1GYl@+D!1&SRj;DxI7E+2P5gs!W?)))bDWEOQsOlw08OJGFG$f_pz;wtXf!QO zPZ_5;1E+Uw$Z<9&d;{jr$A5)r})9*5Z9@j~6QpB(F_3jXI_GFth!;#**3#}a` zFa&JwNkPsphrSx+z8Kc4y`M=ys(%M^osr2V#BF9 z?5irfRT|m*mYMl=P)Z)Q2(V^L=ab8Uq!?(fR+B)TQgxL?Ry~8~w@E*G8j3eckO+5r zb!CU+oDo}5?C&n;*Q#1ICeza`JHn53tur2qZlcGh8F{WZoG0E7>hMLz$7ks?San{< zjGQkfPg|pR2M5o6fUYv|BFRmY`~>n`c@;jvZv05!b z2Srn!gvB2?i?1Ey4P$yB6!I|_8WTIf15OTufXD1Eew(KGCr9qIh&U+=r+`0v39>*= z2|u&zv8aVwSTZz%ff3F>gbB(S!f4M<_d_AJL8!dt7@oZh!Bhh8fZ;?bg6SK?V{^ky zD~~xtWc5*!$G_qvX@Hl<Y;%!n(zS zb5cp1;}X!k*rxd_l3Fjv_$~f+UZMix=y_Itq7u^RMf+kqVp|uJY4yG2a{|h^F+fYXlSoI_mTFNL>kfw>?EUhWPj8OgkJG@L0qDc0w-lYNo zW4*UQgJ7JT2JsKgEzIj6jIp)%^eyD83c#<9;K+n7ew1xxd3k%Q$;vD7@g^wf19BI7 z1wD$SilodGZ~(}@YUbz~Wyr)=Y$|yIES|frF3U^(@@@rX%5OW=+{QzDXhZuVXnJG=1g;PPQD4 zYD7L{pJihdnA}Hb6OIhlr>#g08E(S-5jo&-@*S3Eh=$8man>BBD8sQdI$Hd0(rvC= z#x?59LlEJ}6%DqLB2wm}IDk((b1+RFTJ2nC*0w^hh>knb)jD70DW6H2jipHDEjPT; zK33S;p>>_Lm4RHIG(#XUvhO7jnGr6rkTe9?R$>0N&sxiDo)C~?VlQWGwuYqCwaKBs zs#Jty+blHiOy}iSV&cVNqiK^OF>gg$Ll#Ca7Nfmp@k3i6G`tY27|jP0O64Mz^#|1U znmv==g(n2wwuZC8u%CCqo^2KrvB!F={%k!ztzu*KtQPHTl@bvntNc5Txu3%w(6Sw0 z*Q4vZcH%B1Cf8k&k>I+*J{$;Fw0AW-MHud}HHMFxmD>DT?|1|W{tH1n!L%lr*DEtpReho)G`UqM?(8OBkB&3~px>T> zpm-H<0s#cK$Wv$)k?fC#qvif7GH}&pmBl-Tn_f1%F|AUZR`uyV4Y%@US6pt|$^hDM zA2aBo@VWt*X5mMd^hhsng-`)0wp3>b9A&mP^3Hx}DY1_WyD7O!Cnx@tXAi@N8G4?$ zYmO-kpi3gp9z|`AMf>S!7uaR`_$DE%d8tq?Q$y!CgJ^eP7wV)>c9vDR>h+D~Zlz2^ zzT4sq4@L|Z;jvHJ%gr3M=)#-x1?>{s$29sg)%67{uP0u~n&U`sPmLbZm9l#w2d`i+ za3_=MCZyrg$yf-ccI3!Xrc)&#tQQ}ekPcFKrZ3zTXUu??jTl0_loH~5J@2=u`$bX` z1hEkQez8Wh$c?B4Wa-1kKSIedqQ=qJAlCxwrJt#>y(YnwAyVG-gx{4#(yN7#QDUuH zcGvN)0!Tvyih6ue{{BCVy<>1@UDqv~bZpzUZL4ED=_|Ia4zJiYJGPyUZQFLo{`!93 zAK!WII#ti9+O=1${des-));fGIc8ZDL=Wn$Zz<6w+(@85J@g3Vw|Zh?JZR}{hM812W)}sKw6+CW^^>k(s9lt_Q8C>h`=3DC!r2&YaEO5WVi&AX!fxgG~_y{&&Lp z7ehz+dzj0uXRsDGrB{?huuS(Sk2I1s2ocyp`CXjDA3NVN(j+3iT(ZaJ;-wn|fgoiu zH8)KMA|%=|st6;NgyX-iPCV$@lbpZwwnCg8E1_EOB-w#h!U7RfK%*E{O zi~%mDc7T6Xe*9%^xSD@iTWeHb>8w_T?OO2=Iq}H~dmV#T6iIv>X`66e(=w z3D5GG^cg649Kv`vvsfZ{Pjp{nV%oHM3<5?!FmR^RFL->>)pLEZK4JLP`AqN|;T?O3 z=NbK39UR#tOxsPp0e8ct)lSn&M~rKN*B!q~d7hx!lnBdqN)~9`nlj1!r6vur%sFAT z#!A;dF2Bh^pGbl?8tr7dvIaxxX`M^utaxA2uCY8>{+Rb3thr&*h~9@igb}P3oX;w* z;MifRnF+PwLa-Y}(h5ieW*W7!X2?yIOMAeuhtohDtIHV!cVAau*n%rIF56ACBE$`m z@cZbuRG3TlY#n=kDMRvYRB69;H19gWejMZ@U)?x0WN`ef4^D)@xZcqR(s(8vhuWqf z1~H*dMyz+0?D_iqcF56kGp6uJ9D`R*?zL~1)E~!*$%Nk*qo-}HG=kBDcDzXg$T82F zl1w0Ine)9J?BbHb7SN`kkVOG7AVXm3rgPg84s&X?ly>U$#O{;e zH+`3CWiH4mgE@RERl2rKphs?)@uL|dT=d#(l3(vwbjEpjsdZiD%Q0w+NZ@P{ za!a4lb6Mn6X}$C!r4WqV06RkoQI!cr5ePmCj!9fAR>&LOsr#!VanZsVgT)cLD`y&V zvu8CCs3BsH%>9?MyWIItiWh3%wRPNj)uxy|yFHbT+?1J?EzKp9yt|6lfm-gIA9j<( zgj}D~ML*J*F9HCKPL!W6j#5QpN_g$C827}6-(^GZUbU4tBQBFlMhfTRafaX|lkMgj zM>n^C8}jhZ(%Cdb0!2nGW7MLoO{W>k+bulNg>~&*>6nH>sn}{=_L4IlyN&z^k9att zbX}AMFz=FGl5tRAs#j;3u0&!dGuE~9VTkTz5@kDQ$+_IacW`{Z3&1Gz*CBGU8Y_lQ;HH?u;_gt{?AK`%F1n1UO>%)`_`q9>MW<>!7ild`r_ z5k!+kz8%O(8xl*c@sUr<&^e|&O%cv)i6u<$qFf3*{_cs_0yz(W{53l%D0~FUiWG9ae#Y^a-eKmux2_P|CvbKF@=CSI@7B8X~Wg1F5=c z7Q3RIAusv~OoaQWVtqY3LPP36LkDeXN1#w`3N$^q(QPtj^)S{qhRT+>=ZEc{QPxGc z-U7J=N&$O7pi0yCTOFzqdEPGhq~`F2=67<~nN{{~@o;j|z1hhhScaoNOl~+H4@_Cn zFYRG=`4UcL&oGWK^t{8V%Y2#kZ~FfQBIIJe16O}3&MZ6#2=D)c{1XE>xVSp~-^HJ; zCY}VkA0-ogu&u16NS==MFOFRXozqBgOA>#184(#+74*$u68g=Hp-qSgd-b33IfZo( z_yT^9K|-VZ*{kH_TE2exmH-Q zVi317lp{?`|D7FaGitAPmg>x}J^pr_f?;mK1i$qnyR$3wevY5qmUK03?WSYGpn7Vv zK?cC7Z|)Vc@IkXH;{@U}p?(lxw2UxWL4J6IV{F<@uBm8|=1+q^QjXo=Xq?AUvA2{Q zaqPXYY3X@fe1b*l8_v%p`K$OC3AS}x%Xe%sf?6f3aZ_;u5B&3~G9ozc%6ABW)rApZ z#6S{=ep*T9G32}PULNWrdNRhuk%Q4bv5tb(_7t|0bH7Gy+fRJXHoQtNQjL8N&0cR> zyB%L39+kVuvVWdDp9BdXFyKu4JK>`zV>OQijm+Kw5CH^HX_|0g zeoGS;5E1s+766U%LKEK~N?YuZDw01K1qh!Hk4P4jt4I?`5J`RtS1OSUWygvsR>`3} zR-4!-1Ku^dURj1O?8I$#YGD;Tm}j!H566qjm3jC>FlCwZ;M=m^Q;3?pDh-mRU{A$C zDc0xYo-NvKHC?qh!0+bbJ1d*LEJc~N#aLTYK%_&qdC_8C@T=4XssqOnF336NSsyyh z!nh4SI#z_$B@zQGI51-C6w=R*avp-O_B2%3;DM)@xR2xHO9i(SMW zN-ATAdOBP$dv+|tmz5=(cFKlP_m9OqWr*#XbM}5J|L+G!U3|4NHKp_Hd24iIbU%IlF#IoA zm#VXm@7ibfO}F2;D9I(%isTa469{Lw)+<2EN}x^#!I)naV})>=W}n%76ngOPIVA2t zS@_uB@m84FVRztOqeoU9*PPJ|=Sr6w0)hC%Qh?KNlGw$Fn$~y7KCzy_4?X8aN~&vV zu5xoF$NpLv1q~x-P?1+KUSO!CfiR5d511uDo9y(s_uMGPbS>}~m~LbC=q#s8{Z<(X zrnRD#HRL1)1#{w2rOu5j4jdt9FYki0E2uUDaRr2l{6C{LthC~k+1k(H^uk6VVj!lE z1=B9wACy3ZlD%_7Qc3)ap%EnS-9tzm}-9~C{{;8Ji=csXx11|!cDBfL=s+ei*U%M*mbAtBb5$y(-rjqFC;CA z4MPVcn!Wv35>4cj`Iz({`=$Rb6va*dqNUUSP%u#&dt>YWa6PyG3!nn~u{JjLa|5{-@``3i~~iQvYV~`3iL&el2zr7 z3_yDOKPGQmvzf(57C8W2yy8tCy(V4Wez^+sBg|B}WT`bYd4U^v)hg_2O*sJSKj|}? z5%~GxRGb2xia`g#{cARkimI)3?;@@)qelSjJU1CYa)^Eo{sjQiK`^O6QRdSP-c2*gG}=#( z0GuuceF$upS$k7z&t*9_ANofcdz@*P1-%&rLcsr1_0&7a3}pj- zAL$n5qbB|kah3Ew*;y#yFqT-N_O0-Se(tAArG5#NGX@3jNc!A4a|}7Hk5fGn0X5wV6sgDy$`ldjwb*i631C^4~F`pw+pSR>>xKQC}-+*Zx{YU zw%a(2Mufwc6qUR-L~-=30hz~e9dTKZDl_+;ag`SE=bFYSt6STk7iEf7Et#yd_lDuf z(jk7=oxWxe)99tox%2Mg&=6lw&R=#OOKqH#LWcw!=X<}%(MUJ6%*>!bJ;{a*G#4KT z7hc~aq=UOfmgoTyImN_RbfoX|X%pO(OX$trWkcILTxaSV#@I4b-Y5JL--@Nknm+9v zty$^)*6tK=#7ucm02XJTNfY^%FGGGt6Q*pJAN)XXr_ew9S(M*UDQEw~oayT#J6mxY zYa#ootOzQ&l~Sb!HO|zr;5^Wi0z5zN$yX#m^pgWrS%kn)WgsppiP$)N1Gk`W^WR-Z z$|_*wP5*+9Hyj8E%YW}W@_#c#veiVBLiYob+n_FSiN|$o{Sr>}x2wRU=~Y9WnS*Sk zG>Y$*PoSu@b8~jXiW6Fxe#pYKoURkf6B_%=*IHr7;r^H&PhDgmdtPPpxj%iie0@Ug zaF)Z|lN1n_lLGyKH0x#7ZmKPM*?fx(Rig;w3;LRD_*E8nI$OKyzRKHrP1d|(eYIAv z<9`eiFW?4`eL}4%Th4TP52`ceNn?Toiy}Zv;amFiHtHD)$K-pP$M6QYR5KUy2Q@?TklN3J^Gp^|Q z)%h1I6eNk7t1L9zr1;2Hi3C+{Sp)&trRQ63n(+H&_oE;(dcEeb)#a^}!7-t>TC-gY z9ghnv{Mq$xZkpJUUZx%0HnwY`{l~OJEFv%z2$ym1V{K{QpGvdoo<${KEg?f5pL0~Z z6!feT(l;0BaNY4T9HwZ~jIgghRC-;3b!caG#rBl(0mXZ=K7$TGwP(Pqj-PZ0i+ykx z^djysL%ZCD<8w|a5riR8qBU)0r>%GO@lA0cAN+BwMw;{%7wu6T#VWALxCePrYo~a$ zHtwvO6i}bS<%PjS!+_%&>AjBl3vhB0K4I=8UeVs(vSEiPCV&{@UjU&!e;fh)Rb$6R z+pcocjbT-_}4aKo_D&GwwC&M6(e5h(oKK+W(1DHPVKbb1~jh3#y zpwR|sem9qK4xG-WedqKVkLw+m!)%6cI67IF*)yA2A5yroLGQd!f%o23IZ%GnXTUpK zSbgj3+USxTTDm@0!92N zyw}Ulo)8>+840?gxx@)(N{V8;e}WXo`|(<2m1q-PiIVVz+K+Zi(ZyO^6xIo`J#eki z)st%FnX6Q96?L`5QCY7%iGoJ@H~K74)T`ruBzJ&L`_>cwC>Si5m3tmw{Tc8oV@{7F2zuuh~~AK0DVa@O6?s8z{ee-K7PrqJKYiCmED z{o^2lD#uiI@W~r$idhR>vR~ZQzTatjDLYvi6V)vc&iF`j`SyHmYXkob@)V3 z@E70$B(aWbq*4m9WR$*2>ccplo?vH4&&BKJ$f8l`4>9Uw_QYJTZHAG{#Vor2V!v-DVdHH2A84hDr;hITkN>T%XPru2S+n%IuxOC21O>Y8oSG$A zBXWM824`kwZS87w2V4@4dDd-a?m@*TY<9fM0?tqWFPUNIYoI(aW#4a`sjjCi=iIxj z)XkpHmk&h{_gmILM({ppTIl@D-FDMdRgL6KL`_>;I(2o|`JI@=xyuC~8f$K!Sz0F` zT-E?j|D0RrL$n^0m?6_geSU6!&%|bwDWA}6Z=JY~7P7~*|KF=ECaNJig#p`PQ$guRTpEr=Y3ln919 z1U5CwaE!efj`LJ%qmtBgXQhLr;f*r@tr)K=>V;X{fkikFPr}-#OR{+B-;;O0`v@*m zYW<;x9QTDlVwRd0ZIILCa&O}=idPjoX<_IE$&ecdORRMz5NVQ3VBzcV$=SF0`?O4*UZv*iS1~Yjel| z)anG|6a8unbS*Pfz9cv!K30QLb|Em=E3_XDPK{ze9Y3Ws+^9lt&DMm2yo8TX;nGmo z#X5gAo7tQr0A`wJxwg4iLn?^e(LqGP|Jei9BE*EqL0ck9wbGP{i)G zPatd0ju)R@tsU4IOI|6o7^Mj;gV=2(3E2@FNl*QmNU=k=RhTaszJZz@`HCf>rLCIX zV}F}shp|}DRG9enc)g3(5b$c%Q^tmRYynxGonWPO3Erw*RB8!n#Rq5Alv8&S(PVA? z9?yvB8;#mdK|RcIc7gJG^W$E28;{&Iu`^)$^aY~I%(6%JB8DpFw;xFvoZ%cBt-F2V zBta*!<;Z-W<_dTbdO}W`2hO7JuLE4@=r7pEhU;=&I|Ui8h!Q3D{(-m5#$ZPr&{r_h z@Y@4>Xei=Z@Qm7xLZ)=>(}VJo6XS4U2@&2HVX^qewpgvKFOIw*AyqC6bsXsM) z_&E*wL&WQWps0*6yR0zwviSk#$Rb`-@x$<=NRBwGlzJ>gniInq*1Z^aj)OW5v*#Tg zd^HZ2E^dTq@rr9E=1`C?&=D=!$C19`bn##P8T%}VuH~IYkBKxFByDbu<49UfuC)3LI3cPW`=ctRKO3&B7 zs-P--7s%hCKtRIL{~x<${pa=8Y|V9V^h2C41eggkoUjWcVQ~cLR-WG)vQ-&zqN3F6 zcA|1i)UptTQ34V179@l~kc4>=X zaY}3Pn3uLCfYS_?v{0`Mlqlh}_yq|9KMNwd_XG@4kk8e6Y5 z8q5)lr_At7S7fYHT*wE%wowR!xk3_T@D!V72jrYm_imK$DA**{@|N&e+Dx_l$(G2IQN74%Q2@9%Qb6p3b>?e+Q0W%NaU^;%GEIb& zhAWP=>h=melZSX~m$c1jAe3#EIC!8Dr09b#lDbR`KCd46dPduGCB>Iny{#o&81@Br~oF^!54KLhp@0(MTj6 zeQn;20`Zn^dAQO&F(QI^3JQ@sD8h^Kotz%{CbEHxwIEEbEO75oK4%0NVoeCNp_N7N zYkAsth82uu2W3o=O?tl!SDS8(a)b$Ima$S+yrMgX%{YZNE=>C!#7c=FM?#h_dlP=a zlRrtms6uk8L7ZN2T>TnIE5?ks7;4E+zmEbDdRrRY(Wlh*7^`JAsw^KmCUHE_EobK9 z^j_-leo_=%13QDDIjCVqaUl~dqGv$gaW7Mz+;_&B zJXKU`AVW*G+G*Ah%?4X1BDL@(B6^r}+tX+rVHfYKWH7O7>kSBY7qGzFbQ z`b2P=({N#D_Z{9GqPu8k+(W1DGRe1z7H6h+p(#^n~DL&n|yIc#^pGWo{Q zQl*3{!-;mwPVour%3-B$yn=yLo9(O<0o3?3IZex4XY~$3rwoKB2!gQ3^OtCA=Vy*3 zE<^EsRvE8pCfO#vNv*quze-6Fk1K|c6nBQ?>?K)2jcc*1DjpM-89QI$6t)Yh@xoA} z)p_R=;(B(c!eZGqq#1n6^^(YXxA<`dpfEuM6JdGM{~YX$5ocyGEb*8J>l}#b^@!e3 zIm+?<%4$++xh&JC&!29XFt#RmVLMMoCbSF;cfw1!F4g_VBvDR-Ky_ z-!rJ+$>{e80l@hO$lP0ZsrsHn4RT>Ccn&s+MNCDl!x9ksaMQ^vQeVFjtI6AFOdMbS?s=dtD`5&sw z9GSc#?L_b_99u?Koudo65E<>&I7VhNsy_)W&e}m#iD1|t^qoYUy*9mI&NJuF9p`+> z2hB_9BErs@Ja2 zT*}b?OMPPvc=W7$Fj9a#cZvF{#T)imfV>f4xDQs~H!L-A&G>Rn>v9CVW6jpTO->a! zyxoFrzJJiH&BpXKQVy!qh~p+Njlw=U&l~Wib|m`V3Gsa+6xBXJlxVjqsqQ3T=`bc9a`YDyOo2CO1R$kAMUOp`0oyRoU5)mJ*!Xm60(~f{)(ibU}L8l+WFrs0Q<1_2T3KlIj3jl)3 z$nd}wg6b5-8~91<@yKfM+*;fW8BQm2vA_chzu6$PZ|66lkS!&uCEk;oUJq*x~OHEo~vF8+nLUa3CZ`^FYtJ&x1RhJIvVW|8_&VxS)zXEb9VhSX{|3A1@fx z;E_BwV(sEMm5{sr+whk1Q}XtWFFF0E()u$MN6gRJ)t=jCa@B_;=!f?18p$w?U;F>g2)Sl0*yn5jWjap@bUk%_K7FPB1g*@A7x^XwL4#o##e$CCQpF=+ z=e^;eOHgH=);uVn$fjSlhnb73H+yftq zSB_I>0eyg8F&MsSE$%7p$l{Wlz9&fCJ!lSlJ;lB{q$QK4d!$*j=sslVNwYU$3EziR zDCG{P0luSnAZ6m+0HYu=;1OJ?lr>aZpF!!uvGn!9AtWafTpx>S$Cno zOtyaQTq=MEhI2k4mHQ<{>Shv^6GL+K#Hvl~!B5dX%MbEQu+YaM_Qp|Ix_8<6YM$^X z!nsPjyL+gzbyib8M!-(7O$>2yc`>lgkMEKCX)@RBp)#^fz)J!~ko9k1Idl}J&kKNA z5&W3g=xdeQG1GAq;n@=OG_B>L{T#p-z@osR-|%YEpmJte{bZ@&=A#YJMPd04A@H%) zFxgi?V8nM!I{8=WKF0jfVwGKD$HVV@npv==ZsT2}vFkL#{yFInJT>eQZh4JimpB8O zuaQw~W_>WKpxv}nLfqb6D=0t$Hj~A688^T0V5n`rJy)ZdD|*U9yUmD^Od8tI0tH8d zE#7B}tV7N6;hBz}m?e2EVp^XtFZm^-yPWm>;R55n2KO~QR?kn$I5ED0OKi7LSsGp7 zF~p`ee#XLfe}99KGre`11p)Jn_M@1ng+hDi(9xe*nZi4xl50J>DB|Ra%@~F33x$AAKG{bFH_YQ}j!t{W%`HW5sXTPothMscIo@E%cmc|XWGIHKMyDJ5ek zNA-i_C(Lkn(+3Jq z5mMa;$tPtL`r!%55}!(H@$p8&jW1Hkl1gMlnZQI&+EzaVivFtDuD`L0#$8ftxl5>> zbWSn|$zFQDF@L=JM80Sn430VtAIxp$4o*%Y5Zpj|0ryOWPrtbF2{pauKcjsNH5q#K zZ?u2@jdu3`9_{h~2UQDuCl_Pa|EPMtbeoq%?y}}{ynvSW2ZGqXz%CeuOT#w~Qra4e zN}q${b`;MT*4M9)MjVih%G?q9hJ4yj=U)vY5#}K+&aWoNWv_hm@rmKc$nJ7}nz}gI z{`&L;+d<4JK^Ef=4-SNpW2IkJkZt1IaB4d`v27~ENg37GM+9S#vf;4xu31;kg}m0?uwL)@WhqCJEUwebjt=x$xKhw{ZfkE-b?3In zuHIcux_A$$RX{eZ@;dL(?>r^~QptrRCRanWyxMz7xW zLghlI-=XZ^(|W_1LwstFQ5QOk%WD9GC{N2cPXT{5hh-g6Db@1nxSkU(?KjL&o7g!Z zS0fA|$Hbr)F-?nwHQ|;}U%?sq8FS{45TInE-3uE5FB9=Zc9)+2J1@At2GLV>al=;~ zTsF7^_G;G|n6!2DY9QfU_LIrHz|8~?{nW{xtkpI=;-ZC3y?<|g^Mlg;OlO{aT6>nj zOw1YnmXI6GVBG=7F4mim+Posy`Gr7$ryUEM@<75=E7GnI1cECKL;O+rmPVP22e=pF^CQ?{r?Y2DoLMGNZ5!A@_5Bu0zgSLy6RSDpnK+3B*!f znp_gv5JIcm2a3|Erq>THpYqpTb+GcgKL~Kw+DA;;$QjP@6|~H!`i3jP5t{K9l+pAx zSSXHI^W%4%EUa~@^bR_qQFZ}QNYvn(Wb0#^PiDeO%fTC?+-KqhgrWa4$azyhFumMJ zU$6}|2^?B|e(-$=QObv67B9^kB+Wgy=p$`2G+BTxKTNG(ej2zyWQZQpu~(}S^6S%0 zok!=8`j=xx=tfN%1Nc;rLq{mz$NrS^cjRaE-f86fZ#gQ5?&RTFNJI(Lb8An3!XzR} zWg;XNZovQ?!T-9#W+zN1BEayy0DaYY$ zv6W~@DlpdJH|OQ)?l4{A7Bf*?T))qur1p4`^OOh=bgL++>0 zOS$QeLb@U8-%Hn=tG>|95W5cgF`%vAbV*g6nT+V-^*bA=%{MtNXrUpek774S>*=EP ztTjZHW{gdRxrHKLnJXNqXbgY%tk$QFv#hXDX6Fx&R&aq|(|GP_yd1U1yN8NtN*-6V zT8O61*@P?B+W29SxGTD>j8YkPA(F_W_?8R>h z>_6Ly<0I5#(Gx%6rpPN7S|hNA&8FMqTi@^fB}Q#`J89Jcph|3Zq0T z`$-(KV+7+07@aXaXohAz=6TgB@CcJ4bwJRt_X3li8}~qA;wQS48h1ti(bzN4w%>Rhp8p2gXunT02;IRIp-_o+K#=dw z=@=F{x-dtwB^&=TOH%zF5~3+BVX!~sW^v}s8o3V0&~dHOe0E~Ua%wqq;zsp~o*?Pw z`BpFN6Yw0aoUf`j0-wwyWol5emvZVIGPagia1{<$Dt#R^fto7>16%)$(-VeT|BTUt z5>1yD907~*TM1*;%Kl%DWtG?wX*_?)i3QGoB`A{iP7(lP3sqMm5ho{r=l_p9Ra)L2 z`05y6npVJ3Bkf0wik2vpxUC5K4T_qt8xu|4V&3T2nGD& z5QY#CK1%2!%E|%do1$eBppkoEgDubEAM-*VngV}`iW{4^x#h=ucLLAThWGSG_V(Yg zq4Ur6yg8^%uaKyPaABZ-1SLD0O*3qIYx44BK{dvl>TRc4ns%2~6kf=4t@=vYuM$b~ zp%x6-wG9JnQFA5projAY^M?h!9H6WeqGf{)e?@w_2WbUHBp3i3MFm--4urNJvj?m` zg+m}Q_S$`z%*dn5)&#*jCcmHZj)pEjJ;k!9iC)OICZ!OKDSQckRs9!E6C5xdXUC3F zr4C!RoR|+%=b%DYt+61QByK5!Q|x*K;+x`(6W~XB1fqha^uCSbZ#_Kay1*cZMtsoa zYPGuWl^hc=9~PdMx@wgu)*;QEZrKqJK9}F}7pb1lAX>rkFlVg3a$2Nh-&nCl!xwu5N8Ub8le^cd)~l<2zsm}&hT9sQA;&jexw=jcCN@lE)U3m> zFZ3ZzZ8opEZ*>0N*v*%md@zc!k?03Rz^WIF645LBAw8(xngQ_57U1_lOgj?kXTKnE zlz-=p(z1+}>UXv7hyZs}@{89!qzTj-u80U=2Hqk3I2moCix~`pFy&7Yfn~96Em^xi zY{N^t?mn*;@$2$!y!;D0H3nVr_gIftiewbhb(%YY$JXBmAFvC-(Q{ZQUZBK&JE!5? z=&McUewE{Hkp6~pQ0k4Q-3rgder4P;2QpI(H5*OQJkyPNlGqKQ`0`g{I6De*a;-vKf>Dg4AxS|VHVHQM`c&GQT&P2>M1jCI3AjyvDr=3_H6SJoM%kQL zQPuky8o_t#*WNp1!WvK3Sy3>`#d82gD;S2vQ^dXTFhVGm>);6YRQk*;sN$C`Ngw@f zp5{*e!a?C9Eh2Uw`x0d|8@cB2*C!04H)F?8U~BA1Q5%1cO0Jrt5aACC3aQSgaDe9h zmz#I!Ee%HQkok!|>WSyBoEBSgsVv@j@N!K$3GF^o8Fx%5O{e8}BZ9Le79juPDH?nG z!4o6)Mt}DW~*Sh0-WpE0X?p_dd>@0nT!63p;y+ z4i=GY21GiA`Wy;zr#e+)ePgs>31)Q+w5JHis>k#;sfTG$c?~N0T?%t|*2mJfys^LN zzshvtlWh+LXDw;Uf$@Ko_Ebzu_$DTZFkcXyyuX__TVc~6`Cd?t`vT0UnYVaixoukUW0v;2{t^Mz}az&>@325*Zu&-1vvD6L0xxn#}~AKesEyFBYdOWusB=s3A3 ziml)WdB0G_pkG8}xc`O|&pmJhA2aGt|L|>9(YyKqw2#f7=$&3c_Ma~@G6DQlNPZn$ z&rlZAB1FUl9KF{!;9pVf&zA;aT#E>w?nS-!so-B`mN(O5K|LGeFW}i#Z>{Zp2%mRz zJ4{dC(i5CM^?8o74W=3zsgA)+)Dey=__Ka0j(L_~8Fe!rGfDGYCpp;3^88euda+_U zmO8!QA<|rK&FUdP^l8ExsZyEhVLJAyO1)dRoF)(yURrkO-s*qhp*gnhLcT5^e;HSq zTK>-6qdVqF#ON$yGSVdZ3iN6>o;DIA>)awNw0(jlv(5^^uors<^`nLly@B-;ugI-* z=sWh)OMino#lI1v*sMtIH{a;j?T|gW3Qw+04`$Hwc=N=L97uQ@odL7x4N_Oqq5Gf5 zI*8E_uqZtNorE7!NP47_Y6xKG? z`!@CTbo!)>D1xE(`fMrRB#2+k0tXMS!BoCCHP?xEqqTg91s5 z!>4=v5}TmcNK;Cqp2-HsTqp4#R01(yeU_L5IL+YZ+I%_Y=QrQZdn0Y&42;CMhuwBe z-4K4EDEiA#H6DT!pK`nhmSD;B4Agjd#ob%K18*3OQ}&YsBs>jXa^4euMuBt1T?)qZV#)>P&odQl%JYo}jSO)VG zDSk8f8w(WaeRUrPf!hqlVt2ZhDr(l{lK3zu%J3)H;s8KEzrP72AIDzs@wNcC8tI)1 z0haFygZI+P(y2J8j8ESg9)0W=-8~Emw=~;LWLDlw9bJbG%RJ4L$q(J@#){FqiNN~} z?}cu#UA)$Bhm}UF{2Yxnn4y5bw#^5Tg&c9~`;bDu5=`*e zep(u2?n=BS!sz596RhEzTAr;V0ZHKJvq@`8S^cFc-=N)U{z#t0R9M8+-(gc&cczE1 z)aTLn&=#@&vhMa55Ri+yRJgu|F!S${|Nj4!&gm3-hN&f z**B9nH!Z1|((=X5QWjO5L5YWAi~_ZUy8NK=$+mU_!ktTpPx?hu?OOo%KgGyl+UrG@ ziN8&cCoPVoaMr z5tTj(it(^|a1)?yo&4O%ZN+oKY3+)WQtk*u9g|SA*$9|)=!{@fJKHz8$T9I0l2J59 ziSQUxiVgQG_0prIk>qE$0`rrJfQp6~J<=PDY>ESID>dp0^XBBNeYqZ0pje)FsTY-! zMsY`rS10i0*sD{j-{db;_aKeq&#|y_vwJ&hY`2Bv`(u3V1(-s86E}H41v~d8`RG0P zv#~7ZN2p>te-dJ@2bX9rLD!9p(!cvX@eWCn(27fD!b-K^H4x8u4ZA)Fuek+x5@LIx zLrXq%>t(&`05p3j;mL?PWYq}g@w3MV7*Rs$bz#Uy#4LyVt*$^aoiB?UivA`Hbs^{0`EaZGqViEDr zfv??E-m@s7-$aM4?lilJi`S44|p>DuAOV$SZ-1q)wxFcRB_6Yd9{TBsB`so$p z_PrEO-X4Hm2GEulW8c*=C^hhYAS1%yo<@n|Ud&jUaFZZNV^^fwYV}Yl8$XvjYI5(m zU8>-aHz6tlG)u`rrJp9tERFk*43ID#8W=J)`$ZN&&?gk>3=7GTH4Fj73$67yPTO*m zCXTPeryF67ay5%{mo*bk(&gzc1z@LQztWiV=?1$?kC8&rN;*jNsv|5T9zI7vskgju zHGET8WUZwLF6mq;rRO`Z4m^iHK2*ke!(_rOHe~x_7Qap_$wH~(kVwMG2p#)FuRnNU zJi2{G71u>hDjU@H;w9L%|fWg zkSlUsypA2CyVVVub5X8G8}acV;GYeVEn-bpt#vE3 zc{WBEE43696s&GaK&W6jmP{}&)ESR>_S$+-ben|p@q6c`6^s8V`|!5Z3pd1Ih>`y{gNxJX=(Vl5X zp54SCO}Xx?QfIgc_JIkSs!C5sB=@(F97XDRI2R6{EV5ugA9g>giU*~(T<9kmChw&{ zBFmXajOnQIFkEL5e|EvVXJ>+D7pbzr*S{=n?Xu)V^#5W!2-ttcc=dm6-74b>eZuGh zV2A2-)U<+bxE%b!?ab}JYf0KkZPXsc(toF|C0Me&YGzo->HVdMNw>}H*c5riqZ4gZ zPg&l)ZP!FK_8>8}$s+K#=Q%T`Y65-Af~Wq|k#%mu(G8Gf{X zW(b>0oReNF`IMMR`UBocE+Y{~1Xc?P%04(_Qt>|*X(CK7U*j7H2sPw?C0MF102fQ+ z|M2IxM-@RAXi=gz)Tr2dlB)P9Y8Na;3PC2DErMARmzKY`TvhmyVVXL@SoK~hh~s+$ z^;{U`4s^}W*IJS1bic~tIo?R`xO_gM^mMZ&XHV%h$Be9qb}kW+Tr0RKyy>pB^`u1;O$VPn zN&}cxllI`tehtiSs}&WkH{(t{pF@NCu#+t<-gM$=D7b73ZBFpQr$_`yK?YCaa3w3n zoo$@*XO6Qj<8s3<*->twMI19AdVSe+=ScmUv$2SFO2(O2mK$L=S#Rj?Hfm?WNZic4 zSIRzzmi(@qhm`k2pq5t|j2JkLJEYHya4!-} z@O8qf#(dK3v&>qnjKE`F^xKy|5k<6{k5XnOZ4#Hn8@J-_kU3P{VD08@4CVUQRl=ci ziQqC-q&oU@?6M*%e{_WplK#Fj9*kdroTd2_6m z7IeP;QI)T)c>VG(beHU`lP7N zD5&y^)M^QFVr^$1r`(eu1XC`BOv;KsuM>fF*Mp1R7_BsQ_6YPhh$^1L+M6Fsw*;ankZ$vjkW5#xq zg#ozbYli~g@5AoSk~p|g-CjehzDP}%$=9JXs~v?wzULx8L|n}_Vmdjhc5hWS%7+o1 zt?7E{JV~m`g`lk0hYS=88)*&Kg|Dk-v&1Yw0VYrBXR-JrUPkRGK)ai6^!ozCowaQ6 z%3W8;k@e;fHf7l4+|1(GBFn{+AV!dfOrrKd9ffLLLcim?O--n&|9_0VV{oS5`tKQZ zI(lN;ww;b`+qR94p4hf++qRu_Y@=h$Z|^fTb^g1~%$}-MSKW2LUKiI|pYP_^*kG7* zX`XwAGLtlh=z6F(kC`fxp|)rQq769{+JG>Y;?RucD=ApG_$~TAv`FK+MgWnC^-M}! z#V=~yuc)*c0XdGQ=(j&HW=G4})Sf?I87m-`7hXFr9(mBV$v^CpBn*y781-3G$oPNs zO#Fgop3GUV)h;E??MM6_gy0zs`xZ3mcq?E;BEwfV^{K~|~o#Ze}<;!g3 zMHTl3vp9I(qlK2v;3+EC$trD8Tip{1(oUg_W0GOat2(5OSw5F}Ep#;vUhjTH@xW_I zFX>1qLQH~J&?niU;!}F)8{>)Wepb!FX-Z0uA5f9%#c*;=EOvCFUEl#P?sc_#<#`VL zXr_l|zqNsT(xc`LDo3@RNO> ze$!;JHu8ulc5ps=!*eG&RQLm|9OcKZ(X{WD-Om<@9x_3jbRSEEuKhsiyPq}W32<8$ zqO(e0n<%FVq4b`oN;n;d`O>zhkF|4dKf`m|w!MC4verD~&Fn(xDZ%~P`L?TO?tVSP zvUE$ZGGHBd?T;-A2tU|Y7s2raTQ3=a07bMf@d82W8i>dv5Cp>KiECJEl(v+Lt}^p)5HEX|O1HSl|yRNdrQw;Pl_* zM3#k;0_4D-TjYtUjNIvx=HvzyD%BbnYPPn`^vfMhZ44^p20vO0R;+BVjH-2;R$4YX zTEpd=lTUJ5RE{tDdu-2Pk5%ARV$SXv9Ea*fdB>@+9)8F|l2Kg#H!K zcnEaPdQN^>t;S%c7J&N%p8c5XeuBJyN4%JriaAxB*nLIqj$4)Dd3=BbztmEw6 zOpM|c4acyUq2q$RtODT_OOaU0Q|mh?dG|^RY^+En6;<}O$&G43on+-CO)(A;5Isf$ zjV5(JJ8|?;Wmq3V9hJfjJ%FGEZRR7H;lTs988kQW>>Wes0|QGfByMenv*#?14fvjCl|P)1?__et$n#E}Xs!CC-p<;qbW? z*o`j-BK?<~--Vjp!a>^QfHfdva2DF>G!LJNU1hBW@@HTY_F=7#%mRGkFLos&skEwf z495#3;1W$ebY{6p8u3xYh%FIfQ9f$E6@!lX8*kajjJINucy)+|Yj0YHMa%5u~ul9h& zE$q~EmEy!t9N{b>Y_P-ADUu6dw-_kih&fv@=|I>av{#TOE)iPB*|hBSO5;Xnhc-tO zTXR#+@^)v=zMWf2;5wfa+6=IItz?fkT#@~w>T(ni+ca9xFv(KH((^z@Z{X095<}jq zExB;(8357VM_xXg(j&2N zFs_MeOD641uP~^Rk(Y;NX_oA2J%EAFq=B#-(?wxsbS)BZvo6r45q9zRHtVvT+1mH{ z6H`gn6J-rhr*e8A=O(wCm@xuBxxDms)9;b=(M zSuo8Zqi8rj=Wr+lW|G>C(1D2`jS5?q*VKBv$;Wv}y0xLTTE3(VU9|B*&k;pg>x;EWc-DW^JGv993>sO<3wxjN+Dhz>$%_~iU7b^q+qBu zdpoN|KR0&Bm!Xq=x{~*cXxWY`Bg-FP-8%~Gm~}B*MfXuHUuhx}F8kJO9g(MFNpk-- zXXB-~a17!F)YSXR&xvPA-E4IWAhi1?{iz*$MBDja?RV&+Xi@V~LRW&X$(lWL?+r>$J8usvNnXd29d;zb`7*u*%01X*aFbB5kueq-O* ze#jZH&g#auLFtBKxT46?LGwxl^&S(TZ0w`t!x8NBT_^1vynV=r`aZX z4*B23JJL{YO&21$rG_+gXbuWe&p=n^qC>?35Fmw-y8X8qIo_8RT5?X7L!!CE z54`c3=AV;2_@G91gbw-0a4@OC#yxMqWmU*P~eC`ZN`XcqU2n09PT&(F0E3b<^P ziX;z$3sRO)GCn9J?%`db`=fRi3M3}DeL%adP0C^`3QK2auUK2ZzVPM8MrZ3~rGp~| zegrugS&&`;NEMFAq<$A%g!5M99AKSJjrB=tv_#n^nlN7P!f&?O5OKu`Wq0D1qTsa3 z`Owr%ol8<8I`&i1rX0<)!6H;3_9#q1#uX>N-3o-T;Nmc{h*M|CC@ls{xzeg)lamzJ zZ-Pw1uXG&raj;b+vnXF@Nc!?hI^4*uH7~Wjw!VCP5@nn9D0RGaow#`cjbF|>IK=#& z{tgk~ma#fWWSBo}&m}1dQ^`tPPsBN`D$s{6o)it{UYGxZ%T(_?n>u6)>wsV&us>jr zj|qEvkxN{ybWYW)*!Su?SoUO>jt*PP7GRT)#}t-GUyUOu+?S-E%tFOLX&oIH?8qCB z`ja$Iy16=^hSl7Mf6rOI%j2pUwJG;oXF}~h^B8u92Ju0 zb7QHKJSk5TbW~X{Mfr9#*4c&)vAdnzLdn=ti3C?%b2+MW14w#2{NYrUKn$vLBWQXyd{OLg4`+}) zC?i7g@WMurfBu*d;vjyN;XRBK!kiH(=x8gqVu+&$)}xXeVikB2=A})sz-MSg4D&MS zGtr}uhCvL~O^8h0pl95m&+`nXMnXF`Q~5-3%H{X|y`0?Wpp4lYTZ^;`s44bVq;McH zTP)Dqm1O0lbAu(&YJZs9j9MsAm{GzNq7(~$i7~*Eb)cz}C8`Msd16J2Ku<9?s^zF( zH8=Q|9j#-?cZG(5H|hWu6BybFVSSC@IS&rcp}tmRY%slHjfQs{TWr9^YoshV3Ept7 zJgGII#q2E_bDDBYXnIPc;)PR|k)D8J#F<4?QXo0sdeV-DXoN>56c{C!Pg5(BSgE3R z#l`|SL#)UBmB&|GomE8YdyH}Ux<}qv6tnn&J#u~(bK?a;qSh1YtQu}an8RVt6ZAeY z-{_i8dr^Oc-cehXaYmmo??PO0Z<-Z#nB7Z;^-NYr9NHJHTE!fb8`tz~a68-(YETrn z`bG<0j=wS$xMFy`X?nV0ch-a63)k3y+j5(7w2ZCB;;R<93m-MV!VHWP}c@VlS#cUV$ zUXS;HYlg~GJMQ>i+j_Xezt*h2{nljKcobxNQxyiT@(|llRok+d|H8l=&?v@Mo-VXw zSGWDlZbRp5gXa#vOH9DnRnq}(o<6&6P3y-yU_2e*^TvALAzu&Dv=P+qgLXwj-qGU^ zKqejL8el-o7OLpDhf9fw;*AW!Tq41F=-3SLJF-8)GUOoh+%~nr&xT#xRkdNwmfv;; z1>Tzt=uo5#h%Jdnm<|Y{`IC_G=!+GLsFozuB!M%cm#8@wYREDFpdL0Y(8HA$?2^c0 zh(Q@u#wtVF6alZMHEGP_6s#)Rb&n9Nk5ChJG0&#Y8W6=Qi4=Ql&9FJd7>#7x7m@n- z^qA}u!tN!PBc>&zHqFe#BQ9K{WMI3Tn;T4b{P~R`i``<)>UTgGv*K_f_(kZyOGr@s zkwc(QtA_PgkhY%#QKp8t#nER=`>vM>*NeD%t;a$QkK5F&R2CH7Y8XhZP-!+V^bxF4 z>CyD{A9~i^Pz6_9kRTu;NFX4z|Mjw#jJ>_X|Na71+W|)%>r0N-o6IvBt<+J$af93q z^%Xs2zi8M^2#v#1CR^HiRaS?x>ae-1zKO=Im{X3DZqJXQNdbA{6M4e)r&*Yk0g#H} zs*j|W_hAI3_6P7LOLtt#go@qR`#G!e`;}qm`TOM8*M-HY2{XuR3|CYJrR3q)G_+h9k^OC)hez+;;BIQvs1As?aV7$dYl}| zcjp?*8SNZZCmqnw0$d1CXk5t_W*v;@p(%)ig7^s>H+olc=l-b_@}DP#r~UA7&u44l zdI5aT!Gd|Z>lmCXTem;`IO==gZdQ;+KZ9A#Q%Z2Xzwt~cO~lHMN??w)JiVc^)~SkGS*?eQX;R_5|tYx3Qu*Rd#|vx7%P=r zDhBvY3+6gD0Eg)NTt!AH;h|iZtIjDp)`%-I0x9A5yRq?C?)cA)7VG5Ft9}og>qa&c zI_y;7PNhU8Xfj?cq!*LT(uz7lg|hJ$94L16La_YNB04x!mRhw*K?XD2)PVz5Bj+2Q z1OuT1bfxsjHY(C3vGj=8j7_F|P!BkX>nYM~$NN3uVxfz);%?j0r*Ue)N@51_aTl3F zMm6E7Z33E)r(}-cW*-bQET~lw%$&2V#`jEu41Ry%Sx6aLaRxPF=PcZ@nn8GRpg%Ov-nXfy$xC<6o~Dg9 znrQaBtL2#FA2tDw31==e`kZwrF|Q*8^qs&TBBltJe^niL`UIodc=8HLrRcg%^yiTFuf+vuTu+AFOKXJdmV?QSKWpvg(Eg4fg}8s%wLV{Qa5V>|eJ7wPIb- z4|u@7(pyXZUxVw4WS}kpxSima>V$XNT*YSYk?zo2t{5+0Dj$4(kR=JL;=~u?9@JB* za;`c4N1m{qB^B4QVew?_Z49U-C?72S^)aL?J8|;%6A2V?6ODyib;s|YF&6GT_iMnI zOO=SDW894wa1mgrm|cD&+(K##w|)*`Fhz1gs84?e1CkFM1I;j!58h7RBMn>ZQLDQ3 zPS1F=J=Lh*!A7g~$PRQ=DOs?+aBA&dJWLCMYMu1VYr@0x*x1KS#;@94(C>F}ryg}~ zJS8~_3&gHg3(gSC86%9Gf&=HZSo1a7nLVT|3i8=yh@_f-FFPnoU06!IA1<OOZV(XQlfvJpNwro2lU(9-=urJVhLpUDL zXgMUUVZQLj^XC)ohb7Y z^KHePhWp4;oXElzcw~RF!u-|W`)z=b`++C3C8i_1$Nqh{xw9_EV_z2a0pVBfq@*xP z^Z^t9{U7(Ghe;@^Z@@r6enb9Om9zf`==C3?dI3~EYia3C5HK`?7my#9;84^B3lqxO zh9&z^F6U8xQk%_i|9ZIz@!t3n7>n=);-7S|Z7XXaRb*?PkU8CblJ&keee>&=Kd8o_ z9M~#i1aXxC6cVt?bXBEtTsG9&t+I!w1A&n{&*qrUMGjCl5uR7;K4BPVDRK{D6&k&6ry>5gtW@Tj#J(7+ zGK^hB({T{kAIDZZ)n6MOlcKKj!FL&{+ReTXkz}YlY7UT#fPFznpG~alV{;rmD`^6%Hv-jWEZzaYQBSEi33p25E8-qD9gG zVJAjRw;*rFaRFU!WWXMu=5u|Z-F=W+;nY=u8oPz55b7GGWea^yXQW|k4G$pw3o_nN zq?KXn@4Y|;m2RmKTX9~_56lw!iXvp!O^$y>T$U-&tqzqdEPH(xf;)SMs9#U6 zQ?Ui0+VRRoO3e?BhvExusDO>Zeq`jofVoeB&vVrVTBLNEzuZ9cQT+$+-g&TtI3rS% z8wN<^XE^5=2*EvsaaImQ{0z`wZ+rx&4OaGo+C$PkI0P$0ZHYkDWWf_JijNmyGN~6a z7|D?bPk>(F~KLgAo{l{LIBr= z*dwoox{)A%hnrVE+biu5BxuSqxi%2vJ%*yvqKIP5IS|FDl;St+ph#sl$K)C;YTJ2> z=`>Hfix)SA*H#&_l`#@8Fz0r}_d;d}&y=JI?K+g(M=OV$KTjFj_=5ePt^;z!Q&|BD z0;2Qdzq*dFy`72W|0JL%D{D#*3Lt)|Su-eW-@H>(+7AB|geWhgd{cQVc*!Y|)|E|6 z5Vg|ZUml}Twgf)?cr`BzWOH^r(3F;F) zem}x^4qCLv8J}}u5t=|$MtI2nt@8g5eGZrqYg{}C5D-bgf4#aSVy|vtX=C~yeT&-8 zzW|6|KND=iB8io33RZI0Kmu%&t;mvK6lpL8Bw^4hugC$0hK-rmb|P<+Pgy-p@<;Wb z3?7U*Ehmrjp>04T%Px)Ac9RTP?$MDB zJ2?MTKC@ZLoBn5x;3A`38^V^Twf{lS$$=`6u>(~^1vhYl(Z#_XU=`rKSdzW-7<%_& zP-$_VF%AWP5Eh7!CO$vuutz8c5QNS(?BF{7&a0+6?fx~5KNZ-WAwCC@W z+s(hSZzUnj(03-3qbXjlqD>*rh^4_6^P*MItaTLn@|s*$*!A9IYl&%SPP`GxR8$y)qQzg-Xd-9n50=3*8OE*J|X4mmbD|L|a`PZyr-?iXhAc4uuF0 zZCB2}^LFRXuhU%JD#OU^7j#cs7bvp2T)plo<&^}dV~gdqJqwNu_*Kvflj|F+>YDy~93&-B`uIK5LrLxf8Y46v?(+fc zbQY@WLBcDzg!kfPjm@gsbXIlJc5w9R^}o~pRd)C|6#`gtDs!>}^%x4n(owND6Td}S zyB~knB=lOJ=Hrk?g!?KEpZ|Md7ep)l}X`zGsD5+>i^b%P!v?kbnq!@E7R(#=VJwv=XW# z4@RCBfG`eBv$)0jlPYwIMeb8E_BJTOdTL$dC)zuUDf>^}h|n*{hcOfPI-zOfsS!!W zIF%GhhH3cvL$mN)7(9o)Ir+>RGyi}d;)%1JMyBB&7QyN}GG&bF9{%p#CWkRa+rX+d z!2Idb0mgp&dzXI*9e)op<2AhNQ)URt3f*_k)yi#mAQ5X#5G$6Li{gW!OJAVCfkrh;(Ga2|Zfg8U z4ioaV9Y97;*lz3mt?};HyVKNs^PQKYyl?3(_vq| z(f|8>R39W}HAq5IVh=!u}(3=As4m? zm<`qf@a;vDdUZUbmN90esV2I~Z+bwxyVNI{q*CRoysRE$x0N`HWgfhvH8mc!*%g0ZMBk!?yLA%-u1D!6iZ1ZEUY|` z5GkT;NUBvbE*g?$mU~;D%WF?C%4T*IUP>SboYW2dyd_7nAki9Tl-q(18?Ql;F1?cHu zAI3O?%}pMSf9LZSra1%X4D~}D-x4ts9Hd4XJ!1B5kM9)q;!)nKqm;4T4t+I9V{YP! zOR2o1IK&$zBV=iU4YT4IiH%Xsu`*UpgyKc^vq*}JI9JdQxU*83fSf@hnZ}lb=qq84 z)=`sfVbXvGiC;o^i*ymq0-Wg1``$_;u)IQ*i9kE+w=QZMlp@^R!%cNlorBH%;Bso` zKYvv9ejqa^q!F7DcVMP`BuWJbqeS!dh=ed*$gA$d?u{DCOiBbuX~V4(mY~4uO2$%R z4J-@DQ9+B@WocqL?{~Uq#6=RCmT$h%9n;#6%F{x z%w!(E$Y7;F9VH=gjQus%l2X6)OuT4f-|-^xO`>d0S`{ANuE=L)+Z+i$$%D$zFq}Au zf0YqpJCNgDeb7pr8Ex&aVom&kLLSQ;PRlPBD4yN06P$!Jbe8W98JpxhQUrN6!RqNA!_w; zJ=5qVusC^g^M#PGS&pZRyrd4QTD+*-=Q-VvytKVvXk9ksSCv?)Y`~uxA#GOMPuM@sRkqy99EA_ zh@#&r&`cB#!En^Rt#Byr6>&}%GdKM+_`p1^*I!waMA_|_A#li1rw`Gez&U^s3k?dN z_?xEB@E*ImJBH(QJpB<(DEZn;2e)kW%qACMDNi9|Pj;%U3kFou8uV{1c-S6YS2uoK zMY90#bc%|AaEnFfmdC>#k5JV*U}yzGDAjefipoh1E4u$lV)UkrYEbJtTa;`Dn!FQN z2tgY0X3w{uGYsLDNU`^j=-8XU)0<#1;~+-f;!0Wm#a7(|Pu!1ynP7v*$>NZ30wSzB za2Jd;Y7ug_7>v*rKaLwgLKpg9HXHVkC7-a8PCz6VzJ`uMrBo5TQfW4k67F#Tvjcp9r4J)!VCm1lk2<)0|W5Dn{1 zg3$p?GiwCyK3Z~h!1GO?RC;Gf-6TYtcqjyzgO_qgcE!Y`J+e9=HTm*$C!va5>VfAa z#-=H;%|~98obW>>y20N;c0dw(I~y7ARj_SW{Nqq9KT$>BPwe(To-@RYceffrf`HJX zfPk?7ebUSrNkTx;`3%Gnu)QKTQ&-0G+Sm~ue&$clfg8dJU*V}eBpe8zgfeVo^{)g zY?x~IYCKPGy?1x|Joj*Xt$u$!A%J*0as?u4p$4lq70B+oZH|*(X4$oO92c1*b4}OD zY`dZW6FKvb5>ocCGAy%J|Ebo+t}Jy8maBUD5@_n&4y)Kp$?kIu$;AMxI2zHaVU%GN z8gQz4h4;+@k?eeQb^OPh=^4qATbbOIrQHG7R8LjH!`N;#+@Rp)!l zS+^fgF{}gZw%+Wy(pi9&L9$9v;AV*tj~%kbpTxqsq)Ha@#xoELwf->b6Nja0&2>hXNY+xx*Mz4KU zXB6)EF4yovs4fgLwcV&)z>F3eXvjY1@~meAv9iq_Npe|dER5;+sBS&?-`E1!ch#)` z9tbVdIj~TiQ4I=B*FoboP_!UU2{c?*N(x~x9-M=iHOq7cW>tsOz7D0}P+W2CUH&j5 zs8Gy@Izapr)LvLbA*WaoNd$ut)McJT>NZt`cbv*C3fk!UBwjSYhdbM;O=2nYffjdO z`Bi3t+qe;dLM3WnFW_Pl_OOg=MfK2UZAnMnj<;l=KT3IiY-BZR)Cx6-&_wR! zuZoa!BeYry)N^Ip5LLw1!b##1cSbUKl5D*gdseNtgVu1;)Q$^3Mt_ipME@W)e*f%` zC)S-ZFzdCG2#U>#0f$Ex^4ECEn=48s37RyueCLH0+VU%kzhOD-Y@>C|%yfz-j3oxZ zjAFks#jN2fFlAA65#Uv`mL*SSv25T1jEF^|s3&KO49MRKC1}cvMCCh5PLRQjM)SWB zdGPF23-@dMH+*QGIsMk-55jWfxyy93c%`0G3@88edti8wkN0i<=TMjXcfI>u3?~D| z)cB*bmh=)cu(Yh#`kGk1;yD-Aj8>WL*aO5{ZU4=b_h7)mz3GL?=RWE&Wpsr&+@nqxow5Enal*rO{?5=*QQ&j7Hny>sc9Y6ky=h@V0|wi(D*z zzlF%%4|~13a&;dDC_&5zTRoG)eHpe7XCMtOh(D|+UCUTdCFZ!?qq0(zmwVvx0=!v% zZp%Hf`uEM_X6hblc0m(RwGQLJ&QH~q{B;H+KAU|w7=~3gA1u{K=Cx_TygHI8 zu;td-t~_1*q$!af{+&1FZc>r=mdcrs(pLazFJ5o*J|YCorIHvdZ(Dz4b8S&p)wvcd zk(p+aM`=qCXYB{0|I5)hxf{H5qJu&f2T}5~*d@Bg!V$X_# zDm`Y?lcJh{g>xd>b<-1r0+(Zc?5ZF(m(f>vjsxLGPN6E)?4pUWhA!_+LPd<;Q9u)GHxw_@OC|8m#`Jvb*L$vE{pEyPKMxXhNEm)13n3L_Q%Y|>3-Q%_5p<6_x_5Ihe& z`yU!d@U~TM78NnVVD|e)eBv!?Jok+2ybFCuW;FI$!`iO6>snV56*VM}d#FWh!aAlc zV*7kUSDD&&#l>5j`Te?0@3>kh&tDCEh;^MnA5D_B#>I()DAcA1@2{ zkH5)=)ZXwO%F$3TuR0_2=-=T;FOjCU!1r(l@BV+ff38^T1O2tQ(w2DVD^T`w?ptD2+&*XxbyjE47ErL_cxwrl&2Mh??mgz5Evugs1lGFj|l|Gqzw+Plm z!-+y$(?{87cZN`lkl6B0@78EF`{n@ZeD|jZY)1MOqh|_H8`dMjC8r$|J=O2UJzP%O19Y>D!s`xD{{r@;;03f`Cn%BRm1^{e$dZdc!t%0HGKd`B2>o!3<3B7OW|577-9N2}nz7`;%FLF)4Hgj7Tp2PE1*AL+lY# zv$JK3Fd^k4k{7|Gjq|UMo~W zfQvH;Gf=f6Gmi^AxNy#Rb`I zUBfn{=^`V`3TZ4bDqtity&=#rA=syVh%t4ht49E)(L;N{6ic1Az*!NxXFt|f7k6o$ zWLD!KTF!aV$H-IkPx2jp4YSPIaenjc=SzbXCO~=~2g|P}-3$5Vpd%Q;WNPo{;D^L3 zfGV>SZ?g)oRdVjE2b>zG;sCY#yw#*AK3~E9&h*bwk3!}zrc=hC>_i2wL$aw!jo$-| zIA~)IKsm!wgm19yLCc?1v__Hxq0HO4rpQ+=u!=ahxjYqnj5<;XmUX)F_pzq`>}V`f zu*ZC-Gti`yhmp)uZIf|JS~{BymGGeYlH#T&=cs26?j&tB;SCmR z&S=|*o7q7>Tdl1I31>ZPyQ4a+F8+A$%CymCw>Wz*U5KVl;~($SWO#Y7FhzxD&eUOV zhYFJq&3Vg~R&G*^jD1M3g+~=?+UCy-cmmTC2jWiyJVF!|q?#*26?O6>lnfFhRJ$!J zmQqygT1mUye{xkt6)7T6F$N0>T{}s0JlV)OTMR&nFZ^9aAXSn8U;qy+s0*4u4OOB5 z=LDMbiZgeDgUqN%E1QEwE5dn_A7Ubyhe0UH_s5xvg;*}A1&et20o38TuBx{F(zcM% z2^Cs++6T|BH>Pf5zFN02S;zWOe)%9`@^ACx)>)TmI>k4$r?w#qbP#fNWKe&8T8pZF=*7&E4dVo%_K%z)>5K}9_$ z6vsD^aUJ$=jv`K8`a6DlvNr?=b=3A_NT5!F$8T9h&TxiXGEsG*kv|)1Z7s4rp@C~8 z_r7R>{uFh)u!uLt-5Zc&LIUVu4WR$ayhSonzB^}e(fx~bM=11!r&2Bk#^!^!^pYu>IVdM>H!H zdiEgxFOm>G#L}I{Wy5d#+iC;NK6+@Z(dPnU?UEycp%K52|Cm+zVhfFdfq{VN|M;&w zeNhhwr++^GN9`Y?ylJ;6fXJ80tu7;lotUIe6e5%mp_O-^M9H8SA6HODY`N3g?}V}C zYQhutg_bNpi2nk@55v^iB7{P5yPwXr!DT*k`L=pP@Cy`oUlR$-=Fl*0jCgeJ9(?tw zGzNf&FEevRg|Xl~Foz!C`)G-+K<&fzxWff7HHVL4DE*=G^X66D&ReI$ZRfynjVk}Q z5k46ThXFHgjq$V3_n?gx0mWZ|REqdp5tng=*f77#=k~|qGza-|xK{|OYc}PCYU?pd zm|z#n2xj(&Hq(MYI^f7J-X2dJ?@8AXR7H83Djxn@3SQ!e>=i0uM^c^fLQR3`m3xJ4 zw_4!~=_n`%};(A5$bI(&1jEyoeQY0wmlRM1x-&lFF&# zi%aO~OF-imFDp{Eo{Wn95;&F2>LhSt`NBJRH*`w1t+Hhu#{d8XgC_gn^*2LsKI^@C z|BxqLQsH^QEBLTI>*-$V8G!-n*_m>cQgQ%9(EVnaq8`Ozni0^bru!T1GNJ*_X0f$s z{$$DfH&4>?zH(cn_wN~_mG`7fRO6wH3*I~%NCXtLOlnv1%`dPgO@g%*IvAgP2%`iW zV#GvV;;I1MH|m5`Xi33%A926*lV!FXp@G4rU-6I6Us|8QO!bHBHLK93PU|ZE5tQog zQtzCv|2UTaGhtVT^-pdBK>gS4^l!$(|I4|&v@|%`@4$WV^042&Aa@Yd!-1#@X4Eqh zkmirdS}W`_65N>}Pv&>U{QNKAABr*Vtz^y0T6x))*LE|1ob^lJPi|^=Kyr+|LHM97 zp)O8EgNcqJ686;E4jh$HC&h-Fa5KGg6sa=tIa+=lL!5Z0n7`H-VDhJ|(69UGrisqf zq0ay}K*+zk-Wxy{hHp7Odvd!Q;AaADxycTM_kLr1x)?D;@IP9Ion!u_|^CqUv%~_s-HnUxP;FnC3aR1SL;uS z7+tWv%oO7d09OfV^ioRfKqV^kLS@=oWh_4sTpjr$ARLU7@(#j?k=mS5S0A}m7-cm( z&sNwTSEo@)Q-mqtlCMCIc!-bTLe^;pS%);w2lKl;GBg)e*`^}#tAw7+DC2Gr zooz}_ZiTgiBEYFLTBvjC?6k(4G{R>x0T0_YI|)KlbKwT@Q*u%s69CJ23ganLi7fO8 z@uoc9o@+pqwLlz9@rXkP1+^yLS85`=65t`Gj+oROM&?9bjo~4As%{cnu6)1kbW1Pm zO=I%B%XVc7cFzz2YaqK2#@+NnKm-e=DKNJ2zs#9Zfxcu~K3;eXsq#*KHnj-LvT29h zVnq8j3o@g4T?E@bQi{94U}~OfT28$QTu^!YE4J9lSeNo(u#enlpAFnN`DmC>E%GW( z!zxoi)xOk9_YWN=uihHwa3mvPQRG4LmNIOBeR8k+f3nLDH@WL){t4LNfdA@cl6KB6 zhIYoL_Wy%Wx~YXLiujeO%ghKsh8Hp_Ades7>!&0t`;AEmI)4;*DFlXnPzUIQ>twl@ z0_o?tgLqu=$*q}(CFpa%L;I+@#z$WR56wI~&p7Y>syMGm_y2moum>qzvloXY#xD4& zhw;5xvtyg1^LU)a7yTj3LPS5FQz{Rz>t>@Hg^D`(k;?gj^W%~@zdWShO zC6U8CGUZ18)IzxP=e7VO061^9Cc3)4O_NEp$4}Q<3CeK48)ryJ_)^7V%~dn#tfuIp z-5_;%XIC|2`VmZy;;}U@@3KqhV%cVuhr5lotLMY+U&Opz-Uzw3pKiH>?7!<_9k^&jo;<2 z+3Nth1Gf668>QXfNQEl_k%lFnjBpF{q*>3mJwWZBb_ei-yI8bpc#}LTKWb#0LLrqK zfYWkC+gWIcre}xPKjF59`S$&G?|u`)XXYlk@1--6%uGk2U*+xEWt$%nq_A%nP)##) zH4w668v)#B+NZUjTFP7o1dyrD;VT{;(NsnsNA}^o%rHxUu>hZ_c$Njv1im1#Qy7-@ zG78c#+O>dgOD(@~)R9ALR0P&2td}3T%#R%G%BNUgB$DwOK7x$69E<^yKz7YYwALxa zXu2?NIOB;v3=fHj`y745H#pVa;22Hbh1<10+~e|QD-UKD2mj#%GW=WLBWq!qI0$h4 zJ?jQ_XM+)6Q+q1CAoJ&MMIrNg3j^p$fO`8c+1FW~6|?!Ahb?4beeftaq!AQw`FO** z*pe+~w(n?-MxBvW%ncSW?zM+7ZGE~+6C^ZfF||bS3pcgBgVYO|6f!td z*F73RiqmuB-O3N{r%Dguz#s-M6_av16{aZNpEk^u8@fXUAsa`smhvu2)vv5*VI6ixRU8t<%Bm1civ}<20aVj1&%TotE8>YwZ5Ve&a#=IMmxPv zkYWLrfaFcxris0fMV9NF)W(yxwa1*EvO(g-M_bGM8$h2$F#(!ZW0Q41HEY7^ZPejO zk!F5#$O=vP;f+D9WQ0L>PRCNr-1_?(VY$BM9*z&Jgk*jX;d-8`rCOz556KysQa(na z+dv819nM;y*kJDO0-&I|`>6Qb)yhAAxo^CdJM~VozZeFVWbxEr5xU+xGB}U8WG(e= zl#X;!p-HH}NQJ8WEisCh72^S_w*izAv!TBZydqrgI0&~tVfdv*X zi)4~dsf?s4SC-dsy$;5vxEvHy;(1;`@n7$bRHKFnToMEc7n`5|W2C*ohZ#NhPXPi7 z^Iu2W|1LmeES>*P?Xjx0`@c}{I6R7%VIWjRWWJyf=jw>ivqlFkOo&?-srr+TknhUDv#lGgPyPwa9f!(0pw4>e^K{7H7E>F7=`0_Q$+|##a0a6dCZZ z3~h893#xVP6y9Q;Os6rgW%s^qZ<=ihQ*0u(+!<#y1W;N$0BDpHO!vV&c;tc2R%-h; zP_G(Cl{N2(9trRYeNV2bYXK`dl6k5N;(e(WrP@(iMw<*)aNNoU!`? zCU>OUV+3vTvFl`Enr?&}LX&S|a&yy;Dx*<}v0hO-&MQIzmy!Mp>}0bK<>wg7*!qfIf=$ThFmO10EEh_S86L@ zag)7>g=lMt4zAU8tKVEj;b0r=vEwDYN`zcH*)ft^s%@09l61?(3otIjwrL`v$WRt~ zMX0H`dSN2F=2O_ay>B;YquJCz$YRbWt3Kx0miE{=K$1R9^2tG7LAolRROXr zjk&U)$Xfy#eT5C62=>lXq>~SpG~v}mQz}OCkXjHtqx_Zc(@Gw$Wc4Fiom_;jNpiU3 zbd7wQE6`g}snT7XD~V#k;+N8aZGEDUl-=^Vj+`jJwdh~JyHl%D^!Y#b;!75>%gtro zb_A5)G@}>BDxsZ5OQUNV1+C>#>t(gx$zi^fhJ^?PjKcnIC;qocZdLu=D_?K(ww+qPBPeS5pcO?#D&@7w=I#_kx&C|Ug2Kgap&;brFadwjB& z#}TH_sDlrL00tGcf>Ma-pS{$hUK@K6p7TcIe#KtEp6}sFi`1x&2vbqfn$#DI3;~KJ zSQhMQ)=>KAk^?y6otF9aCdiy!rk57-5m;BxWmHT zT?%)1cPQK`+$r>Yz4yP*7~L29oSq{ua$e+JKJmU0W6lWrsvd#Nn8)Ap&RdN-Xj_y8 zW@!#Mjf&};yc6O`9JJJAa*0LXJ?IU3Su>WPbxlD8KWN~<(_<2XX=qCegPCbexk%{GnX56PE!C`51bZv0sfFG29tr26nH!J4!kLLabG8F3+8t67y4vetaB76qb#q zup?WqkwldJXX0kB)M7+9Zk4xm1Z&nJ$^K|W za4&1Kzcr;Opf(2RgP8S`qn52Kh<@1#SnKMVdN}6h{FZg%S)sL3b`FDy8!wX8$gtR) zLk}I0|INt`AJS5(70+of1hthA0c>my-cn0 zgz@Gn<=-Dd^PLs|PAW=Gd9>mF#<0_zCvo$?;{Q4ua@L|0}4 z!?6<@Y?Y&`qTe>7XK7!uqlAV}RAJ58?RCHv8AiXDs{f498S%=kfuL{2QN5p0L9s;M z+C8MOb~^^$;$}AbDbR*0w_rU}!oxiqhgLce<9A(1K2PYsLXr23Ap=YInLo4lJW~Rb zaj*-9qF*+TzDEk;qy~o;1+A-R3|J5t!hA$TNwN&p;>%0ZkD1f`vGEh<{x}F#Uk}Jd z+&J*G?50x}iC7qwp+Fwk#Gm48+!~8VzxX5w;2({&o0*7Bb2Ql5Y^m6i-f||J zH+{Ath~B*Gi&xF|I4-t@Ld^rMG}N;Z#NCH^F_phS^^7}T?F!5M7}z30CyuKTqtv?6 zs3jtcReLh5cmdtiyF5!(NXyFg6z}tysHB>Q>rO;PrioAUk?Yh|zT`g4xj|ZmwRr@} zbN3;`z~5^sc=P}0M~0RCtVf5u;r-Mh9@#l;3aC{h=-b&tgcQ=aMSmACz721ju@DrQ zB6MTxx|xM8ch*sRmzhdprWyH^9?mMPWW*K^T3?-2Hyyxo!+d88 z-_;f~^Ml{Dd}$93RKJCP$14j#cBii0&beKx&zjUc=3RiG;hPNc!uzV-6$*9iN~Rri z=@bTjxEwNjj>UhOoXD<*Uy20ws@w{Bt*gxJhjVO7eu*C zbdcmQ8OAy|fBYdw1fj7-M1OcK&Ms3zz$fX&+0PDELr8|h^I zLOC>zqM%q{C3(&}$4h@qEchr9Gd7ft6)yR`d087RnYe|UtKpUwxxwCHr3Jq*D$E`o z%L6#4N1ct^<6Xk5}46qQ4! zbdSCYJbH_-Y0N%4BbJw|cA$hw`Bws!nVEu*K^QGRt)fuLZDEod#vQJJCyS;TUpz{| zg$*dksMpjqz?jnO?cS}4?4spR(K0sFZsrJxDw-VEHF8F+jy@SE$Q~6Wsd6Rsb;KGq5bi{ zIZ$1ZOrcOr4;$G{*X0ho*T{~UuO^sg%d0xJlQS*X+b3X(iUgM|h+D$OGSy_hu}r;2 z<8e`Q)O#I%?9tcvIa>Q|dqg_05ZN_&`0^*{cDy8m`AMA-gN1kd(Jt5x3xyHyL+y2!{-77jA%<7h;L%qqFXmW$%;Ym>oZlRvOZOv4x(H4B7M@qcXw{Z*|~lkSjm z>`k+zd#It_y1F&H2jaaWw5zB()_m!Hb#v*Stut{4&J_>3ym#{u#$~<;F3yCRnDT5J z*Q9@YsW7C-@kmuw;eV7Lg6o>c!h=@Z5aw%dReX4=T9RIE9M?sI9rRY=qz`i-UJ6J@ zn4xB;40knV!}^2K5eAu)K!MR=KQ+y~sv^?x3xM4>%SDn8L2^4x9x-!}8*kzAMdO$H zo8)$h!{S{RAoDzX^6AO{p}m=xIscVgR(E^!tgQG2ekJsi$+Z&!0hK*#^cYOMOuJUP zYqxt0hFZlp#?jxNO1Z-xA{iG>HMUT#C>=Lt!#IWOW$;r~Rt63PK?{ya5q^7K-%DCw zP69weGTk7-K;CPz%Eev)vb>>yR=Pv)oa*Ic)H$d{z4pj$T62c{C_9LooB-G?^Cj=7 zIwL~BdFEi*JWm)u6F?P`al@uJhAp9ZvsEphyHFGJq~{T5`vr9^?#K$N-aWm?OT9Y` zU-p*1%dE9x)al6U0W$cG#K(va2AKXqh()C7>It3aK$Or9C`6BvDO1O-eg`12=y!5>1 z3Yda1Z(>!n)%;N*g%m@4p}%vQJ!LQ*K(_E$VSe%XI`#6_*YR4WSDVa&1RZ=OKS&n( z?UO=9^{RQ|eBt0FZ^D2{Bx=BqgKEG?L5*j*-%P1g+acg8KJ;Vsc9-Q{oS?2c8J@@% zMl3Y<4@A(?QWY}LQf}XBu|)iyl?+$^#sT+Ae$-Yq%%fV{!;+}hp?jF6K&{il z;^(AtE47x2hVlp>@~`g@WE^`7QH|opsev)O-mCoP^iO9}+|-Z(ENF}=v5t}h;*Gak zQf*qC27h_OMM>1b@sVuXyWzsD_EGc)Nql(1S9PYu;EzoRD(9mtsOum0z4yn&I zpz2#mf@e9yx9l6Z?NZnF+w%M3uI}4k^mVR9?vTnr&Q$>Tq{V|v3-bor;usG-5Ou6c zo)(7z#gk6oYldT+khqkQ9ZI{~!W@Qc+`wArPn`G!V)#y_H>JyNXkJMK-l{9l3h; z&qoh>g!%XQep30z@7ea0IS)Ov27ZD``U{OL_O#x_T-)Q{cSjq8u)UIui*ba?+S!qX zL3h&Yw2uNo@D)*&IC=mX>VcDByJZ!ELBv+_)4wPWA83lVao}=~-8A27P}uw(rKsr! z^mc5~hN6@&eN&I`k-}CMRJ~IE@IY4=jZ%cB-b5Po#jXW|Wd3AE8 zeJy(6Z|Q_$Zn+Mu5>JD}utDkczd%W4eC2W>)QZD}zE_^5G=`#%;{SdWq1VUKcY+-< zE6rCm13sF>m?)bu2KS^gL3$@fnTY)>@}@=cFlE>3_*io!)rg{ILili4j#Owuwm34f zdPe&2cZOI%QkoPSx^{B%m?v9g7Y}3Nq$`>?Me-Qa9h&x}!3Uz>B8AIOfzAk_*@}u9 zzMyMa2rTdSNBg~j4%Bz4Ifewc;X(R)OGhNBjp!WzinSvUUgxGwVgsFyCNnQ)eo9eA z>L{FfsJ!cmj#=n9j{|1MU5%x}8_Vg$vp=XVK`3n*e{`p1rOBD`C0rEdZd9lRF~_@= z@nMB59cS;Vw<{Db5y$2{cbLf^|JoJeWffnVgmWf*;4WMXPnFw%PDx^sIo7#3$-)SB zI>j=IyX#PzSl7N5DQ!NrDX2ufKB0EM0o)rcc9v4J;k`l~PqoUc`iDdS7$nh;y*9e6 zquhp^Q36q%tSwC(Fv9gG5Ej4s~^gd{c%ymf446K~43qt@O#7suIUd z(cfC51HrVSKajwu(l7fQYO>UwKE~#f%?Ajfjzoz6w;9R_jgYiit z@?FS?qkiznltc0>554uEh>H(@FQFy;xs#RA9d8%%`Tp^KTV%HFd*6Ni-h1tTJuP_u zdY}lVC2h<_t3*>EBf+5;TEy<(S4OeaRzT}xHJ28FgebNTbgX(LZ_^>WjaZi`x3jrQ zRuWBTu%L#eU@w0y6eZ!PeFaWQEn%xoGu$Tp0ftWKcoEB{6PuD!qGBDvDU;ro+tg!} zC8lt4{G|jekiq_9n<=6-cF1p%>kS4;J3V&rFN(Wj%70SDO zP<90g1J>fcpeV<*lwTjpS~Kov_8P}?B*hZNftZ~c83Jmhb(o0xd=9w?zoE!a=3b28 zB-|&X=+GKgg%M5*-LYtdGFT`&EOmJX!uo+lEqt0YFzM52!!z-iT5+OR3cIw1?`3G2 zQQqyM(G;Z0>WZ&X>c3aKO-`P2as)2%z;iB6T(&N-NfpzafeD(9Y_M*QS65r>%iHsv z=2>`r&L9Z!5yntJ0MDAmIOA}AcSHSIQ{!fTq^&f~I~BY)XP%!UdYYJe40GBHkK z8jMYn{=(;bbti~N_&B^;Kbn)I#GcWP&E~HOqjVGQ+<6eOSVN6rPbL80l4q$0ve?e9 z%1v~}ZR}O0<4PXLvY8I-+kQYWVW^;6C`0n}M-S^aSeA;`0fBPOTe$XeaZi;%>pGp40#jK<#}f|&9eCG7 z1FP`*1#-vhG;|PlP&~s_P~6sf`Osk3gpD&~ZHiNBOx3O6z{D+}pU>J;vA(sttSGsj zSM#V*>t?}QXfM>cycU#XT|9Zr{&7k-U8IAM!2syl4px4t;Pa7ndy=}U+~h=^6UllOCWAcbC3dPhnDmx% znn0W1XgFdD!1SiUcZipeO<~s)2Ri>4DRWAglsw*h$rmbm&&p0+N`l-?X3ZTa)2^qL zLyqDl%}NMQzxfp*{i3IlIL1WoN74?j1Ldr6q~8Q(*c6W-yV$Q&iGTo*HQl(mdQov5 zay!8B(;oQ6+7n!drZZ6qmW;Q`-r($hrV}Gtg?L8N9PJs%>E!zPnc5Cxf5Z)D4tt=o zdrqny7*a3DRl>_-Wb{K58lwvLj)D|6cZ4c#dV^V38u(kGlj8-;*dk6eCZV)y0K54m z4&_0Apv$B^izYd{XO?e%~u^auNtDM1ikw zkwda)Ioxh>MH{+dWJX+@F3P+5!-tbycXEA1r`!HB`<2|$AaS?lHZZitG#e1@N~gA_ zOJBv?R8JeH!#~G{@w?!q z6^E3~JLRF-6wfkDDL^JMn?-?0qVi?tKvRt<_n3LtdJ*1slqre*1J#vT_4UCn?T+!H z?QzYysPG4U6iNKOh;ZyAu-1r$#EP~+0KrT0zG}ye8CldT!?+IKIm51UKc-~ZJDxeh zU_z4>cTY3hb$BrE%VUgLJ;#17oQx!OygFd+>Nqhggq|gPuBp6k)yCn-f^fM#YP?G0 znd13pK&N4hHOP550^cxR>!+!&Pf9xO_O|hJOU3sXRDZ{9dJe&h^B81FHwHpFvmURV zLG1#JSTdeM3Pmc=X+TaQ%}-@DkA-HIWwn8*3a4P?Dl!w76DbkPP&UfY@WMeqFdDTq zi0~IFEYz}OlgU|g=rra|x23;7a30j8jp;g-GZlZ@p2%0 zhaPNWIKfJu+K8}`mX=DY4c_{ob?DiG#9G!)WV>c%GcpPR)ZQ-Q`?XrP8c6TGlR*c< zB705inSo>Io!yccjupHWJv7%Iv9oR7A>KNe6);H@;M>6SeMkSu)E~4i+o9U2dQhv-1v?=q{GJMjc(5?Q=WTS)BdFq(FMw5IQnZ_Zf`U zc>9Yl2^MD;6i<>knA!vIaAmvW8iYX%>(QuGMyEP#-B}uoiWJRIOUCZzn72@Itt}5-$KTaLB^n+@Wk0G-uL_g*$WPODHJbPPd0-DT><>ok!$oOc!f?P zA{}j)S?t@QwWU{5ORU;2=p?tZLMq_3W`x&U)n&ti{a82httoBG$vLZz^RYN7+fO05 z+*Q)Vs3w{cJ7Dh@k5tG7t1p%PzfCIhi>^;+Wm4`RN18w7PIq0nQ?7&CNwDCAB{=AVrgnqWt;};;gy%(a?NiCXJ|p1Yc@y!0%Rb1;{;+Z=>7;umi6f{08_sb zA!d0S)J9(?+Z4^*$xbskMmgf*HFO<&W6B?X(#|3K?gbf|_kGj%+v7g8NW~`;v1=zT zEMfJvl?65!h@p7S*;dme{9%yimCwe;qL0*>bZzIlCpJz3 zvC`1>exri;zlcU_T^X#)9h_ndo6$ZzdvA-MjoXloKH@+WZw-6OZc(2_rvY$Tghq+# zpvq{a$-r>s5ZCr@>?$Y1l|4VVJ@D)p`1Ocrl#yW&j{QZjIyYMHppFBB2z)@i>^hjn z4vT!#=T9M>V;e7=Kod4w=AS~peCclC;(fGhrw^2(Mh)=^nGWh`Ua`|Z9r<~`r`8oH z1ZxgduMWj~VM*O|N5LxC8JH^vjtbd1?xS9fzdvw;1cB273o?@`X(h|eAc~W9GnyKK z7phP5?&n>OE{Cw$5H6{yBP>+)`t3q>X?0KwVI?K@JqAOhO*OGkIcT%G*UDR}h5x(& zmOb$siB58PTzgLvcqkxaiVQ@fhFnO)zRXe*djy+Vu?&s6soaiS2H~l8IcVKK>U8{_ zCV!oCE8$yXMVvh6tTn#}r@ZIS&MZsV$>Y|gaLI;F@S+GbLP_vVR=PI#Tq}L8Aoi3p z-IeVjjjgH4eXO|6n)Gl!hR8*G2O6fCG930~b5xGv3oXXi_mrnS0tIeX^1x!5Kf!oPM=~6jzKE<{vcgH5%uS7zZq9DS5oy}}!57pX?c_`Fv9ip*gx^Q7 z`CeWVbhVMuJk+pA}y;LQD16XTh2YTea2PXhO7#M?6UThVrEThA$y|>W`p8^(XrN ztR+l)jKjDQdq!M|J2>D_(E*~mb<*WUUEyv!Wrb!=rMK}9v0!ge273WgbJ8B3V{9!M6)j>tZEKxQJKRzFA3O~QKMpEhrYqTbc!Tm z*B36+SZ?f@U9`69`6R~>X`j7rd|15B@ z|GfzRFKMf>gPWZT)A#>4n`u5nWbOh12DX6+1}6Xi`St&s`Wm~qn!AuIn0s3PpIf%( zX2$;-?=0%*;AZaS_CJRYK=Piy#V~+7(%EA~QRsZ+4AS`w@;_Qc#TT(;>=#NMROE8v zDMyZ~;D>S0nxhWMCQ0`4ZkQ%0@wJ>LK!$z6knmXg^+x@{kl}*_+Ng01;io8XTCNL9`=$owjI_t5~9m;qxH)i*BZcS#4km5i~) zIN!4fT6*C_Am{mJ=jzNV!B04F(pcniRDb<}mThUCR8pC+M5WKGknzgd(*ZP8ZqE@+ zDi}dU#UL~yoIHye{%B$2N0Q#?G>hZ>z4f^~P1SXLh@L*2g;qPJbc-e0;)&RPhuO&Q z2PDDnXe+|gZ7Cnfnt-LCW}Ro~1%uhBNFA2M(C2YSWfO~9QURs8C)cria<%Kvb0Yqc zPOHhZP~*_Wem1ZiTA!zZm9=aEi6kjND&J=~721cxRv>_`Ynd>x$F+o5H*=vS1j&zY+xAVv5-ynJ*Nw(@jOpEEr#23x#r({&Z`$Q@Mh zt3HHIZG(^YZJk*M;b3$!F=>Zxyd8x%Os{0W_GAuNw4BRm>#k_luGGF9%SU96+efZWn-%Y`f0? z?%ceiue-&(qtBwAr@XYwlkUlG-l5j|ps8@;irvU&bslz}gjg^G4!>-75#YYBG5vG0gcBwpy!AcF(#hA`_7=pMCoNd?WCj3Z`>iEozg;@UO84&3jAN2>ZL4wb;uy697FD%%jdiS?$|ADFNHr#kBVg`Cmf;QDV+O&-i&(rX8o0z@tW z&3P((;+K{YAOF!r>{*m&P4qh$7$YngnB;#O+>~6b?Hygr%q>ifUETgYtZC@}>)OX> zOHF9Yo(byGKBls-@PU2-WG@Rx|8(?kEfW?;&J zG#(#hh-s)r$H|4otY^=!lWcOKfPlYnyHFMwVt^65_S$t0W1e7Hf6*%ELDsMXt4$aA z3WnE3aYh0X^oYHLU`9;14xG}#wKztZBAZE$TeXuo-9`pYyAFh{^MRVR*)VZ6R#(^c z#_;t<$IUsjEtlBJIX#+C4b(;(txdi9RWV`VVB_O2*fw2`z2uuh74?Vww1Tu#`epg~ zW6i0Qyj!SMIfoisiex?oW2PhqE`k-Y#yn$wjPX&dG+lcW8U$+YLK75GglExjsnk*p z+R05OYZ`pHwS~qwSLCAX$@;gXLZ$oM3n?&Lwf568of9|As@uI)A|B%Xu)uTUd<}$A zC;N;cW`zpwST=+TU~3DRCxW5~D?(+rNDJAoc#S;REe3LyU{^Q^^?!|H+}s{sgQ!h7 zR<$^_Mq{uC9A`dE{8cz+YQ9jH{uTFKidk$T`jG#AcDfIRF463i4iBVNuBjPTeWRYN zp<=FkatxOSQ;Jc$~aA=j$HHN!iNkqH?j4*xdkNEh4s-K)1?lO7d+B; z>IDkU+6qeft!uTm>82IWV$-1qs^lS{cV1J;l8KtE*UIWo;pP?vA+cv>_cQGQr{;y`^@jr{e1FW3L(9<)cw%5GupfvCLj^GdH5ZA;?e`x)xt zyT1~}U2#>70AP?J2_dcE?-Q5*n;9b})!QN0kf$g+FL9E61f4KOM+kKnOtDky#b^H9 zYT;Zf-o80;m~gRwg!TpjNyeD}z@Aaigz@HQ z`hqVc7}z8{7#P?84^IEZr?`uYql>DWi?N%zrT722C}OXQu89%ANbV5v4Z{${5Stz2UWhDTkMBA zZu{9SM3PL1bMp1g?vu^?yX=$ft=`YSe>K6%Zq)(yR+=ag#1Zxg%DNc)nMawVJubYA z6Chg*Z@=J^=EgGxIOFITI`%Z|6+lZ42mPSH?3^HA+JiV0333mng=~PC@^X#BfkzMh z)upQ#2Q*GlX7koTiO;xQkL_{MIOnZH=K{>lc-x4=;^L@6p~4)bB_BK$X$>Yoy%1BZ zQaWjKz^0}jY@yV2H%Nj9=0IK|&p5nT`Uqw;d z$*=j?N*xwX${j-%+NweN!xe!nBz9K}uch)Bwgu&tj62E{p%WIGNdWm3c913mmn6`{ ziH;ms9-aF;!rN74JiFwhXsP~kOPkG!`J6sCVLx%9;9c@^F~2L4dM)Oi_~1OONLn=J z^psPY36X?czZ67K1RK7U&jb~^C{_E!Mn{VpCx@^$f$fonY9xNlUU9H0>Yp%YV9*yP zmgrcsKRIAT*GTiEnC7P?VB;!o;x&LghpL=AL>>b(+g|*!DTkwpBH1v1X47cHBYSid zPB3y8oRRgyQC|Z}Jju9bs2fUwCfz8ayT<-Qm*^Iy6!7|oTlO&J(bLU1Wpf^-miYk> z%h=*jEnJA+RrVQX-}$#P!(9O*&?(iYDClpGQa?h-25mV`uou~x8^o&FND>5&Cc4AK z#O*S3sk3NY3lFeczJn5Tnl{~%yMsnLgObVW7*d%UyS+Km4uE4;QlW?g%uKKRoneUe zM5di##!g436|u{Sh0}5#yjXqj6HlV-Xg0$nKmFS_+Ztt}r46s&(gndobs%j*M5)?9 zzvQ9~K_j`JaUQ{zlc5nINY*3HW=FtOi(j9S(V2zG6jYSv#PKWhW=(wyp=}bX`Kltr zlkh^?Bs0IFi`3VxL%j9nBxyQq+M**ZScQC6Cujg3-teTs&BaQVXvs5J0iid937CKW zJk?(VMKKNeOGdg-O}3cRO7=BJO#P$3TH{VhL&;+8)}leG3eIqXwL$}7yheEQu0cX+ z`L56hDw{*L49hoKe1JFmHg`r9Xg1hG=}wp$bO0e>zj3-8jF#vK-UK%nA0frZTT?1U zd$d>JUSF_BRN$xRX+JjK41eiHP=ro3AEdhL*1EbXK?vEWD_s>MILYjnb>aB;N~AHZ z0giW(ERcBf7agh>(9yKlbQV3^J8nD7{o(PVZg+Q13X=BYxl0mzAuzBlxByOJtOog1 zr9b@c)SfrfFZebw&nXV(o**MTnAU!FPWZ_Z zXT@0n^#Wc{(xQL~MR#->&f!A>w%NCy90st9K#@VLl>~92?PHTY2o~GTl6k+_7Rn>H zi$n<&-OHxYiackd#f!4G*QgMiiE#VCa78@pXWy5m#SBK|JTQ%J3nRDL zdf@0X5b)fE7R~i$lm-3T&HMQ)lTO2Z_$I9E>nQNw ze@`My)b(BeSMLXxLM@Gp6-aCqt5?`h7)<-`@;0l*91wCjbJFY?d$$tZ$>oU@@oqWR z0dQZ1U7=m2m+ zX#8$ZH6)(?ieibw{%2t;$%jv{$YiLvW2O?L)vyGv2+xg&+x3G*4!g3qdaP5Iu-4Gf z161J626fw-QU}$8=5+Xf7YJZvfg)NhFwnKL-P5}jR%grr4YpMOL&zCn6|Gw?`Kh8I zB~mzDGDSJ=X4?K?ptP!KKSpO?O&a4n-#Gm}P4X~y9nFD?r)>n-P3qrRbC98@Q1)8AT2y3~MOC&%#c+73mmdzT%Gf|?%pYk-Bv zayARUlvdBOAMJk9NU+sPDfx`jn#Z#rqa=y)b~c=qrMVDJuSQu(#4{_aA{jGt+6L z@~rI+LYNiV@8p&K9>42>^~kA381Z0+BWSpyy@4j2;v z_uH(Zs4eR-F!2x@FBSX<+py3HYyTQU%o)5vuA&{V&#C<==cqO@${k3lZ(89dApOM} z*N#0{O)l3%O4-hmd{>e#{g-}84ndtDoI+_rci7)&<}}ca=bLUB0Mnd3p#LrQn1%6! zzn@l_#L9Kd77~Mcx$A_uU(f0@XgE*Frf10HKvQpePdxcTzVU&b zXq8#5SVKgduIvVz@&*2(!#4EOX#uzSE~1O^Zxcmov83VRU-^rg1Kq~e@*lYaJ>L_l zeizwhTs)gqnGqG;F#S8(iWAWe1K?m_@X%mjKmJ>?RjnN??aco@l}gc9L>I;Y?$Cs_ z{8njFA9{@}eEO~(R``RCg~WszGP0)I5G1W_XPLEb|6OQMOnylyhzroo>~$W`n!CSH5;&?D4iLu~lhCqb0VsfD3Vc zE-e#ykfPy&iCGwaj^)0NTD!dO6TkNCE*%2&f{26wCYpJnQUBu5O>bE{ihibIqPd1z5TgOm3 zeKSS4t<^cNn64*e?le`WJ$jY7^E1uLa-fMrU=(-a38HJ8NDylHQF|DWYIFEkniavf zg27&UT7DdIQ19vR)hDm&u68}^y_g&T#aGFMLul^W&$zZTI|R$5345dKL<7~Z5kqU>ViaTUWGDM*fOq=1{YOr@cyU_vU_K3Jl^&`ym#YMo#UPn zV%f67L>oV}8FXHXPM3annh4XWX_9K%tF%q8(}-y zx=dxVNC}B03ZlHEG|2eE*ZzZ#+JPt`jDM8x{vSSs|64x9%uVe)j9vay{r*>riQg-w zhAAq7YGUHz#SAErYQ>mBKXRtP3NLSZQgYB>vSv6hcNK4iUVbw1zV7Z;2}RvfBIbY( z7p%#-UOPNx`#x=FP3INm^@2_9g#p0^i)&~*^f4PL+h`rn)g5GHmuz^`;z}Oe1KVcQ z<;wvgZG|M+KY=i!J$L|DK&ZbLYmL;ph_xWQvk3oi5*1X1@omf`F*$&Q#jv?8DV}Rb zrR?Y?dPh;{oUF)YMv|`W6KKpff`#4#^L08veSWN|glKC^iZqCj94D#xz;p|Z!8j~3 zm0)|fW7u5Wu+ggqB&4XH+>4kKPZB1Gcn0yx(#IUY0uXI8l3CPtb9J6qlCTgRL>aW> zYF}fvEj6v;hH_>IItN%tv2XJ9@*@ysf-`Eb`js+xUvqhit&fkg1ZUwoP&nZgA*|oS zUn+yf?#)>ry!JsY^0u<=HLQUOKKyN21a*?U5k@e8NesXJf!Glg9n#6UP7|31jh){Z zl|x(%nbLJ@<_`8w<{hSitcP}49Ke&WRKY)?n!|;+4zVPZI`)${8>{|*b3H4nvXi8? zJoX5Sz|$LzuQMA}+}cDt&7j<4z9s-JvKMR+RIKbc$48CjXn!XIQ%~K(_|jx-@mB*I z@fd*x$zgxh1Q=)Ju=&IEDpCyHo)vrxDS;6U*8!!Wy3bjECUeXB&Hc85I=Fm6wDi5)yl0 z7jKIt-7d^mT`pseAeGXXJeKGwdbWipB#V|p*pu2U0@yR&)95jUOUG*rVw75Qgv5ml zUSeoJYcr?5yT~XzW)&LA&XdVV^o3j{mdpWWK1Oye$z8(%Q=fgyU`Q@zjtM1Y1)PcB zr6i#QRthK^DO>=*9%3z0$`2y0rqJ(lede`s?{m8>9}I80_~FbJ55uh1UB%7{nJ5Ya z|D%V?7fU82{6EN1!+?Rw{kO>d!i??rjehCEmDsmw z(Q?&M+b~ICpd_SB$d!<>LE@Y)VH=8Q$q>u)W~Bth@3;10|Dr_7nPN zF0R(6b9ZtDzLFoF^N57LO&QSyqlnI+sjpYME*@9Tc^q|AW0@$Y zA6&Ova7(%y)XRkv@v?-@;|QmP!gSz71!)b{6U@)y%U-a{$=?vUHx^{7w;PqW)KnMCUOCG@^vBTSmKLqHZ5HReDftfjC#pc*ji{-G*qGNeDP=J7i{Vv5y9ps|gm6A}K?eIq-w}33Qwrw6! z1~9-ZUZ(w6R9J6=AX~A(Fr&1P3L}A28p2H5QIqWF9JAXK3hgzJ+sMi)(MOT2UxBvM-mHi+G-Vnyjra+XIom z=qARsVC^Fhk=9RBzivovrm&q`@E3Lb(nrPoFd|I$6U7Sjv#89vXcZb!I z1p_+Dz@1#yU9`v1Yb+O)n?KZ>uS5&8e3sR6fW*EWng(sSK7wOgAFc~aY(tziz&S5v zPh{M-^b6@{d?@Q-_V$~evTT{MYZX}5nD=use`V&FdCLl-t2%Uq*_-TFYNL4bIx!ca*HS`OP=lVNvsQIveWis#Ij^GVQGRU6OwM0ua>Z=y%$3MGrgThLhCbu z^*LX9x?kTA-af@Q2ntz1e}O%6PY|BHEim0f5-#8Nii53KPZFWBI{lMu$GybqfvXs@ zW2P&9$5v0+5@?!Ro{XQ@6l!(@&4l8?Buudv3qSe*r8bu^Q9h|C`3$zpmt&#ZOp z??S_YK}-OD=GQ-UEXqU={azlOoWxRgRtr3qGr50fxOR9t^8&lSz>KisO(U4)IvOq7 z5cj--WnF(;IZZZ%o#1RLx3I4aT;itXCcs*qMiyvFBKRXHcJlBw5JT=Tw8x?OMitai zsts>pA}Np2Gnh{bYP{*$t|Z7yYP%a#(}XIcHahTZ8aTI#kpw}!n$Ec+7$p>}VTNJn z?#nEN74h=>oY0S|hy$%Cbsw*}s!buJNv%3!Kg7>gaAY#>{!U;tE(0WREINWJbaG4d z*5ffo-1LShfMS#JkRq(r9^-$?Kf~<2!&}>oUCS)09j;(hF<%o$QUSGp+JNT>?SWk6 zVAToODrNwfh}0GaU)fg%|0JHGymNV7h;IHc2+`Sr*U6K*@_^%B%V1%;L~{VGZk(3& z3RhV5jeHjAc%!XL=S0cT^v35$ zw(ZB+m>HKAg6XLHhU5FZO$U5-yI+Y4HQGiWaR2y<)iIR#7P0EsX*uPPX;wWqr`X`) z_{unkzUD!}u1(%LN_>(jg}3%O#V}mh&;_4Fmh+ik*nc%XFj7K+9rECTK&tOcyx-|N zmo-7zmRmjatIJ@~yr>xRSo|(lO?QCaTk!A0^GkfWT_kOl@8dV`q~?-ilrD~_vq#7* z5C23QskONtk;NX8T7Tq(k&Shdr0cC=fdP|$n40yQ@`=cp2^9Im0d-H5IMF>A27`2G z#*|!ArlUl*XK5ppuoOVpujqNRI7Xs)nw+EilK6Hg-`p%U00SeG|25c`HG;5C?u}bA zO{yRcASC(3`*#4w$ldf&|HtV4*Ch@GN4Ni!nScHMqhE_ySRIqYq|`99Q5(@6QPgQz ztq_7yJ0gb6uSPf103x|1=nv+RUz06Z^dpt>7AI2|**oo5 z4FRs!5WBg8-S54a0VWO2;zb~)oI3JIKi)kX13DF zsLKqB0;2stO~2sw5mvLDN07x9=YkY~oOdOscJNt%@v!7-9~GExYP0dC_U1#o2s`{O z?h@jH0_ZZFN`^7Bd2ufAko@9UAnO%AwYwn|N@3RyDAtn5m z18tU$Ds!Jx8O~DXS!OPo`h%&`Aj(X#A>SuqD#*A&x;NlSlv7>v{phi5;HHt5(L}4C z0x^wcAVIL5`dh7($|<3!K6{T4y1?DET_X`#|D%HTmM;kwc z2L24AQ`+@F#0?9Xb&{V(cKl?w9;NAF4Jw$*+Tg1Dy+HEF(?gQc2ZnM|1MrP zb3exyjanShF>SB^>y-VEjRT-ij)C>9Qv|=x^dOj;Z<*H65jZ^O$v*FUWYZ{_wR^}q zY~lg5BB&zdYQgsA32my&%;J2eO*$ObbN-{bYMB`+(3D(ACF-|aAtgGM)l-mL#M68B z+jtJkPIqi~8)J!E$Nw2M|MSnKy%q(+qWL1?*D5%Wu1#jvwy7hUTKZ?ELj9>=A+*t~ zHI8snZp|m%BY_IJZTM({1ki#O*&#Gy9%UYMMMxg%qxU766QC>qWkL2)An{6s_vd-n zu3Sj5jEsGBk!e14=A!6XJa$gnt~?J#Rj5e+11;edNim$pq{Ess#(ODU@=4fDG-J$l%aV3_jXznNR9YpQY#^}cPZ2k4a%thZ zN1LzSx(r?xDfDPLO&&QX)f}V+Dc3F1a8+M%9DglwWHTXnIV2864GZ|3yjDWCwzBWm z8Z5ZxvK=>JXYVV$fe238+b3*jERwiE2cx0L$sPX>XKxu5XS1z~k`TOs;Ee}&cXxLu zxVyU(AT;jo!5tFZAvEsp?(VLaZ=HSa7;Bw7#<_d_$b+uN+Q zp{JJvPh(P<9@1U|z1bp9LD9S#wff3xk{JhfR8IK``JW<;R??-0-H>qk!lS?a(Q`hc zE4T7>f3!x_uE<>S_=`qcikO`AmV0dC53FX(zs-FYs(3&%@2PAe@G=sZwiB5LokFN)w@4IqANGT~9Qw948bs=q3eACxL!uKm zgA=x$;EmeDN*8*_3b^q&Xk?6nvg#X9Z#4Mo+fL>m_)s+0_syT5EA9DL3a_zETd6o( zh*CL}eqhs5U=urMvmt7`tZ_#&Q;pi->87J?Z^pNVC7b5v1L#Rd^WpStCjLEM?1q6H zToMAR*P8WZ@Og@&bge%1-ovZ%tr#{*tIP{iO>5#FaPfB^Aqj>;qIu==D8|EhdUeuX zZPC)UfHjeQBHn1qZiHG&PW~+0^vGl$`y-du2#smg@r{l-g?|;o%v9#FF!ZCd?gF9d z%87H_U#!P%GdYG)M;4nn0XuPC<)z>L3pA%&jfTIH&2T}0t7XQM81CwGf?L#);|Xb3 zJ&hBx)Oj`gzy-6&aZjWg)}QY1-c@kuQ78oK@lvQ+bWa7*EWyh3gPhyth1_E$&}fk? zME;2n?2(b`cNg0q$=vT()_Z-}&WOPiMgj#|2pk>+j(H{MnSK@~mS&w-rS*d13?1rn1G#3i_MU~}dm_Z9Vc1JM!sK!tnpOY9)hFsv?ek; zZJC{Ec-%JQvFPSL+KEd*wE}~(`L+E+o_~z??)^P^HkE5|r5{|5w#EhphM@VlkD$_E~Z*bZ>;{qr#iC$jzrCX69gO$Y!B}LyAVz77Tr?>s69+h*B z0TTC%dh>DKnxEJKBpJM|g~j=h#5n3;d>F95WQ&!n9O{4oKi6!gx=-jqC8* zS-Fklk4R1XSQSlLG#`^F0 zz6NP_w%kK2aBsVp(L&Hvd=RtYsL;P}X~i3NzPp&OK2Eo+ z(wDHkny)t2EA^J;_vYfh@(_jum9zT_>mOy!r z{d9|2x-Cxqp#5~J#!!o9iV7wA3#8VI>mRsT*7;r|@yBB$$Icm7L3a%*2fDx92D)ss zu3aUQUygq;btan6>6_z>obL(LZiid)*PvYTqc*k#?eGfF0OXIT$y+Mn;Y=se-w@POPtMmE0+>4}Bf#$QN{wne2V+%$7uUucmpj0FRYcE_!S+8$P? z-PL>$F#(k)x1UNmmgDu!hlO0IW6fDxnJ_neS5Ou=%-KUH-KAZoU9>6RTM_?5H&2={NYbJBqHzzVK?JgcTq(x68iXKzOBjQ5_Q7d4uF1^Z7*P z+>Z(iIDUKEZKJoYJL7)jrTCM+I6VX2@=$^ zQi%};r+lYMW7D$pxqWQSH-;O^WWO_&%w@0eR~@08Os zkfzET_MwJ#b9=3RJx9BKy(@mBJ19mw&m9C6&)w$?YC$;Uv!|+GJS0QC&KM%|&3;yOXJg{0pigvRxR34`+uVrmF^NChHY1CIWE_yj z!HPfIzbl!LLc1xMs4u8?X4yC}L^A18iYu^p$RXPO0+mhlp+aL*Ln0c&??N4^C;CQw zoRBXQNFOaEU=YI@eU#)xb|^v0o!U;)F`3oVW@aQMusQa{7p0ZunHbYw+(^X9UnF6J zc(t08ug`aAJ0tfpu|GJTzh?g48!Acqn*BP4yDDKpNgAcH_L~PoeNwME%OuO z-)F3)-Z#a|>`p}rKK}9U=xt!~u5*!l8Y^nXypoN&RtCyO#T*zhsY;2%H@sjbj!P?D z!KQ6Fg=f)JC1sgjyf!j%P7q)a7iDHi@Rk7MhB-)pKc_A7g!O(hRnNfw$jW=xgtm+p z%jqM@@NF`yPaX~xfvqzxElAQv+H(6-nD1?8 zVp@=tOQGW2eP`nO4HmOACC9bB7K_B*x+*=3Ow+bRfpF9%$%jfL%?e(1fJh2H>i9Ib zs^T~=8r~>LlRe-W0KLr42pV2@ctobzSfVL1^bZMbY#PQ&eI_{uH>z`nq-ozeM3Q`a zL=XSc`rwF`9YC&3Iy5&-6_2ziJW@~umZ)K4-Gm8BqxM1tCe~+gGY7>5!EYdZ@!>sD zuu1hK>V3TAH6WE_@AG~z>2-hyZLe_Z{pc+OOUK)$#0t^QD2n$mq&I?yOa>$he7wyyA8m0^b;0h@vYNT>{TMx5V=FNh(*e`E29^=x__T9P*v zA;E^RbeJ*~KqoB_7!e>L=vLqnh9ROf{ClQr0%=ic%Yp{Prv)+w0Q!X+WZP!KH;w4P zj?kdp*vhXomB6YQ<_c!U3Xn|{=UVtC4jotv63jg?XKeA?@F6rgce-l=>U-mYL9at? zvSEIJA({zA=%_lt-5Oh-l`-?ncjP`uT+nkppdA`S7i)`AtjXjfF+5>zumJtNlXEtF zQ;81j1_`!|&7vRsZR8<5>`)Z2Njorxxv$B@TWRW%7O=S+n?-|11A4c@b^`*PVi95I zK>thML8uoaOJt80unm4+R112$)4+r@QEdL7D=H|wE6^FnTa zJlZk9%~~KC0KopoiLzrF23VdBybBHD5OxdpNWcK6CU1^HgJ_7nBRtA6z+zgUMr`03 z(xTcHzB)j|0(;HUpb4kOJTQ5$Gt53ST^CFP{Pg^Wy$(6an_iG$3Zhn_6^q`B*yK%eNO1AD*6%Au zy%(X$o72!BZK77W6}#SxrsPc|NJu?!Z`M|=IzYh^`;3`!1#xk|XTK7dJGr)B1<8g10ft0C%t?NAz_@9ZG0vm7(!~GQi6Q}k{)S8m~SJD4h&|e4G@3Vw85p@926C^$~sMkcz ze}JIElyQde0G1I?O8qwEM42uq_f7EBfHS<}2zfC{3tHqgAQ}qHTvH}s(pvvdMmiWU z{6F6s|5b(-B8crDG1;a6h_S#2&Lw=5cx?60Z%3a+IQ}Ux|6fwr12(_>laFD78F+;3 zf(rB+_|UXnbl9LoU;=ZY%cO@&!T{!{^Q~FU%J-UDcmEp?x48zFlN=h5T5D z_Jeug+qe%vhV+N^`SM`Ak>8c`*&pcz>LR5}|C2xB3*tp%*ZQY-2p`l3iVc;nPKX@n zH^c|`4VEtd&pA+UxDWap&UN@%xpX33v<|2a@QR<8e@y>i`QiP^{F6E4KE(d#eMl?_ ztWWo!&R|vuJs>UwHxL3CJ`;TQVY*=4FxuGcBKg7ai5`joiWkR&)&cv1bHiyvdV_ic zXXCQVHShHOKQ1jfDG}v|gn+<<`yY@3l>aXd2)PC*Gh2vlw}Il9b~L}#@XW%AXZRmB zM%XLJV#%}QhPo6k^$!rD`8l7CVw`v_jdxLjB<$0C<{aJ~*I9h~q>q=^!x|8cm$LtC z7AGT)+QAK@=p2=6Mc^h=;Yn;KTT9zc=FmGG!lnrV`q6Jn6(`0O0mj~zQf<@C+sU@W z7#Z{;{vu6k-nyEi#5@8yWA<8g%<${fr7L!dmmZtvTZ-9F`QTpg?8wqFO7Atci!m6F^ zs8s9V1+O+&OSw^1(FOoq$*9N%X426nX977>k8OVrYT2xOJT&&eh+jU7r`dBWxHa{a zJFsI4gVRxUkm@znF&rS&3Bn(sw&a%s1t2wFl^Iq^R)?RPBU2exi*oPc3I-A>e<$5v z>^%Df(o7=-fAoSq5pK2gCQ0*hT2dM|Ss85fi1ud;eJg5ULl2jNZDYwHfv6tEyGMdt z#P*34${SU04FtxK;9jH?qQID=Gjj0syVCTGfX=*F1ia(jZ`Je4#EnE6+|FYQW&kGm z;Oxt^;geK0e1HnIYI(1vFs`Osb4N#)_CL-C7Md%zGE-R6vmwu=3L#;u_X+uNM1O1= z3+eX&ES+bMxuBdzS|Xlt9U-WUrROQh^J*ecT-VODmjMBjJaPwXn?511ide=t z+Ds)h9{OfKB=LYfk5Dq^+xdBrmPmk0eJe~tlX(0lH|zRaw0+J<@wQ3M3+O>Fl!ND6 zJYOiT5c*RPWgB9FL&C|O67WPBxPqXBT)-F@iOeL?NT*Z#=c)9WkDkMFd-D2AynCL+#S-EL4ReaUlMMYeGBvrVVpm9>!x=bK%EWr;!7 zjyF`H`8k_%3>|1>cHfhY9m;nWvKOv_Pq^LXTs+<6*o#hF8ke5<>#N`88f?dH;71dL zE!c=MibgBh%vBB?Nxcy}XmL-Ae)Dkv&T>SR7$K5`$gl~Qk`Iok&1jZ=Qmio>akU1w zs>=0AtAYeJD=FgoZaJCZK@LGe=TdtutQk6I?nyToa?3ZE*m;XOtMj}S8<^shuAu29 z{W$rAP1^>iV4znOK|c^I%MjV}&uNYduooZ7-7zM<>A2d}dPMP5X>Rdu!kHrc!p~S6 zn4R4X8QFiS_?JYMafjh71wxC2CUJ;y0;S1RD`V=s>G4EhX_UcY!fHR%AT)7Z>EfjD zIf+O;FU|0`(UMW+>dM!eJ`P%~LDd(M`DWJFeFvsJ+cbLc6b9Nr>_)%s_;0+U*K-r*~jmN^!bAoyI8Z1ok#bM8RV|0SRDrB){$?T0-9-=Cr z-ATojI;@oHC}_03fz-QRQ~N>t$%w%m`5)GsqAJr2u<93YTj8!k%W&=NKv(9|0CG2+E4|Guch=xup~rA!EO>3@_U^|jQX5CqYS!% z(jFzyMn~kRWw4UqtW~1H#-d*+5u03k6aM_Q*v$R#0!&ZXR-3T-kM>RYe}w|}o>^f^ z2cX~TRM_H_8Wk@3TV9x?rvQ`Yt;H7_djnol?Dzsf8UY)gPwhw*PzQe-EMiZIQEv*% z#f{VV#7N)%15`wwEBPBAppyFlmDqn5RN|JVHYSF~#vf1qgCy`TSn-L-q$CT^=apcf zp!^{mphhQ4{#EqDQW-uM;ar4=3SoKGtkw3S_<_);3oD8r=zA-HeQqNI?l-cqm}+)E zALF_oX-;^(zVBCn_)bd$G)S$@N;l*9Ig7qhif4gnr&(md&SuwKlyB=!*w{RcF8@Uc zKn*LFYY@6=E3lA;D3YqCSE@*6uU*CIh7Gdno;@WojEgnNjir|ELu$#I%}n0Kam{Ux z{+dpo*<$EeYoZ}W9|#M=5X}evr36WtQ5{A$ePSAg1V6_%ks_ zLO9UPY_eA!x2bM3fFyh1CBRS}pDS^f#^!2cip(B|pBVVrTASNdLC?Dg@RwH%q&fLp zv{Y>!D|+l+uIyv93eNCeVcKzS1iBytm@i+W_)LJbNqqG|b(nKNV&>$VTjXON7jvA< zwqL`}r!*q`IO@y%%oSRTyY!Z(P$0@x*XD8qwO*#j1X(2}(n8kIm(};o4o*t{{-nD@ z;QrI55m3PXyuYus7s2M~CtXXHL!6Mb#>ntc)9Y5kEl_XX*SZhnf>}s-FHHB?H8BY*vHhl{mJ^$Dmu^bn*i^=27iGFXNP0?#+Ea~Y z7;T~E;=LuejYKVrVZ82920v`z8@m2e-A0f=7a`f2RCgYfVK6x(%l#PF{+f4cPG`qE z#P1M%EDR9E5&$v_HVNJ?@A})OaxbwQWL3?=mwr=^5jkpENVXYWPJSH^Knz`Dm)zZF z7WFi|uwOqq2>SDi)yV0Yq4%3rnR$L$RlfOTzW(O>J&V&YLo8CS)+588{HJ?EjN8-S z;wChgr_s!NV+$G_@s90zFIg^G&f*vcl$NeEQQDr#P{EJhGxU}i}W81e!F1!%|XlKZm;^>HrOKPOx?vOG8xc~iq zh`(@kO)X>8o<*v^s7;|cQ(q^=sUg2N9w%|e%RD@Im}Y0(KlYgAiuo*M2U1vrwq-i2 zyOVnqYo@)rx!WrsU2c2hQx?#)(P@kw+w6O!jj_ki;PrpEBTt>b16&`Ak676M0aHuN z#?8|C|AP?;1g!@POTm?>DMgs`-uw{1gssG@*g!|%oH~rsAUf$1ZaK7>LV81g4_@0i z2o7p|qwKez?n)$%#y^y;(eJRy_I2-cZxrzNeMZ_~((9cBGoF_s4wpM0I%{)fsIOv9 z?RMBNGCL&QvU~Yh)8v$E4`;jZHjlX*}@Ba zX3AXJ_}dB^&i^^im`rcKD7Wln+iZxTyL^A0re1^sJ5P-uFkt-&4lRfg8-$DogqQeK zDEr;o=FpPAK~)4gudgz>prQKO5x^4Z&O%hVTNlP6$qa~14sRSKW11@57Dq(iR}U^v zk8Nl@sdP?926Lh9Gj&hse^vf64QM}UDV(g-;t--EJDW zD8bcIdmz4;`6li-+ry?~F3IXjUqK6&PT5*Q{G)&L+l|Irdj;+)s446Zr(If86c1o= zd!F@+I%PO1Ga$G-JfmAzqlJ2W+Mg=yPDnFaAk=`9ga<2*D2?Ap4x||e+&QU<^=Vep z-Yzq$Zcda+=IeJWZ$N$GlI{zXYqxlO*JObR3apMc5#h_M^*p+u@Sr5(1<*|KMtIoyi_{REjvRT+s z-EYV!qwIu{-JNOUciHq=aokN|8yepS@H8)Rs#`D_1mNQ?Jbp`(#a?Z2<F zvM4Htd!v{!j4E#**({d@DqK%d)>%Q^npbXK7C34Yy=0~mN+yp%4opf&`zMa^#hL9T`m#ZpTAS5|jv4{B5Z29Us z|9MBuT=kpx+vhI16_$y9hjERJhV>ohjcBLFlOmFW{-7>5lLM+}p_hj)rB533Rvj+B z&41m~!vnihF!Vt_6tYAC;{bprqXF~C%@^GZUxq!#%;lX;aB}~)V3z}%hVPCIPs9Fz zTkm67yNu{EPo7xvcsp_1XSWqvcOJbKIKb+>6Mtx61=4nQQ)u1A4)2jLmQ>%$V zs9|1;PhGMh!hW0~qm<05{|FhA({OPm4)xe6w+12+e)XQc!%)bkGXfS-Xp27` zY+7<#QVB+a367?2ta%G zX%$*Sq~F(P-LfzCdH3l=s@b`H2ez>KO z(6Nsh`-(B2I)SoJZrZY*Pf7`7ZYG(>K}C@=w;b;3BvpELH@t%Bwf0 zd*)4%NFsIu(C>uZzh(KQQFmWols&ssG{1gGbrV^K68+xc@Lr_*>mS3n;Gd5i@p}UJ zT!l!^(wh`3;ZEs)Pd##QnOL-a!u}Kksn3inE&%$shK-V^ob@;CD3x(jB%|&FsqbzV zwIMIGW=h0fkBkWTk7-`cGsP<`j!}DuEoirAMgdE*swYCKu?2{~fnlr&}-o zI2%BJ;8yg%A5p~A*wn<-&e-%nU3~tvu9?P6ZW#V7hH`->b_!DiUoxOhiYg@u&pvU% zPm89xaK=7O7m%~n*@YF!5A?ZRg!vcy6T7cEnH7bgGu59t#FehEmP5r$7IZ5WQR znQFA;%KK-Rsjd>fddChn#G}C6oCS)|LBPXVHifOfc1ZZX;H#I^K+Ud$_nIyWC{VO^ zQQuaLbd+`;uJ+knNv%%Acp|i4c$A+sSpjSAk$m3vtm0^zy;?o0cHwlJFpM4bKr|&V zG2Bkah%vn)UTx2*b*zBRo-3JS@T;78vd>^F)2$9U+~xgTFd3KR==4xJlmMQqY(lS&P^j5^fy zFl%~JY_+5?bP|?jbhO@hbs{~j)sqU^EugSb^VXgK3?hB+?<)`H`oC%=Pku=wokMlPjcQT6N8 z@y}*SADthkdMu9;Ml<jmN}fIbguLy0>Uks6`PSgj}l0l6JwG#J?huoL)Gpw$_g&(qCNzmG7uwwjLT{PG?HivsgIyH8!{@H~7#jVw zB_?2Tm)v{m2t%y}APcfzSBc`!4?QF`0WqO6q+PTjsjtln2i}l$jUgQ;-a1hR6cTsj z-=8=zcO)Np>h&cpJ7YHP=P#d`q^qsP3%NOiJFkAW74J+hssNE;1@hNLH?Rj_mh%z= z36y&dW$a~fcm_KJU;e|HZUwqt^am6K1R??i1lRu+Y+`m6|J)pF^1neRYwBV8U!2b@ z^x$L)2`F#q)chJnI^gWYNDBEQ9=ve%gB}K8KbBIFnKeI@g%}L*htsn zP;sbKD_d!=^iTZW4sVF{u8UC%22-7MEJmG>A<*kjsbhJ;cWwq=bLFY>XfRz&ZsVW9 zHs1(sOGE2~z|jMPSl!m610Q7?XVBP*oUN2hTDqESoHHF3=K~AM zy6lON9XRNAgR7=K5Jv_Zh$yWeOL;F%V9N4ad2J7KS9^fuE;7PbhBcGQG$=w1?OD8! zHPOYrF;2SVL2jQ1Jkv!ep*lt;P!&v;ei=3XC{sG(p=8^a@$KKQi;3w!HQwkT|VJhV92*cbkZNsx=Q|GI|-{D<82y28%Lc#O0F zIAey9SnnzqOIx;q2xV^+YIuyOP%ug%`}J#VA2Tzwqh{he3aIKqTqUii?x!(eIp^J} z#VQr*Y=AS|FgNE;g%KaD{YAAUE#2=Bjp|{m_>iFF@CggsC|mbkTBjHH zU?8=t;HL~c{kAPAdLN3C80C<%kPUB~wC|^nypZ0E;mnTuBuy{lXsr4*S=CZL%CBrnrWYb(^g`}~+ zw{K~TVL`Hqqjs-S7-@NxXBN%fYn7Hhv~-%t6oX4#e9laYtJrczs_%5dv*o4|f>v1f zOAyAiUalBn&ryI606rEfI#TN}J52-FJ}iL5(8&_ffq_HWdvsJy-)S(8oF(+u;&4^r z3n_Tbuq6GR=YBt!)VFnIwS~&N$hWBN)$KjB({cj5$-t(ZnXiPSnP(9d)z)II`bSJq z2M()xsK|d66Cg6&bOc&LE$J+G?*G1*(PqcFwzHo+zS5XMXMxJfsGt;D;v70LwQ1be-3Aio^o_M@|SRzh`Zjb}*ChgQ;Cj(J&c)S^ zI(5g-tG6=_4+aTX@+P_)ENtoESa8JrfXPcRM_wK${N;@*17=Uns51I3xRH-V<>4tZr%5cfm!S2vS!Y9tD5mHPv#zfITd6O17=mr8ER zM7o3o@kzOktb+xA%;RG)YFI3h&tAyypDdkGu`Xuk97dzI&s$IUkH|mgH}Q)81%(ArF)T zoA!0#tVyD0HNKrBs2%I~Usc1_AM|u{iHyRh1|~zHjvoz1zZ?e}v;!*?#Q8O$irx5`ZA9JWZ*#E(CsG_~Ii z``OMDmKN;bEq{Cl5tscKC+u*cfKrIHqxi{%w5W#s}hYb9(9zfXx&XFeSZTM~xb(!)j=Zh54vvm*2OGC)w| z(4{y-edn3gS=HW`LuRfkoJBmVqs6J-A69rR#R{A`>y?_OXouG}8!ii1lY_?LSsV4l z4vB6z0d_kv7zw2VZx3_MF4koD4zaMfUXQw5pQy0|UJ!yNZPNMeCu}|u;e48uWk!Gc z$+C{Z{p%3ItgTRT#v!5a&lvN_7UuUU;FO%$H+@@-A*} z37G|iL*btwblgY*ee3BI7* z|JWD&g(%A3@)-gG8|Hto10nK%FcMU)?NLA0Fp!$?GNpl);p0|n1oh=~vs#kIJqGC$ zOPV{+zhw{ROjAba3tp+-BAyC6H{{+TULdapi$0 z{$sP0ETe1|vxu%~#%-2;UQXl#escY>M-5nZ&Xx#V`dbYocgyk2EvCt|3B(lPU&nqj zp@;wz0OcJhe0U%a6;1{o5Rf$daERdCX?DX*g${eA0RcBHT>BUD+-P}?F}IgYiEmqO zHG-08?oX5mLwLtd4qsCzPXjF^d)5#_R_d-=NE%K}6!e!bJ+EN64qJsf%1umIkcbQ_ z%A3EeU(cwbIhBZtVHeS~P&IH@w#2IGaZo0dXyR~K=^WXekco^{*|Ai-^7XT-rPq#M zocNC*k5bRGPLi+Yyo9Z+*JCR-;C{cOZ*X%heH%f3nwaJDlE< z+k1bD!*92ADkd3Rr&XVqZ|peRKhw(sR01aP%SqUj=2nFVmBz4ExdDF=r_pQf>EU=H zF^u|SoB%Aw5-~#9o|xu+)RA0Xet8c5LX0~7h+6xUT8F(q;^lsCpak|CgrGw^B+S9N z%Xr3U@AN8$O*v5XoOC@)H&Kh>Jc>iKL(A+{Nmg zBb}jQir`=`IWuV!m{Ge2?%?(~_e+`y7?A$}<1NaVSaQsiqqr(iQ%F*-gDzAcU(jm? zP@cLsFzxmTJR^L$Q5C%r*NfkSvKxQyt}&%#@DdOU7hZXYvP-*R#O3@uDpH0VJ32p5 zf%1V0;r~H6qH*@NpI(# zu}ByA-s2yj&ln#HeBFs*&bDksOfjliDsUdtx#Kclb7t=JBP95g`qu}*2dLIi1bmCp=d}1mFt@ zRvf{&%%-vFG6F83zG;Cmvx0?N>PFq044ut~DVJ8RuVm8V;io>IjR0#lAJpZ5bbDdd z*Knj0gh53@{SaWEl9h0^boZ+Rx{`7RM$xIX@RsXX(6A))O`4sry;2JsDA?sNhh&E( z30TD_8$zT?JAS3R7-JY|0qkXi_LZotcQw^SNSjUz-(jKH@o)V@Co8ZXgKpR3paP~_ zF&l8B12F(sK&ZcnDaJ8-3gw;NgsTm&C3YL8%D7jXUOTG|VYsWf$OIaO=2%LCI1Xi- zmh~p4MPo3j7{LaEPCe534MZks_6v3Kw9Vv=G&)*V1hnF>WxGm{FFA8FIz^qsvF|bL zdGfnOR7W#44DUs!i&HN1oDI|TE22P5@PK8~-H~l!k+1`NvlCaRAgk}{{$8u89`d(K zSL@+TKau!O9m-D5oG0PbRv1TSR%X(=hl3NzIa(|;bCn7+J zAc_K{BFoK30|3Rf*y#D%kF{J)jH^Zr$EeV{UlBY6_t z$F$mI@=T!Pif2rR6Ucp^F5(D2++!*n1!NP+=bBw)+6129ZT+VwCbw&xy>5_$^T)m}3<~Oj0O`R;(nC^0|~p z8YVN%f;VhP;Ljm7DgyrxMB3Qc4gIg~HZ*}XaAkC(E8csV&ii`L*VQpO5O^DoKj6U~ z>|n{5;Gr)ZgPyE=%7w(k#8oy{dE3leL+jCuN)=kR8fLXJ+>YtjuI3j$L+xHUUF+gqv?*`~k z?7)`y>8pg4v|aVcpEcKRZQ1;20BhgUp7Oy1)l`lbRx*>?YP4*mrZ{>Aq4DhD$W5_# zFH>a8NMk!#1I;F55JqXyQ<$-+Q+#<%o?BZJ}0F>C>Edn8$>uN0gv&+l1&j z?uz?449SPOl<`)7$3H!b&I2yRat^Z+RiB+|iOrP7(_PAMCAm_`zmMrQBkO30^Dz+S zv9Dv$-9^wos%vvkm^@CwltuPbc(le;t~ld~8mEby(gEXr>!!(bewfJ>)|IA#Z8dAo zeX(mZpm!|R{mL#TV~}MunO$?SuBcVzxdO+d{Ik6_cbZmkAYB&QYldQ{&c?z~fU$F{ zisj8?n(&{liX!iX=K`Qdla}MxN3<@0rN+1yTYJU=OnXhT=$L+2LwI z2Q2g%2E|BnBrYdq7Gs1Pn{J|fJc?AlFHc+H)m?Dh>>b-}q#+L*4Pk>yt$Ii$k-1A- zsU0!hxkpFRY%~@sD<*#@Z4y<}RdBV9rR!!y7!KM7`nW6R5Jgwl$k7*|I;*&2YeW+% z31P1oPkBBkv8Af9L}Q9BZFOJQh!qF82a%@Lxt#IUF~`=PZ6~r&V&xy)&7aqVkf2^~ zXj5NSeuz_E{O;;!X^&rSzBqMsgpI~VJ{#R7cUZ9WhZn0b!AfGJ{Zk7$7MH7R8JS3V zf|crYh4}bkWj>CfX(NuD5XtAXtRa`eV=fQ4lM3-~A61ByQN!mtlWT8Uk0cTZ@>Z!? zU;zKMkHoYG`sX8GN+XhyWB@FYV{meK` zB+OT{sU*l(i$laCz~Nj;9wn_wGa1s4z;l`K4L&k_PLFO}h-1gRC!cmc+*+#(w3ETQ zw`vhbwj9}2)=k^`TUZyOYEyX^OI_3VI70g_tK-EFNm9PgQk2bP%|@Qk-0kyNp;lMR zZ7oI`m2lKxvqtqu#aB94q^?l5UDn4vd&VvG$i1umUAI{H<0Wcqn%h?R{?|(w|NdpQ z8w4e;4iiyo4lOCqr?#)Xh!~q`F^fnw%2JwLk>bLG7LeoI%ktAuZ+erjf)cki@~&ol zN>}_D-h*bv6dzDUNN5FQE(-^_8m# z$4wQJFiY@!RT>I}l-OFQkV>Ny-$iq?N$~Bt+}W>cLyzv+M&-FR`h#5!%v_N_OqKd~ zWo(sk@wxW*%C6X7o~a4<%4t>HNY0;mT0^)DrSg-OSLxlu+L9ny$}92IS&N3|ROxeu z3a+k7Yd(y>EHVs-WgJl4cu?xHM}@7G<(Zc*o;2pzmdGNY+u!WIIAxX8CmO}4n~)%1 zTnqQUBGghmU8H{R;;7~fb3{J|-R*iJI9%lB{)W^H7Ur(z=y9xhNIojOPx0dTAJ9^gP@NvYvXbU$)-fkaxLA?*?1;6rWz#PybJ(~W zxJojjKgU3QtSHVB5cGotPL@2vP8|p0T#3InK4k>79Iw|kXuYciLpJV5oda>-gTpTK zHee#QhP|CFcoQj}Lg0JdUDAWx@x?=Qsb7!X-p&nT>rI$7iC*-!_RwDwEGteV!9LiM zlbxvzaSO(Ck(4hfmTk0(h@HDZx`G9wlQFVrrmYbN<(vnmf0%n|nB=|^Km@O+39;hS zDYwB-)7M0X$kWX$_^Y?!OrH_sUnim801~Lr#_P%}bZ=-#4`ML#Jibla)tn>@WuR-e z0E#^1Ed3GNQv%kow!%ZCa%!%Kp|Wd)(PI)D^87A8Rh9_Q5YSc9ZR11DB>zRProq?l zD7+;8)Tdq`dI>$HXSVm6pOU6}**A3uFg?iU6VbUeQ$8PY@$xWN?$r+Wwj+C7J#WS+ z^1v-}oH7w3k--1!*d*u})I=5;>C{OEDD#jgO;3A8WJ{o4OS6o3$P{m6kQLbq4tFjP8+CW)?Rp5OvJ2b9YPidgYO=ES*}zmcJ7_ z2VhZV)8AFaXC8ZN7>3`YOHi3%RuVd=>RbX!?Zu~Pd{|PqzAMv_h+t?ztMF<+W#+0X z^O}d!WRnm~i?0<66d{&*WV4oqo0<`qg(p$*Lp;}XP{=bQR%=x~7#x<^l@ark>XmqO z)QKCVg_gu=rWtAAB&klppd_h=ZrNGJz!yH|N(47;1y8|prbmRpa%vTDlHiMo-{JMe z{zWIGo@G{E{bkD7>uJfO$~--R^wAoBJo+4Es{ut1j75F?6dp{7`Oh9>aPb?OV+Pa=)$aDsg!&` z9^<&9dzbfZlgz#5GCBO2UO~mkSd{z<1lZ^!O}#8`Bv{fKoGj%~G?`Ku`oy;*1404j z&?-L7oMuc3VlSz^JK@#w{H1A&0TgVdTh@hqjPd>f1i%l;L3PDHRmdh(x}n1DlfAH| zEPtfLq4qXPY5ca>`=tKcB5sSP_t=P3q)Q)*L@fL~ZqDL?osbQOUeU*&FI>u44HrnS z_!8@~7PcB?ycU*3<)_M>)6ejmDd(eE1&Gw5^;tx?^hzEqi%Qw6ex!-hZ{zlkGs!i& z@C>thE$UPI#gz_r<5N$E9_ds1iQznIwC{`V=-2C99#1`zIMIbYk|^gr&>c#~tPvO$ zNJOA_(%iB-jl(FAkcA7xRte_Ha8(ErRXrFVuAp@AfmcvIlu5gxEgm|>oSvXA4DU3t zYH&X7sheZ&RC$TI!7f|*x`|zuSS8(agZ5G>ENV-`LZyXRLAW4Y$L-;d^Xu{d;q0G+ zbzy>SQE*!8o3?G+wyia7+qP}nwr$(Cv8Ijw&yBbbXLm<*#D1xltg45q8d;ecnIp0G zK`r7>g9g@$2{Ylr@x_K%@qzgg#VmT!-pv1MJ-YO-l{~ZP;d_H_Dm=#YvzN-U?4fuw zZ)!i%_O}(+unPG7p$7>}IwF&6UE798$=n0>CEqD|-`Rhi@5$VQ^yTNxe-GM+Ul_{d2l=7kEqy=Q zm!D6{5{^euX~ z8}JkFTl<3F_bcYN_%WMy9+CGe=!1A;|KRQ)!uW#pk^dcS`vv}ie*^WI6ZuXLs6R{z zr~*N)$G3Oc2inK#7wjMCC+siiKj=s7SL|=`w*p-Qu7%XbZzr@T-Y4sq4afp*1F{9* z#%iav=iYzm$Mxq19s_a%zJ=Mw=D_3tUkA&Eu!gt=wg$chw+3$oHUoKu zXa(i={{%V%c?JJ}s>lfL2=)mDAk%@!0&@E4^kLe8(ji`en)-F~A*X@=^o#bB_LuYz z1&9O1f#AXNptX`a-I4Fm_G|jL0^ET9f$YL|qr2nZ0q+O(hXup|;e+zQc_Y1{++prF^`8a& z_nJU@0p6%@Sa-1dS^aGR|A6*Dd*Qzj-VpDQ_sja{0`!3TKzw1pQa`@&|8H0B;GtYK z9t;SG1`Y^F>VFX;b8mq|u;GD|U=Snfg+M-monH;4_-bJs7-H55F>Ad`L&aPR7hD%wY02t8!yMbcVI6wS$0(1dzB_j z3lZ!tI8Yw|BLraop62Zv=5N8BS)Zr*Xlw2%wBYbwGk~&jEVJA_f~b&8o63mnZ$;o% zSE;h#_+V4VjdE(4tg+ddzmz_On4=D7n0{MS*s-$jsJ$aa8-fzU6q6i~YMSMh3~@fG zZ7r&BnORqfa+zs;403K6w`qC|#ECl~K>{2Q?^%Z$XJXK`UYxtM6mG%2-;oSe)8nJL zVx#F?lrEf3XaLIYQHCPq>p?y3XN z3^}6CzH_TR9I~B--JEM`{YGi+u>VinuqlS@+3E;}oVWvU?R;W&W^6Cgscln=GU_;^ z25(bY!OR0v{(9D(RotaWL!=E1$8FWA6KJD4jtslgXLvsE-@om`Rb_EGzkmQD zO|B=@oqg$<8y8#EmS?VK5-L;Ksyd5Qx>^&n{*6USLa8|N_(1U$k>$u}enTxpBB^>u zBa_~Cfy^ad56NdE)jbxLGK+~2E=ZJE>`G0clf^Om8h}ihT?XM`kK>R6ZvOs(Ra#ZG zd+{|Q2g*1qrbNj?r7Xb7x2?3P@`N;M5@qN{{JE!Tzav)f^~N1NxUux~`h#yN$Pxow zv1-MA2yqe5Uaj*=;5S&!HMVj%86Ry(!|X+=a~zQSGC+A;smeZ1TotR<)mF|*V%(t6 zvr6r@8rPSYQ=m5)ui9j>;x37PXKYqXAL)W&Xfl*f5fw&X2IK6xU!~Ao!^V((&3XD% zx#%HcGUg{9cn<2CmRk+%IT_i9I3E6ndYbiGm#We*eJ!8e0emsMrT^hDq<*Ng~xu*mqI8%YeFeLL6!)~u>!kk zw^ZbO-f>}wR-DUO5eAYZ(-9!)TGVT?g!0TH$}3yKmw7nA$M|8JU&@K@gi;i^ERuzL zjb4nu1s1|1NGLylmc_W@kkHqV6L~{5Qb zcl7uSV+Pld>64d9czrIxCxK=Mw%tl(c>GOIV))ngNZ$Pmn&yaYLA`rEJXh6HwL(@a zF`m>Jli*3U1e7iAO@3p*xDf=R79SH_3R9oLlY%w2`tAKap4Xg*f;Z{QuT_k#L95 zCjaR@3E2MyHu$E&Ow&Reww(hrkjG=kUGXQSv6&-Hvv%pBAMpPsdxfHw5UD{0kG|jpkIM%AS zYq8r5{dKXuuSP*wK?oXU@E$j3iYAS4*ji_ON@bw!F=i806U`>XmNFb+m5IWiOwNk2 z!B2 zEo%U!d)9gvXUbrv)H($Komfug89rm}_SdZJtXx{BZD{cfb80iR z)^a9SbgT?*F=;RDez6r3nJI>@e_Nl66{M_L;@HrpM1Qm@X;Iyz*kG3}4(c$%E0No~QQtF4#DHCbomY?vx2v%@*$3<2-zmEY^?l}d` z23G;=c*AXyLz-WVIi4OCaMmD@hkkAe4B8A5x>ytu01DVHX%X^U@E7#ZdxuNw#&{U0 zYL`hZbyVoaLizR4{~}5qmmbX{r|wf;D%u1N^&0ZKhEeaiv)&?K8`ZuyK9&ewOen~K zi@#USVqUnv=WuL^u6p8C^;HlO7KHxBNuPJwzB{D%xNmSa3_g_K9h0CxOrsN-7C&}q z!e?9+jD9Bag%zpJpO3@%M=%O;VoQLeusoU*`xbup357>T8;Q=%?s&n60DN_tNa07 zr%V0hTh_BV*kznno=bfEf)~mt^y1TKWXwSX>AMln6=+#vT4X69<&K?yS;*E58~c z$?u|?jctgf@+4kU@`wEP3wuC=!pnG^h%;{M*zv^PFeo;NcUJ9q&GEk8YF_;F`*}m| zN6Xb5XRr=c+=A`m3GYB}iQTRx2gw_2xB`#B@2Gn^qNb1ZU%_HILs2O5og!c!K7Cve_cjifH+J4HS>*X`|+>?K^>P;Uj9Wo$8kJ!6~ z;GmQU6~#tJJXddz7Er(gZLjMA8c4jLvKt0SQ90p_AivZa9Yf=5xTjRZsMVuk4bnsl zkf3rDrju&ZU2n4wIk3eV{Ukz@#5k#soU`!`U=8i)%~pmnaOafx)g9#8p>|03tx#1_QQ~@+2t-{rP6j4i*Sa4wzMni{A_=T zhT=e1;~tWUp}A-#1oeyATz$3Gn3Hr+g$22D;bT z_0e{cXZg|-;ZC9_i4zz3^P1ekL7NMZM$U$-ezDUBc+UbKTC+db%`e+xFbC7XE_DDo zqJ?E%&+novn_)$V`PL#Lb<{C1A(9m4WW7@xG-< z@5&qzHcTMQB2HgH*7gj$=2$Kb{EOFaLBY8>$X3Jb9c0J4-bxqIp9%nPWr1_G3Rbt( zK=Wy35y>F{sH)vh+P$5x8N6jr+u07Vw4n}f_S=k)4f0-}JN`pP-pE0k#Ci!egu#ef zr8kOp6~n+37K2w;Ya=WN%@`Sv5v z@AvCZKZGw6!$3XeWGb?&JA#9{C#2jMYQS94xC7{k^qK6#$7l*-nj5yke#ihH|E)K;gsoD9?mFj2qCr-H;}O#sE56@32j1 z2yqWP0bFRF@+n(oHM6hbXzS~)gM5mq1NtjqyX_8$P59t&4tt2rBBgkWVNLo$;O$fC z&vC>zqRV+4I<=7u5+G{8-idMf+PgE7-cn(3PKA=CqMQz7_q_GB-MZ6@@8orcEy$T4L7 z>l0Uis;Arq@gusos8y9&XqdgZ=^IuYQg{oPVP}24975C3&#wQk^PV55iL^7DfJk!_FtzV=$C5fsuGR-I9R3DK9CajADc{<=$sd znR`H+?k6Tik5TW-KV=ZVoi*T9H1prQOj!m;Xa((z(eZQ=@Yf074-=ILhKo%fOwVJW z8vlHzHB}%K!AYY{x{?$a(2!J z#>P%AhX0dg;s4F#>ncIDR{gTHsq$W~ZB@|{WJ!!?DQH1F_R%17zG{?qMPK|u`4#o4 zSb$X4`&Jz7no3oWk|`&d?r}1E+4h?2$k_Aq{Q`?0lEVNFeJx#4nvP|#9-w_zxUip@ z53!-%ws)51ylaEpZVYL8nAl&9FyQLmXzq$hwBR^gk01)hHjtTsGRi!d5efe!)(14C z+Vi=(FhZdyr4a8PMb5^gDlK-{oVhKbvP!F7@`Jbfym zn05V&1*Pn?1`O6^qg59^w~-xoY$NSNCe#^(&mC7` z4c6UxN~F4Us7cf5qE73TKE-{XI%mUVThy4l5U z{PLMse=8}hYzv1?D%W;+01veWP*Yw}!MLMn4a^bxfJdXTScBV3FADgoC{jph?A7d!1uo-ZBn+>j3D>NMMW8>vv}U{_WSqFRn1=Hy3Al>ibUKy1od z%J_8fi063-!Y;C+<{B2hs`ag|+CTvF8ewcXV=WLPxY-q)DlT%2TG$YBNGst-YAfn1 z%R#Vnf}~hih+<)gMP;i6ae^s+dK}?WVBCD-0RIU|2YXz{Bs_Y;A6qlfP62iAwnicq z9rcb%rr_duBD&XGh$sJCu+8e~y6VdDqo=eG;c&~?urEk*l7*;y-;%(_uguHNc?Xvx zcdH)Qb5I}Xa_n{iC>%x!eU<|DXe_i^@rTur&=o@HAuu#KKr1$>$;)Js@QKTj^SOL} z9sJG(147I7hLo-2wJh8nyNa^Dm*bo^ZUg+!=o*;pi7%*2f2}6HZO{UA0W#yFRYG@^K zH$FrWXFym{GZKP%FI{rv6PP|n^a+AE3#7rrFeiq%zK9c!{p6pa11!2IeI4`XQJ~|& zeIObC2rd~Wg1Fay!k=RTWBGg^j^lO-(KIO1Ka!&TnK;1)vZv8Ag378DfTU zJ9LtK_0`!58oe&Xr;Cy8_A+b+P}2uI%*rY~JKp~erE*Tgk(N!LZJKZ%O%b}X47`fE zl7o5!nuOMxQ_LEz66y9{3R(ukS)xZh%p>)AgjGiYQ(H$zSxa3Lk9Z#_c>u%~|Ib1f z{9kLOrgeYP`ecE>^@T*IHvHM5yQrnk$tIgK;Ay_ZAoTz>d!axu2=+2K-t(x4P8wKi<8ceN&-*sG>kC?`Ns)sbG3bR&*=n@V}c_22|}t>+IkKx2bqBEC{6 z%Mw2-!vzh`MM{!a!H3*8h!ePie=*l!*6QYN_hcE(gC45>Pisl6hEB zR8Jki;MTJ2Q?+Bs^1m<&sT;yWMH-h11Dq3F@ENTGtz6cv`(3unO2_&Fcb#PGTQMpQ z{_X1b$JirtMOX^zehTIu{Fs6qm9*y?I#}G-4Ans1W0(#i03R^5&-iZ{U}FJ!QmJ$4 zFAVqmgxHq>yE&v3udWts>4d$fz_}rFD9H&UYGkeVY3T41Hh8cB;&{0M?QvVK*mOfg z+*3z!cntf3?og=eCN3Nx%*aoN?Yge<(e#TxVD4`NeM|U!0X`h4{{w@W-6jH zOBZa;w}*BCWt@?`V6}H~g&BwuRT=G_@Vle5qkbGd-}@6l)&XX)?r^!{sA|EJRhMkF z;$w`GY}x1<`3ufpqVAGmX|V3#x+ynWbjIa&PevwqJ8MWVW`=pdbwk)-$%P;|*CY?? z4t$O{ZZ-4{V0PGdaNSX@t$Y}x3vij(e`k#~Ij0)}iW1%N`Q{Jow8zbhkl|o)$6gT4 zo|WVt$sxR9{KkOiq-xcj|1=)Mc!eu{U~X3oopIP(G9twuQcASY47c><%_;Oq|Utz zJKzTG4a0|OrW$Jy&dKO7T6gk1Kw2`i?q)0+ZVv-qhvUb*WBv-X4ucQ5L4t4fBc4Co z-_6WoDm%ynegpkkKJX79whye8uDmH!b&dNI96My)+^WXmY(WNNqn`yW1kd!tm=P{S zh%Uk!F6Mmw$+2IK5|ktv6iISH!7Nw=@dGI-+H6l5s=qi%00D1V-TLqrFSm(f%r)Vh zD_R9pWho52m|0w)MScTi{c^JU?RYci@#zWOWR&O-f6Xzd;9s01Gb;lL^85>G$^erf zXI@|2RK28GU%EI;i~xBtIa7ii5YgCr*@&3)WX>jxU;`=PmN=nxYZK2GJjWG5Qv@#8 z9zV3fCiUUu0f@ip0%vVy_!uJR!rZQY`C|!Uj9^BXb$o7IE*P3{keC$rUAi1>leQ2k zU!w+5RnKl>r6=nUFw8XwhP&951B4&>|I@rm|`dqc99|hiMbIjR~8xvp4 zg(;oy#(d!=%ONltuf3}?inyK=d`rX>Z|qWm=F)4u$SpwD-^7(6p za?f901FfA2J|kMK1cyuTZVZEdrUJ4?Mk_4W8y~5kCs#TZ8x>V4EC-G1aDb9fOIeH& z=7UQ3)Z;X+os*1FoSZxXlUFvrkUNI#HjOfKkmfu`9WEr^v;Sop*7UVAdFkmK{5URD zJ0c2oC?3VbL2KcQ5n8da38X251~IO#0SzGE?Uab~U+nZtT-U zhg1QG$3i+?99mdP6i(A6xro5mN%FKs!8U#jZDc*jG_u)^vr2Wl_w~n-o&-Gx+dYg} z^hJ8MV|RCS(zLH2{zFE7uO@#4;lvf-5Ej2mfaX{V?3U03bu1~r(j`v9BgJ$ zxeYr|FIStODjNW8LEMu$He1EnwT6hXEwhM5N+rOSgzYZhl_0S5>+^K1>n-iH6dtJS zBowm^CJ!@K?b-w=ghxCN39o7#Eaxi6^2eoqJ1KFh2M%3j zu-HSroGaEb;v$Lk!m$)2706AcRppe`+opu`i$Uj8ecyV_u;is4>;py~MRC*Vj0|a9 z61npO$u3bBJkpM79lU5`RWGz;gC5chk4-#pVMbu#dv}J}f15s%&tV>W+!q(yqxn4$ ztDNj;S}n&%VkHf@%Nb^H(HU|Q+^n7KMjzL#N~U-n^|{CBYP{df=JJ zVPu=2De@Jr(_x71hkj44L+aRzUi^F17V6_DI;)Egu|bJqr52J^grUPFZ!=6N=tugg zB9ZW$8IxN}xu#&=lUy~}^Nca1ynvZh{hj+O-}93SoY4rBW#eoA6F9RcYRKdR z7+8v;)^0H2V0k z#zzffj{c2}nojl#hzfyH;OcN>(OSQKPEWa(+B2kmVyikQgn=t7u?=WMg98YLT2>UQ zg)rn1puKYDWY0&4Q>?GihET_B4^U-*?~%JO z=RkKr26&%CVnOzQ)299RI93$on(WirXBm>2XRsfqvCXh2Infpk1D<$QrhMQ>tK3uuPJ!zh<-RiD~Ke zg5b5{Q_q~*&FwptHmoVuphZzo&(iL&*)b+#4#flw_$&>hTHIJ0$Re-SLIzV#C?{#b z#>4#`lNwX3j@WuTT1ACtiRj90ID!>#<8i5vRaKT(>zcVo*&djFA3JNecdDcE%u1uS zQ2An&?QzsU{fwm>ARO}oz4`Si;HlszZVxDHEBUeA@=ql=_$Q=Z+$(J`dR&zqOEpko8 z2_md)@PTs=VuwDIKA;8gNbV_j)RJi<{#CstY1#2SvvFz^<8sM88bu_1!xc#AmCk3N zgi-|Bmf$nDT`5H1wox_lAO-2J^B&I`5cfsC7tW;qLyWLM}s4^|s-JW?IU!5G`~=VWGM^gKH&AHXKIi#9UbwxNCmWsd34f7&Ex{B1AMR z732=rP*>4b<7lZ5-n%kZVB9I1ouxm@b|sBJa^j#PqgOc8F?XAFfCmBnw(oYo^Uis( zbj>Ym?}aJZNTOM`&;!BU0mz=}FrmQB|2X^0L1L*)GfT>O{gG+yY*}nfEIh0L-h_lW z!si;!-|!|*S3|AT)oCL`g~GL*H*n?%i1r=$J52C`h_12i&pr%3lebZ3}EXxHQd!yDMo05~jo=u{A;SI+CXw7jFjsJ(Kr zHrdJ49nc+HURw_Ftf?Nu9@x~Cc8@v&aOL7f&_b*cUr9*Bo>6v!5#fv}${bUWbr2A9 z&ms$z&QFX#GsWij=ZAQU+ezufyvzB=mMVAX$Vk`6`S`-*#0L4m`zC~G*xJaMVL7AG zp1$B!PPGkJ!BI_~`IN%v7|nMy4|{VdpHHLkZE9&YDUTsmN?Ta^)8(7HMgcn0u~W8b zlqJnw&fCw^YrwcO&-G-zdf-la~y|^x7DKx zt;`$58MN*4$fJsUJeJ`xNLBR@_BIhBm*HV1V&fnyy#isN=EUTVtJs-Qc*ew+;y;z`OK@>=cSx#tYdMQzdv z$(6TVxd2}r`k8NF9gN$Q;>^n}UW&?Mg#r_S#K}iraB6R-XFfl9f&mHTz_KaH{nDN+ zIH~>-7x>XzxlnDyj1gpCu*&&5d`!K^7X8{gU>B$>Uc-Fy^aKq6+SF4u zc@E8NDL}LCnrjQl;k_XXvMpdQVn!<*cVn?{DP}pm|6K$7&m3x7++v8Vtpup5f=x}` zYyJuCR4$yCOR#1>$9<`W`O&X(0CrbLcI`>_(*u$}L^;H5}TInehg9}&vH32}uv z(G9xIe-#a8+SmxLR^a*jMh|@g?3p8nZynJW-pLqyvI=BV41eWZCWwE~k-vS%Qfs6y zq`}ZtUG8rr>h`^Ms@@gkjj}A^hWX^cC__yr3I_Z^aZ-kGaeF+c!H(VhhA8;KfEwMMskK?xPqKoSr;@cHzRFLbtR+H6UqIZf-pf7|R^hx|9 zNzRY$oZcHv#WpG6j3P_`1}^XN?G@JfBkNkhEEU{x>O$S5L+;ny6$|b_Cii6t)ch~t zEx!hQteKa|rIewP*_0tgT7pi;4PI1%BOxJa)>-pFq+T?5h~*9Wx88r?FJq9r2KZV{ zHB{^V6*V(>zRgp3Q_R3-Bg?ha*CXt~d?lK52_=~^iWS$?yrx%>$)ukOQK&#M4ChG{ zfa{6^`^qdcz7@dT-}E)dM*?30EnA&6(Ak6s_IE9Er@@7F<+P0AgEW^YYV*(7L&U5n zJ2xIPCQQ9ub;YO*hpjfP*U8 z#SaUn9w*kB;zea+N8%LvHnaW@Yxj64wl!EAoz3ZSV)vJjbx;h;I)&BL0Z0z6Z@it* zNN5GdkNDCu5O?6orFHhesFSl}A_6mX!!eS+$x@^gA}tIURF-+YBu$(EcY+-^FjRGd zOgnF;6{_M(+TO24O(#DG;OFHdv(zWMqWOXj3=e-Ur3WB;=2eFGwE==3n1ezDvBjrv5(?aga1CJn1PdPaDS0xK?< z7zC9zyC8{7KVSf@Q50-*7{Q--alT*<@q@%5Vj#8A1QAo@0o4PuoeB3BjD~$3CX{O~ z907q@)u)*U6OVbNH-eH~16=sg&Mvda%hRu>TE}zl_LI!4Wni^l3M2=N6Yrfp0826v ziTwU&;;&UbRcow(X>fIV0A>J{M`m}Z>ko7#<2hVwZvUll-uV5F>3Y6f{oi*sUejgo zHch*ET5He%)i9o>i>`?yfG6h`1wMi{sQlhdj{dyt^>Av-S^DJ5&h7*RcSsJF!@!GY zaQ7cqdXqQu1LvTef!XMI{W|4|qGb2=1^P>4 zlEO6fbavnk(s@5wee3#Z_0q&&;@hn}zqO=>zhK(7_0;PrU?71&4PbHv6dRpRGrN9E z>e$y{0mvewcf`^tf; zErcEce;6)G?HJc-|Tz( z^Lz>fL zROGI#4ZlZuENgbJNCD4o5hxYBNS0#B4Ws&=>x}TANpzHPuJW%`mapuATr+wE2}W*m zy?YJ1EqbkZr(Wawv<+%RUk@_Mxw<3ku-{NU=5@Xo&te1LL6KgAu@019*glK;w)HL7 zRcu~k;+*HsxV1O31D&9rvE8_0To1L+MhkJ>DXibD0X}1VQ2^{vM0bH`T=To#3v!Up zwIa!sGQrWCVA@zr65&fEV6R!oxO^||Xb2{RUc ze!@LWqh8m;zWY7E3-=cOeW15qZ#TMg_WA4HT^ z$xFO6&2)?Y_{rCH%69X0Elbu1@PkdR^70E!UJnSosDby#m_%M@u!jz0caZ z+oLe7OF9?NA@~~vg9rWo+i}5fC_M9W!ex2*_=p?&iRkTDchPStJo_@^<+?;WBR?n= zzO7*|`(@v~wlHp4Y@2M1-%B%i>F7XlR<3?I@`@zl*wnx~q|OPh9hg$4|3qkJ9hyjV z%%_}(k&`#fn%|Hzz@>7MU(bR~TX>&h!_=x6JIA)1c{w9NP!)5Izywm&#^taMjWDhtn^0($u$;=$ZOzkNZq^@X~uPdnF! z0y@PsG0&zqrEYIKSKsabkSmUP>F6RlTpFb|vE`iblrE!VQGQ}6YFt>%@-cq+NF}UOQ`=BBgUi21#A}21dPUnC!Lg~ZR;h}p&NcD_OQWfN`JqGk3#Ni?8PmCo z$E3+GK*ehu`L4<57gJ@>qNaWYXvy_MXs@|CDg)zE3AUsjDuDTa>OW3U?3r2h><4VMj+ z7ZDCTBeD5$!~b(te@IRZ4M0IST?+cLrwA-pJ(ZU|eOoSlGm+X|_AF2X5iXh5$N2y^ zK*+zn(6|c2gO0XFOC1{i;UaMbTXuZk)EszD9XI#baRBrFWzjgyLyY=`EgR-)eO*9O zxU58G!;ees4;H>^10%yL{3T}iQTU(>-5#gEFK{UBV25+Uj&0V!MwB_UMKXXt;2L&? z!m-oBnssSONcTpcg$`foY3jh-KmWG`Z7yljK@Ky|Ma39;anNb$v>Buth0nw0_a}O< zGDhW%p3ta752wR{FLH_XLg5Jh_r9}oGyeHgcji00xnn1*e?2wyJLp@knbK{ng7WA7 zRTa%7aj!Q*(!Tg2rgRr9*4#mAEajZI+o8+Wp_OVU%&5uwqBX~e@Ivc#?&Yd2g_+nug-bCZJ`eX_10qrc^meBr~a0YpTdWUm@ z6c(46Tb$ruJUKeF4Bu0Pk_BOB>2?X`eL5?dK8NpWS!xoASu|YsAc@JRvW$1Ktf--y zE<^Z=NMAmE1eNOyK-fvu9F<=Ww-)8`Fj1RRy{LG|>1&ZR5gCcOIMJ9x{pk2J+6g>K zK#&P7u!MHi$OL=E+9);E=F5{84zqGKe`kr|m-=EJHw(U)mdo8ct9^r35#)Z_jp(pKR0EyBZMy3Y8bEVrBd<8eJ74jOVci5?%#=ru6AS>Ukg z!z?s^PuNc#n|7>2DbcXoj^h0ezvPcmYtQ356{AhcB)@`sq~GctJO$X*8+^4@!b*4HDcCnS< zK7_v_ z7c6%mze^G@|ElnWFK&XrsrN=Aiv5J@hMYVU=Gx2!tJNW2O%gD?mV6@7P}shIAaN(H z13i*tn0^!9m)z&SHFiVl?m1Jmf$x@1bi3q&P7LRkx(RR+bd(b#FCac<53^q_z%h{n z31|>cPV#4XId@Cp*2fL;6ZubJ!(u_KH)TE(U-h<+REMl623` zkrCc@#Ey^+RBngqB+cg;!Z(l&k&fWV$%;iDaXDrte-XOdW0f40eR`WSgGLA~(sW<>Aec~LMyW9XRSCh3gL;`w7kgY z$=KG3`etJ*ty&uUVlQME5VWjWS6rvdcgn*E6%h4Faqx-ZV^?}8k_M)Fzg`LVWDNFkw1%AMeSxV$WX$*FU4pT$|_cp@|cI~ zWMl;)M`5WY%Vgqk9&d6+#=fs-ZI~cavJg}58uNCqMm{djHu;COEnUw; zJo=v95N&M!tn0MI<%YhouQg@=zfZ+k=B-#~n}j>@^oRCgwHCjM&Ct)R-DcbF+VxHs zQ*8M*uVY5$?o-xA=StiB`SzKm8})Zd%;L6%M|yFARQPz~sO*06C{69&^oxsbl4)R) zF@P>&tvZ2l?T~NR;YLP)bIjUrzJ<3@jeX?#h|cEFcGz~aU*NQXH(228OTs`|1w?r87JiG)%C%>w^ObCde2KkrKquMS^#U;p-2H}~b zdPYPd(j<$FAgVFcm>t9i;RzQt58*Sb{GMz)wu9HUlbo$PGe(ZeTgmX{FTG`E5iBf#*K@hyFWR?gJiK6 zyP+=@~-0bMkV+L%cFE#8(bhl)8UG$~2a6kYu||jcB}3IcQC& z5+gfwbb08s7^hxpe_)>cbJ1e=z};#XKsjVXh>^Oj4OGzAOYgxH{39;%`ooD_vQE47`EgOnvnyw#^G23()6c+(<*0hQf_m!&zJSh74AD#w?iJ}} zX~lgH`$|ttUBz}>T9-0UP9>L+62M~(^-WvtpssW49oaR!#M*%)F55Q*D9&1tf~?gF zm9nN3B^5QQ$5m*fZz-_u{-mc3dHySXYjeL>lShFV9gFux4TSsLa?Gw~oAXkV9`sIk zdRAZ$H#njg%3}cZV*z$)8>|!AV0n(R$Pr@sE5hXw#9J74WgTn_Zb386Zt@afm+_!? z9W>{@Le-%c$%t%&a~pLeUBWD83ksPHsU_$=KBDn5&XoaZaYEo+3PO;d3Bz$5Azixh zfFco8kO?qLG;|=v1Y&^;Eo+R@Eg&{DWen&R9vxgW#`zK$9jZH)C`_6trp6?9G85E##=8DXIS+OdoKkF0D_RX!H!FyJh_<1-L(=WX1zo;&h$>nc-idh= zV!-nz+5=aoG9FMD0&6ko=(%C1W7p@>^S!G2hSpZVn2+!85I@*MWPK2u@Hsj~{A&a9 z2K(_ZY;~|2LW~aLF~8xjj!{PjWonirx}i;A9a_n68RCBzX6H8IiNUmVV6b>|8y_3J zOTZ)3@z>yhXKRoX4fXQ~&wL8?t7!hVj{EMx?P~SkD(zhAjQ&!98y^$8vl0kqF-m7V z)Ajx%ZgO-BC1zBzb})+lg8nDwg?o>-j=@4Rd>7llacPgKPmd{)j1<;jHiV_F8zDO} zCD%kC;8s)!V_1V1dYNd}hs9mO6h$V^Wj^ z7{O19ViT0kZ|s#Ukz-9urv{Xcbt0uQPO#;ygt6K9y4}Ek8L>sE@%;G`Kyo4U(~!6vpw+%^1T|bYA)JeJdk|O%cv7XcvPjx3ED zA(fhtyppguVyi`aybz{|f4RvoBQWR6vc2W9W=<8yPiq2akdkBvw5cdQ4;YMVnVDEs zb4(jt14lLyX2ton++dCKXO{9cavKW|8er`*tVOuu!sL!c#B14xOHHAanp72*mXJKj zvn&VulFUM8L9S1DoNWhM!>&cEwjpddaD`>0PY>}9+Ia^_QnXeLVzgv)i+qP}nwr$(pg9bR~L8Lwr$=1{`+v_JO6z- zCn7U4W39;fkeOr7G1rP1qknKum`6n?3P0j@qGJ?YhhImLza`I?+jtJQcq8TpD}pIm zCL$kPOICav4F&rQoT{q$^y<{#G9;j+fchCtX}rSrZ57T;t2{-kylXMRlAqbL5NmUU zl%?CLZcj{fkS4K!c{Dav$w+m-c(D~X>k{qAy+*W1A7XhATY@{?p&=!<(hgm(bKmopJiZ&!!-Q70MsWP=m|YmJM{A%{p;=?pA#_vd3d5yoY;h|?;@a6+V0 zR&?h3HUbCwxd!F;2Ypq|5QV*WH*Sk3ZtI5_`BcuUPWw@$`)ipaRd4|39I;CR=I0gC zcVyO8=XUx-|BU`3Ym=*_}~p*{1};p36cbCT$-N*nuv%ah{E@JbtG ziZLp~!v(r$qHS~0Hygo}EwD)@b9LyWDwWf_f_-$>zp$J}n8!R=&lUaqyy#|;Da4k8w5plY73eVp0;AtU7{6|D-B$&-hv%q-i7|!GKLKaV4SCq^Sv@(sgQz#&c#;w{Q zp|uI0m_ar@X6S>RL{~2q^hqn<7-P|iPrkY?@v;a-i(2`4Lvc4CxhJbHQC>jXt)VG> z)caiBU>0B-b;co{jJp*mk8vAB_8tJNbKlQ*BI4z#y?jTC z7kj0psEoQ(7t%dhJccM^EdAeT_w4|5|Upgo4yGMZ?-gyl6@tA zDJOU6iUPOX2c1#+Tilfr-@N{{6*JcqbKY$}xoe)= zsc?_g%@+`o9#A+pGXFPN2$^gmj;@unMpK4HW$OrwI_1ycx_l+aC8cvyEERliQCy5( zudp*^{B=C}OPH3~3=5vF3Kun3FVjS*vNr6V?uGH374z_jWV9nLInxF!zO*m7&#nvH zr_p;aVe@u2#2j1}Xd@F!yC?n7Dl5Vf3E!^Iy$|`&T>x%j1d5`+uXj)^kA;b|M#;9b3T)v?`oRzgh!dSP|tK^S!9^a35Xgv~= z7P$by82Rd<699m?N6-P)6u6?l0F`_J9erh;m)~ri&N|5t?Zi***sKg^l?jfrzz7zpP?BtK;y=+k{OGf`k7IU>W6U= z;sK3{;V))1;ZfF7<%n7xg;10vEJq+S%A|&Q@4hsXZpAn-Bbm!Flo`XFk_=Ru)JIT|PtDq(s<9@fu13D8hQ6-G zzOm+_t_HuUCcC9(qp1c&efEnS*xUeXSHkvI$xlC%M1NKE5{!x93-^mW*pY2LZ-Rg^ zO#u(FD0(;N9rK=1i}Y%3$%Rd4PNyx4rajKxm}cFWhTWKUZ^<;AI~Y7|Y&~s!0y1Vd zv|Og7-Ih)Atf%_|VgDMORk}zKqJPYLzpSmKo&>hf6>>wpxU*M@=>j{djh(svid*TaGb2ElWK zu$3ODrT}PJ4SJ99w_zI;sit4LF|z!0tjfkVg`k;{JIv@p@M!CSg~7)&NErZyR{IE& zaoR3wkeX0*(vY=jwhqZ6p-M(v*#vy_-qOGUVV%#nx-MH|7FuKI&wbw5+#=dLcx?_c zfVPhq=q1z7$Y|B}*n@Ytd4OrG9M?=Cg0UlTS{u4OS>pnJUT6BoK4^0EoKdl@AE|tJ zH7QC$IvZ($L)UL1o#mXS^*|=#SD+1G=B)*#YKwb)l)QV_wK`&wk~&$5hSwXR5Sb={ z`ZV7OousUuAKQW{fx2m$GcsOv88NzHjln)@Nb3SaAu7CZl!_@{m5Yx&f!bnFRC#|b zO|o1UBum=0M5Q}gxQ0F%nNr?J^M#wGQ~Ev8OK^KZwUO2RjbZ2I0B_l zo|M=3Y}M?L!=%G)zkRa2u8urB_c2LG93um{`Wo|94T)uq)3XYc^9tCriiwB$tcUsj zhk1PokB`ab^h=O+Hj4V~ua{1?v%MvN?m!azU z$Bhg>DSBf8(MO) zqyVBR1Cpst!l_N-sZF9O0m3N(j{aTRjA5;jU75gP-i+a>pxY;OpVcuBa>=Jhin}ZZ z*v+=su8WYWUKAWM=(;ZA+OBc^>-vE#rDp+blFuIed@^PPnL=z#RxqqaG}pZEcg0mk zh<31Tta#)z0Lt8RBlL4H8msmsvTkaN@TmrjPTK3bQqt_LP8tQRge4m5sU{Gz6~bN{ z!p$0n*F`ArqeM0&@syTw%2RqK*@fAZ=kg@%Ml%a--uW3yb3+E-zC0SzvJKqms<89r za`>Ouy85RK8m7Dg!jKr?gR5+x>a`nF^@UY*1(TZUW-W4}-kB2uF~7YcKZdC9aYop+ z3#=Cp>*HI#dQ3i5qA7&HZEJ{@3gcUeFU~dO~;Z z?kyP20$p`<@?qzEKkaZj`myVogSQ;)@MZ-6;2&*>HkF27`54!i#uuOcmagCfaSY{M z`|Ln8{cM(!5m_8Eu<3#HZa1)D&_~&b?U|VdSCP1e*-cY^vYDuwn%U2tMLS{(JYrMt zW7URlu!5*8B7&M#bL7(7jq6j_fzeTdxzu54Thj1GIAQ@Cbfj6e#z}9hOxY0a5Ir`D zZp_dbori`y2PfLkSV{@nDa7ZrLD_lB`0a()!EgT@``}9ZIlW<@G)$)Z@v4~H3AC$^ zGUCY>Z2{!4PV+}9lDyOIBV-SfgZj1H!6%H-LMt9;GbbEH+3j*kW?t65^^aHlBVW26_|cHcS11mzq_k{ky+vjPr1X+aAPrNcBu!*mQdQ5D9T@7~Cb1trFW z+;H5W**r)-aP(OgBNRSZ`$CI;yF1+0jBMD)ncLi-`e3osu-nkyNW5YUz^*%F&v<=v zgiwc|>=TKTVdMQ96rGiv+(rQ*tCk>*d4>6&0(#~xvwH#%8lDuSr_ei~2J`!1$=uYn zbQ5{@(icrzyUnwjS^nT$?$|Yi^*;`Az=)1OJ*(xuss-g^Ow|=%7b!j-J56vsW}=Kd z+myrlvkO~@hf3Y1@M_mH$Xy>W%Mcp`urBPcci2fI-TD1|Y{+dgo&Ev72UEoim zePP~|`SUUxx1rp4<5_*+Qe~;^0Xaho4L;@A%Xj|AtZSastXLhZ-cGR%2gr!M@~oo1U#)X#ncysAx~qd=xJet1u( z5~pP=di6QFqY}qsJGzIpqb&MDgsVFyJpi10nq%6g6C&t%l`*0F^avw_-75s$>+OWX zhXU%M)7|Q5^=RL=pb?i}nlh9i7?-|8Lz9!CagPzEtz&cN<|7OjzuumgbPu%l!t~Bl z|Bhh#%3}KJV)~i`?=G>d8dYDqrJ>{=-{79PN@p9y+G0Sl_rTG6j_%#SSISm$E>5BX z_pn1?*9V_9mVQ)F=Mw(3SI3vzVE3C*$jO$L@%8cX$i@FRQlCN$*Wk4KZ9&z+bFv{y z*A$}7a}8gyb!Q37aC6#5eo4ly*sUNJSicb3b#?p{PWZkaNF|r=L&6nml>@<1$Ehc6 z=bAWm0IyPAyS;}Z>Ecqj-2=o^Ral0BE#TqIYvgT>P2F2eOk5K?hX2Kj`9Wg6w?h(GszreV;a4;ZpVuy{SMSq%CSoqOa z@|+Fo(^XuxxT<5e^8xGUL=pC1Nusc%ii`XdBOAX*e2uv950SBf)M2RjvOu2q{R0tu z>F4&;K-&V0249<64>H#2j9*>+37p2b`LSW`Lk#8otbJk0JX8;}nW-SKNh_|Fx_MIB z&%y@h;5&_hKZ*(2R7$}s+Y!&jJGNH*JN8|dE*npf|BvPKuRaaw$jiak+Uh9pjO=z3 zLMBg`N*{QipMBWb^+(^3PXTsX7)K-Ry_ySn+rHk>5KS}fc(NZr$oQK399-UZ6yjn|EKtY2P za0pwb=4# zz5F)k=ov`snBt$iT}%M2W)7g3mVZyUKTcwtKf%6=D49EFVQn4b54hzbNX?(I_kDibPLA_zY9m#^QBI<8jx4~SH{RK57R04P>A)9#Y^hsU zSIrsJ5#%a6AwG8`%2Zn(w6~o9PLy*Eu;QW!YooJu#WHW&K#re{K%|nV#w*~D11BkwEtL-W z`F#Y67hiO^(Wr2aOP2)P+rNdYjFYaPgsi)^TaG;!#c1Ptf>wP z&WQ)WIck6a2tRTPRmZyb=6qT!q^Lwq{8eh&)3x{l}fH(-eP2 zN`FO`NlYFa@d~{UVJgeoptep79|L;BZUzB{<}>5z3Y5ZC(JU+c!t93Nkat^3p;k^Q zRhSgIdQfpKezJZfUIVd;hrlsCA7{40DO>#FrLommzApkk`2F&ZV zX8i7Ug8f(tU*1{OO}8^GCYB!Tjo-6wOKb^Gup?<*W@6x+CVte?;L6=KXBjaLNtGX~;!v&5)w2LwBd;>r-P4N1l$akR{5VyAw!9GH}@aY~z*9^fOc2xt$ z9d!UmO&f#ZsK8HX7EfD);mA4wk;`Gl*OeRl zNO7yusIL#Acn;zhIX#E)OBXnDY2?=jSv&`I3!a`r_+<+m$u#=wgE0OM2%la;`27+% zvT6LH57PKMAdhnd;a4Sal+h@p58`+V>Q=)!gYat*II`1D!ML;g$ZfF^>CC;ngY>IO z`5Qb|;0Ur2m*MF3O^CxJ3qaD2J@VwZq%N6?+CR=)2gSTR7*SS|MusQxb0GIP_FtMd0c57U{e^BOb> zg6Ya#eGa-PmAej{yn|d<%Q%5pUnX!>uuI2q^#}m@H4!sdKGJ7=1A#Xk>x16kg%GCA zJ%tedesI~0V~DI>1n)xjF<9?lfg>jEI*dEZ0MKsZvOY-SDJZ*i;yRFjhQPI(eG!&> zir`MQY5=w!L~sYFnuBd0A-J=x8i(~x5jaw|%f`6V4FJJ5KI?;S--H;Z(LIG2ZXmmK zb&epq#tB^8nnz%{j}W|@nwMa?_Yl0Rn*Rp+8#uao4%Ryh!BTQMMaVS9Cduv52X`LGJj1o?~gL@PlN9mYNkUD%*aEhc^N_KLJ*h?(I>1H|dR=4Ol; zL<)m3R^N>ut%lJvR!RktnoiLEv}X07Gv3B0_?We1(WQ|Vc-bB+?V`yt^q9%jUy|jf zc^PZQm$IyB|NF+5Hg@pFm^QZG`D+SJ9c|x@C+(r3*VmL(BQN3<2_rA$l!{tjESZfD zD$NBGFX|KuLoe(U4HGZ!l+{{Z@UhogUX)`_*fEy@4F#ahq47H}EyE90vHlxj+P@DY zrJ&SxVveId!<(=rS?Q~izGhCMb`*NHqvgNVRTGsNw1iB0R}lX3OYq^Te3wy|-q z$sf26r+LwLBTn1q>(KYuy5`XP!cXfkc!NjVV*Cu9(qi-skA+N6N7#f5@G4YnSHbk9`(R_`Vx0T9;a1_%V`NU;Is%> z|3ttU^B}ta{PJ&Qjkg0^9GY)nU9F0mU%#)Cx)BYdwH+W~nT3v|xj!oB9G9qPS`nDU1QZ@pG`cG~8o-kcNf zLY5>#3a*ZnbpUIjf2mqMYPZX;z*L7`zzJ&ya?Nf@Vl5wYX5YD(xc3Fv<9 zwz#e9;CQI132d{wyzK;90ev;6E~f0Nb;n2(q{a2Vl|Cy7csZJKJhr26V@%D;SdQErT`3sY!P9~Lhg^>?<-1G=ZxZYZsiAT zbgYQH@tmwiymsW~eVQo=Dc>kEU|9pcl9BSUvmjCR77APr`88HG^BN|D3{X=@vyX0c zGFIo8b8*==cK+< z4O~s&s+;8YeweW5X7{jQx8#$;Y(&N-OyBQs@1p>eWNbP&ZSqG_GA#TO5#!e~*^yb_ z-*Ui-T0@e76ceSDGIfqU7V6d{lc?}YZF;_s-Sm+ScFWB0dlQ$eQo_n?Khnw|y+y~e zN2&WHI-S?IV4{Rc#X%GCiow-mhlF|KFcl^^y-jaV@=*EIBdAVSRNurm^ShL ziZj6eDt9@y9fUF+>w-i-9q^Bak%eVZ{7O@Ye53<3bT+pDQ%fNMwcB!i>uy&o=e3Xd zQ4^m}Hdx_hjy7TSb@f9lqua!SFfJqMW3GJz^#QG496 z*x`)))}LnGjIco0FnC3)_wlG0BFh%x0*Dk+RQtz1HX(k%75C8TC((l2;~F)KErhIX zE=S0?AW8P(L+}ZzX`v%k#F&$AuNv?S)XbT3<)cD0p%7=ZCQ~stch^n^z)rurs`k$5 zkXvUyHP%E1kd*U80%B)=87|^v8tsTmqXrt)S7I4g%$7q}^x1Hqla3aT-cz@*EW9E!sd&n4eZZgl zq#h!o0D`TOm;d8q=~o>9>v2eLb=&1kn(GO@S@7wnpz}Nb@L+*F7BWn!lovlCNs^xA z0p;K5IA~YiqXY*7OM&`dgyGCQ9E`2aO#Z{CFZopgMF?@Y2R3=;fwLCI`kKy~;E1Xl zr0%OKB?*IUb#=JEmehZ$*We-cr1HjahY>x5DJ$zs`a*$aQ8QeG%BpWUo3*@XVeVnx zEZ`sT^lgh9pKN-V@pu%t-5k6NO+TSlFi1Cyo^9bc+)`AB>5kEM7c5Ki6I=>lDQO(H zZJ*hN0=rYxQBN-0bHd%dbcOXDr>~}iAy~wI@9+SIZSeF=ug89*c~?iy=t&u>fZlZW zFW=eX$Ji&G0nFA#Epq4(H&(#sSz!iF<|aZatTu<~mhaiv_-5g-dti&tA7pEMWEkK% zFWTSOkX{t=#zniUaCT#Gr4Hti!rNdDVv2B~+bU_x(Q_2^15{~;Z!9ohZbQu6&I+(& zh$Ju^+N&nCj@hv;4sBj*mFQ_HoHDnyafj;nwYq6oFo&h!N*vR7YO9Cx0Y;h;$LbEw z$*_7(zR2TAES!-pQnHquv9N%_9t$0?szmhmQASiwO}(bGL*@Nb*AId}<<7*Cb9|Y( zhP1r?;tRK*Sr>qCYAN7$`hHl5yNlQew?Um1sg8@BJ%fu?nLlURl?Gkx_6EO$;?GiR zRXNn1y!MD_N~-RxT>!#PzW1rfdd{(f;;(ZWyE67xtB4c=wBMg(eH%z})ea?pOmWfs zmnN^Kn?i^(BYMk{PVR32)4&v8`|h;ED#QfsdTjW|)r7le>Gw(dMT?zGzDAYY6r!%Z z<{~O{IcT7+?3%auguvO%O2$U+VNuB3)>c#g;ZemH~|?z+!eOn!*f#=xPkP-toyi>OV#P!$NAsjU3m$ITsr-6pq!H*@@?l>{D2bJu}m$;&Q8K$IX{6)X4)LRT@l1*>OdgAEq8pQR-)L zUB(PH1smG|NxCj4)u?vL#q%o-M`lSh3_I{4L;MY!2)(_-^g-BfA9N2*WN=m zUNCPf=$dc%QT^GRA-C`^&At@#Wf>(2BGHOE9uYQ4!-HPA>gWpYcXW-GKzN~7Kql(< zDtI5+sFhoMSQrP|gc?=xj@*%Kr`sWkv$gr_OMKz(AF%%_;wO-+W64Z?m zPR{{}23KaICCexqTTh@h#)LPlWC|)p9}5`gGsdh6Qjh*RDHjQ;x4mpXY_Q)*0DN@K z93bu2)bk`xw~)lyH-i=^+GUmg=JB?X81nm)d5wJ~bY@uFiz^ZVlOHFfnN}F-S?G(( zljJa#?|9s59M`fYa5Ah3ZrP|~+UfA3ns-NDm1YZf(!xn-n4)z=k?SdZh&M97P!M=6 zq;=Y-2WP5a9?iij+D>Q>%!dN*Zfr=5D6&3aB@)dNet&MKSccN5Ca@p+XHM+lIr^m3m=2zzuikh&Lf9`%)5p|88^*u_lCb%-RIZ!WDG z`sAl^Wn58Dbd4;OVHh&HG92os^)brN zlh&6Wj_xRFfq&NNy=PY4XU3s2z31z9fH4eW&9_f6>oSBMCktyI&W94q4t?~q$wWPS zkKmyUP|yN(uSy2-r;JB*Q@cWM=?AXJ@_2J#z`nS1rxg7tz)iudC-iq^%Gq@_Y%_ix zs(QM@9OX3OkZKd=s`nq(#CcR{LuS}D11{s3w^Vv$2Q-CLkf=~BO(AQrwOyqICUbj+ z+JOnvYU7~GJNcr|wJ)|nLua8X=QCq(Sc%63e{5=L8A-sR+KUYJ`;_!MT=ItEaSqje zPQ{&B@&`tdzj{$k>D^TF3xn!Qx%6j;vOvj;=1y3-vvB)bd*15x8qlQTK=%puhEO&PlADhAu?o($5qs)A2+6z9)JVshM z?{Fv;Z`F)viuel~N=UtUr~O4FGrGvPO?2zZ0VRek?Oa? zJJ#!OT&Z82g&VWud9e%dTy^xGX;W;dO0mNmNvsx0EVe_l9ZROD1~O)c8*&4o%)dGR z6um8UwR9`of3g008#=Ro-m>pVoKd>L6a&{~ry1&js2i!6)&FetbR~UQnV2#KinAQKEWS{ z1*bG%qS@E*byjPZc|B8*GTxT@dLJ`;7$&^s?4qmQ#N!)G_{nVYFDZ5sO_Ww#(PFCy z*+qBrx(EytPiO|%OdfTY&#X#85<_bgy?CjZ#@|)%JoWqL=GWAd5?9)Qn%zLblQX+} zRh+%&KQQB7ROm6|4?hdw53F~fOY&F#@F0;x{}i)%(}z+;{$mJ{ME6Toth9nb!KFniJRgIyb>4hkgCh)m}+sL^P)aa9ff z^n)(V_wCkHE$(h98lTix`(<2IHg0b<3|r=< zSyGNtmvJv^;K-Td-1QsgP~3Y`($ELwlhwX?g0G$OTL<2QaWA_T#Q%w3(OqZL_V9?R zW@a`cJ)2;rLf3fr*<{ z4=(=UYlGT1?%~a?CkCw#=3xitHx1gJ-fORzPvCxy;~V;Lz}U^Dlkd;n9+T%suI`cR z&4xF%7zU>|1lrFp!>L>!5-B}H*Q>anEMp8tgNI|Nd}8V{e7}vU{UQ$krke3Y_C}T9 zCde@B9#849AWH)&FDz3jnJsSTJ9#U2--)I>1I>5r+v8A8x%UW+Vn+eF2ekS-_Oq(I zGRbp~fhGa*yG7uOP3uoN< z(FyNR(*ggH95nZch-d${uyBxIVB-IEaxix_b9S|KaI>&<{ZEPZe+=jOL{6(4_Y1_7 zkkHWT2u7%LkylwQ3#1YQMbonn_^E41hQDP=(qw;nn_UxZho+ zcVu-7yu5QW0WI*lAGzBacZ72jQl4Ig-r~00dj( zL7~>8a9t_rb{{eAN?d{&JQ{gOVXWTU?j2+E^qZLA7;=?aIYhT z#<5ql`XdrCLRc|M9*8G%S;8_l1w1Xkq*&j|fC?+=%%LA{jdX0z4NVXgnpjj_!Ge5~ z^Ffq0PrbLsB+IQ$>z@fXztZZx$4 z!pmxdEHcNb5)Oq;h)o?*wV&U-q*o#DrV1sQXHW7Tn1q)aBp3xVNWz`F z8(d0=7cC<#Vq{k~1+bC&`2f#<)GqdQ*nMJm-Tv8X?fVhQ^uRf;2-`+>rW>hV+fRbP zv=OEx6Qf2_4?d#MeZAi@^hI_D(-j^wKoe?sw zf%2d>jZ~qGycCCS&2;DO-vc&)#hu@{t$G>|%t2_KM%!|ogK^yRF4~rWRVJzEkp^8% zrQXSHyzNrw@2S@@;U09Ip}Zp3$KL0=O(U?-%;S$%^YXfvlGcMYhdD$ZRi)i?B$6Pi zIgbZQpN{|zfJnV~3el>oU){P50nGAFDE`D=`p^C@aC$vIJbCi@@Kv)V&yNWcNmdUj z0tur{rZvDZzr&dI$PjtItX)Z4!$cf+^_l`C_ecJ}~~#5`u@o zq#P_C1v84mky_3o0`J(U909s?ZfS2pvg7GW0V_sS#lkvK4Gp6>$Z@o({iskz{J5Oo z)X@#z8&BAmi4JfJTQIiu00Pm$45jN$(V+lIr=X`(Bj4gQ871EXXyru2%@{#bAmU;^ z<(Y)i6MaITw(8s8<9+1m^vSY4_$)X0jC9~Jp3E0l%!ods4No)ZUv$q)uZR|?;_{Nr>%tq5an7-1F ztmZrey7ccM$*-D}e7(}wO!^92Lg8ntrpNa6IV4pY*geD+Z|s3RyaNW}XdF=Q2emMK zw^&{+BPBO3rT5SdGnek06}v9=V&jT1K!&({29cm4+GU16^a<|~XAt#rg&+T6ZNd<_ zkpuNN1LXfLoI?M@CX}m}quGC$9>}Pm2qAs|gK51mjfqRcB4E-WYez-m2a?qCDy7u~ zS*b2Wn9dVrB%s3-pL2N z%Xv5iyggs={V+2{12B#MQdeU;?X!fkPxhGSUIgf|VpDV2t}C~!oujv4_rd&*G=k_x zM2%uBguKq8ze<1aq<1bhiUiLvDx^M3Xk!2nBTXHmK!h(m*wR&~&(=1jE859GHIY`P zn>+f=R)#eOf{B>tf==@*JqA-2p+X$tjev}Go(prIpwR(XpUy$GEF3kql3Q^SKkOEE z?^F-&A5fDtVL-%w2;lO^8m2GS1n8`CT(l?eCfp8U7a$X4YOG^p*=z;D1e77t9r`9x zN@i#8gqht8omp3YEUIyneRdXJp9OD>tbwKryUksUnNX6t{F65RV8)d5FmCMJ+Iuh| zy53vb8`t9(UWtf2VJ~BnLCMy2_i9vc2FV{rZ4jVtx$+#UV!*?-UKG~+*Xioo%-#l@ z_qkqG@6}b$jD&&M;JF%5f_}N7nJe%4B=bfbZoxxGX_BqL?7~L8$Lb?%G*v&Q9u%KFmWs<<4M^M@!jr%&4B>BG(~Z8A!}2I7ci)Pd>pw8QEDVm!8o5Gx94EO}#TSCoB>jFb z!3L{HV9In44ie!OOX?FzIdtif_K;w|Zs*3d@bQpRP!7rjcdvz0($N&IyW}@eUhv$h zAAkOyUWg;dBAbvIX1X7DX{=#e+3Jgs6zvidT7H#DBvkGaT4 zF`-S{-PdO8D(%&Bi1O2-4VZkoI%w#sXfnT?nfPk@22)MRcd8Faf|E#bf~fA7SYMFm z!`GjI#TBGyTr51dS(ll&=V$((_s_6D&?;>(jWnB8P@bfjYOtLYE7pTnmHQ_!USg>xnq<>v2K2X7F4U$FlHX=Fo&%B z{HK}4JcxT#{RdZdIoc|Z#S%F(+LYVRzFmW@(zIJkt(L;B_n$3i%_3T8!VQ#;pZdJ6 zL4p{E;$RyC3dky?JI~N9z9wQPI2Aeqw^#mnvWVUQSeD|e>{YU$27QpJp=nT!A?K5!NO!*zXDPF#bm3Z|~JBDnap*Z>@ zF=9{5y8RS?MEpx;%@XoJkeM$LAS5`6+9L+B8_4Peg2$9??CL}^J0dFsVNn!dHp8iJ-W$qP1>k$-!c-e z*&lsE8IOKG(&2bnxz^IzjY;}HSTCck5=W06NLjxi_xM~38c6Zm{sJ(^rHdCoF{=bj z-VXLIdJE()%r5>wwft?FE$ZX@zoE#a8}Zuk7mB8Thj{;6C`y_CCkX#>?)z9^R+{27zF=WJ3 zy#(2OymXPh?0QC6+7+>m7(ko8lx0{-X1sW9x9u1ZrQQ3RD${&KK^Yd4e!`gs!Q;C! zXEE7Vt+D(t7H)n&-pMwU09mLs{71xX&16X!BbkrPF9JV~XWGJsx68XCGZx}J{7K>(89JJ(B;;~+@ zxw%M`UxInQS0U|S|FVyei1>VaedFcs1_+eV>IBZ%f%D%5kEz(2D!l|S)HT*g;Wm9>c z{1jGqLN7wh^5KQ2h_dspjS-Q=|00^D0`}ty+Uhwr^CCLCYLRmx-bvx2C6iK(kgejv~dwIZdk7ubfY#vFu@ zSPHSK(un^~l(LEiId*KpL!_Y+G#Yxm=G7Np)SVbRF~vHluR7_!d=4{RF8R_g1-kvg zYeV%Q$$%8nvA%3b%t*SS6d%D7%;-`(E>_0p1Zdkxi{T(QRYf<&+bzTJ?YjMg*5m9N z6|4NZE0?1TR&+hGXigSD(_wtn_!l*s2lU-D$g1ut^_C)CpfgLX25+Ttv=NnR;;l&4 z5FjgrO~(&%5mWzdVkCA$s(Px3*_Tsxrq{PnS_2NEUz;Ie$Bi2w>OBR)c+~bzb!3-+ zF+MudbTCsCx_@k=Cv}5@he8)tN{*J7t6!S576eN$6uCezM2yd zR^gyRP|hN7o=-ji_GP$JK)FrwJZlK0Z3IqPJ6X6uIa@Pkc>68#@TFP0HFDN|JX?%< z1*;a1{rjOo-FkDC3o-J;{l$>X^AWmIf1#>o3(f9M;g37mHg1ZV9eW{uLf1d}t65l` z6|=tL^X~~%w)>^etZZuqCKihut?MstIfGhe~L1pk)~O8zgg zbG9=1j}A)bpH;_?V!EUh_d#`~Q#7%22rYY6xPs9QQ@qe7-8qBO{tARlBbrLft)zr>?Nds z5HaGJBIV+<`g0N;m2z9{7$uZ)k`P5UA{*e~1cZ;=UO|{KW0WSqiLxs~GeJCh>^vsy z!>XS-6ANg_t>P5Rf!Z%cDw$=!*kVznXX2SaPe3BUuL8Y`+HcB$i&kL7*^DcS+|QKR zs|C~?TfdDMxhj`XTA!b?7{&S9cqQtnxjDIs9Lf75Qe`6S3Ii~l!iiol-5aUcZg_tI zuA5u?+KsBETJjKPK93KaXHn1PU&0KJbZ9LV#*zg5_QDOJ&_XAgZ1^TV*#IjPqD8tr1Qp)u-M7tJ?eQ9|D>3MS+ZSK$7ZD;jr@^OOfuNqgR?t~EqAIR6IB z|HIikMOOke?ZPv$c5K_W?M!UjwkNi2JDJ$VPA1NdZEKQD?ELe-XZ;u7cX!TBt*2{s zbzO8-cUN_-Rh`OqXdnL^<~z*)LpN0!2WvNLV>@df^Z(TaUj6?#O?a4o(UAr^hGSu6 zFL6y^bjyC3&3q|w(!`q*rMc0XecJ}q2UXwOE;!{4*c^07{U`N7)*lu4+Hin8@5xja z&(6zC*@9tz|JOI>kP0YXY)brgGhTCJO$g7J3yUnd4b-&gaeI)Sq3 z{($9E7M$w5?||&HNuo&ijNa7Qt5bU$SUszq zgagK~VPzFbu~R1QnXl^U{LVmrV6A{#kk0o(uHu`oy4nCmh*GzuUOqu zsf!in_zm(ip5ZNxNNEC67u0<@T^Jt?@JM_SEAlwA^YiaMPq%TcAp0GTr1l9g<3Cv1LAnXQ}SX5Lr^+ z!{G^5<&%1TBrEXcNMDha6FpY~-9b>}D4V0;q6IeMYU$SV7VoF@k{oLHgmX`g_0#)N z0rGdYd{O~T(H_}mp=MB90srewL&ABojr{LDa}WLhkknUlbane*bu#~TtkJIpOxbRD zQfe>*Ku_D(S*oL3%W;tt-xET`-~%#~>!`|NMYe{2hcwWBNPQ#4Z8v zyhO?7YT+?+oy&7Fb$^iHdHwyzt_Ut{IE`W*j)8th-|oB)*ZHOg!QLB`C%6HwcL6wv}Al z>vNnrqMH~Xv@;z_1aLLj4y_)vTIE=q;na=%mCO@gnTsO~XS^*t??q>1adC~%29zR8 zs*h#=ak1Ri#jwEa<<6HZi2s>uCZ+G6$ipzY@~0&oQz%9zE~627TnsOxacYs1myW1GTIrZb_|0RFuo?KqMMs0g}hGo*mA%Kn1f1+O`?Ip%&4V|sZ>i4xbwU6gD?yOiaer*T> z?{}n9v0d+uB;fR}66CVo2n95FW$0GTyzYhI;-a&4(<@bj__=A}`zag!Ip^`-X!R(VhI_YV)#UNW7GDGoI$k3u;*7K^_7d_X}79Y#MDvgJOkxFnH}Z| z*h-YH3q)%-NzNF@-5v#hgd-{{=O9v`l{w|W{eXI^EUTPWx+wD?$}0GuzBU?sq_pxc zepveFZ$kgqHECA=#{csc`~Sm+h^=xDT^%cLbZUm`+iyK*P>|+mRmo*A>geV|?o#7{ zZ1KNSDnDX9f7Ty|!a0w&70;`G)6MRDn(?0MeFPcq1bpKj?u$VM#)u;Y zo6)P$cEpDdX3Am2umd_+8xiAU?lUjcPH|}~yPWttv`n}gHu1Rx5bniscu++U{kftf zxRfPx@t{uP)aKpBYPWS`7ZCCMFEpAp=n#&4^4|zG8hiLuRfYhbHcff zb*OI*IuxM{tF3R9s|Ta@!WoA|p&W^0CBn{sGzEd~t&q8k!B4%h zHN0B%#lk|395#Bx9nye4k-C}j8ZW&>zEDPOw`PtI&+Z(i%2z|Suv&bM>)(TD(b(z* z7P)+VLOU{A ziXh<_o_rb0!d<9HgjL~^CM4L;-(^U!0Z9A@1#GA7oOF4Ip5z(R4zd)1d#YD2 zLu`pg)RIx$Y)tf0y;hM;bZM(fV$ws*%!xz9;l(Bl>}6$kn&DaqB^*Wh90YS&)E1Zp zcbHrRjJ6j4j%ve7Gc5OHPNR$HGmf&Riy=Ex4+$*xb~QLMY;$bl!|HBEa(8OgHCQok z(Mt~X8?k}K`GXv+=)j9MHAh4JE?uh+}lpa$@gi(aTeInn#7t%Y9@RJe=G6?qSz5a41L(nq)q!!j%>%AvU8p zMZegzD$Td5PA|WbZNrE_Wn5+-R-Yw(RciGvDfXh5Ju)k%*Og{dw4Z643@%4KesspO z@glYMl9t^BWAzeSm-)8AOyB3>nEJbrW;&?vlIMZH6$LdDk_&8v!trm-tVf(kLNfs*P%ge1FeV*bpZ1#X!c;$@EQG?` zazwkZ_JM8ajQc7=!&-PIzp(Zf+K}C{Ly<9pBQG8gZ>6?PcXAY-k+mD-$=6oC?Tf!P zB8&Pqci<1;-hX#V%eUwVzaY6}*Y;;h_gWG=)x|2(ikJm#mL|pjN-3MloNzR`rI5Tc zWX^gE1hVwZqzEXaUr}~(e-^SmS_bmY#a}w*4l$s^^sbIqS~{d?k{0-I#q)iS>%Q;D z@^Yn45s-j&vpqRmkX-0f5P-XwZ+Sd=&k>$H)3hyq~su%heKVz_t&B{H}2k;a=?4WqC5xxg!3Cb z4VlvnNd4>3Z7)9CV4RZYYk54=X*tuIRnY&R%u%c*oC(5RogI#Y6nFma{3a*wri0q31_trGp|6^TZX>V~SfmsxnhB+4X{%<9efjm!3qaBJ9+snwm7?Gc(tSZJSZ zM=YtJ3ZmB7vul;XqN?af`CN}OSE^lqtxvN{5$0aY>W7dmK_|MR_yN;YF6yg z=At>j-ZnS&kMfYowcB+NpSqLZmsAc$WX_MMJ(2DXt<01z95k~{M&MA(5@LFh4~=K5 z33J4onGB$G=#CDZfUM?Nh%Ac<-}rn1Muf`l^g3~NhpZ~#lx!NISgdrwH!vb} zBvB&%5p2pMGK=P2`1PC59^d>6QwKiKIOvk2sU_$1L@7l2;~UM#0nu|7%14&j$dy~7=?fXPI2#bp!EJrgUd(%Q9+`ua@7MPkiOKt`5^z?+eA}sCQ z?$dl}_jmX7f6y#q*qK_J?)4=r%XoZpUSqS2pGwO5M=y`hgc$Xp0ugUf0T9UQi3o`z z#o9(DN=BZj zAPZDDzeR&ot|;r6mzU30{*V~l^DCD{Ke7k-4dC{J-_NMNG#{vvJbz`T#1tmWTBUG4 z%j;;OAhP^CwG4D+nu@nQxdlFhGJ>iM$Wy3S4F?JF13(C`k}=~#lN<1QP`H4V-Z(2u0>MC zw6<|k+?JGMDLXotw)8BRWEcjR3uf$@xw;j)$Mmk)^rdE<%ZQ^An|hFHW*zJRHF>O< zUUAw~DgJ3TBB08I)1-v-mT@-Hr%tlNtNU)DPb1UZQ&8NYcp5DirMyYSa)L8emHv^U z4zd%tGz|P1ahgbwPtgUgU0#)K8CHj*6ML-vM_g~2|H3WKM4TW%VUC7RV8gA#Y8Pbw zsg_ZpDxG8S9+UDZ_J>BR0=93PNRFj{OE(>H_6m*TU3?nHVmc2+B2YGv`YHxXxXA z#7?{LXG`vy$)%6x`*V{X?E2qtCue^p{eJWW6rnluZ1;;NVA;N>73cx`RqIKXOAGWC z3@caAZKGNbbv{FU7#iCiYJdUmwT7b4739xM7qE4qnGjzXLrb9pb^?RK{LG*%Vk!vC zQ&^KIqH=BXGVRHQUn?;{F5SHiuV+Mrddts6pCHlRd}XF`gWbSp1LsOTCZ`|uRLfie zHXj2xy?x3;*ehzS>gNNEFab zO~grx`l#8hsNwRuBSlLni)ULpsavcV;I>jEifp-*D^)eo!x1pTHlQMd7m&mD$ntra z5KtgR60gWK$B9>1vh=q6vjg|C6p4Q~XA0+9(ga24$3f8gwxL#@uPGFESyBbz=7&Jo zcr&6@9(0Nl5Ax**`(KlS0sLKI*dLv<<^648$X`8>t@bt~3ey58gE)WYm*y3Wy5C$= z%+3V_3R8WNfaQIR!ni#Ykk1MXT+EX>r3!<-G4*;nK?U`R7}$LgGhMgsI)KrLC{S={ zi5n?p(?<>MiX1B9NnRxmZW72|&GL;C{N}TJCTJ4FltiHVBJfP{J%PE}mblV?O7!JY z3;^|36GuKpAND(z>XAued-wDHK)zDUz>tv+5RP96g*JCAIzAvCu2_b|Rd*g=7_v$LdQQjC&c{7THbni0mE_T{c( ztcw)=M=C%)lN&Y$gHpceX8dK7f!H%z@@moA<`?k~-^}yJ8!{K5>=0Q3YM#^<+aY$u z&FX;-q=?t1Knn7c+tUvT6o{wTQR*7sa9`dCwk=EU+@t(j4RP)bfrzorla3Qqkte+; zFw(R96N9nwDJd;wtWEgMJFh-S2gb0gFw8uOs}URX(3mYbo;->xa~Jc9U4wIjp@WRB z4Cjui-dy0(RhkXL5XO~yPfywv2ESjKFGfVoW(|+tAA`w-TIc$d8r)#t+MG$aA$R;b z*ZLEuUSG8OYp5p+gPk~-s=k~}Jg;|3E+XQqX$}AOj`rJE2Y1JVhpppn1$uW!&uilH zF3%3nY66rix9k0fjic@9vuls*z56FEx5qAco;=YfK?TbKUm;Ncj=yl6nAd=R;)&3y z;P4%h3(@fllI}}FM?&>qn1rr`l-16JPQ@`;42UOZE+X+OGzY=xO{%R?2CrzKs>bJn zs{C%g`Je>mrpaJ0CjZbL4i&WJ*2$tQ45$-yrvmC`a-!+jW!$f&b}>9`?K|xow)G!x z4m|0;)F#&SguSB6hqS>Ex%nZbg|UYKqHo0=%F16$1+h&6(;!9b%=w$A-}8C$J8k4c zU6T;Yax4d5uA2FO_PsSh)@RPLZ3VN@bB_k=!3n_Y;nUyK7g=~=l86)RYF}!P0kM3& z9KF(qT^bgfRnkxx?|$u$r~7vN@X_$<_~otPldhXMS!kJN$q-Q^K*ytZpe+@$(7X&Z zM<&S-0r4XJBR(J~jEHPYG01p_dppv(2fOOwx#@iSMbCT&n5t=AfgJB=4Wxf7=wcwY z`Fh-(5-f3dEi}g*&NojsE7qY=d3pFgCW75w{zZ53tB;_QS1gLCF#-vK4B)R$oC5fv zq50pq`A>N$SE4caBlsiK)q?YZ`6bohdnibnUHK*XX!wM8_>II^e4xMC__;SuRrk5~ zdKwX={9#ToLks!v zIn=rSHS4vD3UK>qo6Y4T+IbseI}Pe~t$V&(HU+tEf3IfY)G68lAXR;)wqDG=6qdwP4~r3SQnCah!b5fWbATs>gosu>h{7OW%g2@=-%JozGF ztM!9#huHXrLVI`Y3mgjADfuG$CR7q26YlLbJykwsiu@5e7O2~fz2Qc}>ZR(T0-VpJ zY^&PXhgLA-ORV^=Ps*c*($ROU2Vc;4%mwSxcWecB(|4={zcF}?1xqoEnJ@eP_RR-2 zZ!cgT4jl`u1cuyU;xRoRo47+dCOwfxVD)A!<}kk}Ke1w!Pj1tdzVvGPExKTVhU z_2KkorEYUAaLI{zJX^eU)ZIP7RCg>6V7ax-)w&J8%jfVF?$sJF+{p zfUvQ4xCNB8B^b0CxIPZS8HmtH`i}YFT%5kvX>P3(T}82x_U(2N2W9yi`y2aGlKqX- zjp1?oTj!Md_O9|23lcncNOqJ}#QK1J zWc*(!J1qHMmDkJ(LdOR>Q=Xfii@|y(epJ8Ssr*l*NuCv66jEMq^B(gk9*+194s6Ce zmo1wfbFX6|J4;_?-k#Zyx_1wfkLhmp504D*^^cDXZ?lS>)*PdUFm@==UOW446}cv# zK+I>)mamUmH;Is)y!RlR=|Q_MzfOI?pwOt0;+2x>4d~%+ZB_^bs^I(b`_hAVnfuFh z-WO}M6&VSW>=3)Av z`{3hPwER^6%sVFztwBMQ?0_=H9cqP9MX3|xM&9%JQs|Tt80`OU40;U?APYBTdgFTG za_ztYzHy?z>%QwQoku!8IFdbF?i}yT0NZtsu5OrGcV*%0ns#mA>l$};;Om-qec&A$ zb|v8*nuu4pKJ)(YtR69?@-1P$ccXaJkv%^VU+QSyTtB#SggUiD%2Q%&4-X1{iR9Z0 ziB5P7fXvZJRy00CxLlpa0kAxQDqzWDi4g@@B0-~Nay}uyy--G)jshDmJ5678aU)V4D z4^Lb!`Y*YT{zQKz!-DG(f-ivUDS@$pxgN&1h7L>(*rB!z{QAaq0@e(!#|3!|VWFJ} zuV>^C_>UL_!kZcWGzy2DzJdJSr-ML`Kwta{g$aqN>JJ_RE?3zXL=jY`1{#!Z5pp(13U*C23$C0JIELe>-tAayKf2?1X?=hTt9rw*}W@1-k&7q_+yM zM{<&Zv4gQA5o2gzX~=4KR1K7qG;%e*YZ{LO$vw#>g541lF_;1bxFQ8WW5k9asU=8oWfXR)uyUyq&Xw-HqS{R9?iOybjXtIl15kTXVML98WwChF~ zKrzrtFY0g!S@E=QBo9g_@g3Hj24le-lm1%kbWG<9-tO4{sCLe9m?Rcr;! zATb_32NP#>?hX-aS`11ILSi1`p$v|Q%pB|{|A$jH*onyw%z*5ig2;gG=e16`*lF{H%@7sexG#jd6MF_>)(-e^uU`23+_xsr{4q*VLSNd;ymL6ATPY6$7PbyEcdS{M9 z>etqKax~>2K^B9~x!Lv4miq`D*h_B)YD1bz zGSgC!jfl{99{AQ$%2V})H}n!S4kK6Far$o5TH+DpX$X}F>e$j`DdFX;m8y?Hay}vo zwz1d6VjPuqqCMX^3MoUf%_$iLAC<>Y7ZRBJjYh#aQ|8=T>ovf;Ea^wU8Z@BfMgqo2-)^sT^TFRHVi zFtX;c=ggTq0(HOex+ip*Ooywwq5-TB8L%tKnwc3L!9pklbmEkxJe7+uLT2XhUperT z`B<397OElBwtok<&BjA8lo-K<2>r=#iCP=cZd8RlH5!ngO$svoVKJB(d=GXQzuKbR zgvd~Y_VC1jp!04XJU$(cHq8j-7UqD^^5YyqcA)Uckjg~4b19HettUnmhZ`lY^b#!k z%vzY~G|qhnX-V5A+s>u9{`*Iv5ctX1R8P<68Ba-*bq_mT_f6?y5skI~?XAn5Qzda8 zi)uIb9%qMs7&LLvzd785V`lu;@3LE=QIH7dhK4w^!d<{XzCA{$#Os_}rd zBytI`P1Av1Cn8qc`(b=RKkWMkotF%XzWTJ*%qFy}2saEuW@2zj;{wC_8HCVOa#iVM zb8um6PX^;|gm=SBj_RWjsMruY^fkZaioet@2+75@$BFwrBM)ldb#>dP*2~N-4&%qi zv(vMrMiiNMNrXQq4;J8Af142gIxBDrPS^wkryL@`T%DrPhCC()CdG6SstY#@(WUq_ zfl_0qU=Qb<3KiLQjlDvD+d5OvV47nYV!{eq5VGw$XT~2eGP#O67qp*UnuJ#`| zkH+tEMl|65qCN(%Ubx3E2tJZ>-b&QXd5V`nx}Tj6hu6E1$z!7JiAfC>1M!kr7t@}|DhGdIv-#a zvODRqvR|y*lV+W~#}_~y>d#%IJ(QE1l9v`@vV6p=`uhQW@H;r-5D7jyPAAU`+9=rw z?~y#21tms&Y6Yo^YlLN)P-0#bo4wWLnG<$=5<_A&34CH2RY_W+ihnH-_Ox^e2a2GC zG?kxMe3w19gg4J}w&rZ-sE3CR3&#nCk>r}KV%IOX0Q6wA*mI3GVV6-u2gUzgku+h} zU~Hdqymk(i(B>Ec#6Z|mJz-{ID9b&OyVzJpBFQ^2T4)n&&Cnt%W3%3-E!yh<>#q`i zFEcv`_e<>g1XZt7SqfVZh08F21mcwwAy>-d?MSuR}4?_-%`qyRf( z=xk`@;x7cOqJmu;@fh`bswD)dL|21-2|d9a+NZAz)#ek ziPtgYTolUl#@@sa53G3D*d1*mhos6|{454eZXj_zxT~F}mmZ+P7ej9@J%img0!o(b z%iMYq4U-(+ICZv``MAV)@t*U;_mSo<+UD7i;-63Z-ORzma+cbp3Sbi?OFo>JxL((p4+hQj1N^D+h{CF;Y2=#&>k0P7H z&b(py7vRjiFFx9UX=7Ol7j>koHNk@&p>?H(fTfsVu9Ihd4tY(T_d>dq7qyB}S(Ao; z>+~1f+!?|`WxoT57E=0fp{RlbBJSUkOQ=)(5#8?rotcZt;-Wt34<@rGL`7xI`II$p zxW9-lK!wGfD;ZfVED6b3Zz#3JtM_uhqoTJk9}K9o3ykTKDVou9x!1m(w*^!#Jg)}= z*jI@;vH#3R@tk>cnPecA__ck|aSj`aSx-ty<5V~9*@aGAF|X!bGD5|D=@iVRn~E@I z3$%XP)m9iEU9+tAUJ61rl2IWirT%&kp2F`#La|)GMT2i@(p6=yRbTyplBlmgIUwUf!CLz3@25$!eW(TPa8L_X5hE;PE zLj|g42IUr}OnuyPuHuXcJTDa)8&}K1z%ud&pzO@kMv}E_+$d)aGxZ>f$GdZFjba45 ztm?&ZIe0qUwo=pCW{uFkRqhY(sU2RO=qe+WfTK9l?bgaYv!-X0K&jS~WIriQFOg{U zPs5f+rFvFc^|8mwOQRP3w9qu16jQ^7MsT}?@Q*Np0CpW^e9WSac%_1ZNDp&9Y9{^= z0?UfZ5=GOR;>*)WoZq>vGq{)dd^hDVNc%`6?Q zqHPujn#N1x;RSAkJ2pQ_GjAstFi$vCR|yHSuS#8-3f`^3abUTRD-#)~m7*-u6-ahd z%hAofE3W#g?JyxcQoKbIiLTq=K47ZroblR>F_?XD&D9l5*tX2{l|SsPh~Gz8oea%C zEUZ0=U-^W^PsQYBB+DAy-f9G515CSa6b(moV!pQY{>w1!iGBTC=IjDiP* zDOe6C>npkVp|W`FrC5W_V44b%)fX*Gn#(s~)Y&Yh6CtHq_^3N>WI&U0NZQE36F(~^ zcmRXQdL6Y7tWU1)-uI80WcL-X+7Tkrb5>{7Ht}M5r zVkO8h$x2zO>wuVE?poNT5_^sF;lEkl#weK8u&6?gKI9}x@lx#;dznf66dnrk5}G+Hls`sA z1aQd*$XUmt7`XJcfTFnBbP>C@G-7s*R|!516;hf|DUUa+5o_Vs(8#Vfwu^t~mIEel zQ5@@bERs5h4W79$M39p_pVUSzaN+JbE1!nXWL7Bn+6GYwM@3azt9@x>NR zo=pi#&*!ort?j;b+;vb_ao%>(v%Y?T0FCIPa$@Tw<-6GVCTRi4!m&rNBp1h{(iP!y|!!jRIdD33OprJ^n; zF6Hi=$}OVW?;2gu?#ARvcru5Q5^{BJ8E2wUNt47bra1SXRL8#h*kWi89aiIel|>AV z%4gmsr1ELdVSm~6PiA#?3mT<=xH<-t@RJ{EhvVFSK>?Z@4h!pqNk5A1-8+IQaPygB zR~p!oxUAS8U!jk&^fP~4=Kmlkit=#RKISikN6L9Ubr#EHoHE)ScnJ@nJBO;{Kx}9V zAH4&rEv(Hcuv46#4$<2@&=+$E>F67{2+oFiFQ2(U-?haz3?rw*O*R~jEX)8}dOA{_ zCQp>@H6=H0QQh2(!^ZPC!W%TSMuo=}h`t6Y-E_l){Y6n8&p@r9)O%V@b{G$879XFD~|9>j*cyNa^yJr zt77{d5z+*a>1t*PXG1?#S|dt&#b@dG1cUHJ-2PJhd@`T0M*8QK} zgL|TIAUnM&=sfhetKlk2>`mdaA}M6@P~&Y@E#0sgbGbs{MNvYc_bz(L?$}0iMgw}k z9`!+o7M{v983=n#PGKT`Rb)?2#_eCKmSb+jH2mYjSF`8k8h_LWfrZ9A6`A;mWAxJ3 zc?OAf@D78PFh5BK2F9>+vRE9vEb=?c)=I6C;MvVv*BI)zeXidgsDG>o?FuX>vqA$?kQb}2vfVoLhKXzM6b&dzB`)2PQQ_3qV0 zjyZAiaw4{EtlqiC8?Guy2o-q<3u7!I7l&UeWeEFO!nN!t(o%SEdfmr$6S**YAE?Ek zdSQa4!A0RtJ+MjRDoG(tn{VfKASA$QH2rjuo{0T)B6yB(F3)%d&a|`MG zakC5g{L^!;Xx=)-Cl)q8R_ac>!~C|oOX}g5s`;KgO5D(W=@eH0{xay|&b?kxaOYxu z4u=;~+TG5MtdPrOJAcr%5}*+ar{d<=68WEsESbFtML@{1JDL08N!KWFcp^#d#U~24 zZjzv_+?d1e48Oc_u4F%kD%rCh(nVMt3yCo~rf3dpg<3)&no4tXB%!($eggT97okj_ z-YrG)HOXx&rJ>*pZjvKuJ=F_k%C+FF{1FG`4p7ZOb6YtfdrNwIG%9RM{tIDdhM$q5 zITBQ|AWxDbhQfo^+&m|ga2r2?zat=*SFW z2HR61U$ET`{O+$3!8?2;O<88Cqf);?K$=Z2n|)-ckH0-iQpbQO_2ttQ$2vyhAf6fS z#O5J?9V887`IE;T=E*^d3FDt_ck?|>eQxF zYH$3;qL8ova_1)G}| zZZdC3mFiopQ{>4k8~Y!H2(O^(B=#1~fCw*?Ua_|d!0_p$*5T~u_$rd(F7Xcq0#SLEWm85j~WJ!l@Fj+FGHiNV?NJUb{w!iON?|Rv`XaK=PFn)vq!kNP$*D zT}+=yEV3T2`oKq}mE!>{7==rc!&y_SqSDHCI-eFZ2gBz2*=ccvq=iGF$3r8e-=p6lqTe#vR0W5A&(N zf~R`!?Or9BLTKGG8qm#)Hzoj2K(N0PebDB#d;nhO&(JQ-&8*Z3V&^deOB_7zUMjLK zj`Y1kA>~IpXfk4|zLv!lX0;OTRRuRp`p@ld-6Unwl{a~ioxQ)yPyQYW^~SGVUKwOc z;#x_Eln%aIcvG{5yvko+J)AfKJ?6?t=9ScOEu?{G1jjl7>6s$tcDnzbGe~B~0Q{{D zpE3c%v@2?+k_QED43`X&ErzX-bmU~}8Jk}3&7Mt7j;h_rVXYmMdH><3q@#XyeO^tM z<~;;#gMqqus(HCyE;fm(DY-g_i~lx~$9-IexPp-9Vcc3+LCgg=X(2P0 z@{paP0*5E+ke#Xml80g3T0+6rWj%2rJ=f!~g{lIahbw6zDK}^QOnipn(2cSJo=1QD zOj5zq#et#%nn&jFp1cB!r+55JOu^9QHEE$B*UZI%?8L)GWei7HftCk+97lY{$Yp*E zM|4KX#Y&FDUQ4zhsl4}xNMO&Q$ew4hDUaabW4u*t7R@_uaTa;KTI7eiL`X!a0qX*f zO)r5B>2i#^Gkz3zP~|`fsUa_GibT)fJvhj$8qaC^#8km^mt-;kEg zD38d^#mn*=vc|(6z)=JSZxE2aBz6+cf{oMxGBZpIt0Y!Q(x5yHvj9FqxepJ=HjQ9= zhihEwaw10rMfXg7>XX5}P&X?ggSVSCN9#6oOda3G{&+-7T#SmYE;7($jc(hJHJXL5 zUldQm=lVz2LF9aB&5oBX3opJpN-s*b*p?z`V3=C#H0d6{{undB`bJsrOoLuv$A~~V z4zbn0qYrKd!n`cS=O}(i=Xbf;ky&=OFExQ0iu$?)wO~gR7=;cFixtLF@m6)5|GJ&h z_C|A?3YjP(@{?ISmKz23Uj0{YTX?qj82la zp>L#5J@Y=0WU-fQp*bl&@;1;ykudHk=^Y{=K(h2=hwN%-oX&JJ@E$goulgQO)~&<# z`Avax<9_z7YL#Q_YdeAj`~JE6dWo&$!SUw<$+>Lmc4>6B_}SAZw%eQTH5S|HEmt1} zvinu!lLrp@#~)1_ocs5;`bkOsB-U+*Se;6_pAx|$CwKw=v^Ff%e{Hnx!#1|7dV_R> zkFJV}!*Qc5LKQesqj@xy!^5=7z4`-wQA4kTWuhH$;u=8lCI~!+qeUJBYSe6u@&{U_N(N*`Lt9e<8a@+*bOzf#SJXto<4- zv-pGhfK+CYeWHVLsPJ0IH7`)4AmH#(a}ii}S);A22tfhbD&dru)z6OZf}=nhDB*-B z167oy-=b3WDxWYpLre$*@~${u=p0cgxw>mMLDg$$IstHWS> zk>pEGJo|5{Eu<{JQq7~L;r+v*59H*XJ78kjDH`$L+6|Cf@qAY)_8jYWo2pxAB0ki@ z?Zerc|4fpv%A*kBh*a-ghZ(8ls0TkF`P+b-+7)yS%*$;!u4eSQkO^{%d|13WKrnRv zDF_^}dRxJ-%-=&|GddB+)l%c)%c;(=OiLtQHT3vxa#{XeQmV%?#?8JygVe%TCWYI` zHv~0H>%3McUH`T`j%35%z7HJD`61gQ za3Ytk5i<#^pz&lf7m#ETzr)JXkI}vq(>d|%BFVb8}K?xg~=b-3{?mz`OS03__!2_Oo4mc`4DdbYt6#QodK z>4&g?VgIZ(%(Z?)Kg_jp1C6p{0~{sk+X}ZN>stsfJFT20U!y#0zg6F^LM;Y zeJVdLH%O2BkDlkhtLuMhceyWKPC@J{1i3`zhfoJr&z9Ip@kZ-U)3 z>#$Jc-XzM`RCvAcJ3L+=>fd_1XWHTZw!IpZujTN1k#}FbKFq&F_RqY-0Zn@sC|@Jt zj>7NNfSt(aYfeA3{S~`si^z;jz)o<76|kL@apT55vWC--Z$BHb6V$N~el5Of4)i1; zv;}&S84QMBi}WM|c4E9R@C+ioIPm(A{(9l{5&g~PeE+uJYX2-Ue6fBb7P->A7lGnv z1WX{_9t$Uu=!pUdg?q8#{lx!!%=r$n&tU&7GpxIDqZ--Xx>teX=m-Q6|6C3)5PSD4 z{h4-oK=e?fhF)bz(Jd(aL8~9VSf8vfRQNNkPW)VRypBn!Ush4_KS<~w%9sA2U3-B2 zH^lmnz)z76T1e7|8nYFyO2e{>|01D(C_v&r0tx>FK>uB&;6LcUy!wCW>%X#}q93%q z*#A{4_CfnWrP43UB=w&uJO8HC|5xMnf6#w_4}PMi72HsHQ~TIhR7rznxe?XFx%`W{bp|RsvbH>_+Ayl72yUIXU+I!V1kA62 z>Su$cbX^VQ2BC7>4P~Ql+D9ntd1H`~ZPdE_?n)o_CSrn&E(`4CtXtm@*c2&QFUMT$nO zvg3+3Fd5|@>4(FOuk)aJrqv$9PM6uQK=L$6iHF}SN(L0@s^T~k&LijY^&%3Ykom%4 z<4BeB>Jj&Rmh?Dy zUd^b}O$uuJXwvd;`{NV); z5dz@(P738Ik}xE)x3lCl5J{{`zT%1}wU7`84dU`4zbAzt*o1qD|Ws|W=FBYQ|+ zlN*y=~FnN2qvyhvj znqCId!5LxsH|dE8*Rbt_5u%RI(x>1B@cw&JMMPUN=E+WYswiE^zhT>52W5uS-GV-j zSHlwQOxsPYF5##}SLjiyp>MC56~mS;Y47`?(&&wK^k<|Apx4To=N=4EDMjdl<2a4i zV~5ZysLAD>(a6M4z7;qtv3x%gJA?A*kBTrFrz#8m`ia*fL}NbN*^kg=9`>;Sic42+ zk`H)APgsnY8f-EbCqWPgj010{8VI;8ZslV2jq%iKFf8pt;FhJ=$-Y_C#*ROcu}AOT zJR9WC+c}fsecHjr-0E7rx@Di|G zdeD0m8~s~E0_G8t!#6Ol?Qx%HxC?KbQ4qV|XT)Gh%cI&KH#%%AbK159QI*2tI(2a;A{}zqA+?b2LsBKs`MZfmFYgyOvarcGY6|ioNX|S%4-QQj5I1qFpS!2 zDKLx-wBe`{4rf%j#pHX?TZC{Zh9V43Xk;v{aNH*0-YN?47!G{?#CSENoP>JOZN;7H zp?z=_(~x{9Lx9l4428pKDaEjyB-LB6TbYB4V&t;1jW<3=P}}_Tq&EaHU71+_KhDl6 zxU#5g*WIygcGR(L+vbj)9dvBlwr!go+um`~vC}b6f9G8N-`#(z)?0Jcm}9QBs@6NI z#<+TDZ-@(Ned2B@(ocwm%^V`$oXHplsWwSl{Xg~L zKzVPn!N|Qcs);kZEGMa$kBWkvIwKDt>{~0Cvj&+|f(^6k)a^-!2-R5_djo3U$r=c>RMCeX)* zA-@dmC$F*j3)@0|`M1%@oWI$Rh~O=cBQ|drkE3bxYsA*rEtKoTFs#3jt}3=ijY@e! z^R4_~tSq+!-Qo6!7L@>e$S%Y1b8>Wn%KH>JjQXM8w{UWpdH$#SNlco~gJ|>MRbq_Q z*go~@_k%>t-=K$o5TNG;vhFY51V1xAt9V28qUVJe3?1)gKJz6FVW-pd4(aXtX`$b` zq*C>cgQDykbcfosk#7lC6ur|*BCi{a_TMguZoyL-e8)y2i5v88O*o|A#oVP;LO998 zZfc9+iKRksI{u*brW%J_7sCyxS5#6-FN*;x88qVehtE_H8*0^5fpp7gx-u=p3Mz!Z zl9#jyDA`ddrKX3+Rp4)#D@wQ|ttlANbfU0Jvc%(-q!DBQ!^g#T`;#hexAdnPx2hFk zYT9Rv%gKo)5LW(6VlpW@(%>ao#lm-vR)O6Lo>JR%hm#w{5qJ7liQY4WRV<>Q(y5eITG?X432UpeMrlw9Fzu15t61bPzFCo*`cjiY3ObpxBuGYyr(3{GiC=5xc8;GI|Eb$&GLz7$7)3-6zCoNV;ja(xtf`*gDdW1knP`D$yfrz)x^)k${ z2KnmiD<=$Vw)Bgx+LO)gq|X!PF~0I-`flkf<$aq8xF&59>l!x6i`MPj07=YV~XrxLFe1_(q30QA&2lws03BS^=0W%~*cLXufc| zQP|VXf<+TA2B_~64SvD$ak6}F6uJ2N41qQ{vszj`21mCk)6cBpa{2Dm)q!<4&oml6 zx$oymUqJ~;g4(i*7;Tj(Dv{}O6crqY=X>JAy5b$9OU&=f1w4eIs=tS=-uw8w*@j^E`{ zDF;mM$kWLbLx=A$v`7>~9@<6Le+nXa9ZK%jzf)x=5(oXgJGcqvBLusXbJxj5WSnr? zW%pp{rkxFn`vZAX(GAm^f<35shxRP0KVj;@?KDQK4e5bd^|Q|0#9hIn)E>!Gyxaw5Hn{x#Pve%F~TWJOtzA9L%_ek>1*Sf0yYvTO`?RlQ3 zU32t#3*$blRfA!}HchZ6$?#vb+kTPLY4qNmDdVp@Unf4^iJ5!|6mLXXfkFKrmbMvu z-tQlIv5{ltA-K?31NUd!7NZLe<(V>bx+he#J`!o4r72nx3Uh3iX z>EU7!em;H<9Z8)K3IeFOA#rC342 z!Jh1MZt%BWQ}qVkK8pqjZFP;DgHoocl1UvI`*77)q3%Qkmz0fyhD0#>HtB~mlAEol>VCi#Bs}eC)=*oUPTia6yr!&n%dSO*h#tmKZjKcv}WT zl_0Tdo(L_y>_GYrYG4A^Al*4a00k$s*p6t^%99vx}6nT&`CDTz(}^oOu15{&Qss@^b(?d|W2a#0(_iI{jBPZvS;IHXbwaUR z(>GLMf^kmv1yhBY4{RRl)eT%8gC!?u7tl6f1Q3e)s(yKy@Y54^ERA(vw9hg)$dBR z$4W^xfW_zH)EE#4f}psj_r%+$3NDkyYy)YBOHt}L!aA*GEvs7tkF0Gh%UT1nmRsil zw$LBTk4$2h-`pxW2S2PSaf-IVxUDV5G`srQT6m{4>_bwl1+}Uk!w{_%*TT=|ZLRIO z@*e{haeRpe8A+YV?kmO%REJ3BRu>NO2RjMCNOE(^pcW8%v5hmqiv0m1+>dL?u0<9l-*13M2K8E+#S6sV}8dCX8hqT4InhWHk3L@YW0_WzKQqY9?}haBHUPHCpK`kwvw!IIxD^w?^d2Ec7_CJ}t`p+GSnSYZKD7 z(LW_d^S9@oU6@jTZJ4XRBSa5KxL*ev zmd2;|%p5trJZvWxWSrSpoM+!$U!+o8OpQ{u8wgT9xxJMK#x}Kwcs8ErQ=XN>V#7q= z4Z*HY&#(l=-mZv~2F`NFQN2|!!f@s24-&D@u|hm&hCN(a!tkH7yAnl{Xps})9Qpk9 z#)WE4Ni|1)XpWF-HpbH;i(rp(ot%z^`o`YIpN$0=#p<<>GJo&5L*Y9a3(^}^8*`Hj zl8y!YGbQFe#0-1H{gX_xbU+7T$KQc=;PT%`JT97Mv25e+Ps;p>GHT2Fsf}=|*ZMK! zImVDg;;*ybEfPJo2` z<7#c!wn6X2HtYujWt;+E63*WQW{txTM~`ijBj=y_B63vG^T50vex~6c{B=k5+EqA< z!CJz!Sf|>aqmj|u#7&JzVJb&cBbf08UPy!=t`gHU06V|@Z);%yOUgBccbB1DJ&`;u zER%a(C<&Ub1n(0_<}u8QAMq_@aApTQ58;T*gW=_)5LHCKO3sz-@zo@i`u^k;C_&Ps zV=K5r7K^V|2n{PiW}{+`HPI+lI79MeV78ZR!l`kj-J3(+pR47|l!JMSLBdtTsePy_ z$i1>EsIji1bQA4Qk#Y>{$5~OFDf6o#(-#sU>pp~kEqe03T0E5ZpBMb~gz3{SBVreQ zP$E0KZC8)e@zaUJ_0xz0l@@ZGC%6&zF2OE~MwE}zRS~&xH)y1;C^8+xB{{d8$Jc7g zfl!4#V#PZI%9cTeo4@sN9VRXoT&OtdN(t88{?bjvaMf^N9gcW^H%22;AHrILWuy^q zmuB7@o6!QI+D*W=wZL#5bjAKOI(C7f#6nLcgJJv^IS$^?4%WT7)KW-)j85Hvt)mle zH)A%U4!og>a-das{u$+9GeDliga$C7?5>5_;~?n|H{J1!a@1AmC>8GjxI_DIy<`83 zA26sTZ{-@~z!DjqiFK#m{*`Ha=&?^|bRO_-Dv%Bf$YH;VTCE+DHQi&^%vIIzA4el0 zjHaoh5=2)M#)Q#vLyDb~u1G~}3FZGzx#@ie4xk2&ivZ0mN#oc&W{-?WN!?e@dBdF{7>-uACS&?@4WWYofMV%EhzKU?Mm(m*x_ zGK4K(G9N8|XlhD*)UdRKLxl8>Js zx^Dna=1v;U6+%Z@Koajs=cc&ZgmkCuRn6$c z-~-o-Z%-hf#J_)AtC@IwmU2kpiRsazufAQbnbvl~{-?-O{-s-A)>ovCzOQmMVQ2n) zg7_TvPScbB#j7XaU9cx%e-57(^WmpV_qLi<6{jlCX~aa0r&CGYwutn+E;aZAhfL?T z25woK>fQb01ZhuDqe`FjdcK=l>p3T_*S*~YNl%xfM4!5c!gX19@spL!WZDWE^u2>9rdiU1HPHaSQ~0{Kof!H4GfeaK*Iayt&&S48|Usr{~}0S z#=iH_iK5O|JlQ3Gm+v`gK$usgGSbB5ICNkeveiDFC8Toz=kf z67>x2&vgP$rJ65d?K9`v#g5jObF?pbw(~IfsD}3XIZ_Qz?7hU6>83%RwMjbT4Ipg~ zVe>+j;wnE+5!#h{En2M+XJ21awEaBju>&%1SR6_YiifWnghG5^ zN-Y_L@ZPVI0Q|s5PV7ZuKd~@Qu78_;@Tb@IAv--F8%BJRR4)hy#=MT!i+({op5_f0 zeXw$s_#uhEV(+f>1^a&*xa)jD=`Zz#tlnSm0zSN+wZ6XhF8BrKJ$yC5z1CINe&XAd zfAwdcuWI3!1-Up_UznGT@o-bDOj9>~Duf{R-=O07Vh_BbnsYtOVGYmDjd$wIkGlq^ zYJv#Fo7ueKtKe@9uP4>Z7c~4^ooXPBLTFK z_xIE+eHII)-9+u^y(|qy?ZxmN9+uoB{sIQQY+7pjRS2|Y%*Heycq($)qVg(;UYNQo zJyCO4^89U6>rC$2J-4yL&+f0U5WfIR=)%ze`;bYj8XZsR^S?CchF6A(5Whf6=t9%@ zHg0}_e%M1a*gYR@xQA5$9b*`byGF(z?8P;O!UC&9=FZ3l?hqTyb^L*HJ>?H^Or^0) zFi7MQYu1#l4J`O8+&aP?d+A`_R5U~2I>8-)$xUpm$n`g4lsn>MgFB;VH+w+eNx_Vm zr>wy~8@Wj^I}Wb!?=N9D1#IIS){w}V6cPDjnu1)Xmbs-43CmMG%I05oD0aW>usgZk zVED7Mg2|7~3+kMrnASSf+o1=qM{f{r58zb_HTrV3+$uU<09`QM*rukN;oCY#IMzEOkY`;NUe&Z~S@@4a%^cm!; zP4zi?na}hfuT}Wgx!(knw;H7tFSd7DpI;h$h(wuZLG{>Czllf{EADq@fN>fTaTKL_ z5QWpj*gtSBTQ5u@y|hnmoklwnG-#|>7TzQE=M%m2U~FM(EsgAF-fB^%H>hdG`_5|i zYM9|XdIh}SYh5bvaeP#XWX3_LdoLJb&|hp2@(HIQ-fF0ys|f55E~ZhVGHxlZx>2e! zA{NfFQ3oorO*CHpdO2)}1U;A4QYd9i=Pu{jpsIwZd%q<_9bU{HVG*pi9b3p3?sU2- zn?>e);xKNF7G@D*m0h#RL-b>H#C$*ti=FXh%+-bYfyT;udx#4&PZN;@wrkgX3gYNZ z7t_0bFfNQFKEDX3He5k(i{Ph+RC(80X+cAQ)d>>DVPz`SD)h0vL)?USixtu<=8~~; zt#T`Q--Lb>c}J#;nnp)>k5ST}HmSs{g%q{KKM992j<#8q@+MJrGkDw*DmQdMFDuXogn`?r3qnG3DssorubjFxh`o&E6 z`HX@7xFZtj_P{{15pyjYV1WDx$C~wpP{amoSPZ~nVhvGzgvbHki^LFPplL+lIj(_t zn+@r=*NF7<7#Cz7p5!j;0oD5|7gQh4s4|=)OW3bSZqyNyYRhb-%R@hN-rGuoL;_ff zG963Coo>3j5%m)%OwUxGwFlI&? z+kqP~Ziexe%eumT3nzh1kAN%uL;)+q-A$4}>lluiX zho07^3&<7!ckZF==o@Vgn^4>r);(y_Qv^J4W%-q74ms^l1j?~B z(rv9q{H`@(55RVhLz5Th!W!AFo#(gIs(k$684P6Y%uRh2{FH-B$V;tV{nRq}N@t>B zR~03J^FyFXt$@SAGV)B<>mYiqDNMUvkOiP^u&Ng0`8PX^fE!|{y`I%gk`2uJ9^Q=U zDh$?nXV9|4*7#ZHmH7kUsj;;7P0MCgYGvL<@cc-|;zJL)o=p>~YmhxN#yQ5Q2w<}A zu8BqKVAd1{I9MmpMAALX{LRrxWZVVtxG1O1%W;%97Ox$@kY2^Wa+Wun?I1E@>>@I| z?j$n#a+x+3?7(WQsujOLUL|myX4j0@`x~sa>Rk|OQ=l~3B2^w%4?7^XYAi7K76b?B zP{lqcNcGsjwg!Ch1zWxYG->D1i)dKTtAXF9;+@E)-Fj7ByzS((&n#`(?$X?skM|&L z1Ux+*T82AabhC%FHI_Q)@xGfbqV&awT8dxKCxmKgU14m`354V^n!xSo4EdIfKegRp z2rOo+)g@re(x*9szb&VFS=;*aT4$^lXS{&5{80IPS)Da-zlTV!H}d!?Zgvqba;`AA-Dh6EggAh?%?f>U;LHl_mOt} zV1iZI`0H0>Ij0_kCP3Y9sMV_a#Roi$&h22b+BOHeZfI=1>!8(#@7<@b7y^zyPzLP+ zVb^*FyExC^Ydd|1e%-L^Ir$8dueNbv^$s@NfL9&7CM=7t;TWl|eu*SaL!FpyrFEKT zt)Po8dLkw_;%{6~6+Rp_If5FMyBy%^x%b`?zm`AYB0%_5V2UW<`eo$MBx{*so zwx�NwiYL%nc;|0MEO6mg}dK8bOERy|Ic^WD!8#4{}1^2%A-Xey@`%m(r^xNNpZQ zHQz?t_z7C5&(Iv3!Gc|;vhdatkm>Q-671HL&^!sjD|wpGs_h^$Hi-<@5vx;h=Wh;7 zOS4X0{r3pj+lK*og1?BD0p{LGqy+-QoCrNb!_T^_zk;URTcj+-ZsiB+^qD_Z-1H(n z5-tu%3#cii5c=$fS8gD{ryzPs3%DsP$NSCPd`swk%b;Fwu8Z$nOh>e4e9(&5i%2 z99C-VPq%Kc=|-}GN?X5HK^ec*)ouN}FMX~c%eXA^eCmZD)WjgNQ7|#49e%^I{TgT> zSg4j`9;QD2&R#Y&K^?~SApMxUboRMCB}Yq(Kjrx_1#Hejq(Ir2j`xaAJ;iW1L{NKJg9d5DE~{zcg}?dhNhTb*l#W%U+}bE z##zqHS>@1K`eA|C`iJV044qKTfZ78RRy9-ule$@pT7RMXI9@rFO(9ehMhlHi(e^N? zs-U_`C+0zIP|L4{2ft9F9(oDyLb>H9e%*?3T$M7g6>s{ef*Dw$lg3{&(rB^F)v8@H z+FFUYrqDv-rENQeR1SNA$euu`oYi<>&FZQmZ(O!y`q%;r^eJmK#a`C9sPyFQDrh|- zVo!mtwCyNuJ#eeM^k~lcEd_W-gTgGTwD}}2CoL*Jo=KA#MLmex;q9j=QS+pTA@7 zq7|1~D_BR8_QaP$II^#28@oaCRH34xCM4|bCEUu(T-G=;B0F>vpJ#UfB)$2jQ;K;q zJ7t+P=`*FRRWqb%K>Sw=RYT!^;7*_z!E4~Z$#$F|>2Dc^R{`H_y(T7?(OWiPtw)Q0 zsfyr1iKwRatbh*`C#xmzMGtzIqpFlBR-ly65hgjKi&V2vt>y`PE&pqopD8=I^uneB zkcumhvaIe*|5+hxnb{d_UO7pvIbB3?1~;LaS7Nc`(yRlJwpmhd76!<_oZ*3@9Em5M zg2H|eo_YF>{)(|d)4GLSHLAtbDmQY^R32dg4>o%=Sn;{pW@t0Zu#;U9v2C-ZU$k}E zq)*nZeKVZJ?wIbo3bO4udicHyc=FdhnX`FBzyc(`leXYiQ4Lf+A z$;M?K^D{O_=SZVG?lBHiZbV4B9fCyIoLkuy<}*cN*pOS6x|g)!i|us8&uY{Q19Z&r zO4SPzGK-Zng-x1&OsuF)%(G=y!0H6wnNiJxYX;b~7;1*~wJah#Q1IMQ^01%-+;5Qt zuPt1k$u5uht~EWKa@`JQrb&)Amy7uY1jY`t-sE@4t&Hq;E~x74_31oBkk5m2+X$2P zf+i0eag%L2fT1lsf@vJun+l9t&h%8vnMtKB{*yZWW;J__Ub~HI?bV2SD@Y8oL6bal zhBah*%MqoP0L##opd(9uYRz+*zb$snT3u-x)>~HS6K#dwyzS`T(&vYcy7r^Ce*tzI z2W}gG9dv{JTAcF_TZO04HQ7`Kp;PwSiMq=U+Q}&=Dcj{&dU$*F-6;GEr+hUnnHYEfd{{S8>1X1}x92WKeGK_8gotMv|2Wwrp(}+CK zm~iDWcpA$emMmbOuDLhUZFJ4Xi;_%7OnlXN98}7$+U(G^!liCrqr9VTb94QvV^c3J zAg};z5zt%OSOZkgcaAPwK48A_e%4s%@Z_BW`3E063Evtg8|pB)Tc2y$fCd>~LFsE~ zT?=#r80ZmA2R%@NrqGrA#fG1+-Ivex44xWoJdp`l5ixb(GXRXEF zgUDA?$*A=w`ZS*%SNy9S<3ZCa@%0O_aRiOWBW(02dtYWWi@x(asffdMKS0vxb`!&( z_P@m~=k1nW;_Bv3+D4?J%5Ue>+X=UWRx^%Ku~wrnnTyrH9~Y*T zi(=Y1bYZ%9F)cF~E>(=ls*olmTqzP`nMxyAnHW{!DGCOUfz&FwT=+mtK9kgX9syOn zqmYB2x{z9NXifuFk}=X9M1MB_74hn<#Fk%FNYcZIw-{Ck>%)*akDDNNEXHC|$X!LQ z!$hK@#Xg_dg5}o4HlOK&`cyBMFaZ_LL8senmL$&00dO|_D<=6EEM+k%4y8-m*|=3m z>7{*d4lPdXDrVa7Bxc|$a&KY~{&_*m?EmotsuA88ia`oH^LWn5yn) z=q=kV%q&Wgd#QW!6K2ti5_6~^P2&p}bF3oj!3$l&Aa*#ccSQPB9-8X2=PlO`iTT}@ zG1gPW`nwBr7%{rji>5Krm-3&xC}Wa7n%v#Ks1i8r$z+;+X$qNYW#b_o3I)EJp(u-F z3FDGPQVnV=s_W=GgnH;TT|S$`RgLn{JFVoNksM7E%$)5N&6)W4!XfNs8<~~qiz;Wa z$C^FUolp!QJ95}gKGu-w7TiM?8(hQ-@~sHo=vC!nWjvbJm|rB;DrVgg-krpZ!@;-X zZBCvK?%HO)dAZ4&W*A=jJVw=K#I2YYPFbrd`$vm|#jEU_6xw&YuIyAyU4_YB*Br0O z!{)#om*Z0^zx~&7AL?TnjRn^=d79OeZV!CsQDdL?FWM#^tn;SOpE+lqWWzr$wuRRS zMb~=b>Um@9-@X|=p(0`sFEafKBtpwmYUPsV-dD|}5fEMbh)Ru`wJPBfKOatO7SApv z+^3~M*_o@_*Yy9r&E1j@8#l zvri*(%Zc@~wtkOLMOm8?liTrR8yOT zrQLawc}p(oY(h9+^73q&c0$A{zN!<|Ebdk$H&4+n{&Y((?lymUD(gy=Yc25g&|3J` zk$%F^HS=+~QrLBb1qd{7UfoJFrq_I!#igSI1hP(22FM}mb(@a`I9{E ziK-j7lTKfSG}MWQMDFRZDxb_yJq59I-TOZESOVaic zx-{s#T#juPG{MKzpI9HBv+g!!+^hp5=Z;mH1>*Ma^elO?KV+XLlwP-CFJ_pLJiz4CkrbX#1iTC-T%KXupVZ z&%|7#(bxM01ZK@n{bZs(&YWyE9(ia!(tg|lm83;czziTaKb0(fqrkK1!K>^*J}7M#{`-?X`;=v9-d&g!6TK0Wux z9vRl@(tN5jG`OR_CH|NQ?Why*EZ1#bLp_nIc-JKN#?zlr%uW4M8NzL)32Rjor8BKB?;siNHL;L{YG!FbjE!tGAv zy5apxv%wp6pNI2(<)tw6ywtqG2+WTDVKI}NSi5X3Wyd)0mW1MYhi`(%O^UiMMdPv0 z_r1-}WNgR%c?X_swH17O)1eLzsVHi{Zz*EUTeaiLh-lYd=ZOjVrYSs#^947`;zjXi zj@DN^8tQow%6S~=N)xq+fW*8b!9yClRd}`-*&hr~DTNXscg)X;Q#Qa%eu(2w+e9bJ zu58(_PdUL8LqGCB0qnD()w@AJ9n8Y&i)comXCK6?KF^iQI4{vm3Mk5Ra=tH?TKj0REw?E ze0Qj`m0^k^k8j&0YBS}nEtTxaVo{9t$%5llL3)i*ECUmyz5Zaz$+p znp2W8YEws?LUfO}-cIhn!u+07SKT}YboQ&*UGB_8pEg5p1?(8U??HYBNMGr+PU2(p zvNl+bIl`AbKg%9vc)c%*xT~H&C>?K2N9JUyb>cmK`mi%Gb}n*b&Ep71F^&{>*_&@l zX8sZk!4aiez>#SfVLCHGD0_=RW!rR*m%t^ns=h^$XCW%Z9U)7$i7uoVvGiORD_tJZ zqS|E!Sw|OPN8jEjCWs1{fc4nP6}Lr?J5RZ1E8wV}I+@_*FqbZBn?M0r%oprO=w1iY zr@Kc;HRiRZDseh4kW2&?V<4+L8{&ua`G^h?e~Z;uGz|5%923@7$Vz{v8X{n5w29Fg z>pD8BSk=HB#XO8~8%VYxaIg&Wl1#-;Ln*jz(^Z*UrIot^H*K1$UiCj61#%nI_}==w z6?=CgJ74TfxI=JhsX*?TczCoQ+;DjpU!|Q8dM4c^0}%H7Rq8eZRrayH+N+lo6xGJ! z#Z8(WvmMmo$MK7f_BV_chYrWIv?Nsyq#?XORp z$R@WNjy!G6ff`y^^0x~o}6o!Dy*Em{?$`st;WH4#9J?b7O+!eBJEW%DhmJcsLo zUXgz)ZWqa0PxHEah(4S5ePlx%5-#)!|3Uub@6it^0XzEd9GA|UW`;R|zGbZ`7kQCK ziWHT*wn7KCy3l&{Ppz;g6fZW$p=aeqcxo4OPI z;xjORyo6XQ&I`#j!TAVdQ*MdW7{KND>ks7lfpd{=`+%zVrL8PX0yEE_%o4 z|6}|Mu{<#kjBUIRBKS|)cfeiUH|iJM-|_D@KeX=b{VNmw;5VsX+rx6njk-j&3~jl zqEz{B4g9y2X@(%)7F~do%A8#JPhI~)1^-s2NgSDWGX#nSmQa;NmQXY^T+xWu=z^ht zr;O=MsBZtIy#7<@KSBIUEi^+YsrZ*Z&Jw!t?`4T#O9?WxKO|^s0ypM zyng7@5c+$noI)(6$zbOPsImBms+>ZxCCa${5T@bxA5l4lB$p{co)e?-2`E)LMP^Hy z5c(la^YndO1s8!WZ9*wOM2+a1N-+zVC3Q-I5N!n?!X|x+vXD(hG}Kjs3`aao%BVkw z3NCVU$^=Ayj@r-fmnAHaQb`kP`QX{rE^K}-EPgH=elE;@C0Tte#Vmv;G$|QEoE1}0 zSL_C-3@IHMkTSI^lZ?0(bZjP4+NKi{+Q$E6{!ez}652-N5+0z`X&GIPD%O6V6n&O$E3kE=(W5_gI;s`u3TmXTH*4BRoz~aos9S5i{g-AjdE`%@ z71RnuT5seAP9BBRXSD&>yGP_cfeLGlFGH!*Wv#$Zv&kcPU5+x~ zdUmUs(@5$xXDhI`$?R_seHO9pdUlhU)4!Kd=yIgluD=^k9#zt3*|Y*{HyWv%&Hjc_ z;3EJzK*qmoH{LqPauCH);4Aw76QnFhu1qSBPn~YvJ@B8{{_Slg#*EgR9x#hG2i>QW zfRsI_P%g>&&BI+&iCD$Dq~dT3PZv}d6XPi-{t1*AKZ-E#RYJLfmhkUJ?)2ZWu>TxW z^+aQV=Zg?fY^nyxTrnu_>nl2bhuk0cwXE?^Um$hHZur2jpztpXD5!H+;WEpVgq4)S zNGZii(cqxwzhtpM-?%##&ZDHasVgh0y}We>9gI^k_3>!#9Gv#fE_V43)btq=*%hC4 zQSmJupXP)F&hh6~&-LVff3)!KNQy+f&wLY90+W_8o4=1w6o>AHtgd`Qfq&|qB<-fu zE`HM081%|rzW^*Ds*HkqrSc^mYF)s&u%} zAX}wRiZv`g@K*3X$c=-CBSXAc#sI?K2K}*Ab+E1jzev__-_b7u9g#|d^qIh`zqvwY z1#bO@xY1|Ayg+h=VC^R~M0Q|WL(l&92zDIQ(_dmp>;Rb#X$SEZsQcS@SMc}eu7bn2 zXM8RcZ%hJ+EAVS@M??AJLoDGEpRu&}s8`sd@;U^bbHe@WMMNrdk?p+oK zhIXuMXghGH{^?zBCStvBnUDG~y$}Mzm|w*IlNb|hiY`S9{_UFt%(ri1|7%E|5fEr) z=W1{BU#KK?c~x{_6dzIJRBs^>47DM9wZ#0YUFbxd0%PqI8VDJUE75egsw`=1x1@!< zeZL!jY zbH8DdNei=9_`5Vu>BQkwA}KIPSJS178qoC9c@;~9GZ?^1(Uy9iMOsHP;!st05#`#V zjlSUW{^yJ;0wR=Yib;;;inC^?6HZkY*(2&4D^0z}^w;S4HASubPRPt?{6Ki9GZ^%* z@V)m>LxhAtYWcQunT8#;!+)~{bLhISOoJUysczc=&Ts-mtZ_EUx=OZnsimJ0Bw4 zkjuehm};lsUGmThQaN$x_T=9#!uL+gPKS+ zu>t{-tXl;Yk<{Q>x6V_SaFeyBg%xJ;!+|;a(rz|sL?O!jb(i`N0-1rDMH#Cm(bfTC z9(kv^28CM|$IBDk$a~&G)JOK#Y*GQGD6nR`nRdUn0*!EUYh9KGIi-iLq`n7UX3jer zLa(7>8~3+W=ZlIGF=Dia#fe9%1}jVgG=jM*vIrvz=%?Ser2FV{_dD>JggndZeh78QZSuY!iSK91+1B=dsgpTlqKIljW#_Co2hH?ut(Ng07?OD&t0m{#7{m)= z3vs6w7$9pBN|tbDdj?Nb5stTw+1}v7pzCwUyrSuP5Zxl)W1a89MtN2wT*YD+1wU+t ze0!3V#qHubkIgKsHt+vTFLGXcKyH#|{bZEehzwsK9<&wn`~E+Rk@_!(P%7BBZyHel zA9Pp+7a;Jzh-J2li%J;2R~)z-GX3(($MA*aUP*UE1M%r}BqY>M37ywzj;wHiu1#0? zcZ@Gc-~JQmqW^Hp7i;P03XK+upR&C?ubgt$1P0HIFOF9Vm+yiC* zDxEt_HiR5oZQj~euov0}$tsSqp|z(Br3aFa;I2q=^J-p$T&G>Uf9?N=Rc7BlIzR#y zl9N+NB=ivXpuzTkS!HYna0R#`t@L3VE>A*gi&A7s!G(rkG@>r#P>5D79ejlGI2+9b zlRL)Y*pbt_cB*i8WU?1>ZBCu^TO5NMfxba_%f_q`ya>)5n+%qzk&SI~Ya^hQ}Rp3=~v)h~Gz-~Fx6DwYv)-H-nGa=$qj8iV#ns<{k2?NI40N3oubZR~~JHxBW}P8-Lr z@e<67BEtc}X1>svFU>fp;d@1#?M+K^%d2*)$XG#~+VgRr7Q69naaNilLuNT-8)s0* zoRt+s_H?6>S{sb5@$B)@EyK0gj|%>(Lx(=v@trL;j?dy=_KB?`9&sJq1+V`g%QoQP z4nv1?kS`gT6i===xYr=3tyu!1~-)ssmqt ziAQ-``{CNaU*{_--80^P*RX%Khae3fVJapGH45GTVUadbTSQkz?@gIsh#&dDA+krxp6u{|}$`vWQV zc8^5=CiEKPlhsjD)n%*e%I(}ZQTaK^q~AiIY{U5!!uH%N zpJ2Mo-T^pXxO)<2Dnz1>3q7%;S-40)3pP5aRTWtrlUHrFq~o+px~jb6l*KvMyEpSd zp$k2oRLN;&_@hJktt_20S|nml;t7K@uFakHTg0_CVpvrxxnGrKcb7sB+t^kRkG2jj znwOn!rt#t(d9L+!Lum8vT4+pEha!ISNf#R?9hV8JC1oul4VV*ZhM{1IA-NWOux$%>6TFzQoSZNA3jUCwzu`P-gB-kH;2PrP?G0 zW}|1wZ~@Qof?K1CWMO;-j8FD^lKSy&3yY+%J+jXfJ$0SiG%I62$D@=$Jo{?Fd+huF zjRRlHpMeAW*L}hM>%JuZ*Y^L9-OXL>OdQS3|7QuLY$$99p@e=R24qiB7ZZr+P88T8 zVYCTR4+j*AkdZ(}N+o-lq#8P0SRVXvPws{6DX?aw@!SZj|+Z{EK3JkDBw z8J)Qn6!Znt8uEns9-2)-)2M!6o>L{ln`l#Mhu5v2roPA#+%T?)=+m?$&?d8SOt+hw((i6k4XC~C|^ki+(zn82tqP@D&64N2K53Pf{>c zc2?l=u6)-l#QALq^9xo~iixVXGO=92?lLp}0h@Z+7s63(2WPR*mq+W4V zF#v8%b?OKs9SSKJ-Kf86D?7v#I*OLNOFg(-(X>)aBf4&3PUtBj-Nyd=_`{@jT?S#6 z33nhl*lLQ&^a^X_Fm>z>&HTi+)T0<5dLY4-`+*pcWA4a)84n@tu-k(a3t{C)+4Jxd zmA+j#0Ou9HbKDJY*@r>qtXG)!Hn5~ZLL0Nw>6#wulB{Ru=63I)WCiq=Tt-uRwY7&3 zP$Pf0c)=RF49!mL(q3uRZQcfY_7!o}Ab0kp|65IrHFz_Pm&Bx>bL`{bxUM3tO{QZ% zuhEiZ+j1mKD&AvU=_(rElS5ncp7olU*W~qU!PzVvt0r{LUEd81af|LT8$C=I#3Ik2 zcgg)A_f@k^);6=VpQU1y*ed|;;Jan$9*gblBkg8CFOG#WIi$Tx(OLVO;e^8<1K^b{ z8G%|no|?H4FPv(=08_^|)Ex$60hJ1a+I19GncwT93J zSsK>P*ZJ`7aBDHn6sLMCHcA&)Y+6s>QjFIY;pueibbXajVLUMFgiW+~VuUtJzt89Z z#_Z{IN&9N|5vy5D!{AV>QUXy2_vyR;4`=TbpIP*5i^lw7yW^x|+qTuQZQHif@fX`p zI<{>a9ox6}z2`pc|G5wM?8lmGu2nB{jH)qL{i>=BWX7~Kesg; zX$r>cs9=}l=-x-smOmZXsgK$3GQ^@(;efbg_WT+cB>u zpt6+i#n#0c5)9L%S!+?9PKI?!&X zbq)?uf~0i^{HwA1-wW*H3g5sqN`Sb8W>lPxqiLU_Xk@ObS7#_$bAT*$Eg7^Vf2kW- z)`M)c-z83siZ;T@E|g0YfN1#0eE+T|da6gt6ZW@%asjrA^e?x7O_R50<1^_LKkezY zBt{}J)+>})h#~yGDx|s()HSIxBCiApO*ab(N0qQ(F15?z~5f(0A3d1YW z>TBNFwpE_v4$?ntrGk4jswT+z~)~<{xlgdPtZuDrrV`I%VfA$QDpK)|$ z_1-fN(V$zhvWedzS*+(7)9B$P>I|Hep}b)gMircZHWQAY%SIo5CSBE8Mh`+*3+F z#pl%ff4zNy_fp5{reNz)df}6)0G0b(T?k~`RqG`e0%*`KcDp2J+T}xQfI$`TBjM3( zVH#;!2gjv>xzwbTB%?qg^LlKu@d&(u-L7=$O#b%fuu4ZSLvNEUtL1THs@6N7ff0&x zfVC3yAHIdL5MXD6nfKv7%0Oq2A`v_n3+HQ zudI|%ss;GAoU1;tBOW-J)q-Ymo-S_u+Cba!cqeQ^W6EU5`e;frqr%cvQq}=}M7TAM zrj16Y!dF}Bv(=>5oUzweM`CmPLyX1ze03i9svW)`*W08T>4$Q>G^4)4>9idSy1o9Z zQ-l@UekFCug=t%y9`Y!K=Z3>v1%XW5gN*9z->qsUTv&d0{MTsFrs$kYT;ik%%EuB+;n}A7Gc=TI?& z2`;RTl##3m>cKymoSnmFDxu1!5lK44xQ#ykfEX$RfWyNZ`USEiu&m2$GNR@tgDnkb zm?7L^lg{I6ub?{rU!uIk5R6;IlnhFH-p+9iF9BP-Sy#K7X%K!$+@X~kK_ijeEp6qh zTBdgBrbMw*G*-GkagZ%+NQn`*#hymNEMcH246ki0;T)=7P$=vzB4+wjk9rx=BOMRX zJ!`mg3*4|=8p?B1em^d(vu#B2&6`x1H@Pd~$AGA0WrsbFH?zf&(;kuU<`pN&-#soe zY}DhY_g2+zhdrfy7PSUIGTR2mqrA0-Wb!pP`q(mWG!Fv(r2!Enn$^w|j{28AA1=BK z0aI7^qU7wR;{>WL8+J|fm~Xe;*p(&Av3Q?R2H|qq3jdO&Aw>34gQfcNKT$5>c;S%T zN}8}3x8>4)E_uhHF_kJ%UK{K5Tqd8W9`+Eol9;C18Z6vySbq}C4OkDmfYWkqyM{ma zG&}8)AhLroJ5A3MzZvXvAVRXp0d5JmK@ZEQJifs0j~tHNBrMnesA;{8@TtxV*5aj* zk^X^~qhIx-}D*0Co+Vi;5`%D)xqH&_Xil z^9t=m1WfNC)p!fKV8Txy+a{6GM>C9dNhp6#SQ5zYn?bs{rvl2JItOb*B$^*iJvWc` z!@vrZK8u~w86?`IbwL!<5sQgo>ybSE#}sk=P)fv?e9wL=@6zs5u=ThhjesdnruquDX^D0Q5mY|tG{M)v4vG|cMFw6cY=q1t%!F`YUL;& z54-&RFY+tHs9`mjt#AHvVh_cXNCIxE?>)b}O7Sez51fCop+jy-vJ^#*SPh`c85Qfe zhF?1_?LVdcs&Bin@;7!5+DWP=PZYAp*MjtIo1>p`uNmP}LTgzeThL;*U7%qRb$WHp zbL$Dx#?F7jk^eN%<7Nw!uJQWA{L7CojyNpR&0tw>>V8{t^Hq^fdyrcfIUvapB^D7Vf{NJ>j86cT4A{rmeIxHwjz=__L}Hp>IA{s7dn|=`~dMkwhN}S zHFE;wxKpeg0HxBs+)nAR`=5IB5ykUtO_NAU80XJ=|b3$&UUew>}4Tpm+X24?D9=4*ZS^+GEO{eM@mWh)95T z!C($fBo##*yB3UFmKPudzdNL%G*=W%q#X-3H~dn~$opx6mXgOUzRxS#1WB?KaylRg z!=zBkGfDxkKc_-PxObdRA)LAU`CVpF1s|zNB6u#qML_bpjjh^m^iTFYKaVhl@2lk3m(BLcY2%FUQ)@ZUqS}4iXhm$RWR(+++HgM0KSfE<84R zU?As~%*gIe3B$*U1=*!1u=XKZT! z--Uj4nzj7`BYK#At*LH@O(^0p`_;2vfJ09+x}srOS;|B|8JXZ6b0eg3snfjW2l^MJ z4`?4Wg#N2i!~tu{oNYvenv>7p)frFUX{q^Z0e*i_2hdh#Y@@b=O7e+o7#x_IY}3jC z-LuFab!`AXQ@)sRq;|Lv?)AIkg9)+vbtTrbw}z_;#i}r^qI$YuLo}Qi#zj;iJZ6}7 zmymDf5~di2$Q#EMw9iN@;A>Q^!^DBpScNw)%+FXOmCiF_4SLNA8j)7IDni$(D`_aL zuw*ILv-=E+Va91j;a@i2%?X24tnBx=KZ$9z%UWpKSMIPNJ-6gL(8Tsz9J+4gBpwRK z6F8Uxa{vQs8XbnQ#A;O2)}F}}ZpKlETz)50Rd*YJXSgZ4F|%UMygtCdD_N5#hl7ZI ztP!ra+}{Deu+6C4_Z&Sa9$>#~u@ZGnc`!75Z3vpWxFd2nGpEfyOdN9x`;3BU^H|gn zbBq}3#Y9bOnM+tL`d5;ku8NWs2XbGX2x3?=P>IS;Ng8|J)k#jswnMq8;}Q~R)4e+< z+kM8fnRPP0bl|~EK;H>EF!deFA<{CJ*=XN;V6*^R1d0?Z}rLR}_ftfgX|O z)UOuSsf{^fc*R+OW@YQoHE%Fxll?w9UK_;yJ7kFQ)%@zT*YRtb{GW7a(Je=@wV^+p zzIg9K6N%w+m%6K~9PQR`uB}ejlW}@lRY&nOcmyum%ig^5tjx@zt) z69Oe$*0jS=#}`#<9Da&tN28DyUwXqZakqVsWqy)=ewG?mrzc!UjUtfb+EE#TsW2I( zsUs=n(fJO>DZhfd{kcXPhDAESnICq>9*&M+UDGB=ZgPb(j}$8ozIRbz*Gw4o+JZe-@b zitGiqS=vO+k?&WDv2ngQo!li@u5$j{WxT)%yj>e9A~Y0;vZk-dKoV$s~h}02^5UKbP5dDNal7KL3?i zwZ=pZXwx}!>8pK^E?>nEZ`9axggAy=%(5p-vdD@%ibED$oQ{Bq5WOnjyM@TaY~JNn zy(z;KQkV?5wKtys!7#(c(UBynG={+{G<%GlIQo)$_(Q(Y{K+iBn_f$^6XvQj4D)K{ zIrT|?#)duKelDQRc>hQ?j9pTQeC{^!#8+;JuHmXP;ObpJ8p3)jwwH&UD1&Nv8obAk%dhARd127Cspq}pzH(x{=BJD;d3Hp?mgp_gZT(5zU) z!E}1Hm~-_b8c$k4DAH1_N z&ACf^zqlgLNv}-QmA;E2TK{y9&5LiS90EHtLdmt;|cd=?p0jX0$-Ife!1EajceUh~ZP&&uumEfLsI z^M@e?exc^X&1KX33>pAEB4mi^EY#62}NY!NRdM~5k8TSlj5WJ zG=2l8-T?F-qox~J4I@sfd7of2uPhjwq9v(Iu7>q!#IIFYf55^+-UQXsh*DV&B^e+2 zZ=)F_b_b8RVV@trsUSyGjg&R>6Z4fy5`njX^1T0!spj23G)ey=8^^yT!T(cCso1+Z z8UF{Y)c@Dr8Z=E@YSAF_59-)}W}rR4Sd z?*h9*0-;N~?h3K5YL;_F-7FT@NiUbIuG>$x`84JQzQ10v0<=9WhzIJb7G zaQaA*@;NFF49&lsZATEkIFa=31p&^bL%}r(HP5R%2Ws#vfM(T1j<*H^SpCe^pYR*Q zw8z-TkwfY`c=K>L^l6zhTY(oIHGer>XAlOgxN^@V`kgaLVk+SFVq)Fvdl}vF1KVQ3 zvr}{@uOm22p9u$YTwG%}Bi!3##LwvV!QqM4UtnYa8uD1bi8auaMK7#)Q)Y zsmZA&DB31CNr1-8U%vVt-huPymuOLiTWYpVl9UMsqx>EXLUS^uY1{*ZJdh)YsF=kx!=Hr;fSJ?6@k|l=z z3QQn~9Wls)RuZ|o#VTr1{(U#oe#w4~aWIirs;o$0O?H% zDPKKrbrv+E+>>ham>{tj#likWYiIhoHp&~h@;gCXigdDcP zz^dQk!~#}!XDe3uj5o3F!l}r%+$q}pYgTkt7&!pZ_7P%X!(e^1L`Gj2cT#1gxK>Ph zX89?u$^3L>&o@J4@A1(0j`BkpDon7=@3`_0!mI`eU=vpIsf|4cpF84$ zEaSiVMB-doSKB?-U?%%*eQeumP^zS>qcQ_^mkeQO-mQby`x0qc3S&EiT~&(cW22sw zkL+o`gWAVwXlp+L<5@hkfVaG{F7D?rBZR)bA1-8j#V8}>6-&CpJseSGW>?8-lvxkW z25=vA1V~NIf!7ZWw4nwsgr09K70QHSxc-7)0{}E^F#x z>h!{tuI-~?=93TP)}DY8Uo zC{R@C0Kuxt4w}^qSG`*Il_xLT+6AvaEh`swdfwN6wiwyKWBmPh1g^83ui6=wE45qo zV!imkOW$>4>c`@wP@Ne|MaCBPfo2x2X z8Yo;1Ro{Z`p{+UZ7u9PK)gi~1Ayynqx(@yS{GDo89B-&dEf;HM!%(AHaByi&mzo5R zm4RnoV3Gbf3zgx-J06K73GU|NYBuv#vmMGbu(EXZT(iEQnXuZHx@TmFM$_s_cQxMP zy7-=ansaS4Q&Ux2oZhoB*>4<~(os_fmw`*GmXi62@q1G1j%6ctNnLDMmH`o6ttv2e zQGI1s?BSt7uZqLGpSf($=BBZwpLR^NmorLCpCF`Eg?O3 z%kwp=LDfPq2ij}x2RrtY%Nn<)r-P@NtD5c2<@#j9otn@z1+V{8=xkd-7p7UGd*IaH zzOa3H!~>fd=ckAop36ptzYMgWwR#%L)OwdN40vPWjuiBj0Uepm+2lUQ%(mDqV_52Bg&h!Z>XQL0I+9KANvOaGL$rU zsL=cm!#Y+?Y!U>NFlmuc!+A!-HdY;M;#g#HY0^-oc`Cyu)?bnds3D@%N|H%QAtuyX zl8MS8O!L@=RjlgRl(A^a#j1u~ta=$g$qXApuIdCmf#FI{=Tq|bfAX3fwi@FjYNM(0~toLBwO znhL5g-fNyFqzEN~17=Ey|>I8OF687LdboI1|>bD?}PpPA*?eA#3HGu}AU7@Dd*9=A73t`p7THQFx0_IQ zH>q^xI=^~7-_~i~E+rgQL84Q{`#7KTdGrwE)5_1)E6dd_%4JYxz${OXUaBNNe^hRE ztlxEbigFzE>U729*KzO&%!!L+fspv2s7B7I=X1N|Zj6t%!0bGs!9|cfpHDw^IXX5r zg#e*mrP&O(F`3unRJ@DL!_KxO#&(K@PbxvL`;#*%DUIH>zl`%P1UAX|cHh9^@x0>v z@nXjdUSis(y{hWN7i(H@`DCUaejN0>4Q4BZP?*7x*U(+~cO9jneSdezCoD87A_cxQ zXSM^`o<+ZCzezt+$N|g9-BK^?dz~TIXnvAqb`i5B zFt(AiEiji6in<|g5*GF$=cFz_gu_V|k%5IM+=Ren6m9}wBMLVWFdC%;A6O|V|L7Hh zn+Wm9o+MK!*2b2(RXF!oFV-x6#p3A-cZ%Mc)mtIf?7Dic$+sC^^0Qe+61#54rxm` z6Q?7bl}0pp3YJea6RUuUrN8M&%W(ZJo6*+OE`wqm{#y7h+_GCRgWlONgT9f`2cn!) zwlO`-(p?`WPsoc&Hrj(r_Pw1|cIDYBgZ-|VkwrwFoVtM@^qfM+SPi{{ULiQac(DZ*LA zs9F%B7*bD39$I>E{>@mEU^h`&DE|52L#4}%3@{&AVox0&;y7r2pc1V$WwVN%PL9t6E5ufl(& zYr=1)3NFP9;|=BypsXXWM=SwjfpTq&vn8`;w{p8vKsruK3mFTV*VeF>NPAFwc-xUN zk&R@BjM=zZc}K3BX+?rK!V37yf^%KqmJ!#@)hwYS!c8u^RX0adeC8$ml1k&-tY4P*(#eba%oj%|XpH?V1#uCOVe=|BoZD$CjUnol#(Xc43 z38|Ss)F@TFw+3D;et4ZivGt%>6~$Tuwg$8CI=Espx8SxYo-Y;Abfs0weph6PxR<@> zD8kd#kS|I1dbb(6{p~3N^ZiTSXrWm8!tKQe#tzIK*4Q-7s@7*{_ z-bC4^o(A2v&8M+~N()WtoOdrq~^(FU7-P9OZiKzV}ytFdFF(|r=HzCO$ zW*=1xwnIzmN#mpaf&AjeHCE!l!e)oKL)?u<<}TBg`l~r?BVt26sgrLr(35&oN$Ae* zC6^QaW!oz0c_2r11IJJB3fg+~lW|k5f#xIPMTxEB5#dd|%k@p%7wsye`wYR!ez|pb z=v4cm8lCQK+4XRz#Gdlo%@_I>(bv4l@5HO3cER1y;I#8$Yp`>N|Clqm7N%ntyrAy7g@`RwRQ`l@hB z$rSXM%)MpvnIdgZ`q|_MC88-%xUjW6Q}LiaKXZ2CIP-Y_26?BicxGFozsRshu)uJZ zKIX5rCG|z~rq+e?rj~Qn>L*uBn(U|B6PQ@OGm96A_n4*XuTqQ_?I+VC$K#`IPhvNp zRo<;tJP_h1VNdKIy)C_2(TdV~cDY!iTRaf94ctMa1lw7`gilIw_=~~Ng?W!Nj z|732#%_@|&_cqQFviCIZtVbK+aKmo$?`_f_mQO`A7te7_7~V1co4osA^q2if+Ton( z3HxE0;|bTrcbMN;A>}N>d=W44@sh(LF9uv(H^p)%Hdx%`Lqt6I5R~6$))FH_{zyDn zdPMyo!ZGbwz$&}wH6Dl3;_Cn^$2=AdieNe>dH9xU;dY(VbT&NLaXnuo$Z?nx)s%Y^ z#~gwC0p}zJo{5hS@u15warlqHEO55XVs>t)%|s`HIVUl80?O9~TO-CPU8@dyw0P{*pT<_SQWo)q%!( z2VOXVhV(I;-v-f5WEv$=I8k)6=wV_PQ73e3VaG^4JsAr%RNOvRHF7&Obxh9$>k-|K zF~ZTB=~~qKdqc_|I<{+|>fQx>+j8C=;iiR3#V6C1VkotJVw1o$O2N5RYKQxnFnND$ z6aIex{A)BTCl<_e&?sc1|a#El|2D<;TR5CyNq{?yRe>p6bi;}pHk-D|aWt(Z= zh2P;V6@T!&2P4Fqgg+MJP07HTlJRh0X84_cn_-}*PI{onRJ+gA*JY(QU>xc>)fHzh zao^zy9#3$L(3rT+QGuR!EO(Dd$TO*ZY>Ru{vL+PV>-u)V-!QkZ{AcvE_hiV%_et*Rz^n z&X1>>TPIJ={IbB&U8}R)k7H2-+6&gx+>UXwFU84 zc-dEiEB5mQw+-9wqZYz4%DF$8lkT!oa<2QXw~`;Wp|wg%tC**G%CqS2{YmZ;Wxx73 zZAy!6v}GyW=OWjrWpC-?50jjhX&Ja1oQv#5w3jL|M}wTk)52A5`3wW zLKkUbMD5GAm1DXjoK+NT^R;VMg^%0yzbyaIshi~eqVuA!X0Ie4PbsZ(i@u9pGF0Mk zy!!RLS~jN-4Tbr3 zl&e^(>4iE5lm^t!@#OD}hk~rBeOj}-HtGY4NWizvd-htyeQ>j>mT9voA$!);eS=FH zsBu~?Tv{zX!t(Bp8kdZu&zr)Gg7*zWk5+c^SE_%}L!E3|us=_ZLUg#0Pqyp#!>L^h zNLp%YPO{!UgI=0`Lh7a2Yu&~f*U(z%L<3yZR#gDY$UX^&_TcXft%STh0uXZ)b9#Sq zL$6~n4jRx-Mk@xtzu}H)yidWOsDOR_Q?ptSEHH*(vAI*zy23yb=83*R*!eA57#XVZ z>1R{S7kDtc7F4fSr_CxboCmAv5F3A#m`~C_G@Q`2v%1k%2g9`?@=(S_o>}gMKdV#Q;Dv1tky*Y06Os(mldg^Ny|mMG zv;pU_jHZq?NKb|==7jqBWt@BozjZ;P*WuoiloWT7cG+x3v@ny~(V!+7ZadfLyVUi| zvo1jk8F_Z5o&pndQ;*Ev`>1}7IWz_vyhIy}53EAPk{YDsI|5n}hj{|!qV}Za1C9AP zrvr@nxu*k3y%V!^K_s3?+U{g_VFZUTIyfD43Qdcl4aV_yy+K<~D0GyDjhgHvY6H%* z44oAX#Nb&22_Wq8w(vS|2n@{LWgKa)<{8WubhknH$iFM!x{Uq*yt+Kyq(79f0({Avo;7mjYZp;-hkeZ~FdBDT|O zD|V1|T5$P+=_+`U_ixiDdsXF+Mot3pFZzekDZjp{ZfNaYcytlwfU6y7Q0s>y<7A?K z=IAJR-@sS9C(=_X*nug(pj*X=z(*E`<02K-FZ}F4RDp*#NQj$$Vg2sfRSh_doo7zYv2e=Q(@iv7y3e;Dx6hCshJQDp_j0B31UJ$7#jjfM_8`T%@+ZY7s6W&atF*A zJF}VXI)Dp|CnbF;trjd$AXl*BHS<;t9T$2WGp&k36Kdh2FHV`(JU}EY&o0DU}{)xirAUHWnkUK ziUY4o*2Pel-BobKr^&4FXrQKXOebL4(Bi1j8Ji2f?%skC6IhQPL&CZZT?nxeKin~7 z@h85UT$2kcCVe&oc9_NmYwlOrvTW1%EN-;S&(hFc3J6koA!l-Km75BhGGe_aD|{rZ zrm&xL6(!P5+k!MW)vgxB1d#92A^>9W!4 zITkLffK+|xzv~KUQB5}L3#@!i$ya&mukUuFAUu}tb@jUO^dE00KKMP}i{8^A=kFV% z>#SX*aga4_v$odEniNWRsKgA5iMgIFP3f z-$D6_9!mWAz!sQZOXy)lq4hPbTh>>?tD>$PY=kragX$}#5Fc}p`*5eiX~A)-{4kxl zQb9vAULT(&qL*&=3RZmPr@xG!DB`B@QI&^b3b_Jl{d4>$%{Ica$BS0lfnobcLAw5a z`d)ylcB}x2Ol>*!z(z*KgEB?ap2nXK+#bZG34yE0wVT0o=4Af&bXCIbQTwQmwDg^0 zxom`ws^RR%5Nr4<{rg{aB?e;Z4<{xUr=0s}uQ?~K@n5v?rB;u3?AwSZnAoScAY#p? z&2_8jn`SfDEKh) z5TL|ReW5={+EBoM2KhzS%^spdI%GYG0HG;5lM_hMgSvw*8$qy3}6Jit6!yl6pqhdG2hs3mkH>8K85Xf-I zm?X5qRH9VERN_>98aBvrk2fuSwZVRK!T;pM{>snbGc~m(hb&T*wT-73zfDR6=egt) zW3JGqkB0>SDA=Bs&AXf>gU594lKf~$JhsQ-Cx%+zsI!s|luVxzainTmTAylup**O5 zSa$5=H->;+Ed6k9vN#$430M866~?@S*wXrA^UP;xI#w7*Xh2M+@zjm)4j>==RfOo+!FIBLB_4PEfMfaDMx(cxRL@)m)!9@{b;$x4u@Q zqGi+6ww0si8zqhp2KT5h^aYJ({2nm!Y&xqEp)@M}9Q_t{ug=ZJahk%OlOc|}(lRe! zIZtE`7`MO4J&%LFsA2shC0P3pYV64VPSNo!UtXAbPA|tDGo06hWCKJ2dL|1Ep`GftLDfz~E@XPvQ zKMiL8B$y4iee2ARUw!Ba;5S>?0r%}M`Nnvl&idj%WnupWpFOd83(sfx_c*ZIWI+JT zcc{c4@Rlj}4e@}G^+kP}#s0}M+iLr^BgC=sFeB8u`rsd6*LbEMa8-Bq4dFXe;t$)g zP~s2y29*28eIU*HVwlZqJlhFawt4Fj`scbe|KDc_EGA0)k>23hKcQ#!o6f#LKgqMc zNKb25AHIXW{`rD^;IMg{$?tiscQfFBO_LkQLgo(>SVo-PX|bd{y<;Hwqu)+Nf57A$D!9UgCaN#}OS#{xEgA$DT!$Knf!;1;IK%gtW%zBltrYKdPK>%CL<0 zm$U5p@^m0Ot~-C2$pZrf-W}V%p!p#>1{8V}3qRxbPomcd$cSK>pBLd3_8~UlrF?Es zA-?6|Yz_kSn};GU@S9TFC1yxA0Ywu4c_h#D7As@@I5_>?wc}Pj9siT`UNaAvocR6H^+E1q@gr{L01c!2rJC+ z#$gFOj^V0HtT6uQ2yBV?e@VbJ^c&EA;iQhPG896^_hh9C#%qT4F-HcOZ7htaU z;ByymFD6;#sC&|HxXdxNu5vD19zIJ4<3`64(@e?5#V_Dv)AL*wb#yx z+`zpZt}eU_Q!&jN1fw0ZuJnV#fZ^4)cl)k0!1wSO^nIKN>?xT8u#fb`8V-u277v$SlJ*ooj)k%#5cmTK)pB0mOo-E(O2xA$Gv|z$2XG4{kJ5?w}Qt##`Mh< z&1PV${vO!2e`qHo2>R79jA9$Kh~S=H3~1vW&qLy$Yr50)Zw@jJU4nA?mRQNgh?I)bI%ttZk<2c4t<}trdOJOPbXGn7y;~vGFrLdU&Go-jkW1WU+$8j2|%_o7`OzBR; zyyH0k9x_vzne?JSI4q;KxIy1(Qi`sIg48n_5)HZfkwULa8p)!f^#BWDIjLv6@322imAVx70|`#zuup4zpjxJd~{{cPct;#)6jXesjR=ohIh4x zJ_2zRbIpTyXIFZXIT%1CzTLLxdxG6XhhOaOlUJ;$v#oC1pjf%#V+@TFh9^ggPMgx6ei@;iOP zy|~B4#>Xes;HK?iBh?xmzj5Q?)Snj9vaAs4vhfyg5NS>lts%dN*nn3sjir_X7#3u5vL*;HJNgSai{YKL8w`Qyc)W3A^_A?3&MwH;_E zvjtORJ<4Gjd-oW+0oJFoJVrv8Ou4eudH+DIph%lkzh-XYgvb)F+EN?*<<$kT_2J*47kWF*Y)#{8T zlEE@HLLaEkt728IA(wYE=g1T;as!e z1TbNoy@V}MeS`r!(}Tu)jy`jjQHpb)KobQxeS}c70;J%AZ>7AxtM1pPCA?Km?7N%X zMzQ*dnMr!BBhxuo5YsS=%%h4lZF&Q=Qb8}J1K>)*s>cJ|Oa+UdLFRgVt4tDXu;AHp3L<)7fzgLZ&e9U%s z-E8KVnu9uZzQKox5-Pvrj4rWsBzH}7))mT%OP$dYqNyxaDBW}jMbd8Rt`fYAvy~=U z%b#^qaxjd%IZiw99wmR8fc+nI{17!Yw)x*VKwaA& zO#=Ok0#T20TjyMG9pYDh@gkN%JA~xzvJOBpO*Sf9jqojibuA{F_(IqEPs?uU4UtDc z`VSAmhh~9_^WR&MYn)Q1t!Zv2Zqx4F&eNq@-;d9wGa$=1i@oRrsXuJAg*0CH`b6AU z&YjM@3sxH2h&+LhIt+RDb(F&33xJDc4;?2xcQL*j(%YYzVwhop*^xMt%&Ta7yqGmu zzC%VC_R(ca&)s&bL&q}wyJ;hzEo#pmwHmf5nLj1LI5ivlyNSl)>QEkNfgZ2(%3<^0#L&m58v&Jp}0aYIyMgtw!EpA)~g` zeE0eBMS={urDo~VC?*(-T|<>iK>+Y_XVI~K=Gq~MJ5Ls+vjCkd+grto=I!h_H}bRU zr*A81FiU;PL1j*$lozgOZSk3DXZE^i z6ruLWqR|;r5Z_rV2PvTvR7p1&?pK`0Py%(DD#TSsflM#bmmEmp8j6s*JFsDr`c+(MtNKjvMPLb+?xe+pf!lMgHFWgg5jwg7KHnp|nfZWm3X z6;=G7t^?0v*gF+LCYkQ@8n^us0LMhT(FMgGd~CmAQ?GB~%8k)kz-T)B?@4}nU!h|H zx>Cg=cXa|BJgT`(*-1t5Q1PDvHA$&KZSsZO6KT%xqw!v?%G+9(zH}}1MkyDWlBp50 za~3Ji=FjzvaE%xPw3HOM;!}3J6gdlO+(phJ3ryqDYsN1hiTlR*d(PIQBg;Nc5YG5O zENbC)V&G;1vacwsa&Q7O6L7p8Nz+RJB2#U8n7ttkFIZP{QhBXf$se`A9Y+1g73%`` z$qFB8ls3~|qODnl%PfrG9pl+0lAHDCc4ewS{TMmU*9xidTPn?0t?&5E^?NWvUacN_YFH^w|(v9w;0rTEXt!;Z+DU(oOW>|?@` zQAuj@57e!&|AP{6DLYd`V~hXq_6onLS};moEkU5EKA$AY@)3w)bvdf>RAx9WGhqaJ z7sR)u7RQs!o2QEeLj9HW#(Tm4pUw)lOK6l<&?qw=*GpbA?dGmCHQ!$+@dBV#`}L84 z0con|O9!eTNryASRgo*m?g)}B=<)O?w~w|dQ!*1mEgu7;>7Yo@ux8d(>n%Zt=X?Sv zX?*IiS6^WEO(=|{NbyGX6YCm;N1N4zQhrxpT`GFNTL``f=yPkod4VCx%&fGU=Ya*) z*%*9qTgP^|sUfIX7|Gcx&a2VSDy=iVlkm+a&pP{7UqZ21_wuY8B9Y_{XF#m#IsO=* z{%CmD{qek?$!2p{shvla7ac-|X5I%@9e3;4awj<6-+wQ4jPeMW8M zfgSz*Euo0D3Ir?3T+Q3_`!&55T&Q1F8Hc?*V`aw8y6HNUAV2f=-r+M(Hnzo%wDZe? z906-!-o{!Vai`_k4KCMg%Mq8(NPAUn)3<&soJq=VWdo@VS8ALiYkJ-CteVw3{zBD~ ztnzOd=~+}`Mmg8jI07dhXL^4oK*7ZB2Cq1Gy`KD9q z1v>JjHtA-F>_eR7%er6gW@qq4xH^T@XAi-NlDcDr3Qi@8M+)^gnaqYlTC;e*l#EuM zj5fE(Ba>>6WSmJXT-}`N*t%!Wr}_(*=#$%&3Y8B4t#u%A81t23Y1zPSHox|d!rM-t zhIQZMoDq$tojw_8zk`fE5z?%C4jKV^OVoNLl09Z4YbAc9W;9)T_%9l$v^Y8y+s5Cs}}IZ$MX;^nQvK**NasMRtQ}{ zZofkOCxTP*=p*ERBUtS>g1P_4Y1#V!K<|_YD$h{_MQt4YM@sf;R0&P*Pf7yczlT9r z#6%ZllNdYRG*l^CN;cDKU>Ja-9 z;EZCVZO?r~Ayu+}#F@u1!XS<6^cWt<&N{fu>@^G*&;%(mw9%^jhFd{+EyjYBY-CDC zRO>Yv!&1e=rmxzK1!v-z(&eF0T}yDz=irYc1R>30IR~EvIIuI4jq}p!gNWU~ru}v2 zA?T(JR#|wiWw{wl1BU?EV%m=vTJc;eze7A{Y$1CYj)%`^EEzDwC|nXz5yN>@zDcR^ zw|u9*7w=NW$=+k7CaqgF;4?X+77)a zQ`+!BNk>W*4I_~lQN38R(L@p2h>nq5pOR~RP{1?R%5*D;R_a-zszx`Gr97pf_vi=^ zB@R`IpF)7IQ{ZoBFLTqmpV-4R@)fO-7pJY-OS5TLVfpQvZhs8nS!8MlNsyELug(A8#YdxU-A~|FhDYq18yRm7RV9aX^87D zT7f#6iP~#Deu}!_aRawoR5N#fs_e%gJ>UQOLuyutu8EZ|iQp7ID%Kgc4mxB5USAKtxt z`8NkC{xInO1Z(U+_wxGQrpjV~J`iK(n#0XgNv zd7!rabC6=aIK5NIa~uK4-J}f@*K3$7zL(N)e3tT%9`kVdv_<|-m^yqC}*Mlg??IZNT zK#WPrXJpd5FtUq8RlfCu=0-Y)eij6kf%!rr(Q3RG7}57e+(G@l4&CA&>+R~p2$ zsg%*DO1B_ypL(U>h5;(1iw!LFg`Gdy5g<>s(Vhu^z%Oz)E)hvD64dFae4WPblI%ZVs zVVrl4bx{b+_R6-i-@9n1}dW8H>V1zCyAI`pi_wL~1zgV#scCoSh z{msnU>aW{ALsnCKs5~(ND~$)lw6Ur`1ds+lp64}XK~ZK!=N2e^Qm1Mvg26BphAZI_B}#ixuO(mkgTAVh52rjV{g^|B)zTm?VU~FM+`Ku>SCxzDpVYdRhA_0 zhC&aKv}al6{C&QW%b+w>XaM#hUCL*z5{BmL-XS$h3&G7Em> z;xU&>S#rS1C$os&Hl#+^;N(DA{0eB`7Jq+5xFx$32+LzSx~fyoL=>%<|};3I`mr)m|2 zmuQW%;R-iTU))#zGWvcT7p-V_zcMw4d@j+gykzF-ma&FTs~;Nmw_p_Q`squ=0$ zD>1{uHaJwgKmN7QM(@$z`1?Us`}{BNod|<$99;kFi66luD^2uZVRJkbD!r_D z#F)=9(lDg88=e#~mF1}*r#d*2Z~MOVkFd`O-_}D!2tr#QvqPP0C3mLeK$8v5R(rn* zP{Y*2!_g7jyDOI*gYUq_8|=I#V(TPYD@oD&)&Zk+mtFH}?VU9GX4Fmgd_m?6 z;*Rs2^yR`_W9oU@1|~$V_kvl3K1ofqB=Y?lDrF_*jjpM#Qx-$*Rxr%xehe0!7K_nb z)TdyIyvWWB_^x(DFxF{W;%w=C)=8ia52z#Fsl1>lRqIeqW@51P027#alt03{RQGWX zM+h71V|IOefJ%j<}T@gA+Wmo8uCpBdmy9Mr0qgGS}60jYy#t_ zk|+#$Lrj>UxUV3SE=6A#k6;#HqXJn!#5Z>AtEh$khRzJfz6^mYj730ljM;c;ck_^f+?+qe`w| zCiwRs4jWoqhBcAID$UtQ5Ek(thG1**`wHoAr`(muMX6s@i>#K4L-uN^SlnE~nTy~D zti&msqq{tH$F&*YWiZ$d{3ZB3H}&d+p*4O@SfMjsei!oU4)heTjCg*pm9MO5dqQdz zY3(qk7j47$jfYXbRrrCi(yf)@0&5_m~A%^ltL*ToXJ zhz_)o-%n{9%)fX}e}4c$Mh-@Hf89-tlm1=UWA?pKbH(FP(>pG{6BmT_j5(2;_}V9f!g{pzz+H&WS-Uo6)wBYdOP5}q?s|i6t!vI;KaOU z%-ee``>o9T;ixbDou!_te`opW18vul4cmJB8uwSu_!CZUD8LXL-r9UjoqIfijW@dZ zaDkYaT$FKv6pDKccJYYdDWvik+{0BQ;mRhki)(|736gyAEE8|X~NqBywndJo=$aa9MJ+wfO)_T_Qqle%`)TI%eYWZ*^M=_#2?SV0% zC)q`^Cm5`(WmM0zfD9P>)KS#vY{_W zpTQ#O8#k#Aw@z7hygVMg$w11G^wv%aCIB>>biR03rgSRz9 zOuoKAetQrT>T_K*4lAXf9$QY_9;zAQE}Ed6(J`f-+%}^ufo24$b1uy<+%-4uAUxQGyp?;VPZ82#?8F`G7a;lLTG8(9?9ff zDx$&1u{a2Qw$xMG)HPQQMv0~F+HC^_O9CDT&90}xjs??sM|QlN`5d4;Nk(sFSmLkQ zRW#xzzwQk?B1M0c2{$%He7gDW0F;+bA-eE+gqb}^Lf$%7Z5H(`<%M(x$_<@+X?bwc zz1Gd&yQAg{UECF*jZF*C*#{q+-k%9>6al;uM-~DNzeQq3KnWrZL=7KOqU#VbQ{9M? z)GViyB*^W#4WV#*quy2)z6{S1^UU;yaXi_BKV4Du!k`#_LYsglVhQ7x*O0n9EYT48gN*T zlM~;E|5^(#De-*jkqz4=29Z;qp0<%uv z7k0`y#9$`S5MnVHp1$NRhw9JK)vbK+Lit^pbnL_WTEEEkPL&cvCp=eK5ZT%$Z;xvTu+=UgKDkR$cwd^bor9%BDu`Vc-5$ zBZRh-<(MUOO9c0ole~X${qM3QX1p^`CB`^psEeO$i1QUc&Vo_4<{C>5QU&$3-h;g)X-@1yggqE^7N%!lCtOek=CwF@FjT57sm&pBPS_@rADtSb_1(e(hSi@Rc>h z8v|OkgQ*ZD&k!(0ycGWUunc0?(jJ~H&F*{?Znq&o*yoVU-dV*A+>55M3+;!VM}q4+ zk4)QojaKO)zC?KeC3YE9=FmjH&k|_r>o01s4&1ryOMK+~V(J%A8&seoD7ADLW&GIm zG5UI zzcXkp)W4W@{f2~twT01NJV8$n)p6u!y)f6fV1pWuUT;gfA=C)$)SLdV}OxM zU|9WmsvENxs}})TOrAa?lE(a}w0Ib)=FAG8ivNnzOa&*2Y$E&n5MW?GX=!|A3I%R2 zR4k&CctMdvL)GC4lzvJ#dHyS>98IK??SFg24UeZM%#1I--n~7B-6j6iu`KmfGMLo7mtyHznpXve>AX1X0 zx;u)ls<0;N>)TOAGPnCmuM6l(uH3xLC<|0SjEXx!ShF`}=75Qk6~cc-fd3fdfQA{q zGZ$=8tkmBfh45Luxwr89_6;#tN-+HjxQ={f(765MU^o`uy-ySIEtY-|Nny~Ho%%G? z^S<@$zSC?vtv-uJtLA<{4};_QmtPjNF*LUw19_6USYnS*Oq(064@%Yz+BR5rR zfvGvxoY2WS+HpPhg&Q%D)ZT)9nMxR1ss-nGRQ=oJZqIyuIYWbq*=+mb`?9zIRmO<- zTG>sS(@f@Jv1m_z1_T9M--TG<;`yQ|z%F2n_vltw2 z%R6Wnr_plu^BO^_t;^++D^F3^#_1iwXt{pvT7{x233xpSAs1+f~<3p(sNPqGZVlL2I(`vBYA@AqUulI`22y`~t z6aF2gbXxnJVXvrL4O5wvx!5Af*J<_ydt7U-*Vj|O?g-uuZ~X{<4>lVrGgA>v3}1J0 z)U>c!Ve#8CU3P%e)V!-ZT6Xd)CF(B6?AAZ504h6*Cnuz{hJ(DIQ3iFzXuJ}K2U zSV1iX=wlW^-?>g9ruquF83okU8{1Unp~0Q9KPVUrKbmgU?6NkQHM(h=g8J!>nu9-e z`SSUF)jADsWLS}L5X4krjBYn0!LmvjW){rJNN#MB$`wnG1yNQNZUVZr5Id6_vzy}a zL}9X_Q@UKiLXv3@KaE^w$<8B42TU3-->NH#i>$tdGioU@rnQr58Ta3Yjn()Q82)5Q zC+o@(EvqlZ(X3^_LVu0KAJZ~68CgVr5o7S9u7oXYKaeViYHCWb5fHZPU}(-t)-P56 z1=Tc6{W+CHwSIa;2t9%CqeX>Uj` zP}un6I;!1{otJvA45O;Es%sZykkvkGOp|o-^^Ab)0sGC)mx?DZrfR0{uO)ffXtE9m z6pq#L%##SYErvD(|puJ(u>FeLpD z`ITdqO487)c1o;3<7k1%E$CPC5Q*obP@~S_+Pr{kV{0|(gb~k!vu=G#?BsK5kf}*XAfTsvAyM$+EB^6$q5YA*+18E` z`VG_LMigtzDj`qHAXj=Xg(LMU)q2#~$@}f$0;Qcr$A6-O!D=Abe8$fVmac}jrq^)_ zv9{E1iZQ40&|0;j+CP*iP!2N{rgL9E(SqFBQkvb-44q~Q$sJx$RhXUBLNu|Pu%SAZ zs!=bjd&Nw7R$BBI%uJL(0V!c|<3Jdig`#0sAk+3zQFFO!O!ZXh*5N4+Duk^Avja** zO(S7ax81(j zV~hzDiK0=$ZHCVgX4b8l3dmbtLm4bk=~BbApv3VzFBdxL4K|i?#fJhh+0d3dLE!l^ z9E<(o@15Hzzuiz$3xUDpwCrkq|U2X<^{UGji62awcmHEjq>4WfP z%gbn2dx#t36I^7HfklCXl|>jNvDd3{rcbbjSZ=|Mlq!caaDhedDvZ_HJGaRM zt0*|F6iG_c99m_*#=NB295GNx=27O{y&|(nTqyopYh}JvI1W|7%vF`4UzSA#Ae=|e zidH8Jedo@gUg1Jy7h3=R3{PTq;;@x@Cl>XKW%NrJKz$N2;^i5Omp5KUh=SM2uiq>X zffliXOo$ik5h%>L!v8ht6`baUch_UMrKh=8SlO@cwEE&Rl=hhqJy2Ofk-~0>jnt#oFHBS=_I1-gI(I? zet_#(fBq5q`7$xLWE(~rTR@33&AKC30F)<|&g7!x*NFbVI40(J<4MnNj;ZwfC-MI& zu%yiF9sbl^W#r*>=mr0W{2tEGXO%U6J@TiQ97Jn)#*+}0^TpFN+jOi<7# zlS%rxCh6P*2&=aU1;@4F*f6iXT|Q0%8!ggi z>;Fts9lkN+7EmDVI_5+H6S+d#<*+|v=Kty|r|0Br&DesYVIq;W5Gp~hoI#>l z?Oei+cc9rK>IxfBVCa^#6CJF)jF~Ljbz*7mvT;uuJcE+G4^Qrdp*yPgamJ!^*jfHg ziP?ILW6o zOAVDU1B1)y^C_;`$=f2kQv6dTie@UIN@hH_2Fh3ONsi?mtFGl5Q!+*Sq`Vq~MuG4d z!{IfI?bwky1qm<~ej1KMBAy+=gS}?8<h2+N~SD6gQL-_ zyZkGb)RAt5w$6!GuGlRwT%xi}cePWt$#!!Y4wAc>3aX&wQQzja2J)M*hv?h@ zTv1i+&@$r(Ngu)hZXQ^c$9M9F?=`+5@rYbQI1;PnnrO-7xAjOL9<`9xL-=Zd?R(`01j;27CGHrc@bcRs9zMM@uc(vaA6N~z4_syT zz?INYj7ENVxTlcjy^WIaOF9i5XG!Pm?bN#;Ghp>ZZeLbrk&r-{5TlLAxTutsBSMXZ zf|!6*B2ufZZR3paYFtlK6H$@q9|I;JFmcZl74$Kn&){>Q9n(KMVP1x z`eKBf6hS-wr1$i3Dz#lR*eyLQkhWE4MKjukXac$N+r@;lCNBQ88?PBmxaiSfHho4mX+bf6l+u}xyAT#$q$I)KgA((zGtFJtN~ zUcDK_KgaCrWZ+Y)H|Q*yoVre(>_XWS%}7Lq&vh(OOCJdFs0BFT%(duMjp=+G3crbQ z(EofX*YH~Rir(r%o#%6gXhAE@hNMpB%8BzfzrkpA4(Tk5w(Z5P!^ROJzkvazagvHf zm4NFuzgaWN)f{o&B3nQ3V8!1>Yq|IJz10?ca{WHtDPyt?0axw`{@Lj;?6+Y&&j6Qn z`?_*=+LdL06$v9HDvMIAc=}&~ZLfr7BRv)_a=~1~x42C+X@t#-!_L#q1PxtI9Z76k zTZE$B``KDmLsxcGn}tYedJlhD^a%uF^GyE^nw5|Lq9l^DGXwpfb}7<->{2{&$eofo zt;{k}Ddb>^F5llzYGw!oD-9_45e3_=H5NOBRE5tb7re-}T%!hkb)NUlX>~SIq=6a=FFpl ztlf^A4WR!*hu);G5w}a!6=J8%J<_`MxnJQ}C>k)nx z1!Ac|Lmox3&(qDS&ZUIVJ>cy_@&l#5O*M@@%{uin`ukADk5LK&cgSf*KD)@vvz^qI zWIdDL9-e<-!)JPApy>pyyt};5!&hd>~iC#W^MyiN37t zdF(h~tXy-jHG5^11BYu4R(vk+!?vB5t@hwT_^gDvnu5g%hl=%SWU6<64FpH~RkVlD zez0ℑJr-@+Ug;9|_c!7zO?~81(zy6i-Zh#}n-|ArteMw-t{V{~0m_m(Zh0zsbJ- z=YMk%*ul=s%H;pgG^K9Hr!;F;7WL0WHG{F5D6b+5}jTx`CFtL*M$J4Rl*WLg2biA*fPM)Y9$R*G5 z`9w#`m`fC72EGx$rK|NY<47$!y5HsWQbecv%Ef*;xpv?F&ER!24Ag{Vn3qL|kN|~nbk_~h7L4b>WB_(*MACYV7 z1aB^9$!LC>)GcP9y-t|lAt)|DohV*E6 zOy(($;Cp_P0 zFDA#U(JSNmgoo7scIU4rrZm7b{FO%3@~){#0NZ0OPb_aEY1zu&4C0C+@%)Z9BGV2D z;-E9}6{)KjRvV6!o=tku}!O*DfS*+ZO|6rLi*j3|Dy22Zo#BsA2)@Q@bVX`tLuII-c( zZW?)WL;d+iRkWk|fj&wkICr&H``-8C@yk-y;W z%L4-p3$d9dUa^h(`l)(k#n2@46JOW`#-VT)El8@RbE?@0C!ZO(1Ynv)U8K|iuBfK8 z?Z+aC4U^7>ynNK2g0=Tpss`+oQ7!!ODkwlW(*EgHqU;&3uCcIldx|WfeaURaPf_IC zv=~}E&j>S+ z-O07!;|^W^l2Ms{rVKsx!j5?{n!g733@NHjd#4{9Eaa8I$_k9D>LVp;*L34}+wDh2 zR2Hfea*vp%_Xu+gu3_+io*#uIk53CCs(W{3+EPb;7s4^LcPiHT+ZcB1EQhHw(LfVn z`?osMRZ0GM>{J~070_-FTTj{Kpqxg~;~)m?n;9Psw51VHrMqhdH_qso|ZoVibx3DK$H2>4CF#)~++<1VGau zAC9`hRo*=Lfo)!ZlKbu)#SAO(N@(P#GBrjWW5d&9NYs&v+lyA#78wI?e%dmTZ13kZ z)Clr4RA<4LG` z2FlnhP`y+5QB#kBeoF$|gsmr~_$^?;B(bQi%ksn;m!Rv2bK)#$(u}Y|o+)WmGd7i1 z^n4F|Z(kp5;pgR!0?n_~DpzyzciS5x(q#!N> z_+H6SnaLDrm^9;?AUcuPYIMb$GbwQiH}7QngTP@FKfMVK4133QHLJL6D~1^E35I@& z{0y$Xue?;iA;a(iy3@qnCytEzzYdQX3|9AvcW*(@P4*g98vhZAd3gjReZ*c*vB)fV zT-3p;&CwuP%SlSZvMNABXL3;l(oiX3Y*M9Dk@7m6PHQ_02t*M!^r}Z6h=t}jwmQe$ z+*0e>n14}{*ar?zDUEAsu+^O7a$QILBND?Z1qBNz9Y8GWyy*oUvf@$CXV>s^c|eAr zb>`off8W5r{i6}P_%3vOK!}f&w8023@y7(N>2P!+A}+^3(Th3Z>K~2RG)*DDcU?)r(-+uYOu?$^r5}8nIt(e$GoHbcbGCs`K=6)NJr+lR@&g>Z2Sv|JwwvefO^} z%L6J^!XKY%t$!mC69HP<|Lp>|^gCP-d6IJ+x9Y9LGx+M!VYU^_O<_MMP{|49NE`^0 z4E}Kpw8G=k;F5m=f8uT@ILFw@g~CT{zLXEF*b)h&D3B*MGI8Q|-#z|W;r;S@i`Pzs zPaViO+*oN=5BlTJZogz||Hq#lYxx3XW+}7;N5?~`Ou~?*r{wFy2+r?HNiDQlLNS`V z=t&!6m__wkSwVxB?=LOcE`G0arsX^liPu-B(RrSWD5BjTzhmP*vVu&CMx_A50=G4e zB_Sw3AE^>D^ztoef^?|j*$C`Hw+2PWR=9yX**)s8^n`X$i>b3+WKtkM$cCC?qK5f7 z8|WH;Hi*iUIBA2FfH`f22uo_HMi4z|yCeB<2!CeRwJS5FwziCvrz=4iLJd=9UJpJ3 z_>TE9NkW5&Ykr=Jx5M6K994 zip&9ije!#;4Y-g$>UK#~Uga_3M?>Ins2~H`)?AGeY-<) z=mfu_L%-lfKRzLv6(!&iLab^p6VrfkFRy9ZErk;oHf0UJ@zcNZ<(;Ep=(AHqw@AgO zk#%C5M;3bXX}q}rc$fOdvc3~#j@?=#37y8U%}o)(5tBTBe?<_q#)mbW+9ss{A&FCU zo+m}LNzbZfoel9hBjUxs9*V=5R3kY z@x@W@a;U&RvW`Cn?Yaz)9s8=@u@}|Uupy>uYL)JzJUn+CW1gdC|10j9bST9bhTSWz zT%@P3@K({myWf;c(54yO;3jJPDO^7&7$ae0>PPY%eLP3uunE3Kz?KDW3p> zj!22C6*fTvUIy86a21?;i$h?`h94`4B47L&qzG|phf-^^3`24c z^JQ6h92)15MBx#2^3lPMWpyAtO28Uk7CMA5;sXyXBMzf+*)__7?6@RvB&}CZ>Zef& z!?T#&{K5h-w(((A|Gs^~xxMx?ZZHMXqsJXOa2v}N7LS&Rfv9ur;Em`Ky+Y`#pib~) zd{iD0fIjN=FOpy-8N@30n;^79{Tt)YzqgcAl_eE_hyAOSCXNOSk+M?vT`>YnFntc1 zt(*zd$3Vn+-N95lLXuRHlYLp+jZS}969*BD&Hc{4S%MuGD`X4*jOk&LyX{?%wTDsS zfjffqZy)Q!-La7Y+2NXVso-*+j+Mov)%D3*zrBOyTUyoh2&=t@BWSh|)LS#Ly^PEN zU^C7!$Ca6&6jv>M+C6~BaU*#YTrSSv!4S+GOyyToc4RemyOTHWsS34%dYV_=z`9ow z9IOyIl8M0q9;SznAk_U5?zpq2Gq^s}O}%rrFDEe3^vi zV~m7)a|vEzu)Nl+P2`3K8=882+BIGshAUvl22+$GH-1quPCpSg`%Tu9TAr51YVJmq z`Y3Bx)r~RK09(Y4Jsbs)y8?@U{?kzS1J>gQ^(l)<4M*jnUS~?FcdCxiZfL|lgsyKL zg26iT_I+3iQ`Jnz=~1S*nvTB^!PPEuZ~1k zDT9aif}Qohlz5pab%1@LQD<2tOUu@cs7;MsnT>NST zBne(M?G@(sN;K_QiZ!+F+ovPo*JTq&QGHD;JgSt1t% zjJckTAhkl)bKCQA$3sP!ib#l&2K#YCqAsY`v0yxj@Qt)Pjod~v`k*y}k*;@t`CPN} zOWOLZz6c_|q!W?(CPx3rxa^0#Zm~cyLQ`{(sIEneOodrlG{i2G`tVnPh z17NPp#e{Dag4;_Qit=>gBvix9uRe;ow za}+e>yLTf0<0+W60npO;zp?uNCskqMEPV%MjjEzL{!UehL4x`zbaYsOC8xe#jj=-% z4kva?wC9S?821p_1fgCxqFAz$>O#dp;J~Tt=>f->E6Dr#`4X-DLwpt#6kG1bPMl zb4YP#7E#EIqJPc6I3`@qx720ymmTbbGT9ZNgwZNXlb6-1YK1t$*+sI&NFs%TGr8r+ zW_Y79b!Mek)PYcYu+nwC7oW%?^Nv}H?d9TBlFdiv#*nhLOhuxM5()gW3zr%V2ILJH zezps1=_l?tG_G+CjN>v?edmaUz0qFX3ho$*zjU)#HI;Ei<JNL$#v9}co2N+c5q$_4Y)-Z*a z(!MNazi!DA8r|>WkW57Du+q}oLaO<2=!*fFi?BXPRb%JZTAG5z1FX^xDX*`opbs6T zp|>)-%*;?2UE2q>cn&feY;5nw?8KFasJr7W*|nB5ckt3lX2&(joExLh4=W$H&!$yB z7p_DoX2cM#mNzP(D6hc9a;zMyv%w2y87B#-0IZgLX1}1^|CC6@2juM&BLzI+eCUE( z&an@+vWG(XAQHUQaE{a&;3rrwyjNa4Hfxh?vpN5Hs$MX9!z%JkKl05y)$HIZi{VLVKO28 z3U68P-Jr)NyjKNbU4)pVDFlo|0_73GeSt53IUc&{dV0V7M#jm%Q8dX}+dCLqJN&)f z`%g2rE558c*Iz}Rnu@}lH&U5TSYv$NFGrj|7JQeBH`H0VbZmw5O!@kbKx#SzA?tr> zFU~9zG#1U({_M7#4yU-Eidz2$#v@!icf28hK`L2An(=74yun<#HmYv1D_es8+Lx`$Q_WM9Nr)riFVWzR$ei10wtn5frL zrber*-U=ZiGa&Ji@5Zu98)Xw@_=S~JC50xQ8gI?mLb*lLl|nF>oK(t@hbHpD04b#p z0xYID-@MGkZGm*(m$f~Khvw2)(?u;q7fM1mfElsQRK{TV*t@5;3|lFcBpL&pCsm!F zy)Np31N-yl^m)=%RvD|f@*)E=C7Q6X(sCwWJC6?x1pui1Sb`-vSRSPnbv>n(jkz(T zI!pr#dFE#NXILLcYmJ#{UR2$%I_bUG@B}-oaa(956FV9u7IHV!8y^aotiG~j*KQQR z`}YM;OROSPnkqef^Np2=VWeoDfquSNGoRCbd}WPI?h9i)feZm28w%^pMiet0eI!q( zO|Ty#atdy!Y$}@NiOhs6H7=0pBJyO_)j7^>JtlV=lTA`FUxKRwD&Gf_;gZUz*o)j} zR%5{VYMwT4O`~M<-<{5&)iIG z2QxFA(_0a=K<^P&ETWT_gdauGs_jX-NUks+|5}>0!Vg5yBq-ttrY4l(B z<}EnnEva=sf5^#+{ZPs)FimC#JD-Tk_P*{jQxG|~b+-&rIq8^rs<;=-n@{Vw83Crt zNyOoG84b4+4W`iX#cJ`)s{Pn1kMql!_ggbWc{s0B4raWktKO6L2855~-uS8AoJGi& z@`n7y$<=WDx(0yXFi{vZN({P`6yvfuvisIT<`TN&n&PG!qZA(0%%v_&1O+N~n7*RH z%~tpJ2=W(F{K8|1SI zm1vl@u8S^P*`KwS)wv=ned{fY;6;@_-o>N}ny(?rZ|sNa5c+1O&Bmqe$TYv;s>M*D zw?RZlS~?|@QICwbQOM6oR$R7MgzYO&ae7cmt8ovbx_^(`=3Ytq0x#Gy&aIgwVLR46 z2e1&=yIpQqZXE_fuc_68t+6Ua-RNSmx~BX}z%%NU@Z#$Sudo{h67Xy`Q{B$!t)f3N zuWcK^-)S$2f&o*66`V#cbT#YN7~e9TX!GR`7)db6(s;~S*eJa>s<^;}+T=qFOAb62 zIdcmbl|6TkQJ2zd8z($;+Ap}$c{7@Jv39D2${* zyNw*L0&VsEp2$&4))j*q+!!(RezJDtXz9dT?gQh(gm-s=C57?*HKpm5dy%?i=Lb}z^kTzVLH{za~ zE$`q>`&B{g~KS>4Vr6 zYs98hOgS9C?&K*6vRh#hD$cc=VzhLeY0}zplAQMq)Z^o4frelN`+u1~N*YNv8gl4P z>q4}E60LIb6Nh1nlc=RCY%CvbHm{QwT|+A}fXy`TT%D1&Dp5<$%Ftu5S#g%$n2CFg z5l`tv>_Vhev%twBFH9`7x`J1Qv-&z+*`Zn5Y1`O1>}fj5I(+^mvW&E==ZHM21hw|ljp$*x02C1wMdh=*kYHWr(>TO)< z%1S+i7yz;9raOB|y!QI;OhSXGSPIH3sMIGba6k)>u8f}!Ge!0WsY{K`w`9bl@vHi3 z&u?1>P|B*LNDW$U>^PB!XI+GaRI+Aj#H~Nz4rIUq{LoL@(xDS%`ClUyWshaeGp(FT z@K!9RaZktd)cgq#cuLXi6^|MHnAMX)FWt0Rf}Sthl04>#Uq@6py>VI4{6c`+db$VP zEvuuS*C}^dOXZ^7^Lgf+buRNnk$)Y)#c+X=FB6187X{Y}kuC0yl`Kqv*^*jqjG;yv zhh04a6GJ@C+h8-8B~L7vOS8>(F762ve2v95S2r8tYl)b2cFeL3&M>xpb9CU~cQq>) znrEzyh}Zy|nBIbv2-;2boq?HZ3s))E`zHKvQIdVo>GPt`3_S`2 z!daq>C*+yC8>aYodT|z*^;y05D5p*Hq%78q~%v(5%Ry`DqJ_p%Gc-be?vEo z^E*|@nZz7>Yu|SV?A$RXJq2G8dJHQzi$$G^JPNh)bKcGx;T;gU6ZKuZ3+dL`SB_;C z9>i~)QO71#T_t|}=2S(lW|!b!_(|}m6gB86wimU>B=GGoVex^?-52cNc!h!b7iT2} z?0^PFKm&u{zxe8G2ElN-)Li$%<*3L zeC)j+2AO*(;Uv})PKyWUhWjNiw$Sjey@+-es=L_l>s z}zp$p2Dhxl07CsDA71>Y)5raf~SW_hLDFs zk%JlnvS3m56g@vReSAP!cUUZ;8rir*=dr|9ep45jN=!}LU)#Ld+|}Ehf|@p>pKuZ8 zua^DO5(r7#LE{WHq#rMbP!Jjm5aH|`=t;#j7a0X&9Q9|mob}y1&Oew;(SgzLaH_xC zmkh(@_AHq3N#?#?m-t)HzsnJ9FcoxhLxF-F7^5$2r#go0g6$B0DpUQk8f^RKKDNA} z^&4I`QOEH(@r73O-KdPsTj6=_TiXc-XS>p>dS`0=pffo%OM6QNdk}}pePpsws62&W ze=6@OCAX6Sm_r1(chU=%7oBt_xTkdTMicFbv{NKY zKx&sw6*A7hL^Zx()Oif7dsY7)y=5GDie2x*zCe6UTVD*TIm|s$Jp!|qr_Np=bRhnW(z856oiD;Btrs23E0)-Odo6N}_^VFHL zpp%oK=l^s1SZUk}?N64;8h36oC%GbmO29Y-#e~?zzAX9~1=|9P1S`Y0NLfb8yBv6& zNq9x>LJ0i~XIGEounqO7xKnj*HWR=g)Sg#ZRUFmHxT52evd)i0DUexO?r2FPw>)%GP=#L|18 z9*NV#>YvtqNi4bUVra;-f_fHVsJt&ksTt}wHr%+}O=hy@&Ax1$j&X6!s9aXgV07_T zSM-Ua!E+==Yhrx31&k}n-g1{gil=HHM+^db1vJ7U=OZ}AeTz-jIy2d{eU)Trj@V*v zmq_xdvoTNnK|i|oQ01}*q>x$nG|5KFCjG;A_=LQm&np7k!X!>`L>bF(!TmZKSVixG zV$mx>E4g^vP#)XjIwc+@0o#ylPS^xlt&&9NE^sX{lMYbYMW@Ou>$ z=A!`$Z~BANc*EGi5jG%{#w+~4o+BkgJwCoaB7XZv#D)L&17rW?T&mjgKRWWe$*r<7 zmf(#F&GqJ#tS_nYpFb5!91)?Y@?8F8!7H-Mc4ToQdPI8wZPT@8|afyG53BID>`pb)F$RHNJYQyxX+7mDF~-oh&V7y9tZzHnAJ2#+C>mdCvn+4Ett@8fDC9ltL@@sK4wUzfmM0l`Vs-Xr(e-h1M;vpP3EV$I4p?3v$BthZ zfh%JA3iy)ROyjNm1mD&n9r3wUKQ9+mScHKj`GtIX_-Mi+0-+OSfI}_c01!r%`j=G) zP=U4E5F}T=^Q*0EDrK*xf0EID+zC7NUeiFz#F;+=5Y5LZr}Bd?O}Zjw9^FdbYZ`nS z$-DhI8nSgz^_$uBjdV{aFn2zw>^#!I&`{B^rWYm=X| zUtHkgHK!s}u@%eg-R>}J;i6cNAh0C9V*Xl`-+ipDyJ13`a8zX8O5(f;{b@XO@ z7KZ@5X%$$EYHSb1@)TgoiX7zI`N=-80aEHGZ6A8mUcsSDvUed6#=)*A~2!t7HSb*F#l{J3Ny@Z<_`UfA%>4 zO)ldfz6Dc1qeobM6=2l;q9L*EO;!zTT%>@DdBwd|&z0^FSm-Gk+`NN9;SFk(n4w)O zPKmQgEE-84_Z>dfmf7z%WP8CLX6>9#;1x|hkG->46O&qn(J*fxk=XruzC~BJ8AbHW z{V+P7B=0Jnhcx0LuxYeKyKQ)>K|^su{A|2E?kB@0*&Gt_x9JD2@4V$-{@~dF2?j>< zKjNuwXk+~cOUD1;u<4(n6cEEJ4_eKR%4#;K2o3WKI%R&qxA}4rz|gQE_DYIrbH+N( zam^eQVS>o!8y?>aYP5OrU65)oRMx~fyYtl3lxqn=Pp3Dc0W6}Cg&;+c!{;gsW2}3X z(Q7K9=OF418`V}~-R&2Qw7`l=^R;yhD&F21jAT51UO9}x-SpqP;Y1jMJb`W`IDW^l z^d&T^DS-{dl@RAqq(wZcF0TR-c_Y)6#MuV1gvr>Naa&)6Kj7XIwdvAGtv4P*VZX}b z6hOCQJ^M*4e%>3I6PkBq4~;B9t)q}bE;x~3Oqzq@g}Kd-i}eWI*)+i#NAqu*j$^@N zrMSu5X)NaGXNaeK)nttP1oWWWv$cI9N|zLM%ppvbern2wpU$4;p`2uXl>QROt_RKi z=U-9B$bc*@?+ef|gAm8V6%TkW(?ANb9*DdiN+Udd@x6*U+9Y;Sr3zQ?mJLGKJfZx# zw7r4wmvZ;^oMips@|y0!*zC)Yp++2h|J5T1)f4E}Lk8Y$%{8G=rOhTgg=*+cdeoVp zo2g8NmN|Z@j>D|f(Oha_P#;}!fq(@W7ae#k*j1cBq;6e2+Kw+9sK|TEpacg;(lV}7 z<<)(q7cWW-efC(?UmWu|3LGf-h=&xu*pB^0wb{<(q9W9=eEJo6VBve>Z$Z@r*|QNj ztbz|v3~NofP2K6?*6K;WlW3CCdRpO`60+%09`x9T*_Uy^(XNd4XX;$Gfg`voH8ni% zjv)reU%Az6%@sONF^EKp@h;A^?ceFh2`Jn33tZ;wtLc3EQLOUjV-*>GLr>1?{TOH{ zuQhJf1*5UdiDW89${Wm&WiLC$-){lt|+kbCpQCr6g# zeaP1m_WoP*Gp}Kc#cK9yf9P+~&n4PNbz!%BOavorw@C-nLpM^{HysC-GIIn_%!fGD zS*^v6!D=J9S&e;=li%bOtyrS82ROw!W^2C>*b2GJRNt7i8O#u!&v8eSuJ!vSkutyD zzQFz~P2B^phE1Qrz@Gjc|GvL$%Kqyfl^(_FSiG#_0d-`(od7zJNE>4(ImF~!g6XDo z8}?we?n(!QU*yN<9)sK9-kNucT}$;;S&XEy#Pu5|`5(?x&T{89o16Y%6x%T&(8BX> zTIK88spG8T{7T%Jh8b(Fs*@FdncalOkblyb_4Ds`oNifpsWtlavYM7s zt)sO#kj0j*z}jrI`t%e>Tz4!K-syw7@mk#a(!7l1l)O(q931=Y6eaQfyf6=@JU@kj zeM^qeVNlGL@fc_$40~y++)NX~VcLx}PgkH@{s&;2HNmx`C+t32Pi$0i;CSuT$J!j((N2irO*^-6N7)hA z#Z02zd(=$N0b`CvnLB6IAcF>=p@1dZjjl`!CQzA@4QG!kPxcH>_M@@D^>h3{_iR1O ztgFOk84Fu=svbPZJ6=35$<1=}oQrFdsZ(TGu{*7c2XNhN>NxBl+)8a2vjoG;3e2FU z9*)QHd^f9>!EnBvAlZJ)H}D7Vq#Ynf79s zKFpjrK*jWF07Fz_y~H$?E}b1PBaCQ+d)Ch9P^0A zJw80_QommM!;iT|wc6t_&$Z-BSQshQKI=ru#1o0vzg@b|TvAIN_ydGE6d0Jq{}u>2 zdpl!$JCi?IguR{Nzgk;XCENb_^dJp?tg6N8#Sv9~TM$QULL8w)6J2E)f`yS3B$C}- z-CauVKhbS)x7{juq`ia})5nUAz$wrl<#6Z4!H<;eC0ozP&VHFZoiOo#f4apMz?UZt z!NRanR6iqY`C1L*t88lpG<6zQWF2P>ZO5@uXWe#SY%|hsx3#@IEB;nssBOCH^wdmE z90Z2~i6((%;^Bxo8%5Z=wd2uZCzfb>Dpvt6|1Z17B6fvw&@ct%D3d&esh;oNIci?;$TzzVwH? z`e%EMI|X9|ij? zuI(L@m#>{SqRy2^T=sJU{2%IhaSvtgy+!wem?h7KjL~?sOL8~47R~qo92<-~tn}Uv@3uHOS5R9Y5R@m8LW7l5dI5 zR=8F6B*2}-e-LjjB^H;=F^wbzI|8guVuLWz=^i|Pc?}=9=?>%AC;Sc!f5lUL3X{h< z@qi+c?+AMtFp?k^Hpg#3W>9XUm*^JeX^AtrCl(W4noQ^vN|Yn^11#Sp`x6p=@D0ej zK2e$^5I?ItVC3w5vIJ#2k>ozZ@E3j%>u2>=UhL14#SMommGb1oCreHd|NUZ3_V)ep z@Si{|2KL_|<)0o)|Mgv8UFFrkI#0Oc1UtlJGmt_;Nh}GrC`}>FmZymUNsU(UCO`Xs ztES^M2&VGXpGv;SKNCF1|J8irI%@YOX}0;&^qIN_}+0K<^ORr`~!@w zwt{e|cK-<;WbN28Y1`vx)NHfO)w#}Q%kJQFQ`GKj`C@x+9{)2LScm{m2|SZDx;R~; z-HsGkihYcDL7#?y%nBf!-x8eU-lKD5q@mVN$>j|=2Ohbv39z(bKYq_*J!cQK=I+>F zq_-SlNG3N$8j+!DQ7Je{9|H8#)Gb|b=b(o=aX1PxQ+G_^?OK0I~M;BafPq2w@AHS-!C zJ%n&Q&9t%dg0GI2+PgwOJTNTj*PBA4=+={FrGe@&d8;T>yVF@dF_zw)hfjLJlsqNP z%f_s=9-@@yT5k!?`Ee^$a2~^o5usCzStLupi$s$X(RXA0%-h*;qtNYowLETO35DK@ z)ferQ1CgIttNVZm*qGll~5e0sL<6rxzJS(=+gkHstWg*J^UMXfFm*e z-trxnJ@O^!!|AZLw;;RRb^5G_4F;osM^yfOiR)qS`3uLMQ5A-oN9OH<{E-Oyh+NzK zBh&h%}gv_8umct)6%Z5y3yh>x%HaYZPye%&(R>Z~=LV2$Zh z+9zRrh<1#lbLRm|NQ+RFSHM*ZON?PmwPc&jX2J{0qPO-X7+DP9hoXi|=&*_hoiFgz z;Za58(W~{q5V|2xgNG1<5vSoxgAZ`oT>??KNVs%xgAIbzTz^a2YwA)}f37_-nQDTx zfJGKDUh0mOH*$t0jko{R;V~s4vm3z>pBl%sD8j-djo5+)04;R~QS=-nr5MDP|3x=9GVZ%F0}PYt$b*9>4E#j;VHkV6J3u^~)n4p6Zv zlXrqa=OMm5c#C%0gP81wZ)o_=m2+`u9@=nm1s=nhNot{uZuwkdC-(fetZfJ+Yu@}v zEDZjgg-d_=L>oF8y8K^$xN6#eZDx39lOiTpqvMFjV2>}R9ZYM?X9-oxD(vn0R+}Uw zv%!8J+Ym$KX|k;B2m28icC7@7pME_SXT;I5fi>uemE|S2vbucXb8zae_I`4)`2kiN z!HcU5z%~5k)ZYxv6(k-*+%jpp6(u*kCwMA;?#%2nRS+9)Bg33bzU+8YXYqIyXR8Pk ztkD^LzyQ7~Hr7kCkJU^*w1XUHiiO1i3{4I1xp+2WP@GcLzr=U<#cy1ToaW9>Osk>d z_np`%#&r$W$boJ_%v#c|-)2ZABb(n95XX@+M*DF|Y)TLx0@yc5IbZ8~jNJ~n$u!xMc%GEN8w-Y;D{QHt zIRaq`2E{g8Muu-I((!xdMm!TY9>H=FxvH6dbL$`vBPzcU{$a*6U)%czjxWi|mIS&= zAh|p1boIhMHS19sQ9l2kEP47;kAB&To3MB>g`G@Jerd0o+AKxoy74ih`#B^~(u)Tu zozkUcE56QJU>U3E7OBX{?w6s|+r$YIL1lmHMT|U_i-|v;+eC1aetn!yJqnk(LmHei zuFS&RIg50rVS32F1fce5R5{xyduRV^O$#SApHe0ARtf4>a@1Y>)G5#4Do6EI^QW?u z6U1rIdM1fpaCQlM|C--Y49-rO%GS8Ksk8V~o{G2NHzMCrII7&iJfT7&tS3(H3eHLp z{EyJ*B7vY@$S!-(As@~0rz4@EdcEB_T#XtFj!^E~97XA7h_q+9O~8*Wx~62gHDYQA z$26@bGfbwV#dLxVJM!jLvzOxLwi-h|W|`)QQI#K(&*f%`_&_P^(vJoRMm^BM4S%xCM$iH4+6|jd ztW?p4>s#@!#08wg8GCqi>ZAdb0mn_<`8Bw) zrbBeJKa$Abbjs5u^Vx^kJnp8@!bGC_1zisl^19n72cr+#z*$RouJBoPnQSmeEcauL5o+yk>{VKaZS&sUvfzi@95;vn!o>2)=I zGDz?%rX-3swuO>Ua6!TxxLNRaX6TVfGcoN#?H}|jb_69j| z^9cVHIS=6%5uzjlVK9u)51xrq_2Q9+{L@NIcVe%H!;IKY>}N#K&)ap7Gr?KU-$ z>i74{KLI$tA#)HRkac(+^qsARTgN@dx2h_ud5{}VwuN_mo-KG-cKyu3y?B2KXnb+q z-z+T$s*{dAyAq}cG60FO6p;m!4K2UT=3AbeE9MS3n8tZ)>qbGvBJ{hoUrX`Nuu8W) zG$t0gb3MsSgYIOcE7I}k>50P2!i7z-&eRPkbkHcwsfb4O1fb$rTCBazqO7~pr)Hj@ z#jy7wH50=3$|D&`u=CNR`#mRZ=OohTv*w*S7jf2XXVo-V&C@_$SltSYu`s}dAj_-` zCSi(fOW3OuD!Xr@J3tVhmU}ivf0WB!g9+0QOR-Dz{(iwx_J}D9}fwH zCQWN39ZhPa9rf)}&E2|%Z#wjghlZ`~a}>Pf=0$X;srD_Qcy!%5$izbYwl=euJJPcu zfQ$I;u<7PnhFN`k@w22%Z7hM3cx#r|0rKM< zyUyz4Gl4=a3mRR3yJ!3z$lC~xgp4fOtY!liZ+FbI z$lW|B5>g7VIrg~9^xl>HAvd&{a0#1qy*d6!iOo;5hiUq%i(Mn}G|c3uI7D9fni%a* zvpZbqqcN%MkRZR$Myc)h3`syGK^3afRBgSC90$cWC)8O2glxSckEsp#dMg=JB0 z>tT3;6LYp_%y`3N54$82KSOJ}WjmclMkP*xAV~=`zf~@IrmR$qzOCcb4ZY+k(aU&l zyf<+e?#%s&^Q#G>YLfh@m&%0AN$)nR9A^p>c6VM=urQLH|&4t=4N0UrbYf$Uga3?6@3!&pixkiGPitV1ZYj!Im=KYbLHDOCt}KAseFt zxdI*7)YV&ZW*MSL`U{Vm<=6^WO+3=7^a#Z$)#G-!`o}n;+y8RRK$Re@+J>}al5zc| zrLPTKXKZ+`pekAmMe$kUM=92q`WC#ZxHjv6`G9pz47VeVtWqSV7jMDeHI`>eAhFF* zX)3qThcv0IjUwA|XM zW2&fVzt?DhzmW%cS3xN|Y5F`2u&?w>GT#rN4ZY|baKUa3+ zvleY!FD0f`(;U}q@x70o-`Y+U7|Ln`4#LA~;%k-D*yO`P+?H@R3b;A75eYVlI7 zO75@PC21;z;N~NaU$n=9D0*lWg;N})%E!?*op$|iZxjfNm^ZhN19^Ui3~R&SI0u%^ zE7{-?K8c~zNPJM-MV8Qrmfsb$+kxW0t-Fjhej3*o(k6Z~3YH5S8M@I&9><(wcrz84 zfkE`eLH31Y^lg@wdXR%6+4~Q}knQd~VA9olzmkYy3bHPppL-6y#x;)cl~>TNP#9rV2dL{E z>#f7z{#FSdoT%43{R2hNzvB`qXky~*YV?0ui2tJk3|ADop_ofS!538OD1R8XLRL7RCr82>`A(QfjtcOF0AZXoNokIt&}%m)(r`ewG`F(6 zdA4F*xyhzcJ#jG>eb&;_D;_C&Zv#jcwLYbK^>drd)VOX_umco}Hm@dCdwbG579_** zzMHz`<6-pAJZ<5tPJiZ^ha^vB^9agj0EdzZj8s*l)+=$GSLDuI z?z!OA*3cd%**UiYm5?6|xoQ?caMU+UIkef9*ng0{=HE z#lnA5JJbJqA5xXlsNEl;e9vrA%PJ(f?JVI?AR z0MSCVhEys%Un(p$Z>5O1-Ps6+A5W4p33oZM9z0uOTs_cgrx`NX8@KPgcb{UIx5X%^ zjLxU9Q;RYWoy&m3soLe?p@4Sv4LYzOq;aN<%J2d1XQ6mFEHyeB&65pT6h}EdegoNtpRE{98>c+{(%Hqcn!WQ(Wtp zn{(}Mb&~IP2>lq%%-D!6j$tC-T`?b^Cmjg5Lr>w_*R?%uZ&qdM_l4eHWTvKnz;UY2 zDha~FS~al_?(D%|hf)7Nj_wSCf_RFbAq}%@D!H_G1BuUnBvaZPlxniG@{mL+-YO0) z7X@aXQ#1Sbfxr<3iCpOfieF#XGhu6*1^V9*uvD<+F8)V(81;ZoBGK9 zXIB~l7dP5YM&-+=PuW~-xqvHIfzMH7%>DF8-H(GE=VsI{wjehwztjk$9Vq_*{xOg9 zbrK>ZEa_&x4qv?~Uf!-AzmG>41NfRS#sCczNAO??q}0w_ORsJhVam@Hrq6wDc-X-G z>h*g(O-soWhBYf_{zSoQ5eX4m?un|}+J&(C?|n$XBnfP+KtYX}mFK|5{ONkT$XZ)B zqE*Hw3vn^pnptJ1o%XYpSu#Uf{i^Im*X0*C3voS#YP@jb*}D~FKTkt_TXtguG|fJN zmv9FEB5_#b*6aa1aCb(z^sXRz2?EO7ZqR*&-64t@9Ze(I@!WP~@>@utK{3WG-*)t@&i}2TNw#jvh+I2E@0i>tyoDKMAxp4JPAW8Isjk@1q;|bFe zs%x0~kW~)pcVFbCV}EitNAdu8=%6|+3(OrBVkvp%JV`tqRZ1raQMT%Xeq_)QLJ z-5xpi3OkalwJcdSIgjolcJ4z89K&lRsQr)xXd)q*c6gzB(lubgJ`cX5U^(|e4i;h4 ziR)+S?grH+RLn*r%?OWad)@IXAq-anc@*mqBLZ61%g^*M#9nkA_#tf5H*NXV4cAve zM;%Vd&|>qhrV5TCSYt@euaJmW%c9GbT=iIkkK93&NX6I?;Wu3IlxRJHupsHAs|yLg z*-q?tSq4OpV5sL;kjz9qH4Dz^`m_N)BBx}Y_{ecIz{z9cTpkOTR}DelGxXQ&%y zdYy7;jf6aH&Q|Ie{>(DvS! z^45o*C$==TCv)c`rTZN0``<w05yl46z>n-a1LMT$X`qS zmZZbYFoE_o1;rFnQf*pki0xS~&IU3Fd&g)aU#N;Q=#1E3~|uS1znC! z4dKw5_sz_fZ70XlxiH_d`!ryf47Bjt+#alb}Thfm07d8v#tXlhK7B4lar4LB; zTSTxF2UrGNN#ad77DWJ%vM6K_On|eZ2DsD+bXJ)eD`VRQ*U(#;VZHdd1jFb8W*LIZ z!dOZj;~9AK3Z*-dH}LO~_}U7hzjPbn>A9BAAtB0& zwwc$&qEe-6H5%Zi?#|JzMa(Dkqx4$}!=6V3QC!iAtBn<3+0(~}#Z^mBFE_{YDQ{a* zFom9y#zT3ZZn9FAzBhzqtLJXjfVCR7uhR z=K_PC;Gyv8+Mu?=n~jR>vre#KQ? za`nkdu&y{tL61msIHROWi6t=&ORIM7ljk@QKl%{C^sd2BfUqzF;5&KpuirvAn5{X> zY{OMVH@eLCCnnksrZ#v!4$?NkIzyp63*#5^ut&a8JKWd371??=Z`R@C^SG_l3AoMQ zt$WCR)2%w2_!fqZ=Lz1x@Wl?wa2VDrmVnc3>0FC<^^TW3l7=2S=hl1S9?9Or4Xx|q zIjnjUnS&T{8Tyq1;pL128iI~BZW=y&nit_uBnww?63RZIjZ2S>7TV0W7|L9%)e1i$bT`@vKwx(M1gOA z{+QQbUBvl(Fsv*Aw2y9U;))@QaM!=Jq5wZYz`wFrD%Uwum2ac{@Fgd+A*eai_+-uP zVn@v3Vq&YixqQse>(D2DA2^)QW;WB@g~#neijQFBxM&)2&Z56FZeVvqe?D5lvG(wy zoQU6WlH3`$T!MZ~T0s&+E#Fq$#JjO;u%x+9c90Utb^u$oWXHdq(EsXu{Z5u*K(G@! z*%IWp4?~ z4Plj=ckmUhnEz$=-b?)xbBHE^0DBrFMfx~+@Vk9fNWEFPjZYF{W?Ulyi8k(_CGsi%N@pAb) z82DR+AI)_owP@9aF@Tklj0Xyzzy(&0)ci7mPuN`)??dgeYwcUNzH;Qtwv7HqWBKpY zN*$INV%F+O&)OZ>l$}jkDUsUK4mT2Q#HMERnbXN3h4@jzFhMCvm3RK1{uSxjQA!WD zKS&$>L7L-#fwaBdpAWRFv5URafA*#}?N?RM-tBCLE%!~V#mD*%{oJ&78kwns>qwJ@ zimrq$*(p_vn&n%{EIYm29KK8PT!vjjfzkU45;4dc?L zr#T;d4!9S|{X4y`P=4yt>?H)63AGsfLHazxhYU|2vSX^?jNFbJV}z~qXA~wxqm--f zXHJ>)5p51|X#>J&x9r*zNM3cZvsh4y-d0Cf&TiPYUe5`S+}$3-;h*NEsiMX&x4OA)h`-i*JZdJigEE_y>Cl5*~N=ds27xQ2UoxV z(Hgndn+$-P&w~XKy$8m5+te6g7>UbDUsmh4cmiNGWA@Jpeh4aOl>5}sC=czeXh0A{ zYMuCNS{W>Um+9MA>$|epl2647;u3pTBzr`==Oa zPWY3y7V7M8r~q&#%dNBQPwt3Iryu1jt{z8zc9ehhJL^>NgU zTQzFyT34KDsQrAxbe)`IckDQ9?0nAl@uJ?RRT76UDRNXtFHg7~&<)bM`l!39@h;V% z$#&4L$m*Oj#GT}5RhwGCCHS@o95s83K{^jN|`px9XTp*|J2eaI`R#FX2*Z|3@{ z{xoUE0S+%yt_c~4(*)Qrb)Jd7^*sJcnoK^gBcbMZNc>`JF-nQB=T6bJmAU6N_zuxm z4R5gG432jC^oi`Oo=;$u4i`Mxzc3ES?V}$w`T#ZKNPy}QKy*B@B!Ereq(Ea7xG=b# z;gQCS8^JPANfOGIBmrBJDgj%TBLQ0x_<*7*O?)7haAWyfbiuF?y-WTCer;EK{-)$P zc&puXu{$j9V$n~0_8psv(6iadntM;I?5SW6H1d(eoARd zSqkdpFaLn*v$)dQgo77UAx$z$M8+6-%@NFW!K9*^E#t}^WdE#Rm|~>1F&4{2F%K|4 zz!2UoJ)p{G2rM(j*HBW_Hc~<7;0c5ZZBpsTD9syn6E+rAF>7JU=ESs2`{Rg|DadmS2 z0sx?Yi>0zq9(tStp$ZlU;gW!Ee0siH?y;YwtoL4P6dlaPph`p_)K(sn+*mA>|9#D7 z>f77R6T}w6?iV~4V-exo8S-fNXI<)wburp{yaV*S4}5sLF-3IFQ z2v2J0D;(pdWt9&#u6YZuLgDytnJ@*vIC2*4vBQd{Dk;p%rkiw+3^L#4qsyx(VGfQy zDlT9~8)}57!|xTi$2-FB@QGnY+* zzOk7kD>;A&6Bn*A=yQ>%5-cTRdyI8Qt(i3)bkI~dGuN(%zv8BkqW*I(vltf_7vIy? z=5G$a=}{CXXNkgWmY&u3se=dZ1NX%DyJ`9!aE=gZL6@H;y*0!Ea)IyG`|!+)G)rqMl)oW_I8S2ovl}qhPmN|BzZ4U}QB5#5oaimdG|m?0Nq^hlUqC%CPMOp;qiaZ5 zVmYfc-n)SrcCEo=N5$mv-I%}(nnC(?h2;`BRhZob*YJ6<95c&(!pE<;xAyrmJ`6XX zS}?IO?X)D6lG@pdzYQLtohSI@!&ZtNlI_r307inZ%~GI z3oMgLru&J_O_!X?2d2R-@d68Ig1RJ`V{x1>pI)hsa6qndAmkS+t*yvEf)GJ z4HsUsdfNSn4Zpqn#X~yI<}!eUd6+Mo0CyTLf;*N_V%ih?1;HWAgGweW@FnwW7#Wla zYk0|JBT$$tr+52BTsCx~t=c$4^V1pd2(7AU@e3qz#r(K8GVE4qtg(mM&mO~lc9Y-p zWEI)ShjPm&nhgpQj^=I)X^AXm=3jv24>enShY>#q)uwb^@?v%;?_O`tD<;-E49u$r zcTTL;?3Q($EwD8ln2Q^X_HD|%hkCV$>v)D!AlWqKn6+0!GCo-C&Z>7B^#quWJH}&J zWoGNS)pufuc6-@j7aV>DAxBnAL*^V)-PHuH>RbThXF9Y4%69V{HSCXf% zw~3UVw-88jm&cW@&b;U63Aea3d~!R=#j8}{e)GUNp~NRQZfnKY9TsC7KF;`bpl)GF z+$?^1fZy()3kpMh#Z?Ma^rOIqCW=k^H8h4u?Ltuope2o14dfgP_MsOirB)n;o#R0k zyXu6xMQ7#T6vqdXRacZ6LL1X0v+RP|=1^=4WA9n?N(o3HaC#7_2F}xwuE>IKnFRF- z549%0rI_nT6{H9zS~LZ-9DfQ$R|tc;CDMP**MdZun-le3YYoFER_hBurZ=3yqH|p8 ze@6kvIOK*f60d3Yg&;3Tr9ap8aSS>I5NSV<6=j)phvIoeQE%)YySLif->fckN4^<( z;(hPc9K?UHW1@tobfX*{^N~tSc}gKcbP7@1N~CdAE- z3@PWyO~rGWG)yVyiStEszH?kDD?zo0L$2dsoKEIcL4W zeA+Xo!TV;Izvmrti#YvL>(>6I#bw{{%#*}vq=z0_^NuubTy$ms7A`wnF>~)$tcbey zmN=ZafG{-+&~SVsQd_h@fhi<=axkio=68$|Wll@wnG<8TPCpzlseujqGt=uQjx?T? z2{!zF%N7q;I>Hvq#hE}kpqluSHjp_Rvj|1_a*2qw(5HjNMS3&9mcz88_PWrbktjqj z)}7^+aAKx9H3g5EKt7X{Dk`feBRl^_WIhFEi)Ul+aPbnKjv*wrv!tqXVxkeY(L0JT zO$FRVG=KCt7bZ3efCmpVCPHWV2$p84Je*l?E0`rlWIZZYjXFza{8inerU}&)3Mq67 z8N?8>4Zf{HWd$X*KrS^eL(jfm!RR?^tkWOXDrLhtuvK3>7`u(dHh65a&s(ktIvWg^sEvg0HPz?FB45jE zWh1PE@yX2-)vkVPugeSw+WWFd6=T%V?uDPSnHDgJXXah7G;=JJ#!OfKFwWk7#s2ER zlPi1TVeoiEi)J;`W_T*=d;3xL@B>$&{Hv8yYd$uYui6>wFIOkMGFpo%dU?$z*+Zrh zV{{2kQ*P^tOr@XaURB~IUv&ys2SjTGVsm{`63~-ta2ZHO-3$XA+`)@0u8`(L~=LHuM2d1GO&`S@HsOttl_0Ac&=@aJuudIsVHODvX$+sqGP zyw~U%FF#*&T*3ywgsU**hjpRrfcm>(hi(~awuYQx4|;jM5b{HFP(#Q@GljVA6t6+a zOSU-Kc?UGQ;_mT8HQw_qwL$csOh>OX>PbG)nU2n_k0pwFCzFofQAuN*DyPz97^m2D z?avFU;0%uZNK?YbbQWx~v3 zeWXUFhvt%B#q|9Czut-Q@^A?J!ws>c7~q}Wa=e&buq&Emo!7Y;&`G4y7ou+x zA)uU^`G#Yy9WR{`<=I$~stFWPw?HBTDSi5z6}@Z^5czTW;} z!eYB+7!&B4uQSHAa^wx8zCvp)Tf|#GqOw9=AzkEKU#XHg?xkeeLqpOO*Wn$hkkKU{ z>XFM-?lx*cOAMslOL_U5?nPS`?@9d8y%{(#FsA=yk?cQYZ&J=Z3^cmbas@SdK`{=7n213AFtE_)EY9xL8T$-(HtlM@VG`>*(MfmVOhM>)-Ji7O@o;$bgxvcS9I;nQiS|lcY*R7f4oU^}H z=CHz3Gki~QOvObx!JBW5Uc7=7_Up2ZI%Tz{Seozf$ z2mCnvJT5z_SKzwlCnzSk)D?Ud8I96nzTSM6HOYVx_wbP)Em;bX%tw&s(1F$_5)-5t zx76t~g;8djoa6=szo^2Ch3ZLT={K=%oWO-@eUPO5SVU9jIPhlL7={AW$xg1CjG&(r z)H=lys!(9wYcRI*uvFU&z~b}d;Iz>dNytwQW_&UQ0e z>*`U*(&(owGTEKqc~F7?&Pjdxx;EKjY+G~oFo`MLi@EBnu3d5`3y7kjDO{5KC5 z$ZlCN1djj1**6B~!gSf@#P$>0wrwXTwr!gywrv|Hwv#6|PHfxBiSIjezdKdm{F=Gd zySl3Tcdy;ORT-u)s>OJl{fG2d(t^DeI zq=uS9fA%=@u-Uc1;!NR(va8496+##ijJP5N+d^nLO#b;$dDkucUXx#jy?PSVtNfpb z?N zRE4fUqWaep5rwuN7_zb=FT-7xc*yVt0D$^iRZgl~!io6qoa~>XrF9(gZq3h_beNd8A3IX@w7sRMHSRf@l) z`YQ?=`I~?^U#H3&!kX6+lqtn&S9Dt+q~uUY7(}NGl_VAA9Jz>>h*#hY3Jy6N%^Y+g zvxo_(bb|v*^tCIFbLM)#XzT%EXkQ54wNg?30lYd>=zH_=5B7axzuta~-eHb2FO`wt zTV8punCvAUitR4(PH1-IF(L-DH3Lr~VxQQ=jNCRXmF|WEOi5N;P zhhZ=GCyAhPH3uoad~ecmw91?PEqoFv&v66)J`{BB~+=b`0_2_0s-PBvX;pL1REKc77Ed?EX-8zzETQh1G248ukH zef%!VG%YoMl8J78u2ehmlPYGaJK!mlEJvh8!BZxK%Hj-$X|kl@)DLH+%j-E0lGkg? zB84;)v!^PVoc%Slq)L+_UzVdtS!%`&3m4I;eW}0e%!Q(SM8V_(J$hgIB%sRJ6DtkM ztU#@p$VFLRj9mKlE4USy)ZEybqSTdp3#0o%z;RmuyHxNeQVZ66l4)uzw2+oVOb1eR z3#PAsY7^UTT$6$nA*b4q;24eztpKYn4h62Rhpf>07+RXc)VR~ zfP{{MQ~*o1?ff{6V_d7M;KwBRX6D{k%+X;@*v!laNcIe;53*P>+9aSxhzb{f==6^( z5657^>NBP3ARdCBo!z9n)KfaOW~7>~cD@w(BgPM2uSxgz21uLSzrTXq+WB-KN3^op z=qpE*mzEd1+*q~tijbLRr$L*dYd6?(q^?yK3D}H+`I>-^{^7oTBEqyB^I@yV$&Eh^ zy@pgz%HC|OY~2g>gf7#vc+hnttHE_U)953z!cY^PO`=_ItT9~jEJ!=gMKIm$oQrkt z+c3zucq&TWFk6*~&L!f}R~Zy@PXt^rqc5!2LvzusU5YheuNWc>EMzpGO2(k57tTgn zluaE1iH+#lcST}8=x#IkD4#}@$b2NaFZHdf=PnjZwxV-)WRLVoksVvD@ZdrQ63;rr(a zd2`ANco>=)m?5Uh_z~pA-f|k0p}OsvMp8@yXR35szsnYJCe6SJ6OWvug$hE1y=eJ^ zt*#7X{5dD9s)h)u{)(R20&?x5OPf_m2xPPL2~lq8l4MDtnvKAkJgELs35*=Y5XfF+ z=EA_ttSh6e(gSZ)Z(%lpmZi&urgSrwEhm!`A`y)!m6h%5d|Td;jjnqtQA`l6A?*)|eC=Nl7nXLq$Iduw)o z-vKz9;c~q7sst)fE&&ayGNfD)uz#N~GYiG2+HyZ7corqZz3EEZ z6j%;JeDnw4oa5AoTT*5|Gpm9(0%b4anPo+Ue)#nQEk$IPx9ROedIRHO=9bM ze|h(*?eTSYP6l`2=(bdTf}OuiUiVvcZu12`CoHtQfk}XzzngyaR__wt@}DPEH6>;y zdnjsez%9vBk}iKK4#bzZq&A|rHJUugvF9eLy`ez}0%xHU{WGTr zOPJW$`oM8yf9|7fUJXdw{<9sCZY6HlU0==j*zNtXbag0ciFR4!bi=&CUgNupAP?87 zO})zIg>_eJag!|K-)L=na&=%$)76w#1xDX36=xpxdavhkYsstJ330Vqy{l?@MyB9k z3(Wm4^D(tZRkz_YlcdW{xcO5=pSM1n9~}*l#|E2jY$TWBbW*6It8VP&_#tjT$qq`& zm;|rmL3V?F6EQH({y@fHzN$_}|FrHp^qx6lbQK9E770Th->^W zL8Y@=mrkoG z33}6S^{+H^>?Z%t8a6;Sf2^sr1x?dYCUB|9BVSbu-mB@If)aVu(Y^MRLxqLZ>#Uk> z^f+yNi}g_i$SNNwzfSCyPrb*k(OUlIVv`yv*E)~^$4;)h=jP^awK0~dGlz{}cDBdG z2qocDL(h(F<*&v^xtethg&%0??u|ALIF1HZ(Q_4(7E(+fcO3@x!g}WT^lh3{IhH)tEiO*7S?U{q{FDYx^_JSLkD( z^Sl1)U&K4>UAEzw0^h%zfiS9gp_3>0LotTC@KuwyW=n4D0!Tgwo?fyARSuvof7Y$- z%;KZa(Dm#8=-X7c5DrBN)!Kn#YCQ3WMaQj%bZ})y`zBua5Cp$Vbzt!ZD8xWb$n&FV zIzM&cQMb+nOl%+C@>abmkUC<>{N8bbP3Pdo?F(5RWxk=y^8RVmv%?)2+sh$9QQtAn zO>ueXf$=bu?AtBpUy>0ZANQ?xUkv=f=l-I4?TUK!&FvB!c5O*@0EPeP!4FBB=%g|v^}?v9>#7X);)@S z!G?L^2E{x-Bsq~Ax9|2HNMaoOYV-v~VlRxE?^jY<&lJvIz`-pn=`GE%BOYDJR&-=F zQ6hk(>JPH`3Uk({m`K!Lt}$qQb28H!4HeP8P!ZfxP8=$_@?L&p$l{r`cmXiIMo0&Y zR0#xgQd^lGtEGZNGn-pfu5AhS2D0vxd2n3nR}oF&3_}_ht%1C8LygbA;H#bj!~E_7 zLm#vHp9;9PM&Y?5v~(>Oqq~yoDL?|Ab zbe^+@KIM=bpuYa~J!^(1Kfe4RH_{O}{09vxZFTxH`Ve=r?tU>_;rG%fMM+iZC@V=n zNm-|tc$2qCDzC^?87MnRBS={*hA~BdBW58qfGaeh6U_I|(Tm1H8`eP^2U?n*rWFg1 zeH%g>mA0d&yX4hi(#}(;7mC(Mq-zG(<}m)2f9|yNG8(Tn4c-f!JmC1g_pM{tYjT)& z;yU~5+1~~%;E{I2RSi*%7Z#U(!dYTaUcp%+P<9GFr)0NO1XJ~SU9P!7ja!Uq1U9WE1Na4Mb&%c9*p}VpWpVGC$LhFET4bV=xd+uMIfFm zcTc504j*5nROW#mZDudoAVlGw^1{c%8AX%1pwtc~5URgEVo3WvSd!lOE^;~3)aVzL zdgZ0gqqwc?A6ieBBMxWr@I?dv^G809tcDQDHyxpi>;>*bc7q#ujngC%dwSNZ71?p*Yud5I*M!O>D3Ay7cuiHJ zUKZcJL2*gPPM-4iXztT@mn`p@-viQs{s4bc?Tc;Ip@#e*>tj>;!%m}^Iet7kWiq?t z)f*lxwk2m|C&5@Ing6s6*XPjh4Z=2Miw?KjVNt73uk#aMpyGQI{3}sVkO@6^B%b^N zrU^4|a5i60-{yh0sCy+ox6i|?E@tI^0?Y}H23ULP-iA^o3BAZ$FYKFA{A_ZH;!&jG z8-#ytze#z05W9!C4Ip3=aRO{@1=$RX!-wMYw=3(?t(JRUEew+!1dSL*0Bzx!DoRIE z?U@AzG44o$F@hM<8srbdI*5T~M$!##8JOIHw#zq$fjLLU9iF(Kw(}o(-oY=3O$+LZ zj^wmcQ+qVD;m`flAN#|mIQ%1Ocf_m1q+XTH4>F#W-7byP(rUEd|Bci387w5?gMomc zL4ttr{m(hAw7rX~k-dr8zj@M1RrMSerO^2oxtwrvgkaAWMAAVaP|J7ecjlA_+=WH5 zMM_me2N&6%C?HKN?4Tm^gNA8u7cj{>qt%UDe)17r35rCpCLD^tM3S($>nZzUs@9!wUTZR&QNaG(ZGn5c!ryqT&nVCK5e!Gd<6O_`*Iz@(2a1psmu35I zVq#SyxaX<$z+$)?rwb{FV{VlI>fmAm(r`QaUgg|7nf=U%2kkHSGRNe3l?J@Xr>x)VC*mE)|BGW53q8k&VtJ8!k?Tau6- zYSN$Mau*s^CQLm^PN2U6|)IiDP~L= zQ>}Bt-iFa^K2r=fx?@z2f(g8ZqwcqpD|wC~VsmK%j2*SD=(_T$^15=nD!pMdA?=F^ zV5Bp{Li2z~;l396yV~Dc*I{GXLJAjmnUiWYz_4cGgnFSe_FQm`VV%)P;lV2mJIgEU zRU^YQX6P~MF>0N5{-0(w%a%7JW%u~HK6d99Si>+lOkR{jxlfzX532MnazkU?9;WjM z_#S7d6Zqa(lDy0Br)zPFc!&3RkjjR*Lrj-5WrPcx-Pes@M^ILGme2gn+?=J1cU~Pf zXkZ{uyZf{pBcdA+I)tl@1HoE6`%y*gdX#rAylZ(@14sDZ9%+vrs{MTV2X^o<{~tb3 zH!E9Hv;T69*{R^Fq5CsSA5_(Lor0rAH#$}G43_9<}>4x=X1R!`2Bjs39@oW z6f8hSMrKx$fH5Yp?9K*UxI{23eO;RK3w(^R%KoYmA~C3tJ;mh0=STlrR!V2BZ=xrgBZ{{8TI z5w6Ur1YK?lipixXd`Db04rafN-$G95+Hdi1iohsgLNEvBtw zsUCN=x5La;$zNkc95L1&OF+$95`*0*R@w2RF~%4D1#y%d9b$qqkvkZ4qI>xU<|9F{ z;OaYw675K=1I~>=S$qVb3pk}LEYg=s_ASJiwK=RYWeUAOEi>EfpUzXVn`Nb!#WjG+ z5+6D8z1nTpK=gsF&Ut8!c~L@NGL$Ww-9Op6OZ3-(RqtMc;e0U*S+M1ZpWPeg28={d+?PYr^LX!E8#@6QG{#^ zGz-Wdu`eqX%MZ^dNx{+7z>SlA(2z+Rb;3n>0N3!UV5|Q|T_yN)C%FWMNnXw#xhb0~ zT4(r`!%93n>+3`+w`)l{w{rlHC6sQs#hY&M-5lnUfHgs zowp_NxOFLP__HF`-(JiwP^SxOSpNpq$J*dB^|bQ+8}#2d@mhdb^8_LYh&AK?$|&%E zh3S9Q!PT}gR0RUQDBj!0k5TW50?CQ_t-KfdJCTFhv#; zS6gCHxb&0%2q0UyCR`%PC7%B?Rv2jq!>I8VuFXd^WD238>dBlskZxST$i{^QJFzD# z5(x5I79NztFsS?{B02;<$W%-L(aLd@JE~G>jDu8GG)o8%@gxw?KO!pN*cpZGyPG1g zwt^isYJtrr3zYU%)XRxZ)pqr(2ngknJJ5j&7ul=$(?@fbouN@{%#p8Q!^t>4W6Zz$ z8wzT;1_#jMtL|5(RXZTHM0mNlkM^Jx?hS!Jqj3DnOdN?SQ}~0vY8QQhjJS^}C)(?= z#X#po7)?Qi$pCo9MM_P`TV&KI^eT0BbfTTIex!! zZV-xELyns#Je;jg!Xl#05Z)>0zZ^C@RhdW1QcN3e>0rY?UOgO_8b+$noQ=#Eisq_%!6p|S2%Aqg z;0_OlP=$A9GE*FMBo?6wtB))s(~2yns)j#a5Uu!^09+AE3lKj(0Ye2|V`K zt=-4*O*$Xy)Kt8*`x3HYstR3seQ*AYYBk035`Bi}TpM-a$R9i@$c9dob`m$$SP|*; z5IY$YR-vz>y7{YCRj#CPxxrACaB2T|KRgL|R+)^lcHFHb7wKFZl#NwJ-g`DMT^Orf z9An)fxJoz|l^AUEaHr*~Fi^`Uw!SjKy1H++3rEu%Vb@%5*>@3vU}e-$yU0_~ptBs? zt_Mta499b@7CxX{UeZtC2(e%~!N5iG5n1ovpSFYe$LN z80Gfi=FXgY^CyXKC5*bm7SmzJI>dk`^6RjUJmS7snJBE?&91!AYVC~59d)o#pE;Xv z`o>%Yu1&MjPBVy3TO#)A;;+v?N~H@Hyf8|4*_Cc$>@)G3vVK}i#JP!m`r$^xkak5c zFu=nT(9|B|yC|Gn?QjIN$ES7=Z|p4pU5+ArWEH}Wb18rYZS3im&Nxe(t!!iCCOJ|x zw{$Msl4h&!E?;z5B8+PiJuLm9EV2sM&Nj_;GrR6z{fX@fzk=JDh}*DgDqVQg>ujzW zzmOT;g8JK80`@nh1+LhdVyEE#qqgxwpnaVQoI@!6feCr_M%drGc5UXv^F%}^P|7%~ zW$#0B(engG19+IIg)jX9$6mXXgM+oKd0@^FdzgapewLx>dcspNw3Udm2F5F3^oD?* zQp-o5`2DI{cY=hF_lBWw$)`#ln-vy+h?`O!rY4(g3oVJhR8=i3mC}^X?W9CrJWeys zUtYF!>6j;lztAFA^N!u=(!t(IV09jtzUDBAN zSJ@3*Q}wP%NAxNU6T_q2Pt>E!W}U*VZ5xv{@7u->+O{zh6r;&b0M<_dR-NOITz;9a zI6`q2J$w#VFOoDaxc$1(Zvi9zahw2$-GFDbN~=0!|GzPzL)uJPDNS)R3OL=?c$K96 z;`FdrIaCVYE}%oHaJ<=zpL!{QQ+E$l>fkzPw@A;=@6~Zb-phQDeOm^nO^rUjPj*|< z2CQl>*j1Nd6u*0`RIoT=gICrVynaejVO%e!o$=|(CNXYPx;UKbHgKUwExb+(n(7Ku zbZYg6SK0;*!m4s40$8SGOseHWm|de0Yr0!k}YW>=*YC>1n7;@G%m~@#Iz>u?Z=;~@p1}#6g@O9ERw{u7F?V* z$(h*1Xp8NUGO0&QP*6a-3oWEk>%Nfes1wbMe!0T0%6Vu!GCKX@0^x0&JIK?m5vUIE16GFMBI0c zyg7=0mo90Gvl8W;Mm<7kslZ4j?@jJ=1B5aH$dXAinFH6GTuS= zB1WW{Mz(PYau)$Jogy$!hN-o45YjE32sS-a)}v1~j4zp(-7~OWGO=E=v5+#bc&+LV zojIrdV))SHlx?{|3;Ww}(Jr2c(u2$A+ht>XHK}On@A6o<7h=jleCk)0ED2kl4?y_2 z>&$m&(9h9lYd3;b97Kt=s3~dToa+&yTzGThaw*mbmW+>B41EhdTFL#kkctaASLlje(N=hRyB)8pNe?2(<$L@mD-Du+8OXbC{d8*Q5#vb zI%7~ViEQ-(|3ccwe*3DhihRLi0eGcp)86JJ8ginL{CuExbB??jXnYQ{*fs9*MF9BB zP0(|NPO~&XGZdd3jg5{np=cCAYrqitq7FS8!S+%*|7yz{eHphFpn&efD>G<3L4LZG zn}Xn-`mUBOzO9?y-2=QGMY{z)&d5mk00nAn-DM1L-4FPFFaBLxyS(odYgTG47x<*fbhTo*0%;)wHOO5Xc(fPXzI6N)Y>Amcj^Z zWWZWfnDPmMYw3AoY+hUPJnz_4?%}pxnh?#j5RJSZ%^mB9uBigKzpx+T{2MvM30aCU ziX#c~noESYT&+xrx{XtBL*^lNw)D^3j_`yV{*44#d_sF!)Zc63N|&MzO&icP#<(&) zbz7PzZn!3oS5(LRxM=*9#@nJ_o3bT01sOM_1nvtG_e40B!+cS_k=;LS3HT|X_{%c> zqyHx$p_aMip)X0gA-kmvwlK7Ol1$yw@CVo`ZLGRLw-TN-*bJc=u^YF?zg4h5)iq7s zymP~9W}?FQq06E=)AQcP4-ZTaVNjo_Q52JhnZc@<#A}(rs``J63(@4#dA2J=r-TlA zS!AV%u9`}5uO&3{5}P&pDxAZqsPmxhb92NsoyisI@og>M_OMG@~bhxmH{M^QNuJ$p5tX@u_|Jm)_SJJq&F~eC3 zX|p~dUm#Sn8hzL=tNNwinxgOf;A*{{2hPoP(Fr{`sKN{9Dd^hp3)&)bRNyn~puW#v-HS1I5wagm1Yz1zAQroIZLGTfZ{jf(l_LnHP1-%K%RQY5s-m%J9!b>eRvg6dzJXkua zA$0!9I+H2E$MUGQ+-Iiw@qK$Y67vkK5ZBsM`eru5hXu7IwL@j5P9ej({K?d9KOpO76!f* zcX&sgZja>kN&px5uc0dqJToAczEKn`VS}lob)6erKBK8t}c$H6gL;(m`3Sj32)6tABcobO=t+w?0z`Rki zPF*M?y|A4+oh)AbFr81A)yU^gxt@*a>{&)Ps-_-P7v>jyvM(e@5v05?C8Eb8hGQ;H zwJ5-6nJW=PJ*bF85!6zUyf!+V!|=drX^z?n9*nzBK4u=t+CKW8)uEcuEDSezjb*Xe zJ~?+kP{P*Jk^M>mYh?)<|At7ojr^m)or%nqw8MZOWj+kh!s8b3p|g4L@+w{ifN>+V!tA zR_nOpDN=93o873=s6BG*F@!6hN=R7~^e`;#EgZv|uKC)mx6|+Of9bEJ~k+ikzzz*iZ@|R2k$6PJy z5)P_VAPc0oQy#JKtD0I^oNe6?8MHS?CCR?PAG#zp69!1OluJo*KwuM(#~9WOhJpO^cS67IYOA{&eNGCwP2uudoiMAIVF**>=RUxfB#7KM@1?I#NO@f4nn&G#m`M z-bqe9{}9}%Q(<6llW4mR?OEDB)#&DaRR;b_Z%y-L7t2p*^CKd1BhYhDnXi5)Z!n~V zh_7U#Sls97IfQO$3FBDHGi@1X03A|mB*iLYe~MhcQFlp^fY-V|8uJ!*i8bb8ZPy}(IQ?ZD=we&J%t5YqqDR;7qBL|04`8%`l- z7Ed9c7fU6-v&15OXR^*HLQK$U<`hcJ7<`raYO>5@e{euvbbr{Hv-KDHnKxoHT-5dY zTZ%68rZ9ePbu={nv`8&Vow{YArERXLi*ig3^{{Ks3CU`z0P3ak=@^h180_3PK83o6 z3tom4EasE48?$H+%iNRs*`E+Mj7RzcZXX(P5|M418s{fr%1%=V&|r_OY>yw%11f<= zR#%9tJqm@pYNqV9Ms+^G{ktxm2|*oM@(*0%{(+0+|5i!*moQBx;%sGVVJ7Eb^j|9V zfBgc3MVq2v2)Y;;6jhH|qH^aZwJ(IaFP4mLd>Fj#maar~b*W7%2sGH$_y`(7`Tz+? zx!Y`|o5X^dr?8vh=Rfh9ak=hzdC51R1WCJ<6NmSM-N$)LG|bpO$nV%j4ZF$OF4RvA z(`@mqR&(;yYln|CTjb~ggvqs{g6nMBpW+jjZl(qO2$LSQJ5^9wr615U(Qt5HY!$06 z{T+!9pb0~_#7Sx&X4-@pE?^638%F{k(&ShfPuiV2h$IknP+p%3^ zzd&&6j1_egZWJf;X$il)x*r*NCC{~k71jMJ->N~rEelXYyVlWf8cVMF&st_S6K;nw zK3!!cbsudwMG9;kw6^iYelv=w`S?>R=QYYOnkaq=j zc$Y^tup6kTj|i_uia{#VL^ws%v0@=CxjdP{6wMow8dn+a5yL&KAp_FFsoIDmC(l(9qt{ zN3@uO$jj%tQW#(F~mMBb>v14QH_3mnl)8+kXqnr`sSm{O*k{_~> z!uixp6Ix^i);)v}hJZ)s4`O;RWzpDu(5_meNp0Ip>vkQ*f=y&S-{>5!!*~UsEVMf( zGEck!+UFQ6{H62n?rD!#G!zF2`mm&0_q0*J6eQbE*r!CVE{ouzP}<^ z_)vqmS&p{xcov0F&piLJ4vL#v(k{NPKY##UPEbGKzC( zMRm@us@-wqy$!`gyTWEjU!rJfXNMmW2_bQwWNNjgf{@b{auL7TckY5BB-oA? z`t`zI?Hie#kfMh}X?;=dHA|#F|Ke_{aO%aM&*tT`RG1)7jS7$#D1o=EUXnp2v%OMa z*%PTyRq#R;q%sI~qWPTSi-q!4k+Cm5Rj{>G`qi&L3dF1v&-fClL9)L+qr$&EGl}0* zovGfk+5ga7`4NW1Pj92WC!^FVQ2rhOGsc3l%bmVv!H+3=Dof>4Y=3RuV2KW_9=?spi)G?7NS#-^@*^O zGJ=8tD1eQsE{uB27yy7-wIRB*(;c|ob+HZye$Q)R??uiC(fjs&vv0P&naO&UGRw{C z;rX>2w*|r^m2CoI!Bo-hEWL9dWvHcAU*BkHwG%a%)VN@%rna+if1hka^Om>!!JsHu zu92RSruzFa5PTYB3lwZhw3NB3ti;u+J}FSp;r^6ah4N>95@(ojc#v5rE$q8GM6@+c zLujwpQFO4%qzz4Ukm$#1)=m&tJ=UqH?1W5JI4km5k3~j8KT%;x4UeXlx@HzeO%2`e zm|!v~4kBO>#~j8EO%?mckaz0Qe?2ZW5|yydT1di zNb1;VCZ;qDb$Qr4_1ZV)aBJ7b0#;WeqDfVEopg6AqqU^oWK4QE3^gQXqFCYCGK%V| zqLQ1b($ya2%L=4U5f8Yy5CZrl1vi%m8Mx}IAhirNWNrqst!QD1YAIS0fm4_>jK?te zoW=WzByw^yuBCd0pS5*#)H*C7Z;58Y9FIoG8e;5l6L8Dt_$caJjE~09O4k;`0Y!K} z#g`_fLnnhD%?luccP%09HhW{~XlZ{l)G$yp4jGY$A=nrjLByS7K?Y~zsugPV+?3pZ z9086}mx-gCn6R7?u#5*3v5p@{NEHsK8LK6X)zE@yu}jK0_3O7?B3|N}48dcDr5Sa* zSIx}WsUX+RC-)LUUqutYE z{q#-5*@w&NIiJWPIh>{egT;EZ!KVzVdNl+sSu|;cphUil6Av{zG9wocEnQSs2e*1v zpsE?Bj1mH*;!3fyE(cjL;q&4Ouo4$lU@zbz5ofs-|LJ~#$2pLLGZ(-A8a8}dxCe4| zbHlUfu=90=A(=0ZnGf;79iyqZ)75A-@Z`t<+INRC&&lR?-D1x>FG)&QTQnqMVX(E zFm#1g+{Bt$9wR(L1GH}$&)tdGPAbq~C!>N2^9;7&Z6i*-RMiM|#dG^$|LD~Js1t>A z@Ytruuma-oD;!}i6h{ywTj6o)IKpj4`C@a#Rg_^Jj@HA3H?{XzU^(*6xK!+BEJg(Z zOpD>$gN0w_qCXZEc9RAd5>sWZrKCesyU-~4JISGBUkfolC^my99y7 zVhmTe)HPaGM&}m1gcSU-K}qMCL^OedvRi7c-4WbdF|~-nCu+t6U}qstgvf8%<)&FJ z9h=EkC!l6~?yf5OZrz2T$`Dydxn;PgwiwWgP& zFmwc&Jhy^4cja^PBw+otxl6U#-eh{jFO+rbdv05Q+-Tl1bcd@`pYzv|!g}d*D~Mu8 zg>ETkKaRM<{{=a!gfQ2_qkA5bylwcAuX#)Di!p%2k!~0blRBVcEkhHWJTXbJoRY?J z3V+?7NBEMptwI1|OHpe@- zYj_ea!c+($3LygxBO?Mv+Uu@G(S7uWlbU_X=S;{>!cdYhfpUNZ8!N0qMm=F@ zwKa8P3KF7XW-yA1K^zD1(idv0N0;)cGMT8fM@ZikvA^HR9=iQ|nxwStYvx zDN>uur`feQU0IJy3q_OE<$#JE4W#qAhWtf%h+Qy`C~{$Ox+^3{DaA+Q_?=QmtYB3CT5L!1F*|yt&;-(MP(KFUCXv@@{ z^+y;H=PQV>!4(nUE_5leqFu?XPC;`l7#7o|NgmiPO^KzqFUqBkE6Q!8rPNZ<(K}(5 z!&{_=#n0epOD?84G~AieBB<)K9i1jNf=0`erIIos0l>3u8_+6Z?4ejH#?YhX%@*sW z#O2w8O|m74r>RIdf(L56tY%mW9i*pJ)pJ|B%bGh&ir6aJxu==st;*9>8Re_>apbCt zj4B$?HPRS=>gX!#;?nJv&ZLG|0`5hsp*~h~9(AOhnBEi1&*P#fnQtG7Hd(9?B^GNacFw zPVr%h(r{6^nc>0Ei-QkZ2y(rX?0~WoZB{Uv$e- z<5W-AW*tQ}!=1(|9DqG|Lc6(oOsmIJ?lbs}35q%~26znrGM zOE~WV#?*%6bq%X6w)_(^(#!A*TpXa*xT1lxg{{AvR-Y~*jmb;j$6K0g9xe5cHn`b( zIJGX#-Qk?g-8zgRAkWTROqf(-_=2~>to`jt8`(^7>D{5KfpOGam>4uT8Qbk5p0hN; zz*4Dh2TgL@QwFxQLYCx-@;~VGEs3@n;u3uo@hbQM<*C*B9FqqA?t~kr_3!l0AU?<) z%uC=jANR?@epehLg;Qw@EZTqw9+@cH6R<*0aHd$b((sG&`tVhZV1C;Mo>st7nOX~$ zBkamCM_&p(d)JmR@2bLKVHS&pGszzlF>ko!9TA;&u6VIs@|7lf8WA7lR-V&Xv(>`Z z=F%#2INAp;pFzW)+LzSNY*j$>_g`6++D?8eWtbtu#?X;d1kwDm$7RLNJSHKm^PKk$ znU`MCpV@qr)l_`v6Z&Eo^V%j@Ri_q=_32X4y;HkWY36*bI5y5dvhbZ^Jd;g-YhSYC zXun5|@K4i^?4qWu!O_kXMuCdCqxIjIHK;+CKn!XlD5Lek8p%G15;WnAmP;4FD3E_NVps4 zl=w7Cr!FMnC@hYa%wYF%20yx391 zWpdcxpF(>&G#a3EbSSPB9vL6)D1l9@ETYGH&p8}Dyd})9Iy88@gCr95dK@&68Vkkz z(L&A!EsBT4GnI88(BT!d$idas)I~+L-a*$>gQOdVZ|;%$x)wf)yv zR$$T7La&NnJy{%V?o^FEi^Tv{A`$)Z^`1Ti{e0l3&)3!)j~^e~^Oxe{yxQGpS3B0? zrK?a`lMEO)GKqPfIknVLd(w=Vc>Ko#4+|LSq-q8=2G0i*YRb7Xg47=%LJyxaP&9@R0$xOrPL*@w1i#0_}SgrdVSM>CKx zgkw;}k^3F+9BiyUwQBMK`Jelu^a6Co9&2^G84M%a{S5`ByQJ814kcU0JpCniI?WjN zeryO_8eSNL@`d~XrK8Ik(*n$|cQU@D+!A(a)M4Y4ZsGc&hiG(~<8WAbi#44NGF`=Q z$H~aG;Rti~C019{KkNTS(r~bH1jq@e+#jn9?c;Hf>v$BKA1~4i1kZ{shbmuQuCz24 z?>5HL2n_Ewk>0i->@9a8OB}j5@zGmZdr*A(668DFLSPt3y&XkRXtxHlw9aS^7dXK2 zlqaT7B#I1nkEJP{pzHX;7(~5|Epm@SQ40YeR!Aue&45iY!pErMRJGin2T$Paa+Sbz z7gsc8QnnatMd}Rw+Vxj#k;U=xs9fVa85Sc67*uLnI(F`=MKPU&kcKb@#TY-5yx;T_y7PqWFAbhsCoJR&}3%#liqES*0! zb$O%+iPEfV72ya{xGtJ&IeAG{1$-? zEcA2OI49F7K`m*|uJ%~xfwiu~0QU#v@JDzlxz0%c2L)vC;2-8rOd1Xx*}|+33{|VT zk(j#UU44JtV1oG@oNBJ5ir5L;;XLT%6%xUbiN5H8`cq$R!>;1!OMHG#fsSTJai%Ej zgkk=OMw3J;r&b6Z$s+;NKbACzE>pSLL3LntZ(#7pocznR^;6N4i`@hvk;-E%*}km7 z%@Rt$i1GSY>u3m#TU)&G2ZXQk@9DXTV6IWzjA-OuNjl=llYF)x%0bXc+Qr}@L5)j={o z8_8bUMoMmH6V3OH5lKt4{1^dgy2R%p<@)!I_SB|R`E9FWUpEpcJ}YLUZ;aX>f*++7 zZ>9;=OOZh%Gdbf1!u@=V#}kfS*eR_Hy>8IG;!UIFWZ%DmeUxx$6%q5Sqqa7tt$k!o zpJ>6!SxGN_B4DqMH21dy0(V0UU)jI_N*K!BOX%x-na3hIgU6(G9_v1cwp(w=1H`9a z%9m@THXg{BT)GgFoSPvdtPmknD#bHdLB*q+RDxx>Xv(28Sy(O9u8$YSy-eeB?C00S zQ7J;EynGa?>bP>En()rr)MQGMHXkGkWCj0+vUiTrgo(mE+qP}ncE4@gp0;gG+qTVV z+cu|d`)wQ3z29ceh!s>@Mj z(d&dxBcf8yD9$uU#e^YYtZxVPn6Ri3WTc1E6s70JQSlh$Hb&}(jcj6B@(4^u02B;( z_cj1F-7@ZCIX*fQLzx;?+Gv=>;6jmNl0}q*decs_F>c-#vZCm;dXqrUL{j>>sKh45 z;(6ZdoF;78AFX#2Zhm+*vK_Qu38-j#q(T?UVnYr+RA5tt2n1T*>cuNH*4 zaz8@F=*uCXiZ>=<0+1ApJ|pMO(lYoSBF%7^;@L-FDDq<9O2^Ar zeB)^)*=fj$8u1nBnUVNi3%$yt57BNcI<7de$~9Kh_oJa}sw98;-)`%m1FU+?82{^G z838VdSkoBq229)(8o7}SXmyh(#SPc&*D*x%}w5F_ex^5{z!z+ah}5wHzt&zY7frlDOZTTxl* znJ1FT6@PuIvI@&j3nfvPs%B);Go{Xa4rRp@RKCIF7$^|%>f=%PnXCQ$3blVlkv-%e z0qvs%a!w6VTL7%~Bh+c?akbTSI;xUqn&ZXH(!t46l`Cj+4CvHHAUSF(DiJYxphRQV zc-ashbZ@Bo(QI<5OEmtJ8YLm4VUcRA87KH)24RLpW92-jo)S(jkk*nwqx0Ox%6KP! zQ-mN3lQ2rfG94>5v=_*E*cV)Ewc1}SE{3`?8oE)vXEDlN-XEzwuc0gG z5Z$(m(i{0h)0-j{N1VcKAM9V<<}Z3QxB2;V=+U@{;;CJn&i9wegLSK7-m{NeecW#Z z^N>kyoAJh^@DEKDo#;VUYJR~U1C%41rNdF}E<1uVaY)M;AswAjAAuB*p#s)9;6BJ} z0VRj=K5>1h(2TvCdUssmB_SZfDRCsL4jPb0+4fhzwVtVcFBVA?QzKoyklgZhhxrC` zYqA;`*_(yw+t9t8kYI}R;-xn;MK8s~R6AgfQV41+&QPH1JA;<}sq!7;$=}12t|o&B%C>h(%kBB$hp|cen>(>ThHB zw8L=HXBG9MTqje(S;5FyRXmfycpvSBSyaTjAO>{ zH2u~~U$5cAVQ>}dBK!7^czfEZ^#7s~$h62>M7d=@UrcpqfZxO$V%|k3{tM?fvcEm7 z54s}m2|C2bR~>44pitxR;Z{`2p)CXI`jEez+e@8aFHHyXaW3vvGf3+3Rl8yq<4sa&qzn>!*2iZTLoTnHlNEv^m_yy?&0yFg7tj&VW4v5 zNB7Xz6lywO_462%^DcxkpcE?p4lk3>;UU%95MKS%QtnI4oX^MnVJt3}jm>d0lNR+P zpn2q0ooPR9x`4xe=3b@jyJe(rf)rCe-^~w@E`896zDY}ML{boBW;a`of|zY@#Ac%dd%0dSVzFclwb6{0}7 z_x4Np4!Fp;pZLQTJh-g+%J5s7-2@!iC7(PQF5mbIe`9qgjA#yt0lW0tqv; zwc{&vDj9QPMDBNtxY4^8rx>#om$&M4;x!5$(W>9N*Wfo_$Z1zEC3rH5zdNu0-@{y$ zw2R)e_=jorjmhLZbL>m^hK|yiLGwn$#cdT<^_H2Y^Y+) zsz>)PzF$q$0mJ6 zx-Ka8kmqHCY){R!^yA6n6K1S2eI$4~_QcjxkL*mYjJyCOjlT6KI5&|8xSI1)3m?N< zUYO|V%eKnyd<}zQV8NyG!qY$}}^Eb&B!g*EFJ|PjAm$n9LI% zxzp+U3DRkD#aIt}6MCE-a2?~xs3(6(f-dMA6-hP4a@S6casab#;gytGmd&KK4TDD9 z_~B8_F@oG&sZc2g5q)pI{UqG4%?w?_EzUQ9wHV znVw6qzO)hfMq9;32`+1259Z;9#oL=xEO7|#!)sRYBm*-e#|78)+7#&OVDvOgXIotz67!F3;xA%nu4sFwN_7Q| zw$2VWGBM>Sr34KLO`k+luA61p-^+u4IDbveLyMaJYDB8rnwnTUd!Ib)o7NJ!q?)^r zI15qxocF#?({=vta-x&i8afsHGSh;PP@@eE@`BBG zBTrPViBXH0MsAik3mPX)D}PZG-rhjm;@}&BiBpTxHS*8=BNcX`w-bEsvm9Yo{JC(b zxVR8|P<@(PZtHV`G`){eIqK@pvOPX_Sn|MiKex-~gkW7Z!j#+t*s(|ndXVRu z;8!u$a~v`_AfD-&kwkXGAJw-)R^bj&l9fW>wbo(cINY(@?5#Z(XoIs>t_ip+^nBD> z60};)5rt+=R=e2Z%F%GC(QHBRj){2{|CsH4ve9Y5Y3s`y zPF%YzqvVO2jAPO*RJzX=?!KTkPpw?vkm7*rRt~u3y^IVvsMdJ82-I;-EQVD%!SjDm zEVgiLjkxC)m$j!it@MMEB2F(GaCdoqBn(Ua`}TmyfFXc*wk3!d<`DGBkEH_PL^O-j zvN@hx!V9r=C|01bpp_ffQM`EBVjL*0*^)Fb&*{s#BobOwXZ1exCo_+KCVx@Ak{*w& z#k~U_^C}m4Cs|7ZeVeh?c*c=CVqq37Ohq5(Q(TN88||HSRsH!7+qCD+T@KcLvgnyb zNpz*HsUJiIlA@>Gh$^a9J|Pm=`0CW}im}qQ-uKCtvQ9p#ine&`Z)CXwlrGuX^fsw) z;l52^{prbDQ})S=mU5eM`zzGB^W5>_A)WhI$>Jz)N91De0a(Iu!;1SO7iY7D!YOCS z($C9^qAkUrqyp=vL@J5gc`bVO+OkOp z3sY<0>(`(5{(mcW+hck6)U9J!-nR+{fcB-i|!;^Bh7sp(9sSS^dOQzbz9p;AU(hX+pQVA;ep)ZytlN`ZT z`fh=d=TqRXQla>TFws(NEN;5E;sF*D%;Mt_y8a~3C=kCjyKJBK@x8gniRvEXj9Kj9 z@D>>bYD?PwSO(%5kYE{WQbP(x&_95XI!C+S&|lVZ^BuRdXr}dZd9Xj%BO%aMe;1dn zhnBP*(!pK$Qk-RcKuyV5wr&2VqeQiC$g__7N}ryLZFdBCNBTtf!@aAK6mx;h} zXB^!hZ?x7T&I36%-auMF!pM-wFP1uM6a~i!74xC(~2(^>y3mr zsc#8eN%3Bkm|0jNu+qAIj)xwy3(p|AI+amArElIpO!SyW9>@jM`4Q zVsNOp&!zvWFFT7T@3ZHonFJ(mI>}8hu%)!f&)~@5mN5KjUuBoMio!LcF{>>rvz4N> zmNJnPGLLOpS^H_^+=!oeBa+f zz*pRbU?HuCfP$!g=7d_MjHqglfVQ7*|dVhEBLpz0RS*t zG)m@)rxB@qnX5-p!xErHc&U+Rua$QdFS{ZfMFfA|7)d1RgRwT4ycs?oxjm`yS~^3e zb?|bG;iJAbShb$A8BSgqx0I8Axbf0)7OrqF^Q0}3An^qA7rfgAOOZ8=x82&**4`95 z6tz2%2z%)W%|~B0Vfu2P7^#uIBBYr}~XbItG1b-eYNxrm5$KCjFjaCe~SdNkj@$_tgi+;IdmNTps(!GkI<5nMAty1nNAvm*xHafzP`)dy(_$_N4H5 zFX$cXv{Lm!av7Z4WX{1V5JeRs5$YIs;ReKw^IWML3I`z^RAv{Se@jGzj7DiUb^}qm zFiPw%edx~G3uryGVD7i*dCVwK_p?w*aHML4hW{S=Nks!VgQ^lGObWdh&$i#D7$4r9 zCMjD&{tqex%zqc&$!|<^J*yGjgM41_g++i!RtQt}29~k_nbHV>(kMWgQxsvt05|H^ znfE7R`U^Zej@Y?USX}rGGJK$(1uH;J-gsknddms6foX@yw#E=Ayr1X$dU|FZOvyfE zjp#1&s`_t1vrKT4GO0np>F5So34+7q|>6So%(r1Y?k2$y5ROz~~A-%zl$rlDKS zx(uvflABipT8Ds(%pa9gt~VzOEJ0Nvsm31{!4dm#N0-JoGg@IUTw#^5f3*ERxczEl zakQU3u|88B9A$&E#<6U3R>P$kC2Dg`Tr+1UBPw&XTnvNLGR3)#_S*Q&)M+kI=94q_ zwSttsXEr96)rePz=QUgmv#YwGam$*LA|7>SJH#U@X7eg3ZY~xc3>4rjh#ci11@9_a zW1?8i#yBw%G7O+t4FH}n%0awz1g%DlO7kiwVcP_Z%|R>=2%tIPf%vM-pNT(_N}C~x zo`-zWspi~iS7lUQRVh8{Ss8reat0S=4zH@OzEzA1;=7c$<&J9m zq^U-OtC7~3p?x#s{&4Ra=UvKF0V%lv?)4A=f!H*^iFog5L=}dcAPC(ZuG%wAkL8k% zQ`{7`EA4{k1$NWL&m&dK8VO=6X1=uDnfSRnvyiKGXZ+&mgqP?9L`Tm>Vp7yc3sq--i zSU5M>h@7u6g=Vi{p&&YE)U+@~nDLfmr|U$U8jC&^qK^9tH_1EeDo5%9zc!_`z` zf{YL4*mSgS#`PPPE55AZ9tgt~T-F#>%bYV3-h{qE!xc7bv9wOcm514k3l#fuXUv&) zFs5$QTNm(v&Z%em=bwSun6P1bV@Ca`SQ}f>=pq!YLH9=`)`uD~Y0%#Voh;J!z`O#sJ4EZc*xm7_U9Srr#&unuuvwt$N|AN!ZxIzfslP+{W&P z-rmIS+RA|cOg7G6=}B6KaXCB9$p$v2=@CEM-HCer2ljqv^AsP!>YTU%p2x*$_}NMh z3{Q4Xf|O`l0v^$ha_);xf0mtOVD2{)&@OX~DV)rMw$>tJkk5lxf2=O3VgvCGtsB+Q zoO`$%h&2kX=ES)=sNJQ6s8)l@U9XITem1{!drP}5^YKscgZe2oAW%stcDE!5L_o$U zP_~N~hof~jWI>Xq>%~V+LFa5WRc|L-R<9=81%H)-=p3PmsQiG8m?Sl`6y(6J z_09JOWLoFpy5$&%Hm+k3M>uA^KP)asR5!$|LDv)ik6deP!G0LuSxm&IcAsM9ubr8e z97^}0JQ(Chfm*XI7k|JAvFI;Pc)%Bc zV!y^L;p9W@*&dj+-4WClCWsyBdipO+-B4Lmm2s`rPyHEcD_BCZVf@L_$wL7k9DBk+cNB7j;*s9df`(qkY(3Aya*PQLM zxLW^NKLJX4Ab!GrKC*Aky)^IC64??P8#RTzl?PKhO|o|+ARpb!W)g>RIl;@T4FSb* zXpfOP(d8i$z>k^UVNMkWd;EBb)Yw@a+x^rby~BMk13J(wR9}bFSX`f1v7Q}znpGjt zKJFpRIXDQe*&kTndBom0?JE4hY;o*3fUcVo5gXRmF6$IT$x4Sn1Mt=d+ETO#SFct- zyx^$Owqe~)XtjG;_gHaj%@UE5{rE#2bB=KOu!AQg%z%(J!x&Qv~FvNwK_ z4+`Yj3DNTbyF687>LRgsbTmsm;G7SR)tvz~jvaQ{`Vk@vGFs|zmsT2)-I}JxI}?L( zB((Y}{2R2fu)OE;U)S|cH0{%%sU`c?k+$$nRuP+GmJ`9LtICkmPiA}>tEJ>d+ zoiJb%oecDPLqDZqw?}pDY?^NCa=@o*{pzA zHc%Aq?JTQE9jtL*K^*3p1s6SDv-{4@!hy11yV>G4UYtAh2#L!`9A1*yl#SA4ceN6uo#)Ea)boT!5Bx>pgBU z{D8GOp9Eh}0%Y*!2r8^iB02ybRk>lX4@}A)CE*kv2nA6=nkg zt6-Qa5?$0h%m}@@>Vh`l$UxmC@?T6zHjzjtPLgdbR%-6luq`OG5H~wNO2%c(fZ(Ug zAKmo~eB#BqK~1%>C=tdC%oL_-dZew6Hc+ZiwKHc%)w6pp0+plVs3Otjp=0qI8;Hh< z0O%(!feqMWLbD&`nemkNJ1(x_k&3ZB7+O$;(lN)gYY4kDLL9GcfH+W|zR84~!Kz37 zX_Zhy_igk3#{GD-A&_9`Qo%ErO7%Jk-w3|_gbFZY`$4*ZK*6)XNuPTd6(O2rIg)KC zA%vn-$OZbDfB5hlpJi5NM)BGHNT*Y#D(Z@q)7atQADGM>5V>{~F4NVGhML=(nwW>* z*gRP(!ikHt(D7vav>YONG2Q-D9E_I#%?TUY>>h(`YnN2cm}eAnxScPRoQ)Zgq>L{f z&Q@UrA~tf=zz861a{Lsq{tNO~l?tjmiNH=jKzqeKZX@BMp?P_#jbj=3S zefpce-$}8ZCPK4{dA-<`51;ND`$AWK)atWcOWoOkiXY;y$JWH@`3TA{Q|;jNJGTH_ zyTi4Cqc5M9I_?4dFUhymJp`i%y?y8aez{|O9p3=x>0{3(nXdrNiS`DzZx;SANAsFD zE)UwhL8ASHX6pB5JLJIp-uXRK;`_j6r8oHPd86UozTt$^+k4b6|Gw0~Dcd*f^&jE) z(J%Y}+MjgcnZQG(JEM2KFOZ*D;^R+9Jl#P+abj^V3qZ*v+APfIIL3=roFa;CQL`3{ z(D^CP`3v{ZW>G?)zAL(0!t*}#Jghk>?vYjAkMbkZ5n!ak3^mb4Cgp)1Vjhn}8WEM` zQjJ4-7A?@J9GjkI8mmt}HuZc6;a$R`;((=gqRFbzM`AINFv{@$C-YA$YM3s!N+}_; z;;Kc|$>!uCh8R{Yw3|g#!@w9i&>zKZ6pQR$$vE=?;}ErVrs1(+#*5lf*e#uzWHSN# z$L%Z9%bAg&;wuv3zBA($5o6u^m#u+caava&! zC><}C(^xM3Dh1fOs2J)hh5>_9#LW&my*A8hN1XX-a>4Or`!Wo()teQ$D<=rC5L{%j z)S$Xc5rQlLa&#XRxOo%dj^yDZaj>WHB8f=v!ew)V%*(!^FtYk!I==d*&0WSJty!vh za5Utk#s%t&nTNX#ll+TYh8}Esn&AoL?M=C~7I~ydNfR-<60~0%H0qG!oJ?~t>AV6n z%Dq8eAK~N>Z9S5qSEg1A#IHl}7c-hiKQ?SPadmE9{3j*`_5x&`6Pd}haV#1GKbmB> z`DLCd>NAp>M=gCY7fXy2<&{ZkFJ)sg_PVjgZInIv0Cqh=b|13wCFSbux_6%tDi}AiZX7s8FBW zRNgha3WvpxL490lN>kcG)l6HCc}Ec={I#+&jEHokaB5J$1Pkgo8#RwEN`>1`sHyEf zWf6Yh@&t~Lz$qDXkzNxsMp?lC=VNb3sjQ;`486w1pZ}fCZMqp8YHMmQPts@~$KMH$)%f*B4)a3JjX@*yzoO zAy&K(ey(*-t{6HvnK=4a0i!%Q;$d~e;%Dz7v>%7x(_Y@e`1Sr;IHK@emCSGgJR6!1o|rRpb&Y~D<4pwr2-i9jQSkVvgO`E;)aPhSY|L6F)%P&MJ=!#? zV}r_}0+#{MT=br)8$)G{^RVA8ZdVdBmqcxkN&CuHKR7-_$7VMPfYTC zJRa%q_j`RFJKCpDo?u@~dh#|!s84U~Uh5ZHph4RS@dU_^cKL&nIe!ubf3!nW$~kj4 ziN^)q*fd8Y2&;S)kclayceJ-D!3*$$&OA*}W?B%@78_C+2o2@NA?1&1+Y`<8iU&Zi zFWTW+e@^_f5IUI&>t`HXPlQas5(^)_U7U2_xoA!IydRO&aNl6(KMIi{XMZ1)v=vwZ4OV zbeSXpWJdm#^8A-MDMFiv!PzL)kQp_QG8u_?qIh~3ii{}~{GFT>2`U+EbV`aqh*;%< z;0EyEQ&kK#bO9@!27_2kF{u_nwV0%J%F~B=osfX>joH`04Q>C$dPP#BgRC zQ$!)ylm(9*o|9yr>(YF89JXxr0j3! zJo2R8j91gekd7IT7r4EFdkpS`rK32VpFx@k!i+?=!a$Wnpa2HTM@LfL?B@)TnMQfo z54l9ug&Lp@9MS>Z9RSF;38OHm+DuK SIei7156vQ~7LqZ_le@%EOU(SSvrQg0O| z;aD+3S4z5{z~ajfY`UKs<#(fysUKbSuQgFDx`TrU$qDgRF1cNV#}mc|-Cf9xqRU;{ z+lY!|@Vn$~guUsAeb3IYcXPt&^n?b~2Uso@p-II=rDmN6+%|ckX{I#h#+(zqTE&2| zw&~vg9zFPJXHr1qcgTzfxq#5GY+}m(qmbhtui&pN=ET4|^9QAX@a_2@N_?+BRPX%R5uRNtW}YH-LaLq6+~I;eNr_PE-j*EM=zTK-&`}(p2q1Vt#f=d zRn}c{r3xzr@n;D3y8ppzbB%f`jUvWSg`!dVkS|>}2D4%a_Ob7wy!1em-pOaC>=VNF ztKu=u+)YIsY?Wsy?DcMuyR%P3xnlEYseIlgY!;vSez~Y@J@~>N*RP8Z zb5Px$6~lVesjdxz>96cQ@g|#>20!BNlIBpv=EYr^2bEt%-sp!-MnlnN*=%udj6jR0 z1I{M?X;H5U(f?yA7JOTj@V$!*d5P9Qc z^pLiACJJFq*g`IM4~6-Bt4Qg?@TcPkLH!UZf8QBL8^hs%iJ`<9P4 zIvyz-EBkOX{|=Im+nOtA9=9ew5`o~R8$(6Hh7`oXGh`oVi0@Ct<0ff2`U)(^Q940w zP_Xv7(uI5z`(=a_L9c)mju^wlauQrGroqr2IcKn}`Lb_d&u58j*%7{b5T(rSiTmmxIyR5*5+W0^89d z+zLz$b#LOV+_jnWj@;#)kKZ`a5sbAYVw`Y8O8se(L9NQJ|`9X!zn9PJoL5)ne1`*69(thSs zc?+_@I)O#pH$so`Zar8eV*Bm2v8o#tAHOUpMY&APs6sby`$<{I&Z`I^zA;DyLn z{xegs-hXI!O~`D|#;8fS#j&H#e;j-hy{F~{eg1F2X>+Ua8`!l{pY+!J{KDnC?lnli zH2=iy5^f;MW7{R(UfI=`_|AC$!qFRY7iDHX$rs-Tn@`ik;#QPjZ%JmgCl>XE0iZLV zifP$5`sxXxQDq)u>&fmuc2}vV92tZ7N-D8sr%oU){-a6P2*kwzlu>E<0^ft zO|LA!k+4Cxwy=1&jIf%ow6L5nKv?fn-l*bVsZnkF+E3Z*YW-@&B|e5gbp*&ZlGHTv ziDIN>@4h8IM&r^x)_c+>q>=^fgV_`4;Tx{@4Laaa<~Q#N+NgSkKH5( z-zVZn(w}H#`hykanb-TyJR<*&c|9?0VAuMfg4SZ=r5M22FUll9{4dFoHLa9noi0@G z1wr6y4n&m27s|?Wi-@s2chQt?k|^uCeZkAFKWxnGlkvZE9g9?G7Az7D`5iIl`G{s> z7C2`@tb^K6Zf`8594yRqzcY|Fr+(NZ=_8P4>-BcQrg%+UD zJ4;U|S;Yeq@==8Zs~M6nf74W-s}vDyAa@=EiulXGZjBBLq}ML6I#!(XsEzy7;Vmj0AD!woQF#& z+h@L!)fimAf&hOwOn2tv9H2%yFs|@MDNbuqOfen$wiS^%dAN7vm)mrQ*Ua&3+%cshUr38%qq*8lyLH(2 zgNu}xA>@l(aIVqI{RpnQdLyW)?kpw#@-vg`y+D~4gzlxqTH!(EWmv~z3x9adE5qel z#}W&FY|g9CN*&=KyMJalrGa4MR|TTM-@F(XMgwgB5(M9{qL~HGhHb-0 zXSsX+JVGPx3yF;y5O@8GZ$?^Q_521NqEe%6G5Ih$fO@dcaTRTyUu>C~Cp4F}40=j2 zLMd#`?J)te=4PfOC6DIrWaCNI;aU@?cGw}f3%Fuw>rCN`<|1u7Xx=w#_m>8z9;{Sd zBRsEax94bAMIry_W1rp{qS2R8Ay|_npE~%4q%R}Y5u$!HS>HjCyJE>u>~h1xKkZoJ zTnE@RkABq$JB;b-2R)*47=pyax;uqCqv{)ffje+8MRdp3>^mRXcujXe39;5`A>@S1 z!8?p{42*QFnstvQ^IK@9uWa91$?4@bQBZ+ zUUW>gEtM)iJ39>wWisjN=Ps)sQdQA631@ty)lH7(UOG2D0>yf|G^jvbmtGcIX`dW@ z_}41^!duRl&Wp0#O}|TlJG`pLR#m#yJ(z6GYOSDfhSx=ohs*LG>|kBlvd?R&j~tp? zcRp5%d=VCPf#u@k4HU4BoCZ{as5iDN{A#fN;j z$jPUIO9dWn63e&lP>h418nNTg0(&|MUYX?f2BYSy(RtedMVQ9ENRDel+!3-?1RzpB zNrV-^wx#glj7Hs%&Os#&MqCzg5c9a0@v%Zsdzob(nW<)sBS(}_Ku2nY%29|4A1~Jm z@mAjX5(zx7t_pjR3Vylpvdx#zbaK1D zN$!g3`y^qFX^G5_2Buv+;9VN5ky&(NIoD|GZry1{GOJ2Q;2V((`-Pz2F;<7%_UL7F ziSwHadr1>|wKHwp26&z_&k;TdO^nPjp;Pm(nSBBE6Ye$3|E|_2FSTlO8t*Eu?nfwwdzXYEUuSK8cUZ_j=bt%)EVw$Qg%tc;E5lzfZw6#cSNCbD!kFR#{ zYiQ) zFvJr?RRQY2M!9WnbJtKctX2-{m>qE27F)HyR9bFAhyFo-Cp8zqotAuuQA%UIM1uM< z!HhzIX4!L6b<@7hLVCKGq8M#|& z|DU*p@NdOk80RO?ZvKUQi=;=70Q+J8Rz&O-t{FI31t=ZrWaFT@O8QX3 zk9D};H-E$x04#oBbrV7hKM?VOb{Nh@|8=7tA=uCCXfAiCxnqfmIUNq25#Yq%6t0-* zCV^%tqoqg(bp)L}IQom}#b*NYs=)Y*_adzgA)Y-w>N=ICmccm`%(k=4t1a5P%sd-A z%r!LtrL}T3?p@>TVL6{Oehs?k4aO@O)ou%J-v-r9Jvy=@=9bHmM|581I}BmM+5$0F z_r*QU@Pm>a=6vP6$*Aj-7pXBRtY{wF3WO3KeQ06~Y!NH+nlH3KPd8Y}3A}2OmFZ3g~36n0fA)`pXf|)`sv8;;FBK zDF(F?*pu&47t5-~Fqc4$&XQ$m@y@`?Z3fO8<(q)Ct+}i)2a$YPp_dq~P+< z{+f?0u8-=o>bVF}YOZJIH$q#+G4d8J*C;82l=9MQ( zdT`oblawAbH$E42t}PqBb+Pon+pGA#eEbp6PUk**fk3ya6u0# zyQ0LhB!o{|ATe+xgr&?5HVn;#xy~N#`+fQ5WkLUNj1R<}-aFOn1a?Ow_;H;P45hP=E81*JVn#ANpHY>`$TZ%?H*t- z)5^jGGBtB;l9NroYh_2=uwMhyG1vfQ84X#59$^l^BD$|p{uQTJEf&;Lh^GI_z@(( zBRX}Rt{t+p;zo1E>^}Sqcrg+w7~y}z9$Ue>2()14Ej|`sNv4qMmC_AY4jLb*?de~h z$DQkv{be@}-PcWyuFRd;`%LuhkQ|3+{P zvw8d5-!FHSXx0|2JLq~D>_LWu+q3am7y1u5FVHj``!lPEgKspzf`)OvF`^_jNpfBp z;V)m(si$bQAb90T9&TZ(pswu3pW={|Lc0O)P#WR(_QnN>#E14l8nY)}i+s{LmXY4@ zbyyK2y_Zs!Sy70uwnvFYd|TqT1!>}-A~y!5o``|uMM92-c6+*SN8(Z-__b%pPS6%! zA-0tYW08guVC(O2NU`PqIgH3JE|X<7aY;dP+nSk5`?u)WN*Ge5M^cPrMj8ra@o*jN zZ~@MF77M;`Rya$K!u$z_Zfdg0u`D}Vi%{zG@F;Cj8u`@gP9wIhM^H{QJ2abdtV_^Y zO+gqb&Nb81%&JqUOe~ho2_WBwrQk6+NgDXmmI+{B;p%vC>v;c}g~m08;Pf1g+ZT{W zK*+Mdn{#`P?jIu%(4AYjJnv7mB4eV?DWMUgpaEY}Jzmboz5?L!B})<%O(MMV@X*$_ z{3PF?jab+SLAIcyUNf*^M&cgIGVZfMd*;}B=ZjOA+9j*t$L2CsLw#@77k*AcY2hR8 zBGs~!s#_1Xaiu1(z4`5YN)FTLG3@uqs%;~Lgn5eIFy)7~nyEC1dAj6;{M|e1hqZF- zdKddH&|~+FzsqG5#5{|Q%Wh;G@MxZH(>KJIQ{PlFv!*Tl=+3q67cx84f0pCeEcR&7 zHs+ULVA+>&c@M5>8Bob{Pw}hDZq?|j7G%4zHUrHHajelpHWqb?RSwb%F*9K|gs|>< zE}V{7tATSt>FW1)ZDbG6xF!mzQuI_wflXkGyMTeb8Gl0i7QR=UzEanqwgLD`&~ETu zG*7(c@8oz^nSid}yGd%PXeSVNv6a*Qao#5F;I4mizEDAIJ=Wsws>%Z-)n^^mXNh#@ z(`{`0$!re+shNnv#NB`NzUo>Rpe{N_#`_&QA*`7=Z*WqRft;z2=t0Yi#9ocv}rwMDZe znXo6yv`Wq#9dONJxOnp}R`8e$kcz=>m7$=@km5y9V*_wcaZ01Q6=qS*ceA)}r}BRJ z*9r=lAMO6F0r3H^pg)9kh~Cl2p{qu-TOf$nNQCDrP$s)5_LUyFDR@OEyj6>v_lh>T z8QJ^2d8K*hrR7#jt+;e(^c&(pA$uwa05MMO>|g)C$SUa}F9co($&c@ZO z_oA%nVjieaF96ya$C%plNSFzC=3hqkHz?_rY+L-c?wka3-Q+E=3J&P+x|n6Y*c`;z zzr^k&ZLK}k4szXu+F(jX{hcEo=ltx@ha{k9kmd^xi5!1J=X3o0GsX7`j_C=$B5-uV za=p3V!1o7_Bl3^tg`^r2wlq-N#l;%4UV_J0T?E_7dv-8(50-bVjL=+o_^!pU)=w5jgS;HEf)W)4+aDXh{XTD8cCVExtY8CpJYb`b2lr; z|3SS@Q`>U*?-mIj>C69?`KzpZvw#+j4uO@?PKH7UMj93bG4(lSK&jWZ+t8Nu1|@Q+ zCIS}hB_i;s64_|0!c>(y@v!d7;d?q`9{Bb3fjFS>=FUUYN#k2tC~I{cnL)Ndh6kM> z#m^=Wv!X|*I%zL9d5d(#6Pg{<6)}7VBj@dvO=A!`9vYs;tZIHT00>dH9SYz3$#8G} z6h1B%?J)h{NPEk$IM=LMm;{0o+}$;}ySuvv8fe@lID}xq-QC?C0))mPxVr~;_e1vV zne*<KFXkui4IDN7y5uu3s1~-GM2gcf)^?*ADTMQ`%OY@@CMg1ALDr1(E?eS zLx+jWdCIC@6_W5%v6MB{NvSOGAo5b97D()hm@B#P;lqYzFFiey`$gjSuJrt=9>~s2 z7u7QROccv+-tVFNZmmse`!#4TXf=TzX>I%u?|pRIMzdD78SFYX`zcukwxTKL*cIRI zvJGBB%DIQehLsOE)JP_Dm*`zvPp7*yy6Ua?~RiW|PahZ$LON2lKomV%5lxDb+8{BBt8 zIK*g_m>s|mnoCd=2?CQ#s8EFrf1zTG4f&fk2ElicU^aSD7ZpXqlA@ctM{aMhroO!1 z&y0yR_BA_Yhca_5ZIN8nC6hO^@nkpOg+gp|jnnxhtP^#ptvT!1WDHp&E9LxbwM-m;VdNmH&%zBF@ zU8zZ#c{``@fX2?z^xzd=rdU{KDy^AGvDCts8JH?kZ2wxLJ=vN{)mLu2f%w1QkNO{K zq-Fzfa8q@&1OB5x=_-F-A>WwqoS#qh74zF0o6s8AT{!8KXWf9J9#coKeJy*VS1B zWb{=XvPx)iSZ+oINClpQ*|;v_JkZ6@!Ca!kh>0hueY*Mc#mYA@ z=_!@9rPANk#;V0^-%PNT_Di;yJWo9nVK+|%Ma9lb8ov=O%LulN0oBp&m{Ye3_OKeZ zLk$kwq~K^A3dIakdjJ&t)x|1g`TkT`fHUOIw}*mGNZux~N)qtE(o416M4>b8wjFqk zsC?c?+_+V;E%F}t9_c4kO|NaN84caXujd}1nr?7eWr%jtAVR85N*fin)U-}~y0`WT zW^_t6DzZ@lD$~HhV~`Y^UP(@d=M(tg$wr`L5t3hrp|3w#CM+2l8)#oM+efq}Ax2h| ztgnf6(2=>DCHmahj^TN=kEVwPh>r7D7%}zov zL9-+&_mPyXuPjjC`pN=)!t)1_xSakieu1G~b}*LzWPyVCQ5E2Jzi93+7QlQQZkd0X z-OiF~vN@dMaQuS>evbZP0V93f3%c}&#dqFhvB~qrx#4V3_?Qy*VRk-B4v!ADhIKE3 zfxRK3$yCSIk=0GEVisDwf4a-;t~AFl6%YMU7oh-rLky<46qkf_^QUPoyCPr1TTUe_X?yIA~ z$}vP?y7)!F+luy?Dijy%t~ZM9_S`NeN4RHD;IY2UekUegjUUfUaOFi1uE;+Y*2wU% zPuQ5dNO!G)Xhnu#=DiA-QBI{o-H%QD?uaX6Zs@0&?qs67z<}D|76}Q>;wgPbf5*|y z>GyVsB-gPQ@<8_qU6rBZyQj3FiPq0(!wXpRO`L>rLr4RAv~aE;;KAs%=Qh>fT}b6A zI|U1#;V;gD=y7?%KM5N}zHK*$rz3a3rY)1Pk^`VyRyCbETbYzI{Pjy+SXcJxef=_l zZ~pIZ7XD8Ym8+eVxsmz5`i|}Q`YGs{%UtOkd6$e zJ~EX1i9IY~2V|BXNsZ&*DU^}NIC0#~El?G*=zppvH*z<=G+$2d409Cza22AlU^n%& zbF1Z!)UrBLlZlJ{Ht1Df=W(-TU;TV03M)lHsS)VC#m*+E4Q95G)UOYqy9T6fpzgj( z<(n~un%@u~Vd7rf{aO_B*W&Q}P1yr@l|8o34o1L#mPaP!wLF0@%7sP`SUoIFNSKhK z3R%$L;PkvgR&dtA!Fk%9rOip;u2f8X;C@5zr@&#`Uxqf_UfwW@=x2*3Jygv;|B#>a zKg=4Xa7hrWTawqYy^3KWl#TYF?6M{8>D3qRg+&m4F{=>Q*}Qgd<8x4_BJ($A%R=l5 zaB2NUhN|r|r)IYh9arKx@9X0+@GyTN8@dTHDz=+Zg{JC6BAnKO_&9 zA33k4k>P}DEJ-BABr~<&VsZ;6>hqtH$0zRW$FaC{D9jmPC+%YU$4&asNagCP57-N`*>3 zpFz@fPmtt&PPrP-Id7m^A=}78k(@9_0rp^_-Q=c_qUbvoj1qPeroL0v2QLl809c3{ zkD%iyneAXAJHJgK4Uc2E@kxHfeDzKT6~sVpWhjq6kxqk~5r39)rfhMT3(YFK>0UNn z)4BV^n8K?Ev)y*7O?ABXe3<^zBmgnlJ>T+{H0}ImhLYvQ8|n&9)R7SU$UndaaXySJl;&guj2H^b(S=5VPdL_ z|0X~9RrU}_YhQdi5O5K4{*k%OMk>>UNde0mkcukw3vTuL{{8eaVKt*JOF(9_HdH1-uL4{^1 z+a=~r7s7qX&+@1?DF$9Ya^r!~O^mxn7&i%akp|@#NOgT4-z(I_j-1EjD!MbUN#o9j zXroa@6NDvlEBDW-`hq=8+3ee^Z+6fi)pd{%-Dh8aH&BpG+aFMF>{tAwIv^ zM=pzhy?0lAq&A z-ngwMskHzP5L($$eBPlzcY|*vM+%4Og{s%ZqiB76z#GosV&QrC!#p}BHOu}|5|k3j zjpKjZa7!foAx5i-YdOJQ+Kg8*Seg*Iq|;ym#Kg*1=ZP1jph*p?Ro!D=LS9gBAB~GQ ziVb5ahp`m8?AY&JBihcE)8zPskfm<2z62ndr+ve~VXa4;~bzlRh>fP<4cz)A#QV{2n>1hD$mS@E zm_O{sz*T4dKb#WjY{5I|9w~Cq&pOD7U??%@QQF(>Or=hx!!~pE(f6rf#qomWoJ1p! zRNZsoRIDL!;5EG0JUrezXNS*^S4TXb3!x8A70;F4D+V zuZr*l7!8jg6{a=R&MJPrrl|-96r?$vuGnoH+(j-4gC>lTMdu*+?RGbb;gsqr!zej% z(D7L`%>ph6+kw`RPbOi~OUlm+`z0uI6)=Yt>U3tM-gV5&BH}e`%#K`X{C?`bbaL}s z60vJ}eU(>FI_gLtc;L!;T^(uYXHP38|DpGpoTJwEks}dM3-4MM@9wfmGY?=ZRCFoP zW3PAT_tQULH7KmVe2=d*BfyO3pki0dxkUzg3ZrJ)$4)KGbIg7JkmWkmhOO6VZ~7At z9jEBBJ0p~x|Ti0G;?<$BjC?_zp#*Y4~RC1A4M^}9*j1gY+ph43jA z9hGkEPzV6T3JR#ip-v)ZX3sR}DK~0M3i)JZ59=KzxXUlpmc3>Tzf_U zNy1W+Ak7tXIs^mkV;H4Ty^07N9Q^`;yixS&X;Y7hk8P3hX_AC=t!UAlQ-f=a-BTkT zU|cre&yHTFA)XwxJikP>%yll-v^lC(%0m>#G5wyBRRsCMyM<%RJryAzxR4!^gva=S zSvP9inC_5^kGo+kY_VQQli4;2l5cWeuEpnX(O&2=dqU%h~m|>syAI4hv zFJo=$m$8<-V`Y^EG4|#k#@a=K4!5o!Tod$`Pcr-b4hlWypiqRb^a*LoiO-N4OU%kP z^9n$ic+-G9QlrdTvo#`KPYjC9@&%_t!?NQyO3Gj!>>PbX4gB9nji{}Q&Hss%^ekW1 z6d#AaoPRh3BVZsvF}qzfq4C(j5XQH77%6@Nhn zs!a);Qt@W&)EZKM$e7_gv!e$h*5ec?qs;j2>Q4`^yu<}j07O!yiVAhLMr6O? z$|ozv=-^71ujWmjYWYI_pm46M-pbzGWKDoUjMT9ZIE<`^Z*(m~knvYu2hf_I0j7c%3WEe^gjigvRir1s{)Mfvg~N^Juf zqcv0>bm0H|h|pw|zCGE+O`LR-l-4IARo2`|>ucBBmTq;sq&8@WUP_ zq3E-T`z@36>Rq{l+fOdcR4VXG6m<6h4C~Ero9gdp*51;;z0)s{JAP%R0EZ-_tc91b znZ6E(`mns&<}jZq#r`ggOl<4GS5kTj)z zx%bUkdDy?Xe*!n~O%{%H0V)>M<2w{LKjYgMJ}1XPY5z4wpu!%-QjhcA2>* zKrm_gr(qq-b5-Q?@bXL%h< zNI{k^R>Iy;{KVg!#AFj+qW0K?1-8K#c7>j%CANwM3aPz(?Gp;0Pz$uYfAh9#-&`sE znrH#9_2KxN-j9O% zK0ey>T9z>Izuj#9K}x@L%)c*Chh8ZPjc?X134%jPg!)86hDtu_H0Llxum~Dv03kHz zkn)%+c`%vU2sgOHf*k4VJHurGFf#sk_DAB@W}2eH36)-vBWaF5_X%E}&(Xgd{mOMrI*y%TP{$}x`M-E%Uxow{@Y(N@@X%TK6fVt>U&BEALv@mJevH`K`cfRzFbk<(ID~7H9;!;Au^blo7n7N zKTKw3^}NB|yjs%9>iL7vU|tcJs{v|Rv!IPwhnFlUwugXT=qE533pY z_=G1b=D6!{tlogOU{PKq_3bA#x%GK}^gwp%JN-yIJEWR%rNle^c4xig(OH^y zgRZ;o9^V?}5!6w>LaBJNZrZe3b%{Kj`^7EX5}f>Vd#5Q_JYsD_)t&bE33?JxY!N<$h$Zy`f5pkH zv8j#Hf49m|F*7%D`k!;X-mkfyZ?f`_x&ED)=({BhAo>_4Gy~SJRYu*p8FRVOuT_S3 z(rG2@xTGf||3hw=%O^}79mB7Ptv4&NJUdG)92Xa}jVEBAPMiGUYcL$x$yBkrJ% z-F2^)F_(?T;jm#nln3lGi4Ud_Ws-!)g$H`k=ipiFJs)O(Sw2BuH?8c;!Yc@}U{RO^6vEif>w=r-;JSb57=#lUZX+8mqb=x&MY z)FL0q?j!iu{v7GgP?dEZDaa&_8b$O=_~-Qv=i1W*jw;`$$Fw#4?}?u6+6AWFAW$T5 z$=6#XXi%c#CHCIe_W8AMy+C!!rsUx@%}b`}2Z2!pO9PfG34t)N zC)+ueD7`$E^OfPSzJ7wjNiVI-?a$?X5WYEsb_Qlw*Oqc(im zE!#J~6xhy@(d767vR%{(Lw>{{kfxe#*SHR|d4-ltZELa~ExoQY_d z+m*=7q`oznH*ZI52Rdz})=~Bk-v#wr*@ai>Rc2OqO?tUPU>-!8`2Um#7Gf5w%7^-C zF1V1){D3Ekh2}$5k9U<~zu@aZ_3^H74JVd`UeWytmr{itW}!*$w2}g%IFG0Cx>R6u zResj|(vey)acmY1WOl3;!kXn62r;#8nInO44jO%ym#!Y6-TZ79zADtr{_URJW}Uka zH;knBedg1d%h>H1VpE6xHmN<7({PwT#FpnWJ>HqHx z{a>E-Uj)#vHGP1-So+8=mY-?Aux=jlPe^29q<#>pQ!mNL*sB1d_@Xfc0(YgVrVJ$; zqAnt&ay{|r@ChOWa|}a(^v4X=p+$fKVh$-RHqp4M*Wb4-er>GOm3Fcw7owysJd)01 zYN80UyxA^u0a(tarVPyxk^ZN?^HfczU%%G7uHI@Kbew3jOZ!CNZ;1W2FTkaw z4ARAa=XuJ1J=sIlbzerZUm=Vdd!6PAp#|qV-VT!6Lgbz_v7)it{pu$1(C_LQF(sgP z@V`qkhyqO%?3~Q4&E0{<|0>ZhQ~y;!PgXzm{o&~we+Q@fLW}A6WiA0HSSt1s+MN~C z@iW(y!9oCMEB$MM_S-7GUycR9_?eK3kbjO4<_*n5s`K*<>Y7_}LSR?yl(RStBwv(_ zjRcOz+@yXRGxgd`7xZJ~EDntkpZ8m`Qbo46YX&8O4X!;~}J9I<>f?^3!M zcGh-B;B%iH}(dq9V(>xeYU$g6I$bNB#XDsLK7Kb;mx_k&rr&u(!Cpec^v42OX0m}Kf&;OAQ? zAsp}p>WS30JO5k^%YA*Is?h7CdH3M6gmS4}b9oRcLBC?DUu-my{u4~rN-~nATGeMU zxLNC0^%a=zQ+%i05h<~9aJVEDVQU7<%ni^C`}!={*gPS9-E2G6;wc$II}*1wFLWs3 zs|t?OZ5?4!nmM)Le4`DazqTHgJvMW)n`E!9f)Vn!Frc^syF;<<FADBX0W+?KOh7D63 z_-w_8EGZR|EGRue_M-=<4PiiCoY_cH-rhRK^a6fVwl}tGbE_=^Ap~L2!q@~Ah%<%p zNqYM?&C2@bKHvTovn%iZwzn?=uyb;D06PBLWb@~1=xuJ4*=fSga@J@W?Qt2zkXIqG zSd4^qPrH7g2&}{(H04?DSX+WRCPcx>Q+D2^T!>FVl9oM)%sD&Mt;H zz;3G30|S#HntUN*;FCq3Gg8@IaDJBFXIFU!ZY4P7b#x&OVOnsDV^QSheNSw-lcn}n zjn2pv$I0b}2Tx?|WgM3=zi@>&lKuvlhvHFDi#f;MrQFU;r9lHP!p#h?!Uw9cT=bWj zl@gI)KlF?NG>EWADbq|%B=v?x3_(^HBmnLe%%?t5*rgn|d(yfsV`P&D26&Gk?KjMV zD>sE$a#!Fs`KH1zR#+t2$Z%$?4RSN;=yErl@=o$t%;+t8($i_2Niu2t{2-tRIsm|v zszc>@L*_gEV6jS5_^#IMy`ZYbjz@sPa`^t#Mr|akEQG?v(_HO?^x$+;2!-kb^|(A1 zfnqrE$4@s>X5UEgt1(#*gxlWJV%4C}ZhT722Jd|Wm->{MG2Oc=Au;^pam0ISpGFn3 z^;^^S*ov9R?e^qo+%V!1OkkqO+KVigo-;Tw7?YVbb}zL{yQtIPkUK&VnCg*n>SV=; zHYyOp8PaFaQ#rOumq~PmzuA|>**RtU!z-_miEi90OxUDRo+9tFZ-f427#iYVA%7iRHXPMoOhe|7JubVX8QIA!|Mqvsg0jx7IOi zEkCHV`<5k1!~;*p1$f00IL;W8L8QE$7|Op}gYia*wSE8H_F2kN)Yiz^8ff!-mNnU! zBDM&5bd98`N+<^P7gKQwEn=AD9mUY2|7rA;{lnMEWD?3 zV~wpV#${gL6OrdS&55vD!X1{%722g&+29TR7+>Gb0$5sUT(?cXfvJ40Qi)IEHSYSz z`5QSW#ph`W)tMmn(A^Wg@tq&;_8gcJ`XM_m@P0jz4*Mpt@>G74vp|$3a zczq#-kZSs27OIo#>hwe!e+3e`J_o^GHz6geC<-V}dFboC_`HD8F)EoMW1vWXK!50` zKwE8Lp5cj5>Kn4Bl=rbC?o2DY<~{7I(ERpae9a zbTrm8X5WwtWUtjbr#WTP?bJg7s?IXH29vS4Xf9!FhpS=kP;bAUE}${#@e@s2`X|QW zPmOkKdZH)nV!fQ>Q0!T4Xs^6Q6A z4$elegVUdc9;;_N$%q!XI$1e}vQY8;7AvgMCQpn6Mb?g~TW%aeSv4xq{iHyBXSkM! zC%L=hEeyoQTl?WiVNR`Xq^gqC98R}L1%i$~{=2h_0}e1;XdJ5S!|aXR5S;9`q4!!; z$ti)AeXYFm(DkOm!CPF}c$eY`X(G!aU43r1{`tIoa{hD^AF|DFg=pXjE!br9$;zO{ zSEq(oI*1YFK%~|M*%|ggE|xQSP9tF{8*8fi6&cg@RAI!rnqa9-(NyjQ&GWG{d*noD z9ZKZG)>j0t!x30^p{#=~ANd!f9WN5X^Fy6T@L9MF#f@8_)noA2-=A#+8*;MhR6Vr9oejQbt@cf6Rja3f(*c%$-F_O>& z`UcNS1a&*`{#9o7<^u|KIEN(5tc91aGY_jx+8B9Vc$!y3AYie^vIg9?i|q>1cfkBRKP!T^ocO8 z^>n$y8$L^@1b2?Tso613E zzu=G-0_|Ssh#P-%kCU(tb{`_y1#Xl+Ohb>n^jeY+0#XZkR{^ z;n%JfqDT!52%vf3$L%|cb1INW>3oqEQo2|4ypBXsn3(=xLF_O4(=+WNozRp*z?&cw z%=JswJ190xXF>#4m}T&@kN&}9p{LuAg1y+;cdF*`)~FVpt}~!_uJqdc^dQR1!AoA1 z0cJk(x^mA%TG~Q!g$pubD~R#c3Y|?Ka!hMbN_FTMo>aOtVDsrzk}AEQ6~V<3-*j-7 z?`}o}lMmC25L}nSDpMoIn=>ty%#+^8NK7d;qDfo3*r*u=CgZ6bWhNIEER36sw_;{? zZhYQENU^xfRlMjVi6d+(I4)uG(T@Jel1V{xPl5ttKvq+&Tc$eWh&JQ7*72w`+$eO7 z!MLFjXTXH9!e7>`jk!5|yMo5G#T18+=6!0ozS)N(d6Eg$0{R)~*!?R1P9Hlxt5ez1 z3<%N+Y5UBRXmAj^_d}1~hCt3&+@`?;wg{34{O3=U`a-cmt83nU`3$?BG%S@D3*zC8 zd{gQC+#_Hdnz$J=YZNwbwku94ylpl|Cb;N~vY76|M*0Lk4GTL3=7;;FRMpwrHR_%) zv182w7NM{@jUJt?*xP=~5$L)wH~-=&{a2{4|2>#!IG8*A>j({7yk$ZQ7_H|vce2VK zWelAVBi+E2rFNPQfSYA%lO;7L8KAngp!WnLEMX@e;cZ|9C_qDr1+(9K7 zA=@}aj!lDYaN81Jc=v0_f3AtT6X@|G{ymEW8JvjtUj`4QY@C1&HUO)C^%oluBT~RB z#8;Snq$_wjMa+7Lwl`||Wtw(xDe(h{Sj0aG?oYSuzHz5gEer)QlMY9+PI#<7G<$o2 zyNXprA=Hzx*p8_7ZAp%FljJ{7}k;@*T+ z9m+`dKVus)gNUx7^~e={9K-023tfAVk_xmQ1q?GxI{sRmicaCQh1bI9zmm7&|3C7U zdgX4wYqJqHxB2z>+rJ3>S5P}}HP*X(X6{qz@&0|a_Y_;X0X-Tr?Z+mPGJl$~OYfBT z%UZQ8&J@UhtcU|#KhWF;bxW}~`%ByaIKp8xPLk>`K8Jw^U5cPqzP)KuhNlr6t| zSdB^dH975FB&7$hH1t)gaL5)x_Qp-09G*~@*kqTItAG3}g^TLu;R?%K>=ciA=50}h z8-WT4pQZ4xU*0#HA=^;%!A+&fbR|c8^5a5f{@Dvrtvov6cA7V~I6)>W{@5O+LTrlQ zRJDC;cclD-bG)4qQ>^sO z`y-UV=bF&#KL7oM6bE4*w3x_=Czx4=;`2)NW%q^l>LF-e}(l$1s;DMFTvmt<86q zxHA<-)rn&wT?-vwO0Rkq{)(d?LPaqX|6R`icO3mSmGTFC_&A;bK|sF0zLr-Xsh3x* z9?!6qo1Jd{3i=auw^U8sm~+=@nVRm)lHBHe@sSi#@+1>V8n*RF{m zRTvK^OcF1B9EqM-7P0l$c@FWo68AqBl;h0C?u|j5d}4ZqjizP+e#zHh+S)ToXNlkc}FAQ7He|Ya7!pGoK204ZFeKNsIstp6mI2pSM?)UV#q? zef&9h{G+;!)xIc#m5K$kW)rw+)w(+d2qPPKQ60bqtrGk9M4%g1g*6fL`02nW*mJXgwmh3}blu1ASdF0Zk zro~!)m-k{E29mZi8jw8j!P52C8z^16s%VfC)%sLDkvJ2GHT@ekFhd~pABD3Qkwwl6gEt}H>#u8kf5t~ z8q44w-}4WkR~w&lz55&w`Npr%BGHCyn>M_Vp{Mh*c38f?X>Lji1_qYzk@2b#gfbax z*X*1VseZ}E+g#>nxlqr_TP!`9GEb#!wPhi5Y4h4hvX<0f?edl=YIHY%4+xyymQO7; z`%KD+jVHOEw{m;Z92LFc_A|GW`oVo^Js{m|_AUc@GR4cI^`R4U&@`rpLFy$P*1s#mRWcCX;{j{njEs3u?b&^$Fr{Pb$hlCuawn{~SWk z&L^UxSpjPl)7nNMB>H9`E1P0S_iI@l%iQ#1YF_VuhtR*K+^z{QX0-XwV4!Yvjr`-u zyTtU$4goL7s)+Fa5k%jzZ91@@c6Rx<2k!#^eDIE9^O7;@W6Cj$EY=x!oH%Q}MPNo> z)fQ`@7KdeSQ~*q%1L4Y9(+Q?hXI?-_IJGQ`mD!}TvjUaA-6WTthc1o6ZTkTsVSW7$ zKMKCJD^dM9W=1d(tHntZC6=@aQIZvWuvG%1pB7xAm2NLy*L{a7Z@GQ|-;FsA$Hpm_ zGlQQ|;QZ&^o6Gka+5_)zNa`sBNP5IfMIG$G%D+oa=fBYfYVzm@Ezn(h0I3>`HS8^- zy9q9adcJ5V(To1i{me$zTK%0fg1@nNZ-O1@Lqj>xJZ8 z?TncL%*COko6@{ez@{ED{ISDeak!IkcdpsQ&C=c#3kJ(kS~b+Yo;Eit{+XtDyc%SR zR&WgVA*D17YAEz7$Sd&v4odj)=@VTU6X(t{m);TTIz?MJJXXb+M)(TUyyayHOOiyw zBX32lk6#p~F^e(nRiweyHmasR*K8l;mT3!mhn38<^Gr@MFy)c>TSHWu*3`xoJVnJbL(y|C3H)+gtoG~2< zS7#3(Lm!;ih*|;CW%GhSyx{M5!piMQ$=sfj4A ze1Y0L-g3)wxn>2{*uN0qz^@8-!d@^-uG@B3ndJtgqg?ihh-`xb)O?EQ0+nEK!Rn$C z1gfB(v)shl3B*MyreUctgE+6q?{S14hw;P~T=t}i83#fxJ36h^@qXLU!8^)jw!H%b zb9@g5CiwqdA8}_JqhHlg{qN&~qQuECS$Eu#m^Zia|CfNJ6J}R6ss%$B7783X~t*gKkc2!4iFLsX7ZN0$$&o=wV zfIx9_F617XI?1ntnh43+YDI~9%SA;z@Rrs`?plrYWk8dmz|Ja}f!nrgNHh{E|V$t-X@UcWIutc;wSuv z=<4&5Y9hm^LYl5J?J2AU=I3GPF89E>7}tJ9{2-Azs$A4mtel3Q3D%CD6{7F8S@`8@ ze8QkLNyu|~Y|wPZ$pNWhebwpNM&{1_%U7ebPw9CQcT&2a?9KUfUiGa9Csn_(;L56}Q-t};|_He%ikSXlK6VJmD< zOAGbYv(@)kX%(A%k3F9ouj~j%xi`o5+BG#(mm&9SI4&92hkh|k7^4_ntMWGm@&*Y7 zA`IOYEHRXj2Z~7R*R;a<0BaSLV`D%qpBRV}K({f_*gg@P|G3}xOxt4CA+?3eHs~F4qkIpQ0 zv;zji5=;>?-=6eL=E@;v{bIHt`d+?-VcN%M2duf`DqFF<1jNPE0%bGV)htN6fvuTM zzEc3@`R&prUq-eH*CKuc28?HRMy}*E12dc&Sbw5~ak&pV^gmg3SmjnQ-4@dnt;a)X zKolW-CV>rHJ16XjNdQjsG+A(Y1LhNJc1G}c!d&_FlXCn?s$}N<06BozPox5vv z_v0-F7JGulPEF8k*R5er`du+kl$73$#2|C_3#ThzsIE@oX<(L``p^y`+bOfb#Z|(b zM8HxXxBTFn$Y}(6Ek=DtC=O&QXw~+?W3HaS?R)0rzcxc`aGSVgmDby1!EUstjNn15 z08CCw7#LolF3%GRK@<8j8MoAyGS z5GhNRpsSaF=lNMpJWU#ydIA{Y{glU@^0S&}dYm`XW@7Is^m3{1O&}B}#8RIiczDF= zcr_$%w_Rr3ewdT=jh#C|Zt*IUcXccC3j(SO6IUc{$JljvQqBo~|0Q1_Ma!}gr&J$a z;1`wM9&`7I%xp?w3`{2BH3`5(_D7JLY-|tL33QV0`>)7_A;HL$r9xqxa^Z&EFcv7J z@=7ds_}z4SV*L-FbaU~^vFh5hrlnXJZu?^q6$BcK^vS#;y<=fBKoDeG z&u>r`?QHjbNxW#Nslm~WrM`L_>-rRPhCaZo6mv+ST~E#ti<9A?a@5jDf_LG z4sgK)3hGrSmu)ci^hr6$KaUcI4rt*DdD~nXGwKlfEMiIsy@FW|tiy-3%y+I5EP|)^ zMLzj-u!at>F0#R=LsidEmhQg^T}6@htAH8JX8^BsyHX+I-So(*eSQRYex&Q2i>PfJ z@(vjl-bPb(d%+#CkGV)YqMEQ`i0UNQeFQ=n6r_er$-Ws-fr`IP`RSr3D{$s(wGA;+ ziQA-$@|gJDb=luC3C#rc;jN?R2#a;bkqm`iH%h3T)^}Vq$77$rg5d)PG*0^~2=rb7 z!S;8hKj2>(9vRrTOh}@oPqKI~u7O%#;lMW#&^qI@D8CxzvUH20pnpq&LtM-54*SxS z7WgbEGF`fJ@C?Q%rk^DoHFK2DSAABd>mEfJk3GC13`!;6h(lU)A}p@ zb-Uux;j=0LmcgX5i}cZx(@eiwls%hse^ApNFE?UmG~MMs9F9=dx~;pXG;Q{y1udoe zF0x7f9rFS<*Y1tpfLM7Ha>fF(mnPZV3cDF-tt(Tw6Ze4A-d2#w)o4_W7@jvX-2fX;F-olWjESKJWcE)apvu!6kHX<90?pNfb!^x zi9J|)N1=z42#a!jFVzr5g5Syw<+R(mUqYa%gju>?iG{`sQtGa=5h)g1sQnh_TL>v0-}!_h zrbv?UV`V-m_42gp1T3EuG4R}sEIA%Go0@t*JwJng=We)=BrZ3Q6oc%7&kgE@5^01V zAJ=bqKitsA_-**z%_pphdfVa5aUMDiPv$GnvSa1kfA1> z+D4hq_Ey3d_-2#;R=o*MwPs1g$<-_8H zrO7N3f$i*qYTXzko=qKHETErff1Fu8cwA+2fJJ1MHz+jX&|R&EO_#EWu>}IZvOBcF zcp3qnrD-6`&LtAguK+mA`;)P9Eg@u4(?C-|p%%D!@|f{VGe|3RpYm;M1-UCOPPfqo z0RXqcUdEk2EjXC2Y#8RCj4x8r%L6*~3}WozG}c+nj>#ueiz(K!i9%?-*4dy|ZDK6- zZUk~)jV+vo$nD>&&pvn)+O@!@#=|`^ps>3osA(fJTf+1>#5ju5;!l=B`FQ83%F9aY z#k6KKh`oJp*RF|UR%eBZF@|c+yk*&NQa8*;aO-H&s|I7LcoV>wo|U~v;|i)e7m#N% zewZ=HumOauIN*rgi3{HN)u3Te7r@EdD}7YbW^qdBL`jdQdXPLoDtBDqV6?u27*x>Z z7Z=ij0nzp)O_)S5J&~``PrAg6eOD3Y6M6H{{e)NR`X>BSeH7e~=9@26QzWFtBbbQr6Z0D1q@yD|qWROAwPMHHb56OaqDTV$hAqZe$p%E_L}DK58Uvv&z8bJ@_JW;W7Gm8FgW16%`khb^jHeXVD)IJW zIQ+hm;>c4jtAA|^j(lmg6?L<>4|h*1%* z(0deQSYm$F^L8$+i4Tj*EglhTh&5jnV5MaU@4@eLySWNGr6+?v)A8`EGCmA*bfvxU z^MRkDq_UwY!Yjfmi3f}=VP=Zv1tg1~^gS@kL&s|j7HuJ9&&}gZ+ksDm)gy5rqa<8h zvMRYJ9K#?`3KR?uWV3atT7pbW?c(bj*$7K=eDlGEk%r8W-Dl)TJ*dJHK|LPh*^1&a zC!{ZT23F3O>b*0;qFS4at-*|_>l9v)hc&wenQ-+#Wgs=ltr*n%Tdsjppao~SaOnK@ zbV<;JAU2mQV()|(lF^6Sb9M}Ovb4cdI!)E?VW=w?Y6zqq+{Do zR&3jL(y=?X?T&4B*s*Qfwv&!+`*y$Q?CHc8Rddy<=b!n^Ip!Q={Gex_ zZg`48U}mcHpRQ1JtM{mKDv=z?E5LLHsXIrTLeXqY{CM@Gkh-0v(q%73leh; zs3f4q+LzA%UZ6094Tr@IL4X4?m@sVy)PeqI)K#)(T)%g7L#VPR@_u8Ifm;>BTX&~ zx-wi;$4D_2OIm9OP4%=()4)6e?}~jA=SLGA>bM;ge_u7GTnrYCTUcL96-i+mAJ)*? zOIaW`e0}N$17gdg^iwSVTs2&K>v@@pJQRHq$*i9tuL<;rMUR4ggq83@56?*8J#i}$ zfhAExJ`wXMeEzmub`6=|H2$-?>+jw2`_%ZK>aNzOLehc?%72M}jFR#Rxb>u_vMl5; zizk?8CiWN?UJL|w<&yC|LHr-4kf{s1f|g7u0{3ACy=mvAw#$E=72s@R!-SSZrX(?6 zgzvw1ZC4JdYei8Tf|)$lbuH#7l003iS<7 zuyGORXsZfA2#pMrvfQ>#41^$*ii}3_O6nkNU*W@RB|an?F&E`!yhQhz^)+^Y$wt1|#`SS4U*R`Vq{;rNyZS9l zvm_7ysk`=uoiP1(-Ia`d>&bt#h@%G*&EQIyCN_3Y$uBTpP922W2|K-FOq=DWDM`Om=Gfba+^i@-@b&k3QCc@8hm^?iG(;tD9zS!bb@ zTL`!e!41K_0gdLsApfiaA-N-{9OI`l8O+V4(rigd^@rSq(z1_iItbccL+inu%|oi~ z@qX@YJ`|}N08|3Ir!Z~bgv>z6v_(1`>7^)_$xTsNG(0-4lTJpTd+!{U)RcQU+!#Aq zM@3=@5_29UuaJ`5Zo|=MUEMw{68>aLB3sAq9i)8w9Owf?;Ak+uy%0BY_+woM5j$AI zG;K48iAXg0q*N=(nIK;}nzIRv+POfKyGL%tX_jxA2V@j&=8sg{aGsS(TMeG!knmrf z3gPK+Bk_Uc@E3>|pVB;@S4v2~^GyF|PeJuJOLp8RP}@zYYGZuMon*WV74!o$ES2Y}RXeZ-gT%W=^dt&{YM6b9l1b(qfuzw3#iG|dS62w(1G>0oRV z6)*gK1wmxQD3AYV6t^GXuA=Us|2j=h?xQJVzo*9@_&=bJ{FQ^+Iy?RkM5FgOjntKZ1~~{Mvs} z_=>>4pi)>LLx56BDtZIi1A&Gp$5Dyy3naY`>VT1$Vz}S|KG2eisqBA2T;(AeLO_;M zAqga#nhx$hZblBNEn%bpE!KSX56FmUkNFZFc$33{BuJyozI=*1%0PtLM1n@j9jM)~ z_`m=@B|0)h{S@eI-Rj2|KPsi)8}Lg)gerLv{Gonn#{Nn+cr_F-JF1GUnJJMVA$--F z)3xCNnzy2+AAY6eWT0VwtkCaTLW_YC*f1kvri3+0bf&5&q-}|E1NSbCOr{kV&;Qa9 z&7wi7&sAm(gHmv9&6JHRl3aa_dFm_PD0m^a0#)F>!JcC!R?>U}vRHKt z&9LQKkZ<1Kt7)#&PBJg9HktjTTNZ&3IB{s{d>vN(lK2R<`$8C$F)3L#wU6K4Lxx6nysT zK&hFwzf~kq@QE}B{h6E1b$heB#G{?W&CJ>;YR z2L{InmGDvtbON@aK#s<56gJhwk zk`VTGRVur^I@+K^!HXlec%s)x%SU&`V z#h2J$c{NYu1uA$`Bud~TN%^=c3{lQ&&+}#ki>Hpw&4~;X^OCW^`es7+y=vYoQ9@Ds zJT;wYKI)B%2H1(G+y5lj9?1A%MxDx#1Blrbuz08XBdya;r-93lZULBY4n{xFuIJ4d z;MrbLM800j%t`J+!8h+i&QV!kkx9j7WKJWKPR*7nxC~-Wt>34`SK7?ptUtzW1@K&% zuGVl9CNABFqNL?0v}a$ywz!W3&oZ4Wh|9pIB4N=DJJgb%!P(%pZ201DW9yrbCma`f zZe30)qvv#%D4|>z5#O%AOpIGBDudbzQ`+2!^0?RsOmn*n2JY`lIn)SU$$LaCp60Mr zu827cs8`ozvX*)KV^w||8Fc&$%CvTJ?(fm``0HYKT%2X=@W-L`AAKE+L~{b!mvES5 zEgtoN2zBtAE}8VR=09pFv4%fqU%wx;={YP!=G(UX0l*D&G{+B>EwYvC42|lF|IybW zvr)0+|J&~3SHfN30|Nr0MFj$4__tQje@E8TXIB&z)S)hf)pj!FVq+vQQB*WUs9~~N zmjMHcWjI&w%yu@Ak#r7b;Bc6W*Urn=&Rq|9{wIhharZa5c*(EWaS-R6x9*Rf$F3$9 zZI6@bAMaBoT|id-1eD{wR$<95^~6Ne_7ZrC$L&lB$M3r#^(fGQ#4Sp@`W9_Y<{Yp3DOuzmN#6GvpvfC>tjWp4ZF`&PP{0P^>=O>v<&pS*u3ggk(Rz9 zQHRIJ<8INTF_c?TXQg$d<1KG_tUv0y`q}*EtJQAsHZH}G@!%XS^M;Vj2pGg(dEL=K zxnhq{mz^G7sKNbprzp|t^%=f83mfEgi@{{#PvOq^6npy_S<}?0e@ma(fQ4iF>5M?| z^Fg)rW4)wLstjZ@rxriP1qPHn>w#y&QX$#>Q7pHDH;rQ+Z@JXa+TpA8D(%} zYS6}IkzvprrfB>hR3|s9OR`~>pf1ZiF;u~%dJs|97a7Ni>efYTrnvN{X(Q&X%$dkb zP-peic(W+0OJ1vD+t0G?$fuxIom6HfXM{b-ncN#hH9d0ln$c+@`Ru>PE0t0q zEKB{$i##gqfJW%E+Ub>|zKoQptP;jRn6wtsMwqjPX?zo6lca231zvKw37aL6Q@ZLi z-^tiz(&|H=O7#oBhA~>+xq6YN?hD6e)t0Kqlt?A0jow0^Fut}Gy_ibF+*26!{eZm4QLSfinPS2G>PSQ*0V$e;gTqm8R z){P$@!u8S@n$+?r?W=TRqCQez%eu=R23qFukfS%?O*jta0?|r-zavKdrIHoBM1Ga? zd1v9?hPL#XRvCx-%J`!x{6lBsi-0@~Xj*Q#4X-dV0j+sOblcuKiqHyODPPpX$cNfpGRv9gwU81${ zZ@*+msKKzNsFSI4XNaytb@Dy#`#%hh(ARg2bt;gk`%eb zap3(1K3Pex6LrpOs+g`z8`RhVSq)_!j7jt=?8_so>N{XP-NXs48v<0ubCMyEFV{qK z#$=pdHStVPe?KLjtmkJ^stL#S0$r|!oV{u+uEw{8!BLul*HF`s;ZlZ-W<--vii51T zwqRCZKO}V?RoL(9Vn11(smdj;#gq6nbMflZt&EDC^%pBeXrWJy2UpSBh@;ZGR7KFKJKZPEWQHpcc0jZ#mpxl%_4O zD{li}kJB2)8O9m0-h+1_Z%IER_8iu&&0ZIVkM$gZZHZZz$0rOK)^|w6C9x&7CAlQL z1YD9`5}|R*nMlgY#dF>ux~I~_2Ghhp#sOo+i$8gZk5JxlJOe!wy8C;jdL?>Adifvn zOY+I`3G<2b$#)JvrwpqNtBouj`v!PLJgn`t?RoA&?1k(N?UC=9?j7!>?5XXs?=|f` z?jh{S?@jH|?^W$xhVA_g)vngT(Ubi<2Jip=a{N1L9lvk8M&|#-lZldr?qxs?Jba#8 zrrIIWZe}9f{7KyGunjf)R7}Z8Df!&fA*%6yCdeZgOV z_+RCW|JqX#va&U_{A>8%R~8j5TO3i;k3pCAPMIBm^;~@-subllslC=};11%?0cCl#E6oYp$C5PE-{HMjb^uGu@QYEKIPw=K_oDL6@PlUrmQA_)*t#*WvM^FjblI z$2bQWdt$Zi6HzLe$|qyKu>WH0`Y{M7q>*vN?bQ$7SNy{7y>fpht~fuEd{qFAg^sW{ zcbS(9HExGHKqKto>2}p5QWig8->$z6I+UK8r+wa5fV`Xq4MovR>53UW?pm&sgT23| zg-{?{v$^!_8L?}fVUX(JZ&O$`T!l+|3ZmLD?W)|To;lctK55q;;?A@Va=}4XiPHJj zdj3>j6EurFh5o=>N#zN3y`jK+!=M0Mgk@rXPWQasIj-9g>gSjFbzM2(kr zIkk{R?caU(A&y)2l)CLE?<-3o&)I0#ps&I*@^rV**I>?1XyfuozIjw zSF~nt%I}(yJg%3^Sc~FBChLWOzwa%WL2hCf5@E`bft>66f>{;R>m$;E-&3$&>;_u| zPsea<`^aVR?7Nr@UVp%n_;WlV<;PN4{a1C;0+|WR*T(jPROR#AhGIC(C%A%qTba9J z)0eF{WsK+pr}%Jq1=;&ukv2xa(fw;6AbYkXytyTt4mI0vX6XfZMd;dduam?aHpM)X z5bu|(-J2g}dpCBb372ZKf7$mH^UFxSJ0_9NE&yHytCm|Z$-p58dY@q=Y1TBO+WNo< zxf8~v@Kb`@dl2r&g>dRQ(fAH_Wy0Z9B-+?aGB~UMORxykiw@OCsOv@2U9OWS1KCcXSjd4*8|aAv@b`^O#{=8qLofK!*r;)tzTJN+i+LPIGsBuzF6w zJV|39&QpflQDz|~NvUIVnDb4^$ZXe~^d4;@Q@%)@6m;5+djc(Cf%*APKv9Pmc|(RV z9OmVb$hW3Qxg;Z&KE)#tA64*o8Tf0IKI=LkKkh;o#yL{gFpDrBONr0FK~u6j174?p z=P9-@KtO!|?nalB`+|EHZ^pt`1nqk{RFiKq^#g_!oGLY1k2^z=kZcmMFEx zQV|H=qq&i}1^?||1ZRlLcbuQDQammGX--AK$W{Nv>c}7Rn1TXwr-FE2p zs^hbDpTYn6`Tzm6`;-sDR}Xm=VJ3gj>B#HpQN3QghF(sw_sDAA02wi&kfC@g!HIFW#tyFx;5nJDK=cKZwoE;v0xEoXe zEU*$$$ICib)9fx+siX%{@;qS7K|MsIJu(3n&vGKbOO}e1i!pNUix9CGVqwPoTY>Ue zunh{?X=&}G9YERm(8ESFj|;aVl=*#`kAXF(z?{6C(Z+`3Yvpt zhc$9(uU|z*y-j9cPJ=b6KBs6bZIEGo+=nFW^{zG4QQKBpum-HX$L>LZbKTkhH5rjH zjGYoa-y&sF!OMZD5LTmBMOdEY`c(j`s{5yI=q@di?JiG%0LOYg0eln-Aj^?VRJ*@^ z>_>8VKUVqnUANt$tJ}g9Lk2vnnFXpk?ug3{bS1PzShK1c{k#~e4Q2LW!4iHqTs!qQ zRS|GU*j;%CE#}At2TGT|wW5r3+1a91WxIE_XHHW9>Bh8*#%%LirEn4UHQPNyggfwM zn0hy{$`1Z&evY{^bt35|s&R+e-U2E4Vl+WO&dZEqQ&Ht;F__LyhwOF*1T$6^aqQH*hw|R#{PpKENe^-;xG@DoPTi6sUg-ufCb5;oBzbPc=MSPF8Q9FMVznBYLTF9Hc7*#6T-J1auWXscgl9Ibx42)NV)Gi@<+tlb!h*PJi*Ddj$r z9G(6m^MYzP4za+d1B#5M1NP6cMGfjX>%`#^GDO&T_IC%(HYh=_tcX6LC~mLGYF1A3Rye-1k8nq_>!DZJ*MxE+Fjs*Wi9;)715n(=*t%jQYK z<4f7J1iNc!rPrr{ra*>iOX#hxiDCV&P(|R;lQ(%ku4KCK+gQzBVa`I`UZEg)nxNbI za>jLHy(8&bvu@SlB^B>^7Je}Dg&Se%e!yC@2K}cziaSP#WXyiKRpRlQG_~%D<(_Ru zCqdrs-;i56zg`diL+^lp@EQN__MwTVhNhY(v_D2a&aXy_vhRwZlCXn;`6Xn8F=TUG zlmTNh8xc^SQYcaV5W98!qi!ayhSLAHcYu0d9uv);)gB(If>AO4P0adj%ZW$TYDg{$ zBPx4A{0zGAWW{9)V{m*_Z2*iNCRzL)N4!rmK2s#bjxY_#%xo#zD_>vk8m`Gkaf}e# zAQqLk+aGx$Av6>=3wScdfh%@zir!~D zG%;=G{!M#%296tLju~9!RSH9FW;QzR--j!#<>%L%1|NdQ_9cyVnO=4lU@Sc5aS@TO zqCe_FF3q;{ns1?~80$zQJw#_>e#$!7y)=C%&#m)4UR`>@vz2F5_yfRUvT&BxY>jM? ze0jm%!EcQ|;we;rlm_`MBEH2mc?(&{%JQDw3Z!v>+|KyrvsSTxI$|gf-I?n$*Cu+J zGhav1U|U0|(!qqscbI0{XfRaV0@GhYGB**4@Q zLLy@!t-5tjb9}1DpS~wsl`}OTZtyNF7WLA4kfUQ(&Zsfl2Aw&3Y@;8&zd0UjLb_3&?Rhz`F;ZGuXdyZi-z;MXtBiazW*TsXY0 zw;+0R(L2Gy8!jW4a}AaxFX@j&g+1+&44DrIk|885L0RlZ&I~ppCBb3_kT{L`cju#}g4aQI+Btp`bwpW9E|oqPt2%3S zp@pn@dy>kNP?lh;zO&`vEmx=nf}k_AHq=`Df^epV@K1x~`Ol2-HE4xgrh4U?

b+Sv)Q;TVdnVXJ@YV1Ijt*&tWUcMhBG))7*U9pHUg)=J#vOTGre-Y^5kkXi=H zM>9iCR)~vDo{sCMtqpNa$agLFN+`!V%Hhl8K}uNc6ndrs3D-yJZj`}}&F{+x9jJ)CD>`Q=h$4t_Ix;7OZkMb{LB*wfi#0AmC1mlkZvsIMN`?W7m z@(Y9U7idSII>8RoIcMvbLOMRlE6%^h@xppExIgOU=>I;ySk%qV;rCy^$N!^#`QP}( z1kSQ;W}x0$NcmKdInYvDMfOLNTk3;7!%7=Q z4RNsK4w5}(&d|zN4{JNPYgRS@Du;_{3Xp?WsvcdpQBp5Ipj=mklO}N#R!ucjP?u&y zT{&yE%__~mgs$&RHYaL}^%Riwm53!fbzJPmZ`9cZ9jb*$8J#mj_ zU9`kYD~$^JuW%|};yC@QU^b~1+1Uo&)I#CPJ(Vt1$WQE z5?SCUF&@5*fsZ+-J|MrCRliX;Yudj3%_o7%D}+SlKLZX4ng4$oC0dXD1suBi(o8!_ zmGT6%Ef^UlsU#cDlO}VT4@<}F?6Q>qPe=a)vgZE)8iIHEC(ux$?_TAQ%9Qs4C)&pM zoND?cSa{4@x%cKsVkz`dw!&}B>9aCxEd@T60r^H-tg&O?iG)|K}XL56evAs>d$Ut~?SY{`L%UuM%7 zu=|q|3~RV3E7XR-5$Kg`&3>eRR0Z8MyAWP4PFZXQw{cE%!GDL=??*9rL~5ENbdSUh z==>xDtfoQ^PC)0~5(|02>jul6njx#@@SesL4aV}p|ej%AXkc>l=)EpC=3VFr~`AFC{+5s>-|tHUMwIP zA_D4^vvtbDOdH6W!CfPP+awiYKN17)?VhSmOg?=_#Ut$0(#`Wt|MF+Rm`MHO&mdW$ zXIf`g)W7CGmhT6z>g5EIF^JWh;BI-|77&&^!;Ej)_cbT!{JcsQE~PGeJD9`2u`8K*`0Hoj}O7$U(c06JrS%o9nE_*7ORCrTG@@!!_e2 z@F9fwwmpJY8AwWjG@7lW7i_2kzp_VE`^z8apmj$qHL3mL-{I{F>HM*4|AbRAHx>#W zMG6Z+Ey8VLCAjXCa|P!pXZvic_uOOmZ?LPkGMg0?3m$#txn{I90(br6BK}m1kwK^MrfX< ze=u61U%#+4t8}Yuso>-Jbj=mWnTLWfsNOF!jK%@BK-j0>I;1S?z21?c^$&TQ>iCi= zzrv#JJJuc?s!T?}2m?j~M5j}{ON{;^S{2A~c%kY{=!`5@rb#TlE4VN3zH#}ods zokuE_S%Hl)23lCBjaLgH@CMYi4dUUu^giTNU59>G9^Px*kwwbuS8CgWiA;er1PFg3 zUzX8BY!u`*Fq&^O-J&Sc&36cJ;-_jo-9E|lQN>UUv2dYASENoNk^%9b~L zl$DxF@w*RNozyBZXL|a}T{t5V(UEn4kP>;&q_-)clB zwH7D|`njV9D^or)_>8I1P7_HPP&K*i+IDJS52>O!S0K~Pf09cJB0bdxZkiDC!O=4%6Db`NbikRrNe`B$5-k#YMd3#_9yeNrserM+2J z=gi>*dBF-47=D)|eyMw!)FJ(i^r0kbTeak1Jlk$Ivm(#>!H|o{0dgdAmq7pyM&w51v%7wC zxyz1sjOV?HGYeZ=`E$7)h>gkiO!uuz&)b8E^aYqt&j+&anNPMrW0+j9%nLS=R32h+ z6a5me#pNey@gw!7gL8esiPQ^gpRANLR?JWVm- z)bYBpp`z5&Lu?)MD5+)kKgunrI)XRT7g&NP;_$nmGHHWyfzmK@VO;UpZ{VTm1)2M9 zWaQ)?(CdJJ%T0?%S7wUTwQt3s(yM=L8Zk>bd@~7|R!GCwUsh6vydOT^&OZa4Mi+VJ zYGT4o<+o|KFbU59TE%q15Iy*A0+{61HZVKX)*Y>`kf7r+YcdoE;wC-o9m8ClV$)%d zmwNrseIeZ{M5WGBmYC2`VH8y#>T8Dc1e}RN+>6(to*heg@i9Rp1?rC`|9;z=^@5cB zQa=W++W_sj^$FCnq}Gv0GaQN(S1uzbm+{rvW%MMs?p<2Fg1EUKp)$aY7G&i_d>1xQ z;)q3&!gK_tW}*G2IPNil_&JP*BzICAgRe=FqDnMD5xSJyfrykC^N~s0xjs;j4(~)t zSPS!k;}DC#@&jKPB(!uUWNZThMm1E_;xo!Z1hS1wJ;E_T?ZUqhJ`z`&OGzz0q?cHw zp}$TC1u~6hB6VX0FZh5#)RrbIk2A=Tp!Qp0_Rbv_Kciy6OG|vOrruY);bmPlAyy=G zn0EoUPtN&}sQ@=X$iGW7Pi9q$OW2)|T*IyO67dgrgcTgCE}B`E$YIhPBr@}=$~U$D zat;CV`$k3I16pBmYt=Jr2oNz+fR{$Zljbx^13wRfuUbUWSS2wZxYy3|2BQZVW;&yy zf|04R!bHda<)ZU#8JRM)Qb;&N6FhlWkO#Hf^10AkN0+JqB*0-8cEwllP!KVOp&E_{ zoFKstDdotlCdDF+wvVLlu*7@_{b1xrIR({5%f-pw-X`I;eS6$uOrvA8Mk|Mj&3f|a z?u45(kZsZxTYIJuQxo_G_&J|q%~cZL`1E26jlkq7x?OWgnkJZsDSK?-{JJRUKi8B^O$hO%N^_$I+ z@h+ZGA=Q?5W&2FI+`Bp@=Icj$9}r0s(tar(q|@y<{Yh(ums~iz|EdXS3HEm*_u+Aj z+A>fZkF@j*(c^8eEXiG6Tia5~Eek%XRrtQ9nHDGFSkEFX4YqfucC{9?gjR&ncyvIh07jh#)V3Pl4bcHwtlX(^2@rtes`+L%-U-M zC^f)EC}}%sT-|wumUa`W(v(UQY?Q?yy;B!m`CfJiw8?zz-R7IELqlAz))-lrRr0Aw z^VDz})P6M;)dLSm;|)|j(oSn-YA-!*`Kar&dX^e(uw#KTkgxHwqRy)azS8GH8tdr*uRf~mN`O$A~5Dh8Yne&^06E>ersw9D~ z+Z*#f{(vDxIa)#}y>M_T@ZW*i8?DP5DcH0qd|Blz3{o zNeD?>uq-D9eDWycyLH@55kf97b30_}$~>-B50J8~A?qc3PZqMY(@MiWCAt9cS>Q(a z{WEywx_k98wr?nF6qu}Q*w;i}>I|zi&5ScU==8UCyYXXynilUsBZ7e;Xx?g3x1R%- zev3Q^GP@Ipelbd?C{KWQheW*mxELr|A~p%#We-`{QNJ@p!p7WKklrgwuT{lqIr)!D z;ujX1NBPb3AKKe;AFzC<%Wi|$zG&I&Tz=LyP-hS6E64VhNtPyPd?8aX11e8nI#Ul- z=~_-&@1+5+Or1;+MnwcSlkP-SaWS_=C76@tAD zQbD=q*aSeEpDFI}Eo#;NUHQF$jb(bbKK~i-7d8t77k$g?auBTuN()*PTDhO&!ar4) zwGR-VxJEh|izceB0tl!_4$>Zv@Q zKn(*@PElWe2v{H=lKaYS0S>e?G#fLM`$`DQPE2KBHc~@=I1DGHABW8E#3Ces!ntVt8nt3*zIW4A3Xjcz4 z+K6#zH9Mik!%2 zA&kAHKV5rA7XBJ{gDj-7FW#sNko>?oE$sO8#;QnPq?(RWF{!dg0B4xhXBjnS7A0^< zK3X+wqjf8})BVgMrqxkuI9IB7$?uGRC*qq|);h!5sQ8^G@1g1%XQ*T?(~w32sb(8| zb8pFg#4#QWFS8NHN7}awrDAIrK?o&VF@S&8z9lHjH`g>W%eU}Q?~45`j|nv5}aouPydIqldK(0O%75^VZ{m=jCJNWk-|y%@0FzAZ@y&V1+Mwh(1^akP?tM z2MTp@s`rtlsbSj(^|{BCtO@3mk-1>j4xe5uyE`YPErA z8%24NFacTAhWH24#Uqg?i`5q9VJ;+PT2vZTqAPy4Mj0_+fl10?Ttb7UM~L2dkUK(9 zj;@rRNeATx1!?T5g-olm*}~00M1!xmSTHpbLhiE?PUaR=KZt%_NW!}`lLnGD142Ep zQVdE0pA)jzqLiifUy${FaK$%@?Us`2%Qg3q1WK&OS4V=2NX%G?$v#jQ4q5>QEw-%? zWdR29jfx|%wqIw7SXQOj$JE1A`crC9FW@uLTS+B|;b-5NwDJ3~v8D#lIGH?ME{F%h z3PzP5jaZeUusmqer6kp{8-wyuy@xp$XJh4|cifSaF(OPx8C*=~CORfN(`|oT&@yRW zd-=;Adc{v`voJ%C62DOwHKUymN@z-#khRRs-sQjsY!dEL)hl_S^ooxj=l;qi>^)Dc zE?>6ieqK|OB12h#fCKkzh>QxYHQV<|m{velK@|Q;PuM|jh}b0aHCS>BKoX1(MWp2+ zgO#JDx<-q?|Lj^3IdN8Ql^unn!JLzvBQIp`ZPiye34Ld1iMTYf91JWqm!X}j89Vva zqKY;apIIuE$tnf0Jm%4AWqng47(=?a`10s1d@$F0CQBX5B^K#kN6X;c3WvT=ZglD8 zAxpMn=3;WgFnXi({Dju;=(joZNjuTiga?@=}_IC{w2?J4eK-NBjgwhf}=R8i{|2LKf?tIo6C_lRC{rOL|P+ zadoDm3-)S?WZ#}hG%4gVI3q>N%kRyvdM4!uxqcjVgF(3@>09~u9{GC3wu(TFg}(Sj z9pr5NooUn0bv9NY%0pR5hePvh4}@uRg4Dz+@b@FJ=OB`-qIfBBo8-Ii8Tl7^8KVrd z5JtELH)FD_nX|~@qRA;k4MaLMwPY2iK|IzIQ}2!FP$_$b%5h4T4$MW}C2waYUJ!jn zE6ygH1CYnidoG>WYb5Q|Al0;sqeCD93qsjXAt*(b1U>$6zS7&{30vd#)?;{IZ&j_ec0 zcrA_<{5;*WIb!Ti`a2boT2cn=IzZyj7PKT+Upez9b#W$7k!ASf)c0yt6#9(#Q3+)?=v{WOFGEC?Y=*=9kTrmou(M_K8xiyU5 zMl16n3!k}5xl^w*NsjI8aoOMkeX9@sY ziHq;Dn-Q`RF=SQJSY#H%d7}{R+;9POPPFNL(e4G_7LwQLh zp|AMIPf)0o3E}}rQwNkpFJ#s1&Ow%wR!f?~EoxFADu$SM*KpO_<*x82PmJYfW1 zTbE7NRAF2G*DZi=b`02C2V%rk$Tkw6{zSjz4aOLn9emy=p%7%AR13|0>| z`kbP(6>8aCiRD-E7JY);L)PK~Q}!=rL~0GW@YcfTy=l>uU0>VvU`j0U=wzxuP86%u|MhD*$4_W+%82Adi%ecE`VsldqF15hIhDoZ`w0+ga{z0XFj z_(RBHIEpV~zYTu)!)u04%~1}vujG{h}Y?g8xYP)u!`U9of}Qy;cEb^sdkrq z*K@o#Nfcklesb3I4hR?Od>DkgP`k=3NEaer#{k@_y;&R3uYHR2lFsYi%FV@_6h;XWBklc4feuq6+4yj;bCj^@u zY>y%RrHaP#W`? zu=b)W9E{5*kyjo2c2ALZh^a4|m)9W%ZnnIDUw9d4n*=)T{7C0hPZ0s^GVejosijJ8 zO*8~UYog;JoF@Hz#|Em~Rq!TgA7W+5od!pI2p(uni-$iyGtE9JY|h(6_J&N1&*v%| zJ^`y;;kzB;{W$zEZl83~kFlYyXEeXwO;615Q5oek^jWf9@}BixUJ3Nx38tpol2yqS zi6#9U!lWwp5EA+ShD^**K5?W30|L@R00N@>H*nYg?O3m-0b}cM#!G3>95T#pqVTk$V;Wj=+P}gn@>Xam9sxrc2iuPw0DkqC|~#7!^W@ z$9}c{(q~knJmQceTVQQrt2RvO$4p=LK11DkQya{#k}(L0_wAH=5k%*gI(Ltz5)$^Q z7386EM#|VD%e{C2D(g@ZtgSSyxKW}aBbjSqNcwRt-(hek_BL6}7*-0XLzs4i+=v%M zlriAMxa6Yk3$@5OXdd$cVx^S>n9f1~4fx)1+PsH?7$@3N0rJMF)Uf|86&d3|#Wca} zNAzKvD(}Jm{G-Zxyb_xD3p0;3O;(a?Q9gWF#7s18#z zR!NEpnbI;qX|XhIJ@cr(4DC|sfy<_!;<(U~29qm0ATH|gXZK3<9_rDS_J@UW2bi%* za7vvab6#HNxiYbMV15X)Q`lIDNdyV0?=72s5UQ68_xQP>PBCEhA~b=a7mIQoXwFlYkF1G;C$w}#1k>x2!D-IQ?h`M=PaxL z$Y`sShlZE{owcqbB~##C${r$HI~|{4lv}Q8s0` zZ5Y3}Px&#Mw^aHz3VzgX5;Fb_Lc28_Sw~!3%qha&)NC!DE&T&1e}yORI1ku3kMQY@ zo*~Fj=mw%_N{kTjuDWFnsUCjvM&K~n0%K^j*p6!w?6V2I-7+E zS=xOAGrt6`5JZ%OL3&u*nK8WbC0>4tZ(Yi^%SumPiM8}yzHpwI0y}PY{m_wRDt}^1 zbyivgRc!8@ViudGuG#8p{G~^ImBwPl-p{{G&^zLBjE-d6kVK=JKqEb}X_<7UD;}@n zHNs~&xf%XwL2auK;YQxHlWDfEvka^|)HJ-*4|T(Sp|LmE{$3R#U3M@xehpk9ozb^d(f&_yrdXl5#dT?u7lv^n$3F|zQ`Gaienzt=IHjn|< zE$-cIigW6l26pT1zM}$5UrRc`mCA=a{>+F8R;~6}AF;b$W7|uQI)r zyJtEcS36QjEF`cPGlCIT%Z1T8spZ&iVR3QRScM(#g2UXjtlk3an1{Zku*}juii}YP zP!~cwZPju1GRM5at0_WB67QNT-3tXqKBFW4x_QF2q#cqob|BBQalkn??jGD@M~T&q zS*ihBWDNco0?07nsKLl$QXV?)HoucPQnCZLWzgoAs=+YfQF3%1gO!e1UB9<-)R%5| zGR0vs7Aa*%YY%F|;$lZ_ih$Tm5dFKHuKo=siyN-}Q5iIVRS+=6zF%$TrBPS#o>4 zVho&`H`PFeN5L>aL+eCtq{N$I@haOz5eq?61HQWO8UTG!`wi^_zd>O?p+^aSkIQ{__2#j>9zcTsA})iu_;g?4|f!xS@%aNOYOYuaAn zIJp|P25Uj*(zLGc;NXyBmV(cjA#g)Kqiu{>z!gBM8EY;9G~~Atw;R6BL4)S>oON?$ zzVvI0Rp(jUXRG21vR(GU|Fj&*;#Xhbj#CB|ao}gnbMCb} z3y2^FL-%$)mj{1$ca%qP{D3Jldc(zWX!c<~qeLeH%=K;1K$6T~h!R#{D!H$g5;S}b zu+D!>(kxbt$gib)sM!U#JG2sKllA$^)d;FBYk|Pd`Cr9B90};?Z5<$q-KSdE_=-PQ zX-k~e|1eQBmrGg~$gu>=hBqK?6UI9(^Ba92{3)OQ(mv~#vqdth(qZzT^ZOVQjk*Ax z-c>LOXAN9uOK2RSaN$OVss{NRo`2e^5XKq0Z_V^@N#I^$tm0O zLZ#PyyNgimu)i;8wTM>rMix|*V4;RTC6s^Ol6=4`ddsi&2SFi7T~|Mb34k`4o< zYBiMybd|<@j2b&r&devCz_d(76Z{!01r5q3G2-3IY1PB@J0|>r&t_)dLQsjwPUja@ zynC=QM0KdOF0_y-@Eyvkc$R^+;{g(-@H$;Jg&iwW38hgn3s3m(Xv&ip#DB$N<&0^N zdH+gthtNPkjQ=-S>_3}JbvqXvH7tHiCK--Qk3!-=IWb{5|_AMuO?DXa)9ocvo+dQHp+UsT8NHPdagyv;$N)i!5(cd-Im6 zx|)rR++{ct62TA_8Sp*|1=Co(eI|U-<}93y%H&$=oRV8XnO3UtWn=kealO@9vwHa~ z3}Du7abb~0j#U=p{RS-ixON`JoJmHTr@RcCj7EizqaF07IfP4|Srq#mBk__Nd^8IzPbO|Z$dixeRdL54~aT{+u(d+Z2XWtII#=|g{tSC{-uy7u&_crhE zJ5;tid1PY3>9EFhGw|Gz>r{RI5PLe9ocn^QMv=L!)|$UM`4}s9>i4`o2oe{FRJ%O=i<;uj#+^9QZ3G&}9N_rMP- zg+KaAIC+(!_~3@YxQF{z>@8LWPm!e1Cq$P504Zq*YQsj;My2V|O8Rm3jMW@Y%oj~O zfGv+`x@_uM&@r;T9hDc)E3%}RITNM8uSg!Q?f z8szwwlW~PA0B7nq0=5Fa!0ONg>e!w@w>z8Z*anM}!ps|_Dc$bt+e>+vKR4}y&%yZu zbV6oQJnMU_^p}37S5FvX{ln>Q0fE42lryRWB3H&uD48!Yw1Q-=0P^n=nYsHni!X(+ zH8M_;dl*iU7gSiA)gERUQk4LqJ-Onrb(__KC(UHrw3(JuphqWa!WC)=(hipRi; zT2c&28l3~izw@&C65mSM0VVHST4+@fN$@>P73r3&s#yjfPLS;QVC=72n&skf_qaSV z4Rp0<1cNNfKIl4_U^t3*!@0rKw#b^}fgrkUCb>{ZzR4rfsA_M+HBgI%=2guWGa`Mf zq{3qlL!Gtw`vsmz*z9ILm}3du$C|R7sgv7}I9ZE?j2!BkRqWh}L=2%# z79`{ZGwnCVS;BIcr!=bAQ;jBWqxHe?;3v1dBHmh?n82m*$kHy-uhOt8IDU6v9mIJO zsH0;mBG=yfO}daH$O6!vzjAJvRAcIg#&UU2^e*H0_F*w43&={>S6CT03Zs13d!Kp= z!Oi*WHd+ecinwnEp^Mz@6x@bXPBX_Q9%?Tc4QFz-QWg79b}d(oO0)bCT*+eJ7aemq zvfaPOHk09OTop+TTctIo1S`>&PYIFw#D9(`6?(?R3m7K@hyEefJVb3w(~*$$@9QV7 zm(YCs+0W~ZJ4CTLkJi^sJls9s@(8*NJszh~5W<3FYSILA$TP%Tf2$D?Rgc@st;BgU zRuiGIRdB;WbrXsrFzOEhVv_hGYA9PTmxgTG;+Sb1Td%f}XP}#?-5mxQaqw^MtZEtI zp-TU65VWe1vxS-K|H-vR_5ZJ)nDP(x#7o)i&BdFV|5i?9%xfA7d70rq_T@f)pDMaP ze|{zea z<>H4cOe=nea}n=EEJCKQ(#|_(uH>DRTIzaZGc}6M;|oIMzj_x_Wpc=~HUVY_=-Ith z6Tlq8+Ix>&#mgK*Z+m>s3u*XVvChr|AeCtYng-+yz_RD_45`3P`g~o4O>08L=(JkR z_jyvW-M@{Kg4+L8AeR6^47*zX&KGY#UCN8a z6DxqwD9JXZ6NddBOiX(L!yRVm-xN^A>f`NL{)*ClnEw$W_%CKG|24V(w`hSa$_SU0 zh{~L>lwX7i6}Yr5h#b+vf|LrLG77`xS`L?v-O|z(4z<``dL|fOFK7=4nIu&L5@Y90 z0g{EeUrWN;8}a#SI_vgf7Ax=T@wVU>m?Qcj65kEYO&L~jF0r%0yXrVrhSuZ;&E~OL zSYPhE+@_Y%%`hr23<#VGOX=>3_z}RI(AM%13C%MYN<@vR+887WE{eS(qinIw%4L#< z(DW?B9%rMO^=a0q%-nPd*L-!>=BQbng=-*0iPlEkX3qQTd#8z1s0nxI~N|IQx0aqzU z^!5gqBtm9g0%O~7ZBLA%5PtFbbxXc~notS!_)@eqFZC*>4^b*_%f!ht~ zwz!66BpqV+YZuM+8`tL6x`~e=i;V86!l|HT{}Jq!D497u50mVH0GYk)l&UB;k8j}T zIc?W-Ws_h`8vDFr#a?lEpNygKEp7=StJfOR?1|7-ZtNG%z1XN_LVlSfT@V@+z}XrF z_eLT#`wlY#-B0?aV7c*o=rN4SJK z3XJHmDQ|jq#<;z~N1j(isQs2i_IW}>4vL_NyH$OFRd_ERk*{s7BnuF;ToSC{ z6CIA?ZHWARFWpWR0ohG-YG90DFspalROmkqZl{e*N*pvc3&BxT%VuvBjgKkWWRAE#|cxG`ozhWUZM(2{sYWIyi4Dd~=&3l_I-0q@ophCte8 zD!3>9;-=N@J8N{O>}osJw28JZw@jUTBtqKoL>3p0-3qASoJ!jdVdIN*uCT#OcPv zvnyqPu4|13K*(swFg}3j_uyvx=pfj`(mN#A!5WP8Wj6*X40dM}(OXGkg&bu*e9Z)ms*$*LSsl2K^2F8W5tYt<*b!usoMF@;e*Obi%R4 zZ4ZI16x?oAgAdyZ#TkNc3<3EyE6A%)8t7E+@c8S2%YdTqR(Yx!Xs!lHvQfMrM=E zfVnm+3AvZf=H_hrlA5Okn^R5{ZCv)2NrElP52*(69fIiQgwH9dPTedxMqwrIHI_6$x57V|mc4SPaEV{G-eiB26BnN^3&JJkTAHx){&Fjd0=} zc%^yGN(@~@>n^iZ^|U%OZ<+T7@%_7T_72S?;B(i z0I>6u8pmD49;huJL8CPpf^DoHO(vLS+9zR%5yJ(;(O!k}u95Ou_m>gB_T_he(`fLQ zZQxsK91%z>W2Vp_PC@I6v?(oWVr{H3NzpgT5ZeB?pnKtNEy92V0_sQp&kfpt2c5mC znWwyivz?KxmABbHLhqmTPvg}9cLe=AHr_o{6OL{W;pKoZWUM1{lbaT6tWd z7w`SnbN0RYQ|@=SA7(H02Pqh#w&)8v7!i}Jt||kqlOA*AZF03L4P3suly++UpBa>h zsVFEGK2;0p^LB03heEZ=RMi23jzo>|W^){&5G;aPjdAN%)vA?Ua`s9rqfpRLBH_fD zSWvkBcyr4qraEyJx0y;iD{*GzY#TAkcF|Fk_Su1(LM!xrb1in7a^pjdJu@*@*CyqN zxbkt(*aWMB-}>onRf+jFjOF{~`s(x`3R4qgE>R6_r*%USw?h#J53b&K*v`$x4c0c^ zzQWX$YmP`^qI^_&n0|1Vyz1C%cg03YfBty6HTCV46-8NZ$%`Iy4Q<1I0+VgVkO^j<@~-=q+$QH8(uQe9j?=z+n+=tFj{d%4VW4};cdE9T@NGk#e)7b z_#o@ez%BF0!oe%R-p-Yig*O>(hRLd;a-5rK?zy02Vn$?=kydUt=L<1FB0#chMTSlA zm0MW4;2P%GV)z`(w-|&a8oB$kPU}p5x5XIFLX3~p2+s;oZMVKqyGXJPfOTW^eG)BP}GF`ed4I zA+EPn2GTfwhV}>(`dnQ_L9xdTfIROUlNM{jLl|eNLNdSQt0~2NvVw3So-h#_lpyzo zBedW;%J!x4*pv%!{f=2xG$3JMdci1VWW03}6%CBjLLT>(jndrR=SIeS{e_ZupHM!y zvtJ6Oyxdj0{>GtM8eU7T>Xg2#*oCQXE2AdOhs}l*E|noK{`524{0W)lhE@~(Ha|pR zcRn*_Xji8i$BSV1MJvq^${L(Ui_OPPXJ9RplXyP;Neeo6|2J@v?$T-PD@Zrho(xKN zbj`)gfp#*BTTZ}y{tKk-ywwAvC|{E&!`1FXDhy?*LCOugZ}!xpuU&=_(;j)lh!%4H zxTn}4C%G@|cJW??Ooy>4RNK4~yR^BG&QdbDo*N#l1de>C9#6Y1MokXm_vg=MU?&I4Qj<>CdiaBtH@KsOiy@ z10(0tAt9*0Ob^{{I(aC8!NTs^ecE8nCd*SarmZ(-iL3<1luqynnB_8I^r@FCYh0l; zVRs?E==K;7zTm8Pm7(zu=aEUfaOOwOD;@nTrd_7atGiNQC5 z`+XpYsuK3N2k-KVzTeRB&u=E3xX)aI*3AC#IHeY`5fJcskqZy2nPqf;?q5fkIe%b8 z!VMvPA((}x|#Fh4A_f+_n??q0Ty* z0C8Z(ctFS*?3)klH3a)gIZ1@bL1geC{AI;DT7nbr#(0p+9_-r=ym}-2<;6NG0zZ2A znGg_IF*uyC9h3`e!qtrM1hIDgV`8j|bJpDf9=}eGUBZ~LNcb4- zPP!+ne|nwf?tMBhM~6?qIWODA)%B5}slx?rxgU5!V*}JV;yVK8v#Vn1?~FN)(^701 zh>jb$h146kl^D!jgtW$wc)(v#KBJr?zJlwe)B}t`b30?EfpA|w=sO2h+#Er-TLTfK zb8PL+uOVXZf7P{927bp=!a?_i842|_hzc@3B-RAD8oyKOdXT;$Oxo>Wd_{U#i5EA? zZ;($2c)MhbY|p+M{Ogn{6FtiO3IPQ41NlEADt}Wvwz4&IRxxvRwX(PPuh%+_XBS*G zq%VC_t$N>PvxGr1VsJ+kEocnr5FqEs0!dL(2F=)D5oC*3Is>{*d=E<0d7ta9G@!iD zbTmF_rW;Z_LkO%dxGyk((e`P`;XF4~+=%y@U&nhly~FcqNl(ukM(^PZ-%s+Jml>ko zC~?4!k&|{bhA259J{B+Cp)6pSKr3M~8!pFjIULf?$DK0|8kjHx9LmxNs=`~K+&a-% zGCwCX+B&}yjcg!%crl*>lZ>A%)$ey(_Ud!J_h02WJ*!2@0 zPpo3fgKS`HU&jbo56@?2O?q5xRW_1aWw8x36U_F=8-73p5=Fo@my|M}Lc&MpHNWsz zK6M3*MvrPQM)j54Q!#5{0Jcbn8YH<2gXnX_uxV~7cr7}CV6`EX7E(QUEHST9Ak>bu z?%}{9i1A68rH)#1KHsrdvERIL9vQqW6CGkn`y4w;o%zKe1ED+Y$q)e`l^e{JM=e)9 z-?)<{%U#{FGId^U{H|aiF2%p{X%`0oAA57KV63E;56Dh0%_b34hO3ClVz4pyAP?D0 zecx0j=d`Ia_<`exCO9qzqqumBtwJeMhqT!Y3>@4VJ18ngtC&&)zs^NZNhvKIOG%c| z*fZ!m$|QO&vc_GS6mLJpW&{NZ7QI6xqSC{|VCLM|S39JY`fWyq@dKrxlOE6b7s7IT zBa#KMgZ;}kP`kM2?l#p9#fI!7Q*tYXlOMx_ z+iDhq&fv0_FD1bsIQ+7&1;=vE?3m}``Bgaf^^2{+of`qcROS6Bl_J*0A~UfwM)VOf zzh4E%8cLH9p~y;U+;*(E6H@5_rNXQ5c-Tv?u?rVCMx5{k!nK&SYzS*7QnDWeO2p{OpkqUu>tF+%X@9oI%9Pv_8eo%fY%SSY0DHghfKybmnjVMIB5MU zdTf;mxdecnMF*2%e)T8B>AmnM?K{D-?2L@S&h%ytJb&dfD5SLE~SyYd2j zH#1$IbYJdlL!YOnl1M0o9jVC{3l{AQ)LY;rhxBo?a@a50|z7Q{C!o# z*6j~i^&2Z_fK-0!4+nYB7p)onFBaYvV+V}!M$`b+HWYa&IU~k;;q*N!+kn+ zF?pj&I-b|9;iOclBmvI&ie7TdUMi5$eE5AZDUuyjO_3>gln1#3(TKv3JELF2c8x4} zebMk_(zM(6;@v;LUwVdd>5pVn>Yoh`*P;EiyeOsaEjh*26F^;NKro&K=GwOFYdh)8 zbau9{8u$7fF|XW#=gz?HT502b9BXjn)hXkTrH(_vf8Lnh1z)?wNRC1uxM>hQcSw*O zfH^0-ZW#cLg1UAMGWV8o{kYKoc?i#iHD$K3{mq~H9pBe9vicGBnu zu+7%%RZqKIFsYv1Jc+()sg~!DrId`M{f0*T1oo=l7#MJdNiezb@fHx)NslH7_K=!h zUg7bENPD(Ju+wqFSnnA`Kt!-d7?LvALhnjDxvw|@n^n8xJA%jPv}c;?oMFq2Hk)|*!3-kwNIP%jXo-wxiAYbCSXd|>`APLvIg8Ws< zZ;`7A_f_ylBLCf<ABu|?&AIacpgy!T2{grgE9g~Mi;>{Z?#=zpR|T)AaAYO zUe4HHI?rF6gHbQAS_sXi((0tK1sEN7&q`0QPNc|3j!7^FDS<1Yn{nvS_uyEGKLwH~PGz~q=nwtY9_7gV37N_p8D@!)t= z0`hFeuL>-A@+mH;jIq+dD6mMzjICwV1JSxmW>c@hW$pAJM3&Ff#^9c)&H_B=`K@!>Jae z4G>BVU^p#-8q}#R&$n%|(^ITQ($~0!*fPbcfcZn=%MfbPYFmRA_384+qIScPZPl}d z$5B+dE~1$SMhISd0GT-a!<9#w1~1)&FwqQRnO zs)_dl($9fG?}rwR8KY9%(t1o}EvHwJe5&WzoR?qJI>7fi2ASLXq&gSs6tRR4KAxgQ zubtITQ&BzXfCSkm7GPuCZdu^%pG3sXQ~iVq3c2ClLmONW4S;FS~OU7 z18d@Il*5rk#5gi4Cnzov0q}wjo4@5;Dw!SgGD%-enEzrMYT4@6EMnz!hw~Wq=CoRj z=a0juIVcA!h7+4dYlfvC$9obLG9ePZ2WlKX^Z>>pOv@FkAKk`+F>Dp$jfLue$da-k z2rg=XRGbSSap&~?$r^|Jy(s6Dw8X_N{YUUkc>Oh(@=nRwa`yVtCi3z}kGl-6khQ(VwU=l38}eT>(SxFS*f2a0&?qqw5XJu> z6a6Cv4Qau8;w`#;#WhtVsFMnYnALIg+2{;cq%Hq4G$e;hwKWN(>qxXS)}@SAC$%wZ z>PXLSLZ+7^OqS3QUT2fCn&WUw2^`9VT@yF;r#Z<^z`7<2O`cqCqNjR7wEWVfVN zvZbA4BD6!mWMG}&Skl+;wS+2b=7gx_kk<_ce}|@1oYkq;_Tv!9w~xuLNk0}=>P}5T zr+RrGY?{(EFaL4J(PUU@jAx$o!?sy*7uEzopkf*o{cfjCmv3JO*PD%`!C2w#5>Ziq z)0DN@i#WICM(d|x!*11thKEg^p1H(rpOkM6Dz_XbHe0|!P1`{aYS-+wRI`yKzjz0+pTOFM`=p4&Duk#41&k7lK8Ncp*@oL zquJk`)=fA%lUczR+1l9^KD?wRS{z<7pe_dyLC@CZY$ZQNT8<}Zm2*FZ^H3gjQLaDI zNb-VVYUD%bLGr?1u+-slqU8;}IWVR^tRn*C{Y1=DVJY=*Yb0X8K+VvbVomF4r zVNI9QX=zpF9Zw=X2puXSI-mCuXmhD&8FE{~1CAxl#xB@<}w@fdc*kobiQqV`3 zI{4w>O@`VJgiwygLv~V@6?;fKjK`~{-VGB5zhS%w<*zoQEA>0YT}A%FoYt+Q+=CWy zH@4s?^aeaq-1N$_l)9=_`V(%gT1NOxJ%Md92nUyL;im?yHm%ap@Cbl~dOIu_Ix)nN zmJU>e0>#kSPIC}@!hf&WV@1A07RF2fR?PuQf;k^V2&(%%2(-?UQMHh+PmEh*diB(f z+mwj9wK#F+qBNzUBn7{51nbGvH~>XJy1%YU`o;f|Aw)?{M3^asSEkFvRx1d`P(B$ z;Z>h7qHamlV|5Z7=i_{M?q4Q>#@hI!4+y0_-7Ndk=}u;&4ilt*Joe>z6P#SV{;)qI zJg9V}07Vs{a!dh>@v3b$eg81Kb+ zCJ}iC#ZYDA?C34)MF|dxKzvOM>f9`OG2~>T$Ek2Jm1Jym;|B-VoNm{p$=MHVoaJi| z_b}b2d1m(FHA^H~t&f`$C?DB#&21_lZShAYQEbI}+<@g9%@kQhXz^5rr#2ImE4|yG zmfSE1!OJwVPOWK4kJ6)b<6bl0=%dE%_=(|Xo0lpr(YV5>qu)Ngy`$Yw5wPC0Mm+CB zpK+#z=#dA{IBf5-H?&a2LZc>e$!(WBx;r(TiuCqEf?=YsE_TYKD~GYZ{Ek4R23~v^ zk$Y;jZpvY~WfPFiZYzpP`E3%YUofL)btg~$TtFdEd_Y4SR;ik^@~Y$8-%Ba?fiFOA zS>7u<@9b_ROUv0K`dk7#e2Up|CAp#nt_hlpUYYGRJ5KQMD@B>P#RA!{S`vM_(SFgt zl=x7KZE*)SOV+)Rbc}7Fx1X>MpK@Q61$xRLmLIQ9cbQah^r{s9b~tJj znfxe^({7NkD`7Yu+t(vPAWOpDTwa_>H${Flk%7t#c zR`wlgU(vj-Pt}6tFm^-Om;kJ%;C`t40S@((M$EEWtj0 zg2w`pmiBbJc8dp0HBGgq-aUw6hJsTDG{kyS7qa#YK0P@d+%oJ_(wC}2dLV&c55u_$ z#>4RlV8w#UL{;pVDT&9F+xzOC6ZFb5(ahts-$hV`(uSL^S9Tf38s>r&r5U3Ao^o9 z#bR^JTzO?&hS9-1ZZnhTTq3duf1%+wg+1j>qZ5D)W@&;o3D!R5EAgjZuh6{M&t=p8 z;Fz6k*Z^f3!{=}7PCC}mQV`vR$l7jn-m#7vDJ)sfy0RwN6w{36mp2vcq&wF4AQY~u z0p09*wX9{?Db9$r@x4h(%g*y@(8+q;TQ2Tg>7pb`HD^!uS_^{(OSVrvg^twwl~C|} z4yBVKFAe(YV{=8ou-;>x#aM!U^VuN@r{L(Px%lEiH8+(*Md{FkW;s1k2eA(L>=49Q z{3OV6z-z;wAiH2^rOlfQebRa)#6DAU6uxS&@l^UCT8!KD*2WkM+p858(qm_G3r8hq zhRU@mygD^}C#gJq#^5kedBnMXqA)TEP{pF>2YDo{GL zU==`Mwc4M#sMR(X)jMWcVNjg4*(j+u4wf+E2rbQW z#zO^>@;#M+H-74~PntOCaeLf8dJVO=DreQ7 zC?;!aLeQtIpaREjR#PtXS;wQNvwvk1n`LSIBHh@k99A;1NE`ds6gMl7X?TP6gN&JK+mUFIyyd-xp(k+Xwv=fZ!_H={+4b#c}+4rrk zu=k@g<0mYn{QeOau(xUt3}nA|+N`JC&XUy1Z3cBTN=Hj*jN>GjeXcM-q2@6si~8Ut z`8g+6!6-|i=ZOE)yOqM0ZG$~;AYz;`g9zJzs7R`kNMYAHh!^Zr=Lydlc_a;DWh>Ua#sqfj%g{!a`^cfGb`nD?z<`2+&H%-Cv zY0zw_dAk+assvK@5PEiwA^L(Lz68dR9z%W+q*CC&9I`v7=w2FlG5^tsBGS{+@Ls>1 zvrZ=n)Gb&bana=&XTps#xk|M0JCb?8#W)%vj?^%}&lhhdy-;ayl%wa6-^G>PK1Ss9 z^+EY2SJ;0~df9y^E>R4aNAvgo#d!&2n!(fC6}*8zh=ccoHCRc} z0p9wMr!5DEsuH!n1?$C$-flII{w$T?OHV<8aerQ3pU7j3VV9rU=Ro|lqS{O;^3T@U z+W&VA?r$6gz*`dZ83G7s8wm)A=09^3!VV6uF0Rf-|FnJG)Yx>v)jyszp3 z#qIhCN#2azgv|!Z3va`_X~p1{YegI%Iard1&3Gwsx*a%+i$Dt=EG>p~%x7)mxt&{?anYSibnxgj7-9++N6?p%+E3>25pr7@nd!03|H0 zX*wep9&OIikBRp;Wr*9Yt>yz9RF*erNKTuP^LC}B3!=9jSVKW0NoKgJd>mYCTX{n| zib})~C!{dXA?v2{bhG~SOXt;m>uyG3VLS}tSlc^WqPD2xI;PltFybA}7^(iR*(wIx z_251xf?gy_YfJMXEmxTWFeenmKUmNvsuTFDHw%+y=fxNf$)&Bzi%|UMiiMT!r(lke zF7|z^t+{Hvy_-W6> zTCTKgS_nPhvnvdwfHZ}P`ItdzhCw&48L@b_SZ)TZ9ceq9V#^}Po0chmoHV4H<5h)( z^1O?~sH;v9g$r!zYg-eYjS}4zMBTSSN7*^)-T=#wCIqljTGAVv83wnnFLS+hwl`)- znJ|SKjJlH7qUhl4d%ArE3KC3`kq>7^=7H(#ZbTvzW=fx?v^dtD7l>V%slF9JlqrC4w`hJGzawJ{Pv9`9!!6dp94UACd_M+ppJW>|g(Gd0$db zL%Rk7-<-t?ni?)61=_r0Fvq>7z;?6hu4JjipV*V0)?5bW9 zxaxaz8Dv9Ob_ZVB*!KNpMcYgFp##7t3)CGGugNuuKf|dV#(PLLyS)k-|I6XdI}ZDM zx>iuL_RL)>a{=7?b8B zP(a|<4Hd;p2k=Wf_RvVeal>yScYidk?z037P5(Jk0~4<$Qo*6Sd4g9|x_~)S!GXIh ziSDWTvxH+U2CoB_@eB!l!?wKyOsyr(Lk|-bf6n#0gqO6Q&)Q)_xhz2`JdKj2AA(MD zse?v`G)YXa0j)+ruxei)EIFZ*>K7NZM1u|SLCCKodov%XID$g{!!KSo=OSGnd=0ld zRhBzfHYeSnUhcnb;PVVbLAzaggr{w=(As%VGW7;}d%M*{2PbIqE4bF`-^sm)+w=7* z!u6oUgJxsZ;J!<-&+`D2o02mDpKfqICj1k1Bz4~CDWpxv$2vE zqOlN5H!GQVJCIzvZePD0Q&+t>1;5WHWd%C#%TaXHn|xa1Y(qL4ZCW8D)6^3>ks?Dr z7&{|Bk=qz&BH5{JN%9bu2(3;>~VmIA-=w@kwMsPxP#(n^icMScW zz&z_Q&iVqted4n|aNxZ`+Rq9)Wp4G+J}`7ETLoDg=qmLy{IH=^fDY=;8I=%)fu> z^wnOm8V~>h@d^Lu^7#*Y`M+;h{_`<48yDI$NzM6)+-mMY$@@oMGO6=wF@4GaD2yc| zk+yarIJm0hd=Fb<0R{Ck$yR>FdG&L@cZcHgdSo6{B<0qKcg=G9UuRVvTkWjZ3SVDV zDn>($lRm8XTN9b-oUGSfFE0un+ps|MbR4`;A)o^ZDd@Wh^k|En$Ob-QVatRtma~>a z%*X+S-jS@mGFT zyd#pVxT9#3X0$n_w8!UTo{I8!FR?;(x{3ZQtym)lGNL*~EM^+F^VN#|6Tp{sN z-Rmn$;v2~{eDc2#ST7W*jVLDa*`Lr8OVO&DR#fU&5W+~|&K-VwJ49hP_S<5e7m};V zlm=>RcRRa=lG0+9FlY5A;NFySqa5|~h4j5q1xVVQz%|!yNfbS{9B8WVC7{Fxd53)5 zHqJIe)v#%w$9e03;Mg(iT8BY|>~?)=q!gA%iF$%gFM|te_X}GJkLnL#)GTWAXm|_T z8HGR(fo71Etq#&)NNK{2#J7KZ#Uu8qU)-LAlcS`v2$vtyKZ*0@S%xEzNO9$`J6xVb zH?tbHg`ii4!v$oPP^|Rn6p95QT9UuQmh-_sy2;6npJOoC>b7=zje)G849r`Wcj1xM0=>
&`T3q6$=@tL={tEP>*I;oJu zZez%Q?V`R6Fxf)w#X$Ht1QuxTj~m;F7s zFnwFaQZXY!sf7f5?1CZL5xIeutKQtoj@3nHZADQ-M`uxY`69gr%|9YUMr*e%J1_NS zO+1Mi+ovr=ki{KoHv>MZHnLgr1ywh%#BuYXXz0^6mw}tc?-ou&suia?@3WkX6b-&q zZ_}bC$&vpfJ-;jxt^HZ zcSD?^G467 z*?!8|-Fl$lI<+>6K79bIejh}B{?SbhKHU*f!He-V+3JrQv)T7M*+$L9Hm|jm%jR9X z3Fd;2HQ!|V4N;t5xVrV4{^hP@G~{*44HP1QIh=jQa%=4>8_q5>bX*CGB5!rtft2}; z>K||ZykE`AQ8zahWPIZTVK@W)@a7QHwF znW#`ci*x!(H{NG+M+=D)bI1>ZR+9@YcdjlfKYlP$sV$g zb=$f#{V3v2+FkyJhebd5;G4h@#7P$Q5~c4E6ak|Ql3>81f!e%o3sdM7D|9ZaUY?MI zcXj4^*qIEpK@rTNMA84^>>Xn)f!cP>vR%7u+eTNHZL`a^ZQHhOTf5X{8@tRdb+HTQ z{pLGok~5Q>N#@CutYjtY-+ldCxo)`c^gH(&Wq+*5yZW%(nXPKY%6t;a53AGks7Q;H zLMn(B$c=mD=t$7tG`JFW1R|QxMQHTE1(L_cHFcT6G*u)`*}Q0djdyj2ocC{a z#3@0kJk(4tD9z-ZFWS|auXHlz>ND{X5h1FVd50MHc%bf_L$*5OyQXuE@i}3%E1r$v z4YHd}yuDzr)M0WTt-$P@C0E3%TwbDj{h2b#v1;h=C}r~Mv&y___Y6h-AgOKVVj#7h zBHut2p3KiR?!cfq61=f#8=G^e(AZS-nChU~KYw9$+oLdX?nR6sRhxfB$#we~YL3Qe zhfT-`5-Sy1_YtKz=xOBVo__oo0#cqLY?w%~yvg|Za}1MOtkA0*3BX7()+veL>dQc* zbWo&~oAU=j-*}m{0Bcpk4MlcFiQ3DjxPp& z^-ro9>Q$0RZY&(_PuS~JbK-pa~ zhkSiBOnYmZf4(&jz5Sv(BtNgxhXzxSYeR3~=Q7 z!d6q$>uR4`;8{|=_!>RUE`4gT8LMoo6sXM#{91ubea9m2DLDTq*kaHG$FMLOuU8$vlar%FNyHmHZz|Uvz}ba zJweud6tf;Zsy%L21JkEucdl8xF9AxA);A_15DGWQBihqA&x7tQ>J-_9$k3$6W3jl; z`Mj1L70j3N53tv|bh!hit~7){e4w7=_(+W@Gujp7<~QR`{L9AuuV~-90W-#npFbqp zwYBZ}!a`$TxU8pI+yefAWm|3rO<>0$qRRN;zAQ0U8dVqG_%u?XU&0{0Ln`K+9xewb zril7qF2(jgI!7xVq(*qIH6VJ2`jOfzV&*Oh!}Rsbm4g4+5oEf@OE>-(mMhsm@AFy< zJiGJ$G|^lSEnrJ{=fcMYOO9qal=JY@Ib@J7{{Vh=a}fT_HhF%n;QGpH^dIO9y~Ue& zNx&;4a^m45`gshIAhtd zU9+)wTZ&B{^%2Osqx_0Z;rPl`*S}`-6+UzW(wO;a9s0foG;X$c;Io{q*0@=a2!|K6 zaeEv@<^euxiC)>b$aNQj&v@@R2E4EBoDXnT;Ka9yUa`3NyYCIDkH&cj`^D-}J`(V4 zN6}%wgz=H2u3s@naF<0P!0pR_?kQOu z!QJU1>mWXtbc`LBjmom%pEK#?+IPgKfB}G5mt(C^MNM%A8MG>r8eoE(`8c2$K+)w` zFC*9(T5F0jL3DM85#P!A)`iurx7~Jw{S?`C+<^j$#eUypl~uQo^y-u`>{C1~GVohSVDF?`!T+*>J8oTci2Gnp)C&^%D9* zF78@&yCm+=SkVlGF4-^M*|i)s-Env^a{2-EYMhR>n5VIu`tlyUW9TKnnQO|cKnezZ zgIXWQC4dz{vYOj|@I4oWOPd~wwLbuNlLUjAbd!R+vOU=biqo`U_Wry;KM=}qkw2U# zc{IqAr6CQxvN1jZX)3UL1C6ZNB_f<#V-R!+*of7Dy?ME<C+&4`rXBK4T;vkCvK z^GbZ1x4$1XS^%Bm@zIdc2~=1S>3m}vB^M(Hf_=T$^G|H3)z5ed^GncumqWTr>X2F0R7(=>ZZsoyqzEe170p z%zhMd*b9w!PU<`DGO0mZ%r8zIt5V)EX2tYF&NaP7mW33#u@V1i>Z1j!xr36D{1ZO8 znAS0Bm)2wQwAhQpG4Ic(39-5K!_LFLGw}e8tLSe59zZ5!J`Fu!TxT1ch-%o^aq16X9KU#!~8`4hf3qN|=DyWwE`mw4QvjY!SKTJM<-(7so8zYnbwN08sMNG}CNcJ|;bwqtmlCy}iIW2!Lhp1K_1P3Q8l63uha;c?qxaP>X-BB4mK`bJjr^}OGvW!^1Hy-u26GiomcYu{d)KgO zE}9}4J8RuyXU8In(ohayrs942Yofu2R!k!_(bU9qZMR$a0-R$>PSF_5Y(GsbEdAx* z?c(!SpYjL82E%_^p@Rt7T}kg_%~YSwQ=>ZO39kt($b!~?N@VnYI@y1g0!U&KU4T+< zad%;&y5cv)Og5dQIst>~#+}+|lHcrf_SpHj)T2tHH`YV!|YmeW|WTyAug>=gFc(@wic=(oV>Tp5Eef^d?gr zUeDnyDuIKbUZcL=1s|NQ-SCcHqa`BFNQoWcKpIi%&t2hGhcGfHdZftZqNw*$V+gYH!GW0bY zJI6Q@-U_yt$mN=KmpK9KQsFfl_wNK;$OfAm( z+zRayQ`I)V1^pQ9c|?5LPf8{o+8Z^_-Rl~GJqFV5q(YhA>2@;rCR8Nt^?hvPt#eaz z)M!lJJt!LtjPTo#9kHhvb|4-74xe$QkPpr^g4sdhazdO5)k{BcFhCP?O5Dh1ZIfaW zSwNyXbif!biL(Kpq51n{k~x6AO#?;l*yO?V;)i+1~mD(;wHl*}o>hvfdXRQ!>Qay zia^TIU9I0{M67cDXR%Yk@}=nSe($g`RognVHr$JRM+_~b4Exh4h(z-s4}gsEssa=1 z=zfO@^J)$KRf+Od3mza6T(~gILU5pk<<*N4FbXzUKBProYR6OL$Z$>#-9HgDP+CMo zcODA!*B{!Tq9`BcF9meH+5!#oc_o5wanU^90t-OhGfoTXJP@!_Qsjug;D{STlIx#T zw9jxpZLv;%u7uzd7^BcEP#=A#PvR0;x^s-esRKyzX3vvD7iYF8RRunb${t>*^g4T*=&p zKAFABK)*u#;HQD?bElXC^wN2_vKB(XZjQ&{{TP$@rp7-1j==wJL3|HwvFf;@>xWC9 zAeN?_>se~yRot4nFMOWS2;r|F>(#lhF~g zew!IAaVsJ>Bi(-R#JIDLkmlcm2i^{4frJ5KyWalT4!A_u%dYL+j|t^tGWAk0l+p($ z-t>gfpPhP zT^7T04MxlwB{3N;ZjGioyar~1Zjt&AT)Us59Ym<(JC4RVN$}-AWhDpDP!wqWf2?Ew zH4wqN`9*KnIr?gFPJ>mq6%9@g?&#rpB;y>H46*LiO0Yh0Z+0~+c~ROsYc@GltsJy$ z#7m4|ddG?zxm-+WS*0P9AMVg>Cc=?N=_M~%z`Ei16aq7#;zBs&UU#W1v@}T#g++P{TjzQrRknz35`_+udOs`M@7x5`G%#C69 zqB&XK2-^|T%9+iSbx1GatZ&V(!FsI=ovW2IgNgL7HAI0yff_!-oi3}E6WtlltrP-z zNuX1?(vxJ4dK8>k5urXLSsPKU83kpGuMPD~gkw!iU@j7Q1567fdOl9Q5j%TF&m5BI z%;pzAk<&5BGmiZx*^NY>xfp*R?*jNiY99LNYc)4dLjL%lVaE)3fO?yPW@QG^=vkdT z~B1SFuf|waPVMXj`#ql54l{IeiQngb}ndj(E#sV@9R0hUCbU#qsb7OtG9gJ-A z4s8Az#*$+^S+UVIVQPEuL=_Sk-HT@dY#qK{mJ#UibrpOs(8LO*|J4yMWc8mbpBMY< zo%$p}`5~9X{ytMYkD$A=k9fm}!ocJZZEhurIDIe4RO{cA< z&fHd0WZ&r#yy_BVT4jo8hBDh!;3Ex#{7Jd1?3Kn^STVLEn>7!gx{2xvZ#*C$tD}_l zT{dl$fsk;WAQ0uai&kR zsd`Jk#p*S~YIBwPvnq3q`ZEda@5JmCpWs7xnEE4=d-4;S-YvfdsL$3sv?so*^sG=_ z&6BlKvKLrw;|Z+r@bJwHiaXff6#F8^hlU;|p#$Ox-z{ELpjYl6N5=5w-D6$Y zt$ljD9V6oJDSF08{>T-W^!_I*d8LiT;xdM#olmWBH^~pj*xoyBA{EP#++ajU;F=3G zRVi>o)bIMDg|-*24E`Ryu3H-MWHCJl?4F84=XqCi7p(jQJA4driW~XMHg38*e75m( z>soe0vi1TLcRHc=#lNTl0>FeuBM$`p za07r+Nr9Rf;fbc;Lc!s$0PG`M%Bgm=n$2`>1;K#ewguuHUmgSPyM)y5HTs;r{@i3+7qOG=%ZM; zAJj4&!Sv&l3eGqi7qx9XE?41muIgDoP!;{aUG635!GDFQyduO+Xca>lhRVG16UI3z zs|RA%+%Z*+b}FPF6Hkh%hG_0Oxg_(>^v7$|6Be?#B%&&bwQa;eZ_1E|Dv zSD;f)EZ$Hq(;J2Dfu7O>p4O24QV;z~;Tw?^NIFJ$EDLWmtub!Oz7ffvx)>Tuyox@) z#$mtxP3Tp1XEc9Np^!Ch9qo~!T$wjI)ZeL1PRtu|27sI1%c*g3GPmR7E9C95z7pjf zRQ{=akT^kMvrSY%wWQU0-Qv{JG{LL9cU;F%+YygF+Xns?z3T740t8xaAlPGL_^Ij> z&N+82SP+#D)Mtq1##i}>*XJezn061P6%!MmG7jja5Wy9VgFzz*%m_T@tTsHBX*PO%m;U7`8Heidl?bB1{C)A4

dJcXr>Ff6_^ZlAhMEt!m zedgLKMYvrQns9|a%`>_JCB)n|%No<+qokwXb~U7N;+(YB1?MKx+*_wlWHym?K_w<3 zg=6M2s-G}ZXrljW`f4Vh5B1UNFH6xxQ8}K#{^ehC9r`u1lj~Z$Ra70>m(u^ z$7OH>jjRC$UC5ejVj0U{2{##8e#yO_LOW3Rb3{Mp$)iaiqJ@4B{r0kENK%N|*>1Ku z^={F3GwyMkape~d4$Vu{a_;ZC6~5N$8A-#lCa!7rD)#_2Jk(eJlu42N*c9{dlHbNH zaTBASuP zP>m?|2HJztgZ*7Pw14qwi-fJ>(kC0#_KYwsT5h-zT~3=gxPYh%?~*>Y!?KvlF3x>t zTKgMBaS?HRhDNyFHiXCz)y4^vU}e`8cm!?2B;*Zt>3>%?(SB>2T$9$nK_P7C}9>P{kLqH~9rM>U!C4ZpPcdOLhj>?^)mznpAs zdDXGG1TwQr)p>`JDX?(MN5G572>JUQC_60j%e?+NokeTP*W5IQAwZ9l&fxwDpkZFi zeNtE@+s`T51M>4KhdZ+p7Y9+N&i}1?0}Klo8IKZlnk->pUy0BY&X~C4nbgO}~1p?tQ50tS`Ldl_gNdbC?h#gl;JvLtTWAyS77-1toU0cZd z;0^u4pV$ec+OakpaFvuW4vEkq=8t}h8PEgVG-)AJ1QjEk&;dmwZdoE*itQ!XoW;a3 z68x7s6wneCMJH-CHIoq(2hz|9s>6s_{_nq0o`vNTUys=(pD*-G*OtMgMj4?KR6yEQ zl-WpjAw_MkC5TT7Y~4pEJEgMWe98*Gh*Ov75_rf5ycjZtv+{MP^=6~`RVpAa;vvZl zZo^yUB5Y&PpXHJ#)$|-FS{hmE&2;1bI5kaC|F)_dSHd8Eicc!csuM(w=P9EiBjgp0 zU}(p%`m=_@M~q%=ZKu$5YwpcCphD%kZFz(6nP=m}^fEh=XhS(!iH?4kYz?F)f8MUY zQszoR%@4$FfIk|CBm4|WFWVHR#eWR{@X=WCe3&vru!g9tp`{r2)2<(h-qAH+h;J0* zK83Yazk3Wt=aiN=+$PS;T^oX*7PjOxh%LfF#`VI*`q%kSMsDL+PaSSZ#!Sur#!OTS z8@a70m?sxVd- z@G=8|j#VN&&G47t8OI)urfoFT;=EZTJbtH?baiyYV+DoCtPOs%2~N@2=!*^9HDcCs z=hFHP6IxLVghV~?^{lpE@egvqD4*CGS-iB(K8!Ii_K#uZkPe*Mb0m;j(S0QQPOT%l z75x%+Vp~2RmM1dod=ethi8-%w=#5?<42L`}H8=k9>sgv~O-1wKMa@^}5nYo!U%Z`8 z9|zv9Xm3^?d|4ry%D0SIQ#ReQqKpYH^2XK~({^-#di=xgH5(yws2nON;0SMI&R=bQ zpwn?K$zlJg84mN=h|8Lrf8$^=VT}t^6rJ%DV52&$*Cii54Z=B2gPMK>5>e#)#yTLAQ^eckj$#EW{bO%XPv9r`u+aCiX87Q?h1?4!7rI)PyZ2<53|jm zZvX`y5bty_^o&svZw=T8E<(0zshTBmSep9ohj-9Cz{c_zsXyj2E67cNN^Vri5lmu> zP`*-d%0=iLB~q8klNnhx;mo#Nk``3?8Tw@WZHZ2`;Wt*x%Sv>^AS_kl0UvNq( zLI0}5uVcgqgcIAta?ekPN@Old#s9X?WSEb;rc5rc2$ea?|31`tO@)}Rz7vc+3L56e z&lrAp6J_Zu8ZkVOP>mfJ*u^Q6+0D*!;6wdtL1-MR1?H~w&*2n4jLqjqjL0B=4yTGFCW?N3nk34OmG+y;T_p}i$u5`>TR>F&hG~Q zA>FH^^_Pe%EqW2Q0D*5Bqkv^*`X5bSxw7Z&%86MWEQ*%^xFsu=n9(Eg8HJamT+)ILG@r zx7Lkig>n?nR4 zJ)6&@aZY7h=%UnTk0k`sfrlIar;b$yenx{9gL zPGl9v7I`BQy2;FBe3CsVR23-+VJHrZHEkG=g|Q}_DI4z6(vvL&e9)TK5@456Ndohv zff13UX=cZf!&b2oLh@tec|a6DUkHL4Si@UkWR#^P?H4mFIc@b#^dD=)@KaS8SCv)T z)X;Ky7Q`lbva@iRwliKyO_+g2hi-YsMq4I)A#WVv-L}N&_GEgSlI{lhRWAK7r}`78*LZfrO{=9kGvh}t_LWrOv?B|#^ks_JZ}0f{V}ZTCM?wbc1nYVNgDB|3aWvq zIKwHq>G5>>hLKgb6z&$oPUZF75*deiFmJr)v{S5#3A{Wl6R2I3Tg=EKe5}vOKH|mg z5TU`j>av;|qyqfX9bK&4=5R(%?1EkNWrZj0=s`;X5V@etn7j1*to72mYFU=sImhsO z%qC>~dIDf&izesuTH3g;lD@W z&4U1DBskUdJH#=Y_ky~z?|=amC0fEqpx>37l3s~@TAo^;@EGDI-)yaZUssE>yS+mH zMNtZ2M-}B#>_r#=nRnLjj#=ei^;ZxVKCjNH)LVWStDhmIbf zC5q>Jd(5Em=@dtnR0Of@RKeS3JpCSJqMKeoi|)M(=(vD|yNU4F#@8hW#Ut67oqPBtL6o75;+4*i(hG$orWDH9;Y3Sh z!k93H5+Ozfk2) z2kSd*{wdiCi>q|jMJNWWfx9m`p^&A3`S1CdR3g>RFfxGXx$=v#_7#af3%Cf#0P$8t!m*)z3a)XA+Ss;kFFy|&;YCfq%% zj#p1|CI@jJAIhe+^BL}!=9t2RN$VGt#C13^PHa;<_UOE zb+7%!${NX|iwJ{9`Ul(Ua-&*!rP~t3!A5#RgAJq6Y$`HTg;MVaR;6I;D$}Vagf$hJ$WL5dn2=48gZxaQWAVm8P)N6aVI>ko*QGl7;iCQ zVU^JxHR!fBxjXH znP|vkQ~9(0wBD8UqMFwd6Mr8r7_hEhDZMP3BPOK@8UtvgMUSTP?S`i#PGY*4A5gF< zd4+W{Q^ztUWD`m5XyR9+UL^X}w?{rz9Wnj10Xb<_e zLO&_H){QY^s|2DV8{zys#`tWG^+~y_QGGwk*6Q!lMW;cc?42>r(L@w(DZB9juCnq|s@TbN3T(Qf>)L710ll0BH3!a8r2^ zAM_*ZZBcVQB?KwJvwzw!ObqHY@x4wU88lSvtVnBf$oIzGHs&(7}^#$ThFp%B`?Z$&F zS<-=7=`n8+`3dJudk<_Olj-DXH)m_-jFXK{QM3P;w_i)b3-nBxz#kSWuM1P{SUWFv zQSTgKu=4S^@7AHi^8C%07c-LP!8DL6EW;GCJ-GY?E@E{c-^*R}?1Nj5BRS1j1l^B& z@y`M|tP_saa};A_(KAu~1O&3~&>KD*xW59Md4wb>;_4*!8Kt?0>sG(VBI zJvme06tW?fTf3?LJ?e-=BuaAFlogA_cu5A@qlg{s+-_wf?*+5xJ@wPgXzzRK?oeLZ zY`FRR68vlQQPaTqpnnfU=d=5FVz8|{yY5s98;CwfqT;L^5yVv{lYN{soh-%aY{|>| zfcb-?oL7%JobW3wQZbEsdMb~ioq6UCCQ*|z_08J>IN98HYzf!zzV85Q^lJOrj#Vg4Jq>5+J3W-Q} zxU;AN3JpK9PG9}~s>MCfNt?OCKhxp8dH<^a0p@pPszXp36P|W*R-j2AOl9K5VrrZs zdSH&_Xmdt=nQ@Tfmj>4~{C>I59gY2hf4wKi0*8~P9S9lvP6 zVZI*;OXk@U-kC6tnUDJMWH_5u7_9uO5x`nm>`ur5rrweyt@=NIzvPa9M()0vGtvVD zUgc;gHtwg?t34_b)q96?CtfJU}j-eYCD?3ToB?$ zlfn2d2aH_6ek6zRIZe)>>WDFp{rk`!&?mPi85msjVJFB(H%u*21^xrDqvj`)mla;1 z=6rLH<0H7b@mnmsW|?1d5>|9S9BDDBA#sHFVzRG>MQI*agC46ZERCj+YU0nQG;Mv9 zo`KIs^k$)80HP%599{`R#S;=p6-k4`_E9L{ugfS43l^?}49wAYMjSM9cXsn8w zrux_te%iRMyM!D`;U>lx!Xr0*8IcXAXUvyrK_Lj{+Rr_^hWywixL@n!4W4=BF>KP< zude1}q*Ge4BVlmf62l@iy9ceXyI>bwo};10*{ibG?5XE`!L04cQ^&Ga*Rl`*YY(KK zE4yN(i@|Ohv7v_^f}|OCgBy{z3b%npFVCIFZh;FHJ zJ4TsrbmPn$E^Wtf0ILZ>@yQI1=+PcaCWP#S{d1m5t=cq-h>HQ(Gr)f z5CRXkRsy43mm-?7mV~>+_091&w6{w0FLSKEp2~NfT1fp_9xZM{)O-(S#!Z-N5xH+e zT#18X`#KJV;yJgEF=p{2WBX<4@J32mTeMr#9|CihpnslCV|6~-(|jy-$y^2!<LG zU3{7FgJq70vyYa7GFf38BFdfYRR`M|%3yMVEN@0rORMS_Ex@hL$tA8j5&U~7=lZS$ zH(vAvto2v|X%wkZW0TxhELzn;iqs#SG(Itgx%g$HFHDRYM%fTJbX_xmXGiH70i%fO z=8KA^H>8H!2m-YCX+PhbnSB2h4~kiTzK;>ysZ(g0IMQ~&=I{D9wdqz_>6qSwao&$L z+pAzj{#H-x4&I%c%L+L{pT3wUp>z8-s}7&We%KD4o_d43p!H(DhsxeoPG|v;$YtH; z(H)#mKIcvi@5gX%z4=QwkXM)}?3}(BX=NzYF!xbG956V0cG_N;Yed3cZFx1Qk z>v`?JsakL|x0dJZRqaVYC{o%xv)c8mG`y?z=nke{lH18rDU~4rlMN*>anv`r)jBNF z_h3|VT`K?Xp81s=W#0{b&y42EY{I~bjil;_l`W%qZ-os?ZN>QFFb)lJ)TTk!+3cdr z;rU^#-gGgM&8D;9kZQrzoC2eB_1i)|!9=reLidI!p;GNCpa*a$5&`823KPwwwm%rR7~P0(|gv#}NpbzPrMr>C!* zaT)^ulxnX%jl;kRKXa+0U_JCCzJfR5dPHZ>k7?h|#f(Q2K|2RSci=>5V>f=)?Qat` zm$-O+cZ_|=^TUIn`8%e*DEhvTqXVuyutrn#zTo8pM=wS^AFjUe z_B(Ee;(yZzL^}w#hwXPtec)f=s6()?lzvh3BT}z|yMsuDAVZMrhk#$O;V>~M>^o}l zE*5DUs`o2tQPCi%oPY zd(Be}f|QcWd&5O3@=rkHiOz)|WDYk4GMUhr9U*Bm$#u8UclupoplefKA>g@VOv+M? z%cN0_tC7jYWnnjIqo8-HqB78CgaRF_)1bq#JG{VE&M0yuNBPGv;o(kikx~g-YQw(!6qRSnOAZ+EsT?TQ91<^}Bmx%#v};8(M>jq~c*{mKa==i_lo-Oj~h;)>+M2Zg}B-`IQ}We?gCq)Ef3<7(+3Xs?UK=jR(_& z(6NF+kr|GxaCMUFCz1Z>47SFSZ#BP-_YhAK^`Oz zI^Q+$=)gHo3QAS1%vmsb$nOI01oA7k1JJX2 z9oCCIAifd~S_#YBD+f2e5)rk43}u*O4AiAFGvX>vR6wIR^cWq$O#0rfTsIL5OmvRj z2*HUhJemih^ohG2mY2j{{u!MHhI(~-AvFS&lD_taYGr)Zqkec1siB>Y;+8BTq?!dl zv;xJ%0i{v64MW@^4xLq{tQW>1f09j53CW4k0gSB5RCHE;DIw!DbXI*{NDY=a`Lgno zn&hR;DRC`PaVSYljnAo6}i%Vm>jBr zsGCJT3fA%<1tiozL{yd80xXo#tWU6H1mDVniDd+#DLl<%B&bhmbhWLmgyS9oJtR9Pb2dpi4BG3YCJ^2R5nG}$xQfh<{` zenJ0CkMHxCbx1riB}BwKw&(=j-y6KD6Be#tL-8b1St*VXzg$4}BtZS)ldZcB;eSmD z`9TAjAm^H4^X%{u3nuxC#uU>)mS-ET_OzTGe?pBAu?uqiFkU%?Cm6>uLbeb36r@-+ zs%QLT4~9SnqVkXkio|pHWCL2DDfi%z_sPuwp279j^LLR zHw(@g$|*q;7f$prf7$yp!YpY52_ornfm|uvY`Qy9s_CwdrzK9%Ekzb+F}oKr@E!~I^B*eiIVk%Yot(+BuI%cE!OphVc=~3W#n0I{pg}Cv!sB%@D#SD zQBGy^Cn5L>;!XMlw>iM88}`~1{&Px?JMRu}fkl{_K0XG;ufJ18nZBtrN=!Q@gyI)` z@IbY2r;!!aD26vCB6XO{inAZ`Mp*~r2fKFAyAXFPLZCBh{R*BZvF1b;fV6%<(}esp zT<;!{AIEU1`xT)8*a(9DfDu9`y)niX31r@6vQ-^QUZD7u(%skXRO_|5@@gqV@Fssr zEE1{}M?BR^>YMV1Ns@uSxGf~P2;hgUW#PMm(=&#ussENVXa%%8+)!!yoU# zFA#(u`Dk#s5PsgC0UR^VGreKz)cK9odDvke`qIYO$v$g)3xJ|?X4B}YQF3nX+W~n* zU6B>0d8>C;6-@@h{+>Rd*So@A}m@PR1b-rw;{s&(m1GlLUYs>(bQCI!s>y8<4$DCij7 zi3g)1hl`iqz>Sfn{w^?_MvBIO_FY7}CJORu@RI7Yt;oW2i?h7VUf7;nsSP#=L1D8; zpYSaaLH{;e`-ZWgHNhWV0&tYwtcQ@0FT2qUd0mjZ1t`p26XQ1Hpb*(9k9f8mn@W^t ziwossOl@2=#Z9f}ctm4*J}cP9sM1M<3KLG1o+PP?+we)L=ZX={M?>z>I&{*?BUA6&NDS}clE zN?3){sEARrN8!bc`-wBmevii&(MZr)BElInKYai5f4BBzc$P%iLV$r8L;e4&+~WG5 zPU*CLMNARE_~53)W`|vIMorXryG<`G19>$3hBR7w#SnBww9}Dtgoks!y+bI`vmc`>n(AP4ejp$O-B z$E{y_lfq7ei!Iwb$qjH2|C%>w=U#;ZIYto!Ix$~rnWRTF9 zmCg~y2P9>uQ`Dg%z8=6Om{H|w-3B84g5PMa8$iA|)^k<6<$|}RWLiiZ=|rI7dDrv) z?@1}^cHIWGKXskVxG=T}fY|(X+mXPzb$!(buEybA{uiPr!dRaT7l?@Ht!Q6f2?c$R zoRmnSLSe~R*w|!Sn*{_h55D7gEyO+|M|EChxz5@!FDLanqI!pSCj$Dk9Gz$W05L&l zfRKdHggTmr5Dh^C4sT00=}1PW!vRXD|H!-iMwgtn<>26G&ESL=1i4B7_T{`_P3<TO|r&A{Mq$V-f}GG&<2FJGyqm=rx2N3=*>WQ@nC3ymy!*6 z$#XT_jU_4@38v$cwCMDYl15>o16*A)m2fP;f{(~C)+ZHS`{3b9q<^cybtAD|aqR3Y z+Lh)^%Wz41*%O{6D$VT^C5@?)kG>`RiB$(Hesw2X#QkKi?81$;;N*S+7$3cijDP->~$!wc5$0qW8KzGWdGWrMl^5Pj*tlK{8 zlF?%oGe|Fa=xFCr?7j~^Y812CYPub3BI_AeD8ehjkl%Q>z$qB?QOs3+CAL$l~zJW|Uw+B__`m{Li$%)HAYat%>;W~rnZ-{b9r36C~kx%J*)pu#WhqeUuFxP0X z6q0wXtZ`|!y3Xw{f#px2o>lhmNoX^cC(dr@T8B(vj=HWpi{tWdbW_d)gTI9Ci*d`# z;F74R{O0HKYD&I8H+$~m*1Y-sgXW(|6bnMA=8JCfN?ve>%`S1p40jixbw>d(m#RNi zq4-1LEbj9d7axrn$q~B*Bu36NM-dYblSHe-HtK0Q>JD z`9JVt{C{=3vOxYnaC`A5o-#!T154)u1N-s+{rrFZ0u*y}a5MLE6ZtWjWkwBvM16-tyGD<7eaFDd``(rBv%B|g-ar3Vvr__gnU(#2BBy)4f4VJu z&k9`s+qN^YZBJ}llic^*^PRf&onQCdTh%|hc31!DT6?dq zwRiWvF(N`vUd}R0YTr43@d*9Bw-PTrHV7UZzw12{GjGO?r|b;r|;1(g29ajiq3`>wuk`N-HI zjGM)%ByGl_=?awhFq)nS2N3@?9|UJys+WNj&NEhSruCI?FxJ=;L&7;LmqId;0D*#r z2?PaBdj&4xI_>`TZq%>J^H%Zt_%eD`*P`f&FAKDu3sUa4W&*#IPyE*a2iu~^oM zywf>gT64q9GGlfc5UF9Tg8wkEDEf1S(Hx1cjNx?*s>Kq`f$Hn9D4I6fHG>F3eLNP^m=>4Uo$Bo>Wck@Vpfmp{{MmIZDxu`%tb z@hrP<0e@Dn?&((TT}WZF-1XxK*8JGz(?<$Qg)QZO=f>+*bF>Dnwg(>#STx0EIdgm@ z&aq9k=6mlJCn8?_3R6Um#m};DM{2Wc7az@e)pP}yH%aPrmcgS6tf^{V+BO4NT z95eu2j|dWWVg@Yf>kKoEeEI{fMp*R+zMgC$W7dAzUHoIm3R;YbT*__#dtzgj1P1%embxM zt{3MS0LPlkORHR*%=)CCfwZ3}sMxwkJ)#Y739-QgQgzi;9^BlFwf5C^&hS+yA}!hWicY?<)JkGESo_ zu{8-egy-)%rw(yRfxBgZ0BYvUljjz^FKU}lCL*T;B5nq!I-R@NOEn1w^gol_Eb!yXsOjE*?sL7fh|Zd6U)fx%#)?pZfIs+IH=Cu8Uo6Y=UVs5r*%n1YgyyT+KAC!~zH(-02edlnBa_#BsHfNcGTwB*wh*MM`?Jf_>!2 zZqikUGQGbCC%!}KdrCzrg>(SLk|v#3Zm+6UERd!@0wW8aOY}z_F5+A11*bSJA`f2r z-Z~h;^nwj{&R~m(2=+Kv_g_rwYtQP1B0j)0Rqq%;;azG-m4{8tIbOGTH!XGs`EZdD z=I{EOS-Kx$nGQY^DM-1iZ`B&DpqgYH|6c+I1|8Hxc5aH07A>KiXHsS@ z`hO0NK75a_TP=GtSl7lyub~1;uY7)-9ly^<9Kf5zh;o^u5!5#LM{6m>vWB-JD3N#p zknch+O8DtY`aVxse*UE7pXCycZgNAh(TSI|w?SEn*K!B`cBh7KlAVX&6uNi4|DY1l zaq_mlgQd5`V{{Yu_8J5Cf2F#hbvheV@#pVUaq1HL)R{Q7<7#QsB_5Eu>6( zGXj5GydN!7o+%Le?rEt6kL2BE|9UG2a*LGO55x;=@;b@tosJ{Re(jyAM@`#QM!Xth_WxV?fOj}bVu zH841UPY+_#s;!nH9%Cdn8s6MJtBmRu#F61!Ke3~hc3JQx?azSxBxkQ*YMgofATID6 z1EDW8#`s87;7H)%Ye|bu$G<&IZ~9D;iFNf|oa@gYlF$0nwtCco37$nGRN#;C;9YgrS-|Ot7g4{sa+;}#cn;&%#bgoEb;~} zcJtMed^%5Gqafc-gVCqTl%L3F;jof0`75-X`&il8Xd?v76S2EeBY4#Uhe>Pfo2>A3wOa zR+*yhV=gG5V9}$wUx@(sHt0v}xjUD`E_wUJi5rYYnCpY&1&@9>%HxN9?O;r-6iKOm zZYTQc=?mT8HAc;Gp6TOGWmv1ylsnFMioGSXB6b4^ys{#2gm}@EW>hHJfTj|NweNsT)2cA0^;k_BwHUHrWZRNByDZps>fX?Lr!AC&QM1aB3 z4=wKG5ibu`=umz7AO5cyDdI+NOCwnup3}p~u7ayU)&U_9rDk!wm9rf5Pv;RK(5!I^ zHM^XSz=XPda?^)&23j2ZCi+o!fXcJ!H8QqPlI`0AqbAfXIYdrry#R5&hPdt z;g?%|lmYcoZK|*Zb!8lg%2g31ZcsgHlNGHt1(W55;?`;m--s?c*O4k4<#AS+A4JBO z#p`@xPdOvqRX-FAunXpiisnZ2Ab`;!w_K#?xv<33l%*Fs^=QH0dQt>#*8MU_IUc^L zQ2WFj%kru5l}#YSd?;URDwfO=$C8HBl3^%bY$}rM81wkiR8hb*R!qMNwFS3Ovz&_5 z7T(2wYNG{rJWs2MsLtf%Zv?O%+iDl!IiuLEwt678^~-WA0&8um^-m<5<8fe{F+*=; znKOgu0eAPW%{%PtKA3j3E4g5k{?l4Q4e20tmPQ8YWtTB^K|WZWN*ZZWy{Dl!H+AJL zL$xlR>Fmjq1GC!0VAI;e+>S()@Qy^^%#O+F^Y&)kuBn}biehJiuB{jbi|CH+zAk;$ zjY((`9!E}Y^^vxrk^T@DnUTMsm{n>}N(0BfzAr+wAI>2SUy(D)$iAWGrBJNMOA#7M zK9XJvS{x4~2!CF2@)X@u_nFsF*`4m&K#EzdU>d~q+9WUUS=39=Cbzj3!?f_;=~CA$ z+JuyGrQ2jM>!2{kX8S^XI2&#}I!I_uEQ&@f$woYAE-=IYAd&s6sR(?g6l-Zix=)}H zNbn-jhiC6{_rss2HC2U&SRf+m3Yg1hdo3%u%3B2JsL0VPOj$Cae`8*Z_eXh<5^ODY zr^!7Z;Rx&PvRE@i)Vcb{?ze*5_$-{dM`ni086F;EGDb+u7G^iSXE);9gYS6|Lsi0N zBG1bRHe282a6+`zKWae}K4L&9p^b^GJn3`czcx8)k*8Wdh6=DPXZ5j8QDs-Cii9x8 z6THGSqxF~+foITDS7=@i;?=XrDu%mBnAQK}d9&bZBaoLPk@j3^UBF@ME+%klWOot@ zyPR?;=?3F6t%3V|Ax(wDxL$eL-TolO5F&nCI@M<*i2H5JDOo*I6px=nCUr2c%LP!f zA1QjWEw&I0s-AL~*XD?v6}Tnj6qq&ifGu-AOrNr80S^bpq|FUfv_MGnn)VFLpN8@t zIDqh-DBdC$N$Wfoj}RyGNyWp-^azv!N9B;PP8ycWqs2?&PXax@X2lTFDO9e{hRfj?CiU4-U{ZQ zA7uiY@yq8{0G8TC>68SF55LlY1`uXkX|_wK;hq`Z%~8g1E8RM4mJz3}pyik?*633H z@blzw*b>rcRANOB9gZ6POtut3pH>G0dHt&x1Rj?e-KmaBs2gLvok6PC_{}&MJfZMtw@nAzB*!xT_8uH4cbdWLqK`9v z%Nx<3OP7L5%9%Y5kD&EU(f>pm;*F}GAJ)QxcF)KQZk9KwJ=Xg@5_RP}Rgd^RCeHzK zP8mIIMLl_oEYmw5TOU}5iJ7v0j-#)g-0n}k9?|@#u1HSR-Z9|6QU1Gnv&pNNX$$@1 zhb7vNAL9SV>W%3CZKYJTaC3J0&zAqyklDryLjyO!u~UJd)ICTOUN;|8CbJc6hzp7_ zg({OdaRVFZoT;!`kd(p75u`axB%z}&0);k=%`zXC-AH5iRRaI`*j!s6}X&aazS>1B+tm_PyH1I9d`1A06J5;&f}X#5?^zU z+C_MApRz)_70od8IIp=Fcy)N^Xl=-Zg6Dj9?Ry$V$)}eX!eoln@%puR6e>*d>*bc6 zkV^LIhY))C)daV*|wUv!A+`B~8 zMS{#|4VX`eViIw~RKa~L58hUEw>IMoF@5A)p%&oyl8mG9;DQuG2JFe1ohC=G@>Oos z*P=g#OV*)~baP~GboDcZr@AGvIXs?|bxC8GRO9$=BF11^^tUv@Q%CTMWbIv5a|YbrG7cw8(es)WI9&GxdDx zgs8PxyE>~ggRA}Z{Yef% zR~n}y=+rY5D@zFH8ke75Ul6Ua380q#W;QM}Vps4fBd-F=#j!#|3ecINiX&DXtLEPm zCe;Lp6Q>}F$KEGeXrGVUXCWVeDNPunDVvapua_6`Uyi3tro-tr?y2Z9f^DsHkUzV< zGgvU4JD-hU3owWczy@zFsL3IoAPl$V2X`F8EUf57Fw$9K{|Sdj6~{uO`KJh0G~~gp zC!l}OZAFKeD`0*-nRAZ}pKg3R>kCmKP>sl7mT6A)>av`;oC|RKXOF~sIl(82y^!7$ zyma6BMEh2QOqaQp6%jKw@=Gn#gG`XTA~?~6M1Oo-)lR=4$xJU{r*Vd2LuAnYLhUX& z+<%>aaq3}zhPAD6j&Zv=B(OBhgRkimU*G@iFH*kMD`)BYEMa@ z5XLEYw-Q5le%Ub97RbTCU2({}#$*EoUqoMcij)v=9%bf}PbfPbz$QxbT6sFI^^~u~ zF9pFb?GlAO`P9z&FfaF?vuAsVkMN<_=VQ`AlR5DpH^^F@48iXnXJ7*XnBNm)m&@spEKl!EcsU9C&b|4@AL7SX%S3bWxF03DQ zNdjJpkeHhnCboIzwl+K=axW+ifAjw15zYVB0!FQ>M6Swy;`;Xzjp}xb&M57o*3`OA z4gP{#qkzFCaS>tG=EP3oxPY~WcO8J^A;WNPK{PVL>DEeLxb1SM@WC+QLnN9!n^8m76svg1JrxBcVMT-fnMc?*!b z;e6(G3=g-yJ}amy-zV>FwIRg4H;^AS;hQ-72A4b_XL}>l+(oAvvS>b#lS2Ho;SJ~x zNpl$a=N-C#Quyj(T+5Y$hsr=(c*MDn+QXgIoUjRdQxl#13HC-k5+KTQ60}Y_;Sel6 zzRZ@Db&w2+j6q|4?a&?lay1VH{)#44+G5Gx37xmACYL*V3)M?sH>BYgd~^CH7%;@e z`Srh1*+EpLPU8PaWc-KD6#w5-nWVRcnTNZ@e^J^0vZ`CSx&P0P3jQal)%=hAAQ1H{ z%(2UX2^=bV7pr3p+X72eW+Ohn6gSZo&H;7=rm337LU!H3vkm$y1sW+*%BY93_acvN zW_=sId)cq-%OkFXy!Yw#OoCn~U@7*!H9*H!grr6tU9xuxaGz{ER5 zd%ju0DF}7^m>VO4qJTVO0IL{w*1oKEUD2dN%nL+RQ<0@9-dzr^X;#mTE}_|9h_dX~ip9vp`cp&FS>-sH(Do%Ac_|%NB$^bdO|;GPNv@ zjJ3&jM=|sT-;yPZX>`!AMMMRiz4h3qY4(#CABKc@VqW}_X~d&t`v8-x#05gVeTY`K zcnRS!i(w~J>c&7AKX5uO_f;my&_5{y4qIg9B`wRkK2=hrEfiYBS6lTb<`HMDBXHp$R*AqdR0x2!`{;46*R?Jl*4I^ z;x^ln5G0n<=;5_)Z}ls0<&;fMu%(`zkZ%F{WHGwOUQIYl0ZZ0}Ocv{EWX{LCI-0oz zuC-gXJK0dQ6ouc}c;&V$VuSgFeb_AHc(OTZaVnEMZ$J{f)-a>VF*($+Jii@nxd{d> z#nwLqfqHV8$$p3dgB=eONO4!_dO^y(5w-vn_)Q?FQE+4oyYAkPJrHt2xi|cM#fsqM zTfm9n60Rzovs_(Fy90A2lpLeu({l1;46=i^qR( z|34Amq6=rNzUm>UwMi1w8;^<`Y=8S>AR@td4TJ-Ss9TREPs$-iS`QTBAYrrQq?jGp zXKGNy-ds}x9Y9)nA+Ev6vf`2z0jO<)H#u!KI&HEz6|=6-dRZz6PP)IpK~fwPlw`Km zZ{L5uce-A-y{RHyE#V&W}yXx8^!bQkVjuj{BM2O#AjWp@W?_AS(Njj$V9tzSa`4fLy zn%o{zYgbZtj;FK!N8`pfD6Fd&ybmNJb~N}804OMoeQJmc-}0;-y9w&3Xa^qtLxK5j zMAuh+E#mQw`=)mmkrF@kklUsln=$(r7pFdpwg7v89c3;dZo~+>rSrotJdUMzT@>gk zFpnaW+}(|YnJ)=O%riT=umMqy;}Lyt_o-2EM>8{8bh+xHg%5^+Vn@CIW?~;tkmf;xPYg#0H{ag4$LoT(R*b;4OXR1O^lK!cU ztpp@T{qQ5X3#{LE_iz`QAMJ9Zwp$WC03Q3hHb=CrOr&Iy=0AC_yw){ziMR9JcM^=U zeiHAOf@hqjF6XBs)9sRjH2_EP;N4?}Y-tsNd`Czi zIy_`MYb^ahzSGVQ3a`_442P{I!3OJZS_voe%yvHnGZ< zXzE-3>L1H5RwaFi;%Qf4V9N^))|IR9!uXnh5s(d?u zxtDZJ?+f5BCV64?u>sD9)t~tBxt>|Rg=~f2orD}ErMjkgW+J2iNWJS0BY42D$-7jJIWz;wrIt#0*4LXLKcP6G0ff7XQ05QXH92eyIfRAf~&{cgv`%klH zuL8ZSVnYm=Xe6DT64@jrU1!5CdG|-J*luyM!{G2pEc4^PW9g{d(p$WBMqRQ=`%q}N zyd-UsBNuLt%}qlBjXV!DhW{90FDYBL<9o`DYTa>(R+I!QYDl&y(lccRF9Hk)@wg?b z(~VMCq)f8GEFP zPY(JEx2>^8T0=S0@_Z1GDU`i4>EHx=#S6?9K{uzbihgN${lr~r?%0lX)v6ZIPBhV@4-xaZW`?I)=r#n*V zL^vlEm=DEY{7buk=I+v782}506f;{EnpZOdf}8rU1e!I-41(54C%PK;|7z$rznH6Z zC)M=Kwd}t8QuzFuRzd^!51(?Ues5)d8Z8}uPbB8I`doI%-QCdJe4~&H3XeJ=hK{>) zFQ)DPQ9t?C3~~ki>SLtWuxs8usnq;1>OUUW)J>a()-L}3TfVIXR0G>F>zp)4h|?D` zKMJ~B261j@VzT<_kAocP5=|B5+~?bT;Yj4>o8d|9(L(-bG7US!WD*_d zTO+iIot0?)lai+z=?vzExjM4OV_RHHHyyGTovdGnc2EocI95g}#hv65V2@rUHAew& zHkRh4lk!O=?&N7aASFjl?o$B}k03X(IpaK@5@4fu_E$xr#%_+bG65zQ1$%bVVC>eO znha6;>Xp|FGGyD4lHh!0F6>&~rb1)S0vy*mg|?eT@c1Ke_;He23&Kqr!-gG8K-eTu z;iwg(F4Y#GAxo0yw>^L#=v9emAoSKxwE_e;a6LNon10`&GDw}8uz;wObkBuPNYh{<*>yJ63wke6js8Bhf_h5>@T0d=P^Q$ z)T)C=ztA;xpSrP#rwTdt*)0khEr#XsWi>Vaj?BW!^=wsV=eLM}s^oy<7tK_#z|s?7<_A@HlulYy~CqX=5hZ``;fsel>z(m zNiF0+B&WFLnsf+o+4EtDsSV)2Lr*`+9CVv`?Bq7JK>_q;CmISZ)u3l-u#W; z^2q=`6*`!;TXw9}gUVMyGG5o^@t^e3D)?s(2@!ZrvAhaDD^tG=qJ;pPl@tF0w3F)C zpOv2F85Db&wb}Q8h|3i&RwK#Wp7kq-Pi!SecqIWWdmw%mZB8b9H0)t6BqI20a3>L z`#M`VVQ8Qiac|qXlACsdeX|7i-2)#yK&~Wr6uz6(tKK42u;tw~c-3`_)=Rgt!Q}w!O#nP412`zkaCW_+3w6>nVt(7Xp5b7z@WUXxJ z`LRYd z9v7zlE__F|YN@&;cSwMRUr=PTytEU+8y9z|2xp#e0(DYFN}Mxa8=uW!#Wc5iPz?No zT4#29t8bXAAkz72xA!jt0aKB8`JR|Q%XZ^5dtYs{kLS4OnaeF3M(E8v1bmIfb>Vmb zau2{PP||Zu$tutvb+UYTi*r0oRu5VsiU>&lGS(A2&JJU@9?b6w!|%O4Ec^55pARSP zWbWa0H$HCTCf+CfbUxxd`IRkL@ar6&fD7Sur4`(ATotziY@UErSu@)kw~;^KDGDi1 z1IefvpOL4f?O2K(qBtwloxR3Wwg7Z7y=c1ZTCnvQ|12HV4ZnHnMP47tiFKrgME1JBb zp-JgTiPR-?#Bh+v)}VWa$p=IDoKAh>-8U*Z^^15C_$mJSpr$i(Zywe(rN|fku2IZ! z#*8o3qn8t<87OGY2i^8)r4i`#*Y5m_ z+QoeOF}MHEJekkiECyaH<3##}^@u)4%~-hmVUzCIuX;w!-vc(G7iwhuI%k@Zg%o$1 z&&>7S=CI#MCCtQ=x1pwd<>0v#D@#+A=KyjB%`4&f_q!sdp8~FjBznqKQJnPu%$we%>tmQCz?Dc*4Bn2c$50 zp(zyyBukaneqI4_a}{_a_trbc{lQnySwuP}Ka87PoEIJaW`c=Dq`yrBr6mMAcq7yH z<=vzjeLfi;*bEtEh~wD^>^Lfn6vR~pQa})6>kW%c&d)bwBV>91SsxwGG_SDNbnk95 z2Ka^;6dp9Iv+%4)?h_LkN5SZ*)rI&j#UpRtzQ+qxoYx-@VK5zwC&?Gqud>*Yuh9L_z_C~szmjBX7VaMZ)z-R4 z7r|I<)#oc<^B(J&ix(Wj9^54Y!+u@r7)^{d4tJB%(H^3;fvq^+?#wiLtTT!@(q^T` z*WIBl*}-6y%iqgRY-1rdo*b4D&VRmHannI()na?a(Jrmk*IloRb9-UoTlhsT^@xEH zf?%ZcB-{7v<$>@`URd~r@fVW2IhdSEWr0FC%uxt2gAj4OsU@cg{@tzwsbxqJ8b<;w zZom0j6tpM*!!;d8x?CoaZHq~tAE^a(B2tbfr;QR7%(OYZ)a|Y5Cv8Cm4~h*hK6@GX zxr1?>-?dCA1sTT<^OUy-qX?8}p6XiRctH_c(p+X7rM03GySSRDua>dVjAy9KDljw@ z(rFC7l==;+pyWqlLy;^8+8S9N#IXs7qf{_=tS!-M|7am?DqQal#YnIesj0*$MDVyb zFQB%SX&x9{5^}#y1QEOq5yF4GpPHdac|fOWO2Tr%%iv7ewAs@nztiHWrKEel{CLV zaa6aOg0l+)K@W@@WTOz3cUwe*FV99z5+~keiq0i9^h`RCI8wSEKxNK-+Kxm3=h30U zv7KCBn*Xf7-3gxM93D0e*k~vr{iy9ES0{?qOY*m*wO@OP9a#s>&ZgpP*HlF-pcW?a zd$=1*=PJF&$Z~L^qtGatF%I~79N#bFv7tx)wpS#urB3v8iDo9+UK{5#-%_CG3>oGC zSsc{%ZJ*BH0*GGn)0}gD=E;Ro#KN#?^Oea^30g9X=z>1Q2!AXQ^VP=&;3srHsBb=B*wefFRDfLD1d_ z-;?b{!+cC17+Ds$JFDIV z*%?a$j}YS6J3TnsS4i3G?{xuiV5DQLYk&jerr(#VTY2>34s4*5FaA;Y;DFIS4KF%N zR2+R^TZ$7CjpyO{5e3=f(aRHqdv;h}l=G6AWFVaDGa*)ZL^{gJ>zbQ_%|r~vbwD;V znPk<4P#F~)lm+B)q-0td4%>(Lu0_Q3$|D&Z z<;tGBgY!$|?|mcRO=Ykij`b|%O)VN1T^1?ZFD)+ciCX4l%(;~%QkJ_og_Eidso0}q zu3AjsEPDED=Qsb?mh~e0h#VeRIrXPWpr&O3BoOCtgLFj*_dd^#MQ7Y*W^zDL? zDueh|rn=1|({{A6LKY3H2GKsuqKSR*5#dnYvFrk@tnT3bp!1e*`H1pHB^|+-7neLh z4dM?8>}?$f1C_Cig22{=DV&cJht$Hj>;-Rux`XIf^phxs<%8CC?>6-fZ_!b$TaDk` zse5lC^8!X0U*8JZ-?MNd8W)z#wkh6dffaiffG_yJq_5G+bcATO8P4_A%pB?`KIJ~R zG+dw4phdk>WxD>q6Z6+Mu+mX+Nrmj@qII_T&-;KgwqB;>@1VI;vA17((N~fwIK(Wy zlY7P+X^i3M{ocH(0+`L4N+iG1Jp54pT0Obu;qAHlnjNbYA`^wH0Y6$T%Q#fp;J?YM zKdUBOpnfKU_bueG4F0)zCHo5QXv~h~_kA)R=HK@-Zy(PICo)+dGpO`$yE6g=gpjy> zA!PJNpz(0P#46mVqIzHF*4nJW3^2I5-D7k>r!UUu2nn4WAC&>VV7~dM4N3)LjL5v8 zWUy%D;SA?*-NY94VKOf9q$s9lxGN(3JF#SEirH zLI1K;&||TCS3O`3#t0Sg`#(NF=|wsn8D|YFRt!FCDLoe{IKGz%)@i1OtZq%RqYqA) zc3a3xAABp-89W0s3?&%i+VOS-o62$oSwGiO&nHYH`d6^4m_IV5&Z?FYKQg%W6|2yyV4grY}J z*jPbrOM2NY6)j$z5)V|`Xu(R8G+*;T^w6W?4)EH`L{C|iyRv&di#Y;ae%XURZz~oS ztTtZfts_qN0<*!&-Tft591_#Q9Wwv5E=EW|cNDg0sgdlh@LlKRP-*^U7D}FX%;BDM zSpFp8x!s7bzFMXeaadj&LHIB?Q7^N!ttiF5_E;MJ^*%6s7yhgX~$QB%pU&Kj*N(?K{z5c#%@bcI3qV}N^UyNMfh0R=&PLw=Z-t7I%N%{-I zRDW3mZXPiCAGd%82i9Aggsh@1rE6Mwf($bu_o7E^cO2h+Z5v(LJpIsHNAB5VbKMa+ z$AT+%TVB|))3bx3*$0%p2QGsKsOERX2cPBM1LL_Q^&`~MzE5=@jtfFB$itgVTJWMt zp_Yb($&%V`-5Mc%8(p^$Q6deuT3tLcj0@io+;vGxeel)31xiFtBJNvFo6{k~wdIvd zQnihTmohTdvG} z(Mj#Nn*xZxqs*T_KJHv&lL*Jy1Ae70B27KR+ZSQchYiVA?#^Q`*tIA194e(oX)eUX zs{h-k`?R0S6+N)7s2AqdrS?OKrT0aOO+_dRwwm`MSN;&5nhy|Paes18!eG8}ZYHtD zZnn8hu(H6XxL;x}lq0qwG@c*#nVbz?EHE9tjnl#Tj1$Nvb#)}1l5=bV~w>v zbtt)`n+&qSZKwRK?88+dAtdT(ATzQB=+@(A$jti_n-y81-e=f=HWlKu>XZ%wjFt?I zug!Wx1gY`LF4>~dssbpS6b=gfX#AO^6}NS`xaOEzGp*zC5{X9Fl6F|h)ma8s{otMC z4tVT|cc`mUaM=2)oKBWG1H29~1HLhdQ#}5-7NR_JyqeW09~4np_e3lUykiWwRB!2z zEM`ur|GlhWs(LEEtJr8fk$}xw(aFM&b`J=-6X0L)OVns262u*$+lppO|7@;lO!x<< zJ`Sceobi)vHQ2U#pKyKR%!WT)9))R?_3idYO&?~>klq>@vB`xw`;BL{8lRECi}p&u z=q*;K#@Q)uO{vtVo0Ih0z5E)8-C`u1%XXn2y?AelYGyHaOT$o&rCV51cDgyIilI2# zS>PU-TBR^?T*nXaS1E7ojsc|wpfN6PS4=-GGUgu@e0$NY$7(-;*~5}OoYnzZde?%+ z3=ud#3ngkNdr+sRJTXTzcu;PhwUpX`q?>2tffaTdO8?$NNWc#^VBR z=K&b|OyhlWus&9|ztNo9LQUv2;o3dEr~T+fFcA>;a~{pGdq_1ks?eHspR*Wvfw87lLaP1xeZYSwJM{D%Le9e)=w;1(0xZPf$&yA*vmaO1!&o3_Ap zs463L)jVkMW0^){`3O*oonX2`9VTcf0F)T;#Ts|34khxdcXL|Pu_bnR|F zdjB)3uY5V0vZk+(3y)wvh#q=gzHa<)GWSZflm%dngL%7Pea6yu6M?_Pf?7J+Fz3uF zv9+)$1PCMO(3mPm4uZ;SZxn~Xb7ndfxx$~oz-Eap@)GqK9gj3fE3L)B#EC$aCLG4X zcDH>o+!aZ8iT;$2y5|aAQdqV?_e8sOiPO_;Yp!{SOtjP8%mBn z7eQvI+yD;_SmKw3yeZww+kKgSs z$zY*wnj#W}9xaIio6WWNX)K12g=T6nfhNKkSg10|53Zx zKnlCz;8{N4U}o@r4A#IyBJXrWlqr?9W=;#*{wiJh4~P|-5rL-7!Q0RXn-B$>jnYJ0 z3ep$S){?r7!bJD?iN1q`!ZNmK*uf-46f`Pg?1mqb{s5r~=cz)bQRm z(T?O){F}|15-xwO2qv@$^8lP zyRV10zX|E7jKn14GF@TvysJINZ@x7tq*81P@f_-^7?1|?IuQCz7^h#PTfu$W?|W+q zcawA%AV6Ag5=Nv_)v_fEhQn6D%yMeAj;Q0_&JGq-=<<&$vUg${!x^^R9a;vJdc&P43&V$y}3b}!PrDjs^gp%g~C zemtfON>fpnu1-+{G(s=*4FUlPDW;ZM;{R3DO{Hu!wvOD~z{YY!V}Q&1PbZPM9}Jq; zWV+Uh#b`JRNc)Q`@}-a(W~0;A(vh&B#Y{b$O+6pr{c@#DwilV5^bUepbU1$&=V3lh z|A(n{p9bm2r+zt^GPix|e86E&*tO?g6z97>JNXLUbdv9=$!QQBl0pm9f&#hIP%LMlnrqKw>2zJqFDou6&r{GZeKD|fFrbxR#U zqq+6?l5{bmY}Svv_h$T62;o)}bpmes z{YQoF0{;Bc^N`FRTlc1?_LIf&m!Gt`K3DdgQ+u#fc?e2I!md5qBNt}Isr7WhibGO2 zCdj>$-@-&cl^~j0Y-#MYU7wS&Wer1gn_8O4i*(l;g{w!Zv2_Oio^G!Wh?fNuRnR0K zRZG=p83I>kG1Io)bo=f-9EDb=V^u_mIUf_umtZe*91zkpMy(F`L)psQrv7tnDo&JD z^e220%nWaow;R$-ZB*5;R`!Yw-<@enes~Q8kuS^p&7%*sL$2+DA%5fH0 zc83;c!hsbi`w_Sk>tu+M4Pu8YP<6K{M)HB39j6W|vAMqU5dD^)WKQaCPbXMwo~5m7 z+Nq-+_o4z?6bFh_6nk!iLKy6z0Ir_g-_e7S{+NDz>4}yEgTVibwRei~^;`OVdv?#Z z?e5vOZQHgr+qP}nwr&2}wr$%y|Ml*@lfBl(N#6IIt0yCQ>Sl~orRtNaZ=H1dkrk0e z(JwJ9yS)`%E;FE4TxmIN6|E{7J{P)46EcQg&HoGXyKLS@wRAcZ8G@reXIn_?aIwjXa#c|XlMcfNlXlE>dQ{tZphvwXn#q6`Wm$MaU`7cRS};~>P;P-mi$6hl zhJv3wo0yG?Y2|-&-~OV2ohq z2t!X7FxZF7rGW!=N;_aGzXSE`P=-YuF&#WvgEfY%kD{U1)W=R7q%W5|pltGf5_Nur z6arxj=eW`AS5zJy{YXIV9d=*F7g{e7Jxqmv4nx_9Id$G0&JTmtqgrhK8A=OZU|ugN zhxJ3Pj4anzr1rAc8iwFZUh3YVmK|E6O&@JrUFPoN{JUZmp`}%y30e|tOB_m)BQ-RX zH{c*{<&}PKJmh^$@ z$knFwfqIfFIz-W&VUDb|mdC8!=VONI_q7JSCd0G(#x<_|>a55ecq()Yte&9h>e|w6 z;g}G5ZdlORHldT~_jExFY`v+#OSSd1%xYp88-=VI{JBdhiTNZ~SN_YRWJ-d2R z>w^JW16ZQdukl5JPhf=S4GQ=KEEy}MAXMf18o3?nR+*(YSv0Crtzk8tH>o~gLRWiK zCm(wJuK5I%`)LTc!g=f^+aYhg3q)*X+Ormik#0)Ve!h`Fgb6$yS-~^|Ec%@!Vm?49CoOd| zwHWDd^pZO|bh9vb=8hpRE0r~G=>{J3jw({)sTEl8&) z56Q#>IrFp_&CdwD%Dwe!kPWjE+)f0!W>u2|ato0jbzPK8iTdAC<@&Jaagkv4y-7-u zFxoc&6 zso5q!v^RA;Cu(4nT|x`x((6Wu^-CE*^|a|xoW6ZI0#?I?$&G$xXKJ{XiZ=i`C<27hY`CEuXp?M~rFWuhjZ1WzU zpGVL>97?QVjAN_T;*FTNoZXuGe6G!;lw0m>?D3}J9;9oj7UlhFN|!DLY5FWi>K&zl z?31Y$3g;O+88LSr;KE38r3P^%tO6M1nq8Uk^eL71c`TPsnKfyx7ZlLur_!48-+HZs zmy`xUnXyDMK_SLSiK}Du3t|Z4SLc+M zTi8vM?w42=FWpOeSO4?xj;E>~be<*E(9*sl3OHs{-G{JdLncaYMVg}W*K>#6g@B3x z*2b=CV5eyvd^WNL;IS+tz#w=ka%nQRCr< zlMO^5oe!AyH*P^}z5Yf-54vkg(E2}l4`slDy>a}49cg$Y`-olFinLNsy&h!uz{Z7) zEW~$!@Y*$3i15eCR;%8P6vR2H+o(z6;H0Y6NAk_#k6raQFpLM&x&hl!Z+(0J0#a)g z*4FMP3N1ZK<4^ZZa}0&lh+6L*O>Hkq$arZ%8^#EU_Ocy`8?YlxYs*tPCM_IA1w*hU zjKiv8iz@+Uf(b>iRzuqfq#Qs7iTCnymY=(wrtq)R*WesT7n8-Wy#zdE+rs?OI(8(Z zt)qmDeVh3bFz&fr>ji$tD*A%k72dI8M6?=aZe#ZLV({`kL_oZ6sG?(MKfCAwmu#=> zK^mMXdo|-~=+_|+{{_Rn*=bSh%kxv%Q zE#i01o;bo%^w6Aa3Arh&yQ~wT|Dajj34Z*$pA$cTlQ|yl5|UKhhTl7D0E8HFX z!5)vZW}nlY7I_~`f=bx1O{CNn0ie?WZA=$W6R$(;nR^M6bceCP_aCxj{*n1biO@hm z+;~7hV*hP_ldy4gGO#f+5j3|kHn%bRe{&YLAwb4cd+r)oxV0@>gsrxh=1ui`Lg}d`_gS0lB&z|jpG-+ zKS_8aSL$YRFvT~zWhH`+glg)Sz-sA+8eT^q*JXaz{D0542v zJI-<>r72UvC93^OwO&0czK5!`p?Lkb& zNk*WE(q7c^7;HxW206~VozyCgmGztX{gI+Y;UKaY{slSU(=+nHHM1ZgYl7qkG%fP($E=YbGFY1Go$2-@Ih23^zS>*^W)Gg`eNNp@iK!2t) zvhGuN^u%@3$RA`zkbYTkfj!TX8dL&EAx}ox(Vu{$6FFK9Ei;|2Nrl*!zDz@6!Jh;I znOCdMGTKG{oKCnDr{6?zjjrp$W7UZaxJNlqwab@aIf?M{Hp9vwwM_gKg?>9?$NoUk zaJ7-{Cd?mNbP=+Z6D9<<^jc#bGa!ZJBKt4MaW|c6|4S1zVqlCP<<|6KP;A4*=~f|C zw!aS>ef)Gk`ZKNEb2c!N_;GwScnkPpq@U|W?D*tV;*>~K9eXj!Kg@z_Tv9KkpOH;{ zb%^nu;}6fAGQEc##rB^)M$A-4-Lmnv&o~A~LDw2giVOzpUk3rO52<7uW9Ef9>#aAZbm7{bEC5Kf%&E)J;tlxfRYuXB!YJWtxH`i8|wl+!ZWG!vdK7??fJ0z&du?!&%3*gsmgv{w`hQ)P| zAt685d2}e`T_e1H)85O38n`C%*b~o;w(_w1RT68{97MN?>VLtASV{ejr!UM9;`n1q z3R0i)zafY0aOSNkRN>W>Om(zQ|1<9Yf*h7}ix4a07xsZc*e08Uc-p+Uzd#X^J7g(~ zET~ID+K=ns*(TaRTNRHmn&6+N$sMy33T%_@xv0)9Y^^Mhqq&SX$FSdYc&@s9GNU@( zuLwp8iFl@TwqWkAVD4~%J4-{*irS7B={@bJ4|N!?P4LMTpX$8xwX=4J4!YZ7|BN(O zDpM_i@wnk{)9ryU@`k}fT4fo?En3jf75FVkQ_ezzsk_|gZ!Kx(%?FO-NP;=ClES*l z4-0LQwVxSQKL<|q5i#Nog;ID>Y*EF#fr*XWISP*qNqeae8@-aBaa9e5LKfHZjf~|5 zktOg5VG)mH;Bm*$*l^42F?)}7ajI})QKsKwVivMX?v}^qM#A52W<6n09?!RkO$ zAja6x{Z_RH{>FlS=R7H;z>dN3)}c%ERT^%+wWhSqH+Eq)jAp3dqjF$N-(Z$K2uCFQ z@>Lstx%Fl&_g*SLZ{{(ROBdz#hHAR|k{^y$_j3*K8&;R*__Id_Y*|`a-`OVaa2$8b zB3Y*6t%_OS3FuXx;e;q zzOpemR2G~|QI4EM^U4Umd(|q9g0@qs+3yH@v!NyzH{XvNgGn@jfK-3FS%RZJBv^Z->d~bu58l>+SAS$-rAHY^G-?KNDBHrH-YMjJ8RVNW|{yvVQ(c zG5eg=(6kmwq70LcpSRtOz`E161cnHm8d0pTjIq3wY~zrG=U)pY>(`f7(!1Bi^qG!DY!#xa+*=TLcJ3NbGG16v*AeE zVW7VaFgqlw;gHrMcjgyBTi8G0dcmvK@NdO{0eRmKhfNE;qb5_S(fJ>kj|^E}(9?$Vu&c1>?-0{NP1hq9+5ypRivbODXbAxh zDJB92Q>!8F%Td*7wo|Jyw4dR}4UQGLmuC*tA7)a)z3i02=Efem)dS?T2A|KX$5Gkb zlm}%G^YVJ4UUXy}P{#&9Qiy^i1Lk0l#quAwYM;@@KiS8$SU&~7_-kD=` zV&l>zvyt1Fjw|~}=Q#nT%M6{t?WCr{{;j1oqsTfSstdl-j>MuKKbqYSP1|~pvel&Y zaf{+UsCq$m4+(UTV(PL0L08Q2*_c;lcow6Np-W4Cb2GR6+u8g)mBa8i9%=&MROB0; z5bNVh1wOyex$PWu@+o%zm*TNMxI&z7TZGj(Qoz!qEe!llBrJ8Js`au3s#c(C_Ob=9 zQNBuUbAmQJAla84m7eNs+8=wAmTkUzN4VY-!EE2OVUz|1p;ZwASGrNscbxKTfF=Us zmeC3kZ+!yU@oy^z=)CqFREuQU#yA`00k2r4%Y-q1@FO+W`#q+QYi^{y>kBIV?sbo# z=sHhGEY5V@Dy}(NrMSjraMBq(d&v#E|HwzozQm1EA-f^hWc>WNLX^AQDz^~`{y*?9q@kx0Z~Eyw*+hcvqSga1Zz~a z)UZ`Bw)AWgg9zl;^{qv;L^7dG{tijW&?yFC1qb-$gW3Y@g31BXS?T0*-%$8V&5I^_ zhyEQXb(Ze#_x<=s_gsk*Scd8(ywcO@j?=9#-!_i-FMRO6;Pf%g%!FZ-VdY^&0vIlv zuv6tw8^|4&Ru&z5nK9KezX1Wa6>v8QNNO@0kG04J?Or6o#EMY!lpc9VHO&Ib&I>I; zCoFZxfR}P>gGDceP&}J{mteI_=rOyN zmb5-6<{8)Eo?;p>p|zUjPsN^uVMrKq_VRs{h9E=qx>?D>BOn8R1r?5pi)%s!hM`+WKpgME9{sn=#718rD%aUeVZNb=+iyFO9B5hN(4k2=qIU*LAN;f<@gnj(X0ZlSx&7}s%l5U0byk0F< ziDaet)Si-{Zru@y!Z1Wobd@N<1Dxgugaqt?+$N_PqAc#D8vS?!y1j+~bQPQ6RQAt) zpDDI#1ivYy{e=vhyFeHCqjK~`tmMEW$XE4zlOJmFb9rlG6#1704rsw0+{=2D=MK?Z+S4br`s zCIhKrcX%4vv0Ojop@4^uN?Z3WHXv}*>bXv%Eo`!iUv#b>FPgEhzpD|#8k3wd^{%%? z^w@aQmnsd;NKBqqB9m{z8AILY%G*QISdXsLV^G;B!yzozj&1d4ivp<3lIg#F&r?~L zDb@SIaoTON)ou|9e66V9oVMUj-V~$!Zn5|f-5KtkD{qlUG<4g?8_jiNZ_{Wm1EGfW z^0;?fA!_by!4&*@1fzOHBYOIsb+M~*hhe+melvwj{xZeCk(g{1oRrRQi`^EqhyTL0 z>lMlB5u)?n3CJ|xH>LN_jN%%5z=*ZIWs0^c(>!3WF0yC1!e%s{&-RHss?k@A@72F6`u@NN_jzQW_rLrlL|h<8XbB*Y3bh{|1Pa zE2!-Qx87Lzd@{u!;D3)lmR_jF6aT~?9#|kCk^c{ZS^tLoy4C&_fl$6>+hHJ;!YG;r zlT%o&0o6uS*3J1SVND_nC=$|V5y#2iBut@259d1D&FTTtM8Mt~DpA$qcc> zi9XTUVr&l9&ux-cvubC;V~KKcB0p8m&x_nMgbks=QOi7vQ7;+-v8lltFg?v$l-bHQ z{2K{>f)a#D%aTx;5Kg0Mq8605Ulf3c+8su(%`5#^ftzZ9>RGV^ZQYViGn~*@stMg=}QV-b_c`J{rx2@lK^n zt=3h>p1+F&Fs9;&1&s-22#cZB)|Y<#5}L$yD&>%tRtl{M67oy9=S0RHiRi%+5cG zdwdAfZ1hGTz0CBEP_7qRo%T_f9)l^P zN{OXa+-y-NFgwc>(3g*2W4v_X9g{e_WyHJUB@vlV<__(Rdw(@N6i-btu3OYWW$TRp z$|dx|bW?1hoH^KwtZMWaB9NSOWFNH9zWl1+(pe~fG1IK>iP-*~+(k1&<}Y&KY!pC{ zGzU%UI>AaIv&bNgmvIl$$iW~YE-CQadKKMvwkPQ@fu`(eXdCGDef3TwwMQtU>50|o zw52k1ezu-eLHI#nP1k&$8!ARef&OU+3<`i5h_0sUeyPuF#Jt(_=4+~nY{Z`f^?h6x zruvFG;Hc$5Jpm%uKLS(wJYjYNZEsT5kd6N0v_=2|8{I_-ib>d*a>Q9PCH4Jw@#wPu zK+mpzm-*co zB66`~MDWjTm0AEc+#+3aXBtvc959wxtEQDkMNG)z?f0V6Eoe56OtIvzBj?+{s`q_n z!w8_oxc4gpBTQQ!XB<*o(c99BVlK1KAT+XnBxYn|U#Yf>OV63~;a*Os?C{~kq@Z?4 z#ouTeB4_8Joy7Fk`pfa}&RZ1orak1y(;YY(+AMCS+j121`F+L-DhM~U(8s3r>yiN$=-4%lvw|fME z%54Dg_7O_Ve_>%7&!It1tV0Rb@jpv6osL3lV{Z*b7uVx%t`A?0H1|I~WxcfXNV`#m zk992>aTA1-%rJ$-hCLv|A1HaDz4k~qm8t{4U<$O!4iO^i{`?pY%4b}cW|9J!);EIo z=6^EH*Hg`V!)d(2zkUCM(x%)tz$-!m0fAurPnJW7nwwbtUtMR#>TkX%W|%)cYci&d zFvLNHun7*J3724KEltc|_Kr#^R94 z(dwjoEQ~3I;RaIe>=X=IOwrm|ge3-*WLA56u?(h+i&jf8Q`^(CbDCD_BwW@2pQ=7d z$jUaFO3d?8@#h%45<5mWO%4^NWl43iLeT2NggnWOjB7=OG&J?|xhB){uaddkW#IOk0|4e&6fr4>$3YoWGxb_#CqITv=diM z4$p-a1m>Hz;|lE<;r z5#112)QZ*Qv6c9ku9E>H$*`dY*u>}8!kWpJu{Nf>DyrZ4Anh-z0 zaOcY}BE?8)!J`&fBib8mGCW8-3|WdjP(K5B!%Q93z4zdRSX=HL!jG~Cv&D@$)sv6I zTu?;_Jma*7vDqWqnyPPOQOKPFgixxUlX1{nn_q8L>-Y0cSMOLcV{f~j)=B`U0FRvc zo870~Y?x}2NltaBcCXb#fT^zZzbL9U@ytV*VOj{9l2gtJ>>>&GaQ^8R3{^w1GD>`X z(_T`?*0u1T74UXg{C)I}O&xN;-V3I@+?dWvR$)XGPPs4%SGo)4~$4YxDE~HqP|x`zIDrceJ(nTiLx4rR|Wy|gbqG)V=GT&P_u&d zru=1-NcV)$il%T28l&oZ&f!j*fl{Qa&5iM@vNFHvB@~6S6BChX7I)dj5==8+Hik;I z03OOVOj?pAz^VFmed}=@S+hcTu4>ha^ZxL0+6k~}WxA?fBb{=|5R|BrA@GNujP+cm zLvC=9*p-ZA5BRB-a9?<oSvihc=6^?tRlyIH>bhm!7>EL1g z$nMcm(&6*{8@nh&%I5T=sbRe`9vE}`#}q|c)csaU^t1R|MdM)tSmaq2F!DBhvv$7xwsdTaHorSRMn zdA^bvk0u3W?~&ZW&e$uhZuLlI5;#vtzGlIk66F53Fa_UARq}0WLvhP*bYItYFJCit zEd@W4&&0iubP0(Xp(vHZeJJHU2`?z5sk0@4XayqM@ga7~VUtfph>+61y1}J$eLDlp zw!82;3^r&+xO3DaOphO!n&r1hEVC#OxxO}{qT@TJzir{FD;OMrnF>@%(I}vS%LZ8N4oVXbS zF~6AB3reiqnX@v)9noOzwS0hySGZ^hmD?Q4JM-A>?5Y6RP7j|O90(>1=X31s5rG+| zL+T6XM4h@R!Ulep;wkYw`;cQmL74$;H!^@dfJPjGNR>0fVcL0jqKy*fqEGEsB_RQS zrB)#hhYdiu287>ggk%&TP!|{zTf%O60U#f7$gX{$f4UBne-207E z!DAMH(#m`7$n_;N;Duw^R^gH#qbk)l(gYQmeCQ3P3$M06#x-H&6dk+3k&{Qvk$z&M zH@(H{1wt1uA+s80JFkn{BhvYxKtrSdBakMq5B~D7;kMT785;&_qBDhBg>h#GQb?q0 zqv2GPk%^Kmd@e0OFM9-9NP!+D=Fl&V!19CE9(6#3Zr`x?gt>W>_44&ZgV|NTCtyaD zwBZ)nh&XMt$j8rz$*t`M2Y*~1bExb~R?2ypL)6NnKKf~g=ja^cZXj_iPw;S-Fj<>q z42RChr#Y#8b22nya_w-Iq9b}B-iJ&I1MN0s%ko1)O_Q`5PiBwV6M7Rl`1wn<6Db!3 z6Xb6y1M~c?%M2>*(W5&|*i)y&g^=ObUmNcm6Q@{aX4BhiDK~rwEM7Z2uG4F|eS3S` z+|gxej z`+sQ&W-D9EEy$zz*sv{IC%vGf>d6aiapno#p$Zy?kzxRYjq<(I9XP@8^{S5A8fcFM`AbMMB)#)99gfwsTd(q~VMchz8oX6#Tr&e6m~Hn{Vm838^-~P;@!0|TC~w?z<$|s`Lr_8p`H%W-cFO%oV>Ju3?V>R z86kUnQ({|Q3>fj+8Gg^Wx|GXv5?AAhAicIMSuG79iOnBFOn#JdpL8ho<(wwD-y*{} zl{#S(vQ_izOl}!3QCzY$jHCC2#apW@xgwN_M9rP(8u0p)Q+0S~>t5BNuhya{2cWB#Il(IP1I z@NO_hl`}ys?Vsk~5^4%`@g-7}uN0=7BAg{@Q33b82xh%}4EZw22mQ(PFGw*KkNC~mD?As=rao$MEliI59o93?BG-(K(l$#Jaor+>yOZe zg2UN``-xFQ4H>1fLDjI~z*u~#aZIeyh6kSfX_x1Psg8&VoR3Q@(n8LD581*Xo2Fh` z%9#I?AXaaT1};HKLI4ykgIX}e(jy$|qr$ruoU`NYm`j?;k~Z>&I0vqD$o^_|aR0#} zzHL-ihWd01k?T7{YamCkFpFf$w+4H}K@Ej|Bru~_$K{`6>lZ7ia-O=GA3~N zE(noAlVz^VHuAPQ@1Gi##b6!sVm6qQIf7ayh~W$^8Q377r_LzY?NoAmG@(bk8nY6m zsqF7iIHoceQBPUyRHFRh3fCmZUf8#%78K}0y8ja8@-A_VkB(Kd1~9RJ313e@GnA~t z9)NQ|oQlI;ZgVSt+RjtoyRVhD46}!$op=I_5nTA?ijuhs{YQFG(v>tV^xzH3AGU0j zyl&{%!iF^6L*6XVsWKvxU;ejK>pee1iJS^zvbpOYK&*CHzE;0ErR<2QpnJ$i-8h#< zy}#W}o(g(|Q)3>jO)K{BV5h2L#uYc7%Skombf22W0kBNkgq^p7c{G?|2D#I@q;8=| zW$;vOIXko&a#p~|?Bjk_+zwksRrhTkgcgt8*bIV|^B%Bv+3p1;?Z8k_#J71-Iaf)g z@#vW#U4CLb+MO`<1+aF-KBM-uXf*(PK5Dcj6K$=_w(OO(XUM?9eM<5M>AAGfLA-m7 zPJsr+2>Qr6#-%YjgcXB9znyNq_~MRIyLuPPe$oK#jGflCQ)y|w!kpME+~$W?TseoWMimBEm;DMK@5ZfG<0 z{Z6kPoDhYQ%j*2g6mA3xA|>X_i_y?CM*hfjh{Kx7^$F{nezINtcin~w>6vJw1Fgn;8<@m2Sscgg&0Um}w|TXJKN#{hfoenC_BhyQ!2uByh)3!LV4epBengwA4ekDqem&^$$)W0siX4 zV5E-?q@I((e7mtw-$nZ~uTlk7MuN_L@CzB!<0p>?eU(4abL$9R zRE0g-<=B1XHN{7UC|{%M==o5Ki0BW0P;w(b^-jUt{p<3f<%$xbp;lXAufa`TC=aJ4TJkf<;J6iO0DmQd`Im$7%833!MX7B!y71N)+qc9PLGgkDCS& zH%23*%-baKu=?AxOIKYsnx|y6N7tp!2X9j5#gj$n`f=VnJ64=YCTch%HI?&5%9vAz z8T`x8Am2YTgfBEB>B-gpFbfoiJ3)Y3_tK%jfqwSrKcs|_da;BSi=-`!-O3PSkl|L$ z8)ppUPYqD521+%c9i6}|(NY$eH)MP5)=mw#aPnChPU0(uOBO4O%CCq4vcrvK!;ZAx zkI;&DenmLHQm8=iF+rB%{RLwL2sN`y70c(!tUBeDmwGAhEO7|F*bNNRiCAP;A;Zy{ zmrGuri3^4YCuorrzm9CC9&@Y6|yMA;Dl=K zC{sg?N|bdjunWvVQZ6Zh;J0kBq)=pVU(>X7C5vlqz9-|UqV1uG$ozj%A7oRIPx z;7c+9koe+RzaqQ*VF~oAxW+ZPE%Qc>-`v6JT?&{1R*>N9mFTL-vZL&~Cz0$&1p(DQ zajWyd(Ro_!d}&g#839g9f8sTdfcrNxP+_Ze9gNpZ^lX1?wl$Q&wl>zItbhw@8o8cu z(yUWIlhH{I;Dl&;DSTDT;53bJ0%!G|zzqtr`#C-NkT6y23EFQ)Jnm=W63Gd)U!T#y zrZSZXe<@ymGOh(JeT7W<;*V~;=zQvLn)$v@f4gRZ?6VkdHGcFk$aL;6rLF=K@IIV0 zN9}4htksGHaxoTomy48*gfLz7+FP;t;j^FLpN^Nt_&i*l+gIzcR zruEfjvN1?dv;PhC;*)JHd#12Hl7Vv`{q~uO<%P(!$Z<3<4eQmhT{A=$7EHySN- zyt_N!LUvVL?v6=HR|_shOqAJ)oSGAk%`EuvjY!Od8jIl0SaU%v0H2~>vseq}oqe(H ztt7O5rgVZxJNeS=%U3fC-iK*mWvDXz2#K|_K|~EF9c0gu!hEi08a2b91rSJ782@#= zi}NSQ9i)5cqFSH#JNxB`ihW{wo>(S#L4|AGVt95Jqmn|&8(d=wF$QDyO3$`f1#$ZG zJ~;&qNd4aLXsNKKu724uWPDgdE<+OAgeOz%m&NvK{7I22(TE{ONn={XR*>ZlqTJk>Xh|-`P)y@grDQv9I-cza;TO z8ic8YCo6}@fjx}W)t)eoAr$m^CEE92Jk_&Ilsr;1h@_?Bpn|2rZ05;N9bR1nt=nWP zHaN;5P9-n8l62RTB4GX|J@Q!{ zWo>6S@11pBHslS38uf|1NmF0qx=2~)CBBH9_adD7wodcCCWU+s3%-#4AX2PbV%d-7 zt3V51_ynIMVH*?KS&zYKBVsn#XE$hY#LiN_v`wgXr*&&8>q0y>-=UW7lN3fGlvf2? zHtSO$6)-R=XNVD<6w@F!m>4zlvL354@DF}o@>ba8^B|9TnRthz^5{_J3J;DhL#!e6 zTsv}bkinut_0co!+%LhQhM8#(BH=@nZ?d!6uH*)qy$52E0~_ zqL%c(jSek2{Bk!YPv9Qj!rnHk`9{XYqW*BEZ?L{Y9d0ZO+ISvPtscH=UfW7vUi%Izqi@dn6?U{B(~kSsgxhtP+IYhhqRe%6nGl`{mB zOC>F#qAoQfEiu!Q#ZO7g$>iyV7ULd_LWvvhRlykV#ihO#MKM5z6O2P$h&E@3eXv7A zJFsfnH+;2+jMWZy#CZ177P@d>A{P3y&X4Q24K*u@rm2}pJ{sWapv58vj3P-%MqSZA zS7JdJC5F;dZG~PjG46V>N>=}fNWBr+(E`=&IzY?JT*o}oAK>ALeoWY%&^u|zB$T$LLX;jo&x9j=$2NNj2D)em<~9-%-z=j>S!Zx_m3m? z;oMx4X0u+Rujz)SRW&y{cK5>_YXYK$q1EnmlDB`ch>9gdCC#C&uZF}CljkuvEmpkC z_;;wB!E4A1*=xLzR5lWuqP)EXM=W0hB=w4!-XtDbI44k!35E$gm&PM{Ynr{lZK4j) zglf2oJ!|Jo?ygg>T?#i0TwJlX^46lUx5BogjpQudBeKV+nQxh~SD7v^XAGfo065&S zSY3VBPlT#_#mpx*G8K8r(BfQL_Y_Sx&96JTzhr4RK93{ z@qM?XH@8PPcBq!*Mi(sM2B`0iXtLT7211ONPmde=_()G&58H^aq3*nGYzjqtx`G#! z51l%6D=CkCrx^V{DyzS>_c1*okZ!o42e4>YGR%84Sy8PV9^E;lx_>Bi{b=)`HR2%4 zN^2j|qG|W%^6kh0r%cS0Wt=H;ULS^X7B?Wmr2UK_Msr*8&0bl9wS4W6PnwQjm> zV5^kdkDrTwUBy4lM7hYXT%%dB0h!lK?g}_JCRisQ=22DR^n|&6=~2q}QIp}?6O8D- zX)UYqNrn(Q7Md>ahQ7!9v5n}N{b!6N-t8m_G1A1jhr$k^L~kJ^3Hqc7GmeCBq(pBi zBnjeRNDW8A4#gWJW@1YDY8jZe9_=Sz>^=+f3iPG*1H5KzK6)l}nH=+YBYLqemqVdh zkuH0=6qJRl*8Wm!19S(#Cl0i@HM{&CCYo^RHDpi;;eI(gf4Tm4<_r}*AzX>i#6-o#(x5?1Fd?=)V-w&< zQ6o%`3%51wK)jur_}|t^2g!Vij-GHiexx>@s3Ip@c;~9E?AYR4*WB^o9nCr$7tLk#hs^N*==(eX{k7ED;VsQ z9Cce5WwLOW%wTB5;QOBTDw(DK6xNTkgtYNxE@U;k9j0tF$r(X0Z0kIW5aKF=RLCGS zjCDNLc`9HoC9qlKAoh51f>BQ4*hi0n7BDBAc8q9s?sSJlBMezO=mxNC`r9)m+L|xfV=vad0&;?qIss=N+|S(I{B-Hn z_;XaI7c3qNR^2juOjlnxs{$sa@jRq zWa`86KN3MDQKtt6qX-E?Z|7$XDO!_P7kcoFA5(JCA1*fLTCPBBKM!6qMId%$34m!p zx8t-JVgvRF{C;im_8xQT(G#EOoMniFd?hU>tT7kdE6YqM=Xe}3452SKnyhN7Fw$#hpuGsLYZp{- z65q3^MR=@QLT^Qliq4Ji=|)#9w@)wc*I zLwgm_j7K&FN(%+ABfe~g%}$2=Iw^=&ingUPImKxEQBN~jg1ncQG_}Jaqt2Sl4?_aX zHDk5kM{6RwM8B-#$|8P}#4Enz3bnjcK3}b{V*GwW=H6`(GB&l8tPb*V543p`(pS#B z<9$~8=4omAwoLZa4a92X4qkvs-sjtBLyROj3q5A*BD+!>U^w!EGGk%~-x$zdTsy%; z`@Sf{JQgKh$`}v<<2YCSt|h>9kWz%wd2&Mt>l;8NJBRls0WKZZnA4eyL|A(`;im3#AwmtF2 z-QC^YY22NS)4034ySux)+s17pjk`OIyVF3!J^z`PIcIJrx%2oX_4Xx|TB-W2<^CU% z{Qr5*<7q^oh4(Eq^|JBcO_y*)shD#>-C`$Q4Tr}&)l76T=ApmUd{hx$Q10p%U8Y{9 zo+C<{22q~2!46Lqx0r!eq|=d_gsvH^2A|gVx|=IT_iwkS)h%UK{G4R5&-d1U@Ac#4 zV^`>9I$Rp8&KMyk`A24$A09NVnbo4H?>`@nco2XJ8~0fniwRxr$T?7E?UknFi}nPrcC&!d`nxYM!ZnE0kK}`1r zlqw>HrDb7nY~iaNI!EyB^5>$m$=Xe+Fc*tSQ3+x*Z}Epp#C4}PO{%0z?OHm`&TQiX z*iTAwil=!A0lsOt*WAmv3z}IU7!lYHo;J>^>%&jEW*a1G*+@l2_ zs2z}kn>gA!pP$Uk>W`bLEl;GJc}ug^4vROA@q||mVWv9 zx$!z$G__1!?wBZs;3)^KDNjTgI8CLZ7qKHYA!xevjTGbl9fxY@l9LlP^SGcloR_wzm-bh7yEC^4Z`wg| zUX$7+%JDe(8cpYP94({|%BxdAIJwOZ)`Ri-aX#G0ji#%SYyse`8!ecbjQ$8rW}-$X zW<6tGOf;c}(t(!ElzlE|VyDX)b-{WRH}(hFmEy2&C#QKr&sNAhnOo|_#r;YkY-e;Uz+aRHDBnMRo9&+CL2 z!mU`$1fWED-~Gd$eK*0Ej-*-%umA=-j?HU?)$JN$v`w^V2#o$4O2B&YNgfQPa#3ZXOuwUJ-EP;FwaSpMk&L9)*% zJmHky>3$I}`Nk@4J(8l~y$7-u8MYS5AzLZNHIw((9-}MJkc@C(#x@q1>umi@5*puk zpEpfZXJZyaafG5V%RBw_g6W_~6DomDL!JAHQYCvJ`EZo?NcBs+Lg|1Sk<J4&#jv&lVf#)*5IcKqg1 zr*S_`FYqDYfOEYByRH^2vK0llFj)P|6MKn@SBO3UKuK9V&exT=Rtk$;(H7~XxCObz z41-Tq04^)PzBe}x*?Yvmr^_E-oB$4C zhz~LfD9RP^;+$*L441UCd`W(%LPx3Oqcy; zo%a=V?GeN3^g4T#l|c}0KF;=;%5$z*mXhP|+y*>WzoI>=$Et|qGI7{XEWD?wn*Fk| zn#3;7)YaN2MEwUYZZUN+!r4olB9?J8!mb??w+t-MvLPDO$wkEgwjTM&|Nd%(^}7Aq zD4}!l+3dbr(WJt>C>>ta56*aiNYhL*SQN3#Dx4yGH)b!nlEPp}zjqV_1Xt1?@{^-+ zQPVC1LG*O>E;ESHGubUWPDO{EMzIY!L35KN=(WWS4)67#we7faHxAuIK|gRq3GkE0 z9@3w4TcD+pfubl&l$vUtPxSc5J1Fcs2A_w2Q_H68Hlzb$--hMckQ0?UMGIPi*mj7E zxN_ELDwsPlk#=gOJ|`h>zEU`SE#*5DYPF zL)iKg9KlSU6#ZgBF88+9@v$PqB7Ytdqx+rf9#@o&j?C=@BDoDiaaR@gyIs#t9`wZu zBKIK#&Icm@?EHN539qA~&Md1N9mZ9N^MOinT+&vW6%^&D5bdY9^IIEk%`G5wkB_!Q zrU zk*S{uqnk`2jn<@i1E9vU-*>j3V-P-3UQUGD*at)JOHqjjni!ArZeW$-)oSukr34R!jMT3>3!S4{-=y#l%BvzV?0dL3)X+BpV9*Vz&t}!k4d#&l zsP9&N*)pd}J00$+k43`#)_%v`#fgWACMc_T5w}@)QB@yff8qs_p~f~L29|dTek%TC zOqyI@s!E@4ORs(P+%_lMp;!1>!3Yd{7H3ATcBCydJZ@fNXbYzI1W>#@9?fB3Gu!|acEU16N>g){s_0?;;-Dyre_8Dj;R*T||+DlVyvJt*9f(nPe zXP#*Cn5mn|HSOZ%wDA~l$g;-ivmyal*sWc5#JP!o=l;j#M$2zWODa4tunJW$FuDJ~ zfS`tpv4g9VqsxDzbguQ`yfjue0tQmel-RrA$bwj-&^dojOl1EuWF8J+F#q)UYpqJU6C`DAnQTfGj-V1?p&K(_uqQ7>wevC`&$0C%LaRR z`YtRyC41d$X5U)q~5EIAOE;EeBMTGL3 zB|h@t+FSMLf_ZYH;|%d1JobD(hwK(Hk*1Q8(Xhsiwi!NbOJ;N#SukQnjo}^n68-UUoUhiocczi)(sipDU)~>Q3PAMG0mP;H*M4sL6;?mdw#RFBf$73*&pQl z4~LjJ#IRCR0BaUFtemCZ_N z#PViSZsz2y(gU?=Dt-MRdek-*Qheiu7urW^-c~P9gxQNT{alDaxAvgE4?m<#WAI{U z+@>(}#_CJ?7Oy?B1y&iQkz;tqH0`otr0cNr5uza0jPxB0TuZ;se3?Hn3 zrfED#7is5=jPv*vCu2?D9GA0a?KdG;ufvpamNU(uYNSpNCFsIn?)a-mHD%?>S{eP%(~SO2)fV?Xav20%cEc-Sk9}PfbO$XQ@E~+OSit$xmNSw* z`W7c}$j(+M6pt6b0))qbeoN};Hc+oJtc!Dy_tXJ0Z^_KMCd2jTwa=TkV<$-0aK zfHNXDl6h*)uR!Uiy?A#1Wg<<|QA?O?YOHGTuUicf)hg8n*$VbhH{rB_lw#1%ZuY{> z1#w#!{zT*Y1Yh|l%Gw)ba!%6kg&dipF@UMmG=AfG_y_7rlX}{d2odPauARa-2OA^SCuwm+fnLU8mKaz#4iX= zro_y{sH9@qJj5KX554f9%127mVLX6PW%hN5^OT@4U4AVxjYxPJjkZT)ntkh!I2V)& zKJo!8jV7U@-__FXbYoPP;3nQ?3ZSyHT1@4Y@qsz|eMeK33tyUxaO9MC-xV3vE&Qpy z0lc=&@10TLHqR;#^2e`;BRLlu*i;udQq;wx5C$=Y!^gZIH17Ly1L;F&zuz)sxaIDl zY85y)%E#>a<8u_vd828l`pXWcyli0S%)Rl&nhRUm#oQ}$G7s@)HMqC#d3i&)E}Gd51AgX|GTqkf*9G;04S5e68Ow5Y6l{^n5RI*l83Pxsr50EfGTVS8eOe zq_KaEaGPomK-r=Su=;JxstrF7=U$ftN`!reb~EdG_TlsNMH5lIIjRifUvh&Pp_-6U zDO&8pxpRenLKx1m`$^3P2d~@e_B%L{?Y;FaoxUqj5y*<;Ipa5%uy^Q*KCpK8BRx4` zZYMpTKUO7L4C$K&DR9gX77LQzSScfArmKSEqdF<|YPR(oD=_t~Cn9ja5PJzBcz=l#5H)#4g^!;&}J_ap6^vxTOr*5)y7OjGuEo>jzT^h@53 z4Mz{p&X^4`cbZVIrltAXKNx|0jWSiXbV}83iDI{%uAkPO2cMs4rvUpCMHJU+KnkH) z(@wC}MExriZ$8{*xyD%ghGyfq5;wgi0-43VKr7P#<VFE>bM5xe1GUxfDfJOsSuW zvC?`{Y0IRPWrNt%h3(Tl=(%Az z8&AktP}C&|hkIPF&gW34Q;pU3e(aI+6Gx_7r8HV&B$z!@OOBV>RXnI!w4l7Yap6CYF26qP~o zf|xGoz__V_gCpCUe&BXOb57N_AZo9=$7kj@qu`V^98-E*T&mwiQ(RGyUnVP$ynFJ% zWb~T+bQTuAdrf#PdZ#W4prdD4sn#mIY(*12izhUxErXSw&qX zxwt4j=i+WMxxPKkp4L3#ItGX=vt>8^Wigao_jzfZE9LNz#ZBCJZ_Cb+({_w@Q7$a| z58vC`w;@h2vYBoK4$7m7sDzO(*PYFmR}p94TCgo+N%O&)^Cu&sfB;r3>~O+UdAKoR zPVPKqXBv@iDjT|Jx2BrRh(G4+_T2VR`lJeEcWRrHWlsCkud6%nmvm#MTd9;|5s!sWlfIKaMA(x7y#xmLv5bdW z(Q1yr>jN8$509-I!u2s81^AJHm=&~is-3Kajg!Trbt#YalSOxYJegT|DxG^*YGRo~ zTTRXS$OuIO!BMs|w_Pnv`I;0QXcyq|?jYm$@@F>39UNCj7F$jBZpACjGs2BCghw8TH^ zk4?f{bgfH2&O%d!%NF{cjE6yF)aC~fSP6pSYj@YeJTvggA7`a0!W9eS)`r3o-SvOv z`&M5Z7tbxO=Wa?zMx^rBnL3te&Qeo^%NHuogX_BFkY%_@MKDHYq=YwA$C{rx@clE8 zow!vl4_QgrJ^pO=8DnkoK`cl~u8PJJt8t_|JR)cOFZ*9&s2vWa+I0@+%7CtD^6P&Zl*MpX zgxxq5I9bH~;RSMD4ZT+ziu^;2iRq`E^Xdx_f)2SU+TzZh-k%%J2DydIi2SQ>5P=5$ z?hQ#=LVEF|$pL^$X1+c`>T5GQOlnE)N0}TTZ2U-4eT|RW@pyx`_8~Rz zs?Pz_E)_1kELPImQoRu#PO-VVgJ7K$1Pn6St)Ja1MTr(dVtjUZV&B? zxgHMAn|BGoY?`U(o!_L&K+;=0p5d_1#`fhU{jgYs^`Dk=gxp%~hk!#=TSGNWb7I$t zMb$+H!X}Ie>ec)Hnvs$nWO ziuGMII2z6z;02%<%_XOf6>V$J%Bwtc*@g#)HoA&Rhg@^n>xZl6X0#tm9wY2YbDh`N zmKj|PT$HVKGYm@Gg7<(|VDib{wx#z=&nIzVPU)Sx;FXc!NpQ>`oftb@ivkf+LBlgf zJO`lxAEeSMLGi@B?9mmcF?vb=)%a3GYMF&Bo%7QLGrF8szkwua&*)jFwZ8pOjqfcp zH{@A=Dr|FvX_5U99m-)5d=u8!+&ZpVoaITJv#0p$C(bJs`Rld$)gE{mYp&W^ zM$Z#2v^^d$2P>}5i1%FTV;pPIF0L@PBE=R}T#m@XyUS72)q~8olHQ9+3_IQ*+Lz;m zQv$%J*YZ^SM3?Y;DtU$JzBMZ+GIui8BlS`;kKN@j%?CgRCFj%)?$QSHsL62nU%mpv z;7ZHrUGEGF@741W1HiFeInS{{MjGGNB#ou?~1ACGkbX1G%&x z$B@;^b7jQBO+nLe8F;_dfUCNKbH12%s71-@1*`I|lb=n`*^XQK!iGGS;q`#S8artU z_SL)Yc=<>X+!-HKGl}YTE9t)VGUGC!F3};;Zq#KB2zz+Y>7JLuz$~w^*>qrCgQ^4V z^VG^gKDD+gv|Y8+wC!vKgwEC@_k?kSccS^g7pE3??Gb*BtT=6A_xm?ezzXU-2yv+R z*QHAT!aCGK*9Efq-xT+%aAP|1>{E3Yap&#Wk!zDo?THq_V$fSLOoJ-B@CHfpy>pMk zwzc&h^Z3TFdBEyz7P@hr;Ap(OGluc1wH=uYpbnva}r7evM!sC81lftSjP zpAHtIQXha8d|R`Ef(`~aT7F>^(F#E;A^ACv@cAT~R-D$DieD3pPt;LiA$_l!#;Wm` zOoNkGj(DgFCejZo;YU|$5eYsdHsq5)Tt%RzCAn*h3UmA_N zr$}j`IIibFks-T_U!|}efZ{%1tPKRzi!pD)^MtpS9Z(a2hA1(7Y|jc9;mER5K6PH1 z=qLq(@g@z1 z$7(fhCE3boMaz@MfLzoHA+z*#WqSm!fCZAXM8TIr^w!!85!We0#AKE7@G*J8PngY# zFgngvXRXn^=J-fePjreEst9Wuve;W`=LJZ3ZrS{KM8;VTQFQINXCQ)C;YvT^gp9)KXTBXr8dDN^rk zqm(Jel<%Pt#|kEDe#|hlQOK7ZN@)zrTPrWzSkp5rI^j3~6IIPW)&{#~=SLXn=F+k^ zRl8l4KTYoAP3@BWf9?-`RecfPBTUb>oM3lt3=sn&>&uspjs?J2-M?k{C%$ebnj=5qAk{pdfDPZChDd|T+n~-D2;y=a2?qlGX zR#uG?v*E49R>K7P7>|4DD}I|bV49MzE)A@+%Be&$zw;EB3^5Jr1JALjx5oWhO0;y+ zJs;}6i`8KqU#Y!%)-cPuF)mI$pc*1~t}O4K&@uB5;pC~Lc2BYNRiI+T92Z=-XzTSe#%!_czkfM;2z~EengEPU;3Bzy&x#^Qx7*%IQ z6EB1)mSFuei6@ITSD1xbj!{z`E8}twYmIfu01>W-TgZe&bMe-F?HB`z**7sGp@nefa3qQmOK$<%WKrBjV}Bl0;URmFdlBJAK3WgIt60J0a@V zA55d;cMz9%Y`wtZzo-Fzup+i}2f_n!2RuyN{eMjnSw`;~u{Vrp{{?CruOLNdw3wQi z3Jd(9rFT!`pF~dX8+8!ny<)4eqVr7SoiycSmoja1Nh7Ry)N3>4vHs(rfh zsFWCpM%gZ0;sghe?qe6mVn2%8HTdZX)7dUOH1e#P@q_@i8gDh4TX3tm&^5Y$g$QUm zGxCgtdeIV(ymdoy?m0eF8r!|_S?-^`d(HH}BQo#l4W$qHUMRL_1OLh$Ye<&9ZD^!2 zF-k-!(qiRqICVDNb(ZMphk6f(OPUa9!t!*$qrikE?3Cc}#0YOQzu$Wtj2f5-yhIau z>A&5z;SRbECpz9gK5pMqDcMYcEX3_VI`$df=qW@%Ze*!f`Yk2)fhe11oN12MKF076a|NQ_y@Ru`0GZA5f$N5(HLr3ZrPQT6WJUM#%e8-fd@)_diXF?8gW}CHfef<0;_m% z^_r+Aa|j*C3{eX_R;&)nYkvq<(|KNvMW>l{$_haH5{w6<1#iJiiq-g=yPY zaNE;ic-m9R^wXB$pv8-vMP0sqIhKcMPzA2NoU)3VbW<(`WpYhRZQ8b2#cwewePG}n zu8Hp??y!n9E2ZUYhV3b#xFm<*(!3S$Q950qvVJn3XKZ5hCXa0?aXx{3q@0JO7)-!3 zcSIAjGz!hr`c-z@Xoriam;mbSR&iUFes7VmR4?6Al{gydpGSvRKc9@F`M_O62D>y~ zq^^%+jb-%0hu+nsp+BY6>l~ek%*X9e7Ta;n@sO`-KpgR|QE0LBlHCi=}5mc%W3E+^O`wcpant9x=If>$#+%1+{pJMX>E0#}DzE=nFm$h~0O^hYrw#HhIg95)u`n?@NU#vWW|;3@zJ8zE}4MoqG2On_mdtdXNmbh@M4 zX*otZA;weahn%_tC+s5`_=vUU5lkGo+`rtK*6pkSh)2`<6&CC!&>q?m>|3~4 zxH>a7kNLiJ+keS63mNqfJ1om5a+1k&iHlNK#0+{s^rm&_&G?cgee0y$dY*dWQ$SaQ z#Ix3$-qn0%#Jg`&@n4Z>hq`Fks(N@da{{9dRfhLu&5?~5T_L~(w#L)b%fB1ZKF*Mj zmLehp$QN3CGQIu4IKVZxb7_5C$05)e>ERivgZ*Bre|H-B6i(2rO=MJSB_I|E=*3I; zpGY~3RIT$DEEpIb-v7}u=D#E5|J!BEsJ5>grWW4!PSTYfw>TUElA>+!B4%om8(qla z4kpsUf=p2s$S9^BxkntAU7&R*yBt)r@(%hTTm$fHw_&pJ4-ylTVbU(M6207d4-koI z{C+09_q62By`r$@CQP)P+vsvV4cL4AD0uC8t^UsQBOJ8(R2K~&v=IeHmkZCLZT)LKv}2rcBUGD<7G0ihx6Bg}!9P{ejuo3+yxEmk@858m8T0}a77Q7pe z5}Y~pwu$2CFQfx+pyrSXmVA>&Q8^*Nx*2?DCdS3*nt6+Ox{NURZ@i8y-Wn@~Ub{ve z%#>4K2n-gFQP<#hZzE!_sDG8TCjuRxY}uF7l3*m}#on&lK&@9nlE+7zn#s|}*i?3q1xGsxM<0oRY7GC$Br--z z1_m7F8rxrH=o(@T2h?L&NfD4{NzsG<&#OE}z^M|mfeP>SKrLD$Qmb~pX-YX(5PM6o z#&3eJ8_Es32Y_gl_|yIX;?VrXhPp4%+_ZUYk-Klb)r9IfzGU{{w@+a%Xb5y<4#BvarsDXvr3hw|=(~%65tT z+(tN+Y|k^@?h`ixc(->gZQ4gVf!(IO?J_l$Rsl2N$4ahUd@9w%p>NTE<{2L_uhJ=D zPN=u3ta~g?UCkWEZerWi?l3-iW?y$f8EW_Pd)WH_(;6k>E@hg(H zy+^|$867|C22et3uF57R3jZVp2o zWTes0f_t2|{w9QAQ~ang^JyrD{%!Ym2p}NJjJo&^IQ)N=z~evYTY-Tv=7f z3fw%N-z$0U93PX`_GdF-Oj%|mU#UEZ`_pm}-ylIN*)i-2t|sr&o4<5quLJA74Z7O( z;qw-#$-EH|rzc<~ji$En1yx*q^_hfa{s|q?@y=Z3ES}d*3XKJocV%;4`<;C2r?VLP z>&V5sv`vhteqp0?xBqDfTTR;9#P_P{^0t@}kN#5s*(53OML62}6hBlKucdpcBk`+h ze75X}zn0(h{2~R4M=o6_eQm2_N<1-Rv!=)QDvX0A2!YIxU{Cd~E#~&n*L$bY_`3DS zvrML?t(sL2m$#frdA+PsKo4uzkKgEjA^cMLkT(7>ITy1-7uSikew@YT6?9a;*+yFA z2nn{2+T241zm%xZcPHN`Hsov4A7$`!&mtU3Kkz=8Zm@rW1UxfFn;Wr3uw*h~k6KKc zu|-ZhRBMVEiw(gB(hN=-HT)ngZJjlLUbkfx5>=e=c9-sZYIJYgGC>ETTIH}|DJuyR z2>?rS8L2!2HPWw9LXNCU_ZzPuEHGQ)znGi6-pj)dSjODjWMIJ<(t6X6s#1=+-gxQn z;QB{ZNAZkAUA<;wGE)6UU~6*q5>0nim9+LnzYK%wD&^lBPTDM6vNVZ+D5~D2yMbY!+Aj`JOlH%8)BpjAQL^g>gbReKV zOVwW1N-xli{~K>=HOFnlm-i9`5|h4UAsx9>c#TwJc&WF!ApUs2gnXWa-Nmc7AsQL_dY^o+LF@IzeAU{o zvIMtY5@?S4HVh9jZM${WwZUCme3`1U3CpP)!d*RN1Kshi9$|7nW|Ox0-nkRU6b}C5 z4gKMip<7UTJP=LeTrhy(1*3eVgROL4ifJ>!5Hz>onXWCVEGTyiNS>qjLo*cPq7n>a zIf7jS;i605V;vTYxcusq2^$w|{6#rtJXebo{Q%{iKc2#PYKe;_e5!hz(@)RvOgZqk zWz408_z0(Pe9cAh55M=&0P$pwJP>{b-UcgAO{MZqP)_C_yaKddP@w7KfQoA075WJU zt9N>cNBFqMJU_clVb#tcuYmX;_KCHmfwXj;xD#Tfb*p z@s=br9f`NJy&7CcfkaHyLRf{=PH@7JS3)6B1XS z{j3a0i(q!+AavjM-l;~|!r11*cGL3e56Jd4Nx@zN3R%Ks7W+=LDF$X0b7U5(&5tsNfE5V`npq=OJjmSqF=Ml@ne8uN9Z$tiQ!pjG%Aev=M_svw9o0cv?#5g+2~~^c$2O$N0WQ(!1Z5)RUZT5KoU#6u}o7) z3?y&Wep18;?{Jglb@!Pzc5zmMqQ(+Z&&hPQr-HC;bkbMywvR5{q0j{T(|ANqG*Y`o zZ&X(d@ActofkZj5o^!FqqwKj+RvPJM9&r_430JG14ynd7zGO(z?!@zqQU&VLqc z`eC$XA9Q(|PEw%>p12XK#*OdkKQ!lQO8!~V{*!UaF~Dl5Wv*uDg-diKlM=i71_6_b z6@0Zue0gc|9dS#PhCg#KhuNtdx3V#OvLj5ISbFhq!_Ml_`5D$mp&hn{$!Gec;?r(M zNoN_}evU-GMIGo1n6N&@r8BG~yEvyAH!&%~!$;7%=~|-|%O2D!U9sb>4r4dOKt!0w z7@AEuUM1#v*R`yG^o7H5bbPfQ=WSQ1pc98Crrt zRPhO&da)qW<4o|#q|yR>SRggY)-Q2kd(xct&iz2j%)&+o6DR2YEXS#4RY*(oKFIWr zKf6je=hX6caoNGr5CqxKdAA*xCOhrHobLpLuwOal+TI>8Ns`CntF9NUS9qBb@>sBb z+LT9$5*cKSDhq4F_nWO_Wo^1WHQ%ogMD$@d&dlClO>xhu{rmp=U7fu}i*1HYv2ADZ ziu)@Vje!6JNNursQ~JebesN_>w}uxjgYs!UelN6l{MC;j>TtOJ{vVW zGTxM*Et!`OW`YadBikX$gkMh3eZ0BVrwhhphc%s>${@6CR&`wXXBUQ>0rX%N&&WMG zN$r%Z>BUM-;X{h#FVl~0u!*U2S02Rz;`a|L=n5lj;(CuW>25zlLH}8#Mw+!b-NMp9 zk}Csp0ZOV=oeB^9a=R2379KPV7$V^V%A~FJ8r1HR4hBz(%0yum434qhp^^reS^uK_ zJmT6s=j8C_zr(!p)O9n8lk$#7#wncFMF78}ljG#jm(DJtbP30|^Ul0|hn#dfMZhO6 z!p-rm(OVlLwxivl|>~>_!$$YV1F_G85^087`_>V|`|+8+I@* zP83E#KbegnO9&6(WrM<#Kz5{*u)KEM#^xLt+^w*??Kb7R(zP8OlJ!=4Hqho>^edZ- ztDBo6tGln}*Zw4{FaKBmSSuOaOX7d>$NFD)Z++KZkIOri^1+*u+6c`^)q~|Fv0LD9 z@RFGu7ciE6Lv<;ernXAMpPZUH*9f@db{2>oQ#l&oZ2S6Z#eJk@ZppnG=3i_GLQj}W zLs`3iN$oaOczRO|%{s9f`CK~!h=IwagQes+^u~uq2yoB_qOIJxu(oIxMr;&ubWHM_ zn>l}oq$!j|`(q$5(I3?MF)(+lL@$xi$6hF2b6Z-eR11t)l}5c3i~!`K9QX@*FR!e! zLtp_dZYeRpfrG}Hre}Q?l=!Uium{7OH8e~{4(e=*L*eu>tof|ZdA0VaIc>1=jxZEl zQW`vP+2!WKXeJdgCe!3`+&YUOa!ua73^JRD;oWzYMxFFn^fY=}b=3rT>a*|cq#49e zU7mvw-k+<|zM4*l6*Woq<_$64cjzr`1qF8js~vmqvsJ{b;t@9nM{Zg7evB$k$uZ%; zELP*BKAlR3`p72WQ(g}ebMBAWChZ|~EP2&Y8$@j_(=cd#@%`W2NP5@S=WU90ZDEED zCeWHKNYGc-V&>;vRw>uD&Zk&{zgqyN?IHMB?Ntc-?s0meju4%D0@_ zpXOvjW(}luEJRk*BiZ-mIno;zD61y<5QP)K0wh`h+|-#DK!}^(N@XF6x?`r_{t6Ix zeJ10i%!vudjq4-MNk~1-OcW)9pM=TzYHAysGcw;^wH`JTmY3kgNQ#N9A(!ExxB)wHv(j1f$UrA|;cETH&XiCN?LdD%UaEB)?>`nXo zC@;4irmh;-GJQN%ty2_BBwH6cOzJnnW2j#89?IJ%Bgp@VIvYw~uQVEnsdeHqp1m`( zj?2kl?b4DGvzWIS+An95ZO&zIu8+BN5)yZ;f0ZS4O@F^~yj5|g*!{gM>-%i1oRt>m zy7yyOXTMKpl*tR{LsH$nbs%BJO4i>8>jfHA{!|7#l=V68> z%QT5UbXI+XW08O&dBNQ`W4TcnUrl%lF%2ln?oahmA85pI4nQdfX=d z@CpW0I>iUJ*$yI!eyOAok8_%D>-x!!KB@t_H{9w#ba zW8a%Xys>k1Z@n=r`{iEUt!(Cb-JsI6K3!gXyFYDg{cVLRvoZ8&`{r1N!Omcq)}k5NKX^jSxl?%APEKM$d%)8 zRvd(#aiwoy5CxGTYt8t6*2tBbbvKdhUC2mmHL4I7 z+f{X!0b+hcyTN0j_KHLGU{es^9R6h;C%IVU66WoH-tp+C0<~xiqh-AQ4z_AVyQtC~ zTFX96OR?Yh*^^`*`va3`q^tX}+R3VhjbHn^_dSW&F3RH;QA!s|wz>_8dv1nM#q3I< zd)k03p|p0lp}~*rR`w&y>E4@exVU|w&5y~0a`!8+o4#D~7g!x(-M+luh#n;#8MFOq zLR4ISg@YmKx~YLICOvgr^Phv2{xJ|vLa!#-&T!`F-;wmGhEt{?b|kv1)A+cgA`hv! z8=yMcOI#Ziu}Wdwc3z8b-usq-*e+t7_0#Ww?AweIAVM-(J@c^;7z=4;!&mp zUG&v@DYeLJGf>cudn3cOJQdiGC>MZonSL6+{BcD(#KuzRetR_^^B3MM0%A1XMOt1f zvd^EQtnVtry2HXg6MA@dD}LNjf&zRtHbm?mHFq|(4``xok4ybguxgq)-sip1HY+Zy z+c*!Ceql|q2YO$Rc6x6 z50z3Ec43pfxdZD=r`Y2lxHg8WowuY8?c*zwtRiSls-eyUQ7=M9f8=E$IJb=ejkn#v+~|dk8Hzqa+rtx)+x%i>f~XG6c1CZ7t7QG**Hna>^fJf1KJ< zmz{z!pHM6hHPr@{J2nZScVH6r&=g<1Jv)E(6B@D|Glyy!KG#63iXP!tiw}fT#0o|f zM}0xi)qg!)u@#T|FX~X(wsny2mKfJr+?Kl|UX%0b?b}U&Vs0&4nw9pMtEm3K_^0!f zhq(dw;`E)ui*OG^W`ZqR{SKNh&MgJ{NI7CRvi>e#L^WlF%Omp ziN=j>^-5beZCfTlsm%slhcgU|7HD0b!kz>_+^i^K%=?J)C`vfB4AinXxvI88Qa%l- z;{?TD!}IV#7pmJDvl(8Ok5vKuLkwH**zGUv!J5?}>(tpk%rmZOe8avI;O*=lo^6_h zqyB7yMwib`h$x#QnwD=_LUx`Jkym64A82^Q9g%g9G-d;(^;rIg>tg{}1RpDYFl$EC z4WkhH<5B@aIZlvzbMYBfk{!Hou)4qH(=*=cISaAx9DY~2p!L)vNnwf8)WS>Xq8(Bu zp!MuVYe+cAij@r-i+d$w^Aq$@f2;lJXze{jZPAZpm^E?pMzqu}=18Pqcyxju=~Znj z7zzB*tTunx@D26L3-&QwC{ee5L18FH`g+wWFKV=u*|rQ=5cFTw!{-(hk%5k;u6_B} zhsX&~o+ap>&80i>KHH%l38JFg*-&bh5M@f;Sv6~of8Ud*A`k5{y2bfxEtb}A|} z)xlP96#{Y&Xi#1Grvb>eTkMk1fw}T#ubt-6^Isiys`pUBmyYTZ4w|>Ik?KW$q;Qko zaI||edB107v+ac9wKfxuYQqg<4**-hUo$Mx2YTUq?KHX=7<-*Gn-~}(Z8Ynk0R1

pS@4Tiq(TIrhf#N>z*CV3CVU%|W z=|Y13{nt?0m#OSOaWl=2v(5v{Vh52aqI?FW>XLID1Cy2;#FI2a|(wN z(~md)1P@y$hD_F1QzGQyab!c=V0CHRt0HAv?}lLf>fh&FOVll~sfae^+?zT1;+YE` z-Gr%MKmNOj@@$AWGlu{JGlv2LCg54a6!obnuYY@qn;hqQ%*n0aP zz=V8U7p$CiTt<;L6~xROrrbVywyV4eK7Ha0pCb&sC6Tqx`fcR!eTRkHCkj(esoCtMiW;vr^i3Qh4BMGsOy0b7H#X4e3PO zL&~h&^RjVnx{%DS^!mHY9&?f4&hx?{-6tlPFmQ(?u%8{4*R+qP{~{KmFz zRP3Z;RBYR}lgj_Cz1I5A-rvRE`7ux??xi4YN9V&3pvd#*pS`99}9M-)Ju9!ew2J zmvIC{P-M$=i?*A*%ab%da}nJV)Og5}IXJH_kb(HH>@8JnlQvPBw9`|jNu_A6Zdmgl z!RFMRqkiO6fi$jZVbBrVqm+cS`Kao|KX$a4RtH^+xBB3Yh_9i&dnEFbE6Oum zKmXIOZSlX5J3K>#a0dngvI+$P!t>uCSH!{2(aP4$S;frN^)IjQLaJ1zLq|FUN(98YLBt9d>owK4~aWbDv?liqRaxLcVshu$*$_(5&f)LL_gk+Z~pu7#!=N6$pnhtbsBIQh0={NB)O zzqPzcTZ8&Zxi~otKo5l&rjT<8sW!c32p~_980zx|#%Fh?&epBkGeU!Z=w2it{470Z6L zfZuclK%$Hc$!sC(>T58Pw}oIH!D`EkqQE2)6Qkcny{2rqK_6p@=P^uYk^ek9>Qb>= zXgfT_SNGgxv6bm39msG;S;hvo_vKKM$74wA8-6EG6azRgK44=&o7oQfwj0|JTWR-^ zbxAEeazhJCSl@xSCyfZprAJ58kiL6YFFAH=W{jL%3|>sGWJXuP*$NNCWw?D~t=hqx zU>eQ!BV3+kSsIKa4z&sLYLOcTqi;$>2EZ=JPg~tkQhBNr9|KS*a*~s1qQ@u|QQ0eW zbj6j2QP_G}t$`G^YJ-^JZ=P%k7Z3W#zpA^1!Wvo)H9y>o)Q_ej(~aBQ3I}ug%ON@o zv&1%;M2n%38y+$lxAcNrW=jBSGSAouq5W=~PGjyix%%zD%9pcf4l*KFyPRHIe7NR` zTD6QZ7cYP+yRg7sEX!#Dh5)SOE;FYQfATh(K;nq6D9I=trepd!uai;^c3uurdOkt- zYOAFFxp^@M$PA;~he%o1UA`_vpxK@HSfqg~jxE%j%Bao(lO?P8NBcF^FGQNj537FE z*>Y@2KZrvrP+|c`QvVcjGXYq!K)HKYLn}0Nd*bl;QOO*f)zHk$50v0IM4N{h+8dU| zH;A&EDg&GNBhewmkI!(1`XWO92Mx0bgVXqJmP)#K0y~ozSh!Mm$0rxb16w1y=)DJE5pdr(uHx>i)EB8ogPeqy%>&eICvgUAI$r0*E;_(dxgyz4YlK=bT#@X4y`JdDg zqbMgcAc*9rVp3SYrajXMbqlMj=#ZixnH%&yWvk3dN@H%bLF&^W7aZ{qh<`Gt78tSq zM(A#LtHT!U{q5@$!VXL@8eTP7c!#Eh>fqj=pE|LL`%^9-MHRrom_QKYR zu_v=!oe-yMkD`5rUCHXPOS7qojGo$A4}_gMt9zU_&~*5$bJ4i;6Kq;q2{L26J-dMe z-eIUbH!IRH`IE*^QW8VB?YozMkF+F({h^DWP5+?5d{C?`WbndR4dD1D2FbTr%1;m5EmLkDwf{y?k!6~>cy>0-A&G*?F@Gyx{{C0uSkCmhx)a=Ant8l!tT8$wnjK8 zk!yH17)_+ONy!vTp#N)(=478U_6OvUlTB%j&Vc}%o+9HYV?)*Tsg+Et$DEmj4fdm& zie_`yId?6uL;Aj`qke52)hp>pMZoeM*_Z|A6lC_;A!(dbHmv6QsI)rQ6qN45l~nFQ zI*e!05GE&m%U<&f%1HIWlr5VI0)0^ppBy?6t} zJ7OgmC2cHwNgi+Erlkya$n+C>hH<*c{K61dC}MWV^<-rA5o`=2mhf0pt!RzwD&xgY z5mjFEP$9a}!8>Ajij_7>xMWBGs-d~6ADI93_@_`f%FIH7fEXZwfUx{;G_b6hy@jjg zKNax5W@%}>8{nv6eCg|>_N$&Wl|-&%&{zkxRM`6oYYh~U*-?%XX1b^7C(5KVXGA1H z)6`a-zw2%Pe5-WWT-=mSnRB=MT#fTj%H#83W@i*^(0dKx^0?yLzVA8SdVO7PJpOhu ztYEZ|uP&4m>CMW>N^sc7Ywk>gbI#&b(3_7H+O~hg1zeMShz~*VN`a; z8kuz#?#Ed6c|&AI!mo_h4Y)Tj@i0^e?Z|s`ryc{%B87>T7)rT$Ns{`1cgzR(Pr!zS z3`5^!nFsky?lun*qr$ypAWL!?!;f!e{Hr`9mDme`Gx zsfB_W(;J@N94IBYcNT^fhYOUi1l6FGRXYM|^Z_0ct~*_&BHi*Hw1{rqjO;RMw|iAquwar=U`INrOm*ayntgd{8)5H&7Ik?x^wv`fo{lGWIMt*e5;(T7ig$VGwJJiA zidsYIxsV_$sb?BaYO6d@73E53nYkYj#r6ZcDN7uELPCL6h2dJj$-^lPN4!24W`L+< zxLl^YxH>YjT3B^v(n?=N1oomfbj_h_XCc{sZC{zUY)Gm z?f6Q!X1ux5;Ez3*SF{%s7PEP1$43#%Q zZ_$w!x41dEr*o_WwUT5L)D$9E3_D^F3>BQfw2GJCmIUI!c%Dfy8ViePs$hl8ewS)K z6wd9!Ko6k!=BV}t9e300d)iNTsAqEzi|Nd)*=>71(guF#7sccb1--l+Da8?)X+=f9mMje{!q>`m#bbG#0fVhW_L6`^=8r$XX2Bl5$$$X{nR=kOt>$5GtH?e?!RrVqiY~2Z>uP40LIx{$cF8 zVkD@0!pY_Wmdz}=12e02Qkpqqm+(Gzm-Mx3aPcko3?Asu+27|6Ey0vYZBzH#p3x37 zjeS`SH{^VabIe=)$b~3x?r!C%0rGD%9+e^%U^CH(ey!XO1t6}!!|;D1_>qF-JQZB4 zh4JGD;6D~zYlQLR1!VF+1fR_ugDH-O{m{horUjYwnTc}@Y9oE{Tkx{s%Yii+!{mq5Ti(aJbjvX_0ZGEw7vQ^HUF~j z@y#QSK}k7|;f8DmCDmsJN~=$8xfJEr4}=^gH{@^aqr3xq;H z`P&RmNnHvakFY|ld2zvP8#-uK#a;1~dSWLa<^}ufrtb0qJ@p2UczqZ>5v^ZRje|`g0exG(~Hd@?z<&JSjB-T%`(9Hmc@{!_C9&QJd$(nA(+PH0pFLF0qx> z=h;{R&CV{}uhbb=e=IO_!Iz^vZ3S*Q+PH(j7EAm2qdPR_8_E-&_Teovy0A$sdo-)# zHhv$9NBgdWHy^!^6-%S3SRalGdSAo}%CubD5o^Q-5o?hO`SD^Y#9XY_73^WPzOY$X z(Yy{i0<`6eFD<+237V&#lC+3O@cR~eJiaEv7whsO*yn|gc~j7PrnJjs_bybn7^OU> zGuImDu^aHAZ7w^be9D&5`NpEDTvu%jp%wVZk?58s#kes7O0h9Q*KLl;Hr^5qG+$~< zPQz$pl-kwoHPJMoe*3Yk`>wyf9=UJgTezcwo@bp4THdJWuHrm7#cq(}%LPM{ZHmKl2Kb8IKO4m@n~GizKRYHAg^Q zY7$b%x(l$L&VD?TM=%*x+*ZO%52NmA_%yIl{vN>yQz5|_Qkd2l;AbDfi$=QukD~iS zS#)L{19fndj=m7K<-Mt~>AGmCow@FG@;<9$envMqT;)Dub;1D$2!1F=dC>l5@yeI6 z7ZzTSW`H&&oGoz!F;b73$&}CIu`*8v0{v6EsiUIP6C#vvH@F0cB&!Q&XD$c&JI3EV<`P6m z8!Lpl=fe8&AH29Xf*X|GSCahj?zgatRsg?q_T`eg&ZQ#xMOL*o4zjLoYD(Kq_& zK0+Lv0;p{lZg9@FQan6sXn4Jn)RnyY^x^VuOe2KCcQq~wCj=*Y&5CK@hH!t78-!{n z){SZ#&Gf*6o~&zF8IU-`6r?3;3(59zL8PH=Jbm?^qM42H`c9j8bgv@*{9D5G7K{&z z`TvgsGRwaU$lL!&DjfOMdVazlIl+W}+bX{qsf_H!NSHu)miP^kO(q2Mdf1F+$-VvZ z`yZa3m%U)}!eOYJKFrUe=;PGLBxKaorq<6b7yf6Tsi~*?2G*hY%{~2yDn3ZN$uXkf z>cH#>p>()cxt<}rxOY0Al6tC4Mk#`oZ!q`mCC??2F%=Kg@s?E{I z3xT18l)Fro3Y9zII{Hz;$;Tn*XEl*gi>}=S<#GtQtHpNLMIYibrFD2fsr88cKHTH1 zlMa7eN1~v3R@ivG-PtVlKs@U0(Ky+b6aw`hm5CoF&`!S$svk8t)|bP$eot}?3DYw2sJ{pbCdh{d09UkQxNa`+4ew#lN#e=C`TVbd4n{|U_Aa_xY03UsHRn+Mb0$FlK-ns|d7kiIlL z0h7X}pVfm$ZQ?U+b_jux?WaCK*DZO}#w}|?sh!s&%8M^FhFOw;n;mlB?w#2dR_G>S zLsx-P4SDJg)07P7IFS|^7brFe!rlevmMDawq0WEI9<6$VW#;~bpx7oz-yHdK(ueU} z|LSGDDXYn@*_GD9zdNye(4BkWx+kS@Mq7mP6Jyxq5rdo&;In-4O8;3BJv3Q7?eJfr zyJ>{+Y4xwt#s3%RGW{F6{}Hk|^3{62WQPVYA>3>H2XrakU`UDi=>RvNjf1ztlY^kDvWOTAe7s;Gw}f zB1o0t(5D^N^)*HX9e=IXcWxxqeg}OIT%o0ZEvVg-%GukWpIxVPJ`SK;Us0+@@B@J>Tu;UbjVYr(S_nSyXd<`U@Y2YM^D z4w%gI4UKV8;fnkTpZTWIb(yh}TPV~y0OZm!&X?gMx~Zc{;qDBSPxMFfb0PxaS` zMzDUi_6eHH#;NcaIb<~qoUaZyR|Pw!)lzZiv<0}A7^NiVZWH+^R)3)v<5Rn|(0uL@ z_P-Y1I2OAwzW=4ko}fTLSpOHAOvS|6%F$KK-onb>jFkCb?T%`j|A6k-W_=Cw?{u)) z2++L{CH}~lOQfNQh%v!L0KRX-xxIM3Oitc9G+B1K z66#!j(b>~Ig9!gZ^M`wcJBH6{CZ_vz`H4gjQM<*8mO-sOHrrG@{S~$k&>lx#jR7L} zP?bzyjfwIGI>G}dcP!LXQVhldpi0EPslD0IKEmi$DH4iAl7di$)z_|Aafyvy@6ocq zWve7y5_wNePdLEnyb*|k27Q94F$$d}!sDHMugiqEkWrfYr~&~&zJRVF1$AxTSu zvrs#ymxOl%Mch))r2>PvfKI2i)0038i1)Wti^~KVHGM4>uyL{yi86{JyT?Ci(XQej z+yPffos&p(ZRp%P1;kz*tK})}&;69^4K=#+b*=EzC|?DP)Y$1p6=mS2Y4>oSWL0Na zLIbGUxT|f!CB2LW4p?p#x_y@d)@>i4nx=DowMI>1WTQ*GRM5Ia4=Q+4GukrL>ngbI5i+V zL2%y|k&?h`_PYr>Gqv~7&Y}*^Iu`t_(wr)pS4t&c?t-QYO+;wxwv3^ybVo$tP!v%km692(3#l zU=S_<;7Qt|d=j@L^rbcGA|(b(q%efewTgTS0~Y=kW%1Wx0s#g2=Y{*)I0cl~!&F-M6{0Y?bQ zZ=c8a$!VAXdN72vI3EO47Ztid^t&naSYlr&3Fhh22+U8Hi;PZ|U&PSLKfqAn<%;tq zsqVmDl)_gJ=VnQVrlz|5?w>9%Hb1WgK(>rFC7@%WIC?>UhYYLghBIQkvEoLTG5>}> zM3k=bjoY!ckEtxrKkGxR3lFGEDlRi(PU21?jv?QbrNkSjg&mR&^>ZtA_L|EHccq*@ zR+exs%9?PaSLrt%yb-eJ0zU)nmVmewbszyS`Zh1@Vm!~uDH0X0Y3SmPEle_gpzWt8 zs0&1h@}g8QyM60;6*UXl{Nu0fqfjopU|{lbq!6;kh&g*q!bEd*6E?;hb5Ou6_6}Wd z$09B~Gp}8`scD)q!P`Y1@^uoqgWhwKN};-^KvQ4^oCah#X@P(Ez}IGF!?&?ZcHhG; zXmy4zJJ~C}8%5B?(zW6Ka{P#-?lFQ+^Bf-iZGw|)0;tPq2VRUt34g?ML0Y;0C>$Q| z3_8{yaI?p9t@d3->CqV*c`_$jZQ+-B^fOLTJ3l5{@_f!v@1yAo{QBFz8=x+NZ9@L# zkl{f<*#6(|`@i;E{oMsk4dai!J|BP^ddk=nya^Looe2B#%ASrE1Bg)F#{S(@Oib)Q7Gn zeYSb7Eh!8ZvrW8|V^Av`Wx@8Hv&BkUnCi?h0e|^lK~NCxTO@bDXw;P+*EAzCbt=so zWB3km?zF97UQ4OYgESFD^CZncx9WGIz|t#QQWce3m1>G%@etZN9*?}@H-JNMq(ptf zO=X9IUrX=VC|IC_wv|Tef$EzkE85hxdOb8B==hPCte`3S15C9pR@P^r6?#e5pgAjf zJci`(I&VqN!PGzMk~PIy-403bt+eRdVtP!r{wVA9B7O2pb>~Y@Q4z4!C;D)6dA%wJ zClUHF8F1yEnM^1>sPGhZ8X(YcFs>om&GQ8->0ZDL0g()uz;=>f;0QB91$c1Izc31C z?$jgC8>t%#aJ)=s5Q~y~jm~yfaq~R)kkJQt>$xsuchUuM|C$Sjfbc!#Ws15ru4e*R>LIOd{L*mBQw7|TPNp?OO_dKKl99e*2Za&0ALv*Hn| zsD6z#Ujw3}l^mt%Q8lNBl%2XWT`}WIw^0c^1TR;e@)MnfxG@Jx-El!T;5EFY!`+nS z_k)xSFf&k2e)RZi})*VYFeq~DaOW$UJ_U=?ZQqHmg`M^-iR z{xQ!WGgWfmmTxFM#Hx>W(xWmt%9&DWC3ROdUCbA_Fgnv$fSS~Pk#Mnc5ZE`dGK;MJ ztL|h_{&vso)RT|v*{rK@M-3fQYhbujM;yI?^mD+OJ8&67o1)26m}aXt3+6|~p2o}L zp7RGuLc$f$0#t(`J@r%p^@bB_ER-noFD&>+!1+9|F3fvgS!R7#NR$U_jB7v{)JP1jYX?MRMXb}^{J z#yOrCcvjnRLW`0C#}xi-6;q)!TZHJrUR1+sQ&(plBLETz6}2=8k$&!$7&mlNPs{s@Ymq-6mnbhi{GF;!3NfWT(! zZr+vB+t1AcNS`#s zx>w~642H}LqTE5x4kUZrv@vXg-@&gHB%O*LZ2aR|>=q@O<}7};2mjEc1^kw0v4Ob7 zIsrDcW`Z7Iuq}RN78iS5LwokoSM)L;=B{qVxdT*Q5}f?}=h=K-b0u=UCQa3;XmF9? zy&N_Yj6@1tsJ+}xC=To!!B9xIV*!m@7x4Cw?vSA_W#3t54px>B-`HG$qB9DIse;kF zRe0%jw7xi^K2pqWGbX6SqdM=99!++E6sax;K;j59VY`~bw&+YBwd67JAVob|k* zD?k8PF(RBb!#E>l=bbm)&{A*yMsdE-1Fe$iVGS)YHQ}PoERL?*U>x!jkOFX09j2m{ zD>IKbDFWV+U37h`x+2LcQWHthJDfeH4#L(qK5v#hQu{E*a0vZz))DK0EzExJ#$zdx zqWpr}v1E|;%}fu_-XvTTI~L; zm--JHsLT2E!XEaGE8i_9XYxQN$m1j#?pnaFknLZ4QFYd=Gd3k)26HbsFF1ed9UlKa zn4}nK=2Ye{jRgLe_ond#w`%oHYp(_Q(W366HWE^KVvRq$eyuO~l zQw5mOS|EvR2=8_*eij{8$6%w2_&mq;U78qWW%Bj5{|xEDonC z)RnQ67FeH@%RB9YQZNtMjm5w~d&3%SI&^1rymkxYyPpc#-Zh$DX1&IvPok>Nh@m)4 z#rhpR6y281MQ&k42^#QyjGA}F-Mzh3P zL)EU_uok;@A3bKW8VGgt!0>q~VkoJw!L7ideo?dA6;hUbzvD*~LB`fJO@IG+#`xfN zmXwq4;+`kZKOTFQR4Q@c6+qQ6FIpBp|9#~Tr28AmJ;boZCxY+LqUa+D%1<6n;g#h9 zrhZdI2UKjnY(iPL8`RBuRv=T*07XE$zd05eK*hAIJ{-M4Az^@f6|`k1KE?!SO3GuL zYn1wi)f~1koZw}mDE=B0%kQ%w06n4d6|#o(YOL8i*U@A3V5;)#Z-Au28xL23 z00CKr1py)dUjXvoitV4Bg9xO6M1-bL>8K%X8(T04VTKmzlNcGDH9=yjff8%HL@O*w z^F34HidOGm2DzUQwTE}#yn4@qdrr0glv;mI$x^mfhxTe`@EvFSy*{-*uMTW~{dwXE zXndg$;a;G*skk8_taFgpnJI*roRC0eaq(V4{{Dbdx~q#;9_MNrWGSd9Bsqj%4%(fX z6Q@nKNqe4|lM`cJV?4`T6s0xp>N+1WmD1kPH*%2QN|HZir)4+z^O^d3mA@=WC(Ap&&v+VEJse#2w+ z5mu^~9*YaaCI)AA4}WfYn~M#t8%>66bXi!~6=~wYzQ!C$1XCc(`tI5pGlsESuDTP} zTPJPsC)%oBZl@r!!FfTl3644cQhiR^VjaGAq7V2qHD~1>7cMGi%UDM- zz+8wStw?*!QHR7=ddHa4ZQ8M+u}GUr_VXt5$4V52ehkJI7F~ABk()~X)zc_`$Zb&2 zKD9g{@k}?#0;rIRfotW~l!SYF6I3wS#h?^~9(Bj;0R2iR;g@w{IAM&<0;%V_oT z6Hq91x6OvQc8OZ=Uy-moH;NXtRrxmNYI$v?B5lz$Fp#Qn#^Jk57f`%<gH&l@{m|*m4i`L!Hx(}(v5*>+Aws}!Z=Gnkl z2`l!{qhVRqs#p}6V7OMX%D((rRP9a}Vsv1bsmY7Sb8|8pdAm|$T?C{YV;|T`jImp} zo-n#V;TA`2Gzbk;ZaY4lU#>Bz54GCwfiUnapg#8vQ4{sR0%LhdPQ~#=r$&gXY8u1F)@rll+~yjeHCZ74!ZvQ(pnB-k~5EnpsWkXL^ZqXwS1OJ zUmDJzyxTga`DC^F#y2&QQoLlf(*$W;C6dyL^N1e~nWo&2;JPF*I66n}Y+p-45rB9@bSN%hl1s2#26uCCCPFHX=pP5H zMK31r7+pL7H!wK7p3;1VY4Gd05-7O2w9j%F(`y25jd&IA$4^j z%_f5IOU7r)`F_hK4^E|=Y@P^ca=gJGB&930WRc6E(w6U7;i*`iI+l1z#}EIR8S48E zp2v(x0C@x8w-A+DgLok*P!f70XrdXVdD&x9@P#=8=V%A}{W@|m_^V|E&!f`_kIU`l zH`%?7ugK~|D+%*i4`dMZ{(8ZV|9_=r!fFFHo^K!^5PxX}ivNL@Ma_)eEdI&Mq4EFG zmI>Upn;$M-T(bV;#9*jFd(Ll-#EdD+4=UMlLQQj*nE=XBf9m9dLv+6v4N7SXQSZ?= zvfN)?unhe0?dkf)+1C?=@|(Syo4TBtwQNu6Msm>P2cE21I+Ln`T6F^p(7BbPp?-K~ z*?hu{cwZF;Ro;_W6?csFkYl_zivl?`muao7ZujT`Tcq>+=N%N6OHRQ(rV44K!Kfj- zdEL}wx2fd|b*e^Vm|$kXcVcDj9t1H~6!F7X<)(%RX#m29{}>pg4!t;e2mKMY^|g-G zCw^w%<57VubBW2>?$%ib^kK>G_YR1AO8;Plvxi@=)U%}okOPor*}NnMT)vP#)U`OS z>ag`;rOeS@qebE*_zdp_5#+X8ors6@qfsuAX zBZq)UwV{AUgzy9egv&HYVX${Pc?8d_S?lO%r&xDuQ?6MzOWL5@!3zy~qc?|HRMn{Z z-ybY|;j`_!_eC+;3ff_G~|QZe?Niyyrm?Kn7$)3P7PbmhUX| zLfD6VT=wMjROT}*Xf->pyZh_F_32@LhcLcJ#x*-UcOimpP#c{k#AFTO-Rk1N_^#dz zUbwIVfcG7p&SF0n#D`NOG6*(O3}EEmV%@ubxUST`ifVRyCHY=ECE3;rWhAX7rsA*$1i_u4qc)9MMKNaP#;8 z?x6!WKuGbL3iAkUzJ&7q@}`yJwNP)|kB_S^{C4iUjH+g(5GszJWEWjCVI}t`)6#)2 ze5w$v-}NaOsf(@nv|=7Qke~d$<{BP+{WAeq69M6=UOSj=B4tC!zno=h$iCUS4q=If z*~j~%z&+%5G_Qzctr;=BAu+qQ_gZ=8hD#$PWoP}l z6g>ozRS7i&SFl$J&0zdC@B&)g(`p#hIHg|-KFe=M7RFQb-gFRZU#A`bb! zNpj%@D!ya;H*rYL_sm}4IgK=Cc=IqYFSIto9u+EKIB0dz?#s`9=V5C@b4R``mN$LY z7U084MQT662u($-pLyN|pD%%;_9Z34j%Q0_XXa{m1nAcGq0^eTk9DNIzShLq+(5h2 zMA_e-j)f}pm#MA{Y$U?@XTul11q4!I@`GtqII_VyE)zz#+S$T5sj#@@%S8jH8z|lA z`bpPUR0$ZxDA4&OaEEl7#Xo*roh&nroB?q%zHNleSYwF6GiUd4>c4Y+@cIgc`e9#$ zF|3ics`X%%AU9AouS>cdXINRD+=(r{F`Pbzt$j2fS;>#aG}DG{753H8ig9JQ9rdRf zHNvzi1vF!p_G_Tz0uZ)l`?4Ext`>Hx9OH=@WX`@BDEc>$;%e<*L%*$k;D9no!;4k8 zm2~~!5mkH94thQxV061)+QFgSJ~o)ciG2-KJuk9D_EQOV=mR^Lext1E5kxW)m%eqsjTbPK;t-|>C$Hdru_ zvtkR=tzcTfu(wdr%id}CPEx*YV#n!$k1M|^hm7mn7gvXe=vT8FbXg!E!SgL}Hz@8F z`ob>}ZL4(~TjKg|qLYXnC0=|^^b!k6u_LuvbMB#HBs8U!+9C6L^oHw+-BZ>7%=*Ir z0b(ReT~aZ3e_y4ZfhL|}SKBHD8(m?+=82HI%%ydqFcRS@-K`%&PTl$kwHebhTr`F=>O9fs(bQ!71Qh^ak2Vf0@V@)?fz?|hNABE!o;~Fa~Acjd1Cet9fiM0xz%?NU-;j@pA^N=2@CHE()Q)lo0BNmf;;N9bvXi*O-?m+uXGJ zJ9g8k-o8|V{ri4ZOvJj0^{27F2-lN|CYHmMmety$h}^B|SAv6X=yvSd!EXh6P$fH> zSWwd;i27)v39c9>IoYz>$n%8DEimM>&4ZxFeaEOlrKv&#g)Lte#NbAf#_<>;?n$he zFHHp(CM$N;7&{u9r6cRngl@%qTj~)L!#ot4KLbj~%NsZ<6*s0qObqK)&y;>^P#U5| zZ)BV3+Xz`%K(W8Cj~r_^Km4su{A=i~oEnUms54X7gs!{H4KbMhE;wPBuj;_ zu2Bj%mP8e|VwV!v&eR4^j#Atyi>|?)nP#S&tgl)O_Q*i9d{>i}xU6glmP|Fo%xugs zF|3uvkB2OWGgkO}Y%5wCkglOOkPKU1v@(O$%E5OsUR5s}p2uWA#hMdPrdUETGOw|6 zjuJm#)5va|0X&>20=g>wfqqMaDpX1oQv-OjRU6ujZL~0=j|VCypGmSe&$CwdyHM>u z)FG|oHqe{GgJxL?MY5#tXQ!ccN8bcDQU>Uxp~}3t%BW2DuYR9|-`z6wk)p(<*&Tpc zj!1nO^@4*cta6l!Uoql0bFNeD92-~}oHj#Mh|*Db1E2Z5hq10hXfxW+%?u`lh&d(e z+5?1jBx2ZEFY-n=8y_mAEjF_@vI(g~S=z=1Z*XJ^awd8nBg3XQGg=-Ru>VH6GLc&Q zH8o_mQn=^*WS00zy_SAKs~b^QER@ILUtG+05$Al>;mhN8MlC&BaiU(HNWF9=IEu-t zWU6GxmQqhsZkCm7F-WJU92v`1{HhJFa#@Xl`Mz)v9+JD+Dg3*ic@wCDGgRNgcNI zz_=@Vm*MhxJKe8|J<>_(;etJcRhI=W=ZWZy{&}QataRYUxZ!Qu!i287-YaEULMI&N zqsy=tSx`<9Bj4)x2u_dF9Pr~h%|X9yeFfvu`ZLkWUnZJaK6-^Ol97hMds?PRME1Q;$c*wYz{?uf0Bmga|cd{>#;9e#0>e$>BAwrs(;_#NX|y9MhKAi$V%Ob(Gge z7Ox^|Vzylu5(B1BxmlI{HqVC>H!`*CvGg5)E$B~GX>`eK8L>XNtO(mT9>z+Tzx zDcAp!1*qceLdZ;serxn-TkG$xZt_xSS^VIaLmhK_du9J&GHgA>F?~VHS0#6~7bmlx zrA~I5i_`Qv4S6A2(Bh3~#99|rm4QxhThh1VOWM;_mu7#Sks90d8^0?>Xgs!2C@RNN z=TYy6GzuD~%f8Nfatt1YR(wL7DSsBKCo4``pJbY}vwCZVoGUebKCv1U2C2{o6_ z=`%yCnd4?gV`j04v*-+);w#R$_g?d#iF8v~@x7l}cB(B&8{teMZ*+wVvyB1vzW z#f+1-1DGn$z%Ll1U@NsGV24mu$VGV`hX0$5HdzN9Qzv=Pg7@?{Pen4^13y*zHHzY) z2#8R9DrJh6uB?Rec+XUNHl?zTXt~KZgcy%$X8bb8-eUP`%gOv4Y zd6^*3o<@JBG_M!zl*#}qX`OK2i>910^%=lW9$m1CW9A^Kc}zCD%4=FAf?kTBMGnF_ zR@s0madhc?9h%CRIfeE9PF10-`HJ??ZD`?NcfsAE3gvzvRI<##$(hzOU?9Ec6!szxOO#`2dL{8 zSZQ?HFqrM__8IW2=XO`{+*Iat%o;=~*ZT417?4hf4-}+GS6*1c zMgHrN2~w0Yd@v3=QVtLNLOwa$(GhX?qsh%}&ah%ziD{~quah-r>!u~!Iw+AW@VgJi zUiceBLDm~G-@a&JX`KjyC$xgNk#k;5+>rhN(juYLxx!JCv$ZsNx)sal0cemUc zUA7ZRIY3>;#9{6n6qy03}TA9;%hBa890UWFAAm-Q@X zPKT$O@9$-8lYrNUFnJZtP#)~7n5I{{KW1FFNz|vS$KkrU=&3JjFFXJZ!a;wp%2U44 zRz9JSx{vO1>$619pznPi?1v?iwicC*k07#2Gt1pIn;MzE(H*YCJHG^}Cd&oG6RcWY zJ2od>Ds*+n;mxE1ayS6y{AH{$V`8H}5yMPGLw{Pd3oN1A$04}U&~g2CHVdt+UPxB^ zeHJ1z5jZ1HO;)M>qPQ@y+y@WrF1TX%fA0ZAo)ja_3?i0^%jop);&lff1c$_4;4b+1 zlp|*JIC$O7LmDu)u?t2m`lgFkHysJKHpr`__6xtY1p0Lfs-)gHWUaJLW;SSA6^pLp z!n~4vw2{1{BVZcPs>~$*iYkcQ1}dUw28;o2i>d8~9jr@irHzCf;7|*2&_Lca!U_7~ zK=%$WaP)u=RD9q$g8&d3l)6FFbc)lgqm@CWL)(yxjR*-WD$m{2O zV8IcH$tzH=b_qVI_KppbKBw+*P4D1H@2K)b*M|>SCf!(-@!%ha>P`hW2+Kp&UW|{0 zi&@{l3?>Srvu1N@FM7ZrJ@j_}9$#zUi*xG5z7@g&3vpe*F8`(_4ue3kUCSx$+RIN_ z90mQIrYl8a7dOg99aUS%8<31+Awf@=5j)uF|8DA!hvFign4CCp6 z)WvgK#eUP^>ZxS``ZJiH2U+6^&J~8n)j(W308$3VU59SAZCO368vEKr{^SVpQ-4>vBX4F}c>7ty3% zIn%hrK5}7R=FB&Ffhqp<5^?mYb@U0Sgz#;oGbG3`CQj#Y`DAMO);`cC>So3^?wL|n zqt``~+aJN-5n^&+L^d{)j%|F{VTUj1^rcbfpwZ_vga5$j?FYdY1CUp|gN3?VN;Q^E zNlO+NKU52Wlk0Je&iy-3I z?7dY__+cgyRuoUn@myQ*8S4*;48YpbT=`@zuZpgp2W(%+7y4TBk@52%+(P>O1h~$X zrZ#*tsS&fI;+&4m2e@81o%Tz6*!~Y^=M4$%<{;cJjrxZQHhO+fLSs zZRh+p>+F5;uT!UL=AtjUW~Qp&>YJYTNgS&AXT1jRJAnE^%0nR>F%u**KTBal7DsCr zM;|GTCI!w^1kQ*VJqXk*fQS6d;21!Z4ML%xpB7ZZ=aQs8+*ukrGK7a~xxuBPD$~@^ z$}?-!o(8G53Auu&AX#&$pmBc~fZn+!+Sxba;FS2L3_>%fP(W`hGxI0+ zLrE&;6qh6POK?RDaNw4t`PHDJy-cPP*4kOkhEFcr)vyM!yJuCXBnoN$_f1 z{AcAeWCXdyB2v~=Hkl#`@>iZ~ zQXPC!9DFiT<+K|eWd{L2rvij#-?JOqn37H=#4j=QCvki*zn@ z>aRJIk0TIGdP>jC$f0*GlYgoJHcO_9?}|JBp3wtEi^2v-Y$+WTnQEKM0ES!HY*XC? zK{m}&ep2m>`HI2P$~UAB14|Ft(cTUQG&@;G-U|J$+i}q(q`Ok&88;V4^8@KVjkhtw zJ3VH|26R^k+NQRgy2-#Zk!h~H3_EDJDrwXGh8soJ{dbkU`9Wgt{<@BDk zsQAL9B9$5kCK%Dn6sBp5h?ME0QYA%RZ#>pAH@#LxVB>@q8yH=2FG^T$Yh+{wbAC3(jR?ft2hMr4yA{A@r} zJp^|{Y-lzGOPziwzs5OIKQ#_1|u^{XCbs_ZTK49zOyEIzJ@Qmunv3@nsp{M&S{}QZYV{G6wx#7x{H*)T8i90+8B_UCKaqY% zYD*&Z@($~=CTxBu^_W`On)W}THhxmAe#)(T)^aqY(ap@dL_d(?-|_*U;vjE>14tKV zWY7({R=I7ZI5s6`I*={x1jk1Z%BPVl*KNt%7DA)jG{Sxwkr_2GZ2H#C$4q&}{JCOYl)_#mgI!r}{9VxaFM2vm*2oIEw zgaD6(;qGv_4>~;*y83GUr)T@|=Q5LzwwWv-51TJ7Y>rd6SMvTY*DN5fBo5&Eq(Hkg zQ`FyU(R7bLNzKp>QvA4#AT5KvFH{!P3hA>gpBM&)^ zH44oW^)G>0Y*Bscx2p)0n=rSRI+;9FotHJLU$bOP#m2oPFPY3vw<&Dy zVD==hVD3yJc~YN1K}-bS5nXtw* z>m$fRdek@Fz*{oKi)WUT#I~2fE-Q}afX9}qO%Hc+xQm$Q7*i{Ez}~~No}R?cKi?Z5 z^j>V{E47HPfm6H>M(iV(s(I+i0DQZMS@WytsDC`Y(c*WZ8U06bN!wcxL%1%-F ztA4LKL+VgY*t-k)Vq-A#g812+XiKxx7vS@(NqQbP5FnxXp11QO`-sH ziTsCO^AVE2ab_VIAMqgU4d_sJL^eY1MtIeNgd=*3}of)T^6x0?b$M3g=9z z7Y;3mT*Xt2TfFmqf6eFi1D;e@rhjB*mL?@22z0K&+~*eZxp+~KjH z70DSDx=K6)h$Y$CAXN#1Ei&!c2ZxnLb-pkg5;w(aUF@YRxdTb{+=SN3A9|9raw8)| zm%c!sKEJ8zuDY2qdW25Kk3B&O!psnSKQsM_{!jTnnKG^@xGeI_ z)vrZqw<5K|c;uXu0;QuHg!fl*fo@YOu4L0vs)?n&^Cp^-z>Lb#Ws08}b1P$=4x;T|PqEEUGCta8Alg}%VF^m$lN6TsBpI_qY`_F( zs()iKmO?4A!oIrdEe8Tye)bf6wyvzY8Iq$T&U(&6ux8gl28;1Jqu_+hxZ%%5Uj85c z1nkL(aGbKRNr=b#m`dc!#1x`kqmz*o8?!{BJA4RJiuPn_O9IyfLcejehbfO$bHB-w z!M{oePP&Fxnw8T=NU|mFT>23H7N-9D3x0(#VPMf#flSzC;hj~9W~qdB`;0I-UmBWM zi-t9qxtqzpQ| z+foVEJkpkQe{h0pB#-E{1C1%V+}G0~j}L=S2c2aSs0VH?eS$NLp@srP+#`bNHMYr? z8W|8NK(WvG)93%A*(hRxJc z#DKpTn-OCQ>h-;>mmAP$W&);ACglbs>%Pb1*cOzh8EOajwK5oEc}YUsyH3m zH=JXFv#-8I!yk}dSGNrEs4loC;!H6{bB)Vtlq5DSGt6!iGUey_8LPukP%T%_LTFt# zEuu#&8r8rn$rBZwwxJ$h(tblI5Kkha0O|hpQ5a-#w78W^@6ueDIvS0+670wA zg>8A_0)wvwn?q`GX0t~b=i0)s^NXQu48krah+1mr00ZxVv|H;wQhW6G` z?fCJ__$x=wF=4W4>9tiF6Tip`YgrJQn7YntY1vua0zJ^2%$(C98zyHqNJ|Bt5*Xgt zUSL2sP97L8Et(-l4c^jD6!{UzQ`^ovQr1f$fA-Pqd?Q0XVUnnpck^;#?{v*|;(Nlg zcmMN!GW~d#M#zI;!1bje3_TD|0g-vs#lRt4@pXKCY~WPyF{MCBz@CJY9Xw*?J|0ar zV?ff}=0S-SAv$By_;x7r%aok^u)%~HOPZ|5L3n9@K@o;wz|F&fa~%(U6e6>Yas%w8 zQi3+oij?y__0glCNJls%R_YZhN~}!WXz0$z?7@tsg@ZtkFgnDeWmERikE$f9UHv&xvp=%u0{PmLS_Uo9zK$a@+BJQHO_J72FL4zDMe&bDyd0~so_n!$D% z+2CkAVBzKv?}f10U_%kn< z;qR;qLBXXnu?%!wLm@OE9CwCe!#@ZKyC_3Z_5R>zm0G@NdFhrcSGm{-BT{&%QF+~f zXh#4j(NBna9i>LdBWem^@;ZB^-=#E5nyC6$Y zmbP3j`^mCTkI~Nzl4o$oUy4%bY0_-g9HQ)=N0CW?q)%%H?KaY?`W6$n> z*H~}ZqXt{Grz!{tIv|jQ&4?)FA5O+|{YpO=z-=EEUvY@>_>1&GscDR!nTb}V+axUY zi5mjD%^2%pLfQcJ;Lc)rFgu#LbEYzr>&}fqK;x0#*0>ZE)Gmp2JacqQKLzs6+8v^x z%R1cM4sMT1Y;V5AsDG9etL@T_dPCA4N;(0Rv~t~9dpN;uITtHrC3)4WJJuB< zI(?4jAvdOkvny=DKSGP?;da`NdSiMzJJBTdGu?nkn?)Pu&}Q>aFnHcs3w^W{nqg|i z5$2E=`*Pq*jx(STezHQZ-I-kP>V%cve2RXk9d_#<3;8gT*953&KO@E`93I-P*)C28 zH`L#;?5tGqmV?7@k;<#a#4LO;^cEQscfm_Ao(())$~d7GT^k)dY!C-{O zh!}aM4!T*02aryjF&Ewdij3bZ0&T7f3Kp12C=YVpVq@kbM7smqqpZ9m>8u@-m|7tM zxfY1dV454B9{BqGm~VjcXj>LwxLi98w8?#tuKHDM_-NGEM65gP!I18I6)^^t(@`lj z(!73k0545~TIKWVz=_k@S7vOTbRET{Xp5WuFRGlqtbpCfR3^NaKp{3?=2~Vs zW$A>Q^Vi(prR37B6cE3p`|6G$Uk@x&b7_d`?*P9I7Kyk52rpiteJ8!}rOa7;ul{sp zux~ZHKC$2nU<#*dj+8QB$%6Z)DJFn%(Y$k7`xruZdkf+(Cb;josXOVf3FI z(Ys)hP)v9vQCrY`K;rwM-Nib1O62|&9|<^rn;emo^?j3If}f1v`klSnn}W=;nu18? zrU5((<)$$t(}QEC*NLN~~U%-^GWq?NuP-S{{_Q z*Oq6C6gsgoiCe;)$td6dnk+2HhcmT=NjW5sp~mK>=$tSL)vL^>o;ANmRm-JMv@y3| zl7X>cl`h+04RdCZcjNDb8z+lor&Vk-jnnxZ&6LV#T#|OY$QyHfp&|=Ax@(9hQ7Uu% zSR&}?L&-mUUB}Sh)d>TjI#!lt>JVXvw&ES6=IM zSzd))^;o;rY&4d%mAPqutmo*SwFji(ZHTmj^G^Q7LOu` zjhiH*t;@0sAVZ}qTma9`M3qN-Hnn>;IS9$qD_OF#YO9ldpqcD&85tlkypFinn(<(` z)8i>uE^3L+XpZ&pT-Xp2gYj@SN=sW+P7}igY)WJ`l4n1Tk1^sja&TWiPn8aerqL25 ztm%vH1fpv%8+N{$@E6TjL7T_SZkk>3G27T-qu8+=kBjorvfw9O*SCFwD~^3L$<%Xn6jSF&`Ja+ypwcI^t;hY&$+s>KVqOp zTyP3$+WUiNOnwc_$M-;>`IRM|ljI2-=wZz9xVb#qZM0q(K|I!2w_Tmv?M;VY=d}|Y zxnZ!PxuC4u?{$|p`K)!iCd~I)KWy$kX>;2`%N^8RPRVx1*r^_SjeL=0!UCEZ(!7WUZ*pX zGC#jBzqeLI@?iNP{_)FG&{f%rNUUzFY;2qCaL35BcyXa(S`|5;#2sHV)hT2168p80 zfojCSQ`*MIM?^&Yw168CarFlyrwom;k~#ZHcK=TTd++WE37x(18((U!GkFpC5uLFt z1N4=-6VJ$3xe7xT28*%4(}m$10^85A>+1TnvFPPx78ikp+De;bFN;y|vn>Kbg(c2T z^@6Rf;PMVm3cA@9QU zbMmmeW%lCEP+XtOn6f}cv%fMZ!B1Lr&;sb-QqN20I@-~@s(cqC(>pYXz^9=2+UQh>k zruI(mGf!YLeTp%rwVDRK$p^HwrjyA7>e9Y((^KjUmX0azA1GGlPa3?1a>MP5IPeDT&zLx^Q&HO zVgPJCspfZ6u{??h0xw?${6URymAS*0tBuBt?{P-CfEWW`2=mvuL&Tu53ucSy{0>!) zN961+Ejvyh^N&mxPWfEwOgLLGOe}8sGpvF4RjHyzN3bj{AF&d}U5i(-dba(0kL!Vs zxcnDJknc6DYq)pBAocADnrqL&a=;q{3$@6s-H&OFT~T+-;EA1M15r_T!9+P!J6u}< zIPa`iJ{Mu|5;Els8+_gY{s2Ld6IdFqY2z`JQH^To&8seDl|5^vSUhLpHyq&s9ZyO| zt0B4Hced;!Z0jF}g+ToRR0ac3->9ftp3C7ICVv5vP)tYA?_lja^X=Haea7#|u3HXX z*%J5E$y>a$SLl?ZHI)jP+=gkXx}r7EN|%^dr51$6mP>g{-7&f_tDXSyoJtqI9%TBJ z&VY~Y$|v&NvWnF#kL4mQ4yP)W7Q4bMVu8R9>>VL{LF}6&Nl%)-ss}rMN&BLBEg8lbN#)Vw&gGrt=#?Tq-4bw6eRILd;9P&XBT zucyJAGUoC}G}?Equ*U+i!aK1T=KWkh{LO6O*<%jn0{b>l zoGjU#;*RQZb9;xCi0$sD*~r`0UmgOA5QGN(M!!GipX5Am|Cncrxw)qPqP`WFld0B(tIu-8qXf$CQ1F=nmH!A+Com?R`y|Ye&Z> z#q!$d?Y%;YzCOE;8;fDzzXkZ@r)~HE-G%ElPG)BX?@nd=JLio3dFsi4d$+}W;qmqw zQeD2r^VMg!pV{ce8j9|(j0xE6_uUJ}*$n>Mj9q03t0%iPFW=LiOfMAvj#RQ{kpAw- zW;^E64BGbakOQE)F41-Oe`XdDyM^8?`(zv*jd0!$pXo6w&7L7tjhCuS3Aq=i%W`h! z6G5b;iZqIvGl)F2L!NS`8rxp1h;V=ysksjPEvM*Xf*!0;FTleHls&H zA{yYXh(-8UQxb0g)m&j{18Oz`gqmO9Q2Du{G;cV(M~MMM2U-x%?58b+Wl_sMR!Z!w zVW68>Kc5*)%0qJGxBr$yLr(I4v?C+zpTDT1)B75i6H-JzT2=Ss-#umUbH zd@&2LxB?!yJ{bbyAg;{POMW3(7QiTJ^Z2c~MFiHl$D6`}2~$&7eR|KMo&^BYY+)yR z`^5-)C(iT{Nb8h1bO_POgm3!n0m*eqoawjB%V|=3B!O|HS9H>cq}+h~k?%VkMe8T~ zA4|M4h65^32ycx+#50=d^;|NFJamkuw(^W#VY){?t(i|B(698?MR^N`uVkf1UR_ZE z^W!Jp9T|rCtuqb)?w6YARb0D84uh6sx9Wi}zbr2aEsVTj?9L(Q)F^8kZGCF$Z;1#i zXa+#}sta{#37O-y5}jd?-3khOc}41QuXxMu^htoyN@s$$-UiI=89Ddl&ATcvI&;1&xy}5hW-8_( zWjDnJ&^L*S7?8|soI!dZ-9FiJPRZJGj>o8Eb@Gn^)0G=r<-U3HZJ3cx*-yMpxCLEXjR$;1*V zfeRf0fwC3vl%Zm+wT)pzexJ;{i2|*>ma`r2l#W{3o!Uo_zqNyE)_iUg0DC@MJO8J4 zZ`S(dj6A*Bw^;MPfsrn(|jqX-gy5v z2|Tb-y zf)qUTY=muO%-Iw0n%q}B?pAFe3rJ7}`dAVo7^KQDT*#AxkrTk*>rH4|M)BB9O)4Y; zk{DTYG+D@Z2;^OcjWbqTj5m!VRG4=cn8-8a>Mm>umoW6N0%6)fWP>2ld&o~=XJpEf z?ldBaIP!XmBUDpmagrKhW2va$iqE^2@@!~V@>voZ_{J_pgs8)jQRJd*hT9k4`k5e?MwX)`o^vh5) z27^FT$JWLf8rJN&js;0-{F;R2`B+b6ab;%O3bFwgU^!~48Pq}~&1IJ9vDm@HWShs2G8X_sK)t^Z zBv{gdUvZSW7HG4%sz`)%XChgV8=SfWc<|0lbwm=6wpdn$#$w{)MzYkSy;(Vubrf*4 zE>q)wVK9)aBL5iOl*ZKDJ=>7vDAkv9RMG~pG_fSupn}m4qrmGAVuOt>i%=)zh}DaV zwQ?@V+_H}r*iPg={h6En$>1#EqzsFEK$bDha_v= zfVNHNZ?%T|h!_etheI(5CI2EemQ>wo2v?)=W{7gQ5w_~&yfTCu{UV(8((*mxH=(-59G&9lem(d!B%@A>7ul>M>;&i|ynOVl?4HF9&a@bJ z@&&2%+Yvj063l`-jX~n^Q97h&T#oEni9FSU7l*W3z0MuSUBGMaYPy{<;-hxF#7Iow zI_)Glg&_+{0`Jer|!_I1>ie)f*NC_|>19E+8XV~{0R zGD5I-kWSQzy%k>r4q)}gs<$&#Yp>hJxN3*sOou)~zm;+D9P$I6XSCwfcx1k^k}2n6 z@`%u4Hp^YVN3j5Tt;Z$Y!S$+g&r@u4s7M7=Zq*)rD z%7xOsWA)`**gT?*dLc?HB29jtOPOIU5~wR9u;865!dY{9d8DbxCjSx5jFkd5oeO_~0Mki-)m|6swm$LEm z5F;%33q4sFV#*z6a}gXK9SX=FSo&i6Kw!O^`(n0$qMunHo>oKLk#5d|^SFrjNMU;5 z5`+^_E3DAmW`X$C5gEXM^aSQpK|R9=l@cw}KxG9w)aB2jEn|b! zZGz8KxvMr^uypb_Y1;y-X6#}XYM8oO#M)Lb5!gC~XRaNJc(yOaqV0w+SKTZLo!j(o z#Cl|6^~KA3*xk!qWsuO~;a5)S-JkB{y~JN)*Uqf14Coi;sytH$Xiox@hCQCqqiVqe7VSZ;=)o-V4M~+*kO0FS_m8(0lP0W2`=fTrw-7xjnGS?hJ6y5@_PDcV z5(NO(xGt2b`;7&4g*qtZ9Avlvd7^0aIEFciKQA z7F=q~Kt377mzrg)Y*3d0wA%p6b)d&JEgJYoA>}%z=m1G%GOlKUj0vffnVmYm7!Jz+?bdVFUYynE9kVhu1P4K%BsmgGuF%1nK zX2!*zjs2@{vqmNzb5fn@QHPuu2PWJh7xqCio8YNNWcn$wc3NCRSX|;N)?b*6D6=d^ zzy$Olc6*~RALz%oybgLtE~xGzkb3HHyoI23G!O&}0c$Bhbu_TuwY0aLjr|vsAazYZ zh}tEF9WPGAZVi_AcDk8kl(if|PgAMhCG+}LbU(@(Hy0#vzy;0Oa!l~_m1RAd29knT zE5TCKR(0e$N=jaAF5sD+*-G7NOVO-`DG=~y^Fo&6@dymbq41W)Q>z0? z+LQ6C(3UEf&sjptZ&n)7o3vryO3P+^k7`yub&v+i>|3s zYq(0N`Y{0v3K|8`C|c2Tw2A>r-exf{$%iaaD*0%6nk675uOOy(;FE{28n+9iv?V>E z%fn+AzGX)W$p~9@-v%cbofS*J6qz#& z$AMmUSlt3b>kQDmAT(S7Zf!xfszig5JnGSmeQ_%F>rAY=j3=R#zIiE#GtV==pln`{ zfk=96Uh>8O^a~nuUZ7sF(J)CRU`7d@J~?#1N(sMJ!q_o^6&h2z$svN3FincCT_)17 zDtYwpqL~h{ZVXMyRlC$?OjQY_V+;UltJwUO?d+so%@cl2^5*DbQM_I79gJi1vVCHe zaLbmheM2n_ztypQgdM(Z^Z1Zmo5gi=_Ma|d61*6;A=8$(Kylwvhx0kr{+Il$z>V&nIP_#~y7ko2WR;ZmD z6!yoAmo_g-{Ey2zg_}{IHxt76mV?4>0Kpr*x4bV<*lXuk`6pH{gFwvKAI#hKPuS1j zzMx@$Uv7gxk#my;#*|&*<+mUYus<9==Y2_hP4Ys`ubl6UKQuXw^g~WAuU;#@KzQx< z0x>Vw_RGJ_x-I%(1)2o>^R^HV$v%?YH@=lW2ftx_E&WjI)&=_OF7x)&L0?;jV$|{V zK1Z3b>bM49YmDLa++;>^W4T^)jHz}3!?&rV5pDQccd-52YzdpxcOh_PL_;$uPX zdyYtcPP`$AcymW{vBs}aN4B3UcbML%uV{o+2;S8yhxStN-|`hhdnpaCVa(8Zluftt zru_V>TDJ^jp>xTs{OasSdy^|*raTFaSvHejzi7NBYVkZYKy5)wC=fhEaD%O?s@7;tWM}* zINg+E9{6!Cy@La1$0}AmfEQ_sC zR=Ug-ms`?tbfzqmY2=D@uUJQ2kaPZYW{-B-kuxgtepTVD;HQ-)jByAxvpyfwI8059rmTF*Inf+J!zZUVU*XtscK#Ng>2V* z3tBIb$qqkJvrE~{au141unV3w0ZN-9Qr0XLy4n&a)>D*4ElOdjg>rP#xXse1AXlE7 z)xd=dt zIgajChWMI$>H@nwuFp(Tfi<7@Dfiq8pvRoYXU7?G=d>fu?jEq@I(GzUI)S>n?G|ct z-!0bWBb=9;8*8otd5npK>1;B>x~dId+(D;U;zL)qlpi5^R~vEitT)2H8i;$`6J6-L zA8LAScomD+7Vl4bT6Nw|d|Z{kRbGEweb`RC13`Ss=p^OL#Pv%yWoW$C+txk1u=l3+ z6rg~6{P^_Et81)G6!1MddQjEu`=BWB?X<0cea4-%_nV zLR}a(@MI&5*t^Lmm8POPW#q50jMq=b*WT^546M_GdegYu0$Ha%<@ti%0P+%tv((*j zw#1C<;9n=?= z{1S4v%-jL~JM_2QT#}eT5m1i^{5F3=tRiZUlp}pF$>YiI0SJT({r$Ad$Edsb4WpA6 z0!X`_K>di@<5i|!N(mcXT)n?RhQ_#a1NQAH0tlhk7IJRME{B|82XAnoXw%1izCZ1L z1GNT9Ki?^b?w_b-4nPE>)}w8=?*4h+^$wmr?N;y=YhOiRt_78PFF6*KPo4T#PdnRC z*8zEi0et0u!u;ZD|IQ=pH@GRg#p>lDT*<-}wJ~Pzj>7f&u`b-WP(E0_(5g|1I-B3NBJ|QRp|z&F26B$zuVPt9@GGPSq^}w-5i|_aCTsTDum?9`|Ikab&HEMCxeNozShk z=E2?V{kF-g>F4Jg|HZJy;eYjDSkDSYd|!dC$~wH8$+<9w@W)d*!rYy zR@Vx*vxFu{P1Ste%RBP(gZ}w&{=8|s6+qG(p^I5YEFEK?7yC?envEZ>8^B%z@g*kv z8s54F5YTM!^1m;XX8#Mdp;3exr@`Wt9e}s#sM7zG_#s3PwPA0&_Dzshh*#oimc^=nO7G7%y%V!+rDU zb-T}FD~^Hs`TxKXbh|rxR{ee>A#FCS%4wDsI>!D_?iBLiGw>ve@45~eAens;6fG<&5oF6 zbf!|L8mhx`I2$FpD+z-BBXycT9-J;s&;R(?K_mAC{w*m-;EV zaTh#&uMtIV34#Z0!@4vkw;UX`9qbGfB~TWiJicQ@D+sU*ZTf*0-&v#} zqF1DV3#?s)Te17iUzeHd${j8+(7O+P%Y0`y-e?PGf{j%(xa*@xFs(3 zA*Rv)t6E5J6g{(U)OS*Ckj<0kk8vYqUf>fj7mm#h8{8NC%H za?y|8?oCIw9P626uBpo5#-NBzPlTk*$R?ZCUV;9!yN-M#e;`b{8PDi|7?c; z_gUuuCi=?qz<>y%O+B=I_tWwEhY+zN1X3uVfsG8JcIy&eOma3&R};(*g1S>WF3(N| z|B@9VnA^hdhs%LNA6)O0*$|4xbUjl<&&H+Ot%A=>0-dX&^EqqBk|SM)x{Ii*X3dhh zN=zyyE9hy%Oq--$6FU8wCcoh?Xk)Et<>%N?n=IlJu|O8{(?jjFHXf#9^9DlF#wuS;pJb$CewAv3tUAP#pMS8P&aaM|711R7M9`*rer2LZ!XP>mC1yi*`XJ_rbXe zZ_&;-&QPrBuf-9j>3Q>>6l!}_Qo0`U-Q@PNH^|<3G16UDrbs3sK*U=!$>zkHFGavs zyRpIcFc>+uSRkW0v+xeL-QI2GAQwA{2Yp^Wv+CP~TddOC;p)?5zT0%8e!z&N! zs-ntWb_pUGftLGrI|VvgTS&Fc4tWPc!9oGjd#EFYUrLhll`mSrl|-nYeB(?Mqrj=L}z*+xXGrNM|U77&A5?mUObMKrY@& zvKypyy$1+iBX`Wvb9ybcHpl9snrKvfcvFDomhg~zcGc`;dvi>c0L0{JoKLkp&UT&& zVd}ER=s3k@UBloV?~c?yxg%DIx(OU9Rv9i+07d#RcS_gT4(WB=#H{+HJr;HmH3lU; z32;IudP;vq5>ZT9ii;-+;xBvBC$dCd^|bDAvPQT$YQ?$#8-XGAzpyFz|3XA|_9mwP zgPZbi$q&ulGe!SoH$nVspUHnuRu*-4aQY|uZ=cB&m463`e~0R&($j&fbms?_&!-NO z5aDd)gRv1?R!SyN=(n2Mgwb}nFo^u5kna*k&wW!KWt=slNp*BLo6lxH`0RB|e}BDP zaQLJ1E;9M2hq~iH|H3xIZ?ojIwe#8r;L;V@!fYMpny=3yaI9MQo;~o{eZWmL)`#?s zGA3Q35UV2MZt<&GjP!WDHL_gaGW)<>&9>F(b0g&L;yAqG!X#p=S- zN(-s0q?>NeFjgnuxL9JDjpx*Uj)b*JQ!FM9LW!|OmpDSF%}rL}A2XSB2i;cz#uQGq zUmRx_kwzU=hgw)`k06G;`-WYo9^3CmkFHav-db;iZ`$EDd@Nn8Ju-Q6x~1LEg2@*- zncFi)>9)-zC;_1-Urh4LE+mXY7Z0?RfCgb_^c%1;*CyFeL7L=AMjux$I8xc`wb*NMrZfS+3>94m@5@o} z6TzZK95nimMHZaUyP}!*b$*NWqo}*&Hmk&vX0dcF3zlN5Ldz`QeU$yJ#(VxV)dtrA ze4alr@Er))E%mr`hF9&`5XEcR{kFGhLKrNIZ0q0?aXgQQkf+Y#wh8vR)tnp9GK)88 zIp^UCkXSS_XnjHG$9}&*f~M6?1d`nR?*j}YfiV(>QN$XeExFOTHQS@S{2Q;0Lm5lh zqOuP!8Ar#n%3cxj1b>*R6<-4{%fd-2va4XY{|{sD;9YmSbqzOc*x0sh+qSV|+iIg7 zG~BTpH@0nCjcqk{`nC5t=Y8LEe`7q)J;olt{ST~ptvTnK>$=uqVRF;_iqR<&G9W1n z3l880a+*p>SFxF!bls*y>niw~H=L?JdePmSSTP z({YJeiK^|&XQnB)-jTh63kjF{G0`stB<`P2lJlb?V=}V(zufz0`d*w*UJ3~XfT#8b zLy8`Sla!MLQHu>a(*f1?8;2B{+U?a90D5aT2&>C`gZTqo8Yv#VH8^=Uy_f-k0Qx$8 z{>h4?M1`F3o8>!5UX)mp++28EvCWtkTU-t!?ZMH(PfRa_VohU4OCe)&F$SeN$p{AO z7RlCH;B|hD8W>VsJ|;&C(_sO?vj_=#M(+EfxNq{Kar9gs6uf5Bu?+jv5VWT}h==B* ziklVAI3l!6R@jN7kIB~Fva6wAv13Rowow_`r+2??lO{7G5Im?Zwn~ zhNT@V-)6@Vzp}rf7Bo1tie5mN|JcN2&)c>9mPJCs*>e7sp{L}lUD`oE+=(Uj8<*7l z6Wr#dufM2k+S&1`*-;*hZI2VHRlS~#JjH?`AFs_I4%HXn5M4rxS%o<^wrW_3pPv0R zk8dOYjcpmLT&FOLX`nooorb#ITvZGD&;9n<$%Z`YOy|-ESEwNTWV1Io2YB1L{rQgm zQZ?5xKbc{Z=TD9Ps^H&z7Zk7%j)c4X5~WX4m_l_wF5p5rhuPkCD7HPA;S_2tg}EB_ z+(tNtzlk2Pfu5C{Nsx}j;(?hfWT(gu(_9kHU- zyXIEhR5VVOSMx1Qs{M!L zDa=tx5Y4j+`p1-&Zt#_EXj-jMT81(=pZ~^UW7QF8jeoIevvAMX%bkrpH45b zQnHfZqe`+s0*8wXal2V7-+HlE_UF?;Q`&; z_m^JQ>~euaj7?MSi|Z8k?Q4)}%1JUs?$^-?)^nS!M% z+rjDLZE{5xeL?2R_WBwt?@SUM=QDQ_TblSR5l6k74L_c=YNI}!O!Fg5JlABkWt%Vh zJlF;0`WmlafFd|Kd;LWb)6qB0$bAxsV%Qd`R$-c}>;ewuUTX#(JVqGVKM7KX+d;Ek zaIVKrM79Y|5h;#} zI?9>8$xRb1jbfwCT6BV$Wg%r-Abq^u*b*JvT4e}U^9?@Kg`VS+F=TC){TCkZ@h083 z^V35irN2KD&05~XgX7q6bGBg&v0qScabu*CUACWiQa@fg+;o92U(lF5jBu{XzY9uN zj%7z%3hjc*e_^qEOZ%%Q&rhq#rkwRgu7624i2o^9FQOFJt7f8IB%rh0qv=??kYRV{ zZ+AzjCt%E1d>x`Hz~?nZXT8nz-#CR?D2`YE&H8r#mseo^!l{Vj@9)1+s8(laSaw{R%1J& zdXQO@z5y|+Z@7Eh1Y8HmVRTRbxx3@b>g=%|gq{~868P9n+nUcxUQ!Rb!A0m!#TF=# zLtY-h8=OXO5pSBhfRnG>>^%4dAbaf@AJ@L`TQTg8v-ovlE>uI-(g0_itz2JSZO-#d z`7wL8I&OeN-xAx?l*-!L5UM;iwQqtRv!Cp5FM3trsB}DzWnM;_ zlTS$V#HLEO{CrX5BcBb{UBev@?aa=F5;Pt-)Cx5t5R+_FwK`X^9JE|eZ7!bT-5yKUetg~$f zZ1Px6o?ZM!%(MsU=%M4Lc42_&m^J3{_f3|GCNKOM@!TJ6P{72|_-*rFkW2!Bf+`2SUq71bC#29nDcA`8 zniWSyNH}yAxX3hX^AN}yHB=}~J?$@%NChv8NH&Gal?rjBD}Bye8hGZY<6VgSB5$J& z)wPUy-#K(FT*tw6o9?GKJaII`S54i(B1W|N;mw2^_2~&k7F7`tS@~eKARI*7?PXp1 zicwzw6LEzJ;j_pEn#M<`oJz8J2zrue^C$oml9k4Z;x!T z>?i4|Z*6C?NrGe#EXmSkJFXHHo_M4-Kq_kG6nqDiw0ck)Q&QKi1^BF~D2iE9p3JYt zNYb&)hp?Bb!ieS9E4E-8xzQhLr3^zmj)|B!ycp-L8{A~WCL);d zwS<3tdN4vYe}VsZRin*gJS_aKYQo>LgXVv&Y6&|JYuEo%)PEf5_v8&LJYifB6bZ&4 zdm3>RWNOy@@KO2VNtpYQ6y;=(wA6Wa!2Ffean!@dADPBg+Y{otJzm zXH}b||i6O@4 z--HOBcAWKG7CCvUzIEqlNEiW&9f5R!+1&y~Pj4}vxYqD1k(b0Zvl-uskU?W!>vw?k zb-#g132itz&J? z6@-7lfoR%1iL8=}fW+6=pfQJ;x^2%TcF`HVE+u>Yvj}%iSma0hQP&z-&q20Cc49%J z>>rWs1nek1+jY~jFF5$>@8G*5?350J9$q-Rnwq=Wy02TpE%G1 zgbvNquUS6u6U@P59+6^Cm>F2?v6zF-FXo4xr1xk7InyZwutclRA4AzolzVUWg%dAk zMCiA1lUWKKG(v;$QlS$$f;0L_D%e?N$5K^MRTtw#^Ymcq0#SoESOr0E;hhf(z299b zX;CXskmIIT8Wa|ls5WggvIG+#nc-@iCqDL}Dqz}O|@ z4B;POIAyZ0A(hcT+2=l;e|o}9s;0PR;2L@RqMu?ubs)G~N-u&##rmuc9Ff7rBv`$5nX5pn6+`-XSq+U|#Y1H_ZTVtzC-%_Tj@g*gtTfME>qTg)0B`;Eey= zphNV+Vr&N19dU!S>y;*oI68PPR^fwv>q>mH9%sp|qcgz8Gl4f_;oOC=!uA^7tOQ91>0({Yk%6$^= zyz)o)2$T7J@Y1Hw&u~8R)j(xC?VcC#sA`g#9xen1J56>tJu59UI08F(-TCjWo?U-) z_C)G`N#_36VtKIt^9w%z(qfX1F8_I&T+b0r3jLk36HqtU!eReSxtGG3^V2Fk0~lT( zWgKc^5t3rQW}Yh7y)ja%fo+GR{b!la(9$7rgkB=x5k#TfmL4oUH8o^cwXTzAEq*sy zt`-~Z{_o)35^JNz2sP+C(5TXI^j-n$*SfRzazj;|<;nSLMRrXLOJy=+m5_?WMABTy z(#eTsLABj}0j92TOGR4l;QK8D16k}b#95+)-747Nt~v7RIuCw<_?F6wYHxIJ3@e!2 zT+C|Wu@#Y-$Ekt%gHW{^Qx!BoWN79v$mS={F{)@1*Hjq@HGGS;7CdPoE3YgcvMuS? z8+Zt|M^)mE)!WjCzQ;o$g09-pqJjL>fh%-H>_=QGsN|1N^|*?9x?K&0UFUKw`75-> zhB5qRGHUf8B2Zj;WX3~gSj>XkTI=2Eg@6v?cYGYb;5nLc9PZ#ca7*9~ZQQgslmmj| z3f9$!4~ID2?ZN@`AhS5j6p6q(vTFLDE{T))h53?>=~L!79HToi!hQ+~IjicGYbX3a zC~UIUDbMAvanku>s$IZ_8&g}UbV@c!GbtiXW6X;zZBv}SqS?vXRX;d#NPoy)I=+)k`s$c%vsn5GHqct3=8g(C#Omy&oI5c?q?af{2-Iy zr*i5}t9z_k$N{y3O6i{>{er}@0;ZTo1@}uVb<9^ft?o*|Y+|{ekaB!^kFba+VE`NM zRv?DlpJ}d1+FLfYTO(3;P)5Ix^{oX%vJXo8OLFIcXX4kw%d6l)FIloE>j;*?hJI=^ zU6V|i>y~gLvP8@Zd@_Kb$cYM;ysBg6Nx!5OG6m)%Y|1mYrEk{K3vbn}d-jog=laqU zYvrwT_J#Y#$*FuSv`&p{7Pd*utenL*PG@A`SL8V`*7QNRBtN~6JOsENGD>RBvb;{| zqiU3=w&YV*Oz)l@FH=Pw8i;n$_YQ6eT`{0l`A0hJc6y9#)ePjY+-&nV{5jMUw9C6= zxGUHtWf_*mQau{IxK zeI9>k$hBpyv#-MT_|WseQ4Xm^aWM3oSrkG46K0`mZfWlIA7)|j$GR5~6G~1N%>|Ey zA%$t&Hw9>S zPxp@)(YQML%)9EiQrD{c8A5{sYaMP?L~I5!B*VDl*G1JbylYX4&7-HdU_g6%Jrgk z?y~mBnxQLK6|Am!Nt#rKl2HIHQ*cCGr6zD-F*k?>fo38<{EF{AV%70{F;lXU6E#@% zNtQ%d&$J*B9j{eSN-S%p?2YhY6rro_fr@m~L(K2L2w}SS-oGZ$+!eD;2q04~*o}bA zd`lccP+D1Cb>X#-Qe0&Gq|!yoD04lk9`KF@PmK*d<@?&{vvYI3;C0<5F#^(~kY@e<~Q7Yo%+H8_txeRT9T3y3_4)(e|!*@YrTC=O_*;M zJ1h;p$p2KZr&CBxfrQgRM%=_KJGEN>`~|t<%gou&ajojJGcK#H={Km6-y)ct@Ub9n z5M4Gk`zG83%-?M_+Bl+lucKI*IR;i(!jnY?pEbQ#pwL{M2DU`n26pa&oiz|OYoqoz zw}{#sf(EElq+rP886w9qo9Q_aHaGKZy4$e5h;8;V9YYa@X|*6Lk4 z$~Mp%n<<2piwvj35S3Ux{ms`X2@bG{`;7+D-)NxwU(q0LZfb20u>1dwvXTp-#M`%^ zl&cv+C4|N7#6&-!frsr=45 zR=*0+p%F@*jRT|}729^woq52arOq=K?&22VVE*D+c_(y?G)>@cShQx>;6c*C?L%;| zYA)HubYLq-=sWX}kHupWxFBrJTneS~8#13(GDBBMauti_BP(?agm(v4jA09L2irwo zq6&&NlG+&}#^4pq*=g%f%13LiyM^k9jWlnf)C!}68-Q;Dgl8;oy*l7%^;5_gCX7yHXT}8q4$2!Y+lucU z45Ewuv=itH?R)2YQ%N+?>{o9gQa%`^XxuMx;@E4Azi|@oo(vyDcM8+T$poR&)lhyKNUMgEmP7D}t*VU@{I(s+tZkd1b35 zqTvf_yB$|XtOk-VvP2-R1WT)CAWvOXBaB^ z5`c;)=A?@1q~Bfggl&laVzNmpCz;H*Z~1eW$rwnLnmhfo`!?-YPtK9smyV^<^B;=o z<3D%GHGMs&mIZv1y5665l|C$8N#LW1yIVn1GLOVvzqal-G_M{s8rf?z=rw#6@VupYWKUWtHrJ$KT42%(HMl&1h8Rhlgm z73{%Y!$Oc_*oE&F?GZuKU7y2v8`0+73MN>wTke&lfia!6E0Jo0)d_J4hh3{>s$kGc z9PX_sP_KD{IhUK&4qwsU&T|meEIdhKy&*V8$z=pFn#|Mf2AY#LV~W*|z%oS=gC~zz zv=HO_2eCYAeZv#>hwupWEcJSP&4K$Q7_nkZcF2+u{`+!k1@UMzY25>M^hSkUchL~? z6RbL7M106=gHxW=tz1t zd*&kSovf;(7R~Dx2c~Sw_Wj80zQ-7AT5nMpXP*|2c||lP_s)Tr$osUi*ztxG;`Y3s z;wUThr4aD;N>8?BG|=G9V#pzs_iIW=jM2Vlh@bTBfWL-n59@j>%CoFwT%r+jE$A8! zx^H!~O)^K`hPl@4aQH?b?-r82c*;EBqg<6B@Evk4Hcpz!YBbw7DuoDeg#{P6Ly)AL zziFG=k2t)q^?uB&;HmW=HqAsrsWk;<&2B)sBgPIIfsEuYm;qRFf{pLCR z0M>Y<)mwj56_qCaE3$uzR#-GmeUd|?GsBFOeyTbeNh~Hf3@Z%ja0vf`^fBDe?FJ}w z5~8~kQ&GV7q8*dB8P}Yr`Ry|Xr+O{?dbc5&3AKq7MtE*W8#;r`RGHfK`cQ{wPF$7V ztJ-yv@4#ocVl!Fjm*!q&-Sr{3J|u3r!3+c?WSu$8`lNBk=BwSV&vH&H-Sv&t;)pHc z3^FK8u&fI26h}V>1TX;iTj>JL2Qd0|AD0leHHWiB_i%Q2S(R$Izx-k|_kaJ}HJL#t zeHQq?&yLNH|MLsm{}O`Yj{h0R$!MX8Ait;6i}i|Qn!_{FgUi4~;RTUP>hy<^5(Z_q zBjh3?_Q_|;dp)DQay@4^6ZmEGKj!;sa+zw2iZ+j{FR3p)EX;Czet&<2-6A~5w-ON+ zQza1|p~%^g}p?eopxR(JMm0epIEsZ+j1Ll4d8EFpEhn1(R``kGqZDcc2_ z)NH3Sc^9%sO>uT{_j(b_qcxTXh5Q}ZF|U@EK6t~lZP3`JX2{qQzHfzz4(n|xYn^=C zQHD9Fffg|Z8#Ovj$Xmjez|JC&K#f(a6Wz-9x?}e<_bXdl45vHF zsVyiuFhgQ#w5@1`PAt*-W{Ce-tCyamP^mzi!?-@93*Kn_#zQFB^#ehrM&d!b9VL@@ zKoeV+K`2%dth9yI4ZWqw%Y}I^!t92_c$%iScSb%Y=kR^|M+vLB@-~gU`j4gKE}kRQyz)0WJUM6;jOk&NN+FbEwNcUx)k(jXn?z|Ud)RwM;F{?1LWqX~qcxGNzvRmht+6(ZB}RCF)mAX` z8VPE{?9w#*B$%qvM_%`!=Cxi-f1mXPN7>Te^9E@hf>s^9n$`X@tg2~leY zfQ$G4@ml_JcN=d-|W!`m57vSMJq4?f=$GI?PD8B@AS;8G3=_>3CpVYAc#$<|nJ)#GOQbU;SnwkH1K*qR3U zPV<*+Y#$DC=Xk)`yrC8m{Hhxg_KqS$4xrvVUnD-Ne75RObfjL-Z)pq-KX`-nuAm-n zI}#`of$S+!Q$Wo7yZMf9*G z9=HF>-vP>MFmDyzc$uu$LSZ`wJ$e7yf zW^;W8&TN(Vn#XB3)O30`LL>xLUWp>3Ccxu0$U_wjELHvHL}Swz$9(&%;#Q=k=>GNT zgxY8zZ(wB>FK}4DWcvinEL6NZDNhV(*mY6muGv;! z2mefK#+ELG!B)!&JQjwT@m>cSdRN9R&?$Xvm{pOqMJLCjJn~7n*ibP`)HEQH{|r6% zk<_sI-^%lm|F|8&)is(mJ%IQZ~MdO>CRu8SFY1zo0rEdLUMx~P2@<*+Qzyf%Hdiwy+^ z>N=~s{T6jHbKa2w-)=uWO<@LE;#@P(njnp&AVzAv?Z}iYt3RVjn*AF6K6j)_G6hSm zgxCE3G09Ar0!LYfQ&|RD9fYW!oOpM{_1|!-GZ9o7{|&ck*ni-7sEGmGOs)PygY{hJ zB+=hFJ7M@nC1h}MXH^+P$z_A68QA+B;8>;$+{fh!uII6ImRH~-&B)Ve*!Y>}gw!o? z;j$EqmHdxlO?*5GSNb*6L$=M`?;rI$T@8NS96Sht`31g6Aduvfl#zw;_hk~XGjvUK z#H|L-)vh%D)F%WcA6~kgdI;2#bq;lelNe!Uhc99?PC05K&6PgK5WsHIF`S`0!$BcR z5cEysM1|d>t)7VYHe7HKx=_L4NJ6|flYtxFbm$Jb7ao_Qy4qKzg(qP`>)=Y_4Emf! z+J*tZcA44bk5c>5`v!g?mK)ZP6%R1aR*MBZP=z&7*of+!*U5Gk;SBn&`Jls8(Pa65 zH-)y89;y-8M2k6h5VF5X?-;_NEPcbJD7lW0i7k{B^lM%EX)y9EKE#u0NzJam>8(!Y zgpl%S!Vw|BATk-~jFMy1T(HY}fSrsqXWc|Cc`7m-eE#Kz03~u52|H4WJZTT${tlBO zTbZazP4`Zk4j!djpD9U?5$p)#c>J9$DVn&ah44S3g^HzmB& zF1{&l{qz+z1nvSmNprXO&xc5S8f#PIAQG^h+TO!`>&{R%r}ubnsFv%Es&~qX3<_oyO>#Tl8O&_ElDOJ z+a9IYc3sz5>ih2Ms)r$$_Bt zT6CCY1M}uDzK_5H^VZ^|>$>J!)Eg%C&$7t* zvRl3<;k7gz z_`ePKKdCb@2QwYvyl>&8SUu#Bi*(?y>NI((NpMKQe^O_u2n`p5>xdK@0g0LsO&`ZB zg+^;FYsu5$fnjI&w;m5X{zaD8rw(OA0(_kIaqran?0jvD`0Tkk`V?MnSS8y8;47OY3*@O$n~HD_zGJLI#>P7nnlA0+(GgeylVp zJcN)KmcpD?3o`=UmY{WCCI_8D0j+wEib-vy02yofAq{;p^22etwN_FN)jEjgmE`Z5_L#TIpExJ9nmlS;sfR;4e)rY`KJ_aJm71H#0HPgcR>tGO&7`zkhTTG88BC6fu)gift`Dhpofx2@ELoQenx{9V*ijo+EOZf>m&R@ z)HJA#03@ZCu*63>mn5pf+9IDxUiK);dxN-3GVCE%`lTpbsPJ>Nf`(Zs;SVde7+-Q| zwM7Wj+*oq3LG|R<#7L<7KR;o$E($N<9_ix! z`Cpl?8y$W7o0X&*zC#oB8!UMLEMep3V(no0|F=j(!AT8&f`|V|9{X3q1|9AHN!Xk^ zWUW&?SFz7{dH*#suq+yLq0#V46L3AqynLDDYI}K^ip}}3ctsrxR)A2TU>0Xi{akDQ zL~sGM7C+Ir;2d&zOykylPZay0N|Lp z&SVBbKDOr7DfwvF8P1lOR&pyVBHHfeG={b@@fcWj9!S-WH!Pbi0Lu|fanAJqGly84J zA6;i$8({yIk28*16nOzqboIbdOYDe!N}b9pE*(u2%KyodJAATdifHcvZ(;@fNUfS= z~qCk`e;58Fy+1UO(IvaU18-|j1uPRL8aYZshQL{hn@@aic}(vMbH z0gz~#(&M^j^~sM?>&tvjAxY$5bVP@J{`FaMtAEG(@0t$#)M!EUn-lK-Ba1`E;y4YR8AuK}h!sSh_r+zLWl$n6DT9PP7myl}04pWb(>3GTztp=s z8cMWZ(~3ie3=W8wcUM@b@GUFz4fyr+4$&<@FX;pZhA2Wn;>Ik`?J)5B{I7JLV{pXL z_@llSeygvoL6Z?SfzC1e*jzFyyKM&MD5{%KO&zFWJD=W~x{?MMUX(@TnHi|8T+h`t zw&+$W`rZ9!P9|Rt;8(`aYFLa+ydE_Ue}-Va^mCPufMxYMxjqTt-tl-&XGLp0DI?(a zo0gdT5QV-4wjf7U;-*=6GJ&s{G=ao5OM%{rfFT{-*Sk~fZep>NANsYK*OOqwKaPnh zFw)OO!NqVZkD4QG5!5PNO7$$F@v3ar4BUB)&^Wev7=s(B7)k(~-jW@!7WW|*bBz)# z<-5?5*rIUJzjaqRns@z&&IkaaDMG$Idr7m?CYUZVH_g)92F-;}0pkic}of{$~@v8r$9 zwXB3t)}=go3D=}Hr9$J?pG2O!qCc%kFd4i9M;L#qqUra~qr^`Pg<+2R6CSiW;Z zCQ|4?9=|d4@@^SVs||+D=1jlks#J^d9IB(d5qC57I)zEIQ#}wUO=hLaFQNR&Xt?DUlk7L3!G8HH4&ABl!gdn}mWY#GCsw%>$JzdO_nRpn81stqkzf&JAZTvcGVfo3`y4HJPODW_dJ1~YpOC=E z?iQQfJ7krB?;s}$ZHv@N2Ce%Q<*O49YW^mx$|G!H-Cze|h~VvMvX>HjiCm|0zOq;^ z0R(KI-#a;~V3m&phYLQefeIkzoO7o-Z~1(dSv-U93kFIEucVW7-}k^iY|~*6O)P<- zT!a2d)fn*$Q%41U^!w7jeZf*e#TDR+?6Px#Go%y;J{xn)E`p_Csh0gr4TLPmBeRQ! zZb^=O2O3APl9Os*NT8B)B*1}b{-Bk>=``%i%Y)yfmN>EKl;w1eES;j}@VPhTsx*dY zyeCDlNG7KfbPfwbc-%{4kPv8%Td`jQ{v6e}fP*X*7!6tNC_YJ^G-v zC@FV-|Lk}Vn4eW=arZ-aK1n32cR$VD9jDLF(W%gv1GcEyx@N+utDs!eGsq=2VY(+4 zvEMB)c?1Cg}uLU)WM_2+Wg>~8nkXfOLrX>t=@_&vS;=+r0) zBUZ|!Uuesl8eil$sxohAt>MWrsO$s%6a-tlOyZyev&`=Z+3;)(-Cf zSr159|Fa$tI>m0SUu+Ez)vNSm4WFp0N?$5Vj7pkIS|F{`;xvM|W(T5J9yO3^REroo z!3csU{1PTQ?x5ckC&}JfZ8=nTn89=Z{`|H}^~Jyroh3Lk*blaj8aX>eWeWbI>X52< zRwoy->_^rZ0RoT0heaiE(%1q;n9%fD+7dKQPP_@3wse3LIqm#HafSDIkGnEGfRLj1 zzJMXPGraJDC3!}K++1#e7M85HwcjwF@!4vEw@qQSAG2oKl)HFe8`12(Mnit zf)m>(%$BP$SmbNpOk#SDq~w>re9k7y6)?->2|p1z@zL@*Gr{AsRHx&27>NaQxdki= zZ)F?ggP?h?-sfK(6yv(CFqf2KO!ar7j$YH-T^GJ;CoXc%VPnw}rZA#Uj+>pI6gzx5 z2#P(n@MIe%(-m>i^o{Q7rGY5$0hP;Vr&o+8YCi6RvJkuK&sz5y)6wLgEAogw<;VC{ zjWd5;<_d5)((VH%;&j~?K4`x3kI+IiJbXO1t5EL@ROHU=!RW-Y_YQ?`a!$%$P{B!d zo*3nBj=`fh$zyeJ;n>D>xS1QiSJin~*4*dI#8;$$l7EEC8P0O;b)Y=k`J`E1{k5%m zXyJi}an;~pJ@{;$_Hfaz?=;glwMR$?Q*tiq8&uozJ`b_E@aPwI=mG?3({3qFpVz-F zT()yx^r`)B8@Vw5Al)e;;_7N`>0oc};PxMeCa3mi2nB4H*A|ZW_Jyfisupsw$;t#7 zMXr=IF|P*ht~!JS$96rs4NvY@6jdx?y{sPr{nR`4ST;#WsvWt$g|BaG*ZXtE#`5Op zfLBN-98c6E5o%yH5GD_SW{3grub9%h>cIDZ$1>`AqM!b(TW6=k(Scn}B~r{^O5h;r8WX9oWu&f9hq`Tr~O#x@5U+d(}{A_FJw=vHed5+C_jq z!@+Z4CN>cJlMQ!BPe;0TW-;{Wy_~}f!bkefW2S@*k;@5 zTAHAC|E#z!Rq5`G3^Mi-nHUuo`Qg`c?~;|yy2NmeAnT(!;}+^y#n%@g;pMBSS+~K? zV-k!|L&Hr#s0Sj`b<6~qI67ndjIA-T7x5;@dRBo4OO7CULZ_1?%2wI)X?cX*kAEii zTp*3Nsi?C$bCtI0Y)1{1^Ikm&1w0x}Y-)PC<*ff^ZTT8H6d6?G+UrG^Zb_d8ZfzOC z1jlGDqdhR6{6D=rRJ4*5mn!*7z+(XHn={cB%XvAP`Y+*@$9syNQseJf6q{s_3^oA> zqwuliBHmafRO^B#GFpfNro^!Ec^nFkmnfu2-%d@_s*Im+x%vZgE#dv2rM-p}PsHLG z5KhTv7@`95jtGzHV@PFfV;1=rGRbcBQv3P0HwMwq*H$GZq36ETS{N%ESZ57E4QOfE ziH(B%`bI%fWTT+W{6trjl#?imp#JcqoSBCj5z5lcucg(E1j&{@nRW_>aPoqY zbYLTWM7p5MkCa9O4PW+V7Dti56OK6xsnhf+LklFYXasHus6^w!kX1jxQkcdE6eyt& zmz;M4kDSMMTESy>p=;5z$b7ZUM>;a1JNTat&};PeG91yX7QRvyZmFd)aWf344KhXt zU|c7D$)2GRbj$ zssFNo>;3qH{~;gW&0qdm@)a>q5Rw#T5JfbV#D1A$!p3E8yVan?R>2?@+7o}GzEpqp zhtE>IH3fYFNjs(db|u?*f#>yl<9H83PUBX8lLH z`}+0@UD#?H=OM*5%u=n)TVT{=xL4wZjcvxB{>L0ygsFE+_H%|tOLA?e1Z^BtZ@_iX z61Cb#pXIGt@*q~(R))P5CFu3mZ}F4&(-2QIrE6{j*;{6?tAHaeza)gYUl$=Hy1NcD zIqOcma~m*y#7X>)(a|IYnU-F&>lZO>rzYOLd_uYvQu?4h_3y7|n}=%5QQ7V<)!t{1 zHjezZe`N`&#>0sc0wq+~+?VQLe@>*23fgJaY~*WE=?4&J;i?ZhE{3W!@E~$)*7LXZ zHig@?*u!ZaC+WRh3+0T&@oo>-`KnaWe=GbPNfyGQtw7 za&eu-@mEP!x5;<8XH?{Z^)P+3Ev3>sQc!}hVvV~8@i!Suybjp|_!=V<9JAya5|(wY zn@&6*mK}t_`V)U5wnh`t#g*juYG$a6n~H!*$@CGF8Wwn|pMy3p8ao+O9CTpya@RUA z{}m>bj#U>DkQmktDVG>tfzkOI*MPF2HFNT}+{DU<-fzRdSr9+sKVd@u!N$KY3eeQk z!qY&1|3rd-Q?$0COw*_;OM=5#6I72L2_9r(8X1yjyzpr|PM*NxXuQC7J6P~Y=%?f3 zH(Q*F*5onE&xvFHia%xgA>bNf zAK8J~mV@wQYWE6sH&zcgOpkj5X97y0XA<`SBAi^v~9yQ(T8FM4;K*X|1V_z6lNDy7^1@GTVcK>Cz6k z6b3A+ikf>SkXOvhMXe9nfdv7*cd`nPQNaHK?oDNS6;~3n{^qB)YBxU=4N%Xr*kQy4 z&Ip#E7043BM(zPsyh7QOaX$!KeQT^L2JOiDy{O3|!GHDJCu6>yIX&E>{^Cz_V?@gV z5ib@ZKQ|Yd$sW)9AELi_EA^Ed8*+9G)Lz;R+f@D_P4gv4?Q0MEh>Q92Q=d57<|jGe z5L>LkU&H9M>|;L0zCx^^(x%bidAdA4%&f0Rrs?4-^fubPtYZ2$7G6~?gVH&q7-~@0 z7BigGPNm&W-`8%U4ceK9e)f3}HlwUWsa;kAZ@h6$!U9O+7R{F>40PrWTCRH-p_t%k zEaLrY8xgb=WxDO|AILCGh&s zH=ku~%`m1YT;p$z=v+SMFY~s8WVvv^>7rQN^;4QPi)2E-jzrIA*l1gtJcXqx@HU0b zSLLke)_;v?Qr{HKFF56QX6siyCWy?%Cwv=+Is{3s(8WC->ycrr0DZC4xVqA?M$MXK zh1vMiQs&gs}$=u(-NFt5aJ;o^JR$O@|g3~=m9RjP;^Q!o4)!1pucPZ^RFrT^N zg{zt!BTg`y^8DtqAa(7neGuJ3Uy|qV#>d%`6aNBwn@G7DE#3cjqVXs&mWt6JNs@AR>u%x7#)Ki$DZ{ zcUC@2&mw`zn9Ib-&`Mo1fKZP0MQ%R_js-1&STV$nW&E%32 zhqXR>gWW=50;#8lRy3vcI3Do!WC$~E3!*@ZlmcH*yS|-L(Gd4KB#aNS;#bfjhxCQx zK`8=zR^zuA{8TIUX?_}Gzj{M9pS&0Mug0je;%k~@mY99akqC+ZM7JFeF#Aak&VOYJ zV<#D8N!l|9_$fl}MT_c3g%Cgpn;i|gi5MJ9+H;#iG3*nMt5xq{B2V;-;KB6!)K^;H zOT3dg$3jp~sj)pKs`?63!4Blam>Q_kv;5IwzVjTlO1A?1>fUBhF3?#+-PZ0^s~xZL zonX652w+g5qh-6e0DWpR(Yj)sC1>;NZ%G$O@{x_o-(=nh_8+W`t4RR|ZgKV6)>_k(K+ zuMnV$nTVN*6;3iLB>U?~Rhy_G1${XKah)%ZU~*1lQ({uNP6j zEqY}b7jlK@#g04kjmj?B+K%&J_KbM4I2?VI$p>~|?JCc9f7YM@?U`d%^PE{pzq3krZo zwQQeu8+L~IAQ!$uvtzxBY19b<)FxcA3|eH~pha+uQUYls-2b5@XlhR~;hXxU*Y15L z5u6rp%+sUePe)l=XE&vh8AM7+D->d<58i^^`=Atl0la+~fsD7*L#oSm{S1j_T~j9M zX>;g|V^J-zu)V*5bVovgV z&5r9AaE(iK!)o?!;$?AXSVUqT_u4om=un91dtM6SQf`qygW!jP9YMJb_CeS8lma;b z+1tQ&o6FOI4W^uOug#tW*X&2C1WMkb1WN6qL`n~f1jMrVm7@LJX0`F|>EZxs(`l@F^^z#@N(y$T>Wlih0lAlg_<;SvlYKP!gkIbx z;F{}Upd&h-u;wu6oS9}2^BKZq=%nvkwCN@(EsuIi*tJNdEhGv%whA)4)W-MG=tZZ` z8ygJK_Ngc}3%c1ARMt59Tg^v^BT6U!uK9+4H1`#;u?JcI&)io9OBP!{6B!f-3m+>zDoj-0qA=#y&)ggm6*Oo`%%YZ}2P{|?R0cv@-u~)B;6%wSfsFI{ms4s6+2v(? zOp2WDXQ|YeLG2-+N{<|4mQUR zg&R!H&J9Fbu!dU-KgJVrqY#h)pwyuCPuk**Ca=3(oqA0$TmLfle;FOW8TT|vz4|z} zJyHD6w0_Y$@)t87hGo^X4rV{Z#)yL+a>@kk6}NrwdEWP~OQ+S~7`k$en+zGQ+B5CIz>z!d*}Q527ZOmBTmYRR&< ziJG0tMQ)u(bXfV=fB0P=o#aJeREI+xu+kIM)a0Y9TUuil00^xq=F?ER;Ssf)WZm>* z(mO!iMB;_i@~1vG_>&=3dx(H`8IgjM_se9(bVo^PX(kWFhg8||`9PgvIkEW5KXWH& zLd?jdKqJyV4Ig(bgY_)i{YH{eA0#oytPjM>S9cq+#0xll)JYP!^^>0E78y-bz|NmO zxqrVJaqB;HXtQK{LJArJS*LSAN=AR3yA5K>IZleCulX8iPaeHSB3^;K^R<+!Z=Ie_ zGsL*A=>rS3_}wb7;_)XXIDwTtT%e^vlW`2KuHU+tk(;GBdoP-7UO_JrBgdIT0^{E+ zduD%C5A>C0t=Zraz1BV#5Kj>J$meCWPOSCmT(Ayp*Ef2_FVNLpDM9P=+)^DX zur=<R{Ovh3>hzNCa95nuT;-@5K6XPQf7j9us< zHko0}6w5XKTkg}2l5*YEzwLAs^7dAcU}0beVE=)uL|E0u*v0IBXLEm?*RNR7eI_gA z^~(-sN5Vao=qJ9rmm!cLAj>i*u1wn2#oDUWl)MbZ|LT1ID;m=RB#d5)nCgMQ(k8dk zx3kFqS@y8cGKUu3wF>fXZLXznYBaGV!KH^IQ`S~SUA?3lD{%$DgQ=srVCg5%!#i}Y zj?&DSKRa{TO773Kc4@zrn`|(Wa2B}B)fTt8P~2(4rEzQ!C0ltoleQ(rr@xxq)34|% zzFSGOne3a5et`JuDJPL!Ny0LD$awzq%ZF|_(N>3`mdPuaf#^>s(n3SK2w&tYaLs-+&W9!vpfy!>31|o zRugX-;q&IG0TCRaYMjIMgiiyQ%_R)ECypUu@&e(&WSGWlP6O(<{~Ps_@QQC&=;G!zd`sbm!W= z$zPu+NCxs+(*#tV*_Q;mOIkJ}tl3lwhgiK75^&AjAwmisaG5R7qc$A1<7U@CZ7m_Z z23|B~o_KXI`fv~`?ll!^g_&&)uuxH0)qv7{SIvIJ$5kFjm(q;wPV>&PC@C;nZqVz4 z>-9SulgoL18Yxi=bj-Kbr%&iGnxOVf-DsKLmA^&hA@nKht(~x0P$M|MmfF%pHvD*3 zKNjM*DG-SB%ydp$6^=)8dMRqkGlX{Uh%*aAtlx(sq56}~&Wh`)%Oj}@qAN-j@KQ>j z_7Zp-{F$sU+>3yZ(-Kbv%MO&;I^_`DnUK)^sr?iO*_bBM7$s&ZBHClj2c<^H5^za5 z7l4A2@k4)U;Ariq8D8?*D?f@0X%3TUfy$B+<+{u%(Ozlx=R->i}UOE zne|2b$r^KPM2~CS#@Iz=VPZ3$(8j2i?+_l?K zl-W;O6LF@oH`iDeLyrp_T#QVkta^x<5=vf74`J|yr-fk`1chhsppVtsy1Lvg>M}Uc zlDFkD*(4Q<*mj66iVR(f**& zs{x( z2MV#8YzaEl2v$!Q+p`8@Z=VBB!y3B{C|>5P$R^+`u6zIRYWRVR39ZY)_=+GF7sIP#R9e zC2@5=sb9`Z;XBlaRT{z6zxx>JJ>)hHRig;SeK;AH~9;;GUN7C-Kn&v9FWfNIEagK*)x{8R#PQL zaFfnVK*5XVpi z1Vt-`6kV~;71_OJdZET@ed>CkltZy%9acfRw)3XeZ6^clKRm~^DSj}mS#43-cv3(9RBW*;ZcunJD`r5)v0jo=L^UL{zR4(K(HZ6^tI!aS)(IhbE5hY4RcZ^hH6HV`&tG@c zI$FQ1vPG`eY1bJotS!C zrPywhTuoeaICGR_xgI?s)+UIT{iNK>4mFQ8s1k6w8J>4;KgqBkmGk=x=D{|@c?pdB zs3r+;gw!3Vqco0YzTp-uGoO-Qn``^YLUi+)Fn#f4pQIx2fTbI`3*-3>)Z4Rhj@6a(reh=!JRRtxL`Dj+(PY)(97g-aSBTk^3nE|#gfC+raMpH4YN8w;d1WMv zi{4iDvA8o-)OO8I;;IGKFTP|09wFT@E;ef2|rSq~7Fnka$YV2|dteeIrlg(1&XePrWzKB-%loFv@cC za*fwpy0mKbzM|Tt@Vo09eTi&&^{B6&Ndan>4XELc1DW|iOD$|)-|GN*vlJd?I#(=& zR!`onuIDV%0ef_FvZ_6SqtH^&y?iu`A#Q?O#rLpCS%n>ZO{4y5UO_P#aP*{xm6zRQ z?QK-d2q5TH6~+9XL&P{O?32p53XqBGL0VNMOw;~zFvBN<{>dosN4dV8yV=2BfY+fd6SSRoWzq~|2d=7OhO$~ zIjNjdW)-&@Tg_x#x3{k*oAr^EB7%OWl%$F?4^?!ft579-4wamx4+AVZknP@ zhRM`0!y*kcy zg10r$gOPOBbPKtexoLi|UG$Fm*?3mu6Qk+u!d!ifzBOo!zU=gG?N&Q!i9+%3vT=Iz z{~i7Eud-1vbNeg*<^QqC&Qx1ez*5H64_>jP#YV^IM$R$OWrE88p?{#0NvmT0!Qt#V zSfb&Z@2BD?bWDhCY<1oqO5)$}W@6`7gfO!|9BO-gtDntE&F*OTd0Q1FN&BaQI3n@Z z0os}~D2)Te(diRbc=D<|TsHzxN3^Eu`in~1Jqo>GpaFK&DL4LhjaPwz8RaaS≶$ zY3%!0Inu;lm_)Sy+cE_(^iU~I2S&Ss`Nj8_qyudaf zuOM732sUCv0uBo-w@w8$VJUN)UgY&aP@8Q^4{>JP9ue*Ibrmu2Rqv#9zhH{3!U z66vR{x?@wy(IT%0NR4xF=jEx4^{q9xW-cDDOj9X#PKmfj5gyWeK-&V)I{qLwBtlZ? ztrCRz@=x`$cs2V%s`+jj`5U>CMqEdG(I;ov1&)rKXA7=Qd9d`BDja*-DF$j0e4ho8 zY4Li;2o*Mh3hnhGi<%6T8G_{ZQxQNAQ}HR{Ni+&hZ~dux>P$MWZ@bg@ns_utZ|ct$ zQH;j;%-EfdXsL zSa8qrT{Mfep2^ux7O@>Oy~}lrbU#;dLPbpx3&IYjg|QV%$>TUbrXr z@Rk7=+WxJNgmvFQZbrrH#*|rnVw8 zRHnp%kaeIu~9Z&U!ODWX@Btg z-G1qn%4;yamCYB9>cMzsUA``kMSZ<@d4Ap8F9H7>;P45IJQau6HJbYS;JC=NA02*v zzSyAtOsFM87U)2Pfuh`v#}>87lJTx_amnRCKFmn`Lh3eOa-MQLMTrhfkX~#gni04`MeFpAua92U#{OqSoY5~iPz3c zL7MzYl)3bOnA?yOjs9z<9@aq?&Jd_fcx#v#oFog0q>LzmB!11q%>8NmhSpNu(5QJh zaLp!wWf)M-)vp#)Ld8MUhX@gU=eMVh7%%)DuSL@^$Uy~48ye#vo>xLD;qb;{gQsXs zpb*deZD)O&c{<_P;*!tr`x%z)8ZQ$xc6MSxecr^}Pl4Z+=5jtV?Jx%)1=k@KSbB<2 z0(f&YRL;x-eF$)w>7;x@%UUK*dL%QeF9YEiUszcYFpYbC(iK)qE$0en z9T88(l`B?`%#;agwpyFLMY*PoU3~}TjVoF+uoWu#h#N=WBWe%dj?OfZAsU4-g`(?k z8s~57i)zTM;92GKNA==K@#}rvF-!YSK86?B;!F2}BbOcFEBc6w>erhhR5mhGOf?VT~CC2&oXn8KJ}-yVxlsZT77wE7F^{+W?J=0Z2hUk6d;G6_y0rto6ZM{v4>4zxFS6Fo^zA$VnfS(t_Mygf?OC?l ziS^+SY8(2@xriL(HFPq9mR#0LTS700!Bt>M2aV0e*B!PDx7)>12lvqiC-upMacwhm z0*!fYS!x1xV1`9V_UPk4YnUkgI;I5ky~ysY11v5+mBQ6f#{wpmMaVnbG73{VUKMtV z^g?d|J)K^{DV9KEh(l*xI6r5|z$EaTeu85;5>G<3DSwTvl4Y3dqQnNDY1Xi1Y2JZn z6}$4(CKy;lP5B|T$2~k|kjY16q{K@(0^9^~grEiuoHp?;h1#US81`u)#4{4+4t6Tp zS}ZKsE9eSJ*|5QDs!w{>bgeh<+xfGxBh?f?v6VK1c|#MLplsd;5|->a~1= zE`_u95CqW}PeA9_P2!8V0grCRu!MnxG&6>Ie==BdTYR3atyJ{$H+GrUxqg$qh&*5P zA>2&_@#%K$f~;gBh0Mqau+EZ)o!X-}nMdnB_FdXS*>e3_O1zro;?)Ezis$`SuDD8v zfa2~VY$-|MuAqFGAiU*k0JWs5VP{wW%>yvsV*{VoY! zmjLFzgwnH=$r~jVq}o8H0^Z%Qo~m_pr8rZJ{1h$Z*?b9z7k`7o<9@{^pu3_ z()`~CuDy%3bRpq6&fxu-8bwhPH)^&{p&euqS8{@f__2^s2?j&lJGC+g=QW!KdwCIl zVfPM>1D=~>bUCgRiJHOr>eGFP_(TCtHDHfqD!}hFpwD!Hy@a`mPW2W=-gnbeqy~|M z?wMdsu%NdR1V4sa9X74qvtMq4Qf}g^!J0iCH<%Pj^z}wbI{NS2E7(My2qbtI7y_h! z0x?zlkFU30!_XN^9s6b3e35dIkr9^8R*V)Wt-2dg8nQPo2LJ^ck6u_O&Hl3cVcVh8NYLOwOah;DaO7|x!CNQVKC~QTOhK)Ov5n7E z$Q2&!f%CnT7-?)|RCApN(u4}_rjdiBzR0PqoWE!9ZY2J8UnPbFK@Q%w@AJ?7XSE>o0%CGR%7h+&A}wYWABz2==zN4#$~7$t&v zT^d^J9@$AsQ06)i2yh#B!ro!Y+h_~bZtJXa!BdU=V@px^@x+*Y5r!bPdk5$vk(Op{ zPumw|*EaJSS7&v=aOP8?+Kx3(Ad3}co# zVmgUXJhZ3u_-*bvHfK>?xMqt8=@huK&6>FTDT#KGTu01;_8E)$c?__bm^s}VO9-73 zLb>r5OXk`Mj-*P0T!le={6A=ZnuZhWP`hyRu#R$-xGBIVGfBV16?eWsPMgj!duLy8 zDFZ%1_#$h@&2LOxe>?e`s637(K zT_?N)@Sfn}_BPkYjP?qs$0t=6R3ikT14TSf@B{C6c_40TG?7BgAb5K0N4i3=^0E?h z8I9SQ$DVR!ZrQ1S{0@5#^(pER0{8q+8HVfj%;y-}%Um&xzxuPMt zGW6&>gmiPwNyh0RJa!x8iKJ1{XNcNEU#Sz51?U^c8l+}re1lyQKb`L=`?1wL;<9Yb zcC9Nmko&D7)!Q9j!cpS;xbJ8={7)QBjEB7HM2kXdzCi zYvO0AQ*e!L;iTeO>b`TKzQmtjS%h3`gq&LMPT2HZclLSqT)~Ip#>}o`y1kXZi5b#& zGYy!2e@hBxKh5HQ@FBL&1nStnOw99e>D+QAAo;Yq2H3cS()vQ!>ezgl0yRqU z$SdfZ%FUS(CZ^WvtL)oXL>5FJcIEOS-)Hz<``uC#NebT7z`?3OBMj%9$FjVRFETEc zY9Fs>nLFT`Ac}biq{YB|@(2t_;L+6NrvP1fJD%z9F3{iSc=|}4*{r)K=)f47n^?sN zi7l@ROb0b(&|oVrXw;QKWYssb$H%V>T+;y{)R!(S zROWe&0e1P+mf|isXz7D-s%eKT6b~xar~irAc$z%rM?Rz}YKKLylsjCr>yBpFXay!N9H zCWN+q9re{>=o(uR2ZakotcjLj2~(tC-oWHd+S2?do^K(A4z@BZo}?8D%_~wFb59$I zBY<5qN(i`oSE~mH!rrwNm5Z7x?J~)H`z#00Jgb3dy0nE^PbEI1zlk(Tz7;J6v&1gN z@>d?X6I%m-vdu-9uP7)^I$NRuH2~wD7~>05^HpWDS#?OIdTBXl=Q z$)(^RsTa$KI}JFyL!^SSVQ}&!heoS2;Z-ZM))$=d%@9(*HeuHI7hdm<3;I-EGxLDf zkK)nCcjqI@Rmd54?$uS!o3V)lR+NNH?Hx#m4pu5ZJx2XJO?=0=n5DYa7^oLfKkXwv z@qzyJAY_OcpzX+;Axs{72;)>&h#_~4aM$G?@68RqLOH_A-BH7Nxb0Imif zK|AsyYW$?7Ul4`>1=HmD`0c&gYz83rupge5Ty<^dSa#kIHz$~SQvTs(8i_25JxyZ4 zln7z)sbnt#9NvM^>S8r0-1Ct1!+>-fsQkTyTYnVrl=>?{<+Fb?(_=Qx!Smm~JMYs! zuDSm1777vmfzU!#%nW4nUqY&Xh0e)i_6GVR(5)lGinGHKt)gLmBoP`938z6G8esv8 z4*lsEl6=DqjS}fA&V!JL>R;9rwURtr+2Q5HLef(g8||~bj`tRq4gGxY0lx%}^T#27 z;b|A}6+C_CFX@(0nzj~)%zH2^p#eW!zGHG1h3Dd&s~co4smN@=IvF}rC%&` zb5EBts#Cg{C^++7Ts~UN-OPDfrA{mHY?trwEF~GkLXS&Z_Y=FxjTw2{NZ|%K5@>DU zWi>}FT_}DA)j^i|T7K5*!wdA8}L`j;+}Fqd1B+!$gWE6>qyiW9J*(i#j>PR5?MPzWhGJ_^Ywx z1^@U5la%yfR=p+ZVD2lL*pM+y644e?y>=yieX{kM3Tdb2VeLBa>Fro{5+>yR51_u~ z&KOKez6ci`9$WnMq7gSTAs3s59XaB2>e!t`SGH@vsim|^l^j7cl2OjgA6asfuofv+ zP7N)QdsZ7iIdp9}_a}Z7IyC#s>-RY{b2dGG{|c|VRM&6ro}xM9?)=4XiE#0kBt)%E zmAH1^Y%RfCK|ju}2?k$6CnoW9SGr2`SSzvEUPb)d{^DaxOu(f?ZC2ctQOxlkm#4eb zM-SD>)qG__1#*Q?a2EYSvRTw2v2Ud;&?0?dhN64|uTeAY`*EPW_n&^g$6is$WAf}9 zoutv#{es+tKy!e}gjHTwX{1M~P*%sf**}xlY$x+RJY;i#+kYSgB2A7T-`_62V~s-^ z-$=hhnOOHGE+lT|V`0Cfn-07Rm4h@FZyd2cGz*WbREk!?!xNtxd+sA+m4Ps(CTj&I z1#5tISW=tUG#xLuW_GJYyUzK+W~;K|a~L|-CAaBoYn~rfEDex60Q0zHFAEeGZ{zIM zS?tdnJdeABbP8O}P0xgk$zAGOW0|a6mjkvNQKi-g$CyLn>E z2mb_!|H);F?P*6vd6)fv*Kg$*6lD7Q-awE14}{Nu;dlR4y5Ij}18J5+6wR$O0#_uh zAOKkiN>$<}$X*vf>bN*Gx*J5D5LBtz9nB|#KJ-w(_c`mVRJS=3_PO@ERCH;c@=<_L zP>7;1x6550)-tcIJn(y+@2qw_zxhSS1=+|L(H6J zrx{wi>w^k1OGWJ=wya|85Ht4+faW^=Jann%lk!c!fS_{78lERkx7ww!8PA!RU#T_c z7)w2Wx}y6}XZ$Gs$Hla@(+t}!t3p~4i-Ob=U9(jN!Jj4jJ&phxJT*r+Q2_3WR`Yz(Sc3y4s7@ee&_h*vOz2(7e|sh=8SF zWj*yXSRqp$bN8j6l>4#n)*m#zeOl0Jn+lB-#0l^ToQn_xM4FwwgJB@+SO;AogC3M6 zQv_WVA3Bnc=2mJeG`cOC&cz3*4R@w37W)E0POZ)9%fW{f&ip4c{&Yy;LT?@kEEZ zP5TgicpUrAH$h`&_%O88x>(Z?*PUGnnp)P>{WK|je@6jud9K<_<-)~1s4X&gKh88)C`jb)a$e4TTU?8jui-MQ(Xdo%?*U|)-+2nV! z>F)u^6gR&>o<&;N7 z<#9oyUS$xwK)Wzr@qqReQ95P<$qowu z^nsK!7)f-YNlRm+C&keK$tYok!qnr`S1-uSLT}t-5VUGQLspUXt z$x~@&08R&pc}oGZHw->sIJ90IecTb4dHmGC7hTW41DA1ddJ=C(lnMi%>}Q4S#EcH> zgX;rtwx7Fea$-u>92|PIQE82Xs;rbpFh=oRWLqeX^KS1}2AURVX?u;k6P+S_ZjS6+jC&mo&amybd5 zNalvJ?Z>zmS~B+?|NW)%Z%kch2sub6A~e_z&j(4 zR@&4CQvA}Yfx~+x6DTz(hK9{-4UT!6q;JNI>WyR=Qz{Td2bc0T%;K5+gRnB9VBXcB z7FYFo&En*w&;6yD`O5RfKIdoqQ)d#|d`?prQ`T-Hci+WxzTel4BktMqC`P^DxBgq6 zjCT44zEfvlN`YkKVg@}74ZOlgD_icywAngQHs;hwfoot9@EZt^59RV+u&x&bCUrr3&>w%5%L3-{nxr2pKOQv+GO~# zn^_|;GA0}e3|S7bfH$@{n>p3>c+*;nQ$=S5Z$yJb$)N5XhyfCH2ag!6vV_~jd#q(3 zR118zN)<&s{!0ziU0!0tH33l#vwjEHo8eF8BT5%9?B%O~b!(C+jV8sS9qte6T;PtX&kk7DmDw5kN}hg+J^!elxP=eCsH)x?jv(~0h|QfSBC7HN{k*O< zMd2~xNEePDHe0|3U*DhSOQ;v3O?dTgSL~`QEX0eu`Cd~M1OG?|C9t)bs5`gI`upmQI zxRrFC#}d~x)Ws#bhtw7>+0>c6XC8$}J}+6^z)>gvZSB_|3N9$|0H>xfGaS-j>n4Sn zIJd?arxS@9O-g<4TeSQKR>bGGK=VEOK>(LST&_@~KRZCtAtewW!Y##UO2y7zJ7)a! ze6kv+ClO1ixz+ZCc5nyqnmuIDJhdn%7m_~MCAXV~s9zghnL5r1YH^JVj`fUggY3ES zGz7gcQa)Y&Z6XS)lEx~8gMrETCxI`jYL-s+ZvXLW|9iu206~2XkHhRPjBY7wjE;_C z8v@Vr@2vL1tvM35;i=}F=YM6j`zAPMvu_kL_MA@~*P5Ty&&+#`J>K435q{Rg>w*YTQMb>;gv?ukY_rwAH=phL zB(R zQ24${?6$3ZJkky~eP`+02OVWnO=B&_gHN(r!AwPndz{n`gpbuR$@oaYEffWg!3IaL z_m#?$n?v%-aqHN2lYjEsrO@FbaH+zxe;|ct=lb-WzKv)nclo^Uk1+_F6x)l`TNMW8 z!|9?M{>7B@ReKntC}KNSH7(cV%ecT+u;-hnp>nk~L5bB~Oz`VU`hA-ebY=ECPn9GF zJIvI&IBgp2j$clH1t#vwRErr-)sUqkVS)Vc8FGq4*tNPb%?Pm-i|Oucczr)bSK*l1 zBv#dHwc&&wOx%f_kyG(czgK>mG2Hhouk73ROl~15JJ8?1QLlSO{_mGPU%t{s>Gx?S z@lW>KfA!7APR9T1!cJ7z{tF*MMWKzA48lNf6P}`gpF#d+w0T|{c8k6_kZg<({Z&Kg z#+{&XNB&5wvh`$iw;Aw<@2Lfi%{i@^oZQ=ZXz}9f@b!B@{l&LkE0qKNI3b1vhRFIV z)jqYHigdoRn67l*YbFEOju~QH%Wojt2tRw9mS}~+&eOub7GH6V9k-EzufbK6Rj2wEJL6w&5$zJty% z^BS6YA3f^2vzASz4Cm%vfsehx@rX`Nx=O`7M2erge1S zv_4d27f88Yt_7u`l(3*_G=5YQ>w8G-i?jJ|S*T^DtIdPcoyIjsx~cv&){s$LTb*F3 z_&JY*YcWmacPz{IHPalFxuV;xt6EF05dURPDs#t7dt#=sDHC-w+CNb*#0gWe#Um2hT9h*-ODkBBjY!S@Y)grEP}tIYR5 z_3>n$6CaR8OQP|XC|co>E{SsxeyE?fp(4lEMc9oWbcC`_BsH$*hB$td#j2uBk7L>h zli)6`cD0nwm9>&yla1X{*9-qi`(Z1U&imudgZX`(#>&sVVa5O+V)Bni84ovn$%}l7 zn9&+Lo!>`mZ8O0~v58KV7@Xov>>4z_Jte3%`%%R(+9NN(r!m0^;Za+BuCl~jx|Iok z_9l63ESfz}>h;yaw2oy;eeGARk}Z4Y=Z%g|vcMNaRbo(&BAzm79D!IeSMiS@SOm#M z{GYYtNj0!Srs2A+;HphX+0#f5=KU=-WS#x~7BJY#atomRU7VZ<|EMYczf~uHi4_Yk zMTX!+UW>OmG%^IKE5nD0f5DTsl7{yMIJ^e)(^X%Hl`&trzOfI=% z;vJAx-fc8X@DKHV`|xngaJBtaQ*L9>zha%m$&MaLxG@|_Hw>6|nwmMz!u^}wHKhXQSDx|fv{|kEmiW}5a z@OS8abB2BRAL#vO)B<&DFwm6?f7+uk^mNflOM)}PI0N_<9Y3p1!g{cPmIeB70r+_* z{)iTPC7AfwRc7_R)wyf?Zod=t%Jf)kb1(6*+;$S@e8mm&Jub(Cgg<&5fLJ|bt9Xd0 zz~3{o_3M^+GUizq2NFegfr^LzeDEK2kWVWYIac))y$KapilVqR=m-IIW;ab(va-t#%`)67{3Y4QahjK!|M zMes#3;hblDKCkm9^!p2S51VNE>JR&V{-c?;cC_E$plrnvsZ@WUY(L=tkqOty%-H(B z1KTI_9W(Rmuo04-_JWdBR2a}Pi$lqTF;w!41)4U4+HEo@=hqivkJQ#{?*o7Nqy5T5 zuO-}gHxbf{c2HZ(zB8OL{rq_K^!79E9D~UlUgSGAb7fKiq`WLFR8w?W3=ba*Qng=0 zJfFT)(Pz%ukzsl;1OzG-mwlu}>Dtz)IVNXUYg!QSBiY`q^KVH+CI23E`mTZ^zgkN$(wh0AU6X77a zm}>LJT~f=k68oA zZwS^eH}Ndi0AO3@VNZ~T!7h>_@8hhag^Yn6?KO}713Q&)@uTfRa_9RE1NkD=4-WM* z6`dIxVTcW7p8ir03@nlvc^9{~@ZJQ2OmM+Reop&{)dTttLh1I3uY{=NitVbNDHFS2 zvaP`7CRv8b&rYOI?e6-&^y6=MpLG|9roZK=LfEx7>n$hBGMWO)A~YnxT3wu?pfx@_ zmQV?trA?bI#-;6!!lQz^>-eL6ZY|ynf-UxSi%0nEZ1M(n6GFEtn-JTj9>LM}u_lw! z%lUvqn;=kG;TmUDg~j^&$Ui_fG33JCEneY4cv`G{*541X+-a^bVaMMZ2uc-X*W2Ik zlF!?JLS&Qv%a;f;vp4^n(mW}NX-flE92)~?QALZHM#?&p7B8LSDtC_BuwiH%Rr}My z@!zE8g;NIbBrRU%+Zh)=JNy8J{P}^|%G;C+nGSG)2&><$>p-80DY~KC7E%8!<4L1;-(W zG?gYoVb+d%eQ?ly(l;ZJHOHs=zCU@@rGF9Fup}q;%lEaBb$v zHg^8i68r$F>!zb{pUx84@_rVR*PHlBqf9ZFr%{UF($S~PO5tG&)tzL2<1`g)&mvpC z)cwSe1h6YwtZ(P>gpoT{^9%pDa2~$-3r^ZCyotR5fVxpPmS$dHE8-28XgS$_!&;-B z%q>;Xr$$fhO=h0zA}xQQBv>xeFBe!Yn57dGi6|18pPLd6d;kzb#rW>;AK!%%no z!%)W=_rjyVxBGd)RUt$wMsEZZ%)UcPGI)aQG6RS~41%~C*9D+qZ{Q55u>ouR%!_fYMsNWG>eh&4PeObG-i14v_B{8*~ z1Q(jk15XOLap+Jym2-1ww`b9(+oTLQ=$5VttH@_xWySr%z@3k28RqW)%-WzH*Mrsf zDV0@#lr^PhMmi1{2cM#@d~TEH(;)4oh+WONFx1dV2-`$8WH8{}8EJQ!jBR-Madu%> zhfx1O_|Fr~Q$btt?|}nx?1TNiziTM_ALUFcxZ0XIfkFR`Oe@4qN;1iHQf-4ls@tle z%lSx%DaIjBAjfbmZkY7RCb+!S)lK&F!(C2j_fWvu{ChJ2cKrqs-Jsy*KNR(g|AnS$ zRO3q`7^B3(f87O1Eh@-25-vG4uc_^OLjSG*qnmAXizY;^2()Fr+zO&>V}vxj|@#;{L*Ho zg#>&qCbp>2Q6x6*8c`U)z}G6CTq1PkjUSsTNP`?h5c16hH;J~EoRSg47$#9YwdSo* z?k}pdPlbvQruI(e?KF@+9O z{jdLrvv-QltlhSSwIWQf3a0t;9^1ahxIH)er|OtFl)_iI8BB~7w<>W*jIiW* zp|rYm4%4f+kk=l2cd4ao2vc=H_~qd~X5Hn2hNS?rN06x-n^VxdLd8haMC{xf|7%^4 zQOEH}JrLV{6%9^5O2)~R!E*S+eKpH^eT>tA- zLu+$@^uLz_69vZZ>ff=e|55&iu#=I=Kbv9y4k1?h&exjTWGmDWL=xl!-BMgfk|zlR zLqsK|BGnr8B7EOJ(lbM+;E(`O=|7t!b=ZzWo z=8px6AQXayf?jH>`Vaec>ZIrjYTL+FYoH#rlZnY&(fOQ|sx5FXqb{^_hADAv+D&>= z9k%4}4jTRl0?~n=)~BH(OaN1YOz|!Bl-u{+pO~9CBDsS*OQD0#=sX&&bTHX!f9G#> z)yXz2qYDwf4z`sTfj9!oWyEY9xXlAuMWrV4ImhE;iI;O<&y3*rZ`_9c$SSEu%lW9}l3&>_~KXLuOXA4&1^yd=`E5 zaN4D3HNPaMF~#gV_Z0yd5F2Kv^}KxWiC?Qz-)SA6hTH9=DHFCPP3EY4pIqW4O+WVC zur%<9rhC-$0P8s^S}gD%8;z~Yi#^r4)ib|7bE*=wD5vTy76s}lrCZKc@c-iWC0&)( zbSt%ONfyn#BKBgkcF*csnqpy#bp_tZ9V)v#!Vzb7Q||gHBM=`={2jMGKPP#Ds@mRz z&+s8!;STPz@)o_H&mf|T#TN|vm7RCKe?sn93@Hx=s_1-`#ZjP32jsv(8^r@snIg<& z%QMyTcNt1R&8Bw=mgCgHq=wC8nQv)r6p82gLbPJTg&eR6WqpM&MH&qyxinOftC4rn z^P6ols|;lIZ8Bvwpv1-xQ5+MhNE*xU-Qiz|k6zIQRgrh{GJRA6qd34aSxD0cyTD?; zhrOY73qHpEJ(tAROP}B%g{(udGiZL>p`n}FV6kWb(2-Ug!H(ufZx9hwD0i5hQJT+G zy{AC$4vp9@Dri;oEUHTWmy(P&5r+8VUpl!J=06GCkT-KzFt+~xf?NMCA8&DBr%_C%(gM?;}Vz)-Hk zqWInZq*sa$Guz(i8@pPFF?bf|zlA|u3*lFM<;miz73x{DGiT)KJ!iJhHYmEP7cIQU zfs)o&6y6ky$utbOZC_fUeaD>;Z#%Ccd@i{5(%E769Ns#BKXdZ8r{NOu=){{SKqq&E zYNXI_0AMh(%DV5QfX$NTEkPS|%|AW`;olYmTKO1enRVxQdCGI@FS*GwJ>gCOIYv_m zxL17F66iq;tdHMH0id;MZI*i|7}zPX1>2jhI-Bhq`+MqRo^nm1Xd&?BZwp>4@jl06 zw~s9h0;a0XXeIIWhNP1KX;EGmwZL>)#$LHA$~7*!%r@Kpew(Go{_jwUL>kZF*4n%# z{tOCTMt=)^tR5!?Ga!}d#Cu05ZreCh!5VWF{vnh62IbQIIEOf=US#c-wH}S=`+!CtI{xltLY^;VeQfaK%=jC&aDIyPb!f zU>vq2fpW}-+#=QZz{b09=dVHpiYG3VSrRY$B?BkFXo4L9K7$Ht+_lx4F$42M#>rus`u&EyCt42ptm@w5~uy4*1l7YzSZ=HWtgj<%-A;>xc7#` zK*$b&$$fkjF}(nhY4On&9FQZ~J+vonN$+r+A?}1{=nO`4ENA)yDn-u^;t%r(fiW+S z4VCvRvcAvH_FQVJm{wKcV82>Jv{k0?i==m`i1W@Wls&sHa4H-0-^$EZ$#o@$^i+zD zW>T6^E8{ILOI$$~mU*pXPXz3z8Urq;wNcpPpFy{|ssSM^6}0v-!y*~Zp0N8s26;pt*AEq`4*ThPNq0wr$!VA zd#%_+##bmm?oQ~o?JL?7yib@|`-}RsFeiZ?`|>~yIC00Hg4HvM&7yDN5w#YQ&Ml=xIGhZY>4$kc!i~n;jg&xUPWi1Yf=GYyP(T3%{)es6)yVx`s+7 zclbG?i}b0G^l&@`%CyQO7G{VLO?A)ZOPm6h`tyEJIt z!gg#g9y)QMpM`+TziTx&*ca&F%Z)B|oSUb6pfJ%k$@Gto!3)vPpoV&%W13G*)MK+) zIuEh2wJc7Fy6F_bzE9rWaN<@f@wpX0H^Ou7jb&q9sUKpL@rR7_)9GSpTs7|P>um1U zeuL#UcSgkYbd~jrMs){-S2ru+r~}sfG~0qI;R~u^YaH44xb1IhjJ}y@zU{eP;Z}jg zApC*3G4b7m~F9li(*6WiuC7C;R_9_(@DtwQCph zdp=z#e8X0cz7J7-u z^;xI7U`7ybA2~cW7Q2yAh*6J@Vdc2UUKgTR9Xhm9Vt&+nfYGj?lmwo1=$}B7hT;hV zf!&zNZr~qd3xy@p3hr3iHfzJBp&z;yRQKATBI3@59t-#%vX2&j3F-C8Brk_L>2&>7 z6YL5}1>2S<_n%NZj^iu^C>=itB#TE!T$YlE?PMIfO~)qnezCrY!{R`17@oSDWcJ*j+vW94${y0(h(GXvjBG830;i>Wzh}`}6dd9!gx?!G#8~izEs@AGsWc9}X-1 zc*-Izl&Q^mcOi!%QST}4=Xpfwm21JItP)t;gmyBE>`28h`kF=~4Eir$W|h(1qD=yjRG1S){Ew3B1^S=U1WlVA zwMyA}5FM>c7}z|^ccI8tRGpmNZgHVh6!BEJh4e}J3%E605^iHm1{lNM(pJKq(}V)M zETjmQmCmQ+XO73zmxhkVTjB!zfA-T4zf(3x_&EK`c|k$B2h4u3x>n+08Xlhc{?w4Rf0(~%tF_*`Z=KJh zgw`V$HcFN*e2;90cSmn+r@~XZ&+GfwH;`VUS`+9&9Al_`EF;Md{biSn&D-{)WE=A> zl$dGX9hXg54u`5QttUy;nn9)nQRb7(Mi<^Qya|E-@^=EpHu-#`4HWL$&A)vL?azkg6M@XI_Ta~%gr}SljtHnw zF?*bYrf+sL&2|w5RX^`62iF~G_e1NX*;5_iErb^^mI%x?;`-3!EuZ`Q*!)b>es45C zbTz9~Xu`9aO*-N$wOmefxa!xrJ%djcTJ-UyOK~LR!e=GHn>_F-U?1CX4g*>kZ5*fF zCK<^`tiUY*Da1+%DPY~EoU4C^9wAF591-DgW^g5qS_O7A0j2N||dMMbqnRri$V>J&6_7QSA zVAFFGx!1xS3F%7h2Lv_|rxoQ!ow|@F@CvtZs*gXiT6Lmv3HMf(ryo5TmM8035;}Td zN^Bdk{J`HRohd3gkk0lEYwtxr5%ZEq#sTCi66|vlAWOuru{(xWty8f3y7)sT-Q)UE z`cdnR5YqfnOMa?LFdbngAJ;u-E4X>;no3EP(x2{F2ubN*uER-W3nx(c2p;cE8XT@O z-3^x>x8x3?{vp(0+PQ*mN&$P<#5m@;DkwHMzDF1yIy@r#Oi<#3uE%p8x zL&XK>leqkF;v<#Oq?_kM?SOn}<|#SMHA)v`gvbAYm62$d?W*F)O)Uf7mDdwh_L4`g(Ype7SBv$@0GT z+;Vm>c)cF41_3lUb>X9h#E+}Wh&7yTT9S3s{8G(#ZBm2SARRxp48yO2KDa9-9dBtd zEhsFY$gr1VPZdO7yV^BWgeQkCoLyG8Mt7v2S4Nhpl4XdMZn(*Kq!O0OG*VQ%o<;6F z(3X%mO7~2o{`7-TzZ=F54rGEh66G$R^(erPSFPg9mJMAf*DQd{uiB&%A4RE!l*Wb! zMbEc%qeGUN72@pS&1E8FHB*8~>R04}KX@NRmUbo+GNOf$c}Ws;!wP58(yZglIQ?l| zE@>myi6Jvoh>wrnd`C*wL?1X0h{7LX;!OE%4j>dETGJ^w3s}LR2RZ_>@JCqFKv1O) zNW%@XB8m&CIXr2q$rs4_w2ojsaeV{pTJ3Wtuk7I4th=DCi6>>DU}K!1R19eb>fXQ4 z;g9`T=q)+wbc*O@(ji~<=MF|S6z`GV18OT<4aI9-0Z0aJdS2CCNA{=ZRbf>76hHrf zLs1SQjhAclq!(lFg@DOVBSBM_EPx6WRLf0y6odIq?W$g*NN|^{+~H4|18_L}!#94z zhKA6~7&2tpf5Qx>0c0AJpoTrFHu2SnjTsJ3Y+VO2b76PEANTu&1e)scrMciLFt$bd zMO~WI1>kB`jM?eH`<2y;1PFb%Mus|qV2~aeQ&!&s6}js=3*ctyyOh*Dl!B2$Vl4|3 zgN|Bqacw#)51Wf4gn6;Xl>Sq=j9p($4v%9<-M(`51nFZGaw~DtIH>TQ0-qkHyIhrJ zIF&O~s1SNuJC$+wDPgb6)wQ47s>d;p*>e96c#s-g*LB?xhjU{>BJhx1Jy~|7P&+8K z3_~AJ2U(sjiQ;3Kqi44rK1C#p#-{J$Aq2-p&lSxki7EPC2mgL73wyB^kB5wE5mL;8 z8>99{$!u=n6}W+2wTp{$N}69&w^;%1F>bZY4K}}QCBHImNY3GFl~VN!9o8c?1ivl= zjFNbK#st7mIVp9(=GOMz=rZ0%QGsT7!G`l~7=FdcOw)Jm6@<5y_acdBC@cvkOp&G#sxDKTFoEGV-8?&^tH7{HuwrMGbJo;f+t_F4J0$= zUJ-X9e$8Z3rb`vQc0F7APj>Gn7dBbaZ|_lvDUjF2IRH+pjQYV@Wb?=MDnBwcI!Sh$ zYg3)hl$amGg%s>HV{-8J#}V+TPVE#j!JxIH2e96QJvg=ae(_3@ZOrI);WZ`SDWHsU3|QJC8QI!a&?zJ!aklz1v~zRSpL@Y*mn_dD zXjrD36dDDD=h3t0MMZ6o_@SOM)yo`1x0dzq`hvBH@w09?;0Z875(H6z zcsJ0)%MF?Z=_;e9g+ja69EuO&;H+!S2W z@pJNw0U@g8D=5=wai|~Thj%k>Jhu4(tSL0zIFUs@8U5-t^68^Iwe>Vu@G3)X)@IVb z_S{{59$`}G@z>tYvOITl9+i7I)fS;>XXRdXQF`X|!}-Lle(BPxfNdD^{ASk|Eu(w# z&1E1bY;>6*sd=A-|Ah9RQ0e)Fe)C+O6h)(`;<{WO(F*MQ$3G=04gBckMWAcQYzj_ujN4yYYJ2K3s*2B=fNBuX|p8I z7jO6NxpkcF)3jh1)WA$Nd~uHIRsbF`L+$okInA-C-=G))OS-u}Z+W~`&t#f8HX&|E z0uSrP5}ap3w(1L>rsz2N5IbCmQcbWCb1DxB?J89f1$=fWZ}tJc^?g&kE9W3G@hikiOT}S*l8D0bZUOmQJ4yfY*64k?>y9X&=ZI+w$=U&WQ|6sxfTVFYVqxIS> z9slVHa0?66Fvl(J`JmrQzOnA0-TMKZ@5h+%)q6rx)2D!4bFx~V@24S`JZg68?~!Ue zitxc({@mPBVqQ(NsGv+4wyHuI=7O5G^tCL{ZirY^*}vPS1;2Vu0p$3TV(XEk?xkTa zasEDfztQT%+{)4w?GUp+{ySSi{i>1g+ct3lCxIPtZW`M&`Cw-o6e(QYIX7qAH56() zB5NBWs|&vQsg5%xDl$D^tP^XxwhCC`Edj7sxPxRZm0YE(`O2gKY?kS+81(nS?~q@2 zWeg1vgni$xp1%+7><&VYKJIib&M(S)qC}qZ@eU;2kpe^7w`%DE^?gG23fIIg{@6%R z$+b2?2R-`cu)Rii7M3YmjgEe&II%o0nA~6T8C4e#zIrbpkfG?RB%OEr?Dt(B1h76W zLv(=uFm0H{hi@x)xqrYZ^$*%h+@B_K#$MTBxuFIsICoVXRG$0##d0`*5e`uMjox4@ zVNIhoVGy5hx%9gCjc46(0cV{DOr}C=erY@OCEA>i?h#_yGZHDG+zm{Z>l;5el-WU+ zmvK552r&B`jTLwxAFI|fZ}yjqqZgq(q$GCt-^e-sL?c_Ir7eA99n6*}S&YxRwgdWe zv1EUG90sUxo$X-vkWN?;azGrhLVS`sM`OA>ec(*|o_S^%?A~t|O~p)*`s5cz)*Eha zBlXuW+ zYQVI~czp>~5$8aiTz*+CI`%lqJrsD0K3LnnWZcV$mU4$kqS<8b*FNfiMX$x9EtQ?_C7cEv`k8)3Zy3Yf zN~oj*#Jyvd52ZoTFi#=I=NGaxrbZcsh&d=O$V07gnvdrq;he;3TM8S7q^QQ$`*+Ny z?8{?$b`?;(%UJS)z?qg3OPZ>AqZz`TL}dCNNtxkiO%l8ECXbtD3ej~s>l2X*TkB6p z31*WN!6apAErb1am6PuubG*ddJ)CGZQkG{kX!Y#J=)CfO_;MV%)}K7ai>r%uOqxig zyzl+>Rqvcd1#}v|L>G|IEA45E!-sA$X3M<%&63Cvi@*rA^b4D1B9lWBvdbl1^kvg( zW1(2vG^N=LNQ5+dw5}6KJ}v{&U4|J*4Z@6yj8!5)(0QWKhJ_A{2iqXT@4kM`JxP8d zeFs%YJb--F#1=G>iq(Np&}&X}NV=CpVF+srpgr^`F5)p5Xh!jG#xtPZn?~6t-}^w} zr!#5|(SgM4&y5&*cBOVG&Dx>gJOZu z%p~2Q!O)WP>Iv6|z>&dAW2GqmiZ}QH5hb}sW0V;}4Wl9D5g3vx=}{bV;rjNcN3L(< z@R@;YyGkS!`&}UJM&aD;kg_&lR?>4;rcG}kmE1>c03W4?I{zMAzzo7iVxU|3I?U<4 z9PfAPLG96V9RG2%s$eCSjv@Y6FX9>ZPw~m}^IIHtQ#ehW61nVcxE2DeI&3I`0jcEv zdzs9_onUF|kC?^qWMV})q8?R(sr||Fr1-h~dp>xyY-wzY@~jC!1}?EITY`C5AAkj0 zBQL0ptJm(_9KcTl?G#CX63LP+h+P~-aT?G8-&lxoX-K}$?J9~Cbq=gQ|EZt0gs|8m zC^l%Gwhn3Hy-W6h_IehmwWYWYnOw{&P3vVS&*;&S-vNhFVWF7=obUdA4jevD5Bi{M zmhE%yAX>w^_fe}skZcQVTt3J7;wmy&Z04C(3gsEGgZ%^(LES8On;;d2Oylmrw zHhtcfTYQ% z^Sp6+qx8VsT&l|Qgu;qxrxGcQASN88_s+dU>1w247ifo2%%-*lK3m48ccY5LYq1XR ze+o=p(ilQCIDqRSR%2XT-#!o0J2#ka>viuJSn4+4eB0zE%y1-3ggP%Z-?Y%HGx%7D zdh3Autn{-P!M_$B4rfP2v6AV`(li!lSr`cqo*pAEib0*68Z#=2MU}{iFO(vtB(M>W zA&t)B?2pPKQ>Eb`Ev=ZA=k1TGRNQgfmoncRKPRC49E*CZ_bNxV$aHjG+Q>d-IoLc0 zrJP|ss91K$YMYCybSxMSZ#Fl@*C^SJ(wR!uvL5*epU9XSK)HpfDeyKbat# zYIDn%4Z~h9pK;x{X1(T(>70w|)ZE$~l9_Isq~7+Hk#W)*648rw=bsAU@lC;Sv@MUJ zc0J8YqXg4mj=s1T1?)F>dt{?JoDR6E;)qe~M*AR0B(M%KutPorXT@YVF(f+l>c{wxXa~DtF)4_`Y+)2T3=7E^U ziw56)>aZ{dxD>&wzXzy=w1ds$#Z)1(3qcc<{&C}ZaP=kv3(HUEE~0W3p+5~n*F?hi zz;H2!tLi7qicRegZ9`@l__k-f31xGO&<63;U;K=qD_**HG(pxT%!OW~8U#pQfY+e4 z*~KvfuTr@NK1`Ue<#oZYQg`lo9f7=08HeH074Ds+z_!ir2WHaA?3&Bqv1_jkFw$`h z(5JlZs!yr_GR?ki8hH|4H{pfItpoyMn+ZCqECx+hFiz`h0w}9m2O3v8_qZ-ZZi1S> z>64puoBKk!X}S;Ku5{m&JP=(r5CpYW?F`6Y7TS@i*;EET z*~;DY*rEFv>L9(@X6}k?9NffTi9N5i!132#1uEF>^jV0hKX;YEKVn7)f{Ri1r-~)t zB%gvE3nEwZ=x_nGk&2crs7=Zq6eYuMC+e5dwItypSyG-%i^2+`RANXw1`qiHQKMm8 zQ99tQWfid%(uDGS?D$MauSEp$H`(_yiM-;%O)UaY&Uw9VLCsjA-v4Eb^@{?)dIkyt zG7Am@!uxNwU{p+;tN)qD?(_9)nJdlC%uj6;kC|iYx79nYXpjh4h0hS&F+cp2YhdI10%Zz zbxhCp^n2UB^x*$^zd`O3v0XLHGtIy36Ad*wO6%R%A2QqJBgBIfFe|$RvdBJRNngqD zh@-cJW(1RS!p9n;d)#V{i2{z0x|xy8$P$JwUTE-2-;fm&z+YIly(kupeC|ad6XWB^ ztY47@_{=n~KA8YEK*_(S)_8v9U*`p)0~-!W`{&*xmt@qag7Kj`F`*JlQN==53MOhg zC6cb(gu2U?&Jt(+d_-BI?WuCwM-LCB2X=mOA^~6DMeC%#HuHSh=9au6e??j?FGUJ^hBKksgfy3N&&$YI z4EiD+Z}S8nZ3_(EZA5tl8l##;5ryC54>4G)TwJxT!vjEcXup30V z8`WzR-OyJ{5Ij_^V$OPkA1FvIH>9EbW~-c#ovo82btx~Q*r^yI=olm3IVFbjK3zA5 z=(%m~t0Okrz~BCdVXMSn5W=3&%(Vss0kMPl_lB*aX69u7u-}n#F>^9j@0!$TZ{t>jomNp&(9hmx>rN`#pi+|f-C7DrL@boO{U-RUeeR7Zdzeoi z{VtC=(It;vq?@2`jGwF{9Fq}L(J+})GuN3<{J-Dyx35=o4IV+JcI`2slrdy+t?rp+ zAuEoc@QFqG)f%td+9-XIf!E;mS{WJAv?$wY=d!I$9~}#tWYN1DI!o> za9p?_b=Y@0KAD~Tsr2sc_2e1IB0E)--Ig)LYBg2Xpry9(B(rgkhrl%>LkMcaHR1UY zXmM=dr=T?p3Vl8no%9C!WF~y!k#@1)RAC0`U0O6NmIYJsTK*L2k$w!u3^SUsH^-gN zlY2N>S(kogmGZN29WfTvMGl(GRw_b%&K*!8MVQah$v>G^p_fve)!LhQ(B)%jx=NWt zqbuc5NdtMz`VW8_Lq1#E1WPmA6o5%&Yluu85gx2)$*r zSl(=65oO9|Li8_1_79?aQyV4GE5pG7dZ?!m@?QYQ%(`u($WfO_?=ezxH#zBi3?Bj@ zNfXmF;8EZV3+PQYM4k}@PnG0$$_#$z$#;2gzHmP3$=xof@MIb5n-#9dbb?Ap*&G@8 zWXb}FzxK*Ayd<^pA*C}!RVQUUfW6W%5wN4(!`b?A$E?+)(Ekzus%`uG2i+fpsU@9; z5@YM!N2VS^VhoghS8Xe-m$!SdQ)c-*XVWEnF$9+!WY1|Id0|QclhdvIHSo(=(zSh1 zb^(PdT$oA7$k~M%iL6f!k1t68_q-{EYl>31`#0432MP*ibQ_M-7sL*zv;SV=jGBvd&GMpmYT z#Y)0ZUCz#SXxm;w#m$TAS2C7k0)PFYl7196ECvn4#4Om=*(Mc2_m=!oR*g`g+7Hie z9Thirom`6{-&r}N<&)SxPzwK&0lFy+Dr>-E%?=q;>G69GezPwxt26qs5r6%pypUQz5EM~s=ZUUSpTsSvPb#vL*%5x6Ld2lnkybr%>)C$tt3nPT$5M z>3xReHs!n_XK`w$zFVv=X(PyB8)#d5n?y?@Ks|(9S<^QtT>*IA(mG%}#{PacJ&N5B zeX=H_V#+kR&#e3>=2qF*aLoc5;p2jwr!eusTm@K>7{1a%(U~8?xyzkQrALJ zBr^_{Rf+AgE6OnJF8Tb-e!C@{NK`lQ;va01n^cM!PEa2cmvua+tgjfMc2N}1HQ4w> zP32NSZrbT5hzKY3gRn5>5PzUdAlTH@2+^MeMvyMj5#M7S2T2iXB_4rWu z^$ztz_!Px+;PEc^W&qX~cG z{6RKoO{lmwdg`|gu6rTEX1A<*IeNENw-!r#Y4Hd`%==j?>HhmTMS&J?HzoLR$g)eq zidVaMb}{DQO6Lq|!`z#<1`yXS(v`z^g#ub@Q8^#IHbwx6eNc zvn9B$pv+#qv8u+b{X zJF1i7x~Gcbnz1Fh2PaKR|u$m~qElo{^iN zj{N1U&6M%Udxr^In$9~2w<5pZTjF2A`DVs#K5L%4anTBE@w`UGB4Mqb*HQ8)9OL{; zusR)-yU${1vuQCMSB_IZHd`{j(JLaj>M@AKl`gSK#kb|eNf}Gy6sc2HLn@1aYGc;Z zRF8caS|vNu1?mTx%HE5<;^0WdmnSBhuAq^esWuT&YDTw9$HtBaIHQTbBL zvjNvXFW;dgFNF1w&odNtgtySF-ODzF zz({Y=-ly*(xTCeoS)*T$?>JCA_{l`b74`+Lw+!)vnZ0@OExslT01jJ*BDhMg?-5la z6KgEL^*&Ij3GiT%Qc3O;m%bFlo_2<;VJ=TzO4a$YypH=i@XosNC=z}jH+@g03 zX%ls9yj8IgUEhy|uWV4rfm=9@0hx`a9hfXnfrSr(TQI<{>%Ln_1+rg%Jmbk?Z-w{&A*kwmS-$hI~1HC4pMgjt^vRI@0qa5$0CV)XtHC+>k7$fEp7vwoC8vMIdVbO zFdopn!)d^MvgrhP)ra&jV#>P(CEyFl)(Bs(owpt_c>jm_=(UqX2MP4$iE%UM z!JCgW0~$XP-&ZrBbAVBvsc5xb=g>M@NTNK+=^J4Ib*~GJIGKn6KQd z%hSP|PJCP!E`wz>SG<;&^|BC71+F0(JEVWw_P8@KCSldCZ9liIRVy|QWfOx2*NsFw z;6JFLt9LRb_O8Nlu5=wf2N#SVm$`<ct(o)T`TolAT+!x7Q+`Me)p#e%05*07n&vQw4wyu z*wGSbDP^{}K+VF$j}C8XXEc-O>maQiL(OnK)84qEJ#M{arJ$D3rWQHPKC&v|?IHRM z2MI@M+I>Vl^gDEF8VZLiBAMVT-;jL&pqyt}Ym$P;a%j7g@63d#zZWEhC;19%1@U*W zxaA69X{quaQpRLnd04bxVw^X=nLpJFAR58Ft!={BUk}sa(rL?R8#vxRA6JWHCe~B( z)ij1UvFTt%FTG(6dK~H%zQMfmV25W&_5O=rpEWh}PS{__y@Uh-Vf#18Rs8Rp|B}06 z_7+z5W@OC&gzc65KVVBu563wG`*Z&riMkbBL*8>BIoX<_v2s!o(Ek&wCDClG;V*>} zz6}-rlpk)mkHpv)$&$0KhXx02*T7o(JL}I>IxC@ozwZ~QA@bzMSpKMRPGNvPuwBtx zel0&v2bjCEDBhuMyn6Ui$1mwCgRXlU9it9w$cexo#FY$`O4ha_Y`8_ zGPsp;S8Sffov1qLQvtr)W@{8@0?i)WP5Kye5ZULvZGR=nQPP#7mbZ@%%u)7SY(sL6 zGEB;!O>~^bR2pJZrdjCFa>ezVu^VHfk9#qijxo zP-@0)QBr7J1zAHVPII$j)`ibbPscLl)@T}2>M>$ywi?|p;5UL(PvQ1}R%{YOCB#kL zjsaEw4p2>WtMakXYpN(sN$5ocD^F~sBZXtcDq+;5?pQ1jl;U2*gMW%|wEGhRvb6Xn zrl5`FE0RVLi=y@8PX~4}i!+FNI$wa=3sA;Daw1>Wj?pNr+8PleR0(Cur)knDs6}hK-7vv<1uvkP%k&3#>au$i__{~ubVd~&Af^EfL_$`g zjKo{luF)9jPY#1xivU}nb`G5Ka@eJd4V?yE>G8n&RNg|dZ99prdQcat{GLGm1DY?W zkxPStgTFYc^o@Mpw(oN^GvO_yW@LkqogCWw;ScgsC=B>Yo}4>)OlqU^daIB)9Yf!J zVa1nC!@Ln$T-9q;?hv6Ld7-@`i&>waw+AmiJ;S!ut!B+%RLO5`AdNtqFNZq)kz38f zqMCkn|0XR+D8C*epP-|nXKal016EgpKi~f_(e3I6)<=GzrCm6!W;qsj{_ND#!z-&o z31|j@ZvO820lV^hF1rLJ4C5bZmkJ_PQBV@3Oy&CaMpoe^iDZ|`97;u%S^(Ee0ZILcQ5>GMon3aF15gBv>$ z{Y;(O_>vraK&VO{{4OC-FYMDmc8CarxWM-U70lVa_h!PO7KUDXj##V^qKgZ3|nyP8WxcLlI z)tcFV3N^pj`Dos;5GE=EGA%}bVDhOp(@r}2?6#V@TZ-E%J# zDRw*f%N)4&bS(7DRI`TT*Jo~CvezqcFHriu#N}DpTt}P3k7=&7yZS35gu2aZV)H6a zRlzWsW_T^8q@U7H!7t=duJYrg_Y;Z4dm7KLYqACV5B`?sAEHMKwik!v&+ex-Np`p| z{&li>3Pb{a@?ceXnQ^Liu^$YBhxm&TkQiG7!sREC6}k60@O;cZ43k;AV0;G|5D!6@ z8AVZ6&I6mH_X@N@xD5dnLSd+4DYCMCokWae8qRY;Xm7`PA^}4-6x^()i(2a#>MZ7W zbjmq}^h~p(KLL}F*riqZ1ep~ zQ^pYrP~Zr*?|QVDbcnjkK_pAYLZe>QYssGaIsM}cB#|t`Ty&YqLzr1dP1*Z`InNJg zffYzK1@@M#&+04;%TltWmbzKu%p_I_9jJNOspd)#DB0}LET(nkR*MuMT@)^_7@ z7`r$t_T8df_R{5%8pt0NLk|1lnEy)K-Km@I$aZ1wz8m9BN3^glWbV@X42&Hcz$0BB1HoB zr6tv_e^pewO}C$H~W!lD{joD;3AeS^sz$wr*s6Zvc)FVn#?Vj1G~TF zaw7NR{Sus@$j0OvS9h$t-=}2Oe!H()jYSoZ2jgE_tl!*^yK&T?JpZ-vY<;~9FXLRn zqV7X(Y$abdJ(-bxf#x^=UM4jWe(Id~9h&t%S$ia$KN49Ke^jw-Dql~L1wMOB$6kGM z+IN@nxS_=}+yNwqeg5OP^0z-h$hSRn?9g#a%>KmR!`xX<2F8b@Il$sQ62Ac`(B~N zJY=THK#Uf)@Os%&&>9J*_k#4FVztlH-=&G>=0f8!@<=U_oet0z`P1wM$awTK z+FF$mXKJQ#T1G9DA0%BbHCK|=GB4WX^Q#gHdgkF2g&wD6jhc)YSk^U^+o2o0<&x-3 zK}%`5HN#c&myNJt?!qFHP^lmE3fSPv!K>rOA0`PkQ0Cvfy@H1ql#sfP1XD6~q2o2z zda(N!^;{sg$ZL?fJ5F>hC3P9o|Y{NC&1 zw1sg=u~^CjB4Mcx68ECyD6uB8MY!$S;_-bN&rdThQ zmnvD_&}*WPCe_lktg;FWtagT((=0R2&c<{6v@oXop=zySxqxrdV^8!wYnqfng{LtZ zSE78LeGf+J_TejIWl(CfpeVDp$HHr`IFtn)i)+I*)_gyIqq!(Wk*IxmPqv996an(a zmQiyZ`x4W23(*|&2|?188vGNheNpgu$U7h+^;d!_R|S#O)^h|$a7tA7CfOZWUfY!A z?uih_3Aj%%8!E;ayzvEVHnO7+A#DRxNYH0`Okv4aASy_%TDsri^})Y#;DQs>COT~y zIj4Xd?89XuUKA%wF#{Fpi>GJki8xq;C65eDm)59QE%+2zDTbQzN;N!lI`uD)2R*F( zwUpNvzRaD)GB4%w@9e@dK`&Jwq*L3EH8E3<`yUmlN4+_QR$>Ror(Ly1-q(4CwV^(N zB0TDhB&$DlT|@ZLG13I$Vl8{8B3kq<-h{(~f)HvL>^t}?4-|l?O9_z$CbYE7^JJf zzfO`B5jbCY4?0j}-e-f)yju)-6FNnaV%I?_lO{AOuJR55=2 z#_PHj;X%sht^(g&B)#YUkp0s?zkqkc=^6jfSov(NJR-vySP*4`Ntc_)6{27_c z@ZvyD1{Hq{>M{qhg*+!zPv#o^bY8ryDNv{*Pbbm?^^nu-FoJ*68L4rl<2Jr220NRn zw>rH=exxJeL}~o2$$+?WZ{5eQ?)ET5U8t+_xXj6okA918q3 zXu2mG3wR+S2(BANl2J$im`=dk@o$9GV28TsGVj6x*$jPc*&PyM<5YmQInwFG8fW7e z!+w2B&ZHI)*;+!Z1}ZtZ5N|g=alo% z{$<&m!Q5cF6dJWp#4 z=;l|UHGkFGp70@;b!?4h%+Siil^G0ZYQhyHWHu`Ae=R3NlsfrKp4XLs*rh&55sJXp$JU$o5^m%fSht#Oh8I+WVM-$>zz=-5Sc(TVlM=oVP4h|DFwoLm*BHmS8i` zV`Q--(fMR0k(c3kzE->S4cCAB9rJ%ve;Y@%I2tGr5Oqut5T^f#`ditVF_=3z*%|#q z0TgTQIBc+g_nS2kYGP#A<+^xm;VVzsgmo>FRZeIYLuOi7Kc+~;5gWaY`25Y2&_J)1 zLhlY5^wWJuVw;g*ysO7cu7nO!UsybeNMBxNcmbtM7SnW?HBJ;jB%DwjC#c+?Y^p#@ zPR8OqN#?=;n$oAnMZyfC1V<#4_G}V6rHM#pQf#(IVpD=4sRpN>&>ZKVh&?0M7)4FN zISq3_rK!wEf^w{C9yRN@WlXy6lOIR3f@j)pIDFc-Igzy>Z?7(%%5H3i&s+)S2%E8= zJ8?nWK01DH+#BUb0gm9IR<7UuvK5!=60FLArj@LiiL0+ng@zNgLLZr>loRAQOQ#~o zB79EC1B0v(L6U*v@H1`MF{Hre=WuSQ^033|>cxYXXmm9x@yV*rNlhu%3elsVus6a% z%!y(OR;9nXtZ_gtdkBW5ymao|SXPbeoh(omI3LhFWccUh`*?9$`H$pjwDqRdXpz(l zkSL6!F~6gsp;&`>clpavk-J?=!hy~ZUQVS$i16RTWKm=24l-Y+MBKqNN*>a(bXzOmQ8*WWiLb1g64c z4!wlXwSu&NPSx<4#906v{|_ba5GD%LBny^p+qP}nwr$(CZQHhO-m-1GYX0uGc+<0( zd9%*tIbTF%Mn)1sf5xPOQMFPZ!eeS*uBK)Ne~ac{NHmhh*nGt6R1pk5jKH8`{oX_+ zw5g2yeV>mX6kn0-DoW6w7$5Gr5PuWMF_h+hT5`CRm7{M6%h8)Uk`o@7AbC$-oLFV( zCFhQb)Zyni-yg%k=e<3!{5WHH++uncPrg6gZ0Y;i4j65xYkalYfkDzh6I$MuP@=UY z*dsj9XsBHvFABlr76Ig4xD^f_g`(&j8}$)7D|{^qQ0s1cOJ|K+1^0rkzBB^Lh)UH? z`C%HuNJ$e>h{kEV8o7ln+K6^TVgmtx#GJ+FTju^YskHzRo0)OG(wk^;HefMJ(g&&i z+ArNFL4MYGSo6IPn04Y?Ss2$3hEI}#rjo*pK~ov?Fapfezz)N?i!g6D{b>YlyU4%6 zNx25SpIGNwiKL}_(tBa0+u5+<1j_`XG4A&gW4mBGjF;AC*I16#s_jNCxwy!-GY&g1 z_`$?YYRR<&*U5iu&i(Uu8^-1?t_U48o{ zU+YGlZob^->a%Z$u;N-MD7_Qh9N(S)q-h!_zb{JuaWeX2twfeo)|KdnVnoDes++k> zM&MM~H73eWk8)Za-=;-DKM%}fLRtJ)Y33lX*;*9!tuc;sV52^0N>EFwvVCFOTKXjA zp}C$u46mvdf3roZDymu?hffSk2YJXlrS9-pB(5>+r@Au8+RK>z5Ootzs8y=zKU#ia zo_H?!eKAr9h9Q}F*D%`KJ1QnT&p+P1w@eyJXBW8-$ab=bQIBacg{B`n$` z!{a|XMZ=(`nr~?cG7EgY)IqcEFpN5T!(_<2FxCQU6Z084Sp3~~q_$x^gubVnIj35` zI}To&8O;nQQe3Ucr>(4(OJK^m`MqS@t=MsoUG@$JIN3!Jc;|a`!iN&!e1v znI4d{t+(V7eVOJjz*m9XPyk~n=UV9^WI=i5aua#LEOFS#seF|dp1Eu^V7IqhPi?GqyiL~lih}`@#Cfut|?;ls6zIFGv1G!qnJulPS_dfjtllAmD5)m(vJdWQw zY|Dl9<(GB`Pi}PLcPfLQEAJqD{KfIk!{*=RvJ=Mhc+*=+30aI~=0zZl4}9B|{PwOd zwD4|Ret9x*XTyMhEv}-<3KH*5Z{+C-qjM;t@Zli3B(i}%6IcQ*H!Y~0tsdXhix~%< zx9Ia7QcpJ>f)Hp;OTR9#`}CA0}jb}1#h4wM=YI*TGBH&7_2G9^0aNv(U- zStz0U29!$IeyhKdoqpVHQ1{Hrk{p{ZF&(v#LUYLPXC3dm$e>LV{tv!^6|CK7Y4%or zuJfqdF;z*R*bxC6AGroLRYq0yWfF5@f%SapMezIaXD2j}lQD)S^E&K=jRkm&7P0mx z>eveZ&Q7cC6|;IZy{cck{jHr~16}dQn@atY2MVc`tj5odgD8N#+kh1nxgj=HlSwV< zJmWD&8IYE!LG(7UIW6-t6Kh#Tv9AVBD-R1nO9sJRHbD?WSKyl&0Z-5>)~))qCPsKQX)K>W!?#Raf9P~3xP*oxkxyOfO~BMKnz)QF$ApD`pekgn z?IbH1&kVRuxYT);WAoZW~fO?mo^-J}&Evf5n6UeSTc^J}bm2mnA}PyWf7gXEr(3Bvw^>gMmqc4)lL(oF zb$`RBiJ2%!x_NWH#^f^H96+RF6`&7Oc3jqLoZ9Zc-jotj;r$cO2{3Ai|u+ejXm?IdWE z@}23wa?2hZ#5c;^MPBIjhIWtlwvr7;|HLkpNmp1ekRo@0 zac+Zym%v{2G&JAysr>|6K5y_7r9Ymk=2YUitqg z$XpJMcEMu&N7$PEAI`O1T`X<>3twh=b>ojXobH~eLwY^$f#mEHKs+V%irQA0Hy^~u zlg#a29V{wrH`iimIFq({=Lr6IRljMJQZ{#c0W=NTXRC}FHE5DXjTFDi-Tk4=KEhIA zx4>*sMMT$HKFSO=dpSYQoTe`es5BK;2OFz28!D&T=JMxlRfT7TR$!WZXa6Op+B&0? z#DLxef<~KS$rl-3rDL`t0%3_7WV6M9!Wi93F5TJ>~g7`D9-1v?8v+`aw^%hf`_DB;eF;-O}bagS#w&fFGU ztq+n^;BCiLFz;a(Y%O*X1k(l9dOureEBF68Nnc-I?~a-FQeXYjS2^>`4AerGe1<2W zuWzJ-USlb!{^L4hrFj29G9gDWPv#9y5Pgjb9v6>@XMniZ3T~SI!JuMat@E9=@{Sm3 zP(CP_P6231O;z|RXMj9BF5l;qIW)oXiCm7KcE$36RIheO98*N1Q6Dtv%owxm(Gj>H zgh1CXJS_KM#07_O0flY6E64H%FgmReZ|j9_!K8#-kSd5F%Kt~UcY=Mc)yZ$}jdop) zu9j{=My`hc1Qy;52-$9cINo#8wjwYq*Kgpk>xj!Y)ZsvDw?RkWN%_dw2d#u-2Y2E& z8YDbt)xyRXfZf24fxL2jMQ9yNYz@Dv*3vtz*?8hgH%RCWC~bTr)&lVd;<(Y|m2Ru? zbc<__{7vgUc{^vTw=G*$WPnXK8t*Q6ucHb&EtNw`jBsL(je<5hp(1cBooX-0l7T`j z743}vxzWLc(lafgcr5F}(~2IgJsVf>`pX4`e1`Qj-^+$VuGs|?m;J}{)#Oo!!KPWWZHCLAhLGSYecVO{4p5dr%LlO%-?xe4- zDm@xYh#)&~G+6QWt_m3o8t!vP5({c(=9`Y^&T$dhdR6S%_}PKZ)xKbI3lk$W#-E*@ zVZFBYeTysj#nFKVja|CIXtwhm7>kgnAg?rbt7|wm-0jrg$NAP{Dx~%&A*|zoU<^Cm zIZOePriM!m92{~CY)hQGcCNuDeo5-vBbv2_NC5iJH`+lh5XUmqpQ17=zQNN(lv#`K z4_0RWK+)``b1uphw)$*hG&j1)bK(l@rmfsQ$1DvtaRo-r!9h^x73tO%3vJD*NOgO9 zlg!(jgy!=~Y`E%iEka{tSMdSk3}r0bvHRGXyXI5qY>5grfm(COsMX0%6F3{!OOa7m zz>(}$llmP2;7m}qc=JZXmet(QB75R+uQN5qtC`?c`e%%FW7)B)brZ|b!-bs#bS zFuK42)%d^$Xc}G;_$_(+_Uv*QVa{0#mLyH}ctc91f3K{ibfFB91CRPOgmhJgu4ypt z_r~oeke8wBWK1RwzJ0LIL)jRQ(8G5gtkE}|nE7N97|-Dk3N^=sM}~^_fgAyV8_KJ1 zr}A(-0guoK8wTi#7NvwCOH3*)cCivdWs)-BFW#$_uGx#Jr;!)n5LIk~uZ&5zSNL$` z^pvkR1Mz}NYxkkkDEpcDH}?`5B8X5pd#jtj=S%^s!|(v#FCNcWAl4*a^hH+XIDQyl zxa}`#{ah@bGm8x00G3k{2^ba2*yb)I-@IIz;gR-H#{nMnIMFTH_E}v!apMD>kDWtX zcV3JraFaDrcIs7)zfn3L|F5%J&I?d_0~XC#?vfaS*eEC=O=ukg>5Bl(H z@!7Gf|6ps^oeo)peLnOZy^*Frw7s*}w7OE#WRYdj5sxHz=rt{#DNR$lPyBDiCT*;u zy@p;uMvuT7A|&}%rdiNd7-x?U6L!REOku#%M4lwK%cp2ETv3HVv9sxcLO#jAxj<8G z8^BrRs?}4^i8t^yQ>uZG>dL^pa`~ByaB6&W*EuGR>iB&uE?pe% zoz{uLEeb9wOeW}*-*Ddhbv$+NnxIbn4mU^L74~Vhp*sy$635O;f znYC)w770=VEON`9j0z*3{QBj4KeWk5qfU()b7oDgy0S&8tSKd=KQlji^-9W`J4!p} z08n4xq~q_p>4^CP7;~RA3jL>>VeSnsgI;#hVU&SIcMG`8v2JI2Hur9`*kelBLiP1a z528Md0{BN&b{lG)M7ZUXQ6Wz9f}X-td9uL&OzlXI0caz_(voCM`c!>CS4ZMi$K zP!|e`xsyo?8(|$G-$719w`|>yLxJ6+AND&+UZ17sCLQ(xLNSXBgI^%@;h7Jumg-`g zqYqXc(`0;j^E^(U;qR52ECNXe$9*RD%3L^B6Hkk? zZc!+U&~V@n?|}xrLAvL$UhDscK#}?H%50Pk;Q(x$9?9>@M#M_Trt(pwFXx_!vFqYd z@bGw1Frs|>T4bJ^w|2Ye!m@#ZmbL21cB#|&TdEp)kYte|N9aS<5X=4gq>x6_Q~mm! zKtQ$P4a?7>a8H+1&R!9lgFkaxzIs%t%NfkE9OpfJKp-hB6k9>2cxm!XQwHrk8O`j2 zdP*cZts|g{y-(-rCgjyfK{2@Hl;bW4VE;J@n>>Lkf24*&zEJ0T1PuwudVK{t?28oW zfc%rYzO8oXhz}p~d}o=_#(Nt$C!B~%kg~R}_I@{PVqd^MF>o=5(SYNKFQs!Q7A)QR zlH)q-KAsnF-I~#`;>hMDy;#>wp%~El^YB{f$5(HBJQ#incP;AuV~Bm8ua}<=FaK~# z^lK5+FFIF7e?+188PP9xC?!}V$=Va|$|w=IPY|hkut1&ChhO}$!maD8GQycHlTSX; z%2&cuW4uofjh}vr`W{h(eOz}CkZ((x^K%lp>VGqAMf(R#81Y zuzFPdQ{y>%ut6h7G>fpDFsfCBx?~Je)TJF-wfY#fQy=1@rBBPRL`zS0v6S)(NoLTk zmoZM0k$lcjLYoxls9D@=BLG@SB-TuxE5To{4nbK*qy#i%iIfc1FS-aqa(HsIksXZq zNyzUH#2NFbu~s+7opNh{Lp;+S%*Vv(%J9}@X%XA3CY&sntWL}dhdR>P5@OtL*(aN6 zzWaFac|O`d7kB8OAh-qf(R8_1LB2R4xzjOQ1v6+UQD6eJQIXT6lN)Gh3eGy6t(>5X zzMgoVBFm^iXqH{^s4(I8P_WF>P3xw=v>|20w*ou+EOY&=NT(-=oGv>d-_8&Iqo{V8 z)|&C9`AF=EX93MWV#i3_QxuiFtEDW>W8`pq53d1_+L?w@5JXPb!LB310hJM?h7hZ) zNdN^pbjm{)r5jV4tWD@w6b8Xy1BpG%ub&Eolp3Bv<1se9#Y=MDM&EfC-z5M?Vdc!1 z`%VyjB%;b(ahRUt1n5V*TjFRtGf>r^OsiGkH9lq72WClqlSHWihZLU^eH2_0lS+^( ztP=~#90F`h>i?E2YPx=VJL^%1yPU{UoR4S}aF6u8^5v+i@yUbk1r;5 zu={SIQl3iZ%aT|4ob+(P%VeDr4-szG8C=+32CR43PUS)~ApXxBwzq|=ax12o9asf7 zWYUl=i@!`ie?VID0a@cvxtUmZMViIhC0F!g%FsB=#J{wdj$T*>cex6k(uHk zJ((CKzzoh+n z0BDo-%(f5J3e|W8G|yF1q%lQ__k~e%1vO#pA^>h8vVp1#kSR!|l@=G_<$cM{oq_8W z%y!+WV!`YvBV=g0re);JOp_R0yYfm83t3I@g#|HDozqC%hP!;hcoG}m3i9;dq4}l!0^i? zJv`Z-R~@`++rH_};+nxZ1!qUfQZ^n^Ceb9s)>Hb@#xh1)m$H_2G&ylCg{ZO-iDCt> zK5Y=M%`5%a?>i*tyvua|D};|hqJ(YENC=Z%UegIJwJ<7p*~gYH@EFKp4Vfq3-7eP) z#u9wtHfK^o*W1`~B5l3>p+x%@fiP<)PCoTe{0glH(z@bdx*pm)oTa*G-`^ar`OxDF zXDOtV&$zUxu=bwdQZtfXlD7_n6JMu*gwgfr9C#5jd#IY5f!S*^&!oP4+}f{LT^U~T zqsHvvjO>g|c@aPG%$#^}7et2e$h}!Cd37!@aR(xSL_e$DhK`wIFqoj*8|SqU@`Vi- zhc;(I&kKQqO6tdL{Hx1umB!THICJR_?||r;@MB&!M<_^9%qr*J3q}0NQh$mjUtypS z%NDH+hCLcDiF`j@a?9_6Bi8u%MJpt-MF3;3Q;g7ZTX13RJ88=9%4rKuEpnE`S7{YwB@}> zE`S*EN$j;IOk7)GNo0B&vy!4yatN8fmw9#>q)Et9E(Y@~Tna4-$ zt=DJggg$=*uWDW2oy#KvvO9R@c=7|HoG`nO3qw>|j{x03{;}u19eeG=CfQ)*~J*6g5BzMQOf4 zixR4fwsqzj6=NYYS4`v3{)_Vssri`}UXnbz&G&LBYYK|4$v$GW%wF;!-4ZW1K991L zlL`p?HkbKs@3S$hwxdUcO13iV2p?)4g9h&DCZn$u>j>zU6T&2Q8+~doZQbZDSCqoI61|CSgm-dy%#SHYLpZfSIUBq6LceOI%!t7}l-D*n+JVtjldA zU;e?5`&ComT=0^13IaCz$O%l(NMRoT#7Wl^0=$3X5nG?rNggNQ-Uu{Ae;3$)nkTTk z$n`rl0F-%KX~JQDq?IoVhUo7J&-HCyEU=u>b6*4+vX8DAe@ZT>yF53}RddJ$J|m{IA%t^OXSILxI^X=E;Q4QS)yV+&+ zsE_3ErFkb8sh`bpcLqi)+(%v93F(uO)K#wNu=n=<}Z2F1zlgc2u15NNUNuL z!en~Xm7Pb^coQ$IC;rV*wN!P@NiUIjPcB0FlWh_ikVD|6Zmo8nm$bta)RSO(KXwkN z)@>D@_JF7(zpz;r-3`xgXZ?b^`#>jHr+#@3%yn}m)&fb-5tMFdy+bX@Q!k6)0Oej3 z6^0C*2=H41x+k3uYimEM(p_;D z@D*wNhQ0R*{bjTB75ddaDvh$WpSw<(hzlN zVON^qHw1c6(+eKenx>qEamM(VsWF@B_qa08gVBvPba}(O3%|T@fun}Q6-L~%vmtGtu!rN5UR){L|($GU1{s{8Sn>pIWl`M-MT*A`{pim6b1B(_>0u) z*MQ@Z4ATP*f)q4uL^$NqC0n4$E|rI4Ec+PsZFpLK;NG!h^~qxTYtX1aDq&F942^>S zb}dNr!sFMI>WcOkRj4byRV@)YKU2~q_^;*pGwmc3=|{;I4t=(sNTVZNVwt#N+%uJ# z|GIvrK_VZ2sKG(+o#2*VN&X%7aeUU}Dtak5ad4l<>pY)&0uRg%Qcq~;Ha%YX`s1CQSOJEH{b=((`eyqRxj)?vcx5hDJ5h4#tFM0d(Z`=>lbvGT+(^DA{XZ6Ff_j5Y z4Zr{ZM9}`fjm!Vk=>B)@zl?=i4Qb~sHiW(}bpssyW6ads=M_sjRT@r3Yw5bukz_Y6 z+z6d035ftOf$1!{UvGE7q$0eM?J7Qvp}sy}K#lX+4Qw}Rp#(4@GgR!-Z4E@MeJ&Xd zLgED(Ndu{78B2r0B(-Q!!&I@^6>Fxx6#z^LYaljY12#am%(?M(Vzg)!4U(93p@h^1 z$f`{dj5eiD>gYwbcp}5lBpy>CBEq8(Bo2Dbvvm7hr?K@4s#Md^z%W}6NW%7W z2Cwv!%6%pf^t@ooSdfa(4QxXzLry7YbXa9am={+~I_5pbT0%#V;QQ;{ZWGzQ3alPH*;%SaaW3jy(Sizntdr!Qo*8Cj59a zo6mu|^5g>#zC1Yx*I}^a`HJ@|52kD80Ubhwp5rVc0d~+6#G-&zTNO_M-Jt)7^l*g{ zqLGE;rhRDx=@%F+rR?JZX2XgcFy5F3VkSdfdd!vWXy|C?QzwsjS4Yhl0<@*pqH4#1 zQx!XLS6@&X#DdB#9YEo0{u)SAr&Nb&riREU9yc0-*qpg6(r-*|SetXihxi@qM_C@@ zj4Pa(jm+lJywr$)^GZLVl?t0p{qudQHK9tB4jW^-+7^bHArRr7hC{B<(i3c@Du#9N z&4(%vY^@TgD@oll`EjzKXroS&qd42~)Uw+Efm4+xDW3%Np^p9$1`<#fkssTJ?4=X8 zvgrB*E$OTPY)3RJMD06ej@0H-VA2YM-U@F~Ndj%sd+7%Doj1C6$|={nc0=2Z6b)B; zK;!-hASm2ad#te8D;F? zR-SoqOselK_(?@d&(w?Y9!RYaNGa)dhz^V{QanIDHz)%}E{~OTJA54`?osa~Zcz%m zTOg0msF|)p^%69bt|KznbqSGsxQ)r@RnU+)(CysDB#WvVhxTuCEOD^5JzjS<$Z*fi ztD!KCh_kBo+{G*~wo}Nylrf;bBG+MX4ZntWk`0ay5yGo=rrUNlsF;qi$ziX9aMvj0 z5h-WjMWhGV4?v0Nq-N16+~rz;51SseI-U}(SzJiH+!b1#CylF9cXHT@W<0>|1FsgW zab2Y2OB!^xwAy+IF?NV%?RR09L|w?1R5DK%y<_8|g^g9jv)RwNq^Sm2#vkqTK22*< zyZqz6zwi_uPQLi8_0d1p(_1^uN{QI)ij#%D+_4vbUgIl-Sc9@hO!QwdRE#Iz2C08b zPxv=EebqGYT#L^1W9cE;nQW#`ThTsM^)^~DCW$vYCnFq&W`%K&%+l)%ix{SH5xHE# zusbtnyjW{b`k;N85b~pyV)F6NQHZSvqgN&DN3#!;ZAIS?=9H$dk?%v7sj|Tex2@;9 z!;HJAzdx#5qU~X%-Di@%e~z9Cr-^Vb5uS;gwhz9vIp3r>t-Yi#^fkshknEDS{>`ay z&@8g)+0{)S`d^2+{y?8?6zH>?^lT-{+V{A9sdq-vd3L9CYGi&OgaOF;Nr2qB;xX*^0M_!D;!f$*nb@Lam~hw<~;L);78lA zIPP|?igl;&MB{hdl$;--YJMeBzO;`6ggae#zoDJEl0GX2_^T-WCsiU4%u`O?d{`xj znG&67Zn|seMVhi*@3jEEZvTE)<23#$kL6jcfu7E&y>XV^XPG_eJ+*RDGnMRFcx;l+ zr=DEkxTrD7JBH0%e>R1G_yZOBhO6f_x7UT^YULiQ7xE}N1~raUK4g|1qV#)oL`VG_ z$<(a7Su<(jUG6#SP50w&*WGcc}D*1Z24wkw|Z~kOQ+b(*!c)PjA|DRl?IR$NN z1Ox!E0RjNP`Tvir49(4*>1^%Y%#-Ge~vpqW_=+4#9YH-qzS!uBjdT4ND9zoXj3smD_Ym*P%D3!YWhjL<%M8ePJu= zON;fk@LNP}71NfaYej^W2p6Iph^JExxlvw~AqZva<&jjpUL5C6{bE;<_D7ci6H zy0AlGbTg5Np>V`kl|Q453OBmnX?4RwCLb)FUJL=+4*@Ec^0>y$TRrs~9tdFEYAs6? zK;a+5*v5$%3*e0rm$2y*Xu()Z+Rr>m2w*$kS9i1Dnl<;{8ZkP?Hr1MwJ2q&?vQDDe z_qhpfRfp#c)%dwB8TV%cLjZGyYy_GReKmF)3Jr$|7PMisRh0vBcR%(1`Kz+Evc+VMpZ6xw^98YX|OGYzH%|p z7(%Rl4o=5OkascUL_Dk>PX+Tf76nk&m!5|e$=6y&?}g^I<5GpWWmzH+1pQJUtmK1$ zFl)dO1%e#YRRK3ADPz&>KyP@PYO;0e|3&l~Zj*1J-=8sqo$C$y(c9Y6ap`rJc~ zRNl1MX;4sKygP^gpC31JvF%^je?*ooIsgFE|M7O1i>a-Hy_4a8Z*ZjkZ=sbF%~L#; zi?@YLtjLNS=^{aeCtqG1C@CQ%g9HPhrN;GfryBtOK*`AUZ^*AZ+Sk#&5}qA@AqtEH zkwhekkaXs^ERR{m4J8@@SD0Puc*F2M)!66+MPuarXAG9 z`z;+U?g3H+nM{#PLYCbVUV>`Spn*<~I4U{7;_aC=Gzl|;?@S88I}xL!@fEC5Hy7T} z0oLQdIf3ct>G#)zXRe{mb?p7O&*jz+)UcKJiM>Hx+9oLi0a%Dr$}_4s zreh%nV-*bYN(&?JSR=E+pF4r4Yfe-Xh|Y%Vy{FT zaOW}2uV}FgM;(_2$=^&#jKY}DNKnPyCLX+(qTP2;6w44f_P9xt5n!CePHR}EY2&ly z`bYIjzO+#wsUmxLa-h)VvSPnNu4*pSvO6c1 zBj)SQ_4m5_lsFwXq1x4Tp9+sKdli}^I+=lPL>jm^mHhsh3Af8Y`uJqoppVJGS%hh| zKN?VjAD?0vx7=wj>9#sVV{8i2S+419D~Ej=L&U9oEXA2YiCC7Cd4w`81^6*HZb1`x z06h?u6xpW8p8)knwn>yhfb_1rwyv9ByWTwkoxVt{GT0hy-vrT6?RJcr$cD|2JhA6wy^XjTF7 z-KKWy@mKt@MzrMK_~x9w^p;c}U#qovYtWP6j<^~1x$mdf!Hx4#kq2Fqb1Dj-bLN^; zJ^m%*%8&CIe6Ko2-D1&BgQtepG}!pa8+%^mpWTDjg;rMgT)#o6DN^opD^plVw1t3z z-iKMjNhIap7D3dzK#a{W-hI)?=gmW!^gK?KEb(>C%=kXmyIB{yW&bnGXvLlIpZDR}Ej~2?)ZH<}(AFaQ(YJYZ|xeYSJ6Ko?+pl@X&+`GQx#rpa^WbQPfNkO8bIQgIVF0TnnCE^QKZWG0hP1enP$RNE%gDsj@Pb0iy15Ad2r zc3Gw}jCxB!4M^l`D)A8ksHm9K>K*p2$Tq3eI#cbW5Sh(Z#~I!1;%ig0bLUHO_EzFE zbpm6kvJb|0F9|*Zre4kZV*JU0U_G1Bdc7aBGz5oKf8L+NaB3NEmX@9EU4KcBBRZr!+w##a&1t6Ra&_ILLh zfx3^8zMm$ZG2Y#qU=G#b(MsnxK$ zfSPya2&HmKP@~xrxN;F_=9vgqWwLaVy65kqeVq*EZ{Q;%ImTd8YFVl*jQ9zm@U5Aw z9-t&<1g$ zSdZf#04*dLY^{mG>jy}sQc!0YW<1l`p+UoBY-W+pW!$m+?( zBA}_%Df4M>8lO&f?~pF?CbJVUHu0v@Y^700P;oZAfjh=Op@k|A$>P9J)GU_*DP~Aw zuW(9UB{tHM|rCi4qap{bat(Yl~M=~s^zQMeTnb;J12I-fqAl`@Z~)`#G+@U>Q@ zOn7$4mWxyYYwGLw*ut+0s{?Q?n|(lRAp)Vl{&}tB-@*vv;mK%o2AtmV)%!v`Xy>+_ z6`MT@{Ts#cK$qIh^wfg`txMv|-weHNXK#>JP>P)qAxJc0Jc!0(?Ew#>;`~>6>Xu@YWkN{orZcs&Ev2KWHgOGvNkzlg3)J991rx9r0~p*i<6MTo7%$d-fLTnPX# zKk`F%ti<2rudX-+UG*DBdH1E)wsP(ZfUO-#9KJ#zPWsCnoNCOpiy*b!)9r)(8c;8@ zIP@A$&@11C@OJK@+of9Xu=M6zS6Swe%<5V?{?qW_>QDCYbFm}wN)TDvprby=Zo7@- zg3e_?ujEOWJ`DMKGI34!r!%9R>TOi;O(F0@SMNBUg3UwTn#Sg@@L*}r^Zizx*FK>;M?)=;^K@gA zs_HoVO~4Ns*Au`zaRfMM>lE?BYlrq`4W&n=wM*vpq%ltG&{6Z-W7yH{X}54|V+w5M zv@#)$XhdV`>XHeyd3ntpT@#eu+a{WGB~DD-L`{!dVlL*M7#AXJFi<8w28Z#J9I z*uk&JeXOnfIoZz!vdmpM+cS7%{`@m_TFWp0nri_a&e)*-qug`NO0qxbz^#LWlgFZ` z_XE2O@M!Q2!Er>kV*ddDrylSWe_Kuh0RU)3{y*$;|2v*w=4xmB-`7X~tIJiFwLfA* z=>1ZMNs0#TkXkhMcH%>0#rhZh@4_g)g(a;cRgzwElrie}4UW>1SgL6RHh}|ct_O3j zM|`opS=-|j5-OGw15zGzIgA`DQ_6`#GzzLjAOR}wf90UwN)w}qqN$KH$`IlyV&V*= zA;`ov?A@xv$vN(U2v54Aib%}uoS;}a*B#O>@i`FKq?PSDP-+0^CLBy&AyLd^N_9+9 z-dq%UreQ;;ngvRo{i3)})=ti$woS`ubxkyFM@&UE@-&3-U+U2B*#kFB`bYbRwz;-f zq}dno>E0*kP^d}*m5iaW_mFuosWPg#^PuKLg6zEMWMPq%&)1o&G`m|I)-3PF=zE)9C4^U+i95LT90QY}VqdcdVMr%H7SQAregeQ)%sSGWhgj|m|~!LZjoRJ*MVK0E_wtx%S4HZ!C7b;<%=R=CFdF>NKnD7WAfo zG68}&LD-6-{-vpFw*@S@}hw_8Yrf@D8Rr_yF{ysq<1}GtB;Ak zBnEl+T4e_f$j}5%wvJzw#ubp0qUPFOXz0`lTgt4Uaso9fBO59nA9L0T5)h+2aca*s{9B-e#Fl{C;pu z445NUF3jQc4sdGIhJSX~+(x#;z*TO-*e*7j+2aCXjAE^3!o}pf)V=Ed;=6I=Rrwt$ zb=KQs0S%92jGr?G{e1s!;*=x=va;3r-iH>u4=mKjDNnK?#usy=%e-I7;;r}pzWozj2-#%%sx~+XR8_Msl zzTu0XWGT%~mt|I)yrq!)qEdv=W|eyu8Mxd4_P-f6*&?ey-|;XKfq+XoB}pH>{HH_b z^YUtWlR=?1WP&L@2VoaWOEj?oE_OdQet8U z-Nrh5zs4y{-U2H0EEFiKHbZ=&Iz)z|>BkgBydc>5uF4o0g_l+qp%F3VlrlCfqBd7f zAx4&zQTZa*-=~`!yY_5_XjdW=A8(jjw?NAmpfl_dYt}puC@I6bU&@RT5PP_gQwZ`B zY>>>QT*|!Kgjxomf08n6@#7UWBWiaQHfa?!-ncknI_Lv7igBCk%K;sOteyE_b_`_n zx1Y^_EoAh|t5%n_y5t1EQ8PaFw{^OE!^^;#UiB64*;ljUAPnl~5U-sqhy*fbj}VI? zIh*GZ11^wxu*Wz;ZKi}_V@E+_0!9yDN!e_Zf@U(APE;ZohTRgz#_h>929(*&_KbAcq! zu-vdN8eSvtnWT6&h(S3`MZN5yM<_|c;Gxu~o`iM&0pLxq9HMsSX?}s%$B{4uW zsPXo5{NwQ!USa9zub*X=!t7Qnf>;U>MT;LC6E@vDI4u9hnPs6JURhSw*pD{#h^|Lk z&)M<*gO!8o_g}-mKk-stQ^lwz8vfU@deiul@Fndl+R*5kjU|<_-^B=@qg#10r}y=# z^D6tM4Do2arxe9sekhnnz%o5{(t&d54Xec8w@39e1Gh*NO0$p84Eu*lMg*fc>mgxo z0jRC|wDQr>bL}mFsEt<<0UY>Zz&*^LPANpo2aLDC4CP5{CiKcR+Odkk8YQ0D&t^); z_Tq^Y4u*R=KUEG_>RWnTdmWw*UQbW2Egcc%!_-Q6|7&>=&Yv`UwBgLF%GHwco_ zARr?@9s(4=OxBj=S(UmQk2BhmpytIQ+Yz-QsR}mu*>nWr{l; zQ=P6ROamsgciO{9U$r2M3YdDjD4J`IggDaFhlPZ^NhZ-feJ<+Zb&mP9W(U*H3rPge zPL@)dqNn=Y-GyDb+BeJJF*W$RNx?8Adf$Xw<0rD8#A4{W(>Jr}cwecN(k24gA|u zC6XNBu9Stnq@8iATkZVbu2)W>!+|>OL$(TFBq(tg-AuJn)7 zF^SZ(!|d+P!?W1}(_&yZ@eOHguC!&#t;wis`AKTI zP|*6}$c$ky;wCaLWo45Joj-f=FqLdUk#*i<=oy-)m11vY6)w1+=T(b`qQjDWMZr|4 zlU0(CLeYE+O{tJg#mVT(hGR0%kB7Z2w|#{9IkzS`<_V}7$2GFhs&u$Kzqu0eu^OdD z!wz)+grCa8grO-+`B#{po1ydG%Cf0eXh@9cokvk4WeP;& zM3kyN!Hs6`qdyby3aCBr^UTnO>N43UKk5J2ZtkQ>AQf&#W8biOHB_LHp)4`&K+?pT zw;(@_ue9wzZrpxL)`7P)L#m=`i2CtMR1$lu*8Z-kc*gfmP9dd;ekyEOctflf3U+~u z&t=X*x;4SsEN}`j459P(1IDhgm~VoTR}#53r)s#q+rr_a6vYU+rRAI*q;b?@X&Qes zni)Wn;H?J5`sSgkRa_c#&WFm0hl)LEVWJ+{YGQhx^SnD^5cO1t)2v_IWG6F>vTB#~ z@ilt7_U*3Ll?pnWh+RNh_j}#yq`8@F*E-UW4+OpxXEDliSB&(Q4jIYZFm#OJ8Tn;M zXY&W3NK9J&e3loBa_Yv5bl!mR`jTcxmzl_NJ0`J0Sgb7?fu717Li1Pr80JSYlZ&Ts zQ-G-teH}ke>W&~?lFGC+@HMiUsqHjjPpGe3rL+_&&(E(=)wG;vd|ao*H5jxpWl3zv zSQl>#LszmqRI?xmu2&^XOx3elPSEK$ZtsroRY3G1Rjj_?5qWkR3zrF1YUdhNl*84i zxmY2p*L}>+=nH4*ZScO~Vdaj~yOm%)*O+R>B}ZsxowJF$G4YUDDGVNMsG@p{2der` z;Zs^#)ziv`zCJMs406?WLWDg9wVdg-;fcz#e z3q?CPfnnrVPv`rn?B^0(RSK6!Hwx=adZw^0V%0;o%+y2FxYV2QkrXmURcq}&fK}$< zflyBcD|+W02Nl0?eK_@E#z9<4}-fo6lmb~|xU^kt`w;H|85A3v(yWI<;E z!u2X;eikeWA;(G!Ner!DL=F=oA!b|%{nV3N`S3&TgFb5?Cak0g)`8lKFLoLW6p!FD z1tLoZgvcVFfzzkIJ>~8;dhCX5m4W@W=&d8bdWeVO23CLfdF(JZ^5%H|>%0vKgIE8&JgoJAxP+%;jv z@aR-fhv)IP^^&C2q?ZITl_MhIer<(ss`;=yH`>kJ$?(&znLdnpCT>_9Ez+!IwoKEQ zWILbXS-z=UvCy4B@l~%7#s2Ds%3|opLwjWvEOydeQA*UU3DAt{ebOuxD@kbl;ng}8 z6)|i``*K{{&E8a;bO%Oen)^`FYB}zL-_^58L2Xqx{7=>XZdK#ZUQfzrZM1@+X9(o$ zrMvZBO8Vl8P)uAHt&zgAIb*D5(g9oi#Wn%XI(4E}(PMl{x_l*iB}Tgrr_vS*thN$u zd=)GTHOMC)JWaCK=^ak=N?)Os1*&{rWlpK(*M_e3BZ+28IsNGjd>xRHHuVsqot z;GwsKX4AG-OTF9X#hCZe*=Fs6GSBdn-8I6`DL&S5g5?4l0H8;{{|V&(<)$2GGe_`W z&%bHYdgrjfj_te6RPl+lsJFXzU>{UA!K#l#8&~n!h;hRJ%QiG){w;Ls`C*G&wun5B z#N08U_sfs}V_tzom+`O>n>fDm4)pbBBAq-vE%tbyJG92QtcN$knOe}GT^w0_h_C4|~Gq_kM!f`ft z4C0$`CTcF#U!J-kHF!fKn*TvcsLau>C<or7%Xg?b+~`3u2f5k zJB&?x8#=yjUfbz#dHD(&yy6t>s4d#7xA3;kLGm`X*wM!EM$wINaD?J*-*izF(K1;A zG^tO>T?WgF3{dW&D)O%75XNx;8*-ue z?vUYYhgJqkXz_TIaJZE7=@UJ$aev*@=MGqEbHmA%%5bc9g+11!8L`YN2~{Scp*qSZ45_yVZiAtVE-sVdK_e{5En$oiP`xyK2>_!@FoyyEtgKRqXqbL*JnF!Fi z%@QnM8G~;&ZcVPEFjDDK$N>%Wf!t~e+CC@SY*}JIk6^6zf3D?87}TZ z_L|%1lrwYCiOR6tC8+6qO=AjPWUOfZQcL#>-ZB-5Fkcp}_+%GqO^h|8x(z5~QS~gY zJy)05jm2mR#H>Rs%u*qwux3OGn4z6myDZ}q*0zH|3^H^SpmFVsZ?zt+4h5V#Ifz-{ zQI~AJem;8J8b$p&2UnXY(6>V`a)W-eR}+7(OzDG`yN<0g;Ii zSR8q;!Qt_=t2olhecrJVJHrVJdA_u3UZxtws3|(R6KOz)`Q?5beoHs`vwBuWfFjPt zU|n_+eg=N;t2JT<{fcQL&6t8TQC@{ePt!s(1?Co`ORGYJ9>UO1n#pfA1Fog8KTly<)L}a|F zP>AI2CmcIXN|MyC1EZoBu9&D_(P0tU5n!(gaqUDP^pY(BAvvXG4odWhQM}#xnOA$BU4v0`1$d~(y<1St!!VUY z@kQ-1PXyBr+zh~~lYvKAu+k-ZuT9Zw7rq=r^*?!{eZ5YeQ|iH}t4a2ChcnrMD`P>> zAR1NP3)iv+;|U{G{KMsL;jxl|O&CE1wDZCd;ka1nRvV2e;k{d znxdI;Zo{7GNpo{@bO7pam+p|6v^NhFFzXoh7~Ob%#IZZN;mV;)ew(<`Lyhw7^#kRs z7$IPCr#f7kp-@DQGt2pLsMalJikUkdg1j44m40pTN+m55r+yoyBVFmdje^kIS-$YG zRG)O_r1(oOUc=bFI0KfCb2Vpn94}@CI8*!rqx`bQDhGnQjJibjc8k@R{H?)t(RS{# zkGX6tv&&!?Y=-ITx4hd!@)OW4zjY15OCLXG8GBBvw$l#M@jPWO`wpZ7Ktd+>sH)8FE7^Xt|m)iA2W5arr zYEi;Ld~e+F;j;&2eA z$9@s1Qh|Klj+^Ho=J1`wZ&RS&`$|yF`$O@0!Xvlw4C20~SNr1Ys2^*wy%@dNM5|0Y z69l;0yeW{Sgw95z0Y6z zn#!7HEzuhDM4g&s@DLf2Sk*yGSz@UBxW;m#ln)(-1+HD?a))N9mTW$t2Z4(V=-Nx= zwjDFwh#5|=)9X_bsI0a>grd6YhI-cW<%W$86*K1V^d#t01y6JC@V(G1JjB!HQ@V0E zMsMIr6SvLbKOwa9K#LAwcZ8*qT=-}v|4{(pHN!HvZQNiCP?=Hf172{TNDLj$&)bo; zOWTTOUwC7{JrBdvVx}{r4-ZTRZV)Mj&A*$=o81wGcK+hh-sDkoWT2n_zGe4dF6tr1 zlky_rgOZH;!2_8ELqiEuVben=o^=Oqq4l!feTq`&CQWRV#Dvr^FnUYkGTSzx9W|0i zA!n8a6Q*)Aj=mnf{#?7ce!q8m{drc!vPZw%B(e{pm1!`4R9^kTLm}}Su1T>ohEM<* zfwQv>|BZ@5u)Ozv8LsvVxmURu=;Oo6_9JITI$NfG2sHCuDj7D<%=hFA)gy~~=!z0+ zRW3A4uU^6z!jV}ySya^ST4~1|Utt6gdI8j;8>EQn&m!XV>fXMd4ENAN;FmZy30|QF zNg&mo)8t2Ih1ZBF2|Gl7%&71onge^SDPRG->UDr>0%a+*bzO2P3#;@w9^;Ix8gLAr z9Gp^%IX4xuaVSP2;LNLr^Yi-^jfUA&(h3B3kU_%-C` zH&ijfrFtTPu-t%0)ej@MpeDE9x?uD=vGo-kDINzbMFtPjk8hwYJx*KAf6@CenUs>R z&1-Ne5dOVGZ%6fG%O)M`7vx|!BFo%rA;-Fuuai6K0rCr<0x=|6YtCYCy6B(NK&v5) z!qSih!52KpCEzQz|H@ZB9=+Nqp|(nPdp)ro{HX9rOXm2qR$YYdrnyWuT%Gz#<=GMP zh=GsY1;)(Y%=K{z&9uCZ_)^x6B-pB4)m6g1A!qe`V?`gLiW`No9)tzXBhEcse&bY1 zX8{+H6?d`2-j*7{0I0?svs3$qy5Et0?7j00ac0{?aD1KLOG&N9+1J9>4%DXBD6lPD zlw^tUbdUU*V!UY-hCqg!w8q9UdACcdByhPa1TKNq^{RHtjbQNcGNSM6hntf-uLTA# z#M_;=@!p|T1gl=YHFV;$UX2ytI8Y?uIZ!Y}uqSQC3EeQ+Eha$&w~F$dv#?G~l#B(l zm&xeAqq2=x{~lr(yeVxoA-AY;+&)L}V@gn=KI|t!-54kb1ZbCt zDkkl4A!ZayUwtdEodSf#o3kMn4p$Ln`j`|^xZ!J?B9c3XI90y@fO+u?#oiMiL;g0SBqU;U@FNx@mP{4;lmVu$JM z!eY!xI)c*tYYpRulw-b1$VSH>nKTWBpTw=|nREy;$h7aAYsbhS9uN6vA$nBP5?HEn z&<5>?Gu)&LzdWoFT1Q4uhcp5kGNr7B!T?*&YleXlA!v2ES0Vs%0xnsC0dzTXtpKJtC9HRIo0WApCEpkOzJIQ`Xjrnmj=Jc*nd`cJPJOfbBHssI zJZCl-;2SRn2r#5T{EmeN<oDc~zl9QxqWx~ZeW#O@+&)!|3n z!$j69^}%Uha6u`9LsbH7o!@&^$%cJ*)KF(zGWhMeY`nMCh*tiV67$c>-p(_-(FKr$ zqi~EBdcU;}4a_Z83wZ~v7%h8I6Bw*o4EvI$po$SBlf5)G@!?48Y*b}>u3!yEqY1(l zhz*z4!cklx4O%TW79lLUET#$rJ5rnOvv9*d*j_az|91U(nJ7wU*BEi&+jV^%TT;(O zpSAf{!pE_6qoZ2`Bu`>z#!Q=TJ>A;Wkm{O@GX&L)in1OOSuNWyDXfYn*Am4?t_Nhc zHWk~~a;%PZ>toZg^36XPwY2!qf@5&#SYUDUBrPedUH@$bU6$axWmD2H zt@tww0wg|62j3dq3d(g0!XY80%I)fDtsO1i>tWNzl!d}YpUEWmu4z2sk}li|OqXWA z+YzgWw02BRC)UT$_*?4A$D-3_nr)4gD+}Bm3rCcW9KN|Vr7#P-26iZZeCF-E%lmPv z!ii7!^Quxk3rZTB(APZXidmF~w$ygxtCxGgJx@&m8U=OrMLqG|vw^Q08$& zee^-`(aZ19KQ}bKmbEh^umHdj5&*#V$0to-kR{0DuX~ZTfIkkJZZeIXo(hdZ34#^} z9>5gP9#}9t2fjB{!Il(Jw+xMxD51vbqyM&gAu}Ac!7LI!V{8}O+TwoLmgnQ+qJSs& z#?2sxkGnkNMD8)Jme%x6APd2mIWtc`r)*xEX6aj^@WIEDar7+CJqs-nxTBj)1(#c3 zH{SfZk`&}EHSDm{VQs{-1n@MGx3ujhu4GOtr{45yZd^jl%t0`NU5-m=^L~HHn`1Lf5YhV;%&oCV*CzERmr+M^?U0h0%`bvd?0b?Al9 zS@8Hwe7N-yVW=FzB-?gD4~d1p9&ZFymNU3rUUsmowbL52g`#FfSQ@%~omnX1V!w21 zs{N1)v)ZCRv2gLD5}U}b562Crhgq+_VU6?JXZ1p6L?e5dT7RlOWOCbg2w!r=RDpm` zEQ7OKuw8GEy6L|-s08t1v^-b2(zuS?$CC$zug|0 zcM(mRb8+Jr&SJ4lau#gM@wt5v;5YVh`=RO65V7ltg$XE=l1?f-f_=MMNeqL)Z9o&K zcvdc#$Iva;oN?@lK8fAr8~ax%;K$_5YcHiPKheAxwDc8C&l_CFaS3I{ZPj|L$6=HH zj&qPiJFhaCv5&Wr1(|+b*a|Cok7>OaXq-YhW5iSuj$e}_x71_32P(tQ`;^y+r@O70 z(e?2<#t`Z79lURp$K;_2b0?96^SR1VJ=`o%U<8Jn5*H-E>v!uQh@2YaI{zOR)Pf z&?`^~h~>;(oJhj<;sm%gVKK7hq5Ig&L-_+cWIVy{C6UumtneZ$O?xjtA)h}lfAYL< z|5>zhRo=v5Gj4vkvL6atP&GSmignP0XjK`BM88Hc^WpGQV7_;OER@;nlnGs$J?NfZ z{OyO|40ocLZ&+A`wncZZviyi7XC3ky9OlG6uoC-<%u`P$9=rhQ^O`WTA0>ISrHqmfl5Y@rxlbQVC+HL>T4RG~>XFOMjc|txJzgGaEbxva(v^!~=6kEF6k(W{=y~nMQbSDrkkwjZAU%dl zcI3kfA|l7|Lk2(BKfq@P(6Cuov5*r;lI)d1_leHJj3xt@bH{cje)g4*)+cI3lzU8WLoY^Zesed1N}1Q z>=Ug>`b|;^V-E_cb3G~lC>_U$y@_4|Oc%7Oqt#+&Hou#%U-tL+g-q?+#*a6o=bNG* z6wRQO`8_}93{^~x$pskRJd1QzIofT(G|c92`GiB5BH`$A^d7Mq^ErI$R`%Yh2ST@` z8~ti->k|o|_i2$qtb9<}<@9hnWMiFGL1dA|1n;{tzSzBxcYH`VUT~P^V5J|he%U|j zyzOS?&FbUDu5bYdOJ?u>%o1Y)M?<3B2)qDeWub5;bABNOl6nI-Ew{#He;l&+LA*7n zB7eZvg}R^PO6>Sm>QKn$c4Kyrt_aG73FdaAdA)|ed8IM?BVl%u3ZOwU?!_iX;VAYz z{wfg#iGvi#O)^4V0c>M$2XHs>W2e=KcPuR8e*0Pwg#ybRNUqZE9o$@XC{649a6@*-V4RvCt!-5%*a*08xwrLTo~lwp#$tKGRq?> z|1Me+2eMlm?2f)p*aVtQl}RYpippHw{@^9DE-K*C-NX(0g^Ed@k1DukqU90G z&UdnBUmdMVfq^ySai~71M}9mXqIEmE6rEU5G7sk4Yvm{mmzQ#vHA z*oiuB&ue_~h)oU@P=4D380?@+U@VmziTKWe`5A7eK&%b}t}AzKV^iFc{9Xc=PgL%t-DRjM|mdtdh)FVy8{k*MzFB*H>AnsR_SbCI$+Gmb#rIZ zAOXL1hbs%Quk7O_DDd?jc13f`#T&hZd`pS5d!&S9;`UNDxZSnFXR&! zAByq4KKC9UPOR!gMxz%z)Inpy?8wU9lq(To`p_rAe%O16-u9H!o=Zv9DOGeSlN;7t zQq~RjL{1@DAG^;@#YhPHlwVwRqBrm*Ja0rUkt+n%K~L>lqd8mJ|2O?@ME8y{TfR|4=T}qv2 znwo#RA41}=JZ58ai8FqlqXkHdB5Ai%Tuoo#BIY{dJy<$5uCJ&=@|}9uncc!Pjw5Lg zV~Yk7zu>mA-SD;@arsgrP^2%v{5%o8AQJJ>tau|@Te%}!;yV85Y+&HFGya&W)#1>w zueXDBC+i+NcKth1&vmSCl(l45A{rbhs|IWc3AN|5i}2Q0YnM#wbDk?`pV0T`@FUoZ zRVOG6>)*jNqn7mzgL}FTf;jZi>utw)uDA2ilu+~3G+`^YJw)AOnB6PDLiB_#-^hsA zYG3UozM&8d=7o1JV}EV73dTavTRkk;c%pwhidx2kR}$pgSO$X-sm6Uh-pa4>Y2Ja} zUFcOaY=|SwOWu`k)N)z`IY$n2{&xNbVZH#losUam+k}$)A~Vwuh(Y@zNp&ORT^4O_ z%r@}cM#Z+OS`(RQZA~{OK??$5uQuJV;mNe%Jr3&jDTC=6q~fQr&Bh7#hcVcUug)Ky z+O!}&u6eY<8FYXGmtCZy>8UOsfw7Qd#xQETkt%A!MtBQrHo+rNZv-$Le`Bj>Ur~DF z<_@BBo2f!8T*~rmvfYzud3it~g3d)#rARGKgEVbo-3s$L{>8ahsFsJc-|3wEOta{l z^Di8%X;%&fgjVlbM@*1~;=pft=k`uHSXc9<`o1B4b#S4Lmnx+XJ!t5UVRHc9@$RAq6+DEJtfZO$IqTZtx9A1r8?dCT6GOL>2 zpx6N);*^5n+QVn9U}z^({v=tdCT&ErkdpnoNA&6OC3e#5>;;>UVruW*6rrPvxPv6ENx>d_7^Ss!XX?ACmflHfgvV1K;r3(>6v&6Qq@v zXzt=}Koa03BM{*QOP#mEJhY6nZE{uOhg?ya&GOzThaQMKMZI{_UvG-D+up}nF4@M1 zh>^~vugByKkHRH%mHYl|GF^|tE_b#aXCeV1Lux*k@S(FMuVERh&xqGj1ush0_|;_f$dsg7f!^oDMFM?tP-Ir*`Qs#eYt`N%mt*(z;Tz1f`& zigxBEZKn~gyF-C}xN4o#uXnxbJ*FtN(q;hO8jhdbjcdwbY{V8HrbZ(!g7GFyVb?rr z2Fm*u1L-Y3m$xNctf5gcp7dmAH8`*OeDszh8RSc1D%)ze7I@Rxa}Q{JDE3MDD5>1GhAKByUei) zAVCiIg?fMJZP#-Wsg`#gfGg4vm;L<q=oYadCxOpdZ6z!9J>OpsD;w?T zOu$jsjK-qy=5HpaT0Q-+t+iy)#Ux&fr&bkaZl;gVYVR1C@>(JSzl#*RW(8TWAO&S4 zf88Ygkd@gH6k->V_>qtcN8}rd3kenGhTcvx%6RoV`i7(IRs5`jV)*w|&rPc@CB}6y zR}Jj0XFfWGl7Hepu^aE9;p%!3xpAZn!v&51&2+Wu9mrszRtV&41Gpz_ODj=XCpXf0)ALA{pV!RmB zeR=S4e}*s@e_843<8ZISyjy8p@ouX8fRET{Im$(sD*IgRoJE_SIN|kBuq@&->628q zm;BvS^v)Upz?w@CH^JBNaksi*>3|A;X{n>wgYb78i7)rPcAEAk%p%^9j^Q~s^|?() zn@|~5_2SeUR6825pC1hmb710`ZNo2SNui+(M1>L2Ef+PeBgCst* znw581a|jtqJUY+g5nDaCY3_Po0s6LBY+z>MED}Oe&FHoVD6PB9XVTgjL5;+HJ3_ow zIl4uM&PxUpK9aEG|Kid6^*&?^d2ZHobedA0tqF`ei6@A?r?wiXj=DLZ9^4~1eGP8a zyj9`yn+Zg8Q>&Q*d*f{m&&g6z%M z$D#G5&z?@H$v~0TE$H;1ibhvk6bKK5IWm7&TouD!)6Y`aOM~^Rd9xKTUp040gvS*G zz&&e^d?t;ZOtcz|lO0{lpRE}_sSiswBj^ch}%qUwH&K89in3jUF*U9ApB`(p*6dc*KL4k|tU%Vqk zWe9-HRp5oAERedScpRkK8y}f*t=798x#J*QZM!87g;1Z9JS^r6qS}q00P~`l`vEdU z8@f6N7#j+433tWPjPebcF@qhOyB1EwpQ+eFg5ymN-G@&MIoGB(@VACc zXU8l!UQvQ_KD`Vd9u^4Ty3ALrn(#A+0<2@91sUHz(LITsI*Z>riQ%g9_a<<@h0ey@ z_GjZg@y|a?F?wN$hJH;kGM_Q{cuE3h^g;aN=vVn1nwKTaIegwXSeDwUEyOs2l1TyS z%nMV3MORnevZj12Omc4Ry`d7PtXyA4%Zx}vTS4}|Mm=FKBNGR&)s{=Nvi(0#5^XD` z(bB*TMK1LtpqyV-)hsO>z_exEs@2RInYinwx=^q>4fvf^cD`kLJTkFf#h!P`Kh?-g z>&>$oy@DMxn=q4VVEoxT`|_PlVg~6Wb#bjB|7KcWxk&5M zxH?NTNII0#J0+O=T_v5)@(2C6$`6!Tf%6-Pd9NE`@fU|23{`K*Ldofm0*sib>>e~hx&n@V z_Iv4)_H9&oRi{FgBWts?V+Jw0*9}t-hc@C?$hQfqlpa!NdsW7ofW_PPvP4S{$|_;fef!x!zI5r!rNYJZg;HJJ^3R0Nmd$yITQs#O z5H*9QZ&x;EAFP`3JZT>w7Cqeh>~7FNSGqRBncO6g?r_;D(_kdI71~s%L&A41Xxnlt z;KlN7>^OE^3nq9h1pI&~u#=>1A&#_+B)buU3}$(|jr|Q}75g z|APMxPnCcL1Lvdabp@wgVX|<@n)x-ETU$Ah^LiQ;W~DmorcRA~s8#Q}DWwF}h%fCp#Xwx7 zCoZ#ZzI>;Zu}&bloVJ1Nb+JxK-_v&o@##z4n*sc6^j(Q2{B=~g%DW(Os{DnH1?RKa zS9@!d3ESK}Hxx=Q7X{m)QXduuUx5%>O&tp~k3I&75cpA~m!C7}4OU#6E!_V7<;QLx zp4 znVa$`)yH*kgfbmkCv~Y_b2}OaI@|J>E5st_rsh-a2b=OmZ)Cn@*Or%+jT3l;=K}rT z$nx`d$8^4G6%c(IQ%Rs^Vo1rF*B^Ee>Ta|6WSZYlboFW?(kpttxVW($w`8ghx9#$) z)LHNIlX=|TLAB_<;9nWv}h}tD4SJ6I3O7>7p z!6~+=dtSg7$YKYQEB~sP1sffr`Q1?&s$6PMR!puBrJseD=v~Q&ju(6lj3KhYN76P2 z!BO9z2`!fBcqQ(s^C4=nKl9OL$Xgp;V=T#h8O!{l|1{Y=iAh*jJs;lFgQ=-cdfhP znYsz9uom8UL!D&je!7}4i1IXMdT?@+&Cx}pKru5Jh@4ul z8aezi1C3mU3Pa<9 zf$WfU91Rm*ff;qL;#SzO&q9NBwObBj_uDFPp$s6*= zTEdIqr<|82uld|v^x)m?EQ#ptHOc_)2PK5X4XrC+pKx;0z}Ak!s(htNWDT2c#)sB` z45<_GMB5#bg45mp5BkH1!_0Ji$Zt@fYl`SQB;Sq=BP=YWfofj)Gy4_`j>v;c1)o@F zf4|lt$fv=sDWD^?E`&L|2q5lfV=F=y|LkBCXky9q#V_tA3|rD=^sU4W0b^A$iX zeukwwKA1SKcj;Z4Fkgwp@W+AOro!;*$x`OXu3%^{%dNf4u z&UXJc)oD6}^Fx|v6e}UOT+t2DagDg{d9Pa8+)MGzdR}N4B zfE(mmY`}kBuLBtV?`4XjO0rUt>Kd$yQvVm>rB;A?ZS9BW_UHhB5%XPyA*la@pdc$Q zsiZFXAr&0WfPX!kJKpTRT`PKXtLT zxAywWH&lN7-hZFOjz}`13I+f$1zASXpNO@ae>;hP8wq3&;k7msbpYCVI$Jy6L)VHJ z-~bJYa3)BQ@cl#wKk_>|O=l2T0}KL*LEdOtJ6pRr{ul53LBeJFiT{x&0AL{k1rYg( z#BlBpjrl)J$-B|}X*(orwb1|o;h)$}RG|Jx3(A0i<{4Um7*JG z1#JWXynw$ONuue0=nH>1ogZwM#0Y&uki{Q|BzvKs*mA!8sptJkApI~M7Mj?*Q;2$R z!ru*~erV`lCtIMssX6dJ)rL6msf#Q47t`5$f2I55QSjEVceM@;{?`|33W_{eOZV>0g2J*A&*^f8_o)^M%jd5Wx*bx@+ba_XA1USwDpg zS=PZE zaRT^*A(I3O@c=u6rcZf^Y4EsnjZgV z8Gz;x28h{y4enq0@ahEwJRJi5=qK<=q2G)jlA-NE4la@⪻DSAldmI7D@U?KnTkc zB(Ru%VyP6nmqiT(QOSFNwGoBDeh`uAMZ2p6x>EN7D}mgVO>IDC_t^d5OHVH+WYFP| zK{Nh@O|5V*wuTkh(f!xB%sh{MP$9VPh<`se=s&ob%75aQ_eYmf)FI_9B)Y6n?|REu zwR_QJ9o!skLE=C=yL&=cWu8(V0=|uWwM)wEZ=%%9*Q1DP%eE zQ10SZxZaOz=V*2huHV}W_*96&s}S$jtvO!5&3Jz&==*b?kBJFzA41Ro(L=hQ2qK={ zA3pbpEL7FUOJ4}6=RcIBJ@8&oNjo=d=U?M$o~C@jhFBT+ucwpz2RG>XZ{6>&mP8R` zZwdncn!%!9S$qfb){^+HgYSg>p_=@Tf|2~DuL=^LJrC}tX`00QCjp5cSqEzuYoOh~ z+QVOyOtA{@{r5D@nErctQ3O8yeV_)FLw7hp!Uzi;0HFVgLNWXPbOF9cVdBfON;x3b z(L=qP;GhccMb);lwgcUtDqfsJ%@{%g^b^6|RPn6rUTkqk`=>y#wex)zZ+Nq=@MCjB z80~JVpl`fi0URCeK)`#D8IGWYAzdAS7xu16*xT<#76n_n{-{{)i73-%m)TZ`9+D#5 zbw;<&KS^+ZNsI!>!P3R**CYztB(FOlBv=S{?f#dfGdhz`JD%AD9iX5rZPxBhGj*b85-zqPETM&sm+S!5rvi5JP zqL(?3@f08|ZHRXV4|RLLU4YK_n7i$mbm{`4C+#qIwdxfV%>8ZQZ@Yiqav3Fbh*9)$ z?qbiu+>5RGLt}rOwf!2qiNG+s8xox75Dygm37Zq`UTh7|ry%qHWe)?1czNJQ!}awa z4Tcidy-a_F=^vGNxX+Wk4%|b2i+j{_+$UD@#n?4*wzc<;x zc1!=jV4}C|+mt`__#XyyC%>2H|FlcL=|PH1krY7?3U8FVyKZ~b_fq_*5q_`CRAZeo zqY(Ihl)HN+*$nr>%UVdlvihZ~2sAIhs`0ujg{*Z7@pLWcGcv0d% zwjytNe`}QAr;}l<1jKm={4LVmPG|4&Z{h!_d_VK*Nu!6Dec&Hu0-o^iBkgzFY-RXi z$^4M9TM^tHf28=m=>OS^{LNMbPe}lDG(=1LVD7pNzTzLO?yrISngj5zAi%<~cPpGD zrQaLo*SLDNj?(Io!OHw&8;nc!UfdrEQ5|Rjg4pEO=nm-$qK1%B17PlsdRpW6^8ahx z(U%4}T0b5L3V+wko?5@L21HxT?okzMW*AY(VB-+)wvYI__Y0TnJuO~eihr*Jxa4+g#H*(yQxe;^s`5{^&@sE2W9$Wv;pYLx<`>5#34`^iR0Zf%oDaaY zuIL|~koXdc4MIbF#u)Z)W`m2mm*G#&E>?yeiU=UXM@75~t`YNl{rinaZZ1BqehV2j z%s+PPo8#|K_J6f9W+2CVlFq^t{Ky}hUQoz)7k4K4_ap!P_VkaV*!B~tI1-2mbTQoR zm@d|Yuq>tNyd-=TMsxpwUzK(vebuEqc>ehvNaQ(hH2S4-<(+}rj8<-gy5 zh2QpTWJu>|_s=s}&|5MP7_x#3XaIoXPovcOJ=VX+3IscYz|=}0ck6%06v!O-Bj5k% zD?uK=xQ*;Wod3P}kJbrNSV;dw&DH%sQmcTi?H$49APX}{!*(y}(rxmjH$>+~Q10%K zE%yDM^aq(F80-j!G!Kxj!P4`;lguYXko!Sq<_4MBlb=XRNB+l|X@KuHqVjOKLpQ`3 zP#`KO@)K3a`2R?y4rzz%K=-m$bcYFJLnd~DaQ8N!{K@~3?MK7n1_a+r_njjZ4h2F- z5791#pXl)aUu)ML5Y_Sg59w6_F^UZoqgYX6i?IV%EFf6EDmmc}4zD10s3^t~yTq2* z3-%JR_kyvaSg~M@prU|AG#V8}qQBX92YUN<-+N1b{|V}R-cH-unc3N!jBs4?BrZMz zTmyR4E>58oL~c5BeBOFsYqSmC+xE*HMr5wkBnWoS$1v2@#8q>3ne!*B4eEr|fA;dF zR}^kCS?UxVswkTcO0=DB{PWHJ?O|2b>~UQ={F-r9!4eF+@332yP+!wQU}6_DxB>52 zbw#dL!x>tx76kN8_;cL@01N`a05iap4~&3ZqC*}L0lyK146NPcPt+!CZG|_>zAR=| z6m5hKWz9+=2tnY4At%nY0a?zo!_A@9XGUPICMXEjJ|xx!qI=m*JG>k2QaLL|+tcZU zg*dq2Q8Dr`Hh|@C?mlf8vHHd-A{3D*y(7E<+3fS52pfV+S$MTVK9a&wz$mtxHH0a5wrBg z=SLnx^(R-r>THIksKkf`A&YpN@_qdLZnwbd@-1;4nNo!j&tG3sr4mGx?egeofPk!W z7_K9!RoM^)J@muZn?vV;D`^N(Y7aA*8NTd{L;*cSeG&q{SuT(4Q3kS`GrkE;Y|MxY z=W#)(vAbTT!%0MzZI3&yj?Gzp2}-L9g3a~YaC#DeW!mFAf?q3EBluFh6NJ<@IsW+o z*n$gh*Q?l;RfIZ&JTg)rKU@&-==H$bZ4WPzRJ8@3k%93WSZp!uufTy@n3==|%1 z$Fb8UF)qZ|t1P$*&zvJl6g9uK!QEB)X^d#W3VEa;R_@{8$MsNZ*$x*`qnYehrsKN) z1OdBx-3wR;fI0_!ar={))ld^Fh!tGC>*AxLg)u#SW_H;Kt6a(BsQnT~Oq2~5#A-4-E7=aP*7A;6 zjw{#=!H^Iu9FFYS<(k<5;Lh6Nn`vA!D?3g7DZ&S+IwweNpI@X^C&&J@&A=vS|2O z5UkDn^iR$h;WqWne_D221RME7GKlsa*_;uO^U;D}7N5I5A#2P07K~v1<3yxaZE9VY_keZ47GK!U zty!_qbqql;%SHBUeSm{WJj1))mJ#erT}BYG#;C11AJB7nyo}|r4vdI>b`AS-iY)zYWyVKE*K_;!6K znsG6voTzAk7j*WcZ#?oNyj|^qnh_GE>_zdKFWFi<(sTpCTweQPnUzr?7A>#X*ko=$ zKpSX}hm)fXtgF)rXU;cGTa^kb|1Zyb)EZL;J_=$b-3xto6WrocUdE~II7X}hX$fBc zCW+@z{%#iuxhFg>!P5S4z?U~`BCGe1$H_GcIK~O@Gbt&FyZ6HK zoOn_Bp(%`rLGoBs)6|{k6~XeZ@|O40Z04O! zu2PGf0PJ-x8PN{3Xnrs!Agk4%K{Ze|Z^5akvdG#na1JH`bC? zS~^-rL+9>2Ohp~@#T{MA>OK>+a*fEjcx(LBqH9E6D%LQ<1xg5}Y$=Y|$5(c$`;^o3(}2wTOid?fMT|w;vW} z&r9vK+{S9@sC+Bs$TK#4`~;LhDd4%zzB^fsPNz3SiqzlBkE@4u^ z7Y7`*`sK1keo|B_Njq%KQrg_K(*inmwN5u8K9)BO^l?FzDv9u^Lu~QZ ziw+WqnSSOHuAt0l93iu2b_I`gaejG(|%nCR^gsdLMCO5Pa#3v6tQ_0AfyLB42(-R1t#EDoYX%b(R~baf2Qp6YL9-3023f-jH`>LsnQ%&`?MdNdbY7!g z;{Y_h9Ihl6e_;ac#GRA<6|uD8+IJrYPe=pG3JctKubIoH|6^j+st6TDT62_R_nZc@ zDu8Kg26Jp4n_@sO5Qztg9 zbe-r&{#wEW8zR|{tZH`t#1(*a;z4>ZXM*(Cjn+zIZggVxP3Hh2$^!Q%AuHKTPOT2m zN62Yfv&Z6XKV)!n3;c$eeifTP(V|PWWRQu+u3k9<07;fO|K=pKZ8DsYkZy?WYquh! zwEc+wc>fwUEs0hs4Rou!`gz0ql|VO7hC8hA^-^(kT;4=*IRgs*$vF-jJj$yOglu3F z6E_@63{JJcOL>myhT}Fd`$xK7lwUSA4Fwh?hKH zevZh>gL~Li9~LL%dHsTw>BuQw=ap>l>}3Kf9n>SHB|}CxJA{N6Zh>D>mmOe2YOwe7W zjr{5JbfUg^oMMY}vzaU#bnkE@ zzm-!KtYo?s-eZw;yG+cJV7#^~=JvlRBL$_G@0Q`PA`}7`63#EaATvH6XYSJ9-eU|( zIg`@G>**oYrC(ZBMNX;65)a*~-)99IrHG@x7vf&Sd+-^e0oTqhQ_6!3lXYOZ23@xC zx57ukba?RJB1F-0Sci~mneD-EG^&Uf2q!&afuFN0oQg4KEO{QHUG|766JvKPYiDYmE_M$8Po{=|eC zCRfK(^aJ6<-c;U2y=()$@bcYi3tJ|&C)1z%;bgQc{OIC^d!cU~#p!c1!33zU=>JUb z3q}-P9-rPf)kBnesTU&C_U)PGGt=~vLPs;&MV@S&DZRQ#SqXQgWO#kowt)78_oC0k znxbf5?3BC|4+rbd!RTr}?@*@lk;*1cnUEfRxgJ0U3tcv2YCob`R<6qiCXb6!2qZNF zcF)r^1CSgCTy;RF$vnRnFKrWFDmYH>T64J-OC~4D!Cs!3~vQcRT5d zbL)_w6cFQ(?l=wCKu3FF3~{Sx(z&LnC9$I`ZYuR!(}7TQgYGV%Dg;Ct6p`m2$Vo=C zVDs$piGP|7;jo78KHv}BjXjMpb+7`%3U&0?IM#+t&cxAyXWG6!}Bq599$u(7mDL^lO7GG z!ujD&b^HvGbdOcAMz=s$BIWwQuFj}h333fJ!_wsazsb}mP>yO4l#!fkt_|8&88$~Z zJ3K}$$S<27lW``^UbFsJ^sYh;{B$w&IV%p9Ij8gY>Y$>yn&1!{IN(Ma`o7GHBxIx` z50AM?3y!@P5Tz0P@$k;`H?=BwyB zPDfTKDSlwK?PAsc+`%+D;OE|>F06R;Ro!uQNwY;wa6Ajp)RK&1rX*yXiY>wlyTgyI ztK!Odu@38MzN#(K7q&Y+AgWFknQUV}{2tIKzp^b1U3b!g{>KSwq)p(~k)ME>RjS~) zmOaX-CX#T+Z!cW$4hPA)F1YcjLdv*aG2`1hs=}W+PH;~0!YixU1DR#|RW^_=)Y03g z&EuH3Jx&50^W>W zZ=?hP7$1=nd`5c17?%rZ(3u^6Qs^;80#C-os^O0m)*$*>4&$0Di5jFxWY?hPWRUyv zO8=@%{5J>jn!pZ15CvE8$~;;Ljf8t=i@!jSHA$RPU!SRbnu(uIs+H15r<2xG1?)Rk z#XFje%I&svC1^RiS#G{f8$>AeQfWXRBfCV)!Lnetgun;F*my1RsEMG$-7_ukgNCkk z#*4J->C8}Mi1igRp7tzfEIXTDpjk0T7@~(B+$zPie5+rW5dseUjw7BYXf$7(Dw(EN zN_dO5Xf$*={B6V2i-c+VD~88Y?tN^}Zt8s7OD4-z;&~YVCBj5I#Ydr*CxO-(SoSE_l!70P2-aj^|^7yDN3p(--j zHV;n^$OW-Y;#tMWZ^S+~l>W8BGamV(gj!Xe5wG$TBWoPkL#8|3Wxui}a3O~$vAF7D zT`iPPh?3p{-`o$^qy5AZ* z+~qjc6=QCv(m~c_+HUnPUR%l9H=JCzP)A(K9JbJaN1?Xb)mKNx>%E9!lWkHZT$9Ddy85|alkd25fm`3>#G z08H=NkqN8S1CAqKyxA7t>oPlw?jJe|QTKX{)u>j61}2y*T+2W)qQX}Sk9lqBjmA>M z+2Fyprn?yB-ipyTzs4nCRn8=(P`yyx*^dP$b1hEimGILUpd7Z88J^Zbwdh$%NW z-ib`sVqxnf2e5HZ2V9OlL&V7Tr)?+8^H&aQ3`Cnc;Bs6xKq|M%lxr^U_z+DX-Dro~ zqT3MB5I_;37BKVeS58GlqQZwG?vt*CO9@=aR3{xIgQ}b#m>fe)U)M;cBhm6%-;tN|yG8GJxgb?K3|v7s1S85I^dql-%gl1raAfuT^!Gx5W*=%?`%)>K8nvaz2jB$`Xj{HoMJ9 z1uru#0srO}H6;*N%e7HPveq`}ziqMDVsUZNWX>DaKG%htRGcIaja-6yYtZv=5Ex7iwZQHhO z+qP}nwr$(CZDTKc?Pb=!b?&RYbCUbltD2c3`7+Ymm(1R}TE>u<0s@8v_^-z@*jF6j ze|sSQE*TMJ0U8NeQ9AkmHUkH+_-ls2>1u%p2mrtZ3IKroznjSj$V!NcC@Is*h^8bD z*dEfuhu$VUz@mnsh_^_X>!&GW=Whv7mBX8O>2yp0sN#@avM2X`!AgKDLtHsd3>>|k zVJc-qm zq92Lw47ZA88XiRMF_VfA!ekPT|1xFfciQ9~M6`uAU+LUJjfvL8y=>ZAwSaW&CGTDq zaH~LW@B%njUXa8#=pFG|ji2F@mC9{=JcCSgW) zkRLf@ZbT48Sb^DghXU&_FV*V$hQKWR+U;1EGBm3gj$H5_ENHy`a3;KR62#Kmvu&$=iGJoX&VlpKe{LfCV8o?sx_Ew36wj6y3IPsJk{2lvSBN*S0M@oKLPC9$=a3yS3J@kgEQIS9 zA~0PWJ%%sdjsDrl`RdC^==*tf1=NR%4PdNAFW+{PptUL2aCilbYyeI9$IQbyzSoG#r;x#3joOi-h1lyY#JlI|9oh{EZ6-ZpZyW zfA@Dcgb3#n8cV4&eOGM*3M@eiL*OgC3%W^&ZX;&&9zV;icO#RvAz^i#wfv+mmGu}1 zi4>F3I-TBrc9;;8Gm})aUZ#10!2u}?y2=bA(ct>WWarL4`?Io1xrxxRbM8+KJ#v=g zW=REZ06I)|TEv~ju?eHYQbDGf)?t>7X3iAStbBYXLX1m;V4qybH&bWwQW9kXD?=1u zWipRI0&@PAjVttXgz+yYWEhw20MV_A1WoZNdt4WO3JWnO)6_qctqB{wq+5hOR?iHO zu3mwKJ2&0}=gWPvbpT<9G?sU)IW`otj_5d->SM6HXc50N?fT<3>H`8 zHZU@nm}3rBQ?Lpb>m#m$IsAz&%?)g+Bu$zm4pMaUf|`>hAf8f6i=`pW@S(x2u3_{r zbp6mi2QcZsLcwg|y8HWqn3C^unl4vF5roW&cbq=@Zu|1S-u!%DY5lkEcgT5z-zcqx zNh4jU!xdDc9Sxh&kB^oP4Xnn+6hOqJAr-RwC&*kFvEfG7ccNrRIe-SCsQvskrW+oaQ z3`0bodFUH&scz7xG{@$#&8GKc5_N5LTWqMgx{@ulW_<8ia;0X3*77hSH7ccWkryc5 zo79~34~)E_0RvAQFM&{Gvox7$7#xhYkc$`sWKCma1cTl}bh1-$IjQartX;_F*^(Mz za=neP>2W2vREILX-Ctim6K3pEw=WN~pq7tIy}@chYBrF`s89b`iFt_O8cEdjG}X~U z!@^?=lGO(uo^lGX0jD4&$!IoAi37J7u6)M0R=;eTK?8VZ=lCrWFXf(*??UpX zRamku+4R~5P|VC|qw+}Adi9<`z%HI4v= z?u^?Ep4)PK1h=`MPAif#Z1y8$X3t?+^;p;L=YeIT7+hmk8)aYGeR#FCI z!+LgOU&LowVR&{i9D@KT3 zmb0@t&REJVNl5Ctd-7dv;h`84=^GkmwkmmnMkb!e5Uq{klx^+IzR_b#4b; zXPsY|c_X<#=A^&ot_i(biqThIT}-~v7G1$h(tK|oL55XOC_P^&Ty{y02IBq#qd z;lHYY$8kXrr66eO!!uStX5c7u+(ocor#{9=8KoTa(C<5^0VW?OCqCIub*9Annhb+m zf*xb9tRUFCMYMOsQOD~GqO!z86)wqCB{jnx&WR1lUcY-5JdEgZ1w%D1-BDD#mR8-h~qa$L+$vI0;wibump&<+9}e?o3Kmo<;7&X?XBf@ zvR8Yl1KHjmrt&l+I)S1~SC{tN0QV~~YN{4PF293R`>};rE%`kk7#atL_@ujSu6d7A zv1xZ>43@1oTZK49L6{!z>mGM0b)LyagP2ua|6MXLn}n*g3PGrt25~zB-CnBV3?phn z{n$zk>Us}TsnFgTMkvQ*d-x-;SI=x=veb1?+S=&6|!diK^p?ikPV|rM+z&P&{)F1S4>5&f#l2G<;EK;wm8zZ2Kz9bI`42v9F zU|3Y1{`Rjk$#b(4KD>Th72yLwusDmd>+Sf&ekbLh7+;0rA> z2_Y&O9-zX(;6V-Y2rH;;{M%H zSEaN5MOk@xO_(%H{Q$fAIHjL^dfgbEMXUWG#xSxvG5a8;e5eL}T({BUJ7gEM*9h$= zmbeFoeD;8HR@etrP+e1f{G0xxvTDkNHZt@2(gUid>(w0V%$|vgQrFTmpD5MzQo-oP zve@CegvvY*RRYaQK@Gl3S@D-2{w`jIjo;zvK;A~j&mi?jm9kCN)bFa*up;_$vGhfz zMlRvEfuzJ81E7V)#wPHQDg;kFo5%+{W^lDkYth>Gj~@0`g}m6jM*26+(N47T(|0LD z#pT&A=65Ww-MzPgyE*>!EkKqDuQ~zeG>tsNt~IJhl_s_N14W6e9~6GAukdo?zMTuQ z7(yCb>%mbI=iGo&-F=!n(hJF^@--;s@)KKw43f6EAGCiTgP0FBiUIzLnDxJl^Phyf zfT7c00b}GW?`UFdVf0tYC_5V1I$1bd*xCMTD$0?Uks9KM_rWqMY|y+OXd)D^(&vv% zlN11!D*%V-ky|XOEyh}xe(&Q(;eG}BR5&Dt%%wi0o$mGcbC3J+eg6)+kJ$sRbr#AF z?NAKAjcMgxZNDog*%m(GcyPEv@7iM0w9L>DbGtZ^9PwmMuXJ}i)uDPuI(i||Gudoo(@8w|GjXM6qqHo~v?2ekxXWN{W`UoKORuwHP~$m!xH`%Kj0=6JoBp zO1lPEy1=_p|9+97x*QN^6LR3!9Z{q==G*zHWTqu?ka`!{ja-Q{-o4;^8i2*-uU z87wz?A7Sc=&R(max)cSXp64yxuG^Tv{zJB0eyUox{FN;MAOHY#|L9kv|8n7fIx|K| zLk>v+`PQ(l7-Iqgt2#6k)~L)lmgm<6MzJ2`$ls)HuUamKkcLssIR?JYlozc7NY zAJ2sQWwzK5A~daOr^)-2i>c4o?*~F3AT-nwq$Q{4o)J zN`f2V-2!k9QC%S(va{C63rtonz>o^vG$#H-7aQXs=equ7(Sqx@7eUDt75!?**pLiJ zlj#avCh8wSw))uAn*7U&u-L!_%9rdM+o~JjuId{_ zVy!2v=^dkAXujCXLqfX>^51f95RPp6x?xaOG4dhQ=A6I{_8{_*k!pjsw@{Br%WWs{ z!fAenj&}}O`j(~%;8ORqRjy%Oy<|kga)l#37a>HpGM?s7fkt~W~<~isk7{zwA zpcQE2DO3Jbx`G~4aZr|umr4Y!OAZdhG~S3rv`1+9H`@})*#wh_Ry5u_Z?}tm!=Gzc zc!Rbi^9878@&@Q;F3ZtBx=G5Krbx^#3`4qPlG&$bw`04U$xT0zog79CB3@c7B0^m} z`hvL`A)_o3>R~bQzKCHmHWP~(7oNHY3rg+ANu3@ls^C>?bEs83Q{m;qtMwApM6ub_ zI|`|$7Os73QsOQ`Dzu5+Yf_s2){K1BKI*+C9^pR#i+(W^H%=;ZZnK-0O_+rhYn6Sb zVAKNzwaW?8J`|Fi6-@k$SVOcA~?b`x9s>j;Vj(7_SMRl0A=iH^$Epo~u zBq3Qf9v1Dj{@@mTZmSQ`f$v;5A3qorQk!`&Zpb_gHe#F$H^@A5fytPGnVv<}w!S$T zjQK3K&UC02^)T~zdW(l=*49Bt(HK1mFkD=87jl*eiuXK(g?Q#au@apgF2 zlQ&jR2s#s6*r@&H`5FbTN)zV$GgM758iXlHj)XgFnmYe#by#%{uPfJl9QC`csqe`xp&Ob@`2i=qIfB&U$`TBglL=X*=$e!LWY3Y4El$&qy^eLySR9!W{zIlbzc>-GNNO8UH?lbieXxZfIlw89>MfJ$Z}e2);N#0V@}W#OVZ zIL1hbel&xDY8KRnhK5EG-E_o(fkZuPlsH0G-331&0+AU_8ZAOUGA9kZ0q(4;YsfG& zW=cp;*gpM%fuW`kDef_|N>hekZ)3=ACRfHX#c=(&J==V(8Ou;r#xf?E*ZfqS;1He7 z65oXV)!C%R-{3OVQllW|&jNuP|RK%|nGh3laPQJ!St4btT)) zIa6*Rlhv8jMES=h79tqbQ9~!+cD?=)>Eg5ZW)lH9Zwd+kB@n%M&48j;hSTlC=0dbfiBUfLNk zx**U&>4i1e^{M$L`*D;kUKZM=*2Y&QG-ahusR_9vl`zMXLv~W=Kpbzit}|1DoRut; z&dN!1bGiEbaeUc?M~JAvQ#f{^jvhidd%N5SNghxpV%7w=H&uYOJdWZeu|E{jsn051 zV`o>YxK!Q;qe&R|B5OWT%&9hKAr77UM$N~;s-7wSOWyr5%S?osUBqqvUc!9 zMPPu-T`m9EBquD@-FLm!DR+qh#I8tZ05w<3Gp1N+U%w4lZ7@3!sD1RXGf?;HFpGtv z{u>1zAr=YJcAxaEWYc_Ypi54}yHp<_h*@fi<-9aX0-_DsdzQeh`&U~8%p2l{qLqLp!UVZA^C2G%1D+%R>w!|F?C{**qjCYR5zRO8xTIsJO-CDrQM$Z9!@kwZ7?H#}b zI!QhPes~yZVNDi#w19V1Q@ZP_iCz?l#GGcX5PHDJ?R4lbKn7f8JOX;mca3M&fdn#e zns^p}Vu_7ESEHcTVUkrY-x9_3Fn6jt;!4`FdpeOnJSvFAW`2q?JR&Pd!G8YEcrq7I z!po<6`6n2JM{@FCRduonoGscp%6LMurMN|=1DBNN#}G-|5;>2BFkN@Ju0?RSNAScHi``4|Nv&G_U6VuP>|hXR!;<1w{b6|Y3UC&w@o8^RN^ho4ox~)ubh<*^x09vlVH1Y95z33}^ z!z$;y-?<836i~9|P!+0B6<$#6miOR!W-AmQeInnz%uis=$@oNudLY%e&gGg9+I{wxQ3 z3d{9))%*8?t)GJUbqEFkkb&?ICtCk^!B&?2S9PvYyVS#0!|)B&aosQhPXlxa02a&y zbw#RAa1UTkA#NU^)&~qhtaNDPB0zFAJHO&5P}#~<)v{U2Y9PH=rL$QXMLbrN%v4=C zllvk563KHiUDp9m``QPa&Ha=6;_kN%0@2Yvv=J^3)K4xH;Jo=QLz^Zt{FNuiKd>ZFGoia0+qJadVBdF0Dg@(ItiPBS|C-2(5IRBc8ib^Xu9=Fgenk zvXlgrWeVOb?W|=E>(Sw~bibL%{6eYKr~uH(NelaO%?#mqD7PtMb)lPAYEgVlvunao z#xlZJ$2mu^r0dUoML$wg87p7bI(D#`NZ$H^GnJKwG%ga%fqz5*TfUyhjXcB1^Rqpb zcWzQ~AwY<-6Zpu0yZUg-T*2Ot7lx4eM{%TXzU*)fU2%cRwQqona#th{@3!KV#u>%P zT;ZPUQ+MQ}y?~H3)~_Fo($qdFBwG)reyFKV%#E-h&N4#Q%9*etOWIu;ug%0k*(FOl z8Nn`|!Is`6UMsl7Pr2S|Di(p4yY^5nU8SC^E)ag+iSnc&|4@jUQHOFC3XX3OkxVRC zrQi@%w(=GIi%>BN@`9BqQ7!Sy&) z-{sZ^W&gQC?dV^*d}MPr+<4O*ez{$A`%w=}uTtixb{*&t86RQpC;IWuv!D}e5kwC! zrS4nFpkG05U)%W zXQ(MtM3lX-Rb<PZ|VhLwrbaH^cF+kSjX`fr7z)|*i#KFBmm1hP%>ZRAJg2P@9TsWh%|28GDU&?k2LT5S3lpwdFxh=`Zlf*}yUJ zZ91MgCoP#ge6dKR%5iGTwfbY_o}VD!Bf+?~dJf>F>Uw+VmDOF{aADT=p-^V24c6*U zRqcPpT=&95{8{Hget-!5c-a&9`PR7Vh%KjbL=gin#W&UeBIq;2h4YT=xfVH)U!cy! zCb6nFUD*D(@#7l@01@mB^avK}wblkx7;y>BaY1THAAw!H=H(r+QP=PD8)s-4=HE7a z@tHR4h#|X&pj`-^dSwE7M)T~8ez{ZI?%xZ4aO2(h*}v{HCePDvrzMzf`y~EME)NTU za(t&Yb8MyNwJ{z~+ZZd{a@)t*c){JgPqpTytMN!tG78UG3X+(OB`B3P?f~t5=-0Wr zCpsQ$$CLX8W6}PJ`BpU4WE?pE`H$NTxBk#B(SIidA4C8E^8c3!;jfG{wJ`hFoG_xg z<%G0`@l#WrV(oD+0uR(`FVio)3W5M>5rkuCD19`%?hlTPcvyWbqDfdcY-u8hN#4Pd zM(SWocVo=x3ob!ABxH*U-O*Q^^dW`&ut=JtTYBBR^!+Pm@s_Jg7mC`5Nu~^r^OVc1 z@6QMA>5Gq8-uK7n2LS8g8~&K9mPC#~f zpSbF$8??;*M;PEm%3NZn)oLxbSJ^Usbhr1~C#c6~i0d8f`T05l|4B-HF^erS)l&Uq zSz96re`&DLV}^fsn%IKRjC72RWG2zz+(=WgCHi-3PY{LbMdH4SQ))0jr=9vH5Anz; zU(3|8=u>maX^gH`RxBMG3ro&Q{PHNcs9YDo3SEGXSRz+EeYOd#PXu zH2)o*N=hdoFzE!8{>Ne=CItU?cgl)11dB#XdP|mcOp2UFsjV?NOu{UryI0zNT^5*c zTM4;!U(7TrzY7!*@Z zy%B-X&cfL$t>+N&a_M7qfhI{(tc?aezeBBO{D5+=aU>OKu_xQkrH0A?D0e=7pe%?& zwGzCL_JAib3UNKHKNt%4a1=~?ZWtDccF&y%<=NOQ0IupC)Tn##S{r#gM9(ko%6$gz z+Wj>5z%Vit?vX{>d8jV$++fP|{v~d1A>^p0J8HN&iMwID)21$qpZGvrYogszsSqq; zX5C3a1u)^Wal^~le9$n^k8o@#)ZSpud$E3Bi1Q~MZ@hf9hRTO@U1>76)aiyq4<;4B z9m2ka&dj2zlf$di!Iw11qQP$Lb2O}(gm21}iOjHb_Mz7mK>?=n-l2ru4(kjP`$EkK zI@P^`-Q&k02bNu^u2E@~%0og=s#hevq;d$$69Dv}gN0_2njdgB(I3kgdiqiWjMBZFoT!3@eC zLpucM-PYy1BMZRD!@lI0Zc0x z@Luu=i;#p9$}0ME32OPIKK)`}f4o6tIJMLM*PH3g&TMfQdd=RZPz+oik@zKl_Zwra zSMRD1|BAdd)~GB(5BbzP}-yN-(^!v7ZKwLAK#raz;y-@@6xbxAI?tXGV1ocGEe zzsWfs$IdE8Sn1KfpGTy^1LWzM(tS}P*#@8Z=9mOiF6om}^`IqkbpC|242ExD#yDu^ zeIEiuqV#3K2Enxqa)dYVfogo0X<%7g=k*|N83Dgx?}FB+xVAT#PY2H1aCDs!DVRtNWT7ptHY@y{|)4e zJs*yvhEpHX*vV)=M{uWB4*0A=MSKra;4OA4ga9fah2F{zrxZihVII5)Z%MaLsH(T@ z@^_%-Yue*i733)^*X&zFztEvh;_GeL(QHfMp+CQJVKZ}|FGN^h?Y?gRufroR@u*lS zEpt;yneL!tuvx6YwgyTB%d8F3B|dn^ z@*J|*Ok{V$(d=Gu0aNA9>J_>|3WUzgR*!-Dh*4RC&S;i8s9Z;Mu5w;~yE8^#A)l)@ z^#HpH81FEaSDcjRTZvP+<(D6EPd~!SuMFIYKK>!SbDuXX$yctYQNMkM&!)mCnf5fE z0OaS$W$As^N;o(T58#h--0o#Ov&B(OFl@=?ffu@AAolg$-soM^*(aRG^Z59iPq=^Y z==tD>>A*k$0OjBS00RF<9bL)9nfQO5`~7Q2+`!5F@9ux!ElrmCuS2S_pR`^p3Xqa; z?&LBb@St#TRGE5xYsH1)U>JsbSM14tXU#GtLMS8|=Yj|H@!shV2bJq`QClnyS@#C3VOY?+AS!Az zThMloUX zO2{CE#8g*OQY;zj(5txX79`YKiJfd>uqxNL!fc(CYUrKu1eZvBglM*pQf9IZ8G{)+ zYtb!Y)F>e{b}VsG*<36fTuM%)e2Zi(C_?kK<5CP-Td*+-#gH~=@JRoZlN=5P2URP! zNwCK&L24XT)5wE# z-B@6+RBQFfcnR=cNb?F9_cQJ>*x)5Y0iU1b)4=~tySM5_2 z@2i%Qr1Fgwm(SK%bshdZln}hHW=9tDk2rI+8$3} z1JA&o0`OhJ{#_PhtXeUIQ4(kbv-JnVUx@jkTY!IB{mX8lw!Kt=120h$8eU+y<#J93 z&`tWmg@%@Zk2quB>@i6jh11hxIwV*JC$rAB!cN&wJ%GlvPTdjOIEa@0NwIJD96=Mf z2PAHWPP2(7uN$#EL6BCR5OY??=YL?<5FEgK|6^#L>_0bV|C?f1e+Osb|Kk+<*DnzV z7XxbxXAdDe8+!vs182MczQJ9htRsgcfXuU0cRhv*2?oP@m>rAk1|gl|o3j zbKFX?c2g=>olZ>DK3GNd)W0+&Bc2&)hM^u}P>*5eNb=%Ou!6K9s01xj(JnWi-3uth zOy_<-A@jXF6aAomY^6=*V5QNeqRtX&X!#N=d{-SeadE|&M0Ny?;{`DB7_>B!sP!hl ztKHgOzA^lvZ+BqQq`CYksIuNo?z(s6%yeEVp~$XRu_1DlN#SI@wLw&*SQ073UQHO% z97siJYA+o^8&)+=p02VG!+|p95OeT*AbPP$ZuF77eK!8PPgi9j(lvC#uL3TpkFq^K z8|+)NqT9KUF^7FVg5VH;bq~A;R~PZox>7V_aGH>WAlhXUV7UvY)6f)0NYlMS!VjgYN;?Itm(&(*k-4+IWA6IySy! zHD~^qU%Va{ZJ)!#(;{{(S-HB-jpQCS*X@q?v*V71`iB(|PN^pgCQ%j;_=JKx3f&g| zH=wN10>M>XP-pNWXQXc7;~{ba9uY99)-=lfB1!>SV%@=u4QVgGgm1`R*ZHKxY8lL< zltlA`O}>z&Bxh%`n!LB&{vSyPT`}z_Aa^Xtn|zozoWjRY@*G#soXInGfmATE3E06p z@~dg+sB>b6{*-q921ZWK#Mfy3#9o_V-xb3B{^_gy2v+k#^x=3X+>)!XeS&@4N@W~& z5%7mbLD)OiL9Za!)f~)x0yO6E15qYuc3%G9M1(K9LOxMvsZKk6{mzT1Zs4Nv_9w(C zVyw_SP}+rFSKt4UnRKHALWaKp0Nw!s066|XX6C;&y&OdyDZsz!Sfo{DnvrUv2PY8l zG%u(JpGh2*fd5hW$ZMMz_o93oJfjZ zahRReLZO#j_w^=cG2psn3NRL3_RuDu-O(XCdCh?rm;Mkdf zvIwx%ORpTJ4IkG9fw8~K0<<0NF?*}odZ?`SqV@M3 z@fkmU0r>j=B(K zbGxSy9Uq%aqRFFFGL|!I6Eh#LDW_C}{3aqN(#f>6@XoBfui6B#9j0<@md*6M^whA8 zp97Vl0nb;Hcc~DClx4YBV)e<4FO?WD6cW;7L5vO+=K;BwtfVLEKM zis=3AJ}Hj1VXRruc~;M7gGn?Y{23g`DXytt<$<5Uohe0OjU!$v#uX#XntQ|T>SM;4 zHaNSC4;`9 zy{WveXesB00pI!vUZPFtsy2HSlSH*rCr{-zX0%{U%`hopXilEr$CKJZz#5H_LEw~) zHe6Vo$`bd-`$(KRacO9@pq&CgQGuv;PJHb5R>q^rm?7}GW^Q}LCRU=-$%gumzlt^nR)gCuKKWbhhT9sjqqL1VV8qoB+mKT~KbU${Mw?7bdhC;qfwy{|d zRN%pzkc0k6A*^#5)O=UbnHrOfR*9}()QqiVmF0nCqjx*%^~-bRg!k!}DHx~r*V#pD zbGaRi#85y88XJWw21*R653x++2?Siath0m%qSAQ-Dy6EJ2yL5$Z5rj`_dz~8+7NDO zbbE#7*7wMYXC0V0GbID_Y?KgnEX#YUy5vd>lq4C4npd_>cAN=4veP(`F{g*AmLw;j zM%}zJ74<$ncf3xCq66eRSaDR;uqb!iY0+acQ=K$L&mu{UB^{5j61yd+okYi)&#jAY zX;S@@xTVSXiQhPy4nz!_9WV0;iH)q~#aXFPY37m>SLx%^X$+$>sm6~73~U>WA5T+= zPl8@CCa$q%jbzmp_b#T@)d*6?SC*rgH;jxL6aHbb_BgWvr)Rz2nkEXYHz#FlvosLv%|^80j$xMoPN;3>7-BLkI7a{(^JFwr z64za}aE>=z$eqO?Op?=!JfTBCNYuRDpy8UdHh8?3wvTh;t6Mj;D3?y$AJ9=yvR6ZG zHcDcss2{XAWTsUbZKyBH}t{8|fL*XfG{ zJ+Sn=_(qlzSNd1jWh%n#+dl~tDJ*Rn_p1zDSqSC&d?ntS9CMInO?p|CKBuA6^!n+o z)4J05L0@2Rif!umDV9%QG0)JI;-t&Gj&a;=Dw?$^Jtu{BOLYu*a*%mFLO9RLs5anI zzc{-)$F>CDvGEc}cAr?7ux}ZJg?Al-;yxqg{rZqGrHn8ZqcVw12O&SWIfZ}ZvNEE( z3DQieDcyHnVsJUlFG&o3bih-R9YUTE<<)gW3uxgy-WCpK!kL&QWV8YgQHFA~>=v~w z5LSV_hEXO4suXWkEDIZyUDGPrQW%kv)x{{_GTY!2JrDL%(mv{%x9>S+>Am<=3!sGL z(M3_b6Hk8mGT=1Tr8Upe1sTD?It(XIH^?e6B@Wc|h*kSgwGTSax@PN* zdCU&=@^FKoJj)Iu@xt8(aN0ojN+<{OB$fY;cGZXcmQW7SB1K>~Qjn_GfsMJ|k~oqj z#1UfJjYQoJvbu*AzLO>H*tA{}gLM0~Dt<=}C9)~dkn4r~Fe5}ByFgz9p1#Ur00K+FovmPvK z9}~RnE7_cA^%6Qa=!Cd(x?rDgFo2AihyPqEHvb9mX>d`N8co~1hr8xrhth|p(|tY+Pc*caO<9|9Ry-4n>$d(K>fVp|tpX@82*wz>7+M!b#aDy9-7*RG(7xji-dJ1yx-!&ia>K#_8##|Ax9Lg0N9^sHF-b{gSn z4$o~mH9Ge3wkh^3A8_cW1KZB9IH_lObnst@Q;>bK0Cb3lQfK{a=5BOL zC4CwIJrgK?!y&yZA-xH0yKGXQjDNUbnOg2Nr@fP>n{|IHXnprh9e><=#eTMN6iB1l zYCfooy^fH+NeZAZ=(H_3L>}l+zrEap!M1Q0L`J*LK@cHQJ)Chv>}42s(9pzZg-4OE zE`7-9`QArY8Ea|omHzmFN!{yg``;G8?U`x622j;tflsJDVrm~rTD3vBvmK3%VQ(l8 zso}K)&n*6-lzpf;KYJsJ&0Q0n4S*9wGyMZSZ7+t4RX%6uVMeIj-+3Ljydi>?kl*@L zhVaB(*g{*_5^ab_H=oDzM+|lppL+>h3wy^WP|AChW6^g&05d8WPdLPm@PWAVMkvIM z(D9d!m+-XQP~)Pj)Ztp5aGbEQFNkbG)Q}}?l^nsa0ZbSRoFPV73-)TRAW)bK_G*qJ zq5E7Tgc_bJU{^F`;A%1$OC;;0|61`Uy2?{6F!<~->*(tzd;wd-q>xIT#5J`BA(e?Cz_o-Var~rw~#`hN);6O+m2vB-Q(gZ8(JP* z3OC=e<*O`NA05Jrhv?_|C=d*rjKLI=?9FxVK}#r)GMI->n5T^=LYmE?3EQmG1_xCgs`fMbQbSp?v1g=V>;(k@#ugawOvhY%$aZ=> z>KFxzgZL}V5(`ApTx*8AA(kj!!)-rd3)0jwt>@cKpwt>FdT7wMgIZk(#B# ztZTMy+qP}nwvDfC+qP}nw(ZsK)m&|B{vBsu%sw#_XQra6A}{LleKR7no=7u!f{tmk zJ!vo!cul$&$&&2|j`U>yOL;66NTbRS@HfOB>nr*<)uTKF=YCnM zmAkF+aT^8xunPQe4jvMO?QY!Oz`ijK6ErBEbC?h6b=LN{8#gnR)6H9Z)<%1f2-pMl zuA@2==BF-gWo?UmH$$O3t9B*a~uSiG|u?dKZ%od)=s*;KncUO# zViW`Fp0kJYk1>)qj)R+KLABQy1yhZDhfl*&|nj;?nZ{@Khv63ld&KBiU+$H~Vixl$x zF#-K$h3CS;iaIdefX@GlV#q4xg7XPTDm9^_r~Z~PS5zrJm_xDtVBq^m(U~XrF!jq` zTcN(N9C=E3=W|xgT3zQ>o9{gdk{Cu1XgskE6R61cFbP(FK7ix%KGU`&0x*B3iADR+^zTq_+tbB)ES2zmD7*fXNE|=zme$FuT$rAD7XrCRo7JFyTlIz> zy>%pr9pBkt^27AlDVfPjpyK0f%`6s7Gfa1{5N5j=#vWk1=m5pR+(0kj*Cg}F#VYGM z)s<}RoAXsR9`>7zKdaIZOp~N*MM*|`OJ25?&+Mh$kvsEgzb))?T&C>lqeDIK^UV-QV~bT%1I0X@lQDFHFeC=A7KkW}Art7|~!VgTrnQD!{~yb4ZnKX=#)vVPG2DduC| zUGzpWARr5QARvbSA7cLRe_0=BUo`axz*J5HL z6s)74AHo4Sk7pnC=gYO4YR%_vubUpre8@oSBL(<|aa%S|_=Ytic4iDCvN{zhj?EqXku*G9ZrcsLxt- z$12|cL!s;41|)R=yR3 z%G{Gv_-qNY^bRl9_`XL)H{5?$AY_IMn${LP9y6->>W^`j|FfQhLJlADXM?T zg}A+O(;gP!w0*^he+Co&)*WTY-4PnRHDG;Y7ddLd?z1fXR(ooN#L*o}d}T?P$Y=+Q zCqR0+#Wz~?w~*2+MFV*wRl*b{l}l}_7#o``PmmM=c8JdaQs^4+^+Q)`#w!Orz6Ir^ zYRS@@(jbS6R_?cVr<&D^p4E(2Y%?C6j6tpBfy5PQbm`Y3I^j}k;>i#mQD*9;R)7Sl zwL*>3DiZyW&wIn_$wedH{sx!~2wt&rnr1f7%v(}6Pcf?M#Tydcki6OM)H(-l!}hzf zWX8w0=48g2x8_*J<9nW-NvA1ShX=|oNLq)wWJE+0<2S{k}<3ZEZXrZlU|6= zu2V5)Vf1&LFRj?-m?(GUK^Dy9o0ydA;zIzi933$ zqdzb=)HO9U)fsDR4cbcUeU1I*P>$3S5*=JOGz#?fJF0t^?d0nXyFKrVyU8hzKZ0^? z`-YI5+*f^ceg7#*4u&vUD81$m1<&Tp=|RrqY@I-RgBWK%F~s9O1me{l~WIU@?PSpYg`S3bJRf)#X`o^^c#QDjSIf=qTC=mk(k1YpRH_3-+P8M=*id+} zm@86Vj-1TbUk(hvZe-=c8+Qri+8TZ;y!^J=i6Vu;7Ptp<3_mXe5AKkWt4~VB{ad-xsr3_t z^gL6V#b&v2gp-JRon;E862Xu;sf}~QuIe0bovny18u;O;agzzK2y?bVWD_=w>NjP-~-TJF5@8jYNMyFiognzE>|Hvhny$Q4CYJQ3Y%~=3Z z@%TR_6tDE6=9rN3rg^AkE^};kuCH|VFW=g-gv2Hf=ppQ5*-y_J3a~+&lC{mD!gcHN z^=f}gYs%?!Csl)&(i2cKRW*C@fCu1``?K$wKXP);kPTxzY<&39AJ(r>U$=j7GQhAo z<%d?tf{|K#NQV}sg}0ic-ktD<5oyirYGEiW7-N%9C*&r5Kb6n08w|M3M~c=t}z)Wgd5-qHGg_ zFHi0gR7w+NzH8v9KGj+gUhGvSTImX{ICskVQI!h7qj{W_Yc$1S6z4%;@iKL`cnoTP z;6@(W(<1fyFmd%%JiZxM5#0JjYM-Rx>`+k*=E_V=S3KiYUMDmsuXv4>3k=}}!36a0 zeK~%dqXHhVA&)q3uR__voh)AkSHh$e63dV>v&7;w!Xixts*^aSFJA~U*tI7p95uSN=m$X-_~U|8G*XTqB(2RZAh=o8Zjg~$)5K%{d~~LXHJ=&o4|pDz zz)vz<)sH9pI`0?-3vlox$vmbYAR2Zw%nXh;yd#dwi9G)p4Ku0dxC>5yHa-B_3v}MZ zI=Wxe`W!7IdM5YC`DK^}TiFj)#LHkBO*HlSl-k3W$4f|>7U#)pf89F)f}`uA}hv&b)w;t#g99tc^s>?iP(Qs(at*HVLY607X; zo!H!5&6Tur1^g_gR->)e#aU%(X`-WNVk$7xG~|e_eqy69FP9%4rr%Tb{xJLl4CCHW zAj{R!6sT~UTBs{apQa`kA6=B4inq{|&(US`lodCdc$+_`zLTp$*Y);R7q{A2xG8(N zJ%1G+9aXVLPuG;Iqopb2sOYJya0kfTw_bXt;HH93z}AE)O)TB5^#ATah4voj(ZJt% zD|}WOV0m}@?~>CdaCghu--lN7&HamCR}za){(5VhW;XoK1A#!j)!8=dn}bq6j*PB# z2^r|l(a8cXG~(O~+?C@HDuLq!7%WkYkm2KGb~jJY-Py0~C`s-kEG1E_G#>QjxgVjN zoX-3t!PJYzY5y^03)j{Rrl}~&?ZhkbiE@*K+ulzlVaQFHvg;6ExbL0{ z?pq_bXz@kgr);KGl>T0cg3R+T1pFg>%>L~5Xt~Y+57jrTH!e1bq8T>Sc#>1)6N7Qf z3VC_9a88qhEYNGBv$o{!b$q?^Ewp4u0_*W;MS)WMxlOx6P&K`gS-&FlT6}5gbep6G z5M6qm@yZj1Ar=##tm zLbU{6coc^uN&6;8=;)*FwM9)~8KwU3JTAH^+Y?|j6M)efEAxQGCTwB&*TY~SyML>d zdn$)YGiN8A_X*E5F5lX*sYi;1KJoq$oR-|q(L=X4$I}yq%&1>J@H2ZE--!RjxI*Ry zbM&Nm|N0!LgcClw$padyNRt*duwqWYNFW~D2ri!Y%d8`Zb?D0O5V1Jz0P~dKx9b`H z!m4R{l=1@KKiSjBu(eU`0pLUDAwW3cnJ~p;jzGPC3Fm63dr(iF?TS;f&2IRS6U7>r z37>;hf8htHyk00gDJs0!gNV=E@CW%RENxA~Z#A0?$7kpJy-9!%n{k}agCLsUixZ{N zz@Hrg3qQW9l1{vmY|x)}Kr-_`t8kvTSpta>+e9cJnsZ+p86^(Oo`)aV$Dees09*v% zVO}uUH}t~++MSg*ru2W*tFHuK>GiiOs?di-dpF=dz&^r!O!-VQ@nK)Fm!X;>2A`QxkJ2&IA<+GQp5WtZYLsG^C#BR z_9ZUC)_;d}`4G3rU7)#Vx7J^W&;jol@jT)f-z}q0NoUyAhHPrdbAa~{>uP!vRgB6Q z)dO$83{OU;u1v2&>6zSyv_j~+ZTTFAvPcrf+MI{=ve3hki81G;CyQwo&vfCpOX8;J zN$(6+@iszAcnqB`#hYn5Z)bj5pUNb|>Pgq15Mb%1AIZo#{Xqm*u2svEz}YCQzze*u zS6fzM%r6SbHnMe7`2e-jb~Wgg{yBtRv! z3B!s-BgO@EJe~Yh>cABRv5LQ^G>VvlFnB_c*^F2YGo0BDS?=*XP)rFIA^pRgiFO-= zJxWI7T^#0iDzi(#TVN)ABf}9TlgiK3!Wvb6S3lE@el}XUH4NiM4qvFiT&XH|sZyN3 zmNmP|92%?Ij%y(}TXQ1gt+4~@g5`-I^E$W-wew=yLN9w7-c@hL-zG3^uaQbLmuA7> zhV!~W9B&;%(%u1|TwHmf(w_qde0CF*a;g=2H+gK81hV4 zhDATC{^^EEW`Riy{fwsO{SRn`JIqp+G>PwOmc)jxiJBd?v`1Oba=Yjxo*=25#u3eI z8;+8Nas6CiGmMKmD2{TbEhkZ5;A;NGG?8_)9;qG=@2s`?RyYaE@VH@m$=~xWex(&^ zySxjsF*jz})cT4XSyz71GZgbl>`QEZ01tMu0_f8qwt1HKnQ-N4Sctjz8E_^23<^1E z?CJDnIC!tg9+NF?=bO_ZhZG!VM(lZrIQ&=ZCVMgFJl#qhI-=@@H{z;Ww^aL8&&N&z z%^%*TqYG@7_}L}aNy8rFaatke()@JKsULtUC!L>{RZZ`0P}!`}mPY2p!pyXuGqhXw ziS|i+_e|OrsHKOfsCIh38&bOCX%t@IX}2w6ZDNgAB`i(W*mNCJj&^OVv6DjRSQj!z zjT}Zz=9o>2_-_=mL20VlB2T#>R7*)5gDJ88p_T}&5WEBb3S46AgUkwn)=HM}mq9ES z%RFGKefYhbboheXY8O-w zB}?yoRM3lCS&33*sKn2}!3kDX%Dj z3y6NNv0aRl*iqV@u-2Byf2*%CA1){Iqp(bvP`H}nlQO4bcYPVwp8sHl^$Cv^$}9=s z;rjd=_>u~T8s*xI_*J{%z6{T;f|#qh2#>Z8adr@}KulnFzcAB@ zyL6$qY#onGd-@2WIc{VAOj4DeVA6EgBCsiivpuTgWPTkqH<~zW;Yx_6`uKirzB}8T zY$I{S!0&>nXGEkg6*EuKiUz;8;AW0R;DMN2B#vHx)m?Q*5iS>Id9`(7S0DSy_2ge_ znW8^U{eRXl|*IV;7FtPyoAFuMc33 zs8?o*11u|`zX!RoxU4s$91|sR^i}a zMXB0s>jl|Q_GTl~)leq($|Gac5T*J=nX`y~pciGzdc+ z&AHH+;4_OmnGDpXDpg~fT&I%K>6aP(CHr{6CW}o zhgMCMlT&wc6({g|}YI|nMb?l1++zzkQz8+nygL;NPtLqv0!D8y{kMLYkYB_#W zGPDqLY~pq9rU%p7kvf&xljNJ`ME~8kSaIR*wpZ^Q*NBO0_qJ=UlSrsT&#@=AYtk26 z$m{&$j^4i0e&E&K!)$alTxic2ZpUaGaf7+~!tpcsLTq64NorvAiNdHt@7a_co>579 zui}PhP_8gYyK3!O#sk}+^7o+a(%Q36N79q_PNNMvpwb|!qpIWR=5o(7q8s#9t>=#6 znYvf^X2fSz^WNauv{&&v7SKNmx!Q0q>_g+e)pDo&X|-!z5U9BM_aNm3=PT)(ZrA*E zc`re6hjG;5Gp%#NAM|B2AkydN^+4bg!FT)zOTczNZegon==&w)&Ga+oE%-a*E%_U% z-|iR2(21!3Qt7S&f%0cIDYk+4;SOF>p#Oquv;!aAbCQ{G-)Z%rCtl|(?pV%up$S9~ z(0X)>hkIu_k@rS9!TM%7A^a6{EdG^qjQQpx4`td@vrIaG0n z{r-(lVMgAta_Dj?FaGisAY(=5^*aD1Ln3%{upmQ})w?Qh@J+7X0)4VS%b3LHO!o!Z zPtt2?gy0LmFGS*H(OD`SWknYqgGcF1&PRW3rK^p908Zt3tDidUD+Op7?W-6mZ4?KtL zuj`c@7v@SO=ZX$!TV_0VqIq2yjw>_fm3d-(Rwj728=hS%8pD+pm*G6iNH$Q%w08Lp zg$Pc78mM-=>oVD~`L>D_2!6|uxozANL(J3q%pbfXkHF%ZAxFwy$A_z6Xcb!qng>?q zjBAn=cidb$Yt~gZE^x&`MDXSVE9EIe3JJHrv1lp}}A2`9E0 zgSs(DB^Uh@5z zN$C)DqVJ0v?q)T>WjENI3m5Osabk@tqP1uUd19=!f*=^Il~;r-;V##tzYk2&!VrC- z>eITzeE+fqf7JPcb$^p#0;X6>8EuE6*6de_r_Rt6UAM;`_Qq(EOkUL_Eu0Jf66hL!ioY`;`?C*R< z=H+TB`OpdOt1UGkPQy?x=xiMgRszKx;?Jti*K%ju=m)z#Vx-yhPd5`EI9^GLup^z1$It%2=7CQFHW+R3!6$;Il3g#I$jy2vz+FX=3$} zHdoL`Xz0Q?w4p3t%4zQiT7CQ=89Ht$!v}E$-{qT0lHMqZXgdPhxh~gQ;)yQSArIWK zRU-3vsNYH4$bnqPDNQ|5{AgU~qexs493Nx@7>I17}V1haZfN{dJ#kb)UTXH2V_d}6BQ(j>VfTZAS;99&|VV@!BnM30YU z^4`na=Y*_BKOFU#j~^7xS+%N9Pvy8n>6xCe_%XsOoFmm+6xS0F$pO%ejV6id zm`3vN<*vKmWAZy_m6^+5cAz5MOmTdH2NaA@fY6eBo zx?}ekn`fU$^n|n$U+yi|-=~kv-iSk65KcX6PcA1&yBt~a-GGnG)mHn95X@A zr?~_aRZ4bfh1-s4zkxR4F{0Am*)*Ya#JJoMH^CI3I`9##OQDwYL*g9b=!MlxebtBC zxvCP$5I^HOhx19o&VVo?&>~oyQmD~l&vrY73s*yU$jakTKE3w!zXXS~` z(zz;5Z$T)$oTYdlzI-dMC>|Lcf{WLdh*haS%&}6D_@>D9!_}1yOHWE!15ZDm{;D!Z z@IWh(L|spilI(PfznK~&!ssQ`eT^C(Wv8B+ZSbXy2In3H{UF=;doBn(EIfQhVhv`k5rkaEb(PqOrR1 z4;yf9Q7*26Vv*Us3u?>;ttVM=_?OLs+S4*lDqBa|v!)4j))1>sU4`{{h)b}b`2o}a z2~)e#M|+fE=Vj4p=f+tx>J*Mom&3l12u9OcmG8{bvC^ylvioYrt+>ZHQ6XW(#_QrW zhnq}hEc9{*VlDFzqR|SJy?CYiHYg@!ywM!Ag#sk-vAH}Ar}gMvYDis#_?#>x&dfr03_J_zgcS?>Z<-r02~v;Arcn7J z&cm)eu!&xocb(A)?9CPSr2Tj+6+0xy8!p2lDxVLD>-1MYB6HVDz;5iMd}=WX_6Y^v zLwM?yD}NM_HAk$Ea4JzZhP02oWyl?Z12;i9qjZg9I^hXWJ4RBD6g(yDLcWg}Jmq&` zZzCZOSex*F$O*)HHiIZ5BbW^NNH$tiG|nid!RIS@G(sj0M81FxgkB!8XNO_$NghCS zI;2|jEhKEi8Pd~`MG>^InMd8)Nr>y~>rjp^w<-P;s6#uEn?|5j6L(F!O;(RtKb+!> zGU&e;OYy^1C|^63Qp8_Iux;a-35l?!2qMU8@Y)p#&Zi}XtS~Z_cB_ z5JoWH7Fo1fw)F=nUG~bOmmz-eU43`64F3D6yv>;FsH+q53;(v^7nW|lY&I9*Au|8w3a_73}}aKB@jcV0^ZUD@s`Y^Bn8;7*(F4V&YYe#)TlD;)!P1FKU}VuD$f17iy|Cb7(4Kmn=u{Gc3>8Ai?g87+?nA1`|e1^Wm}4Ig zsN#4c%a?@FBY8y{mt{Jfg0=o%cEoXUhUx~N_Nxti$C}9z`vD5!6oBD47=`(hP!e-k zZLIYy)6;KXZ!F}|wkU852my2^Dih-Yb2FTEcGmiT_Nxz}dcGdhU^#4ZiMQ7CT z?jm4^FQnfZh6VU&-Hjyn~FN28O>1&pgw2bYQnrJpK2W zX2iD>OuUpGmvEGeRqFY8`J^@0<4#HOSv53i#LiZt>*1Dd3M$ksWePl9M2Lf6&&m6x z!bxRdJb%%z4}30JNeV_OKfs%`-a-H02)@x9m*Ij=2G{`4U(&YpQF1+T%nzne<0Isc z3~-6-D@rF^N*@FT)PSDeZk|LDjA9*9WOPGK&IeYEE-`rPC<@c*e|q&%97 z074+3T%rHd^z;9_ivN3rnm4qknm7tTeSq`v%u|{o1&#@fNr-3yjAWw{7!8VQ2p|On zUC2TviX~@pGz%uItPOuzT^Uou;CYO^VJaRA9SQQMzNfpUr^d7X-=%jC!Bv>hPqn{e zyb#p=@6YY$JInW_o2I(vf33CSv5XH|H}> zJs=QM6sNvblpx=}A_D>*_l5~-;43@2T5rOocTh%pcc(+F4%yELq7zlgYR9w!ov`R$`L`Uw?xqNN0%B6t zMWAr_C+HqF!GDDoX-5u?T8oL1V!%zIWQYhPu+-3_)-%9|@58OOs+-K0&H#L?Sac;# z%~OvSagu;hQcK8@7LYi(<7ypjEp#f|_z0ZoYLOs!~Nt=l$L7TXa(_;Ny`Myk_B_kcEzt@`~=+skZm?x5nB+{ia+*%gZi|~rYZCma&$~|kL5O2^jf-l z`0>z((dw=sa?ERG+NF`N0UJ_$m}d`E0Uj0{FN-&(o^E5k0Yk||j^Kp=8al(tK z)xzh{B%N6$5HY7bdfL4a8&bj@hq*)!Zg418oUEM*ZrQva0wMzH_f8ZY;Ic; zfDZ{~qFg+E1vv`bx-G1ah@2j|ZFyDfZGzBC+NF4C+en{D50|WxL2ag5N;A{wBZ+_6 zLlT{c#t<>O)_jzKOomQJ)}BvCk&+atj?kP>0ylx#?8|Lv+x7d|h*&kGtq7}0-Y#9E zheRm9JcuKwqnt$#Q%^n{UW$5_vrRc)pPd0-nDj3S?CMO3Po`IjCX*C$IH7iA^}(gl zs@Az@S0t6T+~JV{DjV2{m{C!kokiCr$o=4whk#pjtucpNvn6B}TphF845PL+hOB?+ z-y&lID9Papv5=u|b`|Su?VRNDr1R!*Z9fzvdt@VM+!F+q;vyGs?zSs3RB!7#>u8P) z62C|2i1(a7O}tKg;9A*8*-BJw%b_w!P+WlOyRQP`F|^S=dzV&B zNXK>a){$b_<%z#(+f2K>_*X`q^|BG<7FvlYMzZeX%M$={TN*Q%smmDLNraJnI7+o; zT2x~_l!ak;=4sz$V<+}fB^nYjnFuE*@TivBtlfy$Msx*h1oqXb8StFRYfbPHX6sEG z7njQVt7o17|5G&qxwxRU@IoCf3!HG?-CeFLCD7@jdG3f!q%o1~MwMNWi~#=k`%(G$ z1M2E)jo)b)fxP%LK&1wcl1fPgRm!W$-brRq4AX~|rmM#nYd4c9*J-7kw-1O5B65p~ zIFXDEwsR}AXAhnjuX;8`6x(Tr!QEcn3bL&3N$@YGrf^nc{{ z#={rK>gI`(zdUudiS$v%#>S0os1PGTs*LYGe9XfK+(YGhktUVhcr^^{<93MF&A`$q zH(~ABOu^!oM6TC88QurAMNO}OII5BOSZ11enS&PxvChP)-)W2q?$bSKysdXg%vcp8 zP~5=q0nI4hBN-Rux@6dS4syLmUK_)=40G0!7$!-UwL|W(JO)%8^vf*&DwE8R)ce4; z0(oUZ)GMDfgq&2tCRX~ZFwZ1~R}$r`4`;y+hb(%Ly(kW>!YT(>F(Zl@R!mj2@l+#? zImXX8Ae&k}0(n=&AQPJKnvku+EP_cYj{PX8fmt$Yr5}sTp&yHW?0R?`rk}Zy%Mgz` zadiZyJYRbbhQQ5}#X!+WEPrw)rm@LvJ!5MHwT^13vCz8KHbg->kF72YtVWh!z^C{g z^yTU7^?nc4Cm9l#q(0Q`>qQ+~Izpttvu#mfEhQJ+;l`ZBx$_%Du1;VPdPyvcfm%<>h07}VvOH^EUgfv-<`<-imtB~jh)S98@CFgE7Ce@tfDoh@E#`KfqA#}80O;M3r z%1z^gz>0FMh{_;8yC&Y283DNNMIZXoLiNf#C2r=zemuctRuhm^dmk}*FVYgwrQ7nU zbRZLFp~fxfvomi@SD5z|R`8oNZWrX%+26&2n^>70UfVyieP~it zG{&fK`0{m5d&f&Zg~Sb>WfEML8#sMeZl~rUB8*&Nr&}KrdFk{^K+3u(mY^#}ix2J! z=jkGW=}oCftl;8AGfZ-XaCjfNe259Ik!B?$>_ba9F=vp#qS}feIRPt`Fk2#*{|C&^ z?JG~WG=_a7C$M7a)S`w7+!RZ~t-lIGINVtxHKECKE1hlCWU`IMJC-~ieo?Fd)>TA> zj{j7W@9jhZe6p$0DCmk`O$6i3jh|GSI-$!&KDBYr#g}jLXW7?AVj0Z@Y;RwyIwtMN zu6wd#P9Q6IXPEp@^gWrUJ6tfcTzEdt|*U9GWD-MpQFRsS^c!wT6IB($=<>3 z^jbT&WKEf=7Gny)NDDubNcC$)q6C8$wH`Jc1c6PWp)xo%g-ppl-pHOrRK=_f(Gr$C zeXXjt>-Q|BiyS!zQy(vz?)tFS=sQI(gV+g=$?<2hxLOsQ&W9zgVvrH728JPF0e+n$ z#Ze3W1UAFHTNY;qotf41vZocK%$U4Cy;9!PwWSf&DjS*Ev=Hhm7z#PE zFAjIoEgDUkTP?UZ(|^*Hf`~^*MlNYo&EY&Sbvah{MNvLZSOD1HAg&x%5M8wTt~M6Q zzC?L5r10=bF7m{9RnBNx+_e*-F!IKrE8<4t()P4EcKt} z1|jnSe?rpk+#1>VoAncgEaWd~2o~y(aN2(zz z7B{zI>EvOD@~b}-pJk@=y2x?4GX?S|iR=W-_3(;IN72(qqOdrjNPHFA1emskQE}+T zVjk)HqEQY)9_-Dljr0W4ViNj&rSEliLdu-v^Jrb_AB|Ws>bpHGZPs7!);Na9kt%o! z%HzHzMwAVp&71WN!2{-SS2PN*rC;D?(!3pKf8LBfnu&fP=A#3D`8ms{v3 zv&BgeBkLIV?rw%}ZsVjYAvLHlWgYLWwoB1MvEPcdUfQ#G7bE4neQ}KSMk8Ejli2UR zED4!r^Cr^|vy7E0Q!>VeEdQ#Fh%o*(y%tIFqn!?y zYG;{DY~qCs-2t%y0R=$*flfdr>r@hV>>=-+SWA%A1G7{j469ickacvzL4D#YF9ikQ zM$R3)v;{rzad`ymtN^F)k@rgueKeYV`RpE=?Xx$BSVS)L3=6io&*&Tqtkp_lp z!G&T%&x2f(nH3Ecu)VPcrXG-!7(qLfSBNYG=aW-W`=PO6h^Co50mHUUtn5IKcIA#@ zA&MFYqml0)WT2G|qJ?3NqwPh66 z543&EJ0tS+mdUR~a;m;KG+zT@k5 zerT`!;)3lCsFyvH5vrLdy@4(+92pw{o*4pyO^B)IVm8*3*ELER{KMEy;pc8h7Bg_O zEW#2T!?*k0Tjo$A2bPUeyGX#9USrftq9n|=krpnTVzO07kPA>qm@FlYf5ZgwAwAOl zHecybvGZRJsu!M}A~VRhNvon(e2F)=g!xni?@fn@D1*NQSlao_!#PV3=DY`5z&S;e zHQ5a%V7U%(Ug}jMZ4SJ#e}GNVxcbI|>hf@P5Csk&b+(P^ZX*pJtscx_PgEKtZ6lmk z_luFwCIlMT@#nIITHM81-6scT8s0qv8|?{lBzHhuW|0%@UAy-Fw&BXK)S2Sqc_5-; zGfw!}k*=uc-N-&avT{Uu)G=5Nps$K3?4KvMOI)mPpu z|E{XPTo|dLsw-@c${yi44d)oneQFkdl1iCw>CL)hgyQsX21wD^{$|%UlT(*Cfkl&> zCZn~b%_ZA$uqcAOi|C7@i-?i&+{@yPct)7Q8ldSJw5!?)Uhu@j&!cM^J{3P_%~GXU z|LHZk39V|gi;mCa*0CYI_cIF?EEsdcd5rYwM*U+qcB}ElO|ap2@}-a{5cb=Hkaz(S zTWlEXuef;2j0i^-xRm>CNZJlvaDn6eoMi%-cjO-x;=Y**;(kVHS4zc2MM?{&hw%G*6B3EbN9q zdv7@vB4cjMg*?l6y??O|0T)wDSc%${=gRhBl9#}TU7^I~45pTt2F+PUh2}kw0*ziE z&8wo6ZD-iBm|3Ua!4sh5X(zzj_g*x<){QehLnxR?VvEBWVVi_Xt{An>mN4MnO8)~9 z6RVo+F6R{DMSyuL!=f|G=lV?y9 zmMX+c_t+y>xn3ZDnKW5X#$u1@VMfWLbEqs)F@p2dBOT(r>IZcj_h8*$s~Dg?H#)OZ z^fNj7xN={?`V;LA_3mQze=+t>(SiU;wrCyOb{$)FY}>YN+qP}nwr$(C?OSta=1q66 z_om-k`IEmpcI4iXiUK00|^@PX{FOid~S_N}nz zd!VWiExl{CN&7V;zxnN?kUFJL-f&EVe|jliX)FEk^r#_Qe>vg{xU$8$_Qkp-Z$=|G=d&Av~%;}|m z<=Uixx6bQ5hmC{L5Y%Z^fDY~$1aH-N-NTgwsM=?2RiX0#7=pxv8c?o7#YIIMV79~H zB)AS?o&${uhh3$4?uEU><|N?`%-V-?&CJgwP5iDEA@?sq;o&Tx3N%Uj#aU7rc#o&L^YFk;`*Cmms&tqKMRmTnQyXq4h zuN075{_3_a9Vn+%=(g-UU}iDywv8RKTlQkN=bY^}$sOObN*i{k6l+i8$^BKj3!_`* zCJ=AYW0&-aZ?&YkZw8s|N<}9S=v-p9tXrC8<|Fi6glthpPOgVPqaE%Fl{Z9)uVg5i zCIYH~+H1}VBakf_aIy(mPiMkzbw)n=i`s#1Mm`FN8Xw!xtT!*23{_S6fNmm)b#vZ7 zD4DW$)SkN2qzyoVDx-FBKAeyWZeMhQQq|ZEDW<72;D9P8f9IYH*W_vUZaL4@--$fB zrFsxz`N1yV2`AgM3z$@cWf%D(N>#55twh~rm+|6&b?Y5lSzTuYW4Yx{@q))L`3Wch zOF*>0)9MTOM3ra6{9=W5=ACQ0`1CxpXzrpw@%J)~Pz&01h?;+{1TH$1%fi&5DtR}4?- zPKG6H^%FE;qcWjJcI^y_Cq4LXA~Fq}u!+;mYmgRp$Ql0{@NqfqVYEYb$D*uhteTaw z&1fDBL{;rPCr_9iUunONI#|-HX2_j;>}8n7;h#I~Wh}Gm8&}v12U}Xt83$7XQCgl` zY;~NoCy`eO`!REKUA#}|v}9?^;4fR>*RoY$Fg3u6;3&os?a^6PpRCQ2ED7zfBL{mD z{bP3KbuHfafVp*)L%V1BOu1^UNJz0_-k@es>OujO@FcqLzK1Tk2#VT zt12VJ^&{&mxtD@~m(6(!&eKHa+xoOi49_QEoyp(bi9+N%{r=MSKFrhkbP1^@awI~r zpAG<3R-wUbA9`f4rxp_z0UjSDN5J@p2_HbAjjt2Cyef3&q&kMy`zQ1&0k8#RbotcxpL6@&pGorIi3`EbR+p+9DQfaZ=%xj&`aS&&S)GOHwclg9L&zfQ!5Wm6A zd%Uw`JsIg?dvzV|!17cFx+~LzFcHhZG`AA+1qre1q08_uL5daUxosWgDQ>tSn;Y2@ z9BgcfZ`67ceJumf(i|8)qpK0;A0gH3Z-_Rg*(sW{lL;A!4{nz>B)IH%CRmzaSUY)G~T5No6dQ~O1hTo|D^=oNe z`peT{SzU_l^KEn^{0&+L%zMt|TKo=}Z#!004XIA$$USPR_q0Kq2i!C%%*xjwPF4fER|eEqw0tb*yWm9;Hv@70O$w1$(G1yWU|NQgwm zF)hkI;+;+JWcPcvy~z1WN`@N^lYbn`7q07UiX9QJC7jqeVPAB|>4SFmx+^!&>KwKT z>6ofB%x!h18R8Z%HuhGIkP$w(oUfQi?^mr_0PdQ$VL?QSD30=b-Cs}^+hD36SEmHO zDBM}-$Pv_M)T|@wu+>ykWA!Z*Tp@6=RM_MxP3w@BP0T71i?Cw4ppq3(Rpx84Aw|w{ zM^Hlx$Fj6PZ%f`8#J8|ixlQ;z+eieA#Y9BQ+{&7T+5CQBpxeSK%fodtE5CU}6>4l1&EG0IHkSKdz>v}i-=BFj zbpO-b%LGa*jwA)^onn(asgtk@0XeKE#8 z>AcD5$tA!hM#kdyM;+Nhf&^kUv6!L^LEA^He8FhiRmfgAfHT7MC(arNg*=@~AWls~ z_t3c&e~qV|lOyOr+K)}%swQzhZZ*AB<6x*ZaUH5R7|joA#V;j$WS3Yrlv+0OLXS!t z5{XPxGc-jY5V2Ot6{WHdX4YRt;nV}BaZmWP_hOg)lll{;CdB0q^cCGE(1u*@1A|45 zwF!gamc((dDm9`-4|h9y;0`v^SL2qb6G>*E!xmyYO3tz<8|zlP6A!Ns>=wckUMKft z;uXHzPvdUqmCjpi=4Z~7v>Aefo5H?M3wjtGRvfHoBGHU@G(2Zgk|(Dl~H=vQ}sW?&74La7b= zw^mxz4B5jSlM#7$b>(1}IP#9{@_xp6(QC3jb$6BJuuEdct=19LXUaXwo@WeRX((Q~ z^k>XE7>~RJ1a4GLDd}FxIrOJwR!B}o(`yjsFzpi4qc?&#wF2z7^KS0P9t=h%P2nqb z44^Jl&RGV6npZ@!MsdhvXyw0`%Y^k z2C7(ck!X7~#j!*}m9n4}<{>J3JKU!)67Ly& z<5Yz|?h@~jaV5NzT~*q^M8+1c5$+(Plij@qHe(D5ojHcw)0kNY+4II5`^27K8YQtD z>x%)?f(=W-Q$~Ra6uiyXehoY?kX!|92GD5?)L{=wFWGMLkqHHNJx&zuh{AlWz47p; zMZPn{lD#{-U56~N3W>tRFn37UA~w++q7NBufpz2YvB)u}&me&1^2lRzhxex(+`1Ba z_Hl^fe}+{s#6ZnlW3?1>6qVdt`)j#f_qp^xcYYj=5SQlV7%M**&(BErU>G~Olz&bG z(}{Sy0I>#n2R&-=_7Fc-IR98zcq}Nt99LauG`HJqpLZKod@NVFVYLicZR|dK%u~ls*=qM6*i3D}_Q<$Pt4C%m-s4=o!yVGU@Zp>_;==Wdvrj}hnx_Xs z((45&^@Caj0#zfje`>ethKvTQ1Yv#Cy!W2u(_SKd9i6lx%HP0_4j`RgLb@hBmt_9_ zHe5;jG+Ysb_|$&5$b)j+n5k7o{}~q#j$4Wvfj9<}?D;$tGv9_#SxU4F`C#AuuBz{y zuGesr#3npp;(k3lK9S@2k?9(6 zID=X^wGw+DV;%a0cE>dfnySWIn4uzuZ75{yhIMrnu5+S(|f9>Vhp2Qa>auRPO z(Fu->=|LUsgPhJ8#bzi1lbz@GeonySnUWb~P4Zs2HFV%&${pt#BfpD-;hG1qLUc{^ z4M~t-5X1m+rpMgWmt0?2$vjj(PDOzTUTp$TKSs{9mTrg@G@RQa?*5$M$j!p7{(Qh- zcSPke-De#|9VwaGm!AHQ5iDJq*f80vFr|hH&wS9~3B+ta<>ou#gbNfhO`pC02yr+~ zT0FHUKK&mfo(Y{#Bp=cw-61l8kkJR0a~2jSV=DI+0lrMFkeoM-(E=(oQmm1IRl?kd znUf?4triD27sK)u-A6GE5a5s))*b$jGtGuh0NeD0oVh)JJj#2$q12ycIMV#FfD?*Vgcr%gg3?&ssc9U(0Y` zvt}k#%C?5Z(~EG^j}kux^A0y8_=2Ir#`ZUB9y4!Q4n7{!Z4*2{T(3~Q-kZ+_5NKZD zy@vtBeKaELa4f~dn)d?zLOjNFlza7f-ru_;AD*rKAr+plzHlqAxA=J6`vXqgo|grC zpdnp-MH!myhbg05@D zS$Tip+->RRhh&dRi>1hih%DDi#VWD?T~jNPrKyim}?g zGVw8~t{V(5kMTiWOem!Nl%CInms7H_7bXFMR*POe6w2eV$5b(+DlX zPqfV#So<4>N@!^75cY{!<5iC7Hx{D;NA=9)8>oo!ooa8H6zY2saIoAX!8eRD>YC zB3qdSw$4)1CmYp$sFIkKszgqRTkv~Iu8u_F5Qg8Y9s`4LhS;3?p?FhdW$lhA}de>;S|tHfpto zt${Jpv)v{BTvDNeB)zw0=dw)3JY_{12O4_DT*=T+bODW@8g+k+Oj}SSS)Q(8VGax% zefU8zIE|fEG4(BdOwbZ*5fkg^2wBG>))dL!nNQx)NGsu+54SyRNj(}Z3*v?0zS9)2 zME$w^^|K+yLyPV_r#YhYgURs-X|~-f;eRaENnu)D_(i9kZ->VPlTZ~=6O7u+PyHf` zYTgC5g4d`htW(0vJV#W!Ehg>;=zHUmqo4WT47IG!29|j50Me*2}HE`f@^7hYE zbO(^(kke;`Eu4x6>aJJn!yyVsF>eY^6*PO3F!t2%K(k|y?AaXKEpoc-X@=*+$PGIx zgHGOSF2)wygOS_D&AtBHvK};i7~0%U8OzHyIGu|E4w0>@nDb4zF+P;I8HjB@#fLR& z&1J~aiNSG{`POjiR^g5nhfB*mUhO`+J_U*B64cJ{6|&4|$=vAWi^3&xx(QE-N3-kr zS>ikmez?>?mQk=Lkomn3RzZY2pz(-Xg;|t5x6vTM&2d8hVX;o~d|F8aJR3%g#R8Dh zrw7%ROc3vMzN=io7M0&Gc|KBy3mAvi>1|K#RQ3~`%V_*RqPjc~r|KeXKvP96@WiW2 znX)dEtYbw&UtWse%Duxgl=J%V0v3akUd%BtxWO!ows=ZGu06*f=To&4MS9~2OF%`( zU6}X5J5aGF;=;vPdQ6As{gJM+f9^5lp0MA*+$l=j9GDL}Mp_2wipPIN4*lR6dnOx*JLZ)2mj1Zg# zB1%a^V4zmr(V`qoE4}w{wGYQ;?%hRG>0)O4g6Blj>J#jP@QL#?-tqof!-jb_TI#@l z{mT2RVWj^~!yNS-js8==qEtRSHjGicQFIE)#*3`e;oDJKG*m{c3{Kg{(`C~mHqzOh zLpOn2cxCm8-Vp?c?t}Ay&7Y6 z2bH*aW@zQ?-QmRHdZ$3_Ak77vXxw>y@(#k?5d!cW-ytFJ3>hKt3^-WM%&gr*HGjra z&oCTPa%#ihQUbt1xeE3o0K`DDQs|2I(gTcuM1rFu+4A?2BlHa02zG|<{28dgc2`JK zq+!a;9?QhY8IyuFF+^;@NCQgQpXD|`HIkUCA}>;v19F`G4Mv`nhACY#Y;6A)Mzr83 zU}~r;9refW;J0+P7^%tO=B;70nL2A@RAECwNk#dukm|5U47cGDQk0JAMJt#%CmIj- zw6gMkY;`TPHHhL}vAtAc1_Mca+tb><`L)Zz$9qOug>#`n>@Pb&xa+WKFq;iOUu%DS)Xw8n=1jSnv9n{FjDWKF{MHiSZ z&~aFnd4}*=&5=jD8LG~%)9lBo1R@Kg*os(-WJyGwllbmMFY#V;=;PHqM-ysG6VY&7 z?}e0bmGLUbgz9$?9*jsiJhZbKKtITZGy6W7IrG1NPqslFjK@zDXi%C!e-)OM2PGW} zP1S3yTZ0SLLX#de(k}0)f++?=l`AFWs#gANCja3TBhet0U3o`<)Y=#^mT z8*HxPHE`S7H|>}((jEx28Gj4b9`4F z*t1)M$UVJ3M%$pG=$!3jSMKJr6wN$tuht>Bc1S%i4NuwUAO*^cgo9yrWN0=_hM3eX{j0mO87)iFI)RsrPH6GrTQ zACs)1f%G2(q)Lb}9KhBwLA+I7oCDNbSrpF$8 zl&jAD0km`1`hD=cSSAGbaqllnt) z1J%}?V>l75R_6{|#b(REOh*NT%!kN#4>XI(N4u2J*au3=j>H84ZFPYKue5x8QYs*mtj zbYF8w%&Y^R&efN9wj-L#D8(X zS?03%D%JdQM&PuCx-?pq*xLx;SnRZsSLl|yx{a1%IETMAjUK4qh}fd zc{6SCh`pjA>Toc1P;Db!dto8|i1CPR0^;Y9rj=s@*tlh@5%}D~uB<8bVZJ(^Ie=Js zEtR0a-f0WYs&HEVxsQYJFzkP}EvOOk?8DLmH46EXa%tKb_Il$z^k&10yo`Uy8wES` zWJOGR@HA}6L?iGx^i*NJ|5fFA$fq7Os;#a;2scL|{5-dg8a78E^nT_eSv0Guk%j(&#F&&(g=+P`0Lj*(63)4|3>u{-E99`^yRH==4Fw+uhq#CVVR3h zj~Cm}4u#I>GIV}VjQWR(4NQg)4#}WZt`;FHs3ZO)8H6!lPH}$}M4q7P|E7uFILyrS zn10AS+054I>G=Y_g+5|6w5(X=Q|_zxAE9783>+J&EtRkTO%YCC80eF~OD-^2+EbQ8 zXNl=mQ{k}U!s*po;^B9(!6k9hU9plLbzBmo9EAT~?yMhfZq%W~puHj$T-D+L5V8qT z2{0f{!-K}lecsc6E+2fQ9+kV&+(}}?;aCtTIdX%MeG#%eQraj`W2bHu=q=2Z1nLKEty)5fLU2BJw6%6gnv^nrTNJUVF zA4DWzuZ&qGG;-Flc&kjqL;TU|W$fgM*u?V%F9M4PO|kM7skS_>m$ocjpp^uxBIMxf zDgIi-L=R25ph(8BL7PS9PoPYA;N7+i-c)+Ln08eiji$iHPOSGTZ zEx;{BIZv#Q78>G55yD4sA3$IxDB(a66H765Z@}7+Eei2gIX;97rOm-A3TE4l{^B6&VExez0z+K@fwS(&BKOhStHp%I ziwD)9GM?3!1@B5jBF}35AG?RzyjDaozYTr59HJWF0Z*;r{aU;xq4mS7*gp|Zg#!bq zGDY1dVhT88xB(w{&Pjq(i6dNPhKP8vBtr|;vd~xxqWY@E+rhbcd7d*&LJxrrz4Jse4? z#9Ub|Gp}~tu4=?EN#m^&kYa;a+`HbR2h}{2^PT2|XC*jyis#Qo3(S}2-dHG>8?~~{ zyQ{W)YNczdeK_tl*>c|PY*WIMx+68oyNDBy6!({BwYEVdHOn>q{S#x zk1=7b5!jh~$SSf$H_%(!y}W+Y7&Xs0lNukO2T)ottB+X|;xv4mfQxhz zEl+wJOUHgx>c7pCh1`Al?g*0Y-*MQTrpSCS>yqmubL!Rx%%_DvblF6%k4EcXP=?`< zxn*`=MsAM~yg|?$I7_S*ZgTG`@O0Dg^iUha`_U)5B8RY?qn*b@zZ|i6SU>9WM1xbfbA!`I4N>Z53Kve0wh=$TWM*Vga zFB^1Xz<^-bEm$Ytl2_@(wI}@QMmF%EECE(FXE4@WTd!S%V2G->CqQore|l9^?owBI z@tTNo(j6kl$BP)^Oyvok^%5F=Z>LXcP5eH)+KCTw@f>y-G)02_UJ@hwe)ehE2ao%} z_aTU-bx3WEVj~&3>N1&CXqCnx;oIyBAMn)crT8-v*p&Nv`~3%-IEE*FD8=wGO%rZo z_+OIQeLxcB_3te1FPP;27R-OF4`u!V!uE9@cyd*QzYro=@{v(>Ii7K82}D^o#2Z6> zC!oprgYopnASS3*g8To4Fm19k_9bS8<(J7Rktu#e9hN~1yp}le6iS}cY6g5ipBE*# z8ch;9(?i)TNRIOI*`d=|1;onySrSG%N84}52lW|rmEw6zw`s?M$V{l=D1EQ)*rQcj zLK^ThNpv2x7`Xl(RwAN<_PL_gIBUJ;>7306Vi|;dDc08!@BZ(cvJ3q~238?mo*U_3 z_8rv1%JdV4ZC;On?$0ZAL!%j5ma39HfUAP~X|+cx=4KD>4o-8e0>uVdd&OQ^MQUYp6hhzxZnp^2d%r&}wO}E{LoaW|(Mvv&9Z6;0 zzgz9u<}ESgc9+5OC#Jvlt>TvIN(}~9w%`W{oeRxr+&mE>SU;eARNB}51>&C`$xbRknYM_|1mnT*(=YCCeA>e^FCMzY{#43Ywwe zVY~L@`|cfd7kL2L>JbSV@e-kZn8t(;OaR?zdWX_VU~lPI@@*be8>1*dDJ5tkr#j=1 z4L`f~$d=DECi*1sbe!6VkH^5mdSa>nm7|Mn5W|>eWk}Pb+R;FG&u=C04$o=+Xuw5} z-Jxh^3PuI$*=I9{^E~PrUzi}~^+vvbu4b-m%x!9Vx$pEB^&7qKq2MW_7O+|3t7#UP=a#9^NYe3JN@*%sVha8#&*aahr|^w1*kr6mW01inouV z(Xa*nQ!QJP^A+e*Zuq(%A)n-VwmZ!&D~;vqWaR_QSK*SVE-*7n3QHq!wJnxE1;Pc) zTFH$oL~QFN^DgZmst^_EnvOavLbxj5Q04JlGv#;z^E_>${h1XjQL|;1NEKyiFHllR zR2vIMebb^=dB?S0Y03DDT7?zngU@TXB)L! zHlCNJ&pM3;oF9oYZ=;e*Rt z-yqpP$jbjgq$+T~A1kE+Lo=#vQQ`iphx<$*FHB2t@&qCoqiGJF$zxI#Xmu}*<`^!< zUYW+c%TGIsP#=(YKC>4_zEx;5_dYQ)FzmPB1c@-dxB;7hACS0PRG5qNR3;@P$b=y)2M<=K!p1W@JTk4|Hk`}^);!{rxT?E){3%eiX57SIJKGqfeEV+a;$*Ua(RQB!|*C^tA0P^nVNLabZ+na@_E*!l+m zHsxL++Xj3AZlje~?y~EJJo>TN-X?o?5)QpAJ^)$o5iKG_sN`FqE&XML2Kc`@fBbJ&`~xABBpk4W z;J=rZ3zrC+3t1tE@X0u_)kYBl2l4BO^Irk#QAU32f!EN*lia61RjXAgLLj4g|60Lr zvkxp6EQ_TmZ@rK4rO5SgA*|9N$6Ne*T4C5?*y5_-`T4qX`GqzYwTDtEC>x~*OevQ+ zAca~%WnHIgaK{IXHw-@%9*udy9uebMIdVtOJ!hveQo_!CP?Xbf0fnPeZznRMQW+Et zdQlvWYEgDTbHPlzt(!-%8o6r$<1WFhB%)RfT9uDqJ(<0P42yuHpUgyr(?m>poKCG# ze%1;Vvs}K!;WU9Dyi`vnoOzWo4z0eQZ9)vRtS^%u9eh@9)B@G-2;DK;p;+5NQbGnk zLUl_~hc7gtrA~jN-hhe7(JNfG4U(=FFvEu^o zeSL2r@ylyqAkwPi3n^Oj5+Wc!2^PvMM3HMLpi5QnGD*%UraB1{0M2i~^>zj>dARStd9W zMu=}3&$9x==lTnO0jJ`4b=-1c*07{=AHClajWbz!&|G=NdYpLZPNDA#&D9BTMbDI# znaX0VdC05U3eaXxA23BtQ_n7GsKQlhU~DVQ+A(P^p1J7bdToM=q!uJ0NC@jb?aR0w zz0VnN$Cp|70mJ4#eDT?ngib(^=mEnz?(Q;6?vjdxjaliU^<;AmN1Z-m%+J9jW0Y+~ zdK}60Po0qQh!%6IMfw>;%63B0VOBu-6YloP?Ry@(2n(k2t+Igv^Vw32v72tczK2FS z_4~HrOS6Mb$XCBY!OUzg*9ceh!`^JA(_+i~l=|M(#7qpAPi;o`?8)g3Kq_zeNC5m2 zKteV5%|&HRiD%Zs6V_;l`+5uQYy0j6i*vd`2d*I(7i*5hD0E3y&6IH3Y>+<#pu6<7q&0Kp_k@NB{?)Z{=c?E+yYiGz?$VtHYdmh)WGA8WN4lYO6bAEu9&UOnf`8N7eM^a!&tLc;HpH~NUC>R5kMgkUbp^@SI2}1M1^HP#=JFOCgROYo-ts%Xs z;0RoAfnM_lr`99D`X;73TWq|69^Rh50dzyZ(4o7)sV@UpVd_Lq6Z2!C%252S!Q{b0mnbK=PJ&(h*maXh8BZB#4EdJV_=4XDCh{?X_;r$D=GC#|@w5Xu!WT zGeROMHPS5I;?vaC*w8RqC_1#M@ieT5Fz|6N){*yx-H&T_4oJHXAqWN`mS;(@zhlHj zf1tIcg}DdhT@a6@k$o{+FTnBHyX6}ex-O)%-htuaws`|~QTifs#4%@-emi9XS+>%= ziRmM0&8*#zUD^CIpeJo7#3+AVlLGL+aZW;3wvKMH_C|(g2LG|O@n1&jziP{P37Z98 z_>mtA4m<54`1E_Snt9$VWLr0JSzuA6IDbGz;HO@zgv$P8L#<3HrPf=k-F^TfA|hDN zU);!g%XN`T@_FyI3>*wUJJZoGD>ryN0GqxhKIkzcAcl3RBA{pvs`heDc47D)IuvDi zbgSA{5q)ZK?f!br&>0a25j}~l(t=rK3*)vJkA>Y^3 zJlN#aVErfRZ&(31U)BuBT5EHEWH0>Y^<`@4TkX-0@&j}b%*l2lOc!)VUjnxe^JLTe zUND;i-WFyt%G&i~{1DE-jH&4A@(EC2E#)rmv?v~b)};=FjGh}~t+ zcv9K;pJ-CAhEkpgueB1*sKkc^%Sy&=Y>GLv2UY}z>)}YrFRaTim`!QDKM*7dy3+fK zk#77<#pS9xZ{RC$z@+|b?Ohp11b6dp zd-1V(y~SlD2YhLbt0&u`l8}3j>;h}XxB$b*S~_=V37{N&=_0>FKlmvMq`NN8Kz#4f z<`)ihM;E5cju=FR*1m(Hn+oq~p~*apw) zs|UML{eobRa4^)LJWpH0Dz7`!aM}Q?-8{0Xg@ZXspia4O9P3O-tu8!jGhGYTsn}h&R?By`b#^q|CMs~_BQtaU?bynBmnu4hh~_ZrY6Vhdt`)U#hT!|F!)lJ z`~`DgXEzEBq^cPHOpEJOl;nnR^WssocZkV|P5o);X>)L%;hG#d+2HUctURF)LV>Hm zT?Su6DoWxY-n;~bF}U%{(=z>Yxz3nZ@=0l$OjwjMDLPdN z*mbUv5?zs0K|ID3pSURXA+?g&4ALZLk`xKF(o^chLtD7!gMrvc=qS~&u*~%+79Vvo z5`jsY1BL|GCN+!c*?BX8XH9#Woo^!Q@s{^Z1+vXZL03%ao zeuf)AvoFr{0_n0PPFj5U1f`*ikT^`O?p{hgMj~Pi0Q{jWR?996Jf_QOMXB>;m6Iec zmgJ96)wkhivs3#9{O2H*Oq{m&|DECe4HU)y79>d{S0j5qgMU~NqPZaZdEtY0(3!sd zcL6M^5aH1Vr^VqZ`S|K;H4fmF_l1|EPM4@SeM1mgnAKHThd5K;j;B00ZCri&MEZR8{D|sj%4?QnK)O>ThPCnfI=eYiLu~&sv`=B9Ml3U+-??V1 zHv-J_?d;5l3MSZX47}IyQrrxdb?Wk6|CffIN(O+g$)JCW9++Rh{`~($Ldey~!0G?F zf0?AD<$$b&={@cE;%Hnf0RLxAk_ZaC;sK<=u%W>MJ{ewUSSpkq{YW5Xx=?^!(s(!< z7y8#>XE=YHOM(@#p7xv?jj?4%y!*y`5K#%4@?{dU|WS!8o zKQ?S+z!eaz-*mVS9Coi(lZ#wMW^75Pa2dgEkeQxJ7bQz<|lV+CDSY_mWoG{vV6ax6hL87`x;Wvg(Uf5u)uQto< z{7rJES5uhY$c7CZ8oAnn4$bNi{)WSmj{tfbXhrW9rBL}sc1xPV9Jw82tNm3>n_-8O z5D9LoBP3JLrIS z{dE_MUDu7^?qp({blp6 z{$@WtGWFmrer$KvE|jHqSbL{)Ru}{naU!+iEhEy^GFVbWW?VNW7mHj=f46a;fRg&d znIIHX*iS2Vc3)u-lIt=*%eQ$RoDY(!S3w6byEU$z-%}5~$?@|H>Pw8iC?9rv z8X?K1IX;8z1Bv;z_7ZJ>JxjauoL4ewz>_+HX~ao#Sf^x^-1&rqFlS1FKYLmF4Jt!d zwn>ieGBDK9cvzpDntVvI`WSICLVUP}SF5H|kSgHC!*ikeI zg!7P9mQrCj!5}EzswslI5=t2sj1#p-(5G{m0OEwm%BF*_dEj@{?;7Yd!W@oE>Zh$YWveZ z0acl(u1{&m4v_B_ei}hg;@JjWuS8@t_j0z8K2-5HNBjJ&Z@c@ASO2(dev-iDxi z6`|0D3l8t16tpM{@&<7R8U2o_=sALv(eP?S3p3==V$`=0Tm&DgmZN zwoqb=d0#Qrvqf_BxmTBptkMNC(^*;#wZO%(Wj|dp#|o>7-{2L8KSGuAvK%$JZ@pPg zQ4*$OIg9AI6N`W1Pm&Qu2`2B{{Mp1rzO$k|{iNOFbbrFY$gz4ST!CwT$Ecu`5!IbR zYX`%wAU_4!VjIhHjNKaOZ6#FBOq9uP6EBk>Kkw2~1G&mY#Hdt%TXcd(2J!nD>&ipCVrx>qG* zicog13fs;KKUY|-t@Qw%%MMKxL?x;l^*9-$=I7}V9<14OVp<`rjR}LzJlXLnyJIG_ z^^0x`^{@-`@(xBwHL}+-w%6)o-QI(`qrnTVaE)%CIRN?u!kb?Qdt3>5h2vJ%u9%(g zqQbz4qoal`TZGz!%}yRdG2E9f-T$M8eo)?OSSL*@?2wLe3wHEj_8HdEBdbwt??S-_Bfyj4j(1FszP`4o zlnOMjpT5%rpR_ha?4j>i9{1Y>ZD6%;!*|AY;AHRbHii$>}aHhb_ZZ{!1))!HCUZL7-BIIC$#>r7<3{dXwb1VeS{bz zXa?ztP&(}-`hHibVX}kBNcv$J!$^kd@l@ST8hldTacd*UAWdyv&YjwdaBDUu4K{W8 z%i^O{CxJN}i7~7jXbuIt$zUb#pJh?VTN9Thdpi-|Rt3!&J9lBCl<=aIlVn~ql`Z?9 zi1URjL^(CeI>M5dKW|%^6uQ@MQPZ6B7E%YYy`GHD1%mV&u14UeqZ!F^rpqrd9hEmL zqXc*1Lo0yubDw_Rg_KNNM-005vP z;oMwbhN1t)=w=9|>T@8+mqurY8^FG{<)KcAO%m;u;NcCQ5t82H`Ic!14tk32;3NS^ z<7RXuRv0A(w?#m1Gd&aj0L=F;v-!~7Y^C^}5n#GVnOp~FZG#kWK0I(n3;IeciZ-Q5ablH&wYG&P?%VJXipqj zfgou>GJXVP=0+U}M|e6cqNj;=N|7#6%w^81T*TRIh{;r<4~30Yv-&{Zb%}hnxV4w4 zMaiK&CV3{!#`In8t)A;5SED@jDcM)OcnsAh{^oC${^oHaIZ!UIz`F?#9;IqD!A3}j zqqFgIDrvnwhr^JirC~D8{E(a3g-pKjptcxT&jk{oCW1BFYh~k`*Lj6V7Is z_jftybfEY5ZNYH*PrKnqi6Sia^qB5Qr|mH4@0FQ)h^rLFA&909zJg4sHi-Oisslna!x8Iw^E zM9b5z0b=f7!aLUp%b_$HRaNWgLoONRmsZ#BRdL*#iXDbZQHz38G)tEemxiddoYK@? zre63n4X(=1jz7VjAnHqO$GCBqA?y0uR$vEeM+B%d(G&MFC{OpE=Jw8gjG_^={0_j0 zy`(G0C*?6?YkgiEuJ#gwegk&ih!efQarwEz>h%9We+Rk-_s)#-)%-~o zA2IU;wE9%K7{+4l%?h-L!QSZ%KVv%RZ7t&h?B3J(PG0l|9K!|n%lkF5^aLc%Nh zVLQkzy~zZGnP2OQa{bFvdn9_v-q3k7EtHZtJ;2X^ZkJvy5A3yuHvXp+k8qsoDSJP} zHcIT_E`vsf9eZRpdU8+MVZ6N7n-;G#jrXKP&;ER=+AFfn{6NLY-+n~o*1N;@w}Dt~Mpyrao-OeoR#WSU{)096mbyY@O};2^rdX7`(f4 z#Y=hbB!V*P`Tdt>gkJI%RVp9=z~6wv{-691|MvhS(VuVsvq8&MDeH9t_7{pl#S0e z(=R-`J&E2owx+24tgV>f_g|?@e)mt*r>5%Y1gpW+k-(cVDy9A~(9KIRF9h!IOK;S@ zujBJ}Bk+{rU}2r#%psX?og_iCMdGBFiB;z#WgziF1d(De>!3C+XWr>BbIuZ0Z zQFQ1?l-FG&T$@=nFL~uN8JV~6rOjk^E%}_!!AX`Z^fpYgHg(`q@w&gh!6;%Bub@fZ z;y$0}94nf*y0}3ufvm!$B%M7yLgJLezG`Xu`el_p0fn)Le@wVYkCne?JS`anhQAEr z@(?1JrC4a?rG#*&@+G|M%AgtSGoH85bXKaAPc<5+*kog;I39JZB%Y#cgyZFOSAP)y~yqDRi*EFDTbd07X>nDw4wBOay0cuw@D065b5-8AbIhRlG zb7a*BR{KbwhH_}Cgwm^@KN>OU(j+o-Fs;s`3q&)mB|sWhFkMU>=|jgi!eVgeF2Paj zt5d3;T4mQ%t5(%6HdT7{)*0%r;i-nQX(1(89v!sbw8Mu^ShwlqvIezv%7;dx_{*M# zNWZ;_WAhw!Jf-!ws_;~Ey+`SI7wX*WLOwS%f4G8E9+kkS+QLqp9Nss3;ghul*@t`0 zrDzI))-T)tUG+%-rV6qdEJ)hnCVB^f{mr{51 z&r@_*yUW3AC0J=CL>nXhFqKhMB;YQ@PZ=X(2;j5cpHEr(p^i_M7H=_bWGKC6To;l8 zyaov_(BxdFyiH>Il?761GM&?Hu{`N$dOP`3JL79^GXbY9g>E6-En*|`qK1Njb0Z^%ycpl5W!q;M^mCC8{1ucJr; z$`@h9E3;K%vR+dkY7KMHB5>`{nop~DarApq0lQws5x#WWabd;LR}f9yW@32B0rI3J z0^xSjTL(MkT@kfYmjUG=>&iJ8rX*DntVIme=G{?Ok4zw{sajn`@x;KS3-eN7abuGe z+SL&|Lg7zt9V27S8&@W(Xy~lgYm%i?#1SkuVLZ=+LeG1MN0K;e_3OXeSQz~$kEG19 z0^3+DwHXqnS*mp`T->TM7Y3%X0cJ2(ND;vX6{D5Xg-W4l-dp09MWM@#=$S!EX;Q4z z(rg?TSGv&NEuI%|kZuog)GRINpb`6^q>`A_4eja6-M4u z-dQ<{gSD)^06Gufb9lStc3oOE^k;zwOW;oh_+)?*8^Y*(Try9w(X3=I1nuZmWvR;kg)gxj@}+Y848eK-+pB-q>)#hn<89!W7W2lJ4%rA!2$*J)>3EQAGq zBRp`Ps^pq`(I#A_+TY9}c2-^|QS3a>VHSeFh)V9?!k&oL2&@Ob~aRbwR1HLJ~L`4A;kJh9?m zper%TK%4~wd7Tra_*q$f#KJmeeytcVD=K}aiTu1$tQgJoJ;BT-=i?zEX_IsX6#{6NMxbN7hf<1X|yrz{VecDL-7aB>!W5t1FefcqqE1x2( z`|apF5)t?LR;zpYEqUU597_ja9sY7-dxWIWT;Q$+Inq5lCi2GiXEBL}dH-B?fGERO zZRNmJBNNG6${6he5Vg6s!E(uB(K1m6>fnQy5XLObcwhM~Xe}Eot$0(>LHL?@wICR- zDc!+iV<4v;EtvaYA9d?kADVNka_Cg6xZV0Y9bfQkTp)GpR-g4tLg35+6l`}?rrH&o z8=nZfn_wvSCUNY3TwrM3One~2aNmMkh{SBB9<~?u^Xgq1;ZCYx#Opx}7ryNJT@s-z zlx!@QzLLvtlCNQofq2tf4ABCH!T1xWW!X2*yUvf;Ks3A6KDgh4qB?j13^LC7Wajnr2!$m+BHhBF8P3!00$RJsXz5KI4@)~CcT#g9WZ3IKIPZEq4pw_7ZLi# ztzdIS0!eZHd@cMCXYIyEU$88ia0e9qifZ}!8foxb)m;(pdn&$%67xIn^f33%H>Q(V zmgBD3#y^Zu)JVy8(K)6l=0g&KvjXxxhVW*bKF6B12`Zy;R=T($d_8z7gFTaJ87V*y z(a?-bKGQ6MR3c-e-@Jqwr)#>2?wmV=3T*k(9?4cQFtICNZSw*Wq zCfSxfDNtEY^V2L{#dn0FG2*ACAMS_kztazWX*ven5N0IUFkBdWhpX6GZ?=SfidCU zNF%vp$H#n@Up@f^O<&(}+EFN(bWxsfH;egN+`#EP9?1wd;V%VAp(!2#tF<{$_Zent zuv}tenysC+^gb>fWNi1J6D`|Pb&DR8>a>7OkJrJn#3L`~KzjC2xR%){MfZC9Rl7kV zWbA5Ux0#T-;pl6B7PZX-)Yh6Ek=qq>)lkp@9k%U_VzaxtFuQ>+raw?o!C2O)_aq^L5}W2PD_SqJPYsSyU9I| z7$<>ONF&M*v2FR#W3^ z5ibTOOwH14~zEi2%=1D88Zou5OeVv<#+B@#vlfn9kGGEpz21_(A<1g*ssjn{h zd4f7?I;#_|gsqWbO4kCdRbFU!8}b-}y?jtsRH1>%Mb0RogfaPQQ?{1!F_d(G7tk4{ z9jQmNS!ORJCMDpSe?IS%QV37?@2t1faC^Ydr#11$N1O<9$gfNCvZJHxJf@2Nk)ZHl~Du9j#p7i{J#q>PY+g=wV{IOLn z?>gO?Y98WF9EoXIWOMV)xbE0>WFu_y?raXPrO{(#4VZX>B`WTWC>E3| zj;tzf4R|)xbi*mU&^V)N7EjD-M!Z{h!1K4uN`>7dZvjksE)UP;)Nb^eS67{(5nikt z*H@_zAF(j2eEV04i<{N7y8lYf}lvjoh--KS|EQ8X{O5&{Z3dSqmVzXsO|3 z1a!4|*6fO-XqMZa%Naj+chC#u3O*amI>-Bz35-wHxdZN@iQxQ*^$Z|6WYQRpu*~n zdU@wtC^Z_CRYD9c`IqHBprs0bR->yVqCr^k#}ia*k_wl%1uI2yEkSY`*v{x{i70|m z8^WV7N+kaj2TV~2Iv}sNfHa4+&kEc^z8?rl7@|>N-_zNJaD^{1@J&qY^n_SX{Gwcl9Tt4d$^-GT*)16%GErH z2c3KYgThJg#RxgBgqDOrIR5D=XP~bk9GxqLz8n$H>~FSfZ$r*nTY7r&$@YkIXgv@b zqV7si*33wONf!s&5mx{hfN5gABn`CACj!L$Wj)gD$vJvG7sOhU9S%hocYvR82fTD? zUcakOv208BdnciZ@^G)hjH@X-Hs?sD;tTT)* znw_Y`=w{pe#42L9G!Xzot8O`Xn{j0lAJxM z^bnnM2+uY>ePR`!WKUEi)sclW2eYE{o!)0({E-gr-TpoQYvo zRwzY!sG|{&$#_B57xMh7&Y)z-q%k`~YWgAHq!dK+q*VBQdOG5zZKC`VUJUQhj22{d zd}kYP&(9}iFf%9Iim}Lf%540H+St=eseHJy^18w&8L4pK=Kd$!z#UcWDBv^y!Y4o} z>ml__bM29wMeYTS+r9I+T8^I(q5`f55)G&jE&G=lAN6KFK?2!22SV9`YS-92r6qV8 zcHr@5{O&QjP1R}gih8Z5TPw9yUbH3crUf#MdaNcYA^~4wJt~t0D|DyIV9ofA^iptN z;npK1jhwebP9|}YyNSu@__zbWZRn;G6q9irlc`Isa_0!CM3_emMaD2JVn8*CB3}F% zb;>j&JY)Os^8#qim@{J@op`F4;a=fc8-DI01R#edYb-0S@z&gW}(3sJ!V!oAn zW*UP(_fL`p)5@o?-ICt}(qb=uL)8S6E#2&=zMvQ*6pXwwb4Ocj?PUeQoglx`anJG- z?hUX*;64#u9^5le{}5R|A^67Dh>YIB9x7+?@ooM&gy#+xGupv6GK_n=Y1N6tp}F1P zYRHx3xFlOqx*XVxpmVg6o6HsX!m`8EL{1qyn>^hthYDfV8bgNzLIF$fnIEulRu*%T zx-dCojTRWbQ;)#^3VhTVKI#>*|H3pp79Em((iZ0^lzJ3K!${RUd5LP+#o;*sU)?~9 zS>XI+I(-NIDy2Dcd$0Rj749)@5B^izz0S7Ry*jM{gMa;E6MO#J=dTSKagwfxQg?v71DH%I zCw=zK$+&jgDi7p96y)(7C#u}omrN(*E&j*A!P@3eeMiTSoC!OZUNIk*z;(^eCIHhj z*|Z$2Gq?+F4Ix|L@i^kva zQ=_#+vZ%VJJ38fAWwVJ!8#Yw*l}kRZzQY?mAmX_s+}9WL9d!bx2puSB&}uHECM{js zYDi9w)v?FjEyP;P_>S^S$S`*)m?>m)cE*Fa5JGA4Qx#9Pjbr!K2;VNk#V8Xkc!;;Q zB57thO(6Ja+MgaXnPFKSWa?>Pd>fi-=0zIqqB)$?`8mc8cpZXLC#i zdJxugKR`CUzXAWIs5%)V^+*ytH01v@gM# zwKW-$!YpTe%yECvV~l!hGe5LLhNf-=!!@?JqpjlVXTyZ(49q)DToP^ET-q_?p28Y% z4ULm18m_S#65cuV`b7YgR`kOB)iu;J_95wUPkpWUoZeMQ0jFmyEr&vaxfvdNH8=!k zkX%ZW%(QGKhBCYz68f~Ca0Fz~VSnb66RtSy({p@*d?&Mb9w!G>yAhS?80u=+gwxer z$sbk9lYMY*!R=4%x?e92XCN)@^WO9t<>uicqNXiP%=lOh8{< ze~<1wikiRn!wes~I-1qI*W)K3uVV}KX{oKk;=9i63?%5E>^+v<}Hox^=j7j zXO&PXHt00FQ`*>75}_2)0Gm0Sbs5oxi4Ze4-tsJ|(o8=V8kNja%JQ}-mR~d>*3|-| z?Z}^1Y})GBB5IZN>lLEK$%96gFczo2rkdoF4H*tyD zS&p*4OS0~FioV%^os?xc6oqtU(nKsO|)So2p&j&b9E96)T+ z_|YzYqeJD$X#(|rICcQ|eBLnV6ls|fzyMjh2d`kJ%juYSFd;pP`$$vNZU*0KV=^r# zgKMhB+hWlTwr51OmI$sTC=w>o(R7U2i_@^v%zbke8Yg_oV!}aPVVTb}w<|>2`WBaW zQ8qommYvK*g1RInqggVKE%r#N8*D{!T{w=m*x+-t>=RwE6f}aAsVXVL(5 zPhK>m4V-NXyNz}~yfqA2(TIj&X)4gnqjSbGI5!=7BZ9janMBx~)Ji_-sr7P?{+6$P zP@_mi+dF_Ov_~8EDAqknDEr=JIc#uw!i@rb+8Nn^A|HSNX7ESMnq?o;lgDnpCNSRN z%Xg+Fe@7M3*bzy8#Lm+9ZUGvYnUr@W>1vVON@qWxCUUlRHbD5HkLfd*l87dU5)+S%Wu~Rmjg>l+3UZ`T{Zi;ttHG{ps?&rx zi`vVS=owVt534h&|*mj7bMP4Ai+ zLUCRGi;zU!D}ou^GJ>9pe7Li+>50YyAT?H>;4UiCt=>A($Np0&J}OVs)!92@Wvog1 z!e%}ui*r=bxMck(KrdLW&FV(5Q(bU74ZHYmV2B}Ggd)l0{!MUQ*3)@eLggdHsTj3%rc8tTGqUxCr7?5_v*<^sBI{sN5=62X8Yv5`^Xpisyv1NGy4>aQ-&OTCX ziN=^7#MXX@`ev$`WV7ZcHwpADA&R2;0lI5>&v7|x1lU%O5gjc-S;s5C*p~hF{9N0i zN0a3Utab11y&zh@yqW9Xa7BleT!W9C#-(6>mCggluG0qS19vk#mgapV%}&q}AhZ44 zg-5VViFK2C0LWenHcdV8WSe-1m!vf!B1=LZh?DQX+%7u+5{UN!0sz1T0|21^Uyt+z z&Gc<-jII9XrSfE@9mfR`_^;^{YE5v7EO-fl%;q$eWc~#4b7Hdj?fYE(_8tXW0q>rxyn{68n7G86S_ytp1DCn;NqSN!|fNZHy=2fe`__dP_^i{@e6s!!b;6O(&5dM8`@5X)sn^J+69f-J$(Pb zP@vv6cTQFa66er}6}*l^4UHP-*f}g2*eYpL&}=#9msdh~wzv=&%1?EAvgo~tO?e4X zVn%UW>TTH~89*PK6?N(o=&e9|bGGb5-3tw6&Ot!O#?YFxhbN?v;HcTphycM}p|3EI z7sWv0AhA~(k?n8ykBA{wLWj(ql?w58$cj3M{yEUMHKJHoWeCEmyg6@=r5tiO(#U*{ z{W$Qlijl{Xih7coS~~}Lm&?G8bYshOwuynh*ac;~k=@>g{B|{~8%4s8gqt*l{jMsX zbi!(tio}6o!_`j3Ax2*o;AdFHQX`B7B%BPidg0JxeHE0RElxsFZ885*zDmnHtJ4aFxLT? z5E4F@yo&*$^2PsAtGyZwST#T3<4;TMqG4coyQQuQ8drzo0I4f}*qMZ{4bN;VcvhJw zOnV(~^9+V8_KYlcVCjWK%(JYx>tB?)&Trtt&=Y;gvWQ(YO7b=;c8RxD%OH4yKnDUR zE+=EK44NQ4W(0Ybl{$GpL5fV$I{XT@HxOlDzmSWku@8$hKWV;0nCnC?B!c`b;$luZt0f-E`+S&wW~p6JrI0t#xPHx#0* zYMdy%VJ;W~0;K7KG+?*gV8>++sn50CadW=yInDXZ^Le{M+w+4hSQ6Dr>PX&?5SR9d zK3Wk~XV@I7c2_yFdxh#vCv}7M@IV@g8OcJ@*xfyr~M{3h*TgZCm}61g&(!3V3!y;NsIGb|mXc zm@>8W7uci=6F= zUh{_ab0cGsxOtbX0&5;esKk`=bI1j-P3yjvj>e;gxODw&VX<=g1tBd2Cb2a{yDh%G zChkm!rG%}Ea<4|%eCyHFtHO7k=wWuklGV>%bu2=%_0`yd@D}Syz2!l6EaxoN+n8wf zpUw0!5LjqM&ImAAHv!RPH(jYe-dz*W*y>VA%4OuX=5;tFxGz-EP-$lpcfTnTrUv{@s#9}s{#thcv+(6VE}!W7oqcfJ=Tkux&G|rvGNOl8dZkkG`Zdqd z@)TvID)D!q#i0}}c92|fCZf4Y9}}63DXQ1o;|_0Ox8cGbVt#Ix+A~d5o3v%rAu<0P zU3^O)DU+aejCo|LAw^kkvAap|=L3wlZ#`rNjS~ic` z({OMkotBBdEs(>4|3&X)3~k-Fe#|9e>?b>R#`tvXsPYl8)bo>%#&{G|=C`WS=*&Sd z^lPYTYgOl(ccrZ{mL42+L zJ{HbzO@-GOrVl9051l}a3%PrZY5FqwnV_>Ra>Q?;Y`dw=iFxGywHnHEZ%rdooq zOmgg{zeHHn)2cp34L(beysLOFlwe-=N4r;4_$JH6kho=PvR|lbJw$l zOp^x*Y6YLmPT7O`a26H$s2vtbp9pr?8TaR;X6MTz(oMh1%h?TUUoV#MG8@1pW|$4p zBU#Q`6@XYuTlJeDb*)w$w1y*{GSLSlTGf+?*emGm_Hf!5pe6jEo}k75<;^^mfSvxK zXt!34Toq16CfY|{Ufwrwvrha>qTNlCk=xuB9&SW461dDSyYE{3nmz?zqc8~YLY`11 zCx$s6yzNLSDQL<4syQd(g6*wc1fINnya*Piw8BYN{^csS6DP^v5oB)EU?e){M^Aty z3g+Idq^}kWvmj5Z@G;B_#@k$l%0cD&oqC=W}LX@u#69xg&KHK#}&ASWQ z%UqZah7WprP%p3#Hy?t^)Gmb%ojC-jT|YUW*F>)IDA-dnP69^U5sk3nP}$7-aPury zjO>ND3P09n-a~TUNlDN$fW-i-d18^%f(*ErX0-TMp(j|>G7 zT<29jjGdGOf?N*y$jxMWlGU+0-I2%F{TZ+~x2$ZIE+Yt%Fg%g1IN_=>g7HO?cw>iR zZQ9}Sr|vXmYaAbcUs^;Yx~e?{5+n)nC`^lqL|g>ozF~^t>w(cizCc zZNbJ3=V)#y@~-;n_buj$(>k~6kM`-+L0%+YE&HA+im7oVm>dYyl)_~wtiLOcqlUH? zxG!Vw0AR?qmVZHcL)Mt3^*07!)I3wRR_tMdGDGX(*`c%({sZ?hk z_g?|;v@F=8=%Gh#sP-)bA9}P%8L-L-)GHo-sTz7Ph=+6WivQ`(ryE*WaFFSoBdKoe zqBl(~fu0wsx_&7C60>eAh@k7#1zPyZQ)`P8^UjOjGo`}@@)a_RQtBHK3&BkVf&aXa zL1GPx@CImcOSSc>CI!KF*3UZV>+`YItlTh8uc6j4d7uOOmFj7xp+P zA`wz2yaAS&ONG>&yB}C150L;aLV6YDnq8U{Ne%=4mZ&|I6l%~I!|SGR*qG(3tlYOS zRr|2x5wAe;j3e3IcN_?tIU6r4>a! z`QP>fPoe2J4&Z)^vR{icT2~Wr9cFQRy7YWbw`At_bbkWap~Vt68Weye$`Nh$B?)z# zLm-+E-EAF=sCQ2W)6wqp^yHgCzzI=>g)s&pYMyN+S1&n-sLj{kM_6!#RR5eNa~X1a zC@3k5DJN=b$;hs^vL?PbO=;_ekGd$i<}&&%D*B8bq)b^#-$?4LxKV-3n9)^f0)6L1 zQe98N7lcQ({>5-n#T+6Q9yX8vk5GyJ=lP?!b0;%osTCTj@W-3i-ZX&~oojL;r z8ehcZM&o^UeW8YF`d$$(=ZK3E^+?xPcb3ogh&a}W`wY*VeTJq&6FA4I@U zCq&Bu%dkGzH#_&i77`!Xc)uwT%}7wx>`Fsd=?g8fv*yhmX&_H)nyfTLqjD+TGJ%Mu zr3&-O9S={_qiU{SnGcO?jNlQr1*6yzbFZthNgWN_| zR#pu1x7f6e3ZExU$bk-`kysP{|6&6sw1KE!plaDX}gYHkT z2Pi^(7cCw~M-zX8z$^3Hm?4N9A@&tUSkjHaKtm`1KS@i{cB+<|iXOU@%2QWfdgNE~ zfc+3nW{l1(@)CV=(vwUBYt?>5^1^XLj7|CibYGCo`80GOFGuSX9@Q+dprZt9Q-^JQU$w=8aDhyn^xHv+e{}obgg5v@3nl0l(i{$$qZ-FM9O{G&{30lPt8@B zN(!P(&UrW$5&0SK;~vL+6C4X!u{+~G?dYeq6Clk+r@GaZBXijBMw2q& zsm9bmqXX5d^5_O@>ChN5=K5zHnk7Q~CmxdVJ=UHQ!Lcr|JS#6Eg;-pY%g?%7$I#}e zPK&GRHJYJ}J%_B@2A5Wp1;6=(geIoXN-SoPlOoxZXSDD#r0yUfx=VT$a0+*QDFl(c zwY3P}yaj*XHpi7l%NRB?t^?H}{UT)WGAmUsz)th?6L*RuRt?#Iz9BEwH@%dr(9d z)SE81H+sDTzI%J22X(RV!Lsn14xi3P*GvAvl|}=tk)BSzJIUKCbR&NP^k5gt{rFK0 zV;VfOG>@U&6625u1Iic$ zb%y#Rz%S$+w}4bG03iGUXz0#6QTog5+UCrn*_e^it|gzHYwwQsp6}NyH~?G2Ex+(| zD(F>pts&+>uN9chUQW>0ZRXMEoYijHluQnu(Yz|d<=)UV#@`4wXKALh>--D)dMF_e zRTmv|KVPdP_5iuBM^$T8w0G3QH!@{fr&+o04lDQrJ>lF=Bf1=UAZG#=>_18lL=)Z+iVIFfej=PDIdhdBQbrDxa_KW%V;iPcTZTyzEgWTJC2wO8OsYZtZpJ;;<$^O){1uC77cSsEJz_Q z7S29uLK_kd0LcU&GctRLI>i2su2Pg>d`9fkNpD`0 zfAyH@owd3!@P3WH(47f#1@9WY{l{r8(t#kFtjKy{W6Ycl1jHMl$gS$H{X<%@53vql znFS9-ad&z1))=ZS{)0xW8{q@!=^yimRZic#KS_{Fsh{~OUKkNQz1tg|rX?^N(DKaE z&~-p^5-RFRq}qx|cXn+?nF!?XWbk&2e)cK@*=mja-T<-F|YS(^Qs%FV~i zc*S(R;IjcmOg{j>$VRx$ORANFLbDo7XRcdzUS00*S6XlZuJ`MLOrn_ee?|mxrp1l2 zlJocfv83k*+f-^gyovDU2`e)7o_`oSf_;db0b^4`V_WAqyrhPU%s;9j`M;`R(UVGp zBAACg`N3%l`^bMQGSn3#$e6N{zLDgQBB{{kGr9^(VD1`8tD`I~EB>L#JCE>u_EL6~ z-fUN4hggx&KZ=YbfyDMOpP)F~Y$!lsW~@w)nK&U16q`%07tH&d^EGUDbIkJW-6av+ z`TikZ-l;LJnj_AP_yBpWr)LD(#A>{hN9rt`_N0H=}R;G>2e-z2mXrA>qMY3!ir#y`LN0EsCRHVk-@a*ynTS)%j75Vr_kvSyI)45B$pv5Pb-8sT_ zd8rKJB5O2BSI+;&$jZMlk`}GuUyPLYFXkfo8zWKQ+W*B!uxyh5!ASAHF%kxl@svQa z2Kc`ix%m%9X8d6!lyjQ^a|;%?&UM+}89Ay|`}Z<9cVaxf@TUxl{9$BKm-LXrA4Ven z7bE{5liynLpNt&-FGi~VVWdbE$R9=`JwXX1XurMm(MN_FgC zjBFhJQw5*wM*cAJN;eyCdj5+rD9*RPFfwYrEY9&y6$AtO zPeSvzjTL`snkXWv!+(*a8zrpW3WT)=L{##VLQJ@AJR%zD3o~H?vm!T{UA2cAfd?jH zWKGJQ%ifNWdhhX58<$tl&#U?-HF?Vn6M-UABr|S#x4&Q9JLle?-TU4EW({RQ!Vrp= z(Iv1&Gg7Qgm$Sv{D%F2hBwYParM){!2$+CCMvkgz4WLKq(bwt=0$%g&wD^15q%g(s z)$pHf6X(jtrdlKRX0Cmt+cp{g00qWRc7NQZ2Bkin`OHk|i(ffg_}Uwp^N$+Q>Mi>A z6Gd$n9Lfu~!si{p!rRHcrA2{nPwc|oKX~0Hj|+kZb{v!9)J&p}NJJUX!ujv@d#w!i z{B$(*Ehm4aD>9m(s!m4e?kWpy4WUf$q)2mku9+r-Em$!hGVXO4?=)m(1ToFc#+Yf4 z3s86XtJ0TJUFTdIxCViliS4<P$@K!0@LCCZ4)gPjZd8=3 zTPTJruXIXf?sh7JPutK%Z<X?*coXk_f+94Mc_ zV?yc%^sR{2qdB$^FwBKv_tntZ5$5A2?=x8OTGZu+hI0ou1i*mCP=Uk^URwcp<_vay z(roOI>gN16LHDubSmN>0=b`YM!;Lz|k+Z>GBA0u@XWxa6d9n2Z@zbQZj0M_5?HGWe z1b#{JJpZJ3=l(`GR~#(3_(}NGSoM ze*`=?pYOYd001yY{omHiNzwD$=v%pa82@w1cS+ULPU#5c+xxwJZL0&~2b2}_D4|$_ zv>Fhi<*!cN_#h!f{@xTFmQ=ev6V~f(;QbB>-Nt2w_lWA|p^Qci>jHXUeuRx58fI4$ zZ)Mrb7)@7Eikw%C1$dv{uBn})A=QjubLofIZ9A?wJ&xUu)2`Q!H@vsVerqp zLMt`9IJA&N0YAJX>wPvtJ?(P?ZeGAIJ0QDBnXi*Wu2W-gCXT?6okmk-KzQYT3;Jgz zCqHIdtWarE;8B45kle2;Djuq8%2+Gl#DE58W2PjbYoo|9i?W-L64IcJB+@WQnJW@) zMN?Pm@yPKzksvfty9X&jjhWVHROxLZ+RlSIRK}q9c9vW);g+wpW@%$+rodU6Y|`kD zyEzk1ZCVn8)JjmdRaRarNHb~P$|D6g_A0cYUMdus4;!i2zr9*l(YAyy!@*X|E>?f* zEQls#WX2dXD`GG?3s;${TA$WTpc@|-SfQq)_2RffEL-CgBjdf6^Qv29*r zfI|Gi*pTutAti=|Og0v!X_AT1l=f`8SLaIZCf3_U?ravyv)CIIg)hk)x-^@&G896r z%3>AQHIO@7ogzeqEk>NrI;82G(n1I9Mei-@o0TQ5y^Qs3Y9ykWJ8K{=SYVuWuf|=n zjTgHjNTJ}QU?&hM-pJcH00}XmrtDLxH z9V>R9epo{ojzS%R0!Nhg3Kpz%|KKGk!uwQVF7%V4W|)~Grx%bV>%N0>3fIV1>6R!b zAy$wgXIOt*9m7~rJjDhaGDrRv=|gE~#!jeD9%bY=RqgzJPIJTwovH)U?M(5~0b4!+ zMY-;7>Lq>QI7Xb(y&Rm00mp!KW(SMP0=+-m|4FX*qc0SE>+4h_#6P4H^Y?hCjPLBl( z5l=+fNMgE;EzwCuOxZheidl>4(=3V1^RrW!(_(^era(5px!_Suf-y0r5j-b3iOcF5 zOL<=^Qk%A5oDv$m~t-+UCPbmpDi969^ZId~qU#Sf_`^KhMT1M|A97jCaa1F3o^ZQCIGPg{j4C(&KH za*Lj2#U;^Xx@-rV3LiQH?V%t|Ae|afk{V%{8gZH$g6e8TXsSJ&)r%9oWPP*k!3T#w zs3{s!n@nLgn4~)x!;Xfa!4&<8$KQ41m8D5^=R(7(8lwX$j;g0pQsS(xHD^;I9lYg- zstUw%#<4+Kj!1Lvx9|F~te5Zf(3 z2rLf7K0S~h9hj4E(6G8@!`hfktMIpi{4Agw(o%-YNF>Nl&kbXgb3fjee3@1Q?BaJQmO z9N}?z-z6*;)}x`{j)}m<`@f+0htKZ^|4~t=?sZu?O85sU+u3SDG$OA6X~s|{*oqtK zv$M?A$SAZI4MPJO6z=wsvDV#vI*am+bNXA*f1zDnWJr+TxCWQU=SGB-mdk=utc#k&IeEp%m= z!b)6G!E!v*(zaAM#kGj`#Hdp4LZlf)CnvL!?N*@!7+IK3oobHM*I>uI_F5!!CvG8f z0C6TEjNVq90R>}7U5!u|QX){-Z}H>jX?G^aZSG|}Q)!%&_(?q_($xstza3Kfr%Zdy z`o?%5SY|eSgAw11i7&|Vi;PkTJ!)}Q?SZ2Cp7HU{*!jZP>0TU`b^Qq7>4|{tiJ4$Z zWh2adVe0Tl4yW+cxC4KF>lOYGB!8UKI)5O4;t8z$o}H7of7J1vKPPPdK=g1}nkawx z7Nv)3>cPAbqCV2qLzm?UmdICkCyIUa_C0zdwm+)-fqgNuT?Bm(&G$Xr1Me55{y_ie z-b4Hgn|##UWBqW%JB~n(`|(eZaTkJH=Bp7+-Gv- zn_j%^Bv`@5ikcFq0&cPaLz3R7J7=4;%6BbmX>k^pN%lr?WI2M8K^!Kx+YO~2w)qAb z5L6b*$W8zChR}v1>PRR7_OIvV)20uU@w*Q=+-8J4v3P`X1VzUW%;I@mQ9%5)sdy6C z-l&JT;M{1kydhYpoKRvHlH0KyQkAJP?Z0BziaX+5G5`SpFoFNw+kw5kwV{Qnp~=4> zYEqMC<**qM#(#mcb(6y4g_hFfA?X%SG$0+NRUu1DYfuOY5W(%t)&bLFaXI%Y;1|;m z1E^jWz%PVVb9c`aDxaDQ7(xGgXvf8^iDN z`7uY?>FFc)q)Cz(HrBTA%zw5oxsBRRC7!lLih9x3Ono~pIKNp*aAeRF+s*`cq!WEG zByEXynP-J`ii}xR#E^kF5++68$ZOs-8iE^ z{eyn2>HK00#W3DBvzHzesAcTyzgnGs<;VLDWo5%>c$Rp~T+yI=Ks7*_kKY)?%wv3i z^~S5h2#?Y-@KRYIO7mwn=P|Dtly}^zuw)F?C+=Lu_z}A57?k=D6ops+`(G}^R6B&} z|AT0@2KaA~Gyll!@c#sw6*0E*e*#e{p|Viip$Li!LhQ1*@XLmCWtmO6C_UuvbRg-x z1MsDIFpE(JQ2?hq`}cX9o!x$)q7ERg0drXVy?EC!wlI;BrT6gKXWxMctJS!49C1#* zwD7~ZBw}|WBC>nD;JMr^LN-=K!|3Yg9(5_N_IW-BFHYCf99gO?&_7NW2mk>6|8HF7>|K;iUH;FI|JwFQf*8ER zcAHVpR@zhw+O?(7y)~?3=rN6KjD&?f0$CT^0LWZj=uI%OFN63;*?$Jp`7WSIvi_6J zWuGRiYK82UY(MAcd^W?`{9m_?ct-@GC~U5s_J-7;z@WNquH$PJ71EPKaiO|lC&-u5 zFDZu`bzp|$hkZ`M=QoMSufC+bhYqLQ;jvn}j-*q+tJhnViyLaEW@FlUMj zXfV#Im(g1L)=>?E*>tXG)q>I8#+;&-4Dz6xBb#DsLD3a_#cW)(0f%;_kQQ^3S z7;NMXa|X{W?%qQ%Fw!Nu=qAgl_Z@ppj(eolVY)R88hGlp%7S^xZly|;eMAe~Z&`uU z0~UU3$v=>wwrj^{HRJm~%Nd0Fwd~_P*!Y7jP*+qbe&KFy(W~oUG2C=-+;bJuVM;Fr z_-!N5m6y}U;7#@FC636KENYI~gv@k*dgyoCAXv@mV|_U+Ym9}9!4Zuph{GUw?26z;%%l`d9E2WxscFZ!T) z3PnQHA~HFIKcFC0&$mc_7XMibSA-&X!X29?ocVYImPrkhfHQz)_%@}PJnm02p-bM0 z)p+y!mx!B;%=((P{~bLh`v0>`c_$N7CsULE8FTX=GbQ=2@5cXGlrVI*_^0Z>?6p*B zS8A96gLjjwd-jt(_ZKQCC`tnn(WEG*hy!DLH0(_iopq@-tE?!#lsyQYF94p@Z}(ZK zR+ur6USDpG6EeJ~hmRL{W6TJy7-M=^U~nWPQZg*i_fpz-AVE);+ms=vfEGinY6-oy zVi~DwQX7vj)atjQcgjp+I04%NV1C46vvSW+(gq zu4f_bPJr4!de#8_n>Wh;i=O{OJlX$%1Nml;jg47Yw`lRx1qIbE1$b2BuLMzqcv(<- z?37t!%OY=*jP{=jLP7li_@{7K_NrVLc0e~XH{0*c@Vy>Cz2FWYwz-xVT4^j64$K(9 zRbxG3tzpP$^FV4se{r{aj}Gezi@=Pv-YYL18&U{`Jx5-5FJcN?^=}v{SPP0XnAonP zzjW-QHBD~kNpt{(d(+bGlKjY3DbAC|HQ`RwjebuguN`b8HR5;r8vdDRw=)r!Sgd5b zK4!ECfePM(FuIJ8W+$YS8BDqV2*92YFz3h*^?7?shOIG9K6F6C3hU)ROy(HI{BT3y z8HvYg!+2bE!;ria;Sb^e)-H`im!TzJr23^8hx?9ZNgB0Ct}{WW=n{zHAyCH(O5H2i zflKrX{FLDY1=}PQ0uZgBAdCj4Q8J0KzM%W(fj{FfV&SJrDuT1vXGB?X97mbwzPvlRte z10qL>rEy1vLso7m@VUgetul zGL|6IkZE&L&s*2e`nL7$$M?^AIY1aAx&U?DC*sgHpv*x=&@te92}u|-U{1(dfMib6 zG9_nD+Az11hdP`ie=HJUR{!}p%N)xA%^W31{XtMb6}Q?>4y}nRxj8h9B_}uXDlNIS zc>QhM7U(pic`QY0QTeUT)=ZX~8INO?5hIZTyOmZMGueY!huZOvCE$Ld&J=>hlFT%R zYDO;OO$c9dDM`s`l9!uSyDTR?r0<+5GkwE1Xrc?Z{YZ^VoP{jrFsBvCY)@u$i39=C zUBAMvTPNKtwq>2ADdBL~(DxSVcNQKkhZ$Q{rf0a;izRP1Zw+}pR0Q%@*4G%wS0!op z!}R#GMV7$R7R4=fwWDvHP9ko#i+J=?jh$%`<1`~v7%!6;_+W?dn56GqtMWqmfv?io zY)mOPXm*}k-DF)>JscWYqVuy&kmg9ua$6!(GF&pkL)c$aM4Uia)6$MI%g!Cu@20}? zvNH|1L8aDob#;0>EweE*&CU+TNvl~c(!H$I0Gr^*KOG;9hd@s5Ad11KxKbw_R1Kw5 zbfrsPDNw{va7}>2jF$TPZKxN1ji|%p&Y;|uG8s+dhY z=%6XqUTp?4rJM3Q>E|xgm@g_)1++D_bLJ(#Jk2g^E)EH??u{EJH$;00)a8VoGd@Ud zcBiuC9s}BW#M@s*Z3nHH?k?jK)7kcwH9UEb6?F&==Z`l!Hm9I_Q*Qzd+8gv=>jS^H z8v~8O&5$1NfIx}j2#JCE2OpWt@N7;tA?d~`@nZ;I9=O>Y*7R1z$k~8z`Oj0ugxw7% z&KtSFFNvTYctd{+jvD7u<2&+JisUAd;gP&5NN;})M!0_PC^oJ82i@-w*JQoMyTJ1N z;%$L-hoLLBudUxc`nzJs2J6NZ@(T8XnVd|F2#vTf*=J>VHxg@ zNC*-Ht&gOgzwk`Y_D#tWt}$#B(r2FI`gu)devbEkd*AY5>R!tWWW#@Yx;}p5GkbEM z_qFocwPEXY%_N)0RQsW^_K(BNBfSM>Plosg>ugWjC;wJ|@0R`ooD0u1Ph^9bCkiTg zi8SBQ>KkJv1gjz{!&owCU_oSt$suV*S6ui40{%fN-jTLnsG*)&o4c`*UAK zkiOA|CDD3!6(afq#Xa5N5!=fc7haL&QpLBj&+@0eDz6`j^EZqOzWc&_if;LNcq)&_ zS^9n1UQ!--RxyfX58TIZy|{8HP~#D6*FSKn4`r1^x;6BJu5j{jbOj%fS|E3-bk_mi ztNds8$mj*-7RsHU>&LZ98vw&`%Gw7F*1jkI^1bIf>?s6~6#yX5_uu%q|G6mtU&5fy z`hfeWsJ=dQzV$INdrA^83?zi;B}0-*1Ox~K1Qe1;07)ofBms&g$>d}z+<>Bb(*rgJ zV4S%;JKyDgU2Z6Q z0}v?tTm=#Lv_r@J;gj1-e{(*v_NQ8^kXY|b&%>nq=RoBWI0wFXeZY}78Ln`PLKUWz zD7qh}n0`pd7i6w%TY?;|DCb9oFWbu4vZg`qa>l8GUHlVzH`N&Xq9ZyrhL)H zQ$?oC2#Qsv)CjBfOu057>XK#q666DhT;@3RSxW?#S=RwQUv$*Qi#IrmbA>BC%JZ2k z-;6I=Z@y?9RE4HwNL-oL>9AzN+^We5xXwJ*x7Co9U@#jh4R>ch#ZJ1*)PG6UP< zq*SG*NH2Y<9osUTeNluRxzOHmQ1(lWe9g}JsC^qUfIH_)PEVA&X7z&$UFGoU7->Ci&s|LTg=>B zhr^Cu?tSqp!Ud}jKdSrUb#u$R^U7aOp+RVtbygJ|1oYLlHLgat1T@dYt{#zw-^c?m z&GUMNdiBa%R-wV`BO?^@+gu0(UY^1b`ZFW;nic$c3- zXt$?RxF2&bKatUKxF1Mf{!+*vA?0t?bN}LVbFSZ?qOMdnSe?_lBbRqYl#EEb}6d?$i zCOmR6EO3bQph9v0N{l!n-a;VP5e9OHUlMC6C5nu4gGjW%C*LUoM#ZhXD1nZbc0rr$ zk}xbtdq-ELGaNW$ionOGffx5VP1NjLlIj=e0S=aH8V*_#;Ak2^r_@(CqK69s-bf5= zHfgjEWaXnfi4Rc|-EV=?r?`PzOBe!N9{xodcmyHG1>1dupq|O!GQz*7-h%@E zYVBp9Nua~yAk^TbSm__-zsjNy8t04#mX`9DD3uWBRf9S_)(Tc9;yl9N)a2y1>fQoG zPaA=vz9NRI4jguNMmBTeqQmfLN7&)je@#JWvoeE&6|1xHbQ5Gm<%dWBJC&&#!oy7{UF_8l5q3CX@9}g#DGeH$3cn6Y&!%J;cG=ijDiI9rZ-sr zq^`%HS`9E)^_xWE8(yt8(PZwrx^dk#acG2n9oB;{B_)$PSObb_q0@_|)i8+8=_d)x zY_Iu@@-6nwP8(6bM}-&%jsBSNSHe1!u(sdgXx!6o1b;pqz)i`rT?oV%9lRS@R`zoU zVUh4KqKFha83`*U@Daqb38QCqEa->r1CJ{9t!)D3tRi)722LzRmcSPYX+QNR4T?pb zz@w?67Zo=+AiLEv#R8tkHG1X39 zim{@dEE2me%0l&;{9ulRgqo0>wcJ3i7?RrCLHij3i`Iw<@jV)Ov-1~4c!AW+S|d`% z%wj=VM^b=4N&uk3Ub(8pB<(|0SjZHh2fs2#ObSOEaKbvK)nck+>WRWh>5@Bh+eTg` z?i?KDIM|7DC`gfsw%;Zr3TTB-DfpC+h^=vWXq)Z�Z6K4!O&4dnTe$f$X!6K zN|{op-ry!Ks};B}&d|v*6AJ&_^8@O<2Gq|*dd`!mD0;F2 zv(pz9y`-|Cc};(xj{ZglE=7#OD{oKhA{W@1ib5LEiiEY*x_0fClyue*(ujh^L~e-< zzLqXQj?zQl8F^j$SgJ*>Xe1Xf{}c&_cFOiV_7<>RbBstbk zvT7aWMI;x7lzuDHhQ@N?ItsywZRnQ~;^nL2+zzw7xV1+e&#YiII|u1N4UM}Oe2SOg zMe@3*P zZJ%C8 z1PQmnxTmH~og@T<)*d#ijVrzSyX1U6S1N$=(uB(*1qmU|uJy@<{Dq1b6JP(nFSsa` zz8Tb-e0mYDe?_ksc11iU{Q51=m`a{re&TOh(KDy3XDA)3B~-EWK@C&)^$Va))&mEv zi!tSGvQ|rj5x3Hr&wOInfNeKY?J~!EQ&#D0mGdV_vXRCWD=yvfCAjp^`@viZ!!A)$e1{ zEICA0F2dh3RHj1nke*NS;QHK7+Q*lq-I+OZG^H@1kC%xTBSf}r5n9%mQdjK{dL^7G za)v4qdDxFm;o@LW3U03}cJMNW;*d^NkS1!hyI9Sk`eVeK>A z@;tF7UM0KXncI}I#4cyc<0)ui=6R4K)h{!MwL!;>ST^KH3y<~}O*QRP=mOu(9xFuC z4Sk9z6=Y%Yxt1xMoK;P&>!LWiuWTv{3x@8A_T{hC%${Z~hnuC+H)*O$nySozr_=Ox z&KuzD%-YE$0Hy6+bc;kqjzmKZUZF?Fhq3fBdF0A9{ZufAz#ze}{Q1L|u!y=RE(^?^ zB`A>A^+pSq5MSXFvQa#So#Tg;)WYO6t!B_k_a{6DpM_}tS}p@cv?*Gicu?0xxCa|x zD^A2HIlSd{fx~-kF+SKNp}!~t%4DR-mnrIVrJkPu&mV%%UCacY%f8HL?P=6cQhB;` zL4D`SoQtX1gp$lq9aP571QF}$6qU-9Xx%UH28q3${a6@JmYecrP|SPrEBzCL16#oi zY@N>pxUZx!<&rn<#Asrinr^zDun7cUi@ZQE>*1{{qE&<85rSaS zMmI3)`5mz90WK^f20!FXqX;@V#t{XtNu*(zGgHj23!{j`4aFFPCm1){?l}W>#+*Yo zH}0*}W2IcQc{UI2Jd=n-qm;q1hDI=EjUGZd)M8l1u&72L!Z~wLFlrOgFpO~~n7b3Y zVBWbl3?m*Ydd5IE{=ic#qV!&a249qT@yfBVv7)KDGM})w5`za^gal2#-CTikk)b}+6vdR#cVQK2q+kxwqha0XyW^ux4G)gzpQK}16d=MV{nLn3_;6(bYV2B$O=)HL=i z^dgT3S6o56ml3E(sf2T;w=R!(gmdWW7DVZVbtveH2GOyPkc2KvkC?zLYvM}GcpOh0>J2h-7-Z&|>pfZJkKxd+=4jqD3D?@?C4 zypnlOBUndPT`Sm)Iyj=k>e@HJR~*{8@fn`xouNqITm}=~p(Zu5VHgoxhyUnT1B(WK zhkmajye0QtZz8o82YSv-z(dNyM|i?~wlUE$j06{9srm}vyX%-1w1murvqcf*+J$*Y z@W_l3!(bd)CvVt9h)2Tw8vTl)|9t}u0{ffvjeC0>38D}7H_g|)VSeL0%tMC9Frs#% zIO38VwlP-XEn~wlA{XYnSvSl>>V~ZLjfHEn6BHTtPs~PfbZ^)O`K$4OL4^5cc?bGy zB@8YGR)BuRPYz)oI(@=#e4jtNzo#VoH33U^HXR(sQGv%I0(~UEVj1biY!}Hg&)7Bh z`vuhk_{hcd2*zx6ua}*lG)@YB49mqdorw#m+TeW09PW)&v_bX=^+v|-j)^Si`N(5E z))6ZW!eo6iMh)`_Hc6B{R7zC4aC_b;4}(kFdMye6Ci`a)qFQn4R(! z{*F|=`qJXPu=jGUCuZyS)W%SY@_DzZA41a%We_3PEUO@0gAubAG^FhVf)dKh%I#sB z;9YCpH?a|Xvv(zjsBJrz%h6{CD-f zU!{Kf8c9s!Us>V7PR!wk)eEC-A9fvdQeS2RYVnC$UmHZ`QxJ+A`gJW;W5dyix@2fXA&KJJ?uYde@1o^`9zR&!!&(U5Fo z&q?lzxM^Nob%1uMBU+tjckiEY-o#nr6t)HJBsx8I>ub5fL*{l$ncH?%D%23_Wd z^&2u$gnE}}E_hi{Y(JJm4`n0VkcakC+ALOUXVFSJ7I|qz+@)&H4fP0UYH9?EG2xbb zQ3&OA|FV?EWd$ibB?pxv|1#}MT|%Qzayxis(g)dLvsrvUK9*28C>6|_%4z$*6;Ar* zLRhAL1%2(D&8uc%nRJmP7R+2F;N`7@cxAnZ4aV31Tq^1g6S07OpG{_74jBl}#B0?w zy8yQqvmh<|K1IQT>_`xP7&H8VPvp|+_mwOF zLUkt5P`FR$hJ2JVn+c6T^J_LeDum2!`HE+9A+Ks{X)x8t(lKkGv>_cOtp#hd=*-?) z2lel$<7H%BRQzcf-3Qss3G9k~s(iuu9org-=zJwqRF#*y36`H#S3IO2|H1smVrS>0 zYE2U5U?{DOCUKpOlVHFqUV9tANtJwt`Yr?zgE*g(_AdKmGP0k#Ix>dg%#xsC#MoY9 zlmu>hwK4+x;AY?QIPsq!lU3v-M8Lz>jQj*^jE|{`mQUzo{5#Dx_`TU3r5!`(=@3e~ zA)J*j%ypV*;7ROPx(~WQ&+6bISv^$EWx5MAF-ym;)RiL}jv_k>RuXHxOvIZ#tkP=@ zDobv7)pe4pT5x1bZ2Sn?3^jHhlg-I$K9*zV@1yL0BB5L^7tz~=kX@&V8uvb(aQsEs zGGgbf9!OBcpn%utf4EZ@IdAbrby3^WBUpljs(6jkc(@z-L)$1noR69MiS{9#-Or0? z?0UvVIi;Ju+PFkt7!9W!jnVV$?RVn+l?{D{MZWj?_x|q^qbmUqu%a9w+~?vT)J+sb z)1rS4M%7*U+M}J3#*e^IR!kDeqe?VW+>eugL`IZO7g}7H^J8?`Tm`#@i8E*Sm@a0` zCvQk&{M(b<6^rlXWbN+OG-%68CwcqIfGyYJCH)UO!nlWqIZg5(Te6CRgXPSzrCv<# z0A9j!>A@vXoI?z7IJUgfBk!!uL{~#yOS~FaE^*@)A zPhKgw1Iuke8*oxupKc1vOl`qx?Iz$?6ORJsdJ+c7g}B*3xf+2wKPb*f-k$Lz(8;Vo zA=6lZL?*ESiU=WlHSwSlU0=s!2;Jhbd<{aK0fuZ^&U64s=fn%!ufa=- zR=+Z3(kJkUu%o48*Y*Ss>D?aGf0+wW5pwq4gOBde+yKQ!j_d^iE1Q*yjx?kDv(YazMT;I*^ZMi1?kn^2HoW->0Z@c$PM1^s-qq!rZEf}*4W;N4)7FfL_?2sPlS4&iBoDr&D(Qej60G0scz04XNM!en9TS zAA8ekOT4`Cle}_HY$PM%(Mtp5q6tCj`T~Hf2q0ohfE99*506#5)}9~vPIc;)-l4=h?c@KpZ{ZLcj={M8ds3M}{GahazlXT+(Z81(LZ$6B{1(Y=> zjww!vUoiaz!?iX9xZuLBY*u_fRoR4WV8d!1!(dxG;b8AhkoT9rJB$Sqa zvalm@nzsobcSOKs*!8&=z1J>$3uYjp9WM52v0a#qg;^or&OB5jIb1pshi{VHr%B&d z1h=Kwdw zv@>RQ#nb^spJ;^(!b$4f44T*^p`e4WOK6m|(oLWev7(cPe@Vu05@BpvKTbMWTCy9x zMZrtAsR<-hiylvml{kA7K&tbC3d>M5Qo)Db=IKZ6;rPaWe}xlz<#RE>7hw8}@YxV= z2=JFX#!P=u)%lQB z7iCy2{h$J+D4hqsDwU)n&KJ773^B}z5M*`)EBUmDTz&|#PDJSRh9!kDiKhEzl52Vj zE}1n+pw61aR!Gh1Ax%_GVoPO*5NdYJH;c}x7#pazE2Z%;!65|(g?0ovJYU=pC0BOT z#IZP`w)wKgm7xR&%^uj6t#V4} zH38go4DVYZ+;&pn>J05mEHpkW6+STThsMK$hVbrcTZ%5}jLce@#CM^Pc^6Ok(*{_1 z@the$NO=Nk=0n4F12!ifeVJz>v~nVFa}#R36T$+Jr?TRhwZVpsk4+*L9<8@i+m)i* z&;!kEt*Ri^)PNhtjbdbf>n#J&DJ*{GYJx119B*^AL4b`sF*dUkHxzvAB|0wUw!%;3 zha*XE5rCOSzPPF&JdVICI-?s^)&UlVtZfBF4j!s+j0{i;hzEvH*&3l8T7mHh^x%Q$ zLfU)?3LmtnJ6*}sN5W+K00Ult^2a7aBz^F*CuYuMa6^px04EO!wFk!9!)ZfYeZZ^_ zsm*cq0i_>d&j5YW1pn+vX6zSwF-=(OBw4g!>;%CA1JA4((G%50R!#a%r4ZT9oZ047 zH!X#D0XEfjF+;=#wKYSM1sfMK*>q2gFso18%i2K3V#%t4c1)@rz^bw}LAaYA09RS^ zsHjZLHr(Zc?Q9qQTGb`4nH9@dGwD89)L)bo5JLi_p z`rera+xSsq-K)nxg(}Nl2 zhyun3yy-#C`riWF zM=|uu(c?oG>1gtSj|u&eHL#rsz);O?D}U-RCbb!vIlm?s3V(gQhdlK*T07 zFRjE@p366d*yon8C$19T1;rSZ&t#PeIk~aY22RIbtu_p6x@Wll)FKRU`(k_wLqYmQ z7*LsvG@+i8RUC&kBX?mg>0w0_MKap@NJh>1G1a(12!hdHYRVrx4Okbc4_Gez#KkZn z^;FvQD{qilfdF`-U{}&NOLm#eX5g}-m)wPou90_~@Qaqe6}RGX5m zOPEHU;ICI(g%13vDuWxfj7~IMtS^RYFths5j$H>ZId*gVD=mkT2XE^)&y`?YvG`*`Y7Ff@~K&m%a3;bLKa z<2doRk=uY!5MS7LvwAn94d;d~H#lJf$5K7Yi7#ErO8O+fjHFss;UX3|)q>?}ac+p^ z+ghvcLJUIc$u&fcg;XmAg)2)Rz{%Fuzji9kZudz@vi_*CMJWERx8p2L#Gh&J{f`B# z;;=Mi0dNB|D^w9r7TkuDS`*0|PTmU4G^LidCX49+z(X~yM(9;56v)B`ofARRBNQES zIZ*Bqiw%k~fwM+5JcLdM0Dj)0G6CzOmJewZA@QP!K9PCv?M8SwNalm}L-69GMrF~Y zHL)5~N+JQV2RCylZcsw?w>bfWf~^5kpnr(Xoxp-U4cwo$12I4O_Oe_*#q z=2FR;sJ$a{!x^)p6qn|el2|FLF{HGs^#Y!Oa@FEL0YKYAq0(eVLOwxx0Y zU;tspz<#5y44?chF$m|dG^fX%jw8z1cu~Jlj>b#F#5L7sigPvrX6IjB+OM$Pp{kLW z%8nBs2t+o-k#D5*e0yYf%+KwS;e~QF_&#$ zO8MraU?k83N%w%Sb4zw1Ki6zJ5)$jo+zpDbflns+RN2nTe9{X* zlA4evnTO{6(T-GxAJ*6zu)T4cetXRF;};fDIqFN7w+`f;x1PwOW>{LQvi=o&Nlk$n z@`q$?m|A(?a9Xf#-0$ReWKlD}4SE8d5xbQV#tWEJ%v*6A0A-UG65@^PF9F1P!X9?4 z2Nj3a=|BS;sq2E~#k=uj9CQKL%A_tn>?w%~<59CO^k%-g@J3q#Nr4_H)PcB7jW$Sd zf{an13ssytO{mua44uNtDAfTrooem)adqf*B37YfAJ(0!ZBXuneWi9Ec$(7k0CjxE z9CDY~1o&0NGtsFut9ophMgL$3-5tCWnSvMcHi42fkC~Pw{D9ecM#i(*g-otpgH^ym zB~%er?5)x!IuYm%;yyYGC z15Q^-tJ+QYz~-`n`i7f;5=SEjNb$MN$Rm=`b+&}nQ6oKWen3+zS3#=Cex&G{jStlJ z6uRrl=3F^@j$5K*)Z7+O6BMEVz~u?`>oS#mx6w%^s;gE?4V@pjvKf`mDNLeG&q)+z zjb?6fk>ahb2%N4a$OHoQX~fG^soQ~6M^RhHL(r@e>LPR{27`;zo)&?R&C@=d8nUvW z<~RHXTu`f)xe9#SxzcJ8A4!B&DSza7oQ7VI<_!39LXcC|3!#brVIc*aT4fIHNff_~Fw?nyRb>_*^-UU?8>?;>% zPOLDYj2OS#BHyLJNjaO?0Ihh;4`l5=z)D)NzBXY<-Z49M9=SwEDEEond2*Lf>k9;& z;x{aLq?J+b3uk`#t2qU)Q1b=hcZ$wUBQFMHm*kWh(Bs&P$vhy19ZYQtQ@y z>o>;M;fF6Xt=8ZKtg4a;*||7VV9=QPyk68@sEeg))EC|5-~fONU13>Jt!Ghe79frb zu)7+#AdU<2y-LWS!3!!@A#B)^1E9h1r*hDG0jd3yC_}rxp{OJd;sd^INf}FDb6oh*eOShM zWVsI4;fZDW_5~vy6mGr&RDT8XPp;uMxv9G547wG8C?S!lm zliJ;;&9n*HR)-h$Xk~Fh8W;F@rIA4t7yRSOBcoQXK|af?(3Y%I+2L<-=}w|smOZge zUuXjdh%FKq3(r*wPAD7KHmU20!DTXsLLE-WaruVbji^q7HD^GzF|4%=ueM};{VI-3H4Pr0X>Vn4lD>)a|=Ro#`Cmp(6ejunJgh2ew( zF@sEPNaB?f%FvSn#C$l>3sOe09@yxWl}3>soVoEn)QeT*BGir$+;h5KE^^cYyN}jc zm-m3>jpi%P(RKjRD~~rI`!*U{1Mm`Bi+Em)f>>6Y{V{h;;tDjZov zuZSmSM05=-#70MurQ-1ydjcK%ruLcMKd(t80O3o; zTDQs17F6e)g6G9&*WcI&l2wXR$AKl__o0O?*K~SL8OxS8Y888c@T;7&=%S3LY2nWB z>cW|q>AXhr&CqO#k-Is!F@9bw+oZe>pAAFDLr$k~9x+S&8ooZP$wGDM~`owi>qrgBi~tff)y~@jx#kzzJ6tEa<|X3i`n5pJXlS`$2hE+ZNq>fiy3= zD)aY)msjHr3g6ICGy4ZXFU%UnesI^T_J@Km+#9ui@byO-?W_0)xG&`I3g4J~>i&WB zD_ZZ8#;<}mpcM-HV*pc-)&%Lj!FDasYs-ubO1yBe%gal&1bG}LPa}lc99i*0uB! zh$Z7|Hp9j^0p}M7kCUw_<_vCwY;a2JyjN>YN49vq6VLAveQf9}*M5XRsU845&k5=f zULHDs4ZLAj6tMF8?1JI84aR#9o-r&o+`VErCk&^AP8jQ(aM`qC(DdUM)yCJ=j2FVV zw?d)2kJk&)@#dF!z?xKX?1=VE_{gjIUafJRX~ok)Sb$k*0g+lnE=oEJ+t&cWEN~5T z&H_EPm}pj%2x?pdPqo}R)UN@nTk{;k)4;G<=y?iYL$GWSZkEA@ueHc}s!a#=TMZvd z*>K>ji7&?YW(s4?{%m2aiDA!oS=z0MXOm#w%aLsI#cc7VWIi|_3ci}xj20ii5@!=R z+aDh&zalz`YUB$o%dT!8(z~EN)r%e!n*clWI9N z7V~U8j`{RaXL%eZNTO{yDTxcGLUzIy^-AtPsQ5MH?KTJ{coJTW3Xt=vMso5+qP}nwr$(C zZQHi7mu=g&vHN`Y^?f)I-S>7!%o#E3Z)T2BHKQ`eAh(6;Xsvv0#mSjL`vH8!1C5U5 z`8)J$^?^l$sOunx<^ zF1m&{U#hAROpieyUZWm-W{uoxe}qHtVUx4*k{fN?T2VV@=GBIASkwNrGMQf-ld59(VQ#K)IjsGEXJ z6u}-C6a;Cm6%5&z#F5(&wb`vgo08azs+B#%01q{T zB~zA4ryDfHjT%GCRsrqhA&M>T^V)5Ya_fsjmK(dw(yic*3w}Hw_GUQwYBGFE6iqt{rt!xDNqqa_n_E1>^1&%eo_$oI!e!A**+{9 zC}@}eVs<)8O!4r!Cq-Tl^@(qv>7Hf=T{A=P?w@C@?Hg;=1PeZ`$hS4^$TxPZ{ht>v z5C3Au-8~DiyD!AMIU(~K(DSMv$isEQ2p7ITke4`^@RhZqiEf0PBBVXRbAMePw7q4aUIL;%ao_n36pW^ORW__yk`UTf?ttLk?FJ453KTqmE7bv`hL1)8`!Z8 z=q@(-A&7rcE;%EaPnS!{L;g79Z_1|;m>t?(beG?QJSz8eAL52xx%PJ$(KkNkV8dbv z7TSj5{v(AjTVcr5Pm`+olYi5?y3ai`&#RkP;@%L{YpLfoHeRD;hO<=Wc_om*F8y*_ zaK%fYco%to9Xx2t|J5f$^k<@5vZh*);{X*^MR5VWxsnNoh7ahSmNNSeGU~rZE?+x8ZPC%eV&Z2ZfTnk3A{h z?haHwiLVy0zhr%2|Ej5eb%-J*Bm~ifSXbp;(o*xXNx}W;k8a`7z>HvV!nrXy2De2910W z;+G|Zo;>j6mx%_Ed?52HWx^8qU{xDD$mCrdeyn~0O@ zx;`+c675$ITxPBFW79wlmFq_StDSQ0!kxuwN%=D^&NgFf`h`dJ^YAF2#`b9BsstyJ zV=f!8drP^Nth~8)GQZKlj;sBgCLZns@UWiNelc#BM)rl$&q6e-MUt^}N-Ci>nQO1d z_#w?|du9`-3XD{=X$POnj5?265yEWRj8#~N7nwfrrtsC)d-8Qtc#~U-UW|6lAgmGi zUFXq-@qH*@=YT0grVJwVt)&gku4Uqw`t`Emg-9}7vp7UcxMaBVjXY6`MblVriJHRY zJQF4T*5(s%?Mh?N7hrjl^MoX?(i3Vu;03x={qRUiwZ!RPf=?9MJn48f*BJ{<>iK4KgRU#t~>bi%Ek z#g4PH3)20Vo8D~qSM0}T_5eFSwJ7G-1~Wf>8B??^lijCK~;a_ogec{!`HYDuVgTf zfcNhyFz0LuukMfN-Xra1@E<3uo%riR-Bg70-b3>u7XFSu=shVcMh`cw#GLV5>QT!7 zlmPGHqm7>lQ6$b~d{_3tDmLThe9DQ;(X}wh3;O?9KD*}yHh$u9$nHTez5Fof`@^AK z&TaK9qB?nSZ*tj=i>)GIYf4Xl$s;lH3OeGAWO7d4MDU@oP`aikd0=0Cvi+QhY-wF0 zDeKC5FK@Jn_e)`x+z(|pLi;Th%K1w=*Y-!{ylYMC_OE}HD0XyhvNc!J`so(8n>Dg6 z@L3L>S3(P|1iIUhs!&3#9E>XThAZ^?^HiMQiR3}3;NksnPV=gzHHJf~TG6O5p%rtB zcklPdi3RwA?!4F=IYsf>40x>vyhP|34{PO(Jn@onfM~$y*@VjS9t)XJ9O&qJs9VtN} z`oduS{KD9cb+4&z_CAX_K^ipP&m z3wtXu+l}UhaY>*|DCTMoKgLa6F@ol6KWYZc`3wGk4qFjdCwC$P0swph{ugt~3O26J zLN@ls*2*r1PA>nO;6$pzKY|kozihI|v6U;AgX9TSrO_kVhlN1|Ac$ICdw)vn$~KyI zP<+Vg^FZML88Vk!R4J(-fF5RN-h*$x!>_jo@LGpe7;cMCi^mNUt9ognPYMr2Rbz4? zvK`eiVPp028E{Rb~#m-AnvYN%tJ36CQ5V**`c^Nn22d85B4pOH8kF*!>b* z??rb;9150K|0Pc>{Gs*@jWXtV?gl76E0P;06v(cS+!XM!^C93)irJLi~@Qhmqlb zMEQRzdMFG^0W+fHtnJ)hy0x}~_Y0Juu~O`!&_@x_5Vbt#=?cf%B`3|Geg8**{Q~ew z&5RKhL7X!@?U|ATdwTnOfH*|>1q1A2^kQIzT!>Oz%0mnqmwC!-p&EtySqHWnL35Sd$>oX9ZRqjb|0?dN76$~O01P5pf`HDKd9UM$Iy91BiIbz|FR9e=ASw%j!Rjc#BvBs`AR!$ z6_kw`9w*^grbIDlmk{v;2?A9Sq^zmIm}=2{6@g_i3Zo7XSvt< z|J&3y@~lB<8xR1%3n%~p&;M&colK2QO-${KP5<>}Vrl%Zkp8=*N2!XfJhC8y->a6E zS582{D*7@Mgsms%U=#s8Vgwm$h{a~YK6aLzYMebjicS9s)dm8adR?#LE+u8%A1TK6 zDTKX8s}Q{Pw7PXmR;nTTrzqjPLfiVuT-{q~-?$vdR(jyD(y5I-7m`HPWK}7Z*G_B+ zQeO8^5~|y^P$P`~AfYhCE^VZe!9@JdG75ZFNpYI&Vkc!Lj}4fTqt3!JT;ZlcRagCl zKO$`!G*;?F99Akk;f4K#JGxbsN|zk(2l#N&NvQ|t_w6`%AY60NHn)#L zQbmG(nmdm=q;M;LKtoiF>mQ{i)9rF6N!#GR1GeDYuiUnz*{74%9;**imw7S1u8`*} zJ3;|zv^v6mN%9zLe&YpZTK^>6rnPP)$;rNR_1GM6y4W*cS!ce<`f9b_LnOmQ|7D|q z37sl0Mxw}9UkPoPsD|mw`?^-$3kl=H_M>DmY$5Y#rrgFFrtqA5q4GlUir%>X5YzL+ ze|kRAekM2QsM``#3%Uk`3B%i`h4~~0@cp93m^C4-lQBH@I~4+^F~N3sFl&+8y1!k>4`qbWXdOMOaBXX z1oAMai@7>$(ag1JVL}`idmk9^OzHJ1moU#VCIR_8M9=k zL?hOVDSY{FS`!b&C$QK5BA065|B7J#69uAfhBp68%^+4u*BV(6g*Ulg_Rvb!wYary zbJId63h;rJC`H4NFvVywXi%*@TM~C?I&5cS-k+2&Bq)e~0H1Uh^Qk2E^<*}4_+;1l zwwK?}?+0$5G7E0C+Mq*S6!VSRp}}yd3=$s4hSP!}MTrsa1v7XrHhBq6ol8n<$-I_C zSu0L7+Ir!f6XSR*DayeNe-CN&t*N-|8(t?z!!EU_xjB4Ab&mQAUuqX``D zVwxx}v;7sl9MVD$(%7y~{h559W98X-4GQ=2esN&TiKV=Gy0rWG5OF%Himr|*Yr49D z(q6sYyZ$8o*|P?2RH?smv+OfyujsaK^07BlIDoWh*e>~|UHd#%qcDY8jYj&9qV%{J zp1pF7V*PU|`i1QW^?tzK&T-qXS?c$bXX9VgBbZQq>RE;7-m@^J^eGv`Oe0)TD0@hJ zL$)}^xOWYqZyko3+6e3ifM_d{0L|xUre`!-gmm@`xI&=OG5bZLVpbDG#Ji0asU1u! zoz|)~|E@DdIp*>C@-HS81pBXc|NkfcOjXg8M^;4nwbSNO5J#ZtIM5ntqalvy*bh@p zVw8}PlGq>FZew?taVg8}WL?=i4kn|CB>6|G3@n_xtAsdjKbm z86$`}K#<3PY06}F$Op{V!^Dsf7&`vzA>>1uZ0@sh{JS$09Ze4kCPY@g(@(Y)Bt-g7 ztimeytsqXS>_v?grPo&&E*t3LY;!G)2>K(HVB4w0zGwb}=`KWe@4U;Y~qrH`KdeyiBKgm})D{rxbd8=0BKl zO^i|MY;*?eqY%l_<60iH4DPQ@7}k>eO7@KQS)0|B2{ZN}pYP{<8x#k$$?h(eb2c(Y zQU;ea>;w${kB8KWkftj**ldnj+NZ*^`EDRL)RFP|v*8w6Z13Di8MFAhGYht^U?S4k zbKT_GjMDnw2tUwC7^ks`EYs4_oHwx^SV3hxV$VpuS`zd9fEd(rx56l6?uit7d!kJo zS(~DlRX165aa$H{UAv?Vh;F6!`99hm^%q&zsl5aGVh`bFoWn;ugO~Xh`MsEEajO`&Y8{zw zqZ;`V*teX(8`xeK_?a5QVT^ANwj*}Zl1c6keCVGa3D0%Fxdfq2xPpNmU^YJQ5j86!{GWzC{}Ax68T$;|7Hn+mYPVV~8+(`S zLv1OAVq=CDAt^ndESoTwDKl}hRNv5~$mu4Nm7%?!8Rs%>4d7 zU+{)#u?V77>H7zQfnfw_4-^C9fg=EUjHoXqN$-Jub!*))A?Y{JVJB^>2REK;NarDM zN=>y*t^><-?2A`rA8_2Bmo zcQPa>KJy779h+{ancPlfD80+^z(codr<>nsXB$(j_NF9<>KVU0Xnci|hT<_Y#QKt} z#Z4TuDbPZ?eig|oaV6Ne5Z;v^i0le64G~HqSl6)6zu~bCpwudxJig`?)2DL{pY1>l zG@3t$?x69=f+&_jz+NWG%#n9OK2dt8x_~sZ_TK?}Bp*RTG>!g|9pB*{F;^Xli0O{WEbyS)lRWlEa&5aKyjsx5 z?)Xz1pctj6eItASW`fbCAuWgv0FWc_-`d6hw=(*_Wx=(f4N^v0`OIG*?(BX^8YY0C z0fEFIyCLm^$$|s~1cd~U1O!4zH4O}tW3xp7tJOx@qVCw=g?tQZ;Z+6*qAE~rYqhkk ztJ_xjdfKgORquLLqmKQ|fBo43lLW3?+f$G8Kg?%yIM1d%XYu`g_9u2kSfccw6@$RN zDg&NQ7$xw6dwpQg$B#5Td1u4N5a`h#6)>Sq9WtU}=w1^ihGFn-OKI>9My4e2U(OVP zc(w41anRe^+CJYs=n}^?JaY)*kwL912Tl1dr+`79NrJ~Dxp()IVe|Hjeex*ol6`b$ z|Fs~ZFCU+()VpR>#alADW&zB-9Vr&VWA@RRV214y-_!(h=o)PHWk~`zb3j#}GLGXp z^{N=$_vFKx0_ODzYkA`B5m2LHd6&VD1vc!7m_sr{H@`=U8#k_uS96LDx8}upxo3i> zPuB&XctpjE^YkbUSNG(g^SI>Exhr(%&AE377Rb3dC<(_lt(y=xf6&2af8t!sCN@5s zys`>jdq0=Kv$)sFTv+6kk9beRS>AZ`>LchDz(fCh-||W1p+o!7Cvqr;)g!+t1@j*p z+dH}YuqY_#j)}~ewtRV6F!AJBT*t6AagxnstoGtt8jlij`nZ?yRp#OR&S4x*o_UdN z>^i@fQ2x26bcL0*HX|)=ZOkguDC*-;+(XgbQ5ij6vqqoGQ{pvuJ2|Ynj9gL9@W4_g zGr-CYu(2cA#YyF*E?1S$)@by&{8Lp4XMxLXN*_<2CU%$_xoP1E)Xi9_L947LTVf{K z;$~IiD9Txu;RF@3SaQA1!MTc|i4_kNgU-+?AGI=;HOkBhT@ITYJ*t(Ui&!16#lmF+M~$pns{2`AA)m=rUq*x}$Mi?GP3d#gk$pmAy~C3cBz zEJUV^zCCWXVyQqg4HA~6%1i`X$!r-@D|$qM14a5s7tc1+bb%R%e4Cfj)ZpdTQpu^Q zZ`w#C1ymzeqj=T43o2z&7oh>%Oe9O4!9%?mbrzSmgA*w%3KWV;O?)O2Hb(q16+Wla z#iSL?JiMr)TF^^(SvenF3t$;Ilr&J-slxUH)%0Z7-UmHnsVB(Wi>f|UPliT3>eyb1@{4vUs;CwEc=2q6KN)#9LP?e# zw+b60MfQ^xJg^hzoQ)#P>xzjZk9eBM@*-9v>?~GJN_}7N^`h)9^3#rC<8T}JPnCaz zgudQXBs*IC&P;!I>o@Jw)7+Iw|d&u&s^F+QpSA znrM4kIxExg{fl6Yz~g<4fw1+>?!7R~l#L#zFRRK(V)877+L1yEZ<4H_gDI?vRjhGk zIE-G-BrcL=hQ!iFl46Z`S!X1t=$y71d0|!MODGd*kONd+jx;uVVkkW@P9J=`cD!4JRS~UIfN}iC71~RvlCsS2QNi>hn{eJ0)6Zj?*mSdY-egJDyz-k~n<_&) zjMOS0S$onKwiTfL0o_-q7Yj*;2Ul9Qx}?olAY0c@axkDoHcIu;ONQyi)Q=ciPBB@j z9m<+YI-f}Q3K$*APX^ij!uAN%cb4J!S-g#S?BG?jPFa4h-a}qT(f@`cxGbq;I>DI( zP1u^g$-N#ifi~xuta!BQaf$9bD4^Z>yl?! z$H^$a-R5v@k94w-QHD3uEk=+mCfme`Qxm%q?4nlJ#}XO4lBdyH$FNP*DvA_2hxG}y z+!{WhrBf$L&P=6EWH%FL)g)i_m4Ps6xif99QrB_HCAs=o>G3b>Z+@ zv0Ny9#tp7z`QZ8*JiY)qhfc8H$1401?{;Uopzi%)Ii|4JmhH=|W8nl}zNhUOEU~`e zO13M>hS{}vtaVAAXn(jN+Z%MBJaS9!6Fp;ou|%Huv&{OPV#1Wja-r50Yx%rJ0V2C8 zD9-?UcM2X}r{x% zJ63YHgqD0;?hVtocKqcL&2!6sA^Sl>v7Cxsuj-tf`RZ}gZqhS@hROK_)f zk^RH-myqpd01@LmXu|sn#p6H~%!|)I`_Wsp_}uD7gY5`p*kG^&3;qL`Ahj*6-nA zn-r-?#E>Pf4=>OkNd3E0S)qAIT%UwXgXB25!G-x4X0DqG1RT=ITmQtY?4}~NVY?J( zIhLBtNG4y4gQ4L!P7a=Oc6KOqq5Vf&Dq*TQiJCsdQLyCe^K4yli0#C%FbiI)vBRO^ z+TVMhhMOUR`B6d6W!9?96qriJL>nZsE(+jDWLJlX&om+OnZ}mdH!2(bp_cwB4MfYD z<+SUXHCa2u@NbN>adLOWBf_wZS&K(5Ec>G?gPsf3+_Bv_X|*!Yoto{g55yic^V2x- ztMn=Jt)8LBIu9plRq{US{)TAZ>$}s(gY{Xvm$ALLC!udDeR7rXAamE+W9 zFWux%7E1q8)De-YERD<;OkFr4a0Q7;`+?KQ z0|Tfor1x?~GF?%1V8d$wi=uGyWy2FmT*!r@Ma9}C3y-8mY;oCUXM!I!{o4?0V5@zO)Vr-^RXirdQ*b8?JUH-rM4`r^jlDsJa5Ef;fJCdArsszLZD#z#*=wJP2^di-dY&2{piBgRfz z32#f~AF+7qK@YD6)?YX}v7f;{(2#tKO1#}IQ(XqNOq5I6cWi7YM*`RYSk>*Zs%Um~ z4_#6^fa(IAnO~^jI>b`Do`UQOO>?8$Tl^6Ca7;8Jx#{=OglH3Z)GgkkSVmSqaKMuZ zaH^oi=i!rZV%iX%W58*Z=jFgfS6tdp3jldufK`VAv2O@s_JDaXm-BBy!Q~*@=y$^N zps!$O^0S4o?4b92LNVl-O z(mw4A_zZWRVQvb44ZlxO+vfvn10W3i3cz5ra~pbzywOkQ(C^7D=ARw56xOk-22}Vy zKY}b9-a*RK-Y9aI0+;okA#kPR7cIPE$QI?sKP9V5bnkuU<~8r>s4qCVuw4&7o`ceS z$BiQKnp(|yd1LUdc8LR&y4q1Oa0fD)5HUS*h>6Bv9?dYx-k5p)_-&V{x^bO^53DC< z7=g3}FyFmLye%ynCC)Q=RTM5^xZaYu&$qR3;BdK6pw3$78&|#I(lRyBTy6`FPA87p zuz>tIO^{QIwIl2bA&f05sVPQvJ@O@3&!Mz;?$vm?K=4|YbA?V;E4H;emr2J~#nWy? zRT>s9n4c%J_qV0ZJM$iA+#%93pK_aCrj^{IqP!`{&)v=k3$`i^b>x)?eZ=QgS@$Bv zD+$NYS%PlKDO4^c6ZaQLJlfm>Hnr%t)Gr!^_AR#TB*wNvY3~z2cTdtROFJQ}gq{g^ z`HE~lnRO<|7?%45DEUH#e8r3}oU>&9ik2_#S+L0v3w=?8J-nI!8S$b6!jFL>w3`XA;!gMMv{aC)3N1qADi`+ktzj+Hy%%^m;p#Dm{c z&7Q6BM9_!k>elFrv;Dplqg50mb88b$g?8d8SUaNB&#O?pp*{Bn03#T)uba+!Ayo2( zZyJN5FCsz5$rnLk+qq)Fc|o4|OS$0phA$ZJG^gV8N`Hty=y+)@C9whXrZWXEQF!ye zuvxtEJ>HcQa9Q!A7CdMU>m%(x%{$(dE8f%VFDTr2y7>!UQfCDt_Gm%dQhR}^HHYW5 z;=T!8Uyr`qD$Eq%T9c+iT-~~tTF07&Zjsd{E@+JnrNw(LTi@Fq49p1 zEHul#V&$53HTm*NOEoM!Jqu6ia^8UbMS7PM1YfNTfGWNK5$8_RLuv;%1tcfQ40 z4WEw)vDZHt9n`nYR|H6_g>2PAv}+^sxqwF812oX=yV(_e7mP7dS#X3!UcHs&6sU6x z-l+|)2#t7KXjFIJR|N2=Rfp5lfoJD@5$GG`>K5d_}swn8{;1zy%W+M z*79WGpCo@U=Kb07fU-ZU+#YK7B&RSF!(%ojh|_Mf|Gly7O0L8t;D~h?mPNO`Kk>H$ZW56p<=iZ^iP+VkKB_ z>EeWBgXOXc0^(14hApZz_;Q`jHO{eEv zh*jWz7#RajpP#uyJfD7?ndP38kk90i4~kHSqhv>3m_;nLJ0EPKN=HP6TIf6I8MjAQ z-UsHBG&8#|T8xMoSM@+Z5zu-Z{v@^v2K^4iSz>6$fw9XNhs8cOXrjj)_yzJL6tF!x zE*-0-KuZ4iVxa{nd>kC~H-qO}{Sq+n_|cptfWMs3gZ{LC;XxvuyqFi3q}*ywXL3Ps zE3kF!+*pZX_;Z67!OHv`FXBc08Esu6as|s1n-|fVuCtFQU+~Uhmk$x%IKGb~M1RSi zW);_eeg)`3UWh(vOTWn#JvJ0Xt^(pqep|5gY3P^9Qo%5nrPu;Jc6j?8jjPjwyilSh zgo8{9^b?IMIm2eIj`?mv$-j7#n%*_^CpnP#97#lwoaz||(+!$n(S4Y|bQ$+GpNK$$ zPWq=DYCCQ9D@gA6(zd#NVHuYN79`hffmIjFJW|_is8nozUH8y2VhY-WZAs&u^sWI( zf1o$f$Iw4?0L%voh-vtj(hE2S+$RWeP`rB}edrex%_=)6NVsF$1tFblzPr8UR}tMK z$>@uHX|sSCfRCDRinfJ^7N~tDWNV0j$;#|vR1Z*sQ0&NYR~Xdri{QH)Sc;(R!@RgX zxJqgHtGcxOYW!a(2|B2+GYg*p3+Fue;ql0Rp-ne{+2&L)LC;TW(cn~!#`v--n5& z2Y=YkeRYpapO&8SpYt1X2EPG*BKZPn%dTg!KIY}(>qzY{b39_7$W2d-A~yZ1^c}Tk~iC zh<%O2=0m)J7y1bKwsZ%8Z}gVREu0Om)N6)M!C;_dyXWP zPNXK87$sE`syhrm(87Qg%%>g{p=l{tf#hIQQd?*kzb!&O>Mh%)lJ7%WVo$(!&BGRy zRMlvmC2Av>Lssz;uvWQjI=9ojKx0Cmp-$+o`b^;Jbh9gg#nhpsu_D<^CIeeI2`(bG*J%Y(pQp|G5VUHgG+Ih`#z7I}C^4yEdaigpRwo zV6O;uDd$SQRf#?qa6zxOq_kCHc}a<0c(%suDlmF!*OtJm;`&xyTINd)6wR)#Hoyh; z0P=mQ)}An1M#Vvc1$`scHrs{Rfk(+vUg^od?S_TThcCnzY@>i z6c+@65nUgfyza^ewq4rbKi}OcXw}BM0J5c9ckp$e%W(9FqIRFtv_zYSh%}~!Y*7*W zHrLb%usM>xD5xKK)LFX6u8o;30k4j*VAg{F=`+Qy+~(JI#xFbD9nBD3QZKjwfRWyK zn}^GVPYHchGE|trref4(|U_u_|AND&BVI4EBS1lO2uj| zWfqB8Kzw`CGaS}VjQrl5ZO zH!eylL2u16Ox6~}ht%bfs3logIyiItLX=_!weKJe^&)<}7q#%-$)2RJ?GQL_7}7mj z#w<~dUF6!ON}5+&i&^j+rWzDkbTthq90qoXR(%N~xZZGzGxlYG zfUG1hz}q`Q=A|LUGqFRmHe~@`s1d(d@bZyE;?_P^yKtlX056}QvI=x=o!cVM3(EHK z5{h#7h0qI(ZrSTR>Lq8FUatUqb#IGgugF_<`T>@|&|HQ3x#<_*a#_v;PY=pIYx!@v=9M+E1MHiZG90Z@wH!nF6V~P^yfU zj4=mvuMCOk*d;tS8B(N934prMwQ0%Oh6GsCzQ*1WURT-VDZ?eGW1}CP34czpARr%F zgVWbaIQSI4)Sx}gA;B3DZNlFLUl{NjX_vfYCF_6QXRnKR@x>=g^84)4W;=>?W>2LHN@ps2p8E ziqmWQiCk?RgWF=AjU`_Omchhs8}DoAhnFLs?wqW;Uhr$~a40v09kDGTyz+fIEC}Hm z{~VZ}%C`8h)wBEqjt0gEPV(|T5mPw?uFor4FalH*^FbCl;#eeojD(lMs(^<1*+udR_RZejT9%0mP0BS`N$TA96ELlxLis6%O z$9`kbO}a%X=pVvAOpZTq0g;7~?{m+Nd6a8eixyc(?{0Rx?LFIjHtX~E;{rMW)ub3> zv?nrK5rhF>5T%`np^+hGOCm#}inUZ`Q)KiHk?{C|+>9v(<@E!wU3+Q8mS|u9ECOX@ z)tKp4x%Ps8x1z~G*-ZE+NorMs=Cq^?Zf0s6t7uCT9)me zBR&-A`1HmPU8ScM#t6pndj1CAVJ+66b^}6l1IX4NT#tS1+?Bjs?ckUQwH!H z5MheCa=G5BUzVP8t`)uWW{%e0Z~ewQHrG?><`d{w-kmVekwcb1SV#PHNY4GT%hu*0 z`OpB)3i*fz87fYTSmnw9#1M;lDdgO8c{nhefW?)@4a?>rZt;dbBKNs*c?8vb%n1cO zB!+?WEVP%@cE8!u^sym{(Sw_4ijAfDM+y=V;Sh34;#$1z_S)dP~K8hh={02fMFJkuQ z?zQq3Sg(lp)E5-%r5o<7gpp-kio8Y~>JpXVsiMtAS9z#NB$P-9HhSU)w+1X3fuB`MWL#LkH5q5=DmMmc?M#l-8FKtO;(OF11hd`f3 zP5t3y7=?mD$U#AxTaT4f8hSn_@g&-HdYZ!qto_X*m z=|;n}U$39c1RBnyG{;D0?sUT`U3f#4+FAv3N9>)wt7Zk7?xj067w2 zvyY_O`CM){hS>!lkYp)-E~B;JF*2(6)ZxGz0J_DF{NViu{? zWaNc+DXZNyuKX`26FR9p?#}tbeP|p|-!fykC#?R3#sDX-==O4B1ZiceONyiNa8uk< z#|Xo_AJC$cfH?jqtIv$Wr#`vnn554D!-syAxU)x(_Qcy|U%LRX+)6TR^TbzS&X0@Y zE8`}D&b-^XzP3W5n9AS4H!MaMia)1w;?8&WHC0CzT@^r9X3a=28*X^P1$HRiwqmf{?7mpI)CM$ z`@#zX@FS+;IYPxj-Y|eYiiC3Qhd>KU+`!B!HDXNgXAyytnejsW(L(YkP4o$|HYUE| zXurUV4srt|+9>skw3`ZbZSW(}=ZWeOA*IC(M=F#Hc|ye^$RECO$&_iylrdCfO4vlS zAcjlE5LZ~(Jc3TxD5#Y(@m%S@7(V zDfI$cS?xcCs)gZoAqt|{AuTmQSeD3Cq?0=v9fKA}u)CTf{%AT>F9Y!V;;Ol!dRl_0 zBWXNG580l~cwet;bJzgg>orAjtyoeT?)TG!&Vix8wk)_%DDCcGK&1qjlz}G-*ef$+ z!lqwN7|o@4WQj50imd3f`R!UEM`o^DA zc;WHRp1*X>-pj^j?q&2O=!tQ?#k0Jkvcjh|u1GUumm6oJb-uWV;XA{FVbNiGTogyC z!RtX^&uA44%uka*xSSWZ(YLtfOlmH?6J*L3rKD;GwlE!2-P+gQx&@{8P5Sis(6Z~l zmR|k4F5#7%n2Nx8p1{BCMok(*iTb~MAo5IU?l13eiJ@u$Q{UDQ_!T;gdIO;9Sxc|dLO!xZiCey-LD;3fAL9`dJ@&(e=THpSj=nFL zgBb5`S z0h!J2Iw}SaawN!%rf%ROUR>y5PLd%Jf&>+{l=cDwPvHRi?FXvoPs6v35tEhZaa7nE z`(UyR6CZ{ZY}Vpv&BKC%$1>b#A#X#mTqA%AJ^Ay?tJ}TF{q@%J=HjB?c=lNLTQ~cA z+ouQWH52Fd_q8q{sLT4)fbC6zAqj=xig&lOW{GQqisCW6xN!IQd{?1|zk_`UPZ#TY ztK1-Y%wPCkp@9)8296zU+Jl2@n+JXL>8Ekfz(kw@4=v61NsEO83&~hnCRtjV1B*=? zHN|wJbE`YJdM1&f^Avn&c%dLOW+`ck7>o`NlI$GmU=c#_IsCBLaCU!4QUeMoPjfim z&&b=q9+fjNVdHwIRsd5)fjRnv2x5p83Qg-c$&6!h-8Rf*+HsmS#l&D`*2&n4k&3qh zev+@M2%OlA$mFq>#x#6Zg9It5r-RI&Bqb&s*(@U=b;rmkE832W;h?Om22D%Ji+iYS zU_q)n6p)#tBmYc@&TMo3p&|~SY5M{d+R6R-(u4$s!5Exl4F!98KZ+7c9gG;Unmo}k z!o-;tkb@de>JL%4P~`8r>O{x}I_ynCi-#Agibs?6;f>mSf-eoVHlsh^0Z5W}=qfTG z`!HDdSqI=5eUV*n<9Udq7$c`ch8#MdTb%r*)M#q0+h|geWDC<;B=a~6Sx{npj_-j- zP%XK%g4xIz%Pa^d4pY1MFnDrW2c6yEyRmt9h&p>N+J-^&*H`d>jaU8a5ZP!{U2zk}5o7}rVnicemigO#7OmY4YJ9uu( z5i0~L-jGE9gpwnwn9LSe6bN<05hY(#B3WHOKg!^Lw(ums@@45LDE~uA!xQA`L*I+q z<%W_~?K;6o*bLHx7wXd|6j6_~%C8vn*8p+W5Oa3`EN2Yu@JNweSqa^~d%gU=8wPlNX5r`CAGYo!>dL0F zrtxZ{Kb$vpe1<<<3fF!W(Fe9~t)M543DH#4 zQoi7tkS6I8kN0nn^t`*|r>T&4DbmTLkqWhHg<@KfnkL-^?OBe~5Yn^CY<1dm2!C%9 z6o~|I-CCk>fVBQmjgMl&8N`*^aR&$uE%GYWkQGUdG=w#)VXbU%tG!%97^Bgc9rsz( z24~FA;{>hgK6QMVh;5t;GR~O>?;5uwPT2-_33BL$-kWH>_;d5blwn8)BWQo_E2F^sUD3qc=&OOG7|TV9zH#Qy5THs-Ub?P^(qY z=~P)I3*}B>+3E8cE9bGWJ7#r~5=^8p_R2^+a#Ms`&2{glrJPprFnQ~oDF|tC!>g1$ zdq*IDf^9+rl}68*`sbjMeDbYfrs7K@_~Z=~qZ5kJj03fZ61BL~H{YEy$n8m}NJS49 zQ6is2%CrOsD8)nD)v-r++In7&G^V%x68EB_^SrdT|1ZkEDN3|uOSZJjwr$(CZQHhO z+ji};xy!a~+vYpBU-uY&PxtHl`g7&yoVjAf%oQ0i?-F4u3(P0mxiTq?ed zR;0NDy7}C3u>y_5Q`L!^@`<|N5Caj&<4+8CoTnLKt(*Wa+&Kp_Lc0JxK*GOtdkX7! ztHh^WE7Ar6mS0d%9>lbO9&bQ*_^I6PKEon5&rc$8eB$O}3bB0D(Qh!^C8#{j6L@vm zc*6Gf)ABZH9QGU_}IsKFgqnIegc`f5(3(NFMYw}>}@UBcDIs?eAG)k*378H}5jwG(AE%V7k zWMGxIbIiP|Gaz|{Tn;$7;$v_7Z1q>uWyUdJoTXzZ%@n61xLUPuk{C@ayq)>sRiDKv zd)S;y$jFxQu^SW6jNrAL!Y3j2869v~O9)Nn1|t@_rX6tys2OUg93CZo1Z#VTDWu+n zM77SvhbR*B(^T7I9Z1Lha`&Rv98|-s#f<#F`w#LM10Ff zyuEa2wG#n=y8bI5ppLOdI=j5&`+~M#0O@oxwh3-glBjK(y3;z($9er_CLb8q$xJHR z!)sXVcE2`A55LgrGR4mz6G^=*5sCLr1?yNA|yyt!34#1VQEzlQHgUV?wxz z<;bZ)x@jjrZdt7mww}H2aM!bPY7{)xCcADsp`xzL&Ru7v%4kTks%)VG!9vn8*4n%r zyrKd2!EtSvIuCa!ZQ_`-7;4dy5F7q3c69H*av{GwTWfBj$vAa21gf@?s$5`$Z`>44 zEEoF9tnE+4Fi?m{ZgiS4t<-BQ34#8MM6af;+;IWDvXSVFYM`WI%Ac`g6TbsY$oGx!Em7z;~o6F|U?|45S+UM?aGFP_VM6FX9Ex-gD^k z69uw^G&%>}{;GH;p(d4z6}X4>7E4zXxJM;@$Efq-g_EM*H!J0E)GfD^8idS4PDw?U z7WG z3#XTs4dim-uix&BIO(LPuAXg9HVZY>;iZ?#^qfW!i~NbuYRA5%Uihe1&KM+%v>lR) z@SYbGz&&VC(344u5S&imCzdIEa3CS1Thu?ygkQAuJcTN?{r*oK4 zZgjpyrS=Kz*aigVWno5-G_#vNkR)CRSDHWGhlW^<8Mk?|sB}zVG+lsekSaS@9Yj*v z*Bd2(^zRPkGZRPILJH9r8-oRf!NBy8pm)k9nlN0-SYZN8h57=(;IrDAH4+R z!V(Pi41928XZ%fFx1#KKGF`uRej^7*1`zISTS@4{Y5`3tnK2Xznr;{GD61aj8XdVG z6eD|Bn%E`~%x9vDyRKtlk7-Ah%1Bm<=#!!$`Ss$VST>X~Uv*`O$$UcivEHG5E)DM7 zx*=)Z!U6v<-_c>S+}a_H+~Of&)g|~lB`yteR-rNeusA@lW|}9OBC%q!+baGUA4FNT z`OV#wMTsvHkEPuGC}_@>Z15Whg;fRQiYa0~Tl})#)27nu3*Y73VY?7g#^g6gDJ7!g z(0sBuliIQ5+z^QIZx2ERXrEHbq-OF=85TUT4A4eKtvqSc#Qva){y{B=m|IQ#-yy#@ z2TO?ki4{p}hIN%_5DTN$cAML#1p zl5-fB5E1e{Lp6u_+H(8cv^At)ToNl%D~?E6qkMerexsZlE9DUG+Vy71xcru{o{)60 z>_R8=$&ARdZ;Vje(^RTPrYtk*2|Rf5`?N2KXg$DztkhGt7zXDSGStW5&`L;xIZU1u z%sPcfYTV~>vH3GLxTLGo?!ElMy+AV#EdnV5jV^PTYCP!t%U7kveJ^i{S^+qFLR04T zql6!a?ra?qq}*OH)tyzrFS*o{rX#U7OtjQBvdOd*)02DC440rF%dF5Fz`3S3lm}rW*v9wrTB+0CJ+qGtInZKlFoXQ1~I>SC^*@t2olDhgpm3vOq z*ORL93DIC1c_%f28Y<@%Kaa8IMU={VMtaAoIM+_9O;x&(P|c)kEyAm)5F#{-rZtWG z9UF17cC$&s-6N~|&-U>IrIYL_2e+nrK*S&hwsVte`P8(rrG6UOxiKJ7#=QvR9h#g0 zisg$Qhxe7aO$sFTyM)&Kyn_@Kv!9RD#4_r23(2CBq1JTds0JKhWz>;rQ-eo8z7|(a z1DA$-ootSjaxoV~N~q_)u}RHYaOurV^ar<>2WtiCz|OKt$ICa~ zTK~TM;}j*6Xw!wovXEpqV5LYh(Gzt8t+~aKQ{M0S)X)JFkf3$BO&rt4J{2@!*fZaV*Wd6YAcJ_{+`et zMBv3`#&;{q`2ZRv#`b8aw??YiqI=Vk48v(`#}#`A zGkoHpfZD^Z6ahiKa-^$F5Rcx0UBLjY51m;6T5TQCUUb60G$GW4eK^+bIr6Vtaq zhQGNFaZXzN19Cv!VeFBk+6OV`lBjPig!O^OE2gnT5}1dUP>RSo!rSQ}R9BQ2V+)#i ziHU@ac#TfLkEhs7}sr(+8@FYty7b5JNTedrV8EP`auxSro))uhJjmde8plWkt<1|0Y^4N!M zI>6PA7?}$xGdSRIK5#o#I675Pv0q$klWZm|aIFhQtzCbHK}^wVn;)LL4Tor<{Xnc%e{KT_1ZcuK;Cz#4RF9-DdhLku4UW> zQHiHR0#?Hjn@fJd;;VrR>?4=pJ9 zf~eVQvbi$DA=mm#e&jWLfXX9Qj!%mqM|k#5kZwxL2AWs6n^7N3O)Y%Y&0nixQdC<~ul4Rd zU6Z3m2(|@JwFEx(xe=f3BlejAD~qX7M{oX5G_QF3@NsAGP@)tMXHRb|d3~81byHAg zqW0*0{66HE9Vm!XkzoXTZROje@!1Zi(ll>vy9!05nYCv*M`O5CJ#;i5gxfCq!P#OK zN5GS0M(n^~XL)hv99rS#7__|?aoTbb@a0}=7IBGq z<^_;LC??MzM@QC|B+>{!!=tzt_=L`Q0jl_aItZ73y zMmWLKd-Bc@+`gQ}ZYLP1ICVpz=$$#*BXGErjq(Y2CpIUjX`zK!}5@=OWH60*LQjIf{eiNZM36{PtN`8$orB|Mj2kl?C{q{^s{WxI4(5@@AgmCyV&X3#Dg4$NWQd!r!^d zJ5%)gZ~Qnt{UmxIdExy8Ja23;^N*~rv&h>~FEUPT$AbeunaB3HLTOE{b6osuC`eN) zr!;#%e{(z%t_TmR0RjN9`2_$#`#%jmiUzi3|HV=wMoH@*qKWU)4J?-ec`F4#J8BRT5SS03}~Z`2P{;#k^?QUiPRm367zM`w3?@;2BlQ(b*mCZ-!%G`u8yhr zZPc1S3l^$2J^huiB8(flF`ztoODcz+7 zXU^0IIy70uquLD#*v{Xr|JBq12ZQqqi%2}sB*cw)RdPevf&vP6~b zn;bI|o~$r3-w9l&uFlxq%=ez;}ivwds-y-#6dC` z@o1o=Lj0J&LVzPw^{>p!32hrhQZhJ#6j{o!#hptQo9!lgZ-z5!2NSw6-0Yo#7Yg=v zLV`Rbnjg2!ix2k|RaQc>;4!b{k#^E4<=7-trM|TdlXhxDqR)Y~Wl!`e6rbOK*--7D zo5~CS!zsd@#4eyHbGrv3l(D!b3dMSU2$J?5Ve<&j)H!GxRMp<|H$#=#eSSPZ0Dy16 zfAL>V(Z$;2?@nql@)ME+{K!15wX5jWE$7{pErg+wTq5uS@PcSIXZ>5uu`a|iSuABZip{S-OW&`h-p6`-B<{Zj_{t-Z9K0D1Eph5zj{@ zjI{_WPr6oUi0D8e#h}8=g^JMmpO99W#avFrNza}zCZqY%KEF) z)c?BDcJ}|hquR1dbF<%W$zmYrP!?f0KDZRzR>@0KDCQa|i7C{#(tmPPqXM5YIQ}yx z4SIj~_6)lVy#)fe&cMk43qDt(JgGLmTX&E)`jgv zbm2o2`YgbU@SefSG@L>Mqm|2~L_u{D=uo*@PAOvhMA*6+2_?vhv^+EPAox1ZC75_I zE<6WGQyL1Qi65m03^+^(kS2rw3w(A`U=Q9$NZ;t0?{pW9Lp_r?DL8HHb8vED($9DZM z);~%zN(=JHJaiIDT9Bh*bCj<%O@Nc9izwlYg+e*7MNO`Iv^IH19A{8T#o*thY_yz{ zZMR^rMc6l9SIP4TgCaL-Gk;z-lMYvI@c00^hcr=HaZM726+;V6Od!XR7wi|*4Mv5x z{L@UL@EGbCg`V;EWw!Sl#xO{ELj>S`D@c6>*McO`n|bs|1rBjiyUMAM=Js;)#O*3u zDg`c}pF`sb*NolO*ztPAq$&s`-*t|pq-mEis?-dk!ll)M4?DMHgTz>R5adl-5Lv(S zuf+t9f(lbEqtXl98DcaKC)9}vlP`c-nCtzXV`j;gTH5Dh5>bVGYvHP+)Zr6oD1-v% zfCvwic|XFs(Ow&_a-L^{Uw#UfO^!0{9*V~$lK8!sKrBbVila2h=YyuhU!Mf11Le=h z#X_A&sHWo8Xk!)TL2do0_+(l*)c?vB*O$>I6C(T>0L07@qq000=` zzcAWSGO_t<-NNWUY<2`>h5udopnln{gXp5Xu)Q=e^58 zPd|{AxuY`{i8HYp%J_Ww$7{O%degU^A^_sD?QqfYuXAumJoj>0UUW%A$tFg z(wKo1PZpVFRK);WPI!;7wlC>0FciwsrzbS$zBfrhd$5F@*B@>i7;pY|kKB-*PLEcq zS*;jSj5jwiF3^~p#tjeRz!Dj6XoQ)YDF1jkM_M=kOaDy%_Rv9mUt`hV9iK% z{xSP7pGCAI=X#nW_o;Y=7XG{rXX65y9Y^oG{z!^77g4^$2}4R>CGScdIHaCng2w{Z zM55wqqGallVxjrSE26wzwKTqk5y&d86O}DBsikdV)nb82@5Tjt;%Ykct!v1fEA2|k z=a&lz|DCAyIfN?=HsWfhw@vbig#jdUHHaMpG5mClR?QiimC8=JN1rq*H563mvsI38% z`Dr8b_EUVbF7(k>{^81YAvF>yU!bDl$mZR8yn7 z!j7slXtFq%7)33w0}WLXTXW^*GoVsyHoHN`j7bD0KEmQ;{#S!-2! z2mv`#ANaF%$>*`9q!1@_Btp->($HLEg70(%AP%>Y`PJqt5zYhyM5JGvofD)_ZMsb^ zA5Lh}Qix71N1N|Ms5NkYTrTRcS-8vgJOVzmLiZ&NVZ7P}W}0&hWb@aK@1eTION5>2 z)$A9R5wv2{U0y6!so+jmL*yW<+Q9_jyn>L#^G!&cR7PMg7jFY&<@Oe`kvMsSX3k%+ zJ6ybR^>$Zh??k*M2GU%#2VC~Qd!)&6^aSGgz1umRy;1ZQ?&!N~_C#=Q_~SzDMxRJ< z{;-dGUFd%5)L7{zr{o_X%U-kvsN(DlwmNSPPH>XmLTmXOr>RCL^*VAnn^y@ZUJf@i!HW`w@BLjt?(p`bD{WzpX6iwy2i`-2?ZgU)|FF%tW5( z0ZQh^&jqGE0LI&@Kz0JjCFBaBl-H)0FmUC`KBH~c2U2m#>HCmS{rRS0 zvA5o(FTzbVL5zA)g{C(j&pH|ZIZ4`!EKRzaN_F4rs_bbMw$rO>@)5@^AXlo2J~ob3 zx|4)NjqQR|iVu?51M~A&l7*$XX1;bS$ejUAlp_p>ub5%`1A4~^6xSWP%l{Bppgr&| z{Gda=6UG@b%iIJYE0<3>#~nl8N<)&63Q0&Kk}!x&NHHW|ih$A<%PEv0xTJGR!D3J@ zl29sah`9vBCpg8d`pX`z(huON^HHqN_Qxw~`h@Q7nd;g%hUXO~@s%ca-;F(B=K%WK zpWrh=Pxd<)=sWu47Hu3LCz>J(eBV5jvPcN(8R(lfLUowd?V}A!{sv{;1ImFt2>8>> ze|dehmg-?!^O#3`Tj0J$FsFb(L`YJo^JhoEn9cGh`K5QGyax&H&N~P%==LfkE0TTd zn0HqU?f&fNmS;JNk%VvOdcry3nZ6?`!_XV>9q`iJ(ih`5BSn%kj-|b#X@wr8-5pF- zMzQ=Xo4k};iLOi?nFs&_!efjL9VKN@18|sqdLToGLvWKnUU8s5DrpLu=vWBYSc2`c zxRq5wSL8ZHRZccuJopSmd8x6`XjUo6sGPaZL1w%rD5BeXvVIn(lF>MfwdCO9S4BL6 zrL70b#ph|wZ$ralv~lBh(nx{oAOb`4-F52a4^Xf2#r!ol#!3QHAGUY(8v)fm*+(P) zGH2@nhHGOH;_*H*d+eM09GSXO&@*GLHJ>rAu`L0<I+-Yo`dJuf3`^;aJV-19jMH~z<0u4%ezpP% zB>6x!&X`fTg2)ZgL$6hPq@_?p;ZlP-gBnZWgK1R41?SF1jg@?bNV`4@>tq_`B*l~# zd1~UfERfpUsZdMu`dMl($SkGvC3uvtO?kUC2b^Bb{SFMk`d=!BRL|iy<|rF?@%kR& zI_!j)^!U%a&>nFSrqB_n#=?lPt6*x_y(V`6y6`sc5f{PirzeMq4%Q}UxqD24} z1GmRH$zBI_`E+#wu@C76Q@UcFveZq;<53iSrWdT^Nn-@APLl|cDTwNxOk_B>8kRzw zk+PzAQf5=$+tJ^;Q5uLLD?8R61Qd9xN$3?!n}Z zlI&5&RNvjt;e9WnGd7{lr&VMq#e99^x1&+@In^@9RWCC>jbkE)A~n!rHtgA^8*-S-j{#r_P4crFp4EGivG<4kQgooy7ZGZ38E5wp?RI^H zepa4`X?Q~P7E7}Jkn)cw1cAm{B`#L7Y(2G2Q$VA+M_>>sA_wRD+c6K^+Dp+t_!#hC ze2n3L)da==pkr2Uf6=imwahv3`&EgY(J^9~hw`maI2Bv+QkCe9#i@Y^z2=3#hF&A; zm@5a9F`O7_#Q9;E(qvX2d2^|LBIemP5I=Bga~nvFJz(!$vD*kdK-LlSnYEMg?}xIUKQQ`ur30%W0x&`buZLdjAirGEiQH>|ETYH7NTVr@spvyb z2JGV(b|oEj2im;-WCn0OcCFD*?0KVW4+6Pw14GdU!u92S+sLP6#gQ=H+$DJL9S;(< zeTc%x=({7*SiGh9r(#VMIRb{6ax~1yZA~|uknsH2uMe4Ocr1rx*pO~oMJZa7uGXTh za$H`ZKs2wSuUa6oqieoa*I6#)z{#)Hl1b_c7N*s=-sS)&IBa2yDJrffNv9qm7V3{q z3gw-ulzA;mKvXX5DpJl~ququ*T8AB}QVQk-M&QRA@A! zcR3k709@-?H@RHhHP2b!Aj(7wdcG&Jn6B+a;F80q>SLPF&4ZD8u6l%qFn<|5*1 z6+J<1wwf(iP@S!>LB|BYUll~39pjk39zt4PNVDkkb2A%&~Wa}NKG-C8Z$gTb-er-_BY+3DNh?2dB9*%{!<#^l@{^1|${ zJ%Fr1eEu=vKYIm?1s7hi6dHlMBf^o}9l@+QV~-}n^d;Q6xj5}r(A9KX;M-+C=2!#g zl-(_ky)a+CQ~H!10c$HTg2YQ4LZdZ*ZI8`*4e1rK3X_PHW}^a|-7+~UY?p(ju@wu* zsdP^5tyx+h$8zdgV55~t(gsg&56tQ~&0u_ReuF`!SE>NEfR!TB^gZ`tN|(IkWy)xB zY+pY(8b;3gw(@5c5MIGJh{(W(phXhq%(K0V}kV2na} z5Yr6du%V)SURkuHA|EkJfV=>GoVp)KO~Htqsk{xCr(hVmIe#8LYN}9i_oX~@2o&#H z|G+N)JMoU(z4s$a#4H*|eo*0bAN@H}c2ES(OqK0jm`*mLj2{0z!4%iS0kJkhX?4QM zW|ATXzR*0FvQY>L7ofn(nu67J#T;c=_~D|3DuP0}mQ{zl(F5=`4kI{;!63ig$7BTdH1A~-CPUGHy*Gh=(L@ef(Srs zWC>y6ai^9_{c@0TTkeP;XCwpkF!Bcv3Cddw0Z9)%WFhxpF2+g9GaIDjIC-8RnFT4t z1~g)2l}Ca@5hakrxW}41oFvr_qgE|szbH%vqH!1NIFUu42!8}~>v!<1+EHdQ6$X|U zwIAmSGDwLdgK&q%yDjo*PuRHGa)H(n^(BDHZvf7<8(-HxT<3zUGUrQSIJE{M!;(Xz z(;+3@I)=xBJb{+v{xq+VcF#qrp&tt882-p=zoHnynA~Gpx^Y3v?yE<%($CVKD;(hd@Yb%WsOU9j0?3i9s=R-#8L>py@o zGg_jja^=j(M9fe@$r+2`BR8&f_b{!In}o7~RzK7&5|An+$|WT55TjdHfo!jT{$gVZ zy`Rs}*D~cvfSGSm1>2N+>oGEePrqIVqsrmY#+o6Rqtb5E+kyuS5l03mPEl%inP0jo zX^BX5NAlgFjP`BV_i?n5Z}wsCaJNsqB(jVMKx`T7z6-7<*y~d{|yuPZxR~*UGt{8Ww)V>{IfL}aj_dh zd&Q|f(Gf#)Sfm3QJyX2aVI7kE7-W(;m}{R?IXKiDbf63zJjhfl?hTFm+z+ov0=do^ zL4Y1u_9X-EP-7Elfkm?LtpBm?bo2E!{g&fod%t^==lAOt=p2GrK@omH8^|PE$=p+S zxGNe8Vo(87K*XCaa-{@;I{XF;V*R@``ggQQsPM)fr=7jj33SysJRL^rUMfam#-$2q zg&~{y(O9Oz-O=?hrdy4<6{&_Ii`L3yWu{TMhko*A>YM?()>XiKxxs0sRFvgXLz@yq zvuxNxspFJu=C3J7nX)>MmR)C>mDSx}Q_YYr-2$#cwX3NUwVh-y0aS<@rRN@_Y%ECz z1dV*#CN&!}6R+`HL%diWq;TPpYgWYkT27ps2JF1>@y>T_YRn5&otfXb0}*}H&`3XM zH$l%mK)INM8skUfAo+_ir6uh;Vbj#G(VJY%+K&?+C)B!(DO(Iw*qr|sQa zHPJdk*WkZqcopj-TZb+fEwpoy)Q(0@l)Hk{R1V|KMxiHU1g^@k2Oj9mqYsDtWys0s0L`1;>EK8jlH^;Dl>jM@ZNno>^Ew4L$<@6vTxy zF(VRA?>qZ9qwABD%%46QOW}8GWq!iVz=c<}3DI5$+b03q246M`I6|@`?fOc(Ixyz^ z;M8MGkl6tzetSonP1Jsr=RIc zTX4($w47p+!08fe4JOHk6c7INr$;xQgy=cb1w$&3GrPduxm^p&#$58>Eo6xE2pr&) zZ*(qt!&6>U008QUs{% zMUm^yn{Kb3k3U{FU7v3+bUlFVaYrDD>Cr^76cpRZhMWOW=@Pp%+KGkuLr3J>v4*Sx zAf!g_Vl`^T8FdXhU86n|2+pb6U-8zl0M$I_zpgKXA(wz**@wT(Z3A@&_9D zjmPh@^)>nB%u+$31#Om4**Q<#p9LE`3_sYaP|{KlwX3b$rBZK3eax8yS<3o|S26}E zv;bvH=N_zkb~R%{r-~xlGxf%_ba&t1Vj{YB9)Zd%PpW$-(k>~lWZUH*fV`N{ZGxl| zRXWtQRlP4^CQ#C4@S+>d)2vWt-d3N<;2iWh`azN~JhnQH*sS&Ri;3_nXB}zPYqTJ% z?rj2;avH%0wKaY__9nu8wp!vX;i+9ampv-2T{72qsO)=G*6bIm>o-fu4mAdr%B5H0 zTol^p*09#IRMsz_^Mw_RCK&y>ycPaighC>}hGRP16$av@y|<9qAT| zRm({is78K`RYN$HGJAG!u6mSAwv_3yp5bhu_K|}g(!kK)*WfnW$9T**O)~@-DQYv_ z=iH7moD+sPk>2c;L6T~tu>byv%kI4+tI)CJy#Aey?65NU@yZ{mwW$L?d#%HKhpCGu z&SygzuqVp{&%RcW1~eC{lcLwun=!!IQ&urlcsY9-;Hy%mFg?TeWx3GP-_i(ok7G}r z7P0zl>Z6iU5`||NdLw=)ge%n=u99{W*6Z%+$M(X;UPbu9olcAE8;!v0Ex{YX>1k3_ z@ieYmnR0cE89;CX07TZN8$}Y6+JwQDF82M(% zTr%c$)|A^1)igT!f4w3cV!9d)`2M?WZNo2?bY=L4p*!Oe;s5FBVxh=t{qSh_Bu zgg#`E!W@F;5W2lZoKadzv_R|sIIcLM5GTmRh%|&gyhJlIpFPrEN4Yagh;;1smPY_# z*%?kL;#Va;4ocAqouD&Vs<`pHBJl>I1-?xxH7|aQNB2j`@hy6cSM3u_^OnFbDEj`l zImeKq*C3~bLf4?2Ja^(}at*}wo7nq3ZYt+W#U@_3Ul^?qSJnrv&amsX+u#Yw1;cj> zJ+X(ueJD`n*d8UQu_@Hr|F3nu3GgO~$VC zsNevWq;9VS3$~z-!VJ+j+&`m{#NDBh{4W52J(zzXf%zL6{l^Vm8z&?cWMA3FGu926 zC=w9?YUIdb2`R~;NSmr<{$^&GdO*tjeb>h1p)1=>Ys%L#zCEzdLw2C)*Aj5Wcns?E z@F32E8-v@U%_%9UU;+$#&Oa}kUNbx=9JO9fCwX|bKKvl^NN`^e#ej52%}k0EFlJO-AyM@XvpPBA+Yn5?(-Db3C^&EN&C%qf_1*5txV zaNKwc)^d+XQD2BR3$HSuL9zoY+iJbyV&58ikM=)z5MMYK6$Jv?XSj`{gL~J#A&%w`=dpr^*yoFql10 zO?6@ME*5MP7bdi7?^iW_Om(-ekVXd)Qg-|(s*N;ziM`h-c3R_7<~tqgA3yx#giW-G%>4@x9wYsCMUw(Wm=%HaGF}j-tXcc!qDl5XsN8E;%MaGTn|@s`IzE{E zf=iM7DF!$3|Z-ktWz$$>&i{`IaU6_-a6srHy=;Z4zK{3-AUWrU>w^#KT&gJAeTf_ zt7}h!oJB}9o`7%2t(#`Umq>Ac&#=|E8{x=K#PsQa**-1X)i1LIoNWy#kSs(-)cagK zt!LB60$B=|2QE8czHe|Wk$a}4e&!XRxDEFw#F;@S`j+TYDQRW{t~kh@Y0wADi@2d4ShR*r52&E6J!f3NXXmfHq`fxs-pf~Wkvk)HBr;0zsiFb0Di36F=3A!q%?zR~ z%BbNlxtSADLntx1EVWvdVq4zDsoBS&J&1XCPpWj!3fl9Jdtr%XbsJUuB=2n;#7;_% z)d>?!KktlH?dO1IEI?F+pMKmC+~9DoXQ*bI^>xO$n7{i*Y0?>DYMfjWXnVc6@ut)p zcrR(A=f8(RHy+Bzn-iv*bcV`0BeO2ct55P32()cP=rKsX){p+~^?gS|dQ7uZj=r%o zb+@ad0Cc_4y{@u*r{Ii0ettV+=sUhL69OBTC3lEXYv2Cu_-9JHA7X~!l}(V5)EK)h z42O~Qrtpe3^U$%sWTja@5=RNrxKNW~gvwjc3cI4Xwq>O?E{twi4}}v(|{Z5?tyyp32WaN=D>p@m^GK}fwTVs3jFJDnU`Fj zqQM$K006MR{)Kc%$;A1;iW6n*7WmL*c<>LAoqJwBJh4q+E!7L=}`9NJ`|Wx^MZa0 z6NG#UFVOgA3ZpvXvAAhPR{Pc+Z%6?Sy<9jAnW0~RpWnd?#^JMlkmXN!6J0#&1B?sd zCF`r=MDz*uGtck1N}uSKCrs3A+jo$g>Btmh8$b@Yz98jeg^AJ1Vr@Z3Ef=34V+e4c zaJhOOV}NKFPvI1z-NoV4FHfebz-bn$=@8?UNupc1cp4`N_?B3WD2FY|TquYKGZrXI z0VxyuKGx39FHDULxlG`+{EUQaN3&mh5LGx=uoMYuyWm3_o^@KbK63^mac9mA>Rfe| z=btNBahmCav}LL)W6G%dvf6iF9{2k)Q_@JC7@a>y zY>l*7v`5Hl0-hkEHKm6_XBKOp#NVP3%E+AZRp>A=>Ki7(N{h**YRfcVE!J_W()hr! z!B}9l6Wl0`Vd{`wr`H~M3O*-U%>F=F{(!juVsN~U?h40(c;hg+XphY7a*w<;egP}> z=aS^aY6#D2Gr5jZCroBA&xjXM@bs2=%MsQRNaGW+Rop&c@Cae@s3g6Z{@Z~{P0HO+ z`(MLMGXwwt$N#uREo{xKO`Pp){~xJ9IVu`TSYq(spg2^;M~GqJ1oZ^O2#D&WtkZxF z(CAp!ARyojui;xDb=sTB12j9Wot>SEm8THBkhWn8aRwZd!Sqxo5%kx)HGy70eI`@OG-CNi z#AY*P%o7Y(i{mB9I(67a5^@%?DVVKKRS6E!*(~vm*kAPvMDX=4W%^o4vB#%ChU)n( zG7~2*TA4?+7#=p$i;atujH0y$7}F>LPhI%bfu!jMesHPPU7(B8oKG19>$~=e0T>YUV9d%e83+56S0> z@ZJp84t*xw_Vn-6nTzSRl|tpuA8*_vN_};WuX=uxKhnvplX7a0xQjiM=A=}rcz7C% zs!hoi8V?oq?IJt}k4=p(gk)4c;v&D`Lb5lDjhIjZI^4Y?R2OD-CU`rM_0SB&yze!kq*h@W3&ZYqH1MV`UoP23!~e&1kQ zACPl5a6UlRRrsL4)NR>&m^9?w=(&x*%+oO{r8wG|oh<@|w?^8%z`H1ZE&n^=5%4z_ z$zB`m9T5Tktmv7S<~zF^Ng^GB1nUid6+v52gUxI1Ku_fPaYI>n7r(wKR^JZdm<)Nu zu}pAoyTJC|YHt{xJ6f}QoDsG}(*Y1-a$kInljXiYm4tkM3UL+VdWY)SE-hMjejo^) zq5S3j-V23zhgCi{&gmomPEcMczmZZ?_AH{GphxWOFqE(GF5CcG3?&{t$?y3x7>qaIqWo}cZQ!%Y|TP`r_-8CDQdH>XE9w6vfTD@4K%V- zl={+Rad9Z0(Ao&P>y#(Fh#f@ts_u~C=LAh3T8TSCnk+I&lP|s`UeRM7Gvp2g-FWFG z^hts6Js8O6#TXyFhDdK~?y8`OBUH-Hn8>}G@I0d>SWlUGsztZ(pX|0G>w zZXfk(S}ePGej)E@+XjhN`4|O8_vovPeBXv(Q6Vq;uvLMoRSRt@wM`bMfDvs*QbGwgjj) zhrPh{G+p&OgG|Q1=z};X5kGK3yQHdK#)`m5j%2-FFT&f=^4yPVf_qBk`cl-&RcZ2u-STnuj-aln0hplJ9)Hz%?*2{fdv3OM{SUQoK0Y|EBp}Pcnm3EMX?!MutNHF}Rzb_v#46+#*G6{Pi;yMb;ODw=0H{ z)D(`)lRhx%m6QQ}=ydlDxC@d44b!A_S-eU@e4;4fC1CChHxm4+xC1HJ885lH*cd@i z#9NU)PvJaP$-p*$L&ebIb{{cI*Ob9nV{NkfnVg}i!GRR5H58hfB~UT}#-|RZnvHN@ zp*~%7B8Kq07Gh0)r4(6hzNhO&s?Klbgj+0{wBRLARPaUaBNJsLeDmcMDZo2zV9;ig zfx{N}Pk`=#%Q-#7KCdS1LkS?(C^iyn(Eh+56?V%%RU~k}#6>rUs;ezMe>*3Gom+Kv z`m2nf|0*r^9~Jy>;;2&;CuIBik$H>)B&&gVZ@=)ZDS0tt9D*1aM3E5tLHm7sLB$GK z2K4*++GhvN_xszPi=tagcmoCMqr7f5bDfSSb3A;!JVEQCSb!O%vBC{WW0_4|CePAW z>U-h2;A`Z=pfj6wO~SFg%wUBI$+gQCEtaq~_L@muNTYOf*ebOl{cu=`&b6B`Sfg^S zS2tB;Ec9;H8V)Gu9Ib1#JFD8LJ-1b2R_H2JWpfFZ?2~u698xYpWk5GHf3yyx_$f&% zKwL_^jzFHFphed%MmwA+z1`06vh4J3t!xEH$|FRm&(vA#164v}{4S^BvA)}&GXi#t zBq;{bB^_I7l5e#^Cn!G|qsGY+P2+KZMAf8A&DBBhs6X~d;&41k7+&*0kK-Z@vsGD5 zL^(3OKst*IK|qJpXD9We3)g_A7N2t_7dLk(J{k|eP+(6?(J!4$;PPqiqK_HDsQGLnKr?q2_=vZcv^^rzcaB)KRwwzd71JRX15$fr+V{$7<&g`VVY%IaND+R+qP}n zwr$(CZQJIzZQK6#{P)g%Gw+`F-kFK0>WJ=+=!nW#mAx_}SJraZzDlb5f44m&)>ah2 zKen&@XVU*W+pE}Ho7y@5uL_U<_bB(SO|5%y9+YYeg`QRq#s6?ZrEe~z`*FQycs%4|!zq&nW=rHCv+*p2#IGL`WcSjHe)Sg(&OdK+zw!B( zdTuyAZy^4EKuq^f97g?bjrsosVgKXA)Yz0OyJZHH@n5~}PFTYmw%eF1yWSX6Y*511 zEP-W&4oG=yq5cqd22Hqd=oXqsSrw#S#j|6v{NpTZI|~nUxR}t6&0EiZ+O) zY`b*i<4d8(o&AF0Ua~6=t<1V^HfOCF8OL$t_1;LT_%JmmRuLQ=)jv zAf-d|b~xXI4yGv4Mk^!OE3(@nAq9zNpR(qo#QZvKCM*vB#OcUm<#_kd*_DVue9AWh zIkaR+Z-J4GQIWR6RSQ9P-4}RmIg_oSi_MvfqUKoz3r+D4Lj+LJ(zPUP%V_ zpga6|4{)Rf;!_ZxJKn+2A-9JVyjAQ61_cm!sK#{fTwlBD%LXN!w1z5oKe%A&yBWc| zOHVCr{zRzM_)>a;Qp}jrexeu5i?+7$abj;Hq9)4NYZ$W8;-f9~kDR4GTv$ z`y3j+3z{(oLN@m|U%@zd#Y}B3s;hqsVy^AkaFAw0*_V)mHj&k0aBmO~8 zoh4_i^x+6~2i6>OM_QHotA`>#7+Lu+)X8tK|Bgd+dU$a3e}b(IZ~y?x|9?3AFBGcY zId6#|_@Z;C)o_hOkkR65C6QU)bi_(h@*^w79HIphgHO9&r76wDT~ef$5=>^M9mpR7 z=fWk!;O-MMe=82`jtZ-k!#spP_4aPHwg;#ytUtiPg0exCg*EQT9nH3& z>?l60i+Z9mQ=O^QP;k@~RTPyRRYf&LoyOYiC_muqL!*O7@hLWhf;IG{pxL9^hvI@U zn8@tA3ke%9o`}Rqxh%6?yF>$K#w3dwh&Q?&r@e*EL0L3{If{*Pt)?2l5bkp9COe}H zyEZHCvcpO(V=1w0T&l5;_}~yOGh&(TB2@?@G*!khPk=-bb&zq#kt!yBnnuj^Fo5puCE~khs-?lW$ZS+-j)%}1qTsj2# zimvU#DAsTik=)s54Eu>oZu1x?j@FCj(z$-90y-pY%(d%Q2?tHig%TO|=z_1TThg7P zm43j2JAqeA2U(~*Lr9xtf#So}%kRkm8SGUEy=2N%Df+>jd_or|+O z>PmG7S;gI-Gi>a^=@S4iiA#R(5h>Z*eB>gj*8E$I6jnL(Ku!de!Jp(u3iF|?Xp0xg zE-Fs3T_0OOXsoYPrEO!aDJ9kaBo43yV#!YPfx~(X`X{#!)dFvyFPq4VC zU#z%Feko)1iN)&k>J{)AxaR?Zp2T=iI@y#d!Z+9~J850rL%J?u zn-u!xUqTnpS0>rff9L`S@c##0Q@6A+F*bBE`CoK)iSpLSf*51@8@lexEVAnwwyd>5 zLXudUwM)sVN{Ar>*QUZC_Q|seakq16lO-DcvCsRZ;Jm0j?||P5huqntMlA*03_bpN zp1XT9e!mYd*a74o3Zda;sgkZPUFa8;Oiaj#>{MR6 z{86y%M*X}7_j<>~gALosQaG~Bg$urXGL=i`Cy?^sI?~zcqVqV@ki)Mpr}d^>QXP0N ztm?>?CUvin(+<37VF!!+3XC-gY-lVTsrVhO=lQt{4(+SAOdy!-lZ^yW{i`=_TlN$g zIi|xrrUonZ=1Q;-z8LAFUSoT|Y8pkHliljyB|R#m$Bm0hv6fENJKodso+(bOab~JO zRbwA2s0kC?=k3ptiL=B+>mYnP2~?3fWB+z zNdKTm;i%Zi`R)^C9x>ppms|cC#2lI5zWS zC2SUj&c4J~u(Hn~dgSkSqKE#SDp!8DnQiGe5MpttJ`dL@oS-b)Lg@s7>#9=bZ?UP? z$Tqj-$UoC?|DEz`SJ?T2{O_0mvp@!I{fGIWg8iEew2+~tji{58y_2$w(?1N--1EQS zkfJS*tcWoFTaLa)S4v<7iU_E<-Ik&X(z9YgM1%yz7{K~&1b5X~W8*$~fi`H)AG9}U zjvpd|9%Ie>WrZIz11M4;XV;28Ak2MJFgt5L_cAEH~H? z#qkIwrb4lRs-xx*T6mIXnSE;;R*^=@I6{_d9#q(Cv1!z&rithjS+C);r0D`+68wtUgN>tma~(dDdnv>1!;L&bq})E!00PYb_-uPS=&xUDTs$pish)151c$mc8vY zVX8VI)ZrLtmmaz9a$PAaT;m-bG+v7lcM~PsPAQ}7R%jME*+o8B(M0z)b#%dt5FWVb z?JQnkWTEZS*~m7&{$oRlQChtJ!Q3VaX-0vQ6C%^$I;n}Q7vR(3Ii^?N>1@_Iq-*R7 zA3-HAZ+Be{Ri^}6t=%@b@QV*O)zEdmQbKr}t#`2eWySp%WPeYhm8imyHo`%6J}9Xp z?2fd~ILLs8RbZqR>y;vDhdtEvuCMtFVz00pe=m18MKHW@pp`_3eCZ+2bH+OTGtRE# zqHoQL410wfmIo|Cd1HFOX40&c90zfA8}cVQkpT4(kuB22QnX&JYxJh6uZ1c%UC&!7 zmN5Xtm$OG7W`>$H*0yEY(u`%incLvb0#Pt(4DI(GZqrg`4R;~%h-LWUV;;cO}+wbf8y6#ANxckCmq)t(vP5B+4EDgK`e=LjH4}vCFeDb$s z4v{<(Ops?ai`*B0@3|2BL2jWdHPIfT`odEz4(41@It<=oc7ViIXj9JCU}NWSQ7I&V zhlmB~kdKU@G|HnSF#QX7-rB%2P)|s_JKcaZ;Uh35s4t+Qf5I!^Xa2ZBLa!tL)}Q>> za=!(aoj(wxK?*SFKEKiZ{~{cqLz;8`^AG$XApR}WcFY=Jo%Ce60#`{~=6J$cK|?*}k|>J)M@x>&@!r$kmkFMIV+X^A+GAs6weO+3;_p0ds6a*L)}!L+vqA5D)da{++s24 zmXb_83p5L=gb7?MAY`fCvIR0kKPGf8C4c7>dMtmA;o?Khdunx=(#f4X*?fi4ay&4t zgDvhv>2^(DjY(wGBvXwJ7_gCt)qvNmWKa~V5-Qe8>Z=?lHPnE|pB4)^UbxFsa3uhq z&O`RRickfvN5;Akhv`bUPXG&yoQq?zLf6f_N?kDH>J0@HgW5(`35Q~g&C6{l_+5%k zF=DNBY12B?n{W*t@I+??R{=eh)48BguXwVgkB*AVP7<2+HP5~RTGk~{fhO19-b*YR z;&q<+he)PjXGe&DnAM>Ny3;JET;xzRbER=DG@)%cs+HG`#o-_Eu}Yfz;oITbI-ggnSsz zU6XQj<}Pf*x{z8&rN+`wIw&R-I&I*w|1OP`ircKB7{DAdh#2(syT=do`u}2IpQZk} z_G9$_0~n@>F~E+EkM1*Y#6ip)2dTr!f!v(^7x0)H3z#A1C}Bq@5D{lQ3!DJ5m;+g7 zZ%`R*dtiPB4Z|APPGkS9j2Q^)ims#vzd7&=V?^WV9-wi|D`Ksn6>ks(3G0ec#%P9| z7L#mD8K=Bv#oR)6SSDqf{X2~QxaI6⁡K1i~E=-KB%vOlxb}`Ek~V}v)+&{i5VUm zV_74*A*2LksR3uv`)1Xm{CBAp>u0f^z_Lfw%BU}JRj*`JJBiD}HL0%^=I-ym2=i;g zPv9{T0RTEl{!RIakgJ)QsgtS6f94wg`AL`>+Wy~rq*MdiL**pX?>CbVgz4pETC5LhF4X+c+Htt80gLAGAl2dPlVya)Rhm zYfGzpOKov&w{6FEy{*l){iaPTrEkw)PEL*t*(HDW=~(CQ)9>D|-<|u<*+x5_H_iYO zD|^HX$L%;j_`zVLQ_b*^6X(S^G9VA82xeZ=n?K~^V~b87O}aVNJ*tl?pO36w{>#x7 zAP>3->_4yVC(`|2j`o5q1{^Uz%s~4>?Y|y;K(p?HYWP$GJ3SWT)4$*45SATy0`8`M zOgnw(1Ma4Ozy(>3C^P>G^ZSt?w6^%CPsg7;B7eN%KR)LJezOGRgWl^4_5|gt@1wt$ z`uMEI>w`9;kRLlqnfa-@{cYv>5u>Z0Jn27q!E#>*|6eaaL{N-EM@JM76tpEzxG*(!|3w#<1Lc@wJB7k%uiaSg8Y zaP{W0undxsNspkYS~pQ}LX1{6ywaAUaIx(eY!7Wm#_Cji!SpmO?U+gq%jtO8s0`Ue zpE<=%H$~YzCZ}oGTXuNyR}}Z6XT#p?rB6L}tNnTAw^;}lMQ3T-b1{&OV2o@1%=KYA zpj2x5eKB$&U)?IXJyb25&0QO8J4KHfLB4zz#W7OaRpOe1nCoc~(cNrTcT$E>y{9D6 zuYoR8BEvHJc{YLzx@S)gV{*8ks*9^Z< z1^*HRI9>si@Z_ zNrrFJSZp~*#&8N>h@35?L3qRuWlOjN6V&VkK}jN{0aHE90T`+VqAo+Wbmp%|)C-e+ zms?e9>DCF=6dFMYj9ZukYF$ZJF%TW*1;mMFb!v%^G+X9+s&ETr zCBZ=z6e?7uRKww{3pq;Id6|%(^4_HkVeUjHfLz|dhK2^Kq@r$fF`;T{)rnFu17#NF zadBp!MOyXDc_%b_*b0dcVF+(k+P2&^F5z~_mGC=t=eCfRYOceV<9F+d2xX!omn2Ja ztW@r?v#3gibeSo1e-(Ts*y_1@vanSY#YRaP(sYLimXaak65loa0fQav0o*n0ftzQq zdfqL1iSVCW^~3}#wPhg>0C6EJt}Y=es$F2rk0AYAWF_X6nS?A-leV7avdMb0@D-du zHRh@7vb-BJ`ew;z8rdX0-&aaE$2zQRj#7l!crMG#sa&do3=ZpPvf5n~>#KA21@>13Z@~3NXw=;K(v*fF=O1kA)DR5(qlR2PF z!?oHj$Zv5@{ZUW-=0soJOU$dTQGd?eoXME%ANW##jAwOtr)IKH+i}# zq8TG8upD8_jHt~B<2=s?SHCwcB*(uv;6Ei7sOCVb3g;Y}#^Y=KY>omBhw!lf5bb6H z>__tkJ%YE&HJNAPv9w+%unwlxE<}q&j`k6YMuSVyvE`0JUytJ1KTM@qmyT<0g!_4) ze0b#;!=uv9bag1ep+_lnJ)23|w2RW5;AkqMJ;4!jP?1-xqd|D2LyKer8-)WK@zxSM zR~rdeM2rpzLo6l~Csn=UcI2sTMN9b%gw{+6s)^Fknox$Pm9o(aI$7ZemkKfR;=2(- zb4`HZ?&!7q-f%Te0k%A;wZbnS&n8-u+y?D&zzlaEIxLc}gXg=8dhlZzk0Dg?f{%(? zcUhCe7QdRvq+x7@DYU~U8_jJ&fTkUdLBMw;-g+ae_J+=VO4f{(J5v6Q3w+Y7df?15 z!>{BkP>n)-$ElNL4nWEEJ&vsUW2G1Mbj|2^jg>O5RodhF%@13ORqn)#4VxYIB@s0E z1qxs6$6qzvUUB4(?{Mu6$g6aXJ!*3Ne!SQ~ZMf1GS)o+Dg3aaFaM#HA zJ2SRzyD@Xdd#H;Exfl9^lsnEKYG{0wY_S<0wnn~iY~cu_%$v}@W(?uo5ZVs$^dJ~w zG*(;@g|QX4eXbZj4-+olswD&ugjeX8VdIcuq6CF!JjX4UBwv6VKRxd>LnObTA)Yy% z=hl)dHS4xWp=Y&I>5Afi<@S8Lwy7mn65oB-fcB~NW9*;uYX&Rr6V)2GVnJlz<`+R&BNo5D(W~~#v8Rw9 zAmv+1od|TxtaJ^FzF{VZD9#ttS~YINy5j0cOCOedPc7PqazF7^zp^4aT;-ql0yZ7u z*)PI07JB69RJEE!cV`iZ%$O#CTK4JqejZr(e6yvvdgKIEK>i_6b`CfnM)8m>D9MPY zmaNf>t8hAG6^;%n8)S?n>aC%+O-i-1a)%+KJRPn2BUxM0JACCec^z*V=huue9Tp=q zdKFN7dnY#TNy3K|S|e>U-rLa>#F)@0;RxP}_HD6ZsiKOl!#2-9VJt@O;OQ}%lXI;^a}y%k=N=n)cs>dJz;R)YQ^zd4mi)+HqXlriMUQh;sCn__bhYBr*>{L z3uE2qw~*+K^)Tqnc;ap2X8n3|er++XZjm)8I~Ub{NX-5$omU-7fcFhxd4oEvl}l*W z%Q!?$+BlWfrV>u=D7#&>jGd_zIaN@mRamQ4W27Bply3OWSk<#_!>daj59|D?-;eLDLWd zGnf&f#8wcsqfSJ0$>2B~faDs$ZVho`IP(c^exE4rX>boAuiL;Jb>kh|wk&xV1w9G@cePH;_TTHn-K5}l<*1lk|wX&vMukl6?j5H4o zW9n>}Ol40w@t(JtWOjn}JL25*%vmvuHS1#FK=B1(S+Sf|pm*v|o;bP@f~^qUE1eyh zv|~3`kZ?~w(U(3j)oeUvbSofr#E9(8A(CJ;#^eo3U#;>JjuN~F30^~Fs(E~&{e&OE zB=7c_(6G7C*tz!kZhex&^l>{O137I??23ae6FD6M>upYJZ3(u;^A^}UR$ITuI%Eu4 z@y3?!PYiSDx!IEQ2%BvSIcDahvJwcUZ_OXx z+VBqZ)+YEJMuOfr`EG{&xTEx1zT&7!ZP#rfmy_u@Mvlj`6FHd5n(R!R9SP%`+>P>< z&EAK^?Np|;xZNHPnCH8_;zDI+6|O8E;ld8QSSnyMD%pxuZHGWFopegN8iZ`etXxUE zR=jikf(LL~>sypld2ko3=1nTQW!;I_&xMRwa>Ph)78gcJ+qt>;Y_B@nO5$R&*Brf~ z+}w+n*8Z$L%^Wb7>u_BV)qYke^SeqZm7`>>5S&)Aq*cSHRne$b;;dB!70>H*#fN4% z;LNUZW!1K_Hg}6&GG~8MR6BPKTgn9aq>qtUg8$9n(BF@Gy4NJ;Mq_JZgw|h~^Uki% ze64Acur#LDS=%-n!Sb;&32UgSh^T~TlHeDK<&SZ51>P;S6Dt3t*e$mkRC{qGC+O}x zmFq^72Tv%hXwEm)#;w{;6r{-)`b6G0ATOzap&0So0M9WIevDip?5UD<%m+Zn{ z{NA<&-*M{pzaJmUBc!39>&$md1?Uf+wm)hD*&QzT&ZYU=CHwyRi+`ZzF1U$Y;w$RS zj1M;@!TiDgzH}PR%1|0-y8YPvWAY7v*&iFTH_5>nBE8D{r{H#2_427hYbW0J60jVT zL`l5<25xW5wvRnw>x*Siy;!T+#vpsPXVks({)RhNa%%Lh7x})sDPu2u53xV&+r*OF zCB(hb1o)_G=JSw>*p5q;YARGo*eqTxeK^Aio*f@Y-8C$W#@Yh7BYF_@k zD+==xgZQ1X$tNh&0%PYAGUHNmyxhD&wK+;=f{$0i3D2PgXTe8{KX}`c{0wlmJDuD1 z1kx#)np2yYxU>|pj~%h;a>__o{Sxy7RqjyjG1x9rAKF8}MxCN^{PWqGrhv3S1g z>J!T+G)-kW20uNMcz+1#W%tY?J#$!J`0VA^xgx(n*d6E#@%U0UVjJy_UH;kVk~gmO z-6;A)yo#0|<7Ysd3&Q$k(6wnm%@F@05iu?BSEyDg58@6R&oNG^6{|D+7MP}`d&^1V zhj52Y3?xnT-xXB#6Sz1hpU860Qc+&3XsIpptc$07L61S^mWOX+nPrWF`nEvJxT3zJ z^p?A7l6UsHGMBFAMVBhi+_ta(xayYmAoUsrC5rue`6YxZnNyQ_>T4n%nL;G1i+S`z zj|qp*_k=O>H&*N|w)@TZ{sv&Zh|8$e8%KR62N#Bso@sHX46$VabY~f!$()6sv81az zzgGkw`M{=%2XNJ33yu4If#=Z|_PwtUmuf}8=S`^!9WnlVH1hvT{lT~dM}6f#cQ^k` z(*IMEMcBg7>3`0x*vj)l4KZN&>et4u3Cb5#(}9f$o>M>w4HoJVq$wybp%`iLWCw%a zD;Pxyr^D?X4SpT?=HFex8iZpGhZs&FAZrlrsPf?+G{{+NI~eA)_3SurVU$x%Qol@m zY;ag=7beSUMX5Q^$_z4_r&1yb<*dfeo=93sX5ilqc}YfDZFxdg5AN=YX*`#vYg0P& zn{#43a{tYY5PdZjhGKmY)spa1}n{U6tAZ)KapD3 z_rB^`|6Vn%^L^a+r`H2@fcC=|0Ly@6imm-`kRnXY>plb0ps}e9$G|QFLB zKeL_T>Utw@=n+?wi*0dT$izEZ#Ka*p8Yxe*G!_zWHgYFRV@=9%rNv1X3Jtkv$wR(7 zEpb_2g2Qg&?WJc(w3H!TB&s!K%fu(B+B5Oi;FmC(lQgwVCNnb`mu%(9kjKhpg&WWQ zu(+6pO)7EbH(L9+D9pc-f+@!k=3z>ed?Ig<%+nT*|H+<#TuHyDX6DM0l>UVjQ1<9_ z7(WkZtkLyA5gq2D`(sTK_UcYPcAOIxdnrTEnse|gZv=Hd)HwQZ5}Jj?F#~W9$C`|l z#vC|R`by=I{>WW~?c0R3ePC7vN?MMscIqi=<|W(FDN;8+X4z?{*?3Ns0)XxndqDbKJ#PN2<9Y;>{<29DB7>~^TKpD$OrpvP{dXlwAeAbrp zjirR4!Vud!hUv}nx@2fMVSf@N877$q892C91}$)v0cJ?Q_t|yD0U?|PXUzC|eV+9j zC>UyY-2oTBAyi)pu-*P|#JPiK9nyi1( zAtW#=Aix@4Hvkj1fAmfQ!32;JlAqZH`}^iXhPGW*D)lO`+k#Ze-+ig1I5KG+#kQQX zHWHlD{t3ei?CnChMDH%*=daCDLhbA|sC9`R)ADAu^0YK5eD;o=@C6JVhjrbGh>_tH zJ7Y$!d!kFDh}*BuV}l-4tBe7S?fFI|$d0c1DPo{QM%E$Hjk_IpVTW-go9>^pKl31{ z`SgrzW#!?Zc21Y_(IRS}lATF-ZKP?bHGFAO4)Pu4e;R1$e+D}8PS*=VwgG7_&zp;Z z>0AzzAqtY{DN@gNo@q~o(0P2UBj4_OlaJi2w+&+%xBR+NmGIivx@#TNan_f2@vg}} zF|$e~#zl>7kC8-1Qp2ea=Z0r<$eoPwb|*mJ9~I`LuilVq!-QCbUpM#5lhqC2)r@1V z@dRLXfAq9;tv5yRZ-Gh1`5-)<_-hv@i6n~S*FJOQ>S^6D-xmX@&2Yo zL+FSLhm2#U$O6YknE0UOOc7o6?>-8ODPj`Li5-kfyuC7^;As)%m(WPA;-o`rJo>65eXKGL5 z^q*#}@kUHdOv9|{j&O3DH$ioJpkaHQd+BNBaROzBS+EHPSi}aUD_nvK)($hoR%2Ar z#?UK5$Pq%8J2LB$p{`Hpu1|_KX9!roXV4q>>k(vqAj=!k<_L5aRrPH`C`*3l!E18r zLaK|G7$kN6WC&NeueKmh3PhLHGhjpBq&PNY;B)#eLOmr+8f2IZ2dQOgrV zjrGH#*3uYAc2{7I)j}r|c1;e&SQjSs%KRKcbvmLF{nov-U|ylDTdehp|J?WzHC;0W z)Diu)yJ+|j0brMck(9rE0FkZ94X;|+FYAr zEEV*c@;%8&S{ye;*Zr7}o4P#r;j+>D4t??68iKx6>tlu!*l{E9_Y}h9pO|e6?)a|c zDIQ6{O+x^VW9xC#_9xlqr#hszGoA42JjKfWaxYHqBqy-Kjo_CscLPbFiRI;RJeI3w*~3;JJbwHUeu33QS1@L~0A1A8j?Yjshu( z37|ZU=C%xK>VlNl0EF}H|6y@}j4dpq z4AEo>J*5;mT-ns>DqdGahZPlOmo;>DXIa7e%$VH9=?cwkz7Ms#$zyiF#?BZ*V?yZK zVJTz!bxeL^_6L7Z{Dk%$Lh;h{Ab+5xX=w=pt)*;X+>_%m%(}})lmZk%-7dlcR1D@) z3>Q<3kFXf1H6q@;sN5kO`a?-1`onAwIuZ;09%N%3%OSQ$&0YIf6Yh=X5`~8^TC9l9 zYEFL$S?!*Eavs^L>DvxUu>NH0J~#a!x8De{NI#9z^U?pm^AeMH(;`Cpdu7@}8W*u#@Qfs^9Ys{q>e%7@oA}g5RL>a$Z z=aD*edyZHbr$e@@IpbbrU=y!&T-4d2)Ry-J6cia1w;6vK11+u$KjYvc-x8g0>owqB z-D=g^c#0P5kVfk{$QE;%d$9S##VY!5)IpbIm+khdN=Ge^$jmeO2x&;L#?@}PGIJaM z(WBGXU|fr}S#_Y=(A;azm7<&XpG{V3)7a5{l9^gwrh(NrGfw3Z%~O?3_-kfmRorRA z5vPi3QTr7pZnxNVSGS}{I!6)F_hz=;b+c8M><+a=F;V+Q0qTBTo=F8&*_NfVpYSTZ zttlyVoA9kdr#r`F<-`{^`x*9OvZ;~kc>xIV{ijC!j_mE%>RF;n@I8?z{IBS-o6PYA zDVi{l^PQKUyRwd%W9TSDbTv&~BSxV*kCobRICnqzub$^9(~t1PU-)SvL>K$1Q%Ihd zQ}4csN>-ObbzyQogWBv}EoPh7FR@{uzSaAPQFo7;fPg6m&^b0tRM2QF8XC(AgUmu3 zSYHgohs1$U;yNN&Nf(F~d=mM69pzUF>8Cd5Xv<|v*J*e+0TxnNNhTQ<4a94V>l2fz zK>?-X%=YpUTIJpT<{`&|Rc8;A-3MCxodQ=h$8->-N?~&6d~W)oDYwD(y|FPlk*-Cy z8;kePQ-?S|PI;kyUlEI@3e}SQqP=qnx#U>0SeH;676%gk5_#ii-HxUMU&xCYXc0dF`(Nma`o-1C zrblojoKNaazhE>#PPeLg2o9>~3(=fo;+%prXRP>2ir^>;V}R%mLHJ|7G}A(9K}K$L zx&t1*w9#}2KuSq*NK2CiqE^2q(NZ=6t?Csp$!N(NBFm_irjk+0FWMq&2&$j27Fj6s z4xvqk3o0UiLxI87C&S-5$vQEivO5JTxom^FF0eL->kYs*>qjKs!1cU3#|t4V9j_AK z9EJMuisa+jzfxt^2FQ1~Au>0CDsBX*K1f1TBYSKnbRpk)c125-=AYf*!D(cuc1I6m z=g@&~a9wiefLC0F;5Anmu$t=+mJ?la!8i?dN6iVYxcTA#JWa0|cXT$__##1DctyH- z(&l(zem-*|a+f~zg!_|U;P$}z@h|-yn2`@>PJjUb*unoz3CsWFm;UDv@;@}h*s3VJ z_G9>y9!Tjdj0oK!WhJfUC1th+0(=%o5?7nD6p>WtK#Yy!lO(SkP2+@-0umHR`9}IH zpj6RzT^6NAC}Bt(q+dR zsep*c5ipx^w{j6Mq|mKiN^34UQoY=KNwn>`>U3M1pj%a^GwWHeM6Fqz6$w~$sWpKf zf^IaLXD^FX^%>Bqc`hhdqRdW;G}}#aal*Duy9pb>UtMaoYB~RcwA*E_Dh6hF`xgxC z|0W}Ge50&5CwXtv)UsgT_L$b5<|MTW`d#RR(LrbnP|85UH248wgh#6S;Re=uuG7@z zv9AM_N*M|k_)j|dDxXd{&$ZaMyphQ&EbmgN6TFllZkGj|F6T9px*Q_r$JJ(BmBsOq zxLk|Z$4SauWTb6uF?oYCE9NXZqUHSQe`CNsOIT-&5axp7s5e$ibDlW&P!EHiSlBJ! z!fpTlGgfKVX|r0pRJVpVzUk1L+$IZj;4-&2hQ(UHTI<@qX2Y|M;t0(BhFSu-QD&{) z1loPz#J*n&hH(1Ha{Rzk9kLhaM*>JMeE={>at;l64*FX0CZJ0Eg0b)h`3WdU?^!_qc@rNn zCDWuf;TNFG_YKlb3OOr*)n3qo(6uWT?_uxz4xm~aTeTRgHxId64p1#xgWyk|oL-1b zn?s{3sIi@J;bCzo@F@O_S*b9IkAk_UHrnd{E46bKS?0D-xX-QjOeI= zqfQoUili}!SYw2tniF!g$oxHM>y_*=S-d`77o2wv5@!^V88w`TdO+uhj!1}P*oJFq z{m#3Pz^=8Bv?pyfNxg*I9{R_ZRGm?P6u-4VUrfg=(6g_M|4qSofoRdnbL`;*xa!eX^+Q0FKMuS~Y2sF`)H1KiuaYWOrr@%z-F3f)OH5by z(nE@ATd;jVHRjrgo4Y5tBVgqTm}pC=VVAt;5R0oFb*ekJ=kG#LAHm=B9(sMnrS<1W zY`RnJ95deFyG-a-jU(mU|C;u!aGlmYeo;dE}Z`{}m#Z7=iJ&-l7M z09vCVLDUBy4MFK36T(nWT)2}j$!t;FI9D2pkI_D5o~4J~Kne^pJxhdF|@j0d+ve3UM=JEIu=huyoyDbS5J4qo7^P5^BZyL zEdHPHwv&w>n_g->j#7f$>D(t&po2%J&n1SytEZRZB;`~B+EVO88o=R|RICaKfBX4^ z?v$#93KU;?v<o%ZVKolmO7k}qM!833(TbTzRfW~`sk$Q!>O(@uHE63nX@7= z!B|UVOdg{QpXEWVT~{|#8?RG=GFcj9Q7rY9^UpvSo@$L{gw}_!=eQ()KQJ>e24KTB z>Oxq%-5ywm2RJQZoz%{eID**1v~8Vzt)o%EOST%R9}_oLsdiYxLOsllk#XzXN=0zu z&*-++fcsC3FrwV?N|1wfK^QGmKTaH#wq+OK%Ff4;O7)RvrC-AoA{b`FuMvDTTbQ)G zlgSOnLA)4b*ot!A7=_ksa0e=in72z5c%!}}gDhLh_=?X^3gO8i#=Wv5WLOGEU4VuS zBR?TxCH;}bkblOdLS~ERL4&ZK)^ot~g3`5Ht z;!&hnWUGiXIG1USXOO%KtfGa*4{P2nQ4!DiLnROEeFtBRRS)mbMkdKnU2n<%`!=|x zbGzF?0svU2`ZwD^#MI2t)yCz2Q~I2%3GJb>+WL1rb38LM2Z0I0!~+Zg!V@7dAc+7= zs3?)7M-X9!kaJ+dgltYGU?Kgrw5^S=`mkFyyQ;#15u-4wZFH?otGCv+wbs_Q{%v>n z_4S~)74@L5bv~0hlOY+v?=;8v)%d3MZ|iIS8lE2K>-H1C=|Viv`JlJo^^pI>5kK(v zJH6$PqSh~MJpaXebI-5%xc>2vI<9~6+un}{{%`fsFA&9$H{ZALE`CIhKiECJ4@JH> z@H@mfkO#u}hadz(A{o3oVCN(;tvx`pM0%P$dA|?HvEqPs0%5!mCU&86BL1>PNq!Wi zhCuN-OsN5&$0+ZNVhVsJQY6@VMxh zCSTTR@dcB6Sp1Yn;dyto*=L1Li3|!LDp7%wma3oM43q+E?oIIqUlww}VB6rnDi4!- z+StU(Xv`@+XV_CN_M0OPveqJ{2c=_Z|wtIvH&&& zanXTEmEu^IG43hdtqmCpXKuC9K}sgcj721Qn_9SWrA3jKxU6dtcUqK7)@p8|iMOjW zjVYO1NOARy3hrPx15$Je8lsLV^E%QttW|YUn#7FF)$O(onOZozcqjQEBzt)A^)O__ zk2da|N?XL;TB0{Q=^5ICTMMm55bL(qaQA1DnUYMRl(ZAQHl7@btVpg!vIGNwwlSja zSrEgagk{KLwrp);##U`%o;5^C(L=`D*6yZ49J@Q%!$6I314B^AX6ihN)-dIptsHvR z$OadS#7=N^t=GusG2I)pbe5D8lhaVvGb|y(uRdRI>yGKy*t`ZBviJ6p;>GtlRiG~J z4P@&xI%oCtjl8IlHYBp!Z8SO7O3KMJkDltxVPYDgbW%09Xc8lI`Po`&4%F&8{hhKQ zxgbi^J-acF&y?+D^=XFiAk5q24XqS-p-4IPrea*_BraR>5E_fELu=KJRdLO74C>Zl z9@zqLi7Bv6+QOPR-ySf?4yP4?*HHHoHW zsPwhvMQ&-awo}#emLLTq%T<&(k=eKt$Y$cDIXz+;Zroll$;q{vPn2~8AIP|tHiz&OEmW$2 z**I0+-0UO8TrNoS2{V~Eree-9CYm)0yE{3O8I7r|&aU$4H?}2T$`fGsne9=uu6Le? zK%B>#hc)7yW$k<`%go{>F`06Xa>p_I03_*|3d;)x3hW9#>)%A}Sr)D|Ez55;YEh_W z6*MLl?#} ziFO`V=Ji236UjF2l-}JE+w%ZG)8mptQ|CuqOR}wD-P11Bm&Q2M9mj{$=O*O0naVK= zL2CmyB^+Ih$eRdz^m&cJLedUqcqt^5p`~i%0}Uple6o7ccO3bfN?p%MjWkC)X+{87 zw<=Rq4dC1^5`oOi3+@Oy^kFG_@p!10E3$+wh@ z*#Tq^rSfVRDG4NAX9ozH+c*(!647+G@wV|pZsXEO87F#GQiIBU6>Td9KsW4`V7JSb z=>oh>+l6t9b~$|ZbL3~qU_McGY?s*Q5tvUF-?4)Er8ACptY^=zwSuc-M_%8-0{W%Z z5KHnCda+zPhI8o=h+mv~%y(Kp^MzO!WxjI~tlEf#B7&t2T-fqMNMQPXhUH{I_GMbG z9jgV~XVjr%Lu#mCyK-)ruH;t?7xW$51zw|pUK^;mea%QnxS;Zr2dW+Og`ejDLH*L| z%xKt8EZu3vwgyMdWujoblJJf)-3&0zQXe``^3lUmZWV`=FlJOSOAgan3f6lvzp01DJ1&dS^h5O9V#lbHmTB_mcGj`vqb!Ua9ztmi2=1RHCFkDbCo39`ZlR9;o${ zY9|EonGd%mv>BZ(%SNVlU82aq;G_fnFN72d2rjn-u*#MJWkRC0L=Efmnv33^7T z-sUK#GxdBcJ+MX}bstep38bAWyVt7B(`+|}$V@LMB29-=!T>4jW=xoRR9&J+ zl@n=ssqD0fVhm?w8XmF3X=vnBPEqdSi8Vk|Czi_cSWA}Te&4xD{L~y$C2fi28lw^L z*MTb?L@qhqHJf!3T)Tz#MNQM$s#oxC&l|#UVKY>_k<6ru9L@M(ms0ZQ(2>L>zrte@ z(U5j@LR6r=lzzJ9u3AhU)z*C7 zO`F1ywYfPhT^H$VMPWS(8iC(m zs2p1#-5ku~07H_kRem5$$Q-IZJyI|lD9!DF+@`*!GE+7(KP~AjZ>LuSc};AxcgN)U zIWaS1{oy}Ch5XTcqr`p^n3{B~2?TN9g_xoGTb?(<@=;2bqcBe9#+1nW4SKj@RLzQH z9g~%N`N(030j?!ur2k6Oh3~S0>uyO~eKM8?U&X?UURs|hd*D`vy=%CcaBLx8-OSSX zuC${SW~$0sZFf#Jn3lr6i`^EA497c9!JT+0ca8X&8oXKLcS@1CU_>~5l=hTutmc3F z$&HO+&uvfh-0w!=mnhX(_aPF`GVMRiVeCVvZ#D(QYuL00T0jact6q?y?wZXw#;)%_ zhe6rk57XL|mCmf@n{wB_Ti5WS9ma0x_Q;Eo+z8D}a2*D1*XaTDecD~_^)#EjM~HXk zP07zctpBpm0g94}50KiO7pPXp3%stpU3$>`jWbvj0Hw!=?zeK_{B5cseebC=Z9nAN zv5VFop>Fy8nx_6dn45^C(2G749Gw!fxJIKL)Y2C~-vraH6D6N;V%7>atu@%B21glv zoMt<%HDI54Lb(TTTTeVy4Wr2~5HpT=D0CKcL=TJ9x;)gi*U6xQRQkZ#Xy%f_rU=mc`qOcMJh%2<+z-XVB8y z=I#5x7<2 zYV?dd*>HI~t=!nItgP$IwXQy`KpJJ17d&20X zViuKc*AM%R(*7JM`}>wAb0(yj0KHb$3{$Z{H{*>J@{_*tBW=unS;F&82y8_I@6S4; zX=6hn+5oUp7 zK;v&VT?v6LQU5Jjh?^qgEd?Py+2T92)IBYnxDYn|eII0#M<&8V{@4OYh5Px=7#;nA z$$`(rxHnDTIFTEIQ8nRv{Vf2`^hwf3Cc`$&7s_kAaaBB|!0%HH=x;9p18PQ8020B` zsYp*Qx+md7x^2sV@E6ns_H35_l*x$~P|)6|wg=;=x)kM4?qY(AULpg3N5M+2fKID# z(S*oVBG0AeviGprX(-?NsdN}_>X1n4oeDa|99@oFvyzC*yg0k2!Xp|Ut=EpG{-fJd zle?OHOsnhO+Fs}$1j2v=6COcL<9@|}tB)9N{*gq-vC<#30XH;MtlxV|9j<&6s02g$ zxRICFs2xOhOTx_2@LctXF&)@^yCjLQSQYOAbZvBYs3Nn~(wp2wZ;GJiN~ANTWxx`a zLB&5H5mG~9R4d{)(NlmT9^dXt+Rs#k#D0&Fu@GM8(^+Nx2`nA#J1 zriZr}tyv;Bw!LVRi`HlhI4gGa7;`+OI&rNb#X7acI`o%q2^yG9n0d-w(@n_e4SDX) ze-y5$w9}?HKRukP4Ld62`qIrw%E)Uw3UHh)jA-Uj9~9NkWGzS_#P*Bj5po$~3&6yc zP>7`qQ@Z9wdAUQ~sa-I%s<=+*ny0bD!d@x3lAW2E^QBa#sKr`&-0a!z5i%8>f)Z+V z{2!a?eaD}N7IZP9?cx=*M_OWq1~_=*G0c2XomJvPoIrH|&=K~=m-a@N_GXv%L&ja8 zn2Wifc&$r~wv{I9o$+QaDf|p{ZP!i~SHxv+7WBd=Qzy!%T4j+9tq)KjG}Wp%$?;}B zd+9F^ehBZOw*^o>gy2uWW%GhcazDcO%=wEFnnHkvwD?1mIJ~|YF>!D_n~01E_L+)Du7I7P zrs_@@zdRj#25gG)`p2p)xC60~JCGsINXs)G{e^ow;{)}d011ExI*r7dl{(Um1{>rQ zIv*bff%=`AmV2BGWn&KLodbAj>Jb3{PJkQFW5p&+?YfT1k_gQjH=wQ%ni1w zg));i^%JlySp@m}%)3=xz)@%XxS6L`rtVbV_F)PTo*khPM-;sMo7ndA83jRN)?pto zWQ3RDwQu{xWzb%iwgRJh$?O8TulW&GYQy8IP2yn-u^dWjtA3EsC7=|?`fnmwnaEvQyM%UtZ{7z zv$1YMYKEz|vrZ9j!mic(EquFP0Aj=zX~JJ4%0+~xI@Jvw-|(GL$?J5i`Y#yP7LSHs zAMk0G{Ua>b9-8I7F>xEWPMyz?@zpOm%8tORtNn%$e;xk{KsfB%h5KVi3^2Z3K_N=u z6_X-R>y>4wAHrUGf5P;e(~mw>3@!_KQ7GmX?!&@ zrTAhfEMH8FkHOqvv2)ATo(ymAJy_3uTk!m@6>P8anZ-Y7DJ%m-!k)hac^Vd!Iu|Ni zY3>=SDO;GZ`z=@;*W1NrlEB%b%0Gp`2l{^m{}n0Az?zxD009D02Ll2k`|pr4Q*$Fm zQE{RFR}@KA$6=8P?K>yUTuwzj)rqj!Y z1QrD|N(y!MlWN!rqp&hWS*$H#_U$t_CgJgM^+g8AuBCabz<6L-9K*^sYpqFAg%@XGO>(pRmCh~kY(HO zAunQhA)9dZmlHV;>@RD62+}npO#`5sLc8!<>z~+s7r^27nine{YTd=w((n|ab{7n% z?S`;GGbs0SXVyh%{PzgA+JZ}$(~d6q{tk%KO){trlB|dZp0wU0Ig6bQM^|Zj-z9j` z6F@>Z7lIU34tKWEjuduM93egqgR0{p;A>5DK+n4DZFN*JV>;I8*rAfSW~R8vJ1#7m zU!@(by4B|I`J|)fGpjT{RCG1)e{|jV z88~IAU<(V$nymC2VAt6wYK2Qspwgt*u%Mh!H=mkx_ri~Fo^Q6ntc&jFwexG+1QtXW zU#k)>kl+i`8Y{13XSj;L3s~Pnc)s!Di~VQTCj|^KG0bQ9eQsWS`}1#4Ke$f6zxy5f z!1s{+>xg4vad?=kNeT!4Iq&MGq6Sbzl*Otr1o&zP)st2lz?*T{xLWa8j60~pWrW@J z;b4UiS!o1rhRopQI*RsDUrVB^X#|dk%Gq1?qN8VR2Q2Pu!tp)jM^4#V_PyZq0hreR z;f8DZ+JvY_(^q7~sE;%hWBghbU2k>v=aEKV`MwE+>JCvW&FF>l$#p}LwnB=xi`JG+YO+2K#cU zEvdaMj`|i>#zN%{T_%^ccG+Szk#1vAyc77&_fZM)87sXB87rsJn#sFpFm2Gjsjfoo zWgE4BcZ|NpDB1e$`kVpCRSniqB9K!43xK>he)g44gdBeIH6Yj7{N1$iH18SV-T$X$ zKZi~=iPX*DLs3F(VEzHAf>BR?_xrgJW(hjK#nm3yMPq~$ZA+s>{oti7KXni^yg^}I z3Y@;Yi3cEsntR(^L)Mze%(9y!cNA(bC-LAT+Lj!w)Q$mijSv1NU|WU{v$Q78lk6C2 z{MCJOpvhRGe&}Z!XG;cXbK+N52C8C%jS4;;7$vvu z6z($(^@b&j2_Q~LB@`f`2^oqm=ofvl(%p8Qjk6CL9IYL|Z`;7jSwgK9NMA5&TvKk0QatRn7=@C^b1RM_Yl} zm%5CVQa%D4w6{sqcL z%g}>-!H)=CNm%h(NLarwLF0b~P`FbO%mdPy>YG5(tMJsb^-`YlS7&e<7RW3S%SOLi z8-u}B%9}eI1}D8q8EU;*(kzv@!Ow0?PH(!7E(gxGQkkbt;&lu z63@-NiHZ4Bx6v9`&Pt(kZb*@|hXKf-yt2C{=UI5Iig+W6hLz#V)Q*&=KBKZW(E{Gi|w_O!V^D(%97FY0};en$#WyZ^mSnT34{U$j0!$;+4zJiq(8CA?06cl-+c z?VXZ-MwzgC!sdn#fwHo-`3}mUZ&VQ|7tqhS7ECDOo;BGiLb^| zMe7`X94h4?SvQIl_CWIFm#~e~%}Hwe(gI#J2IcqK+V)+KlhHV1z%dZohRHGxvYWK* zTlL2OTdTV62&uE%6=rQV|26Mz$t)q;96LlVZk1=v$f{Ro$XSOW+C+P5jw$7nVGc=# zrXa!(i#v2g6LVx>uQ*#H;S9R7Ip}G7lnWw?7a)f3>k!ZHWUN)gwnQeX524+e)&n$E%eDLwOWk@pg-40 z{D5LTUBzByWX3getllb6N+PUk*HZ7NGi+tH=vAVSPE$in0n$SaFtLZo39m?Z60ZL3emd5qIM)`8 z&&;CsMSG>%Fy)YY2eV;JSFwk?hOVz0vM(2GKumgzW@tHCx~SfTTAbNzUuC8-bDT8- z;5t0NfOtpJ0MqH4W4znbyVBQ9ZtAC()XKLJxe~lp<`lsME5krO+byJMuy6wRF)6%b zdCooSiTSb__z433CbfB10N=M6M_vmqFIfDA+Ux-0#OYh2Oe;msh*3uv^}w~Ux(78D z{lltq#W?(+vGJ~hO$SjXZKA}KYjOw8>D7j*eS3hz!y5v{g_JIoYK@Uhl!Sx|A|>V{sn8rQg)wyDvpd!`8zEP$Z$DNxHner@ z@GZ__N!nD(1=lmb#d24q^KW7IE0C#JKk#qgdS`t?1qnWVF@J&AK{jB1fOvv@fNBPV za`WM%O%~$u#SGCr0jc6a-7O@=$cwrd70A$S)5ISUA(m;zcYTD1hlggG2n$fi^ZRT1 z50CH@ryT;XNuCS}Nmu0D9dZlGzt0n8^ai_5B%F?X2NNiq`*LD0R=m;bO{Ax;-p5nxm_lr=xDdnxHg>0 z6nY$*>Ra>ptoBO1ma)$j2=ulYWY1DgXp*nFp6^mH=4lL*&x#JRw@k($S#+`^N)49A zx>%9KWODv;pLG6klV%KMlP%LJCZmIL@kW>Ox_?e@B;AlxBJk`bGaDd<fGG4~Ul_kU zWBj;CDyy!Na$sTXw&{@H&cs~Q%&JX_JxuD10bNg*MX6$#qElH&zU=CZA;ao#qu(K? z1PCn-fN(MM0-1l^HpjK(&<`&zttXZc-wJ8ujA;$twcYb4MO?Jb_Tq`p9QQIR)X0e| zE!{liWWKquq{WG(jnQ0ldj=UB_;mhLib?+EQk*q2^1PQs2M^S|sjzubvSi;B{svXP zneC%u5xdQa&ul~7*@XtfCRQgtl>Uzu0s9D_Le**;*VkeXd$Pdzv4KMn?eZm_t`ksun$X6kjt8uSPth9NbE(W;i*J4|+11Pfc+~ zjQhSlJU`({`eslAU1Q&dkvudkc+Rdv#H@V}n!X?Wwql5gPnWh+eg0hg$YdJ}08SbY zD~^XbR-nu0;##^*)vwGG(K9e7=V?Fq!rLSL!>IVO6D~+%2|Bjk_lW=klQnruq%q+2 zbYDRI3h%W{;8EJ1;r;fKNc+;As_#)+Kc{il8_OX@{iLQ^J#xXngjvn0gp}hBZ8IS} z!p7$ZG_YMLybENm`Y{>I>P7$T0*&Dejb3dPWfcUBb}ewJQiq_|&-0|b4rI=rj%fG{ ze!uYG-?vSAzZQG0Ys2kv$%OisYCuJ_40w4hu+~x(m*W75qcGw(Wg~d`JsU&PH-6hb zPTL_w7IPp*lo4jKE6!u34ya%&PD5Urn)G$yUvG*U9Drqi9=I}i%%^srG4jm3M)7cf zBi2u5hj4#(q&qaj!Xw5(0BGlvo=7J}ny)9O&8lM3=?=&Yukn4q}X8?Iw;>8#_4 z2peIS0^17cgnA&tC^h?NcEnt)GaRSuHbZ;N0H4RxzakC(h4`M|l5fptx4wfk^GJ7FCdr=y2L%6<4 zv%EEui{j9@qfhD$&I$N8;W&7ql6oKzARquJ5D>@ztxWxg?DqdMcl_VhDF3BkZ)bxl zgyhc=<;Be?)vJCfvm{!rUuRc0>R%0oLk$)jog^%1|C`;u^D1nzOcgvJIap3;fN(DX z#p1Tem{v{js)vu4cXsN^_v`r_HjsjYK1GqfATkS?)49WnIK-NOR(zM`z@5M&*8Q;p z_xiXYZHPz294`tuWEHx&a5fczBp_fhp-g%*>8?}VUn3SO{Y$-~g=sWH92?yx z{>lcM2BNz*B~lwWUlwee^GA1hkNkzuLGa4DZe-$?x+1#=p_}Ko^;pBv3a^iE{s*vz zh@cSl;6?ai1!G$&b?Gaq%8=-Wrs&WpHy6i*rlF?5_xHRi1Ss-+k1nnNM{LJrqINY- zwUp0umHyyLzhg{>swP`>MAH7U#m2$V2_KuABb>6z0|@N-i<`edS>#;7ao>_IupZ?K z8<(3mfU3h7~ z-Q5Mg17q+~N(d>f7_?#0e;Lf&>>&>4G^@}bOu9lIPrvw|NqydaOOB&(QDUV%DS)8we zVmcHVZxe^DNlJxqIk+klF8uk)P1_nYYn@`CcI04C`!FcNAPT;EAXl#tI7sX-xi=3X*$2n1gqG@+9g;hk0ntcAXP_7Q4z*otskt3GCmf=B>GSQ#X-GS2I-TcN^LG&KeTO`=ynu@hmc^ig)!6sFG9hN)FN?vnV*ftpIn;7(g9CG#pMA62)r zI284%MkEe)k?F=uWmI!t6|!YOFSMbE9uc^yU++Mdsqp>qb?Y+YH2pAX-?~!8;mnCtFv9JSAe| z#FI+WQCIX_#+Bt$$n!ML;`qyS(fsDtf7qT(b7ZIA&9=v-ya3!O_-PH+1M~4Ud6Dxf z?wpg(M5*jW+UX1#+AKOCwFmLRwVuiY;%riDimLcjcZFI56fF&41We0Ad$o%RjyWYX zuUN@20t4kJ7ld{i1JG>aud)NDT;nZNTw&WVeSb<~7hz`hWWbT1Xt-irVEP8Rp)U!w z4q@IKb%du$wy0ilcA4-uhYxg^b1c;CZxvsk?US5@2537f3ee?@M-J*U@>!v?e;gu` ze}XTJsp+nODD-C;t6^cB<&(#1+66AxjlLS|bfs)mG^ySWoSRC`Q90VvtSn^ZOk2N= zfYI&!C0k=A8?n&;PSwq~a1Yq~SgsX&Q{Y@p;}mNZ1FXU8Jtth*3kjEuU@Qz+gAv4t#t* z)WCixcc+_9_)Mm}$RD1q7i6?WHfbNu?O=^T@S5Wy*+D%jzo4{BO_q?-cEy`Nd&dF( zh+D#8b51fULDU=0T6wi*ZGp*C@6;)e^ACq?y*wiJvqi9{bckO|e@&?Kh92R1AmX5m zB97e>u3IB0bJwKuJ=QZPq`EpT%1ZEN`V~owCB$D5**v9!3Qoib9<5rmwiTk7v~`GQ z4Yhpq7&<1f=^W}^rVYh>wo~~EQ!(&Ql?DT8oRDdqtPnDr#gv7MKCZ6r`CFaj*GxHH zL3anPyHGkJs$v{@>%ne{sx%7hn-Z&eSD=2CgKNr|{84LS7o(^v1X*2YT)fK4B2a>wDo zu7>1J*Zt*E;0zyT-2N;;6LzwJU4vRQsW|-1td9pb*I(%i^;Z_=4ivzv8A6yj+HHI3 zW4n=`F_)j2Y;fYZmakyP32|SQJEqL_Eh;%GfSCHMbIuhApVO;qY1k4+^@aIw#sFd^ zGP}S)K!5%x>F)n14YFpQX3qbo(O7xje(@Jlj*M^)mWYfcbxCPr8d`Z_Yp5r!u(U9( z7z=n33bn#=slXI5AReI`L?DC_m_QktOdv2p_^MM$O;R$fbhCRi7ytJ9YSv)aAIKA` zJd!0%E<=;qaZ?nd8hmU1L(h(8qXt`((=mbgM>Cwoe^9Y;Tk;J7@$eTnAOSBmv=I*q z;4__D?h}`*old+O%{RbwI-z()8PcZ+*+ZCEek3lzcjip~*Y9d-#NmiHyf)ym>!4Rr zbO$*FYb-(dn#q)Uuz{x=2VxTQzU~w@{YC6ggJrwTjgJ3RZeT>;VpJ-eUof*&^|YaT zktlu?e*5`fO47*}$L(m5X!w|*VuJ5!cM4lp#E~)mQT~XaMD6Gz!dBqPv1)Ia1 z|DMm$kETI}|CSP1gK#5+xYf}AWj!Wj$LPhYdfxY#L;o48YO(v(g2JuRE9ez-zGN|} zwRQS&Uon*6++N{=nR>pXRCOqNI}4p*c)l{mXdsV-orY>*?2-{w97fBuey&#NFt4z; zL+b~|J$Nb|%{jFu6QYVL@yx}w%L!(fudY;7EX6U`(pmG*L56)k6OP(RittB=1OlQ(`X4;1M6HZ0 z>>XTOtxWz8zj@SZ=y>32pns3zSe49kVFKK%N9A+b^p! z&oyx-lO;)6S)H#Pse46aZo@d}+d*RLM$D4x&~wl*^zIfQzd;l}A$BwsUuUqz+-=w8 zZ1m%wxh?X0`@gRJp7Fk}Lj?OE_n`bM5A9{o&jws$7ebVLwQ79D;0%`U+4)L_hGp@< z(bzS4%#~0kU6j_8bYu5XCX6eF_X=p-Pr_zaNA|?`&+n8L8ww5)WG_9jVw~vqo3Q!{ z_p_N@_jQbKH*pB&@9@EyZRY7-QRTg)f>>&aoVVOM`?M#+~wfYRZ+A>x#)dFUo(eMbWyGmovU(Gn19oY_7@OJAoGm$sxxYcIc zVJ&pqFXGqp9JteXhh4_4_HDaurZw~eI#^87t=kkt)03y=wjH_%6gYP2X0kGq=JyMA z?Y*yxIt|UO`}AgVn!b9D9UOT^R?wY?vyeePfSHHLMI50Y_4pf?D#(uIgq2Ak3!e`GC0prfU)&gN~Z8>J;nUX48Qof2Y z)OiF@kZjhD2Wzb{{ISz4^pEK^5c;5z;ofzY>!)ETL^j=|flwr^J6@g;*enbOWQ6&) zo?z4{>7>}MLTsNkFxl=Al#Dd&{?WfOk@dJ!L`yJ@#+#egy-1Oe|wdAY6$~bkA$DH;c z`l)>I4mhvK{<6;SpJ4U4uhjl!hPx5v^M7;(k7w^;|LM#6&|whR)CA4ES8zt~ZD@1; z_VihS2Hw!o{ybqh8R3*M! zC{->OB{Ix2vxybf)S$0GvbVQ=vq1Pra0tocSU!jV`ZMuGVJJva)vJX)B9-8fTo<3< zGbFKi2+AxV^khlDSJvb&F~lpWUH?A0wUl`L02iXuZXzAL0;wJ$4|!Qtvd-1%p#1(b z(vJE*N4Tcyeb%1plYDLir6*KF`U$b*v9V;!w!PXP>&THNFN(6kLZa2{fUWHj;SD

*9q4%@&;xtiUg$r zYl|+5+6d1D%uZ%a0W1)wfzk;of}A>eY!8*CLdy5|50tR-I~uu~v;?-kru54J7B$*J zIrAUUt6`{qL@m+tzg)bYX`VOxorg<{UiOUgj`Z?)IP%IYci2&SR;7ot#vOS<{foXi z{ULU>TD9_uuJrVcp%@63Q83!ZzmaEI?Z29;47B|5z8WA%#jMOdDxdfmn&P5977YZjlWbM!IK+I|!Y z5|Bp^k#-;L`J~DfkT(`};9Nr&0MM%F3bEb=KPBLDXwpCIwu0kE0QM`sGM1$_+YS*w z)O!SWWdB=w(QXIB)n%~NcvI4-v5!bfl1a;cp*i9e+fsw4$y@+H(!A|6;KvtNcbl)P|Dm<@{30z zYxX%R2o39tEMI%x9j+Fb%9OKbt9gI}O8Jutg8E*NZ@S+px0v1VJ>pppqiV7gcSrPK z>I~Zx-DzK8yPEbzqVCvT8BJz4f4%^PsaP+PTm40U+pX?bho5oV>ULR-`c1^X^+(-* z^IADG4?qnL_aGW1%Ef%+-Xk=G;V0iCj19NVDQ9i(V|_(NwvnQ}A!@VBT6f$rE>!Hr zG;Q=f{TnYPvkSd<6ATDQ9~KCR^}j2!;zm}s{{{Q}Gq8(!nwUBMZ}=xy-A)5n49!2B z%oYVh5;kcv{*FP>n!c58{5Jy`?p7|5^5(_t9p>u@;uBuM8MpUc z4B@BJMeep7&KQL@V$S$B?~8!XE$i&mRer$N18$(dFQF*qO{9@79IBA4G0m?bN+H*M zmHUhoCj3A`R^TaPdCwEvhCngseDL{5a_C7Ebr7h1`#_=_7#lUsm*Klk$qvSbey9WB zl}}Hq{YfGMBfVe}qYz?4v?gn1g=!x-y79RdlL{;P(z_y7MQ5g#DSPT}X_D%*%wZNz zHh$JAMZuzToc@tH)Vyij%+R98nxr*+Ni6VO8*cW-3am0~I0<#^p2Ih`eYRQaE*^t1 zyNc21rli)yM{4r!Dy%F|_Z2cA?kwk1`8Vw;bu zfjPBin(*%pGmSPnoJ`McG7l{3@~pDoh5Fd+-plwZ``*U3X8xHf>8kY!xY6?SoHn8k z8E7d7EcQkzS;>XQD&i0c zVLHi;VRqq~YVTKKXelI#g`z&2fTBEDRXUIe7J@h7syJKqrROsCa&u9y{>uiiL{?cjwrN^VLH`f&6r|7S+ zIC}OGDIAu(L*DS6)r|-^W-6d^asM(OWp0ml=~^UwpQ&;Ov*r;U&nlCA17RkRmrRWFGNh27?SVQwo~P{zM%{OGHRMusjv6yfo%bl%rOEz-Eitpem$FSMi zRe5EluD=;HkVA7x)rnPCvO8)jwuyRugtt{;wDjGf(!wGy=LNv9%?DE#QwO~j$CD?6 zBsaT~1(1OdwTI&=tHrK7F4NoizN8Ca!B;JN$K`xCt1 z3B1y4V%)I?BZ1P;#bSSfX>I{$ho|oW1+ZaIwsbvOa#CHEg{_zXQs+JqvKmA)-7xIy z<0g6big|!oac=p8C^2}$Io+XlWtNhua|Z(={}2a1&E2sc@#GXajcxA##VbBXn-=Lq z*u91N3cRt5xJ4@z(d81BnfE`0sue6^zCaeh(JsCxZ4raMJYe21(A15_c%S=q2zCF; zjLs1)v{ly6{QlV`lCPMo2|&}MlGGcp{teoWJIuas+EDm!nXt`TVB3*`<2+aZO4 zWR@L%QCTFM^QFA_F2h`s!Af)tfok~{Ore4U9(;!qLHL5U;RZ5zLZTN;?Wk|hc~lo_ z!rwz1J`=MauRTpp*G-m)jG8!V3vn5L(fu8A0O9_7l}a9Ip5OztZ*2g2$M*d0JG%YG zr)ZM0!&Ol`h(dWaw#Oy9?{9=Sa585UBLoq^Iq*HuT z|Kc6CrSq>}NuBl6LLPNXU<8SmT^2TgfVWb;dI^s2RksV;{)i!{}3w` ztYPZiPQ(-c7{ZD?!J1~9CexI!B-%pPIADHw_+orhY>PQ<&U8BPf{0M6CxBFiZzuy; zNU;KI`@gOx3b_9c!*>K2(D_qy+t=V=i)>I#fTxAKkHS!HnkLlONgB{>UnQkMR@^v8 z!xI=~<-2`C>ziO%TsCP$grMENO6uCw$fr`7oUTv8?G47Ty4RFLDx8c49%1s%= z>Q9}saHq+!oF)wc$bL)*v%SIpHQUT0Gc zBe)=LcA#)D-5Bb5R8H$v|Y7dA^VM84is|vEw+H7KkF-lRQyv$7xwXfJy z6!d=H^t6Y)f4p#7#zzMZ4ZK-FdtI1qzW)ta*rbl2qy8tX6~O-k7xZ2P78SaKj8r~Lm{%^G8i#N?3~^7SGF{kl-&VeF zE7`&PX7vR?8AzM6X}QKlZrZJ%@XJrQ616%Ec_LOUW9$l%P}uHhr+X)v7u&lSAn$R| zM%Wd4VMviS2p#s2TR0!|-~hu#x~8H|mSk$z%uZnnVHdo7)voCvCOo?c98zmWH6mPB zzBL7YA=bytP?A$m;(2N`o^Fms=}n{8*tC*e`@@OqzLKktEbjLTEk;b`g^gD|lo)7df8X%cEP)oJ9Q(;9>*iY>LUx}!5b ziED(H!b$rD1s`+(CK&~7bwEc4VT-+wA(<^K1EROUb}lZm*%Bv`R%V5(Y zXHEv#0I9g0N-}b#I%!hib6c>C<)=2X?I~D$x z8DnI7cMyC#PiJ|{2dOImNuB?JeoIY!3H{U_`+@Z`7!xyc9Z#4K9p&Q7*D}!{=>Ae2 zAYd|L)NfJ{&r0N5Pi7ouEcs=k$AJbz{TqISg$7uMAyp|e_QaBJRiV{Yh~l^@R_A2= z=8dJZD3I1OD3&Jd2v%te6Gn^7v~g7Cvqg*R$x#iM@z@O9m)wB-HtplhTAz5UioNb$ zg;j6t5GYTVw)^tIvs6}nyreHs;}J)&{@9I0kFglutA{xO_rX(zzyX6p=SZ!1R1U{s z*%LQ;XQ8Xv%fRRMmxP84EowV6`c0%5syUPOGx(6y5WER;a(S_hL$!~w@d3&xzt2XO znQv01?}M+yYTn9B5Po+yFR#vKrf>JAwzf9!F&itoTT7*f59E8+=1c2@@Z(=A*fok+ zKhVN6%LP;I4T2cNRZw9r{Qy%xKR#@B5#(ZjfF#q7$!MHmNs`rfAU3b9s0Re(wC1O# z0v$FXTUSUSmx|i+Vv?e9gTDKoqrZYB7YsK#txH8oL=Um8sY#*8D#$BFib_mtrlnxox3@gbFp(}uGT6-+AK#M&CZ zOz4y%-3CVMjtwX&adWE;CEiIO6bBbdtauz#vd14jj%fJrAW7S?KW_s=vdgSg9rV^X zL*lyEwkeO!^7qW~gUAyb4hRI$D-j{{``#5SX!saT*~0UQ(5G%1t4{;?9u=e`AhoR6 z+?K;WA&^7ic=7NMLw*CEk|)na06b`NXX~nMF6+nu-BeJQEU9Db`s@0gjwGf#I#p*^ zFp2BQhvLMd)u!o#i>%DrA+82*2+iC^0dh%~HCuij%h+fkQM0nrhuG(nUZO<6|<^nt-3RPWgD5cq|EAIYAsQ%^pB+BE(H-1 zPW~uaI^?qEaCzm?yFbQD)*rpYhE?v@qPVb92*=^7K<7Li5>a>htPw zQ^9I{lQ_>Quqbj0dFp~i8(UaXz^9~jvXcLN4p~hGOAHEK{Z8z*u^%(f92}Rq_WQ;# zVY7#WkTZ#_G_Ef_?^7fp8^DblohpFJO&pVSlF7AdVX#MtKv%h<&3=k3!G>9Fk3cCi z6@tb?=+b?Y;KVIlR~OeQ+#JrCidMp<>2C(8jju0+_OrA|AAZ)}h4$ z>W!VZi4Gf(Q=Onf!8ZrUKnjdE?ytucJ)wY9En4w=v&45%BdjL97x6Y{%R%%~MFM;Z z_x3AbQYx!lqy%P-3Th%U(vByv&!Nxju(b#{=a5T7n_xZTF{muX=|4DM+@C{9#vPH? zG5lG5WWt!5#Cb)OX+NVUX*xB^H_zPTJkGX+Biy>QBxB!=&L?eHU3rAtFI-dJ-Ec_m z^!Z$VB0f6Q%0<{}9_76i%b~>Fq$iwI#^{B`!|;j^c}A!U9~rvzPPZ52AQ=o8dA7yc zN!7}!Iq^8cLyZ1#zyo3%Iv3SdN8)sT6T1?s4OCgub+Kf@MO3S*D5ol(=#p>nR6jzV zg~}V0`nc|ZB6sMO%|dzMgz|Qve!pyr)yq4~XY=^shIV)8lruuVT#5?ir_nw;BjSv! z@>s>iW&u)wmFYUt<$vvIRb|K<6i~W(=gC|4D4Yu1q)+zMEH*o@TM;wf{e`MtGMm2l zR6It@6_9@+>O6n?jn}V!`sMrU6!#1v&SB||;=2Hk;LoXOd}26US*;aiQ578YR#ib~ zsdd;4p>fBN!JN$ef847^h=Nk*^oOmvZ1$5rk^I%<;_C1Kmz%~VXKIe>x4U5kEP6?-xIsw`HF zr15O&S*}?S&xwLx$OFG;9g7XOyuN}Yp9xNB*KiGLl#-WZF|yb>t4i@`!XCw5g(dN6 z!1A}+8%T3#p|{tvnT8K@?Gw3Ck~N(nB=%lz2VX(|%lnfvl0bz2A@L9HHfnfjUS{83K` zr`7}$Iew5%24Y_z7f(X43QJ6`lDy;ABefBY;spjWBbbHJH3HU!FAMUDpC2(kqIl}^ z%J>zkEGpR$t^JquCFTLmIHWD)1%#uvfm>L}^zb>P2z&~4%|{4LpG zF}{Wz_;MJ90(mv5T>8k}TxJnn(Ux&i=<{LJa(zWbbpu7XbzwY;5q4s=(I{43R-$JlIZMq`KZud+T>qU+ z4|;oL9-PVqwYN48^Gxq=ON~<~MvyBm2SGVOVeJ!@T{*xo zXqI>)fH3cwi{ysP{uSlsmB*f2>}65oOo**Azp&ouMMcvK=Sj;OnSY1#jQgFU_m{&0 z_B-#{z15t5*Xaqgv4%htxi;LzAE>DJv%LxU;cxa(GYOlO+Y?O zXU*QKDpR{o@$N{P0=In6fV(lhqBHV_tXQ;xQvm$_yLT3&mAs)qV%$h69o-I4jDwk3 z(D%A|BRsEJJ)lnXVJy;F-t8LuYTgoB( zir#KH%e&9(tY8VX@%mB+-Oz!>S#!t15RD}zd$8?^d?}a=sbHY^&Uj&xgetJ-Qm453 zpuChPT=DDxbFI!rYM`jTp?kjj)1gX?pzP2^D`P_jUwrHAP&IumUOVcec9ae8y>zHn zCnj?Tjp7H*vV~W=7KT5Z{H-$g7uesMaw-tPV)my;Hs)0x<2o?`f8ihiXL6iLFMaIp zgh+KJ3;DZQmZ(1f{+`;{MN&h;|78EAsUqYU)GiBg$XEBRja3F9EhPKE?_@-L%ri z4QjN0Ej?hl08T)$zw>jkV~bP^DZR!4WWB)+B|K(Z{z+YIjMyE7Ss@2TU_)K=?N7T z`jMDLI<-B`i%e`yRkNlJ7TOq#T(U)DT%C3;qJG;cwl^4>y$iP5%lL6O^1?taepEoX zY<%mUW_rk>3kTp7pPBUlIBg=u*=@zsb0DfhfOl6d`m(o7JC(b9(fS$7j)$Lp-U-yI zC(3o__h^^1by4yNYlWTGpt*e6@bq2oHTR%7T&4sW->Ij)_a}zgFk`!U+%6^~tG@@R zqz(cRGH>W;^J?={^w{;@a#F~uUmLc8^Hc_ds#6(wcQcHe0B_=NS)yK`CPv6a!Iq#; zC3Pb5%wKi0Fh^$IBLcb>gzipddiJus=g#td(aASj@&psk54bSjgquF!qWA9`(0iHmaNNH`=Z* zHgA4OXHY9FdWq{G`ND4T*g>mGn8CUlYma|7 zbsosthHb~#1U~&D>yb-M6xXaW^Kx5{%@{@t$mM7`>Re+EfpK#ca=)AXgNg?bk7l0Q zYOHdW80Bjd)R4FZ;GblyH{<4%yL`J|WkX$i-CKj6+0jmaES2%DdmVqjdyaVfk+N6) zMg0L0nm?J%pdQHfEIimY>9`cp_~$zBk79A&MQz*Kx1Nb;)ikfQd*@Zq$)BSZ@cyrR zCnw6t{J}I`N|ZB!?~dzVcv_RMg^9bu*6)0v&g``sm0aU@gUnD5ynoxCd9$fUE?*J3 z@dUJ^@p|6%D*}z9pBDDekgy5RqQDpk&cj>Kx{-}OmT6`%B!o9Rp_sPUQ%4$ zmrF!o$K8HV-roRS7xw6n`eM1}azoB~R(wPgCKcCGy9Ae(g+#{-*P6bnN zH^2G6Ds%YXVE;FZK)DKM;{*l(@QMHcK>Z(Cg#Uw#`xl`wqOPrgt%~u3h6EQJ;sk9= zWvlE(SX|JcRRM)YnU;uUA&n5Ve2A1y2yPX2~)T1T9%G=(S{tsM6t zcqF_nX73sbqA&9_B8iQYZkXO;2rTz-GuAOdj*)BPA=RH5PbNx~xLOr_0!T>&UH*Wd zR!VBGp(MQPUrT9KE-TPbTm{7TQKU6aUE!tiBzO1M3Q|>_s4;EVlRFG7e5O=few=Cz z8C`+fs&rzNc8rCS;^HwFV9Y>jq#j9qbjYc!Qp{CSy^tzm&0LaxB0iLfHsLWPeUfM) zLUfZ27gp*hqP}N3!9jx!Wqb2h zmf7^DHA;vW+=Oy*m4~K(QFL7?K##m2&4;drF|Hjh;LY&2oFScwN;hX2=EH} zX#w3a#76l$BPUhJvDCUFez&Aaa4)IdDRQ!GeMN>h;9@~eHfcP2Dn)k`d@L?$pw%pi zmueN&=T+3muoFbaYC+Kt&90C6g8DYFCBhd^-L+Lv8g@-%!gN80I@!SBJZ#D6E>v7Z!)JtEvoIp)Z#L$5++ER~L?Cj^2T$029okw> zj?#m-WZN|FpD%XuNY)emq}5$e;z(EMu+`+ia(r7Gs2pNAYTRAHTh}&tu$01m3Lw~G z63NU9##v*dBy11O8qv)#`KhCDKt-r4ATq#$4Y<4@GW4kZen9Q;;QK{?=sp?1_>s=X za3pgE2z>&F-hhJMVo&iIg7q)1`jx#=OhKN>Tne)GdBi0p6+x`YcAzs{=rKtDu}{>6 z-gvP$VbX0#wac3M8#mJz&jhcCw?U5}QFK9?A-^zw(A*n$|2=>MF5CgUO?RwZW@{Pr zpnUGLvPz)hu3lz`IZpHC6WCC$OOL<*K4Bzfs0#-W91_m^plZk9m`n&fa_ETXU`xj! zGTGoext4(c^2w*soN8J62a^evP9Al2Ce+di0qwOsw0hG0=AGhX+0(5j$DBEqP)R}u z$4ZD?q?{RLwH);|qX%g3d%WTA_`eb0I>vkc?Z5y4b`SsnEdMce$=Rz}8XDP{3fj2* zyKJbc?T#Xf@Z)9Q1d{}f9bJe($9PMcJ`iug;vY)dKWG-TfdIqj%9b;;e?85$l!g3F z&3DlR?-eU>kHgFNnZ<)ssuaR$-{;yEKbe`g}UiLCaZGXndKCyE}12BY1wFj|`o&2g(A7mPWXCmz*a z`I@O7YG}W7KPzh2Y~!^x+iX0(?n7|tG}i^#+k6y`E7wJ9@N}^WUCmmzsWbiwCc`@R zY#bYAE5t)a^*Yjk&#$g~Ca4nz#C}G7m%+#}hEVk{JLr(Y%7aMF(k8+H)t-YTsrn1{ zw(e0t(X$03X&bK8#U%Cl$`@;lhQrCBop5u>k$1B8rfL?9xqb>}2B%&ZF(zFvn`CG} zY5a@uiac7R%CB;>S5+D4h|oh!7K6lFB+)|55W_CKPq2*I2MM=c(g6!#?mezu*S7WM zj_S(Jeb;K}UFGEHHJk!jjD4zyw)$qjZm9G(kUCEX{#x~CPClU>cO|c}FE>Mpkd_BF zDl{>N?E!7H)*FB_%$a@?4sf#R?jUb8cDsE+Bq=m24Yr5Dfpj%SxwgiLaOI$g#<+a& zJJHa+VB!?EyhuRWKr2|A`952#u-)OtfPC8weAJL&*j8FRvY8wON}REhjlT)OfW zELTYtO1NRIyplL#136Rg6V{so7aiKeT8aezc^cXIV`J-#gE5&FE zF#5&M+=4ujpgk61qLY=>c@|*NglE{Uxx^-14Xy2zG)ua zcj)lkCm^$zqa1icCzv8a=qn>4u-wlga4#No=|o)r5+aATSM0LB#Ot9w-k|yG11Jjd zSDZ=NqxP(6W#kupx>DXEGX{OU=^Lc6)T;IK^}#T7I;0ADk~fFuPEZQA#BadB%P~Y~ zu3>v`Zf5n}L2_ZjGK-k2<6XR4+B1}?cid!ev1N>#uy}K#OIQknYrciOtCR8nz8a-X2L<0KZ8js+n}04Ox^2+oNt#v){%jAYpQ9W735No&Zdt)I-@``IoIsHJtY zxuSJtw6(RBt;Eti+c7iG!)D7nxf7*hL->zFO@ks>mtgodKn?qYIjdFP2d|{Y4WwpG&<8SF0e3h1PNX06PHM_$PxGh@% z$Ec%?)?~N1)b5EFY09)mP)JjR+y&eLJC!WrSGgGwx6##Bj!FRUc9AYtz@T>`a+iz% z!E5z|8NAIU(Q%O0E84+zjXVVhqNwx+4N>(64R=4O8iC(ctSF4rQ*(J)bLPoGo5oqq zqglyVmZmjgioA*>c@HkfsI(juL@^LiZ7s+rRKGxYsq{usA-&n;^+q5I+#nQD`9-s9 z3Jy`bD-T%-e=;=m#Nz_lRlacSnI>R$OJa10xQa0~QH42(nY^NC9jxlrNBrx5LL%E7 zRY$l7_i%Pe0abi}9UqkiNDAKNLLtsxsYHT`(4RqQvDxcxT2)sz#q^Ix59))zw;Wb) zEfUZ7N4+v!xQuH_)0`ev?jFKELOtyZV$8#`naxJZ3Ft$Vs@vPLAGvbW(2NIh$rza{ zwG@dzg1j}G!@o&xZuZvyfc6mIJO6=?b^0}xn(d#$%?~DCL@K`8_F5QVHHhki z!oM9+Z?AG*rszPgcb9vsbF6#3kkEn-;JRVZwM&1!SeY_+_JGX2x8K(d#(4)xVOMmk z9c=?SaX%O*njEr?y4P<@Lg&0LlHb4LX(<9?R5=2?7UMqw1~HnSty*}{9CHS_)g}`z%K(E15-DnV_oQzd8aCM7vZ@VxUmXykgcgvb^}*fWVZgIF5^IoqN+ggKA#(^J2Q)}ODY5*MsHSJ-zK`NbtjHayhb@41H%RRdEo2Az15S>TO=k5= z#z>;4I304Oh$OBvE^fW=g{~PePLA>9f(p3U2}$5qGIq&(Cs2_l-pS*AatvP1i&k3c zgY0@Xc!0g$5=`%5<>vjm`cxnG>z$@}H%3$`S{$;tb)lUUmz^yPn_v^apHXBalAco( z%F`GwU{U~J!_EnQtoe-M_K@VuUb^AKd^|=YcLd23UWPEX#KSwZmphI)(HS)Y>mwZ`;xA8oq$k})hF_=cKe|0dr0#7*f5%dKT?lD(cEBR%LYQM{8x zy080X?!(iw7o=BRbKLs)I)?t%ISH zp^N?h;;C|!rlf`hP@=V~8WZnl-riJfl<=ryltn(wWgxyoh7@Pgb%sAfVk0<%0n!UBK zd_ly*1`)7^KS04N$Knj&my;XBG^mGxmm#t5&<%`w4D?fAD@kq8)s zCFB3|hlK_9|MY>Alc6V*h=ifDg|ev&$v=;zi|POAgXB4RWJQFrKKrhlP1Ar)qR2kc z_!~6MiZ^5t7%Q7V4~hkUbo?7>%^kZ5x2^sGevvr6ZjX5pmIaX8A4aT~ebJbwR3(ZN z0h>kkPqWh*AG6aNzxPi|TmdxpBL)XSadNQ}Wi>kV6Q%m8y10+Mq9lyrj0A=_{r!FND%{$DoD>8BzgV@Ev#KzQPq! zdZBPo>CnM*L6LI?$2{_HOLTQ=pqTV`S(XOk7*thltVfrRYO4>v;@SDR^0CdKD>9sr znpLgCRZT2#q2U>&x9+FFdLC|h^!BP}A)|QAJ!=j{W?b18HqL>D2vC`puRF`jKw~X! z96}6S)o0YKeFV2$O!}3mej<3x4jAGv&xY#)P`qGp1#1HANMK-&u?Np?*Bes3Y8stK zk&X9(+|-~AI2@>P2g_^`!GF;IR@u}k-^tMy!gdy8iI*xm+;4NS|B7yjn2)=6J_w?6K5DVYLa3Se>F2$ z8Lo&?s2fAWOKJsMLe~&{#YZ7xmo}p)`g@6Idkdp{g(JWF5&t5S3uO@n?UN7^`Xn+y zeQ4X5XSI@CunB@ggpr9?4QG(U5@$5;A`g3utmz9X>kG$pLoSet%Xro68|oP^btSa* zi4r%#F7QKyMrdZZ^PWI*V6{wl)u`!cN z27ZT*{^VyxUMjA`MGmk<4&b67*58UvpkXp&*d>4QWJ4Gz>Ai%QL?B1?3TmSjB1C9S z`w5H{IZ)kYavTUGW3Y%43pB{LXgYPMd_v;321zj4Ml+Aps$ZK@cJV zNf3Y#07e^*z4dgN__viG}6YUx(j&A+JnHzQkbR%TG zT)XHblYuitQ-j42HF*Ukc5Q5EL7|(RYFu@12RC-rHPooVK}NyN*VxzDcBr&-v&!n~ zGD{mI!b|YBGxt`5XBk%?{qh<*i!ec4AwaIv!geQ;bY>JPcVt%+w#o|KUQ0`Rt=kGV zod)gL=H}uO_F@YR+pF*fb$=bRW}UH%XmbrW&LxDplc-@ea+9{fgJl^@zp;mY88hOp zl6iFo{n#S1**)ygccDK2l76JPv%AC74_3oc`$MY^dW_KZXEX+OONd(tHK*YAZ=|Yr zYl!v{LA~8Jgi>}d_U9;f;em#@yg9bZ>Lz+jXxEXtXhRNxeK9vuTrd~9q?>EOt^ze1 zN-}#&3RaAGe)mX%hJ~~_M$`zHV5^m$67jf>Qm0&eNo^UdPa`(b&cSGJE$x8?Gcxp~ z?YSkaYb~@W@mF!mipsK zzOi84o0GNcaQe^}eTTO%Z{6%_f|Sv*U1XS6oi^rW%7_1v)gW2<-(Od`mb4yT9DURp zTj1j*DKRZ|OR^wj-e}KI$fjmtZ%Dfmr*kQXy1UDpVHpo|60|s%wjgnk-`4)oWY-}W z=FL{%)NO=YyEym$!u4rH;(B@RIs_Zj0t*)^yPN~NZ5c!zoOGRH<`*+0$b`W~KlH)m#s848%bs=mzJMKaRjJO`e;B+}^@F3Y|sVKE8*| zbR|*2-DVdfngkM9fw=OW@*dGCR)BEUo8d*)8mlNBfNDExzi1Oo|0rJt3@uEi3 zon9H_B%{{e*7EK=C>ODg`}1Ma!VMgG##3z5;?V{=3WZss`aFx82@s3%l@v!JgQP7U zaVntHEiA;FM8jt^st47@B*3z#uD2c(P|Im9pu_tcX@3XXB37)c#ONw14>puU zjiN%kyxasRluJ*8`e+$}B*Mwz&x?#U74ipo6-IkVOxw%rNVVvrJPdR2za~Mayl( z{gbz5HMF^h2r)u=cb8D&q<&}VC1G(dX~n~7eYIg7l}piHM7g#Gs~N6VC0NkJZ~E*1 zN@&a9yM!5W4QFG4#LOILExK*jEI3f$=_QY68%tkGGn31ETQ9beBt&XgTy5-MF=OYB zB2%5Kwz*wyG1%b7h=hB=nPUe>NE$ZcAnGpoECa%-4<$R@Cn({b>_Ntp1Xd~C5A`YA<@aU>l#Im zyDivlRzSu{AJ50oB~U-=xX{NvisF4eM2T^r^UBG$pHpekTIeX9Z;GTekYL?Jbe5;Y zF^#8zTf{{bf_UeAi@v63D&cmX8*ak;LexcpS;Ub+=@?aGfX_KD@OdHJ9wG5{&XGci zCPdwq@j@hb*#{~{8wJxT(u@X`+7QLjIWAao0e4;BUR}EkLX#ctk;&)>fzLU}urdyi zk}di&46r!R(`}YlVEH9a2qyT6gJBD%6xkQZrC9=d`x+}d+Zyj6;XDvY+OU8ztY2Vm zGdM3klgQH^1e<3NU&_1c`#Hic<95( z8US{33aTEH;M3+~Mb`=Fn-+$6)XIA51)p(g+3>^_`M}A0^eXYu6GMYj9(jW1B5pFY zI=CdgihcJ+-sL*t=-=}IS|dO@+Kw(cO+#};ob;$+`ytP%a=YT zi?P!7=78+Oy1~7H!1APCwTXp~NsmBNu-L$p3DTKmquAD*;$Kc0IRLVDoq52lAXB$) zeOUW?p}rx5rby!xRkS=)1`AejTfM9`E`cRqw7Qx`sV7S(J)uHUf`!MebG%xqJV;uq-kbd@H9H2iMl@7kp{TU@8Mgo@&6vC- zWOYE*Q)*Se)WS(0I&fF~Zc%v7>mavtVsk-8YNz?d)-tm%MdHI4!XK|y_!1w~tleBB zMzN0qV4$1D5k|>@Y4jIa*J^BrP1usja=O|vEb?7@ZsCpD+MPSc<9ZzcyF6^ia>=sZvK{D?20-I3~F7PHD*q1-j6i#`|Lu&K%{w_2Y0S zgZ4#CRB89t4sw0LdvJbG2GkU50d|}u?g7?$%Svoa+i*KKtruP`{#SGv>Y8E5f+c^g zz{7}e{i@{VyiA4D#ajs(nuvY|G3d?!d~zR)HBBze4nj0x0P|I_S`h(1cZI{_`S4MWt|H=X?|z{^{$e_);Aw}p|S(a;aocAfQ|iMKq`si6e{@+ z_vpDLm*7$Z1a_Cm6Bg9I%xZAAc@G`@Fs@vGO`eE&Ck*>~YF{!3Jth@<{{Ogt<5Jo3P5tk~e$HehxnEjx=**%at*5_jW+pXtwV zw?TLDdc&?eM~hK566|r8yR8^xW!ELA1QJjaU?E8L}C{73E1kXfzShqt}bd*)UXYz*M^qqD!BWW0vS;ME-gU zc-j*+0+SO~N&OXaW*S&Oo^n^7bh!#=+Nre90T%9-H(#r6a%S!VCMHp}`I9@i${Se2 zP3ojIksK0qcFzv%8&>5Yv;t8tM9(RT9Gdqrq?qY%qRG`gcAUk$W~{D2_Lp|IFh zxfetD9{`~W2|Eqv(IS1U(9F+7;oGEuszALX%D+H7BK2Yy@Q>*~CzziSTAf&P%mybx zjb){nM2&434}=I`lCPVa4nMSZ!66Tn2O7HxJ+fVMoEGO!9d=|ZU;c^1`eZJ& z0uMQN8xqVIBg9<@DLyglr(!PVEE4{}AnRlG!pP#HiH?}~J!4>thcGt6^{5xwN;IUY zA!f#B)jBbqP|KiH_BukE)?iCx&RAPCv*Z>|D->*PhU_7=6PC8&q=J53q8xCfME5kp z(j0bagyioK@qZM-j6r1{CpTL_QIsq>WXcTabu3Mnh897sHs`BBUMcrQ@82_53@cAr zbdFMk!j70=Yfs0DX7k7@NU-(FcP)^TY_`1$Yq5GwSONv|LIfiX#QOeuOlB1iQt%* z0DyI&9kuo7ldUeKYsmZS6>3tWIFZ%1m7<-DcosSC|~?%jkV1$P4%bv;44c-`9yvJuU@Phz-RHn2|)AEcGY#GS+ z(sJL9NoEbMNf!P|(W{9$>^QTSMu=q&Ox4d^mOhk%0{O%HgTtnp>VTrnn^X_-L3(@b z$~*&&!V`1C#e+Oa%5!e*gQgO2ZO1p2WO0awgXz`hp^-(#)_={WRM4JD7G+IAP#aQb z!}Mjyv|PmpjVjbp%@n1=Q58Lvm=QW@R`|ytB?Dl-$zmrFQ})4v-J@ALFVnI-K^<>t z<(%vZG}sq{kCO7uQ6`r`XMgMwLIj-IvNuipvOB+M|&0i_0^zA41nP@HPnoLY;%Vp+Jl_Ou1cDnZ=IDtEu2FK!`dqYro0m?kG&7#QnA>@f@@k4=e z3fvx0T%*b?$#-uCeEK*dYhN*oSoFj43z>$5?o~1jA(;XvVW#3qRm{?1o7CkP5J{Vz z0xOy|-2*QuHj(7&)z=*8CNt5`MJ<{kROm6+n=Yz!JK~jnQjo7@EeX*RhB$*41vh@C z(5d&XADMR;gkDf@QGplST|{0<9dLd)pF@w&6x<2g5(PPgvVd}9kZ$s4%yDB+eQ;&# zZ238-1+#`9nDoZq0w-)nhIwID*MK{mkAc$++RBDHAx_8p5;jI4GYN8wDh@qO5|hSB z$`tGJ3OWu!wTShKt%s$afwn^zo&L2YT>6z1u~ zD%Iy_K}m;3RJh3-B)}m z=U0=MA7)VeG5gUj@h!3`?^H$cR6vgWTWqH}odCA@6Jiis?|*~lkG7dGO%Z5;hO}%S z+@vJ!n23@rMI`N{tUL8VUz?sYXL}US?C8Bh0?jWT54^kc8pC$hhvfFy__(TguPwcO zz`eWW_GaqGwwq9JddKX0m+|(7p0LHPvxzD9b39cXB2P(sUwFhr#z#&x#W7~qQlZjMP96b&?)VYDO#e==RusDrnw!yB*HAobI5+-<8W_!;G>fgUeOEOnCaE10SA;!?e{F)W~5^H11wgoG~HwgvgxmzR?R-=lUu4&--pg$_G}=qd|+l<8{kFLf8STQe{kQ-%9PK zJo;WU)HEY!40Q~sz+>0jGk&^9KDx$qPLwcWi2!2bH0gDgA__7>np&4KLd`yUKcFk< zUw2J)LJP&|aQV@P!{Ho{BPMtgpKYe}BHR{<<`aGNj7p$KP~$WnK}Q(bL-;ra9qbeM zjA&rkJyiD72_Ryj*>4q6eoyZwxk;u^R6}L<$np-SBMV0~wJCdJZVsUq5qbk&Xe66V zi2jSD*`}wde1&-f)xgmE16YJ&O2=4Em~tL#N_twd*?Xm*Q$0`RX6`Ia@k@PK^Jpq_8%&9f z@=fC`+a!&|5j)OoUc+U&M3ih2ZB!%_FHK8&sT57scOY#c%{GTDckrKgpWbDf$7+@!ySYGi9FUgYF&BWR##AnB8UFAhr( zB=4ip@97aGx2WC${1^hP^jhR~m69otWfCmHsfb~CGNzBI7!}1>p{Vy5q z9}FcCwR3Uu{NHVbwC#}uF?fNKr#ew1*DGtZXeWi}t!43>k??G6Y$87dgb%ZTWzLsP z3A_pa>RTFF7Dn**S|ghyV8}^=MI7ie>HKKj<3nT#c#o>>wXGfi8Oz!!(@bBQ{95s5^OFI5;_at zf*7t9(`lwieXxcmvU2l9C1=4z^!SCHHn~;|f99_t-6Gm^3PGQ@bu1m%E`MXlyp1gu z)z~NBB}zOAP#)ZJZa;wxY`yeuwRAkXYO=q$g(> zMk!i=-)>z`({eSG+Dye`NEj=HM1CT3N;lWYRD0lxmlJO%!`m;2pk0TtVGl8f`OB(n zFfMCdyk=-gp6J7XP&rg@IkU-dHLZcXQ~$Mxbr11j2+T-x{uy{+wr*N6se6fjUc?@% zoeUl9k$jZgl;T`J`GIMa62m;)hlny}8!1@`%&k$}j-}WT>@A=nTIEubHeAm$pw(eu zO0S_MKyt!oXu>T@d#p?WL-8YWc*J>1wTFmJ$SFjXS1Mr3VLB{v;WmcSL0u5t=#Es zk2_hCi3HPrJP2Z;jF~@$P$VIUFi1qO|7m~(s|;D1i3t<4oEe1yt(N7L@z&ywl6qHj zs@BrZA{s47a-_@DYOl7f_03eTTiZ_WULokxulwyxP6mlF@y?fD$LqH9?6=o|XCKc{`aJgq^z$g552U+Krnuh11$EZW0a^W` zx~JnNJF3#N$P4Ue*d z=j>eHYGr#D&)zvd`3v~(HOyS!6S99w`_7*e<;-2ZG|SXoKC0x-bX;=vT$j=f9)#}8 zC6M7{DFU;t4=+>61V_R!e|L(4)%F@=@>S}{TunCgZM7%k0 z;mE%R8)FUudiMmHX=$swx7F$|nf47-b><)bjiFt8uC=wi+4KMo)&a&QiYPGQK$Rm` z3dGt(t_bjwh%UKwr>l`SJ-j%NSL8>*ja_bghAmS*knozZ&2XT}+}*M2YZHpEV&+1C zFJq<@R#*aswA5N|@e^ehvGmOtha-lqGq<(vW9??v$&(Z}ez{qAgDd*4k&L6c*rj@& zv@_PmqqP?L(;Db_c-shatCIrkg68V*%Z)~eVfYsV$&5Vd;V-~J5@ab%hZnMx5QxW7 zqeT}M9IaZJYglrj#0~=So5RpEfj9x3c!E;ah4@`l1}O^(n3v2-x`ySbD!aMi!N$A? zD#>{@lBB#CA!P*@*l@z;r_!4_IB9Mt&m-%((4ABchUHCdRdm<7&^D>{hd?cDEe~6! zXcU?pQd~<1<8U4n#(_)akY)L)f5zpM)lErEgO@@{;F@YkQ=o(ot_8Gp1q&+~iTUF3 zh5?!#jCtH*F)Wfr2wwJ^^tejU#^R3J zm0lN%D!3seI%<8?LmM4v0kFR>0z`G&P$eisA{ktx)~z9TtE1F_VsHjnN@E{+O2{9v zJlB~Yt?*5aWrXy1Tm}wNoPYef8?=MuFGHe6*jFMLVGCtX_ZOWgSIaY>OEH;Vi{xxf zkOm=|REz*F!6<{5b0x1wh@wtv88O}y#2?X~5%9qog*WN^bQApfkx_<76jqzA1O`Rd z%!itFWins}a^=Kk9O1M-)JamMQ@Mb2Iv4fi)zWD7E?LR2sNUNrzTC!> zg#|WsMDC>GNRgAE9zCHFzK?&{<T z$RxbX_S>a%;jrB`UkdMHPLLFhZcz*^a0+ycdZ&fokwEOVc~C&Fc~M~3_j+s0ShX5F zL9|o6E`VJ=D4^K6EVzQK(LMRcY9;f+aSDUNjyl2O$pkwdB9yof;Y^zft? zZ16GRn0TFG;=i+IyA+eWg@JI8TJbR=?miqzGGt1YMSqMFpIB$-GIbG&CxG`eTGUrn z+17;f^5g2;rLu+KZ;IJ12hR|h87a*oqId@5pnGC=V>2B-aV4@qCC(6=L{b0JWKR`Y z!Sl-HC8EbO=0=Qc6CCM$y&BEUc$8{KYwds4Pah zxDxI}^Z|?cxykg0ObhqSOus{HIaGq@1yp9}grT$~vUXs5v>>*)2zh3Z*h`Ww$zS}k zNelCjsmmt~QO#77D^ak&I={BMu+Rlg(m$d>L+Zz5wZhKKdv~_#34LBOybipO(%F4f zPSkxwld03%Wx2`Wjke<_Pz7patfsv_43)8>NYWzbCTm-%nuMt`OZ>JV8|VkyDTLMGo z8ITQEr7w!`@*x;mQZNq=Q;UA0_=A<1_=OUMUT(*We0mcspSEw@83Iy6hW>5|TQl;e zz;}|%1!f% zHcp%9Bn*1&CU7f8>0a8glsHAaG_>2@V*7WKY-@|Jg4yW82hbh#)pX!U1;qu2SbYA9 zQ%p*MQ*1V0onOrA9CHzmyoMKpOU$eO6r>||U>K{KhfH|*l(y7Jr)~^+$Qd?;&?W`q z*CyB#4u}UHP}kpJRro~-BvBV4Gtw44Rg><%9RCh|0ugwB+|fGYW|e7*7WfF0WHeu7 z!J~wxndZZ|s@S46)LGZKM6|y8W%a zRVW79N|}ic^$kB40X+{FeK)UlX3`)y&@z)0?Fbk4X{P|ze}ys@1U**Ck~vGQsaWxW z+G(*aT1m5L>CPwckAzn|5KkuIIZFzA<#oJ1uN10B)1#o%!@9eP$#vEMLqNR0VYqA< z!J!z_B{xDD#x_o1hGV6m50>GstAhoy&UtXb^&=O?N;qzp1$zZY-Ai+r9XUC+Ran6m z6mZ@eyRiyL#hR93x@6TkoELQSs}>+j$xR50@+jL?aUUjDRQ0#J5q{E`{H$}awac1k zB??y03r+>r4z3p{_6){tkd_O&8VoCh*@VG9o9IhZ=E7qvns_VZ@gN*kfwUH^2?GhS!huKcTX zv|YS}n!yQM{z?o*mPJjW8g@+(n-(~2vxiHZNEH;4Z1bt{AG8{?afXv!>nT=8ZVTm) zd{Il~&7OF(NeuCM@k%YP+*8v9?G({FzHX1-H%Ng{%EU8Ke#s_4iIt_lnkprM-<)Ck zZ95CJ?v0Q-ZP8Oq$9d;Dsj=9{$F-bt|AVn_h!TcbvMk$neP!FWZQHhO z+qP}nw(Y8~Y*)?ydV2btev9r|B^J5Kh>V*N_uO#W%$UJO%e8c6Cg&|VMQEXYqL50T znTzKXeD)o~g`Cu~D|u+o1-c6LCt!Nc@&$RiXT&4n`Q;FPQU{3Gp!0pS-e z=nTyAKx)Ajolj81GkPhx*di<)x>wva;`SnVSaNat|Th%fYVxJJ;|3SG^m&r zT#^*7YiVKVm!o{#fu+abZ>+ts$0augW&fQEbS44tH!i(##5BrJi0nMEG@^Sed93T7 z0s7=&wr)bEsnXaL6r4&vL2kagG@-$a1Ts#Up+|21tF+QOdz$ivI9Ha?%LmKbJ+ih` zNq>CF+(34yA&Gn^Pb7xRsOiB{896CM;BL{_1ocBTDTTJ%tTC}~TShfCK;ftwrxB<) z-%Mg6HWio|y-T`dT2d)ysRmjfyJc8#hXpg*+ClZ0?o@9cN`FSvwTH4cGWIgW+z!6^cMxCl2E=I0kL znfj0UxEHw9@WY{Xj7{i{upfZJkMPNp*>OtWpR#Tcq?{5`Ps{>OP`zS5kyl?hvM=uQ z2g>kIr0bxm*C0%s1hz?OR=GuOvEMeqMeU&IOX3-o<7!_CC-Ba9o5WKnUI4pP=CkRR zy+~6ISA3BKgn9y@qTo7tuColR7r`KkSJ@5Jn>m3uN5_saehZa{SK-vI6y)}*Dzgcr z##cz}sv$I0@KMA}b;B29DQSuFZr#yTPodo!q%p9SY7UKT)ziacrF(~Cb9oZMw8yS! zoMUZ+Ns0;3s|)#eB)kd3rx)*jV_T%RQv^yK7<8);S%QPYk;3#;NO&4_l4Tl6i!Ab? zvbr?aNKEH4^0F=w(48>17w9}#O*TzRqjUm@YeikMiz)oJ7GCAv8IFNi>|q)E?(_} zSvw2IGqxTJ z2&f0Yk~qTXnP)4@m7PETCcYvYo%^Bw&r%lUe-K~!kKDn(#8=Er{-X!ae~iIH*ogDnS310$0nOOE+-!#9RaV&9aPiwyGq+X7j`6<%Lagqc`g+<*6(`(Q6yhOZsEF0p3rR1=OmCfRI}Kaxt5hdz6kFAQtf3{(`FZL?|Js-^q4^XOU-`r|!$Bn>>pD|OV8KHULj-Opv zz2c!Bl%I5>4N<{CV{AlTZi0$RWYtgUB9*6=D~)f2nJ`9%G@hVH?q+SZPaG^JYGYl` zmbWxmtnnVBP-!DOT~{0uc=_95cOPJAZ2V{nuHoB9DPhjFD$uHm$y%6OYMN#X5{2GO z#KAM;({;@*&mOjCMw&jho>cY#=OTN)a#ncsC_6`Jn(gT{JQz1NbD^Xxe7i9yYoUA> zCSQWqnRlq?Hd}D8TBMNj6pT%dI0T}TmIvIp6v*sSvt@kd1fI-BCn>om2E%U}D~su= z=x1IDMnqu4JZbCI-(t2xBb(I5*vy=M|BZ}8r?Oq`-Ay30fQx7s6U-DRusO6+H!8%o z0$B}Sl!292e+P6cWE>%e!=2q+QbY>X{zB%qF~sd8CqeFV$YOTRh{!v{q?{UVYF`9vm zlp6``q3;VIHG)4C} zMaxhuS<)6#TTCPI=udFLHPSKb7I9g;I|pn`bgE=r7Voi6d+tk*t^#9J-f+Zo;74(& zV2@SZH*^k|IARZ!dh@n8??~#iZJ{MBDP4u8P}g01tjK!|bYpVCB79~gycO`0s@d(g zLOzyo292AiTJW!$CsOmgT!@P4meNXlZ^b*Ug}6-<*NQ-&zXN>QDMS1wcR-2hy+#f% zL&6{RL6Iq_4j#B<3wLBOTn{vD?1Wge_x0C>pcZnR)jp!|IjfnlXpV{!yGSL_X=P6vLON24_~|F=nZJ-WC+fMm`9Et69p2_ zuX>XO>KW_W?qNvr#eW7WLp;kD&^P}6qzBCv0|$=*GFxaSYB@kj%@%x&=#V4ntexM= zP|@dN;*#DIHFp6=6hjR{S#xIK%)%strHP(diB2|dVbE_$sBTSE(u*hISh^LnpdfVG zE@zMgt7WjKv5$Sgke{_X${P5S5C%ygQ>Fy7hN){|m(y}Y~(iVD{4u?^8A;%JhCCsIvJ$C0s4L2W8Mh2P6OH{C=iOTI`idF-a zXNM+nhzIrJN_qz;c|J>*F#k*qQ$Sy3N%4{1gGcHIm6Fb){{v*+p^s<}9kV|7F^$BI z4s;9K@8tt3aaS40YXGBs#cf|g2wC^+A-LHvfUdX}S{YwnNcBwFblh9=ah?#)&hTm$ zmt!_?c4lD>%K{=@Z%P_*9Y3bCJT0KAl}ngD$fh9gO-Gibi?#wg9Ty8{D*snTDWm3t z{`9gYgs?N=l91peCM4O6@B!Z$Lh=U3OJ2sFJLIcgYlTn-M@UHUM0urftRP2BxH+e&1TFPxhBqm! zT??x8atilWt?k(SOpq6z5lC$O%)s*#0{PQ?p4^;Hf8j(-FqV}n7IK+sMyDAyc2+$; z8eHM5CG~~E-6yZJqH^wnqry+m*;Dp$i$7LSDi?-wfao0~hG&-b5rpZN7%&ei zkaRAEa)Qb`(Ixv{ytUrz>Iw3DmCTqyd@6BlzSO-t6fC8C%?eP8Sg#J3ld=f#(m36T zF=1+=i#5u1L`Qbo%1}3hTj+pKs&TF%jC<}KeVtj4xF)SWPEUqKthyf&dhemgrr}va z%s9Geo)adCcW03I{JmKw2amgkIIf8P33*;9vPy|AE(~s46o04*-J)G}6_*do$E&Be zr@UEBD$wjKCO)U!t}Fz%@dMGm%euGQE%c-3N3{~GR)adrP1lR(=8X9bdhvrq>OhUa ze-=R@(}sBQ8ZePt^Q5pfc~7wi$NbT=Ty5WO>4j_9lg+q`=Fgm*{N8c;FKE+r9C07h zE)0X8lfWMPOk9jcE-ZfVG;Nk#N5IP~SaRFjfCUWsTw#$YYi;EZktS!Cc^H&eGS@Vx<9bHX)eh$y&ib@Y>VFQOx;aYKH zM0`Jf#ya0ZOW*uVU!&b`%-Q!I#hjn~BtG&F^2a^J=i?anDIc_tLbYOylu|`SqZavm<#6$u=Bv%~rlp|) z_Gk7>Uq1Y=X@k!|sRu7PHP5oD+UFIawn5k?r5u`0xYHLU+LwO%U5E8U6+tRz2x^mf zH%H%}U_40`>iK2kd7!q5)_;P>F*ifhabP1-!~DV$if~;-nPnO$Dw#*C=S%EDuJrfm zZ&wgWVh*n(x#{4s!)Ex3A>?YLsWpO4!D=6eej|suXx6tWv6$@q$9F4A!tWoS!oJs} z_8j3$H&uW~D#iO`>- z@KbPxn1_YHHG~nUg~(OIlppnA6b=lYAaUdkXy1mown!EIWyG&%K0jN)x2>PQ0ILr| zwD`@NV9O)0kKBcYER)N!peEgtpg{D^@JF|Q9#Sc1HKUTJpeXxBe*dkwg&iujJO&5= z&<6a!kx>2tv13$6NRPxb}(<@N&03MtRf3 zt?zMO6d*>eMIEP+yL{gqYXf*&Biq7(V86*V26XdTT?ex2$*F=rwGc5GcRUfVy6UAt z7&gYZ#$AV|<5US9;7T_I-XJ|gA4f)a*exA{SK(e9MSN2#3ajjEZtVCHD%;%UEkl%G zlwtm;!S4*pDTQnuC!~V`b=1g?S}t>3U#y$$z}iaN*l$_jM#2Wnn`cwV!XA>jHK?ZtX)0ZF%GQPah^PtFzUI|PT>k!6zf zZ%zgu(YrI)q<&Pd6Dl`1M5{2?ymek-zIv1?!XCNC0IFgh;e4Q!Yn>w6bX0w)6$&PT zsy}e$u_oPSwt-0k!a_zY$#$li3droL!;bFF(WqYaEX*vSRhs%eNp)88!tKH=sE^D6 zZR1OLiy!1=>owIduC;P?ntk=JDT<7O)2EWV-Y)=8*Yn?PCKf97@UCC}Wpu|J5KIT@9@N(->6toRC#eeptI?Hl4K+fhuGyDNe+e ziiyL^)hV05;7a-O<;lsfSvOd<7I!)~=>G=M#WKa(#rC3PBL-vYy7MOtj-Q`L&Az4W z|Mt`8Hg#pRt8HgV53C}0@m%EZd##!Q+L_=^DMWZXlO&B?`? znvs!_HZ~a*rI}ec-WLN95anOD7o<^(w3Vh9YM=t%clh(0X15y4LagVoOph@4&}Lp> zgD;tCv^ATs^i-!fUPRkgoWhp4tMe<#PM|6SdeySF>dBvHkQHWEP^M}k?U&JzxE2SR zE+8;Zt_}}+i0Bck|JIjUts)*qm@xt`HD9zRls(gE$f%7?O+8k62xQPLN{X$HQ;b-Q zEW6*(N~#JFiRU%^`7;X<6Jedzn7sYWU@{VNkFMggKna0x1f0y#_o1LcfpqiY?kdqy z@Q<0Z@-XxjpH>e6tC8~uJzINGtc-R;Su&D5eA!;UFvXpyBlG^<#No6p} z$dt_+Xno_Cp~+SOvc;Q4-I)?GDXBuuW}GyYK!9Lc36 z3Tk5XtQQ)!Rc#KN(PjOvrfI%Egfy?CjiGoE(Oq?ZEt2U~n>={qZt66u zO_SX2b3b!Y-G|Rrf$yS#Z%u$L0uwC9gs)r1F~aZ5L>3_$@Xs63b=43u)HKF()4qoL zdEF4?su(*#YR)Q}zC2P-at|U%_=3?Z{Av`=w`nZV=(`DVHTGvi*Np0A18DuSsia}- zn3u5NZ~MI7yas-ZotRPE0$xvuJ~K6MUs$VIylG(rnYrg1^49{eAHj4{4_3(vh1WAd z4vGyZB69++U^uOCGSpreQL)*uDvATfHcEZllxlO{7ioAw>NT@R&<|;BvBc)AV;2&i zQ1RF+aON<0iiB>a*M0+-&GC}?5N4KyD_m#8ZQCoPP4gQmT<5u7M~5`YuQ2^PcyU&A z?icPN#t{6X6E^O&XstUo-~}&ngXD$)%UNWzVYWG$g>NnCH3i)@MYl+RVpB2KJzUMZ zH3ws{>H(-H5IeKr49cyi!rVL@Tq)Hu2M+W#d3}0EBw6BXe^o1yK_&!ULh#gi@z4sl zS|Pdxuko42e%x|N$KOR~Cf`NbTHNX*TbR`jwKdE6t{#A#dPB z8V>8c)%s^o>z>PvLb$E_nF^!3V4n)kMA^9Vtj;+sxgE%C=ZMjr`oa_}9+4>Qu6tjuyTWETsN><_*xeblZKzWP3I&=(8%Pc#E3DJ-PX#0Zx0s&EzPqX&e6 zbsy*Z&(@W{=)@=QBl;~}ViCA+nd*H7xt}yH4=RW45khaZrSGsFo}t=2yqgZAR=zZM zJ^c7Byf?2;MQ$Ov4xc`AOwDuL1$j6W>O$p2-KWkl4{P-py7{mksaA5oY$zlB~BK((9L5(J~ck)WgZ=PLlhBOjVx;O;HIFSJo zk^m$Ipa=w#L?BRLf+|KLAQ)K&d&VCHEh{Ulsuo@?bWJ^T8GP%g8W8f;HP`EZcHO#K ztah$jFI!ixo%B*c8-Jbp?0Pe}$?w}fm2TYf&ayxI&RlMfxyt%paQs6Y{UEkV@&N=O zL)1|$5#huYAb2TOMJ4!2&)byGl0l6P$I9bJ_Iy99n^jcO=FCNO#STvW>3WZLsYHH>GrWp4ma`OdK zYUD?wTRH^HcFJ2tHJi^HHlU$X?vQkT5z=6_w5xxnS~f^Woou>Eh|{K^YHF8`NM2`E zfT)5NQv{nRA|$VD)}lpx&6Om*cH2?UyR()f;XHEL zh17%lJhPMG?kPvzL6F;_ENg-zZUf^jkFUv<0LR*g$Q$Yi}~L#mP}bsI&Btohp6lc@^{rN{eC-A~T-t1y9kl%hovsRoa%m~&$Z zdk(^TMJuzVB#d|lg)UWg4ozLFIXBwPif3RDS*<^c)z}bst!>mZx}Lcg!1gqF8SZ3< z+in}2lHrG_C-d*ATazW%$+h^Xk;SBv%?rZiRdM!;DYPng#is>pOyxbl0luD_H(fwF zPwxA2$aO>x*AWW;+g#STxywU1vX!YFb&_NisY1UnX~JJLYQCXrhg>^;*Le+!^<#IH z=hXY$r}hP0Hi?_8$|danGDz^(HbRN*wOr6!H?k;V_r!{6T43geM~C7#EKUmeA@1Zk zF0NbuE0kbPMID|AceU;;aJ;{g&TJ=G4L!C*-C?aAJ#?5-uWWYv>a`QV*VkLQ5@Vd~ z*Q58ETfd(QsAWezJ`>WdNv;{PSKBFjyLfvEl4EKwB}*@{hFt8&tQPBB%P%vOc!e^u z2=?ceX$0a=N|q)&mL|hdV*6(m#EZ>4gS9`W*;KTwJ07?OM`d0|xUDarT_0@3wuc`; zObfMBwqRlYSA^BL6*tn<3p-X?OgXi53tOq;SVCr2gR}0YLWN#tj5oPR;N*IU;F^!K zvsuy4rr9^$kFZTSWZD|f-3tXq6Xmn_;L$n_yE?8Ica|xiF?;i7>LNIPxzR4S)p~6{ zY=9l=ZpRWfdo=`x%YQ0N6tNjCZc?H0QYNPBz35a6*J3yHFwR2N?)Kd<@jH6rVml0% zho?S}CT82QWREajm~H)*i*;S6%o#?Lsgosi@NaFSTt*@$h8a{HCM(_LICl}JhAV7P zFKv3Wka{3PGf{VTDx{@VCWdFX|88R`syl;4Z69OZtq;y{UowJg{!O$En02g6zcMiY_(>pP`v+9oq5H@u<{JmavFokYG`zulum}9nPqA4+fn}+}{(BRBTVgd`tXjO(;ES8yHsU_+u4WmB%o5;H&t$&$7}_3 zj3M|$w}TEkUZI*LI+$BUQxC)LXm5}`WXa%4$vD<>A;Mjq`aZY;xUz#th=AqPdbMAl z-c@yGo|ChgK?ajI>&7Wv3InmW3D72q^Xq;`**y2)n4Y6F`oY^H!^(d{Fss84Chn<2 z`_F0z*Q$YIx`VDox)ul*a!;2Mq|Wi>C2OnO$QxRmU6|1+GYG}FdN*0ZUnAy><*bMj zrk=D6Zs!y#+2T%*D;rYn?#Zx&6u&gW5SbUA{K`%b7|2mVZQfJqNzMYYoWBueN#9gE z(@oo{9wDcbHWfG7%K^f6N@{YUwcalo_ZHVuZqFra=~r~cxg3a6nE*GpmlwrE&Nro4 zoUZok#1Rd-x$UQ`k7J9Sd`WFGHM;L-VPoUujzdkeiE+%2iNZ=739>FW(^H#Rqp~lv zxtub5Qz#FH<0RWW6Y70AdLpBn0h>`1s;k%Z;9Pdm0I?YmhFl=HcZGsFF#Wf1kI$AU z$LfJD&|Mp*$A&$zK04>7dXY6&Z!K@IB60kMEMW#bGpXP_)KTxCtM(>LQ1+m<`c>Px zG0g`BELl=?mj=GqxjVFb(FdzziXA!!_EfQ+xv;naWSUG6x2I(@(>P37%%ihQ)3rBY?pRxUOu(12yOVu8;wrkC+ylvY{`!Vii zxctSehc#ArB1F9cAhxfQ{qtKK0Hqj(Y6)+iKs|tG`pBk6OsWMt;jP(6Qg3#29|uF) z*-=^cOd~mAlG}s4@Rkgiw?;EngB@DtgP$C7_EiT5n1lvY{_r5}WZ9#{z9PE@W`Mxg zVc9gT37J3&t~dj(*gdJ46eyji^AK;fLIkabK&>YOYNe2%eIdawLtwlm--uzZTx%DT z5geN}ElQN&3i4cu?%Bky`N?M0!)EJKIER`>4}ygu!BQLQ)l6Lu>(+=N_)=WAfA(72az}Dv1~_n69mXDxB=QmZy>wTp1%A+`E_k%pS~fk~{O_ zV3>^OM9W?e{h3#I%f{Z@S7sPhUAU?=aFzw;C(A|Tn8U%@gkdlR@CYaWxX>h9))uTm z4TUipOYlUVjK=Q1^-A}es7>K1X7YpH2B3Rht3VdHc@0mfL}3RQ72*Iixm{HTU%1nY zL#it2tyqCC)r(P=ZyB?v6;=0`yahp5^M=mmdDWfj+jJqMYmFiP7jXGubVVU_DJ1LK zrjSyjaJ%bc|2$LZPRFNwYL2QC%n}VZ5503vLKsu4sbgf{Ne(Su+JMb-U_c_OI{P+!FW+Mm3gEu62IiK7<)f;v1 zXfFCahc|TISX)NQ4{UEV{!zSDi%Ok$b&GSrYr`qYoR>YH6`pd!AnHN)QS-vloYgF;UE7n*+NGDK5~u-Ltk0K$p;YA901w;FOV{Hd*Dx8 zl8;8GWfu=q+!;6d6K8p7Y#)}$R=1{mc{|LcxwS(xbBC}CYp@+ySvQy+F+R>5+l!@R zt^kBBTMNP>;Z>TI1121mHTtNnIId0EIObuCUI0AS+zW$(#$KTygdRl>&&_j)zBJl3 zbROkL!oW{^jTvZ&)>W*NYzlie#S$Gv%N>u@8q)O&C| zlFOMl((9M4<*wU1oIWWY_YwZmA0I@EC+wW`SaKK11BxL-%T&(BigdYe#k)?9BS@vL zP`qCs_r>!GT{XFv#J^^^7-U1Yd@&Qj@;o2hm&8!O?tej^waW`Me!EQ5$uii&&1bJa zV$$9YFuTXkwCNr^jzp3xojBbm3p&eBO@bpVV+b#h&(fi07-01kd*|2=+F3cnH*uQJ z6qA(SqwbKC_hCQA{fzhcln(r2J$H+g>&Xi$kH6?J%EuQ`9T0oX`_sje8pP~aI!|d5 z50ldQ_1WF$Q{EB9PEFn!!&HPtU*2QS56UNfQCz}AC-7=kZV(&$L^k$HxBE!0`i#g~L&~JFX=|xoCS6hIqgF@uPA6dtfAgZ8 zfp8|HcBW`SuB032T@%D@4l%EB!ioPmtZ~qKONy1A+7O?y&$?H8v!EHZg<~gWD%WX6 z<(Z&`@5+rI^=Ag#31}o^cxRv7vhiBdwL3%1-ovo3NWrr=H4k&kx>&>P zp)N-dB1eZ_8VM?Ylcy3`mHbtff?^NhB})aaeT!bdckhGBG5vdAOc;+NTy{}-Gn zQGsT)*AnN-dGan=p@vh^d;KjDlriB(%O|#SON5*fCTCdK1*dEu6^#al=&SDvrSfWX z`rYIVMxn)LRKTzQ4dbLR!rw?1HJe2cJ^u0RL*Cyrw*54ZGOp!*tOg zOB~b~H`E1996QDC0=1CTbz8i1B@5v=LCkxNW%XApWD{J`DQxfi%i664ggaJX0{^fZ zfSqU{;xFKrkqz!XdBmS&-Ld zMuFDH8hWsO+Cxs}X0~oLHu})BDX;WjJ?M|eoSOrz%#tu^Q?Ay@vf5-^^Auh4G+hY^ z#Mk-Q_=zYt3D|cQ5=Rab`%#VHVaKA}RNrr@+retbXx*f@W7v*VzBFwId?!X2iCiOO z_-RNTT9e1R5fC-vH)$LX`3pgKxtAx=>0{+9pzGmh=|($K?f<}pzhG8zTb%+~T(edM zBe=pC32q8hlNLpKXWVpd$cL6rMuk3nbD4Pu2IIop07>ad+}FOEvUN%ETg$Fp%dSb8 zo8y8g?_n4_(9sWl82wVGewqQB0~FM)GsVV|6u(tp$b+hoCZANMz^x8n=oFUj<}s$< zw~TWS7E18?8qVp+r-(}?tZz?BoBMUjCMLYza8z+c^^AvpK)kmCM>o9(9Os;koJJ6z zh>3Db`Mg1x5A>DCh?!qS?ecg-h8+GW4&knDAV3=AC zCJeU=-LQJLxI^<*;ZN^r%Esf*++{rQ&L*JK>F5j#yoY+~a>lw__lQrAnZGRC<8$_0 zMU>UzO@D7ziZ{;+;V0+KFPnNmNE|eyTkixW5V?I%xk{&C=U4dL1LK4Ql_?+3u6oj)Vk_x&3 z*oRtLFyFMka66I}dbNGeLsZJ|ej)sB#qE2Y4u8tc5TwMi!%il+;QNHM)PC1HG{3hU zWc#>cO|Hf2yIqgQaraPtqiFSyZjKUWKw|BR{kPyoK({sBx?+nRxecY`MnT(xoR|F zF1Tm}kpD$Bots`(C ze^#Z~tYNfnA}t)t435)4DOBIsIAh8>cUm) z%5`JdT4{Y`w^5V2+Y$Q3rQ?^tlZn^Zle>fG&K{Ze#LIW|+0-RSF^i9UYKqx0A&k5XeHJNxVlkrhvfKaDD3rLycbDOB()(mbJ6)W@|pq>@@$ z2TipH8QM{XR?w0)m$UXE^$d-4$lo0BSTK#RYPQrLuxkmFn>J3Nywvvg)eY9*1y|y~s;7I^_+nMg=ZB zs_GIqZfvYtJ05b@249;Xm8h~%ha=u+C*o||H`eKD+DXZfXKjZxZbLASW%5O5)Q*9e z_2;*vu_LNUftuEz4%K#QLhOOCG`F;JGnjf246QMv^S*Xx6&`fqq;f5b+pctzH87=poeYF zm4ZDJbL!~*R=QKZbcGo{^TsdM7+(2^+arutqao(-0@@CAZgz*SC*%un2a z$2M!aw70P;o>^^I${p;h#;n2Hz@(k*vw}&{+i2iyKZus?W^Qdv%w6rLJW;)_;$%&Y z*-l4!!gi;80qb1+pO=cuu=Ih+2P~bK)-DH3 z*D~x_B`ekDRK*B}K%R;fwzXn1-eik}m0_!R!W3BjQD3;1)c#&`M=t_S*DW)fo0jxCZwOQn$RE@&7cEHMvbrq45ZN-m zN}aeomDIBjVHwvt1q=yo(j}}it?qUF^JPj#lHmCZdb&pNEG<<(b}yZV zpt!{(GbB`N#(70KXmAEwi1hi$?FkxKQs!Rfh)+OCD(fxg2XiFxINo%N@b0!YKY%#) zb;NK}Tj(csLcM#ceU-D9B1QEI|GV=_FG%G3(xO|I@`a*Q&GyjvSJI6tlvTcci<{@KUXQg)7{q1AMuT@C9|a`PRbd-Ig#i z>U@pXRJApG(vu?S5~_TufE?-T-x_iO{iWajmnMM%5C8z_|5gh6uPRVTe3$GHJxXYt z)MhD7vZCfqTa57lk|C9#VZLGsRH!(NwXLN{Eep^rsSk@e*`v9>w>3sjbt|ZWrex;0HM&siA2`o!TBjE`%s)jV# zr090+;B(@jZh;Ti>P6Ktg<2}V3^tH&>Rhz}!>BIlE(c|hycN&<3WKa|2+ZDL0*5uRQA`u$sy3YlE4uI0|3l{0st`lkC!8DXC`A{YhY&L zNUUgLWanu7pNwRjjXKNvrqk^5c!4p-lu*a$OBZh(!9N%Z_9G+9 zg$vs$6Re2V&S6DV3!u(-jv&EGnkOkF0T=7T-%FJ*bBp$WIcRsipq;X4H_Y zE7@li5YgKlfN$3_*0Y4RpI~v*>x3N$yGHdLcaG#l=EAq%7w$U^&M z_P~k;ot4UQu!bCXnt<7?I0U!pJM5tcmo?R@Ep#BO=l(bfzFNjwi?zw@%{SSSjN$4c z?%GpjQbvN?R&j<3O(f`8(XQk&JpGzx)@7(_xE~js?qr&Un=C^j*A z!XA4iPbx^lN@~_PYc+4_cqD_!(lfyjuf1{Uzw46@3|*6hhdf{LFaC2ZrVLFo!<{Tt zunkU%DRoJ2krh4`QAq;+Tgz>obBjpES4o76)_9eOxhV)@fyxNuOlXh3yv2}3KtqU zKTOxDC}s!E`2_6T9!tdM`5cnP8QZ%<`%Z`orCkDkhlT2#BiR)IX-qHCrck$!%3e66NJyj>&7+-@BsVfJR&%>-aE$?9gaBr^mN-=aR){jx zWto616%qEe%AJ2Gh!bnLAk3K>xe|=K`T)`3q5SSHJ#>r(XXPFw7o{P$a%9O}lzI*- z>r9!OZrTw`cmA$0G>y3%r{tcBQ+LEl@VPUR4P|-TpoE97sDJ&WGqrDpAxfmGod3Bb zyGx{E=rlir8niV@dSfJCNT7Law!>8Zfd-UqPSH-8sRnJyzyPwQ0fiT0be!-bt!1a9ov6s1I7H)ygI90XVg94ac55qNKa| z;wiJ2W|wME+u1sIX3A4qhv%^?!f6KP^$UA}`S;p%<@J!S84?wn= zVAwk$E@f}v@b9Gdz*la+H2wu6G$ZPR#6Tj4{X{0pi~wnsiOrbf40l_>Rea!xiPe^r zZCT-58*qk^xqoMg%1){xtJS+|X(1Vta<_zPVm#%pV!j==tX4?>%XVR8+eJ%<$*eFF zYm#<)L!;>gWUE?&d4O80fqqv7lq8ZNI&fwHnLr1e4@`0+6xN(mUx_gnW`vFU2-h^5 z-DYjz$0;y+dDz9-`g148av>21(27@gVqPnVtU#&d_tpp`@Y)d==>6{kJ#d?TQvi)& z2iPU1H|5MsuU#y}LlwK}zD~E9KC&&T!K__ecz007%nnY0&t8Ady4gPP6}`Q7zw#z3 zvv$5Q81Jwc&YB|@Bo*4^+ z0x{WTsjwIP`>y=^W_eciZ|#oCi`np8(QEdJl%@7?nDb>?OshAaSdb*o1!CBZWiFQ zC02-oLWb2Ibq|0a8osSCh^xptIHp~K^Mpsn{p={;s1zm11$hzk$_vF3k>vFJvl?Dlx+Vx0%F$aI31Y> z^Qc{J3U@1#+T_{Omlwi_uIAHXXKyVfUrngiQ6E~9u$1AmHtGC4 zz7}WfC?s{!;Ua@JsdAbV=Fmg!YAb{G0;?vCsk9-FxZCSP6QfL{i7_12Bp@Y6u*8UN z#t`S}9+?+AE*9On zvV8U6e~!)mW-OSM6pS)=<0oxSR&qhVA-%F`#4-5B!7|3aQE<4UW96G%Oi!6Sqbkf<3NpT85Sqx!HXJ$P~{5&X=r=o+h7yfFd?QvK=6$y0LOHOP}5KE7cE_IX8~IW zWPnd=f@VY9R*n&>FK(me`a_zF&(zIa8+uRo-(%JH!LEGPKk(2Q{C^|%%D7lNTl|-P zQHrvT60#YJ@9tG-NIkiza8WXp=s38;H@}rcW~yLCGJ;J|L+jB9lB+(+hA}I4=~wa( zklhwGrgQ0a4%0X6H<3&8i-(($rFe5E`t2CU9LE_4{^!s8T+$wZ_Q(!|C{@{Y}rb9bfH)eN@n)csYa z$?}!yh`OB$kJpa1G3aQQ&J~+(SB4k-=k}0d&P4z~ULR^5XBP5i7^GyqQwsk|yx(n)O z6x)K^q|xmA@U<2Os65seP4W>xgp${+FfJDm-*e`f*d}rl=*8{=a9>r2cf`35O?QZ* z#si>_Q`V7pkvQBAojt-4?CsXKq??jj@njVIXC2zFy~3Re(f zFBhuxZEuE&kUoNdf??< zD6k)*8U5}H`m=fpW?2iw8@}>4<4X=4%W`7Zt-J@q@;tlc9s4TRQ4?$Pk|YidKURW7 z5-20lKND5laV)A9tJZMziq;+26u5fZGB$dW?Kq`2Y!&69+mxj#T(*UWv<`x<&$X;v zw(T9}(D?a z+8%T8bRTQZ{yFI99AY%QI@>`H>bBvESPR*S<@LH3mrDbe40w}@Vjgz;%qDr*!+D?f31|(Vt>hny8%8)ZaaE^{!c!(ToT^X*N8mWRq@~PqVP+N?*JxtY%stkf zplTN$B(Orgn752+;9Yt-evrQo;GlH=;DRV=@|sf zc(SX19UZlg!9Bnl29;39tG#D80_vk%Dp(NCTBT%bN!*gNveb;bP`Jh(N>uP+G=j-t zd6`S?tfh*B^TqzNxf{gCMxJMm6e)qwXJ-6{t96~Y?v-^1tvm6FG=hXZ^bV~EbMeT% zPkhQdtkk*2Mf7+5AQ}S1jvmS)JHg)NWc2u5_9^s2H8aUazaXY*^BsN?Uns`o1oTrd>uIO<6=!~{@s!BVcNvxWNP={BgfJM=m-NK z0Km&XgY3UX4p}>ASr_YnZTSh2mysLbNAQ_c168L&UoK^}!lAw$iY;&iV z%8pMgCNYs3(0km2zZ1hWTOKQp$ehYd&we#EJ3V~7#_6M(`SXX1-NBwx9-Bl1OOb7X zSpJr*B-dg)qq6xxGc1K7WMZze&m(@ARCA74Q$$I|_79n8I(dW|&}t8=c=bWLvLEQW z#DKK`!{jY!%>xG@rm6i<(u3xC^r-od5DqCP6r1YY%l5qZ9-$t+>UlKGqwT|G4k_1; zdn55?)#l?@EdTL1HXj^&8QgZ?@enk~%eTPhvmUAKoD-gRTyIuUOX_^N_U`zT8JiV9!JqmI1oU)%qPK z$p9}LH8b@eP4*9>wpVPliUAZf<(Qj7gBtx~bX9)rr=bGSoYUt{V$}+F}VbD2(Sh z(_VV&z3Vp9z3b=w4we_t0SOFEYHOsB!3B1cF;_27*l4_mD^$OIeQoWcwFz^XO@Fb; z9sWJWtKM2j96mhOC}w!IQJa;2IrECz@%!b$%q4(BZD#xF8})$KEey zpux)(%0u=>VnD;-CeFwLcbu{>MODN*~J!!(@K%KR2jofw`?+?YN#>MH3~^e!Jy_V$dYIl zEWIoBMIecFwR5{!_`jGTL@ZCa#Y<&Jp_)rFTZ+CvWdDvVa3pT|1CW3C4E4Y4CFLBA zO&m>(|IJ=9MrBtCTLpzz3Mp}pM&A5Qp$sjGgfEXma`~YWznIm=Uz40-sY0p&c+=K( zaXPfVu;*>T4DT{3uIDa~35v;e1dopQjg2FAx819eS|V#d!0R^0>&ERL89w_wzphJZgbx? zJ!k~yH&98&?L9GQrtqPipJsbuvh7$2Me-om>_emF1k2QU#=a5NrQl+Q5?PDgnR2r7 zM<*Y;+R>Da)$}A`jn_&&l&e;rJbgo%yvsP%zKF_B{ll`ltB?zOBQ`=@Zim+Lq~rwS zizdjxZ5ANHdB@eYG2=Kx*42;la*2AC{BpfPfyQ(-EAi&6m`l^ePRGpDhb%x>`9gnP zsc@-1%+&=QiP(vvR!FJQwb6=8?wJr>5$5Z7j^M=s;9Nx(PwFJ2Q42`;g{6^H)2@Eq zAhX!y7R#Ud?r{8%X{~B!Vd}L{3dC}&z>sK)DkZ}<+Hvb}gyl3yT3}q`WsRzLOMyxe z>$QaFh-i`dMFueRl>pRjjlIKG@lRJtdRw%;3KbC+llJz!Hg>zuxN;vL2PXJE- zZWf=wr00rpN~}xg3Ci;(*yXy{g1p*$mE@o^q85!WH6`p^o}}F(YVtyof<^XN+MYH6vt9Eyjs(}FPrshvUnVNh40<(^?MW`vNWX{Sp<+MY7W>ek*jizbF z-Us5g0)?2H1Ih1Vn_$qmKe<$kg&=udnuN^)L{6D6x6U2(Yv?;r&xUz8XMHFF#xDO6 zZ%7^Cg)Ck|qOXjNmKe%M$N(P!S^*owL!)pK;0%b)faLRo!amJZs+_Q>h{#wtQ*=*g zy4>g@pOX7oJR#pl-Sxp73*h=GiG;Xri~`c<=$-@Q4NHO`%qnq|H~bL&NrGpBN<9W& z(TkwxL3lyP(;MP26!PfsCwbt#PoFP;w=?jEG!ZY+SP>6>?o>GOV5NikT&NPTGiqf# zzB^=NDj((JD8;3jN&RI)4_LIs7uCHp9nrvP2l4qE)Zb`a<}qJndg#CuwUWuwWfF_w z{G-mm6%)O2T8WWOBSxr9jf-c+6xkxCcoW+<)N*M;Cvlt*15yMu5YJ#Z4*o6BbgwxJ z5`N-+WC;kBJO2DDuQ_IM;{D1*gbMOGE$wfrd?0^G;!$L2RxU1iEgG8aFwggHQ=UlHJgj-NbRg|jWH$WXsLd-Iy} z2iG}iMB23wwdlU%V()!L#Wsedj90qZ!lcBcE-L8ZFtj!SZlwh=SlCzQpO$JRmaBAq z#CpIFc|tnLBM2h4ffb;-1k$xJ$TgCoyCO-RTp^T~X?)rq^oL`nuN{XNBF;EAIw$Q3 zromMGchsEuTHq4wBoZU%SX-lFkSkvDC@-(V0_d*8YS=_`>#Iy9ERbImO`;bT!X9o^ z!%UP*+~eV{2fTlemadiu;1>Vn;I)71CgT4ZE#(a?{*`lMly#htRWN*I-TulfEc^0X z<=3NFpPMRmYC=KD0NGF=*gy+$PWh{m8Q8Rvu6L}{2eJLYeFJ(fLc0ia&Geno^o~2Y z&)AO5k%H1a+<4`5;mtnwn7tnR`E>)-g~z~v)yIdziKWZIcxRA$QwTLC^_yrboG_X2 z%I#MKSYc8Z>gSQ;7xkTuB=@P&4^Xon0H7?ABt~$4F#8hBgPg-uPOn+xo@x=#}&fv(yyLG4ZJ%!?3 zu{U6@0H2O0A~$8`ui9sau2yio7NN&ayv;wAe%4B<-;UA9BM&DfDO0_m&K7;zXsxV! zRmC-J5y|$|-arz6*{{NanC>hOx2S}@Qs!6rG*Y}0Ic~-ATdK9V9_E{?)<{o7?IoR; z2=*d@={kw+H6?omiA6W;eGg1D>-tvu)GbJ0KX{iOI9j{ha*0HWFroJ1ENyA!Waw$m z>&YjO$`%?Q_`sxN49E8^ZL(mu*&cwYN=q9o_hm$6wwW8^v`TM{;7l}{YCSGCR%i9y zIXHq%koNSYD(}RZFjLo9+#5qVWVzg4+M`KqfD-Td-!f8AH(Z;{>`h=RM(R+A5K2y9kmQn~NnxHc#VRTu z{L&(PhLa(8Bz&N^1f*ks5$mI&b17y-%X_Q~VT4EweT=dF}T=qjQOdoQB;ayVxoDkAssr@tXRlLcD zIlLY1Is8n@*-aXUzizH^Kk<+jmxI^)ACmbyua%?rpCZK`_BTqgEmy6iHGG{uvxt@F#K}DsyI;Gco~4c+Ipv zs2Z}|aeBTF4XemS0eqhj!HtV3%DnNShBltosV{HK>B-A09>8tg5X8JWrLZnB(NYt$Qea@fPX>rgk)b<2A-{waFI8b8jm3# zh13M}Swbv&Jz(XaS5gC)kLdS5=#qMWKxOl#UBU=`@b^Z+=T3N*+&9v_9YyZLLPqb2V(4EfSP1s8m-HCa7>Nqbwkr>Rw)G%!&zFX(u_A$xMG(=! zyehQ8<9nhC#7k+899A$j)%R35pf#!)t`LEYQY48+=Txd%r&mGOJPAB*h+Ky!nfh_V z0qPf|YJdb@BZZe~*h~BLr4`Ng`i~r?%~@d{@DJ!?!Twi;u59AutY~5OuL@mR#|>Eo z!?y%-n^@-~L9q(e1h7)VqJd^8jy0d~J6;fp9ZUdDZE>1eXKg#;uS8RCxK$T_-02zo z9hnC~PospCrs8^3wnuO7Yr4njt>5n(bRR7Dm_Cp#6cuJsT?Xm&G>RmqcL1G|dfi=e zh=|e0#(*Z+CAn5vk^SU=DAaC|{pLU`%poe@!!wu=VXEI${8o}oKfTH`k-}}Lmmtr9 z7obIh%K7~sgIZ4l_(T!!9T&`Mg<`Wd?m2&ZcVcBn!i&&;sPCu~UPSEAj$QIN$C2-l zJ;Q6&N7ic__FdUAYtuua>Uz=)hL_A4@Vc=&#)E<(A3~1nC+lB|i(q9R+cC3Gbh)Nx zIwHo+4m~HPJsd$crX#dUX)wBtQW!P4%nQZaG25j-vZ+TEA{M4%eI>TC3yy2N1t$aN zOUcEs4}80GTV4u>*K0Jqg-|#UF1ztM#}PQYAe+D|k)+`3Ey7xmaaQw93{lW9kHYQ@ zq+M%Ns9HL`?BR6|;F)MG*49R02CtB6hy*%rV6J3IJoC;?&NCD8u4VcDhJSXJ8ILyD z9`ZwOt+$40w_}uI*4x59VHWL{;a_qVJv7QT%JOBGV%W(-7lcS6g6f>K@2JH^oR}O> zFdkrLK4YWzf8cTU$G#79h~66uhvanhB4P{SgBPzir&Xme%!%#d z?mS-L+P;2aZ@x5GJ;T1r$n+4vtp-g#*v^I111NuMpw!}v(dWI5^iAc@_{0*Ws0yFA z7VPwE4If25;zlPRz4`q*Z~Z;I#jUWQ-^%c&NXO{}Iy_yk+=wfB3DbFAu?|-oRUhPv za!5@RY`a8DTBgKv+%wb($9*4=T&LS)6ECGaVWxaDUQDy|pHU6cgaN78<=R)}`%w}Z zDzJk==~YV4_<|;vlmn=#mS!jkG>ME+2Rx$UZINfNXoqoie8~u;4DcMa4YzW{@4bJh3Zq(a&WiV5X zz@W*oN`8U=oyDibV?2NVVX+wa|7xF8cC@g!Hu?Xxrh?O^Fv8DFBFUzT_+QeJrY44B zTVsxLb8%=3?bcX{G6IGv_QmwM(ncyI&`rk3t@Hu9fPGBe2LPA=7$*9=3Ei#J2(2#ayN8yDVP@My+0Ya>Mt5|nuB~Ge zj*!F%FU%wdJ$~_k@n@ZP(t z)~?yl1{vxSu`yb2!EMNlPM~r8jeEg>^~Oq$*g=;OtORSH76PwGG8qN@W};Wu12_xU zd{J0^|4@EYN+>LaGq+aIlB)q|Wy2&SU+s!59tj5%5LKLjWU=gMX`-_*8oTLJ(VVFF z-6M8r4eq)-s1cff-YCQ&U`t~|Mwz-Q&ypbQRk})3J;-L?hr3CB5Pa+FYe$>b7YgV! z&WZ6pKg7kl3~b&RO|}-pPW|}&eZJ9h?tENc6mXhcrIL-6>iMFg@8CE3vI%4@{aBJi1B;J0Te^=$>Nuv8PC(j7-$bt;U; zG*KSSY0myw{A=0ZYl!}5fZjLbz-Wf)iA5|j#PEx4AFKOPKnJko2glham+?aKk_4k^hj^G)dW+B%dgze# zzP<42|$vID2AUrF~!o$ zmtQe2yd5xMG%2BXS~`cxH>kOoA$25WMjA^?;$)tEXzf)9DilgjiR7?}DJW4b7GG^y z_^^FODfRvO=Rf2~Y@MbT5;OpS3_1V+$$zD_inE2a)4y1)2Fb6zjLKuz0zXJ*FdiQr zpEg82I7KWj9RVbePyrZF7LOlM?HdXy{m0ZE&2T~8rnp*lad=}zx!OXz&SpuNhB$Gg zW@E!=w`rNSrA4*VYEjs8=X0>HFCJmw^LE#5xBE25Yu6|DbjR}o!=HuyTp0Al@+}D& z?P_{(yG;!4bz)U+a-WyuYI4x5P7m&t>mC3%sUggfJ}D^7=}v=8_0S26_g0ty&fI+{ z!jQXSpMgx9ojgp)(Onx(PJElJ_yC<5hD(8zEt0YB5?}e?IjNUV&2R;eQJ$DK5ID1f5ikz=%5uTy$_Trm{85^d>vEbrCF3H!I)jTqcKEYR+}&* zeSn!58<6gh=uh;W0pz$xYXW^6NkIS?v{8T08Gjw`pC0l*SJ0wJ)N;GVHP{9F^fn;{ z2^rKqY?D}H719l0(e{auG}9IIy0EIlOC5`3J{PKR3qAt$xbz=EINepx?!Ab4gULDP z(zfF{s{nr3;!~<{f%%su&8!yTwT4llc2;MGAvfXk{locpFfWZofA<_)9E2+8M`bke zu_WU4N}dWXf%|`2yfux+nTlAC;J!SUlSj@4PKJ~a7q;4V6Eo;6}P1WY{nAr4b>{=)Q|zpIGtJz!z|h5TW|xYu3D z4qbKmg7q`Yzjy=xDc{%Qv>obo`2x3tpw3mXEvjT=`BEEo=*mCzV<)gbvFLGM8!K)3 zg-J-~!(DJr!d|p-)~-O=^fH>qIS~{09Z4|kmiZ0T*SldksP4Z+z6i2~wqh#KJ@J?* z-4*-#VEXtBO0W%3$vK}~tWArcD^^+OJf?VZa-UNqFcXwuEAd#b5w%TvjYf=&S5RY( zhM70md5>N$+~7|y-B*^^Ue9~Hp?EUZC00CnH13WxZR_4*GM{AW?65aEg%msV=6r)( z{ClQiVTp9-*`R~L^rjK7qv6v#IT@|Xt%ZFQrrZxwUQw7RB8I=w>qK-b6T=x3SvO!& zomP2aC3cbQjy2iHY(-dV72wope7w}0RMWHgr&qaC6@>xKD(-5DwZK9dkhNmAxSU4K z*RbJ$_scZ z&c!;R<_5kq$&o3;3D?qK?$MZGGegLI|E4(32T&=kw*|;S=N1py zaUDJ52-JkQFpJG(q19RHawVPk#2P+0X0!=%q~J?UT|4iTuh9&n4WLFYMl@HSH&-h5 zS=X(d7K{6U6Lcf1WfnovkudglE)MfVcde+5R#>l4c23?jT#qhSYQHqAz@--I-}9m* zBh~gI5Dm_8h1TUPZ2646q90c_gyjO2B26)X7NZUx~J3{6Sy&&XtCQ4yNWvD9CL;HR!bI(W>4C~!j#bFUZxgCxkA;;kW7VoK$Hcq zn|NofQtG-EVhgAwL@bRb!q9}Uj=;jfZh}j$jjURfn!SUhA39_;N_9% zVX2k-Bib)4=RoOzP5??;EKS6{*d1!M8~{<-1o1@Ch`3l4&zK%tqJZMspn=*PThQjT z3~{2e`8Atoj#4ArI?WfDX!(h7l2%KUMhBfs<{q%g3-gNbP>ExC;>zt4o}Y~vHV$)Cmd&dE) zF3Jh+k~VoSYS&v-QMPYN!W%%s&6g@Ib&9`}DtW5oOx3v=sX%SCs{LX8)p3yLq%JFNI@jCp6fi?>Lz#J8C}2-&0-AH>Ym^TuVXRhVY1|K|?S0SdZG0qYPf+A7RTj z>XC8rz6`!>oU#*AJz)q^5Z8rMJ&@NcC<#7$hl{&9;I1b}Jgm(!+hNFi9f z1d;7862sd%zyBesnL9FIqJaPa48i`3>!z%os)d1}wTXbW^MC%|Ro0S27C`v{Ytd0h zM_7_<4=04O33vwj@Mj><4lV*LE~+)P2t`mj0rC3UUItcx2HW0Q5Q~Yl=}?a^q!&}U0Ghm& zK2CCJ91o;QN5QyCK`#5ZxQ>F#n`dreY(&QIr`zvn^G=(4z$n!>9THuR;l>bPaD%$7 z4Qy2e??Z?B0CV}*%*18jrfI2!0KKjd(x&J{iH!BRPV-PdP4Y^|%N!V`bs;G+YFDOp zC!>CuYs#kdo61>!p+r7HFjj3tKdACr>LdM(KcD4W8}s$d((#P&av(kj$V12z;i-vV zY|y14=&1!ZlfLiTJKr8mv{H&rm0jSKql!)eN}dr`OBhV5O+dEhdxs#mneC*L^fe&HhW;kteWZRYdlvV@`}>4PLz4Ie1yul-oVnPsiDa# zM!2~twRqWt+o#L+FecFqQx1>Eq>CY%NQS|U9B zR)KX|6h-hkO&d!A1rZXLf`Y*6iv<*bps^Lq3!cMLCevr30nM0rStdUt;+Z6P*Pz!Y zy@uh-{K7y(}mmJ_u^0*-|KPqjt?l`0|!+B zje;56cPiy^2-z>f3{Nrm%%_Tewh(gun!9hSH;!UOXF6iPe3NA#%|l1xiJF8tf6p7o zgv*Sbt;DIF2{BpK$r#TR7<5gw*%#-^k)A}sfYThXwO_Od+d2&tsC;|McDr`XTy&&} za5)H4pLjud6{1C3XRpM`cVQy|-kneBxhC}0M3*y^(_|IfJ!NRGM zoIzix>d4Os?;ivf6-kCGn32P6iNFwjW1(GEb|EhG+|cO+8_bu%(<{e&-ujo@EF_9S z3xxHFPMZ%yMw2L#6KA(wj~ANORy*{L+n4M6QXfh?tSVPggT|7fh=ErO3$_1M+58J` z*(C4XfSt4dr7yEPTKOi^gNUk?zoquaPX^7+^_I3V5PfU6M)@}hUTC5TLl&KK7=&Ap zv;LPdNHASfVP+Jn^SSU$8sljMM(lMBSy26N%B9r$%i7=LZ{tlDSImYaX`97L@TW|6 zl{aAxJZJt|NMtVjZb3iVto%tM7?|9EnCP)V3AYKrhS7V_Zn;F37!)3VP7P0J)_1sa zHREFY$eqC>;Hd-V?q>5U!O_$wleE5ejD3=A3Z(rNE12()r$aaPc)T0IbZ_7lk&-15 z*T|_Ztl4e>+HO#q5Sl;6dE>NjE)H#Sl`nhrVs2?gyttb@mK>aM(8996oG6FHKjDAL zZ^aFR>!h^U2_+uh3%`>*HU2~)Z)W-evYicFEvB}fPo@b|6!eBmF?>IhXE?vBcY$-^ zud#v|Cy*bW{iZr37<;FY8$qcRq*bzGi!eM;E1lN;r)ji>g&V2bKj}*l6aawv zf3n=dM%hlu!t+0sdtNK3N?zE0kI587i)NJID=gngBkqcGa7#L7w#>Ur>HtJ`a)-D&3P=i~dC%b#V-Ex!K>;)FhT z#0DIax~TpL-z0;Ay3~OzrV}Rp+H!icDy0=8GA9w1u3{X%fG|@tSdihUMM!fJ(2iB) zuR$|XIZtXBr@ZZFK&57W3hX}bVFks~l|_I{=sql_>gb_61lAuj<2&d-`(9dBL(!e)#Wg-I?Ev9A^u+vy>c5AV6Q3cM8#u4p|Y$i5@M9vUF z%$Z19t;J2R*TC+mf*I=&{j85BA3{&b(`*2zk{Z1UNzK8A2a?^V zq;@M3TZgFbqNRNC@hrwi#E{sL%rkHVojBHda5Bvt)Dnewt|G$kSZc>Z@!FBhq#<(B zuQS^U;J%zV-X_g6>vtf2YgGcKBrbE5!b{Yc;Sv15166|TH8dL^0AN}M z0D$1X1NC1C$P?0AdHMO5<5lJ%aeNZY0Ma0W8n6KYMA&2zqO6}7UzomZFCJ!Gobm6B zw0f1M)^Kz44b+V$HhNVVLbNidg7vi_+ool!o|WJBm&<2!Z;NK~@9Wm<$@HW#>Cg|~ zMCRvB=j3aa*GncFukStyVMgK3-J(&~=8@M|k-Ho1^Fuf3?rACY=drH^4}LZHw>vj+ zT-c*IDqrHL-htThVCwNaRov*&`e^XAXSTDEk*kRC@ipIMLU+)lIrZ;)m&f}souk{0!Z^lfg$q=GP7iHY) zkppv_-f8Xl;U#Hrf>`qx?D}!$!?o7KTT53D1zr!4m7m(!yDqL@t{;9}tMMsb##iB= zpZZ+12LsGUSKQt~K8E*F(%&gLmo6<_KDKvsU(J!<<5Rtd+M?$gn9pYFyIs;>QD48z z!Ct2KS6|B`zOh4|zrSK1zvOYH#;1G|m$L)Wy<>;94Y9?8U?UW{+zm(8x`RR{lWI(aUtL8TqCkD#aCyD$TjJ z2iqJKV7-bX-O0vDxSI&yIpprut#OU3d5E?|^sSe6+}5U1zm)troF&KC5A$kE_KgA_cAS=40esvhUS(xVxC=HV};|t=ov}SmZa6;ayf(!8*_xk82>C z!+`H|$a_{`zdFWzuVI2WtyYH$^X)$eLdBc9SrKC0M6Ft9Dq~)P5A$pEZSZkxs;z7_ z+dwXlZD2$T&A}9y;IzlW&KRk5u)vauNFgigBJ&8>}SzAV0&9dq#5n@0K zE4?^_q=?d5d1_uamyj>47jrWT1@_r%%^2B6Lay!Cn}9Z02> znCyk5UNWr)I!O`PXvPcf>nq8vmgqNe)RiOi5MdEk2h21Fn2YaZ%j$Iz1>2Cp3}g3A zIm+@85_RWA8S1_z)V~fI84IjbJhBJDhff^ZnmO{H z2ZqPma(we&iF^mnve159|T~d}GNE~Xq&TlhA3ric>leeDMZ1i zO=)9UYs!($x*j^b&dOCMSObfrL34@!Wm$$&$|e%X1p$XeNLdkAb1P#yo4Bvw{}ULx z{>%g>H*5UWMCrWAWTljfRJew@MavoAkIy=$cK_{gO-;x%zq04B)rK2HVd@5Cm#~Or zQwHA(x}OttE{*{;;3tmj#$WAiiUuIr-KMl@!6bvDc)9IgfuF)?CkX}uQOteM|9%g)9?tExt_6FB@IrGPA!-2Z z!uQ1XCnAjXb}rbn+6F5E0Yy};D*JpOA7*G^U$n6@s5-$FL}X5HwewO2iZGK6e9$KC z-2X^_q>k>I(Ed6bA~SEj8YkeaLR{Lf7M|^F=5QivR3yEco|Z=ki8|PYo7D#5e)(@E zy|uwexLTYQoM@-uK7T1w64BgQEI)O9bvc&Iaxjy>pXBn5b|RurN}%J3*R# zn0@bHY{TWH%`DA*NbGE*p+h)(N)%~~pzNGOI(?dFYBG=eQbfpf5tD^$VoU>Y_JMhQ zY_NbWYrtXsJh+xu0mGm8?sPv5c}BQB0d2L84P8$gax}p!CG^lXus?hZ8${;g9QtQf za(xi6bGz4oEG3;}EAuE@Cj*XPK|~lWEGXywJ4T)Z^WtV^7inp0h5NQOZae09CA^^S z^qmZXnJLu?H1dFMuNBb*E*;E~o%L=vW!Z>DG64?=K*D{BQ=1f-Hh42%$&mFC=q z0`d7ks1zn^E%-<#L$1y?k41YvD zMcFxc!!hi*aApGpX||hN2YMhjvl;|MGO|qOT9Eu^utU9zC$o{v`|A_XVRScGodS(7 z?CsN{$3Cup0VKgHhTdqn8fR`2hzo6;@Qvys?V2&DBVj_AHa$B!?ju#HF<@$8h#<91 zYR$pjUoex2F}UL`sS>9x& zpIifVV zvL&TY;}M68`^`g>HXDNJ%W`$=+>uhXxC6*ED>SMZmOapXgsPj*0C?=?2{^6isTeO* zKgx?&=GkamdMp{19is=J<%->;T}p~q7Wt@M08bofx{b=Amqk)G@O`kChta+xeGSU} zH;m_R&*9O&6>x7CYGqtniZxX-Md%i!N}x0?nZB_&<54zBa%N^!5NX_P*};G8L6ten zO*iE)^GFTKA=os#0PIy-5gsW}o8Oo{ zV)lV~#6GD})p$xaEa&;o_0W9ye_NJVd%#loB!)a7FgrVm6x8>Uw-uxBz)GbjGq?he zt6Ml|SC_}lJ+z21sKl(5Zlz9%*8qwrVBBT!)$^N39 zK9a8)YY|Oo0q>vRKd_2B7*pv2A9Iq>#)(aM8OmoV{T^N^0Qk-*sxeF{z284DN0rW1 zrgUWVnB6ny7?fud!mW-SYPw{X*R(FdUWg+4l8>D(qf@*5{U|85Rc%$jP<6>1F142; zg~dU!AMe_vaRFym-j=Y_@Mcsk=9^q_+fV)>jci5CD8ES|=sX&<4RC~SEWS~BO~3G! z;_(OUiq1EHKfo3{!%Gk5>rQCcvuL1xM(8#!4>Ngy9EHuJeh&XYI=~=jv8!7uyqKH| zG8j=zxj2rZ(H)#dbE_Does1#wnxD-hA5@fs;^66;R`gm@D*1r=BiY8PUtW5?nX*xt zQ@_-B!ND#pb*=On0&oTjvjh0tRsKQq9Eh>XQ0}Xc>P8YXtAl0@D*>TrTOM>#nc}H{ z;UnJ-vf2qC^}wPYNJah3h})&4?6ZQvE2aG><&_aF=ltzQwdgUv7_L&SsCh+E4CUyS z#YI_HuXSm4X_bakTyOe_Y(d8lzfNjn%B8uEj&F^aoCW5bwVZ;KU&rucYApYF~+*FR=WhI z5FvgX1cnmW_(0y$KX-hA>;6Fgf{i{$aEI8VCYyKM1sQ0PCcOl{CPX5{ZLE`2gQ>Ea z3B#VsB!{Xb`dvChic~35^P_~E4hv;<&3u>smmO+U{+*Atg4@8JkrBh4N1@n60ja0&4 zFs*D!3ER3#uAadaVBn;4#USm5vpbJ=@1YL@H5yU)v863&RGsf=mUHS^^c6l}B{*;> zS#5OJltXvdHONvrFvB_zFL|@~XA84qso`-Z3XNH3%I(qeI#SDRArCh*lJ-Kazg?j5 z7!-be*2Yscb}Gl@chrV#d@xWwn@_5fsN@wxf8%un0a+zm}c2+N4I zDFsX(E(MP{&?&gDEyHuKJ*7qY7HP_~6a0Zb1oqlm`kb$1(iQpZE0&#DT#}K>78OlK z$jLv!P}I=F9X88&qT{=TY@HdPx}pO2|ncaVTHj_3j{8w);{#rB2{e%I2Fxq5RZ0bpxx_*@i7XqmY#ZsH7OC z8p9e54}tknVdM3fA`W-*h$8PY_)Hz3wkKN# z$fONP(BDw27}QsY!QR(W6XI_-N{lmrBUGQRG)Fxj)MDd8~kQ*rAy~1 zU}F7&B~*#Uub9A4nI|K{0@S_J&2&-q z4h#Evclp%6W{?b!B?O^37yw1FQwy04(>Q`Ti>8_z_9&aF(!hq>(YQo1`3w$1ml0}I zafRx@FX&+{U~n{PZCy@3#P&CL^zLMUlQKe^-}&1=4nq&fgY#E0u%>n;pMA_*+3hQC zK+g+C!bhbJibI4YrS~P~bnis~LqNR0brCIcls0cP2@A!(eEhSQyS|-?e8BQTNvn-I1qO?wbN&qKM1NxcD{P;uXgLn4jZerf3=KpW{_qVkPrp z9LYW3Nm?4qVn1e23u5Wb)B^hxexOP-^%ik_8|}C6(A*s{VQTb+TlLT z4fz-3>HHdmnCq1_YxWfuBMBp52tXw%?PDgF{t{RNGESx){G5ncTP zJxPw|T2D6kzQ}g%MUi|(N-`gCoIO&zgYpDNDqmP{IsSdSwF6D<^;vxIh6ak{iN5`JfDYK2k1+8R%O&`z zb0ql|9tNI_zPzxx)|-6RIf@#CTS6&iHouuI$pf+e1Xh3{D3gGwkU(ZgbGYLcQj=2Z z2e((1Wi^&XHd-K_E;!D-W{qmV5okDvhTi6khAG3n0nh(}eASCsM+EV9Ho!P$p&0Z& zIdZV_^*2hiOjcXlzk<@#@PWr&H(Ne);X`*B@gjq;vV=9NPJFgK)ozg{USdWs#P9C{odCHp&E>VFAs0YvxWpM4gXE#AANzq{mx zty;tXksS4D41=y-pm)4hnky@x6zP`^?$;Cbs{@ft1BTc&UK4`v>nOB*Y0Ufo5cUpC znt;u+VE43bYudKo*0gQgwr$&Q+qP}nwx(@!y7#*`&c?<)v3oY6>JL0okxyntMW(#+ z%f->txvAKGT00@Wu3(1`;<0lwlud!RkKc#AZH3>FjrnW$RsS6Ecfy10c!TYPU-y(8;H)+b{n3fgl|w+kQvd@8fB^6e^Yuq-pDa(Ju&>T`)lH3-+`bC)?`SJ@~A7-B& zHXB&~)Y_Ju$)T2UaA5lXVZA^9IU};17BZ~V;>%m5`%3;kE?=pbHBqd3|L4LoWSBG9 z%xRi|#WzK!GNG36y6_76Z%fmzhcrd>#))f88G~HYPvN-YAOu3I8iNf^!x;#@-@B&P;^G@@Zh%8(_VT$bd)-oX zunH78pK)N_^66z!36!O#bj2+QeYiJQQ~az|5)Au4msY_`tIGH*-2_$NVzx`uD;=p* zWw;lsg6OBZv8l(bgqPd?E1xv62c+_py+M>8N^|Jw7~ zkm8w=iR$$~RlkmW?)XS-vARkxe$ARKDfU$yQi*3~KccK;WLr1>V2j~VoE9Jv3{wQ_ zXCvjVJKg_Y4Yit&+?V~Qm`Nh=A(3f?xD=(!R(OJ3qp@FAjjInUmffW8Tr0?xM$+aV zQuBs}@tAi&^7HHcAMMlCZDxe$x&g^J!(S2`HF!$JX;L;6%1$URlK5p2i7IfV8q`&j zPBETIK|H7VI9`cZdQ>cq2^iJb+oDuS!{*Ra*A)gX7C(P`niJz zUNIlG_~S*k$s_4RuVd*lUf#GVNbXk}@H%Qtg<}Qar0ngsCvT87$`K5&BQi3+G7T~< zLW0rzC8)Vm4j6dg1F_bM_-UyTbs@g$)j-mv(L@O+au76d!m|EPv~ojAvFRi_XRH3X z0X2FX6VY-t=FPM`$zq&VY7oy6m3pdTAa35MqV!bc`Y|Ix_K3Wbk6iBW$LA&7RIAEm zbfjM1H1(zSK~1BIH>I8s@D$Z;@tKP4OgTUHp30h9*VX&-%7NXc1dnB< z#7tS^l}MG^6Q$5hs6MBpQJ2=9tY}OAx#j!v|Gd*;jg&|TBuPbv4ua_2K$MyTf|h1I zyXX_)9ID%dehe4zNpl)a+G`ojtO^R5o0Re*e1(K(8v9%fvuon%IcZLVZh=Ybm{iak25=S5WmLUH#u*qCI1NE7E1$ON?J8ON@(4e2U+& z<9HQf65439rsP~+3_0a$OC&8}7pkl^YNv|PTGDBiu20;q{MRa%=KxpYokE7ft`*1& zb-eTX&U8Ih+sscaA!CLS$;hhNLJojnI68kZr?V$VUWD%FPF;MvzWsv!sl}4Va?5SF zXDxpR+BsrwIP@k*EGyNkFm80V(bIW^=;Iq{rrm=iwES*3m5<1gOs;+3**Y&uRWpy3 zI*rekI3D7H*o0&ajpuDqG@oO6O#nek83U+vYXG?}p&_H|{xPu_AH?8wWb{~Oj7K2W zZwv`=iNo%Fa9YDMu)qq*bG_(K$oO;#iSlodc-JC7so6#$Cw1Q=^|kT{a2~qk46y=}vj_ zk0cFF^-VfkEY##}h#E-}7f~TyvC__8;6A|0l@*u9O-b`|Ym3y*RpX@DXKUFtFLmVA}(Btrpf^}=d;}UE87>H(Edr0 zY{f`Gl5~YoexqQxr1(E20vCCByf|aNi5NV$cw>!|cb)JAeIl*EyoG5w&oW8Lo-5PH zr;%@~>496l4!8FLpTH-ryyC1<@}gLC@d(b)q?_`;t04Cpx~2J|1;KqC{B@dEhv2IL z%%dcKiFwIyQ{e)yFK<)Y7fci&C|qGrWb35Lx_7haloFS#$plf{E46VUioYk#Ce#xH z{>_x~gdwqr67pKNsPqh|QP3}*#$^wJM7_;Ic;uJkbE3}Uivf~WAnvJMpW#7({se#V*!w>V4`5Sg*GReI&eE*tiI zU=EhrD_3|WQ&iqAus!>h&mkFP>;@V2$(mQ>DF%N))F}6b?0K${`r+veaylea_>knx z(k>{;jh=rdlm?F9^AGoRN6opy!MlxCP+L^ zeIOKHw?<L0NwC_4l5FD&GWAui>X9iR09{4Vl&ZMgU%|7|zyh(M( z``Qf|G73>~@xj^u&pT}1qvizb5x~EvY27oFa(EwS={XL=mqalN`OJv4GIEnN(jB4A zHg8lTKH1PydFQfW&ZobB#kxn$OifB29kwQGq~XQO8C**D^-cwU;T?j-htxVvZ$bra4nI zE;U=VsxJUrI2G%~mWd&@g}UbzdBZtzFWH zQPvbXie2J;;M+i08nxJEjtzf+9!ns04C$iQd`-PCSwD#qA^*#2q~%nXn7c+sv`q?i&uE?zj0rG6g7L4NrC4McJd(R~TJ_qe-0IO(<2bhH)?Aka zWnL)ua=+lknVSp$hk#innV!Jfh>Zd)&fZh(n-v-yLDOA{XP6j*YHjkBV`7`<+t;6i zjL*PrUCGO}uNobh>sC3aqvZ~+yTKG54~`j2(m*&A-xvA5EGerABzxnj_Wn-Ed|ZdK z#r-4tlVRh<2-TxbhD}+$5V`HSzs0UKU2ALUF;dXApus}C@v}I@#`n%)7}qLku;l9N z(lZlKg>b|7l5;^*5vw3Y}|Hswl>zicP>a2fe`3td6Z+ zK5^z^JnN}4oOeLGfE)bw&;)?R7A$brLi@nuGKAYAW8ud%oPwZy1Ir|J zWh%KekVB!PhRBot3j%mXt=;AdpdCF_@dJl^5GEq{Bu`M|s<*?#xT~afKvK?!qjhQ-wJaS5FYqAI(+A_Lu z^3aw*>6JQ7GYFvkoBL;jNSXOj@aI+;E7HRPYSSj3m39{3)hPHM_wz4kCJ<7g6 z(4u0dbL%*WmW|+Kfv!Kt>5;t5l59Ubn&)oc>xUR%1hNXi;~c!%*-~kZmZ}Og;LK6+ zOmJ&aAASkwoPKf{$4s8cOemQDOx_qHUNP(eZ}pFBhB-GMVw7;sk;CX(1rD-@TYR>!c)t%mCqqncJsK8#}#l zU?`lhSiE3b-!Bdb{d}cK0&mdD*AacY+Shd^NuL`F&1hW{O1?cikU@_(N>v}pdXGl!L*ytIk#>Aw&{5&?gZ6X-L@hEuIb z)}hN2krRHi6^;-;CdH7k9!-agtafg7GU}plJ6KmsL&#EyMIH9Gs5w?(w5(d$w)m~A zb{0Nmf9g@NCTD>6>WAvzblvuP&+$$D(7n#G?(yFJ4hU`rbE-%my<`*2B?r_xFNti< z#nM_P;i)f}NGVgqif1}m(&(s2HWjOarfobt5O7k=8v+P!M#(CH*VZPVq#=X&o;kG2@h%7GAGK^PEN&@m zkHs0cqYce}O7X8p16^bo2&7#B1gAn6^ve}`Cv-`rf|(de>ymbCixYOGAHEw#sdpYJ3f!Wl9?q8R?3aI+CQ@E84=Pt= zZU3&Ft6*U+;9~VPUoI0@feq>^lqDeZ$-W^@RYu4uVyP*Vbys6#tR63VZ6-imZIBMj zr6CC&8M23T>xtc0y$+%?+n$6jMitr-XDx} z{0@Z@W>eDOoX<#d7~F)5q)k3b0J=cow5|qVtwT{1c;bPnKOip6SbpPG_XuY>D=n+W8K}3}eapWXaaKdSMWc z$UaZ#s4ifqW~mN_RC{PQ4$=P2!oO={|C$K~;H^ub5^VC*p%=n&w+gwW~U??D&1!%zu~&o(^tSpcBYsY5IZDi)oJ` zzmEZt$YH0NbUf9QF{Y@eQU8G)$HpsUJ zA39@Zzx;#}m8PYV#@bi4HM9BWH}*5Et{y*y;d4K3={~Hxmerx;P0N7U?ZvE{y$6_~ zlviBZSlQj-LX{Am)o&}-UF>I$I^vkdmy4K>Ezi0ofGDPVQ6Kw$tXr5SR>AVEc=2B4 z5RqlA*5PwFtB?L6g81~E(#hU#ABEsJfY6eCh~4>JfLNe?6+ZkQzjLpx?5`D+(K5s} z2%(&X&VV_cxxo^@YTJ0!(b6~u_7R8Z=Mi&WClWk zUl_0D1QD}FT~Ma}bxrS5?Ou4s4_KE@Qry}B@7q=|%sZM8Q{L{ z(CQ)Z8wtLU7ef!zQ+P6kW>bFtoG_>LtXZ|p8z$?=9PqX}okGAtN>RjBrLvn{W0tap zN|6Rc$cpf|X$%QbP|*T%kdJpC!#Mg_Q7lR(Zttr~L$L^g_7Ns$ios_I z#eDq&)-Y(Yd-lQI2!|-@Y*6Q%LpK)*3|-!Kx+??`bZCw5gQ(`>cO^9s)nrOH%ua(sEWfu0@-{vKGqU&T)@uQF&mzWrG=l+|%R1G*PZT8F99MwA*d!X>nlLT8?dmHR6oi=&JZ>iaKJhZ<&Fbn|nf1^J~-QM>YX zp(R-&Q){}I*C#$w(1+N!@|_%j;LNkPkgm!|-r80V%v54MfE{}UB0d8HxK?N3t#y!3 zkM*RMIo$OK+3aJ;i@;5X^)A621^DeCZ3LGCH?Y2>6GY3KVO_|Z(K>`LbOF{o=er5F zD#5-)`?W&=gB<*(91`w&%q9GGNEU*Z1UNO$1-RwDm5%W|qL)U9XGjiL6O)u7>cba7 z_IS}iJ)P%r{@9r#*j&BD6F8rx0TLAM_K?q_@-#Ca{ntN?Tbw8OhGn9|ANUXYXY41Q zZmW6cC-0nG&NGP32?_!L0nQUi&oHRxZBU>f;}R|8Kf$-jL2=0M$oO^DSKJ%O9w~pe zJAS*hbM|1jg-ggDL3`XMN{3tek0v>LyeBNLvERWD@Lp-h#eIw3g(-bFPwrlO`ABC( zkiHUpfQI=WvR(hl`VO+>fZ)RNz@4|v{?jLBans;$^43`!K1rG90Rs!c-?+O5HD&(M zzWf7O`2SdIn^xbhFn$VhAbn}nu-U1O@t!i$pPemx^UYycS*LIj+~7q-gL~0;d_}tu zK5%_?bnu=Ku9?{^syBsbSlD8t!V1&0z!CUZ4b6Q5wqP(mlOho4qdQjcwK&EGn&IMK zoI>tL(hz^cYO|SBK$l8ygq2RIac4#|c~cQyH&W8K+0myxrZ0;n{gYOA6jfsO?5v}s z9;Davr3u#)W^Lv@a00J#*gt{^lrX14hj+Hn!)F=VCF3-d$U}zK{&wy>&zfe+ zko&D#9lO>76Q)LDBeVRn)FK(ip_x@^{fW1!Vw+hYJ0=~*A36z0Ql*Z?Qgdn11f%dARzX;)nN`XxXsH6{%~ zlMu-{o@x5u1OFarRj|Tv3oiJbU#>#cqpB-`4E$*>yiO_Ka@>pf#SF0AcU{L? zD|}{K!btPBakl6}`nWfn0jTYcWyJ_bAE$Z|QT9m+2M}67m=u&Fi;?MYIk{J0O7W^f zaYuNd__0I&VQOVXHZzYW%CXfW$&l9$epfR$ckXeGGhF?^z!#WLL8#LAwwh5Gk({Ka zrZjrlnp3pOvDI+CH`-3F@R4`NIAVcXoRKJP{Ed$o7XJ+Cb8R&9AX99G2V7xPb>1+~ zNLGu4J+}@XP(Ie%PN^;vgMs@Iumj%Xr>41Vn|xJ2x4LIWYrh#J;oE-PDyQEs4uD9a zc^$9vF5_4xx2JwLDZdAaREBJgd-;p7v) z&D8I0-3aKaq5)|!K1jh@ru*e7!to%G$!L7&Zc^85*CuRNw?mC*vR7QabvDe^OpE?RtL?x=Ri)6+M7*3 zz!WXT#XGa&-pTS?ca_%yTL3rvBCYHXx1o=W5G-MUhDc>9LZI0*Yn)QT->2df6^?CH z171qU;roy7s}AeykIZBH_2|h$ySgFPV4AkjU>}E^YtN#;wPhtE@NXJ&pC7`bm}Ghh zB4Nlv1R1H__-eSaj&o0x-OG`yq2DC_27+B11vevNFF(i*Y{qm{t`?)(%fpuJ56!DiwLttP8526I~}V9kd8(X%bZ;^Sub!CSmXr$h0t(5H(P0_8*#y4+?m z3(zVCgV-{G5z49wOI<5epgG7?KN6H{@PIDmPZ*4R*xIS`M1|l#34lG7f*DP^rdJb< zoQNO1o@p4A(q%8%YNlb~s=~G*Z0)crh*-WtZCkrK6{pUHCwr~pTzd=n`i&glsc6(UT&J?)l;Gu2Zo4Gc{b=pNGu{jH zVUvD+^7i#%%e6Z`IWL*-7z4xi_n@1F@AzUpSlnl#uI3fh>`kaM?cBtm?{=rBx&Z-& zAzx<)=bBbL;V}Aba%yq0*IJ?aOzt$x(}UM}1U&I|}}Z+@;;aCQV#pg>NfiX&Fu(X?uI-xOO3!H-0r3Wicj&VlR0mSds8T zFtDr}s)%nSju72xE0%;}E0|fcDMwUJo^zDfgLQPGX$FcgDB7{b0I0=S z!5d1YBSmj?lR|vcLWuMH=p4xGm}ff(*25321B&pi5#$m%e6sZY?^hz@_TJG^w5Hbj z0<5M)A`>QVPccwJBN%6FrV=X|tkAL8URt{+UD(phi)TDv_3g7Rnabg$rSXv^#T|U^u^LlPfOyEh zggoO55z~_z`!oNs)Zi5Rww=kojnQkj+JwzL^CB?nPWx+?LUTm^9XLZly^TuwS0$K} zf^IgB(GRiD+)xQCUQ*XbJ98vB805gSJ0UFXP{ql;_}G;4k;UV_eR<@ySoe!WPpZko z<>K(BIa+^|cGbu$QJh?4prChThz#jmD3%xLdT^k7_VAu3b%*OJ-wvNpn0>A`@wX$m`$@h3i=iDR z@8Oxa+x0ywCEkVsr(x@LDLvImYkZCcUKE#GszZ9Y2}k9|MBMO^Mp)UM?`X|LyYA5V zPQL+9-oXnR#}m2D1+AuYpTUl;Nq9G?zu9D6Q&sPPmTxk_y?pl}eeQGZ;(J4NFnfB0 zz`p9PFBsTcd^b(j>6}rSSI$`6U%DsAI&^a%>YHXp=A5kzXU@kv%-e#IoQ1-BjwNJs zIEtJLe(nNj+g ze?K?40-ZT_?P{CkL4%YeUf8d)0VmgbEK+D{6K=gx{Ph4k^6yw_;b#%J3)bcr*WWR$ zXfEi6 z@6vu8o5l~XuzgUII1lePGs^>HSgu8er0lFLSOPKRS>+SPfnOdFPH=k@q|)yB%weX$ z=`eqSJ_8=Bb7W&R9T!>H(~PcAIH3O z=vVKix?I5mSu33+%GK^gqM57Mn;eGE9B)}%y?GzVbDBfvDl-%3VqI(m%;23&>Ie_7 z2Ao37*{v`bR;q&Q4Frj$bf0j-&G&vH-V5*Z9M!fo8f($dNS-!w>yJ7;Oyj5LK_}LGhPdz_X~Ca<;dk9-t_T!e~|w(x0oz}e?at( z5$ENQQv=q!td2K%mnb|Qew$H6RGu%FE*R+%lrv>xteA}&2yX+D0GWgNU$dCwv2t1b zpRPoKu^~ry6|DFXhfzeCqzW|p{%;Gp30=p>X$3-!u`41dPI?jIm7;Xa117B?ZWmt6 zc^0+>*b)w;Ce>+(MNYRX4V-vpT1^-p5f~i~O(_qN_sSP`wEawDwz_TMdI%d8W{B~f zFxD)t`sc&ylzG9bBfN*^H;xV+Zmj~kG&Y`;q)D&~Bv!qrEXLKEukZC^x1*3+ItP6`9?>1sPDOt_ zQje+m^LM0tL~f^KmHwrcXt|=dlp+lOva?I$O>4qe%DsvS_Kp+s_avH~5b~)vVZAvn zcDxZ|a}E$_3*QWGKH}>y@GHnhO?7#@0?}&&~ZktE}I2SIkEtwu8fYA*XsHA^4aAD6#<4=uGGx zWVXo`LL38{BDFO6Gf2B~Llx7q6>q+tgMYqwGc2Sx@2BT>z8$+fGg1D^6+QZf`#;yV zjBS!M6>uOR^o0MvYg;8(8&hF>I~P-r|1z(y(SY+&Ucvdv5%2z zo1Hm7;b^0dy78aw>5+Z32;Pb(I?c`Y?j3vg-+TR;y&ibI?Z)e;!+$a0a6>P^UmHKr z;`UKT+)HMxO>Vkmzk2GAB^YT3<(xeDbb1R6(X$B9}#x6?hS49 z5Prc6q}kstIrKrX)(^!({e29;Cv-U?=0xngum5_Ez$Z-AB1|4S!p%i?24Yw2F1cdj1lvBF$;YUr2_B%T9Um^0VgM*uaj#Sui;Qa>q{OrlXw0 za^g-nv^$zIZ4lTU+3X5gIlAEu6ca1t=oZrVa^oLyh_|80&zpHzQ)M%eIClP>V`!kk zk8&R+ahgjSm{Hl!Bsw@03KX*6!M(7Kbf3RnT$P~fE-b6!%eH}Q74J}5#YKWBvJRSv z__C!3#Kb1rf!e!8U;LR+K2wYPr|~z#!^KCL@!=>_wQtIwrSlIWLa1?f zahYX?ZVbf5qo=6@RY~9HG3DGujRwwOAuaWEd+|IKNX-)(Z5fHB3TCE)VQ8nNoyQTQ z7tXZ7N6Fq<`xQch%xdLk#4FiwZldgA+910S;EHt+GgKYH`_FU*Nt>Cs3&1nggpZjwUcFEwuQX)mu?BFs}y$-5*dJe5fzr@n~yoUOk;9%xWZHwd%ck0uyBr zQP5dwga|RyFWF%jaARCKT1A3)b{-$pQLwlu=6WC!&wrsQjkEO^BH%h|isNP!*lGuX zVy7Lz*Vv$zW6$btTa&*Gy9B;}w{k4jh3P|QbM)0zPz&_V{ux+AD?b_v@n9P24nzT! znvyR=QLZ??*j-&AjGu%MCTOuxAU}8ShSwIubHZ7@EG0b?9GaHK_nQD#9;p9G?~Bd`ZogSaPO*F=uV$|M3KHNcHu1{)A`Y^lMK_ zLrS-i?~-2T(q|Lc;;XBVc_$IG&{vsACs;b$Fo>N+xrF0Fy=qo;ZgzX(=SZ8~-4s;x zJbV=3-T$!>_Gw8piN@ga8)o41z;~5V4w>YYeDrh5wJBjIoos?NAtJt5Vr5|wZW^}q zs!fKMufXYOHJx8*PK5!lF$TrF;3BYg<&882 z$c^d{$U12R=wGG>qe(yrd(cTy1aFa1(HRlQA8-QvqOSpd;bj}d54X-r|5yQPheeBT zVuLB>%8QPOy%Wn*jGwMwdH@dkc#yL~fO(ZpQyXm5=$ z7S}OT?hXIDR{UK4@Tr?>v^J%Yu1hTY@@h!QB`z|92~p!>tLpF|;n z9_lagV+JFo@%rcnvZw`GQQaCi&sXB>5~Wp~>#RZIUGTMuIaExRc&pv*zrBBSPvin8 zNcA&{36kI~sg3gL2DQ_U2z2vhinm*Zh}tAWx>yFbHrinEDZ_2jgTPJ zF5{*g#w@@_yt+1@)GdW?b6RmyE&O=cVt)UI;kn+jQqGoBF+-ftLMuVB6 z^}fnPuxFz8#C{1e@+I`6b>;6~LER6hM9og073h-behpohGru~1Qt*?%&+iF=fk1eB z80D7k{o+1VBKH9i5`1G+4i8pKfu=DH3;?68KQO=aw(g}h=4sP*J=Wf%R3?VGkxwj-h2eQWW3S z84R;Xw1OK$n(lzf9qW#Ji+NcxeGD+z0TvkFTU7DEbF$JHn6-H9f>x zF}j#V#gL{6equvuHss-zZOQ~B^b5*@-~XAgJq+DD#Q}+LuP|jLbIZmQ7f&V{5Gh7J z)vmoyYSFTsoB#A1E$C5+7p;{g@K~dOm&e!z<21f7XlG7oK``&QcA|G@9?>Aio@m?r zR#1^BehT9Ugcm~N^7a@gWNHut_!~+!WUd$-tpqRIiXb~mQ(YR!K~6rIGQ#@98J#wN zSkmmFi5P8N@76xjbQQuQ&2X|0Js|4W)!4AE zLqx${7N(snOxN7Y=9rC^S*WgXV4EhXpu`Yfdtz8zOt&nbsvw&5mPHHVE{>+XR`3ObRkX|@5rkRg;{f!jNRiq62K7Wv_ zz<^i6m{)`WKG}+ZOqh2_$UT+v9W%wkQOGF_;hiMm8#Tl>&ByL!CFeLR!%g*XggH}K zx_(EF^s)^D3L7T0;tNyQ`Va)OEQrz&I~`b;+rs07%3yiJs#H1Yw`E8{9uQkbEAJ5} zyb=rqOtC;nJd$5!zcY2u6Od7u(MUp5VHS!%z^Ga^96Worf8#i;L3Lfyqdg1bv?eS- z`J=|%a5YWKJtD=|o zM00)O^V^3y@jj1x4|FODar2O;d`pEN23{C~Dp#!p3-Y9sTQr>^-EkQDhYNbgI}lK; z-!^Rv^iQ}nYoZ%6=p(OE&X#L6%>&r?l)Fku=_R^7sVa%srg0J1I$3 zt@;-)V;0s%zjA8(;h9m&A|}xV=I~F<4INiR82n+<%yrZ)scP;csWt1W^k-?VWUz7* zRo`g4z0wEOo!a2tn)cS>je%>oBMqNS!IG!+RhQvcuiq?)y^UdhE1CpO2VBmx-+zg` z1qES;LDdjo?SqW|s*vBa&%A1yIL zMzhA3hx9!kgl^s2O1DpRs-zfj@ zg_Y1ja1am^`2W!U|4+*AWbffA>f~ha^nd!Kv;gh@S76m>UT8lB8MIRVuvDrPJy}ab z3o)z&0ynOM>Rh?z3Ee0db69zvGO3Z)Q-1 zV172wO`Kk@_+IB8c}};pzV7$r`#{>mgzx|f3-^}DIkpyq=EU7WHrQ>+A0TXeXCbe3l`wRqZMg>-a0%*UIpn#jP*7)OO&8bBD8 zcWKx}Sn%kP<{B*py!g7F(WTcQEzjU46WaNT+%##r{LTI=g& zLigwD#_qOm6(3{F;E?ucsh4^|XOUrN!>V$DbghHs7K)f?2U|)wu6!ORoZUhLp`(_o z_Lf6bLjYl8(+tM#y!@4jQfdyZ z{;}DeX@&qcX2jfbb5VI-t>EkDZM@N9YK2tK(!xReCm2}JS8wLwY$pk7nqBS0W}g!E zS@uDEfZvHUAwM*3(e9CbbFX1zBr~Rqur*dH7wb!Xet9=7yM}fiYg(nzI@}s29$uco z+MPsK7&e=#O6urLAuH;B=>)Kh?50M{FoLE`)Ek%Wn5|Sz87j{HCVe%F`p^IHC?0J9 zks5Lk8PjB>=5d)4(sjN?Z#F^&eUfa5go6sOjaH&Rf39RdgD8o;k){-xcbJVYLq35l zxjwHvbeYx}%xXM*@Ot01_K=Vw3v%8xz`j@sjQyVLz~t`M@|;qXMsYb*-R(r#9s?t6 zh3*|(L&4TCQ1!CzJ5rL>33|uCV3y>8p zv%1`sS6!}EB3u#R+WF*NSc_4YUBp{A+=0b;o$nIp`}YxdrFM98!R^=%lNKkO^|XU?{BZsix#U)S&!Z}Fbn3o_eyeJr!D+p8;@;Lp{W zVDba~ks@CcNTmmsqPp{le+`|tlFQnxS0-yJ&)p*_-WT_HRW9WnY)3JH^}^{g36<19 zG~XDkhEU|CvKLr;pskajrZ8j=9Sd1B@OJ#2`=^y27M`SdT!V*NcXeLt_b42uZ65c9Rk_TQ|$_H5(SD~aP^DqvprEV#P!YWUDVPUdJ z5Q##^Jcsmvt}WddXh(aY${_Hkt{72KUPlq?7|4EsjuB6FP3^ars5ok)Pel{Rv_e|zx)Ql&wwwGuXgN-)1B=del9XGQe5PoT>T#*0oe&T=B z=aqQ=1BN4T|ARxElBu!1xt-#=}i?{yHbl;`X=kYEFSv>cuN znCrZI+I{=zmpzl`|L+^V-};RUhM>*M0ax0ooj%-@yE|(b_GWzPwfM;om3uoC>Xq#; zdo?VXZj}Z-_lr7^)`42zRa$fDe$vkE(`wb0D_6In@iLrd4Z;i7ZGyQc%o;|ZSaq?{Lr}l=zM-)6J+dLCfbM-In?)6Dm%jPCeyGyNh<|8#*sIPowJXgv9M4q%+jb+h zSS8NDE?XcRkggE?eyVYSk|!_@QVT!H7w zsdaz+tv|~b|0AB62?R$FL)kJ^63N?}8%F2E%gEAmlS?3r1l+be{_v+>>J3xcBI+D+ zs-Lz2;si50+}MWJVMLD{Fz>l#FCbdKvn$;uPakf1P zA(uNaK^^Rlq@~0f=E&#IhB)QQP1&$dsU7x93*tGsN{Pgf`kVfI-z9N{D=S7 z$+r1nr;W!hIiQT9q6AAULX}O;MjYz5r31VKUfOMJ(GL5Uds|un%83^|cV~52NIq+>*RIVdsG8wU3f7z1c(c>Yw8FM*p@m85&w&p~{Z7R$HCHwq^}$73gmxJ}g5hLI z{s~H@!+>P+_>_WKiuX){7HW+SV%Jx4x>Wa_o%?yGXYK_h0A(ouG(7->f|?@K;dQIOE{LQg%ss@Jsj8=hzy4teHZ1BV67EQy zY3@;|t5YpBb}TUpJN|cIv5ofv1ooMGo*Dnhi_C%@?+pdBeZ2pWSsnM zpX$d^wbpcE!U{K z=Gl=4*;8}!#p0Cg0nru#tPgws&&cH9_eeL(QAYQL`22(XKd0i7Hgrl50R%+p_y3S~ z@&8Q4f7$VB!1$oA5Pa{NaW-whuL%VV2f&Fqq5TmhCx!+k;|NI^R_MnJpCQxSAf3G- zDOas(-4wR2vbDW7Y^mWZhrsx?D7AvQyz73sqE(l>X;p=(_|cOwadl;eRd~?$So`z2 zw|2D9cGE@FFXZS3LVF((Bo21BFHWuzNHjGk|Lg0EEgMa1hY@at2`Nu`VKQLUqPbcL^=vl zuM(p^90CGVrZp-gp@8}G$2NYv^;e;;fsQswu8ifqN*$)`rRdlj zB@=b(dgYk68~nL-a{&iEZ#kUIS-ZkkDSL@>q6M0J7KBtMDVQUDDu)Lk-FI^_kqI(0 zg58SEmZxSyVoW(PeRnwv+FE;2ab|rX8_u4gNXsX`g2zBt?VHqD z3{crN_W5*;bHZ_wzoi1b$3u+0jvq^=40slwk##vZ-{+Xia&Co%?yGJpGv1LJlE5b5 z0HAd>atm3FCj4>z_ST}=ZLThB;z2DsQGy9}r+9HF+UYhs`*M=rk)bVpnsu>>wu44* z6FE#&p{3VV-lE;Nm43?Lh#9s`AheZE;Tb^8B-^aQLFs7a?5P-l7R^b`%X9VOVn&YF zF-pqG^vxqQKC<9nI0ok45H!$7S~%CzDN!FmKB>e%4am_`{A;N$<%pNG%)|iLhf;gw z{dosOOW2d)5F67Whl^v$nCi7w<|~5)e#MkinRDs~E`1Uc=}BJ3z$0W_S@8 zt|8(8sBe!)r7hWn-kZzG$l3G63E3?#$WU7iL_&aUPbPrO!s;y@AB=igO!es9j45T9 z9cudSRCO8cOii+J5faw2^y&!eU*;!KoNj;7Y(nEjm>we%7k>WonyEQ-m{N+5EE0z7K?R@^`}W?)F(eW3|7xDk<8lV)Z�zbqg?l*T{XKl~YosN^E$8mX@oWdPw|M@_#qMGwl@xuU1V14}$^)b1mOOD#QfCzt zGa;dlZLgE7oj8^rl+c{vZPQx&AX{Jjg??r7TiE6JrJkRRa>nzSvHa$vEiicd}lEto0HsQ=5I(6@~#BE#AndAa9Ir-9( zngfBbNQ;x9UtD!QQcCd)kLh1kaPQd8aN5}^y?iU%yLw%T)JD!KtJMxxUHw;R`i+mt z8MS9WTI&yF&c|t?!-WFwbWh}-{$XB!^wvI~JT%PE;2zV-6Xl2>)2ctrGR$8dDRU6) zV<85qTHUMB*-|J%3apu7^(9u|E#O8av1+xQVe8X6$j$*6ju<0S<58|s#8!4psMR=d zI?_3sQb`>mu$sitVSnq~-~*B(Du#d#%QvNi^lD3u>#6Xob{z^xWtmNA2Zzij%c~0! zXIMLt%_ys;B?So?K+D_^)doaTj^9msHRMnoWrss=5hQX-qU4(!$mK?1=@n7Qq7oQQ!#1kZD6ANY zB~i-5Iv8NkOUbN_Iau7IDH!>K5GygSB1G$C=FD#vO zihLZNwN^fpim!z6nqGDAv?Y1Tnh?I8)a<(rA7YsD}e*dCI z*`b&t0%!jK(jfww)1U}x5tNyk#<{p)78s}zt5co{P^1(zka6wFqg6&Bq=eS=UFQXt1^9{8!5zJNh)sNNPI*r1OUVRh%4ukC;jPlcWwV4<3^ zx*zm8yuE{L?@DdjT!b9Y+}ia~No2e{wo*>CxpVJs%AwD*4vh;=jbs`fpYmZ*GcwjW zdcUfCiWpZ}L!K?gZ_+bDnY&fbr=PO}V9OjsFCOrui1Ov!HNAHRX;Yv}1ZPa`Ulw{n zkw_D%yx62ljS$p+u?LzMPSBPt0;(>rm-oGet=2+kJvIzQHPM3PELi5->WpajD{1A( zwJPD^<8t|p0Edxlb@NHi^OxXxorv?r=2P zV&-3g@ZfMzu7TNQnIU?$yDJ9Xzu-+HnUg#aNe>`*q);!mg*_QC55v~Evh+|J!;^0U zM))=pC3ClV3@Aby2Sa>c;bQtgot_W*=hmPn*phrxYdA2Pm#~ zy!oDNNkxATZ&Iw*90~ZGKN0WrJ}IoNx?y4v__i5f?sG+;yN`E(&vWU1}{!V`sbEGm8E9_hL0Tmg>PF9nlg z5K2c1?_E))PXLg?gTd)wPIIvQ z0Aw0lv=T{;>sqc-H)==M*G6)rFm@z!Y669Px1}U5nG$zC7RHhS-$9P{sIqmUf?9$# z+mqRW%#D~tW=;)0ae*5QlgcFov|*PGE5= zz8XtXJLVrD+(DKg1kxMA@;*aTv|5P~UyHPaA`{2G$9kX}d}R2tZvrupSeCO+kA45C z;9%VujB0ZXs;kWtX-ww(uhyA2(Y&WsP<4-vY)|%`eYW!|HqDrkLC%Z5Q>>u*{lZk7 zTj-3zC|uq2iS?G< zUUbU7|7&MzBSHNOdabDwxF%w5c-ckb&PHb`a3dmJpO&1NV>O~JvNYKamnb5H5dtye zVo^gJ?+-e|1)os-H_-=LX#{-1sa%nDclLrWNG}?>K{15FP+ASC*j*gl)&Q^64Zo!$6 z+B^xl5AT*vhT|s)=fk->4r6cNN~a0Z*>VPK*{0ZT$&Nh39(O#PKD}r-lwzN#fEfDJ z+SwR-CdrN3ILARzUq4S&v+2+knXygO2!FtM(Mhd zo(YrNq=>XjXjwsMS@cWyUIWVUS}j^E%bx(mL2VeZRI_bNDa)PA>K2}|o(!C(;Vn_B zmIP0|VRma*wRH_?Uf&{|RMlzxp*q}){E;9Xydd6!K)XFHwxxScNm!2#ZYTSsmi)>6 z(xrf`=cX*fE2aXOsjdVRo3w}e2+2l@s(tmulsS$Ry>`mYOZm3p!bHTc&K0BmZR1yx zC)eZLoC9QGj*}qbPgjigc`&*8)qF^O#o#@1gnmG z;0{VXY~kVCL^gLo=SrE<_ZuCjcl;~%;eg66Vu{ypK!O3Gsn_q7>3q)qhJF$d6x_at z(%LppxdUk`7i_HYVKFbb81Xu=vxH<8oPbCtX()q&zOR2nm%rz%+ih_c$fyu^X^QRDk8G+(ZKg%S1$sh@9yPjvh>n&X0hl^-?PEU<$%f|bK zaGGr|b$_~WT3x&h^msofVPxDid{=P`=F5KEfql+T5!G|tPYtH`%-gj9{GJLq zzC{Cg`|;^M?2>vdMx@-bem=Jat38*gv33k?Z{O$>>mC=dcRZ7I9S&wZH{lGwj6u_V z;Q`YBJ2s5Le>bfQR$&KOc>X`s)P7D+%XF`5MmZ$!$b+rR;D?U}IIvDj9|4|3?3_fWzL zM7$WMK>6@F>#}00^lsw3hs}%1$&sLkA^Rl) zrVsNi@!1sNe$n=EsClxB{fZVIp6nOIC^L0 zf=en;sOO8570urSPsxNDZdoGpY%~(3LQdxsGfhdbM?edXijX*J9~W|bEBxJQ$b%*o zC>Dw*0|`DyCCG)G4HaUz`)ppiEgkP95s&3;&S5hknlBN&mQT!Gr)85E2G>@&j7JbA zYg&@U$b9aVUVK+oz zq|@lG#O%)+&$4)WYb8ab8yCQBICRi={~X8op=Hc!LRNZEg?-&jfIeuXxltuK5uB!Rf@?*qr-kkX>ym{ z7jBo-0WWnH4|4TKT-sI3t|{Y^&CR7^gz?Z80Z1I^=?w1B91W7r`Yt{k;~+ZhV{HP` zZxi8+isxA?*DvRMF+hjMH#qGqTqD(A8C}m&va5p^VxP@NCvP-6~y(*4s2w#=%f&%sB+@%-ee zG#c+@VKzwi?6zn#@J_y-F)x+|ACYd2oQ~^H$erzvh3U38NN&%zUw2fRb5R9(ferh# zzr1*`OPcVkX=JzeBD1?$7@no*^w*egT|UEeSh7)OUy3_>)GaT7 zRwwrf#n%Gfmms1xd!z)JhPK+#CP_6K^K)skekDqL35D7+PF4goZeUpp&hou~y_@eM zDq}vAL?B0u?Yg-#UFGcp&LNrd`fif}@5khdut@I1Nd?bi zORCME z#?H<1vB6T0#B>2Nvnbb|dRSnOZc0+hiq}9IGvnR{=RCS~(rb1uJD=XA`O_1do{0Os zJBLRMBb(FywTqc~KOrj6FCtq4SvF1)#kNo(7+R4lt)M}ZO2z!+o1#db04aVUD@0YF z{vnrgab8vCpVA`FD#ZBC_bSzqNk!FL49Tuk;xhGkZko~9MCW*{Xu;hSr@*Xado^fF zy;>D1JSoX+5_*U0=4WPm{E&(MV7 ze#qC#!^8Dc0#1AXFZvSktiqTb)S?%yL_mr7^tx)MngOkAgfLN7yd2sG?-$3PAH?`^ zOdK7{yR$Qsz%cqlLh5ZA<|Jka)2Ws8moP&KvAhFLtx^03$x&LvtfeV*s4jYNtGbM( z@BK66D519Bk@JJL-V^&^3YsQBp$$&-eiZ&9JGiPQ@;c^cA)Lh?G)lY&b>`V+ zUWX3j3krG0TWOt&VMmAzql{{UIZn#v$uLzT8JcnIr$kgL6Uan%Wbl3!H;MwKd2{2h z{w&bCeKuZm<*!zYre(U5rxw2nhu_g$o-=Z5Ibn@v&0~}A;s`bQu@t=F#KfDT6>llx zlp#hdzroKTu68n9Ho~M-bX{T_67-76VMj?&@AZWXFTt>h+Ua%{n;f3sH<$O8FLo>} zmaE;OV%;EQL%l;*SS`X?#uQMiS18tx>QDM%HQWrgp_?_{gP&Do`Ef2VXNsESu5pGm z!g%jDPxt6tb_8;3>hx-=p(lsGt1KX!wqnAn+tpZRHLjOWZE&DhYEEPL*DaWSEc>B7 zynoktm3zLSr^@*%&VMCnbJs3-8~Rx= zR%*9>hQe8ug+F3#vC_uJ@^=;?cj*qxze?#1?yEkH+q)9$^BjC6?tWcR$f!%917at1 z3?a*66kCh4B#Ih-&+ZU`9ZAcn2l~SMWXNe3U8|i>6JP>hPY7r_eOFv zmXDF;o_VfSXT`w*hGH-nJ5z;FZW&8;! zgCc?oBFfL7UTdHQ6xWnz(rAw!hJ%c8kqw-2&g*5uj4z!_x7`hPpd_?qO77f zm>~&cjtGCiCMMl0smp7GJMhOa&i-^y|2U+4y0i$sv$zG+$~dvBV>8TVR%OdzZM$lF z@)e(E9!~#zkEM=1iDG5wUX#e41KbDX*PVR3za{jI~dIbr5T8+4jHtTkR3afsUi4F$TR zbhn5^cZeEVTJkWF2RK>~J5Z6Dyx;;~&1g>TB$c$c2P#zFU|^f^(srtRkG8ui>k^;m zIppGHz-2gC$<8S>oOrpC-;-+`qK3JE^((57mvns>c5;a7DS2{{E2UR^ys|nqoAyP# zG(7#1*;o>&HmNkxma>gM?I|O+0-vo1bIpWtR7oi<>C-VLLZY*qZP$(<;!Z40of&uG9hx0&1lGpfy z7oAls*bBjnOQL-8uz$43#==cg)8CNH#GJV00m2^67?#nW-%82|1XN%Oh-o)R3^oI~ z9=^%`4s$Cf{={oomjpU$;ZOslqBrJyOIrYN$;zj#<1tDGu3~@&2!1whNp$*FBLKFDJL6vZ-?Vee{S4!%RzhFPO;r5>Y>;eBQ zJpFV5+2nEI&!0c(XP0($bf7Z|EX8y;V~|V~Y!;Z)4g`Kb+vX4%`7<+(DU+t)c?fg( z?`v(B=vt!qzwu)N6a?hw|IW)O+gX|Xv;4*-eq6SXA2oQ|S5t(J%I(|r9w-|dhsP)Gk7+(L*|(R+&u<{d@+v#(ekhr zE+MjlKdW50^Q=U3XO_4o{kEDlQ)K#QoIzit)@%#Dnrj|$NTkrCnM3Oo-OBrgb2?!_ zi!^n9*Xda=*0&a<6WO=U#|Aeu_8VEbVAo$c9>{oE@~#O?=d|=aavnPHqAYWo;p?(m zKjmO`dFxj*X7nKh9#{{2svCX|+rG~4;P=ZqQ1RI-YbRrr(qmXM9QNTcvhG?Acqhh` zq@po+3g^D+>2Ww%_ccl5&Ny`1Us2rAY<%+@`wBr;e_L02x(eHjAbkhM_C-<22eZ)p zqz!%xA~h!M9Uq~Fp88o$#u1U3aXvl$0t>AhFahf4vI4?SD@ZvfQ>LZZTy!N)D^(@F z8PqBMwEOcF4YA;&!0un=nR82|^Zb{h1^o|I4gZj*sDY8Qoum8z6f5_m-7)|Y5|S7a z;0lTF3aKj!*>sSB>vlJIe}PN&EDF>^5U($^L?K$%N$6G~a%q@rc& zX(nb`M(G4Oi5eo!D3pb@F8%e@*vQ&x8HqVy&PjQY=px`=M1S*&K51ASsQEFN98Onp z(`Ry%BC-^aMn?gj@rQR=qgBPFs?8&rKSevf_76KWdO?G{(waS9I>KVx`gqfb&-CwnJu%2; z^(q497IgH?_gfjG2lwY6U_G3FcprE#7+D(K8lPV3XW+gIaEwJowlt-MnPS9IHmIN= zwZ8iHCiF!oqQhAUU@|{YcKPfb(dAH8IO3g`ujP4F>cM2F6F&#;n#Vbn0mY5C$u#0I z$k5om1lf2Mqh2D#u$GyK(&ysMKb4hH5Zx`E?uYD=Y0^BWumcev)`TS{Z45?uwFR8r zJr13_-nZ}EPJuoiOyCtaV{C9$XO}`rv(7qd)l6oG$e0KiZ^}H1A64gn9B35R_4P~h zcyzP6;9Y#}@!&(YaM@>^SNe+kX&WS#302?$gF4nK!-vH(x~!nr>(;c&z@cTnvhE-uDfK(t!~q{JGNAgGoo|!sO?G{Ax|- z1wtU%F$umQU7zZ?GxdS6;q=?5yX)J=zMvtM1DOLIgSZ;#@~C05DxKPz$G-A?uC|T0 zLcFF5f$=eDe`ZGy^m0>urDaN7kIme05M}XUasfOy59#eo-SfmkpNC@<+r38HQk1Ib zx;AYmwe=KPibTUlOaE;Z+k<>ovV4JM?-NYOqsU@JA=H&X!FGgsq38 z<#Fn3v$waN(vZ832qgj)gV8qtVK*UxYBad$3YhwEmgGgVhp$9QNEW_}#{`L9@;OM! z?|%_ zU5u!zcY*}sq?(;PW^+ZIat0gss;Fd0RTp1uLTIrT9bZ6sZf>I2oly4v!VfbWyA707 zQX7hkZl0*IblFa(x4zY}U2dsdvnrxh_%jyxmFb;+%uXQ%kVO97a@T#@76hc#E?x3qqQ_In zrZMz?BWf#Re>foS1^3nr030_%?c){S;A45+lx4lSEZA9cYWI*Jg}r6b2%ScJZWfm+ z_k)}Bpijly%=v&_kpHWCgXEqx!v0;<7D0l5aQt`FThYYc+Q8_4X?BvbmYkBnkIyFB zPKQ`cDhP<)@U@7B&{BSC^lBk@?|fuK ztMhB6hf670t7$K$N9$Re*WCagNQOvv46Ywe{UL;Tn?J^Da9lT>^@T^YWH-?7qG1a# zQLb)P2jcAafG8+^PSat9C<(ICoUxAj>AfoP%TW!QI~i~^llttsl;>bVaJ!Mk7)EuL zi4QA7;?mWn!F5AM5o=Ux7;stRR`>6XKuqlz6so+^aK>H+EFNB(tf|To9LGwMia~X@ z=DvGTo`wN^DGP|HnQR4RfSSg*gSWy|(#YPVxqE8^$$9w7awOXBQsbP-rxp82%cZC1*0K6`Y6K1QQ4Q| zd6qN}hZ%V)PpJd0zYhj|J1{<1U{l%X964#&M$+@7FDWC`^M`9xO8lt zSOo3YzlrrV0xn~t-t-Ppa-yU0*mUpR!ca$f?Fh#wwuqDg-aZoy%kGIvZm(gBg@#F@ z3v#+}2|iTNR|Yb_;m4%%)Ts+3#N0RN=IX?~(~@xa#p%@87nZCyAxgFU!dS9F*%GQq z@%R0O`iyV0lK;WxF^bG~iPsV;{u zisXn$J1wqhL%*G_6W(Vwfy!;KkAZ)YAyNVH{+RCrj+8&JOT~^@#`4-I$tT^`gQKvE zqxijsZ#$&_d?O%2+WNaKauvZ(4SsmH_0L+Z^FPeS+d^+k1RW(3@2llHu@4?rt)E8j z{4{*C{hx+)iRCuS>M*vn`h24{y}9o63T=iC%;;RZ3J-Bt#O?$a{}z8urZgY5{~FlO z?;s$2|2+fy$Ag@`iKD^)TbG}uqM?NH*Urekw}({>{6J#yH(ZE?y6pi23(Y@~B!LNa zYlk+{mSHQKgmhOK-yg(?;oGT4)P%@ep!)60H&$n+!3hebS(bi!n*G_ zb0cAy=wK9*>?Qhs2=TLzC2Ndodd$j4NdP8$lp(7D^qw>T8%A)YmtKy~@B|5M{i7EMqvB{mbvD$>?i^Jr?;HrX$u)aFoWIdF@uN_lUHL}+*nNdoww zo*8-~Dn+tYKZBgrXm_$g_e5re@fq9!LUf<|j`iE2jaXHw4b<+ezNGE6b{4 zef>NLsGo1D)Uhb6roif)1r%M2MRm-eS+Aj!{a|0c{cpmg2!21>rWZ;tH_ z>jVGJ6~vWzsKYuxM2?5@N)3X= zQ59~vtvK=N25AS;KjMAjIIW;6m#;{-P4?0@$g8snkrtllA=&4JSM~06Y!M$o##caj z1M@!6nYI*ew@~sBrOf3xG{znpPlWyJDni1AT)$~^SQp^-0@gl^*F}GuA=PaEtKqVS zHNMuPf`9;-KtSmJ8;1M;L4H_Dv z)j|`b{uIWdczh%hWU^3WuHw+L}tU+NO0d8LVaP*>~o?VqdWmZ8Db# zZq8X1d*s1aRfyx|Y~-t|Fd)YMV&2%(^jx|qW<%EaW-bHLtdOLBa0pUWuPvzR+=Y? zBvG&XF4%;<+1Xm79{4fjGOpR&Kx?HVL1DFuzlmidDV)G7bIyF~R%iCjU?N<(jt@n~ zluZ}~jIpofB3oRm&}jU1q^cv12+~uH-N4$a&_+t^?JUWL$NOVU%NR2-U)oZPW^_Wv zN}|X!9g`-`c8#-yk~el@z|%+uk7*LsXjZrQ_u`nwJnYE~T{p3#pyJ^nPg1KU^@~_qpVcgrvy(RdBKi`!a~g+~ z*@Gm0V!*@D0$!tmAv@9x9mRA(SDV`f2Pzw_>?-wyEIYCh{=s!<4sUi!So$L6ES!u@ zc;4Z~6_Pj2#=*4OXCa?iij=ojdJ4|K!)CLTCPkJMIcpY(C*8zG)R8>2>OKdilP_3B zB`9)fzj)F(7`?fk-#O8Y43rANjI~fcfNgE3r#3(5TUfbxK4+$Z&QRQU0zb&RQYT6d ztEzn=!EEgNRpZOAncV^i8*gHQ?o+|2vjy5U{IZR4F<}8Ufl|R(BRh(^96KP;`OeRR zM;Un@wO*{Q;CnB1T-OvU088Jv$JrZZlezcoOc;()pU{%YJ2Yije3?5^(!j<93YJ}U zii#s^qUd$gfX=nzL}#5Yr7EsN7!=tvlPtyBOX{R#%uTD6)kp-UL`MOvRKE#FBWfo{ zMb$Ia{POFA*%FRcq74GA1Ej|xMU%zE(Eo^N&D9_HL_(sJg<=qRdXS?B2k_{-$8%+i zh}FJ23N3MrlH8gT`5fyX@J&sCaQjYDPJs%BHQdUW5+wkKQZjqp5`HiTn2^};jw3FaxXxf z-=!HNMYe=`Y@TS0S5pq=Zr*-gQZ`ZLL}-b)uMJOjIa?od^=3O!8gds^G9)KY7H5%i z6l!75=+0d4h#5=NUYVpIy}6Ii)T8F2V)=K$$D*m z)xXu&{}N%o^mcu3X>xV*r;K%k!Q&6V(7;F>uh-Do+F`V+ul1(xw4CG%o!D{feqzazXUYX4!_ZOn%+|SWmnf2mi33-`1 zl#oyTFp$*6bX`HLl!Gs9H)J0uTFF-T9kE_DH(VIH+*-L&(VCfc^fbGQM2j%$%er`Z?~Sl z0Jj*H82g%r6IZW=J^&BVq$kw9o&kftX=soKWC+a}>Im;iV`)SiEl)Uk;Y+SWWvm6K z7=2vmqcF<&94}aBt+DAL5U8HxeVxr9sA}NN)oA4;lWu0jdP_oH@Ak_nN<^znEZrt^ z$aLDY8F+d8&=oqZkR{@Pi($jd-|{+FkPzCm+1I^8JjY|POX)>p)v$qSIE#RuiM(L7 zOM#G<*gzrD@qr02+BK_0eAiCA!GyDa3U zi6+41y=pjVeBhc2LGR4AnA3THguO-w;1BUd0I!lK?HX>ZI`n5VPA0+C2ZEWyI!|bw zV?^wVJf%G{M4sl>Inzjuel6G^l`)>;p>L9(!Eyd6-JP9U;IGTL&ZzxNUs%F!GL*42 zqsa>Wv$%mYZCw#3jVcHKu&nNe&v)o#Xhy#}`l4AKMjqnZ;`xltS9+av>KyKEAonh(2WiP+whIk$eBiUQ-E~O2 z-2sx$ec0?>;iF$lE$?K{(&1J_*Zfs==k^m4XECWK@~Qgba{OW`g9ij@CY)B8DtgNu6Yp;JHJ>XP3Z0QLn}(zxieDwW=OmqO+*r+bKPPaeeu5trQaz+ z&~nDZ<*0tjq+2n4w;`Cnxw)2}<$va&37R&L*4md)K8X`8IOC9nTfs)s#Fwd%zJBGD zyn(&?LT2VLf}QlhnJnT&v72p5%9dB_Fe19@)zS!W=5$;me!3f3XH9X2WR~s-FuUsi zK*(`&FUd`%c|?lDqk6E@aHl__m~uuNX7vNB9Z<;BAYg+zxkQA!9mefU1Hu&(gE<7f z#)37aA0~c@?3wxW!HdarpiNWUr?UJrlBs_sWSPMEOb)@617&Ds_Fu+mMC*9+vsd+KdeIw#_YZWWr`J7KEC|4!k%$VNw}z8P+2U8Tv`|~+X`kD=eBIw%vy^-qunfKm z+rW3`%04~x=>Ic8w9LF8ZH9h|d*D~zA$xur5KY^3PX{2N_=Q+wE0hfiaDmUsc}bt< z{DYq|?f7@t5EOYD5(qt61H!H!R;|0G-9|-8WS@5P2GTi+Xh$Tp`ibLv@MutX%3*qF zO#^?Od(;Dcu&8Dlf^tu|S^l ztob6ZjePpRj-4*yR)uI!2_YPlQkVw4Q=`~hwK}56F^>&zraR$XCv6s`do40*bE@kc zomnss%_ZYn7SS~-s>u~S2eJi;ZM{6J-8^yZsT?gf5aDMWPyj7$+_2qUXk6;NbZGS0 zhl{E>QK@yplW>97FEl!x<(X%TbBk8d11Lovjdp}$&)22n8ac!-!cWDsqSU}%XWyB7K5^THt|Ak|aHqxeYKb=t8V1Fh^()k``Mo-w;awsKF5+TP&H zaIG=&zoVH|T1*}pz2*cWIG)OXrYvGWv@?@~g6jzQ+xmIKk>joeaV}@te=GD9ZQ1jJ z$#IPZ7!jB8Mk-HmA06pAU?Uhz@gO!k9lHl?;ZEhuM5lf1Nld@V8&BNdWR2{V^2M}l zP{pBHH@?8gaktjCmS!tGG)|h%Fi&GVKy=0!iOn1n;q}-MGM~&(0XbWMtJerDPLPR7 z0$uKO^5zJE?l9{PDZcfk2ieh~hUs0W5^iP((Lbw7d!-xV1!?gY#&@z1`52b|pe{xhNn zsGh-5w+zA1(}|IrA3&D$0opRxtTYV4D&cB(><{5aE^}i}fMo#C4NMzveU;m{+YkU4 z7knNp)tGmh4ujr6A>YX$o1D|w()TIZ(}+gFKk`8SjPlgom`eFGy$^dK%li!TiHC)d z#C=1GC+GD_qSAut$lj=yb)e!nm7$;2?*%v(E=J=Bxh0cLquJKzSR25Q@fsa}VsU|7 zB>TKipXcJ|4WVB*$UWiwG>l={vY;u4IoCY#6-=dHV>cf*x8e!P4|u2;GE-=l^Wbwl zJ}b;5i1}G^h0|Jiyk~)?)gn$sWwYBXDW}D;VjVmo6dK4AVn6Rw2{*oM$rX)`z(N?a z-@vru0d8}lT37Qy!tM8dHzVLSzO>S~VbDMz{PA-GrPP&g2@pNUa}jXN+-I0&s^COM zlfD<USIaLs3p^Y^&uynTixA5eC)h+TM1D=$rT z5wTPWZQ|Ja06*{2aoTjdQ6Tj?32#oP5X?cqu}{6&Y;V08B%WU}q7!%o9$_h-K&w-4 z1V>-IfAo*l7;3?6kJBcu;_gQV&5)Jrd5k)OtFihVU(2FPj8RBKo zz{)r^L2&v#NcV~Zko-MN33%Nuynk*Y;;BZBV=|AwQ>>!Oueno-peoj7P2MUTnguJ5 z#8=us#Zr;gQ$l;-Wfc$zavP2qt1efOmr-T43{M}tRIx1AECk9|EI2O~n3hJCt2~u@ zJl0ssU#r$Hep@WeR#7Xjz8T9YtyqLT!+y+9w@S6lo-GBk0+knscp7`uE77sp%S47C z8&_5;lQAh97gnm0ah~p;11yUWI?Y-l*CVrVPrAkoY|-#%44OhUTYO*AJANtBhwN25 z(enr1KSOqn=H0QgC{#0R$aOCguIbZyC-tARYoth>z&qTsz= z=B9rPDOS!J1q*Ljx9wyXLNcKjfNk&Z*kQ=s%@{bnd#CIZFaTxF-{`6cMgl^IT9yYn zP7{6U4PLP*=)lMw$Zm4_QBn8~gnPqwc=S%a$$UsGgv_ALHU?ty9+u1QGkZlf4OgV3 z-{o#CCJqW6Rp+87vU`!UVX>sr&QRl4#&dz+=*at_b6Un1d1G3cSzzf767&U`HtGTLKT;Mwe8TN zB^ZVm4$^L|t4qaOi$VeMm{u^#b=QA!U0yqc=HSB-b7<6_tF`*YQbaHkyFmnC}n6mQUH>ws+7S4A6 z1=^q@<+QGh$*Y$99=cg5lg{tUuLs$!IUk$M=DJxpWHHZWDkE|kMgQDcGA>mw1LWF z79{Qbp{k6X^?5Sob-U=|7V|o&)M2v9^qEY%feB(toob0i#+&F;6C81S+k6vkrzzij zRg*%I$+~rh>-ZS-1iYZn@0ozkH@=&@MN1&R{H6RB3Na((0C%QTvffFEM^hgA`DzI_ zePnu8g28PZ{|1IXzwvk_yGy&#t@Ic?b#1PPg1>!1_yJOV?6Aj(0!B_z$ zbctlHc=n3LTBAn6mh9_1vgS1LC7l59CvdYhZFA{^p>lfQ3bA33R(@-@p2AMoOSi=ZS$;suCvKm=C;z9{>ek& zt+tuFM^JYv=4chE_7i2NZgXUtElwwxbvw!@IW4gOF@9=~@ci=>3X)80BwxzCp=b4? z#RXTjWst7ycut&TwUMa!OSr-^7rGvHjJ@?34^3HUNcV(Vid-MMXkqX0I1h_2S(7GS zg=uA)V?1pp8_-s~95X*ONIu#CED6)rkTI}?k*EKdabwUCE@LI!0J<3*{p(E_m4go# zasGhsr(XpilN0YjbCTodpYZC{<%}=lx6`pzR^!w2;B|pPbi7P@3^C0XvqKK?N^(iDK`AY1l4jAlAAvj61M~G~jQ%jmjqa0+^|;a|M*(6`Ht)INYG3b3 zaiA(<`eUQAeg>N*p=iMk;ZL`|5)_0{tQ&no8wH)A!?HhOD2QZ6uNB&cN(^Xb_(sg6 zoCi-Vyk{k)v}BpR>7r(cBEWJqpN zWj_GOdwtu&&hrUBZR1}zk$HBr_LLJ(v8KK-ykC}i9I*4|z+RRi?{e}kxon>apO2LN zFEIUM(rBZD=mhfC6T&e??H*YR9rFp3zD>x5j;75GddbH4Hkbu7{&r@dZmc8=ftdjI2TT&ZPKH?BQ%`0}VBhkJQA^~v=q zm9KT{3*%pXOc0HUf52a=;_pTIAAQU}02^iPjQ?2+<`TgO)5rfqioe)7^2 zv0|xmdiu8z|2h-rGI5FeZy+F-VE+Mn@?U2nVPkLo&)SIuEjerf)Zs6%M3Dg;)rLiM zdPJC1AYm6pU@D7^p?^ad&Q^$vf5A8%79?_a5+LnQFXB!JQ#M{H1ii4X>E|(1H>Vlh z6XNe{5Y*ZTOZB-q;@#LDU9mVG& z?JUASyc{TSPAaDnn$3$p#&+Rjcup;M9IL02;4b}`i>FOO2%orAs0l9!Bq7R0XZBUs zsTsXyc6H&b_NnsXThGIrOgTIXjKRJpRm3<7&+3R>awS{OAg&$O_kHF+7h8S+$&we( z#jsq-4H%6@J-aNw&f7V@m1x^%Dh8P=mDHU&ZK{BCP;!UROI1nFo%?1K#1XndrCoPG&aAP!~C zDs5RIe!~f!yUryS*CQ2V#)FQ6uA?#*Gmk!oZPsTe1Be*SxxqFVrm)1Gni*hI_LO2x zAO`^8jwsRVEWuRsW4{8#!IujQg3(KO^zA|u-H1qQd|3uDBub=pPhP|=N0k!@8?6_{ zi2qgk%DvvZxPPUu0Q!H@bjJHQLjf#;ZkT}_ALk4!4Na!K#0?${GSH$5b$GDg%B>LGTTyuc3>5>m^L ztu))(O!Q84(qrNVu^$W1mA5XY3y@AsK+#gJN|t$(o3Jhxs>d#9SW2BCQq^^Ki>sfY ze>d?csw`mj&&c_Ynd$#`CjNVUSFdP3X<$BhA)noz9}EL1CcddFVRpxJJK?2rdtHN2 zW(O4^AtQx9jIce>9X07PyCJ4BnnS@!p37u{9&WU>Bww+aa!oR9J% z{jh8iwAdyY*WUBkcaf(pEoujE20IKAV^sc<;yX(<>y)-x{QU=)FZe#EK*h34{!ULC zw<}Iie(1XTNLxQDKp+c;n1f@2wlK_;_6fD-yu{d}KLA*gnnAi_6Ejj#_5U6kI(ykG+(I>al)V?BJDzietD=C*%4g~q0Y(^ zBj&Isq9qK54Li-jq)QI&;L$V4Ylm?K=XLk#cH-a#zfJqryF%R+j4o;wt=0P0g0;Cb z8?`xy#7%_ccz#u88jpPGcKJ@Za(L-{OU~wUI`p7JaDn$K4V*7bw5 z^0ttU&7HZW;~5NM-f3 zg@!L9xKEwYp$>tmn~fI)I_HJj^S`2oY4$}Mo^?gd!QQ|YNyM4z9wYik|`6_%6&tr`I4K1AlW zdrfWRsVB+OADK%#O`b_7lz{=i^wqotR6Zls(VEb85qgXBfVh}0`he?3+-VzxI~&8t zw6}^lyN@J1VVKzF*fYJ7)^l(wyazvzv6JXOyF+H)%x^vCzWks9U2Iq5%6+4JmW03? zaTV=i=Yld*4>XOPd}XRQ7Da#dbdNG!CryU~rVXpdkK5uNLITe0q9VnT2Y0zYfF403 z`#j}kCJ|x2dM_D*?lW<{S|9BGJ2u^>8Pt*gyuatb|3XCfcWnM`DG?R7DGf{yA0%U^ z2L^T<;sS(ZrQZRMzlGjOU~ahY6=Y0JzG--J)$g_oaw9u%RgRTkYn@Q+kj{QDWt?2` z1P6q6^33*V*C#8rz^@kU?Sd>f5>%J&YClTOqC835LPYF?*cj^<0p2+)N#t*#$N%}v zk$}XhTNy&&a&2}U4D~hogMq_*RCUn z6|(B_0Qq+d7k7JK`Tmt`2JtV7zW&|9DteX{|ILwAN?HmED#%)-zjlLdA>%P;y;XXS z<|}^YfraaX7Ddg9_n3LL4RE?1#yirWiEqD=zOJS4FlP0Nan9$}e2{(y@swKR1e!YW z_e(M+culEYcrC5HUEq9rf4=1ak=*?7V=-DG?IeVTYR}7z+3xNe4*Qd z>%Lx%172;h?ck5oGPVgaylo%A9@OPla}@*)H7n_?)l0^V`wL3IsOC!9xrKEj_ssHH zQ2tPO@^(3^C&2-Wsc(1-HU-Z$qffmU?Wn#5sm;m$X^)FEJ#qLtoPWb6gaTP+aNtH( zuY;8T_3pC{$@BX9)jCKCak;H;$NXfW`HF4(JpG0b<#p5j1bnYV_G3GIZZ|1i&|)L2hWgJwoVa-R&K3 z*1ncAT|E?~UlvmSW=p+(qmNv)s8f1^yOGF4k6Ew9UM~A#8U_xBGPfg8que&b(40a0 z0@(}1IJ_Ts0UVF5k%m0v55LV@@>w@)A6D-1k}x0U6=)NjBE#d8EMo%V4Hqm14;KvE z0zcbP65P3xA2)V}l?9w7b}xC7k*hPU^BtGw-f{Rn`*bFme3DRTMgr{7_OrdD-)Ll5 zRESNl$QQn`B6NtXm-Nzi$XGOC-}u3*%*9mc_JMQzg#>lOzhI1d?n1Nz?Z*;~^Jm^xWo{1;coO#c;G4t1N&3mP;IlHL@f*!}Dv*M`dW=T>yz z9?08Fw>I2-=;VUL{Y$`+8O7rZa(_Mi1T(SxG)(GCDQOR|OWh^wPmSGDH|fXGb7j#& zj8}Uj3bZrJ;&NOUe=jJ*Bpx=>$S^kRh|5(XEH*kx~^Vzp;@W2y&-tl z=ZC*`ZNA#Lhr!or%&Ixu6871eACFp11BMh1ocGZtkj#(3+E?bPVUZ6x00sG=0yjHj zzbuokPa9QPaEl*ef1;gh7$6{)|CO2k zhx?@T91TqW(_Vwh<1R=_*qa4k*ZMFaw_=_);y<>B-6^z1#o zvHkgZYXanPYX?d%Ig$#{qX5R+g4(@=x7V{AYE<1q7>`t)60}8aVA$FO0&~!3C*Ai1 zz24=AxE_}!G)#xnV+!n8K?VKN1jR%pAlu6aXz}+nGfDORY_>#UEH}eaHn!6lDo<S zgnH>`?=&rUo+KE#w+f9kq%DguACjHaCo$1zQ^U3%Zz62`4PZ`E6TMlbZn#E`l$xOZWfW4^gl1VQekKdu(n&EGzd*44m;L0JMTz5Vxz>5R zdrPeR3>^PjSGnE|fURcY+0oi9C)m=?(9Z4u2U*c+20?sv2UqJXmEm_|5 zEhCRCPBd+zg)u&lY$)qz42KGLc4l|sAvt3>l4S%~QZEG=oBy(*XXy@9HA6a7^)4Z# zN4BG*{RWR)d0LDQ?X?pPQ+S>Q&HBZ1twI2{GPiaVC*fMPlJSg$RBpM>Jhqhq|9UE^ z!nFCF@HkabM=8z_XJrz_qf(p>Cwqzj7ai&t z>iU8Ql!st{f!Ym$js$(z5U{3N>c*g;6^BkTDUrReO~WPEwe3}Z;co-q#w@N6Fkb)n zY|$-$CP%#4oSlrQ7aRN=iE=xraD_IzGx(B&8HNKxbBsJgcZ^*_9bJsLYi#WH%$B@9 zC;t+$8#~c;_6PlR*QuFDwTmHa?ICvT0XOU;N?T;K*xZ3G4paaEBpc^$!ofwL3Xcit z`yD6et{jlS1|`ihi~E^K_}k#wpD~+#P-92c;wprPef^NQs)IX&~;AnJ1flXU)u+Ym$a;yb8e2l+me&zyjt z(u{uv)0OBXN6+v->z*;1d(gu{fq*gq|L!ls|AU-m|MSLwMW)J||5G(~C&$fY2o29xxb!4)zd`1c zn@4fF6N_II7%89D#pW^dp0oQt?c;v6rW*u($ONj*jxdmB<6xNDyJI}2(EsVC%O7Ln zZavd~U39dy;Ad}?^912$G~nvafNzL9Nb;)106~I*C-xR!;JEFoE!bc>g^JMZI8q8c z*)Vpz2BXbb`8SMKgCl26k=AhIy#e6T$Zm)p5bT%cpRnw>-t_0a6Lb)7wy4HPm*A%^ z;uH#~h)e1yZeK;|*F)zV$_IGqrrl{zxDN#)VY6DzXX&nletge-ZH<>2M9|TVErw!; zdS8e%!*VEUGU^{a8BgwQP#QT~^1n)bcX=r~s7@6Mjzb-hZ~PX=96Y%1UzP8INK1fb zkeNqj`0M8Y^HO00F@wAA+ApzjeNh$@#hmBXr}BgJWEKNdEq$g9ar6nJo3vvL_i~_w z!isjVEjB+Xh9bpg6p3-{FyJUfGiem6@sT!Tzmm>|Y=r|`JG`oEv}bpL*xTZIdcK}{ z=C0@lE&^9rB<8q?8d(V<}m9zBSn$&RXa;*eIJ&+hOA0x@En)`mBa9rp{t zOZ65=HdS*Uh2H~vQO-(Fe$tEa_rrUQM;x$*TNLfCRC#Ac#f8wsq!t@YZXgd!@*NXl zrLK@U1pdcL9loW2_}>OmQW5V=p;UJ0|U%8SF5 zxC-6NYgo4Lmk-fn5O8@ znmsA#Z>m*Azt?^4i&#yuG{_%`Qf;{+jZ?bYO|29@##-L8adl|N#AWMD%h|gd{H}jS z)Xu9(vwr1?KM}hZYN6rL-xE_H$!79=ummYkRsy2{F?^dy7m(-wL9$_W=7I zmWLx6R@xZQBSZ9imui_Wb_)5WH2~gA#_5L?4`ZM=8f}8%fB*r_F_7IpZd$7y4UgUa z#rxinn;20^992Ww56ESCMBaYZ)Diq6|6qGkRFs|@+4zF-$vv*>l7_I7H6KRJ2rb(bTrPcu`p%!vcQM@)Th^BqDlgn z9{y=dV~_UtH{L(-xiT;ikkbDre*2FY)X1JlK+oFR#!<}L*~Y?1!BNlN@vmw^|3!Xs zQWAeD-__2ms}{|hga#sZdg17PXcx{&$dOg5ug`h3!LqbXCR;YYw)~0b|LTH3zY3*& zCN!bPvo+D-CdR|Fz4a|KSGx~*3B(d~#qegjD86{Kwh50Xv}~tN_Njw+ZZrXXXn_r# z4q1H^hhCAt?1TdgPV7l5Di4z+F7!LUM2b%U7-KxSKoxPKxTJ#-V?&Ie} z=H6eHXrQt{u?k0B%=a@z62g*zkQ5wL0^L8)N7|2^$RPc!Qs#^B{>e+xCbKCF&jfP3 z!H-ilM^?G2VDC#ImQq7KRrVR`cI@8#z4W7?p&+ z4MK1N$9(-t=+S18!ZZ@8Z@=(jjWNS)G`PK+Zm{#k_Azq?)$s-xgc1G<_1ine8ldE4 z7mO?dV;{yea7y!K@JQ>b>yfH#sH#owM_s@BC*kCiT3bm6|?cv`&j_y0UW$x}G;+Eg1(cm88 z)26gNQ^CE4jPF}!iMW~q!9C2V;*xV8j~FCQ8R2416dV?4nskSFYUVikllHg`aNf{}SeoY?RXP{laH{zhZpTpLsfb*n6>q6o969hJK&5WnqqiAc77IP_| zZsa-?grM5X7OP$El2|9B1al?sP#)F^AJ>MQ89}Qb;MmzIZdLUuk(z50^BeSGqAdkR ztZ#HAFNfKy{BrG_!gAa?$uq7D;`eL{;T>Ea*;h}y=Wz2>7xWw&TyB2`H`^+`H_>`$ z$5XJHMPYb3MPpup1YqqsDWub31iN-qtojH-q#eo4dL-FelGEGuNev?ymF4705? zzkDuN)gVGJhg4o1J250NslbSU-kiB`gkGS7z_E*)R8;gR`n6OEl@V85E)=FpHW$d} z!G&NIRIA(3YdLr@MV=>0MxSIRuBJ2p#C~`}9IJt271-+ClqXtSU1g+1*Sw-hTtQhL z#ge*9Y)nP8Ju2;y&kh=K0G|;G7NAs>j8espT0m;H6r)p8If}11&TX`( zv5H*eEXOjedqEI0;5-e!dthbcwnUeIL_6-7Uz)GAash)y1+1TLCG-MOo&Pqt+|6oN z5JEYaBj}_yaznM`-PG2lW_2e0E>q?R-&Q#yojW zFoyLT-ql#yNdc>%hPr_MV>(rxlf4Dkt_Q#k>2*bT5V+D3BDmj}06A(xeH-2~2|Tq4 zz4c@D7aBNLzSPeKT&h8ga+Ak)q@G<-E3)X=Yn$0dV*Xu<)81MM7}{`Vx~v2cOuklC z1fg|(7{q$%L6Gk0y12#JSaOQ?4`_1XogVz-az12bwP0@%NW@|MEPCXBldll#B zFpb-eyB}|22+v@ty?&XJETNW0X;`w5YtTnc_BWxK)Q4J-3>|Io47ReNTR;j;)c(xO z#|zOl)x&h5s5>%m)ZoMRe*Qn}?@{v4pMRy?W)FTk*QBT^0y#SfW zFoFcLB{SwxDC>#6qLRu3p{_y9$BY1P7kCI{6UMFl5ZdhlnEPdtP(~Y&j>YtrZ|bx3 z<``gAipo}22d*k^ZlJ35E5^wHh|5s0`)Ly@lU{0zM+Q>_=srs?Bpd zVlxWnq*vEwb6JXk5(cw!{@g#=)1r~kvCr#NXO7I<+Q57u+ zOrYwLeMQ>W36ys?AuuY|lo?;#%~%Uy*i5o~7kWvmrP*ec-9+eYG%3!qlTR@n2IYJ?Dp4rln}QjDwrp#<6X8DZ zO}!g#Pg>Y?3`)q504ZZv_e#j`lcEG#V>^PS0EM;Osg$A${~`oz0f~aTIb8`0{Q$`~ z;!+afajAPUZsQs5Mi>Qeu4YV6ycR6Sz-;CdJ$KNseM81nvZmEs)>GR3Tci-Oi7rp?O8v%}MCRa3G%NuKN!9ebGi4_1NayCkYkU{E7 zV1*UkGrjl54(anaTUM5MlEsb6bH&e`4hu~n=VU0fF7L6Da|E>-iUC*;fOmX%FvvmK z_|);4;l^kL+GQ840)!kKs|g2p0skp;s+WOBk|j9rpod=`xQi`m`E^1iJj@m;ZgWl! zORI$XcI+_g%g=4=bh$>12h5MPod8eK<9=}7{w#1Gi9k;c_3d54vx{_Y&vq1u5n@sm zUvx>tQV^M)g9p`#jUV{o`+>dFcMhVg;JnjMnD1Jjxbb$`@MDVEK1sSa(DfVF_i&Eh zvpaZqF@R6{Zo?ay=MwNQN$0~x?9+gcf=%U1{H1M z2!7iv9oA>oca4DUolfv>a*t2cuc@8w@wSj=$jq=B5`Xa&>+`-e*j%I#cJV63*@T{* zmKQil5cfkVZ1TWndar3^Oe={*VHHA$o^o!(j(#8HtS*w*F6b2cXPNix-s}6EG?FX^ z-5p17{;HRXY5+^_9vf#2M>|%*A&hOn)vxTbQkO~ajuEY|TzT>YN&#-<9O5zsFIoeN zDP=X^YFV7*`ICT|5hwjCr=FGH-R*E=$2XbVYTy#ss!R7&_??6fra5j#1db*Te%>p% zsW#FRmS*$@^7@Nu)O5SK^|PxqEzm#PiPm(h&7t=dfk{V=PLTxdza?-Dg_-^;rp4;y z%DDOD?tv?mX0(2p8L5{<9T#c9sRh?7KaDs~4n~}0WNA#leSWB2Xej#iz=RY>F%xjV zWSO##*U}6CD=#R;#aftD*;e@7WU}-k_E@RIl`5+=J!o7=PxYo4XV8L@>_i!x2o%!x z8(1^>uuXWv*#tjDanU$~E_-Pb43%{_|NI~rT~J`G9(JTs{n9a0jm`*Weyz*@eE;(` z_K=0?+B9yxK?qA-S*YJjv#Dq*pChy?fmybA$X=P*7|g$F&=N)P^wLZ1-A?Q-d72}I zeDv($2%x=a9K6r{gKDs8!oM~$D)uc9+uUWldM>7b9l>+d1I$L4O8r)odRcQ(?uMjA zrzOs-V&odP;{{eK~TbrHpM^0f0>b%^x0MRBvc4cZg8G$$H8Fg_kaC6SBZ(j&^&zvLk(qLK#RkREoWjnfH z9FznSl9oo0Ps zr76lOpZeKAC^=1Kd#)OICEC1LSFL19anMkp7L^_OS9mZ5-xwFrla&or)uZ%e4Ja3E zXU!gfCQLzMQtx;~*Zu536qi;6#QKqA9}!>{%}S)(i|K>$iZ{L1OI&5(-nc;GwqI*5 z?&+LY((hR=5=Yo5GrZbNZ9BX3=S;2(=N}8ThU#bFeq0GvK89d~8*XJ5okJCHe8IrFctzV*g?r0m=vq}#a$G2(vbCAT?(b8Jz;syk?PxXgQ8T>**_fjNu9DOp+-4#YWafC!mCvoneY8pGToI! zLI<=9Op>!N=)tL#{mvzcv~DJOw+}A`I7#B^3?S+C0GUhm41`jgHauOGQX;SqCx)qk zZL7`7Y&Q#?exZVSSOqb~np-Nmbhz?!S6DaG33BI#{9R1(*D>Z54ci96Q;nCG2H*|e zmOjrtouj_KWYV1ECM=Trjc`|%Z7z{vtpNEXsh9Nv^!uEduGc7IhsGJL-Fc?eKBp61 zj-?`hr8CFaqR_X0{~jB^WV+#4nDv_Hv;L%u6sGojZNdPt;c=egNRi@zi5z51p*>H? z)S(=mHip!mz50YV!!LhzS3uQ1MO*ZAQHCq_=GgJ909PFDuGoW8wz$l#P-9F*S%k^_ zJy(eKLBwf^rU30d>4{TyQHv{X=cLmEKCSL|oj$ORSeavDuE>RUCL)J?NOxF z1M-a#G&`i}BV*YRS7x12iyK=1Yfuq4oZ+N`%byTUvXI;&(}a>!LT)!&ueeLB-@d&8 ztHT~lo}-MgBWx&~z5%CnsPi3p9r!tW10AD+-bei0!<&uieNzzif~5Ywv*$;2eGV~@ zqboSPHZ(Wgvx`V74aTl6mMV09Y-(0+)N4uMT9vK%swNEr1syG?T9UG8lftrc{)%Qc zd}?oB5xQ06Eh^J3%iHKvHcu&7T~LTZ=<+K~1RDinW+ZzQXt$=k^c@BD9`l0(qge(j z>j(aZi!`TH!1h#{N?i_(@VBZTEB#Tct&lqKl(uyt!{Z7?Yo4tcok0Vl- zdl2KI`v`|^)3hy*AX%!QA0(@y~0i+bO@u|F!fmSbeZjp)l{9u(}X3vB{(oIgkBQ9G2Wn#d$8>mnMu) z86{cxdy^=?GF=2R7g1-|!_L*d%>|FM)gh$qDn$2>EkWrnj2(MurJE|IY@347cd+zv zml^x61k7iewxDb9&1)jX^Zj?QGSUq~2%eSySL(*N5 zF5lpB!2uUXzuzIYdt&aY-qE**b&jYlQ7T&sIMk(6X0A`3Zl*(aIBb9jG=YMOfc&X% zlv%T)HEI30(!S4ej=cPNNHPBP7i85U6YT1j*G!cN`6qZe2-*^Sareyko2lM+wFOY0 z1lPSPGx(MCi3jP*{cDOXsJgWh#bPvgbgExEQ6GfdkJCih6NJ_3 z2r5^1*goAJ*W0q&+p)`Qx+Fadg$l{}uff}y{ZbbbXDjUsAB)SMYgb^8S8T9ZhJTw zKUh7fxYx3~P{*?KsArOX?sX-tZc0sOM2i zsV~b5+JQ~ynf>VptInI3+o}r<^V`jG5R4)6a}W4UJm>qx-DQ#*+R_^!rzL@f$jNzK zWvr3$o%^7~S-i2X=7Qc%PyVZ~@pFu4egIejf5gaUfK}$k6n>;i_^50j`B7b@rztTe)PN7 zBNE&5H>qz77sOGVzQkDzIZtshImNphLf)U?P*pZ_#Gby&_xNiAA*bG-sNCU5%i5o? z-2Kz}q3+SzyT@{Dojusi$iUzDnkb#8^dqptZppu4{ndLorR zqh24kya;`QbZzZ+-?w{aZqL8JcPS70EDhc2G>*Reu|$29PabxgjX&e5Lw_k~jDIaT z-^p3gc2_vvimc)D6nFLwH*K31b{5n<<{dSSI4I;@H_w^$_})oJCrSvfHEHvl<#+gvXKPki75S%Phkw0ArLtVoXZLQ0x9?X)) zQV${z^{m6zP3E@7DW)Cs2$fW?@M>9b!jx^9{2XsT-$nyLWoU2b}avUjV8qxP*A%b%O_2}Xx(pKJ2IN2(KSEf;=%;1Jz zX14e?yOEAy9)YVbkiTz4gMtXYZ-W2<-9P{VG5xPLqW}6Spr~i^5B0p0l&2lm1(83O zoC!pf;)kphF!^oc<=+d!pc;Q>z?;q|w%N@iX26n@u5Q^aj>QWnbByAkUuCk{?)3x- za0Y-@C0cV2tF84uHbe*R_!y3VgA2!T_|>@h-7 zsm;};kBCt;tYs3^;XxfI)&Et#lMaQF(x8g^_M1w%z7>B$X3$=NPpFp~tOB4$Tl<_o zy9@YrCa_nphK&QL*-LJh|*I9pK?qpf_qRJzsl;kxl-7LbuOL*A74{k6vDaS#RO`xE0!j zw%oq3-F#wO*6M%yWpaM*_p^zZeosx5!1hK|7PIZ)c~J(CJK zVPQmGmf2E^*x|)bZ@o_pExa(CU&N{FR3uqZLN(tdSOLGM{oGM%+wmeCCmSw8byjH5 z`gU@LG&mdYCH(=1)n4SW-$Hta8PX@Ylw@oXsR@3Mq8Uq;CI$;9c(vcw-w>UM<1wq& zsUMk@d8y*7BUZd0w0O`y<6)sTm=nNA%Ur37YJ;_;RD0k|{$vqxD6OcB#BsiNa%dLpP2qg`?e$OQ;(()t{vOWG4lkcQS6 z;I2eNalN}#NqQ26$Pow$rpzJ0DrwwnNW2m91O?mp$#;Mrz-CudP_;yB=ev0KPdCI0 zIx;wo`chPIL80LrX9Cw!ybB`VQ))tQ{AW*4=^~#i0qfsC6zJ=bQ$B>lVJpMyr{%T+ z+Ug<<*sZ?z?}avRh?-LN;T9QvAI^Cg?7XAy3Py0-)|oZ(l=|>m7cgSZXCO34Gws55 znWq$L?#{bLEMoHkCHEFh)W7<{uo&}Bj==5%dN40o=nZozT5Ic<+cRj-!z(aJ7 z*TDPddUwx<;7d|aJAC-Un8{dScH<&Ol1Y6PCsHV3KI3;ycBO*d(v#7`_b=eT6NW;b zQ*|;h5YQ0lzsqy~Z^HOP7XO&Lsh}Z)#E1M@OugxB<(Hh{FaAqR4XPqkB~lmxDd9|l z5@D;@sJ_zDCBsPk$1U&kMR$*&DM47*?M{sRw4@5tj9|j#L>l9h<)Q}kU>cXKFdGcD1Mn#udLt>@x7^kq z8!i{&0Ih8`y^~v4DObR#NcHvDFhF21!pPwq5Voi*LHCfPrPE=}Y)(*lmrjaoDSN!d z0jK2@9S`~I@`@pGjPGP+BSqWvntEMtJjGgIg#o$G+MZnKWL@d1(z@jx2a$0-sJ%Vi zQt0k=igIhgbM4rQVHZM8N%?z(!{$WsN!P}0Mm!(sq^`2RvML&_CNbBW##H{V~;s+%T#4#qQ>KHW8M%8NHPD4pz?dYTeckB=$+ z?&mIyajXbUfcF8q@RWqtl6@yNg0U^!6UItEfP`*J_v4GsEcuHjn&NGGem;P_93SiI zDjl(MI)h1j%M!{bLN9L}>sM^JW@YYAxW<@l`$(fu}Zzb{9S1bntiBTLa=eB6;sehzn;BD84are@e(?T;ZPsJN6@e+ zPZ_LHo27RIxuob=c}crTCUNYaL7!C=V%`GJHg-5zseQ%9G>&xBsU=Ma#e_xp!SvzZ z;ornA?LbsuGz;J!SdSn~xe!e(>JGMFBNuklrVFK)m=%f@;OP%Te`z$>b*)~{O)T7p zrGqI|a7r-e=nnoOYHD;!LPsc>j{eKwZ@If z*vo>XKK=>gCRcVKZ{L7`fY z{;0?l{M4v{3n_v8c%-AXTJ333`6VuPN5!Og!FhPp}(w zEaFyGC)*N;nn1wS)DGqn_6B91?;^ocREJIWci1ynOaMEzG?PHleKMW{?;gWVyLu4A ziar?^oKdL`9Th80NXDCD2NE1uzb0woo)p6w^y2=9D0-`FPk~o8zd^}VCJ$hn(b7>L zO6n^XEx+QP+a0 zwoq5;zC%{tLKG4+e0!cAy@%h&h~lqQfe5`zxW02T#BVXpO0kiF3pLW<`j!2v3{>fA zZXh&9m7QHThKNGN-djQ#4S+>6OxgrGOsK)0GnzHmlRZ>r^tCT~py1c;hWUw?(@(%^ z@T!7e%rHN!*`o^eT|8w`Cs@qXaM5^vP&!_GGI17<YHdjpmgrH~#R+iTC+EtudCmH4LclNNh=a^D? zGKUgt{1&KNHM^SZ< zTt1QyXL~Rg*Nm-xD z9^+q4r$~6pX(+g`>7WK(2aNjqfrIrf1c>z^P%@_%&|4~R&GhmK znp%mvl6-sKdFy6}c9v$1^wfe}+}U9MHAi@HzKC`n;+2Z(TPK59=Zm-krDk-=+8;^FX8;xD&hZ1+W&8B7;9&6SZ8o)XK)%}@UDl2_}sZn#pCym_=Rvp zVQ>@}ssGnDF@1x;S|Wb z5oAHKF}2iCh(eycNa{PQ1}OnNun*1`Tu>nVW;6aB#?Ov_nrqI+SoB|W-T!k^{mZ#3 z8rfT!S?gK;L#=|CI;cK6^uX(yraGrtG+P8gG?2ZcD5Eg>H^;hwzOQB1dV4XviMJ`J>OPpfvkEa`1(^@JA zUg%=E>^YUC(M_x_7Rk9=Omo#JRzM{D!8@C!=WJkaNQ!zI|5Y?f4wv40ce~2fn42i1 zA3T^-A3``JeOpc-kt^se_}^#i;yu}e^(S^cf%p#x+ALugkA1BWsE1j;|2> z#1w@VGwmmC&XeQEx2l!Mg~x0f`dP&xb=_x*P27ACX9WIPj5qUr%`8ETbAEa1p@3sk zBpwA{=ZwqNF}iVi&0;(H<#?O-6S&qNFCn#uln{NGMaNkfez+gm`L--nM2v%s+%tIk zC4)a6@c3Mv+QY+(eRCK~s}o8VCM)DQ3U8CW7*mPa-{fzkVl?S!-t1XVu}(ctf7Vv1 zKJCOP;tVsnBo(5U*Pdl?-sYLRDy3C~^Zfv3-j)U3c>W`m(I|M=>RidnG&%^-ey5mI=`HAT_$RcUVx*@ z5FFr-@lZk+3Ey@=dM4cVy)BEv+H=R`T$y$sRz(cWNx6Y3o~`#ojk-ObvbbQK6!S&v zLP&`<9p#tRa2hs^Abrelh2Yuv0;TM0GZyU$t1&b?TV+kv;WZb2MPfC5^D*OIA<=OK*iokV&1`i>GA zhLVDVIt#+Z;o@)UGAtX!is_8JB>qC8@>lZ1b8(K$fOrEe<0Jsl@ZDpk->x;BOm#wp z8G!7$XI3Qm+^cS3)k+88@WHu$&9wU`M-k>T~$v{|lg06;*$zj1}x!Im(q*FI@R ztSO|{r_oBFhp2#vyHANAL*yA{ba(gVH38X$a9b64ebX7pypa&QKMt4qBiX05yB&WQ z+?be3U%HPA6}>({2NJM9dt){L-rueu@RVGNJgxy|6r$%{JDPdrW#gJq7EHpLtc-kB zBqKXZLX7D0yrz$F_u1%BW(I~HW`V@LQHs=uOta8gzbvl=FVQXV93%}~Y!;OO6Wa@@QpiTB-sRZ2Vx!b*=_ut{=61KI^ve=qOBTT9 z@4?^Ek{@HY)}K0#-DHFx{{+VuCv0bmH`$q|+mg`g5sobAybVhyB%w!jfx5$^?Bygz3rPr=jpTtUlLV*W9T4H^_v%j5UqurY?hT3^Q2jwK zoa?k~M+0wvA`L`$2`PqfDif-cJVgs}-zU@N8S<`E9ouS_vt85IT2U|c)n?XK>bbGX zw6DQL>xS2Hg+`E_<>og_=8lhSz!gd#9pfegKTR-cT@os>X_ilLbejw9YWe56tXdUS zU;jj_96aZx3x7she-;u%|C(t2=i?C-U1tu=hb(kGzv#r8PNrHiV{F9$e?b-G3oko^ zhtAsHHqHjpUm2?-5eIiuKxCtAN#(E5q0!-RoOOSAjMl>`Os}mEfE3Gn7v?bH(u<$u zsiF*I(&n^FhgQMT!A-i6Ca>GC`%x&htD*5CgziufPfFg%6r_4hndD*YK0yUnWUQZF zh5PJiOG#ygBe}L}DRkn|{=1q2xlq`#kV%G#uWU8vz*JJADG! zu`#^ygL&8=(gm26y2u9?Jab(Gxef zzwQb?2l2n-(vIyDBY3Oz<8eI~pqcE;1SPr&^;c+94de)ITJXGs`JmKwhaG-re%I>f zMSf81U`~Pe7YRG5sbT zlHvr-y5hw_qXPJF^^oJ9kw{EJl^~Z}!nq9zmgj&5==e4?9|`9y=<21$TN)%)Gd34n z|FDd3McY8kLX+YQ)K8Pfhn;Ccp&M%Otdp5`XJ%wH zBC!q;BYE=UroN`-tazrRC`Vw`=zpr=#2mK=3<;Xn!@2$y?g>$8DZN~Ax^FVpiSc&{ ztvRi-}I(vRO1|MV_;# zF}^1M+erM^W)Ms@5vhfB$cQq9nU2GYViC|;-<(U-+}VN%KU}9y!6quZkH_qGoS?Zkm{fF~o~^PtY;_J2wQ& z)7M8FsZe6S4e#nE4npYZT2eozOcBqNIBsiTq>kJHLFP`QoEB$%aUuJXoy^PP&dQz0 z=;(ppZP)D>EJ%(KyuN`ltQtbuw!%78IOk5Y#l8hKkzvsmQn+U~ovdws1N%n88Pg7G z0%^r+dOG3L#2rD_2v4eBbpVGeW_JfW=9~k-?$R`I=#b4>usY>(Brj{)2RR~#*KsEj z0q%UHXzPJ`Dyf7oyL4V5@8K~tYT>B5dNE$=tb;~bS~FZ$AJv_oc~+qwCI4U&gd9>% z+-NUAge2FL7Huf%>m#UEY?)OvOkG88JiP=Qci*hKK(sfF{rQTJE!-S+m2rrm}nPD@_h11Slj!j z8`EYO(X6adX>t;Y>!K!SE#~C zYYy$!r_*D`&tvl;zdA2}eidSO9N3BA-jj*0GA{9j&|8P-Q5Y|v-#$XM9+egz@$c38ZnmjGCyU(wHM`5BqWO3 zEib`}`a_}kH|Xip*DZCEd;`G}(+h>AYhKwmqU9w0b7$2|+=5_lFPe7;GM}Q3ENtSO zKp@u~WRS5-d;vpvYXl_mqka-IqqQh5lrwP%zIITxJ3`Iy)T(Vl2R8nBx)iOAu>q51 zxfQPkOoHh^R5IwvO1hIykvDNJ{2tXo4EN=08@S5q;1-^Sf|Ho)Vp881S<2mEw?G&C zQ6*ml6gC`QSmqSVnlt3}u_-b{-&dMAfSlHi?!L6qOi*Xmj!c@LjJxg`9Ae=pr2SXI zxQv>vl=_MR&vppqCmiBE6SHF$>;pD46t1bU#&4dah>YL!C0fjk8_sw!Mz>7Xi%4Xl zCJtHXZe}#w5>o{ZbW>Rcx1ddY(p!maI~koE?wBSnvfa|eJW~WzK zNF`Llna}eC4&9PHpyLO~k$2XRmN{ z#K{zkhJVw3RKj@&*H9kHXk=5e746GnF&6{DAeBAUKkAqhsqam*8E}Fs= zcc{%y@;4eRaq1heY%D!^F%$AcCPUF^@m&%}@hvC%E*=<64*1ga|5^?bTh)SRydyAp z=P^aDkGf3RW$^}LSDYY<;U6WjM|vHU?paUBw~zvq(6Y%eVM`~nIyLHiV`=@3857&n ze_vi5^a&fhL$>m|1tK*`^*D>FZp_yz!{EeF`*160pi{RIya8d(7pRC$Jk`TZ>DwvD z%@Jno9zgVqaN03p;t65C`-%%Lm7ZUnBt4d7Y*#0UcYXhlpuWhjmuE>)VSTc#P zUxf`l&xWPtMn3MK-cnmsO*I_4P>i!6aSo z&H6%BYkw{aD5^I+!?OLR8S{=oJMfSmf+;)ru=|vmY1uALL4d%n<*%3+)9Zs; zYSnrHX#%bBa@}lEaqNpPDH|= z8jl3i6E5*q?$!~@Ti1#u@e+;F{f8nVJT{{N>6VklIA^8e*(LWVC6hAcqbhrp+0nan z*v-c=H?ltQ6$hl-B7tzTry`G6nu~5jgiJP)yDGcf8C+29cSiT53Pf?$2JZmB4vqDN zmICMPtAT{B^v(xpx$zkkyQjQOueh#1o2DkG+yA!FE~9g-ZU0#4hW~c?sJp#^vpK!G z{69|;#{cZY?O6RspyYGH@%n^;QuX?K2b*CcdhPmrY8Vk=pgVd#G7O(htyF_aBhE7$ z;ZF)e@orG}hnU1k3K13hOi*Ta`j_eJ@y6Tt!Gs#nOkKA>w5?d#zHC?r;Wgvpru^}g z4Pd0aJ}B6#)S78a=2DoO6FIz5kff2(ZQ$^hcMU$7Csnf@1J;gRa`_W)8PCz{k2>FZ z(>v@^4JSHmWci|@9}^eyGYG$8Qd>qwR3%?aLl}L+QXo4B=03Y7%!Ufj448@L5eIAT zJV{>^xZVdjH()kmQaZ$Bwaxfgc`|s5-Dl=5*|AH5Y*-fi%FJlc{lgCis{n+#KHZ%* zGMsqkV+<I_-8&hPiTkv5gB@C@>-ey6K9W4lX5Bx4PxsXE> z_mZ#=o&EHDOsVd50o!MW5HY$83F+hE*2%y>jg)n5YvpbLYHwhr;!&pW#{+E z@mVZ(+bHI5qFwp{y@fg_Q#GqGL}j}N?GfMeu*3PU6Y3OaVlP-ApmH1_AnN}q2L5w0 z7Rnn}1@HSiQ^wSdiIQ-td8@fskzy)aQ4-psJXl`87%nY%9v!iC&7#4X8OM~Im!osM z7!tp1lyuYp6)_?%7?BQ*6D$cwp%YA*2N4aiY$tQANTMBTgPEn1R^XKXN=ELuX z_x<$4Z3{*}o@SvQ`^%uJ*WVJ{h0U#EI}Xsn0(&uQ-x5Q#S-7VQ_hQki7aH)5=$n_r zuRIm&jC$}3j(=GJ0O7(D9Lma5Kpg1%bYBJ-ah~j#Q+{X#kT32g5`HS5ebWS&3c73h zc!~}*avkUQV9xA$EOqAX4H@G4=T?^KLKnjK$Pe_u?wJ&SP#?Y6!u6;I^~~u5Zna3i zx$$~N2$Ubh;A&4fFu$;Mm+FN>zA_SCl_33=Z!z#DZrcGp=jt&lLa&*Co(pqDp`Y2g zsvJ9tnIGk;OziLC15DOpzXN`Q=WWHV7S_%}qh#p!9`_c^n?0YU%h}-izY(Xf?=rrL z{$yalkWsBRTCz^#lg3q1znu-ekK(6fs>PtkRdY;jnJ$z{wlsE*xgWJ zrhXH3mm9b9`EEwg;;F`FN^hK7$z{|_WOZFjlY1rUq-_jIT16f;6{&eSSxL4TO$AGP zT^!R*>f{xwwdc&dgZ>yveA@8Gs+xk*y?e?Uz?SM($F+%m54 zHDM|=rHs-}2*f&=Yy4{lD&T_RoFbDPf7Ime?hBnrs+&$>;WQN!T{($Q4<1V|k#A?N zSQIXbO!8!bOjKGA&qeE2)vdool72_5t=87T0cj`84JDU;QT?v{XT>D&b?i(=!-3nt zhR_YV2~L80E{VumF>!F$<8Hhoy9*bMkCae3X3W}&E4hKI0S!^v73|@`as0OB)tQ_( zOHiv}fo)4yg*Ww+pfCp-ja!G3A-GD0#CpNRKlBJ=x5xw9CDT?6gILu)mnOA&q&DF2 zJMf-S&|$?1T)aNu>6F2qY8h)aEGa;)mn7q% zK#6XC^msq0eu)VsrSKBFf)Yu3rv3x+_)6Fz>*0(=Nk-PJw&IOZkeOWYD{R}CD!-L` zKI>YB)X;T3nlVaPT1DAtGR~qQl(4v=^j>FJhLFBt?dK0EU-qG?cpEBA%3%*xI zq5JZ_!Dw9ABqDCy&T?MXG>dV{y?Tq+Mh&4tx{R1Z=1{uy;3pngimM@>PKl9gojMca zLNqBFr>8~wfXhi`YS}6<`W&#(@whJW!~tRDn8ReXC7rRG&~wTf;F6fJ?{W-kE6#@! zAa$z(k~wQN%Ci=`(iKBM!jelpb8P_vee*Bkfd=&#?SGA0cG!nBKHZn_kv z>FUE_GY+^cmk1$DRx}a+s|=NLbaWwHP%+?)EVGt4d1~71-m)mR^jP0kt3ZtwP0IWX zEQ*WMsL!}@Y08VAk%~gUg#xS$S%HR|))O#?XeI+GZd1JG4nH=LMYODQ|ChmTNg;WY zhgl(nImBdFQ=47G<4W))V-ZV{9tMsRaRXL^1R@$y^4Z$ht_gW0Exm>|w$SaHLIVq(%V)op>8nWlDC zB@S7IlO_hs;>K}4^PlR_0hfCTp7Z&hWh@}eJB>G-+Uh7~DSO*E+Q`O*i5(@}!N|B= z;?t9RmqTOb_Om>u>-+j*hk%`A35vDfsS*0>)tRguozAurrnp71Gj^&vOfGqkA$gC#B+AipZ3OEfK!Q+4<8FEmEb9x@-ME-)MMnK6{*JO>VY8_v{#IYr zK@oll*Jw?@b7C`#vO-HgN_s1nhrH;ZTm546$R7i{`(E~-b?pURK$XqJ#iUMTHdv^11MqZAO!P18Wg9 zrXag8|NCc=WsnhNq#&}8aR)qs1Mb#yJk^~SO~l_P$G|5S%x!j_<&Q14=-j%2xm6_u z61K3r3V|W*0L5`yy(=QA$pUzt7^@!$MK#`F&I??jwT;Tu?~;~9_`FV6Or(e@%-%_Z zNlmG6b0Y?9+}=w*8XKCT(M>=Bl9Jb)bdfobSL#(Ft0wvbExYBoiE<=56^5kW&TzlM zWZ}ARw8&gP;jR5*9vVw9QGS>!t!^Z`sGv|9LGKqw9wSY(Ckqc~&X%n*bB_ze&qD+3 zX4Upv2g^t0_&41Y6{-#h?P{pLfY2Q$A(okH?0Js(nJgBup*G-N2eh)lv&bFj9|FxK zgw4f{n=3RKL#q7B5?|w^2}f5E830SdWafzV8zIv?BF4rC!sv2F{QL5Bm4-x1)e7Mv zRHo*OZhL|ZT$~1vXQM6@H${kuO4A)J*ONL~;>(VgY!AlMjfuPrp2 zEliOk$l!{If6~|$Y~Pv%)E?8@mOk5-jEMDz9p67XG;UbSURX?2a(5i}Q4)`BOkkA; zZk0%EAJ$^eV;3}wv~=HEBUUaV#AbgVcP+~42v+;I*@0jiEXG)J>uS&$&^YFfwr3sG zD*SM|C_)5gWWWMe?eHn~?(ZPM_b(#t$AUCn_6i{u*mrL9%z7}=s9O!|AuO8U&|B1L zV>vZ{d`h5a*{6tl<(gg%j?6Wyq=h>P?+D)wV>rI(83w=9iJVuOW+a*wiLkb{VRhSK z_^g347SjWrtO;Gn6OSW|(eYe6S=%`X<P0}m7&_6|L4hi5C52!acvmxUS z8u*OwI{q9HZxDw>y@5#M@y#it{FOi!b1%vGN+oXbT98kk{i9tbxN-+qaOp_)0X;dA zTsER2{IfK;`6RpyhuenawvWC!f#wV5=02)@UFw9&5B7@9NUAHB&+x~pcK>2f`Ik+% zG(E3&peK)z9w_>LH}np)8z1zc)=GfNG=%Qx|4T1_6Q$OpL+cNvu@CiU`i zoL{VEc{O(L+=keh?hDnoE#J)@QPuqs)|blmee!-Le!q%bybsWMu0xIKCs^xn<2|kK z(NymX1s2}`?haR!QQGZmMaDVz-JCD3mJ_*ZGb#%xF*JF|i*6+AfVxOHy5cKh3`$&Z zB*4DY9y(Ix#z;`q(q^$>9nmV>U{WC>ez%~E&Hs5=zKKE7mnT4brR8AHDe!P2r5kT$ z#rEa7SC)RwtIj7*+$*e)KM)dM>Bl$N!yR|YJuvBsgz~}yd5M|k>!j6wCJ|qESFm&R z-fv85WGpLcj8bm~3{JraKm3*o{YpSRg3mj&yzKZi{jwH z?*%mS8@$8@oi^u!_R?dGcj@JpSMp=oHlJ1R8{JnWk`E)~#t3~(aKP0Aq490jej~vR z(LGf2ThKeVXQOKy541p26JNf|{FeW}dYE*Sma{@h}`R)nVr zqxkvrkGq>{5w~DQBX}KWHJ*5%WZIc_|GeJA^%AKniW|{|BW*5{+2!l!n-?Sr0z18* zctgWbT+G(nPxRYd(Pgw68HV@Wf0nEs+VsVhUp>&dXaB6dJWs6e6 zJ7CM(3Bcj%3IBQ;iK$RpVd7|PHZn#q;$THBq)##V6S_GL7s|KGAg#8>LXf4{@C-&p zLsu-`Cv>Z^)#w8ZzV+Y%29GaUr5ztd6ZMO=G9wS~CF->K%R0V)PWXMLLntJOisXvK z5iQFnTu1OWS3qx^gIA2zIWE|8B;nX}7){Tzi_Y!__06kCnVYnmh9#I~1A4G9WsaMzG3mX?1Pk>qEUnm6c0!=wQd$rUmO8DL{TJt!{vr1K!VF&td! z4?EtW?BD2{M7>~+!!C6b-F+gO)}9yriZOrk<}=;^BTGwkpEYp8c9*~^+N#x@uB>87o`3lK$SLe|Ib}i z$!b4dN=IlvoAhH{>}_9y6d`~V;A$2bDIqBrTalJH)Tk0#mSRi?*VISoc$TwyEr@BG zCaQC`o;o|qrze2?)hHCm7AZ70kxo@s#Jz$y&g zCptY9$MAsz1{Y(ozGj~k@OUU-O4yMmT>`slO1~z;VlP@(iD6Z|GYQNtd}ysoA+gJKZwFwXS4QPFxxi+S*O+tX{j z^P6q5KVgs*g-?u;&al|!4oU^Mkid)II&-$i2AU*Xt2jG8I&OaK3kg7NJ+*fJ4(ZFT zVg-Bhb^Mb}LCekHUMpPdRuP)%B$TOlwzF1#HiV9+wY6tn`?=I`i>HBH- zw$B@2O_D@)9Z(W`g*|puP5xreX5WR*EYx5&P(QBo&-lju`E6ouHx65}fHkrE2ieFa zz7i~*&edUZk}O*J9^Q$AW0tyC$x`V)uB+MftZ9zxF8PYNI)La$^)*8ZTmoJJlM;8% zc>NNOV9m-D0@g9TdO=qy2)s?$*;ldonOs7{%Hl$GnFRzw49?1LcNh<}@K}I3^Q9P( z^A}wAu^f`_Ns2L-g${MzoLUCA@R9TfLv(!2vB0kwV^8NzO(qwS%_h!!bFXDgEXIb7 zgf=21c@f*>usGSyuMo)!5~k-dVMZ~zeC%xD^#`lebu&U4*#SO%;Ea@260K>1o*MDs zmvXpb4rZu}JpZRujPcwcanDtfUvAU7IaHAvqWgbh88hT+(V-A@?9B&C9m2liv5Hm2 zN8F`|l4sHn4@e0mAc#}2X2_)BnUgj0>DDTZz6`{6 z^1FR@r4QWj|0JEBD}!k)3Tw=hXe=AA<#QUB9w(v6#H+|8v&tk2e@qahcf-GP{ZkiK zxpBIfwK;Wdj5QUBM+n`up-L3jl37G7AJ02iYg@TlNlX2jBjv=E6`50RQ6*Cio6t_6 zpf(gsHOS)hI6mA+Jc*f+lb)25Rt?L~L~+Z>=J_~X8=1-B_!!?nIVqXW_BAv7X87Xe z@O#aF3om~o7CVWRna<_=t{rgmxqTdbGb5jw!RO`YcKLohlXX-riX8w%z_H@A+q6dI zH)Z7)X5=?S_aA*m#DGy&WycPqUWh1Sg8)S zmn4HIYFrt0JC+slBV3+hm_)RdMwp*2uxfOS23%lQSQ)f?QVfI&&ryd&v+t{?WPpQpTFHdOSsIlyzD5eTl*N@4(ji_gu{xZW_H|&W5GPy9DAuwI7O~0q` zhqL$BOZ-VVMeuyI<3jC2V^GA;IKtIXC(mB-be9~UOUs+MH)@J(h3ae?19vW7L8c)g zy@#!mCY~}EWzgAi&&l?#ZTfWP0`haZ?b-3DuBkRV%c`)7i8KE?Hf^FpvcwC?aGtj8 zwUgm}b%tW1@nX|0m=O~fQo0^h1@fQ2qUbK%ap@My){)M0V@8SgJE!=ho*NiA?JXCe zyqR_8AINkT@7my<+{XB?OLa*)e}dgKy!Ftz+)L?F0(X+8KSUzX%(XeL%oTIro9HU! zzSR%v$YV*)OS>jze3C}fJF}gyLF?@lh1iCH9&w3>T}HBmo>nWBVb%a!u5_BYIa;rr zmZigC8F|Zm2u`ubCon2HnlDa{C6Gr;IXin@GSU)TtAnfj_Q>I2FgG2c)0V-|iNu(x z`!>}JXBQ2pJg)AMHkl6!Gene`e~wbV4%8Lspl}gVyG`~sH?|{7mOkXzQUwH|Oqxau zb$%T4qH@02=c}HbKVCuUJbTjVbiKKFGREy~7M9;E+kf-&ndgl&sHJ`=rj6%dO&J_+y03yXzcQx}j86WKdK^kjS@7E>rEdlc#Enh^pp6OM8Ii27o zmC8^_kB&!S!KO5By=J(!MLHF;Ux_ing+8&ZtjHEw-+9U5LF|NTmD^INVN^Siqjhap zo+NVHGUFVNQ)+iOV826V?eMhdji^ zbx&tK-T|}`f~EXun`A2Q0X%^pv_U`9`+%$56EqxXwxmpy2VGsjRhb`XUYuZFI;nkI z2VQyW!bN`RHxgKj2h;7kiXYJ&%2iYmTc_BUO^{F&VYi#fDHnp^D1hebnc`I;t3QNW zanPvF2`A!6AnbcNziw=zJtfcn0s8?8f12}J%ob(%EGIHFxKjGilKgqq`iU}(14qRB-%u{yyW_{Yk8gS#- zw((gwocic4TowlsO3;Tyulm9?dc@NZ`y=rV(vRl&Zh_g)2r#IFg7u|e{SU693!iWzL?)2eGLgRIu}&!~-I*15!WRnq5YrM-@?_n#9pOVbhWpXFrL zYm2qSVHHfr?h(K#STwhcg6?yOY;fV{^Es~|M1RhhkYMuPUo!hi#~t;{4M)n>VG*2w z79)k((~JtEU@4lj<{vRPtEvu_gh}>;!REv6`$8RD)`l0|B7fWz=+WPqLyo^{?*Ik| z`MJHfwpEn__hvt<_&q#Qg%+R07D}eF$rd-}R~_cFB?@I++xAw(zRBA;B}WkEIA1)F z#gHe6bLr$aC~qjnrQ;WVz2h)F@$Xm>C8t^N5gRz7jWZItC;!;5cgciZ%N%;zZbL9M!UsGHEYUii0CQGaS z1CIyr|1QwxUl94P&`{oTQW8P^vElPv*pP~}fD%PS%qOuq8DgC7MnP=a@%~n9lcDP;o5jT_f^U?xtN`o4hSZh3aaGTC)^!t9exB)7$ z-xxwHd1nlB!_#(-A8d+zv9|!aIk*QhWw08{3ZjNT)@gn1UH3I#Qp58;%p;j4_zR)} zk7KFzCkgER*tu+~^G}U@8+l6+!D!vtigeMyv2~lf)Nd6_)}&r}k0$M$=fqNJWK!l?{DvvB8h{TeV! zoe)#i-$wN+N1nJJK$z!CDX{DF@_xm$qYx}lo&2s^ImS6!DI9t$IR2uBo zy+5GgU&N8si_p>Q&8xyXyUM9q4qO4je8dd*(#s7B@hsJ&00v+)uRz*3_%-L}yIbab zD$fO?G2Wf$4;(p7Ka1QeZMn8%tb6&X(RVc%N8*^nT_cDU*U~T$rG+W^4A()}XtguX z8~aVoO1z~d7n@4120wW+8+!x|?H7B@S}0ilDOc8rLm#bQw9~O-Bfi&9DJMO6SzN7`z~p9} zY#MdX2Cp~V?tl`eVpcpBA~eV14&Ltow;i12sO!8jvjQx$VTwgc|dOUwD4L&i72nyu#JiiHX1Mp}P#vpkFf265$-$Ab7=}welcc+UU z&T^qH#YnmXCI&VHYKY&l2RexCh-ShGFxn#=p z_GMVqMH9Q~!KP4lS-d6X>Sh(TQARkCJN%EBuYvq`E6V>L zO#NR|%m43cuB0QSC4k6_vp~W=9BVGa|EUTZJrpvJ5e_b_2^pM65>{)~K(=A~Jl@W} z(ugp6h$RIR z7{fQd#KDIUOFe|}Z)udH383Hfsqqx5CdA^Qy>YxrD!Cd3aA>evuwP!m)PU|XQ)Cmy zEEAt@xr+KOtbQ~z(^qs+CIwJrZL*<@T1a`}v&X?IZBO8)ChciQtQnAt9cA-ohB$t= z64QKBn5bc?d8{1EI31?y`8yOB&zn!YN72*shEYyzKyMNxZ&4|S!dd)gQ1;iF3k|!# zFvh@$1p2(PW?i&(G{N@5&-q2PU5jkYnm5W?$yd?0RZ1wd`9Q^SRG%{xb>5=tur7iX zu;up+4|^lp&P|c~F#!Gsrim|mkSno~z^_)~lIOB18oeBCcr`Rm_9XER0;DOy zk0c1yj|FPn=*(5}fcbW}|BO(#-|t3ZyIh&j~xe|HY7DZGqeOISZnzB{;#7#)18M! zkAL8y0RaTW{(l6Kf2sM7l9mYSHyeK|VH+4|M#lHxx*>3ql|}tHef6p;OVOh6dV!C6 zLbm-zrdRp|_;_xAqR)ehnKPBkaCz;w7?;x122KR6K3NRC*~~TPNA^t?ubJP+!y5g6 zYBdG+><2`6hoy$1=6X#7>2H96W@F;Go+= zJ74U^-&1c~NdBRLbe?MMt~>#4c-t3hqr{VYcPXtBTB((zM@<1keh`SB?h;v1D>gM! z2|;T+zbop&R4#6z5n8!?fk*M2+Cpqqgzax9_PGj~K2{;OeqXnt5DS&$j9n?#b-Jci zP=!s$$+4u+lj|W-ONfU2Lc|M5T2IndvRpTNuo0vgvlmh+lflqrkh$`l_TKfVsMzWy zS62K`(7n>9qOxx7ew2JXL{mV+v^=o|h92C60S>Ai)S0D9;(~+=MsHn3a$sTs)tKm3 z%Zs=MP%1w^%|EcdjQd^r-$;wqA47UnRhc}^6==uC(QG!{T2yAg(wVeeOKrhoVvu2y8meGT7R6NVqRDAS}WfchnV2FIm`_ERIR>3>%nM$&jkHuSf0!FdC6JX$!c_J zeF2RjVB9tBKJXx(`CALJ!Jwq(={xmgnST#Pwmv=82fu2yk<>9mmPwDxOSqh|Nl`w+ zMbzAnFcixcME^zn-l9g(kUtqKpm=mOY`J?!dFDt`TGr~!_jaQ?OsaUHWvuOql7MKT%ip&pKY?dQ1R=aGkwug?Ck+JRhi%r1tebVRF0!#a6Sm_QF}Qy!Cw^TZ zmFH}MkBrgtPND8`7NT{f?OPME08FGJot%(zKP>pMsm)oz03RMnC!sj0= z7{!isD~P7dKRIPK9X0ZBaI$g(bX}-0EJ9$SUC}#Wf}rM&>aaAzTIJDtoj)yaDMf1& znm@a`X+&XLTCZQ`T8vKL&= zT8k4gpH(oJ0KV-iwAM|$JAo>uHN533WYsaV+9q#`^*0}x$KfqCqiB6gGWFfiss#hY$!%Pi#Tox-=SbmUqF z){|;VhCEYLsmE_4{i03ismJ^W?BbX0F4@3*?zz_RZD9?rrrV9zDx^J33)Z^X8xEjD zsrUy_%Q6h2piOQtgI%7OvQe3^NNfPvyUkGbmAL>1V6)^ydJe}F6&wjJ>@EEHfoG3I zKW}<7X=MgECKGZ@=?rZ{aG%2F?wc&dMT~ot2>MNWe>@63%rIFfnPKoNGT0uKUV{75 z@qrndfbj!v=nEx_yZZsG*eC+ydtWQ>awyBrC5w5_Nfl0Nup`T?j!N}jRh812uZ8aO zUkh_woX$aJfPsJ>!GM6c{!gLvuYbY+ZgK+XXqsS-V#+o_n~s_zXeltz+9*nXLxCo> z`e~D8@v7^b@++#L0 zXz{$54mgO)O@>J+QjE>%f%eRplSb(P?@qrPH7RPC$Ae~Jw?yiHA7wI$j91POk{-sC zmf!7*UfJZ@F5ecf))z?f5{{0GOeJPId2fg@F&YMTqD3-YIcBQ_XTRreQRj%;GbXA? zFcS0fPOadb`>XjkH;f2k8levhRI9wSoChm<&nVq|)w10!_A^Vzk-*F_O0?5R8?5)k z>M}H;^T`6HiNdtp!!-b_JxvrsdI}b1dny>QX*E+zvPR}%Sy@2?TTNYM$<7Of+eg%i zwz3rChLx~3Awx zFOe^c&f5K_b#PV`5eRoG7YT!JJ!dq$$3H54ACkSD;Cb}`RR=$~G~U0Lh@>}P{+if9 zEI|q56xA^_7bLJx=6UJjh2cazVRkI3^E?X=zXE)M`3Bs*!sdfMc~(A8_Gop=xWAu~tkG5`Zd?132Utw((ZQ{^M-er{8Ws}Lvu2HLbL>Viv z$#vOhrLpi%i=wn3-KFnocrVyA!)#Ejn+l@Snos^_xpR#=h4L7H_VX(k6)Xs^AUL{w z;5eiL-@K_F{wvK@i1a96^FLnE{(lmzRcsAiOillhQez==14pHQSYu%Ge@}<2c{nRC z;eB6E8nXfZ2&$U_2O&YB5C;A-hJu8|1}m^;lef`jq902f%XDBGD5JC$a6N+sp5!#kc3CyYpdZ zMHWQeem=x^^v8a$-|BOvZX{^SE`5q80MGCXL*3+`n8W&>qI*#h3wQ6L-? z38DQQQN+J^FnS*hv}rCoqco8)R^fc%2vwI&73=x(adLFpLmQP0zEN( zcx3)XJ4{zNC&_Nh6kU|ABbwPfXGNvVO-?y*ZM&H@q3Oy#^EpAgF&lO zA~I6lV(nzq-7+v~%gCkpe&k#lziw(w>*31o(}o{;%~mC6)`0`ZojT5TZD~1qe3i9i z#b*1LyJ^N$akEHd;S|m8Tx(Eqm2=bjs<43aNu*Yb1XfM`kSvAQ!0=oQf@)rAf_fg^ ziq2ig4uWgJRZkb0C1veuv#oCwwf|zn%FS_oYL+CjB>iyFq*zfFRgo*ltGdY}C%4+_ z#W^EAE4lfsv>{9?Ws&WrmDHr<&9s$b5Y{uAM8#{lcHU+2Nf|V(HZ?3SRD;d|c&3t6Dm;P=$NAwm_Tt06TAT@YG`1+gj{ z=NL79L=6GhttTiy_^+o#NZ$x{)N{$tgrdL4PSrY<3m)8SY6UIZLvmkP99|qIWCtD9 z+H#w92Psn->SL`Sl^(x3vHE6iRY6-KlUs=0A^6o=PXkZYOggZNvZ)!FJXknXPpXv_ z>$tXLqV-vFq(*EkM>RSoTSYy20`+!vu(l(2eJ&N#cnd736Vu0v8bLj+h9|KND+Z4r z10}_jxNY&FUicu0i@zE`6fa5XHcZQ9J(u?Ew_erd=>bdO19&j#&;e7u~gH-BRXU z&r6MfWKc*-Vfz-!tB@t#6+TeMy;k>U%}jQkoS9#>hI|Sbr}}g~7KcA!Ja^q5C(VD+ z@~%DF8p!x4d+^O9q?J)=)aA}-ln<+{Rn@0Xp4>>AJvY0?N1wY@S)>!M7+Ms0eK?jF zk0LrCsfZ`7yY#CsE1FiC45D=P^o>5nS$NR`mlkx3EVS&58ml9Qv>gR@>Q98QNYIGB z|0Esg(_L9J(pB2o*paoUJ$$>eE&Py6B=GYip?ZwuGF{m^c-f}s@lXBAmN5lle(P#L zhE^j2&7J+tKk3~c?x}k*eX;pw>DE1*t+ko%JgOm~WnZ%w;XHBY$wwO)O8q;nOc67X zYxpk&yFIKEcstVX2b+DW{ZRwvW^;B3O8A4Ds5n;Eii1Ib=1LXQ=mz) zopJN$95yA5){tHD3)&GM9yrx>Hpt6hVpUU1+jVlwuDR5n;GyDD0yGJzCe$5#chCss)fN=|ph6-nkcw09KrS7TvENBz`e75X4x!Zx+ zsV(xK!V%KMjG%I}t~=rgh{lpH7+j&1G1+iKhK zf_;motyEOI#O~+g)n5g_RmF2!NIW3Yn}n=O2hsp1k_i~JWb~34pydZ=&K(g0d2t{2bO_LV7kT_FGgIt`)7=7v(%deMT-u{PC$_!+fiyR zkTSx9G~ZljHL0b9BnIcU>Hs=C;A|T3Rz;`yd4p)7Ax?MYt9PJwAp(UEIllDQ{-^gI ziKF>RhbTNyJX)Z++7~@~A;y+d!Mlq6)<*R*pG6wk$<$79Us(dvoK0acIkm^EKi0h- zh=^d#FcH9WVl6RrJ2)&aWKQ@0S7~Pg57qbo|1tJmc2SmuWXV$5QnJU`m&6BS3?^en zGh91@@u5~xtLL2V7-0UpdDZzN?-r9_o-G^$`r<>79 zrG9URhKUGA58K1_TqB<9mJ#$mKtkK$z$pQpH~H@01MgKxY5o zK4()7KlWi;-D#nT=~K707mOxdt?O;(*zQ~R`8#5>cy&LEM|?_i)j9Pv%2oPArChCh zz5${#^Oj-T%(d3xFUamH&KDjYc!!ydk!c9`UT%*L9$rQ{B~c@N>z6i;xuHrz*@nH- zLsi$Uf*YsIrDQHAu&JpHSmxL~);Y?5n7_yHfvnpBZ|SLG)1y{;4E8LN0(!}7;|hxm zB8yt#Tj}ELIB`}s^~$N=#~;iev26cQ&nBLR;Te@V3}?Hl8SYX)&Cz1NTgpc8Uf#$u>WWM^gjPRrz^Ip7xQ1qOP+wza1Ugo~oy!|4* z#jpDFOYc}zu`&1TtH0Qr%rj_JUZek$>*?Lx?kTNCgwQv}Ls6u+ac^b*$%_tAv3GeC_qM90mv&-Ib7YAyLdp=SYHrse-A!MOF6B zq64DK(!a=q$C?^8oRKgIo>;nS}lqAEPN z_n_D;-#2f6|=!LBV9}TNw5T*xlVv}PM>(3^| zy!qU1a@-FlzWSKluu9?gh+S$_S z5`6zyDkGci6k>nL)(aL<_D5MCP8oP7@R}L&Z9VVZC;hnmY;Rs~<*SZvcK$bxiEx4U zOqv~$Qmt-$hlkOO*CQU;4a`} z-I_9hi8buL0^cCB>3(jy31(_9p*+P-D|>%bd!Nl1lVX0JAd5NIq56pAVg|9eu{G%^ zpKF4)j=49qx(&`t%UTVIac}Oj-?KZ3i0V4?^rCe!X8wt*$ooed9XE(5U5+XuSzkL- zkodGWbK~~ug^N!f^*S^jxs#s}?SZgS!$+GG2X4|F(6)a{a4Ea|86|d^mE--rf#+L- z*T{|{+LCel{?5enM{L`$Pt!#)B8grw4#})-z8MHMuGX%2_0;+TTweUf`ZDW*l5T_K zz22MweIA)QR@z6N3gza!%MMm(xiK^ES{Y;O%*+w=-tXw^CUfVV{jQA}nfJ{Zv=c9D zOy3sps?5KUs}>RZa$|MNZIR~t1A>WOLbhl*?Oxw5xrQcz8@wk>IyoP>^1kutcw70F z$#Hi=(VoP5wz@BN{F9fRZ}j-@Nl@tP)J!ZY3JMM8@A@g?R$`SKMPzpG%iGwz6;bKv zagFz*n{Txj_Q|&b{ixd1gzDhxHMrfQS0f+z8@rRlx3NFjr=BhIj`e`j&kP<5lZ#C3 zb9b)XkQCbrYkE=?(0bNROK-+uTV+oVr`ID!!4$=HqL}KYA-qi8c2sj-dixob`TPS_ z(jW9UDzjHjic$V2z?n93O3w!m3LmAeu^Z2*|2+;(TO@3Uqju~7&nic zvp*em74EW$C!SV$IwUCFZqKAu^>Du`kF9nxzcJl9iGv#ALbH>;o4lPYB3cz9luf0_ zD)og^S;}w?ubc6c&f@AWADch*|f zMsDbs(;n&Hnm;o*F^Kq$!+JpoT*dkG9C5U6gPt6Y(jf6 z*-YuEJ^p1&v~#ejo|p)Z_&)UbiVXkoX+ayOwdab%|9Q)0VzDp2c~h+hr}T_4r*H%| zwxyb((oR)E?{-Xa?)cT@qu)XlP`lB5J0wuA*Xix&jIG_yr-!|R!=@lBOxYyhlUy?_UzhOJo zp12Eh_^hMI)>EgB9z9#mFRL4A9@TMQ7k%+*`l&L*fgNU$u5@UIVZW^XCtD1Fa<6OlZxU~~257hWSReiRy5Q6$ zvjk(K?%dYn+mBkoowP2ePF4wbT1Fc3?B(DVtB{{myDM4l_K9^N^l)j{cF`S@PM@6P zf9cffmY7Rz8Q)-fRlpDHNa$tL#(vmwsb~~#EWt(axGR6%HT;EC%>fmM_d+v~H9Wqs zaA`)~y=~>E!oBe8!tReqMK&ottGJdWs0DYvcStyJK*RF|hy6Muy~3Z?HC)Y#jyjUY z-1WsoRdY$i2(NtCtu7A^$xmy1j79hv9yM`a-crQV^@`ndHtjP)?PXF<90q>g@G&Mo zH%odZKG%4(dsAgpsFlWZx0^zSzdk>b41W5}qeEr3S4_VveS@%#P9<~m&mF?9F70}? zV)3Wx=jFV2$Z1AA=F6y?e|5%*nIvw4jvb&Ey=SD)y3M9WDQ-kDe?-yFU*s5}!u~S@ zUWMV)5rw4dNpfq{YS(n^mlX4qy6^XG@av<(m@S`o9lw2Y?A`=E$l!JAr`ra_+t5ya zMnv~}rSB_*h~jtBGSj4FZv?O-9BH*kzP`Sao@>s>{Zx-v2^Xo|2;+Iv zGw5=t()zixv=uHWs(e2yVR4-IN8$Mm z#olJ-C$l_0cxve0og-ZR=k^`x)}SsGb5~oPS1kD#SVSa!vI2xRptD%-?l@ikq5s5V zj+_0`u=^z%@$NpHr!=3wVl%eTBTDLgGW4LZkq|_b4R(I#JZo|)%%#TH{jMb4?$<$49?-wZT^q55O$jm-|5nAJh;6@sq=QW zfa&_yEn+#B);9@@RSWmy*Y=;PT;J4O*!sG@d&>FFr6davjz-waz9`8@Ik_8ay9dWp z=RV!e;9!fNh=hlYC_Gn2H3}ueH}8HD##7fS^ELcS$>GLLlY1isCK(20Rp=~{ycaI{ zWwnU7oR1j~7}eKtewfk!z~r>yTk{?Bo}Rtu*?Qg^cCI})Qy_60Uw4wz zp6hf`?h^&ZR5r2aaTVBGHK(+SYa%=NIqsC8Dg)YZtd-Q;kx zb+)V*y!H`?n}zL3lSuwH$vnbd4&6lgNO3KtdW&1(d^xi-s_XGi4Cy{B>an|BQoOrQ z2p&x%=+_1x6S<7Haue~2Rf4g<$~0Qv`zEB&jPJ}wSD~Pr@#DjV9r*xjW09Jiss$WrN#8pMaxW$f-!H55eZ$s#Pp|qp#~Y2-DtVhAh__|YD8p#E}u=Mb7#~I$Mp3s`_FP;tu&j`xs1OcTC9Nk^7TMYAL)g? zSFP)QgJatxj|9L&@+{jm5b0~pcFCv*K4DUQ7WC$qQlCofnzD=Uc;5N0!;0?D%;UOg zxg)P5Hf&4tyVu`I^1kyCW7kS>rT0S8gf>($=2=`}?Ym6@!wClxoiBSk zWCm4+RaOgmvrqnd=c~i;rO)Vwsr0w9;ul1*Y(a|yGSRUi=IL5L8ZPAHcb(S|^)=jZ z?Z6uKa^X$o435Y1YA(DuZ;*sb!tD=Xj3sG&`P`&p{jK=mer)SF6It~^)yBEi*7tjEeM%1eaZ&O@wgi0to#XHQckJ{e1U^TeNZYTW zd)tn`#y^tXjpyeJ=G(>x@u9@c5t|Ne=zH${UH;4hPHi5qHh*>8Tv~i>N><9k@NDAu z_H#L(${w8Bh2j>Rh-xe6=9<$N-azc|3UNU6zVbe>IXeqmWBUB2NcEff)CK0{V;^P5 zo|${V)h`&t4yg`!6w)IAizI|B4tNYL+HkkG@QB5nh520)>TtQAmm-gAa7aEKEEW;^ z+`!@GNdG+z>uKK|lXUZiMcec|xV3NOoEw;*pR0uhHxnc7GFWn4v#C)G7jL(IqaVb} z5s9wzN(()-A+szu&@L?Wrc1&3*nMNMr2A^#c}macj+7J=b3D5?N6zS9u#By{kmi|R zer)#a1LbSVwpP#NDu#WJqxK3r6CPVQJu9v8t4yjI2@+{;o1f6VYA_nj!Lep{OZj1u za&h7HTk1Us&DOiFip{M@eg-`+dGyx5D}oUQlLUUXiMlD4HtEi^e{ad}D7GUtbNDDa z_?{Wk(7L#y?Y6@~cLB_rOaBZo3gjAM1NI6E4?B7@zmVy@ej+^gW9qiIF;BU-ex>*G z;Omaay^?z>N@hMOzUzYaH3fGcb;P#Dx&E-j<3rCizjf@iJmds_bP}U1KbbC6^+pN# zsfJ^RYR2hHeQ~<=N@4hWJYUD8f7oNEvYFP1WV>>>TeV#F$VfTYac|4ZmMtAc!q!vy zvDFNF!g30tjg>h_#$hoGMR%oAzibYbyHt?BTR#Ppn}3qcPRrMbDXi{PRstzYM4*42 z>3Lhp9q5hikCN#ni85%B`@ufa>z>^8=B+!&F000h>*0+S*gkXT$c1=s#s?L(FN3v^ zcRr*1Zr}6b4aK3;y03B-DBrbht~Gbr6~(BNY4o71j^XD+QU5{r2Ro%Me#w=sX_WHF z9ivyfKUJeya!OXiRzR}p&T-sYLPnNd2_|wRQ<-H_JPoa%vi^oxk6NHZ%0CwmJ~o&A zIQ_CS$Y%q~!t|c|sww7cy?WoB>qR)^#h`pQ7VA7=N@Uf)<-GT7-iX3oyP??9&1J;{ zoxG_;-3s}nU4F#=Mr#}9o&otJ&MRx4m=ZngE%y0(49k@Z?rtj`bRH5hdl?`jOPJxx z^KRc;-+3Btf3@2Fd~W_F)-X?veDd7wVRwWV6bsB zqf7Til#8KxGeg?pKK9^qcS4!x)WvS;ajw%S^ss5>=lUF|<9s>5Hu6R4n2MSE9C6Fu z#vX+AaJq~HKJv&7O|cYtO-*lw4Qt+QL+mit4)J(vj`ROKaAS1Wdd|7+Y5Z0RSiCCg z)rUPWqINPFZg)^7icXQay0ZxbuPNlc5dOxPAYuwQ!o+R9n`+ zN8A1um+tl|p2A3GPMZz7`)6#}uG`xv-55^Ci%u)fcsRF*Bibd(l5&!)MiQ?bO}Q8h zJCHgr^}*vy1>=uPQW2^BzJy+6b%InFuW|p56Ly~C!!KS6aJCmuHu{Xv#ljWatem|@ zYnl6W&h9$hJo4btI-gf%0q8fZ42Pn8UV7jNy{m+%;^~Cf_a9tPh-VKQXyoX$J*#KkRg&HPUC;l^S+l-QLCu_B z3-hvjMr8F0Eav?OjcWz><};aXzw}K1zKg^yRM=r-Ozxw&y1KRWX4PHdKG>bv-J1_L ztHm|XJb+i8_bGLBVVBc>SsLIt(ZGl=v*vi1eW|zM_-FixUwKSk#hHa6{}TfR(~a34 zl^ab$?zmhk@t*agFHJN^yH;vYsuHJ-K0EH@78+k97$~~ zq&(1FK{@5D4|A-Ui>1|jep?-*URGMd80LPZ_gGw$t$nRv*-6q-ahBN)be|?4EZpIJ zc!lnn1MJy#roDRB2@iga!arA3$_{c#_04_I4iZhSn5i-j(Hm~6J%Xn@-}|MQDcE-S z3h}0)-Bp|YIXOo|Z1zdyF?ai(VL0@{dGd5+%B<8+y`F{Ky2I8N(C6RLt7J`M@rQ2WAFKXzUEg(H2=wwhUtjewaAiOTac+ns0YgH1V9|e^eB*ao z&uq6I|FO=r2GfU?w-9X(br^}KRUB|Vxh?y5JbkrQYxv9o{yAU8A9TOurf!bMvFsa~ z;~c+0|E(kN+YILj)A(BNwd)c}BV27l^AR?0W{lNtTyJ&mc&K4`_xtcj9^s&p$)#uF zzgRZhmHP5BU;+2Sc5mHFIQRF9+qt#WLXHKV;U*@&74`Gf;40S0V;*k_nOgf#1)pW! zi@LqLM&<7e?X}QNpSf~kv&BSYyLiFwZ$Sfh@5kp8nqD>5KF-=C*PVOvC38jHPy}jM znqI2T<^IC{l74x)THCi!lpG8*w zXTZEli{_<#n;g__zrU+1{#P?gmFW&pT#mmwbPNuIF|jVLVxX1ckM~mw_QN*VBThZt z!qYIlS*qI?TW6*$mtfFixEo%#?@>{8@&TsF2jv?E=^T1~ebYJse;Jv0rD%`l9egXk zb=vELg!N~E!l{!gLO*;Ddz-7?){qI~IW8SF0W+;?Ikoe3@Np)eM?n{Ia=c_eHIE+Q z`d0gV*nLuDA)_-T5M_Q)rcyE@VmNQWWPHTj#v_e?T|`Kq4&l~wZZ{@ndQ!Lb&JQ83 zJZ&tqYI1%9%^4E61QdvSj>C}^Za4bn&&fQItIRmg(rkak?4BkMxzOG<+u%mn~g?kGWXSw5>9C%AbtyL=v6BOOA7g}$-&RFxRP-w(S%|+m2bTm$B`>lbg z=(85h0rFB@!@2d>o~Zxa?jeXoN%b>TV$N(GSi^a#fp?S7M_Y?ZCcn5d&Ejv5e0XNN z$uHM2t+CcT1wXZqU?emUe<7dFetal3v<*IVJHjY=Q_+B*# zSXbGra$_HzXzav?Vteg>LcXwG+Yykm@vzxxb3|hLlXbHv$GMBkcm88Od9XgsVtvYH z^{oEwbBQ@N1xD?gdF|QlODn_vi3@s|tyEj~O?1hl$?d|a``n-paD{~}zLfGe9_>%S z`=bdY44P;gX!+1gagTy2!rG+BZW5tj_V9%b7xU+Xd~y8ooctUQOX3k%I1~loF1z{T zIXE8HJg|w2Rhyp1BEFc<#MeHK`)tOpV~d@RdpyHCy@wE6!yC6p^$Bm1*^vl$I=Z3sOJ zQzJuLJ0*nCpTZcBFvBuoU-)8-hq{Ap*&PY+KYInJ`#6`~R|N<4CQ!EuEW33CeCtxU zql^$)hTx;8fIyOPe}UOU3FGO7gv6{}hB-i)pg$gbik6E{87fm++#^cnjcCBMMhGC=eP35X9FKVk$&+Yv?aKV5ER_c1LGZl zA%&RYyzr}&k=^XaT!AA0U|T!}a*50<0!1bekRgk62V90BnFFPwKVS6t$~l7Z#{#q`2>uTH(lGM2^wn70scLcL|w zY9(5zwrG;{e@?Ub8xtha2l(ykau063_0$05M1c=tuuN{PDy?#hKN%hfKw^P9*2nw# zBY~!irw|mBTbuMf03Z4RIH~3`#mcRJrwIAM#xBq;5V&A4qBOW6`)#LDW_57ROxG?U zz+#tx5T>yVEN27_Hb4{61PoxLP$bz|E&a*giL0wrybmru3|J`r-$SIARFX|;RY4Q} z+vOWff1y1jPzD&r3ju>EEu(vw(@tNV&F7^^3Em#bL0F zUEug5O`?^(I$8F_!JlD3jmq(W-Q{Bv6*9&z08823TXzmtmjjLK5&)K61};yh0ww|B zes$=E@k$OMz{XzzKa^PpEy#w3+J*R2U{3S2MQ#O{sSM!G8lDHmB;oz2bsGxvD!xB~ ztLs3sS6haTD1bokMiMco-B>)z7cd2Z1YwWRW^;jA;OY$B;xzb@KxZK}0SIjE*L@^z z069ztb~mmfC@i_ihB!|Ma?w+7{!hTwSvqhd@h+xDre+LJdgklo0NVoxccs}S5bS^Z zPFwWvstknet!?flFWdtp1n9xh$*~kd!T^oJ5CJbS1jdXc%5=wUU!fj=?98$vbeGXU zrwrc5&WU0Mik+~cKI1BB07KY(ymow}0AO!h2C&VCsvyXQe*Pp{(-%*|CYgu2QqSzT)9Kya0J8+Z!O2W@9Tc6MsZ%z1LED?U+rX7L9XLyOsfVEc-Wz8z zBVCC z)CIV%T;MqCewQj6I(KwW9#%*NOnq-fYW23A2K4`WGHKu^2xq@`G2G;sC=6Ds3--jm zj=zzikekr&K640Y{o+8Usj`e-^o|-`-v^1qp|Mb|QyqR=F95&~F@XyoG(i;~Km>X_ zDEd3;vZ`I=?44=xeg7r8)@N#TQ=BL8k0*5zR!7T393V2!1oqC(8EPU3yeArpZFN#D z?hW8R<10M>#CPg!0wjBuIW^!jz@B9U*KEU2YHTaKKeeJC^o+76Pw=c`1_y;-3sms| zL?3E=+_uS=Z&x)ifyi1W=C{2Jj4thA;%!0uSUyBx?Prikh2~-@loW;WGH+wGePif+u;I z{<1;tV#uLB(!HcObi@IWEP;dlK3nYXBv^jTnznr1y1T+fmiN>MOG*1^!<-Fqj0DOkx6`4Gj z8?@3Q-!Kq~rCiCI?gsyu0X~Nw6Sx*kkhIXLl|K9RY1?l==@ATIeu5`e{vR1PrTGTj z^}S|#qKoql%dOMx^*8+0={t|-?(_t#qs|B(G@v5>hOUo9`A~4Af+F7>ax&El^eeW@ zB-S3INrK!dO6y@sdN|Mj)c$I=Zwl^uJOikqL;r4gxKsev6aU8oAdvp6YbmwHrn|BO zwIvG9tm)!t)}oLWZG!#G0pj1>I-t6ip^H)=(ReQm7K8Jq9 zp#N9S2Q}y?@OCSLas@sp$);qVfI^m z3?cysj{@i_|AKGnkA`Sj+ymco$RC297uyom1VcmuCh8?`=?P9`TPi7zo%iqrxh-E?kuyy_fsz42 z1Wh>;S66=^5nwZ|$Tf6&sPJjXnYLbec2x*;CCZ2SQ?fMDDid>`vEcw z3z*#aiXxeareUo^Nz5y5ku@+_tu#2YjJ}~nru1?)n0?noLYCNg@+}!(GLo2 z@XkmpM%SS!KZjkSL(9 zqfl>Zd}Hme0B76I46e5b7V2myLxdn4uhan^@oL3-iE<8#WE>HXrNt>Glmb5u1O0QK zJQ!ZXO#yCz_Cf|=Ni_I!ns7(EAW(7#z~s`);A2u0;N;a(21a_6qD2_VOtTNLFmlBT z39XIL#MOCn^j~BD0ot$tC%D$E6`^_59@lc}0k2@<06)1k-$sK5g@A0BYxW$7oVQj)PH!Cu=gY0wVK&#wJe~1f$UYl#TwY z_j>|)g^}wD`^cD45cs|H3V|W|AxS77D)<*4*4Q2g@N6saLI{d@5}M#oKm*^&V++U( zYH(AG!X^L@41GXNa>9Z#9x6Pgp5-oO0V-2c=Fhc$WTKN51>zqkjjb36_HtnfK!g0Y zVwFRdI~45?8&H@dL_I%J1K1SD0v;dl^M(Rj;2kjJHyVHeK9!oocMYr~ueKWD2G3%9 z`chH=TzN76;{ymN3Jp^8t_za_P+bH#cwL?bkplPkV$uKBSkXEWBD+iJtT68SR)<@}U-+>XxU}F0H(arNC^)&{dZT{_ z_0_?m(PrJ`14*h^tmtegqX7)`bSU4|$;}zA1MNWfNyiC(Y|v3bjZ6qcVkxBA3?fap zMPRT4+gCgls-cBS(YN96whWQI&l(O+h~Gb^0;b5^61b5|8Zd0Lk1=C_()Tlg zQ;}Prpx9PO3?;X?Sgm;#FtR^AxCir{h9(k-XniCOg~no$i)(Knd-|^F8y|{+t1mF{ zoCRS9QU`kK&qb@qhrB%CBoo+AJikI=?a)M$4W$(~&77%M?5n96>O&Xj<` z9wLA&dzr#bUPuKBYYdNi|L_CyFAjBZhnlViiKCihD_5J!Uk3a_as7(Xv?(N+>aazk ztLNBlAP#wQfG2JY>?x56_}~y428_ELAK7RF=oWUcb6D@AfF^f&-Z%^mkwUxt?3ykh zA|0d$H|YUaXgI_g^Y@#Nwdnyli;V=h`F(Py3SAxfxglis4bVEmguqdPh@*t0R%efd zlQQCg&(5`CGCeDZI@1wD@&Q`5MF53iXZ@II{(7L=!kNLHbw~(B`tN@cXcE!@NkXp9 z4LDgYl?ar&9S$DOJ`bnBg_v;6nv(j5yzE;M4zB;JhoNYYQ;kjuo4%9B@RlnQ!4F5E zpiqO%M|Sf!$qV2&tYA7tKr$g_1`bGt^+x^sz<9?p=DA4fOdE6%0Rw~@X;T~!AxOK? zUU&l9l0xbRyZEMx9q_Zf6|cMmNB@PC4cZ?I1bj*==#G6eE`W*z^Wt$@f3)Tkf2H#K zhsB4d6hp_>6IDO10fVVh)fK5%0_|d}vm0i`__=__vzZ-it~1#b*uUS;{NKBN!n#qS!`5m+^D6VuMT{bZ#KLGr=;&sr*0-CT(dklfm zc1_`L@;L7o&|T|?yhE#X+8 znV#nWH`BhlztXT;{Ke`jW6CphI{^bZ0QGCXOr^T<|Dy8y0|8+U7*OHxBvV`<5S4yE zUMBbMs~5pY&}t32N83Rj0W2#5rs@Bu2>+8cz@v%4aGLyFEWi(K1dOU7P4dm@cA|nH z3|6QP4)nau{~zU9U5tU=;OBFo1x&7(nLa;2i`eh=L`ZLlEX4LwKxqnK+SU~>=Clvf zq5#k#T7vh)0s1+h@}gHfu_zm%iB4X!M;Rabi{JeqPFD;URT7 zz}0IQcsl9+D<}!FoMpWXheZmhSt)Zs6*l6_1`~K0wkbT4IOZ zn#c(t{^m=82SV%<)QA-3Ap-7oZLI|gSg|6x(D@04{NqeCO2f9S6w-z;Ad+7e1y2m` z`b8s}!XwY!kEF=Ux~x~MF=k+Apbfn2{5CKjWhU2jn-vDrESo9lP(DKr3@DyMiowV6c66Zc8RX2Pk|2J*>=Tb187g!C8UM{RV3vO2E0Sz5|(Zc1?(#O6)sU% z-_&pMiwCx_AZ^C5a*IF8H;XL0*?Q&8_&>J%ql}SThB*#}`ThOREdZD7wb~|tOZMvc z3+lgH0{#hKZ9ldp_(hAqgD>sQW@X06L}v5>f3_iDFc(YmmSz7BP)h>@3IG5I2mk_& zNmEUGotS4X0RTfg0RRvH003cgZ7p?lX>2VwE;24^VRDqaQ>-Y=)-AYf?q%DyZQHhO z+xA|zZQHhO+wSk&d%Dv}_kV6qrDp1-#!P07%BY$TLtY9P1Q`GV0s;U4Kmg$XJs<%< z0Axgz1!yE>Md^Mf004jiwGRO3-&^GWM=Lg~$^ z=c^3Oyp`X^SgDA^*g5}vvB#`o%@ba{_N!{wf@x=JK&MAyrwoV3EN0$uXNi~5z!l3s zDhxL4nhU;`wkEyt#&A@ZtBAWIZh!>6IK!tXxw>kb-TqyKV%jNNCN_rl9E#IKr#@gg zxye(hN*yp)$u9`shFxIiK18G8-d0XeCFGOroOT`upQgR{%!dtukDoPoA@gbbDu@9R zgATPtz={dvS$SQ9+u=g4Gc5793<|j2R)vm`fj9RIkqazh@^M8C1kqHr9IRO^8ujOx zgl$9-ALmD@0+G6cOYLijX(XC*38->ZqO)vQF)DU|u4+ohd-JyI=K0z6(0g4*^i(4g z5TijWuyI8WfN=%PmSsx~0&1@xz)kd~n-}T!L)labgdx8))j0^4j?sz8azna+IEv%$ zz3Vyk%KaB{mwpn`Zh!y)ZovPEIK+R##?IKq+Jwf!*3^#H$lAck$tG3?E|4A}WVg&e z+^=Y)TDQ+oI=b`@L8LMOK!)6W%YhaRw@(OzN|co`$`19hXWXmj1|C4`4TMf*lrf=_ zoQncn_;2OclnVN!kV6Uq8t1VIA_ zzp4Z+2~R2grv_NS|5PQ&KVj!+M)%)bX}Eu5>P-KP~M((<~kxv_JxxvcrU?SAcMvm<4a z===Wq9-68?&GDU~LCX2L?S@B~XTK*YG?1ZbHp)oFLo2h@LQ>})+1;Qo>8`Hvm9`ae z*pBSrj|xLC%vS(QXgnZX_Ln=cVGuy7Q4~095AwU;pt~(C42Ry>Sv>GS zHbFH}U8X+-1&Sfu|FRweD)JuKb<;ACm>B3NaNkhfx~!_5!Pi^HFJECTJw`)8Q9V^n zO-)m-DX*@euAw623_=$JVjiN1f*-JywfrX~a3q7dd5B!p zp^@aNVnW0O)E}?*xbi8#tfm=ufrq`y!qW`pZ-9Xh@@FL~fXp3W-bcp?%4a5nv7>nu zmOKG-E$bX34WqRXR3>6=r2(?mtsZU}oq!Nk;rTuSSEXwLHZG{`)KkYuM+-)@u;vGl z04~KFd$AF;)`dQlZpP4AUemx#BlIHLn>jZQrc_pmzfC`2#wTvZ#F!SA`DD^u9-+lK zEq;YVMqJ6_em!%He;+*>)x{-n^UnuVnQWfEjZ2<}Wqg?F#53uTBJDJg(1z*?;Kd!p zQ|MafkxoX)<(2y1VQjqP_)@ew+NXK+$fRN-f!^MBIPsbl&yK!b#Zx_9JaPqXxvQel z5QL!ly;>_{W_%k7vy&paKI&!7RpyeqB@yBFN+Ax|X)}10(7QTA9V@N8ej$INjEzlw zDG`FkI@!Km05DvcD9?F8%_(#F45p%{?&|h!+(vbzUgp%3MFs#gbBlY0l$@66`6=-? zG!cFV$3hTY(aB;|fG+v1*bJM**s*0ju8|YT}wEWPRtX6Dm-v#3o7d>$SNA0 zd>8bO4bkiRCCSR-AMllycXOIzDNtQ-BG@p-%I=yTQOgy`dglV;?%H*|ZYTvZl&z`z z1qu9>-7ReMU|O?Mk-lc7Bv3@n)&$uEez_1k8ymfy3eRE+WK`Fjkp0Xv=ZF`KXl|N- z6Hh-Eez=**D0X+IY!Yz0fYv0q4{Y~@mJ%qdh#w2Y(=4%5DeoT;GVF^A9_N6jYs|U7 zMD(x*b3_YlIZ7JvsBq z5YbnVR`nAp&MDf*mZBOiu+uU$HW07P9I>!R;^oJoT!iEKz4(G6x$|Ielbf4Bjz_Yh zg$1$7wC&t`X@5r6bhcxgt-*}!^pg>Ntd)$cGJET3{BR7_b(*pLkP7D&MrUiFmsZ=x zKB>CAm-?N)V_2ReTeL8WMR7`m@I|+0$Z0Oh4)`}GNM9G+7%>t5QO(Aw2hHQQPQ;#q z*h=lyDuk_RbK|t2i7y+f<36wWV&+!5+S4JwD%b2 znLsuo0V5kNDZHMtqQ6Mz9!Jfzgf5JKPm0lek|i`8cP7OhxlBxOJyAw_H}rOK<9FQo z%L?n1*zme#?S>Ytp4jxU6H`6TXxa#fh3G59ZOfdJ;&$@g4$o{n7uY&nIe6Rn@>XwaivOt+afe%8da^XZHwQYHiXKxQPZ%(4LC39#BfeRpu@) zsy?vV_^O_1R;0K0iv7l5P(D%VzLDVSRV>P9vOmV?RRi>>RHz4=ILF8NrKpCeiQ)$o zCMs%L=66?i{>U>FbIcW-j#jw$0{x%_d1^ggnIn`Asm(9#5E4zC<-F?#?*(Kl^AGwH ze!)o|-9SP5Z^W-hAy?*2o681^5FUY_Z&I8V?xe$%RHmPSu4SEuP zj6uWSG}$9CrsBd>WUMetDjM$FR%%^j^g9|8T9#p=JQ*jn_q2J{Yzl)4ysts9YB#tQ zHK9Gxq+N{0pTWb_P!EQBJwshBanc=wqyY=d+(rtnKGZv>lFm;8c-ov%!dJzoXEY

B50i3otmMbT6J9T0bTp7(Cp^tQ&vqPjJ;SU4L}q2>lw^MfNU5W5Km`*pdOpvqk*FwsiR`{GEuo$rkoZ2$%_t4F-j8}naWGc0&xw!G6Na zzJ{tLS@s+N%*^M!R8=u76#U|2#9@7$p@yVRnZ2wjWa5noF0SPhD*+=fU}xgd^1R2V zvG}w<}p}FtE%)r%2(|XnpT%9BUQ(PxYOAP=<1ukx@XzMY%CrUNUF$_y9yeyT5I? zr{|KPnJXvG*%mhL1lSu!*LLPMs-vCt(5k~je|pv}>*XFQC=Q?Q82hK1w51DI6Ay;utgSbD9`5Zd^fJU`xnWMuxwz%stYupW) z`28pJ)L@|#<1xSa2?cg>^0>)Ju#|J%GMw>wP|t2otYw}XJC7MfQkO}G%1!(?u73(p zNxrL^1`zOUI1;zNJw6H8znmkJfL5M^by54^P_A0dN(#-LA9 zQ&5tR%skeW^pX8zMamMe++L)W2_H^UQw;{3#yp<&SF6za_Qx729iyPL0%|bm1iT7n z(N8lNYZgs#y+GULOT7xVQa4;Zh*7&t3~GlPR-b7Q(J5YIuVF=GmVZ2p{yu*&F$9V~ z_eHW2v!Jh4@Of#ByIU0^=@^4`RV~A@=N8K8@P4eNVNR$$2!AH3d&c+#v1DWb)A*hnvGjrdpu)}p87~8#;msv2 z`$zg=np>(1a0cdQ70%8$drhs0E7s)AM#S;okl%(ggX>*TnQ z;tUVMOht6*XQYjqLpz;+LF(RsqFk_MjS{X<+gVy#*;w7PSxG)C-c`)c_eB~cS$jFW z5Ik;RWw1;O5$52XEjmSX%{s{v$*`A9i5)v4x z=;PF%O+yAMYB3;Ib7?0DV4+M|b|*a*J|A#gtjFGR^~D)ls%Tw}h|Pp6LDTgGDdZpW zIsg3_ni^g3s8K%AaQ&mZ-atDG4>v9NBp|JUntcsEhd6)zGJD=d|CdDzoV7gw_2Oi; zQL#M&HS_ZYgzYf4X+7uU^AjRXGJ8R| z2i<~Dj}TA4coH-=K@{0COT}KWWtubeY7}Pu?0DG~fv&;|Cb{WSNP!9lJd;1dyIVdE zj4OrmrPjtpkwT;Y?;$@;S4N8Gv!H?lse5=H5E^TSZt6_f74x*kV-e@tRxQ#YhF!M&|K-d|2*YVf{r#yj@)_ z)N)_@&q~TeMO-fSB-;WtT_$;L>k$t`9$pXM=X#x)5o2=B2;1;I();<0=l}uHG{=*w z#6UVPrD|Npn?`tIyN8F0$)gzx%8AGOx_(DR8XE0bHTB(t)AJx7bFwPxkqYXO*ZaFY zbU9g|n_noh@$S_?;uay@E|kESrHO~FgouI3K9F?pT%aDo?l<7i@Yu7mkAlR`*26;L z!0?{~S7`a+uZBQJy56Y_=E^R5KRADF135+hiiwjG)x2lPZ|QnV0Sm;8s5!@G{9Mn> zk_@Lwr^!=g_xEO0P|WJUL`k~d8O=OmH6xGiyne?5HCEK*DY}=3Opxa^naF~@K@qy% zVNF&yRZ#pBzzv%fpb(N2(Bnp@M}RxHu&W0_jw4zth_RY1tvvR5f^aVABedicw4-Cn zmmQ*TR!uwl%19@v2bV`5RPpbxg-N{lc7?keRN0Q2=DhrNSWjmBhTjMKE5y$+#n`Tf z!BS_KHQE9RtX^QFW~JLg1qQ>xHBVA-}*S{pYp3W7+gSNpMG<9v%~@1 zeD^ms@yMmm&^fsA7CXcD)i0iRXlu#7k4C{qv-DR{$9vU*>%-r@Uox}br1`p;#_KCB zoQAhI_(V26C+kiy5xmWFi^Okn|KB0M)J4ApwL1Kt-4|eT7D-(MclH(mAClsR)Kt>k z5dFPdf?Di4YoET|GsHEV_teV?APXkeU`vaE@RcTnAcX-T_I6IWZrDOLT- zh1KIJRY^CBI;-pFwv ziS&^q(cW?7%{zaW$XPA%#U$|Gb%4v``QxdEtD0N~d@6-tGv>!2N-%#&VW~vciM7n~ zyHhvQYy_a|E9xcDRYfSuR9*ttT^o&Sd35i`D~2ort2YD`7Nq0UYi3KTkjB?g`h^4~ zmE`z`p}O--TB!C_&sY>r3*Fvk9hy|C9vsg)<~3V!yJNnZ**&$G`@@%v)+i7cMsvCq z|BObVLPCiTlPfEA19_-*J$p*qjhh$}u87mF!LMSZ{NCzPu4y#GW1+kg3uNsL>dK|C zj)7xWnX+`ec zRjw^NzOD1N$k6%J#Mvz4$1liiR^#3W+PW?8hH}maVO6?eRl2J5@<3Hqmu} z#ZKSGcberG%MO`x&0c&cSlE)RwVj|7DTpKY=HX7>R#nzy9Hp{#ZvID2C?6s- z6KvOL9;Kk9;ulU)$Ah~o9BmCk)V2&qIlDmc zQCgvm$0hoEx=KDgqtn7zaOVS>H?XiWEbFEdH}zrtgf-c#Ao^azy$pwK9cu9f*6(bg zM^s`SBR}hZ6X7ION4JtmO10^2tJJ#6pwy2{Q~=)8Q~>An)Ib5mJU3Tm5MJyrOqAAYXsd3J>>^r8SZ?{J zF7ii#zSk6bs2yXPThe)cFXRnR$IY*8!uY8&FH{h{{VdKpn|u0gyo-77Bp^6y>oAR= zwc)kEf@A%;TE))MQX4NXfSIv3iGspE=J6v5;2G`-EVDfGaitt&x5GT`)Vkp>d~}}O zmp0zdTCe8?{bwV(IP;_R2NZE-=c> z6jPOEM|hdUd`qEnjr_?1F6wK39GqcTgAOf#G)7-lCJY znc0XgZ8Opsj%+ER!aQ)(oq0j-$>=|4p-e+_%=(jy4aZ$oC;O~t&((FFCJ%5cAZc+c`?TQ%Yaj@S`KiK(%p%c6QI1k}*&0yE!7Bt4 zlIN`G9O}wgnXKS((BPg);~g6VGb%={$|{;7FK84o;5wN1AA)gkoTruJvU2UKZxMC}?6&v9BZc1%j=Kd-Pq;&p`a>T#YJ{%Hpc{EFq) zZ9k&7<#HRz&Y)kgea3h4YS+Y`SUy904EW}EJ+OTyKNs1&5x#@sO8g!io`7>je-A{S z3K^Sa?00nkZo=Oo;IeCWCX0(a(aE2i?Dp&Yge%fnsJFNYoReMa$bVpv)JQHQK&gCy zk_e^vii|54`weCpl|}kguEsab+mU+!p7K5EO# z8xiJnvJ((w+=z+t5+!&fxV@KPHbAHL9|V=8kNt5bS#O&raAY)=mmk10sE zX2uy_(4KGK>hHu>t$o+3VV_pnq#xEJb;CoY0VPhB^kifNcdH8u8c-F;G-+>IIRtnm z=a)`_10+5Ri{J*u4#lcxX;0nB+WedV;SA32$MNRN|7&nBimhq9quU;wgwlrX3B3Z z#&6A(uiLJ$bV)DOUD9BIN{XkYiU%=#boZ@5rz6Ao=K!a44(H-<`FO%Qj~Bdb7W5*7 z-v^iZM%-K*k+}vPQ%x|WDw$FpW4W$S+YpAG7!-RF&L2?o5lBA>)rjB2i9+-$lBK^Y za0w454@(eorV-eTkV<|yIhfagw#MqAQ1nxnl`KPwTA7wXw0-C=IV}Te+3aIHEs_>Y z2@T+4>Vmv0OpM{^g1V|m5`uo5zVC$DWO=a4Ak9i24Om|r%F>Xf1(BA7mO74&+338B zb&BDM_B2FTIvcqwgG$5n8`vnRZQUUL#)*J5sZG5$*_AY|n>vi|hT&bU0)}=4#us>7 zruD?uILvIeVWQ!j`Kc_!etS|na-n1CUt7mgT4L9w9|>zvMM}f#_%%y$YgpozRM7^P zuApoIw)$yjwoSp$ zwe_bz3#F~=?c#AWg}H0*k{QQi;F{G>q@u^*E!uuAM<10$t9m%?KNn-qYN~M2GM<|x zt59aOYs5Tsykr)Zr=1dd=mGEqDS5~6@u*WR3U;mpu!3Oh89Fed&Jk9?tmA@N^@TWL z?fs5_lLS*N9l|psNn7pXJW!4tSdJWEw`TP4sM|UIFGi z0?W4}mTiG3+k#R0fl{^#3lvWXnqW+67e3IJ^K}YA3o4ZF^d?t(X}kE^gWGo_we5gv z--fo{@YA+nz1;}Vy24}6b~EQ*yBvcoE*44wGYh#X{z98iOevw9`4uK>iPMlp+O7jZlg#u+AzzIY`l9lP06 zld)sESOjZrCyqxSorRjW6Jy0GDKr4av1&~giF7hEjSE*^FHG5|nG&Z@6sPCYR+|}S z3-|Q(;4K5t3_VN!X8 zkfUyVU7N?qrlz9c&}YQgc!1}rXiG|WCo1$_JsuI9(Hp;z4QJ+aj(R68(t{gk^0WO* zE=5T8UlVFiJLN=!tNNcW@#HVe5I2XD)+sS{Lbk~~*({*I{(8%(y*ckIzi~=JyfL6j z&!6>791gw-xDv6(BfOwPiMA15#gz?Ev4%oIagPvVO-;&JtH@_V2 zw;POIH2+&@igY{~4DW=f+jO{HI@2`uNga#(Zu$Yoh$}^{VW!tQoSQENvbf72OI$z< z4;bMsc;i$dgGvjK+I(uEmYF2vkmN3S%|RKQv8>l%#rS>SH&g&rJ!b&aCsn z0h&B4JAM(S?vA6X#9WL-n^{sK%6G?={=Ph43c$~&G06j&=InMQLT~rX>8-G+(eZ$$%s=22JQ_^9f>1(7Gn5LXr z?IGKiAwBAs!d6CZm`M@Px6bbLiCT;@z_GH0^$QO|{rMnqEq2$1 z;zE*B^mb!LJ0|bdO&H@3oPEEH3laOzS`uFfh4R_K${EvL z+#JN;q`4d>!^mie?^ap{%%~4vo7B>6Oz*hYnWdk*B*z+pCp)vn9KQ~%UWWkghbE@e;IH_@CjDZr1taa zAHIcIWB!Iu@8#C5e=)ft|7Ovg>al0amW5RpRAqe2U;#_oUFdK~+SRufAM&5$xIj@$ z-4;v-H*||N(x~X#f&MK=+};F#^8x>4H|eTy85P2b>IH=Vjs1Go;*9n?Wh^g`m5?#X zCHKboMBePu&r7nk^%dssvER?fR+i-JxTTEj4D#V||Dt_>*%|!VAwq7x{l+E#0oeVd zd)vWFsQCuz{bUmUfnJ!&qhV`K#X|nwI^&E zd4(_`z++^JM9Tt$5D*FxAPSKv6oD~;hL1tg%^>j=p;{IpQ0rcK z=G+J^e>5BUnf7fobgyvnZs8!T{^ilvtnT2H+?4J9!*N~kS9r$D*R-UiicqJgB8<^g z5Th|KdR=}5`_2@`oe8xI1AHeM+>X?r4auVCuW;p%u_g^%G1G8z9(3U}TpTMG_r}$o zdE;Q=G-{mu0T9DZky2Z~Oq5o70vwi&t;6gOw*H;&{%GtAVxc9yWla;U)Kb@8y%AyD zCE}rJphG*e_*8)bVcdcU?{PR#2BGQVz9Afp5v@wNpeE(XJd^`5OJR>@@*G7$GA)7$ z++l3A_ukpnM=K}Ayz0$i_j3TzE$}Wp%b|})mxVOiC%;GR1C2Ew8Y~E0Q!H`1mAKvg%ak4(}M-b>kr)px_)qZv+ z?8E%ju2=YRqX1)+cx|x$w{!jatdVfb3<^taK>ZKB~rGb5L z$8aiDpi0iNNryg56q9hzHI*Rf!gDy$#pUyyA@g9Qn3@(>S~20Fv%R@&sac}=d;SX0 zkzJ@mJ7C9aep?poKFqU6P2oZpW}&uLXEAnh-28*Md<5$Q%sY z<9RwX_l8Ty{LaNo&}0y>Y#1<`BE$n^$w9MN_88LeMS^xFawx-P z8R%gS`Zp&;o|!^a2i|OgAK}wN@BB9Z8f2|=*81u8YFq&Z$;U3|tVz(V8~Qg_+gn?^ z)Ag0-scUX0R*)z4E*?n-ViQmcFF%k`5VJMv~c>qQCb_JQ(A+ z0}Dv_1v>JG5ggQw#Mnj3_! z3#<+}I&`iAKwRZQTl}9xF;9tY;|U}7Qkb}!;ueyaVA#qXDnN2L?Oj#Ieah$byei|z zCM2*|MDNU}<`;&OhR}qX*1=*FJdIk{(zc+$wn5!Ha9-WOlpbW^y^|)Tpl}_aGxJh_ zx%o_@@87u4vkQN5)3?wW-cuFJ0bWxL7ctt zEbBE}U93KuDmhsRw5oXr}=gJ{qm1~~g$F|(}$ivv|;Ww|e2(9iE9{5o31 z-A3Sg8DnuSf1G-xJ4Njh+0HP#v$YG=+C^)xF>&G|Eo#Kkc;Rt*+2<$B(!j|w6i|if zUo6pRPrJ)cm(9Z&Y}?5^Jngg2Qh35I6XKA%1yJ}@eyk)8^bmgImejt4zlUd&34H^s zepBEOe?uw1z+QjwK(==1@Y@Y}q{LyUA~v4$d$J z#cHv-&vW0t*3Tw`5obLyzSrc50%92ctRollWf~FJOB(qr7Cs*p8qv7j7DHG2UlpGw zke&ZY?&m1fFvr4^OuerUk*}`5h&ClqwyNSR)EJ8s^rv5DFOO!ZAeLDiEKvu~)Fs#L z8$alT!)(SeDDC2zj#8PUkGDb(vM6E*UWm?T?nf7W7qvXf>sQm-OwLy8~NQ&NoIg6 zizziW0y05IBGQ$Jv8NU9sn5}BayjH8+?ICpIM1`G3|pPQ7;jGi?|Lk+Rp9@QSx+{c z%HJQ+<{Tmi!FlYPAu3-YF9;E3h^`3Mj8POZ@eK#?WoCHBnoXTA>W@; ze-Rr<|5}t+W!JLgneHm)Li6GFHLFmpERKh6?6hB5r!~XzTuC&3>3e*B<0zIij#);S z!-O!W0cnB*(u4=h1^~D;z@^NVO0!pprm1Coo76Bwq%>hV~S=v_cmg> zrk^RujHap4caIue)^g7zY12aXYp0a>fl_IJH?&pwO4-$d=1s*u)o$?)W^g>gAjmqY zB+A)nWv{Eg2kmep5{&yEyE-EbN&t%|%V2jCFdq z3o{=Zk=k&8igR#}bXaix9oSGS9dO_s^rbRK?>KS%;O^X3EIFz zLa<*XWmCLUlCp8pJC<`g++|?!=zRajOPn$k*%jy-_)LL-O1nZ0{YtZ6z;nK{sHT?oSk`UTAEO4{oxMqz)) zI!>l}^oQkQP3yWIOa#ZNlP6RR DW1j}b?`{nQ3nUSlO(VzF6GNT)pk*k=p;GR6? zTlj>-AkTxn>yI%O-zsPHSc*pIGO&~Sur>$ysdrsT<`>082R`frVERk&gj#N4j*&s< zzxNO_Ot^&_BZthJ?t!t4IR%9=r4}Lg)HrtBa@8YIO}M76iT8}zCS8)7(W_^KhpfRTF#C4aN8DLQOx%|o(pdP6bGYE`S%n)*50yu7BWh1k_po36yXm8EsU#S` z`b06e8nO3utSfI3)6BjG?<2KUOK&kWtUrUPn7xfEr1hjp_0J{xrI}z{i_r#XnYym& z>rpypBlVjyc%6$^gSZTb>qw@2FAeEqJ1n1#uhW26Tu%6(JJP1VEK2TlPV{k1VGqJi z=zL?+$3#+ODddRzwMM<)oiao;N(I}_wBK=nfZr`0U`8YL6E^|+=O)HBc|BS$wsMZZr^}g_ z7WanMO};Td5?F56un;-nHQyL;xnj@}>T);en3+U;F=&k=5a6Eg$$`1tSil}y!uYvz zRWHZPYV{wj8|tL7O}pKx#Uu9S*RZ{3vH~!76FCJUs||kc!{U{58m9O%G zSowA9fcFXixLIA-Mpf7Eh6a7kfWj@`UQ-Kh=N9gdVq9ojhi^rij8mVhc~BU@K=vlu zXP11J&?BY+CjHu$3REb>OWM3Gyg9qb(6Soy62G}(*1VIuhsm=E4G`Id3e&+Jn8QRm z@jt-Uk?JdNP_f_wU1L5k;(oE~%dE?u0JH=0rFKu!zGEcpmqHGs zEK9uKz6#5t7pMhGbk=raiWBnm*Y#1DOM1jy6im1o2yEb$dN#(xTIzV1p1 zILG*<&Og}+&RHWg6;~Sp!i+v?TXYycC1U(Hsyw$n7%W4|BH07RBy)5DxiRwO@1%Lp zd1>*9V+BiP`A${-w5fkfCe@i}W9ki)+QOqL&i6ETEHA6tJen!bw;^jCdt(AghKzOi z`jqJ3(MIXU5E(|T^?g&U%plMD_D~#~8k5Os7whcy92;gieXc2_by+9i&RK5A`%~kW zruKwxarA~|7LGTQ-N8p*6@6M0ZIcBy3-W-{9Y_tC?X!pn5 zJ^kFddV#YzC%Ny>C!pbETJPN_tUrG}2>fQc!uN;5MhYQ|+_Ka(LB)DL*w5I^ zcW#dk9Gh-%k(FQ^zvo*w| zciJ)cxPz%zit(@b&eM<&haACr?b+J#sb@kg$rcbtVUREy9rt!q?^ zSFP2%<%E|0uOjJBIwxc^9)b21t3l!UJUHq)Jk;rjTMB%fcc*&H9G5tAqz`x>VyO1R z8AV^8>LRu)Xp)mx{2OpddcRzvz;Y_iZn0z*#C_XDSCy?+J$8pk9V+NqP>3Yd;yJ zZL#ZBWWE0O!p;z6&k#VJB3ae~=yn$QXb7Rs;YuADw=O4oWU|dCaZb8|r7$-LnH?E7 z4kfqIvV9mlM0=VJreq+D0eu=gLVJ|;!x_AyybOJ%>GK3L=avoU(mSX)pO2oQ-z(&6 zzvX6Yxnw`qGI*DsK-4X+EL=Sd8*J&F$!7fFb*oI@`) z`fu3iOvAcW7sOb@gYQuy@vTVxK55$p~l&Bs^!+KPx!-eN7kvs=CJchT~1uGA6Vy48ApD%Nn z5eETq_j?1;Z`djx;%Xyh6GR=1sw%jPDGn76!8OrpDW%RPffd|^R7fhv96ISmlyqkk z#ws2XYH%tkYR)E+6C`@gY(@%qOJ217wY{=IW< zxwX!8+bE!wA@Wiqu(Y_YN17T zNrv2pzqlJGT>2t!=0Ym@-m5-bDPz`UFK%6k?!~N?bm*5nI_wXP|EkRe*}VZml>8Rr zph{(Fk5n?qP6$Sw&5Afuw9p_*mhWdOJLvBJ1(wN8nY$}f$-4ml*U21x60i0BQ@8qm zbl5Td&nF{fZenC5U~6mVY~XBRXDeWAVDJ2Yo6cfXca@OTkbi9d=!6TZaX(d7*;N(bH234T`A#xY?U)DIA2Knal4fGDVLt^ zP-pU!C#CUBZaekfcJ6YVCi?w+U%LUE-US*gH8?p?bx$zOAmzkb6$^G=P20@`pf50)H!NBNKAS%9XM0Wet}M|Kk>1XtHChEXirJC4JE89oz>-%sdtXk6$bzU=Y4M*(hn))F+@bb( zOiodtZ#Vd@)G=@1nVnV8$!Z-p2`tRmXjJ2f^?N*&w6jK8iEP{h3hA3j_LCr{p=cvA zxT${brE=tT%dFGd8rRbJ?a(Lynn=w&DUPm2J9d7a?_(yI zbkObZrcb>G&DLLJko9Ei_C)A6d((d7f0bk31X!n-Rn6rH&7{e;FDo98!H1?b$BQrw z8FoIB;E!2sX>QJjwL$Sli34rxpKI~(6P;7-#H)gkX6PpH9;c^oul7-%hqlbuc2xez zE3wG8Mz_h6I#O4%BDEm1JDX%ImmLdLW^&dk%b;9e4y4Yn;3`dp<*3c{&es-q@=_>_K$bCAu$63dxo>Snm(?`k8Z>hWI%LnL6aF0o z3Xi;$?`a-ilU=36keM}mS_KJw0+lx$%%|kkogAt?!$_SxwRi;ema*$@X1)?5!{6<& z_vZ)M$(< zH&e)H43MLc`zs`L+5Q`OCp-@B?bI<71L8OQJ>Sd20MElvs8V!k`C+0}HMiqcbi#|W z2iS2Rzp}_460Q8FlHgMk%$o!_xt`q1$e)=e(U6pMV=;LY*CX`jI%9{hj@V{gCI9!z zDnc>`o9D5}B&CVl*1ll(^!D}rlgF1-bo+KNUU})XapkOn9He~w(z!X7IugtO>?&0B z_Obkfo060kI!<5n&w{T6!UlhW?$zpM?0s;K9Hd)}=X?54?2@a;8sNQCpHSE3w1M3f z?8l0OzI&r-Y}xt?hNmpiD-h3HiGV|GZQH66HUgil_prb?4$0R+M_&!KNUm!AO7J&L z^Pp5lzZz_0AucPCX+rlY-@l}}%pmZd6BGb|4EjGwGyVTTnuY!)#Q#=hVupX~a{Qzu zFavVv@1muaRr5|Q1TC^9;%E(3pmNB7yFhVZr!(f}k^^CS(}7X@AMG~)-Z-m!;|oFg z`QWUqudH7#3EkaXJpko8X>zN@7w1MQU@yANoN&AD9F=^v%dx!@)JXOns|YTtX-yJn z!sH*VT#*d=SDBrQ$R2#-kA<>J@BxYOUs-7wGsUK&@{gIl5JF3MV?DBmWq!2{E*8p$ zG*O1Gj(KwYAnHdq<)SNEgKp*DY5}3)2}@CqRfq=<7=vVbV=hWk>mU8kYqN^rrZy#z z3M8n0Glf)cGn`6m_PIImV23tC|&W0Q&100^uBaF9`j&w_6+1TX}hz@3bdv%Gf>FK+sm*bV@ku*JcZTI!Fj>&1)$L&4$JL^;TW75v|2OU5f@sOFQ@eOr^DN#o4Ajh2C!h>pu6`<|Cwka@m#gTh@HD4mc2skJU!Xm&_*%@Q;{*x8U; zje0G&#V&Seky)Q1t28L6lL4hJ!Oo@2kYVVcThC&!Aa7$OmoqY>zfZm4gie`DX{$f&3;}$3`6`#ty;_Q zN)zpdGEQtg+=?AJ0xbnL67$J<*nzi8VKcUgLHO*J~B+iVH(sCtg>udjjY!JcPKOkZd;LK9z=;@ zEG*a?(u$`_WZVacoQ#Mcbs%%Z(Qak$y)T(>5+eook;ItW0pvts1s` zfc5kM+UXM2yi+hNtJDP=nE_!WTXF`0a}&H`fEC`EkRv3LNL6HDwr;c4sE|!moa!0c z$VCws!DA zryN8DiDz85yM!_lIfcr3=qkPDC}*qsUtAgCgYZbyHL{C{_#I%2M1Qd1=ceZ(PZfd@ zy9Gng+>2mr&KM}mq?=*QN89o>;r&oKsIYIWv1Y>_DrJZFXYD2MNu>6DOU{K_pZMTl zVCYOP4MZW#6`_z6bUwZb1opKMxW6c*SVWu|lN+&&Ht_E^nmRU&*~ig1r3&j701jER z35%Oko+9v~+J;Hsf;b+(nS?}u_nh01muZe$`aexPYdFY5JEf%pKusFZ(&wh%HMa37 z8<=DnO3pKB0sC$+V=)EdSw~p*KP?m;y~a2@uhO>6z8<=L-v|J%r5%}>5T#TZMQ>N^ z>SQ`YaEw>TE%7nB>Z`z=GJa=XD>MG012G)UIbYb=P2lb0#xyGc2EToOnk^Idabr?f z=zw9w6JA7RShnWi`$tq=pU(Vm}Bjs zDqh(tRQsiU?>A)BrW{8Xc|reTe$DMQ%isl2&Y?pzLS=xw5zJXFgORrdeE0?BYbNFM z=XYVx@B>&l$e!318M-I31$RYG6vCrMh|h*J_?IZ~#tBbSOk#~T{b2?6%wm~O0Yt}e zLKf32iL5l1OY$Uh@DV8e1|*`klBPlet7oLfU&voEEq%QPKji2S_&cc(DIN$4Gx5?M zqmqkIdU)~koo}u|dF*w>0D?tBQq#NIaU^-Ed~~Sw&su!G`v>r1A?6>>ro1&M`#Kx@ z7$swASiT}5if$zI{8HDNan_|T(TCH}3gjub`{{UecwS~N9y~#V96Yi*%eJ$?6o_LA zN#KGogXZwB=#h~~Upj6rc~5+zaZH=&RlPdx7B3Iy!a<&)|bvlCy}cQ3plGY8lZX@Ig&I))ngIp@XVo{G&I# zo>8V7J9zzYjm)LJ6p6%tNne+CH0Q?}5yKeoX1_%5tGPJc*Kck#zk$BiE{r=9e?ZED zRk_2f+&a1+89@lagCsD#eQP7?)H8#;?MxpDIptYQYn^U$}jS@;^Euuls(d#~LGp zn!HG4uc8QCU3Gr)7mm14Utivt@-As5xthjSjUq zcO~-0Xb^-r8aSW8#QJt7o}MzxlorI3Va#9ER7jh|y*+RbhKx9qYeEEFbQu4bav*6r zsbu2{PX|!-iVWMEyAAspqqRLhM zBe8v$Tt5+c-kT`YB7K_KU1%`Ek2k)-AX$NH4dq65;+yi&BQ1mTU>l|2A2 zH%~0nw->YUIdB|0w}ksfhw155Hc>n%;&z zH%^T)>@5sfzjaw|;YC6{?#qET$QmV{BRp`0MvHEwRs&71S>(E%N2%L2 z5(#VN?XIImDnSz1OnsHFM4b`K%CONb3}6NSF^GVK7j<&6`b*^)G!+4P4|8`xrm zv*I6YbvC->2_8kJNTc5WV-!k8S&9m&5zy=fay-=LOd#7u*rqC_OGV+fNPP+zyxR`xCuDMqy{`B9ngzoKFWvQ>l zJBp8Mca$_)W)0>nlILY17i6E(y^4hX1WsbBuD!l`s_D#d#PcJ3R)zVi4mMxOewi@3 zwZ)2PQax?|)8ptVW(xNQXwy>uPG%ICl=Y=MqdW`MY;R(C_$EW&pLXg;{_Cg6`mKcI zTZYFl+DcRYx^OXa=Gbv)_bi6JsQ!JG{Azy%uRAJC+u`BVrQ-_Se^CFba!YXq5DFJ0 z{P8G$$kOvLW57#l*I?fdB0&pW4jz3YPp(P291Y*uyt2~;ZCRLtU6nfu(2ofWmay)b zK{)T#ym(b{Sb}F9*`HLEPCMSX6^e`L=}K{k>+oP6V^b|=&8g6$W$X2mI2T4u+HsC8 zhwh&j6K61f5`E3P=`ax6Iq&oiT^Xg#lY3dMFs@PIT9N8s??d2x5q~AE^$p>R9@^Sw zaZ3u~2uF`R60JLmRXsDx14)-ZJMK)@8U;}Q1X+??w3ehvutu8=yd>oE0 zb=;wL*+EgfzPm~oboh!Cu>K(W53?o@!NZWbx3uga1kX}8(P~XYznn00FKqy>=tMw4 zhN4l8Zr=$ltd2!Vp&^q zp~O*PXe(VGpPA&0b>#U+iJ0|_p=4cxLJHI-TQ=9;1n~!*gsFImV@7Dgtr}sOnMaVA za%X7WqEC@jkr2x7$}t0X!W45KD5;-8PO>ME|Ii8izg`*5#*#$!|)UaPQ}dk5?+bnBq0)Fz@!0OOH&8yJbNM+ z9tR~K876yNA8?FNH*JwY#IThfIcHC@{tX}m`#jEpKZO>`u6e<7AK`Aw|6a#Jv%{hE za#$LHptiolb0XL3eg=~eXv|^X|GxUu0pn9Fe#4*gnqsXV|*dJW~EwH}qK+_}e&SQ3#c3w+dX2&%jA=Bn+x&Akq z#d=tp!WI$va*q(3OUR@tPkV#Hw3tpbB}N7g>t9+JRd-c54Q{5SO(~n@sFUXu$llJ- z{633MLhCP~JCVX5?PZ?IxGvW>-iq^wrs!pnHJA1HL+h1t#C6>W^5tuk*H7Gs<=qP(_{3 zSxAN8l-IYtB&k=(RUY*(OGp4sU}^6nsrLa|vOu?aZ8=R5l_NUGGb~4?4|P4`fqR6G z-iZdugs;_Zz5?OU{ryT_QK{~2%2&QALp6NVT+ zD3>m%ZY;0BSzM9+1^Ir(3Z>uRw$}i-$F>22=NP)cGqI*|xe5jqQAs!PUJ4b`44oK3 z>0QQ?+oz|&kk99$LJ%DhmG<7ne=nw15jDZ_XN?;FWW>^^!9;>4ZHp*>A6XUHVytAl z8sxsK{Czg5YZmO6O20@K_lAc?%cP*F<_tN5le+%6SsJ3tBC61ZYGd@uv1<1^1qC5Z z2@7c*K~|>UjdO~i@CEgfH2w*7!uvj`L~_S9ylPm!dAJlh-;*I@W8NfW_oqcSfMJT$ zVEVWvc=c)3)D@1MAh>RsfXA#>zDKKAee#^s)QS>UcX#Zp)CT0CXO~26lh=WnTMyO_ zks_QTp|H~|b_qqnZS%VGXYICjtW$fNU~XXp)2>fc?%dgok_uJ)_k=3IzyAqc*>%KUAm1p?&??EW2QRUYFD z5F(1hVuy_T9{7c0hi8uV@Usyk!Wihp8d#>J%)PNm?C50_YnZ>wSH6^(SDxbeZQvxn z6C&JYI2BtS4J zuc72VCVnwE$EDtcZ^$BR4mfiVK3E>3`y>BnB-MAGgI3=8C3;OQ=TwLK>igP@(6)pO z&oz&MF&)C15tn$|PG=e3AbQIu)-+eaNBcX^<#@rUT%?|cIvs@f~VYlnB8~{ zJRg>4KiO6S(zgNBUr^@8(eU$r{rgXZVTAxk!$^(ya?eGcqgim4kuqgqMih7h5{m(e z`A>T3BlqNupYax?B1{0Wx*t@%iWD?;BblY{4+X8;JW* zh9iwH`0D^iBVs&g7_euQzkvk$?ilME)N)y&I&$}I1aW8d$dTcg4)LcZHq;bGT7|MH zfy*Q{+l5JIx^}hgb+LN3tsbNRshXg_*YrXuAnx?uM9Yf!l^6eb){9nC={Ifq9t~EF>tBQO60ZRH(6koLS*CgPMmDh#Td5(p$&(*v~NVrec zh}xCkrdxr-Z~9ne`>jm6{YS=nN~dIX5y(NF-O5sS^yE5f(b~iDDEw(g>F${al;Xs(DQ1>|HCVHAvQWOSpQk zY50TYZ%-jS26IkU$Xu1g##1=EZpUTttnkq-gM&=TkP zEJn~QL>#c-!)BK)ir2ysq!JfHLgr8dvhm3B4%(RBT~3)dSk5tJptmCjdFOJRz<;CL zk>fBE2^m5T-awUmrls!Pg^Bc`%RJGk_Si7wLWhb?qeh~e&0w6Oh(ztPe%zr?6M3Xa zf*&Tz&W&7nxS(EkBjB^eQRTEa#W{+Tv^dor%&=YKZ-=DHnYB$Zn6p??vo1C-c3Gp% z{qYavrsX+6wK1BX%_=B}wYjZ^X;^EWIb^TGxhJs?WrKOhR#XIRDqL}zW2K!2r_brhju!haCZWoO>Xt_ZwWoQUG%AB@R2aRN5!|DIk z#I{hbl_i}m$EqdnSuwfN3VL|@)|*+@D*e^e7hx(xweUFH0n5tbMJpg-E{GU>3!cR} z(Qb4m0}D@!c{J?`|GEip`rA$FOJ0)R@oy}Q?N453C#Zr6%`Uu3^YV_!YhZ!@j*lVl zC#g*Fv}SSXW{sSIG?gM|eM_vM@LXDZtOaS&n%^2t=!Y60x*;jLUr5*s6=Pqg5wI7& zd=JQt>Zf4gH&=Ppjw#lzWdi8M;7ou+d99ne)|D!ky?Jo|fk;q&jq_|5x)mpR@XFGE zXvvaQ!LT=eLxX8#KX40g=^uE7K84&L5fH}kha$P}w{x1Y5qdDjTXVrZ!W4_G!;lVU zmxPVuk;6x~vz(I1Fc9=hzW$CL^quYb6KB^FHD)gVqk?VHvIj8ol##FBkk^{YB)yQ3 zyyWWQV)?fdzsco=>0u<>+dV$eejLOlqUJH>bViXNQ%Eqb)$16nVmbiE1oaRGp)z8q zNxq{<5-yxG(R}fZVHNu20BfJ7os38`QRl3Fe2jSke;JRgz=>P|8%kxAw77C%$qRUt*_Nslj{mDqHM1%)Aw$X1 zP@4nbM%gi=*d8Pd(VY+XGmROdKE+?wRj*F{Wtye&hvr2%ydc(113bWhp})bI;V*2q z|6l*F6kTdGfi2J@+mR2hpsCV^9aZC{D4wi(d0@FzG1ad)eU3=aFo#ZRwhg@tTeh$C22S$B=Y~w3Mhg zMnRl7>5+1c$}2+={t+l_e*+BsBbI5{dY#T*XwhQ*;oP-6Yu=mcwu$;7r2CzbXoc{8StcSX`|DyJZP} z#K21UGKpn>^h}(M?N{EcAGMk-Xc+=GXa~hp6x|L z2xGy?lc6%F5$!AOFluALn55C_Y_1WYQJi!}P{yc1t?0w5)(z$)jpVSoSG^{utpLb}D8$l-PEQZMw%a%g|1}Xr=x&FBOq-wm5q?d(X}Y)BEW` zxHA83&bZv?Z387Bq4VKj@NiIWzRG|eLY?I2Y=|JHo`ybvxud2r>S>07NnLH!)Upxj z8WH}_+7R{z-L)j_oe<=%i6l!jIWL5lPYQi3mF>K{pyidW>j7hwtfo%|GNo)|)U4_d z$|6&X`<+cLRD8)&bU#FgNELIPt)EP(X|YW#0NiQt!^%mYHb;UrFv zU+3aR_HFnM59rH?$N3(F*BEzzKKhp{^{jl^zak3C+UJG0E0)eb71qChKQcXi4$tN< z^9yKw+l%phwccbt?h*J9o{lDm3>Fd59p~3w4`CB<-hVqwdomE0oRg#bMdg*CMI?)L zr1ClrKu(BtYjtBQs4|XV%qZ%W>qp(D=xdeiN8+aBI;A$_VsUvG=VwCHg{5YawtuF!1KhP{I!) znks2eo|04hi)b~$Wc^6BbSo9LHk)*UM{@cuKt!naTT0CzdiPjE-0(sEbua3L(u z;jb;Sp7*x-VE}+> zA)ShOKzc$(si7a{HD$BL^Z+)2M7d#+5lDtn??onN8AhF=fy}`|_W1(eZwZz+15nT-62I z4fE%>|Op zqdO`%)~*nlUUB*NTiS{n24>RyAkG~IdQbH6*=A`&;a5oRg8-4ldu-_k*NpNHyh|EG zk@`Cf@zgq!T3^t=)Fa23=!L5C!4zw5TqV4_+{<8&mBG7oZm9Sw?4Abmp@>VQH&Hy8 zUsb-lKh+jEJdtO|4^_I8g}?Da-1_2Xu)kzaREN0K`nq?dC;7nC1`va8j`5WhAmMc9 zurk#Kkr33DitV}&3sE|>24Uhqa;#BERx?s}v78w&`K&M`{ret5RDc<>?ap{@gnJ(m zou^)FD+WK+7Pjgzg7; zh28YcJYJ@9e3%}Qw|}ExI+~?g{P)#(M%2c7nq8B3>aSX>N~PmV5-;RD=8N{F4sDdp zxm)t|B_cWxUb&m|@Qhzk57!W!bTT$XwmbcJT_aKo4!xFJM(|cTcC+0(?6iIM zmbpiIL$CNhzHC#vF65zw5G)0Lgq+;9(>)DUGB^bje``csGmp)q=(Qs$JpOH`e?}EVOyT#u9TdgXCUH-c@obR-X>pvLVW=NDO1U!P%<%Db~b?VYu?L()t z9MZ87n@80Ho6CF{C~i;~Vl*Tj_)OUvqSpn}vS*sa4PMtDe)G@GtCq)PfQV(mC$7~E zyH_T*Vr+xxW`7CK6jRk}o>N8sOq(vFWZ(fn6QOd?f+`3X%Q) zv6P5gnc15DZ&F z)4OP*Q*{|Bj(`E!L-cuyS#2&Y%=A3j+sC08^qB3@uwpIb9&8Ac&Mmw;|jO$Ob;>bI`k2W2;-?4-YU{4Kt%FNCp7PTu&rZ+3c|0U1P0=$mYXfqG)Sn$I4K`1 zoQUip_^4AfF_UR0!7Vi%U2U4G%>9rra62dad}St>^v?;0{az{e{o9vT+kjn9#69z7w`H#(~abwzZ^X3ZZ+v;cev35tAf(H zP4)1uUN>Q+MIZe{vG_d8f^LQMxwhTQoUJ@glXEXQ9CKmPS-48a^&3!UJV&#&7P39n>FMBPzXkA!gH5pw~1*=iifU&e5&O2d-c8^qxqlFistf0Mmh zWTKl&O?_NJG5tTP!j#t-QIiYRCwI@@T=O+K<=jO{Y8l2IGu33jpDEgFbsDg~qs?6P zsCh`o>tuwRi;?`ED?38oYv%?CLfXLe6vSJupl7J#fG9Bkp+BgiF7UDa?_n6ANKdhp zh23C|3H(Hl00|hkY^-OX^H+pIanwxGx5O{ycPr*pa>^76a&zp?C5Y9PQ?tYN;sH2@ zT5yapHC+-UuH;5w>}B}#VgDF@^Z>5Kf!TvU7yCA(@{B1mvI^{uH9>z0bfV}h2gXON z(=_YcNdKAs0Q0L((jSpXQ>G-@eVYa>%L??iFmFQNoRDKn;vFsX_$~GwBoQ#V1&8fG zFOhC5o1Op?O8gq`Q+LNVYz;qWM{KST=4abE5Mn}i&+hC~vSYeH$Q4lmM1Js8uvwA9oU<4m=Pg!M)94?ifAWp6P% z|M?i{VsL&_%uNRq`y~4W_cZO@Y*Ai=6-7Xq1tJme1BUe@@Wymv$BYgFnHjyvzhq^k zwXLIPQlY-Xnc;Uj>f8nAF^^So&Xm+Qx;m!E5hfbkV3{h{h@%XYeuR`0PKgDL?+(;z+#KT3=H=mgQiBLw{{Jk?UARrl-{|=!{{}+VHnYmgznEoF=%Q|%* z7gRO0-@nQ17bsFzMSy6OLdj(I&mKout z%w96*m*Px0=bB7cTQ!;ZxtN`6C!=|s_XA#r&7mv2yAUSaN&~|jVN$+5XcC66jj_fG z{((gu-yoKe?A^r#_V9L=>iVrSfO+wSt9ZDiwJrl`OMBI2SP?|_$D96&)vB`Ha4oIV zEAClY(S|D$a4XA|?!LA|(^^{dUuNu(McPL0*|^_8{h`0{hCJGhAx4M4JKHA3B0-2e zSiUbk)=Z?>gj=m5k#Xr|CUCTLVZiU4a|>GiX|1`52`LXw(M2`D_~8S=Gu&loSnXma z%Wx5Ec}w2M5;c(>V=9xK`hEjiUnz8XU$*Ez~|)>2uO zx!;L3v1)fwyQ{X5os9-#O(JUXbH*bsE5ma;v=0A;c@i=W{lci?5BxdS#qZR&sB)(D zdKlG@$mYH^*FY8>Z*P(THVL@A6fvuOg>dD^D*>`ZIqT8JX|_u7GNbzJY9C}hbw;X) z_8RMa(i*)s{UWzCbCQ$_ysn$J)zo0xT}#jAT7r@i?2g(tom=W_mE`-HNj>EB5~cM> zx%Wz}vkJS7xXDgCj*x#a^#>4nEL^jVqDT&tCNwb}7sdh3<>I(%(=r>kl>7g1?T*(H zaOKp6HjajxJ%uEPjGvFNLI-k#d10K@rOAjCendmK>J4;)gospkk|pg~u43YUx=ob0 zFIPU>53oWgrGQd8N$5NP?i^QW;e~3;1aN<|#QjA4w`KnA)QkB9J9pVgR5`;H#a72a zS8&qIP5)u0loR8MOf5lZde`x-5?3w(AXIE zAJKnd^WNZyY;w3*`os?@<(c*`oIYZjYbkJy`i`aj96ceDkMx211uUWc;-0XC$JoE9 zpxX;e9-xd-4ZogiB<|*uizvtR@SIb*{KgVX=_}V{mi8DFXo|U$ALtA|6*BB%zW7CR z4EHhHPsM5sPMWp%$G$VS#!F&}7sI6-Gt*|kb&N3ovBygNjt2UK*SqqNzCpaQ2KtaC$6$KrjLEv$H*qGLy}V28}FXs`}wUQ}s^ltjm-;1+qbFNk5n9;O0VCd6Z|M!{Jdz}ALU#}5?M;gdP zdxp>|LT7HRPLzfG^l7N! zcT;|t9{Z*X8NymQ+S$iE=mSYKi-#uspUN#|L7d&kkK+Y$cI zy`4hs@69<50Oll?EbCrt%`(p?>a>}Bbm=VZI*fkFKhH=>RYE(-`{Q;H1!ZZIzT@v! zD;40w%$G87e=*-?#UJ5zC0Q9v0I#-f$jUXhI4c@XMV1d$#H3N!Y{Vi%Y~5yvViU}! zkg}1M8H+c8R>(qFLXf;D$jQjg1E|Q+aHGJm#6!$O2%ge3<?I9{vL+XFl2wDy&oT7#Djdw^kySvJV288TNtzJRDA;<8y*#a8bK| znx|Z^M8>J&;}Z(hBYph7VHFEZr+kAUCb2F^k4}t}ga5HXV;?6kTY3>{5hjdO1 z5<|*Fr(`IGI)wJh#nylupr%+4?v2E1L$NYdV1$yNt_E+66l*I+N>`4bbA4?05k0`x zF1?s8Ca}fsbG0!@+^jbVz0y>##;`h|mVAh|l70Nu?wz;;y!}_^uYh8#^8f@0=oS3G z6A{Dzi-`WGdfI&d>&e{RpV!`uEK<-?MnUXYN$9N5-JDXQphQCyVqiAeWFjmnawq1Z zO=Pg{4I#2X<126?U?tn`X1!O3b+;Iuzk2Ix9_wunc`mnp4!fkNqWyba{XH&ceL8b| z-?<(c1-_3fOhB2XWXyndZ0pf4}^@6YS8N-M;e4B0Ngc?*VlWW8$Sf9f>(#qLk8#}U}TaV^tMW&;5g6| z0(3e9JslH$ot~~2TC{&136uzF0x}LU{QcGP;f6%azt8{Qrp8%$nsLK544h=$N=0J^YI@*RN z8WTeUEdw2c1z;}^?O^-pBmBemlPI8h``X$OK9nDYle=%8XeCci$1uZCS2x!{Imtjt zS5HSpL#;2=u!=HU_uOYb&uu`tUt{y)12PC9K+hN}w6%1Srq*V5qSgjJ+GaK_BV#j7 zV;z%0UsFA}nSYfY|I$dYf&H@b&Nmo;YOY{2apYhH>wtR}3I`FDruL4m)>ifoK3eux z!*?aJlI`dAV@?5Qfn=boZrT_73W+nLQ9&fO{w{JIv_=T39t`)gy8gMneLwS0F&1g~ zi_l)WlKwHwm(Jf*+{i|PaRx}~x1A`&Qn0hJZP4J{#{Q*bqcM#Vy4a)2A!L(^CprY) zT*8-d17FpK6k1lMv4nI3NpQzeYtoB|M#j%>;>Bx`&&5Pior;cSoScPhu(5xUAe)8Y zj6#do4(Z6-PWb}ys4(|D|A(UpT_xM^`XEdcA*?F^H00P&Lgqf4o#4kB>DIA8^@_&U z&I*=jm85r$9Csx)R%pj|I~`&R8r7kpKBWYOg98aJ^o#|NhG(LiI-AK-kKGhY+c&Yk z8~Wr5L|Wa46{5W{?N zq!e%!3xMER?YU$;x$PHU}!W@yCE{x&#$4GO%n3rj4W6&M- z!-c{&_zgxOA(WHTci=Qg2TmGC!T%ZZSzV`x0j%{~#c|D0l+EP-CEYbQ|&Ni$!*2 zO$ErF%HVdgfj9>3&}=6Yp|4X4mCzwl-MRDdmW7up&H5Zw;Og_4kxfcrTW!Oz-Z=N= zRd=sYqHg~}J7Qjm`!2q!8mH3i1Vq8U4Cs5eBn(dQ*OLpZY%CrkhTH9)>K}_Zq{4E8 z%)*P#exek{=qlwl)WM4=n@YeDVXarl%u4jX0`W@9@S1jkb@%b?Bd2aS{v-Z{;ED^v zL0tC$#pH!Zpcp8*1M}W;tl{!>bf0Nb7!ulgK&r#g88sJD0J(}7hSL=$M^DnbLqmRc z=DZBfYm6~H$0Bvm>?^B7Phi>*$O0ow*a8;_e_CoH{>3(sVCe(o_*oMl1Y;gRhK?gL zJ@c|@=~E$k=7zpYL{`Fw0Tu)TZ17n$(rTc;OX|JsA2(=-@<$_ZRid5};UgSRLLn zsn0doAV&B;8zsbkZ}C0z6Q@*6cFf&Yjf3aijf1^`;<^j5$CAi@kAmN(#H}7QoG*=z z5sMfl#+>fmpqTY=Q>YV63f3Qg2`r!)OA)BWlir7bMv_+*T}?H~L2LG3F*HCDtPOpOUPoie7W@@^cQ9Lrf!!?J|?z=2^;rouv*)%; zQznB>kC97bs4x=Vw!tx*P*hfY@D-*H^M_)qtHu-X|LWYtr^pEs=WVv*m)4QgEK#gv zJ2q@V*|3!z)IPtby` z{GS%!j)Oxo)uO-{A1mY_vYGdGDEV-S*lpkCBxqiQ%~p@#DI(XlEvC-+Y~jbv9F9Ru zgBWS~GhaF(5+u7+OuHlqc;6U)MggbjTQa?Jva4~(u5JXr?v$3awg4sn#Xt5$8q>zi zPV@rSQPfZLQyQBQql}g6*!VixC$VyD^Ixbh0vzC+5WXJWP z6)gAyQPBkL*pZJp_a@l{ffd_B2z*){UXl08%ZQ*)6xVnxqEe8~`bP}Au`+&ThHY&$ za%aA>eYa#P>DkEBUFH3^oMR*1gvbm@wQn4~I~h5WEzNfLE>U4@46U2@P@nipP>pPg z>XxFx2DT_9vgl}^c-OFvo-GS%v>)F`P}kZTBRH$w)K>Kq$;WiyiTh4(xXdx2^ff@4 zS5?5LH3!4YNkS%v)J9!4gK!#Il)%ghgTbIn7W(~KdE%BZf~xx77So&F%kyK_P`K7o zh5iq7yHS74U%Ak&YvYdQ?MOu*q~*%ywvg+RVhzxbX&)^e;_irRi^R2Ce)EkA#;Gd; z67Etv8fu&tNGFNDO1EIi6o=m-J`B6&}(Ci<4wy3_wHQNxNSGvYp*7}?DLKjYf z_uQH~HhEmrUy%a5X9F9kg@#G_D3)jw4NeUh4n3mwWBUs5KVhqgNkk=|`9pYZoPo+h z`!?Dn$R(Vdi!&b}pP-(QrG0&f^@=)6M@D}}eq?ta7~rHo^pwP?m1>BoqHCQt*}q2Y z1KgVSqCN;dd8TAnG+WlhSR;>5=wFg3LmC5$2h$($w=~a1%rOi<0;L8ofBwAl>5s%l zLsT}{mfM=#aH2N}I>cIfUP2|f%aF^msGf~qDMm^XnnDi;=w6||P~n~R1G>Zl@?07j zP&PA6&EfC0FwFE*yK${ORE%dHw=GA679v?v#EFNi)d>=?SPQZl1ki7P=)h=+yvwG( z(u|d-0KNzDiF>2))t-8DM~rR5nPQ~v^f#r#2PL?u3tr`^lMYLz!fTM0sT|hhsm4sc zxhWJ~onvmrAmkgL+={WSe&%im+3f{cepD6=)OsEd$u__4N2Crv0aL0n zCFf~0kz%4L?5gIl$fJnu5GQUx*um!1gvI(@#tT=7?BO}NJJ*L*zKn|wl~}785Wzpp zMw_;3speOzFLRU2dc1OANW7@NeL%hPkd6Y66+>%dowOx_;JHm9+m<# zj(1UVMzV@VawD2`I5vcviCTn-OWQKnd+VNWtfDn1AzCim2G7Fjc3MNy|q_-16rWMi%*Gfn97ZB^42omn+=z5_R}a; z)J~>#eYqIo&(wOfb&0E~`CB@NMl~k*XFB4-%1++Z==-E!azl!hwDsM~V+tsEteLo} zs15WLwXY8OH^a1=qC z{$eD1ntF|@bTC~}aLlh0-D`hcU-yag+K9&{6?0OB8=Ld;Qg>ZIyZXX9K>4tx+Zh$-Jf8?E3%8KuWN*|!jY(kFB&o@BB5XQvnp z8#c44Ep7COul@F5$@}9mZ)a1WE;-~lhxE`W%dg^M?g2wGEuV4}38-SV(>~tm99!^J z`**(CIj>wdE2wt8$!|t%S{7$Z?LFmv(L+=`la)+ztC>39{dG5f_V#Hek#&CHgYHRr zanoV7S9~OecrEOqTv0m+zm+g`TJoFl_Eka3Yfu@&p1!RH+Z(+-LtL<(QIUAcvorrZ z98Qf9h$VuL_X9I4`)#8C0Tc)g<9po4h^+<_2%cCguR_pW*^H!W|D+b&Las`rL=Ijs zG2Vztv_52guaM7ldo+hyp$`I%N2>QmyAYV_w?+#NC7@p`s3=f_LSqO@mgK+#FzR;S zxN68b-ydBmfaoTg)nh*&?#~&bSAu_m!l(#a5HJIzUhpLPtz_u$W{e2TV~_Y;cxx!1 za|y2o6Ov-;Y}Q&yyf`qjX>u~)7j(W;e`7H||9JIE=!aN8ENajRh@&u+oP%vuNn`vE z(%vyR6QJ7`PA0Z(+qP|+Px6El+sVY5*tTsYt3BP$yoezJ$bL-Zv-n*-M*Q);2 zUA0!#Ufnz%#nJ6F?|r@dsUP2{uXG<8(J)>}`k@E^uEK=PGXJ2)%|W_nFb>^FCXDf4se_Z@a`b!HIHV zcKWg0#blycJ0yTuzIqeIO8qV1B!PPWo7z@|lQ)l&W&>#QY=N4a2*lJ-$X|X!*PE!_ z{rCmGI&)T;A2}^SoiiId!C6iOAL5@R5O8*a3%ayt8hI_-5${0oO_hBHh;}{q?fc>H zY6G(5C%YK4C3xm`>D}~Ulw|BOtEK7)WehE25UmIDA@spkuT42R8*L`v@nOwRjry%M z#?MMY%cq(xR=^h@cOFtn9v)kyoYE9u-T>0gJbM|o5bz5TM2t9_6w$VL00bLaexYuP z1je6g!>@Gzrmtm(-M;L~9rsJI^>cdANT+3o?6!5}*_)iXgNJQ8-7Ye0L57ogAdh@h8L@5n70Z{t@xc`kk8ML+vFjitwg-GoUKO&d{T53;i$kY;R3K4i&8a>D ztIiQ(-At?}oZo?l-$WHH++coqTzqCFiPYpncg1T0vH>QxYKT33jZ!w1I-y+_gZyJe zoGzSnOrjD)FT)b0qvKzOB5a}jFc8c4{*!m98*;r>CfTH?sW_}6T{bo<9*skmwXv`t zewH4VM;27I8Z7`Jw#FAXwv7?am;k384%A_vM5T6~gQXR31Rwvm-FWD~bmUC>w{nC% z;=I&@x=5~qav;#98F)t>e-~LjRakSuW*n1me`L0ux&uD-&4iT=ETAHtZ_`mPr0GB99wz>F`vim)Z%aJUvdaLb-e zHp2@wUa5Bz2vCv20lV`^`h7>8n`h?_+qib3Jn$#jc}=S>)ED9BC7$-kpO&#puZ!@D zJsk?zi#;9>!|;ADIp9hZ;KH(Qr5BESx;LJw9|;|Fx|fTcO$QZZ?jkU?iSX4CoWo?9 zI|H&N>ZxWttMDnEBylFZg2@BN$@<$h37P!&$m@woQD=B-w^{t0K7ibng zO2R9M!-Gg%U!rZ+WK(Q9%y(&Qx|QR-hKIt{Cv%OYlfJCC#Z@pm`P+=Um?>nRFEhCf zR)H(B;WmQZxtAvsJn?jKmxw$mjB1}ONf8if{%KFXoKK5S$C zpaq5Tzm~lR?&3zNh;1lEcyr+g{-S4MN7oJh)9U$~q5hFo#8%s(8$QAQmCv)B$s9C9 zFTGSV$I47zDbY|6@rt|t(60Gl_qEEO3jIB6G`Whu2ZW@sCg)2=tQOP45wdXs2kJ8p zov2@U$xMZKUzO<03k2>=(alJ&KcLL^1CnYBq&i$%4-7|JkUm4vb$_#P4x9$lq<5NV zZ_6R)y3;!o!54X!!=F}jZk@TA8aXNEZ1A*)o#|y&T?|Uu`Er>R;V_AqW;jk(>pAaN z@hqk%Bm!4~zG;0g!m5f>BFBqbM5fb>>?5yafqvwN#!Ry8LBU(c){G=Pr-02vjI~~9 zKKDW;2tlkh9;7u-NVB^i)cZqq;Y`7*ew0ZML&|-ry-tR(bRIla`xCqTeweSXPL{h% zv!gSi4}WD>$7e!pUOjKIUqrZSf4Tp~ePyr@_P?ck&B~qOHkv?1ro1D@8`IMv>n%Xcs!+kfzAoK|u zG=-Yxg6iRDc`M6Z_nk7P%{NLG&yhKKVe_(b@*jZUd8WmTjh&jTjq<-jr$Z4vafYN8 zn%YYa{rLfT$H1IsFJaI*U$WUk9TI~E&DP;x%=sxQk+AVI^LVcW2PuL^S#A&Fi(+qw z+N2kB05qWV&K?M)&2&gpI16fT;OK^IsZ4QZECRUjYb-ESQu4dimix3D`tbP8pN=YVL;qP#KyCP|r;DmPcm2lz4clm)P!u){A_#_i8Bo8vrTDg5fdIg95{CrfmB4 z8-?Ll3*lF6;a4n!m;mfA{OFUJ0r;;UhYwf~Eow*Cm@N?h!nWC|0zz|DFR$NFl561* zW)--|pnHbFo)A-BWkm!(B@J{KW5gz}cPWhL|Bi@0v{%$;6`1BP;kyR0>2vNlNXUo! z)S`tuc5R$x#DQCI)RAJsEjkNZgT&qGhEm-cR$L&oNBslw{yY530QfBd6soz&ze7*8 zl{d#SN?>1dLP(_@&h!{Auz;Z4>GcoY>t&iP>WS7oNTsf??dSf&J-k~n!Sx3#-c{g^ z>OnZUb?=?o2|@nGIA5*Zv;M+sOa4Yw`1^i~{!R7~_YGrs?B7z1U#s2ZA7W5q`5h_o zu0_AzWBRXDH^DN>!rb96?LddfSc+#9Ni_Bf`S-U>FllqTK6MZ&O8opr;y!O59q#lg zF<-Jl46TSgDJQi;?$10qjJbe~z1RT{B;PW?y{&~}Mq`*k>ri0q<1hSL^l>~>6i42Y zlU^aw{0}PFqY>#zC1q+MiTn?&!+U=~(hCmibx$4U9>fRS>R?YW!v`H5@k`-Od;Asg z%nv<=grCg`+0~lxZOL1Mc9`TF<8~tcqWfNSoeDIYPve&I%`}^es%`oV$y;j!ydfB? zQAHIK0{zzq4sKNo(d7INI0k^-MQ-?ccI;5kZHG+&erzxDG!VW-_f@fZ(S~+os3VxA zEmt7VVI%*X=r86)7o{rHp8gHWGj>?2{FtvO#PW$3%`V?0B7un%`{44%KjwxF+>XsL593DXi0YD4z_+kQ}4-ttx1*QExB&ic+$4(t##f!9gi|1_-GQeR$Y^-7~Ay zyuaf!WbA_Yg~D@+@=EI(U*zS21A{K0Hjaz5TWgLLOD1AWMD)783xG-EPa#v@`NUsz zs4b85xE;Jm7Q7fwX&=9IZqh?(>Pb<354;!+KZ7h0x#n&a0bkm3Otjzw7a@qubNobc z5rh{FIwli=i^~fsbCTGdi|nh+hFNHmP>!gR)Pt{Vrl`}T3t?DSCAX&8EJ+?9{F!ck zH~dQrSuKnqGVKvqD>2y}G6D*OQ#wLOE3UCR|I8)GFni8DV+S#z#+$k7Q35oEtB5zV zD6FUnz*(nF8gMoa(S$=-w{DZS383Q;O0bDh2yxdYUo*idbgT0(^SC2x#10uu8lsLu z?uJ#B3CXTQ6;Yv5C#Aj)Rt1MMPd}H}DbXdLP^V~+Z`MvJPx1L?#dzngKuiFGkq&Q` z4hNEp!>v~lt6N><8*D&dG%H94!0lOIk&^d@w5TR$g8<8|!!#<^O_D3z%2bo_t}19X z=0f--asp}5w@e=dX~;(Z9$M3o!6e|kyC-q+#!iFyA65{_Jd!c(m(r?G2o)B9W2;mrt9}*?dvqK^jKh*Bh4i7nBVhPNzNn|}Noe4a{+QGV$7>CHFWsCJNIVb;ua03B2nr_Z+*KHcO(gg9Q`Ece#eVVCgdtrrfFO|w5K4OS2ayaGTH+D@ zLRmj>=x_sZYNnd2A{6IygdT6Ud(;p>78W0{d|_J2!DQ!^${8=11Jc=U+J?kr{w0Du}U34&!LA!b=}>gDOl_~JjKb#kH)9C}=}Cvp<05VOVr}118m>mDhlNDk zKG9*DatB1OuQubZo*o(6488H(tZsU{*E=1=-!ZzXx}oeXdqQ{i-i2&# z;5h9>=LrLC`vSl{aGaq{g-LZAoBP#J6NMVnKGKV!b9EwbaT=Z(`VME}n(bjI{f850 zy>U(PVZ^d5<1@IG6}GXgMdcM6ox(YfRKoOBtY~M5y^d+E0kD0IUQ+%3BDy{SO|Nu7 zIR9Vy>pLWJ@3Y6lp?YaZCj%JA10mD}c!hK^vV)9$TUJY2^b|`-cMH^KiLA@I&C5n9cE%?9v(-=4ugi=ZJC@90Rq(j%52k5Y*D z{Na6z5ivcJXR8W1fsV4z+blYJZ5z4FmHO0U-*z>HLcN4X8F}#0VBZf>dQ8av9dDRR z+$}g1_h^JI@QDYs`VJ($>TxJ}=25pX%KPwv*=VlarM$9f)8hk0BIQHc>n!YmWIE;j z(B|4T7np`s>W&+Z3buw~F;O~x?Dpgx$|jWlccNpa+ppPuxA0EKFofca5ap>F_3Wt-I5E#1*Vw^aRx@ zKc9?pMmhn@SKQf`j}(Y5Fx6sK(QF2+qWEjYH${;;q{|AIiXSWIWy&D)ogf`efu$#> zIjD>Jvse%}6(0{x4rrdrRC)NKdIReB>^CY_=Oh#@;&APU9ccXIhaH;dCqz!B-AIIr z41Nvv33KyMWl9B)IepHf89%;a8axNV);04W)6g>=WlaqBQgce2+!}6~+DcQoLFc+m z)l2TnE5B^fOYcibziiQ`^JZJNaMDll=A%dGT~5K0TEWj5_~k!&?!a*FkT`!6y!dFw z?T6Hcc&q$ocJpO;8jO5^g~_JLR0DUI&#!GwwimLf5Q90GyN+ z-JkYfGzZD4QeI``mj}+?BpgEhA{4wap+U*6+J`rDjY8iOomYcddwx=dF3#Nj-12$v znk))dl%A1~yb110mx<6Ub7+>{EF)9J$Ts77wOQH_XlYfeYF$~_U`&a&Zgr({8OC-b zixJk?>|k%r{F$3uT)1maG4wKxR6d(#K7m9tczMiKdatMA--!Pb{(ExK6-q+bz|wqa zX22FKh|dDRiXu)lAj)UuYf+jExa~x0jAeu-<=}M}?GD+8ha2ifmDp}qXXs!`-mzJh z?(|G8YXBVN6S@%uNu5Mhh%O7Vd$!xT*eEHlCBi}znHpXTOy)*ATvH~Xeabd@Zy3k@ zAy8bX6|Ae%hcNdoKfni*-53PLC+k^b%Y?J6A;COX&! z(qZe5`{KTs%{9h*qN=9$Kn8*^L36o*=?@|`WY)XFRh@(dAG|}E-Py_dt@50U%5Am1?a-P|ae%uf8;qAx~}TVzOlb*UY8MtZRDGT5Kb1ugZP#C#vD ze79U=6?^c=2jVYX()g(l2_hAwEmYZeVZ&J52{0ooyf~-72-{(CHzLP2KrP8o8&MP9 zI>-Xat4efW*x?QWs?H=k&_BED=ojc;&`a;Jx~|*oz6R{R$hL7bj-sm4$Nf(|TIqfNp)n4=0(OZw8-Nep>!{T6c>65c7{l{kESP z)N@9^WItWe>$&nI{L$=&-Acb0FXwg!PES{Ei@xxpqy6B7mi(cJ06QxkUE=D2mzANY zefQjLP?{B(s_XlcT$mPr_HN4dx41qGFj#!a+KK__uF^@*@DB4r)OoL(QTEi!eguCh z!+Yq)siT}vo=ua6(p96Ct)>M>m5y#a8|jD5Ms!J&ze})=8p0hOIu-dJf}52+ml>l; z8{|{ur8WxuNTl9$p4~s`-?eM?HNK&{M}+3QeSR!FDydw2LzutyvXX&C?5{T!dA#3Ad7oz)^cd&m*3^Qj^V(Cd^{tG&Cv{Sog z(R%@npA~b*-iegPs&iA{>TCv5HHeBYz;@SA&4KER%J|+J02INmjt=5ddQR^zR1+mb zq;EVg)paj^`Jm6|n*vDK8Wu>^==wMlvw<^7#Aq&uq`iN^K2{*qpDMpl2|a zm2c%FQ>p>%FQl*disCe>>iSq+!2Pc;PyyG)iKN>TQY$v32c;rpH_4p}nV6IsLz-X~ zxU6AYY1U)nvy-(Nu_ImXL*3>X{jc)2KS#kTvI1zQKiVSEG3C(4TMAvsfvA+;3c0WL zBrxgJF#FfYb&bIhpxPn+?vh{PJwd>Y^KZSPo)@FVECk;j5sPI-2eb1^B0(7QQ;x5A zu9Ob`)u=CRSo^NqQKB?^<&>?CP;{X+uk9k9bZYN~82O}b@z(gY0L^!^Dw&u4XFGX& zA*sL*F|6n;f|Z%mCT>*mR>ylVPypMSsHW-InQSz7pEf)r00y2TrJq5PXVZlKS7$Wr zDa&+V;jrgzmP!;RfTIhr*mo1-_1d{-fmj$pfdiUio;3r+Ww6bk8SNre~kg2d-o z2jy&Nki=3F<3AW&Ajwg@lTR>^X<(9Z9hQqD7AXK{gn^W1Rt^$DB_}ZUgifH4HhGmU z9x?W6L#q1$E=G<`+s6zuPcB{Ri1RpFB!wvux&Qzd;D6+fAtXC<!q|TEf5r{lhk^dd?$el!e4a`W zCCm@hET%qPm<$Tbcw%aiO7Nek`otpAL2fYps4miVia}4PwsFRV;F;uY%0W@&ZQ4O{ z zgA8D|Yj?p!Q4;nFoNZ~A`nP4tG)rv$d@X& zbwN_-KC+iyk05juuT6ClW4F<7zVcW1RY5c5UndwoB2PVKyGVo9Lg+K8i!{H{d=kP& zW7$kb4uTD8gewV|OvPgb3CR9HoIO6peJwycF@F*Lr*(T@FKEP&0R)8K{C{fQ=K5c? zZrhu=+M560{s%>qpXO@j|NW;;vYMU}nh5$=34?7phNV?hc&ct?H*uBn4KivOt_(6! zg~IL64nCTwg)Lxb<-OJi7Uho!@;CYn%MX!ZwvABSWhmdYHMWy`-<>Qr?~jWEDv&5` zptj3q%XO1m$X+)JL-q@A$Bm9^?G5z7lG?~)u{N9M3Ywi`l)Jc?2QK!&A*274>)2Jx z7d`bdvUwV!7DJhM=(x~=CARr2q&_^0eJvgKe5P1`=!vddX4CK0q$NPwITTngRadK6 z2s%SZH=Rk6#yGw~z3qga`Dn@jOfscdlfD%tOY9i9sWb&k=pZ&}nyGXMH@g|yXn7Tz zbVZf(IVvCKS=G-k_ePnms{DfUt|Jedd5q0XZ)*541Cs=HyNuG{7pMk(&-6hFeN|IN zHuTUt1R9CV369n)_uwl5FPrch3!auxI`;kr1nn0FtBQJkg}iWd*C_YSe#-g3J+5?z zxQk3wjsz4CD!}kVIY)J?iemm7BWim@JM+)!W^xk-VJ&U0C3Q+vYU7BG zsvNKZ?E&kJ%?E=9fa^%n%wUOm6hqGf>b3KBLZ7!2JY(i&^P=ajyxnSx!$rc2*SQV_ zZtM;2emw=mJ^Lv3CmTx9(&LzH<=>z^w_O1_I9WK)>$s;MbG?bxgro-98&V}+!JE${ zB`+)wg4FGJ0o1chO+;vyT4uKhFQbVUSO@&Tz+Tm&=qflWbVQBvD%~eA;4xyHKh!?8 zEm6KS|H^14k1&Noe5CG~=Q?7YNBH_let_dVj8AFjM=z@1mxv)BN$sAdFkzoB?-K)p zV*?>LH(Aer8F>h2t-#Fw^A7^}|B(mb|9g=C`v7ZQ`J*r42qfAzOaP%xNGyp@!7&oI zf|H04*70>tPKOHXC1V=tRclm=R%_DFFJnc9fJaa~ADx7K zA$_iYVf~!xc5mpURJom)_Vvwip7yLj2$K$as?m}CZF4Y-z z!xZ3KSmOg0$6ierN8!-ra6aT5LFQYvIe$4dTh21B8pcLdaBOn=KXeoxx0RcGGFWtW z?o8^+Ou!Xrip}0wmNmn9-=`Fc@RgZF*bp$S*CcE5!QpZ~`ksJiva@TW{mr`2IqL?V zr}5DKSUO27b`oa7ta6!m8o5e3@4=;>4gc^!#d>*q$W8n3_Gt!GJyq2FMl9FGfnW4qRTr0;r zD5Ehqi%E84)}jeIpVac+q8gLM*_$?#D&uaBBdINant&r06Z^$r`%R&XTCU8l>XL}g zJO8v;+-YT$zv6RU6fqK)S$SaA02jl6d$FIb*PgOFJbP~}u;X^`iS^=m$qVK6kEIx- z8SfHT_z8r0SM%d>e0%4`U3AeOAq|$OOjXVF^lr&>%WOFyI#8v^i9d=XfnyGqvU96s zu%}OD+_GLin@_@4;=beqO6tQSR3~}laKcqSQNQ)ChHUY=L2vz7Ei_SSfoEm)$Ad^z z|A3PO3dVQgLYoLnsY>7#fsXE}Enl{Lo+Cqy|E$Doopyf8_U|%VrP3s*^q@3agByfmM>zjO0XHERvBcg zF)vZFYxMA{2>_I@^Qj_qjR&|oIr2;V&QIc~jtGw199BWOOp7fJ4@VtSbrSL33fCdN z-dv#@*CMPhzsigK(Z#Hf&QX631Xng1F?k3p~{c4p~b4b2Ha|clQ9;`@!efps=Y!XT0-~v=gyl+$q6! zCtJ=YVSi(j3R9_*rS>+j5SyCdZ!Ga~tbsISO4i_C1#$k0Efsa(KflF)9W*jYW_vLd zb>G4=sjr1$GXC7^;RqebfZcOd;q~FdZAgwa_h?7nUc8g*qU`OOT3lOmcV7DusH2P; z=VgDxFhk@3JJjXH%yW`afJ{rf&oxNsUWD?tQdY!)pd3@`Ii{K;pjdYka>vL!0ZDuU zvG~Bdt)lENHAayH20@EOUq;0h9K|cBj(HcBkV{Quu!{Xhq#M&eQ4d-8GsD5-mjtNx zT9-R(uo86NizQdE9Is&i2k zXTy9f!UHtJVA_^_bUH+T9y-mLDv1x{w)Q#t=@g8g+?(ylITKKd-n|ze1q0)3!tpBa zY`x$H=^SRHUjIyWKWku2e@;P%?=pJGQbQaf7#ZaTUJvDNeeA~cTkpm{^>qjGZwNEN zmnKZY$8!C0N@0ADQC%uf9g|p7l8nxDO3gA-aerER31U&9(A*N%eMUdU`yGSZl;$uNL!bun2@#*)@Wf>VcV}VcdMR?0A{jUOSz7-~ zO@W;2bz-*NlBcknm--3G+FAiG(!})g>3QXY|xkZ*2ktXir9=53TQO=PH)-69OI}K6_=BC`Q4=p2xo%9S}^d z%AeLW*(`cReN-_bcxl0}z@Gs(B(OW;Yiqa3GTe;wa>_OM8Ay~$cs%mi$@G$cik^d9 zA6833&>d8>N-mj623UzRnXx+cBf>L{{~#ZpAzk0RJyf%BTw$jSEpd!Aza1F;P373( ziE2SQQ_m8ST5{W%ZdsKM_2Ptcs0$0pEuRLsI`TXp(E+3477RpaS|!h+th1m++*OA8 zAn79Qi&|##K-Xw^q8+Jl)P2oUIwjhqif!{xnZ*;WqC3?L_LC4LFd|u=L$ie4Jgq6e zuK8K@vx?_;Nr^DvrbY^tXGS)RW z%&5}4YvsdMH4p!iAk!K5{MSDGNUU-6&LWns;ilRJ#O*wsu81X%sNNk(cneRPX%i}w z>w^~-+%Jh;z{Q zyL*4V#-%vd7u?e^oD@v@>uPuB+Cbh5v67!0uG2a-`-dpM$2g0gM&yr{7G!L^^VzvT zRdm|m+TennIC@I~6tt8WnyX?yBKT$&l_$Q%0qoT?$C?vT;D1>}{6yn&RmbI;;wDz3 zXS8Wjt$4JcudU0^xzzgQ@XIlFWziD^A?DuLLMlx&f5$5UGcO+^Cvai(hsg0(j=dgE zKDwt*7n;JE%mfXt?ibNe4B^%v6fqmAb8zBN%;R3o_u zc9=(kY{ORHRAZMv{)=pvxTNe#{Uar<$o~^Z;r?H66g4XgI};aIr~ij;H5g%}39xJ7R+I!rkW$*?aSUNjEunB31_ zAC^Ae>)7Fg;vI+lf3pQouDo*=ouW8hQs zu;MhdT22A9U8SV^+jY53EtehR*3bm7R3){<2#^8rrZpv9*ZJD#zp=&{&2YRBS5S~m zqzT(kJ5J27zjL8rkg*FstctALJ8IbEiAuFXS7nPer&G(Ooy-AOv0Cnh70c3OOy3J_Tr8?fU|niHn=-8G}@qV!Y(Xp5p+l(~NGj>Cm6ErOOFx6oVAWeRTF z0TEMZSBJ0dp3Bz5j4-S(@EN7UO@m4HvH2GL_HWFSG^U;C-2?XqQ3$XWGF@oPA4{Sq z+V(Z%)Hq&>CDtO_L#I+BBXM+Lc{HFwge}`Pvhqf1{6@bS-pHm zBbCjl1(nAHaeM_2CCJUlecH$hj3j%I+{;r%A9${+TFrQlb-hdNT9#uvre4-&rEQ{JYm8(Y?3p;5GWQq@H2jZG>l^Cg~6GY z`j5&{Q^D)B2E|u0_hw&ki4F+%$^d$i78_c~&BnN*X4%kN70YfpGWIEIYH44gIv00pAT2{`m z-+y_NVMd`{5Wzq|1R(#XJIDIJdFTH7ITkG#5A}c2fD0}i9;wg7h(O;s0R&VD94MM3ac6#y3Q1+Z|4)7vSL=4RyUp$Ya2;5-h9DPlv}#^j)}(Wrvu@6;c3Y~b zZfRL}-tqNJlsAMuTVXOf-tyk*{^t4q`~AN2p7XZbCj~-}iky}z@2Ww$(TZC?=_%=5 zoO4|>3EYnupT#6|Q)J&ZKTo$Ehju#^z#-H4vDZPmB~&}FGl!`(!X{Hgjh=RFZQG~m zo5|>~FAa@PGh;qV02R@m)EOU+Fj5_8IiOB~fU`ey+%%C)31X6i)nArFQdbcGigGOL}Q;+n9pN$fCNmle@KRma9C zK`+u{P63pgTW@BuJnF{out0jn%7C259%?FO+=VILLI#{0V@O-$+Xc^Z>bl;c``dE4 z%tGsZAivrKIfA%kOG{NXG6gFX78EI&VZ|Oeco(&BI=Y1%|COSzhl+GJHL^L~O^Mw}zRlkvJLKFu+nH~hU&VVDgVM_`u`niB3u$ObJLNNV&8Lb*# ztubf56G0SO9{!&!2*wX9ERMhpEDXDhL{*koqPGG*wxv%u&Y5{?8MKHDz=C&kL%9AJ zIoy^)g(;L%(gsD!C1Uepy(`lBL8)UR{8#rz0=v?(Wr6fEHR9;ztn^YO_=uDeBFY6D zNV$!Z>Nn|nH?Y;nVHu~QGcl;FAu5(QU#By`sxDWoN8K&MPpUH?g6hR@(P+130DV(GoTi(^YGhG&J{?2() zr(Ko8C@emucUUno|lo!*$R5bGKt^WU-*NFfpc@C>&KyW8VwwqMNM5g z0~WVBG`)(T_S2}9R^fVUs^S%MC8hgb-a!@7^~|knzDRZ}4w>uV?$4z%%*peublEqm z=YGxK$;~WDU50d^pH3Sdx2}qH`xXk8hiReSOT+Jwr4`N!EIWm1H%kt!{oO*Xpz>rK zASU|yh_6ulk5;D|m4w)*hW_Fl_(?}a&`2=!rms?7F6f$*M%R6X{M1Unsd!DjD#(P$ zmxJwPvZTnT59EU%F3W8z)RQ#|ndw_U6USF>tUW~UydpP9OwtU}88kluvL7Hy6GDS89Vjy+&6%r?q!0vXAEG6CUNS2#G0SS~9KiiWdga{xH>Ay2nOnX>3*T z<9$FsB`j-Gn0G|wxWj8Wq$D{)>!q-Z)b>qmjt?0B6sS!ja7AoW$2JW6aSVS9?!R^T zp^djKO}DYV9Vp(8MbQ4K&^Ha7!3mT%&cG4{B{Q4=Rf{O*ihQ#$ej>d6lDV`iyOY0p zu#vSSI16Z>5W|FvPTp&~b9*tAjpb=7L`v9IuMnHiyD zLe|3m^d)0@u9Cr6wH%7L-yR0*&ikkvd8_ zK%ck6km#9+tO>E4o}0$jP{WDeDf3P{n`aMi%6~?5@0q^FSz+|ykr?}W*6u4K%6j?# zDhw{u%R#RtloSVa*{+8sw&W#Fl)mIQ_ze5e)9yqiJ_BU_etedV_sNU0_}3oZXt29; z_;!W@R&?Wz#)H9laER(jsPuLrH06S@oRU^3@jhUmk2=rq-|NdfsD`hGoH$b-Nt?h5 zkx0A#AXO4E)ZS*Vhe^)!D$I8fX($;t`KUjVW{`2qZU1o;2Ec3j zDwS|a2yzbBEL?((jN_uqT$FH;R28C-I)}$n^{$V%MJzD(tv|dcmmRe8FmJ@Ncs6

v|aJ*4QZpt5-U{VV`WyNc)wVeIsuO9V0-_E_Y(fS@W)QUs~+RDpk2#RqbQ1|B^uV|Qd0MupaeOgzAdzIcVc1+dXCdp5 z*^V(Klvy;Siid+z4m?#a-Eo{Gh?)e>M?n_*F+u>7^X>$`hhN3&Egp|lB*zBCc7HS2 zt;4nsBPOaFiU}6lGkHHFPTq&Dl&4?is<4)0a*-Mr`smA)h~5cHyX*aYp8tc7w=!WF zHa%IQHLFf6P64L#AndUSL6Fa}c$u?!8K7PB{Rl=XSWz!HAPU<1PJjx*mf=q>QILjN zT?V{jF{VwK1lV!pFsK)}?c_iHZ>_pRe{HcXAIpmCR^>A@5g;a+4^fXo{+ZYOS{<#SnXgQ0HTyeUl9AN0i4fY{u^P>ea;;o zEA_|Z*^W7Xp>DSY%|fH~N7+_0e1-^ZsAPIlZYRW-n^m_xhZEk>gHJDHb6P4-C#>m# z@Yb`VnrQxdb%9i2T}YU_V7(n`snYhE@&`Jk&9q-g|6$jFr%?4TSP+mkod1bklmGv- zYh@=ZyZ`r*SY0b)bqUm(>n<}x(SYDk;Q(qj;LnMFMzRMXZ%qPN{qW1FJgW6D!J$P+s#l=w9rn0>9qk?(@nU{}qXQ zcb*Evm$96hS{~0~`A;Nv&x|b+`(7qXaequn_UEE`G)-~J!4-*wF-r>za-eE`UY7$Y zt+WROf`ZlBd5)#K1vyRG2Qk|-dKJ#uXehgBqqxiDA4}SpbQ#}J%-6SJA%BuVSWI!i z+@q*=#iOEC2wt+9Zj=>z3?u2UVe|!NTtn%EVOU&&04zpaBXJCRJ<(!%Z~BCjsw+jx z0lYO(ECMZYG#xu|nv~4J>)OZ?aD`g}uW#$I_?%6&`?HH!?H{6UzmF`ZS79nXS%!LbB~lcn60+<-adI3*sRNdo&{*jp z7_2Gbqo>BIX8*=WOUmqE_>q2_JtL8}Lb3$W3WKr^^hFgQFNi;RvR!y-oDN$GMc75T zFcN>o8Diu5*{^ZA?{wVqYAdm8ZTg&LN9c@=-($tUq-U{5^eq9T2{sMw^BcbfC`1mBni1E4NYj=H|fhVrGVHPiyEDdfDT%iTphyVp|K>cOT6yLbOf^VL5W@VIQ2OmUrIsZ7DyTc5Q zc>h%*hD>cF;Oe^2ptI+}?{z4QcjF+!2A&Si^_k?Dd=jtrsm`X0E;3p$1BW*!p;%bK5Daj_+{N`>Q%R?*T)?E7us8@CPa0u7s zuD8ov!GP?o_`hJb$;q#c-piC#=Cp6!w6Fbiq+1W!C%MNecb@{;1UfdqeY4%bq4dWQ zo%--%)|@mQLto(`R-_gPDy#MuUPJk2ohA^^!d)|b?_M)<1_LRfI8Ozd#^yAS@(RVb zzDB*aL>UTc^*-9Amd+FM4~H|DQ);hc-K$VszIrc+hQxa?1m6N&k)B7fZlK){uq6lH zD3^Vc+zZLvwcG>mpx)yy_Ls$W^q{u$#U}C1M<$hu2YmLh`Rc5?$R%HKl8ItmB{`uJ zCc4rQ52A)izX&fxbuD+YIJvpNxBMR;rj{^d!X7Ixp!rJ>y*vf(^7=pxjULc0c?b!g z{Ui@G>7x==Vn_az8wdR4#>x^j^(EL5CeqN`I%Gm5&P8&2==6TSbUPY-wW*J}K=(*H(LG$t<5+3@Hok7IE?>KV_u;uaH5v64$kKUwW$Kl+BfOLE zem9RMbSoF$P5N=R>|O9CgW_e)9cW`?F&vKrmAf)26$g4fmm>Ibo$(7WdWyNQa(eOc zw2uenL(~wg{|I{NB4ujuF&t*=6?LZfct!daC)h7OQsFS4-6EK^h`!){=FR`$n-zA% zlITZm>DBWxDb{`ua66*1C9#^H!A{M5gd2{a{EVz0Dcb2x)oBE#KkC6Ve;F#51Cn#= z4Z2|b7hBUg;3miN~HY3ZsRDuHr#G zX7Voh2$;?Q^gsLQ$79nCZ{((;zTNw`>F;@J)5oHWG#rC}m~z3oUi$e$oV)~~_rsrN zE8TU^>2S7w9((=T9T;PTK6xp+)N-VI@Z**Al3nm^)lps&*7H!7F=lWU zk!t;1VxJ+hWF3KMmt%nnwa*)M|6Hd412mg+LYOMYJkJ?4ts`2Ic0#z6YC_n; zHzjadlDQct8?F)FMerZYS>|5_$b20=ZSk+dIKw*I&N?A{LR&Ovb8>*ISgv3M)$xyy zff(2VbPb5u z6Ja@O-cW@7+MYm!>el)PH3B)AKJVW9hn@Hbi5(ER5nvpZPuM<@onSUPMjdB1I$9rP zHabz5Vcr2(%Qg`NS4%YcgFu;P^5+*|WpL}*`_zqicNB?PeIqEJ=q{(3y_qSnR%mP% zxtE|H1mPU3i=Ud|zLNeI&L&m#t|?d5uBeH}R&yDeJ`H?si9;0gIr|e;OL2Kmoqk>M zI=VlAN8$6=5&IKU%EF?*c1=CR=dD@_Ko&M-L7i~IiZ=Jiy1Tl>r9P_!R!31yO#5Hz z)T}L7j_=WzmaC>LB$@Ik#F+B*lx`;#{sSESt2+KmSOO_9es0Z?zD37n?M9m4t;53E zy$ID3iHYs`I)~`l4F1(LrRTOR90Yj#2*ys7FVd8+O3d37^4XKB85t7*L;n~WCG)>) z+RrWui|I~(D2>nITbMU&#@RRmX<8|nL!NWAM$`xugu4yV((aukZWl=vj4M>ol#wha zmP+`g(6j~0BJybTR+}@1Db<{AYQP|dI5Wiy+T2%k3s2U`B{z7Yd1rV^;BCflSj@vp zd`Q$J*)@LGGe-4mFJ=s_;`#|MH+!2zAXK?(x1hRfbGil&G<2&YMy=)Gn@rZUYPbIM zEF5Ssf3zzUM21a6jp4hwBz6N+;HzB&6SG_u6TP<1rD|KntFU6G$Z)G?Le^fKPr})o zj)8@5W`9>0%l^Ok@W$9m9O%osQ`=M9wr$(CZBK2ycFI?GYP(ZSZQHi{>UMwsO*Z>! zH#xcYCO7xXNv_UDrfKbdQ@&~K-UGn2_Tay$-PhWN<7U85nFiM_s(Tgb@n)bbgCP#v z$*gty;?C4~ZctHS{pb3NN?k8!{piBEBXUc71GBefRG4BuJ~K*T1!#+9ny)nDRG02;ILa+~qV8%?{EfaRn$;qoN3*E41_NqSVz9vc*IATmJ=0$W; zF8QaB7fkW-bs!r?d5X-*s!XDo`F`JM547R~K_k`twm%%QGf|UZ$V$2XnLIg)&a5$e zmoT)g=M(KDjF>=nIHmjH8k2c9j!CXpnLoGy%jEsz2}>p@0yM z<0!Kwo_Tk~_Qj4hYq`?+BpAiU8-)BB7=6sXaa)M(()zHWM`C?5cOJ#v{Jdt`{Arri zAHR#@2G#VQ7}Xyg=~YrW!$X8vino{h`K`-}rI&bs$Jz|zeeJ=M;e_PT*J^Jo@&VYd-`#oc|Xh>k&@`ZjL@Gf_kc?#LgP1T6q zcoUpV*Z3ti?jXNX2)leT$>kj##t#s?gla+<{mT%J;*GZx3%d zgZRK3?+C-Fcnq~ggGPWuZE>!l$mmOJ##Y~hOp$6H^PLQ_J8~W~3|@0iBOW1JYQYgD zdk0H37FX%BnZlunH`dg3ES8Gecw93oUQb)!q(SDCr>E#=4VC8( z)f$OnQnl8KRWfTrFO^78V5SFC{d;OrE)LeQ*A47_>T}gNOnn@*k=XaON#4qAbB1>4 zw+Gf?RCeVoWeBw2unVc_7GJ-&)5o|Rzp{_3c-Sq3+De3Ps-B)BA|9_a)@N*Gw5jTh z%dKN|0FGiU7B-}j;cL4gL3UlHb=uuhYi>UU=Pl9zHFoXZF>4C1MXg;{!0;{?c1!{p zhaTx%`FjS@2A}8VMxtI3Xn0S3gosG;HM}TW$L~riq(_>jr?>t5zqw|m5^_|E<8kIC z@rBI~hCE`1q98O9VtVff3%~fCh`R`}edmI|rThJS!qsBG1vHya^ zD=C_{d&tu=7O09EwenVX_y+x|`!8JbITgBR&O3&-`PGP@%N3muW>ACdP zqI6whx2)@YglX)Ji3(sHo!5Y2)xu0K56OR}Q8rq-7p`%WW{Yn_KXWR8-V(2xU8ldw z-`l-{l*}jF44Ue^l1wjM{RWWd6VSZsUFh`JI+i+uV$ck5%J5Qyv)9MNMZ$-I6gec4l)8uRyYYRK0*9=I;Q(*Y#P=IhA}s;@69v4KUZ!M`gu zdOO?C^U#zWQWmA3VZGCBtO@(__HI*ErL{SS|ica!vE-E<; z!hX%8WNRqWNG5s-=Ty;C)e8$Ps!Y|rRK+@co;R-U7^s9N+8`U6R^D*P%=^?6HBWA4 z{F6UXvDgkV)|E;ZU&FE19$QABy-+dC=@IG5xJ0m`TI?#w$mLXkd?2OgxG4d5V6|?s zDVCFExzZmNdN&NK z2tF-OmHjRW)@>o}RE>`Ky8U?O$d5+5`N(va9RfVdy0Pkfu?=;H;6VA2lEoC!OI2tN0KZAI6O; z?RnnyeeksPK1|STQotREyP@TDjfQpfT5Rxlzdf(y5!H%%U0g}%f(QG6_s|80(D|}V zcGxJ}vRmdMA@h{MTzCnuZkFso^PZq?p<|e!dE6&$e^`I?fZwC#j>*uvTlyLI<|3Ce zIicx~$j`bNU!}|*JBz<7n}KH=LzuZ-6%};pKKW&3VQel* zaWS@Le2tyaVXcal^8pb6!ex5Qn^L&Sbz-}qUOFs z9}8tb$e!E_IHep1xE#O+CSHXMN=2?2LW-^dibXD_G=z;Z15%3KmB=I>{RDCxpaYIu zB|&YHi%t)I0MLQ{qhiLni%Z!FK0=2Esh~mz&Z87VVN!%{+jl2d4XL1J2DuAG$px;Y zu}{`#LCFN8LFYwglvKD$$W8bfjfzk;ow`6cC zWKC`po<}!k1Kd5m9g;;mh9A6~#+7?lgR&J;N!<)~*MssJc2a#`v@d{?2!=srAFU5h zP5?&d&mcw!JHQ9tB?DaXuY7PLM2d17`UUVpMp7`C38p9)0PT{9ZVbsme}^QZ5`zLpO4$Q# zqqH z!0A+3ppleWu#(gsvWSR{q4z<{C4x}^e!|Jn>cYs-^TNy)AL{h+QI0?^)B7Ttgr{PF z5dd<3td<;7^hLO35K6DtKNYDNEqD!f4 zx>+{J=wPrV}D@nS8 zq5O>Jrdu2INlTrI=cdOq>y{kT8rBCG4EW@v%Hug}yYU{Jqzfe7Aydv+Zq zExP5$0P)`R9kF*rl-lf0+KoAPMwI69{OJlT+vOIicR`d3@hCJMiFcxu+$=&Rhg8E+ zfbMA5@>Cplp`ydm_#`@CyaRJNC({6fF?w8}iW~hwnw-(pOM2=GTaeM%1s+iAUTWAn zy*ukpm(tlXKxu3P&zsg4_P~kq#^O@*-f37+&dGYS>JWMO1keqPOlZ(k#ozu>biI2o-x)qpcZ$r^ z3sbd2L(A894!eHd*1VL|ymVt%wbNGpHQV@RK5?$hSS1{lXMFFjae8vfauJ*H4Q+~< z9+!gqDPcTx{|);;`uOrWC;2os7+4+G|4$#M`hV-=8kRmDwpOs{0 zKRMi{!1)0OE)HQ7i?1e9DU+u0qs#;zEg5~3Qer`&@JEpej;G;J81#l+byt;Hm>Kgb zJ?Of!Yk8^Vy0h!;Taou9|5py`X8=$@ufNa9`@2hr^)~y%^u{qGyZc=Q@TLP8Jseku z3r&z?Kg?On!`0mA>1Z?5UYYUc>R8j_c`z=1p=YnX6qy}3h>LVnxbktl9;vk^&2oN& zjOXvLluak)C8@idDjz74Oez+cl?RmumBUxja zXpB@;w2}>fz`VoG6}FlxI?6L1#y^WnHN2LZ1nIsGy!u)e=-A6vwA8njV)5Z`A#s=+ zOHPY%&*`;@*ww3*GHMi>1kFdKh1uj2H2qty?I;B6vTj zs!6XMsTaaqSXd)eRan^2QrXhlU@EF&ac)@`Bo6jng1h`lS&||R@4H=$) z!QF@QI~e#azc?tAJveHNVK z=6j(PnN7hdludO)fFr(O8<{?Hu4h7s7rIn?;pB&?n<_$t_(n@F0-M5&PjLb!$wFi5 z3(lwW?>J<|L`56{!XrA82yPhb4~<9#z6OMfraGg_qV^IO3w~~=kMUTdE?5~>bE~U? zXiQX!+YCZW0DX@A(z)@hWTNGTzMlA#P}twt;Z`lFm8x7 zBDDS@-cy3gE70>@O&?28MD*pt$b3#3MQhUeW$b*5oXd;9viE$C_@qbn7sJAYRJ&0q ziVXXx^Ay0x3v_H17;V1p5dv)E{D50p`r3@t%InsDe4!KS3R~Px`J}}CP7jMQc02UM zY9b0FvzM2^*J(1(VffIL(3uqcA2CrfVB_uDEKRZT*O-;N~CyY z4ddv4q3lNZ11F2Xwq&@6RQR_L34RPq^{#1u$c#wfQbWQNLT+K`+d7OgyH?FY<8bRS z02i0}s;i-=LW@eQDx2VWa3ocy~AL+xfu{PU}ya}GDp6++E z9a>o)6`a-817N%d!6LxHX%-#AzV`z4q*G2}7%M~)EAF``A3HGD)?vo%6hRQFqa0 z_k01cP$b&+wtMLl0UzH0G9N^M)GngyBI?>;-(M|k{wbM|46x}Q>30#(sEtQReN8T3q3|V!H7MhF)fp`o%3Ru2R99rYvcJS6C9BH{TZEC0-t@^VmKG z*^&}};V1s4MG7hAYj&P&i{8UBCnwjxr=%DOF_fVc;Sv~G^u}a>-)%ZaCJ7x|_I9}@ z4qkA`QN6(#{r%=B{F&6nY)QTJ*ety@bMo-c`l&U|lMf7s_OF|>7pXiPO=raAIw`H@ z(cFR>NjT)C9vgetCzjq0S4Brp6u(nZm^nqiWC`kL$}oF-eQon&Nwq0tp>lQiID6Gt zVoSSsS=GF!u0FVwt@C#4e*ferw2l{Jg>pJRm++H!rYiUvF3(E7+1mg;4Q`SIj? zB!wbbGoV+oz2p!#CY3w`+Rhp70I~t)P3#ms?0J61&VkjDH@dVJfnHob`Y~3xmP#Rc z|E3d6_1AoncN%*Kf0MK3&jc~E8qG%BKv+O=qVI71rk#f@UX4qcXpCE>OuQ=64{_86 z{OGDihY&x%RYuyoEF&%)hz zZQ?y#zpi^=x{^JPm4q z6%h|ydc!JY&>TJDSYFYFUKQ8#Lzz&D3%FE$H9R)=KlDya5+v8KQ-XQ=T*`Et-%~^T8aKFi(hpnbX_T8kdxlN_8Ex1j0M%$vq`rhSNz$s9#cHRUa1)fS z(mZkp9m;?sg>+9AX+}M%0=L%! zeV9~_m{e9Ej|{%Ng0scQdCnkF*wB=0EGC3OcE)bVzc0$Xy6ub-vQVum@{amOC7!vZ zrpz6E{>XCw1rQpgzo>8ceY>bbSZk22iIf>y#4pr>!&XWbRcHSTryd+|#f_hg=EOgv1rC@3a5*2R- zd&%v@M8-G%b*Lk3n-y&~;c^P@I)*-X+ZKZbUphg<3JU_4qPA;$@lG{|Rz4T_TVKDy z6t{A!0Z`Ru$$_K{ln?8%=y;q$@49GbL8UG^mc2#l+>GKeM|1Ehp-o@&j$Vswn50T= z0<@BE&6Vsy-k1IzlN!vg(xpzjcp$lbVvQrM3kZo2s6Ja?cGs?Q?e|i&4)7be?^$|y zudVG!HZrvFLn+&=EASB=OY!YMh--?4cl!v)+U$?~)a*c$PqOA`)a~{16{k|+kYSkr1=6x zlri3U97HUj_VRY)U(f@{)08AK=GVg@8f5J!_e^YN5KR)D!gEUxm&YI0&vTmTA|*E) z=1%~2&k_hq;1-dy@+V-zDbA^8{Ia#uf+Q6qUx_Phzn@}hs=2VyU9AZZbs{==Sq>Hm zrQq!dCVzAl6zKHu$vd7`J80!xCO$J{HWVQ)ZH+H=K5O`vEn?Wl`2%DM1n9e1>Z%(H z1gx1ne#wPS06c|`Ia$@XRLP_LeEDe4tCmBr&o$XrrDz*Oki`A-|HafsBqWTqaC%a9 zTBjoTQFj`f%8sRAJs4Ip!^)0CEX$YIZh?>1PSGLPXvQSvJBg@Dn)-@rxf)q&+1f6P zv3O#k_5+TW#abEtZYcYoino0#Fv{+apLjJ3gb1ebn7n;w4!M2fSR095jUK;5z&wZxa^cYjEqFke-n4cg!&9bge+C&V@(;@#NFb3NbxWb=ZA7(Uq0}~1 z2=z`VEDTQ{iYK~rCo3rpWkKu5IySWy*;#2)Ad|pG`6OjJ~tp7q4z$FS! zZBQgcJz3ElqsO@4Rf7zxqM-0ON`%V{MPB6z#1~NqsI6We9)b(C z9brsiM){|CRJarpPv6Xive)g1=$zi&Y`KDbdSxe2ziW3}ppq zvAnUSgTX{49~{*|!z@;L#d#YNy~SXx9}nUAm7n60E+e*On1v5+N?b03afYr5;{mlk zUS$Vi#xO6nO*VWsLCPlrofMv#K}9oc&N?+ludV(3VMd0C&&X5hR?=?9W;9*rVnh^~ zX~WBm5=A{v+t5k#8wf}{!CBZ6=x_)!QWC{}$%Qkv7O^@+b1H|1E8E(H^6)>&QztC zNUv9w?DSmcc$WPpWe(2Yc7PoJvv9Nh5(B|Zf>N>1n2k|+5eLoks?{MVgCStTQUaC6 z5WSR3!-XB-9U7ks?uwXBPB(;}&VI~wC{(O!{kRhSkjNxGr4NH^ml)c}l*~=%AzNNw zv$%>%o=Tq5$k0^+!P#V7Ef%|)S8zGp8fh19-7>5mM?;iChm*qKo8URxIOxGNMPEn5 z2#S-+I^r$J?GA(XEe&!Nfd2tG$xO>QPIoy_t$-)08)WVtsA=eJ+G5+N()Kcbl98by zCtJOc>ax;v<`;I|**~V5!MnG`e8^Za-`$uRJD)3nf@?Wc&|U0)f1HVvj_F-!+oTHQ z-r-otEhRkslZ$x%VA_lZxqpRMUAySnS z^849=Ug82opiZ4m=?`$NMe&SN4DM4xf~E`1rtv0dPOu0sKKcKuzM%hI1uwa zv{;9`Azc6p4~Y{-a5~}luS7O~ZNg%GLkW-c^%*ShaeIMYT4khL(}4EeBJt}=P-6}r z5lb}nIJh@^_D5Ci_%GJMs&|%sBPnwK?NKD}o4V2`kq~`9lKCUq-s9!exY*C?YX&%q zh1c&UMIRi>z==aaoV#|0v@aAUnqR)MRIp>wu zdgKob?~LMoq)9525ql?WtE^{im(4EGC%TFXug-gI$S=8q!~l-s^13Xt{J88LpJ|4 zZBFT~IjT6Ypj;5bkw>{%So3Obcb~*}jd%w(=SFmOkxF2J(41K#E zc@G#x!N>vwpV*X8#xTKVRY=Z(zmJ|9^^{WFOi!HK!v4ceQHSl_!*$U>^UHphO^^>v z1zu6wXqlW!=MOidFx`v=7Czl64jprL_ObHf5{trrYIKmuyw>0@GJdHGUg}2!{5CO5 zvJ+1Sajkk-rG19{XUM~Rc%`c~0gQO+v zkMV{YD})!8sPjkg_A1XSI!o$;oPVF+(y$Mk^}EIBz%SRc|b) zT^B7MzO*W;2aLJ85|qCzc+LCGiRK)7e~b}&*cH`-C+hJn~dG>2?%JE z4Spm0yj5)fqaIdOR7p)lx$R}s|K}4^dV0G2oT?VsVn|xi8*SI9w zbzSz(qZtWsgVchaQ`8}C6Ul2cP54H;04eLl1rOtQ5!#<3SEuotcJbT7Z%HmxKx!?^ zb~MKw&u?jnJiD6?`u$p7C~nRoDt@p*F;qxe@O*@_@d`9_}4Vv#gv$R^EcL4i4o@&F)pS*2he(K1Tu&oNy1aJ3^y|{^T zk(dFqt0Rz4q=L;1JghuEKNEGKTwDaTxCp2*eEg<{-L`G5d>Vx#R z6jPS zaL0j&&xq46#$c)tcB;{tdGcXh~C*)?*#JE2_dXrmQ7(~{Y@@|BR_+3#>YyvfJ^bJQotG;LoSXUc2ER&fY zRrsa;<@~H0X_~QTmm%wy&Oc%Drt1$Q@HrsUz_D6S0X&@i!TOTqPu6>hwFBrtwu*w@ zGJXZ9{`TZRvViryB;C&A&6i}gL{W)0FDS={%tPxZLf4seZcy=&5tc75WXKC+2%p!X z(hyu0pF_+n!eh|Vr$Oi=!IV_+ly)g}=u_(f3f%G9ZiAL1JF&*Vmq92>u_Ipp^}C1bG(iVU@0|d^p0X~jYkJr~=zQ4uy-JUw=-nuX0ppu_>oVfnIUp+K z9jbsg>=C+72RMZopT(1GYv7-+R;arzA^4L%Hejl^8`JrT+TvVZSh*teC^p`QtfD;1 zn=Y@NJZ@yMJtfa9DJEMD76ydxL|2hT zsA#`N#+SlaR&$BV2V^XqSX1jvWGwdA!4e!7d;F;{dDDPjnyafsO64}JMR`wv|y zgwo+Z{G!bLA%q#OS8HlLpK!_0ndVq3cxm95l9%Lh{ro7@ z6(VGfGmqjx5ND3lKzeD$1u+v~@ZbDR%lU48O3?ttqMjXQv>{**4qn4n5rOPVjqvc>8ydEp^h#n8_7Jd3(zUN*-5 zJTVGI6TQmB;>Jz3E65RcRfgzAtZ4DPzyKmad?MwG6;!~wM60w+nuBvlqP|<(W$OfY zl^BUZatiC{r}&+-hUU;}+Jk;fAw671S%3MMKB<(kv*ndvgbQl6PSx>F)gf1ebEk^2 zbT=t>OYK0T&cg9cJz*U~==DowyyvE(npW)RIZJeoE*O4f3X0U1^;7+l-p*uuvt>ZgzU;HwFD%{%(QRjpZ7CUSI46EnVnTW}Q;YUJdAy zTigc^KR5!}q)zOhPV}IToZlzSZEZuNZ@bV9#ALAGzc{cR#qXEd+JJKA2TCFMMQ2-9 zNorZ^5>DSifV$s&=%m7ScwUoUsKP=ko<*PDIdRL8HZ>LBjyc|SEEU=RS>_H5X;C(v z1d`^x=gzO;$ZYgHR!?U!^|R|nkomCdhuBohzClsQh3nv?ltgUF>Q%R5Ss_h?RF?T@ zP`bC9VkmHTClt_tOA0O_a93WNe}i%FgycB)PkHYi`!h@tpZ)-mPnmv8ynzJwN0dj& zzOL}ysR(P+$s>BEk30$Q5o&ThN7=OGnG+TM-JPG!bEmo4PCweUA&fb$#;}jXsAHgG za?bMUTK5L1>-kaZrxeate%a!osQFVPv^*5E6Pa=157_aoxn2nnC%;|kz=bS&*B|if z>hlw$?CL=pZ1i(^G^VZl!iVG_xwymf*QDhwO$UNGarV^^UV7(U4tRl(LU8bQB?&L* zmAriU{bBp)k#5J4y%edtdh!IpO}fBaz3cV))`074d0^X4A#L9EU3_a>R}Qcl*L#GQ zUsnz}dSwI}j<2ZxF8r1SJK-x4qtHEr8CT|NXr{5*l-xOaGm%?!9Zg0ZTU&$_L7WU*@C& z<=?NT-__UO4F8ixCi8e>mi|8R{64v(Hv1MYgf0IrJpJDH!Vmvq-v6)Qpb%-xxMFW` z`vC8}-Pb3cDrZC`WMm$ds9>*~IAQPeOhqWf>W*jp0J0)}WPvPSr0SK#?l8uKj98)k zp40f9tt`BxsTDHKajAE#rCRVksAym9J=hZl?1 z1Z7k0-B&#!z}7g5HUBGzmoRXe=!kul-6tq!#?1MFd{sJt>z#kbZ0F7aL@bo!o<9L8 zkc6`T8o&emC;iYN06By4A|Mn_9Mcxj{Tt`b?IRE|U;DP$hmiSz`rfUjk+=kE@n&q? zROgWp$(p$VR))lugJtpL-s-DG>H*&UN5Cg9DPH2aOGB;>20Sd3x;YwXT9#A3UxnsB49@b>~zN8eopnqDFO2x?O=9u62wNc8a%}XYXv6u%f&y$1x zs>PQK4BvIBwNCS_l;6|H9H@vCqq-r*?bITGE{Qx_pM@mM?VK&7lL-PpZAbNRC?@G0 zEzZ~Q{?cFmLY8=m&hL$LV;!=l}+ zW8GToJDk7QySUYAr#?rm9{Z}MWc6Iz<|UR1-~^U=y|?z93`>O*v%{b5sjHN8@cnEB z;E{x-#9xUR;zn#?g2Q58(uj^QyC$)^?}=O z`@Lt49L*}gQnU}=BndUcn8D|zG7xC33_U19@$a2*V}+;;*OP)az9$i$YMyv5pRpwC zYjUHZ9lNYnvHuiSLseMyvSVb zw71q6SWh&Wt15C({LeBTI65V%Y!S!mh4{5+u=HSfBO!5?*PmXm&;m1=90|daYajN9 z_umC_Ur2=idP|Q!_>~JP++V8>iV=f7oeC<{a|h&7{yjSuej^}!mVYulhJ$)%gJarf zQI4a2*Ocl{p^dL>4PIRaS1a$DFHXNq**SGS;e1G!=qPalBVMY9P8b51=FA^!HRPHp~4?XW45JD0C+ z1^oBK3mt?San1j8)032y7*GyEQw*0tB&0F3tvyw?Gy`MAbk$&lQOk=i=OhD1mn%^p zGU3xlEOgul>y$t&C6Hj-`tO!gV$J1~OeVr6Kb2)o%||Ofvvg(oOOISwCsBE#ab&Lz zFn-7y*62BInuOV1`S>BE&zM!`n?VOSROd3`;>bL}CSkM^w>SG0;RZXSoIiUqWtjqg z%x?P&R~}=KCuvI211>Mxj#g`J)4<5&NBfe`pbYB(rO0sPVW`K=5~e>avqILaQct>J zcL{SdAwv@Spu{X!%>JMSpNK-C={^gehC*q@zDqV~nQF>!B3>h0zoUf!4CX{PTH!5~ z?N1AD;&JtXZL?rD0SJ%jGL96<143ee@n0J%b>*YGW~Dd!|KHjx*3_u1PuQaX8z=);Ij>YsN;xRJg0v; zcT4z6Sd)IyjJKOmWc9S}c^>S)3?9_Cg*!M<6YR6h|VFbVea>Q7u+(GeGG zf{6EaV;;=tS2wf;|H*yOiGfgPTScG;vHf@H5-JIEl%%FN@6UFX`6ylHCrCa7GW_iJ%Vr{%nrCtl2a^2oZExuSXX35h@iAuBfS(8L3-E4D9_nFj{^);ckCujCfvNd?Np+cL;)!z9%-;VPE zN}%?_LkWVQ`*^4?e@ni%(?xrs^Mc1QCZ=1l5oyrW_~KfJ4}7jLQX;T0e6Wc_UO?J? z@F5y`My1pz#coJ`kpwi)cgL1TY2->i#FBlSUyy$_zVFa8HM;K*lp-rQbByzkE5fhy zmvLprqgPN?igG&Kx3yII43?hIv4&-8)LOiReYLc}ACE3sX`&7RLQ%dzqi{VwXAwSU zF+S%xn>#LtUBYpUn}LKLNv!?AN;G1Upf6~rM{z#!fI}+ee>Ro8DPY#M9@84BCg*2z zp!C$~EIF#i$c?@0Gy}8$rjJCHK@EF86ZiF-#p!xQ07Rq^v^Y=|bvA~CS3BU0^->^n zQ@|TYnq+LQHevtXGegqzN_@p`F!H^dZ)1hVbXvLOv~A4I-o+)k(h6vN&y4n3%5W9XYNYjjGaG4kpZ1! z4DsfgMhqQ2Yk-?z#1(xqx4I{&{d* z)i-19m-l^FeB5t^^Y6VvZ*WkGM7jGB?W!YzbX(y$Q>Sv{->X3y?>&p_LUDm;`i0BM zTSXBh*_lNgaDTQ8T@J8)JmkBdGR4&-2@O)S6802?P6=T?QtF|esEiHBg>>|Q^BUGg zv(hL zk?7m!(}d1UkQCPg2#+3y9DCpo4rU)WB;cpO)Caw zCNG+c$ESY2lR_J+3zz6%{Azc;gnBELPR=wd9ch(X&K$XF2D-LJ+fo4J-N`&3eUeh^ zy=@g|RI9H08CTy;^YPq4My+{L=%|)zire8mES|q%>7}QmPo4cg|H4>#fEq{)E0eDD z@~CU~*l|gjj#*An79mRRmE6h5zd_d4jlA-N)miSro?RaSQ0s{~*^sVQ!*RCGECid9R{-EyZ&81^|cX%hsU(?s|7Wl0fIcv zHnjyx&QK1p%%K(W2S+EOg#NdZ$;kx$Zg1yI7M^ z+L>XKOKAK!pZeAXUsxG!&ejD}w?7A0>g8hND9`2V_+U}kkvrJmlsfo? z>C5UZ{KHc)vDcUTTSTR+iTq(NNX|{uSVueTjL=11yFaW>bhbNVSbwEIwyykDi=VO8 zsf_OeTN&waEsB-@!Qe|-H(pqHV$sW(O8>e>@2g>5t?eJ!lYhB|G45>z#Nbc_DbPN; zcVlbQV&>EE%Tl^c{_>+L@tS+j+Bm=;-!|}5(lIAJd0}2ozQ$89@Rw*{w%t>Af9e5C>F6??1JuZQ}M*Z)JQGc!xO+~kq}BoKCu;F0izzGZP$nzu~vQTIgG{eAJ6`vmjX^sgkbj`IS}au-oc zz**&y{1Pct4a~*5w8`jn!Sp|l8eb4%5S>n#UK8t%IS^i$ zgeLDc=~v|({MluztQ;H2Dxq6RbeCTq@q!`vm!Zwmpf7x3dYD}->FMg~F zh?&*n!JTvUL0vKMfcG~fIN4tP_+ybJqx3r-%iqO^P^29y*Anrjzj%J^sFrkEZ!)q$ zYf8;#tarDjjaiqgbPMK&fkHXfk<~H;!(uO0oBN2a<_tOOSx}=mOSvP z$r4kL4&N-1^byEM&q}!f3q(=bMy;L}i0fEiK2PP3pJ%CD?tYo&5hEDax*Y7S({u-7b!hzii_nzW{1xQpmHwBk8&RjCOI4pv?$ys`(@L^P z;Yba(F{a2MPtg5Uc{0aR<kxR#m-X0zc$N^K&O`TJ^QDM zPj38nBS>z$U4Hy_*{+lm&kKt!Kbf}HQ}FUL@HV?$yw12U-6|gWw&|6cn$8x_~{rx+djZq`!t5I1gI>K)PJNWI?&8`QmBpDQh0S^t@m<*cTL-6q9=7^ zo!=+;-HxcaZm;-gUqJehPfoeVXxH-vPHamW6Z7NV9{PnQ*ja zzA3) zFwMCz3YFK4zhccxBwVA~wUE_&kK6SP>WYG+Ugy#SNY_<|s*GoI@j!{{e~6pku#4U| zFztKAV?h2Kn?XS2R^fs>kg-kYCz|C(aH-k5b2JWcUhh$G?rVTC4Y>@{oo>Q5{2c@T z`I`5od+yLVXHYSq^?v!LOItKYQHYnXMtoC`0s;T`F3^)Rtw;LXX#PWwjoe_q;$!ZM z$Q9|e_>&x|;aisc3MimEYUBMix9lNtE57_41k@ z{9mP=b#xp_lJHxyWJwk?SI?8pi$^)cB#`7AKTtrkjdiIhryLa(P`>F*5(5xQ7;_3JK;n->ue z?fYP%nno`!58PUIV!klW0R*E-hzmd-Vc z{Tr|krdId$E(1_J3-;C$oJ;E`q{fV%6!)Jw*dmTXyCkzHvm6w03h#%2Zz=Ic#p*Z) z<~w(76sPwF8MyXcs-#&@%CQX|`EgcoyIp2C71atXqD|B?uDH##Nf8FN>H?MgTv{!1 zGjX-Z)X@@gwW!oj#NnzFsqRQZSjJF>fbNu5sc(9zZ<;v<8|P4KY>EU8XT;gCqO6I( zQD$Nt(FI`6EEA3IUx+u{Qk*)<#>#N4L)3L23IDbv4p7 z5>}H~?nhE%o-=KfOlyzg6}Q7A&&tXenStT6ns0FflRi-^X(8kapl#Y0Lr21?^8CQ- zkc#s*G>as^WhuTxO@C~$qO>S2zr8n4;B9Nt&{7 zDP;@}0BACP(&P_$QinRD=Is^|h;~!{%{WfO?8<$=l?wv;m2nt;VH`VMeM3GgD{BW` z2UBY+B~yD-2Wz|kezm!!^t=qx@DJyALt?u5dG>qnK8!3`D!}2`aE6cqeZmDw&gnW8 z@+VRkZiCrxo)c^qMC9?o8FRU`B+G|V@tDura-aI)Jk@%9KfRsZ4J4tPil^N8&}eqk zL&}yNnOC$-uC_@xz7|!P96>Z*#@?o{ROzS6kXeQWq>sjhIX6T%LA#-4aLdaxhmIfI zWXcvaNn7V7g`i#~oRw=l)gWQ)yHh%zF%KHF=%i(gFse;Wl#d!00QZE}mLY8v$;Uxc z=Gpj-79SwM-nb+yMa@8+3+Yeq1{aF(`k_Au+rbd+N+dS-IV2qlvD9(Py|P06d96u6 z@$y(*i8(OFWalEB1CdWo2zZ%jqlZ@7F=b8v6xpATEqn*!m~ZbuJe!e-p{rS66+wc^xi2Q= zenkTWED{v7q#T4=#;c@ky_O*mFnyZ*-ZK+=;IopEU*kHW0#`?1QK$e+xu={@d|Cs0JEjks7`X@1HG1v=Xl7jAtb>i!(K}yhaY@#XS*Mt^PoObsdk5c z*!Tz&tyGK#;)8#JBDn{&zY7&<8RnrpQg)t>Uqwuf(8z~*a_IVDh8ZgS_+chP7B9R8 ziVx|86)#LS#F%$m7JS5>26Mv1Tm^!XGbNE9MeR@VoXMh4W8g;K8*Q}>5Mvv!1Ds3J zuhiI6uYDn4(F(6fwoGRi+)_a4uze=zpeITtDM8#Hv+}reU~vMwK0moXMNGxIJOB9o zMdpLv#9lAJ06-4dubhke-#C}RZ?5#luNsxroiP{DH;0q*RZ>}_A{N0dmWxtI>mnf! z&ty!%1JFJW6ZYb5tkrThk&MKVN5;j>*qOFA`l66NxQ|x`%8&}G?KG-1z;ZU+wcZs{ zL3)9OzhT@wI~_X_y&)3dRrq_7&~rgY8CRwyT6 zQL#9$xtPKat_`|^uO%^Jo2dq+lKxB{{8%IN168m=e4mS+!vb<9 zGedp9I~H}y=Gw@F6bBy!V2Jb z*kdVQlK36iqWR5xvnErd=&VOuQtEH_-E0NpCus4&XW(&TKjlzur&ArtZ`82OhE(pY zezElp>5>uD7I9>7N_$o>sx006jDJ@3*+JIG`F+m&7FTgzLCp{`Vr|be8FBjg9;Fu< zD4jto)zY{maPWQMx*IvVrw@S?{vQlV_ZA&mMBNNQ)@_hCS>l@?K1Zm-lx za}SmkT4&bha0JZ1TJwcPF+*Ft?HCiUGIZ*rMd%dYJBAcw<>FK1;1u@?^rFW7TndNE zfy$Yqqa1-FvLf#4EW|(I3q!i*NwY#`Qd~idhs{;uNpG_2o9O1Sz-lq z+w|I!CD_C92xY}e?S3m^gqU1L4*aey|W%YVzKS( z!7)JU%U*S`oR(rRWsX=yH}4=79_E(c#+Q<8s0C?xm zq1!jeKIw6d#EvkR-+YZT+vi-uiwNH9P-Txk#HyT9A zl(zp}I<6r@w&kcVs32^@ESt(uk=lf7g|LwReOtNb1oMC`d#wTCi6jQsQSP)&V+xX_ z&~(mH*1?SE9Wj{n-oyt1h3b|}_h`9AItw9PC&fgg+|4@xbb8uWCha#|;@RVNr;@QZH=KLeG0B0I>?NqFz7K;vWjE}qsnv+f(Td)O`QI^-AvLWEcKN! z{w%djuu`hbcCxIX^s7zr$bM0r6E^g^z}0L)XX6UZOc`qFMIpOs!*o4qnWg>wiJN6? z9QE9OdNGNC@rI)T1WP2%4vMWlIuG%M!eFS?`)>X?eUc8W)KmVaBNn-r{+{fl7@hB4P0%0 zobeUO18#%Z`K1TfrEw91z>v9I!$un-^>X%DDlr4tWw#$(Lie6R_Xe3G*t(_S2GL8C zDI6dV#*HmFa1R1|)C;|@t;uws%Q-hbeidgP?udYA*^2*YgY_7$_7a^s?fr@5yOJUp z9S+aYG9Lg_K&-!1_gx)vkULlR;9V}RV~^E8SMA1R46L^p-1ikD~ zza0jb9Oj63%)PwBZZ_vKcO`r5lPBM6n%wEE9>&A7@8<{U0xy&Yea529qSq+AM8weL z>oELml-OydP!BD^E(MBPZIIUp{2!9B*{|97+DE#lVK1E|aV%vd^o7x^(gqOKH?z+W z?1z#;L@%?R9hsE-nPMu^%cVbX#GYsbUU-%^jw!=wNbkj|maYR@^^oJ0oSHnKe|ywZ zwhoCz|9s3pBK=<;^+KkG76xw~^_2>i@(8j>Ueq*s=V1_LaVjxNHDLyYiD@*cj|j2( zg7IGseHEAabdm;(vS+$WQ{z(8PQmW-)22Ere0*pazWQ$3Uu8XWPPKj@H6J=3#ARHkmt;mJX+ zTLpa$z5uQl%yv@lvE*cm{z=O6m$ud(T8uVIwhTQL)8xewZfCZz2q_T+ixVT86fn0? zKDH|baYw>c#R~q_U0}2!e#~UkGP`?Gn;wGvcU9DSSH>|97k& zS|bb-L3{yF&FEExj`|q=8XFTc_o>S+?3iI0Ohm zw()p&vBy+qn&2a7e$nYJ>AIbpxD81zjizYva06KH$;uPygS3z~uN#(xJCBVD>@g-E zs0Fgs2pRzB9eq^?#d%97OY2Pnef3QWUvw^U)L0d)r5R4|z40v1wh8k(ydMlDaQo74 zd^G4F?A6x^l$p`kCBuDtz2SNM+u0|HYm3K;Q!z}|7w50`-d{6E?4N?6*Sm}<#&(f# z+U=kVB&&g8x56beLM|+NB=utNF>*O#tR>d^9cSzWMi^qV!R3$Z7se2M-fGEOMmWrG zFd|f>!_0EQ%RiDU%l*>WNOplV_P$%f4nuQ@*>9&=7^fSC^em?ZQS(ltcoF||0#FMQ zvzG5mB0nbc30dfcsaSR|@hAU9#=9QqkdkCC2_I3E6&T;u)6IOBD)=to~3OlZV-E&$6C7mW<@}q zyB9_{@of_u221XkfI}hp3qQyZ-0foX`^^N?dN$66YDFM6W;&VPQl3pW%=2C07Gr0H zK(R{EGMyvc>8bV#WYpIAc)P<8$3byYe z*JUJ#<{OW$6sn{UFE9xCy^0|1c0|3B-D{_dNSh7KmyZ=G{c z#ognhA~H8id%9%PI-_X2Ihl z$`Yi*TuudEAihtZ;Na?Rm0U|ek-|QZ=YW^^e0a<$>FBaQj-50h9)*2+{w7>+Y&6BS z&9?2j;og2;xqd~QCm<*@v!`HGLd8_08hJdl;LA+JF_$!L8rrlKfuOux(z?5#lby6^ zR~nHWkmFfgww}Q0ha#ovF=J$?08cpQhtZ~iyKLBCAT`+lNzNrFZq}L)au3GlWhBQHEw{n;9otqcWIhV~-lYl_V1XJnWfVy#NEie;5>^uN;G zDU(+kB^>0GZWkM<@Q4(jxfR!LneG@V0IAzo!WUpda});3a*J}fd`<0mlBg$8HZ{|e z(Qer=-L=@NxsjeXUU^)#c(D*n!VtG3KSpk>iP}@KuXiQD?9VS~#LX0zJOprT3vh28 zq!T;nhX$}05VF$?e1m67Rd1!LDrz-*y#wX5ti9}N9IpJV+FZx5G`qMozdnn3ZXrU1 zX`pM>mlSU~QlkDvO~~@(M76#lg1YbMxMuWJeTWFCEv5^tVEQ!IJGIbWwM{~1bi^d4 ziy!hE>5bwM$kma_Y6^sbRIZ90W8){yS}cxslx2uBM}UJ1L*uwoz5#++yZEMeWR=l& zm9B9WpUS&Z1?u#1Qd2IrCp|u<%OE;2i*SjwdLv})bR5xh*#$N$Aeu;*GIeJ`D%_4XTf zUqiembROteI#0sjsYbmMEmL^?(_LsN5(Dt*m@bnh;x(1Y8;P>BKBu+o*Pd#+$6)I% zV;e1F2XUw`TwLcF5JVM1e%1BqR60LV8fK?QH*A;MvU<&VMx$|q+Rz)Rn&H(#sZ3{* zR;^&Kg#|seeKolS5x+ILRWA@LX0Pv*=c66gNzmub-ou`3f81e(&MGKgNiJTEEdE-F z=TGUPkhS++ca;}C9%#OEPbX{M-!+Q}eIk2O=b$NoidX(Xl##CehBn=oPN2Zf+|5_Z zpj%5XS&LuQxG}dNZrpe$R9YOSY<+@Zl8_}F7X4V#o6@nm$h4~?S$}-$M?O!J{|v<) zEb&7&W#<^wkM#|@_J)GBOdNC>2EMiY=&@zhT}KB6;Oz1RPY0jrL{7*{Y!pVd+RXc|$B0A|ClxA1nz=fbT592=fj! zX|)xg2A)qz#|vSw(FyIt{Vs>z4Kh&8Y`BzqO-?<>jbkVne%q`f8d+CC4K1;OW($Ih zNiKe%M{}tMtyeS%Jv;?%a?@GH!5>6zPA#Vild8t7wB`Mcv;S73O@lhHkmEZPod6hS z0D;KJGZ31OeK0;J?A9K$F>kg%OJN3nkXAzh(*ag7Eaii!%Quyr`;h70Kt#X-*yC59 z(|GESg_w03CItlWr7?40H;sZ~V3-?>V$B^hnKYZXe26UU{gwK;lTXNTv9@Os_^lKd z8QCHHtPn0FsEOi%f!ciKG04=Y8$D3%<~}28cugEqnws=8z>dulA7rZsfKsASQ(@4H z#;@H=WJ6or(2JHa1LcTP1rgkrgi-N)Nme?sgIu8$|$SLd&;KN5@T6=QHkCi zDA+{E(}c+sm0M*nQ$JvgmQl=z!dDBF$&tP?Qsq=Ch?_#VrC1FBSLVk<&#@M(Hi0P3Br6FqE8QrubFs1y#<&BPB~kPJIlc z-oq@;Ex|)ES+R2c>5hVi*@_A-OiGMQIm*#RLCoVu`lgUX2WuD&>hTAKL2)j_fx(#R z^1DcFHs7(RR;Ph{N^UH*a0h2#)jRX>8C5*a+&x zo?7UHq#TDI0|r<(qio@6soxI~G3aIJ>g?Q}gwz^jOz|HgpU68!i2K^2(^6-{3&ejV zeuOdADTqa^ny#UX$4?RE$d@CXVRI@d52qn7W;r}pi961IvuKPI_L779D zncDjOlT|*(Gx{q^Th7xLWqI?F=8`b0_OO>N^U?FxxT7sTRI8G%7GCYP;*nADYNobq z9D$PE@f(N1cejVgOXCzI_$K&59oX<2K3>n*SRvvg^yD)i+9IR!*G=DTsw(WhDk=Qq z5K^Gj48*?2b8i8#*RJPreE(_^K6T0R`STXo_9a+5Y)g%~mAS6DRlsSwLuKz}^NjWc z^`*Pj*Dso?Jj?60&beOLg0Gy6`mY;+trjnvQ~OoUuIcn)DyIP9p4a@Emi5BmRq6-Y zsz7Qd?;a=X`t?_)&)hH^(k&~UnU^t6R}NR}?S|W1`pqwdtgat2I4}A}(NRcv50wyb zZ`pK))n1*| zdY@nIGpnqL#xyOHI8w$9fs=iIaF;$xQoPNUm7(STjf<776K3E2er^5?zfGGf)`VIGH+Y_Pmd-133J?i($FoQ zpRwd@+?h7mQLP{G<*Z{7ZUV)IPpf6mb9ue%V+qPiRYk8mQ)tYs_$y-l$xtZTD z&Tm6|pAk7~k8VBw~Axjt3 zQu&T^(#0svI0HN@@*GaAhL2I$PjVJS6|7DL{M)g6M15*Qv9UpfWfhnnEf%b9s8cAX z=l04fC`8z4SW`djS@J^|=If6&2u+DY{14jd8fzEcO91yFv@`1#*wN;L7?1U++4m9t zzP2F@LAZzl;GKovrp{eTkh@Iy9xv{b~V_m(7PLXEb^x^D8+ih?`fX)X+p;}7EZ62?FUY+0e_WBM?8;T07pQI_F{!Oz} z=C$B>K22U~xlD+x)6g8$b8)RI@h)Q*O>sPm7aXcN=-ICi*uQcVv0Y6Q5f0f`d=+pQ zWSOG4nFY_vZJr|JD77P2Cx5LXrBkZ8!1I(kg$BAAA_UK0&8zG{7IOc znj!)mmo{y}p<=8tDO_Z`bWEt#51;p|472a(S9AbrNH68IFii>*;Bn z5z2hJB6$^QyW`D?lAw)LFM3Ag9%oQX9iDI2EfDjp7_v-(hX94gOjw&AJj*M6vG&o_ zwLi_1*M|73{$$N5{c6#GM0**sM!%yFF?wnNge`|4WTnhZD|IZMtY$M-KRu{hp~*M< z!MbwBg9}dUweY!?(L7NUNrXA%*d2p(5VJ_6N@WN6UQHIqH2GrCw!?g77^VZ?^%!(7 ze$m~ud5fVS%f-kGA0vbC0)A@A8ey{G!1((1)A*UhcD+l>>!C32I(3?odBi_HitHKz6)9y&4h z*USL$-h6LA;Z!AAHRt@`F6C1*Y~3&bewheUjz_*BxH)}S17;BS0HF1( zI69`L1Z50h_v{p!I}>vnURQ%==Vkx3&l`NVm>zm%w*}Vxsfxn47>Vh?N3pLpEBx zxNrZc^kL&Q-~eWGr9VZC2fD=T9)rm?^WH?DPLZX2R0@}9wF6@q+Ehx1-|311CyIAF z;k^Cu!7}y=`I{Y3*DQ0_dJBQ4bM*D5z30Pg*Ybh8Kv(mR3q%j8clV}A52v}?U!(3t z37oifvX2VRDR5Mx)4#lDp>ExGcc~~E!c@77-=1$Jo71@bn6~K>|FJmKej7i^uq@L! zd8+0(CB7br^z%x8*c$KBG|lWl@+_qMEi%j2K4cnn6!-n_)A?MmFZ)VAr}JdLoX-EE zJn-)>`#fg8GYPOrBLuJWr3a_tBb`%I$04O_a0E7ccFn zGW#=i_vph1Ct8nhE?zILY%e{2cvpA4-vRcJ&LK+0ww8?2v=**oS<|mnGOA-rjvQ1E zo>Llt86HAZVoq)f)=yPoPOStyL?=wR z^Ij63g9z5JWWUS6Z7ys)kuNQ+Av9;8As^Uy=+;MbR0H4aD7o-f{g92}f@OVj0fB-m zEB-d1Ug_Zd+$B+0qLeyVW@uRiRV1AgaQM1@s7jMSPF`)ZY0Jk%A6R4kOjKl6WdOFt zbuLh%BB?+=Y@-UXFrROmyo**Sitp`Ek*=vxJSyhwS%0bub+jgL%4fj{1dB6{OpnCa z8hm(;_|s7Jea^17n}j|VXnEfVcQKtoAk$*w+_MDt+(PD2&qAj=`myCpK90aTIkKao zo*_9BDbe9z7>jsvkCkA>H6v6-5xDHuGMB(TCE|QPDjP3-c?d%(&16gZG*ybsM5}K$ ztqL7b93Wp1xtrC z--$Rbs!%~QJjTa(;zTarK`ygjY`CJfB{SN#*aB*8K`nzwv7M>A06{+c0hQyR7jG$` zlV^B|nI%#Bs{q3600Uno5nuvzV+ULE=t}R>$&S?&l9w?VJ^oZn$ zuYKo%?U8g%U9KZ~@+Ji7 zCqF(C$GQNyUUMq%j}Eke3AeP-WHqpULtr3Go1k0eOgRjCH>$GWuP>48%EB*t~L2$$d1usdWDAMcKKHkgrdmB~f6 zZHkPmM6*r_E%~$dt#4Wv9`yl>4{FEld|i~+2{BiYFCOXA@ubHeW{{vV?4i#6eZLs7 zRaKMqFEEl*7|Few_YgGB0IM)F zPdl;}N8amHy~&SA%i-2p6qPs(ty_<=(Sw2HQ=FRGG>&kh#- za*WzEI4r_^&H2kSc9|_r{T?@J55~KidB7bjz)NjSF|MY`m4~NWqz`t+7fh4+b?C^a z*41C%o`6l8jmohkZxy{FPKr^S#A7a$;Uh^EC2+#vJ5GL0oWgIzDOJ(?oK4L}oTtEB zCO*!XI|V>VQy&Ls7+>`^-YL>gF@TSNdGCL`FUf&L2Fflp3owl?mWeD@HJ^riQT5{y zzCITB(>CmZrK_>4)W_~nGg_R1N@}q=ED?>A(2UIBUDcwF_}I@hXb}}RNx3>vOYqSc zL>n(1aV0`o(}Eo;-5eToTz6igJ)cCDA)2}DTZEbH$CFuL6Rcb7_?~nNUh8_`E!$ou z%$9TT{9UVE(bVN$RQOV!sFiV%q+RIGSJrHyXNV(k?9Xj?ou3pp=r&vmNVj-&D!fOI zz7jOm;M7-n^>y9I2d?E%rKdsGfqo>c$|m6vilRSaN%xy}PI4>WNtZ}H$Cd|FV!fhE zzGf7@LSnoWmQS2R8+4L=QIF`;jOz1B{Nx_3SGG4pS%i}CR2_0-9_1}xXJ*APu)W9K zx((xz;c=BRG-%+xoVU<_eP%I-<$8W*iV`=@;$7}N|M7AO-Kz|F_*>`P-T>ugTC&c& zYLXpIukTT|-?^rGact1Phg3cy23;FR?9hblRTCqalMQi2$fufBx%yfV%ybUr4xh$s zFy*|aIF0295OwPzkRUQOlWAiVfAYlgz6SOt_}X31{=N6?k)(WIyCT>-e#*NB@`vLm z`<@sqT7#>8;$VxR(4(Y|lX4`zK}XlHi?}@bu{MW=J7zY`jPa}7uqHFm?O;JRZ*Q8u zK)qUu7j=optA1lF;Az%m+ed%Ik_Q$XCyb3_$4B$?S4>zA^-E{+$<1BA&pU8hbLGTb zIHy!ZofsK3z$@ft@Cf6N{&g_QVaU{neL}6YS3dq?tzZ#$%dBIS?muWz7c$!x2QhA0 z!#Oc(gfb%Q2icPa&m#n3_wc|>>-pbvBIv-tPCqxQOogK?yGFD92x049WYeZ_zYtXj zLswKIAd;}OO)X<8!lp7cbJ$uifsn@{l1LoYzKSaKI-V<=wZNFdrU8HIdKq?nxQYmTtJqZ=4Bae zo0-cJsc;7;*@mc1&#{YoqWhvKY z9D=jA{a`ec?+18wYLD{q^+Hxow|G={?l2ykmr``U-Ae}c!`Ka^n@8Zs?vi0HI}e5E z=JA^hY}oM|TPE(PWi5WeC)ui(6^SBTVOR=R^lekjk=8((%-cqLj`m~RH5HT#55vtc z%JtdF6{D9gHH#fn?Z~XEv|ob8YOcsDQC@L=3rIn>g~jPU002}dzXhZ~z(|0%At~^G zg`^Mw5Ww3e`ALC~N>oaS=KI(`i=KOnN;m@n0M0*SU8FZ+Tz?joOK0HQaL<$n#~lb8Hcy@$Ud z51t8EdVZd%{4_b`8zdpyzeT>uIQ}cSMAz?z9TWf{f&3#w_aBX${a=y=ObqqS|B=5B5V8j=a?1$@K^S${;%1;r(pb&eK$KQEA`Xt8BhQK-5WNq>95&;{Du5) z%#uwq)Ux~2;154lFu%cHSp6sXKfe9Fjy-k1VJ#{EfYb1gwXJ_Ndb|HX$KS)TXH4H2 z{Oo|tPy4gH?LhXwf&YF-`tJ@%=~^1f8~%&2U*wRqBmDEU8Sbw=Xv+H!l>9v$Vg%d) z=BK}Ny$1kT-@w(%e+~biRfYfVg#1r8GyHoke=fvM|3=GH)qket@A-c&WJCCdPucdr zEAIYR;-7a?5WgWx%>Hl7;rx~P=M4x{Zyi}jK8A)oPPcG{>TpguhGBQ&+u0T ze@^;(dzVRu{I!CA&guRu`_D-$Z`po`zh?h)64_tbe_r%{>pa+~zh?hU63k!We_oz@ z3uh__`((sH-~T?=etxPU001w#pTiLF{{T=+0|W{H z00;;G0*y&iYhLCGMy~(>Ys3Hm5C8xGVRLOQbaG{7EjKPQE^1+NoVo+BF42-Tczo{B zz4zF*ZQHhO+qP}nw$1O@wrykH>+XN1qi6a}bXQbXR8;KDsEEDxTDfzloFph1640+- zzkq;%_<{cO1qB2KBrUAOPc1GZLi;@d1Oy5sCkgq_aX=vdF+}dabw>J!{ol^g{4(Mq z!iq{X(jw7ulD2_#e}itsK0}dPCHl?FDpZ89?swzJgyh9N&&K-_I_O!NrEz|GNTS7x zcorX~y*L@6kgJ5OKT(x3#Nh}0Han)<;K43aXDW}MLYse4JY01CCPy)Q_e ze5cB^Ge1@Pymt*iabG@$yTap$>WW-;-lclcGF4f^+Nf%px~@&%L+NTkL#sBylSC^P z|3jvB3hyM$4oLw`R?b3K$_c$Tw>lID0LgDKT!!PV&r@nS(5bj;rCsTk8ST~7hVIr7 zytx}WmNA;sT3+F8k17yg9CSfLUs{c-d4T%oY%s?@eMtcW0oj54tJ#46hgsViIa?W1 zo7b~SmqK!}Jy)PEK@b$=FBEgmM^ zFA{YR?Q3->6f7#~Lzsv_s5o*Ha)XLM@WV0>z*+6F?GKHLnh0TKd%Wl)g0r79Zx~(oT zi~7fp_Wes>f&Ig59ZYHeCt&>jA3nAQ7RH87uEqxcBX5NNm$$y7_5UaW>i=g1CkJEW z|4|s~|2_=AfuoayzM+%2jiZf>t&#D+&v3KanWLgI($5x^bSBsYfmyS%Jt2{q5qz=$ zb5l83!y<__HO*@V4`48E@ep|kPEL!@ZYPyChSA498<>LT6mr!5^N{ri$o}(^rI0b= z`&!z~U0l&TdrwgJ`licqhW*8Ly8Wfc$MZZ95lCfM+P^q>U#*RGCRC_<%cQhWN3z1cl21{eOL77X z<%UUOL`-Jp5@&_*&8vS=sxXe0KB~>Ydw|JduT7qT6MT6ke0jKZ+@P91J#(67ns;VQ z_GF4zXVDcSI6OhBq_5$OYULzLCG+9E+r&L%V=VUk#liU^s!QL+e%ybxfgjjZZ!Hg3 z)vy?qrt031oQv9S7lU`;Xs%g_xfLC@$XJS15!r@~nq)U$XavBpTULuu%11``K=!2; zLs?viIq{qTyB1qH(Etal1~7TDSKMvj1I>im+EhWqCUQ2(UnNIn>aDUZoQ(xa5w~Q8 znyi*igwzVIG}0~Oh8lBfv2)WiugonwC=N%J9qX8o8kci zt1thkJ5iu6xq*j()c-^nSL{$n&ucfkad7*Xq22Av(7p2&TFGP+L^eWpq*@Z#<~!hs zFKac17^ZAeVZ?Ce_47_=G!Dvy_a!il@XDCpIU1C)L+yT* z%He8@O-@LZ+f7IZJ61Jjt`OQ3fSiUF2B~%?M?QsG#5H|V?{y}*>(#Y2fxvgoE*40W z?~`taUaI_&-OZ{+7N&V9Ss)IE|;h)^N zwF!6$6m)>$nQWBD#7;If3`|+uA6;-0$%&qEXfGZRqStH`Ze6DDTbL;mn(DbEAudT_ zg>2pgvzJ^)16Upl#T%-&Dws!Ox^uJF8w$~yFl{Ft!mEv0wDVV+iD|*p)F3Wz@^i?wHB&g~zA-1m8rLLL|`s}l_aHX~_g`B|8wC;fq!C1dy z?F+L|edSsXp1`-R-M$H-uG(?hhr;QKdBwx&N_7o|(Bk(jhR_oBY=+e0!F1i1*8*=R zyn3mR2VweD1`>>34roY2gy7OqXW$4m?)c!3lyNG)^-6c|iq-Bd><{y>UdWmPa zr^tmty83h19x8N0?Ei6_ArX`jazQzwsjS9TT#u5Ce4rov77fo8z}`e5Qk0RIko{Smi|xTWdCT99RjPua^y^0-$@%NX&tC8U8wFp z{5Oxt8opPt*r_QYwQ;%i=+>kY%*nCYOuvfSF#D&-(Zl0L(v!=ohonQ51X~mr^xq-} ztQR$K^`cPwB(?L|gArkUB#Gq`%{pTtG^!VoDchbY+cc84xy7Q>)fL|%jWtT0VBRg{0cp*gEMuncMy+K}}YDQ`B5R{;qi(Vif5Ajb_y~B!kT<5NW@)8s6XE zzv>WVoCSsn;Wq_~$k+a72o`oM{!X(9c4w#0Mc zjNwYk^qrUKYuZ~jGI+k9@y}V0V~^*w>x|1V?e|9^B@j zV+qOnJ^O~u#e`9VJ7w;KiAN)03siBPHG5OgxvX^Y{aO}Ot-wKZrfZBQV@d$Mfh;jH z2EzS%N()1wwfCDA1><_Y9HU`z=@oe}bBqf|u0$)bf};DmP5Iv|T1Gpq#@Xi{cM7*pFL6aZd#U1B2`U)>Aj@J40?3I*zw`Baflh|D>aRyN@@fp-`X12MAtF3)!D@w^FEuEl^?;{Q5kWe*ry%h1^)<)kfGHAJ>FCGBBQ)y&v$||@Vj#UlxSfJ!SpkzVDFz^B_4T#;l>gL zivSu`%ZuV3IkpZ0+-hYFt2L~PI1}hGA-#v!Y6eoD(P|}yC&lW?9;I&x`v$=*BXn^9 zeftC)M&wQ^R%7BH`zCd9bZqGU-3Tkm$*A&0!xkLIJ0O$WIwt_3*9lr;O++h=tQt`4 zo_)W!JbpccET!cS(7l^L%$pGqd%7D$&c0Ye{gB_bnR3QabrZo27G-G)Bb_~qACbt= z)EkfpJxS$B&`?vH8@?YCiO{nG5Pp#H6Lci=&uP)tjncz8gG9_wZR0@?lZwn_oEF~{ zMne=a^-Hk`qlyp5XtfAyDJwRS0(z&bhr#M}c3sFqhkt`O<(BP{!u>ULW-W3;UghnJ zajBy?XNtr|Te>NOTehcL_tF^ksBMgekB_9}7Zn5X(&UV~1jJr3k{J~9=3kw@j9IP> zOfK8PYlh`NGr z(C7GTTB-XVr~2to>JpK##Jl|lQtdJ!LF50_xu&>D5UAFt&Ja9vR8xmj9^F+3l=3y5;_ar>?@I z%FX}2RUeY$$IO3r68FrQA>h%SJuei^YE|e7Mf2`6KnYC;z#lIiXQ2;Yp&5ezL&y%S zCPL|kXJFF$D93g`#`Yo8h7fxEbRBFy6W4$6YPr%>n$zSp*=Gi@)Q8vQ;{|NmJ@`S$+aL8v5E9CY?ozl9J930w%dPlt*7+t!9Xa^b5)cf3O#|mPo-eux%q&>8< zPENpUa!Gek$6kna9%v8#Cd%z&Q98sRvyW;`f7pR4wyHZG8#%M zQWCwOCEg%mZt8zi@b->BQKk2a9M?Xyj=QoIZj)h>g%dmk0X~0M8Hd0L~4R|Eo*Cy>^P`gQHx|)PIOi`+wptv63MAtUl8LZ(c2J4x7(tqjeKYg>@R z>j=aO89+gHxlZgOyb|U+E*!JgE9$Un52SLnML=%AeOYkjfnW95Mb7PO4yESaxgGO- z({aU?N?N(xKK8|?=fAMw)W(f}a9t^z+=I%zQC9Wm?vr84uDT?MrezVkK)_cz3BIL0 zxOfG3ajTVDdVMsQ@kTPfQ|uqz{GQ{#UO-0G7*LOicWgA<`^7$5%8C@@;VK%v_0UEa zYEQx4s3u!n$|`|eX@oN~b1=;QsL*POqK%((cC>wpU_)r3+PSxNVM*`g^t{aUmSM3y z)KO+ZpW(%UhX9qE@$tGNFJkb!Ip59*N0)jFjl1j8PxmgY*F=oxu-CCyU;hh*|1+j{ z(Pw7x+n@SQbpLxO{K+({Pt$h1O^Vq4{ehWTbFS;vIPfSpdeg@s$qEUuMkYJKA1>^r>vnwWp*Kdq1?2r zl3%S*YjFXb?S8Xin>q!ddV#yfYP+rVk6ZT{%j?k|=!hwfjtI?y3T8Rma#@sy?qb4g z@3=v1j0sS}^fs`!t`~R)0UvWMQ~c}rZSY%+?ob3Hzjeytv%TktC&05E9r_~>7(^Ly z-uvmvgiXp0!fT2Izf`1bBE!pnq=7v?j(1j#f9%%>CTNeC`K{1^4%CNgy>ZdO7?%^kDumQbfxyja*u1Co-Ifr2l#rzjNj zr@!Bkz(eM1;krrV6#zPIZ|qTz4Z-{G+6+_Xg%mxtclnN$g`S>L1w{!|AobQ()pi1S zl#H^UkGH14yPIzxZaGmAa_XdHAm%O;Sq)9PsUBMw4x8MBJ~qsnxRt zq~!j920kTh(jM5Km=hxr{mZ(J@^Pz2=BYhXhEg3h+mIRU_rrB#Cy-SpL8#I zRujj6K;YRMN_hA#<4ICEi715#)X}~K(vdFqD`iM2rGkr2bO~?Zc3{FdQALJuG1a%|l{X-&?;3vH)NaMYiJiq8nxLeW_XL}vld2)-yiDnvfKeyK%MAzHK z5e}Bmv4yIhau}@OVjwt?sXV};^&5g4e`LRkJ8rqJ@D_cf4=cPslbqkRC z1bJ!XT$C#for7g=TMab7WZ)*@8yFj~nt->M#LH6sq}J4HAiB2kpGN=$=Ro*)9@`ft z$~|DO7bO^0LTay)MDMIDX3%|W26?q9c@~YP6S1VNX-{W#WcF6~j~|B_Rm+`;=OeEt z=3@5ZPkz!&Da3dU1ZKM>2#!TfJ2ULFXxf~jve^}~3D34AVnMI1{w~45>M|_9rb1`4* zrMfR}i~Te0Bl#eYLXUI;E!An_U#}mE6l=BTw+@-=a9Ux!JN!G7))pn9Pl4~=f$unW zOk2Gq*~lZmX12!5VjbyM(rZ1zY#{4vJpgCujs`nF)8trHQc4GZ(weL1`HtTgcjFZi zt6rF=9)dp`TH|uqpG&#_g#3QSb5kIE!|@w3qP0A-1#ULcsMN3id;=-a3c)@d{IG)Y zO=`Y(t61(??#XR_Ze0ep{{(f0@_$$*_ z2Ns3c?Y7%-e*FA?20-)-%F4xXCO>~`1Qu)Om48-Xc_+B0U}vU@=yZ%_?)Y2S@ex(FY@M5u z(_8DqM3d14ncA&INK$Um$xpG|7BN5MOv7~t&WFOZp4|om<^$rj%`W|RI>Emot(kF* zKMtyFLKXIws9GYloGBIL9!uJ()NCXamZ^-By$DRt98!-%5e|9BSKKJB@fWP$&VG@- z5e3gBEa?&3w~RrD>k{0@0T2V7SS7axH)n1$xiL+>Td27)QPq}BD?QneobrviS2QIL zjJr@}o);^?i?!Q|T6R?q-PV*VMQw&NephWr@ib^xwpxmgaINZg-#k$LfcqSvvUlt5 zH+iy86t0!x;S*x@^{x<&S|APo{; zE;H8aU^8q@m0BdMRwC2fz$;0$CRYKw7Wex7k>>8Z`C+Fh=FRAk55ap?Sz z#X*xjm7bMaGc_*hOV^)Q%jWdQ!3{ftQareYKUjj8oDAbU_!$#row)q(#vLUK zS!Uufy^I&RIOpxfb;(s!`g0OPb%p2x8R#09Hj>=&kNG-HP5gRO>7AzWx=m+rGKd7k zFpk78KGESP(uR+P95Y{stU>^l@V%o13qh>y{3jC^f<|W%Fb`nr)Rptz(Wf__F~{;i zf9!A2;S|<i3u9hoC&9d-YsBdiwR|Bfvrmz`8953rVIhn9OS7aTxrY_UOInHNqE7ndT&SA ziGuk7qr?K4w2>W6)iK~(n7Q%8+RPSdpD#w>h{Y*Idej~X&MKrp@t5p7$9k3|+FcbA~H{vJNH>aTx>G${r0d`uXYkq7TH9{NWjix@i zq&ev~G5n-vzS;FqXjhO6zG@5pcjWEPy^L2CsGXKZF?N7qR^SIat3ewcglM5f7Uhg` z#>sksY(8O`uZ*Vfd;BKHc3ZxyjM%5W;p$!m&4Hbnq-xNr5zd!yyy zq%9_062O;*cs$0hlF;%Pqzb+fb{Wm{<4sukPTvhz!i(At6@?%aGUoMAS0hqoMY>sZB3iH2OYiDxFG=}wV` z);X&?7t7L-<^XF<`A9(0^{H?1?u%KVX4?CUO7Oc8cP^-%g2`8(SIums3V1 zO<6>Kr0)_{O%&)bWeH`uAeZ(kH`F*V`2-ShoQUXGy@oBBi3FF{wd{bNw?MDiA%TK$ z-acGoE=KEh4g9{qKO1kyT0IZhjErATA0M(nl5YKUS02cN)Yx{CSa%8f@63X3S(uqD zwJDAW<-Z|8Iqe!8YapOpg{n`Cr`f`F`b7&x9W^MBVgct_Y zPRRqqJy(pAhqMfc30U?QDEJU5hGCL>IohZ+&S($qtMW2%EB1o-vUEa}T9-amiUM?r z#Yti2>RmLae;G=XHunv7s8OwVNF37D|>O&;jE3g9S)Gm$;(l}HRv&)p7v1w$>oq^!=RimjJdlI-UL%v#r#drWrK(oJ| zh--6_Gg&3ODiWq+)jET~+yx;HkxltsA-4OY3YoMcQpq^3U95}0x6%qmRO!%IDE zzX5D$2;Qbbs6K+)J3`wouP%iu``b0@6%KC#+JGt)>6w&|7W%i=cp8Tdc*<4eQ6#un zyRKapu1MQ;SJUfMQOFQ}sd>_fk&fllaMkuuNnJMSr z;{J2B5Z1#UErS37c|-g!ix2;{TK-eMDEf?96`@pVsa2z;R|%ID zBW+g9TzZRw@$crftsVB4=V>R7I^w(7_7lz%ubrEZ>>IBg*LB{nljI%Hu)Vgv;&d%r zriE^Rm_{qwmOS?kZnHxOytXYyd5&~C&fp(>I?t{Y1+g^ug^9z@+h0=l(kNH+kk0-R z-04jB?!fSTbNf34kjJ0#4jFEUJ7{>w$CNbmUyDvJ9#-6WOKa| zZ>2tVaW3EvYg>(g`#M_rRn zD6N;{F3SL&8aoYLhM2BU@1EbNOz*xQ)=*3u~H zO6$;Ni=%Y7mxj$Suv}2@|R2zI`_0tPfA!pG2_rP85<|P zF+vVl6cmAUm@LbJdLu~Hi}VpHiXK7XfHgI$^4C*REN1D{B~lY3bC|(PCSdEJU1?`A zaB#b^qmQn)m>nb%h)|`-Hoo#Kq|tew)7L@Nf?ePg>69J5;`y%F^;#Kdzh3LPq-10J z=IzY+Q`(;ukl`+7;rleIR_x-h?x2Bt711w=IMF=2$@pMM)#fMBkvHpyiS4Li47map zJ2YC~Cl0TJ3U}D{-d_$Z{D)B`hI(==K~|$^;P^S!G?^#siY?JuB9~XtIMq2)0tw|@ zd1$TnRmA-=X!2s`Hm7&zNQO>`y|zh+|HvC6_N4#u)1j;4t;LQl;&LuZH}5gZjo-Y| zzpAH{M$eS@d1Ux&(o@K5c_q)>D;OBAy>vQg<9~EAapdHKmpOGvRVeD58WazBND_uC z;`M8~#5j)!Xn;ISWG}L~I26#!KjTv`-vxdV<;9e&NB3Rd)H|l)t9?c^zEi=1#f{=+ zYHW8+$pJ@Nwjp~O_1PR`NvpL)x*fQA!SRmL7H!j3t}?EkspKiU3~_=F7%%8)EsZ&i z?bEC|Je#l5Y@C{GR_jC7B+hf+3vNocN$hXaI{NCag&dkh;P%ej5!xiYhcZ2m+1w1* zu8FVOfRC7BfHug%|$t&nhT=ali9xe4ijj=D@3gJhK_DeXt3raWyrc6T0Ka@}4DpD1{ zPva4`ND(}TvH;9730a8FB9vnaFwHXrsSzDw4X;5*N%aE%^$4iudkMyCkizTG`LVsFuG7oFwzvqObH%zp;cND zIAl_>w#Dc_mv$EY8zm%TAPoSl(giV-(kaUvJTq1&6$UIkQQ*{Gn`t z$6PYTl=lis@Cl#{mm5xWHX!_hQJ49hYdDnT3eOomwc%b$xH^Acd56>QBu)T-PmU6f zm_~B4k;Oi$pb>jlCz@_OdV0);vNv838Pncm&@hBtoknGkJe1}+M=!i2Sp?pO!&(+; zLd>beSw{FoDoNszNKk-!&6LNITYLv~Q}A=Geh}n zYTF493>ITGKy`r&A?RRNh!qK3CSR)#PsnOzPwRN#3cxj_jn#{uo zg@9W$K38DzD`@8ag|Ml)OWUs}pw|UX(Tit`vefrf1fAxQWQug922Aq~EbI!&Eh) z*$b}vF>ac)k!{) zJM@x*2B0|o@FKFi_Q=9JKmXP<*Rec$Dh>$*l=k<3E6xu3*2e!SjaB|X&2n%==H*Es z_2m8hBx5t=AxGp-p}FT0F=A;*rN9cLCX2!j+`*bPpQymjIN8lm^ATX`fCPDb5QFo4oVqj*Z#$XQ80^+=qmjY@T|QS`1fBg z&C#!WqS>f4RB<#l_QBV7Q5+xAW@*0rVPB&#NF!D`bv>K z?%l77q{Q@;+bYH-s6N#q{7@xt^Kc*Nbl;)h_ZrKLLEt8v*E5gqr=X9yt_ou<>@($C-&Q7vJR70EA4s#`tv}={3SY}x#Z|=JH9>Geq zmTD(vtu+EW1M)I!y&;O*O+jX?>Oua@vMl&&+m3g)kxRf!zP8HYG^ z{-b07rB*d(QXCL(=t=h~c#UoRfnUiIh@<7Q+%ztOg8CHI@!E4ub(AiItP^qqg8S zlrjFi6I`@3s=Ms=0d6ltM{@DPRAZuPm?;s079)5Ct0;yxa4UgeqMB_ZX*~r(mOvyy zCP_ULgW!}aX7z`y-jeJB{%E~lFNUBD-6xUiA$(0GQj2^CsNv;}p;J~rzmYxK!Qy11 z;%;@=0bBljlxBI-d{PpaGMNQ4d1%2g{$vsHl1PrEJkR~+VicZ6fD2sjeQ{&Ybpy6h zv_Ur)sA6ffd8Y>fJzNC^uws0rSyvfBe=)jSh@|K~+w=p)%vzb!v3XDUQ&iv(62~jl zR&S_VZ%mCiQDp^3Fv4>RjBq%%qSeXQ#r^ye^MXxsfm7XrpW@^d^5rnt^B$9qUKfeZ zsHGdI?LKu^Z0jBVcE52}f2$iR>ut_}X??K2b`<&Z3<|gfR;)ZpJcr_W4`=IM-bIeT z3;GH(rN3@{=@&~<|M6Vbq4Wb6WZZFB1j@>-u;0IY{_2UMcjeADqkYry(OK9s!tXLqr~t zKRkO}J`)+fBhijH96X{Jx^+)IL@|kMk+o%a%)YdH6z;9J(E@jWOM4*!Vc&F^P05eH zGMC4tcls^Hp1CE@7$rRBQ)E^Z3@lZ5e?L2V9fq_|U{D#sja>4iG=M(mxMvZz@mr@R zC9s=kKj6xXeT5BvO;gWx-swGnHwg!zsMYuZKqz`t{k$_rRNTiofupNuUJ@(CEK7w(nE1Am2H04gqKZBm9xx~-Nk>{ zaKpnfexF4ZO#*6@=kV`i$S1x~|4I2m-#Q(+;6Okqu>Xqk$^U!G7dN*t{4d1|MeTnU zFSr*{mTO>|)cC={!HTq#kePk_iTvnU$i4Gf=3W^(rS!%dBwG}tYZwV2|#^$#WxMZy$|z8Mn^y*Vr<_B&(ShQ8$*o@rg(d3 zN)tP1@CTDZ6fc?&(;piuE=kTOYQI=QJC^vt^Ge5WCNw%a@W=c1@^4GrbRPCMN}I|J ztRp|PY14kP$bl_1E;etFRe)YWgZQ&_oU}FshXzq&m<2IcqzMxs=WrIxKQ$yBg}w-& zuGqK~xaM8(6k9-Rv zUvtArP+>PpIwMBz)UJCJU;0daa=IhaF?1;2o)BwvW1G)wZro6Y-E*lGWQMZ*U9w zI83)tnRf-BO))T7SI~;!46URt@ocJ4<1+Lz7bI=I2o7P)+(W^7imF#JPw%1!@6^MN z?;te7Oshk|NV^Qfnm1U6G7bdmSqv+js-W>#WI`i8U@;*psoRlvq@a8Ot#2*8C2ccg z*7IKS;NZ}8%|Bi*W2ZVjPFd$Bo6&B-F>`J-;$B?t`?q3((go+82?!97J^25!AyM4M z#@OM%v@=92NZT&(A!SK#LS$Zk}~LE67sC&2srxX)_K&1n1CwbSF<4Soya9MUcH5?JI)K6t6|3=;o4KuQvw z=|JuLK&;4|lRXugF62$po;W~k=xcWonu3f`$=4JI10fJG(%(qUN}4HKs&0O#H_OqEf#kBzg-oim$RxEcP=oVUA>-85!Iu5L7+M9ttXwH@lF zRP@I8J4BiDRy4^-h+ibhFlcFJH3RI9syvW5Ypoy9etNQt(NZs%;>22Iat`@VfY27S zyNv+@0ZD=WD}b>4D}XrKNSQl25&fSRaVKL3eJ5Lo|7c!S1}I>PB5`m13L3gaaj0|G zKzqbP8R^=;Wr84(GvZt&VsVcaf*c_>DNFZ{qiU4t!ON_g9EmJ~wo_I>HZ@i~lD#i0 zY8WFaXKUwX{DmF~q#(ZKUHXyZIKxf-_4>JS1FXd#lRrbJnke$@KemP&X{FG-oY+Dx z#89m1q0x9*6DU4`QeuHwf*X~8?cY&M@=(!}(RG&fR2YSSDa2|`o$GZc#4h;6p963} z6#Rf{^HD;Wl(!3!(6-uOnEWe=n`lU3xmJ?l*kT-oe8&;kpdUd!wc4~{;$&szXIM3g66F){0#AVHfRDMYjKh+IF_r$982VPRah6C?LRUVBX?P&6hFIzJvG z_+cD1JhizlrBem7m9sV-4yVcJ#9ISF?^kEER**M>Y_?h-*qrXl#L+C!kjvag!c1`a zEXY&trBhY$Cal>k4F9P1_n_fHS!scKCd%j}dVpz{V}rbhtMqtz4EOGc(a8#f6=cXH z#Hlfwd_jpf9(;H@!Qi*;Sio3f?S4a&xxdZz-jaP=ayca>tn^%}=3mW%$da=Zx~q$Q z!oebt>l~d~Msi=6R^qwVGFO_s08X3Jnt-T{l-6+}2th&e+9dUW7^+UuZDLbtMU#;A zrjp3XV-J-y*F`Ixu>}#CDcbT9k`)~}@v{!P(uWFTsBs!!EQn1yTGSl9;6CYvH4>FtNhE~TfEUwPc0wOkpF;jPnR!^fHV8{&HN z)}V+9q8Yg-Y=~>_kzK@gDS!as-4IPk*84ME!t1*jx^Mr?w+QtIB>pqj)TOSILWj+C z#6*}!8@@;Hv7J7VQ{w0@OevLK44!Dteg+oNyWpbdz4#;c;+XOfa-s=-yDe^^!bd!| zxK~z~EEoIY8v%v61~BhSp}%TDAY}Y5bch@ChyLUXBBe*?@Bn1|`i2k;c!d`g- zw5W~}+dkPQ1+q&8MKK+(EVvdQwXJYiYkqob;sAD$-3b)5u4v;Z-vg422Y1SOyglx? zJ!<2On3<|<#Pp5+vNE@~ni<*=(-jE;w-AvC(f;XPw{S2iOmMHd-;{eDee12mx36D3 zf}iTII$U5LKM@`=N20{o2fi`yX_rP{jbJ2=+Wl`7`*MOJ*kVIjsDcz*4VE`=(eI^xM z=3|*%17>$B>zB}Tc{az8ise{IM`bY(oH>q&S+X(DWyoNNSo(4y0P744n0Ogc)-yf zc;Bxv3S!HPo(o*xcAjK-vw>rBslbBT-Q$dlIqrGzN558AG(nmJrFfaN*!t%Er z-0yFz!$O`Ist!QNK*M7|rC>CSb{eGj_Z9q6cvVjosNQHnkEpVBpC(=fvi^4#);ZJsG@dqwK_a*C!{jECd@d4JU?s$e5#;<9Js>x> zoJ-M&`L#^z-~h!kC%ws5E2_0xz_t-pjTM!l2xc5VZZsd3Ij&F;xWU-clt%U-Li}Lj z#Na#tFPx`$C4d4E{pe=Egd{DTe$Jn)sl9r9C~>UuCiFcToM?6J5j;eZsVqf7ePP6b zVL3%hK7B#`YHvzJhl=Wwr&_*AdP>P1yUs z!(GLe(cPL{nGq6|_HN%1pU5s-2E(*4d|Zj->e*!82G@}q9`kC(SZ9+h)w3~JR5Ufk zIUj_aB^8Ugn#o*>Xc~jN9u0FfgfOC}(_W2y*rK9Eqbg$ulaXXBTb;ln2z^SFa6A-w zt%k$&c=t#4Aj?b}-IX({UMOJ*$uz+9up+@d^A{=af<@tfi(iV&dA*RoP1gd)03CVBBK*m|G=*|M8V zZfIl**}6NFPuLak4;nM>oAGYkQ>`l7!LFe%&=$1kXQpn{%Yjp3_vaH*eAp`Y(`-%Q z?UA(oUyEJC8z@>`7`SSB^(O;e!`?77HfnbE3!lTD|M(1HIW=rQO|!ps@y#%?N%1@? z|4fzfN4NhS^>v*glA+cW!yjNKUwb{p{-?R=*QVsf=-3`q8+^%8*t1$@y@iL8?BxZ= ztV&aQkF0%6^b16=$1HE4MDGn$(i?5Qx83SAMwHL+y!&TehW9Q0{7Wbdr8-xIMT|o0xEPd>CVQ}n8enrctQl^j4 z6?cjAppbHEs0GGGV;RkK;f-S#PnZIiQ0O1W%Xlt6=afQ@&2SvJvI&qOj(zQRtA(4S zP0P%Y>o@XSH63WAOcX_rX&UR|m8SV>7zjHam=pcl59sR*TXTw7uu|mAg(Mckhd{4= z_+g?0;C;D-=G6`+Y>NJz!DIY+1ys9_myfNNNG*|BqZ19hlLjb&TX$b;ln!?$>oRoh;&-+yl`n&Z&;=Rx154GjIZeytAn<%r@{2DSY`#r}LDN!rie^P7?O}U7MyWxHAFZR@? zsw7Mf?U2?j1pOJaveO9s-jyxdE-?TQ9qPC46AlklAZG}*xLw<_B}+K)xu zF^`m49Q{?c^M!hjk)iH_xAdYN#rkn?}39H!rx+%B+X!1M^-5Zy4; zeveOnqFUVAy4?QMC4Q%JUvXOvJ3Ga1X@3@D{l%0ecL^HOPkvAdXcJwS)}s~+CEJ&P zJ6n}Dd!gd%94$ovRQ5NU@}LxGPwXWfwFy(mQ6_(aR|qs#^@%L^qm&O{DTG=#+>VeG)chx@OKq$iLQSYc@vu3 z5?@(6n(_Jwu1|&dkxuC>r59?qhWM=$d~$pmcW}NsO_YV~seWFReL0fvgfWP0iBANU z#nDv7S=#|zn8f}pkNjhNr-3KEy&86E1+mw>sm&W25Aw4v~_+-^g%XE zSIovo{`=mROGqv>P11FcG!*KTEWm;GI>~UAdX#(|N&=yTh)~5dgHi~^NCe?~r}6V< z4i9s+H};}rF-jVvjbkO)mSDAxIBgWEYO6GH%Cy1-wBG>9Dj;B(TrD6V8@~k!TvD-4 z9NH!2by~je0UT_j!B5Xqx_=-=?>o}$e4?M|?b@qbi z+?n_39qj3y=;!S@wUC_Kv}sE$q5( zZ1at6I~{jy+qP}zjnT1f+qT`YZL6a`bA5aL*tO2CS*P}?8a1lM|Lej1XatBRh0uK# zszYtZ8|%GDdJnKl>cG= zMQt}D=x{=j{azE&4I2LG8N^n0mz~(cJlpk96#ot0i?jTpFZ`s$?Z-sQ6PrF69aM1{ zFte^-f~Mpzw&{R&sZ*ZZ7Wv3^H}3GrS#!lC^s>j4!q2xa-+ilKg&X-Nd^61d&Q{V! zv#=oLSM@Z>Dp1PL;N?F{B+fVkaLceDAa+>)Khnhiy=J&+za)ghpB23X!IcKKubdA7 zH&e2F04rK3S%6kfeJvwGPH1P5(&>CPzJqZ;3VjcPCQXs>1|k5*(#2whMnTKB7ydlG zp222e?h)|){sP{w0zlC5pXc(uOx`@;Z$n_i2axV4OR`kP0@ z9J7tS;@do|b2AXMhwkwgOeK?%Y4r$jm2t@H#RIvWgn`N*Xozm4_u-sc2v>n5zfTJv zjU@HEbx91^?6Qy{hupnjf0JNS`PW^B#@&T?>Q+O5bUhIqZFkxVX!x9Qq6Z@1;Huh*jv)*n|sebaNv-7AVcde5p(q)Po zxOTSSFJE0r`|3%%EG!ST#beZ&fCOS_A&V@HnQ7VZf~G6i)TB?p9|XvEiYr^@^J+7> zi*nC9-)U0JM_6&;j1Rq?Ssks1_eBf+2S3M}b%i-MdhI&(*c^+Af=0X&7@eDc3An%K z(ujkuXP5+FtbL{&IwzO+KpdrkB;AIuz8>)_QO9=CwFDm=K;D2N~Tk-f+(=M||E94=T49(H^7dgk6H{y~2SU}ps?;i}iMcaHd>S}q{wXQlZNVm4Bvo@=HL8wPd{RcIVE9zc) z%}<5}iNgYOTxWFxC@{g`*WK!NG^qTNAi_5GS8HRi_ z2L7QYI7|m=7E_2eoK%KHGd-o@9Za*z&1W&0!7sXtx3#=gAx-0Oqn*4^VdJMA;)oKf z{_vI$T|O0{D&n7GABrjDhZ=H|eO13nG#{C()7(|e-(7>X*P^3JC)&D{DeNxXNJ+2s zHBigl=qOj)8bBMt_~38RFvIjGzQ1Am#+X`xRqss@#98V9fnZqqRD=Oep4T2Sb=M95F__q zW&cRK{g}SR+&01^?MNjxD zdHSu0;0TW*dIGyr-8;{^WVM1- zV2UmOOJ&2api^Dj;f$YKKsa8&DNw-g0R;Sg8MO9XWC0w(uPhX6dVx!7ub&mPkF*~1 zi8vZUpDc_ky=>eTGv=Wdd@|8Ey;0e;h58Z-Hj4U^vOscTs9g=6U94(4-nGq-zraqV z7E!D~ZHNQTn-@@%auTVFMo#tQXKkG4y;g?x!r~g?z zF_$I79-v1hZ7T!fw%f zZv95wL3m+{X(_GCxv=072~c867X$v$#V*diiwQ!_C3e z_AJv+hRDXLMrX3rX?kq39@$Qay3?+N@Ng&fl7CxPPD|~0bx%`Gd}vcyO6FD7VcM3Z zN0?V#u$nQ&7aFYMDm3M*Bh~lSl{Ocq&gH+{b-BU*%%^rQ_?UT1n0{A;4#^Al<~&9b zaI&eAH&B*VMb03>TICq2&Q>2&G2^?I{HHCC|He63cx*&SRp9zyIfoG8lliy_n@sc zveyzzTG^0Tg%m2&+X9e3Cy6$ddPFs^9=$GkFr)SaW@_?}-B~F6NiO+lcIGNd!@+5B zN@mrBnLrc;CZBB(Gn2rA5lyU8cS7ZnY)kA4k$T5DqHFzHNek_!P@=L?LR=OiNZ}cM z=2@|;0RLXt8-r35%50^G&~{n<2_Q|kc0Esn*f%|ZAACa4U{M)SCe;$F?%CxB^9+NP zwpPcoSH%*?ht-lot2#ozbErT@#r%V`n(AvB7YNsVpwZ-WWv9;iY3W~}hHH^~{s-Zy_e8m)&(!uYEHm&AV|A?b z%qzkY6Awq(&p1}oA&0b=ShA3U+wuC#Y@=#deQ{?Ez(G~2$sZd8B@P4gh>8*4AF<>Z zkStH=d$(E~zzig`8h%lvuJ~!Vn>*(^$kJg_B=7ZQ;7P`lgbLoy%*u8jk@^ zIhTzoLh|VfG=LERg+)50;1iBH?~l!+@7yk(R_oMT@5OzJ80w=txFJ(yZ?GhU$zPh| zAxr6_fx$G>%%(`083;@KbEQ$~t(S@fgq0KvygfHMt-V(GwXT)IW2nTxJnV?-UhK6U zMmGet#+w0>#0oVfrL5DCJGMg=7fi)iuz4Yi)P@B_xL7!gjg(!BaIt$;e-rVCgmgwK z>Imz?LI6Ipq63Ziy=sExjt*nxYoQ2fx=Zy_K#NE>khtbx_RRrt>&;3YpusD(dh2$l z#e+rajFl~^>jH?lLnkrD!U#5*?3_+>ryO^Yhi-H>=~qx8j8R~^ijWx5L%WDl=YTMj zRz>QtQ2t$ZdhjL_X7eBwLe7LKYRM_s1byOBVluAPJU9(1qKpaxwj&FQUa}1_Pr5Hr z)CfiM))5u%dqEYZuQs{r)p2t?;614N1+%D5!70NR_NaYklQ!oo0>7GT;zW4eNfx3J z?`LL3uzPS7Rr`v56_sjInH+5CAcRW#h+D=y{)v^sc+aOwOYjSh463ihl_sTYs%3*r zi|hLk+ZN(B+!2)6YcWYm;W5&+8H1md=1y_1t()zw-^A4i;(vkPxBo0$@(=jR{|f%U zNC;We|JVG>Kj%q&qi;%8D>4G|J8ChxXuQ%4}qazmBh*PM*ehXAE3?IpZu})Qtbo8%VH& z?f6?DqTQ)mfewDtSt#-;easf<9?A!kg~;?`3a}DxA`5OzH)8gll!O7a19b^CoN|ob zi#NPcLK9_}5RAdOl%`cA+5nWBA(99#WH?F&wM0M6OA9(+$hug!T$_^p;C6262zTZ9{ z2BQPa9L;2%!+1w3NK7bQ+pT1Uem8Hz{Au&Y{vi^%jY&fpeaPpyRB>Cmt8ndC;>D-0 zgZ4*Ja&y?}MK7#RJ2-w%&XQ9J;!Tu)5~>*<%FY~AYX(^+Hn`Fa#^sHE6-Tm9P$mDt zK=%YvG@8pFut37kZ?Nyveu6Pll}B$tEvnzB5ZUSy^~@8GM)HChwFes~zVncnu2dQr z5$=n(xQA{ICF?80G)<2C0d%_vtu0&=D!OXeH+r_w0f z(ne}&@x$KKbcV~H+1Kmrb%VaXzYup2mk80h1{^&PENvsu`kXc9GZ+G=<_i^%J?b^n zU;K50MTQ9Hbjk*4S%Yo1O66Ra*GI%dM`nS)O)iIxVLFb?3?YGzDw@59=}h_~{j@F` zD{4qf32E#o z@s=skA@PWbDD14{q{i!<4hd0^CNQIn*b@3BN9+Xqf1-Y0QQs(SnE;9yCx@8LOo1R( zhWcX|gqH2`R=^C@?V@y}h`ezT1&pDm`MGsqTAl;aBd+;Q)QxAEQiQ99obsF7*z+-`^s()>^TB$F*ammrseX9ykIkQx4qWZ2Tp> zMQx+`-&|GY?6@;Se@S!2jRKi^!tA0Zggp3=svBi;MHiYF+D!k%@p)udOv!tr9b8r-a43Xa{Lk;6k~ z^9LcauJQ$^1~#k-$h=Z{7`8tl`d|bsGHnFt^7RAhc zD_wMOS>a_ChfxtZyUcF8^gZU>X8Zj8K7Ew{kuREN)_0qXwBDy=>TL5QF$=FSRcm4UjixaGDCD1I*s664^}!MM7~pkV(q+NH@6sQop{8#*Ec-&7!LIjq# zEs83}jk~8I;owTf`?L zZ{7(dJk;%16c&0hS#14U`?RAe@|WuL0z@55+1MMK@9PtjWK*MFuWLtj^fvj%NyEA$ z!L70W5T+?kjQ!(+m*`dyL~WEsATVqpl!eyDutN_9SmZN{%MT_@lNgSIb>2AMrmrq~ zJGsAe%nEUax{IKCPSX~ZT`jaiyIH#F2s7`IdIB_Jwp-_0(qweB7)Z_l64}Vws7$Hj z$^Xi0qs?Ya47canD``u;A*z9V;5#Fus~=k3`1JP@9s{Wn+t4b3@vdjVz}vwVz(M{a z{GH%T^bwGJJrPBQNgR+o3*^R!`64^|5>OeF^s&ciN_qu0xDB^W5F`h>NgtpU zKn>LHaobB3KFako6ryHZqsB^35?aAd>RZwL^DVW`#PCz=S86`Jsq9_}r5Wc>C($x!Vn-JQ?UQ$rx^b~`l@*CNnsS6jMEpO&nVPLp$HjYH{qPWq(=xN?Rq4#83%K!88g`l6G&uE^n&%tw>%s!0BbCo$H2=zT1 zk715AC+>XHe0AThZmh`=!A0UE(u%uulk3G)7E20v0C~!=KbGJ;18PCtJUjRzvg`gK z$8&k#Z#)j^zRNLGpx}Q_%>I>7u#pL2=H5Z}pyHd@#Sho8P55`bzZHj7!EO~PkHvS9 z5$m%or^2kv$99!>xb!zK?L)dog78lv5h7z;YP0!aLM8^+pLf~Bps)@es;$NL4Vmuo?CvvQz)eAjVW z)hyFbUeq3dN$s!1;fMN)10;6H%W^i=Qq{S_pH0rBSf`95d!K#Ku?>`jw$b66!>PfD z0EDIxK~Nm^QuhSHs~^u1B)g4zJA_D`TLE~xg^oAshjM1Sc<)#NM!fiRPX{#bM&|v) z;|a#bhQ%|n42aRq7i)|8333UOCUL#Umd_b0gFbqHqab@wjg)pH9QAf%6c=SGV9a`w zQtBVZfUM#z}mqqsYJs9xPMO(q!;Bq{l3%4$*+ z#c0D)6%a1cv_ErFr5&SvR_b|5TStGj00l@7P(kA(JVGC6Tv=rrDzXo5aGZZq3iR{s zpdNsx1TdjSJ5}yRaJ;v5DTDzCr`DdB{tH)3U~}t#C5aIKnk2IP8?FvEZjLG*R<0(N z|3lWOPbuSRpzv#;!3?iK#niJH-bG6a3p-t*KpXvl8*Q+?hp7W3uS()Z2N?kq#oY&J z;T<0Ec1(ZKCsqbJ5vs2IiB>)4P0Z$17Q#|4pl>uh+id%AeLe4ey?E{O=YeoU2lkh% zjZ4%_15;b_Y%U6yYoih;Pc-l~K=UB-Sn@*5EB9>o<3}6p3?a>El&q z#8b-YOxc8KX1mt_L&-+3(UHnVJwB=2RN4-~;=#&zi<8cmVuR_b=@JlXskuX31U%;} z@-cVZ+vgePcR>s@B7GLf<_oaU!-6}>qEXIZi<1Vly)m8+f|31oRp){3k-m}1kc(>M zGe1a--Ta`7Z}WrW8gEh1TUXr@0alv&IsQrf8ay1cgMIh+C9AO&7$PWH zy|xOiMjAaF5Cm~sRE$f`?6v{Knq|5c zNjRg-=R*)Ta0oerk=U=AXKWqi*RlOM-pQ#Z%S8JRx%lf40#EzkvcHO)L(odC)+=jt z%#{Y%!32<|!9fci1yd&XXgvatklcS&Wq+- z*IF!JYx1jK?ifJ)sT}QytSZcyI_fK~-x{`CvJD#}gga@V#%m^b6jbziu({9VZbfqx z1M??kYHCu}PLbeBlL6qL^j-i|QjJg|L(t%BX(SI-yw(H`uWz2Jt>Yk4aZ>|=t4_4k z?jU=UX&z9Ov(r0NkDWBX4R9G9Yk?}W&xN^MYF(;(3EriaRI#oqSqye&&4Y=VEam6& zT3Wv~mEb#dHIjQAXuZ_bIenwmD-wPao&kY$^m9X8JBTE^7}CPMAfEyySHbbh62G+z zS`umwxfJ9o4g>)ggKc&5n7G?Re-z(9{Cv)k#gH7mkcR}hYcD@v3d4505HP5g`Je8r z0eb`mDtNTuEG1QeWD}vGWG(x+sKv`cm-3BV%*s+EgMSw%w<~--Fv@=`$?^6}dv$?L z8&JX|>xprRv?{mZy`z5Bp`yNxa2>{lBaiWi_|M58ghC(B?%8a)?sIKzjJ<8ax4oU* z3RV!hQg&Qw{-{8HL7}1KP?}n65JA!wzJcO1d!q%8q zdf9APY*Fh(&(^{lgW`s3acaPS!m=zUP2IdwSbhz*a?4a7MQM#b44okQxy2}=r|h1!UGa|?4ip^ZGVq-rHeMbk+KXr1 z$=IgHB*~9N?s73d+3}g}aj|$Eo2%Oe%^sBiGeFG01DfRSmU}*9+DE5#>o)}dj=K9% zw{?8b(WdGG?y;hCV8skX*GZRO1>e_c)#Q?Az53wqpTN8f8%cQ3@7%ID%lq{xz2uCS z@?)RZ-X!s-OO2ZR5E`;Au$U|rXN@#O;J7`moK=_R_xIE0gtS+X(Stz__7pr6!jtJQ zA*KoRxtzc#mZqXXCGF1DMpz|A4q&zHtcUhs5v^s3Ho%7&9uXShOwuC4Y`nh~N>rwX$USw7)lxn*xX?3OZ^H?xmqTf7k~X*b6oMwV5*t$a(hgbzdz?^? z(5o;x@ICY4{z4?3&KN4B6M2e~w34*!LXs-e{HTw3jSE)OSsioSmT?};w(_o)JjHH` z%+SgN7^{E6Q?5mf{Y141i?9`_PgIWXWX~&r4aWR`G!+C^lH#9_$oSD>71fkYy+T!i zla;0n*KTa9181YS+@?K++|VLb=_>Ku=MKS#O4UvrWiVYkV@|Fu2bBQvcD#}NTN>$K zr=;r>Z(%Ht=;G%%rBn(ELAeLGl+z6y+1eVbMXd0SC}839H_Dn3m{?~P9^y1SGF0h` zd{atpQM(DA^*ET%K^?#;qH)UglD}%$A}r?A2XLj9f4#G_bTi1=Ym#%RTty0&|C54f zR7>IsgM)wsK>uI%+5VSl{+C&jv;6l;R-&HGe}*!WiX){~eVaM`#pLX)lQ48$a+Ilm z3lpzIOEY?NNsYQrx>L(w!ySBY`QtKurS3YSZkAhib055IYcEXzUJ#8Vxph>H@ul!^0*3Nf~beDUO4pEO~ z$gUO^=>MWfXSoIRPJJwCa2la5=gt$YlzrMpdi|0xIAD^?=yfP#P+{_CO^)&Cc0 z{##$+dHF8={`&Bf%H%|pkkO&G>xgH!Vi1F{g^$2jwx=(I#1zgH&X+UJB=xkTrB!L0 z;QVExGHjG{Fi?!Bd3*CI8;Q!-YhQ|Lh zh-eKSI+{nq}8)O602+ziqFZ)0Po z%NQQQ2bh2GZAD<_P%p4Cqqmef2tg{d@*={6 z&#nn_o1_r$1qd^K>c;xT!B1XId1ic}Kx$~17Sy?@g5LBHSKk)5I4L~Iq9)MC#78)A zM6GWKMh8-YF5z|$#v2ouWJhd>aU5@&);wsbyvyp~sT`7puF53gl)X7QA6DQQzpF#>Xu_fVgcK zB*&8l-n#=3L|7#&E65vo&qUIX%Wu-(J6W5`4^{8MLf7Li^3X}D%y_blpeRh_A4v=n zq~R;J47&&xNkU4pOrY^hA1reTuB=r}!tJeHv5uoL-g%;R=iw~KOgpnEfxzH`$C0)Rpp$sw0kPMMEZRm5h^hV5yckt}GJjI4*;IR!K`$A% zRp$tjHwXwIQ~fhb498PNI+f$A>l0Htx1bQ`C0)_N>V*UOi{b{!R*i@sg?V*B3S?n~ zn>RwK++`1jXb2G!`+AjZP^gW=-D%FX_l3nb!aV0mJz`^p|Bxb0o03&L%h=ZAGM3zs zOMz9wLs{#usaZKA24W6OW$aT`GuXG+6WB32QGG1LH(I|LNNnNC#Q;KLm z6vf#Yi4wO?w3e*Cg0WDsK8pzQm~jhF!xME(dsf{#J{f}JB97rx5>K8Dun2j+^WipV5|Om0=DK+19{VeTPjOq54hi*d{yF4O+@RBF{FR4&@F$BTMob}3D$I&u%6Z^wI`Fk z0T?QkIW}u3XNp14&c&|dfbTN_Z^8_tJdhTEz(^>b@ll`MKB?xl708p+y;b-D*b*x5F5fVriI|~pH}^!|B3$v?=jR4W zr1NPDqA-UL7E^FC`bsORpNY^l1HJ14hN!lR{Bt{($0X?(`%D>O&!~OHwrG$u&g!aQ z*~9scXft{#S^KzDZ%^9zW}MU^)xD|e?z`~(2q}hrLwrq{mwuI59q}XXOkxD-B`%UB zaVs|Z&Ncd4gb?ioEECWpLJZFJFG&48JyPywIwk1s$fzqly8;dvDrH6D3pV4}VG;vv{SmzlOB}iW` z5WuPRB7QiZOysX8q#?#66g?nw!E<20pS{(h#2urn)1t=G+RJ)cREnlZiZm<}83h-i z`-;mrXPED54wdAka)x-dporPvjt4>^Dr0 zns=E=hQopFZP)`$B?N<9N<8nC+GkwrK0tuQz34)4U4n6=m{HaHyhN}Uv1Jl}Fx(<- z-)O#_slkLoT}dPO@a%0-jF?V7zn#R{rP+M&h`P4*P?T>QShi1I325fzI?_9aXXjdX zf*$32 zd1bL~TMsM-b~O=I@z5NTpe6N}@maV!MO(d!$J@%D73I0>QTc1MW7(#m`E zb!+%Z6btX9L(6<(dFU@H ze?2=fYc)n5ryug&y0}uMvL|guWKdUoi4tbr{YWL&qEbB`BDBZx?=)$T3nrtodAqz_ z9ZodRewB|xw(-i#)_Ar1^7C6l@C!*N^sg6`ya;I)h~3LbNF48w1oFr481l?LDw>;7 z8Ynz@eFBDKEsE(u8V70>b61_THJ-7EtM`!n?`K5kgDWT=-b7oFCMcSL$Q91W0lbg& zuiomJXMv&_lNUCbivWbWUzoN91OefFr&~|Zp)3`(RbO&51BS#HFDn!ANbubxjFSzw zaP0#?t_cM=x!&k*_pQ(}w#gY?W3DNkSoDe+_CclmHDeWU za{GDmTMpRVyYVOO-slNu1Al*ZoMGR|m|sKoreC7jL|w_qzL@da9wavt8Z-=L zml&98-;?oP1*?m{G!PIP60`XWk7tIs-PE^b$}|%Qh9z|{eZ&O3ux$A04!z!=)*fy+ z6(l~>F_IW;-lO#fvK_X4x^V9g_1@r#bZN>uL4FmPh|AC4b$N-)UB&ZFUlg`VR2xrg z%!s=SGb~a*qES^$TEo9s_wC{(&dz#mmD*>PzT@ z)XV?H$vi%$pJtY`d>U($q={hkx47Y8Zw>+jK&o~5aV7z(4P~c>{GQVYtoLW~og=Pn zvTkf(6313kKLkR?F?~e;`K)`oEOlA)AP!6Mc97B*B-Ywb$_^dUX%!({VL~~wAf>|E z6-5JZbz_Kp!3I4=g=d~#0xf`!QEH2qNLVSn-?ORQK~ z=iy${)S{bW-e^+q=p3&J)cFxTKiBsJo;sRhM z`~2w;x9BcJlAvWYsji@>mK#T?(X>b7rLf!*QP!zsSiean=&f-F=tGV(%dzL-Zi#NT zAnA}OBpZUuHVO(>)BvD=31Q&5dDoHh&{Xo2y8V9of0dZJ1PYfm{m^X;buL~Ltrf-n zBRzrCA@VF z+pk>QuCK+cwIm1>@vn@?VeWpL)`A%X2*O_5ilRwSt#q+=(-H5uI4ud8 zT6#3+0JU;IK{#<9;iJb8zmJ4Yc1LPBaXZnCT^O%tgV1-(G*?ohwRV2CzDFt8+C((u zg9P|E*O)6$W-zEDV9k-+RRa3RnB_@=s=Xd!)R;!LpEW&-%#bm4sKb->%@KqBz>**2 zBMIrRG@7z@$EKFR<^YlCrAn?X@0MJ9p~*~?{C%Z<5J}nl z&vVrJ#AJQUG?lJ%h^_(qH8PhAaBmvNiM9Ly7IxMMM`K--0z%R2yL_njFi(q*WK=qf z=AoU9;K4-^L$Md=hrMtY8s4f`1gg{XaI!l~Jp8C%N{qY&&W@Z2BOS#n=SPi?2fD4Y zr`-sItEy^H0ZR`WIDDWA(rL6IWtOw_5(-QBT~}=0KQqPYy8FtEpBx#E&ri{~mArZ3 zz4wB-d!CmoUXMbOmp9?>3V|)+Q(;@vR~jtHcE*_X|?z@t|S2q%V(|1*Lx2dQm%qC z-?n&9jN`&;?iA1+N3jN1^yBaMYi31D7!DFPE=z?n22}N{Y5b@|$;0YOwS6d2j)({Y z(FI^kkyXtpJY6Ywv7`1Sv;TSYt?2f39@OF0?sk;>+1v5I;dgPWZo;+Gl(*xdDs^Nk zyC}mh3Xj<(6qtWF#!wbtpwOsGx`{3pFYsr*?X~DiVKcNq&L(nBN91n@-noSyymR-B z|Asxi;G7z>1ZiNZA;P+l6opj5ED4hq3_K#R zDf#JVj|l{{gkw5rO^sq;+h|6YVwqaR5Qlcy*?giY309GH8~j2fqty*srCyXje+Le) ziGf&OwYqju=mVDuk(H1i@6qA4(~P`9j<(bG7rPYW+eI$N7*qP^0<&$hplKOv(XH08T2P8?^D)u z%dsDm&a9dx&<)@@T1NW<1^SoRmr1X7w zGe!8Ur&$xworhq2SyPO;MOph+GYD+yVZr*EK3a26(Jy;mU@wQ+a58Y<*>x2kzS|(t zEg2*CPyo>&@HdR3{Uv6YJQC2{FjP-~{K)CPt8Wm-t8(^%#TnSlJf@(C#wUW_JsbPT z=>ScpP{;vBAH;U1WdP;Hx%55Z>@-F#tK~!&tMd}6rh$>!h|fcf@_nx9NS_?TF>mo3 z+3;&u;v5@hZ~DRW^z}Sjn@H}AY^TpbXRcMc|_)*VJE2p$^>pknl%VrAZ?#q0-R(L8E1ToD1Ow8YTT04 zFuY1iEr<7c=QwfOr)0@xy3={#1|fqUhxU1=XdTRB^C2>k-yJK@)ge<-EeA}6So#1Z zz0(<++fxS0%tFTAyz=VtnB8dDOg;|RqrshJV2idCgY)%Z74lx^%>4H6xJzrYw<`7S zOsdzz&W}Xy84AWnVvVOrx(G3zGBNE8%+E1W+>ELTQrjew+8*TnPOZ}W+Mc2rv5phF z`Ksdab6Q*}xH>JH!BmsHWbzz2F_Pj)(s8V>{Dj<)VUQkRNfNZ4OegcNK(A%Ba&`ZR z*$EUau>{ox`{bkAHMsz~-H@?J_-K5y8me|I9Fx;`1> zSO+7kE|t8a86a$nM8O&U(}4y59UhuH41Ln_21BWkYGP*|IzXf(EraQI2GdFgQ4yOU zRDg^g7Mnrb>QV%vzsQrVF)rxC?fsoWGV!Xzh@Ee=Gm7`43Gz+^25H8%Cwhj{p+GOJ zq6{8FDraR$ER@VM^gf^e)d0h18}}hxTR*)&Qw-(ddP%YeY%bBn9ebN51B1N2HeFl` zzKxpLOFY~yl0Tj8kPhIO9g=GX#H>ZT@}}MuB+ed8nMx6I~^qZ~Wxi3cxyqmL(4vxrc$n4YN zN#c+%-{9$vf+5c2y4{l19rxoTNEe#U(LM|%l#xQL`cz1 zdl8x-MA0sL5=zll#;ES-r^dP5$T3gW{I-9S-D;`<*sDOO+iWDBL8fRV6!wB&W zKjs~Yb)~Rxaml0wbumtEgKdtdgMxFC?f~T01{VqQ4oh>!FWw}Fz_VSjN;u>hM*i_t z47+y0^J1#n;-_XTc{GgyNhgjNtW}|sH#OZMoe~LmOv3)~$Rgo`PgJiGfy9gVqGja_ z@-!dJG%FieCNr!hILs}^C2q_ejMzi?=+=o7pD?VLc_F*4ftFz9CFrBTc%o3>OiyEV z*oRGf-_9dqO{u`sx7r8dn3VR~sQrVi^{T>l^(N%bhncQBuL}iz+o3+0;kt(rbBvb_ z(KQ=HLpiDXFs-xGw$hFX+GBqk?tl&nkhHQz41dg}oLseQDY#;XPgMFC zTR3(vXs6p~QwWS27(_2Hr4k(A9hP8r%#|TL#9?n@P`Cap!DPn39Qb7Qm!i#v`<`My z=SgP&WfuHp$@y0U@0<5RkX3FQAukdBO9C^P7FL%C+?_%dY|)yjV4HR8zwe~B{AiZ5?N7^SZW|HJ2u>>Fg{i5?w~rj|=d*8wDr+uCWE`WA$0y4xfv`yXWS_b-eCFJ+~80 zee<(vEr`=e&^%I)Te8`>V-MW|VlMyGH8YED-DPt|%sDX$H75dKkp1Bq? z{y^@4Ws4EXm@anH>MPI_j0judz~#WoSBx*@HqzoA-3ic#GPkdB0QrkiAH(Z_f+c2@ z;sA$5xtmrVvo3OJ81De#zLZ;{o5J@AA%e^})>A{5b_d&L1o}YyzMx&tmzCfvM_zcHS=eybnt!&K#&CLoyTw@F;P~y* zz4!iI$WWZks9p}9`k_)Gz(vV3_DrrFDNASi`=^n*T{oo`(qE^auA=Pp4^JTvqSIpW zS>~Xr|Lcb?klP=rT(}@0#=`&Bi(>efUX+8Wo2{9cr=x?j>;HaNa#vwV5#xK7r+gD` zT8@%7>daJ+O7-)eG&CXtRw@IT&1b)YZid?Za?y=i;O+KJw$X}x;m!Pj$c*pRzT8%3 zui(}H^~rblYqsNb^|o#owE6(H043Cot(>4!{%NHq!@bK$yx=mkYrucZqiwe(n3)#( zqZW>|82OrP^Xb}r+n0id7GvSD%vG=r-K*Ei>pFt%QKaBmsRNOu{f48*GNisZp7IJ|3acu#3|kW4Cxe*IyP?tu z?wtIfz1j%P8jQu1LswWhUkg#H7@|`;LIRplL!+5-?RE*Og|yayXy^jHQhzf#SP@qm z8BfhAiP^6e9}|A-ntBaE z;$Xfm8ISb_cJwS-AKQq$phz^``jbgUldtV(sOU%)*L6AigV1PO`$#CxluhTce9;N8 z?mCl+io?;i&pP~{KoZ>1G;p|b9&bu<78>`l_^4;E5OWHmC}fNQ_yvIIcm&*8v&Q{A zjEC=MzeVPLFzW9z_7xwceg;gxCa|UYr1SpruIfHv9=K$i-7=jyo!$|~V!BHx63>gb zX-ASIzoCi@4M`Ur3<;KqX&LAvCo|i#2AXg&&sE*I_P4>N>a%uUl})*KrhmYu2$0g{ zKOnaC5)%y})B6p6p;wUjnipe7r?~$*?FnprO27PnN;~VQxR$NK6WoFm+}%Rs?(Xg` zjk|k*0Kwfg!2$$#cXxLQ4j~ZS0%VfhZ{`8>Zsy%Lhs9dmf7Gu|pR;S%uBt5u_ZJ05 zSTtTZ>lItFcfU_c4T9G?g+M_-;=%vhlhWVP`7gQ5Q!}^2ok4zBJ+ctfe3=+y?PoPq z$x!QR>I^O)Kubs53q>i;i%MVB%f=TL9z=aU;Ax(%jB;zUy zI0kDHj8TRtid7~ngv}MCzDd~Ni#}rOJCY>Ps(oYjQE%+v=$rGK72ER@wzk`szBI?F z?M1~bvzd4crlfFJ*d&J@yeqop@HyUuT%4JVwI*UOAq3yMU$T1kMW?XAX-Emkzb`eO zu9eFghY-Z#v4)<$I;-T#Gwof3f=f)nIzWa#7|4wogA3+;0qiV4HDM)@kDDDC?u{R{ zEpqfYwecP%wEu8u*sPzw_TI4(3mpem@CCF)H9c$Yh9YekVZgR>(d4Q%W1~NMfYUG~J5UELBH<=(5FeeU zg4*9iThy4Q<*QU)I0Fxh`Z!#CPi_nik_4LeYozc+g*nl3$OGko57UIc$)Oh0{w*pIzganKE+U9q=x8MlBxih)&ws;VMn=nqV z6TiMvp?+PP_pM0XSW4}ax}ns83iCXQ3TMHtv>ya2cQ>S@qiKj7oSm;`n%o+yeBOYI zCwHN|brep>QXZ34IjuJzfpEtTbgi8wZ2|WOe;mxNy}j^gm!HQK+ zhZ@^O_D|i;Wn5Lol@A3RX>%}o-F*XCO;W;~?@fB=IWzGYiqkG);_qyR$S!^fcnliN`DB(5sGMAMy1&T`i!+UR3&+d*T$ z49>zyG}c?`$W3NfUKqwE4wu_fYUv3ofRAzC^2nx=Ae1_5-cM37o5!i^5V%3rOsO+* z2kt>>Hbcv3=?aIrTP7Xxvy9Rfgr?f@~B5@?Z2-xptI>4I#;dI$-!HD zk=reXp}5fnmD}BiC@HTN9mF33=^?tO^R70Wp#~5@9yE?g46kXV5~Ir!6t*{woj}}J z-TQ^Ov8Feb7$qD)iEJ_1_dro(GUv(dGH0*nEG@>`a87?U*_Ek=Uzg)ztPSt?O5H;cul)6ir3q6aNYQJw5ut# z=Bi?!`AfCA6;^3x=Y@!sp~@X8BftRg+?F&c+Qsy?4oF1zZh;svuHDpT_#;)wol>}gu_ z894lm?WBxZ-nr7214NPYfXpZr*p+@OJEzZv#l4N?)Bvf&^s-U=ZHw`7wdwdFw$m7= zkm~|g<^?=TseqwLKtL0p407ZGWF9syA@&P`iC}IKk`v~beOI&2zH{3n3bT%HAti6x zT|5xhqnvKMLUDOX480}0zR9^kzUfW7hVzjaL%pWqZC~3PSv!&Qc!2uR{^Pg-dIPQ^ zIaBI{z5-S+6yZ+GDt|ZPx~%xSJ@%9}GlvhQ*bk4;dy-heX}nUKtdCqf`?@Di1RjlY z+$(CCN0w?7s!f~|JBK8)4O?XU9H|~oKf>@+z33%RTdAv*FW&r`)&o%}Dg2>8Ksr(W zTZh)))4H4;(B`qpIrcjeIiIeW4sKeNhC0;j!%|84MpPIQv7xkhqC>%c;FOCyRoIeE z)r#Tq=eEt1+!?d%lV3Jx`jw{gBDvkX`&wGY%}$#8*}>J^H;~!m@N%>OXPN>$7&Z?I zi^M4#bp^_R8XR0FuZlU85eGX$W+Wo-EnKd|Z_Ed?nUdp36vks>rGte2vkuF3qn-l& z^FJo374}#T4gw_loJE>s-s44DQ$n`peXq#qKcic+6vIs+aIg-aw3h6AS+*~d763G2 z);CSARVU+a6aGAJ$jD)X_K7n;MwZC$6DRoxGGrmuK6FyD#!`|%st_9_l!*_0E9hUP zMsBUROM;u&dbiHIKK(X02P6=OrtRJV99ywo!f_u^&ch1r2IymczdU|hiqUuNM7!~T zcC`M2@A6^q=du9#)$8cO5B%czLt$UzBEQX=Q${D?z&MURW9R^oP% zkLoO_+uO~Gm<+J=lu>NNXC2`e z;ZnTqW472W=jCl1`GPjwtFt_`fsly#3W1`mQ!voLJV*u(5%X)z1G}vXS29{!c3GJl zIwF&L2T9P)QClT^LJEd3{U;($9=(@j=rS#lPKRDP=hv&{OB<){wt+h`qJ9Ld-c;vT z9O{km_*Xx0jj<=>A@3rfIjEWU(C!6b0QaDjh8_>qv+$`56dLo@Wyq&~m!ya8S4^-r zhwvh4s5V zz}U=ZATK321UwsdG8IphXvy2wXEA6gj=`?@Iye|et;vi~^pKtV)G2$q;oyME>#?+5 zs9g=GzY~$P%g{BU{ggM^|37(C1ll{BJ3N**#U+5|>gW0EfU$u6(KcIv3Q5^O12){$ z1V+1xbIIAq*vpO&IE(&v>yL$NfZsR&xhQQvbCUpTu}h*>^0{@tSHkZ=r0-QwlQ5ScmV=}@JDZ8 zra#Dlvz3#%fGp5ez`?=5{dv-+|B3Q=WifO*p&14WODWleu6hT8PFjdW0&@;QI&uB0 zxNh{JO%+$*@ebmt04gtvjMrJuukth{@*i^7JCZWEt};E38eB~-=f2tUfeah?8T29+ zCF-qiM=tLIhY&0i;o0k0u}jf7At6FT$&Gw z-8Ig-gS4V2!)=ss`?MoTSYe;2WjJjrhxe}8RSc4X=9CF6pBzB{RV2VT5|8L!#gtE# zVT4(Z77lQXOV!+tdMeG#WRHi%uf+!4Rh=LrIEUXmqMC=LsY1&+{o^LSQLRbWoHGZ@ zW`be4Q=Fzocr`eiF8N*V5z@XrO8n|5Nd;{4sR%4|SyF4SrSMb9$WKBSK+75=;QeJ3%;SlXRlMUJp408JD|ZLqGv( z15~oz$kfVEijgEkwN1+I1)`5H0hwx@y^sKX&4uShQBjs2eijI=VJZOiKod!A3}{vg zWI-Ac)vu24_%NPL6fyn%q)_@T^vdprB)Bv3j6c{rITDHfH(;;T;fc`gL)Hef;z@E; zl(k5z!vOIk4t@fG{8&BPi}VYa-S-cO10(3*~- zv8B`fFipBbehp8rcNbVmF{0t6sUQP1Y%wuSz6Oi$1tIbT#vxJ>2<@E$f(-=oPtawO zZuKX?3Bi3SNam4LM&>l)U26-q?Z~m-Qeo zXw^{TH_k^ox^%k7dY7L2TA4o}tmI8_ZQ0`5`CG5nXc#sPpQ9UBgi4f3w@!?Q@wSVo z25B1y7EANN(xq67Qp}s~NY&BWhu!gr@``X@)x!{ImAbFLyqn;Ds}OEa)0(iET*9H8xjBL|zSUjJ{xS zM2Ed$w(+61li&QZ>ZzfvMa#jW4n+V(LH0G0Zh_CiHe6HIL*EPa>Q&lpqN@Wy8 zZt?o{I$AT&`8gK4!E7p-BqG{lCfE3Cg82c417oGbMLeXSd8%Sfi zmDSPZn5$Qc7qOI>J?iizm}*Gxh!tf3jBomkUrdd-F;n7xa%z3K8n9QpzMJ(GOMB5c z6mGdNX{Z+;EK`!TzSZQKYgBdOTzbtu&w{6nDVTV`jpxi#$S-uCT``#~ofluiFxW8i zj1jbx@Y5S@LI|v8Dt~QK=~_e79<2aWTp5pg!2xA3Scf>Uwum8y;Z3|wVYp0%W+_ZJ zQlDumnfl3ku(zZUut;BnMYY-u;nhy?Kk$uvE!5)aUCH!aqBTdQV5)35 zq#z$qMR(A}gUQXi@Gc-`#}^t8I8@Q_vXnfEc{{bWZd|-ONGag0my&Bk1k}oC+z)yr z*ofO&Ueh#qL|e-Gkfm%w>~)XIhlSZb%gyZ&-vi>$cnl66fzDB;xwM@zqJ&#%ryQcK z-YIBX(HIIn0TuUZ=AgFZSL(*bLZhE0Ls6 zr&rpbJI|rDeH!gPkj2-ViBKP1a2@I`GRI;cIw!sx!+H^-F#0M);I$h5ro%SY&>D`_ zyYUf=pVQ$47%I19c^-YyB}a}cmE^Dj`oEWZevUhj8I=V;%eKfR>55 zclS|qQFgzPPKmO^g#}{$<*;&@Dtm)?Sz(*uDa2&Jn!N#CbG(l;2O}@$%?qP_`wEr^ z&?LcYurqY+0dI_af*&`QiT7Z4633>&EBmmC(+jYPh=o}(ovT3PZw8oGF*_G1r_!zM z{h=(kEz%OuXuVX4z(6_$;u7+Z-pUpI?iFqjHH})NCAOGt6+636#4`6m<8O1`n`9Zu z`0idDF)gdnP(IK)Bh7o{rsL4=k(;hXtD^{{Ypzmx1wf@ARHX)Wp4pdh*i>9JRC7K+ z{8A_c6COfmKh2|<|28j$vV*&j!B1@>Xk}|;`6zBm#aanR1}CR+C*_rn}-zX&PJgrD`xUL8KV=vkX=0bg@`} z68H6S8eoAt$<=Ve;;J2U+6Dk-2Dp_swb#rw-yP1{^;Ao{YJU#vpFn@0M~12o&L7Yw zO`V+xtwmx^pPum5NBuHhnlmvcyKi<{^BRwcnWjXGA)QS_zOFd_9T&*n`3qk(Z*Z zG-@zkwM$xMrPyqDC(~i(T$&-%d5Y`>w2{q>OEN{?bD@;P%sWc574_L8(`RYiR3uV>gY-g=f2&fkbYfkwDdGi)Eurs7)ea(Kj?{L-;|Hp~tU( zz))38QN{>&3iM;q;Jjr~`=R)Ty9Q#mRzc5^Ztu$MesT|;N~dQtl;w%DT}aV)#vP%6 z^o{pTDw`DJKj@EM`5DoST`Io6>Nh|&OJ&2L>!Rb}mR=-1gRUl(ks6IxWt^12YR;d| zFgJ}5T(pzEXf=W#St6sLDNBP0(ZvUi_*nZwaOKr>KqC$JLN$rIoLlT}>KMcqXLi|= zye~ZbE$9^2chQO8t7nulBfX(qkfgs2<0}<)~v((I2Uboip>tA-QGNXZHBsR&kjdbpv0kp{ zd$rUl#b-naD2@LZ3>$97bGT^apYvgt3i8V_>c%=_5I%T!T$|5FcQ@CL;|GW?mRDaB zFh~eLg9)|>4Za!|6BmlN-9Xju(HY*vcZiqj_t;+5C!#>=fNmRl5ukBNW`RvkUArF| zL8fA+(Th`EcnM7LjBbLH-7Apu14yK1zh9PX$6Klwo!AeXY9pk~B+OM_Ls{)@;uS&l ztAG{RT+i&oR4!Xz@u0WV(R`Cs8%T6fUx?4N$huQ|^6-{(@;GfQEelxBIdtq;1)e!l zDYk>Q*sOS<@cBwVCBFC=T$6Ni2TNB3GD|>l$Gl2@!}cX&MWon#``mmtTm0H&Z6G~Q z<*xV~Ed^b~Qm$ik(qXXJ3Z<6Q0LgcSo+jNBAC;5XaY?X2t5dv5x_R}$x(C}|s%l2J zxnAz)YF_V;s*3&(s>-p(%E{eL1Af)~0TD3UyHaNXr}f<#Bqyp!v^^<2 zH$nU|o$;5tu+HKIwY}S18nptHJn)UZl@uzOoXz4;K~2$#!P zeta(g!>M9*sOL}S8)^fCZD(aY}nB&qD@?!PL`%X>Pc*I z=CxII<@fb~K|`H%%aY^^Iu4K)8TJdn_;93c*{ieCP4m(Z7uzHT|Ugca3oy z3JP<=tamZp)I(_3KZZ_tB|YfwfEToeMnP+J^HTE6&gf&#)OB@1$!ENqm1*CDQ_3jY z@C|RiQ#cm%+Rnw>R1K5V5*T60DeMNb#eV(>iz!7ippKo&G^?U5wAyaU`9mR6RA5EioU1{TtTy#_#ZQ zR^+mwlb@D!)*tr<{t#aNOPc;B=fAmuWzzPi*`oKLpp!N+=xq>!f`DG~yzXk&iKasr z#YEa)DbY7G`1;{wz)@uWCnGvlji4_aeWT}Sx?|?q=t9cF?fpH4FOb_P`DJGi`m{>} z3GZ<#P|PzCnxl@IAg%ecm=9IK88~KFCsqhfjd(yUiEQj9{ASCxRMTba2ta-}rYo9S za_-cs%^*Q!?>0gO(t6}Vr(!gE;MGIKZjo{Z-BuA~DgPXUFA*3vibOX+ny6w^6#lFS z&PwDGHWfhwzPSCpl){H4G=cu;@cMaG0kBPRi941c;kR%h`oX4-%GnE%U1|OXuRSIE z=w7?{d(1>=3sP?@?UIrK1^~_U8Af@KHYA0cua3qkmkwFI_>gMt!1x#e5il9uBl0V{ z^q~ej+7j?@BQdV4GZL&kr|H%`W)oMmM>1GC9&Dwf<-B_=O=n&mVqYmw++ZCfMH7w* zD#{WK0rFj@nTUf1%?Wm$CV?KQ{X>~y@IhZR)HVv&mVPh*LqNR0A_Pse+6wo1D`l)s zWFMRE{xiCDdTH^mrY26|`aRftV5}1{m%E3-=;p3wn2stMFl0`4p5>^*FeB--T0@ z0g^K8=v7Cqa{_jEzpRoAPMfSx_-ZPLyYDcJhT5kzVV}Z+6c^Q`BIg45Is2S0O=Vu> z>M@R+QEL!L(pOaLu?Bq_r*azHy_2Ys(h;4B2TNa3lE z_^73&ECK2uC?XsZ=mDy8nQ_suy6!gl9@YI~5=8BTfcToeweL>!Pgb zFdH_$OmGXa#XUP(`PHMBX$3h7Vh}QZlJ!%%pWpu&)5C?WCbsmFCx=%_MHLfu>6U9M zd#STCmAo7gs-)WJon|N2Ba<@Lb6Yc_BWTJ=X7#?i1f&(t;xpKXuHpcZ%Yauce|if-Iyw}167rLE ztAp<_(OpG0CzO3-Mz(oXWqU|D24u;?zZMSc!RO@b`(bA!lSf4{E$+aEN}2~h=sS_m zU=5co?~fuHRJU2$)o}o#vx+V4%9+MP%I?s_TJz2xE^)xjI^w53Rj)h?rtrIBSDwQVp$jm@>AG6U_Ar-MU`A20Q{T zqw8B7HtJhCl-5>*l8 zj=8c1Vlz}^m&}^tY89cH15fsQ_2e;0zAHzGRfXXq^8E=JgseV<^SYSCMjiv=KKp}~?r6FXa#O&(OF9Ed=NY_s@BoZCP;?4E+Q=~?l39+Q2(15?*t2kb- ziH?BrAm7gAPE?Ey;-PZo&)y9GqRnG7q_I3z5GT3^eiy#}R)rd`tK0@g5C%h8tThjJ z8?+-~rgr0Jj}o_=RAkwC1&jkIGRGfF)~p%ceot{y3@ArT5Z+u!=*T2p_ehaTMh3yZ zEzSbOSxHY|NcHDaNC%^2Y-)U8&Nq|g+sq~Pi|HUe=t5=fc;`eQFBw=lOhFs=f&gr` zDOrM*9GZM1=1wQv5~XG2s`Q1c4=tO0k3`ZwHGKCm7Kj5pNh1_AnYQ}QY94C4Kv^JI zQlM4(TAKJfqc2BlDP@vQacrBqbsU-Bug4|o_E@<;d%wLf%)!PmYD5*6RZ(oIPdYW! zH@Cl_FJq~tpAEqH7P05g6JifBrVGGUL&*EsCFvk93VcwF#*8!S+cJ*Vvoy3WWp`y| zlE)4e-Eg4J=x$(&7pLfpZ#9$tK&Kp_#`%oaYmzZCz#N8C|7vPu)qW5y_!`}8XV*;2e3k= zq7g2LD%v9}`luS?@s<;BF2cVgW_H3O{*z$EuDyz|-i|QKo`Q^rGFWyi%4c-nC4PD{ zJ2mF|m$=0*^aYkOonJ;33=S#R9*BHB;d|p=6;K0^n3m1k+e~mv{$i&shsSA-r7|Vz zfqZjwL7t9mI0x%NE`C#@m;jkxs666dB$9KB?;dKu9H^z*JCGTB0K?~zX)nCoVERJq zT}$ukd9Z~y%jM+?f^Ewtlmnt(9oQ8j*pUY<@q`VnB?aua7e2n+)fl$-FBbhCGCD2^ zN8g&+*kHKZe{YzH@IAMjUnn5w8T=?#^F@^vjz)`_X3aviG0`9oU$xeSy_qGF!2I3O zj%!@nz1kdGTY_Ay6f=_=@! z7{3o%jZgs(^dM0*qQ5PBlvkVRs#uxX7L@cB)MpN%R<&rv;%KiXQJ-9p8_IsU_m>jW zybyy^gnSUEb_h2L9x^0|9Z8bB#>52KMw~aF{Jq=)x0aYs*6{a&mx3Qw- z{Jc2d-H|RAY<-lN?e8K#1w`y`mBF2!pvTr(Y$-wXkNI(jEK*I&zIAxDWge~9PqdpB zA?Xujh<#HX;+49=*+ATnN<+Fck1-wREqEoTn&BBCAz%swk;xITcQxJ)Go^@ z9P`H*J?0p_TsG&^D*wA!KkZw@tp_{=Te|)=Dt=zX(<_DVkQQs;+jP-u;5Uc3rzbO|yt<^!W#p~yz=J$cx%F4bpEocE5^d!^+xw*$ZjoL?z zxI@TRIc*|8ZR(FDY9s2$U_OP|S}%hhNeOZve`O}vReb55wpf#B4z`xndY5JDc15m(8szEO3LArxSkxCli1aGLfq(4;H(( z;+b00z=XUVw(?%7ZKDhrbmfsn zNAcV!E|H6(%)9$7M^doyF=NUl4N3-YHtGtY=y3+N?F_8gc@9N+V~I*h?ixqs;=YDS zhO(HP1RCPHgC#K#0b-T4Ui!;(P9naY?^X%Nnr5W}l&?vkBp($9-#Z={w6AbetEfaB z=AR#2rcS9-XU&uzbrEkF&24N~&8lXks?CLYm=@o?1$RDnp9#ciX7dokqeGeJKD3gh zxR)A187#{00h?l}t8tt4Jkwhw0!%xLHZRGY>);jL`qiy2m$^^mD}9<$DhMi7+C@K| zY2L=YUmCDnhTUSj%V>`o39hX`W+j>P+?nH%cJ%c2+V*n#A}}#HZdi6x!{IsNmTJ_- z;sZaG%HI7MLsqGNJZj;k*oLrF<5EUnEP}P#4Og{)Srtxo$yKWcL~kRxxI}!wbeu)r z;hV}j8ow!et7)Yaw!8qk>AK`8pQ%-SWbQm?zA~q}!}d$A7cZ3+{ePO)RliN^7ocb$ zPbC}leY``y11AAX(!tKY>)PTS?T|$sf>WEgs6y; z@@pB<=Y->ue6#SMpW1)^BR?Sw{p4Ezc5$FHw=uE(Yc&1o-k;uiBIsXZ^iM4ip0pSQ zeLkxH)E4DQTWGLNv6~w{vhE5 zbTDwTb@=yvJTBHs_N0#i&i`j0{~r0c^5#?IvG9Lb^YuCMam_=@CrB5u|Ga?bbL!)= zZ;VfVx=5_vQQvc`d$61OQp0MBP z{}&n9pW`2AT7CNDZGrzN@$qx;<7}KyWg6D}AEa%4j(r?hN$~{BXZar|_IwU~91r(& z-X9A5hY_03(T{!dpWd%|@P8a$_Z<4zpZe)b93i3kiKXuN2&VKBW z`IIe@@J}6apR*tPy)ivu%Ow3*4xZ0Dc1zGcf3Us(9QSyU`E=S{t^Ch7Oys2?Ab+)?{rQ*mbA?u0^XuFH0#Hi> z1PTBE2nYZIjY(5SCpKsD0RaG`5di=b0001Cb8RhSZ*6UFZgVX+E;24^VRD?hV~j3b z(=OP%ZQHhO+qP}nwtcs;+qP}nHh1@KyL+DJ%$JkFIq&3~m8ztEtgDhrRVB5qT8c6t zpr}BQkdQz?Kte$O*8>Fv3M40{DnutGFV65i2?PWJq$mUN?>!*k|GJ|1zXhZIqyG~u zCnPT=E~cVNFDD+CEaR}nh!}bc{78V_iB^9+DzqRNjv!Yg0?b2M^gAO@eR-D81SrFcuZvi)7%#A z_oUY<|E+3uQz*muY&*pc+J^REhw# z{}d_cKkDFY!SLUybol?mI2c=-nYenG8UMG?$p0_2k&E4b%LDfR(u0YEot=Zd%YVxg z-GB5XWa@5ZVkTs2I%TW&*yeJkr5P*D3 zxsAK2h$ZQ&3T}dj*I-%H*xJH_3%k+^&Ye=&4k7wCj`uA`krfqz)Wpf6SBessKfuj` zaTQzIbD#Ja{*dwKUeF^Dg2DEY<30PU1@60L>XN}u|G?pJ0Q^l83rcj0T7$-W1$6E5 z6m5#0@c0!+$J70*q-w58N3YNJ?{%puFRtCuhRN5$jUeeZH(e>BEKV2`sDlLM2EI)V z9JojnXwYY>OvG(EgrIe4*fPIkMmNa9 zTTRM4W7<)(fqqqn(IJ|iMI}mOBRIX&0aDCCZiEY?(7HkfOO<31(tv$SpY?9PG=buLli$ZS=v%U%LL!i>-pc*>_i0kfO;m^sr#v$cGxrAu3<|CKX|b<{9A&I24Wu3X#t^DTfpav)T*11sg%a)bBtV&Bp0p;} zG_~)-*ceCd(4F17xR!3-nquw*Vo4|vSrs#Lr)-v*c`i~>=?Wf6gfuAmGj}TnJh8w@ zpeH;l5Io5~6-zThrH$q34-jD4tEyKc*Qb+Kk!eMO4v)65(%@XAelzr8cQ;IWm9em( zoOC@iOFpS5^5Hvr=dDFSHbDG%%J)5dV!i`U_hK#;fKTg~nT-h;lw*D|XIPCcrS**} zDL3{^LC(@CgITR=tV$Qu%}%eHzQw@u#cG?Z-0H?GaeG!n*YOC{6Of?O#D^Eteo7WI zBcRbVXRp^gN8&fI5FL0H>}Id1>6(q&X>^@frMwC9xGr z-sO%J^{v&*V~jn_R%Qv?W)_HHXx#CcW}Sk#@X`%%bP%vFAm&l$Y`KFNL&oI<%H8t( z$US(RWOeT!K`UJmxxzvOGM0)Y6II@(mfWC^Gk)c8NH_3;agzt ztHibs!>hRbG zlMaVCDXu-Vol)>8H~ryGrQ&|u1PRJPYC(u#V945j?z=?#(S(p+VBfgBz%lMTQvBo` zP}zd+3`K7VeZOqFxbc8o=#=>GE5&OfmAlh1iMR;(7}*c-YlNt`OBg05XwD6X?39UjkX))kk6<>$3Wx-!G&g+S4_ss51_2uqMJ+ zKg9qWyX%X&=4XTf#G_V|ah4vabhvH%Gk1hGi}s!=8^7(%_ML3@v)dzGz=O2E%Uvo>V{R%ITi93b?slQP0%*M}0`+EFw@;Tx;+ z{31grB9Ly-y0lk@tocIez~K*E$h|MpiBh%!T7X+uSMrdhj_xEk^y1 zXirD^5Xv)KxFZ7=FSy(0>IGxfgLvj_H>??kZCVk0r_jDbyci!d;=K0ykrdN0cgsPZ)n&y&-0uv9B~}C zCAv)+5ZIV>D^s$9Ica(9)cSNSHCE3QX-0+d#MP3%O>nlOV`F`KSAoe@0SM)-M9@s*exh zjw7mI4!h_ee^zVw?s4Ki14Pi{14A~^Dl+flc%wELxRakVbrZ+AHYNCwc!2(fJH#nc1{ zIk5PL@bf4b%w{_iC}zZ|0xLkYP{n?YFs5-kaq9({OB>>mIbpdSIBy1H>V=WojjXuB zt%Yngq`2>scczo|ptLPLwP)K8DlL|5K&Q{EZihMS+o!Ky=yvZ6>HhBIJ(YaRBsno@ z&zNErdHUVVrgqD!cATsmy`g3f5s>7-lA82l-Jq~&Oklu@FQK{pdEtzxNpZ-Vs+iXj2&C)BP_Y{FLq=P=}X1oedyZgBfa;F>P&!lf+k!knD`EoF1kvln;;E%_%!Sn)h#(g1f2?eO}u}y~ZjziySj2nr=oPy}W7_7@TR?Fgic} z0_$v3*0^HOd}+9EYtz*MYdU8Mz9`vvtL|ZVSD$WV)w&;Q1S(c{1Cv$X?MB0Ms)juJv;b z34YSM;O^sSg?0qUV<(b%S2Sx*pKN=cyvj_H4_v}Y@h-NXC^kQQVF#`~lLyIF&C-<~ zSv42BpBVZYmdR0Ln0#4hj%Uj#-PCi*%3F4LvIYm%$ z8h_3NI;mn?)#T*o?oDzYP2W9VZebyHh9SN4cfLX5MFIw@!oxUVG5k%A_w`I@zb#{j ze8lPbS;_ZFiUVnj9SLa@X-W`pHmuL`wm~E{Ju&*)R(iwIh~hP3sG+;Rzfbnuzsrkv40|7u#Vw7@|umV zn`<1%&zQ~p0#*cZ>r@1nGmIop38yGUf|xKy!Vy?vqglIhE(ifp|J2glutaC%ab5w7 zngXNNy#(21pP_dSm1KNv17GlTb9Rnb#JHU!66;d0jP^0LYXdpL0wupuIaFA!$T8cJ zgY$A&azo$&y>OIL+{UOeE#vGca`C9s*+!i7 zX_8D>4NDuSYZx<681qZ&x;FI!>=Wxl%yOEu~9 z80>x!DT7!kO;hTR>wvGrEGR9p&~IdqMNQBb=tBNJ6YHJID`ADl@-7E#3pA`lP^`G4X}82&S7VsB*Y2% zS60rpKhh^m0}~SkC4r5Q21$v65eaSKKv^+?$&4f%CBscgO~V0Fnz}$@)M`pd7~7ht zCQ3R=Qi7_Eb*o@4)UEVtI%qX5t3@qeG(TJ~Jpfs9l&hDYlaHJ3huNNg`4WG9&w~#M ze274#b_AnL0Zol#1OK2l^6t^D?r{`tHPs`7mdQO#YHGVj)@q$wxb#(TU?YclP|@=} zuQyqT_FV&>IgI6hylyLYo6G`kW^@~l-AH6*wNI!g?~Lm2b{{G zR((Rs8fq=mx$QqmwM!!g^Rm)pC!4mXe&6%_xtcZvt5e%j$;S4J>#d(#+Sj?QsnN&y ziyVc|L~8C0+-&PAAq%XJ>~f}gqNQWT6nO$E^{8!Rp!HH`^#~pnl9{V^P*B$p|BQu_ zStSJ-^T4f;;UP}e{|V6&OTexW3>8>U%J6I8XBCV79IPjoumFLIOJ9`ykql|YYe)r$ zmk3!(d$&GQ1l744rjck2N^3qrw#KiqyKq3~%!+U$yk}{3)f2=hzi%N_7ZhNT%RLVT zVu%=wUi(7T^txa^bN{lu>LhJ8(N7UT>G{XF`u zwhsyit(ORGB}Ljsj4adjCk;F#&&_s)Jl1hg67v`sL{^_;5=;tRHFmzqZ;UHXkx#1{ z$|RRDeyytjrw;ozP#;g#_+lB9nR}?KQ)DyzjYi;AV`@)Hk=9)kjubEqj*H& z1W*(oR?16Dt_p=?k1lul1UOv1HZ~r_re)!mB1i(EH;Wku?$^9VG#)9lFx)2 zu9h0E{2Zufu6D+l4)fw=$v2;*sVd9mvx|p8*REzDF3Zzrs%qW@lcAmVG8IH?qMmaf zV1{wxPpZgc+X9=adBH|@->B|Tp)T#{w<%Ol3|at~F@djDg&3?~cNO6KPEUfN)?0$e zFiVE(N`YSmgWK4n3FklOmJ}v(jK?$6VIKEidk715NJa+d!vr%Xl;AiEDM%XOSP46N z8G}jmQwl{&14Z>wdFxIE|NHk*}9bgct4!_yd_B9yk z(fimE;!Zx*!g*rb^t5!)WJW34>yKZ}s5O|zuc^pXSZoGwl%$zWmCjM+a@AO8g3w1b zJw$p@I&7bZ#u%l*tPX`;XJjNr+hBL+6~NF(1ku*ek$ye%Y@M>RGA!U7x!~qIg4v#& zzdl5+R6Viq9eAVE5(AWb3s9tDZN0blu_<)n);bF!O7j2U?xgDD#>rKw8W*m>t+LSl zf>;&iA&UE9k|m$+tgilzmh|B+YqXFuZ;@*rGu%Ne&M!WXV{J{CFRffe;{eOHsCGZ8 zMcieCAmh!zmFZE_R!Suc@kkCux;n2{pK#*AvG7>cE<$dz)I33bu`p@S?_cV}uP(#*7Jiy^Ie%tvWjP^a+sWreHuyhYj9Cv)Uvy9&xDJDRo^nMJI ztIlap_S{u2D9d>qI)t0*b5|lUhy>bDxdE0S#P8t8d#GT|ChV@C(E>~037cp5oKM>E zF;2%z?QSY+9HDI05m$D8$I3hO{6Sk$_s|X1N6xk4Bb>I|qJCan{sa6;j+GaI!9_#u zI@WG#vkS&BMALo4IZ-LHVmfFeNyFqkMh!JP#918YzZ`ZMTM*ByfvV7B!dEg6yd zI1kV9PPAz-wC8n3mF!VKrMj0lqWiaIDtK4$;!kot&d7m zB-7MDvZNIa-+!t;w`-$sQKvONn#CZF?C}zm=u<+i`b?43@^8FCW{U1#z8vzqlrxcc z=RN^kXb8SvGHa{+p}l%bzkRXVsFLS}lQ@}+tolDGkJ#ii~m)tgnt1#zk}JbDpPW<9m?hEpubS zMden~AlhQqMZc3dcZTX1{^E35r8lrq@37V-Ig`Hn^_?ac6mRP~Bl?EW!~`tCjAMZW zLeq0KPCjC6t?Z-oEAGCju6eObyahK&HKRbuRM^Ck76n@Zxg^%$l%yl9T$?1if_+Uj zSuyL$OBLJh)v5R0qDbLVqy^^uC(-B#|^ad>8a1e?!>8+HXoa(sB z<}oxoSpO;|nEJIxm*kwklDq<1M&|$~+<5Z7;z^fye_0ZVWvPk%cn~WcEEmTBV>q=# zcW3TyQ7KOSNj0(j^}3O5KsrY(jc@i;0e^S2lhNxh?u`_=37SE%m6v3aB5W$fC5|I~ z<;n8kqyE5!qx$@P$$EMSb+g*Hc^wQ!g=%2TpT&lOxds_Fv2B)pXRqjO7BDL9?az~?F>!|v?hf$5( zMYZWL_4_bv-1!V<#934RPurOo-1e+|6}<9lpJ_q#Iw|zs(b)XL`f}tQ>p^Rp=H^;>G23BlTJBAJ@eQC8 z-49+onU8R&yCfN{jjVdFJ0!&Yy=%mo2HGMeBA&)pqi zgN`6+xgb%Tm2oq+fxRkFSX$IGJ5d(55r)no3O^ZDp)>b@ciwvXy{DEDgxsP4R{Z+c zi6c@MQWx<(j=W2<3fNkQ)2^+9!C%8$B0!hm^4P^Fu~w|7i;#=$;iLFh!y`#3Yg)H5 zB)9xmJ0?9k3@u3$&{sY52c}4UECw_rnK_W9OucN{%QOcjo1xI)+5s@;0}D5alXXra^SdAQL|cc-cw3Z$`^^OIZs7Zy zux1rNz(rMdtB;wK&(bJLGrnxwHb2fi@#js;gTs|3GHc&lirp>emSn~6{f@%(mG7%p z#Qc$dB;_Tt@1-!(vG_n_Za=qw z#{^WwX*S-H2dWL{dEJSoni3eby=ik_-XndO-E!g*>gCTb3O@Ml5_|~{5e|+<{tFA) zF7|ioD1X^cKVb_$=`2pE#;;h|%em+W*cf-!Lom}WE7t}Wzw0Dl0#saGYVedgckDW0cj{Q9KIz8z574=To;byF{|c&0kJ3xdNnbR< zc$=2QZh%Q?Q60I5RI~(=Q&6Q$&u&1K%HM9jwJMFeq!l**z&Sn9!G`|?A-{5L?7l?v zw7F@gc2Ozal=lA`*)d%F2th?BGa@K$H@ro@{3&hJRaM$03Ku^P#9XNc>Wg(BNlog%w;_en+hLcAXPoM zk9ZARi2XNDqajpGB(ex?Qj1blb)2**-J4<7;u??XS2`|sB=8hbT6?kVCirmau|>C< zAvmd#!+Dfn5rvC!Tuw4J9~m1m>;;xoZiV}$==Y#8L~osBe$H?Cq#SsBzZY@Vr%@(j!J+ zk!aPZ%a!p`CZ_DsVLNP;J}IV;+!0UJc8o+55O~4?_+VRHSgvJ^vjauSK_kUjdrSp@ z6)^fB>ZCkNePr|`%RmUpK!k-5qQnq5F38%U=6c-Geg>$3?8X&V4y40A>1*VE3MX3c zF!L*qyh^l=!oZM%^cw{jCxk9EiCUmca4ThVkY}U0MT*!@3?Y1WSy3gw7`$aT4%mff4(fh+^WB6qpt2;)LMaRKm=GHIL@A z#h93cPq+PBA!HoPKmmU(4t=WR74v%5 zp-FhgMsQ5qu@K{3QHKNa%7l4%|NJdvr6QH+^(yU8TH7M_={;#~ zbS*sfL;!Ht?eA=rhV^hhoqa0d-+w~&L;oTMCq6Aa+=#sH~*ibGWQz-?OUV z(825(s|M8%PnR5ZT%DGVJ*{Xd&XvV%l-_Z%*w#b$?f0QoX`X%a!Sd?|3LEmgAg?Ib zDH3L!L=hi61~F26ba6u?CpJwCRfHx|FJ{?>m^hO>g=E^PXKVCd@{<~ua*spu0P$sY z25_25bgicpofbST^)928K=vzEI}g&C8cn3RPbK4T%-;tn^91>%5?Cz{3hKlvxgp&~}Y+iFJxzEBFY%g(_wgUvC z*C=i4;>@j8C@-R&-;!s(?e8aosR8mFQ)IOWqeRJ z^7M{d|EvkbHLF{J2{z9>la>?TAQah{UIALOaf|v^Gz{^@*n98-eU;a;7O*n^n1`hP zt+LlWKUoOEKKCc6{iuQwg}=Defy;V`K|kLs#w(gX6oMk(c6dO@*qse$blJYuXE=6D zK!5ftUU%G2Fv8zGChu=X>Ia01T|a}wFmd&AdQNb>) z2Mj299^b%~(b6pl#s(9n4(Z}bBh)E$yMu^7FRlp1UKo<(#W~J>RfxSFE3pu zQdUQ3yiOPp<}fIWc@_gwu_YnwMxm=g+%1qkg$QAiE~Zf=6iW-yXZZqFO;T*{ z6gN5Sy1T#|8@T!vj|Jyh*Q;o=2c^Dk-CyD)QtE`dhLQIB$U7J6d<7YjK?g+?s7o%v zk|D|qFy=9?nuOj%Uc@Z#R;HX{+jz@~T<7!_6(3UPVsr*@tCd>}al8H%xp=Ue1;jKo zuk`B{J6vx(+HwtBzS)zDKJFE_5Pj=#I~{{kLui*g%C=A$VzuV_FsprMK&C`-ELN$X z{gh2(+nEj1_|Mp-SXj)0M@8{->u*1!^S2b8Yx56m-Z#Zg!-HbspYVb_O-B!XGa=$e zqFtmnSUi-nR$G?al1KX342y*PkGj}utgTz0{MI`nKl?vFwU1aV%b#s}yaazfSvwcb zMNBy3qS$xS8L=)2con};rQ*Ywf792VHy2}B7ZylSWxpDk&Q-tp{|elIMTkGXMfnzm&_ z55b`Ns-wospH+?{E6iaHj3y@sXe>zh9TXUrVwbUcl)A%`+tjp2JjfD$z<12aIc2XOlv@S}th{|xbVpMx7Y3ns5S=ReFv#UQ$f2Z} z=_vv5Wz2H6_JOD++U1^bOUqlwf2kX$0iJ(FUqRkx#T^^)F_*M4AQHG~_VC zY$UAzsNiMjr9_%vAI_#`kggZ;i@0rDx;OBZRoS#cA~k@RdAznH>*qA=cO{8U0^gmC z7in${h)Dms(|PG^Li-~Kvuk?6Wj{~~xmtu5+wES^LmJ%Do`Yr7@11*0Z@M&W?4M^l zaIk6(!zD=ccKM)FkFl0-r<}H9U8Rb$@u60`DH!r#+(>=(Yjol7yrz^1HQc>b=3EKB zman4Qn*s`|`aB6J8_oHz;8p5#``}fobM|0*)p_?|di8lUP&>tWvK*;|{n9WyPb+MD z`w^N(ox4f(?NVhLmh<>oeVL}Ph`;aZ-AZ%?e^P45pkZjxDg%oH>3*H>1%1wl64HeG zM$Dlo_cH-ERZNc;9##k(Lsxc>0AUh`Pg*l6&teoGD>G?(51VQ+_!DbHS1?mf9&61>Lm>Gv}6Pr&~iIr>vpNzF4$wNS^JE*ZSpsWY%$o+ z-(b*fh8*JmDe%7>tFNf~kRR=Bh2Y4g;dLt;4ZI27}X^ps*iWVj+3p?TMI(#BvL36mG8M%F2IedA?zKieG?=Evdy}i;ICU~qG zhIn_sEA(%AmFyGwRO*xYl-gt9UnM}yKd(7d`Ko`F6`>cczg8GFB+AG|@a|m`@tIdvGqIS}0O&L_LrinIMBk@{y@I?- zZ5hYL)y$(FvrCu_(JCkD42WNdvQ^0?4WRsA$A+8nGOlY9rSX0ls<#fBWs8%ul}$OYmvkp-0QW12CIN2}jyjaXza z+5Wu6EL^u+CL|`H0b^NoHwLJ`9?Re-p#VCV6D1r1I;8Gdy^eW0cku!rc>`}h_ZW7G z&&pKt@w~Kx6+Ijw1o|BjGU?O>R$qSsYn?mfM=_Oodep_(0f5Z$%)-Ma!|{x<#jHSv zbigG0=_C9rb`h+Ab56!B&WL;$Y%*8b5ldsRJ+um%e!obOPyT_;6ToO}?{oS) zZ%2}~1ZP*JPj9(O&yafTKTXVV(oquAlnZ;C=~h+FwoUi; z_x=3Bd**m2P}ldX03$$SJ#|7}jQJ>{#`8Oq4WG%SLXtN{+HA37JpMOeDo&fq#d~uS zya1J5m3}p56uCo`e~67gEFHOlE@Gy*bg;4dp|+k$UGFG(ISx9`DF};61)azw7+EUo zqGCMHmpv)(FgQF&lsjp!*3oAiq#<)PuElbj_f!N(F)}>-Tfh@M*b=%3o(A>vO(ZI3 z@KlUcW?TFt)-y>uEWSi?p-G0NO{)fKFhLyz4w5x_&kUO1g}bmI9Q86`QaLPXeqOO; z%Lc%u$VFzIQK*!vS2*9%`~lpqOw@n3>r(~*S_|c6q$z{d4Vxw8{)^O^FzAD#jYG zq)4DfrHe^pe?2q1@WOo(0_CDREtmDE$gd@WIP8R5c?b{%_)TV|(=kdCsE=hSf_h&b z<-X-IIMUC)RpAG2-#<1?T&smLr)|F5m=V`G(z8^BE_0V*Zn#fwn=$ytx#YxbC4UmC za)%sNWnFJl;yzDz;Y#js+Vb+z(^P<5NvIl_;<{~hatID?7qM+`NMVl>QE$Jx0Z#>q zo`d}0EL{@5>4~`>mdR<6Da92tv2 zlO+7x8=^6V()tu$i%WwtP(kMNiGuuydTSlvON-vB*{aH%C}Fbr|0r zb0&JX4=ir~&Y`1r(fX5)iat5bBYmTmm$y`F0rMQJYUzmk#`a8b77cnFdW!Ct2cr=g z1!{aKy_3rQy;6yOO^G68W5hXC7scMe0-rVDZri*!kU<)uV{g}dboyBKjlWHkmS8wT z^nD7PKVnXXwr{Ypg(s0**OI@@TTLV?SB87Af5=l*d$!n_w;s@{+E~;_9y8pXKx>BY z{>(@dKsj_?O3&7*pWUv6HRD`(NrZ`^AhUqVJLhaE6Hc~T6>vG3eXW;kk=>>X^B?TXRp92`=&-k!wfG-2>bTafBn ziqS8Xo@5UE=LL1(%!Sye|7TK3nYzV*%O7W$*ME%Uig>4s)C|N^#b`W5*lKZk_P`hM z>~FfThg;NSh%!*02Iljzl9zRP?o^jhcL+?(NAWQ#PCr5Ad+~9o7Ego#xD(Rt18zMD z3(*9=%I0`FP4ni%x4 z9IOh8ghYu5+N;Ga@b=W2$S`yb#oey`Os$&6ux?({L&;vC0(bt^$fqZ!PDbbmM_VNq zKZ=~d!Yjs^HJ40;sb0a=7P8I(A%CDVPs=uSkG;NM82rLB*EZ9l35q6qseuIa`yWR9 zV!Sb~-ZskEKlBmq*Y2FnNAkYH-$x+<#p}gG<08GqZ=f9h7nYXvwg&HZy}IslR}OCc7M z@v5*^!}%81M5OkF>7Pm%F|Mk38-WMj^G($G4IH>SjM>k&!t7pu!+|sR6~;MX7<&%% z$+JSPoO1`waw72%ut!ZIY#i1bsZ0Ox>im-rm+J6arn7wG6J7&VE~N|?)wP&Kl8F^= zKJ7^P<(eEy{4tG-IJ;Ee;N7Ij1He?n1%W79!YD3D@6Mj0^-hW%T$sJ05k_{u%2!Zp zppAwh`dr%96N~!RTmw1-lXssnslNG6Z{)9>Rrk2fcfBs-T0N$<+Dsdb=@uICA61L?_jO+m76_?8SZTht^ zrYVXKCVi1eh+zC)gDH4zOZ=W%s2=-=^OCz(s2==$)^5(!_gOcm0B()hN@fm5jo4CepcY3{S z9mA)8ukLeifKgufYRqeT*436m_bl0@;XPvnXVXP`6~+-N1!t`!`*e$R$Gv+h-)K*7 zi7z+kgYt%O!c=YNNBuQHH?D=*uLcn%^^vKG=FmiX_2A#;s1uW;fIDIU+P~i9nT~x< z5`QXzsqEAV9gM4L1G_b0--Ni1`>F&d`N!c?5a1FJsN#y={fhNPl&Il0S-~P;>HUrg5Y#(c$j&71 zeFE3ZnjsFP^PCjS$j-iTqDauJ7|dbYFga!6qvA+$%JrRd_n5?P z$gg*w&blbla&!^G1h3WdH%w5=bSe}7(@huW3%;3-^Jl8`oPi*8um94lmV58Z|{*&6uZrZmp`k(8Ae-zv-$oay?EHrEnBn;DPIWso(t zoq0o2u!&qno9bs*AipKGYUNm6ay5lL_aZMzc-Fm?46w6K3U7~Yy0|Gg<;(+8^}8## zH7!s1yH)MMgN%pxbf#TwczY6dhii;@+k0%9(LE9@V{${QLOLb3!c|e|fvw9z$-h8hJ#7CcX98N)!ZRJ4gsd^if89)NzVP z-Pq9R=sH4DD;}y9!wA(7r&Bg(1`8;5INh4fY0}yxuCBx^C8*~HC!0kke<|ZK!zj*( zjq#kC!!_Q~?@aC-^Y*~wEaKKRcVTa|ty-Vv%rAz@&CKKs@w6Zk#ST&D1U*W6G&wLR>VddX;I|&EHH8DqP&V^b&?kiGMVGVgOP! z6jo^W?Nlr?QWh~DJy^;$olEjMnq#%KXJ1JBP4cnEyJ>F5&-WYCUuf6vAY-rY2)>^X zmXgiY*V=-|Ma?;94w5%}(=q4hyaOxNW@8U@fx3EORTm}t+x(k4$-BP3;3x|>V9n~> zU2%clB~3PIfHPW3bEMqJG)JAWC+zAKd!JX-pEs2J;;-N{Rzi+8sR$#p0zB+Ub-ryS zuh-hj2rq<-A(O1U=V77nT7kzfZMR4@3F6K`w-Q_(Nifb*VN6qSp84=xv*8w>j@Bb? zKoyZa6W$q`oUQ@u;%8nw`ro1`wK1*gq%D7ktvbPBbVm^=v9ed^{;z2V4q{AUi04D` z>!Y$o#*Tpc*e`wrrF?Nxb&evXMuL}mI%`{$hJCE+81|#ji%Z{}k(vf`klU+(klnt4 zsRM=7zZByHjM8%WMl_>LffAW3#h#H(7P#hG^eFoOL(UgF{)(@4XmV#* zvW1&1JgY)mBh4E@bITo=PYR3oBq}kOqH-izA|+#V!oQ)yOfiI3AUUh(d;#0n#AX?( zMkwh)XQnB&Q8VZ-%QIM0;Ml~_Hp)wPixpk{PSrVv-LI)l`Wz`j97!^!*j{ix!>W!_ zu05hy_-)wyj^X$+V)?N5gWE?pk8L@+CR4J$UuaL6(P+&)8Iv(<9uMKT5=((Zf`!L9 zJIO61mq&DMvB5BBgpgOZ$zX`gFsXX0!q}s#f~!KklA2*XDZ^Hxf>!lwTPADu$o~J} z?45#i;lVD^wr$(Cx!bm_w{6?DZQI6f?zU~)cK7-InVT~gXD&`EE2zp%sB&y3^u`EX216*_WcRL}2Q6@MB%U7{!&Wm+BLH#YQ_*ZUEXV=$jx+&TYd zn(E}q_O9d!4R$zxzy40`|D_ee(zoau`>b=bde69Xo1eVq-wFax5lthx zZ`m%6-#!Gn_dk}YQD=get2$jpb&`2FI>)BhIAwMtGldnNn5MQc&%5i`$f!)bJJZw3YbB9 za1#wB_7p#EtVUd0DRTX0onJX@aPefQ)9Yt!H2nwWUb-A8gqHMeN zNTbBdHeyIjWMzp?>$L^&YpiXa$EwR;h`GuYpSIHk0f#}2ME58gW>^TP+XyT;f{Nfk zII2j%t(F!ehgB|~UtC-ax4Y<$u&)O@hNaVQuG(B2EXG@7i3AC6l9|_0RSSz}_a^wR zKl>yu!f{}A)h3s~9b(#e#Ag}Y_-1m>+{ww(Oy#n|9HX+Vpj4OgM9zS8|B5$`8=g>C`27|0+)Mc-sXsMv{y|!PA4i z8+sq}mr;p2^75SV%5fq_tSLwRPCB@ZgQh<+0y*|1UQ4O>?kXFRg|LhiAte+ zW(=p&r=@;m?zeHQA|w1RQ(S{7r^9g8!Qo=KD{yT_8X}Ffb`eO&TUL zRFO76KCw#D)zX)iYia5$Xmxa3EBf4+gQu*#*y^S+b$RO;7OoKILFbwNwLkM=Bbs5mYrZDRfiNwD$CTB^=AUqEG^EL*J^Z> zrA=9rVAEAm*P~25)j$(%B^l~8B~2ld*wG&&V&<$LN7iD4bY+j!;GW2%`oi9pQcykd zLG@{Q+6_;YrFH^x(bJJz@hh`nirk3Q^c0p}6SF8k+)&6r+~!s0-35cm3{;W@@x;S~ z4Y~DP#M;F8m%5W2b99$`$M$-%Es#=(Ii}sld@q%s7K@fvZ5QVPQiX4)dO6{gHtst=PUoa8DAqQQv z6yVW7xq)ggt_C`1nqMY3?qBe^Ocw0vGRb#j1)@@_kfwS43NTbs81=V0;jmMuJCdlXeq*B*Cu<`^|}Wh8BYIk~m3m z1ucMRaU_c{Tf$eW1{>8x4eSkuOeP51867Miflc+sm9;6}!BM6)VzNIIxpeOXeF2%6 zN@!p$Ht2Wc@QE|l6fr#^@yT9zgE_8zZge4q(~s7y_3Z?RIB?g_4q+wD8X-}BVV#oW z713ZJ;2@DNkIt;*QF zQGdZLHwWm~m#lt1BUIbKzB5_;f-vhu)F2k-lm<#_BGj?$3GNAQ-^skCa+t#ga9~n_ zzm7^QI}!!0+gqGjkJX^;8WbB&jQqL9*C)6b+*pOSH@0ssxD+9?tK{|z1pXVm+r3ZE zb)E6ZiS(Q3HBqZe9%J|kiSxnGL?5qGlzBc*h$@~6fvr-B#kt4W5TEw%)0a*c2d4Gi zR~YjP9lRuI8Qfna75Y&P;>&h=Az+b_Eh{1Q7xG$z~@8rl}Bu zOW`f`^(lFa13I+$eRW$g^-_$*cSiEuf{6?grzr_ggKT4;7KR$2=uHyABi91Wwi7J++vh^}2i^%MyAFmJOEF=(;3c!M~`%$$+1t1*2kQva?+x-gT@u*4USV ziL7oOICW7qN^=wVG|C`K+;D}J5DpxhAUtm5$N2c2+G1{8o2IM0po%}Re;$C9cVrYH zR4rU>CWT;YF6hqtw^7dECifw-scHUj(4Nenqz#Ur6v#%%Ty{(_FgBp5XAPb-u`ETc z)feFH3pgFEr!k*MuMQ37p9~2ERg(By*~ue$r{i8JtIHia1Q&!#)y(^*dDs->7PQ(R z_j+P~AYseM`SN57UsQ84iO!J1h&jPtt52?z!NRH@;*K?uE(yOgE(Mm&&qsJT@7bP% zxsJM_xT=JcSr~c*BN}94$VEgevosmcYr zeyShU>1GUi&q}`1W(tvB1PvsD2vW^dNlc5TC&ARgyf)e#A96HJW{hdyH@_-1UiQ9q zi?TIlViHJ7a|7SVVn%PS-oC=wFf>bf526wQ-p6Qh3EP0xz2Y%mo8L#|pj?B!_+< z(p(*xg2d}$ylbTBe-rx>$~7$hqQ}`CyB%ko{4rOTn4tSvS{VBS=scmIkgtn$SGB`# z=`#-?uKa|POkF|%T_9^+4Dd_p+6{1#x@IO4QT~J*OIZT#aKf>wo)`&C8Xzdy;aHIi zJ@7ut6J%T4z9J~{3+aRiHcgcjk~`apA7gs07+^@R^O*z3!lbd~Bb_%zxz)57ogx^O z!yHm(B}LLD>{?(UqA^Q0bqW_GH<1Cx`5}*Z|<)0r5$EAT$B@y8YG_ zyJO12&1DaZFglSE)mOLKxSVb9H>YV-9f(5Cwv59q#PLjn1cme4UYM53TEJHcxUz6_qv zL!M-)dAy{5VZviyMOgy}3I0Jh7CtDwnsF5ak6J42#jTUr_mE-v`ljh*{BXyyF&+xL zo^48R0u(rTZ9V+5NyQuwwSAfBS${V|bjg3=%6?RHF8k7J9J|6_Me+~4M#Ky+U`9sD zP`{ZzvVp&Ye`?kVMRg1z=IfvA2ZN&PD^SpjaApm^R*wVNiG|Ty+>MH~4jG{q=lz_4 z)kFFmP|I+Ze1t1d(Qgl=AiPt*F{1|c8e^4iskPU}x6!OYO9lrA(akBaH|0n;J*u+H z@7SIo_subgOS>ak|BC(qF#BZ{j?Vc&`4i1?u3WX5b6y26IT3ijx!!>i*+m!Q_Xq9o zgs*bZ1QJdSpUE}Ms9el6Bn&c9WprKLGMzmM#&a0s1j$yN{G;ou>5cCS=Oklb=eRdP zg+DHbYD0=q)ueYAje&Va_J;B1LWA_K(_wZUT|>W(C%6Ux<9|ZgFgNrPsUKAPg!9zM z+Mn!jXi-6T`lYABh_bP$&<1dZR*BhJ3~cWqN1jKkF5W~0M`B@ATLPELWeuJI-T}FOQm(L`@R3T^ zcUFxf`|0C+j2s+r0DX^K=`+_Jf=i`7FnCgT>7PCA4Z9LCZC(-Fsq4Us+1brbsoL+= zaFA*6hh{b5LLa7oBk7amsI;18d8eZhT!idzoNvdP;18(~1~gT&eK;nP-GX>8kr{^u z;jYSSdvY~68Z7CsqUsec85TrXm1RLa@Z|*7c5sGva{ri6lNIs+3{SSAHMGM&Q6@@9 zD81+dfZ%r2S5iFml$E0)A?x|I!QuIrOFpLdjm!lT9?n$cV&$#6Vyfz2|BMRAHiWhD zAvUq47*O^#DFcfUea7`&UXAt)A3FJ28!MH3+VkkhC%>$)fqD#s?l=<-y<=48}ghT{F*b$|}|M%>h3dy_`iSja!2gM8+@sWlSnLe=#Ed0tD@l z7dQWCa?cSqyW{dy^KsE9a6XRHa|N!=Ytt<+J^Sg+O^RBbY2ZVB9OJ3(Ok6!SgDs!` zm(sX$ywif(`e8xfxzzN*I3gy1$s2z0t{o)A*lvvy^Jlm@9^_TsSYGTXY%ebJi)^zl z^UG|1R`cEn5BnB$Au7M(xf0ggbROh^dM4OUu!3!h&|ibtAU3yzn|Ozm@}q2GYHbuZ z@NXYzEubm+FOFH>SBUw~cCT95(U~EaXL8OQpS-+9))(lpB8<%Y35PC^WZ} zn|Ozn@}q7x(_OEx<(XK{iqWSV;@;^?@6O)qQ#|EJ?J|$_qMYpicgwxKH{j7;(P#$n z?~deE&u<{~oO-h`KA*%+&wCgALk<0q^#f9l6B%kE&~$r1>YcF1G&~|>Bn{S>%Mp}V z{7TvlgBb}p;c$P3Usdo_x0of5R_XR2aBvf7IJ)`p% zBK;dJ{!yIZ(spDl&4bL5$OA6M;w+pvl~o`eS|XY_g;3a2oiat+AUFHr`R@cn%m`MW z^QE8D8{7%DBi^CZhc3hW$RXKNa3BKOiq?x46D_h3T6|YI=4OaGagCI}Q;ZuS88hn= zjZ2HO!|AEmK?d#SEU^1vX+W0?M1xKycz_Q7K??@Wa2G1gNNH43r6{pXlvFNKd~kch zfpnkXiCRx_H`v?p!|sEpy2NIblFh2(bXcXKS35iPl)n*?rPnjBCsJum%N^Bx+u(cI zdk)P3$P<1jV91F$_IN<3=d_vlPXkDZlTpQ5X^%P87ULE-bZr;zOF;zO%b>uBSB?cJ z7oT9*O1K<^)Y5nutdIzTBCFXG^7hFwEyBNEzKNG(w>A4mC-bjKg3a`yy8tiGC@*es zgB`3;LIHTl1L7!pBx;IbkvmdwrN|B9?-Ea^_F$D;2)*dRVn>ECfDlbKbwY{MqHJ*a zfQf@*#61#U4O6$$D_a$F`fccy?y8@p1Tdk(gdToq@wQ@CcAps|+?d>M4M9oysrdf5 z9aeI7BX}$e6Q{|eMxzAwgAxwAQ~Y4Fl1HOP$G(NJ;qe{--Qr97X~L5!Z#sOv28O{b z{ST;)1#;IsS{HKYjK-gQo)w=r_zBdak_2AOXu43$PgFfy5>UvC zYbZ>bphr!mOi}dhO!&4e__4AEO0xcS8T$1Z@;HuYe*Cnz*~481`X|-h#N3RJe_yD8 zmBqJ=k%7X0+Twwnuxo%Jt%HPXu*I7(GRCy0%>wSqkdhT6R+YdY0Y#j{3q+1TI+4!s zQ6hmI9#2Xh-7Z*C$t-ZBg_Th)^X*z>14{lai6x-qT98^bsA0`eF=m(aNod(gXvJjy zC`mEGQu@fqm&6_lQcbFz$3(Z(<1&~wmtPxq8?bA`B{1>c zW7>qAZTh|g;RUL1g4}0+0e$25UMqT7ESpd$o=_>7phnY6F!GN{A`q7Ho@*Grf}g}X zjEZ&()XfuSm?cR+O`ME|?l_BS;w)Eaj+%@;Ye|{vNnu~mTfJ5Uc1h#l6?7*wJyp`2 z$;UFgS#GvlVDjwEU(MDz(tbxTX*M8T)L?e|#`)Kjxucla6US!OPwC9a?H93OvD&aBgCtCf-#6l4Y)jf@^lm;mKx1@iWn z=GqvD{y4j%D#j!0J%P;vb=%rkIG&PBmJ zh+P|p+U&~X%K+2>sg41^V;AV;w9ip+D|P44Rkn*uIOm}ieIA(97W8)qEX1QJ*1v5g zh|R+eqZREA9pfRTaDpuq>b>-B0Pp0DCrWRtN11OG&9)4lYq5{Fb= zju)zkT+hbjR`69I;(zf4=sBLdBOhki10UA-qY|!&AqpYPVxvC;Az*_NwRxO@{T4-K zDtgxJ+45e6Ah_!_XXs+F5i6Ein-I_p{fYI>L2F>pi^x(j9@mJ25W8*AD^vSbWMb0@5$mXxF$+SO4-fi= zx1v4dGjC8u|0v6xSwckt zWiit%Ka!-nl_aWFTwcLwio<^d-$8&guqQ)5PqrK)$8M&e*2GHp7e&S=tkak3KFe9M z=cxuoAv(PSp4?6%GWeAKzDbiksI;R@+7Velnz_e#B?{AvzvKb_rL-feF2*mzBwoup zz7w0dJqOgev-6A*S&g)h_K#CB{LXZTSG(4q@J-!hT9g91R^fMqqAftQate@HCj3Nh zgi|{G1Sd*?&k)41zG&wx#_JlNK+^)mu|&i}1AP=D@B|HRN^peJ7y^la^Ipig~IH}hY%!yROM zdXH9P;+nh?qeqyYjf%VICZfdNoapnEXhC2GKfK4OWTu4teqCpjut)QOq5NRuA3oWA z#z8he7{Glx-KA$+Cl8=Gp~@VPec3q#ht&$`@At$!-eBxqWzziO5N^L$Gr3it)Z`^r!FC-|q>FJ@$jk}1m{ z2KL>2u3xY{!k!z+(49tv<3J$7<+>~%G_#H}UJHlM${^yGK4_?7 zpz2njjM(grNC4Qp{}9mczT1dh)$e&Zbloc`Z&ybujsIuVR5nDejWV^Q%cS8f(h2DI zp@*+(bHGKZl&h;p>0kT=4RH$sp%VvsLH zD_4Xl)eaS#5-!Uy`>z5f_$;|C64bWahORP0ck~&xY3<7}6HjFatXq$@V^&#u82CeO zn(vsr>&kGW=86-1fX9u(*QtBp1*ZI>+G~cc%sp5UnlLEI< zzzf*IzqL}dY-(F9Y7>{}0bjPboh<}8)UE(?T3qe{=;>0HgVd{l;N3ew6_KN5tL978 zvr=|THe{nF4l=yjsh=Eg-n#=5GNh=CZ253$8i6VSz{Oj^v$f>rElL05+f8 z-ohP5B?+kUhpB*GbWFI`?156-W;D6+vWW33_p8@IP<_skN+pL(>6{C!nqxkl5bJA% z;a=2*)^57BOqk73N9r)&tX}%YcOg%kQ5w2DS5es%(^LXBl!_}zs6}$;k!=@>!Qrn5 z7Q<6JheO*{0?C;G?`~ph&>Pfn3jZcthj^?ze&#*27rR|j1#p%LHrGq1{GeY?v^13SzZtYDaTf>$CQk0O+eNm=lJ#rB#2-1W>}hpzHlg5SAz3Jt0l-#ebG`gO9CVjU@%W=BB6 zAL>OT%CC?F#fJTJLGPPIBepprEE>>q|C{(v=UU?(AUeN1a;opxk@p{6P0x3EG24*a9f|T z(TK}CjC-Kt=JF);aN1g@MlHwX7LW4-g79#Pp_*kFG@n`Kjx+ITfH?Tp=q?oCja=hq zCkB2PpAzO99}E={Aj078;vB_37MT(j;bXHR3G`~#8)269+HawhDE~3>!4l!dlYG?% zK*s!;j1jL!8x-<|gOlh|WDGL)rITHWHp&t}0GuBdisl2Qeu!4gQwkRgV5?uK^w%qd zCipSyM!(`dTRM~r;JaP08vYePC0H;Y5ccKZ3-z5nC%-Gwhp9g^yTk3p^NW9Vf0lj= z7eMTrJ7o>mA7Od&v?=IIIKE`J3DKYYcNQJuF3Rj_>5#}%Z4;_1T-xu4Mdz~{;JtLN z2ctUJ>cgchFPpBHGsCEtfwb`^S(4N!e)kytMpfW1!X?zR9~9qsl)gmUAjIc7NavBe zXwQ~3Ry@UZ9{En9ynq==6B10N$LupOT0qD1@j$s1oC3VaQnjdjj~pYg1n3`X z+vM&gV0j`>n;=Q+;QoeCE(Ax)?Q1ft^nm6aP4bN{Mj}jd6*Rq|Pcx9zoI!Z!Lg#9J z^H2*0Wd2PUJ3wtvkO)J4CmU%~0uMw}fWUjfh`rudIaqfsz^rY~LDfbWep77~mPVEx zcFY{(dm2AB)}e^sWbMSs)O7}yWoR^eF!83)NgQuy2pDJ6p^&)B+zOPv`Pz`>`W9DZ zerMpJRAt)=MOvf zPCTd3ca0EIXs@c;m^>}|y9**80=k^Gv8n97-`&fsH_Ifi{JeS!}0 zbFg;jA`enbVT>|=6JYu3|8bajs@@r@l$g@l^tZY{!$b^UsFPLaO5!5nnAmMN< z)JGz3L|H@@9*$XK5~)wT1ESbJ+{_U)k*DX0J?_&T>eR2Rep+ zZq{G=8*kRXXs_Ry29ckEbbVS#v!?g-K~z%*oCUvvGx}o(oX`iIbbV?_^``gP;P;Dk zKeobObkVys5h~#!IF=8k2?#0w^cvtQ(V3taMn@W9U}`1_p$X7mOvq84+bdYiXhhE24=CgDVM{ zs*LIh<0+!;!yLpNKxov_*6E_X(lYe??N*Qva|pZh+!B2!G%S2nSf43%UCTv5Wx&eiPrX}OcBU-p*MCj#*_*~`LdKMYhBwr*Z@JVSl| zY|7&O4*K&TlcC+_R^g(Z&c7KmPaPI<{azrT*pB#Yy4`Z$1N(noePrf%>x|t~6?qOGH%7 ziufgQk&})U8>22&|lXY3RGys&+jsEY&jRbnoyPeN!?rQ!!mCnVt1R6Xcb!}!hG z=Qn5Gw;59NL0DQ~O{8p(4F{y7&$Bc2wt4x}DB%S+$1{_BT4@3K%yellXOf1uQ#Mip z`Z3+;#D@T}gslU2x<0euY1&n67bAQ2h_qnc4Mp)m+U6@Pg|T%RP{MrNT> zKqe3%1R`9BB8%P(6llRN7g?qerC6;Q8M2r%{ZLeoLY_hFQ(e$6VkWIMRj9Adye`73 z=7}*ef-XjoRXm{qt@q8!&!&-DFWkALaaH(Uy`hR+SJ`D(%p$NCgy{R{7vWK_9W&UU_is!LO5|QT2dtk$;O`x5`~Llrx=pQ_>AK zWz~;%j5)+$ovL?((J?}<_(iZh??v7}#z8YcQHEt-S>IE{wpZw#dL_^?k&#pLlco@o zqQD0<`k0pHb&B>55_+t{|CDfXXtYmztRfTK%=c7*!vF6`_zg*B7gK2JZVvTg1jA6Z zFXMVN$X9v1ota&bG{`IsLmjQyR}=}h+p;@C7tlZ^s0nm?ujx1pLteL{#ggXnadKpe zIEH5zyQjq-blRW^+Nb4&MBt?l@5Tv-M0cNR+(xti&qJnmtOUamb=1QS*PrnkRfw_3 z+j_jGgp5(9evryQ&FGvx#oP;-3DuMnMCe(C5MLGEU)8fxP;pHqh7_1LAj+eNg=1m` z;koGiir5J2=^g_(VKw@1BRKg7)32Uu31!BZJC)>F z*i=~PHB{5#tq0#`TB+=b;4$I5I@KBhPx+wPACIZWPP4X><~%=!bcZAcv(WO`!TiG@ z%chMeb^*FIi+dgmyHpHTS6;{hA24H!Y0&IG_MY*{+zJ zu*+cVGa5m#4M<-qUu!HrtbAMA(3@+2=5@kQ+FC=4Mqi}Y1~8)Zd8zk9)&4A96@}_w z6rjNW;R~R`Qe2=T7kfFPL1{CyLR-Gl{BWXaRud~;)8W}3;)XF2Jr+pPGwgI8vXDm` z8f!5|@)HM_nKTQ^JKmq?gHXyIWofN?D4Y*$ib_IRdO$yl*p|KuSHmSG#kJ0fwh?fR znvL$=O4q)~R@!HLPAvt*;jFNyQL4%mlNOoK8SMWv&WYgrk!Ak`KMqs0tDV8hb7_lX zh%!6!7)R#iUhyW5v(Y7@pbyAxtpG{p04Fsi#55kNsM>Uw=f2u^e`LEak{`A-uWDb) zW|a5SCF5kUuN+Q%xj5Yig51`fB9Guk2F-!d8F8r0o&?hkk@pQAAj!w%sP5;JQxB!- zxhKdQY2M~bKFtZ^M5+Y3tNS;lfNY}{9Z1j@cG~I5Fe_-ww>}_b1jiWgPi(L10`dON z)u%Tb^Py-F!S6d5JvmfjD?(C>=)H>Lo4+T--s#a;-%A^pJ(k%Awx?S+H_)SVtL(C^*x=U25k{ncT3{j` ziab#UchT3mAf5vl@)Uw*vf{r}X^!wZJ%QF~V}{~+@{u;~P;0ETX>H!u6Pu4VreLHF zfg&9Gj5zapHfL4OZ3-}9lzCZfS(q7>yM0Sb5UWKrHw87)b!N!h5L$I7KO6* z`pjK}PDw}@z!<-&2hrTL*p zUFIwqN-)t{z31*uz0nT7wkcwvg|`GM1_SyCj($ z@Cn+tk3N#@TguW9#f3lhim~5tiD@QtR+lACCX}uQHi1kKGU1Vp4Fxu7cA`X47i+35 zIA?HF%$9=+*Xe~5DMRI;(m$$<*O(ThOmJQyyhPR8#%sb$UJ_m@Z+k5ewJY`f(55nU z+P+s>#y+sSfl|RMkolkioM`}Bb6+UJ7iMK!#nS#K>mi}ShCrhbq&npZPyj1* z075{$zdieBoBo1Om=q1V<`;2+5mF8~HJ}be4^bPU3%2F_S2A2e$Y{=Pj<&#Eh%l%R zZe9Xr{I{R9T} z-ND4L`dLt1a4YOzD9;CF!aNJ92L;lW2_N)BaWWk8L-E8}__y=PfBmK*6j~?ae>HRS zTfdwRhP;o8qVJ@0n3^l}A|C@T2r8EeKhj=NT1aIv34Of$ArTSDX0py}kDP7WgEobX!?<8mOJ%1$Qz z$I=Ws`?2unzUTxUH!z60-l33nrGucWn)`t=${u^Np`08g-+=?Mh?atfj$)FZ81YiX z*D`z$A{(O~ve1^{_jlnf!5(O#EgJVwVJ*ELDPb+ADLm-J*HWbj9gwL!dn>fA{o92X z(+HL_YWxncs{9Vk|6!zRXPKDZU>P*M{338t>A4?X@%bdQC9jI0L;gRg|A(@wouCAD ziDgjC@(X&Xdulbrt=X!bXJUb}i@;COWzg5YD##mGwcCG4e+w=Gy|u^fVD1m4A#Mc7 zyUB;GQ{lF3rQNI}>{VGDR;kvbkkYBKx&9{A!rZ4yL)f(>j1OiZA*zQ4HqEJ^H<=1g z`3tV>DnAE?-myRbh~d1JsI>tB4BEJ7~tNo*J>-%v#&kgw?#2(8<{NYuT_EVB;$W zCABo`VLF=}c=-T&5ot8lc2J4zzu6+D(}JcWQNeTuTKhjp$JDd|lS=gev%mgl)i?zC zso&rv>(Dv^XeAng`0xhaa7*h5tHXO$$*bwjM$X%cjqK--7Q`<_{?xE@ddrTTDDg8} z%aWbIhbKEW{aay;GhfRhfdGojA!-9cDV3*t)d~Y)zuthd^H9|i$O_GTVPRj6)H4m0 z%D#k&Co$E+U0K%WV>TTFG3Tdcm)5?(lZ$N2ydR|LRl-v64-V}L`|`_Yy%xz`5zY(7 zdE;-jE-67tnP>NNjBhCVMZqGSXT~#*Z)@$s{DRbv_!enFDfbKi1*>4oSfjaZak^U8 z1Iyk-N!*x?mhlRO>{xM^{P~m{PUUmpVsd_qmZI=jIIm#~g?Mbtg+p2|Rnj6mHIrorEcMQmet!z2Jcz`|=jYU0kwm#d*qMr$IuMJ(H zMojiOy9s!&T}=__KOV2eH6ciNu9$nIUTd4e*BD!uu{EjiSiPsf#`ria#`*Anz#n5= zlKEKS=PHL#`0N73v$2e4GKXBgW;q4+F$U)uj!ALd8GWwUE`#U_yWLjv@pWEL$<@T) z2Ug;LyRsJT>o{B%meYP5T+0OYw$DJn|boVZum0}pjkxp$6D z;;m^`oxr*0)h5R4xOF$#(=WT~4!!DdI|45;?9RGUal3Dh&{lEmE;hzaJ8zCvS50zG zHU{Z+c{@w3F%9;-B6T|Tk2&hxxwl;-HoF9l-RfpLgD-h+&OQPeUA_l$^~yV{u}8OM z5#2d2oe`UnU)gjV7#~6UhOfrbvvRo?ucm#c1-5qa20^Ec_{Q)igQxHLhUxYS0mO)` z<&O@m{iIVc1eP-=N+&)u?3fQ&&d~Af_9uMK;PIRq_s1tZeS=&58d=17rUJvs)ATv^ zb|(3q$>kZ{9#x(3-zPGB+kE|EC&CCBw~ljsQ>;_|j+O7s>DdyUgI`JJnWG+U@AL*{ zKZDNI4=RtV6DcTLi!7z4j)v-yWT(Y(l6ROl|mqOl`w>R7ph^wtb-s<-{M$XtxHLm%D8=T8xJHG!<>qk`m*3YsB)*J zXnQ&pLFw~0eW@ppwk-)7b13>!oR2ix1MUkn`f`^Z@V5PtoWqxA#QF+49-y{|a!+)- z3f}L55mUbTxt+JoQ*<5v0jQauux$gFnvg-Wbissea2kjyjFhodK!4gKoQ1|F&b&N3 zi2jI6YztWnVFunLkD?S&(iKOKcVj(~K&_+ortXMpJNZWdVA}Fc%VFILOevu`WSWY> zc~zT4Y5?^iXUf&sYYghF4YoA^!`bGnz*^F+1J`mYTe(qErf(C~;dqsV_Bpy8X$@v$ zDnYSm58mxT-ZOQ-u$zP2mbCoh>7MAD)9oj$+rV{$f^tvBsUMQ5zWkTRNtXbF!r(8Q z-r4#^!FM3v341ZwQS%oUAArKB`BNA>J{OwKY4d8WiLw4B{?m+uYZi`irV}F7HjRM= zf4C;aP4*6h>}_~OsK+3t%J5_UAe6_?ygM(Dx!+}Pk`4W^2HG0 z0?jOHR(iY=4Pp7p z7rogX^u^S!3g=HE!+t(hvVS4nPioUU=fv^uaY7f>$Q5yHTR2;m{1`FxjkiPBkspxx z-uHh=Wot~9cKFBp`Q};r#$UT42kn%x76kU_3kM zaNeg<;|6m0EY`oVH@l3X8GvRvU!fI61at9!`PHLtAgHq?fW_A8g zuI%v0F*-YB`u1Khvwb^&mw1MRy8Aly(E<#q-M;LmnOtNjorI5&p;$EGvMV-XRSZTg z>x;t;PE~$mxK~ZtKkV*GiHuS55&HsdUQZu={5~IEgZG}AYkh@h#m(-n2em#A@JM@h z@^bSlfI&w2gwGjP$Rgc-3zI_n;42YbTgZq}9&xBQ5nXeLnGr+!pafA)CyoRGlzG^D z)iPSJ>ek%xZy- zeWhyz%W{WxX?5T(MiIgwk@R4vQ6H#h8oLGYeVAu!@#ZutZzM;qa*MH5A$Ucv`k!}#Bj;^|`04IeLsZVbDw zfH+Io{j61H%T?!^wg}W#+>S;!(k%z(EqiG96}tnOmTb?0wweiGkA!RWU=R5ER0Ici zvlVnt_)W5@%WCh%Yf~%QBHf`rB~SS1B?VJN*x^$|^N^Jk3YL}~4g7b8kd@_`LUihq zKW@&FB(OKYZ*?%Whz}wxEdnt;QqO78O;E3mBnjc~_ZPmk#aP#Q5h=tgH?-;pO-Fe` zH>FbYPXQ^yTl@J&WpEkPy1l`2Op7N}fRBLWsvfCV`FA8{$>zDBYP!qL6?IvUQ!^loD%BVa}3fY;-1aOnPb{+ISGMYQPZxZK6v<>h$nC@wA? z`hgBztRisgE{|O3hq0DLK1q(hcx-F&lK5v(mxY0thn9v;Y@H80VF(Jt(X73W{tEbQ z>L2G?pLXAB{4c*1Lb6nFAiDKw0S2RE$L+0Nwve7C%NNW|AAcvF-@Ksh&T~xLek*XYp%FF+#CXt zMz0!HJpTw?x&GS0$QQi*B%pLW@Eg(}&5r?nJUjF)Nm>9IOh=p4^Pmr!->;bKT$id$ zJ*Ls5C@$MPLSv-Uow2(2W*eRby(G6ew$wGSKLBVRS2<{h>d&z@hgYq1aFl&NPxTgk z)P!7t)1tGwrwd!EbB>yP=!7ME1>X!p;dU+3MOq%;3UEF7$oZ+nuwpyD2&Jen_G!QZ z4wWsQ5{0<%xU-NYY>HbPRj868gS*fIj;KC5H znKoJqEuNwmaA`ZKv?E$dBm$LhK+<`HG79*yG-S$;5cXb#GRAZ=CAas3p@)+?d>$Vw z4j_;fz`+#09KcP9bE0ZKh1`#)1$we&ks&sd(bQ8qi{inODPMuUv-Rv1HC!hnpahe%_w=OB$j~@GCQ}vLT z=G^e9j%~3QUHfbi7*UyQ%b^3dC4q0tf}e~#R8G9h2+qntCQJ6T>?ivo;IV^Q2{{u9 zH^WOcDo;OBHJl_;u%wFhS=)xoSaJd>2USuwjZ@|k zqLl+TOY;3tFJctu+M%AoD9m+B_Ww|bhZ4N_u5o6lW}hHW4wZ4f(;O|NC5Y+Cx;$xB zRQO^8mh8&@zDr(I?h5!kaaqLsqQkDLEwcN->QvN`^!CI`K6xFL%eJtKrXhn;bJ8En+?3EG{-Ef9V&yo>u~_pIdQBfq0@R@j-W;eE%O7y6I$NLs(OhgQ(+>}+1vs?34` zo>-D@$$z7zO1T7$Nf&eA(mN{xdPd(cEb69XSn}%<&)hijZ$aOCnu?-!mDSI_Xq@91 z`z?v@cU8o^BMzR^h!A*K3rBkhtWaPbVvrB4_`SQqhDhfTRyhfKActBl`K{4l?`_G3 z-27i{VZ*132y2}Lxj}{oSqO6RVej$ChKTq-)nLPG{&Bboeqw~@E%~P-!a4*YA1Dy` z>H-^nn1^HMAoKzqtg;ZeMuxVHL*DQ5bN}N=GY`+oL6G}(|AB=-7a8hahkPK%|Je>U z6jemHa~k}LPw=e@Y< za2fMnAZ)N35qgN-OqeQ0gTOb*fbc{My&IB}Cef{XS1s%*epEcX?NX+m5c#Te2jruc zt-t4=)vYH!;MQ|g)2;i$Q%U>fTfMuJ*b#diakcy2)$RVTi(Qw`HnyI}HkGdeE=iF0 z5JM32g(6LnF0945ne!45Zd?PjizUUOUCP9dHf|Fk2}aDHOH!oo!nY{jXwmayvJ)T8{Op@wQC{y{9PLMj z&2v&EFZ5=(N?`31{UX}3lltG!JWKQ0mNR`Ek2d0i>&(PPYn3mJ!{iMyem(%4@JU`KLjai6AJC(>> zGG0EmcY*nD`}N;TV;I5}Bl?m|q4Huz=0git##CG5s%=R%_GFXRdq-rK2UXx?Ho!9h zg@$t$%P`_8nKo*}yOmn@)F{~C&?o(Y$oXr>h-lJHsFMF>Tv4EMxmloc>u5{tf^uTe z?*(Vf>h_$}>M$a}_4<(1?1#M#`L6;}uxQ5H3)W&N(7M3G{~YsZ&<`qaz5f#mIvh7T z+aH*zYa*Z)oEwX#4JTq30%{ios>VN`Qol#3emCqB#SQlB{*Zl0l551$e{Tp zr4gE9io?aE3bO*O=zRf|f|%lNHIg36rO<(-gti}o4j2~tBA4*C(S4ay;_D_Y1)QH2 zN&7t;&G^tifWZB;CXJYq^x)Tzb_rx3-#>t;vnm;%sE5_OOI9Zucxl=_$rA&-qU@ek z$FNH(W5jD|mt8n0dXRbO)jt$d%slv>Fou!0kYR)@+00uq4O{G7^3W;M+FN3HL=A<$ zryyg5!6F=q7-hTOFwvxEVey2~#x{N$wy9l#HO>_;`}Wr(u)eQzFbB#sg`1eegx+3q@5){>zA*4MGUyXAzhB+& zrwDW4@32Atp8xCTsAv+J;+t18k0OzuTbfnX;aJx+Pr({6q z0~P3jo`k*Zy`D2WbK4fmZDid(Mj|S#Loc!+4FTzjf4UKo>PNmxl^Sj>>u|?~@LRr0 z|FGrrHHQUlOhAiEGZd${pMYW_ql{_c-^=nf0-x?|S z{eG(Dv<&r+94XmHHOE-_-$;l9dVWAG2XzzmBeu5Ov&h&1!gDAK$Lu? zD$|mg5vR_beEKKBKSEpWCY2tX#P&!ZPsldJoo*Ni{z{$PfHQJv`cL7&5)74Q-7G%b zqb|u-^~(r&IWa2^?1Oou0iIDh59O-GS>?(mlG-4lLU^24JqR@BiSwPlc3ef6M@A)X zEQrkP#2+3LT4e+ZN#+RvLW2e(2E2+I$NrdLT@pc&Bt|)2*}CmRvWQd0+!I4vwD?1! zf$6X?YNAdos-P>0i!JGo^n}nobG|G+FLYCgBJkkTyta+1 zp^vE$t4F=8_;jB%;j&;(p<>{(yL`NOPNR8aBZwQ2qE zlF=45Y;Q2l+C>2VZ_nFfX4p!q&y=K$F1|UC%ylZHjXBz-RwnA4!bD_T7}-XE6a56q z70O?(hNvsCLt{UgP;EZ;kClXA9(PLk#rm}V?IO#9zeGzcFfkdxnU+Bk?pPn($(diw zGRqA{tGGRRww27wk#PnWpNPTok5+`f$pLQ^Y=(d5yrr?zB<2S0V{SlD0WpD>NE-4y zXF`0a4%KlZp5;zg@~H@DKDPtErhmi}(~u+uF3VG55PhV)2`Gm!Sxqzb4o{AX6-wh2 z%P0k{ZV_ErOjLz6F2W-^3Io*>OWu^u?FJ1JzKLMojD(7g>>D0AMSSFX`M{(plO`pw z{Z#SC>gDn0C6S~o&dWbKo7K6oV$Q(e0b=d8s;V2mcyIcl4M%Ec#spQJkP6v}D_=}5 z>VmxK0Xd5f=Aw^64kA~_2hye;!t)qDh348m%x#*G8gpNXzkH)C+mx=CQ8pTtOIuW% zG@*f%J#Fp-Y-?|oI|jAaN`?`3YR)R%Tj5B0MFDGAs)h`*D?B*~ZFoo6%k*NH2Ak(u zv$282!L6 z8ZI?8L$1J+Qr74VYhG~BqVhd%AX0yI2Nnt^K)QB7ERoy~kaD{yjaODSLT|Pcs=^vn z|H2tHX!|7f^oQcb!zkrP9dtdi&o0l_o0MO5fsq4MO=-M|kV>=1v0b6~eWjsh{C zJD+-^4!}Br*g7S5T~_IHd}0;kUD8Y-*9lo=NIJ;)rV08wqn#gbyx1%-P0Sbdfm&(_519pg3Ycmzymh$V}ht^xWiI z#C)aGTX#p4%xbfXjegW>+y|o5ElVvMZu_I;-8(YgWQ<7AR|M|^c5&@Cee%1{`!5+_ zR*&4wZ*g?2xB2uIN#G}+Ff2o0;z*DqB?f{~jXepfXdEU+l`_R}2r4{t+|idyA}Yzl zabaIZ1#*Z&ExEsPl06ex^0wRzZb4AC$HGBXkncLwmtGwVlU;M zmPW#9JjZc7Myl<{>h43 z4u|ph?%Dydp~+$cZZb~0OYA2~ng@NzElZvI4X#y0lXi#A5BKNye`LMHYF(lTuwTF0 z;Qs$rME|d>_g{IgTf@tHX*uO5cbxcfEKZH!%s=i9E-AR2KORIN7YRZDA{Y!MhnR1m z&)AfcX@9?Jvl6voxK?#@L-RhM|KcW9N4(f#c)P{?x^}yzvufM5W_$Z)L$kHE=5I;k zu9F{Ah6E#`Jm1IY)YfkIX|7l9uG5WOuGfo|-0wE^T6E%hqj8TuE2e z)?rRAqj}@!%F^fOd2x12GPKsP0}8JF00>gp?Rd#G*Jms)0ME(}hE*#MIRZ#%Vogk| z1IxKUf&M4LNt0`^YY52WtYB`%^Sm)7icE-5g+28}fEvAFU$TC+W6h0Z2+^{Y;C;np zQN@rhe53@set!^RpkUJkPfme-b43*k;-TXb>8QD){6hS8197%xqmoPD{J3wUai*>E z;IA2CO zgRoBbASY6!t1?>&(!ROacE*%7CMQc%`!76x1gg3pKc1BVcaTqe1EA|X9X+eqmp$%) z)9vOp4(=)AoDgXH0{qh=X?p|L2418FF2RISWpWd`-9^*Uocv5`a&kA> zoQqKuY2-Gg&A}s`6tQaqqm*y#`3x&$(*=uZ(cRFE4QmzQ3lCf7C5sm^!&(&(UsF}B zKR9gJKI8Uc%*hv$@f&t#&I(RP4dwX_UEhPQbJl!Y=}8Xqvcz8UhojumP4rks-cJYK z{pr;66Y`FoXyhu8Bb3X6&?RtR$3SYQw9rxrOQR*ua^!fWE^Za`sxB@gEHoge zr?65_DZ&=A^Hy?YI2wWHriGq7JiHgE^E!)al+2Y>yH*%$ zhOy61jrFA!s47Bx)vLL5+493OWw-#pghsFs0s|d_U)9LXCiTm_sx#OQN0sY|A1uu3!qI-ON!y=OHEM(4z3L@8!EX zkuQjZd5^5WpI+>lcXpdyvkI8-#vRFJXFQ|bjF_>w+!%P?ij!hB&NESRtVx8lD7~n` zS1-yo-F;WN@Y-g#Npvk>fY*B5i8H`e6itRH7z)w4jUsQ4e9iw2&#Avl2O2W)4tZNHxId$< zTV?hF@+%PtN}SZs)E}`MB12lh4wq-#y9zzZ!1tR$1>x~Sq#$M{vobO;H}i% zmNp!u{p#a3mxC{}fhJ3=2sj70V+*ks@h9CF^%R3q5ktV8_7hGlavciYyAyAf_NbqW zxBOWt5hu#B*lBhiH`km|6t8a!qj4@%j<8dymVs@yqpF5%ki+pz{l3tL8Ljs;r?(AjmZ4$Zj7zFR^EB6g{<V+?Kf4d%MqMEnzzu9NpiXZz9BVpT#QmjTx=`y4!cT9^HueWv3q{`4>B8-vXG{ zFbX+hPFL);_45_Clmb`E)nVW!`nplgyi)6x=(V|_43JfhcR+YndEo|v(;IK z?w$AsVI)kk^rcvfYugRgdU`(z9wjz)M%@_&j~T$7N4M^~abQ28ZcdS?wbaa_<7v{P z#?1j?h_MuHvS1G7A&u48WzcX$Y^|)K$u7Z|aX&{;-MCc8K!PjRenN$BM(Hd2AF`Z@ zQg@S3UC{#vEw8xB2kzA?LM4k-Wy?~KaTQ9C(+Z!RsCr3Avxbe-kR+3W#^*`4WD%o^ zMvUoA-Ib13YLsoQRGai}M??iORHs*%9++a1rX!?`8tYS-4)K#7f_SW6=>(6!bC0_7 zAfZX?jN4Jsie(xz_fc;_w7XK`w0lbq`h2w^qzu}DL=jnW-R~VYkHz{iq#CE$casY0 z#hG_DYyZvUn*0{;Ij`O(^XDTdjqzM*z5DDe6(a{m3o>S$R5GNlv~kM;Tc2bB99WSF z>zu5k6qZ5lm6-mO_Y#5Hr5BYA_zBM`LpfTRf8IVp4j)RSL3Z!F97kqYZz@bbNu2i( z84Y`(o*0d?0g|xQ8K7k2pp6TSmTmqZYCDg-NDc794C<3EyPG?94Es^W?e1irep>-rzUiJZFzm-GDot=j@zk3N-h?keH=lK zPW@RS(qQM=a&pfu(uHJWO;yxd+A!<{tN<01e4Z`-m!A|p+8>Ho9gNX+MW$P#ZTIZK z6k6VG*t(vZkz)PvcGOc9(5-+0eg!&GFfaOwAs2{YpY(-=sB^Mda_n^zjLtV3Bb1hX zZRdO#gZ1f*6gZ+Oehz(6Cd4BH&PslgkU2!7Y{852OrlmyhBgIrQS?J3%Ka6!eT(05 zNmXgRPC>}CZpF~ao+YJ4 zh(KRKPWDWRK0Qz4efLidIYiKy6j`)Qi!D*o6;w1|+eKpp|JjI~5?{Wve}U4Tw=-)k zsDqZRz7p}dBVT#SKew>vj!YeiFtUFoAa!z7Sa#699{4A)z+1S)5-I!3Xqb#-0oxrK ze4MRqQat!B9Vm$=OfW*EhLN8E^mdvT%diO1x+^qz0?v(m(;qj@K(jl7b$ypZ1(qfG00b9PP|B2i4Kpl)JAbJ zIC@VGZEcWGU!$)sCOS9>5D`r)EOO7jSaPS7;^BF%qRTJ;Sb}Gw@~ffRnObDyX=?yB#K3LSi3@=Yiwv8KM8- zJUPV9_KUQ*lVRT1g4o}LI3#vG)bNJcHMsCbZ$0o3-FdMG@3FIPjp;ls%Eh4^uZpOc zA5BQ}B1%!&X@<8a3-?(H$eZp%fI;&EWN0UfBGs9iR<$MQ!5svq%-+W3C z$oXzYdJNn~FF!_H!o}!nTaXjO(CM(WF7J&XBl#={x7=%5{YEJ9==oFZz{LbM5mRI|7 zFM-=WOtXD!JAN?gdqQ%@^#0VVA?Mrg`5+}aR%cJ>JI(H%!~YI;vfmm##0I%$$6)F^ zd-VZ)@JjR#{;`MNF;(soI`YUWf27SFqTn9P_mBml3x}qY_^pe`rh9mMsYp(uNSqYB zicFCzcBIWgjC@?LO%V6^P=ttMmTODG`^ohkl<-DuqFDU&kd68(_lYB-xWM;i6M5TPHtfH;+a+zl`V0Xk`qCSik zssmk=mv9J`6vcq<7Mkr?61;&k98*UawTrH>n^qb!bNW;6c zAbX?!){9GCdI$dIpC^nihkErYX5Oo#xfsDl-69T%6uXbKHCo|wMZg{4k@QW(Jm#ZB zUc9?p4THAyd1tCv+4Z?N_P@oHzD=T_@nNhfQ7es0dB9U>PQ7SwWC-J7Y*nM>=~wJ& zFoEd+?$NC{+}~cy;*Cw|)frxr&0i@4(Cu;2LhF>4zRCPj2KjXMvvJ5*e^%GFQh6R< zK_qyj zQNhHVFbx{TewDuwQZq2cRS~0{5sQ}RbOjuk(KpIrX^u!5)m3vL+Q01EN8RMF?G!Z@ zsIitDbGQ(=D?@?h?Phc6X=Xo~IQ_gelAL;ny5==EpFJ(VyVUpsGLzWry-d7Lvc;Ml zd=u4yG^WeIlE<*JXDhjdYKN#b!mxG->5@oC-odm9r`HIy>|?Z}R3ZCQRj=Y*uh6jB z?iT=ZEln?L5p|Nui6eJiL-&Q=$>d3WS&2iPjbVsgYNTT>j3T!2lI;qW$D|pvT$g{^ zMQT&DOv=0!UB_*-*U8u>uRnL_@{+iY2;Z8$b#9W89n-p~-6pFZG(4nk65Nh-SCu{b zScKp*yS#$fIilDr*pH0B;u7e^d|jQCjKQ7eXQj6!F;~)T2d+~YdoN|AP zsZ4dJ6J={Pn@2sC&d{fZ|4d*lLeYM#&2o<{8$M{3{e<+3Y1Pj8_3xGW)H0%OQPnkG za8=nU(3(A)P$#ipA1qr>>;HLFZ4jI)569SJBoYY+Vp0sY&r0|sObDgDXuc?uOc8`? zN@jC?rCcq(h)e~QGZ`CwWLDQ+(U$K=cmZu4Rr6E+Cr{h#U4Hhz=#KG+U6pu*U%#kH z|Bdco{#Uv~+Rn(pTGhbO!obkl#7W7{@qZYP9Cb)95@!&+b1S7=F#gye*hLI_F_WcpqZ4Pood6sc+ufmCmD%jyQK)Tjz z>9$ZW;31_gbQ@}YO+HAjvotS2P1&T6R(s0S>d?|t=DY;EyFJY^<>*&1AulBYvS|bY zx}MBPq;~exGPJOYlOxT^S5)S!+Y*ynm=eWHmpFd+ch;u!#{0I?MGgI_zzTT$>~~1p zRE!X>uHuaHBum%Q*)RwIMo_xghyVE z$hy*gKrSF`AR6MV92SUd z-mjcyu91SW2oH}&In^hT!kk9#W^;AAduAStOL!5Y9FqGT`z|Fqhe8>_ka4Wb?4c^; z7U|Nb3ovK#9&|{Qx1ciKHVM2fX(=U#!y^73zXR0V?MR!4t6`vA>kD#eo7FiORqCut zX3025U>QBXAE-Md`IrwuO?1!3ix>C3W?^Eia|2sN`B@j7dncT>3AO7rvkkV570?P>LnrBTH|6WG|6WY@y)~v`9CFCQ^b~Y{ZJ+? z;?+v{`9+I1f6TkkWu6pusL{+4h z?rqLvF4tPsad9Xp7bBtT+NDbwdEes}CufDkPzT2UJ(IMgrV$n(GeR%`mNBET7o#PH8 zU{;yAIZfaGFh9W{?ompSte+BqDl3BTiU)wR{V&}!?o`~Dhrrrbz%g(S&S(A3=L z524xiDsxvYr>@P0o!ihSNqx2ZYZz+)m2Nj*^fiSn}eCvbnJ) z*B#kFG+;LyVa;b}6ddb28h(Awy^dwl`brW!)Oo*+33Dki5ZuF6S+ zy%Nl%EL*&WN`)?K2O;7-dK!*r-u5Z03;AJpYiZ-aC!)TG6KR;tx#i`L#@Z7IRri#BW_Knj2#(=t`JUh3D@>lpbRa`AUsg^lYZCg-jQ zx@s;)wHCVpdefnTu__c#N2HGC(4tJ4$w(gpJsm7Gxq>4yu~ZX!jbeeTqIpqXD{!F0C z3|nwPaM4c98xGK(fIASReS|Men9L?J-*E=H#N1oy`fFhQ1Zy-&Pm9?fYxG+$Kb(J! zBk+cyS7jc4Z*D1qm?xz$%oUjnSbPua=}_{j;Hk=<#ShFRY54hbJ?gfx+E?z)k?Lzl z=2rMj3fiu;;v2?AB@}B){z+}*^uwZT|M4rC)~6?JjCBT*<1QoLvLNqx+9JR|j&(7< z5OHXPdS1EN_D@(=iYdv+d33f#WT(8c_Bf)WI&(YQT{CyxywuPN1QMB3PfF?zYiv{M ze)YT?m{|-SGrJQv6UVM$VKx0;LmKYLv2xb}qPo254L~wDZf}A?ViD!E)8dCmc&uAe zv&w;ss3smSr>q#&G_f9Gm!3g$%w4AjK_lEQp^@SO%hDw56ZuwC5pE~7f%%C`_8X}I zpvwiQa%6;#vs9s#Dmj_&s-5q?^VGqk;nx9#=#l~K@Rrhp^1!?&K+;xeW7@tOq7fVQ zf5*vB@j`hghhB+MtU_^@!d#TJVWtP&%A?9a&7`YJ>0`!LlXUgeNmrE!u#qoXTFk*t zG;3lVDV8o%qOYO45Cz@t%_;)%cX(rME&#Ys|A?`g4sK;%x@Vq5y7Xjw(W{h7?bJ|m zes)Vn7#VKIQkj_w79Gh=2~%nJ=l9lWqcV>-?wN~&_ww{$lE@~tP} zRNailL@a>wmYAz<<-|d?BQ!p1{UD;Hpbih_m75kIb{4xg?!P)?$P2c+sX-7x6g{PY zwr_^!4u4+<=!N}TupJ+oiWiaTu7Z5T%n*+k{d5o7fwl*Q z>Rt|Us2AnJ_77g>8j!KUeEIP6(j$381Be?rG3U5N%*G-y=R!KMOH+V^q8uhoz!Euw zvBDC=jRxv7^NaPJ;yY9El{&HMqHl(+-F)LNIp%GnGPjhZ;X|wYjP)PbN8oR!O${-gP!{%U<|0( z!_L;o?Q66T`B52XXn0SN8fRr+YEX2wh#{0fTo#j#nv_1-LV9Htr7!?OA)85g&(K0Y z#uP*O<2LHOx06-h5x3;rl-#2{0#gUA!s-IDQGQFcerh#JR1AD3YtR7Jklq@{9%c=3 zF|Ec1bIfofI5jqOx-JQwgJ`6UyWA$5wXU1^_UAI z0LtXXAz;EFf~%*oh2h?5>8DtAkT%sIvdurfs^4273bZgbU#nn=Q%x}8H@B=8aWk43 zigvbwJp*cTZBtQp1hGyU(U02*hzT#p*ucguRZPx` zZ+qZ1Gjqp@0Y~Xz6gMUS{Vo96_=f(EItduNb$bNn*RQsJ9WF`wuXU1)iL<$#@qblD zDQX(-$g8N|VptkR8mxl)fiTR(Hpm_b>G6%2L}*5V5Mbb)rJ?oF#{y^xBqq#;kTw!Y zq|*x_q%z3j{?x?Igs>p-u+03d3&b+3x0hCWA3i=8QhFP7PrXb=8jU9Qdeuy)xsQ2H zKTbF8t9d>i=TQD$cWRJLOrnJpi=AR|Sszb3q%>@ED6=py#tZGGuG?3$PacF45Lfem zEoCTZa?Cf3*FE=B<&CtakfV@}npyLuAZ#%ju9;(uIUA${>*QvKV~kg(Fp#h~Sw}aF zbQ*42a&&lbZMasOU0Q5oU7T$;wkQk|uXEXwDoR~xq;^A&GgBL0jaU@kRRF|`5=@l) zH+RC?Se+zogE8B_uL`F@(GtGO{aUc-q#p%q3n;dWS8^xBTkZR*LkuB+ls|FLmz?p- zX*Sq)0>rE}51HtTB+?RRrMM0nW45PG0pag z^gi6|kbumDG59kJCGVuBxwZllWpp<}LQTo6f?L^n@ARdSh&x^%Za2D{#66Sp7}Yvi zShX!Dj2PZ6Se1dUEK|jV+VFfU+D*H@b(z6e1SCx=Q1fF`X(D_xLb8@!Kr6q6LJ{(B z#XT}NEb^~NWOIzUK3Z(n{ zJxg-?FNKywSExEqR;!cOeY8RK*LBnepUTOSc-Yd9^@#FOx9dA2`mvpgR-QY zd4A3oj^*=L+TINrkhmKFNiE9Wyq$z zE{hKPzwi*D!9jJvzsf`9v>kVkWhZL2>F`RxC#m;qCD)@>-tIB*>iS>qd=!}*EB7RF z;e=*O#aqPN0D^JzZxq}#51yjV^)BP}@u9Bzfd0gD(;31_QMvQMR%j(!E1?fBr*WQT zgj(m8oJ}&uenP{H4+o6dh=jk~;J=EhDSMAigR$)#XH~B59In6If73XXBPx;QP z^fx#sL0dVz0;#+MwEe6~9S)ib{UT%~{nNVRAnpFEqOp~P_KU$&KL64P5bL8z)qApRp~@)V%MkIB8T z2OvTN=*c?e{lpXAfcsng^7De3>0OBn0-%d*Xo$)L=ycB&#_u9fBK_5SyJ-GPh0&}d zxuvO=ZTUD0f0Bs$M3RU;TB<1<*>svGbas}D?6EDvDXgJuBEq3`@Uj;8ZM0^~EVKfv z$T=1muP_sxAg+6rQu%4GmrL1r(|5?KK9e@jC7_dXq4ZI4v&e+X>gOC)lwO-C(y7zB zzVT#Y`ThuU&1^K21|1nxx_AV5=W zeuu7WcVy;d_%n>*m&&rtV9Ye0OKF!?>Xul$KX_&pZBf#s&h$<^8v^Z~mevyg(15}P zYKNSX+45w|bu=OOnzj$rVuhll$}`kpSXaRXkgF}CE?X@^b%H)&Ts?xNfN4495lwz; zH7VB`ao^(BDN}m1*?JwHFp=+&-vW{J`YcJ~^CY zChhh?bc1xlFn?jH>+fOBz#n;UQpq|5Xbbw{*?r=f8H=Q(ut8?JV zVECY?SQKNPS#JB-r))Btp!7gOyx}Nr+cbLgTLaLFjqoaOTVKar**dctyoMorX==Qn>pq@K%Xj!_=e^dcbR3Vi$=a}wQIB4{a_5uL znxh$CP%ef<-T^3OE!_?N-9^ZMV`aHrJG}W4wOIi+)}VDr%OmYG<%Tw+CS9_DJ8gQf zJ(%1oeRZwn{jEW_YXj?h99nG%IL#qZ{gW+sx_6N@+R(oG(DD4S!m&Y2`66ZTKfR!51Ai@HY4}mxyVq27Lk+&riTV?6Y?Fs-1lOfd}TU4f#;$ zK8wEZJD%b^-~7_P@5(%TZ@nB7Tgc?&`No}OcTPQSKHfa@w0~w(+cRyKQ0JFg79U5S>RvI8$!2=pTZpXmhzj7_@b@N>FrnuRH^m3x&28Qvq4_lz9^(0wYxQ>Zh`gK0|zmzY2Y%Z_m`xu z5d;MS61C+{K!w=lce2$NZ}2Q!s_1*qIzVc(LOf$>+@QoLhwcOaK{AAaa-SLx_78_# zlAA1=SEkHia*A-SoaHo>C2<1E(%jUZLWf7w20;)ka-Jhbppa`XSEnvM`Gr+4DX2+B zzb9!{&^t_K?1Q?JFQOm9u?rE@O0EYqc@iR)Uk1=ab5vW9j&XrKVT@Q(zwI9mk<33n z+50j_y;HhrY9s=}*4YBkc%$-TT&tK15&Ori5kJmT$g@adXABh=YIn*<#S-`K7Y`e< zkHxB3!%scWP zz)1YcTLS$=kYXsf-zcc$R77gk{CLNpfi;bhl^KwX9Z}R+E~b6co5Ieh_Loj+Mpdim z*)0J^H<_rBl7W#*B>K%AILMgdyeB0WTP#N5zBsDXxDcJ}SZx-Ecb1uIr9pQxE#PtL z5%4SaAl;dZ@D|5O5j%|=Slr4ysib!d6NO*YEHps3v_Mp;DCoRm3PONZ#@LAB?b)w~&>Q&52wGw6N1E+)?NPSCC6#uxrYUG0+brEH`e4{E#L_MqWOh!b+nm?x28M8?BZ{WBlt$yr z^Un`1=gIi{u3tDZ#!2QY2c55tHDc|M)WjN|+kAe9u*dQmPi(KpKNp0n)Yw<}QPxj-IjPYAd*ta_7&mQUpLy4qMnIJ3i-J*WBmT-M?uX^Q+F#%arI5s8 zGWgvG_>gOh@U+xV8yWUloBA)T&+Df050VvblpcSV>(BDpL*Ubs`-`#_{|Bu)fK9Ki z=8k`-xz-rT`d^3aTapeL1B==(Bu>@SdDPqdrdT91T@of)sd7vb zartOrsHrv^g+V1rkgEc>#v5MB=A5X!af!sdK^s-5ymWGoh%NU#gJw7I;lWwOw-Ea* zrwuiu5SU0g3XwD9LAKfFGW6SGs2#R8vjXV-qc+ZnAr*oWNJ(LwSV{f6?LeF9vWYdu zYY4WQxVqa3qci!4a9D(NCEPbus>a=pEYpKb%+DB`#+kRW$siT|0rP#4%k)?XMal)m zAM4V1B+rjI1;xL6l9tD>x4@pXxWwPe88S$o5}>kNq|Ej<>w zUWK_2&hMCYMuMc9BFW{nhj*md+1)1)ntr-zFXASog>1eYbI_sKWt@5wbRGr-GgqInk=(Kh%lTiLvVo*&IwY;`kO+g zsj%B!lGcwkh3{TN_BJ5?o|dX;^m6?G`u)`&EQZi-uIwJnCyQ=>XpaIv>JZ;kIbk)4 zwKtoGSyp>^vGKPosznF~YCvoHHgxr8RIziw?E{E&hRZ!Gp^ibxsm6M*;*u-!l;TQK zHHL&^A0bCR1SAQnrn5ibCHfCX-^FIpO+-$029Kq`y<4-yO%q*FrlI76yX|+3GwQN? z-H~XegZC;ss-jTvLTh|fsf`broYdFF01RKk=;{gf4@+J8Zn8G?a_qgOcg`?H9jI+O zoPHc^+mWWbN6+--(w7cqU&y%LV4+^guU^MRkDm>i>5pBeH9!5JuY0&@FWV~C9*K=a znhgxwG@U^WVpEUZ&78p{A2VO#?PH)2Q?ybCYTj9p5u71>5tPNZ%x~&jsQh^7P@mTKkVcFwt-b=-U+z#K5 zVg63px*ylJs1RJWY1wFI4xZJ>thM?yWpZeuF?qM-d;5hl;i@e59x4Eg5!Dmp0wKhk zG5VW(=b{2eWrAKcHMGG|0K=+;j8LA4yBrx+odhQ2&A?`}f9pE9aTC&nEkRd4y))t| za9~TO7#;P*c6hps7}K2Z*(IOK;?yRj@8dra9%g-We<5^#2Qv@mpWHu-PfOjXNCNd@KGb>W1VtV|7pxx}B`kd9wC-%yLn<~O!X#R7JT ze{OM^g(U6sQLv`6FiotBw?badIsM_?0q~SKmwMR!G^U@wU*8X*uU8&3jg91JSZ-~1 z?XKqThS$StZ!gbpdmuN^KO9_`~+N}7PV=`z9s=XO`Ee|}g zMr8+}qYDHee{Enl_V!TCsjM}Coj_2cX(I@POGrxX^Db@ zxyDjK*Wln}%&xIB@>zlknou@JW0ebb$3k5(?ggRwgA)S#f)qEe^3SCV0HMw}M*_1W ze&nGzaqgJ5$^&MU!P6&NrX&KarcE55OQyZQk*-s>F=J$(M93!Cho{<7ZmcPc*415q zoSLlM75eBBqG9ydiRxQ3mNi>u#`+Apst-wU5X+Tr*1ah+0Hk51PA!9`nME`Tz|-al zvNb@RD-`DqHcq+^h5O}~6({!EqYT3TeA5}+%>Iv}%B~TIjHINjS1e^6=N4@Sd#ExK zcP~Wm+pfVqCfjE=kRh0#!Yy&POAX;#+o=iC91FpC@ZYn-blr+T{X_5kAaFN*;7uo` zr4MMGe@mh;pNE<)UeHQQje%x`@jv8_NNbSbgGzplW1^QkcnN}5B7ttUG|p%$rwna2 z`Sw3s6EfAi=D4(qaOef3s%JFVIb{hQdsnT-xNXJ573ytqkouH_(i z&zm=IrAjV{dd!Sxu=32-8VK7?`QjE~Z2FdL2V#x9ZJE??xD`$AxYu5Q)Lyg&T3Nfd z?4hi_%0J4A!+QI}lfdzIEYI?CVIghRcweNJ&~jO+r$ZScN#rIaj>Sv&n&iP2(-VE! z-Z|_YVh7!n_nkRb1A(WnN8R zaAfKkCv1&Xr<_guTD6YlGVtL#PJ+yRA|G~yP_k{qrAtO;uS0tCC+Hz9QL0PtE|5bh z4=-+59ug$AT^lgf*4^_=_F8OGy46;`^ibvC>olDWbp+)y&=D`D;ibaux%5L5&YncBgX znronURf0iCSYBmb#CWV1TB;#r$bko{g0fyH_JbpMUnDOa7b%$wO)YUKHR}Cj*pGmt zT>}i83&4m8Y<*Y|gn-1`Q6cc5zm^Hv!t$hP`mxz}Eqw?K?f zG3$77N=`q58`BP)Ip>DDj&E4E51=d2eId6NRTf%d?BdTbEo&05Xs1&asBSui-(MD+ zQriG^B0@<_zY*$E=;%*Fk~}D>l^2n3@2a}rAYq&-K71y&!+f^vWkb)S?VYyWCG?tG z(}TTIf=qcFYKBR%PBpLWg%63uhPy>By4s@@9Z|DyA3%F-Zhnv8nP}^YdO{xx;Zs0) z^f)}_l`eOO-oACEIZqTG8tJ-tkK-%a8sM;ie#RrZ=))Uz1#!2f&UV7#bP5HYsKY>6 z&Mm7|gJ2s$eLZktIroy+T(pHnOfw5?txIjf?k#=gaKIvKzs?5n&Cw+)8a8ZXDx!+* zyno6_;d-T|oHt$D1h}BrWn$-+&HzeJgZxckG^#>L7_ZN7G6;Q)x9ON*+!*0V~Y4o@AX`NyHBA3_&vwA z=2yo@<9XGkh~D%w)wPOZ&}AbY$e8PzF4(=v7CZrde58pi#Jm{ZESP~$I{cMBb}FEI zbmiQZ&|2qo-Qtw=&yqSo3$^*%Dy+yMAqF^R@X!iI%Dr~V)PV5_=;ot~Sq8~!a8Z;l zVn!Ms^5wvV+R}$sF)Fl(qWa&KL_r)R@JFAv3nM7(i_zw9XmITIs_BR6YmAW5ikBt) z<;#gIPTb@DYJ%)AgFlJJ8v_?Tpkj%REjkQ=`iPA*L2(M%prqdL2!}1>If~IDC#=K1J5gPLMM)jB5oqyTl((z#h`_Fc|R50i#O*L zm6R$Jb+%g}*Ryz#A5>dJtT=Dz0Vv)B>ekXeH(4kq`1_5;!$;^8{1KA0ybH{tOtA#IPOs#SF)GLCj1>7DOQFZ zm=2M+S14EE^*jV>a(N7zdGy?f8i$$K5yyL;f>Q8VVtksE<^!g|#8BVeK$EF2$YEd! zppFY|*c7_72J+1~yn0zlU z9L2qC4E3?}u&3k!e-3pYQ?GHsYn{TV$f6f&McFla&b-F=Eg`3{A|QBMy{5BJpb_Ar zB`(Y$mJnv?V2JlEn2YvZ@5*YF;~GRuZe-y->r#b@y5orsO&cL1F^)$jcpobM9RDMQ;$&y-@>gq+pH0(j zWh*;G1$Z7b@?xzzO_Z00p5#4Xt!lca@yd=dNXVph<`C-plp5MjQ@8pbTX;p`>%vIB zethHX-y0MQnq*!ZxSY(6CexkV9*=A7_y8vN83tfBEQ-&EskZN1Jcx=rFO(N-TWuy} zc$(QbN$H$cSh;cxv@m9zqC`jb2lSyAK2RS?Krl?dSABVu^sQvdqC=x3u_>-diFoiX z*QHiDyDd^JGh6xg-0x^AQprfqB5qv1udMfOSt2bGY<4I}t3-1kBI>uhw~g^|sF?mW zH^mY|#yAx0Yx&ToB&(82WUPp2kbr~d0Y?9Kn2GBYbU_H0nqm8=t7k%VGxnU>L1zl3 zR?Tuq80+SFr)F~4{j>o17%_&BT?7#O0KYDp0TcDb_kGxT=ssbzr!YZ#CXi!$pTJ!c zLjc|BMV(e@hsz=r7oO4@IPv*=Ft!CJV8S;Wxywl2s0C;-kVKRT7ERQhGV8UN6&0E2 zJ?d!+_P{&n!l^CF5T#FXq;D5fGO0+4s|TtR8wF<;wx_Z!jqMZwKxZ5dhIZGAbaInI zKynv6)C03EM($YzUrMbj3Hu~!%~Vuyh-DwUN30Gu0uCoX-xy3rLS?Gf`C&oFv*FCB z9ZT=wpOB+ENFi^(6X~R`@XkLEN_Y`O#*v>%6+J_WuK(zaC}q=Qii#^=lhFY$VfOT~ zh=s%CNGYUSTA-;d$;7cl8XvPY$}qSwEZzDlt74bt%t(7<)nNp_!>3H?I%08aGH`G& z200r8Ae1JIexE6zO3g}m5PSa&`CRDpIq6kB*Eft>ld1m5crIj?No*S_0`vq&CI)Xy zfScnD6e-tOBqt{nf4N6EI~U=H6Y^-J+Z}{2WZZJvIgxy+&thiMzzL2ln47&Lnp}J{ zop}aJILd|TvpT`#UQ+DW1hUoP9-=x|k0UBt_}LL$d397)lF>w4!&rG}GzW=cBXN0Z z{A!MdfO*Nih9u??DrZ*W>dH3JG=d`gEwXF>Zy>g>T&T;g_IJAOj9ITy;k9jzoEY0C z^T}m0sRp~`=e(jfxP|POdpX9)Vwm4a3fXa=qW07YsfepX6Xw7fJcFXL95q}qf57}H zqd1W4EIfb6Xg=8g)^$`ia5OV<{?&C<){;XKKz=XzZmU+I((8!QydJ5j0AJ*u&k`h4 zLE4fOHymn{u4})+vQd3@C1;Y z5(u2~nqOr%kPjS96rJ0(aLqt{^y4^auop)z30Y79VrcIN?Lg)ed|Fn%8)q%Wbahlo zvXiEVxT0>YJy9Y$BE~tp4`~OU5pkw|gUY750eY-b>!5^#vbKnFA3G_!;L^Tl$ePnh z;y2Uk9g!g9C{j@-3Y^^BlHd=bUXuqJ$c1jM?)z<_DsA6Y5*8 zw8IE6KANhay;q}r>w#uErZ^dsxVP_FJEH5XhHQ=S9QwM)1KE{e%MyHLO$pj)rSdGg z@b!MievS+HBqOX;G$oYU^K%m~iCy9z?T?Bv@e_Eogz_tJUy_|*v!<9{nKMKcUSaSF zbuzoE)Y!;25OJxmjNzCL@@N#Hb1SLFp>5t#rK9U)*NV#>RmmE#DWBBnzu`Mmh}FU1 z^))AEM0mo|OUUc}kmuE?kVu=-waisSS4vB{p*N+c$_#OcrfzLIve$_~9EMrHlRP6e z3_9(j*>olMF0l5;;u17GhA&sqTTFcoxu$#C%0-ZbcmaU=q`n^WBPRTTnJTVd#Iliv zpMIGPorouuqz_&7S{yOTV2J=-D1eWALc&I8A-^@gzQT1Pf41i?rA&*0suh% ztt65ER+4@;oU_%Syp@)4-eohGZNQt9A(1sf1)wD)$=H)rD_QbXE&5O=AZ5E{=ne%v zl@gH+Y@L+IWV4-&YCE1OC4Ho;ZA;8iOyHbtr#ZG>$~UrXr+Ky}%3r2APHn^$srzf& zk8|$$cAj&uKHff#H%qrY-SPbPh2wkYZiMA@x~w8GWOS@umoUb*JIuRU*=)(q)+cof zYiFe02|&hM(Kn(x(hR;YRx2#)+CMHbWRf_l=RFZmv?xXciTDrUd`2EP3&bJ;x?D7+ zqNuZf;G$c>|7v1YoE`;94OxY&*T$1vcb*e6^sEEKbmzZval+J{CfbI}M4|`M54sB} zXLDVMz2HMG6#OVIL=7Wt>do7m8At{SXPG7a=!QxG|N=x+K91nH0johtDcR^M&${%m!q&*T{ z5(BzZD~}U?%#8t9SPod7x;_ACA*`9w6p?!1F{_t=9HLOQrb#%@Rgqy@_lC!DXdgsU z(She=gi&G@2%An`lr7^TDZMUQy>ia#N4<8U2M~lI`kT6cIV0wS=Zog_JQbhpJRn~> zQIj=;Qs({^qvq$Vg5#`;;}Zm)V`d&QLVW6X7A8pun#0R;el?jkq6mF!!Im9H3*_iV zGD3x3I~?YG;zC7JP5iogfJ)FGMeIw<%i!1w`kRqXAl!56o2*Vt73oBRT+7e1Ax;vq zEke$P4YSh;B-gmg3ipb-)_LcXkw2o-vM@&VEN;N~rUzbpR>STjT=GEBj?iOllu>eN z%fV6O9Yw1s<`(!$CJ#U|HrVSmWvmkfF6Vabn3Uciv_g00cXtTo*Xv6>4nfa2S88sF zLK>ykHmJU8ZYl(@3z-;u@aKBsI7CwzTQX{)PQC^;M#0*IBAl@~S?-}TJ?kLLzPnB{ zpI@R^?hP4hs)_`bHwqoSx_bLXjykukvLb!%N9%h))RZen~j!w+B4C} zBpP{B1%xbDQmW7Mie8h1xpf+hTRIhfiuRtN;C=Urv2O|ppw~t&W}_C9E7E^ zsDDT4;>GFTj8)(!YH)deuSOo?7%5ZU*_qik@l&D&*=R@$F&W_+x_l%V6q$*sK#7C zx$xe*zbskPn`)H;fvG`)@=-IY1jv-Z!*ro4on+b8N%bS^^x>6eQRm7zqi?V-Fkx=rM;Naz#k(?N8u2TBn_6~4hgho=hV0WmucZoj;fRb6NeNm^evwMQ!Tu!q zOlGEXSDWp-OpfXtqi-OkU<4x*izDpO=52(T|Dc8(p5Lkm^=VD3v|GaH^oKp>Rk7DV z!^cmQWuFd!%~88wPBCHHILS zpjZD6k4g@38p3BFC2{eQeRC^_6=jcgl;W9CxLzz(XAWDCcJ#VrJBAidkN8#w zCZR>+Qlh@0Xn05;eC51hMPFf*!9Vb?MMhIT9(?Y;2qluwM z3Rs!i+4Tvtu7bpAU3vgHbM=UVeslphD2a_t%}%@aPV@HCd8JcaVHulZGYcOFc6AkQ z+DIzy&9zA8?RCu;fODQS%$k!#V4*VTRJxL!faXt;YYDDa<{!@LKv%fFsrR4g434`xpd$IH)E41y()q7bp97U&>&c_c4wpt2PR zPi^9@$2I__yTleo(Zb&OyZ}reojs-~SYH)2rh+`*8GxzsJbzbs*NGoiah&YLS_t^@ z_KiS#j5+hE%@!m#!M+pB9(UKZx$ep-nkBYhr}nf8WAL2Pt(XkG=Dx^52vg5eO}IyT z0PBpWaCM1d*S2EsL|^`0muBbtY}~d=yO_P1YX*t3>dv6~c9i3$qz-h08Qal5D|P<` zu{B&Xy|B-w0LrF>;8nrBo6--xVbV>Zw=JpeEwPA@=TjHHQ(wO%xI3Y}$~Hi_Jycz} z%VBV@M?<_Fy!~i#u)=6%RU*Nr&eJj2Nfpd{csrcGF|PmfMppOhPXf-5b9dDA3XP&nho;xD12^^07P;p&}vvc@`m9&#*)>s9CPn+cLC zhS?5B{T(7u%YmMVQ$J7%zDyDSfHmE>AYF5-ho7Mz%P~pdIVQ$HbJ5oFCrfHDK9iVs zb8v;1LnsbIzYD)-b~w(DP@gF?e#OPzP1_XoDM3YHo;z4tr~50`^?Snwqy_|j?C2Q&M_cQR{1y|5|XD@L4dWx2C* z6NOJ^KNVPA^^jICZ8-80s@R&O^b6LYG{qLP?ph=tT%Iz-cap7#@4M>`WHBY2er#pq zA2)5@i44AxP2JHZ_cp=y`4Vk79hK$bzT_9P5F<BJJ_K~BW)cKDQhngvOri^1`c-Kl!`~q289!Ce zzU6!X|7i`QWBa$ZK>`4<|Mp6f^S9Qpl8K{*fwiiMqmzZ5t%`)OfU)tP#Y~+|Z2oN@ zD{nO2|e*;i8)R`Fe3 z88T#pQu#u4qGeN1g1uyp8ft{a=1liyS+JTktG<9t(*b_BZ$9E?l$;r+Qn#%V;K)@3 z^q1q8ed1O}0;qGm7;6*YF)&ZHiF%is6=ns6 znoWW6EM>PDhgUM?yhRD6b*iM7*eW^zUM#J2xba|zmV~VQG3mDO)uk}hPRFov3&7cXsgWK|LYv2N;CA57)3prNSjw zVGD53b8;12~zsCpE^&@{ck1?SCJPh;S zip#&RVALVpm6uT7?HHNtjt)T+2mna|5$z2a;RzuHfC%I2pkV+_C?b_Z4@fZ5NT#Lv zsYROdTpO3yDchQgoAbj^v?O}L`-&IWhl;D$mkL}gs+&7iwN8DnywWlH!^YKm>>RH; zzi&FtJYR8q@T89H_lBh4|d46oHOV=ZOPu(A>yGnaAV#*7|GV3C>>W=Gbr zB~93Mx7MgLP?d^IpT!3e&F(7WMU*5>4Ncrt&SHE9GT}&-Q%75eM@+jXaN}W0V=Tnf z27+BfvR`5Y%M6eL?TS`3k|9R6s?TH91J}B0-m?82Jj(zF%cXJkO)Q@YB>w0DAFY> zQ!y%njIRo6mZg1$Ka+h)3cY7kJ4aT^V)qzkF^+SNHdk_@E> zBr>_`DunRchnTS=c1*L}$7vS4-!df(8pGq%1eXv(hC_Xa`+T%W{9wmr7oKOC*g_r7kT$(5|w&M z2w5ghU$ih-$d>1GNq604>=jT?y}8dYHDtk&Khe-%Ls-X8x}m6CTVGIw`ua0pSZWg; z^HR`T=Up^%4IsFr+paiN8`q6-0! ztGdj`bwfRWU3iMxJZWTRLTM%a}mDH_(Z~rY&V?8I#4!Sxz&GmJMc6&7V?wM?Z_1*-55H7^1n9 zxW@@kL)Uw@YmMl7EDyXKVhWbeh7<0fUOzARnar%AUoAM$T@n#OG!pgy2r3T!(!+bq z@fq#Al;nn785?Ooh11%6JMa@-WVfe1za_8u)b2r+xtL|5zwUVw_k;eI0DWo_Bdy{s zgL{c*qPL*TcGUz-+!A%h9C3+#)#E(u8eim{g&XjggjMRTD!a5$q}O(+^5H~+%ofRS z9?*Po9FJ{SD%=ap;${t7y|E?$+mX4~w30&$p#(Oynmil5iSCj)g9}aFe8tBzmJqG! z5}N0?wK>&ApF?t`re?C>7gJV0lk&=e zhff~Vf)V^HZ6TkrQ zY6OdJVX|7xKPu1RUiMzn0@pF<0+TuCP?5uJiL_FG-jMw=W>?N;`P?>a6?HXyi(~%@ z`=!31tL?gd08XlJ zHt@&?nBP5slv{!<#}Ju!z}(ZHzbnk)iRar5;+8iCL01Zx0mxBaFl=Yhngx3SBuKue zE*D}!0vR&)95c2;6Cwy1$+LZ(fcrX_K|N@{^cCmCLcbrU4FT!?7?p0pY@DCAOOJP9F!LLoeELj)d?Hty>3|V}jmcD64 zwO#D&HTg!)`A;PKdS|Y_+Z?t+`!O)Pk1HgNnr7Y3B9Xg{y5q9-bS%yVP6I+viuO{8 z8n%`7nk$r3XBk^aZm@u15$~66i*D2~wDq@GL9@bQYFG{x=xZlZMj*bcYid298c*uB^YRp@=G__o;rl%OMEnrqW>!ueU6G;${0 z7l6;$ZU`3oJ4ZK|na?yKTl4LiSc=S@w`?d0N|PtA8CwoGd{?gPzDBc2{0qS6ee?2% zBv0GG+Je#Ua4fHBs^J@&joO^{DW|Tpr%K*q1?>YT9FcPEk4{?VMGM|kIjy;1POk&) z81q4qvr48ScafI1)|$AzU}U#Sgxt9iChL2SH2$VTb+j$Fap6+oY8LqcKr@`sAK$gIo|s*bs(fu5MG-^wQP`(~Vcw$ytlUwqb@2J-AD6_OeYxyQ^+S85w@6Jjf=L zJJ0KGT^L6(WZamtZf)G6xl~5*X3((1YP<(+5CGIYMtLV(>kV|e&N5_&8KF<1{y;no z2?x7o1fVc3fDW|$_)-dE)|DwdG#YITv2H~8`c@HKn$vDOF5 z>tXxpA_~oE0l?p!HTq8y2Xc9p?rlxeL^L;c7W?wi}7@VrrRC>I~08<(qMf zgs2_N0bYQseB5ws*lUU9kq8TLuS45%y51S3OERZhe%~><3&Y~@bJI)ke%|bDUdLr% z|G;G!&h_}rdrVnl2))z6j5jQquIG zJYAS-CzoR!|EoocDOBgkC^P`T3dTP#7O!MsW^3T=;`mFZTC>WA6Osw?hmXyC)2U$o ztROidzF~$gkS?V>$c7|=qGn#xA*FSqO-G#d_+<$VctoByG6G6h&q?5qyy=suAdGYe z@5fljSD(&I=0vF3Pk0R+FWF9?kKJf@)_mXJ?&tutZj$-MhNVTB63v?1cuHkNRj6>H z&E`*|3`E2pit^Vrl(?#jm(nq&p!va_Q@GSU20&9Cw&pLOc`QLSC;U2QDjF+#87l|1 z^SsO%uDgLAN$^-0pQ@xEnLG~$@8bX$uOd*<}B`D{rs!I!eO3HQ7=V>xUFOsL- zIzlOy#6`tyl-LrwJI%FPmDrrUQhU|i9nNz@>|Weyfc$#h*&luLsMG^Y1Qzw%{bq8O zH4M%Z5OY>JweVGyBQb8N{WKTK8WOjLRb{xM zWODw@bPuo{2k!;49P$a*u1yW|a{sWIsYX%Xq|K0JXYjKH*PxZvT7`ucQU{Mt@yGc! zy=cH{bf7d*F@}VfW}ksZ*A_a8xv_`FNf0X^ zd1+`s5LVdm1P)O-u9}QawzLIgLX9h@uUOX0QOqo=s+BB|sk@_i zn!_cQxOgxoYcavby%OGiYa0gq*(Q6j7bXp&{9f0&25fJ!Rx{q3sa4P6xH!lo?Fa81 z&q@6rCCjXe$eM2#!5h2f+y@gi@7dgr`P#I%C~9xIIg19-7M=>V?H%9mI_9XaIu__b zS9G?b_Y}XUk6CLnS%bD7tl#bm`ByvkplPgYwb69*9nH?%5Duy&s#R@QM@pa8M7~pu zlzyFRfHOvXwZfc-v0=~5F%D_Ca!fYfv!vwvv~6I|oXE1jo|iLU=g+k`66;Nqap+jQ zNUcWoVRf*6`03C|Ox1nb)rcbZyL&^fZu6^M7&DuTGq3R7Rm_?X@1Yvds%YB9Q^36E z3OrREqCGK&IC}y#3B>jO9-vlJO3*r?4O~6tEqJ~WT>T^9vIr-hzE9B^AS-Q&i22aIAhQ_w-r88X&eHbN{>gVHga-cgNLBse|kF zomHQ&2tU#se9d43Cs+^RjTZZ`VWFiuX(dLO;-WZlL~C$tT%QgIqFc_dLthE>U_*OE z3t#zek&@!{)^Q%PPsptAm3&)*SSc7GtkBm_^6;*@GamdKI#cRHgRV zr|)cV$vmKBIUuf_lqiV32c+Cdv^+8tVd7=KLKkZqKJXVG>`Rc5Qapz*1+jA}iRB9C zL;2iSIvhnRYXp?fKP#?vj1@$n&8YS4jaArtQfwmcCdm2iuit%b;kdd_w&y4!Bm9#5 zHJeCxyiZA63QlfxOFdj!3-(w_BAYBsBRFa7Rua*bMX~X)$TXZyEq+m(B?;<~JT=U* zop5}}l00=>p$73#QzU6(Nq(C~tX(2W3q_$uS*Tqz#xayUS6R4S73MI|G097~T@T^V zlO-whkbDRJa3g~Jwa%TK2UoH?@>bz=(|d&K*#|+?2VvL-&ZrlwN7U}=GD+##-{{e3 z?ui9;xHkUKapcO@fAO835iix24N|e!omsNijTfS})Y9r=B}vyz34{jd4X*(y4ANMgu5MqCbt%cL~$;6aGwFuGhZ6KYuYPm=iZ zNkxdF2wlqSt3-`;+s(^X1>85ivAfO5_}&s>Y(?*76=tSvSQ`wZdwpDPmz!QQ?PixX zA8%I!WB|twYWP>r(G9K7%F6*X?HVoXQ;t(stURqcHS1F;M0lemOO_IRFa-8TXLDW- z68=Kn?RRq3$0_`p7uKx=_Vy|s;aQzvl5%x*N8FLRJn)ie((%4&YO5 z5_tus@{AZbWTGsvQ|mWj9E|oltwEBU)jXFTDXd*FycjdkhZkaC_c7>(9WSm_mlS!v zj9srLYwbFuL|2V{4Mz*!!K=sygn2@vM>_iq9MBLf!S+ zb`Gh%=x!jP^SW$EJC_qLxN-J((DO{`Aa&N%HQ>f3{LtUcQJ{bgXM&O!?p@jbFH- zRqSnr`KqW+4Muk}LkL2*8vv%ZETSoKgdW8waQjNU+Zq-(gXDtH*crr(F=Rf@)qw#3dOrP5X6b(~v(5(27Df`b7S1vzj%NQ)YL#u2uvCyg(|W8C>x@f5vVcOZ zY7Vdx^q?g3Ye{Nlq6#KQ0I77YY4;m8WM0~W7v{cVyaRn%Pfu`m9**VQk9Fk!{>QE% z)(oNZn%VZsamqQ)a=Oy<{rJJ<_q`W`d87Q`e4dxBigt3n=eq>0ia zBx!)Adw7CMMQN{#r}K4nxATZ#pl+$m<*F09syD7G`~;8?zn8voU1}~Als1XFgB+g? z!?wt){l3plYI4RlBo6@k?zCTHmJ*C1s~12Ny?;AL!jjQP{$mMD;0rFYC`*SRlk^u_ zjscZDZ^{rmVt7$W5opP*Hd*?h`wj^iZg^YhA?N(uA4=!v_qO3lsSG5FvXx;!D0U3x zVcYgIT0|0D2ENiJQ&N`%5nN`VNPWkQX~J^3L_Yl7-^6`+{_Q{-c%BSpfTO$@4$AfjQ*# zaK7;n3WqJXv2r@|EDWDOooW`;x)1wpAe&-*CA0yETSXz#TSV^{5cAjS!9P)W zKnYf`#@p;Cq;T{fIGo=xE@LXtAXd9z=-s_>-Kca)V5c<+mjGIXOB1XYlaY+x6J zOV}7PWpPc&Wc%FTu%-HqF&?O5(3U67D!sn56Q?k&4Mts&Sp z+m{BvKrz@zOAHaevEot-kKOF%qR*LgC7MJ%#$3x=$9zShQ5$7j?BCC2sFE>`F(^Nw z!x_u4<|1S4B<>90I5&WJX^XtN=CC9mDA2*e&6#jKPx!!xxiD*@Jm8Vjg>>-^C$!<` z-XgA0sw$yINL+5tw)?l9vWf15?B_=oJ{Fbn_HRLbTyIm?qKv>-Cx2wy z#Y~n#WetC3gW~IrMQn^nmSwVMAK=bu?a-BT81dk08&A`RL#?|y#UvM2Xs<%Rdu z^C1wUb(SrQ#j+mBp%m4VH-vIvk zwWhM8hmgS^^B`z#XJqxSLg&N`|2MmQM+s2@`Mm}oye5E;l1hd|QGFKW#UNO5d=(nz zG%%*Sg<4Hr$2D^+a_M7Him8N&hneTQ7)y3Hy(u^ez1J0w+00vyQ;*Zt`|HC6mmhWu zI`c@ih3sJ%)*f)JdB@5ubm^SEwcLgqABu~?Vk`9s`?iyI-M{@QD`t2i z)B%7|RAEBlhVY5d0(=-D+P?N<*d$1-MXp)9Hm%m4&WAtZPZ0;~hu9`IYr;93^Wbuv zCHinSqZRuWp!W-y@JI4=@v5=)EO&~~eBmEWJLd}&nw51<@0BAoxWhu2?usvX8 zZxsYZC(C4BF=L@zUY?;ua_^gr%vJ+$vG{aeC`-0XZ50%I;g;$Swb%?eaxc_$?-pFn zqC}skvuL5odF-KDcckfn zW)692pxUE$vQ;%1Zp)9gLNYPKnV)00hdee(vB*?q{mOusppll|GssGb5ko#=)Zao} zxh~kQFY`6tINAUx_9n=6yD*GJ6Lc?Ad1Zn1L$k96f#nVuO2c#dx~s7+MLI$|wuvHu zxRwqjaqsyH+B{ZqSgNBhK&B(^zusjuW`@wN0097m{5G4M=|7d6e>X+^!$ZLm`O`z$ zHohn+zZ1|dg((4K-6)ZQJe(|h|Quo(`n>_4K~B|5O@V99%Mq4 zAy4)r!+k<}Mfy5jRjz}m83+GT=k(}#*?#plx56t2Ky>50ML2 zt|JD`rBfk|J@aP7g|>TZLg2t>Ho}m?5I8E|b5&$mko`>XQuHFaj0V~m<0$r;&o}bQ z0qj^0R7aB~&TO>yIiHS>AxQPE&KJ{rLe3jwh^4h0hyxIgOIB+5^J@7U(u7y2%!HN( zI)IRI`-jyk;jjXIygHfV0Mo3k%Xf_i=;|iJWaaa@Hd^P4k{iV5zxjwE2`#0L} z-1&+Iv?8z`YV=94E5Uu%k%N(MIi@+{ud<^id?i~YPfS=UaYUBmbq4JpE|EFR@dHZG z>6f|)sHmi6Mia_X+jr$cUX))%S54EB3X2JYl_-q#e)uokBT2kz_IVs$(rdF@4&tfa zD^FHPGIDQ)~Ic>|j5fKDtk;7$INXx%UlM+p)9vQpCS zGSJc{lOLXauQP;2irL1BUVuBK<9Uyg&t1D8!f7Cr!TC+KM%Dm5hV4z>`o5EkaQlGE zKWAWczYjIm=YfYFb&DRkp4T#9-J+%7^LcpXXMA~*Y3#+s;z%QGr28HTJw5o6P7sw& z;~X7zisNyk!&ktVwU;znlODlcT+>!K`j36KCW=6|Q=NHX+U&ivv=JEY7?&=Q6e|jZ zarSax6@@;aV1*N8*~C`C$R@r@g=BfzLNf4Z3uiE?(ZuA-bpPI&Fqf=lj7~>N6*X4a z>n5K|*z^1niK%DU>MKLc4b7R2@*N)@CUz;xG*L+nQKDk%%DFxMnJ87(l6b|G6>*Bf zFPK>aoBV6XOBsBOe;`E6FbbzyT_~TL+eRWbH;R)tiZc#L0F!RO z5YJ6%iFy0WQrJhm#|!$yo6-Brn*jtu2KZUbf&MM#5C9+mKYubJ$^tYJvZ8eFU2KauhLom56ZRzg%nNtsqg^j8D|!Ck4!KmJ4d0|ntH1c*PzWb@a1 zFB%J5Q@elZli!W~=aJt5{Yzo}jso^43NqkdRow4r5r3jJ`TuDDRFA)-$NY)D^uMM5 zzv}f@oY)M4@-=t>04P!b0Ggk0Zhng+VC-sPWb#wj+FzmD%|Xe^{}`JbD*yo9PtYVN zzlF9nu=eo$r6umK;L4Nfi9i0B0LwpiA>&WrRP?_Gx3zQrtEK$U@vmR;eV_)hp#HFh z$IySBmtdyf;S2o<_Iqf#*-lNwKZY#?|F0XL|N73je+zAAY@%f0X(HlkVD0i>w)L;@ zX1n)P9e>PBDcG+g*B1UAyr_kVwejyq9#>k)MMMAqa4GYzaK(Rp4r0H9{(s)UZie|J!SUR}tKe;CgBmBru9W2T>22m=BB6Teqp3KZ;5a)LkpjG+Jk zD8T-(Zvp-fP)h>@3IG5I2mk_&NmKI%7mM%<% zE;24^VRDqY1B`CbvMt=ZZQC|?o4ak>{@S)}+q-Sswr$(C`<-*|{a^m%y?1iX%Bo~0 zYu3ojnpKriHRPp0Kv97pAt8Z)fCPa4{|^)pD3FYZvH+cgtSH0xI1msBkh~PcKlcFv z|IZfr|EDwRKkR=y%LvFyh>9pF)60k^E6B+8Gh%i#togb%WJ1yWHew;ceO>+44{2~d zq_nneP-Izm1^(&Dy(j?6il521V=E0dp@8!Pm29FIG)uHn%}gmWJV!A0XwL+NvIYlWf2zKN-unAi%2TKR(SB*57MX_+n{+vE>bMM?Vt zjEZ1i#g7ZnA6`^B<&j9*X~NlUsUcI z>>SJ*-=8!z05=f@S9Rni;RO)|XS$3C##+V1;%GpgKz>R1Sjr@Oj8#r43M+g}IR|F>q0N_7@ zA^d;C7}ywCxjULW{zrK9{|27GKNlVVhyMl$BgNd;83DYG^$q}{XX_SY&%b+bZ8lyiH2ZTY_%Jr+3=q)gMnwyuG zn|-R*pDL=nPBvdmertduZ+tVkzifG4bDm^>Wv8nB@iZXtQ#w@f4x6 zm9#kpxfCz;j;f(*4*l`Ml*_mK$g#?y2T=W^Rv#5sh~O_py_N&1p^(Z$n5tkTaM5|{S@Nx` z37S&Go9Z?z2Nn$->CykipVAG|SR3>jdf9{WIQAh1P|9tX=A znN@tWJE;<7$Jl01@L(^YHI4e`6JQ}KWd{4>le;)``&qb?)X}Wv(V44nMCWgJisHC_ zsJwZI!5mo$_j%Nj6UYj5L!dB)+d1T{ zy6&-HfWI*!1`YyG*J}T>T(rX{slqXkk!Cr?r!+rzpo~nhBnO z6y-sRV%kouPuFy|b1zj$FF=?;eAK5(X?O`Dfy-Owq=78b`0`zMP&KUOBciUmOmCX? zi!<|Q=MszuCs5U(90>CN3!H`?3AUmIt#KeiBKq5@Hcvoy=i%}I{S8q-S$0SImIX|! zqTs1dv^g^k5JtNN)9`!Ey|dhk5TlWW)iqlHtO}i>ye?&_HQ;F095~2Psy_dWF5gwH zUEg-dg8^skDC>YrZ3bajZ3>>P!>$NJEu2St?{%{Hc+dl|MeAQ-A1%JC`8=RrO4ZJG zIN+QjVMQ8sJ}{>J`ddq=AZ#EMN0?^iK(rc%%U%A_mN@E62C8f0kdCtztL1m>6rGik z>2rNuY^lu%Uck95AIF8NBoM(h`cLVw+95*bVeh*cN+OhsJv9sjj*1zP_A{rDErC%7 z!izy^rp>o5boC>3Zw}Q}xv=x59(ez@O{b3Vjc{4HSSNv={)|@yo zx^>nPrVe|9K#)a}vI^9O+yR!sherSA0S10b(ns~gmD_aW!rZi1u%^(Hm9+NQ3vc&R zgUdZpIDFm<4+i(hA8B?i%oq$($g4d<(q_VHZH%ZgTqg34UoIiseRssqP+wm`cWb)^ zbF6ezO&9%H99&E8ad5Q;B1Zk8wvxFVL8%;vci?-MKBsk|avH!t*%(Ey ze0>|G$df~Nh!nN#WHV|9bp5P9oIN+bZt9(*(1?FfZ(fm39%wyJAdLRbe?o~JH}$0< z{E(#I<9)JtmX|_{opZB)Zw9)8t>c*}!IS5E1>c

TTg z3f8RBLW`_MS#UUzn{=KzkmPXKW5{|ZH7qOQ5-Dg*)!)#vLNY7tIcTD(7?0HuSim$+AD3kzn{6t4e;OA8MF9{x>Oux%Vx#kEiDt(>w5m`R8aI*;Hj9(mHZ?>X z?IZ^)Qp9MePdJtFm|Uq^X=+NF2xZ)ZwS%@a-3JNI=#nLivGPJEk75_{7cIG2vh)Kf z8(#D4npY@HgpP}ISCZ148CUuMVYI=8lyH>>LGmqW0k&A_mutV3GlPxAkS;cZuvAwX zVUmjF>8BHe*?7;&bKcVBh0e9nh#A?VaG#-)YbkXbQk}B8`E%T>>VYPn+OJfG0q=G_ z$mXaIVe_ANq3b0!daBKs7a*{z``>8*R-_#2?u!$4n|bKCFY<^Gj=V z-RB@~gqs*Pw$&93k#Oo_K%Ce1`AE@>b4+)Jc7=1>Q&|9Csl<hrQ?+bM~~^}hPaGX*U$#cM5!m{ z{D7JU7GgRX)dSUQRyvvSzfh4Mt4Ha-x8Lg*|Fk{s73;smDI>kN-o6X#@uiyF4{|bZ z=XXa78@XVrZqc@l$2&EgYV;>`vmwwff2`0YWFPvLpUf784vD$>t&321yl)LCdUAJ` z7M~PVOr+K%6Fui^)BF79KF|Ca2zbj9O!^+L<8yIBAqlK_=%TD&qac7b_V6$hVL@*v zoF1c6labcyGt8pk?ePDr;tQTs!q)j4`Os`AVAiy>Fk5em@E>QM;MA3npzA`g&wT4s zSKq!G>(im&++T&+c3K*XIj)Ls9y}Ln=#M6f3IYeIgmz=(=Nkz)%>20IdN_0Dj&;gG%ZdIsmTTiE-m zgZV|xq7`bWhhC<5-jAhJF%;!zFLp{@G5{W6Xw@^io)Pc);~Q5e}lZQ4xcy zcsu5uR8NEdg0l6bO)|_ZefYZD{iqAlBJOU znZ*ci@xp$?)#(&GU%paM6AIa9=^Zx(-mtvqVvJ`j;fP1xdbBLxjF$gh5ngN0JeFOW zEZ;nvkK8^{NZCiZ57M7wF!XzMTlGQ19Qi%uXzDyFTAtfm{py{g(gMjE%Jmz`{flJX zh0MVNG|qds^8S+-wAPlqID)s0-6OGW7g(5Ao%i49^vR)KtKU9S(;i9SvB{CPB|gDSH$`O$Bcb1`fl>S^v%CdW;( zCfy6c={LBU=_}0X?!1ZVvhZAOS8QZ^hu6hxkxv3q#+KNNr zR>@iQx?UZv6<5@bsi55jiS#sd`LrAfcV|{o*J_HDu!s@dE;jI9>*8kAG`G_xU@y&# zf+l;vty@*HcN0uHxaK8ZmjRAH3sC{`wUqbg+h?mmLXd6;e4 zeZ*D+M6tg7m3b!EA2#mBkPhKrA0A%C0Kc?yGZYx2<|pO$8|X?QYgZ6e&@+N1=BEVh z@tJpMjltsCCEQhk#%xOjp$LN`@|p`;S`1VGG4qnL+;1jPK1Bs3Q^XQ%@6sp zhlH3ZB#hqD3;og;1OOjbyxT_BX>o0NR|_S|Y<|U9!!||?l1mRH(zWwR=)ESgj0Uo- z6`WvId&Vn43SqY%s7kX@Ft;MwQJ5*j)ACTd=0hs#v6=RqVrw+Fkkx~ZJPjB)c8F>R zD%ybryHc-IGd&}nO?CYCJqPa)Pq&ZK^vu@Eejg0fz!i(SrA+m5M7mMuJdGC zV{LlBoY+?gV-Ey8Vn#aZ65;oW-qG=k*=I#<4;0>kV&_KOvcCN(Sm5L=!Lyg3Z{e&V zvlnCs>93S|YW@apX|g_>N!6-Z4)ax^eKb|r^LM~OG(22Fx;FOg!(B0Zt?c!^;DWt&9|(W;BK2m*EZn9TL2tV(?uNz`y~Z1{`do$`P{dT+ z#u%Z@>;&a-UH8r4BJ7{UjNDZhfb`hm4l>~F^n1p*-qMVOy6xOyd}>1Y6nNhG^8wwQ z3Egpfqx#gO?h1Xed!rv;v?4iNy-XI)E52K2AAj%l{jXaq~vc;j)tvd?pGHCjsFN*-c)|V9m|Ig0jmM#Oo1uvLRGLHM zW623!#UZ8*3T3f8vB{BD=6Feac?>qA-_f1+jOpGMkJ2t{fE$!Pr^x|K2Cy*(*kYOE zguWhN*|aoog}}Pj&KS0fE!kCXS2E*T7&XGm5o89H{R1+w6r-JTA^cVvrd{9%!)VzU z^2QfylHc13bwHKV?I&1(X1d`h)!Levcw#Ap%YEjx9bfSGv^A&P?T~qTut`$2ZJ19w zALdl>5Hlw?OKfR058<)Fta&t+Z<#_7&d6oq&tsh=J^s{e=o~KgXc1wmM_K+HXUy?F z2|WbB3F#vyqz5apG{u$U)rT$SJS}57gFy1(3>ni&UQYxDt3fF_{XyMQ(w7Cv=7Xo< zl_x?jd!mUmeaH@7g(TTvpq_xIRlEt!|;&ZiIzxRghK>(NNriQ@P<0?ZTKyJT>F}#5^mGj|~^uioHY$apOd}lvVe- z9lwj?Z#`Q58kKfnD$23rB7ipdcw}cKkK#9 zw~j{+;k$p5{C|$5KWiQbe_q5k$~nMDeb;=!d9O#M`SB=u(0p?42K6Ti3@tV!6!Rl~ zK1Xt*_grL=E;_^YoED<{IPz+KMf5ac_SB~NSa(()&@iPpO^f8PoE%V6jwJmUkF;8{ zo%Q-muzr_wg2J-e;%p29Uoga$m0G2sKQgn&`dCqfzH@_Pl@AcRFbCTbqdAk%6q%7P zP!+6NcWc+?POP3MUjTRB+89fn^4wKJnhO47091ASdbqb~HpI=YD;ymmM}Iu5d*rxQ z!IGs$Hr{s1H%!bl9Oy75WU1_AU946bUl5px>?SdUnd+uNEo-KvY)r(O;Fez&FAM=O zlA=@=o_{%mi~{_r%(42T#4)_eF?gYLixhHke9;(ow>^ShL_g@{XXFh((?j1fBW%t+ zl-SrjtdKV$($w2!K%?z};}q)lf*ys*FqG6V$pFr=Zu{!Y#@Wszfx~w0wN_x6QzElJ zY&iTuy%_@?yK~@rGsv=V^depUXkznN%sze#Zp8gr28f|$*OJ2F)y}+rfME~+i@A=H zdv*E&_zghnp_6bWJIk&&onLOj6?#izw@d0o0&7e^IF-Q^VsBuv>CkwnD~$lKq}**ox~Z zjKk21G(Sv2r_vyvA{1gvZv?sBDY#@q&lCA1Z|(H7d+*OTz*}7Y;;1o|pF9PcBS}fT zKUKg7C_cJyuC4^(r>>)S1xJa29w2!faG#D{ANnK_@ycaniLHs542&h(aEz@hm4%@2 z4j*O@Gk^L>b9U#5#JHU9%`I%n0w&_;d?l$KAKX+K&E8SAZhtsIdQygKita3VZ!Q16 z-pS$k&MwOu- zSHUi}2Mt`5%1vU3Z1MmT>+@QmRH;IjR29c9HxP$+5tCPT!8BS7Qf37xE|tQ8xbevu z8fzrD=qcrWJkep&dFSRUwd9M#z3yk8i6k2KOzp?kc;9$R z2ET>0E+erDLQhtblQX++r*Tg%vc+lvDL7b=W)10zE zs?l#D0KlwJg-XD<>S(rJochsiIy=b^d1B!i3Y=SvWHKh(nXaot*D6A721{1Q3-4Yl zZ?{lI)luQf1Gn5xE4mvOA$L`Tp1$1Alt1HnF{GA!{QUZ-MlfK0DK?4$1mw)|uQUSo z|3D)UaWewgIhotq{I^(8rMw}FqJXLs+AKeASGNgjJA?x;!$J}+6sHY?3eiUumhs~J zi$4)}Ce4f?kn;(a^97XSjfgDc#Z3Mk<*UDG%7t23m2f(cwdR@AwW515x%TjNQo;a4 zUf0WL&P_H&Zw0fpfdm%a0hn?Onr$7LQmfvxa5wKnpT4iAr_MIevet*~+V~_}k+w;0 zB*8%n@o)cxN*|v+fF9XBF1FK#=F@GP=)E2p?)i3O=2k|BRhXckx3p@4%Jb@b!makC zNwKU_u!zZX-mZm>6{ane({Az*_u6uVtw_SzFg}%IkD!sCP3A}4f<`_y6N8aOQm~z{ z!z#{bvhr~d=CnXW3Tdvl)6pgF8hamxz4*yL5jQrPsB?AeL-iML@*gE1Q+$h4=9@%Z zXuCmP_zr1<<^Bo33?IR@_Ha0)%dX`}{+m3ZBExz~-->gsxCu=&-oRANkLLsE)IgT}J$7<;!=6ZIZ zN}Ilv8-%m=Xd_OCe16vFRV^qxW?0`3=o@Z52<7Ir~)aW78tc`V+=iu*u9J2bk0X)T29fuqgnXLl;I;&q*Z09WMT zqpO2p7D__-as~Oyt|i3eo78`BEFk`o6UHd>pz2ktcw_Y{3aJTto#%>l%!_+4C~;NB zNHjY)EzG_;P&~%jAEUWpvfI5epyiaEz-Q5H@NCV_A~&LxL?6l+^A&zZ+3Z{J+jeS_ zh}P{EWzZ`~Ejh@5D%u*PMKWr|%N_RluZ)PmOJk}61_FWw|5q8o_HSf_pn;>gk%WyC zz`@P|;Pn5gXG`ktZrUnnKU=AQ6xjtQNp=f!&1R>Bgl5!mQaBCrys`xenR)(5bY1eA zv!pX=E@lZRLhjLSAv$7Z4Pzv6NpYI0MoKWy!dyz|(lE*}b);m3ucVnB@0+QFG3kW7 zs;|$3N#2`nN8cCPmn_d4o}V)mIbe0aSg_;J{=a~{XXgxtb_GiYC6bGYd;=L8r9g{Z z_WjtTrE;X68Yax8ejS789&|{5{28-_1QX>L>dKnAV(|8+i1G-Qc^RE<`B3ySM9MKk z9T*3&VpHJ;LOp$la`%xcB~RsFnXbVjj-nUiM^C^D^Ja_d8$}9#KP7!<3967ssj@dx zKwt>tv^F_fru^+GYe^7LK%vQuHTdCNQdrLj(WOyqT`cC>{jkF!rKz(oBt}pFqQOh| z8@O;VY3QpO)-oZ^axL*oDMwp8nSL}@m;OO*)GybbF&JTyE&-lv^2w*i&D=<|DiPqc zNY&$KQS8ZWRzL{n7qTZun4BVS1g@@=(qhbv(b%(oWNT(ycEeU%>X`i)9Rky433*Iw>3NG-V#KB90NyQtY+*37sY{r)2W}NBelGWD^EGZ-3 z{&n(WMbc70iRlh~xmk=|Ih>L?BYFgs2<%K;d zLV6={ZzS;LGYJ{t>lkv{mcB>^C~4{O-^aoww1jl9^7$P^h>Gc*G@#JPf3xLl7Xp+* z3){FCxA1r8X+j^Hg$I17JloMqV4_l0&ovv*iD>Db3nPl*VzOP~D~GWhe6u-)1-hKn z=ja(p=FSkFgQn-lQ_Sa@^58i)BH;bY$|9FsK0xs1k6B%u#;cB4EM?+Hbkr6}4)lz{ zB89rwmx{okm>8;tvFyt3H6X=LpQxCpNSpei}VfNFfK=+5!I!`yjZ? zJGfs*k1DEOO3luyn`9fBG~Hwy=rq?U)&*!U1neLm>4!7DGb6B?2F7dJ4qh zLYKDlOS88T9zadw-IJQ~wsP_>3m`^9A?9}8-oZ5yjnM0h@2e9GmvKV3D+8|s4$_gR zOk7R0_S*}-`O`-T!c8jrt#jOq#uF~De>AAs7y=*bulX1C4wl%KM<>Ne_u+9|#_yFg zm@ENc?2!(J>xpca2@A%-Hp}^2d$|hHIls&A5I|$53&VWSN#nAc4Pu3V2aHF)kA0jm zk_%hdh2ziL%G3Tb(Ag-IaXibLNp7>|8E#)!YQfgG zA}{GEt8~}}bQ+!zeW3V97hLd&I**i_Um83hKaYFf&VWvH`XNopYHTOy_C_j3>p?aW zOj@8ecyM=9qbnyj@1qDMw>N{fPNze7)k3s;XBecl*7iD%jL=Bi8QE9G*u>Ufn^k+P=KI z&~29Ae^K5X5anQ-DGm6%S-Y$s$XPCzPzjNhMo|GQ?_9xERDC!L$QJRQ&EdZNLe?6| z5IUn0iYLTa{l*SQ`ucWyZYQLm`dl(5IDK%Sbpq4&m9K|d@zRChWqE3kqTahD!7sTr(ikV>#rC;|@5 z_Ml8bUZ0ujavA^V>9d0YG%;s1Si)z4?Hn1=qAk^mdafKx3E1LNPstMbwlw47FcK7T z;}U4z+_h;YYF7~#=kp3$W@uA+EA>uCtQl#n9 zuJlN`3~>7$1B&GGoEu8-)505=r4F{dP7 zi!u#pvo_57U|2E3SZfu=&7fvzaLE2FPu}2_YH8Gi{Ds}sJZTpIm4c`^9`8HwLWJH1QFW7&e3zUF zk3EujlQ_b@LOp0h7>x~zoghPZlYV&*zRn+E!qL8@H4R|`=dIKW^Ciq@lfvb|rw#+* zJaZvfd%9*QTDk_e`V-ow-=S@3(6SCr>e^(p-}*Ja`waA4oVI;<`G_3-Jw;jaV_)_7 zBC}afFK-kx+SYeSIB`>kBcf-lHPf(loZ|988qg#SMN);9zN|3cuy`&#uc-(@r5^B-Z8i*FB-a(l+!p%{zZZ%_5&Xq;E1##QHUaeRyah8VBRLdr18bwbQ`|t zJykOW(mFs}gOPD0mXh#Rm?(Zl`W`9frywgw^3?r6$&unMxfQlIq1y1+a6f`BlqV9O+k0U z<2I!g);995n;6eP=J14@TxZP75cezp~ga78O*DhdU62g6R?nF{ck#Dr%f@A*R5DTIxt1)blM6QhO z!?5x3CrI&Xb>~L<)yVo~1>Y5WoV3Fj^}UjdR>HMzEDmC16ech&mlnAp$-zEKkt#;5 zd`KcfOPN|qxN08>e`tbvn1`36imp3#W8bq4F80h*!vW;bA>`7c+jVlGAX)DoxF9TP zL8x+3P2D)En3s1FO?((SVMCPObyU-f!9lwEDJ))~GzCkU)sgtwYR`UVzt$0Ghn)u8 zGl$XB6;A)f<&)8+{06I+`MAUfDlr3}o%IKhh9->|oKSIJRCl+Z@ zZ3s=LMYSDqQPHQ{*$!u?>lD`DjFK%9Oky_b0i1q7r{9Y7pClz`a|{0pVdBXJ1)sG) zz>?Bkt!slDF)K2iV8x2)sf`~~@r!kZzdUeze*2>{w>mt?N>Z}%s;ykJllaR@f~?23 zh}J(}#ox|rbcDZ1f!M~IS8es;4Bm+K*O&CQvobGttQo2t@S<4yeCCZBd;5}ReX?*g z+)_A3!u8!nfsI5U?91(~y+h{6PkVRr@Prh7c`TCxjYFA|n!&-m|>d{qN0XTX2c`)Fk?B`p=yZx?#fAdo)LIK>T3mNXb-Ef{M!G!V|b zxiozn=;h_aN0uzvjK#!RNoTKHjMsGv@A&Rp{yGdXQ(W&kEZ?Mq_0+=j#wb~nY26(z zOJB<>+p5H#pSL4+AX9ljSo5Vvr%IRkGMDR=#irR#gsas;6pl=}Yq%<5xh2)>&BK9%@vtwbj7$FGSd>3%T?8eX-0wHJRRg+P;k1ij(aJXs{t4oEU!fecn+jp z>dTDecr3`OxBI*y3g^UtX3J*hj)i*GO2haPtpcZsrVva50-0H{mifa&9I~Z0+NCkG zTZwrDv#P^{F+T(VzwAI5?CNZ;L`ANFvL+eBkFpzDy*uae+wH>@9;X`|!2ycyBYAXF z!c3v^w;bS`rcqQ|QuDHMiN%q-rMy_@cR3u9PlX8!b%?cnWmdPLsUHK&U)_BovNQK&`HVC?r}un0CbVeY;YT4s zc^d1m9oDjY(@!iYEW_1&Yd{@oeQlW1JZZr{mfW&-+*=_eAMt#AD)}nH{((Itn zUcLahp%8d2NynV)rRTrF$rVbn(5Lg}O1>23KFAc`>GGb^v%ON2o&_}gN8bEIY393l zS2MO&G;3GZIccfOt&29a9}!Pp7*Si@_*Db6Qd^t)s}B8*Aq{jt_`8FmcI*483B=lhE=>grxlG zf&53NR#2_PJKp5{`+28HD_l}-{*c~+4{79_LL5>(B9*(qOHN3fv`U3U9uJ=||12t% zKg1C)AV5G_5dW&EF#p>{1>j(AV*bC!2AXhg%1h}!J)WisV~n40!0-X0A`&5kki=Rb zf*?>Vn8+FYpz{HVhvcy2%wv4_Q7-(O;xW#FdJ3TLj;ISx#%KEnpG;xl7PN) zt@etFt*QPY0XHjnj%5BZ0Ff368N@R<(h|mL(2f;f#p(-R8_TXrBHetKF+>j(OWYJ1G z6VgRWo!uKSr@3``Q}H-fZd@peW5~zOPfffvzusiEY`qI1bcas6J{ruTcl4$p;wcf8 zCx2!dtMF6F5*+MFP1& zLAdaHaUMl1YZenYhh=%GP$n8*%D-R>-^w@0Dy#Z~^DZX&`s0KuGs9~ExcE{kS|An2 zc_|-pk&ay)5GPOpfh%k{iQy``ZBEYC5<~UB1^VPu&If|i5&~i3C@`CI^UDWA?ug9B zN}xN@RFYv5_XVZA;(OIJo2+z2H0t*ZD-5vh`|m6Y#=@}T!7a^cyjww;JFU8sP+>(J z=#CtW#&q#NqxH8CW%)!FGK6Vy&1F;7Cw~?$Yr_q&X92p>f=tO3^jhte?e&F?jqR1i z>%k%UlXxLYHdFYBhR=PV_OI#=W3l@LL4h8>QOhyko2vw4E8OCqQ8yQW8jqs+` zM8%Tu48OCvuw!$&#jP)Z4s4?K`F&nDxB|PDVAEjEAwdnzTif0PVZ!u_N*P6Tpaq3h zM`ALymv>X_36YgM)M8439=rq7Us;d{v%=}k?N})i&$HDqsQOkYc5#j=(mv#aH*GeD z)m1NR&INo^8pC5Ie0|j7(sa(&CeR{`NEz12r}7HT`H^0&T*M&&Qb4W0478fO#G;Ah0AqT)fWE7{8?Ae&6XPar(^QS4m2K*Lx+E1jI8xe24 zOP96P6>BY05D9Q~w{Q7vLYH*W$KEEE*UU!JKap3fC&vi{A8XI;LRsoNikaToB+GqI zurzvetNrd#eKY?=p1+LPhvn|=A7;nUU4rP6a45ABWOU}${U@3~m6WQt|2FEu7T-d9 zHXgDCRVC;kJTH6b%J*5ip#&aK%qgT|C4941f5-U3mnE--dPK)V-}Yh>>!3Bgb>5_VQ>wGhEQAzZ|rmJp3Mn5NkBd$P;q&fF2dw}SMCD=h5=g}G9+p! z!$POakrY^>!jik-SvFeMskb@s>OcRaq1%SeUyWttcDvcj{1g*Xe@7C61mZ*&>m*vk zCfPxpr~0%ndH#HW+X*Q|C2qP6H~-d>@_DnK)xg>u%y+NVL8# zT6C@zWt^M6yX|%>ML#Ae84JPw3HF3vI9}#D!R@BDHFE=X{zsO;T&C277$WDZ5S?e$ z8_W0)!FBIn3z-`j7caw5j-W2n%cUob6-P?w#Scb-E@hTa0`D=ei8z{mq=Nc_rtL5~ zxH;p+NXB(Ui!!*O{&E|bu4z-`_7wXr&)L>XT~5}$WAQNZ646CrZbP`?0J}(kN6sRc z^BuFW_xHga1dWP9?X(=MN6%*i^;(F6uXSF`XO7{@)zCUQzCkuP-`$!W+&_Pf82ZE~ zd)+1nsU*FY-J%LWFB5}taXjJ0EG8QQOp&;P4_S+HLU72)`#CC7qjM!qiTPm_Jcrl} z$0Cy)e6ZM?AWm*rI`&+pjnX61uL0)<>qAa{XILFyKg@u0J1>NLnvObS1n9Kn8v&#S zcTdK@Yn1``lbtXH_#=gAN<%+o0p4j(#(g$YTm}fowJZ!vx*1kGI-?==9gOo9l#NHz z+p31Jz8L#HwEIP_VEDzx1m&1k%FO4beM+T(Bs(&nFf=zPnmg>B8~fgC;mCo#c38^D<64=fa}L@4!ADot?hZ z>EbRO7%G!EvxR)L(@S+)2fkwW%|k>>f9LkItbLBb7K;U1nzB| zBjSh>-TKNVv0Yw903BoC7VELKV8SyFth`baiG2b6_>BO@HTDPw-$ASCMRqQvN?c=$zK#F0 z<*}*vqessx3HWG{?RWf&@x}h>Lagjr`%~&85BFCamp1kz1?>Q}%Unj})|P{AE#c`+ zV-QOXH`96ptglXpncnQr5bT_5z&C?wm<`-J zyGdBJl6(EU>`wtTSoyp-Lm{$xx*X&W+^p$`S{TI0IDe%h(=5g zwM#nd##yPXxh}=-@2zRGg{xO~2S&Q4l42`aLI9I%% zs-6S{W(1Ip2qI20EL(JZTQmbt@xKLMBJ~Iuj2^=sLVRMR7bzx4dG)53VGgyuGqOr% zk{+HW4uQR^wM!XNs6752o>h>RvP<}HlLx;$JH&TOL_BT)GP-q&FJ4Ark0_70v`1Mz z>~OZyBWafISW`ANL8MYDx6H4cG}*c8r(&!sNH&f~UAXFXVlMqixDQ8TIO+>xE(=LK zH{!caMz>v!3CIEMDM@F#Vr=vA(XPfQcx%RDDL13Vob~B3R!k&TQmQAT$NRWmcsscj zB?WT2=(KC8U{Wm=6aBQ-DyhNSgOp=}3S=lnD`5Shsd&s?03AZbWOS^aKII|hi9s09 zPX1n)pgDrx|6YM+D#8y|!O~UWY?as(UhBe>a^{m9@h9ZJCxJ*rl>gVq%=|Sfh<#Of zzx>Z65PKS<5;JXo>h02qGgT`M%WUhR*aI6js|Vi-h#gg#m)8!B^FHrld!=AO=B($9 z;VRJLB?xVQT9i>6cfvR{yE1f%^w%kloDHo2tltEVMW{uJA;I;C6GzOj2^e`g)U#BP zb7U$}@+tZaUg%nd;93AGk_14MXtgnEeHPLvv?zfy2Vu?t(Hjr^gaxiatBbNA2RR%^ zALWS*lU>5zP6zUFfk%~e9C|u!l|#6&af~pat}&Bo!uKIOiA97xiHdJj5j~;!RJ3DZ z{jAZ($v6{lO;xOUD*npJxC3|Xmzc|FJf6L=7~UF`*mE%n3FlwPvhnb^@`OTD^9no( z6?@SH{G0qa(SQD8j4#&%qGz?o9R7Px&fjWtzSD(s9GxI!V6lchxvOY)yyI8JWvHd| zN&$hby~c(6VFM6MVCRb+a(HLM6YtXN4Vm&EEd>E<&?B>3gu&Xkj=+Me{9N=o{B<`YQ``5N2xI?|bM?;$$@`ae^yOdW zyBq4ru*R{@woVgH0Lbj^1+)`T5YAt*oIA#j-K1|n=}80=!Cj);5Ip|4mLf0dOivdP zGwow1nonjaPfnCNw`1U-N7&lqB_dRKpTwf`kw;SM#CpTjZTOY|5}P=jS|PSQrnP>$ zPAG<3cl)gPTkd-mheuT`3cW->uYtoO@Y0`0#=a=;E%nHBFQ0A!aNH3qX6WZ#Jk&TW zrQC0h58&@$1aFL#UCz%eGWMLZ3j2g=n#gJkP9J|?RvoSCfo$zi-nLGU9dgf22lVg1 zM$QVLcbwL?=`%8!O!?`{ouNmrqm&Ew`D-3lhFl|!2Nz->Lgu@CSt=16=-KxWP|c|;P|o?0TiZ{^Z4h! z;d!a##e^mWvdt4cPVDx{DQp4agq~n&-q)H?4-zAb^^rm(bO%YH{j1`chTCF$7gk>v z`LfqP9daN^TL`mwuT5}Hoa~EJUec0veiMt=>?I(03kYW1KA!DMK6YH4->RNq-z&qJ z1~to;dr!NrB?I@*fvg7ll{*Mfky&$7-`9WsYqkdbg6LWd4+OM8@vquont!7W7IiSN z{-3ehkS2^*(o*uD6Q7#&F;X5wzc@c3VPq3z5+X2YOh`}`Y-G~3xZJdVG`?68Ci^^o zrIcuB1X?0h4Dw;G;uWug$d!b-D)KAW%QhVr6&u#xmDP)yRn8kW6WN|Uw%yVu!TcCODs zUwiXKM6RiFAi||-JB+S2Q)BP!t=t9O?r5y3Z7Xe3+*|(7^IUVD`2DAYTyVyqnX`82 zwnC$jZ^U!*{Ez~a(n>f1jrIOMOU|8`qLi9{pT*sqFfHJOJ1he5T`8i6EkI$CFGy@= zG!wXX2yVrw5tH)I&(GirE6qFt@k9sIaVX>df`QK!KcR!Iif%b z7Y@ipM9M{I+Og&!s!d9T;?Mo{;EJ5T9G~Lmo|=Bp0S=uW!o<<|9Zb8YO++iFWsvc8 zX$LNS#5G;$Zg!voNk-BiUh5GQ5^cHMC)jan5TNv z6JV!d=-rxso+u|s8>ezkEBF>AfL{TkyA#rGR#@zoMfZv_oC2KjQFfTDekr30Ku*GZ79h#|KQx10M~MxkdLcn z$&7*j17uJ~#|Fl2g6;53E&X3y3(Vj{-je|^ZBsZwKK6HEJWKZxE@EorMI0nq8cBRX zZ-yYWPFr3-`&DX<99qIO!vx~03 z;suDutK%}w@= z-PPCPu_TkMSpa`F&BgYDJ$O(tXF^3J@f{(o7dI2jJqSPF0T}&qki_c)`D`K410CN!mE31HY;4K zlH7zcGQD^a>{q8$>{p8}QaPpEyRtUNu86~`t}gHE=A4ls-i>Y?DJyg>y<1L-`2JM` zv=cAugpe;p#uOEl4ZY9CTPhgw*AQk!a@9RfORzAPg&a((#pPx%)LcSV?;kUxaVnK- z@Cg0%*LnzWAepq@r0KvG`5}Bk_EgwGRXhKyv~P^EY}v9-+qP}nwr$&XWmc-vwr$(C zZL5-%Hok|^J#O9Z?~eCgpR@n$Gsc=L;)GX3%!yqcHO!k$Q`=n^-|GDxj=1U*q~`KN z+|%~;>KTP+YU>7*3q>q6w`HKgOX61%^Blu``3K<4t`@67uy1Xc$E>}fBahAv{6Rqe z-rF?~yD@)%(3wB^>XK~k^O~Ue>5qGtH=67I>!(`NZVB;OZi{2W^|`L@+R*P5OJ%2` z9%E3Ud*Bepk^W_HmC)4>j(`x>U&*U@du;Zh>bnr|u>~&dICfy5&E?tt9TWKMNi~{# zu69Dry$Dgy+|tx99|a-NhFG13f%vy~wqp*FqTIxv<61jjjAjaFJx<5(x`y1HW9|+3 zeKBL_V5~p;zX@??Z%CW(I*L+Wv22!q3vFTAExBgf5egaBK9^WyeTXJ7@&n^P`Mjs& z^s*tHr5AO<`he#mbUPc-|6nx~qfthin2agcj=g4w=AL(Y>EM{RxOnqOxf ze{>1!th|CwcdgL}vMP80{u%kbCJF34&E^!zS|7E_jqsa0iG&?<9obMkf^$r!O4R%E z;t_Nc^6jw8j;ojl3WwrhiBIOsIgjmr?1ykqo9pxYC2m|TC~DBdX}3-0nJx~dRU=tm z@G{!Yk>ZU&e8PU*4>(zi0T^rwixT0slE8JJ#zmus>+<_!Ggs!5d>fOX^FSLTTNr(m zW2rSOuKuinU$!vv(h%_&+sIueyBydwLpSeZ#y4Ty$G6eQmkwpsG&^v#`*6VtFUU5$ zq8c22FhW-MTuV;IyE|jqsniq4obG&2MmN|r#Mpsvo9^@*^$tlV;YsitiKp^6z1NlwJpY;B zj8mc*C5#XE!hk7o*vf^%SsiOgcyuWy%O{gh5AML#C{Aumuh( zw*nwzjT010s!~NnAwW-11*?hY9q^F-c~3HPHj$gs83zy8Cu5i zBpzrxnb7RV#61~Q7rUH=So|!E@=u}uV(0q4YjJZt^k)K@ZFp7LH$%$>J`Q)A2LPH0 zZ2u$RS|@xKynU~`aj-2x_u0@N8KGlMcu@z_mQE)m;L}X43yK`BC}74BrPX0Gkpj-# zd3#K{WwSc*K(da$Mx%Vf4vz?#U4^Cm)3!QfBq?x`hsz4~XzX=#&PJZtRdisJON(Fg ziOAf{j&N|roOQIvIF$tcI&E1QgRnE!VaA#%uUa@AZVM>hNF9$PZd+s>)=@+B&Ia28%<>VSeiwHPhfY92hcjFf zcf~y}ccSjwzn%!9Q(7Y0zY_!=H0S5Oe5g) zBc);vqemVbsS&ZxOE^Rxo}`8>fbFf1Y)&0D&hIeg`uP=&^rL}tiib69;z&{}Hq*$q zgsQA1heHx}>{P`#7A}+IBXd5*#VJjoEPNhKN`78QAQbt?uV2&KiCZiaPhNr;wD__hygIpDde%IJlxHFt_E-`TVhh$(&9W4wMPwHl;g*;sdC|xL(pq_!Rbk7+ z;EC#6Z1)S#10{0i5DJGnOs}eMc-F1lieal4y^l2j8+bsqFBX!B@%nF!Z-6&rt_8O*;iR=#q8>gr)b9@_aljVu7ThPGZET`b@(@%|( zb)*8&`uk+u{NTyf@C&*y&5E#YE*N7GdXuAZ$UyHQ1LsU;_2V8LjhuU@&UQVFOP3^+ zUxA^&fScrBtNGsNL{z#Mv{x_}7NMQ@gW}2bR+&&L7c(LBx#Uy6woG1{!frQQhMki% zhD0fwI9^a-7-PF~D5&UJkuN-Pr;Nue>|{|Kor%(i@`a{UjZtw2dX<#}D0{H|X(5asq}fNRAY@41W>m*~f}DH|UzgHJ>(6qTdNh-!02 zz(SC@yeEx}@!3d1V9IjOKz%vBF*M&?@X>_fAN_bZ5q+ISlki^b={%M=cI?@j8Dai3a#OJ5~{k{%`XmbZ+0T?{Yiu z_Xynn_S`0T`m0;l5V+AQWjM6+Hj@cxM%|4V(1l_=!dAnl@#tAf=CIs{z-&_(QZN?z#bl0Cgtrx(hfyFcx3eQ2K4?MIUSKX7jq?cQA4_1=^BN34_NIt3y3&`gQ9OKtix z673JB#&JC&oY1$+@AuV?5Ic1}p?)d6-pCy>_=$6Vcc--9-x}BUh`FWiRDWUJF7b)l zNaR0oQ|*6KrRK{u+kbUlG<@U>c+z&modzbKq;#Y;=Wqd&JEA9^OQQYa9 zS#rlL&0$c9VT^e7HdFoTtzSC0V39TB)FQ(!i84cOv|&@6F!x+PwTV~j#nEES$WN*l zs>(#Qyjyj#g><$$sWYmIPO_M?2{|@2)xxG;XN{(6Uv;|FRjctGRI|#d1-pb^mQasXMdS7z zsW+P-X*Ms| zZ$cNtbluovDY#M{hoojrt7h^#cWgKNp;nctC7Sg}jscIa#9#3UYWM=dSFF`eH%Nz< z_y+Qsp&v}~MES1O^ES6+6yH~xw7THgn9idmi&sS!qn03EF;=j2IA?Wh!TPWLOuU47 zp(*=VN#?C$?(nEjW}W~u%@=GusDgOyE*+HX$aMDo{-HU@3- zB(5E%g3>si@E~PhQ61lN5=SzK9PtISNPYK*VBr+OJod)x;r_{PMPNfBjw!gw$#{Hs z!^bCAaC6tpvl<&%_6Fp^!{!|~p>r*K)cXP6^F%?cDp54euQ|8WBbHl{v)!q!v0qCU zR<-rk1fjOy7f!v&I?%eu*anQ;cxjB;d#T)rJyxg4ybB1$L9kmkt?N%%*Y5fqJtQ4= zSyvm7u<;48E>7W@^%9tC`VzOHl^7nd1xzR~Wun+p_u)~QU#Ne{VGKQaO^PMiUnm8$ zNKeO2Jw@s=bfoqe9FCgOeK7m0U8PNHP&#AJ5ks{}b?G=}Mf#5DXEt&N6nPT_Zb|dC zz&fqDIn=-9ediT`x%3BXPS99~vtFf3D1u6**w%fCSR?8~|3+-d_+mAOpC&3tBiXo^ z8KKD$?t%uDRZl4VREL&OgZ2Trj*&(rqKM`b#ir*Gt%RMTfGIvnM?pUwXKsi5OHMeI zkgooLtAI*u)Y>sf1}y5H2RQOFfaZ}##@ySG?VP-2O}OtVlOO;*Oz2!=@?L7;9}iB% z+P=o*P;x*7gt~zm6!Y`cOOx@T^r{lXjRym8# zXv9x>-2z?c0LWtetB4xozYtOTAGcA4)O4K}#8A4zwGJ0eU z0f(aHt%5GQQcKIf(7>RonjLOaei)8vyKQEZZ10!x;ajIu@SH%xXYX5Ayf8% zu=F|O7xNX$ORl?yx>WE(@|)YoSl^GX(W`c{lLITHQKsbrN!k!HD-4Uakg^omktoHh zCVDW+$ujgZ<+mV}FHzkk{!hEhh%qxtH{<_;2UiYZ<3kp@-$K_GuK@ z$*o&fi$Rsw>5vmHYZ*x8I#}D=?9)CrgAu>D#eQLCpp9G_WI^dE%D9(#yGbt(wHt;X zI!2NXV3=WEJ`D?04pWBBF@R2BxjpU4?`DjD$Q!o_(_8Fwx(PQ{6NO%TL*9sHw#uy3 zWt@sEkjNsJC8Z?I!B#FjyZI;|PF1Fc8&LKo=i=9<=i%4gDtz^`ht(u!e8u>^vI{J# zfK|sjW&Hqk2J@UM2A^<4z6#T7=yU96u`otO&FUDthYSL6j* zM(;at&6M-`-YbNVeJ_}Lsd#N1lu-L2Q{oeJIYaf-7~@nzd0;+0pBeLwOyNpcRN2o5 zjT4FQlfHG#Q>*uk>{0Si^PZQBNBUm-#|aJ3$*Y%N$sJ)f^tDdm5rRBkHqh&}vOHq; zgbV3l1JI1B7VPUUtzPpVp9=byxPR{g?Dnmn3V{ItPGJA43t;&dx&UQMTL&A{-w&5H zsY80GETMkdo5f3#1qKQDOUz23#%n?lfSD`+2nrAQ2Z;jxNH|0ckz+D6>sM=v&_=CV zSJQR?bj+i&DkKCf6Vk2Ks&Z>+)UwuY(W+=}TF?4C>1MYl!;YPKoM88S+B(YoviWNJ z)%L~n%HeyN1h8Z4ut}KoN71$Au*}>MxSGZedA2k$tJXk3~fv1k4x21fYFH1YD5-hkp zoxY4lY-Wp}?$=(%&wKlT0CmTIb#p6mMWd<;Wjep_HcV6@QNoX_Gh&yi8+i zB~+I~m9S$@f^2g)e*Wqv^Q_Mj<4BB=ufV~|s*VbY5lc8OU4+6=#jcWF3u(`y0owdW zx1e3*Fh;Oy&iZ#_?r=#(CDZhut^<%c{k?YAx)2nZiH)`ErS=DXNOPs@fi1s7h0nqpMRrzZEP7 zN==US*<^|{oozv(|HMH*P(+R*WnowGyw+6dU_`&0lu#9mB&$!oKx=t0Ykmv`zpE3N z*3D(ex34+9FLgv-B7&!4P(})-!*8v1k~``j->sBdP=TjosiP-$#XQ#%MqCya^qQTF zPt(#m?{lg|2%s*irm)tqQkEq`v$M-w!-{5sSkS)^lb|vLxJ<__)^t>p+zj+u*atg< zYHGxo!a+%zl#(B(`)+Y=#S&i0TKqJ~0>2#o(Qv~l-Hlii6ReG-Xtog>pIuK;2sxF{ zHlD0H@BFcPwi4zQJ}oCQB?^y3L>Hu1ELpSv3bUzYgMfprNnMHVE{t{C&nGAWJ5zp?;Z@phF!-p{~?8L#ol$?-^4<_t%bf)6F;Habv)?Wx*bLPOjUzjwiNEM z`6KLlT!Utur+Y3cgCyoo#^iK*8geW`v#OI~v2l6Kd3*~@UepK zU@CG18)fF&uWp*5cUd3lGmnbzzc4m8w6!}`r?-kVE7jI}apyUjr5_$v?Wo$7*)B|A zF|oBv`wiB-jN|tii%pk~nSbGNY&{mfB zAR!P^$k=kjmgV95aNx)x94lJPImvk(&WO>t@czIrMIWB2R>Cb)nY(dOUj_I%n=)N{ z6*?TV&tT6Cc*}0w^a~U(@g!FHshyFAHV?2F%EKoe_dUvRPk3mX}aYvIkf2=r?r z%Jwwmt+$vZPTprkKbki`A-1yz?cf}Ja=q`xV?$eJfpdeI7S_(fSiGN^3dUm6BGedK zOcg17(l$N4^ugfzd&f(28VrkrQw!df)=j=!2VQwU^!Hf9UTZXu$EZso9(UQzh(D|P zEz6XR$oA-eU^;q{P5HBLauDe*Z$M`@rioe`&B6AEEEk>FvTqM^U(_|4sJOfEzKiJF zaqyjH66|DBERjq+>s`dRzZD1#m9&15aZ;CvtqbL|iVb8n&`gjrfi_f;qiR!-Mi3hl1t=wkDT+eTCT{CFVa*K2jb19E=sepXR^o;Q-?ZWtth% z=eSLMQ}EC8PmZ%6(~v>=PC9aaR}q&*aY1K-7X#XaP}4StL_`c&b1$)L{xc51%>s3R zgP5xNq+5nh-0M#K`8@#0oLuh4ZTJ2?(mU20pdiIH|1h`I)uQP|Yig*GFNm;I7!j-S zHLMsx`$OWN&n15-9wY)eUwu_dlTLM7H*cEt`*;%p->5lSj&Ub3*WMFUJDkjFM3~-} zUZOX!H{+W2&&>$R?eCLasmVaz;cy#dLK!lS8IV_ZD87*VzVt^=7p7RxFv^X8`xA?u zrCr9a22E54tdDY9$tJDi_Si0 zt`l4K0H=0Mu7nNQ>V4{CZ3eCjfYYsqqb4%!Schw%^7tb-=h}rYC}wNMj%gZ`gQ0TW zCA$-J-iOOHo&WR>mcxs5$t?oDDI#i`T?=@4s}1W9?AVp0f&yE&nFz#~{L=n$#^v<{ zc6<bNnz&lE*Quxnt|mg4>urj5G3<-WY_rbgUPhAym(GKjJwawGOOWVo-c;f0XCT zF!N%?ZaW(AleaY=SlzA(r@KR&1;6-;EvjTS(cm1f=;3<2&!tD@Bsrn3XI<*dAXImhLiOiPmyNvz@;O5}7&t+~@|Kw}e;(TA^g~`}e!SToSfnsy@ z$b4+l#_0HsEA*O&k1+9D#WaSfKDQ%tm+kR3L3V=Bj*viHE@EH%}jVZ@L()Z#^JOPuB*w{bLT z9j5k-0+?$3p+;n?QPXk>h#}Dn*JN)v?V^{un9nu~Y(4~d(2wE_xPv8s(@JuKwO5_I zJBIA5A;=gI8}gTpd7fiyY0psO%XwQb16QT2|v^`6})`l7ID!Nz^VF+Blxh#U+WjE`K zM={nPyfZ=%MBa^XM+-D zlV2|Fk=8EixK}&wA~nt9kr_L2j0;&WT#BX)Wt78vy8#Y1FyPlyS`ClGc~{v(IF@tt z@}*4PaJ_d8#e|>B!<>d2*Lejmhg!3uLoUV=wZfBA7mQScMo>V{Ra!C#(E85Rgfi8M z(^#ryqcGvvpcQawMS7)D;m`uLr>APKk78fE;%Bxb=w-cWUZmVnjdXKt*}7yU-*PRx z&&cVyg+b<-t)9%@3|zxyaX#qlp?gKt)1ZHh+nX_AF{}M4_x_nHS4ecm-~Xsyo>Lqw zOe$3l%4vlwR?g8$1(;*i$ap+=kjL^U>arvb?+en|@4!#9*+ zV@4l*TYkwPnXR7X@|L$3MVb(`kVJX3bup_-8uW=L%T#}hf4PI{>mS!*#&RW1HDLe% z{Bizj>B9Z5EM063ZETcHT}UKdOq~o}?42b4wv_qj3MNZgS{_*u<1MwRDQ>XQ!wE|nweR6t<$NzV_*%1ps1l?ei;9_$CV6sqYAR{ zeD;X@%mMd_iQmVk6OaL71dO|_xpWIPcm5eKI%-x2>y~rS-y;~lD-I7zQ5j~E9u1)K z9GeYULREI<1~BlqM{h&8-4m_}+S;$}`X-ib*if+{=BZv465Tp_T6#tpJouGF;|z;g zx%nx1J)&gfz!;5Rq%}m-u{JsWo8}=0UKPcqkP`*jTb-CF`I5G}icM8h42|se8X+x{ z3?(1dt8OF>eF!V1NFRk&7HLr=V2qgp@bi)+bNJ#Q#FB+T=TO3n8*DluJ~u)qmbOpZ zL-Ul%TgO-Ft9L%l@?kMY+?KR=Evyd~nBNqY&E2&dq@&KGMGQ(0Cl7Un2(QhddMBuqTWAJtfu}65Ej4XP`)2$2qiOT=_Rlz zDSo&#Iz8`>(g*ONjO&+pPe2#PE58gq4|S^Y2$9lo7m6`OhhucXD+q-q0rvt4PNGj7 z&91UbYlqB)T4;!)4!8rgE+x$tNw9**iYNvfq!_OwVEv=kx>Zgoo+>t(@nbCp;{%ZO z6-x1lpm6b=-|yOzB(Rq(N(Y=sn@Q$%BhFO+bpz1MWG&|>_wZx_@np7CODZWR*@!vI z=X*!(8b=c_-<%ac@_}*TO_G-H0E?Ut5V7DGSpdt>=KWLbxkm=X4p_gkOoA))z*HJO z5>ePlbRJ^Gt0#^-Krx}Au*~Z5t2dVq*s+AaKi2e^Df6)>X$y}<4GUpyQxgg zowui%9)QWcMuEIjyPs39s?Jm2b}G+9R%U%ySX>7fFKB0O?~g(ZHzta6f)G;mbu>$L zS*x+zZ8@-ChX|_c3uS0`>L1k4vVUP&^AV$|t5Hd#25bu0BRK}t**zX05j$9Eu!&c6 zpJMqfRIydBNs@pz>O}~QH%^6|K05;;e?)>Bk+g)Sqhshc;!No0moRjc?*R-mOAu5L z28PaK^aX`57Rf>me#lNB&sq=MUQXw8xCG(oCZXfv=3(OUZj5(pEAMj z46t@3%N{9I)kr+;k-zCS7N=Cfj$CWgk)!i3=$tU>HQ?L{RVcD=p%&O>IDN~o4(YOL zP3{aGe)l_AZmJfp;EF!Yu7jOnZp%+QGMmt4p3!U$L(UZWj?b5(X~znZ4R43_%lPf8 zfae8t_7^8V;8}Ujp;IE`?PxH6qEZB;2C~*sKDDi<2-;|O;nXwD^xF5C?NGkgq zVV9*+d>WbVu_S!3ftcr+BVO`LW!>eJTbxyv9(B$^OyuSdXpAqN} zjH@rnTm9v4*!alLp!2&2SDvAE_OaiEUZ}eV?4R*)-~D@Th_Cj^H%Df8JDRaC*XfSS zAcwOCohuhRtUKo3Nv*hqvLf9sF&t5?Fkh8AODL1bhif#l3;xN=M(qPrJ^~B?Km+lA z5DuzHm}uM8A`T|vh!-NfcUcc)~pkJ%V*X{p>7YMpWq+J?XOwR@F|8E z4#jgM{F`eh*&{7apO1WRncUyre=WrTRQAFKpw4N?+csL~V9LOx>NbgW*03QmPiYYw z1%V&rdDwchU_z${Nc2ZOjL3Yp6M}$g*)`Fbihj_NYfEuOIm}mDmrgKrRqb1k_W)(e0Syuz=j23CAhq3-&5%E}i;gBRHsmm2-8ChOPjkA48yw zwKW+M&ns^ftF$tz8@R=!t>UolI|j^;mnVmDmN!gC7$tfr1Sti0lYs&6)vg%_Z6J&F z+wh^VydmdV3`V^u<%|R}ZHfXTbrI%QkH^oU)JS9v{Dg=60RS7B3LI`5AWp?>;0p7Y z;i}3R93^8m?0A3|@IKuvu&^Di8^aDwlPZ^oE?vulA91rp)1R5Y5%mVLdDzLzVp8q_ z+S!VqgeRoxSt|oJo|X#+NRMlO&TbrXsAgbJc9N7mT5J7FXk5N#AWAGxEFWuF@rj_8(qfj-D66lDe1e59euDF zK5#5GM#}}AUsHXqO`-{53a!ngDDsCk17TmM zqR*vV!mSm?f9!C?gZ>m|HUa6t4oNE_midW3Wyp4|!mbP#2?y=H%2r#poC~h0F zo{5hK0(=p6GP$?OvA$v)ld5*tI&P%IR6wqp9%U;_s=ew3zk30X9NpV9 za_^h%Lq^m*B)H1GAu^JFJ36+oNHbEl11M4kT;0&v>GV5etsso5dW|-FK~2c3v0|?n zEA(UcBM{co^vB;9k6%2fcU98UnY?ZQx?}2N=$#X(8B@V&byYHLv#j5uTjrrB`Z0d} zrDE`tEjf8J@^W(jtx1tBWKv@seF>BPk(j3Ht4&}fiwARL%nQl8u>B5*y@#US&Ku}r zknVJW8)!gZ1IWbI20S}<_*2T@x*0p83-!`NN6WD~rJHc1ita4!2yx<)q5-iH01QB}7eM^(AEkWu*z|Xss z+~_)_zj}#N*Cca27R(*GBmVD{bhL^QLwF7U5Oz?XouoBY0?WD zbVZ@r8U~uLd$dlob$ADf*KY!^rU~!p8LCM6Gq0cyyVVKJG~1pq zk35#Ayd?~cl{$#YI$12XDp*4VhH!0OqhnX0f$BZQ;TLD?V6e4D^Tan~yA#jWkAmx{ zzHfbMile6j{&4(Kfm%w7Dyvph3Q zIMpYR_@aF1ZajyZ_IQ?`YmBEL@T{uZ0@{_Y6>K;lTh4hh<=|2rpZ5@Lx+BFc__%eZ zVdu+4KN)$r_QK^UH>0Cg17s^vrdFeDE5Cz1=3>he4@bDh+XMR~*x3~=Wc=j&#U{dV zvCJk`8C>V=Fyw)c+jPp&k6Be{=>J-&NZ68m{KYs=M43fQTZuQcLWZ4I5>HSFqhTZ4 z7;s`?ux6D1IGufg|~F8N*iPO5|TX9e3+v~(oosGk!2zw8fj5tdH%}q& z)k-1P>cneVgDoSkyCk==4}Sr3S@MkK!%RIXEN!4xy4Irw=Dbr+vagm{6Hl(G&V(Q> zy!A_GGQE|j$V5t!OVttGgmbm4)ci3wd4pQ9j|c3)Gvl<=6lZ6!{=PYzwmGY~9BQ0n zw3tn_*lZa`d=;(@XKTAa4Z^L(Vp_d;!%>_V2lLSmx73N1b#m31vGwh0I*(R@&83;N z)&9_TW2J=>E#-vB39S+Vpj9oiR$U9SHcfP*HV>dUfpfaaqc$lxs^SZwr5W27iWH+<9?#+>So;4Q+;E z&`&w3(VP1gu6l+0cGg+S0sDtXw5Ez3xtd1LU^S{jYd6h?(Q5mKOA*%8an-4x$@=#5 ze}9H0kvFn_@Td@c)D^E^M=HRusfyD*Q4n!V-3kreyVZWsb=Mh=OK0>s=pPI+Y7-IC z8MZE|-N%Z)X?-mYW(j9^UsO346H5)ttt64`{lx&EbIs5?2H(1KK==;m`ixw$lOwsQ z-|G1_etFB@8Ps#|9VZ+0Fe#h$0djiQa;&lOUcv{R?< zmqHu!4z&pBA&`@K)J@GnxU1e{zU$O^V*Iv9(`n9akmd5VOL;Gka-dI1uFLs!0B9<~ z>DQlJyQFPga#^>4(@=l#CK?5Epv}d7Xqac!7BkjWnLa}ivcSG|o2czi_X@H$17z@0 zFLxr8H$#tTg)+Wz`jQ;3REqhoL0ZzQxz9URZ?so6E7lgq9Gy z;qG}FWOyWJy4ArPkZcBUX&_*B{kX!sXL{R=+cW1;(bR$mZI6O(-hXS-Yn6d_{-IN6 zrBP2)34YOmYkl}rhQkxnflA|cP$iRkZ8s8~3EVm{vEd8+?=w2vFzX>(NC1F9-2dw# zj(^VR{=hkuq^v8CEQrFp0FRa&EEJIcT}riZ3r!8B$3GThuuu|IRu)+ixx`f&t%cUj zo$*=zevDn;gE5Z9%IJOx-?Woix1~s`H#Dow`()ess?GiQq=tS6VA42_P}yE3(#vv# zrlV#mzAvkMk4-9uRE_qXXm!;|gl9N;F_P48q|B1zpCf021U8ZogMdyU9IA83=%+&N6k zL4vB4u?Ddl!gcWbuo?`n%|Qv|uo$L$u&Q#NPH9W)R!Oyd)JVrnph8lK)F7TqVLtNF z+8;|990l??21@9&Yh=^-1Z%_5$~d)h=U|3By%=~>(Pe6Gy>Zu^L9T%4l-jjX^K`3v z^K|>hw#}9D$XB^*g%6N@Jfuf6Toy)vvNXA2hUr{r{j~K>y#;Y4O9Rq}9g&#&)r3dD ziXX^iPr$Cc^O|ugw6Ue~LxaAz2~Y25QIqgN+k{x#Gd|)!yZy)KpYRS$$i`&+hHW71 z^F;Uvkn9#37uOUNiq6gp?Mj5R5KEaE$fLIjs8dAtQ$2v3QIr9RqISOS_%XxwY}zb z;j(74ZB>AJ5IM8Q1t1!EWBH{vlJurN*`~({XdNu5tu(+P8iV3shyp%u=>SYM6|Z2w zDMmpZ>hw@;U3VzZLKR-oK$v*AsUjfN;t0(7(EZ!R@ZGYRI{G#q;$?k7=}-7 zvLAcMsi35e_2j5q%9}zv1D*O9P7ea&jQ0s-g!zt|#Pu5@zr`&rBL+OuP9qBvq_Ez= za5?{+$K-50Yqbjqw0}zOV_Z&&Kq)5H-=_PtFzge-=d%-~kA}vWix(c7z7q2Z%~MLW z(f!nk1}7kpLZO_~+da$%L8x^{btdoa*8$U{lURD-C3PUt_h_L-2G!s^bQH_w{_1g* ze=Sst!r8%=CBdHuf^!b;mjzj_uE;IsOT~DahRD)a;hBvOyKBu2{>VH@55EVH;&$r# zLGQXr8W6w>kqk(>q5se&!KG(m#Dj^SqQv2KlcJ%$11!k3rde`4R*D##z&K{2*TmGe zZLvsA)k;ZC#~XUEH#OEfcT&vEe$M{AJl#nrSiVgYjNS*!){E7$>2xS^M~4SOW?rt( z1f=v-(5t)nfqI7PS7TFnOeHh5MT^%Lo7${lm%z$9RdjXkh*x-a9&4t=-m<=9T!qgg z0+`LDKpaB~q%-IULxA!#l=8a~sp$>mYryJ!qlgWxL9A+{Yy1sl-h!+U-rYSrHQ_zV zWjgW>?u>&Q@XJ@}+(H9X#(gt~Yku$y619$T0rR28alCHKiwl`5`-?_mWlrk%uTf{n zW(!hu-91Au9^DJ$4cv2ihSIvX=`XR(RSy_uw#ba(=Yq!PgNL4U65Y!l<9N&TRjh6V zkMGdySK0#-P}msQjE%8d!&r1B*Qryi@>(lqWhq_X?AVz5F#%gV(@S`uIik5y>kU zc|siFQylOYVq1fuP|BYHBOp$-hm|%R;=W_s+(8Y=F_*KWZS+#~knpj!Q;rN2<4_*U zI|Na#?+of_ZT~}amdW~3O%oUZK>9D)v;URLzke9ta3dI@`UOx#PRJQ6)bxaAhhtI@ zDrtZo`S%)ZG#YBxD7;vp;dk=}NtAiOwkMySJ^?t4L{~#BI2WtfPV-o$za5~cYI5Do zY(My*@|OZ4vQ=#ccey0f-}MoYSDy44I)NCos$LZZ`LyuI)I$l|+#hx(2f=_AD+p2; z7Ka>-!bI@@J#VP2vBrUaOL+po-}43tgaYt;?*2<_7xEwS#DDz!Cb9iz(?1G1qx@#Z z3-G@SZ2uYIkDAGdzai`a{x9;+f0g#%Q{D>n&vN=J3%K7bE`a`AvVUcZ^qVae@c&jT z{>mQxH+!4^)&3u~>aRcuegkRv|AYK@)%-KsA4L@CenSHW`FF*Y|BU@dI(qisu*q@$ zV^M}bJNY9WCfjdLIPw2ep8G%h_#-VI}v^v@ptxSPfOn}=|t z|I?oCpB?=%CjNUf%QXFWx61yE{m0eW@6D{o@;{ET|JlhO=d`~Ub)55mug%HDr}`+rbN0|W{H00;;G z0*y&i17h4&Gq3>wxxfJc3jhEBVRLOQH!d?|&9WtEX1mSI%*@Qp%*^&@ z{LIj{+sq7YyUonZZDwX>W@eiIzPU46Y2V(}OI4^;sW_=rR-B9zm8C2P4uK8=0|NsB z0wMzP|7@@z5FiQ?>LLs>ijquUV;~^lAj)#k|MCL?`?rho|0fvzAN#*x1rbFVNeMM| zMg_@`csa*DX5`@OggYY5tU8*qa#<04xC0O{^@O>OefovLxCM95k9S^xg^EhqafYM2 zrIvgY0k4wbEUyU=Nz6NqGD&k8b#&t9(lVnz^I1U7>XJE3hKKCg zoL1-Di9AFwns3a&qz@tW-lmQb{%l&1veKV+07@r$Nj~JjDfagnL7R|?gqZFE*UKzi zl$*2U)*A@c8+GL@T+_5kLJB0#T$nWzQ#q^Z&9|vplGJavZR)9*xJa!BeS=e z*w0|I=i4U>5nYOC#($e$*0U&G7o{(78yrBrPz*!yln%>(AOt;wDYDQ6p5nSscxl&? z{W{xA$ix*1dFH74kp01bVXy7M`s9@4D?q0(cYyY<1=d>fH6j890r>#^PYaClj{zLb z-RvzGY#b~d8O`iXTwVDd9hU{6prF*C3?-qiHa?Q)veY*xIvai6p(G`tdOz}kT_1&& z3kQjJ3*=n`d-^6&mXc7Kt^#BN;F3^Nl>_zSU4;XC>LyT};kJIQwtl&`C{Rd}P&NoA zpL-Jzodbz;`RXW8jFM2CMzu|d2ng~B%snGL;N&UT+SusW+JXE}Q1R4g;8fbFc-66> z{}nDbxVbrT|ID@LKZXm$Kj!FS#q>W3F{FQE98GO4%zz#irvD=}>i-LE;tKd5jX?9? zjv(UT;0QDU+BiCB+PK;P9bNwSn~6!*Q$kZg9{w(2Kwr{0!BnQ9^#qI;Z8_#@BRAywK6$PIFI9$)LNoV$x4h`{)q z*0?6RIPN&_`1vOENJDbc%xn7F5-{q*^4q(VeHBWAs0z8e+!KvgNNZ)plb% zkGLvybSiUIC5m+Mlp3S6*v_DJXx3;FEVIe(tABbD4V4mi>2h2o_5fM88Cf#VoFK~^ zXp#0|;N#$SG__^m3GL7|Da-TF%Ya)B?o~^S1goT3pKgQ zN)WxR{zeHuz0Afs(Xo=-SzwHOeacISX5$_gTHtxhv+pv6vCihw(P}vZ5P_ov`okL3 zpJN2leM5iG_m}DSYAA-FrzEP-wOeW|u@fKanNd2`&`Fn9m}oc?+(&nbXy+WXza3qr z0O{HSBvLzN%#a_73->Um{b5?l=A#rNe)Gh00ro|@RGBx1y?YZc-6GqYDmy51z)S8% z$~q+WQ*Ce(U{P~RM9-0@A#_mwz^tAA3_P;L`C7#dr>LVkC+s$;IPY~~ z?o7|wUm#%!H;Ma8&?j%}9<0p$MGpZk<$?DMJAybj(c*v1keS?tO%nLe#zBQBbIcRXJ6EG5&DC+wTifj-(=|og+ zyFC%jg(BksJ#JtQ$WE?OI=7U}TM$FwxjV3k%s)=>ZBq`Ps{@M^$LR7qi;uciS$MK7 zUUlp*=GyCH>e~0`_4aNUk>58+0LF{7u?XpE6aTEcP2*-K{aQ2s+;T@J zEum663R`M87nPYMyNxgmJ${^nTqTnwrF1F$ixi=eyN$V&_)hZQ>@GmWpJHh}C3WMg z(Af`MHV7+N`OE7lPI2!70zFcJ&sjmxS<;<|!VLI{(cS{@5}eIGFH#xIhW*d3<0o;S z0%tXrHfB`0C&0zUMbvL|EX?$TDB)aVAJ%rV)RXIqv37t;?7Czmt;ujfTITApzSZy^ zi)*%UBM>Y@Zz#=*^_giizaInwzi78euai1p8yp(0%c*zdSE!C*YXX&2L)Xm?Iu9)- zHQ}`-xGS}!68n~gyAR6`wL504`>va@gwPPfgMKQh5^t7CTm}-b20lrH_O+zhaG5A0 zmV_6)!W{bRh=Gn&uHU2Dopvs$N>lXSWksCQRTqMbITCDFHq{-K#6v(sSKM^I-dU3sLhbB)3^42@223j&>(|1hapi z%To~3$@8Bu~2yD|7F2{ zH35>y$_LKvG4eoNOCI)aUS+S2fDurpX>LHhgXh)PBXJLI~{#V-!$gL(W8ENn$$lLA<6Vs*AI@C28({w+!mM(rgk#c zq@%)D06evp^KpiB_o;=~g0rq#gsM*5#sv2{XqoMyI%pZ{X}ZJXbn5z8!fU-fP;Vqb zxlYt9C0d5d>FbuI`^Y>{vD#sKD5NKJ6Nm_>$ykUPKcMtgWPR;4SGC)TVpqXTS`+W4 zJ)|gV+IMM#Y>{OIw+F3Pgr}#Q-2K>Rc|&zbi@wUWNqb~~UtU~ZVsCfG0$514cfQmg zNf(8MU*ydKmNupaQEe_L!G)}lyC$i~^OFUe;)ge(ieU5~qHS2NEEQp1Y^w1lqpGbponrWGTW~1I`%ULFxNX5!`0apJ5ysG$T(aP||Usz084}jUpP^Qtm%T;B~0m z@VbSs5O}A!he#`DvoTY#v3n$^y{@^j^ZHWIk!kp(rKcmhmU+l5HyTyCc2HJm_LX>e zw#*}wp)#VR3fXi;dz$}g<-p^-{rLOhp1o5dt{(hB6!T$=S}87ap~0mvbZN8)X8Z=3 zMQ2>nUS+aGF*2k_{+xa>OHw=OBuxWAX>u&e;ACjy=>kHfioUM8PSEwz<1+i>beSUp z>5Ss17(F52I0 zB9;Gk7g74x_x!eEbV_~lbx-Zy=8R9HO5wQUei~l@flED1dv6uy`?z7xKJl1vYh^U#$Oyhs6CWPsJK+Dqx?SKSUOOyRh@e zeYK;S-g_;K7~C>%ln?H*?81D$Xw1|wR8d|ETt&5HYUo|*EPVK)%ovn?V=eEqtK#}e z9AwK?Kjm;Ui9Otv*epo#y;@&)KvnzE9n}oC@Vo2kG8NYtkyOE&ode;*vy;y`$Wg_m zt&h(+^Op&Bg5ODFI3tP5Gssx$qCoT?Q0!O{?gzIs356>cSPe@Qrnh)?)yfnFHKa^&W|9RE}ynvxabykCjlFX8oXe! z;24TseTN>e+0W8c&mEc?*f=Jil1J@bj4YmehawD$smlnt8T2)~j=rhm^*YVk^lEQ7 zy>G7~0A8KzrkSflM^tO279FnGKu!LCe0At+_8n2_wbtqFhD=xGQn|G09e4ieNL!`v zH>=B1NX==~_D}4Vsnz!n9jrI3rzam*%9%h78@X%19G|}9f=w;SwyCR3qPI#s#FDIK zAHB1TI-LAa2s(`9z#P{db-~ox;tV?UJ#6f6Rb!!R&D43O>&He|H9u9SgHs;AMU$6P zVy*jzK5C$$wOIE>LfSRsj;4MeBK$z^xMf${Wb=*xgxT(oyXF2fci}Lfn(i`iCPR(Hp;m_I38?8@N-$IZ5;3=(EfB0?>7=>=V zlef>lJR)Sq6m}zXxXsCKY_%fA2)Nmxn%>-KjaDKp=S%O2T?fMDHRPe?x-dH_ndLY~ zCxFl?VGmWVTBIoX4RJ+I+&?qWt={Zua!hOx$tb^J(4TH1)Ava)ThHa!ctLUqRy+ui zKM-T4mct0uaRU1NSSIZCXmf{M{u{>Ra_LX{DM*+CtN)W;1CHm-qXNXakqFw_IdIsT zr{ZE`kemsA=?zw`&84aIQiT7ztuBO?g4u z_eX5R6=TC|smP{vmwlraQH$B~YaYrAQA8GWNdd{cf>DemM4kwK)s$3E)VxT%IS9_6 zEFMFbt0ivq@F^9M~7+ds?XGGP!NAhnG)-zW%PKS=j7v~BxFU(J8 z=|G)Y00VRs9+Xn2#)t6R6{LMUF=K!jyK&^oz&n#@ptX3Obsv8nPEkJ3cy`d~`mYIf zRd_*8T_v`^?9HVO=}o@nhu)EG<9+Tc4q6qDa8ftK#cu??K~`b6AI)%58NX98I*k>^ z7aKcwDC}@m=aqY>E6cVMdi81aOiftfAK1P;dgvI*jtmBTAtMJIbgyZ%XsacaFM4fE znKYy9>Lm5U9`}YSC7W@DS}e+j#_?e}yd|3AT!7-E`E1gLMZT)ue{<=}2I!|OqOd%w zlUy`-C5q^F_`@YYkD0fqVYvY@HS6cdpd}jxM|9c0s0`_tR2lHr%L*nB{ll^#^iveQ zJWbmCu4@07GDmN-k2{y~zk3gvos|K?GxH+!cFhL)89b;kBRguv zH)1gM2qjmwYBe{abTvcn2mD%v5%LXu5xe%zWyy|p<_fFuh8-`5IAYE06*#_@e|;z7 z{BDoh^^hfV0MbVB$;`%XyB_SV*}eP%ymLu2;O*h+@^;Iw`ZcM~5ZRo4+~u-&a+Flv zi;38H*)itO3R)uD_}WpBc@soY4Uq6&m_&Y}XFo9y`s1<Rn?D6nk z-^1Mc{PqMbiNaE$G0aEn47a^)sPylm+^kuqPD8eRsK12a7^47k@aDg{@$eSExp95r zIj;91t@Tyk;Y{sa0p^%KV)uM%ODC?ibxsSF@J&f@g43oE+8d`g^E%CL1}Z3UrkhdU z*B{RI(a#9HA37TMeI}rWZHAxcbK!MD=qG(@A%+3p`bDOwwhu^K8bOtPKD4}uma!!% zpBpbkosRFfX$$w@o--sYTHFtCCEGGHQR5#|DJhKIWIM8X?yz`nA!ap6wMMX$sZ)4~ zg1(JB>ip3{93`=i4Y5(OVl__C`p_sat=L#Tn?ZjaAoN+W^?y;thI}|+fIyK2tHX=A zjKhIwX@azxgT#Nix!rE!7@dd*h%v#xqQ5kjQ7_zO`1`>)|COcd!3FS6&06Lv-kTL@ zhEJ}^>sL;SR$%sUR>7!n3T^7bdN=Z{Wp>$2Su z#IQ!X`qq#+`@s8G0sCnbdS!cz`|a6#C$O5K@Fj}Kxh4PQZv!pU})SHqF^HFC5iMKsa zn<#7a6q&`OH0V(_Pi{qonXvp$rl=_wm#7HxU{VEbu;-w#u*wYOP)I*I&3DZvrySB# zJL!qX0(sO$j7VCoxm{g|hvrLK=Q`<}^>4^RO>EOhGA$PO6jrrK{9?ZKM-x%2#CauI zhSWMd^~3yVcUS{QDlBA*f?BdjgS1|0t6!ZVmu)K1%L6HEHk(Sy7>n$RDQq+qRmXit zRJ{}A{Um1(GVWXqIW3DiZO7-}6MrWmgj~}pvQ*jBC#X`)A2jSvpIHy=}kg0S2 zn5@AA-1PnkCBM|?79-MFG@KVstjdcEO(u1ivkm%)wkK|n8nXWN%oj|+{PjDiBP-%vTCQ>6-G0jJ@SBw8a*@iTbLPlTjE5(w(T;~vB1&W_4^sc0k1Py zFDQy(E>jKJSnfep{kPoY(WzjMi>K&`)DM=nlOvESX$&Y9_1!;3m913Ro07q*Bk=(D z;SJw^6x#|_7*~(;Dy?lspr+-TeigI_oqm&gCQ9VK{)09x!lA6Y*p+#BfxkG_F-#0# zUp4wKQ|m1f$Db-K#^$!~@vcMU=DVNMMbd1#uILE5Ri4q z|3pQS|4hYy(@;y!6@3}w>#Eb+(mI%&JxByMQ;XDwTp2+dTZUQ`Y$ATPpD-Sw4_2Tc zmEClq>Y)*V6m|&7>$3ilR%5i!t5sD-7PD+6J{wENGC^T0rF&X zU&Go#-nhkL%Nbkzj^E`T)6ywe#^%)UBqKLI#$41hBQc%0G1pLCQre# zNxd45d~C5|iDCRNfM)+3%x7%T*=MVAoV3om-K)S_(|UV#ldVNT)ix9M(q||&R527N zZ)Ujjcm=?Z){3@$)VRu=QfB1h`s3rC{D(j$!8$W(z*Bjc@dWkK1hvgf98oHwgB*)7 zmF{R|tuN^Ge>2{h_&l_iUOt z<}ieNimi^aI%2p-@Km?=hgB(!_E zNndG-p!Yh>`0=aNlD#G_x0tILkLmi5wf!cC6?copTQ=XN9kyl9RU3;^J(*0_;BgD} zUEhUpLJqVBWuBbk%g8&#pjI>=L8w&ah4h9)gZp@)WVe z0t!a~je0Fj8_X3g_`{B9S5H1mxN|y*4wpe{&51lh-cwuigW2I8FL~KpMHG*w=5B#2 zZ|HT|&M9Frkzt-_3MWV{8h0~)|BdGIYLhmt&5*P|vm)s;^oDYG6C4D;uoKHBPq`z| zU@?!~5)CxS@d3d~L1yZA0w(n9Q_-jDtplN!VHIMqBvQW*$1MTw(ydMuba6UAps5m_9LTPAE4ZMKW3*ns4Nsk4;J zhrVx(VM%{rWlySl^%#8D5vx+*G45tI_avNLv)H=WW=!>OBrLl-u&!QY{`3ZY0*_7# z_ll9uU0mM3q<9-Tz8boKt@~fNZ-Q>d!qXpMg3*K^qksvDgr{8L<7IMQRa|F9m!q^OV6eB9q ze+h}dZ|f_zQC`n-rwE6y+I{ye#ZVCx>1-~6R;D^ zOr9=`>=TUfTe*I}P;>Q;=)`)7EK8v|6nl2}BLX{B%0UqFyK4d{N;-iMe2w~lo3nqw zWaJ3AwQ75C{;Z_F(4uaGIRtkh~eWd%;_HWzr1tLbp&2bH8uy5?M^T` z^H(iyw9r;xC(XCS_pNhB)>J6hcwd2~wx@3&tG_V%|H)RycOQMq$G+;L=`7`UGYPO$ z;oyx)OaF2(qptF zmk=5O7I$RntVD_oo8b_J&>3mY94!Xnl;VS5a51##17PQFphHRUsubfQ0Pa<6%3oYd zh!R664`RFD@I$=_IODokg1+~Lg&pY;keH$@hVXkhnCL;UOXerQl$0M8H#`_Be|77x z6Ln+6sqTck1nzj)kJDq6Q*U{lulCrW=71p1&OWeP@Swwc>Tu?KsX)m-0l37E)%*Y^ zI-$&M6a`U3xJXk=W=z7F`d_!e53+~n;(hcf%IfLj#v^UsKe3M%stM{qG%n7Y!Ave_ zb{i)z=C7-y?czq=qp%cp|7rYh;pv~|f!P0(n?rDE&9 z`@WJ5;Gh_Q-qk+lb7?-A+#{%xdtT_P@6IkG;m=!}9?tsA=6!)a|75Z~*dEytve(MV z#Z#K06Xhl$|KjI4rK>cP zGpQQpk3ezLCM}*=i}T{$M;&cjA3d%t;j-z}x6i37Y#=<%mC^{wuZc&1FDZF_nkwqx^m-xXeGYQ7*a~m0NYt|Kf|5q9Z z>ctRwLW6(=|M*X7ob(6X9bD3 zK(pwKd!NYd!}^o(C+}J5e)u%c^VMbuRDPfN^B>n~udQRht>ez;tH+-|FL497hagGY zKEz!eE=c$^JXZ+q6cv>?u6fjI!?>po5n({@^A1kUT`4oNtpy90aBg7LQ#App@DPo+ z=7B~SR8KH3Z0_#)%rwMr+$$+l#qil+e?t|-OoD64ArN<&KbV6`dswd+50 zCKapgPS)86c2JqKCmzU^DAZ2-a6r3u2MA#ag_T8h!G_8*f;Q-CV>2Z`*~FrY$})VnrZAy4I)VAa`(WpIZM5207vD|AC8HH!a zlG=O&D~bQ#Jc7OssS$w1(;Ipf%@*Evdi7p z>BgaQ2A;Fm7lOR4jvc~s{lcSsQgE+8chlp1r=u!8ghtx)htV?!#5_b;BBy<*K_-UF zar27^b4w3@Q_D#2C&9#eC5*YY%0L3J)Iv3`s+lvmHvMc`H;`yc262^)MzUb{nZ~aL zbJp&T9nTu7XorhbeCxaTY=y(2%Yu-ki6#JOu7n->kl}-D%Vj|{tD?#f;a8D`4thMF z-s2PuIY|qK!Rd51KMx@3e6H+(j8*YAl-0o5I29^vggETO1-6 zN((HOK+EGKi?)n*MtA4z8F2BJtEI)V7xEYlFoD$UAl3}hUdQ(D=qq<$>d?Nkg_M!~ zK)!NO`O#5YED9jEth`;bP*01mP%k(5xH!7nly_FbX+O=&l!g0^PFFW3Ata`mxIDB{ zQF;|_UGP!e`2+^Wa!G4@NX)87VtWi_hB6dwa2HffiR}c!3kCDxpv@i3o46V9u71M{ zy#)Uk1tsNo>|?lSvk#hG?NX6*LWP@uiwthIN|(5phfSj$hc0l5s*D>8#U*lU!bZRj zp)*aIO#Ej+mZZu!-=q0LFuE>tb!!t(9lry>{Cs3f@>tq|H;O^$t+hL}6>87|`q})Q zd{{TZQOX?jFLY+IljRP$>o0|t*_qu;i$?smEYfX$x9s9j$h&Nk?fFQ@i3p9MmaWD5 z)kPPMZ@E;#>eZdHH@N3f`3?e&Pm0v9_40CGx5oO!1=I1pRPpWV#PY`okU&2ois+<1$ z#2C-A7*9@okpjJXl#TP$i*&7ehZ=K*=!!WEyG`J67Py7?DwJ7qT6ObMrO(4?@D)C( zoYvtzl;tb2GG{uT{`T| zkAtLV-+c`G^Kf;*Pf$4{v3IA%>WY&$@48v3NYqeMZ}W%LW}qC`9R*M>PG8)opvXu> zsn7AX>9}t0E|cgDOkNA^dt-;c9ABrk+V&lz^L(z7@yM~$xkB@k9pwh=?u#zwk?T)l z_x$m9ZuZw`;x|0tgJh9`c81Eg)ySHg5hxEM1V~Yywz}&5kAV+qR+nPpW%yhLW)&XB}fmz686aVC%V40a{6!}B= z#Ks(>sa?TgnkaUv2#Nf7yeNc4ed{=DDjwHDsVU)?&L-D+3Dh!!z*{V@_|T&>?5mA< z;oE?_{t-f>${|~IgLbuZi@uyG{a~wF%~>#t3`Au>;e%-bMtpLRW_+@yo`{T1h$yQi zq6wsh*OnvXKU)}AH~#jH>bH*TCG>P#VD_nN)>WeG{Ru;+f7JYLgXJMu1?-#~U;HWv z2X?H^(4?eiGytCzu3P&TvC=D?N|6;e!IBxPMTJY>cJY$BqB=-2;bAmBpf6kk@iZ)^*86$O9P@C}lGS94cy?L3!X)r{-fBZ`vMO!X^ zO#Y*WDp3BDI3oLR#gW**6p^ZhiTVHeRc4_J+81N+iQB*UVhhk>m87pKeL@Zj!7i*^ zC>Bo`86Yj{5@;;V3`>kjKk)S@QV_m0T)VnUR9K?UbU5;J)#v)n{xv&ev#Sjj^i#0A z`{(SmWl1)h)l?Rbh2_)T0FlQwG|1;_353hp8P;=!UMsudYAfyES|f@6djA4jT3p=X zVY>3YM?qo;Q8_wI&4-GzQiBc0Nc$7;&?1`2w@6Hghx&fm5=LweiwA75!ekRH7{Ey1 zteNB#K9wvrs9I`twxH=;I;+Y6?8`8|pxeN~EKLaR(+*}Vb!KBO&}G$JB{rO-19-QP z!G@VUnJ#EKPzYHdHGy4cyzr8d^g~Zslh*^G88$)_9(Kly47O0ozQ`Zw>*d#3pU+~* zkt|%RIwREFPATXzm@jebO`h-KWpkO|P7{U6F@T2$;Y$-JZD9ZqGm`-9PYg-?veH(5tI#pR?NpYz-i#Z!|Lml`vIS*x6fG)@TdTY_%3v z)5=IW=?tOnh>zosh3Kc~$ET^t9B|E3#{WWy^;WGu2E zgV^JLRySNv;r19F_)L3gVA&h!6n|2fE}W)0mAbSrfTgt%U24>6s?*u*Ji(H3dl<9m z)+C56Ph9Yc5UsCbKqk>V5Rf;MCj6Yygh-dlH0-da;gvYkDIsmMRUcX8_UJ{o|A-h) zr}RrKJ`Q6%ZP=~?=HayIdE#lSV&u}c6f&&OX?7hZSr&i9A1@XaQ3`Da_dH7$%%u7qRYe7k zMoWIPO63y9<0Ca~0lBXYK;r{jzX|{+FstApN7ZDcKTCmuczG&0p}pqN1Hrm^T2_6o zrQ6{vK}H&>MUO{|Y09p{LLuRhw!cz%Gad@T>lSV7aP39LP#Ol+n<2@v`AXV^>SpC; z$A%t_%gMJ=u&hE{3oCD$EOdL8wKe3J;s^N@Bv zXwIDv_9Y4*On4Q=0}H3N(yefi6b?7UaML7kr=lKTisp)PYxE~qk|0?(`JQsQJU!~` zPb7J2$E+Nbl$6_0sIuT$ppG_cdCH*b`4~U#v|-UbZ)z0YE+vUA7vGKrUFz^Tfs>G> zoE_f&W0Z0&Zm{jyW*!FlqUk4NS?TixZ8n$eil$W)1)`K2YDxz%<_!J7R-#Co&6>~W za@=;Z+@aK=vVusLR#R_jD<-jg2-*V;`?bWAmvv4CU#v)G6NwMP@k(M#y*L1ML8F0y-GRY4Llu?znJE_@hy9r?R|Zc=Lk#a@jsp`L0HD*b zo+ftf7!f^^rODKfI1Rt4qBm1|?7&lO|D`=t5^-zjk-{ZuZyYn9VrIu+*ERpJ!dhP? z?z1~iKRdQhy^Ed^J8sQXoZ&1SZqyl3Q2cSG*&B+rQ6VYLS-i?HA1a3xpEBG~bD5Jd zf=g4&_6BkdV6{-N53fZ}=qhj4si{qY_}9fNxH`WUvFKgbY`do=$ZtDuqx~Uysrn*d zhsbHO#jzG^RET7rYEh#K8SRGXATQ~Ux{3kcn96x~{tZuv8Xaw1xi)?JC!aUiB_Iom zp;Zy`lSZ?>dHE{@aJ?I$lI9MdhJIF1WSYk@~;;aO$)5pQsk4>z?chcvG`z! zQ0B3WbVRGGM6$sqj!&f&ong>0WkxYR-0Bk!**S=PfO%)(x}H1csIV2<>8SW_B8Ro- zhMS!6h!L*Fuz)+M4wz$eSc^s_Y%(yw!y&9b3F21j&?HLxl>rz`k&Mi0N%mc9i?#R8O}m&O)<{S2~Km=A>VvBw)A9%SyO< z1T`W|7c<>Wa|4|}sK^F7O|s*)byk@FNscTM474` zd?riK6t^__*_q5W-LG{N6WH8PSbG5wXo0DO7dQ?llSZP%+Vz6jiT(O-kF(WMY zylzs-LS@4KE_&(|YI-LWPBla-@&w)zRIbJ%s6&J8#VS$X4HQVwMw*oPhbBp+5aYN>Cx zB_h8r=x`lr0~_|*`%zA7*Vg~TW2vj@%4IYO9f6wM6}K?zlzbDC;Nfm`lfX#d1@xSr z>yQ(%brr|W!O^=OM4+7)zeO4D?ow2nHOc46yeMxFwSl0}q zC88Av&+io=tAmz5RT_Rs+H=srhfvBc`j)48>;8ReGWj;%dt|6NU$<0s+H-Sx;e0`Y zUgwPwx(D)C2k37J2eKA;lWy?$M6kWJ-U9E>G({I^5o2ik_^!t40|xU7?a@gB@4GU9 zpLvJK6{*+WwD7G}{0p0)#j<}9cDEh&FIBywD^Oew8p>ItjIoAhJypibvLP-Wi6~W= zGf%qQ$(xzD2Y14vcKlh}4f^`=+b=?lj?A){;7*+bXKsT9!C70s>^04PhlG+WPNf2H z=JIN|k}n*5OBd*R5i;zNFsZ^9h$vdk1$Ns5NHBvns9`m_vpOx{qgua0eVA~|6N}9- zgV7N7d>)fdTv1KrjTY^~a#FJ}Cu!|PH%5Idx`tGg!qYshg_JG$x-v zwdFNTs@6hmx!&Kk>?ZX471tcae4lz&4+k3?o9i}AueeUfc#lzN2vh2xmn>ch`EzoA zcH-kkhVvh(uH2(nUK&QMT+Ty$x9U>l2=d~5Pj%W3Ai@!M{kVIKk{l84FkUe;8>5Ew zaZtcND?odm51<1tq|gJOP_P5VzDz-udd0`XFRbo}dV`E+a^B!QeGj;PeJBpWd*2P@ z1Uir}CoG_1&!RX3D6D=mc5|~71V;)WjEOt`;;*3BERfLH-TNuwca;?>3fBK^n3FG; zjhip02ITp>XeP(_p_#)XQO&ocS(eDezIAtS=Eh;@JvY#SL3rmR;d4~L_MUcxdpHnf zqT1+6;W9iPyeC7DTItAuiqzF27ABelyfT>FuJ}^o$P8=D?Yi&k9>T%6#FDuK+UW32 z9&5a$6-}KBb_Q{(#cf`gu}wnXM16YIoYoHSk&a7VcpZO%x}T$KjUslc(sqhmls z`h5Q2y5bKODB~tA?wYmc=?)AB+Oyo)_p5By}J<#my@Az?s0FOOqh zcT2(GXrMi-7<8URxRomSR1V0>8|#@OaJ=##j?#Xg2Q}3PALIz))Mg=eWg!ZtGdFbu z_|_wMZ}y3h+SZiYGI&gxqwFK=n$9q)<2zh-WiHglxki3Ryk%(F&WfEdi#+8?OX!fj z?IzKtb_B-=ckN@?0M*X}Pqj%(u6RHQ$hweuHJKTTbdl|5Zxt%vD zMhBL2nW7 z-RWiunPOYki_j#6G!V`FA=*1l)I_S~huq8iW-vYdku5fjkv=UpZ0Cp>Q&U=19`Szi z`e#2PwdJa`)n~Zx)PKlmeRUJ-F7I92^E)34uAT92K%QQpr2ipyh)$R>?;LUjihT?z zY34{PE*;;9BJLm?r%T>@?!g7$fi4yW5%Jc$E`NAoxj;>}hWUijYaml!rYdSHKXVZ3 zP3wVHsjd}@&S(3MD!e1qkS-VX4~~Ob1I)rl+WY>uD;|Tw=HmmD$YczJcL#1OqCi!D zcB~)lp#FBa?5pK4R~uwOOg9KeaWkBWgR8NF)b%A(Pp*=-0nNjA_TJxFF(CY%o1=RX zVe*ITH#Eh&;mdp3sqo*gvs*Yo@j8T6i;uUb>L3euRtJHEOI!mu6UKhJ?g0Hn>B^M{ z`f4d$8wHW}%2w?HLrrpT_#Ck%7}WmUr|8<#aUdRMz6+J?*F%8M8rKCAP*TzNb3 zB%!)lSBP9*7ovtTE?uuG^$O~CEJ@}sj8t{)DKN3*NQhQq&4%S z*+5lGmcCNO4(8Azj#z2;wHCN17LE2Hvh2)hdA9Vp(?2m!`LpQYT*kYM9*56NaH4L`B76^e$kY+6m&@z6IzF3N;^Ah8rK?D-YC^< zH+qK*nEN5z@tv<9eVvw4a3{*vVND<*dvvU2{UN8bhyzDh%0U#T%bLq+O}jLMj=NJ^=8olVIyLwXlaeBjaqCM2M{_ ztLVYWiZRuW8EVkQ=NV&#wyzvKs%?Yf%bK)v9rg!q!Z}-IOb*ZCUJQT&Wg!oOw+a}V z15)rVq(j?hktL@(N)C)pCTh}3?vM6JZIMyub=8jyn;&H#&X%GSLr?0OP9dBPnWP;U zaOpPiW#sK}WW!?T_AtcvXy|5j%5V?dJpC3`?hyqkdn^?hv-NokfCB?_ZDx{8=`4R6 zE30ncj;-tTS?E?|vpHBFs?}K%Q5fZN$GooD8HAr;&y z`Q(7UoPiEoi_puzYz1uX_On|%mdm*87jT!`P!#ilK;6yd1v5VTtcRcw`l-leEvE8n zAfwM{08oV>5ZuSDvw+=Jr~|18lP|}B*!M<^O!jzvwO}&M9FI?xeT+TZ}){VD`k|8O0jXpvIInIO_7cd z+jI_w-X#lqNjs2DJhhn>U_MOfJoQzMN6#vOAHLPkjFA0pbqxG@eMXx#?Z8=EqIVPb zf=*KZd>5vY)E8%OS#z5pT`Am5ql0Tlm&zrZg$}Q%o@&di!9tpTjK9LT^UHCX*6Rm#F3NRQxeG-v7Wg3@w_LoDWn1|?_IcGp zFD>TQj~XRFs^q@P=;7g0b8X?x^NjTMVaH9LU})Xr`xN{#R4MSg`sb-n`;Fn)(6IZs z#CC_6z(S^gU9HuMYoBzS0)X_f)BYxC=7y=+St9uKrY9 zruaZ?2DXpp73L^EKD10JB)Bi^r8K}i@I{@Czz$^HoEY;qmZZ_sN~GEMxb?6t90gg; z>QxYA_udI2nEzKB3O7=uV^?lG##Rer`z9lA@voa25<1TP-0J*K=9DY;;Hw!i)B-Uj z@&IX|fj*E8Iomi|Q#pOIMMT8MzJ&y~Ma~{lv4fmb0V!-mouoORRU??@R>qF-!NogQ zhy+|7WKOvj@eKprA>?asx+m+AIhUBPFqNx6S9x@C=93Vc^_UQw*9s3GA7=@HtZU2? zdmf=^-FQ!tezw-!mW8SI70HZFe)zeI*^F*tZ&22KgqlfHr!Ee|u2Q=NBhxX|3a+^a zsAr=ZUUhdz6Vl9k166p%d)vk+Xv(YTe1E@rjQzZNA$Y$ou-rCn;owIh=w|3swIbc4 z{xKfKVOU9J+?b@;GZSxUPfP%NQiBO&>O_1v7z$YVTS2nS8R-W*aszu)OZ)XZh-lsT zNl~w9um_a^8hzz&6*F{TOJ66Lp3#qD?5__9r+N%b^p!R5+S*CZQefm>RnjVg-_12r zR-caw!j%}0KSdosOtQ$sO#Oq!U?Kd*nx1Y5~Trb$VJ8XSBuK9gf;$b!6h(<~FFaL|L zZwT(i0h&zmV%xU!i*4JsZS%#pZQHi(P$qzw6mCI|;70EO{LB4mf*V&q6kw1mKo!Jv}7P?7`IqiFX?&@05Kbbq{8 zwwM1Z)SEziKPs`sPD?)D5lSdBu|ocG)ldTJ8wtUc`Z%C27+?umC`h9otsxgH(Kd1AJ7vUxzB|HUwp+CgMcUR^ zZVQoU<|kbxn_81%WCk3xV`D5iA3Qn^do>U942*Q6QXyY^qqawt0hZa9>#*ed$xu0w ze)kU6;maOgdP+%f3Ddqf24{;YTn1d+7r8?M@g-dcykoTAb^K|`V&2Gp<&kwv5; zL}l(P`P5P>M!5;9<>?}hY@L7xOZoU3^$wxzp}bc=@6lLn{&jiz5Vn#p z(&mq5g>@d6l1@}6Uo)+j9J4r?r}o1EW<>$0WGc%cWFRHDzq9KBNzPNY*NAu-ft1-I zE|+|W?=LYHmSN#`L7TcFuS7H|J_slt^XRxb@gav?q&TcUM^`cPQ-V5!nhi!Y>sdra z5%YYlO7UoiYuXv}Ep&NLme(x;wVx^Z391~fV*<(5p|DQ0a=VW6QMd>pN1&9WPXkA! zlIwu09vH1N?xSKCu}dEcn*=e3bm z0bvdUytL?I3u+#;qt2~^D2zAtmNYzQhuV7;$f6ES8tH_FlD=+`G!zOUsfm z-MGSjJ-zrs6W{@$8JO>F+BR!Bku#uJ*q$^c{ZjFPxN zD*^n#d7^w$mKaj~)(P@T(8N$4JP#S#du_f6bMnaKKGsLlZL|iT3ilI(vI%wDgDh7A zvklH}l$(2>i3T8YF<`K-}masLdJ<80jGm0<>!~#BQ1S#emUkmd)4_dy?~pXx)5_QKn4E z>DEY|Y6Ej1p1bUs;oE~7GGq@0wFgG{-uzOBD&tV$(;m7yHtw9YnT8h+78KbPDLr)pb$)XJ0jpQ0Ktz4%SS#&EH3}Fqa=+2mEv{7|^gv2Fwhs7*@tFCfEl}U_83)8sW0LV2 zK*SdKJEfi!q=hoBSDTZ9@xuq`Zo+uSbB2# ze((oW9XYq64nOo-;<=%gBhmKUZdmpN^?mpUY#xCfe~O7dLj+G0J)+zGt|Q$iw_Ra` zSKlW)enRl~_}~d~(kLGpuLSvVpa&`}AwH6#@lzA(_rPf~{Y2;oG|ot$nBE)qBaa6< z&KMu@S(4o(=?R1*><5yLkS~$UiMb=c2Z9I6_YfX&KeD>`-SMs?_u)5)R=WZVgmOVB zIAy6OsEK&Hv+)F4H_`FeWWv7ey zPeNJ(?DIlitaOEKgrU0448B<}MR|(>Pc*|XD7@i6a#0yV5KV-5gTyYBQ~6=_vrh)A zF|h``=a_0DV~u~#In;#4ni5<5OzX-lHAKq??F0a5%NzT2cAIqvVWrF zi!mO~UH;_BG91I4nexanoKBsI?FiKEQOwJJxpZ6hEW;V)UZ??Gf0HOggGv(W=R~H zkDC>X(3B!KvW34QG@o?Se6n((3rWinwHHJ%K~6PY(w&cy8EwS;N2Bb^{QeG;X zuFIdf$R*YH2?J-)6RwHWrQ)5fG$xu{gPmP+w1^K`w7H?dI$Q6`ukxi=`%YIKBSA*g z%@Rc+^rxtiTfG~ciI|OVfH^>BP)!{G!l#i<2g&YrFl9REVN3A?bO-kbmj5_437pm? ztRFc%#d|&AEsiPWBXl}wn?}1L_JvF*Fs?%OW=xBIu%r(&5I+J+hIsr!pITu#Le%3) zK*tzYITNSHlEKXHJdY%7PQ#ks_f3FHK^)-oBqtfsf?a*MD*D}s-Z1in(2b~JF&-m| zk(H6w`lTh&N(&u038W9wrGjLAP?7G7;MDK;O&Er$lOrT1c_Pq3jM5@!F*zW~7 zhNy@4;CJ%EH5yR*u;IyA6NTdh=tpG&g;jjQZ`vKl;Y6>H7ji`hrInq7txuvVW=uL9 zrhQ^pMB1jqxYW>E`z;`sZqch?_`glvPKuF9AvB|4!d+{ToZRiQsexT##-=&D zIMF~Gls(Hdp_$GXXPO|b$+kYum1M1n-;sNqw5f^VL~tpEZ^k>JT#exw`b^HyB=i7J zL!8&ZcO<=<_SOH9fbIs=qVX5oT=Y zqx8#xhM>bhA}1U+MV^8UZr5oRW_5*Jr&K|t&JhIl4u*VR50MZ)JwR-PA{snm z#+0Z<{Z1svXJ3!LE)=r#h(uSeEiB}`ti%3?`$23^-aYC!qK{KUL zuhc1yFHNW_DV!+)NAgDIdU=NE)G8|~#EYhv2HB=VksEw{PP9fyl5z^#!{&tS6=jF@ z45^V9Gl%U6uv_MFnE+)ke~SPT->1SL_A9ufPl1m6kB@^HFwqFJh)mHCI|*A%1+=~(%)A7?)M0SV-3ojv=R=^Ot}7y~ifk594c3=Rh_D8x zdn+yQT0xL@`Ulb9@r#UY6rtQ(+tE@67EAn>NO3_o>hT&N!H?-~p&p$WoB$Oj@&Qgj zY}hJn@GfllgVm_t#ZdEEc;KQo)KA7shCo@s5Dl7ubUb+BbsfynlOf1fASq2w851-> zkm_JUHFRn{2kuB}zmyuB^_r5#SZrWSHMpJ0lmSguNKQ38zWLO?=+okk$@3kswSS!k z$=A3Vq5(}rTn(|I8|;M1SYqxKhtlwBVnB5StWjBl;gt(hV2&txW77o73j$Nrcha=s zrNTh2JbD9+1l%Wvr3pp?>y?oclXe{Ri;@%lrohigEg?oz)wt=El@n`M=vIF%nS1@^ zIJ_J0`h*>6Tz%HW(iQmwPgl;Bq4tFJ_@etd+XLgKT(6mX(r-Opa)Wopq}w9P3($<& zYf++awwtm1(vA)L^;~X2(7ZQ)GH+bMK$HvDrm%Tq+QQObw4N{Z`JumIZ1o6djAtu( z<9wxMjq+#wYh_P7FIYYPzLE@v`jV!)@VbU@mGhs)#B@%+mz20r_R_mtu+n>E=i1`q z*b-?!HrUiXKZ|)W`4DQ9-|Lj4etOp{6B{_)T9fnQ&Q`l233k_GXKCCmOFdc}yH%o8 z_g3;EtaLob;R<$=BR-Z&jQW`>(`ku0N{m%cLhkm8Wv9|))dK0cWZ!ydDR;ddXGL;(x${osu z4}@qx85HYJSNVO;pu<8o-4Ub82geI2PXP3eoa&u>L7+VyB>7$^r6*$vX|{jVyb=c( z8^bCYmmvdbukixzTE$5|_KcvlASG~^_=GeSnT61(*aXdgl?gsN@t=|uu_TF6QG>*5 zw1gNf2|5}zLlG(UWC+sx;ggU`O5FYAFGPfCbDw!F-z>cWi z*wyDm7uk8k6NolL;+19)Jfb|1IT+{SA^K_7%#kolal;d?_C$phc^>o6?-x+nf_WK-0Wt}?iMb2r`A3lA>gdj)9^VX9qNgu;2>Y;4Ib z2wm!q%#Q@C=lSB1r-!$PhBTAoB8-t6Eegb{k$JlrQBlr>bQ-xm|GW*uS$?ykAOu%B z2%U{eXuTrV^#Z9aDzY#U+IA5!W#*BcQ|XlK>@O5oD+U+KxZlF?%cE^FOwfhNAK0db zazMDEA>xv;q9vkjG|pt3IhoTBDvqRi!4&KjdvgLE=F%S# z&Hx{|0s?b@zcuDW(c`!x#XS)0AyM=x2_Umk={@XyXaWnq@kOUilvdai&F9Lxe{1M4S0p@z#anXbdUud#ncVB!Q z39Ao}}}{m^)a4{=AhJhA%2#1R)?$ks$zV%NLN0|x_1*E>&R7=fVPv3O$J zJIDjRzQoU76xCsh%HhiLT-qWcf2P@k*G2~Wyr2&nUg+(@mj+h`fq2<{(iyEE6K`Uk zP}Q53){uM&+!u=%E`Lt#z|+JRtQXfW#Ww_hm|p)}iTd5+*@_QdPp-a*{XWK$nIF{l zz`v!i_YxQOzG%KN_QkY!@(=XBz}~sKQiA)-v(qoH58l3jpFw+*5%&h{A4u-ZTeFxi&(_So zez_Bd#-(?BH~POBx`X^98sxgia1-O!K%+}LZ1=g|<8=Qls4=F(W-njV&$ZTAF zfSkL8UBZ|EXf!{48(-2)Y%~7r8i8vW;%C9TdIi8Yv#WVIHNCwWUtsJgva z)$y_0;NSlA8g9|JFP^mYZYQlZZ>Y&!{B?U>f9iX(u|FUcoTaP(ZciQYZ4*3Zw8o)J z-yYT!;&|74UtWFyt2y>gJeCTKMv2Nwn>zL_(pjoHx<-n3r-@5!OU-r<{hW2#Wqj4W z>V@dI9k}rh|D<=^m3_mFnR_PQ9bU-tZH%ab@*wpjiTkwa>nyF*(Ghu)UO7&P#PSq( zj4SJ{oNs|+bDY|ydI``9i!n2cc9$7+ zv%}E|V&-WC;Gt-6v5u5)S5%*MU`T{e#B_Htpag(W#DI9c+vyoFfzH!hr&Qd zDTKBy?4KzQSz_*&q%te!ruuJC;+?C7w|_zrEHbL+?tsouo2EwXISD1lx^23=Y@oJT z4v9>!twj7ViD=5q2wnibqW1r`I@}qPwJxPo{BVdFKv_)4?I%@kfeljO#WqTH{p6 zeijd$WyxwFAVpOBjD!0X8Xy{I7ufEj4?HC5Q3`d?2u+nLJ7^Bh*L0r|HB45In}l+g zu~7?s-t@=KfmsCR4zH0p-f{}yW3M?jyxVkbBa)dNCqM)r>`5|f=j!-y#j}6lZ|@u# z*YtABI~n!n6FgT3WYzgd`FitR@BEvV$6r)Mf{pw;HMe`T{fj}cd+-b6;r`PjK8^Pm zZx`itJ8a`s9{)+%X9?F6?e@r{tYzj=8jvUTSatJw_&YlkN7wwAdG54-ez#THjQ}Iu znp1|WZFgE>ooN|FtDg0MPX&wC#$c{bTsMV^jV;F46#5Dy<`)_bPRo%GJX%I0EU>_m_e+6 z^x)(VA;Bjgm#3PgsdrW#E2BIohP(>Ah8eOD1tk<0rhvqyl-@|9uuc%-9#KWHqK1{t zA}pJKq|y0nq`=Ip_NMwId^K8?)V&Y&(R~;0gcmn{E&igEnD-RcN$Oj3P&+tqVdY+t zm|!T$uC}#(gMaC^Yc?L?I-~*}$Rx76j0ZQx9pZ-3WlhV!T9Z*LIUm_KU*OQ+c?FQF zFI%;{qR%QITwS@j5s;BQpTD0INz1%)k|%Qer|RM%73f2kzBx(3>5xfS*NV=*q-?rC zAI876zh1E@-CI;o2&S>6(cYAiOO>Usqp=quXw4azefDA7SZeRsn>3@IP3y$G+mScl z(tmT{VZ03}X?N%CjIy-ZfkhG|h$UL>$@pOg-j0v-(PIUe*gf!s;ZLBu#hG==^>4R8 zxtz!TQ0p!c{6r(S{nk=_yAO#u+K^_sj3K$bj=J;r?g0u;O11_Z%G8_ILm)vr!Zx{~gEB-2WL zv=q{}X0Hdc&R53I(mP{Z@?!K5KE&KgpL~$%{*5kzKFBq&MDR=r<{lHtH*tSMIjy35 zYAI6~xhK}563c(Rp++|G8DMCHFfGW`H4R-#_B&6YDJ9ahK9+I*Gg={Q7jimY(ZHYT zscUcA_9VeYowB{yGICE<971J3tl6!g0wex64CiUAvNL~8Yk%g{#oW3EdAYquRKXCL zb9^2<6(N;4cv#4A6N)J*`Kob4&w+WobQ9oW8~W3r(7BEhE zY^Srt8<4meM5dKXAMGmWh$PKz@Be0enubvCNVS`65fQa+8!b9gV}%Pq;hMYY#-c`4 z)zhEwaGYHjxk1ME0}I8`6(V69s1GFO)}Mxv7vnD))DP)iBqUQo>G zyqp|%(4{EjZA^RpYgShsq-X2cw}wsu*1@>hp>Gov6u5=Vz3|{yaNX2m)=A>YFL^CA zDQ0fALQ!C9h%jA`CfAk7212(6JPexBmJB zpVEza9yLS#ZZd{uRAZR(v2kvx&meQMyeGx6v|h2#$S0NlBk*L~7uPX@U(w>Ay&_bf zZi!FS`jLN*fHQ?`aRuT)#^~efKDe0LNPin@K0xd~O)n+t9%ebbM znN`zDpH$Avf09X-_a{MIgBum_YKB)BCy`yVoszlqDpV;bX;va8tz0Xe8o6XGmC9&- z6{eEmE?|$rT}z$XxTHPRa%#Hs@oT$FW)*i-X_66^rH+kRYkf-T)p}K`QtX$pkG)&u z-X&|){S)_So6%f?qRl=$$Dh^kYR)&_9 zmzi8^xa4h>XjYV0OfM`UUBO%Vly!@8L+EDJegl5>^K+~0ROXaXF5WL<(z}$u(>AL0 z%6H22if@z{)bdmr6!jJURn=ADEF&ySuf(lPEzd1atq>qys6Rs!DAzr|uLLtx>YYw0 z+21`Y*grih+dtqH?_6=KY~N(%?;O)8-@eQ!-rlEFar;CqhxyQ~g!ztD$iK@e=Dp;U z@*RGbeN^A_Jd5t8T;T7vTvYYqto-w7tEl^It+ao;QTx62mhfGAP5(%}<$Si?mA>Ha z{kJ0TOHh^f-mA+0`d#W*;HuQO&{>&h)w4p-LUeJvL77(=^Zecn#V4C}z}#}q*T*)6 zF@N$txRKE7a&4h6qrHDwjCVV=xccGL(&UrV;_U0Wq1tP=q1(H&@uwH$T2jB@CD-2n zrOLkiCF~Ya_a1KPyI0tClU-=;A!g~(=kOUszrrv67T{ZNAIUe#zT!3AHnefw_0Mb8 zJ@2{j7VOe=H)=Jkmv@z-H?4ZkH&0Xlov-cZ&Uc@pm*HE7=c0Y@)ok1@fL%j$$alKX+0SEq4nFS2U zsl{=Cs}_$#Mk|LyP%CM@XcfH8QY!e(N-Q)_^$Mt)^%O{*RZGBJOJ~1&`mZ22*L>j$ zM1`9XpfU$l=ARt=XmhCn!u7I&{dKcJf}7ZX+nGJW-P9b~U*h&;uJZij%{~D5R zW73k*;C4kSh}Ts2;6m!gN8SIcC9bs>=GZW`wAaCLB20C ze)Mfni~Ueh%k3aaOWx4&M&16%#_hh=b(}+x+#tmni%;V0(C>1}UC5hW9mJse#lYRO z7pYGYKAio+(%|bw#y-Wf-2S$g+8Y=z_g88z%8$St+}}!f_`fBmVSkspM83TKNu=)3 zCDy%0KdpL;ew_D^1U&_g7kC*Kt1v`PFMo=lU-*(tJ^3XZe*hCn{f!tNh{QlZHv#NxQ{DS-=~@uIHb5^Gm1 z(6ge0#TG8y(1kGon(~-1qpyz6teU%^dE0iA!4ic@7tTK7N3o4N#mjxN#VPkJuU&Kn zmPrYZ-L%4^WXyl%=^LE)d5LeGsS6^s zfB%V7lMR}D>9nC*+JGs#kinqR+JFyNLce%BUwcpS>{4heK6ot<(VKL*XFMj25QPYT zw4sNVFtF~>LqC9Qp2PzhS6qL@?R~&iL0BIJaX{LT$QQ5T7P}i9J}oV(6U;s=;=xJr zbzUf~SC0s0*2zFvGhB2qv~y?yxzM_tf0#_9Z5~j30Vk+$0UcfRTWT*9H;<$sv|G4T zV9tT(HU4Mr%K_hH>TsTZKh$I$N-v{T?N)Gilxg{&m-z>LY-1Cjls4%p55pG~Y~ig@ zrSlO7W$M#&I4-pLl;>1b^5BG%-y;dLVMuYEvyZiP}8w%#m)Zq zywI+A?Xj5G(25c~m$>a2ndLe5JHFI8gi&|GoFVqNf=USIa`k#D@450;$u; zeW`moj;}a=AacS3ymJGsF8)d5S=Iw&S`fZ(h##nBz5w8(n`cjzh~xwGTLb^(MEtfa zZ%;dD(1Q6}WoHZzu_wik9KktEq^>BBJFH^??g=K_SDKSkL9P$gTI{|57w225HY4u) zw?KcY-9Vo2xM`xoQM`$$^led3qu8(7(LlHn)r z2)M3B=~w%~pMBfCgf;soT!X2(;iTk=g6Mh!_0i{}NdtJp1RE8#*~<2M#?=S5qpEkHGLjwE3$r1(mC;9qC0Wl`#e%V{jXR_t$JN+ z+I`n-gCexHvqmmq#XHhf8^IKxp)GlW%U${%K=Iv{+oYTud+|g6^Se1$aO3O-F;{U- zaGS;(8AG13E|wG^cdaeeGjhbWj{l5`sUAlxL}ZtVh4_Ad)mWvXC>W6ng`T z40{U8DP53|WsvDE$>4(UID?c%W#fv&EG0uK*bqZ@eC%|RgCO1@4*Ouky9HuxL_S*A zo$=mK@VH%`=`Yt>NnRjfEt@2U+{cgc*g_u_r2sGZ7nbbFasHXY(UK=5u@mRv**rh( zhX*&mPXPN_eH{B+)v56Zn^XC3PW93*VcLa#V$6s3k=M6vqwL?Xt(w0wH!J)^ua~;p5biQpoJ%!Pa1iG~AEO9|V*32B8f zA*HHOhw~jGSNhuGBeI=|T(+{Lq%;qTCIE-Nx{MjpvK_7p*UopeY?<-a+*+XvWmF;k zGZ6-jWl9ygl}^iii37&kddRD7sDYrNn|}7$eECfNe3pnDw94fir${g~QzW6mQ;16u zfeH4vaQPr<)b@OIaDZEg}0B z$AnL4ylutMTW*tpnv}9+TTz{lWbzbJEjy`x!N6o0NzBTOTs90T!6Sw3VXX)`RlX3m zU8%@;)MWwa#WTv>{4x(GgZ*T{Q+lHX0x?MWI3yDy#D=|eCrprtD4UuM; zWAk#OR++}Q;$1it`S_d(@BlfX?FO~lyyJ34&ynFtMA!pGEiO39ocI z*FY(?p$4TqN+U3`mLb>*lwgnvM=)jg>C39Zndu_mh zxJzo}ZGJF4MxW+!c7AyXhi`dZj`ka$WxuT}bF@QU#pS_C_fBzJ_^wDpO8VS8mkJWJ zeiRl6%bc3WGzDsBqJ^jA*;8urE&*1AB044pFf)B8AHX?aQKb0&WHE zQ2C&laWkxXcv(RR0_-{Xc<(bH4qn59?W11?)tVo&=3x|;ole0!rth3iOhn#gZ9`Pn z;Pj3Utozs7$7cE>=IqmNkB)lU(&T1tFNwZimD$1a)PF#KuwQ*g^Ilhoz9I+tZb~SK zRDBn;YBy)+18&0(QVig{S91Gu@Xl6dQCk|t{B+LH5H?}X0x81Xxf^Fc;e~6DlEuhC||39$pjhXG$mR) zffwr?nX_VJDs?m^5@^o3amk+}!jFM265M%pwW)m~FtXcZ)c2YSSo`WAde`K)Wm`Li zr4_?fScIq_P}H72$co=sOJ3qn=TPHIVJsyi zt{{y$?(tG=p~mSMVR2ppvx$M|K)={QGC>e?g$0^Ke0)tm@dTz`+Wwd{l|l>j!B|N4 zOCu`KA?@0%0|T$=`ds#@$8D}7ErHqq8R`pb^ab;R82XYOK`PRA<6C`;$u{_slVhUdoH04V^{Hd#d{3*0kCG+m zSX>^Led}FWp1>;=Y|Zp*T!KwJz3&-|j#rwvn2_ zRGr4H%;;EVd9+B)kawP;XW6|H^UmV6XjspIXN5TfG=;~qe$Fd3G34016fI5vIpgtR z03a0Frw5&Nofv#{a0W9r1y`IHm}NjTm3u-;vm=H6_3 zUvWMT>AvNQ!^q{Da;$RMy(c|>5HhNVZnQ#wQC znW#N_o+$bhbR;t{S$nXiu4J_MlxU8rXNf|k2zud zf_G-!wZb23%c8i0m^pkeI+^%_dQ|!8_UvTIA`k}eROH2T5A213Xq1xpLvVRz7k@#E z6WU0P4iS^mbq&i#ut{eW)!-JSZPpmoU;~w@6x0(Qs-CQQ#R&^v#c09Qu}k;p>|(N} zE`uTm zVfAyZJ0gjDjnNcp+=b~z{sXP>my*xx=m6p|a)sEkH!B17Ln5@QJ%prw(U;zR>K~VV z2Kw=7$9bLB)Fyx7_Rqy0c-$;X!fcG*3F>t9BU71C>BLlaK!&OW(;`kL#Z6Lp-s|To zU3xndQ+@voT+d*Yn}5`Nn{s&hCj>E8{jFN}mT82Ad3b)9Tqp1SSZ?vDJG=`RfB|%* z8xr=>=q@WwAhZL&GXp2T&l4+}wJuockviQ(4^BRVymJ8c05!ea5lYd7NEb}b|$qd8<=A$9gicjz~69HM& z^w_x4BUIzGPb5pae3IBBRtM>5k%bH~?>awT0!dXOzfI?4j5w;&+^nErkwLual}@xI z=G%mlTke)OJ(=%1m-L%!k(gS;!IwjRB*RA0a9R)k;-)C276?c><*Jd6 z4rZ*Qqx{;c(9nPEw+8K!vDU5x#pM19v}0g(&|%Sx5|yrU*nT4%|4av6nNiFTBNE=X zkvHu+4(DIR4180hIrh?^6=bHR zMvZWvr8~{YoXqra_*6||tsP{+54W`htVx-!o~SwW9T@N^tbQ=>IdvQ z&0v`S4hPHF7qWli4%qm@Cr{@W$ap82X8s!(ax$l2{sVzMqrb25$gW}i18<)7w;#+5 zM#lu$rWta}La}}3VBUtReo1K`YYM`f7H+52@JBl9Z+zlXLV!M6R)MT#F_v+wwU6v0BHeVI@Py@;E`e z#-|Z%$z6MUJV?#`A$%o>$Ml($L9=K4Q_cPXsY5P$e0IMP?K7y8ef|Zu7z!cYu=~ zKkP^3x&FRlI1jNMhR`=3-yL6>gxT7i6lJ57?$)2SC1&Z0XtfF60n?P*t|u=%W?`g_ zG?Eld{W>Y_{@N3nI*mw8BSmE^kV_Ims7xnGuTR12_f56%L~5$h$tCpr9pnODtVtr= zoBvhS%o+P zb?^nlf3b2hUSFP#|49C>9bLOmPiL_|)@Izk9$hMsv}!r{Jr?}XU|Gk^w_O$tSk#Ew`wfi_ zC-mC{7S=eCe?}!?N1>mJbs0K3GKu1d3ijM?sAE|n2@Wd5J&<1N?WByXq!ZX%w|n9q zdF=!+t}Tff?TZ`jD>%~P0^b%8c7V+YHmS24;Fa6M-sMz;C%EY+Ewl%W0$4`+s$}e4Q^7x9I zG8L~uB{axh4x-l)_{JQk&B(t*zA;wHOq@9IZz@EK-JtOuB0t#<2>{n znNH#$AV<#XIpX)hobB0N%9L}6g zH}$0y5cb+Tclp{7O=qhIh)Q#VCZ8ICh&?%!Q=Yo-JOWol+9C1t;5XqzYSB(4;I=qr z&J&bzr~%cev#K;BEF_}%uIErUv(A#yPZ9?M#bzT&r|YR2IT5=Uhz;vhvh>(?GGsCj zF3$Y6qupx8MERqhh3a6begOpp{+Q-sh$pUKGR^xgGgOY+ay1>*lL;#x2vxzRDfQJ4 zbX9uV6)B;rBeKUbt5@inpaexFONgOz*Fvq9Xn2*Sc-{3`h8YOkg5b6eFut3P&PKEOIzu5Yjgl zpHNuic9WY?WW%Yy2%oT*<}$Uzki3+L$$m~4wUkWDf?RzspQK@As(7F)J`O7&;gwzp z#`X_`?(0WjOIdrgB4>n>sI{7-oA+Gr2G9UMK)}DxeC&(`olnw`$!5(>0Jr`vP>O)d z=sOjPt#tM>eQZ8KQNy{qRDoPCd^|&}N+V9x3N(enRP8t4VtX=atkXhkEhZNp$6)m{ z#^g`)8v-B7!(B7n3`rtApI%m>HK76vC-*XOd?&&_iXhu)R2BfpUF#VM@1|~uT z(@hd#gx6FaIo=szTL4K(1ePu8BNK7K-ieOW5$LmMGrL%|DS9If&c7%=qzirdf1HHx zUIFuyzy`oS=zk!V4O%=;)cVu~i_pJ>mlk{J?}bNEFFHjCQY?$U(`*b4(>PKV9yjS7 zN0YBY9af8Hm zknGd2!}r($XS<;|Y|C;1yC;)uGqSPtKwuA(+0DVHlh8YR;|D>sn+CPpkzp5zXZTNOBWzc!4FQyT4~lJ=IB(g_BPCaICj8oXDADAK74dBm>SI=9(Q->-*)G`1PrG zD3*HDtA{+mBnu-Kg8C`YE{5oEl2D09fuZ>6@~BRtU;xl^43_IPS+9X##!5{W`gQ#9 z(*MX!oIT1T+^u7&*&bEB4g+ zH_48vBn8ia<#g-_ZJ4Gwzm^3HlcY2K+npl@A_`-vlOYIOg-$ztaRx6e)N@Y^-u2!G zQ`(@VTX2C|h5iZwDz|JJ7TZyZ5#A$MM>i^Lbt z@)Up@TS(Sbk*xKZuLh?Y2V) z0Rn3O^Z!E}>HiyX|CgV7iTb89iYe-^ujNKFQ4g7=Wfn+Cp{$U!U+8L#o3t6u(^u|3=a=lGtiIk4?0&oMp?Xf^;t%HgN2|f^;?d8PEu-K2=ESKP3(lEo zOng9S>EmSLVxHQQ%5QP~Q+{MY$Ce!hyJ5LfBle#t#&%?BSY2alaRQTa&f@b-&)@;P zSWz>~)-dDWW7LfM0hl}t`sJj&j{IXtP8(jkhryT=Y7c+v&)^$-% z_m#pVC&flhRe7eTkTGguSTQx;L`4YSwP-pX2Ml3~iu%%hHoLVM1#7cv2CTsS6?&6T zO#IVUURXLgmtZX;3CwH59D|*CrC^;GJGd%FA1jQ;?`4?sO3RAiiSSIKIG#Gu z3ZFuslDj86?GhJNJ;qX7ZUDk#EIn|#kCpKyQwj_xtim&{aI!JCRU=D!(RWq^V^Fyx zlA&ILUP3b&MH$&0sh>Yu(Vu`(?$Hp*3;u04D^x-9N77CYc`W#9X$ABD#o0FoS;A~v zR#%tJQ?}V<+eVjd+qT_h+qP}nu2VL;?D_7zc`@@M-rR_pi2Re0`(LiTR%UKwguGFt zuuNh@x^#zB_K9podvLyqD2}GSa7PZK=o=g)Uf&ECw3Z`vj2O5mniO%(!Mj>fsXp$=zZ`NL_AOAhEL?KpK%ixejOaAs z;8V=W<0F;Q%^AY7-O+n7TqFzvNESG1m84=bR~~3m!q}JEIxyAX2>{h8M2}q^pKa!t zi!PlOS?6sz6pS-Z#Bi}f2Iov&$kmbxrFSom)KGoMva*Z*JxIPoSZu28FnD?f0lzI9 zq}!&(b%4Ukk|#9B>Cr~=NXyz|NW;V8#iuN$*(1??RPo+;cJ>Ksl2pwcza?fg1ufO( z-|Vx0sbTCClza0e7SUplvgl-w-3Q9i>@VNNnV?8{iZQZ$Y4AO0I=PBSS6)}6`MWae z1bLs$jZdo~Y~xr6desNiu%9g|ZmiRHcoh(}1cr7F907E?_wRl<@`Z0YTQ*OPWkUif z@KYl6`LHf|LCXu?>rY`mi2U@iI)%RWmEy zxBf-g6^MsekneRG^|T}G7aJr^`VB9Fs%ljq*{Q#U_TDi5VwXUMol?HDT?vsykMZ~0Gs28Tx#w>?+qg2Ab zpB|!=eqti^*c9#fQwrf&aP;{R%v=0fTm6G*!!d4=Qq}I$I6OXT$L$I0UXd^*8ptpQ z)<}D2nkmC9$wWBDqICX~*g;bO<&YW~sO(AUnalqIeSTwPJj34+xb_paRdCPs3H^2o z&vi7^7QL$$dCUdOKIq&b(f4i(fL5R?ktq~6mK~v_7>BN*>C(8t^`pS)CWKa(%S z1r6iSm&lRxSNkjC3ITCSmBeZ)guH4c4ezqfBTOku61+RMr?SYDyj(RmTuLfQ(E}vOSA*(mEX!Ln z1MLK9HmPsp)O$>nta5pqBjLA9QyaW3yQRjKjU{QUcx869rw<7ku}I0oz9|x|SXenx zHDeOS|CAwC3GoD{?!vSwYD5|kGXz4gQA54ZVo3S}Z47ii+5aVQc$dN!aS$LNW&if` znSaw?UCq(L`9J%s4{JhuD=#nepY*V&kKaPALtqZ{NTeYl{YjEFPE}`v4)jM0HHHcr zm&r=^Z}TVCOtrMpaS+tj+^XnLh=?Q}21KniUstab(>GIJYd$94WO?0md8EsL9$ESA z&gObNZ@b-aemh-j+}yq(B3?uIBM*el(O_5N;p?=MD`%@?Xmavk))MeF(__Z{iVQ)` zPs2gVcjJ}@gdPAzl2$OGUSnkkSHM;wkl&iN**N3FL)6&Fg??5FL9-tcCOOxQ!Q+Dk zsS>Lzoy_F8Ed-uTn#uFE!~CFuACRB>`}>~xSU!ihJJJk1(94DrfGAD^mA!v)p(b_# z08?8Dfts&s*{WE89a((ScX6S_&sD^ziXk~9R}DM(%>ZWCRZoPr_{#$bHQr9Gj+cRL z6OF`I%PXiiydagKfgU@I$~=(Yk#QM)L{ZFOm?a&oRLMPF-6$Nftmt`>X;S-UYWMW_ zc`4-1zBcn_GcrAj9;th%~jrSERD?+x!o3PK$V?&bCDk#c6wCRo4Kr!js&Cai83w? zl%@eGn*3Ds2&x?Tn^Gw=#gO=j(Yp)`vCYZfBDytt!LJP2U*)qy%Q|?~c7JG`$b(6M zLn2^}O^#zi!9R*MTBOA8h5NAsWh!YoAgH?t_r#lGs`)#wxU#a~Sr6bR;s`gjm*L6ZK&R4u$FXgZ(2nSW4Ko13@XL$Ry)h_0iOMh-$p0_)D7_906h7K$m`rt6OTWIEky5!E}uc*GPk78VM&l%HM^kR4sTs z5`QmT-Z!EbrH3bxCfyhW!G~?-Xcvc%3flxG6SXos4c>;539|&T7QYc6P})lqKy0 zm%M-~!)v0l=Dt$TLsoWH_pQ+gqe4F}=MvIbh#Xs%a%!$z^UQgNah zu_0B$Ij+fYHc&y05}BV2kW}G^U%cFr9kv`ne|5?m78<+k$+K*%dB(!7i|cTzq05E2 z$YFOUCagVO!JJ`d0-Vwh$zfvyec{ZQ%S~0%?hB+xj+4Bpqlu=bMow-K)Z^zK`(LWi zgPGMkfQFI22wuMl=w8wr;-rxwOq_hou4}56fEOpaZD91R`jknHNxdScw8I&Ng$~@R za$aHOMjhDwsP!0b@tG&TMP$jo>%&~G*EJs0Xlp3sUBV~X#C@lzPU@y3yZ(x;O>DG} z2mXk>oOVu{cR@zb*~w7o;sV$bewG^Bn>IRVq|uJMCY;HZFE;5jBr&yKq4gqW7jD}_ zz8bpFq8(;AALqw1MqP#}LOMrl|7zHyZ`*`*8?d9rED&i+wGvIfp)#i>>P^jqP2|*>svC^^f-Dkrto77FlEB3) z&7Z|#Lur0_WHBg(C^vIqlYC#Rd7jR_Qn)7wpC}!wo}thU)99F*xm6CP$LCkpg1(u- zOajL$%D{h>Sw9zm`=I?zm0Y0|Cr&PMG?AXQEFU<{XU-3 z9;c{el%zjn`BO~n>?~;O9`plm0sc$L-K52R64q{D6Y!D`Y4w5!E46gWdw!r=J*|7% ziZU7K*9X}ya+Ir$0C9oMYzaRt*W%JrYWo(;=tff1x7W*z3EGr#6b(5Hi*~{6<0t;Nw!gU4nFCtfmZF5HlH`}u5A-O{-DbRScZpGP(h>Kp_~hEUUWe#{Hl;^ z(vI~YQzy6vyB_LU3UAtVL5=x3yux$YZZalhRmXGNZaSqjEUK4@5JSBap9QdyrhGdH6E09JXhQP!2imM>a-3!xdr>^B_tsH?~pF z=IcLZgrjliY@Xu_OrrjKEL!N&*hCktk~cbEZ&EeaMo|X7B%0c?7)T#3W(IE5oPS1z z<3_@|1W1%_4H_;M-r&j%Npyfw%ANQ$^qL7@?B;+rv)|!fasLQ|bW5a4{G!I%hGrtz zj~lnQn1If`w-{Yd3O}&U8vHmx(cQ4?Obem-8^-Ryr}2KS?=2P`94=j|We5c2iO{hm z5>Iu$fu%Nb*C=4( z{?OMyTc8TFEwv|8ZA75-J)HLqnYc%3AZPj?Mk4;z?#eET`;4R-MP}hEN-57T1fXfDmQ5V)j*nG%aQRw2*#9Da` z;kpL6m4{rwtQo50P}iQG@nLl%RXpnl*cuFX+;K6j=(D}8SA?$|9>Mo7 zlyCsLo1B{wy*l-Ep#8D9DA$2iV-6h*oJ+|@DNv!iVvs$F*yuX{hVA0|J%s;(M`QSF z{d!^rn#2Wk{Y`lqF5hdUWW0W3h`PNZp9gI#vs(u#d^LkP0J*y8ga7}R^e zMenQ1-nK_~E393;i7#t`7r1&~VoxdKb1`P=)A)sW8vs+ zz2hjWBG09>jT*NXT7s(J4GkX4*02#iuoHVBoXh+5{T>^&G$)y1x71$>$m_q zr5m^!;kdW^qc6}F#4?XMPd3-4U(oL2q3G#p#7vEqdh~bi;n;%HM#5d@z@1(Ingi`tc{p=lZ%X&)UxE@O{n+^GQE6QamKg?!2vw z*WO5*zpU%^BF@bQym3nK$1Pky2$y>!%+LD!rktH9P^Qirc<$PN@@U=={2)1UO3J!{ z9OG9>90GrE#T~Okv4^ZH0w1~D3ke*4nm98fg3Lh`WjU`%=~u^uBosf;ut`u4A#Xv~ z?)yeE(af_Kf|Y?w;)U?&)L+;afR))sNWo2M?u#}}TVHH8fOUpx%LE2r%{7-&?aI6! zLt{S3kengnzr7<9y{3&oN}S3|G|5L?m@y#t_?O=%i>#^LwIIs{5e(7}jFOB{p9+`_ z;)bA`p{SO0(kU#!-GlOG8u${{p_->7Q{RTO`rv%mkK{LdR!u|76K7t!>*)yVnnq%z-r{5L-3IkJMUMZ){K?7p zEYa+DZOU%T|H3t$qi-7oD)xu}+;kzRG8vyd!2vX5PYA=E5z;dc=PeI1v;d>Th$1Zv zKszvlG{jRjN^g(P`)fd`4Iyfe{6P|Lnj+n5tgoO5kzLx($|{q^E2$8ooDM%5g0Xzr z2B|cjN*o%y{jdy&WuY9cPgA{=E>BUbZ?D0ph$N*^#?tn4&SwwyGLRXgwJ$VZ3^+oX zwF-Aq8gel~I*&;H`2ZwTQnmIsEcAYy}b^qZYh zoXRIT5kW%_+@4Ik7&x_rkdnI)+K^_&!~-8Dh=F^3+vJ_8#HUTeiQWfjlT*@>H^-dK zTT!+nWW`7=t#YM=ZmY(!U5_W)VjQw$aUO)h3^&3ZfN#)12-Iq}4=PF%inU$*hj)Cd z4XpEgyxEF*Tj%biC?8Iq-hueP&#z>kWD+whq$=ClGYRDK?)g%bd6B&zE+Z53e#wRQ zbBsG&7qeCTjsQ6_huor={cNpMGHMT%iJxxRGP4b^ok$H!StRk`=Mk7nl#OlIv9whq zCyazqhX!{CbuKL>FdHPy5@uu20_758v|sBJ-Y}a08mv{@?6F-jAE6XU5)uUPVjn^` zg{mFZt3I&}5`daa*RlHucyys+1VUm6n(?d~AbPN^@~nyf$i4zHZcd}`qDW=e;Z^8VW(xg#xbd%2fOZ{05deaZkV8QGx}y^y7C~EO96N;i3C{uwR78^udl@>I2#3m}~ap z(gOeBXBN*U^U$0-I<6%6jd7l?#B&d1nFe=+KfkELe;2kr;d)KywjGtDv<&bf75GgK z{)GenrG@{U*XNfB{U!Ccw313o=Vx&nxY*t!!mW|be;vTQDVFVg0=nzrJ!Bv)d=H$! zLSGZn;Vsw>Z8|kXSvv9VYk3SlLG=Y3+K<1TjYTe%YBCJ5M@AXKM9uG}0c1y#&b4U@ zJ*a$JKka7LeQ$pj4j8M{ncM^=Op~W5DEjZ9qpp?st^Lf6bF$%osZs2Q@$w~j*vSea zd&T!%kH}W5O6%*O8=yCyvL0u*4lvZL$LhB15tyC*H3muj=WW3r9|MxacK*U~se`!N z*2A(_eER+2<1PBt*7NHFjkv_A2Q{95xE;#NdTk*?OaAlf$wy(R(?dPDJ8@0^+Ez-Q z9UbVuY~Au<&Tq zim3><+~L~o+^1J*!ssEg`#mp}w39uekafK~@+J)(l_l_wN*}d7ho-6nKGF?XG;C>` zF}oc&%0IwZTLf+vd;h{Ks-8Zi;cfh&PAbhOk8;8}!;l*5!ks22T9cbU{d^QDi$j?- z?_Td&6(118zj8NLcKg;nl9F%nJP=Mjn$C-S%M^xslwg1)Vaz(#${f9nOKl zep5mb-k5AS4_(T5C%Y?8&Au%h?xR2lprKej~6Pl+%?5C<{0WHla< z(;~gfFg(yIZlCMGzCkAK399->^N^nhHC{NZhbZ(>zo6-dOZV8kv0Dz9?J4+Rr(ew6 zp>o5-Z^751ehnnPF!99I_DR1`cSZUIxZSe8AnAqud|`jV-xXl*2?=>&d%L|^!`tou z@q!O=>q9&0w`cwVG71#uz~UdVa0f91Qn(ZQ#!?($w4wD5S$@E%9rQ9p?i#}N4l>_* zZG}VL5&w8$4sdT~xD_+l#~TRgMYr`m^0v8&;i?ltlIvM4HMWe-2w1`nSozp{+&=?z1^0tqINoG~2b{ zkmDl#-DQ#@D25g88ZgOskOqf0_s#EWFp2okNQO%dlV0R-joOCH2T{;N^`cqdA{@lZ zG3}y}P@^UH)eqbm=hbOrnNuRh#bVM7m!itWgk+d!&=zB44MQCQ(k!+psblc=M@K#l zQ|`DKH(v_VjCkmdvHbuP*tL%@e->zd|%PgN0R-|u@v0Xs>drQx$F(LqJgfRgC9yd9Eo_(FKd%o}2 z&~NWaKid<2*$;|wKi9ilnitgr?5}&U-s=;7ub)fo?M@t!3Z=cUvs&7wm~`%fd0EH2 zulQdI{`PVxzYs|8%+3nV!FtkBQiG&F0`tC5I>Kr4)U}~~9nc>DYJ(txAt`NuiD3XH zRZS2ct}H#DvaA=*%(x7{+s;JWISiw?x~!e)t;npa?T`3TeNlex!RJ(a@bH6e4aI z`(6xba9jsj3tSK3y|h__A#X zY1U>V)@JSPN7_l!$Alo|8d>U8iDA-mz4am{|8ZT)#Es1>P^Bf`ZG6@)64>US>*c`g zj18SD!%kY*=9`Ut*VdA!OxJqW=3>*f&HNT&rOvp8B%<==-&*}K9lr=SQxZP5{yA){ zrzfl>NVcRFHa@Q#`y}lu7w#ph5edcqZZ+UrPgJO%m99@M5XC|vjW(rI4mNF1rES8Y zBq;tH))3CNG4BD`z=W}Z3tlKbEHd%Ln5p-@f&y&?jkHcK>-4;gvT&WVhk_C;t#XFfn6S$mct`^p)&x zop$RS&!OMLFQH>=qOo&|XWN9BPh3U9HNDJrB(=ewvlB1fw%J6=VZ7CA-P^;Ic5$+9 z*+FS+w<}T)rNzXtW6go?xS7+#=E|e_Nya`tff z!}r4;_f_PJW0n5H-64HD z)&}2!)#|_7h7N(fX^`YVp^Vu&hC4v&4&;B>K>+vP;#-kaXP0zVyu2TNsGARzlZ|1R)D@!to2>};K!4Q!o7JZueY zER6mm_~WB;p@gG?_HBbtTeO)FYL>Agq?kz?*VJrDr4FB;v~)WMB6VSvs6DV!7nK_1s1 zamrpTwL!aMME1_BDr42$?7YHa*l0Qt;g5*6ggpOnt^_KubcIzi7lY zP)!;vVSr-zN?xX~)uD)$h(y(>Ili!JvzRca%OcxpsCFK)?YLR|v?c>n%zY>jV6Ng5?^d?_I?(L9bYxGWGb zW&=f7lD8bsh2BIoP;3iMk!N3?3S(S*F5=lokyl!f6s&FVR7GiS#&o2%kZ8=%UR%Mq zK+zS#E4^kU<@Lp2THB$~XuC$diu-tC&JC?Iu02r;Z?H|-_)I2onD=cC*(_REQq6Ep zakMF?8xi>RgPS2%4$HV~>3X5U)FfOYJlq%^&VVCOF#69Z`gX;-Mn{g;x*j!^H06_= zPUVWbdM^F0qCL301G}09NL)o;v>CB`N+!qDd5f;B8LjkY**IqMAE`IeLw052Ep@O} zM!Bgh1<2hZG1d)fNO4i2iwae(=?s$+!XVhJ`{3Y3S@W4XjSvkc@-yBh8LdW*oa#T{ z_DreLM`$uYRWNqOX(uA{Rst*A)c&X}!yKWFl0^_`-kltorAN#3u*@iZe8mZIEj=>` z_=CFH3wQS>osYD@RK-1dfH};2&2zJ1#H`bk`%oor{bfXR>R|m?C+=%Ta8%D3foY9C z!0y(wYrNPAcRH2O4$OtB7RS1!N7Z;GP_-(u**vA`Ji+u#+l@dB#&nyyo~nDo{1>Eo zwto*}kD^56oe#0x2Kj!Gj?f9Tx5(xR8flWm`yYD)uGum~driD8!7#3-AbJ;fNM1{x zrl6M_XB>g_%|B*@(v0NeC$fJTLlCxva2NHRotE!Ja>4dC76}6WaHRVG;mFMU!;y=$ zojc5#Dw`o^OQ8BKF#0LB2e`A>1yjWR$sQg&>fNuSd)x5qh3Cz?k5|`z9%CyGmi{FH z)|SZrW4;Bly9IH3K>%^j0fSd;rHh40oTA1($PUWA)o7utI-26PsI51=NFKqVHdEf) zNrKgpvW*s{%d=mnXcTpRb2pz3Ir{_O@kK3IXnKfv;6ds9$n1TMOuN;F#VXaDm=k9s z$krdP0qFA!uA-L^cLV!*c|5Y8SQQMDPyDw_oh z73ab};@uYcArdkyTSU3x5`QUNZyt(|B*`0k0U=t8d81E^qPOtRce?I6tw8r&;c~?l zP}@BZ;w{k=*aunx#}Y-7>?)7Xg$n%)74Tu}rbi-2g33VPmHCoN_?}EtDvhk?_CEY; z0QtB0!#>41a2AuOdp*OskhL+U-&bo6MC^OuBFykFP5fzEX}3&fw`l{rpLe`&x-l*z z;T7uh^=}^SZV#9fyAQW{A5uF%<#xQ&b zpifLS8 zJC3*8E>6Z7=R30_n{kfP-HQAp!-I(gg z+1cW2D{jgduZX-Xvq0!AJv`~28_k#=44pid5ocCz)oEniskF{WAyLa+ZnGwXg5CD@ z%GVZfb3{TDu6;XwN?Z%xsWZK1;OoI&4P@m$-4lqAttQ|lF*D&YqSclaPqaFTzmw8! zC3q{&_(F(Zd+0%Zi^DHnlD8T4!Kbd5d~`y)&|O>)-Z;BEsy#b`@`dyB>1h%~OL< zo_{oMcQ;W+s)eI=iGXLSa*n-e7Zh{w2)SW|edo3m(bk}C{cFTS+mX@V*>`aW$vFub zQ30pHT=;V~v0&yFCV=CfDrFFvRb|s*lNP5nh3V6AO_lZ_Hf8^cum+QwUt0q47y~{x zmxeS>%&noIZT~7|nkqI#kgS>?rw^0MvVVWma^-B6tUka~(;*e)gN_9%jld#l2%BX) zVewjfX4`P)0GGgt@8p{C`6vtIT+%fzjhLIUWK1rSs=+IePSVv;CVrddpLqmTgZmXr zQQ~FOKL)%dV>97Y4a|a^<1_ylHX21HbkK!XO9S38O}R7}UxA z4Wj_OlAuI&WvL%#z8aBvkkyZsqDFOPs2@6r{kBN3lcoJ@6CtWAP5U=WKm8Dc0^q@0 zKZX+_imyldGsWB@*>A^RXU&pRwF!225JJb;jHD}$o#&Sm#!+;%mB~vh148e$KCjqV zi{j$YPxr<+;FV@n`r>1U2z5sR{x}W$2-`3%+wjV(ph2clYvct1DPWaf=6v`u?luO0 zeHo1e6~3Xkg%SJgU}MC*P2LixkdeRI3N1Nzd06!5M6wV!uBZ8*<zj1c9 zXx5Xh(Ao*(A0CrWZ5WG8UO2rKDP*%(Ry@NYA$drKR?oSNjXqHC$)79X>~lBnOTY;s zEIo|3?~e(szT*a8N^$lnBO{H-tKiv~({=(JI7A|R2toIgPh=HUBr6=Oe=nZ{Wd}}h4I;e)E2dyXS&FT$**L>?Tv^vO+v z;?3+0dslsHGt@fhhO`U4E9%Yejd<66>ofE`h={}w#V_)~@BwvKcxyBiHz<$956v(7 z!SVrj*LdqRG&}eUsTZo2!WY`N-S)dn=!5!0bNkCT`vv|*cE5EGe#ma{2B{aeSNwyo zE8ZQoEB)o?i|~Hj9^DYZU@np$jGx#CXIJtI?2Gii&8_Qx?Ots}Z$K~E7s6N03*?LV ze(qjwgkOKD@5ejXcUVEOA}sPGXKx>&^!AkIktPSXhfZ<^+8;_m#PDeqc#D1vBw!WN1{DS=O7Hr>iv3=WU& zBVgsEvMUQvXRz{(gLN`Grel{?skujq5r|(Peo1#z93=t~XXKf7$KJc`Gp~>Dqw*l` z;Iq)yGUl zJ?3R6VKrwOS!~Ll!^B%LD`Vz52BQY5jO)md8_6_tlJEej77Y#2O3G#!h+^CW1Ka@1 z*Aj55_w>A%ja0=v`*Xt4MpBXuM?0`<{#pqVi0U+5S1EK)>H#(*Nr)|GY zm}uKU-QkU!!7?yol&Oll%Do5X8F%OT2|g~f?#O=g!j)%*=i1X7#0%m1^E1B_^}_)- zJj@*isoJFd4iBLRNCYWf<7~y_-$Anqy+_64bPZps`StT(*g}zop+W=&0g(Xv|HyTB zadb4Xbry3purc`$fQ9_Fv07BXN|h(Hv&4D$T3Z)Z+ufApCQ7CdK3M7s%pLS6hpPCAj>-o`d z66VmU6V>sgXQ2^rk4i6S&95d!JPc1ELFGLtf1WArrP}?3dd7rzuO`_RH!J&?dj z;n@s|Kt5x!Ht@ktW)ufg*DyjZ7)`eIIJrc8S6_?W*V|;}Kf#WO8}6{1uCiG-x<_ge zbhLQDy#Frm*XeUoz6EvZ8Y0(a+_b9ZO1t}1vN6EJ(QFPOFANmRToY0(w7I-Za#^y6 zbW2aY6}U+A1Nj$17Q(*B=YD{Ig#X)W2;Bb`A)>}+{|gx|3X^gJ0%*L#ieMFllm#za zl7*@mH7vGJ7Dq0|U7)-eBS{p6Y2{A^j>VL zWmINF&}0RaOBn>7U&e8XBU51m3t&oAs|?99Rkv8SrGvZTccbs5^lbbi>lp=>*KqZL zWDc4ojY9VzQArpqM+sRq3L~tl;jBHog(+<`N%D@AMdYO@MW$9mHRL%)|MSgf|A;r# z^`HF{2mOEda4`!LYvcb?&62G=>$E6_>4THWfh@H@CYh%Q6<88uf+7RklvL#Z6FLoB zNb+2+ZTx8jS%s8nGT>=N&aV3@C;bbYLFp2U=_fnFXCc?8E19+YJ~VUpWlhm@_j3}`AB471ErBT1zRHAhUAREQM35+{Z- zoPl~4JyY2xuc#U3CTeNmg=an)^{{<6;uUJ6{BwsOfo9pltq&Epd%9rxofvDKPk8}e zv9Sd7S>&KaLk@#+QT%kS&yLmcXpa+#LR^)-*_j`=N!m^5le5qENdVYWG(tcry9IWP zjcv%ZKh6)om8PFN3@|u|H3&d0@X3OW-Mx;nPk;#^p?}m{>w&pNirHS97_Lx}L`bS| z0j``J)#0iwGiCPLk2K331^_**;+TEhCV?0C2I5cS2V7^hji!s$h2F$ zqo(!Sz`k)qh*0fI65JLMR%V%B!3W;dU^01)ND&??`4Q%k(dg&A)j?G?B%p}!rXxRa1CTFSVRQW zZ>gc)`Byi|DlsBP#=&P4pMCy+xqc6Uk#qS!w9Nwf|DNanKVAR-%=4dCwX()lL;tFA zyV9{6jkCTeiojcy6YAWFG5RgFF=D$hM0y5jg?8woKc0_XHS$V$lvzucJpgR6$x3@9 zi7&FT#_@Sblqr{&#S9}N38^bUHy^PV>=PkDfq@2;383$GGv{b*;@CwY?T=pbI-P7e z^*CL-&19wSe$2T2xExvqW7`~#!1sD+i+8(W5%Y33t2ptuKpP2-DzQQ{LIS z{RagRP%No<8xGq1#jx7-Z)>`%bQy(Yt7~lLZH!CMO*Q#w!rFvFvH{XrXXT2W_R0iw zm`yx&Z%g`RbJ9M%x$L+n0)n~rU?Qq{I}4@JRjj620CQ~SEbn<9$=ffvLL)i#4lQc> za?<;jqqdHt;|C!Ni^l3a?(6u3qYR}`xSXCR&6KP28VfDZP3k_FU9WbcmM10}jf*8q zZoJBE+-$yf853zT9GYb8(dmK%h_gc-HPE()jIr}KnMvnos4NMUaeG&R9S3>AR&q}W zi@uCgpG@wsq^q48Z7@aw!i)*mBNpp{W3U4AReR_r&!s$J;h)&WO~u3-lM2IiF>aM{ zF^}(V?1!;>6mtoh@I^G0Kq>9yKJGsYbBCM~R_0GdSJ*2a3kc)Ia|L~|${~dG8P&S* zH$!q~$Cd$;ze>;tSwWjiNYg`g7VbJ7szZw)fu7`}*(});UB4>1J%aFWJcJgqS#)+o>JaslBs-$u|J8zAbfTh96vQVityGhTV`m=m zo}VMjzNr~mfC2M{Meks9)RP>fbGHN4fHOD06KazsJXRytnhEbKrdFbpe_L)iGT97bv$@;v&Nj6>`wU@Xgm>~mv28XJHne-N zz-4NQ7pDzglV?sItx@vglPuC5IJ>J4mU%11@mvTZ%8n$*eXd~g3zyeq^_h5hos;U7 z7ffm%Jmq<$=Ij22U7XX?<#oU{ySRJ&DIe>mueJO>K7pXpryN)&k?|-U8W{YRGNTgh zX|nO1%7yYG=`Etmv^kk#8Sm-P!8C)inNnD=fNAx|C}2iP>`As$EOxf2rz@ibdazrC z-xkA3Y@P@k-mO0==GpeQMfD=1cQm_fSWu$teDX{~a+&tA+#qiodjQrJ22tkAm8SS^ zKl=0L2dxzcP50`-k0+XqnThH+uTp$sl{R!oA$>g;KIKXOQ2BkOg@aNJoCWVOv;ehA z*NJat6qUFq=9j9XwG%MgH^e}#0CYqlP`Ztz|MhJ0W9>9~ z%2l;ls_*j&(}5j6EyY30!g7i7&B)cUZ{{K-OPA%SvA5iWZj$oJ>C3q&XedvI`gk1) z7J?!hJJxpZFePabr?@Csf~(Pno9{cH-CYthDOEn~)cw$KV8-PzPgS#lf43S+)+RP%AIvECMW+Pf-q zv{QG$#cA!c!112;dr{){8-EF?8|pwnl$(G5UZ2dGJ@B(FBgC5yf51y3^Q6q95fZff z^k))j5QbDo0C1Vo0@0RKn`&Exs-mbn$uhYK@$-Sg+^Lsf3(lp;x55TMclIECI(2)q z2WpF#KG+Z2KSN}9G8Y2k*Ew(xHJe-bES|pLBy)0b+UKD?tYUb5YYmyE1d{+qK)AoG zuwc*?mjV{6KR4=_vqm~|opMUcM!8zO+1U<9x97@5(4o-6zJC={1@=HH%QNidm*Mt` zhWduOoa(u{4Q`oI}2 z$#)8DW~)e?O%&cGrO{K+UG<^>t&L#_k6y$XVFYeYgTRDA$u5`2*ioVG$VT-Ik;;7U z!NN&q%(9NYLs>l2VvW+0$^Qu)G!yMo;k#>1?#%&T8+|*R5%ef!YSyLBHu_uD6Ylc2 zm_Bam08J%VuYdK1xr;#IjR@8>-sGOd892HYlbYV3W)D)kPwA@(aGnvz1=8)FmAD3@o!k~qbLWcEzPX5yu>XmByvQIr;$>R?XXCTd(?_u)vFr@)vaw<>BRvII zvtwY&N0^x#C7Qswd4ZjWpUnM({ljEu3WV_B3lfBk6tZ*+VSI6raaWP1ngt8BCrxyy zMH!j2Z}MY4-#&jHraQGdY%fGY;wL_h&V(fE;@-DQVvVUb613L?O402vaY$t2h*P!ea10M1VKt8BI6 zExB2F%CBc2>(uYahH{@DdJhpDtF>8g(W%oEg950Qee1z(=5z;gK7CJ~DSsDPlm>sN zY-Am^r|L)esjQd%M>!ubNkc8=ABnd@`FApp_utF>|GvCed%=1u4==xPx|v>Y>#&j1 z(*7|f2}jgN1qBr&M3Rso4&Wh#CY&{vOsV^0Y~(s9ErU!#%!eEwhKR4Su(3n|CL(I1 zQdJ%2n;W-c*}Sr%s%52B-K_bVo9^ZIOb+(h^R4FFyUuwsmCkH7lf~&|YU>WXK&AMx zN9>JkAko%rVC_D9uCL>)V?VqmP-J24V3kK@4nka1g-`__Z3D-`9$l?K3&8-EYB-i+ z6RdocioT4dw(tuyg&_eJ>V(qLVtDFfp&}#7vZCVNY^I#5oSvGQlWY|ZOx#3KMe!)G zKq$ga4D)rm+cJ{AlAih(%Q}B)zS&GgS|-wqd~g7?i6?_!#NJFUcj_(JyP6~My|grH z(qd6Oc6Em#y^Dgrq=(CVv1oEb`X!JlnW}oMHU$knKeqr)b?O(T3ey;moG5iQWv?=d zd$fX_kz^oYCe6Ud--N#j=LsJR3B#fxtjmp36~>MDi5iLwo#rLjBLcTs>%uJ@<`K%$ zzA75Bf})}pioWed+{g(i52DLbtf|=?PO3c_%KX7!4u;EBky5M$nY4=PLSWe854lQ0 zq09zq4i;a1c}8YYK&nkqKtnuxd5@zm!rr|7Qm zDxAa@8{j1rq%#WpL-i|vu+KXSPtSt~jI`u2X+_}J2H?@RVcA|G|E(;j&Oe6uX)BEB zeZ2Qu@~3Snop?s3uoQ~WO4-5=lcV@^e)r*w%OL*RMc{VbESfTA^ip%EW`Tj@ykMbN zF4Z`^+kj5P;5HPN&)O9I-u`#UJnbFX0ygAcUXn4!i-QU%? z+omLkdEG)!3?+yN?j4{t=2I~8lyxSx)QPAh9cO8Sz>-1(`L*kAK1DMo4A11xX=bPA zE3D*;CO1aKFXlk8WF*pUtCGi>%LYlH!sJf({{pke=L-D1(XrM9tePgv0|0!n@}`e^u$ueV-a+=gxi^yVpovgQ6MlVuo~ zYm6Yo0XshrM7HQG>3drfOvvJJlghXVx;RVX9TaLw<9OGCZ#tAwyN)71c!Reu+`7M+ z=X5BUiv+aylR$LIpQxT;F(%72p!wbe@AgC5s@n?r6&d{!iEnAImXr-A_9@grycGVK zIL{?0I#l&OEOFi?BbU{e;sC*-KEgPw8HrCCYQEy`s5X|^Id3X2PP#*TNS^1^=qHrk zAmE(*LvJqse^B-gKE8lKvgn+#ZR0n#ZQHib*tTukwr$(C&e-fVgV|pLTE`GP zwX^VSQuIX$i`E&O3ADNPb5%o77B?^2)m z+;~jd5_hrE+tlUIoASip_35EbeJ)FJbqD3DShn(i%*tT3=9^rnT#iCG5(_b63*#Nx z=s>=Y8>~zbU5(Lsu7+iBjpxXXp&LICO+CD2%R6#aAZrm;TSOzW#bJ8pok?2YJYysD^`A zm6HuAqANonH3J7T2WNK@+S1bPtk~5Ay>zI$tlf0mB3^*pY{`kNdnm)hIe7NOroCe-hA3+ z4S5f0I+p_`kIjpK)YN<(p1tx6^V~ypZcbUaRV4)2>=W1oI+G3RC zaJcBIE#t)RDiYZLMlxw5-$#B1|=qn^jmYjA1Ml+^j52*_M%crJq>Z zeg{_(N&Bdlvma-8z;84BPVj#x*YgJz%6+N z#H7M0ENk74h&r{Er`#qWW1z_0Dy>H*e>1Hu31NR*Cx#nQrlh_tDk`->3dA<;sZJe6 zOqHvoYidkKOtj{|@H@Vn5mOG2K$_?T?~I`4_R z*Fz8vNgS=#)|SYM-&wLOIE7w*?{abZ!QzbCoEMFoiG&xSZM~OyY~qvTeH|W~jNw17pa! z)&pZGf;HoAmiF>6UkirbG4Dl#K)5`Kw-Gj;usdn%H^3nI>tUYJkP}p%~F)vpYWo`ib>96i~W5)NO~9WoAI)*UBXck&$*j-H&mDE+&W%^S}kwT(O4 z!3vu<-vNyEo6sPA(?JxZzV%QVV()S&J&AV*TpdyOYA6JxzTwd2SsQ*Jv>p^(rJm^u zQI(#}idDYiimcj>jH3yj+O=?bbN>n8og5SmMK?(w@oS9mVvYyeia<%9UoU^}hj8!t zSM2lgwi$zeByUpPHwkBT!1J9moUj57OmH0thc=^4i|O$zL)>6VaUF2^QZ#>KE^gQ) z(ux~G+jzqPjn}yAiP}2pxuv*Hc6lvs=p_^fW8v(-^pFHuOlZn0@d<~e0g7@M;rAZ_o`d zjJwm&FM~e{FYMOU!|a{+6}jUPa(fL9!S*K7iapQ-M0Xxm1_Ys0nCfHSd)i+H7d&m> zetg7Gs_B5Gbo(A*U`T4L;tmp5kxMS5Xuigo2YiecuMDZ}*wHicJ7edr-9{is<4=># zlFO(l=K&P5Jdqlob7o=l-lDV1MFaoR8l3zd`No~v2W5U^@FaMZb;DWKwAMi44<1Y} znc%H1Kp$7n%?xLIqLU}8lSjIfXR4EjRX2lAPrAu7bj@w(@q}fB(K1qG<{(2)&a#w- z6KEDbHmB^@3^{q%esa-^F{P^9y4W#Jv~Q=jFpwU^srakR!=(Ps4S3NBJqU;sJY)F_rKJJhuZWxR` zG2O7VJ<>lV+w3su2LBrtQRgGd0WF#lG!Bf*dg!3xmP8J<14q~eG)p)|Y$&Ty7tGPB zdNA(#6vrcqo6(lZ3HXJZUE>b)A}fZVyaWQ6=@?-!hjIY?OpGwh(fk7kYP9g>87@1i zK3dU$;Tv_g1mYOZ*)qGuaX%wF`5EcSKV;7gtrL%5=&x;O_iBiMJt> zm08DN`jLfRghrp--$JS#`GW~+E>O=%W{V!Zq$-sKs0k=f-I4U8yD3V~N?)ci15Wt5 zAuHU#K$V?KO#YFKjEiDKq9fs0>fN!+%Dew)(@RiPWhRKLMF<)m{KFY}_uv#lz}W7* z$H!cRKYP|9T0aiu{toDbJ99{iHbwQ7|3P!!=)PQfa92L!s{rjy2-h}5j5|d27`Gwy zE?66~4+6gbw>#oZS+E|R0cl987<7?cVugx2b;I0~3N<6HZ zj9_2tl%VGh*lqcVM1b1?d+NX*;kKv_K|OBH38vnV8>HjP)e}4(^O$?g{n$YpTB9%K zk3BsgV?Hd5_DF)x0yil7v06Q?If?O8!zQPU-=qPJa4SXvtr=KK!w&{v!6m7H?wh5y zYqhLZbz{IOSEuYcMUrr>2XSwrL|?Wj^~XVSHdv3WXn9JKI4?D9P3^sjwn1U)#3FL% zxNEFGX6yi0YHREWYMi7SG0)$T2Wsk$Qmeeoko5M?Coc;*n zj@$~1-=BYI%<@0JzH-y#d2$gH5g&cpayeTE+G{S(QQOJjpVCo@f+W8%7vdO_3Z+_| z+?S-DxT0UR&Q4i)X&l*JdCeYnZF5_;=iLX~_LJQ$#|J_Vo~9*sH%S1mSEgMa(^smq zq>d8fOd2MsES_Os!Q-zF*RLM}-N?NW1qFC{xRzkxChxd286?B?&>ZInV!|{)btwZT2 zl_x3Xo4gcAZ$9rQDdjs_@j;&2@XObn^sLWSf#};eKQ)*Yg7X+z}pxn&gQ5Moz+$r-sYKfu)5?Y%#gBq>L|s z^QV~sN&5?8SjmPt*`lIRTq4+$3Lm2-=6$Hy9BP}{{uEmJHk`HWOxYGR#ULr`jRZUR zgW5VBL9UJR%Xi>-;@FfvjVy)-q`vC#A0x6wcJ9)Rj%>-?%|nW`cVn1c+~z%3YKXPd+aifj zb*jaN#m~BEq7Hafg4WTbvgugj9Kz5ao2*BV&Ym1rHE4XMtwk@9t*Sh=e%ydlxdswf zixGC!EQ$1=K&hJ96l>F=+V~*RztP0H;nlOXq*_Nq`q|_u<{5H~@uNAO*mEP2I{r|s zoc+g=>0JaleW`;#j4FkAi7~5a3qmx}_wHE~ zbaJ9rln6&68r$(UZRXr zU$(?PLt+D*w{Bji9_6)#t(GCbTg)(vLk7xP`oT6&O#0Nu7l`AlYC@E3a+G%ECruW` zIMy~HP46v|HE|3PWk!)KN&Y^_u5ejQ{lz|Rs$V;;*g}x(4R9CiGGG0(sBE)t6Sn-b zlbb;67C_#{8^#1T*ZhK;Q+a;v%yOB#(M+sYFsnwA^%Zj}t z!|#Hse@qEeuG9^CvQU-E#-vi!u={GdqiMuz!`oZ6$d+H_ne+GIii+~;w7TMk$)Zun zZP`Oy?z5j3grK^N;#yTJMq@;{LPy{vt;%2J2>m2q)bJ-sl~v6Te1_A4-?QJ?s^@03 zm7fpI3lN`W0;PA<{`{K1u&rmQF>rk=Z0%*=*GE`M$X0}gn#p({R0(!;QhE)VvuEa2 zXnQLVKpeFUr^KIJxl;Xu4~AoVM($|qV}&xY%Ke4MqZD7pQB1UQ9#M%x znNlx|*}D_^zOavyQGw*&zI%)+AopX-f@{K*yA})9(DykPgii!Rhqwo?(DM609NmHN z67sd`%T%f8RV$|@H%;DdkJx?k_%$I9iNrb%a-KT z-c2S46)y?08^a1!pl>&3zaR3jgRyTe9=lC+eFYa@%lSE1a?A(@Z6oG%Hif?28*Vg( zUn|D4oiTGT(rvcm^E$km>#C~|Z++4GZK64wT8x^`1 z9+3|=wAImTLut*V=wT0zHPeUn9;s}a8NKFpFkgI>PHJVTwza--YD0yT8E+vW8zzUh zG?|Mqd-Yidu$-a8vl+*kxm|D`ns~V>=(b7yw7Ju?g3;Et7t61<#!5=3WoJr90X7**&B zl!Qr(=}dr}s?$;L5r!={uy#rQ_6GV2&I5z>h|CLt@e0qILURemGr{sm%o~B>kx5`- z+(h^Mk(w#xND-=#GwTV0FT>&qohtu)NG9EBCtlfz8L-tMH3$8Jj4XBXKspI4rce;v zE<;qC9{&RXtdw+`RK2BKM81y42Tb-NMz~G7M^3f&cY*Ttk6MtggI+tZX?g|9uH>47 zZOUsTybxy<^$h>Bpl`CgR&CYrilJTdow#Ub*Od4~vs&IYi?h0|C$P9}G;eubmv7l? z%(J>}Qg7L7VsAlM-`XVDuxmlsAZUqO&uivyqHl@YSlcA$k^H2VU-+2~v&efu+T?!| z-8}U!d;yzV`Wbm_sdI+{(VK{$GtZ_s3tVH$sWSmbV8pO9`*u(6co$0@t}~hNhLJwl zS_ksQc)SC#X5gO=y(4HW_#*0D&oi+2@<^T4H|6_*V9WNK6uX<-2>GIBQ~GP2Y~Q#r zIP_)2Foim#k%~Dq8a_dzo!oG5gp@^*+E_u{UmD@bCLFBM``1ME&{2)Huv*R9;7>!u zN(61Sp0%-{1By1KYrU!wJ4{5^y5+rYtC)8lYb1n5l6!i{h{GzeT;tRTbG2%BKk*?N zI$B*5+daBA^{(ON$n84#J<=A%&txrPUX$$M-j!5NgR8E2mJ55A)N4KMhk|R1FZ|{F zocY?BZ>uW7SnZ0_I-3vER;|4G+NJ0fxDR}nV&0hTB1U7@v*{IjPwe%)@Rb_xwCjb< z6`K!lm#&?8*E4*p_}=WxMO(ekhn#C)Zy0a>-VFXSLZkTe&{gp62>ud*S=^=gM$-q> zOZE>ccgb&hPdWdb-lDz{{bgOF`V;$l)ra;K>o@O9=MTh7=nv9o$q)NW=?|8d_AkaK zE9u28CYRTmedBBN&sEzf;Y%-Lu#dspIzcqjRop1WOF_Z}AKPd{A13D&iZRpY^0;vx zR>&2>Xvq~!QKF9pjj28s>y=eeKn|Ol6U(3uGn>7c_2J;zey0N-do)h_R(lLNmfPLW zs9>%&?g{0nv3LGG4S@;I90X3?9nol(clkY=JTu?F^HF_|(0c^m*duSg?SzRr7n8?F zT#Uy@yrj2~*mLjVBL6toWc$9NWOL5SH_S0$zSV~4IH%=@`&e+kfrd=|tM=?ujzaH5 zW0bs$^}#aE+7pjh?RV!f5S~4U#5rf4nJC;c_l6_$FQqXY@6;n+zO8#GdY7-9H13}X z>TK-1OVsYgjNz&iUdW~c++MiQLrag(19AiC+Jrj)FNSGm3nPtaGL}xHjAq(+Yr!s} zoxmGldbY`|Bo25I+D%E0wL)HDZ`lVnQeSbFre~7E3bh?LW0}mGf?hHkiu(%u59iMA z%oU4#UAxyugR^~NtlL=3&@NM%v_;P*(kchvSBjqhqh0Xun)V-gxq%2Qvu{02A5|0 z=lM8@fgxdr=(^B|ZX~jMv+)x{ihZQ?v7-9Kx{wubUY7fq3t*n8QVv+}b~RnE(-ERf zxvxl^QJ2+ELk-?%%jI4z-DE60%jb*2X(F(-Bk&~vO%SZm7@ppoW4ovOoMIgixziKP zpq$_@O`Rvz#;{Yn=hkbaaQ&$FNX{ASa@0A|b8X9rmC4kTecp4We0mW|;+I~TjB#yz zIQK50K4mTmkMciZ^?w}Q+4(dpp1ACGvj-Q({We%`!WQ1khQlUM96^t|h0S?^#Wc?& zWjRwWGt==RW{w^nX_;x>ht)HTGA{v|mf+e}#yN+1?qul%K!1gxWOJ01%a)fjO0;V3 zPcXvrDsq)piPDMmU#EK;I=YaRbd(~cr5q?JL)R3oP@X_3vE9;8;$?3fK^S^LbiULu z``rc=d~sk;yY_i{aYpZY;>TVGO1|JJPTYn}zksUu!wAQ3L(F^$n(qnbn|&eKQEusc zQC}Wx-{|Pi zsRiSl52FTRIjhoiLm`aBsmwi&o8U(o$S|!MIcEjB(q1nER+B6xd(P}SEnN+ukyUjh z$K!P~!g0^rfy;ZULJR24LN<^g8+TUMc{6*(t-NJ1Y=4}U(uD)#c9=TF&h}${axH>I zJ0kq|as7DH-ntS3pn=Xsi${dvn(K$!RG|T){PJ9XZ_h1=+G^Ok@_pS4R$GFd=wqks zvNxb?COfy`3$zEcxv7)eKNOKpuq3fMfoxMz`%kXVfom~qHnR)d2ZkO5#j9E!?0_kO z?b=>O<*n;V(O6O%$(2ZA&DTM~C@IBA6HFz4PYS?oO3e1^SvhGX(azVXp0G^*xR`1x z=Z@1HroxV9$OND$|7NAfyN@YUZqxt@p$|taG0acu&pk7Ac%CrwFR;>EA7P9i)agUr z_*7qL0_suY8BLc2Lpq@|1u{GZ8XX13w?T$PAb0{W&WJ#}7Hj~{h(fz^@t|)4IKqOY zyAEua@}Z8qWHv-y#N&aF!yj*e*r5I}1qZEI5gF40<)}dmcIx`mV<^^Ak^?*KV-K_Ui)j^QutHgr8u&>I(Xn|n z&~qKA!%Y@w!RqXDJ2BKmR+&=8ls+AlSZQpoAyWGk?&d-Z* zrv|6z>c{0`VS+oO8AU&HA3ez%F|PppQ@cWU;KJmGG^Ps$csj*Dl16l}$d81wDueEd zU`e8^G8i!V8Z^Pk4dGu_P%4on4Ei)dQ^StsXspNPEq8cG{ zQr5O%PAM}{cFE4_-+j^w^=V2MkOKBd^pVEzJ}1>-MbP>%Ykt<;^Bz(FZ%QM=e^TaS z_zC%P4(~{)IX1o2)cws@ z>sgpls2GKO*RNTDC@53W6ryn93PuEvp9Its5w4YO|E z^lhF2yhib6x(uqX>bgO*^OpKisjuyZbe)nSfzr>ZAKJ$TS2DQlRvN3?iV+ zkTk3Vl?FD*IMVxtP@?A+8x^Rq3{?lYpwfKOr=bt2>=h1g(A9&)o%;a|DLPMD6o_kz zklS)}p1d?@>;f_EP{fC38q}@=UF%Tl5xVud+wj~JD}BmWq-)y{dl+sC9Nj4RWyQZ^ zM*NNrh&zXfWKmxB;q4epBIL1%dqXh8J{)3SKw}XEhX4oue~}OlrS6bXBP1GhvHLs= zF;T;eI#l>0p7$JV;pFv_*~2l1lqahm$A9tvs`xbyO)Tf_$K7K+_o3H?hjmd`)9Zj9 zG`(ei8EZ>z%2Q&9ePuNN3IF}CgpUpbq&XL2AfR!f|4F}s|KI9YBn)j$tWE!4ZTgX zJ&};P3V;F7?0^R~J$1#kqizpWgf6+gacr7pNP@w^jfuVMbR+LC)!nTz&ah7`E(|UG~>`?l;8Ou8BRz9csGRp1h?tR(r2n zG;;0*w?Otsv3Qs%Ar-j_P;xk7D8GoDpwj>MPaH`!v}PVMnG{n^jQpGl@zF#9?=Z<{ zcrb#)14bw0K`Cg+neG5(BwO)q*wgShyQGo= zr=XqS3ANCOgjtJl_Ufn?=2EFw`n;KE`Z7}jzHV_5ugtTi>||gSYKPeUC2G_Wpfy1d z#C$$k24g3kZ>f5Y<&t^kL#7E|xgf$-bmEa_)&n*Q0LE0wJEE~xhLjX$lOgAV*X;~j zcjOJ^et7!l?lfSZ9e5xqXrCTLz=IC zsE)3nc27&mVcyX>Se`5-B_spWvidYuBRxgM(A`4%J7L|J?wVu7Dc}Cl*mCv31B$!_ zc*%)|LqMAnb<^m->Z2Xl;U;Ta#7;+od_kB#`zs_3OVoS;Num@6bjVehyHZ*-+qCI2 ztUzBA43Fm~Q6)`yj2?*=khGQ1LuC2t%&RuT`0s3G!Cx%n8sD|fz9{@Tw#6W?A-90= z#rVfht`8(L_S3!(uu4DgisKIn8xVRGX?1d?c{#kTGveq{Z1SFn37fmKz8?SK37Gbc zJ=0mJu$~Po6-%BuKwLbR&V7M5{&AK0Fp5pS^(wS1n%#Ox`~s9%*#(wji?N>}^kt z&S92jE>%wU`ek6Vq37hZ>egti&kKRNU!$7-_dvN|nTQ}+}ZJf*qvljCrqsmLt}sCi>bI4Yg0{>qh;Ak^U`e<{42= zUxZedk5_j{mp!`l1n-r;&0aap8(E}~YJ#PZdW@x#YJ#nldR$2v%F8#j=C1yMEN02z zfra4VHoAnm4V%F8xxkHcVLZy{4p=WHYfhSVGEn$blBW)4$Wb{gRl) zn6}p8Ix_62;4xgPxupt{$F3MK&IMoXk!$8cfdV)waFB*o-n*!!Brul_FtyYrx@F%; zY%9O8c{-q>zJn)N@`~mRgYx!2dPAH$Bw@QZM!kcw_8vW(yQdrzRI?sjiV5-^QM>ow z>s7^7LX+OtN`zIqY`qV)(S zY^=(&es|dI64{Dv zD2A1`ge<|NH;aYqvjhT)ixJ1gZ8nVrY3!2PYzSwxliCDwh>7Lcb8wupK-kd0ra;;G zp*alml?8LMJacw!czwrIw{6<()@G)2?Px#4rVI5@#sEROuW^Bm z64Hrv4*3p<#mumNZ2{-@@(fB?jTPWPJU7wUM!#M^yws5Q-vq`Bkgzf+_1Vr2DX`r9 zw>z8rB4D?Uv1R?i9tT!g*F@UNDCLi5H1|Ah4ONSfu45!U^F)i|lkiM*q-~0Vf|8so z(4M0Z8p!Ba%;cQ&RcjQVbnY8dz{A2i)ogYnPYI$aWfl!vnrOkw#I#&yMp2Q;S8gWc z0e79y3P$PYQ`CvQhHTl~$6s{;>-5<&jZd0%rtrBEof%uz+{BWBkH28tiM)phCnu< zaqnNL@08gJdfko{6-8vo%4oyXJfI?lr=WD4UJL#*x?;#%QBp(L)e+pzVmQQFZJOjg z!CG#jYa#3QTw%GHqS&Nm*_LS^Q|XCxTHn-5Xv4@WJXfRPN(Z7ax|=i*z?H!RlF$z>wS4yH0;lDyE)#xuEJn>RbV4@-3)!M$KS zgl>%nyZ#6kJDVr8Wx!Q{+(9FiZ5{~S>l>;Ntm>m6?MzNdq-D0US`uutkapE*1c5ef zOBXJnbUtUoiD44kf#86H3MfyzJEWb$qlMI$^|4n20l_>5PgJRT>N6=vgcH6_>lD>Y ze~{ICK@~}!u3M30B0|vw1IDErQp;Jmak+bAKdnuEeM5kOzg3_g%FV&Pty2p}DYOl|5@&47&(c7skMxa{Y zQj+eO!SZRvaGe^i|3jcFVbN$lz&$>kHi?&Fwv<{(>UvD42?|xT$jZ~=dWP3`4Nvnl z1I%_o+4TxbnW@w4$#@u!x@YTlJCc6Q$D(y?#9iI}i+qO6!N#9}9~CQ?Y1E8SHXfnz z&MnFL!sI{L@Xo9(s>^5QR$jI&`WBjWwY?YL?ga{WsFbjPJ<+GdkB)V=i`cUfOzYrI2t5OKze-RrL*bBtB6CAX_l z3*qu3yS-?x5&arHSiN>n5$20s!~6$m%4*DzBW3Pyvcg4Mhc04)GT2P)xtlDCJJY&J z%vY1J@!MPx)b*{L4vqrmeot-~>*ZqLsJ>mUkvMV0gsOcN4>9ZT7zXQ+v&g|C1laH9n zQjX)Huvi(sE{h^AfgbnivTl5Jq4MdDoD{~^P1%tmn=lo&h>A#^PnD@CCkiE;7t8Am&P~eWhV=ypB zQb~67aaDji#yBg0o+ch6K?Y8+o6SxL?~UJcHoS#&RN}R5ehEZ)+*a5Y@p7M(LM)Ea zGbTX~)kufzDhQ(-QysD+xL3lC`s0&Pa>~MlC3ENWu0?&=+*jPs1AoBs96}h6k7y%z zD|icToVsC>p&p8?p|zhk7X7l9gn~1!h4-Y_Ny9Hf#is0$P1=Gy9b=>#>T$ghxC;ih z3*Ne4Y{)k1k2OBlhBKJjE2gzKSM1r8M-qB($UBV8%v6@;7+z?_D-_460zyt4Ucip- z^D)-&Xnn?UEw#z2=YT7?t`5`ANs60wRI7^9!6D;v6@jmZ|6J$rx^(cq5gIcvi4 zh^til@E@GnyA5PPe25#dx`S1y&6%AC*7iA}8oub1?Y<$6-TtwMAD`mg2_(HxX+33< zJFmPk@7`_IqmwEKS!?S8OM|Y4@Bgv z*H{KU!g3gDs0f+2#DCxdTeKJKc-k3m7;cwXYH8+BxxISY8cVeCE9dqWVa`Y|t+9q+ zXC?4XP_fg(ZU?TnwhM7v6$?FYyXhhndjrN5(ESxra($3SKfV~P7ii3t;`S*1(Djvr zwSjzo5Z)xWaQp+^ussk06owJUUlZccun8=bne!GCjz{^zl#o$h;QMj3X$NkaZ1Fhz zg=3r4oJJ3bJd_yps0BkJ{2G?1XQ?L1Gma#xnbC?MCZ|c&g|FsLvpz%_IS=fySfsIB zVJvPrTy8;0&wxah(3UH}H=+m9^%ZJs(~`o5^xzn4aB~FqdeJZ=^H}z4tNz>g@Ksi5 z+xJ_lgf&>9d$bu7>AizD_bL04WHIDXWmlNF13#K_RM!XWrlSKxmR#1V%;>&Fw9^3P z6qH>PC;Ky-c4f{#k){<3O8w@#j_}Y)pA*%&!YPZKE`l+Juu%xI*rBa`7W%xX2%nje z*~5^SK^Eki3|o!w#gzdDW7xK9Jv-&PT|r_`4Al$##YUIPa@zUpDcakH*AvQ(&+s}# zcvBq1FkfleD>+C1kN?;P!9D0FtoTNrec(4%U1Q%q@+UQ)iQiC}$tm-TTSBof^d(}+ z2^6#kqsu>YQOjwKn6f*(OwIi8I9Z(>sqtR{qu-LkPlAzir{pFm6~wekQY@E5tmi~9y$Z;w;zRF+lqI-#ngiAd z?u(Z%n~_^aPlF{|SDEc!cp)tIadlzEebC-Kh0q-&dH5!AgIEu>$rPr7Se40`;+o_k z9czc`EjP>C*rW zO7_O=iG~P2+0NdGZ2xA&#%4(Li4f!p^v4n|$8WP`iKvNZQVn9(e9;Gks%GTKW@so2 zI9Ew4hqz|=E>Z$9QvRDHtw$FvK{XEz6U!{ow~q9Lp?9F2xHJo~II8GFG$cY8ml9** zcx1m{oC=>T0Dh&taQftp$*>bvP4_JqE^4e}oHhsF4{-GZrU^>j3Q2o{l27Dz4=_X? z>%T-lDjB~~O5!wZ@>{(Hvl0qG1)kbD{B#1BL)3DNSp|euan0Y9n?W%PZZ>H*m^C%T zbZy9EtV2RKiflIu8Q5Ahti5QbK_cqSJ1$iqKdpJ{CHrx4>g)xyQ-^|6TwhJ!bDCY( zzF2DW`vFaFjuTD766jnow9&E1yR5IJGPx%%gINy=lbg1>g9`QGBQm={%kTqb!__ap z|6~Yzf_N8eFhD@lX#W#Kp#N_ff~=|YUptfk-=O1bITS%u-X#PIQ)&$XR7L8yg`LG( zg141IfIol~%-28WhjHU3)^x(PY-5M&-sfu!L6!!P`3BS*ZD-KkWaFRAN|cqu>2#Xw zHJ9V%_xpVh(=SZJSUzZVQP?myvyWnT7P_~{?om%(txIGXx^`TLAhQc^eE=WCIDLyl z0Jp=qU8?6o>s!AA)yiKT9&7b(j(Y^#^c$g+_y=>;=Vl{R`7~F4lzoL2yfwkw&;AVX zLu1DfogRDesVx@8Y+_X==>|OKnuQkXOdY-VD51U0&fA3t8AoIa9d)~&H!)iYD=gh= z1yP4N`TpoYXH-$?5_&H9C^hCva|1^*g*aE`0(O)xV+^beXjk{5LnJ;OznJPkEtQO1 zuytB9q7ai%oXhy2q=K$B6btQ@LrYO=7sLVU7A*lUJe_37;3x1;Z(}6Sc!X6}qoMShERRIvy)GA&kI+4FJcPJ3|$Q)_jK!ZhC+liBC+)CHNZ0u$ET6ME9Mv=~k{U zkBks9`pgiq=yXPI2q2Es!35ZeCt{FHj0iq#Nj#wN#1LMFftZja>Ol1*{{EMbP$~I5 zQV=i@(Dnc5G)4M9%=iBptE}PSt+Il~_ez>1IVg}gA(RM(&DL}%uoEB777SGM#|RlY zptF0rJ2+q>%OO?3x@28cy2j86A4_k1W0ae?V0cp>88te%~xedlh>4LeAWm+es3Zn!N3 z2f{|*cHtu3K-WJ}AJ+ee@()c*H`Oc&_bm1y@_L2SFlgo?V_Ey&q8{MVk z^<}h3(5XE}(PppJpxSqQ{wShck${y>7PLsLI`@{MSV+vn6EGhBj%h(s6#dU_MgW-% zW)?bg;B6!N0mn!7I~tw+!)$wH`L>W#MN0-d#n2eH*nfwjj5?AskLqz;GFcq#D=z4F zg50Q?@H&Yel_~UD!ONM0(J1|XR@avV86i}b+yir_<+GOx2^I`CoU1NfE?q$|R&c3~01;Tv@XRqx@6l!)DP7ku-ePoLWOwtq+F=^)XO#OH*` ziW~T04Fe*$nQQE3O=%x=hrTd2C#if8(w?42-4?HgH3Aoq5U?1!gO|}_6xkz2uCgj3 zHdaM*Z7oxJE?bQ=4*0&%_SE3?MjXJ|Ng0^u(4WtU$?y|<4>VpUgS+iFdjTPg@6_k- z)9L4hNK^-*L>_`1hY-eMbkCaAh!E#TsB|S1`Vk^dp&q^u7o=$~mx>nFDs&qW*xZ%E z_Zw;g(#onEaw%KA(R_34zC@x zQyN@akRq^s@kq;Q#bNPhqQgJwm()3*Qv8z#TeAXo?TIhK2Bm#)KG<{DHnw&)4y)K&!u**_X=iYor*frc>X|Cj|2|+YhuAh_}!b zvgEq)sT6lqX=8GoaN@~5iCpJl9Do;A$S=VNr(Wj@A|L4od;`3J*?A}g_vSW6kLL_an3 z3w{A4x!CaYIHQxCvYaex34_ei4xy6^)VJ@-VrlkImxi8?RUi?GA7A|a_@HtX`Wzh( z|M~4(nwY4{fIbH1>0=w1FUI~BF4e%E-@g8-L0G81c~h8d{Ad39ll&Jg`jVBQHQqN& z5miU@WC#-9p;-olT4l5k4xz`J?h22fD*o%XgVu$EdYp38m@czi6wl431J?oKiVdF# zxpi(n%Fbg_^LkS8D4<0y{&m4@E~vS><5`f4bS9`vco}4>$DR6l0c%;_d;zFs2`o^( zN2J-kMydPlOl8^v^T?wJQ>-DgPJj6eh}LQCs%#_d@4K1X0%*=VMX8)esLoee37w>k z1-$APlC&_Q6DEhC{iYlYWpY5@_@F+ht6V@|xLP38*cgn&fg9vV_#|MR<|nE6<#Xd3 zeI3`(o;><$kW!aSK*uo&*zoJ52-a2&K1DjHN%@MqEVCP#tS&I22djIEcK4)N?4E9{ zq7J3|K2?!HUHDI}pt9VK)O%c+i=?SX(Q8*mM?-Y~V9LTR1!e+Pp;5aDpa{=t_6wN-9$SMt~ zx`Sq8p&gpn8Jg|47_%P8zU8^Qy^##=#JwOtif?98U7j5FLRe zWIcrbOWQ&$yBTmU)TLfGG>QY6#Ln#4yevdr5_M*T zS!8R(@<9<{BCUM1s{x&Uv{1T}!AZF4SR+*EByuaW#vgDSl|i+Ch%|+f*!X75A{yJa zo**)I#Y(Qev5-+3NT~`ghR$N~x~}mse;knn=rWK@WCHfFIPfH_66xB4f`OQT8QCkqT(h zOVFc~qC_i33YQBhROfX*uvaAUM>qasTlhl6rgn|LFqo9{JP$~ON<`X4 z8Pb>gnw);CU5hd9pV6*zq1)4Pp>3u*A$L=%DF~H3ws@tMf)j^LKNQu2(-)kI(i6r` zUP*qcoR+!LSEk)iRfCls_DcHKVtHcGJMz}}1*-8P)de9NI@$%xza?tYRxMnc`7=en z*Tb-jAn50RI^%9IiVYPg7F2&>t=1MtmPJ zE{j7KmL4aVq8=`l6zsyu+5&%Z3$KDGKI5cOyJyiDcUQRYwP`hzwF{Rys9+MGLUq%_ zh^C1$tMJg@E|2wK`Zg=8R@8KBO82P0EQz#xF&#E=jh@IbA1~qJtVhoImJwBu9{!U>U*5KZlGzo?bi-~aNm4TFrlMt}tZ`o{mice?yv z4CDXEwfBK=Paavl%ka1iCACIIhop3dq$DIPNCWy4NJIfeC_0{KEd^6_EWqz`u*zd&5Z0#`fh=tiZWM8TluK^e>i!kCS4eqO?S0z z+cuxJ&DFMT+qP}n?$x$!+qN<9w~zK5%p6Qrk{^(&+_`g*i%uVwE2n(+sWG#0L+YxS zDIAJm<$f}CNn~6<><=i zatgZv(Rq4wY$i`PhL5ol?G*#ZZgzYU+HwZ#u8R&5r%lLjrux$5u>KEBec0@j(Ojje z+0fU-Bf!r0A3;Kr*Vo9SFkYyEpPoWOZ>g#Gzq;no0Ghq3yHn7}Z)wakWibv%s9B#$ zpMZs)_wYsms#r^r&sfeag~6w zM#uk8Te;a>L3V!TO%;97t!a)?oSk9(T9>1R$|KV?G2pNrTWH8wFXb(jP=;z8o{;)G zJgId$Gw0%_xvC+&w!$S~qY;{4G9VK`gIHeuyqu~6UatkZ6ueQAO&IIhnodViPx;{8 z0Q|VLh=c$xnSc((et6XouveOj`d(#Dyd{v$|3-uX$`>af`ao9AXaY*0rd<4s;FNi?g!*7yDNXwZ7F4JToLtaWyyY}XB zr?XgKTd=0%{ulL4*3SR^GkQo>!9+BLvIR-j$eYbX z)F{40A@Go7k8BmNs~0;KPQEt4J&L-lqV@?geOdW^40%G_`xolnWYRvPwH&<^UvFCz z4#6_Iz0&fxuyLXz^B(joIWZAiaj_Vg4-RUtl#1rBMK&rYx-xOq zdt}roF`lJQz=lN&S&&wBA^R@(C>(}>3#zhU3=Z-!u#2rkpfNR_t1yK|+Y(%k$##e> zCg~-;>!5&!=6NvN~|TfZ}wW+KXS;~q4)7?uSW zKwH#a8%nK{NK43>4m;*hJW*fEo;N7ACg&_)0sGTB@z+>;WjSRsXd@eQLTsk4%2g@4heO&78xB~{)qjIewN6-amCTto}gD{+tF%NvPRj_6KpTG&rsT3zD5CzTuXVR=CgC_Luamk`x+bq+&(jK z!ldR5`7X+081IT#Jtg-!35h5Mju$Sib*_S-?(#UiMWnEGjOt#NPLKS+`+jI9bx0Gs zZiD7fH(BA0n%bU~1hU~eRsI6cj?9P+oY5`B?kZwZGqeZ81&XS>E`yL7o&+X6m7=Hc z(f(PJ9|KF0lwFC~^iw{UxDlK}^qzLjw>A@&ng5<`-_UPjfHA%{;Xw9eG{3x-&Vhat zH|qKL6{BE$9S1qqPt8E*Ff`Z52(f4exGW@-5mIv#lUP38(})!D=2^uzigptw$Q%PD zbj#9l=3gjqT?+8H(eI57tkW6HnZ!CSXDCN9(C6tZZNbGd&aRI^|9(bQ-Y=)1?NiQv z!R+9#W12E?M`lcwRO_NWn6^ncv9Tnu)}uKs_N|;j;X-Jrs;s|FTtE}Zv7aaV=kQ^v zc8U+S0qAL6~Sknb)LE5%k-qan&U~tvKiD-t7v= zp{e3dD`HvvS6Exx>ZB9X`}pSJvChH;!(MxC)bxbX-eRnxkw=+2^3hRHSN~O=heHmN zGXl!#uh7aGa?ma-bMHDn`(Z!3`QOZa8E*6Q%q)Em2VaP`x!O#^f-8Y(zGLogA<^ zO+G>WbUd3VoqZfi-^hxErfHr!ZVp4s4i*uNRQUHJuw=$Ssc3DTmcXdBF|zWZk+_%* ztWs)Q1j@OJB}6ra*cs=Pq3$_BiyBcM;(}U4Il?J*l7~JJbu9McYNA}hOg*rT(HXX# zNDo+6{gvS!1B#9VIe^EZ+#Y&_oce9e+whPW(3}Pwl;rvd5jollTWXfPOg~mNRfqE9 zWYUtABE>F#Sh)(xuvJ~1SY~Yt*6goEgnaEE;5}c*|G?vuH_&MUq$+u(ToU^1)MTj< z)H+jH{h;;V zneNyI+L|_xsLf!J;pcdPk%&z~9T(-@(6-~cYr+)QHAf_-fFZ8SdBiztbJgMG*YU99 zOhLj4PY$*wfSz^?)k!spo0Tn)o}&g!9yzM9YHhiAqLzrkGoX&EN;MZX+NJ4o?HMz# z2?tT{KtIfSV+|~rKUs{C8Utc%Hk@AJ4s4z@rL$QQ84ansbg?2swA~yh6o#`(K!mJ|6`gC1d68K|9h?8Ns>Xw5*j8T?yw*9NJylm- zO=hG^RzXD(O{;6kv?UoOICBm@A*Jy)6Ih-~%vVucC}qe&Wtfe|lP91#W9OQDYKjow zs5^--k`vHz3Yx2yK7s-SzS-|8wqG8WU^DN6c?qKoKJoYQ2of7niBu%|j11SdHHvNF zj=XtKf=gTtLm9tpdfOVIGjHj&$d*GZTd%&m9p<5ba8^FX(Sr+xZCC9O;bORTDQYfJ zNf?q^;HPv^s(VdbX38wip{ZZV?Pu@dag)2e>!RlzK3<&fvbJMm1yBR=`y zUZ`AAyD+tjdSrfqzwtxDv+P{&bnb8?^Fml3o;O+GPs65TPvnfFS0v<=Q8|K4x_?RA zLbFR()v*y~+Q>bBx)l1(^1_+1&k&!_f&ZM6JkU{jvc|>4%1JSr+B4USnr6c=t)$~&qoJDX z*U*=f-O1)@cQ_HR&h|j4pe+q9NpSk+P%x&;4Q@XLXBM6W^HPTMF^LSG){wZiEIs;x z=t>qHLIM86z$)U5OWeM5;k$T%8TI}HAPPTYF)$v8p{#XUgzFBr8?i~YTi}331jG$^ zUd>B+j-&oq_&S2ufc00cmHYcOI5BMYV9{ska8lDlb@H10{lUNs>h=G#VH*nB>#hYT zhI!3xa}v(1zT68{)>vh77g~m0xvSB3dvNMQX?l^={Q1y8lRAwEXmDo^LQ3GEayE5W z#3O|zyTu*mc8KmNveLqhkitv8)b8jkXd5y^02E7jzX&;$YQ5`whpd!@e))KyWT@!U z$9-(`$PRJff}FopIy8Ff`uAiLhGxAc_4v@+t#_069muW5zx{z2J>X=9GwWy4-tzzG&n4xUZR)NK-xVR1T83LIjet#6CG$_vXp8i*5Mq zOLAADz-Dj}mO~{Nr6>qCljrNyKk3@a{sIQfqRoJen)kp_MQ$}_Zd^GUuy1=uT61R! z%2aSoLW2c*Q+u9==XrM^I1+2|#klgtCM__)BrA8k{~KdhXF^F(+84JA>%UW4I217p zKLWmA_lB?W!XN&EWU`r?J|*(CPZ3&jGk>DNcrPPnbEtVFsX_A&dEi&$P7vYX$jX{M z6)pBv$mzhf6;l&*kiXN^@qzxMqLX#kVZAUq4=<&HVfP5_oKtVr@I)`H_JY=k_B0pK#_1w+BkBd8R3I~;VA*jr4yGbAY|IRtXsV()Ue4ow6A33@RmJ5F1qXBt zsC6?sJIM=L^Eu+y9Y)Qm9Y>ur?^sMzxjvQ*U>r~WU4~NQUJC`vJ#;Gq{uh2jxB3t@ zzHZg;MRNp8;{b8GstnP6wnr0cuU=?5z*l%4Px%!gcsVx&mZCDIAXEYcot^^CP%}iN zaz`W>%`=3c#L>>H*CfyF9(sDRD)sG(M=eneOAzwe8KCm_FlMP-S1f-;rA)gf2ttp# z$7a!xC#bsDv1Mrs^-k-phE%$3zbBR(srgNFOIkyKPtM`dh4Qv5cUSOZ3j(T&MbGcF^lRwADLq6}zJ9$6rSt{thF)kiL_6`|B!f4J0b!ie>2jD; z(mdIlSme6~^tM)0UU`<`t;9W&SSVn4b9U%fvixdnZ5eT34vmt7xi%w`CB;oTY7(HxLXdLUCFQB3(?!p%&#cscd+295pJfO}ug0;vhwzL}*_jro?3r+RzqA$#eZt$l+1@u=6GBNFXoX74lR@e&? zi)7RT*Lp7&5i1pHjBGG%bp~&ICrg2@T#KUf&X~`=FBsmpzjcb9TubJ%OW|U@pcP2( z-8VzY;S|cFQF|3>kLFCx+O$FQen4rXNTqa0BT=ozlwTQPp71E!Uu^80B?sylF5D3& zpl*V7cgy;Oh}Tw3N~47n502Wa=aKk6HYPRRzsL$1fKRs2`2Cf$>ag_f8ka!3!oCHr z#T<=di_UCE)#k>*sEOstt(ATy^JmPB92Cg6st_~5CQ-?{FlET>0J(hlKZE%o&Az-KWmZ)QOPgj};Q0p=tg=5Kh z+6TFd&Zf5$XIc7(+R7drXN+Cm{IW5x3Leqt%bE?PWw*=^IkjD^i_W-@o2s5!e_!I; zx(6WpQgyq0O@LBAzeEg9^#`27`(G7)*w85N;1B$Y9`Op_B9+{d6}+O47?s__rH=JV zUmh96cafCSa_>Y-e(B?X)OpnpJU|R(07_zhoEfBd<&@JZUoh%C@&}+d0G{ceL%+eh zJw(jCflLNPquFh^CZ*Q_4;D9J1Cl9uS*zfKC3QbB6x!;r@}aed6u5i}b|FGGxJyeM z0|iSl5?V>wsMX=q#wGV|BzmbBK(4T^-5)j0E+XvI(JTmknXZ3!Ah7bo)kvzPr8j{o zapG(PWSXVdE$3D&q=v2$t8!6*Zr5M(c5e)A7L%4~>{y?zKQ&5h*dz8Lm8+J)mtCs7 zu+gipebXb2GTie<40RArEk3l%R)7II;%+#%CUiHYT}ZdNKsVO@k(^+tALde^Bl*2| z{lmuvRoVc`cEopY6vcs1pRsYb z2;>6-_CfRyBJ@-V-*ifwVO4r5QtnL3gr&1WXFw{fLI(tPA1c`a6^!)ep01?9ktw;U zRSf1@4-hS0Mfvt^et)r;E=8ZyWGp4a3B9|>=mN=7DmhjeT58l{yi2CO8}ZLHBvK8YO}ZX1dM~QsyGi^Q zLsE{{>}@_U`KpLC5*Nt$$vviQMiXtZsI+3+A{%hc!4x|%Fqb!R1o9)>cL5T7X=F{*O><=jH z!KXgQF2dov5@tO=T+_RfpK2FOZv~@u%J|*Ao zvM?^w213jof#L~2tAAKeT=^{t(UU9bbw5I(dN-`i8_is9FL=(|3a4ipOJ3FK7WT^6o6?2vCcFk9d?mletxcZI%fU`YCJ zLsY*LW3rSN7RC9O5nvGqtDIek{|79bIYO3}T77${5XYS`z)q+Y;aNsRP@rhYFv}%N zV7s;S#?U4&NHB58Sb3^@D!YPq%=QG&a#(A>%0pHgzREswNrahQozoBFGX88iOWm-v z!bDS-#({UIdt`+H_RocaYLxnRsHGyBC9~DN$y_Ey)Mo^0w3;*-qOOb zoA(TCVG8o3Y;xsn+Mt_l;W_<`n52)K`r0^^ z(Yc=g4fP6@*@zndOEN z69?j%$2C+K>h;s_viB7g|JX^>I+yTO1t z#mL5IU5MUHEkG_dpYODrHT@1+4{RgWL?+z`YVkN>Ob$5E!btAeU(so^!TdV56@nK# zl1!HH%vRkRfW^?prFMb++7`*T4VyB{qd$<;C;wukKI-i=|KMzm`H3T(Ah-t;gorDI-7I2u6*{QGqHuSiB2T3}+E8G>>OSq+j~;;Z`a>olN`PBgdD`p;{WtmM+IAg_DUWo(zY z7{*{eT`hLhgQOEIbu_d!-&~Q@Kzt%U-ZtxAiPr!=g{6<@HL>mLN*?9XU*|sVP_JQk z^e#t7ExzADuVr$};EtMF1icH`gCUv?y5w+X7#77ItXdSl$Zc4ARd5Dxmqs3BHQ9Gd zWsl;_IlrLj()g=nPZ%sbzHT>Ae_CAGf3YTet==)^(9P#J{%HoXD@iY#LZP|vi}3=e z^QJ@y=yGL)ip+uaMj=g-?y7u8)0)wdQb>}TskOo!d$=^*)dG1w?Bb=lZdLJ z8cc~&ki#n^SMmHcggR|eXV9otTcyCr&7;{G4#3Z3yk8kWXIfN5L+G%yyQKd8FqP6a8dkTxyv98O5Z7Q>9%GyzbWR5J7E0s*dk zQ%e&>`6>b}s>f@0P>7Q=YBRPmMy8f)LIKDl6B;2K7isM(3x6?!ZP@xznj%2)0nzMx zfmm<@>u3ZmmY;`m&EFy|-?r^#qX0!%p+HV%s_+2Pfj{IL1u@WrwaW)$7=qEQn29YV z@r62|J}prO$*s6EE2@34HfU4J^a04%it6P$;3^v_u?Bfn?9!;6ed||XmQBi)NjKD5 zs9WrDshg%g1FbpcHlc!$FWor%(V+6=mg@4OpmA+$>Wb;=;m}nvW|*o~^4O`QF4}cN zn4vN|v6Y5d4kvjY5G}=QNZ)SuScblg9EgV)Z4+@+%QO~di$fAYB>l8U<{hF zM7{pIt{|>8i`Sp$bUTp}o3jXYv_OJdw1}iT;bs}6_S8syanj!W_ibCq!8hPM2HV&(+USKEZ z;2|^RMPEw+mJum|CqexsQnMW1cQmcFg~*A$Ewo5AL0$bHpdKL7X_Pzd$u+9P_aqPC zock@)CA}PYMrS;N^RTcb5d(?k9(3UwOcKf>`UauSEb-0V@y%WU$$|5bj1ILlV<#w- zGFTnydp{&5Z^YqWle5C^NX2_>!#5YyZ-9P>Dg(0r> z=9E5BnVV2hip>$V%Kv?x2WFGvSzUL9< zm}X6oj-)`WDbuOH4u+6pMk<(0vJVRgK>QjQ7?q^l?k9*J{a583E*YpLN!R?q(`*|aMoD@H`uNWAC6o@gUqgZ2XEI`z7dK~ zLvRmB>u&qCazyMykY+805nOgXH569CDh! zPW6siO7DTyxb#~=m>l*P?TJs)*o4%Q3`-32Q7IstyGArp`{}QuB|DG^XYE|e6)Mk$ z#ZM$~@hQpm3Xui&0wgYyu3rf!HIyZB625OaP^m3A-D<}lV>cp9zX zNzcp#8*`(qxzNNe-{Z?L`dEFiQO}$7L-=rN7fuK1Jdu_zebwvI2IPEzSkGY$dUfH$ zo?!Msy0Lo}G5V2wn7%3*1WS67Tjm*zB|nC%vwE^iF4F2{eAsX*^hQe0(weY7Qmg}f z*{YW6Or)PL*GxX1uPndCRO@zUS}&fh!+LVB7jKPc&$HGAePFzFb|?7eZVi4H+8XMf zyw-c4KSSE1MbLlHd`t6m2|TK@}!pQxD`7S+SBCd)5QsK9o} z6yuWrvt7CV0e65Jo!%3pJ%b)3l_ zz{Cf`_wAt^%OEJg4~cPJ*9Z4uRWIj<$9`$ohyIOdy}BES_rX0^-U~B(cH1xb?c=WW zOVL$E5X$f@zX$QH^j7!7x%kYWZuM=2UgbBqak<+l^38oTaI2B|LxPk4J2Z0ETMzY7 zS1>5fKEIz85vWB;?HvTCwql0n2(IJRwY~7c+R#Q`R@9FbA@q&lMN+JE!vqilDD<^6 zVZq!cr2$x!US){Yq+g?9J`6HKSIg5n78cnh!6mm7Mcznh4wX%s-y&t9L*(eXukz*h zBbFENtX8zt>9q&$G3EDHCCm2Zm7zxS=_ZuA`VHz01~^0E7U3{ZaTMz3%)z8%R_ZtI z`q;!^w+w~`;EU*ZOx*WHi}1Qko&DRQK?AlT!dnyeObPPqN(Yr%;SbS+f{5yH7ci-IY?< zgsu|{X;jlvfom5Yp9F4#Z+0a9s_l!9`6@3IJPC>$FQ#HN5#t;tv{AGH<0vNlsgbTp zbl~uaxwzhJfH{%T(FCo7YiICx{(mK>XvQ+No}eL*T#CQpH@?S|zlj7IM6dj^?BbBI zo|hSXLDP`Xh`>g#XphBc*VJK(k$=1DwO|9bZl`zs#AWNNv%&&v`XyINFo+5Ml@I@w z59pN-NeqiK-bg0UVnkza^g)OOfu6CHWB&{jb`?-z-@LKz*Rn=~QliHwv0o_Cx#GWw zFrG}0UE@3-lxTIU;%Xi{ShHC|L2M(V$zn#%%V4Bes?6=|xkWN)^+aG&iNf_5UCjpxlHZd%i7G!G2q}P=hmpPnl7(1Kkg?ES~!dO};H4(-x2IS0KDvV$R z4$C+rT0I@y6r;h2FbNaQImYQ^kf8vXhRs(FI2lBrx@D8v12PH-n#MJ!Xde&u3|Ax@ ziVa$Y!0Y0ptd|VK`E{X>%TuWwa?3VV3&UZ$tLEB~fX#7*w=SSQz&qm4!sOaFCt*re z*IttubtM}COeB~9mmAR(q&WaRF;yBAD}aEF7*evPhUhH3XUDu{cd8S7sz6F;rBC@P zouc(}w>2VAHH?w#)n|1D#2ER}Qi#2*Nvkf{&~j7T8m-UcC2VqryTa4iTE(F0W6}_f ztAlMb-5O{^kwjPNXFHgz+1Rv#IJP=zWnFKrm~ywZtoLtf&Ty#p-Hxb)`x?DE`jkT6 z!2i)HLtCUl^lTI~MR`VO8}l)v$sP3P*fBSz5iy?9Q9mgR?R0JZ^Q{`rW*cTZqA4L7 z#nIOe5{lziSh%rANv^#Ds+x^?J_pC;gg`COEYEY2{CYsw z;HxNOCn#eVf_Z3r=I_zf0N3EdP>i$rV|O#`K^v#!X^hYs|NX{z_(w*VqibKDq8-B{ zt!=eoDR<+*)7`rX>(F8ZG?OV{L(XdC>4YWm z{W^_4ZmNqsM`OIG%AtUS>3!oA9j!2I^90$M=8qB4_F7C~8^i`yoS>Nz!xKBcIQKWG z<+Tsv_!@#z48{Di2lfohPbk-!M)^qnh0q~q`-D5bEwirw^GH)+&jeSj=Lf@?jW2-pP)XT0uiyR9skpCq4%j;a^$mL>+HTJ31D&_zmt!^ySwFu% zXXqaeuC}dxyI;`YH;%iN0e1x|oi~JKH#A)pkyNmgQnCa)ZT7vYoIHrlkoD^x5mdEm zT+|%e9&gsYll;Lc>VIJ=ofb~kbDJ^urIO(OqmK&$)1;mxTv#C(Ss~2724mnmhj(N)7$kX8?nP0Ut zWU5Hy4Oj$g#R9XLQk~mz7H(X}_RO4z%!6Q_#>6xo3F+tD3UF$7*BlqAsA81J_O+^v zP1_mOAu($#DlT(0?fYYBJH6}XrdW%nZW5ex^EAZ^Xe-Nm5L{-67 z#}o!**$D^ShSJ}U=^bQ0r}3~x#~ZxgDhfy_(3{#9%`|4PKlDe-yJW@@xk0NxhB1ET?R7jp0Rt$O&tCQVXLMf)2_a{ydrEuLD z;Pr>H)yq+mES@!;eWxwvK6Tw1xH|JtVXGx*kOjY7>IPmf7{#7+vOH`K5u+sZ8{moX z6>K58(TI9Agq~OXzO22vIl0J?JQB<@s2Cg`L~@@Axeb|=KBheFQ?c>-&SX z(_q$Z|KfQ~Qk&GKpkG3<^zAZKu9TWGeXLd*^rpzJJ62NzoxBy@@O-F?e9B-2a`u(oY?sT}T-N zBi!BVH?Wnj2b&tp3wF5k>=~NWYi37hR72CBq#P9VrR<%mEMPYjNu|QQ&~&i#Dx|74 z9)AIIQ2(TC2U)|qJhaY$0`#`Y5^@K05d8R!A~hj5ydNFkt9QFxD~K^j&2HN5#}CgDAHudQ$w$^ zKB(NgcFa*G;l(ue?yoU-Kp`ZJy3L~p{lh(W0Gl|J3TIan%=Um9%Bm_z^1ymxX9!a` zX<=OLLOQ-TRX48<@b|!(Fy^IYU+sr}lxyGG`m>sevrir!aKjELqO}be{VU9GWgDc{ z@NM%b=)UU>$;a&0r*=YOKQzPvH7K*aHL88$d%~1wtoulnYR)&wl>wpd_UMsn+S3P~ z$Fxb47zu zf(N5b7M^gEDa81J|MY>r7PdPr9x@%sN1-0uSfhTsiE!$1pf(NkOp9i}hpN@7|0>9L zFmbWZluNAVp@w)%T&Dnv?LHw(17ihaQZ6wm0U^yq;_Ljl7)`TjoH!fv{f44+x==o)}UdN5Mns!Ed_cwC{Vtso>kmF z2+ZuR>PQ2rcjF$2f~qdr`}Qb!h>C8^!esBmXJE{N+iHD)c}|nPSFBd_cd0D%Rp@>P z5YZ!>;BZ%*JrItM6)-cMU=YX2%Q2~l^mD}vaqKA>5LD5M zG}|Hu`fxWu@TY{}oF>VU$`(@=`j!ZB+pJx{IS;9#08j}6$u0;7tr5(->$|`JH|O?~W2tDkhcLkn5J+5AhrmNl zm_AJ8DPj-0#-jW977G1?u?-nCEhGJ98S+LyZAB68B>B`w3q)&*Q;_nf4rr^2j@);^ zam>n=;-M$p4}I)6roVSyz>e)~2sa<~_b1A)?O^xz&5i2kJHW5G*N8Rsu5`zqmvpy= zJi@FhSWH^KmFSH;6~CSwOaa$g3*8eui&TCCqa3?V0kd+7zlRwO?_5YpmWA*Mp5y#U z;*xPk?a6!uJ1QCgkHvt)rHMkgof!_B20~IT8E=#WmQ+1%-{S(WtfgcSk%EAs1y(E-}lfjz2faY{4jW_O4VgX=U-O?{We}1Orna$QUjyzRh3Dm z$yS}DGTLFN6WJ1+qBW_W<3glms57OiHM#!00_~dA5?RwiZ*W;7y-se$);Ze|&8x4| zkKa~n{8>}A4zt2`Wpb(c(&2)~ZM@yb*KBJXR+F`6wlaMMVFmXY=YrE#z3G8tMR%*e z{CKnCd78bic2?zk=LGrzb6%>a0!pyB*sH{<1gXht$KVVd_PDNludEzo`2yLiVV*{I z1=JI2jiEi&8c|kAe((e?ez@fS=2z}72X$s_gI-4w89|d2PW5@a-&VI0565qJp9Fo0 zPbvyRJ>W&S-m%KLJcOw3FSD>Tt{|JkxR!!lfD)IwPx}4X0quWH_|p$HChrxwx*E0W z{=Jxc^epS$?P(PMf||Ut*!ky38|HFIVLd zg^<_%^s9iK=Oy?gyPz3Ex1f3t)W%|nB4MnZV4J)yoGqwuCx7|vhYfmIegGx{2Oe{c zNFAA{hit*Bz6o#i!j1f6rfjw?n5u0cOIN8vx`ow41L8L%Jr-M}b%Z`y zpNg=lbdKV-o`INO9B}pBBA;)dnFilrtQC32#Sg3#P4PavZ{U)Zy1!Q6;FHz6`;M0g z?b9D9E1IKw@LxTXCo>Jx!Jp|*oDtT{l%yG+-<_Rdw+J%>n~wV2#pj`cuWmQ{7Bj6jnDxPBZxIc6+w6) zt#~l_++V>YBS18v0IJmJ{mzn>BQJFZlj24Pg{#@HlAD9LC1IMlh>($iPu>^%;|ptj zVP&dV{tmQ`t79)SDf+dzbz~Pp!RyZV#ZQyqZyA)O8Hqc0Wnm&P6|c9)kPu_gPXa`K zq6D!!`eCR6TJn{}@rfvNnS(b$PB0IMZwqMb2`_wq(0>CoaxAy!tfSo1bs?FDIi_`s zUt;C!IM$}(2>xL^Zul{WVVQ56hLB@iPu=hQ%6MP?3aHbML0m~tPB2vJ$Ym^FAoDKA z;1wnZZ~V)Vq%<@bdx1~}6kL<9cY}?g<|I~*9Lb9yuf^I;sbFn<{YMR2Bo~IqML{6 z*}MTR%@mEbuJD&8r;U6s#_a1eMoQNe)}=YKk=MYgCh}%!YbD^$YClcM zmV7c$YP7d>JHYXcJOh3S#34wacte3dTInZBGIFr7A8`{Tk}2ScZ)B6ly#2%(bV{dn zlu=NECY@2gVr-=8&LCY%B)hs3<>+Ib8mC=KwRv) z(2=lEPhsI*NVw^b^&a5RHMM^8Ps?-zJ&@Cl)j{h|@ViaG z0Pv@2dGo43iw{k8qg~MQi)Q;sZP?{Ccm9qp!sRtw{qPRt>zZ3X$cyQkfnCV<%l2LR zwm-TN4v*0;1jG6OQ9!Q0=04gd&|Bv>VVBXZzuyM;e*HJ3x5aIsT|5{6dgpv3PpGOf z!{gegx|RQ;^^vmu#-iy{1{U=`QvTvGA+pbaP)$4qR_gHQ{zv1i}L6iK|;1$g$itEr%G+qn; zY3VnZosHl4=7!$A@u!t<=rYvbJ6s)cEc8Ewd~fRWz|l|N3>ekjpN#v z4R*D_mv|b5&1J8hljJjUPUWgWMFo@YAR=bn)?!ncng{$fm-WNx>eUtpjF^crd6Tu zT1t`M377QmN=nu5#RLp^$QKVL8Ex%w$K&$A40bh!UP%i@p2jCQZB5Wq|MPC(Ozlq= zdYmp-koq@FO;P>I_{XPaVECfxB~FXTevnr)DV>JLdeZW2ztPZbPgXPi!Ruan+g+p1 zV!!M3pSa7d`f{qN$(0VvZF>W)OMIR`3k=9Nd}qTb&AKCMOc*A+_vQuoRi z5;aiCq>70mlFVc|l;<*%7*ZXI3o(f|$xr#k6lC3$k_E*8vWnzlg+w_?a55hba-Rxk z@RSdQ3jP%EzmDW&AXMYU#_^|=r9l-OBB)`C4wx}ma^sLvVv5X>iSx6@}Rh6JW$S0>kp0h>9^kk3bFG3gBDBY(@4HBqP6=xX3$keHk2f);+5eM(e+LV#U z%GHR&f6!<|8zLhy!VTHbtV9}8Q>%-Nk;tN`G=!$5sWimKLz1;c#)p$>%Z$O`VgpjL z7gQb&k?$QUzSL_JJTv6Zj|;s+DD|kgbjTK}M8DaHyH%TYDHbk8FX0M*#1wiYlkL#Y zu@iQL(qjU&L?3O4yLD7+QZ%~DwI~+2MLKkg-6EZXBmh$CfblE(44c$-^bDJn^=;-( zaZ@o2yFVS~Crd`*B9fk!6<5h^^QQ-%pu(vfTF&^3lagh$`;uvBZI6h zVjr1eui?zNs0S3{ZHk3&Vjsm~`G_ZrhLv5BkLYp_)nW*dX(wVI-KDO4S&!h9_*TP$ zNi;|$;5zj2XLl6UQJ zL=i`Ot+Kcr=p(L6Eh>bwp31ICU0`gavKA7-A+)o}Nn;9x2xnd(5iC+)Yh~em~!b@_#Vy0{R>_D~_)o zcs1_%(vGulR-CF_TP!9L_px)YMTkHTUY}BPSzM}8wUlZVg257h9e76hfDO70Aj1O% z%6E{{so?h*pmEzx)kEEM2ovdzrGU#8Ljn{s5<@YYF@-Qo1a6fE$*!CIaN~#qMKBcz zqe1-pBDR5{OR20n{v9o4x>$I;L!hnS!EQz##P{v%^vcdMRc}m}8mN+O5A0*c5(=s{ znEaa-8+7H-$SKt<3$ss_n!|y!Pystx9JU=YG~HDY&{<*(QN#3>(T|MuD>!83o`Hn* zX#Fkf6Y=zee9s-Cdf}zBFIVHbh@BZ>jA#~HMZqhTp0dOoVAOH~E_&{Eqhnk#v{C_7 zn<%;Iz;VJO?05Cq`mrAH0uSg^oKk=wtF-Da-zfxb;rN2-$PlpTQM-*Rv} zdQepv^x^>}vYxZ1-Gn7zz)tPju1+g)5@&dnC5v(u1xR-o-SKfm+ib{|5v#!}iJ3wo z7W_salD}odqwR|eSw)Lcha!FS2zTPJnhk+;9XBERJuOz7Nn=ghD?|fW0)n;r+8IOuxvo#AltC+8N?$z(IIDMz;l3vRqWQJbxWks zL)gvmd8Tty&?eJu-2J8B-PZZsLp}j0e!c!(LE&j*xLK;PZMhA<=ba~bJzwgSx<;C{ z7x2wH;9%ZQViV1^GOEG(1}X3(9g_FTwJ!{k12u15q4Mz^c;ZJ4r>!^zivA>K(%o3Q zN`1>9^(FdLBP)0(3JA`ZS^q3K{U1%RO;Qc%2l-C}{;#y~{{zY3|I>v3Kgpm(Rl{Ce z4T*QPY5bvyEs>?wWHK@aU}3B=p^iKtT~B9COhPjfNybUuFVDiT&?Np?oB?RWYs5Rr{f*Dm7hcv&>c zg~wjsj$haN_WP;rxBmrvU=GMQ$b#wD?m=9h+v1nCWz7(yoCo0p>qre-vZo(4NeF6( zb0OQ11MpdZ7#?m}v&){(dTt#;7T%J2tmoSPuhyz@X3*Lg9xcf_jS^R-3F2moLIC4Y zE*k1*Y>_95+0A{hP{?X{o-gp(<#c_qhfUw@QK4{-?$kFZJ9!qO+HA*j)iRL1kgf70 zH|ycSe}cn8El9_4aw4sbTn0@`j~-GxRZoOD?VrWA9GfGE|#|q?A@G@tm+yEh@h)hT+y_<3eFsF6|Cs*)4{ed6o7s zbsZma<2MXO-KIJj^R$lxtwElqz@7g*!pem*)?@Rz`8dg~Ju8+ugT$r+k98$iXX88( zU%k1w`kLLfjBYr#aE6G3I`Sr-ygP#(&wMfLbz7|v6imv){LZUK40^~X?o(+vgtFHKyM#rakOs@Wyo`C`9TikQdLM6xMcp6$^^Vsqx|WEC?&2B^R!Cz1hr{uCA@h) zGcrs4y~yJx^|M1rAI+d*dEy&569#V8JJbrwyiZ!@X39fBJ$*>l#= zF7NLUrtKo!>LFQpiz3i&tDFML6GOW>cPLnjo-XEIv1e^h1*Wa=w$7+c7w-MAiAPM) zr;i>Q>E9MT9R(c=u-4aId=0;!Uq8V4Ns9$1#0(ZJVL3$=xxsk>^Mc(q(ytHWc7^gd3ng!>&`jdhI|H!5@H^xLuVqE8Iv!Y|WN@=4{ zVO{Mem?u-u=CghNEaAxf=2E<8^c1?FiR)Lopo!P3e1ZL~VT#lbJ*VV@M=I8_n`LL^ zgT_U*b=oMn!JdFL(fYWaDi?azNcrmBLi`&|_pGs&-XncPP5YWQM!>xBA5_{m#CN%x ze@-v1>5lbkI;lg1*X^K1?Bm%eqzCY1$`f7E3EuN(jOE2bsq&%xIygYuLlt06eJqf; z$S6Zfc~gd2_G!_xq2`0Y@Sw(@)f4(`pV9b@jy!AnC$?5&qkR3Jb((9KAjxs;5h^sE z{Qm0C%Nr^iwvfNs1CqEetl^T$V~ob^z9z)QCxla&ePotgA+6l{N##7_OR=U@b2r=$ z5B$QvCW;+S*DW5lD<9eK>p!O=!%iE-bT*kxYf=Ttmn?nyFGslP~gnSQEwz~$GV%dfY9R62kb0WQOct8H!H3ofJru#4gEk4&$xN=QJO55eDJJ+sNgW=k&x-7oF(`y zq`M){k54}LVKo+dT@uNICeOGXTUl`Gu`8NWXhV!gU(LSf&!EmGE@XGknlc};Qcz$S zGXmZxuE_yUs3>&>BIM+w=7?`4nT0CqWB$)moauADjhyiJ7xvg;&2g?{^x28=JPY}( z)~Ol+mz`4Q8jZfYq(0k2j)%e&33JMZjwV4sZF7-@Jl>zvbDY;2|0u^%&GCl%rs4WW z&ttCf3diK)z0l}RUs~(chp{U<+~Gcjnc?FXX4{DsnMfvTk!6!;HKvl-_SSc zXg$sg^B4nGALHeJ5pqvcUeU?bNO`04c(^W`iLBvjMwzSOjHc)Ch-{nlricQY%tbld zOW1B%V`Qa_w&TT{&1+tN(*<|}%i4?}7FB;EN_}O(0e*1~pp4IuJYal~|9!=}1}03r z79>vm8d3g;ZGPd|i!}tvf@D2DHD6HuXZSt{e4+bC&Om$$xBp(SzXIcjV3jpzY5GcOVAp#jkvAF?wZdHFg~xvBZ52&9oG9KK4uyuqCmi{kp0M~Y6Sj%Mbr|J?!DXTfA*s-us+vkY7hCaWpM0-%{2u9H_D zuk$W8tg~*2%YRz{*>4%i85z4f=y4-y$xO(yTxY?thI18qA6;g7ym`*}Is?@Eck6!V z`xbnDt84bxY2LZ_;(L7$Wf(NmS4;O+)I|Ax#t^*oW-NC>Ggi5Cde9qgo-ZoNA*>H_5SyxOtOZP3tp?C-<(vx+8^b_o2+QnokB)nNxr>jbzW~iIS zPi92W)UvbB*ARH4Hlj+5yp*hRgKrJSg&7kWzH{=zvo=Fb{BB>jJ8=F$=3i6w5^NLzHCjax_lA;iboA)!=f^~_`umDRXu&0zWF zh%|>E`5ALunX2>HT6;KXz^mDl zNhjI&Xw!>zk8=;-8;1Xl9kbYs6ePrHz)yLAFGVqsIKxFNj8bO!J*dxFbHyH+v;(n3 zrg>b3zjnHDSk9_CU4pm>OL;U~W7cK0`;sQ!#j%5SBvcb11zXjmxc)LwtPI`|RWQ^& z7eF%ks8*{M{ZkTQ5lWTzjY=ibGJSA7Mxj z;TIbSo<3r37D?}a^h?CWi8@FlK>-Xi3zx?l2MBS7{{Ztq#9(Qff>e$aXT;vnT5q(9 z5~sozrr-YgB#Rt}5u-=n)Y#A})tyv?>uW>`B(sl)RD`j>7}k*5f-^~5Vt_LVDPe<4 z4-yW>qNqE^k!~|R6MA5!59HCccod0l)vICL|0qDi=+29h6snmk`tzstXd`6Yb1uko z-r+G)-D+*trD?8IznX){kEfQ;+N`#F{1>1-(Ln4=XKmc@>hV8o?OS@g9E8;(q%$Lr;w|Uk(iFbZa%8GCl=YW8#+GL($sppy{S)1f zU69W}A|8RvkB7&`4{Ad{;a3gPlM#^Matj9WHRt#aC z@kEIJR5DRru{{ot-dr+~gV98Y{XzRZt)uxwlD+9f`Gp6(;L_;39gN@IJpXtcKKBsU z!WhY3r}mG%&M>N6R|RT6I@^T)lr1fs7mBVtce<^+NcOF}Sl@MjoU|3am647GBTwpZ ztKr}1qU_hbQKb0qg2qM(?<3^b_d;e3FnbS1CXu@js3tbgq6JSwQcKq0^;hJHlo2p~ zm}A0LM<1}8JIq5XPb|6LPSwrS8%jSaDviMw7~N7BSnz)e?=kq z$@2=K{9bM;2imyMb2hB}5o|vNFQN)yrjGn@c=Nt_W+ys+&sY2h=?jdSFwM97?X*fD z$(?0L=quPg-@HY(mvIg@vVi=`8XVXnOZl7ho^DxQW~kb28Xc3cp5vIXPhv$-%yxpj zC!U(so0_eliVbg9=;-jUq`5ct=+PqpQpZ@gOw;43rsB1t5u>a_ z^;=PM@z1R$O03eqN*IEgm|EG(g_K|0M*8@bk_0vCNj^P59imEVf*Ot_pDv&dL1iA& zYL}ETP}2O(Cj-Up30duYWqla$LugP2tLwMzZggp0>sLc@GfX;~@Lu#+olaeMd3c)( z?59bSU<1`54*KW*rnz#z9!wn$KLo#u(+{uI{e^v&;IZH_yq<)8Kj1qYF-rbVuK*Bu zF!kYFP%sJv3H=Kys4y7iha$Q!_X{jA1t!9*dhqi=2S zvmyGx@}M&W#=tGe-$sT`Xue(h=-jA;!BfzhknJHjxUc$v7er82@Ek+|!waF)idKRv zlxH9xlxGkQTwTx{+iE~#`&guBC7NSBnqvXl9V-wg_#KJ~1_k%N zj`>jEdiZw|8q*~1lNhauuYPHL^|O0iU){=@`&-v;sTM!CdRi`Bz8Fou6*MfXV>8SkbQgi2?lf_9ibdCdWd?4US=IJeKB4vfbD^fSiS@= z9zf5)J4|1k7YE?Kz&k8ok{ADCazkO5d996&>u08tPZ za1Z^%9^?rbfb$^$+=KZR0-*qZp?{cy6d}JcKiGi3U;=)=;Ora1e__3>1qb~2s@>lU z3SfFc+5ZLqh4C^M{0;F{z26`7&G%vJY>?)DCbn8^#mD*y-+NDVKD>YKP9 z0*rta#PrSFm);+OA_32*e~|(y2PZ(2kbA*6StARFWy zR1N2n%N7mizm{9~uOJOk z`h-E)APwNwz!uaUSd71D;hs{_p3*s2>NywY8xrLkQkcJJl)vbXcz|jBt;WBBCW3TE z=1(s2R!M=UTpu%lAu+*d26g!3NhP~se&x|rvnl!2BOk+Cdo>Oy;LX@_f%WHSanjel z)H_h>-K^wYt3q23QBumH^; z1jFp(S@G<&4o?YP)neji zVi#knAX<`h)K~h{6`guR;FRO0lQ))ZimR$EArd52;kD^Z&}l!r?XnMZIc zrNc|lWH6T}wPK?B9=j^nO(|!YNocyf^%zO(QrW7zyLi)+5p9#q0F@*@3Ko&2XT$cV$X4DAe50(%p-@8xsF_gg@3NK0dD83AC&qW|ViSn)oA)?xncE&X5E7Q&1b zx&tB(=S8X_P7{g$X&DH-H&VCUbJ#M&s8l&<#))d{{&&z3OzQ%v1)n7Q^kx1X{-?*# z&89-}$pf+*a-PP_g-@;|%92L;o`Bnpb_Xr1v}Q$0Q)!*1?3t=iHdhZDug?Js+0*mLFSU z`!|1V$GaG=^S5-g46{TXNA{R)Zc2dAE5YnDjC4O41c73@2Tv<$ zd<7zHlMVt4>MyAOs%c*thMCA8U|cdqM-z{U-dF|H(tOT;JQY^W z)>X!+sU(>g2XyCe+;Ope-9K)5f-yM{I$gu0#Q*APlsu>vxQ=|SS?Vu9^7|Kp+vV{X zhXehOhB^E*0zy@a6>}}wU^}wZi2LJgg614ik7zD2Kl=4)l4J|9X6ldkU;Xj>oq!d<+BD9{=5&za`#dSi2=lC_lz>avhspcLa!-^*#c=E`CPv!)sdsP7{SWLm=g7 zDRX&UX_xOu{}PF^Vq)mLw&xQuX9kc^#Ne=9{xF316jrpS7qOx0E`zfhRy|%F)l^o$ zVW?IC(!j?rwpbB|#fz78%FzQkC3A_lbD!dRwT+(&<5Np$Ca1^3R;e&g+U zCY20BJoH2gIP7Qs-BqPJSx8eY&rqTwDN_qm6RD(L&*&_nSIJgUSJA7)NB%=ciMJGe z?Iiv;X_a;IC+p}eE4=Jhpt?wUhVn61*fXt6bYm>;RAeJ!SfZK?zU}18#iW>IQt0$&e&$Xe=SDeXvX-pu;O|Jl?=Y-OaO^;rmOS(@T=KrWp>w5dDiMI)szx_>~rM#feKwUJJOjZ~~|7oIS zmhm%22uyp~Etx8cf{5y(VWtB)lKY(dw!=t^{m*P9vYQen6bUb{LvhhjU&2(ZA`#Kv zZ?^az*GrAPZ8zS`v%AlQi!S$Sg*@IX&nn)z4-g0q442a4=P@VeN~~2)53?GUHck~G znwDnFMhyHj@da3n<*%W<2B#Ik>O7h|fw+XuY`) zjT5ZOiI!Y_88R~?n{!E4NEQ0!+BI9S%3{PXqT^~RtEw*L!H5V|t;jP)DAfEEz8@AR zFDoKH;~r-WNiQmuj4+458H&1i=63CwH$$f$>t70L{_lQhfehXf{N^ovChjz0%Of4WIQIHJSi{t;z8 z-^MQy5S1ODtXMd&*6Oot+nh`UYJ+!7G21PheAl z$iaY^L@0Vd%Q|@PZEmh_{kJ4yqy;zKYr-Zy8QtroiP^b-`q-aGq|M&g2dfH>W5zLC zdk>eQv)f9a<|56BIE9;}u-%@U?>p7WoyQwN#K0gy%ghD~)yyVM2GbRPh@(uw@=6Jb z)WQxcYOxI+C2Nnpw{hf2+QgOT%+EDxTL~>981DW^{IuYQGnQLSzllal^1X6|Wa!9wgEjWE2QksX9fSU@s=M zIQeVfn0Eu5K{)|tYo*7W&^hXBz4Ec7Iw-<5X65n92b5W>w*t%JLF{o}6-b<(J7me4 z6!(kf2bc3c=nPO1S?=;PNi=B5Hs-PvW{kx{aTcK+QLHljcqrS8G&ec$&O^0keT4J~A`LM5V1s)`9YS++VbtdrBSnTRUDv7_%Fq~)D#3=f!F zPs89btjTlOaB7!$`58u5?KTutUto&Or}>2)i#q@=ep#jF_)npTGw+l+AVLe>u9>Er z#jyJr=0TMv-m`gGB1(NC>_kwx`=HAYjI3M+eKyqldHO7W@0WzLf4_#Va`*QB9IFt; zLb`j!97qZF%y6!wlbTR`SB*m3gdTo+D({Rcc`=-QvJ?e(;QTjSB?ARHfW#z^4=cF?H}f;!_*<2QOU6K@I`Tg2z?mO5?g6QN+Fy6Y}U1?+^zD`swM6;E9fR=A11Q zFQpC!MOR}YV!2Km-)c_XkrUy;2w6!o)$ z1?O%$q|sjv{6jUN*v@c4M-#UVnADkgVK=ojgf(a5^+yz5!F$S$OEJ6hC>)v`E476U zWk;BgNa_L-I9ii2=iEETrV@h#4rH2K`Qpma0F0i+##p?pu;vN63fr$PW_45Uh_$RR zSNVAPy3-uLvrNC~w0MLW!7AhAtLzgh4XA3XYZJ~MZ7JjK9?MeUwfvLvX+zt4z1>q* z8E80G4@Hk&IZgbXFtcUF~>t!nTj?vw~^Hs$oSW>8S|`%*0I_| zjs!Bw+yrviiWsBoD2psFATj7TA$`S;8>?>1#jL5$#Db==Sr?^Dr%zTJ?^llA98@?& zbxSTUTt0XTvc^SP6(PJE_F^pupL(SZ5j1Nt0a&Ly2sw}1JrFR|A_;jg3;bs~Y*eM5 zSm1+^p?~hw@}1BOfI8bEQKQ6-S0c|V^0AdTf2XUym}1Cszg2s?%C*c{zM?%_fj`!{$2N zW8&9jrNY}Uy^z*m9D4M}40CFUcpX@EYhj0cTDV3lu{vU`?>NyeDh^FpmAuOy$>;Iz z2D{mQwB(51oNDElveP{k@~K8xVaczv3fy6}>eMQyxacfbOZf zWvk9K6==|;Xo7(#tO~CJq(xa~ zeULx?vzwb+J~o+qG+|<94+uhdi=%(eIERvAG+^o6HgvcQhEp>NQ6&WzPN;M~)y~|5uj6lCpK}^$qvj z3|}NxY`J!Ec!duAWPk0Vw@YfOr8$u6>1jQz>hHE!RX!l}NAQGln8h%W^R!B5rJ^|( zAoQ98{4uqj_UK;$pjZ2#PO;o&+#@wAac_x8#@v65;I+Q)A4`=@EseLit4q8p#xMyp{~iHp?mhbH!`iWuq+!F zMMvLDy_hk4+>I{WBLHY_kkmQC{6Ov9wlY7DYp~ozwOlg0J^D_8zWg+psepR-j}z{CgX=Q@ zZLjhvzJF=imDgtO zi#rg)%$*nx0(77Z&Ifv+hC?BXel>Z(5{3@KWTs|{1iUM&K(7ngIZ~ttD8zgOr}x0R z9GHxU$PTOdY;92>i!&1nwRyP$v~i7__PyZvTf|ZQc(ni&*zhO=>)v5Hn6vX$vg|(D zMoL3?fY`xfT-oquqB1nqpdU(FW>(>B@Rkfn^GnQ@wG)DI`Ja)^%O?P5TT0@gp*xR* zTah8rP}$_@Sc+O7x2QGH?$EWeG;DIW+*{9|{Cd>_@>7_MERv3azZ>@PX-9jirr}iy zgoavH6g+fA%o!Q-$hgEqvH}}*0D1s>@@sc)UmjM|3i4SSaI8^SS!)3qrv*s2TzHx) zYhz=lPQbs+(pE;K(GE1=SsfxDyqRBy=3@Jy6APY~|Dp zaWP=9St|QO>k2Mx$ej7rg4n9^Ep&oWv!XBW)jh7H2p0;>xE$oiy?yO0hgKinlOzXiY1Ui33&V%-(mScP%iA91?TS(m>25f zJ{b0pFrn;?GwR4a2sIrz#0^6XJOKgnVG@k7_~WY&0AGml2hd&+OhHE-Q28aSXEfW$ z0Rvl=cM4wkp1@`=9R7%ZgO4x2y}(JrGC(w5cm*MvAaNgmJca#g5LKTf?6>IK7eY^5 z0kHdpqlbwfR45hR4@=@p*0j%dq2I0~^^)u1Pbp27r7^2nyW+$vO#|hoW8Sa1?ox6zS`J1S3yh*+@aS3Nr`7-LHUvqy}%qI@hYEXqx)vx&Bpv57p-+rZx!rR7D# zc)#yz7@f23fs{u>4)|@|<4*XD@hIWV!#7HeMrVw$1P&oo6j-HuS(gktXrB3jEKp0cJ8P!NAFq%D_n`?CP&Hnl6{@C5BI+1kYRa zfJupfmzh`60(@s+>7qyicaiPl8JOfqSSagK5cDzJKK=+zNdRKNQIRAVc5)vaQyxdO{#><@>lXyCXF!kjqW4(gdE=Zr zuHYUsL|sYt8$W>vNZ0KX#Q0n|UCe@vGmjgEx1LY{NuJmz08hgB%re$(9TxpgNF-O@ ze6mklwPyPh%9fOXsm1>1n9e!5?T?&g=+UfWNu125nLK7cWZD9!mWHn5QxDIL+)%M~ z5!?Dw<{sp?LmynJAa8bu(4~8vHl&2+7u<4A`j=@rXLj+zCWDqqrQPRwa@*)5`#V~K zP6O;tgGGj{?f>*FDTEq}&?)CT@?sYV(lhE;b$dUAH}cOtS8cFTT;Q0~Z2U*H!XG)g zQ}*y?4^=eVuA~9u9z=mmu4}0so%ixIq%n3gPBv{LmF%Y%d_}bNQ^Lg!aLoJxK60A@L)8 ze=t1<*e6Sx8?h#xscZhk`hee-^|%r&t5_hJ64Z?Kf@HU0G>-&S1gpmn%aYXn=tAc0 z;N8Ci>nGyJl85_>(-FdxI%f9Q2xdzYb3~dRheY5-uQNl8ki$+8@!}ntN_;WGF6yNx z`%AdyrtpFT*i3$`T=xz6Py~q+{awn*1RLBlkST#iCma=Z!WT4)fgPg94yVCJZ1PHh z+V3)?+zD%8`5UqUC3z5WlZX%SBJ__WlUV5N-v@NfFuF-D6nFJhO%aS>uAHdq_W{Z( z>tb&bNA&TOr#F!u;(q3c_D*UPHy39Tj0A0SXt^WBA2tB6oivj!X`>G-0P&@a9rk?Z z(|h*Hgkz#I?L)Zu%DnB&(HPVxZ#}h7`I{*;PPcQ^wcb3=G95OP$^Trl5|F(AqcQYD;G zO@L!T;1uk(eq22e>BZ7&WUUZs?`PsGRm+hsB-I`j*ef#ls3(nMKTQ4MJLHF8sN%ip zOLE|T`wN_JM8V)MIMlDe0&Cl2NVSdO{!(0T8UtvE=7~$jF~dnzfvw5aW@ikC^o2(h zPz&ol9Ic2qgrKJ|p8+0DF63sgs^lUFGpnFd%ZcVzu&zHfqFG~xcw;8KGvM!a< z0e4LmpPKpymrTdBM?Id_xKYc705zQPED^hJgyThDG6!4@E~EWS#&Id=0hj(CZT{Zvg1>g>uOv zZ#sA58Coi9jcmE-7}p@Sgg2{)>-7{X6&-wS_cf1;jc)H|)lR@<_w2=vWBQUDhUQ=l z;ehKsu_|D)Qk;vV}WlHV~NEyH44T> zFfW2U#Yi^<%m${J@T*0^HH7z(n-pQL(!2gnZ>Y-<5!&dIfV=Xc;f%0rTxUmr)yS&P z#T;f#0iPK$As?`&y(5Do19v&WcKI0gY-5}FAR~Q%-Uo6!kzxlL(zjSMubI$lO2}By zYBo`uUEzXa0YftQ1I~(L!)im{5`z+N6qfGZGa9>Ka6Y_{cPs!FMIQdR$=+sT zGeu~S)4rLB0~WGH1pgo5U&p=!4OSuhU!DP~Rjqlx;`m0G2`_L=?ybDvr+SRx*VtT< z8c@zG9mACKvU)Nao_MjS8KRBrVj7-M+B?{tAuLpmtH@OhS5@gQXf1n>rIbA5;TbZr zCQ9w-v#Re1=ny6=_>;*lT<@;d#TM?^j>2oAWgS3lcSKv`p{AezG0~<0W$J_v{xVbf z%~Zbqi@kWsqvxh#J|a~HhAVi&TG+fwSV|b*I^zhC7lg^X-EcC4$%NRobmg&*uN>NY z@bHYoW+K}@s-S)m6k^=U^|VJ*b{-1{9}lE;?fy~S(2tVry!X!c9F_9QQ)4w_I&+lB zLs!|-{5-5vAR*R3T`ss7pTZWn_o)-=HW_?nQC_GotVC3VQ7kuN%5X8WK4B2lA&V9% zaeJp@NNXvy>=lxVN-$@$jE#PA(YbG>JGESdZzV-MMXOSIn}M)xGm<&&C(08F1sa9M zcXJ|XeskrorNf^~DgkAsUvd)&R~5ma3cTCjNq@5b)e8D0D>4E2Lzz6%|7NLd3g#X8 z%5DGXyqkrUZ}!OUXb4Io7rSXWDU@nR&pj8g< z!Q(ykEU877N0v%`?oHeBz2e&jHeI0wN7Q?w^=b$`?M7$3Tk8c!-O((6<{W?hco~+) zy%YjC!Qx)b8|4l`u@O4tY^dw-)u?F?i-X(Ov-9PtP)wy`2$9z)zw3xtoSief)GCor z@d2mZSX?CW&A1os)5J~4_ezdCvFFswK3$tg7^xE8Rb=)0)WXquB0xLno&!$J-oKj! z6h*w-B4+qnv8YsIrnh$BDYrejBbp_^xftw@8jT4Oh_<}`KAKf=ZYg|mj`b3JEP_9jZY-voIgx+ein1OT3uzrN%iC4JD zszk*``Ng;z+!CVmEYMX%l&BuDAr@nJ4Kpmn4(@V+8gIKtnG|82i$PFhrW%k-CT^c<>}?&{suYAu1aV!GHSGCQn^gTCq91Vlag_FdpQ^rR0veIE;PL zE>&_`bp!dyol}=Aa743B;f{IIWlnLjA??>VSY*dDwh}XVuni{LK`MQNvc5ugtOXKs zcj{;UOBlugLkPqZSdWlG!L-!?n(qchWu0;*zjQd)Z;eaDmqmDZ9tjo1S61X@KC;AP ztH!V<`WUZttjQ)dz|pFv8ZwXBmF-72zkqb1$)&E&lgh^!O_tPMepK!7u=+QX9PAff z*u&UvRnvOB)sKVCxzCrFeT;g^` zUp%Pi<+P46Ozt&)3wYJ>W08pO=ae(GoYQr%cKpt<#&4sbBL|n-UZuNrOFHs4?c@!M z(hqwLyFH5)y)96HncdmcYJSSl-qvQI66%y)V!j5ZGGap751vMspLxGAH7?XpdhD=D zNF<=B0HUrl{CJuu+=Ad4N2=#1FR^l`t}^ifVGHq0Mny-m+*|i)ND!el459Smp=y&Z zKqw6Xr#f{MJxCl~uX$6GZY}c*rb|xs{7}T~kAQ)YMKH#b0tFgszmVaHj8~HT#`~V4 z@&TM~6+{0-UU=;mNtxVu^o#@6-0q(-v5V?@m&j(_#eu2x7>9aedUrsW8B|zMO(~Ie*`#-6yOx)} zL$|b%@O@_|V*Vu*6TplkiA6?Q1pe^$4)zTvOX{gD#(z{yGwv2M zt=JHG!0;)d8IY1&+|FywhhS@&Qs@6c+JnihKo26?GpAaM8r5-PVq291P3&=VtC4~q zFZ4^(QL0@pl)k*s@55I#e&B&Q2<@fsUV5m{TkQ9EqdIe|?Vy}put*l1 zg(%=eK&(Z4XxU&^;Ukc+b(LarHO}WuaAP5vyzuU9LuH)D7LL$`6?GR=}y15E}ktUNp1&VwkEbcGPcJxO&|9bx7e)we!{~E}g89Dq)*8Ssn z3gKrh9KQ}*i4~gEn$Gy)+90mIfV4+3LpXhdmj0iCUYJSD4#gmA`0bE$Iv31$YAF@M zXPd?SCWvp+g*oCh>WKti z+aqL6ry-IzBJTNh@W{g`&UfdSPhX$x=H%#UJh&smnw~mIx1f95Y+bA=KJ9+M!-%nJr0|PMCD7f5KsZ`Lt?xP;6U28^nKu zdDx(A%7h7|_xz1So0R?=nWSQ{0iyeA{cpiP+UUaS7W_&6<)jAGv<_(xt3ITt)bovkt0=)M`FoOuaTi+@HO)L!`b?+*Uy!fF}^#0)u7?MUv6;md&QXsBU*n{QC&rfROGY1 z=KPMpX(3JjqSvlC!(>5LS<=%#lvo^GMIgr|>=fp7Ogr=@3-spo@SN!+%ri*?zw zDT8_SDU}Sc!e17UM>n^p|FF zk7ki>c|<-u&#g`)DeyzGZ6L!qX|7@167aB{O>y*t$*ow&igG5UVX99_`7qAcBEouA zChNbgC0VidsZFOc{?r+9>ar=hkWy>fpD@jwA^pFt6#Hm33tbhKB~CtQ6fJnafx3*% z-RXF%C6vr~jGX$kT@NRXb_@4)`}~W-$ns5o9b& z#uzwZ1{;G4CBj|}*(8JxVZ0!hEX~TB6~%@j=2Sfjb)~A;)xj@ZbJ}V`&>pDLzNvYo zvQ^%<3YBsBxUsp})$^(CedhabYDS(l>WjF4Z${v(`;2dIqpul}$@lps1uQ+y=Q)(?krKJ{mM@wg8u(8R3otCcV-gu;_P<>U}pOjU)>MI$F zWMP7;E1Nlsb;?9#gZ9c69S2VgZD~J0X*P1=H#NMj9@7-a?cuJX zzP6HV=lZ|^Q9pHgbyIcKk0!ZJLp={0)^dzk_o5f~pTWxDm^^-0a zE(POMHG%d&qvNVtoU1aPbRc1Vo@6IFzLf0w8pRP$vT$cXI$+2}H330q;t2 z6otufizxnksI-@e=m7GDqtVHdLV}-C1&tiX5>wnHNsT2cPnSbup{lBf%0|B_oYS8V**)v>4AUc4pOVWpwFaloo(iDe;tF#kvQhw_>nMF4?blAQiJV zH6q16N=lJd0eZUqkv1kgc_K=T30G7f6OeB;w5)v0X!jFGBAEa^=~jxiiH-hw=Tck% zpS2M!4q_?1y-{g(X<|LYzhhU7uJilVXm7$Iz%tzNAixY&qQ|t6LzF zMZZpZjcpJYNgs2Xk%C`}Reqr4f(VW9gzR2RoD7Znpni6k1Av(+2ed@VfVvno zmt*#6#94D5_S=d8Lb8p-CW;>j_5pXTlFW*kGS0NEv#SxVdRD+efV$-PsE`Ws%J~J!U zgyYE8=3~gcf;|PLTiMZVQSPdW-L9rf zWh9>Otoes#-!~$~G%#e`J3+2hcsqsq1|PxJUubhlj1x|tCG^BMH}Na(ZWKe9kgl6P z7;q{xb%*m@A!OF6mnm4Us1w)hTh&@pL0B02nQ+%!zkBoMc_J67J;giy}v4A^1C2UxfVee^qm_4Mjf=eqoLZ#r0O;3Ne zWpRi1BW8^k%y$jqBps|P^k_^GnfAuTzwxqpTv2Wu=BWHD+XzF%ETqPlV zXGbM;VB+1Z))HQRm2IX^V>>;7qOX7S=5D;|J@)Z=s4(r1Qkzw4G#_{0YwDa%ABMz;CLSW+z^Ll`l7FoogZr?^(-Bf&g=b?B(&8w#9#$B4*UERs$JUBZa1CG>(#1h z7A-v;EdlOH>)`(sO+x|Bpimmlx3QI-TFs7oeA5F-0^%xUwU$fWgs-to|DVdv0;sNKS=c~u_uwwU-Gc{r4esvl?(XjH?!n#N zf=h4<4nYDx=bZQIgxn6z6-Eiqe!}~?(Fikfomn$jd~KG8yfl zlY>?|(}kr}TRF{`aTS0HVJTY45{^#}@bSA(>^V0-2sbP$dLS_+=Opkm5mvD3MGf&! zZ5M58Ik1;rHpFD?(be|~jh}}EAS%3{>!aH=W17jKW)Lb+!^TJ;e(@(Q6miGov4$<7 zOF1Ss8I*)H$YMp zL3aIodW*Er1n{R4b^a8DQ?wFG7~z{dsJnP?9#uzSpAbMSx;dHO$DWBWuZVrq!{qWm zD`sB7b4!7_Mc`Tun7~`H>@h^-8VQ&naf^V-6jMronIL&T5Rl1(d};PA*e}yBmdcH5 zY6e>L3@d|vhvQ@0?PccsPTdwKOH$sWyFPS))(b-V+sPPm$Ix+4IAREZXbwUm6)p)*Lk-uYTX zXmo0c`f#AP3j4UCwaWV>GC1V}S7NY+n5MvOEfoR@S$#)Xp~drD&<##~=i`-|w<9&A zC=~^kL8QtlV13Sb_kezJIx)yM5yRpUp)gyGDtDyfy(jIjsq-d7h)|=Gxi!f49FhHu zA}vsp(EFnq>Owf?l(gV$o(opOiF|Ma{prZ>1G2$ zn)A*B>Ru}G-g5kU!XoQ>KI?MZ%@p|f1DVG+QrSbd*(df34xs_ci={RJZ=rVpxLqk_ zsQ0eBL3o34dYLj=?$pZTjE%)nwlTq{vG{wu9Vqgc@$i=se5SDbzD?^34geNCGjN5Q zOoS)2IA~mYAz>3V#>?ybYaYh2?l>N01Yb!}9~45`x{kv|jK2+?&X&Pk($V7G9TRiG zk#m^}qbT+x4RZSk^bHn3ixkQ~N??+wqF+L+9yeftJxeH&>I9Qm*ab zL)MKD5-;lqz=EK@B&viB;A-eQ9k%*WiYk?~2t*TP@;;0*6hoahR;bhu5Dh`Hbs_Y2 zGVZ4+hFh}LsGo!En?%Y696vC%)0pJt9}bBYPF=Q&t0%^kqYZbW+AKi7;tb*D=;AA$4jaNusT+m|(f*YdE^&cCSc^aoo}w< zrfu;#^{p14bs1cP!S(7^l<1Vm}GPazckbfl{ z#~M!di^ehH_a>!B_{7kMDN>xy`Y)tS8t;SLzi6+J)& z-PfHneILjzG6R`W7>|A#KZ2X(Q*$9?^F{&pGV^2dv6+4jh70q%%PQldb{!5*%4>U* zb^G=X=xB9p3vW(OgnG6-?YXSWos8E7i#L}#ZQzlI|8Ne~0z49Xu>E1sXn=J#R1#hF?g!hK&~qZ3N^o%$M}s7wVSF1)Y1n9-+>$2Tp)4#yPC+DmY+g2Dm#}g_MzEcC2Xvwk#jwNL)4Fj8HU`oW zu{gQjQTn`W(k@{(e5k-X!UuHZ5Xsna{P_b)LW6nP1{Px>v);o9@n!%XYMtb85FEas>O(w9(Ji`0W%>$$L|DiRF7=Au$X>am6{ z8DsmS${X-vQW^z*yfe>? zXO%33r^@AHe{X8No5qD4XhB$uOWhyP!mb=&_cF%r_tO5`&mQt+4W&~|%i5``LRT@K z!x*~wO!H#uOj#_K0jFP;yq-jWZFOhvs|B7&H};7? zPfKUv*kIY; z#k$S_Hr1rHz%EK!u{~^BA+sATo;JBD1*%+d)_oqbwCv7q^A|rcZ`L`*Q*}+OSa6jm zUK`Lkc~dsY^`ZC%Vcyh}C-;Vq+0?V3^!lxxv8RCe12V*kty+Hasw|zdL0bwjb`vCZ z3YR)PuDazI@aJH1CNYgtgCTJ>ULW6E4FyCM2RSglAZ$(#mO zO#-oY9a(~cWT8YeXHP^$U3cYIP(+g@}%M#+U(-yuh zQL@rR9Oa3d7Qufv!P9~*`Ca;HX40g8w`d~>rT>P3X)i;kBF-OHMOAJenm%pQal7t; z;c?tSBu;Msf&*&R<1GtX&Dp)W9kvg26GYSpcj~FV0$2M{IyaDcnF}8ciA>Xt=_6W! zbpZa#Kut@Y=(Ql7HVFND1Nj40=|UFW*mf9vcaEhiTPtr5>WX<)E2=J*1M_LxWsx1& z?GfHy0hCSoDIQCf)GqR+#vsA!;IN&6{G8x<#u5t5UIX+eMxs!vi|*NMHO7%lZRTtx zvv}??@ai#939#7-ik<%L>V{+T7}JcF>ZlV^j_@zXX}(GN;v+hjR%=EFx~k-f zFhOhK$u7{BSFDg%I5qrO`7YBYRl9PvX0R&GNrd@xI|8)Vslh-o)6fBA^R%@}#ImHM z(=xUR9>G+h>?oLx3lS(Lmryl{Z!C5XRBy*VV>7yQhbanTx8K1Qe&9g2zu)CxAaViB zMQ^`Drw$fGx4+wMSwUPKDq3b)H&2@7*w&KT77Wm~Xi}?~gzeE!a;=D=s2j4v#Q9XS z$E!?JMjfxl6kjt2feadIR7cH6vC~c4kPWrUikpZpIBQN1#{LZV*Cr`F^_TEvWB`DC zia%|V;{A4$R9f54=%0;Je+5%pIR#{OmdJnr9{%k2_Z#IBmi%IXGROQg%sh28%aa%- z4UH$}h~lv>w!TzF-#S^?*$<1pfe4fn_-@46#i65X?z2XC%+NdmKKa?F+r~D?lHsPf zj&r$IwoNP@9Y5Vqu&e_W@vHC}4+qT`1Vrrj>~7slmnYzq!bn9Nsi&aZ^2%&*=3vj6 zAgR!+ER(gVRZ%EK-xebbtbjGQyovVKQmx`#Kv2x`I~E53`6?|WROO0?bpQzo1^DsC ze@R6ZXtn&!Sb8A4ivJu5zyTfnK9`j+061Z78NIEtimZrSD5*WacX>xL+MVzC)Kr}X z`8%T^(k%m|x9v~6(h8M8EtMsoeRayLd%O|hLN=)UV2<7^fg~!KOK;HXTFXkWIoQG; zq96;}*GfnUjwOA7A+wsA);-+%aOUsVg)SAbj|3TF0|rQ#f<6;VIm&60*Y6@um=Q+4 zaA2BhI72`Gkt%(~Ro4ru@jL-SIT9>>VZZ<6>>M#vJB3uDE%704)5m0vob0?sCLFu9 z->Dy&3$o#C-}5jDtmrlil|w)cyB2PMkx)w|hzjSpA4jGad)qt6{%YY6Y8B&5a)lV+ zcFO%JCi??lq)G65@2+5N_m!~180y)UDKA74>?!dvUy`W_lNCZ%t6&8lHS_w^bo0$Q zKOS&CV%W>b1byeBckeV-{6S)d8=LXs>721aFcaIs>r>%il*yG03NElUw@OvgMtvct zm@%7?%UaIs*pQ^c_K%esF*z4DyGYDX+XI!NnYk)Rg%(UTM|2j{@TMNlvVy{$s^&sY z-r{HH^W&i6N4`_yK1CKTwC9h)WrWVo=4D|#k7J)CO9QF4R@C1=5chd+-bBW=95&kX7EESaHH3YytW!Z>bkr zK54dVoo;&c`Fs~ONr(+T(xFTIWeOxH-XuNRlV7##2RtouO{vzS6muk8}fN|VcF7Cu^o(w@*F+vdGyB`S9tX2uNZCs5+Pp zszJm01gK}n%`V|AV>yS6X~P25Mom74!i&%*OW$uO9j+Hs$%nbR+3LUKsSoL#Y6u1Q zEPH}dJUZa4ud?{dG0uYf5*o5f`jSJBr)`46aBzR%=6)C_pmZQ~O$TZ?=dQXUt9MgO zWNrJZ6s#-lB02UXV*4E(SzP-Lh4F*Iq|=KlUwuZ3WFm3feJ!AIh<@gp}=()l97=Ej7nj(Bphs6^G zV@*F|e7q+)c*E|G8AI?CiZ0MHe8vH?(qGcJiLqo{J2UJZi_}NC^BGwG|y$9yoH(R!J9iY+F2*{TZUI{>%ze73aL_{)ScxaL(Dg}$~%xss|NkKG5}X zx+jE^%tHWHB&|@F{{v!76e|yrv%V#Pm;vz!vtLqtYjyl!i$Pn+iI%v{R>m8w4PXU% z8RRISG;&!GkQG*|z0P=UF9!bu^zeB?=lIWn3gwI)8DA}3TwENlmpiy0!2w7&VZFZx zu;J2Ou+;g~=j;$3Ug!|DG~|&j*^kX$`RoGm;aTywlLZB%dl1bI#vXihTmmdMqfEvWlid&CU4>-L zoRA?NE***I%_X$kAQr@9%9ju$e%!C$)Ip$68jsB6fVf{3ZY?sdAbUtK*)tYgM~@>h zZ?GbfMa5B&SEy9N3H&%@hm~$4BM{LagIKRY056{>?jjp3#)KpT)RKSD#26hvQ>Bfbr;D+6y5G18#M z?!D+n6?l{!+h~e@7bKd3An8Cp@+Ec;=w_?Xry*ciw%xj!YkF_@YxJ*I%{~gX^0pea zMv(&7Y|0%LTJ}#H>gDsH}UwGaDB?NNGY~rrAy6S=@I? zH#j64HRPa_=@sR(qdNIk*#q=OkcO>Iv^WkUHQFk`H&(NLCMBUM`N=?+gIch?lMm9TZ?6Dnzx!|n-A84?<#G}1=N3gmxBy&R^TOreAYNS-X~JJY%p&l11l{Ll zNoj+e8B4BzHtHFm@Ti=5i$2b@S?PEN*!q6qmQ$^1;&HvllG|OcYr=`EV?+5767vd0 z{X9UsN%+}wV;`j<1Wd#KGwUU#ApGMsr(Z#`RhHv5;D?UtVso>vj)Ip8k9hs`mpoX4 zto%NT4YF0k`%u7&r>tI2@Y9es@Yo{n%Ynme^iZRZH3ea8EIUsIzlU1G$rG4auKZd|Z}OB_IZeM|7NZ$t`|MGhm>`BZMBHoK?ZrMF4A*NLMhO zS%mY1aMlq7u7D;%gxf=O(Xmxo{ZXzl$qM#@bB8y*Ooi#A90aq4yh#y3t>LU=2sk0& z-SdK+TKoV`W;|V>ACw{i>STG`&%pSsDBv7=u#DohE)4T|ht$o+uW6Ll5unI3DOlTk zX^zlo6q;hX?Sq#y;vYOUOL~~pJz+NaP2G_!B;8>P1C4lDcn8j_v(HFceSO_W(JHQn zNaChupWqb@a5@jAmQS;g7D#x$Mk8F{qm$xBLtM}$<7Y;dZp+1xW+H^U2@G$W)`)(M z<=9?6F$DWGvhHYC2imBt**eiw{`_~s46{SxdH_%WfGW8E|C1KB#)cN!cJ?;Kt_?;^}oi zp|*qwk^S6q!g76ysPG(>7W7T{FlryBLS0p(K#{VRP7xZuN>4u8@I?cAoRO?P+GjOs zOR;?fDzyRW)^|Aqh}Y1tpRcVHzUK`G^ZDP^Il7c!TMQcYeUcO8UT4B{YRO zn79oof2Y6VMTYe$x(jiN@30}k0<@L_enUmh8fl>5+$dJvT031ENT{+hChVXoIdr{y z16=|Yb!C{=7>b**?@EJssl81~J%3-cGDX@Rom$B}DVRLK27fz}{M<>Gve+NZpru6c zBOUtOnU!tryJ9YZ$kG+eo@vhmSFO0QtPp3qWx=|n9$JlFjLCQY7Q{s%PWqz^X%nOv zknb}|BKZO)yx#80nkx)>8)!9RDw#{#FEWoO3sXYT&~Nmy%{Ay{mVh}I>#b;Oi5X!$ zp~r^_zPUiU^u23tAV%h#nqI~J7)(mZlNYQHm*16FQL54{MMMxZJG|9%$fK`Sgs#4b zW(N8_TySBF5E!jLC<`-Qt&%7)6K+ei?rieoPL0W=y?%>Km?B{*b3irFmqzRm9GT`E zzco|%lH5rmi8+u`=#vL%H;M5xTBCHpa>I9YDV?b<+|T5ry>EKjnD#gIXi!80bh6al zn!o27l={GbE+{K}`wcxvaIOk*>~4^u%cly4^Bu8Csmi<0JR*c4Q_=&@AB{%^!pLFZ zRK*$*GTt*WbG7cpaHA5$U+e}IREXa=AaFY$d>ddumCyMkYd~XeUY&;{pGqKu65s0) z+!<3MVq+2hc8(O5x_XC z4c2QxsjT1qZa_E$Ez= z!#IpdqrdMU^LpuIuWfTU!8p!v(!q5EYoN-ceJq-gnEl?iJvOOtG=BZogO3$T)Y(&F zF7E1(ubUZ=Q}lEo0HJ{Kv;vvJj|20W851;AGWHE7!rmvcYl}S#q*F4QV~_FWz=J7a zQ|xjof8wvTrz>2Z&wQI6+lV%<6`v<`IY3yxTEl)y?_F^awHP$7w6$qY;j}31p=@ei z(oSFxOykN)=-Z}1$Ej}%m{F+t43pEPy%>1VYg&j*P_I&&n}=ItSi=-vPo~so;EQiU zXqZm+UOy?$jktIekPj8cK;sF{eaO(7qYqj5<{q$&_E4~cv{S+qwY<1aZIR=xfM~l< z@hGW9;VfCrNde`NT(WLljl1;|Yus>CI{(0A?Jg|==>?1=RP1-VB{1{}T^5ZlLZ^5Y z4=*Jb_%}}255c*Wn^a5KefU^C2v`Fgtll(d)a_latf9BofVdm8Na^xx7@ zHpLA3MHGE1mC)M_VAFJFtWw508_Ph~YwuWlP`^;BhS*a2T~dZYnIU=TVNll)QCFLP zOFrqOF?(B4)%Go(GotiZ+{GIyEqTf{C;ZZSbf2)g&ebu<70KW+1f8*)1!9iPZpYsdtuzs< zj`Ss>=|KG{3eo?!QRsE)9(jqsR!r`MLkR?W9BZt&f+W^_0Ll)Ofo_1zfU42FABYPH zgaoMN!}Eh{is!!0A6p>eZVty2DsWkC0|{tfZ42e^C(pgTA&G&*khr~a{odSY zsL|$e&f-;CL`%t5qhRHb5ftz&evI7oncB%du-6FBX(FD2-tuQYXTDKnTR`fRB+8XM{j)IuXw=tp@l;}UP4(2O zys8Tx2hRLW?FS1YX}YY-k)?Ojb*h-&#!Mjb-z+ec(d75uEJLY%j&3&^k1e=EUL%xP zvG-rqYl1UVPFM#8S;&w2TS2{zVlXLLq8zy)GrX4;ZrVr$ai?$;sva0 zZqbOZid^B!lm@ikLr6iz8KjQ`1p`%TY~ziZ`mDF|<}BEYyYnvr{;T!Sa`)SqcjilU zXhH(C;>Jo-UosjuflC}Ih5E%X6!X5UEl1<1n!3Xe9~f%=sas%pxZoaZQzLV)-ld_NAGb@;vm=i z&NOQe+{BuPZ(6ueU`qF-nctB{I*yR_RVIpV0oLIU)JL;@REu0Iwt= za&7}MdfbG@Va7gGie%cPMR?rxtx`mmhd|~J14i3T+7KO?2f}wx7*P{07oklvfNOcrN_v~m zutbs9F$2n-%X8%dLVRHN3u~Y0YNI>(Kf*tQ;~`h#`ghII8|U~oT0lgTOUmP zIHx_ww1{RYAz4*CZuHnk+n6;8dGEX|Lp7|hwNj7?IN|Dx-I~gLtpLo@4D@anQgc2H zCh)d$n*~)uLU>ww&Gl&Qqk$m*SW+^>2?R{Lu;)^s3Xd;0{Y)lo4o)_}qtWzaJD4$5 zN}nMNU|kkd5XLMH56V)o7ELnI`;=DyZW-QzkfNYc?Hk!H=^<02@MBpn@|dz7z%J(` z(JQ{LK`NvD5b^6%PT_%Ak3nA+P=a~=8B!;^d=#Zv1S?Ps*e!|q1`DCBzc&IcBVfsnL#o!VNcok zqdRNz#QvfU)3!EVJdHBC?$Rj6em*kyUO29{b!1a1;+n&}mzpwO1yyTytL>wEm=s*k z05*I?04B@b1F_1uMUFW(b|yPr%31zuGZpBUsn<*#%*BqVHxB^iVEE?UpC=b7$BnEvxAb zkvlkUT3na}ona<=PCAq5Tp{_ZZl%578fq2Q$5DtTe2;SKqgw1t3k2IZZolS>B+vQ= za<6GLi=;Vtf!a^t#%Rb_Iv4)S!(^43Ob;DOasoy()>kIz5m;!B6ao&$`s}5HBwT^! z9zk$of(jSbX5<0Rkgc@s{nal;x1FKto&q_0B3D4c8XqXAYpZFTgIrLi&N(q`+I9Ff zcJD>|0#-sFh$Q(`=<%%B8&$^zoZ~-kGe!0R4eJ1|M1rL`yf32k(D?GHA!F72XzBz@ zd(Wg@vW#UijDj_cg;Q-cq*Qg$SuYYJtyIaku}Py`$yY{doJ(qa=!7j(y~pW#n!Vp~ z_mMfel3~v=`O=kT$eGVtJxa};aWJpvZ3|iEQPl0wJs{Y4pwWQPRqp`KI9K0dAuEze z)Yu3wYB1z+H&Y!4<4~7HPlXu>3KPHuMC&25Xf61f zhDMLg%gy81>9#_vpY^^6l&!~Hb6|Ddw%&6)F%HD-q6yU>^~8!^=3NQqmR!PaeTsb` zMZGrs>Ul0~A|LbZvrsq8NUh_6=L;$M zX1EUP(Pzmf?xJfcIJ*`fuI#MLxYZNu(i2Z1Ni491dbtK4_vak<=UvTKPoo)6PGnBe z?_{Jo_nv?G(IGuh@B?4~fQrATM}R=c0Ixmi|1RJm0Du8rHwgiGUMdkuL7J!G{}lZ= z@1}_h2mtW-@vH!LkvS0D||APFuU-dIE zO1;$r^~ExpUT!1xE8rWC-vZ?=EycBM4D|(^boH(N8l+#dv5)zUBVUHM2=1SPga0;r zuivu&XhzL=TeC$kP z!P!FF+*tS5P&AqY7JtxxYA?OEA-aM;L2ZAHLzI9dgL@ewVc>rXHvQW`%725i(|7uf zeNP8Ikxah0t-sP_zaA6q%3s3lZEW-{>;!GJ&3|L!Uv@YH9A1b*K>uvV{@Z-3e?=6~ zGyDyy9aq<`_(d;9u%C6~ul*HC&{*G0?~nGf8K?X);e}?3__MumHU5_PCwuv%g)m|N z@*ofL0RUjX0up!p68QW3W(>YUcl6TRzqY4wUZHk2e}xj(w$L;C4GZJ>x?1bSJbJ)? zHW94d-?M(x$8}!p!_O~4cogO5J`NrJ3MQ_tqi^S|l_aC|YJ^YXM@?$~e*A_?x3iw;(pA6>5a;&eF+~WHs z@juHl|BU>xwA*VXZv}sk{F9PD*1~vgK2MUrBEBjR_cQFrB>b=K8dv34upf(Q{LK0> zYx?Wi$ff%4S^ts5|7Yxvzv6xE79zC%5A3gjKc0Ad?E@8jeg*vZr@;SVFF%~5z6dTf ze#ZZ}FGK$dA6@ca;{W&<{@ z3IG5I2mnP`fm4M5>}=R(001C#001-q003`rWo~n2VRB<=EpuaXX>fFDZf7lUVQFq` zWpi^ab8u-bF)lMMFfCI~K~PgjPgE{yVREdxW00mnlc?R+^t5eb+O}<5^R&&UZQHhc z+L*R&+jjS~?Qb@Azq99k_nbw zAphE6|8_YsRUtYld2t4C5XFBGyyXRsnt_6VSpStq``-oSgyf~f#Z*-3<;0y8do4E@ z(K>M75d-vpIWMRt47@aG^VbX4WI$m(**?iN}K<&nq@-!(k7&?M$~O{`8!H z%lq@^`cx`9#_FAj|n74FYTWkwkR1>zZRG zzB}pe%^%z^KQV2c3PgFbofFSc%pFOg*nqiwl5yo$e@fr-_6ab$g`9~e;8Q7c9Na)! zow5a`G3=-3kre9w85DOU9;8U=nixBC&Dd}=4;>_cq=|8|<`cK^^dNH_(t0H4K7ed3 zG^g^&xbhrr)p)K32*W7k)8xe_+1Kdbe}x7k-&gWqmiyTusBvr(>`%ea5( z^WGf@Bss4E_s(C=h2P~|3LBtr9?qAyd0Wjn;6E}oY98M{bm!QejGAk3w#GU={J&k* zT={564xUMObZY6*U8QnVG`hJuQSw*Td~Z9@NSa4`|V&8ZYXP4T`sD)1oRjQ2&cY(buMXhH)%ctS+o4ZXV8CA6HYBCrppwYa>IG zD{ti-Fzo2o-|A*YdpaB2#DT;#T0XnmE5X8GA}-;uCv0%5T(8jt@}zZ*@*|(;)0G&C zR2o)(__zb@Go1FP7!6OW+6nK*a`fR8je6LJ{OnEdTSE=mYOBd3P9bcOuK7&R#=c9N zgj8d7Z${^D@`$eI$A>Eh3m)(zKtHt5{ti?v_bbSTaq>45p#yQ}pFzrjG?a5Je2L?} z?EA91J=*W7*?LF%pT=N73Sf3RnWoffWv}*M%$(wZqqr z4aA(qVCSJ5q(b`e0*XQJH4iQg_dG4dO4wc-)}N61_T_DV5Z>`yjI5~yEUmPk>o!Ia z8W3Q?lxDfSltcUq>~Oy7FqCWEC3Ttj@=_guvXYtlM#9+2Q-n_6;z7*(YAB(nX1~e5 z_MBWOA{^UR{Gc1OKe;pVWqq87rTVigGnZx$LrPcdciG2e2v&RgUZ<@E8pkWSIU`xDBIvkzAXj$p}HoeN%5XGB;?hhYy4A&p0GNzxmR@6!G- z7m^D5Is?uOrEY1l5+WM=4eX%(VY+0T@d|qboR6>c3yDkwsM2=NlANCnhb;&XuN%ac z(5zTSq6a^^IceVwsy1?m?BV6AWjrdnsL2|^zIS0y8e7_)#qF)hqW;8Qx~V?K@*B)} ztt4BV-x3HG=1Fk0HC4ag9+$^IyHi#>8*!d#lQ5bu`=HpEBH_ zBxg{V9D>wB+Umba&>f*UN|z3i5ZGK*HHD?i<3C|t1l0LWY2vL8#Tl5f(nW<#Z5nXx zHynYghZe&r*K_kDu|aNStYyPLgHE+Xc>9!eYsP1z5E7b`A5n#K<^E79(jVS8k^Tu) z6eWxYOpB8QELuZ8rjSf8zWiYgj(v9Kbs6cIR?gY*3yZy&Eir^1vql?0RTLMKM36py zDa~#pe>`%G4&smzfGVgt-m8(UuwQS!dG^-|e;65D96nn$rI4w|OGa0f zipQ2D3i6v8*bBo1;R;d#>px!vNrByko>^A)jMgn=*_65M2t?fkSYE!5q~<#sSm?thL8n%QyOCP3;asAhj|e(?dOamyg!M5Kn!ky%

O z%N3{|G=S$NFYFL9xHOjCIhq$S!p{=K=fxLPDGGmu2^pGK>9I2ICq)*#Tn|57`&gG< z?ItTeXLe=O%Apz!ZdvcvyR{2vEeY&s6|O^l;2&psZMjt~9l_k0St@)>gPdo~OO&;r z^V9~&pr3y6qUs9cF_=N%4x#52bfwc|cI8>yhOyN!jkfeDY_9^3!U%D2M%>&cGdk)* zw&b6(M8~*Kk*c*M%}qnCsm6@F;uhiu3h&sw3}U2DwhQ0{jY>N7U_a~fy!Id@N}ut1 zl;nO6X4^F;W-)UeBgLS<&!VV6GRY2>#A8vZn2URt1c{R#rZKqtt2sFLpov`tYsS+f zHNkbk_hMFko2yl*#DeQekK{>K2svtSra^>VjZuQJSA@1@=`A$VBxr@OAhl)(8&Sd| zF7*H6s3TEWH>KZIAkvE0CT5m|t5TlZW5agO6;}`(d_wwlPTMrFf*t@oK*PUynZ>&6 zSv?|JbvSpG_;^^jS>obM0o+)ksS_vovggz^+(NTBp)s`(7&=vKUW$xFmMMfX`x_~F znTG}QMe+dk`)<4UnZ^3W$&cHM2l#|52O>|SwHLlPN3*ws)nuWbbXl%C+7}vBbZrG# z{v`B=H0qNDt$GqFsjNFwL*7zk=4+u7n8DN)HdLprKC=tyUMOZ9a{MNhm!vHQsa3cE z7n87d8sdu>pf1Oj#gFpfj zwwe*elC$Rxk^yd6)4MwBk@7xYojSVDgo&5-Fnybx$?U~)7 zae{=qIUZ3Piq8^X;s&&uVR_+PWVbRKGOg;xmra|i6g8BUY4P&TMjXtyrb&|_Hcyqf zOZg|RmrJ3?RN|VAiWW1)P#C4Jl;Jspp@w?NOvZG>?&!1C336Z(a+emwWR(yA3YCrC zWKinAC2($Y;?<8>rzJ<%<>2X&m&EcAAUdX<+1#Gq`Oas!XNXyvg7%Z7wW;qu#xK`Q z_(kqrd|ynB-YtQjcagckPeE3mj8;U*Dn~Pb0N@zY-{3N}?|@QkGBt_noSaf!Bl3%s zlB&d(PY~t!ao`M;S2cLm05?3J+x-MU(V)QgkT6&X^st4e{Ha}CNP|M2FfC;b`q)CO z9c(eMzsq+fdH1AueZg?QU>${u%s@qp6cw>AmT!w+*u~c7@m#~E03E5(eCGEaSS`WT zp|+IzKs#v+=lv7c5?`!GN_IF5zE35AG%e&F?@9AmW?#=WVG#QJ2IwI$!4{Nff^*a^ zxEn<#|McsSoRrB3KpzG0Kj|qyfH@E5_c}yo2t@CXhdVJiR71uI=u{3m@}UaQE{!`g zUJf6A#BvZ`4h|1)ZW=mwcj!lEw!dqC9%XA*`e67PR9E|5nsp~LH+wS~8ZqR+N z_3G~UN*6xay!CPJJX(gv&>yPjPStdWoXNE%;?sokTAZmx1qR9U;5G(j(!%5s6Ji4` z*Uf(#6t~gt*|;)c9}bO9o9%xDgo91p<8~4YON`k#5FlUC^`(G!`R|;#ynFdDddQDK zBbWq?Fn=|r&e-_M`6F|N z1Qb}m>hC2XarK5pEGu&q9$J!Rdi+6Y%_ng?l0)i%?Ae*T=*6Xh_78LjgZOYmIyM;q zv{e$Y>Lo0FK71bmu?xC(48)wqzTVWOm9B&}5BBS5xg$Y8qowA*jB8Vve|Y|Vy?Wl% zBRrnX#0yz0b$swU+qAl8-BQdAOi==7tXkD8EN3{6-~$28wfy3GF@Y7ysGeQO6~G>y zD~xRC9$XtZky@j@!<)v?)jbC)WE6N)xN1Z>P-V=DwaS8;H6>x!0F-mVMd51h#>gkG z{=)HR?WH=Ek)_-$AQM$cETDydm&Nj=&|~2@>T~K!v^g`+lwO>vzJ~ibw3t5I+CWFF zM6^5`L7gpOHE)xiiw902q7gIs15qh*zV+s4XH=Yf29z$&8fWyKKXNFFU0jdTuSP>d zQ)9XlN}`*=)^0ccP+&Xnn}s$=n8$Su!$HmEu3ga>{}y@g_6H_C)w>niyoGSyBGm68 zVO;*8GVyrs7ww7?M=&@DGM>OsC_8F&(W5`924=|5=FGblo5T>P`Q=xyzvVsX>8oX* zLZ7~~k}^EhLF~aW*bx{? zpDhF^zf;LiJ8lRj5gQ#gMVgxuMmV%+ZGsw`8%?^pU=ytt7^Uk9JYIWv5?w3zr5GI!~6pM$Mig!w>iDxZ(8mE3Ic-qzo+N&3aWn- z^nc0DlVh0>0~nElpFSgSO=^h}+pr4*zXg8!yRc*uDsUtd9zBa4^FY;}5WcaB-P zE|W-t}l*{W$+rA-Fw$Y6PJEIvT(yS$p;ZLKQgihAeJVO)yg+Xox&jJ zE`;X^oi^Ks>OwkC(^rKVn`8l?Etj2jzu0vRhs+j!Qz5MXUbz4`v_-%n7VaJd;|+N; z*V0h@g3tm+attg#(o8Qqx96km?~+`xg&VCw|NA0E{;7%t@weMKn*Boo>7Qi~|4qiu z-o)0)1mI|7{-26t{YP8~w|mQC-UkRRZZ#Jf%^Sz-U#s4)!PvL<^5O4kSDZ z)xISdbUcj|I;yy?!>y6VMFZ)W2=kz8RK`b0KM@)wQ?=Ml{*Z(G6uYX9UX`LuRf>m? z#oLE(lg0P_VaF9@dhd-&*qjCF$Xp_b*hOyxjap_Ucwx2docWkVG(>LwUIJ?kB`{H` zQZUq?N`jJ8B-WJStpvT+6P`xTz0kVNaiVT`EM>5kF1~CzgTPvsx%T&v&yuTL`;0%9 z2mHNbWuIjTUyP0<7rBL}a!YHvs#IDNIUmDe^2EX*_Izr}=}TeCbq8rlHYFsQ-+l(< zyy?m0f|UZzgB8~j^=5_FE=G}Wgj+)ZlTwmpcWj}-g+%0BuETBQxPp_l;`q7lX{w&5 z>Pn2wQ4kxGMnc4Jk2shSY#uyU$pO2QbuSD2xzhv6agdCGL0ga^tqM*07LPo-=J^9vglpYjvFTK@^qJ-^Ht)SxD{2?P2v1 z^Z}3;`kOAENsz9p6BR%I>;r>xNQ3yclM5u77OAQ$f0SBCKF#^_^f(|*vou61kCHrb zEL|(4+FV$lX$r>0vxa=f+r#ROgm^Flq9TZ)FZYVd*i?BLan8sT+Y?17by)9E4Kv~W z^1~qmlf_Ria@n4B)3>yFlFIfl79RFZ*hiCBnR7}u3CS0wNiQjO4#}^e@I9N!I^+z! zOIrJ_2fRIMpe~^;17Axj9e4R~XNogf>1c3qi+D%k%6;6^s@ME|ptSCKtm@j@!xSC~ z{48w(>dnH+sPdFy&MQUj|2`Aez-r<+#gg+FArp~UqFo6%Q(zuAmo{yw+mM(`I$l!@ zIp@Qck6~phAu~gb#`zi_Q+8OMRx{sKqGV9pOmD)Y!TeRklYd*dmL?)+q};-7&}+Do z%LKdXola>Rhqkh&9la0(R#in*1y(|H9=Sp5j~%x8*6nARF?aY}3UZi2C4VTkk; z0z@G2rEBD#L$kFr3asqZ1+xK~7-#>^OdOUCmiAS7cBNrDdy>E#)_{^RyQkZCmJk&4 z0Ivp(?UzUgj3+?Ocb!g*sftU8q8MtI?)^$x)2ht*(84z+Qf0CiUU82g&T+_Ebu4pc zwuD%`d3Mc2PpmS5#&wklHZg&11;S@3x|2$~F5(TtPhd@4B6MtqyV|e!wBudvdY{p= zJ>V~E74k%^O`%*Hm0#BRSoJ$)@3%Oflx3Ob>3rVy&k>_uXrpaV8_#uYz&j=GTJ}OD|zTJ8uBN z0|jZhL}%{(3Sks(Ks*6Au)a$e2vakcwS^gbR5P2KwC9Xd14bLm2*1{v`HoK8YD6$3 z0^F2vbv!0%rD(N^_xiDlsb92A`L@Q!XHS^RB_{+ zqWO5gdZtv6P|*g^6GlXO`q6CX*YG7XKq-It&srkPB!R_PNDE=kC4tq*slavBmZ14v zWZ;ne5MJ|d1pUJJ-O)m6VV7|d4f7#)i3Mc{)mH8Ueh>Z#wSsA=q$Y6Cr}prJ)MzZd zdYjRQ>*X5-6?m%8T+}kKV6kK-{u++7+9+j+T{EKsCUZ!R)%C;|wfWjqurnAD>q29qOhGEvf=K6b=2;rhU)DsE**FVwu;}-|c;=e@Dzmb&Xzo4_?zYWCPfPV!n z>}>y)%q9xG(%?ey-WHAPOOXN3-<$HBZkmcQ>wcjJ^SKeE)@CzLz_VzbHoISgzf(E1 zq1D+qN)n99Y`uNHyn|i)MS_9%Wye z_cb?Zv|1d03&ADUZKjRgg0(xeu?eb@N`TR5t}i#Z9MF8I6p>0>C~?+q8-d`;VJIi% zDhsCwD}Vd_Rhz~z?%VL=i+dM;X2=oOgprV2*k&2~6`FZ}eg^34j_}uBRiLjc3kQ^KK+t-BG6+VJGyu0N$!%Ngs2^n?Mh zFR4T%j5*NYe*dHlPRjC& zLR$>runkE#H^e6fwDM=znmoNJW{?_MTVi?yIvywYpuJcH&~~Qa}nPzZ6-YZ?R#L7C|ex_HoPHz&dfF0um?O-W#wheYKhb= zZj%^-2sw?NS+tahLZPQ`|Li}N7GTunRc@lH=h2(fu(%*0v zFP$GTC$%fTx>~RiWe-0-xixmwLOfwzhig?EZN%fkXfA$Z-nBDTx~88bHN@!x;OJ*2 zURF4REO#hkH`G?{$~{`(n`VTc(AbGHS*^YVon6l(&b~b6^fN2_k(4rX2Y-GBpQ>Tv zeu94W>oUqIYR5f9zEhErdMKa!Wi+AqXKb`51Q%BrCM(_^XB1P&;s zwZS1lM8Z9xjxZyf7^jXP)Fj2kkY)VxpzP|ZYu@dxt3oI>hx}WCQ6_&C)m2&dbbv-t zw*0u$^KVzO9}kJy@duoN^{Q)Y1RAh5VXUg{bq1_iMk|_nJV_;-_fayQyG8K~X?vxI z+$@f0dOIy)G7G+3OuiA&B#lDlVqpBiG>*o4CM#=(9SvBV)FIVH@%6yw1DFuf@6eWH zbuFXHe91Snoc+DU>D0L@CIFxX#sWYc>T$V{Zw_Wj64(Gqpv;14krVl?N|kcD@D|1W zD=iz{krpc@jp}f|kBf9&Lf(|8Fq3*y>EeLyb(1;}uG>4FTnWFowtQ$1P)7IQ%Dn*eY+(k5pzvjxTrW3?Q=se^pSe^Q>&=Is|+Cd4yZ| z8S+@k1d-@6IvVC6qVAB_wZl`)tLGhRv!0v-x3##8+Y?P>yXX9!1pAr37}jB?Vykg7>$L6GH9Q*~ zJSCZ?2}F^hJ+R5mfk3P7en2lm@8w0G=cXGWq4{&%KIsOr-}#WFW^$yh70v)91TqZ? z-DLz4KVSt@OLQ~Sj5GK;9e&p-rhTp(?}d7kHXNy|@^QH!TD>iVu$gw3im;$`H?wWJ z5l2S#4o3&4%WS2iw7}{)9^mb%1XfKe`_%P2b`zpIw#yRiBr*55=)-7@!b6FIz-x(H zvN@qXsgKO}8dL$@{$Jt07;4~h^L-ryXgeW$Z2J+TUAn?PK`}Pjzg-Y}To!V)dFSY2 zZ|0A%)oXeZ;uZvc;;`f4|EUBtBS0M?nY4uFoY1;#4d)otGH_#2MNR++`@wmkCjbi) z3&=D3RE&`~Tx4EuVe|EF!3{ozVee^0$5bAt?kO!!qS?dRZP%nNK_4wq4!VODW9K9$ z-oeb1GX^=#?PBSh`Hk&q!Z8i8$t+}3AZX;sCJ_l~uV~~-7m7&(^EFiA#=i<4QP6wX z3JSNjDEfT#^=n3c(cI3H*`e@3Z2O`Wln@u?OMEP;`t%UgFuo`%60pNXjtQT>sQK~v zk~kB;@=U3?(|?siP^xK+OjeG?q(Un7ID5GEC@G7z+;0hbg@4-Z(si6GHQ&@pcuPC@ zh$&lz%j{ZtMbdkP5?-T1o#tf^bqX$j{UgACIT;86(Lq3Vm_b08{<{E|wF4NNIQ}ca zJGEimR2MtHKCUJ@U71@~DbmR}zCl311HwgNn2Au3$gsdrAcO}vjp;{wj47sM1+^_2 zwW}7j*VmrPmmWGZY*mZdK$R*LiW}F&E=mfTT^ zcJ0Y7W1&fRU!(bYTae*~_T)x88$Zn_!cfLi*c<7ttSxQy>boqw?=qIA8s*N{QfjP7 z(Hg9uuI`tgh6D3o7eBj#ZCm0B`>KdBaR5U%Zj^t+!8>U05boY!gn5x`Hge>>M0*sv z5EO;3s+9xmOaJD>xbUZ@H2`Wo2Fs5v7O_zuG9p#j-VtY#$2@7Q z+<|E`L3!(t?}witYe~}<)J#@X=J9S=19NE#@yHSyF*1tg>;Wp&<9_E<8PxrL2hrLm z3kE^y`vh0}f-{9;oD_$A8S@lImm3nSIiMpPUcw2!%&uT?)Mr>n&Z|nAmtt<-ryRh8 z@g%Wd7v1ZI7;48(XydZ03+Ao*vuegZr^gX{*Dt)VAf3@2qKpfTx$Psr#N5@l204bI zyK?*l)xC7jOTJHbPv$LZ)S(=vWnQVStp) zyN{ARXas>mwVv(lY7H=8#SKx_h(q7Qt-icHJPg`bS!ZO|!1_&3bNba5If>nT#oU9H zpzh~}X1EJ7ROWE)5)>vqNmlv{saha?z3L*?pOaZQxKUjDx)6C?Va32*KZn%}o0L0F zqNV~FF~LC4t-F|m*WWO>9WqEev?EL&Z+oUksJ!7lEWHC;!o7JCB1r4ltV`W&rqC)D zHcn1URL*~09}`dwRWez>pmF43Y9mgKJHJF)&@wYDWwq7=8RKzg?;q@9eUpx`L5AIR*TGWL2)n~gr}F?yVWJhs=wbS0gFi%WtY9UVmAD(1URB3boe(Z?L&P9()tAS z40A2c8$0hxFL}n=kE!8Q4^eXIN6Wl@SFN^X^I0G5HU_P(7BSLCN_+rF=Lq*( zA_kO=H5~BsZFRi|Dz|LC;FcqwyZmRe$-9QvupKT!Klr>w=xqng)HaHCo%-NjPC@X{ zKLvY&8IaVWs?HqohBFo`DSlU#EQdXbg7tO!=;O?4Mqkf2N&0dUZQO`acq&NRBjUInpamK@5Cz;jn#+GbGRnOA=Uah^a zxF-)51m$Yg0{u1E$ZL5Y{JsGcA@0-KztvCaaNVWBOJjlD1y!Q!`mx8P<8Qxv9^_6b z-&E=HOcB{04P7i>4^#TQD@^ES+?q8Btfn) za$g@lqKA;=&oh$0ujMH5+90ScgHEsv&pt9bG_2E5ZnaFSLYm$wF?^dnRk44}IR2^N z&8ByV{iLfHF-GLqE@!FpF08NT7*}l5&JU(o1lT~^Z#em= zN6ist!#cJaFdnw=5Yyi}ea6`Ni%Vr<%W~_{R&m69SGw7ZqwYJA6mt6JyU8Xdwlu9F zS=J)bei|%XC$wmd>`f@%IPRes!ZmkM;;fRVwK_o5SU#isOybAUxa;^D7$j8zu65OO z${kUR@>}xWDWbK<@LqUwZY2@-;f{9!o4j&$fR~65pA-;Q5 z0AX%76%EYHEYu$a=-{LZy4pm{!hwvqgA&6o zF(vFsr>4mJB)GWb69l8D)x?*M?KJBxzEb7FALSUXu`7A)pawYSe zf%8FTNL_qp6F4sRQ8bT`4`n!Vs9c9TnYEFY!*T1C8na3AedNt0q?jm+7Oa8KF%+3Z ze;Oy=Br&>dGopo)&*tfum#_g3C-*$T%h)lYCh>m61Q`j-?+*p#*Z5WBbBBmF*Ajty z&7G$aYYLQq_LUs32W@c8OmYRbNC~umhYc>b0?RiRY{tmb4=@(#FwoWmHUYjJ0%Js0 z5@qy1<)z*NkQvw`W#~f9SW?Zc8f*^nP+Ud|#H$4!6139NXF5#e7$%RdNfwL)ukWXo z8X|<`cIj@V$#DK4IO@^K$B^0VW&DgR93bU^?=7UYtVQn|f8Ukl9HBsCrGpcb8aUkb zi&kGvhHYl&x~@eHYLWZc+(RCKDS}Bf5-%gS9j$Hv*hbTpq#IFFZRu%IkN^_(PFa}j z(Emu|GC;&}8T`^)$tJ$DkPgRLDl`$hWuv(xR}>A6H?N|VjL1#EPX#2^ez+YYn$~0q;Q}wl*jFOB_>H@(Qle*ZU{sg`jAl=QF#UZlscI2;C%r*ij|@um}B%3v%Pe+d12p+>3|=_yJac9?4nRAZA#m%f8V zG^JDf!syS^=77<0?y(byj{r(mPII`}A>nJOcE{5(ajbX$nSUR+8t}AF5S~~=R@f72 z?|4}Kg7|&EpoSTh;0e1Wmaa45wH5xsXesgh^0?+339j%i%q0%%Fyt9fxm9`N&ZCvL zVKeFrzgM2?z^`L(I8Glex~FUY?(M?HplBYO$uCZ*aS9h*U8GRN*Kij#U{ldGxZTdS zV+Q}l-d(DQN88$2)}13Ihl)FDSUX3EP&=WbO%jj)v8rz~EQ~gJet^F*jgGA+LN=Ks?N1y;Zdk-A3 zA`}*?__%PQz_^voEuaa-QIjYfKff)1qViopuc_!wC1pDz5OsJHa$7&Y&k{X267%_;O#E{7NYG&S3xi&XVD!@XmnFYG>q=i-M83}71HGll8Zfa z51r5F{w>`4EfwBS#wfg5wCx)?g|As&ow2}s*B)rn zup^iPrqAmmgaQGft<;)!y)Bd>lyY>k3C%Q3G;)j_f5a`rL3$RoKO@z95<|s%hpTzK zgm;JEy{@%NJO_#R9lmKlbX*ZDhz%4XD~DS^T5q;b2hQ=RDT>Jv~l=i*W2Ca6so^# z&N!L8?*{R`D?$`P&AlS6VK^#z=m)YuGB_35jS5?sr`V3_t+rXYnilJAnxlHAhup}_ zxs_Ju>-WwzG^2M*ZWD&QS+YOL4zPH4@TOnRnocmpflRv6W zu`o@G0T+d<5LldTRLER&M-Cp_cWK!B2GT90GI+c>oA~l92yN+*CF`zO^PA6 z;t}0lp4Wzd86~aKZq+lRzgE0Olr+1zNe0so)S=Qx^o_6+ zpQQx}x)v5O9~|0&i6De9mdH;6x!1K0H5P!F( z3^xRwAy22y?}1x=j30?a?$jDTDCNqFcrCyVWfDsTGlYZZPE|=JN%S%llIO%&g_ujl z*Mt&_aLqG-(35dp!Ysvhv(~4>tP-!-lgaw|;-}55+r;xE60q49GZuA5U~F7x=4Jcv z+|zY_8TiLDKZfDBw`Zj*@bQ8Eg|Y11F0&NF1eUJ4BKR?&XFy)TR8E3Io0wnEZ(M7a zoWeYKz9lmFc6U!qYp0dPG}(72KaNe?may~AvpRvDEaDuTb|uqX-SQkf(5I18p{MG{ ziNMS|i`U}E>h7b9&qHUVQQ|IEC&(eus-BdIuawcq@#X+Ywhuj_# zt^QkTk8?^Ug!n21C4iU)=p;;XW&NXv3KZ4vo@%iNs=uTou%L_4uN`Q0Ni1&xx(oc2Wy(@u0XEw{F7Ak&+qDVtg9i0YrQ42LXw%M@NFz4=8CSy9>HK=O& zn@}_}ADfnK{U%(ZUO)BHKim~A&eV2Gj+K-4Q@<+*!H>w*PM0R8SEnyw>-Z61aGKZH zaZ8kEUGU9KEefe-i76zrV*J7S?qMbgQwzgM!5zK+M0l-7#H;YuX>H{9!Jh0P=8RK=ge za+GfixAZmE2F|~2=rWxP#+p>k=9D{rH+GVhRSA?`EEP3nS5}eV1OyE%45H{bLM1r9 zCOYE8S>SF5RSWMkmPCBf)$10A|8TBonf)xZHOvmJ!y-4zDmMr5MZ0s|yBa#k70mYL zRsG~axZNm|V2Y3U@^5NYNxpTzWzwZZiZMjlClves#Tnwhq>gWgFT!S?)~V&~h&RdN zKOd*NIdZm)&!Tl>pts3*Ca+I;iseCTp}&Vg*h)udiX@!Q2bb6#rm+>I7 zjm3xuM>cyQ;xA?E7x(@;D{sla(*T5~xoqk;lbL!g9! zFhLg5z}L@FZ}+YaRNdeg)i;Q#_Sl?~NeF7I8s*cNez0hBDP@|TO}mMu%;W~o^CBI5GGS`N~JPn zXo}s@lBNnu4+>)?zn!vMFm2M&<##Hh;>V@T0W7dP>8tWwi;@$_CgXLLBM*|bi{UJy z8+5fP^{CWj@aLfJ>Mk=kKxO`<3sW5w5ItFABHV1bTrO~Rd|_Se$m>ZeZHN-{(-pMu z{?gZXcl)w{iery2YC(uEItl`DA}{&Y$9DfshTjP)?b~WfOCmY@VUbVD`~kT-#`nwBS|)7ydowTS z7n6s^{!;%R(&1{~TJy*M+#&}k_N@>;yK0goa5<-UHXUM z^s>{q;~FI0iTx-75UAiuwH%P1$}csBnV~To7@mtw?%TE6hMDsH1S2y56uZpVEy+~K zwx2NHCA&1kJ|MrAI*X^?ZK>T$5tP-0K7X-<+vee(Qk;B&{omi9Q2rMsT!5{u9S{Jt zu(SOS&rw+NJd_PVK|qZD&L#BUpGMTg-qFMe05mZc`d=Ua?QTy=PN7c_Eh|_!A591X zozH!rG#}Hh6jq1}lz_35!9*n`iOvGkH_43!ZBdLHt=$N`&NJY4E9I<$}6tuXID-fHEx7qfC zq7&u^liBEpBtISg4A5LE5(`U_BL+K-t0Ui)8rN^xO8%rrPkxQcV}5_g9tr!_E%P(V zvydmiPyns2NVttvvmS2@V#)fKgre2Eo^6|DC*f2Q&|(AIpbk{Q>-;Tq2ocLKr+4wQJD>+@jF{z;87bUk)uZyacf>!K64WiopF}e9^gXE z*?X!!KZ`Zep#PKq;^4Q($}IC^NNuuGIP3Wz+`G(M4U-ZO9{h)TV;irR3M464?UKYEL#swl{@Scl** zG@+W?Grh5%55BH7!YGi)XRkh~_Oma*s0YoTYwKOFay&0vSzIq`wzqvjT0##wxsf7> zvIHsHusoVypsPcK&BI2u!RY1+2(6`ZHaI73Ugacvt6NKWH^F#34H2MJQV$tYiv52NFHxRvhk$ z&<4pvAg7UdFf33SlLl?Jkg9957kK97JN4Nontub*o8Bk}zxM-MNpkXYbChJLWBehY zm_!fl$5F(i0qlC*+03aHPLMwwBMdc_bh*%1H%fd(^<{oS6v8VlTYnm#C^Uy6v4*WXtSGy0Fh*pk_!sx(lR=q=0s7(w2swza@W4WY-AaH1{CwhmHfkNmtui_s%1 z(X6gSAET*&Z=Gv(cSJVEN}0m{3RsWfK4cYU(l%8E)WZkg(O5Dk&_@Ga1hn*Rr8G|F zED8ZIHEC?UGb#xoRG!gtfSMb*Y+i5j`mqORTbT!Z&i>03AQm?r`XnPV-rg*_#c5)m zGnOH53rft`JWdbeN`~1o_ZZtY++_2UOuJb;Od@)!#8jd|YPlDwFHb#}SOAk{AOhpWEK6mZKZRNtFKgb2Q)JiWT;^=M(<_K| zY!t#qiP|JHQ6S^*77_IVSdx~4pRAWhu=z0jDB^(mRl6`ouf7xF1l-V*RnDHlt}gF; zvnL04LJdUwk7LXzUkdA$X$Suz5FEn+7O9CJ(?1hwMhYw6+yq|;2&@!yAb%6Q6xT`} zc+I+Jl*GnXWL#VLU5XS*4c}`RKY4BbuuGk05j!s+Y`g!bCxhLz6LFP);f4qPKk^MJ zTT2ropopEV6A)kv{11Rr){(&#!s1I5T^@+8)8EspK<_}~rn|2y2+AX>d_@|KL}Xr- z-5=pbbukP6tmF@ygZ35lRXyd%EX8_|!f@$2=Q@5f{rr4$Lfqk5k{Kbs%HJ`9>4)D> z-P697JV6k!&>Nd!hQFe!`a>_`<5Z#52NOXc%`d*oB5QV7riNUTNmbxs0u*K}>teQf zPzhHj15ZYpSTns={Ua>U1O+Y@mA?nr7>1T*{FC0&FQ;)P8o}AIqn8gPOAMma8w_3= zf|D3VY(&+jG^A;$;-kRgO)k#L6QU34bGR_i1(3Mmv;vHZd6H-Z^4P?Z#*il%RXsG1 z0RsJs)}=8R?9t@?uOOdOG7EZr*;P4;;zI|I#e|!(^~`xwVPrP3tcyBA z+Ldo&sEYxv)u9Kfx%*#d{Px!HXI{<(zd9EfE%3&`57$z*_Fnhe>ZG4C3(Zpd@hF%e zJp|G`DCUHz3XsOcLGxP#R)c4~8S}9qFIS;q&ae@-jP`czJkxLj-BA!;-)iMh-$Xmw*7ee8JU>MKY&Vt#jE# z_|6?bA2Lf}B(y9ReeT|Vv>3cgZ>JV?C_NvR7YeAw)^M@0SRn9Ju7Tr;u_eFe71?UA z%CLa#|I03G^Z&&zhki!A>Xn8B3tr^v+Vi4y2NQDNXPO9~5V2__9gJj(a_Y$Sj%vey z#j6&pY~d@%qfNOKuLx6)k{q>IAgpc=k_|$$q+;@4u=45)vd{0qS&sdPkcrs4WimFJ zR}ey=v`427DYatvVZJZcAx6s_DCJ&Z3i(XDuNs<(42P%&B&@&{-%58iG)&a0^4_VN zk&ZC(@~}+H*z%Oa&cO_trz;tnVZ)heto)?N#(5YJ_}#e{S3MJc!uegqZL|_Dl-0ntE?DL9IUrPjyg3ZluBLSX+cD>1#A7?w8LY+Fj+p*HN1_$+YSNlh%8L+-& z>j1gkk;Z<{hGaHbX=*wCP#SR(TCVn~x0^f*dAv`f9y_VML-7xg$L81kyp`q?zd*U4 z;wkV|4469i&uy33q|O5nz5kh*2Uc+9N`Hw-_t#A%{$IGrzaI+0n2@{=tx_RHZ`LE3 z2+$B8_v8$P6;V-j5s^@0O)T<^EkTTm4;S>i7*xPO<#|k^|xiayS4>D zb$3TZptPV{nQy5Wtz!lwck*XMJ{JhbdzjI1q-i|+Rl)=L?P>&+sX z6j*>EKZ+KibkH9CksM@|AcaXizvVW?fV|ZwzrWcwWk3z(%r`MUkzwW>fGQ`Fp!!NjahZE9aJV9PC004+x3oOVLn-XTY^ zmfMTk>+PFH{70)v#6$a-|BZ1AQ2!&V{hto;|9~{LO*>Qxq%WJxg!*h!#XD@OpD=U? z@|^sA5>gpMoKWyDd2Db&<%pSgd$d_s@~)mR&kAFxcHVU5NmS9IZwt_$6spdTdl1&Y zl+}zaTzPr!GPftDraHggo?JmF6sPkMLm=nH1)#7V`3|RB_Z_-VX{&kBs|~Tz61r69 z;l&+*qrxHic%pKYFu{c12wE%TM~bh)IbHS?xi_A`2b1Bw%+!P8?ORczZ)0xxV-1l_ z;*4sw5Bf7nz%xMuvHJACy;*zt^Y-T?Nfs0xPu#`z+QKzrO-4}i-h?eY?CEmMkLpjF z*NpY~$IzbtNbpgC7BJptj6tHq*3ilKNT+z$%*AVTfHz=HtMejPc9yG9Sm6u@>(9e| zj*v#$Mm+%9W11Gm(q)-)e%$08xfhIfIxfg+_h~F*R>L^E(`ErUH{`BNNjgKq9I^c4 zd$mGF2QV3g12Ce6M@Y9&q(vQerI65lvMd_L*rM@#&6HmFiUn)Q8Pk&>l-$M}*htvs$hmPo^H}-@>ZDXhbsySy8 zI`6ZU51Iju5Hy(L10hLe#*3nSl3yDQdN&Sq2CXerLVRC)oRTu2C96V5=G&pknc4Cm?<6OX z`sl7=T`Z5neigxw0ulx_k|B%l`uJ&!6IC=)cg^;FkO4QGdE9dk%jd z(LOy=7@b)4|Fw3NQE@KWHVF<5!D&3W1a~L6LkO-3?$8ii5(v;Z!L4y=T!Xt4B)EIy z9w0agyxjL@CUbc+v+m5??qA(&)!zN}x9WW7RGqzR5IUlP9j*WTj&;;vVnDIM&}(aR zPYk#U8D0ZLw9Wi;22hQblX%6-d{Zk!CJZpdcyFJ z9<8B0dL;eFC-dLhwEoFl*4)=3*1)_sFjVN&lxtIi!4P`(4~Meb^kqhWrZ0+VLKi7|bv!;=sS={#HiB8XE=Bu%(xAfAL1 z`CRcR?3J*F1u#30N(p|O4GNp`E&dj_jG|9o6@QSBeWAQY#Nzyvws<4;I8cv*LwJQ_ z|C5iuktCx|(TofTC6QYn#i@0m0<^{!l$_th8aZ4>e}8g>#>?!#C(F2dt1{r3=KMd4K1rkwP(aJ^#TLXfJhwb!2^|EkmLP zAV_&a9dD;*yXm?otho%CRE7Wb4XIMp^*ynxxs%rg)6A4TAP& z!7pUkgW|v*q-gacpbKrCZ<4e0U9h5(f|wfnlWgWrebyPI{4_B96aEiK?gbP%CRjS) zWR~L1G4LiDWLWlXPsA5mr$TW`h`|8e>298_5r1WjLg@zbGFl}!lNy0L`}-6ie=$?T zYZnB6%j)E!oUqWom`(ROgk^juZvo{V7`zRbcL7uL-mf65`;J_5V%XW{wZ9tF0K(H! z8KMDh&1DSnWmX8at$BrOFLIU6Mdnnrmtwq(8W3jpEgA3uv+|Stb%(keeqdXc{POXm z*3fLX;l=H#MXN#QCHjl;8SpezAp!Z4yWgQ1*opliELe6D3~i)Wj(LVjH@c zHiMHvfGEtI2a4sn6oOe@)n?I5NkP}F>XqJ&fJ%$^#*V4t-M6(ucKs7~&%k%PRo9Oc zxDZRjOg9rYD8_E&=|uGord#L;b9Fk_X0I z>s#Z?zy;;Tw&VsXVmZpWc1}aQ)?0{WzOT~V2fRD?e?7=2QIf1Gdiyc;_3KF6`+GQFkGSj|v1b99euii4sa~H&VRB&$@f$5ILD{*$n}j@O zjZg|q72F#vY3hIizjr_Y`$|h$ZoK|BH#q8yV=6`9IE_5u7|@l6(_LZ~f?B7!B;Uel z8-HC*Z7nb03funD~SE0!rLfK-wrnH3>gfrTWWHp!#_ z{sOlY;AraeUgP&?bi#RI4R2O;!r9@4ual1_hNaoVx4GBQ zOd~CD0_hFA6)2Jre?I)XuJakI;tP-yDN0f`h0-K>#*)F2E_3aJ^_`XL|>cZk? zO&r%`BPg*2Db#f}kFAWof;56Fe>cej7?oJVk ztLc*A5Gyo4cl~5+{@CRkqQsUcV#A1m3NKT{ivVyF;<9AIST>BC!GOa|Q5MZ{7_^fQ~bv3)zc-cPMn zoDBKUYZsFx^J%1fL+t$;52jlyo7eI_BDauo#w6lFiW?P1=C0c0GoQ!9He;q*Gjcrg zj}p2`3)hvtx9FO4vnkb&FKG&ayq}1!$K4a(e2-vNBhVEZJ3JTaKvYB@?-04uM_Xx-< zWqoo05ygPW;^h--dq@((|0tou%5%iWVzfb?Dfqsh)IsuNYnM^Gon=W2V{T9T>^UU= zV7nsREs8FkomEsFQMRr(PH=aJ;2PY6yF+jbE*p23;O_4376{44-Q5Z9E*sa&>2uE* z-RE|{+^UDFm;bA=s@7AjHRlKT9CH5yMh>H{)I%1g`=Ne**U;7S?=$`%}f9A zOU)5vztBEh1Xqy-ge2~yJ9SzorLunW0f>6V-}9+~d`D~SKGscd&0Ees;XPW}^j@O! zCZTtRY2qSyL=m5^Xl1eo^VwpH?rDS3jR=x?ffHY+lP)%2A}H0nnn zPb3w68!nw)s*cR>Bl2Z8NjY=nls98ol88Inu|g5H3T{2kwRTMVFod*$tpS-!=`Po7KSGf$mPHGh)#37N{HyTsM& zO>XoF`(Ae%c3y_K+f|nSTFC<9J>KGC0G@#j3ks0f8vcsN;6_&z!s?&jTJ?94;;HUA3q$u#EFVEBqRTBt5BF)(SzERO+M1Afdb4^wn8w2#`s^& zd?JV)qX~9z0b}o+6_KscFM(y`v5mfb73#@QX6onDtlA~>6bdMg5tU@Q=aJHggE!Sb z#XVP0RK$|FZe{;2pY!}lW3xOD)$dL)zFo^_NFIsLKTUkqoHw7(Z2DTCX2u4K;cZEw z`C#Y!;C!3*CgK|smhKW}R-!pB2Why9W6u_jqO5?wl!J111!(pxWY1zHm#Q@zICINt9- zWCSs?-Gk}dBw7}p>aR8v($9HFtOG3XV;I1`Uq8u*M>|RO^~TB98q7C>VaV!@i-|JDY-+Sj>8lY@2cg^E(e-iIm{S(}EanTtSZ5KZJrBVp@>MMa(e8i< z&#ln>KaY!-;BQ===9|KSHWEW)UPJS&F z)8q^!_a(1eN(SbRA5=o;#=8#l39lLEVoRKh_naePphf)VvYAO|9;dz^EygLH+ZicQKU?mMKpkf)5OLoW=3T7LIQuZ{MhRtWZqazLR1iX zk`wZ?&0=rkWJ0*&D*EVf4lSl-P<{U!z>chHLvvJ}$fO=gRshdG24RB*O;Jpox64oB z=~b7`%Dtsedf|}rRv2vt#_U!izlVG-$d_wlk4;;s~E9nEO zcBfcVJ6vaX9Ogb`YTv+deq}t8DbE0HvGVUwRN#F=p~p4opv3$3v$mr2wDao#=&(@B9)wFj8qJBL0*GTk1eeS@3mgVLU zR_eF{f+u#j_fq+QUq!l!SEAScG@8BKSh<#12ya>4dH6Lp2wb5e2*)`Z6{i9GY*T<; z$k0qV+n4aX0b3qxt}1raGa8v#U(D3jhmn#QeZX1G6G=yhRRHPJyxLbgERtESn+s-P zMy@v_dhK^2kotQk-jQfnu@C7DHIc=@O6B=kJuUW@W4JpRe^=9wzt*dCmSoZ{(J1oE zu^Dng*d<+14_%Kh2S1BYd#l?Spkmv-fX||hfI?q2LXXdP^mvR&7DGT0J=S{&crblp zK??7jx-4XdB{87z<{3D+9?{3Ft##!dDJZT{id`|At$@hoy!|sm%~GM?`Dw=G%iaI)HCAG9*Ngswxs-SsN`isIgPHme_VV(0T| zHdl0NiQs1R7fUX=0{m&lx2~8udP@GDFNxabMd#PM!`m6ssI<5_Wyk!h$K}iHcUqGq zKKCpe>uNA-VkyWn>e&oHn;FMt-5=U+;wQR9NNb>E0`|>`-KO)Hr9Cm#ddw?8o#PK* z;w9SDEh>#52^#Bm%(`4cA5GmROPBl3L@JLQh1{l^#J`caV129pLrL)~6i#vt*3IPF z#mkuZFcEoMw2eK?AiVW~r%}9SOB_uIzw=>|-qkLZK9nk)WnROPIF|WE$7gg!3DcR* z0bBMXB5}|cBB?Jr&-=2}v>k;+o{YGVUkeKy5t354_d?^=*fbmv=z217n{~-rK>fL5 zGwI_>dX=lv=QK|feW%YO&s{NmWB%(UFgz`;9XPJj*3w^?z7XIXS9a0sgRjPxbf3At zs#VH#OX{;MN~p4A5&lWOD6EB>Pp4SLuV6eL4(R5Q8;V|JyOmV@Nz?rbPTg5`YpIG$ znSiL}54L}DzazQBc*ML}wW1X8nuB`a-p!XapGZY1^>-~=c2!I2Y1fopCU9=aEuV>~ zgcK6f?CK-PF;yQd9x#Kt-kiR5ek3|}MJa@-^KAu&tfA>yg;%~SOxip0*~=xpvuCV6 zZ11?a%n-AlF{Y+S(7>jE)ex_S#f6#lxUU9kd0yu`%W?Q$;&GO$@RJt(0061 zW?pGm%va>>+2rFdfN38+I|>kRP}%pZVg5*|NTfnW-e)b0j@7f~Kc{!6NKh-N+7CJk zV&R2!kHz)&=0C8=^2{4DUz$KQsjA2sS3H>Z2!W7(l}Ipk8AK2LT+Zc|xl*rn2AUyU zsF$cNn!g4IWHt&GB0(Ks_^| zFq@iQbulI+oE^`DIX8+x{ISoE(#$WZ6Ch-yt3)Y57`Xo=x&lGY$_Ou{YQcguV?(Z-n>H}8-e0;HydxIPweG-Wu zna*eDgFlJdXpS=f)n+7nwaaX8NpAwb=^?=|_w|^1*dr}K{ze?BMI<<6z0-Z_nAWlj z8ImWW?%~`A?pTv5dytUBY6=YssC1SjF-3;F=&rCmCvi=Y=(p!|$=Rfv5WlKJ(a?2T zkNT6 z&MNN|x;wjtev2mQ4e^sudNl|FooKE7B_!#Hnbj(d6=sZLRZ4lWCWO?{9yC$jyf_Ds zNvO~Hc@*T`DzOu9mIW8E^MW&Qv$6^h7iYpi4PU;*cEQRzTH`OW;Wxd;yW?&+z$4D% z@Kc_r6$q;e)CwpLf8K7b2}FGN&{}Q+zNT2;{WAFJ*hMb$Q$d`gKPUTN_0j1{P(3OG z<0WJGNT!pe zO%Zx6&Iy6sys%b!B@x-89o|46KA4RiQL&y}m&};>4*Q9o!7`M&JV95BR{Cg`5!J3y z9tr6v+8oq_xN+ipzcw(crxP0|3~;6S+7g$NUPyPLZFqWJYHJ*RaM!4)}MQ{Q|rnt0GDK z#4#sGO^Ib)q;V&VD#{=26$7?cSW3o5%i%nPTRfeRm8O-A**eBJC8J%77P}&bdHI~> zkv`XL-rkn>FCIcg3EQ~eC@}drgB7=>Uy{jE>^OG}P>?tJEf91@KJDNT;tf}cu$#Dv zkTHau{tVx+nfA{KTotpY_K@j_++ljP_7CH=-2E{Cez0iJ!L2VzUOB<+06Thvjmi?{^p$ztpW3E`HdxX|*;>a4rTTcH}hoK+& z{VSwhhdYzSIw!1==}km>TKclZ`So4#R=bui({D4Wr@kk*r<}w!P2F*|W0J;6DXSoR z!(VIEuMc+``>&Kj{?`|&-}}RNeHrIZFLl<0wEjX2qle0tnyDs_NSZ47n_Dwue9SiQ^tS(?d4j?IUU> zMi@OyH$t%sfqwrEdJuPJQ}4z}WU-XfE7eXPBRt?oH5}VQqLOJa6=$N^4DLHQI z%k9mW;1jxw3Re@1*bNIHJrDo&m#WIA8|O!EutIFGZQlG$r3tu%hiPoW63Gc7gs@8* zRrpy{$rij@t@u{Z5T_0(j+n5PFy6oPC5#KTP%NXv27wyAKzklxELxPrAs>ZI1<;IH zWxaIqts#N8g9$uC*>Il<3aiT;EAjB?)6)BajijX z85O746P?UD*2=O?ZC0#p)6#o@fl);#-WXT3IJ@u|5!betV*ddxcxaq6P6=5!3G$>H zu>9XXOc*T+Im5(7&&zj7+)DtJV&2ySVb9qDCpQ&+9w&7*f=}sY#;5$~n z|83(IUR9laeu^|tb&}s^27J?ko@v`FmLzOhq6TJMg%)n$zzAg<))TNzlKFy#%pU)D zhRHtjQYw<{YpcqR_bci7Q5A|dBBuwew^ug|<3aIvJ8-he*zI?&7-!enHqe)xTUA3` zk~27;Hn8jZX3c}ZldhmaV($@$;!eNr)>i`Peb?yV;Vv}{aI>VFX+1i6WBi0$ldDQj zk4S(UR&;pomP6k;2@lPKXuDXrZX-W5(TV=Ew{@KA?5`RwfoSb^AINFEc$2!~o~1#RR~8xMY6?)@!u+Le9! ze^nl$@}mlbCGB~=BQv4u23m_Gisr&aiAjJ)lFp2+dt|q@Qai)o6D0!{D4LejFhVWG z!G6;W0+%O3FlHn54S{vvY8$=!B!b+@+pQbjgxX|g&$^Y#egCWAm>0>!;I_vewm(?I;9b$h>W?muO-^{l}2QSw7sOOsQW zzI1um)>Fbs^T0Ug8P$j!hnx#dc~x7~J)nVOtB#4_0vXG9=2g9{RTq!>J1+C=R1umE z$#>7|Y>CjxxTxQ&1=05m_YP;@H`q$zGpzWpTpO0y$d_Cbp|qqDy`!2O#o)t}g{^_{ zn`o{LzE0|LedOdYfwe2)G4tmy*j5tfuV*|p<9y9RFL22E=Dw%c+0!AebW&WU!{^KG z7oM?ae;cQlbCTSx8NzJ=R2Caz+dsEY_mm|(lkin=_2dzE@UBp3z`M{z6v)b9Ct!m) zus7o$zp8UGbRC&G)&O1EJMqR94p~G2_qYo2H@h z7zE(a1;+*33tzhD)b567EsJy-K~8qfgorNxijXXX*c)3x;Te@yvW><15^8(gS!{&dhG+Sk?(OK1*8qjMb z!}}c_Hn}XXCD;Uo=sYCiWap;lOA3`0c7m_RWt@jhI2JVMu1JRv*R(UrAlXtEO88oi zrS|Qp$Zal1=zPpsGu53PT_}(xjAQ;D-FH|-AZF#eEL|GO!{PJk_6BZqV zfz`*Qf1yybL-@ST>`uYq9!taMD_wokWU`T{K&TMUQNB$239&n9mh$rO1P0buQVXex z2a7bSRM&b^IixB5w0L^s4;P&j?og`UT(R6k#>4rH<~%OB;zwh`&uFSBp@UUD%SRrr z+r;4ud>m;RYiKNO{^q|B8ckjtNqi;4k~iV^Mm|cL!pJ6Ot6YzkNY>_t*b&cc_s>2t z_qhYzsB?zPpXI4@t{r9IQZH=M17G*Z@`ho5goeqV-PcfN({fIEe%4d=kUlW5{VK-D zfs%FsRaVJ?wA-UH58D9AkQJPzsNXIj?6zNzlXXz`vAYmul;HbuGO4XS>!D!nl7=r> z<1EPkwkXYbA5CORd%UXM!4L{Hdm%o(OUm2<@qsrl^(M6ndpx^S>3eMy2wq3a3;*mq z<$%qykNrDs(N!#T9cJuTEqF4FI%fpdKMQ9Zx1Q!GKpYuIN)xNU?t${lWiyTum|O6D z)~Auo&Go8_`|#ZT=7i~&+th{y^NPOsI1J;gyK3_8vfOak{0~Qr&hR95Nkx3#T~e;5 zPVEfCHZ@xOu0rfR%u)4ZCnQJTt1%B;q=NMcujyx-yOXIe&@!zq(9E;3H*L+_xOS~u zeQg*BJEEN|4Xp~4nv#5gN$QMOp&{1vN>t#n(Y}(BEprC+b{`*M&BBgg-Q?Wr(_8b< zn@`iSyhIN$@e4rhJ(PsXZbTEr2~XOrCUVPQtIWH^k{)(Ryif-<%Su(G+s5+hf}IEH z$OlE$fls{$NvCmuh&LDQrh8M$uBSNtGtnKax~G+Aq79=5|DE2WA5cDl33F}iJ)ELA zA_Qj}=CdoQ>>f$ACaWPn5Jk}fpD}L;^=$E_o2I#kNF@gqaw!D`owiejvcp*-63GZ- zFyF7)AI&Dr;O9_-xBn2(v1K1QfgUP*2eUbGeBE#7az3vcgl=#mwX$yqhf;+YZ7RPk z(JEo}w}}YG)-cvnLv^MjCrnFb@}O7GL|W+<{blp+R)xW`#oHL7ZEzDbshRYk- zZrq4wk%Q`>(D^NK5JJ-;o}sIQb6hyxw zCcih&`vlxfu|6)ohsA|)UzGnvNd1(ua$!1PmMK)SsG_ z77Y&%3EmeZPsKLFo@qJ*nq1z3*OJ37Q5RqRx<@?FJPrEC~ew@WKHAL?5o4|2c{S|Le}d z*44twj>**Qzr8v3PRtHYA2uB)7b`R8e

03sj@V`00@e`C;6F0RV9R2lH_)4h;C< z|6h``nTe~Dm5V2{gT1YliRV8vq(00&&j|^Xd+(nB=<5govVUPH!U6!c#{cZ?`G1`~ z|4f67kx?r+cFoq|hX#~qLj&;sh2{v0@X^!4#ny>~$;s8u#R_P~#KFw@Ap!a@|NL)? zf_F~31v3Odj2{Ak_Ae9&Q51l&Gmy#E#mbh6oB7MXOWPRiBS_W(05*Mq|CLU4K>bJB z#mUU z3iN*fjO~nVJ)N!oBLV9BUk&zp2*B?y2ms>05L7b%oA7T&{_iYI|C22atdYh1|8g=Z X%fWoi0ssKv<81qwX^Rd2^-lf|qy??x literal 0 HcmV?d00001 diff --git a/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java b/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java index b7c742a99..ecfe3e34b 100644 --- a/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java +++ b/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java @@ -29,7 +29,6 @@ import org.junit.Before; import org.opensearch.client.Request; import org.opensearch.client.Response; -import org.opensearch.common.Strings; import org.opensearch.common.settings.Settings; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; @@ -165,17 +164,16 @@ protected void createKnnIndex(String index, Settings settings, String mapping) t } protected void createBasicKnnIndex(String index, String fieldName, int dimension) throws IOException { - String mapping = Strings.toString( - XContentFactory.jsonBuilder() - .startObject() - .startObject("properties") - .startObject(fieldName) - .field("type", "knn_vector") - .field("dimension", Integer.toString(dimension)) - .endObject() - .endObject() - .endObject() - ); + String mapping = XContentFactory.jsonBuilder() + .startObject() + .startObject("properties") + .startObject(fieldName) + .field("type", "knn_vector") + .field("dimension", Integer.toString(dimension)) + .endObject() + .endObject() + .endObject() + .toString(); mapping = mapping.substring(1, mapping.length() - 1); createIndex(index, Settings.EMPTY, mapping); @@ -194,7 +192,7 @@ protected Response searchKNNIndex(String index, KNNQueryBuilder knnQueryBuilder, request.addParameter("size", Integer.toString(resultSize)); request.addParameter("explain", Boolean.toString(true)); request.addParameter("search_type", "query_then_fetch"); - request.setJsonEntity(Strings.toString(builder)); + request.setJsonEntity(builder.toString()); Response response = client().performRequest(request); assertEquals(request.getEndpoint() + ": failed", RestStatus.OK, RestStatus.fromCode(response.getStatusLine().getStatusCode())); @@ -215,7 +213,7 @@ protected Response searchExists(String index, ExistsQueryBuilder existsQueryBuil builder.endObject(); request.addParameter("size", Integer.toString(resultSize)); - request.setJsonEntity(Strings.toString(builder)); + request.setJsonEntity(builder.toString()); Response response = client().performRequest(request); assertEquals(request.getEndpoint() + ": failed", RestStatus.OK, RestStatus.fromCode(response.getStatusLine().getStatusCode())); @@ -298,38 +296,36 @@ protected void putMappingRequest(String index, String mapping) throws IOExceptio * Utility to create a Knn Index Mapping */ protected String createKnnIndexMapping(String fieldName, Integer dimensions) throws IOException { - return Strings.toString( - XContentFactory.jsonBuilder() - .startObject() - .startObject("properties") - .startObject(fieldName) - .field("type", "knn_vector") - .field("dimension", dimensions.toString()) - .endObject() - .endObject() - .endObject() - ); + return XContentFactory.jsonBuilder() + .startObject() + .startObject("properties") + .startObject(fieldName) + .field("type", "knn_vector") + .field("dimension", dimensions.toString()) + .endObject() + .endObject() + .endObject() + .toString(); } /** * Utility to create a Knn Index Mapping with specific algorithm and engine */ protected String createKnnIndexMapping(String fieldName, Integer dimensions, String algoName, String knnEngine) throws IOException { - return Strings.toString( - XContentFactory.jsonBuilder() - .startObject() - .startObject("properties") - .startObject(fieldName) - .field("type", "knn_vector") - .field("dimension", dimensions.toString()) - .startObject("method") - .field("name", algoName) - .field("engine", knnEngine) - .endObject() - .endObject() - .endObject() - .endObject() - ); + return XContentFactory.jsonBuilder() + .startObject() + .startObject("properties") + .startObject(fieldName) + .field("type", "knn_vector") + .field("dimension", dimensions.toString()) + .startObject("method") + .field("name", algoName) + .field("engine", knnEngine) + .endObject() + .endObject() + .endObject() + .endObject() + .toString(); } /** @@ -348,7 +344,7 @@ protected String createKnnIndexMapping(List fieldNames, List di } xContentBuilder.endObject().endObject(); - return Strings.toString(xContentBuilder); + return xContentBuilder.toString(); } /** @@ -410,7 +406,7 @@ protected void addKnnDoc(String index, String docId, String fieldName, Object[] Request request = new Request("POST", "/" + index + "/_doc/" + docId + "?refresh=true"); XContentBuilder builder = XContentFactory.jsonBuilder().startObject().field(fieldName, vector).endObject(); - request.setJsonEntity(Strings.toString(builder)); + request.setJsonEntity(builder.toString()); client().performRequest(request); request = new Request("POST", "/" + index + "/_refresh"); @@ -430,7 +426,7 @@ protected void addKnnDoc(String index, String docId, List fieldNames, Li } builder.endObject(); - request.setJsonEntity(Strings.toString(builder)); + request.setJsonEntity(builder.toString()); Response response = client().performRequest(request); assertEquals(request.getEndpoint() + ": failed", RestStatus.CREATED, RestStatus.fromCode(response.getStatusLine().getStatusCode())); } @@ -443,7 +439,7 @@ protected void addDocWithNumericField(String index, String docId, String fieldNa XContentBuilder builder = XContentFactory.jsonBuilder().startObject().field(fieldName, value).endObject(); - request.setJsonEntity(Strings.toString(builder)); + request.setJsonEntity(builder.toString()); Response response = client().performRequest(request); @@ -458,7 +454,7 @@ protected void addDocWithBinaryField(String index, String docId, String fieldNam XContentBuilder builder = XContentFactory.jsonBuilder().startObject().field(fieldName, base64String).endObject(); - request.setJsonEntity(Strings.toString(builder)); + request.setJsonEntity(builder.toString()); Response response = client().performRequest(request); @@ -473,7 +469,7 @@ protected void updateKnnDoc(String index, String docId, String fieldName, Object XContentBuilder builder = XContentFactory.jsonBuilder().startObject().field(fieldName, vector).endObject(); - request.setJsonEntity(Strings.toString(builder)); + request.setJsonEntity(builder.toString()); Response response = client().performRequest(request); assertEquals(request.getEndpoint() + ": failed", RestStatus.OK, RestStatus.fromCode(response.getStatusLine().getStatusCode())); @@ -520,7 +516,7 @@ protected void updateClusterSettings(String settingKey, Object value) throws Exc .endObject() .endObject(); Request request = new Request("PUT", "_cluster/settings"); - request.setJsonEntity(Strings.toString(builder)); + request.setJsonEntity(builder.toString()); Response response = client().performRequest(request); assertEquals(RestStatus.OK, RestStatus.fromCode(response.getStatusLine().getStatusCode())); } @@ -676,7 +672,7 @@ protected void addModelToSystemIndex(String modelId, ModelMetadata modelMetadata .field(MODEL_ERROR, modelMetadata.getError()) .endObject(); - request.setJsonEntity(Strings.toString(builder)); + request.setJsonEntity(builder.toString()); Response response = client().performRequest(request); @@ -756,7 +752,7 @@ protected Request constructScriptedMetricAggregationSearchRequest( builder.endObject(); String endpoint = String.format(Locale.getDefault(), "/%s/_search?size=0&filter_path=aggregations", INDEX_NAME); Request request = new Request("POST", endpoint); - request.setJsonEntity(Strings.toString(builder)); + request.setJsonEntity(builder.toString()); return request; } @@ -779,7 +775,7 @@ protected Request constructScriptScoreContextSearchRequest( builder.endObject(); builder.endObject(); Request request = new Request("POST", "/" + indexName + "/_search"); - request.setJsonEntity(Strings.toString(builder)); + request.setJsonEntity(builder.toString()); return request; } @@ -795,7 +791,7 @@ protected Request constructKNNScriptQueryRequest(String indexName, QueryBuilder builder.endObject(); builder.endObject(); Request request = new Request("POST", "/" + indexName + "/_search"); - request.setJsonEntity(Strings.toString(builder)); + request.setJsonEntity(builder.toString()); return request; } @@ -865,7 +861,7 @@ public float[][] getIndexVectorsFromIndex(String testIndex, String testField, in XContentBuilder builder = XContentFactory.jsonBuilder().startObject(); builder.field("query", qb); builder.endObject(); - request.setJsonEntity(Strings.toString(builder)); + request.setJsonEntity(builder.toString()); Response response = client().performRequest(request); assertEquals(request.getEndpoint() + ": failed", RestStatus.OK, RestStatus.fromCode(response.getStatusLine().getStatusCode())); @@ -945,20 +941,19 @@ protected Settings createKNNIndexCustomLegacyFieldMappingSettings(SpaceType spac } public String createKNNIndexMethodFieldMapping(String fieldName, Integer dimensions) throws IOException { - return Strings.toString( - XContentFactory.jsonBuilder() - .startObject() - .startObject(PROPERTIES) - .startObject(fieldName) - .field(VECTOR_TYPE, KNN_VECTOR) - .field(DIMENSION, dimensions.toString()) - .startObject(KNN_METHOD) - .field(NAME, METHOD_HNSW) - .endObject() - .endObject() - .endObject() - .endObject() - ); + return XContentFactory.jsonBuilder() + .startObject() + .startObject(PROPERTIES) + .startObject(fieldName) + .field(VECTOR_TYPE, KNN_VECTOR) + .field(DIMENSION, dimensions.toString()) + .startObject(KNN_METHOD) + .field(NAME, METHOD_HNSW) + .endObject() + .endObject() + .endObject() + .endObject() + .toString(); } public String createKNNIndexCustomMethodFieldMapping( @@ -969,26 +964,25 @@ public String createKNNIndexCustomMethodFieldMapping( Integer m, Integer ef_construction ) throws IOException { - return Strings.toString( - XContentFactory.jsonBuilder() - .startObject() - .startObject(PROPERTIES) - .startObject(fieldName) - .field(VECTOR_TYPE, KNN_VECTOR) - .field(DIMENSION, dimensions.toString()) - .startObject(KNN_METHOD) - .field(NAME, METHOD_HNSW) - .field(METHOD_PARAMETER_SPACE_TYPE, spaceType.getValue()) - .field(KNN_ENGINE, engine) - .startObject(PARAMETERS) - .field(METHOD_PARAMETER_EF_CONSTRUCTION, ef_construction) - .field(METHOD_PARAMETER_M, m) - .endObject() - .endObject() - .endObject() - .endObject() - .endObject() - ); + return XContentFactory.jsonBuilder() + .startObject() + .startObject(PROPERTIES) + .startObject(fieldName) + .field(VECTOR_TYPE, KNN_VECTOR) + .field(DIMENSION, dimensions.toString()) + .startObject(KNN_METHOD) + .field(NAME, METHOD_HNSW) + .field(METHOD_PARAMETER_SPACE_TYPE, spaceType.getValue()) + .field(KNN_ENGINE, engine) + .startObject(PARAMETERS) + .field(METHOD_PARAMETER_EF_CONSTRUCTION, ef_construction) + .field(METHOD_PARAMETER_M, m) + .endObject() + .endObject() + .endObject() + .endObject() + .endObject() + .toString(); } // Default KNN script score settings @@ -1101,7 +1095,7 @@ public Response trainModel( } Request request = new Request("POST", "/_plugins/_knn/models" + modelId + "/_train"); - request.setJsonEntity(Strings.toString(builder)); + request.setJsonEntity(builder.toString()); return client().performRequest(request); } @@ -1316,7 +1310,7 @@ protected void addKnnDocWithAttributes(String docId, float[] vector, Map Date: Wed, 16 Aug 2023 13:15:36 -0400 Subject: [PATCH 130/416] Fix OnDiskInvertedLists file path for windows patching (#1044) (#1045) (cherry picked from commit 9702dd0ff19adc37844fd91201eb392fd2d2fc39) Co-authored-by: Naveen Tatikonda --- scripts/windowsScript.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/windowsScript.ps1 b/scripts/windowsScript.ps1 index 93945b29f..2b0d4f799 100644 --- a/scripts/windowsScript.ps1 +++ b/scripts/windowsScript.ps1 @@ -26,7 +26,7 @@ git submodule update --init -- jni/external/faiss # #ifndef __MINGW32__ # #include # #endif -(Get-Content jni/external/faiss/faiss/OnDiskInvertedLists.cpp).replace('#include ', "#ifndef __MINGW32__`n#include `n#endif") | Set-Content jni/external/faiss/faiss/OnDiskInvertedLists.cpp +(Get-Content jni/external/faiss/faiss/invlists/OnDiskInvertedLists.cpp).replace('#include ', "#ifndef __MINGW32__`n#include `n#endif") | Set-Content jni/external/faiss/faiss/invlists/OnDiskInvertedLists.cpp # intrin.h function like __builtin_ctz, __builtin_clzll is not available in MINGW32. So, adding condition to include it if not running on Windows # Replace '#include ' with # #ifndef __MINGW32__ From 67df3aee57601ceeab81cf8b159749422c155468 Mon Sep 17 00:00:00 2001 From: Martin Gaievski Date: Wed, 16 Aug 2023 21:39:51 +0200 Subject: [PATCH 131/416] [Backport 2.x] Fixed compilation errors after refactoring in core foundation classes (#1038) (#1048) * Fixed compilation errors after refactoring in core foundation classes (#1038) Signed-off-by: Martin Gaievski --------- Signed-off-by: Martin Gaievski --- .../org/opensearch/knn/index/KNNSettings.java | 2 +- .../index/memory/NativeMemoryLoadStrategy.java | 2 +- .../java/org/opensearch/knn/indices/ModelDao.java | 2 +- .../java/org/opensearch/knn/plugin/KNNPlugin.java | 2 +- .../knn/plugin/transport/DeleteModelResponse.java | 2 +- .../transport/DeleteModelTransportAction.java | 2 +- .../knn/plugin/transport/GetModelResponse.java | 2 +- .../plugin/transport/GetModelTransportAction.java | 2 +- .../transport/SearchModelTransportAction.java | 2 +- .../TrainingJobRouterTransportAction.java | 2 +- .../plugin/transport/TrainingModelResponse.java | 2 +- .../transport/TrainingModelTransportAction.java | 2 +- .../UpdateModelGraveyardTransportAction.java | 2 +- .../UpdateModelMetadataTransportAction.java | 2 +- .../knn/training/TrainingJobRunner.java | 2 +- .../org/opensearch/knn/training/VectorReader.java | 2 +- .../knn/index/KNNCreateIndexFromModelTests.java | 2 +- .../opensearch/knn/index/KNNSettingsTests.java | 15 +++++++++++++++ .../memory/NativeMemoryLoadStrategyTests.java | 2 +- .../org/opensearch/knn/indices/ModelDaoTests.java | 2 +- ...gJobRouteDecisionInfoTransportActionTests.java | 2 +- .../TrainingJobRouterTransportActionTests.java | 2 +- .../TrainingModelTransportActionTests.java | 2 +- .../UpdateModelGraveyardTransportActionTests.java | 2 +- .../UpdateModelMetadataTransportActionTests.java | 2 +- .../knn/training/TrainingJobRunnerTests.java | 2 +- .../knn/training/VectorReaderTests.java | 2 +- 27 files changed, 41 insertions(+), 26 deletions(-) diff --git a/src/main/java/org/opensearch/knn/index/KNNSettings.java b/src/main/java/org/opensearch/knn/index/KNNSettings.java index c47a2b197..6c7a80c82 100644 --- a/src/main/java/org/opensearch/knn/index/KNNSettings.java +++ b/src/main/java/org/opensearch/knn/index/KNNSettings.java @@ -8,7 +8,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.opensearch.OpenSearchParseException; -import org.opensearch.action.ActionListener; +import org.opensearch.core.action.ActionListener; import org.opensearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest; import org.opensearch.action.admin.cluster.settings.ClusterUpdateSettingsResponse; import org.opensearch.client.Client; diff --git a/src/main/java/org/opensearch/knn/index/memory/NativeMemoryLoadStrategy.java b/src/main/java/org/opensearch/knn/index/memory/NativeMemoryLoadStrategy.java index 5ca8f195b..93da658d0 100644 --- a/src/main/java/org/opensearch/knn/index/memory/NativeMemoryLoadStrategy.java +++ b/src/main/java/org/opensearch/knn/index/memory/NativeMemoryLoadStrategy.java @@ -11,7 +11,7 @@ package org.opensearch.knn.index.memory; -import org.opensearch.action.ActionListener; +import org.opensearch.core.action.ActionListener; import org.opensearch.knn.jni.JNIService; import org.opensearch.knn.index.util.KNNEngine; import org.opensearch.knn.training.TrainingDataConsumer; diff --git a/src/main/java/org/opensearch/knn/indices/ModelDao.java b/src/main/java/org/opensearch/knn/indices/ModelDao.java index 0573b87c1..fe0deb1cf 100644 --- a/src/main/java/org/opensearch/knn/indices/ModelDao.java +++ b/src/main/java/org/opensearch/knn/indices/ModelDao.java @@ -18,7 +18,6 @@ import org.apache.logging.log4j.Logger; import org.opensearch.OpenSearchException; import org.opensearch.ResourceNotFoundException; -import org.opensearch.action.ActionListener; import org.opensearch.action.DocWriteRequest; import org.opensearch.action.DocWriteResponse; import org.opensearch.action.FailedNodeException; @@ -44,6 +43,7 @@ import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.concurrent.ThreadContext; +import org.opensearch.core.action.ActionListener; import org.opensearch.index.IndexNotFoundException; import org.opensearch.knn.common.KNNConstants; import org.opensearch.knn.common.exception.DeleteModelWhenInTrainStateException; diff --git a/src/main/java/org/opensearch/knn/plugin/KNNPlugin.java b/src/main/java/org/opensearch/knn/plugin/KNNPlugin.java index 54011c259..53e9f3105 100644 --- a/src/main/java/org/opensearch/knn/plugin/KNNPlugin.java +++ b/src/main/java/org/opensearch/knn/plugin/KNNPlugin.java @@ -8,6 +8,7 @@ import org.opensearch.cluster.NamedDiff; import org.opensearch.cluster.metadata.Metadata; import org.opensearch.core.ParseField; +import org.opensearch.core.action.ActionResponse; import org.opensearch.index.codec.CodecServiceFactory; import org.opensearch.index.engine.EngineFactory; import org.opensearch.indices.SystemIndexDescriptor; @@ -43,7 +44,6 @@ import com.google.common.collect.ImmutableList; import org.opensearch.action.ActionRequest; -import org.opensearch.action.ActionResponse; import org.opensearch.client.Client; import org.opensearch.cluster.metadata.IndexNameExpressionResolver; import org.opensearch.cluster.node.DiscoveryNodes; diff --git a/src/main/java/org/opensearch/knn/plugin/transport/DeleteModelResponse.java b/src/main/java/org/opensearch/knn/plugin/transport/DeleteModelResponse.java index b4879d881..35f5f6cf2 100644 --- a/src/main/java/org/opensearch/knn/plugin/transport/DeleteModelResponse.java +++ b/src/main/java/org/opensearch/knn/plugin/transport/DeleteModelResponse.java @@ -10,8 +10,8 @@ */ package org.opensearch.knn.plugin.transport; -import org.opensearch.action.ActionResponse; import org.opensearch.common.Nullable; +import org.opensearch.core.action.ActionResponse; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.ToXContentObject; diff --git a/src/main/java/org/opensearch/knn/plugin/transport/DeleteModelTransportAction.java b/src/main/java/org/opensearch/knn/plugin/transport/DeleteModelTransportAction.java index a325d8b38..9828b5025 100644 --- a/src/main/java/org/opensearch/knn/plugin/transport/DeleteModelTransportAction.java +++ b/src/main/java/org/opensearch/knn/plugin/transport/DeleteModelTransportAction.java @@ -12,7 +12,7 @@ package org.opensearch.knn.plugin.transport; import lombok.extern.log4j.Log4j2; -import org.opensearch.action.ActionListener; +import org.opensearch.core.action.ActionListener; import org.opensearch.action.support.ActionFilters; import org.opensearch.action.support.HandledTransportAction; import org.opensearch.common.inject.Inject; diff --git a/src/main/java/org/opensearch/knn/plugin/transport/GetModelResponse.java b/src/main/java/org/opensearch/knn/plugin/transport/GetModelResponse.java index 4df2b7f03..f47638560 100644 --- a/src/main/java/org/opensearch/knn/plugin/transport/GetModelResponse.java +++ b/src/main/java/org/opensearch/knn/plugin/transport/GetModelResponse.java @@ -10,7 +10,7 @@ */ package org.opensearch.knn.plugin.transport; -import org.opensearch.action.ActionResponse; +import org.opensearch.core.action.ActionResponse; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.ToXContentObject; diff --git a/src/main/java/org/opensearch/knn/plugin/transport/GetModelTransportAction.java b/src/main/java/org/opensearch/knn/plugin/transport/GetModelTransportAction.java index e47a42d8d..8f8276746 100644 --- a/src/main/java/org/opensearch/knn/plugin/transport/GetModelTransportAction.java +++ b/src/main/java/org/opensearch/knn/plugin/transport/GetModelTransportAction.java @@ -12,7 +12,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.opensearch.action.ActionListener; +import org.opensearch.core.action.ActionListener; import org.opensearch.action.support.ActionFilters; import org.opensearch.action.support.HandledTransportAction; import org.opensearch.common.inject.Inject; diff --git a/src/main/java/org/opensearch/knn/plugin/transport/SearchModelTransportAction.java b/src/main/java/org/opensearch/knn/plugin/transport/SearchModelTransportAction.java index 4d9f67059..89114d763 100644 --- a/src/main/java/org/opensearch/knn/plugin/transport/SearchModelTransportAction.java +++ b/src/main/java/org/opensearch/knn/plugin/transport/SearchModelTransportAction.java @@ -11,7 +11,7 @@ package org.opensearch.knn.plugin.transport; -import org.opensearch.action.ActionListener; +import org.opensearch.core.action.ActionListener; import org.opensearch.action.search.SearchRequest; import org.opensearch.action.search.SearchResponse; import org.opensearch.action.support.ActionFilters; diff --git a/src/main/java/org/opensearch/knn/plugin/transport/TrainingJobRouterTransportAction.java b/src/main/java/org/opensearch/knn/plugin/transport/TrainingJobRouterTransportAction.java index 941d95212..9b2d3a9de 100644 --- a/src/main/java/org/opensearch/knn/plugin/transport/TrainingJobRouterTransportAction.java +++ b/src/main/java/org/opensearch/knn/plugin/transport/TrainingJobRouterTransportAction.java @@ -12,7 +12,7 @@ package org.opensearch.knn.plugin.transport; import org.apache.commons.lang.StringUtils; -import org.opensearch.action.ActionListener; +import org.opensearch.core.action.ActionListener; import org.opensearch.action.ActionListenerResponseHandler; import org.opensearch.action.search.SearchRequest; import org.opensearch.action.support.ActionFilters; diff --git a/src/main/java/org/opensearch/knn/plugin/transport/TrainingModelResponse.java b/src/main/java/org/opensearch/knn/plugin/transport/TrainingModelResponse.java index 2886c30ae..6dec1adcf 100644 --- a/src/main/java/org/opensearch/knn/plugin/transport/TrainingModelResponse.java +++ b/src/main/java/org/opensearch/knn/plugin/transport/TrainingModelResponse.java @@ -11,7 +11,7 @@ package org.opensearch.knn.plugin.transport; -import org.opensearch.action.ActionResponse; +import org.opensearch.core.action.ActionResponse; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.ToXContentObject; diff --git a/src/main/java/org/opensearch/knn/plugin/transport/TrainingModelTransportAction.java b/src/main/java/org/opensearch/knn/plugin/transport/TrainingModelTransportAction.java index a3c4be16e..379d8a809 100644 --- a/src/main/java/org/opensearch/knn/plugin/transport/TrainingModelTransportAction.java +++ b/src/main/java/org/opensearch/knn/plugin/transport/TrainingModelTransportAction.java @@ -11,7 +11,7 @@ package org.opensearch.knn.plugin.transport; -import org.opensearch.action.ActionListener; +import org.opensearch.core.action.ActionListener; import org.opensearch.action.support.ActionFilters; import org.opensearch.action.support.HandledTransportAction; import org.opensearch.cluster.service.ClusterService; diff --git a/src/main/java/org/opensearch/knn/plugin/transport/UpdateModelGraveyardTransportAction.java b/src/main/java/org/opensearch/knn/plugin/transport/UpdateModelGraveyardTransportAction.java index 99aa1b23a..df0c26624 100644 --- a/src/main/java/org/opensearch/knn/plugin/transport/UpdateModelGraveyardTransportAction.java +++ b/src/main/java/org/opensearch/knn/plugin/transport/UpdateModelGraveyardTransportAction.java @@ -7,7 +7,7 @@ import lombok.Value; import lombok.extern.log4j.Log4j2; -import org.opensearch.action.ActionListener; +import org.opensearch.core.action.ActionListener; import org.opensearch.action.support.ActionFilters; import org.opensearch.action.support.master.AcknowledgedResponse; import org.opensearch.action.support.clustermanager.TransportClusterManagerNodeAction; diff --git a/src/main/java/org/opensearch/knn/plugin/transport/UpdateModelMetadataTransportAction.java b/src/main/java/org/opensearch/knn/plugin/transport/UpdateModelMetadataTransportAction.java index 8c95c2db8..ec909f443 100644 --- a/src/main/java/org/opensearch/knn/plugin/transport/UpdateModelMetadataTransportAction.java +++ b/src/main/java/org/opensearch/knn/plugin/transport/UpdateModelMetadataTransportAction.java @@ -13,7 +13,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.opensearch.action.ActionListener; +import org.opensearch.core.action.ActionListener; import org.opensearch.action.support.ActionFilters; import org.opensearch.action.support.master.AcknowledgedResponse; import org.opensearch.action.support.clustermanager.TransportClusterManagerNodeAction; diff --git a/src/main/java/org/opensearch/knn/training/TrainingJobRunner.java b/src/main/java/org/opensearch/knn/training/TrainingJobRunner.java index 774500311..a999735a4 100644 --- a/src/main/java/org/opensearch/knn/training/TrainingJobRunner.java +++ b/src/main/java/org/opensearch/knn/training/TrainingJobRunner.java @@ -13,7 +13,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.opensearch.action.ActionListener; +import org.opensearch.core.action.ActionListener; import org.opensearch.action.index.IndexResponse; import org.opensearch.common.ValidationException; import org.opensearch.knn.indices.ModelDao; diff --git a/src/main/java/org/opensearch/knn/training/VectorReader.java b/src/main/java/org/opensearch/knn/training/VectorReader.java index 9b7db6d99..c104e2df3 100644 --- a/src/main/java/org/opensearch/knn/training/VectorReader.java +++ b/src/main/java/org/opensearch/knn/training/VectorReader.java @@ -13,7 +13,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.opensearch.action.ActionListener; +import org.opensearch.core.action.ActionListener; import org.opensearch.action.search.SearchRequestBuilder; import org.opensearch.action.search.SearchResponse; import org.opensearch.action.search.SearchScrollRequestBuilder; diff --git a/src/test/java/org/opensearch/knn/index/KNNCreateIndexFromModelTests.java b/src/test/java/org/opensearch/knn/index/KNNCreateIndexFromModelTests.java index 3a4746158..c1c52e63a 100644 --- a/src/test/java/org/opensearch/knn/index/KNNCreateIndexFromModelTests.java +++ b/src/test/java/org/opensearch/knn/index/KNNCreateIndexFromModelTests.java @@ -12,7 +12,7 @@ package org.opensearch.knn.index; import com.google.common.collect.ImmutableMap; -import org.opensearch.action.ActionListener; +import org.opensearch.core.action.ActionListener; import org.opensearch.action.admin.indices.create.CreateIndexRequestBuilder; import org.opensearch.common.settings.Settings; import org.opensearch.common.xcontent.XContentFactory; diff --git a/src/test/java/org/opensearch/knn/index/KNNSettingsTests.java b/src/test/java/org/opensearch/knn/index/KNNSettingsTests.java index f6a10616b..4adad3fe6 100644 --- a/src/test/java/org/opensearch/knn/index/KNNSettingsTests.java +++ b/src/test/java/org/opensearch/knn/index/KNNSettingsTests.java @@ -46,6 +46,7 @@ public void testGetSettingValueFromConfig() { .getSettingValue(KNNSettings.KNN_MEMORY_CIRCUIT_BREAKER_LIMIT)).getKb(); mockNode.close(); assertEquals(expectedKNNCircuitBreakerLimit, actualKNNCircuitBreakerLimit); + assertWarnings(); } @SneakyThrows @@ -63,6 +64,10 @@ public void testGetSettingValueDefault() { actualKNNCircuitBreakerLimit ); + // set warning for deprecation of index.store.hybrid.mmap.extensions as expected temporarily, need to work on proper strategy of + // switching to new setting in core + // no-jdk distributions expected warning is a workaround for running tests locally + assertWarnings(); } private Node createMockNode(Map configSettings) throws IOException { @@ -93,4 +98,14 @@ private static Settings.Builder baseSettings() { .put(NetworkModule.TRANSPORT_TYPE_KEY, getTestTransportType()) .put(dataNode()); } + + private void assertWarnings() { + // set warning for deprecation of index.store.hybrid.mmap.extensions as expected temporarily, need to work on proper strategy of + // switching to new setting in core + // no-jdk distributions expected warning is a workaround for running tests locally + assertWarnings( + "[index.store.hybrid.mmap.extensions] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.", + "no-jdk distributions that do not bundle a JDK are deprecated and will be removed in a future release" + ); + } } diff --git a/src/test/java/org/opensearch/knn/index/memory/NativeMemoryLoadStrategyTests.java b/src/test/java/org/opensearch/knn/index/memory/NativeMemoryLoadStrategyTests.java index ce08e0350..fb91266ab 100644 --- a/src/test/java/org/opensearch/knn/index/memory/NativeMemoryLoadStrategyTests.java +++ b/src/test/java/org/opensearch/knn/index/memory/NativeMemoryLoadStrategyTests.java @@ -12,7 +12,7 @@ package org.opensearch.knn.index.memory; import com.google.common.collect.ImmutableMap; -import org.opensearch.action.ActionListener; +import org.opensearch.core.action.ActionListener; import org.opensearch.action.search.SearchResponse; import org.opensearch.knn.KNNTestCase; import org.opensearch.knn.common.KNNConstants; diff --git a/src/test/java/org/opensearch/knn/indices/ModelDaoTests.java b/src/test/java/org/opensearch/knn/indices/ModelDaoTests.java index 082757eb5..cb6628c16 100644 --- a/src/test/java/org/opensearch/knn/indices/ModelDaoTests.java +++ b/src/test/java/org/opensearch/knn/indices/ModelDaoTests.java @@ -17,7 +17,7 @@ import org.opensearch.ExceptionsHelper; import org.opensearch.ResourceAlreadyExistsException; import org.opensearch.ResourceNotFoundException; -import org.opensearch.action.ActionListener; +import org.opensearch.core.action.ActionListener; import org.opensearch.action.DocWriteResponse; import org.opensearch.action.StepListener; import org.opensearch.action.admin.indices.create.CreateIndexResponse; diff --git a/src/test/java/org/opensearch/knn/plugin/transport/TrainingJobRouteDecisionInfoTransportActionTests.java b/src/test/java/org/opensearch/knn/plugin/transport/TrainingJobRouteDecisionInfoTransportActionTests.java index 2eeaab418..15d84638d 100644 --- a/src/test/java/org/opensearch/knn/plugin/transport/TrainingJobRouteDecisionInfoTransportActionTests.java +++ b/src/test/java/org/opensearch/knn/plugin/transport/TrainingJobRouteDecisionInfoTransportActionTests.java @@ -13,7 +13,7 @@ import org.junit.After; import org.junit.Before; -import org.opensearch.action.ActionListener; +import org.opensearch.core.action.ActionListener; import org.opensearch.action.index.IndexResponse; import org.opensearch.core.index.shard.ShardId; import org.opensearch.knn.KNNSingleNodeTestCase; diff --git a/src/test/java/org/opensearch/knn/plugin/transport/TrainingJobRouterTransportActionTests.java b/src/test/java/org/opensearch/knn/plugin/transport/TrainingJobRouterTransportActionTests.java index 4fc4a1ccd..56c50aca1 100644 --- a/src/test/java/org/opensearch/knn/plugin/transport/TrainingJobRouterTransportActionTests.java +++ b/src/test/java/org/opensearch/knn/plugin/transport/TrainingJobRouterTransportActionTests.java @@ -13,7 +13,7 @@ import com.google.common.collect.ImmutableList; import org.apache.lucene.search.TotalHits; -import org.opensearch.action.ActionListener; +import org.opensearch.core.action.ActionListener; import org.opensearch.action.search.SearchResponse; import org.opensearch.action.support.ActionFilters; import org.opensearch.client.Client; diff --git a/src/test/java/org/opensearch/knn/plugin/transport/TrainingModelTransportActionTests.java b/src/test/java/org/opensearch/knn/plugin/transport/TrainingModelTransportActionTests.java index 6013913dd..950ce1fd0 100644 --- a/src/test/java/org/opensearch/knn/plugin/transport/TrainingModelTransportActionTests.java +++ b/src/test/java/org/opensearch/knn/plugin/transport/TrainingModelTransportActionTests.java @@ -11,7 +11,7 @@ package org.opensearch.knn.plugin.transport; -import org.opensearch.action.ActionListener; +import org.opensearch.core.action.ActionListener; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.knn.KNNSingleNodeTestCase; diff --git a/src/test/java/org/opensearch/knn/plugin/transport/UpdateModelGraveyardTransportActionTests.java b/src/test/java/org/opensearch/knn/plugin/transport/UpdateModelGraveyardTransportActionTests.java index 29ef6fab4..b1983b964 100644 --- a/src/test/java/org/opensearch/knn/plugin/transport/UpdateModelGraveyardTransportActionTests.java +++ b/src/test/java/org/opensearch/knn/plugin/transport/UpdateModelGraveyardTransportActionTests.java @@ -5,7 +5,7 @@ package org.opensearch.knn.plugin.transport; -import org.opensearch.action.ActionListener; +import org.opensearch.core.action.ActionListener; import org.opensearch.action.support.master.AcknowledgedResponse; import org.opensearch.cluster.ClusterState; import org.opensearch.common.io.stream.BytesStreamOutput; diff --git a/src/test/java/org/opensearch/knn/plugin/transport/UpdateModelMetadataTransportActionTests.java b/src/test/java/org/opensearch/knn/plugin/transport/UpdateModelMetadataTransportActionTests.java index 6fe9222c8..eb3ecf168 100644 --- a/src/test/java/org/opensearch/knn/plugin/transport/UpdateModelMetadataTransportActionTests.java +++ b/src/test/java/org/opensearch/knn/plugin/transport/UpdateModelMetadataTransportActionTests.java @@ -11,7 +11,7 @@ package org.opensearch.knn.plugin.transport; -import org.opensearch.action.ActionListener; +import org.opensearch.core.action.ActionListener; import org.opensearch.action.support.master.AcknowledgedResponse; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.metadata.IndexMetadata; diff --git a/src/test/java/org/opensearch/knn/training/TrainingJobRunnerTests.java b/src/test/java/org/opensearch/knn/training/TrainingJobRunnerTests.java index 3edff82c6..9acdc7b36 100644 --- a/src/test/java/org/opensearch/knn/training/TrainingJobRunnerTests.java +++ b/src/test/java/org/opensearch/knn/training/TrainingJobRunnerTests.java @@ -11,7 +11,7 @@ package org.opensearch.knn.training; -import org.opensearch.action.ActionListener; +import org.opensearch.core.action.ActionListener; import org.opensearch.action.index.IndexResponse; import org.opensearch.core.index.shard.ShardId; import org.opensearch.knn.KNNTestCase; diff --git a/src/test/java/org/opensearch/knn/training/VectorReaderTests.java b/src/test/java/org/opensearch/knn/training/VectorReaderTests.java index 0b1e15289..74008a043 100644 --- a/src/test/java/org/opensearch/knn/training/VectorReaderTests.java +++ b/src/test/java/org/opensearch/knn/training/VectorReaderTests.java @@ -11,7 +11,7 @@ package org.opensearch.knn.training; -import org.opensearch.action.ActionListener; +import org.opensearch.core.action.ActionListener; import org.opensearch.action.search.SearchResponse; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.ValidationException; From 3c56b74dd565622a1730aaf9d1096a010132e153 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Wed, 16 Aug 2023 13:52:15 -0700 Subject: [PATCH 132/416] Update Guava Version to 32.0.1 (#1019) (#1022) Signed-off-by: Naveen Tatikonda (cherry picked from commit 478d87c19a43dd2ebaaeae36b2599a86bed9b7bd) Co-authored-by: Naveen Tatikonda --- CHANGELOG.md | 1 + build.gradle | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ecc1736df..4ddf9fd2c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,4 +20,5 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Infrastructure ### Documentation ### Maintenance +* Update Guava Version to 32.0.1 [#1019](https://github.com/opensearch-project/k-NN/pull/1019) ### Refactoring diff --git a/build.gradle b/build.gradle index 3303cbde2..b70958daa 100644 --- a/build.gradle +++ b/build.gradle @@ -172,7 +172,7 @@ dependencies { api "org.opensearch:opensearch:${opensearch_version}" compileOnly "org.opensearch.plugin:opensearch-scripting-painless-spi:${versions.opensearch}" api group: 'com.google.guava', name: 'failureaccess', version:'1.0.1' - api group: 'com.google.guava', name: 'guava', version:'30.0-jre' + api group: 'com.google.guava', name: 'guava', version:'32.0.1-jre' api group: 'commons-lang', name: 'commons-lang', version: '2.6' testFixturesImplementation "org.opensearch.test:framework:${opensearch_version}" testImplementation group: 'net.bytebuddy', name: 'byte-buddy', version: '1.14.3' From 8c7cb1b49d53e2add0481f57980b0d38a07ad30c Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Thu, 17 Aug 2023 13:45:34 -0500 Subject: [PATCH 133/416] Fix TransportAddress Refactoring Changes in Core (#1020) (#1021) Signed-off-by: Naveen Tatikonda (cherry picked from commit d75cc9575a36e9bc68bf2ea19f5d12eac3d14d0b) Co-authored-by: Naveen Tatikonda --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4ddf9fd2c..3dea320b4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,3 +22,4 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Maintenance * Update Guava Version to 32.0.1 [#1019](https://github.com/opensearch-project/k-NN/pull/1019) ### Refactoring +* Fix TransportAddress Refactoring Changes in Core [#1020](https://github.com/opensearch-project/k-NN/pull/1020) \ No newline at end of file From d89c85e71609c11f74a18cf60a2f063e91295c04 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Wed, 23 Aug 2023 16:53:52 -0700 Subject: [PATCH 134/416] Improved the logic to switch to exact search for restrictive filters search. (#1059) (#1060) This change includes: * adding 2 extra advanced K-NN settings on when to do exact search for users to tune. Signed-off-by: Navneet Verma (cherry picked from commit 4edc1bf5d7d2fab39fc689ea35d0778b559f5ebc) Co-authored-by: Navneet Verma --- CHANGELOG.md | 3 +- .../org/opensearch/knn/index/KNNSettings.java | 49 +++++++++- .../opensearch/knn/index/query/KNNWeight.java | 44 ++++++++- .../knn/index/KNNSettingsTests.java | 85 +++++++++++++++++ .../knn/index/query/KNNWeightTests.java | 91 ++++++++++++++++++- 5 files changed, 263 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3dea320b4..3ddd7fca2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,10 +16,11 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Features ### Enhancements * Enabled the IVF algorithm to work with Filters of K-NN Query. [#1013](https://github.com/opensearch-project/k-NN/pull/1013) +* Improved the logic to switch to exact search for restrictive filters search for better recall. [#1059](https://github.com/opensearch-project/k-NN/pull/1059) ### Bug Fixes ### Infrastructure ### Documentation ### Maintenance * Update Guava Version to 32.0.1 [#1019](https://github.com/opensearch-project/k-NN/pull/1019) ### Refactoring -* Fix TransportAddress Refactoring Changes in Core [#1020](https://github.com/opensearch-project/k-NN/pull/1020) \ No newline at end of file +* Fix TransportAddress Refactoring Changes in Core [#1020](https://github.com/opensearch-project/k-NN/pull/1020) diff --git a/src/main/java/org/opensearch/knn/index/KNNSettings.java b/src/main/java/org/opensearch/knn/index/KNNSettings.java index 6c7a80c82..a1958233b 100644 --- a/src/main/java/org/opensearch/knn/index/KNNSettings.java +++ b/src/main/java/org/opensearch/knn/index/KNNSettings.java @@ -73,6 +73,8 @@ public class KNNSettings { public static final String MODEL_INDEX_NUMBER_OF_SHARDS = "knn.model.index.number_of_shards"; public static final String MODEL_INDEX_NUMBER_OF_REPLICAS = "knn.model.index.number_of_replicas"; public static final String MODEL_CACHE_SIZE_LIMIT = "knn.model.cache.size.limit"; + public static final String ADVANCED_FILTERED_EXACT_SEARCH_THRESHOLD = "index.knn.advanced.filtered_exact_search_threshold"; + public static final String ADVANCED_FILTERED_EXACT_SEARCH_THRESHOLD_PCT = "index.knn.advanced.filtered_exact_search_threshold_pct"; /** * Default setting values @@ -87,6 +89,9 @@ public class KNNSettings { public static final Integer KNN_MAX_MODEL_CACHE_SIZE_LIMIT_PERCENTAGE = 25; // Model cache limit cannot exceed 25% of the JVM heap public static final String KNN_DEFAULT_MEMORY_CIRCUIT_BREAKER_LIMIT = "50%"; + public static final Integer ADVANCED_FILTERED_EXACT_SEARCH_THRESHOLD_DEFAULT_VALUE = 2000; + public static final Integer ADVANCED_FILTERED_EXACT_SEARCH_THRESHOLD_PCT_DEFAULT_VALUE = 10; + /** * Settings Definition */ @@ -154,6 +159,22 @@ public class KNNSettings { Setting.Property.Dynamic ); + public static final Setting ADVANCED_FILTERED_EXACT_SEARCH_THRESHOLD_SETTING = Setting.intSetting( + ADVANCED_FILTERED_EXACT_SEARCH_THRESHOLD, + ADVANCED_FILTERED_EXACT_SEARCH_THRESHOLD_DEFAULT_VALUE, + 0, + IndexScope, + Setting.Property.Dynamic + ); + + public static final Setting ADVANCED_FILTERED_EXACT_SEARCH_THRESHOLD_PCT_SETTING = Setting.intSetting( + ADVANCED_FILTERED_EXACT_SEARCH_THRESHOLD_PCT, + ADVANCED_FILTERED_EXACT_SEARCH_THRESHOLD_PCT_DEFAULT_VALUE, + 0, + IndexScope, + Setting.Property.Dynamic + ); + public static final Setting MODEL_CACHE_SIZE_LIMIT_SETTING = new Setting<>( MODEL_CACHE_SIZE_LIMIT, percentageAsString(KNN_DEFAULT_MODEL_CACHE_SIZE_LIMIT_PERCENTAGE), @@ -323,6 +344,14 @@ private Setting getSetting(String key) { return KNN_ALGO_PARAM_INDEX_THREAD_QTY_SETTING; } + if (ADVANCED_FILTERED_EXACT_SEARCH_THRESHOLD.equals(key)) { + return ADVANCED_FILTERED_EXACT_SEARCH_THRESHOLD_SETTING; + } + + if (ADVANCED_FILTERED_EXACT_SEARCH_THRESHOLD_PCT.equals(key)) { + return ADVANCED_FILTERED_EXACT_SEARCH_THRESHOLD_PCT_SETTING; + } + throw new IllegalArgumentException("Cannot find setting by key [" + key + "]"); } @@ -338,7 +367,9 @@ public List> getSettings() { IS_KNN_INDEX_SETTING, MODEL_INDEX_NUMBER_OF_SHARDS_SETTING, MODEL_INDEX_NUMBER_OF_REPLICAS_SETTING, - MODEL_CACHE_SIZE_LIMIT_SETTING + MODEL_CACHE_SIZE_LIMIT_SETTING, + ADVANCED_FILTERED_EXACT_SEARCH_THRESHOLD_SETTING, + ADVANCED_FILTERED_EXACT_SEARCH_THRESHOLD_PCT_SETTING ); return Stream.concat(settings.stream(), dynamicCacheSettings.values().stream()).collect(Collectors.toList()); } @@ -359,6 +390,22 @@ public static double getCircuitBreakerUnsetPercentage() { return KNNSettings.state().getSettingValue(KNNSettings.KNN_CIRCUIT_BREAKER_UNSET_PERCENTAGE); } + public static int getFilteredExactSearchThreshold(final String indexName) { + return KNNSettings.state().clusterService.state() + .getMetadata() + .index(indexName) + .getSettings() + .getAsInt(ADVANCED_FILTERED_EXACT_SEARCH_THRESHOLD, ADVANCED_FILTERED_EXACT_SEARCH_THRESHOLD_DEFAULT_VALUE); + } + + public static int getFilteredExactSearchThresholdPct(final String indexName) { + return KNNSettings.state().clusterService.state() + .getMetadata() + .index(indexName) + .getSettings() + .getAsInt(ADVANCED_FILTERED_EXACT_SEARCH_THRESHOLD_PCT, ADVANCED_FILTERED_EXACT_SEARCH_THRESHOLD_PCT_DEFAULT_VALUE); + } + public void initialize(Client client, ClusterService clusterService) { this.client = client; this.clusterService = clusterService; diff --git a/src/main/java/org/opensearch/knn/index/query/KNNWeight.java b/src/main/java/org/opensearch/knn/index/query/KNNWeight.java index 4bbf61f25..7352dd436 100644 --- a/src/main/java/org/opensearch/knn/index/query/KNNWeight.java +++ b/src/main/java/org/opensearch/knn/index/query/KNNWeight.java @@ -18,6 +18,7 @@ import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.FixedBitSet; import org.opensearch.knn.common.KNNConstants; +import org.opensearch.knn.index.KNNSettings; import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.codec.util.KNNVectorSerializer; import org.opensearch.knn.index.codec.util.KNNVectorSerializerFactory; @@ -115,13 +116,16 @@ public Scorer scorer(LeafReaderContext context) throws IOException { * . Hence, if filtered results are less than K and filter query is present we should shift to exact search. * This improves the recall. */ - if (filterWeight != null && filterIdsArray.length <= knnQuery.getK()) { + if (filterWeight != null && canDoExactSearch(filterIdsArray.length, getTotalDocsInSegment(context))) { docIdsToScoreMap.putAll(doExactSearch(context, filterIdsArray)); } else { - final Map annResults = doANNSearch(context, filterIdsArray); + Map annResults = doANNSearch(context, filterIdsArray); if (annResults == null) { return null; } + if (canDoExactSearchAfterANNSearch(filterIdsArray.length, annResults.size())) { + annResults = doExactSearch(context, filterIdsArray); + } docIdsToScoreMap.putAll(annResults); } if (docIdsToScoreMap.isEmpty()) { @@ -170,7 +174,6 @@ private int[] getFilterIdsArray(final LeafReaderContext context) throws IOExcept if (docId == DocIdSetIterator.NO_MORE_DOCS || docId + 1 == DocIdSetIterator.NO_MORE_DOCS) { break; } - log.debug("Docs in filtered docs id set is : {}", docId); filteredIds[filteredIdsIndex] = docId; filteredIdsIndex++; docId++; @@ -369,4 +372,39 @@ private SpaceType getSpaceType(final FieldInfo fieldInfo) { String.format(Locale.ROOT, "Unable to find the Space Type from Field Info attribute for field %s", fieldInfo.getName()) ); } + + private boolean canDoExactSearch(final int filterIdsCount, final int searchableDocs) { + log.debug( + "Info for doing exact search Live Docs: {}, filterIdsLength : {}, Threshold value: {} , Threshold %age : {}", + searchableDocs, + filterIdsCount, + KNNSettings.getFilteredExactSearchThreshold(knnQuery.getIndexName()), + KNNSettings.getFilteredExactSearchThresholdPct(knnQuery.getIndexName()) + ); + // Refer this GitHub around more details https://github.com/opensearch-project/k-NN/issues/1049 on the logic + return filterIdsCount <= knnQuery.getK() + || (filterIdsCount <= KNNSettings.getFilteredExactSearchThreshold(knnQuery.getIndexName()) + && (((float) filterIdsCount / (float) searchableDocs) * 100) <= (float) KNNSettings.getFilteredExactSearchThresholdPct( + knnQuery.getIndexName() + )); + } + + /** + * This condition mainly checks during filtered search we have more than K elements in filterIds but the ANN + * doesn't yeild K nearest neighbors. + * @param filterIdsCount count of filtered Doc ids + * @param annResultCount Count of Nearest Neighbours we got after doing filtered ANN Search. + * @return boolean - true if exactSearch needs to be done after ANNSearch. + */ + private boolean canDoExactSearchAfterANNSearch(final int filterIdsCount, final int annResultCount) { + return filterWeight != null && filterIdsCount >= knnQuery.getK() && knnQuery.getK() > annResultCount; + } + + private int getTotalDocsInSegment(final LeafReaderContext context) { + // This means that there is no deleted documents, hence the live docs bitset is null + if (context.reader().getLiveDocs() == null) { + return context.reader().maxDoc(); + } + return context.reader().getLiveDocs().length(); + } } diff --git a/src/test/java/org/opensearch/knn/index/KNNSettingsTests.java b/src/test/java/org/opensearch/knn/index/KNNSettingsTests.java index 4adad3fe6..9432be33e 100644 --- a/src/test/java/org/opensearch/knn/index/KNNSettingsTests.java +++ b/src/test/java/org/opensearch/knn/index/KNNSettingsTests.java @@ -6,6 +6,10 @@ package org.opensearch.knn.index; import lombok.SneakyThrows; +import org.junit.Assert; +import org.opensearch.action.admin.cluster.state.ClusterStateRequest; +import org.opensearch.action.admin.indices.create.CreateIndexRequest; +import org.opensearch.action.admin.indices.settings.put.UpdateSettingsRequest; import org.opensearch.cluster.ClusterName; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.network.NetworkModule; @@ -33,6 +37,8 @@ public class KNNSettingsTests extends KNNTestCase { + private static final String INDEX_NAME = "myindex"; + @SneakyThrows public void testGetSettingValueFromConfig() { long expectedKNNCircuitBreakerLimit = 13; @@ -70,6 +76,85 @@ public void testGetSettingValueDefault() { assertWarnings(); } + @SneakyThrows + public void testFilteredSearchAdvanceSetting_whenNoValuesProvidedByUsers_thenDefaultSettingsUsed() { + Node mockNode = createMockNode(Collections.emptyMap()); + mockNode.start(); + ClusterService clusterService = mockNode.injector().getInstance(ClusterService.class); + mockNode.client().admin().cluster().state(new ClusterStateRequest()).actionGet(); + mockNode.client().admin().indices().create(new CreateIndexRequest(INDEX_NAME)).actionGet(); + KNNSettings.state().setClusterService(clusterService); + + int filteredSearchThresholdPct = KNNSettings.getFilteredExactSearchThresholdPct(INDEX_NAME); + int filteredSearchThreshold = KNNSettings.getFilteredExactSearchThreshold(INDEX_NAME); + mockNode.close(); + assertEquals((int) KNNSettings.ADVANCED_FILTERED_EXACT_SEARCH_THRESHOLD_PCT_DEFAULT_VALUE, filteredSearchThresholdPct); + assertEquals((int) KNNSettings.ADVANCED_FILTERED_EXACT_SEARCH_THRESHOLD_DEFAULT_VALUE, filteredSearchThreshold); + assertWarnings(); + } + + @SneakyThrows + public void testFilteredSearchAdvanceSetting_whenValuesProvidedByUsers_thenValidateSameValues() { + int userDefinedPctThreshold = 20; + int userDefinedThreshold = 1000; + int userDefinedPctThresholdMinValue = 0; + int userDefinedThresholdMinValue = 0; + Node mockNode = createMockNode(Collections.emptyMap()); + mockNode.start(); + ClusterService clusterService = mockNode.injector().getInstance(ClusterService.class); + mockNode.client().admin().cluster().state(new ClusterStateRequest()).actionGet(); + mockNode.client().admin().indices().create(new CreateIndexRequest(INDEX_NAME)).actionGet(); + KNNSettings.state().setClusterService(clusterService); + + final Settings filteredSearchAdvanceSettings = Settings.builder() + .put(KNNSettings.ADVANCED_FILTERED_EXACT_SEARCH_THRESHOLD, userDefinedThreshold) + .put(KNNSettings.ADVANCED_FILTERED_EXACT_SEARCH_THRESHOLD_PCT, userDefinedPctThreshold) + .build(); + + mockNode.client() + .admin() + .indices() + .updateSettings(new UpdateSettingsRequest(filteredSearchAdvanceSettings, INDEX_NAME)) + .actionGet(); + + int filteredSearchThresholdPct = KNNSettings.getFilteredExactSearchThresholdPct(INDEX_NAME); + int filteredSearchThreshold = KNNSettings.getFilteredExactSearchThreshold(INDEX_NAME); + + // validate if we are able to set MinValues for the setting + final Settings filteredSearchAdvanceSettingsWithMinValues = Settings.builder() + .put(KNNSettings.ADVANCED_FILTERED_EXACT_SEARCH_THRESHOLD, userDefinedThresholdMinValue) + .put(KNNSettings.ADVANCED_FILTERED_EXACT_SEARCH_THRESHOLD_PCT, userDefinedPctThresholdMinValue) + .build(); + + mockNode.client() + .admin() + .indices() + .updateSettings(new UpdateSettingsRequest(filteredSearchAdvanceSettingsWithMinValues, INDEX_NAME)) + .actionGet(); + + int filteredSearchThresholdPctMinValue = KNNSettings.getFilteredExactSearchThresholdPct(INDEX_NAME); + int filteredSearchThresholdMinValue = KNNSettings.getFilteredExactSearchThreshold(INDEX_NAME); + + // Validate if less than MinValues are set then Exception Happens + final Settings filteredSearchAdvanceSettingsWithLessThanMinValues = Settings.builder() + .put(KNNSettings.ADVANCED_FILTERED_EXACT_SEARCH_THRESHOLD, -1) + .put(KNNSettings.ADVANCED_FILTERED_EXACT_SEARCH_THRESHOLD_PCT, -1) + .build(); + + Assert.assertThrows(IllegalArgumentException.class, () -> mockNode.client() + .admin() + .indices() + .updateSettings(new UpdateSettingsRequest(filteredSearchAdvanceSettingsWithLessThanMinValues, INDEX_NAME)) + .actionGet()); + + mockNode.close(); + assertEquals(userDefinedPctThreshold, filteredSearchThresholdPct); + assertEquals(userDefinedThreshold, filteredSearchThreshold); + assertEquals(userDefinedPctThresholdMinValue, filteredSearchThresholdPctMinValue); + assertEquals(userDefinedThresholdMinValue, filteredSearchThresholdMinValue); + assertWarnings(); + } + private Node createMockNode(Map configSettings) throws IOException { Path configDir = createTempDir(); File configFile = configDir.resolve("opensearch.yml").toFile(); diff --git a/src/test/java/org/opensearch/knn/index/query/KNNWeightTests.java b/src/test/java/org/opensearch/knn/index/query/KNNWeightTests.java index 20ee69744..9cc624377 100644 --- a/src/test/java/org/opensearch/knn/index/query/KNNWeightTests.java +++ b/src/test/java/org/opensearch/knn/index/query/KNNWeightTests.java @@ -23,9 +23,11 @@ import org.apache.lucene.search.TermQuery; import org.apache.lucene.search.Weight; import org.apache.lucene.store.FSDirectory; +import org.apache.lucene.util.Bits; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.StringHelper; import org.apache.lucene.util.Version; +import org.junit.Before; import org.junit.BeforeClass; import org.mockito.MockedStatic; import org.opensearch.common.io.PathUtils; @@ -86,10 +88,12 @@ public class KNNWeightTests extends KNNTestCase { private static MockedStatic nativeMemoryCacheManagerMockedStatic; private static MockedStatic jniServiceMockedStatic; + private static MockedStatic knnSettingsMockedStatic; + @BeforeClass public static void setUpClass() throws Exception { final KNNSettings knnSettings = mock(KNNSettings.class); - final MockedStatic knnSettingsMockedStatic = mockStatic(KNNSettings.class); + knnSettingsMockedStatic = mockStatic(KNNSettings.class); when(knnSettings.getSettingValue(eq(KNNSettings.KNN_MEMORY_CIRCUIT_BREAKER_ENABLED))).thenReturn(true); when(knnSettings.getSettingValue(eq(KNNSettings.KNN_MEMORY_CIRCUIT_BREAKER_LIMIT))).thenReturn(CIRCUIT_BREAKER_LIMIT_100KB); when(knnSettings.getSettingValue(eq(KNNSettings.KNN_CACHE_ITEM_EXPIRY_ENABLED))).thenReturn(false); @@ -118,6 +122,12 @@ public static void setUpClass() throws Exception { pathUtilsMockedStatic.when(() -> PathUtils.get(anyString(), anyString())).thenReturn(indexPath); } + @Before + public void setupBeforeTest() { + knnSettingsMockedStatic.when(() -> KNNSettings.getFilteredExactSearchThreshold(INDEX_NAME)).thenReturn(0); + knnSettingsMockedStatic.when(() -> KNNSettings.getFilteredExactSearchThresholdPct(INDEX_NAME)).thenReturn(0); + } + @SneakyThrows public void testQueryResultScoreNmslib() { for (SpaceType space : List.of(SpaceType.L2, SpaceType.L1, SpaceType.COSINESIMIL, SpaceType.INNER_PRODUCT, SpaceType.LINF)) { @@ -327,20 +337,27 @@ public void testEmptyQueryResults() { @SneakyThrows public void testANNWithFilterQuery_whenDoingANN_thenSuccess() { + int k = 3; final int[] filterDocIds = new int[] { 0, 1, 2, 3, 4, 5 }; jniServiceMockedStatic.when(() -> JNIService.queryIndex(anyLong(), any(), anyInt(), anyString(), eq(filterDocIds))) .thenReturn(getFilteredKNNQueryResults()); final LeafReaderContext leafReaderContext = mock(LeafReaderContext.class); final SegmentReader reader = mock(SegmentReader.class); - when(reader.maxDoc()).thenReturn(K + 1); + final Bits liveDocsBits = mock(Bits.class); + when(reader.maxDoc()).thenReturn(filterDocIds.length); + when(reader.getLiveDocs()).thenReturn(liveDocsBits); + for (int filterDocId : filterDocIds) { + when(liveDocsBits.get(filterDocId)).thenReturn(true); + } + when(liveDocsBits.length()).thenReturn(1000); when(leafReaderContext.reader()).thenReturn(reader); - final KNNQuery query = new KNNQuery(FIELD_NAME, QUERY_VECTOR, K, INDEX_NAME, FILTER_QUERY); + final KNNQuery query = new KNNQuery(FIELD_NAME, QUERY_VECTOR, k, INDEX_NAME, FILTER_QUERY); final Weight filterQueryWeight = mock(Weight.class); final Scorer filterScorer = mock(Scorer.class); when(filterQueryWeight.scorer(leafReaderContext)).thenReturn(filterScorer); // Just to make sure that we are not hitting the exact search condition - when(filterScorer.iterator()).thenReturn(DocIdSetIterator.all(K + 1)); + when(filterScorer.iterator()).thenReturn(DocIdSetIterator.all(filterDocIds.length + 1)); final KNNWeight knnWeight = new KNNWeight(query, 0.0f, filterQueryWeight); @@ -406,6 +423,10 @@ public void testANNWithFilterQuery_whenExactSearch_thenSuccess() { // scorer will return 2 documents when(filterScorer.iterator()).thenReturn(DocIdSetIterator.all(1)); when(reader.maxDoc()).thenReturn(1); + final Bits liveDocsBits = mock(Bits.class); + when(reader.getLiveDocs()).thenReturn(liveDocsBits); + when(liveDocsBits.get(filterDocId)).thenReturn(true); + when(liveDocsBits.length()).thenReturn(1000); final KNNWeight knnWeight = new KNNWeight(query, 0.0f, filterQueryWeight); final Map attributesMap = ImmutableMap.of(KNN_ENGINE, KNNEngine.FAISS.getName(), SPACE_TYPE, SpaceType.L2.name()); @@ -437,6 +458,68 @@ public void testANNWithFilterQuery_whenExactSearch_thenSuccess() { assertTrue(Comparators.isInOrder(actualDocIds, Comparator.naturalOrder())); } + /** + * This test ensure that we do the exact search when threshold settings are correct and not using filteredIds<=K + * condition to do exact search. + * FilteredIdThreshold: 10 + * FilteredIdThresholdPct: 10% + * FilteredIdsCount: 6 + * liveDocs : null, as there is no deleted documents + * MaxDoc: 100 + * K : 1 + */ + @SneakyThrows + public void testANNWithFilterQuery_whenExactSearchViaThresholdSetting_thenSuccess() { + knnSettingsMockedStatic.when(() -> KNNSettings.getFilteredExactSearchThreshold(INDEX_NAME)).thenReturn(10); + knnSettingsMockedStatic.when(() -> KNNSettings.getFilteredExactSearchThresholdPct(INDEX_NAME)).thenReturn(10); + float[] vector = new float[] { 0.1f, 0.3f }; + int k = 1; + final int[] filterDocIds = new int[] { 0, 1, 2, 3, 4, 5 }; + + final LeafReaderContext leafReaderContext = mock(LeafReaderContext.class); + final SegmentReader reader = mock(SegmentReader.class); + when(leafReaderContext.reader()).thenReturn(reader); + when(reader.maxDoc()).thenReturn(100); + when(reader.getLiveDocs()).thenReturn(null); + final Weight filterQueryWeight = mock(Weight.class); + final Scorer filterScorer = mock(Scorer.class); + when(filterQueryWeight.scorer(leafReaderContext)).thenReturn(filterScorer); + + when(filterScorer.iterator()).thenReturn(DocIdSetIterator.all(filterDocIds.length)); + + + final KNNQuery query = new KNNQuery(FIELD_NAME, QUERY_VECTOR, k, INDEX_NAME, FILTER_QUERY); + + final KNNWeight knnWeight = new KNNWeight(query, 0.0f, filterQueryWeight); + final Map attributesMap = ImmutableMap.of(KNN_ENGINE, KNNEngine.FAISS.getName(), SPACE_TYPE, SpaceType.L2.name()); + final FieldInfos fieldInfos = mock(FieldInfos.class); + final FieldInfo fieldInfo = mock(FieldInfo.class); + final BinaryDocValues binaryDocValues = mock(BinaryDocValues.class); + when(reader.getFieldInfos()).thenReturn(fieldInfos); + when(fieldInfos.fieldInfo(any())).thenReturn(fieldInfo); + when(fieldInfo.attributes()).thenReturn(attributesMap); + when(fieldInfo.getAttribute(SPACE_TYPE)).thenReturn(SpaceType.L2.name()); + when(fieldInfo.getName()).thenReturn(FIELD_NAME); + when(reader.getBinaryDocValues(FIELD_NAME)).thenReturn(binaryDocValues); + when(binaryDocValues.advance(0)).thenReturn(0); + BytesRef vectorByteRef = new BytesRef(new KNNVectorAsArraySerializer().floatToByteArray(vector)); + when(binaryDocValues.binaryValue()).thenReturn(vectorByteRef); + + final KNNScorer knnScorer = (KNNScorer) knnWeight.scorer(leafReaderContext); + assertNotNull(knnScorer); + final DocIdSetIterator docIdSetIterator = knnScorer.iterator(); + assertNotNull(docIdSetIterator); + assertEquals(EXACT_SEARCH_DOC_ID_TO_SCORES.size(), docIdSetIterator.cost()); + + final List actualDocIds = new ArrayList<>(); + for (int docId = docIdSetIterator.nextDoc(); docId != NO_MORE_DOCS; docId = docIdSetIterator.nextDoc()) { + actualDocIds.add(docId); + assertEquals(EXACT_SEARCH_DOC_ID_TO_SCORES.get(docId), knnScorer.score(), 0.01f); + } + assertEquals(docIdSetIterator.cost(), actualDocIds.size()); + assertTrue(Comparators.isInOrder(actualDocIds, Comparator.naturalOrder())); + } + @SneakyThrows public void testANNWithFilterQuery_whenEmptyFilterIds_thenReturnEarly() { final LeafReaderContext leafReaderContext = mock(LeafReaderContext.class); From 50fae344401157df85cc7c32c5af16fe245cabdb Mon Sep 17 00:00:00 2001 From: Navneet Verma Date: Fri, 25 Aug 2023 09:44:38 -0700 Subject: [PATCH 135/416] Fixing the backward incompatible changes coming from core in ScoreScript class. (#1056) (#1063) Signed-off-by: Navneet Verma --- .../knn/plugin/script/KNNScoreScript.java | 21 +++-- .../plugin/script/KNNScoreScriptFactory.java | 8 +- .../knn/plugin/script/KNNScoringSpace.java | 83 +++++++++++++------ .../knn/index/KNNSettingsTests.java | 27 +++--- .../knn/index/query/KNNWeightTests.java | 1 - 5 files changed, 93 insertions(+), 47 deletions(-) diff --git a/src/main/java/org/opensearch/knn/plugin/script/KNNScoreScript.java b/src/main/java/org/opensearch/knn/plugin/script/KNNScoreScript.java index f190a3e1d..d7a84817b 100644 --- a/src/main/java/org/opensearch/knn/plugin/script/KNNScoreScript.java +++ b/src/main/java/org/opensearch/knn/plugin/script/KNNScoreScript.java @@ -5,6 +5,7 @@ package org.opensearch.knn.plugin.script; +import org.apache.lucene.search.IndexSearcher; import org.opensearch.knn.index.KNNVectorScriptDocValues; import org.apache.lucene.index.LeafReaderContext; import org.opensearch.index.fielddata.ScriptDocValues; @@ -32,9 +33,10 @@ public KNNScoreScript( String field, BiFunction scoringMethod, SearchLookup lookup, - LeafReaderContext leafContext + LeafReaderContext leafContext, + IndexSearcher searcher ) { - super(params, lookup, leafContext); + super(params, lookup, searcher, leafContext); this.queryValue = queryValue; this.field = field; this.scoringMethod = scoringMethod; @@ -51,9 +53,10 @@ public LongType( String field, BiFunction scoringMethod, SearchLookup lookup, - LeafReaderContext leafContext + LeafReaderContext leafContext, + IndexSearcher searcher ) { - super(params, queryValue, field, scoringMethod, lookup, leafContext); + super(params, queryValue, field, scoringMethod, lookup, leafContext, searcher); } /** @@ -84,9 +87,10 @@ public BigIntegerType( String field, BiFunction scoringMethod, SearchLookup lookup, - LeafReaderContext leafContext + LeafReaderContext leafContext, + IndexSearcher searcher ) { - super(params, queryValue, field, scoringMethod, lookup, leafContext); + super(params, queryValue, field, scoringMethod, lookup, leafContext, searcher); } /** @@ -118,9 +122,10 @@ public KNNVectorType( String field, BiFunction scoringMethod, SearchLookup lookup, - LeafReaderContext leafContext + LeafReaderContext leafContext, + IndexSearcher searcher ) throws IOException { - super(params, queryValue, field, scoringMethod, lookup, leafContext); + super(params, queryValue, field, scoringMethod, lookup, leafContext, searcher); } /** diff --git a/src/main/java/org/opensearch/knn/plugin/script/KNNScoreScriptFactory.java b/src/main/java/org/opensearch/knn/plugin/script/KNNScoreScriptFactory.java index b686a20f0..63b367b2d 100644 --- a/src/main/java/org/opensearch/knn/plugin/script/KNNScoreScriptFactory.java +++ b/src/main/java/org/opensearch/knn/plugin/script/KNNScoreScriptFactory.java @@ -5,6 +5,7 @@ package org.opensearch.knn.plugin.script; +import org.apache.lucene.search.IndexSearcher; import org.opensearch.knn.plugin.stats.KNNCounter; import org.apache.lucene.index.LeafReaderContext; import org.opensearch.script.ScoreScript; @@ -21,13 +22,16 @@ public class KNNScoreScriptFactory implements ScoreScript.LeafFactory { private Object query; private KNNScoringSpace knnScoringSpace; - public KNNScoreScriptFactory(Map params, SearchLookup lookup) { + private IndexSearcher searcher; + + public KNNScoreScriptFactory(Map params, SearchLookup lookup, IndexSearcher searcher) { KNNCounter.SCRIPT_QUERY_REQUESTS.increment(); this.params = params; this.lookup = lookup; this.field = getValue(params, "field").toString(); this.similaritySpace = getValue(params, "space_type").toString(); this.query = getValue(params, "query_value"); + this.searcher = searcher; this.knnScoringSpace = KNNScoringSpaceFactory.create( this.similaritySpace, @@ -60,6 +64,6 @@ public boolean needs_score() { */ @Override public ScoreScript newInstance(LeafReaderContext ctx) throws IOException { - return knnScoringSpace.getScoreScript(params, field, lookup, ctx); + return knnScoringSpace.getScoreScript(params, field, lookup, ctx, this.searcher); } } diff --git a/src/main/java/org/opensearch/knn/plugin/script/KNNScoringSpace.java b/src/main/java/org/opensearch/knn/plugin/script/KNNScoringSpace.java index 16bf6e204..0e4c9f815 100644 --- a/src/main/java/org/opensearch/knn/plugin/script/KNNScoringSpace.java +++ b/src/main/java/org/opensearch/knn/plugin/script/KNNScoringSpace.java @@ -5,6 +5,7 @@ package org.opensearch.knn.plugin.script; +import org.apache.lucene.search.IndexSearcher; import org.opensearch.knn.index.mapper.KNNVectorFieldMapper; import org.opensearch.knn.index.query.KNNWeight; import org.apache.lucene.index.LeafReaderContext; @@ -29,14 +30,16 @@ public interface KNNScoringSpace { /** * Return the correct scoring script for a given query. The scoring script * - * @param params Map of parameters - * @param field Fieldname - * @param lookup SearchLookup - * @param ctx ctx LeafReaderContext to be used for scoring documents + * @param params Map of parameters + * @param field Fieldname + * @param lookup SearchLookup + * @param ctx ctx LeafReaderContext to be used for scoring documents + * @param searcher IndexSearcher * @return ScoreScript for this query * @throws IOException throws IOException if ScoreScript cannot be constructed */ - ScoreScript getScoreScript(Map params, String field, SearchLookup lookup, LeafReaderContext ctx) throws IOException; + ScoreScript getScoreScript(Map params, String field, SearchLookup lookup, LeafReaderContext ctx, IndexSearcher searcher) + throws IOException; class L2 implements KNNScoringSpace { @@ -62,9 +65,14 @@ public L2(Object query, MappedFieldType fieldType) { this.scoringMethod = (float[] q, float[] v) -> 1 / (1 + KNNScoringUtil.l2Squared(q, v)); } - public ScoreScript getScoreScript(Map params, String field, SearchLookup lookup, LeafReaderContext ctx) - throws IOException { - return new KNNScoreScript.KNNVectorType(params, this.processedQuery, field, this.scoringMethod, lookup, ctx); + public ScoreScript getScoreScript( + Map params, + String field, + SearchLookup lookup, + LeafReaderContext ctx, + IndexSearcher searcher + ) throws IOException { + return new KNNScoreScript.KNNVectorType(params, this.processedQuery, field, this.scoringMethod, lookup, ctx, searcher); } } @@ -94,9 +102,14 @@ public CosineSimilarity(Object query, MappedFieldType fieldType) { this.scoringMethod = (float[] q, float[] v) -> 1 + KNNScoringUtil.cosinesimilOptimized(q, v, qVectorSquaredMagnitude); } - public ScoreScript getScoreScript(Map params, String field, SearchLookup lookup, LeafReaderContext ctx) - throws IOException { - return new KNNScoreScript.KNNVectorType(params, this.processedQuery, field, this.scoringMethod, lookup, ctx); + public ScoreScript getScoreScript( + Map params, + String field, + SearchLookup lookup, + LeafReaderContext ctx, + IndexSearcher searcher + ) throws IOException { + return new KNNScoreScript.KNNVectorType(params, this.processedQuery, field, this.scoringMethod, lookup, ctx, searcher); } } @@ -127,8 +140,13 @@ public HammingBit(Object query, MappedFieldType fieldType) { } @SuppressWarnings("unchecked") - public ScoreScript getScoreScript(Map params, String field, SearchLookup lookup, LeafReaderContext ctx) - throws IOException { + public ScoreScript getScoreScript( + Map params, + String field, + SearchLookup lookup, + LeafReaderContext ctx, + IndexSearcher searcher + ) throws IOException { if (this.processedQuery instanceof Long) { return new KNNScoreScript.LongType( params, @@ -136,7 +154,8 @@ public ScoreScript getScoreScript(Map params, String field, Sear field, (BiFunction) this.scoringMethod, lookup, - ctx + ctx, + searcher ); } @@ -146,7 +165,8 @@ public ScoreScript getScoreScript(Map params, String field, Sear field, (BiFunction) this.scoringMethod, lookup, - ctx + ctx, + searcher ); } } @@ -175,9 +195,14 @@ public L1(Object query, MappedFieldType fieldType) { this.scoringMethod = (float[] q, float[] v) -> 1 / (1 + KNNScoringUtil.l1Norm(q, v)); } - public ScoreScript getScoreScript(Map params, String field, SearchLookup lookup, LeafReaderContext ctx) - throws IOException { - return new KNNScoreScript.KNNVectorType(params, this.processedQuery, field, this.scoringMethod, lookup, ctx); + public ScoreScript getScoreScript( + Map params, + String field, + SearchLookup lookup, + LeafReaderContext ctx, + IndexSearcher searcher + ) throws IOException { + return new KNNScoreScript.KNNVectorType(params, this.processedQuery, field, this.scoringMethod, lookup, ctx, searcher); } } @@ -205,9 +230,14 @@ public LInf(Object query, MappedFieldType fieldType) { this.scoringMethod = (float[] q, float[] v) -> 1 / (1 + KNNScoringUtil.lInfNorm(q, v)); } - public ScoreScript getScoreScript(Map params, String field, SearchLookup lookup, LeafReaderContext ctx) - throws IOException { - return new KNNScoreScript.KNNVectorType(params, this.processedQuery, field, this.scoringMethod, lookup, ctx); + public ScoreScript getScoreScript( + Map params, + String field, + SearchLookup lookup, + LeafReaderContext ctx, + IndexSearcher searcher + ) throws IOException { + return new KNNScoreScript.KNNVectorType(params, this.processedQuery, field, this.scoringMethod, lookup, ctx, searcher); } } @@ -238,9 +268,14 @@ public InnerProd(Object query, MappedFieldType fieldType) { } @Override - public ScoreScript getScoreScript(Map params, String field, SearchLookup lookup, LeafReaderContext ctx) - throws IOException { - return new KNNScoreScript.KNNVectorType(params, this.processedQuery, field, this.scoringMethod, lookup, ctx); + public ScoreScript getScoreScript( + Map params, + String field, + SearchLookup lookup, + LeafReaderContext ctx, + IndexSearcher searcher + ) throws IOException { + return new KNNScoreScript.KNNVectorType(params, this.processedQuery, field, this.scoringMethod, lookup, ctx, searcher); } } } diff --git a/src/test/java/org/opensearch/knn/index/KNNSettingsTests.java b/src/test/java/org/opensearch/knn/index/KNNSettingsTests.java index 9432be33e..17a58cbca 100644 --- a/src/test/java/org/opensearch/knn/index/KNNSettingsTests.java +++ b/src/test/java/org/opensearch/knn/index/KNNSettingsTests.java @@ -122,30 +122,33 @@ public void testFilteredSearchAdvanceSetting_whenValuesProvidedByUsers_thenValid // validate if we are able to set MinValues for the setting final Settings filteredSearchAdvanceSettingsWithMinValues = Settings.builder() - .put(KNNSettings.ADVANCED_FILTERED_EXACT_SEARCH_THRESHOLD, userDefinedThresholdMinValue) - .put(KNNSettings.ADVANCED_FILTERED_EXACT_SEARCH_THRESHOLD_PCT, userDefinedPctThresholdMinValue) - .build(); + .put(KNNSettings.ADVANCED_FILTERED_EXACT_SEARCH_THRESHOLD, userDefinedThresholdMinValue) + .put(KNNSettings.ADVANCED_FILTERED_EXACT_SEARCH_THRESHOLD_PCT, userDefinedPctThresholdMinValue) + .build(); mockNode.client() - .admin() - .indices() - .updateSettings(new UpdateSettingsRequest(filteredSearchAdvanceSettingsWithMinValues, INDEX_NAME)) - .actionGet(); + .admin() + .indices() + .updateSettings(new UpdateSettingsRequest(filteredSearchAdvanceSettingsWithMinValues, INDEX_NAME)) + .actionGet(); int filteredSearchThresholdPctMinValue = KNNSettings.getFilteredExactSearchThresholdPct(INDEX_NAME); int filteredSearchThresholdMinValue = KNNSettings.getFilteredExactSearchThreshold(INDEX_NAME); // Validate if less than MinValues are set then Exception Happens final Settings filteredSearchAdvanceSettingsWithLessThanMinValues = Settings.builder() - .put(KNNSettings.ADVANCED_FILTERED_EXACT_SEARCH_THRESHOLD, -1) - .put(KNNSettings.ADVANCED_FILTERED_EXACT_SEARCH_THRESHOLD_PCT, -1) - .build(); + .put(KNNSettings.ADVANCED_FILTERED_EXACT_SEARCH_THRESHOLD, -1) + .put(KNNSettings.ADVANCED_FILTERED_EXACT_SEARCH_THRESHOLD_PCT, -1) + .build(); - Assert.assertThrows(IllegalArgumentException.class, () -> mockNode.client() + Assert.assertThrows( + IllegalArgumentException.class, + () -> mockNode.client() .admin() .indices() .updateSettings(new UpdateSettingsRequest(filteredSearchAdvanceSettingsWithLessThanMinValues, INDEX_NAME)) - .actionGet()); + .actionGet() + ); mockNode.close(); assertEquals(userDefinedPctThreshold, filteredSearchThresholdPct); diff --git a/src/test/java/org/opensearch/knn/index/query/KNNWeightTests.java b/src/test/java/org/opensearch/knn/index/query/KNNWeightTests.java index 9cc624377..6b7bb3208 100644 --- a/src/test/java/org/opensearch/knn/index/query/KNNWeightTests.java +++ b/src/test/java/org/opensearch/knn/index/query/KNNWeightTests.java @@ -487,7 +487,6 @@ public void testANNWithFilterQuery_whenExactSearchViaThresholdSetting_thenSucces when(filterScorer.iterator()).thenReturn(DocIdSetIterator.all(filterDocIds.length)); - final KNNQuery query = new KNNQuery(FIELD_NAME, QUERY_VECTOR, k, INDEX_NAME, FILTER_QUERY); final KNNWeight knnWeight = new KNNWeight(query, 0.0f, filterQueryWeight); From 865f6131adb06b5d77f714038263b3315444a297 Mon Sep 17 00:00:00 2001 From: Navneet Verma Date: Mon, 28 Aug 2023 16:35:51 -0700 Subject: [PATCH 136/416] Added max distance computation logic to enhance the switch to exact search in filtered Nearest Neighbor Search. (#1066) (#1068) Signed-off-by: Navneet Verma --- CHANGELOG.md | 1 + .../opensearch/knn/common/KNNConstants.java | 5 ++++ .../org/opensearch/knn/index/KNNSettings.java | 30 ++----------------- .../opensearch/knn/index/query/KNNWeight.java | 30 ++++++++++++++----- .../knn/index/KNNSettingsTests.java | 26 ++-------------- .../knn/index/query/KNNWeightTests.java | 2 -- 6 files changed, 33 insertions(+), 61 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3ddd7fca2..9e358179f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Enhancements * Enabled the IVF algorithm to work with Filters of K-NN Query. [#1013](https://github.com/opensearch-project/k-NN/pull/1013) * Improved the logic to switch to exact search for restrictive filters search for better recall. [#1059](https://github.com/opensearch-project/k-NN/pull/1059) +* Added max distance computation logic to enhance the switch to exact search in filtered Nearest Neighbor Search. [#1066](https://github.com/opensearch-project/k-NN/pull/1066) ### Bug Fixes ### Infrastructure ### Documentation diff --git a/src/main/java/org/opensearch/knn/common/KNNConstants.java b/src/main/java/org/opensearch/knn/common/KNNConstants.java index 6d387eec4..59732cca0 100644 --- a/src/main/java/org/opensearch/knn/common/KNNConstants.java +++ b/src/main/java/org/opensearch/knn/common/KNNConstants.java @@ -98,4 +98,9 @@ public class KNNConstants { private static final String JNI_LIBRARY_PREFIX = "opensearchknn_"; public static final String FAISS_JNI_LIBRARY_NAME = JNI_LIBRARY_PREFIX + FAISS_NAME; public static final String NMSLIB_JNI_LIBRARY_NAME = JNI_LIBRARY_PREFIX + NMSLIB_NAME; + + // Filtered Search Constants + // Please refer this github issue for more details for choosing this value: + // https://github.com/opensearch-project/k-NN/issues/1049#issuecomment-1694741092 + public static int MAX_DISTANCE_COMPUTATIONS = 2048000; } diff --git a/src/main/java/org/opensearch/knn/index/KNNSettings.java b/src/main/java/org/opensearch/knn/index/KNNSettings.java index a1958233b..4356a0610 100644 --- a/src/main/java/org/opensearch/knn/index/KNNSettings.java +++ b/src/main/java/org/opensearch/knn/index/KNNSettings.java @@ -74,7 +74,6 @@ public class KNNSettings { public static final String MODEL_INDEX_NUMBER_OF_REPLICAS = "knn.model.index.number_of_replicas"; public static final String MODEL_CACHE_SIZE_LIMIT = "knn.model.cache.size.limit"; public static final String ADVANCED_FILTERED_EXACT_SEARCH_THRESHOLD = "index.knn.advanced.filtered_exact_search_threshold"; - public static final String ADVANCED_FILTERED_EXACT_SEARCH_THRESHOLD_PCT = "index.knn.advanced.filtered_exact_search_threshold_pct"; /** * Default setting values @@ -89,8 +88,7 @@ public class KNNSettings { public static final Integer KNN_MAX_MODEL_CACHE_SIZE_LIMIT_PERCENTAGE = 25; // Model cache limit cannot exceed 25% of the JVM heap public static final String KNN_DEFAULT_MEMORY_CIRCUIT_BREAKER_LIMIT = "50%"; - public static final Integer ADVANCED_FILTERED_EXACT_SEARCH_THRESHOLD_DEFAULT_VALUE = 2000; - public static final Integer ADVANCED_FILTERED_EXACT_SEARCH_THRESHOLD_PCT_DEFAULT_VALUE = 10; + public static final Integer ADVANCED_FILTERED_EXACT_SEARCH_THRESHOLD_DEFAULT_VALUE = -1; /** * Settings Definition @@ -162,15 +160,6 @@ public class KNNSettings { public static final Setting ADVANCED_FILTERED_EXACT_SEARCH_THRESHOLD_SETTING = Setting.intSetting( ADVANCED_FILTERED_EXACT_SEARCH_THRESHOLD, ADVANCED_FILTERED_EXACT_SEARCH_THRESHOLD_DEFAULT_VALUE, - 0, - IndexScope, - Setting.Property.Dynamic - ); - - public static final Setting ADVANCED_FILTERED_EXACT_SEARCH_THRESHOLD_PCT_SETTING = Setting.intSetting( - ADVANCED_FILTERED_EXACT_SEARCH_THRESHOLD_PCT, - ADVANCED_FILTERED_EXACT_SEARCH_THRESHOLD_PCT_DEFAULT_VALUE, - 0, IndexScope, Setting.Property.Dynamic ); @@ -348,10 +337,6 @@ private Setting getSetting(String key) { return ADVANCED_FILTERED_EXACT_SEARCH_THRESHOLD_SETTING; } - if (ADVANCED_FILTERED_EXACT_SEARCH_THRESHOLD_PCT.equals(key)) { - return ADVANCED_FILTERED_EXACT_SEARCH_THRESHOLD_PCT_SETTING; - } - throw new IllegalArgumentException("Cannot find setting by key [" + key + "]"); } @@ -368,8 +353,7 @@ public List> getSettings() { MODEL_INDEX_NUMBER_OF_SHARDS_SETTING, MODEL_INDEX_NUMBER_OF_REPLICAS_SETTING, MODEL_CACHE_SIZE_LIMIT_SETTING, - ADVANCED_FILTERED_EXACT_SEARCH_THRESHOLD_SETTING, - ADVANCED_FILTERED_EXACT_SEARCH_THRESHOLD_PCT_SETTING + ADVANCED_FILTERED_EXACT_SEARCH_THRESHOLD_SETTING ); return Stream.concat(settings.stream(), dynamicCacheSettings.values().stream()).collect(Collectors.toList()); } @@ -390,7 +374,7 @@ public static double getCircuitBreakerUnsetPercentage() { return KNNSettings.state().getSettingValue(KNNSettings.KNN_CIRCUIT_BREAKER_UNSET_PERCENTAGE); } - public static int getFilteredExactSearchThreshold(final String indexName) { + public static Integer getFilteredExactSearchThreshold(final String indexName) { return KNNSettings.state().clusterService.state() .getMetadata() .index(indexName) @@ -398,14 +382,6 @@ public static int getFilteredExactSearchThreshold(final String indexName) { .getAsInt(ADVANCED_FILTERED_EXACT_SEARCH_THRESHOLD, ADVANCED_FILTERED_EXACT_SEARCH_THRESHOLD_DEFAULT_VALUE); } - public static int getFilteredExactSearchThresholdPct(final String indexName) { - return KNNSettings.state().clusterService.state() - .getMetadata() - .index(indexName) - .getSettings() - .getAsInt(ADVANCED_FILTERED_EXACT_SEARCH_THRESHOLD_PCT, ADVANCED_FILTERED_EXACT_SEARCH_THRESHOLD_PCT_DEFAULT_VALUE); - } - public void initialize(Client client, ClusterService clusterService) { this.client = client; this.clusterService = clusterService; diff --git a/src/main/java/org/opensearch/knn/index/query/KNNWeight.java b/src/main/java/org/opensearch/knn/index/query/KNNWeight.java index 7352dd436..4778ba25d 100644 --- a/src/main/java/org/opensearch/knn/index/query/KNNWeight.java +++ b/src/main/java/org/opensearch/knn/index/query/KNNWeight.java @@ -375,18 +375,32 @@ private SpaceType getSpaceType(final FieldInfo fieldInfo) { private boolean canDoExactSearch(final int filterIdsCount, final int searchableDocs) { log.debug( - "Info for doing exact search Live Docs: {}, filterIdsLength : {}, Threshold value: {} , Threshold %age : {}", + "Info for doing exact search Live Docs: {}, filterIdsLength : {}, Threshold value: {}", searchableDocs, filterIdsCount, - KNNSettings.getFilteredExactSearchThreshold(knnQuery.getIndexName()), - KNNSettings.getFilteredExactSearchThresholdPct(knnQuery.getIndexName()) + KNNSettings.getFilteredExactSearchThreshold(knnQuery.getIndexName()) ); + int filterThresholdValue = KNNSettings.getFilteredExactSearchThreshold(knnQuery.getIndexName()); // Refer this GitHub around more details https://github.com/opensearch-project/k-NN/issues/1049 on the logic - return filterIdsCount <= knnQuery.getK() - || (filterIdsCount <= KNNSettings.getFilteredExactSearchThreshold(knnQuery.getIndexName()) - && (((float) filterIdsCount / (float) searchableDocs) * 100) <= (float) KNNSettings.getFilteredExactSearchThresholdPct( - knnQuery.getIndexName() - )); + if (filterIdsCount <= knnQuery.getK()) { + return true; + } + // See user has defined Exact Search filtered threshold. if yes, then use that setting. + if (isExactSearchThresholdSettingSet(filterThresholdValue)) { + return filterThresholdValue >= filterIdsCount; + } + // if no setting is set, then use the default max distance computation value to see if we can do exact search. + return KNNConstants.MAX_DISTANCE_COMPUTATIONS <= filterIdsCount * knnQuery.getQueryVector().length; + } + + /** + * This function validates if {@link KNNSettings#ADVANCED_FILTERED_EXACT_SEARCH_THRESHOLD} is set or not. This + * is done by validating if the setting value is equal to the default value. + * @param filterThresholdValue value of the Index Setting: {@link KNNSettings#ADVANCED_FILTERED_EXACT_SEARCH_THRESHOLD_SETTING} + * @return boolean true if the setting is set. + */ + private boolean isExactSearchThresholdSettingSet(int filterThresholdValue) { + return filterThresholdValue != KNNSettings.ADVANCED_FILTERED_EXACT_SEARCH_THRESHOLD_DEFAULT_VALUE; } /** diff --git a/src/test/java/org/opensearch/knn/index/KNNSettingsTests.java b/src/test/java/org/opensearch/knn/index/KNNSettingsTests.java index 17a58cbca..4af62a4c4 100644 --- a/src/test/java/org/opensearch/knn/index/KNNSettingsTests.java +++ b/src/test/java/org/opensearch/knn/index/KNNSettingsTests.java @@ -6,7 +6,6 @@ package org.opensearch.knn.index; import lombok.SneakyThrows; -import org.junit.Assert; import org.opensearch.action.admin.cluster.state.ClusterStateRequest; import org.opensearch.action.admin.indices.create.CreateIndexRequest; import org.opensearch.action.admin.indices.settings.put.UpdateSettingsRequest; @@ -85,19 +84,15 @@ public void testFilteredSearchAdvanceSetting_whenNoValuesProvidedByUsers_thenDef mockNode.client().admin().indices().create(new CreateIndexRequest(INDEX_NAME)).actionGet(); KNNSettings.state().setClusterService(clusterService); - int filteredSearchThresholdPct = KNNSettings.getFilteredExactSearchThresholdPct(INDEX_NAME); - int filteredSearchThreshold = KNNSettings.getFilteredExactSearchThreshold(INDEX_NAME); + Integer filteredSearchThreshold = KNNSettings.getFilteredExactSearchThreshold(INDEX_NAME); mockNode.close(); - assertEquals((int) KNNSettings.ADVANCED_FILTERED_EXACT_SEARCH_THRESHOLD_PCT_DEFAULT_VALUE, filteredSearchThresholdPct); - assertEquals((int) KNNSettings.ADVANCED_FILTERED_EXACT_SEARCH_THRESHOLD_DEFAULT_VALUE, filteredSearchThreshold); + assertEquals(KNNSettings.ADVANCED_FILTERED_EXACT_SEARCH_THRESHOLD_DEFAULT_VALUE, filteredSearchThreshold); assertWarnings(); } @SneakyThrows public void testFilteredSearchAdvanceSetting_whenValuesProvidedByUsers_thenValidateSameValues() { - int userDefinedPctThreshold = 20; int userDefinedThreshold = 1000; - int userDefinedPctThresholdMinValue = 0; int userDefinedThresholdMinValue = 0; Node mockNode = createMockNode(Collections.emptyMap()); mockNode.start(); @@ -108,7 +103,6 @@ public void testFilteredSearchAdvanceSetting_whenValuesProvidedByUsers_thenValid final Settings filteredSearchAdvanceSettings = Settings.builder() .put(KNNSettings.ADVANCED_FILTERED_EXACT_SEARCH_THRESHOLD, userDefinedThreshold) - .put(KNNSettings.ADVANCED_FILTERED_EXACT_SEARCH_THRESHOLD_PCT, userDefinedPctThreshold) .build(); mockNode.client() @@ -117,13 +111,11 @@ public void testFilteredSearchAdvanceSetting_whenValuesProvidedByUsers_thenValid .updateSettings(new UpdateSettingsRequest(filteredSearchAdvanceSettings, INDEX_NAME)) .actionGet(); - int filteredSearchThresholdPct = KNNSettings.getFilteredExactSearchThresholdPct(INDEX_NAME); int filteredSearchThreshold = KNNSettings.getFilteredExactSearchThreshold(INDEX_NAME); // validate if we are able to set MinValues for the setting final Settings filteredSearchAdvanceSettingsWithMinValues = Settings.builder() .put(KNNSettings.ADVANCED_FILTERED_EXACT_SEARCH_THRESHOLD, userDefinedThresholdMinValue) - .put(KNNSettings.ADVANCED_FILTERED_EXACT_SEARCH_THRESHOLD_PCT, userDefinedPctThresholdMinValue) .build(); mockNode.client() @@ -132,28 +124,14 @@ public void testFilteredSearchAdvanceSetting_whenValuesProvidedByUsers_thenValid .updateSettings(new UpdateSettingsRequest(filteredSearchAdvanceSettingsWithMinValues, INDEX_NAME)) .actionGet(); - int filteredSearchThresholdPctMinValue = KNNSettings.getFilteredExactSearchThresholdPct(INDEX_NAME); int filteredSearchThresholdMinValue = KNNSettings.getFilteredExactSearchThreshold(INDEX_NAME); // Validate if less than MinValues are set then Exception Happens final Settings filteredSearchAdvanceSettingsWithLessThanMinValues = Settings.builder() .put(KNNSettings.ADVANCED_FILTERED_EXACT_SEARCH_THRESHOLD, -1) - .put(KNNSettings.ADVANCED_FILTERED_EXACT_SEARCH_THRESHOLD_PCT, -1) .build(); - - Assert.assertThrows( - IllegalArgumentException.class, - () -> mockNode.client() - .admin() - .indices() - .updateSettings(new UpdateSettingsRequest(filteredSearchAdvanceSettingsWithLessThanMinValues, INDEX_NAME)) - .actionGet() - ); - mockNode.close(); - assertEquals(userDefinedPctThreshold, filteredSearchThresholdPct); assertEquals(userDefinedThreshold, filteredSearchThreshold); - assertEquals(userDefinedPctThresholdMinValue, filteredSearchThresholdPctMinValue); assertEquals(userDefinedThresholdMinValue, filteredSearchThresholdMinValue); assertWarnings(); } diff --git a/src/test/java/org/opensearch/knn/index/query/KNNWeightTests.java b/src/test/java/org/opensearch/knn/index/query/KNNWeightTests.java index 6b7bb3208..3a5e984f4 100644 --- a/src/test/java/org/opensearch/knn/index/query/KNNWeightTests.java +++ b/src/test/java/org/opensearch/knn/index/query/KNNWeightTests.java @@ -125,7 +125,6 @@ public static void setUpClass() throws Exception { @Before public void setupBeforeTest() { knnSettingsMockedStatic.when(() -> KNNSettings.getFilteredExactSearchThreshold(INDEX_NAME)).thenReturn(0); - knnSettingsMockedStatic.when(() -> KNNSettings.getFilteredExactSearchThresholdPct(INDEX_NAME)).thenReturn(0); } @SneakyThrows @@ -471,7 +470,6 @@ public void testANNWithFilterQuery_whenExactSearch_thenSuccess() { @SneakyThrows public void testANNWithFilterQuery_whenExactSearchViaThresholdSetting_thenSuccess() { knnSettingsMockedStatic.when(() -> KNNSettings.getFilteredExactSearchThreshold(INDEX_NAME)).thenReturn(10); - knnSettingsMockedStatic.when(() -> KNNSettings.getFilteredExactSearchThresholdPct(INDEX_NAME)).thenReturn(10); float[] vector = new float[] { 0.1f, 0.3f }; int k = 1; final int[] filterDocIds = new int[] { 0, 1, 2, 3, 4, 5 }; From f7b4415153e8c2829f0037fbfd61583289154669 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Wed, 30 Aug 2023 09:25:30 -0700 Subject: [PATCH 137/416] Fixed the github actions to use the image used by OpenSearch-build repo for building the distribution (#1069) (#1072) * Fixed the github actions to use the image used by OpenSearch-build repo for building the distribution Signed-off-by: Navneet Verma * Retry switch user on the CI using prod docker images Signed-off-by: Peter Zhu * Cleanup debug commands Signed-off-by: Peter Zhu --------- Signed-off-by: Navneet Verma Signed-off-by: Peter Zhu Co-authored-by: Peter Zhu (cherry picked from commit 8994de6ab6f5d05694f5d937022089a7963d1686) Co-authored-by: Navneet Verma --- .github/workflows/CI.yml | 49 +++++++++++++++++++++++++++++----------- 1 file changed, 36 insertions(+), 13 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 31d16f21f..d16287dd3 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -16,27 +16,56 @@ jobs: strategy: matrix: java: [11, 17] - os: [ubuntu-latest, macos-latest] - name: Build and Test k-NN Plugin - runs-on: ${{ matrix.os }} + name: Build and Test k-NN Plugin on Linux + runs-on: ubuntu-latest + container: + # using the same image which is used by opensearch-build team to build the OpenSearch Distribution + # this image tag is subject to change as more dependencies and updates will arrive over time + image: public.ecr.aws/opensearchstaging/ci-runner:ci-runner-rockylinux8-opensearch-build-v4 + # need to switch to root so that github actions can install runner binary on container without permission issues. + options: --user root steps: - name: Checkout k-NN uses: actions/checkout@v1 + with: + submodules: true - name: Setup Java ${{ matrix.java }} uses: actions/setup-java@v1 with: java-version: ${{ matrix.java }} - - name: Install dependencies on ubuntu - if: startsWith(matrix.os,'ubuntu') + - name: Run build + # switching the user, as OpenSearch cluster can only be started as root/Administrator on linux-deb/linux-rpm/windows-zip. run: | - sudo apt-get install libopenblas-dev gfortran -y + chown -R opensearch.opensearch `pwd` + su opensearch -c "whoami && java -version && ./gradlew build" + + - name: Upload Coverage Report + uses: codecov/codecov-action@v1 + with: + token: ${{ secrets.CODECOV_TOKEN }} + + Build-k-NN-MacOS: + strategy: + matrix: + java: [ 11, 17 ] + + name: Build and Test k-NN Plugin on MacOS + runs-on: macos-latest + + steps: + - name: Checkout k-NN + uses: actions/checkout@v1 + + - name: Setup Java ${{ matrix.java }} + uses: actions/setup-java@v1 + with: + java-version: ${{ matrix.java }} - name: Install dependencies on macos - if: startsWith(matrix.os, 'macos') run: | brew reinstall gcc export FC=/usr/local/Cellar/gcc/12.2.0/bin/gfortran @@ -45,12 +74,6 @@ jobs: run: | ./gradlew build - - name: Upload Coverage Report - if: startsWith(matrix.os,'ubuntu') - uses: codecov/codecov-action@v1 - with: - token: ${{ secrets.CODECOV_TOKEN }} - Build-k-NN-Windows: strategy: matrix: From a67e9874126bdf831f6706c8b55805f1e2b1f86c Mon Sep 17 00:00:00 2001 From: John Mazanec Date: Wed, 30 Aug 2023 13:01:47 -0700 Subject: [PATCH 138/416] Update Faiss engine to allow PQ and HNSW (#1074) Updates faiss engine to enable hnsw and faiss to work together. For HNSW, code_size must be equal to 8 (refer to https://github.com/facebookresearch/faiss/issues/3027). Therefore, the index description string "HNSW32,PQXxY" does not work. Only "HNSW32,PQX" ends up working. Additionally, adds several unit tests and integration tests in order to validate the functionality. Signed-off-by: John Mazanec --- CHANGELOG.md | 1 + .../org/opensearch/knn/index/util/Faiss.java | 160 ++++++++++++------ .../org/opensearch/knn/index/FaissIT.java | 119 ++++++++++++- .../knn/index/KNNMethodContextTests.java | 60 +++++-- .../opensearch/knn/index/util/FaissTests.java | 111 ++++++++++++ .../opensearch/knn/jni/JNIServiceTests.java | 98 +++++++++-- .../org/opensearch/knn/KNNRestTestCase.java | 30 ++++ 7 files changed, 491 insertions(+), 88 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9e358179f..a421e08e4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), * Improved the logic to switch to exact search for restrictive filters search for better recall. [#1059](https://github.com/opensearch-project/k-NN/pull/1059) * Added max distance computation logic to enhance the switch to exact search in filtered Nearest Neighbor Search. [#1066](https://github.com/opensearch-project/k-NN/pull/1066) ### Bug Fixes +* Update Faiss parameter construction to allow HNSW+PQ to work [#1074](https://github.com/opensearch-project/k-NN/pull/1074) ### Infrastructure ### Documentation ### Maintenance diff --git a/src/main/java/org/opensearch/knn/index/util/Faiss.java b/src/main/java/org/opensearch/knn/index/util/Faiss.java index 6c091da03..71eed404a 100644 --- a/src/main/java/org/opensearch/knn/index/util/Faiss.java +++ b/src/main/java/org/opensearch/knn/index/util/Faiss.java @@ -18,6 +18,7 @@ import java.util.Collections; import java.util.HashMap; import java.util.Map; +import java.util.Objects; import java.util.function.Function; import static org.opensearch.knn.common.KNNConstants.BYTES_PER_KILOBYTES; @@ -64,9 +65,7 @@ class Faiss extends NativeLibrary { Collections.emptyMap() ); - // TODO: To think about in future: for PQ, if dimension is not divisible by code count, PQ will fail. Right now, - // we do not have a way to base validation off of dimension. Failure will happen during training in JNI. - private final static Map encoderComponents = ImmutableMap.of( + private final static Map COMMON_ENCODERS = ImmutableMap.of( KNNConstants.ENCODER_FLAT, MethodComponent.Builder.builder(KNNConstants.ENCODER_FLAT) .setMapGenerator( @@ -76,62 +75,111 @@ class Faiss extends NativeLibrary { methodComponentContext ).build()) ) - .build(), - KNNConstants.ENCODER_PQ, - MethodComponent.Builder.builder(KNNConstants.ENCODER_PQ) - .addParameter( - ENCODER_PARAMETER_PQ_M, - new Parameter.IntegerParameter( - ENCODER_PARAMETER_PQ_M, - ENCODER_PARAMETER_PQ_CODE_COUNT_DEFAULT, - v -> v > 0 && v < ENCODER_PARAMETER_PQ_CODE_COUNT_LIMIT - ) - ) - .addParameter( - ENCODER_PARAMETER_PQ_CODE_SIZE, - new Parameter.IntegerParameter( - ENCODER_PARAMETER_PQ_CODE_SIZE, - ENCODER_PARAMETER_PQ_CODE_SIZE_DEFAULT, - v -> v > 0 && v < ENCODER_PARAMETER_PQ_CODE_SIZE_LIMIT - ) - ) - .setRequiresTraining(true) - .setMapGenerator( - ((methodComponent, methodComponentContext) -> MethodAsMapBuilder.builder( - FAISS_PQ_DESCRIPTION, - methodComponent, - methodComponentContext - ).addParameter(ENCODER_PARAMETER_PQ_M, "", "").addParameter(ENCODER_PARAMETER_PQ_CODE_SIZE, "x", "").build()) + .build() + ); + + // TODO: To think about in future: for PQ, if dimension is not divisible by code count, PQ will fail. Right now, + // we do not have a way to base validation off of dimension. Failure will happen during training in JNI. + // Define methods supported by faiss. See issue here: https://github.com/opensearch-project/k-NN/issues/1075 + private final static Map HNSW_ENCODERS = ImmutableMap.builder() + .putAll( + ImmutableMap.of( + KNNConstants.ENCODER_PQ, + MethodComponent.Builder.builder(KNNConstants.ENCODER_PQ) + .addParameter( + ENCODER_PARAMETER_PQ_M, + new Parameter.IntegerParameter( + ENCODER_PARAMETER_PQ_M, + ENCODER_PARAMETER_PQ_CODE_COUNT_DEFAULT, + v -> v > 0 && v < ENCODER_PARAMETER_PQ_CODE_COUNT_LIMIT + ) + ) + .addParameter( + ENCODER_PARAMETER_PQ_CODE_SIZE, + new Parameter.IntegerParameter( + ENCODER_PARAMETER_PQ_CODE_SIZE, + ENCODER_PARAMETER_PQ_CODE_SIZE_DEFAULT, + v -> Objects.equals(v, ENCODER_PARAMETER_PQ_CODE_SIZE_DEFAULT) + ) + ) + .setRequiresTraining(true) + .setMapGenerator( + ((methodComponent, methodComponentContext) -> MethodAsMapBuilder.builder( + FAISS_PQ_DESCRIPTION, + methodComponent, + methodComponentContext + ).addParameter(ENCODER_PARAMETER_PQ_M, "", "").build()) + ) + .setOverheadInKBEstimator((methodComponent, methodComponentContext, dimension) -> { + int codeSize = ENCODER_PARAMETER_PQ_CODE_SIZE_DEFAULT; + return ((4L * (1L << codeSize) * dimension) / BYTES_PER_KILOBYTES) + 1; + }) + .build() ) - .setOverheadInKBEstimator((methodComponent, methodComponentContext, dimension) -> { - // Size estimate formula: (4 * d * 2^code_size) / 1024 + 1 - - // Get value of code size passed in by user - Object codeSizeObject = methodComponentContext.getParameters().get(ENCODER_PARAMETER_PQ_CODE_SIZE); - - // If not specified, get default value of code size - if (codeSizeObject == null) { - Parameter codeSizeParameter = methodComponent.getParameters().get(ENCODER_PARAMETER_PQ_CODE_SIZE); - if (codeSizeParameter == null) { - throw new IllegalStateException( - String.format("%s is not a valid parameter. This is a bug.", ENCODER_PARAMETER_PQ_CODE_SIZE) - ); - } + ) + .putAll(COMMON_ENCODERS) + .build(); - codeSizeObject = codeSizeParameter.getDefaultValue(); - } + private final static Map IVF_ENCODERS = ImmutableMap.builder() + .putAll( + ImmutableMap.of( + KNNConstants.ENCODER_PQ, + MethodComponent.Builder.builder(KNNConstants.ENCODER_PQ) + .addParameter( + ENCODER_PARAMETER_PQ_M, + new Parameter.IntegerParameter( + ENCODER_PARAMETER_PQ_M, + ENCODER_PARAMETER_PQ_CODE_COUNT_DEFAULT, + v -> v > 0 && v < ENCODER_PARAMETER_PQ_CODE_COUNT_LIMIT + ) + ) + .addParameter( + ENCODER_PARAMETER_PQ_CODE_SIZE, + new Parameter.IntegerParameter( + ENCODER_PARAMETER_PQ_CODE_SIZE, + ENCODER_PARAMETER_PQ_CODE_SIZE_DEFAULT, + v -> v > 0 && v < ENCODER_PARAMETER_PQ_CODE_SIZE_LIMIT + ) + ) + .setRequiresTraining(true) + .setMapGenerator( + ((methodComponent, methodComponentContext) -> MethodAsMapBuilder.builder( + FAISS_PQ_DESCRIPTION, + methodComponent, + methodComponentContext + ).addParameter(ENCODER_PARAMETER_PQ_M, "", "").addParameter(ENCODER_PARAMETER_PQ_CODE_SIZE, "x", "").build()) + ) + .setOverheadInKBEstimator((methodComponent, methodComponentContext, dimension) -> { + // Size estimate formula: (4 * d * 2^code_size) / 1024 + 1 - if (!(codeSizeObject instanceof Integer)) { - throw new IllegalStateException(String.format("%s must be an integer.", ENCODER_PARAMETER_PQ_CODE_SIZE)); - } + // Get value of code size passed in by user + Object codeSizeObject = methodComponentContext.getParameters().get(ENCODER_PARAMETER_PQ_CODE_SIZE); - int codeSize = (Integer) codeSizeObject; - return ((4L * (1L << codeSize) * dimension) / BYTES_PER_KILOBYTES) + 1; - }) - .build() - ); + // If not specified, get default value of code size + if (codeSizeObject == null) { + Parameter codeSizeParameter = methodComponent.getParameters().get(ENCODER_PARAMETER_PQ_CODE_SIZE); + if (codeSizeParameter == null) { + throw new IllegalStateException( + String.format("%s is not a valid parameter. This is a bug.", ENCODER_PARAMETER_PQ_CODE_SIZE) + ); + } + + codeSizeObject = codeSizeParameter.getDefaultValue(); + } + + if (!(codeSizeObject instanceof Integer)) { + throw new IllegalStateException(String.format("%s must be an integer.", ENCODER_PARAMETER_PQ_CODE_SIZE)); + } + + int codeSize = (Integer) codeSizeObject; + return ((4L * (1L << codeSize) * dimension) / BYTES_PER_KILOBYTES) + 1; + }) + .build() + ) + ) + .putAll(COMMON_ENCODERS) + .build(); - // Define methods supported by faiss private final static Map METHODS = ImmutableMap.of( METHOD_HNSW, KNNMethod.Builder.builder( @@ -158,7 +206,7 @@ class Faiss extends NativeLibrary { ) .addParameter( METHOD_ENCODER_PARAMETER, - new Parameter.MethodComponentContextParameter(METHOD_ENCODER_PARAMETER, ENCODER_DEFAULT, encoderComponents) + new Parameter.MethodComponentContextParameter(METHOD_ENCODER_PARAMETER, ENCODER_DEFAULT, HNSW_ENCODERS) ) .setMapGenerator( ((methodComponent, methodComponentContext) -> MethodAsMapBuilder.builder( @@ -190,7 +238,7 @@ class Faiss extends NativeLibrary { ) .addParameter( METHOD_ENCODER_PARAMETER, - new Parameter.MethodComponentContextParameter(METHOD_ENCODER_PARAMETER, ENCODER_DEFAULT, encoderComponents) + new Parameter.MethodComponentContextParameter(METHOD_ENCODER_PARAMETER, ENCODER_DEFAULT, IVF_ENCODERS) ) .setRequiresTraining(true) .setMapGenerator( diff --git a/src/test/java/org/opensearch/knn/index/FaissIT.java b/src/test/java/org/opensearch/knn/index/FaissIT.java index cfda93113..842c5af7b 100644 --- a/src/test/java/org/opensearch/knn/index/FaissIT.java +++ b/src/test/java/org/opensearch/knn/index/FaissIT.java @@ -37,7 +37,12 @@ import java.util.TreeMap; import java.util.stream.Collectors; +import static org.opensearch.knn.common.KNNConstants.ENCODER_PARAMETER_PQ_M; +import static org.opensearch.knn.common.KNNConstants.ENCODER_PQ; +import static org.opensearch.knn.common.KNNConstants.FAISS_NAME; import static org.opensearch.knn.common.KNNConstants.KNN_ENGINE; +import static org.opensearch.knn.common.KNNConstants.METHOD_ENCODER_PARAMETER; +import static org.opensearch.knn.common.KNNConstants.METHOD_HNSW; import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_NLIST; import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_SPACE_TYPE; import static org.opensearch.knn.common.KNNConstants.MODEL_ID; @@ -62,7 +67,8 @@ public static void setUpClass() throws IOException { testData = new TestUtils.TestData(testIndexVectors.getPath(), testQueries.getPath()); } - public void testEndToEnd_fromMethod() throws Exception { + @SneakyThrows + public void testEndToEnd_whenMethodIsHNSWFlat_thenSucceed() { String indexName = "test-index-1"; String fieldName = "test-field-1"; @@ -150,6 +156,117 @@ public void testEndToEnd_fromMethod() throws Exception { fail("Graphs are not getting evicted"); } + @SneakyThrows + public void testEndToEnd_whenMethodIsHNSWPQ_thenSucceed() { + String indexName = "test-index"; + String fieldName = "test-field"; + String trainingIndexName = "training-index"; + String trainingFieldName = "training-field"; + + String modelId = "test-model"; + String modelDescription = "test model"; + + List mValues = ImmutableList.of(16, 32, 64, 128); + List efConstructionValues = ImmutableList.of(16, 32, 64, 128); + List efSearchValues = ImmutableList.of(16, 32, 64, 128); + List pqMValues = ImmutableList.of(2, 4, 8); + + // training data needs to be at least equal to the number of centroids for PQ + // which is 2^8 = 256. 8 because thats the only valid code_size for HNSWPQ + int trainingDataCount = 256; + + SpaceType spaceType = SpaceType.L2; + + Integer dimension = testData.indexData.vectors[0].length; + + XContentBuilder xContentBuilder = XContentFactory.jsonBuilder() + .startObject() + .field(NAME, METHOD_HNSW) + .field(KNN_ENGINE, FAISS_NAME) + .startObject(PARAMETERS) + .field(KNNConstants.METHOD_PARAMETER_M, mValues.get(random().nextInt(mValues.size()))) + .field(KNNConstants.METHOD_PARAMETER_EF_CONSTRUCTION, efConstructionValues.get(random().nextInt(efConstructionValues.size()))) + .field(KNNConstants.METHOD_PARAMETER_EF_SEARCH, efSearchValues.get(random().nextInt(efSearchValues.size()))) + .startObject(METHOD_ENCODER_PARAMETER) + .field(NAME, ENCODER_PQ) + .startObject(PARAMETERS) + .field(ENCODER_PARAMETER_PQ_M, pqMValues.get(random().nextInt(pqMValues.size()))) + .endObject() + .endObject() + .endObject() + .endObject(); + Map in = xContentBuilderToMap(xContentBuilder); + + createBasicKnnIndex(trainingIndexName, trainingFieldName, dimension); + ingestDataAndTrainModel(modelId, trainingIndexName, trainingFieldName, dimension, modelDescription, in, trainingDataCount); + assertTrainingSucceeds(modelId, 180, 1000); + + // Create an index + XContentBuilder builder = XContentFactory.jsonBuilder() + .startObject() + .startObject("properties") + .startObject(fieldName) + .field("type", "knn_vector") + .field("model_id", modelId) + .endObject() + .endObject() + .endObject(); + + Map mappingMap = xContentBuilderToMap(builder); + String mapping = builder.toString(); + + createKnnIndex(indexName, mapping); + assertEquals(new TreeMap<>(mappingMap), new TreeMap<>(getIndexMappingAsMap(indexName))); + + // Index the test data + for (int i = 0; i < testData.indexData.docs.length; i++) { + addKnnDoc( + indexName, + Integer.toString(testData.indexData.docs[i]), + fieldName, + Floats.asList(testData.indexData.vectors[i]).toArray() + ); + } + + // Assert we have the right number of documents in the index + refreshAllNonSystemIndices(); + assertEquals(testData.indexData.docs.length, getDocCount(indexName)); + + int k = 10; + for (int i = 0; i < testData.queries.length; i++) { + Response response = searchKNNIndex(indexName, new KNNQueryBuilder(fieldName, testData.queries[i], k), k); + String responseBody = EntityUtils.toString(response.getEntity()); + List knnResults = parseSearchResponse(responseBody, fieldName); + assertEquals(k, knnResults.size()); + + List actualScores = parseSearchResponseScore(responseBody, fieldName); + for (int j = 0; j < k; j++) { + float[] primitiveArray = Floats.toArray(Arrays.stream(knnResults.get(j).getVector()).collect(Collectors.toList())); + assertEquals( + KNNEngine.FAISS.score(KNNScoringUtil.l2Squared(testData.queries[i], primitiveArray), spaceType), + actualScores.get(j), + 0.0001 + ); + } + } + + // Delete index + deleteKNNIndex(indexName); + deleteModel(modelId); + + // Search every 5 seconds 14 times to confirm graph gets evicted + int intervals = 14; + for (int i = 0; i < intervals; i++) { + if (getTotalGraphsInCache() == 0) { + return; + } + + Thread.sleep(5 * 1000); + } + + fail("Graphs are not getting evicted"); + } + public void testDocUpdate() throws IOException { String indexName = "test-index-1"; String fieldName = "test-field-1"; diff --git a/src/test/java/org/opensearch/knn/index/KNNMethodContextTests.java b/src/test/java/org/opensearch/knn/index/KNNMethodContextTests.java index a3011cef5..0b12980ab 100644 --- a/src/test/java/org/opensearch/knn/index/KNNMethodContextTests.java +++ b/src/test/java/org/opensearch/knn/index/KNNMethodContextTests.java @@ -26,6 +26,7 @@ import static org.opensearch.knn.common.KNNConstants.BYTES_PER_KILOBYTES; import static org.opensearch.knn.common.KNNConstants.ENCODER_PARAMETER_PQ_CODE_SIZE; +import static org.opensearch.knn.common.KNNConstants.ENCODER_PARAMETER_PQ_CODE_SIZE_DEFAULT; import static org.opensearch.knn.common.KNNConstants.ENCODER_PQ; import static org.opensearch.knn.common.KNNConstants.KNN_ENGINE; import static org.opensearch.knn.common.KNNConstants.METHOD_ENCODER_PARAMETER; @@ -140,22 +141,54 @@ public void testRequiresTraining() { assertTrue(knnMethodContext.isTrainingRequired()); } - public void testEstimateOverheadInKB() { + public void testEstimateOverheadInKB_whenMethodIsHNSWFlatNmslib_thenSizeIsExpectedValue() { // For HNSW no encoding we expect 0 MethodComponentContext hnswMethod = new MethodComponentContext(METHOD_HNSW, Collections.emptyMap()); - KNNMethodContext knnMethodContextNmslib = new KNNMethodContext(KNNEngine.NMSLIB, SpaceType.L2, hnswMethod); - KNNMethodContext knnMethodContextFaiss = new KNNMethodContext(KNNEngine.FAISS, SpaceType.INNER_PRODUCT, hnswMethod); - assertEquals(0, knnMethodContextNmslib.estimateOverheadInKB(1000)); - assertEquals(0, knnMethodContextFaiss.estimateOverheadInKB(168)); + KNNMethodContext knnMethodContext = new KNNMethodContext(KNNEngine.NMSLIB, SpaceType.L2, hnswMethod); + assertEquals(0, knnMethodContext.estimateOverheadInKB(1000)); + + } + + public void testEstimateOverheadInKB_whenMethodIsHNSWFlatFaiss_thenSizeIsExpectedValue() { + // For HNSW no encoding we expect 0 + MethodComponentContext hnswMethod = new MethodComponentContext(METHOD_HNSW, Collections.emptyMap()); + KNNMethodContext knnMethodContext = new KNNMethodContext(KNNEngine.FAISS, SpaceType.INNER_PRODUCT, hnswMethod); + assertEquals(0, knnMethodContext.estimateOverheadInKB(168)); + + } + + public void testEstimateOverheadInKB_whenMethodIsHNSWPQFaiss_thenSizeIsExpectedValue() { + int dimension = 768; + int codeSize = ENCODER_PARAMETER_PQ_CODE_SIZE_DEFAULT; + + // For HNSWPQ, we expect 4 * d * 2^code_size / 1024 + 1 + int expectedHnswPq = 4 * dimension * (1 << codeSize) / BYTES_PER_KILOBYTES + 1; + + MethodComponentContext pqMethodContext = new MethodComponentContext(ENCODER_PQ, ImmutableMap.of()); + + MethodComponentContext hnswMethodPq = new MethodComponentContext( + METHOD_HNSW, + ImmutableMap.of(METHOD_ENCODER_PARAMETER, pqMethodContext) + ); + KNNMethodContext knnMethodContext = new KNNMethodContext(KNNEngine.FAISS, SpaceType.L2, hnswMethodPq); + assertEquals(expectedHnswPq, knnMethodContext.estimateOverheadInKB(dimension)); + } + public void testEstimateOverheadInKB_whenMethodIsIVFFlatFaiss_thenSizeIsExpectedValue() { // For IVF, we expect 4 * nlist * d / 1024 + 1 int dimension = 768; int nlists = 1024; int expectedIvf = 4 * nlists * dimension / BYTES_PER_KILOBYTES + 1; MethodComponentContext ivfMethod = new MethodComponentContext(METHOD_IVF, ImmutableMap.of(METHOD_PARAMETER_NLIST, nlists)); - knnMethodContextFaiss = new KNNMethodContext(KNNEngine.FAISS, SpaceType.L2, ivfMethod); - assertEquals(expectedIvf, knnMethodContextFaiss.estimateOverheadInKB(dimension)); + KNNMethodContext knnMethodContext = new KNNMethodContext(KNNEngine.FAISS, SpaceType.L2, ivfMethod); + assertEquals(expectedIvf, knnMethodContext.estimateOverheadInKB(dimension)); + } + + public void testEstimateOverheadInKB_whenMethodIsIVFPQFaiss_thenSizeIsExpectedValue() { + int dimension = 768; + int nlists = 1024; + int expectedIvf = 4 * nlists * dimension / BYTES_PER_KILOBYTES + 1; // For IVFPQ twe expect 4 * nlist * d / 1024 + 1 + 4 * d * 2^code_size / 1024 + 1 int codeSize = 16; @@ -171,17 +204,8 @@ public void testEstimateOverheadInKB() { METHOD_IVF, ImmutableMap.of(METHOD_PARAMETER_NLIST, nlists, METHOD_ENCODER_PARAMETER, pqMethodContext) ); - knnMethodContextFaiss = new KNNMethodContext(KNNEngine.FAISS, SpaceType.L2, ivfMethodPq); - assertEquals(expectedIvfPq, knnMethodContextFaiss.estimateOverheadInKB(dimension)); - - // For HNSWPQ, we expect 4 * d * 2^code_size / 1024 + 1 - int expectedHnswPq = expectedFromPq; - MethodComponentContext hnswMethodPq = new MethodComponentContext( - METHOD_HNSW, - ImmutableMap.of(METHOD_ENCODER_PARAMETER, pqMethodContext) - ); - knnMethodContextFaiss = new KNNMethodContext(KNNEngine.FAISS, SpaceType.L2, hnswMethodPq); - assertEquals(expectedHnswPq, knnMethodContextFaiss.estimateOverheadInKB(dimension)); + KNNMethodContext knnMethodContext = new KNNMethodContext(KNNEngine.FAISS, SpaceType.L2, ivfMethodPq); + assertEquals(expectedIvfPq, knnMethodContext.estimateOverheadInKB(dimension)); } /** diff --git a/src/test/java/org/opensearch/knn/index/util/FaissTests.java b/src/test/java/org/opensearch/knn/index/util/FaissTests.java index 01841363d..35f06027c 100644 --- a/src/test/java/org/opensearch/knn/index/util/FaissTests.java +++ b/src/test/java/org/opensearch/knn/index/util/FaissTests.java @@ -8,6 +8,7 @@ import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.knn.KNNTestCase; +import org.opensearch.knn.index.KNNMethodContext; import org.opensearch.knn.index.MethodComponent; import org.opensearch.knn.index.MethodComponentContext; import org.opensearch.knn.index.Parameter; @@ -16,12 +17,122 @@ import java.util.HashMap; import java.util.Map; +import static org.opensearch.knn.common.KNNConstants.ENCODER_PARAMETER_PQ_CODE_SIZE; +import static org.opensearch.knn.common.KNNConstants.ENCODER_PARAMETER_PQ_M; +import static org.opensearch.knn.common.KNNConstants.ENCODER_PQ; +import static org.opensearch.knn.common.KNNConstants.FAISS_NAME; import static org.opensearch.knn.common.KNNConstants.INDEX_DESCRIPTION_PARAMETER; +import static org.opensearch.knn.common.KNNConstants.KNN_ENGINE; +import static org.opensearch.knn.common.KNNConstants.METHOD_ENCODER_PARAMETER; +import static org.opensearch.knn.common.KNNConstants.METHOD_HNSW; +import static org.opensearch.knn.common.KNNConstants.METHOD_IVF; +import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_M; +import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_NLIST; import static org.opensearch.knn.common.KNNConstants.NAME; import static org.opensearch.knn.common.KNNConstants.PARAMETERS; public class FaissTests extends KNNTestCase { + public void testGetMethodAsMap_whenMethodIsHNSWFlat_thenCreateCorrectIndexDescription() throws IOException { + int mParam = 65; + String expectedIndexDescription = String.format("HNSW%d,Flat", mParam); + + XContentBuilder xContentBuilder = XContentFactory.jsonBuilder() + .startObject() + .field(NAME, METHOD_HNSW) + .field(KNN_ENGINE, FAISS_NAME) + .startObject(PARAMETERS) + .field(METHOD_PARAMETER_M, mParam) + .endObject() + .endObject(); + Map in = xContentBuilderToMap(xContentBuilder); + KNNMethodContext knnMethodContext = KNNMethodContext.parse(in); + + Map map = Faiss.INSTANCE.getMethodAsMap(knnMethodContext); + + assertTrue(map.containsKey(INDEX_DESCRIPTION_PARAMETER)); + assertEquals(expectedIndexDescription, map.get(INDEX_DESCRIPTION_PARAMETER)); + } + + public void testGetMethodAsMap_whenMethodIsHNSWPQ_thenCreateCorrectIndexDescription() throws IOException { + int hnswMParam = 65; + int pqMParam = 17; + String expectedIndexDescription = String.format("HNSW%d,PQ%d", hnswMParam, pqMParam); + + XContentBuilder xContentBuilder = XContentFactory.jsonBuilder() + .startObject() + .field(NAME, METHOD_HNSW) + .field(KNN_ENGINE, FAISS_NAME) + .startObject(PARAMETERS) + .field(METHOD_PARAMETER_M, hnswMParam) + .startObject(METHOD_ENCODER_PARAMETER) + .field(NAME, ENCODER_PQ) + .startObject(PARAMETERS) + .field(ENCODER_PARAMETER_PQ_M, pqMParam) + .endObject() + .endObject() + .endObject() + .endObject(); + Map in = xContentBuilderToMap(xContentBuilder); + KNNMethodContext knnMethodContext = KNNMethodContext.parse(in); + + Map map = Faiss.INSTANCE.getMethodAsMap(knnMethodContext); + + assertTrue(map.containsKey(INDEX_DESCRIPTION_PARAMETER)); + assertEquals(expectedIndexDescription, map.get(INDEX_DESCRIPTION_PARAMETER)); + } + + public void testGetMethodAsMap_whenMethodIsIVFFlat_thenCreateCorrectIndexDescription() throws IOException { + int nlists = 88; + String expectedIndexDescription = String.format("IVF%d,Flat", nlists); + + XContentBuilder xContentBuilder = XContentFactory.jsonBuilder() + .startObject() + .field(NAME, METHOD_IVF) + .field(KNN_ENGINE, FAISS_NAME) + .startObject(PARAMETERS) + .field(METHOD_PARAMETER_NLIST, nlists) + .endObject() + .endObject(); + Map in = xContentBuilderToMap(xContentBuilder); + KNNMethodContext knnMethodContext = KNNMethodContext.parse(in); + + Map map = Faiss.INSTANCE.getMethodAsMap(knnMethodContext); + + assertTrue(map.containsKey(INDEX_DESCRIPTION_PARAMETER)); + assertEquals(expectedIndexDescription, map.get(INDEX_DESCRIPTION_PARAMETER)); + } + + public void testGetMethodAsMap_whenMethodIsIVFPQ_thenCreateCorrectIndexDescription() throws IOException { + int ivfNlistsParam = 88; + int pqMParam = 17; + int pqCodeSizeParam = 53; + String expectedIndexDescription = String.format("IVF%d,PQ%dx%d", ivfNlistsParam, pqMParam, pqCodeSizeParam); + + XContentBuilder xContentBuilder = XContentFactory.jsonBuilder() + .startObject() + .field(NAME, METHOD_IVF) + .field(KNN_ENGINE, FAISS_NAME) + .startObject(PARAMETERS) + .field(METHOD_PARAMETER_NLIST, ivfNlistsParam) + .startObject(METHOD_ENCODER_PARAMETER) + .field(NAME, ENCODER_PQ) + .startObject(PARAMETERS) + .field(ENCODER_PARAMETER_PQ_M, pqMParam) + .field(ENCODER_PARAMETER_PQ_CODE_SIZE, pqCodeSizeParam) + .endObject() + .endObject() + .endObject() + .endObject(); + Map in = xContentBuilderToMap(xContentBuilder); + KNNMethodContext knnMethodContext = KNNMethodContext.parse(in); + + Map map = Faiss.INSTANCE.getMethodAsMap(knnMethodContext); + + assertTrue(map.containsKey(INDEX_DESCRIPTION_PARAMETER)); + assertEquals(expectedIndexDescription, map.get(INDEX_DESCRIPTION_PARAMETER)); + } + public void testMethodAsMapBuilder() throws IOException { String methodName = "test-method"; String methodDescription = "test-description"; diff --git a/src/test/java/org/opensearch/knn/jni/JNIServiceTests.java b/src/test/java/org/opensearch/knn/jni/JNIServiceTests.java index 6d52e5544..d1a5be741 100644 --- a/src/test/java/org/opensearch/knn/jni/JNIServiceTests.java +++ b/src/test/java/org/opensearch/knn/jni/JNIServiceTests.java @@ -14,6 +14,8 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import org.junit.BeforeClass; +import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.knn.KNNTestCase; import org.opensearch.knn.TestUtils; import org.opensearch.knn.common.KNNConstants; @@ -36,9 +38,13 @@ import static org.opensearch.knn.common.KNNConstants.FAISS_NAME; import static org.opensearch.knn.common.KNNConstants.INDEX_DESCRIPTION_PARAMETER; import static org.opensearch.knn.common.KNNConstants.INDEX_THREAD_QTY; +import static org.opensearch.knn.common.KNNConstants.KNN_ENGINE; import static org.opensearch.knn.common.KNNConstants.METHOD_ENCODER_PARAMETER; +import static org.opensearch.knn.common.KNNConstants.METHOD_HNSW; import static org.opensearch.knn.common.KNNConstants.METHOD_IVF; import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_NLIST; +import static org.opensearch.knn.common.KNNConstants.NAME; +import static org.opensearch.knn.common.KNNConstants.PARAMETERS; public class JNIServiceTests extends KNNTestCase { @@ -765,28 +771,94 @@ public void testTransferVectors() { JNIService.freeVectors(trainPointer1); } - public void testTrain() { + public void testTrain_whenConfigurationIsIVFFlat_thenSucceed() throws IOException { + long trainPointer = transferVectors(10); + int ivfNlistParam = 16; + XContentBuilder xContentBuilder = XContentFactory.jsonBuilder() + .startObject() + .field(NAME, METHOD_IVF) + .field(KNN_ENGINE, FAISS_NAME) + .startObject(PARAMETERS) + .field(METHOD_PARAMETER_NLIST, ivfNlistParam) + .endObject() + .endObject(); + Map in = xContentBuilderToMap(xContentBuilder); + KNNMethodContext knnMethodContext = KNNMethodContext.parse(in); + Map parameters = KNNEngine.FAISS.getMethodAsMap(knnMethodContext); + byte[] faissIndex = JNIService.trainIndex(parameters, 128, trainPointer, FAISS_NAME); + + assertNotEquals(0, faissIndex.length); + JNIService.freeVectors(trainPointer); + } + + public void testTrain_whenConfigurationIsIVFPQ_thenSucceed() throws IOException { + long trainPointer = transferVectors(10); + int ivfNlistParam = 16; + int pqMParam = 4; + int pqCodeSizeParam = 4; + XContentBuilder xContentBuilder = XContentFactory.jsonBuilder() + .startObject() + .field(NAME, METHOD_IVF) + .field(KNN_ENGINE, FAISS_NAME) + .startObject(PARAMETERS) + .field(METHOD_PARAMETER_NLIST, ivfNlistParam) + .startObject(METHOD_ENCODER_PARAMETER) + .field(NAME, ENCODER_PQ) + .startObject(PARAMETERS) + .field(ENCODER_PARAMETER_PQ_M, pqMParam) + .field(ENCODER_PARAMETER_PQ_CODE_SIZE, pqCodeSizeParam) + .endObject() + .endObject() + .endObject() + .endObject(); + Map in = xContentBuilderToMap(xContentBuilder); + KNNMethodContext knnMethodContext = KNNMethodContext.parse(in); + Map parameters = KNNEngine.FAISS.getMethodAsMap(knnMethodContext); + + byte[] faissIndex = JNIService.trainIndex(parameters, 128, trainPointer, FAISS_NAME); + + assertNotEquals(0, faissIndex.length); + JNIService.freeVectors(trainPointer); + } + + public void testTrain_whenConfigurationIsHNSWPQ_thenSucceed() throws IOException { + long trainPointer = transferVectors(10); + int pqMParam = 4; + XContentBuilder xContentBuilder = XContentFactory.jsonBuilder() + .startObject() + .field(NAME, METHOD_HNSW) + .field(KNN_ENGINE, FAISS_NAME) + .startObject(PARAMETERS) + .startObject(METHOD_ENCODER_PARAMETER) + .field(NAME, ENCODER_PQ) + .startObject(PARAMETERS) + .field(ENCODER_PARAMETER_PQ_M, pqMParam) + .endObject() + .endObject() + .endObject() + .endObject(); + Map in = xContentBuilderToMap(xContentBuilder); + KNNMethodContext knnMethodContext = KNNMethodContext.parse(in); + Map parameters = KNNEngine.FAISS.getMethodAsMap(knnMethodContext); + + byte[] faissIndex = JNIService.trainIndex(parameters, 128, trainPointer, FAISS_NAME); + + assertNotEquals(0, faissIndex.length); + JNIService.freeVectors(trainPointer); + } + + private long transferVectors(int numDuplicates) { long trainPointer1 = JNIService.transferVectors(0, testData.indexData.vectors); assertNotEquals(0, trainPointer1); long trainPointer2; - for (int i = 0; i < 10; i++) { + for (int i = 0; i < numDuplicates; i++) { trainPointer2 = JNIService.transferVectors(trainPointer1, testData.indexData.vectors); assertEquals(trainPointer1, trainPointer2); } - Map parameters = ImmutableMap.of( - INDEX_DESCRIPTION_PARAMETER, - "IVF16,PQ4", - KNNConstants.SPACE_TYPE, - SpaceType.L2.getValue() - ); - - byte[] faissIndex = JNIService.trainIndex(parameters, 128, trainPointer1, FAISS_NAME); - - assertNotEquals(0, faissIndex.length); - JNIService.freeVectors(trainPointer1); + return trainPointer1; } public void testCreateIndexFromTemplate() throws IOException { diff --git a/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java b/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java index ecfe3e34b..1c754a5c7 100644 --- a/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java +++ b/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java @@ -1126,6 +1126,24 @@ public Response getModel(String modelId, List filters) throws IOExceptio return client().performRequest(request); } + /** + * Delete the model + * + * @param modelId Id of model to be retrieved + * @throws IOException if request cannot be performed + */ + public void deleteModel(String modelId) throws IOException { + if (modelId == null) { + modelId = ""; + } else { + modelId = "/" + modelId; + } + + Request request = new Request("DELETE", "/_plugins/_knn/models" + modelId); + Response response = client().performRequest(request); + assertEquals(request.getEndpoint() + ": failed", RestStatus.OK, RestStatus.fromCode(response.getStatusLine().getStatusCode())); + } + public void assertTrainingSucceeds(String modelId, int attempts, int delayInMillis) throws InterruptedException, IOException { int attemptNum = 0; Response response; @@ -1224,6 +1242,18 @@ protected void ingestDataAndTrainModel( Map method ) throws Exception { int trainingDataCount = 40; + ingestDataAndTrainModel(modelId, trainingIndexName, trainingFieldName, dimension, modelDescription, method, trainingDataCount); + } + + protected void ingestDataAndTrainModel( + String modelId, + String trainingIndexName, + String trainingFieldName, + int dimension, + String modelDescription, + Map method, + int trainingDataCount + ) throws Exception { bulkIngestRandomVectors(trainingIndexName, trainingFieldName, trainingDataCount, dimension); Response trainResponse = trainModel(modelId, trainingIndexName, trainingFieldName, dimension, method, modelDescription); From d8c5c4708f3f35cd9fc62a24a1b8197da9b8832f Mon Sep 17 00:00:00 2001 From: John Mazanec Date: Thu, 31 Aug 2023 11:51:53 -0700 Subject: [PATCH 139/416] Fix locale in FaissTests (#1082) Signed-off-by: John Mazanec --- .../java/org/opensearch/knn/index/util/FaissTests.java | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/test/java/org/opensearch/knn/index/util/FaissTests.java b/src/test/java/org/opensearch/knn/index/util/FaissTests.java index 35f06027c..99a153780 100644 --- a/src/test/java/org/opensearch/knn/index/util/FaissTests.java +++ b/src/test/java/org/opensearch/knn/index/util/FaissTests.java @@ -15,6 +15,7 @@ import java.io.IOException; import java.util.HashMap; +import java.util.Locale; import java.util.Map; import static org.opensearch.knn.common.KNNConstants.ENCODER_PARAMETER_PQ_CODE_SIZE; @@ -35,7 +36,7 @@ public class FaissTests extends KNNTestCase { public void testGetMethodAsMap_whenMethodIsHNSWFlat_thenCreateCorrectIndexDescription() throws IOException { int mParam = 65; - String expectedIndexDescription = String.format("HNSW%d,Flat", mParam); + String expectedIndexDescription = String.format(Locale.ROOT, "HNSW%d,Flat", mParam); XContentBuilder xContentBuilder = XContentFactory.jsonBuilder() .startObject() @@ -57,7 +58,7 @@ public void testGetMethodAsMap_whenMethodIsHNSWFlat_thenCreateCorrectIndexDescri public void testGetMethodAsMap_whenMethodIsHNSWPQ_thenCreateCorrectIndexDescription() throws IOException { int hnswMParam = 65; int pqMParam = 17; - String expectedIndexDescription = String.format("HNSW%d,PQ%d", hnswMParam, pqMParam); + String expectedIndexDescription = String.format(Locale.ROOT, "HNSW%d,PQ%d", hnswMParam, pqMParam); XContentBuilder xContentBuilder = XContentFactory.jsonBuilder() .startObject() @@ -84,7 +85,7 @@ public void testGetMethodAsMap_whenMethodIsHNSWPQ_thenCreateCorrectIndexDescript public void testGetMethodAsMap_whenMethodIsIVFFlat_thenCreateCorrectIndexDescription() throws IOException { int nlists = 88; - String expectedIndexDescription = String.format("IVF%d,Flat", nlists); + String expectedIndexDescription = String.format(Locale.ROOT, "IVF%d,Flat", nlists); XContentBuilder xContentBuilder = XContentFactory.jsonBuilder() .startObject() @@ -107,7 +108,7 @@ public void testGetMethodAsMap_whenMethodIsIVFPQ_thenCreateCorrectIndexDescripti int ivfNlistsParam = 88; int pqMParam = 17; int pqCodeSizeParam = 53; - String expectedIndexDescription = String.format("IVF%d,PQ%dx%d", ivfNlistsParam, pqMParam, pqCodeSizeParam); + String expectedIndexDescription = String.format(Locale.ROOT, "IVF%d,PQ%dx%d", ivfNlistsParam, pqMParam, pqCodeSizeParam); XContentBuilder xContentBuilder = XContentFactory.jsonBuilder() .startObject() From 0e8b48fe1aaeee5bf0cb495bd6a9eafb78c54e96 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Thu, 31 Aug 2023 14:21:55 -0700 Subject: [PATCH 140/416] Fixing the check for max distance computation happening to switch to exact search (#1083) (#1084) Signed-off-by: Navneet Verma (cherry picked from commit a8eda139e2a1bd0996aa3d02634d970333ef5648) Co-authored-by: Navneet Verma --- .../knn/index/query/KNNQueryFactory.java | 9 +--- .../opensearch/knn/index/query/KNNWeight.java | 9 +++- .../knn/index/query/KNNWeightTests.java | 51 +++++++++++++++++++ 3 files changed, 60 insertions(+), 9 deletions(-) diff --git a/src/main/java/org/opensearch/knn/index/query/KNNQueryFactory.java b/src/main/java/org/opensearch/knn/index/query/KNNQueryFactory.java index 65c15499d..b05098f28 100644 --- a/src/main/java/org/opensearch/knn/index/query/KNNQueryFactory.java +++ b/src/main/java/org/opensearch/knn/index/query/KNNQueryFactory.java @@ -79,14 +79,7 @@ public static Query create(CreateQueryRequest createQueryRequest) { if (KNNEngine.getEnginesThatCreateCustomSegmentFiles().contains(createQueryRequest.getKnnEngine())) { if (filterQuery != null && KNNEngine.getEnginesThatSupportsFilters().contains(createQueryRequest.getKnnEngine())) { - log.debug( - String.format( - "Creating custom k-NN query with filters for index: %s \"\", field: %s \"\", " + "k: %d", - indexName, - fieldName, - k - ) - ); + log.debug("Creating custom k-NN query with filters for index: {}, field: {} , k: {}", indexName, fieldName, k); return new KNNQuery(fieldName, vector, k, indexName, filterQuery); } log.debug(String.format("Creating custom k-NN query for index: %s \"\", field: %s \"\", k: %d", indexName, fieldName, k)); diff --git a/src/main/java/org/opensearch/knn/index/query/KNNWeight.java b/src/main/java/org/opensearch/knn/index/query/KNNWeight.java index 4778ba25d..d24075aa5 100644 --- a/src/main/java/org/opensearch/knn/index/query/KNNWeight.java +++ b/src/main/java/org/opensearch/knn/index/query/KNNWeight.java @@ -124,6 +124,13 @@ public Scorer scorer(LeafReaderContext context) throws IOException { return null; } if (canDoExactSearchAfterANNSearch(filterIdsArray.length, annResults.size())) { + log.debug( + "Doing ExactSearch after doing ANNSearch as the number of documents returned are less than " + + "K, even when we have more than K filtered Ids. K: {}, ANNResults: {}, filteredIdCount: {}", + knnQuery.getK(), + annResults.size(), + filterIdsArray.length + ); annResults = doExactSearch(context, filterIdsArray); } docIdsToScoreMap.putAll(annResults); @@ -390,7 +397,7 @@ private boolean canDoExactSearch(final int filterIdsCount, final int searchableD return filterThresholdValue >= filterIdsCount; } // if no setting is set, then use the default max distance computation value to see if we can do exact search. - return KNNConstants.MAX_DISTANCE_COMPUTATIONS <= filterIdsCount * knnQuery.getQueryVector().length; + return KNNConstants.MAX_DISTANCE_COMPUTATIONS >= filterIdsCount * knnQuery.getQueryVector().length; } /** diff --git a/src/test/java/org/opensearch/knn/index/query/KNNWeightTests.java b/src/test/java/org/opensearch/knn/index/query/KNNWeightTests.java index 3a5e984f4..cd0daa54b 100644 --- a/src/test/java/org/opensearch/knn/index/query/KNNWeightTests.java +++ b/src/test/java/org/opensearch/knn/index/query/KNNWeightTests.java @@ -457,6 +457,57 @@ public void testANNWithFilterQuery_whenExactSearch_thenSuccess() { assertTrue(Comparators.isInOrder(actualDocIds, Comparator.naturalOrder())); } + @SneakyThrows + public void testANNWithFilterQuery_whenExactSearchAndThresholdComputations_thenSuccess() { + knnSettingsMockedStatic.when(() -> KNNSettings.getFilteredExactSearchThreshold(INDEX_NAME)).thenReturn(-1); + float[] vector = new float[] { 0.1f, 0.3f }; + int filterDocId = 0; + final LeafReaderContext leafReaderContext = mock(LeafReaderContext.class); + final SegmentReader reader = mock(SegmentReader.class); + when(leafReaderContext.reader()).thenReturn(reader); + + final KNNQuery query = new KNNQuery(FIELD_NAME, QUERY_VECTOR, K, INDEX_NAME, FILTER_QUERY); + final Weight filterQueryWeight = mock(Weight.class); + final Scorer filterScorer = mock(Scorer.class); + when(filterQueryWeight.scorer(leafReaderContext)).thenReturn(filterScorer); + // scorer will return 2 documents + when(filterScorer.iterator()).thenReturn(DocIdSetIterator.all(1)); + when(reader.maxDoc()).thenReturn(1); + final Bits liveDocsBits = mock(Bits.class); + when(reader.getLiveDocs()).thenReturn(liveDocsBits); + when(liveDocsBits.get(filterDocId)).thenReturn(true); + when(liveDocsBits.length()).thenReturn(1000); + + final KNNWeight knnWeight = new KNNWeight(query, 0.0f, filterQueryWeight); + final Map attributesMap = ImmutableMap.of(KNN_ENGINE, KNNEngine.FAISS.getName(), SPACE_TYPE, SpaceType.L2.name()); + final FieldInfos fieldInfos = mock(FieldInfos.class); + final FieldInfo fieldInfo = mock(FieldInfo.class); + final BinaryDocValues binaryDocValues = mock(BinaryDocValues.class); + when(reader.getFieldInfos()).thenReturn(fieldInfos); + when(fieldInfos.fieldInfo(any())).thenReturn(fieldInfo); + when(fieldInfo.attributes()).thenReturn(attributesMap); + when(fieldInfo.getAttribute(SPACE_TYPE)).thenReturn(SpaceType.L2.name()); + when(fieldInfo.getName()).thenReturn(FIELD_NAME); + when(reader.getBinaryDocValues(FIELD_NAME)).thenReturn(binaryDocValues); + when(binaryDocValues.advance(filterDocId)).thenReturn(filterDocId); + BytesRef vectorByteRef = new BytesRef(new KNNVectorAsArraySerializer().floatToByteArray(vector)); + when(binaryDocValues.binaryValue()).thenReturn(vectorByteRef); + + final KNNScorer knnScorer = (KNNScorer) knnWeight.scorer(leafReaderContext); + assertNotNull(knnScorer); + final DocIdSetIterator docIdSetIterator = knnScorer.iterator(); + assertNotNull(docIdSetIterator); + assertEquals(EXACT_SEARCH_DOC_ID_TO_SCORES.size(), docIdSetIterator.cost()); + + final List actualDocIds = new ArrayList<>(); + for (int docId = docIdSetIterator.nextDoc(); docId != NO_MORE_DOCS; docId = docIdSetIterator.nextDoc()) { + actualDocIds.add(docId); + assertEquals(EXACT_SEARCH_DOC_ID_TO_SCORES.get(docId), knnScorer.score(), 0.01f); + } + assertEquals(docIdSetIterator.cost(), actualDocIds.size()); + assertTrue(Comparators.isInOrder(actualDocIds, Comparator.naturalOrder())); + } + /** * This test ensure that we do the exact search when threshold settings are correct and not using filteredIds<=K * condition to do exact search. From d1f230680955b4c56e3eed8626d00686de72cd47 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Thu, 31 Aug 2023 23:14:10 -0700 Subject: [PATCH 141/416] Add HNSW PQ configuration to perf tests (#1090) Signed-off-by: John Mazanec (cherry picked from commit 036d06065933f2a46cd6b8f06e98a912fac2102c) --- .../release-configs/faiss-hnswpq/index.json | 17 ++++++ .../faiss-hnswpq/method-spec.json | 15 ++++++ .../release-configs/faiss-hnswpq/test.yml | 53 +++++++++++++++++++ .../faiss-hnswpq/train-index-spec.json | 16 ++++++ 4 files changed, 101 insertions(+) create mode 100644 benchmarks/perf-tool/release-configs/faiss-hnswpq/index.json create mode 100644 benchmarks/perf-tool/release-configs/faiss-hnswpq/method-spec.json create mode 100644 benchmarks/perf-tool/release-configs/faiss-hnswpq/test.yml create mode 100644 benchmarks/perf-tool/release-configs/faiss-hnswpq/train-index-spec.json diff --git a/benchmarks/perf-tool/release-configs/faiss-hnswpq/index.json b/benchmarks/perf-tool/release-configs/faiss-hnswpq/index.json new file mode 100644 index 000000000..479703412 --- /dev/null +++ b/benchmarks/perf-tool/release-configs/faiss-hnswpq/index.json @@ -0,0 +1,17 @@ +{ + "settings": { + "index": { + "knn": true, + "number_of_shards": 24, + "number_of_replicas": 1 + } + }, + "mappings": { + "properties": { + "target_field": { + "type": "knn_vector", + "model_id": "test-model" + } + } + } +} diff --git a/benchmarks/perf-tool/release-configs/faiss-hnswpq/method-spec.json b/benchmarks/perf-tool/release-configs/faiss-hnswpq/method-spec.json new file mode 100644 index 000000000..2d67bf2df --- /dev/null +++ b/benchmarks/perf-tool/release-configs/faiss-hnswpq/method-spec.json @@ -0,0 +1,15 @@ +{ + "name":"hnsw", + "engine":"faiss", + "space_type": "l2", + "parameters":{ + "ef_construction": 256, + "m": 16, + "encoder": { + "name": "pq", + "parameters": { + "m": 16 + } + } + } +} diff --git a/benchmarks/perf-tool/release-configs/faiss-hnswpq/test.yml b/benchmarks/perf-tool/release-configs/faiss-hnswpq/test.yml new file mode 100644 index 000000000..dd88affc3 --- /dev/null +++ b/benchmarks/perf-tool/release-configs/faiss-hnswpq/test.yml @@ -0,0 +1,53 @@ +endpoint: [ENDPOINT] +test_name: "index-workflow" +test_id: "index workflow" +num_runs: 10 +show_runs: false +setup: + - name: delete_index + index_name: train_index + - name: create_index + index_name: train_index + index_spec: /home/ec2-user/[PATH]/train-index-spec.json + - name: ingest + index_name: train_index + field_name: train_field + bulk_size: 500 + dataset_format: hdf5 + dataset_path: /home/ec2-user/data/sift-128-euclidean.hdf5 + doc_count: 50000 + - name: refresh_index + index_name: train_index +steps: + - name: delete_model + model_id: test-model + - name: delete_index + index_name: target_index + - name: train_model + model_id: test-model + train_index: train_index + train_field: train_field + dimension: 128 + method_spec: /home/ec2-user/[PATH]/method-spec.json + max_training_vector_count: 50000 + - name: create_index + index_name: target_index + index_spec: /home/ec2-user/[PATH]/index.json + - name: ingest + index_name: target_index + field_name: target_field + bulk_size: 500 + dataset_format: hdf5 + dataset_path: /home/ec2-user/data/sift-128-euclidean.hdf5 + - name: refresh_index + index_name: target_index + - name: query + k: 100 + r: 1 + calculate_recall: true + index_name: target_index + field_name: target_field + dataset_format: hdf5 + dataset_path: /home/ec2-user/data/sift-128-euclidean.hdf5 + neighbors_format: hdf5 + neighbors_path: /home/ec2-user/data/sift-128-euclidean.hdf5 diff --git a/benchmarks/perf-tool/release-configs/faiss-hnswpq/train-index-spec.json b/benchmarks/perf-tool/release-configs/faiss-hnswpq/train-index-spec.json new file mode 100644 index 000000000..804a5707e --- /dev/null +++ b/benchmarks/perf-tool/release-configs/faiss-hnswpq/train-index-spec.json @@ -0,0 +1,16 @@ +{ + "settings": { + "index": { + "number_of_shards": 24, + "number_of_replicas": 0 + } + }, + "mappings": { + "properties": { + "train_field": { + "type": "knn_vector", + "dimension": 128 + } + } + } +} From 3f70116ce5f27065c9d0a3dd7bcdf8e3d61d17a1 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Wed, 6 Sep 2023 11:47:27 -0700 Subject: [PATCH 142/416] Created release notes for 2.10 release (#1094) (#1095) Signed-off-by: Navneet Verma (cherry picked from commit cc5cd7e31759ee50397e8522f84cbc6f5bfc7034) Co-authored-by: Navneet Verma --- CHANGELOG.md | 10 ++-------- .../opensearch-knn.release-notes-2.10.0.0.md | 16 ++++++++++++++++ 2 files changed, 18 insertions(+), 8 deletions(-) create mode 100644 release-notes/opensearch-knn.release-notes-2.10.0.0.md diff --git a/CHANGELOG.md b/CHANGELOG.md index a421e08e4..f0832aa6e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,17 +12,11 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Maintenance ### Refactoring -## [Unreleased 2.x](https://github.com/opensearch-project/k-NN/compare/2.9...2.x) +## [Unreleased 2.x](https://github.com/opensearch-project/k-NN/compare/2.10...2.x) ### Features ### Enhancements -* Enabled the IVF algorithm to work with Filters of K-NN Query. [#1013](https://github.com/opensearch-project/k-NN/pull/1013) -* Improved the logic to switch to exact search for restrictive filters search for better recall. [#1059](https://github.com/opensearch-project/k-NN/pull/1059) -* Added max distance computation logic to enhance the switch to exact search in filtered Nearest Neighbor Search. [#1066](https://github.com/opensearch-project/k-NN/pull/1066) -### Bug Fixes -* Update Faiss parameter construction to allow HNSW+PQ to work [#1074](https://github.com/opensearch-project/k-NN/pull/1074) +### Bug Fixes ### Infrastructure ### Documentation ### Maintenance -* Update Guava Version to 32.0.1 [#1019](https://github.com/opensearch-project/k-NN/pull/1019) ### Refactoring -* Fix TransportAddress Refactoring Changes in Core [#1020](https://github.com/opensearch-project/k-NN/pull/1020) diff --git a/release-notes/opensearch-knn.release-notes-2.10.0.0.md b/release-notes/opensearch-knn.release-notes-2.10.0.0.md new file mode 100644 index 000000000..5ac0004df --- /dev/null +++ b/release-notes/opensearch-knn.release-notes-2.10.0.0.md @@ -0,0 +1,16 @@ +## Version 2.10.0.0 Release Notes + +Compatible with OpenSearch 2.10.0 + +### Features +* Add Clear Cache API ([#740](https://github.com/opensearch-project/k-NN/pull/740)) +### Enhancements +* Enabled the IVF algorithm to work with Filters of K-NN Query. ([#1013](https://github.com/opensearch-project/k-NN/pull/1013)) +* Improved the logic to switch to exact search for restrictive filters search for better recall. ([#1059](https://github.com/opensearch-project/k-NN/pull/1059)) +* Added max distance computation logic to enhance the switch to exact search in filtered Nearest Neighbor Search. ([#1066](https://github.com/opensearch-project/k-NN/pull/1066)) +### Bug Fixes +* Update Faiss parameter construction to allow HNSW+PQ to work ([#1074](https://github.com/opensearch-project/k-NN/pull/1074)) +### Maintenance +* Update Guava Version to 32.0.1 ([#1019](https://github.com/opensearch-project/k-NN/pull/1019)) +### Refactoring +* Fix TransportAddress Refactoring Changes in Core ([#1020](https://github.com/opensearch-project/k-NN/pull/1020)) From 870cc2a8fec9e7976f4b0f49c7a5039e7afc2414 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Wed, 6 Sep 2023 16:41:31 -0700 Subject: [PATCH 143/416] Adding get-ci-image-tag dynamic retrival on CI workflows (#1099) (#1101) Signed-off-by: Peter Zhu (cherry picked from commit c051e550905ecf1217ad338ffdebeb84563df4aa) Co-authored-by: Peter Zhu --- .github/workflows/CI.yml | 31 +++++++++++++++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index d16287dd3..959a0a400 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -12,17 +12,42 @@ on: - "feature/**" jobs: - Build-k-NN: + Get-CI-Image-Tag: + runs-on: ubuntu-latest + outputs: + ci-image-version-linux: ${{ steps.step-ci-image-version-linux.outputs.ci-image-version-linux }} + steps: + - name: Install crane + uses: iarekylew00t/crane-installer@v1 + with: + crane-release: v0.15.2 + - name: Checkout opensearch-build repository + uses: actions/checkout@v2 + with: + repository: 'opensearch-project/opensearch-build' + ref: 'main' + path: 'opensearch-build' + - name: Get ci image version from opensearch-build repository scripts + id: step-ci-image-version-linux + run: | + crane version + CI_IMAGE_VERSION=`opensearch-build/docker/ci/get-ci-images.sh -p rockylinux8 -u opensearch -t build | head -1` + echo $CI_IMAGE_VERSION + echo "ci-image-version-linux=$CI_IMAGE_VERSION" >> $GITHUB_OUTPUT + + + Build-k-NN-Linux: strategy: matrix: java: [11, 17] name: Build and Test k-NN Plugin on Linux runs-on: ubuntu-latest + needs: Get-CI-Image-Tag container: # using the same image which is used by opensearch-build team to build the OpenSearch Distribution # this image tag is subject to change as more dependencies and updates will arrive over time - image: public.ecr.aws/opensearchstaging/ci-runner:ci-runner-rockylinux8-opensearch-build-v4 + image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }} # need to switch to root so that github actions can install runner binary on container without permission issues. options: --user root @@ -54,6 +79,7 @@ jobs: java: [ 11, 17 ] name: Build and Test k-NN Plugin on MacOS + needs: Get-CI-Image-Tag runs-on: macos-latest steps: @@ -80,6 +106,7 @@ jobs: java: [ 11, 17 ] name: Build and Test k-NN Plugin on Windows + needs: Get-CI-Image-Tag runs-on: windows-latest steps: From 1a16ee34e0d90f72a210e6065f1db62c4cfe7325 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Thu, 7 Sep 2023 13:09:52 -0700 Subject: [PATCH 144/416] Downgrading the version of request lib to fix the CVE-2015-2296 (#1105) (#1106) Signed-off-by: Navneet Verma (cherry picked from commit 47ccb1b085e9793844256cd1d243d9f40ad61f41) Co-authored-by: Navneet Verma --- benchmarks/perf-tool/okpt/test/steps/steps.py | 2 +- benchmarks/perf-tool/requirements.in | 2 +- benchmarks/perf-tool/requirements.txt | 20 +++++-------------- 3 files changed, 7 insertions(+), 17 deletions(-) diff --git a/benchmarks/perf-tool/okpt/test/steps/steps.py b/benchmarks/perf-tool/okpt/test/steps/steps.py index cc1773330..183a37cfa 100644 --- a/benchmarks/perf-tool/okpt/test/steps/steps.py +++ b/benchmarks/perf-tool/okpt/test/steps/steps.py @@ -796,7 +796,7 @@ def query_index(opensearch: OpenSearch, index_name: str, body: dict, def bulk_index(opensearch: OpenSearch, index_name: str, body: List): - return opensearch.bulk(index=index_name, body=body, timeout='5m') + return opensearch.bulk(index=index_name, body=body) def get_segment_stats(opensearch: OpenSearch, index_name: str): return opensearch.indices.segments(index=index_name) diff --git a/benchmarks/perf-tool/requirements.in b/benchmarks/perf-tool/requirements.in index fd3555aab..575ca566b 100644 --- a/benchmarks/perf-tool/requirements.in +++ b/benchmarks/perf-tool/requirements.in @@ -3,5 +3,5 @@ opensearch-py PyYAML numpy h5py -requests +requests>=2.1.0,<2.6.0 psutil diff --git a/benchmarks/perf-tool/requirements.txt b/benchmarks/perf-tool/requirements.txt index 5489e63dd..5981e6355 100644 --- a/benchmarks/perf-tool/requirements.txt +++ b/benchmarks/perf-tool/requirements.txt @@ -1,23 +1,15 @@ # -# This file is autogenerated by pip-compile with python 3.9 -# To update, run: +# This file is autogenerated by pip-compile with Python 3.8 +# by the following command: # # pip-compile # -cached-property==1.5.2 - # via h5py cerberus==1.3.4 # via -r requirements.in certifi==2023.7.22 - # via - # opensearch-py - # requests -charset-normalizer==2.0.4 - # via requests + # via opensearch-py h5py==3.3.0 # via -r requirements.in -idna==3.2 - # via requests numpy==1.24.2 # via # -r requirements.in @@ -28,12 +20,10 @@ psutil==5.8.0 # via -r requirements.in pyyaml==5.4.1 # via -r requirements.in -requests==2.31.0 +requests==2.5.3 # via -r requirements.in urllib3==1.26.6 - # via - # opensearch-py - # requests + # via opensearch-py # The following packages are considered to be unsafe in a requirements file: # setuptools From c83847510cc19238f019e0439befd588ae35c819 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Thu, 7 Sep 2023 17:55:49 -0700 Subject: [PATCH 145/416] Revert "Downgrading the version of request lib to fix the CVE-2015-2296 (#1105)" (#1110) (#1111) This reverts commit 47ccb1b085e9793844256cd1d243d9f40ad61f41. Signed-off-by: Navneet Verma (cherry picked from commit 7dbcbcd961d737eb42f8f6a4f1ec979d33b4e03d) Co-authored-by: Navneet Verma --- benchmarks/perf-tool/okpt/test/steps/steps.py | 2 +- benchmarks/perf-tool/requirements.in | 2 +- benchmarks/perf-tool/requirements.txt | 20 ++++++++++++++----- 3 files changed, 17 insertions(+), 7 deletions(-) diff --git a/benchmarks/perf-tool/okpt/test/steps/steps.py b/benchmarks/perf-tool/okpt/test/steps/steps.py index 183a37cfa..cc1773330 100644 --- a/benchmarks/perf-tool/okpt/test/steps/steps.py +++ b/benchmarks/perf-tool/okpt/test/steps/steps.py @@ -796,7 +796,7 @@ def query_index(opensearch: OpenSearch, index_name: str, body: dict, def bulk_index(opensearch: OpenSearch, index_name: str, body: List): - return opensearch.bulk(index=index_name, body=body) + return opensearch.bulk(index=index_name, body=body, timeout='5m') def get_segment_stats(opensearch: OpenSearch, index_name: str): return opensearch.indices.segments(index=index_name) diff --git a/benchmarks/perf-tool/requirements.in b/benchmarks/perf-tool/requirements.in index 575ca566b..fd3555aab 100644 --- a/benchmarks/perf-tool/requirements.in +++ b/benchmarks/perf-tool/requirements.in @@ -3,5 +3,5 @@ opensearch-py PyYAML numpy h5py -requests>=2.1.0,<2.6.0 +requests psutil diff --git a/benchmarks/perf-tool/requirements.txt b/benchmarks/perf-tool/requirements.txt index 5981e6355..5489e63dd 100644 --- a/benchmarks/perf-tool/requirements.txt +++ b/benchmarks/perf-tool/requirements.txt @@ -1,15 +1,23 @@ # -# This file is autogenerated by pip-compile with Python 3.8 -# by the following command: +# This file is autogenerated by pip-compile with python 3.9 +# To update, run: # # pip-compile # +cached-property==1.5.2 + # via h5py cerberus==1.3.4 # via -r requirements.in certifi==2023.7.22 - # via opensearch-py + # via + # opensearch-py + # requests +charset-normalizer==2.0.4 + # via requests h5py==3.3.0 # via -r requirements.in +idna==3.2 + # via requests numpy==1.24.2 # via # -r requirements.in @@ -20,10 +28,12 @@ psutil==5.8.0 # via -r requirements.in pyyaml==5.4.1 # via -r requirements.in -requests==2.5.3 +requests==2.31.0 # via -r requirements.in urllib3==1.26.6 - # via opensearch-py + # via + # opensearch-py + # requests # The following packages are considered to be unsafe in a requirements file: # setuptools From 47783cab4952d0db5fab4347cc4e6c1e3c80e842 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Thu, 7 Sep 2023 23:05:16 -0400 Subject: [PATCH 146/416] Updates demo certs used in integ tests (#1100) (#1113) Signed-off-by: Darshit Chanpura (cherry picked from commit 3173b8a1693cbb214a1d8d8b185501c54d589811) Co-authored-by: Darshit Chanpura <35282393+DarshitChanpura@users.noreply.github.com> --- src/test/resources/security/sample.pem | 51 ++++++++++------------ src/test/resources/security/test-kirk.jks | Bin 3874 -> 4504 bytes 2 files changed, 24 insertions(+), 27 deletions(-) diff --git a/src/test/resources/security/sample.pem b/src/test/resources/security/sample.pem index fa785ca10..a1fc20a77 100644 --- a/src/test/resources/security/sample.pem +++ b/src/test/resources/security/sample.pem @@ -1,28 +1,25 @@ -----BEGIN CERTIFICATE----- -MIIEyTCCA7GgAwIBAgIGAWLrc1O2MA0GCSqGSIb3DQEBCwUAMIGPMRMwEQYKCZIm -iZPyLGQBGRYDY29tMRcwFQYKCZImiZPyLGQBGRYHZXhhbXBsZTEZMBcGA1UECgwQ -RXhhbXBsZSBDb20gSW5jLjEhMB8GA1UECwwYRXhhbXBsZSBDb20gSW5jLiBSb290 -IENBMSEwHwYDVQQDDBhFeGFtcGxlIENvbSBJbmMuIFJvb3QgQ0EwHhcNMTgwNDIy -MDM0MzQ3WhcNMjgwNDE5MDM0MzQ3WjBeMRIwEAYKCZImiZPyLGQBGRYCZGUxDTAL -BgNVBAcMBHRlc3QxDTALBgNVBAoMBG5vZGUxDTALBgNVBAsMBG5vZGUxGzAZBgNV -BAMMEm5vZGUtMC5leGFtcGxlLmNvbTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCC -AQoCggEBAJa+f476vLB+AwK53biYByUwN+40D8jMIovGXm6wgT8+9Sbs899dDXgt -9CE1Beo65oP1+JUz4c7UHMrCY3ePiDt4cidHVzEQ2g0YoVrQWv0RedS/yx/DKhs8 -Pw1O715oftP53p/2ijD5DifFv1eKfkhFH+lwny/vMSNxellpl6NxJTiJVnQ9HYOL -gf2t971ITJHnAuuxUF48HcuNovW4rhtkXef8kaAN7cE3LU+A9T474ULNCKkEFPIl -ZAKN3iJNFdVsxrTU+CUBHzk73Do1cCkEvJZ0ZFjp0Z3y8wLY/gqWGfGVyA9l2CUq -eIZNf55PNPtGzOrvvONiui48vBKH1LsCAwEAAaOCAVkwggFVMIG8BgNVHSMEgbQw -gbGAFJI1DOAPHitF9k0583tfouYSl0BzoYGVpIGSMIGPMRMwEQYKCZImiZPyLGQB -GRYDY29tMRcwFQYKCZImiZPyLGQBGRYHZXhhbXBsZTEZMBcGA1UECgwQRXhhbXBs -ZSBDb20gSW5jLjEhMB8GA1UECwwYRXhhbXBsZSBDb20gSW5jLiBSb290IENBMSEw -HwYDVQQDDBhFeGFtcGxlIENvbSBJbmMuIFJvb3QgQ0GCAQEwHQYDVR0OBBYEFKyv -78ZmFjVKM9g7pMConYH7FVBHMAwGA1UdEwEB/wQCMAAwDgYDVR0PAQH/BAQDAgXg -MCAGA1UdJQEB/wQWMBQGCCsGAQUFBwMBBggrBgEFBQcDAjA1BgNVHREELjAsiAUq -AwQFBYISbm9kZS0wLmV4YW1wbGUuY29tgglsb2NhbGhvc3SHBH8AAAEwDQYJKoZI -hvcNAQELBQADggEBAIOKuyXsFfGv1hI/Lkpd/73QNqjqJdxQclX57GOMWNbOM5H0 -5/9AOIZ5JQsWULNKN77aHjLRr4owq2jGbpc/Z6kAd+eiatkcpnbtbGrhKpOtoEZy -8KuslwkeixpzLDNISSbkeLpXz4xJI1ETMN/VG8ZZP1bjzlHziHHDu0JNZ6TnNzKr -XzCGMCohFfem8vnKNnKUneMQMvXd3rzUaAgvtf7Hc2LTBlf4fZzZF1EkwdSXhaMA -1lkfHiqOBxtgeDLxCHESZ2fqgVqsWX+t3qHQfivcPW6txtDyrFPRdJOGhiMGzT/t -e/9kkAtQRgpTb3skYdIOOUOV0WGQ60kJlFhAzIs= ------END CERTIFICATE----- \ No newline at end of file +MIIEPDCCAySgAwIBAgIUZjrlDPP8azRDPZchA/XEsx0X2iIwDQYJKoZIhvcNAQEL +BQAwgY8xEzARBgoJkiaJk/IsZAEZFgNjb20xFzAVBgoJkiaJk/IsZAEZFgdleGFt +cGxlMRkwFwYDVQQKDBBFeGFtcGxlIENvbSBJbmMuMSEwHwYDVQQLDBhFeGFtcGxl +IENvbSBJbmMuIFJvb3QgQ0ExITAfBgNVBAMMGEV4YW1wbGUgQ29tIEluYy4gUm9v +dCBDQTAeFw0yMzA4MjkwNDIzMTJaFw0zMzA4MjYwNDIzMTJaMFcxCzAJBgNVBAYT +AmRlMQ0wCwYDVQQHDAR0ZXN0MQ0wCwYDVQQKDARub2RlMQ0wCwYDVQQLDARub2Rl +MRswGQYDVQQDDBJub2RlLTAuZXhhbXBsZS5jb20wggEiMA0GCSqGSIb3DQEBAQUA +A4IBDwAwggEKAoIBAQCm93kXteDQHMAvbUPNPW5pyRHKDD42XGWSgq0k1D29C/Ud +yL21HLzTJa49ZU2ldIkSKs9JqbkHdyK0o8MO6L8dotLoYbxDWbJFW8bp1w6tDTU0 +HGkn47XVu3EwbfrTENg3jFu+Oem6a/501SzITzJWtS0cn2dIFOBimTVpT/4Zv5qr +XA6Cp4biOmoTYWhi/qQl8d0IaADiqoZ1MvZbZ6x76qTrRAbg+UWkpTEXoH1xTc8n +dibR7+HP6OTqCKvo1NhE8uP4pY+fWd6b6l+KLo3IKpfTbAIJXIO+M67FLtWKtttD +ao94B069skzKk6FPgW/OZh6PRCD0oxOavV+ld2SjAgMBAAGjgcYwgcMwRwYDVR0R +BEAwPogFKgMEBQWCEm5vZGUtMC5leGFtcGxlLmNvbYIJbG9jYWxob3N0hxAAAAAA +AAAAAAAAAAAAAAABhwR/AAABMAsGA1UdDwQEAwIF4DAdBgNVHSUEFjAUBggrBgEF +BQcDAQYIKwYBBQUHAwIwDAYDVR0TAQH/BAIwADAdBgNVHQ4EFgQU0/qDQaY10jIo +wCjLUpz/HfQXyt8wHwYDVR0jBBgwFoAUF4ffoFrrZhKn1dD4uhJFPLcrAJwwDQYJ +KoZIhvcNAQELBQADggEBAD2hkndVih6TWxoe/oOW0i2Bq7ScNO/n7/yHWL04HJmR +MaHv/Xjc8zLFLgHuHaRvC02ikWIJyQf5xJt0Oqu2GVbqXH9PBGKuEP2kCsRRyU27 +zTclAzfQhqmKBTYQ/3lJ3GhRQvXIdYTe+t4aq78TCawp1nSN+vdH/1geG6QjMn5N +1FU8tovDd4x8Ib/0dv8RJx+n9gytI8n/giIaDCEbfLLpe4EkV5e5UNpOnRgJjjuy +vtZutc81TQnzBtkS9XuulovDE0qI+jQrKkKu8xgGLhgH0zxnPkKtUg2I3Aq6zl1L +zYkEOUF8Y25J6WeY88Yfnc0iigI+Pnz5NK8R9GL7TYo= +-----END CERTIFICATE----- diff --git a/src/test/resources/security/test-kirk.jks b/src/test/resources/security/test-kirk.jks index 174dbda656f41b10341adb78ab91a46afaae8a1c..6dbc51e714784fa58a4209c75deab8b9ed1698ff 100644 GIT binary patch literal 4504 zcma)AXEYp+vt7GZ$?DyT=tPUf>Rt32Rtcg+B4PQKLo)5nT`xBt(f8 zz4zYx{`1az=l47B(|aH0%$a-V&c}OZ28N+d1QLK?7-~f#Qh{)-@KbUEVuBnDwFn`G zTJSH-2g86X{uc$#Cd7a<{=zALBY_C=KPs|Y1i%~&Sotp~4}12H0!$9GfJy&blEDNC z=>%hA9@l)1y-8vD6#cH^U}=KBI0FdeqXH7J!^nt8{(B;j6byi|5|P@4YY{kr2nhrT zsl1TD93_M516EPM#9d4EG(rsFKtBW4^r*(5KwKbTLB){+^0E(}Q+A7HoW0lrA)@i+ zydGtY^95cAh7C?*2qIcESObb&7%#|($|(-eXIiQ#0>bYpj@=?*4?U=5@-ISTdSa4x zOtEjIWb0hr)D^1HVpX7-CjwnsDG8#WM@AVZvyufeW?}`^GtGW7WcGsVl)G*$?lP3S z^GYelg04B!ZBp4GnwCzq@uOLfB4xY#hE;StB61*Yd8?%(Nl9NW{s3+HODy#ik72s%Hj($a8 zhF0>hs}=106=eHlR<&9zT@LuHAUIZWLFWrKQ#$R3^=pv*&-7e6{O_Ji`|s`^^4v@-Hr>`?(V#!ktZ-$-0?Jt1G-G? zE9HvN@-0iPpKSDRsLacPB>#JY4d$KM!zs7xPBvUu4HQ}!Bz$qc)A`=Ver4EBC?!g7b zuW7GvE*puJA=;!bv2_S?8ZQx_n`M?F&kkb{-h zKwO=OA_@auvAUmAsQW~NjYK|}m{>`{*n^45MJ^ph*%K9}8GnxA%-;D^^-}ih8oWP* zXJ#vzJY3e4?&oSey+_=qv19lq zeLI>%Gjx=y!qVzf%Y&c7dgkjEw?^rl8^KxGs^%{Fd_(b51&l(wYCO&Rc~ZUl5^~y> zc}BJ!4+n2KaS|<{vd#M44my1W|M0Y-gfk9<&l%IBje@31-Sr1Mt!fvT(Pe+Gt$Bz? z_up@HJf$b!)YfI|4{%l^JDxgWvp75|nMzg7E)(qZ%=alvt zXMfZg7Z=_eanGP?tBXFKyvFRu$?uMAzg|k-(32orZccxnHGr$(gM%4Hgc&3blJCi; z6j@^Y3XVg*doBz7pms~Jn7 z9>1&oI7bPBOnn7vyV1x>YahPMDy_bySw!71ij);ebzBEUSZK&o1y43I-AuJKXJ~C3 z{ScF0neCZB8?5r>Px#3V%} zq$OY&i2FZH#6&q5i2Yy421o$-o6P@Z2>vgd4p$sB)+@I7CAQvk>m=OVG#EC`^#8Hx zXo}&oS5+Eg(sw4>QN4_Cy_0U!W9o!pxS@}|4s+L{ow)59*P>fYuDV~JqCwTL5s{)3(v zzbM`$E?)E;`zu*Kjpah> zgQl1ucOJOd1|%MDBk_Lsu64*-#r>9orWT19xT!DnCoNv_AnWczl?5a3@Sd4mtPrx@ z;QPqXK#%ve%3=_Sa$)(zJ)mvCYW0$Uim6bQ!S}#H@uPFY+qvmT_x`cr%&q*~6sufG zKKVZ8ebd?WhVYT)or=?jzV*~PLH&t?CH^KO=IX%=oHNr75%vVz=nN9ipHOrX*7{h! zNkaI3@a@JfTINcbD<@;DNwqa&=S5v4pM=tBEMN8HU3}euq?(dEFWfNC>H+2C+1dBA zFs|s&27315cK^vG`LRKX~{Ugw!|2K~TP_VAqXtzNY6)j={rQ zv73v$!psb1ph9o6`kKlGjC8GEdFX9+@{I}q{33}%?v>$a-cw6HGOOLVnv3ITN_D~k zo^QL%)6K#_{j)b&>8Qy@Eweq=Ne8rKsjJTe)mfDw?scqlc&US2dxU0@o5$(Zu(GB4 zujr5^yZdwlP>E{wrkq=NiW~PQZm5`fJz5m&9I}B^zPVNSSa9vWcXu^m%+bU|aOg5q zK%|a72J^vxGy)&3GlNod=Wt|FBG=mgP)o%{(2PCL$9s$dMvIcv^FdM?hbNYQrX%I| z{binoW_?J27M3L2H_Y4n0!3PGL#b*UxRbpd3l$RLC#I})-32((m#4}vP%kHB3Q7PGLpvuro4~7i2u6z$3ar+YSP2?_%+^%f* zR}5Rl@nUnDVdT&uE_ZP%NU-(Zn*^k2*4S;xubW_f3f-cK+=>uy-sK;&F{mRdpgwIgSHfJSw=22paH-mu>R=3Kf9cR*A_Sjg7q#MM< zqobyHu#q_oM3;REOf&nTGa=n6MK4QZ{pey;iGwX&bnAUCVq`=c0{gykLm{VZo%ulF z*n_LEk%}KbmVW1)L+Ab3sSZPR+Fe*5p$^HC|Oyb{_is> zsuD42;l;BT-a#X6fP(~C+`TP&(``5KD7dp9)GD&EVfNN4Bf@5N63j4c_IOZZ`^gF1 zphj9>;b1JVOWrk`HhO{mmk*Lp>wXpL*r|VQth!^2ajO2-Q$=;E0ZcMzj9V;D}3k7ej?g$MEOSvfr*p<&b z6B?7p3F^a78y9pEd$#q2Pm1b zU#?c^Op~TXSZ`3z2a{A=UzcS`zB%Z|XG2xth@1`h=wY$wyp|u2)s&QN#af+k>`vF! z&{oB;K{Wblwtcc`JH%E!TwV2q%vd}p>iZ9d@C(kwR>Dm)p? zV-i0tv8PP66)jD1#I*Qm*`@U`^o)}|58+bGD1y(EEM_dJh-O9xP^xdF-_Z#qZ&m{c zbC6W;iNU!24Cvnj14>>_V8a{IB$GXu&z39rEKNX_07*3xp*W3rJo!}pp2M0Hwe$#* zi#HgV_>>SSD;YT=uK8*Lu|$a+IIXPF$${!eaPU%X#jh@y96VcWEFGqB#<_hE8QPmQ zO_C$p_nXzGgQtqVrC1t-5`*juoj0Q%VLnw`@Yt&eCg!x)84Pq&N%`@t**O@LYz3OR(@+})Hu&$>gJ;6oxdO{ z&KR3!hDx52>YBb*JE@4B`8}j*yOg=37>&zbSN}#T@GA6n9+dFcA*9q_l2eI%Xh*7~ ziU87?k{%5!@e5oasj8xTY|ysPyOMR3W;w?vvG}prD%~$8wf$j!6&K4LI%aD1$6B&8 zG|Bq_{em<75I~pVeMNJ6Dv9e{<=x@Es?2r|L;d(lJhNv+5~$`ps7`1lAq>B{Ot5Ga z6qD6CeNHKADuYBeC(!$C>E5yJ7O5IFfdN*2lPV*LTj(fX$`T*h6!l7_BFQ%HhbJFp zKUVk@Dl`5ZH)LoQ^{7N6?HyY_;Jo?*Uu#dn_XW`49o!xdK!+JJN_3KD7k@2J((0h0 z?0!++a*3VkR_Y8-s+o<1M(>PCz=|sJMqa z0+r0sNH_$gvD_@AC}TCb8}m~2v}_leWOtWdheZwxJl0i{OGIRcO0iVJ-B>5CgP^O-M7OYVJ*8(0|euX~UGp`sq@@gaEw*bHD4*Dj8_ zPO4*=dce-k-f;9Xl`P>A2U6SzIPhFWQT>2(PjqTMlBf}zL3<&dS*!E0mM}&jbXhc- zAb9}5!V(`=H1zl4fM|8TdAE{XwAuTJ>dTw3o}wzSb&xhxCijhe4Q#{|l(FXGy+A)j zH>IZrWy4|#?wJ-1?zBm;cKLHK*H5ngXeiJE?k?6Lz1i+02rcMG7kNDQlDJ_??0D#; z(Bju>vbV@>IGl97vC?TD(|fa!E?NjDA;*m&#_ZiX>Vgi+wr`atYOngkRp_w%?M~sv zUVImV4>dX4Ih+MO4LU`Ui=K%20a~JOwq1$6)KUw@81y#uUGKMV4>O0ioDGDvtZ{Jl zmay)x!zLD>Hl1jqnzX9b_da}w9xr9S`kQwUZPAei4I5Ao#$N}f9I10=!}MXIF!F!C z6+i+ofRKI2Rvlk8erCmgYu2%A6S_nSX7!cGJQ6pQ{xw*Iw(KXQGft90Ft(YQ<7nw! ROz*Khv5A{`^It3We*oUlR=)rM literal 3874 zcmY+GcQhM}zs8e@RFtSn>{Y9_XzfvZl*TSQG6F9FNd(wu zFab99cRY+FKt2CO5E21u`*&mo0s{VisDB9%$qk|Z?*;}S1PKGv1*1XCugHHEKp;B6 z69Saqd|bch;ZdXcj@o48Or^T{VjiQWQ)um?koax&EW2Jd6%cmO+99&?<0M#TkhMY0 z>TOc9NNj$5o%GwnI2>ZpA<-syd;YVlrkqVstJxqe_w8#F0dlKW!#D3WVDWfwaN@uX z{)l!>hgv`=r)M_tPedAH8wS zrMCsCM3^vbf3iWkdUoK)O(h9`bxp3s^zq4CU5%IJN;Y04OLiLfXPS%;Duo}L?EKtE z$4DyO?uRf+Ovm@OBmMKYjcI;;3k(jA`wJ`_W&){Es6Nv(A-s;NYZhfPTZJ%tBZ{1@ zc|_(P(o|Du6c{sJ4@Q6w- zF)*aVb&dDqmGoH8(8Y;T2S?DR9+P|nUT>q8177|so}DjY7IWc!jB(9r?rJ%YyVvh5 z4`BJLeFX6F2g1N^WT?dWin3^|1>$*MQP~CSqFMgQ4m&bJp``1>I(!5Pe9&NB7{wXc z+p)Bs6Durb104tWmIOYRkBU~Waz;l#k`+@Fye00vbTIQq3dY*R{KBH-UF3%r{=+v` zqu(DD1~xv;*N0vqhN9l+bCm(5u37KF+&JF&or0qB&J%}ZmdviHekDmr#GlPK60J4Q zJ#vSZYt1pSxEPM~S27`bL-X}ig&?t1ubwy1&P?lEwQUs|t?a7>dqM7^&@^5tSL9pMp+&5H?jk>BGMj!JcQ+3*rxFcY4MY2z z4C?1*^xq&(g`+u7JnXS-Yuq8?$%DG-Zs#VDo=cTmcJRfEFTG1T4~(u1j$Snc+7Cs; zyB9?mE4rqbq_*xqj?#OlN%@YGt*PgH+-~Fy+blur5jn zu_S?>vGKl_57zp6>#CW5Q&HHKl|qVToNrM`8!zz5n*{CQ+r2#n4{2tk@;0m{ zM8pbY25rVQv1<0iw2CPT?uG+>NVZVLalVoRSZQdC(&M@`0$mC@6l?zxF&LAM8XHR1Ah3S zb?4&7@N$w<+PVC^0ws=h2pqrozQ!=b!?Zy2@uQjFh1)BEPT$JlDa9Q8(%YHT_r)w# z<4bW`j)gX^ktonho#Uf=U=ZH5QT!;ug%qe!Fi?N(OjphEVY3YTU5B*j^ZMOg+XmnL zPpT%`zoHjGCw~=w|5zC`KWOFwsF`=Jjwez^hwA2rgTt^ z^10Gp<3*%@mI37QZ>P3$*PX4;4LpFQqK9AnvMxAg!|B)unEQ{13w`0LO;;mgV22L5 z=Y8bwo8Fch2UFgZEqeTdMGZMKmz)4Uzb#-R)&H4zUC45?<4&g?`6XX-=`F2|(~Esf z4P+-+Y;J{*hV8L55?o`K^wL+ zE>e|WH7ZW48)vi%Zq4nbkLikeTd&2pCr5A#jJC9jypS>*@uF<#i}Xp$3X7~b0>bXQ zd@CV7FY-$A{IR_m5uZie z+ckdOpNC4bjck=wZ@3lTl5+`W3~_4oPuGx4#mk-f?CsbGulgu|BAb)LTI|hBYM==Q zPLdu6@x)I_O{qq^{%cI*Q`-C+WZjpp^GjGiWv(#7Vr(pZ@A532u&Rn|3@4+xgKqNc zMhtgDOn)7lv}KZc^U}jD!KU{3;=7as(>uBwDx5}ii8iIz!F(WDlbe(V`WH5PS-XhZ zPJFI;eV}4{aJ?&?Sv%?zMZJ9SRFL%?ZZ0C(FdozY2R@i=1>&&E< z<(hauSRE!6;QE6ujbYrYrWNm9;!ixJV`}*=J$7wZ^0l>rTb7|)`olK^*^m3Ex%nq2 zL({r^1)T=Q7qM>-F~1lC817t!PNhq1c&?{#kiAuiMtlDELuI?Ut6LMQ6()675@U5L z_g(P7&7MR-N3z!C5a+qZ$!xmrg0qbsQn*7vqc!v-^yqc6`tlc%aQl-Fe+IYP5Pe^K z^%zx2w*a+^&+F*;<~HZ&=XwRTB6z)Uec2XkH=^cl)cHs|VxGqSQStks&td*NQbTPW z@??ewN#dRVCH?t{p-$)JDIxkVF$#9Q?iS!Qqby9p zttQuw3k2_4Hs9`5TG}3Jwk97Nste6#I!jG)f$b(~xI#)Bs7nQ7es#6RzYPh=8vCY$@K;aE z0JYYxSm&6)?GS&eI-ibs8vhi$EXK)Yhv7%bHy2C$czjfz?F4J+b%lJkXj+1&h?Ti_R;#D>}h%qh-ltN3^kJE=J$q9lGN z97&*c`aeQNBG8(G3ADz4#|D3&4&?Ix=oLK>L?VDUkp%Gi|FbTdf2<2!*X4kUenR(; zb%6=se)ca%eZ zOyn3`1eb66NoONNlb!Qgq|BuMxwULjnW>4u2iuhj(ZUV8fC!eY=nsZF*}w6V0(LxJ zVJ|ew^cV0%UizR_Y1yOEtM1}iw*f#fPAX(#E)%*G)QD7W7O$XT5e!*pv0krMED!yw zv)_h?54B@8<=GZ6ukEmkmrx<@jaUud2Y%EQU-vBcCChZ&9Xf`1Rw3w4G=@{y>I<<5 zr)BfiiXe`(Z@ksE4@BqB5d!$>pA(N&9b7XX5GBfr?j{H(J6=OSr*~9Ff8Zh0^d;HS3|V9O<+-Py zxI&YAI-gM^t2+X1O6JyQ*^8SfuZ5{?m1F14fGg;0aeF|P)4c8tw{C;?*J)`bjV2~qOsSjk^$@gQ1{3jw}OGfYhan!3#Y zHIQX-5|4fmT69zTvDd3aW(AkQqj4t}?Md}bd>>Q>N!29V@klLOr#L%^gPrlgw8ASS>!fstf*6i;ka?xLu@MUq>?r_mf*HCZ0jHy2N^B`x>Y90Tt5-jn7*G)Ai~?r^6!i zChFK}Z-Np|s#K(ct1NYcNSoxM%p~ng6bf7}uXm#_v&(wHHp4Tljgd6EW$Kg0xZkkr zi&o;({o`MC#=#JXFx-Py14vyFMbGypX`-a>1F9n21b`MXKk|zU$zEO&>l1Rjkx$4Vg-UeUetqM3xCVt2 z#4}QY$t__sQxkuq9U8E_JbjM8#9JvlSK48A@`?q^I*~JnT-!@f$l49YlT>fpGqYJ9 zr+k*tw-oT8l~Dr<$GT8lt$6D+{n7Af1%CX7h0*}>N)s;I);DZqq{57a Date: Mon, 2 Oct 2023 09:47:11 -0700 Subject: [PATCH 147/416] Removded unnecessary call to find live docs during filtered search (#1177) (#1180) Signed-off-by: Navneet Verma (cherry picked from commit 78aba559da1eb56d91931f0061c255f390de3b93) Co-authored-by: Navneet Verma --- .../org/opensearch/knn/index/query/KNNWeight.java | 15 +++------------ .../knn/index/query/KNNWeightTests.java | 2 -- 2 files changed, 3 insertions(+), 14 deletions(-) diff --git a/src/main/java/org/opensearch/knn/index/query/KNNWeight.java b/src/main/java/org/opensearch/knn/index/query/KNNWeight.java index d24075aa5..c166be3c2 100644 --- a/src/main/java/org/opensearch/knn/index/query/KNNWeight.java +++ b/src/main/java/org/opensearch/knn/index/query/KNNWeight.java @@ -116,7 +116,7 @@ public Scorer scorer(LeafReaderContext context) throws IOException { * . Hence, if filtered results are less than K and filter query is present we should shift to exact search. * This improves the recall. */ - if (filterWeight != null && canDoExactSearch(filterIdsArray.length, getTotalDocsInSegment(context))) { + if (filterWeight != null && canDoExactSearch(filterIdsArray.length)) { docIdsToScoreMap.putAll(doExactSearch(context, filterIdsArray)); } else { Map annResults = doANNSearch(context, filterIdsArray); @@ -380,10 +380,9 @@ private SpaceType getSpaceType(final FieldInfo fieldInfo) { ); } - private boolean canDoExactSearch(final int filterIdsCount, final int searchableDocs) { + private boolean canDoExactSearch(final int filterIdsCount) { log.debug( - "Info for doing exact search Live Docs: {}, filterIdsLength : {}, Threshold value: {}", - searchableDocs, + "Info for doing exact search filterIdsLength : {}, Threshold value: {}", filterIdsCount, KNNSettings.getFilteredExactSearchThreshold(knnQuery.getIndexName()) ); @@ -420,12 +419,4 @@ private boolean isExactSearchThresholdSettingSet(int filterThresholdValue) { private boolean canDoExactSearchAfterANNSearch(final int filterIdsCount, final int annResultCount) { return filterWeight != null && filterIdsCount >= knnQuery.getK() && knnQuery.getK() > annResultCount; } - - private int getTotalDocsInSegment(final LeafReaderContext context) { - // This means that there is no deleted documents, hence the live docs bitset is null - if (context.reader().getLiveDocs() == null) { - return context.reader().maxDoc(); - } - return context.reader().getLiveDocs().length(); - } } diff --git a/src/test/java/org/opensearch/knn/index/query/KNNWeightTests.java b/src/test/java/org/opensearch/knn/index/query/KNNWeightTests.java index cd0daa54b..79e41b52f 100644 --- a/src/test/java/org/opensearch/knn/index/query/KNNWeightTests.java +++ b/src/test/java/org/opensearch/knn/index/query/KNNWeightTests.java @@ -425,7 +425,6 @@ public void testANNWithFilterQuery_whenExactSearch_thenSuccess() { final Bits liveDocsBits = mock(Bits.class); when(reader.getLiveDocs()).thenReturn(liveDocsBits); when(liveDocsBits.get(filterDocId)).thenReturn(true); - when(liveDocsBits.length()).thenReturn(1000); final KNNWeight knnWeight = new KNNWeight(query, 0.0f, filterQueryWeight); final Map attributesMap = ImmutableMap.of(KNN_ENGINE, KNNEngine.FAISS.getName(), SPACE_TYPE, SpaceType.L2.name()); @@ -476,7 +475,6 @@ public void testANNWithFilterQuery_whenExactSearchAndThresholdComputations_thenS final Bits liveDocsBits = mock(Bits.class); when(reader.getLiveDocs()).thenReturn(liveDocsBits); when(liveDocsBits.get(filterDocId)).thenReturn(true); - when(liveDocsBits.length()).thenReturn(1000); final KNNWeight knnWeight = new KNNWeight(query, 0.0f, filterQueryWeight); final Map attributesMap = ImmutableMap.of(KNN_ENGINE, KNNEngine.FAISS.getName(), SPACE_TYPE, SpaceType.L2.name()); From e9e6146f849849cfaa9b78f9e96c60fd0f55379c Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Mon, 2 Oct 2023 12:38:02 -0700 Subject: [PATCH 148/416] Increment version to 2.11.0-SNAPSHOT (#1103) Signed-off-by: opensearch-ci-bot Co-authored-by: opensearch-ci-bot --- .../workflows/backwards_compatibility_tests_workflow.yml | 8 ++++---- build.gradle | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/backwards_compatibility_tests_workflow.yml b/.github/workflows/backwards_compatibility_tests_workflow.yml index b5c25fd6c..b7ce283d1 100644 --- a/.github/workflows/backwards_compatibility_tests_workflow.yml +++ b/.github/workflows/backwards_compatibility_tests_workflow.yml @@ -14,8 +14,8 @@ jobs: strategy: matrix: java: [ 11, 17 ] - bwc_version : [ "1.1.0", "1.2.4", "1.3.8", "2.0.1", "2.1.0", "2.2.1", "2.3.0", "2.4.1", "2.5.0", "2.6.0", "2.7.0", "2.8.0", "2.9.0" ] - opensearch_version : [ "2.10.0-SNAPSHOT" ] + bwc_version : [ "1.1.0", "1.2.4", "1.3.8", "2.0.1", "2.1.0", "2.2.1", "2.3.0", "2.4.1", "2.5.0", "2.6.0", "2.7.0", "2.8.0", "2.9.0", "2.10.0" ] + opensearch_version : [ "2.11.0-SNAPSHOT" ] name: k-NN Restart-Upgrade BWC Tests runs-on: ubuntu-latest @@ -46,8 +46,8 @@ jobs: strategy: matrix: java: [ 11, 17 ] - bwc_version: [ "1.3.8", "2.0.1", "2.1.0", "2.2.1", "2.3.0", "2.4.1", "2.5.0", "2.6.0", "2.7.0", "2.8.0", "2.9.0"] - opensearch_version: [ "2.10.0-SNAPSHOT" ] + bwc_version: [ "1.3.8", "2.0.1", "2.1.0", "2.2.1", "2.3.0", "2.4.1", "2.5.0", "2.6.0", "2.7.0", "2.8.0", "2.9.0", "2.10.0"] + opensearch_version: [ "2.11.0-SNAPSHOT" ] name: k-NN Rolling-Upgrade BWC Tests runs-on: ubuntu-latest diff --git a/build.gradle b/build.gradle index b70958daa..fa7eafd1a 100644 --- a/build.gradle +++ b/build.gradle @@ -10,7 +10,7 @@ buildscript { ext { // build.version_qualifier parameter applies to knn plugin artifacts only. OpenSearch version must be set // explicitly as 'opensearch.version' property, for instance opensearch.version=2.0.0-rc1-SNAPSHOT - opensearch_version = System.getProperty("opensearch.version", "2.10.0-SNAPSHOT") + opensearch_version = System.getProperty("opensearch.version", "2.11.0-SNAPSHOT") version_qualifier = System.getProperty("build.version_qualifier", "") opensearch_group = "org.opensearch" } From 0dd42ef283b4a132d7b1344bc31858949fe76eb6 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Mon, 2 Oct 2023 14:03:20 -0700 Subject: [PATCH 149/416] Upgrade bytebuddy version to match OpenSearch core (#1135) (#1183) * Bump bytebuddy version to match OpenSearch core Signed-off-by: Ryan Bogan (cherry picked from commit 0f93430c703fea25a118e4791dd24ab0709bc9dd) Co-authored-by: Ryan Bogan <10944539+ryanbogan@users.noreply.github.com> --- CHANGELOG.md | 1 + build.gradle | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f0832aa6e..1075b68c4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,4 +19,5 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Infrastructure ### Documentation ### Maintenance +- Update bytebuddy to 1.14.7 [#1135](https://github.com/opensearch-project/k-NN/pull/1135) ### Refactoring diff --git a/build.gradle b/build.gradle index fa7eafd1a..1960dd220 100644 --- a/build.gradle +++ b/build.gradle @@ -175,9 +175,9 @@ dependencies { api group: 'com.google.guava', name: 'guava', version:'32.0.1-jre' api group: 'commons-lang', name: 'commons-lang', version: '2.6' testFixturesImplementation "org.opensearch.test:framework:${opensearch_version}" - testImplementation group: 'net.bytebuddy', name: 'byte-buddy', version: '1.14.3' + testImplementation group: 'net.bytebuddy', name: 'byte-buddy', version: '1.14.7' testImplementation group: 'org.objenesis', name: 'objenesis', version: '3.2' - testImplementation group: 'net.bytebuddy', name: 'byte-buddy-agent', version: '1.14.3' + testImplementation group: 'net.bytebuddy', name: 'byte-buddy-agent', version: '1.14.7' testFixturesImplementation "org.opensearch:common-utils:${version}" } From 39836c69154387d9784710d90451ca241187ac3f Mon Sep 17 00:00:00 2001 From: Ryan Bogan <10944539+ryanbogan@users.noreply.github.com> Date: Tue, 3 Oct 2023 15:07:14 -0700 Subject: [PATCH 150/416] [Backport 2.x] Add ignore_unmapped support in KNNQueryBuilder (#1152) * Add ignore_unmapped support in KNNQueryBuilder Signed-off-by: Ryan Bogan --- CHANGELOG.md | 1 + .../org/opensearch/knn/index/IndexUtil.java | 19 +++++++ .../knn/index/query/KNNQueryBuilder.java | 57 +++++++++++++++---- .../knn/index/query/KNNQueryBuilderTests.java | 14 +++++ 4 files changed, 79 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1075b68c4..f0413a82c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ## [Unreleased 2.x](https://github.com/opensearch-project/k-NN/compare/2.10...2.x) ### Features ### Enhancements +- Added support for ignore_unmapped in KNN queries. [#1071](https://github.com/opensearch-project/k-NN/pull/1071) ### Bug Fixes ### Infrastructure ### Documentation diff --git a/src/main/java/org/opensearch/knn/index/IndexUtil.java b/src/main/java/org/opensearch/knn/index/IndexUtil.java index 60156b4a7..f9809ba70 100644 --- a/src/main/java/org/opensearch/knn/index/IndexUtil.java +++ b/src/main/java/org/opensearch/knn/index/IndexUtil.java @@ -13,6 +13,7 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.Maps; +import org.opensearch.Version; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.metadata.MappingMetadata; import org.opensearch.common.ValidationException; @@ -24,6 +25,7 @@ import java.io.File; import java.util.Collections; +import java.util.HashMap; import java.util.Map; import static org.opensearch.knn.common.KNNConstants.BYTES_PER_KILOBYTES; @@ -32,6 +34,15 @@ public class IndexUtil { + private static final Version MINIMAL_SUPPORTED_VERSION_FOR_LUCENE_HNSW_FILTER = Version.V_2_4_0; + private static final Version MINIMAL_SUPPORTED_VERSION_FOR_IGNORE_UNMAPPED = Version.V_2_10_0; + public static final Map minimalRequiredVersionMap = new HashMap() { + { + put("filter", MINIMAL_SUPPORTED_VERSION_FOR_LUCENE_HNSW_FILTER); + put("ignore_unmapped", MINIMAL_SUPPORTED_VERSION_FOR_IGNORE_UNMAPPED); + } + }; + /** * Determines the size of a file on disk in kilobytes * @@ -195,4 +206,12 @@ public static Map getParametersAtLoading(SpaceType spaceType, KN return Collections.unmodifiableMap(loadParameters); } + + public static boolean isClusterOnOrAfterMinRequiredVersion(String key) { + Version minimalRequiredVersion = minimalRequiredVersionMap.get(key); + if (minimalRequiredVersion == null) { + return false; + } + return KNNClusterUtil.instance().getClusterMinVersion().onOrAfter(minimalRequiredVersion); + } } diff --git a/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java b/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java index b69b3dbfb..11912870f 100644 --- a/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java +++ b/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java @@ -6,11 +6,10 @@ package org.opensearch.knn.index.query; import lombok.extern.log4j.Log4j2; +import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.commons.lang.StringUtils; -import org.opensearch.Version; import org.opensearch.index.mapper.NumberFieldMapper; import org.opensearch.index.query.QueryBuilder; -import org.opensearch.knn.index.KNNClusterUtil; import org.opensearch.knn.index.KNNMethodContext; import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.mapper.KNNVectorFieldMapper; @@ -33,6 +32,7 @@ import java.util.List; import java.util.Objects; +import static org.opensearch.knn.index.IndexUtil.*; import static org.opensearch.knn.index.mapper.KNNVectorFieldMapperUtil.validateByteVectorValue; /** @@ -45,6 +45,7 @@ public class KNNQueryBuilder extends AbstractQueryBuilder { public static final ParseField VECTOR_FIELD = new ParseField("vector"); public static final ParseField K_FIELD = new ParseField("k"); public static final ParseField FILTER_FIELD = new ParseField("filter"); + public static final ParseField IGNORE_UNMAPPED_FIELD = new ParseField("ignore_unmapped"); public static int K_MAX = 10000; /** * The name for the knn query @@ -57,7 +58,7 @@ public class KNNQueryBuilder extends AbstractQueryBuilder { private final float[] vector; private int k = 0; private QueryBuilder filter; - private static final Version MINIMAL_SUPPORTED_VERSION_FOR_LUCENE_HNSW_FILTER = Version.V_2_4_0; + private boolean ignoreUnmapped = false; /** * Constructs a new knn query @@ -91,6 +92,7 @@ public KNNQueryBuilder(String fieldName, float[] vector, int k, QueryBuilder fil this.vector = vector; this.k = k; this.filter = filter; + this.ignoreUnmapped = false; } public static void initialize(ModelDao modelDao) { @@ -117,9 +119,12 @@ public KNNQueryBuilder(StreamInput in) throws IOException { k = in.readInt(); // We're checking if all cluster nodes has at least that version or higher. This check is required // to avoid issues with cluster upgrade - if (isClusterOnOrAfterMinRequiredVersion()) { + if (isClusterOnOrAfterMinRequiredVersion("filter")) { filter = in.readOptionalNamedWriteable(QueryBuilder.class); } + if (isClusterOnOrAfterMinRequiredVersion("ignore_unmapped")) { + ignoreUnmapped = in.readOptionalBoolean(); + } } catch (IOException ex) { throw new RuntimeException("[KNN] Unable to create KNNQueryBuilder", ex); } @@ -131,6 +136,7 @@ public static KNNQueryBuilder fromXContent(XContentParser parser) throws IOExcep float boost = AbstractQueryBuilder.DEFAULT_BOOST; int k = 0; QueryBuilder filter = null; + boolean ignoreUnmapped = false; String queryName = null; String currentFieldName = null; XContentParser.Token token; @@ -153,6 +159,10 @@ public static KNNQueryBuilder fromXContent(XContentParser parser) throws IOExcep k = (Integer) NumberFieldMapper.NumberType.INTEGER.parse(parser.objectBytes(), false); } else if (AbstractQueryBuilder.NAME_FIELD.match(currentFieldName, parser.getDeprecationHandler())) { queryName = parser.text(); + } else if (IGNORE_UNMAPPED_FIELD.getPreferredName().equals("ignore_unmapped")) { + if (isClusterOnOrAfterMinRequiredVersion("ignore_unmapped")) { + ignoreUnmapped = parser.booleanValue(); + } } else { throw new ParsingException( parser.getTokenLocation(), @@ -168,20 +178,20 @@ public static KNNQueryBuilder fromXContent(XContentParser parser) throws IOExcep // MINIMAL_SUPPORTED_VERSION_FOR_LUCENE_HNSW_FILTER variable. // Here we're checking if all cluster nodes has at least that version or higher. This check is required // to avoid issues with rolling cluster upgrade - if (isClusterOnOrAfterMinRequiredVersion()) { + if (isClusterOnOrAfterMinRequiredVersion("filter")) { filter = parseInnerQueryBuilder(parser); } else { log.debug( String.format( "This version of k-NN doesn't support [filter] field, minimal required version is [%s]", - MINIMAL_SUPPORTED_VERSION_FOR_LUCENE_HNSW_FILTER + minimalRequiredVersionMap.get("filter") ) ); throw new IllegalArgumentException( String.format( "%s field is supported from version %s", FILTER_FIELD.getPreferredName(), - MINIMAL_SUPPORTED_VERSION_FOR_LUCENE_HNSW_FILTER + minimalRequiredVersionMap.get("filter") ) ); } @@ -204,6 +214,9 @@ public static KNNQueryBuilder fromXContent(XContentParser parser) throws IOExcep KNNQueryBuilder knnQueryBuilder = new KNNQueryBuilder(fieldName, ObjectsToFloats(vector), k, filter); knnQueryBuilder.queryName(queryName); + if (isClusterOnOrAfterMinRequiredVersion("ignoreUnmapped")) { + knnQueryBuilder.ignoreUnmapped(ignoreUnmapped); + } knnQueryBuilder.boost(boost); return knnQueryBuilder; } @@ -215,9 +228,12 @@ protected void doWriteTo(StreamOutput out) throws IOException { out.writeInt(k); // We're checking if all cluster nodes has at least that version or higher. This check is required // to avoid issues with cluster upgrade - if (isClusterOnOrAfterMinRequiredVersion()) { + if (isClusterOnOrAfterMinRequiredVersion("filter")) { out.writeOptionalNamedWriteable(filter); } + if (isClusterOnOrAfterMinRequiredVersion("ignore_unmapped")) { + out.writeOptionalBoolean(ignoreUnmapped); + } } /** @@ -242,6 +258,20 @@ public QueryBuilder getFilter() { return this.filter; } + /** + * Sets whether the query builder should ignore unmapped paths (and run a + * {@link MatchNoDocsQuery} in place of this query) or throw an exception if + * the path is unmapped. + */ + public KNNQueryBuilder ignoreUnmapped(boolean ignoreUnmapped) { + this.ignoreUnmapped = ignoreUnmapped; + return this; + } + + public boolean getIgnoreUnmapped() { + return this.ignoreUnmapped; + } + @Override public void doXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(NAME); @@ -252,6 +282,9 @@ public void doXContent(XContentBuilder builder, Params params) throws IOExceptio if (filter != null) { builder.field(FILTER_FIELD.getPreferredName(), filter); } + if (ignoreUnmapped) { + builder.field(IGNORE_UNMAPPED_FIELD.getPreferredName(), ignoreUnmapped); + } printBoostAndQueryName(builder); builder.endObject(); builder.endObject(); @@ -261,6 +294,10 @@ public void doXContent(XContentBuilder builder, Params params) throws IOExceptio protected Query doToQuery(QueryShardContext context) { MappedFieldType mappedFieldType = context.fieldMapper(this.fieldName); + if (mappedFieldType == null && ignoreUnmapped) { + return new MatchNoDocsQuery(); + } + if (!(mappedFieldType instanceof KNNVectorFieldMapper.KNNVectorFieldType)) { throw new IllegalArgumentException(String.format("Field '%s' is not knn_vector type.", this.fieldName)); } @@ -345,8 +382,4 @@ protected int doHashCode() { public String getWriteableName() { return NAME; } - - private static boolean isClusterOnOrAfterMinRequiredVersion() { - return KNNClusterUtil.instance().getClusterMinVersion().onOrAfter(MINIMAL_SUPPORTED_VERSION_FOR_LUCENE_HNSW_FILTER); - } } diff --git a/src/test/java/org/opensearch/knn/index/query/KNNQueryBuilderTests.java b/src/test/java/org/opensearch/knn/index/query/KNNQueryBuilderTests.java index 236cbd644..1381e19be 100644 --- a/src/test/java/org/opensearch/knn/index/query/KNNQueryBuilderTests.java +++ b/src/test/java/org/opensearch/knn/index/query/KNNQueryBuilderTests.java @@ -7,6 +7,7 @@ import com.google.common.collect.ImmutableMap; import org.apache.lucene.search.KnnFloatVectorQuery; +import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.Query; import org.opensearch.Version; import org.opensearch.cluster.ClusterModule; @@ -41,6 +42,7 @@ import java.util.List; import java.util.Optional; +import static org.hamcrest.Matchers.instanceOf; import static org.mockito.Mockito.anyString; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -297,6 +299,18 @@ public void testSerialization() throws Exception { assertSerialization(Version.V_2_3_0, Optional.empty()); } + public void testIgnoreUnmapped() throws IOException { + float[] queryVector = { 1.0f, 2.0f, 3.0f, 4.0f }; + KNNQueryBuilder knnQueryBuilder = new KNNQueryBuilder(FIELD_NAME, queryVector, K); + knnQueryBuilder.ignoreUnmapped(true); + assertTrue(knnQueryBuilder.getIgnoreUnmapped()); + Query query = knnQueryBuilder.doToQuery(mock(QueryShardContext.class)); + assertNotNull(query); + assertThat(query, instanceOf(MatchNoDocsQuery.class)); + knnQueryBuilder.ignoreUnmapped(false); + expectThrows(IllegalArgumentException.class, () -> knnQueryBuilder.doToQuery(mock(QueryShardContext.class))); + } + private void assertSerialization(final Version version, final Optional queryBuilderOptional) throws Exception { final KNNQueryBuilder knnQueryBuilder = queryBuilderOptional.isPresent() ? new KNNQueryBuilder(FIELD_NAME, QUERY_VECTOR, K, queryBuilderOptional.get()) From 46b54d698c25fa3256e79ec8105a88f26fc162b1 Mon Sep 17 00:00:00 2001 From: Ryan Bogan <10944539+ryanbogan@users.noreply.github.com> Date: Wed, 4 Oct 2023 10:32:35 -0700 Subject: [PATCH 151/416] Add graph creation stats to KNNStats API (#1191) Signed-off-by: Ryan Bogan --- CHANGELOG.md | 3 +- .../KNN80Codec/KNN80DocValuesConsumer.java | 49 +++++++++- .../knn/index/codec/util/KNNCodecUtil.java | 46 ++++++++- .../util/KNNVectorSerializerFactory.java | 2 +- .../knn/plugin/stats/KNNGraphValue.java | 95 +++++++++++++++++++ .../opensearch/knn/plugin/stats/KNNStats.java | 29 ++++++ .../knn/plugin/stats/StatNames.java | 5 +- .../KNN80DocValuesConsumerTests.java | 63 ++++++++++-- .../java/org/opensearch/knn/TestUtils.java | 3 +- 9 files changed, 278 insertions(+), 17 deletions(-) create mode 100644 src/main/java/org/opensearch/knn/plugin/stats/KNNGraphValue.java diff --git a/CHANGELOG.md b/CHANGELOG.md index f0413a82c..1b17c8f63 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,7 +15,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ## [Unreleased 2.x](https://github.com/opensearch-project/k-NN/compare/2.10...2.x) ### Features ### Enhancements -- Added support for ignore_unmapped in KNN queries. [#1071](https://github.com/opensearch-project/k-NN/pull/1071) +* Added support for ignore_unmapped in KNN queries. [#1071](https://github.com/opensearch-project/k-NN/pull/1071) +* Add graph creation stats to the KNNStats API. [#1141](https://github.com/opensearch-project/k-NN/pull/1141) ### Bug Fixes ### Infrastructure ### Documentation diff --git a/src/main/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumer.java b/src/main/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumer.java index ed423c5c8..901328766 100644 --- a/src/main/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumer.java +++ b/src/main/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumer.java @@ -10,6 +10,7 @@ import lombok.extern.log4j.Log4j2; import org.apache.lucene.store.ChecksumIndexInput; import org.opensearch.common.xcontent.XContentType; +import org.opensearch.common.StopWatch; import org.opensearch.core.xcontent.DeprecationHandler; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.knn.index.KNNSettings; @@ -33,6 +34,7 @@ import org.apache.lucene.store.FilterDirectory; import org.opensearch.knn.index.mapper.KNNVectorFieldMapper; import org.opensearch.knn.common.KNNConstants; +import org.opensearch.knn.plugin.stats.KNNGraphValue; import java.io.Closeable; import java.io.IOException; @@ -51,6 +53,7 @@ import static org.opensearch.knn.common.KNNConstants.MODEL_ID; import static org.opensearch.knn.common.KNNConstants.PARAMETERS; import static org.opensearch.knn.index.codec.util.KNNCodecUtil.buildEngineFileName; +import static org.opensearch.knn.index.codec.util.KNNCodecUtil.calculateArraySize; /** * This class writes the KNN docvalues to the segments @@ -74,7 +77,13 @@ class KNN80DocValuesConsumer extends DocValuesConsumer implements Closeable { public void addBinaryField(FieldInfo field, DocValuesProducer valuesProducer) throws IOException { delegatee.addBinaryField(field, valuesProducer); if (isKNNBinaryFieldRequired(field)) { - addKNNBinaryField(field, valuesProducer); + StopWatch stopWatch = new StopWatch(); + stopWatch.start(); + addKNNBinaryField(field, valuesProducer, false, true); + stopWatch.stop(); + long time_in_millis = stopWatch.totalTime().millis(); + KNNGraphValue.REFRESH_TOTAL_TIME_IN_MILLIS.set(KNNGraphValue.REFRESH_TOTAL_TIME_IN_MILLIS.getValue() + time_in_millis); + logger.warn("Refresh operation complete in " + time_in_millis + " ms"); } } @@ -95,7 +104,8 @@ private KNNEngine getKNNEngine(@NonNull FieldInfo field) { return KNNEngine.getEngine(engineName); } - public void addKNNBinaryField(FieldInfo field, DocValuesProducer valuesProducer) throws IOException { + public void addKNNBinaryField(FieldInfo field, DocValuesProducer valuesProducer, boolean isMerge, boolean isRefresh) + throws IOException { // Get values to be indexed BinaryDocValues values = valuesProducer.getBinary(field); KNNCodecUtil.Pair pair = KNNCodecUtil.getFloats(values); @@ -103,6 +113,12 @@ public void addKNNBinaryField(FieldInfo field, DocValuesProducer valuesProducer) logger.info("Skipping engine index creation as there are no vectors or docs in the documents"); return; } + long arraySize = calculateArraySize(pair.vectors, pair.serializationMode); + if (isMerge) { + KNNGraphValue.MERGE_CURRENT_OPERATIONS.increment(); + KNNGraphValue.MERGE_CURRENT_DOCS.incrementBy(pair.docs.length); + KNNGraphValue.MERGE_CURRENT_SIZE_IN_BYTES.incrementBy(arraySize); + } // Increment counter for number of graph index requests KNNCounter.GRAPH_INDEX_REQUESTS.increment(); // Create library index either from model or from scratch @@ -133,6 +149,14 @@ public void addKNNBinaryField(FieldInfo field, DocValuesProducer valuesProducer) indexCreator = () -> createKNNIndexFromScratch(field, pair, knnEngine, indexPath); } + if (isMerge) { + recordMergeStats(pair.docs.length, arraySize); + } + + if (isRefresh) { + recordRefreshStats(); + } + // This is a bit of a hack. We have to create an output here and then immediately close it to ensure that // engineFileName is added to the tracked files by Lucene's TrackingDirectoryWrapper. Otherwise, the file will // not be marked as added to the directory. @@ -141,6 +165,19 @@ public void addKNNBinaryField(FieldInfo field, DocValuesProducer valuesProducer) writeFooter(indexPath, engineFileName); } + private void recordMergeStats(int length, long arraySize) { + KNNGraphValue.MERGE_CURRENT_OPERATIONS.decrement(); + KNNGraphValue.MERGE_CURRENT_DOCS.decrementBy(length); + KNNGraphValue.MERGE_CURRENT_SIZE_IN_BYTES.decrementBy(arraySize); + KNNGraphValue.MERGE_TOTAL_OPERATIONS.increment(); + KNNGraphValue.MERGE_TOTAL_DOCS.incrementBy(length); + KNNGraphValue.MERGE_TOTAL_SIZE_IN_BYTES.incrementBy(arraySize); + } + + private void recordRefreshStats() { + KNNGraphValue.REFRESH_TOTAL_OPERATIONS.increment(); + } + private void createKNNIndexFromTemplate(byte[] model, KNNCodecUtil.Pair pair, KNNEngine knnEngine, String indexPath) { Map parameters = ImmutableMap.of( KNNConstants.INDEX_THREAD_QTY, @@ -205,7 +242,13 @@ public void merge(MergeState mergeState) { for (FieldInfo fieldInfo : mergeState.mergeFieldInfos) { DocValuesType type = fieldInfo.getDocValuesType(); if (type == DocValuesType.BINARY && fieldInfo.attributes().containsKey(KNNVectorFieldMapper.KNN_FIELD)) { - addKNNBinaryField(fieldInfo, new KNN80DocValuesReader(mergeState)); + StopWatch stopWatch = new StopWatch(); + stopWatch.start(); + addKNNBinaryField(fieldInfo, new KNN80DocValuesReader(mergeState), true, false); + stopWatch.stop(); + long time_in_millis = stopWatch.totalTime().millis(); + KNNGraphValue.MERGE_TOTAL_TIME_IN_MILLIS.set(KNNGraphValue.MERGE_TOTAL_TIME_IN_MILLIS.getValue() + time_in_millis); + logger.warn("Merge operation complete in " + time_in_millis + " ms"); } } } catch (Exception e) { diff --git a/src/main/java/org/opensearch/knn/index/codec/util/KNNCodecUtil.java b/src/main/java/org/opensearch/knn/index/codec/util/KNNCodecUtil.java index eef9e6863..02ab2d833 100644 --- a/src/main/java/org/opensearch/knn/index/codec/util/KNNCodecUtil.java +++ b/src/main/java/org/opensearch/knn/index/codec/util/KNNCodecUtil.java @@ -17,30 +17,72 @@ public class KNNCodecUtil { public static final String HNSW_EXTENSION = ".hnsw"; public static final String HNSW_COMPOUND_EXTENSION = ".hnswc"; + // Floats are 4 bytes in size + public static final int FLOAT_BYTE_SIZE = 4; + // References to objects are 4 bytes in size + public static final int JAVA_REFERENCE_SIZE = 4; + // Each array in Java has a header that is 12 bytes + public static final int JAVA_ARRAY_HEADER_SIZE = 12; + // Java rounds each array size up to multiples of 8 bytes + public static final int JAVA_ROUNDING_NUMBER = 8; public static final class Pair { - public Pair(int[] docs, float[][] vectors) { + public Pair(int[] docs, float[][] vectors, SerializationMode serializationMode) { this.docs = docs; this.vectors = vectors; + this.serializationMode = serializationMode; } public int[] docs; public float[][] vectors; + public SerializationMode serializationMode; } public static KNNCodecUtil.Pair getFloats(BinaryDocValues values) throws IOException { ArrayList vectorList = new ArrayList<>(); ArrayList docIdList = new ArrayList<>(); + SerializationMode serializationMode = SerializationMode.COLLECTION_OF_FLOATS; for (int doc = values.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = values.nextDoc()) { BytesRef bytesref = values.binaryValue(); try (ByteArrayInputStream byteStream = new ByteArrayInputStream(bytesref.bytes, bytesref.offset, bytesref.length)) { + serializationMode = KNNVectorSerializerFactory.serializerModeFromStream(byteStream); final KNNVectorSerializer vectorSerializer = KNNVectorSerializerFactory.getSerializerByStreamContent(byteStream); final float[] vector = vectorSerializer.byteToFloatArray(byteStream); vectorList.add(vector); } docIdList.add(doc); } - return new KNNCodecUtil.Pair(docIdList.stream().mapToInt(Integer::intValue).toArray(), vectorList.toArray(new float[][] {})); + return new KNNCodecUtil.Pair( + docIdList.stream().mapToInt(Integer::intValue).toArray(), + vectorList.toArray(new float[][] {}), + serializationMode + ); + } + + public static long calculateArraySize(float[][] vectors, SerializationMode serializationMode) { + int vectorLength = vectors[0].length; + int numVectors = vectors.length; + if (serializationMode == SerializationMode.ARRAY) { + int vectorSize = vectorLength * FLOAT_BYTE_SIZE + JAVA_ARRAY_HEADER_SIZE; + if (vectorSize % JAVA_ROUNDING_NUMBER != 0) { + vectorSize += vectorSize % JAVA_ROUNDING_NUMBER; + } + int vectorsSize = numVectors * (vectorSize + JAVA_REFERENCE_SIZE) + JAVA_ARRAY_HEADER_SIZE; + if (vectorsSize % JAVA_ROUNDING_NUMBER != 0) { + vectorsSize += vectorsSize % JAVA_ROUNDING_NUMBER; + } + return vectorsSize; + } else { + int vectorSize = vectorLength * FLOAT_BYTE_SIZE; + if (vectorSize % JAVA_ROUNDING_NUMBER != 0) { + vectorSize += vectorSize % JAVA_ROUNDING_NUMBER; + } + int vectorsSize = numVectors * (vectorSize + JAVA_REFERENCE_SIZE); + if (vectorsSize % JAVA_ROUNDING_NUMBER != 0) { + vectorsSize += vectorsSize % JAVA_ROUNDING_NUMBER; + } + return vectorsSize; + } } public static String buildEngineFileName(String segmentName, String latestBuildVersion, String fieldName, String extension) { diff --git a/src/main/java/org/opensearch/knn/index/codec/util/KNNVectorSerializerFactory.java b/src/main/java/org/opensearch/knn/index/codec/util/KNNVectorSerializerFactory.java index f02da0949..5c1e4ca9b 100644 --- a/src/main/java/org/opensearch/knn/index/codec/util/KNNVectorSerializerFactory.java +++ b/src/main/java/org/opensearch/knn/index/codec/util/KNNVectorSerializerFactory.java @@ -56,7 +56,7 @@ public static KNNVectorSerializer getSerializerByStreamContent(final ByteArrayIn return getSerializerBySerializationMode(serializationMode); } - private static SerializationMode serializerModeFromStream(ByteArrayInputStream byteStream) { + static SerializationMode serializerModeFromStream(ByteArrayInputStream byteStream) { int numberOfAvailableBytesInStream = byteStream.available(); if (numberOfAvailableBytesInStream < ARRAY_HEADER_OFFSET) { return getSerializerOrThrowError(numberOfAvailableBytesInStream, COLLECTION_OF_FLOATS); diff --git a/src/main/java/org/opensearch/knn/plugin/stats/KNNGraphValue.java b/src/main/java/org/opensearch/knn/plugin/stats/KNNGraphValue.java new file mode 100644 index 000000000..b33b59e36 --- /dev/null +++ b/src/main/java/org/opensearch/knn/plugin/stats/KNNGraphValue.java @@ -0,0 +1,95 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.plugin.stats; + +import java.util.concurrent.atomic.AtomicLong; + +/** + * Contains a map to keep track of different graph values + */ +public enum KNNGraphValue { + + REFRESH_TOTAL_OPERATIONS("total"), + REFRESH_TOTAL_TIME_IN_MILLIS("total_time_in_millis"), + MERGE_CURRENT_OPERATIONS("current"), + MERGE_CURRENT_DOCS("current_docs"), + MERGE_CURRENT_SIZE_IN_BYTES("current_size_in_bytes"), + MERGE_TOTAL_OPERATIONS("total"), + MERGE_TOTAL_TIME_IN_MILLIS("total_time_in_millis"), + MERGE_TOTAL_DOCS("total_docs"), + MERGE_TOTAL_SIZE_IN_BYTES("total_size_in_bytes"); + + private String name; + private AtomicLong value; + + /** + * Constructor + * + * @param name name of the graph value + */ + KNNGraphValue(String name) { + this.name = name; + this.value = new AtomicLong(0); + } + + /** + * Get name of value + * + * @return name + */ + public String getName() { + return name; + } + + /** + * Get the graph value + * + * @return value + */ + public Long getValue() { + return value.get(); + } + + /** + * Increment the graph value + */ + public void increment() { + value.getAndIncrement(); + } + + /** + * Decrement the graph value + */ + public void decrement() { + value.getAndDecrement(); + } + + /** + * Increment the graph value by a specified amount + * + * @param delta The amount to increment + */ + public void incrementBy(long delta) { + value.getAndAdd(delta); + } + + /** + * Decrement the graph value by a specified amount + * + * @param delta The amount to decrement + */ + public void decrementBy(long delta) { + value.set(value.get() - delta); + } + + /** + * @param value graph value + * Set the graph value + */ + public void set(long value) { + this.value.set(value); + } +} diff --git a/src/main/java/org/opensearch/knn/plugin/stats/KNNStats.java b/src/main/java/org/opensearch/knn/plugin/stats/KNNStats.java index 66b3f215b..07d129652 100644 --- a/src/main/java/org/opensearch/knn/plugin/stats/KNNStats.java +++ b/src/main/java/org/opensearch/knn/plugin/stats/KNNStats.java @@ -24,6 +24,7 @@ import java.time.temporal.ChronoUnit; import java.util.HashMap; import java.util.Map; +import java.util.function.Supplier; /** * Class represents all stats the plugin keeps track of @@ -84,6 +85,7 @@ private Map> buildStatsMap() { addEngineStats(builder); addScriptStats(builder); addModelStats(builder); + addGraphStats(builder); return builder.build(); } @@ -169,4 +171,31 @@ private void addModelStats(ImmutableMap.Builder> builder) { new KNNStat<>(false, new NativeMemoryCacheManagerSupplier<>(NativeMemoryCacheManager::getTrainingSizeAsPercentage)) ); } + + private void addGraphStats(ImmutableMap.Builder> builder) { + builder.put(StatNames.GRAPH_STATS.getName(), new KNNStat<>(false, new Supplier>>() { + @Override + public Map> get() { + return createGraphStatsMap(); + } + })); + } + + private Map> createGraphStatsMap() { + Map mergeMap = new HashMap<>(); + mergeMap.put(KNNGraphValue.MERGE_CURRENT_OPERATIONS.getName(), KNNGraphValue.MERGE_CURRENT_OPERATIONS.getValue()); + mergeMap.put(KNNGraphValue.MERGE_CURRENT_DOCS.getName(), KNNGraphValue.MERGE_CURRENT_DOCS.getValue()); + mergeMap.put(KNNGraphValue.MERGE_CURRENT_SIZE_IN_BYTES.getName(), KNNGraphValue.MERGE_CURRENT_SIZE_IN_BYTES.getValue()); + mergeMap.put(KNNGraphValue.MERGE_TOTAL_OPERATIONS.getName(), KNNGraphValue.MERGE_TOTAL_OPERATIONS.getValue()); + mergeMap.put(KNNGraphValue.MERGE_TOTAL_TIME_IN_MILLIS.getName(), KNNGraphValue.MERGE_TOTAL_TIME_IN_MILLIS.getValue()); + mergeMap.put(KNNGraphValue.MERGE_TOTAL_DOCS.getName(), KNNGraphValue.MERGE_TOTAL_DOCS.getValue()); + mergeMap.put(KNNGraphValue.MERGE_TOTAL_SIZE_IN_BYTES.getName(), KNNGraphValue.MERGE_TOTAL_SIZE_IN_BYTES.getValue()); + Map refreshMap = new HashMap<>(); + refreshMap.put(KNNGraphValue.REFRESH_TOTAL_OPERATIONS.getName(), KNNGraphValue.REFRESH_TOTAL_OPERATIONS.getValue()); + refreshMap.put(KNNGraphValue.REFRESH_TOTAL_TIME_IN_MILLIS.getName(), KNNGraphValue.REFRESH_TOTAL_TIME_IN_MILLIS.getValue()); + Map> graphStatsMap = new HashMap<>(); + graphStatsMap.put(StatNames.MERGE.getName(), mergeMap); + graphStatsMap.put(StatNames.REFRESH.getName(), refreshMap); + return graphStatsMap; + } } diff --git a/src/main/java/org/opensearch/knn/plugin/stats/StatNames.java b/src/main/java/org/opensearch/knn/plugin/stats/StatNames.java index a098dd8b5..e9ed2b126 100644 --- a/src/main/java/org/opensearch/knn/plugin/stats/StatNames.java +++ b/src/main/java/org/opensearch/knn/plugin/stats/StatNames.java @@ -41,7 +41,10 @@ public enum StatNames { TRAINING_MEMORY_USAGE("training_memory_usage"), TRAINING_MEMORY_USAGE_PERCENTAGE("training_memory_usage_percentage"), SCRIPT_QUERY_ERRORS(KNNCounter.SCRIPT_QUERY_ERRORS.getName()), - KNN_QUERY_WITH_FILTER_REQUESTS(KNNCounter.KNN_QUERY_WITH_FILTER_REQUESTS.getName()); + KNN_QUERY_WITH_FILTER_REQUESTS(KNNCounter.KNN_QUERY_WITH_FILTER_REQUESTS.getName()), + GRAPH_STATS("graph_stats"), + REFRESH("refresh"), + MERGE("merge"); private String name; diff --git a/src/test/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumerTests.java b/src/test/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumerTests.java index 58f4b6e39..6af83de87 100644 --- a/src/test/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumerTests.java +++ b/src/test/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumerTests.java @@ -39,12 +39,14 @@ import org.opensearch.knn.indices.ModelState; import org.opensearch.knn.jni.JNIService; import org.opensearch.knn.plugin.stats.KNNCounter; +import org.opensearch.knn.plugin.stats.KNNGraphValue; import java.io.IOException; import java.util.Map; import java.util.concurrent.ExecutionException; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.doNothing; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; @@ -94,7 +96,7 @@ public void testAddBinaryField_withKNN() throws IOException { KNN80DocValuesConsumer knn80DocValuesConsumer = new KNN80DocValuesConsumer(delegate, null) { @Override - public void addKNNBinaryField(FieldInfo field, DocValuesProducer valuesProducer) { + public void addKNNBinaryField(FieldInfo field, DocValuesProducer valuesProducer, boolean isMerge, boolean isRefresh) { called[0] = true; } }; @@ -118,7 +120,7 @@ public void testAddBinaryField_withoutKNN() throws IOException { KNN80DocValuesConsumer knn80DocValuesConsumer = new KNN80DocValuesConsumer(delegate, null) { @Override - public void addKNNBinaryField(FieldInfo field, DocValuesProducer valuesProducer) { + public void addKNNBinaryField(FieldInfo field, DocValuesProducer valuesProducer, boolean isMerge, boolean isRefresh) { called[0] = true; } }; @@ -133,9 +135,17 @@ public void testAddKNNBinaryField_noVectors() throws IOException { // When there are no new vectors, no more graph index requests should be added RandomVectorDocValuesProducer randomVectorDocValuesProducer = new RandomVectorDocValuesProducer(0, 128); Long initialGraphIndexRequests = KNNCounter.GRAPH_INDEX_REQUESTS.getCount(); + Long initialRefreshOperations = KNNGraphValue.REFRESH_TOTAL_OPERATIONS.getValue(); + Long initialMergeOperations = KNNGraphValue.MERGE_TOTAL_OPERATIONS.getValue(); + Long initialMergeSize = KNNGraphValue.MERGE_TOTAL_SIZE_IN_BYTES.getValue(); + Long initialMergeDocs = KNNGraphValue.MERGE_TOTAL_DOCS.getValue(); KNN80DocValuesConsumer knn80DocValuesConsumer = new KNN80DocValuesConsumer(null, null); - knn80DocValuesConsumer.addKNNBinaryField(null, randomVectorDocValuesProducer); + knn80DocValuesConsumer.addKNNBinaryField(null, randomVectorDocValuesProducer, true, true); assertEquals(initialGraphIndexRequests, KNNCounter.GRAPH_INDEX_REQUESTS.getCount()); + assertEquals(initialRefreshOperations, KNNGraphValue.REFRESH_TOTAL_OPERATIONS.getValue()); + assertEquals(initialMergeOperations, KNNGraphValue.MERGE_TOTAL_OPERATIONS.getValue()); + assertEquals(initialMergeSize, KNNGraphValue.MERGE_TOTAL_SIZE_IN_BYTES.getValue()); + assertEquals(initialMergeDocs, KNNGraphValue.MERGE_TOTAL_DOCS.getValue()); } public void testAddKNNBinaryField_fromScratch_nmslibCurrent() throws IOException { @@ -174,10 +184,13 @@ public void testAddKNNBinaryField_fromScratch_nmslibCurrent() throws IOException FieldInfos fieldInfos = new FieldInfos(fieldInfoArray); SegmentWriteState state = new SegmentWriteState(null, directory, segmentInfo, fieldInfos, null, IOContext.DEFAULT); + long initialRefreshOperations = KNNGraphValue.REFRESH_TOTAL_OPERATIONS.getValue(); + long initialMergeOperations = KNNGraphValue.MERGE_TOTAL_OPERATIONS.getValue(); + // Add documents to the field KNN80DocValuesConsumer knn80DocValuesConsumer = new KNN80DocValuesConsumer(null, state); RandomVectorDocValuesProducer randomVectorDocValuesProducer = new RandomVectorDocValuesProducer(docsInSegment, dimension); - knn80DocValuesConsumer.addKNNBinaryField(fieldInfoArray[0], randomVectorDocValuesProducer); + knn80DocValuesConsumer.addKNNBinaryField(fieldInfoArray[0], randomVectorDocValuesProducer, true, true); // The document should be created in the correct location String expectedFile = KNNCodecUtil.buildEngineFileName(segmentName, knnEngine.getVersion(), fieldName, knnEngine.getExtension()); @@ -188,6 +201,12 @@ public void testAddKNNBinaryField_fromScratch_nmslibCurrent() throws IOException // The document should be readable by nmslib assertLoadableByEngine(state, expectedFile, knnEngine, spaceType, dimension); + + // The graph creation statistics should be updated + assertEquals(1 + initialRefreshOperations, (long) KNNGraphValue.REFRESH_TOTAL_OPERATIONS.getValue()); + assertEquals(1 + initialMergeOperations, (long) KNNGraphValue.MERGE_TOTAL_OPERATIONS.getValue()); + assertNotEquals(0, (long) KNNGraphValue.MERGE_TOTAL_DOCS.getValue()); + assertNotEquals(0, (long) KNNGraphValue.MERGE_TOTAL_SIZE_IN_BYTES.getValue()); } public void testAddKNNBinaryField_fromScratch_nmslibLegacy() throws IOException { @@ -218,10 +237,13 @@ public void testAddKNNBinaryField_fromScratch_nmslibLegacy() throws IOException FieldInfos fieldInfos = new FieldInfos(fieldInfoArray); SegmentWriteState state = new SegmentWriteState(null, directory, segmentInfo, fieldInfos, null, IOContext.DEFAULT); + long initialRefreshOperations = KNNGraphValue.REFRESH_TOTAL_OPERATIONS.getValue(); + long initialMergeOperations = KNNGraphValue.MERGE_TOTAL_OPERATIONS.getValue(); + // Add documents to the field KNN80DocValuesConsumer knn80DocValuesConsumer = new KNN80DocValuesConsumer(null, state); RandomVectorDocValuesProducer randomVectorDocValuesProducer = new RandomVectorDocValuesProducer(docsInSegment, dimension); - knn80DocValuesConsumer.addKNNBinaryField(fieldInfoArray[0], randomVectorDocValuesProducer); + knn80DocValuesConsumer.addKNNBinaryField(fieldInfoArray[0], randomVectorDocValuesProducer, true, true); // The document should be created in the correct location String expectedFile = KNNCodecUtil.buildEngineFileName(segmentName, knnEngine.getVersion(), fieldName, knnEngine.getExtension()); @@ -232,6 +254,12 @@ public void testAddKNNBinaryField_fromScratch_nmslibLegacy() throws IOException // The document should be readable by nmslib assertLoadableByEngine(state, expectedFile, knnEngine, spaceType, dimension); + + // The graph creation statistics should be updated + assertEquals(1 + initialRefreshOperations, (long) KNNGraphValue.REFRESH_TOTAL_OPERATIONS.getValue()); + assertEquals(1 + initialMergeOperations, (long) KNNGraphValue.MERGE_TOTAL_OPERATIONS.getValue()); + assertNotEquals(0, (long) KNNGraphValue.MERGE_TOTAL_DOCS.getValue()); + assertNotEquals(0, (long) KNNGraphValue.MERGE_TOTAL_SIZE_IN_BYTES.getValue()); } public void testAddKNNBinaryField_fromScratch_faissCurrent() throws IOException { @@ -269,10 +297,13 @@ public void testAddKNNBinaryField_fromScratch_faissCurrent() throws IOException FieldInfos fieldInfos = new FieldInfos(fieldInfoArray); SegmentWriteState state = new SegmentWriteState(null, directory, segmentInfo, fieldInfos, null, IOContext.DEFAULT); + long initialRefreshOperations = KNNGraphValue.REFRESH_TOTAL_OPERATIONS.getValue(); + long initialMergeOperations = KNNGraphValue.MERGE_TOTAL_OPERATIONS.getValue(); + // Add documents to the field KNN80DocValuesConsumer knn80DocValuesConsumer = new KNN80DocValuesConsumer(null, state); RandomVectorDocValuesProducer randomVectorDocValuesProducer = new RandomVectorDocValuesProducer(docsInSegment, dimension); - knn80DocValuesConsumer.addKNNBinaryField(fieldInfoArray[0], randomVectorDocValuesProducer); + knn80DocValuesConsumer.addKNNBinaryField(fieldInfoArray[0], randomVectorDocValuesProducer, true, true); // The document should be created in the correct location String expectedFile = KNNCodecUtil.buildEngineFileName(segmentName, knnEngine.getVersion(), fieldName, knnEngine.getExtension()); @@ -283,6 +314,12 @@ public void testAddKNNBinaryField_fromScratch_faissCurrent() throws IOException // The document should be readable by faiss assertLoadableByEngine(state, expectedFile, knnEngine, spaceType, dimension); + + // The graph creation statistics should be updated + assertEquals(1 + initialRefreshOperations, (long) KNNGraphValue.REFRESH_TOTAL_OPERATIONS.getValue()); + assertEquals(1 + initialMergeOperations, (long) KNNGraphValue.MERGE_TOTAL_OPERATIONS.getValue()); + assertNotEquals(0, (long) KNNGraphValue.MERGE_TOTAL_DOCS.getValue()); + assertNotEquals(0, (long) KNNGraphValue.MERGE_TOTAL_SIZE_IN_BYTES.getValue()); } public void testAddKNNBinaryField_fromModel_faiss() throws IOException, ExecutionException, InterruptedException { @@ -345,10 +382,13 @@ public void testAddKNNBinaryField_fromModel_faiss() throws IOException, Executio FieldInfos fieldInfos = new FieldInfos(fieldInfoArray); SegmentWriteState state = new SegmentWriteState(null, directory, segmentInfo, fieldInfos, null, IOContext.DEFAULT); + long initialRefreshOperations = KNNGraphValue.REFRESH_TOTAL_OPERATIONS.getValue(); + long initialMergeOperations = KNNGraphValue.MERGE_TOTAL_OPERATIONS.getValue(); + // Add documents to the field KNN80DocValuesConsumer knn80DocValuesConsumer = new KNN80DocValuesConsumer(null, state); RandomVectorDocValuesProducer randomVectorDocValuesProducer = new RandomVectorDocValuesProducer(docsInSegment, dimension); - knn80DocValuesConsumer.addKNNBinaryField(fieldInfoArray[0], randomVectorDocValuesProducer); + knn80DocValuesConsumer.addKNNBinaryField(fieldInfoArray[0], randomVectorDocValuesProducer, true, true); // The document should be created in the correct location String expectedFile = KNNCodecUtil.buildEngineFileName(segmentName, knnEngine.getVersion(), fieldName, knnEngine.getExtension()); @@ -359,6 +399,13 @@ public void testAddKNNBinaryField_fromModel_faiss() throws IOException, Executio // The document should be readable by faiss assertLoadableByEngine(state, expectedFile, knnEngine, spaceType, dimension); + + // The graph creation statistics should be updated + assertEquals(1 + initialRefreshOperations, (long) KNNGraphValue.REFRESH_TOTAL_OPERATIONS.getValue()); + assertEquals(1 + initialMergeOperations, (long) KNNGraphValue.MERGE_TOTAL_OPERATIONS.getValue()); + assertNotEquals(0, (long) KNNGraphValue.MERGE_TOTAL_DOCS.getValue()); + assertNotEquals(0, (long) KNNGraphValue.MERGE_TOTAL_SIZE_IN_BYTES.getValue()); + } public void testMerge_exception() throws IOException { @@ -426,6 +473,6 @@ public void testAddBinaryField_luceneEngine_noInvocations_addKNNBinary() throws knn80DocValuesConsumer.addBinaryField(fieldInfo, docValuesProducer); verify(delegate, times(1)).addBinaryField(fieldInfo, docValuesProducer); - verify(knn80DocValuesConsumer, never()).addKNNBinaryField(any(), any()); + verify(knn80DocValuesConsumer, never()).addKNNBinaryField(any(), any(), eq(false), eq(true)); } } diff --git a/src/testFixtures/java/org/opensearch/knn/TestUtils.java b/src/testFixtures/java/org/opensearch/knn/TestUtils.java index 06d38cdf9..a9721ce14 100644 --- a/src/testFixtures/java/org/opensearch/knn/TestUtils.java +++ b/src/testFixtures/java/org/opensearch/knn/TestUtils.java @@ -14,6 +14,7 @@ import java.io.FileReader; import java.io.IOException; import org.opensearch.knn.index.SpaceType; +import org.opensearch.knn.index.codec.util.SerializationMode; import org.opensearch.knn.plugin.script.KNNScoringUtil; import java.util.Comparator; import java.util.Random; @@ -278,7 +279,7 @@ private KNNCodecUtil.Pair readIndexData(String path) throws IOException { } } - return new KNNCodecUtil.Pair(idsArray, vectorsArray); + return new KNNCodecUtil.Pair(idsArray, vectorsArray, SerializationMode.COLLECTION_OF_FLOATS); } private float[][] readQueries(String path) throws IOException { From 037da30c12c2f76ff338be4abe56d44c118531fc Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Fri, 6 Oct 2023 07:00:50 -0700 Subject: [PATCH 152/416] Add 2.11 release note (#1197) (#1202) Signed-off-by: Heemin Kim (cherry picked from commit 276dba79911cce11053fcb5c1bd45ba2a6326c92) Co-authored-by: Heemin Kim --- CHANGELOG.md | 5 +---- release-notes/opensearch-knn.release-notes-2.11.0.0.md | 9 +++++++++ 2 files changed, 10 insertions(+), 4 deletions(-) create mode 100644 release-notes/opensearch-knn.release-notes-2.11.0.0.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 1b17c8f63..9f79b0414 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,14 +12,11 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Maintenance ### Refactoring -## [Unreleased 2.x](https://github.com/opensearch-project/k-NN/compare/2.10...2.x) +## [Unreleased 2.x](https://github.com/opensearch-project/k-NN/compare/2.11...2.x) ### Features ### Enhancements -* Added support for ignore_unmapped in KNN queries. [#1071](https://github.com/opensearch-project/k-NN/pull/1071) -* Add graph creation stats to the KNNStats API. [#1141](https://github.com/opensearch-project/k-NN/pull/1141) ### Bug Fixes ### Infrastructure ### Documentation ### Maintenance -- Update bytebuddy to 1.14.7 [#1135](https://github.com/opensearch-project/k-NN/pull/1135) ### Refactoring diff --git a/release-notes/opensearch-knn.release-notes-2.11.0.0.md b/release-notes/opensearch-knn.release-notes-2.11.0.0.md new file mode 100644 index 000000000..8cb6dde36 --- /dev/null +++ b/release-notes/opensearch-knn.release-notes-2.11.0.0.md @@ -0,0 +1,9 @@ +## Version 2.11.0.0 Release Notes + +Compatible with OpenSearch 2.11.0 + +### Enhancements +* Added support for ignore_unmapped in KNN queries. [#1071](https://github.com/opensearch-project/k-NN/pull/1071) +* Add graph creation stats to the KNNStats API. [#1141](https://github.com/opensearch-project/k-NN/pull/1141) +### Maintenance +* Update bytebuddy to 1.14.7 [#1135](https://github.com/opensearch-project/k-NN/pull/1135) From def209e3fb4e9a922cbe993dc10739ffcf43fa4a Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Fri, 6 Oct 2023 19:18:43 -0700 Subject: [PATCH 153/416] Switch from rockylinux8 to centos7 docker image for legacy support before deprecation (#1204) (#1209) Signed-off-by: Peter Zhu (cherry picked from commit 4cbc2f1709e205b9414a93d888a432c3b6552444) Co-authored-by: Peter Zhu --- .github/workflows/CI.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 959a0a400..f56dc7a49 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -31,7 +31,7 @@ jobs: id: step-ci-image-version-linux run: | crane version - CI_IMAGE_VERSION=`opensearch-build/docker/ci/get-ci-images.sh -p rockylinux8 -u opensearch -t build | head -1` + CI_IMAGE_VERSION=`opensearch-build/docker/ci/get-ci-images.sh -p centos7 -u opensearch -t build | head -1` echo $CI_IMAGE_VERSION echo "ci-image-version-linux=$CI_IMAGE_VERSION" >> $GITHUB_OUTPUT From 9d2086cada4c6bb352a56366373387dd41ddb6ae Mon Sep 17 00:00:00 2001 From: Ryan Bogan <10944539+ryanbogan@users.noreply.github.com> Date: Wed, 11 Oct 2023 11:50:11 -0700 Subject: [PATCH 154/416] Fix ignore unmapped minimum version (#1239) Signed-off-by: Ryan Bogan --- src/main/java/org/opensearch/knn/index/IndexUtil.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/opensearch/knn/index/IndexUtil.java b/src/main/java/org/opensearch/knn/index/IndexUtil.java index f9809ba70..4e2dc38c6 100644 --- a/src/main/java/org/opensearch/knn/index/IndexUtil.java +++ b/src/main/java/org/opensearch/knn/index/IndexUtil.java @@ -35,7 +35,7 @@ public class IndexUtil { private static final Version MINIMAL_SUPPORTED_VERSION_FOR_LUCENE_HNSW_FILTER = Version.V_2_4_0; - private static final Version MINIMAL_SUPPORTED_VERSION_FOR_IGNORE_UNMAPPED = Version.V_2_10_0; + private static final Version MINIMAL_SUPPORTED_VERSION_FOR_IGNORE_UNMAPPED = Version.V_2_11_0; public static final Map minimalRequiredVersionMap = new HashMap() { { put("filter", MINIMAL_SUPPORTED_VERSION_FOR_LUCENE_HNSW_FILTER); From 89a23bef181200bb11fefba00947aafb005d5f9f Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Thu, 12 Oct 2023 18:34:40 -0400 Subject: [PATCH 155/416] Tweak docker retrival method to be simpler (#1249) (#1250) Signed-off-by: Peter Zhu (cherry picked from commit ca699c7459041abd0e02b0a579c6292e84863562) Co-authored-by: Peter Zhu --- .github/workflows/CI.yml | 25 +++---------------------- 1 file changed, 3 insertions(+), 22 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index f56dc7a49..e7835a1f3 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -13,28 +13,9 @@ on: jobs: Get-CI-Image-Tag: - runs-on: ubuntu-latest - outputs: - ci-image-version-linux: ${{ steps.step-ci-image-version-linux.outputs.ci-image-version-linux }} - steps: - - name: Install crane - uses: iarekylew00t/crane-installer@v1 - with: - crane-release: v0.15.2 - - name: Checkout opensearch-build repository - uses: actions/checkout@v2 - with: - repository: 'opensearch-project/opensearch-build' - ref: 'main' - path: 'opensearch-build' - - name: Get ci image version from opensearch-build repository scripts - id: step-ci-image-version-linux - run: | - crane version - CI_IMAGE_VERSION=`opensearch-build/docker/ci/get-ci-images.sh -p centos7 -u opensearch -t build | head -1` - echo $CI_IMAGE_VERSION - echo "ci-image-version-linux=$CI_IMAGE_VERSION" >> $GITHUB_OUTPUT - + uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@main + with: + product: opensearch Build-k-NN-Linux: strategy: From 9597e49ddd677ab3a1a2833b95d5a0578b9b38f9 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Mon, 16 Oct 2023 13:12:35 -0400 Subject: [PATCH 156/416] Make sure not hardcoding user name when switching to uid 1000 (#1252) (#1253) Signed-off-by: Peter Zhu (cherry picked from commit 644902ef4c3a9787a22bc53e8d5c3d212642590c) Co-authored-by: Peter Zhu --- .github/workflows/CI.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index e7835a1f3..d5a6e3f11 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -46,8 +46,8 @@ jobs: - name: Run build # switching the user, as OpenSearch cluster can only be started as root/Administrator on linux-deb/linux-rpm/windows-zip. run: | - chown -R opensearch.opensearch `pwd` - su opensearch -c "whoami && java -version && ./gradlew build" + chown -R 1000:1000 `pwd` + su `id -un 1000` -c "whoami && java -version && ./gradlew build" - name: Upload Coverage Report uses: codecov/codecov-action@v1 From 5e15d16d69c1f78396b787c19ed66884f61c7baa Mon Sep 17 00:00:00 2001 From: Ryan Bogan Date: Thu, 26 Oct 2023 13:06:45 -0700 Subject: [PATCH 157/416] [Backport 2.x] Upgrade urllib to 1.26.17 (#1282) * Upgrade urllib to 1.26.17 Signed-off-by: Ryan Bogan * Add Changelog Signed-off-by: Ryan Bogan --------- Signed-off-by: Ryan Bogan --- CHANGELOG.md | 1 + benchmarks/osb/requirements.txt | 2 +- benchmarks/perf-tool/requirements.txt | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9f79b0414..faab7d08e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,4 +19,5 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Infrastructure ### Documentation ### Maintenance +* Upgrade urllib to 1.26.17 [#1278](https://github.com/opensearch-project/k-NN/pull/1278) ### Refactoring diff --git a/benchmarks/osb/requirements.txt b/benchmarks/osb/requirements.txt index 423253c71..1d918d881 100644 --- a/benchmarks/osb/requirements.txt +++ b/benchmarks/osb/requirements.txt @@ -85,7 +85,7 @@ tabulate==0.8.7 # via opensearch-benchmark thespian==3.10.1 # via opensearch-benchmark -urllib3==1.26.9 +urllib3==1.26.17 # via opensearch-py yappi==1.2.3 # via opensearch-benchmark diff --git a/benchmarks/perf-tool/requirements.txt b/benchmarks/perf-tool/requirements.txt index 5489e63dd..ca0d30b91 100644 --- a/benchmarks/perf-tool/requirements.txt +++ b/benchmarks/perf-tool/requirements.txt @@ -30,7 +30,7 @@ pyyaml==5.4.1 # via -r requirements.in requests==2.31.0 # via -r requirements.in -urllib3==1.26.6 +urllib3==1.26.17 # via # opensearch-py # requests From 7c935ef27e4f96fe2087daf68e498949e081427b Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Mon, 30 Oct 2023 17:40:52 -0500 Subject: [PATCH 158/416] Fix Windows CI (#1281) (#1285) Signed-off-by: Naveen Tatikonda (cherry picked from commit 4d6cc169698c7bfb1435b2fa004840b28693cc2e) Co-authored-by: Naveen Tatikonda --- .github/workflows/CI.yml | 1 + jni/include/jni_util.h | 1 + src/test/java/org/opensearch/knn/index/FaissIT.java | 2 +- 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index d5a6e3f11..84b4f09f9 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -108,6 +108,7 @@ jobs: - name: Add MinGW to PATH run: | echo "C:/Users/runneradmin/scoop/apps/mingw/current/bin" >> $env:GITHUB_PATH + Import-Module "$env:ChocolateyInstall\helpers\chocolateyProfile.psm1" refreshenv - name: Download OpenBLAS diff --git a/jni/include/jni_util.h b/jni/include/jni_util.h index 8d6f2b6f6..b4dd44891 100644 --- a/jni/include/jni_util.h +++ b/jni/include/jni_util.h @@ -17,6 +17,7 @@ #include #include #include +#include namespace knn_jni { diff --git a/src/test/java/org/opensearch/knn/index/FaissIT.java b/src/test/java/org/opensearch/knn/index/FaissIT.java index 842c5af7b..41ce2be99 100644 --- a/src/test/java/org/opensearch/knn/index/FaissIT.java +++ b/src/test/java/org/opensearch/knn/index/FaissIT.java @@ -199,7 +199,7 @@ public void testEndToEnd_whenMethodIsHNSWPQ_thenSucceed() { createBasicKnnIndex(trainingIndexName, trainingFieldName, dimension); ingestDataAndTrainModel(modelId, trainingIndexName, trainingFieldName, dimension, modelDescription, in, trainingDataCount); - assertTrainingSucceeds(modelId, 180, 1000); + assertTrainingSucceeds(modelId, 360, 1000); // Create an index XContentBuilder builder = XContentFactory.jsonBuilder() From ee66aaa937b1d0bc25e917f33197bb5b85f1127a Mon Sep 17 00:00:00 2001 From: Ryan Bogan Date: Mon, 30 Oct 2023 15:43:04 -0700 Subject: [PATCH 159/416] Update developer guide to include M1 Setup (#1284) Signed-off-by: Ryan Bogan --- CHANGELOG.md | 1 + DEVELOPER_GUIDE.md | 51 +++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 51 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index faab7d08e..e3dd6155f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,5 +19,6 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Infrastructure ### Documentation ### Maintenance +* Update developer guide to include M1 Setup [#1222](https://github.com/opensearch-project/k-NN/pull/1222) * Upgrade urllib to 1.26.17 [#1278](https://github.com/opensearch-project/k-NN/pull/1278) ### Refactoring diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index 4fe31487a..582041e1d 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -63,6 +63,11 @@ One easy way to install on mac or linux is to use pip: pip install cmake==3.23.3 ``` +On Mac M1 machines, install cmake using: +```bash +brew install cmake +``` + #### Faiss Dependencies To build the *faiss* JNI library, you need to have openmp, lapack and blas installed. For more information on *faiss* @@ -78,6 +83,50 @@ Additionally, the `gcc` toolchain needs to be installed on Mac. To install, run: brew install gcc ``` +#### Extra setup for Mac M1 Machines + +The following commands enable running/building k-NN on M1 machines: + +```bash +// Go to k-NN folder +cd k-NN + +// Build to generate the necessary files to be modified below (will fail) +./gradlew build + +//Go to jni folder +cd jni + +// File changes required +sed -i -e 's/\/usr\/local\/opt\/libomp\//\/opt\/homebrew\/opt\/llvm\//g' CMakeLists.txt +sed -i -e 's/-march=native/-mcpu=apple-m1/g' external/nmslib/similarity_search/CMakeLists.txt +sed -i -e 's/pragma message WARN/pragma message /g' external/nmslib/similarity_search/src/distcomp_scalar.cc +sed -i -e 's/-mcpu=apple-a14/-mcpu=apple-m1/g' external/nmslib/python_bindings/setup.py +sed -i -e 's/__aarch64__/__undefine_aarch64__/g' external/faiss/faiss/utils/distances_simd.cpp + +// Install llvm +brew install llvm +echo 'export PATH="/opt/homebrew/opt/llvm/bin:$PATH"' >> ~/.zshrc +source ~/.zshrc + +// Set compiler path for CMAKE +export CC=/opt/homebrew/opt/llvm/bin/clang +export CXX=/opt/homebrew/opt/llvm/bin/clang++ + +// Build +cmake . --fresh +make +``` + +Next, obtain a minimum distribution tarball of the k-NN version you want to build: + +1. Fork the [OpenSearch Repo](https://github.com/opensearch-project/OpenSearch) into your github account. +2. Clone the repository locally +3. Run the following commands: +```cd OpenSearch && ./gradlew -p distribution/archives/darwin-tar assemble``` +4. You should see a opensearch-min--SNAPSHOT-darwin-x64.tar.gz file present in distribution/archives/darwin-tar/build/distributions/ +5. Build k-NN by passing the OpenSearch distribution path in `./gradlew -PcustomDistributionUrl=""` + #### Environment Currently, the plugin only supports Linux on x64 and arm platforms. @@ -114,7 +163,7 @@ Please follow these formatting guidelines: * Wildcard imports (`import foo.bar.baz.*`) are forbidden and will cause the build to fail. * If *absolutely* necessary, you can disable formatting for regions of code with the `// tag::NAME` and `// end::NAME` directives, but note that these are intended for use in documentation, so please make it clear what you have done, and only do this where the benefit clearly outweighs the decrease in consistency. * Note that JavaDoc and block comments i.e. `/* ... */` are not formatted, but line comments i.e `// ...` are. -* There is an implicit rule that negative boolean expressions should use the form `foo == false` instead of `!foo` for better readability of the code. While this isn't strictly enforced, if might get called out in PR reviews as something to change. +* There is an implicit rule that negative boolean expressions should use the form `foo == false` instead of `!foo` for better readability of the code. While this isn't strictly enforced, it might get called out in PR reviews as something to change. ## Build From 8aad0d21aac1cd042a4c8e9c8b5f1f9f696461b1 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Mon, 6 Nov 2023 11:19:43 -0800 Subject: [PATCH 160/416] Upgrade gradle to 8.4 (#1297) Upgrade gradle to 8.4 to support JDK-21. Refer to https://docs.gradle.org/current/userguide/compatibility.html Simplifies Jacoco logic. From long ago, jacoco support was added through a series of hacks. Some of these hacks are no longer necessary, so they were removed to simplify. Signed-off-by: John Mazanec --- CHANGELOG.md | 1 + build-tools/knnplugin-coverage.gradle | 74 +++++------------- build.gradle | 10 +-- gradle/wrapper/gradle-wrapper.jar | Bin 61574 -> 63721 bytes gradle/wrapper/gradle-wrapper.properties | 4 +- gradlew | 29 ++++--- .../org/opensearch/knn/KNNRestTestCase.java | 6 +- 7 files changed, 49 insertions(+), 75 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e3dd6155f..59dd618d8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Enhancements ### Bug Fixes ### Infrastructure +* Upgrade gradle to 8.4 [1289](https://github.com/opensearch-project/k-NN/pull/1289) ### Documentation ### Maintenance * Update developer guide to include M1 Setup [#1222](https://github.com/opensearch-project/k-NN/pull/1222) diff --git a/build-tools/knnplugin-coverage.gradle b/build-tools/knnplugin-coverage.gradle index 386cfcfda..eb3582dab 100644 --- a/build-tools/knnplugin-coverage.gradle +++ b/build-tools/knnplugin-coverage.gradle @@ -3,12 +3,15 @@ * SPDX-License-Identifier: Apache-2.0 */ +apply plugin: 'jacoco' + +jacoco { + toolVersion = "0.8.10" +} + /** - * ES Plugin build tools don't work with the Gradle Jacoco Plugin to report coverage out of the box. - * https://github.com/elastic/elasticsearch/issues/28867. - * - * This code sets up coverage reporting manually for ES plugin tests. This is complicated because: - * 1. The ES integTest Task doesn't implement Gradle's JavaForkOptions so we have to manually start the jacoco agent with the test JVM + * This code sets up coverage reporting manually for the k-NN plugin tests. This is complicated because: + * 1. The OS integTest Task doesn't implement Gradle's JavaForkOptions so we have to manually start the jacoco agent with the test JVM * 2. The cluster nodes are stopped using 'kill -9' which means jacoco can't dump it's execution output to a file on VM shutdown * 3. The Java Security Manager prevents JMX from writing execution output to the file. * @@ -16,66 +19,31 @@ * cluster is stopped and dump it to a file. Luckily our current security policy seems to allow this. This will also probably * break if there are multiple nodes in the integTestCluster. But for now... it sorta works. */ - -import org.apache.tools.ant.taskdefs.condition.Os -apply plugin: 'jacoco' - -// Get gradle to generate the required jvm agent arg for us using a dummy tasks of type Test. Unfortunately Elastic's -// testing tasks don't derive from Test so the jacoco plugin can't do this automatically. -def jacocoDir = "${buildDir}/jacoco" -task dummyTest(type: Test) { - enabled = false - workingDir = file("/") // Force absolute path to jacoco agent jar - jacoco { - destinationFile = file("${jacocoDir}/test.exec") - destinationFile.parentFile.mkdirs() - jmx = true - } -} - -task dummyIntegTest(type: Test) { - enabled = false - workingDir = file("/") // Force absolute path to jacoco agent jar +integTest { jacoco { - destinationFile = file("${jacocoDir}/integTest.exec") - destinationFile.parentFile.mkdirs() jmx = true } -} -integTest { - systemProperty 'jacoco.dir', "${jacocoDir}" + systemProperty 'jacoco.dir', project.layout.buildDirectory.get().file("jacoco").asFile.absolutePath systemProperty 'jmx.serviceUrl', "service:jmx:rmi:///jndi/rmi://127.0.0.1:7777/jmxrmi" } jacocoTestReport { dependsOn integTest, test - executionData.from = [dummyTest.jacoco.destinationFile, dummyIntegTest.jacoco.destinationFile] - sourceDirectories.from = sourceSets.main.java.sourceDirectories - classDirectories.from = files(sourceSets.main.java.outputDir) - + executionData.from = [integTest.jacoco.destinationFile, test.jacoco.destinationFile] reports { - html.enabled = true // human readable - csv.enabled = true - xml.enabled = true // for coverlay + html.getRequired().set(true) // human readable + csv.getRequired().set(true) + xml.getRequired().set(true) // for coverlay } } -afterEvaluate { - jacocoTestReport.dependsOn integTest +testClusters.integTest { + jvmArgs " ${integTest.jacoco.getAsJvmArg()}" - testClusters.integTest { - if (Os.isFamily(Os.FAMILY_WINDOWS)) { - // Replacing build with absolute path to fix the error "error opening zip file or JAR manifest missing : /build/tmp/expandedArchives/..../jacocoagent.jar" - jvmArgs " ${dummyIntegTest.jacoco.getAsJvmArg()}".replace('build',"${buildDir}") - } else { - jvmArgs " ${dummyIntegTest.jacoco.getAsJvmArg()}".replace('javaagent:','javaagent:/') - } - - systemProperty 'com.sun.management.jmxremote', "true" - systemProperty 'com.sun.management.jmxremote.authenticate', "false" - systemProperty 'com.sun.management.jmxremote.port', "7777" - systemProperty 'com.sun.management.jmxremote.ssl', "false" - systemProperty 'java.rmi.server.hostname', "127.0.0.1" - } + systemProperty 'com.sun.management.jmxremote', "true" + systemProperty 'com.sun.management.jmxremote.authenticate', "false" + systemProperty 'com.sun.management.jmxremote.port', "7777" + systemProperty 'com.sun.management.jmxremote.ssl', "false" + systemProperty 'java.rmi.server.hostname', "127.0.0.1" } diff --git a/build.gradle b/build.gradle index 1960dd220..8c8db8340 100644 --- a/build.gradle +++ b/build.gradle @@ -36,9 +36,8 @@ plugins { id 'java-library' id 'java-test-fixtures' id 'idea' - id 'jacoco' id "com.diffplug.spotless" version "6.3.0" apply false - id 'io.freefair.lombok' version '6.4.3' + id 'io.freefair.lombok' version '8.4' } apply from: 'gradle/formatting.gradle' @@ -123,6 +122,9 @@ compileJava { compileTestJava { options.compilerArgs.addAll(["-processor", 'lombok.launch.AnnotationProcessorHider$AnnotationProcessor']) } +compileTestFixturesJava { + options.compilerArgs.addAll(["-processor", 'lombok.launch.AnnotationProcessorHider$AnnotationProcessor']) +} def usingRemoteCluster = System.properties.containsKey('tests.rest.cluster') || System.properties.containsKey('tests.cluster') def usingMultiNode = project.properties.containsKey('numNodes') @@ -135,10 +137,6 @@ if (!usingRemoteCluster) { } } -jacoco { - toolVersion = "0.8.7" -} - check.dependsOn spotlessCheck check.dependsOn jacocoTestReport diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar index 943f0cbfa754578e88a3dae77fce6e3dea56edbf..7f93135c49b765f8051ef9d0a6055ff8e46073d8 100644 GIT binary patch delta 41154 zcmZ6yV|*sjvn`xVY}>YN+qUiGiTT8~ZQHhOPOOP0b~4GlbI-Z&x%YoRb#?9CzwQrJ zwRYF46@CbIaSsNeEC&XTo&pMwk%Wr|ik`&i0{UNfNZ=qKAWi@)CNPlyvttY6zZX-$ zK?y+7TS!42VgEUj;GF);E&ab2jo@+qEqcR8|M+(SM`{HB=MOl*X_-g!1N~?2{xi)n zB>$N$HJB2R|2+5jmx$;fAkfhNUMT`H8bxB3azUUBq`}|Bq^8EWjl{Ts@DTy0uM7kv zi7t`CeCti?Voft{IgV-F(fC2gvsaRj191zcu+M&DQl~eMCBB{MTmJHUoZHIUdVGA% zXaGU=qAh}0qQo^t)kR4|mKqKL-8sZQ>7-*HLFJa@zHy0_y*ua!he6^d1jMqjXEv;g z5|1we^OocE*{vq+yeYEhYL;aDUDejtRjbSCrzJ&LlFbFGZL7TtOu9F={y4$O^=evX zz%#OSQay8o6=^_YM(5N-H<35|l3C7QZUF@7aH=;k!R!Vzj=bMzl$**|Ne<1TYsn?T z@98M0#ZL9=Q&XFBoJ_Jf<0Fn;OcCl5x^koelbG4BbjMQ>*!nE0yT@6k7A+ebv`X1w zt|Xjn4FVXX9-Gr+Eak=408_Fui&@?foGz6qak-tHu>2o@ZVRQ-X;HZhb1Hw|ZAoxx z!)Cn4hxBI}ZbBCOTp3L63EU3Wv1dxk@J?)0_#oYR7HOP5Yx6W3jnagH;c}y$G^}eN z_gNT{1AanZ<}mw2ELMxx@ZzZ(2RvE4c)lH8c7Gi~3R2#hx}p9!hKPMW>ekYbK86>N zL&7Ky#*zv-P4iuIQ5RV(+vKjmwl+P}KH+$~xd=b5Dx1{hqqu0tbG{fYWstL&Kcz*d zOc@$}f?5vBmO8f3pj<+2PO7R}Jd6N{qRexKo>ElNYgVeYkyhIUY}X%clJ>unwsuOm z;;>SVKUJt$Kgz4Ax?PKY8F>##IJuP>EQ5R;Cq6}Xuvz;%La(_I4j$jv%s z_v}|apMsrN_%S~~HmEwu3RG@~x!CES{G~n#-()k{<4D?L%JT%I>3r{ML&;j7U#{u0 zJ?Wc+C3`^378b`@&yD4v8!cjFCp`ed7Vun)3h1Mkly&;(&fuUsq`8F2oWWnBfh9v! z%)WBwE2S9RJJIEHjIzyFh7TbyvbDRCqs zz`u%UBFGa1z6^Z;hSo~r?|SGTS_dE)60uPS35n|LB018jWS`wU7vFvrB4e$T&m zHc|hf8hn9fWZKeyH(lwiTQ1#0@gld4;-h@NX+Rzmyy}R9oxYJVHoXb zyV@nf36;c=c`b21vH@(g3?J$vx=?@!?R$yVrnPrplW!cQS})U%>{%lmdXH)bK|}WB zcslr*h|XiL-|~x4Ki6AvE3d+lTEd33pE)hY`fn@yv8^AoR52`*L^Kh!TF%3Zj&Vo) z=)bDG$a-IkN7fJsTT4x6FFNyqV+gZs@`P2OIF#{#7x)$_Cxj2bW2H2c)@w~>M9-`> z4Rw#yV$w+Qv?+!cb>ZXasldjG=R;#7T0@G-UcsiUBp%^VX-Dc8J_GSU8yDRiKwU|c zWvpbDr3EA4NPJjox0F|pxJqXQs*5zW32Z1yt8f{bm&ngF4za}c3?5YO)hu10?0t>G z?ULZt7!+Z}hMH(DP{TvGVkLv~GA_zNQf_1_ni6^ym;89EzQ5#iE4m6n-r2uEvoizl zq5cbd{wH>EyOaK;1d^KqLzrk_GD1tax$Dq$Q})b@IuYAblTIlc7NyShO4+UxQ!h@9 z`1~UTW%+i=c#J0?vlJ~q&h%e?Z+*S2@M z9)%F6JI5V&Z_>NgLbq|?usS;Lz#Hcsr^jx;DUTy_azC&RZ=O&Cop&s-TL-CH84KYl~J8>BsHHR%FFg^brE_t={xLMsXGwF zIyCKUONvr-f1;TKTPsMS*((XEUx+LCFvCe!sDD;lU=eO>tQ@>$nrs^M^q((M>TR#Q zOI>o=R+r!OkY1EKbUNuYY&$~TEk$WBzF19Z=DLh}j4c%g5#bz8au{mO(Tbi7uvF$Khaa+4M=?LiGQV#Lt>t>bsPrzJ1l+$MHNZAg*yv2Aj^GPdOj?yc~aVqIC*@K@(1i)SWh_{G{A zG1@USpgj^;P7~3AZ~V|GoHJ2?7%^R(%z)V*M!^T-q5otVw?hcavR3}JStYt4!&fXD z1+e)IzeoW7Z+C(-4G(4Cs?Tv2T4LY_Vi&j`Y32s=e7#vP1KE&fqM6+)W7s0H-(S1iQEl`JtY37ONAZL+Nu$hJdF28aC@KL1>?4iXE{ODGHT*$J!M(}w| z?iMo7ViHWSXq^tSRA9d49%mjWkK}6`jDOB=bRBJKkM^)P5DObI%N@QWmwBtA`U5as zY$MJ>tCT^Cl?=nqgIhYmmXxgSlTJp?*nuQde}DXE0r*uaEGzc|1QO)--|@1i^EYRU z-jUJ0(A^Onr66{}m%_N0m8V*Wgx!(Y+58UA>yEFY)xg)=ABaIlk4IPQu;Ff z^U0cjG$rBb6bPd4&~HD7 zuilr*e$ya*bYJ1slNQmcQRBfYGVv^7U*TP&1&j+6K!Gtya8k0ZVXlRaXonBQud{(- z8{H;11N->}EsfRH&PRJ+Zvv6nmNL5gZt^1ycQR+y^$-cE4ysf=aesOre{qVP8ZE-N z5b!{I@h=~}ezVU}r}w|kH1)|0eTt{uhLWwJF_ooj=394^#ps{7%#C64V{PAIM-QlV zWljWxJv?vy{cg$FR1<-R)1ooe&bh%H@q1B31dgl|L#Hi%;b1m+v3-Qi#xKFwtej6F zMD#OP7dy=d7x@>b$WbMbmRN5H4!ud^fkEiH^4c)#SM=rlV2(hQC})_B#wcQlF8lZe zG5d9j)R?jGyvJKno5h^QKFplNMt_2USAR%e+t$izw$>w&nxaUtQ<^8j*4Y`hJ=&70 zX!}IKNGDkF?b-aTbUt6IUAZ-_H)qqB}z z!Oxw~3$9y#kV1rG*7QSA92I_QlZsfNs`aV()|gms1UM2eQcsq<@USs>c&Gp?rddNQ zEV(xadXNq%+{o-xVl40Gp9^W}smgI{@XyRnBS|vC^n18J$sI&VO$Z4O<7O!Q^QmAM z=VJ|CUZTSd-k)5(U*-_`!=NxqE$3{g0d$9+KcYE)<3axb{$^F! zy^*(#FX8*az%oN7PXD!W!#xk;cyKXPlk#REJfCc@D3GUbxUdbf3 zgKAiY3UkwLeALOY#IYIP>YMzVjl!=0xvd{+phh(_O7tE9qy4gb>yre|RzH3^lT zWrRQ??y`cGvDufpSH>KBD+)tNgKaf$kj^Of{&pP#R7K8Q)1rNc)c#pAknYFKm6g5g zOW=*;dhTx-*{h7*GlF>Xh!oxu^ZvA7xfcsG7i<(iMKq?ht{pz!I?YZzNOki^74gx-@+C`zFrDH5GU4uDsNnfkcmY zQbAo?mp6?L4ni5+PG2%Zz&h=kLQn?S^b(Dt8DLm&ns$jXoaqk)El;XE@SK;iXX0wQ z;Olbo>zZ$ds`WKqciZ7*g0)utwY8VaYRl@26NmB|nw(xe&+Db*ldXdLA3d+d!5Pld z#$pjwmtrF~-?5pz)jXGt4sqBp0!26N_8b8iD|4ubbY3_O)aT;{K-ll#%wV!e8E)Ff zZt9=A;m691@9&~gi1oqV5Es86S%S0^+zH~VOTzgoDcz_X@d(}Xq%@uJsnC0)Q&1IY z-slwRxI@HX4M(nEzsE&vZxtyFLZ+F_)>Ne2^$IA3VfO}gAb?iJd!u^Zp!ak#LpeXGXMcSS#4&+DJBT91RSM<{qPz8@SJTKl;oJiy+6QQ@VK$5PjOa zD+x}7a3gCeP*X}*EGre%RbJ1fDeIQx!HOK|aONo)ukFgyfI!6{f)z*54Oco>&mI9i z;18~KEb$7_mh|HUv5!txYFdUQRaHc4J$-H^`SruU<8nJI(%i<(vp!&63A z!=>cO@-l5t{(3p5DoxawpiZul&;+*%46Q7W8tOty9cNCiNcm!@cTBA*_Sge^l>@eE0yb+7& z_G2$v0AnxOpW$Bfw?kEjDNw8x$j1q>M?gh4yM{&(@rM;tUsM8^hWY_z`J5riM7;CK zXlXQxK*Ska!rCWbb;(&bgG;Hb5qw>0eZ#Y?eVJDrz8L6*knEMm4+N7N(`k+2TB6u{ zP*lDK>Mi6JLU|r2J~*(|iBapcCaxQF(%pGfoCzq)y_CA_cws+oJ%9&=jAXjQtbN5k zAkClhvE(E$F&65^ij?_t*1kpm7|9VZEJ95(6bfqN%+8`g)#l5IQpmhG`ofn;5>7hk z2xnq?L2V}~_8;0Ll(dVlX(LSJO0x+1jr6Vw{Bo%vNJRugYT&*KUaL3&}YH4OWt#%tJVil>0MY&zxM zvAMLu22RDvj^Z_sa*ao26u32j#Gbhope{6`+4?eF)` zE3QBt`YUPT2C^v8Lt3;Or%uLTrW8xK5 zqLEc(9k<4`l{8L0=Vea0-xQYvFOQB(duQK#S=rMa^RK=p>fI!(^ef$BOyb)qUF|i~ zTl#JvRhkRlzl}D@lzj(;62K{qy$1rr=B~=Lb$%JgnRkS6>I{yw{h}QBka+IE&GX>% zAJ+|^G*Y#^rb6nMgMPQ3GkuC1B4U!BUk;Dd)rpy`_Yr1&E2!i z^7vz6B1W#bfEhpYDh3<@bGEu{6Jux__bwaZ2^g?PY_`Tg39vJlA>bfG>_pQj^Zq_6 zi#$Qa0DQ}Y6R}vkCm%Lt0&{NR63oo55%F%pOS?lg^XX1ghs3MiQf1Dt+2j*IGJMZa z#;0K^rLufIwaWc(uyfHqLcf`(@H^dMl)6c&#e6xWQ_(k zRz=x*OVFt#$cTpB?i@m*D8nm*lFVev555nBCQr+JihUaz;5fsw6-=qeW9iHz&hX|F zS&VP=r( zbO+X0bOM!y4TuJgS-&=u(*nR@cH5dzCPjGU>oS0CMPQMj^F@SYX(rvl+Y_76GURaR zp^G)7`Er$dE7Z-tH5)^X|2PfO8!}okjcZz8d-)|VT0R3v@@&4{g70e)0cTWq;*xOm z(e039+BRgcLB1nuoSwBO|5QIk3DjemLfsP#H=)+^8#8+J3)z15n?g%BFq#&yf_7EO zfboQ=qKNN1+=K$ZC!5;4mB7lqUt<5XQQP&I?f8PVp{Ss!{*_G;r@nDPQ&mY8R2sjM zxw4d?#_I?))gJ4O*V9&Rsx*U{fp-ncs_ng#Z?c5hplhQI$TVrp(5v3H%;YCL3+Ss1 z@~NQVv3~ibw5b*z1+1!z?twQOa?Q`OS#VheAa&;=;`&|UHmni$-h(qeO3wV5F;DBM z>Rzon?A7Hk;9}!a=XHn0klvPBC)cbM32aD#8!3$18Lf;z1s zG}(1&!y$ehWEo1unGS_G3z!!A`(GAjnMmxq6>>m{LCm?+e-_slha9vVFc1)#e+&xO z{}k7K4#<>CZWN%#E?`9x{d+x~OoDohJ4$Ssh&WVN)-)Gf);hNw=GQ`HPus_XphMt>}b*b=*@rzV<@1ijU?f6raCIlI+Jv) z_0^LwE%@~_m9Py3lW*#h3gZajMH(|r!5rbOj`l3l7#$X@_;ot*I=44BnR^WVW+{|f zt~onHYA&99JI6s+EY=zmEPc^){`=&kUD;P{at;X{_ARTe zb*LtuT`NFT6Gy-TS6^0$;50mdO<$$Z?t=u8bmqZ0RE46zk=w{TlhFPSwqLyMMt7K2 z%Xg6IA$cy(qYA|k zb)SKGwihPbq|>C0fY40>&8}gl98cThVt>8?(GfU{+og%;xM7#A#h_x_&-6#Y!tAf80_?y=XIxJt2Q&4q!8vC7 z?^~enOF_MOt1-6R5rje3P%fEa>l`txDAwOh$KS`=Bk+;j$DeuIoDi{%Hr*1dYJKUg z1@ddnOA9vBgGilNZyj|9f)XpAPPHx(go4{{KYs`#5%s~11b9v)@UYZt#g*C#j`9(# z*s!3d_`Ot_ek2y5cK*F{kXLdukiN@AE{O(0_zWb3m?Zb3p{gD|EM5}mrb)9VXKe|T z0?TD!ZawCi>si-w93t>jw&I?a!^WwqoIfVWxOt@cl6BJ z9Xl_11OE;aC;o4y$JGf7{3p2eau=Jc)qHMN*LA^w5D+YLtcBgj#G1UE-CP;fk|)dt zfy<;ibE&YHTwEe@3;iZ)lLrGyo!>mtWnd^#Z|@hdpzFf9!=yf}|C;j`PO>3gt3XC7 z#CF?=MEI1bm3~D<=R9(Qk9$m!)0RhFTHden(}ClhcnVr?j+EdoMt%-!sn{C#FT!3Mr`9asC7OOBkKx)@ZaE+XxKZ*xJ8L>uixI6iBh zKUc6oC)GTS)SciDQbhnvHur8HUtwTsFoRfVBx zND}|`cdIj36VJDmIW1haD0==ic!Q|+{Vrmd60J?2*7nU~Jw526CG7mpcM^D9Z@Vhk zK2Ntl6F|}%t4oMlc-^|JC+#vh3=Q(W}UY9Jo^1{B~gIY24 z0=mOyd=lVUu3W}us9s0D z{J*xZHKGUkBI?n~O}$@9gzpR#;(T0rtYDbPT{hlRan>z*%oZFuxGnU{ls$ECJm9UH z>BXmC*me*j;V>t%HpXHgBw)Au0BR!#tGk0vAw8@Mw0F5oo1sKKa#@+f;elcwo_p|i zf4zh1(PPF;vHKJm!Y}szf*YVt0CEmRp6t)d6`pxRBz!!1u_4dXst;7PqakTnr&yb# zy5R0SPn_YGvQuRQ1KHmt;Rg|7lPy&9=MNW@sgdll7K$pJ3agxoXmcJ1Bx`J6&_6PL z!oi)a7D|1iLw|mQJVW#d7Xziw&2yruRgPgk>;o&9C!vx~#WD|VPTrYi{lI7Z=t)~q zxvr6u_Y`)br5%qsy>llS%aIK2j=5Y@(nyb2w zsH`8K_@s+-Wt0x zEHp8g-ad7(dJ^(Jj-xbu1N);g{@8BcEE3FavmjOQn0uDn@%43f#smUoy(L{@OBP~_ zspPQQXkjuTnwRK(A;aV&A-#q-0p5ZJZ!m1Tk#ci5)_Gf z-!|L|W^Gt2u8&+SJ9Weu6C;9p(LXJLd;D^@G>K}79RO>Sj7Bx1*~i|xgr9GJVwFFM z*oST)uxtKzO`Ni}yjp?VJeLJsA(76F ze}2NOjg1)CrQ<^^Fk>zqr~~`bB;YN>fOYUs7DJ14AcvSzh~c99I7Qz zvf#)6h3UvIytr|wARx4~ARv_g`w>VWqnW*lt81Q)jj`TZ+IKv|#nb{*4jL7TIf_o? zwHHiK=BQ2{1oNokAjyypbo7@!ohCWi6nS`KsPGnzT#E@*GN@?!`;C7x{T3|eSCQv!&ugyhg20UDg1^u4<|7n{e8v~h+j^wp z@;=MwPeYUsKI@$pnj=2zJ@9SkR7HEVfuLbisk5Xl+ew5)i%A0A0*#FMycc;@T6_iJHNuhjtinw9&QSk0TF z)>0Yd#5Yq~&LP@b)&R{UR=%hBZEd({8IxVrp7~nov|wx5s#G)bI*ez&r$1=LGNk)x z=uSi%YSmL};Jc)a|B-hdZYtEsF5)=mO8&Mg~ndT{dj5?Ua_g^DK4wGAqwD^9n^0wTT%=+EHSoJ z!PP+cszWE*1f*+no9GPTd^rMC3;2uB69^nl9T!sd2U2DQVrQTHt$dgNZpG$MWNXwS7B`M_O7>WCgcfzU z4gLmu*mwix+Y@J#n^I^J+)TyENce+W#Hg#m>5i-05n6XzqOsLBc`gU|my@INVPL3t z7A8b$Q?{>eyRhcw^RQYGpPL+zh}mP{?5O-1)-DWV>UT>}@91Fj$nzs%)lPy>B|wSd z+*&gC;VzNwda2y4HAuwA$u8enHkQB0*|zjVMP>x5flRL>PLy2wN3CF579W!f)OL~* zxM0NSaF{#Z({GiM2&j$fOqndh&nst7cZs#aZ0{%pF$72TU1xG6Q$7D&gqgIo+Lq+3 zT$mOp`AbF$S3ois-io~}YrTgJ!+P)wy$nVd9VYCzBmu~lDKA`ZH_YAi_65~pGXfrs zxJV8#Keo(o*%#r1+_It?bs;?dm*r{hl0T+yrPV56t{QWazt$Igo<=1-tH58%77&>8 zF;0^=Ezh>NX+2?@Vkw_PnW?`j1dIO2KEK6U7vWld#P3g>>rWe58mS{2>WR3O8?s%S z;3kfzBS|ApxFx09m27tCxMOk1x#M`KxYh%NdPObrN#~|QwmW4F2WQx#cEG%uU?#r{9!X$A%NlnuM zbm@~&UwMu_;c76nrZwtmw*NZnx+>QNl)32w()1msIGX2@?JW3;N~{BFxkXqydPjlD zS0_FaPYiO7iFhyxK86Z4I(|@|O~x{@X?1i=COZ|NTFuCMsBx0T={u#Vglk+3!9|p5 zEW`f0^c~uOnjOoj>uKcu^y~B;5>H(~#*X#WZs$hw?W92ZPL25Ui(Y|t`$^A(z`C-I zvFh0P0^6T%QrqpPnuAtQO<@5pBn#kAg3G3rSP|UkUE^ky{xaca5rKK?7>`h<-_qQx7YR_N4!|zc`@m|)gjvL0QLZGvVMZvHuDbq_7kZGY)^I_sFCB?jm-T9Z2I>m z*U=wB(d0?W}1#g=l!qus4$Xk4k)Svul8k}pbG_&G;N0ANuif%WAR*S$K@ zw!*1wOaXPo_iA#5`mzQCY$$LfsZ(fiHFdLnL~aB;x&4WYm%W!$;`n=R$g2h@yOj!n z<2sNO%Wpry@m^09puOh>w}Yf!V(~L0$46SU3sUyABc8n$4~hF8*Yv4W;frKE)a}+0 zD*I!nHUh&Ymfun;N5fifef_7-Zo8opQRODhPPMQ3`ARmLVT78*<h-gwf(YuMTpacqNgSyG2=nR1QhH+2ax1bbjX~wwhYy z1ml%qPoUeL>g>Gu2o1RA-;buAcS*=X`x%$Z<^V<=^DzMZ0_+k{XwY2Lf=kyJN}ZFk zv}d}2a~H5f7`^<>;PN#U`kY5sYb1$|VMUi5;Rx&IsLXY1&F>9EPd}|1P_J14%XocI zv>HQv0fV~w#Im^G?;ld(Z&veQme0F|ilV2jp3-JcSQ^ah00*pTu|IU`qO|%lXXS3n zWNrR-V|4&|eK9Pck2UU`+AC(fV|1*N>}sL>T$e`>;YEOeYw7xxQ=eDBonm@cWmivC z$d-DZr11h1Ef{@2PF6MJp`y74)v@Wat|V}oqj-(cjG^l->d{HDS3QynIhhc8MS55Y z7GXPm!kJF}1pw-yx8`Ouyfj02FfLd@D#@`gFZI(_uG2^__&i&Pj%}rWr|_aA^$C-C zzg+MjVbvgp^+W1p5>j#{c5flgNE@B;MKy1j@~vYdPztrT)hNNTwb*+HO5U|@<>4kl zy~?jcrn2nN?pb>@e0LYw^y&wcJ^mX@u16!7*NVxH@d0*6e1e`lG2xjtQ#dNocjbr? zG_9WuEzNlGLqTC@N7;SUI+fa4&RRkU`E0I^naoC&w(5zFcYL7ROFUC_OD&RO`aO5^ zI<>OdpEPdp%D1#g*DFlpB~vPVA&E^|H=7Mr?xuFvRe|3ggf2~IewENZMD zWy^0umLP7`Xh;a>+}bgjmq}!ymHVLXkc6llH%XkT4TBCS;2QuL?>h$A zO=9^^U2w2H%mAox4>R=;Qv!nyJ;H;=1~{tgL7CF0E*U=n*0{R2Up`|j#gHay>3_x*zLks^As z4{DVs=>T5JMYNg`Ib2jVzwNf*LV)~K5sDP8PX1`LE?;j(qJf3AESX4GT`isjy1Ksd za#&Tgmo1j824DH~)uTs|Jru0p-ib#QEYMMN54gr?vb zI}Rf=5>6#9jT@`x%>(6!wQ+N;B-Q$XZLNiEt=XVatW+bRuQQAx>0cQ55<|j2AVMdPgs~Nx3C*w2;pZ$N z**f#|?k?x>^_-wjaPmEB>egW-h8}sW+N@({F)1c~6CBc;5wpIbt~Bh&q@zWINub zD>xfG{A&S=#VQJVlP5ZdAMQE7XdI&1o{8jf1~{POKNkLGj?@(I#bkg?bZ4h$sHqLs>BZFN zdbPV5EUkV=*0ZQ*u`Q-b|2*IDlt$s#$pw$O02x$Gy(`IsLtb3q`V|7o?<_4l=@?MiG(0dFeV(YETtlz{=rf*Tek(1 zSdx|f!?So9fYB)+)P!d~Fitjb_hbYVHg$Mx*?NorFgK z#us}*O<|*P)#LQJGO$9S?&rYrY6+>B9k1duYBp||BLo2BQ(5c6vX(mC!e8g78vRU~ z#LKbYTs;O)SL?x#4Y*3DNewhQ@MnY0#GD+B?44~{$C|`{zi9`gRv|a=50F}-#UoyS zG{?>}rSPdO;T5c2n5<5~BMVJ_{kHt|yALSe6_LpSg&je}d=s#+ zHxb*YRC!@i{F|khl+uu*zMoO>kLdUTf=-~(v}!NS%pINSmR>V~(~Q5D)ZS3f1L0oE z>pdR9Rfie#DbqL|>~rU(nOE8}LcK57zwxKoUkNNx)}Cx_f56S|;S@S@v-#(9@0D_6K8gA0{x*4tnbax7>#T zOY8m{M9CZ6HM%;&odxZKZpPk^xFDcN*5%vuBNr=gaP|Z!@=s;e^M~1z`iWzW>RP`^ncxsp-UY2&+-}%hSy=srh9knmjX2Ng)i?zLM3DGL*VU`Z zh#`Bkw3_ouYHo+`f>4O1MO`{$>y7*(xbKSo+0hozMU9IVPyM+U3(roD1HPPy;&@tB z_-NUuOEyLOsi;04(DqEHa{>k&g7%wUIc1wIZNNHesErepVq*!QJF6elioGY}|4cyj zk7ofURP-|csQXBDarH=?Cv%_1m(F8_Lams+ekz;pILR`_578nbmr@=AApl~d4FrBt z!@2|6*~qC7pO1v@3ZhcFgX;jftS&cbeK)Xd%k$P;-*R>Gzl07KbTVCijM$smfXVI_ zID^x%y?+%AvM|qa2DKK~!;q06Hyk?w1!JSZ3ZKXUm~;NOieeYZR&Aa5c0tZ}K=vu4 z#rYS&dH@PVBCTc%pf6Rchk6@(d&~aVo=;%YP|_u5%h6IIMyMYrjA`bpic)!Y|- zy_U+KdCg(p(bTt|7IJOhK=$=)KTwwRKpb!}^$Gm1eppJt8BWV@y+^2j!oLGEGO&Nb zKl*c=76Pm8|0M<7v|j#S;=q48#FRl>-2ZLe*^>QVJu#wrQu&^Lq*&CyaSOJTds}>< zvWc6uI>5xk0^n+5FJ^6FW@iET?;cs2x}FxE2Ksk6xFxh0lUfr5t)x$o{5Fn{h+I)? zrfOX|4X1FKgh7OJcCH62+Cpw1|NBt^F>o+Luo8(zF5}}S0noKTUS<=AL}`~dv-kP? zcDv*K>elElh%>~#`C`HhPV8|sFscT#J}YzXK+G>y1a{-uW_}oN- zzstd7YIx!!zr%UrA8FBpDL8eYwu3in^`>6~i+Phnjf<^~T%;TWsk+kT4tC+!I){MI z5SfUD*T%r8wWTSHT7jIV(>Pzc_!`e#S53-!fJLfvPnYZfwc|vM@)5@%_ zmu(-hm<{$z%P4T=aT<)@Qmc2D&?FN&tAJbBM0^Cp)clj2OjFL)T28Vj?SE6eNNognH=FibthG z`YBIiJIOjg$3Ab}fGrRQ6zh(NQ;xzl!fGN`l{3Mv8l~&Py`9Icfg8XM8LX9qx18maYTf%gsvQ|Q>NdR3+m&^`L(lyJE-=1)g+%Yo>mubEh7(QAz%E+m)j z%t*58Q5Eati6k^X{=5pQvqEo;g5uP?3kwghE(wi+gx?>p{$*?r{OO!Bf`DhI-Qgl~ z^~wK``tyk&FQJw5)H|p3BWm-}56lwX7k6nigOk&Febfw3N%*FJc%yXBKW$U)Z%x?V z!9F8-+rx_VdL}FLM#-!atP|8u&xlVuG(tGd(W$P%waUHOSZQ&(vIf|C&3uuM$H1&s z7X7^w9zXqK=@>mB(9v_xO>I90qX7rI+PRIigf|1X$RW|3B#YO!xxa1MWZRP_@-8tN zc8M{=8`D!kwL>9+`ySMv=A#Js#q8Fy#4Ey8;2|cro537VE=IIh;ZBSaPbOEh%Snut z(u#BhKkq^4G$`+eb_4qH;&RDV%9-o-;rZlLy0Z)lX*m1`xbhW6uNt*M)(XbsbBY=k zW3Wf%jCf{KAZs7D0xs6F81$YmZBwGt0Z|hLSI@R7S{@~{fg_7p66(Zt*g5YEC-uVO z7g+Miydp%J=i?G7D5(O?fQQN}hX^q;JX zitgBu$iEgk&OhCU;Qv-8Tcy0)q64)6CeF?l0C5{vH-L?)yPJ)ZqXxiU%*pXzRdD>ObjV$Sz&viz$nu=E?RJQCOUiW>Yarq%av_mmaT=&S17>$3(^=t2{380C(0551jmfkZgt*2hvF%{ zUyMu+YYw9bFFI3|`3fe{q20hy#S>9uj$JQB)yo?RkKB6VG6TGNCTcXs#pMBBod7OBz6_B>N|0NHdwf!rc(X z)|6`l3m7FRs7XHtqL%Bf)k{In+g-%icG=Mu<>g&-jdJ|#RZRYy6GGA=wY4o$h$C6g zy3GGmgz7<@sEe4$gX2}u@uAW4ZKuXeDYRU5dzf|0G1tZm8}qNrT{MYR=H3l81CoS6 zJ4I4G9fmcb8tbfnJ}pvN3r1yK{B1)-v+XgYJ>(}KX8hl5?=cE3FmSKRp1Ts;ZEf7F zmWBUo-<>7aAokJWSlEkwIBQ0svmo`?#MczFJmO|?m-SZqVtoe_qK!6M*+U_R!i(6B zvKK(f=hjOc0!vmagR@gu7ityBUBBByfjNQxi};sJV3tTSKIII_oODIT{9ym+9rRSu zCQpn?vIiFk(5zF2H->+lW||x*2`jTa=1T4nMcmZ|h+g%KEg3}yYE(?((cvko zG@s3_z&DQaN{?y^{-JqH8^(x6$&AyXGm7r0a!OzBlCuYXlgI`3f(8*&i_@$cx?gs? z)p_fidF5^h67c`7kEBC@%o`6J_mB>eN zORD8d)_f`fuH`VG@Y^)D1rnPMdh}rlcgKjewMBN-c}iMJRP#~{zh{`4Gkx0ypG{t~ zuaXZsaf-M??w})`U<#2%>En6Xyt)&n#WH+Jf6GsJ-|N@ZEL*z97p7F%SbQzozhp4r zUw*b|8l({I^JoC&=FR6MndV;NEA1|o{Eto|Q>Y#izgk$J{k-m_CBQa0sd+bK9*VUt zp${49PPx$ka2(RXXd~ZU*FHo z3JRnrfOF2cs(V}yq~!mmVoWHoi;8$Oaf>n(r?bxB+b8ZLiaybh|)ak{MX~F-lPH3nfTvzj2uSXN8rls|oB|{E#|HCdXYsAk80gvcS^Vlul|B&PX{_#+l5KUU(u*@?HiK3bI%U94%*{#yCeWSvm!d zNU4SX1VR%%l#8159s()ZVfz2a)j3Aj6}Q_yjT+mw+1S{zZQHhX(Ac(ZG>vUrjcsSA zaeDLKbH=#mo-x*!^?l)a{_{8I{K<-&tCe_1wCy-*??rdu` zV~ci=Fwte~L|<9mGHoBWVm&>Vg9~lQ-ZHhTn8h>W#8Qg;E>qbsQG0P-rI4gFF;(^2 zWMjSGNe1G(zT1x~>BwJbRCzU2y$ z)>w1eVh zC*|vy*ZXwI(W81S6|AUqkpM{R>!fLKb!==0-NShiaKC$<%oisn#ftHNz~LG~zLbnsvrI$NmtaIkvri72296&WoTLTaK)RO~ zEN@5qjFXSj>DDsZUCeGU%zGV#@ss8mBY&O;^CYOko~AN*)){CxfDP9(q>0v}af=9D z?L_ykdV%^u25N=t8H9k^Irzr04F7j&_h&HiE&1RryhDM*IzU^s6c9@&F=#y93`ggF z@#pmOv)W#|o?tmybEi}?`x3L3&}j-^_5p(nuiAd-rSjEfT9ZNbjX`z58)9!c*z>qO zdAo_wpu+LRss`A2@mD9WMNgH{L8+(l+^tH&XM!nF647yWm9cI?_;f6dVXxwKOB;J7 z8Sa+TGf5s=RS|@{x9;XsFIQG*vBa6FLH7H+f%hp##mCoV7SDQ1adAF!J_hlD$&s5i z_24cCT@`h{ueL=}h0FdrwqIDIiw%Jtq4U_XI@NLEy#ctTdxZt)v{;R4<;-<6`PJ5O zzJ+Te5+mTOK8#mJp}#|YMuZI%WMO@^A}p$h6u=dLAm1?RU66%0DEqyP8OADCy^l*0 zg(H9~!6Kv4ocRbS0v2HGh)kw7_Re?18&VxU{RmGqTNK z4~C@Rz3KKbeI63?rRC;kNrb$k_Sg+5x9r{a5P$~cNe1=KB0F^(3t(LWuHX5#)qO%b}j;A4t z{%6sGJpOm3Y-DPdAbHDINuE4k*dT>(<)%N{pN{ilr zwWa9jw)1h?{hBfRg7a!9+Tl;Lrra#rKm2SF;9wOi!qk1Z#nxZN=qV!%f-Kh-?P_P2 zwg9a9y?+rBmC_n`ElG~Ak2(&6ZdF|abBT0a46GKWWW*tjB6_SX zB2x6jgI~q3)jkj>F8MINA^pINir}9eyySb}oDRFAA36@)dctm8Nga>=41I(AXQDW{IQ~ll(;%defD&}PVx2tW$dN#GvblIL3bzJXe*@RIc_vx z_}!7J3#xNpdpQN>pix5s$>S=}o!DYaT46sj4Wjuwn^Sz$;hEHWth6K9~I%K;rNeLNK?j5L?!^DF2HT@(am z0j-<&5%?Fxtn?X{M|6pBEmC^-$5qUV4F&lF&R#v^pQxOishMA>6HIU_nf4=qTmw~1 z3j=l~jtFZMM%E<9-6YFh+QWK5)=J)ktt}?Sj4MRB3Hs1RE)T!_HykDEMS;Cf4_=BP z7tM*OkB^ZRG9xQ+Ydb?F`P@~H%%Z>KmHZX*q@)8m*J@P4ppYYQ*-fRCp+|Tl=9Q1k zcI%v|2-uUdtC|rupWyt>IB8y1`U=2&F-n2ohtVm87M5U+%`zHRno=#sBy-57CV{E# zQ!l?Spp0{veSfclkxWl2lUOvMROVpIq9cvHg@ULrTOuRnMQwse^k4%l- zX7Q@$NSO~!I?`9+S~Xbrzx!e>=sfH$9+n=xnYk|(9yhD$LLUgb3^LGh#_TeK+7SL; znw2L-UdT7}XAls?`&~h-F&Aw{B)}>#Wxbf)q%3C712`%-z1RYj{*t(O1ki3)5M&*_ zBk@IB;Q@LW6L71F>Hz^le3kxWB9G?JkJi0N8F8O>Y0tq%ePulAU8t{*ge*cxW!xAD z4bZlmMgdTqcR6&ss^&OjjNr)DKoeiZ_?vXgP|AfhNC&x|{kZv-jm`no2lDoq!|goc zJR^=K8uVi=S5e6IEY6R2Bhg%cHi0b1{RSUpZVZ;Z==9EUx7vIB7JE@!P5!}p@NK;gnMk}+A4_7&~DT_m=qsV^C0~I;A)F(;Du_!R9 zU+B2Q0KZ(>TGMb9daHKIXd=&t+sPO?B*p1}?oaaqT03YuJ$j0%-DDHy1$mrfQ} zdF&rp;jxtaeV*_az=7;r{zhqJRl07Kg0dazoK#UC*borX)4cBVzO#F@6r6}^dKB-A z{K8CP*}R=u7?H@N9Vv*=8V}m)k__P%Utw+x;!mG+m%OW%yT{<5VM(ZUo%uNoFdnco zKvr3e)SclCbM;+}h`gf<%CsWx8nV1FZY`d>W)Ie9W z$j`4bYO8zdFWgV$k3vxrEFf=)v5On}oFhomyU2BloHLrQRSI^q4<+{=3-^hbG_KTF zeLBo%hDin@%pr|ToaR=cpcS==Ra*oBA=hOyczs%c{{lxv2#`2%GAKe4_UYN0p<0B1 zAsZ24s+5R)svKG*u_X9vq}W==cUUP;DC!O|m+WxqpZlnA^~j5wumAqnio5_pGSB>$LTzez$NXs6Q22BV?{!%}=>gJmyRki1Wdk+WFP*0Nh( zkMj6sQW~w(+LFe!U_y_MLccDq+xf@8HCi{le&xD)`bp@i`%e<|Z5J=A?cT>ok}USGT$}eOdRq z`L-1ReEZDc<0eUTEYbSNiO(s$U*5>1TR>_!*4;~!OVG^Zk!$EwO^QV-yZi#XZI{jg zyui{J@Rz$o;%sz@cJYJGi`{a&yx@s%MbN7CX5E8NE_0f4czE8if;H#Z89vALLfZzw zwtW;}>y;dyhv_g2*J|ngi#=Ux@uKjAdv{OpI^80AMpvLYY85l_y^@4(PxB!#Ja5mQ z*YWAL)Gzb0P0xa9)hm3ae*RAiBO%@mM(y`fAa2q~l7&_lsv2u5+9yZ(pI%l}f-;r`17hVGGy0i~GZT#Sq zf%CXXy7MgwxY63IWo#?jgBD~MhS-15k;JD8r{~9{mZF9`f*aeQM5&m|{$A^5N5t#w zc{$C+NU~^e@BC`CTwKW`)Lr+5$j$Z^f-+)Er0=Ep;bXJ<=o5g%x5!;N!f z1;EOlgvdp&{H{0L*ja8ZF7I}{DBF(Z1HSThZg4$5U7cQEo}VK$x7wd;V;k+yh!(lh zWyt8ft=2oQf``tPE%17`%3=q zECeyFEWb5o3*IUTdfniYs~LZoMPBwdEGOe^Sc|_+<&w(k5#X`|bf>J8MrKOr1@V5C z!CU;mGIMy_ky)WF%H_m?y$N%M04_54E4ZhzvcXTwmU|b#u*6*tT6TW$P^X(DW;jbnRhyF{yr+Q+3Un~nAO9R_fRrbGkQYu) zkd+QLP|CQi4LT7MrW#%qgFnK3YFDXhaKI}UzHuh$nF1ZlbCaAfTBc@e+=dPgKDzZQ zn2mqJAwmB9BO~d`var@(>3>u3rW#x9r=5hv z5y1RI^i|jl(toUx&gK*&61YfKgB->{*=vD>7#e*s=yi^#|&T)8tZ%C`2(j;Yw+?j33JXCVOSesfKP)WND=39QQ zr%OS~ka2uWlV>`|#wHsyw#!6+t(HSDSOuq+s$r%|CYToi0h`7X20RKj;vS{ln<^S< zweiayX|;V9jJ=WKg9y;!#)MG)Xd$sAYhWheda{sJhYD%UYTVsbTVkBPs6LyBUgZxt zV|{0II7L8~42;ROn9>Od@byx{oSQ~tbMkE6wFQ+$Nn7#*j=%z zhXrR8&na5IG-iLQ10F5G?TQ^Utzp=66&DsLO^+8%w8WC>C5oSFu!x*A*ASkEt(9W! zR`Q{y(>R7iCg8TdE~atQ_vX7SYox(f)29o@0i4}~IJa{SFnTgAG*1Nj$z635Xb#V{ zO^|bZbs{`JtHJZ4TP)Wo9A)xR9 zGM*nZaBLUwZX6;sKy03sdU9@bJNjGhQH-7_jVd6;yL$C zPuhaS00f5&1c#ZDMCeGq{&5=OHdi2ds%&I~@zQ3jci+{vxcl~!EXDZ)e^PF6o6R}z za}LEKf8qICNW9BJf#Do8V&1MPH1WxIRDNbdM5Q0R>#KEa&ya(Ed&~X>FNy{GK(Rx# zqpZBK3)$UD2Mp~>4u8+zn=PAByS)$(7VD7>N7^@~19Ix3_a{Ws7yGTV#F_5BU2>1V;xmpzK#0g=P%T_B`)R*2;}{GFU?;dvBV2tt2kY{9|x_EQ8pZ%)XNW9p{hq=x%-#8<1*xR{XfU^eKjYwkSwvmXzOu z2D{43g)pXj>|H2G~Y0ThIgWY6i zfLzb5?_bZ{Wq0%f-^8Wp5_V%q-(IqQ9Q$W(fA5J$R1=+VSE8_oWt z1C;9CFX#QtUqYeQzL2vIam99^(AM`!X64Z%Y31A{3M znjfCmzj%I(=&fCV`UaB<+xL6}f+m7x49myC-J^Tf`}pEqHYBigoBEGhhRqCXYSDa% zHH7+6LOBApV!Sfjis@Bsb^079Mok0Wp+V3>D<7BHmescdAAUj)-s2oDk-fIf0Zk3X z9bSK`n-~0lvqY&bu1o}|^bF%bas`89>}fyvY-{Iv?CMQhuS}${O%*oNPWCZS zALXPCGrrN<_FnD6{uJha-1HD%{?%3C<6E84NhV48TP>tqbE3y?JXVkBw6m8XQ2Yk*7k~MVkYj8gj_j2&08}kS7K#V97WK6^` zGFESge(0cnWm&rPumDN1p4r503pLep%P4CKSN)`h5{vYLPC=Wvn9A?F&$J>!v#o>w ze%Tl0gIv|d~gn3GO^aHE!aZKN)jPn&vOd3}Fogcfs1rd*It6!Gw z*^VGZ#E)&EpPVRoEk??vQYBx~;Q9 zxtoVcf3kGys)Zz=Mk}0x^`5Hbi6t)jspntRB(Ucs=c*gW&x%2;kGhjCl+e|AFe(K; zWHN;&Zux^&KiQLZTs16MvktNfiYjX~RG?~AYGzuwO0?C1W!mar7jI1o^=rG+gz+o) zN?!_mBiX)#pvZL)>_Uf4QVDUnN!fMB!J%=6GY>DNTzta3sxB}`CNoJbOo3>$4FSk0z!U`ZcewC;{lZnzbHOZOd%#D<>3~OBqTN$}l`TninpOvvtaqdHAU>YR- ziXrHJUI6@_;uu$j4o6T$QE~Yj*~lK;*8b2ZvI~!J@${L3kuqHZd7V5Kflg`5KY1;s zQ^|^XcW0-;0%G^){Rp7N_*BPh(7v;~Zu{gOQ$0_0@41L&68mEJuScnDw0z#`Rd8!C zI~d#|SVIsQ4TDM+9@59wT>Tj8#iC42IALR6Ul)+--*SOPa2LmKNox)H59KWV16RUQ z9*&-(;vo*|3Y&r!hhPOh8CTomw)iCEp@$zy%!MY+*de~(eRAiFAg03%kCm}=0b6Rw z|8gX=Q#1%UTbnf|7jzh9ZGSV=E;oJM5Y(1XSGZc9wK7QdCO>=sBytb#8*nJp)_DMH zd;)?F*n7cfs@002Y(O}v`30d69Q-1d1mr-8+8>mn%+uw9Rb`Aae%X5}lJBrk6TvT( z86OD#E3iS6EY!h7bpjHWRA)8U!D$^7xgRi$HZCuE+r!d2DykO%lDrUQ4!L%A=>{&b zdrDY%>8j+i9&-^&|2?KEJ`qF+>I&3(H(=dU7X{;>as7Q>{7f)~{;qzULXw8u+(dG? zm3y+S#W|ImodmX5_Ej#~_<8aZ017!)6(O@vqZg`;6b~$?)%ZvyOFX^5IGw!sx`5XQ zF)3MEz8O7{3uXt|_=d&qC(S>^tM%2G-VMjWV_+IGdy9` z)6g0ypVQx;NuLvF8R$7->wCm-Qdl3F2cAxUNNbwI^?$ZQ0-P^&QZ-Nkwuc4QhHD=6+XOheXV=qnia5P`2xGLic0q!$Czj>tG<0}U_fS)3f1brp@5<&jcJ$u^)VW7<~N^#GU zqjm>Y_eFzUo2;~kC*@?_|&@}m|_l?yoxI06k4e^YL)Yxv3V<}xUqT5r#wHC z=`@{9um_yc3R%!G>8pNKQ;~M1r6aZGOP^-^lA1xYZHD^x{!URPDlQ0qf-E&BCpw;f zkcb)I@vhS+eXrR+161KYSDb74rpMjFmL+@ViW|T*I*at)Wf43@uAfBI9r8QrUajCQ zan|FQ;yvE@SdbSUio}}81PoNr zaJJpPNzK@hoj~G3f60ai_oj!(c0PZm8A*Fhwi|Vi$lwTG2e)oGmAH;^Y6=KA^e{D6)EssBzj^?Jw|C^-F!O%7MM}JEX;0ZE0{+{XI(kINw0X zkwNs-K}4E9GRbgdl@s@hKI0V4L6&4u;A`!Vm2b5I*)s1q1rw64l5A#jOO=hTxZ0uRP7Z zcpsL#@s_CKvxRQ_@wyYtO%4^U+*q{b7j44cUdE)9w;ia_ON%U>DdJ2ejCv&w6O4`@itcXXSSw1?zv)qZ()b;XeK$LPC#}lQ;~g!qt+3e@oXm zUm%l;g%TqpSzlL3vc$=pDq%yPZ}Hf98fMD*>)H#7)`!XQQFt3x{7Cj$&)eop77k7% zcXHY3eA@ch_S|`Y+_?dQaR;{hTn<}9vqD?q@DCbE0qDcjW2}^%HHLu|VLk|KE^(fw z?hy|@d9()zR5)@!+6s(ORPlVA6Z=bj_@hs}JhcZOyn?jdETpZZ$Vx@_;fk#VGc=5? z)J4$;Dq$ChIB~)9 z;!~_>JhKh8&ZBy0O(j5VLgMJeISC8d^%YF=TvxYa)j2^kzB8-!dDXI*8D1Yw`rK2q zhQH}eNq)6l_HFiCa2^_HQQCFo*;EgNYz%{Zg?+H~BU(hNlr^WX5N~UOg(ORk9Tzg9p7p?ePhI3t95VTo{Sl|P zi3u2Tql^4B>8h%$3xl#v>I3nu(wY*v$3kd&nVrj%|+x~o*ljX_wTsJ^L0B}Wp^Xkr@n6*cwRMC1LfLW80+ z-wB2Jt}1H_lLfH2B)=)C>}_{;iaJ zC1wx-k!FMapJi^2mQ=w^wy6|1$U0+}<^7+mn zzmA^sW<=Cr$+);uxvZ|)OEyXvl9%DsKK?hg{x{9=nUA-JVV4jVy+;7+!XSb5 z2_D(wjg8ZzwKO#wu>uRPL z?sqe=MeOe^AkuBBm~Me5{#?q{il|V^b(-IX48Gzc)2nI@(2zzE^zD@eq6ID1%o!#8 z8*r2pBZq*Lh1F=?W{R49q9i$)w$TeTqOaY!_lkJVriR~C2f<^O*kCnwi%DCd z^4+hs*OZ4MYp;@dB*twe2boSM_k8lLu?<6G&E1#h3(X9`vZD}`5D3W|#+I}G#M$Q# zfya>mCzm=P=(cp;EJ6UrJHJQ3zWRa2y6AfHK9hc@7^}eIH>?p*1BTBsPgKiJ_24F2rV&y}hm>kSJ{ab+zVU6U{7UC-*37MG}w zqc-^cgh%Ezh+pS&w6R(H(3j}#qP)Y$UK?(|QTEfg)U9h!q{@<*FAp6kV4QIo1hTGD zuqd_mL=+2{D}t;=Lf{PuMlzmEWr{{tS9#b7VlFu9rL1r* ze3INmX~hl^lRxIraL;v`pL)(eT+=m})h6u9W)K=3WjsdphB{G$Z2W{n>XDp;Nc9tO zVu3wQ<)!d`>Ra>u<+laHI2I_nZ^t60f-W_osDBkmsZDT4oDr3PY_OI#RN3yD@E)K+Ky9SPU>c<$cQ)VtZBSrU%-lvu<)EcIA#je*I8tEm9R*;pn8 z=vK<`Ax{=>Q8^1AVlALEs^?q8q9ytc-}+tLGoMO%qd-IF0u9N=Y>RMO3(k;%XGU}~cZ5(@yoGQL;1_+Cc?B$Jo^LQ)BjC>zT)H5bK`E2s% z6)l(f@zz}Qu$w3#Ki#J0bMoN~+fQ8ZBdI=RRGlcG*Uj*1&(`cZ0NF5mcJ=P@-Z_Nd z0d)Jl3q;%_eS+*$DgNvg>zJ0OTY{Os65i!U4_uQ)?U5gPjkt8~8*IJs3wH}xk|jQh z2TGsh67|S#d-}c*^{fsOrza}HK;)-H=HK6nFaxuM$nk+1CvRO#gZPIB0oso|na_dY z#7i#;GvNa7-pD`^iQdyv!2l^DfI;5OATM#^)1U#~F7p}xeyP7npyc641%HQoz|>^? z1Nyz!f^7QjFwtjIc>evp=5w|8JG&4$@SXo+uYUZE=g;8ZnWs2GIn5& zuRIN!OpQ5jCkV%dP&dib(s$m2%2L01(kyEUBPxRt!k^H>&K4!aB+tr{rAq(@e!O+- zOb!%gw4%-9*+TGb)0fZGg2i|xd>^)KnTK-CxZC*ZT4`38Ap=I7oFke67!M;}ElzC` zH8bU0CO#?;hvshlrd44o+|xQdAcxL)kIJUpUHcnV6>fmc#D9c87x?qKtZ_?jaz{NI zex!B)se?tCII5IWanhn<+B5X^2%k4ZDC48)OE5U)M9=O1Ltw`|U6#N&mC<;x!p(0a zI>g?&|5ypOr~k}0JQhU-Y(dsE#5u2ruBIjG2RfGpZ1{vk%(VmwwmEpBFa*XCv9U7I zuoN<)Uh?Iuzl z*^f-sX>gDYm@AEAte;M}q~!;Lgdr!CTP(A(7bR#{TFPOHtDRkeRD0I?7He`DQ8O!6 zz~uJPpUlHU*fOK4&Tf&ixREuH$!wR)kenj!HXaDbf2j}FgeUz$jOm5 z2`9AV)~_Gu#Om9D$RDJ_s;y*okNuApy3q#~C&COVI5iH?ZQ$A$0D-cF=we+ZhC!^v z&mc$-){w9CC|>Aq2K{0Qw8)3GTZxk+&dmWN7+Aph7i`{tD&<0=2fkBU6}~Ks)w;#= zKV41P_Nj);C>$#Hk4uz4{8dGU+=EwX4g;G(4TQhJKq z`0;NhsHSqTi?mzWxz78?|N78eCKj>f%!A3nf3wb@6%_9~+1 zO_1UVFZxXi#Jhl}LW9H2F{Y4_yS@PnHn*~rWuT+wKSR464=5|TL$^`sFZaPGC&9-* z4gdVHXB2GS(_v+3$O0bD$wG_wYfI}yvoKuAPm(6M30jU%2K(Eut$8n5rKwy?<4764 zgET+b1?uK2 zN1}euHFy5AAA#Gbif$Sfy&WoPcTQBP9Ke%E&QSFTo!WuTV9=FONo{E&yQ1(qg9S*a>EmNRgrVQ6^E*{|( z&VRXp>r_63=x`_S6Bcu)>9iHvKaPmyl*E6%V0O+Du_OMP>)G?&H}@aOjS${D_2;jC z;GR&i0&kdf8ccgH-aFSPpVu_T@GkIH=o_gd(9rI-*DFk6D;k2kPk0Q~@`!ZJ17_ppZ7uY;^xU9wUGOwG*g-PRYv5XnNm*d>fu5lT(F!&e)9s8(aC86P>2x5=vHvP6*WpM{T=IK>=?%93X+{!`zyNu>p z*67^*vwRqE+oV5P1YGOrwv@XshI}c~u?e0K{)HKsMRWDD#$_ zaC-5~bv1jPg}9caA1D)ZWwwHV?82|Q676+6{cKY!R}L0l#cbpUYiite@IN=3i>XiM zx<1CzeucgCHY2GK+@X}gg%LtHxN@w>Q+4-TYn6s2*Akrf*>4H|217n6tx2m3fVIuu zoSr%14gmUj15kC>)A%Qlv|5mR7ROrBmG-rAu(`bW0DCovyX_y3{4!l!-}Fd<_gIIX$~1 z@9yzuH!RZ;La3J)>0`Gyh?G8Gp*m!6dZzxLVva09;b(>!59}>-JH*i@#wK&fsLHfenDqt~v_jT(Zy`0grYU;3SD1=fGe69gv5+TN z^1{UBtf4)+bx~zY758-O(Lh4)lK;EwoS|GBV8I&{|>|2 z6w=I~slaGU9wcvnU_s+!msh5Knnft7hB@AmdtQN2?IwAmFJRY5P!e$2BWEZI1R+2ZYO zo?#Sl#m-e`AUIm*_t(zgfx0*(_{L3rPElT2>~Th8XbKqxb(?8LF|IP^rzlx`*Y9u& zw*o~*!eoE5)O9==%2xn)VLhKi1)IUumvsT3IFcSucRyw1Uo*N?;>OF5mzM4fzjGfH z!WU9}UlLN-OgVEk|NS^`1-^!M=_o>2w8ph&c16C;XK8XeUE>mef(U}+k$Odo|nX}fyq z;)8PXQxG1qWla*jEIFQjwdA=Gf$GeV$)xpnX@JZOPKENfZH%qxLwt-1h3iBf>Jy^8 z!$|boym3u^N0t@nQMMr6iSZocBgtV}uJN*iN#K3`CH}Ou@cyyYlpRdA{~Tq@1h!a< z(69QMC704^DV7?Wf?C!bc+3*d4-b0(i~HYEXQL{{I%xI zEN~ve3)}cQ#0_S4@Y#pCeJt`RxXIWhEjFRLdrn_?7Ag4?#d~6cxTvcsDtt^=;|1l2 zScA`xXcqTy#1&Jcu7K7J&Pz+)l}4Ca8PWe6xjB~nE17^;iOv9eb(&LYW!mkL@C^!L zv1G*#z&q+b>YnsR)?|;=iq`#i(V!ZOSg4}X zd?ALfDk;Xi4!>e?q#8WdYRHk#@Vbs|2!<{FDU1LDm0oj3j~ICYOCr_+Ifz>;8=Q?_ zL{T&Ymp!>BCM`N|0FU~Zd2p(JPLpxuh3#~5aBN!e1VtXUjevgZI+Zsg-zSiN7o5Ttkq{*7!=Y{GETe!wmpv& z;(_GsGH|ke!M{{crv@0KfLF+KMb6&ppYb005N0LV!dL0^4G*C9LylU=;IhXb)HJy3 z4sKtwU zH`)YtSRq^7l(JkEU!0M>lIYj4Zy?$Pa33y$5WE{q2nA#f0q{D~)^8T2;u?&y8w+TJ zd}^|Gdytl^^R7-V*fa(J!|wuIZCz14-y~PhvNPJV_;2PQbIGP&;ufD7fj_)bj*}$I zO>(2$UekO8>#0yK*e@7yGajM&*%kwt=b|+TZpqi=5V*J>As{|LM%Y%iFSE58vTV^V&B`O>K);cR7CJWxtmG%k(e2ZVc z=O=O+XnaUo(L*vxm9z@Q0e(5?Z`3o{6h!LVX1;1hh=a8(lVLAVKa0+|z@BL4@TPOR1&PMS zx|(Odg@iOl`r()z{LsXl%)tfvG{4XuN7Jzf5~_`BHDxSrDa#f!I)_+Hn)0aWm3?L7 z*7!OL?*5J?qoafHR4@k?71L^0q@1MF!P8EN?$&;5A#gc<;f+&|brE8D(jsh;JBAP8 z_Scyd6^}AelX5snpnN4+e6vKZ&Gt}I$>567X*h@+zpeM%k6@SVi9q;r4o!Z!-*Swp z$mn!;5Y1?@ywKf6cB56TTgOYy&HI&zd`NMEu3A^gVNad6UHBe7-xK|q?S}vqFgXpm< zFF}fIzIQ80-AHU9#k5YsQP@eO-H~Xlz~rVi^`S3_kqBqlhGb{@DiHF@Yy4`-kmEMo zTN3FKLInL|@am4|Bp0xkT-c0t!xbBlqi%^y=^_N#Zg>%L=1oh^yu=Q$B`yN`%C?-A z5!UX;kjE0Z9U<(TYh)aZLDtzmXF~A zoumoLY3~n5RpT_E8z`I(Ad#7j0D^PIa3-}liEI!|O(vGs!XjpBA33 z!)z;~Fpnh9KDu;6CGoW>bPa3zmmTTA(a5eSCmks1m&|u;<5+!b>~ui<)`F{ z=E&+kqIp}2yiDZqYy?yJAlfnjme5ZfL{gjnPpanDz+WYmn&ci7WNxW>$u?HMV+C=w zMJ$n@pB7a%PNh|K1*BEe6X=PTQ)ax2xmiy=1ctrAmvh49t!HcxO&a4yUY@@)lyIeg zC6Udm3O76q|Ap&?9|SwMfM$98-AP<3)mh8}$3=4)j^2mOWQAXrQDGag|1Eu%Lo5=a zxt}fvdi}_EmgP$Q_ae+yh{yNZb8Bhez6W;shqF*@9oB<~X2f~%G1K~}BxVO5sb36D z7jq0SBneD}MUy25-HfS<$wF+lz%FL}?^@aiEG4uO%5I3FvfHg+BZ%qsz+Ny(57M3h ze^8Vc8RmnT&IlC7uIOnyj1f!d%u%JApkndlnxtl9e%)TC8{=$I_FPY>wQolNG7(4aw**KwoHVV`gmq z=ynxt?lX-wkT#Qs^?79qF@NbmHfno#-)gc<$M?Rit(Il{u>;)Up2}C;e`LImXZ(bz z>2adO4&2}UgZ*Zvq{S|j%j_1;l3)Y8LgFpgaJ->86D#QHy53>*@4Wv{U0)p+)%L|p zcXvyJbPe4|3(_DUNK1D~3?U&y58W}M^cCqGx{+481%v?xP*6eM==FCm-1px6vCls1 zthHn9edcq{K6`z?tzNM;PF(!KH$+c(U+W$eeM-OIPBa%(o*D|QXz2U26qeyoAuzwq z5uAHMnrv89U1r`tt=C@TSQC&#Au&HuBj1^8Ty|As{Z&t#K_fSP!g?3B?X?Vih5GiZ zzWU9@YY!DBF5{=A8Unh?;1-(V#w-dr36<4--hm4Zi+r4S%2^Va!=o?PO^Q-eP+cVDu$}Ss#&RUI(PlziL-#O2aF}dH|I}nM!=twm3*$TVD74-Ek;f`2Yuf6 z$`07dv!+`WG+JoU?|XhtF1mygx2h-7xP7~1BvQ8Cv=6xPX7RyQ+*N|fUI?rp_P7)5 z*`*8Zix$d0yqEG(+#{KNeVvJyPMRQbHJaW%Q<3=*O{cU0uvz;uu!)Kic>Os)CUSKr zz*5_|qDeR;ZCn3-(uXP58n%12F@y740@Lyb0jPkJ{rVKKDL%InH#da`E(0&CqA*TY z2hH}F-IQO{Pa&)$FMQhpzEI9$QCV!=4Ml+ND4R|ht@-!{c!OV3PcKU|Fy-{@KwqP) zVDymHUnUdCE%&~ZyBTFbYb(E@Zlng=NgZe=0PLEWbDy~{xq{WjfQXnbW{jMVC0I-L z9&%2*z;LW^8LE1}6QeK18;TdQS;mI0P3FbBGYQN1nm!@*%v$+XDV2cFVE2?VJZYrky>gwh|#J3)t#|;@u!m0DS6(hsp%t17@ zVIb2~8c2t-+cr5}l*IVCEn)4pp`Q!jX?mWvkFTE>#NA-o3B=VOv6j>G`XkR?ETQJU zBqpQ|X_&^s=3s-T&FD13_EjFBzE`5$G$K&npyPgpg-(MTPP+%-pbR?dJg23=rEBxv zH#kRxR|Pu2&@}6i7bJ)4v|N6@{56zj*~DwYQZ#b&CVAZ}dNyE<|GJ-3roomX! zx1m@aQG;iKNWr;Bc<&gyynpRJsX3y4SKU2wo3_Wee6W=i5iKuI@C)Mq7k&)18~+c) zf4b4WCG7`tnaB)cYZGQ0si%M09nvsileKv+Q4Qjg^bIMj*S-3vexgRx_i;L22$azF z+PBF^YZ0QAE4q?<1W91ysE1$t)N&2&@V8G6i)H_I`l(aQU*e+u$5GHt=mpFlDX;rl zK-;F8-ZL%0gixvfi-5XV0OuJ{hqxGCHvz7Qb?DuL(jdT-vzbh;8hWud*!kBst(5v; z0?*;u0--p1<=veo-0NEGrQGzer zV@~Lee)18n*{H5L?4uL&N1vcl0I7O3ncC@kl1y#}nJtJoXU%*V`(d>^Q+{W0PLr($D3Cb9IV*&nu!s zpWHVAS16RaEmIIl*E&@IeHEZ@Kf1wI-lfvW$Z@1V$&UnBN;4$qm3Ugc&WqrW(oML^#X}wDg_yJw(bUB5tDUH7o*-V<|2*gzHp(eDb(v}N-NvC?L ze0gFGa&Ks@vDss(rVJe`ZL6E!;8g`7E94I~8Vp_SM zT!topq5#>jlvZ~p= z&+PZ6CnUZQX%?Cc*}hv6Pr52(-w{o&@_s~twZ4D7|FLN_s@bqPVd5U`h7o0brSbx^ zfB45a5ik+Y^QnlpRc&4Z8Lll);70UaEp^Rqdri#%$z{LE(J&}wdO{1pZvjOLKIOLf z4fgXnXND_1Fw-5iRwT{AwZ-KKzH4-TIg}o<$+IOclc7CBI-Vpe9siE#L@=j;N#QK% zI4g;{XC-MHOHBTI+<8(CUP98eOO#LWIV|x5Qy;1S6vd;}sAK%W%LtoKui=~tgOiDz zt(@l^j%=TEHUu9cCH7gNscy=Ctl187CpUpp3GKSC=A(JdiCMFcr};Uvs03Z zU-GJ$TC(X=eomT_H0$wsD%_Y@ z&4oP}6@VwK+DX6j{H5p-cAQY8cAa~Mvpb6^zbxzlsk#liF=r~}h1?#5gRG0Nz4?6# zknP7IM}c(OE%rPWu+B_`kKfNZ$wc3n|hiR;SO0bT~6EFn*3zaYdBs% z);gn6zIE!Jhag2^V7Y`my_r`9T< zHjmlHKt)^YRbx`G(!~W*r$%={$@-%i9ts&HTEB7R;Gu*Wq+o`q0Sw~k8tw0vK5_cp zs`;l7=agyb$gh83Xx>3b2+_&@-NGSnshnkpKx2E14Ln7#I7*n z=^6YksAc8mh`%ZG>ihK;2hu~R3f`swcdt2`g>o_dCp(i^C^PSwZN?DK=wFJV=?}xl zoQ0f)$m~oUiqh~0fei_fInE~Rk&T;gzv}91tr+?@;>_S}Ccx3hr>K1{B@D-_-mt}I zlgl0_d}0t(6Lm{ZtbaLNt_MpCzZw=lHL5&;<1cZy=5~3Rzz?y2<}iZ7 zv>}Uerg`m}G$73r%AJ}5m)9&B+V-`(aT-4cM?akS&zQUQ#!qWMmVj<>xoq~54 z8_kh;J_Nw3*K^}v*w7`5ajnp%Bleq3 z$*oNJ{q{;d$kY@wQC4iHAu#f1XY(zO8|v#&nu=7zEt-MV@+gvIYAMWTg<_O}{QM)I zy5ENG^)UWK-n=AyDNkzHXGq!+u?r4=gx)E2vJYhe?Gj^#pit%0E`|n6c85fVvR=@e z)NHBHL(Ek*>22NV;qz1G%%ruw<9P;{JC(*xNUryWp0&yM3<$c=4659B@uj83FQ)H% zvq>4#){F{lW{3__upyV}&+%R>`ZBs!!npL1SIRu#z%!sp1gr)5k2^%V8AqlDi}K! zDXlUNCQ7zN65>O1+)^mOQ7{lxqa_qd$jK&3Hb4TN>R^#N42k1Nw=QeUMJk*wGoqysEQJa66vzFru&x!} zz=Z?&kaPo*n-r5N#Y2!|dm`JF#(xkIeUH-JqJDIC`XAc9Og8~k7&hYR_9cP_i}Tmh zZR!ao*mi~ph#gGkKz{S6ZrCMSosl**mFjZ_hMFjo!U+B=EyZNsmNB;oY^S_K?bPt` z2|vFKqLiHM7s>P?IW(*l0myl?_ijYM)ac|L9Cw{JuK&SM~~CY}b|0+C|4j z=Z#e7#p(sj=8?=LQ5Zn6Jk~h2c_ztNgR`gdLA$9UHZW0*$b(Yff@QNIv|YRJfQ@c| zmNji7frMf`HdajCB$maFHBbz=I#yWP4rln;9_7Ery-^)NJKC|5<*bkA-f34Vwyr+q5ko5u6r zUO9L<2@|-mtREU25h6K07IW!s+DDC@d!o)R$74lOxG7VZae^hwvZ0%2XZc?Jl80ey zVV5Yk7e8hH3;aWxgy^8w?--?gMlhWq|A3&NdguB{Gi1GyN&N~dCnr<+ z1eoW*EQ#y2gk<0oL5>C`&EzHWMo3u`Jcm`o=DC-7F45#{H7%(tX*9{BH?A|$sU;z< zZ7?8$KxUXKu6$p8(Ew0G3vsBW5pHtE2=U!23TsICww|eYC|NWhRG$D1&VQcAWA?F{ zZEkgJHp}Tjx?m$O;21T2*z49~wf_7o zPwm3fSBsr#NEjy@os9W>>0`_9bD+q2>y-kbV&(;)o#=|bCTGW)Nz8;7Vf_hO780f@ zlj#9n2jB29n2sreqXV~3|;cUc~WpB1!G(nd*=-!F|-jGET#9}(d6P;{> z@y70OfpW74ih&&5MJ@0fbyVF3bBF-Uaankci60_BX~n!Td@$(?erTj8gY%=Bsto94 zg1Mm|j~%Cy9z!6c_>KvE)>4BBJ!A~^hB@%q5hrSaVyXO5%6nYS;0Y_;)7ocw?s>aR zquS~WhlW&krGuTL%5jxj8tk6MrE>v{QMo;7gI4aJ)Ml^*_klBZ3j&*tFA(7&V>*Pf zxSR{`qhl|*>?(`Pe0mS3rX5LgL8(BpDPfg|4Vpfl3dQw8?O}E|ZIB^C9@H3vQx`hHw2Pg$4T7Z8sveK+8ecHr>IBq zwW!%^3_Nm;H?zBn@84E3V_Shn02bXm6)+Axh4I6=73dqF>&T@HgG0BO?G-W*61qN>vu`f>(VMA=&tBF4u%)xZom<(1*Unz%bj775&+`UWbee5E3U~b|I%?8d(qaIqWitb7Lg3L3he zFk_f=5xf0+TvoKI&S&1hyr5k&^D5k#s3vzin@+qR`i35=-pIrNm^yFrPEHQLvwkMo z&?l!%lYLMf>ms1!m5Z<5V#i++qRqW;DbxG*$GPpfiT7H|AHLtcr0G5Mqj#k_t?{rf zs@c>g3^*X%kKj%WjzHAMiQ#H`sEM*A^~a$PS5U-fzqtA z_E(dTDd{AVb<$b8{Hgew&R@`V8=iZ6AYT(%gqiwSVOAGtRWetVe$oEWn(?!z=8K-` z+PW#GW5;NNTaj;*Gq59dUZ%!3cD|#=mm4LxV_F(2Rt1>O*R(;uenH*C%Mi~b~9Mgoc*OHs=vf_fuk z;)P?^i#+U4gBuM}N_jl=IIYmit~i*!KO+%`CfxAFRNV6_Kj6NT=W;*+8~5ytJRzM~ zxd)}|3Gq%aCkO|{LJbPiE)kmbA_>-a{ArLa1PGL67x)ok}))? zz+zvL-QuTsf7#OkyK_9P$scp*MwYzBhGb2cKS+rs0Wy3bO26l=t%5flcQKce84hiw zu9CyOPiB(YAH6dF*V~gT|LN=)UFD4T`M`Xem)<`OlP{;#CaX5xnBm{}Wumlted=?A za_+a9?Ew!2)jFhW>Pn1NXezuHYasQ=f%+<)hB|?0Z{#Z+t4iM%9NNPaMoV64e}%_) zI6(TTUIShH`-^k7pbA8S!=95GZ!-gAQ&&}Q&TG7w>!MY@4!alrXovzd4(>Fq!Oi4J zTHkezFIe{cz!ghs5zX8deeF?7T8SoM&~shXKi}#4&fU%%yNy`f?sQltINzv@cq;>&pPz07fbcjkOULeO&m_h zM+e8Mw{~7M*8wSW8&v!3n2wlC7p6Wr@KDk55 zZ_#vw4+8972#m4SaEYb6}_h2iLOGNB>y?H5VxS0GrQ3t zw!pb4%@RgF|5mROSqIGy8&FuL*{S~(r4EKBG71^$-$W0ND@@2_DSXsm$OlPC;hi9hjZ3qJj4TU061&Qgz#Uv%h`V0Yw*n$Im(g{c!QG#EV0eR{_o{( zKjNfcJpjknm*yxLf=_xO|)LmikxMT+&>`E3P^g5|Y^ebP-2L_=Y$_ zT>=c;!nmb=hfDsB`jj+yN=g#kwT*GBt-qP%!G$~IC=;^3D@PE>)BYV2sq^=^{ljVd zo0mKF6FJJT!XM4s&HPP*jObM}0uff|PQ9%Uemh}FiTGFDx0-r~B=?R9f$DD)0TqV- znA{=By+UHcPSOx33{K$VXHsB-bwoJn?^#N;Pk;h<1~cwc{Sllvq2c}A03sxq0+S2a zV*mCa-(fDf(@@i2s&vZ#UmlbHy8XXA2>&Y#5^nE-D2bNh|4oMgzF8x`N*%pIJ~Wo`c_h=DuZr z?>08@9eaf!ggpZSD)_egZ^Tc;91lg@O(J*H$AMta1L<2O|BEn)gv4}5wIdQyGCla@ z;8&}z4_Ht{v%njv!vBoC`5_Amdp0=yPzrIqJBRwu@cr@uZ_)2gX(MBRuMdAw(NMuy zP~auMg?g~te!LS!e5d-Q(%ap8t;go%p0X zb+J_aF~t9;cUDI%C{G4{i*t{D&FLD10DJhi0NTy=e-(b`ThyJxVIzNx@WE!szkCTD zx$Ub5)4wktj?jaor-Z<2D}x{F(!eI~^_3c2h;C4BN%%wM4{G&`hsv7%DZk5%b(wVxUR7hDh3;g)F$NKg)je>(rQ%S5ojB1!-c;Q4s_}T^0c^fB-6pnC+5QRsQ z!uw7+But7hfp-+wc1FuqXnkrPNLr4wXRsU3%t4nsSzZt8ZK1TwEUBW6X-v|Jh15x? z$g&kO&p3DG`rH~sw~D?*->F8-gA*)f;eZ<$oL%iStr@@IO@>VN)t#-KjG%jlDwLYH zXy@MLxId-^Ea~3dJ7f5Gtj^}i=h`vy$$2~ATHG|c3ix<&d z^@?p~tBiiMVQe}>^Ews-z;OI;i!%=^Q9v078P6urO^8>DT0<aBuUP zGRH$2`WVB%Xp@MDrZs|`YD>OrW*U2bQL({tf~tjLEy0EUs2r_|nt0}Er6-WJkx#`8 zuwura3^-xnbZ}Q!GCjWtCmY(&)b+aEG@g2F7Xo>9on$`pp!KbtKQBzltVAUE z|ITNhuFd@kT&S}?j*xR_o>xlpTx#}Oxju|dfpFI^25=!w#Ve-ioxfCVU{aO zHLoop_tPK5>ep2Pc1iJuJnk)vdy8?xB7ZH+!?X(xV7$fRn ze|_?UNnAzVGeD5CzK9{F!+&OI43Yd6)F8ppMA`hq3bqj|Mt0oORonuzZC(U(_?)X& zAJZ99WScZcF?lukuV`ZNDl2I9*)9cKHXM4u=veO#&ImdH*Hy{kovrwS_1iXwsirE)*ohdM-swhL$d|e)<{3 zyk|)t{}s>w9bWAo#`&N)QW3yG2}1;R?9-32$Ca_Qf<#CQGML^uD4J|k{FamgOJQD8 z#fafbMXAou(vKz(vM+|2LPdt-4&t>iwrQ;?r}?NqgQ|BX{fL&(jr55Z*RR zf!Xjk{Nf#oxHB4jY16@e3I-xIzA`*Eta`(fB3;+885Zq(^O-6cLl3~A`hahhoQc5G z!)47XkJMucEgpz5@#gp$P&1vV|5yb%M>}+H88DNU@RlW)R!mtxxWkqnzbIz52pn_Z zHnyz==n45B^5-d^C!@CNyZaQIfNZbg2&3>QNF$4W(_Z-J_8B#4`7`}3ODc3~e$47S zPMeaL(Y-4rw|y|HMg-uP?C3gLB_fEG#8LSyaecF0+36VH)rV|hTqCvbB$^vm-uz5H z@RS(*4wN`E``ENk)%E7>M09AsV>FOAYHbVm`gm7x`?UCaO{I5F?2Q82~GSeWL}BJvla2`h;Q zc67R-IwilL3-BNF?hfE?q{=c@(u>kUF%P7^q%|AL%LiH|(*{+An(NSu({UxX&Et}w z9IbF8SRb!(sS%~j5uzVgf_Or4<{7pwsp3NkpXnB3+ZgqPjo+)x?rN_=rWM;^() z)MTD+tPTigprFRf&rs%vd180}vljig@50dcQGkLdOo9IOFi6-AOqg;HQM<)#228Di zH1@{rBdmAWfEf2OqGYz7KX&Ce^HMhDeiSg5>m*AP^8boLo?zE*;AYdN@aNkZ4w#!a z#UaCDxwUo*YZ!-=W<(ez9-cmuDc%}SUCa#pSe0@Ysn{sr*bJDX%XXRz%-2cWerPF0 zN!)BgA0XZj@$d7Rq#)lAOIo$gvHFIp7oD$cHEv~#Zc9}bKkv};O{JzmTVqL&c}7If zw6oo!-d_(SsqUqs^xRF;#8q2-iDo zuq+>+fCs(PauI{<(AXlF=~ok2Tt-)=qljfc#WJ;_IFZ8rh`bdxP_JttVh}0#?#U^Y zFYStR4es#Zu%%ufkAjHOs+{Wd&cl?gPb7j{xaxURrIeE}rDD7uK<;x)G*|=j>$PGx z1v0RP2*sMy{SaLXSATF!;w2>x5%DdB_(7ep78&E7@LaP~C=FM(|M@n6EwultE`qj& zh{i00B`|D-7?YProY7@@Rk=aQKKIq5Wbex;WEC^sffT=X!(?2QXk_5R|bmj?rvH zG!^N}vPGacH`qKeJ3lVeT#M0-VgU9i0oe^-_xyrChUJ{U28xl8$4U2@tzzZBBku={ zkMKBzvMy@n-Ix{N3NikTWtRXNZ@&}@B7SrxIJ5GS^(^{cbYLU~y%Tpp>XBt=u}-G3 zj@FS5)R9kVRx;{)6&QJn5Pj2P4VDRE&{S)_CzTei#6>rE!{kmJ=HM))r3R#TcgJ0& z34@zRQG=Uilpjjy7p94>*Gawp4zhF|@Wgk}!Nl{n^q&xp5vCJ951+Hu@fcbCY-k92 zu`!eg>NSoSd?1y5Xs38TLOfVTo`Y2@1QXBihF*ZOa>gmjpWP#$h@7+e5KaF4uXgpn z6Tq#*ExOznFlexwf93WHZ@JN2AX*9wEa38w%(4m}w+`(T5|LgsWvXV| zcB_yGdKq10g3H(2mDrOc#N*6c@};Ym;k!(yr)7|HT;2Ct6+) z7l&Syn)Vm{3OhyC#n&X={L=te(GBtOlr>n-V2nXpL#blNlN}nkumT$1WgG z=o#aEY)6baU5*7_hBQCLzc<5%fAi-JLQGrr-y+mg1FfW$KFG)jTBd>twG(9WR5?sx z)>n`*$@<+_F|SQRuQApU?_L_PMq%2~W}OmuD9;-sy72S#Ug7?Cz0oW7!&!m`1EWF% z?Tb)@+Aj!!8SOJK3=PcB9isY||Fw63D?7-8Dv~^Dx61x=Zc=45WA>`H*6xWcEoa5w-rZ67xpT z7}(@U;}xR@e{AHOE!0g}LP+sg?LiGhUJo;ZY>xdsvED|IFYHIbY}?;qe0-z_hy4G- z8VT!0jNJx>jb%Q(}h1+HjOg`t(UI{4Ek87mbDcFrawqzasl|!IU%uf-Z*0Ze6m4CS?qj&aj zsPnt9#NYg$cC3{&rD%*zI@;!G3lZ$m#EYGE<$1QHuPR||a}{ebt9X9>ud8Y+X>Yv- zH0OAV2&K<1OVOE+Y1|mwl-kyyN{bM^_ow}Tl_)7Tht?$C=7gP4c9Yybd@C{M8MG(8~;68b2%8bFDeDi0t3KJ9lhxaNtkp zG3u$kz)NIYF`YF8UIG0>0 zmQ{?rzFLqpk8y4}rFlq=wnMY80ab6JEdsNN)g9jg|Fd5g&Iw*$eSRx9;!v3hczYA2 zf~hq&G~~JXgY9zFs3@fZJTnjRA1tuzoAFD_@rcZ{!Yrn4H)SF-iq zocXo?H#~CV)7QNr8N-(;i2+j2SE*O+LFzBIg$F9mxKfZzLNj?Z^8z!rdEy+@C^aUM z4k87CRGf^;f<2lN4&L39n+7|pfc{ijuu*#flx^9<=AT&+r1VvmkxrxH4?mkcnZ`mh zTag9l33ifS$JwIO0;Sn9ZF8U%MM25Y@_x<1 zyP$>cnWIiUk#CGXWg%?qKe`+kR9`}M(H#SUiq{Fes;BFAs<%s7@IGB<=*&z2(OP8EE zSgo!emqGChCgB1cG!&jK4y7=w1UZ&&R4zBGnZ~OWU`gw~eViG!&6MTvdZR{y%4axI zM1M1AMW7OR_%v=)9z5{@_n(SQvs*UEA?H!E677g4x;FmkNAMw+1z&|Jo3*4yR4^qH z3fo_3M9FEQMkP_>R};QPbH$RN=PLCtk`pIj>*9!h8P=mgb1Km9j`K0)t{NuJp?)@v z@`lkupM}q7uNzzm@-kHj_cAqz>Bg|ryUf+zMcrxJ&-&ISND^{K$r?sxCC(s zf498@*BvHpj!6(epbI$DLFoJezn1J+eMTs{;Dd114Sf~dN+5U@Oi7`HKax3=F_%0x z&Wk%*V^(o#!JO{_csHUqZTR2O(v4Utk9?=rzQcnG(ei5-r6dtF_+m5D%&tn2=t)D6 z%`!BSj)5!jMKa_oyim{lcpGU5k<~78v1LeAFZtZ;g)}zK$6!-ziKfS^Il`$RXS7@Q z(CB<`EW+#zq=m^xL2T06f{w7HCurTroyTa8pW!^3Gxi_^fcX44ZqgBuw0?wZ!UBD3BJN<6`A(*_PFzQXa3%;949&C{Q8`%}gr!rbu( zq66L^(aB1lsy{tdb7JnWeClG@QXd|?_lB?q5ac#WxTdgJEH9i(b(W z#W4`6yr25Bbe04u9juN<37j6gyh)=(55m9pqgV(i>HP|#47HH)nq6`WJZZVg@9PVM z$QVeD$Asrwq$$&(qxDdgg63Y?NJ*ZQk*8)Ao6lj~bu~wCgAHYdcuRE_TrvQj!ky4# ztyHtF8yN-W9$}j_#%j|q>MAxYeU@4$rxc4x&1-FC*dGamzvvv$crn_%y}&)Z8G?m# zikfazx(Go`I+t#&v+S&y4*dcxX;`VP+YPp)5ED`T@xm_We;m9QkXscyf@mYwXqgv` ziOgh?(^A;o zVLR%hM){dzmJ&k?<&czpnL0ZnB1tt<8`3F{g)uTY^sffvJu)V|_Rs}@0vpb)hY`)> zp5ia%bWV40Si|)di9Dehk4dpwT3g>YUzRJzEp-#RHw>p3;^v|amQtm04>!s9T|=9| z`a4ib*+6ncMiy#5V&ij43+|^?baSM3!z0EWk*1kFGFL)Vd8E*Np&EwOt340Lm`)n1 zG`!p&{>UWQPF#@|QyUy0%84;=^hIRLfv|RD!I>E63$l~Uu+QurFT^7bK++r2yLiOZ zd=u%M|3#mP!+!srJSW*b&6L4t)EcrEhcZLcc$U=%_q~I0%pD=i@i=2>1(x&cDO$9f z|3hCur)xR0^uijLRagWm6GGl#3+(-O#zT{#=WqOx$s4xSKYC1f4@C}B-HEtEqN%>j z39KQ;k`4@2_DfeVYeR`C@7BZD{SYbx+gKL<->w{K_{9Wt&8Y`lno@#O?y&d`q{8(L zkhYsoO6p>bBR<5ZVyPVXra6)Vjm1vqif@{sp`z@POKRwmrQo<0o#wz6i%q05w*pnq zIj!Gfa-8S7pVhJ=oIx4!{bkX0>5cdlS^sxI;;F?{Yd1e43U$c-z&*$U+G3?rr4jCI z-I|lW%zKm`=^ha?m(D4r<44IAKUO9aAa*{{YQ_6JiHy^$J8?)n^5n6_HDVjuRVULP z-p{bqlX+?YQut^!O{VM)KpdJEzrzA%+>jjC+$fc_Jp*j+dBrkfI_2Bxr5Rl=k;a5b zzJHZ@4rK1!i%lroP|7}EHS4|7lBVZnRN-7>uqo+OoRgLyf}`-r8O@0g%vp2+Fox)U zd2A1cL`x9KXOq(BvTSarqwHsDy+zlay_H3(OaSc7*@w{9}AT9GNOo%jb1A}>N z@_*$VG`1~pQSu%h?fv45)BNU-lL`jP>+W=~1_+dH*_@iE`{Xq{va95B@n@?wmOPf+ zd^$%m&9>!u8}vSpKZx77E(~k|>JxmVQCjlHj^#5k{K0}8Cz8w*45d64O#C8nmK@S* z>F+GHGUQzdmOtuY@zlCtf&00TS=Ahif(BNKb)Jav^dj8c!Y6_GpE7Z%E{6R-=DMX@ z#U)}dacI6GzmZ|WOnKglBm0oGf0x6wYL7+Bw}J_9P#<`WWl?!rnCvBWx<{@QtjJzW zMw=9PCyo*v<_b{zy&kAnSt$dZCY3Vdd5VqffzW%cu~`jkL0qZcjVie%V>HC8izG{~ zIM*cm-4wcx=Kj;nI180k_kqH-*dXTA>3{njVk5|~>t713`jiD=#jf?x3`! zj2U>nx}d^GSP$PDgt!AA%JvO48kT8+L8sq5VmQHqqp8GBW(y675DsGw1SgN$Z|WPX z$d5f~MN;IVWiptX3Yc}f7Cdw zjX#c>Oq0i7V^|H%j%*drmrEYldgR7ShO$Tyq2Y&t9;&UfA>gn5)w|!j@WObHsF~a8 zcy(4caWzi+dLy4e+U0kY9dF>7CDmE|JAR5p%YMswD(%__nl!B{eoL94F3=dyc85)4 zkjvwDP`OWSSKhv&(eXU)u&=R7kyf}J?4UFwaiRM6CAdfQA$9+LO8+Eac9X+r+4xCW zcvVBLjlg!sOQEYU+zxUAXA|4&1tqSt-orN~8kP}kY_+81E|20TGmHE#9k zIqMgw*?f#U{ZZ^zo7+0zImCpX%R=WRHu?&$)$09M!$PJX88oRGA~P=4Vb!S1A!6w# z@!vjVY|nU;cD)Ph=)wWZlcOqxYWxBu#rlsG)iOrOb(IcWg64`*rJ>qn z6jf%}XjGSZ&K{@ACe(%nHf)loa6gg$x0`1&^c|LigMk%8fPwM+x0|B~N<*Lq;5F$A zdV{0<&xntgf%>u%v@i4x_*DEig!p*&y-Gc_wnIaekYHFOW|Tq$Lab|8S;;aAaQ`^j zFdyIgu5A-ZejTB{HIjxzSMUe>I?2<;2-_@EC}U-1Y1R8?X}Ki~03XmSkyZh?L6xA4 z)g)1sj8Y0q_wArk6V1qoB2){~U&-zCrYD@+Yqvq9qoI6ao<7+C@GEVqr?UkcDqhRT zSo(El{7z66n`ka74Y%w**CjV^kS|pC&W=mWcjN9dHFva(sQWOxj*;$7C zQc_QG+_+{kgb|m`$S03TU7t9DYe#n~P`T|KyuPDV!-J^$fE$0iItty%2X#~A+FXb; zv1)4pf!eM^-Mj3Rk432 zRuRK6q1d&658~f$-IgI1t-SowN7;{%rPIy!-d{J68ox`)(^$5HS;$op^+iX&VTXze za3xAcWTeYNWB(Xuam9Xli$7ew?C-~5z-y0Ug_ayxV=3DQXvNM%yYbjQ@3gqvu58^W zJuo>J{30G#A58!F;@*+K3-c^s4=Jb5U<)mRPG-ukR=zv71tDv7S>c~~Gn3(#=Nm-_ zcEZPjEM;^xV>4#kACW?c>2wIpM^5FlBfn=Z{Z^hi5L(ZZCrY-V>sPN}h!;xx7Hxw| z_32HIMBWY$NGqh6sA7a;F4_=Wu@yE4OFY0V+y4QJa>D%$<#329WAdDozu*G)mtQi0 z>0Qa-QZ-QCRTh?^OpaI*p>5`nFMyKW_n=I45#fL-QVs5X7 z))E6+k=T_+yHl3(O zbBK0f>A31QSX$D=e+Xn%#&Y7R15I{g`(^SD;<0HMoJ+%XsLhiu7_7MA{XAxtX^G8?fb| z(KAH8A2a`e7n^mV8@PfdVvOckd;P}PbQ~i%#+DuLfc~u9?$@}%z7La@-`A-cE@CH% z#+ggIj56H7LbTsHjjMAugK6hJ8dH@@`J1?Z#_Alp6A1TA~=K%gq-%jP%vp?9;&|3Sn(sLdM7a-lYuZB zX9LUwPiB6+=i{8JHfS?MEL1O>W}JU2%~QHCv|(LK%|$qFn)k757kHJ2nj{)l!ZU@J zT7M@4F_33x2*LnT$YcLHjLah$^KoudusV|z2GG4b3J@XcV_fYHEfD~r5P+buk@1^T zSH+Eber#Ed!kgNQZU=*6c^0OEd7T%X5)h^4R~L`M-z}Ykeg%v0hUcv_%R(vYU!r*+ zWw?_Ux8`-+q@?H}Xva(19%Q`dNzxnSaa5Zjyl)rFlO;ZNUtBjgGWp@X!mh_LyK~J` z9i}y2W+j(jFtc!j2$9=oog7bVZZ5FF*eIFi1!bXykQE1x^9Q3nl6F(eS$Oc|j-I4( zsa?@C=bjdmRrF_Am=&~j7MU#WC-VPJ&^3sogixRvaaq8>UEyR5)g`n+WhZyQnrvY> z%wCSAYkdcSqejlh?x&~ZCy4u}^+C9B|5pnErzt7|3Te=@|0D05&#ceZE)e+f#W^S~ zj-!f;t-|tdSQ1Hz@lv}2gN<46OrjpP()TdT*fXS1PW#GtQLr2L#o$TgF<>^ObKu)6 zwWtqY9tALSI*kC=VOA)r^+bl|!uDw>tL5R37QxhK?*YN-Dk+N?u!i@Vw*)t8MjF{E zrmB@q=Y#=oc2|1iT2fS;kc1;WsN3VmBm_5x<9hT+c08fdq!-BKo?G;TKf*k!CDdJ~yj;e45OBIE z<`NPnU;cnm_~ADYj^BPG5O99Mo{^SEg@T|Hq5%+bJ?kkT%m_RyH~&yt*R$jzzS{MS ze?PeCb6(4$lp0`1QdC5B=2K9p!mP9k4PY{}H*s^z)$uV_U+MS^n3&$4n1K={>8F_*C))_6 zK^w%!=9K+{4>?4pn4E1s&Vpe{CAwV^;!!!WS{vd)TU*v@3&k0va&C8_YFFJVRNJd* zwDsaIr&a5KrSs1J4;8mej*0qfcFp^ebK`%a2b}PG847(ont-t%R}wZK0!rt1R!^@n z@xJ~A`(i|n{so8iuhJ*CyBvQEhJ;=3sDxGv4>5>%d5ZSch=yD^aK@YkaDF?7pafGD znMIoWQ0k4*xY!)5vT6z#E-ZIx3A$p2*A>gT_Lx>AnVhrg$Hm^t5?drC8BVO3r^MU5 z2Juf>E5N&B9zfK`n$VlA0A@M5`)Ubln4xM|yLacTA`>>X*%>iIY*8qLYs2-$s)|eI zjqw|75}?W~`6X5@#WfYmB`k|VDhWR9MR?VkV%-^a*(j<~zBIZBrwNMslm$A~I!n54 zHnA($2rbL1XL*C!WSuIl%888*ZQ7?4#Rpi}V^$40f*jL~Px zl+a2kZ5~0ksfA4uRBEVGrp_-lX5m_Q{?8cSqTs=!<6GK=D!qrHj)=)Ax=!U(xNDNu z`N|QU0uwea=l%~o19bRdQ%iXkO9?3fT)4Hd;sNK2tg)10u9e2Dc} zn9I;+td0(Ino@eTwCAcj=-kE6*76{akcHK8-T`X{!`%((4lbC%8S@|O_EF+9frnW& zN_Yjnb|kizmt7)>*{sgA-12Q{#>>&E)wY_;4mH(v2CXiD#Z#5W6jK$(YuZ0tnz-UL z7Dfz9rDWG&O6<(rs6}l~R2F?vG#X?Hw5bxt) zx}uVfuV@w6S~O;u$7Qn-pX)Z}!8IZoXJRmQZ%-4?QlJ#}s=Cx%M+BO_Ii$8j5XbtRyg_tBRfgq zFZ&zNv_zeBRasHc{|2Ew%m!ZmGuCY&Gn4!w;tMVQ^#Vl!St$Mpb3BypnUYbbR24rT z?z|^D8dIz^w0nHKc=n7no7JTXrM}d%eS>}?;e;H9qYq2j(TpQ^*LU=E7}4DAWrv7E zm8afmM+7!fw<^vWO$ok3I&30J^TE5Ib^Cq5ohrsHd8h<0kCY?aEz%mJbt4;M3sUWU zYk@Vn{8{ak!rU!)*?ABcTWNw8<|bUVS*(x&Q9=5xl}RUpbJw%PEhJ0Rm~-iX?_n+> zlN?i>(>s-uuSbt6m>1p7X2q8JF9Im<6qaSONJ?f_axt#6?cfBAt(ME4jORgf0>Qv< z+*x|fG@a9$q%6WB)qMU^waiMGtgLBm`|}z+cE}}o7^z?~;su>8%g5TBTFpG(lvIex zXo?u#=x{IM^o{Zk!>?ztTn${R-vune%3Vjs`fJxM_Ncb)_ok{|G`1zf&bJP_ztzNT zS)1Z7lGQ?Aj|P1CH2%RK1hX`n$OQ+u!fccfUxiGHBoK_9E7l_0G;SpR5Jx;qWQ-7( zAe9A9@X*s0@jL&KsHQBc3+(5KmJpO$(3*_s>GizM{vc6CxXYB2YK^Bx=u>JDxW?I2 zyJIrNVo}okO2;9%hw9KoGq4&*-jMNb{a^-u_@J$VJ#Xtd@) z=RESId|VHm@VdJmMOu z>f41LZX|7+B4Vf^G*%rMb;AoZz~WSR*Ki(1ohs_R2xGg-J$FebH3D^kWT}3zZ#!~P ze@W2c760{@nKj}vO%Sh2U8)!eW(uE1j8g3jHac(+yH3i6WI4KUtuI)qnipdQx@R6A zVDF@K|HL(Wb~E5UQ!DPees-j>1XjC8znSI6vAW4oqtJ7ruiGzbvH~Q(62im`KDPdei$s7E)C+{IhfUw?xp}4q`S#5l?xp7 zxWuO2qkjTh`V$>fr_FLyG(b-D?~yuwC0KgmGppaekJz4A(xuRq=!kDfJ}c#8XJ4bJ z_9-tTa(KOE`&32F{Born@k$Zvvl}O;JF7~A9Nd@Q{%Y;0Sm97#YVt)!?>*Z-OIrXn z?spsXp~J_S#nKegb`Sx;>9d?TcI1~OP>1dmW4wdPyBXmW9d-!r%ZPezjSyC;$gzIp zWLq4W#Q?KFxjkjerO5m_z5HWH9uyF9sTC5-tlxKA^+cql89DxP?onyDF^V)I5g|aT zv{(@;$r$rnA`~6TS0$gFCI+;#5t*zr%k4WbJqMuR!*Ayw+4Tc8g@w;S2@qlu@gMoH zkfr%2WE?4Gln)Q06BB1ds?2>TxHVIW$1f&$ShQ1)<>%a?(_fbMGbwG+)!z4#tPkOa zFFS1To|KEa>G8E&{c1vT>Bq3Jb88MBa!Y=ybANA`#c`i)Mkyr0{{zqS+VYHO*Y+n^ z^+y4GnY|E|SWgMSygX9Fh+56wHd4z>Q*)_ra$(_alBDL7)5bf$QgOY#;C=tDrF^RF zU8iA$-0FVMb1s$m2nlBkjeBaOWpv29yIl4sqjZOU$8ejzs*OS--OJD_GhWZ%*)yR);aLN(6SDy|8z?SvB2#D~Y`@A&m_otw=Q zXNpsSP)BWSLE|!%^dd7U*myTeUTNo9SbRF9H^W0;E_CP7bq=2iUoJq(zK!qHG?)TC z9v5~&wqFAAz%W^YKb7LYQ46oD=oKwk-aq?mVtUUePLhv*6>>%B?M$b>SEhm|TgY3L znym4wHH8u&Xg51FQzce7_YJ!&QV~kk>pcnY%nl$`dDgWZ%_$nqd?r>$>b4k&^r&Wt-AGFD6dr$l%nPZl@Cdjz zYYrx67uFKY<2jwnQ1dW zX$cu1+~a62CJrJfuc%7zVelY`+>_J%#QyMRZF_I4;&RIA6uS?=wei+FdGPFI{Kn;% z!z2WNX}K_OBf;$X!0Y_3N%gv|Y=W{J2;XQ22@aKw8NA|e+4(f=D!`A}3F%2>tLeL< z>Y{WflxKL?NK7$6o7opZLac0s`tf{vJR8{G9N#!@OAU;3{I*hOO}E$Y8MiqN|Ec_k zq7j)?%NcAG9I^Up*o5%ADH?UFbSi z$>;AYfHCtVug5VXSYCAT*l}f!3=rh8;8r9pol`ha^JD3qJbZXG{sLD=HXt5TuzfO{ zy&Ec%N}Syogc&GN{l@WiS3h0HPU*17t;4#p<&-04^;j`6xpwYKW;NWNuorLDO+VNz+X}aWtJvjI~o;B>9@+{NjPp~ z3`=;Fa-0UwSDc3qeDE3*SN#ws4rBgRHmp_F>(xUM`+iV6=4V8lH&Xh(R*64K+$&6$ z^OqSm#Ijh482NA2uM1QX>l9YS2IpwW9_tL*s7c7~<16cA+~jm9q*MvFe`)Q3RI7PP z<~?{HthMo*1_Pmfr{s^7JKN6e$lEfyf(hM1w!x!kC*9`%VgCq>P0T&ddLZSjH~=I@|mG2f6~`#M$BPm0hDi!xOv{vazLu zjKRjftd?hKSHHpD=9D^<^IXk+3YFXaGy~zFj*fQI(o8$h_K0c;cRp#4I=J2l39UhA zPIoo1u|xfn`8wu!;t|~%huMKGshvC2fil30)XWSipeLpo9nf|^Wte+J==KPdD_ zgjY?wGcSaO;fW0+Pm&qnRD7Q&_K6_3J)w2ZHKy_9Xtwj#95N9K|MiNimQb}2AjOgw z)2md;2r?I*yMi6M_hh!@P9f|kJaELi^LB*Tcl!xGT~99u{tq50VDxQBY}HdgrfZs3 zk+_w^J;GX%(;hZ!s?@Z{2E{=YhDB4#P{~c0*r>R?zl zPCUMpi}6HGcd}MGbsFzIMn(Gn{NsjlAaE^;W{bInlW2GM6N2&6AGzJ1SLiP))41j% zyMNuEkUz)I^u=PiTrIKyWu=rv1m&6dikx&6Ry)?X2BEk?X?8E0EC=O*m-i>3IMXmqt6&*0`z z=T8#36fWP`9mK+&*unY3!B7kvG4ll3AMLb53!UJ6)G0$CiRiU*J5=k~qnpWT@8n+Y z&tUYE0j}bQT-1uV;PpF8V88Wh`_0uWY2dJR;8(Rf0V}umUC3_Zh1wF#os`j`07L{< z&TCg%!{G!Cp{cD20h1exKVgNrCrC9Sbd?CxxMG{Q9!|RJ1mFA_d1LeSHV^aI4k4&(I+^#0cGfL;Rw7%A39B)e5 z3);`!jOa$pbn32!B;)K)2m?NMq%}(Q#Vm{}QW_lq^`(d+t(fglmZ}GrQI+ooLZ?#q z2=3;3q|Grr15F%`lgnHi?J0j)w$a0KVEXGb@!b`-cG#{dYoOHMPx^4D(i-I50ic3C z`sA&?{fp*}!Mm!-x@p~CCAkENqaXBtV;fI_DNPwxnE8dp7%8^sj0UPJxW#IMpN`CT6* z_$UQY5?LYHzmB>$fIwm0lonmT`BrKXj7z1arIDt?GDCH)it{%xu9Y-hmW;p@c?LP~ zsHx;BjjZ9Kmzr~$t;Rsg82>#b+&JL}_csIp;RVH}qpy+boj~{2OMj%};uh}8E6nGE zcHxAk_*r`rp?JYEm(6h&x#?aymn~AQe0qPFwkf7p{Vh#Et@-d)7iL05o#h)f(LhSV zzEH#$`YgO;wfpW@r?R>LRjnTy@oH4FfhD^JCYqagSEHX5xgD>-9e^Cu^lrqFgYyaW z9nP|a<>LADDH&v9>#zqg_w~S#5+l|*wm0VYr~H|LK@0rC2yvt6<34eS-Nj&*Q zJ@Iv6;2C~35N#xpOtZ;#XOP|?#Ub`eztyER*PfZWtJc1);mXu6Ty8AYzW(61cB@uX zPsEj{EAi|zf_=S(UoJfY|N1q>-zR&NK5NqBZ0`xsgVKKjoK3_Ah_L^b$hvRvn#3*O zA^vTfDOoT5$F^DEEnbs=ZO4Cz|JL0=2=@5@PvW;H$N3+KtGf%!I`H4r3im&Vo}gf7 zLO|EXSbD$!Qx7MVj0XtU6ODL4k%G;8wKx-FshQ6 zD0Ut(i;2|HlC!exkEmA^_n{(p6W2|(&YOhuGolShoN1Am%#MVPU9Ydjz{lT1%AkkW zx*&m&FY$*M82P$9P6wjkIIJ2$!E?alS#tQ4Z+Tf)#-?Br zr`s?*;_M8x?0yUrJ{B1-wpG@QimhtGaH@u-p8S(**gDX7kb2|v;2XJQlF20DWQO!K zsB6Jv7)3r}k@7P!s-i;sw{WUPojD;Z;`n1>EZ0FT2R-T=&ZG-@<@an=vyI$Gf1=G% zIDcz1LJak!HSM-RWpy8K#!7(8mP{2vSW~=*$lv|ao$uG=a-8Y0tCpK5knEvly;`TZ zQQp`S?{4)Mj+Mh@Yn1HTy1k3*dY=IT!G3$K-F_w_oygKY<*t9Ur1j+ub3*$efIx23 z_Vok5R;29tZ_q%jnh|-HcIzE#xxBxf8B{eFaj^bOt(J!K?%MgMiX(uNRd(r;v)pc) zalH^BRm$ylDF`K~C(Uj*>0YR*wY8k1ekZOnwM-TIRMoV6be;W>EkGOd6`!o?yFDZW znuGhbyV|Z|Y6UB1wdjt*L8ml_=oYD*fFCLcWPsdp&%z}e>V8qQ++CXVFP}}$s~sW1 z`{A+2?+^1v#PV~Pwd4Q=R5N?+lVOlxx!6~sq84E_>G zXihi*{J0+&qtT$Lzu|wP_fx$VKEM3h{poaMVy^PCfZXYDz@dVB5iKN|#onA?(R&Fw zFhA~P97v_QX)|dzU~9VQ2CYlO!FtX+a_O9MdqqV!hi7lHiwq5TH2^S{*IGxhR(_{r zH1zYQ*v{CI&hh5?u5hP)oEszNRWUNc3QaDlRNoSS$Z2Lj^Bf4sRq>zRzj$HvoU_zn zPL#_C?sJARN1t;FV;8f{B`aY2p(Y>5qT9pB+vO?B;4CFSV|GQx#p}EqXTIkdM=n6J zF7Aq)gc=aO5e+di5_-g=RY{r|T;&u$>93l8{vRq&D&Zx3c$u9u+6BP8qkF&}+MB+2fnV?UCMbMG@#lcT&a)}raSPV14q)0NI;li& z$FnQ`85=2RF=x(fR)%&GH9cFpVj-a5>sWvg4BV!&P7+%+(qImaxj6n-*`4H2TcDt}vXv=q}`9>9m&l_B;=}#&=DSfgams>eYF&p6?nR1878+6zipuT z0SMWgv4G4#h$(^sn}AZ;Qv;DHOxKqydTu7=&*Ddi8&S%f{>@#5RHOxT#vfiIo%Z{o zTs8t&uexkayzHybm#61VF2zAJ<60Q&beMZ;PXris_f@#_izg>5{=s;X4mw#`BtYY5>ARFYQASXqQ$;14Y8?J z!?q*(LwvQ7TkHKuzThf`BLn}3C}RW8Q2iAO42%Lq>cItUsbOg0eiFHD{9yRbN*5K< zASwff!Q7(xLo<$M9*Pm%p?Q;Og0#lb^USh%-u9-pN8ly6?E~(aa-FSZDc!RS03AN^ zcge_@cK1(jH0?gU7;t|I{r$@tRQxI<35LODI0OZUBbJfELTQG7GkGhL%0h0UDT$bF zu0QI>_A3#<8taZl#WvR;&~WMgDypbl7w&8@2!pQ5M%GShM#xR+zaC}iPa%R}qOv|2 zQ%r))WFj*f2u>1F6!wqwHObMD%i8RknT{7y?ylyg$o`5xL67(mtFec;rLl9=l)6aP zcG4mdN@KDzX*Y^BvpV7zcj}eq(U_#p*pEU zZ0$euh!rr@Awoo5oHJg!)Y5qcOy1c~F0W!=`BhVJ@5U(&mABuYzrPS1rq(f>4cZE6pVluo}otiRXZ4I zva+}!gdhGo+6=53eHGI`LvNKU1X2Y_*K*k-#cJTq+g5A5u=U>exwJryB{2Ka~D*0i4|$T_dVq6E&0$q zJsr!VP(5IC->Qvtd})&_sY@Z3X0T}8s6>^QyE1zBbgt>N@GI))ilkOeaF7|}v_jIZ zc%zGT`Dpe9p9V-hKS@3OE^Ts|pBzvK=B4Un1i88BhIZ~(#y(=%?sW~`eAg|h$Q9? zZD5KC$A_f^CWE*HGJ!0nkeIxLq!{ zrI4#i&3aY9cSI|i(rCCl6aaqHpRilSA9TJ&AGh&zgJ!|jB;BAAa$X~Et=8xrvjOWw zA8e`9fVq{5Pv*xuvG{jDEJY;fxvN0!w8V{BWG52&u`oZ7Czd=OQKDY;54Q^{Wpt0| z!Y(-y+I9)fH%1;8esRwXKee=fIp<*rwY~I<{@= zjcwbuZ9928>Daby+fK(;$F}X$-_+EpnW@@8Vb|XG^IYp%w?)D0iI*>G{f49GHN;an zsoF!e$uEM>^Ap^)p|1^yJvF>E>`z?1!r0GVL7Yc?zdr;W#pqUH$XJS;*{bDKOV)9M zrO;1IpftiUi5o;6PIM3!)V$8e5HJaZB|i{Ru|Rmip1?+-ILAykdN>tMf12)Hd#D6| z|F^5*`VKqM*L&=L47EfPXvxg~I&!O!|C9>0gF5DGm5_} z3@3i#SUbn_hD*f!2a?)%77|InoX_H(LU^KD15D(tYla8suoh3v}2o!512b zzK@J&Yqi`$a-{qxY*)XG(y^lgMfQX2md~?5>G6ioAAqKXg;Hu6JqZqP(Ql$tt`%Ja zvnljj>?@O1gP#}C?ZWon+LrT^(qd$WVcjR@VJUi{32$Oos-HO77p%W~5_L_ZP=sHn zquPOgRkKiZv$K`c)byS2DWFvs)-^8=LyqH*4>Q7d>^vu?Fw9G@QeG<@+7aqc@)7|Y zwr2^c1E%^k9zz~}3tRcv+l6h7MX4JQ!v}zV=8i-Q%^x@?BW^!0Iqma)W<$v5P4sEO zp;j8f6Jn={j0p-%V?AtiRU2*f!P8l4qU8W%Pc;$qrW}Q9{<=P_wTJBM(E6@WBc-zX z8t<1p4;CesS2lhE&LPV$9?9^x8|mDW09I9VR%?b5(*g9C$3usXg_3al*Q z!g}(VB){yl$b2Hc01F+$Cd`e{DcBo=L@Z4p6{u&GKH}*M{mCL_n8OlJy9XKY~ zl0^qcWk>mPo!M~ru5Tt15IS>Zn(RH1rkF|r;*P~~dy=Pzcre0Ytf-Qx6(k5vET-i^ z`x28NF~)IiDPc4&j6t+W8S3LO-KaknQe_c%-7DfZC3Ld>8v6PX#?BO?)XJ(@<0t6w z;z6!+@N^8l4NohwOCaf) zB~esXWb~_dP}B~%oRasx^>*lLuYW*%#m1LYGRG4*g#-S=hEyCy`~nep3E#c2`ztjs zACc|)oV9tJ#dpkQ@yzf2wJ`HTzR+}sO1zLYi+wiKBiWlG;`yd0fuHe)cp3+5unidF z0#N{iG%SIfrEwo@uR9bJBZG`alD6oyCWu&rCAnjAc!`Ry;ihYHa<@`#%{;@X=-TaE za}k~ZaKX(fQ!Pr(bmhQXM=y2)&1uA&5SU?1I}4L#U~F5eCOpJ3KXf0nRZGF7Lnph*TAPT-jEVf#T|L9&U79YssezB7?rx zoc@KI`_ZeJj0Uvb!aTxMDL^@>*jEXG&3-*k+qD#|b!VszQ3^qY%)^fIY6Tvp-$fLs z#a!B&sX^e??sASYJFzl*`_CG4qVvXTM?`T8GC*uvqs5KXegf;{;n)ZK4>JBExR@t|i2ZxyqSsVVaGX z3Q?3+A}qDfNHkb7O^(7~rv*kVvhyu%r~*RCbmCOA4kmW65T>T5HZ;vS9~=BL3&<)k zk0Y8E@zAT0A~??BHqQz!*qyEsdYY$VJq*Lo7-Ut{E63(|*({&_70uTDrdluXHW42O zsDec7#g%{N_d^tX zvgHrb0U7?ej_NR{^Bf?bJ%AJH&h&!#==4}dNa6w*wmB1z?- zMV#`I({HGzV@9_J+wZTRvA;R-tQi@lAc%)G%q?MU{}Ilu$incY(l{m7b%HoT>}A6A zb(8yYRM*568(3>rX!M`Mq-gqtB@9o!|3~{0TBrK9`48m!Mg8Lk@qY$_|K_-+w4r_f z!*TUYIy+Es5MvWDhB4B}JVB8df-0e)K$-T-fQnt_TB(-$;X@6F5Uyg&7&B{0oDzr8nqOtv5U9`hWprzXDq zz0UQC4@`XA@Vo;(vwjxaHX2cS>i$7SLZct>P|Ul@CU@Qp=asRrM}N}9xm z2I>--YlHCh2g6`OhyC=pV`V~%%ojDP4VCul(40gXloiTTJ=EG;v`zMdnYo?i>i8+4!fsgjjPq%%qy=+lI-py`*~r4wjRPw0_-p`)VT!04ambG*_epY!U@R1W1Yfi`IG zLIlq|DU$N2@=N%xgykz8QrzlQIyb&xgyHI-=~J%iF@&fy*&+>!Gc|?h>%rl}c+hl7 z9lG!>4g!RMzvz>H65t%!C;O;wRo!8L@F`d}t%!VfsU6x8ypiYD-hsmyPzkQ9^+=lO z<)@-;?e6klX{;f6z)|s*-NcOxC3{qK$sERw4+f6bHK+gVkKkIz_8%ID;1)nx&=vkLjUPp}* zlkUB`Q5CuuGM>r^&|KeWC@HP>@sV(OPMNNWUhmXd&*bsxXl-q6X>Nwd$ov;;L!`AW z2{p9=)5`PeON)!!ZG7_FY*+-%hE$u%D_!(7T!`wrt@vIXp=ep!MdJH~`fsG{;!g}O z4OOoZv73Dg-u{FkSKB3K8cOwdh!CRTzeDNloZDi)4a8f^S%SHnDh-P@# z3=tto?>p6T&>6XJgK2GWo5nYi-AzS%!-NvdR5L1;ZY2Z$F;Mz8C$xYa>y{ptMkWkN zAma@$DfUY0Q4fAaC@PUJn0SiwQ#2X6YVP@$td7Uks?|gKV^?I<&9eXf? z+6!|qgN3yKQ-H#1c}?29AW)Vp3u+iz9l6EtVdhUCw6#>7*I}-T0y7lB`-Dc)feU@0 z8xd!iDo1_bpQv*xR7vaJDLm=;%eg=Hz-B(Z>Zjp;Njn)vQ^`$QZ(uLd4{AnSbhCo( zu=M^qKM$oREijk+Eq)VG+dW681aq5J)&e8Gpr5yC$a*Z6tI04<-K6}CL=}QtE#gQB zqoJK}{iXrfOpl70$ONpR%=XN0i9}n!7Srh{(f zppi8<%Zk&}$=vugR%1pZGbh_uWwp1su{@5UV@*%DPUHrbT9EyV6}VM(Zpz6x zMK$5|TcVF0ch`Ro0YekEGnGclq1fj5C zM5ImG7H|FqqDN^rll6;r8(4J<@U0xl-Ca*BIi@ zJyCy)5^tSr#@Uo-H2XyR1IwQ;1v&WP70oZ>Vd=`TnK#$y3dPT6nnB7&O7aZjV(p8$ z9lb%8SI>&|mJ@MzaAbhgDY5Ir))U*Ccl-JAYXni4=K(a@`a$F^9yo${J1F@*hPOz? zQa+cPULFU-!s2?A(q@1a)?0R4_>m@lJ@13_8NweF4L-Ou&%pMDLOA;}@qr%QOA|8s z3LTw$P|Es+L#X;<6`i|VYx6<3oxz$e8Jb<5XL3g#{BS*I@|7pKIQJ4}^{@hU7K$cs z$vYm(#y>E_6hB~s0~DBXjKU0ZKL!d{h#$yDWsG<-l9_anNns0q3$ij_th|Wv(}biV z1%H9ybD!4UXD(-dG1uS2AI$A)w(neK{t-!Ms<~|qI`-3nV}M1+Lv7R;x}kx{dO_%N ziolC5N;PkRWrGc3VJ~f!4EZo05-5-`2{D*0GMKBK6O-|n2fna<(Jy=fY&y8Ld6r*& zjD;H>H#ajsnD^fsReA&hDYj}u^B>aYLLvHdr_TZ<8!FzM{`AUwm-G_$5WruB!Diun zm&675GQ;LSZ1^CSZw{ z?F;#v6PCft)jP(!{{VBgo6RRwgpDU~_ba2hqde2onN@3s^wfm3t*=SbM%08Xha1!M zO%I`a>@;>*LY#zX;TqDG@U8|396WMA*|af>$Ki?8JS}F$Z5yj$ zA1DSE+;)=Gn7x***92_yXEfunRaw;05H5lI z{1leLzz}@Ysz~6pW-QK|r+X;c6K&)!8s(1aF7c(d6@$f3iM((pTe52JKNtVWUz1T@ zrvlQr?ll^AlA)0z$Ip$OaQ_tzY>-GjF~VzGV6F*N2ro4{3%98+T%4*@X3V&dk;OnXszzVKxn+$vCN0YxRCjKzF8=OuJ zG3&M%(nsE)%R2pVq1Z@&jJgxfq2J%*)c@V``%329`6LnetQNTcc>(jvJ9 zta>ip5u24z|CkzAlzbYL>0)*S;PuJ&pzbbKxAD*zB-b?9MH_2np*J{JB0Vad@U~As z0byxK=&)?$JQ$(wbrFdqm0l&Gf( zUm0!k9R45`OZNNvk(p~Z%2CBW$zm>Jl%H(05l1+JlmL|c95s7t#O436xqNUxe7nm! zUW$93$ib4p&vd56q^IC-iww+&4yqIxuH^$_6I{eXp(nKEU=VprM9=6uGO% z2b_&|^XZd16nuXzfExr!$Z*#&7lEJC#^#IO{;$-BH&d{r2PeS$eKXUTE zrE03qYL79pJ#M$3>wO$KkxO5hQ@w{B9Rh(q` zp|--mi{f714WuCQ)2Ef3<;a=?Bdm(ankTvANYoPk>oHD5VIj^%1-A0a#u_4ut_9%t zUzIR9KgT4@tH%>PLLIU+Yv2>d?l^E!fXgz(FDq&J1;M1Fs0NL(mT27c@Cs%$|9F5x z=dqj!xYeLR+>7l50npK9=1Gi}dG-{~5XYp%{?HR9Q#K7sW_Fvc()4^kS%QwOTCuF#Okskg? zDTHEhm6Bq^{g(f8+_2`Y-S8MSwQTc?cF*Auz3`yAj8}Al7FdBe7X_1hk2FuWP%m9tWL`s>G=_3<|AL1la9@G-Wl=htDo_rcMt9puvD<_F%yRH zeDa}hL$_VpexK_mhgt~*Vl20hS}I**0j|Jz8z4ssp}iZV@x5 z)lbyW6}MdZ2nK=HiHQq-(_nydm*k|md+Tf(b(vJ3I}NmhTB?V3U+ci>6TO_=XCghv zv=s7_NuM&usFY`Kji~wY!eVfi`{#377-`tU*1CUR*le~e;PGdX(M03-w2xIx5H>B0 zki9!O(6AU>qHIfUiGPWSML?($A!<9c`gCgv+WMwFCjQKY{N&=yq^X))=ol~Widzf` zuTcCTjpF(*7#l3KPS-hOqp9Y#-7v2ZuJ1UN9j`VA;5j&=ZI{d zE=edsB^x}9a^5uW#8`VUBtlFO8l{y9G2r>*qvKb)pA*CsbsN68RHb7>^uV z@#0sT?LZ%zBZg2(P}ris*yBDj8;~piVLLF>{&I>_+}Gd2bawh^UxQZ+kTA=8pR01U zILzt%(Vi(EoVnqUTCGC-SovTQZUC>rWo$vr5&o@Ef{XQUo^vPgMiECUZ_Eb;vQD3u zg?QyWJ*fwlg4uPavSK5xBO$46-6zJo2889FDRs<7;FE>A@@G-1JpaslBwBn&;`9nK z&kuW3vehV@E%8YXQN^*5EUt%mqsAjcEY+BwG4;mu718P~1l4pGm(HCCc;Nrsn5PWA zXz+qvY~d5aY}W941iQR>;?mCpet}`*t8IbQ6Sx|0p5GvV|EO|NXLRnz%I)~zP8*anwVW#0VE0pX} zXxu~5O`rgx@_gXPA+D7!m%833gNLlLGqI{=>@dex=&1xOe!_3eEL)|9L8@BM+0V-Q z@=0I4N?M|SBucQhFBkhHxD(ualV2lG#AO`P&on~uw(+UB;yBdhJL84 zgP}j(lP9Xu1BUeIx+@5ZY=>6I;PM|<$uEZaf%~|PZ#C;89PpFvB60Nv^EZz*y5>Uk zcf?8EHVdv#O7kn@mmf0OEEfLO_1M4T0P7_kWclVG<1=uoG3&mj5AQy$}CLvh>UUn8}GAu^VOb zD*hui_j$xlL@b>B9~m0}qDo)ypR+vS-?bXae_<$fPmqalPf);Y4KH6b4gBw(3#JKE zP%El@TuWT)78xQf;dT@RSm+9#OocC)H#GjhMz-b{^ zIs#?ht~AKZP*xyfOq9AQ8D2c{KwVHFv#F`4Q&g`#{4P0fZ<2@p$l!z*=k%Z*bAo4Z zlzmr1==$*vLRUnuK471%Em7(!!HGj3ci=qj;gPs2z^q$oB=g36%S(p*s$VDl#;(iP zhuqjxf_%@D5|Na6V`BYz%oB49P?%dSxyQ%br7`d>>;KZgfRmPK7z zkngmIot&w)?H1TSNogNtr%0Eb3cotNA}X0kqccwreH0y0PnS2;Dm9}lfYyn|)plJ~ zSD~JYc>;R7=nzxihRQ5Gz?D@lS6z;gfm`#Mg;-M*@QE5ygb|;>otzfaf@7pE%o~;9 zFEOUs7uugqn+NcaupnbV=5NqkpOf>jZJwF=fPdJjXBF_*s&F*d$Tk=>dQ^U9g=Wah zPv|c#du6XbG^T<|hgv0gipZ6+Y3sB7rj*GXvjPf870{_kq$E>l%<(m}fB@LZb`xtZ z%CjTM01TaRNkL=s%xvML23fl#CpP^>^D)`$F(XZ?^rOzo&7rL8q_xa!fE@eQ5V=9+ z{h_wpx)gOpo6BNN6+GIda{M{CfqyCZrBta4>0hZPvbGjZ*HPuyo2ArQdpd+s^S$ab zC*b1hECSnMgXBw6kO~R`kqxDsf($3X(ThLLUfkme(VA(QuT(-Its=2rYBS{&3>-qy zZHT$0I$gdgI`Gfg_$ukMCN@urm(cieN%$BP>-uDxQ8BbM535CAgv?7*AS6?TA+Eh@ z9dag`?a{7Rn@~T7t=epeJNSA$xmIum8ZgQzwQoQ<%zo-bT(=QDYCkG4)G_7muPHqB z3tVmMej^OQNKoL+?Ty#gJ7`bYj?7C#VC_yNN>9noIcMn(?n_CK1ez~wEX@l$1(~lz zzsD_KfS`(WGd}qCDB_u zy%(I%SdOZ_D{^4Uw(2cWYkCX{A1Kc|JToMEVockjgS6^gTvgABxzilT5_u zFAj(pa>_NOe5lRAYO;`7L09yQ!+}SMIW~HWEs+xe8Pkj6p@mBO4Hv#ix{)visvb_o zL4VV#%VTspK>=nwc54J&)dH_18)TIz8I3Y#wcXrtmYc;C&GR%_9q^hhQg%g5$|aZb z>*M`dSRyfpcF?-0)GJRi?;gcD_A@`H*izrr6%~Bkr%8EQA4&nF`<9)m7QkPig(Jq= z8T1v&u4?4we&FbxOp()BD(jQh8k0pWX^Guq_cg{_ckOgDIvEq7rq6gXJUDuX{8KHk z3}Jc)=^|fN)(w4U=UVqAS~m0{Lm$lCP*T+T*g?nOhjGHt2qW$Jip-eFN&;>MQob6O z^==)}q{E0oX2lMLT{@MyY{0xjmkld&6nf3bzU)pzb5M_$f&nVjkmNLGJNV2Nr>7FP zgP#q%B!2cZhWRfC$4lhMv=9iC#G=g#k3H)Jsb*4I7{acXjhT7_yq~v7^OU#-c11V5 z+eqQ!IEo8;f0fSc{1>wCAgWA$;#ipUecfGl>Uuo9N$vWS?MQLp>jTf%QZniLQ-p^M zq$eh5Esel&zpgEsXe03c3NiU znrcrc5WSLhAn_=%i04`84EHB}I|1_KC~V+AG!vr7S@yITy;6w|MkuMWvc?RoAU(wn zY$G8=k<20?#E`5qLI6)e3aa{GE#XiEBo|<apV@dKp=(WY!L~cVC2vWO6;znh!)s0A7J4vTzBRPjf}N*YD6s^Z|3&IL#h)Y` zJ>*f3GUhL?xD8`1%L#`4CZSe-DBQX>KDVdKcY-Ae+QtxepxkW@Wo(XB zTvxD^o}FfN-{IUncNboj+6Y=W87UNf#ByIiZNpSB$^~kM@z)c)53A!Ltvg0?hKZ0s zN3)g>i1)S;GC3hQ-9qtI&zK1EBIxmk3*9ongp{FRJAAQQ^ zKgfVwPxBgaMfgP7?@;?eH@f(uL#QW(Dgj1LgiC86dwld){Zp#h{Ji96j<5&`N%xwU z(g-_ZL%?FBZV&gcg8X@TmIK&TN@BA7cdN{R)u+ikA` zni8tBVN(i$2pSIgNaS58qKg12MLvuAFV!_^b>6@%$V6jHJjYWvxQ=@yxMw=(#JeAh zv-+@Iz4m`E13vx_8ykQv@Br=e&o_qs4`&Db|EWax2^E-}r1u|FO~|%Yt6s;ZaP{_! zTCt=Kj1Ei&(lC}(7=;*vb;izS9h{kw)tG(d7pX(rtZ2d9kPO3z(g;iAI-)c@tK7`w zTJ|&7^;-JoKi|>^sA9j#PbW09GzCSfu{xp!F?=&FdThzO0+aN9OHbcXbOxQ!K%<6^ znQfN|E1(u`3~kPAu-;W0zJe{x_A$J6JhXaS*IJ#m-Ha8xWimmHF{1(7C%ebyBcBmr zSrc`aTk3ny4fgV##Zs{KgpFnZ^!e}6*q9)T;G_+r=rhanYJ3`IHMfjCw_`^gPlwl* zO5JALtL66NKfGOKgzj?Z{-Z6_Mw(N(SB_a{x4=^Q1uFrQH0;1A%L;k(bsq#Zywbqk z7q?xm&X`rBup`)OEahhQRz&t!rPZWL@tZK>&F3d`AA?o9kkfCIMT({PVh$Q127NH>(_iM;R-q$chX)9*!qhqe;w*hH z;V&4p{=dJp%IQ_2CU|TyG%W2QDjn6B;MNYPgyW-$(BzRpIgr;3C07gRr1#O1bE&3t z1!+^{kv``KP0PSEp>cEs0|CYfS}^JGrogljfZ!BaR(ZNDyRd-wol!a_>$p$(o`z*_ zzD;hPSnR2=YoCo-pA8Mzb}7OXBO6#3_>`e;ZaoD zB%DCq+#a5}L|7pvs=&pir@;q){QVF$dCs}rQ(js&CBhR|tucsI^ZUaX36$NU0Pg?h zNi1Z{wpY=9`~U)e{2==8_Y@vjt@WQcdP#t?v1{yR6%-_7N(tJO{9*zupBM}}rz3c>`M(j%-Obg_faT2=*ZS%*GQ$eX<~AK0kLp!z8=doIH9NJQ?{)`s zW+?*DFMi1@ziWDn&s>+8POp#OKG&25Tpu;L?suA2FvZxEQ=V272ZcarNfsr2%GdHB zN*mr`v5!c|kAvfKW{(6`A%WbU<-9n)_snEP7O+c+F9YVb+?f)N7RAb--nmfb zUhfCxCfXjlM6^aq^Z0I%`?W&328{50E)uI>?kFG>w!W&&>z3V?exe=Dyu6ZZktnPo~(Nv6)j? zlZZ20QKW;?S_3uI9jdk+rz1K>;Lp!f95xhNTXwVDmT=XX&=+$pY&mF1#jCm-=A?s6 zF4bfQZT5s2%m@fV%`(oK>g>h5_1j!=)Fs$Nqu3>a?x^K)mk(?DOMv0#8ZumR&_z%4s zI?DSqgMc(!DKWz>yZ{Ai0?FbbYHzGPXZ*UQETFXws^*cM-t@s4V<#g)h{HPCU>8-O zX@ne+RZ9_yQDvty{4BI(4ODBU6l{~O9q8`?x|I%1i*ZoOg)r$RHjlyg5HrTZ-(3KZ zg&<;OfJQ=CAugcMY#paiWmk6JhV?5@+=umxWTEZSoMdA;+ou2VmNU+6gs3`{<}xgm z6)0bNctjD#$d3s}J83FIvSIbUx``t%xUR(RHz`4LJ~$5NpZQSXi57y(!D!gQ!y<%= z*W@%NSf8qH^-=84wy;|{Bpu;$jESgrkrqMP&|G%O%GqyOuPiPqL@t1>K&~x|jx;z~ zBi}g8tg{rwU~Ga?0g-*1RM{ah8(2)*3PjMYm@_e)$Fh&bR;K|uYK#%G31j<_RAjZ} z4&sx7<(Qow7Lq6x{$psz&{~mIp^lT7$Nu-Yt)2r;wL7z%cJ*yi@^GbbH*-!HPD(LG z8>ys!-ojdxXpYIGTX%F-$WRZ_kO*Dm~Liv3gqj^VX7tRn9xQ>npnHIrn&xU@aox|1)WR-clfqs6? zGIEtO&wwP)`({_c>>y~v*tN4HO5Ij%9?Q_6abp?# zp)d*ZwJupw0BW9$6O$md%yAu5$%xlj}q)jcbI!s?*V5!3C;6sLF$w{UVp_^ zB-Rii3m;KiLZ`}d{h|QErjvNEQ-%qFbleeqwM=jCUY~vNeML-r+&stEWldm31M}_? zU*)r(@ASC0r=BGn6^;pdgpT#S<4-phlDqmhlu0=Ysklg2aX1qDftnjI1$)g=j&zno z9rbhp`&{%~j^=gje(I3bo^|k)LbDeG^#rE~)6+HQIpH+L25rq#$3~jRpT2^}KJ5y} zgtvOK7Wismtk-0kL%-)@DFUXSZ$EJe7hggA?l0 zHTQ!bsM(19MH}@}!0{5t$nm{?&&-8o!%UFlmKSwB6p0-?&my4kdT?3ir?mcZ?L`5Fr<=+!JW-_yqf=#XjU>#_k4^M_VBm}{JRiZT1z?{0s_VfZx##TXuf*x4# z$ON%^Sz&l-HyV{E$K_PODK|?|7btVTwHh5l`s^!m56GjEK+kfiu{x)<`8gJJhlG;a zgT?ll=)g$|TGhSDL&0?HJu0^VwSq!_fUBnQz&2h?@xCS$Z#2!GCHm?QSBoARUI6|a zr!>XN!0Wvm@Qd0QUNvX#)#9USS^%>z)};n69TXw7m9O-k;U!i0L}UKL`y1L6uHkE62<*vw=ewOynCau?ETQAv?ge2o4s<$$_Tuqv9NyG zY`lvLhNRzix1v;3ug7B1z}kDlAtr5JE|!o8rTtV_;1_9c)ore;fADwJy}?Uu+LqBu zDtGTVaN(8BAD_J11iN@a=Obj=A%b*!uh)W(x5$0|l`x_6yZpZQ#Vq?9aFxTmifsdl z_8ZlC!RadLWmjM?rPa+l^{eup=R+^=_9XPjLezd$sG`3`33-{4!!*}z>|4}z@DBz`Sb;AjKS0vDxo_Cx%YISfR2bx&o(*`vYY zOFg}hoXeO5{VIg^Wm~dK4OzGNn@sKB+f2D#AZRIFUl^ibWdp&)|%N*G&~6K zWQckUOV0IOoPqH+lS)46* zfu?p%3j-^)t#a9LXRy{#_0=MP3o8E>_gp~pyH$)a{J7N$U5rXjQmjA>-zN`10vio=>HOB@6ktb?4c;{gq*xoY)Rg zXWjbWfJ5P-9?1Ge6*m`V0x7MMq4zuhu>IE9eGOZ{fWBA#uZDIZNI$W4P$Dfe8~ue< z6R1)t!TT37ow!gM4k<=#zIgQNd=#yFo2SxT9GXmITYJn|ui3mimJ#eDB*IVhOs;fa zU->{BPh6h<*9760=+j*3?Ub=Hg@P7+y>I{*F&SdJW+GZUg$r!DO6RVsO1P0HkQ>=- z3z7i#$ufDv&kTm*k?xJOwtuLP#z&-cmFuO>&tAX9HP{ z49&csH%>l}bH(Z>!;-i=))?C@Fh~oRsqh9t6FxFsPyNlUF0;W)^!L&Q?%0Nl?wkff z6gDz7JT9rT83&D3Q+m4a+}2^VeWSzQpPzMj&cD^uZOT9}-6RHkEg+{U66H&>RfRNyORw5 zN_>?B8Wo<->CZpU-OM-m6(y%!lR8)p^W)TOIKA-x4L=a$!R4W~qMHhUbFreCDpaxJ z*Zci%JrjjrK#ycmjwpq?0*P9VH%hxM)+dTyByzmDz^jxi~+UM8EHkahI*sK#1m4?tVIcvCl zp^9aCf=lugfc9AVjad&e-1Jl9q%vXVA09HE>o&3UmDd=2>3T;YjC4=}EDHrs$X%WNh-mDbzxj9^|w-wVnUG39wyir9k?Jk$I1Aa1m|Pv-=_ z?YcMGJi3Gw#3SG1D){|0DGp=S`$Y8_z)H7+%~ly&Vu#>qT4s4quD|7SU}ePdDG$Yk zGNJ_mlB(2vrZ;Yz2`odpbT4cEN!tOE4ZE|K?TX@;U_M(~mPJ3*qsMqOyxJUTJy#oO zg1hr~%F%`eZymbL?WZFqh_!~{1LM}xzp!uTfX{KwQJsi5j?S1t`{H zeJx&j&h(oyesyc7bsw>e=OyoqwX_rC8=q;Y>(OK@K4^<4J44TwI8_oP&%aMO{qUlO z_}I`x%Q~96QjdZ*&{5J{YD?GpPP(SR@7apgwM-Imf5HNBGI`OX<@zehuYbXsveQpu$Tev%NKj?x6c7Y08d<&d2& znl{(whpHJTQ9{r2K0ctLPkNLuj(z{^*YzMQkeJ6I5Mqe$?1;r6T^3G=s&HRGah1Wz zd)$C=+WiE=ULesuY=~3JyiC=LTsLhNr9w9SY+{YQh~v@aUoD0d{MJvyriP12`@u08 za*Mej^v9KO;B>c2#$v$|+#jCyc&_jP3+< zG7~|ALeT`Hh=Zak;9E5pU>E}V2`7m!ei1n>CO;s64>WUx14i|>9PQ6fb1x+gfw7Ou zk2v-xg@q`q~RJ z@WiI~&2^95tF!AO7VN>kp_aaaw3-eQl6@tnHLnid%mj*gxC2L z*sCN!0+_v^**8mCD*u9=DgtgD!?2pr>C=3dXW>lYqN)k2!n0bjvpS=;;E{Ykv}J#| zCm+z`!$i$nR-zXbm^qfZlbO1I)b0d>CGICm>2+=WHzU<2#_1K8=OJSmU##LMzVHM1 z&Y9Scx94AHGUJ8Wzs}7wpsi=h;%Gs;E&8j0ze{FkK;#KIt06B9Kmo+uzx~1A2o#^@ z9s2s=JGgUlR1ASskpZEqjD-;5Z$A0~-ALuP9$IxP2{RsGzJAz*`=#QAhdAANbJp-y zL0g68Sg1s8E|~Fjf5XT#Lm`Ep7#HhV_OLw@({NM_FG`O(%Zfv_&nI9e39LIweL&xV zO9=u(M;b{_`xk(JHvf}PAM_%xyOmVdUC6vjvW5y~bSanF=2~h2QSzE%m2s!#Y~2b+ z4Sq9(Un?^9TpYEOvF9-|x?xS2^}bv9zZ=FY4Ien_e4@qY&Y@a36nVR$DZ9iPZ+d=h zO+)BFtAEAohj_s%7X0Bl80b+hyNN&ncGKA0(Y!S>x2JhcY=Ot~zE&!C zKzJfu`_r{|#Bv`zAad-`2`I$*{Aw9PHXc_A%6C^q*?%cGRc|o}1x>Iw!NMR-c8W4cK zN_N=qbVVD-(L#ty=f?i@$9&T(TXZ*bS@J%>EBH2g@;ji(Dv`tx%aVXy6Ps9-Yre== zyC?))l(H}9&pAfqE977AAvYyT%$0hvUJ3)b-YVM!)4ll|^+lr{$8G9ngp&wsE4i&< z&;D@i;J|N?IL>UNPuEW4d|JN&CEg@EGrr0lb7J@Q4IUiYs1qMAvcLP~o#fc8knnGd z=hm+mNTLh+9+4Sm;K9?IELgom6-ghuL|(s(dDX~FW5c2ipXon2{6sgxm;$WkEIFjx zLzq|Hofkl#SFA+6f@yZ>S6vR=emT*gC}?~$m0By)lQAgd&Ee&Z@nRnWN6|7yR9p~l z3#vxM>b?+@M#P}zIF^X*G3DrS0$g7G)Fd~HU1xk0BC`Fim585V`Nky(`&`^@myA=h z2(+!RW9$roN4yD++0bWCpqWJM zKC9<-Yd*f2Y<*K(c+wU?qS}hFERS{Q(afb`n{!a^*I!jX@t^T2B`vUa>`9IB;piP7 z``cddoG#spOo4c>4N1OvrYqF^nkzLbzEFfxo7j|FF#M#RrWrGWlt)~Rua1;3nRr1r zR6`}dnO4)El%j;Z!tAS#nO+2uiaKxl>re^d{FB8FwjX)nOO3xE|C>f=G#;t^4|IV1 zjNM2%HV*PXsDr%S79OyFtzKG$|LAJ|BflV+zyJbqm(cl40HhQ9;KNdN^{B(p^>kwx zQqW19MWQX?=yW`f;@Ll@C2=MaC0UG9hEC<7ZD0ZK%F@p%S0&`i5wO13+lzPS-7F?! z?9KrU4p%-d+kD449>>$~&!vR^a66p*F#A~Q+Q<^|p-YUax(JDG!J#OKX2>`U=e*@% zo zp}UO_Tl2T7Uw;l%Wxt-7^jH8|jTnas)les#0z5AE#|dl_Tc9o33OJFj?=bV1d?ReX&{F7!92WY`*8016b+gjexZc` zq987rX@jwG^ja!z_~I%9HHu|AF&6x_&y2zp!80iA`PoV^jX@9wRJnR(SYTti9+{yR zW|VUG7DqR=XYjb-kPheo8i!oy5#FD^D^Ru`bb(wZyQ~E}(%Ksm^tYAaHb+QkgIhr)~~ueF}r#wJBuy`}2{ci~c* zxZ+gZHMxSl!Ww-&qQi}4pk5%aX{;^0A_&J`>V>jY&!p|-xL7XR9%$i^d_&A$rrA{@ z#WsHE$Sh5csWw$stvB~z6T|&ounB*}!eyVfo^sT+_cYu_i{RgXohm#O;*N2WgCEUs zxv1+=WtD&JdXz+s7%~q78qzy!ajk~Q!6dUidEmIP)eD#{c!yWQb>Z?xO&!FSW5uH6 zlIf5g5>?Sx!hloj0|p#zyNDoKooDTR<@z5yMMT}pOrqwSbe8toh`>5&IQnD=un*}0 zEHz*n#CAdJBdz4QHE(7E>!UgwI8v;*>UZo>Vs+5S7ARR?|6_6nP1rS7~2o3@Y2BcbO(xi8hq7*@@ zNG}2ckuC^-;QM^!y~$cLH|Oqsa!+O^nRUxP5ejn?uA@~SO{&F`T;jT7CxnRSye&na zC`?x|D$1j7Fr{g-W}<`o#H(ZX`tHxD(@%x|KJT?Q%KcU7(0++&%nYgXf;Vy(Xl3(t1^nAlA(6uF_UJL z18TQf?ASl#kOJgx@T7Z_0knOE7zN1mPEhx|R zOlou2Yg2p?1**3XPf zh;Y-Ezi_2GZTZ62Mmb*oER$-CvNqS-UeR+-9Y6ZMO6;Y>p0%|rs8bN}$3^(q6L$cI z{1^!DN3;GWIa>7QM0361_K?>;3eJ>#Sd3^;mA$M#DN3L;-y29y{y5oaNM`-W^juteTN-66In;gFY3{66|JWD)H~#(9pXVcd1f*lT0Wboi(!02 zz^_BZRZ3}e?~Y+x1}kA=x(4*N;T77>=iU}|z9e=23=e59;RU@}VE{BZn0&AotSmSW zm}qH}Q_fl|D9ZOBnIWsjTxt=IJUJPT@I;pHw|j-mTP`3sIt3)u9A*cMT{nCxmjSj2 zF}h(m(IG!upB>63nYDS}DouV0**-*lPYzYqzaP$kOrxluPUQPVBJgp>`4`8@P$ST_ zb3CPwzSQ^A4AH57P^PeCa|kFMWdP3I`~L`e{VpjGGsK9tM)s>P-0e|w8KqnF+hd?* z7|qa7HNcPVg%p^0Su{dDQxedvFLq+x=`zM|I@MlqDqW~u zQy%Hx?3uXUfNw**twwW~k3f;Cu@n~5d>1bje>k&iLCW4`7&1xxQ9)|uQ#?IPOeHMT ztY3I+;FSp4QW8awmE$v4%C-DTgyV5^DiPPSE$eqi$9Ar~=^^7* zcq{1C**O}R^O+kK+$U;~-7t75!wQ=*c3~CDy_8<3mmZR9P9(vDZprWRMZT0a-uq-MDr$`UYi&O5 zIeK781o=F4_v}=)>q1Q-{84u?6J5n{u5)!wR7o3DXeWf;^!B(P(_8~t!;@=aWWae%n3@M%3(+nfE|G!{-Kevdw7 zRy(@dK`53aEa5fV&!BP*Z568s1}Zyag?D}Pv52}G{L-{-H`F4?G&Hq8&A&T3DCb=I z=tcc%Vd*86-+B$@jFw0vo^R`Cyd1w<&0RWTd9{k@d56|URuZ)$iWw>8tKR80^+Lah zxZaVBO#-%uGGyKIllgayP~xBNPOZgtiNpYt?^EzNFtln%Wb%1lek>tQ#az`GuXB## z^x2q=Za}<$G=7t?)pO|XUrs)`>WU~4j27H9?cFN=sEA0pYHbtwg&Ln1PyASoZYD3N zm~Os8FE;Jigb_~LsX{7Y(ed+PyJj@x(ufe1#r(x4sOY`MaA6-+Ze{S8pW#?et=&v$ zJ{=xY3|qE}-w%$bh1e5Hr-rCJVs9rD)ea@@xTTu32g8ThGf7t|!vv@`=EI51>JW5stuKXtPIzYX zsq{}8_P1tf0b}HNc=?QYc>E{Np!K_q=*Ol3HVku&M|vfxBb0!R3^xr~wQZt8t+eQ9ZT_bumwcP4Js-*iN_D+&f6mhPO-b9Jd76x6Nl*RSp8 zmam0Ku5{ldK^zV5#qHEj)J*Jr2aW?fLQxfv8<$=*fbu(SN};Mn=s zfSef9$aPm}k)apUwbvOCQpOKW_|PKcc_vcV`y0xKwU~=hyOX~T0tk^4%+0C(!IEpW zkxrgeQyz-Sjp=Wr*3Ys}bj*?l+?I>*!|u|czp^z4`71W3N2&XZJzTTw~8+q-65tn&7Pjf_bu4o`rY$tK2zMlPP`=KSJYT4Cwd z<2-`hPV|6|V9V*2WS)%F7C3zOu*l6on3<6jd#k_dzhiJ^W$b+MXLT!sOPA3ebk3*#kJN@CAP z-hi_?pKRApE1R2KpB}ZfZ!ZHS9nX&i=&?$>NGdqSfEkz^EM~sXB*{GO8{09ni0dx* z^lQ@#&$Dre zD7m;y4S^mK8RJ#Ww9~@`fBTX;=u=2aupOZq=c#a!J{6p!(k~oRnvj=v4lQuLJ;On} z5?1|zHvaSX?WRL+Qf^O3K1#rt=Yf$iOl$^epPA#9MevEWOacY7bIl6Dl#xNrJH4$_ zg-2B{@s(DEe|Il>jCFygCt%HQ9aY{8`V1d;&J$jzo9;W}L)csUhh83mj$f!*J&1o~ z74;-W%p=QXG&?FQsjN95|tBwH@%33;F=AIU}-zN0XyLCWyb+gfx$QK)eoesp& z@x`-fCdiY;0(cdVU<2sBoBq7KPk_E@gw+hb&+=Et{Gr;Q4DXgL zo33`Hyr%ACg18ZH$>yhnPwg*sy7FDLHtUQngkI$8n$^L-v>69OuitRFp3{xOg&^9dudj@iV zIM97p6tI0R#vu8e$Z{hTf=+!<<2^QFe``!+yz}dqA5wY{tR-VZ%i7h0FiDO0`9OQw>GJK?W}k`C*%T@4*t9jzt?$7ce5LN|%XNRfRs8n&ZKSV9 zz2{T&yPcWqLGR_z&w%=>A%jtbqen1G{*aHBrJnSH)XD+a~ZXQq!{ zGh&8yt6F%dQGEI}`8tm_Mb*ohMY=!!Aw{JJdefIV*;XOs>H-ahC&NwL zn20q>*!li2gW8T=6QyfZv+kt$2k5J~oGU8NE{%h0d4-YSu6LR6?71+^SjHnQICm%~y^HU?Te=@w z9?$+fJ6-*Bx}GB0PI_)onjuCyrPX5Py*z7cd4#vp^L+ICmdd!pJDv>PF|?g6&TXYw z)~O{VbDIyoxE(3Nw(SX$$7&qx&LGQv8ZMZno7=E0#p|5RA+b3ZZy8~(X?N~Id0n(( z$R2%~ej?(1uUw>%vlGwU07Ic7i&&(*FxQ~YJxl&pOMxs;8WtG^v%+htRuNK{bi@XE ziM~-LwcDdhue`?l9!I<5ZwREevTHauPsIsXZ4o7!RobzbdZstM;uN=5iggo|EKD|D zREINa3(Sb#zUn1e*at^Wy-gF^Q06M(iFPV>yr)=Pp@2zgmj7uT#H>sGuuIdxO~(Xj zxEh&Ql3^|qS%9&R5O|jAY^7}{6jsVk`+Uq+b|V1Y7Iq^$;JjCC!+`97u6ph_3M;H>UBVSUKm2XyNGn8m^vQ1ZPjGX51?b_B<# zoBA|PaO}#S)N8sFzseL@I@yh(chd!Y#bGS%6K-`JT<Pjb31^_dz`X294HV#6Q2LGW zU*H2v84K6I{n3ImxOZMWE+ZJBz~tgFiv1fB++$gxK=e1oQ)%rMcU>BQ&)+OhnJRs4 z(gEcq_~s{#P9!g>mhlQyc69Jl@XDj;0*6!aw7BnYhpB`76uJg2Fjd^-obbPI4;+>Gzgq(3!ph*)3r@EFvt@@3dHp8Cz#!LtzzI}upBNqn()=d?97WMT zK8^$6_WMmD2ZDll{zE(w4g|Yf{sK>~6b`XG=ns)Jl=(kI9LX{q;!E^zA{iA<$lxMQ zIKjI|e^&yYDG<`X&rbtbRSuj~?QQac@xr#?1hr334A(MP{@{W@bLcZ5V@vSV6^FCb z6Q?Pz7+@U-db|MR&-x1-C}|#`vMqheE6x;n9O%cFpjXy^1=qR_Czz=}Y4G7T9T)-a z4xHd&(}{t-iwZE?VL#;@XPgMfxlhY)3jLp;yFgDhoxu8z+$j$@{bV>Ex_SVwA+i6Q jFr01y9D;B^02>p87$Atj-#$D%S?~p?$HUVbKYsT=?Xdg@ diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 58e3982e3..d54311bfa 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,7 +1,7 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionSha256Sum=518a863631feb7452b8f1b3dc2aaee5f388355cc3421bbd0275fbeadd77e84b2 -distributionUrl=https\://services.gradle.org/distributions/gradle-7.6.1-all.zip +distributionSha256Sum=f2b9ed0faf8472cbe469255ae6c86eddb77076c75191741b4a462f33128dd419 +distributionUrl=https\://services.gradle.org/distributions/gradle-8.4-all.zip networkTimeout=10000 zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/gradlew b/gradlew index 65dcd68d6..1aa94a426 100755 --- a/gradlew +++ b/gradlew @@ -83,10 +83,8 @@ done # This is normally unused # shellcheck disable=SC2034 APP_BASE_NAME=${0##*/} -APP_HOME=$( cd "${APP_HOME:-./}" && pwd -P ) || exit - -# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. -DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' +# Discard cd standard output in case $CDPATH is set (https://github.com/gradle/gradle/issues/25036) +APP_HOME=$( cd "${APP_HOME:-./}" > /dev/null && pwd -P ) || exit # Use the maximum available, or set MAX_FD != -1 to use that value. MAX_FD=maximum @@ -133,10 +131,13 @@ location of your Java installation." fi else JAVACMD=java - which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + if ! command -v java >/dev/null 2>&1 + then + die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. Please set the JAVA_HOME variable in your environment to match the location of your Java installation." + fi fi # Increase the maximum file descriptors if we can. @@ -144,7 +145,7 @@ if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then case $MAX_FD in #( max*) # In POSIX sh, ulimit -H is undefined. That's why the result is checked to see if it worked. - # shellcheck disable=SC3045 + # shellcheck disable=SC2039,SC3045 MAX_FD=$( ulimit -H -n ) || warn "Could not query maximum file descriptor limit" esac @@ -152,7 +153,7 @@ if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then '' | soft) :;; #( *) # In POSIX sh, ulimit -n is undefined. That's why the result is checked to see if it worked. - # shellcheck disable=SC3045 + # shellcheck disable=SC2039,SC3045 ulimit -n "$MAX_FD" || warn "Could not set maximum file descriptor limit to $MAX_FD" esac @@ -197,11 +198,15 @@ if "$cygwin" || "$msys" ; then done fi -# Collect all arguments for the java command; -# * $DEFAULT_JVM_OPTS, $JAVA_OPTS, and $GRADLE_OPTS can contain fragments of -# shell script including quotes and variable substitutions, so put them in -# double quotes to make sure that they get re-expanded; and -# * put everything else in single quotes, so that it's not re-expanded. + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' + +# Collect all arguments for the java command: +# * DEFAULT_JVM_OPTS, JAVA_OPTS, JAVA_OPTS, and optsEnvironmentVar are not allowed to contain shell fragments, +# and any embedded shellness will be escaped. +# * For example: A user cannot expect ${Hostname} to be expanded, as it is an environment variable and will be +# treated as '${Hostname}' itself on the command line. set -- \ "-Dorg.gradle.appname=$APP_BASE_NAME" \ diff --git a/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java b/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java index 1c754a5c7..678f490fa 100644 --- a/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java +++ b/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java @@ -8,6 +8,7 @@ import com.google.common.base.Charsets; import com.google.common.io.Resources; import com.google.common.primitives.Floats; +import lombok.extern.log4j.Log4j2; import org.apache.commons.lang.StringUtils; import org.apache.http.util.EntityUtils; import org.opensearch.core.common.bytes.BytesReference; @@ -52,7 +53,6 @@ import java.net.URL; import java.nio.file.Files; import java.nio.file.Path; -import java.nio.file.Paths; import java.util.ArrayList; import java.util.Arrays; import java.util.Base64; @@ -108,6 +108,7 @@ /** * Base class for integration tests for KNN plugin. Contains several methods for testing KNN ES functionality. */ +@Log4j2 public class KNNRestTestCase extends ODFERestTestCase { public static final String INDEX_NAME = "test_index"; public static final String FIELD_NAME = "test_field"; @@ -135,9 +136,10 @@ public static void dumpCoverage() throws IOException, MalformedObjectNameExcepti false ); - Path path = Paths.get(jacocoBuildPath + "/integTest.exec"); + Path path = Path.of(jacocoBuildPath, "integTest.exec"); Files.write(path, proxy.getExecutionData(false)); } catch (Exception ex) { + log.error("Failed to dump coverage: ", ex); throw new RuntimeException("Failed to dump coverage: " + ex); } } From 785e477832c46d661bd570eca5cd7b158916114f Mon Sep 17 00:00:00 2001 From: Ryan Bogan Date: Tue, 7 Nov 2023 09:02:12 -0800 Subject: [PATCH 161/416] Increment version to 2.12.0-SNAPSHOT and change imports to align with core lucene upgrade (#1296) * Change imports to align with core lucene upgrade Signed-off-by: Ryan Bogan * Increment version to 2.12.0-SNAPSHOT Signed-off-by: Ryan Bogan * Fix spotless Signed-off-by: Ryan Bogan --------- Signed-off-by: Ryan Bogan --- .../workflows/backwards_compatibility_tests_workflow.yml | 8 ++++---- build.gradle | 2 +- .../opensearch/knn/index/mapper/LuceneFieldMapper.java | 4 ++-- .../java/org/opensearch/knn/index/util/KNNEngine.java | 4 ++-- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/backwards_compatibility_tests_workflow.yml b/.github/workflows/backwards_compatibility_tests_workflow.yml index b7ce283d1..ad90a79f3 100644 --- a/.github/workflows/backwards_compatibility_tests_workflow.yml +++ b/.github/workflows/backwards_compatibility_tests_workflow.yml @@ -14,8 +14,8 @@ jobs: strategy: matrix: java: [ 11, 17 ] - bwc_version : [ "1.1.0", "1.2.4", "1.3.8", "2.0.1", "2.1.0", "2.2.1", "2.3.0", "2.4.1", "2.5.0", "2.6.0", "2.7.0", "2.8.0", "2.9.0", "2.10.0" ] - opensearch_version : [ "2.11.0-SNAPSHOT" ] + bwc_version : [ "1.1.0", "1.2.4", "1.3.8", "2.0.1", "2.1.0", "2.2.1", "2.3.0", "2.4.1", "2.5.0", "2.6.0", "2.7.0", "2.8.0", "2.9.0", "2.10.0", "2.11.0" ] + opensearch_version : [ "2.12.0-SNAPSHOT" ] name: k-NN Restart-Upgrade BWC Tests runs-on: ubuntu-latest @@ -46,8 +46,8 @@ jobs: strategy: matrix: java: [ 11, 17 ] - bwc_version: [ "1.3.8", "2.0.1", "2.1.0", "2.2.1", "2.3.0", "2.4.1", "2.5.0", "2.6.0", "2.7.0", "2.8.0", "2.9.0", "2.10.0"] - opensearch_version: [ "2.11.0-SNAPSHOT" ] + bwc_version: [ "1.3.8", "2.0.1", "2.1.0", "2.2.1", "2.3.0", "2.4.1", "2.5.0", "2.6.0", "2.7.0", "2.8.0", "2.9.0", "2.10.0", "2.11.0"] + opensearch_version: [ "2.12.0-SNAPSHOT" ] name: k-NN Rolling-Upgrade BWC Tests runs-on: ubuntu-latest diff --git a/build.gradle b/build.gradle index 8c8db8340..f3dcff3e3 100644 --- a/build.gradle +++ b/build.gradle @@ -10,7 +10,7 @@ buildscript { ext { // build.version_qualifier parameter applies to knn plugin artifacts only. OpenSearch version must be set // explicitly as 'opensearch.version' property, for instance opensearch.version=2.0.0-rc1-SNAPSHOT - opensearch_version = System.getProperty("opensearch.version", "2.11.0-SNAPSHOT") + opensearch_version = System.getProperty("opensearch.version", "2.12.0-SNAPSHOT") version_qualifier = System.getProperty("build.version_qualifier", "") opensearch_group = "org.opensearch" } diff --git a/src/main/java/org/opensearch/knn/index/mapper/LuceneFieldMapper.java b/src/main/java/org/opensearch/knn/index/mapper/LuceneFieldMapper.java index 94e42ee7c..b28b93028 100644 --- a/src/main/java/org/opensearch/knn/index/mapper/LuceneFieldMapper.java +++ b/src/main/java/org/opensearch/knn/index/mapper/LuceneFieldMapper.java @@ -23,7 +23,7 @@ import java.util.Locale; import java.util.Optional; -import static org.apache.lucene.index.VectorValues.MAX_DIMENSIONS; +import org.apache.lucene.codecs.KnnVectorsFormat; import static org.opensearch.knn.common.KNNConstants.VECTOR_DATA_TYPE_FIELD; import static org.opensearch.knn.index.mapper.KNNVectorFieldMapperUtil.addStoredFieldForVectorField; import static org.opensearch.knn.index.mapper.KNNVectorFieldMapperUtil.buildDocValuesFieldType; @@ -33,7 +33,7 @@ */ public class LuceneFieldMapper extends KNNVectorFieldMapper { - private static final int LUCENE_MAX_DIMENSION = MAX_DIMENSIONS; + private static final int LUCENE_MAX_DIMENSION = KnnVectorsFormat.DEFAULT_MAX_DIMENSIONS; /** FieldType used for initializing VectorField, which is used for creating binary doc values. **/ private final FieldType vectorFieldType; diff --git a/src/main/java/org/opensearch/knn/index/util/KNNEngine.java b/src/main/java/org/opensearch/knn/index/util/KNNEngine.java index 776ea5366..197bb87f3 100644 --- a/src/main/java/org/opensearch/knn/index/util/KNNEngine.java +++ b/src/main/java/org/opensearch/knn/index/util/KNNEngine.java @@ -6,7 +6,7 @@ package org.opensearch.knn.index.util; import com.google.common.collect.ImmutableSet; -import org.apache.lucene.index.VectorValues; +import org.apache.lucene.codecs.KnnVectorsFormat; import org.opensearch.common.ValidationException; import org.opensearch.knn.index.KNNMethod; import org.opensearch.knn.index.KNNMethodContext; @@ -40,7 +40,7 @@ public enum KNNEngine implements KNNLibrary { KNNEngine.FAISS, 16_000, KNNEngine.LUCENE, - VectorValues.MAX_DIMENSIONS + KnnVectorsFormat.DEFAULT_MAX_DIMENSIONS ); /** From f32f25ab5f706a5c6d4168ab66ec40a839d39c45 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Tue, 7 Nov 2023 10:14:46 -0800 Subject: [PATCH 162/416] Add parent join support for lucene knn (#1182) (#1230) Call DiversifyingChildren[Byte|Float]KnnVectorQuery for nested field so that k number of parent document can be returned in search result Signed-off-by: Heemin Kim (cherry picked from commit a49692671389a3e6e2f35f227bff5ec627a46f1b) Co-authored-by: Heemin Kim --- CHANGELOG.md | 1 + .../opensearch/knn/common/KNNConstants.java | 8 + .../knn/index/query/KNNQueryFactory.java | 65 +++--- .../opensearch/knn/index/NestedSearchIT.java | 202 ++++++++++++++++++ .../knn/index/query/KNNQueryFactoryTests.java | 40 +++- 5 files changed, 286 insertions(+), 30 deletions(-) create mode 100644 src/test/java/org/opensearch/knn/index/NestedSearchIT.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 59dd618d8..feb1411e3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ## [Unreleased 2.x](https://github.com/opensearch-project/k-NN/compare/2.11...2.x) ### Features +* Add parent join support for lucene knn [#1182](https://github.com/opensearch-project/k-NN/pull/1182) ### Enhancements ### Bug Fixes ### Infrastructure diff --git a/src/main/java/org/opensearch/knn/common/KNNConstants.java b/src/main/java/org/opensearch/knn/common/KNNConstants.java index 59732cca0..184067c35 100644 --- a/src/main/java/org/opensearch/knn/common/KNNConstants.java +++ b/src/main/java/org/opensearch/knn/common/KNNConstants.java @@ -15,6 +15,14 @@ public class KNNConstants { public static final String NAME = "name"; public static final String PARAMETERS = "parameters"; public static final String METHOD_HNSW = "hnsw"; + public static final String TYPE = "type"; + public static final String TYPE_NESTED = "nested"; + public static final String PATH = "path"; + public static final String QUERY = "query"; + public static final String KNN = "knn"; + public static final String VECTOR = "vector"; + public static final String K = "k"; + public static final String TYPE_KNN_VECTOR = "knn_vector"; public static final String METHOD_PARAMETER_EF_SEARCH = "ef_search"; public static final String METHOD_PARAMETER_EF_CONSTRUCTION = "ef_construction"; public static final String METHOD_PARAMETER_M = "m"; diff --git a/src/main/java/org/opensearch/knn/index/query/KNNQueryFactory.java b/src/main/java/org/opensearch/knn/index/query/KNNQueryFactory.java index b05098f28..c073450af 100644 --- a/src/main/java/org/opensearch/knn/index/query/KNNQueryFactory.java +++ b/src/main/java/org/opensearch/knn/index/query/KNNQueryFactory.java @@ -14,6 +14,9 @@ import org.apache.lucene.search.KnnByteVectorQuery; import org.apache.lucene.search.KnnFloatVectorQuery; import org.apache.lucene.search.Query; +import org.apache.lucene.search.join.BitSetProducer; +import org.apache.lucene.search.join.DiversifyingChildrenByteKnnVectorQuery; +import org.apache.lucene.search.join.DiversifyingChildrenFloatKnnVectorQuery; import org.opensearch.index.query.QueryBuilder; import org.opensearch.index.query.QueryShardContext; import org.opensearch.knn.index.VectorDataType; @@ -86,10 +89,12 @@ public static Query create(CreateQueryRequest createQueryRequest) { return new KNNQuery(fieldName, vector, k, indexName); } + log.debug(String.format("Creating Lucene k-NN query for index: %s \"\", field: %s \"\", k: %d", indexName, fieldName, k)); + BitSetProducer parentFilter = createQueryRequest.context == null ? null : createQueryRequest.context.getParentFilter(); if (VectorDataType.BYTE == vectorDataType) { - return getKnnByteVectorQuery(indexName, fieldName, byteVector, k, filterQuery); + return getKnnByteVectorQuery(fieldName, byteVector, k, filterQuery, parentFilter); } else if (VectorDataType.FLOAT == vectorDataType) { - return getKnnFloatVectorQuery(indexName, fieldName, vector, k, filterQuery); + return getKnnFloatVectorQuery(fieldName, vector, k, filterQuery, parentFilter); } else { throw new IllegalArgumentException( String.format( @@ -102,38 +107,40 @@ public static Query create(CreateQueryRequest createQueryRequest) { } } - private static Query getKnnByteVectorQuery(String indexName, String fieldName, byte[] byteVector, int k, Query filterQuery) { - if (filterQuery != null) { - log.debug( - String.format( - Locale.ROOT, - "Creating Lucene k-NN query with filters for index: %s \"\", field: %s \"\", k: %d", - indexName, - fieldName, - k - ) - ); + /** + * If parentFilter is not null, it is a nested query. Therefore, we return {@link DiversifyingChildrenByteKnnVectorQuery} + * which will dedupe search result per parent so that we can get k parent results at the end. + */ + private static Query getKnnByteVectorQuery( + final String fieldName, + final byte[] byteVector, + final int k, + final Query filterQuery, + final BitSetProducer parentFilter + ) { + if (parentFilter == null) { return new KnnByteVectorQuery(fieldName, byteVector, k, filterQuery); + } else { + return new DiversifyingChildrenByteKnnVectorQuery(fieldName, byteVector, filterQuery, k, parentFilter); } - log.debug(String.format("Creating Lucene k-NN query for index: %s \"\", field: %s \"\", k: %d", indexName, fieldName, k)); - return new KnnByteVectorQuery(fieldName, byteVector, k); } - private static Query getKnnFloatVectorQuery(String indexName, String fieldName, float[] floatVector, int k, Query filterQuery) { - if (filterQuery != null) { - log.debug( - String.format( - Locale.ROOT, - "Creating Lucene k-NN query with filters for index: %s \"\", field: %s \"\", k: %d", - indexName, - fieldName, - k - ) - ); + /** + * If parentFilter is not null, it is a nested query. Therefore, we return {@link DiversifyingChildrenFloatKnnVectorQuery} + * which will dedupe search result per parent so that we can get k parent results at the end. + */ + private static Query getKnnFloatVectorQuery( + final String fieldName, + final float[] floatVector, + final int k, + final Query filterQuery, + final BitSetProducer parentFilter + ) { + if (parentFilter == null) { return new KnnFloatVectorQuery(fieldName, floatVector, k, filterQuery); + } else { + return new DiversifyingChildrenFloatKnnVectorQuery(fieldName, floatVector, filterQuery, k, parentFilter); } - log.debug(String.format("Creating Lucene k-NN query for index: %s \"\", field: %s \"\", k: %d", indexName, fieldName, k)); - return new KnnFloatVectorQuery(fieldName, floatVector, k); } private static Query getFilterQuery(CreateQueryRequest createQueryRequest) { @@ -181,6 +188,8 @@ static class CreateQueryRequest { @Getter private int k; // can be null in cases filter not passed with the knn query + @Getter + private BitSetProducer parentFilter; private QueryBuilder filter; // can be null in cases filter not passed with the knn query private QueryShardContext context; diff --git a/src/test/java/org/opensearch/knn/index/NestedSearchIT.java b/src/test/java/org/opensearch/knn/index/NestedSearchIT.java new file mode 100644 index 000000000..dce006b50 --- /dev/null +++ b/src/test/java/org/opensearch/knn/index/NestedSearchIT.java @@ -0,0 +1,202 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index; + +import lombok.SneakyThrows; +import org.apache.http.util.EntityUtils; +import org.junit.After; +import org.opensearch.client.Request; +import org.opensearch.client.Response; +import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.core.xcontent.MediaTypeRegistry; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.knn.KNNRestTestCase; +import org.opensearch.knn.index.util.KNNEngine; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +import static org.opensearch.knn.common.KNNConstants.DIMENSION; +import static org.opensearch.knn.common.KNNConstants.K; +import static org.opensearch.knn.common.KNNConstants.KNN; +import static org.opensearch.knn.common.KNNConstants.KNN_ENGINE; +import static org.opensearch.knn.common.KNNConstants.KNN_METHOD; +import static org.opensearch.knn.common.KNNConstants.METHOD_HNSW; +import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_EF_CONSTRUCTION; +import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_M; +import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_SPACE_TYPE; +import static org.opensearch.knn.common.KNNConstants.NAME; +import static org.opensearch.knn.common.KNNConstants.PARAMETERS; +import static org.opensearch.knn.common.KNNConstants.PATH; +import static org.opensearch.knn.common.KNNConstants.QUERY; +import static org.opensearch.knn.common.KNNConstants.TYPE; +import static org.opensearch.knn.common.KNNConstants.TYPE_KNN_VECTOR; +import static org.opensearch.knn.common.KNNConstants.TYPE_NESTED; +import static org.opensearch.knn.common.KNNConstants.VECTOR; + +public class NestedSearchIT extends KNNRestTestCase { + private static final String INDEX_NAME = "test-index-nested-search"; + private static final String FIELD_NAME_NESTED = "test-nested"; + private static final String FIELD_NAME_VECTOR = "test-vector"; + private static final String PROPERTIES_FIELD = "properties"; + private static final int EF_CONSTRUCTION = 128; + private static final int M = 16; + private static final SpaceType SPACE_TYPE = SpaceType.L2; + + @After + @SneakyThrows + public final void cleanUp() { + deleteKNNIndex(INDEX_NAME); + } + + @SneakyThrows + public void testNestedSearch_whenKIsTwo_thenReturnTwoResults() { + createKnnIndex(2, KNNEngine.LUCENE.getName()); + + String doc1 = NestedKnnDocBuilder.create(FIELD_NAME_NESTED) + .add(FIELD_NAME_VECTOR, new Float[] { 1f, 1f }, new Float[] { 1f, 1f }) + .build(); + addNestedKnnDoc(INDEX_NAME, "1", doc1); + + String doc2 = NestedKnnDocBuilder.create(FIELD_NAME_NESTED) + .add(FIELD_NAME_VECTOR, new Float[] { 2f, 2f }, new Float[] { 2f, 2f }) + .build(); + addNestedKnnDoc(INDEX_NAME, "2", doc2); + + Float[] queryVector = { 1f, 1f }; + Response response = queryNestedField(INDEX_NAME, 2, queryVector); + + List hits = (List) ((Map) createParser( + MediaTypeRegistry.getDefaultMediaType().xContent(), + EntityUtils.toString(response.getEntity()) + ).map().get("hits")).get("hits"); + assertEquals(2, hits.size()); + } + + /** + * { + * "properties": { + * "test-nested": { + * "type": "nested", + * "properties": { + * "test-vector": { + * "type": "knn_vector", + * "dimension": 3, + * "method": { + * "name": "hnsw", + * "space_type": "l2", + * "engine": "lucene", + * "parameters": { + * "ef_construction": 128, + * "m": 24 + * } + * } + * } + * } + * } + * } + * } + */ + private void createKnnIndex(final int dimension, final String engine) throws Exception { + XContentBuilder builder = XContentFactory.jsonBuilder() + .startObject() + .startObject(PROPERTIES_FIELD) + .startObject(FIELD_NAME_NESTED) + .field(TYPE, TYPE_NESTED) + .startObject(PROPERTIES_FIELD) + .startObject(FIELD_NAME_VECTOR) + .field(TYPE, TYPE_KNN_VECTOR) + .field(DIMENSION, dimension) + .startObject(KNN_METHOD) + .field(NAME, METHOD_HNSW) + .field(METHOD_PARAMETER_SPACE_TYPE, SPACE_TYPE) + .field(KNN_ENGINE, engine) + .startObject(PARAMETERS) + .field(METHOD_PARAMETER_M, M) + .field(METHOD_PARAMETER_EF_CONSTRUCTION, EF_CONSTRUCTION) + .endObject() + .endObject() + .endObject() + .endObject() + .endObject() + .endObject() + .endObject(); + + String mapping = builder.toString(); + createKnnIndex(INDEX_NAME, mapping); + } + + @SneakyThrows + private void ingestTestData() { + String doc1 = NestedKnnDocBuilder.create(FIELD_NAME_NESTED) + .add(FIELD_NAME_VECTOR, new Float[] { 1f, 1f }, new Float[] { 1f, 1f }) + .build(); + addNestedKnnDoc(INDEX_NAME, "1", doc1); + + String doc2 = NestedKnnDocBuilder.create(FIELD_NAME_NESTED) + .add(FIELD_NAME_VECTOR, new Float[] { 2f, 2f }, new Float[] { 2f, 2f }) + .build(); + addNestedKnnDoc(INDEX_NAME, "2", doc2); + } + + private void addNestedKnnDoc(final String index, final String docId, final String document) throws IOException { + Request request = new Request("POST", "/" + index + "/_doc/" + docId + "?refresh=true"); + + request.setJsonEntity(document); + client().performRequest(request); + + request = new Request("POST", "/" + index + "/_refresh"); + Response response = client().performRequest(request); + assertEquals(request.getEndpoint() + ": failed", RestStatus.OK, RestStatus.fromCode(response.getStatusLine().getStatusCode())); + } + + private Response queryNestedField(final String index, final int k, final Object[] vector) throws IOException { + XContentBuilder builder = XContentFactory.jsonBuilder().startObject().startObject(QUERY); + builder.startObject(TYPE_NESTED); + builder.field(PATH, FIELD_NAME_NESTED); + builder.startObject(QUERY).startObject(KNN).startObject(FIELD_NAME_NESTED + "." + FIELD_NAME_VECTOR); + builder.field(VECTOR, vector); + builder.field(K, k); + builder.endObject().endObject().endObject().endObject().endObject().endObject(); + + Request request = new Request("POST", "/" + index + "/_search"); + request.setJsonEntity(builder.toString()); + + Response response = client().performRequest(request); + assertEquals(request.getEndpoint() + ": failed", RestStatus.OK, RestStatus.fromCode(response.getStatusLine().getStatusCode())); + + return response; + } + + private static class NestedKnnDocBuilder { + private XContentBuilder builder; + + public NestedKnnDocBuilder(final String fieldName) throws IOException { + builder = XContentFactory.jsonBuilder().startObject().startArray(fieldName); + } + + public static NestedKnnDocBuilder create(final String fieldName) throws IOException { + return new NestedKnnDocBuilder(fieldName); + } + + public NestedKnnDocBuilder add(final String fieldName, final Object[]... vectors) throws IOException { + for (Object[] vector : vectors) { + builder.startObject(); + builder.field(fieldName, vector); + builder.endObject(); + } + return this; + } + + public String build() throws IOException { + builder.endArray().endObject(); + return builder.toString(); + } + + } +} diff --git a/src/test/java/org/opensearch/knn/index/query/KNNQueryFactoryTests.java b/src/test/java/org/opensearch/knn/index/query/KNNQueryFactoryTests.java index 4dccfd087..a6b915a85 100644 --- a/src/test/java/org/opensearch/knn/index/query/KNNQueryFactoryTests.java +++ b/src/test/java/org/opensearch/knn/index/query/KNNQueryFactoryTests.java @@ -9,12 +9,16 @@ import org.apache.lucene.search.KnnFloatVectorQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.TermQuery; +import org.apache.lucene.search.join.BitSetProducer; +import org.apache.lucene.search.join.DiversifyingChildrenByteKnnVectorQuery; +import org.apache.lucene.search.join.DiversifyingChildrenFloatKnnVectorQuery; import org.mockito.Mockito; import org.opensearch.index.mapper.MappedFieldType; import org.opensearch.index.query.QueryBuilder; import org.opensearch.index.query.QueryShardContext; import org.opensearch.index.query.TermQueryBuilder; import org.opensearch.knn.KNNTestCase; +import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.util.KNNEngine; import java.util.Arrays; @@ -33,6 +37,7 @@ public class KNNQueryFactoryTests extends KNNTestCase { private static final Query FILTER_QUERY = new TermQuery(new Term(FILTER_FILED_NAME, FILTER_FILED_VALUE)); private final int testQueryDimension = 17; private final float[] testQueryVector = new float[testQueryDimension]; + private final byte[] testByteQueryVector = new byte[testQueryDimension]; private final String testIndexName = "test-index"; private final String testFieldName = "test-field"; private final int testK = 10; @@ -69,7 +74,7 @@ public void testCreateLuceneDefaultQuery() { testK, DEFAULT_VECTOR_DATA_TYPE_FIELD ); - assertTrue(query.getClass().isAssignableFrom(KnnFloatVectorQuery.class)); + assertEquals(KnnFloatVectorQuery.class, query.getClass()); } } @@ -92,7 +97,7 @@ public void testCreateLuceneQueryWithFilter() { .filter(FILTER_QUERY_BUILDER) .build(); Query query = KNNQueryFactory.create(createQueryRequest); - assertTrue(query.getClass().isAssignableFrom(KnnFloatVectorQuery.class)); + assertEquals(KnnFloatVectorQuery.class, query.getClass()); } } @@ -120,4 +125,35 @@ public void testCreateFaissQueryWithFilter_withValidValues_thenSuccess() { assertEquals(testK, ((KNNQuery) query).getK()); assertEquals(FILTER_QUERY, ((KNNQuery) query).getFilterQuery()); } + + public void testCreate_whenLuceneWithParentFilter_thenReturnDiversifyingQuery() { + validateDiversifyingQueryWithParentFilter(VectorDataType.BYTE, DiversifyingChildrenByteKnnVectorQuery.class); + validateDiversifyingQueryWithParentFilter(VectorDataType.FLOAT, DiversifyingChildrenFloatKnnVectorQuery.class); + } + + private void validateDiversifyingQueryWithParentFilter(final VectorDataType type, final Class expectedQueryClass) { + List luceneDefaultQueryEngineList = Arrays.stream(KNNEngine.values()) + .filter(knnEngine -> !KNNEngine.getEnginesThatCreateCustomSegmentFiles().contains(knnEngine)) + .collect(Collectors.toList()); + for (KNNEngine knnEngine : luceneDefaultQueryEngineList) { + QueryShardContext mockQueryShardContext = mock(QueryShardContext.class); + MappedFieldType testMapper = mock(MappedFieldType.class); + when(mockQueryShardContext.fieldMapper(any())).thenReturn(testMapper); + BitSetProducer parentFilter = mock(BitSetProducer.class); + when(mockQueryShardContext.getParentFilter()).thenReturn(parentFilter); + final KNNQueryFactory.CreateQueryRequest createQueryRequest = KNNQueryFactory.CreateQueryRequest.builder() + .knnEngine(knnEngine) + .indexName(testIndexName) + .fieldName(testFieldName) + .vector(testQueryVector) + .byteVector(testByteQueryVector) + .vectorDataType(type) + .k(testK) + .context(mockQueryShardContext) + .filter(FILTER_QUERY_BUILDER) + .build(); + Query query = KNNQueryFactory.create(createQueryRequest); + assertEquals(expectedQueryClass, query.getClass()); + } + } } From 7ccb10cf5339701217c4019932bd26bf304a93aa Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Tue, 7 Nov 2023 12:45:16 -0800 Subject: [PATCH 163/416] [Backport 2.x] Update CI to use jdk-21 (#1298) This updates the CI system to use jdk-21, which is latest LTS supported version. Coming from https://github.com/opensearch-project/OpenSearch/issues/10334 Signed-off-by: John Mazanec --- .github/workflows/CI.yml | 6 +++--- .github/workflows/test_security.yml | 2 +- build.gradle | 2 +- src/main/java/org/opensearch/knn/indices/ModelDao.java | 4 +++- .../knn/plugin/transport/KNNWarmupResponse.java | 2 +- .../knn/plugin/transport/KNNWarmupTransportAction.java | 2 +- src/testFixtures/java/org/opensearch/knn/TestUtils.java | 9 +++------ 7 files changed, 13 insertions(+), 14 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 84b4f09f9..c807e348e 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -20,7 +20,7 @@ jobs: Build-k-NN-Linux: strategy: matrix: - java: [11, 17] + java: [11, 17, 21] name: Build and Test k-NN Plugin on Linux runs-on: ubuntu-latest @@ -57,7 +57,7 @@ jobs: Build-k-NN-MacOS: strategy: matrix: - java: [ 11, 17 ] + java: [ 11, 17, 21 ] name: Build and Test k-NN Plugin on MacOS needs: Get-CI-Image-Tag @@ -84,7 +84,7 @@ jobs: Build-k-NN-Windows: strategy: matrix: - java: [ 11, 17 ] + java: [ 11, 17, 21 ] name: Build and Test k-NN Plugin on Windows needs: Get-CI-Image-Tag diff --git a/.github/workflows/test_security.yml b/.github/workflows/test_security.yml index 3813ff675..ff3ca1459 100644 --- a/.github/workflows/test_security.yml +++ b/.github/workflows/test_security.yml @@ -15,7 +15,7 @@ jobs: Build-ad: strategy: matrix: - java: [ 11,17 ] + java: [ 11,17,21 ] os: [ubuntu-latest] fail-fast: true diff --git a/build.gradle b/build.gradle index f3dcff3e3..fb56692ce 100644 --- a/build.gradle +++ b/build.gradle @@ -36,7 +36,7 @@ plugins { id 'java-library' id 'java-test-fixtures' id 'idea' - id "com.diffplug.spotless" version "6.3.0" apply false + id "com.diffplug.spotless" version "6.20.0" apply false id 'io.freefair.lombok' version '8.4' } diff --git a/src/main/java/org/opensearch/knn/indices/ModelDao.java b/src/main/java/org/opensearch/knn/indices/ModelDao.java index fe0deb1cf..eada08b44 100644 --- a/src/main/java/org/opensearch/knn/indices/ModelDao.java +++ b/src/main/java/org/opensearch/knn/indices/ModelDao.java @@ -654,7 +654,9 @@ private void removeModelIdFromGraveyardOnFailure(String modelId, Exception excep client.execute( UpdateModelGraveyardAction.INSTANCE, new UpdateModelGraveyardRequest(modelId, true), - ActionListener.wrap(acknowledgedResponse -> { throw exceptionFromPreviousStep; }, unblockingFailedException -> { + ActionListener.wrap(acknowledgedResponse -> { + throw exceptionFromPreviousStep; + }, unblockingFailedException -> { // If it fails to remove the modelId from Model Graveyard, then log the error message and // throw the exception that was passed as a parameter from previous step String errorMessage = String.format("Failed to remove \" %s \" from Model Graveyard", modelId); diff --git a/src/main/java/org/opensearch/knn/plugin/transport/KNNWarmupResponse.java b/src/main/java/org/opensearch/knn/plugin/transport/KNNWarmupResponse.java index f13fbd3db..d9e607db2 100644 --- a/src/main/java/org/opensearch/knn/plugin/transport/KNNWarmupResponse.java +++ b/src/main/java/org/opensearch/knn/plugin/transport/KNNWarmupResponse.java @@ -5,7 +5,7 @@ package org.opensearch.knn.plugin.transport; -import org.opensearch.core.action.support.DefaultShardOperationFailedException;; +import org.opensearch.core.action.support.DefaultShardOperationFailedException; import org.opensearch.action.support.broadcast.BroadcastResponse; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.xcontent.ToXContentObject; diff --git a/src/main/java/org/opensearch/knn/plugin/transport/KNNWarmupTransportAction.java b/src/main/java/org/opensearch/knn/plugin/transport/KNNWarmupTransportAction.java index c55c9b619..a738527ff 100644 --- a/src/main/java/org/opensearch/knn/plugin/transport/KNNWarmupTransportAction.java +++ b/src/main/java/org/opensearch/knn/plugin/transport/KNNWarmupTransportAction.java @@ -9,7 +9,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.opensearch.action.support.ActionFilters; -import org.opensearch.core.action.support.DefaultShardOperationFailedException;; +import org.opensearch.core.action.support.DefaultShardOperationFailedException; import org.opensearch.action.support.broadcast.node.TransportBroadcastByNodeAction; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.block.ClusterBlockException; diff --git a/src/testFixtures/java/org/opensearch/knn/TestUtils.java b/src/testFixtures/java/org/opensearch/knn/TestUtils.java index a9721ce14..61c0abe4f 100644 --- a/src/testFixtures/java/org/opensearch/knn/TestUtils.java +++ b/src/testFixtures/java/org/opensearch/knn/TestUtils.java @@ -208,12 +208,9 @@ public static PriorityQueue computeGroundTruthValues(int k, SpaceTyp public static float computeDistFromSpaceType(SpaceType spaceType, float[] indexVector, float[] queryVector) { float dist; if (spaceType != null) { - dist = KNN_SCORING_SPACE_TYPE.getOrDefault( - spaceType, - (defaultQueryVector, defaultIndexVector) -> { - throw new IllegalArgumentException(String.format("Invalid SpaceType function: \"%s\"", spaceType)); - } - ).apply(queryVector, indexVector); + dist = KNN_SCORING_SPACE_TYPE.getOrDefault(spaceType, (defaultQueryVector, defaultIndexVector) -> { + throw new IllegalArgumentException(String.format("Invalid SpaceType function: \"%s\"", spaceType)); + }).apply(queryVector, indexVector); } else { throw new NullPointerException("SpaceType is null. Provide a valid SpaceType."); } From ddf988d7f722055cf27de6eae0fe9f39454f54d9 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Tue, 14 Nov 2023 14:42:53 -0800 Subject: [PATCH 164/416] Move data dummy var to struct method in nmslib jni (#1308) Signed-off-by: John Mazanec --- CHANGELOG.md | 3 ++- jni/include/nmslib_wrapper.h | 2 +- jni/tests/nmslib_wrapper_test.cpp | 11 +++++++++++ 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index feb1411e3..f1350e532 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,7 +16,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Features * Add parent join support for lucene knn [#1182](https://github.com/opensearch-project/k-NN/pull/1182) ### Enhancements -### Bug Fixes +### Bug Fixes +* Fix use-after-free case on nmslib search path [#1305](https://github.com/opensearch-project/k-NN/pull/1305) ### Infrastructure * Upgrade gradle to 8.4 [1289](https://github.com/opensearch-project/k-NN/pull/1289) ### Documentation diff --git a/jni/include/nmslib_wrapper.h b/jni/include/nmslib_wrapper.h index 6d862048a..9b555580a 100644 --- a/jni/include/nmslib_wrapper.h +++ b/jni/include/nmslib_wrapper.h @@ -48,10 +48,10 @@ namespace knn_jni { struct IndexWrapper { explicit IndexWrapper(const std::string& spaceType) { // Index gets constructed with a reference to data (see above) but is otherwise unused - similarity::ObjectVector data; space.reset(similarity::SpaceFactoryRegistry::Instance().CreateSpace(spaceType, similarity::AnyParams())); index.reset(similarity::MethodFactoryRegistry::Instance().CreateMethod(false, "hnsw", spaceType, *space, data)); } + similarity::ObjectVector data; std::unique_ptr> space; std::unique_ptr> index; }; diff --git a/jni/tests/nmslib_wrapper_test.cpp b/jni/tests/nmslib_wrapper_test.cpp index 96c3e1d1c..3a21e7401 100644 --- a/jni/tests/nmslib_wrapper_test.cpp +++ b/jni/tests/nmslib_wrapper_test.cpp @@ -21,6 +21,17 @@ using ::testing::NiceMock; using ::testing::Return; +TEST(NmslibIndexWrapperSearchTest, BasicAssertions) { + similarity::initLibrary(); + knn_jni::nmslib_wrapper::IndexWrapper indexWrapper = knn_jni::nmslib_wrapper::IndexWrapper("l2"); + int k = 10; + int dim = 2; + std::unique_ptr rawQueryvector(new float[dim]); + std::unique_ptr queryObject(new similarity::Object(-1, -1, dim*sizeof(float), rawQueryvector.get())); + similarity::KNNQuery knnQuery(*(indexWrapper.space), queryObject.get(), k); + indexWrapper.index->Search((similarity::KNNQuery *)nullptr); +} + TEST(NmslibCreateIndexTest, BasicAssertions) { // Initialize nmslib similarity::initLibrary(); From 90934fa3960ce2c2af6f3f61bed1b5dd7581675b Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Mon, 20 Nov 2023 14:44:09 -0800 Subject: [PATCH 165/416] Add release note for 2.11.1 (#1314) (#1315) Signed-off-by: Heemin Kim (cherry picked from commit af8c9fdca96b830c4d8bbd54fbad06c0b7c8ed19) Co-authored-by: Heemin Kim --- release-notes/opensearch-knn.release-notes-2.11.1.0.md | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 release-notes/opensearch-knn.release-notes-2.11.1.0.md diff --git a/release-notes/opensearch-knn.release-notes-2.11.1.0.md b/release-notes/opensearch-knn.release-notes-2.11.1.0.md new file mode 100644 index 000000000..238d05bcc --- /dev/null +++ b/release-notes/opensearch-knn.release-notes-2.11.1.0.md @@ -0,0 +1,6 @@ +## Version 2.11.1.0 Release Notes + +Compatible with OpenSearch 2.11.1 + +### Infrastructure +* Make sure not hardcoding user name when switching to uid 1000 on CI.yml [#1252](https://github.com/opensearch-project/k-NN/pull/1252) From 7c1782adb406f48420eee8a716365b9e2d0c0130 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Fri, 24 Nov 2023 20:29:26 +0530 Subject: [PATCH 166/416] Update CVE-affected dependency versions (#1319) (#1322) * Update CVE-affected dependency versions Signed-off-by: Daniel Widdis * Change log Signed-off-by: Daniel Widdis --------- Signed-off-by: Daniel Widdis (cherry picked from commit a45054f13494ca647523768266dc94f24b724a27) Co-authored-by: Daniel Widdis --- CHANGELOG.md | 2 ++ benchmarks/osb/requirements.txt | 2 +- benchmarks/perf-tool/requirements.txt | 2 +- build.gradle | 2 +- 4 files changed, 5 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f1350e532..310d8951c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,4 +24,6 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Maintenance * Update developer guide to include M1 Setup [#1222](https://github.com/opensearch-project/k-NN/pull/1222) * Upgrade urllib to 1.26.17 [#1278](https://github.com/opensearch-project/k-NN/pull/1278) +* Upgrade urllib to 1.26.18 [#1319](https://github.com/opensearch-project/k-NN/pull/1319) +* Upgrade guava to 32.1.3 [#1319](https://github.com/opensearch-project/k-NN/pull/1319) ### Refactoring diff --git a/benchmarks/osb/requirements.txt b/benchmarks/osb/requirements.txt index 1d918d881..1bf7aadbb 100644 --- a/benchmarks/osb/requirements.txt +++ b/benchmarks/osb/requirements.txt @@ -85,7 +85,7 @@ tabulate==0.8.7 # via opensearch-benchmark thespian==3.10.1 # via opensearch-benchmark -urllib3==1.26.17 +urllib3==1.26.18 # via opensearch-py yappi==1.2.3 # via opensearch-benchmark diff --git a/benchmarks/perf-tool/requirements.txt b/benchmarks/perf-tool/requirements.txt index ca0d30b91..fdcff1fa8 100644 --- a/benchmarks/perf-tool/requirements.txt +++ b/benchmarks/perf-tool/requirements.txt @@ -30,7 +30,7 @@ pyyaml==5.4.1 # via -r requirements.in requests==2.31.0 # via -r requirements.in -urllib3==1.26.17 +urllib3==1.26.18 # via # opensearch-py # requests diff --git a/build.gradle b/build.gradle index fb56692ce..49ede4f0c 100644 --- a/build.gradle +++ b/build.gradle @@ -170,7 +170,7 @@ dependencies { api "org.opensearch:opensearch:${opensearch_version}" compileOnly "org.opensearch.plugin:opensearch-scripting-painless-spi:${versions.opensearch}" api group: 'com.google.guava', name: 'failureaccess', version:'1.0.1' - api group: 'com.google.guava', name: 'guava', version:'32.0.1-jre' + api group: 'com.google.guava', name: 'guava', version:'32.1.3-jre' api group: 'commons-lang', name: 'commons-lang', version: '2.6' testFixturesImplementation "org.opensearch.test:framework:${opensearch_version}" testImplementation group: 'net.bytebuddy', name: 'byte-buddy', version: '1.14.7' From 6b92dbcbc4959f7807786647056005bfb95346c9 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Mon, 27 Nov 2023 10:04:40 -0800 Subject: [PATCH 167/416] Fix Spotless Transitive Dependency and bump aiohttp to fix CVE (#1323) (#1324) Signed-off-by: Naveen Tatikonda (cherry picked from commit 5e2f899ccafc3adfebab48235bdfad088b06e9f3) Co-authored-by: Naveen Tatikonda --- benchmarks/osb/requirements.txt | 2 +- build.gradle | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/benchmarks/osb/requirements.txt b/benchmarks/osb/requirements.txt index 1bf7aadbb..7d7cfcf67 100644 --- a/benchmarks/osb/requirements.txt +++ b/benchmarks/osb/requirements.txt @@ -4,7 +4,7 @@ # # pip-compile # -aiohttp==3.8.5 +aiohttp==3.8.6 # via opensearch-py aiosignal==1.2.0 # via aiohttp diff --git a/build.gradle b/build.gradle index 49ede4f0c..ceba9a038 100644 --- a/build.gradle +++ b/build.gradle @@ -25,6 +25,11 @@ buildscript { dependencies { classpath "${opensearch_group}.gradle:build-tools:${opensearch_version}" + configurations.all { + resolutionStrategy { + force("org.eclipse.platform:org.eclipse.core.runtime:3.29.0") // for spotless transitive dependency CVE (for 3.26.100) + } + } } } From d5db2ae725853ee3bebc6ad6888ef1229e8fc8da Mon Sep 17 00:00:00 2001 From: gaobinlong Date: Sat, 2 Dec 2023 01:11:14 +0800 Subject: [PATCH 168/416] Add more description about running OpenSearch on MAC M1 to developer guide (#1334) Signed-off-by: gaobinlong --- DEVELOPER_GUIDE.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index 582041e1d..cbfaf0f4c 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -127,6 +127,13 @@ Next, obtain a minimum distribution tarball of the k-NN version you want to buil 4. You should see a opensearch-min--SNAPSHOT-darwin-x64.tar.gz file present in distribution/archives/darwin-tar/build/distributions/ 5. Build k-NN by passing the OpenSearch distribution path in `./gradlew -PcustomDistributionUrl=""` +If you want to start OpenSearch directly on Mac M1, make sure to use JDK for ARM. Otherwise, you will see the following error: `mach-o file, but is an incompatible architecture (have 'arm64', need 'x86_64')`. It is better to start OpenSearch by running `bash opensearch-tar-install.sh` instead of `./bin/opensearch`. To run `./bin/opensearch`, the environment variable `JAVA_LIBRARY_PATH` needs to be set correctly so that OpenSearch can find the JNI library: + +``` +export OPENSEARCH_HOME=the directory of opensearch... +export JAVA_LIBRARY_PATH=$JAVA_LIBRARY_PATH:$OPENSEARCH_HOME/plugins/opensearch-knn/lib +``` + #### Environment Currently, the plugin only supports Linux on x64 and arm platforms. From 06d52d55f21d054dc267f637fa279ed1b6b59e08 Mon Sep 17 00:00:00 2001 From: Junqiu Lei Date: Thu, 7 Dec 2023 15:49:57 -0800 Subject: [PATCH 169/416] Allow nested knn field mapping when train model (#1318) (#1339) (cherry picked from commit 2e3ab953e3a9a3bdba83464f93d325e8eb12f567) Signed-off-by: Junqiu Lei --- CHANGELOG.md | 1 + .../org/opensearch/knn/index/IndexUtil.java | 41 +++++- .../opensearch/knn/training/VectorReader.java | 46 +++++- .../opensearch/knn/KNNSingleNodeTestCase.java | 54 +++++++ .../opensearch/knn/index/IndexUtilTests.java | 137 ++++++++++++++++++ .../action/RestTrainModelHandlerIT.java | 70 +++++++++ .../knn/training/VectorReaderTests.java | 41 ++++++ .../org/opensearch/knn/KNNRestTestCase.java | 83 +++++++++++ 8 files changed, 465 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 310d8951c..3328dda52 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Enhancements ### Bug Fixes * Fix use-after-free case on nmslib search path [#1305](https://github.com/opensearch-project/k-NN/pull/1305) +* Allow nested knn field mapping when train model [#1318](https://github.com/opensearch-project/k-NN/pull/1318) ### Infrastructure * Upgrade gradle to 8.4 [1289](https://github.com/opensearch-project/k-NN/pull/1289) ### Documentation diff --git a/src/main/java/org/opensearch/knn/index/IndexUtil.java b/src/main/java/org/opensearch/knn/index/IndexUtil.java index 4e2dc38c6..b3a24d2f0 100644 --- a/src/main/java/org/opensearch/knn/index/IndexUtil.java +++ b/src/main/java/org/opensearch/knn/index/IndexUtil.java @@ -13,6 +13,7 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.Maps; +import org.apache.commons.lang.StringUtils; import org.opensearch.Version; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.metadata.MappingMetadata; @@ -26,6 +27,7 @@ import java.io.File; import java.util.Collections; import java.util.HashMap; +import java.util.Locale; import java.util.Map; import static org.opensearch.knn.common.KNNConstants.BYTES_PER_KILOBYTES; @@ -61,6 +63,37 @@ public static int getFileSizeInKB(String filePath) { return Math.toIntExact((file.length() / BYTES_PER_KILOBYTES) + 1L); // Add one so that integer division rounds up } + /** + * This method retrieves the field mapping by a given field path from the index metadata. + * + * @param properties Index metadata mapping properties. + * @param fieldPath The field path string that make up the path to the field mapping. e.g. "a.b.field" or "field". + * The field path is applied and checked in OpenSearch, so it is guaranteed to be valid. + * + * @return The field mapping object if found, or null if the field is not found in the index metadata. + */ + private static Object getFieldMapping(final Map properties, final String fieldPath) { + String[] fieldPaths = fieldPath.split("\\."); + Object currentFieldMapping = properties; + + // Iterate through the field path list to retrieve the field mapping. + for (String path : fieldPaths) { + currentFieldMapping = ((Map) currentFieldMapping).get(path); + if (currentFieldMapping == null) { + return null; + } + + if (currentFieldMapping instanceof Map) { + Object possibleProperties = ((Map) currentFieldMapping).get("properties"); + if (possibleProperties instanceof Map) { + currentFieldMapping = possibleProperties; + } + } + } + + return currentFieldMapping; + } + /** * Validate that a field is a k-NN vector field and has the expected dimension * @@ -102,7 +135,13 @@ public static ValidationException validateKnnField( return exception; } - Object fieldMapping = properties.get(field); + // Check field path is valid + if (StringUtils.isEmpty(field)) { + exception.addValidationError(String.format(Locale.ROOT, "Field path is empty.")); + return exception; + } + + Object fieldMapping = getFieldMapping(properties, field); // Check field existence if (fieldMapping == null) { diff --git a/src/main/java/org/opensearch/knn/training/VectorReader.java b/src/main/java/org/opensearch/knn/training/VectorReader.java index c104e2df3..aeebae129 100644 --- a/src/main/java/org/opensearch/knn/training/VectorReader.java +++ b/src/main/java/org/opensearch/knn/training/VectorReader.java @@ -29,6 +29,7 @@ import java.util.ArrayList; import java.util.List; +import java.util.Map; import java.util.function.Consumer; public class VectorReader { @@ -180,13 +181,7 @@ public void onResponse(SearchResponse searchResponse) { // Either add the entire set of returned hits, or maxVectorCount - collectedVectorCount hits SearchHit[] hits = searchResponse.getHits().getHits(); int vectorsToAdd = Integer.min(maxVectorCount - collectedVectorCount, hits.length); - List trainingData = new ArrayList<>(); - - for (int i = 0; i < vectorsToAdd; i++) { - trainingData.add( - ((List) hits[i].getSourceAsMap().get(fieldName)).stream().map(Number::floatValue).toArray(Float[]::new) - ); - } + List trainingData = extractVectorsFromHits(searchResponse, vectorsToAdd); this.collectedVectorCount += trainingData.size(); @@ -225,5 +220,42 @@ public void onFailure(Exception e) { listener.onFailure(e); } } + + /** + * Extracts vectors from the hits in a search response + * + * @param searchResponse Search response to extract vectors from + * @param vectorsToAdd number of vectors to extract + * @return list of vectors + */ + private List extractVectorsFromHits(SearchResponse searchResponse, int vectorsToAdd) { + SearchHit[] hits = searchResponse.getHits().getHits(); + List trainingData = new ArrayList<>(); + String[] fieldPath = fieldName.split("\\."); + int nullVectorCount = 0; + + for (int vector = 0; vector < vectorsToAdd; vector++) { + Map currentMap = hits[vector].getSourceAsMap(); + // The field name may be a nested field, so we need to split it and traverse the map. + // Example fieldName: "my_field" or "my_field.nested_field.nested_nested_field" + + for (int pathPart = 0; pathPart < fieldPath.length - 1; pathPart++) { + currentMap = (Map) currentMap.get(fieldPath[pathPart]); + } + + if (currentMap.get(fieldPath[fieldPath.length - 1]) instanceof List == false) { + nullVectorCount++; + continue; + } + + List fieldList = (List) currentMap.get(fieldPath[fieldPath.length - 1]); + + trainingData.add(fieldList.stream().map(Number::floatValue).toArray(Float[]::new)); + } + if (nullVectorCount > 0) { + logger.warn("Found {} documents with null vectors in field {}", nullVectorCount, fieldName); + } + return trainingData; + } } } diff --git a/src/test/java/org/opensearch/knn/KNNSingleNodeTestCase.java b/src/test/java/org/opensearch/knn/KNNSingleNodeTestCase.java index 792ebde69..323442fff 100644 --- a/src/test/java/org/opensearch/knn/KNNSingleNodeTestCase.java +++ b/src/test/java/org/opensearch/knn/KNNSingleNodeTestCase.java @@ -81,6 +81,35 @@ protected void createKnnIndexMapping(String indexName, String fieldName, Integer OpenSearchAssertions.assertAcked(client().admin().indices().putMapping(request).actionGet()); } + /** + * Create simple k-NN mapping which can be nested. + * e.g. fieldPath = "a.b.c" will create mapping for "c" as knn_vector + */ + protected void createKnnNestedIndexMapping(String indexName, String fieldPath, Integer dimensions) throws IOException { + PutMappingRequest request = new PutMappingRequest(indexName); + String[] path = fieldPath.split("\\."); + XContentBuilder xContentBuilder = XContentFactory.jsonBuilder().startObject().startObject("properties"); + for (int i = 0; i < path.length; i++) { + xContentBuilder.startObject(path[i]); + if (i == path.length - 1) { + xContentBuilder.field("type", "knn_vector").field("dimension", dimensions.toString()); + } else { + xContentBuilder.startObject("properties"); + } + } + for (int i = path.length - 1; i >= 0; i--) { + if (i != path.length - 1) { + xContentBuilder.endObject(); + } + xContentBuilder.endObject(); + } + xContentBuilder.endObject().endObject(); + + request.source(xContentBuilder); + + OpenSearchAssertions.assertAcked(client().admin().indices().putMapping(request).actionGet()); + } + /** * Get default k-NN settings for test cases */ @@ -103,6 +132,31 @@ protected void addKnnDoc(String index, String docId, String fieldName, Object[] assertEquals(response.status(), RestStatus.CREATED); } + /** + * Add a k-NN doc to an index with nested knn_vector field + */ + protected void addKnnNestedDoc(String index, String docId, String fieldPath, Object[] vector) throws IOException, InterruptedException, + ExecutionException { + XContentBuilder builder = XContentFactory.jsonBuilder().startObject(); + String[] fieldParts = fieldPath.split("\\."); + + for (int i = 0; i < fieldParts.length - 1; i++) { + builder.startObject(fieldParts[i]); + } + builder.field(fieldParts[fieldParts.length - 1], vector); + for (int i = fieldParts.length - 2; i >= 0; i--) { + builder.endObject(); + } + builder.endObject(); + IndexRequest indexRequest = new IndexRequest().index(index) + .id(docId) + .source(builder) + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + + IndexResponse response = client().index(indexRequest).get(); + assertEquals(response.status(), RestStatus.CREATED); + } + /** * Add any document to index */ diff --git a/src/test/java/org/opensearch/knn/index/IndexUtilTests.java b/src/test/java/org/opensearch/knn/index/IndexUtilTests.java index 7013ef261..dc9c980e0 100644 --- a/src/test/java/org/opensearch/knn/index/IndexUtilTests.java +++ b/src/test/java/org/opensearch/knn/index/IndexUtilTests.java @@ -14,13 +14,18 @@ import com.google.common.collect.ImmutableMap; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.cluster.metadata.MappingMetadata; import org.opensearch.cluster.metadata.Metadata; import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.ValidationException; import org.opensearch.common.settings.Settings; import org.opensearch.knn.KNNTestCase; import org.opensearch.knn.index.util.KNNEngine; +import org.opensearch.knn.indices.ModelDao; +import org.opensearch.knn.indices.ModelMetadata; import java.util.Map; +import java.util.Objects; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.mock; @@ -67,4 +72,136 @@ public void testGetLoadParameters() { assertEquals(spaceType2.getValue(), loadParameters.get(SPACE_TYPE)); assertEquals(efSearchValue, loadParameters.get(HNSW_ALGO_EF_SEARCH)); } + + public void testValidateKnnField_NestedField() { + Map deepFieldValues = Map.of("type", "knn_vector", "dimension", 8); + Map deepField = Map.of("train-field", deepFieldValues); + Map deepFieldProperties = Map.of("properties", deepField); + Map nest_b = Map.of("b", deepFieldProperties); + Map nest_b_properties = Map.of("properties", nest_b); + Map nest_a = Map.of("a", nest_b_properties); + Map properties = Map.of("properties", nest_a); + + String field = "a.b.train-field"; + int dimension = 8; + + MappingMetadata mappingMetadata = mock(MappingMetadata.class); + when(mappingMetadata.getSourceAsMap()).thenReturn(properties); + IndexMetadata indexMetadata = mock(IndexMetadata.class); + when(indexMetadata.mapping()).thenReturn(mappingMetadata); + ModelDao modelDao = mock(ModelDao.class); + ModelMetadata trainingFieldModelMetadata = mock(ModelMetadata.class); + when(trainingFieldModelMetadata.getDimension()).thenReturn(dimension); + when(modelDao.getMetadata(anyString())).thenReturn(trainingFieldModelMetadata); + + ValidationException e = IndexUtil.validateKnnField(indexMetadata, field, dimension, modelDao); + + assertNull(e); + } + + public void testValidateKnnField_NonNestedField() { + Map fieldValues = Map.of("type", "knn_vector", "dimension", 8); + Map top_level_field = Map.of("top_level_field", fieldValues); + Map properties = Map.of("properties", top_level_field); + String field = "top_level_field"; + int dimension = 8; + + MappingMetadata mappingMetadata = mock(MappingMetadata.class); + when(mappingMetadata.getSourceAsMap()).thenReturn(properties); + IndexMetadata indexMetadata = mock(IndexMetadata.class); + when(indexMetadata.mapping()).thenReturn(mappingMetadata); + ModelDao modelDao = mock(ModelDao.class); + ModelMetadata trainingFieldModelMetadata = mock(ModelMetadata.class); + when(trainingFieldModelMetadata.getDimension()).thenReturn(dimension); + when(modelDao.getMetadata(anyString())).thenReturn(trainingFieldModelMetadata); + + ValidationException e = IndexUtil.validateKnnField(indexMetadata, field, dimension, modelDao); + + assertNull(e); + } + + public void testValidateKnnField_NonKnnField() { + Map fieldValues = Map.of("type", "text"); + Map top_level_field = Map.of("top_level_field", fieldValues); + Map properties = Map.of("properties", top_level_field); + String field = "top_level_field"; + int dimension = 8; + MappingMetadata mappingMetadata = mock(MappingMetadata.class); + when(mappingMetadata.getSourceAsMap()).thenReturn(properties); + IndexMetadata indexMetadata = mock(IndexMetadata.class); + when(indexMetadata.mapping()).thenReturn(mappingMetadata); + ModelDao modelDao = mock(ModelDao.class); + ModelMetadata trainingFieldModelMetadata = mock(ModelMetadata.class); + when(trainingFieldModelMetadata.getDimension()).thenReturn(dimension); + when(modelDao.getMetadata(anyString())).thenReturn(trainingFieldModelMetadata); + + ValidationException e = IndexUtil.validateKnnField(indexMetadata, field, dimension, modelDao); + + assert Objects.requireNonNull(e).getMessage().matches("Validation Failed: 1: Field \"" + field + "\" is not of type knn_vector.;"); + } + + public void testValidateKnnField_WrongFieldPath() { + Map deepFieldValues = Map.of("type", "knn_vector", "dimension", 8); + Map deepField = Map.of("train-field", deepFieldValues); + Map deepFieldProperties = Map.of("properties", deepField); + Map nest_b = Map.of("b", deepFieldProperties); + Map nest_b_properties = Map.of("properties", nest_b); + Map nest_a = Map.of("a", nest_b_properties); + Map properties = Map.of("properties", nest_a); + String field = "a.train-field"; + int dimension = 8; + MappingMetadata mappingMetadata = mock(MappingMetadata.class); + when(mappingMetadata.getSourceAsMap()).thenReturn(properties); + IndexMetadata indexMetadata = mock(IndexMetadata.class); + when(indexMetadata.mapping()).thenReturn(mappingMetadata); + ModelDao modelDao = mock(ModelDao.class); + ModelMetadata trainingFieldModelMetadata = mock(ModelMetadata.class); + when(trainingFieldModelMetadata.getDimension()).thenReturn(dimension); + when(modelDao.getMetadata(anyString())).thenReturn(trainingFieldModelMetadata); + + ValidationException e = IndexUtil.validateKnnField(indexMetadata, field, dimension, modelDao); + + assert (Objects.requireNonNull(e).getMessage().matches("Validation Failed: 1: Field \"" + field + "\" does not exist.;")); + } + + public void testValidateKnnField_EmptyField() { + Map deepFieldValues = Map.of("type", "knn_vector", "dimension", 8); + Map deepField = Map.of("train-field", deepFieldValues); + Map deepFieldProperties = Map.of("properties", deepField); + Map nest_b = Map.of("b", deepFieldProperties); + Map nest_b_properties = Map.of("properties", nest_b); + Map nest_a = Map.of("a", nest_b_properties); + Map properties = Map.of("properties", nest_a); + String field = ""; + int dimension = 8; + MappingMetadata mappingMetadata = mock(MappingMetadata.class); + when(mappingMetadata.getSourceAsMap()).thenReturn(properties); + IndexMetadata indexMetadata = mock(IndexMetadata.class); + when(indexMetadata.mapping()).thenReturn(mappingMetadata); + ModelDao modelDao = mock(ModelDao.class); + ModelMetadata trainingFieldModelMetadata = mock(ModelMetadata.class); + when(trainingFieldModelMetadata.getDimension()).thenReturn(dimension); + when(modelDao.getMetadata(anyString())).thenReturn(trainingFieldModelMetadata); + + ValidationException e = IndexUtil.validateKnnField(indexMetadata, field, dimension, modelDao); + + System.out.println(Objects.requireNonNull(e).getMessage()); + + assert (Objects.requireNonNull(e).getMessage().matches("Validation Failed: 1: Field path is empty.;")); + } + + public void testValidateKnnField_EmptyIndexMetadata() { + String field = "a.b.train-field"; + int dimension = 8; + IndexMetadata indexMetadata = mock(IndexMetadata.class); + when(indexMetadata.mapping()).thenReturn(null); + ModelDao modelDao = mock(ModelDao.class); + ModelMetadata trainingFieldModelMetadata = mock(ModelMetadata.class); + when(trainingFieldModelMetadata.getDimension()).thenReturn(dimension); + when(modelDao.getMetadata(anyString())).thenReturn(trainingFieldModelMetadata); + + ValidationException e = IndexUtil.validateKnnField(indexMetadata, field, dimension, modelDao); + + assert (Objects.requireNonNull(e).getMessage().matches("Validation Failed: 1: Invalid index. Index does not contain a mapping;")); + } } diff --git a/src/test/java/org/opensearch/knn/plugin/action/RestTrainModelHandlerIT.java b/src/test/java/org/opensearch/knn/plugin/action/RestTrainModelHandlerIT.java index 4b996be28..b2f429e2a 100644 --- a/src/test/java/org/opensearch/knn/plugin/action/RestTrainModelHandlerIT.java +++ b/src/test/java/org/opensearch/knn/plugin/action/RestTrainModelHandlerIT.java @@ -342,4 +342,74 @@ public void testTrainModel_success_noId() throws IOException, InterruptedExcepti assertTrainingSucceeds(modelId, 30, 1000); } + + // Test to checks when user tries to train a model with nested fields + public void testTrainModel_success_nestedField() throws Exception { + String modelId = "test-model-id"; + String trainingIndexName = "train-index"; + String nestedFieldPath = "a.b.train-field"; + int dimension = 8; + + // Create a training index and randomly ingest data into it + String mapping = createKnnIndexNestedMapping(dimension, nestedFieldPath); + createKnnIndex(trainingIndexName, mapping); + int trainingDataCount = 200; + bulkIngestRandomVectorsWithNestedField(trainingIndexName, nestedFieldPath, trainingDataCount, dimension); + + // Call the train API with this definition: + /* + { + "training_index": "train_index", + "training_field": "a.b.train_field", + "dimension": 8, + "description": "this should be allowed to be null", + "method": { + "name":"ivf", + "engine":"faiss", + "space_type": "l2", + "parameters":{ + "nlist":1, + "encoder":{ + "name":"pq", + "parameters":{ + "code_size":2, + "m": 2 + } + } + } + } + } + */ + XContentBuilder builder = XContentFactory.jsonBuilder() + .startObject() + .field(NAME, "ivf") + .field(KNN_ENGINE, "faiss") + .field(METHOD_PARAMETER_SPACE_TYPE, "l2") + .startObject(PARAMETERS) + .field(METHOD_PARAMETER_NLIST, 1) + .startObject(METHOD_ENCODER_PARAMETER) + .field(NAME, "pq") + .startObject(PARAMETERS) + .field(ENCODER_PARAMETER_PQ_CODE_SIZE, 2) + .field(ENCODER_PARAMETER_PQ_M, 2) + .endObject() + .endObject() + .endObject() + .endObject(); + Map method = xContentBuilderToMap(builder); + + Response trainResponse = trainModel(modelId, trainingIndexName, nestedFieldPath, dimension, method, "dummy description"); + + assertEquals(RestStatus.OK, RestStatus.fromCode(trainResponse.getStatusLine().getStatusCode())); + + Response getResponse = getModel(modelId, null); + String responseBody = EntityUtils.toString(getResponse.getEntity()); + assertNotNull(responseBody); + + Map responseMap = createParser(XContentType.JSON.xContent(), responseBody).map(); + + assertEquals(modelId, responseMap.get(MODEL_ID)); + + assertTrainingSucceeds(modelId, 30, 1000); + } } diff --git a/src/test/java/org/opensearch/knn/training/VectorReaderTests.java b/src/test/java/org/opensearch/knn/training/VectorReaderTests.java index 74008a043..209c9cc73 100644 --- a/src/test/java/org/opensearch/knn/training/VectorReaderTests.java +++ b/src/test/java/org/opensearch/knn/training/VectorReaderTests.java @@ -34,6 +34,7 @@ public class VectorReaderTests extends KNNSingleNodeTestCase { private final static int DEFAULT_LATCH_TIMEOUT = 100; private final static String DEFAULT_INDEX_NAME = "test-index"; private final static String DEFAULT_FIELD_NAME = "test-field"; + private final static String DEFAULT_NESTED_FIELD_PATH = "a.b.test-field"; private final static int DEFAULT_DIMENSION = 16; private final static int DEFAULT_NUM_VECTORS = 100; private final static int DEFAULT_MAX_VECTOR_COUNT = 10000; @@ -345,6 +346,46 @@ public void testRead_invalid_fieldIsNotKnn() throws InterruptedException, Execut ); } + public void testRead_valid_NestedField() throws InterruptedException, ExecutionException, IOException { + createIndex(DEFAULT_INDEX_NAME); + createKnnNestedIndexMapping(DEFAULT_INDEX_NAME, DEFAULT_NESTED_FIELD_PATH, DEFAULT_DIMENSION); + + // Create list of random vectors and ingest + Random random = new Random(); + List vectors = new ArrayList<>(); + for (int i = 0; i < DEFAULT_NUM_VECTORS; i++) { + Float[] vector = random.doubles(DEFAULT_DIMENSION).boxed().map(Double::floatValue).toArray(Float[]::new); + vectors.add(vector); + addKnnNestedDoc(DEFAULT_INDEX_NAME, Integer.toString(i), DEFAULT_NESTED_FIELD_PATH, vector); + } + + // Configure VectorReader + ClusterService clusterService = node().injector().getInstance(ClusterService.class); + VectorReader vectorReader = new VectorReader(client()); + + // Read all vectors and confirm they match vectors + TestVectorConsumer testVectorConsumer = new TestVectorConsumer(); + final CountDownLatch inProgressLatch = new CountDownLatch(1); + vectorReader.read( + clusterService, + DEFAULT_INDEX_NAME, + DEFAULT_NESTED_FIELD_PATH, + DEFAULT_MAX_VECTOR_COUNT, + DEFAULT_SEARCH_SIZE, + testVectorConsumer, + createOnSearchResponseCountDownListener(inProgressLatch) + ); + + assertLatchDecremented(inProgressLatch); + + List consumedVectors = testVectorConsumer.getVectorsConsumed(); + assertEquals(DEFAULT_NUM_VECTORS, consumedVectors.size()); + + List flatVectors = vectors.stream().flatMap(Arrays::stream).collect(Collectors.toList()); + List flatConsumedVectors = consumedVectors.stream().flatMap(Arrays::stream).collect(Collectors.toList()); + assertEquals(new HashSet<>(flatVectors), new HashSet<>(flatConsumedVectors)); + } + private static class TestVectorConsumer implements Consumer> { List vectorsConsumed; diff --git a/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java b/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java index 678f490fa..6f08627ae 100644 --- a/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java +++ b/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java @@ -349,6 +349,38 @@ protected String createKnnIndexMapping(List fieldNames, List di return xContentBuilder.toString(); } + /** + * Utility to create a Knn Index Mapping with nested field + * + * @param dimensions dimension of the vector + * @param fieldPath path of the nested field, e.g. "my_nested_field.my_vector" + * @return mapping string for the nested field + */ + protected String createKnnIndexNestedMapping(Integer dimensions, String fieldPath) throws IOException { + String[] fieldPathArray = fieldPath.split("\\."); + XContentBuilder xContentBuilder = XContentFactory.jsonBuilder().startObject().startObject("properties"); + + for (int i = 0; i < fieldPathArray.length; i++) { + xContentBuilder.startObject(fieldPathArray[i]); + if (i == fieldPathArray.length - 1) { + xContentBuilder.field("type", "knn_vector").field("dimension", dimensions.toString()); + } else { + xContentBuilder.startObject("properties"); + } + } + + for (int i = fieldPathArray.length - 1; i >= 0; i--) { + if (i != fieldPathArray.length - 1) { + xContentBuilder.endObject(); + } + xContentBuilder.endObject(); + } + + xContentBuilder.endObject().endObject(); + + return xContentBuilder.toString(); + } + /** * Get index mapping as map * @@ -416,6 +448,37 @@ protected void addKnnDoc(String index, String docId, String fieldName, Object[] assertEquals(request.getEndpoint() + ": failed", RestStatus.OK, RestStatus.fromCode(response.getStatusLine().getStatusCode())); } + /** + * Add a single KNN Doc to an index with a nested vector field + * + * @param index name of the index + * @param docId id of the document + * @param nestedFieldPath path of the nested field, e.g. "my_nested_field.my_vector" + * @param vector vector to add + * + */ + protected void addKnnDocWithNestedField(String index, String docId, String nestedFieldPath, Object[] vector) throws IOException { + String[] fieldParts = nestedFieldPath.split("\\."); + + XContentBuilder builder = XContentFactory.jsonBuilder().startObject(); + for (int i = 0; i < fieldParts.length - 1; i++) { + builder.startObject(fieldParts[i]); + } + builder.field(fieldParts[fieldParts.length - 1], vector); + for (int i = fieldParts.length - 2; i >= 0; i--) { + builder.endObject(); + } + builder.endObject(); + + Request request = new Request("POST", "/" + index + "/_doc/" + docId + "?refresh=true"); + request.setJsonEntity(builder.toString()); + client().performRequest(request); + + request = new Request("POST", "/" + index + "/_refresh"); + Response response = client().performRequest(request); + assertEquals(request.getEndpoint() + ": failed", RestStatus.OK, RestStatus.fromCode(response.getStatusLine().getStatusCode())); + } + /** * Add a single KNN Doc to an index with multiple fields */ @@ -825,6 +888,26 @@ public void bulkIngestRandomVectors(String indexName, String fieldName, int numV } + /** + * Bulk ingest random vectors with nested field + * + * @param indexName index name + * @param nestedFieldPath nested field path, e.g. "my_nested_field.my_vector_field" + * @param numVectors number of vectors + * @param dimension vector dimension + */ + public void bulkIngestRandomVectorsWithNestedField(String indexName, String nestedFieldPath, int numVectors, int dimension) + throws IOException { + for (int i = 0; i < numVectors; i++) { + float[] vector = new float[dimension]; + for (int j = 0; j < dimension; j++) { + vector[j] = randomFloat(); + } + + addKnnDocWithNestedField(indexName, String.valueOf(i + 1), nestedFieldPath, Floats.asList(vector).toArray()); + } + } + // Method that adds multiple documents into the index using Bulk API public void bulkAddKnnDocs(String index, String fieldName, float[][] indexVectors, int docCount) throws IOException { Request request = new Request("POST", "/_bulk"); From fda94bcfe7beaaa02bcabbab9a49a0fa3310966d Mon Sep 17 00:00:00 2001 From: Ryan Bogan Date: Wed, 13 Dec 2023 09:25:13 -0800 Subject: [PATCH 170/416] [Backport 2.x] Properly designate model state for actively training models when nodes crash or leave cluster (#1348) * Properly designate model state for actively training models when nodes crash or leave cluster Signed-off-by: Ryan Bogan * Fix merge conflict Signed-off-by: Ryan Bogan --------- Signed-off-by: Ryan Bogan --- CHANGELOG.md | 3 + .../java/org/opensearch/knn/bwc/ModelIT.java | 2 +- .../opensearch/knn/common/KNNConstants.java | 1 + .../org/opensearch/knn/index/IndexUtil.java | 12 ++ .../org/opensearch/knn/indices/ModelDao.java | 1 + .../opensearch/knn/indices/ModelMetadata.java | 80 ++++++-- .../org/opensearch/knn/plugin/KNNPlugin.java | 5 + .../TrainingModelTransportAction.java | 6 +- .../opensearch/knn/training/TrainingJob.java | 6 +- .../TrainingJobClusterStateListener.java | 176 +++++++++++++++++ .../knn/training/TrainingJobRunner.java | 25 ++- .../index/KNNCreateIndexFromModelTests.java | 3 +- .../KNN80DocValuesConsumerTests.java | 2 +- .../knn/index/codec/KNNCodecTestCase.java | 1 + .../mapper/KNNVectorFieldMapperTests.java | 2 + .../knn/indices/ModelCacheTests.java | 12 ++ .../opensearch/knn/indices/ModelDaoTests.java | 15 +- .../knn/indices/ModelMetadataTests.java | 136 ++++++++++--- .../knn/indices/ModelStateTests.java | 3 + .../opensearch/knn/indices/ModelTests.java | 44 ++++- .../action/RestSearchModelHandlerIT.java | 2 +- .../transport/GetModelResponseTests.java | 6 +- ...oveModelFromCacheTransportActionTests.java | 2 +- ...RouteDecisionInfoTransportActionTests.java | 13 +- .../transport/TrainingModelRequestTests.java | 1 + .../UpdateModelMetadataRequestTests.java | 3 + ...dateModelMetadataTransportActionTests.java | 1 + .../TrainingJobClusterStateListenerTests.java | 179 ++++++++++++++++++ .../knn/training/TrainingJobRunnerTests.java | 19 +- .../knn/training/TrainingJobTests.java | 32 +++- .../org/opensearch/knn/KNNRestTestCase.java | 2 + 31 files changed, 706 insertions(+), 89 deletions(-) create mode 100644 src/main/java/org/opensearch/knn/training/TrainingJobClusterStateListener.java create mode 100644 src/test/java/org/opensearch/knn/training/TrainingJobClusterStateListenerTests.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 3328dda52..cfd852424 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Bug Fixes * Fix use-after-free case on nmslib search path [#1305](https://github.com/opensearch-project/k-NN/pull/1305) * Allow nested knn field mapping when train model [#1318](https://github.com/opensearch-project/k-NN/pull/1318) +* Properly designate model state for actively training models when nodes crash or leave cluster [#1317](https://github.com/opensearch-project/k-NN/pull/1317) + +>>>>>>> main ### Infrastructure * Upgrade gradle to 8.4 [1289](https://github.com/opensearch-project/k-NN/pull/1289) ### Documentation diff --git a/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/ModelIT.java b/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/ModelIT.java index abf44e016..0ef9c4c91 100644 --- a/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/ModelIT.java +++ b/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/ModelIT.java @@ -257,6 +257,6 @@ public String modelIndexMapping(String fieldName, String modelId) throws IOExcep } private ModelMetadata getModelMetadata() { - return new ModelMetadata(KNNEngine.DEFAULT, SpaceType.DEFAULT, 4, ModelState.CREATED, "2021-03-27", "test model", ""); + return new ModelMetadata(KNNEngine.DEFAULT, SpaceType.DEFAULT, 4, ModelState.CREATED, "2021-03-27", "test model", "", ""); } } diff --git a/src/main/java/org/opensearch/knn/common/KNNConstants.java b/src/main/java/org/opensearch/knn/common/KNNConstants.java index 184067c35..e4d59a00f 100644 --- a/src/main/java/org/opensearch/knn/common/KNNConstants.java +++ b/src/main/java/org/opensearch/knn/common/KNNConstants.java @@ -45,6 +45,7 @@ public class KNNConstants { public static final String MODEL_TIMESTAMP = "timestamp"; public static final String MODEL_DESCRIPTION = "description"; public static final String MODEL_ERROR = "error"; + public static final String MODEL_NODE_ASSIGNMENT = "training_node_assignment"; public static final String PARAM_SIZE = "size"; public static final Integer SEARCH_MODEL_MIN_SIZE = 1; public static final Integer SEARCH_MODEL_MAX_SIZE = 1000; diff --git a/src/main/java/org/opensearch/knn/index/IndexUtil.java b/src/main/java/org/opensearch/knn/index/IndexUtil.java index b3a24d2f0..a18c228bc 100644 --- a/src/main/java/org/opensearch/knn/index/IndexUtil.java +++ b/src/main/java/org/opensearch/knn/index/IndexUtil.java @@ -36,12 +36,16 @@ public class IndexUtil { + public static final String MODEL_NODE_ASSIGNMENT_KEY = KNNConstants.MODEL_NODE_ASSIGNMENT; + private static final Version MINIMAL_SUPPORTED_VERSION_FOR_LUCENE_HNSW_FILTER = Version.V_2_4_0; private static final Version MINIMAL_SUPPORTED_VERSION_FOR_IGNORE_UNMAPPED = Version.V_2_11_0; + private static final Version MINIMAL_SUPPORTED_VERSION_FOR_MODEL_NODE_ASSIGNMENT = Version.V_2_12_0; public static final Map minimalRequiredVersionMap = new HashMap() { { put("filter", MINIMAL_SUPPORTED_VERSION_FOR_LUCENE_HNSW_FILTER); put("ignore_unmapped", MINIMAL_SUPPORTED_VERSION_FOR_IGNORE_UNMAPPED); + put(MODEL_NODE_ASSIGNMENT_KEY, MINIMAL_SUPPORTED_VERSION_FOR_MODEL_NODE_ASSIGNMENT); } }; @@ -253,4 +257,12 @@ public static boolean isClusterOnOrAfterMinRequiredVersion(String key) { } return KNNClusterUtil.instance().getClusterMinVersion().onOrAfter(minimalRequiredVersion); } + + public static boolean isVersionOnOrAfterMinRequiredVersion(Version version, String key) { + Version minimalRequiredVersion = minimalRequiredVersionMap.get(key); + if (minimalRequiredVersion == null) { + return false; + } + return version.onOrAfter(minimalRequiredVersion); + } } diff --git a/src/main/java/org/opensearch/knn/indices/ModelDao.java b/src/main/java/org/opensearch/knn/indices/ModelDao.java index eada08b44..1d88c9a00 100644 --- a/src/main/java/org/opensearch/knn/indices/ModelDao.java +++ b/src/main/java/org/opensearch/knn/indices/ModelDao.java @@ -287,6 +287,7 @@ private void putInternal(Model model, ActionListener listener, Do put(KNNConstants.MODEL_TIMESTAMP, modelMetadata.getTimestamp()); put(KNNConstants.MODEL_DESCRIPTION, modelMetadata.getDescription()); put(KNNConstants.MODEL_ERROR, modelMetadata.getError()); + put(KNNConstants.MODEL_NODE_ASSIGNMENT, modelMetadata.getNodeAssignment()); } }; diff --git a/src/main/java/org/opensearch/knn/indices/ModelMetadata.java b/src/main/java/org/opensearch/knn/indices/ModelMetadata.java index 0d0c79bc3..04836f184 100644 --- a/src/main/java/org/opensearch/knn/indices/ModelMetadata.java +++ b/src/main/java/org/opensearch/knn/indices/ModelMetadata.java @@ -11,6 +11,7 @@ package org.opensearch.knn.indices; +import lombok.extern.log4j.Log4j2; import org.apache.commons.lang.builder.EqualsBuilder; import org.apache.commons.lang.builder.HashCodeBuilder; import org.opensearch.core.common.io.stream.StreamInput; @@ -19,6 +20,7 @@ import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.knn.common.KNNConstants; +import org.opensearch.knn.index.IndexUtil; import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.util.KNNEngine; @@ -34,7 +36,9 @@ import static org.opensearch.knn.common.KNNConstants.MODEL_ERROR; import static org.opensearch.knn.common.KNNConstants.MODEL_STATE; import static org.opensearch.knn.common.KNNConstants.MODEL_TIMESTAMP; +import static org.opensearch.knn.common.KNNConstants.MODEL_NODE_ASSIGNMENT; +@Log4j2 public class ModelMetadata implements Writeable, ToXContentObject { private static final String DELIMITER = ","; @@ -46,6 +50,7 @@ public class ModelMetadata implements Writeable, ToXContentObject { private AtomicReference state; final private String timestamp; final private String description; + final private String trainingNodeAssignment; private String error; /** @@ -54,6 +59,7 @@ public class ModelMetadata implements Writeable, ToXContentObject { * @param in Stream input */ public ModelMetadata(StreamInput in) throws IOException { + String tempTrainingNodeAssignment; this.knnEngine = KNNEngine.getEngine(in.readString()); this.spaceType = SpaceType.getSpace(in.readString()); this.dimension = in.readInt(); @@ -64,6 +70,12 @@ public ModelMetadata(StreamInput in) throws IOException { // which is checked in constructor and setters this.description = in.readString(); this.error = in.readString(); + + if (IndexUtil.isVersionOnOrAfterMinRequiredVersion(in.getVersion(), IndexUtil.MODEL_NODE_ASSIGNMENT_KEY)) { + this.trainingNodeAssignment = in.readString(); + } else { + this.trainingNodeAssignment = ""; + } } /** @@ -84,7 +96,8 @@ public ModelMetadata( ModelState modelState, String timestamp, String description, - String error + String error, + String trainingNodeAssignment ) { this.knnEngine = Objects.requireNonNull(knnEngine, "knnEngine must not be null"); this.spaceType = Objects.requireNonNull(spaceType, "spaceType must not be null"); @@ -104,6 +117,7 @@ public ModelMetadata( this.timestamp = Objects.requireNonNull(timestamp, "timestamp must not be null"); this.description = Objects.requireNonNull(description, "description must not be null"); this.error = Objects.requireNonNull(error, "error must not be null"); + this.trainingNodeAssignment = Objects.requireNonNull(trainingNodeAssignment, "node assignment must not be null"); } /** @@ -169,6 +183,15 @@ public String getError() { return error; } + /** + * getter for model's node assignment + * + * @return trainingNodeAssignment + */ + public String getNodeAssignment() { + return trainingNodeAssignment; + } + /** * setter for model's state * @@ -197,7 +220,8 @@ public String toString() { getState().toString(), timestamp, description, - error + error, + trainingNodeAssignment ); } @@ -240,22 +264,36 @@ public int hashCode() { public static ModelMetadata fromString(String modelMetadataString) { String[] modelMetadataArray = modelMetadataString.split(DELIMITER, -1); - if (modelMetadataArray.length != 7) { + // Training node assignment was added as a field in Version 2.12.0 + // Because models can be created on older versions and the cluster can be upgraded after, + // we need to accept model metadata arrays both with and without the training node assignment. + if (modelMetadataArray.length == 7) { + log.debug("Model metadata array does not contain training node assignment. Assuming empty string."); + KNNEngine knnEngine = KNNEngine.getEngine(modelMetadataArray[0]); + SpaceType spaceType = SpaceType.getSpace(modelMetadataArray[1]); + int dimension = Integer.parseInt(modelMetadataArray[2]); + ModelState modelState = ModelState.getModelState(modelMetadataArray[3]); + String timestamp = modelMetadataArray[4]; + String description = modelMetadataArray[5]; + String error = modelMetadataArray[6]; + return new ModelMetadata(knnEngine, spaceType, dimension, modelState, timestamp, description, error, ""); + } else if (modelMetadataArray.length == 8) { + log.debug("Model metadata contains training node assignment"); + KNNEngine knnEngine = KNNEngine.getEngine(modelMetadataArray[0]); + SpaceType spaceType = SpaceType.getSpace(modelMetadataArray[1]); + int dimension = Integer.parseInt(modelMetadataArray[2]); + ModelState modelState = ModelState.getModelState(modelMetadataArray[3]); + String timestamp = modelMetadataArray[4]; + String description = modelMetadataArray[5]; + String error = modelMetadataArray[6]; + String trainingNodeAssignment = modelMetadataArray[7]; + return new ModelMetadata(knnEngine, spaceType, dimension, modelState, timestamp, description, error, trainingNodeAssignment); + } else { throw new IllegalArgumentException( "Illegal format for model metadata. Must be of the form " - + "\",,,,,,\"." + + "\",,,,,,\" or \",,,,,,,\"." ); } - - KNNEngine knnEngine = KNNEngine.getEngine(modelMetadataArray[0]); - SpaceType spaceType = SpaceType.getSpace(modelMetadataArray[1]); - int dimension = Integer.parseInt(modelMetadataArray[2]); - ModelState modelState = ModelState.getModelState(modelMetadataArray[3]); - String timestamp = modelMetadataArray[4]; - String description = modelMetadataArray[5]; - String error = modelMetadataArray[6]; - - return new ModelMetadata(knnEngine, spaceType, dimension, modelState, timestamp, description, error); } private static String objectToString(Object value) { @@ -282,6 +320,11 @@ public static ModelMetadata getMetadataFromSourceMap(final Map m Object timestamp = modelSourceMap.get(KNNConstants.MODEL_TIMESTAMP); Object description = modelSourceMap.get(KNNConstants.MODEL_DESCRIPTION); Object error = modelSourceMap.get(KNNConstants.MODEL_ERROR); + Object trainingNodeAssignment = modelSourceMap.get(KNNConstants.MODEL_NODE_ASSIGNMENT); + + if (trainingNodeAssignment == null) { + trainingNodeAssignment = ""; + } ModelMetadata modelMetadata = new ModelMetadata( KNNEngine.getEngine(objectToString(engine)), @@ -290,7 +333,8 @@ public static ModelMetadata getMetadataFromSourceMap(final Map m ModelState.getModelState(objectToString(state)), objectToString(timestamp), objectToString(description), - objectToString(error) + objectToString(error), + objectToString(trainingNodeAssignment) ); return modelMetadata; } @@ -304,6 +348,9 @@ public void writeTo(StreamOutput out) throws IOException { out.writeString(getTimestamp()); out.writeString(getDescription()); out.writeString(getError()); + if (IndexUtil.isVersionOnOrAfterMinRequiredVersion(out.getVersion(), IndexUtil.MODEL_NODE_ASSIGNMENT_KEY)) { + out.writeString(getNodeAssignment()); + } } @Override @@ -316,6 +363,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field(METHOD_PARAMETER_SPACE_TYPE, getSpaceType().getValue()); builder.field(DIMENSION, getDimension()); builder.field(KNN_ENGINE, getKnnEngine().getName()); + if (IndexUtil.isClusterOnOrAfterMinRequiredVersion(IndexUtil.MODEL_NODE_ASSIGNMENT_KEY)) { + builder.field(MODEL_NODE_ASSIGNMENT, getNodeAssignment()); + } return builder; } } diff --git a/src/main/java/org/opensearch/knn/plugin/KNNPlugin.java b/src/main/java/org/opensearch/knn/plugin/KNNPlugin.java index 53e9f3105..3edff1879 100644 --- a/src/main/java/org/opensearch/knn/plugin/KNNPlugin.java +++ b/src/main/java/org/opensearch/knn/plugin/KNNPlugin.java @@ -75,6 +75,7 @@ import org.opensearch.knn.plugin.transport.UpdateModelMetadataTransportAction; import org.opensearch.knn.plugin.transport.UpdateModelGraveyardAction; import org.opensearch.knn.plugin.transport.UpdateModelGraveyardTransportAction; +import org.opensearch.knn.training.TrainingJobClusterStateListener; import org.opensearch.knn.training.TrainingJobRunner; import org.opensearch.knn.training.VectorReader; import org.opensearch.plugins.ActionPlugin; @@ -197,10 +198,14 @@ public Collection createComponents( ModelDao.OpenSearchKNNModelDao.initialize(client, clusterService, environment.settings()); ModelCache.initialize(ModelDao.OpenSearchKNNModelDao.getInstance(), clusterService); TrainingJobRunner.initialize(threadPool, ModelDao.OpenSearchKNNModelDao.getInstance()); + TrainingJobClusterStateListener.initialize(threadPool, ModelDao.OpenSearchKNNModelDao.getInstance(), clusterService); KNNCircuitBreaker.getInstance().initialize(threadPool, clusterService, client); KNNQueryBuilder.initialize(ModelDao.OpenSearchKNNModelDao.getInstance()); KNNWeight.initialize(ModelDao.OpenSearchKNNModelDao.getInstance()); TrainingModelRequest.initialize(ModelDao.OpenSearchKNNModelDao.getInstance(), clusterService); + + clusterService.addListener(TrainingJobClusterStateListener.getInstance()); + knnStats = new KNNStats(); return ImmutableList.of(knnStats); } diff --git a/src/main/java/org/opensearch/knn/plugin/transport/TrainingModelTransportAction.java b/src/main/java/org/opensearch/knn/plugin/transport/TrainingModelTransportAction.java index 379d8a809..33b420e2c 100644 --- a/src/main/java/org/opensearch/knn/plugin/transport/TrainingModelTransportAction.java +++ b/src/main/java/org/opensearch/knn/plugin/transport/TrainingModelTransportAction.java @@ -26,6 +26,7 @@ import org.opensearch.transport.TransportService; import java.io.IOException; +import java.util.concurrent.ExecutionException; /** * Transport action that trains a model and serializes it to model system index @@ -66,7 +67,8 @@ protected void doExecute(Task task, TrainingModelRequest request, ActionListener trainingDataEntryContext, modelAnonymousEntryContext, request.getDimension(), - request.getDescription() + request.getDescription(), + clusterService.localNode().getEphemeralId() ); KNNCounter.TRAINING_REQUESTS.increment(); @@ -84,7 +86,7 @@ protected void doExecute(Task task, TrainingModelRequest request, ActionListener wrappedListener::onFailure ) ); - } catch (IOException e) { + } catch (IOException | ExecutionException | InterruptedException e) { wrappedListener.onFailure(e); } } diff --git a/src/main/java/org/opensearch/knn/training/TrainingJob.java b/src/main/java/org/opensearch/knn/training/TrainingJob.java index 27a1c6025..8a2af4319 100644 --- a/src/main/java/org/opensearch/knn/training/TrainingJob.java +++ b/src/main/java/org/opensearch/knn/training/TrainingJob.java @@ -65,7 +65,8 @@ public TrainingJob( NativeMemoryEntryContext.TrainingDataEntryContext trainingDataEntryContext, NativeMemoryEntryContext.AnonymousEntryContext modelAnonymousEntryContext, int dimension, - String description + String description, + String nodeAssignment ) { // Generate random base64 string if one is not provided this.modelId = StringUtils.isNotBlank(modelId) ? modelId : UUIDs.randomBase64UUID(); @@ -81,7 +82,8 @@ public TrainingJob( ModelState.TRAINING, ZonedDateTime.now(ZoneOffset.UTC).toString(), description, - "" + "", + nodeAssignment ), null, this.modelId diff --git a/src/main/java/org/opensearch/knn/training/TrainingJobClusterStateListener.java b/src/main/java/org/opensearch/knn/training/TrainingJobClusterStateListener.java new file mode 100644 index 000000000..45d2197e8 --- /dev/null +++ b/src/main/java/org/opensearch/knn/training/TrainingJobClusterStateListener.java @@ -0,0 +1,176 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.knn.training; + +import lombok.extern.log4j.Log4j2; +import org.opensearch.action.index.IndexResponse; +import org.opensearch.action.search.SearchRequest; +import org.opensearch.action.search.SearchResponse; +import org.opensearch.cluster.ClusterChangedEvent; +import org.opensearch.cluster.ClusterStateListener; +import org.opensearch.cluster.node.DiscoveryNode; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.unit.TimeValue; +import org.opensearch.core.action.ActionListener; +import org.opensearch.knn.indices.Model; +import org.opensearch.knn.indices.ModelDao; +import org.opensearch.knn.indices.ModelMetadata; +import org.opensearch.knn.indices.ModelState; +import org.opensearch.search.SearchHit; +import org.opensearch.threadpool.ThreadPool; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutionException; + +/** + * TrainingJobClusterStateListener is a ClusterStateListener that is used to update models that are still training when a node leaves or the cluster crashes. + * This class also sets a flag in TrainingJobRunner to block serialization when a node rejoins a cluster. + */ +@Log4j2 +public class TrainingJobClusterStateListener implements ClusterStateListener { + private static TrainingJobClusterStateListener INSTANCE; + + private static ModelDao modelDao; + private static ThreadPool threadPool; + private static ClusterService clusterService; + private String oldClusterManagerNodeId = ""; + private String currentClusterManagerNodeId = ""; + private boolean clusterManagerNodeRemoved = false; + + /** + * Get singleton instance of TrainingJobRunner + * + * @return singleton instance of TrainingJobRunner + */ + public static synchronized TrainingJobClusterStateListener getInstance() { + if (INSTANCE == null) { + INSTANCE = new TrainingJobClusterStateListener(); + } + return INSTANCE; + } + + /** + * Initializes static components. + * + * @param threadPool threadPool to use to schedule update of models + * @param modelDao modelDao used to get modelIds + * @param clusterService clusterService used to add a listener + */ + public static synchronized void initialize(ThreadPool threadPool, ModelDao modelDao, ClusterService clusterService) { + TrainingJobClusterStateListener.threadPool = threadPool; + TrainingJobClusterStateListener.modelDao = modelDao; + TrainingJobClusterStateListener.clusterService = clusterService; + } + + /** + * This method is called whenever the cluster state changes. + * It is used to update models that are still training when a node leaves or the cluster crashes. + * It is also used to cancel training jobs when a node rejoins the cluster. + * @param event the event that changed the cluster change + */ + @Override + public void clusterChanged(ClusterChangedEvent event) { + if (event.localNodeClusterManager()) { + if (event.isNewCluster()) { + // When the cluster is first created, the cluster manager will update models that are still marked as training. + threadPool.schedule(() -> { + try { + updateModelsNewCluster(); + } catch (IOException | InterruptedException | ExecutionException e) { + throw new RuntimeException(e); + } + }, TimeValue.timeValueSeconds(1), ThreadPool.Names.GENERIC); + } else if (event.nodesRemoved()) { + List removedNodes = event.nodesDelta().removedNodes(); + threadPool.schedule(() -> { + try { + updateModelsNodesRemoved(removedNodes); + } catch (IOException | InterruptedException | ExecutionException e) { + throw new RuntimeException(e); + } + }, TimeValue.timeValueSeconds(0), ThreadPool.Names.GENERIC); + } + } + } + + protected void updateModelsNewCluster() throws IOException, InterruptedException, ExecutionException { + if (modelDao.isCreated()) { + List modelIds = searchModelIds(); + for (String modelId : modelIds) { + Model model = modelDao.get(modelId); + ModelMetadata modelMetadata = model.getModelMetadata(); + if (modelMetadata.getState().equals(ModelState.TRAINING)) { + updateModelStateAsFailed(model, "Training failed to complete as cluster crashed"); + } + } + } + } + + protected void updateModelsNodesRemoved(List removedNodes) throws IOException, InterruptedException, ExecutionException { + if (modelDao.isCreated()) { + List modelIds = searchModelIds(); + for (DiscoveryNode removedNode : removedNodes) { + for (String modelId : modelIds) { + Model model = modelDao.get(modelId); + ModelMetadata modelMetadata = model.getModelMetadata(); + if (modelMetadata.getNodeAssignment().equals(removedNode.getEphemeralId()) + && modelMetadata.getState().equals(ModelState.TRAINING)) { + updateModelStateAsFailed(model, "Training failed to complete as node dropped"); + } + } + } + } + } + + private List searchModelIds() throws IOException, InterruptedException { + List modelIds = new ArrayList(); + CountDownLatch latch = new CountDownLatch(1); + modelDao.search(new SearchRequest(), new ActionListener() { + @Override + public void onResponse(SearchResponse searchResponse) { + try { + for (SearchHit searchHit : searchResponse.getHits().getHits()) { + modelIds.add(searchHit.getId()); + } + } finally { + latch.countDown(); + } + } + + @Override + public void onFailure(Exception e) { + latch.countDown(); + } + }); + latch.await(); + return modelIds; + } + + private void updateModelStateAsFailed(Model model, String msg) throws IOException { + model.getModelMetadata().setState(ModelState.FAILED); + model.getModelMetadata().setError(msg); + modelDao.update(model, new ActionListener() { + @Override + public void onResponse(IndexResponse indexResponse) { + log.info("Model {} marked as {}", model.getModelID(), model.getModelMetadata().getState()); + } + + @Override + public void onFailure(Exception e) { + log.error("Failed to update model state", e); + } + }); + } +} diff --git a/src/main/java/org/opensearch/knn/training/TrainingJobRunner.java b/src/main/java/org/opensearch/knn/training/TrainingJobRunner.java index a999735a4..8884f8102 100644 --- a/src/main/java/org/opensearch/knn/training/TrainingJobRunner.java +++ b/src/main/java/org/opensearch/knn/training/TrainingJobRunner.java @@ -16,6 +16,7 @@ import org.opensearch.core.action.ActionListener; import org.opensearch.action.index.IndexResponse; import org.opensearch.common.ValidationException; +import org.opensearch.knn.indices.Model; import org.opensearch.knn.indices.ModelDao; import org.opensearch.knn.indices.ModelMetadata; import org.opensearch.knn.indices.ModelState; @@ -23,6 +24,7 @@ import org.opensearch.threadpool.ThreadPool; import java.io.IOException; +import java.util.concurrent.ExecutionException; import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.Semaphore; import java.util.concurrent.atomic.AtomicInteger; @@ -79,7 +81,8 @@ public static void initialize(ThreadPool threadPool, ModelDao modelDao) { * @param trainingJob training job to be executed * @param listener listener to handle final model serialization response (or exception) */ - public void execute(TrainingJob trainingJob, ActionListener listener) throws IOException { + public void execute(TrainingJob trainingJob, ActionListener listener) throws IOException, ExecutionException, + InterruptedException { // If the semaphore cannot be acquired, the node is unable to execute this job. This allows us to limit // the number of training jobs that enter this function. Although the training threadpool size will also prevent // this, we want to prevent this before we perform any serialization. @@ -106,10 +109,10 @@ public void execute(TrainingJob trainingJob, ActionListener liste logger.error("Unable to initialize model serialization: " + exception.getMessage()); listener.onFailure(exception); }), false); - } catch (IOException ioe) { + } catch (IOException | ExecutionException | InterruptedException e) { jobCount.decrementAndGet(); semaphore.release(); - throw ioe; + throw e; } } @@ -130,7 +133,7 @@ private void train(TrainingJob trainingJob) { try { trainingJob.run(); serializeModel(trainingJob, loggingListener, true); - } catch (IOException e) { + } catch (IOException | ExecutionException | InterruptedException e) { logger.error("Unable to serialize model \"" + trainingJob.getModelId() + "\": " + e.getMessage()); KNNCounter.TRAINING_ERRORS.increment(); } catch (Exception e) { @@ -150,8 +153,8 @@ private void train(TrainingJob trainingJob) { try { serializeModel(trainingJob, loggingListener, true); - } catch (IOException ioe) { - logger.error("Unable to serialize the failure for model \"" + trainingJob.getModelId() + "\": " + ioe); + } catch (IOException | ExecutionException | InterruptedException e) { + logger.error("Unable to serialize the failure for model \"{}\": ", trainingJob.getModelId(), e); } finally { jobCount.decrementAndGet(); semaphore.release(); @@ -160,9 +163,15 @@ private void train(TrainingJob trainingJob) { } } - private void serializeModel(TrainingJob trainingJob, ActionListener listener, boolean update) throws IOException { + private void serializeModel(TrainingJob trainingJob, ActionListener listener, boolean update) throws IOException, + ExecutionException, InterruptedException { if (update) { - modelDao.update(trainingJob.getModel(), listener); + Model model = modelDao.get(trainingJob.getModelId()); + if (model.getModelMetadata().getState().equals(ModelState.TRAINING)) { + modelDao.update(trainingJob.getModel(), listener); + } else { + logger.info("Model state is {}. Skipping serialization of trained data", model.getModelMetadata().getState()); + } } else { modelDao.put(trainingJob.getModel(), listener); } diff --git a/src/test/java/org/opensearch/knn/index/KNNCreateIndexFromModelTests.java b/src/test/java/org/opensearch/knn/index/KNNCreateIndexFromModelTests.java index c1c52e63a..8fdc55766 100644 --- a/src/test/java/org/opensearch/knn/index/KNNCreateIndexFromModelTests.java +++ b/src/test/java/org/opensearch/knn/index/KNNCreateIndexFromModelTests.java @@ -61,7 +61,8 @@ public void testCreateIndexFromModel() throws IOException, InterruptedException ModelState.CREATED, ZonedDateTime.now(ZoneOffset.UTC).toString(), "", - "" + "", + "test-node" ); Model model = new Model(modelMetadata, modelBlob, modelId); diff --git a/src/test/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumerTests.java b/src/test/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumerTests.java index 6af83de87..242aeaa17 100644 --- a/src/test/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumerTests.java +++ b/src/test/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumerTests.java @@ -341,7 +341,7 @@ public void testAddKNNBinaryField_fromModel_faiss() throws IOException, Executio byte[] modelBytes = JNIService.trainIndex(parameters, dimension, trainingPtr, knnEngine.getName()); Model model = new Model( - new ModelMetadata(knnEngine, spaceType, dimension, ModelState.CREATED, "timestamp", "Empty description", ""), + new ModelMetadata(knnEngine, spaceType, dimension, ModelState.CREATED, "timestamp", "Empty description", "", ""), modelBytes, modelId ); diff --git a/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestCase.java b/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestCase.java index 6c7631216..eb9b4fa2d 100644 --- a/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestCase.java +++ b/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestCase.java @@ -204,6 +204,7 @@ public void testBuildFromModelTemplate(Codec codec) throws IOException, Executio ModelState.CREATED, ZonedDateTime.now(ZoneOffset.UTC).toString(), "", + "", "" ); diff --git a/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java b/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java index 1f3598781..2de98d803 100644 --- a/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java +++ b/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java @@ -163,6 +163,7 @@ public void testBuilder_build_fromModel() { ModelState.CREATED, ZonedDateTime.now(ZoneOffset.UTC).toString(), "", + "", "" ); builder.modelId.setValue(modelId); @@ -689,6 +690,7 @@ public void testKNNVectorFieldMapper_merge_fromModel() throws IOException { ModelState.CREATED, ZonedDateTime.now(ZoneOffset.UTC).toString(), "", + "", "" ); when(mockModelDao.getMetadata(modelId)).thenReturn(mockModelMetadata); diff --git a/src/test/java/org/opensearch/knn/indices/ModelCacheTests.java b/src/test/java/org/opensearch/knn/indices/ModelCacheTests.java index fb810d969..3146d898e 100644 --- a/src/test/java/org/opensearch/knn/indices/ModelCacheTests.java +++ b/src/test/java/org/opensearch/knn/indices/ModelCacheTests.java @@ -42,6 +42,7 @@ public void testGet_normal() throws ExecutionException, InterruptedException { ModelState.CREATED, ZonedDateTime.now(ZoneOffset.UTC).toString(), "", + "", "" ), "hello".getBytes(), @@ -77,6 +78,7 @@ public void testGet_modelDoesNotFitInCache() throws ExecutionException, Interrup ModelState.CREATED, ZonedDateTime.now(ZoneOffset.UTC).toString(), "", + "", "" ), new byte[BYTES_PER_KILOBYTES + 1], @@ -133,6 +135,7 @@ public void testGetTotalWeight() throws ExecutionException, InterruptedException ModelState.CREATED, ZonedDateTime.now(ZoneOffset.UTC).toString(), "", + "", "" ), new byte[size1], @@ -147,6 +150,7 @@ public void testGetTotalWeight() throws ExecutionException, InterruptedException ModelState.CREATED, ZonedDateTime.now(ZoneOffset.UTC).toString(), "", + "", "" ), new byte[size2], @@ -189,6 +193,7 @@ public void testRemove_normal() throws ExecutionException, InterruptedException ModelState.CREATED, ZonedDateTime.now(ZoneOffset.UTC).toString(), "", + "", "" ), new byte[size1], @@ -203,6 +208,7 @@ public void testRemove_normal() throws ExecutionException, InterruptedException ModelState.CREATED, ZonedDateTime.now(ZoneOffset.UTC).toString(), "", + "", "" ), new byte[size2], @@ -250,6 +256,7 @@ public void testRebuild_normal() throws ExecutionException, InterruptedException ModelState.CREATED, ZonedDateTime.now(ZoneOffset.UTC).toString(), "", + "", "" ), "hello".getBytes(), @@ -294,6 +301,7 @@ public void testRebuild_afterSettingUpdate() throws ExecutionException, Interrup ModelState.CREATED, ZonedDateTime.now(ZoneOffset.UTC).toString(), "", + "", "" ), new byte[modelSize], @@ -361,6 +369,7 @@ public void testContains() throws ExecutionException, InterruptedException { ModelState.CREATED, ZonedDateTime.now(ZoneOffset.UTC).toString(), "", + "", "" ), new byte[modelSize1], @@ -401,6 +410,7 @@ public void testRemoveAll() throws ExecutionException, InterruptedException { ModelState.CREATED, ZonedDateTime.now(ZoneOffset.UTC).toString(), "", + "", "" ), new byte[modelSize1], @@ -417,6 +427,7 @@ public void testRemoveAll() throws ExecutionException, InterruptedException { ModelState.CREATED, ZonedDateTime.now(ZoneOffset.UTC).toString(), "", + "", "" ), new byte[modelSize2], @@ -461,6 +472,7 @@ public void testModelCacheEvictionDueToSize() throws ExecutionException, Interru ModelState.CREATED, ZonedDateTime.now(ZoneOffset.UTC).toString(), "", + "", "" ), new byte[BYTES_PER_KILOBYTES * 2], diff --git a/src/test/java/org/opensearch/knn/indices/ModelDaoTests.java b/src/test/java/org/opensearch/knn/indices/ModelDaoTests.java index cb6628c16..1297dc184 100644 --- a/src/test/java/org/opensearch/knn/indices/ModelDaoTests.java +++ b/src/test/java/org/opensearch/knn/indices/ModelDaoTests.java @@ -136,6 +136,7 @@ public void testModelIndexHealth() throws InterruptedException, ExecutionExcepti ModelState.CREATED, ZonedDateTime.now(ZoneOffset.UTC).toString(), "", + "", "" ), modelBlob, @@ -154,6 +155,7 @@ public void testModelIndexHealth() throws InterruptedException, ExecutionExcepti ModelState.FAILED, ZonedDateTime.now(ZoneOffset.UTC).toString(), "", + "", "" ), modelBlob, @@ -180,6 +182,7 @@ public void testPut_withId() throws InterruptedException, IOException { ModelState.CREATED, ZonedDateTime.now(ZoneOffset.UTC).toString(), "", + "", "" ), modelBlob, @@ -239,6 +242,7 @@ public void testPut_withoutModel() throws InterruptedException, IOException { ModelState.TRAINING, ZonedDateTime.now(ZoneOffset.UTC).toString(), "", + "", "" ), modelBlob, @@ -299,6 +303,7 @@ public void testPut_invalid_badState() { ModelState.TRAINING, ZonedDateTime.now(ZoneOffset.UTC).toString(), "", + "", "" ), modelBlob, @@ -334,6 +339,7 @@ public void testUpdate() throws IOException, InterruptedException { ModelState.TRAINING, ZonedDateTime.now(ZoneOffset.UTC).toString(), "", + "", "" ), null, @@ -371,6 +377,7 @@ public void testUpdate() throws IOException, InterruptedException { ModelState.CREATED, ZonedDateTime.now(ZoneOffset.UTC).toString(), "", + "", "" ), modelBlob, @@ -420,6 +427,7 @@ public void testGet() throws IOException, InterruptedException, ExecutionExcepti ModelState.CREATED, ZonedDateTime.now(ZoneOffset.UTC).toString(), "", + "", "" ), modelBlob, @@ -437,6 +445,7 @@ public void testGet() throws IOException, InterruptedException, ExecutionExcepti ModelState.TRAINING, ZonedDateTime.now(ZoneOffset.UTC).toString(), "", + "", "" ), null, @@ -472,6 +481,7 @@ public void testGetMetadata() throws IOException, InterruptedException { ModelState.CREATED, ZonedDateTime.now(ZoneOffset.UTC).toString(), "", + "", "" ); @@ -481,7 +491,6 @@ public void testGetMetadata() throws IOException, InterruptedException { final CountDownLatch inProgressLatch1 = new CountDownLatch(1); ActionListener docCreationListener = ActionListener.wrap(response -> { assertEquals(modelId, response.getId()); - ModelMetadata modelMetadata1 = modelDao.getMetadata(modelId); assertEquals(modelMetadata, modelMetadata1); @@ -548,6 +557,7 @@ public void testDelete() throws IOException, InterruptedException { ModelState.TRAINING, ZonedDateTime.now(ZoneOffset.UTC).toString(), "", + "", "" ), modelBlob, @@ -580,6 +590,7 @@ public void testDelete() throws IOException, InterruptedException { ModelState.CREATED, ZonedDateTime.now(ZoneOffset.UTC).toString(), "", + "", "" ), modelBlob, @@ -646,6 +657,7 @@ public void testDeleteModelInTrainingWithStepListeners() throws IOException, Exe ModelState.TRAINING, ZonedDateTime.now(ZoneOffset.UTC).toString(), "", + "", "" ), modelBlob, @@ -686,6 +698,7 @@ public void testDeleteWithStepListeners() throws IOException, InterruptedExcepti ModelState.CREATED, ZonedDateTime.now(ZoneOffset.UTC).toString(), "", + "", "" ), modelBlob, diff --git a/src/test/java/org/opensearch/knn/indices/ModelMetadataTests.java b/src/test/java/org/opensearch/knn/indices/ModelMetadataTests.java index a2e5c6bbe..219710308 100644 --- a/src/test/java/org/opensearch/knn/indices/ModelMetadataTests.java +++ b/src/test/java/org/opensearch/knn/indices/ModelMetadataTests.java @@ -38,6 +38,7 @@ public void testStreams() throws IOException { ModelState.CREATED, ZonedDateTime.now(ZoneOffset.UTC).toString(), "", + "", "" ); @@ -58,6 +59,7 @@ public void testGetKnnEngine() { ModelState.CREATED, ZonedDateTime.now(ZoneOffset.UTC).toString(), "", + "", "" ); @@ -73,6 +75,7 @@ public void testGetSpaceType() { ModelState.CREATED, ZonedDateTime.now(ZoneOffset.UTC).toString(), "", + "", "" ); @@ -88,6 +91,7 @@ public void testGetDimension() { ModelState.CREATED, ZonedDateTime.now(ZoneOffset.UTC).toString(), "", + "", "" ); @@ -103,6 +107,7 @@ public void testGetState() { modelState, ZonedDateTime.now(ZoneOffset.UTC).toString(), "", + "", "" ); @@ -111,7 +116,7 @@ public void testGetState() { public void testGetTimestamp() { String timeValue = ZonedDateTime.now(ZoneOffset.UTC).toString(); - ModelMetadata modelMetadata = new ModelMetadata(KNNEngine.DEFAULT, SpaceType.L2, 12, ModelState.CREATED, timeValue, "", ""); + ModelMetadata modelMetadata = new ModelMetadata(KNNEngine.DEFAULT, SpaceType.L2, 12, ModelState.CREATED, timeValue, "", "", ""); assertEquals(timeValue, modelMetadata.getTimestamp()); } @@ -125,6 +130,7 @@ public void testDescription() { ModelState.CREATED, ZonedDateTime.now(ZoneOffset.UTC).toString(), description, + "", "" ); @@ -140,7 +146,8 @@ public void testGetError() { ModelState.CREATED, ZonedDateTime.now(ZoneOffset.UTC).toString(), "", - error + error, + "" ); assertEquals(error, modelMetadata.getError()); @@ -155,6 +162,7 @@ public void testSetState() { modelState, ZonedDateTime.now(ZoneOffset.UTC).toString(), "", + "", "" ); @@ -174,7 +182,8 @@ public void testSetError() { ModelState.TRAINING, ZonedDateTime.now(ZoneOffset.UTC).toString(), "", - error + error, + "" ); assertEquals(error, modelMetadata.getError()); @@ -192,6 +201,7 @@ public void testToString() { String timestamp = ZonedDateTime.now(ZoneOffset.UTC).toString(); String description = "test-description"; String error = "test-error"; + String nodeAssignment = ""; String expected = knnEngine.getName() + "," @@ -205,9 +215,20 @@ public void testToString() { + "," + description + "," - + error; + + error + + "," + + nodeAssignment; - ModelMetadata modelMetadata = new ModelMetadata(knnEngine, spaceType, dimension, modelState, timestamp, description, error); + ModelMetadata modelMetadata = new ModelMetadata( + knnEngine, + spaceType, + dimension, + modelState, + timestamp, + description, + error, + nodeAssignment + ); assertEquals(expected, modelMetadata.toString()); } @@ -217,14 +238,14 @@ public void testEquals() { String time1 = ZonedDateTime.now(ZoneOffset.UTC).toString(); String time2 = ZonedDateTime.of(2021, 9, 30, 12, 20, 45, 1, ZoneId.systemDefault()).toString(); - ModelMetadata modelMetadata1 = new ModelMetadata(KNNEngine.FAISS, SpaceType.L2, 128, ModelState.CREATED, time1, "", ""); - ModelMetadata modelMetadata2 = new ModelMetadata(KNNEngine.FAISS, SpaceType.L2, 128, ModelState.CREATED, time1, "", ""); + ModelMetadata modelMetadata1 = new ModelMetadata(KNNEngine.FAISS, SpaceType.L2, 128, ModelState.CREATED, time1, "", "", ""); + ModelMetadata modelMetadata2 = new ModelMetadata(KNNEngine.FAISS, SpaceType.L2, 128, ModelState.CREATED, time1, "", "", ""); - ModelMetadata modelMetadata3 = new ModelMetadata(KNNEngine.NMSLIB, SpaceType.L2, 128, ModelState.CREATED, time1, "", ""); - ModelMetadata modelMetadata4 = new ModelMetadata(KNNEngine.FAISS, SpaceType.L1, 128, ModelState.CREATED, time1, "", ""); - ModelMetadata modelMetadata5 = new ModelMetadata(KNNEngine.FAISS, SpaceType.L2, 129, ModelState.CREATED, time1, "", ""); - ModelMetadata modelMetadata6 = new ModelMetadata(KNNEngine.FAISS, SpaceType.L2, 128, ModelState.TRAINING, time1, "", ""); - ModelMetadata modelMetadata7 = new ModelMetadata(KNNEngine.FAISS, SpaceType.L2, 128, ModelState.CREATED, time2, "", ""); + ModelMetadata modelMetadata3 = new ModelMetadata(KNNEngine.NMSLIB, SpaceType.L2, 128, ModelState.CREATED, time1, "", "", ""); + ModelMetadata modelMetadata4 = new ModelMetadata(KNNEngine.FAISS, SpaceType.L1, 128, ModelState.CREATED, time1, "", "", ""); + ModelMetadata modelMetadata5 = new ModelMetadata(KNNEngine.FAISS, SpaceType.L2, 129, ModelState.CREATED, time1, "", "", ""); + ModelMetadata modelMetadata6 = new ModelMetadata(KNNEngine.FAISS, SpaceType.L2, 128, ModelState.TRAINING, time1, "", "", ""); + ModelMetadata modelMetadata7 = new ModelMetadata(KNNEngine.FAISS, SpaceType.L2, 128, ModelState.CREATED, time2, "", "", ""); ModelMetadata modelMetadata8 = new ModelMetadata( KNNEngine.FAISS, SpaceType.L2, @@ -232,9 +253,19 @@ public void testEquals() { ModelState.CREATED, time1, "diff descript", + "", + "" + ); + ModelMetadata modelMetadata9 = new ModelMetadata( + KNNEngine.FAISS, + SpaceType.L2, + 128, + ModelState.CREATED, + time1, + "", + "diff error", "" ); - ModelMetadata modelMetadata9 = new ModelMetadata(KNNEngine.FAISS, SpaceType.L2, 128, ModelState.CREATED, time1, "", "diff error"); assertEquals(modelMetadata1, modelMetadata1); assertEquals(modelMetadata1, modelMetadata2); @@ -254,14 +285,14 @@ public void testHashCode() { String time1 = ZonedDateTime.now(ZoneOffset.UTC).toString(); String time2 = ZonedDateTime.of(2021, 9, 30, 12, 20, 45, 1, ZoneId.systemDefault()).toString(); - ModelMetadata modelMetadata1 = new ModelMetadata(KNNEngine.FAISS, SpaceType.L2, 128, ModelState.CREATED, time1, "", ""); - ModelMetadata modelMetadata2 = new ModelMetadata(KNNEngine.FAISS, SpaceType.L2, 128, ModelState.CREATED, time1, "", ""); + ModelMetadata modelMetadata1 = new ModelMetadata(KNNEngine.FAISS, SpaceType.L2, 128, ModelState.CREATED, time1, "", "", ""); + ModelMetadata modelMetadata2 = new ModelMetadata(KNNEngine.FAISS, SpaceType.L2, 128, ModelState.CREATED, time1, "", "", ""); - ModelMetadata modelMetadata3 = new ModelMetadata(KNNEngine.NMSLIB, SpaceType.L2, 128, ModelState.CREATED, time1, "", ""); - ModelMetadata modelMetadata4 = new ModelMetadata(KNNEngine.FAISS, SpaceType.L1, 128, ModelState.CREATED, time1, "", ""); - ModelMetadata modelMetadata5 = new ModelMetadata(KNNEngine.FAISS, SpaceType.L2, 129, ModelState.CREATED, time1, "", ""); - ModelMetadata modelMetadata6 = new ModelMetadata(KNNEngine.FAISS, SpaceType.L2, 128, ModelState.TRAINING, time1, "", ""); - ModelMetadata modelMetadata7 = new ModelMetadata(KNNEngine.FAISS, SpaceType.L2, 128, ModelState.CREATED, time2, "", ""); + ModelMetadata modelMetadata3 = new ModelMetadata(KNNEngine.NMSLIB, SpaceType.L2, 128, ModelState.CREATED, time1, "", "", ""); + ModelMetadata modelMetadata4 = new ModelMetadata(KNNEngine.FAISS, SpaceType.L1, 128, ModelState.CREATED, time1, "", "", ""); + ModelMetadata modelMetadata5 = new ModelMetadata(KNNEngine.FAISS, SpaceType.L2, 129, ModelState.CREATED, time1, "", "", ""); + ModelMetadata modelMetadata6 = new ModelMetadata(KNNEngine.FAISS, SpaceType.L2, 128, ModelState.TRAINING, time1, "", "", ""); + ModelMetadata modelMetadata7 = new ModelMetadata(KNNEngine.FAISS, SpaceType.L2, 128, ModelState.CREATED, time2, "", "", ""); ModelMetadata modelMetadata8 = new ModelMetadata( KNNEngine.FAISS, SpaceType.L2, @@ -269,9 +300,19 @@ public void testHashCode() { ModelState.CREATED, time1, "diff descript", + "", + "" + ); + ModelMetadata modelMetadata9 = new ModelMetadata( + KNNEngine.FAISS, + SpaceType.L2, + 128, + ModelState.CREATED, + time1, + "", + "diff error", "" ); - ModelMetadata modelMetadata9 = new ModelMetadata(KNNEngine.FAISS, SpaceType.L2, 128, ModelState.CREATED, time1, "", "diff error"); assertEquals(modelMetadata1.hashCode(), modelMetadata1.hashCode()); assertEquals(modelMetadata1.hashCode(), modelMetadata2.hashCode()); @@ -294,8 +335,25 @@ public void testFromString() { String timestamp = ZonedDateTime.now(ZoneOffset.UTC).toString(); String description = "test-description"; String error = "test-error"; + String nodeAssignment = "test-node"; String stringRep1 = knnEngine.getName() + + "," + + spaceType.getValue() + + "," + + dimension + + "," + + modelState.getName() + + "," + + timestamp + + "," + + description + + "," + + error + + "," + + nodeAssignment; + + String stringRep2 = knnEngine.getName() + "," + spaceType.getValue() + "," @@ -309,10 +367,24 @@ public void testFromString() { + "," + error; - ModelMetadata expected = new ModelMetadata(knnEngine, spaceType, dimension, modelState, timestamp, description, error); + ModelMetadata expected1 = new ModelMetadata( + knnEngine, + spaceType, + dimension, + modelState, + timestamp, + description, + error, + nodeAssignment + ); + + ModelMetadata expected2 = new ModelMetadata(knnEngine, spaceType, dimension, modelState, timestamp, description, error, ""); + ModelMetadata fromString1 = ModelMetadata.fromString(stringRep1); + ModelMetadata fromString2 = ModelMetadata.fromString(stringRep2); - assertEquals(expected, fromString1); + assertEquals(expected1, fromString1); + assertEquals(expected2, fromString2); expectThrows(IllegalArgumentException.class, () -> ModelMetadata.fromString("invalid")); } @@ -325,8 +397,19 @@ public void testFromResponseMap() { String timestamp = ZonedDateTime.now(ZoneOffset.UTC).toString(); String description = "test-description"; String error = "test-error"; + String nodeAssignment = "test-node"; - ModelMetadata expected = new ModelMetadata(knnEngine, spaceType, dimension, modelState, timestamp, description, error); + ModelMetadata expected = new ModelMetadata( + knnEngine, + spaceType, + dimension, + modelState, + timestamp, + description, + error, + nodeAssignment + ); + ModelMetadata expected2 = new ModelMetadata(knnEngine, spaceType, dimension, modelState, timestamp, description, error, ""); Map metadataAsMap = new HashMap<>(); metadataAsMap.put(KNNConstants.KNN_ENGINE, knnEngine.getName()); metadataAsMap.put(KNNConstants.METHOD_PARAMETER_SPACE_TYPE, spaceType.getValue()); @@ -335,8 +418,13 @@ public void testFromResponseMap() { metadataAsMap.put(KNNConstants.MODEL_TIMESTAMP, timestamp); metadataAsMap.put(KNNConstants.MODEL_DESCRIPTION, description); metadataAsMap.put(KNNConstants.MODEL_ERROR, error); + metadataAsMap.put(KNNConstants.MODEL_NODE_ASSIGNMENT, nodeAssignment); ModelMetadata fromMap = ModelMetadata.getMetadataFromSourceMap(metadataAsMap); assertEquals(expected, fromMap); + + metadataAsMap.put(KNNConstants.MODEL_NODE_ASSIGNMENT, null); + assertEquals(expected2, fromMap); + } } diff --git a/src/test/java/org/opensearch/knn/indices/ModelStateTests.java b/src/test/java/org/opensearch/knn/indices/ModelStateTests.java index 1cc8a0e82..1527de539 100644 --- a/src/test/java/org/opensearch/knn/indices/ModelStateTests.java +++ b/src/test/java/org/opensearch/knn/indices/ModelStateTests.java @@ -31,5 +31,8 @@ public void testStreams() throws IOException { public void testGetModelState() { assertEquals(ModelState.CREATED, ModelState.getModelState(ModelState.CREATED.getName())); + assertEquals(ModelState.TRAINING, ModelState.getModelState(ModelState.TRAINING.getName())); + assertEquals(ModelState.FAILED, ModelState.getModelState(ModelState.FAILED.getName())); + expectThrows(IllegalArgumentException.class, () -> ModelState.getModelState("throw-exception")); } } diff --git a/src/test/java/org/opensearch/knn/indices/ModelTests.java b/src/test/java/org/opensearch/knn/indices/ModelTests.java index fd3173431..c015e8d62 100644 --- a/src/test/java/org/opensearch/knn/indices/ModelTests.java +++ b/src/test/java/org/opensearch/knn/indices/ModelTests.java @@ -38,6 +38,7 @@ public void testInvalidConstructor() { ModelState.FAILED, ZonedDateTime.now(ZoneOffset.UTC).toString(), "", + "", "" ), null, @@ -57,6 +58,7 @@ public void testInvalidDimension() { ModelState.CREATED, ZonedDateTime.now(ZoneOffset.UTC).toString(), "", + "", "" ), new byte[16], @@ -73,6 +75,7 @@ public void testInvalidDimension() { ModelState.CREATED, ZonedDateTime.now(ZoneOffset.UTC).toString(), "", + "", "" ), new byte[16], @@ -89,6 +92,7 @@ public void testInvalidDimension() { ModelState.CREATED, ZonedDateTime.now(ZoneOffset.UTC).toString(), "", + "", "" ), new byte[16], @@ -106,6 +110,7 @@ public void testGetModelMetadata() { ModelState.CREATED, ZonedDateTime.now(ZoneOffset.UTC).toString(), "", + "", "" ); Model model = new Model(modelMetadata, new byte[16], "test-model"); @@ -122,6 +127,7 @@ public void testGetModelBlob() { ModelState.CREATED, ZonedDateTime.now(ZoneOffset.UTC).toString(), "", + "", "" ), modelBlob, @@ -140,6 +146,7 @@ public void testGetLength() { ModelState.CREATED, ZonedDateTime.now(ZoneOffset.UTC).toString(), "", + "", "" ), new byte[size], @@ -155,6 +162,7 @@ public void testGetLength() { ModelState.TRAINING, ZonedDateTime.now(ZoneOffset.UTC).toString(), "", + "", "" ), null, @@ -166,7 +174,16 @@ public void testGetLength() { public void testSetModelBlob() { byte[] blob1 = "Hello blob 1".getBytes(); Model model = new Model( - new ModelMetadata(KNNEngine.DEFAULT, SpaceType.L1, 2, ModelState.CREATED, ZonedDateTime.now(ZoneOffset.UTC).toString(), "", ""), + new ModelMetadata( + KNNEngine.DEFAULT, + SpaceType.L1, + 2, + ModelState.CREATED, + ZonedDateTime.now(ZoneOffset.UTC).toString(), + "", + "", + "" + ), blob1, "test-model" ); @@ -182,17 +199,17 @@ public void testEquals() { String time = ZonedDateTime.now(ZoneOffset.UTC).toString(); Model model1 = new Model( - new ModelMetadata(KNNEngine.DEFAULT, SpaceType.L1, 2, ModelState.CREATED, time, "", ""), + new ModelMetadata(KNNEngine.DEFAULT, SpaceType.L1, 2, ModelState.CREATED, time, "", "", ""), new byte[16], "test-model-1" ); Model model2 = new Model( - new ModelMetadata(KNNEngine.DEFAULT, SpaceType.L1, 2, ModelState.CREATED, time, "", ""), + new ModelMetadata(KNNEngine.DEFAULT, SpaceType.L1, 2, ModelState.CREATED, time, "", "", ""), new byte[16], "test-model-1" ); Model model3 = new Model( - new ModelMetadata(KNNEngine.DEFAULT, SpaceType.L2, 2, ModelState.CREATED, time, "", ""), + new ModelMetadata(KNNEngine.DEFAULT, SpaceType.L2, 2, ModelState.CREATED, time, "", "", ""), new byte[16], "test-model-2" ); @@ -207,17 +224,17 @@ public void testHashCode() { String time = ZonedDateTime.now(ZoneOffset.UTC).toString(); Model model1 = new Model( - new ModelMetadata(KNNEngine.DEFAULT, SpaceType.L1, 2, ModelState.CREATED, time, "", ""), + new ModelMetadata(KNNEngine.DEFAULT, SpaceType.L1, 2, ModelState.CREATED, time, "", "", ""), new byte[16], "test-model-1" ); Model model2 = new Model( - new ModelMetadata(KNNEngine.DEFAULT, SpaceType.L1, 2, ModelState.CREATED, time, "", ""), + new ModelMetadata(KNNEngine.DEFAULT, SpaceType.L1, 2, ModelState.CREATED, time, "", "", ""), new byte[16], "test-model-1" ); Model model3 = new Model( - new ModelMetadata(KNNEngine.DEFAULT, SpaceType.L1, 2, ModelState.CREATED, time, "", ""), + new ModelMetadata(KNNEngine.DEFAULT, SpaceType.L1, 2, ModelState.CREATED, time, "", "", ""), new byte[16], "test-model-2" ); @@ -236,8 +253,18 @@ public void testModelFromSourceMap() { String timestamp = ZonedDateTime.now(ZoneOffset.UTC).toString(); String description = "test-description"; String error = "test-error"; + String nodeAssignment = "test-node"; - ModelMetadata metadata = new ModelMetadata(knnEngine, spaceType, dimension, modelState, timestamp, description, error); + ModelMetadata metadata = new ModelMetadata( + knnEngine, + spaceType, + dimension, + modelState, + timestamp, + description, + error, + nodeAssignment + ); Map modelAsMap = new HashMap<>(); modelAsMap.put(KNNConstants.MODEL_ID, modelID); modelAsMap.put(KNNConstants.KNN_ENGINE, knnEngine.getName()); @@ -247,6 +274,7 @@ public void testModelFromSourceMap() { modelAsMap.put(KNNConstants.MODEL_TIMESTAMP, timestamp); modelAsMap.put(KNNConstants.MODEL_DESCRIPTION, description); modelAsMap.put(KNNConstants.MODEL_ERROR, error); + modelAsMap.put(KNNConstants.MODEL_NODE_ASSIGNMENT, nodeAssignment); modelAsMap.put(KNNConstants.MODEL_BLOB_PARAMETER, "aGVsbG8="); byte[] blob1 = "hello".getBytes(); diff --git a/src/test/java/org/opensearch/knn/plugin/action/RestSearchModelHandlerIT.java b/src/test/java/org/opensearch/knn/plugin/action/RestSearchModelHandlerIT.java index 64960cedc..3aa6085d8 100644 --- a/src/test/java/org/opensearch/knn/plugin/action/RestSearchModelHandlerIT.java +++ b/src/test/java/org/opensearch/knn/plugin/action/RestSearchModelHandlerIT.java @@ -48,7 +48,7 @@ public class RestSearchModelHandlerIT extends KNNRestTestCase { private ModelMetadata getModelMetadata() { - return new ModelMetadata(KNNEngine.DEFAULT, SpaceType.DEFAULT, 4, ModelState.CREATED, "2021-03-27", "test model", ""); + return new ModelMetadata(KNNEngine.DEFAULT, SpaceType.DEFAULT, 4, ModelState.CREATED, "2021-03-27", "test model", "", ""); } public void testNotSupportedParams() throws IOException { diff --git a/src/test/java/org/opensearch/knn/plugin/transport/GetModelResponseTests.java b/src/test/java/org/opensearch/knn/plugin/transport/GetModelResponseTests.java index 42adfaf66..6296a7192 100644 --- a/src/test/java/org/opensearch/knn/plugin/transport/GetModelResponseTests.java +++ b/src/test/java/org/opensearch/knn/plugin/transport/GetModelResponseTests.java @@ -27,7 +27,7 @@ public class GetModelResponseTests extends KNNTestCase { private ModelMetadata getModelMetadata(ModelState state) { - return new ModelMetadata(KNNEngine.DEFAULT, SpaceType.DEFAULT, 4, state, "2021-03-27 10:15:30 AM +05:30", "test model", ""); + return new ModelMetadata(KNNEngine.DEFAULT, SpaceType.DEFAULT, 4, state, "2021-03-27 10:15:30 AM +05:30", "test model", "", ""); } public void testStreams() throws IOException { @@ -47,7 +47,7 @@ public void testXContent() throws IOException { Model model = new Model(getModelMetadata(ModelState.CREATED), testModelBlob, modelId); GetModelResponse getModelResponse = new GetModelResponse(model); String expectedResponseString = - "{\"model_id\":\"test-model\",\"model_blob\":\"aGVsbG8=\",\"state\":\"created\",\"timestamp\":\"2021-03-27 10:15:30 AM +05:30\",\"description\":\"test model\",\"error\":\"\",\"space_type\":\"l2\",\"dimension\":4,\"engine\":\"nmslib\"}"; + "{\"model_id\":\"test-model\",\"model_blob\":\"aGVsbG8=\",\"state\":\"created\",\"timestamp\":\"2021-03-27 10:15:30 AM +05:30\",\"description\":\"test model\",\"error\":\"\",\"space_type\":\"l2\",\"dimension\":4,\"engine\":\"nmslib\",\"training_node_assignment\":\"\"}"; XContentBuilder xContentBuilder = MediaTypeRegistry.contentBuilder(XContentType.JSON); getModelResponse.toXContent(xContentBuilder, null); assertEquals(expectedResponseString, xContentBuilder.toString()); @@ -58,7 +58,7 @@ public void testXContentWithNoModelBlob() throws IOException { Model model = new Model(getModelMetadata(ModelState.FAILED), null, modelId); GetModelResponse getModelResponse = new GetModelResponse(model); String expectedResponseString = - "{\"model_id\":\"test-model\",\"model_blob\":\"\",\"state\":\"failed\",\"timestamp\":\"2021-03-27 10:15:30 AM +05:30\",\"description\":\"test model\",\"error\":\"\",\"space_type\":\"l2\",\"dimension\":4,\"engine\":\"nmslib\"}"; + "{\"model_id\":\"test-model\",\"model_blob\":\"\",\"state\":\"failed\",\"timestamp\":\"2021-03-27 10:15:30 AM +05:30\",\"description\":\"test model\",\"error\":\"\",\"space_type\":\"l2\",\"dimension\":4,\"engine\":\"nmslib\",\"training_node_assignment\":\"\"}"; XContentBuilder xContentBuilder = MediaTypeRegistry.contentBuilder(XContentType.JSON); getModelResponse.toXContent(xContentBuilder, null); assertEquals(expectedResponseString, xContentBuilder.toString()); diff --git a/src/test/java/org/opensearch/knn/plugin/transport/RemoveModelFromCacheTransportActionTests.java b/src/test/java/org/opensearch/knn/plugin/transport/RemoveModelFromCacheTransportActionTests.java index ae89d83e1..5d30f54bb 100644 --- a/src/test/java/org/opensearch/knn/plugin/transport/RemoveModelFromCacheTransportActionTests.java +++ b/src/test/java/org/opensearch/knn/plugin/transport/RemoveModelFromCacheTransportActionTests.java @@ -68,7 +68,7 @@ public void testNodeOperation_modelInCache() throws ExecutionException, Interrup ModelDao modelDao = mock(ModelDao.class); String modelId = "test-model-id"; Model model = new Model( - new ModelMetadata(KNNEngine.DEFAULT, SpaceType.L2, 16, ModelState.CREATED, "timestamp", "description", ""), + new ModelMetadata(KNNEngine.DEFAULT, SpaceType.L2, 16, ModelState.CREATED, "timestamp", "description", "", ""), new byte[128], modelId ); diff --git a/src/test/java/org/opensearch/knn/plugin/transport/TrainingJobRouteDecisionInfoTransportActionTests.java b/src/test/java/org/opensearch/knn/plugin/transport/TrainingJobRouteDecisionInfoTransportActionTests.java index 15d84638d..9f64afebb 100644 --- a/src/test/java/org/opensearch/knn/plugin/transport/TrainingJobRouteDecisionInfoTransportActionTests.java +++ b/src/test/java/org/opensearch/knn/plugin/transport/TrainingJobRouteDecisionInfoTransportActionTests.java @@ -19,15 +19,14 @@ import org.opensearch.knn.KNNSingleNodeTestCase; import org.opensearch.knn.indices.Model; import org.opensearch.knn.indices.ModelDao; +import org.opensearch.knn.indices.ModelMetadata; +import org.opensearch.knn.indices.ModelState; import org.opensearch.knn.training.TrainingJob; import org.opensearch.knn.training.TrainingJobRunner; import org.opensearch.threadpool.ThreadPool; import java.io.IOException; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; +import java.util.concurrent.*; import static org.mockito.Mockito.any; import static org.mockito.Mockito.doAnswer; @@ -51,7 +50,7 @@ public void teardown() { } @SuppressWarnings("unchecked") - public void testNodeOperation() throws IOException, InterruptedException { + public void testNodeOperation() throws IOException, InterruptedException, ExecutionException { // Ensure initial value of train job count is 0 TrainingJobRouteDecisionInfoTransportAction action = node().injector() .getInstance(TrainingJobRouteDecisionInfoTransportAction.class); @@ -64,12 +63,16 @@ public void testNodeOperation() throws IOException, InterruptedException { // Setup mocked training job String modelId = "model-id"; Model model = mock(Model.class); + ModelMetadata modelMetadata = mock(ModelMetadata.class); + when(modelMetadata.getState()).thenReturn(ModelState.TRAINING); + when(model.getModelMetadata()).thenReturn(modelMetadata); TrainingJob trainingJob = mock(TrainingJob.class); when(trainingJob.getModelId()).thenReturn(modelId); when(trainingJob.getModel()).thenReturn(model); doAnswer(invocationOnMock -> null).when(trainingJob).run(); ModelDao modelDao = mock(ModelDao.class); + when(modelDao.get(modelId)).thenReturn(model); // Here we check to make sure there is a running job doAnswer(invocationOnMock -> { diff --git a/src/test/java/org/opensearch/knn/plugin/transport/TrainingModelRequestTests.java b/src/test/java/org/opensearch/knn/plugin/transport/TrainingModelRequestTests.java index 09ca6c73b..7465ccc58 100644 --- a/src/test/java/org/opensearch/knn/plugin/transport/TrainingModelRequestTests.java +++ b/src/test/java/org/opensearch/knn/plugin/transport/TrainingModelRequestTests.java @@ -167,6 +167,7 @@ public void testValidation_invalid_modelIdAlreadyExists() { ModelState.CREATED, ZonedDateTime.now(ZoneOffset.UTC).toString(), "", + "", "" ); when(modelDao.getMetadata(modelId)).thenReturn(modelMetadata); diff --git a/src/test/java/org/opensearch/knn/plugin/transport/UpdateModelMetadataRequestTests.java b/src/test/java/org/opensearch/knn/plugin/transport/UpdateModelMetadataRequestTests.java index f38273f15..a41ca900a 100644 --- a/src/test/java/org/opensearch/knn/plugin/transport/UpdateModelMetadataRequestTests.java +++ b/src/test/java/org/opensearch/knn/plugin/transport/UpdateModelMetadataRequestTests.java @@ -39,6 +39,7 @@ public void testStreams() throws IOException { ModelState.CREATED, ZonedDateTime.now(ZoneOffset.UTC).toString(), "", + "", "" ); UpdateModelMetadataRequest updateModelMetadataRequest = new UpdateModelMetadataRequest(modelId, isRemoveRequest, modelMetadata); @@ -62,6 +63,7 @@ public void testValidate() { ModelState.CREATED, ZonedDateTime.now(ZoneOffset.UTC).toString(), "", + "", "" ); @@ -100,6 +102,7 @@ public void testGetModelMetadata() { ModelState.CREATED, ZonedDateTime.now(ZoneOffset.UTC).toString(), "", + "", "" ); UpdateModelMetadataRequest updateModelMetadataRequest = new UpdateModelMetadataRequest("test", true, modelMetadata); diff --git a/src/test/java/org/opensearch/knn/plugin/transport/UpdateModelMetadataTransportActionTests.java b/src/test/java/org/opensearch/knn/plugin/transport/UpdateModelMetadataTransportActionTests.java index eb3ecf168..11961f6f5 100644 --- a/src/test/java/org/opensearch/knn/plugin/transport/UpdateModelMetadataTransportActionTests.java +++ b/src/test/java/org/opensearch/knn/plugin/transport/UpdateModelMetadataTransportActionTests.java @@ -65,6 +65,7 @@ public void testClusterManagerOperation() throws InterruptedException { ModelState.CREATED, ZonedDateTime.now(ZoneOffset.UTC).toString(), "", + "", "" ); diff --git a/src/test/java/org/opensearch/knn/training/TrainingJobClusterStateListenerTests.java b/src/test/java/org/opensearch/knn/training/TrainingJobClusterStateListenerTests.java new file mode 100644 index 000000000..7994e73d2 --- /dev/null +++ b/src/test/java/org/opensearch/knn/training/TrainingJobClusterStateListenerTests.java @@ -0,0 +1,179 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.knn.training; + +import org.opensearch.action.search.SearchRequest; +import org.opensearch.action.search.SearchResponse; +import org.opensearch.cluster.ClusterChangedEvent; +import org.opensearch.cluster.node.DiscoveryNode; +import org.opensearch.cluster.node.DiscoveryNodes; +import org.opensearch.common.unit.TimeValue; +import org.opensearch.core.action.ActionListener; +import org.opensearch.knn.KNNTestCase; +import org.opensearch.knn.indices.Model; +import org.opensearch.knn.indices.ModelDao; +import org.opensearch.knn.indices.ModelMetadata; +import org.opensearch.knn.indices.ModelState; +import org.opensearch.search.SearchHit; +import org.opensearch.search.SearchHits; +import org.opensearch.threadpool.ThreadPool; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.*; +import static org.opensearch.knn.common.KNNConstants.TRAIN_THREAD_POOL; + +public class TrainingJobClusterStateListenerTests extends KNNTestCase { + public void testClusterChanged() throws InterruptedException { + ExecutorService executorService = Executors.newSingleThreadExecutor(); + + TrainingJobClusterStateListener trainingJobClusterStateListener = TrainingJobClusterStateListener.getInstance(); + + ThreadPool threadPool = mock(ThreadPool.class); + when(threadPool.executor(TRAIN_THREAD_POOL)).thenReturn(executorService); + doAnswer(invocationOnMock -> { return null; }).when(threadPool) + .schedule(any(Runnable.class), any(TimeValue.class), any(String.class)); + + ModelDao modelDao = mock(ModelDao.class); + ClusterChangedEvent clusterChangedEvent = mock(ClusterChangedEvent.class); + when(clusterChangedEvent.localNodeClusterManager()).thenReturn(true); + when(clusterChangedEvent.isNewCluster()).thenReturn(true); + + TrainingJobClusterStateListener.initialize(threadPool, modelDao, clusterService); + + trainingJobClusterStateListener.clusterChanged(clusterChangedEvent); + + verify(threadPool, times(1)).schedule(any(Runnable.class), any(TimeValue.class), any(String.class)); + + when(clusterChangedEvent.isNewCluster()).thenReturn(false); + when(clusterChangedEvent.nodesRemoved()).thenReturn(true); + DiscoveryNodes.Delta delta = mock(DiscoveryNodes.Delta.class); + List nodes = new ArrayList<>(); + when(clusterChangedEvent.nodesDelta()).thenReturn(delta); + when(delta.removedNodes()).thenReturn(nodes); + + trainingJobClusterStateListener.clusterChanged(clusterChangedEvent); + + verify(threadPool, times(2)).schedule(any(Runnable.class), any(TimeValue.class), any(String.class)); + verify(clusterChangedEvent, times(1)).nodesDelta(); + + when(clusterChangedEvent.nodesRemoved()).thenReturn(false); + trainingJobClusterStateListener.clusterChanged(clusterChangedEvent); + verify(threadPool, times(2)).schedule(any(Runnable.class), any(TimeValue.class), any(String.class)); + + when(clusterChangedEvent.localNodeClusterManager()).thenReturn(false); + trainingJobClusterStateListener.clusterChanged(clusterChangedEvent); + verify(threadPool, times(2)).schedule(any(Runnable.class), any(TimeValue.class), any(String.class)); + + executorService.shutdown(); + executorService.awaitTermination(10, TimeUnit.SECONDS); + } + + public void testUpdateModelsNewCluster() throws IOException, InterruptedException, ExecutionException { + ExecutorService executorService = Executors.newSingleThreadExecutor(); + + TrainingJobClusterStateListener trainingJobClusterStateListener = TrainingJobClusterStateListener.getInstance(); + + ThreadPool threadPool = mock(ThreadPool.class); + when(threadPool.executor(TRAIN_THREAD_POOL)).thenReturn(executorService); + + String modelId = "test-model-id"; + Model model = mock(Model.class); + ModelMetadata modelMetadata = mock(ModelMetadata.class); + when(modelMetadata.getState()).thenReturn(ModelState.TRAINING); + when(model.getModelMetadata()).thenReturn(modelMetadata); + ModelDao modelDao = mock(ModelDao.class); + when(modelDao.isCreated()).thenReturn(true); + when(modelDao.get(modelId)).thenReturn(model); + doAnswer(invocationOnMock -> { + SearchResponse searchResponse = mock(SearchResponse.class); + SearchHits searchHits = mock(SearchHits.class); + when(searchResponse.getHits()).thenReturn(searchHits); + SearchHit searchHit = mock(SearchHit.class); + when(searchHit.getId()).thenReturn(modelId); + SearchHit[] searchHitArray = new SearchHit[1]; + searchHitArray[0] = searchHit; + when(searchHits.getHits()).thenReturn(searchHitArray); + ((ActionListener) invocationOnMock.getArguments()[1]).onResponse(searchResponse); + return null; + }).when(modelDao).search(any(SearchRequest.class), any(ActionListener.class)); + doAnswer(invocationOnMock -> { return null; }).when(modelDao).update(any(Model.class), any(ActionListener.class)); + + TrainingJobClusterStateListener.initialize(threadPool, modelDao, clusterService); + + trainingJobClusterStateListener.updateModelsNewCluster(); + + executorService.shutdown(); + executorService.awaitTermination(10, TimeUnit.SECONDS); + + verify(modelMetadata, times(1)).setState(ModelState.FAILED); + verify(modelMetadata, times(1)).setError("Training failed to complete as cluster crashed"); + verify(modelDao, times(1)).update(any(Model.class), any(ActionListener.class)); + } + + public void testUpdateModelsNodesRemoved() throws IOException, InterruptedException, ExecutionException { + ExecutorService executorService = Executors.newSingleThreadExecutor(); + + TrainingJobClusterStateListener trainingJobClusterStateListener = TrainingJobClusterStateListener.getInstance(); + + ThreadPool threadPool = mock(ThreadPool.class); + when(threadPool.executor(TRAIN_THREAD_POOL)).thenReturn(executorService); + + String modelId = "test-model-id"; + Model model = mock(Model.class); + ModelMetadata modelMetadata = mock(ModelMetadata.class); + when(modelMetadata.getState()).thenReturn(ModelState.TRAINING); + when(modelMetadata.getNodeAssignment()).thenReturn("test-node-model-match"); + when(model.getModelMetadata()).thenReturn(modelMetadata); + ModelDao modelDao = mock(ModelDao.class); + when(modelDao.isCreated()).thenReturn(true); + when(modelDao.get(modelId)).thenReturn(model); + DiscoveryNode node1 = mock(DiscoveryNode.class); + when(node1.getEphemeralId()).thenReturn("test-node-model-match"); + DiscoveryNode node2 = mock(DiscoveryNode.class); + when(node2.getEphemeralId()).thenReturn("test-node-not-model-match"); + List nodes = new ArrayList(); + nodes.add(node1); + nodes.add(node2); + doAnswer(invocationOnMock -> { + SearchResponse searchResponse = mock(SearchResponse.class); + SearchHits searchHits = mock(SearchHits.class); + when(searchResponse.getHits()).thenReturn(searchHits); + SearchHit searchHit = mock(SearchHit.class); + when(searchHit.getId()).thenReturn(modelId); + SearchHit[] searchHitArray = new SearchHit[1]; + searchHitArray[0] = searchHit; + when(searchHits.getHits()).thenReturn(searchHitArray); + ((ActionListener) invocationOnMock.getArguments()[1]).onResponse(searchResponse); + return null; + }).when(modelDao).search(any(SearchRequest.class), any(ActionListener.class)); + doAnswer(invocationOnMock -> { return null; }).when(modelDao).update(any(Model.class), any(ActionListener.class)); + + TrainingJobClusterStateListener.initialize(threadPool, modelDao, clusterService); + + trainingJobClusterStateListener.updateModelsNodesRemoved(nodes); + + executorService.shutdown(); + executorService.awaitTermination(10, TimeUnit.SECONDS); + + verify(modelMetadata, times(1)).setState(ModelState.FAILED); + verify(modelMetadata, times(1)).setError("Training failed to complete as node dropped"); + verify(modelDao, times(1)).update(any(Model.class), any(ActionListener.class)); + } +} diff --git a/src/test/java/org/opensearch/knn/training/TrainingJobRunnerTests.java b/src/test/java/org/opensearch/knn/training/TrainingJobRunnerTests.java index 9acdc7b36..4876b1562 100644 --- a/src/test/java/org/opensearch/knn/training/TrainingJobRunnerTests.java +++ b/src/test/java/org/opensearch/knn/training/TrainingJobRunnerTests.java @@ -17,13 +17,12 @@ import org.opensearch.knn.KNNTestCase; import org.opensearch.knn.indices.Model; import org.opensearch.knn.indices.ModelDao; +import org.opensearch.knn.indices.ModelMetadata; +import org.opensearch.knn.indices.ModelState; import org.opensearch.threadpool.ThreadPool; import java.io.IOException; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.RejectedExecutionException; -import java.util.concurrent.TimeUnit; +import java.util.concurrent.*; import static org.mockito.Mockito.any; import static org.mockito.Mockito.doAnswer; @@ -37,7 +36,7 @@ public class TrainingJobRunnerTests extends KNNTestCase { @SuppressWarnings("unchecked") - public void testExecute_success() throws IOException, InterruptedException { + public void testExecute_success() throws IOException, InterruptedException, ExecutionException { // Test makes sure the correct execution logic follows on successful run ExecutorService executorService = Executors.newSingleThreadExecutor(); @@ -48,6 +47,9 @@ public void testExecute_success() throws IOException, InterruptedException { String modelId = "test-model-id"; Model model = mock(Model.class); + ModelMetadata modelMetadata = mock(ModelMetadata.class); + when(modelMetadata.getState()).thenReturn(ModelState.TRAINING); + when(model.getModelMetadata()).thenReturn(modelMetadata); TrainingJob trainingJob = mock(TrainingJob.class); when(trainingJob.getModelId()).thenReturn(modelId); when(trainingJob.getModel()).thenReturn(model); @@ -63,6 +65,7 @@ public void testExecute_success() throws IOException, InterruptedException { // After put finishes, it should call the onResponse function that will call responseListener and then kickoff // training. ModelDao modelDao = mock(ModelDao.class); + when(modelDao.get(modelId)).thenReturn(model); doAnswer(invocationOnMock -> { assertEquals(1, trainingJobRunner.getJobCount()); // Make sure job count is correct IndexResponse indexResponse = new IndexResponse(new ShardId(MODEL_INDEX_NAME, "uuid", 0), modelId, 0, 0, 0, true); @@ -88,7 +91,7 @@ public void testExecute_success() throws IOException, InterruptedException { } @SuppressWarnings("unchecked") - public void testExecute_failure_rejected() throws IOException, InterruptedException { + public void testExecute_failure_rejected() throws IOException, InterruptedException, ExecutionException { // This test makes sure we reject another request when one is ongoing. To do this, we call // trainingJobRunner.execute(trainingJob, responseListener) in the mocked modeldao.update. At this point, // the call should produce a failure because a training job is already ongoing. @@ -100,6 +103,9 @@ public void testExecute_failure_rejected() throws IOException, InterruptedExcept String modelId = "test-model-id"; Model model = mock(Model.class); + ModelMetadata modelMetadata = mock(ModelMetadata.class); + when(modelMetadata.getState()).thenReturn(ModelState.TRAINING); + when(model.getModelMetadata()).thenReturn(modelMetadata); TrainingJob trainingJob = mock(TrainingJob.class); when(trainingJob.getModelId()).thenReturn(modelId); when(trainingJob.getModel()).thenReturn(model); @@ -115,6 +121,7 @@ public void testExecute_failure_rejected() throws IOException, InterruptedExcept // After put finishes, it should call the onResponse function that will call responseListener and then kickoff // training. ModelDao modelDao = mock(ModelDao.class); + when(modelDao.get(modelId)).thenReturn(model); doAnswer(invocationOnMock -> { IndexResponse indexResponse = new IndexResponse(new ShardId(MODEL_INDEX_NAME, "uuid", 0), modelId, 0, 0, 0, true); ((ActionListener) invocationOnMock.getArguments()[1]).onResponse(indexResponse); diff --git a/src/test/java/org/opensearch/knn/training/TrainingJobTests.java b/src/test/java/org/opensearch/knn/training/TrainingJobTests.java index b15bf8207..daf9645fa 100644 --- a/src/test/java/org/opensearch/knn/training/TrainingJobTests.java +++ b/src/test/java/org/opensearch/knn/training/TrainingJobTests.java @@ -52,7 +52,8 @@ public void testGetModelId() { mock(NativeMemoryEntryContext.TrainingDataEntryContext.class), mock(NativeMemoryEntryContext.AnonymousEntryContext.class), 10, - "" + "", + "test-node" ); assertEquals(modelId, trainingJob.getModelId()); @@ -62,8 +63,9 @@ public void testGetModel() { SpaceType spaceType = SpaceType.INNER_PRODUCT; KNNEngine knnEngine = KNNEngine.DEFAULT; int dimension = 10; - String desciption = "test description"; + String description = "test description"; String error = ""; + String nodeAssignment = "test-node"; KNNMethodContext knnMethodContext = mock(KNNMethodContext.class); when(knnMethodContext.getKnnEngine()).thenReturn(knnEngine); @@ -77,7 +79,8 @@ public void testGetModel() { mock(NativeMemoryEntryContext.TrainingDataEntryContext.class), mock(NativeMemoryEntryContext.AnonymousEntryContext.class), dimension, - desciption + description, + nodeAssignment ); Model model = new Model( @@ -87,8 +90,9 @@ public void testGetModel() { dimension, ModelState.TRAINING, trainingJob.getModel().getModelMetadata().getTimestamp(), - desciption, - error + description, + error, + nodeAssignment ), null, modelID @@ -159,7 +163,9 @@ public void testRun_success() throws IOException, ExecutionException { trainingDataEntryContext, modelContext, dimension, - "" + "", + "test-node" + ); trainingJob.run(); @@ -235,7 +241,9 @@ public void testRun_failure_onGetTrainingDataAllocation() throws ExecutionExcept trainingDataEntryContext, modelContext, dimension, - "" + "", + + "test-node" ); trainingJob.run(); @@ -301,7 +309,9 @@ public void testRun_failure_onGetModelAnonymousAllocation() throws ExecutionExce trainingDataEntryContext, modelContext, dimension, - "" + "", + + "test-node" ); trainingJob.run(); @@ -366,7 +376,8 @@ public void testRun_failure_closedTrainingDataAllocation() throws ExecutionExcep trainingDataEntryContext, mock(NativeMemoryEntryContext.AnonymousEntryContext.class), dimension, - "" + "", + "test-node" ); trainingJob.run(); @@ -438,7 +449,8 @@ public void testRun_failure_notEnoughTrainingData() throws ExecutionException { trainingDataEntryContext, modelContext, dimension, - "" + "", + "test-node" ); trainingJob.run(); diff --git a/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java b/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java index 6f08627ae..57158be6f 100644 --- a/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java +++ b/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java @@ -77,6 +77,7 @@ import static org.opensearch.knn.common.KNNConstants.MODEL_BLOB_PARAMETER; import static org.opensearch.knn.common.KNNConstants.MODEL_DESCRIPTION; import static org.opensearch.knn.common.KNNConstants.MODEL_ERROR; +import static org.opensearch.knn.common.KNNConstants.MODEL_NODE_ASSIGNMENT; import static org.opensearch.knn.common.KNNConstants.MODEL_ID; import static org.opensearch.knn.common.KNNConstants.MODEL_INDEX_MAPPING_PATH; import static org.opensearch.knn.common.KNNConstants.MODEL_INDEX_NAME; @@ -735,6 +736,7 @@ protected void addModelToSystemIndex(String modelId, ModelMetadata modelMetadata .field(MODEL_TIMESTAMP, modelMetadata.getTimestamp()) .field(MODEL_DESCRIPTION, modelMetadata.getDescription()) .field(MODEL_ERROR, modelMetadata.getError()) + .field(MODEL_NODE_ASSIGNMENT, modelMetadata.getNodeAssignment()) .endObject(); request.setJsonEntity(builder.toString()); From 8b7f6ad4b0b1dfa9119214deb821c54fd7a08400 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Thu, 14 Dec 2023 11:01:26 -0800 Subject: [PATCH 171/416] Increase Lucene max dimension limit to 16,000 (#1346) (#1352) * Increase Lucene max dimension limit to 16,000 Signed-off-by: Junqiu Lei (cherry picked from commit 083ea2ba44a79eb9c592ff61253cc1aa65158888) Co-authored-by: Junqiu Lei --- CHANGELOG.md | 1 + .../index/codec/BasePerFieldKnnVectorsFormat.java | 5 +++++ .../KNN950Codec/KNN950PerFieldKnnVectorsFormat.java | 12 ++++++++++++ .../knn/index/mapper/LuceneFieldMapper.java | 7 ++----- .../org/opensearch/knn/index/util/KNNEngine.java | 3 +-- .../opensearch/knn/index/codec/KNNCodecTestCase.java | 2 ++ .../knn/index/mapper/KNNVectorFieldMapperTests.java | 4 ++-- 7 files changed, 25 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cfd852424..4e53ecc62 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Features * Add parent join support for lucene knn [#1182](https://github.com/opensearch-project/k-NN/pull/1182) ### Enhancements +* Increase Lucene max dimension limit to 16,000 [#1346](https://github.com/opensearch-project/k-NN/pull/1346) ### Bug Fixes * Fix use-after-free case on nmslib search path [#1305](https://github.com/opensearch-project/k-NN/pull/1305) * Allow nested knn field mapping when train model [#1318](https://github.com/opensearch-project/k-NN/pull/1318) diff --git a/src/main/java/org/opensearch/knn/index/codec/BasePerFieldKnnVectorsFormat.java b/src/main/java/org/opensearch/knn/index/codec/BasePerFieldKnnVectorsFormat.java index d10ad9821..c8eb22a97 100644 --- a/src/main/java/org/opensearch/knn/index/codec/BasePerFieldKnnVectorsFormat.java +++ b/src/main/java/org/opensearch/knn/index/codec/BasePerFieldKnnVectorsFormat.java @@ -59,6 +59,11 @@ public KnnVectorsFormat getKnnVectorsFormatForField(final String field) { return formatSupplier.apply(maxConnections, beamWidth); } + @Override + public int getMaxDimensions(String fieldName) { + return getKnnVectorsFormatForField(fieldName).getMaxDimensions(fieldName); + } + private boolean isKnnVectorFieldType(final String field) { return mapperService.isPresent() && mapperService.get().fieldType(field) instanceof KNNVectorFieldMapper.KNNVectorFieldType; } diff --git a/src/main/java/org/opensearch/knn/index/codec/KNN950Codec/KNN950PerFieldKnnVectorsFormat.java b/src/main/java/org/opensearch/knn/index/codec/KNN950Codec/KNN950PerFieldKnnVectorsFormat.java index 66dfcd46e..d9091b2a7 100644 --- a/src/main/java/org/opensearch/knn/index/codec/KNN950Codec/KNN950PerFieldKnnVectorsFormat.java +++ b/src/main/java/org/opensearch/knn/index/codec/KNN950Codec/KNN950PerFieldKnnVectorsFormat.java @@ -8,6 +8,7 @@ import org.apache.lucene.codecs.lucene95.Lucene95HnswVectorsFormat; import org.opensearch.index.mapper.MapperService; import org.opensearch.knn.index.codec.BasePerFieldKnnVectorsFormat; +import org.opensearch.knn.index.util.KNNEngine; import java.util.Optional; @@ -25,4 +26,15 @@ public KNN950PerFieldKnnVectorsFormat(final Optional mapperServic (maxConnm, beamWidth) -> new Lucene95HnswVectorsFormat(maxConnm, beamWidth) ); } + + @Override + /** + * This method returns the maximum dimension allowed from KNNEngine for Lucene codec + * + * @param fieldName Name of the field, ignored + * @return Maximum constant dimension set by KNNEngine + */ + public int getMaxDimensions(String fieldName) { + return KNNEngine.getMaxDimensionByEngine(KNNEngine.LUCENE); + } } diff --git a/src/main/java/org/opensearch/knn/index/mapper/LuceneFieldMapper.java b/src/main/java/org/opensearch/knn/index/mapper/LuceneFieldMapper.java index b28b93028..173f057e6 100644 --- a/src/main/java/org/opensearch/knn/index/mapper/LuceneFieldMapper.java +++ b/src/main/java/org/opensearch/knn/index/mapper/LuceneFieldMapper.java @@ -23,7 +23,6 @@ import java.util.Locale; import java.util.Optional; -import org.apache.lucene.codecs.KnnVectorsFormat; import static org.opensearch.knn.common.KNNConstants.VECTOR_DATA_TYPE_FIELD; import static org.opensearch.knn.index.mapper.KNNVectorFieldMapperUtil.addStoredFieldForVectorField; import static org.opensearch.knn.index.mapper.KNNVectorFieldMapperUtil.buildDocValuesFieldType; @@ -33,8 +32,6 @@ */ public class LuceneFieldMapper extends KNNVectorFieldMapper { - private static final int LUCENE_MAX_DIMENSION = KnnVectorsFormat.DEFAULT_MAX_DIMENSIONS; - /** FieldType used for initializing VectorField, which is used for creating binary doc values. **/ private final FieldType vectorFieldType; private final VectorDataType vectorDataType; @@ -55,12 +52,12 @@ public class LuceneFieldMapper extends KNNVectorFieldMapper { final VectorSimilarityFunction vectorSimilarityFunction = this.knnMethod.getSpaceType().getVectorSimilarityFunction(); final int dimension = input.getMappedFieldType().getDimension(); - if (dimension > LUCENE_MAX_DIMENSION) { + if (dimension > KNNEngine.getMaxDimensionByEngine(KNNEngine.LUCENE)) { throw new IllegalArgumentException( String.format( Locale.ROOT, "Dimension value cannot be greater than [%s] but got [%s] for vector [%s]", - LUCENE_MAX_DIMENSION, + KNNEngine.getMaxDimensionByEngine(KNNEngine.LUCENE), dimension, input.getName() ) diff --git a/src/main/java/org/opensearch/knn/index/util/KNNEngine.java b/src/main/java/org/opensearch/knn/index/util/KNNEngine.java index 197bb87f3..8d03d9a9e 100644 --- a/src/main/java/org/opensearch/knn/index/util/KNNEngine.java +++ b/src/main/java/org/opensearch/knn/index/util/KNNEngine.java @@ -6,7 +6,6 @@ package org.opensearch.knn.index.util; import com.google.common.collect.ImmutableSet; -import org.apache.lucene.codecs.KnnVectorsFormat; import org.opensearch.common.ValidationException; import org.opensearch.knn.index.KNNMethod; import org.opensearch.knn.index.KNNMethodContext; @@ -40,7 +39,7 @@ public enum KNNEngine implements KNNLibrary { KNNEngine.FAISS, 16_000, KNNEngine.LUCENE, - KnnVectorsFormat.DEFAULT_MAX_DIMENSIONS + 16_000 ); /** diff --git a/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestCase.java b/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestCase.java index eb9b4fa2d..40309027d 100644 --- a/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestCase.java +++ b/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestCase.java @@ -338,6 +338,7 @@ public void testKnnVectorIndex( writer.close(); verify(perFieldKnnVectorsFormatSpy, atLeastOnce()).getKnnVectorsFormatForField(eq(FIELD_NAME_ONE)); + verify(perFieldKnnVectorsFormatSpy, atLeastOnce()).getMaxDimensions(eq(FIELD_NAME_ONE)); IndexSearcher searcher = new IndexSearcher(reader); Query query = KNNQueryFactory.create( @@ -372,6 +373,7 @@ public void testKnnVectorIndex( NativeMemoryLoadStrategy.IndexLoadStrategy.initialize(resourceWatcherService); verify(perFieldKnnVectorsFormatSpy, atLeastOnce()).getKnnVectorsFormatForField(eq(FIELD_NAME_TWO)); + verify(perFieldKnnVectorsFormatSpy, atLeastOnce()).getMaxDimensions(eq(FIELD_NAME_TWO)); IndexSearcher searcher1 = new IndexSearcher(reader1); Query query1 = KNNQueryFactory.create( diff --git a/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java b/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java index 2de98d803..4ed231063 100644 --- a/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java +++ b/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java @@ -301,7 +301,7 @@ public void testTypeParser_parse_fromKnnMethodContext_invalidDimension() throws XContentBuilder xContentBuilderOverMaxDimension = XContentFactory.jsonBuilder() .startObject() .field(TYPE_FIELD_NAME, KNN_VECTOR_TYPE) - .field(DIMENSION_FIELD_NAME, 2000) + .field(DIMENSION_FIELD_NAME, 20000) .startObject(KNN_METHOD) .field(NAME, METHOD_HNSW) .field(METHOD_PARAMETER_SPACE_TYPE, SpaceType.L2) @@ -321,7 +321,7 @@ public void testTypeParser_parse_fromKnnMethodContext_invalidDimension() throws IllegalArgumentException.class, () -> builderOverMaxDimension.build(new Mapper.BuilderContext(settings, new ContentPath())) ); - assertEquals("Dimension value cannot be greater than 1024 for vector: test-field-name", ex.getMessage()); + assertEquals("Dimension value cannot be greater than 16000 for vector: test-field-name", ex.getMessage()); XContentBuilder xContentBuilderInvalidDimension = XContentFactory.jsonBuilder() .startObject() From 77d5da193b25aa975ba3f0030d0309187c7c7031 Mon Sep 17 00:00:00 2001 From: Navneet Verma Date: Thu, 21 Dec 2023 16:03:41 -0800 Subject: [PATCH 172/416] Tuned default values for ef_search and ef_construction for better indexing and search performance (#1353) (#1362) Signed-off-by: Navneet Verma --- CHANGELOG.md | 2 +- .../org/opensearch/knn/bwc/IndexingIT.java | 36 +++++- .../org/opensearch/knn/bwc/IndexingIT.java | 80 +++++++++++++ .../org/opensearch/knn/index/KNNMethod.java | 8 +- .../knn/index/KNNMethodContext.java | 12 +- .../org/opensearch/knn/index/KNNSettings.java | 17 +-- .../opensearch/knn/index/MethodComponent.java | 17 ++- .../knn/index/MethodComponentContext.java | 10 +- .../codec/BasePerFieldKnnVectorsFormat.java | 2 +- .../index/mapper/KNNVectorFieldMapper.java | 32 ++++-- .../knn/index/mapper/LegacyFieldMapper.java | 23 ++-- .../knn/index/mapper/LuceneFieldMapper.java | 3 +- .../knn/index/mapper/MethodFieldMapper.java | 11 +- .../knn/index/mapper/ModelFieldMapper.java | 6 +- .../knn/index/util/AbstractKNNLibrary.java | 10 +- .../index/util/IndexHyperParametersUtil.java | 79 +++++++++++++ .../knn/index/util/NativeLibrary.java | 2 +- .../opensearch/knn/training/TrainingJob.java | 10 +- .../java/org/opensearch/knn/KNNTestCase.java | 16 ++- .../org/opensearch/knn/index/FaissIT.java | 107 ++++++++++++++++++ .../opensearch/knn/index/IndexUtilTests.java | 2 + .../knn/index/KNNMethodContextTests.java | 18 +-- .../knn/index/KNNSettingsTests.java | 35 ++++++ .../KNN80DocValuesConsumerTests.java | 2 + .../mapper/KNNVectorFieldMapperTests.java | 20 ++-- .../opensearch/knn/index/util/FaissTests.java | 3 + .../util/IndexHyperParametersUtilTests.java | 44 +++++++ .../opensearch/knn/jni/JNIServiceTests.java | 2 + .../knn/training/TrainingJobTests.java | 16 ++- .../org/opensearch/knn/KNNRestTestCase.java | 59 +++++++++- 30 files changed, 603 insertions(+), 81 deletions(-) create mode 100644 src/main/java/org/opensearch/knn/index/util/IndexHyperParametersUtil.java create mode 100644 src/test/java/org/opensearch/knn/index/util/IndexHyperParametersUtilTests.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 4e53ecc62..bf61d49e7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,12 +17,12 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), * Add parent join support for lucene knn [#1182](https://github.com/opensearch-project/k-NN/pull/1182) ### Enhancements * Increase Lucene max dimension limit to 16,000 [#1346](https://github.com/opensearch-project/k-NN/pull/1346) +* Tuned default values for ef_search and ef_construction for better indexing and search performance for vector search [#1353](https://github.com/opensearch-project/k-NN/pull/1353) ### Bug Fixes * Fix use-after-free case on nmslib search path [#1305](https://github.com/opensearch-project/k-NN/pull/1305) * Allow nested knn field mapping when train model [#1318](https://github.com/opensearch-project/k-NN/pull/1318) * Properly designate model state for actively training models when nodes crash or leave cluster [#1317](https://github.com/opensearch-project/k-NN/pull/1317) ->>>>>>> main ### Infrastructure * Upgrade gradle to 8.4 [1289](https://github.com/opensearch-project/k-NN/pull/1289) ### Documentation diff --git a/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/IndexingIT.java b/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/IndexingIT.java index c8d0cac93..2df79a3a2 100644 --- a/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/IndexingIT.java +++ b/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/IndexingIT.java @@ -5,9 +5,12 @@ package org.opensearch.knn.bwc; +import org.junit.Assert; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.knn.index.SpaceType; +import java.util.Map; + import static org.opensearch.knn.TestUtils.KNN_ALGO_PARAM_EF_CONSTRUCTION_MIN_VALUE; import static org.opensearch.knn.TestUtils.KNN_ALGO_PARAM_M_MIN_VALUE; import static org.opensearch.knn.TestUtils.KNN_VECTOR; @@ -16,8 +19,13 @@ import static org.opensearch.knn.TestUtils.VECTOR_TYPE; import static org.opensearch.knn.common.KNNConstants.DIMENSION; import static org.opensearch.knn.common.KNNConstants.FAISS_NAME; +import static org.opensearch.knn.common.KNNConstants.KNN_ENGINE; import static org.opensearch.knn.common.KNNConstants.KNN_METHOD; import static org.opensearch.knn.common.KNNConstants.METHOD_HNSW; +import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_EF_CONSTRUCTION; +import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_EF_SEARCH; +import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_M; +import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_SPACE_TYPE; import static org.opensearch.knn.common.KNNConstants.NAME; import static org.opensearch.knn.common.KNNConstants.PARAMETERS; @@ -28,6 +36,7 @@ public class IndexingIT extends AbstractRestartUpgradeTestCase { private static final int K = 5; private static final int M = 50; private static final int EF_CONSTRUCTION = 1024; + private static final int EF_SEARCH = 200; private static final int NUM_DOCS = 10; private static int QUERY_COUNT = 0; @@ -78,20 +87,43 @@ public void testKNNIndexDefaultMethodFieldMapping() throws Exception { } // Custom Method Field Mapping - // space_type : "inner_product", engine : "faiss", m : 50, ef_construction : 1024 + // space_type : "inner_product", engine : "faiss", m : 50, ef_construction : 1024, ef_search : 200 public void testKNNIndexCustomMethodFieldMapping() throws Exception { if (isRunningAgainstOldCluster()) { createKnnIndex( testIndex, getKNNDefaultIndexSettings(), - createKNNIndexCustomMethodFieldMapping(TEST_FIELD, DIMENSIONS, SpaceType.INNER_PRODUCT, FAISS_NAME, M, EF_CONSTRUCTION) + createKNNIndexCustomMethodFieldMapping( + TEST_FIELD, + DIMENSIONS, + SpaceType.INNER_PRODUCT, + FAISS_NAME, + M, + EF_CONSTRUCTION, + EF_SEARCH + ) ); addKNNDocs(testIndex, TEST_FIELD, DIMENSIONS, DOC_ID, NUM_DOCS); } else { + validateCustomMethodFieldMappingAfterUpgrade(); validateKNNIndexingOnUpgrade(); } } + private void validateCustomMethodFieldMappingAfterUpgrade() throws Exception { + final Map indexMappings = getIndexMappingAsMap(testIndex); + final Map properties = (Map) indexMappings.get(PROPERTIES); + final Map knnMethod = ((Map) ((Map) properties.get(TEST_FIELD)).get(KNN_METHOD)); + final Map methodParameters = (Map) knnMethod.get(PARAMETERS); + + Assert.assertEquals(METHOD_HNSW, knnMethod.get(NAME)); + Assert.assertEquals(SpaceType.INNER_PRODUCT.getValue(), knnMethod.get(METHOD_PARAMETER_SPACE_TYPE)); + Assert.assertEquals(FAISS_NAME, knnMethod.get(KNN_ENGINE)); + Assert.assertEquals(EF_CONSTRUCTION, ((Integer) methodParameters.get(METHOD_PARAMETER_EF_CONSTRUCTION)).intValue()); + Assert.assertEquals(EF_SEARCH, ((Integer) methodParameters.get(METHOD_PARAMETER_EF_SEARCH)).intValue()); + Assert.assertEquals(M, ((Integer) methodParameters.get(METHOD_PARAMETER_M)).intValue()); + } + // test null parameters public void testNullParametersOnUpgrade() throws Exception { if (isRunningAgainstOldCluster()) { diff --git a/qa/rolling-upgrade/src/test/java/org/opensearch/knn/bwc/IndexingIT.java b/qa/rolling-upgrade/src/test/java/org/opensearch/knn/bwc/IndexingIT.java index 982a05dd7..73adb6db8 100644 --- a/qa/rolling-upgrade/src/test/java/org/opensearch/knn/bwc/IndexingIT.java +++ b/qa/rolling-upgrade/src/test/java/org/opensearch/knn/bwc/IndexingIT.java @@ -15,6 +15,10 @@ public class IndexingIT extends AbstractRollingUpgradeTestCase { private static final int K = 5; private static final int NUM_DOCS = 10; + private static final String ALGO = "hnsw"; + + private static final String ENGINE = "faiss"; + public void testKNNDefaultIndexSettings() throws Exception { waitForClusterHealthGreen(NODES_BWC_CLUSTER); switch (getClusterType()) { @@ -48,6 +52,82 @@ public void testKNNDefaultIndexSettings() throws Exception { } } + public void testKNNIndexCreation_withLegacyMapper() throws Exception { + waitForClusterHealthGreen(NODES_BWC_CLUSTER); + final String firstMixRoundIndex = testIndex + "first-mix-round"; + final String otherMixRoundIndex = testIndex + "other-mix-round"; + final String upgradedIndex = testIndex + "upgraded"; + switch (getClusterType()) { + case OLD: + createKnnIndex(testIndex, getKNNDefaultIndexSettings(), createKnnIndexMapping(TEST_FIELD, DIMENSIONS)); + int docIdOld = 0; + addKNNDocs(testIndex, TEST_FIELD, DIMENSIONS, docIdOld, NUM_DOCS); + break; + case MIXED: + if (isFirstMixedRound()) { + docIdOld = 0; + createKnnIndex(firstMixRoundIndex, getKNNDefaultIndexSettings(), createKnnIndexMapping(TEST_FIELD, DIMENSIONS)); + addKNNDocs(firstMixRoundIndex, TEST_FIELD, DIMENSIONS, docIdOld, NUM_DOCS); + } else { + docIdOld = 0; + createKnnIndex(otherMixRoundIndex, getKNNDefaultIndexSettings(), createKnnIndexMapping(TEST_FIELD, DIMENSIONS)); + addKNNDocs(otherMixRoundIndex, TEST_FIELD, DIMENSIONS, docIdOld, NUM_DOCS); + } + break; + case UPGRADED: + docIdOld = 0; + createKnnIndex(upgradedIndex, getKNNDefaultIndexSettings(), createKnnIndexMapping(TEST_FIELD, DIMENSIONS)); + addKNNDocs(upgradedIndex, TEST_FIELD, DIMENSIONS, docIdOld, NUM_DOCS); + + deleteKNNIndex(testIndex); + deleteKNNIndex(firstMixRoundIndex); + deleteKNNIndex(otherMixRoundIndex); + deleteKNNIndex(upgradedIndex); + } + } + + public void testKNNIndexCreation_withMethodMapper() throws Exception { + waitForClusterHealthGreen(NODES_BWC_CLUSTER); + final String firstMixRoundIndex = testIndex + "first-mix-round"; + final String otherMixRoundIndex = testIndex + "other-mix-round"; + final String upgradedIndex = testIndex + "upgraded"; + switch (getClusterType()) { + case OLD: + createKnnIndex(testIndex, getKNNDefaultIndexSettings(), createKnnIndexMapping(TEST_FIELD, DIMENSIONS, ALGO, ENGINE)); + int docIdOld = 0; + addKNNDocs(testIndex, TEST_FIELD, DIMENSIONS, docIdOld, NUM_DOCS); + break; + case MIXED: + if (isFirstMixedRound()) { + docIdOld = 0; + createKnnIndex( + firstMixRoundIndex, + getKNNDefaultIndexSettings(), + createKnnIndexMapping(TEST_FIELD, DIMENSIONS, ALGO, ENGINE) + ); + addKNNDocs(firstMixRoundIndex, TEST_FIELD, DIMENSIONS, docIdOld, NUM_DOCS); + } else { + docIdOld = 0; + createKnnIndex( + otherMixRoundIndex, + getKNNDefaultIndexSettings(), + createKnnIndexMapping(TEST_FIELD, DIMENSIONS, ALGO, ENGINE) + ); + addKNNDocs(otherMixRoundIndex, TEST_FIELD, DIMENSIONS, docIdOld, NUM_DOCS); + } + break; + case UPGRADED: + docIdOld = 0; + createKnnIndex(upgradedIndex, getKNNDefaultIndexSettings(), createKnnIndexMapping(TEST_FIELD, DIMENSIONS, ALGO, ENGINE)); + addKNNDocs(upgradedIndex, TEST_FIELD, DIMENSIONS, docIdOld, NUM_DOCS); + + deleteKNNIndex(testIndex); + deleteKNNIndex(firstMixRoundIndex); + deleteKNNIndex(otherMixRoundIndex); + deleteKNNIndex(upgradedIndex); + } + } + // validation steps for indexing after upgrading each node from old version to new version public void validateKNNIndexingOnUpgrade(int totalDocsCount, int docId) throws IOException { validateKNNSearch(testIndex, TEST_FIELD, DIMENSIONS, totalDocsCount, K); diff --git a/src/main/java/org/opensearch/knn/index/KNNMethod.java b/src/main/java/org/opensearch/knn/index/KNNMethod.java index d46e8fccd..2d3672d87 100644 --- a/src/main/java/org/opensearch/knn/index/KNNMethod.java +++ b/src/main/java/org/opensearch/knn/index/KNNMethod.java @@ -63,7 +63,7 @@ public ValidationException validate(KNNMethodContext knnMethodContext) { ); } - ValidationException methodValidation = methodComponent.validate(knnMethodContext.getMethodComponent()); + ValidationException methodValidation = methodComponent.validate(knnMethodContext.getMethodComponentContext()); if (methodValidation != null) { errorMessages.addAll(methodValidation.validationErrors()); } @@ -84,7 +84,7 @@ public ValidationException validate(KNNMethodContext knnMethodContext) { * @return true if training is required; false otherwise */ public boolean isTrainingRequired(KNNMethodContext knnMethodContext) { - return methodComponent.isTrainingRequired(knnMethodContext.getMethodComponent()); + return methodComponent.isTrainingRequired(knnMethodContext.getMethodComponentContext()); } /** @@ -95,7 +95,7 @@ public boolean isTrainingRequired(KNNMethodContext knnMethodContext) { * @return estimate overhead in KB */ public int estimateOverheadInKB(KNNMethodContext knnMethodContext, int dimension) { - return methodComponent.estimateOverheadInKB(knnMethodContext.getMethodComponent(), dimension); + return methodComponent.estimateOverheadInKB(knnMethodContext.getMethodComponentContext(), dimension); } /** @@ -105,7 +105,7 @@ public int estimateOverheadInKB(KNNMethodContext knnMethodContext, int dimension * @return KNNMethod as a map */ public Map getAsMap(KNNMethodContext knnMethodContext) { - Map parameterMap = new HashMap<>(methodComponent.getAsMap(knnMethodContext.getMethodComponent())); + Map parameterMap = new HashMap<>(methodComponent.getAsMap(knnMethodContext.getMethodComponentContext())); parameterMap.put(KNNConstants.SPACE_TYPE, knnMethodContext.getSpaceType().getValue()); return parameterMap; } diff --git a/src/main/java/org/opensearch/knn/index/KNNMethodContext.java b/src/main/java/org/opensearch/knn/index/KNNMethodContext.java index 5f5a0f232..d4df713c2 100644 --- a/src/main/java/org/opensearch/knn/index/KNNMethodContext.java +++ b/src/main/java/org/opensearch/knn/index/KNNMethodContext.java @@ -63,7 +63,7 @@ public static synchronized KNNMethodContext getDefault() { @NonNull private final SpaceType spaceType; @NonNull - private final MethodComponentContext methodComponent; + private final MethodComponentContext methodComponentContext; /** * Constructor from stream. @@ -74,7 +74,7 @@ public static synchronized KNNMethodContext getDefault() { public KNNMethodContext(StreamInput in) throws IOException { this.knnEngine = KNNEngine.getEngine(in.readString()); this.spaceType = SpaceType.getSpace(in.readString()); - this.methodComponent = new MethodComponentContext(in); + this.methodComponentContext = new MethodComponentContext(in); } /** @@ -198,7 +198,7 @@ public static KNNMethodContext parse(Object in) { public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.field(KNN_ENGINE, knnEngine.getName()); builder.field(METHOD_PARAMETER_SPACE_TYPE, spaceType.getValue()); - builder = methodComponent.toXContent(builder, params); + builder = methodComponentContext.toXContent(builder, params); return builder; } @@ -211,20 +211,20 @@ public boolean equals(Object obj) { EqualsBuilder equalsBuilder = new EqualsBuilder(); equalsBuilder.append(knnEngine, other.knnEngine); equalsBuilder.append(spaceType, other.spaceType); - equalsBuilder.append(methodComponent, other.methodComponent); + equalsBuilder.append(methodComponentContext, other.methodComponentContext); return equalsBuilder.isEquals(); } @Override public int hashCode() { - return new HashCodeBuilder().append(knnEngine).append(spaceType).append(methodComponent).toHashCode(); + return new HashCodeBuilder().append(knnEngine).append(spaceType).append(methodComponentContext).toHashCode(); } @Override public void writeTo(StreamOutput out) throws IOException { out.writeString(knnEngine.getName()); out.writeString(spaceType.getValue()); - this.methodComponent.writeTo(out); + this.methodComponentContext.writeTo(out); } } diff --git a/src/main/java/org/opensearch/knn/index/KNNSettings.java b/src/main/java/org/opensearch/knn/index/KNNSettings.java index 4356a0610..81a8f6f2e 100644 --- a/src/main/java/org/opensearch/knn/index/KNNSettings.java +++ b/src/main/java/org/opensearch/knn/index/KNNSettings.java @@ -8,6 +8,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.opensearch.OpenSearchParseException; +import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.core.action.ActionListener; import org.opensearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest; import org.opensearch.action.admin.cluster.settings.ClusterUpdateSettingsResponse; @@ -21,6 +22,7 @@ import org.opensearch.index.IndexModule; import org.opensearch.knn.index.memory.NativeMemoryCacheManager; import org.opensearch.knn.index.memory.NativeMemoryCacheManagerDto; +import org.opensearch.knn.index.util.IndexHyperParametersUtil; import org.opensearch.monitor.jvm.JvmInfo; import org.opensearch.monitor.os.OsProbe; @@ -80,8 +82,8 @@ public class KNNSettings { */ public static final String INDEX_KNN_DEFAULT_SPACE_TYPE = "l2"; public static final Integer INDEX_KNN_DEFAULT_ALGO_PARAM_M = 16; - public static final Integer INDEX_KNN_DEFAULT_ALGO_PARAM_EF_SEARCH = 512; - public static final Integer INDEX_KNN_DEFAULT_ALGO_PARAM_EF_CONSTRUCTION = 512; + public static final Integer INDEX_KNN_DEFAULT_ALGO_PARAM_EF_SEARCH = 100; + public static final Integer INDEX_KNN_DEFAULT_ALGO_PARAM_EF_CONSTRUCTION = 100; public static final Integer KNN_DEFAULT_ALGO_PARAM_INDEX_THREAD_QTY = 1; public static final Integer KNN_DEFAULT_CIRCUIT_BREAKER_UNSET_PERCENTAGE = 75; public static final Integer KNN_DEFAULT_MODEL_CACHE_SIZE_LIMIT_PERCENTAGE = 10; // By default, set aside 10% of the JVM for the limit @@ -447,11 +449,12 @@ public void onFailure(Exception e) { * @return efSearch value */ public static int getEfSearchParam(String index) { - return KNNSettings.state().clusterService.state() - .getMetadata() - .index(index) - .getSettings() - .getAsInt(KNNSettings.KNN_ALGO_PARAM_EF_SEARCH, 512); + final IndexMetadata indexMetadata = KNNSettings.state().clusterService.state().getMetadata().index(index); + return indexMetadata.getSettings() + .getAsInt( + KNNSettings.KNN_ALGO_PARAM_EF_SEARCH, + IndexHyperParametersUtil.getHNSWEFSearchValue(indexMetadata.getCreationVersion()) + ); } public void setClusterService(ClusterService clusterService) { diff --git a/src/main/java/org/opensearch/knn/index/MethodComponent.java b/src/main/java/org/opensearch/knn/index/MethodComponent.java index c904f15c3..f2e2d878e 100644 --- a/src/main/java/org/opensearch/knn/index/MethodComponent.java +++ b/src/main/java/org/opensearch/knn/index/MethodComponent.java @@ -12,9 +12,11 @@ package org.opensearch.knn.index; import lombok.Getter; +import org.opensearch.Version; import org.opensearch.common.TriFunction; import org.opensearch.common.ValidationException; import org.opensearch.knn.common.KNNConstants; +import org.opensearch.knn.index.util.IndexHyperParametersUtil; import java.util.ArrayList; import java.util.HashMap; @@ -296,11 +298,24 @@ public static Map getParameterMapWithDefaultsAdded( ) { Map parametersWithDefaultsMap = new HashMap<>(); Map userProvidedParametersMap = methodComponentContext.getParameters(); + Version indexCreationVersion = methodComponentContext.getIndexVersion(); for (Parameter parameter : methodComponent.getParameters().values()) { if (methodComponentContext.getParameters().containsKey(parameter.getName())) { parametersWithDefaultsMap.put(parameter.getName(), userProvidedParametersMap.get(parameter.getName())); } else { - parametersWithDefaultsMap.put(parameter.getName(), parameter.getDefaultValue()); + // Picking the right values for the parameters whose values are different based on different index + // created version. + if (parameter.getName().equals(KNNConstants.METHOD_PARAMETER_EF_SEARCH)) { + parametersWithDefaultsMap.put(parameter.getName(), IndexHyperParametersUtil.getHNSWEFSearchValue(indexCreationVersion)); + } else if (parameter.getName().equals(KNNConstants.METHOD_PARAMETER_EF_CONSTRUCTION)) { + parametersWithDefaultsMap.put( + parameter.getName(), + IndexHyperParametersUtil.getHNSWEFConstructionValue(indexCreationVersion) + ); + } else { + parametersWithDefaultsMap.put(parameter.getName(), parameter.getDefaultValue()); + } + } } diff --git a/src/main/java/org/opensearch/knn/index/MethodComponentContext.java b/src/main/java/org/opensearch/knn/index/MethodComponentContext.java index cf10e9a64..66952f448 100644 --- a/src/main/java/org/opensearch/knn/index/MethodComponentContext.java +++ b/src/main/java/org/opensearch/knn/index/MethodComponentContext.java @@ -11,8 +11,10 @@ package org.opensearch.knn.index; -import lombok.AllArgsConstructor; import lombok.Getter; +import lombok.RequiredArgsConstructor; +import lombok.Setter; +import org.opensearch.Version; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.io.stream.Writeable; @@ -36,13 +38,17 @@ * * Each component is composed of a name and a map of parameters. */ -@AllArgsConstructor +@RequiredArgsConstructor public class MethodComponentContext implements ToXContentFragment, Writeable { @Getter private final String name; private final Map parameters; + @Getter + @Setter + private Version indexVersion; + /** * Constructor from stream. * diff --git a/src/main/java/org/opensearch/knn/index/codec/BasePerFieldKnnVectorsFormat.java b/src/main/java/org/opensearch/knn/index/codec/BasePerFieldKnnVectorsFormat.java index c8eb22a97..b9e280e2e 100644 --- a/src/main/java/org/opensearch/knn/index/codec/BasePerFieldKnnVectorsFormat.java +++ b/src/main/java/org/opensearch/knn/index/codec/BasePerFieldKnnVectorsFormat.java @@ -47,7 +47,7 @@ public KnnVectorsFormat getKnnVectorsFormatForField(final String field) { String.format("Cannot read field type for field [%s] because mapper service is not available", field) ) ).fieldType(field); - var params = type.getKnnMethodContext().getMethodComponent().getParameters(); + var params = type.getKnnMethodContext().getMethodComponentContext().getParameters(); int maxConnections = getMaxConnections(params); int beamWidth = getBeamWidth(params); log.debug( diff --git a/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapper.java b/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapper.java index 346d4c238..5b427517b 100644 --- a/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapper.java +++ b/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapper.java @@ -7,6 +7,7 @@ import lombok.Getter; import lombok.extern.log4j.Log4j2; +import org.opensearch.Version; import org.opensearch.common.ValidationException; import org.opensearch.knn.common.KNNConstants; @@ -79,6 +80,10 @@ private static KNNVectorFieldMapper toType(FieldMapper in) { return (KNNVectorFieldMapper) in; } + // We store the version of the index with the mapper as different version of Opensearch has different default + // values of KNN engine Algorithms hyperparameters. + protected Version indexCreatedVersion; + /** * Builder for KNNVectorFieldMapper. This class defines the set of parameters that can be applied to the knn_vector * field type @@ -139,7 +144,7 @@ public static class Builder extends ParametrizedFieldMapper.Builder { b.startObject(n); v.toXContent(b, ToXContent.EMPTY_PARAMS); b.endObject(); - }), m -> m.getMethodComponent().getName()).setValidator(v -> { + }), m -> m.getMethodComponentContext().getName()).setValidator(v -> { if (v == null) return; ValidationException validationException = null; @@ -167,9 +172,12 @@ public static class Builder extends ParametrizedFieldMapper.Builder { protected ModelDao modelDao; - public Builder(String name, ModelDao modelDao) { + protected Version indexCreatedVersion; + + public Builder(String name, ModelDao modelDao, Version indexCreatedVersion) { super(name); this.modelDao = modelDao; + this.indexCreatedVersion = indexCreatedVersion; } /** @@ -181,11 +189,12 @@ public Builder(String name, ModelDao modelDao) { * @param m m value of field * @param efConstruction efConstruction value of field */ - public Builder(String name, String spaceType, String m, String efConstruction) { + public Builder(String name, String spaceType, String m, String efConstruction, Version indexCreatedVersion) { super(name); this.spaceType = spaceType; this.m = m; this.efConstruction = efConstruction; + this.indexCreatedVersion = indexCreatedVersion; } @Override @@ -221,6 +230,7 @@ public KNNVectorFieldMapper build(BuilderContext context) { final Map metaValue = meta.getValue(); if (knnMethodContext != null) { + knnMethodContext.getMethodComponentContext().setIndexVersion(indexCreatedVersion); final KNNVectorFieldType mappedFieldType = new KNNVectorFieldType( buildFullName(context), metaValue, @@ -278,7 +288,8 @@ public KNNVectorFieldMapper build(BuilderContext context) { stored.get(), hasDocValues.get(), modelDao, - modelIdAsString + modelIdAsString, + indexCreatedVersion ); } @@ -292,7 +303,7 @@ public KNNVectorFieldMapper build(BuilderContext context) { } if (this.efConstruction == null) { - this.efConstruction = LegacyFieldMapper.getEfConstruction(context.indexSettings()); + this.efConstruction = LegacyFieldMapper.getEfConstruction(context.indexSettings(), indexCreatedVersion); } // Validates and throws exception if index.knn is set to true in the index settings @@ -310,7 +321,8 @@ public KNNVectorFieldMapper build(BuilderContext context) { hasDocValues.get(), spaceType, m, - efConstruction + efConstruction, + indexCreatedVersion ); } @@ -346,7 +358,7 @@ public TypeParser(Supplier modelDaoSupplier) { @Override public Mapper.Builder parse(String name, Map node, ParserContext parserContext) throws MapperParsingException { - Builder builder = new KNNVectorFieldMapper.Builder(name, modelDaoSupplier.get()); + Builder builder = new KNNVectorFieldMapper.Builder(name, modelDaoSupplier.get(), parserContext.indexVersionCreated()); builder.parse(name, parserContext, node); // All ignoreMalformed, boolean stored, - boolean hasDocValues + boolean hasDocValues, + Version indexCreatedVersion ) { super(simpleName, mappedFieldType, multiFields, copyTo); this.ignoreMalformed = ignoreMalformed; @@ -469,6 +482,7 @@ public KNNVectorFieldMapper( this.dimension = mappedFieldType.getDimension(); this.vectorDataType = mappedFieldType.getVectorDataType(); updateEngineStats(); + this.indexCreatedVersion = indexCreatedVersion; } public KNNVectorFieldMapper clone() { @@ -610,7 +624,7 @@ protected boolean docValuesByDefault() { @Override public ParametrizedFieldMapper.Builder getMergeBuilder() { - return new KNNVectorFieldMapper.Builder(simpleName(), modelDao).init(this); + return new KNNVectorFieldMapper.Builder(simpleName(), modelDao, indexCreatedVersion).init(this); } @Override diff --git a/src/main/java/org/opensearch/knn/index/mapper/LegacyFieldMapper.java b/src/main/java/org/opensearch/knn/index/mapper/LegacyFieldMapper.java index 868aec3e6..d67ffc73c 100644 --- a/src/main/java/org/opensearch/knn/index/mapper/LegacyFieldMapper.java +++ b/src/main/java/org/opensearch/knn/index/mapper/LegacyFieldMapper.java @@ -7,10 +7,12 @@ import lombok.extern.log4j.Log4j2; import org.apache.lucene.document.FieldType; +import org.opensearch.Version; import org.opensearch.common.Explicit; import org.opensearch.common.settings.Settings; import org.opensearch.index.mapper.ParametrizedFieldMapper; import org.opensearch.knn.index.KNNSettings; +import org.opensearch.knn.index.util.IndexHyperParametersUtil; import org.opensearch.knn.index.util.KNNEngine; import static org.opensearch.knn.common.KNNConstants.DIMENSION; @@ -47,9 +49,10 @@ public class LegacyFieldMapper extends KNNVectorFieldMapper { boolean hasDocValues, String spaceType, String m, - String efConstruction + String efConstruction, + Version indexCreatedVersion ) { - super(simpleName, mappedFieldType, multiFields, copyTo, ignoreMalformed, stored, hasDocValues); + super(simpleName, mappedFieldType, multiFields, copyTo, ignoreMalformed, stored, hasDocValues, indexCreatedVersion); this.spaceType = spaceType; this.m = m; @@ -70,7 +73,9 @@ public class LegacyFieldMapper extends KNNVectorFieldMapper { @Override public ParametrizedFieldMapper.Builder getMergeBuilder() { - return new KNNVectorFieldMapper.Builder(simpleName(), this.spaceType, this.m, this.efConstruction).init(this); + return new KNNVectorFieldMapper.Builder(simpleName(), this.spaceType, this.m, this.efConstruction, this.indexCreatedVersion).init( + this + ); } static String getSpaceType(Settings indexSettings) { @@ -103,17 +108,19 @@ static String getM(Settings indexSettings) { return m; } - static String getEfConstruction(Settings indexSettings) { - String efConstruction = indexSettings.get(KNNSettings.INDEX_KNN_ALGO_PARAM_EF_CONSTRUCTION_SETTING.getKey()); + static String getEfConstruction(Settings indexSettings, Version indexVersion) { + final String efConstruction = indexSettings.get(KNNSettings.INDEX_KNN_ALGO_PARAM_EF_CONSTRUCTION_SETTING.getKey()); if (efConstruction == null) { + final String defaultEFConstructionValue = String.valueOf(IndexHyperParametersUtil.getHNSWEFConstructionValue(indexVersion)); log.info( String.format( - "[KNN] The setting \"%s\" was not set for the index. Likely caused by recent version upgrade. Setting the setting to the default value=%s", + "[KNN] The setting \"%s\" was not set for the index. Likely caused by recent version upgrade. " + + "Picking up default value for the index =%s", HNSW_ALGO_EF_CONSTRUCTION, - KNNSettings.INDEX_KNN_DEFAULT_ALGO_PARAM_EF_CONSTRUCTION + defaultEFConstructionValue ) ); - return String.valueOf(KNNSettings.INDEX_KNN_DEFAULT_ALGO_PARAM_EF_CONSTRUCTION); + return defaultEFConstructionValue; } return efConstruction; } diff --git a/src/main/java/org/opensearch/knn/index/mapper/LuceneFieldMapper.java b/src/main/java/org/opensearch/knn/index/mapper/LuceneFieldMapper.java index 173f057e6..831a23f4b 100644 --- a/src/main/java/org/opensearch/knn/index/mapper/LuceneFieldMapper.java +++ b/src/main/java/org/opensearch/knn/index/mapper/LuceneFieldMapper.java @@ -44,7 +44,8 @@ public class LuceneFieldMapper extends KNNVectorFieldMapper { input.getCopyTo(), input.getIgnoreMalformed(), input.isStored(), - input.isHasDocValues() + input.isHasDocValues(), + input.getKnnMethodContext().getMethodComponentContext().getIndexVersion() ); vectorDataType = input.getVectorDataType(); diff --git a/src/main/java/org/opensearch/knn/index/mapper/MethodFieldMapper.java b/src/main/java/org/opensearch/knn/index/mapper/MethodFieldMapper.java index 1db710bad..d2db7fb5a 100644 --- a/src/main/java/org/opensearch/knn/index/mapper/MethodFieldMapper.java +++ b/src/main/java/org/opensearch/knn/index/mapper/MethodFieldMapper.java @@ -34,7 +34,16 @@ public class MethodFieldMapper extends KNNVectorFieldMapper { KNNMethodContext knnMethodContext ) { - super(simpleName, mappedFieldType, multiFields, copyTo, ignoreMalformed, stored, hasDocValues); + super( + simpleName, + mappedFieldType, + multiFields, + copyTo, + ignoreMalformed, + stored, + hasDocValues, + knnMethodContext.getMethodComponentContext().getIndexVersion() + ); this.knnMethod = knnMethodContext; diff --git a/src/main/java/org/opensearch/knn/index/mapper/ModelFieldMapper.java b/src/main/java/org/opensearch/knn/index/mapper/ModelFieldMapper.java index 0fa116937..2f138dba6 100644 --- a/src/main/java/org/opensearch/knn/index/mapper/ModelFieldMapper.java +++ b/src/main/java/org/opensearch/knn/index/mapper/ModelFieldMapper.java @@ -6,6 +6,7 @@ package org.opensearch.knn.index.mapper; import org.apache.lucene.document.FieldType; +import org.opensearch.Version; import org.opensearch.common.Explicit; import org.opensearch.index.mapper.ParseContext; import org.opensearch.knn.indices.ModelDao; @@ -29,9 +30,10 @@ public class ModelFieldMapper extends KNNVectorFieldMapper { boolean stored, boolean hasDocValues, ModelDao modelDao, - String modelId + String modelId, + Version indexCreatedVersion ) { - super(simpleName, mappedFieldType, multiFields, copyTo, ignoreMalformed, stored, hasDocValues); + super(simpleName, mappedFieldType, multiFields, copyTo, ignoreMalformed, stored, hasDocValues, indexCreatedVersion); this.modelId = modelId; this.modelDao = modelDao; diff --git a/src/main/java/org/opensearch/knn/index/util/AbstractKNNLibrary.java b/src/main/java/org/opensearch/knn/index/util/AbstractKNNLibrary.java index c5a3d1d1e..f97d18810 100644 --- a/src/main/java/org/opensearch/knn/index/util/AbstractKNNLibrary.java +++ b/src/main/java/org/opensearch/knn/index/util/AbstractKNNLibrary.java @@ -35,22 +35,24 @@ public KNNMethod getMethod(String methodName) { @Override public ValidationException validateMethod(KNNMethodContext knnMethodContext) { - String methodName = knnMethodContext.getMethodComponent().getName(); + String methodName = knnMethodContext.getMethodComponentContext().getName(); return getMethod(methodName).validate(knnMethodContext); } @Override public boolean isTrainingRequired(KNNMethodContext knnMethodContext) { - String methodName = knnMethodContext.getMethodComponent().getName(); + String methodName = knnMethodContext.getMethodComponentContext().getName(); return getMethod(methodName).isTrainingRequired(knnMethodContext); } @Override public Map getMethodAsMap(KNNMethodContext knnMethodContext) { - KNNMethod knnMethod = methods.get(knnMethodContext.getMethodComponent().getName()); + KNNMethod knnMethod = methods.get(knnMethodContext.getMethodComponentContext().getName()); if (knnMethod == null) { - throw new IllegalArgumentException(String.format("Invalid method name: %s", knnMethodContext.getMethodComponent().getName())); + throw new IllegalArgumentException( + String.format("Invalid method name: %s", knnMethodContext.getMethodComponentContext().getName()) + ); } return knnMethod.getAsMap(knnMethodContext); diff --git a/src/main/java/org/opensearch/knn/index/util/IndexHyperParametersUtil.java b/src/main/java/org/opensearch/knn/index/util/IndexHyperParametersUtil.java new file mode 100644 index 000000000..af842788a --- /dev/null +++ b/src/main/java/org/opensearch/knn/index/util/IndexHyperParametersUtil.java @@ -0,0 +1,79 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.knn.index.util; + +import lombok.AccessLevel; +import lombok.NoArgsConstructor; +import lombok.NonNull; +import lombok.extern.log4j.Log4j2; +import org.opensearch.Version; +import org.opensearch.knn.index.KNNSettings; + +/** + * This class acts as an abstraction to get the default hyperparameter values for different parameters used in the + * Nearest Neighbor Algorithm across different version of Index. + */ +@Log4j2 +@NoArgsConstructor(access = AccessLevel.PRIVATE) +public class IndexHyperParametersUtil { + + private static final int INDEX_KNN_DEFAULT_ALGO_PARAM_EF_CONSTRUCTION_OLD_VALUE = 512; + private static final int INDEX_KNN_DEFAULT_ALGO_PARAM_EF_SEARCH_OLD_VALUE = 512; + + /** + * Returns the default value of EF Construction that should be used for the input index version. After version 2.12.0 + * of Opensearch we are have reduced the value of ef_construction in favor of better build times. + * + * @param indexVersion {@code Version} of the index with which it was created. + * @return default value of EF Construction that should be used for the input index version. + */ + public static int getHNSWEFConstructionValue(@NonNull final Version indexVersion) { + if (indexVersion.before(Version.V_2_12_0)) { + log.debug( + "Picking up old values of ef_construction : index version : {}, value: {}", + indexVersion, + INDEX_KNN_DEFAULT_ALGO_PARAM_EF_CONSTRUCTION_OLD_VALUE + ); + return INDEX_KNN_DEFAULT_ALGO_PARAM_EF_CONSTRUCTION_OLD_VALUE; + } + log.debug( + "Picking up new values of ef_construction : index version : {}, value: {}", + indexVersion, + KNNSettings.INDEX_KNN_DEFAULT_ALGO_PARAM_EF_CONSTRUCTION + ); + return KNNSettings.INDEX_KNN_DEFAULT_ALGO_PARAM_EF_CONSTRUCTION; + } + + /** + * Returns the default value of EF Search that should be used for the input index version. After version 2.12.0 + * of Opensearch we are have reduced the value of ef_search in favor of better latency. + * + * @param indexVersion {@code Version} of the index with which it was created. + * @return default value of EF Search that should be used for the input index version. + */ + public static int getHNSWEFSearchValue(@NonNull final Version indexVersion) { + if (indexVersion.before(Version.V_2_12_0)) { + log.debug( + "Picking up old values of ef_search : index version : {}, value: {}", + indexVersion, + INDEX_KNN_DEFAULT_ALGO_PARAM_EF_SEARCH_OLD_VALUE + ); + return INDEX_KNN_DEFAULT_ALGO_PARAM_EF_SEARCH_OLD_VALUE; + } + log.debug( + "Picking up new values of ef_search : index version : {}, value: {}", + indexVersion, + KNNSettings.INDEX_KNN_DEFAULT_ALGO_PARAM_EF_SEARCH + ); + return KNNSettings.INDEX_KNN_DEFAULT_ALGO_PARAM_EF_SEARCH; + } +} diff --git a/src/main/java/org/opensearch/knn/index/util/NativeLibrary.java b/src/main/java/org/opensearch/knn/index/util/NativeLibrary.java index 896c2b889..5e264ed12 100644 --- a/src/main/java/org/opensearch/knn/index/util/NativeLibrary.java +++ b/src/main/java/org/opensearch/knn/index/util/NativeLibrary.java @@ -62,7 +62,7 @@ public float score(float rawScore, SpaceType spaceType) { @Override public int estimateOverheadInKB(KNNMethodContext knnMethodContext, int dimension) { - String methodName = knnMethodContext.getMethodComponent().getName(); + String methodName = knnMethodContext.getMethodComponentContext().getName(); return getMethod(methodName).estimateOverheadInKB(knnMethodContext, dimension); } diff --git a/src/main/java/org/opensearch/knn/training/TrainingJob.java b/src/main/java/org/opensearch/knn/training/TrainingJob.java index 8a2af4319..7b5404f6c 100644 --- a/src/main/java/org/opensearch/knn/training/TrainingJob.java +++ b/src/main/java/org/opensearch/knn/training/TrainingJob.java @@ -174,7 +174,7 @@ public void run() { if (trainingDataAllocation.isClosed()) { throw new RuntimeException("Unable to load training data into memory: allocation is already closed"); } - + setVersionInKnnMethodContext(); Map trainParameters = model.getModelMetadata().getKnnEngine().getMethodAsMap(knnMethodContext); trainParameters.put( KNNConstants.INDEX_THREAD_QTY, @@ -192,7 +192,7 @@ public void run() { model.setModelBlob(modelBlob); modelMetadata.setState(ModelState.CREATED); } catch (Exception e) { - logger.error("Failed to run training job for model \"" + modelId + "\": " + e.getMessage()); + logger.error("Failed to run training job for model \"" + modelId + "\": ", e); modelMetadata.setState(ModelState.FAILED); modelMetadata.setError( "Failed to execute training. May be caused by an invalid method definition or " + "not enough memory to perform training." @@ -208,4 +208,10 @@ public void run() { nativeMemoryCacheManager.invalidate(modelAnonymousEntryContext.getKey()); } } + + private void setVersionInKnnMethodContext() { + // We are picking up the node version here. For more details why we did this please check below conversation + // Ref: https://github.com/opensearch-project/k-NN/pull/1353#discussion_r1434428542 + knnMethodContext.getMethodComponentContext().setIndexVersion(trainingDataEntryContext.getClusterService().localNode().getVersion()); + } } diff --git a/src/test/java/org/opensearch/knn/KNNTestCase.java b/src/test/java/org/opensearch/knn/KNNTestCase.java index 1b0dbefef..3f20981bd 100644 --- a/src/test/java/org/opensearch/knn/KNNTestCase.java +++ b/src/test/java/org/opensearch/knn/KNNTestCase.java @@ -39,6 +39,11 @@ public class KNNTestCase extends OpenSearchTestCase { public void setUp() throws Exception { super.setUp(); openMocks = MockitoAnnotations.openMocks(this); + // This is required to make sure that before every test we are initializing the KNNSettings. Not doing this + // leads to failures of unit tests cases when a unit test is run separately. Try running this test: + // ./gradlew ':test' --tests "org.opensearch.knn.training.TrainingJobTests.testRun_success" and see it fails + // but if run along with other tests this test passes. + initKNNSettings(); } @Override @@ -53,7 +58,14 @@ public void resetState() { for (KNNCounter knnCounter : KNNCounter.values()) { knnCounter.set(0L); } + initKNNSettings(); + // Clean up the cache + NativeMemoryCacheManager.getInstance().invalidateAll(); + NativeMemoryCacheManager.getInstance().close(); + } + + private void initKNNSettings() { Set> defaultClusterSettings = new HashSet<>(ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); defaultClusterSettings.addAll( KNNSettings.state() @@ -64,10 +76,6 @@ public void resetState() { ); when(clusterService.getClusterSettings()).thenReturn(new ClusterSettings(Settings.EMPTY, defaultClusterSettings)); KNNSettings.state().setClusterService(clusterService); - - // Clean up the cache - NativeMemoryCacheManager.getInstance().invalidateAll(); - NativeMemoryCacheManager.getInstance().close(); } public Map xContentBuilderToMap(XContentBuilder xContentBuilder) { diff --git a/src/test/java/org/opensearch/knn/index/FaissIT.java b/src/test/java/org/opensearch/knn/index/FaissIT.java index 41ce2be99..b373832d1 100644 --- a/src/test/java/org/opensearch/knn/index/FaissIT.java +++ b/src/test/java/org/opensearch/knn/index/FaissIT.java @@ -267,6 +267,113 @@ public void testEndToEnd_whenMethodIsHNSWPQ_thenSucceed() { fail("Graphs are not getting evicted"); } + @SneakyThrows + public void testEndToEnd_whenMethodIsHNSWPQAndHyperParametersNotSet_thenSucceed() { + String indexName = "test-index"; + String fieldName = "test-field"; + String trainingIndexName = "training-index"; + String trainingFieldName = "training-field"; + + String modelId = "test-model"; + String modelDescription = "test model"; + + List mValues = ImmutableList.of(16, 32, 64, 128); + List pqMValues = ImmutableList.of(2, 4, 8); + + // training data needs to be at least equal to the number of centroids for PQ + // which is 2^8 = 256. 8 because thats the only valid code_size for HNSWPQ + int trainingDataCount = 256; + + SpaceType spaceType = SpaceType.L2; + + Integer dimension = testData.indexData.vectors[0].length; + + XContentBuilder xContentBuilder = XContentFactory.jsonBuilder() + .startObject() + .field(NAME, METHOD_HNSW) + .field(KNN_ENGINE, FAISS_NAME) + .startObject(PARAMETERS) + .field(KNNConstants.METHOD_PARAMETER_M, mValues.get(random().nextInt(mValues.size()))) + .startObject(METHOD_ENCODER_PARAMETER) + .field(NAME, ENCODER_PQ) + .startObject(PARAMETERS) + .field(ENCODER_PARAMETER_PQ_M, pqMValues.get(random().nextInt(pqMValues.size()))) + .endObject() + .endObject() + .endObject() + .endObject(); + Map in = xContentBuilderToMap(xContentBuilder); + + createBasicKnnIndex(trainingIndexName, trainingFieldName, dimension); + ingestDataAndTrainModel(modelId, trainingIndexName, trainingFieldName, dimension, modelDescription, in, trainingDataCount); + assertTrainingSucceeds(modelId, 360, 1000); + + // Create an index + XContentBuilder builder = XContentFactory.jsonBuilder() + .startObject() + .startObject("properties") + .startObject(fieldName) + .field("type", "knn_vector") + .field("model_id", modelId) + .endObject() + .endObject() + .endObject(); + + Map mappingMap = xContentBuilderToMap(builder); + String mapping = builder.toString(); + + createKnnIndex(indexName, mapping); + assertEquals(new TreeMap<>(mappingMap), new TreeMap<>(getIndexMappingAsMap(indexName))); + + // Index the test data + for (int i = 0; i < testData.indexData.docs.length; i++) { + addKnnDoc( + indexName, + Integer.toString(testData.indexData.docs[i]), + fieldName, + Floats.asList(testData.indexData.vectors[i]).toArray() + ); + } + + // Assert we have the right number of documents in the index + refreshAllNonSystemIndices(); + assertEquals(testData.indexData.docs.length, getDocCount(indexName)); + + int k = 10; + for (int i = 0; i < testData.queries.length; i++) { + Response response = searchKNNIndex(indexName, new KNNQueryBuilder(fieldName, testData.queries[i], k), k); + String responseBody = EntityUtils.toString(response.getEntity()); + List knnResults = parseSearchResponse(responseBody, fieldName); + assertEquals(k, knnResults.size()); + + List actualScores = parseSearchResponseScore(responseBody, fieldName); + for (int j = 0; j < k; j++) { + float[] primitiveArray = Floats.toArray(Arrays.stream(knnResults.get(j).getVector()).collect(Collectors.toList())); + assertEquals( + KNNEngine.FAISS.score(KNNScoringUtil.l2Squared(testData.queries[i], primitiveArray), spaceType), + actualScores.get(j), + 0.0001 + ); + } + } + + // Delete index + deleteKNNIndex(indexName); + deleteModel(modelId); + + // Search every 5 seconds 14 times to confirm graph gets evicted + int intervals = 14; + for (int i = 0; i < intervals; i++) { + if (getTotalGraphsInCache() == 0) { + return; + } + + Thread.sleep(5 * 1000); + } + + fail("Graphs are not getting evicted"); + } + public void testDocUpdate() throws IOException { String indexName = "test-index-1"; String fieldName = "test-field-1"; diff --git a/src/test/java/org/opensearch/knn/index/IndexUtilTests.java b/src/test/java/org/opensearch/knn/index/IndexUtilTests.java index dc9c980e0..c8b29b6ef 100644 --- a/src/test/java/org/opensearch/knn/index/IndexUtilTests.java +++ b/src/test/java/org/opensearch/knn/index/IndexUtilTests.java @@ -12,6 +12,7 @@ package org.opensearch.knn.index; import com.google.common.collect.ImmutableMap; +import org.opensearch.Version; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.metadata.MappingMetadata; @@ -59,6 +60,7 @@ public void testGetLoadParameters() { Settings settings = Settings.builder().loadFromMap(indexSettings).build(); IndexMetadata indexMetadata = mock(IndexMetadata.class); when(indexMetadata.getSettings()).thenReturn(settings); + when(indexMetadata.getCreationVersion()).thenReturn(Version.CURRENT); Metadata metadata = mock(Metadata.class); when(metadata.index(anyString())).thenReturn(indexMetadata); ClusterState clusterState = mock(ClusterState.class); diff --git a/src/test/java/org/opensearch/knn/index/KNNMethodContextTests.java b/src/test/java/org/opensearch/knn/index/KNNMethodContextTests.java index 0b12980ab..1330e8da0 100644 --- a/src/test/java/org/opensearch/knn/index/KNNMethodContextTests.java +++ b/src/test/java/org/opensearch/knn/index/KNNMethodContextTests.java @@ -66,7 +66,7 @@ public void testStreams() throws IOException { public void testGetMethodComponent() { MethodComponentContext methodComponent = new MethodComponentContext("test-method", Collections.emptyMap()); KNNMethodContext knnMethodContext = new KNNMethodContext(KNNEngine.DEFAULT, SpaceType.DEFAULT, methodComponent); - assertEquals(methodComponent, knnMethodContext.getMethodComponent()); + assertEquals(methodComponent, knnMethodContext.getMethodComponentContext()); } /** @@ -275,7 +275,7 @@ public void testParse_nullParameters() throws IOException { .endObject(); Map in = xContentBuilderToMap(xContentBuilder); KNNMethodContext knnMethodContext = KNNMethodContext.parse(in); - assertTrue(knnMethodContext.getMethodComponent().getParameters().isEmpty()); + assertTrue(knnMethodContext.getMethodComponentContext().getParameters().isEmpty()); } /** @@ -291,8 +291,8 @@ public void testParse_valid() throws IOException { assertEquals(KNNEngine.DEFAULT, knnMethodContext.getKnnEngine()); assertEquals(SpaceType.DEFAULT, knnMethodContext.getSpaceType()); - assertEquals(methodName, knnMethodContext.getMethodComponent().getName()); - assertTrue(knnMethodContext.getMethodComponent().getParameters().isEmpty()); + assertEquals(methodName, knnMethodContext.getMethodComponentContext().getName()); + assertTrue(knnMethodContext.getMethodComponentContext().getParameters().isEmpty()); // Method with parameters String methodParameterKey1 = "p-1"; @@ -311,8 +311,8 @@ public void testParse_valid() throws IOException { in = xContentBuilderToMap(xContentBuilder); knnMethodContext = KNNMethodContext.parse(in); - assertEquals(methodParameterValue1, knnMethodContext.getMethodComponent().getParameters().get(methodParameterKey1)); - assertEquals(methodParameterValue2, knnMethodContext.getMethodComponent().getParameters().get(methodParameterKey2)); + assertEquals(methodParameterValue1, knnMethodContext.getMethodComponentContext().getParameters().get(methodParameterKey1)); + assertEquals(methodParameterValue2, knnMethodContext.getMethodComponentContext().getParameters().get(methodParameterKey2)); // Method with parameter that is a method context paramet @@ -330,12 +330,12 @@ public void testParse_valid() throws IOException { in = xContentBuilderToMap(xContentBuilder); knnMethodContext = KNNMethodContext.parse(in); - assertTrue(knnMethodContext.getMethodComponent().getParameters().get(methodParameterKey1) instanceof MethodComponentContext); + assertTrue(knnMethodContext.getMethodComponentContext().getParameters().get(methodParameterKey1) instanceof MethodComponentContext); assertEquals( methodParameterValue1, - ((MethodComponentContext) knnMethodContext.getMethodComponent().getParameters().get(methodParameterKey1)).getName() + ((MethodComponentContext) knnMethodContext.getMethodComponentContext().getParameters().get(methodParameterKey1)).getName() ); - assertEquals(methodParameterValue2, knnMethodContext.getMethodComponent().getParameters().get(methodParameterKey2)); + assertEquals(methodParameterValue2, knnMethodContext.getMethodComponentContext().getParameters().get(methodParameterKey2)); } /** diff --git a/src/test/java/org/opensearch/knn/index/KNNSettingsTests.java b/src/test/java/org/opensearch/knn/index/KNNSettingsTests.java index 4af62a4c4..4292b1a4f 100644 --- a/src/test/java/org/opensearch/knn/index/KNNSettingsTests.java +++ b/src/test/java/org/opensearch/knn/index/KNNSettingsTests.java @@ -136,6 +136,41 @@ public void testFilteredSearchAdvanceSetting_whenValuesProvidedByUsers_thenValid assertWarnings(); } + @SneakyThrows + public void testGetEfSearch_whenNoValuesProvidedByUsers_thenDefaultSettingsUsed() { + Node mockNode = createMockNode(Collections.emptyMap()); + mockNode.start(); + ClusterService clusterService = mockNode.injector().getInstance(ClusterService.class); + mockNode.client().admin().cluster().state(new ClusterStateRequest()).actionGet(); + mockNode.client().admin().indices().create(new CreateIndexRequest(INDEX_NAME)).actionGet(); + KNNSettings.state().setClusterService(clusterService); + + Integer efSearchValue = KNNSettings.getEfSearchParam(INDEX_NAME); + mockNode.close(); + assertEquals(KNNSettings.INDEX_KNN_DEFAULT_ALGO_PARAM_EF_SEARCH, efSearchValue); + assertWarnings(); + } + + @SneakyThrows + public void testGetEfSearch_whenEFSearchValueSetByUser_thenReturnValue() { + int userProvidedEfSearch = 300; + Node mockNode = createMockNode(Collections.emptyMap()); + mockNode.start(); + ClusterService clusterService = mockNode.injector().getInstance(ClusterService.class); + mockNode.client().admin().cluster().state(new ClusterStateRequest()).actionGet(); + final Settings settings = Settings.builder() + .put(KNNSettings.KNN_ALGO_PARAM_EF_SEARCH, userProvidedEfSearch) + .put(KNNSettings.KNN_INDEX, true) + .build(); + mockNode.client().admin().indices().create(new CreateIndexRequest(INDEX_NAME, settings)).actionGet(); + KNNSettings.state().setClusterService(clusterService); + + int efSearchValue = KNNSettings.getEfSearchParam(INDEX_NAME); + mockNode.close(); + assertEquals(userProvidedEfSearch, efSearchValue); + assertWarnings(); + } + private Node createMockNode(Map configSettings) throws IOException { Path configDir = createTempDir(); File configFile = configDir.resolve("opensearch.yml").toFile(); diff --git a/src/test/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumerTests.java b/src/test/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumerTests.java index 242aeaa17..eeca1e5ed 100644 --- a/src/test/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumerTests.java +++ b/src/test/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumerTests.java @@ -18,6 +18,7 @@ import org.apache.lucene.store.IOContext; import org.junit.AfterClass; import org.junit.BeforeClass; +import org.opensearch.Version; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; @@ -283,6 +284,7 @@ public void testAddKNNBinaryField_fromScratch_faissCurrent() throws IOException spaceType, new MethodComponentContext(METHOD_HNSW, ImmutableMap.of(METHOD_PARAMETER_M, 16, METHOD_PARAMETER_EF_CONSTRUCTION, 512)) ); + knnMethodContext.getMethodComponentContext().setIndexVersion(Version.CURRENT); String parameterString = XContentFactory.jsonBuilder().map(knnEngine.getMethodAsMap(knnMethodContext)).toString(); diff --git a/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java b/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java index 4ed231063..4a5db8c8f 100644 --- a/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java +++ b/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java @@ -93,7 +93,7 @@ public class KNNVectorFieldMapperTests extends KNNTestCase { public void testBuilder_getParameters() { String fieldName = "test-field-name"; ModelDao modelDao = mock(ModelDao.class); - KNNVectorFieldMapper.Builder builder = new KNNVectorFieldMapper.Builder(fieldName, modelDao); + KNNVectorFieldMapper.Builder builder = new KNNVectorFieldMapper.Builder(fieldName, modelDao, CURRENT); assertEquals(7, builder.getParameters().size()); List actualParams = builder.getParameters().stream().map(a -> a.name).collect(Collectors.toList()); @@ -104,7 +104,7 @@ public void testBuilder_getParameters() { public void testBuilder_build_fromKnnMethodContext() { // Check that knnMethodContext takes precedent over both model and legacy ModelDao modelDao = mock(ModelDao.class); - KNNVectorFieldMapper.Builder builder = new KNNVectorFieldMapper.Builder("test-field-name-1", modelDao); + KNNVectorFieldMapper.Builder builder = new KNNVectorFieldMapper.Builder("test-field-name-1", modelDao, CURRENT); SpaceType spaceType = SpaceType.COSINESIMIL; int m = 17; @@ -141,7 +141,7 @@ public void testBuilder_build_fromKnnMethodContext() { public void testBuilder_build_fromModel() { // Check that modelContext takes precedent over legacy ModelDao modelDao = mock(ModelDao.class); - KNNVectorFieldMapper.Builder builder = new KNNVectorFieldMapper.Builder("test-field-name-1", modelDao); + KNNVectorFieldMapper.Builder builder = new KNNVectorFieldMapper.Builder("test-field-name-1", modelDao, CURRENT); SpaceType spaceType = SpaceType.COSINESIMIL; int m = 17; @@ -179,7 +179,7 @@ public void testBuilder_build_fromModel() { public void testBuilder_build_fromLegacy() { // Check legacy is picked up if model context and method context are not set ModelDao modelDao = mock(ModelDao.class); - KNNVectorFieldMapper.Builder builder = new KNNVectorFieldMapper.Builder("test-field-name-1", modelDao); + KNNVectorFieldMapper.Builder builder = new KNNVectorFieldMapper.Builder("test-field-name-1", modelDao, CURRENT); SpaceType spaceType = SpaceType.COSINESIMIL; int m = 17; @@ -237,10 +237,10 @@ public void testBuilder_parse_fromKnnMethodContext_luceneEngine() throws IOExcep Mapper.BuilderContext builderContext = new Mapper.BuilderContext(settings, new ContentPath()); builder.build(builderContext); - assertEquals(METHOD_HNSW, builder.knnMethodContext.get().getMethodComponent().getName()); + assertEquals(METHOD_HNSW, builder.knnMethodContext.get().getMethodComponentContext().getName()); assertEquals( efConstruction, - builder.knnMethodContext.get().getMethodComponent().getParameters().get(METHOD_PARAMETER_EF_CONSTRUCTION) + builder.knnMethodContext.get().getMethodComponentContext().getParameters().get(METHOD_PARAMETER_EF_CONSTRUCTION) ); assertTrue(KNNEngine.LUCENE.isInitialized()); @@ -260,8 +260,8 @@ public void testBuilder_parse_fromKnnMethodContext_luceneEngine() throws IOExcep buildParserContext(indexName, settings) ); - assertEquals(METHOD_HNSW, builder.knnMethodContext.get().getMethodComponent().getName()); - assertTrue(builderEmptyParams.knnMethodContext.get().getMethodComponent().getParameters().isEmpty()); + assertEquals(METHOD_HNSW, builder.knnMethodContext.get().getMethodComponentContext().getName()); + assertTrue(builderEmptyParams.knnMethodContext.get().getMethodComponentContext().getParameters().isEmpty()); XContentBuilder xContentBuilderUnsupportedParam = XContentFactory.jsonBuilder() .startObject() @@ -485,10 +485,10 @@ public void testTypeParser_parse_fromKnnMethodContext() throws IOException { buildParserContext(indexName, settings) ); - assertEquals(METHOD_HNSW, builder.knnMethodContext.get().getMethodComponent().getName()); + assertEquals(METHOD_HNSW, builder.knnMethodContext.get().getMethodComponentContext().getName()); assertEquals( efConstruction, - builder.knnMethodContext.get().getMethodComponent().getParameters().get(METHOD_PARAMETER_EF_CONSTRUCTION) + builder.knnMethodContext.get().getMethodComponentContext().getParameters().get(METHOD_PARAMETER_EF_CONSTRUCTION) ); // Test invalid parameter diff --git a/src/test/java/org/opensearch/knn/index/util/FaissTests.java b/src/test/java/org/opensearch/knn/index/util/FaissTests.java index 99a153780..0e7bc6482 100644 --- a/src/test/java/org/opensearch/knn/index/util/FaissTests.java +++ b/src/test/java/org/opensearch/knn/index/util/FaissTests.java @@ -5,6 +5,7 @@ package org.opensearch.knn.index.util; +import org.opensearch.Version; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.knn.KNNTestCase; @@ -48,6 +49,7 @@ public void testGetMethodAsMap_whenMethodIsHNSWFlat_thenCreateCorrectIndexDescri .endObject(); Map in = xContentBuilderToMap(xContentBuilder); KNNMethodContext knnMethodContext = KNNMethodContext.parse(in); + knnMethodContext.getMethodComponentContext().setIndexVersion(Version.CURRENT); Map map = Faiss.INSTANCE.getMethodAsMap(knnMethodContext); @@ -76,6 +78,7 @@ public void testGetMethodAsMap_whenMethodIsHNSWPQ_thenCreateCorrectIndexDescript .endObject(); Map in = xContentBuilderToMap(xContentBuilder); KNNMethodContext knnMethodContext = KNNMethodContext.parse(in); + knnMethodContext.getMethodComponentContext().setIndexVersion(Version.CURRENT); Map map = Faiss.INSTANCE.getMethodAsMap(knnMethodContext); diff --git a/src/test/java/org/opensearch/knn/index/util/IndexHyperParametersUtilTests.java b/src/test/java/org/opensearch/knn/index/util/IndexHyperParametersUtilTests.java new file mode 100644 index 000000000..508b8765c --- /dev/null +++ b/src/test/java/org/opensearch/knn/index/util/IndexHyperParametersUtilTests.java @@ -0,0 +1,44 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.knn.index.util; + +import junit.framework.TestCase; +import org.junit.Assert; +import org.opensearch.Version; +import org.opensearch.knn.index.KNNSettings; + +public class IndexHyperParametersUtilTests extends TestCase { + + public void testLombokNonNull() { + Assert.assertThrows(NullPointerException.class, () -> IndexHyperParametersUtil.getHNSWEFConstructionValue(null)); + Assert.assertThrows(NullPointerException.class, () -> IndexHyperParametersUtil.getHNSWEFSearchValue(null)); + } + + public void testGetHNSWEFConstructionValue_withDifferentValues_thenSuccess() { + Assert.assertEquals(512, IndexHyperParametersUtil.getHNSWEFConstructionValue(Version.V_2_11_0)); + Assert.assertEquals(512, IndexHyperParametersUtil.getHNSWEFConstructionValue(Version.V_2_3_0)); + Assert.assertEquals( + KNNSettings.INDEX_KNN_DEFAULT_ALGO_PARAM_EF_CONSTRUCTION.intValue(), + IndexHyperParametersUtil.getHNSWEFConstructionValue(Version.CURRENT) + ); + + } + + public void testGetHNSWEFSearchValue_withDifferentValues_thenSuccess() { + Assert.assertEquals(512, IndexHyperParametersUtil.getHNSWEFConstructionValue(Version.V_2_11_0)); + Assert.assertEquals(512, IndexHyperParametersUtil.getHNSWEFConstructionValue(Version.V_2_3_0)); + Assert.assertEquals( + KNNSettings.INDEX_KNN_DEFAULT_ALGO_PARAM_EF_SEARCH.intValue(), + IndexHyperParametersUtil.getHNSWEFConstructionValue(Version.CURRENT) + ); + } +} diff --git a/src/test/java/org/opensearch/knn/jni/JNIServiceTests.java b/src/test/java/org/opensearch/knn/jni/JNIServiceTests.java index d1a5be741..185f2953d 100644 --- a/src/test/java/org/opensearch/knn/jni/JNIServiceTests.java +++ b/src/test/java/org/opensearch/knn/jni/JNIServiceTests.java @@ -14,6 +14,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import org.junit.BeforeClass; +import org.opensearch.Version; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.knn.KNNTestCase; @@ -840,6 +841,7 @@ public void testTrain_whenConfigurationIsHNSWPQ_thenSucceed() throws IOException .endObject(); Map in = xContentBuilderToMap(xContentBuilder); KNNMethodContext knnMethodContext = KNNMethodContext.parse(in); + knnMethodContext.getMethodComponentContext().setIndexVersion(Version.CURRENT); Map parameters = KNNEngine.FAISS.getMethodAsMap(knnMethodContext); byte[] faissIndex = JNIService.trainIndex(parameters, 128, trainPointer, FAISS_NAME); diff --git a/src/test/java/org/opensearch/knn/training/TrainingJobTests.java b/src/test/java/org/opensearch/knn/training/TrainingJobTests.java index daf9645fa..6b07b2dd2 100644 --- a/src/test/java/org/opensearch/knn/training/TrainingJobTests.java +++ b/src/test/java/org/opensearch/knn/training/TrainingJobTests.java @@ -12,8 +12,9 @@ package org.opensearch.knn.training; import com.google.common.collect.ImmutableMap; +import org.opensearch.Version; +import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.knn.KNNTestCase; -import org.opensearch.knn.jni.JNIService; import org.opensearch.knn.index.KNNMethodContext; import org.opensearch.knn.index.MethodComponentContext; import org.opensearch.knn.index.SpaceType; @@ -24,6 +25,7 @@ import org.opensearch.knn.indices.Model; import org.opensearch.knn.indices.ModelMetadata; import org.opensearch.knn.indices.ModelState; +import org.opensearch.knn.jni.JNIService; import java.io.File; import java.io.IOException; @@ -39,6 +41,16 @@ public class TrainingJobTests extends KNNTestCase { + private final String trainingIndexName = "trainingindexname"; + + @Override + public void setUp() throws Exception { + super.setUp(); + DiscoveryNode mockedDiscoveryNode = mock(DiscoveryNode.class); + when(clusterService.localNode()).thenReturn(mockedDiscoveryNode); + when(mockedDiscoveryNode.getVersion()).thenReturn(Version.CURRENT); + } + public void testGetModelId() { String modelId = "test-model-id"; KNNMethodContext knnMethodContext = mock(KNNMethodContext.class); @@ -149,6 +161,8 @@ public void testRun_success() throws IOException, ExecutionException { NativeMemoryEntryContext.TrainingDataEntryContext.class ); when(trainingDataEntryContext.getKey()).thenReturn(tdataKey); + when(trainingDataEntryContext.getTrainIndexName()).thenReturn(trainingIndexName); + when(trainingDataEntryContext.getClusterService()).thenReturn(clusterService); when(nativeMemoryCacheManager.get(trainingDataEntryContext, false)).thenReturn(nativeMemoryAllocation); doAnswer(invocationOnMock -> { diff --git a/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java b/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java index 57158be6f..70605b599 100644 --- a/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java +++ b/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java @@ -15,6 +15,7 @@ import org.opensearch.common.xcontent.XContentHelper; import org.opensearch.core.xcontent.DeprecationHandler; import org.opensearch.core.xcontent.MediaType; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.index.query.MatchAllQueryBuilder; @@ -50,6 +51,7 @@ import javax.management.remote.JMXServiceURL; import java.io.IOException; +import java.io.InputStream; import java.net.URL; import java.nio.file.Files; import java.nio.file.Path; @@ -72,6 +74,7 @@ import static org.opensearch.knn.common.KNNConstants.KNN_ENGINE; import static org.opensearch.knn.common.KNNConstants.KNN_METHOD; import static org.opensearch.knn.common.KNNConstants.METHOD_ENCODER_PARAMETER; +import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_EF_SEARCH; import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_NLIST; import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_SPACE_TYPE; import static org.opensearch.knn.common.KNNConstants.MODEL_BLOB_PARAMETER; @@ -698,11 +701,26 @@ protected int getTotalGraphsInCache() throws IOException { * Get specific Index setting value from response */ protected String getIndexSettingByName(String indexName, String settingName) throws IOException { - @SuppressWarnings("unchecked") - Map settings = (Map) ((Map) getIndexSettings(indexName).get(indexName)).get( - "settings" - ); - return (String) settings.get(settingName); + return getIndexSettingByName(indexName, settingName, false); + } + + protected String getIndexSettingByName(String indexName, String settingName, boolean includeDefaults) throws IOException { + Request request = new Request("GET", "/" + indexName + "/_settings"); + if (includeDefaults) { + request.addParameter("include_defaults", "true"); + } + request.addParameter("flat_settings", "true"); + Response response = client().performRequest(request); + try (InputStream is = response.getEntity().getContent()) { + Map settings = (Map) XContentHelper.convertToMap(MediaTypeRegistry.JSON.xContent(), is, true) + .get(indexName); + Map defaultSettings = new HashMap<>(); + if (includeDefaults) { + defaultSettings = (Map) settings.get("defaults"); + } + Map userSettings = (Map) settings.get("settings"); + return (String) (userSettings.get(settingName) == null ? defaultSettings.get(settingName) : userSettings.get(settingName)); + } } protected void createModelSystemIndex() throws IOException { @@ -1072,6 +1090,37 @@ public String createKNNIndexCustomMethodFieldMapping( .toString(); } + public String createKNNIndexCustomMethodFieldMapping( + String fieldName, + Integer dimensions, + SpaceType spaceType, + String engine, + Integer m, + Integer ef_construction, + Integer ef_search + ) throws IOException { + return XContentFactory.jsonBuilder() + .startObject() + .startObject(PROPERTIES) + .startObject(fieldName) + .field(VECTOR_TYPE, KNN_VECTOR) + .field(DIMENSION, dimensions.toString()) + .startObject(KNN_METHOD) + .field(NAME, METHOD_HNSW) + .field(METHOD_PARAMETER_SPACE_TYPE, spaceType.getValue()) + .field(KNN_ENGINE, engine) + .startObject(PARAMETERS) + .field(METHOD_PARAMETER_EF_CONSTRUCTION, ef_construction) + .field(METHOD_PARAMETER_M, m) + .field(METHOD_PARAMETER_EF_SEARCH, ef_search) + .endObject() + .endObject() + .endObject() + .endObject() + .endObject() + .toString(); + } + // Default KNN script score settings protected Settings createKNNDefaultScriptScoreSettings() { return Settings.builder().put(NUMBER_OF_SHARDS, 1).put(NUMBER_OF_REPLICAS, 0).put(INDEX_KNN, false).build(); From 46d001d1a184569184b8eb20a45c908ebc8d6353 Mon Sep 17 00:00:00 2001 From: John Mazanec Date: Wed, 3 Jan 2024 10:07:38 -0800 Subject: [PATCH 173/416] Install security plugin from individual artifacts (#1369) Changes how security tests are executed. Instead of setting up docker container with security enabled, we now can directly spin up a gradle local cluster with security which we can use to run tests against. To enable this option, we just have to pass `-Dsecurity.enabled=true` as a flag. Along with this, some refactoring was done for the ODFERestTestCase for configuring the client and cleaning up. Signed-off-by: John Mazanec --- .github/workflows/CI.yml | 40 ------ .github/workflows/test_security.yml | 92 +++--------- CHANGELOG.md | 2 +- DEVELOPER_GUIDE.md | 39 +++++- build.gradle | 132 ++++++++++++++++-- .../java/org/opensearch/knn/bwc/ModelIT.java | 28 +--- src/test/resources/security/sample.pem | 25 ---- src/test/resources/security/test-kirk.jks | Bin 4504 -> 0 bytes .../org/opensearch/knn/KNNRestTestCase.java | 34 ----- .../org/opensearch/knn/ODFERestTestCase.java | 119 +++------------- 10 files changed, 207 insertions(+), 304 deletions(-) delete mode 100644 src/test/resources/security/sample.pem delete mode 100644 src/test/resources/security/test-kirk.jks diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index c807e348e..b58741b92 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -125,43 +125,3 @@ jobs: run: | ./gradlew.bat build -# - name: Pull and Run Docker for security tests -# run: | -# plugin=`ls build/distributions/*.zip` -# version=`echo $plugin|awk -F- '{print $4}'| cut -d. -f 1-3` -# plugin_version=`echo $plugin|awk -F- '{print $4}'| cut -d. -f 1-4` -# echo $version -# cd .. -# if docker pull opendistroforelasticsearch/opendistroforelasticsearch:$version -# then -# echo "FROM opendistroforelasticsearch/opendistroforelasticsearch:$version" >> Dockerfile -# echo "RUN if [ -d /usr/share/elasticsearch/plugins/opendistro-knn ]; then /usr/share/elasticsearch/bin/elasticsearch-plugin remove opendistro-knn; fi" >> Dockerfile -# echo "RUN yum -y update \ && yum -y groupinstall "Development Tools" \ && yum install -y unzip glibc.x86_64 cmake \ && yum clean all" >> Dockerfile -# echo "RUN git clone --recursive --branch ${GITHUB_REF##*/} https://github.com/opendistro-for-elasticsearch/k-NN.git /usr/share/elasticsearch/k-NN \ " >> Dockerfile -# echo "&& cd /usr/share/elasticsearch/k-NN/jni \ && sed -i 's/-march=native/-march=x86-64/g' external/nmslib/similarity_search/CMakeLists.txt \ && cmake . \ && make \ " >> Dockerfile -# echo "&& mkdir /tmp/jni/ && cp release/*.so /tmp/jni/ && ls -ltr /tmp/jni/ \ && cp /tmp/jni/libKNNIndex*.so /usr/lib \ && rm -rf /usr/share/elasticsearch/k-NN" >> Dockerfile -# echo "RUN cd /usr/share/elasticsearch/" >> Dockerfile -# echo "ADD k-NN/build/distributions/opendistro-knn-$plugin_version.zip /tmp/" >> Dockerfile -# echo "RUN /usr/share/elasticsearch/bin/elasticsearch-plugin install --batch file:/tmp/opendistro-knn-$plugin_version.zip" >> Dockerfile -# docker build -t odfe-knn:test . -# echo "imagePresent=true" >> $GITHUB_ENV -# else -# echo "imagePresent=false" >> $GITHUB_ENV -# fi -# - name: Run Docker Image -# if: env.imagePresent == 'true' -# run: | -# cd .. -# docker run -p 9200:9200 -d -p 9600:9600 -e "discovery.type=single-node" odfe-knn:test -# sleep 90 -# - name: Run k-NN Test -# if: env.imagePresent == 'true' -# run: | -# security=`curl -XGET https://localhost:9200/_cat/plugins?v -u admin:admin --insecure |grep opendistro_security|wc -l` -# if [ $security -gt 0 ] -# then -# echo "Security plugin is available. Running tests in security mode" -# ./gradlew :integTest -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername="docker-cluster" -Dhttps=true -Duser=admin -Dpassword=admin -# else -# echo "Security plugin is NOT available. Skipping tests as they are already ran part of ./gradlew build" -# fi diff --git a/.github/workflows/test_security.yml b/.github/workflows/test_security.yml index ff3ca1459..4d0374a08 100644 --- a/.github/workflows/test_security.yml +++ b/.github/workflows/test_security.yml @@ -12,87 +12,39 @@ on: - "feature/**" jobs: - Build-ad: + Get-CI-Image-Tag: + uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@main + with: + product: opensearch + + integ-test-with-security-linux: strategy: matrix: - java: [ 11,17,21 ] - os: [ubuntu-latest] - fail-fast: true + java: [11, 17, 21] - name: Test k-NN on Secure Cluster - runs-on: ${{ matrix.os }} + name: Run Integration Tests on Linux + runs-on: ubuntu-latest + needs: Get-CI-Image-Tag + container: + # using the same image which is used by opensearch-build team to build the OpenSearch Distribution + # this image tag is subject to change as more dependencies and updates will arrive over time + image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }} + # need to switch to root so that github actions can install runner binary on container without permission issues. + options: --user root steps: - name: Checkout k-NN uses: actions/checkout@v1 + with: + submodules: true - name: Setup Java ${{ matrix.java }} uses: actions/setup-java@v1 with: java-version: ${{ matrix.java }} - - name: Install dependencies on ubuntu - if: startsWith(matrix.os,'ubuntu') - run: | - sudo apt-get install libopenblas-dev gfortran -y - - - name: Assemble k-NN - run: | - ./gradlew assemble - # example of variables: - # plugin = opensearch-knn-2.7.0.0-SNAPSHOT.zip - # version = 2.7.0 - # plugin_version = 2.7.0.0 - # qualifier = `SNAPSHOT` - - name: Pull and Run Docker - run: | - plugin=`basename $(ls build/distributions/*.zip)` - version=`echo $plugin|awk -F- '{print $3}'| cut -d. -f 1-3` - plugin_version=`echo $plugin|awk -F- '{print $3}'| cut -d. -f 1-4` - qualifier=`echo $plugin|awk -F- '{print $4}'| cut -d. -f 1-1` - if [ $qualifier != `SNAPSHOT` ]; - then - docker_version=$version-$qualifier - else - docker_version=$version - fi - echo plugin version plugin_version qualifier docker_version - echo "($plugin) ($version) ($plugin_version) ($qualifier) ($docker_version)" - - cd .. - if docker pull opensearchstaging/opensearch:$docker_version - then - echo "FROM opensearchstaging/opensearch:$docker_version" >> Dockerfile - # knn plugin cannot be deleted until there are plugin that has dependency on it - echo "RUN if [ -d /usr/share/opensearch/plugins/opensearch-neural-search ]; then /usr/share/opensearch/bin/opensearch-plugin remove opensearch-neural-search; fi" >> Dockerfile - echo "RUN if [ -d /usr/share/opensearch/plugins/opensearch-performance-analyzer ]; then /usr/share/opensearch/bin/opensearch-plugin remove opensearch-performance-analyzer; fi" >> Dockerfile - # saving pre-built artifacts of native libraries as we can't build it with gradle assemle - echo "RUN if [ -d /usr/share/opensearch/plugins/opensearch-knn ]; then cp -r /usr/share/opensearch/plugins/opensearch-knn/lib /usr/share/opensearch/knn-libs; fi" >> Dockerfile - echo "RUN if [ -d /usr/share/opensearch/plugins/opensearch-knn ]; then /usr/share/opensearch/bin/opensearch-plugin remove opensearch-knn; fi" >> Dockerfile - echo "ADD k-NN/build/distributions/$plugin /tmp/" >> Dockerfile - echo "RUN /usr/share/opensearch/bin/opensearch-plugin install --batch file:/tmp/$plugin" >> Dockerfile - # moving pre-built artifacts of native libraries back to plugin folder - echo "RUN if [ -d /usr/share/opensearch/knn-libs ]; then mv /usr/share/opensearch/knn-libs /usr/share/opensearch/plugins/opensearch-knn/lib; fi" >> Dockerfile - docker build -t opensearch-knn:test . - echo "imagePresent=true" >> $GITHUB_ENV - else - echo "imagePresent=false" >> $GITHUB_ENV - fi - - - name: Run Docker Image - if: env.imagePresent == 'true' - run: | - cd .. - docker run -p 9200:9200 -d -p 9600:9600 -e "discovery.type=single-node" opensearch-knn:test - sleep 90 - - name: Run k-NN Integ Test - if: env.imagePresent == 'true' + - name: Run build + # switching the user, as OpenSearch cluster can only be started as root/Administrator on linux-deb/linux-rpm/windows-zip. run: | - security=`curl -XGET https://localhost:9200/_cat/plugins?v -u admin:admin --insecure |grep opensearch-security|wc -l` - if [ $security -gt 0 ] - then - echo "Security plugin is available" - ./gradlew integTest -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername="docker-cluster" -Dhttps=true -Duser=admin -Dpassword=admin - else - echo "Security plugin is NOT available, skipping integration tests" - fi + chown -R 1000:1000 `pwd` + su `id -un 1000` -c "whoami && java -version && ./gradlew integTest -Dsecurity.enabled=true" diff --git a/CHANGELOG.md b/CHANGELOG.md index bf61d49e7..7839d9825 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,9 +22,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), * Fix use-after-free case on nmslib search path [#1305](https://github.com/opensearch-project/k-NN/pull/1305) * Allow nested knn field mapping when train model [#1318](https://github.com/opensearch-project/k-NN/pull/1318) * Properly designate model state for actively training models when nodes crash or leave cluster [#1317](https://github.com/opensearch-project/k-NN/pull/1317) - ### Infrastructure * Upgrade gradle to 8.4 [1289](https://github.com/opensearch-project/k-NN/pull/1289) +* Refactor security testing to install from individual components [#1307](https://github.com/opensearch-project/k-NN/pull/1307) ### Documentation ### Maintenance * Update developer guide to include M1 Setup [#1222](https://github.com/opensearch-project/k-NN/pull/1222) diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index cbfaf0f4c..2e7f7322c 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -264,6 +264,38 @@ curl localhost:9200 } } ``` + +Additionally, it is also possible to run a cluster with security enabled: +```shell script +./gradlew run -Dsecurity.enabled=true -Dhttps=true -Duser=admin -Dpassword=admin +``` + +By default, if `-Dsecurity.enabled=true` is passed the following defaults will be used: `https=true`, `user=admin` and +`password=admin`. + +Then, to access the cluster, we can run +```bash +curl https://localhost:9200 --insecure -u admin:admin + +{ + "name" : "integTest-0", + "cluster_name" : "integTest", + "cluster_uuid" : "kLsNk4JDTMyp1yQRqog-3g", + "version" : { + "distribution" : "opensearch", + "number" : "3.0.0-SNAPSHOT", + "build_type" : "tar", + "build_hash" : "9d85e566894ef53e5f2093618b3d455e4d0a04ce", + "build_date" : "2023-10-30T18:34:06.996519Z", + "build_snapshot" : true, + "lucene_version" : "9.8.0", + "minimum_wire_compatibility_version" : "2.12.0", + "minimum_index_compatibility_version" : "2.0.0" + }, + "tagline" : "The OpenSearch Project: https://opensearch.org/" +} +``` + ### Run Multi-node Cluster Locally It can be useful to test and debug on a multi-node cluster. In order to launch a 3 node cluster with the KNN plugin installed, run the following command: @@ -272,12 +304,17 @@ It can be useful to test and debug on a multi-node cluster. In order to launch a ./gradlew run -PnumNodes=3 ``` -In order to run the integration tests with a 3 node cluster, run this command: +In order to run the integration tests, run this command: ``` ./gradlew :integTest -PnumNodes=3 ``` +Additionally, to run integration tests with security enabled, run +``` +./gradlew :integTest -Dsecurity.enabled=true -PnumNodes=3 +``` + Integration tests can be run with remote cluster. For that run the following command and replace host/port/cluster name values with ones for the target cluster: ``` diff --git a/build.gradle b/build.gradle index ceba9a038..2ed70a66e 100644 --- a/build.gradle +++ b/build.gradle @@ -4,7 +4,10 @@ */ import org.opensearch.gradle.test.RestIntegTestTask +import org.opensearch.gradle.testclusters.OpenSearchCluster import org.apache.tools.ant.taskdefs.condition.Os +import java.nio.file.Paths +import java.util.concurrent.Callable buildscript { ext { @@ -13,6 +16,19 @@ buildscript { opensearch_version = System.getProperty("opensearch.version", "2.12.0-SNAPSHOT") version_qualifier = System.getProperty("build.version_qualifier", "") opensearch_group = "org.opensearch" + isSnapshot = "true" == System.getProperty("build.snapshot", "true") + + version_tokens = opensearch_version.tokenize('-') + opensearch_build = version_tokens[0] + '.0' + plugin_no_snapshot = opensearch_build + if (version_qualifier) { + opensearch_build += "-${version_qualifier}" + plugin_no_snapshot += "-${version_qualifier}" + } + if (isSnapshot) { + opensearch_build += "-SNAPSHOT" + } + opensearch_no_snapshot = opensearch_build.replace("-SNAPSHOT","") } // This isn't applying from repositories.gradle so repeating git diff it here @@ -43,6 +59,7 @@ plugins { id 'idea' id "com.diffplug.spotless" version "6.20.0" apply false id 'io.freefair.lombok' version '8.4' + id "de.undercouch.download" version "5.3.0" } apply from: 'gradle/formatting.gradle' @@ -51,9 +68,91 @@ apply plugin: 'opensearch.rest-test' apply plugin: 'opensearch.pluginzip' apply plugin: 'opensearch.repositories' + +def opensearch_tmp_dir = rootProject.file('build/private/opensearch_tmp').absoluteFile +opensearch_tmp_dir.mkdirs() + ext { - isSnapshot = "true" == System.getProperty("build.snapshot", "true") projectSubstitutions = [:] + + configureSecurityPlugin = { OpenSearchCluster cluster -> + configurations.zipArchive.asFileTree.each { + cluster.plugin(provider(new Callable() { + @Override + RegularFile call() throws Exception { + return new RegularFile() { + @Override + File getAsFile() { + return it + } + } + } + })) + } + + cluster.getNodes().forEach { node -> + var creds = node.getCredentials() + if (creds.isEmpty()) { + creds.add(Map.of('username', 'admin', 'password', 'admin')) + } else { + creds.get(0).putAll(Map.of('username', 'admin', 'password', 'admin')) + } + } + + // Config below including files are copied from security demo configuration + ['esnode.pem', 'esnode-key.pem', 'root-ca.pem'].forEach { file -> + File local = Paths.get(opensearch_tmp_dir.absolutePath, file).toFile() + download.run { + src "https://raw.githubusercontent.com/opensearch-project/security/main/bwc-test/src/test/resources/security/" + file + dest local + overwrite false + } + cluster.extraConfigFile(file, local) + } + + // This configuration is copied from the security plugins demo install: + // https://github.com/opensearch-project/security/blob/2.11.1.0/tools/install_demo_configuration.sh#L365-L388 + cluster.setting("plugins.security.ssl.transport.pemcert_filepath", "esnode.pem") + cluster.setting("plugins.security.ssl.transport.pemkey_filepath", "esnode-key.pem") + cluster.setting("plugins.security.ssl.transport.pemtrustedcas_filepath", "root-ca.pem") + cluster.setting("plugins.security.ssl.transport.enforce_hostname_verification", "false") + cluster.setting("plugins.security.ssl.http.enabled", "true") + cluster.setting("plugins.security.ssl.http.pemcert_filepath", "esnode.pem") + cluster.setting("plugins.security.ssl.http.pemkey_filepath", "esnode-key.pem") + cluster.setting("plugins.security.ssl.http.pemtrustedcas_filepath", "root-ca.pem") + cluster.setting("plugins.security.allow_unsafe_democertificates", "true") + cluster.setting("plugins.security.allow_default_init_securityindex", "true") + cluster.setting("plugins.security.unsupported.inject_user.enabled", "true") + + cluster.setting("plugins.security.authcz.admin_dn", "\n- CN=kirk,OU=client,O=client,L=test, C=de") + cluster.setting('plugins.security.restapi.roles_enabled', '["all_access", "security_rest_api_access"]') + cluster.setting('plugins.security.system_indices.enabled', "true") + cluster.setting('plugins.security.system_indices.indices', '[' + + '".plugins-ml-config", ' + + '".plugins-ml-connector", ' + + '".plugins-ml-model-group", ' + + '".plugins-ml-model", ".plugins-ml-task", ' + + '".plugins-ml-conversation-meta", ' + + '".plugins-ml-conversation-interactions", ' + + '".opendistro-alerting-config", ' + + '".opendistro-alerting-alert*", ' + + '".opendistro-anomaly-results*", ' + + '".opendistro-anomaly-detector*", ' + + '".opendistro-anomaly-checkpoints", ' + + '".opendistro-anomaly-detection-state", ' + + '".opendistro-reports-*", ' + + '".opensearch-notifications-*", ' + + '".opensearch-notebooks", ' + + '".opensearch-observability", ' + + '".ql-datasources", ' + + '".opendistro-asynchronous-search-response*", ' + + '".replication-metadata-store", ' + + '".opensearch-knn-models", ' + + '".geospatial-ip2geo-data*"' + + ']' + ) + cluster.setSecure(true) + } } allprojects { @@ -87,6 +186,10 @@ allprojects { } } +configurations { + zipArchive +} + publishing { repositories { maven { @@ -182,11 +285,9 @@ dependencies { testImplementation group: 'org.objenesis', name: 'objenesis', version: '3.2' testImplementation group: 'net.bytebuddy', name: 'byte-buddy-agent', version: '1.14.7' testFixturesImplementation "org.opensearch:common-utils:${version}" -} - -def opensearch_tmp_dir = rootProject.file('build/private/opensearch_tmp').absoluteFile -opensearch_tmp_dir.mkdirs() + zipArchive group: 'org.opensearch.plugin', name:'opensearch-security', version: "${opensearch_build}" +} task windowsPatches(type:Exec) { commandLine 'cmd', '/c', "Powershell -File $rootDir\\scripts\\windowsScript.ps1" @@ -232,9 +333,18 @@ integTest { // allows integration test classes to access test resource from project root path systemProperty('project.root', project.rootDir.absolutePath) - systemProperty "https", System.getProperty("https") - systemProperty "user", System.getProperty("user") - systemProperty "password", System.getProperty("password") + var is_https = System.getProperty("https") + var user = System.getProperty("user") + var password = System.getProperty("password") + if (System.getProperty("security.enabled") != null) { + // If security is enabled, set is_https/user/password defaults + is_https = is_https == null ? "true" : is_https + user = user == null ? "admin" : user + password = password == null ? "admin" : password + } + systemProperty("https", is_https) + systemProperty("user", user) + systemProperty("password", password) doFirst { // Tell the test JVM if the cluster JVM is running under a debugger so that tests can @@ -258,6 +368,12 @@ integTest { testClusters.integTest { testDistribution = "ARCHIVE" + + // Optionally install security + if (System.getProperty("security.enabled") != null) { + configureSecurityPlugin(testClusters.integTest) + } + plugin(project.tasks.bundlePlugin.archiveFile) if (Os.isFamily(Os.FAMILY_WINDOWS)) { // Add the paths of built JNI libraries and its dependent libraries to PATH variable in System variables diff --git a/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/ModelIT.java b/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/ModelIT.java index 0ef9c4c91..be20d18a2 100644 --- a/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/ModelIT.java +++ b/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/ModelIT.java @@ -10,21 +10,16 @@ import org.opensearch.action.search.SearchResponse; import org.opensearch.client.Request; import org.opensearch.client.Response; -import org.opensearch.client.ResponseException; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; import org.opensearch.knn.index.SpaceType; -import org.opensearch.knn.index.util.KNNEngine; -import org.opensearch.knn.indices.ModelMetadata; -import org.opensearch.knn.indices.ModelState; import org.opensearch.knn.plugin.KNNPlugin; import org.opensearch.core.rest.RestStatus; import org.opensearch.search.SearchHit; import java.io.IOException; -import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.Map; @@ -55,7 +50,7 @@ public class ModelIT extends AbstractRestartUpgradeTestCase { private static int DOC_ID_TEST_MODEL_INDEX = 0; private static int DOC_ID_TEST_MODEL_INDEX_DEFAULT = 0; private static final int DELAY_MILLI_SEC = 1000; - private static final int EXP_NUM_OF_MODELS = 3; + private static final int EXP_NUM_OF_MODELS = 2; private static final int K = 5; private static final int NUM_DOCS = 10; private static final int NUM_DOCS_TEST_MODEL_INDEX = 100; @@ -66,7 +61,6 @@ public class ModelIT extends AbstractRestartUpgradeTestCase { private static int QUERY_COUNT_TEST_MODEL_INDEX_DEFAULT = 0; private static final String TEST_MODEL_ID = "test-model-id"; private static final String TEST_MODEL_ID_DEFAULT = "test-model-id-default"; - private static final String TEST_MODEL_ID_TRAINING = "test-model-id-training"; private static final String MODEL_DESCRIPTION = "Description for train model test"; // KNN model test @@ -139,22 +133,6 @@ public void testKNNModelDefault() throws Exception { } } - // KNN Delete Model test for model in Training State - public void testDeleteTrainingModel() throws Exception { - byte[] testModelBlob = "hello".getBytes(StandardCharsets.UTF_8); - ModelMetadata testModelMetadata = getModelMetadata(); - testModelMetadata.setState(ModelState.TRAINING); - if (isRunningAgainstOldCluster()) { - addModelToSystemIndex(TEST_MODEL_ID_TRAINING, testModelMetadata, testModelBlob); - } else { - String restURI = String.join("/", KNNPlugin.KNN_BASE_URI, MODELS, TEST_MODEL_ID_TRAINING); - Request request = new Request("DELETE", restURI); - - ResponseException ex = expectThrows(ResponseException.class, () -> client().performRequest(request)); - assertEquals(RestStatus.CONFLICT.getStatus(), ex.getResponse().getStatusLine().getStatusCode()); - } - } - // Delete Models and ".opensearch-knn-models" index to clear cluster metadata @AfterClass public static void wipeAllModels() throws IOException { @@ -255,8 +233,4 @@ public String modelIndexMapping(String fieldName, String modelId) throws IOExcep .endObject() .toString(); } - - private ModelMetadata getModelMetadata() { - return new ModelMetadata(KNNEngine.DEFAULT, SpaceType.DEFAULT, 4, ModelState.CREATED, "2021-03-27", "test model", "", ""); - } } diff --git a/src/test/resources/security/sample.pem b/src/test/resources/security/sample.pem deleted file mode 100644 index a1fc20a77..000000000 --- a/src/test/resources/security/sample.pem +++ /dev/null @@ -1,25 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIEPDCCAySgAwIBAgIUZjrlDPP8azRDPZchA/XEsx0X2iIwDQYJKoZIhvcNAQEL -BQAwgY8xEzARBgoJkiaJk/IsZAEZFgNjb20xFzAVBgoJkiaJk/IsZAEZFgdleGFt -cGxlMRkwFwYDVQQKDBBFeGFtcGxlIENvbSBJbmMuMSEwHwYDVQQLDBhFeGFtcGxl -IENvbSBJbmMuIFJvb3QgQ0ExITAfBgNVBAMMGEV4YW1wbGUgQ29tIEluYy4gUm9v -dCBDQTAeFw0yMzA4MjkwNDIzMTJaFw0zMzA4MjYwNDIzMTJaMFcxCzAJBgNVBAYT -AmRlMQ0wCwYDVQQHDAR0ZXN0MQ0wCwYDVQQKDARub2RlMQ0wCwYDVQQLDARub2Rl -MRswGQYDVQQDDBJub2RlLTAuZXhhbXBsZS5jb20wggEiMA0GCSqGSIb3DQEBAQUA -A4IBDwAwggEKAoIBAQCm93kXteDQHMAvbUPNPW5pyRHKDD42XGWSgq0k1D29C/Ud -yL21HLzTJa49ZU2ldIkSKs9JqbkHdyK0o8MO6L8dotLoYbxDWbJFW8bp1w6tDTU0 -HGkn47XVu3EwbfrTENg3jFu+Oem6a/501SzITzJWtS0cn2dIFOBimTVpT/4Zv5qr -XA6Cp4biOmoTYWhi/qQl8d0IaADiqoZ1MvZbZ6x76qTrRAbg+UWkpTEXoH1xTc8n -dibR7+HP6OTqCKvo1NhE8uP4pY+fWd6b6l+KLo3IKpfTbAIJXIO+M67FLtWKtttD -ao94B069skzKk6FPgW/OZh6PRCD0oxOavV+ld2SjAgMBAAGjgcYwgcMwRwYDVR0R -BEAwPogFKgMEBQWCEm5vZGUtMC5leGFtcGxlLmNvbYIJbG9jYWxob3N0hxAAAAAA -AAAAAAAAAAAAAAABhwR/AAABMAsGA1UdDwQEAwIF4DAdBgNVHSUEFjAUBggrBgEF -BQcDAQYIKwYBBQUHAwIwDAYDVR0TAQH/BAIwADAdBgNVHQ4EFgQU0/qDQaY10jIo -wCjLUpz/HfQXyt8wHwYDVR0jBBgwFoAUF4ffoFrrZhKn1dD4uhJFPLcrAJwwDQYJ -KoZIhvcNAQELBQADggEBAD2hkndVih6TWxoe/oOW0i2Bq7ScNO/n7/yHWL04HJmR -MaHv/Xjc8zLFLgHuHaRvC02ikWIJyQf5xJt0Oqu2GVbqXH9PBGKuEP2kCsRRyU27 -zTclAzfQhqmKBTYQ/3lJ3GhRQvXIdYTe+t4aq78TCawp1nSN+vdH/1geG6QjMn5N -1FU8tovDd4x8Ib/0dv8RJx+n9gytI8n/giIaDCEbfLLpe4EkV5e5UNpOnRgJjjuy -vtZutc81TQnzBtkS9XuulovDE0qI+jQrKkKu8xgGLhgH0zxnPkKtUg2I3Aq6zl1L -zYkEOUF8Y25J6WeY88Yfnc0iigI+Pnz5NK8R9GL7TYo= ------END CERTIFICATE----- diff --git a/src/test/resources/security/test-kirk.jks b/src/test/resources/security/test-kirk.jks deleted file mode 100644 index 6dbc51e714784fa58a4209c75deab8b9ed1698ff..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 4504 zcma)AXEYp+vt7GZ$?DyT=tPUf>Rt32Rtcg+B4PQKLo)5nT`xBt(f8 zz4zYx{`1az=l47B(|aH0%$a-V&c}OZ28N+d1QLK?7-~f#Qh{)-@KbUEVuBnDwFn`G zTJSH-2g86X{uc$#Cd7a<{=zALBY_C=KPs|Y1i%~&Sotp~4}12H0!$9GfJy&blEDNC z=>%hA9@l)1y-8vD6#cH^U}=KBI0FdeqXH7J!^nt8{(B;j6byi|5|P@4YY{kr2nhrT zsl1TD93_M516EPM#9d4EG(rsFKtBW4^r*(5KwKbTLB){+^0E(}Q+A7HoW0lrA)@i+ zydGtY^95cAh7C?*2qIcESObb&7%#|($|(-eXIiQ#0>bYpj@=?*4?U=5@-ISTdSa4x zOtEjIWb0hr)D^1HVpX7-CjwnsDG8#WM@AVZvyufeW?}`^GtGW7WcGsVl)G*$?lP3S z^GYelg04B!ZBp4GnwCzq@uOLfB4xY#hE;StB61*Yd8?%(Nl9NW{s3+HODy#ik72s%Hj($a8 zhF0>hs}=106=eHlR<&9zT@LuHAUIZWLFWrKQ#$R3^=pv*&-7e6{O_Ji`|s`^^4v@-Hr>`?(V#!ktZ-$-0?Jt1G-G? zE9HvN@-0iPpKSDRsLacPB>#JY4d$KM!zs7xPBvUu4HQ}!Bz$qc)A`=Ver4EBC?!g7b zuW7GvE*puJA=;!bv2_S?8ZQx_n`M?F&kkb{-h zKwO=OA_@auvAUmAsQW~NjYK|}m{>`{*n^45MJ^ph*%K9}8GnxA%-;D^^-}ih8oWP* zXJ#vzJY3e4?&oSey+_=qv19lq zeLI>%Gjx=y!qVzf%Y&c7dgkjEw?^rl8^KxGs^%{Fd_(b51&l(wYCO&Rc~ZUl5^~y> zc}BJ!4+n2KaS|<{vd#M44my1W|M0Y-gfk9<&l%IBje@31-Sr1Mt!fvT(Pe+Gt$Bz? z_up@HJf$b!)YfI|4{%l^JDxgWvp75|nMzg7E)(qZ%=alvt zXMfZg7Z=_eanGP?tBXFKyvFRu$?uMAzg|k-(32orZccxnHGr$(gM%4Hgc&3blJCi; z6j@^Y3XVg*doBz7pms~Jn7 z9>1&oI7bPBOnn7vyV1x>YahPMDy_bySw!71ij);ebzBEUSZK&o1y43I-AuJKXJ~C3 z{ScF0neCZB8?5r>Px#3V%} zq$OY&i2FZH#6&q5i2Yy421o$-o6P@Z2>vgd4p$sB)+@I7CAQvk>m=OVG#EC`^#8Hx zXo}&oS5+Eg(sw4>QN4_Cy_0U!W9o!pxS@}|4s+L{ow)59*P>fYuDV~JqCwTL5s{)3(v zzbM`$E?)E;`zu*Kjpah> zgQl1ucOJOd1|%MDBk_Lsu64*-#r>9orWT19xT!DnCoNv_AnWczl?5a3@Sd4mtPrx@ z;QPqXK#%ve%3=_Sa$)(zJ)mvCYW0$Uim6bQ!S}#H@uPFY+qvmT_x`cr%&q*~6sufG zKKVZ8ebd?WhVYT)or=?jzV*~PLH&t?CH^KO=IX%=oHNr75%vVz=nN9ipHOrX*7{h! zNkaI3@a@JfTINcbD<@;DNwqa&=S5v4pM=tBEMN8HU3}euq?(dEFWfNC>H+2C+1dBA zFs|s&27315cK^vG`LRKX~{Ugw!|2K~TP_VAqXtzNY6)j={rQ zv73v$!psb1ph9o6`kKlGjC8GEdFX9+@{I}q{33}%?v>$a-cw6HGOOLVnv3ITN_D~k zo^QL%)6K#_{j)b&>8Qy@Eweq=Ne8rKsjJTe)mfDw?scqlc&US2dxU0@o5$(Zu(GB4 zujr5^yZdwlP>E{wrkq=NiW~PQZm5`fJz5m&9I}B^zPVNSSa9vWcXu^m%+bU|aOg5q zK%|a72J^vxGy)&3GlNod=Wt|FBG=mgP)o%{(2PCL$9s$dMvIcv^FdM?hbNYQrX%I| z{binoW_?J27M3L2H_Y4n0!3PGL#b*UxRbpd3l$RLC#I})-32((m#4}vP%kHB3Q7PGLpvuro4~7i2u6z$3ar+YSP2?_%+^%f* zR}5Rl@nUnDVdT&uE_ZP%NU-(Zn*^k2*4S;xubW_f3f-cK+=>uy-sK;&F{mRdpgwIgSHfJSw=22paH-mu>R=3Kf9cR*A_Sjg7q#MM< zqobyHu#q_oM3;REOf&nTGa=n6MK4QZ{pey;iGwX&bnAUCVq`=c0{gykLm{VZo%ulF z*n_LEk%}KbmVW1)L+Ab3sSZPR+Fe*5p$^HC|Oyb{_is> zsuD42;l;BT-a#X6fP(~C+`TP&(``5KD7dp9)GD&EVfNN4Bf@5N63j4c_IOZZ`^gF1 zphj9>;b1JVOWrk`HhO{mmk*Lp>wXpL*r|VQth!^2ajO2-Q$=;E0ZcMzj9V;D}3k7ej?g$MEOSvfr*p<&b z6B?7p3F^a78y9pEd$#q2Pm1b zU#?c^Op~TXSZ`3z2a{A=UzcS`zB%Z|XG2xth@1`h=wY$wyp|u2)s&QN#af+k>`vF! z&{oB;K{Wblwtcc`JH%E!TwV2q%vd}p>iZ9d@C(kwR>Dm)p? zV-i0tv8PP66)jD1#I*Qm*`@U`^o)}|58+bGD1y(EEM_dJh-O9xP^xdF-_Z#qZ&m{c zbC6W;iNU!24Cvnj14>>_V8a{IB$GXu&z39rEKNX_07*3xp*W3rJo!}pp2M0Hwe$#* zi#HgV_>>SSD;YT=uK8*Lu|$a+IIXPF$${!eaPU%X#jh@y96VcWEFGqB#<_hE8QPmQ zO_C$p_nXzGgQtqVrC1t-5`*juoj0Q%VLnw`@Yt&eCg!x)84Pq&N%`@t**O@LYz3OR(@+})Hu&$>gJ;6oxdO{ z&KR3!hDx52>YBb*JE@4B`8}j*yOg=37>&zbSN}#T@GA6n9+dFcA*9q_l2eI%Xh*7~ ziU87?k{%5!@e5oasj8xTY|ysPyOMR3W;w?vvG}prD%~$8wf$j!6&K4LI%aD1$6B&8 zG|Bq_{em<75I~pVeMNJ6Dv9e{<=x@Es?2r|L;d(lJhNv+5~$`ps7`1lAq>B{Ot5Ga z6qD6CeNHKADuYBeC(!$C>E5yJ7O5IFfdN*2lPV*LTj(fX$`T*h6!l7_BFQ%HhbJFp zKUVk@Dl`5ZH)LoQ^{7N6?HyY_;Jo?*Uu#dn_XW`49o!xdK!+JJN_3KD7k@2J((0h0 z?0!++a*3VkR_Y8-s+o<1M(>PCz=|sJMqa z0+r0sNH_$gvD_@AC}TCb8}m~2v}_leWOtWdheZwxJl0i{OGIRcO0iVJ-B>5CgP^O-M7OYVJ*8(0|euX~UGp`sq@@gaEw*bHD4*Dj8_ zPO4*=dce-k-f;9Xl`P>A2U6SzIPhFWQT>2(PjqTMlBf}zL3<&dS*!E0mM}&jbXhc- zAb9}5!V(`=H1zl4fM|8TdAE{XwAuTJ>dTw3o}wzSb&xhxCijhe4Q#{|l(FXGy+A)j zH>IZrWy4|#?wJ-1?zBm;cKLHK*H5ngXeiJE?k?6Lz1i+02rcMG7kNDQlDJ_??0D#; z(Bju>vbV@>IGl97vC?TD(|fa!E?NjDA;*m&#_ZiX>Vgi+wr`atYOngkRp_w%?M~sv zUVImV4>dX4Ih+MO4LU`Ui=K%20a~JOwq1$6)KUw@81y#uUGKMV4>O0ioDGDvtZ{Jl zmay)x!zLD>Hl1jqnzX9b_da}w9xr9S`kQwUZPAei4I5Ao#$N}f9I10=!}MXIF!F!C z6+i+ofRKI2Rvlk8erCmgYu2%A6S_nSX7!cGJQ6pQ{xw*Iw(KXQGft90Ft(YQ<7nw! ROz*Khv5A{`^It3We*oUlR=)rM diff --git a/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java b/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java index 70605b599..c2a8a1010 100644 --- a/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java +++ b/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java @@ -23,7 +23,6 @@ import org.opensearch.knn.index.KNNSettings; import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.indices.ModelDao; -import org.opensearch.knn.indices.ModelMetadata; import org.opensearch.knn.indices.ModelState; import org.opensearch.knn.plugin.KNNPlugin; import org.opensearch.knn.plugin.script.KNNScoringScriptEngine; @@ -57,7 +56,6 @@ import java.nio.file.Path; import java.util.ArrayList; import java.util.Arrays; -import java.util.Base64; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -77,15 +75,10 @@ import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_EF_SEARCH; import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_NLIST; import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_SPACE_TYPE; -import static org.opensearch.knn.common.KNNConstants.MODEL_BLOB_PARAMETER; import static org.opensearch.knn.common.KNNConstants.MODEL_DESCRIPTION; -import static org.opensearch.knn.common.KNNConstants.MODEL_ERROR; -import static org.opensearch.knn.common.KNNConstants.MODEL_NODE_ASSIGNMENT; -import static org.opensearch.knn.common.KNNConstants.MODEL_ID; import static org.opensearch.knn.common.KNNConstants.MODEL_INDEX_MAPPING_PATH; import static org.opensearch.knn.common.KNNConstants.MODEL_INDEX_NAME; import static org.opensearch.knn.common.KNNConstants.MODEL_STATE; -import static org.opensearch.knn.common.KNNConstants.MODEL_TIMESTAMP; import static org.opensearch.knn.common.KNNConstants.TRAIN_FIELD_PARAMETER; import static org.opensearch.knn.common.KNNConstants.TRAIN_INDEX_PARAMETER; import static org.opensearch.knn.common.KNNConstants.NAME; @@ -737,33 +730,6 @@ protected void createModelSystemIndex() throws IOException { } } - protected void addModelToSystemIndex(String modelId, ModelMetadata modelMetadata, byte[] model) throws IOException { - assertFalse(StringUtils.isBlank(modelId)); - String modelBase64 = Base64.getEncoder().encodeToString(model); - - Request request = new Request("POST", "/" + MODEL_INDEX_NAME + "/_doc/" + modelId + "?refresh=true"); - - XContentBuilder builder = XContentFactory.jsonBuilder() - .startObject() - .field(MODEL_ID, modelId) - .field(MODEL_STATE, modelMetadata.getState().getName()) - .field(KNN_ENGINE, modelMetadata.getKnnEngine().getName()) - .field(METHOD_PARAMETER_SPACE_TYPE, modelMetadata.getSpaceType().getValue()) - .field(DIMENSION, modelMetadata.getDimension()) - .field(MODEL_BLOB_PARAMETER, modelBase64) - .field(MODEL_TIMESTAMP, modelMetadata.getTimestamp()) - .field(MODEL_DESCRIPTION, modelMetadata.getDescription()) - .field(MODEL_ERROR, modelMetadata.getError()) - .field(MODEL_NODE_ASSIGNMENT, modelMetadata.getNodeAssignment()) - .endObject(); - - request.setJsonEntity(builder.toString()); - - Response response = client().performRequest(request); - - assertEquals(request.getEndpoint() + ": failed", RestStatus.CREATED, RestStatus.fromCode(response.getStatusLine().getStatusCode())); - } - /** * Clear cache *

diff --git a/src/testFixtures/java/org/opensearch/knn/ODFERestTestCase.java b/src/testFixtures/java/org/opensearch/knn/ODFERestTestCase.java index 230c1e7dc..c93e537fb 100644 --- a/src/testFixtures/java/org/opensearch/knn/ODFERestTestCase.java +++ b/src/testFixtures/java/org/opensearch/knn/ODFERestTestCase.java @@ -21,41 +21,28 @@ import org.opensearch.client.Response; import org.opensearch.client.RestClient; import org.opensearch.client.RestClientBuilder; -import org.opensearch.common.io.PathUtils; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.concurrent.ThreadContext; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.DeprecationHandler; import org.opensearch.core.xcontent.MediaType; import org.opensearch.core.xcontent.NamedXContentRegistry; -import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; -import org.opensearch.commons.rest.SecureRestClientBuilder; import org.opensearch.knn.plugin.KNNPlugin; import org.opensearch.core.rest.RestStatus; import org.opensearch.search.SearchHit; import org.opensearch.test.rest.OpenSearchRestTestCase; import java.io.IOException; -import java.net.URI; -import java.net.URISyntaxException; -import java.nio.file.Path; import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Map; -import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; -import static org.opensearch.commons.ConfigConstants.OPENSEARCH_SECURITY_SSL_HTTP_ENABLED; -import static org.opensearch.commons.ConfigConstants.OPENSEARCH_SECURITY_SSL_HTTP_KEYSTORE_FILEPATH; -import static org.opensearch.commons.ConfigConstants.OPENSEARCH_SECURITY_SSL_HTTP_KEYSTORE_KEYPASSWORD; -import static org.opensearch.commons.ConfigConstants.OPENSEARCH_SECURITY_SSL_HTTP_KEYSTORE_PASSWORD; -import static org.opensearch.commons.ConfigConstants.OPENSEARCH_SECURITY_SSL_HTTP_PEMCERT_FILEPATH; import static org.opensearch.knn.TestUtils.KNN_BWC_PREFIX; import static org.opensearch.knn.TestUtils.OPENDISTRO_SECURITY; import static org.opensearch.knn.TestUtils.OPENSEARCH_SYSTEM_INDEX_PREFIX; @@ -72,15 +59,7 @@ public abstract class ODFERestTestCase extends OpenSearchRestTestCase { private final Set IMMUTABLE_INDEX_PREFIXES = Set.of(KNN_BWC_PREFIX, SECURITY_AUDITLOG_PREFIX, OPENSEARCH_SYSTEM_INDEX_PREFIX); protected boolean isHttps() { - boolean isHttps = Optional.ofNullable(System.getProperty("https")).map("true"::equalsIgnoreCase).orElse(false); - if (isHttps) { - // currently only external cluster is supported for security enabled testing - if (!Optional.ofNullable(System.getProperty("tests.rest.cluster")).isPresent()) { - throw new RuntimeException("cluster url should be provided for security enabled testing"); - } - } - - return isHttps; + return Optional.ofNullable(System.getProperty("https")).map("true"::equalsIgnoreCase).orElse(false); } @Override @@ -92,37 +71,19 @@ protected String getProtocol() { protected RestClient buildClient(Settings settings, HttpHost[] hosts) throws IOException { RestClientBuilder builder = RestClient.builder(hosts); if (isHttps()) { - String keystore = settings.get(OPENSEARCH_SECURITY_SSL_HTTP_KEYSTORE_FILEPATH); - if (Objects.nonNull(keystore)) { - URI uri; - try { - uri = this.getClass().getClassLoader().getResource("security/sample.pem").toURI(); - } catch (URISyntaxException e) { - throw new RuntimeException(e); - } - Path configPath = PathUtils.get(uri).getParent().toAbsolutePath(); - return new SecureRestClientBuilder(settings, configPath).build(); - } else { - configureHttpsClient(builder, settings); - boolean strictDeprecationMode = settings.getAsBoolean("strictDeprecationMode", true); - builder.setStrictDeprecationMode(strictDeprecationMode); - return builder.build(); - } + configureHttpsClient(builder, settings); } else { configureClient(builder, settings); } + builder.setStrictDeprecationMode(false); return builder.build(); } protected static void configureHttpsClient(RestClientBuilder builder, Settings settings) throws IOException { - Map headers = ThreadContext.buildDefaultHeaders(settings); - Header[] defaultHeaders = new Header[headers.size()]; - int i = 0; - for (Map.Entry entry : headers.entrySet()) { - defaultHeaders[i++] = new BasicHeader(entry.getKey(), entry.getValue()); - } - builder.setDefaultHeaders(defaultHeaders); + // Similar to client configuration with OpenSearch: + // https://github.com/opensearch-project/OpenSearch/blob/2.11.1/test/framework/src/main/java/org/opensearch/test/rest/OpenSearchRestTestCase.java#L841-L863 + // except we set the user name and password builder.setHttpClientConfigCallback(httpClientBuilder -> { String userName = Optional.ofNullable(System.getProperty("user")) .orElseThrow(() -> new RuntimeException("user name is missing")); @@ -139,7 +100,13 @@ protected static void configureHttpsClient(RestClientBuilder builder, Settings s throw new RuntimeException(e); } }); - + Map headers = ThreadContext.buildDefaultHeaders(settings); + Header[] defaultHeaders = new Header[headers.size()]; + int i = 0; + for (Map.Entry entry : headers.entrySet()) { + defaultHeaders[i++] = new BasicHeader(entry.getKey(), entry.getValue()); + } + builder.setDefaultHeaders(defaultHeaders); final String socketTimeoutString = settings.get(CLIENT_SOCKET_TIMEOUT); final TimeValue socketTimeout = TimeValue.parseTimeValue( socketTimeoutString == null ? "60s" : socketTimeoutString, @@ -182,8 +149,10 @@ protected void wipeAllODFEIndices() throws Exception { for (Map index : parserList) { final String indexName = (String) index.get("index"); - if (isIndexCleanupRequired(indexName)) { - wipeIndexContent(indexName); + if (MODEL_INDEX_NAME.equals(indexName)) { + if (!getSkipDeleteModelIndexFlag()) { + deleteModels(getModelIds()); + } continue; } if (!skipDeleteIndex(indexName)) { @@ -193,15 +162,6 @@ protected void wipeAllODFEIndices() throws Exception { } } - private boolean isIndexCleanupRequired(final String index) { - return MODEL_INDEX_NAME.equals(index) && !getSkipDeleteModelIndexFlag(); - } - - private void wipeIndexContent(String indexName) throws IOException { - deleteModels(getModelIds()); - deleteAllDocs(indexName); - } - private List getModelIds() throws IOException { final String restURIGetModels = String.join("/", KNNPlugin.KNN_BASE_URI, MODELS, "_search"); final Response response = adminClient().performRequest(new Request("GET", restURIGetModels)); @@ -229,51 +189,14 @@ private void deleteModels(final List modelIds) throws IOException { } } - private void deleteAllDocs(final String indexName) throws IOException { - final String restURIDeleteByQuery = String.join("/", indexName, "_delete_by_query"); - final Request request = new Request("POST", restURIDeleteByQuery); - final XContentBuilder matchAllDocsQuery = XContentFactory.jsonBuilder() - .startObject() - .startObject("query") - .startObject("match_all") - .endObject() - .endObject() - .endObject(); - - request.setJsonEntity(matchAllDocsQuery.toString()); - adminClient().performRequest(request); - } - private boolean getSkipDeleteModelIndexFlag() { return Boolean.parseBoolean(System.getProperty(SKIP_DELETE_MODEL_INDEX, "false")); } - private boolean skipDeleteModelIndex(String indexName) { - return (MODEL_INDEX_NAME.equals(indexName) && getSkipDeleteModelIndexFlag()); - } - private boolean skipDeleteIndex(String indexName) { - if (indexName != null - && !OPENDISTRO_SECURITY.equals(indexName) - && IMMUTABLE_INDEX_PREFIXES.stream().noneMatch(indexName::startsWith) - && !skipDeleteModelIndex(indexName)) { - return false; - } - - return true; - } - - @Override - protected Settings restAdminSettings() { - return Settings.builder() - // disable the warning exception for admin client since it's only used for cleanup. - .put("strictDeprecationMode", false) - .put("http.port", 9200) - .put(OPENSEARCH_SECURITY_SSL_HTTP_ENABLED, isHttps()) - .put(OPENSEARCH_SECURITY_SSL_HTTP_PEMCERT_FILEPATH, "sample.pem") - .put(OPENSEARCH_SECURITY_SSL_HTTP_KEYSTORE_FILEPATH, "test-kirk.jks") - .put(OPENSEARCH_SECURITY_SSL_HTTP_KEYSTORE_PASSWORD, "changeit") - .put(OPENSEARCH_SECURITY_SSL_HTTP_KEYSTORE_KEYPASSWORD, "changeit") - .build(); + return indexName == null + || OPENDISTRO_SECURITY.equals(indexName) + || IMMUTABLE_INDEX_PREFIXES.stream().anyMatch(indexName::startsWith) + || MODEL_INDEX_NAME.equals(indexName); } } From f519fe2acc64817b83ea1c3aca2c1b5affb9ce3e Mon Sep 17 00:00:00 2001 From: Ryan Bogan Date: Wed, 3 Jan 2024 12:40:19 -0800 Subject: [PATCH 174/416] [Backport 2.x] Fix flaky tests (#1370) * Fix flaky tests Signed-off-by: Ryan Bogan * Minor change Signed-off-by: Ryan Bogan * Add necessary imports Signed-off-by: Ryan Bogan --------- Signed-off-by: Ryan Bogan --- .../java/org/opensearch/knn/bwc/ModelIT.java | 2 + .../transport/GetModelResponseTests.java | 51 ++++++++++++------- 2 files changed, 36 insertions(+), 17 deletions(-) diff --git a/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/ModelIT.java b/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/ModelIT.java index be20d18a2..fed2d39da 100644 --- a/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/ModelIT.java +++ b/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/ModelIT.java @@ -77,6 +77,7 @@ public void testKNNModel() throws Exception { createKnnIndex(testIndex, modelIndexMapping(TEST_FIELD, TEST_MODEL_ID)); addKNNDocs(testIndex, TEST_FIELD, DIMENSIONS, DOC_ID, NUM_DOCS); } else { + Thread.sleep(1000); DOC_ID = NUM_DOCS; addKNNDocs(testIndex, TEST_FIELD, DIMENSIONS, DOC_ID, NUM_DOCS); QUERY_COUNT = 2 * NUM_DOCS; @@ -109,6 +110,7 @@ public void testKNNModelDefault() throws Exception { createKnnIndex(testIndex, modelIndexMapping(TEST_FIELD, TEST_MODEL_ID_DEFAULT)); addKNNDocs(testIndex, TEST_FIELD, DIMENSIONS, DOC_ID, NUM_DOCS); } else { + Thread.sleep(1000); DOC_ID = NUM_DOCS; addKNNDocs(testIndex, TEST_FIELD, DIMENSIONS, DOC_ID, NUM_DOCS); QUERY_COUNT = 2 * NUM_DOCS; diff --git a/src/test/java/org/opensearch/knn/plugin/transport/GetModelResponseTests.java b/src/test/java/org/opensearch/knn/plugin/transport/GetModelResponseTests.java index 6296a7192..a54605d24 100644 --- a/src/test/java/org/opensearch/knn/plugin/transport/GetModelResponseTests.java +++ b/src/test/java/org/opensearch/knn/plugin/transport/GetModelResponseTests.java @@ -11,11 +11,14 @@ package org.opensearch.knn.plugin.transport; +import org.mockito.MockedStatic; +import org.opensearch.Version; import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentType; import org.opensearch.knn.KNNTestCase; +import org.opensearch.knn.index.KNNClusterUtil; import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.util.KNNEngine; import org.opensearch.knn.indices.Model; @@ -24,6 +27,10 @@ import java.io.IOException; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.mockStatic; +import static org.mockito.Mockito.when; + public class GetModelResponseTests extends KNNTestCase { private ModelMetadata getModelMetadata(ModelState state) { @@ -42,25 +49,35 @@ public void testStreams() throws IOException { } public void testXContent() throws IOException { - String modelId = "test-model"; - byte[] testModelBlob = "hello".getBytes(); - Model model = new Model(getModelMetadata(ModelState.CREATED), testModelBlob, modelId); - GetModelResponse getModelResponse = new GetModelResponse(model); - String expectedResponseString = - "{\"model_id\":\"test-model\",\"model_blob\":\"aGVsbG8=\",\"state\":\"created\",\"timestamp\":\"2021-03-27 10:15:30 AM +05:30\",\"description\":\"test model\",\"error\":\"\",\"space_type\":\"l2\",\"dimension\":4,\"engine\":\"nmslib\",\"training_node_assignment\":\"\"}"; - XContentBuilder xContentBuilder = MediaTypeRegistry.contentBuilder(XContentType.JSON); - getModelResponse.toXContent(xContentBuilder, null); - assertEquals(expectedResponseString, xContentBuilder.toString()); + try (MockedStatic knnClusterUtilMockedStatic = mockStatic(KNNClusterUtil.class)) { + final KNNClusterUtil knnClusterUtil = mock(KNNClusterUtil.class); + when(knnClusterUtil.getClusterMinVersion()).thenReturn(Version.CURRENT); + knnClusterUtilMockedStatic.when(KNNClusterUtil::instance).thenReturn(knnClusterUtil); + String modelId = "test-model"; + byte[] testModelBlob = "hello".getBytes(); + Model model = new Model(getModelMetadata(ModelState.CREATED), testModelBlob, modelId); + GetModelResponse getModelResponse = new GetModelResponse(model); + String expectedResponseString = + "{\"model_id\":\"test-model\",\"model_blob\":\"aGVsbG8=\",\"state\":\"created\",\"timestamp\":\"2021-03-27 10:15:30 AM +05:30\",\"description\":\"test model\",\"error\":\"\",\"space_type\":\"l2\",\"dimension\":4,\"engine\":\"nmslib\",\"training_node_assignment\":\"\"}"; + XContentBuilder xContentBuilder = MediaTypeRegistry.contentBuilder(XContentType.JSON); + getModelResponse.toXContent(xContentBuilder, null); + assertEquals(expectedResponseString, xContentBuilder.toString()); + } } public void testXContentWithNoModelBlob() throws IOException { - String modelId = "test-model"; - Model model = new Model(getModelMetadata(ModelState.FAILED), null, modelId); - GetModelResponse getModelResponse = new GetModelResponse(model); - String expectedResponseString = - "{\"model_id\":\"test-model\",\"model_blob\":\"\",\"state\":\"failed\",\"timestamp\":\"2021-03-27 10:15:30 AM +05:30\",\"description\":\"test model\",\"error\":\"\",\"space_type\":\"l2\",\"dimension\":4,\"engine\":\"nmslib\",\"training_node_assignment\":\"\"}"; - XContentBuilder xContentBuilder = MediaTypeRegistry.contentBuilder(XContentType.JSON); - getModelResponse.toXContent(xContentBuilder, null); - assertEquals(expectedResponseString, xContentBuilder.toString()); + try (MockedStatic knnClusterUtilMockedStatic = mockStatic(KNNClusterUtil.class)) { + final KNNClusterUtil knnClusterUtil = mock(KNNClusterUtil.class); + when(knnClusterUtil.getClusterMinVersion()).thenReturn(Version.CURRENT); + knnClusterUtilMockedStatic.when(KNNClusterUtil::instance).thenReturn(knnClusterUtil); + String modelId = "test-model"; + Model model = new Model(getModelMetadata(ModelState.FAILED), null, modelId); + GetModelResponse getModelResponse = new GetModelResponse(model); + String expectedResponseString = + "{\"model_id\":\"test-model\",\"model_blob\":\"\",\"state\":\"failed\",\"timestamp\":\"2021-03-27 10:15:30 AM +05:30\",\"description\":\"test model\",\"error\":\"\",\"space_type\":\"l2\",\"dimension\":4,\"engine\":\"nmslib\",\"training_node_assignment\":\"\"}"; + XContentBuilder xContentBuilder = MediaTypeRegistry.contentBuilder(XContentType.JSON); + getModelResponse.toXContent(xContentBuilder, null); + assertEquals(expectedResponseString, xContentBuilder.toString()); + } } } From 3043ef7a58c5ee1cc5827f75ead0fe4282b6907b Mon Sep 17 00:00:00 2001 From: Junqiu Lei Date: Thu, 4 Jan 2024 09:02:28 -0800 Subject: [PATCH 175/416] Fix script score queries not getting cached (#1373) Signed-off-by: Junqiu Lei --- CHANGELOG.md | 1 + .../opensearch/knn/bwc/ScriptScoringIT.java | 3 +- .../plugin/script/KNNScoreScriptFactory.java | 57 ++------- .../script/KNNScoreScriptLeafFactory.java | 73 +++++++++++ .../plugin/script/KNNScoringScriptEngine.java | 2 +- .../knn/index/VectorDataTypeIT.java | 6 +- .../knn/plugin/script/KNNScriptScoringIT.java | 117 +++++++++++++++++- .../knn/plugin/script/PainlessScriptIT.java | 20 ++- .../org/opensearch/knn/KNNRestTestCase.java | 36 ++++-- 9 files changed, 245 insertions(+), 70 deletions(-) create mode 100644 src/main/java/org/opensearch/knn/plugin/script/KNNScoreScriptLeafFactory.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 7839d9825..b2ab6b446 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), * Fix use-after-free case on nmslib search path [#1305](https://github.com/opensearch-project/k-NN/pull/1305) * Allow nested knn field mapping when train model [#1318](https://github.com/opensearch-project/k-NN/pull/1318) * Properly designate model state for actively training models when nodes crash or leave cluster [#1317](https://github.com/opensearch-project/k-NN/pull/1317) +* Fix script score queries not getting cached [#1367](https://github.com/opensearch-project/k-NN/pull/1367) ### Infrastructure * Upgrade gradle to 8.4 [1289](https://github.com/opensearch-project/k-NN/pull/1289) * Refactor security testing to install from individual components [#1307](https://github.com/opensearch-project/k-NN/pull/1307) diff --git a/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/ScriptScoringIT.java b/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/ScriptScoringIT.java index ccb30fa1a..2932f32fb 100644 --- a/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/ScriptScoringIT.java +++ b/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/ScriptScoringIT.java @@ -15,6 +15,7 @@ import org.opensearch.knn.index.SpaceType; import org.opensearch.core.rest.RestStatus; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -107,7 +108,7 @@ private void validateKNNInnerProductScriptScoreSearch(String testIndex, String t params.put(QUERY_VALUE, queryVector); params.put(METHOD_PARAMETER_SPACE_TYPE, SpaceType.INNER_PRODUCT.getValue()); - Request request = constructKNNScriptQueryRequest(testIndex, qb, params, k); + Request request = constructKNNScriptQueryRequest(testIndex, qb, params, k, Collections.emptyMap()); Response response = client().performRequest(request); assertEquals(request.getEndpoint() + ": failed", RestStatus.OK, RestStatus.fromCode(response.getStatusLine().getStatusCode())); diff --git a/src/main/java/org/opensearch/knn/plugin/script/KNNScoreScriptFactory.java b/src/main/java/org/opensearch/knn/plugin/script/KNNScoreScriptFactory.java index 63b367b2d..4f6a1a6c4 100644 --- a/src/main/java/org/opensearch/knn/plugin/script/KNNScoreScriptFactory.java +++ b/src/main/java/org/opensearch/knn/plugin/script/KNNScoreScriptFactory.java @@ -6,64 +6,21 @@ package org.opensearch.knn.plugin.script; import org.apache.lucene.search.IndexSearcher; -import org.opensearch.knn.plugin.stats.KNNCounter; -import org.apache.lucene.index.LeafReaderContext; import org.opensearch.script.ScoreScript; +import org.opensearch.script.ScriptFactory; import org.opensearch.search.lookup.SearchLookup; -import java.io.IOException; import java.util.Map; -public class KNNScoreScriptFactory implements ScoreScript.LeafFactory { - private final Map params; - private final SearchLookup lookup; - private String similaritySpace; - private String field; - private Object query; - private KNNScoringSpace knnScoringSpace; - - private IndexSearcher searcher; - - public KNNScoreScriptFactory(Map params, SearchLookup lookup, IndexSearcher searcher) { - KNNCounter.SCRIPT_QUERY_REQUESTS.increment(); - this.params = params; - this.lookup = lookup; - this.field = getValue(params, "field").toString(); - this.similaritySpace = getValue(params, "space_type").toString(); - this.query = getValue(params, "query_value"); - this.searcher = searcher; - - this.knnScoringSpace = KNNScoringSpaceFactory.create( - this.similaritySpace, - this.query, - lookup.doc().mapperService().fieldType(this.field) - ); - } - - private Object getValue(Map params, String fieldName) { - final Object value = params.get(fieldName); - if (value != null) return value; - - KNNCounter.SCRIPT_QUERY_ERRORS.increment(); - throw new IllegalArgumentException("Missing parameter [" + fieldName + "]"); - } - +public class KNNScoreScriptFactory implements ScoreScript.Factory, ScriptFactory { @Override - public boolean needs_score() { - return false; + public boolean isResultDeterministic() { + // This implies the results are cacheable + return true; } - /** - * For each segment, supply the KNNScoreScript that should be used to re-score the documents returned from the - * query. Because the method to score the documents was set during factory construction, the scripts are agnostic of - * the similarity space. The KNNScoringSpace will return the correct script, given the query, the field type, and - * the similarity space. - * - * @param ctx LeafReaderContext for the segment - * @return ScoreScript to be executed - */ @Override - public ScoreScript newInstance(LeafReaderContext ctx) throws IOException { - return knnScoringSpace.getScoreScript(params, field, lookup, ctx, this.searcher); + public ScoreScript.LeafFactory newFactory(Map params, SearchLookup lookup, IndexSearcher indexSearcher) { + return new KNNScoreScriptLeafFactory(params, lookup, indexSearcher); } } diff --git a/src/main/java/org/opensearch/knn/plugin/script/KNNScoreScriptLeafFactory.java b/src/main/java/org/opensearch/knn/plugin/script/KNNScoreScriptLeafFactory.java new file mode 100644 index 000000000..1caca0d4b --- /dev/null +++ b/src/main/java/org/opensearch/knn/plugin/script/KNNScoreScriptLeafFactory.java @@ -0,0 +1,73 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.plugin.script; + +import java.io.IOException; +import java.util.Locale; +import java.util.Map; + +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.search.IndexSearcher; +import org.opensearch.knn.plugin.stats.KNNCounter; +import org.opensearch.script.ScoreScript; +import org.opensearch.search.lookup.SearchLookup; + +/* + * A factory that creates KNNScoreScriptLeafFactory objects. The factory is responsible for parsing the parameters + * passed in the query and creating the KNNScoreScriptLeafFactory object. + */ +public class KNNScoreScriptLeafFactory implements ScoreScript.LeafFactory { + private final Map params; + private final SearchLookup lookup; + private final String similaritySpace; + private final String field; + private final Object query; + private final KNNScoringSpace knnScoringSpace; + private final IndexSearcher searcher; + + public KNNScoreScriptLeafFactory(Map params, SearchLookup lookup, IndexSearcher searcher) { + KNNCounter.SCRIPT_QUERY_REQUESTS.increment(); + this.params = params; + this.lookup = lookup; + this.field = getValue(params, "field").toString(); + this.similaritySpace = getValue(params, "space_type").toString(); + this.query = getValue(params, "query_value"); + this.searcher = searcher; + + this.knnScoringSpace = KNNScoringSpaceFactory.create( + this.similaritySpace, + this.query, + lookup.doc().mapperService().fieldType(this.field) + ); + } + + private Object getValue(Map params, String fieldName) { + final Object value = params.get(fieldName); + if (value != null) return value; + + KNNCounter.SCRIPT_QUERY_ERRORS.increment(); + throw new IllegalArgumentException(String.format(Locale.ROOT, "Missing parameter [%s]", fieldName)); + } + + @Override + public boolean needs_score() { + return false; + } + + /** + * For each segment, supply the KNNScoreScript that should be used to re-score the documents returned from the + * query. Because the method to score the documents was set during factory construction, the scripts are agnostic of + * the similarity space. The KNNScoringSpace will return the correct script, given the query, the field type, and + * the similarity space. + * + * @param ctx LeafReaderContext for the segment + * @return ScoreScript to be executed + */ + @Override + public ScoreScript newInstance(LeafReaderContext ctx) throws IOException { + return knnScoringSpace.getScoreScript(params, field, lookup, ctx, this.searcher); + } +} diff --git a/src/main/java/org/opensearch/knn/plugin/script/KNNScoringScriptEngine.java b/src/main/java/org/opensearch/knn/plugin/script/KNNScoringScriptEngine.java index 42e0e90ec..61b26c760 100644 --- a/src/main/java/org/opensearch/knn/plugin/script/KNNScoringScriptEngine.java +++ b/src/main/java/org/opensearch/knn/plugin/script/KNNScoringScriptEngine.java @@ -39,7 +39,7 @@ public FactoryType compile(String name, String code, ScriptContext KNNCounter.SCRIPT_COMPILATION_ERRORS.increment(); throw new IllegalArgumentException("Unknown script name " + code); } - ScoreScript.Factory factory = KNNScoreScriptFactory::new; + ScoreScript.Factory factory = new KNNScoreScriptFactory(); return context.factoryClazz.cast(factory); } diff --git a/src/test/java/org/opensearch/knn/index/VectorDataTypeIT.java b/src/test/java/org/opensearch/knn/index/VectorDataTypeIT.java index adb222caf..d98cc2a14 100644 --- a/src/test/java/org/opensearch/knn/index/VectorDataTypeIT.java +++ b/src/test/java/org/opensearch/knn/index/VectorDataTypeIT.java @@ -350,7 +350,8 @@ public void testL2PainlessScriptingWithByteVectorDataType() throws Exception { Collections.emptyMap(), Script.DEFAULT_SCRIPT_LANG, source, - 4 + 4, + Collections.emptyMap() ); Response response = client().performRequest(request); @@ -370,7 +371,8 @@ public void testL2PainlessScriptingWithFloatVectorDataType() throws Exception { Collections.emptyMap(), Script.DEFAULT_SCRIPT_LANG, source, - 4 + 4, + Collections.emptyMap() ); Response response = client().performRequest(request); diff --git a/src/test/java/org/opensearch/knn/plugin/script/KNNScriptScoringIT.java b/src/test/java/org/opensearch/knn/plugin/script/KNNScriptScoringIT.java index 54a80926a..214ecd158 100644 --- a/src/test/java/org/opensearch/knn/plugin/script/KNNScriptScoringIT.java +++ b/src/test/java/org/opensearch/knn/plugin/script/KNNScriptScoringIT.java @@ -5,6 +5,7 @@ package org.opensearch.knn.plugin.script; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.knn.KNNRestTestCase; import org.opensearch.knn.KNNResult; import org.opensearch.knn.index.SpaceType; @@ -25,9 +26,11 @@ import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.stream.Collectors; import static org.hamcrest.Matchers.containsString; @@ -449,7 +452,7 @@ public void testHammingScriptScore_Long() throws Exception { params1.put("field", FIELD_NAME); params1.put("query_value", queryValue1); params1.put("space_type", SpaceType.HAMMING_BIT.getValue()); - Request request1 = constructKNNScriptQueryRequest(INDEX_NAME, qb1, params1, 4); + Request request1 = constructKNNScriptQueryRequest(INDEX_NAME, qb1, params1, 4, Collections.emptyMap()); Response response1 = client().performRequest(request1); assertEquals(request1.getEndpoint() + ": failed", RestStatus.OK, RestStatus.fromCode(response1.getStatusLine().getStatusCode())); @@ -487,7 +490,7 @@ public void testHammingScriptScore_Long() throws Exception { params2.put("field", FIELD_NAME); params2.put("query_value", queryValue2); params2.put("space_type", SpaceType.HAMMING_BIT.getValue()); - Request request2 = constructKNNScriptQueryRequest(INDEX_NAME, qb2, params2, 4); + Request request2 = constructKNNScriptQueryRequest(INDEX_NAME, qb2, params2, 4, Collections.emptyMap()); Response response2 = client().performRequest(request2); assertEquals(request2.getEndpoint() + ": failed", RestStatus.OK, RestStatus.fromCode(response2.getStatusLine().getStatusCode())); @@ -555,7 +558,7 @@ public void testHammingScriptScore_Base64() throws Exception { params1.put("field", FIELD_NAME); params1.put("query_value", queryValue1); params1.put("space_type", SpaceType.HAMMING_BIT.getValue()); - Request request1 = constructKNNScriptQueryRequest(INDEX_NAME, qb1, params1, 4); + Request request1 = constructKNNScriptQueryRequest(INDEX_NAME, qb1, params1, 4, Collections.emptyMap()); Response response1 = client().performRequest(request1); assertEquals(request1.getEndpoint() + ": failed", RestStatus.OK, RestStatus.fromCode(response1.getStatusLine().getStatusCode())); @@ -593,7 +596,7 @@ public void testHammingScriptScore_Base64() throws Exception { params2.put("field", FIELD_NAME); params2.put("query_value", queryValue2); params2.put("space_type", SpaceType.HAMMING_BIT.getValue()); - Request request2 = constructKNNScriptQueryRequest(INDEX_NAME, qb2, params2, 4); + Request request2 = constructKNNScriptQueryRequest(INDEX_NAME, qb2, params2, 4, Collections.emptyMap()); Response response2 = client().performRequest(request2); assertEquals(request2.getEndpoint() + ": failed", RestStatus.OK, RestStatus.fromCode(response2.getStatusLine().getStatusCode())); @@ -673,4 +676,110 @@ public void testKNNInnerProdScriptScore() throws Exception { assertEquals("4", results.get(2).getDocId()); assertEquals("1", results.get(3).getDocId()); } + + public void testKNNScriptScoreWithRequestCacheEnabled() throws Exception { + /* + * Create knn index and populate data + */ + createKnnIndex(INDEX_NAME, createKnnIndexMapping(FIELD_NAME, 2)); + Float[] f1 = { 6.0f, 6.0f }; + addKnnDoc(INDEX_NAME, "1", FIELD_NAME, f1); + + Float[] f2 = { 2.0f, 2.0f }; + addKnnDoc(INDEX_NAME, "2", FIELD_NAME, f2); + + Float[] f3 = { 4.0f, 4.0f }; + addKnnDoc(INDEX_NAME, "3", FIELD_NAME, f3); + + Float[] f4 = { 3.0f, 3.0f }; + addKnnDoc(INDEX_NAME, "4", FIELD_NAME, f4); + + /** + * Construct Search Request + */ + QueryBuilder qb = new MatchAllQueryBuilder(); + Map scriptParams = new HashMap<>(); + /* + * params": { + * "field": "my_dense_vector", + * "vector": [2.0, 2.0] + * } + */ + float[] queryVector = { 1.0f, 1.0f }; + scriptParams.put("field", FIELD_NAME); + scriptParams.put("query_value", queryVector); + scriptParams.put("space_type", SpaceType.L2.getValue()); + Map searchParams = new HashMap<>(); + searchParams.put("request_cache", true); + + // first request with request cache enabled + Request firstScriptQueryRequest = constructKNNScriptQueryRequest(INDEX_NAME, qb, scriptParams, 4, searchParams); + Response firstScriptQueryResponse = client().performRequest(firstScriptQueryRequest); + assertEquals( + firstScriptQueryRequest.getEndpoint() + ": failed", + RestStatus.OK, + RestStatus.fromCode(firstScriptQueryResponse.getStatusLine().getStatusCode()) + ); + + List results = parseSearchResponse(EntityUtils.toString(firstScriptQueryResponse.getEntity()), FIELD_NAME); + List expectedDocids = Arrays.asList("2", "4", "3", "1"); + + List actualDocids = new ArrayList<>(); + for (KNNResult result : results) { + actualDocids.add(result.getDocId()); + } + + assertEquals(4, results.size()); + assertEquals(expectedDocids, actualDocids); + + // assert that the request cache was hit missed at first request + Request firstStatsRequest = new Request("GET", "/" + INDEX_NAME + "/_stats"); + Response firstStatsResponse = client().performRequest(firstStatsRequest); + assertEquals( + firstStatsRequest.getEndpoint() + ": failed", + RestStatus.OK, + RestStatus.fromCode(firstStatsResponse.getStatusLine().getStatusCode()) + ); + String firstStatsResponseBody = EntityUtils.toString(firstStatsResponse.getEntity()); + Map firstQueryCacheMap = Optional.ofNullable( + createParser(MediaTypeRegistry.getDefaultMediaType().xContent(), firstStatsResponseBody).map() + ) + .map(r -> (Map) r.get("indices")) + .map(i -> (Map) i.get(INDEX_NAME)) + .map(ind -> (Map) ind.get("total")) + .map(t -> (Map) t.get("request_cache")) + .orElseThrow(() -> new IllegalStateException("Query Cache Map not found")); + // assert that the request cache was hit missed at first request + assertEquals(1, firstQueryCacheMap.get("miss_count")); + assertEquals(0, firstQueryCacheMap.get("hit_count")); + + // second request with request cache enabled + Request secondScriptQueryRequest = constructKNNScriptQueryRequest(INDEX_NAME, qb, scriptParams, 4, searchParams); + Response secondScriptQueryResponse = client().performRequest(secondScriptQueryRequest); + assertEquals( + firstScriptQueryRequest.getEndpoint() + ": failed", + RestStatus.OK, + RestStatus.fromCode(secondScriptQueryResponse.getStatusLine().getStatusCode()) + ); + + Request secondStatsRequest = new Request("GET", "/" + INDEX_NAME + "/_stats"); + Response secondStatsResponse = client().performRequest(secondStatsRequest); + assertEquals( + secondStatsRequest.getEndpoint() + ": failed", + RestStatus.OK, + RestStatus.fromCode(secondStatsResponse.getStatusLine().getStatusCode()) + ); + String secondStatsResponseBody = EntityUtils.toString(secondStatsResponse.getEntity()); + Map secondQueryCacheMap = Optional.ofNullable( + createParser(MediaTypeRegistry.getDefaultMediaType().xContent(), secondStatsResponseBody).map() + ) + .map(r -> (Map) r.get("indices")) + .map(i -> (Map) i.get(INDEX_NAME)) + .map(ind -> (Map) ind.get("total")) + .map(t -> (Map) t.get("request_cache")) + .orElseThrow(() -> new IllegalStateException("Query Cache Map not found")); + assertEquals(1, secondQueryCacheMap.get("miss_count")); + // assert that the request cache was hit at second request + assertEquals(1, secondQueryCacheMap.get("hit_count")); + } } diff --git a/src/test/java/org/opensearch/knn/plugin/script/PainlessScriptIT.java b/src/test/java/org/opensearch/knn/plugin/script/PainlessScriptIT.java index da10578b9..15e3732b2 100644 --- a/src/test/java/org/opensearch/knn/plugin/script/PainlessScriptIT.java +++ b/src/test/java/org/opensearch/knn/plugin/script/PainlessScriptIT.java @@ -155,7 +155,15 @@ public void testL2ScriptScoreFails() throws Exception { private Request buildPainlessScoreScriptRequest(String source, int size, Map documents) throws Exception { buildTestIndex(documents); QueryBuilder qb = new MatchAllQueryBuilder(); - return constructScriptScoreContextSearchRequest(INDEX_NAME, qb, Collections.emptyMap(), Script.DEFAULT_SCRIPT_LANG, source, size); + return constructScriptScoreContextSearchRequest( + INDEX_NAME, + qb, + Collections.emptyMap(), + Script.DEFAULT_SCRIPT_LANG, + source, + size, + Collections.emptyMap() + ); } private Request buildPainlessScoreScriptRequest( @@ -166,7 +174,15 @@ private Request buildPainlessScoreScriptRequest( ) throws Exception { buildTestIndex(documents, properties); QueryBuilder qb = new MatchAllQueryBuilder(); - return constructScriptScoreContextSearchRequest(INDEX_NAME, qb, Collections.emptyMap(), Script.DEFAULT_SCRIPT_LANG, source, size); + return constructScriptScoreContextSearchRequest( + INDEX_NAME, + qb, + Collections.emptyMap(), + Script.DEFAULT_SCRIPT_LANG, + source, + size, + Collections.emptyMap() + ); } private Request buildPainlessScriptedMetricRequest( diff --git a/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java b/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java index c2a8a1010..f97396303 100644 --- a/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java +++ b/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java @@ -10,6 +10,7 @@ import com.google.common.primitives.Floats; import lombok.extern.log4j.Log4j2; import org.apache.commons.lang.StringUtils; +import org.apache.http.client.utils.URIBuilder; import org.apache.http.util.EntityUtils; import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.xcontent.XContentHelper; @@ -61,6 +62,7 @@ import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.Objects; import java.util.PriorityQueue; import java.util.Set; import java.util.concurrent.TimeUnit; @@ -810,12 +812,13 @@ protected Request constructScriptedMetricAggregationSearchRequest( protected Request constructScriptScoreContextSearchRequest( String indexName, QueryBuilder qb, - Map params, + Map scriptParams, String language, String source, - int size + int size, + Map searchParams ) throws Exception { - Script script = buildScript(source, language, params); + Script script = buildScript(source, language, scriptParams); ScriptScoreQueryBuilder sc = new ScriptScoreQueryBuilder(qb, script); XContentBuilder builder = XContentFactory.jsonBuilder().startObject().field("size", size).startObject("query"); builder.startObject("script_score"); @@ -825,7 +828,13 @@ protected Request constructScriptScoreContextSearchRequest( builder.endObject(); builder.endObject(); builder.endObject(); - Request request = new Request("POST", "/" + indexName + "/_search"); + URIBuilder uriBuilder = new URIBuilder("/" + indexName + "/_search"); + if (Objects.nonNull(searchParams)) { + for (Map.Entry entry : searchParams.entrySet()) { + uriBuilder.addParameter(entry.getKey(), entry.getValue().toString()); + } + } + Request request = new Request("POST", uriBuilder.toString()); request.setJsonEntity(builder.toString()); return request; } @@ -846,15 +855,21 @@ protected Request constructKNNScriptQueryRequest(String indexName, QueryBuilder return request; } - protected Request constructKNNScriptQueryRequest(String indexName, QueryBuilder qb, Map params, int size) - throws Exception { + protected Request constructKNNScriptQueryRequest( + String indexName, + QueryBuilder qb, + Map scriptParams, + int size, + Map searchParams + ) throws Exception { return constructScriptScoreContextSearchRequest( indexName, qb, - params, + scriptParams, KNNScoringScriptEngine.NAME, KNNScoringScriptEngine.SCRIPT_SOURCE, - size + size, + searchParams ); } @@ -1105,7 +1120,7 @@ protected void validateKNNScriptScoreSearch(String testIndex, String testField, params.put(QUERY_VALUE, queryVector); params.put(METHOD_PARAMETER_SPACE_TYPE, spaceType.getValue()); - Request request = constructKNNScriptQueryRequest(testIndex, qb, params, k); + Request request = constructKNNScriptQueryRequest(testIndex, qb, params, k, Collections.emptyMap()); Response response = client().performRequest(request); assertEquals(request.getEndpoint() + ": failed", RestStatus.OK, RestStatus.fromCode(response.getStatusLine().getStatusCode())); @@ -1131,7 +1146,8 @@ protected void validateKNNPainlessScriptScoreSearch(String testIndex, String tes Collections.emptyMap(), Script.DEFAULT_SCRIPT_LANG, source, - k + k, + Collections.emptyMap() ); Response response = client().performRequest(request); assertEquals(request.getEndpoint() + ": failed", RestStatus.OK, RestStatus.fromCode(response.getStatusLine().getStatusCode())); From bb4976c0f4ea34766bca21f36337e5e2da417b44 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Thu, 4 Jan 2024 16:52:18 -0800 Subject: [PATCH 176/416] Fix TrainingJobRouteDecisionInfo test (#1375) Recently, we have seen that TrainingJobRouteDecisionInfoTransportActionTests has been having failures on Windows. The failures are related to an unintialized cluster state. This does not have anything to do with the test itself. Most likely, it is the result of state dependence that happens with KNNSingleNodeTestCase. This change refactors the class to use mocks and a lighter weight base class, KNNTestCase. Signed-off-by: John Mazanec (cherry picked from commit 5c24d99ac6aa8533e1924e16ca1663d1c7bdcbe4) --- ...RouteDecisionInfoTransportActionTests.java | 120 ++++++------------ 1 file changed, 36 insertions(+), 84 deletions(-) diff --git a/src/test/java/org/opensearch/knn/plugin/transport/TrainingJobRouteDecisionInfoTransportActionTests.java b/src/test/java/org/opensearch/knn/plugin/transport/TrainingJobRouteDecisionInfoTransportActionTests.java index 9f64afebb..d07be2070 100644 --- a/src/test/java/org/opensearch/knn/plugin/transport/TrainingJobRouteDecisionInfoTransportActionTests.java +++ b/src/test/java/org/opensearch/knn/plugin/transport/TrainingJobRouteDecisionInfoTransportActionTests.java @@ -11,98 +11,50 @@ package org.opensearch.knn.plugin.transport; -import org.junit.After; -import org.junit.Before; -import org.opensearch.core.action.ActionListener; -import org.opensearch.action.index.IndexResponse; -import org.opensearch.core.index.shard.ShardId; -import org.opensearch.knn.KNNSingleNodeTestCase; -import org.opensearch.knn.indices.Model; -import org.opensearch.knn.indices.ModelDao; -import org.opensearch.knn.indices.ModelMetadata; -import org.opensearch.knn.indices.ModelState; -import org.opensearch.knn.training.TrainingJob; +import org.mockito.MockedStatic; +import org.opensearch.action.support.ActionFilters; +import org.opensearch.cluster.node.DiscoveryNode; +import org.opensearch.knn.KNNTestCase; import org.opensearch.knn.training.TrainingJobRunner; import org.opensearch.threadpool.ThreadPool; +import org.opensearch.transport.TransportService; -import java.io.IOException; -import java.util.concurrent.*; +import java.util.Collections; import static org.mockito.Mockito.any; -import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.doNothing; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.mockStatic; import static org.mockito.Mockito.when; -import static org.opensearch.knn.common.KNNConstants.MODEL_INDEX_NAME; -import static org.opensearch.knn.common.KNNConstants.TRAIN_THREAD_POOL; - -public class TrainingJobRouteDecisionInfoTransportActionTests extends KNNSingleNodeTestCase { - - ExecutorService executorService; - - @Before - public void setup() { - executorService = Executors.newSingleThreadExecutor(); - } - - @After - public void teardown() { - executorService.shutdown(); - } - - @SuppressWarnings("unchecked") - public void testNodeOperation() throws IOException, InterruptedException, ExecutionException { - // Ensure initial value of train job count is 0 - TrainingJobRouteDecisionInfoTransportAction action = node().injector() - .getInstance(TrainingJobRouteDecisionInfoTransportAction.class); - - TrainingJobRouteDecisionInfoNodeRequest request = new TrainingJobRouteDecisionInfoNodeRequest(); - - TrainingJobRouteDecisionInfoNodeResponse response1 = action.nodeOperation(request); - assertEquals(0, response1.getTrainingJobCount().intValue()); - - // Setup mocked training job - String modelId = "model-id"; - Model model = mock(Model.class); - ModelMetadata modelMetadata = mock(ModelMetadata.class); - when(modelMetadata.getState()).thenReturn(ModelState.TRAINING); - when(model.getModelMetadata()).thenReturn(modelMetadata); - TrainingJob trainingJob = mock(TrainingJob.class); - when(trainingJob.getModelId()).thenReturn(modelId); - when(trainingJob.getModel()).thenReturn(model); - doAnswer(invocationOnMock -> null).when(trainingJob).run(); - - ModelDao modelDao = mock(ModelDao.class); - when(modelDao.get(modelId)).thenReturn(model); - - // Here we check to make sure there is a running job - doAnswer(invocationOnMock -> { - TrainingJobRouteDecisionInfoNodeResponse response2 = action.nodeOperation(request); - assertEquals(1, response2.getTrainingJobCount().intValue()); - - IndexResponse indexResponse = new IndexResponse(new ShardId(MODEL_INDEX_NAME, "uuid", 0), modelId, 0, 0, 0, true); - ((ActionListener) invocationOnMock.getArguments()[1]).onResponse(indexResponse); - return null; - }).when(modelDao).put(any(Model.class), any(ActionListener.class)); - - // Set up the rest of the training logic - final CountDownLatch inProgressLatch = new CountDownLatch(1); - ActionListener responseListener = ActionListener.wrap( - indexResponse -> { inProgressLatch.countDown(); }, - e -> fail("Failure should not have occurred") - ); - - doAnswer(invocationOnMock -> { - responseListener.onResponse(mock(IndexResponse.class)); - return null; - }).when(modelDao).update(model, responseListener); +public class TrainingJobRouteDecisionInfoTransportActionTests extends KNNTestCase { + public void testNodeOperation() { + // Initialize mocked variables for the class + DiscoveryNode node = mock(DiscoveryNode.class); + when(clusterService.localNode()).thenReturn(node); ThreadPool threadPool = mock(ThreadPool.class); - when(threadPool.executor(TRAIN_THREAD_POOL)).thenReturn(executorService); - - // Initialize runner and execute job - TrainingJobRunner.initialize(threadPool, modelDao); - TrainingJobRunner.getInstance().execute(trainingJob, responseListener); - - assertTrue(inProgressLatch.await(100, TimeUnit.SECONDS)); + TransportService transportService = mock(TransportService.class); + doNothing().when(transportService).registerRequestHandler(any(), any(), any(), any()); + ActionFilters actionFilters = new ActionFilters(Collections.emptySet()); + + TrainingJobRouteDecisionInfoTransportAction trainingJobRouteDecisionInfoTransportAction = + new TrainingJobRouteDecisionInfoTransportAction(threadPool, clusterService, transportService, actionFilters); + + try (MockedStatic mockedTrainingJobRunnerStatic = mockStatic(TrainingJobRunner.class)) { + // Ensure the job count is correct + int initialJobCount = 4; + final TrainingJobRunner mockedTrainingJobRunner = mock(TrainingJobRunner.class); + when(mockedTrainingJobRunner.getJobCount()).thenReturn(initialJobCount); + mockedTrainingJobRunnerStatic.when(TrainingJobRunner::getInstance).thenReturn(mockedTrainingJobRunner); + + TrainingJobRouteDecisionInfoNodeRequest request = new TrainingJobRouteDecisionInfoNodeRequest(); + TrainingJobRouteDecisionInfoNodeResponse response = trainingJobRouteDecisionInfoTransportAction.nodeOperation(request); + assertEquals(initialJobCount, response.getTrainingJobCount().intValue()); + + int resetJobCount = 0; + when(mockedTrainingJobRunner.getJobCount()).thenReturn(resetJobCount); + response = trainingJobRouteDecisionInfoTransportAction.nodeOperation(request); + assertEquals(resetJobCount, response.getTrainingJobCount().intValue()); + } } } From 722bc634b59a16836d32c70f44a318a9f5e3962f Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Fri, 5 Jan 2024 11:36:11 -0800 Subject: [PATCH 177/416] Enabled Filtering on Nested Vector fields with top level filters (#1372) (#1377) Signed-off-by: Navneet Verma (cherry picked from commit 271df52ea5d95d0f3b5f8e8b984878ba4b23b97b) Co-authored-by: Navneet Verma --- CHANGELOG.md | 3 +- .../knn/index/query/KNNQueryFactory.java | 20 +- .../index/AdvancedFilteringUseCasesIT.java | 549 ++++++++++++++++++ .../opensearch/knn/index/NestedSearchIT.java | 62 +- .../knn/index/query/KNNQueryFactoryTests.java | 60 ++ .../org/opensearch/knn/KNNRestTestCase.java | 26 + .../opensearch/knn/NestedKnnDocBuilder.java | 90 +++ 7 files changed, 753 insertions(+), 57 deletions(-) create mode 100644 src/test/java/org/opensearch/knn/index/AdvancedFilteringUseCasesIT.java create mode 100644 src/testFixtures/java/org/opensearch/knn/NestedKnnDocBuilder.java diff --git a/CHANGELOG.md b/CHANGELOG.md index b2ab6b446..aafc2e585 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,12 +12,13 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Maintenance ### Refactoring -## [Unreleased 2.x](https://github.com/opensearch-project/k-NN/compare/2.11...2.x) +## [Unreleased 2.x](https://github.com/opensearch-project/k-NN/compare/2.12...2.x) ### Features * Add parent join support for lucene knn [#1182](https://github.com/opensearch-project/k-NN/pull/1182) ### Enhancements * Increase Lucene max dimension limit to 16,000 [#1346](https://github.com/opensearch-project/k-NN/pull/1346) * Tuned default values for ef_search and ef_construction for better indexing and search performance for vector search [#1353](https://github.com/opensearch-project/k-NN/pull/1353) +* Enabled Filtering on Nested Vector fields with top level filters [#1372](https://github.com/opensearch-project/k-NN/pull/1372) ### Bug Fixes * Fix use-after-free case on nmslib search path [#1305](https://github.com/opensearch-project/k-NN/pull/1305) * Allow nested knn field mapping when train model [#1318](https://github.com/opensearch-project/k-NN/pull/1318) diff --git a/src/main/java/org/opensearch/knn/index/query/KNNQueryFactory.java b/src/main/java/org/opensearch/knn/index/query/KNNQueryFactory.java index c073450af..7772ab582 100644 --- a/src/main/java/org/opensearch/knn/index/query/KNNQueryFactory.java +++ b/src/main/java/org/opensearch/knn/index/query/KNNQueryFactory.java @@ -17,8 +17,10 @@ import org.apache.lucene.search.join.BitSetProducer; import org.apache.lucene.search.join.DiversifyingChildrenByteKnnVectorQuery; import org.apache.lucene.search.join.DiversifyingChildrenFloatKnnVectorQuery; +import org.apache.lucene.search.join.ToChildBlockJoinQuery; import org.opensearch.index.query.QueryBuilder; import org.opensearch.index.query.QueryShardContext; +import org.opensearch.index.search.NestedHelper; import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.util.KNNEngine; @@ -155,11 +157,27 @@ private static Query getFilterQuery(CreateQueryRequest createQueryRequest) { createQueryRequest.k ) ); + final Query filterQuery; try { - return createQueryRequest.getFilter().get().toQuery(queryShardContext); + filterQuery = createQueryRequest.getFilter().get().toQuery(queryShardContext); } catch (IOException e) { throw new RuntimeException("Cannot create knn query with filter", e); } + // If k-NN Field is nested field then parentFilter will not be null. This parentFilter is set by the + // Opensearch core. Ref PR: https://github.com/opensearch-project/OpenSearch/pull/10246 + if (queryShardContext.getParentFilter() != null) { + // if the filter is also a nested query clause then we should just return the same query without + // considering it to join with the parent documents. + if (new NestedHelper(queryShardContext.getMapperService()).mightMatchNestedDocs(filterQuery)) { + return filterQuery; + } + // This condition will be hit when filters are getting applied on the top level fields and k-nn + // query field is a nested field. In this case we need to wrap the filter query with + // ToChildBlockJoinQuery to ensure parent documents which will be retrieved from filters can be + // joined with the child documents containing vector field. + return new ToChildBlockJoinQuery(filterQuery, queryShardContext.getParentFilter()); + } + return filterQuery; } return null; } diff --git a/src/test/java/org/opensearch/knn/index/AdvancedFilteringUseCasesIT.java b/src/test/java/org/opensearch/knn/index/AdvancedFilteringUseCasesIT.java new file mode 100644 index 000000000..9ce994a78 --- /dev/null +++ b/src/test/java/org/opensearch/knn/index/AdvancedFilteringUseCasesIT.java @@ -0,0 +1,549 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.knn.index; + +import com.google.common.collect.ImmutableMap; +import lombok.SneakyThrows; +import org.apache.http.util.EntityUtils; +import org.junit.Assert; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.knn.KNNRestTestCase; +import org.opensearch.knn.NestedKnnDocBuilder; +import org.opensearch.knn.index.util.KNNEngine; + +import java.io.IOException; +import java.util.List; +import java.util.stream.Collectors; + +import static org.opensearch.knn.common.KNNConstants.DIMENSION; +import static org.opensearch.knn.common.KNNConstants.K; +import static org.opensearch.knn.common.KNNConstants.KNN; +import static org.opensearch.knn.common.KNNConstants.KNN_ENGINE; +import static org.opensearch.knn.common.KNNConstants.KNN_METHOD; +import static org.opensearch.knn.common.KNNConstants.METHOD_HNSW; +import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_SPACE_TYPE; +import static org.opensearch.knn.common.KNNConstants.NAME; +import static org.opensearch.knn.common.KNNConstants.PATH; +import static org.opensearch.knn.common.KNNConstants.QUERY; +import static org.opensearch.knn.common.KNNConstants.TYPE; +import static org.opensearch.knn.common.KNNConstants.TYPE_KNN_VECTOR; +import static org.opensearch.knn.common.KNNConstants.TYPE_NESTED; +import static org.opensearch.knn.common.KNNConstants.VECTOR; + +/** + * This class contains the IT for some advanced and tricky use-case of filters. + * Github issue + */ +public class AdvancedFilteringUseCasesIT extends KNNRestTestCase { + + private static final String INDEX_NAME = "advanced_filtering_test_index"; + + private static final String FIELD_NAME_NESTED = "test_nested"; + + private static final String FIELD_NAME_VECTOR = "test_vector"; + + private static final String PROPERTIES_FIELD = "properties"; + + private static final String FILTER_FIELD = "filter"; + + private static final String TERM_FIELD = "term"; + + private static final int k = 20; + + private static final String FIELD_NAME_METADATA = "parking"; + + private static final int NUM_DOCS = 50; + + private static final int DOCUMENT_IN_RESPONSE = 10; + + private static final Float[] QUERY_VECTOR = { 5f }; + + private static final List enginesToTest = KNNEngine.getEnginesThatSupportsFilters() + .stream() + .map(KNNEngine::getName) + .collect(Collectors.toList()); + + /** + * { + * "query": { + * "nested": { + * "path": "test_nested", + * "query": { + * "knn": { + * "test_nested.test_vector": { + * "vector": [ + * 3 + * ], + * "k": 20, + * "filter": { + * "nested": { + * "path": "test_nested", + * "query": { + * "term": { + * "test_nested.parking": "false" + * } + * } + * } + * } + * } + * } + * } + * } + * } + * } + */ + @SneakyThrows + public void testFiltering_whenNestedKNNAndFilterFieldWithNestedQueries_thenSuccess() { + for (final String engine : enginesToTest) { + // Set up the index with nested k-nn and metadata fields + createKnnIndex(INDEX_NAME, createNestedMappings(1, engine)); + for (int i = 1; i <= NUM_DOCS; i++) { + // making sure that only 2 documents have valid filters + final String metadataFieldValue = i % 2 == 0 ? "false" : "true"; + String doc = NestedKnnDocBuilder.create(FIELD_NAME_NESTED) + .addVectors(FIELD_NAME_VECTOR, new Float[] { (float) i + 1 }) + .addVectorWithMetadata(FIELD_NAME_VECTOR, new Float[] { (float) i }, FIELD_NAME_METADATA, metadataFieldValue) + .build(); + addKnnDoc(INDEX_NAME, String.valueOf(i), doc); + } + refreshIndex(INDEX_NAME); + forceMergeKnnIndex(INDEX_NAME); + + // Build the query with both k-nn and filters as nested fields. The filter should also have a nested context + final XContentBuilder builder = XContentFactory.jsonBuilder().startObject().startObject(QUERY); + builder.startObject(TYPE_NESTED); + builder.field(PATH, FIELD_NAME_NESTED); + builder.startObject(QUERY).startObject(KNN).startObject(FIELD_NAME_NESTED + "." + FIELD_NAME_VECTOR); + builder.field(VECTOR, QUERY_VECTOR); + builder.field(K, k); + builder.startObject(FILTER_FIELD); + builder.startObject(TYPE_NESTED); + builder.field(PATH, FIELD_NAME_NESTED); + builder.startObject(QUERY); + builder.startObject(TERM_FIELD); + builder.field(FIELD_NAME_NESTED + "." + FIELD_NAME_METADATA, "false"); + builder.endObject(); + builder.endObject(); + builder.endObject(); + builder.endObject(); + builder.endObject().endObject().endObject().endObject().endObject().endObject(); + + validateFilterSearch(builder.toString(), engine); + // cleanup + deleteKNNIndex(INDEX_NAME); + } + } + + /** + * { + * "query": { + * "nested": { + * "path": "test_nested", + * "query": { + * "knn": { + * "test_nested.test_vector": { + * "vector": [ + * 3 + * ], + * "k": 20, + * "filter": { + * "term": { + * "test_nested.parking": "false" + * } + * } + * } + * } + * } + * } + * } + * } + */ + @SneakyThrows + public void testFiltering_whenNestedKNNAndFilterFieldWithNoNestedContextInFilterQuery_thenSuccess() { + for (final String engine : enginesToTest) { + // Set up the index with nested k-nn and metadata fields + createKnnIndex(INDEX_NAME, createNestedMappings(1, engine)); + for (int i = 1; i <= NUM_DOCS; i++) { + // making sure that only 2 documents have valid filters + final String metadataFieldValue = i % 2 == 0 ? "false" : "true"; + String doc = NestedKnnDocBuilder.create(FIELD_NAME_NESTED) + .addVectors(FIELD_NAME_VECTOR, new Float[] { (float) i + 1 }) + .addVectorWithMetadata(FIELD_NAME_VECTOR, new Float[] { (float) i }, FIELD_NAME_METADATA, metadataFieldValue) + .build(); + addKnnDoc(INDEX_NAME, String.valueOf(i), doc); + } + refreshIndex(INDEX_NAME); + forceMergeKnnIndex(INDEX_NAME); + + // Build the query with both k-nn and filters as nested fields but a single nested context + final XContentBuilder builder = XContentFactory.jsonBuilder().startObject().startObject(QUERY); + builder.startObject(TYPE_NESTED); + builder.field(PATH, FIELD_NAME_NESTED); + builder.startObject(QUERY).startObject(KNN).startObject(FIELD_NAME_NESTED + "." + FIELD_NAME_VECTOR); + builder.field(VECTOR, QUERY_VECTOR); + builder.field(K, k); + builder.startObject(FILTER_FIELD); + builder.startObject(TERM_FIELD); + builder.field(FIELD_NAME_NESTED + "." + FIELD_NAME_METADATA, "false"); + builder.endObject(); + builder.endObject(); + builder.endObject().endObject().endObject().endObject().endObject().endObject(); + + validateFilterSearch(builder.toString(), engine); + + // cleanup + deleteKNNIndex(INDEX_NAME); + } + } + + /** + * { + * "query": { + * "nested": { + * "path": "test_nested", + * "query": { + * "knn": { + * "test_nested.test_vector": { + * "vector": [ + * 3 + * ], + * "k": 20, + * "filter": { + * "term": { + * "parking": "false" + * } + * } + * } + * } + * } + * } + * } + * } + * + */ + @SneakyThrows + public void testFiltering_whenNestedKNNAndNonNestedFilterFieldWithNonNestedFilterQuery_thenSuccess() { + for (final String engine : enginesToTest) { + // Set up the index with nested k-nn and metadata fields + createKnnIndex(INDEX_NAME, createNestedMappings(1, engine)); + for (int i = 1; i <= NUM_DOCS; i++) { + final String metadataFieldValue = i % 2 == 0 ? "false" : "true"; + String doc = NestedKnnDocBuilder.create(FIELD_NAME_NESTED) + .addVectors(FIELD_NAME_VECTOR, new Float[] { (float) i + 1 }, new Float[] { (float) i }) + .addTopLevelField(FIELD_NAME_METADATA, metadataFieldValue) + .build(); + addKnnDoc(INDEX_NAME, String.valueOf(i), doc); + } + refreshIndex(INDEX_NAME); + forceMergeKnnIndex(INDEX_NAME); + + // Build the query with k-nn field as nested query and filter on the top level fields + final XContentBuilder builder = XContentFactory.jsonBuilder().startObject().startObject(QUERY); + builder.startObject(TYPE_NESTED); + builder.field(PATH, FIELD_NAME_NESTED); + builder.startObject(QUERY).startObject(KNN).startObject(FIELD_NAME_NESTED + "." + FIELD_NAME_VECTOR); + builder.field(VECTOR, QUERY_VECTOR); + builder.field(K, k); + builder.startObject(FILTER_FIELD); + builder.startObject(TERM_FIELD); + builder.field(FIELD_NAME_METADATA, "false"); + builder.endObject(); + builder.endObject(); + builder.endObject().endObject().endObject().endObject().endObject().endObject(); + + validateFilterSearch(builder.toString(), engine); + + // cleanup + deleteKNNIndex(INDEX_NAME); + } + } + + /** + * { + * "query": { + * "knn": { + * "test_vector": { + * "vector": [ + * 3 + * ], + * "k": 20, + * "filter": { + * "bool": { + * "should": [ + * { + * "nested": { + * "path": "test_nested", + * "query": { + * "term": { + * "test_nested.parking": "false" + * } + * } + * } + * } + * ] + * } + * } + * } + * } + * } + * } + */ + @SneakyThrows + public void testFiltering_whenNonNestedKNNAndNestedFilterFieldWithNestedFilterQuery_thenSuccess() { + for (final String engine : enginesToTest) { + // Set up the index with nested k-nn and metadata fields + createKnnIndex(INDEX_NAME, createVectorNonNestedMappings(1, engine)); + for (int i = 1; i <= NUM_DOCS; i++) { + final String metadataFieldValue = i % 2 == 0 ? "false" : "true"; + String doc = NestedKnnDocBuilder.create(FIELD_NAME_NESTED) + .addMetadata(ImmutableMap.of(FIELD_NAME_METADATA, metadataFieldValue)) + .addTopLevelField(FIELD_NAME_VECTOR, new Float[] { (float) i }) + .build(); + addKnnDoc(INDEX_NAME, String.valueOf(i), doc); + } + refreshIndex(INDEX_NAME); + forceMergeKnnIndex(INDEX_NAME); + + // Build the query when filters are nested with nested path and k-NN field is non nested. + final XContentBuilder builder = XContentFactory.jsonBuilder().startObject().startObject(QUERY); + builder.startObject(KNN).startObject(FIELD_NAME_VECTOR); + builder.field(VECTOR, QUERY_VECTOR); + builder.field(K, k); + builder.startObject(FILTER_FIELD); + builder.startObject("bool"); + builder.startArray("should"); + builder.startObject(); + + builder.startObject(TYPE_NESTED); + builder.field(PATH, FIELD_NAME_NESTED); + + builder.startObject(QUERY); + builder.startObject(TERM_FIELD); + builder.field(FIELD_NAME_NESTED + "." + FIELD_NAME_METADATA, "false"); + builder.endObject(); + builder.endObject(); + + builder.endObject(); + + builder.endObject(); + builder.endArray(); + builder.endObject(); + builder.endObject(); + builder.endObject(); + builder.endObject(); + builder.endObject(); + builder.endObject(); + + validateFilterSearch(builder.toString(), engine); + // cleanup + deleteKNNIndex(INDEX_NAME); + } + } + + /** + * { + * "query": { + * "knn": { + * "test_vector": { + * "vector": [ + * 5 + * ], + * "k": 20, + * "filter": { + * "bool": { + * "must": [ + * { + * "nested": { + * "path": "test_nested", + * "query": { + * "term": { + * "test_nested.parking": "false" + * } + * } + * } + * }, + * { + * "term": { + * "parking": "false" + * } + * } + * ] + * } + * } + * } + * } + * } + * } + */ + @SneakyThrows + public void testFiltering_whenNonNestedKNNAndNestedFilterAndNonNestedFieldWithNestedAndNonNestedFilterQuery_thenSuccess() { + for (final String engine : enginesToTest) { + // Set up the index with nested k-nn and metadata fields + createKnnIndex(INDEX_NAME, createVectorNonNestedMappings(1, engine)); + for (int i = 1; i <= NUM_DOCS; i++) { + final String metadataFieldValue = i % 2 == 0 ? "false" : "true"; + String doc = NestedKnnDocBuilder.create(FIELD_NAME_NESTED) + .addMetadata(ImmutableMap.of(FIELD_NAME_METADATA, metadataFieldValue)) + .addTopLevelField(FIELD_NAME_VECTOR, new Float[] { (float) i }) + .addTopLevelField(FIELD_NAME_METADATA, metadataFieldValue) + .build(); + addKnnDoc(INDEX_NAME, String.valueOf(i), doc); + } + refreshIndex(INDEX_NAME); + forceMergeKnnIndex(INDEX_NAME); + + // Build the query when filters are nested with nested path and k-NN field is non nested. + final XContentBuilder builder = XContentFactory.jsonBuilder().startObject().startObject(QUERY); + builder.startObject(KNN).startObject(FIELD_NAME_VECTOR); + builder.field(VECTOR, QUERY_VECTOR); + builder.field(K, k); + builder.startObject(FILTER_FIELD); + builder.startObject("bool"); + builder.startArray("must"); + builder.startObject(); + builder.startObject(TERM_FIELD); + builder.field(FIELD_NAME_METADATA, "false"); + builder.endObject(); + builder.endObject(); + + builder.startObject(); + + builder.startObject(TYPE_NESTED); + builder.field(PATH, FIELD_NAME_NESTED); + + builder.startObject(QUERY); + builder.startObject(TERM_FIELD); + builder.field(FIELD_NAME_NESTED + "." + FIELD_NAME_METADATA, "false"); + builder.endObject(); + builder.endObject(); + + builder.endObject(); + + builder.endObject(); + builder.endArray(); + builder.endObject(); + builder.endObject(); + builder.endObject(); + builder.endObject(); + builder.endObject(); + builder.endObject(); + + validateFilterSearch(builder.toString(), engine); + // cleanup + deleteKNNIndex(INDEX_NAME); + } + } + + private void validateFilterSearch(final String query, final String engine) throws IOException { + String response = EntityUtils.toString(performSearch(INDEX_NAME, query).getEntity()); + // Validate number of documents returned as the expected number of documents + Assert.assertEquals("For engine " + engine + " : ", DOCUMENT_IN_RESPONSE, parseHits(response)); + if (KNNEngine.getEngine(engine) == KNNEngine.FAISS) { + // Update the filter threshold to 0 to ensure that we are hitting ANN Search use case for FAISS + updateIndexSettings(INDEX_NAME, Settings.builder().put(KNNSettings.ADVANCED_FILTERED_EXACT_SEARCH_THRESHOLD, 0)); + response = EntityUtils.toString(performSearch(INDEX_NAME, query).getEntity()); + + // Validate number of documents returned as the expected number of documents + Assert.assertEquals("For engine " + engine + " with ANN search :", DOCUMENT_IN_RESPONSE, parseHits(response)); + } + } + + /** + * Sample return + * { + * "properties": { + * "test_nested": { + * "type": "nested", + * "properties": { + * "test_vector": { + * "type": "knn_vector", + * "dimension": 1, + * "method": { + * "name": "hnsw", + * "space_type": "l2", + * "engine": "lucene" + * } + * } + * } + * } + * } + * } + */ + @SneakyThrows + private String createNestedMappings(final int dimension, final String engine) { + XContentBuilder builder = XContentFactory.jsonBuilder() + .startObject() + .startObject(PROPERTIES_FIELD) + .startObject(FIELD_NAME_NESTED) + .field(TYPE, TYPE_NESTED) + .startObject(PROPERTIES_FIELD) + .startObject(FIELD_NAME_VECTOR) + .field(TYPE, TYPE_KNN_VECTOR) + .field(DIMENSION, dimension) + .startObject(KNN_METHOD) + .field(NAME, METHOD_HNSW) + .field(METHOD_PARAMETER_SPACE_TYPE, SpaceType.L2.getValue()) + .field(KNN_ENGINE, engine) + .endObject() + .endObject() + .endObject() + .endObject() + .endObject() + .endObject(); + + return builder.toString(); + } + + /** + * Sample return + * { + * "properties": { + * "test_vector": { + * "type": "knn_vector", + * "dimension": 1, + * "method": { + * "name": "hnsw", + * "space_type": "l2", + * "engine": "lucene" + * } + * }, + * "test_nested": { + * "type": "nested" + * } + * } + * } + */ + @SneakyThrows + private String createVectorNonNestedMappings(final int dimension, final String engine) { + XContentBuilder builder = XContentFactory.jsonBuilder() + .startObject() + .startObject(PROPERTIES_FIELD) + .startObject(FIELD_NAME_NESTED) + .field(TYPE, TYPE_NESTED) + .endObject() + .startObject(FIELD_NAME_VECTOR) + .field(TYPE, TYPE_KNN_VECTOR) + .field(DIMENSION, dimension) + .startObject(KNN_METHOD) + .field(NAME, METHOD_HNSW) + .field(METHOD_PARAMETER_SPACE_TYPE, SpaceType.L2.getValue()) + .field(KNN_ENGINE, engine) + .endObject() + .endObject() + .endObject() + .endObject(); + + return builder.toString(); + } +} diff --git a/src/test/java/org/opensearch/knn/index/NestedSearchIT.java b/src/test/java/org/opensearch/knn/index/NestedSearchIT.java index dce006b50..21f3bb27a 100644 --- a/src/test/java/org/opensearch/knn/index/NestedSearchIT.java +++ b/src/test/java/org/opensearch/knn/index/NestedSearchIT.java @@ -15,6 +15,7 @@ import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.knn.KNNRestTestCase; +import org.opensearch.knn.NestedKnnDocBuilder; import org.opensearch.knn.index.util.KNNEngine; import java.io.IOException; @@ -59,14 +60,16 @@ public void testNestedSearch_whenKIsTwo_thenReturnTwoResults() { createKnnIndex(2, KNNEngine.LUCENE.getName()); String doc1 = NestedKnnDocBuilder.create(FIELD_NAME_NESTED) - .add(FIELD_NAME_VECTOR, new Float[] { 1f, 1f }, new Float[] { 1f, 1f }) + .addVectors(FIELD_NAME_VECTOR, new Float[] { 1f, 1f }, new Float[] { 1f, 1f }) .build(); - addNestedKnnDoc(INDEX_NAME, "1", doc1); + addKnnDoc(INDEX_NAME, "1", doc1); String doc2 = NestedKnnDocBuilder.create(FIELD_NAME_NESTED) - .add(FIELD_NAME_VECTOR, new Float[] { 2f, 2f }, new Float[] { 2f, 2f }) + .addVectors(FIELD_NAME_VECTOR, new Float[] { 2f, 2f }, new Float[] { 2f, 2f }) .build(); - addNestedKnnDoc(INDEX_NAME, "2", doc2); + addKnnDoc(INDEX_NAME, "2", doc2); + + refreshIndex(INDEX_NAME); Float[] queryVector = { 1f, 1f }; Response response = queryNestedField(INDEX_NAME, 2, queryVector); @@ -131,30 +134,6 @@ private void createKnnIndex(final int dimension, final String engine) throws Exc createKnnIndex(INDEX_NAME, mapping); } - @SneakyThrows - private void ingestTestData() { - String doc1 = NestedKnnDocBuilder.create(FIELD_NAME_NESTED) - .add(FIELD_NAME_VECTOR, new Float[] { 1f, 1f }, new Float[] { 1f, 1f }) - .build(); - addNestedKnnDoc(INDEX_NAME, "1", doc1); - - String doc2 = NestedKnnDocBuilder.create(FIELD_NAME_NESTED) - .add(FIELD_NAME_VECTOR, new Float[] { 2f, 2f }, new Float[] { 2f, 2f }) - .build(); - addNestedKnnDoc(INDEX_NAME, "2", doc2); - } - - private void addNestedKnnDoc(final String index, final String docId, final String document) throws IOException { - Request request = new Request("POST", "/" + index + "/_doc/" + docId + "?refresh=true"); - - request.setJsonEntity(document); - client().performRequest(request); - - request = new Request("POST", "/" + index + "/_refresh"); - Response response = client().performRequest(request); - assertEquals(request.getEndpoint() + ": failed", RestStatus.OK, RestStatus.fromCode(response.getStatusLine().getStatusCode())); - } - private Response queryNestedField(final String index, final int k, final Object[] vector) throws IOException { XContentBuilder builder = XContentFactory.jsonBuilder().startObject().startObject(QUERY); builder.startObject(TYPE_NESTED); @@ -172,31 +151,4 @@ private Response queryNestedField(final String index, final int k, final Object[ return response; } - - private static class NestedKnnDocBuilder { - private XContentBuilder builder; - - public NestedKnnDocBuilder(final String fieldName) throws IOException { - builder = XContentFactory.jsonBuilder().startObject().startArray(fieldName); - } - - public static NestedKnnDocBuilder create(final String fieldName) throws IOException { - return new NestedKnnDocBuilder(fieldName); - } - - public NestedKnnDocBuilder add(final String fieldName, final Object[]... vectors) throws IOException { - for (Object[] vector : vectors) { - builder.startObject(); - builder.field(fieldName, vector); - builder.endObject(); - } - return this; - } - - public String build() throws IOException { - builder.endArray().endObject(); - return builder.toString(); - } - - } } diff --git a/src/test/java/org/opensearch/knn/index/query/KNNQueryFactoryTests.java b/src/test/java/org/opensearch/knn/index/query/KNNQueryFactoryTests.java index a6b915a85..62d2db544 100644 --- a/src/test/java/org/opensearch/knn/index/query/KNNQueryFactoryTests.java +++ b/src/test/java/org/opensearch/knn/index/query/KNNQueryFactoryTests.java @@ -12,11 +12,15 @@ import org.apache.lucene.search.join.BitSetProducer; import org.apache.lucene.search.join.DiversifyingChildrenByteKnnVectorQuery; import org.apache.lucene.search.join.DiversifyingChildrenFloatKnnVectorQuery; +import org.apache.lucene.search.join.ToChildBlockJoinQuery; +import org.mockito.MockedConstruction; import org.mockito.Mockito; import org.opensearch.index.mapper.MappedFieldType; +import org.opensearch.index.mapper.MapperService; import org.opensearch.index.query.QueryBuilder; import org.opensearch.index.query.QueryShardContext; import org.opensearch.index.query.TermQueryBuilder; +import org.opensearch.index.search.NestedHelper; import org.opensearch.knn.KNNTestCase; import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.util.KNNEngine; @@ -131,6 +135,62 @@ public void testCreate_whenLuceneWithParentFilter_thenReturnDiversifyingQuery() validateDiversifyingQueryWithParentFilter(VectorDataType.FLOAT, DiversifyingChildrenFloatKnnVectorQuery.class); } + public void testCreate_whenNestedVectorFiledAndNonNestedFilterField_thenReturnToChildBlockJoinQueryForFilters() { + MapperService mockMapperService = mock(MapperService.class); + QueryShardContext mockQueryShardContext = mock(QueryShardContext.class); + when(mockQueryShardContext.getMapperService()).thenReturn(mockMapperService); + MappedFieldType testMapper = mock(MappedFieldType.class); + when(mockQueryShardContext.fieldMapper(any())).thenReturn(testMapper); + when(testMapper.termQuery(Mockito.any(), Mockito.eq(mockQueryShardContext))).thenReturn(FILTER_QUERY); + BitSetProducer parentFilter = mock(BitSetProducer.class); + when(mockQueryShardContext.getParentFilter()).thenReturn(parentFilter); + MockedConstruction mockedNestedHelper = Mockito.mockConstruction( + NestedHelper.class, + (mock, context) -> when(mock.mightMatchNestedDocs(FILTER_QUERY)).thenReturn(false) + ); + + final KNNQueryFactory.CreateQueryRequest createQueryRequest = KNNQueryFactory.CreateQueryRequest.builder() + .knnEngine(KNNEngine.FAISS) + .indexName(testIndexName) + .fieldName(testFieldName) + .vector(testQueryVector) + .k(testK) + .context(mockQueryShardContext) + .filter(FILTER_QUERY_BUILDER) + .build(); + KNNQuery query = (KNNQuery) KNNQueryFactory.create(createQueryRequest); + mockedNestedHelper.close(); + assertEquals(ToChildBlockJoinQuery.class, query.getFilterQuery().getClass()); + } + + public void testCreate_whenNestedVectorAndFilterField_thenReturnSameFilterQuery() { + MapperService mockMapperService = mock(MapperService.class); + QueryShardContext mockQueryShardContext = mock(QueryShardContext.class); + when(mockQueryShardContext.getMapperService()).thenReturn(mockMapperService); + MappedFieldType testMapper = mock(MappedFieldType.class); + when(mockQueryShardContext.fieldMapper(any())).thenReturn(testMapper); + when(testMapper.termQuery(Mockito.any(), Mockito.eq(mockQueryShardContext))).thenReturn(FILTER_QUERY); + BitSetProducer parentFilter = mock(BitSetProducer.class); + when(mockQueryShardContext.getParentFilter()).thenReturn(parentFilter); + MockedConstruction mockedNestedHelper = Mockito.mockConstruction( + NestedHelper.class, + (mock, context) -> when(mock.mightMatchNestedDocs(FILTER_QUERY)).thenReturn(true) + ); + + final KNNQueryFactory.CreateQueryRequest createQueryRequest = KNNQueryFactory.CreateQueryRequest.builder() + .knnEngine(KNNEngine.FAISS) + .indexName(testIndexName) + .fieldName(testFieldName) + .vector(testQueryVector) + .k(testK) + .context(mockQueryShardContext) + .filter(FILTER_QUERY_BUILDER) + .build(); + KNNQuery query = (KNNQuery) KNNQueryFactory.create(createQueryRequest); + mockedNestedHelper.close(); + assertEquals(FILTER_QUERY.getClass(), query.getFilterQuery().getClass()); + } + private void validateDiversifyingQueryWithParentFilter(final VectorDataType type, final Class expectedQueryClass) { List luceneDefaultQueryEngineList = Arrays.stream(KNNEngine.values()) .filter(knnEngine -> !KNNEngine.getEnginesThatCreateCustomSegmentFiles().contains(knnEngine)) diff --git a/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java b/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java index f97396303..d32ce419d 100644 --- a/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java +++ b/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java @@ -222,6 +222,15 @@ protected Response searchExists(String index, ExistsQueryBuilder existsQueryBuil return response; } + protected Response performSearch(final String indexName, final String query) throws IOException { + Request request = new Request("POST", "/" + indexName + "/_search"); + request.setJsonEntity(query); + + Response response = client().performRequest(request); + assertEquals(request.getEndpoint() + ": failed", RestStatus.OK, RestStatus.fromCode(response.getStatusLine().getStatusCode())); + return response; + } + /** * Parse the response of KNN search into a List of KNNResults */ @@ -495,6 +504,15 @@ protected void addKnnDoc(String index, String docId, List fieldNames, Li assertEquals(request.getEndpoint() + ": failed", RestStatus.CREATED, RestStatus.fromCode(response.getStatusLine().getStatusCode())); } + /** + * Adds a doc where document is represented as a string. + */ + protected void addKnnDoc(final String index, final String docId, final String document) throws IOException { + Request request = new Request("POST", "/" + index + "/_doc/" + docId); + request.setJsonEntity(document); + client().performRequest(request); + } + /** * Add a single numeric field Doc to an index */ @@ -668,6 +686,14 @@ protected int parseTotalSearchHits(String searchResponseBody) throws IOException return (int) ((Map) responseMap.get("total")).get("value"); } + protected int parseHits(String searchResponseBody) throws IOException { + Map responseMap = (Map) createParser( + MediaTypeRegistry.getDefaultMediaType().xContent(), + searchResponseBody + ).map().get("hits"); + return ((List) responseMap.get("hits")).size(); + } + /** * Get the total number of graphs in the cache across all nodes */ diff --git a/src/testFixtures/java/org/opensearch/knn/NestedKnnDocBuilder.java b/src/testFixtures/java/org/opensearch/knn/NestedKnnDocBuilder.java new file mode 100644 index 000000000..dca58838a --- /dev/null +++ b/src/testFixtures/java/org/opensearch/knn/NestedKnnDocBuilder.java @@ -0,0 +1,90 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.knn; + +import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.core.xcontent.XContentBuilder; + +import java.io.IOException; +import java.util.Map; + +public class NestedKnnDocBuilder { + private XContentBuilder builder; + private boolean isNestedFieldBuildCompleted; + + public NestedKnnDocBuilder(final String fieldName) throws IOException { + isNestedFieldBuildCompleted = false; + builder = XContentFactory.jsonBuilder().startObject().startArray(fieldName); + } + + public static NestedKnnDocBuilder create(final String fieldName) throws IOException { + return new NestedKnnDocBuilder(fieldName); + } + + public NestedKnnDocBuilder addVectors(final String fieldName, final Object[]... vectors) throws IOException { + for (Object[] vector : vectors) { + builder.startObject(); + builder.field(fieldName, vector); + builder.endObject(); + } + return this; + } + + public NestedKnnDocBuilder addVectorWithMetadata( + final String fieldName, + final Object[] vectorValue, + final String metadataFieldName, + final Object metadataValue + ) throws IOException { + builder.startObject(); + builder.field(fieldName, vectorValue); + builder.field(metadataFieldName, metadataValue); + builder.endObject(); + return this; + } + + public NestedKnnDocBuilder addMetadata(final Map metadata) throws IOException { + builder.startObject(); + metadata.forEach((k, v) -> { + try { + builder.field(k, v); + } catch (IOException e) { + throw new RuntimeException(e); + } + }); + builder.endObject(); + return this; + } + + /** + * Use this function when you want to add top level fields in the document that contains nested fields. Once you + * run this function you cannot add anything in the nested field. + */ + public NestedKnnDocBuilder addTopLevelField(final String fieldName, final Object value) throws IOException { + if (isNestedFieldBuildCompleted == false) { + // Making sure that we close the building of nested field. + isNestedFieldBuildCompleted = true; + builder.endArray(); + } + builder.field(fieldName, value); + return this; + } + + public String build() throws IOException { + if (isNestedFieldBuildCompleted) { + builder.endObject(); + } else { + builder.endArray().endObject(); + } + return builder.toString(); + } +} From 7c65643551156d58b83cddbd2f7607945e1b81a7 Mon Sep 17 00:00:00 2001 From: Junqiu Lei Date: Tue, 9 Jan 2024 14:11:42 -0800 Subject: [PATCH 178/416] Throw proper exception to invalid k-NN query (#1380) (#1381) Signed-off-by: Junqiu Lei --- CHANGELOG.md | 1 + .../knn/index/query/KNNQueryBuilder.java | 6 ++ .../knn/index/VectorDataTypeIT.java | 51 +++++++++++++++ .../knn/index/query/KNNQueryBuilderTests.java | 65 +++++++++++++++++++ 4 files changed, 123 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index aafc2e585..a9c34eaf9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), * Increase Lucene max dimension limit to 16,000 [#1346](https://github.com/opensearch-project/k-NN/pull/1346) * Tuned default values for ef_search and ef_construction for better indexing and search performance for vector search [#1353](https://github.com/opensearch-project/k-NN/pull/1353) * Enabled Filtering on Nested Vector fields with top level filters [#1372](https://github.com/opensearch-project/k-NN/pull/1372) +* Throw proper exception to invalid k-NN query [#1380](https://github.com/opensearch-project/k-NN/pull/1380) ### Bug Fixes * Fix use-after-free case on nmslib search path [#1305](https://github.com/opensearch-project/k-NN/pull/1305) * Allow nested knn field mapping when train model [#1318](https://github.com/opensearch-project/k-NN/pull/1318) diff --git a/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java b/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java index 11912870f..0fa3835ac 100644 --- a/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java +++ b/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java @@ -100,8 +100,14 @@ public static void initialize(ModelDao modelDao) { } private static float[] ObjectsToFloats(List objs) { + if (Objects.isNull(objs) || objs.isEmpty()) { + throw new IllegalArgumentException(String.format("[%s] field 'vector' requires to be non-null and non-empty", NAME)); + } float[] vec = new float[objs.size()]; for (int i = 0; i < objs.size(); i++) { + if ((objs.get(i) instanceof Number) == false) { + throw new IllegalArgumentException(String.format("[%s] field 'vector' requires to be an array of numbers", NAME)); + } vec[i] = ((Number) objs.get(i)).floatValue(); } return vec; diff --git a/src/test/java/org/opensearch/knn/index/VectorDataTypeIT.java b/src/test/java/org/opensearch/knn/index/VectorDataTypeIT.java index d98cc2a14..96486e707 100644 --- a/src/test/java/org/opensearch/knn/index/VectorDataTypeIT.java +++ b/src/test/java/org/opensearch/knn/index/VectorDataTypeIT.java @@ -24,6 +24,7 @@ import org.opensearch.core.rest.RestStatus; import org.opensearch.script.Script; +import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -425,6 +426,56 @@ public void testKNNScriptScoreWithInvalidByteQueryVector() throws Exception { ); } + @SneakyThrows + public void testSearchWithInvalidSearchVectorType() { + createKnnIndexMappingWithLuceneEngine(2, SpaceType.L2, VectorDataType.FLOAT.getValue()); + ingestL2FloatTestData(); + Request request = new Request("POST", String.format("/%s/_search", INDEX_NAME)); + List invalidTypeQueryVector = new ArrayList<>(); + invalidTypeQueryVector.add(1.5); + invalidTypeQueryVector.add(2.5); + invalidTypeQueryVector.add("a"); + invalidTypeQueryVector.add(null); + XContentBuilder builder = XContentFactory.jsonBuilder() + .startObject() + .startObject("query") + .startObject("knn") + .startObject(FIELD_NAME) + .field("vector", invalidTypeQueryVector) + .field("k", 4) + .endObject() + .endObject() + .endObject() + .endObject(); + request.setJsonEntity(builder.toString()); + + ResponseException ex = expectThrows(ResponseException.class, () -> client().performRequest(request)); + assertEquals(400, ex.getResponse().getStatusLine().getStatusCode()); + assertTrue(ex.getMessage().contains("[knn] field 'vector' requires to be an array of numbers")); + } + + @SneakyThrows + public void testSearchWithMissingQueryVector() { + createKnnIndexMappingWithLuceneEngine(2, SpaceType.L2, VectorDataType.FLOAT.getValue()); + ingestL2FloatTestData(); + Request request = new Request("POST", String.format("/%s/_search", INDEX_NAME)); + XContentBuilder builder = XContentFactory.jsonBuilder() + .startObject() + .startObject("query") + .startObject("knn") + .startObject(FIELD_NAME) + .field("k", 4) + .endObject() + .endObject() + .endObject() + .endObject(); + request.setJsonEntity(builder.toString()); + + ResponseException ex = expectThrows(ResponseException.class, () -> client().performRequest(request)); + assertEquals(400, ex.getResponse().getStatusLine().getStatusCode()); + assertTrue(ex.getMessage().contains("[knn] field 'vector' requires to be non-null and non-empty")); + } + @SneakyThrows private void ingestL2ByteTestData() { Byte[] b1 = { 6, 6 }; diff --git a/src/test/java/org/opensearch/knn/index/query/KNNQueryBuilderTests.java b/src/test/java/org/opensearch/knn/index/query/KNNQueryBuilderTests.java index 1381e19be..5e62abec2 100644 --- a/src/test/java/org/opensearch/knn/index/query/KNNQueryBuilderTests.java +++ b/src/test/java/org/opensearch/knn/index/query/KNNQueryBuilderTests.java @@ -39,6 +39,7 @@ import org.opensearch.plugins.SearchPlugin; import java.io.IOException; +import java.util.ArrayList; import java.util.List; import java.util.Optional; @@ -149,6 +150,70 @@ public void testFromXcontent_WithFilter_UnsupportedClusterVersion() throws Excep expectThrows(IllegalArgumentException.class, () -> KNNQueryBuilder.fromXContent(contentParser)); } + public void testFromXContent_invalidQueryVectorType() throws Exception { + final ClusterService clusterService = mockClusterService(Version.CURRENT); + + final KNNClusterUtil knnClusterUtil = KNNClusterUtil.instance(); + knnClusterUtil.initialize(clusterService); + + List invalidTypeQueryVector = new ArrayList<>(); + invalidTypeQueryVector.add(1.5); + invalidTypeQueryVector.add(2.5); + invalidTypeQueryVector.add("a"); + invalidTypeQueryVector.add(null); + + XContentBuilder builder = XContentFactory.jsonBuilder(); + builder.startObject(); + builder.startObject(FIELD_NAME); + builder.field(KNNQueryBuilder.VECTOR_FIELD.getPreferredName(), invalidTypeQueryVector); + builder.field(KNNQueryBuilder.K_FIELD.getPreferredName(), K); + builder.endObject(); + builder.endObject(); + XContentParser contentParser = createParser(builder); + contentParser.nextToken(); + IllegalArgumentException exception = expectThrows( + IllegalArgumentException.class, + () -> KNNQueryBuilder.fromXContent(contentParser) + ); + assertTrue(exception.getMessage().contains("[knn] field 'vector' requires to be an array of numbers")); + } + + public void testFromXContent_missingQueryVector() throws Exception { + final ClusterService clusterService = mockClusterService(Version.CURRENT); + + final KNNClusterUtil knnClusterUtil = KNNClusterUtil.instance(); + knnClusterUtil.initialize(clusterService); + + // Test without vector field + XContentBuilder builderWithoutVectorField = XContentFactory.jsonBuilder(); + builderWithoutVectorField.startObject(); + builderWithoutVectorField.startObject(FIELD_NAME); + builderWithoutVectorField.field(KNNQueryBuilder.K_FIELD.getPreferredName(), K); + builderWithoutVectorField.endObject(); + builderWithoutVectorField.endObject(); + XContentParser contentParserWithoutVectorField = createParser(builderWithoutVectorField); + contentParserWithoutVectorField.nextToken(); + IllegalArgumentException exception = expectThrows( + IllegalArgumentException.class, + () -> KNNQueryBuilder.fromXContent(contentParserWithoutVectorField) + ); + assertTrue(exception.getMessage().contains("[knn] field 'vector' requires to be non-null and non-empty")); + + // Test empty vector field + List emptyQueryVector = new ArrayList<>(); + XContentBuilder builderWithEmptyVector = XContentFactory.jsonBuilder(); + builderWithEmptyVector.startObject(); + builderWithEmptyVector.startObject(FIELD_NAME); + builderWithEmptyVector.field(KNNQueryBuilder.VECTOR_FIELD.getPreferredName(), emptyQueryVector); + builderWithEmptyVector.field(KNNQueryBuilder.K_FIELD.getPreferredName(), K); + builderWithEmptyVector.endObject(); + builderWithEmptyVector.endObject(); + XContentParser contentParserWithEmptyVector = createParser(builderWithEmptyVector); + contentParserWithEmptyVector.nextToken(); + exception = expectThrows(IllegalArgumentException.class, () -> KNNQueryBuilder.fromXContent(contentParserWithEmptyVector)); + assertTrue(exception.getMessage().contains("[knn] field 'vector' requires to be non-null and non-empty")); + } + @Override protected NamedXContentRegistry xContentRegistry() { List list = ClusterModule.getNamedXWriteables(); From 7900dbb339ab56a56a09cf07c50381aeba084cbb Mon Sep 17 00:00:00 2001 From: Naveen Tatikonda Date: Thu, 11 Jan 2024 18:51:16 -0600 Subject: [PATCH 179/416] Bump lucene codec to 99 (#1383) (#1386) * Add Lucene Codec 9.9 Signed-off-by: Naveen Tatikonda * Fix import statements for Lucene95 Codec Signed-off-by: Naveen Tatikonda * Fix SegmentInfo Constructor in Test Signed-off-by: Naveen Tatikonda * Temporarily Ignore Old Codec Tests Signed-off-by: Naveen Tatikonda * Add CHANGELOG Signed-off-by: Naveen Tatikonda * Delete Old Codec Tests Signed-off-by: Naveen Tatikonda --------- Signed-off-by: Naveen Tatikonda (cherry picked from commit 45e9e542aef60ef7073ee726e6ac14dec27bfa04) --- CHANGELOG.md | 1 + .../KNN950PerFieldKnnVectorsFormat.java | 2 +- .../index/codec/KNN990Codec/KNN990Codec.java | 61 +++++++++++++++++++ .../KNN990PerFieldKnnVectorsFormat.java | 40 ++++++++++++ .../knn/index/codec/KNNCodecVersion.java | 22 ++++++- .../services/org.apache.lucene.codecs.Codec | 3 +- .../codec/KNN910Codec/KNN910CodecTests.java | 22 ------- .../codec/KNN920Codec/KNN920CodecTests.java | 23 ------- .../codec/KNN940Codec/KNN940CodecTests.java | 30 --------- .../KNN990CodecTests.java} | 20 +++--- .../knn/index/codec/KNNCodecFactoryTests.java | 2 +- .../knn/index/codec/KNNCodecTestUtil.java | 1 + .../knn/index/query/KNNWeightTests.java | 5 ++ 13 files changed, 142 insertions(+), 90 deletions(-) create mode 100644 src/main/java/org/opensearch/knn/index/codec/KNN990Codec/KNN990Codec.java create mode 100644 src/main/java/org/opensearch/knn/index/codec/KNN990Codec/KNN990PerFieldKnnVectorsFormat.java delete mode 100644 src/test/java/org/opensearch/knn/index/codec/KNN910Codec/KNN910CodecTests.java delete mode 100644 src/test/java/org/opensearch/knn/index/codec/KNN920Codec/KNN920CodecTests.java delete mode 100644 src/test/java/org/opensearch/knn/index/codec/KNN940Codec/KNN940CodecTests.java rename src/test/java/org/opensearch/knn/index/codec/{KNN950Codec/KNN950CodecTests.java => KNN990Codec/KNN990CodecTests.java} (68%) diff --git a/CHANGELOG.md b/CHANGELOG.md index a9c34eaf9..68041d7ff 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,4 +34,5 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), * Upgrade urllib to 1.26.17 [#1278](https://github.com/opensearch-project/k-NN/pull/1278) * Upgrade urllib to 1.26.18 [#1319](https://github.com/opensearch-project/k-NN/pull/1319) * Upgrade guava to 32.1.3 [#1319](https://github.com/opensearch-project/k-NN/pull/1319) +* Bump lucene codec to 99 [#1383](https://github.com/opensearch-project/k-NN/pull/1383) ### Refactoring diff --git a/src/main/java/org/opensearch/knn/index/codec/KNN950Codec/KNN950PerFieldKnnVectorsFormat.java b/src/main/java/org/opensearch/knn/index/codec/KNN950Codec/KNN950PerFieldKnnVectorsFormat.java index d9091b2a7..05ce7271f 100644 --- a/src/main/java/org/opensearch/knn/index/codec/KNN950Codec/KNN950PerFieldKnnVectorsFormat.java +++ b/src/main/java/org/opensearch/knn/index/codec/KNN950Codec/KNN950PerFieldKnnVectorsFormat.java @@ -5,7 +5,7 @@ package org.opensearch.knn.index.codec.KNN950Codec; -import org.apache.lucene.codecs.lucene95.Lucene95HnswVectorsFormat; +import org.apache.lucene.backward_codecs.lucene95.Lucene95HnswVectorsFormat; import org.opensearch.index.mapper.MapperService; import org.opensearch.knn.index.codec.BasePerFieldKnnVectorsFormat; import org.opensearch.knn.index.util.KNNEngine; diff --git a/src/main/java/org/opensearch/knn/index/codec/KNN990Codec/KNN990Codec.java b/src/main/java/org/opensearch/knn/index/codec/KNN990Codec/KNN990Codec.java new file mode 100644 index 000000000..4b8a1d3cd --- /dev/null +++ b/src/main/java/org/opensearch/knn/index/codec/KNN990Codec/KNN990Codec.java @@ -0,0 +1,61 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.codec.KNN990Codec; + +import lombok.Builder; +import org.apache.lucene.codecs.Codec; +import org.apache.lucene.codecs.CompoundFormat; +import org.apache.lucene.codecs.DocValuesFormat; +import org.apache.lucene.codecs.FilterCodec; +import org.apache.lucene.codecs.KnnVectorsFormat; +import org.apache.lucene.codecs.perfield.PerFieldKnnVectorsFormat; +import org.opensearch.knn.index.codec.KNNCodecVersion; +import org.opensearch.knn.index.codec.KNNFormatFacade; + +/** + * KNN Codec that wraps the Lucene Codec which is part of Lucene 9.9 + */ +public class KNN990Codec extends FilterCodec { + private static final KNNCodecVersion VERSION = KNNCodecVersion.V_9_9_0; + private final KNNFormatFacade knnFormatFacade; + private final PerFieldKnnVectorsFormat perFieldKnnVectorsFormat; + + /** + * No arg constructor that uses Lucene99 as the delegate + */ + public KNN990Codec() { + this(VERSION.getDefaultCodecDelegate(), VERSION.getPerFieldKnnVectorsFormat()); + } + + /** + * Sole constructor. When subclassing this codec, create a no-arg ctor and pass the delegate codec + * and a unique name to this ctor. + * + * @param delegate codec that will perform all operations this codec does not override + * @param knnVectorsFormat per field format for KnnVector + */ + @Builder + protected KNN990Codec(Codec delegate, PerFieldKnnVectorsFormat knnVectorsFormat) { + super(VERSION.getCodecName(), delegate); + knnFormatFacade = VERSION.getKnnFormatFacadeSupplier().apply(delegate); + perFieldKnnVectorsFormat = knnVectorsFormat; + } + + @Override + public DocValuesFormat docValuesFormat() { + return knnFormatFacade.docValuesFormat(); + } + + @Override + public CompoundFormat compoundFormat() { + return knnFormatFacade.compoundFormat(); + } + + @Override + public KnnVectorsFormat knnVectorsFormat() { + return perFieldKnnVectorsFormat; + } +} diff --git a/src/main/java/org/opensearch/knn/index/codec/KNN990Codec/KNN990PerFieldKnnVectorsFormat.java b/src/main/java/org/opensearch/knn/index/codec/KNN990Codec/KNN990PerFieldKnnVectorsFormat.java new file mode 100644 index 000000000..abf40f2ef --- /dev/null +++ b/src/main/java/org/opensearch/knn/index/codec/KNN990Codec/KNN990PerFieldKnnVectorsFormat.java @@ -0,0 +1,40 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.codec.KNN990Codec; + +import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat; +import org.opensearch.index.mapper.MapperService; +import org.opensearch.knn.index.codec.BasePerFieldKnnVectorsFormat; +import org.opensearch.knn.index.util.KNNEngine; + +import java.util.Optional; + +/** + * Class provides per field format implementation for Lucene Knn vector type + */ +public class KNN990PerFieldKnnVectorsFormat extends BasePerFieldKnnVectorsFormat { + + public KNN990PerFieldKnnVectorsFormat(final Optional mapperService) { + super( + mapperService, + Lucene99HnswVectorsFormat.DEFAULT_MAX_CONN, + Lucene99HnswVectorsFormat.DEFAULT_BEAM_WIDTH, + () -> new Lucene99HnswVectorsFormat(), + (maxConnm, beamWidth) -> new Lucene99HnswVectorsFormat(maxConnm, beamWidth) + ); + } + + @Override + /** + * This method returns the maximum dimension allowed from KNNEngine for Lucene codec + * + * @param fieldName Name of the field, ignored + * @return Maximum constant dimension set by KNNEngine + */ + public int getMaxDimensions(String fieldName) { + return KNNEngine.getMaxDimensionByEngine(KNNEngine.LUCENE); + } +} diff --git a/src/main/java/org/opensearch/knn/index/codec/KNNCodecVersion.java b/src/main/java/org/opensearch/knn/index/codec/KNNCodecVersion.java index cbf6680f7..505dd50a5 100644 --- a/src/main/java/org/opensearch/knn/index/codec/KNNCodecVersion.java +++ b/src/main/java/org/opensearch/knn/index/codec/KNNCodecVersion.java @@ -11,7 +11,8 @@ import org.apache.lucene.backward_codecs.lucene92.Lucene92Codec; import org.apache.lucene.codecs.Codec; import org.apache.lucene.backward_codecs.lucene94.Lucene94Codec; -import org.apache.lucene.codecs.lucene95.Lucene95Codec; +import org.apache.lucene.backward_codecs.lucene95.Lucene95Codec; +import org.apache.lucene.codecs.lucene99.Lucene99Codec; import org.apache.lucene.codecs.perfield.PerFieldKnnVectorsFormat; import org.opensearch.index.mapper.MapperService; import org.opensearch.knn.index.codec.KNN80Codec.KNN80CompoundFormat; @@ -23,6 +24,8 @@ import org.opensearch.knn.index.codec.KNN940Codec.KNN940PerFieldKnnVectorsFormat; import org.opensearch.knn.index.codec.KNN950Codec.KNN950Codec; import org.opensearch.knn.index.codec.KNN950Codec.KNN950PerFieldKnnVectorsFormat; +import org.opensearch.knn.index.codec.KNN990Codec.KNN990Codec; +import org.opensearch.knn.index.codec.KNN990Codec.KNN990PerFieldKnnVectorsFormat; import java.util.Optional; import java.util.function.BiFunction; @@ -92,9 +95,24 @@ public enum KNNCodecVersion { .knnVectorsFormat(new KNN950PerFieldKnnVectorsFormat(Optional.ofNullable(mapperService))) .build(), KNN950Codec::new + ), + + V_9_9_0( + "KNN990Codec", + new Lucene99Codec(), + new KNN990PerFieldKnnVectorsFormat(Optional.empty()), + (delegate) -> new KNNFormatFacade( + new KNN80DocValuesFormat(delegate.docValuesFormat()), + new KNN80CompoundFormat(delegate.compoundFormat()) + ), + (userCodec, mapperService) -> KNN990Codec.builder() + .delegate(userCodec) + .knnVectorsFormat(new KNN990PerFieldKnnVectorsFormat(Optional.ofNullable(mapperService))) + .build(), + KNN990Codec::new ); - private static final KNNCodecVersion CURRENT = V_9_5_0; + private static final KNNCodecVersion CURRENT = V_9_9_0; private final String codecName; private final Codec defaultCodecDelegate; diff --git a/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec b/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec index 5c44d5756..308b37967 100644 --- a/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec +++ b/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec @@ -5,4 +5,5 @@ org.opensearch.knn.index.codec.KNN87Codec.KNN87Codec org.opensearch.knn.index.codec.KNN910Codec.KNN910Codec org.opensearch.knn.index.codec.KNN920Codec.KNN920Codec org.opensearch.knn.index.codec.KNN940Codec.KNN940Codec -org.opensearch.knn.index.codec.KNN950Codec.KNN950Codec \ No newline at end of file +org.opensearch.knn.index.codec.KNN950Codec.KNN950Codec +org.opensearch.knn.index.codec.KNN990Codec.KNN990Codec \ No newline at end of file diff --git a/src/test/java/org/opensearch/knn/index/codec/KNN910Codec/KNN910CodecTests.java b/src/test/java/org/opensearch/knn/index/codec/KNN910Codec/KNN910CodecTests.java deleted file mode 100644 index 1ec28d6a4..000000000 --- a/src/test/java/org/opensearch/knn/index/codec/KNN910Codec/KNN910CodecTests.java +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.knn.index.codec.KNN910Codec; - -import org.opensearch.knn.index.codec.KNNCodecTestCase; - -import java.io.IOException; -import java.util.concurrent.ExecutionException; - -public class KNN910CodecTests extends KNNCodecTestCase { - - public void testMultiFieldsKnnIndex() throws Exception { - testMultiFieldsKnnIndex(new KNN910Codec()); - } - - public void testBuildFromModelTemplate() throws InterruptedException, ExecutionException, IOException { - testBuildFromModelTemplate(new KNN910Codec()); - } -} diff --git a/src/test/java/org/opensearch/knn/index/codec/KNN920Codec/KNN920CodecTests.java b/src/test/java/org/opensearch/knn/index/codec/KNN920Codec/KNN920CodecTests.java deleted file mode 100644 index 8cdfc2d69..000000000 --- a/src/test/java/org/opensearch/knn/index/codec/KNN920Codec/KNN920CodecTests.java +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.knn.index.codec.KNN920Codec; - -import org.opensearch.knn.index.codec.KNNCodecTestCase; -import java.io.IOException; -import java.util.concurrent.ExecutionException; - -import static org.opensearch.knn.index.codec.KNNCodecVersion.V_9_2_0; - -public class KNN920CodecTests extends KNNCodecTestCase { - - public void testMultiFieldsKnnIndex() throws Exception { - testMultiFieldsKnnIndex(KNN920Codec.builder().delegate(V_9_2_0.getDefaultCodecDelegate()).build()); - } - - public void testBuildFromModelTemplate() throws InterruptedException, ExecutionException, IOException { - testBuildFromModelTemplate((KNN920Codec.builder().delegate(V_9_2_0.getDefaultCodecDelegate()).build())); - } -} diff --git a/src/test/java/org/opensearch/knn/index/codec/KNN940Codec/KNN940CodecTests.java b/src/test/java/org/opensearch/knn/index/codec/KNN940Codec/KNN940CodecTests.java deleted file mode 100644 index 805edac9d..000000000 --- a/src/test/java/org/opensearch/knn/index/codec/KNN940Codec/KNN940CodecTests.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.knn.index.codec.KNN940Codec; - -import org.apache.lucene.codecs.Codec; -import org.opensearch.knn.index.codec.KNNCodecTestCase; -import java.io.IOException; -import java.util.concurrent.ExecutionException; - -import static org.opensearch.knn.index.codec.KNNCodecVersion.V_9_4_0; - -public class KNN940CodecTests extends KNNCodecTestCase { - - public void testMultiFieldsKnnIndex() throws Exception { - testMultiFieldsKnnIndex(KNN940Codec.builder().delegate(V_9_4_0.getDefaultCodecDelegate()).build()); - } - - public void testBuildFromModelTemplate() throws InterruptedException, ExecutionException, IOException { - testBuildFromModelTemplate((KNN940Codec.builder().delegate(V_9_4_0.getDefaultCodecDelegate()).build())); - } - - // Ensure that the codec is able to return the correct per field knn vectors format for codec - public void testCodecSetsCustomPerFieldKnnVectorsFormat() { - final Codec codec = new KNN940Codec(); - assertTrue(codec.knnVectorsFormat() instanceof KNN940PerFieldKnnVectorsFormat); - } -} diff --git a/src/test/java/org/opensearch/knn/index/codec/KNN950Codec/KNN950CodecTests.java b/src/test/java/org/opensearch/knn/index/codec/KNN990Codec/KNN990CodecTests.java similarity index 68% rename from src/test/java/org/opensearch/knn/index/codec/KNN950Codec/KNN950CodecTests.java rename to src/test/java/org/opensearch/knn/index/codec/KNN990Codec/KNN990CodecTests.java index 8eafb6a4a..307ebbb24 100644 --- a/src/test/java/org/opensearch/knn/index/codec/KNN950Codec/KNN950CodecTests.java +++ b/src/test/java/org/opensearch/knn/index/codec/KNN990Codec/KNN990CodecTests.java @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.knn.index.codec.KNN950Codec; +package org.opensearch.knn.index.codec.KNN990Codec; import lombok.SneakyThrows; import org.apache.lucene.codecs.Codec; @@ -14,24 +14,24 @@ import java.util.Optional; import java.util.function.Function; -import static org.opensearch.knn.index.codec.KNNCodecVersion.V_9_5_0; +import static org.opensearch.knn.index.codec.KNNCodecVersion.V_9_9_0; -public class KNN950CodecTests extends KNNCodecTestCase { +public class KNN990CodecTests extends KNNCodecTestCase { @SneakyThrows public void testMultiFieldsKnnIndex() { - testMultiFieldsKnnIndex(KNN950Codec.builder().delegate(V_9_5_0.getDefaultCodecDelegate()).build()); + testMultiFieldsKnnIndex(KNN990Codec.builder().delegate(V_9_9_0.getDefaultCodecDelegate()).build()); } @SneakyThrows public void testBuildFromModelTemplate() { - testBuildFromModelTemplate((KNN950Codec.builder().delegate(V_9_5_0.getDefaultCodecDelegate()).build())); + testBuildFromModelTemplate((KNN990Codec.builder().delegate(V_9_9_0.getDefaultCodecDelegate()).build())); } // Ensure that the codec is able to return the correct per field knn vectors format for codec public void testCodecSetsCustomPerFieldKnnVectorsFormat() { - final Codec codec = new KNN950Codec(); - assertTrue(codec.knnVectorsFormat() instanceof KNN950PerFieldKnnVectorsFormat); + final Codec codec = new KNN990Codec(); + assertTrue(codec.knnVectorsFormat() instanceof KNN990PerFieldKnnVectorsFormat); } // IMPORTANT: When this Codec is moved to a backwards Codec, this test needs to be removed, because it attempts to @@ -39,10 +39,10 @@ public void testCodecSetsCustomPerFieldKnnVectorsFormat() { @SneakyThrows public void testKnnVectorIndex() { Function perFieldKnnVectorsFormatProvider = ( - mapperService) -> new KNN950PerFieldKnnVectorsFormat(Optional.of(mapperService)); + mapperService) -> new KNN990PerFieldKnnVectorsFormat(Optional.of(mapperService)); - Function knnCodecProvider = (knnVectorFormat) -> KNN950Codec.builder() - .delegate(V_9_5_0.getDefaultCodecDelegate()) + Function knnCodecProvider = (knnVectorFormat) -> KNN990Codec.builder() + .delegate(V_9_9_0.getDefaultCodecDelegate()) .knnVectorsFormat(knnVectorFormat) .build(); diff --git a/src/test/java/org/opensearch/knn/index/codec/KNNCodecFactoryTests.java b/src/test/java/org/opensearch/knn/index/codec/KNNCodecFactoryTests.java index 2ec953b18..29dae6085 100644 --- a/src/test/java/org/opensearch/knn/index/codec/KNNCodecFactoryTests.java +++ b/src/test/java/org/opensearch/knn/index/codec/KNNCodecFactoryTests.java @@ -9,7 +9,7 @@ import org.apache.lucene.codecs.Codec; import org.apache.lucene.backward_codecs.lucene91.Lucene91Codec; import org.apache.lucene.backward_codecs.lucene94.Lucene94Codec; -import org.apache.lucene.codecs.lucene95.Lucene95Codec; +import org.apache.lucene.backward_codecs.lucene95.Lucene95Codec; import org.opensearch.knn.KNNTestCase; import static org.opensearch.knn.index.codec.KNNCodecVersion.V_9_1_0; diff --git a/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestUtil.java b/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestUtil.java index ad0cd37a0..08dedb0e7 100644 --- a/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestUtil.java +++ b/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestUtil.java @@ -363,6 +363,7 @@ public static SegmentInfo newSegmentInfo(final Directory directory, final String segmentName, docsInSegment, false, + false, codec, Collections.emptyMap(), randomByteArrayOfLength(StringHelper.ID_LENGTH), diff --git a/src/test/java/org/opensearch/knn/index/query/KNNWeightTests.java b/src/test/java/org/opensearch/knn/index/query/KNNWeightTests.java index 79e41b52f..a71f25822 100644 --- a/src/test/java/org/opensearch/knn/index/query/KNNWeightTests.java +++ b/src/test/java/org/opensearch/knn/index/query/KNNWeightTests.java @@ -181,6 +181,7 @@ public void testQueryScoreForFaissWithModel() throws IOException { SEGMENT_NAME, 100, true, + false, KNNCodecVersion.current().getDefaultCodecDelegate(), Map.of(), new byte[StringHelper.ID_LENGTH], @@ -270,6 +271,7 @@ public void testShardWithoutFiles() { SEGMENT_NAME, 100, false, + false, KNNCodecVersion.current().getDefaultCodecDelegate(), Map.of(), new byte[StringHelper.ID_LENGTH], @@ -313,6 +315,7 @@ public void testEmptyQueryResults() { SEGMENT_NAME, 100, true, + false, KNNCodecVersion.current().getDefaultCodecDelegate(), Map.of(), new byte[StringHelper.ID_LENGTH], @@ -369,6 +372,7 @@ public void testANNWithFilterQuery_whenDoingANN_thenSuccess() { SEGMENT_NAME, 100, true, + false, KNNCodecVersion.current().getDefaultCodecDelegate(), Map.of(), new byte[StringHelper.ID_LENGTH], @@ -617,6 +621,7 @@ private void testQueryScore( SEGMENT_NAME, 100, true, + false, KNNCodecVersion.current().getDefaultCodecDelegate(), Map.of(), new byte[StringHelper.ID_LENGTH], From 85fb70fa8eac74d3049cc78130d3427c641b9e62 Mon Sep 17 00:00:00 2001 From: Heemin Kim Date: Fri, 19 Jan 2024 12:19:24 -0800 Subject: [PATCH 180/416] Add parent join support for faiss hnsw (#1398) (#1400) * Add patch to support multi vector in faiss (#1358) Signed-off-by: Heemin Kim * Initialize id_map as null (#1363) Signed-off-by: Heemin Kim * Add support of multi vector in jni (#1364) Signed-off-by: Heemin Kim * Multi vector support for Faiss HNSW (#1371) Apply the parentId filter to the Faiss HNSW search method. This ensures that documents are deduplicated based on their parentId, and the method returns k results for documents with nested fields. Signed-off-by: Heemin Kim * Add data generation script for nested field (#1388) Signed-off-by: Heemin Kim * Add perf test for nested field (#1394) Signed-off-by: Heemin Kim --------- Signed-off-by: Heemin Kim (cherry picked from commit 709b448d88bc168e1d7c137bf2cb539c33b28188) --- .github/workflows/CI.yml | 9 + .github/workflows/test_security.yml | 9 + CHANGELOG.md | 1 + DEVELOPER_GUIDE.md | 7 + benchmarks/perf-tool/README.md | 56 + .../perf-tool/add-parent-doc-id-to-dataset.py | 291 +++++ benchmarks/perf-tool/dataset/data-nested.hdf5 | Bin 0 -> 74496 bytes .../dataset/sift-128-euclidean-nested.hdf5 | Bin 0 -> 74496 bytes .../perf-tool/okpt/test/steps/factory.py | 6 +- benchmarks/perf-tool/okpt/test/steps/steps.py | 155 ++- .../faiss-hnsw/nested/simple/index.json | 32 + .../nested/simple/simple-nested-test.yml | 37 + .../lucene-hnsw/nested/simple/index.json | 31 + .../nested/simple/simple-nested-test.yml | 37 + jni/CMakeLists.txt | 39 +- jni/include/faiss_wrapper.h | 5 +- .../faiss/MultiVectorResultCollector.h | 69 ++ .../faiss/MultiVectorResultCollectorFactory.h | 26 + .../knn_extension/faiss/utils/BitSet.h | 54 + jni/include/knn_extension/faiss/utils/Heap.h | 255 +++++ .../org_opensearch_knn_jni_FaissService.h | 4 +- ...Custom-patch-to-support-multi-vector.patch | 301 +++++ jni/src/faiss_wrapper.cpp | 48 +- .../faiss/MultiVectorResultCollector.cpp | 67 ++ .../MultiVectorResultCollectorFactory.cpp | 24 + jni/src/knn_extension/faiss/utils/BitSet.cpp | 47 + .../org_opensearch_knn_jni_FaissService.cpp | 8 +- jni/tests/faiss_wrapper_test.cpp | 85 +- .../MultiVectorResultCollectorFactoryTest.cpp | 78 ++ .../faiss/MultiVectorResultCollectorTest.cpp | 96 ++ .../knn_extension/faiss/utils/BitSetTest.cpp | 52 + .../knn_extension/faiss/utils/HeapTest.cpp | 86 ++ .../opensearch/knn/index/query/KNNQuery.java | 24 +- .../knn/index/query/KNNQueryFactory.java | 9 +- .../opensearch/knn/index/query/KNNWeight.java | 41 +- .../org/opensearch/knn/jni/FaissService.java | 29 +- .../org/opensearch/knn/jni/JNIService.java | 13 +- .../opensearch/knn/index/NestedSearchIT.java | 35 +- .../knn/index/codec/KNNCodecTestCase.java | 17 +- .../knn/index/codec/KNNCodecTestUtil.java | 2 +- .../memory/NativeMemoryLoadStrategyTests.java | 2 +- .../knn/index/query/KNNQueryFactoryTests.java | 24 + .../knn/index/query/KNNWeightTests.java | 113 +- .../opensearch/knn/jni/JNIServiceTests.java | 102 +- src/test/resources/data/README.md | 6 + .../data/test_vectors_nested_1000x128.json | 1000 +++++++++++++++++ 46 files changed, 3333 insertions(+), 99 deletions(-) create mode 100644 benchmarks/perf-tool/add-parent-doc-id-to-dataset.py create mode 100644 benchmarks/perf-tool/dataset/data-nested.hdf5 create mode 100644 benchmarks/perf-tool/dataset/sift-128-euclidean-nested.hdf5 create mode 100644 benchmarks/perf-tool/release-configs/faiss-hnsw/nested/simple/index.json create mode 100644 benchmarks/perf-tool/release-configs/faiss-hnsw/nested/simple/simple-nested-test.yml create mode 100644 benchmarks/perf-tool/release-configs/lucene-hnsw/nested/simple/index.json create mode 100644 benchmarks/perf-tool/release-configs/lucene-hnsw/nested/simple/simple-nested-test.yml create mode 100644 jni/include/knn_extension/faiss/MultiVectorResultCollector.h create mode 100644 jni/include/knn_extension/faiss/MultiVectorResultCollectorFactory.h create mode 100644 jni/include/knn_extension/faiss/utils/BitSet.h create mode 100644 jni/include/knn_extension/faiss/utils/Heap.h create mode 100644 jni/patches/faiss/0001-Custom-patch-to-support-multi-vector.patch create mode 100644 jni/src/knn_extension/faiss/MultiVectorResultCollector.cpp create mode 100644 jni/src/knn_extension/faiss/MultiVectorResultCollectorFactory.cpp create mode 100644 jni/src/knn_extension/faiss/utils/BitSet.cpp create mode 100644 jni/tests/knn_extension/faiss/MultiVectorResultCollectorFactoryTest.cpp create mode 100644 jni/tests/knn_extension/faiss/MultiVectorResultCollectorTest.cpp create mode 100644 jni/tests/knn_extension/faiss/utils/BitSetTest.cpp create mode 100644 jni/tests/knn_extension/faiss/utils/HeapTest.cpp create mode 100644 src/test/resources/data/README.md create mode 100644 src/test/resources/data/test_vectors_nested_1000x128.json diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index b58741b92..04ec15fd4 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -38,6 +38,15 @@ jobs: with: submodules: true + # Git functionality in CMAKE file does not work with given ubuntu image. Therefore, handling it here. + - name: Apply Git Patch + # Deleting file at the end to skip `git apply` inside CMAKE file + run: | + cd jni/external/faiss + git apply --ignore-space-change --ignore-whitespace --3way ../../patches/faiss/0001-Custom-patch-to-support-multi-vector.patch + rm ../../patches/faiss/0001-Custom-patch-to-support-multi-vector.patch + working-directory: ${{ github.workspace }} + - name: Setup Java ${{ matrix.java }} uses: actions/setup-java@v1 with: diff --git a/.github/workflows/test_security.yml b/.github/workflows/test_security.yml index 4d0374a08..783b4399c 100644 --- a/.github/workflows/test_security.yml +++ b/.github/workflows/test_security.yml @@ -38,6 +38,15 @@ jobs: with: submodules: true + # Git functionality in CMAKE file does not work with given ubuntu image. Therefore, handling it here. + - name: Apply Git Patch + # Deleting file at the end to skip `git apply` inside CMAKE file + run: | + cd jni/external/faiss + git apply --ignore-space-change --ignore-whitespace --3way ../../patches/faiss/0001-Custom-patch-to-support-multi-vector.patch + rm ../../patches/faiss/0001-Custom-patch-to-support-multi-vector.patch + working-directory: ${{ github.workspace }} + - name: Setup Java ${{ matrix.java }} uses: actions/setup-java@v1 with: diff --git a/CHANGELOG.md b/CHANGELOG.md index 68041d7ff..961ab62af 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ## [Unreleased 2.x](https://github.com/opensearch-project/k-NN/compare/2.12...2.x) ### Features * Add parent join support for lucene knn [#1182](https://github.com/opensearch-project/k-NN/pull/1182) +* Add parent join support for faiss hnsw [#1398](https://github.com/opensearch-project/k-NN/pull/1398) ### Enhancements * Increase Lucene max dimension limit to 16,000 [#1346](https://github.com/opensearch-project/k-NN/pull/1346) * Tuned default values for ef_search and ef_construction for better indexing and search performance for vector search [#1353](https://github.com/opensearch-project/k-NN/pull/1353) diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index 2e7f7322c..a62437a8b 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -229,6 +229,13 @@ For users that want to get the most out of the libraries, they should follow [th and build the libraries from source in their production environment, so that if their environment has optimized instruction sets, they take advantage of them. +### Custom patch on JNI Library +If you want to make a custom patch on JNI library +1. Make a change on top of current version of JNI library and push the commit locally. +2. Create a patch file for the change using `git format-patch -o patches HEAD^` +3. Place the patch file under `jni/patches` +4. Make a change in `jni/CmakeLists.txt`, `.github/workflows/CI.yml` to apply the patch during build + ## Run OpenSearch k-NN ### Run Single-node Cluster Locally diff --git a/benchmarks/perf-tool/README.md b/benchmarks/perf-tool/README.md index f98227e27..52590d22b 100644 --- a/benchmarks/perf-tool/README.md +++ b/benchmarks/perf-tool/README.md @@ -270,6 +270,26 @@ Ingests a dataset of multiple context types into the cluster. | ----------- | ----------- | ----------- | | took | Total time to ingest the dataset into the index.| ms | +#### ingest_nested_field + +Ingests a dataset with nested field into the cluster. + +##### Parameters + +| Parameter Name | Description | Default | +| ----------- |------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| ----------- | +| index_name | Name of index to ingest into | No default | +| field_name | Name of field to ingest into | No default | +| dataset_path | Path to data-set | No default | +| attributes_dataset_name | Name of dataset with additional attributes inside the main dataset | No default | +| attribute_spec | Definition of attributes, format is: [{ name: [name_val], type: [type_val]}] Order is important and must match order of attributes column in dataset file. It should contains { name: 'parent_id', type: 'int'} | No default | + +##### Metrics + +| Metric Name | Description | Unit | +| ----------- | ----------- | ----------- | +| took | Total time to ingest the dataset into the index.| ms | + #### query Runs a set of queries against an index. @@ -330,6 +350,36 @@ Runs a set of queries with filter against an index. | recall@R | ratio of top R results from the ground truth neighbors that are in the K results returned by the plugin | float 0.0-1.0 | | recall@K | ratio of results returned that were ground truth nearest neighbors | float 0.0-1.0 | + +#### query_nested_field + +Runs a set of queries with nested field against an index. + +##### Parameters + +| Parameter Name | Description | Default | +| ----------- |-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------------------| +| k | Number of neighbors to return on search | 100 | +| r | r value in Recall@R | 1 | +| index_name | Name of index to search | No default | +| field_name | Name field to search | No default | +| calculate_recall | Whether to calculate recall values | False | +| dataset_format | Format the dataset is in. Currently hdf5 and bigann is supported. The hdf5 file must be organized in the same way that the ann-benchmarks organizes theirs. | 'hdf5' | +| dataset_path | Path to dataset | No default | +| neighbors_format | Format the neighbors dataset is in. Currently hdf5 and bigann is supported. The hdf5 file must be organized in the same way that the ann-benchmarks organizes theirs. | 'hdf5' | +| neighbors_path | Path to neighbors dataset | No default | +| neighbors_dataset | Name of filter dataset inside the neighbors dataset | No default | +| query_count | Number of queries to create from data-set | Size of the data-set | + +##### Metrics + +| Metric Name | Description | Unit | +| ----------- | ----------- | ----------- | +| took | Took times returned per query aggregated as total, p50, p90 and p99 (when applicable) | ms | +| memory_kb | Native memory k-NN is using at the end of the query workload | KB | +| recall@R | ratio of top R results from the ground truth neighbors that are in the K results returned by the plugin | float 0.0-1.0 | +| recall@K | ratio of results returned that were ground truth nearest neighbors | float 0.0-1.0 | + #### get_stats Gets the index stats. @@ -369,6 +419,12 @@ python add-filters-to-dataset.py +``` +This will generate neighbours dataset as well. This new dataset(s) can be referred from testcase definition in `ingest_nested_field` and `query_nested_field` steps. + ## Contributing ### Linting diff --git a/benchmarks/perf-tool/add-parent-doc-id-to-dataset.py b/benchmarks/perf-tool/add-parent-doc-id-to-dataset.py new file mode 100644 index 000000000..54a40b281 --- /dev/null +++ b/benchmarks/perf-tool/add-parent-doc-id-to-dataset.py @@ -0,0 +1,291 @@ +# Copyright OpenSearch Contributors +# SPDX-License-Identifier: Apache-2.0 + +""" +Script builds complex dataset with additional attributes from exiting dataset that has only vectors. +Additional attributes are predefined in the script: color, taste, age, and parent doc id. Only HDF5 format of vector dataset is supported. + +Output dataset file will have additional dataset 'attributes' with multiple columns, each column corresponds to one attribute +from an attribute set, and value is generated at random, e.g.: + +0: green None 71 1 +1: green bitter 28 1 +2: green bitter 28 1 +3: green bitter 28 2 +... + +there is no explicit index reference in 'attributes' dataset, index of the row corresponds to a document id. +For instance, in example above two rows of fields mapped to documents with ids '0' and '1'. + +The parend doc ids are assigned in non-decreasing order. + +If 'generate_filters' flag is set script generates additional dataset of neighbours (ground truth). +Output is a new file with three dataset each of which corresponds to a certain type of query. +Dataset name neighbour_nested is a ground truth for query without filtering. +Dataset name neighbour_filtered_relaxed is a ground truth for query with filtering of (30 <= age <= 70) or color in ["green", "blue", "yellow"] or taste in ["sweet"] +Dataset name neighbour_filtered_restrictive is a ground truth for query with filtering of (30 <= age <= 60) and color in ["green", "blue"] and taste in ["bitter"] + + +Each dataset has rows with array of integers, where integer corresponds to +a document id from original dataset with additional fields. + +Example of script usage: + + create new hdf5 file with attribute dataset + add-parent-doc-id-to-dataset.py ~/dev/opensearch/k-NN/benchmarks/perf-tool/dataset/data.hdf5 ~/dev/opensearch/datasets/data-nested.hdf5 + +""" +import getopt +import multiprocessing +import random +import sys +from multiprocessing import Process +from typing import cast +import traceback + +import h5py +import numpy as np + + +class MyVector: + def __init__(self, vector, id, color=None, taste=None, age=None, parent_id=None): + self.vector = vector + self.id = id + self.age = age + self.color = color + self.taste = taste + self.parent_id = parent_id + + def apply_restricted_filter(self): + return (30 <= self.age <= 60) and self.color in ["green", "blue"] and self.taste in ["bitter"] + + def apply_relaxed_filter(self): + return (30 <= self.age <= 70) or self.color in ["green", "blue", "yellow"] or self.taste in ["sweet"] + + def __str__(self): + return f'Vector : {self.vector}, id : {self.id}, color: {self.color}, taste: {self.taste}, age: {self.age}, parent_id: {self.parent_id}\n' + + def __repr__(self): + return f'Vector : {self.vector}, id : {self.id}, color: {self.color}, taste: {self.taste}, age: {self.age}, parent_id: {self.parent_id}\n' + +class HDF5DataSet: + def __init__(self, file_path, key): + self.file_name = file_path + self.file = h5py.File(self.file_name) + self.key = key + self.data = cast(h5py.Dataset, self.file[key]) + self.metadata = None + self.metadata = cast(h5py.Dataset, self.file["attributes"]) if key == "train" else None + print(f'Keys in the file are {self.file.keys()}') + + def read(self, start, end=None): + if end is None: + end = self.data.len() + values = cast(np.ndarray, self.data[start:end]) + metadata = cast(list, self.metadata[start:end]) if self.metadata is not None else None + if metadata is not None: + print(metadata) + vectors = [] + i = 0 + for value in values: + if self.metadata is None: + vector = MyVector(value, i) + else: + # color, taste, age, and parent id + vector = MyVector(value, i, str(metadata[i][0].decode()), str(metadata[i][1].decode()), + int(metadata[i][2]), int(metadata[i][3])) + vectors.append(vector) + i = i + 1 + return vectors + + def read_neighbors(self, start, end): + return cast(np.ndarray, self.data[start:end]) + + def size(self): + return self.data.len() + + def close(self): + self.file.close() + +class _Dataset: + def run(self, source_path, target_path) -> None: + # Add attributes + print(f'Adding attributes started.') + with h5py.File(source_path, "r") as in_file: + out_file = h5py.File(target_path, "w") + possible_colors = ['red', 'green', 'yellow', 'blue', None] + possible_tastes = ['sweet', 'salty', 'sour', 'bitter', None] + max_age = 100 + min_field_size = 1000 + max_field_size = 10001 + + # Copy train and test data + for key in in_file.keys(): + if key not in ['test', 'train']: + continue + out_file.create_dataset(key, data=in_file[key][()]) + + # Generate attributes + attributes = [] + field_size = random.randint(min_field_size, max_field_size) + parent_id = 1 + field_count = 0 + for i in range(len(in_file['train'])): + attr = [random.choice(possible_colors), random.choice(possible_tastes), + random.randint(0, max_age + 1), parent_id] + attributes.append(attr) + field_count += 1 + if field_count >= field_size: + field_size = random.randint(min_field_size, max_field_size) + field_count = 0 + parent_id += 1 + out_file.create_dataset('attributes', (len(attributes), 4), 'S10', data=attributes) + + out_file.flush() + out_file.close() + + print(f'Adding attributes completed.') + + + # Calculate ground truth + print(f'Calculating ground truth started.') + cpus = multiprocessing.cpu_count() + total_clients = min(8, cpus) # 1 # 10 + hdf5Data_train = HDF5DataSet(target_path, "train") + train_vectors = hdf5Data_train.read(0, hdf5Data_train.size()) + hdf5Data_train.close() + print(f'Train vector size: {len(train_vectors)}') + + hdf5Data_test = HDF5DataSet(target_path, "test") + total_queries = hdf5Data_test.size() # 10000 + dis = [] * total_queries + + for i in range(total_queries): + dis.insert(i, []) + + queries_per_client = int(total_queries / total_clients + 0.5) + if queries_per_client == 0: + queries_per_client = total_queries + + processes = [] + test_vectors = hdf5Data_test.read(0, total_queries) + hdf5Data_test.close() + tasks_that_are_done = multiprocessing.Queue() + for client in range(total_clients): + start_index = int(client * queries_per_client) + if start_index + queries_per_client <= total_queries: + end_index = int(start_index + queries_per_client) + else: + end_index = total_queries + + print(f'Start Index: {start_index}, end Index: {end_index}') + print(f'client is : {client}') + p = Process(target=queryTask, args=( + train_vectors, test_vectors, start_index, end_index, client, total_queries, tasks_that_are_done)) + processes.append(p) + p.start() + if end_index >= total_queries: + print(f'Exiting end Index : {end_index} total_queries: {total_queries}') + break + + # wait for tasks to be completed + print('Waiting for all tasks to be completed') + j = 0 + # This is required because threads can hang if the data sent from the sub process increases by a certain limit + # https://stackoverflow.com/questions/21641887/python-multiprocessing-process-hangs-on-join-for-large-queue + while j < total_queries: + while not tasks_that_are_done.empty(): + calculatedDis = tasks_that_are_done.get() + i = 0 + for d in calculatedDis: + if d: + dis[i] = d + j = j + 1 + i = i + 1 + + for p in processes: + if p.is_alive(): + p.join() + else: + print("Process was not alive hence shutting down") + + data_set_file = h5py.File(target_path, "a") + for type in ['nested', 'relaxed', 'restricted']: + results = [] + for d in dis: + r = [] + for i in range(min(10000, len(d[type]))): + r.append(d[type][i]['id']) + results.append(r) + + + data_set_file.create_dataset("neighbour_" + type, (len(results), len(results[0])), data=results) + data_set_file.flush() + data_set_file.close() + +def calculateL2Distance(point1, point2): + return np.linalg.norm(point1 - point2) + + +def queryTask(train_vectors, test_vectors, startIndex, endIndex, process_number, total_queries, tasks_that_are_done): + print(f'Starting Process number : {process_number}') + all_distances = [] * total_queries + for i in range(total_queries): + all_distances.insert(i, {}) + try: + test_vectors = test_vectors[startIndex:endIndex] + i = startIndex + for test in test_vectors: + distances = [] + values = {} + for value in train_vectors: + values[value.id] = value + distances.append({ + "dis": calculateL2Distance(test.vector, value.vector), + "id": value.parent_id + }) + + distances.sort(key=lambda vector: vector['dis']) + seen_set_nested = set() + seen_set_restricted = set() + seen_set_relaxed = set() + nested = [] + restricted = [] + relaxed = [] + for sub_i in range(len(distances)): + id = distances[sub_i]['id'] + # Check if the number has been seen before + if len(nested) < 1000 and id not in seen_set_nested: + # If not seen before, mark it as seen + seen_set_nested.add(id) + nested.append(distances[sub_i]) + if len(restricted) < 1000 and id not in seen_set_restricted and values[id].apply_restricted_filter(): + seen_set_restricted.add(id) + restricted.append(distances[sub_i]) + if len(relaxed) < 1000 and id not in seen_set_relaxed and values[id].apply_relaxed_filter(): + seen_set_relaxed.add(id) + relaxed.append(distances[sub_i]) + + all_distances[i]['nested'] = nested + all_distances[i]['restricted'] = restricted + all_distances[i]['relaxed'] = relaxed + print(f"Process {process_number} queries completed: {i + 1 - startIndex}, queries left: {endIndex - i - 1}") + i = i + 1 + except: + print( + f"Got exception while running the thread: {process_number} with startIndex: {startIndex} endIndex: {endIndex} ") + traceback.print_exc() + tasks_that_are_done.put(all_distances) + print(f'Exiting Process number : {process_number}') + + +def main(argv): + opts, args = getopt.getopt(argv, "") + in_file_path = args[0] + out_file_path = args[1] + + worker = _Dataset() + worker.run(in_file_path, out_file_path) + +if __name__ == "__main__": + main(sys.argv[1:]) \ No newline at end of file diff --git a/benchmarks/perf-tool/dataset/data-nested.hdf5 b/benchmarks/perf-tool/dataset/data-nested.hdf5 new file mode 100644 index 0000000000000000000000000000000000000000..4223d72810785f11ba801ac0fe819b0a72d0ada6 GIT binary patch literal 74496 zcmeFY2T&DF+bv2EK@cR01OX8cQBV+3VfR|(3?c#w0+JO_5haR%iWzgxVnWP{m=lU% zz#K7WF=xf>neThw_f&me^^-Fs(O?Vj%G-Lt2s`|0OdYxNXgPj7WqLshwd z9V#kv&Ez!yefrPWpT7^e_VWKO`k(o~E5!Ys{&iJOZt~yx3WdK;PdO*E4Sa`$32Qoxaom zXXO1=kd-_#>tAPCBjQtsNojmm)`--EY^nIK>;Kg5`d9XU`zplhap`(=Yz_xt!Fx%D=BGDE!r+%HK0Z zQC_<6*J1E?`gfA|pV|MzxxT@@J>}(^|Ihh6x%q$p2mb0UIk{@xe|r9ZUElTZ;tP8J zeZBsF*Sp{PzpwXC=b0(~ukZIy=|Ar+PwAhM|1;_TZd~>M+kXD@|Nd+K|Him#_xBSn z_|GT(e|B8`U+s^-mjStd`s1Ix;@|T~M;aFYTFm~Nd9~-S=dOC!p3RXb6qz>mjBx7i!LwogsAV(;0|VN#wrd~* zIwr5Eh2o<~q2TP!^DSG@N+plkA0ERfx|7R!?-#Ns0sc(vH<*JvZ^Py# z#++J_C<|#G!eFlt80ut?6o|AjMukBvejub#`iQ~ zXk`ZHY;)lrlT5alQ6(0BRTH_N4&z0UJcmqpFLExwf_%U|+0AJIjI9}r4jM+NmABzI zh333;`5x}Q^QX$^K9E1;LM6v25#RAUTGSkNX-Miq_tWJ#?>-y#j*TwE2DIbptGlow zZ73qvhEe->$kAbrT(L+i-kcF~=Qu7qSo9{B}^D z!nOuew^l>p;t{me?aC|lU3h%v3lXFl$8R6Huyf-|^mLeoG|z0RbgdQEo!ZjlaVjgL z$u8qRiPPWmu=%SS6ZHo1;z&bkZZhL*tJ5MgyAo+rM{~iGI=nlZz+oQwTzGplBMiJa zy2PHrecQ2Yss)md#L4`U(zqxnhmC8vO5O(A8d2-4$O`yO1IWR15MSHQSTA5+pj zXu7wUx^;RyS-M8#PnrywK`*{3ZHEF~LoQmi4o7=D$J!?f=rUq9mO35A`{lQwme&=J zE}OI8{5N7szsWe38qPs;y;ztSf_-l_S)b^^k9XcWx9psaLm{oWWQz-r1O#H|iZ{r8 zI@&BC+IY$1s&^xxoAfb+eY-{k_kJ|Hm(C){Q{UA z^I7<(1fpH_6IsDWdwzPD&b!+mVosk6&|6TB@(rOJJ?1+`pGd*lBtI^EKN6<%b8z`h zARi_>aG1F*Efohd?usto_K)JM8N+Ck+>YKyZp%KT_aIudz(uKi{iIfmt=C|mMthDt z+KH=GqNvn4jVXOoxYi+n{r`;M2N$4eQ7v4SoI+yN5J>rICt5Wcu=mv= zB6rXSF<^jxeQ>uKy^`m=&?s{5tlVof1zPGKA90aZtOd!Qh28ID9UP-!)dE{iW-e{wJ2LjB;^g z#4{K;(vHJUq4B#MP=9r4>Jhy4~9qPED6Uz9K6;@uE>*JLv2 zZ52#5XmHo@7{08NUH9gRN~TDZ0NJtj{|WxEtDGvaWZ?tk-I*r* z7Y}5#LoZ%bsz9*UMXdbZkJZNG{4Trx z%9po=EsvMmiwN0poEW1`GmCFxtByI3sw>0YD2*pu4;3SHqj1P-2nWVRa^B?@d=gr~ z<&|UMA*~&Ur4`^{rXYf z*Ht_XTPqCmD`i1tHdyhtE3b_RrG5D=SXw!=MN~Q`Z?R|DZ3C_?_d@CuWo}<@!@h?G z(Bow=f4sHEmVJS=dz^qW<%ignyc%ga-I;7Vn&U$*;Mdv$?rbU(3jX(G;>VEimt77xZGO-#dG4=>w6W%Wp&PWynv)oQLibqypmYCaFq1^yK>}m zcV6q5En4&}=J+qkuzy~Uep?*b-oFUHX3j*DsWsJS?2wf`{fR@{1t0j>h^5lpZ(Jl# zn^%VD+u4S;Uwqi!tPAyV35_Ywa3MZNR4L}s!lDZD=XBuaW5MAwqPg`HIL>ZAtR8h` z#+%WsHSiL#T8g}Ty#b|#ew+}x4GrH6**8v(rS(%0v8FvnH&0oz8H2IGp)hwZkv%%2z*%#PxF&c%GKZL>qH-!+ zmJH#t@wMo=(TD{ihbFdZOtg2QnR}y1s8(Xb=7l1Az+f7rgm8y>H<7x=mTJn;oZRah znkCIe*vuHNC`zQF{W)>Ta0VuRkH=Wg!PI;D4i6fJP<3oO29^Y))lV;OJ~tPMnFr7% zOM#y)RtlpnePMsanZ0}GGVy>bb2V(Kv_A%!+kG*x)nVDHh^t6BFdu717W2KwWSDNw z#vOAbj`hi53xhXchXOXM*#h;@_i%gag&8kPu}n3QuKj9obDo80RW+IwD=%YX;dBiB zQ->+LoM_q zOKF|Va@H&)-R;H)ixgOM`xcH|)S`BE9s2(u@4xvV(^+1?=l(?;)$=O$Y>46)4KMaO zn!=S;Eot;3kw@E)rf?oE)S5={WrsSHd=C&uGj2j*!5esL^oPP*J${yal(-+JypnFn zQGRL2PM(Ppv-R+uD8tVDyBM>jHv=MPi_l&Dxas+19KEZ}2RGJY!>hiWoi;|CaB0tx zhh|}>_CTHrQ4=Z$idpi#6`Rz`;JChnXjW0gU(Tf%k(N!Ld5Kha2;$^HP8?9&gF9R9 zLwYB&^70!Ly0*oH&!_Nboj?40nWNac89Vna#qN?o=4czjXGtb^n5A*Mw>fnO7gD~r z2YU7RjW&}8^6jE*Mh-uP*;D$npH4eo|LVnd1K;7N=N)M0_Tp|a9a)ifIBDyMX&C{0 z+{T{1HJTWwP$s&)3t{Wv2%b!A5XPt4^I_FEJfG;mY&8w+w(KHicX*DgVkrwQIXD)Vp zJtc4Yg`uj=zy!4G_G zaS(IT^0|F(G?)2|n}XO*Ev72 zzwLMER~yp$$!@V?c@kG_ca+9^2>%?tj;N6HFxDEtW?qGyWPSm2&Vds#PwX`4$ml`e zF?VA#YJKm*gBhdPB{3T%13joVNgt1+jhMQ26!zrhQr=?(@6Tz;jbqz##rjLaQzwgl z!85R@dI>D_&*Mpo9A!OL;D&WI=6*ec-IA|1c6c{zbsQ?1cU*!|7VpvYxC8yT6r-<= zNI{nK2YsW~Dl+BoqAkp|R^s)hzKj!c ztRAh$-=6(B=zcL(kLT0l+9)b5j)c{aFwUMC$mogo)IXuiS9{lp^Mg*{i*79zp4or{ z!-&Y~&!9Ijoa6hw#*8he5ZA39p~l0xMOxQqHPzvQ)+pW`@5U$vQ${U*hmyCJ7<}v_ z<}S%&p8?5Y*smD+xensIaBn&cOQfybQ8-29BEjN{i+ zjWDWtDO+6Ci75|t=;atKsNuw2$6jFk*SV4%Y=E-51=C9W7&7bwv_?O`ZaGWd88;2e zLEGSQ&x;Q3yU_RRd93lW=EG$@Xcj-1%8DIXwsHUtJ(G#SYNud$u@qL)97=7kjbUou zE>E+o5ZM(XJkCqDdXO4j!!o#d!68IEa%7cB3obb^9@C}%-g;RcQ%2`gMCuEb*ILrp zGi8$8PrO3btI+dp@aoOIZ=+>+! zHXf1D`S@A!=z2O|gys?#-ij?N{du#xJKxv^v8P;1o^IvLdq$4zxvn*@8FuFxF2>U( z(JT&Vhm3P4U{)Q5*+HH8D8?V=ZY?=>UJ_M)d2&!|4JPaO^OpZ?IPG36I_93lo8|+k zq?yk?_3~_$*^FL|9(0MlE>!!QaZaH(w{5xy<)xibY9z;)ULSDr&|cKb%@cROJEDDj z4ScpQM%EgCo~jlwZ?zFa&ic^x%TyF@+l7#~YKTq`M`C0L7Q2kZM=7R0IWLG6j}AiH zBUtE8{2(jp-kBA?ZxNi{3f*7(@xsnX_8XK)Y<_@4()^2kZi}^l9^-utc-=QpwyeP& zCnprs{FpKq>}bnz26ou5wM68MHRP|$v#?svh>n(hG3@km@z!IlSUl+_UisOua@JUU zcbW&yXWumA~{fIeb z?-5=%15f8Eq2&V?E-r5mwN(YUp3oVer$o`}k{)X$n-W!a8lAQ~&}+RRr?0d_-@_T4 zazw&h>_d4^PKW7te!%9X65sZ?i~YM7pkI;;=I}T6tAh7Zr-*qKFOd_MPOWQ7SfYMG z{Bk@2)!OzLzeS5DYhTM&MfSa-&t*cCqw+G$*ayhL)$} z#m4qmU?atw=6H<5y`?$fs=-qZ+3dCZo3NO2Pu6>V7DqlT!Ru-j9&2e0`wtFmYB|aI z)k1SPHY`T`?%ybKvg4I!zT!*UfiThN&Ff3Enf6eh3p0P9ZM+HmHCBl7TT+aXSq1N7 zgZXotKBqanh3VEoToKj`pUk^qd*f+ryYmuyUuB5C6Z2=p~ZlCxRUA5`*P9LYA0woX$IzqRk#X& zYLE2dx0>cO2utAzdwKqPZ_cG79c5cij28)=4q@?ESB&u(&65jdRMU>;>vnxOr7>Q( z<<5amO&sTKbl`HQ57=SR4w*$S!~}xiF;>2ns%e2_#RudK(D02Qp$YhhRplQAzPvyk$=*&(?8qkMttlwbHfneSpqr+#KyJY=? zoFMyrR_q(ymFE88R6BeXJ8Z$BTW<<&ol;rc>dUB2Z-@6qM*JQ15J`95U{vuej9;-s zw)A%ZTa@QwMS=_K^%sb9ty5(t-(6^-*IKgOO_=aBh2C+od_Eyh+&gp?yDw_equ2^g zir3+MMUN|DAHrsDN3OM5hx(iVE-un!S36JUX6tf*xjREX5kGIrp-*WRzEpjcFm`!5 zYM$yL`m%D=7wnHI;OA~*WfN*kMZ2s0xb>(q)E91s;cFL;3T;V`hqi40 z{3V=|>(IUNozR~CQQVk+2Mboo7<_Okrreq*i%sn=-re(KRC7lvrN;6_Lkdm%?tz(v zg&b3`#H+rJ+`Dj~p!fX$7y}8SL1?lLjR+zU=x`_&H9&lV!Vbysm(4*Gh5HJtraNHDh^~ z5`38)K=sF6c=)0@?KJ|}?1wxL7f!{Nc}BdwXgf|9R$_WYCeyWh^5W&@ur{z`Mp`t7 zkG+E67BSRLv*rE+seJr4n|H4I(k`?CT}oVKvXWI8@mUWQwR&tmra#Imb1-w&J%oMC zM`-U%4(#c~wpL-h)}+T-QT7}auFKi;pF*arLb=6B%zr8&@E<;}lus&vS^cHDZ}T z1~s|_(QBsU$L9EQ*7g({&j{nq^>c7OyEB5dJ9Az7I#JqS2ji15Tm@e~He3vcr-H|y zmBMk{5ISG^j&-dUqI;wR$DDA6l4&OUT^T3dd~QRf$~<28Fr~|yg?QK8PP`624SBg? zTvDRL8y?L#D%+pmAG(SD!)}XS!+&B%mJW}d2w;iERWZ2RBaz(Amby!aa9ijcq};oL z2TDmepsCA4e^fYQ?>RhSD&zGN*zm-j-BMMlwV)3(yiXw}XFD1Ux^b0~DYTlrXf`YX z>y}$^{>vh+b@`1g2DXfJ+=iK}vqY4Z1JWl2aKS}=CT(_R`AbWd1Zv>;($i>e;L08g zw?gGg2kK`uU`#@LM2K9(XY59gWeuphG@O$bti>wHU;gQH6`JLhqA+x?aC~mVh;vfB z`}zsyT<=P~4f1^PY&FgY2}Epe%d%4vj~CsMYGWe!@Y`y!H#VM*!-wCU}DoBVY{R+H}2D>&520f(;3D~ z$!T0OJA^-;e}VNaX+3(<9lER9@wmi4ym$saIcmsr_31Pk=gLUUwIbK-qv$%iuh`k4 z7XozFWBA$>);xFS>8RFh+PnpO?rg!&`u==;Es%3ua(GKunKya_^V2D9EHK(F8yI{- zT*&OgYeRlv{22wjS(?vbPU{gW{{X)mtfknr6N8Ulz|`fYV!8fIY`)QtImMcMJ@gks z%=+`pt9r>C=&gv(GY*LZ8wYn|$LVG{-R^{{I7gbJu6+n0Q=aA3u z%a$@Zerat?s~78_yx*5s%Y*1J-bwcG)-h~#uNOH(wfSLr8kbJZVCwddIPgx37qX3+ zsqzZ*uTI3cqH@4+6mOfBptElczQ=}BOMfK0Z4ReXsV}=}8Z&8hAcNOrik>qji-gQD zI&9WrPisf6*dHW&F-eOCyF(eSk`31yJ>HjiJMl7+XTNu)cLxK0+M7h*xsIIBs|0cF zGw8c=EE;aU!*Bl-E-R1Z>m6RqeLsjfMML=g;!@-&IItncgXR1C^WhCO?CT}Z&WcSk zmrX~-L!G7Qw``qwuA;z>{d2iyha%Ve?GXyuj(nL{Cp<>9=EFtGG^!elwomQE(16kG zI_WO%z8Z@*i=8<^>j|oQ>_+LdbgpnrWWqoNRx}M@_!b*>t_)`T@va=a>;^iY>x7V$ zV9uC%P4qaF#xCvWA=*rV_Q&kl+s2&cZmC@B-jO3SuE}&fjd92I6P9&6iDw6L_$MQt zvqoHm%4!9Ea__^DCWARpeI0ZUHDDk%A;>+I4@=e~sjLRYWtWB7*)-~E-Nd_gmeo-Pm{b=@`7;*m9IzEpak)bOW)M{+jwSbv9Vh-?1+~@coMM^m za%ZDGBMNo-yvmE+MFX5qHo^9;7cV6Jz^7;VoG0^Ok#or{?sXrpOB>8MHS$i?2rK@NHQ)Uh8`V zuTQT)xk@*#`BWhr^uZDRUk|6lML)K*d5(cyx8TLp!OTvJ;j{2>aFh7Z#<>G%+a`}! zrUbFiX&E0l1yIsRaqjecNHvS1^^ZB?cB|fa`plSmofPpsN(mM_h9fR)G|u(ADSr8_ z#qG5-p#IE^#zRe+EBRqfwGGg*t;Y6s&qdm<0=ih7f_D98{Mj7JVRICj{<n-fpO1*3WqmU`9v#kC$A{8zn2)$ReI5=(+{fz2@syMNoU>nxsAuHI zFZ0UqwaL&qa;zn$?}&l$D8|;|^IMfGsQ-CN9FqCksntvSCej=3&cbZ{}^*|A52x0xooywYb)c?OL=_KDwhkHucE6yDp? zih~|ZL_?b*Iz>0hD+h=Y3`@pRKbgY4>(60T(I6;g4xwZJ-YiOx zNxs1ovB9`C^+&abwKOfq_8H1ET6we@-%E&XPCWLm6~kQ_PO7vz_Z~yE4LN#N68HL@$IkKxIA7UUVK_~8gT^^x={g<#_e%4D{c%5e2aa;XWxu;sHV#JXnhspK5UKS3L%O7PRV{i#3m|MExIiUbK;X zo_8_O)=lQ#Tnny=y#v?gdDK0t%0)g;QQ0$?fh*dfYK|sTyu-Qt6`1Ntmd`nlOO{g1 zt!BvLjz`gIeqVNp5sY8cisqL`BJuN86xld&YOWHo^jQX?53|b1t__Y$ts&~R+!CrV)N%(I`Dt`~l z;ItuIVQtrlGY>0Z^!=ewTxZ3}d*=v`hnw(T=>b~wyoR4m2J}jZq5sPdXh^n`t-cfj z`*}9lw=s(WgTLV2MLQl|1!`rQV?w7exP8fCdP5G^tNC(CxC!pMjpoa~nH*sq#_o?5 zxIKIgRLm9esk|I#vc4jC&M>wMms zIa>r8^ZMv?{@Is+fD|X%9ME94We82(y}1)+{Q5?f^VH3dK13N_Go$g#cqkXU4rapB z7*Y9VKE4+e&~3tN%-oR0j?P-LPVaQF^Vn1*Sr+lGZ(Gjx%VVooZ(Me5=nj9?ew<*o z6#DY!nC0|Zn0332v26{x@zGmZQ$-M`K8#^~O$c?{Ji(CFCY0+MM>7W{PLSqmhDr)A z>wbZfP^4;*KlRJCIqQ6_n7(y7YV>-dg=8-Z_myGXT?&oIR3v)-8bd2w$2E-6gDiBz`Kxg-5Slf}Pt< zSUvG&lhZfZu*E7Ivo%N9MZ2?n>qwMK8U>YC?wH;Et;o_@Ay(WSi(j)PJbcJ)c-`A6 z>%00k4&F`UgraO-ZRUrg6{d9Wz8ACi+QVYMF{hoYL$NFo5qn=?_Z1Vy)m%ZlPt9q& zz6VE+lQH+66%BvXV0HEzoO~_$Ul-CuhsQb$th8ab%Mn}`6fJ4GzKi~z3&f@DS8y9B z>3<&2l+E$9M3>F?@pQ`?5hP(*Iq5D?2w#G8Z}fP3!8k;o=tsF9H)Xx_Kj2-1B?Got zF?W0tU3p7nI)N|c454&5oCA6XvHqJ8{Y<$=j8DfE&n;7_x@T)Dpu z*H_IG;}h@TxtX-Cn#Un(qcn~}wqeyZXDlh1iJ8@x@z!XqOUo-}OrCNE*~4NOQ*{oS z!8M|Baeu~3eG%{7hu!Vh+(=+pRPgc&(WNDz=R%kBY8MBl~tMRS_GX_R_pr#i zKx(ruqn*vMR@;C+4z=Q{pAPp79!=AcVT|uE5WU-};#JHz44@Iyx7xAOyjJ|Zb^?m` z^yIc(50IPo1|P3wvt~+XgzhlJh3PS@G#$-uU*l+5-Y%CUN`6L&fkzO`S{}k zOCT4Y&4>{;w6-0>aXWXSF6o7sw4xI8nml=Isya3oP81&xw&d*x6J)E~zk^o$-6)&* z3}Y+0N}Pa%S?4!lmAVELtB-+iB~7Z`K%NZQfYs4)bPtc`_wR=hcCsfPY3Jhd21DAF zTtP|6XM}5d(eHL!TDLQ0P2biG_Wgq=~W*;IZm_d)PtZ?H0>4{|un9gg=L9 zd5V?01P4|3p-Z#}n)8}xA1zs{3#xn9_9I8*NgN=xl% zUZV$(mTG)Dt_=;hEP>Cq42F$#WaISDcsigOMUUI#gMTPmJJn#yoL{m%D*l|_!-oA0 zcZ*3uBUyGi5^p~A;c%-C?Cqz|l*(lhow%=+ zC68uBvia9%FbP-S@Y%bOI&i0OF7e`k4?FSnNE)}b8;8|vf?$)KjH?A_#nT~{T=T75 zl)v7OXU$_NFZtqOlFoQh>n!R`>d!0dlvpg=jPW|n`CxuDXB*Xv;B*zPvD6jjr#o`& zmqhkb`i7w&lJLDWlK$JHnBTY=2ZMj$!z&LISUI4MSE15L7r);)L+?c)WA61uXT5iL zuN%Z3HSbZi_!2@DYti=WN(|mLQ}~{lfJ?I)usv=xlJ@CvTeF1-(e$Oo)FN639fr#b z2NsHShSqAc;gSii9eIswae)j`-zY?{-7s1{n6pX#PfYFF zf=(+78MIZChmS>xn)?fouX<8^e%piYRzuNp@(TU(7@5>{c7NPsaWwNnL<|16GGkl2; z6-xp|M|&k+Dplpi&-VD_cMGqryEAR0GSAYVN5a9rdr9vxETwGvqvK+AmguN5I;9F9|9}UM`@?vm0SDw2i(Lwmc5{X% zLVibxX3vN7_nk{EcH@=m$QLoqGO`tBvZc*lJ^^SZh#ke!a z-o)vh_hokTM(|*094G9tL26aCm^;Or#gC7Q6EB|1B95;Ru1zU;@@gASE;%4^gF0+) zs)G%qThpwA9X93N5u5j&LWq4Ol-{hu2a^Imy}S&usd=2<<}|iQxbGcPZElk=A*bR3 zzKtvt@oOS*ZM6*!+YMpZ{WDNHYJhJuM&PP>rR=A@5~d&QN0<(zU(ROe{+uAXjZ=o| zYdKyp@4=+ogSoJwH}AFy=9v8^tbZ~SDW?l~-rI}Ea`ULIVSw}J-Kml7&H>7iG&Ng@ zp|w3}x-yPMW$W?7>WFwUbPbB5?_p_t5Z|YoXL1r}BvI&^vlWTw9?0Pw%pEV(0KMDrA_l-mx=VKRzRDZ(4GsnE{8q zmhQFK1I`kEuID@%-o0zFx^9G|VUlnN#SS>AJ{s>YxzJ>N3r?Bco`sUnw)H?B6(lUX z&7heQznjCt)O5~tXo=fq&>ZZ+%^Zsl%$A+n&dr)~_9o|ng#axf2 zV);NbTH8#6dqp`;UK!4`S#q=tZNoKs-8pws5$|s-B#MqlB(Zy2Y##foqY)DU>f7+WfD~wrs%L^4fM{tzR z1hK-S1+HFFxRBx@q^+Chv{4QO*p8rA^Uv3n$2g%d!@@z5p>lQp19nB3| z3A}o3GfI24V$P)IoZmYWE^!%RSk4oqZPP;g$GMzZ(uDOBa;a=E3Az{F38T+NJh|~6 z^7Rj4Gmqm`_zS$1cyxzFWzxDMY5)4UG2Uq@Zg;3cr~V35zr9$@oa4@C$H&XEZidtU z`U#OYxIaq^t8rBOtVox%f|nXKX&a}&w_43<|63hX8Y1|>a47PnK3z8SJ63;DWcx$0 zJS{gB+bwrtQHcYM&4SsttQ4;*N8w0cL7ff{@UVCxeB5F=c}lei+b{$3ZXR}7yJ|Il z={aJ?Y!^maRN~L$&p5F3IBuV-#k&bl@bT1Hlq^i*>ZNU1Z|BVz?R4lFgfaSMFx1Cb zvkl&g>Ybm(R5K-(BzNJ|V=Blvo6Ambui)`~IsWXG#!;_U<7c!f#E0JEXv1Z=%|0S* zzct8|rgf3dc@iyS1e5f5>>XPp5+_Lea(Ycb zX;d6LD((?#5sB=)!jNeyUa+Y9< zoyApqQ0wZ#`7y=3+rA3T)CWtLcS{79crf+F6!?uc=8(dHOdT^9!JG1B_L?O)(Ort; zvVe$*g3*(_0(s9wFXmsgLtDfhdF1QIotjl?tUnjIiz?~<(M*L zYj?n`;Q`cb|JO=vXyGen4jIfbIYmr$RKUCP zIDWY)>ES2#gLb3DGis!A{LLtI``Upiy%Mo!T^8)CA~|b!1;P)G;~-Ppv1Y<*lb80`u6iWd3RQ_;iT7l|26tR*y8y1CFND<;1>V`#oADRg^6vcKxM*}ijEu5F z`qeUw4Ul7k{(7v`P=<+XEEOc3fRAlEo{4w>+3dc&p6e*>t?kOKl>>Qpk2@|?TE}}U z!TnewH6nM3!B14Uciwq8B^qFzPX;e9E^#)rlKf27U-+6k4wY(T*Tg9HF$!l>sU3V~ zNc-gcCyEvOtD&?inmgJj<89b3wDpOgh2;p|8ks_Eg`w=KFbfaYyD~61n~zFu*-V<&v$K7&ukTN$X3qb3t@nR>*O>!};xbJ}>W^ zi~&Onx!XUVkJgW%!yE?|t~PpwPTY7})3` z)=r)84b7kMqk0@$D-EUOaFOqYa7? zdip(Xe_0Bnsllvz6pC${cg$Bi5bl1P6uA7_uvyJ*KH* z^~v)noU}}~Qqrr&oztaH!4BA1`tU%2l~9oI4mbZb_!_thg_g<8TU;sIZnYWZ2REXm zn+NYdj>n_yW+-%4!aet&5NALBd(YpT9lxNudAls{{02n7^yak!U7mT=mN|bGpwd*1 zPX>0ONkTs^moWdQKKT}0qbgt~X%Fre6|;1R1uX_jai)?R2c>^S+q&iA zW7Q|ryqtq}GxfRoWh{plER(HWlgo_Tdi-@GfdTRhFlJ*YcYl70!H$8no;Hk~I{5N{ zx`aFR^Q7siUbKk23dhiwcrh$VazFY%z5tuNBBuMlg7JDkG?*udq_00Q zY`zjz3#9!}lGgUrxreasn#s~{HE1Z@fM>}&gzdE`Y5g8b%Pe{Ep)81!2o)O_45QBE zJJ3`fi^Z{nu`;S0Uz{7E-QEV%KKG&4TV?iK5zltpwqQe19JeO@LHCYTSkTd#m%kWs zr>;HApBzX0@tLqQ+^jW@FM$ zX^+T;0dy_1=KrARyaTCz|0rHogv=s)res7R+~=G`8WcrRl+lop6_G@P_FhUUN>h7% zwfEk8Ylt+p_xL@(|EsHeug~*-&ikC#Ss5K5BF4y!&h&YrNp@e~x_$Ya>8!#HHojDw zco~KdtI@^6krTv2JV-i+v=&;TqpmT+vnPvHPv!e-_YDqb)VOkGKI3e4cxG}}<}Ju# ztIdNst5Jzx*Z6S5xhBlgn}}(j!a4845Js>y)#oW=L~eUdbIZXbIm@Mon_{71v3QVj z1}0}!Y43IciKVT@jR2tePakYQtH*Dj%sBt?I0o7j3Lon=Sllv*bN5S~SNi&cWuENj zVyUnEI3Su%4`)aB5Ki0m2ve>NV)wN{n6lCyeFhKb=b$?5v@eJ1-atHUG=Z7vZwwEK zW0k)Jcf2UhWX&Zi?x)Rm<^)Tf3V<@k;<1+IU{<3n% z`85M+opc%pFXUZ(T&vcNa|j7{u+Xq@QHXE`@`9ogd$==&+=K23wji z;`%hn5OCz()45z|+LWEkPT^L;VceZ-1l#))k#e{NR3>?I@zkMQdN+ncHV@~2x*mMp z+?uMt;^As_5m6ao2tM757q6?rN@YCSMXIvz*(9HzH_nL>lQh|~m5F%QtWgpAU z%;&^asm!Q4iiQ~xOg^Lu`}b++yxyK8W5)Z|dPnoebq79M-;Q^mcH@z@&7_xqGA5a} z!{IJ>u+c*aAL2?-;+;*c`)&DKC7lt!g4q4zNX}^+su(M?a2*c0<4&K^yu8;6alv|g zdq{BEH#;70nyK(g=*8|J(A9M5RJGB)vs{65%aUUej4}{sMbPi5!fs%p2 z^y#X~2U4r*q_bOQQD0zMttsd4+k#`G+wy^CI2V4m=k$TkMWCyKGcp62H6@P5Nk?GQ zKaVPU&dfY|3u&4@bUS($TgO?-4CP|+v$`dxZ`9zVCx21c+XMB3U%~pUCi`s*#_Lms zd}GFDwui+F|CzpQKClN*6fO`k zkDGI+d`>Se*)1*w$eFS-lEW@8!#mp%Od1v{bY0hA={7UmU8l^OTiy6*h&r6Q_^}~! zJ8rIa$HmwGVDGR5Hs8Woe^{9X0ZJ@5>??g10jym$kh744fDgee4zQv6m^@BfcMgwF zbYV%E%(6Jn6!R`#7b~L_9DceBKZbNe<(6bj*w&X{3SAVVRxQS+ZV!dSBTL?2G@KWY zjYE+|bH2#w3{&Ny9Co7aPX+oiw%Q-K*fOJBdwp?zYvVG%ZG4CXZ(W8N~$#ub(EoLA8oSIlzwZ)XKM z&dCz*J#C>~=gmB&{U{tY62?tuqvrHkSdG{z3>0}BxvW@doX_A|eRtMujpSLKzC1iX zhdFMx=(6PuA`1Fo?w5QnH8baU*JOD%1!8cQBTyX>PMh*>Jm7L0eSR8p=15ImygC&I zr>2UswJ)K&Gl*VJFVIQyBig1+LdIX2;c6=Lbz%J((yLtIaoQf9vztOEbToGU(4um$ zXdZoS$gulo#hS?XzB2}!%8c0vrrCeS=v#Nh#Fu$2JWcB8Tk$~lI;^>)&Upj>LZiEp za4CC>kRvI0d$1#ZZXP0c&^o?AEhNj;T^!PCX zxmH7h`uYcIPbj~ zchr4F^ypYdYFtCt9+DNY!-eO{K4R|)xvS90=a}K2<$c|SD=Qyk!Luw}Egs7-9T)zP zo`D`?PvN`acwXIe4^1A^d7NMoe&LtM5;7XZ%bIl$!D`;Jkb5X zXj;{$@V@0YJg*o=9ZPk#-WEfhTDi-Oo(JotJ^0~mA(}m1ASUG>L326J&bp?;!eK-3 zY{+uqTDeTVkAwJKu?r)*yHnYF51M5E#X+51Ha^RsTKHRRoNde&QCDHz(TBg26FF+d zYfL;RcMcvhgT6hHK9gpm*X0NVYgdY*sd;F7_9@zF{6mJ!gW6BdVYh3)eNWuV;fZTt z?9yq9tWl3(f3r`?AU@KLrYNh z>L|YNQ{sSTomnV#-gCht#MYZ#eDAxRzzMe_7;ya`^3ITN_LpG%#}t|bnKP=BJs&7! z2IqA(X4jA7uR|)lAhQ%9*(O|3dmam$3};4e4`wDt(DaltgNMZOscKg?&XJkQV`)sX zw5N4eD%TZ&)#sz(R%FBx*KPQFQv%)A7mHt8CgINQKpySq!vjubSUBYicFT;n<36dA z)K{UU!#d1b{|^&>MB(^42VNR-0H#`YT)QBahy1$m^^^Db;^NA>b3;(5u7VLi61m6J zhLPTXFw&zn+s&_n>jMwo%rD2CynNa#9fn`#LRMRosCN z4o2MJyc-APz3JjI4frpq7Qc$(=%RfUdfG9*iT5fHq@%{#ZH>?jIe}*HO;GgKnen&% znYD5aUOo@zqe1x`+rvfXlhSB+L+0e#tBI7lAtK~0c(Yjk-v099=Mb3@k$1EWGdJUP zO*|Lek^h&0v(R?_ExbK4S6JCOqx4Tt_K4~tuJ+ssjlW~XXP;qgJNvRwZpMK}TlTMpYUJ!{J`6+CB$MZ~hdQE9W8Xt1Vmp zY0dlZ(z$l-YsGiH1Xf5ddR=KdRt)dJ4!*u(u&p5u$$dwqK_8hNY|dXNPeDz7ZdH0M zQQoIJt;Sez;?-DoEbYV&ooqN$w+&|u{)?)^iL|-B1O1-(FuUh)8b!6nx=#zmgdh8b z=3iTmT5=gRE#+<}wE(e>n%rSFfG4F7{DZ|mMBGzD^~cFRLmp3u?VCO%>UuTYYC%indiUyV5I# zCo(=EZ=K|oJ&57$*+ckM&xCQopI~q6$GeYW+2N=+7defk%KdbjWIPlhF)>uzuw95F z)1b7d3rfGKOFiQ#mbLa^+L)0TK2GlQjk-g*pEIL8Y(B?I_nNTF;7s1Ac!6JUhjQ}oWGGch590H#*c+J6K{|%;?U5@Pc2di~WXzfXWnN54 zqlkL(rfWJkgl-Y;l3nsP zGL?JUMoDkRNLoeCfur4MF5W#GJBsg$x%GC8wAzIuvu;DbpfzI#ZbFPL8SD`&+S~Tx zh%e#Hkh#y!4GZw~oijImK8aSz?~pvn3>%$QnfALaf4!Lrl0r!t(#LMzay)g{SaL&FI1Sdc=8MlU>=%&12?2h>rCSB;W0s+;S1XPTUxOjblW`;_ zh(}td@Th^zxP`WpGq$`BO>E7@E;8e~(oU2*eZs|)P1t{xC9f{pg7%bYO`&EM_o zJ|&GW-1g#QwDkW>nB|_N z%>p}1G2jIm+dZDzp--`}z5==3CZhkR<{Z_|p7_&_e_+5~gy zwKYG~gNwWL=RB$XT|Z#XumWYd*It0JQFmaVb{}`!$=B*Q97_({@n7|Lc2A7sfJ?hk zbHx*LcX_~JWeShQh4FdqU|MJDGG|i}dQX{+RUMl1WnL@Z9e)RJyUN}A+d(|#z8Lv0 z?urrV(qDh$1eBWnMtRX+(e<_o#}-XPpQajUdu26Zrh9O}ejhAbH;(oi&k!;CqP&McAshuaZ)xYE!c+dIC+vKKv=t>}pU*OV9~Gd~-n3b5?@HI&=wqJzx_ zJn@}~u+ufT7#+{nwsGY9Y^Lmef!m5jqOuG;{(KOpre4BFHFIuUN(SuCW5Cw&TrWSr zK93vGJM6ER5U?20)tb^T@Kfe7FXLD7Yb=pjkzrRHVIUbP+s3D(sa-2h`WS%GcFMeE zSR)Jv74U+lrrgb~RCGUO!kTB#6wwEo(_wf%-6dW@ZV>`gm!w8|Pq80D9^cA|>f-uNBlYuoq*2`{-71d|aC)XVDc^z3<^bs3ZwBX7K8CcZ*AnH1N z5u?{j&!*!HRJJhV;zhHtTWuiM@A!-qt>>7dHyhSR{CVE91s2-G(AHmzUe20aWMRwd zIR~+Jrys|~4dEN%P1mK{(EglcrfKH!y42h}v%9e*!Bw(=H0h}H1fO*W@sV;AJ$L!> zciSI|nnkgaarqb)m68{!bxHWVFUNK3zDV~B5uK~9VM&Q6vre4Dbp0~4c_+Q43zFns zH-%$Zi+|@!Q6>Ga+grBgu$eEg#-u+NA+fasX zl>7Co2l3l>6r1nu%fRM)pg(N~ruV2pf<*#`*dBsoAL-Am_vP@Y=f1C{zp2h4iM_4b z(K|nqOOIT`>ivyCt9#-{yLNce(uy0>jp%i8x;U9#ihq0EG3WSS;gOKb<8nW-M*7!x zWP~ZM%Und`)uV8czOVq_;Y|^ zBii;TVA9W64wlc}%FPxm3DZNC%=3;amEME$>-aV29V*f#)7f(};y<Xuv+Oq=vdx`uG%RyKK4?S8YxQ-fMiUpyp6AOKjNPBcO~9>0G0Y(zU6NW zU}6%>8iy&Oer^XE>m@SQ`R+l?oCPe!raul}^&i3cZiMeMql;^`)1(em+6XusTs zk9o;dTW!tXefMC6oCW7h>%}fz3}~n^A8+H_xxM8q1itCd`1!9E-_3bC#THF`i*2z48>2b`Peg!3Y>)oNDsd?iudRa0Z6*HIIR=OJlA|~wn$4qz)9_oJaBWkBc0)bj zckVij=3hb2%@dI@XgOSW`l2ByO1|zzyl!;o{S_bZc*Zp}i=Bz6(!pHkpUFSJ3^Dvn zGwN>{AvVo%BmYjon$DR~lW-6w! zr@Y(uVz1^tG?3nW?aUIm4j)KGwTbvR<3Ege7luLO2XgO}Xoh$j@cVzYQ0?E5i!&W~ zaJ$UHwAm|j9{Ma)5h7nK5bkG!S@!EM%D?_Vn9kED2!op;@?JJqwjLoA_>9 zBl#hxqBz<|YG>~}SXO?{H@?RZ8jbMh^8dU=f0N( zQ~EwQF7*5)L*ctAbu~tcmCao7Vbg!;^SA*9CL6@B_6|IO4On%06d#XGlV?I6+ivf} z68$ft$#M^tzkZ0~7ZEgj6d}DnR@ApxffaUx7;L0Ux2eW7zIRG|-mS`z4P}Tf31RDh z1^5)>NaOxeM^^Dc=*&@!jFY+f`-W_?s0Di;%j0cRS01pCS+`x}M1v?g`kCO^KwG+P zYQZ(@B-bU_9_`fBSf48QD9@yppnXWxeskfO;E!TLdNll{-gfSkAMW^{^R-rQ$HvO% zxYO>o%#sqr#Zl}Z)tOtXJK*ByoA7D=7>3=hVp3TedripY38f3Z_P_Gz8Q+cDzxRUP za3iKaRp51$mxHEEB^Q=LhEBi3@SqR?@kZce| zC}V2B;n=717<-@uBk%OU&xAJIaYVBERt#e6mzyzp`Y~kJIWiRknAU!=7-f~rH76=D zcAyU)Ta4jpujvT8*(jOShWK=JC4!yFjekzy#k>dTH?S#p;;HBrpU&xe`Skwt9lDK; zShwUin)Q1wj;C!CE6w{tDb|$hVg@#t#tEY6Fgu9b?N^Br;3-hjWo{zL32b8w*a!zJ{9PDCU8vif1y z@+)H5;Nv2wej83^=Ch#AL)>oAf^jFAe~XM3O|%vv-z$q&TV{yyfuAs?+MUO@N3;LE z>rm`=#;WXHmw|DBH&!=o^zAeMiy%A`A$CQz)Ls)oq3);)wceg-I z{`Xpmaid*nn&`_*F@>}bYJ+q83TSgMSZtPjd)=4j)IOWX4b3*fF|N0`Jhm-QM_Kah zWOW`r-H+Gi6(hD^AGSRp|Gk>a==VpRSr)qd6av)q(;_m~+^Ra_VLm3yIfQ+mQa z7htvNFrIem1*1u;vHom4Uq$Coc~>SE{gC&ctOUj$ZO<=v7og#t2G>|lz~)WDZA{BpS!M@bmFAC|PgI&No z>?;{!&rAQpDOup7vzb`YTaz|PD==@40iFdVFaX0CZKlIim25inS%NhdE_}CTBd}mA zx*>%*-dWtTR)fE*euxe7JgF()3-z6O9AzBG=u0bMx^sr;Jl&EbTY~BThBCQ#6()Uf zVd01XUYOgCwVSqM=AU3Ldm#DFGN1M)(2nI(cVfb_X8inftC%>oGoL>&;sePNyZNaF zdbB(sdb(HPPgGO--(QQ=on2_x*c??!Gw?IHrF?x0_W$I=PT@Kn8Pbzo7ev!yqueR` zbVc~c*%-I5B}+UK=~UZ_v$v|CZOTkkNoMca@%bG1xv%W=xQdX49}uJaR5&hnV}3@4 zXt7Z~)@;@y#iS6ucDLgiS4WONn8DV6?U-tn!kW+i^w#`@3!esa)JrGs*c!<%I`>80 zt|pAVqsNC*5Bl<>LHxRLPq;acpz4@&7?Y+W8C)xI@?1Eld-h|j-2FcrVa~PXZtOdE zq|6&dbNgZs+;MHoTXvoK&3U%mZ@R%L=_m#z+wo!NV;G>+5w1@zDhj4+ibvxlOLkUM z-Yb&K`M_6~1Z&YXMJiZK&{*hY1ln7CW=nk5r^ zT-Gav{)#~6ryNDf^JIp(NS17+D%Vuhp|F! z0oL`a7cT$Wvf@k}pTCObn88~xta%(Bd5+`&@AVkHPK~y0d$Y*@jyUA+A5ahlk(ZiQ4QZ zig~RH`SH2n>1QkP)XbZWDI4)LS09hRXo;y~X5yEv^j>LCM?>2NC^n>1`@}7DEE~xF zPg-K6{LUVJY{NBUKZ}2!t#EX}Vw?>q7qM?gaZOPRrf-qYvAP6aR@x=L+#Sk=((itJ zVZNNH(lPK`9(CjyqtWfCP|3fFhF|AJqx>$Zwut5BgM+YZcQdGZpT(i2o=m9q7p-@E zMOyJ$B;MPCb$Kzo+TuS<(VK~uZB`;IpbK|iFBVhf^C|sb9$QV(XHmvF+@K3?U(A@|EwTho_P!i`$#@d172gdUx@{o4+1> zW?%z@l!1KzUl~-M3=-aLx^nC`UuqqUqUM{XTz)c_h5Gjpb1|1kT8x3$>@F;_Ov3yb z)p+@M6pQM+aLeLJqWRH&JZ=%d^q8)k7WhNBZLJq+a^5n%+JQ49ujRqm0{RpN;G$$m zw_lhnf;Xihr0ErO`7ZCF!U?Zj>u{q|kDpazxN1)qamW0DIAdZ@gJD*vmi`+P#R#@B zY0a8-OEGEKP#T;YLys?UJl9O#N0owjf4H)o%{y|+${w7eA$jLNjN#iDkKFVa8sF&x z3rauhwpvWQmQT+dOWM4tQEZ$N$ZL-dVaq!Uz8(D+MSOt8k-rt|qatbE>>tKE>+wSR zX7QyU9ol6fl2J2NJhialw9SQF-_C$tj>WNmvL$o9ZwTXaZ!qkk+{Nu4uUHX29wWLo zqvhL`FfWbamNS9k{I7k8jZn}^^B}x_gyLw{9mR>-UOe1RGNsavDJ)j^k{MVv>Yub= zP!id8fEP2j`s2s6Dv6Z@Fh)#X3*&oB;P2IvvySM(dYm)6Z25%UI~;ts=pDl03JqFWCer%2 z8?0o4VTfA}%hOu$&SXDE$n4aH*dA;gqE4Imc=qWa^^5QMaR2&M(XdjRU31;}ah@&B zR}bZpUZ)gZpEqIPzp)(nNSp81`Eri!7o-)<$A?{=VSZbWePrHXM7H#LNWCsY_7y~4 zEQ6`kyy|YYV6noUn+p3etuct^*$SRdYKg$PN_;l&GR_UsF+_2YZ)`Ll+i_ zm_2?x{2C=)S%jDJ-1(B$-6u)vlH2OFAxSvgqN5HKttE(( zy1DZC#b`FAKaU2DXM4v;(Wdi!I4zfc(}IEgEx&YMGOMt0oCch0oA8y+1T5TPCfsW0 zBIsN-O4^2SVND3bHc4iLd=^Z#&f*)H>B~;|iedkBrB|RAPneZp*0~bVs$>QxZs{r( zU6~?QB?fcn>o+3oKpk!-r*p2hF2e)+aMR{Tu=?pB^{`%0>so=670t0JWdX)jO5gV% z9iPidC(z{DW!#@U7InL2-@xiZzIm`hQM^P4E%zmeyRl!fQu_An`Wf*0@|j{qNn1vz zj^@8%mtp&_jR-G&=sR>u4(|*J;locxqUY9;eEei8ma4_@Tj66|dVdS;Yqz4|_hc-a zq>6x-jfkt%v6bKWWcv{_(_dw15ye;qhExLyCLXPysRL0Y= z%T#E-UxxjiYejL%9C1r>;>@J>rrrG}+|up&+NCXfylh2-gbu7KDT1b-3SUI2QT@hz zgnn@1r5>Ab@oOvI4-!y%lF02UvsX=s`xcYu=)D&Fy_>N8vkZI-j%L8Tq3CqK4(r#gl$~wvjCSZN z(vm-6`>~PKYws)l6>7|t?2LdZs=O$(4~{clA?9y?sI?Tb2gZ_Jj&|h4bMufnRB}O- z6R3X8l{qp;H>PDF9=)H5nXM&1X>|`q?e>F)@|Mq&|2r+L9C6H{rAJi}02V z1<$*kq>;idpenOA&fCMEa>&8IaG2pS)1`ua%GMq~a-J2p4>I7xO~W|4VkthpmYk7*?rgtvsCX!QSEifI!)^Tl zPP*_OtxpcXPx;*Gl#|T1p^j)`t;{BMtI;qqn3raFa@OH6T1(yFWxeFy?Ytl^|I(*n zb{IasxCL9YJ>s6S{92sh7-EgEu2V`8?cg&m&=CE>{F*^Q@&NmM9zZx%L{2tC);-b^%Oj z{TQd8l;fH7Y*jt%%=2YZ=a(F1v!w56a@&thw#D7$w2Rx*Rz zj)Aj@HM_YEXVZi7XHNbH6WP!8$KH@$X0_1kTnnEW1y~qh#g?U?agiEOJ7XHF;7JXm!W~WEJ;U9cQ zBy33)!Q(Y~>)>6)A2ATYDJO9q&McE%LeDK5@LsZy<{0GDW2_%HUWtlr;@vv_eC&5qDOW)9dR^xg04G^l6O!eW%k(QIhN}mkAk19d0->=1`!#T{V z%VEnI%@j)&cciu%jQAzV$Zq}y$__(#v$gD#JgAG*!Vai7(Uu+qWY5mIcv}0-MyzDP zcntW0%}alXj^=W98(W01n&+q(Y{Lz|m!XYqHusE4#e%b*&vzap=a%&JXZT(h!nXbkFmpIXdMATAz>LmX3XCAorgypFYFmcHL+`bcOh{ zX&@tC??BD_ccO<{lw<*_N^Sc&7PL4cwS38%HSWeZr8uURORnfVS01U4l3bYMi23gj zPWB#)gKBlyxZw!+$zL+B?Zvfk8f;#<2`kDcz{*5s@IJO>R?=pq9bFBTwVOq@O_J}I zj2x!AXfoRH1k?stNng7z_cm+8T@gPKaxQ`gCD*rJEb-@kFs( z?U(5IxI(P(EMV*l$-n5Ez==QPY?@-idt=ST2E*2@cp1qBjps$P5w{d`J9)6`dJ&v2 zOQ!dd-y(AS0B&EF%Z^YJKJ$mm`FsvsKg$_#t`?6h8pnN!KCD+ChrfH|U3S|ZJdL)% zd82`RZQF$BrAMfx>l*Q*pEmQ)>N0Oul;~NT!()enxNzrSEEcw0dc}uh`{?qa>?*i_ zLH3z-k=YT+lxtpIgNS3vi1kM8Kg1` zZ~F`orUm=Oh&8P+ecm6K?ir0k|39)aW{v8Y8d5Q_hg5 zp5=3RwK4O8Wp-RA5JN1R@#KV3?5y#}p-0!y>Qa@smwf{L4lPH8x+YCyo1rF%AAE6;9PvG&+$pta`-&LGs69sN6>CnGbGbbX z>D@bp1-nP{r_VfbZbF6d(7c3=a@Kr#awP@?U4_;84=9)27iR{K;kPTh#e=#QC{J|Y zPW5Ht%J4wRTfKs97w#!~wRU6U{$PIhYs#+%vZt@P{6008jJ8EjeUme;z#(if64h@Z ztjvJhD=QS;bqv_)v%oN$DD2CcfTQ!Wc(#58@-mI+uWZ156SraI)Z;i*zZ^SPW>T+9 zGA7>`%_ADNa^JT?Z2x)#Qlf!xaFQq+(}AHPkf(F9d48SjvHAdvaJ~bpzs_8C%L&D5 zlX0Vy?DM<)7;jg~`>)|n)Xg!L{Zq0B?O=cO^^`qb?G0JJy(zB`JTBUVnJM(=jKrT8 zk_olppCVy!EbrcL!}-!-Xw@kjLDzfIElye9K_=kC?tF$n&0?~jG3PBgq$vGu zPqnhP{OKgw5K(r_|M@|7^Ss4NZ6!MIa%cPaQJf|Ha(5OQ@Z|yN`&$+cC$H~tGLp>J zE60)j+L1F|PKu>!^Rcp8GJxMTWn!<{h}znj57j*hf3Y{8zGgckB{ zUf5CUXPPgebZ;7hI-C%rUudJZc!|>2WB-kTdz1`PP&bOIpHxy8*q@PKu`|8n9>cT6igiFynb6j>mh@WI-Zx8vF8T zU4LdI>(IsBQbbPbPp47M_^!)v8lU=y-M_jp%p{1xp7({B^iWT4vH<#ne+c)gCRlAZ zj1P}Iz#WS_B699A@okR+cc%D=o%-Fm!hM|Nq9ya;LzzL)`3C!gT{(T4kx-I+$#YBE zGk=sTx3;yVx@7&Y{nrc=3;M9%uN9bna6T4xtjB=(=_qV*8LOTy6ZR&}Id;@VT(cg} zeMjb)~KLAvpaT!W+{2vSN-mFSi&hUj1jrwF^rXzpt9GGB+OA%g1rQ+gS9u zB7gt#1E~3o9P`GH?)S&B&E4Z@e%OhfE@vo)^qY$fbDOjLV<&!QBCuOeahvY<)ZXH(USbPas*p1-YD}1Sxnq)Cig7aJU#0-93;=fIe)6$H|Mf* zoi#Px+7Y8hi0!Kb(0F_a6p~*SIcy#5R*y$;y#aT>uw~;HT`FA(RU|?Bfwz9gpT$*B zydT0)^%%qy<O7v&R^D8YVT?lOu{`u}3zApaFut3< zdM(F94aM3m7 zf#G?)xn#NGeyTsrZ;g~)va67LZUwXqx^r7EZQeQ@$$Q6xnJ#l7K?iJErreA9mw&>~ zRF!&5&SFPW8Vq;((t2|MxW*D>9wXmn;w={xa#T0g1|3g^G_NZv7%xq5#)$w5$M<+X8iH7v%wmoHFJ zJCZN-QW!l;Re?e=} z!AWv)pB%)@XK9@4HCOSZ)h0N0-zpAI4q%(ZMTlwMlx1#n6gRcA6{`j=!~z=^1~W-! zcE_@MvA5!~vl-7w9)ZF4G_+Zz#4DC#SQeq+B==c`t;}e9~V*LA~_5mefVB0fxCMTW2W5w8V^2!iw<%p z)l`=o;^k-NH%Isu8^JQJldoS>EqcmMgFkUkaHL%i2HvQ|+0jN!oKoi7R(4C3YmA`7 z7b7$YR%2R&5*_UC3xhK;Xt*IW_z#hJ~x_9UF?Yddh$pM2go?#;@d>74a022Cfo{Qnun<{xudencyj-~*aPwxYdc*Omt4 z)2!EY#l)4iV(h$f+?Xl(nOkBdzd4X)L5E z?6mp%02ckQVb{Q=C{W5^;@7P(y4sEPj)Az>pv55zlc_)65eY8$74~Y0aGxN1isz?_ zn_Y9c?7Rid!+*i=>PHxLE5oLTDG2PWi9N?;WHW3Ef;@)^2oy|F;h~+Ws99WRL8-rq|Hph~P+@L7a0v ziN-Fm+>_Xz&2(i3bBivml#ifQ`e<(Y)}KMrbKWt@j-&gX6rU?E;qNCQ`;opVoKz-b zwof1%*2>NtX%%SrY{Fq7l8JA53ZolZ@pQZkZPLAANq<~AM?0WH~ctc9F8y7PG2djv*Wuw&3^ z{Qi@{OX{=mX68~1oH+wa^Dkg}@J0kYaN;ScPc6y1AnrL2g>dnp;c6`g_n3t2Z%Hh5 zlXLcm3;5C3jGHgI<3@5f;UJmdz8c!xBXt0q>UwB)aAo_!tvGME84NBh#RZRBV#w8I zJiSvgR^_agDCUUODmTT<=ewlF4y0ZPlKU2a4m~nK(JgK-Dz)}vw>XQnNpf%es|PmJ zNPRC{ekbmJMM-o5d*!{tn-fNCYW55vdb(`)wi%~5j-b8g2%grSg$C~w4j)@2rW@$- z$D;lCuW1~u4h%+li!WH;B!O$>ndLg&j;o9Y!7-yFPrvHP+3O3rruZQ=GwNV61F}Q- z5ZZf~!uIqfXmxoaLWg}2`E|0xF5(WJ*ent2&0C3UOop4}td9)rD_-A~`NP}Ycr;)c z95+g)hEFwo4F|IK&fe5sIUf~rC!RigJPW0kTqL>8QCeN-Io%EI#;y@v)uZ7u(wHx< z)!-`}*#BFZ^qGCf^^0*F_tBRn#_LdLR*$P1>Rdi$JBANv#qzB!MPIXa$h)W^wd)*q z_#8{CPVqFa>&}w-$Br8 zXC?|IgC(=#v$$%h#Q@Jpt_hQSdC7SC^23!GyX7uF{fEN1UldDA=b=>U##f4cg@)xp z6qLWkK+}F~=8?ryD(z_<<-oh{Z!p0x2(icfc*5h27Hz`)7g{tdOW@FR(u=u$iQN4dG4K9h7ETFa zVjjPJ?E?-lGF z?LnvZrI5bR|kZm++B~EMv-(I6V93wp`1FTv#4LI!pR3i71vD0QfsW#-P`q+^IQv_ zva*4h^mrIU$9mfe;E%b-&&szLhExB^W9oeMU3i#Cx zlHC9wVc}ke5ZTGMT;&il+bQwq^9uBTY`_^2Q^kJ~*3?R02iqI}Ywt^-dR)7BqmUso zrIaEhv*yzO-9Lm3X)uMPl!!#lp(y6c?vE#A+&|9js18Q$mFy8@Q&O@xEEL$_P#!DZ84nh-1MWL)1wjdJeM zoglHVvAQho*L_Pa14Zq;T3e`v(?N3j+z~^TT~pS)5Z^g6?$D%K4H0=)C(aC*(})LC zX!dq-@8xk*I{siQ-Ak#B&>LM5A#(i0Wj&E}X9MLl>V|l~k`(M{ghEcsXh@&}8gf}V z<5Gtnl>SOZW8R2+u3a%g^m&cFC-$Qv`{Ck@mU z*A10kH-}L6*^c!1Xf}1LZ-&KAf0y&Gv|#RF71;FaQ- z=|JJhC#iSy9`wFkecYTqn-E)<*TZk;r*&g_$0Q=+o9MGENGV8IJXR(Z=F9VfG6OOY_CX!F7>Q-Vl$9^}t9kFFOCOB8K>v z!o=X#czAmx3ci}cIM{-Qx9)(LbtSR7Q#c;Z5xrDz8{y;IX%sLrpW2klqJ^Eu)AsY* zmGosDFePUgx`}W4d}QBSI-W|rywsfO%VO+2GoFYemy*X zTNQ;)d*VZNU)mK^f=4pTsQu7}sA+bYKKoZgV)J;|Zs>qsE``y^xV-4E5KL=xt6&0&ewPJ< zDduE6UW*>^gDRL)*qUBQ|FiB=D_@taN zzo68pUG8L!_@QFY@$+-~ zw9%Z7%%4g*mS?EsyPae*tp>iXx=W9}#TooL$0>H8#Pnap9+Ro~Cj3(AbFU43Ts}>lhdWF6OkA*UUoku$G@Uj_ zTp{nRP4Oh*ciQFF0P6$A`Ov@)*zwH`J&e{;LJ?PVPA`j)I3G-j?u+ngnYtM>k`b2I z6!t!gb(jAr1CR6xO7hWEx_B!Y4#gYc#O!s-h~xDrx#ANF=w1aQ%Zan3O%lnbbF8TI zeu)yAR>NnP`83mPKh;mqqXyzEM}nx~^<>aJ3f`7V*S^)lQ0EyGW@z9Z=%1) z`)5i}(nzcq>)_Z11&aNkwm2SDMAQ{LPBrg~^JRsTQEf*Eg`Fw_gV;nG+v<)kYC~x( zFPpAJJ%|xC?K_aor;>P5x-=?<#o@epXWZPRYx zt44XjTAbG~?68i4_NR;1p>32tdm52%(PYt!&zy#qv?hDQZ{%(*=8*g9(6?RpXieF# zv}5x#YUTP|w>HrkCtti!_J<#&Q5E`7%3BZW-=Z((F0M#V<}Rl2)xH=v<~zlXIiL&Z zHI{0$DvjzNTH=GZIBT)Lu;*Kc8B|@I?KztMzMC zXW54iXBR`xr^&QxSuVABZB5@-i(KePm|46vjrFWS1?~nES!X&G%y7r0T}9~8g%-Gf zcde+wvPKDyM53(WedfGT%@Cm zZK#q!P3OzUp~0#VD9A62qdl#ueb2iT^Gw{`{+LNUn&c?f_Fky_ygUth zHbm5Yt3@N4E>!Z5_rl~2w`s$v08)~H>ZglQ^koC2in`v{7xhw1o{AcDZAa4~M{m^M z6iaQZc1MqMxyrafI^fE6IwilF7 zZWEQS%O;3=zrQN>XNsZp&0Dl)@S&G2cE6;b&&h%sZujhi+6z|PDG1-lwy8RssvsPXgU zMmP9v4X1f;#GZpuA2iwXT=6ZPOM#n))4AXZc@whwkT9Njhc)eOAGt8!Bvw@lznF@nL1QP(C(fz z(ZvEC|1gEk>H0K6RH_D!gL(sT+Klr4VL&3G_G}uziY4YbO4d&NW!Y$j-l(D{u znlqj5p0&s5=MmU6=mdG37rpT|i26&#tYMyEhsRCK5wSS{X_n&d+O$qsnfsj_T^pmD zr59SPD@xTbom3WlK2Dn-e5V65g3xqe9p$jDK3u=-BPDt;j@7D*4Pg((K3FSU>{=X! zVmeWlou@c2cS^}VY^1al-^n{a6|ocR0*JlYwKoP*ps1TN+qEO^WccD(vq(BF4m}SH zD2YSeYE#k1X-ZUGW9pFentIsg(Z^a{5gBHSPNAJJX6JIHZPOZfKd&mDhX1bkJt7<` zw3|K~tfk}67m<77Guq#;65LW~|2t75>DxC739?1+tOf|H z-VK}c#?j{(ZydX4fs{(sXtwSml^!PU?)S4t@QP35SR@SlMBRs%75<>$^{w&cwGYOR zx<`$xl*QvI;$B_aGsW9-G}&zRhv7|MT<#<8FTPqvX>C_2cA^eP)z?utw)hLhRvw9W z8y?X0mnW1C=@pQ;#R~fdR>sG@2g$2QA&fgWnlkNeal+^{6*}4rM{5cno!5$@A0AQ4 z)e(IhYGu)@`=ZX_@K2sTJ7sgw?Pm6aNvj}?Z<&CJ39IOCQ)7DQaF_h9_eQBj=fxe;nslm1 zH>@`-M2i?Vq zP_sVn)VI|=?p7aj9H!Iw-Z3=XxjZaxm&dddp=fZGaAc0?t7$L#SMD`LeLHV_8)A{oODnr`pu>d`G~qCTdo!)4$nI6-%((j!Kg5<6Ya|I5qqRBXz7z*XvfKwWF0vOU7i>sWkoEW zJiMt4ORfNu?82Da@s+3@UzyezyVJqbi7 z^JHP%%4mXFVy|-kq!DN=&Y>91uPx3%+lxKG<;t7+e(*lFo7`jjQoF{YFWHy-w6s*F zZq^`U@>|`LuBT_wxF=SW^-0wK9A}L(T}3ZjPf`xGnxP~)8Q`v|=+EfskI<!p?Sp>iVZ7ql1am<04_Wva|NbcUAn8i8qjPEh^S-f-|J3M0|GHl&j; zdXz9j|N7j*$)JlRwisv8 z+%?TmWKcMKBE|k`h1pahB^gii*Xbff&C9Rl#hs7mC&*#uH^r%W7+PF!0$bOn__jEk z?wJQ*O@-!Ya?6i=_a0JS+gal6UL#~30%mz6iZfpUI5(pNPIYh>=iR2zqTszsf=zcA z)$2@#4^Hczjtij?!<{ia3#e_<5%;R4Q0x~&8h6D)chWnMvRg+|v!%5#V8c!FCu0na zd`br0{V?@r0uElS3|gD4Ox<4w57!0Z57z@UZqOmBWl$d8&0Of~phV0Rb0ysD0*gQ| zbotGYEb63D$Qn;9xe$fvo~P)Hy$QVY3*l|554>$+ME(ED-`9}#+;mZH4y%bv%S656%@Zj)XdHb#=81{zN5JSyH@xdU6ca?x50|s`QKF3l zo`$Zc(2&`p=5rfVuF@0{W<4>k-vRo)Y$vz}dcxqh*<{_$9vj>;D56?@Jb2aves;q| zUB}Hd`|@dOBuj3Wi#(8^tBiqUFyQ^)hjv~+!6IWJ}Q3Kn^T@eGfcBLRd(3K zVyUe=#*A=A)%K!?>6HmoWBd)8KEDgn2xfgPQ|IJ$G9_JgwqM)2qj0LOW3}W~(tBUsndF2ZbnmDvH`YnU$30m1mIe)L^V` z;z+g5?4nOfO^iFch~C{3wU{?_!MPkeC8uix1c*CG<%adc_ZiXX_G=xw=mr$u5QN(= ztBV?t(WqGC7sV`e5?yY(m-4rX`-dOIna9}|sB4ejSk)twQksal6C#yaRx_1JPCIG& zj#g-Nc_3ML*P&t87&2S4kzR_r0zS6a>CKLoI4JJXEs7TXV@1!D-AzPa-1oImxkn0J zyeDd0Y}luy56Y&_mU*J)j5~anS67akI^boc-b!lRak{kB0LdTM&>vgwD77t$)4BQm zU_9RiU7F0KB`>@%`k}ac{Jj|rMeSVA7iDnqmcP#Wjt6B#1|wVCiOan-jsm~c(Cz4L zgWZ$b6Oqoi9a^5MA3dP#t!R(t;;h8xf>;c`UmCOWex--^=Th^eBeX330CibkSL~}6 zr5oZq$U)orx;9%x4V?vq!CU$%HSk&K8GVaJn)C!Y6r+mXg;2gl8!~$DN~v+%mBD74 z$;rkIom@&#t(M&|ARrJi#^Jag-WV3sF466+wGgv6%d?W>8pR^`Bh{PJ2p7wiM`v*^ zGirP((&@yScG+QSR@MZYa{i#v(-p)_ct~L{yr@h$56b(zNl~_xN2U2yDfx|*#l;c8sO2(Th!XN85~-OTw2v^PR-i?Sd&Z z!NeQS`{|GxGLddHv!^xWrZgDvk?yt-cM>YK#^{UUe8VlV4>>v(x?{lJh}zwGf^3yN0y%R93Cn9A%qUY z%AMsYK2(Q&4FTUy7d&&Etgvf!C_EqRRL*2aP}z(1@&4W-va9owuC4Bi6d!9mzj{~M z-l;MyL9{$Cjgn&GQT(6=#zZO|9uF41ye=w5n>nCR zSsPJ@uRPq%c2bhjD>|H+P8}l}((VjXnk#B%3~zgr-b}NF=>l_nD;$H+I7eLE8$tEZ zjTRSo!_Y&9NbLQX_Fo&06QWMjg29Hc_&5|E8RzN4n>?k}p-6F0Zy1K!deQU)6X@~S zlXTl+H`Ns9-s|;=$LQ9jXvoeWq(GgZT}hb9^~`-04JC+q+YfxB{him(DPG8jKPK&&lrDdgbJ< zOl4gDdvbmeAZnWz$5X>SN}gXK?E6+6Q_6iM!;F4d8lFS@z2d2pe|H=gXM3|PqR4a8 zc-`g;qQ>dOXSDF?NHn!?iYn(sjo*e15p6aMS85(5bJ6!TyJS}sEC@u0Bu^NxeN4go zUnpDWET)g09Z*%5LHEZe)AdeAXwNtU^0<;i&%_<7-QqmzxZ+-LJUC04Gi4DC5%rTy z{ie{_yACLOts^}gEoyYH4#LY3Ls0MJ3fiQz!#0bVboSmL)O)8x`>`u1&h!#xh1Y?R zt2y$;y!d6F6ZR;L;A=kwb$#cNLuOfwo->CY=iZ=A^&XK=rV}3azDZ@DHuOARcQ3UT zIXu(!rSiPX6?*ohG!ka_r=#NB?LB3Uu7qPC-kX1+XMW-w?wp!1ZGD9b|DH;2UxjZ) ziGF;#Jf+j;cv0)HG*%RyL;FmsC`LI!*pfa&cjsIyls7lQ5Z8RA#L#teQ2|Be>$C1o6?=@!_ay^Z7r9KUM6NJcu1)7HG?wV?Vvre zZBh5qBwD>79!-bYk_mNz(yloo-yNeg;q#f9ixmGI#uRqtmTutIp*UTkKXyBLV!_G1 zRIZ&k=XbUPGQ_#n7taeLDph>r3@D2EX2a-xjdQer;6CbXm8eu*)SOIa&Q{`AWYCG8 zH4#wU4{Kf;;Z@C(;@tmw>TEPgS2v@lf`KtgJ5d{?<%K}}=K4yvtbypq4ZkIqLj67tNcFVCA3cWPaR#uk=0wGBmoJ_e ziu$k<$CLB%D-`A56E2lRt=l&BaMHzuI(4zZmhAS(KNO4TA=OdD?m8X#G>Pi>7!2>! z04lS@367pN_>>cd>FatcA18!hS4uE03@(Xvwk>qWg6g2n&^0vu%TlW0c$Z~XaN7LF+?g-yX7Cl*e zInnfn;=Zv_Pvj@3Dl_cNabMhnH;o^L%+W3gn2?Olb|Z0T=@nZ0)fr>gR=|i{8_?J! zYVxE!S@#m>sl@HN^TV&}T;0UEuz}_#%4C?)qMy1u+K*nO_?!^6ko!BqZ2bzlFt0Y|e9og@21dyB>xbR<*XyP&6@4Fz*TeXS zMQBIgr{s2}F=_>b(%IAV>HDp&BQai_Q~My!gMY7hn|hzm zQM!g2(3~gll~Lo3VANdn@x0y?cUN?Q*WF%pYF|^VK0Qa-e73D}CTJwW{om2Kj2ycC zRn)z_dW*LIS`R0di~Y(nLos{16+Q7!q1t;UD&JauqpS_yNGcSG*70uexEx2iHbkK# z7m;W0D9kmlf?Fr2D)w$i>C8({(f_Rt_FN00T<765e)U#S8&LEs5qa&5Jq(sD~+&Cq1LxV(-A^>2>8kp&b~(@gZK6ul8g z+hOe7H_XgK>HJ;W*Yjg3vJ%0x!#0mR; zzDm>b8zLyQF$Pwu2iHR<=A6*^q7TVFFQ)#qy zp%q4W{7T_VKGRjR(Mrcj4)|seuk76@>NPgV)_rT3O4I(ROra}U;_mJw968VcNuoY8 zni-LWxSud9Z7^1i9EnVEX6VMRCCSsPAIi2UiMnUIqQt7on6bPHUZN^ZIZzlk9=c=Q zPSGnTd#ciKdnpWU(-z|@UZlt8t6<>!Ug&&jBZVgo!?4vZ$Y}15vkQID$S03VU2#Ok zhnCdY-%U5KaTSy_c2oSt8SI=9HL&Js2r87bLE8Kt;ylj+`Xkv0jYThFk0n;fJy!_5 zp5IdLjIWE&r)?2u*bZ&2d|*5*hx)ZN!oUt8C@8s+`VY-g_K3TR_V-3mdbX&UJhl)e z_Pjx%I$&w^Fg(pUP8BTsqRq@)`mlLCxt_kR zWOoijYcap~99x_U`roGxc1_`C*930HHz{v$QJnbl=iamN3R8@9bnp~5!}sGeV3J)Ob`CLo8Cs?tB|6A8Uvw1>MnR z_+9FnwU8D)83N}^(bPlKE-mq(EQ*aW!>J`n*kU2>ab63B*}|vv=KEthHT(iSit0^c z#@|p1n}pNom08rTN_+SR48!z!lgQ7kFdf~VD%Pt{$T-Fjzdmb)C$;)upV%*)Std@@ zDKNqCt7ZtOA-)4H^+Of=&x+Fa2^B1;MPI_JDxWs>#*j;|m8zeIW83wmx|+ReV`x_o z+T+%k))x`|;%sxNr+*JrjI*Q8PRZytz64V1Wza9h&(OxPIn;F6HG=4inYY&oO&^H< zu4%uL`Ek*oM(k}|zQ0TK94>=rjY1Idy)4297ss)ZT@~k>Uz8Ms3$*M)38a-gO_OtG z=zPQ;XR#AyDC^=7YG^o$+U;6I?Y>vRhv)qiml-9g%gE=-r*4A?jVq(pnM9a9D~@DQ z^U1iksJqinoI}e3qQ@avwLA z7Ju!G$fs+mXuYn;h)%|*1CuEG)Dk+eDFSCNZPiuF_)N#`v&g9VMsZ)MBl35aL24mS zWR6)a&eEBnMeXjG^!PEgz26z@BaTwHtb5Asef6l{n|SQm)(ItUh(3Vgn@IH+jc__E zoxa@&Q0{EdA^XZMGJjD6c54TcYsOs)ez;Veso73F4;4jvyE0hXwFS%@wm@j1H0tuL z0gYNT6mDmV;=-LU{Boxw9bVa;?%FoODc=y-?-b{790L$mtT_($v4C!4Em}F}iEdM2 z6Xc2BVQ&vvQJHOG4_xdQ7Pl9@bHseAtjBOTw-(HcPB_`s8^!v0!?$b&-M#moG&L(l_j-66M3f6A_tdde^l2B}ad!g@ zE#*!NX15^wwqNOSehihdjH0uL3SoTB>PY(>j_K`0-@eGUX!hn7y{uK6k}^u-UDcIz zEXEFd%DPeW=g}x0Y=r2{v&!?s{lvE(W2|~A?zM|`@#T`GU?sjmxsRzP&Y<_f;^`A9 z>{=mgJXsTS>a>N6embM>=a=1Ou*E<=SL}!$RY&02kfP!|(teuY7=~724>W$kSjERc)Z9IEjC5TZ!fIuE znC-O0Hn%3YX**GOJpCx0&K7gJ)>fjYjEK(`qITP!Ksfp@rfS8#vGi?CEZ7@Hx{B2? z*}V{w#N5kb(+%O}O;sjr>_e;W zED__s72ZYW()@`I7}qP64y1IW%vZsRx5j6F}lekC(@HtQKzWbJ$mb3 zjTTznQxbZYgU^>*w02@7=9jLG4yDBRmS+KYKfOFv4J=FHh6yP6xQI5)`Q&lqktgcv zgihAHq2xRl%Gxqo^jHXkTgeVsy4)6hHs7Zl(UrD$=e<-v#~MN2^>KWuGaTDnBPn7E zz3Q1PYj`4X&;KKS3{tuS8frDfJ#^dVv-}o3VtZ#gL z#9xqZ?*9!Zlf%*Nzv24CMnpva_0PuX~g9MJ>vcI@X?5lxee^%X7 zeK64Qr@}cq{kNVOj{Q${15JD|U4C-buVvT!u&B^7uCQrd$8ZTsSe0+%oWp>B{mm%fXmkB>H1}Fg}^QjE~`* zRrb{f1E$7s9uhxbeEbU+N&L$AQ`M)5BgVo{E_&_8e&r^i%UD=vH$6ZW4f*qIruHp{n_!u)mdL2XUwwrkm^t3_p5t)94%z&G^~ZLTBa7@=hlGoS|*lx0Mi{V&*t`fgzIHv2U1{}vZ%d0=H zF+L65ALiA>FWZgfaFF-`(`Eh1#u<(N)bYq&WnXzow%Ww{o@k1uBY@Ns6#80yIstlj%RvP_j z?565ZHh=uhEJAvSq`?Zvn0-04#uobvo(5QIL0TN z|FNDm@=D@}sd0QbN%&*EFkRU^k>zE-@=yigcw>AFC$k&N!JNwM!FpkN-6e8p;8cH+ ztwWhE%PX5dGd||;r`G|hVqa{mMmR4-K3eZ$CSKmgONCe=1qIys96_)&Y!<n|><+SCWl zJ`&3-i*x2rV-LwZjPn}nS+@RQ?)gcUhnOzI$^44nYUoP*h56%O$ox-6SLRpj2aL~A zZ$4l++4_zBp5@R#-(`I661vPE+xKVVT>rX^{a!Y2VmX*U**ML5X1~(E-e)*jTrxF= z({Eqavn(!Ua{O#Qm96t+`1FsLtY?}2Wcb`9^9_b$e6A8W)*nCVr_1=%>7-^{;CN$u z$oOMFV0(C|X-a+2tmBzK**u%&)xb&OgW;I&KiHSYX-WQLd>n5-?WYoa%)N#`i5#py zwukJyp9W4H_htM18aP#6HJxhg%lU;lmE|GE$8zY850*ogCuR6#-wkAN&guk^{ffC~ zyUF}d2IusXc5{%#nJ|dv)K;T}=r3 z_3Wkw9K&hos`wlvde-Pqh0{L|)5t5q$NFPGknM{y96!mv3-Mc~D_g&@y!;FO>wSik z*+ZijHGXCJUS>BLKIV`0qTlZ|ekzGK4PBK#)m=3Ch2dC#vVBaJm-XT-S?@C(^XD${ z0~uWzK89nuKRsSb^diewvV9e%%X*Q0A7p$iuai0<;&_w6{lmDR=BuC0yRvbC?ZI-$ z;+(b1c}O;2V|*;HZ2yJz$Czd76ozBJm(A}vK3ERf_{MOIPc|;d=*s5pGJLY{1{ydu z-eh@={ebl%+b?D~*0anW49ECna2k14{$%Sx4IKNQ>c;AW0n^p+*XYOlAAcKYzGJgJ zWb0oIU3Hw1>5rvkKak}~4S%Y9zin6B(R z5=+T&GP*K+vb@W1oOku>kLk+tB+JX3{_MO{Hcqo2Fh1FM#BW)SfB4SC{wx_UWqNUv z%_c^KP`;qP1o{ri86Pd5K!{jr{9^2+>KHjia$oL}_EFU#R3 zS+BCZ?5DE*9F~JQ(9a*k$?`MPWw@VR*Xp0IvAq8p?gYUrx#ZrQ$>22SNqwoca2RpDfGS%0i&*?LgpFDiek z8_4yi!pXh|Fh1r~Hco5g<@lA@LnDU@CtLSu;8gx(^LFNs{aJS2hq>oClC3K=a4KC{ z9LfAjwk~5im_L~xFdX}zY+cLpvLERGPRtU?&hs-qhLg?jnLmb;(dDQ(3R{9fZWPYH5Q{~XV4$$O3z4L6wEb|wZm*HgJQ>6A#Cr<1KQa!7$ z)d$V_A@=8=AAf&Duzq~9ePx!H_4fmt_WnL=gY72UFVVy~`-`e*=8yHldE+1QxH?~x ztzyGP(@M^2*{;BZsO#Ssbz5H275Yst=m;=xn!t znD4S2lJQc*pQ;zxdPb&a*>`N2US#`(EH7i0eb-?AG;r#Kob|%^INoIAC1YXz=|88% zzfjYVCO`9A#;2-T1IO{farqD9jA}R8zOoFTY+PkHjbBOh$8zwKn*KC$@LR?w+oxuG z@GoTZ4dzc{H+3Ln`?4Qs^rtGC;g~MN>0e(mf3i4Yx-743++x3HIb{2t%%29Ins`|b z=8FAFHlAzrr{)*gc{rAX@yYyN1E>1m&*oFvx>h5HiqE0Zj}b{KuL`GsACv8?f4`XZ zEZbMn$gAc*{rf6xUm1Vw2TWHTE;Vt)aI7KOI!eQzYG2uSDZ?j=Hw`|PS2ctBpqcM7 zf3p0d!N>glFt0{lhGTp`JAUY&-?Ker--($T`-^Pfg5!woCW|+ggXzl7N66r0`xXqx zbY<&R=8xr&RdbQw`(EQjnqm~7nA|NV{mlYO6NdHG3p zZ;|!Key^q%jb4~Pj!XUXPL@}8&W-W09QyM;O~r^cIpKVbg! z=L3zus1s{mJYrTmLc~)BV{vm+gzPyo^u( zc+MUn%LmLK>sg&s;*}PMu7xsHKVR8J*^ddX&sDV@MrvJPJ`>Fo@NtQ@9 z&afQp2eLSl@h3YstbtSYA{(a}j_JzqF@G$tY#&bJS1MiEc&Wk1{vw%&Y2Z|Q$i53{ z;8eP@_?7vo%%5d)$l^^#SN6SI1E$ZKjoklia{YAo;1u6y+7 z1D01duao(KY#yPJSJks@{ME>z$}8hf>Iaf}9ovKBQYMFnuF9WG4wjek$?~{HUbe3! zJ~aHPbY<%@8Go|<4b}_uC-Y|wK2^^$|0`rr#Gufh^)UbX&;Q@h!1S-5ni?Azi2sTIn*8-=5%K^2{7vD)eYj~_Ps;U7Q#zoD@EpYoNh3e`WHOBy(T|Bv_}HF(|Lh0QGd@1HZ`fe*&wsNO9nrT>WS9uOkZ1v^Ha{@n ze~*pm-!oBtq5bzbfhznzzxmrYeni#Zxc~k0Uk&`Lfqymde?kL4^2@(}|EneThw_f&me^^-Fs(O?Vj%G-Lt2s`|0OdYxNXgPj7WqLshwd z9V#kv&Ez!yefrPWpT7^e_VWKO`k(o~E5!Ys{&iJOZt~yx3WdK;PdO*E4Sa`$32Qoxaom zXXO1=kd-_#>tAPCBjQtsNojmm)`--EY^nIK>;Kg5`d9XU`zplhap`(=Yz_xt!Fx%D=BGDE!r+%HK0Z zQC_<6*J1E?`gfA|pV|MzxxT@@J>}(^|Ihh6x%q$p2mb0UIk{@xe|r9ZUElTZ;tP8J zeZBsF*Sp{PzpwXC=b0(~ukZIy=|Ar+PwAhM|1;_TZd~>M+kXD@|Nd+K|Him#_xBSn z_|GT(e|B8`U+s^-mjStd`s1Ix;@|T~M;aFYTFm~Nd9~-S=dOC!p3RXb6qz>mjBx7i!LwogsAV(;0|VN#wrd~* zIwr5Eh2o<~q2TP!^DSG@N+plkA0ERfx|7R!?-#Ns0sc(vH<*JvZ^Py# z#++J_C<|#G!eFlt80ut?6o|AjMukBvejub#`iQ~ zXk`ZHY;)lrlT5alQ6(0BRTH_N4&z0UJcmqpFLExwf_%U|+0AJIjI9}r4jM+NmABzI zh333;`5x}Q^QX$^K9E1;LM6v25#RAUTGSkNX-Miq_tWJ#?>-y#j*TwE2DIbptGlow zZ73qvhEe->$kAbrT(L+i-kcF~=Qu7qSo9{B}^D z!nOuew^l>p;t{me?aC|lU3h%v3lXFl$8R6Huyf-|^mLeoG|z0RbgdQEo!ZjlaVjgL z$u8qRiPPWmu=%SS6ZHo1;z&bkZZhL*tJ5MgyAo+rM{~iGI=nlZz+oQwTzGplBMiJa zy2PHrecQ2Yss)md#L4`U(zqxnhmC8vO5O(A8d2-4$O`yO1IWR15MSHQSTA5+pj zXu7wUx^;RyS-M8#PnrywK`*{3ZHEF~LoQmi4o7=D$J!?f=rUq9mO35A`{lQwme&=J zE}OI8{5N7szsWe38qPs;y;ztSf_-l_S)b^^k9XcWx9psaLm{oWWQz-r1O#H|iZ{r8 zI@&BC+IY$1s&^xxoAfb+eY-{k_kJ|Hm(C){Q{UA z^I7<(1fpH_6IsDWdwzPD&b!+mVosk6&|6TB@(rOJJ?1+`pGd*lBtI^EKN6<%b8z`h zARi_>aG1F*Efohd?usto_K)JM8N+Ck+>YKyZp%KT_aIudz(uKi{iIfmt=C|mMthDt z+KH=GqNvn4jVXOoxYi+n{r`;M2N$4eQ7v4SoI+yN5J>rICt5Wcu=mv= zB6rXSF<^jxeQ>uKy^`m=&?s{5tlVof1zPGKA90aZtOd!Qh28ID9UP-!)dE{iW-e{wJ2LjB;^g z#4{K;(vHJUq4B#MP=9r4>Jhy4~9qPED6Uz9K6;@uE>*JLv2 zZ52#5XmHo@7{08NUH9gRN~TDZ0NJtj{|WxEtDGvaWZ?tk-I*r* z7Y}5#LoZ%bsz9*UMXdbZkJZNG{4Trx z%9po=EsvMmiwN0poEW1`GmCFxtByI3sw>0YD2*pu4;3SHqj1P-2nWVRa^B?@d=gr~ z<&|UMA*~&Ur4`^{rXYf z*Ht_XTPqCmD`i1tHdyhtE3b_RrG5D=SXw!=MN~Q`Z?R|DZ3C_?_d@CuWo}<@!@h?G z(Bow=f4sHEmVJS=dz^qW<%ignyc%ga-I;7Vn&U$*;Mdv$?rbU(3jX(G;>VEimt77xZGO-#dG4=>w6W%Wp&PWynv)oQLibqypmYCaFq1^yK>}m zcV6q5En4&}=J+qkuzy~Uep?*b-oFUHX3j*DsWsJS?2wf`{fR@{1t0j>h^5lpZ(Jl# zn^%VD+u4S;Uwqi!tPAyV35_Ywa3MZNR4L}s!lDZD=XBuaW5MAwqPg`HIL>ZAtR8h` z#+%WsHSiL#T8g}Ty#b|#ew+}x4GrH6**8v(rS(%0v8FvnH&0oz8H2IGp)hwZkv%%2z*%#PxF&c%GKZL>qH-!+ zmJH#t@wMo=(TD{ihbFdZOtg2QnR}y1s8(Xb=7l1Az+f7rgm8y>H<7x=mTJn;oZRah znkCIe*vuHNC`zQF{W)>Ta0VuRkH=Wg!PI;D4i6fJP<3oO29^Y))lV;OJ~tPMnFr7% zOM#y)RtlpnePMsanZ0}GGVy>bb2V(Kv_A%!+kG*x)nVDHh^t6BFdu717W2KwWSDNw z#vOAbj`hi53xhXchXOXM*#h;@_i%gag&8kPu}n3QuKj9obDo80RW+IwD=%YX;dBiB zQ->+LoM_q zOKF|Va@H&)-R;H)ixgOM`xcH|)S`BE9s2(u@4xvV(^+1?=l(?;)$=O$Y>46)4KMaO zn!=S;Eot;3kw@E)rf?oE)S5={WrsSHd=C&uGj2j*!5esL^oPP*J${yal(-+JypnFn zQGRL2PM(Ppv-R+uD8tVDyBM>jHv=MPi_l&Dxas+19KEZ}2RGJY!>hiWoi;|CaB0tx zhh|}>_CTHrQ4=Z$idpi#6`Rz`;JChnXjW0gU(Tf%k(N!Ld5Kha2;$^HP8?9&gF9R9 zLwYB&^70!Ly0*oH&!_Nboj?40nWNac89Vna#qN?o=4czjXGtb^n5A*Mw>fnO7gD~r z2YU7RjW&}8^6jE*Mh-uP*;D$npH4eo|LVnd1K;7N=N)M0_Tp|a9a)ifIBDyMX&C{0 z+{T{1HJTWwP$s&)3t{Wv2%b!A5XPt4^I_FEJfG;mY&8w+w(KHicX*DgVkrwQIXD)Vp zJtc4Yg`uj=zy!4G_G zaS(IT^0|F(G?)2|n}XO*Ev72 zzwLMER~yp$$!@V?c@kG_ca+9^2>%?tj;N6HFxDEtW?qGyWPSm2&Vds#PwX`4$ml`e zF?VA#YJKm*gBhdPB{3T%13joVNgt1+jhMQ26!zrhQr=?(@6Tz;jbqz##rjLaQzwgl z!85R@dI>D_&*Mpo9A!OL;D&WI=6*ec-IA|1c6c{zbsQ?1cU*!|7VpvYxC8yT6r-<= zNI{nK2YsW~Dl+BoqAkp|R^s)hzKj!c ztRAh$-=6(B=zcL(kLT0l+9)b5j)c{aFwUMC$mogo)IXuiS9{lp^Mg*{i*79zp4or{ z!-&Y~&!9Ijoa6hw#*8he5ZA39p~l0xMOxQqHPzvQ)+pW`@5U$vQ${U*hmyCJ7<}v_ z<}S%&p8?5Y*smD+xensIaBn&cOQfybQ8-29BEjN{i+ zjWDWtDO+6Ci75|t=;atKsNuw2$6jFk*SV4%Y=E-51=C9W7&7bwv_?O`ZaGWd88;2e zLEGSQ&x;Q3yU_RRd93lW=EG$@Xcj-1%8DIXwsHUtJ(G#SYNud$u@qL)97=7kjbUou zE>E+o5ZM(XJkCqDdXO4j!!o#d!68IEa%7cB3obb^9@C}%-g;RcQ%2`gMCuEb*ILrp zGi8$8PrO3btI+dp@aoOIZ=+>+! zHXf1D`S@A!=z2O|gys?#-ij?N{du#xJKxv^v8P;1o^IvLdq$4zxvn*@8FuFxF2>U( z(JT&Vhm3P4U{)Q5*+HH8D8?V=ZY?=>UJ_M)d2&!|4JPaO^OpZ?IPG36I_93lo8|+k zq?yk?_3~_$*^FL|9(0MlE>!!QaZaH(w{5xy<)xibY9z;)ULSDr&|cKb%@cROJEDDj z4ScpQM%EgCo~jlwZ?zFa&ic^x%TyF@+l7#~YKTq`M`C0L7Q2kZM=7R0IWLG6j}AiH zBUtE8{2(jp-kBA?ZxNi{3f*7(@xsnX_8XK)Y<_@4()^2kZi}^l9^-utc-=QpwyeP& zCnprs{FpKq>}bnz26ou5wM68MHRP|$v#?svh>n(hG3@km@z!IlSUl+_UisOua@JUU zcbW&yXWumA~{fIeb z?-5=%15f8Eq2&V?E-r5mwN(YUp3oVer$o`}k{)X$n-W!a8lAQ~&}+RRr?0d_-@_T4 zazw&h>_d4^PKW7te!%9X65sZ?i~YM7pkI;;=I}T6tAh7Zr-*qKFOd_MPOWQ7SfYMG z{Bk@2)!OzLzeS5DYhTM&MfSa-&t*cCqw+G$*ayhL)$} z#m4qmU?atw=6H<5y`?$fs=-qZ+3dCZo3NO2Pu6>V7DqlT!Ru-j9&2e0`wtFmYB|aI z)k1SPHY`T`?%ybKvg4I!zT!*UfiThN&Ff3Enf6eh3p0P9ZM+HmHCBl7TT+aXSq1N7 zgZXotKBqanh3VEoToKj`pUk^qd*f+ryYmuyUuB5C6Z2=p~ZlCxRUA5`*P9LYA0woX$IzqRk#X& zYLE2dx0>cO2utAzdwKqPZ_cG79c5cij28)=4q@?ESB&u(&65jdRMU>;>vnxOr7>Q( z<<5amO&sTKbl`HQ57=SR4w*$S!~}xiF;>2ns%e2_#RudK(D02Qp$YhhRplQAzPvyk$=*&(?8qkMttlwbHfneSpqr+#KyJY=? zoFMyrR_q(ymFE88R6BeXJ8Z$BTW<<&ol;rc>dUB2Z-@6qM*JQ15J`95U{vuej9;-s zw)A%ZTa@QwMS=_K^%sb9ty5(t-(6^-*IKgOO_=aBh2C+od_Eyh+&gp?yDw_equ2^g zir3+MMUN|DAHrsDN3OM5hx(iVE-un!S36JUX6tf*xjREX5kGIrp-*WRzEpjcFm`!5 zYM$yL`m%D=7wnHI;OA~*WfN*kMZ2s0xb>(q)E91s;cFL;3T;V`hqi40 z{3V=|>(IUNozR~CQQVk+2Mboo7<_Okrreq*i%sn=-re(KRC7lvrN;6_Lkdm%?tz(v zg&b3`#H+rJ+`Dj~p!fX$7y}8SL1?lLjR+zU=x`_&H9&lV!Vbysm(4*Gh5HJtraNHDh^~ z5`38)K=sF6c=)0@?KJ|}?1wxL7f!{Nc}BdwXgf|9R$_WYCeyWh^5W&@ur{z`Mp`t7 zkG+E67BSRLv*rE+seJr4n|H4I(k`?CT}oVKvXWI8@mUWQwR&tmra#Imb1-w&J%oMC zM`-U%4(#c~wpL-h)}+T-QT7}auFKi;pF*arLb=6B%zr8&@E<;}lus&vS^cHDZ}T z1~s|_(QBsU$L9EQ*7g({&j{nq^>c7OyEB5dJ9Az7I#JqS2ji15Tm@e~He3vcr-H|y zmBMk{5ISG^j&-dUqI;wR$DDA6l4&OUT^T3dd~QRf$~<28Fr~|yg?QK8PP`624SBg? zTvDRL8y?L#D%+pmAG(SD!)}XS!+&B%mJW}d2w;iERWZ2RBaz(Amby!aa9ijcq};oL z2TDmepsCA4e^fYQ?>RhSD&zGN*zm-j-BMMlwV)3(yiXw}XFD1Ux^b0~DYTlrXf`YX z>y}$^{>vh+b@`1g2DXfJ+=iK}vqY4Z1JWl2aKS}=CT(_R`AbWd1Zv>;($i>e;L08g zw?gGg2kK`uU`#@LM2K9(XY59gWeuphG@O$bti>wHU;gQH6`JLhqA+x?aC~mVh;vfB z`}zsyT<=P~4f1^PY&FgY2}Epe%d%4vj~CsMYGWe!@Y`y!H#VM*!-wCU}DoBVY{R+H}2D>&520f(;3D~ z$!T0OJA^-;e}VNaX+3(<9lER9@wmi4ym$saIcmsr_31Pk=gLUUwIbK-qv$%iuh`k4 z7XozFWBA$>);xFS>8RFh+PnpO?rg!&`u==;Es%3ua(GKunKya_^V2D9EHK(F8yI{- zT*&OgYeRlv{22wjS(?vbPU{gW{{X)mtfknr6N8Ulz|`fYV!8fIY`)QtImMcMJ@gks z%=+`pt9r>C=&gv(GY*LZ8wYn|$LVG{-R^{{I7gbJu6+n0Q=aA3u z%a$@Zerat?s~78_yx*5s%Y*1J-bwcG)-h~#uNOH(wfSLr8kbJZVCwddIPgx37qX3+ zsqzZ*uTI3cqH@4+6mOfBptElczQ=}BOMfK0Z4ReXsV}=}8Z&8hAcNOrik>qji-gQD zI&9WrPisf6*dHW&F-eOCyF(eSk`31yJ>HjiJMl7+XTNu)cLxK0+M7h*xsIIBs|0cF zGw8c=EE;aU!*Bl-E-R1Z>m6RqeLsjfMML=g;!@-&IItncgXR1C^WhCO?CT}Z&WcSk zmrX~-L!G7Qw``qwuA;z>{d2iyha%Ve?GXyuj(nL{Cp<>9=EFtGG^!elwomQE(16kG zI_WO%z8Z@*i=8<^>j|oQ>_+LdbgpnrWWqoNRx}M@_!b*>t_)`T@va=a>;^iY>x7V$ zV9uC%P4qaF#xCvWA=*rV_Q&kl+s2&cZmC@B-jO3SuE}&fjd92I6P9&6iDw6L_$MQt zvqoHm%4!9Ea__^DCWARpeI0ZUHDDk%A;>+I4@=e~sjLRYWtWB7*)-~E-Nd_gmeo-Pm{b=@`7;*m9IzEpak)bOW)M{+jwSbv9Vh-?1+~@coMM^m za%ZDGBMNo-yvmE+MFX5qHo^9;7cV6Jz^7;VoG0^Ok#or{?sXrpOB>8MHS$i?2rK@NHQ)Uh8`V zuTQT)xk@*#`BWhr^uZDRUk|6lML)K*d5(cyx8TLp!OTvJ;j{2>aFh7Z#<>G%+a`}! zrUbFiX&E0l1yIsRaqjecNHvS1^^ZB?cB|fa`plSmofPpsN(mM_h9fR)G|u(ADSr8_ z#qG5-p#IE^#zRe+EBRqfwGGg*t;Y6s&qdm<0=ih7f_D98{Mj7JVRICj{<n-fpO1*3WqmU`9v#kC$A{8zn2)$ReI5=(+{fz2@syMNoU>nxsAuHI zFZ0UqwaL&qa;zn$?}&l$D8|;|^IMfGsQ-CN9FqCksntvSCej=3&cbZ{}^*|A52x0xooywYb)c?OL=_KDwhkHucE6yDp? zih~|ZL_?b*Iz>0hD+h=Y3`@pRKbgY4>(60T(I6;g4xwZJ-YiOx zNxs1ovB9`C^+&abwKOfq_8H1ET6we@-%E&XPCWLm6~kQ_PO7vz_Z~yE4LN#N68HL@$IkKxIA7UUVK_~8gT^^x={g<#_e%4D{c%5e2aa;XWxu;sHV#JXnhspK5UKS3L%O7PRV{i#3m|MExIiUbK;X zo_8_O)=lQ#Tnny=y#v?gdDK0t%0)g;QQ0$?fh*dfYK|sTyu-Qt6`1Ntmd`nlOO{g1 zt!BvLjz`gIeqVNp5sY8cisqL`BJuN86xld&YOWHo^jQX?53|b1t__Y$ts&~R+!CrV)N%(I`Dt`~l z;ItuIVQtrlGY>0Z^!=ewTxZ3}d*=v`hnw(T=>b~wyoR4m2J}jZq5sPdXh^n`t-cfj z`*}9lw=s(WgTLV2MLQl|1!`rQV?w7exP8fCdP5G^tNC(CxC!pMjpoa~nH*sq#_o?5 zxIKIgRLm9esk|I#vc4jC&M>wMms zIa>r8^ZMv?{@Is+fD|X%9ME94We82(y}1)+{Q5?f^VH3dK13N_Go$g#cqkXU4rapB z7*Y9VKE4+e&~3tN%-oR0j?P-LPVaQF^Vn1*Sr+lGZ(Gjx%VVooZ(Me5=nj9?ew<*o z6#DY!nC0|Zn0332v26{x@zGmZQ$-M`K8#^~O$c?{Ji(CFCY0+MM>7W{PLSqmhDr)A z>wbZfP^4;*KlRJCIqQ6_n7(y7YV>-dg=8-Z_myGXT?&oIR3v)-8bd2w$2E-6gDiBz`Kxg-5Slf}Pt< zSUvG&lhZfZu*E7Ivo%N9MZ2?n>qwMK8U>YC?wH;Et;o_@Ay(WSi(j)PJbcJ)c-`A6 z>%00k4&F`UgraO-ZRUrg6{d9Wz8ACi+QVYMF{hoYL$NFo5qn=?_Z1Vy)m%ZlPt9q& zz6VE+lQH+66%BvXV0HEzoO~_$Ul-CuhsQb$th8ab%Mn}`6fJ4GzKi~z3&f@DS8y9B z>3<&2l+E$9M3>F?@pQ`?5hP(*Iq5D?2w#G8Z}fP3!8k;o=tsF9H)Xx_Kj2-1B?Got zF?W0tU3p7nI)N|c454&5oCA6XvHqJ8{Y<$=j8DfE&n;7_x@T)Dpu z*H_IG;}h@TxtX-Cn#Un(qcn~}wqeyZXDlh1iJ8@x@z!XqOUo-}OrCNE*~4NOQ*{oS z!8M|Baeu~3eG%{7hu!Vh+(=+pRPgc&(WNDz=R%kBY8MBl~tMRS_GX_R_pr#i zKx(ruqn*vMR@;C+4z=Q{pAPp79!=AcVT|uE5WU-};#JHz44@Iyx7xAOyjJ|Zb^?m` z^yIc(50IPo1|P3wvt~+XgzhlJh3PS@G#$-uU*l+5-Y%CUN`6L&fkzO`S{}k zOCT4Y&4>{;w6-0>aXWXSF6o7sw4xI8nml=Isya3oP81&xw&d*x6J)E~zk^o$-6)&* z3}Y+0N}Pa%S?4!lmAVELtB-+iB~7Z`K%NZQfYs4)bPtc`_wR=hcCsfPY3Jhd21DAF zTtP|6XM}5d(eHL!TDLQ0P2biG_Wgq=~W*;IZm_d)PtZ?H0>4{|un9gg=L9 zd5V?01P4|3p-Z#}n)8}xA1zs{3#xn9_9I8*NgN=xl% zUZV$(mTG)Dt_=;hEP>Cq42F$#WaISDcsigOMUUI#gMTPmJJn#yoL{m%D*l|_!-oA0 zcZ*3uBUyGi5^p~A;c%-C?Cqz|l*(lhow%=+ zC68uBvia9%FbP-S@Y%bOI&i0OF7e`k4?FSnNE)}b8;8|vf?$)KjH?A_#nT~{T=T75 zl)v7OXU$_NFZtqOlFoQh>n!R`>d!0dlvpg=jPW|n`CxuDXB*Xv;B*zPvD6jjr#o`& zmqhkb`i7w&lJLDWlK$JHnBTY=2ZMj$!z&LISUI4MSE15L7r);)L+?c)WA61uXT5iL zuN%Z3HSbZi_!2@DYti=WN(|mLQ}~{lfJ?I)usv=xlJ@CvTeF1-(e$Oo)FN639fr#b z2NsHShSqAc;gSii9eIswae)j`-zY?{-7s1{n6pX#PfYFF zf=(+78MIZChmS>xn)?fouX<8^e%piYRzuNp@(TU(7@5>{c7NPsaWwNnL<|16GGkl2; z6-xp|M|&k+Dplpi&-VD_cMGqryEAR0GSAYVN5a9rdr9vxETwGvqvK+AmguN5I;9F9|9}UM`@?vm0SDw2i(Lwmc5{X% zLVibxX3vN7_nk{EcH@=m$QLoqGO`tBvZc*lJ^^SZh#ke!a z-o)vh_hokTM(|*094G9tL26aCm^;Or#gC7Q6EB|1B95;Ru1zU;@@gASE;%4^gF0+) zs)G%qThpwA9X93N5u5j&LWq4Ol-{hu2a^Imy}S&usd=2<<}|iQxbGcPZElk=A*bR3 zzKtvt@oOS*ZM6*!+YMpZ{WDNHYJhJuM&PP>rR=A@5~d&QN0<(zU(ROe{+uAXjZ=o| zYdKyp@4=+ogSoJwH}AFy=9v8^tbZ~SDW?l~-rI}Ea`ULIVSw}J-Kml7&H>7iG&Ng@ zp|w3}x-yPMW$W?7>WFwUbPbB5?_p_t5Z|YoXL1r}BvI&^vlWTw9?0Pw%pEV(0KMDrA_l-mx=VKRzRDZ(4GsnE{8q zmhQFK1I`kEuID@%-o0zFx^9G|VUlnN#SS>AJ{s>YxzJ>N3r?Bco`sUnw)H?B6(lUX z&7heQznjCt)O5~tXo=fq&>ZZ+%^Zsl%$A+n&dr)~_9o|ng#axf2 zV);NbTH8#6dqp`;UK!4`S#q=tZNoKs-8pws5$|s-B#MqlB(Zy2Y##foqY)DU>f7+WfD~wrs%L^4fM{tzR z1hK-S1+HFFxRBx@q^+Chv{4QO*p8rA^Uv3n$2g%d!@@z5p>lQp19nB3| z3A}o3GfI24V$P)IoZmYWE^!%RSk4oqZPP;g$GMzZ(uDOBa;a=E3Az{F38T+NJh|~6 z^7Rj4Gmqm`_zS$1cyxzFWzxDMY5)4UG2Uq@Zg;3cr~V35zr9$@oa4@C$H&XEZidtU z`U#OYxIaq^t8rBOtVox%f|nXKX&a}&w_43<|63hX8Y1|>a47PnK3z8SJ63;DWcx$0 zJS{gB+bwrtQHcYM&4SsttQ4;*N8w0cL7ff{@UVCxeB5F=c}lei+b{$3ZXR}7yJ|Il z={aJ?Y!^maRN~L$&p5F3IBuV-#k&bl@bT1Hlq^i*>ZNU1Z|BVz?R4lFgfaSMFx1Cb zvkl&g>Ybm(R5K-(BzNJ|V=Blvo6Ambui)`~IsWXG#!;_U<7c!f#E0JEXv1Z=%|0S* zzct8|rgf3dc@iyS1e5f5>>XPp5+_Lea(Ycb zX;d6LD((?#5sB=)!jNeyUa+Y9< zoyApqQ0wZ#`7y=3+rA3T)CWtLcS{79crf+F6!?uc=8(dHOdT^9!JG1B_L?O)(Ort; zvVe$*g3*(_0(s9wFXmsgLtDfhdF1QIotjl?tUnjIiz?~<(M*L zYj?n`;Q`cb|JO=vXyGen4jIfbIYmr$RKUCP zIDWY)>ES2#gLb3DGis!A{LLtI``Upiy%Mo!T^8)CA~|b!1;P)G;~-Ppv1Y<*lb80`u6iWd3RQ_;iT7l|26tR*y8y1CFND<;1>V`#oADRg^6vcKxM*}ijEu5F z`qeUw4Ul7k{(7v`P=<+XEEOc3fRAlEo{4w>+3dc&p6e*>t?kOKl>>Qpk2@|?TE}}U z!TnewH6nM3!B14Uciwq8B^qFzPX;e9E^#)rlKf27U-+6k4wY(T*Tg9HF$!l>sU3V~ zNc-gcCyEvOtD&?inmgJj<89b3wDpOgh2;p|8ks_Eg`w=KFbfaYyD~61n~zFu*-V<&v$K7&ukTN$X3qb3t@nR>*O>!};xbJ}>W^ zi~&Onx!XUVkJgW%!yE?|t~PpwPTY7})3` z)=r)84b7kMqk0@$D-EUOaFOqYa7? zdip(Xe_0Bnsllvz6pC${cg$Bi5bl1P6uA7_uvyJ*KH* z^~v)noU}}~Qqrr&oztaH!4BA1`tU%2l~9oI4mbZb_!_thg_g<8TU;sIZnYWZ2REXm zn+NYdj>n_yW+-%4!aet&5NALBd(YpT9lxNudAls{{02n7^yak!U7mT=mN|bGpwd*1 zPX>0ONkTs^moWdQKKT}0qbgt~X%Fre6|;1R1uX_jai)?R2c>^S+q&iA zW7Q|ryqtq}GxfRoWh{plER(HWlgo_Tdi-@GfdTRhFlJ*YcYl70!H$8no;Hk~I{5N{ zx`aFR^Q7siUbKk23dhiwcrh$VazFY%z5tuNBBuMlg7JDkG?*udq_00Q zY`zjz3#9!}lGgUrxreasn#s~{HE1Z@fM>}&gzdE`Y5g8b%Pe{Ep)81!2o)O_45QBE zJJ3`fi^Z{nu`;S0Uz{7E-QEV%KKG&4TV?iK5zltpwqQe19JeO@LHCYTSkTd#m%kWs zr>;HApBzX0@tLqQ+^jW@FM$ zX^+T;0dy_1=KrARyaTCz|0rHogv=s)res7R+~=G`8WcrRl+lop6_G@P_FhUUN>h7% zwfEk8Ylt+p_xL@(|EsHeug~*-&ikC#Ss5K5BF4y!&h&YrNp@e~x_$Ya>8!#HHojDw zco~KdtI@^6krTv2JV-i+v=&;TqpmT+vnPvHPv!e-_YDqb)VOkGKI3e4cxG}}<}Ju# ztIdNst5Jzx*Z6S5xhBlgn}}(j!a4845Js>y)#oW=L~eUdbIZXbIm@Mon_{71v3QVj z1}0}!Y43IciKVT@jR2tePakYQtH*Dj%sBt?I0o7j3Lon=Sllv*bN5S~SNi&cWuENj zVyUnEI3Su%4`)aB5Ki0m2ve>NV)wN{n6lCyeFhKb=b$?5v@eJ1-atHUG=Z7vZwwEK zW0k)Jcf2UhWX&Zi?x)Rm<^)Tf3V<@k;<1+IU{<3n% z`85M+opc%pFXUZ(T&vcNa|j7{u+Xq@QHXE`@`9ogd$==&+=K23wji z;`%hn5OCz()45z|+LWEkPT^L;VceZ-1l#))k#e{NR3>?I@zkMQdN+ncHV@~2x*mMp z+?uMt;^As_5m6ao2tM757q6?rN@YCSMXIvz*(9HzH_nL>lQh|~m5F%QtWgpAU z%;&^asm!Q4iiQ~xOg^Lu`}b++yxyK8W5)Z|dPnoebq79M-;Q^mcH@z@&7_xqGA5a} z!{IJ>u+c*aAL2?-;+;*c`)&DKC7lt!g4q4zNX}^+su(M?a2*c0<4&K^yu8;6alv|g zdq{BEH#;70nyK(g=*8|J(A9M5RJGB)vs{65%aUUej4}{sMbPi5!fs%p2 z^y#X~2U4r*q_bOQQD0zMttsd4+k#`G+wy^CI2V4m=k$TkMWCyKGcp62H6@P5Nk?GQ zKaVPU&dfY|3u&4@bUS($TgO?-4CP|+v$`dxZ`9zVCx21c+XMB3U%~pUCi`s*#_Lms zd}GFDwui+F|CzpQKClN*6fO`k zkDGI+d`>Se*)1*w$eFS-lEW@8!#mp%Od1v{bY0hA={7UmU8l^OTiy6*h&r6Q_^}~! zJ8rIa$HmwGVDGR5Hs8Woe^{9X0ZJ@5>??g10jym$kh744fDgee4zQv6m^@BfcMgwF zbYV%E%(6Jn6!R`#7b~L_9DceBKZbNe<(6bj*w&X{3SAVVRxQS+ZV!dSBTL?2G@KWY zjYE+|bH2#w3{&Ny9Co7aPX+oiw%Q-K*fOJBdwp?zYvVG%ZG4CXZ(W8N~$#ub(EoLA8oSIlzwZ)XKM z&dCz*J#C>~=gmB&{U{tY62?tuqvrHkSdG{z3>0}BxvW@doX_A|eRtMujpSLKzC1iX zhdFMx=(6PuA`1Fo?w5QnH8baU*JOD%1!8cQBTyX>PMh*>Jm7L0eSR8p=15ImygC&I zr>2UswJ)K&Gl*VJFVIQyBig1+LdIX2;c6=Lbz%J((yLtIaoQf9vztOEbToGU(4um$ zXdZoS$gulo#hS?XzB2}!%8c0vrrCeS=v#Nh#Fu$2JWcB8Tk$~lI;^>)&Upj>LZiEp za4CC>kRvI0d$1#ZZXP0c&^o?AEhNj;T^!PCX zxmH7h`uYcIPbj~ zchr4F^ypYdYFtCt9+DNY!-eO{K4R|)xvS90=a}K2<$c|SD=Qyk!Luw}Egs7-9T)zP zo`D`?PvN`acwXIe4^1A^d7NMoe&LtM5;7XZ%bIl$!D`;Jkb5X zXj;{$@V@0YJg*o=9ZPk#-WEfhTDi-Oo(JotJ^0~mA(}m1ASUG>L326J&bp?;!eK-3 zY{+uqTDeTVkAwJKu?r)*yHnYF51M5E#X+51Ha^RsTKHRRoNde&QCDHz(TBg26FF+d zYfL;RcMcvhgT6hHK9gpm*X0NVYgdY*sd;F7_9@zF{6mJ!gW6BdVYh3)eNWuV;fZTt z?9yq9tWl3(f3r`?AU@KLrYNh z>L|YNQ{sSTomnV#-gCht#MYZ#eDAxRzzMe_7;ya`^3ITN_LpG%#}t|bnKP=BJs&7! z2IqA(X4jA7uR|)lAhQ%9*(O|3dmam$3};4e4`wDt(DaltgNMZOscKg?&XJkQV`)sX zw5N4eD%TZ&)#sz(R%FBx*KPQFQv%)A7mHt8CgINQKpySq!vjubSUBYicFT;n<36dA z)K{UU!#d1b{|^&>MB(^42VNR-0H#`YT)QBahy1$m^^^Db;^NA>b3;(5u7VLi61m6J zhLPTXFw&zn+s&_n>jMwo%rD2CynNa#9fn`#LRMRosCN z4o2MJyc-APz3JjI4frpq7Qc$(=%RfUdfG9*iT5fHq@%{#ZH>?jIe}*HO;GgKnen&% znYD5aUOo@zqe1x`+rvfXlhSB+L+0e#tBI7lAtK~0c(Yjk-v099=Mb3@k$1EWGdJUP zO*|Lek^h&0v(R?_ExbK4S6JCOqx4Tt_K4~tuJ+ssjlW~XXP;qgJNvRwZpMK}TlTMpYUJ!{J`6+CB$MZ~hdQE9W8Xt1Vmp zY0dlZ(z$l-YsGiH1Xf5ddR=KdRt)dJ4!*u(u&p5u$$dwqK_8hNY|dXNPeDz7ZdH0M zQQoIJt;Sez;?-DoEbYV&ooqN$w+&|u{)?)^iL|-B1O1-(FuUh)8b!6nx=#zmgdh8b z=3iTmT5=gRE#+<}wE(e>n%rSFfG4F7{DZ|mMBGzD^~cFRLmp3u?VCO%>UuTYYC%indiUyV5I# zCo(=EZ=K|oJ&57$*+ckM&xCQopI~q6$GeYW+2N=+7defk%KdbjWIPlhF)>uzuw95F z)1b7d3rfGKOFiQ#mbLa^+L)0TK2GlQjk-g*pEIL8Y(B?I_nNTF;7s1Ac!6JUhjQ}oWGGch590H#*c+J6K{|%;?U5@Pc2di~WXzfXWnN54 zqlkL(rfWJkgl-Y;l3nsP zGL?JUMoDkRNLoeCfur4MF5W#GJBsg$x%GC8wAzIuvu;DbpfzI#ZbFPL8SD`&+S~Tx zh%e#Hkh#y!4GZw~oijImK8aSz?~pvn3>%$QnfALaf4!Lrl0r!t(#LMzay)g{SaL&FI1Sdc=8MlU>=%&12?2h>rCSB;W0s+;S1XPTUxOjblW`;_ zh(}td@Th^zxP`WpGq$`BO>E7@E;8e~(oU2*eZs|)P1t{xC9f{pg7%bYO`&EM_o zJ|&GW-1g#QwDkW>nB|_N z%>p}1G2jIm+dZDzp--`}z5==3CZhkR<{Z_|p7_&_e_+5~gy zwKYG~gNwWL=RB$XT|Z#XumWYd*It0JQFmaVb{}`!$=B*Q97_({@n7|Lc2A7sfJ?hk zbHx*LcX_~JWeShQh4FdqU|MJDGG|i}dQX{+RUMl1WnL@Z9e)RJyUN}A+d(|#z8Lv0 z?urrV(qDh$1eBWnMtRX+(e<_o#}-XPpQajUdu26Zrh9O}ejhAbH;(oi&k!;CqP&McAshuaZ)xYE!c+dIC+vKKv=t>}pU*OV9~Gd~-n3b5?@HI&=wqJzx_ zJn@}~u+ufT7#+{nwsGY9Y^Lmef!m5jqOuG;{(KOpre4BFHFIuUN(SuCW5Cw&TrWSr zK93vGJM6ER5U?20)tb^T@Kfe7FXLD7Yb=pjkzrRHVIUbP+s3D(sa-2h`WS%GcFMeE zSR)Jv74U+lrrgb~RCGUO!kTB#6wwEo(_wf%-6dW@ZV>`gm!w8|Pq80D9^cA|>f-uNBlYuoq*2`{-71d|aC)XVDc^z3<^bs3ZwBX7K8CcZ*AnH1N z5u?{j&!*!HRJJhV;zhHtTWuiM@A!-qt>>7dHyhSR{CVE91s2-G(AHmzUe20aWMRwd zIR~+Jrys|~4dEN%P1mK{(EglcrfKH!y42h}v%9e*!Bw(=H0h}H1fO*W@sV;AJ$L!> zciSI|nnkgaarqb)m68{!bxHWVFUNK3zDV~B5uK~9VM&Q6vre4Dbp0~4c_+Q43zFns zH-%$Zi+|@!Q6>Ga+grBgu$eEg#-u+NA+fasX zl>7Co2l3l>6r1nu%fRM)pg(N~ruV2pf<*#`*dBsoAL-Am_vP@Y=f1C{zp2h4iM_4b z(K|nqOOIT`>ivyCt9#-{yLNce(uy0>jp%i8x;U9#ihq0EG3WSS;gOKb<8nW-M*7!x zWP~ZM%Und`)uV8czOVq_;Y|^ zBii;TVA9W64wlc}%FPxm3DZNC%=3;amEME$>-aV29V*f#)7f(};y<Xuv+Oq=vdx`uG%RyKK4?S8YxQ-fMiUpyp6AOKjNPBcO~9>0G0Y(zU6NW zU}6%>8iy&Oer^XE>m@SQ`R+l?oCPe!raul}^&i3cZiMeMql;^`)1(em+6XusTs zk9o;dTW!tXefMC6oCW7h>%}fz3}~n^A8+H_xxM8q1itCd`1!9E-_3bC#THF`i*2z48>2b`Peg!3Y>)oNDsd?iudRa0Z6*HIIR=OJlA|~wn$4qz)9_oJaBWkBc0)bj zckVij=3hb2%@dI@XgOSW`l2ByO1|zzyl!;o{S_bZc*Zp}i=Bz6(!pHkpUFSJ3^Dvn zGwN>{AvVo%BmYjon$DR~lW-6w! zr@Y(uVz1^tG?3nW?aUIm4j)KGwTbvR<3Ege7luLO2XgO}Xoh$j@cVzYQ0?E5i!&W~ zaJ$UHwAm|j9{Ma)5h7nK5bkG!S@!EM%D?_Vn9kED2!op;@?JJqwjLoA_>9 zBl#hxqBz<|YG>~}SXO?{H@?RZ8jbMh^8dU=f0N( zQ~EwQF7*5)L*ctAbu~tcmCao7Vbg!;^SA*9CL6@B_6|IO4On%06d#XGlV?I6+ivf} z68$ft$#M^tzkZ0~7ZEgj6d}DnR@ApxffaUx7;L0Ux2eW7zIRG|-mS`z4P}Tf31RDh z1^5)>NaOxeM^^Dc=*&@!jFY+f`-W_?s0Di;%j0cRS01pCS+`x}M1v?g`kCO^KwG+P zYQZ(@B-bU_9_`fBSf48QD9@yppnXWxeskfO;E!TLdNll{-gfSkAMW^{^R-rQ$HvO% zxYO>o%#sqr#Zl}Z)tOtXJK*ByoA7D=7>3=hVp3TedripY38f3Z_P_Gz8Q+cDzxRUP za3iKaRp51$mxHEEB^Q=LhEBi3@SqR?@kZce| zC}V2B;n=717<-@uBk%OU&xAJIaYVBERt#e6mzyzp`Y~kJIWiRknAU!=7-f~rH76=D zcAyU)Ta4jpujvT8*(jOShWK=JC4!yFjekzy#k>dTH?S#p;;HBrpU&xe`Skwt9lDK; zShwUin)Q1wj;C!CE6w{tDb|$hVg@#t#tEY6Fgu9b?N^Br;3-hjWo{zL32b8w*a!zJ{9PDCU8vif1y z@+)H5;Nv2wej83^=Ch#AL)>oAf^jFAe~XM3O|%vv-z$q&TV{yyfuAs?+MUO@N3;LE z>rm`=#;WXHmw|DBH&!=o^zAeMiy%A`A$CQz)Ls)oq3);)wceg-I z{`Xpmaid*nn&`_*F@>}bYJ+q83TSgMSZtPjd)=4j)IOWX4b3*fF|N0`Jhm-QM_Kah zWOW`r-H+Gi6(hD^AGSRp|Gk>a==VpRSr)qd6av)q(;_m~+^Ra_VLm3yIfQ+mQa z7htvNFrIem1*1u;vHom4Uq$Coc~>SE{gC&ctOUj$ZO<=v7og#t2G>|lz~)WDZA{BpS!M@bmFAC|PgI&No z>?;{!&rAQpDOup7vzb`YTaz|PD==@40iFdVFaX0CZKlIim25inS%NhdE_}CTBd}mA zx*>%*-dWtTR)fE*euxe7JgF()3-z6O9AzBG=u0bMx^sr;Jl&EbTY~BThBCQ#6()Uf zVd01XUYOgCwVSqM=AU3Ldm#DFGN1M)(2nI(cVfb_X8inftC%>oGoL>&;sePNyZNaF zdbB(sdb(HPPgGO--(QQ=on2_x*c??!Gw?IHrF?x0_W$I=PT@Kn8Pbzo7ev!yqueR` zbVc~c*%-I5B}+UK=~UZ_v$v|CZOTkkNoMca@%bG1xv%W=xQdX49}uJaR5&hnV}3@4 zXt7Z~)@;@y#iS6ucDLgiS4WONn8DV6?U-tn!kW+i^w#`@3!esa)JrGs*c!<%I`>80 zt|pAVqsNC*5Bl<>LHxRLPq;acpz4@&7?Y+W8C)xI@?1Eld-h|j-2FcrVa~PXZtOdE zq|6&dbNgZs+;MHoTXvoK&3U%mZ@R%L=_m#z+wo!NV;G>+5w1@zDhj4+ibvxlOLkUM z-Yb&K`M_6~1Z&YXMJiZK&{*hY1ln7CW=nk5r^ zT-Gav{)#~6ryNDf^JIp(NS17+D%Vuhp|F! z0oL`a7cT$Wvf@k}pTCObn88~xta%(Bd5+`&@AVkHPK~y0d$Y*@jyUA+A5ahlk(ZiQ4QZ zig~RH`SH2n>1QkP)XbZWDI4)LS09hRXo;y~X5yEv^j>LCM?>2NC^n>1`@}7DEE~xF zPg-K6{LUVJY{NBUKZ}2!t#EX}Vw?>q7qM?gaZOPRrf-qYvAP6aR@x=L+#Sk=((itJ zVZNNH(lPK`9(CjyqtWfCP|3fFhF|AJqx>$Zwut5BgM+YZcQdGZpT(i2o=m9q7p-@E zMOyJ$B;MPCb$Kzo+TuS<(VK~uZB`;IpbK|iFBVhf^C|sb9$QV(XHmvF+@K3?U(A@|EwTho_P!i`$#@d172gdUx@{o4+1> zW?%z@l!1KzUl~-M3=-aLx^nC`UuqqUqUM{XTz)c_h5Gjpb1|1kT8x3$>@F;_Ov3yb z)p+@M6pQM+aLeLJqWRH&JZ=%d^q8)k7WhNBZLJq+a^5n%+JQ49ujRqm0{RpN;G$$m zw_lhnf;Xihr0ErO`7ZCF!U?Zj>u{q|kDpazxN1)qamW0DIAdZ@gJD*vmi`+P#R#@B zY0a8-OEGEKP#T;YLys?UJl9O#N0owjf4H)o%{y|+${w7eA$jLNjN#iDkKFVa8sF&x z3rauhwpvWQmQT+dOWM4tQEZ$N$ZL-dVaq!Uz8(D+MSOt8k-rt|qatbE>>tKE>+wSR zX7QyU9ol6fl2J2NJhialw9SQF-_C$tj>WNmvL$o9ZwTXaZ!qkk+{Nu4uUHX29wWLo zqvhL`FfWbamNS9k{I7k8jZn}^^B}x_gyLw{9mR>-UOe1RGNsavDJ)j^k{MVv>Yub= zP!id8fEP2j`s2s6Dv6Z@Fh)#X3*&oB;P2IvvySM(dYm)6Z25%UI~;ts=pDl03JqFWCer%2 z8?0o4VTfA}%hOu$&SXDE$n4aH*dA;gqE4Imc=qWa^^5QMaR2&M(XdjRU31;}ah@&B zR}bZpUZ)gZpEqIPzp)(nNSp81`Eri!7o-)<$A?{=VSZbWePrHXM7H#LNWCsY_7y~4 zEQ6`kyy|YYV6noUn+p3etuct^*$SRdYKg$PN_;l&GR_UsF+_2YZ)`Ll+i_ zm_2?x{2C=)S%jDJ-1(B$-6u)vlH2OFAxSvgqN5HKttE(( zy1DZC#b`FAKaU2DXM4v;(Wdi!I4zfc(}IEgEx&YMGOMt0oCch0oA8y+1T5TPCfsW0 zBIsN-O4^2SVND3bHc4iLd=^Z#&f*)H>B~;|iedkBrB|RAPneZp*0~bVs$>QxZs{r( zU6~?QB?fcn>o+3oKpk!-r*p2hF2e)+aMR{Tu=?pB^{`%0>so=670t0JWdX)jO5gV% z9iPidC(z{DW!#@U7InL2-@xiZzIm`hQM^P4E%zmeyRl!fQu_An`Wf*0@|j{qNn1vz zj^@8%mtp&_jR-G&=sR>u4(|*J;locxqUY9;eEei8ma4_@Tj66|dVdS;Yqz4|_hc-a zq>6x-jfkt%v6bKWWcv{_(_dw15ye;qhExLyCLXPysRL0Y= z%T#E-UxxjiYejL%9C1r>;>@J>rrrG}+|up&+NCXfylh2-gbu7KDT1b-3SUI2QT@hz zgnn@1r5>Ab@oOvI4-!y%lF02UvsX=s`xcYu=)D&Fy_>N8vkZI-j%L8Tq3CqK4(r#gl$~wvjCSZN z(vm-6`>~PKYws)l6>7|t?2LdZs=O$(4~{clA?9y?sI?Tb2gZ_Jj&|h4bMufnRB}O- z6R3X8l{qp;H>PDF9=)H5nXM&1X>|`q?e>F)@|Mq&|2r+L9C6H{rAJi}02V z1<$*kq>;idpenOA&fCMEa>&8IaG2pS)1`ua%GMq~a-J2p4>I7xO~W|4VkthpmYk7*?rgtvsCX!QSEifI!)^Tl zPP*_OtxpcXPx;*Gl#|T1p^j)`t;{BMtI;qqn3raFa@OH6T1(yFWxeFy?Ytl^|I(*n zb{IasxCL9YJ>s6S{92sh7-EgEu2V`8?cg&m&=CE>{F*^Q@&NmM9zZx%L{2tC);-b^%Oj z{TQd8l;fH7Y*jt%%=2YZ=a(F1v!w56a@&thw#D7$w2Rx*Rz zj)Aj@HM_YEXVZi7XHNbH6WP!8$KH@$X0_1kTnnEW1y~qh#g?U?agiEOJ7XHF;7JXm!W~WEJ;U9cQ zBy33)!Q(Y~>)>6)A2ATYDJO9q&McE%LeDK5@LsZy<{0GDW2_%HUWtlr;@vv_eC&5qDOW)9dR^xg04G^l6O!eW%k(QIhN}mkAk19d0->=1`!#T{V z%VEnI%@j)&cciu%jQAzV$Zq}y$__(#v$gD#JgAG*!Vai7(Uu+qWY5mIcv}0-MyzDP zcntW0%}alXj^=W98(W01n&+q(Y{Lz|m!XYqHusE4#e%b*&vzap=a%&JXZT(h!nXbkFmpIXdMATAz>LmX3XCAorgypFYFmcHL+`bcOh{ zX&@tC??BD_ccO<{lw<*_N^Sc&7PL4cwS38%HSWeZr8uURORnfVS01U4l3bYMi23gj zPWB#)gKBlyxZw!+$zL+B?Zvfk8f;#<2`kDcz{*5s@IJO>R?=pq9bFBTwVOq@O_J}I zj2x!AXfoRH1k?stNng7z_cm+8T@gPKaxQ`gCD*rJEb-@kFs( z?U(5IxI(P(EMV*l$-n5Ez==QPY?@-idt=ST2E*2@cp1qBjps$P5w{d`J9)6`dJ&v2 zOQ!dd-y(AS0B&EF%Z^YJKJ$mm`FsvsKg$_#t`?6h8pnN!KCD+ChrfH|U3S|ZJdL)% zd82`RZQF$BrAMfx>l*Q*pEmQ)>N0Oul;~NT!()enxNzrSEEcw0dc}uh`{?qa>?*i_ zLH3z-k=YT+lxtpIgNS3vi1kM8Kg1` zZ~F`orUm=Oh&8P+ecm6K?ir0k|39)aW{v8Y8d5Q_hg5 zp5=3RwK4O8Wp-RA5JN1R@#KV3?5y#}p-0!y>Qa@smwf{L4lPH8x+YCyo1rF%AAE6;9PvG&+$pta`-&LGs69sN6>CnGbGbbX z>D@bp1-nP{r_VfbZbF6d(7c3=a@Kr#awP@?U4_;84=9)27iR{K;kPTh#e=#QC{J|Y zPW5Ht%J4wRTfKs97w#!~wRU6U{$PIhYs#+%vZt@P{6008jJ8EjeUme;z#(if64h@Z ztjvJhD=QS;bqv_)v%oN$DD2CcfTQ!Wc(#58@-mI+uWZ156SraI)Z;i*zZ^SPW>T+9 zGA7>`%_ADNa^JT?Z2x)#Qlf!xaFQq+(}AHPkf(F9d48SjvHAdvaJ~bpzs_8C%L&D5 zlX0Vy?DM<)7;jg~`>)|n)Xg!L{Zq0B?O=cO^^`qb?G0JJy(zB`JTBUVnJM(=jKrT8 zk_olppCVy!EbrcL!}-!-Xw@kjLDzfIElye9K_=kC?tF$n&0?~jG3PBgq$vGu zPqnhP{OKgw5K(r_|M@|7^Ss4NZ6!MIa%cPaQJf|Ha(5OQ@Z|yN`&$+cC$H~tGLp>J zE60)j+L1F|PKu>!^Rcp8GJxMTWn!<{h}znj57j*hf3Y{8zGgckB{ zUf5CUXPPgebZ;7hI-C%rUudJZc!|>2WB-kTdz1`PP&bOIpHxy8*q@PKu`|8n9>cT6igiFynb6j>mh@WI-Zx8vF8T zU4LdI>(IsBQbbPbPp47M_^!)v8lU=y-M_jp%p{1xp7({B^iWT4vH<#ne+c)gCRlAZ zj1P}Iz#WS_B699A@okR+cc%D=o%-Fm!hM|Nq9ya;LzzL)`3C!gT{(T4kx-I+$#YBE zGk=sTx3;yVx@7&Y{nrc=3;M9%uN9bna6T4xtjB=(=_qV*8LOTy6ZR&}Id;@VT(cg} zeMjb)~KLAvpaT!W+{2vSN-mFSi&hUj1jrwF^rXzpt9GGB+OA%g1rQ+gS9u zB7gt#1E~3o9P`GH?)S&B&E4Z@e%OhfE@vo)^qY$fbDOjLV<&!QBCuOeahvY<)ZXH(USbPas*p1-YD}1Sxnq)Cig7aJU#0-93;=fIe)6$H|Mf* zoi#Px+7Y8hi0!Kb(0F_a6p~*SIcy#5R*y$;y#aT>uw~;HT`FA(RU|?Bfwz9gpT$*B zydT0)^%%qy<O7v&R^D8YVT?lOu{`u}3zApaFut3< zdM(F94aM3m7 zf#G?)xn#NGeyTsrZ;g~)va67LZUwXqx^r7EZQeQ@$$Q6xnJ#l7K?iJErreA9mw&>~ zRF!&5&SFPW8Vq;((t2|MxW*D>9wXmn;w={xa#T0g1|3g^G_NZv7%xq5#)$w5$M<+X8iH7v%wmoHFJ zJCZN-QW!l;Re?e=} z!AWv)pB%)@XK9@4HCOSZ)h0N0-zpAI4q%(ZMTlwMlx1#n6gRcA6{`j=!~z=^1~W-! zcE_@MvA5!~vl-7w9)ZF4G_+Zz#4DC#SQeq+B==c`t;}e9~V*LA~_5mefVB0fxCMTW2W5w8V^2!iw<%p z)l`=o;^k-NH%Isu8^JQJldoS>EqcmMgFkUkaHL%i2HvQ|+0jN!oKoi7R(4C3YmA`7 z7b7$YR%2R&5*_UC3xhK;Xt*IW_z#hJ~x_9UF?Yddh$pM2go?#;@d>74a022Cfo{Qnun<{xudencyj-~*aPwxYdc*Omt4 z)2!EY#l)4iV(h$f+?Xl(nOkBdzd4X)L5E z?6mp%02ckQVb{Q=C{W5^;@7P(y4sEPj)Az>pv55zlc_)65eY8$74~Y0aGxN1isz?_ zn_Y9c?7Rid!+*i=>PHxLE5oLTDG2PWi9N?;WHW3Ef;@)^2oy|F;h~+Ws99WRL8-rq|Hph~P+@L7a0v ziN-Fm+>_Xz&2(i3bBivml#ifQ`e<(Y)}KMrbKWt@j-&gX6rU?E;qNCQ`;opVoKz-b zwof1%*2>NtX%%SrY{Fq7l8JA53ZolZ@pQZkZPLAANq<~AM?0WH~ctc9F8y7PG2djv*Wuw&3^ z{Qi@{OX{=mX68~1oH+wa^Dkg}@J0kYaN;ScPc6y1AnrL2g>dnp;c6`g_n3t2Z%Hh5 zlXLcm3;5C3jGHgI<3@5f;UJmdz8c!xBXt0q>UwB)aAo_!tvGME84NBh#RZRBV#w8I zJiSvgR^_agDCUUODmTT<=ewlF4y0ZPlKU2a4m~nK(JgK-Dz)}vw>XQnNpf%es|PmJ zNPRC{ekbmJMM-o5d*!{tn-fNCYW55vdb(`)wi%~5j-b8g2%grSg$C~w4j)@2rW@$- z$D;lCuW1~u4h%+li!WH;B!O$>ndLg&j;o9Y!7-yFPrvHP+3O3rruZQ=GwNV61F}Q- z5ZZf~!uIqfXmxoaLWg}2`E|0xF5(WJ*ent2&0C3UOop4}td9)rD_-A~`NP}Ycr;)c z95+g)hEFwo4F|IK&fe5sIUf~rC!RigJPW0kTqL>8QCeN-Io%EI#;y@v)uZ7u(wHx< z)!-`}*#BFZ^qGCf^^0*F_tBRn#_LdLR*$P1>Rdi$JBANv#qzB!MPIXa$h)W^wd)*q z_#8{CPVqFa>&}w-$Br8 zXC?|IgC(=#v$$%h#Q@Jpt_hQSdC7SC^23!GyX7uF{fEN1UldDA=b=>U##f4cg@)xp z6qLWkK+}F~=8?ryD(z_<<-oh{Z!p0x2(icfc*5h27Hz`)7g{tdOW@FR(u=u$iQN4dG4K9h7ETFa zVjjPJ?E?-lGF z?LnvZrI5bR|kZm++B~EMv-(I6V93wp`1FTv#4LI!pR3i71vD0QfsW#-P`q+^IQv_ zva*4h^mrIU$9mfe;E%b-&&szLhExB^W9oeMU3i#Cx zlHC9wVc}ke5ZTGMT;&il+bQwq^9uBTY`_^2Q^kJ~*3?R02iqI}Ywt^-dR)7BqmUso zrIaEhv*yzO-9Lm3X)uMPl!!#lp(y6c?vE#A+&|9js18Q$mFy8@Q&O@xEEL$_P#!DZ84nh-1MWL)1wjdJeM zoglHVvAQho*L_Pa14Zq;T3e`v(?N3j+z~^TT~pS)5Z^g6?$D%K4H0=)C(aC*(})LC zX!dq-@8xk*I{siQ-Ak#B&>LM5A#(i0Wj&E}X9MLl>V|l~k`(M{ghEcsXh@&}8gf}V z<5Gtnl>SOZW8R2+u3a%g^m&cFC-$Qv`{Ck@mU z*A10kH-}L6*^c!1Xf}1LZ-&KAf0y&Gv|#RF71;FaQ- z=|JJhC#iSy9`wFkecYTqn-E)<*TZk;r*&g_$0Q=+o9MGENGV8IJXR(Z=F9VfG6OOY_CX!F7>Q-Vl$9^}t9kFFOCOB8K>v z!o=X#czAmx3ci}cIM{-Qx9)(LbtSR7Q#c;Z5xrDz8{y;IX%sLrpW2klqJ^Eu)AsY* zmGosDFePUgx`}W4d}QBSI-W|rywsfO%VO+2GoFYemy*X zTNQ;)d*VZNU)mK^f=4pTsQu7}sA+bYKKoZgV)J;|Zs>qsE``y^xV-4E5KL=xt6&0&ewPJ< zDduE6UW*>^gDRL)*qUBQ|FiB=D_@taN zzo68pUG8L!_@QFY@$+-~ zw9%Z7%%4g*mS?EsyPae*tp>iXx=W9}#TooL$0>H8#Pnap9+Ro~Cj3(AbFU43Ts}>lhdWF6OkA*UUoku$G@Uj_ zTp{nRP4Oh*ciQFF0P6$A`Ov@)*zwH`J&e{;LJ?PVPA`j)I3G-j?u+ngnYtM>k`b2I z6!t!gb(jAr1CR6xO7hWEx_B!Y4#gYc#O!s-h~xDrx#ANF=w1aQ%Zan3O%lnbbF8TI zeu)yAR>NnP`83mPKh;mqqXyzEM}nx~^<>aJ3f`7V*S^)lQ0EyGW@z9Z=%1) z`)5i}(nzcq>)_Z11&aNkwm2SDMAQ{LPBrg~^JRsTQEf*Eg`Fw_gV;nG+v<)kYC~x( zFPpAJJ%|xC?K_aor;>P5x-=?<#o@epXWZPRYx zt44XjTAbG~?68i4_NR;1p>32tdm52%(PYt!&zy#qv?hDQZ{%(*=8*g9(6?RpXieF# zv}5x#YUTP|w>HrkCtti!_J<#&Q5E`7%3BZW-=Z((F0M#V<}Rl2)xH=v<~zlXIiL&Z zHI{0$DvjzNTH=GZIBT)Lu;*Kc8B|@I?KztMzMC zXW54iXBR`xr^&QxSuVABZB5@-i(KePm|46vjrFWS1?~nES!X&G%y7r0T}9~8g%-Gf zcde+wvPKDyM53(WedfGT%@Cm zZK#q!P3OzUp~0#VD9A62qdl#ueb2iT^Gw{`{+LNUn&c?f_Fky_ygUth zHbm5Yt3@N4E>!Z5_rl~2w`s$v08)~H>ZglQ^koC2in`v{7xhw1o{AcDZAa4~M{m^M z6iaQZc1MqMxyrafI^fE6IwilF7 zZWEQS%O;3=zrQN>XNsZp&0Dl)@S&G2cE6;b&&h%sZujhi+6z|PDG1-lwy8RssvsPXgU zMmP9v4X1f;#GZpuA2iwXT=6ZPOM#n))4AXZc@whwkT9Njhc)eOAGt8!Bvw@lznF@nL1QP(C(fz z(ZvEC|1gEk>H0K6RH_D!gL(sT+Klr4VL&3G_G}uziY4YbO4d&NW!Y$j-l(D{u znlqj5p0&s5=MmU6=mdG37rpT|i26&#tYMyEhsRCK5wSS{X_n&d+O$qsnfsj_T^pmD zr59SPD@xTbom3WlK2Dn-e5V65g3xqe9p$jDK3u=-BPDt;j@7D*4Pg((K3FSU>{=X! zVmeWlou@c2cS^}VY^1al-^n{a6|ocR0*JlYwKoP*ps1TN+qEO^WccD(vq(BF4m}SH zD2YSeYE#k1X-ZUGW9pFentIsg(Z^a{5gBHSPNAJJX6JIHZPOZfKd&mDhX1bkJt7<` zw3|K~tfk}67m<77Guq#;65LW~|2t75>DxC739?1+tOf|H z-VK}c#?j{(ZydX4fs{(sXtwSml^!PU?)S4t@QP35SR@SlMBRs%75<>$^{w&cwGYOR zx<`$xl*QvI;$B_aGsW9-G}&zRhv7|MT<#<8FTPqvX>C_2cA^eP)z?utw)hLhRvw9W z8y?X0mnW1C=@pQ;#R~fdR>sG@2g$2QA&fgWnlkNeal+^{6*}4rM{5cno!5$@A0AQ4 z)e(IhYGu)@`=ZX_@K2sTJ7sgw?Pm6aNvj}?Z<&CJ39IOCQ)7DQaF_h9_eQBj=fxe;nslm1 zH>@`-M2i?Vq zP_sVn)VI|=?p7aj9H!Iw-Z3=XxjZaxm&dddp=fZGaAc0?t7$L#SMD`LeLHV_8)A{oODnr`pu>d`G~qCTdo!)4$nI6-%((j!Kg5<6Ya|I5qqRBXz7z*XvfKwWF0vOU7i>sWkoEW zJiMt4ORfNu?82Da@s+3@UzyezyVJqbi7 z^JHP%%4mXFVy|-kq!DN=&Y>91uPx3%+lxKG<;t7+e(*lFo7`jjQoF{YFWHy-w6s*F zZq^`U@>|`LuBT_wxF=SW^-0wK9A}L(T}3ZjPf`xGnxP~)8Q`v|=+EfskI<!p?Sp>iVZ7ql1am<04_Wva|NbcUAn8i8qjPEh^S-f-|J3M0|GHl&j; zdXz9j|N7j*$)JlRwisv8 z+%?TmWKcMKBE|k`h1pahB^gii*Xbff&C9Rl#hs7mC&*#uH^r%W7+PF!0$bOn__jEk z?wJQ*O@-!Ya?6i=_a0JS+gal6UL#~30%mz6iZfpUI5(pNPIYh>=iR2zqTszsf=zcA z)$2@#4^Hczjtij?!<{ia3#e_<5%;R4Q0x~&8h6D)chWnMvRg+|v!%5#V8c!FCu0na zd`br0{V?@r0uElS3|gD4Ox<4w57!0Z57z@UZqOmBWl$d8&0Of~phV0Rb0ysD0*gQ| zbotGYEb63D$Qn;9xe$fvo~P)Hy$QVY3*l|554>$+ME(ED-`9}#+;mZH4y%bv%S656%@Zj)XdHb#=81{zN5JSyH@xdU6ca?x50|s`QKF3l zo`$Zc(2&`p=5rfVuF@0{W<4>k-vRo)Y$vz}dcxqh*<{_$9vj>;D56?@Jb2aves;q| zUB}Hd`|@dOBuj3Wi#(8^tBiqUFyQ^)hjv~+!6IWJ}Q3Kn^T@eGfcBLRd(3K zVyUe=#*A=A)%K!?>6HmoWBd)8KEDgn2xfgPQ|IJ$G9_JgwqM)2qj0LOW3}W~(tBUsndF2ZbnmDvH`YnU$30m1mIe)L^V` z;z+g5?4nOfO^iFch~C{3wU{?_!MPkeC8uix1c*CG<%adc_ZiXX_G=xw=mr$u5QN(= ztBV?t(WqGC7sV`e5?yY(m-4rX`-dOIna9}|sB4ejSk)twQksal6C#yaRx_1JPCIG& zj#g-Nc_3ML*P&t87&2S4kzR_r0zS6a>CKLoI4JJXEs7TXV@1!D-AzPa-1oImxkn0J zyeDd0Y}luy56Y&_mU*J)j5~anS67akI^boc-b!lRak{kB0LdTM&>vgwD77t$)4BQm zU_9RiU7F0KB`>@%`k}ac{Jj|rMeSVA7iDnqmcP#Wjt6B#1|wVCiOan-jsm~c(Cz4L zgWZ$b6Oqoi9a^5MA3dP#t!R(t;;h8xf>;c`UmCOWex--^=Th^eBeX330CibkSL~}6 zr5oZq$U)orx;9%x4V?vq!CU$%HSk&K8GVaJn)C!Y6r+mXg;2gl8!~$DN~v+%mBD74 z$;rkIom@&#t(M&|ARrJi#^Jag-WV3sF466+wGgv6%d?W>8pR^`Bh{PJ2p7wiM`v*^ zGirP((&@yScG+QSR@MZYa{i#v(-p)_ct~L{yr@h$56b(zNl~_xN2U2yDfx|*#l;c8sO2(Th!XN85~-OTw2v^PR-i?Sd&Z z!NeQS`{|GxGLddHv!^xWrZgDvk?yt-cM>YK#^{UUe8VlV4>>v(x?{lJh}zwGf^3yN0y%R93Cn9A%qUY z%AMsYK2(Q&4FTUy7d&&Etgvf!C_EqRRL*2aP}z(1@&4W-va9owuC4Bi6d!9mzj{~M z-l;MyL9{$Cjgn&GQT(6=#zZO|9uF41ye=w5n>nCR zSsPJ@uRPq%c2bhjD>|H+P8}l}((VjXnk#B%3~zgr-b}NF=>l_nD;$H+I7eLE8$tEZ zjTRSo!_Y&9NbLQX_Fo&06QWMjg29Hc_&5|E8RzN4n>?k}p-6F0Zy1K!deQU)6X@~S zlXTl+H`Ns9-s|;=$LQ9jXvoeWq(GgZT}hb9^~`-04JC+q+YfxB{him(DPG8jKPK&&lrDdgbJ< zOl4gDdvbmeAZnWz$5X>SN}gXK?E6+6Q_6iM!;F4d8lFS@z2d2pe|H=gXM3|PqR4a8 zc-`g;qQ>dOXSDF?NHn!?iYn(sjo*e15p6aMS85(5bJ6!TyJS}sEC@u0Bu^NxeN4go zUnpDWET)g09Z*%5LHEZe)AdeAXwNtU^0<;i&%_<7-QqmzxZ+-LJUC04Gi4DC5%rTy z{ie{_yACLOts^}gEoyYH4#LY3Ls0MJ3fiQz!#0bVboSmL)O)8x`>`u1&h!#xh1Y?R zt2y$;y!d6F6ZR;L;A=kwb$#cNLuOfwo->CY=iZ=A^&XK=rV}3azDZ@DHuOARcQ3UT zIXu(!rSiPX6?*ohG!ka_r=#NB?LB3Uu7qPC-kX1+XMW-w?wp!1ZGD9b|DH;2UxjZ) ziGF;#Jf+j;cv0)HG*%RyL;FmsC`LI!*pfa&cjsIyls7lQ5Z8RA#L#teQ2|Be>$C1o6?=@!_ay^Z7r9KUM6NJcu1)7HG?wV?Vvre zZBh5qBwD>79!-bYk_mNz(yloo-yNeg;q#f9ixmGI#uRqtmTutIp*UTkKXyBLV!_G1 zRIZ&k=XbUPGQ_#n7taeLDph>r3@D2EX2a-xjdQer;6CbXm8eu*)SOIa&Q{`AWYCG8 zH4#wU4{Kf;;Z@C(;@tmw>TEPgS2v@lf`KtgJ5d{?<%K}}=K4yvtbypq4ZkIqLj67tNcFVCA3cWPaR#uk=0wGBmoJ_e ziu$k<$CLB%D-`A56E2lRt=l&BaMHzuI(4zZmhAS(KNO4TA=OdD?m8X#G>Pi>7!2>! z04lS@367pN_>>cd>FatcA18!hS4uE03@(Xvwk>qWg6g2n&^0vu%TlW0c$Z~XaN7LF+?g-yX7Cl*e zInnfn;=Zv_Pvj@3Dl_cNabMhnH;o^L%+W3gn2?Olb|Z0T=@nZ0)fr>gR=|i{8_?J! zYVxE!S@#m>sl@HN^TV&}T;0UEuz}_#%4C?)qMy1u+K*nO_?!^6ko!BqZ2bzlFt0Y|e9og@21dyB>xbR<*XyP&6@4Fz*TeXS zMQBIgr{s2}F=_>b(%IAV>HDp&BQai_Q~My!gMY7hn|hzm zQM!g2(3~gll~Lo3VANdn@x0y?cUN?Q*WF%pYF|^VK0Qa-e73D}CTJwW{om2Kj2ycC zRn)z_dW*LIS`R0di~Y(nLos{16+Q7!q1t;UD&JauqpS_yNGcSG*70uexEx2iHbkK# z7m;W0D9kmlf?Fr2D)w$i>C8({(f_Rt_FN00T<765e)U#S8&LEs5qa&5Jq(sD~+&Cq1LxV(-A^>2>8kp&b~(@gZK6ul8g z+hOe7H_XgK>HJ;W*Yjg3vJ%0x!#0mR; zzDm>b8zLyQF$Pwu2iHR<=A6*^q7TVFFQ)#qy zp%q4W{7T_VKGRjR(Mrcj4)|seuk76@>NPgV)_rT3O4I(ROra}U;_mJw968VcNuoY8 zni-LWxSud9Z7^1i9EnVEX6VMRCCSsPAIi2UiMnUIqQt7on6bPHUZN^ZIZzlk9=c=Q zPSGnTd#ciKdnpWU(-z|@UZlt8t6<>!Ug&&jBZVgo!?4vZ$Y}15vkQID$S03VU2#Ok zhnCdY-%U5KaTSy_c2oSt8SI=9HL&Js2r87bLE8Kt;ylj+`Xkv0jYThFk0n;fJy!_5 zp5IdLjIWE&r)?2u*bZ&2d|*5*hx)ZN!oUt8C@8s+`VY-g_K3TR_V-3mdbX&UJhl)e z_Pjx%I$&w^Fg(pUP8BTsqRq@)`mlLCxt_kR zWOoijYcap~99x_U`roGxc1_`C*930HHz{v$QJnbl=iamN3R8@9bnp~5!}sGeV3J)Ob`CLo8Cs?tB|6A8Uvw1>MnR z_+9FnwU8D)83N}^(bPlKE-mq(EQ*aW!>J`n*kU2>ab63B*}|vv=KEthHT(iSit0^c z#@|p1n}pNom08rTN_+SR48!z!lgQ7kFdf~VD%Pt{$T-Fjzdmb)C$;)upV%*)Std@@ zDKNqCt7ZtOA-)4H^+Of=&x+Fa2^B1;MPI_JDxWs>#*j;|m8zeIW83wmx|+ReV`x_o z+T+%k))x`|;%sxNr+*JrjI*Q8PRZytz64V1Wza9h&(OxPIn;F6HG=4inYY&oO&^H< zu4%uL`Ek*oM(k}|zQ0TK94>=rjY1Idy)4297ss)ZT@~k>Uz8Ms3$*M)38a-gO_OtG z=zPQ;XR#AyDC^=7YG^o$+U;6I?Y>vRhv)qiml-9g%gE=-r*4A?jVq(pnM9a9D~@DQ z^U1iksJqinoI}e3qQ@avwLA z7Ju!G$fs+mXuYn;h)%|*1CuEG)Dk+eDFSCNZPiuF_)N#`v&g9VMsZ)MBl35aL24mS zWR6)a&eEBnMeXjG^!PEgz26z@BaTwHtb5Asef6l{n|SQm)(ItUh(3Vgn@IH+jc__E zoxa@&Q0{EdA^XZMGJjD6c54TcYsOs)ez;Veso73F4;4jvyE0hXwFS%@wm@j1H0tuL z0gYNT6mDmV;=-LU{Boxw9bVa;?%FoODc=y-?-b{790L$mtT_($v4C!4Em}F}iEdM2 z6Xc2BVQ&vvQJHOG4_xdQ7Pl9@bHseAtjBOTw-(HcPB_`s8^!v0!?$b&-M#moG&L(l_j-66M3f6A_tdde^l2B}ad!g@ zE#*!NX15^wwqNOSehihdjH0uL3SoTB>PY(>j_K`0-@eGUX!hn7y{uK6k}^u-UDcIz zEXEFd%DPeW=g}x0Y=r2{v&!?s{lvE(W2|~A?zM|`@#T`GU?sjmxsRzP&Y<_f;^`A9 z>{=mgJXsTS>a>N6embM>=a=1Ou*E<=SL}!$RY&02kfP!|(teuY7=~724>W$kSjERc)Z9IEjC5TZ!fIuE znC-O0Hn%3YX**GOJpCx0&K7gJ)>fjYjEK(`qITP!Ksfp@rfS8#vGi?CEZ7@Hx{B2? z*}V{w#N5kb(+%O}O;sjr>_e;W zED__s72ZYW()@`I7}qP64y1IW%vZsRx5j6F}lekC(@HtQKzWbJ$mb3 zjTTznQxbZYgU^>*w02@7=9jLG4yDBRmS+KYKfOFv4J=FHh6yP6xQI5)`Q&lqktgcv zgihAHq2xRl%Gxqo^jHXkTgeVsy4)6hHs7Zl(UrD$=e<-v#~MN2^>KWuGaTDnBPn7E zz3Q1PYj`4X&;KKS3{tuS8frDfJ#^dVv-}o3VtZ#gL z#9xqZ?*9!Zlf%*Nzv24CMnpva_0PuX~g9MJ>vcI@X?5lxee^%X7 zeK64Qr@}cq{kNVOj{Q${15JD|U4C-buVvT!u&B^7uCQrd$8ZTsSe0+%oWp>B{mm%fXmkB>H1}Fg}^QjE~`* zRrb{f1E$7s9uhxbeEbU+N&L$AQ`M)5BgVo{E_&_8e&r^i%UD=vH$6ZW4f*qIruHp{n_!u)mdL2XUwwrkm^t3_p5t)94%z&G^~ZLTBa7@=hlGoS|*lx0Mi{V&*t`fgzIHv2U1{}vZ%d0=H zF+L65ALiA>FWZgfaFF-`(`Eh1#u<(N)bYq&WnXzow%Ww{o@k1uBY@Ns6#80yIstlj%RvP_j z?565ZHh=uhEJAvSq`?Zvn0-04#uobvo(5QIL0TN z|FNDm@=D@}sd0QbN%&*EFkRU^k>zE-@=yigcw>AFC$k&N!JNwM!FpkN-6e8p;8cH+ ztwWhE%PX5dGd||;r`G|hVqa{mMmR4-K3eZ$CSKmgONCe=1qIys96_)&Y!<n|><+SCWl zJ`&3-i*x2rV-LwZjPn}nS+@RQ?)gcUhnOzI$^44nYUoP*h56%O$ox-6SLRpj2aL~A zZ$4l++4_zBp5@R#-(`I661vPE+xKVVT>rX^{a!Y2VmX*U**ML5X1~(E-e)*jTrxF= z({Eqavn(!Ua{O#Qm96t+`1FsLtY?}2Wcb`9^9_b$e6A8W)*nCVr_1=%>7-^{;CN$u z$oOMFV0(C|X-a+2tmBzK**u%&)xb&OgW;I&KiHSYX-WQLd>n5-?WYoa%)N#`i5#py zwukJyp9W4H_htM18aP#6HJxhg%lU;lmE|GE$8zY850*ogCuR6#-wkAN&guk^{ffC~ zyUF}d2IusXc5{%#nJ|dv)K;T}=r3 z_3Wkw9K&hos`wlvde-Pqh0{L|)5t5q$NFPGknM{y96!mv3-Mc~D_g&@y!;FO>wSik z*+ZijHGXCJUS>BLKIV`0qTlZ|ekzGK4PBK#)m=3Ch2dC#vVBaJm-XT-S?@C(^XD${ z0~uWzK89nuKRsSb^diewvV9e%%X*Q0A7p$iuai0<;&_w6{lmDR=BuC0yRvbC?ZI-$ z;+(b1c}O;2V|*;HZ2yJz$Czd76ozBJm(A}vK3ERf_{MOIPc|;d=*s5pGJLY{1{ydu z-eh@={ebl%+b?D~*0anW49ECna2k14{$%Sx4IKNQ>c;AW0n^p+*XYOlAAcKYzGJgJ zWb0oIU3Hw1>5rvkKak}~4S%Y9zin6B(R z5=+T&GP*K+vb@W1oOku>kLk+tB+JX3{_MO{Hcqo2Fh1FM#BW)SfB4SC{wx_UWqNUv z%_c^KP`;qP1o{ri86Pd5K!{jr{9^2+>KHjia$oL}_EFU#R3 zS+BCZ?5DE*9F~JQ(9a*k$?`MPWw@VR*Xp0IvAq8p?gYUrx#ZrQ$>22SNqwoca2RpDfGS%0i&*?LgpFDiek z8_4yi!pXh|Fh1r~Hco5g<@lA@LnDU@CtLSu;8gx(^LFNs{aJS2hq>oClC3K=a4KC{ z9LfAjwk~5im_L~xFdX}zY+cLpvLERGPRtU?&hs-qhLg?jnLmb;(dDQ(3R{9fZWPYH5Q{~XV4$$O3z4L6wEb|wZm*HgJQ>6A#Cr<1KQa!7$ z)d$V_A@=8=AAf&Duzq~9ePx!H_4fmt_WnL=gY72UFVVy~`-`e*=8yHldE+1QxH?~x ztzyGP(@M^2*{;BZsO#Ssbz5H275Yst=m;=xn!t znD4S2lJQc*pQ;zxdPb&a*>`N2US#`(EH7i0eb-?AG;r#Kob|%^INoIAC1YXz=|88% zzfjYVCO`9A#;2-T1IO{farqD9jA}R8zOoFTY+PkHjbBOh$8zwKn*KC$@LR?w+oxuG z@GoTZ4dzc{H+3Ln`?4Qs^rtGC;g~MN>0e(mf3i4Yx-743++x3HIb{2t%%29Ins`|b z=8FAFHlAzrr{)*gc{rAX@yYyN1E>1m&*oFvx>h5HiqE0Zj}b{KuL`GsACv8?f4`XZ zEZbMn$gAc*{rf6xUm1Vw2TWHTE;Vt)aI7KOI!eQzYG2uSDZ?j=Hw`|PS2ctBpqcM7 zf3p0d!N>glFt0{lhGTp`JAUY&-?Ker--($T`-^Pfg5!woCW|+ggXzl7N66r0`xXqx zbY<&R=8xr&RdbQw`(EQjnqm~7nA|NV{mlYO6NdHG3p zZ;|!Key^q%jb4~Pj!XUXPL@}8&W-W09QyM;O~r^cIpKVbg! z=L3zus1s{mJYrTmLc~)BV{vm+gzPyo^u( zc+MUn%LmLK>sg&s;*}PMu7xsHKVR8J*^ddX&sDV@MrvJPJ`>Fo@NtQ@9 z&afQp2eLSl@h3YstbtSYA{(a}j_JzqF@G$tY#&bJS1MiEc&Wk1{vw%&Y2Z|Q$i53{ z;8eP@_?7vo%%5d)$l^^#SN6SI1E$ZKjoklia{YAo;1u6y+7 z1D01duao(KY#yPJSJks@{ME>z$}8hf>Iaf}9ovKBQYMFnuF9WG4wjek$?~{HUbe3! zJ~aHPbY<%@8Go|<4b}_uC-Y|wK2^^$|0`rr#Gufh^)UbX&;Q@h!1S-5ni?Azi2sTIn*8-=5%K^2{7vD)eYj~_Ps;U7Q#zoD@EpYoNh3e`WHOBy(T|Bv_}HF(|Lh0QGd@1HZ`fe*&wsNO9nrT>WS9uOkZ1v^Ha{@n ze~*pm-!oBtq5bzbfhznzzxmrYeni#Zxc~k0Uk&`Lfqymde?kL4^2@(}|E Step: @@ -30,10 +30,14 @@ def create_step(step_config: StepConfig) -> Step: return IngestStep(step_config) elif step_config.step_name == IngestMultiFieldStep.label: return IngestMultiFieldStep(step_config) + elif step_config.step_name == IngestNestedFieldStep.label: + return IngestNestedFieldStep(step_config) elif step_config.step_name == QueryStep.label: return QueryStep(step_config) elif step_config.step_name == QueryWithFilterStep.label: return QueryWithFilterStep(step_config) + elif step_config.step_name == QueryNestedFieldStep.label: + return QueryNestedFieldStep(step_config) elif step_config.step_name == ForceMergeStep.label: return ForceMergeStep(step_config) elif step_config.step_name == ClearCacheStep.label: diff --git a/benchmarks/perf-tool/okpt/test/steps/steps.py b/benchmarks/perf-tool/okpt/test/steps/steps.py index cc1773330..75404e354 100644 --- a/benchmarks/perf-tool/okpt/test/steps/steps.py +++ b/benchmarks/perf-tool/okpt/test/steps/steps.py @@ -313,7 +313,6 @@ def action(doc_id): for i in range(0, self.doc_count, self.bulk_size): partition = self.dataset.read(self.bulk_size) self._handle_data_bulk(partition, action, i) - self.dataset.reset() return {} @@ -359,6 +358,7 @@ def __init__(self, step_config: StepConfig): step_config.config, {}, []) self.partition_attr = self.attributes_dataset.read(self.doc_count) + self.action_buffer = None def _handle_data_bulk(self, partition, action, i): if partition is None: @@ -409,6 +409,118 @@ def bulk_transform_with_attributes(self, partition: np.ndarray, partition_attr, return actions +class IngestNestedFieldStep(BaseIngestStep): + """See base class.""" + + label = 'ingest_nested_field' + + def __init__(self, step_config: StepConfig): + super().__init__(step_config) + + dataset_path = parse_string_param('dataset_path', step_config.config, + {}, None) + + self.attributes_dataset_name = parse_string_param('attributes_dataset_name', + step_config.config, {}, None) + + self.attributes_dataset = parse_dataset('hdf5', dataset_path, + Context.CUSTOM, self.attributes_dataset_name) + + self.attribute_spec = parse_list_param('attribute_spec', + step_config.config, {}, []) + + self.partition_attr = self.attributes_dataset.read(self.doc_count) + + if self.dataset.size() != self.doc_count: + raise ValueError("custom doc_count is not supported for nested field") + self.action_buffer = None + self.action_parent_id = None + self.count = 0 + + def _handle_data_bulk(self, partition, action, i): + if partition is None: + return + body = self.bulk_transform_with_nested(partition, self.partition_attr, self.field_name, + action, i, self.attribute_spec) + if len(body) > 0: + bulk_index(self.opensearch, self.index_name, body) + + def bulk_transform_with_nested(self, partition: np.ndarray, partition_attr, field_name: str, + action, offset: int, attributes_def) -> List[Dict[str, Any]]: + """Partitions and transforms a list of vectors into OpenSearch's bulk + injection format. + Args: + partition: An array of vectors to transform. + partition_attr: dictionary of additional data to transform + field_name: field name for action + action: Bulk API action. + offset: to start counting from + attributes_def: definition of additional doc fields + Returns: + An array of transformed vectors in bulk format. + """ + # offset is index of start row. We need number of parent doc - 1. + # The number of parent document can be calculated by using partition_attr data. + # We need to keep the last parent doc aside so that additional data can be added later. + parent_id_idx = next((index for (index, d) in enumerate(attributes_def) if d.get('name') == 'parent_id'), None) + if parent_id_idx is None: + raise ValueError("parent_id should be provided as attribute spec") + if attributes_def[parent_id_idx]['type'] != 'int': + raise ValueError("parent_id should be int type") + + first_index = offset + last_index = offset + len(partition) - 1 + num_of_actions = int(partition_attr[last_index][parent_id_idx].decode()) - int(partition_attr[first_index][parent_id_idx].decode()) + if self.action_buffer is None: + self.action_buffer = {"nested_field": []} + self.action_parent_id = int(partition_attr[first_index][parent_id_idx].decode()) + + actions = [] + _ = [ + actions.extend([action(i + self.action_parent_id), None]) + for i in range(num_of_actions) + ] + + idx = 1 + part_list = partition.tolist() + for i in range(len(partition)): + self.count += 1 + nested = {field_name: part_list[i]} + attr_idx = i + offset + attr_def_idx = 0 + current_parent_id = None + for attribute in attributes_def: + attr_def_name = attribute['name'] + attr_def_type = attribute['type'] + if attr_def_name == "parent_id": + current_parent_id = int(partition_attr[attr_idx][attr_def_idx].decode()) + attr_def_idx += 1 + continue + + if attr_def_type == 'str': + val = partition_attr[attr_idx][attr_def_idx].decode() + if val != 'None': + nested[attr_def_name] = val + elif attr_def_type == 'int': + val = int(partition_attr[attr_idx][attr_def_idx].decode()) + nested[attr_def_name] = val + attr_def_idx += 1 + + if self.action_parent_id == current_parent_id: + self.action_buffer["nested_field"].append(nested) + else: + actions.extend([action(self.action_parent_id), self.action_buffer]) + self.action_buffer = {"nested_field": []} + self.action_buffer["nested_field"].append(nested) + self.action_parent_id = current_parent_id + idx += 2 + + if self.count == self.doc_count: + actions.extend([action(self.action_parent_id), self.action_buffer]) + + return actions + + class BaseQueryStep(OpenSearchStep): """See base class.""" @@ -449,7 +561,7 @@ def _action(self): break query_responses.append( query_index(self.opensearch, self.index_name, - self.get_body(query[0]) , [self.field_name])) + self.get_body(query[0]) , self.get_exclude_fields())) results['took'] = [ float(query_response['took']) for query_response in query_responses @@ -486,6 +598,8 @@ def _get_measures(self) -> List[str]: def get_body(self, vec): pass + def get_exclude_fields(self): + return [self.field_name] class QueryStep(BaseQueryStep): """See base class.""" @@ -591,6 +705,43 @@ def get_body(self, vec): else: raise ConfigurationError('Not supported filter type {}'.format(self.filter_type)) +class QueryNestedFieldStep(BaseQueryStep): + """See base class.""" + + label = 'query_nested_field' + + def __init__(self, step_config: StepConfig): + super().__init__(step_config) + + neighbors_dataset = parse_string_param('neighbors_dataset', + step_config.config, {}, None) + + self.neighbors = parse_dataset(self.neighbors_format, self.neighbors_path, + Context.CUSTOM, neighbors_dataset) + + self.implicit_config = step_config.implicit_config + + def get_body(self, vec): + return { + 'size': self.k, + 'query': { + 'nested': { + 'path': 'nested_field', + 'query': { + 'knn': { + 'nested_field.' + self.field_name: { + 'vector': vec, + 'k': self.k + } + } + } + } + } + } + + def get_exclude_fields(self): + return ['nested_field.' + self.field_name] + class GetStatsStep(OpenSearchStep): """See base class.""" diff --git a/benchmarks/perf-tool/release-configs/faiss-hnsw/nested/simple/index.json b/benchmarks/perf-tool/release-configs/faiss-hnsw/nested/simple/index.json new file mode 100644 index 000000000..a982afc81 --- /dev/null +++ b/benchmarks/perf-tool/release-configs/faiss-hnsw/nested/simple/index.json @@ -0,0 +1,32 @@ +{ + "settings": { + "index": { + "knn": true, + "number_of_shards": 24, + "number_of_replicas": 1, + "knn.algo_param.ef_search": 100 + } + }, + "mappings": { + "properties": { + "nested_field": { + "type": "nested", + "properties": { + "target_field": { + "type": "knn_vector", + "dimension": 128, + "method": { + "name": "hnsw", + "space_type": "l2", + "engine": "faiss", + "parameters": { + "ef_construction": 256, + "m": 16 + } + } + } + } + } + } + } +} diff --git a/benchmarks/perf-tool/release-configs/faiss-hnsw/nested/simple/simple-nested-test.yml b/benchmarks/perf-tool/release-configs/faiss-hnsw/nested/simple/simple-nested-test.yml new file mode 100644 index 000000000..151b2014d --- /dev/null +++ b/benchmarks/perf-tool/release-configs/faiss-hnsw/nested/simple/simple-nested-test.yml @@ -0,0 +1,37 @@ +endpoint: [ENDPOINT] +port: [PORT] +test_name: "Faiss HNSW Nested Field Test" +test_id: "Faiss HNSW Nested Field Test" +num_runs: 3 +show_runs: false +steps: + - name: delete_index + index_name: target_index + - name: create_index + index_name: target_index + index_spec: release-configs/faiss-hnsw/nested/simple/index.json + - name: ingest_nested_field + index_name: target_index + field_name: target_field + dataset_format: hdf5 + dataset_path: dataset/sift-128-euclidean-nested.hdf5 + attributes_dataset_name: attributes + attribute_spec: [ { name: 'color', type: 'str' }, { name: 'taste', type: 'str' }, { name: 'age', type: 'int' }, { name: 'parent_id', type: 'int'} ] + - name: refresh_index + index_name: target_index + - name: force_merge + index_name: target_index + max_num_segments: 1 + - name: warmup_operation + index_name: target_index + - name: query_nested_field + k: 100 + r: 1 + calculate_recall: true + index_name: target_index + field_name: target_field + dataset_format: hdf5 + dataset_path: dataset/sift-128-euclidean-nested.hdf5 + neighbors_format: hdf5 + neighbors_path: dataset/sift-128-euclidean-nested.hdf5 + neighbors_dataset: neighbour_nested \ No newline at end of file diff --git a/benchmarks/perf-tool/release-configs/lucene-hnsw/nested/simple/index.json b/benchmarks/perf-tool/release-configs/lucene-hnsw/nested/simple/index.json new file mode 100644 index 000000000..8dc749c39 --- /dev/null +++ b/benchmarks/perf-tool/release-configs/lucene-hnsw/nested/simple/index.json @@ -0,0 +1,31 @@ +{ + "settings": { + "index": { + "knn": true, + "number_of_shards": 24, + "number_of_replicas": 1 + } + }, + "mappings": { + "properties": { + "nested_field": { + "type": "nested", + "properties": { + "target_field": { + "type": "knn_vector", + "dimension": 128, + "method": { + "name": "hnsw", + "space_type": "l2", + "engine": "lucene", + "parameters": { + "ef_construction": 256, + "m": 16 + } + } + } + } + } + } + } +} diff --git a/benchmarks/perf-tool/release-configs/lucene-hnsw/nested/simple/simple-nested-test.yml b/benchmarks/perf-tool/release-configs/lucene-hnsw/nested/simple/simple-nested-test.yml new file mode 100644 index 000000000..cf1e4edc4 --- /dev/null +++ b/benchmarks/perf-tool/release-configs/lucene-hnsw/nested/simple/simple-nested-test.yml @@ -0,0 +1,37 @@ +endpoint: [ENDPOINT] +port: [PORT] +test_name: "Lucene HNSW Nested Field Test" +test_id: "Lucene HNSW Nested Field Test" +num_runs: 3 +show_runs: false +steps: + - name: delete_index + index_name: target_index + - name: create_index + index_name: target_index + index_spec: release-configs/faiss-hnsw/nested/simple/index.json + - name: ingest_nested_field + index_name: target_index + field_name: target_field + dataset_format: hdf5 + dataset_path: dataset/sift-128-euclidean-nested.hdf5 + attributes_dataset_name: attributes + attribute_spec: [ { name: 'color', type: 'str' }, { name: 'taste', type: 'str' }, { name: 'age', type: 'int' }, { name: 'parent_id', type: 'int'} ] + - name: refresh_index + index_name: target_index + - name: force_merge + index_name: target_index + max_num_segments: 1 + - name: warmup_operation + index_name: target_index + - name: query_nested_field + k: 100 + r: 1 + calculate_recall: true + index_name: target_index + field_name: target_field + dataset_format: hdf5 + dataset_path: dataset/sift-128-euclidean-nested.hdf5 + neighbors_format: hdf5 + neighbors_path: dataset/sift-128-euclidean-nested.hdf5 + neighbors_dataset: neighbour_nested \ No newline at end of file diff --git a/jni/CMakeLists.txt b/jni/CMakeLists.txt index 29a844ee0..04dca217c 100644 --- a/jni/CMakeLists.txt +++ b/jni/CMakeLists.txt @@ -79,7 +79,7 @@ list(APPEND TARGET_LIBS ${TARGET_LIB_COMMON}) # ---------------------------------- NMSLIB ---------------------------------- if (${CONFIG_NMSLIB} STREQUAL ON OR ${CONFIG_ALL} STREQUAL ON OR ${CONFIG_TEST} STREQUAL ON) # Check if nmslib exists - find_path(NMS_REPO_DIR NAMES similarity_search PATHS ${CMAKE_CURRENT_SOURCE_DIR}/external/nmslib) + find_path(NMS_REPO_DIR NAMES similarity_search PATHS ${CMAKE_CURRENT_SOURCE_DIR}/external/nmslib NO_DEFAULT_PATH) # If not, pull the updated submodule if (NOT EXISTS ${NMS_REPO_DIR}) @@ -134,7 +134,7 @@ if (${CONFIG_FAISS} STREQUAL ON OR ${CONFIG_ALL} STREQUAL ON OR ${CONFIG_TEST} S find_package(LAPACK REQUIRED) # Check if faiss exists - find_path(FAISS_REPO_DIR NAMES faiss PATHS ${CMAKE_CURRENT_SOURCE_DIR}/external/faiss) + find_path(FAISS_REPO_DIR NAMES faiss PATHS ${CMAKE_CURRENT_SOURCE_DIR}/external/faiss NO_DEFAULT_PATH) # If not, pull the updated submodule if (NOT EXISTS ${FAISS_REPO_DIR}) @@ -142,13 +142,37 @@ if (${CONFIG_FAISS} STREQUAL ON OR ${CONFIG_ALL} STREQUAL ON OR ${CONFIG_TEST} S execute_process(COMMAND git submodule update --init -- external/faiss WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}) endif () + # Check if patch exist, this is to skip git apply during CI build. See CI.yml with ubuntu. + find_path(PATCH_FILE NAMES 0001-Custom-patch-to-support-multi-vector.patch PATHS ${CMAKE_CURRENT_SOURCE_DIR}/patches/faiss NO_DEFAULT_PATH) + + # If it exists, apply patches + if (EXISTS ${PATCH_FILE}) + message(STATUS "Applying custom patches.") + execute_process(COMMAND git apply --ignore-space-change --ignore-whitespace --3way ${CMAKE_CURRENT_SOURCE_DIR}/patches/faiss/0001-Custom-patch-to-support-multi-vector.patch WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/external/faiss ERROR_VARIABLE ERROR_MSG RESULT_VARIABLE RESULT_CODE) + if(RESULT_CODE) + message(FATAL_ERROR "Failed to apply patch:\n${ERROR_MSG}") + endif() + endif() + set(FAISS_ENABLE_GPU OFF) set(FAISS_ENABLE_PYTHON OFF) add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/external/faiss EXCLUDE_FROM_ALL) - add_library(${TARGET_LIB_FAISS} SHARED ${CMAKE_CURRENT_SOURCE_DIR}/src/org_opensearch_knn_jni_FaissService.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/faiss_wrapper.cpp) + add_library( + ${TARGET_LIB_FAISS} SHARED + ${CMAKE_CURRENT_SOURCE_DIR}/src/org_opensearch_knn_jni_FaissService.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/src/faiss_wrapper.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/src/knn_extension/faiss/utils/BitSet.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/src/knn_extension/faiss/MultiVectorResultCollector.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/src/knn_extension/faiss/MultiVectorResultCollectorFactory.cpp) target_link_libraries(${TARGET_LIB_FAISS} faiss ${TARGET_LIB_COMMON} OpenMP::OpenMP_CXX) - target_include_directories(${TARGET_LIB_FAISS} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include $ENV{JAVA_HOME}/include $ENV{JAVA_HOME}/include/${JVM_OS_TYPE} ${CMAKE_CURRENT_SOURCE_DIR}/external/faiss) + target_include_directories(${TARGET_LIB_FAISS} PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR}/include + ${CMAKE_CURRENT_SOURCE_DIR}/include/knn_extension/faiss + ${CMAKE_CURRENT_SOURCE_DIR}/include/knn_extension/faiss/utils + $ENV{JAVA_HOME}/include + $ENV{JAVA_HOME}/include/${JVM_OS_TYPE} + ${CMAKE_CURRENT_SOURCE_DIR}/external/faiss) set_target_properties(${TARGET_LIB_FAISS} PROPERTIES SUFFIX ${LIB_EXT}) set_target_properties(${TARGET_LIB_FAISS} PROPERTIES POSITION_INDEPENDENT_CODE ON) @@ -186,7 +210,12 @@ if ("${WIN32}" STREQUAL "") jni_test tests/faiss_wrapper_test.cpp tests/nmslib_wrapper_test.cpp - tests/test_util.cpp) + tests/test_util.cpp + tests/knn_extension/faiss/utils/BitSetTest.cpp + tests/knn_extension/faiss/utils/HeapTest.cpp + tests/knn_extension/faiss/MultiVectorResultCollectorTest.cpp + tests/knn_extension/faiss/MultiVectorResultCollectorFactoryTest.cpp + ) target_link_libraries( jni_test diff --git a/jni/include/faiss_wrapper.h b/jni/include/faiss_wrapper.h index 284214631..078526000 100644 --- a/jni/include/faiss_wrapper.h +++ b/jni/include/faiss_wrapper.h @@ -13,7 +13,6 @@ #define OPENSEARCH_KNN_FAISS_WRAPPER_H #include "jni_util.h" - #include namespace knn_jni { @@ -38,13 +37,13 @@ namespace knn_jni { // // Return an array of KNNQueryResults jobjectArray QueryIndex(knn_jni::JNIUtilInterface * jniUtil, JNIEnv * env, jlong indexPointerJ, - jfloatArray queryVectorJ, jint kJ); + jfloatArray queryVectorJ, jint kJ, jintArray parentIdsJ); // Execute a query against the index located in memory at indexPointerJ along with Filters // // Return an array of KNNQueryResults jobjectArray QueryIndex_WithFilter(knn_jni::JNIUtilInterface * jniUtil, JNIEnv * env, jlong indexPointerJ, - jfloatArray queryVectorJ, jint kJ, jintArray filterIdsJ); + jfloatArray queryVectorJ, jint kJ, jintArray filterIdsJ, jintArray parentIdsJ); // Free the index located in memory at indexPointerJ void Free(jlong indexPointer); diff --git a/jni/include/knn_extension/faiss/MultiVectorResultCollector.h b/jni/include/knn_extension/faiss/MultiVectorResultCollector.h new file mode 100644 index 000000000..a11a278d9 --- /dev/null +++ b/jni/include/knn_extension/faiss/MultiVectorResultCollector.h @@ -0,0 +1,69 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include +#include +#include "knn_extension/faiss/utils/BitSet.h" +#include + +namespace os_faiss { + +using idx_t = faiss::idx_t; +/** + * Implementation of ResultCollector to support multi vector + * + * Only supports HNSW algorithm + * + * Example: + * When there is two lucene document with two nested fields, the parent_bit_set value of 100100 is provided where + * parent doc ids are 2, and 5. Doc id for nested fields of parent document 2 are 0, and 1. Doc id for nested fields + * of parent document 5 are 3, and 4. For faiss, only nested fields are stored. Therefore corresponding doc ids for + * nested fields 0, 1, 3, 4 is 0, 1, 2, 3 in faiss. This mapping data is stored in id_map parameter. + * + * When collect method is called + * 1. It switches from faiss id to lucene id and look for its parent id. + * 2. See if the parent id already exist in heap using either parent_id_to_id or parent_id_to_index. + * 3. If it does not exist, add the parent id and distance value in the heap(bh_ids, bh_val) and update parent_id_to_id, and parent_id_to_index. + * 4. If it does exist, update the distance value(bh_val), parent_id_to_id, and parent_id_to_index. + * + * When post_process method is called + * 1. Convert lucene parent ID to faiss doc ID using parent_id_to_id + */ +struct MultiVectorResultCollector:faiss::ResultCollector { + // BitSet of lucene parent doc ID + const BitSet* parent_bit_set; + + // Mapping data from Faiss doc ID to Lucene doc ID + const std::vector* id_map; + + // Lucene parent doc ID to to Faiss doc ID + // Lucene parent doc ID to index in heap(bh_val, bh_ids) + std::unordered_map parent_id_to_id; + std::unordered_map parent_id_to_index; + MultiVectorResultCollector(const BitSet* parent_bit_set, const std::vector* id_map); + + /** + * + * @param k max size of bh_val, and bh_ids + * @param nres number of results in bh_val, and bh_ids + * @param bh_val binary heap storing values (For this case distance from query to result) + * @param bh_ids binary heap storing document IDs + * @param val a new value to add in bh_val + * @param ids a new doc id to add in bh_ids + */ + void collect( + int k, + int& nres, + float* bh_val, + int64_t* bh_ids, + float val, + int64_t ids) override; + void post_process(int64_t nres, int64_t* bh_ids) override; +}; + +} // namespace os_faiss + diff --git a/jni/include/knn_extension/faiss/MultiVectorResultCollectorFactory.h b/jni/include/knn_extension/faiss/MultiVectorResultCollectorFactory.h new file mode 100644 index 000000000..45c0338b3 --- /dev/null +++ b/jni/include/knn_extension/faiss/MultiVectorResultCollectorFactory.h @@ -0,0 +1,26 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include +#include "knn_extension/faiss/utils/BitSet.h" + +namespace os_faiss { +/** + * Create MultiVectorResultCollector for single query request + * + * Creating new collector is required because MultiVectorResultCollector has instance variables + * which should be isolated for each query. + */ +struct MultiVectorResultCollectorFactory:faiss::ResultCollectorFactory { + BitSet* parent_bit_set; + + MultiVectorResultCollectorFactory(BitSet* parent_bit_set); + faiss::ResultCollector* new_collector() override; + void delete_collector(faiss::ResultCollector* resultCollector) override; +}; + +} // namespace os_faiss diff --git a/jni/include/knn_extension/faiss/utils/BitSet.h b/jni/include/knn_extension/faiss/utils/BitSet.h new file mode 100644 index 000000000..0c8079d37 --- /dev/null +++ b/jni/include/knn_extension/faiss/utils/BitSet.h @@ -0,0 +1,54 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include +#include +#include + +using idx_t = faiss::idx_t; + +struct BitSet { + const int NO_MORE_DOCS = std::numeric_limits::max(); + /** + * Returns the index of the first set bit starting at the index specified. + * NO_MORE_DOCS is returned if there are no more set bits. + */ + virtual idx_t next_set_bit(idx_t index) const = 0; + virtual ~BitSet() = default; +}; + + +/** + * BitSet of fixed length (numBits), implemented using an array of unit64. + * See https://github.com/apache/lucene/blob/main/lucene/core/src/java/org/apache/lucene/util/FixedBitSet.java + * + * Here a block is 64 bit. However, for simplicity let's assume its size is 8 bits. + * Then, if have an array of 3, 7, and 10, it will be represented in bitmap as follow. + * [0] [1] + * bitmap: 10001000 00000100 + * + * for next_set_bit call with 4 + * 1. it looks for words[0] + * 2. words[0] >> 4 + * 3. count trailing zero of the result from step 2 which is 3 + * 4. return 4(current index) + 3(result from step 3) + */ +struct FixedBitSet : public BitSet { + // The number of bits in use + idx_t num_bits; + + // The exact number of longs needed to hold num_bits + size_t num_words; + + // Array of uint64_t holding the bits + // Using uint64_t to leverage function __builtin_ctzll which is defined in faiss/impl/platform_macros.h + uint64_t* words; + + FixedBitSet(const int* int_array, const int length); + idx_t next_set_bit(idx_t index) const; + ~FixedBitSet(); +}; diff --git a/jni/include/knn_extension/faiss/utils/Heap.h b/jni/include/knn_extension/faiss/utils/Heap.h new file mode 100644 index 000000000..2aa19da52 --- /dev/null +++ b/jni/include/knn_extension/faiss/utils/Heap.h @@ -0,0 +1,255 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include +#include +#include + +#include +#include +#include + +#include +#include +#include + +// Collection of heap operations with parent id to dedupe +namespace os_faiss { + +/** + * From start_index, it compare its value with parent node's and swap if needed. + * Continue until either there is no swap or it reaches the top node. + * + * @param bh_val binary heap storing values + * @param bh_ids binary heap storing parent ids + * @param val new value to add + * @param id new id to add + * @parent_id_to_id parent doc id to id mapping data, see MultiVectorResultCollector.h + * @parent_id_to_index parent doc id to index mapping data, see MultiVectorResultCollector.h + * @parent_id parent id of given id + * @start_index an index to start up-heap from in the binary heap(bh_val, and bh_ids) + */ +template +static inline void up_heap( + typename C::T* bh_val, + typename C::TI* bh_ids, + typename C::T val, + typename C::TI id, + std::unordered_map* parent_id_to_id, + std::unordered_map* parent_id_to_index, + typename C::TI parent_id, + size_t start_index) { + bh_val--; /* Use 1-based indexing for easier node->child translation */ + bh_ids--; + size_t i = start_index + 1, i_father; + + while (i > 1) { + i_father = i >> 1; + if (!C::cmp2(val, bh_val[i_father], parent_id, bh_ids[i_father])) { + /* the heap structure is ok */ + break; + } + bh_val[i] = bh_val[i_father]; + bh_ids[i] = bh_ids[i_father]; + (*parent_id_to_index)[bh_ids[i]] = i - 1; + i = i_father; + } + bh_val[i] = val; + bh_ids[i] = parent_id; + (*parent_id_to_id)[parent_id] = id; + (*parent_id_to_index)[parent_id] = i - 1; +} + +/** + * From start_index, it compare its value with child node's and swap if needed. + * Continue until either there is no swap or it reaches the leaf node. + * + * @param nres number of values in the binary heap(bh_val, and bh_ids) + * @param bh_val binary heap storing values + * @param bh_ids binary heap storing parent ids + * @param val new value to add + * @param id new id to add + * @parent_id_to_id parent doc id to id mapping data, see MultiVectorResultCollector.h + * @parent_id_to_index parent doc id to index mapping data, see MultiVectorResultCollector.h + * @parent_id parent id of given id + * @start_index an index to start up-heap from in the binary heap(bh_val, and bh_ids) + */ +template +static inline void down_heap( + int nres, + typename C::T* bh_val, + typename C::TI* bh_ids, + typename C::T val, + typename C::TI id, + std::unordered_map* parent_id_to_id, + std::unordered_map* parent_id_to_index, + typename C::TI parent_id, + size_t start_index) { + bh_val--; /* Use 1-based indexing for easier node->child translation */ + bh_ids--; + size_t i = start_index + 1, i1, i2; + + while (1) { + i1 = i << 1; + i2 = i1 + 1; + if (i1 > nres) { + break; + } + + // Note that C::cmp2() is a bool function answering + // `(a1 > b1) || ((a1 == b1) && (a2 > b2))` for max + // heap and same with the `<` sign for min heap. + if ((i2 == nres + 1) || + C::cmp2(bh_val[i1], bh_val[i2], bh_ids[i1], bh_ids[i2])) { + if (C::cmp2(val, bh_val[i1], parent_id, bh_ids[i1])) { + break; + } + bh_val[i] = bh_val[i1]; + bh_ids[i] = bh_ids[i1]; + (*parent_id_to_index)[bh_ids[i]] = i - 1; + i = i1; + } else { + if (C::cmp2(val, bh_val[i2], parent_id, bh_ids[i2])) { + break; + } + bh_val[i] = bh_val[i2]; + bh_ids[i] = bh_ids[i2]; + (*parent_id_to_index)[bh_ids[i]] = i - 1; + i = i2; + } + } + bh_val[i] = val; + bh_ids[i] = parent_id; + (*parent_id_to_id)[parent_id] = id; + (*parent_id_to_index)[parent_id] = i - 1; +} + +/** + * Push the value to the max heap + * As the heap contains only one value per group id, pushing a value of existing group id + * will break the data integrity. For existing group id, use maxheap_update instead. + * The parent_id should not exist in in bh_ids, parent_id_to_id, and parent_id_to_index. + * + * @param nres number of values in the binary heap(bh_val, and bh_ids) + * @param bh_val binary heap storing values + * @param bh_ids binary heap storing parent ids + * @param val new value to add + * @param id new id to add + * @parent_id_to_id parent doc id to id mapping data, see MultiVectorResultCollector.h + * @parent_id_to_index parent doc id to index mapping data, see MultiVectorResultCollector.h + * @parent_id parent id of given id + */ +template +inline void maxheap_push( + int nres, + T* bh_val, + int64_t* bh_ids, + T val, + int64_t id, + std::unordered_map* parent_id_to_id, + std::unordered_map* parent_id_to_index, + int64_t parent_id) { + + assert(parent_id_to_index->find(parent_id) == parent_id_to_index->end() && "parent id should not exist in the binary heap"); + + up_heap>( + bh_val, + bh_ids, + val, + id, + parent_id_to_id, + parent_id_to_index, + parent_id, + nres); +} + +/** + * Update the top node with given value + * The parent_id should not exist in in bh_ids, parent_id_to_id, and parent_id_to_index. + * + * @param nres number of values in the binary heap(bh_val, and bh_ids) + * @param bh_val binary heap storing values + * @param bh_ids binary heap storing parent ids + * @param val new value to add + * @param id new id to add + * @parent_id_to_id parent doc id to id mapping data, see MultiVectorResultCollector.h + * @parent_id_to_index parent doc id to index mapping data, see MultiVectorResultCollector.h + * @parent_id parent id of given id + */ +template +inline void maxheap_replace_top( + int nres, + T* bh_val, + int64_t* bh_ids, + T val, + int64_t id, + std::unordered_map* parent_id_to_id, + std::unordered_map* parent_id_to_index, + int64_t parent_id) { + + assert(parent_id_to_index->find(parent_id) == parent_id_to_index->end() && "parent id should not exist in the binary heap"); + + parent_id_to_id->erase(bh_ids[0]); + parent_id_to_index->erase(bh_ids[0]); + down_heap>( + nres, + bh_val, + bh_ids, + val, + id, + parent_id_to_id, + parent_id_to_index, + parent_id, + 0); +} + +/** + * Update value of the parent_id in the binary heap and id of the parent_id in parent_id_to_id + * The parent_id should exist in bh_ids, parent_id_to_id, and parent_id_to_index. + * + * @param nres number of values in the binary heap(bh_val, and bh_ids) + * @param bh_val binary heap storing values + * @param bh_ids binary heap storing parent ids + * @param val new value to update + * @param id new id to update + * @parent_id_to_id parent doc id to id mapping data, see MultiVectorResultCollector.h + * @parent_id_to_index parent doc id to index mapping data, see MultiVectorResultCollector.h + * @parent_id parent id of given id + */ +template +inline void maxheap_update( + int nres, + T* bh_val, + int64_t* bh_ids, + T val, + int64_t id, + std::unordered_map* parent_id_to_id, + std::unordered_map* parent_id_to_index, + int64_t parent_id) { + size_t target_index = parent_id_to_index->at(parent_id); + up_heap>( + bh_val, + bh_ids, + val, + id, + parent_id_to_id, + parent_id_to_index, + parent_id, + target_index); + down_heap>( + nres, + bh_val, + bh_ids, + val, + id, + parent_id_to_id, + parent_id_to_index, + parent_id, + target_index); +} + +} // namespace os_faiss diff --git a/jni/include/org_opensearch_knn_jni_FaissService.h b/jni/include/org_opensearch_knn_jni_FaissService.h index a25264335..aefadcee4 100644 --- a/jni/include/org_opensearch_knn_jni_FaissService.h +++ b/jni/include/org_opensearch_knn_jni_FaissService.h @@ -48,7 +48,7 @@ JNIEXPORT jlong JNICALL Java_org_opensearch_knn_jni_FaissService_loadIndex * Signature: (J[FI)[Lorg/opensearch/knn/index/query/KNNQueryResult; */ JNIEXPORT jobjectArray JNICALL Java_org_opensearch_knn_jni_FaissService_queryIndex - (JNIEnv *, jclass, jlong, jfloatArray, jint); + (JNIEnv *, jclass, jlong, jfloatArray, jint, jintArray); /* * Class: org_opensearch_knn_jni_FaissService @@ -56,7 +56,7 @@ JNIEXPORT jobjectArray JNICALL Java_org_opensearch_knn_jni_FaissService_queryInd * Signature: (J[FI[J)[Lorg/opensearch/knn/index/query/KNNQueryResult; */ JNIEXPORT jobjectArray JNICALL Java_org_opensearch_knn_jni_FaissService_queryIndexWithFilter - (JNIEnv *, jclass, jlong, jfloatArray, jint, jintArray); + (JNIEnv *, jclass, jlong, jfloatArray, jint, jintArray, jintArray); /* * Class: org_opensearch_knn_jni_FaissService diff --git a/jni/patches/faiss/0001-Custom-patch-to-support-multi-vector.patch b/jni/patches/faiss/0001-Custom-patch-to-support-multi-vector.patch new file mode 100644 index 000000000..b07620c7e --- /dev/null +++ b/jni/patches/faiss/0001-Custom-patch-to-support-multi-vector.patch @@ -0,0 +1,301 @@ +From baa7e23c54637d68adac45f09633939b402405a9 Mon Sep 17 00:00:00 2001 +From: Heemin Kim +Date: Wed, 6 Dec 2023 16:33:52 -0800 +Subject: [PATCH] Custom patch to support multi-vector + +Signed-off-by: Heemin Kim +--- + faiss/CMakeLists.txt | 2 + + faiss/Index.h | 6 ++- + faiss/IndexIDMap.cpp | 24 ++++++++++ + faiss/impl/HNSW.cpp | 27 +++++++---- + faiss/impl/ResultCollector.h | 74 +++++++++++++++++++++++++++++ + faiss/impl/ResultCollectorFactory.h | 33 +++++++++++++ + 6 files changed, 154 insertions(+), 12 deletions(-) + create mode 100644 faiss/impl/ResultCollector.h + create mode 100644 faiss/impl/ResultCollectorFactory.h + +diff --git a/faiss/CMakeLists.txt b/faiss/CMakeLists.txt +index 27701586..af682a05 100644 +--- a/faiss/CMakeLists.txt ++++ b/faiss/CMakeLists.txt +@@ -162,6 +162,8 @@ set(FAISS_HEADERS + impl/ProductQuantizer.h + impl/Quantizer.h + impl/ResidualQuantizer.h ++ impl/ResultCollector.h ++ impl/ResultCollectorFactory.h + impl/ResultHandler.h + impl/ScalarQuantizer.h + impl/ThreadedIndex-inl.h +diff --git a/faiss/Index.h b/faiss/Index.h +index 4b4b302b..13eab0c0 100644 +--- a/faiss/Index.h ++++ b/faiss/Index.h +@@ -38,11 +38,12 @@ + + namespace faiss { + +-/// Forward declarations see impl/AuxIndexStructures.h, impl/IDSelector.h and +-/// impl/DistanceComputer.h ++/// Forward declarations see impl/AuxIndexStructures.h, impl/IDSelector.h, ++/// impl/DistanceComputer.h, and impl/ResultCollectorFactory.h + struct IDSelector; + struct RangeSearchResult; + struct DistanceComputer; ++struct ResultCollectorFactory; + + /** Parent class for the optional search paramenters. + * +@@ -52,6 +53,7 @@ struct DistanceComputer; + struct SearchParameters { + /// if non-null, only these IDs will be considered during search. + IDSelector* sel = nullptr; ++ ResultCollectorFactory* col = nullptr; + /// make sure we can dynamic_cast this + virtual ~SearchParameters() {} + }; +diff --git a/faiss/IndexIDMap.cpp b/faiss/IndexIDMap.cpp +index 7972bec9..0f82a17c 100644 +--- a/faiss/IndexIDMap.cpp ++++ b/faiss/IndexIDMap.cpp +@@ -18,6 +18,7 @@ + #include + #include + #include ++#include + + namespace faiss { + +@@ -102,6 +103,24 @@ struct ScopedSelChange { + } + }; + ++// RAII object to reset the id_map parameter in ResultCollectorFactory object ++// This object make sure to reset the id_map parameter in ResultCollectorFactory ++// once the program exist current method scope. ++struct ScopedColChange { ++ ResultCollectorFactory* collector_factory = nullptr; ++ void set( ++ ResultCollectorFactory* collector_factory, ++ const std::vector* id_map) { ++ this->collector_factory = collector_factory; ++ collector_factory->id_map = id_map; ++ } ++ ~ScopedColChange() { ++ if (collector_factory) { ++ collector_factory->id_map = nullptr; ++ } ++ } ++}; ++ + } // namespace + + template +@@ -114,6 +133,7 @@ void IndexIDMapTemplate::search( + const SearchParameters* params) const { + IDSelectorTranslated this_idtrans(this->id_map, nullptr); + ScopedSelChange sel_change; ++ ScopedColChange col_change; + + if (params && params->sel) { + auto idtrans = dynamic_cast(params->sel); +@@ -131,6 +151,10 @@ void IndexIDMapTemplate::search( + sel_change.set(params_non_const, &this_idtrans); + } + } ++ ++ if (params && params->col && !params->col->id_map) { ++ col_change.set(params->col, &this->id_map); ++ } + index->search(n, x, k, distances, labels, params); + idx_t* li = labels; + #pragma omp parallel for +diff --git a/faiss/impl/HNSW.cpp b/faiss/impl/HNSW.cpp +index 9fc201ea..5b5900d1 100644 +--- a/faiss/impl/HNSW.cpp ++++ b/faiss/impl/HNSW.cpp +@@ -14,6 +14,7 @@ + #include + #include + #include ++#include + #include + + #include +@@ -530,6 +531,15 @@ int search_from_candidates( + int level, + int nres_in = 0, + const SearchParametersHNSW* params = nullptr) { ++ ResultCollectorFactory defaultFactory; ++ ResultCollectorFactory* collectorFactory; ++ if (params == nullptr || params->col == nullptr) { ++ collectorFactory = &defaultFactory; ++ } else { ++ collectorFactory = params->col; ++ } ++ ResultCollector* collector = collectorFactory->new_collector(); ++ + int nres = nres_in; + int ndis = 0; + +@@ -544,11 +554,7 @@ int search_from_candidates( + float d = candidates.dis[i]; + FAISS_ASSERT(v1 >= 0); + if (!sel || sel->is_member(v1)) { +- if (nres < k) { +- faiss::maxheap_push(++nres, D, I, d, v1); +- } else if (d < D[0]) { +- faiss::maxheap_replace_top(nres, D, I, d, v1); +- } ++ collector->collect(k, nres, D, I, d, v1); + } + vt.set(v1); + } +@@ -612,11 +618,7 @@ int search_from_candidates( + + auto add_to_heap = [&](const size_t idx, const float dis) { + if (!sel || sel->is_member(idx)) { +- if (nres < k) { +- faiss::maxheap_push(++nres, D, I, dis, idx); +- } else if (dis < D[0]) { +- faiss::maxheap_replace_top(nres, D, I, dis, idx); +- } ++ collector->collect(k, nres, D, I, dis, idx); + } + candidates.push(idx, dis); + }; +@@ -660,6 +662,11 @@ int search_from_candidates( + } + } + ++ // Completed collection of result. Run post processor. ++ collector->post_process(nres, I); ++ // Collector completed its task. Release all resource of the collector. ++ collectorFactory->delete_collector(collector); ++ + if (level == 0) { + stats.n1++; + if (candidates.size() == 0) { +diff --git a/faiss/impl/ResultCollector.h b/faiss/impl/ResultCollector.h +new file mode 100644 +index 00000000..a0489fd6 +--- /dev/null ++++ b/faiss/impl/ResultCollector.h +@@ -0,0 +1,74 @@ ++/** ++ * Copyright (c) Facebook, Inc. and its affiliates. ++ * ++ * This source code is licensed under the MIT license found in the ++ * LICENSE file in the root directory of this source tree. ++ */ ++ ++#pragma once ++ ++#include ++#include ++ ++#include ++#include ++ ++/** ++ * ResultCollector is intended to define how to collect search result ++ * For each single search result, collect method will be called. ++ * After every results are collected, post_process method is called at the end. ++ */ ++ ++namespace faiss { ++ ++/** Encapsulates a set of ids to handle. */ ++struct ResultCollector { ++ /** ++ * For each result, collect method is called to store result ++ * @param k number of vectors to search ++ * @param nres number of results in queue ++ * @param bh_val search result, distances from query ++ * @param bh_ids search result, ids of vectors ++ * @param val distance from query for current vector ++ * @param ids id of current vector ++ */ ++ virtual void collect( ++ int k, ++ int& nres, ++ float* bh_val, ++ idx_t* bh_ids, ++ float val, ++ idx_t ids) = 0; ++ ++ // This method is called after all result is collected ++ virtual void post_process(idx_t nres, idx_t* bh_ids) = 0; ++ virtual ~ResultCollector() {} ++}; ++ ++struct DefaultCollector : ResultCollector { ++ void collect( ++ int k, ++ int& nres, ++ float* bh_val, ++ idx_t* bh_ids, ++ float val, ++ idx_t ids) override { ++ if (nres < k) { ++ faiss::maxheap_push(++nres, bh_val, bh_ids, val, ids); ++ } else if (val < bh_val[0]) { ++ faiss::maxheap_replace_top(nres, bh_val, bh_ids, val, ids); ++ } ++ } ++ ++ // This method is called once all result is collected so that final post ++ // processing can be done For example, if the result is collected using ++ // group id, the group id can be converted back to its original id inside ++ // this method ++ void post_process(idx_t nres, idx_t* bh_ids) override { ++ // Do nothing ++ } ++ ++ ~DefaultCollector() override {} ++}; ++ ++} // namespace faiss +diff --git a/faiss/impl/ResultCollectorFactory.h b/faiss/impl/ResultCollectorFactory.h +new file mode 100644 +index 00000000..b460b20b +--- /dev/null ++++ b/faiss/impl/ResultCollectorFactory.h +@@ -0,0 +1,33 @@ ++/** ++ * Copyright (c) Facebook, Inc. and its affiliates. ++ * ++ * This source code is licensed under the MIT license found in the ++ * LICENSE file in the root directory of this source tree. ++ */ ++ ++#pragma once ++#include ++namespace faiss { ++ ++/** ResultCollectorFactory to create a ResultCollector object */ ++struct ResultCollectorFactory { ++ DefaultCollector default_collector; ++ const std::vector* id_map = nullptr; ++ ++ // Create a new ResultCollector object ++ virtual ResultCollector* new_collector() { ++ return &default_collector; ++ } ++ ++ // For default case, the factory share single object and no need to delete ++ // the object. For other case, the factory can create a new object which ++ // need to be deleted later. We have deleteCollector method to handle both ++ // case as factory class knows how to release resource that it created ++ virtual void delete_collector(ResultCollector* collector) { ++ // Do nothing ++ } ++ ++ virtual ~ResultCollectorFactory() {} ++}; ++ ++} // namespace faiss +-- +2.39.3 (Apple Git-145) + diff --git a/jni/src/faiss_wrapper.cpp b/jni/src/faiss_wrapper.cpp index e8fb4de20..8e9deb07b 100644 --- a/jni/src/faiss_wrapper.cpp +++ b/jni/src/faiss_wrapper.cpp @@ -11,6 +11,7 @@ #include "jni_util.h" #include "faiss_wrapper.h" +#include "knn_extension/faiss/MultiVectorResultCollectorFactory.h" #include "faiss/impl/io.h" #include "faiss/index_factory.h" @@ -50,6 +51,10 @@ void convertFilterIdsToFaissIdType(const int* filterIds, int filterIdsLength, fa // Concerts the FilterIds to BitMap void buildFilterIdsBitMap(const int* filterIds, int filterIdsLength, uint8_t* bitsetVector); +os_faiss::MultiVectorResultCollectorFactory* buildResultCollectorFactory(knn_jni::JNIUtilInterface * jniUtil, JNIEnv *env, jintArray parentIdsJ); + +void releaseResultCollectorFactory(os_faiss::MultiVectorResultCollectorFactory* collectorFactory); + void knn_jni::faiss_wrapper::CreateIndex(knn_jni::JNIUtilInterface * jniUtil, JNIEnv * env, jintArray idsJ, jobjectArray vectorsJ, jstring indexPathJ, jobject parametersJ) { @@ -195,13 +200,12 @@ jlong knn_jni::faiss_wrapper::LoadIndex(knn_jni::JNIUtilInterface * jniUtil, JNI } jobjectArray knn_jni::faiss_wrapper::QueryIndex(knn_jni::JNIUtilInterface * jniUtil, JNIEnv * env, jlong indexPointerJ, - jfloatArray queryVectorJ, jint kJ) { - return knn_jni::faiss_wrapper::QueryIndex_WithFilter(jniUtil, env, indexPointerJ, queryVectorJ, kJ, nullptr); + jfloatArray queryVectorJ, jint kJ, jintArray parentIdsJ) { + return knn_jni::faiss_wrapper::QueryIndex_WithFilter(jniUtil, env, indexPointerJ, queryVectorJ, kJ, nullptr, parentIdsJ); } jobjectArray knn_jni::faiss_wrapper::QueryIndex_WithFilter(knn_jni::JNIUtilInterface * jniUtil, JNIEnv * env, jlong indexPointerJ, - jfloatArray queryVectorJ, jint kJ, jintArray filterIdsJ) { - + jfloatArray queryVectorJ, jint kJ, jintArray filterIdsJ, jintArray parentIdsJ) { if (queryVectorJ == nullptr) { throw std::runtime_error("Query Vector cannot be null"); } @@ -255,6 +259,7 @@ jobjectArray knn_jni::faiss_wrapper::QueryIndex_WithFilter(knn_jni::JNIUtilInter // value of ef_search = 16 which will then be used. hnswParams.efSearch = hnswReader->hnsw.efSearch; hnswParams.sel = idSelector.get(); + hnswParams.col = buildResultCollectorFactory(jniUtil, env, parentIdsJ); searchParameters = &hnswParams; } else { auto ivfReader = dynamic_cast(indexReader->index); @@ -269,16 +274,30 @@ jobjectArray knn_jni::faiss_wrapper::QueryIndex_WithFilter(knn_jni::JNIUtilInter } catch (...) { jniUtil->ReleaseFloatArrayElements(env, queryVectorJ, rawQueryvector, JNI_ABORT); jniUtil->ReleaseIntArrayElements(env, filterIdsJ, filteredIdsArray, JNI_ABORT); + releaseResultCollectorFactory(dynamic_cast(hnswParams.col)); throw; } jniUtil->ReleaseIntArrayElements(env, filterIdsJ, filteredIdsArray, JNI_ABORT); + releaseResultCollectorFactory(dynamic_cast(hnswParams.col)); } else { + faiss::SearchParameters *searchParameters = nullptr; + faiss::SearchParametersHNSW hnswParams; + auto hnswReader = dynamic_cast(indexReader->index); + if(hnswReader!= nullptr && parentIdsJ != nullptr) { + // Setting the ef_search value equal to what was provided during index creation. SearchParametersHNSW has a default + // value of ef_search = 16 which will then be used. + hnswParams.efSearch = hnswReader->hnsw.efSearch; + hnswParams.col = buildResultCollectorFactory(jniUtil, env, parentIdsJ); + searchParameters = &hnswParams; + } try { - indexReader->search(1, rawQueryvector, kJ, dis.data(), ids.data()); + indexReader->search(1, rawQueryvector, kJ, dis.data(), ids.data(), searchParameters); } catch (...) { jniUtil->ReleaseFloatArrayElements(env, queryVectorJ, rawQueryvector, JNI_ABORT); + releaseResultCollectorFactory(dynamic_cast(hnswParams.col)); throw; } + releaseResultCollectorFactory(dynamic_cast(hnswParams.col)); } jniUtil->ReleaseFloatArrayElements(env, queryVectorJ, rawQueryvector, JNI_ABORT); @@ -489,3 +508,22 @@ void buildFilterIdsBitMap(const int* filterIds, int filterIdsLength, uint8_t* bi bitsetVector[bitsetArrayIndex] = bitsetVector[bitsetArrayIndex] | (1 << (value & 7)); } } + +os_faiss::MultiVectorResultCollectorFactory* buildResultCollectorFactory(knn_jni::JNIUtilInterface * jniUtil, JNIEnv *env, jintArray parentIdsJ) { + if (parentIdsJ == nullptr) { + return nullptr; + } + int *parentIdsArray = jniUtil->GetIntArrayElements(env, parentIdsJ, nullptr); + int parentIdsLength = jniUtil->GetJavaIntArrayLength(env, parentIdsJ); + auto* parent_id_filter = new FixedBitSet(parentIdsArray, parentIdsLength); + jniUtil->ReleaseIntArrayElements(env, parentIdsJ, parentIdsArray, JNI_ABORT); + return new os_faiss::MultiVectorResultCollectorFactory(parent_id_filter); +} + +void releaseResultCollectorFactory(os_faiss::MultiVectorResultCollectorFactory* collectorFactory) { + if (collectorFactory == nullptr) { + return; + } + delete collectorFactory->parent_bit_set; + delete collectorFactory; +} diff --git a/jni/src/knn_extension/faiss/MultiVectorResultCollector.cpp b/jni/src/knn_extension/faiss/MultiVectorResultCollector.cpp new file mode 100644 index 000000000..a7564d3aa --- /dev/null +++ b/jni/src/knn_extension/faiss/MultiVectorResultCollector.cpp @@ -0,0 +1,67 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "MultiVectorResultCollector.h" +#include "knn_extension/faiss/utils/Heap.h" +#include "knn_extension/faiss/utils/BitSet.h" + +namespace os_faiss { + +using idx_t = faiss::idx_t; + +MultiVectorResultCollector::MultiVectorResultCollector(const BitSet* parent_bit_set, const std::vector* id_map) +: parent_bit_set(parent_bit_set), id_map(id_map) {} + +void MultiVectorResultCollector::collect( + int k, + int& nres, + float* bh_val, + int64_t* bh_ids, + float val, + int64_t ids) { + idx_t group_id = id_map ? parent_bit_set->next_set_bit(id_map->at(ids)) : parent_bit_set->next_set_bit(ids); + if (parent_id_to_index.find(group_id) == + parent_id_to_index.end()) { + if (nres < k) { + maxheap_push( + nres++, + bh_val, + bh_ids, + val, + ids, + &parent_id_to_id, + &parent_id_to_index, + group_id); + } else if (val < bh_val[0]) { + maxheap_replace_top( + nres, + bh_val, + bh_ids, + val, + ids, + &parent_id_to_id, + &parent_id_to_index, + group_id); + } + } else if (val < bh_val[parent_id_to_index.at(group_id)]) { + maxheap_update( + nres, + bh_val, + bh_ids, + val, + ids, + &parent_id_to_id, + &parent_id_to_index, + group_id); + } +} + +void MultiVectorResultCollector::post_process(int64_t nres, int64_t* bh_ids) { + for (size_t icnt = 0; icnt < nres; icnt++) { + bh_ids[icnt] = parent_id_to_id.at(bh_ids[icnt]); + } +} + +} // namespace os_faiss diff --git a/jni/src/knn_extension/faiss/MultiVectorResultCollectorFactory.cpp b/jni/src/knn_extension/faiss/MultiVectorResultCollectorFactory.cpp new file mode 100644 index 000000000..f4c7c0656 --- /dev/null +++ b/jni/src/knn_extension/faiss/MultiVectorResultCollectorFactory.cpp @@ -0,0 +1,24 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "MultiVectorResultCollectorFactory.h" +#include "MultiVectorResultCollector.h" + +namespace os_faiss { + +MultiVectorResultCollectorFactory::MultiVectorResultCollectorFactory(BitSet* parent_bit_set) + : parent_bit_set(parent_bit_set) {} + +// id_map is set in IndexIDMap.cpp of faiss library with custom patch +// https://github.com/opensearch-project/k-NN/blob/feature/multi-vector/jni/patches/faiss/0001-Custom-patch-to-support-multi-vector.patch#L109 +faiss::ResultCollector* MultiVectorResultCollectorFactory::new_collector() { + return new MultiVectorResultCollector(parent_bit_set, id_map); +} + +void MultiVectorResultCollectorFactory::delete_collector(faiss::ResultCollector* resultCollector) { + delete resultCollector; +} + +} // namespace os_faiss diff --git a/jni/src/knn_extension/faiss/utils/BitSet.cpp b/jni/src/knn_extension/faiss/utils/BitSet.cpp new file mode 100644 index 000000000..33e9470e0 --- /dev/null +++ b/jni/src/knn_extension/faiss/utils/BitSet.cpp @@ -0,0 +1,47 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +#include +#include +#include "BitSet.h" + +FixedBitSet::FixedBitSet(const int* int_array, const int length){ + assert(int_array && "int_array should not be null"); + const int* maxValue = std::max_element(int_array, int_array + length); + this->num_bits = *maxValue + 1; + this->num_words = (num_bits >> 6) + 1; // div by 64 + this->words = new uint64_t[this->num_words](); + for(int i = 0 ; i < length ; i ++) { + int value = int_array[i]; + int bitset_array_index = value >> 6; + this->words[bitset_array_index] |= 1ULL << (value & 63); // Equivalent of 1ULL << (value % 64) + } +} + +idx_t FixedBitSet::next_set_bit(idx_t index) const { + assert(index >= 0 && "index shouldn't be less than zero"); + assert(index < this->num_bits && "index should be less than total number of bits"); + + idx_t i = index >> 6; // div by 64 + uint64_t word = this->words[i] >> (index & 63); // Equivalent of words[i] >> (index % 64) + // word is non zero after right shift, it means, next set bit is in current word + // The index of set bit is "given index" + "trailing zero in the right shifted word" + if (word != 0) { + return index + __builtin_ctzll(word); + } + + while (++i < this->num_words) { + word = this->words[i]; + if (word != 0) { + return (i << 6) + __builtin_ctzll(word); + } + } + + return NO_MORE_DOCS; +} + +FixedBitSet::~FixedBitSet() { + delete this->words; +} diff --git a/jni/src/org_opensearch_knn_jni_FaissService.cpp b/jni/src/org_opensearch_knn_jni_FaissService.cpp index 1b79d9114..a7b24fcab 100644 --- a/jni/src/org_opensearch_knn_jni_FaissService.cpp +++ b/jni/src/org_opensearch_knn_jni_FaissService.cpp @@ -77,10 +77,10 @@ JNIEXPORT jlong JNICALL Java_org_opensearch_knn_jni_FaissService_loadIndex(JNIEn JNIEXPORT jobjectArray JNICALL Java_org_opensearch_knn_jni_FaissService_queryIndex(JNIEnv * env, jclass cls, jlong indexPointerJ, - jfloatArray queryVectorJ, jint kJ) + jfloatArray queryVectorJ, jint kJ, jintArray parentIdsJ) { try { - return knn_jni::faiss_wrapper::QueryIndex(&jniUtil, env, indexPointerJ, queryVectorJ, kJ); + return knn_jni::faiss_wrapper::QueryIndex(&jniUtil, env, indexPointerJ, queryVectorJ, kJ, parentIdsJ); } catch (...) { jniUtil.CatchCppExceptionAndThrowJava(env); @@ -89,10 +89,10 @@ JNIEXPORT jobjectArray JNICALL Java_org_opensearch_knn_jni_FaissService_queryInd } JNIEXPORT jobjectArray JNICALL Java_org_opensearch_knn_jni_FaissService_queryIndexWithFilter - (JNIEnv * env, jclass cls, jlong indexPointerJ, jfloatArray queryVectorJ, jint kJ, jintArray filteredIdsJ) { + (JNIEnv * env, jclass cls, jlong indexPointerJ, jfloatArray queryVectorJ, jint kJ, jintArray filteredIdsJ, jintArray parentIdsJ) { try { - return knn_jni::faiss_wrapper::QueryIndex_WithFilter(&jniUtil, env, indexPointerJ, queryVectorJ, kJ, filteredIdsJ); + return knn_jni::faiss_wrapper::QueryIndex_WithFilter(&jniUtil, env, indexPointerJ, queryVectorJ, kJ, filteredIdsJ, parentIdsJ); } catch (...) { jniUtil.CatchCppExceptionAndThrowJava(env); } diff --git a/jni/tests/faiss_wrapper_test.cpp b/jni/tests/faiss_wrapper_test.cpp index abe4ecb20..5fa5165bb 100644 --- a/jni/tests/faiss_wrapper_test.cpp +++ b/jni/tests/faiss_wrapper_test.cpp @@ -40,7 +40,7 @@ TEST(FaissCreateIndexTest, BasicAssertions) { std::string indexPath = test_util::RandomString(10, "tmp/", ".faiss"); std::string spaceType = knn_jni::L2; - std::string index_description = "Flat"; // TODO: Revert bach to HNSW32,Flat + std::string index_description = "HNSW32,Flat"; std::unordered_map parametersMap; parametersMap[knn_jni::SPACE_TYPE] = (jobject)&spaceType; @@ -87,7 +87,7 @@ TEST(FaissCreateIndexFromTemplateTest, BasicAssertions) { std::string indexPath = test_util::RandomString(10, "tmp/", ".faiss"); faiss::MetricType metricType = faiss::METRIC_L2; - std::string method = "Flat"; // TODO: Revert bach to HNSW32,Flat + std::string method = "HNSW32,Flat"; std::unique_ptr createdIndex( test_util::FaissCreateIndex(dim, method, metricType)); @@ -135,7 +135,7 @@ TEST(FaissLoadIndexTest, BasicAssertions) { std::string indexPath = test_util::RandomString(10, "tmp/", ".faiss"); faiss::MetricType metricType = faiss::METRIC_L2; - std::string method = "Flat"; // TODO: Revert bach to HNSW32,Flat + std::string method = "HNSW32,Flat"; // Create the index std::unique_ptr createdIndex( @@ -186,7 +186,7 @@ TEST(FaissQueryIndexTest, BasicAssertions) { } faiss::MetricType metricType = faiss::METRIC_L2; - std::string method = "Flat"; // TODO: Revert bach to HNSW32,Flat + std::string method = "HNSW32,Flat"; // Define query data int k = 10; @@ -218,7 +218,7 @@ TEST(FaissQueryIndexTest, BasicAssertions) { knn_jni::faiss_wrapper::QueryIndex( &mockJNIUtil, jniEnv, reinterpret_cast(&createdIndexWithData), - reinterpret_cast(&query), k))); + reinterpret_cast(&query), k, nullptr))); ASSERT_EQ(k, results->size()); @@ -229,11 +229,84 @@ TEST(FaissQueryIndexTest, BasicAssertions) { } } +TEST(FaissQueryIndexWithParentFilterTest, BasicAssertions) { + // Define the index data + faiss::idx_t numIds = 100; + std::vector ids; + std::vector vectors; + std::vector parentIds; + int dim = 16; + for (int64_t i = 1; i < numIds + 1; i++) { + if (i % 10 == 0) { + parentIds.push_back(i); + continue; + } + ids.push_back(i); + for (int j = 0; j < dim; j++) { + vectors.push_back(test_util::RandomFloat(-500.0, 500.0)); + } + } + + faiss::MetricType metricType = faiss::METRIC_L2; + std::string method = "HNSW32,Flat"; + + // Define query data + int k = 20; + int numQueries = 100; + std::vector> queries; + + for (int i = 0; i < numQueries; i++) { + std::vector query; + query.reserve(dim); + for (int j = 0; j < dim; j++) { + query.push_back(test_util::RandomFloat(-500.0, 500.0)); + } + queries.push_back(query); + } + + // Create the index + std::unique_ptr createdIndex( + test_util::FaissCreateIndex(2, method, metricType)); + auto createdIndexWithData = + test_util::FaissAddData(createdIndex.get(), ids, vectors); + + // Setup jni + JNIEnv *jniEnv = nullptr; + NiceMock mockJNIUtil; + EXPECT_CALL(mockJNIUtil, + GetJavaIntArrayLength( + jniEnv, reinterpret_cast(&parentIds))) + .WillRepeatedly(Return(parentIds.size())); + for (auto query : queries) { + std::unique_ptr *>> results( + reinterpret_cast *> *>( + knn_jni::faiss_wrapper::QueryIndex( + &mockJNIUtil, jniEnv, + reinterpret_cast(&createdIndexWithData), + reinterpret_cast(&query), k, + reinterpret_cast(&parentIds)))); + + // Even with k 20, result should have only 10 which is total number of groups + ASSERT_EQ(10, results->size()); + // Result should be one for each group + std::set idSet; + for (const auto& pairPtr : *results) { + idSet.insert(pairPtr->first / 10); + } + ASSERT_EQ(10, idSet.size()); + + // Need to free up each result + for (auto it : *results.get()) { + delete it; + } + } +} + TEST(FaissFreeTest, BasicAssertions) { // Define the data int dim = 2; faiss::MetricType metricType = faiss::METRIC_L2; - std::string method = "Flat"; // TODO: Revert bach to HNSW32,Flat + std::string method = "HNSW32,Flat"; // Create the index faiss::Index *createdIndex( diff --git a/jni/tests/knn_extension/faiss/MultiVectorResultCollectorFactoryTest.cpp b/jni/tests/knn_extension/faiss/MultiVectorResultCollectorFactoryTest.cpp new file mode 100644 index 000000000..3177360bb --- /dev/null +++ b/jni/tests/knn_extension/faiss/MultiVectorResultCollectorFactoryTest.cpp @@ -0,0 +1,78 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "knn_extension/faiss/MultiVectorResultCollectorFactory.h" +#include "knn_extension/faiss/MultiVectorResultCollector.h" + +#include + +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "jni_util.h" + +using ::testing::NiceMock; +using ::testing::Return; +using idx_t = faiss::idx_t; + + +TEST(MultiVectorResultCollectorFactoryTest, BasicAssertions) { + int parent_ids[1] = {1}; + FixedBitSet parent_id_filter(parent_ids, 1); + + std::unordered_map distance1; + distance1[0] = 10; + distance1[1] = 11; + + std::unordered_map distance2; + distance2[0] = 11; + distance2[1] = 10; + + os_faiss::MultiVectorResultCollectorFactory* rc_factory = new os_faiss::MultiVectorResultCollectorFactory(&parent_id_filter); + faiss::ResultCollector* rc1 = rc_factory->new_collector(); + faiss::ResultCollector* rc2 = rc_factory->new_collector(); + ASSERT_NE(rc1, rc2); + + int k = 1; + int nres1 = 0; + int nres2 = 0; + float* bh_val = new float[k * 2]; + int64_t* bh_ids = new int64_t[k * 2]; + // Verify two collector are thread safe each other. + // Simulate multi thread by interleaving collect methods of two ResultCollectors. + for (int i = 0; i < distance1.size(); i++) { + rc1->collect(k, nres1, bh_val, bh_ids, distance1.at(i), i); + rc2->collect(k, nres2, bh_val + k, bh_ids + k, distance2.at(i), i); + } + rc1->post_process(nres1, bh_ids); + rc2->post_process(nres2, bh_ids + k); + + ASSERT_EQ(0, bh_ids[0]); + ASSERT_EQ(1, bh_ids[1]); + + rc_factory->delete_collector(rc1); + rc_factory->delete_collector(rc2); + delete rc_factory; + delete[] bh_val; + delete[] bh_ids; +} + +// Verify that id_map is passed to collector +TEST(MultiVectorResultCollectorFactoryWithIdMapTest, BasicAssertions) { + int parent_ids[1] = {1}; + FixedBitSet parent_id_filter(parent_ids, 1); + std::vector id_map; + + os_faiss::MultiVectorResultCollectorFactory* rc_factory = new os_faiss::MultiVectorResultCollectorFactory(&parent_id_filter); + os_faiss::MultiVectorResultCollector* rc1 = dynamic_cast(rc_factory->new_collector()); + ASSERT_EQ(nullptr, rc1->id_map); + + rc_factory->id_map = &id_map; + os_faiss::MultiVectorResultCollector* rc2 = dynamic_cast(rc_factory->new_collector()); + ASSERT_EQ(&id_map, rc2->id_map); + + rc_factory->delete_collector(rc1); + rc_factory->delete_collector(rc2); + delete rc_factory; +} diff --git a/jni/tests/knn_extension/faiss/MultiVectorResultCollectorTest.cpp b/jni/tests/knn_extension/faiss/MultiVectorResultCollectorTest.cpp new file mode 100644 index 000000000..934d708dd --- /dev/null +++ b/jni/tests/knn_extension/faiss/MultiVectorResultCollectorTest.cpp @@ -0,0 +1,96 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "knn_extension/faiss/MultiVectorResultCollector.h" + +#include + +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +using ::testing::NiceMock; +using ::testing::Return; +using idx_t = faiss::idx_t; + + +TEST(MultiVectorResultCollectorTest, BasicAssertions) { + // Data + // Parent ID: 2, ID: 0, Distance: 10 + // Parent ID: 2, ID: 1, Distance: 11 + // Parent ID: 5, ID: 3, Distance: 12 + // Parent ID: 5, ID: 4, Distance: 13 + // After collector handing the data with k = 3, it should return data with id 0 and 2, one from each group. + // Parent bit set representation: 100100 + int parent_ids[2] = {2, 5}; + FixedBitSet parent_id_filter(parent_ids, 2); + + idx_t ids[] = {0, 1, 2, 3}; + float distances[] = {10, 11, 12, 13}; + + os_faiss::MultiVectorResultCollector* rc = new os_faiss::MultiVectorResultCollector(&parent_id_filter, nullptr); + int k = 3; + int nres = 0; + float* bh_val = new float[k]; + int64_t* bh_ids = new int64_t[k]; + for (int i = 0; i < 4; i++) { + rc->collect(k, nres, bh_val, bh_ids, distances[i], ids[i]); + } + + // Parent ID is stored before finalize + ASSERT_EQ(5, bh_ids[0]); + ASSERT_EQ(2, bh_ids[1]); + + rc->post_process(nres, bh_ids); + + // Parent ID is converted to ID after finalize + ASSERT_EQ(3, bh_ids[0]); + ASSERT_EQ(0, bh_ids[1]); + + delete rc; + delete[] bh_val; + delete[] bh_ids; +} + +TEST(MultiVectorResultCollectorWithIDMapTest, BasicAssertions) { + // Data + // Parent ID: 2, Lucene ID: 0, Faiss ID: 0, Distance: 10 + // Parent ID: 2, Lucene ID: 1, Faiss ID: 1, Distance: 11 + // Parent ID: 5, Lucene ID: 3, Faiss ID: 2, Distance: 12 + // Parent ID: 5, Lucene ID: 4, Faiss ID: 3, Distance: 13 + // After collector handing the data with k = 3, it should return data with id 0 and 2, one from each group. + + // Parent bit set representation with Lucene ID: 100100 + int parent_ids[2] = {2, 5}; + FixedBitSet parent_id_filter(parent_ids, 2); + + idx_t faiss_ids[] = {0, 1, 2, 3}; + float distances[] = {10, 11, 12, 13}; + + // Faiss IDs to Lucene ID mapping + std::vector id_map = {0, 1, 3, 4}; + + os_faiss::MultiVectorResultCollector* rc = new os_faiss::MultiVectorResultCollector(&parent_id_filter, &id_map); + int k = 3; + int nres = 0; + float* bh_val = new float[k]; + int64_t* bh_ids = new int64_t[k]; + for (int i = 0; i < 4; i++) { + rc->collect(k, nres, bh_val, bh_ids, distances[i], faiss_ids[i]); + } + + // Parent ID is stored before finalize + ASSERT_EQ(5, bh_ids[0]); + ASSERT_EQ(2, bh_ids[1]); + + rc->post_process(nres, bh_ids); + + // Parent ID is converted to Faiss ID after finalize + ASSERT_EQ(2, bh_ids[0]); + ASSERT_EQ(0, bh_ids[1]); + + delete rc; + delete[] bh_val; + delete[] bh_ids; +} diff --git a/jni/tests/knn_extension/faiss/utils/BitSetTest.cpp b/jni/tests/knn_extension/faiss/utils/BitSetTest.cpp new file mode 100644 index 000000000..96ad6b3c2 --- /dev/null +++ b/jni/tests/knn_extension/faiss/utils/BitSetTest.cpp @@ -0,0 +1,52 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "knn_extension/faiss/utils/BitSet.h" + +#include + +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +using ::testing::NiceMock; +using ::testing::Return; +using idx_t = faiss::idx_t; + +TEST(FixedBitSetTest, BasicAssertions) { + int ids1[4] = {3, 7, 11, 15}; + FixedBitSet single_block(ids1, 4); + + ASSERT_EQ(3, single_block.next_set_bit(0)); + ASSERT_EQ(3, single_block.next_set_bit(1)); + ASSERT_EQ(3, single_block.next_set_bit(2)); + ASSERT_EQ(3, single_block.next_set_bit(3)); + ASSERT_EQ(7, single_block.next_set_bit(4)); + ASSERT_EQ(7, single_block.next_set_bit(5)); + ASSERT_EQ(7, single_block.next_set_bit(6)); + ASSERT_EQ(7, single_block.next_set_bit(7)); + ASSERT_EQ(11, single_block.next_set_bit(8)); + ASSERT_EQ(11, single_block.next_set_bit(9)); + ASSERT_EQ(11, single_block.next_set_bit(10)); + ASSERT_EQ(11, single_block.next_set_bit(11)); + ASSERT_EQ(15, single_block.next_set_bit(12)); + ASSERT_EQ(15, single_block.next_set_bit(13)); + ASSERT_EQ(15, single_block.next_set_bit(14)); + ASSERT_EQ(15, single_block.next_set_bit(15)); + ASSERT_EQ(single_block.NO_MORE_DOCS, single_block.next_set_bit(16)); + + int ids2[5] = {64, 128, 127, 1024, 34565}; + int ids2_sorted[5]; + std::copy(ids2, ids2 + 5, ids2_sorted); + std::sort(ids2_sorted, ids2_sorted + 5); + FixedBitSet multi_blocks(ids2, 5); + int parent_index = 0; + for (int i = 0; i < ids2[4] + 1; i++) { + ASSERT_EQ(ids2_sorted[parent_index], multi_blocks.next_set_bit(i)); + if (ids2_sorted[parent_index] == i) { + parent_index++; + } + } + ASSERT_EQ(multi_blocks.NO_MORE_DOCS, multi_blocks.next_set_bit(ids2[4] + 1)); +} diff --git a/jni/tests/knn_extension/faiss/utils/HeapTest.cpp b/jni/tests/knn_extension/faiss/utils/HeapTest.cpp new file mode 100644 index 000000000..97a30babd --- /dev/null +++ b/jni/tests/knn_extension/faiss/utils/HeapTest.cpp @@ -0,0 +1,86 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "knn_extension/faiss/utils/Heap.h" +#include "faiss/utils/Heap.h" + +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +using ::testing::NiceMock; +using ::testing::Return; +using ::testing::ElementsAreArray; + +TEST(MaxHeapUpdateTest, BasicAssertions) { + const int k = 5; + int nres = 0; + float binary_heap_values[k]; + int64_t binary_heap_ids[k]; + float input_values[] = {1.1f, 2.1f, 3.1f, 4.1f, 5.1f}; + int64_t input_ids[] = {1, 2, 3, 4, 5}; + int64_t group_ids[] = {11, 22, 33, 44, 55}; + std::unordered_map group_id_to_id; + std::unordered_map group_id_to_index; + + // Push + for (int i = 0; i < k; i++) { + os_faiss::maxheap_push( + nres++, + binary_heap_values, + binary_heap_ids, + input_values[i], + input_ids[i], + &group_id_to_id, + &group_id_to_index, + group_ids[i]); + } + + // Verify heap data + // The top node in the max heap should be the one with max value(5.1f) + ASSERT_EQ(5.1f, binary_heap_values[0]); + ASSERT_EQ(55, binary_heap_ids[0]); + ASSERT_EQ(5, group_id_to_id.at(binary_heap_ids[0])); + + // Replace top + os_faiss::maxheap_replace_top( + nres, + binary_heap_values, + binary_heap_ids, + 0.1f, + 6, + &group_id_to_id, + &group_id_to_index, + 66); + + // Verify heap data + // Previous top value(5.1f) should have been removed and the next max value(4.1f) should be in the top node. + ASSERT_EQ(4.1f, binary_heap_values[0]); + ASSERT_EQ(44, binary_heap_ids[0]); + ASSERT_EQ(4, group_id_to_id.at(binary_heap_ids[0])); + + // Update + os_faiss::maxheap_update( + nres, + binary_heap_values, + binary_heap_ids, + 0.2f, + 7, + &group_id_to_id, + &group_id_to_index, + 33); + + // Verify heap data + // node id 3 with group id 33 should have been replaced by node id 7 with new value + ASSERT_EQ(7, group_id_to_id.at(33)); + + // Verify heap is in order + float expectedValues[] = {4.1f, 2.1f, 1.1f, 0.2f, 0.1f}; + int64_t expectedIds[] = {4, 2, 1, 7, 6}; + for (int i = 0; i < k; i++) { + ASSERT_EQ(expectedValues[i], binary_heap_values[0]); + ASSERT_EQ(expectedIds[i], group_id_to_id.at(binary_heap_ids[0])); + faiss::maxheap_pop(nres--, binary_heap_values, binary_heap_ids); + } +} diff --git a/src/main/java/org/opensearch/knn/index/query/KNNQuery.java b/src/main/java/org/opensearch/knn/index/query/KNNQuery.java index 5ac207c43..74a289994 100644 --- a/src/main/java/org/opensearch/knn/index/query/KNNQuery.java +++ b/src/main/java/org/opensearch/knn/index/query/KNNQuery.java @@ -15,6 +15,7 @@ import org.apache.lucene.search.QueryVisitor; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Weight; +import org.apache.lucene.search.join.BitSetProducer; import org.opensearch.knn.index.KNNSettings; import java.io.IOException; @@ -33,20 +34,37 @@ public class KNNQuery extends Query { @Getter @Setter private Query filterQuery; - - public KNNQuery(String field, float[] queryVector, int k, String indexName) { + @Getter + private BitSetProducer parentsFilter; + + public KNNQuery( + final String field, + final float[] queryVector, + final int k, + final String indexName, + final BitSetProducer parentsFilter + ) { this.field = field; this.queryVector = queryVector; this.k = k; this.indexName = indexName; + this.parentsFilter = parentsFilter; } - public KNNQuery(String field, float[] queryVector, int k, String indexName, Query filterQuery) { + public KNNQuery( + final String field, + final float[] queryVector, + final int k, + final String indexName, + final Query filterQuery, + final BitSetProducer parentsFilter + ) { this.field = field; this.queryVector = queryVector; this.k = k; this.indexName = indexName; this.filterQuery = filterQuery; + this.parentsFilter = parentsFilter; } public String getField() { diff --git a/src/main/java/org/opensearch/knn/index/query/KNNQueryFactory.java b/src/main/java/org/opensearch/knn/index/query/KNNQueryFactory.java index 7772ab582..2ab0e62af 100644 --- a/src/main/java/org/opensearch/knn/index/query/KNNQueryFactory.java +++ b/src/main/java/org/opensearch/knn/index/query/KNNQueryFactory.java @@ -82,17 +82,17 @@ public static Query create(CreateQueryRequest createQueryRequest) { final VectorDataType vectorDataType = createQueryRequest.getVectorDataType(); final Query filterQuery = getFilterQuery(createQueryRequest); + BitSetProducer parentFilter = createQueryRequest.context == null ? null : createQueryRequest.context.getParentFilter(); if (KNNEngine.getEnginesThatCreateCustomSegmentFiles().contains(createQueryRequest.getKnnEngine())) { if (filterQuery != null && KNNEngine.getEnginesThatSupportsFilters().contains(createQueryRequest.getKnnEngine())) { log.debug("Creating custom k-NN query with filters for index: {}, field: {} , k: {}", indexName, fieldName, k); - return new KNNQuery(fieldName, vector, k, indexName, filterQuery); + return new KNNQuery(fieldName, vector, k, indexName, filterQuery, parentFilter); } log.debug(String.format("Creating custom k-NN query for index: %s \"\", field: %s \"\", k: %d", indexName, fieldName, k)); - return new KNNQuery(fieldName, vector, k, indexName); + return new KNNQuery(fieldName, vector, k, indexName, parentFilter); } log.debug(String.format("Creating Lucene k-NN query for index: %s \"\", field: %s \"\", k: %d", indexName, fieldName, k)); - BitSetProducer parentFilter = createQueryRequest.context == null ? null : createQueryRequest.context.getParentFilter(); if (VectorDataType.BYTE == vectorDataType) { return getKnnByteVectorQuery(fieldName, byteVector, k, filterQuery, parentFilter); } else if (VectorDataType.FLOAT == vectorDataType) { @@ -205,9 +205,6 @@ static class CreateQueryRequest { private VectorDataType vectorDataType; @Getter private int k; - // can be null in cases filter not passed with the knn query - @Getter - private BitSetProducer parentFilter; private QueryBuilder filter; // can be null in cases filter not passed with the knn query private QueryShardContext context; diff --git a/src/main/java/org/opensearch/knn/index/query/KNNWeight.java b/src/main/java/org/opensearch/knn/index/query/KNNWeight.java index c166be3c2..180ce1b31 100644 --- a/src/main/java/org/opensearch/knn/index/query/KNNWeight.java +++ b/src/main/java/org/opensearch/knn/index/query/KNNWeight.java @@ -172,20 +172,28 @@ private int[] getFilterIdsArray(final LeafReaderContext context) throws IOExcept if (filterWeight == null) { return new int[0]; } - final BitSet filteredDocsBitSet = getFilteredDocsBitSet(context, this.filterWeight); - final int[] filteredIds = new int[filteredDocsBitSet.cardinality()]; - int filteredIdsIndex = 0; - int docId = 0; - while (docId < filteredDocsBitSet.length()) { - docId = filteredDocsBitSet.nextSetBit(docId); - if (docId == DocIdSetIterator.NO_MORE_DOCS || docId + 1 == DocIdSetIterator.NO_MORE_DOCS) { - break; - } - filteredIds[filteredIdsIndex] = docId; - filteredIdsIndex++; - docId++; + return bitSetToIntArray(getFilteredDocsBitSet(context, this.filterWeight)); + } + + private int[] getParentIdsArray(final LeafReaderContext context) throws IOException { + if (knnQuery.getParentsFilter() == null) { + return null; } - return filteredIds; + return bitSetToIntArray(knnQuery.getParentsFilter().getBitSet(context)); + } + + private int[] bitSetToIntArray(final BitSet bitSet) { + final int cardinality = bitSet.cardinality(); + final int[] intArray = new int[cardinality]; + final BitSetIterator bitSetIterator = new BitSetIterator(bitSet, cardinality); + int index = 0; + int docId = bitSetIterator.nextDoc(); + while (docId != DocIdSetIterator.NO_MORE_DOCS) { + assert index < intArray.length; + intArray[index++] = docId; + docId = bitSetIterator.nextDoc(); + } + return intArray; } private Map doANNSearch(final LeafReaderContext context, final int[] filterIdsArray) throws IOException { @@ -265,13 +273,14 @@ private Map doANNSearch(final LeafReaderContext context, final i if (indexAllocation.isClosed()) { throw new RuntimeException("Index has already been closed"); } - + int[] parentIds = getParentIdsArray(context); results = JNIService.queryIndex( indexAllocation.getMemoryAddress(), knnQuery.getQueryVector(), knnQuery.getK(), knnEngine.getName(), - filterIdsArray + filterIdsArray, + parentIds ); } catch (Exception e) { @@ -296,7 +305,7 @@ private Map doANNSearch(final LeafReaderContext context, final i .collect(Collectors.toMap(KNNQueryResult::getId, result -> knnEngine.score(result.getScore(), spaceType))); } - private Map doExactSearch(final LeafReaderContext leafReaderContext, final int[] filterIdsArray) { + private Map doExactSearch(final LeafReaderContext leafReaderContext, final int[] filterIdsArray) throws IOException { final SegmentReader reader = (SegmentReader) FilterLeafReader.unwrap(leafReaderContext.reader()); final FieldInfo fieldInfo = reader.getFieldInfos().fieldInfo(knnQuery.getField()); float[] queryVector = this.knnQuery.getQueryVector(); diff --git a/src/main/java/org/opensearch/knn/jni/FaissService.java b/src/main/java/org/opensearch/knn/jni/FaissService.java index 5dce15d6e..abf3e052a 100644 --- a/src/main/java/org/opensearch/knn/jni/FaissService.java +++ b/src/main/java/org/opensearch/knn/jni/FaissService.java @@ -74,16 +74,39 @@ public static native void createIndexFromTemplate( public static native long loadIndex(String indexPath); /** - * Query an index + * Query an index without filter + * + * If the "knn" field is a nested field, each vector value within that nested field will be assigned its + * own document ID. In this situation, the term "parent ID" corresponds to the original document ID. + * The arrangement of parent IDs and nested field IDs is assured to have all nested field IDs appearing first, + * followed by the parent ID, in consecutive order without any gaps. Because of this ID pattern, + * we can determine the parent ID of a specific nested field ID using only an array of parent IDs. * * @param indexPointer pointer to index in memory * @param queryVector vector to be used for query * @param k neighbors to be returned + * @param parentIds list of parent doc ids when the knn field is a nested field * @return KNNQueryResult array of k neighbors */ - public static native KNNQueryResult[] queryIndex(long indexPointer, float[] queryVector, int k); + public static native KNNQueryResult[] queryIndex(long indexPointer, float[] queryVector, int k, int[] parentIds); - public static native KNNQueryResult[] queryIndexWithFilter(long indexPointer, float[] queryVector, int k, int[] filterIds); + /** + * Query an index with filter + * + * @param indexPointer pointer to index in memory + * @param queryVector vector to be used for query + * @param k neighbors to be returned + * @param filterIds list of doc ids to include in the query result + * @param parentIds list of parent doc ids when the knn field is a nested field + * @return KNNQueryResult array of k neighbors + */ + public static native KNNQueryResult[] queryIndexWithFilter( + long indexPointer, + float[] queryVector, + int k, + int[] filterIds, + int[] parentIds + ); /** * Free native memory pointer diff --git a/src/main/java/org/opensearch/knn/jni/JNIService.java b/src/main/java/org/opensearch/knn/jni/JNIService.java index f45fb0c73..beef9f927 100644 --- a/src/main/java/org/opensearch/knn/jni/JNIService.java +++ b/src/main/java/org/opensearch/knn/jni/JNIService.java @@ -101,7 +101,14 @@ public static long loadIndex(String indexPath, Map parameters, S * @param filteredIds array of ints on which should be used for search. * @return KNNQueryResult array of k neighbors */ - public static KNNQueryResult[] queryIndex(long indexPointer, float[] queryVector, int k, String engineName, int[] filteredIds) { + public static KNNQueryResult[] queryIndex( + long indexPointer, + float[] queryVector, + int k, + String engineName, + int[] filteredIds, + int[] parentIds + ) { if (KNNEngine.NMSLIB.getName().equals(engineName)) { return NmslibService.queryIndex(indexPointer, queryVector, k); } @@ -112,9 +119,9 @@ public static KNNQueryResult[] queryIndex(long indexPointer, float[] queryVector // filterIds. FilterIds is coming as empty then its the case where we need to do search with Faiss engine // normally. if (ArrayUtils.isNotEmpty(filteredIds)) { - return FaissService.queryIndexWithFilter(indexPointer, queryVector, k, filteredIds); + return FaissService.queryIndexWithFilter(indexPointer, queryVector, k, filteredIds, parentIds); } - return FaissService.queryIndex(indexPointer, queryVector, k); + return FaissService.queryIndex(indexPointer, queryVector, k, parentIds); } throw new IllegalArgumentException("QueryIndex not supported for provided engine"); } diff --git a/src/test/java/org/opensearch/knn/index/NestedSearchIT.java b/src/test/java/org/opensearch/knn/index/NestedSearchIT.java index 21f3bb27a..df5256fb2 100644 --- a/src/test/java/org/opensearch/knn/index/NestedSearchIT.java +++ b/src/test/java/org/opensearch/knn/index/NestedSearchIT.java @@ -12,15 +12,12 @@ import org.opensearch.client.Response; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.rest.RestStatus; -import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.knn.KNNRestTestCase; import org.opensearch.knn.NestedKnnDocBuilder; import org.opensearch.knn.index.util.KNNEngine; import java.io.IOException; -import java.util.List; -import java.util.Map; import static org.opensearch.knn.common.KNNConstants.DIMENSION; import static org.opensearch.knn.common.KNNConstants.K; @@ -56,7 +53,7 @@ public final void cleanUp() { } @SneakyThrows - public void testNestedSearch_whenKIsTwo_thenReturnTwoResults() { + public void testNestedSearchWithLucene_whenKIsTwo_thenReturnTwoResults() { createKnnIndex(2, KNNEngine.LUCENE.getName()); String doc1 = NestedKnnDocBuilder.create(FIELD_NAME_NESTED) @@ -73,12 +70,32 @@ public void testNestedSearch_whenKIsTwo_thenReturnTwoResults() { Float[] queryVector = { 1f, 1f }; Response response = queryNestedField(INDEX_NAME, 2, queryVector); + String entity = EntityUtils.toString(response.getEntity()); + assertEquals(2, parseHits(entity)); + assertEquals(2, parseTotalSearchHits(entity)); + } + + @SneakyThrows + public void testNestedSearchWithFaiss_whenKIsTwo_thenReturnTwoResults() { + createKnnIndex(2, KNNEngine.FAISS.getName()); - List hits = (List) ((Map) createParser( - MediaTypeRegistry.getDefaultMediaType().xContent(), - EntityUtils.toString(response.getEntity()) - ).map().get("hits")).get("hits"); - assertEquals(2, hits.size()); + String doc1 = NestedKnnDocBuilder.create(FIELD_NAME_NESTED) + .addVectors(FIELD_NAME_VECTOR, new Float[] { 1f, 1f }, new Float[] { 1f, 1f }) + .build(); + addKnnDoc(INDEX_NAME, "1", doc1); + + String doc2 = NestedKnnDocBuilder.create(FIELD_NAME_NESTED) + .addVectors(FIELD_NAME_VECTOR, new Float[] { 2f, 2f }, new Float[] { 2f, 2f }) + .build(); + addKnnDoc(INDEX_NAME, "2", doc2); + + refreshIndex(INDEX_NAME); + + Float[] queryVector = { 1f, 1f }; + Response response = queryNestedField(INDEX_NAME, 2, queryVector); + String entity = EntityUtils.toString(response.getEntity()); + assertEquals(2, parseHits(entity)); + assertEquals(2, parseTotalSearchHits(entity)); } /** diff --git a/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestCase.java b/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestCase.java index 40309027d..42eb81759 100644 --- a/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestCase.java +++ b/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestCase.java @@ -13,6 +13,7 @@ import org.apache.lucene.index.VectorSimilarityFunction; import org.apache.lucene.search.Query; import org.apache.lucene.search.TopDocs; +import org.apache.lucene.search.join.BitSetProducer; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Setting; import org.opensearch.index.mapper.MapperService; @@ -162,14 +163,20 @@ public void testMultiFieldsKnnIndex(Codec codec) throws Exception { // query to verify distance for each of the field IndexSearcher searcher = new IndexSearcher(reader); - float score = searcher.search(new KNNQuery("test_vector", new float[] { 1.0f, 0.0f, 0.0f }, 1, "dummy"), 10).scoreDocs[0].score; - float score1 = searcher.search(new KNNQuery("my_vector", new float[] { 1.0f, 2.0f }, 1, "dummy"), 10).scoreDocs[0].score; + float score = searcher.search( + new KNNQuery("test_vector", new float[] { 1.0f, 0.0f, 0.0f }, 1, "dummy", (BitSetProducer) null), + 10 + ).scoreDocs[0].score; + float score1 = searcher.search( + new KNNQuery("my_vector", new float[] { 1.0f, 2.0f }, 1, "dummy", (BitSetProducer) null), + 10 + ).scoreDocs[0].score; assertEquals(1.0f / (1 + 25), score, 0.01f); assertEquals(1.0f / (1 + 169), score1, 0.01f); // query to determine the hits - assertEquals(1, searcher.count(new KNNQuery("test_vector", new float[] { 1.0f, 0.0f, 0.0f }, 1, "dummy"))); - assertEquals(1, searcher.count(new KNNQuery("my_vector", new float[] { 1.0f, 1.0f }, 1, "dummy"))); + assertEquals(1, searcher.count(new KNNQuery("test_vector", new float[] { 1.0f, 0.0f, 0.0f }, 1, "dummy", (BitSetProducer) null))); + assertEquals(1, searcher.count(new KNNQuery("my_vector", new float[] { 1.0f, 1.0f }, 1, "dummy", (BitSetProducer) null))); reader.close(); dir.close(); @@ -254,7 +261,7 @@ public void testBuildFromModelTemplate(Codec codec) throws IOException, Executio NativeMemoryLoadStrategy.IndexLoadStrategy.initialize(resourceWatcherService); float[] query = { 10.0f, 10.0f, 10.0f }; IndexSearcher searcher = new IndexSearcher(reader); - TopDocs topDocs = searcher.search(new KNNQuery(fieldName, query, 4, "dummy"), 10); + TopDocs topDocs = searcher.search(new KNNQuery(fieldName, query, 4, "dummy", (BitSetProducer) null), 10); assertEquals(3, topDocs.scoreDocs[0].doc); assertEquals(2, topDocs.scoreDocs[1].doc); diff --git a/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestUtil.java b/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestUtil.java index 08dedb0e7..e602501cc 100644 --- a/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestUtil.java +++ b/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestUtil.java @@ -333,7 +333,7 @@ public static void assertLoadableByEngine( ); int k = 2; float[] queryVector = new float[dimension]; - KNNQueryResult[] results = JNIService.queryIndex(indexPtr, queryVector, k, knnEngine.getName(), null); + KNNQueryResult[] results = JNIService.queryIndex(indexPtr, queryVector, k, knnEngine.getName(), null, null); assertTrue(results.length > 0); JNIService.free(indexPtr, knnEngine.getName()); } diff --git a/src/test/java/org/opensearch/knn/index/memory/NativeMemoryLoadStrategyTests.java b/src/test/java/org/opensearch/knn/index/memory/NativeMemoryLoadStrategyTests.java index fb91266ab..798c64a17 100644 --- a/src/test/java/org/opensearch/knn/index/memory/NativeMemoryLoadStrategyTests.java +++ b/src/test/java/org/opensearch/knn/index/memory/NativeMemoryLoadStrategyTests.java @@ -74,7 +74,7 @@ public void testIndexLoadStrategy_load() throws IOException { // Confirm that the file was loaded by querying float[] query = new float[dimension]; Arrays.fill(query, numVectors + 1); - KNNQueryResult[] results = JNIService.queryIndex(indexAllocation.getMemoryAddress(), query, 2, knnEngine.getName(), null); + KNNQueryResult[] results = JNIService.queryIndex(indexAllocation.getMemoryAddress(), query, 2, knnEngine.getName(), null, null); assertTrue(results.length > 0); } diff --git a/src/test/java/org/opensearch/knn/index/query/KNNQueryFactoryTests.java b/src/test/java/org/opensearch/knn/index/query/KNNQueryFactoryTests.java index 62d2db544..1bb17cfae 100644 --- a/src/test/java/org/opensearch/knn/index/query/KNNQueryFactoryTests.java +++ b/src/test/java/org/opensearch/knn/index/query/KNNQueryFactoryTests.java @@ -191,6 +191,30 @@ public void testCreate_whenNestedVectorAndFilterField_thenReturnSameFilterQuery( assertEquals(FILTER_QUERY.getClass(), query.getFilterQuery().getClass()); } + public void testCreate_whenFaissWithParentFilter_thenSuccess() { + final KNNEngine knnEngine = KNNEngine.FAISS; + QueryShardContext mockQueryShardContext = mock(QueryShardContext.class); + MappedFieldType testMapper = mock(MappedFieldType.class); + when(mockQueryShardContext.fieldMapper(any())).thenReturn(testMapper); + BitSetProducer parentFilter = mock(BitSetProducer.class); + when(mockQueryShardContext.getParentFilter()).thenReturn(parentFilter); + final KNNQueryFactory.CreateQueryRequest createQueryRequest = KNNQueryFactory.CreateQueryRequest.builder() + .knnEngine(knnEngine) + .indexName(testIndexName) + .fieldName(testFieldName) + .vector(testQueryVector) + .k(testK) + .context(mockQueryShardContext) + .build(); + final Query query = KNNQueryFactory.create(createQueryRequest); + assertTrue(query instanceof KNNQuery); + assertEquals(testIndexName, ((KNNQuery) query).getIndexName()); + assertEquals(testFieldName, ((KNNQuery) query).getField()); + assertEquals(testQueryVector, ((KNNQuery) query).getQueryVector()); + assertEquals(testK, ((KNNQuery) query).getK()); + assertEquals(parentFilter, ((KNNQuery) query).getParentsFilter()); + } + private void validateDiversifyingQueryWithParentFilter(final VectorDataType type, final Class expectedQueryClass) { List luceneDefaultQueryEngineList = Arrays.stream(KNNEngine.values()) .filter(knnEngine -> !KNNEngine.getEnginesThatCreateCustomSegmentFiles().contains(knnEngine)) diff --git a/src/test/java/org/opensearch/knn/index/query/KNNWeightTests.java b/src/test/java/org/opensearch/knn/index/query/KNNWeightTests.java index a71f25822..fa0613921 100644 --- a/src/test/java/org/opensearch/knn/index/query/KNNWeightTests.java +++ b/src/test/java/org/opensearch/knn/index/query/KNNWeightTests.java @@ -22,9 +22,11 @@ import org.apache.lucene.search.Sort; import org.apache.lucene.search.TermQuery; import org.apache.lucene.search.Weight; +import org.apache.lucene.search.join.BitSetProducer; import org.apache.lucene.store.FSDirectory; import org.apache.lucene.util.Bits; import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.FixedBitSet; import org.apache.lucene.util.StringHelper; import org.apache.lucene.util.Version; import org.junit.Before; @@ -48,6 +50,7 @@ import java.io.IOException; import java.nio.file.Path; import java.util.ArrayList; +import java.util.Arrays; import java.util.Comparator; import java.util.List; import java.util.Map; @@ -150,14 +153,14 @@ public void testQueryResultScoreFaiss() { } @SneakyThrows - public void testQueryScoreForFaissWithModel() throws IOException { + public void testQueryScoreForFaissWithModel() { SpaceType spaceType = SpaceType.L2; final Function scoreTranslator = spaceType::scoreTranslation; final String modelId = "modelId"; - jniServiceMockedStatic.when(() -> JNIService.queryIndex(anyLong(), any(), anyInt(), anyString(), any())) + jniServiceMockedStatic.when(() -> JNIService.queryIndex(anyLong(), any(), anyInt(), anyString(), any(), any())) .thenReturn(getKNNQueryResults()); - final KNNQuery query = new KNNQuery(FIELD_NAME, QUERY_VECTOR, K, INDEX_NAME); + final KNNQuery query = new KNNQuery(FIELD_NAME, QUERY_VECTOR, K, INDEX_NAME, (BitSetProducer) null); ModelDao modelDao = mock(ModelDao.class); ModelMetadata modelMetadata = mock(ModelMetadata.class); @@ -222,7 +225,7 @@ public void testQueryScoreForFaissWithNonExistingModel() throws IOException { SpaceType spaceType = SpaceType.L2; final String modelId = "modelId"; - final KNNQuery query = new KNNQuery(FIELD_NAME, QUERY_VECTOR, K, INDEX_NAME); + final KNNQuery query = new KNNQuery(FIELD_NAME, QUERY_VECTOR, K, INDEX_NAME, (BitSetProducer) null); ModelDao modelDao = mock(ModelDao.class); ModelMetadata modelMetadata = mock(ModelMetadata.class); @@ -254,7 +257,7 @@ public void testQueryScoreForFaissWithNonExistingModel() throws IOException { @SneakyThrows public void testShardWithoutFiles() { - final KNNQuery query = new KNNQuery(FIELD_NAME, QUERY_VECTOR, K, INDEX_NAME); + final KNNQuery query = new KNNQuery(FIELD_NAME, QUERY_VECTOR, K, INDEX_NAME, (BitSetProducer) null); final KNNWeight knnWeight = new KNNWeight(query, 0.0f); final LeafReaderContext leafReaderContext = mock(LeafReaderContext.class); @@ -296,10 +299,10 @@ public void testShardWithoutFiles() { @SneakyThrows public void testEmptyQueryResults() { final KNNQueryResult[] knnQueryResults = new KNNQueryResult[] {}; - jniServiceMockedStatic.when(() -> JNIService.queryIndex(anyLong(), any(), anyInt(), anyString(), any())) + jniServiceMockedStatic.when(() -> JNIService.queryIndex(anyLong(), any(), anyInt(), anyString(), any(), any())) .thenReturn(knnQueryResults); - final KNNQuery query = new KNNQuery(FIELD_NAME, QUERY_VECTOR, K, INDEX_NAME); + final KNNQuery query = new KNNQuery(FIELD_NAME, QUERY_VECTOR, K, INDEX_NAME, (BitSetProducer) null); final KNNWeight knnWeight = new KNNWeight(query, 0.0f); final LeafReaderContext leafReaderContext = mock(LeafReaderContext.class); @@ -341,7 +344,7 @@ public void testEmptyQueryResults() { public void testANNWithFilterQuery_whenDoingANN_thenSuccess() { int k = 3; final int[] filterDocIds = new int[] { 0, 1, 2, 3, 4, 5 }; - jniServiceMockedStatic.when(() -> JNIService.queryIndex(anyLong(), any(), anyInt(), anyString(), eq(filterDocIds))) + jniServiceMockedStatic.when(() -> JNIService.queryIndex(anyLong(), any(), anyInt(), anyString(), eq(filterDocIds), any())) .thenReturn(getFilteredKNNQueryResults()); final LeafReaderContext leafReaderContext = mock(LeafReaderContext.class); final SegmentReader reader = mock(SegmentReader.class); @@ -354,7 +357,7 @@ public void testANNWithFilterQuery_whenDoingANN_thenSuccess() { when(liveDocsBits.length()).thenReturn(1000); when(leafReaderContext.reader()).thenReturn(reader); - final KNNQuery query = new KNNQuery(FIELD_NAME, QUERY_VECTOR, k, INDEX_NAME, FILTER_QUERY); + final KNNQuery query = new KNNQuery(FIELD_NAME, QUERY_VECTOR, k, INDEX_NAME, FILTER_QUERY, null); final Weight filterQueryWeight = mock(Weight.class); final Scorer filterScorer = mock(Scorer.class); when(filterQueryWeight.scorer(leafReaderContext)).thenReturn(filterScorer); @@ -399,7 +402,7 @@ public void testANNWithFilterQuery_whenDoingANN_thenSuccess() { final DocIdSetIterator docIdSetIterator = knnScorer.iterator(); assertNotNull(docIdSetIterator); assertEquals(FILTERED_DOC_ID_TO_SCORES.size(), docIdSetIterator.cost()); - jniServiceMockedStatic.verify(() -> JNIService.queryIndex(anyLong(), any(), anyInt(), anyString(), eq(filterDocIds))); + jniServiceMockedStatic.verify(() -> JNIService.queryIndex(anyLong(), any(), anyInt(), anyString(), eq(filterDocIds), any())); final List actualDocIds = new ArrayList<>(); final Map translatedScores = getTranslatedScores(SpaceType.L2::scoreTranslation); @@ -419,7 +422,7 @@ public void testANNWithFilterQuery_whenExactSearch_thenSuccess() { final SegmentReader reader = mock(SegmentReader.class); when(leafReaderContext.reader()).thenReturn(reader); - final KNNQuery query = new KNNQuery(FIELD_NAME, QUERY_VECTOR, K, INDEX_NAME, FILTER_QUERY); + final KNNQuery query = new KNNQuery(FIELD_NAME, QUERY_VECTOR, K, INDEX_NAME, FILTER_QUERY, null); final Weight filterQueryWeight = mock(Weight.class); final Scorer filterScorer = mock(Scorer.class); when(filterQueryWeight.scorer(leafReaderContext)).thenReturn(filterScorer); @@ -469,7 +472,7 @@ public void testANNWithFilterQuery_whenExactSearchAndThresholdComputations_thenS final SegmentReader reader = mock(SegmentReader.class); when(leafReaderContext.reader()).thenReturn(reader); - final KNNQuery query = new KNNQuery(FIELD_NAME, QUERY_VECTOR, K, INDEX_NAME, FILTER_QUERY); + final KNNQuery query = new KNNQuery(FIELD_NAME, QUERY_VECTOR, K, INDEX_NAME, FILTER_QUERY, null); final Weight filterQueryWeight = mock(Weight.class); final Scorer filterScorer = mock(Scorer.class); when(filterQueryWeight.scorer(leafReaderContext)).thenReturn(filterScorer); @@ -538,7 +541,7 @@ public void testANNWithFilterQuery_whenExactSearchViaThresholdSetting_thenSucces when(filterScorer.iterator()).thenReturn(DocIdSetIterator.all(filterDocIds.length)); - final KNNQuery query = new KNNQuery(FIELD_NAME, QUERY_VECTOR, k, INDEX_NAME, FILTER_QUERY); + final KNNQuery query = new KNNQuery(FIELD_NAME, QUERY_VECTOR, k, INDEX_NAME, FILTER_QUERY, null); final KNNWeight knnWeight = new KNNWeight(query, 0.0f, filterQueryWeight); final Map attributesMap = ImmutableMap.of(KNN_ENGINE, KNNEngine.FAISS.getName(), SPACE_TYPE, SpaceType.L2.name()); @@ -581,7 +584,7 @@ public void testANNWithFilterQuery_whenEmptyFilterIds_thenReturnEarly() { when(filterQueryWeight.scorer(leafReaderContext)).thenReturn(filterScorer); when(filterScorer.iterator()).thenReturn(DocIdSetIterator.empty()); - final KNNQuery query = new KNNQuery(FIELD_NAME, QUERY_VECTOR, K, INDEX_NAME, FILTER_QUERY); + final KNNQuery query = new KNNQuery(FIELD_NAME, QUERY_VECTOR, K, INDEX_NAME, FILTER_QUERY, null); final KNNWeight knnWeight = new KNNWeight(query, 0.0f, filterQueryWeight); final FieldInfos fieldInfos = mock(FieldInfos.class); @@ -597,15 +600,93 @@ public void testANNWithFilterQuery_whenEmptyFilterIds_thenReturnEarly() { assertEquals(0, docIdSetIterator.cost()); } + @SneakyThrows + public void testANNWithParentsFilter_whenSet_thenBitSetIsPassedToJNI() { + SegmentReader reader = getMockedSegmentReader(); + final LeafReaderContext leafReaderContext = mock(LeafReaderContext.class); + when(leafReaderContext.reader()).thenReturn(reader); + + // Prepare parentFilter + final int[] parentsFilter = { 10, 64 }; + final FixedBitSet bitset = new FixedBitSet(65); + Arrays.stream(parentsFilter).forEach(i -> bitset.set(i)); + final BitSetProducer bitSetProducer = mock(BitSetProducer.class); + + // Prepare query and weight + when(bitSetProducer.getBitSet(leafReaderContext)).thenReturn(bitset); + final KNNQuery query = new KNNQuery(FIELD_NAME, QUERY_VECTOR, 1, INDEX_NAME, null, bitSetProducer); + final KNNWeight knnWeight = new KNNWeight(query, 0.0f, null); + + jniServiceMockedStatic.when(() -> JNIService.queryIndex(anyLong(), any(), anyInt(), anyString(), any(), eq(parentsFilter))) + .thenReturn(getKNNQueryResults()); + + // Execute + Scorer knnScorer = knnWeight.scorer(leafReaderContext); + + // Verify + jniServiceMockedStatic.verify(() -> JNIService.queryIndex(anyLong(), any(), anyInt(), anyString(), any(), eq(parentsFilter))); + assertNotNull(knnScorer); + final DocIdSetIterator docIdSetIterator = knnScorer.iterator(); + assertNotNull(docIdSetIterator); + assertEquals(DOC_ID_TO_SCORES.size(), docIdSetIterator.cost()); + } + + private SegmentReader getMockedSegmentReader() { + final SegmentReader reader = mock(SegmentReader.class); + when(reader.maxDoc()).thenReturn(1); + + // Prepare live docs + int liveDocId = 0; + final Bits liveDocsBits = mock(Bits.class); + when(liveDocsBits.get(liveDocId)).thenReturn(true); + when(liveDocsBits.length()).thenReturn(1); + when(reader.getLiveDocs()).thenReturn(liveDocsBits); + + // Prepare directory + final Path path = mock(Path.class); + final FSDirectory directory = mock(FSDirectory.class); + when(directory.getDirectory()).thenReturn(path); + when(reader.directory()).thenReturn(directory); + + // Prepare segment + final SegmentInfo segmentInfo = new SegmentInfo( + directory, + Version.LATEST, + Version.LATEST, + SEGMENT_NAME, + 100, + true, + false, + KNNCodecVersion.current().getDefaultCodecDelegate(), + Map.of(), + new byte[StringHelper.ID_LENGTH], + Map.of(), + Sort.RELEVANCE + ); + segmentInfo.setFiles(SEGMENT_FILES_FAISS); + final SegmentCommitInfo segmentCommitInfo = new SegmentCommitInfo(segmentInfo, 0, 0, 0, 0, 0, new byte[StringHelper.ID_LENGTH]); + when(reader.getSegmentInfo()).thenReturn(segmentCommitInfo); + + // Prepare fieldInfos + final Map attributesMap = ImmutableMap.of(KNN_ENGINE, KNNEngine.FAISS.getName()); + final FieldInfo fieldInfo = mock(FieldInfo.class); + when(fieldInfo.attributes()).thenReturn(attributesMap); + final FieldInfos fieldInfos = mock(FieldInfos.class); + when(fieldInfos.fieldInfo(any())).thenReturn(fieldInfo); + when(reader.getFieldInfos()).thenReturn(fieldInfos); + + return reader; + } + private void testQueryScore( final Function scoreTranslator, final Set segmentFiles, final Map fileAttributes ) throws IOException { - jniServiceMockedStatic.when(() -> JNIService.queryIndex(anyLong(), any(), anyInt(), anyString(), any())) + jniServiceMockedStatic.when(() -> JNIService.queryIndex(anyLong(), any(), anyInt(), anyString(), any(), any())) .thenReturn(getKNNQueryResults()); - final KNNQuery query = new KNNQuery(FIELD_NAME, QUERY_VECTOR, K, INDEX_NAME); + final KNNQuery query = new KNNQuery(FIELD_NAME, QUERY_VECTOR, K, INDEX_NAME, (BitSetProducer) null); final KNNWeight knnWeight = new KNNWeight(query, 0.0f); final LeafReaderContext leafReaderContext = mock(LeafReaderContext.class); diff --git a/src/test/java/org/opensearch/knn/jni/JNIServiceTests.java b/src/test/java/org/opensearch/knn/jni/JNIServiceTests.java index 185f2953d..0a05c95a0 100644 --- a/src/test/java/org/opensearch/knn/jni/JNIServiceTests.java +++ b/src/test/java/org/opensearch/knn/jni/JNIServiceTests.java @@ -29,9 +29,14 @@ import java.io.IOException; import java.net.URL; import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; import static org.opensearch.knn.common.KNNConstants.ENCODER_PARAMETER_PQ_M; import static org.opensearch.knn.common.KNNConstants.ENCODER_PARAMETER_PQ_CODE_SIZE; @@ -50,15 +55,19 @@ public class JNIServiceTests extends KNNTestCase { static TestUtils.TestData testData; + static TestUtils.TestData testDataNested; private String faissMethod = "HNSW32,Flat"; @BeforeClass public static void setUpClass() throws IOException { URL testIndexVectors = JNIServiceTests.class.getClassLoader().getResource("data/test_vectors_1000x128.json"); + URL testIndexVectorsNested = JNIServiceTests.class.getClassLoader().getResource("data/test_vectors_nested_1000x128.json"); URL testQueries = JNIServiceTests.class.getClassLoader().getResource("data/test_queries_100x128.csv"); assert testIndexVectors != null; + assert testIndexVectorsNested != null; assert testQueries != null; testData = new TestUtils.TestData(testIndexVectors.getPath(), testQueries.getPath()); + testDataNested = new TestUtils.TestData(testIndexVectorsNested.getPath(), testQueries.getPath()); } public void testCreateIndex_invalid_engineNotSupported() { @@ -590,12 +599,12 @@ public void testLoadIndex_faiss_valid() throws IOException { } public void testQueryIndex_invalidEngine() { - expectThrows(IllegalArgumentException.class, () -> JNIService.queryIndex(0L, new float[] {}, 0, "invalid" + "-engine", null)); + expectThrows(IllegalArgumentException.class, () -> JNIService.queryIndex(0L, new float[] {}, 0, "invalid" + "-engine", null, null)); } public void testQueryIndex_nmslib_invalid_badPointer() { - expectThrows(Exception.class, () -> JNIService.queryIndex(0L, new float[] {}, 0, KNNEngine.NMSLIB.getName(), null)); + expectThrows(Exception.class, () -> JNIService.queryIndex(0L, new float[] {}, 0, KNNEngine.NMSLIB.getName(), null, null)); } public void testQueryIndex_nmslib_invalid_nullQueryVector() throws IOException { @@ -618,7 +627,7 @@ public void testQueryIndex_nmslib_invalid_nullQueryVector() throws IOException { ); assertNotEquals(0, pointer); - expectThrows(Exception.class, () -> JNIService.queryIndex(pointer, null, 10, KNNEngine.NMSLIB.getName(), null)); + expectThrows(Exception.class, () -> JNIService.queryIndex(pointer, null, 10, KNNEngine.NMSLIB.getName(), null, null)); } public void testQueryIndex_nmslib_valid() throws IOException { @@ -644,7 +653,7 @@ public void testQueryIndex_nmslib_valid() throws IOException { assertNotEquals(0, pointer); for (float[] query : testData.queries) { - KNNQueryResult[] results = JNIService.queryIndex(pointer, query, k, KNNEngine.NMSLIB.getName(), null); + KNNQueryResult[] results = JNIService.queryIndex(pointer, query, k, KNNEngine.NMSLIB.getName(), null, null); assertEquals(k, results.length); } } @@ -652,7 +661,7 @@ public void testQueryIndex_nmslib_valid() throws IOException { public void testQueryIndex_faiss_invalid_badPointer() { - expectThrows(Exception.class, () -> JNIService.queryIndex(0L, new float[] {}, 0, FAISS_NAME, null)); + expectThrows(Exception.class, () -> JNIService.queryIndex(0L, new float[] {}, 0, FAISS_NAME, null, null)); } public void testQueryIndex_faiss_invalid_nullQueryVector() throws IOException { @@ -671,7 +680,7 @@ public void testQueryIndex_faiss_invalid_nullQueryVector() throws IOException { long pointer = JNIService.loadIndex(tmpFile.toAbsolutePath().toString(), Collections.emptyMap(), FAISS_NAME); assertNotEquals(0, pointer); - expectThrows(Exception.class, () -> JNIService.queryIndex(pointer, null, 10, FAISS_NAME, null)); + expectThrows(Exception.class, () -> JNIService.queryIndex(pointer, null, 10, FAISS_NAME, null, null)); } public void testQueryIndex_faiss_valid() throws IOException { @@ -700,19 +709,96 @@ public void testQueryIndex_faiss_valid() throws IOException { assertNotEquals(0, pointer); for (float[] query : testData.queries) { - KNNQueryResult[] results = JNIService.queryIndex(pointer, query, k, FAISS_NAME, null); + KNNQueryResult[] results = JNIService.queryIndex(pointer, query, k, FAISS_NAME, null, null); assertEquals(k, results.length); } // Filter will result in no ids for (float[] query : testData.queries) { - KNNQueryResult[] results = JNIService.queryIndex(pointer, query, k, FAISS_NAME, new int[] { 0 }); + KNNQueryResult[] results = JNIService.queryIndex(pointer, query, k, FAISS_NAME, new int[] { 0 }, null); assertEquals(0, results.length); } } } } + public void testQueryIndex_faiss_parentIds() throws IOException { + + int k = 100; + + List methods = ImmutableList.of(faissMethod); + List spaces = ImmutableList.of(SpaceType.L2, SpaceType.INNER_PRODUCT); + int[] parentIds = toParentIdArray(testDataNested.indexData.docs); + Map idToParentIdMap = toIdToParentIdMap(testDataNested.indexData.docs); + for (String method : methods) { + for (SpaceType spaceType : spaces) { + Path tmpFile = createTempFile(); + JNIService.createIndex( + testDataNested.indexData.docs, + testDataNested.indexData.vectors, + tmpFile.toAbsolutePath().toString(), + ImmutableMap.of(INDEX_DESCRIPTION_PARAMETER, method, KNNConstants.SPACE_TYPE, spaceType.getValue()), + FAISS_NAME + ); + assertTrue(tmpFile.toFile().length() > 0); + + long pointer = JNIService.loadIndex( + tmpFile.toAbsolutePath().toString(), + ImmutableMap.of(KNNConstants.SPACE_TYPE, spaceType.getValue()), + FAISS_NAME + ); + assertNotEquals(0, pointer); + + for (float[] query : testDataNested.queries) { + KNNQueryResult[] results = JNIService.queryIndex(pointer, query, k, FAISS_NAME, null, parentIds); + // Verify there is no more than one result from same parent + Set parentIdSet = toParentIdSet(results, idToParentIdMap); + assertEquals(results.length, parentIdSet.size()); + } + } + } + } + + private Set toParentIdSet(KNNQueryResult[] results, Map idToParentIdMap) { + return Arrays.stream(results).map(result -> idToParentIdMap.get(result.getId())).collect(Collectors.toSet()); + } + + private int[] toParentIdArray(int[] ids) { + int length = ids.length; + int[] sortedIds = Arrays.copyOf(ids, length); + Arrays.sort(sortedIds); + + List parentIds = new ArrayList<>(); + int largestId = sortedIds[length - 1]; + parentIds.add(largestId + 1); + for (int i = length - 2; i > -1; i--) { + if (sortedIds[i] != sortedIds[i + 1] - 1) { + parentIds.add(sortedIds[i] + 1); + } + } + + Collections.shuffle(parentIds); + return parentIds.stream().mapToInt(Integer::intValue).toArray(); + } + + private Map toIdToParentIdMap(int[] ids) { + int length = ids.length; + int[] sortedIds = Arrays.copyOf(ids, length); + Arrays.sort(sortedIds); + + Map idToParentIdMap = new HashMap<>(); + int largestId = sortedIds[length - 1]; + int parentId = largestId + 1; + idToParentIdMap.put(largestId, parentId); + for (int i = length - 2; i > -1; i--) { + if (sortedIds[i] != sortedIds[i + 1] - 1) { + parentId = sortedIds[i] + 1; + } + idToParentIdMap.put(sortedIds[i], parentId); + } + return idToParentIdMap; + } + public void testFree_invalidEngine() { expectThrows(IllegalArgumentException.class, () -> JNIService.free(0L, "invalid-engine")); } diff --git a/src/test/resources/data/README.md b/src/test/resources/data/README.md new file mode 100644 index 000000000..8d62d361d --- /dev/null +++ b/src/test/resources/data/README.md @@ -0,0 +1,6 @@ +# test_vectors_nested_1000x128.json +The file contains a simulated data to represent nested field. +Consecutive ids are assigned for data from same parent document. +For example, let's say there are ids of 2, 3, 4, 5, 10, 11, 12. +Then, the 2, 3, 4, 5 are ids of nested field in parent document with id 6 and 10, 11, 12 are ids of nested field +in parent document with id 13. \ No newline at end of file diff --git a/src/test/resources/data/test_vectors_nested_1000x128.json b/src/test/resources/data/test_vectors_nested_1000x128.json new file mode 100644 index 000000000..0dffc35de --- /dev/null +++ b/src/test/resources/data/test_vectors_nested_1000x128.json @@ -0,0 +1,1000 @@ +{"id": 322, "vector": [99068.50376381158, 88573.2072597422, 15853.064675079664, 28632.989469555538, 35982.04456596793, 17738.390408696003, 81579.19998990816, 83550.36160149651, 16172.577774468078, 12407.173011917082, 62466.586202863895, 14944.28048334243, 51895.63040821629, 67411.49186977767, 82874.69624450797, 70707.40727542483, 54196.68741962424, 78245.5888156245, 52369.250870967946, 65187.2164525083, 13661.213228264323, 7040.9589866832985, 52418.47489527644, 34149.32870217484, 23047.848728567198, 9304.158200408918, 50714.429512049086, 27010.477967660252, 86000.97337722874, 19262.619217617048, 9025.864553964348, 52005.39507542871, 97222.62476129965, 12068.742178114533, 13160.922768044004, 79288.55247190826, 17289.211990437892, 17174.8624999444, 34737.322505056254, 27604.347388084192, 73737.57504407635, 10400.290963994252, 24774.79771851735, 81682.86000017286, 72730.83555053356, 4914.223178227384, 87645.81997374613, 87089.3814877381, 66771.76429098121, 76129.03264603007, 25307.513408494255, 21826.91796770343, 59295.44771322758, 72510.33111259321, 60536.414950457984, 56120.82790694709, 94411.84525648673, 91670.3096586618, 32349.549287113543, 32952.103729973416, 5108.999867738706, 98370.51585049079, 59725.65333734072, 33108.0741272476, 69122.69592593945, 96699.04035309899, 82247.36640939685, 97635.28044033359, 54850.6938342168, 13681.643886540583, 41755.70190007814, 54582.94245407739, 43095.66746738402, 67773.99769093192, 71645.7331194456, 86773.25308830473, 70885.46423317467, 91737.86055941659, 87067.17726086797, 52777.694477546145, 98655.09331800397, 80226.62727292119, 59952.330623489914, 12260.897754625055, 54474.004086384484, 4954.9713299904315, 78447.24339101047, 87801.74515242745, 37016.76744235859, 59024.69837713164, 73863.08146016131, 73000.37209498162, 77376.45825121911, 81444.46246162163, 70344.649517354, 11516.665047529506, 82911.03643269501, 60083.34823774171, 54022.7244240484, 30881.786721398606, 90916.90296596587, 26634.436329025557, 98446.81454349941, 45057.85079473461, 46184.386953657355, 77884.38584874007, 96437.1547694247, 76501.62101631636, 98932.33322067512, 25925.705275967426, 15356.58805391542, 61202.14624958974, 2421.288542598132, 12535.995455880899, 69540.69649848856, 72143.37123737848, 20262.992060173845, 92348.37537285395, 84848.42968418587, 48787.159705339625, 90914.29958986014, 92254.20209409502, 8527.774382362362, 38954.90398963415, 37422.72317763923, 76494.31213884517, 74822.10049643254, 34750.53819160653]} +{"id": 571, "vector": [45518.91613130162, 26576.640067248238, 47907.61270098416, 96055.72494689011, 19753.506751210105, 20627.621096022718, 63395.69929423381, 41942.417186873725, 2332.710176606001, 90362.35047122953, 25423.19777990937, 5246.908823541652, 13665.306814018551, 44644.84883987933, 99641.7915924587, 90468.3353375929, 634.6964749400996, 72304.46320476061, 84695.9753598577, 46664.61882420013, 46799.64933195806, 99283.10915171122, 16449.764839052194, 10191.28717700445, 69848.2230363387, 10545.99114076611, 49534.939715658074, 82036.10817804359, 25066.580867159828, 68817.27177278258, 2383.998401567966, 53393.14394887404, 43861.25859947717, 85310.18719133949, 49405.28617159584, 31364.623019516846, 93555.48495954402, 58313.14882053672, 59262.978826089784, 98255.53373225071, 38932.613351920045, 84442.1656738434, 37211.419240085954, 38420.81799561128, 3289.1199241183335, 73995.85835837593, 31621.59651080354, 63871.97029817826, 87835.89379683956, 82640.18003694105, 41398.270575836126, 28983.78575888342, 56577.73879850081, 66025.36972538164, 82817.17525312766, 16730.99553217454, 52261.52630085136, 82848.60156710738, 49472.710789434626, 78060.97547248221, 19211.627225903307, 84241.24760216472, 67231.6074387088, 83898.12002644969, 77473.7730938889, 42651.285042557, 32298.99649285932, 13881.51813950803, 75980.11387721862, 37989.33384437198, 38332.73412479814, 50976.859503958614, 37720.06185935408, 99193.46351252029, 36853.47757489766, 31327.204142920607, 33027.36645960469, 43754.23530096614, 98264.42380751023, 78593.72751651118, 23454.63174696979, 3232.2324125558466, 76336.30393831164, 23455.463690765555, 49636.089897134974, 30594.44656199004, 78957.91474476666, 55291.17173231656, 59482.264865226774, 36631.62701453811, 23535.935800019946, 40494.94230087256, 93897.71235524236, 9336.484756585727, 24868.543611674755, 40189.96580210098, 98752.86859990322, 54237.06113096801, 9369.889193582349, 67183.90744420802, 69460.07977474207, 62486.31567420663, 22411.40841870628, 76126.22085983853, 91280.2458172559, 98832.81162818873, 10927.784309266886, 64360.83482222591, 6681.061624074003, 5435.547915613748, 61398.69591733693, 4950.231175964692, 44038.85060570579, 40223.161365633954, 71375.50944880412, 7519.084776290497, 5959.316456896546, 83380.00473421182, 53609.47207066573, 27407.13813583957, 25659.123363286595, 62258.57734375094, 5167.049909781363, 39159.57039810029, 37716.746192275554, 95411.41474439763, 10820.610899237348, 36814.42095430023]} +{"id": 164, "vector": [28763.422191592592, 8422.724805662729, 77514.60659938685, 18331.89877663529, 75352.78095722066, 59348.18048384885, 65196.78586390411, 2006.1008426069282, 22326.371318470607, 26911.103813461577, 95636.86487516103, 18859.068261288892, 71603.83066918081, 48757.274929393, 41051.778247434835, 90955.43490253083, 62105.06364462943, 71624.72662112884, 36634.38314465399, 52951.95088502563, 17098.09444096181, 93947.39703834956, 16769.564692711592, 64725.58720343016, 70741.5041448602, 9679.182618752091, 39075.00085166311, 62810.5687310301, 56359.134327920416, 39694.82124689279, 94891.6489444261, 40828.82741134367, 45595.2339446529, 88940.72291768978, 29957.529513134385, 70173.69492312126, 64991.14200963969, 2408.9748817743616, 38820.67203099201, 13127.56051124916, 49167.483821063026, 51505.3943168003, 92024.01194874776, 66122.45798123065, 90781.13237971468, 3423.711407635699, 38008.61896772719, 18760.560409843096, 36767.38392456448, 19543.2485093095, 83790.19016946203, 41049.87227017078, 96926.50998222546, 99464.95440043347, 50992.79327018047, 57214.81564072638, 92763.83125530514, 28494.985730477707, 28054.66174380642, 8221.14798817407, 93567.72153072682, 35393.71490238884, 34028.81957850555, 12609.596130925793, 94921.5807994374, 95943.85423803715, 35522.420979976974, 24114.89810042752, 18998.390524453167, 721.9858179446592, 39999.68418398419, 33056.24692299456, 26540.673053265007, 47743.246244629336, 159.2033710386187, 89270.61855901364, 6966.848709057849, 8919.062306402626, 51797.6705111746, 82413.54867412568, 91282.37203830693, 88600.80338837321, 70192.51958752691, 21382.171332703936, 49472.70315719634, 15125.943966549326, 40226.43511050353, 96226.34484326141, 67735.72852627149, 22653.416501803626, 53418.71296212689, 30136.90320051756, 93294.96662502506, 92897.27845535311, 41375.4024166923, 4342.656546180312, 35938.33715402125, 32214.59467874532, 81874.35736004938, 69858.29288576824, 77467.83144899538, 79497.24399046, 70803.80505339631, 56445.728830622786, 55163.55972504984, 73231.52753716854, 77455.61351541939, 87493.89991354039, 15924.722739478948, 1558.1429347547694, 28402.443990105097, 6817.364911101731, 51494.44020903198, 23403.959090894245, 25047.45618665025, 63740.52528576395, 78125.38748791492, 11010.156953187767, 32846.500840966255, 39298.02659485334, 61858.31325436125, 61374.755351834334, 11004.596760594453, 41817.975216121726, 23563.62024809502, 6899.687993667647, 637.8567207981911, 44317.36574640524]} +{"id": 2010, "vector": [22791.10162898287, 26950.652130723618, 26125.17357875298, 39652.34625271533, 98609.88871463083, 13369.479002815766, 73592.68766542341, 2461.0035281964592, 89234.00756226206, 48453.86894513495, 81819.75650966275, 95216.7866293796, 19029.227213019796, 2457.1759194561005, 96112.83210556045, 97917.14926089617, 75789.56826875213, 13344.519980174518, 13199.693859665618, 16696.24616477631, 48350.45035023057, 73317.484872905, 56221.55278717057, 80687.81642827138, 65860.54791751069, 20525.390091220685, 32081.483627898077, 11289.048459195672, 61672.62083241597, 8739.19627484968, 96773.06738654534, 85271.02356600389, 9301.040573118713, 2484.5412262682885, 73390.82820110639, 32034.75785306298, 41936.3869677533, 32342.388301290404, 19335.66606713053, 32620.7082774708, 2295.602383698303, 23077.216371628783, 20182.589512854964, 48782.5719131646, 38076.04483215576, 16489.671819075826, 92717.92266633772, 40236.120209509885, 64467.54993530334, 3998.8061040336697, 25397.877620666277, 4086.9894887612945, 29556.581774691458, 8456.328830180693, 27555.213759603113, 60691.777394275174, 52765.771783905744, 12956.489005652084, 38361.704919385054, 41848.35103127804, 39660.866281002985, 46015.98244441966, 23965.3049325391, 44734.211695030426, 98054.71728096697, 42286.074125975705, 6915.987826552916, 86761.84756550887, 1582.6172938999573, 57676.879743331076, 74344.91331466564, 90326.19241711246, 74006.59643162787, 75208.24175141187, 1424.095530306424, 74641.32619630331, 67353.11109145578, 39540.88595464011, 72406.76161752158, 60557.09598110144, 19689.453644105924, 8959.229264066504, 19446.979236136398, 19291.000856698814, 82505.24191993658, 88711.06820748902, 97374.91880773533, 56437.65384391281, 99686.26320547718, 30817.568369333436, 27638.18979861138, 7125.539887683041, 16439.450906883714, 22887.075438371518, 23620.89604057467, 68886.23008494181, 90143.24106914537, 73137.8999698065, 36692.46183375412, 20042.075067407415, 65566.02086878338, 95461.9791130794, 4712.557533137973, 91449.2491620533, 97795.65458181467, 93946.97625631852, 7863.3792999446505, 71511.25895388954, 89246.66252248708, 45337.676041007246, 58825.85609532649, 67655.05739611814, 32788.393364690404, 37557.13469032088, 3206.4422679570102, 35481.374848641666, 48533.346559158585, 27274.75194989253, 70565.71349541213, 34880.51067237524, 54562.15533698289, 59413.82557954193, 4028.4431514461817, 34583.0820454349, 43830.66592533811, 37717.38106092212, 78072.99798748721, 5147.460454547381]} +{"id": 2030, "vector": [83915.72523892614, 48454.77150000133, 8376.754153665277, 18210.949810092792, 8601.103480754868, 82791.61248197446, 88907.35971212534, 89535.5217140671, 94383.2728219979, 37713.549428875136, 17835.43835996777, 79515.41828733121, 96543.36164578603, 3333.784223986291, 13561.90993358597, 93048.60647116485, 71784.49857982223, 90657.19403482041, 2431.1785766047087, 93038.63510275398, 8370.165243736627, 88260.61564929425, 93888.84458725678, 23916.89529369935, 58166.086241031124, 48058.15552867052, 99263.98680516194, 17123.38765044168, 82348.31194780453, 22914.14247122713, 40272.360237091954, 35814.42030372167, 95351.21456020803, 3296.4725535953976, 26677.47145007784, 59929.72907007167, 93311.75372463198, 64834.074927899965, 73170.21180432811, 2156.640732011905, 63504.26891815478, 74761.88857784611, 3280.741989746516, 70005.16541708942, 10094.722715174197, 43686.61453798928, 34530.84881675197, 12515.589145424667, 39476.296342334004, 28207.787053635093, 66053.34434024476, 81644.51822468395, 94825.09880052124, 20603.01071730839, 26292.76076507704, 26003.48491132354, 26707.292507426595, 30735.610017114468, 91640.43318661652, 47735.30339128753, 74238.07234832508, 23582.248424847596, 16805.189856384106, 53064.89489180377, 34450.23088632075, 92715.48789150712, 43062.662017214585, 89868.49763186867, 38296.87102388142, 24225.300678219675, 91713.31268637898, 9387.844449503258, 98626.84742641031, 17970.98690157245, 48814.05282468571, 65930.07648699221, 76071.2692392162, 7312.810419896576, 47813.80260553564, 72563.4022849717, 47302.11006880662, 79012.14130793714, 79446.77254132084, 71819.78614917603, 88233.68397106712, 74663.44180114231, 79691.15538275978, 48307.57463927773, 33268.75242263766, 56000.83914674521, 63419.37262323268, 24694.652966860252, 62911.1109788442, 87356.99308505142, 1582.023049756387, 42585.92303014358, 10545.945933864032, 81973.93699999976, 50308.007501495835, 32204.602548775398, 36934.342281522295, 29538.600688200124, 34778.19091945995, 38407.70641012099, 85219.86726216078, 80765.74973671878, 42440.007225365574, 80115.95824064396, 2448.0415970348445, 90474.63499219829, 44898.9055534965, 69048.98469963104, 50326.82217873836, 51306.15841902632, 43126.23611999043, 70220.93945376929, 43831.82848291514, 16252.100000634206, 56607.849415391, 90672.11845259224, 34794.09897763569, 78189.12242423426, 40190.286471880856, 99203.64840584507, 43184.495356161875, 41649.78937681798, 8059.722582673634, 56785.22232584508]} +{"id": 1860, "vector": [90179.0011338516, 88817.23970501067, 17473.31786086096, 36386.631869836514, 38762.7685536473, 34594.19300835991, 35819.70455068715, 46475.72282644284, 78494.50253408325, 79.42386124558665, 85085.46161795943, 61109.849726139786, 49770.86417942413, 48321.27509467302, 59926.635756615186, 55658.91023986328, 73435.33319934146, 92683.03686637533, 93533.23163894872, 61049.07500828395, 75708.60166834918, 32094.761528776027, 44195.645791463234, 97050.09181831389, 40909.66815301136, 53289.24896136776, 85640.41512900068, 36222.84417924124, 55016.55256527764, 90965.96035183914, 53531.49975237403, 39380.28539846508, 25652.563969764175, 62438.18418005773, 69367.0461560958, 88224.49041822318, 62712.44476163822, 80763.75921848703, 47446.11199245575, 95689.24155404174, 34355.824770049345, 56403.915508892445, 29266.843842276914, 31129.57063872431, 20491.52986497701, 33452.20716670143, 22201.0376808646, 87604.26477435422, 28264.931014282578, 52311.787976717016, 9941.10168497685, 21157.534902828746, 51951.11817264824, 87986.34708211705, 53562.808740451306, 45569.29758931706, 44342.4962729814, 75694.37708452051, 95795.56514304635, 61476.82844511751, 82380.29223392467, 43759.7432455696, 69087.08132233549, 88264.4732716139, 47046.39957241018, 56949.80394873742, 57564.57493328676, 34278.47113179621, 11515.851734988903, 25076.862455734285, 59584.84127989343, 89775.64212362144, 75407.63339341823, 92348.2902513662, 74613.22612624534, 21401.924920604975, 35874.83521248989, 4905.357585200032, 26454.55793439675, 99713.63374882283, 77572.64304519931, 76597.6691387502, 42565.90682102723, 4467.968422901658, 92273.50807513026, 31590.411657868834, 94858.09962681565, 35321.450387313555, 2581.8947374601107, 20742.462465108878, 64780.34670221049, 72254.62946996564, 97303.44902251092, 42639.267895863355, 25125.140831570236, 69722.00540778329, 29162.99577717473, 95717.93247803707, 17363.385608519256, 2220.2838677203963, 75693.1241593888, 25080.50150621505, 82306.31913979635, 32851.01616028987, 48176.243336941916, 45010.477878612124, 48723.73878520647, 3824.9959099359353, 62149.09733301298, 36894.12769643871, 20846.99128852391, 9495.265713513978, 21236.57167848495, 46697.02329558737, 25282.637978110124, 19429.589355906952, 53958.011040859696, 35929.222393485594, 43897.061308863595, 21295.17772666466, 7247.017473227946, 92007.35290936462, 89900.18631489317, 48062.84482334672, 43348.51284643799, 32225.924592760603, 9688.119970250553, 8563.788066559875]} +{"id": 1245, "vector": [58161.42290697215, 13747.684853688603, 53659.399814000186, 26324.263477942444, 96575.32032442733, 77912.26435833056, 10262.097708041118, 54811.68877169053, 17536.610887524206, 9475.658422233024, 15895.790839647294, 13606.244872824047, 96744.43023347348, 35852.60817962173, 39287.59862343951, 84818.22093271647, 58758.45841838366, 82131.61916975546, 80595.4211909791, 2141.6627747209427, 27400.604452097254, 30853.162020744694, 53273.33049305398, 80532.66238256493, 79506.36791133191, 81468.92497762965, 88049.77027055509, 20310.74617934141, 62280.574149765496, 4810.696608529608, 68478.22576836364, 85829.49927545027, 29879.81162102049, 56351.83064896013, 29266.006391486364, 1075.3191669713335, 8580.238396301498, 81144.21299076268, 49881.72084626602, 23367.67552830421, 39344.97901607006, 37803.79773583573, 55829.014105875576, 50324.92767804092, 82794.25273569446, 28956.538437674073, 71070.82793396433, 24479.630661914965, 17903.003446681098, 90781.65336836386, 79859.89572201167, 95275.29986080113, 72340.18348982008, 82282.36153630615, 15885.530149825388, 20990.96321202881, 30849.954037637726, 80576.16661187296, 43942.98155213767, 48951.166757547646, 41868.80457176123, 1179.9383212666603, 12303.043695374605, 14392.600050683946, 22084.738793032764, 79830.724531305, 51815.04369517389, 93328.84112203731, 6812.088090378876, 71829.16658529853, 56173.41658665993, 30521.918989529608, 95910.08819527946, 78652.1255261042, 74490.14388910546, 10671.07941519988, 10510.602386900347, 41941.96504209365, 41618.44349504125, 50517.44337070063, 55672.168631151464, 19704.609299340136, 21279.76084450587, 4371.760695499594, 77631.16198487082, 700.5231038409887, 10578.944674482627, 25104.07284609948, 51171.12130578521, 16043.857746699152, 23752.43653493593, 56935.40143996144, 23358.485455288002, 44786.92032320817, 41334.12640697885, 81255.00195743087, 7692.564316814365, 16027.43008983014, 74722.51488269748, 13034.456440065545, 28244.785413759866, 93293.16680608047, 46561.69942371767, 2193.8195955360175, 816.84072943633, 36993.96532442873, 58207.014664290225, 23747.683773035587, 73334.58152920083, 29378.05380342733, 46543.76734640314, 46470.43659284548, 76125.86538100574, 99565.85731658095, 93823.86039391728, 85435.57274313775, 5280.549533995349, 49614.451817076835, 48013.72021497321, 15160.60972049147, 33412.62283023866, 72572.70824945754, 29608.30676118903, 37376.110332814605, 9599.644333286828, 20732.602798182943, 19285.467520050115, 87760.15656227963]} +{"id": 382, "vector": [15698.054036410636, 64344.89401337323, 42690.97826406322, 89581.43027970842, 20838.186051120723, 43243.526444857525, 63489.805460216165, 33881.562410165614, 35854.59790703878, 87422.02369020753, 21910.69958381503, 57219.6304028058, 59065.017197846755, 81889.69408915809, 36534.772426475916, 64898.67374276712, 97523.50560339201, 29727.285571685625, 30628.172865824166, 11977.547488919483, 60688.70862189056, 33829.84793815261, 40514.78066594839, 951.8140350107008, 70101.18353052826, 21222.19089980485, 81113.74453032434, 79448.89121002037, 85869.5866745864, 73230.51653046419, 52463.704837000136, 24885.834634672123, 95731.28618303771, 28621.943346596734, 87856.29352939622, 81571.65821795589, 14938.611944327396, 43464.06379427596, 50297.2153109935, 32453.492322883336, 22519.4501950583, 33429.897814626594, 74039.2805026494, 80587.83425582734, 53197.88176862611, 11427.121153040987, 83882.33312910616, 44852.15668917239, 12104.264249194452, 38110.637894724074, 29233.705666350717, 85249.61072330007, 68081.69732871251, 5829.596426035333, 67993.82152791129, 50413.70772742959, 76372.35794744903, 38611.39388931834, 58564.31622081919, 49127.79799048789, 3870.88772609423, 33734.28073025323, 87052.38017523395, 31868.874211781873, 11457.965625951816, 99592.07680116006, 73326.63177558982, 36132.580864273434, 51102.207721200895, 26645.11558757634, 99039.95589586877, 35072.42124077933, 22508.371422935015, 88164.16487999281, 67209.1495019686, 82461.57665061111, 79730.93968046815, 71973.22833379815, 17678.013188590412, 9104.93839512172, 3697.148856115795, 56008.7216429953, 54491.66877439926, 85597.13142810807, 22553.346570933718, 31890.219266235898, 52684.17741754421, 28130.706523582703, 33179.93215096105, 57381.871076576805, 56004.92003563492, 43077.39898228838, 3632.881167144941, 27364.56532410927, 85550.15883793983, 59286.27155458492, 14801.122519115084, 17446.078355219874, 48783.73689229957, 9198.126541961372, 9523.579133329597, 55655.57372710196, 20645.9725839339, 91864.83914837024, 67501.31091859109, 56117.67972349311, 18408.936879295135, 89914.73310328071, 124.12467614085764, 79176.3527215346, 99544.84454026142, 86984.61551192697, 80776.46181434806, 65982.74489634986, 7443.96108937162, 38707.335826095536, 26988.688248761428, 57448.04239945116, 22270.864717597116, 27186.7080530521, 68418.7894812716, 86484.35364582165, 14652.349382033002, 69648.97745257574, 76232.37280904778, 87175.7260514941, 98520.43048817369, 64173.334670062795]} +{"id": 1267, "vector": [13032.45280740556, 60522.58660548381, 74479.88934911693, 85123.06800821766, 44655.68112097423, 47576.99390314184, 13529.340117153532, 84138.13168798518, 73455.9638600225, 31140.65671295555, 70565.22079874406, 34312.55927647191, 67186.34498162477, 81358.0281548459, 93218.15316846392, 35278.0433114383, 91667.5701111531, 65904.82249100484, 5762.067110747294, 45596.134075384965, 2672.023914337074, 87924.68137620995, 92475.89843977834, 66492.56144387687, 40327.62091268488, 64749.0137123183, 54724.63703883816, 57060.75493589505, 23775.845023931808, 14770.294502305815, 6627.5549339410045, 65655.57700766463, 62155.17815136441, 80005.85037218347, 69513.22371890115, 91760.73223395673, 43208.35105997434, 93163.62103716655, 42692.41123446824, 25821.675068533423, 85472.7250697604, 91285.93619506287, 69730.91761045849, 49041.86924828252, 94466.62364259112, 42790.027595540756, 41956.31072221009, 43290.97598834305, 14235.286135446946, 23748.784705313086, 24972.385591002978, 56726.9738900011, 36595.34355675885, 92325.44205231126, 78385.54306092391, 18353.344163841044, 31710.09327850316, 86895.97103729464, 5230.052357346571, 79907.56563623824, 54076.473564372995, 94448.09526041646, 64892.74765156361, 75984.16601715404, 90809.05066121282, 69433.44017827357, 17596.21081150954, 20695.93322557822, 70072.78826152642, 77999.17066737018, 8007.324599663701, 81606.16407811825, 57735.373161747484, 25241.597833650732, 5783.55753084483, 57037.91294163928, 81167.51814170646, 82127.42073307256, 2503.994355129413, 72037.92492043902, 91805.46504031948, 74745.53895540201, 31859.44748313855, 24386.52528695021, 95963.08900923796, 88851.05323715195, 93279.4467053454, 5899.790834687102, 88983.00325214509, 32950.73399408256, 766.1216510681523, 32364.894051065497, 68851.94591234547, 71676.1074116219, 32491.105220620564, 96958.95507572542, 57913.02744310796, 28371.043937955354, 28252.49307945069, 80857.30460573423, 81787.05244629818, 53994.65485756313, 78188.87255624331, 96772.12332153202, 90673.32249707692, 43903.453683488966, 91852.07618260631, 63060.9793873777, 78500.63743813832, 65063.03710920636, 7402.606850666371, 11787.764147451107, 1452.6732495996896, 67696.69704094852, 65841.85879798864, 3322.319415461217, 28158.715213609998, 46008.23796571804, 23758.88880725332, 78683.71760526906, 71347.00009473616, 11508.104268671581, 90301.81661309244, 61412.63626309908, 56215.86088953224, 12666.6325814198, 20275.54239786006, 76554.52234971736]} +{"id": 739, "vector": [17342.583785051236, 63879.12803597602, 64437.795768661, 50319.23967500005, 5892.88265754917, 22237.12701063145, 84469.01542140519, 13899.741318135882, 56051.49386682995, 37687.42611741911, 2983.199212984888, 59112.93908011017, 57543.117935640854, 65436.94277931978, 92889.12952790862, 5221.451016972323, 4446.700143380455, 16413.96310236626, 83982.9979091796, 31723.457832007796, 14614.236883669319, 80030.12184312692, 37295.625163589364, 34698.68310328006, 72495.0797974275, 67176.03568004597, 16562.504273782208, 37816.89932063246, 72551.94753376946, 397.9257479011955, 37140.17275144411, 88684.76920060361, 15725.135757042275, 38795.740600905905, 48361.94926871151, 15223.676273854015, 58814.61009895013, 73086.19509459643, 81347.82102800877, 59391.34866975594, 91016.61797839239, 82394.21270654666, 31005.4062035238, 3026.025022335732, 82379.44715748928, 14134.22101697801, 37525.59612751344, 62649.368147863235, 14258.505878225835, 73091.84965799606, 96517.08523677988, 79648.37138227717, 90851.50388157491, 95144.62332891335, 4655.205650962646, 53127.96304657081, 5243.810397269055, 7522.68802378383, 64650.30871761597, 92109.59466158767, 37609.179384329414, 11596.401195547856, 91689.46673552261, 68773.70942176423, 29972.71223919541, 1884.558813757653, 62886.29992828574, 53583.14901734264, 71302.57001409454, 68361.38524163186, 49953.72183068542, 26266.24241973483, 65765.84267517191, 12999.599380152738, 35329.8533260219, 92766.06320695583, 2319.980356461049, 52236.86911079274, 84163.45818951611, 81298.76317903641, 26860.720542895655, 76278.48960070261, 25426.31114134617, 58327.777520161006, 40877.1829447209, 24266.33118944259, 19051.382046825493, 35420.533912878425, 80527.63952355491, 1957.59701180932, 26983.509951146378, 81524.93043633143, 13544.222893586677, 75432.39280753695, 22136.162753180775, 89816.94370552893, 46382.273835706146, 58484.679745006964, 23251.580643427293, 11353.871032933594, 20409.912630278883, 76211.63235419504, 63154.517481765935, 88509.935851043, 78760.11448594478, 91964.93818578898, 34936.09833989123, 95723.87062191757, 32891.834069795375, 24363.5798196702, 27779.611761165957, 23447.615510871066, 41566.82808562675, 1222.6563653725652, 437.3035264654601, 97000.48673746748, 75111.53710179073, 88857.53248194294, 56103.26334762935, 83423.41770670778, 54955.47577992488, 59180.2772186245, 92968.1850253585, 66245.83741286845, 15181.999126595858, 15380.951725503766, 43079.40633533691, 58239.57807904497]} +{"id": 955, "vector": [74135.70239453326, 15913.873723905781, 51231.84468770682, 80312.37831393168, 53942.140172138534, 734.7538203433701, 59184.15453263768, 84863.81529442527, 7001.790134023733, 99287.90734428063, 99419.10507196441, 87673.2243565731, 56106.001033495835, 77068.78637690491, 57621.97150753171, 49472.66313821412, 40281.08810570401, 49496.44348750531, 19717.468354417877, 40057.58800733326, 94362.10560368646, 34954.823940999035, 64271.74659346413, 9482.599585138296, 9949.486676730956, 42287.51425179477, 34845.766552200585, 65161.8342278345, 89068.12646190244, 53804.55056570555, 96034.14353825289, 29448.776745749637, 86957.29016419513, 95869.10312045389, 9324.873549399183, 11705.094311080944, 13778.515891152432, 97220.64937631495, 40381.649651915155, 10901.272213962033, 75154.19914281479, 76432.66574597031, 81547.83889592519, 64565.75090611455, 20759.66244136339, 75213.36398012182, 50665.02390111269, 14162.152764474622, 86150.21233368648, 87510.76238125026, 66032.27329501302, 98536.92806792057, 72558.49710996935, 44344.7417689763, 63725.46074316304, 36122.81416742268, 12719.464216178334, 55419.08835386027, 99126.2959813233, 85831.59267586698, 40613.85815886417, 95749.96135149641, 37444.52734539345, 95032.95202766232, 54064.10245559283, 21223.061361758035, 29143.027965245903, 60652.82665247213, 39104.66286481667, 24093.275998753317, 61714.028537433274, 97768.74411757107, 48452.96671903374, 63495.318101718214, 98675.72856919773, 3116.1926346556565, 74415.03722467652, 26903.922203710874, 97711.91302787384, 58172.047166215576, 52189.30664203467, 8075.1136935595, 77762.7003808875, 73645.37790700186, 19626.454155019503, 93937.68726402006, 55429.36035485021, 89439.57767481286, 3475.5591151315434, 47474.73523568183, 36791.04909145745, 93900.18911455771, 84699.85266444023, 52729.745935756204, 82138.86332584958, 40590.58312919414, 81006.35608546792, 52840.30766216886, 48964.17735136792, 67338.92899609626, 39860.92405156464, 11664.763853227245, 3092.5765161253894, 31700.418546288623, 6309.116546810922, 67202.2966734673, 51975.62891939836, 37736.0886445675, 32947.04302842718, 55901.645131512145, 41720.490278755795, 79929.05260576594, 34593.66410673349, 80714.4745231601, 39358.68847342154, 52464.61915371153, 45745.32751850654, 43567.60948395735, 84621.25981391838, 97027.7606143268, 58436.10644805412, 54787.74585314119, 73897.42465700101, 28813.62373923072, 18238.593908578947, 93484.479017114, 28252.614375829922, 53655.051484401]} +{"id": 377, "vector": [6181.167172246815, 13387.517196436682, 90478.55528904524, 78339.29671102407, 97907.9074803903, 34204.710264648565, 93230.70087335363, 57361.49451436833, 38729.67789559805, 62062.60765380681, 19148.246899247813, 14001.648671810008, 92760.01280612333, 51024.62225398859, 55484.91980937329, 91337.47577364037, 2653.71179801569, 10957.528649523696, 82333.40499726315, 97781.80639684734, 45615.584771486996, 22452.986518515438, 88530.5342363891, 83833.95204151033, 24258.105003914978, 3734.2190343727766, 62031.566109887506, 84570.65709820992, 45336.453690679846, 34313.12196368089, 68802.28093741403, 83630.89530709333, 48280.27030343458, 22069.609213031337, 25474.977320756876, 52967.71160524776, 36777.32317839293, 67965.42707080788, 14914.852807988344, 47541.936477035764, 17093.584623847135, 49343.08573914516, 89515.02707100016, 36252.46559347157, 98039.39075255318, 48330.20418500925, 30040.89118890164, 47796.96217028872, 9374.456237835571, 24.87551145294864, 69306.6621846738, 27695.83978869885, 37636.64037747841, 97081.1969167608, 6223.532194259152, 82286.47511359362, 70661.5956559405, 27320.308075407316, 97967.9247526486, 40317.081413818836, 84883.89608304172, 1192.8295925706323, 3862.9261198158792, 70967.34326853245, 5025.14616972618, 2700.0909592709068, 54328.518009823776, 66957.41430190182, 58774.308163340895, 71802.3103546073, 40994.75076854586, 93180.52472422959, 21589.765437528684, 74511.98430666824, 83003.26610221251, 31615.8644526903, 47897.33544018024, 16453.573599299933, 43194.901326209576, 58237.60249415296, 8788.453034734079, 12514.490930028, 5412.916490563902, 96376.28995436647, 43078.12372762133, 64707.59164583802, 44694.39557424031, 23282.91499098486, 54262.90761452054, 22550.225688001225, 41107.12643724559, 77366.12015903133, 20082.63086816784, 1396.4755965200416, 47658.933025681894, 12373.747486905251, 81537.97527115738, 28298.72620384608, 17159.3654084517, 89735.74673191686, 3582.800301670963, 29563.507547551195, 4447.23741351084, 77934.64994955495, 78631.41334707018, 79909.48201998182, 74722.2273866798, 91874.0073332505, 63720.889647644995, 99940.30144338317, 23810.778833063574, 91577.9927019933, 33876.412348527316, 98879.63349828294, 73579.59746315885, 53987.93512285616, 3772.6301217424816, 41691.572827626136, 87500.60369376438, 44659.722690227165, 71968.4490557214, 27472.44121364214, 3197.486997723309, 93597.64808786022, 37474.88444568676, 83658.1761446599, 18817.772289271594, 74240.3943321634]} +{"id": 621, "vector": [88010.7927327726, 37181.26621337821, 50130.47379545709, 21826.698299031912, 91253.1243359372, 81515.70276388504, 66833.66996883831, 79450.8434418869, 54332.463798354256, 16247.446215256612, 34677.067683700894, 12717.42427582292, 70143.75692829108, 74308.83790550422, 64731.62830776091, 7162.99326574702, 37248.037689952405, 32924.018896583664, 16450.424417767128, 87522.45186600514, 65616.4505557275, 52044.058551855946, 50077.16406876601, 19410.072528370838, 34332.214080500686, 79943.22597758332, 56036.4327225772, 55944.21966837605, 19451.030101361754, 38615.90956767929, 14643.078954056766, 60856.62260681746, 73334.36810158672, 5348.014457586514, 24878.16329697705, 10772.807983785471, 46817.38144215091, 39401.963995970145, 45045.164095753156, 86638.64574858946, 37103.794956526195, 75940.85079767628, 37766.43160033697, 65511.67171205562, 93808.02411793658, 10478.89965832799, 7653.112879428115, 99584.8991061937, 6030.858833989283, 56.59379394985508, 77696.67271369816, 94312.32152000157, 60771.88142275991, 2807.3153526294136, 91858.93891268894, 32865.04421437314, 12284.697751594953, 46296.45168080233, 53133.80723648581, 93516.9442578574, 72445.98797020201, 41356.53783981451, 56459.36749727447, 42821.8321113253, 47911.357373998995, 58268.943812564525, 56133.94487182065, 86892.2547733164, 36947.9680986988, 51731.181465711416, 87419.22565227313, 14133.049638265738, 48711.24703378545, 67407.93000470074, 49806.52373357294, 18211.936529219798, 4498.497321086603, 67472.44858297842, 46183.08684087611, 30900.527847734094, 1754.8471008077881, 49498.79478054183, 47401.7945089914, 72277.7655410185, 96988.06143294556, 38650.88229952859, 26607.012550053645, 43651.65958540976, 98159.4546069052, 51567.31130222038, 74642.12839926506, 44614.811122123196, 47475.19829516962, 44826.06914967817, 61936.556110316196, 73522.71355429126, 49562.074734098846, 8071.771219727552, 68895.43339458691, 511.86412391219795, 4716.440742533035, 90623.15610721527, 71805.5149401253, 840.2486370204798, 16455.41991964389, 20224.981813363764, 20807.74689495618, 96096.01195666159, 8983.075753957426, 58785.268385972355, 4096.171073581522, 32506.386597900295, 89885.20377643917, 79681.05705588353, 93728.93362266223, 33933.820038688435, 43481.79427244219, 50426.44738387867, 17183.689935085524, 3228.9792596183474, 69146.69323668738, 36101.21315653184, 48400.24296901164, 90506.56685456201, 98202.99048320945, 51915.110039795, 2671.424980595061, 53243.295793185374]} +{"id": 1379, "vector": [80800.27654227188, 28544.61594111376, 36758.2114220381, 42675.61376730804, 50721.60513010742, 92865.57078084596, 75745.92085872933, 60091.08789576815, 51648.961032235165, 51731.70155439975, 66206.64854975694, 65809.12336765752, 6675.524270938859, 5981.738474956522, 65662.42746904341, 15434.496469671776, 84331.34446942763, 9499.361739958667, 54014.38215669384, 47066.43711385782, 22932.374669179746, 47328.29742434285, 88980.11827325099, 75630.18746084072, 52902.15180278101, 52845.33979467917, 46281.42832567772, 67481.59256827705, 47792.22553866709, 71255.90644898178, 55301.77838865591, 15294.526089531868, 3815.0425800106636, 76144.5206129364, 92994.36111267399, 63291.05607636396, 69031.89034082627, 75551.34867887526, 49155.88970775572, 99472.34903232353, 35567.81224731215, 91291.1928129539, 3343.8012953043717, 57575.64764330202, 61611.585515953484, 48043.3240841198, 80950.20360739641, 21655.845242237036, 49290.71483423103, 37548.45321529886, 9993.726019421656, 22935.447496185923, 23953.385803076155, 38646.31941120358, 25042.1343429437, 51985.29002383926, 34131.48247827899, 55396.31633870492, 41931.36375327097, 41900.589386989115, 6903.541159826532, 5699.362768272876, 55778.243799452466, 11878.017492007453, 5024.551073987327, 25141.953037821062, 44163.87620226151, 49248.93232325454, 26459.5984332478, 87999.85676076724, 57987.03641727293, 63434.67370369483, 78279.85767351766, 4844.73760182802, 33064.31919387448, 27003.82406214863, 96314.69670619743, 56024.42503376363, 3012.0396067032207, 82440.85605540531, 34188.53796188015, 19538.171498681444, 53030.27033607318, 76545.55290517915, 88469.90079082995, 39560.87539483782, 30431.48453039236, 45958.45384714674, 39543.64716742588, 1064.8259282317385, 16744.421823457655, 66245.1223071462, 80070.14847136002, 44369.69603311408, 47406.86002249559, 76986.5839029628, 13268.246194496958, 73688.56025851174, 25418.12675932983, 31615.76303303454, 83106.28613655767, 33569.00038501447, 34453.00870078153, 97036.86824207104, 14378.951443384858, 90419.48214872317, 4836.769995636037, 77709.51397176844, 3278.215708782284, 30165.881709444166, 66498.92167590666, 27693.861988237335, 93076.14454027182, 21308.651664143596, 5911.649359401916, 20576.742336252486, 8336.667859846992, 261.4971694196444, 93022.36581171791, 9725.081304856842, 31844.059585739615, 8038.586580527774, 9265.556481792637, 23910.803673898474, 21444.878285753188, 44507.69099379894, 40011.379559775196, 21538.123553907128]} +{"id": 139, "vector": [48418.82329987354, 25173.7740706086, 97135.46463586677, 60309.26654089296, 65387.10633955882, 24115.699203343556, 9461.705456762993, 95621.11982841269, 20663.405262925848, 40703.314387632774, 37003.340550955974, 63273.79277172493, 52217.74580341509, 13527.787825081361, 21909.868082970173, 24430.899369142422, 33376.639663095186, 6739.488249737003, 66445.61250343852, 57877.94022336806, 972.9407853063865, 13986.551905528211, 57905.47723526864, 48800.95100148101, 51481.76358824351, 88951.28537267142, 17523.096727832253, 49217.48490119711, 61241.10886451062, 15399.182946308809, 40593.44728499892, 6023.927334063661, 52711.55361692396, 41946.580657156795, 62757.2110430115, 77460.92654242642, 43460.78203890075, 16648.310693309366, 61603.27463636289, 14127.884397368573, 85970.86948575694, 68700.47037291677, 65035.14711973232, 59169.67333045233, 66595.98930855386, 34281.62662283979, 17143.131401361723, 13410.32707602361, 66536.66938011604, 50454.25508185077, 89524.95753778542, 16447.774512031177, 9384.552762986243, 18893.95307433238, 33968.7288880863, 67423.70481458787, 20822.740241478597, 34378.200824268366, 2371.2868910403363, 79453.46473480023, 45221.27687324118, 57443.86031214274, 60361.13353319566, 93418.76237927702, 25547.9293292867, 89984.38409078313, 65106.399213832454, 25443.282995852802, 46465.9564947848, 95034.71364255242, 51612.92469384338, 40946.264136401696, 1121.5691722115673, 77369.4551570949, 9552.63754552076, 27591.642884950517, 44015.04997724582, 70191.15963533807, 41662.332327904674, 64713.38862264249, 16906.07332571822, 37173.45620949021, 88715.95478858305, 88496.13540424069, 68343.77431326831, 85453.13628510605, 32869.191390976215, 49885.42675807232, 37754.878405969284, 67915.4089955484, 40048.98900480821, 47053.5226456151, 43952.911125677696, 81384.38883324884, 88574.25061661458, 5317.281777476434, 18386.283893709122, 60500.57533303077, 95230.23462819918, 72753.63665192318, 48458.63072417866, 13090.006771449469, 92709.48471931102, 39104.770382974275, 55400.76774732085, 32997.66883399519, 2031.3808683653733, 94841.18863304575, 82651.10554029989, 59914.47414741641, 19876.71390288177, 93114.98689155429, 633.5346872592296, 81095.23983483206, 91252.41361201351, 35721.07068157655, 35290.506580774185, 22228.36823687493, 89809.08376921322, 38693.76068585054, 79862.12877076036, 68719.2596653289, 61669.22409779908, 33113.29129951482, 3590.6526060808797, 85016.26907001907, 59461.938167717635, 96892.34763442788]} +{"id": 1958, "vector": [40311.483035652716, 61688.01775799592, 58926.11413968708, 61029.939220636144, 68399.33912160524, 64442.19849190033, 50932.36011542741, 45502.58771565205, 95750.68726485428, 87522.6302072072, 17515.292004052608, 52016.04935792881, 62188.47837951952, 21603.818480043647, 85097.43142120488, 37504.54561070574, 47084.071450462616, 28787.92355434695, 29752.435949760904, 42279.06843689211, 28211.124804530875, 3625.521945549115, 30700.647657661517, 77254.4751291172, 31050.10640094029, 22071.620455584416, 45866.512130501294, 39398.48650117299, 18444.601264200734, 59143.7159118772, 78506.93445660164, 41054.98031311293, 54110.80225987733, 60739.07939050202, 32040.979179450835, 53103.48869217958, 5072.996489491199, 29112.53037986806, 75755.29717386354, 80018.3093538323, 78202.4958975787, 95519.97698711931, 73968.0125972993, 3981.2415951864355, 31610.771217109766, 35382.33007878039, 44236.48013713879, 85853.22380077348, 33797.288271672856, 88797.13536698175, 21153.771701637626, 69516.14267050785, 89099.2496170238, 85133.14198917372, 36703.32340903951, 99656.66661959434, 59766.2140391812, 55791.338089265955, 3207.8795267327932, 68388.75820942997, 73982.25655851224, 67053.0322583231, 70329.81586716503, 82299.3884158151, 87818.75124110183, 84511.58882525083, 18871.311299307516, 69043.79525947038, 47551.3049221972, 3374.1126586831483, 71562.22807485005, 36938.5047017483, 69671.83096106583, 55889.075279164645, 70510.21241675488, 17856.68622569696, 87225.16978122445, 14433.227794972969, 27920.624565284903, 73008.15494512027, 91502.26654410004, 95155.9153892865, 54706.99730015454, 1830.0713453346384, 84402.25239671758, 73058.35187728264, 38389.559346386806, 85910.16084719109, 31567.967416723142, 20309.75163544918, 7641.009506934993, 70219.41930878864, 23247.196196719044, 74802.03266878103, 6233.898579021557, 34142.86123826685, 95191.01553242648, 96282.55924229116, 26189.015273995497, 97678.0124418688, 58315.80974437304, 36400.22306028115, 99410.64128746108, 28298.71564090437, 46654.91106543338, 70894.60227507082, 11702.701980574582, 89497.57380312751, 36157.16677381084, 42573.4488058424, 82958.24542636487, 69686.24790125, 9066.292836290391, 35469.58564539861, 44964.004340020416, 69748.49756164981, 79272.80342800425, 34008.96742593855, 75744.5480595755, 13044.36674554077, 90147.02738761027, 62039.353278559916, 56272.28131460096, 89527.66854840622, 78517.53243279713, 66941.41084597661, 68775.12664109605, 6579.257251338855]} +{"id": 476, "vector": [45726.80399926197, 17660.670723237516, 99462.0163454675, 94696.25817024923, 72739.19053348621, 67588.15002807637, 78900.9536724074, 97206.05400212416, 50110.6359683116, 22250.985294914106, 22889.358984954066, 4793.188379994184, 31037.88989001861, 60200.09464433697, 24910.848063009515, 68458.13735906448, 73602.30716597028, 48194.45935240161, 46318.81417866926, 53852.014906283985, 40276.925465497116, 30883.524017992968, 83814.35648743795, 64222.332732929484, 98761.74043978332, 99588.7587712333, 71744.99528741711, 73536.75958177136, 50073.02631794468, 86693.99790022131, 16435.90021637852, 82236.8376113506, 10237.97978035269, 8271.930664605554, 28109.9824350456, 48633.44642587932, 70168.2486732479, 24684.357515469736, 70316.89297489804, 41829.366319942004, 29720.93566263396, 18872.33607527695, 58931.714985074694, 74530.07485970577, 40131.04214148475, 9650.804177494376, 16865.107357237965, 74841.85373525765, 48510.207740587655, 89722.44426247758, 66862.94974136601, 80980.65592398001, 77441.84565893089, 86907.03581709365, 27374.70852938483, 70682.96466082748, 72407.2054210653, 16676.607616174842, 99790.0323080679, 75615.46863303008, 20648.467093999312, 83184.72534157606, 37450.17446460436, 66431.71706140721, 69893.80924296826, 72488.05053654984, 12936.751293736814, 69812.36974920548, 38728.21422826614, 32027.94813916211, 58816.85200743728, 57958.27201194929, 28475.286775478715, 46024.76808699024, 30297.658286423677, 24576.355820848028, 54897.618561768446, 15592.476633498487, 49101.9046645068, 26610.029029816353, 20785.97817620459, 60607.360142959566, 18090.687505797665, 65076.99325410934, 41196.87098155819, 71846.03090580618, 80697.72791527984, 6401.21833021885, 44967.2734480615, 75468.39624667961, 70747.91513131882, 40563.96878188106, 48529.51675352495, 45540.25488037621, 10167.048304545046, 62092.001766566544, 642.4257265049249, 32888.58639609498, 25583.800927858545, 95539.02324590107, 2352.209800570393, 39308.0640235852, 23494.162826225627, 35123.23009576532, 80921.96045869804, 74646.23525193673, 86613.32056591999, 32357.006793432975, 52060.06958479455, 88016.86584118326, 50872.01731093497, 53870.35400976602, 59326.52678539766, 35434.91661583096, 53819.02296373704, 48474.08725423584, 18288.27288309092, 22356.05792275349, 32244.82780325204, 17474.256197164606, 42039.34882550001, 2658.888719093988, 52095.33164137479, 50778.22430110681, 21849.99868617071, 37735.99145672897, 31768.791299856402, 15303.668735669573]} +{"id": 45, "vector": [96955.07626130774, 30442.858429628104, 3291.95594943541, 28098.64970464988, 9558.440605727614, 9451.475926522857, 6005.381345379135, 6096.029277201431, 79457.79501820519, 35482.57990209546, 84334.80653573328, 32805.596059558215, 21464.550530697536, 1206.6126617651162, 97650.79727881824, 53967.07306335049, 71585.83073140396, 51879.73123437533, 4843.112103106295, 38698.463685164694, 51330.33253320364, 77915.25583577441, 80083.82906680959, 57417.38246359288, 81177.47315824749, 1365.6728429856214, 48293.95748275779, 3365.010746740005, 11471.362583113232, 3354.116871799107, 84934.54734024269, 84222.16414913406, 9770.372285716978, 95229.12304673536, 5239.828444398709, 10356.159904716633, 38097.80648135267, 14831.064928172033, 47884.65994144111, 38587.30091193726, 58401.62509660717, 2475.5454220355186, 80282.06015559853, 20328.967745234273, 17196.7378688036, 89127.89084742617, 46082.60342791471, 14073.630711246376, 11670.651191629178, 68075.33142650586, 89.82598927919084, 8602.159218560857, 49473.81862726354, 48395.84232904525, 23792.28772742431, 56784.15518893598, 30160.718420499077, 31481.401759853044, 98918.34055748452, 23942.941228796775, 28990.98852653351, 37939.06885432522, 91838.88846599581, 23302.637704400055, 77479.75701374504, 8447.17775160535, 51798.420148546684, 8793.432317189798, 9602.826224873563, 73967.84576842366, 18103.126590616102, 50954.00826769038, 72125.81776378842, 23405.817169169786, 35211.867835737474, 81532.24149026403, 72099.20123873779, 85186.27190240177, 3365.176538917625, 90402.39311409113, 84418.07459687044, 83549.98827631406, 44544.56196871409, 51257.26187206774, 34900.88573882899, 2346.05602102963, 81841.30871154595, 90557.34696117042, 71341.78589243941, 15641.678940493408, 82891.02083259508, 73238.05285437345, 96541.18926761388, 72985.46122373366, 41284.045218792395, 52362.19172038875, 56792.638530065284, 16726.470859420195, 32216.940042091723, 83479.92283723503, 67762.72649952603, 50373.63063319983, 55747.03282966127, 47412.97835421453, 80674.16717966925, 63719.17133958035, 54454.650817604714, 48444.261157897716, 8368.644527049262, 73236.88937717104, 25237.208643414964, 71848.33203687044, 52462.18593156481, 5539.259837978106, 20823.56620065272, 75856.30323882766, 53448.14967090265, 35068.90394098159, 21214.76465313251, 51061.183631619126, 42399.969875717325, 34145.67871154151, 23588.140726924456, 60329.53730593024, 98271.79964694293, 89499.1294822449, 83075.51856072072, 83616.38479508267]} +{"id": 273, "vector": [74378.11196419857, 51612.062633215115, 63258.281051345286, 73315.062378286, 73634.9686244373, 9957.103909230525, 44821.95372399506, 70083.62979508069, 42616.310677398775, 75484.13561211227, 82998.04211180139, 38518.42288803665, 26030.192752410196, 45547.48263538836, 45031.67603206412, 22906.55430930786, 70966.68120984768, 8860.188801769375, 44379.74223404384, 84471.88915697265, 19179.04179908214, 41350.73924180445, 56251.41970004647, 2256.4483803360845, 95825.2969322208, 59524.14417441881, 95212.50457174676, 75349.70607773971, 7507.412063614715, 70327.5105470024, 52949.3132047712, 16639.69265749423, 81693.75247693389, 62105.87079892777, 53183.03956335855, 20049.130404809766, 33278.943129200634, 54168.98964382997, 98412.65298546615, 18346.749652240902, 93218.02523232155, 37297.108463783756, 89078.33308683787, 72515.32939233855, 21934.616133136675, 39448.0028484999, 99283.56450676243, 96719.54798621946, 23090.941518385378, 90882.64722928965, 59008.26741552633, 53772.07363650042, 12292.00890951534, 91473.19242020829, 10380.343317256069, 86798.74906376384, 29719.38948772619, 14522.89107419429, 96437.3066872022, 1525.3985941967362, 88069.83837911581, 78916.38231988167, 61723.600949125524, 51225.71523440132, 30990.8078559341, 90222.3370157027, 39764.914837429766, 6275.47250930931, 59710.44237097564, 82309.89143367221, 13916.195702349232, 23947.995567007285, 71052.14573044996, 83754.26002528277, 78939.27784318662, 35360.895379053436, 40649.21756598212, 17489.610733976868, 6578.8301067809525, 7463.7376787190515, 78254.77409082814, 6785.623210469072, 90360.6810157413, 95576.86897874047, 33116.67569186811, 14791.850910176452, 97631.25871033789, 7602.561400869267, 71087.5819897638, 21690.515047795354, 88536.31180830611, 31810.743768266915, 90340.061175233, 7084.873618507881, 75778.29349404058, 86724.83017097182, 15539.328494181658, 46084.126931553896, 23460.312769817392, 87288.39880447173, 24484.328618089745, 99557.99009935849, 73419.30527090885, 66545.53854285514, 8162.07293061485, 50055.71845622137, 59580.12624141521, 23241.180672884642, 39917.339492026425, 53880.31619461047, 76865.03075219043, 53085.10280205212, 55291.00814884121, 70745.89958697796, 66998.44721314301, 16670.742424035223, 12170.902148400242, 43477.3994681047, 54284.33794888975, 21173.198363533873, 65961.19582429658, 48096.43433908412, 66405.9618124556, 18105.386313240655, 47307.52294930503, 50638.45735875658, 98537.39976118252, 54632.67001850894]} +{"id": 1761, "vector": [9661.538753457022, 7832.942269700405, 46945.575596270806, 18889.144112394963, 80880.74868826637, 38356.075670789425, 75359.56977549492, 86698.89664385299, 32193.187896276522, 28666.362296951098, 69528.04704580878, 12957.544226583006, 38098.96204496108, 54289.53726402286, 39845.95357756775, 24366.14679586525, 85170.43696954027, 93637.0183339006, 98674.53749566116, 40445.170006659784, 2150.819937237669, 64546.45617557575, 47279.69915216297, 68322.21155411113, 48921.33033005709, 5462.759245308246, 39836.369375422684, 96228.19479733847, 26443.245200455, 57302.32112082497, 60185.11186518169, 58474.94606966259, 26601.124312408952, 99512.67081512143, 69650.95121586649, 55176.376795032826, 90090.71147544366, 96464.82516053453, 97184.4270254603, 54897.561146581975, 75052.80749548985, 16295.684866631376, 80818.25946755112, 75010.18643802487, 49340.6595589452, 61395.79456348281, 62104.20590364966, 79380.78167890514, 17106.01946542565, 59425.75334531625, 20477.45475167394, 70591.08166038895, 42937.55781362902, 89125.44260929378, 1180.8884620204462, 33227.06624246444, 44240.17380581927, 83116.70707520195, 75918.00694244399, 93788.87720931188, 98435.28995966142, 41398.85309134429, 70992.95114753132, 28781.004185026602, 69160.27904262852, 87193.56031085148, 80471.03855276672, 1755.6752283308886, 55171.632394036395, 21943.521503193097, 78156.82915261082, 2535.0326261520563, 40721.060784815236, 48114.60081615213, 9096.374652486516, 87389.94957639278, 8525.622423549894, 69794.6868497061, 69175.5945909366, 36331.72001180731, 97598.15888357072, 53162.8876230036, 91720.09637467859, 41280.110018463376, 18003.589091622107, 25959.27995177325, 37807.743797793024, 60390.239232690765, 41032.08429724276, 92028.66078456689, 37805.27230195114, 58195.24814405634, 47985.08221163425, 20672.04535139735, 46972.832566556666, 88286.09403986961, 56023.42821678054, 5713.61134135695, 7581.95519165229, 69612.99080111347, 84258.00688029756, 92770.75429438557, 91051.96828786517, 40831.45882427165, 6769.31082661909, 53107.906545141625, 69440.6234797504, 67088.61104505676, 54989.295417689355, 49046.36323081178, 24049.729725280187, 98686.15109214526, 84309.24636230436, 31406.930875683734, 7462.201916113431, 94870.72886416368, 94666.41840210205, 93570.06698391821, 77047.44252923364, 65544.26979853668, 15772.528980599487, 13112.956386488606, 72397.43751602684, 17696.3912645388, 18648.273039117146, 15515.960022274034, 37743.61457379133, 51338.531500729776]} +{"id": 225, "vector": [16597.34288931408, 18863.004150389086, 6298.03790654333, 81596.48861340039, 8602.920351259425, 52967.82089275325, 41558.268041563686, 68088.11126101977, 46058.61239640143, 74213.6648237508, 20114.470296725372, 11355.735544267898, 63627.222849020836, 93092.97356686852, 10086.020445730559, 59676.21006978475, 51363.605014986715, 23538.750721422464, 4598.656849999527, 87747.44677674779, 36133.116650254924, 79929.89173366671, 15733.144386099451, 65440.94786572252, 45318.79149631471, 43364.29605869877, 96057.84984628655, 56844.40447758111, 4290.64401220316, 74483.94159771875, 89894.70732516676, 86540.04628593764, 39751.95956701079, 78447.54381691401, 51597.02016372834, 51016.32966592348, 6204.02876166215, 17074.135287730864, 12712.610785681367, 40667.996466531455, 89983.13070870381, 45380.83122290623, 81360.57749436202, 43988.492702750125, 58135.668987844205, 1344.7432206856736, 98687.52597998464, 10980.421868848256, 59836.048600158385, 28657.431530967424, 12948.406103443056, 41457.7089474057, 23448.251125594612, 32781.93416240901, 97735.31524691258, 76498.6299268169, 80626.86299059029, 69736.80480066076, 73577.00573689616, 33869.362087635025, 86572.83202902121, 79078.60073525649, 99072.54124436936, 80822.98643497369, 35505.55901770118, 28779.618297000685, 16466.977764911393, 15002.271449537286, 1862.3930589732552, 14605.801964360953, 5021.601879087667, 30816.653918598637, 67879.97820161114, 26721.364043486017, 72810.17639850074, 13897.884142406869, 80970.91829639715, 56156.511702987, 34329.068112051864, 265.3932683438942, 86997.92456062685, 23035.417713162242, 75585.63545815344, 83296.45334576037, 42490.76015428256, 5464.186994723441, 44334.77229144075, 92948.28324481043, 91692.54696140432, 10200.365290346925, 33660.89224084411, 66193.58210074353, 82000.94535435205, 46639.77274537346, 78671.33189222512, 79031.6341094046, 68002.56312991474, 75607.06447872559, 89339.1532155563, 32536.069162510583, 75845.00772902995, 1950.3477458535313, 47639.201544893076, 39616.78838666196, 27920.24143611367, 7960.118478410294, 13620.723590208161, 13140.07047651522, 62849.88395645848, 40106.382066253944, 19496.1146551062, 31667.140107785097, 12082.737063503357, 97522.82661102376, 61938.80411439412, 58402.33002224299, 77962.68434990542, 91986.0738688766, 79512.76496909716, 50058.78039612448, 96602.1122267131, 8696.408296214531, 57303.93935576176, 6896.543291274793, 32422.199363208903, 58798.90402022143, 77218.89981964625, 6412.310510455643]} +{"id": 1764, "vector": [17121.024465473798, 58194.224658575185, 26590.769177927865, 11894.813955568407, 98682.66666028547, 25608.433653485263, 65390.797204540984, 64146.511556120255, 76822.48772865882, 61992.11170849007, 4721.5813099723555, 57640.834132571385, 53888.37028842605, 4330.298669806875, 24943.01395030446, 74729.46769966658, 69458.51887431036, 7053.364417410179, 18411.34523782195, 69646.32914179887, 41878.471132944585, 55080.97688574878, 45705.3056925248, 50633.67718906021, 56346.902004317366, 18060.647950753817, 40323.956282936604, 77195.79110738823, 19207.51674339626, 24354.53466764851, 12450.897215112167, 77234.54734210676, 27421.591311966407, 4473.154031811655, 5042.105230995197, 9236.67725472158, 43953.71051942435, 22332.83969524873, 95716.52585805985, 20155.363980142003, 17808.530821097578, 82955.63772266365, 46770.92092737345, 94967.42892170935, 60403.052362300834, 33225.51077184558, 97031.55530985644, 80367.28433615694, 58676.671142700245, 21778.529136170855, 31652.550207937, 25904.70506305307, 28694.69585489891, 36818.41523616342, 49135.14201393204, 7882.135401854728, 15862.810462790521, 87948.90107957103, 79299.23633313947, 87068.17818292617, 24073.673934684135, 58047.736972801824, 5146.268429259871, 75321.04206201374, 3558.349872853106, 19100.388065510022, 96189.13165641925, 93094.18600126886, 406.2479735761015, 95903.90976750357, 37468.671179017045, 61267.40654672957, 54824.84126639252, 39339.65740122395, 12050.383299400492, 87378.67042210733, 80592.51876584966, 86170.30474972846, 45515.04377691543, 67650.62141104677, 24481.801365967014, 6849.859753325804, 1077.6671845115548, 44362.914174113655, 44414.401281699946, 55265.20364916712, 3065.6532272190116, 35119.88850055105, 48387.39724651549, 77961.23165681779, 67911.23299453229, 42041.3900715942, 64318.292972147974, 78149.65021803202, 5521.416999003714, 3466.4994626499856, 69719.08264350159, 53627.726016180735, 26946.864852281506, 19752.47224378279, 23829.67953631454, 30089.307405597043, 80770.69921096406, 11574.045702052383, 70348.15217206368, 81203.60785185645, 18467.900600509012, 11401.014939028575, 68419.42509040979, 42093.28754522019, 81878.52330751189, 69946.59390758217, 19200.278819653715, 98173.15987932397, 91220.14442701018, 95689.60583126756, 68422.64225933165, 90627.05619491146, 92135.05976933804, 94802.19897390639, 19829.722085665104, 98312.90506751995, 77540.85453894583, 87217.6071708152, 79105.32796935152, 62162.61335388085, 18398.316372607358, 26050.851955037633]} +{"id": 1233, "vector": [81537.14085079118, 9763.707794190635, 14945.759260830604, 51081.188422822524, 3170.6850300177503, 99223.5239338023, 29065.674520948804, 35172.65991666855, 16780.686131469556, 16951.62116352038, 40864.79590748935, 50133.46816985923, 94207.20386814693, 88190.15229994287, 77241.59166416936, 45045.75330688756, 44679.004226757366, 40063.485400956015, 96680.07780897246, 8207.248948545897, 34718.02735111489, 38665.69424607286, 50115.89660900554, 58385.401855910466, 61504.85200420873, 92131.64164417003, 26189.704299607085, 82471.54003460634, 47331.86455147509, 15123.579347131732, 73284.60083764505, 70047.00432055687, 68572.62945148315, 94475.3112574595, 44468.149825523295, 36557.345197699964, 833.088229411727, 72794.89772838562, 3902.284228726316, 96112.39014544785, 14817.375555947454, 52966.80754494507, 5807.992210399127, 32413.272432705875, 78763.5869276458, 92723.9329895176, 97050.942354516, 21118.4872864838, 55088.07881455126, 92278.13171792273, 40656.30205155149, 51271.308821798215, 68661.44577451501, 45314.64443818415, 17544.734882005043, 5223.821492484127, 38111.646695383795, 85849.09138689745, 74987.65143571667, 29007.301336478497, 60478.532329933434, 8275.789416441427, 42503.50073855962, 96593.70020588608, 56512.80856193751, 51047.51225119636, 67857.9122005943, 81597.65880572058, 27295.324686792166, 89194.32026998723, 75433.09466923388, 79497.7345812759, 64273.37248830029, 22013.89406134184, 69558.67771360252, 49603.80236692983, 72089.33318725268, 55354.5907038175, 61217.11892477012, 47783.42347565715, 44523.963161481515, 33503.75739585499, 5736.746146620552, 16337.055502524589, 85713.38767498881, 34148.487487977254, 58105.42578181509, 65112.649219402665, 82454.01867062427, 92249.52770681537, 68719.54717853437, 79778.60735461474, 51591.1427909142, 50333.26423113996, 96076.57313798471, 5020.297025727516, 8506.19952324343, 5097.796125884346, 7571.953815967836, 38039.504127600456, 46706.367740925794, 69247.68967010725, 41598.358441258875, 54081.92943447159, 29098.663622125176, 6471.418584048283, 51361.680950354814, 72656.85272769081, 20883.67726706458, 30077.912748577928, 49443.42004004194, 49682.1995912207, 70475.50307004042, 27921.835850773856, 61161.07662426517, 48679.17978223044, 28358.808891283028, 63101.68942816731, 86133.89961258514, 11042.301130511645, 35419.40850394297, 47458.32684777037, 83909.00926623639, 14678.402854618977, 14006.511094522079, 7131.629457030408, 30007.012329976034, 85079.63837636574]} +{"id": 582, "vector": [55353.7512140156, 66523.37120654796, 97868.32879177514, 86116.67904241265, 55420.80099669669, 86284.07939159212, 64225.07281508076, 61231.0806220847, 84620.45290732408, 70836.06687809178, 7301.812752769232, 92080.809446595, 13440.337691353943, 47422.986832353185, 26269.97503358841, 5468.964051364167, 77312.3350725433, 20593.14763634581, 51787.98678691941, 72717.49896204918, 30310.083141227762, 89433.34678655629, 38536.20932631491, 68184.8689658525, 10462.605703766138, 8227.363035398594, 5480.996449035935, 87770.28078594293, 53958.15733282737, 4873.929322781012, 74010.52660409997, 71963.76974488141, 95902.28780006107, 86839.60253772058, 62083.428593861325, 38097.70246746479, 84886.44302033956, 58874.4475486804, 50450.45205504177, 92082.10490612619, 16802.75600428419, 42602.83878426837, 67519.82054771944, 20399.018143620375, 91114.72485161858, 11550.214158892435, 12892.613498726558, 26419.758332842037, 59682.63769261688, 63989.39153609753, 52629.08689820658, 45688.29135485298, 8308.7523420308, 40923.02973381594, 10486.68955686276, 88928.81055965394, 25853.949668294885, 36986.84034690992, 58473.296386092734, 86738.80907857776, 71342.4702329004, 81178.99624830973, 5963.042814347741, 39037.81893193154, 69053.5874720856, 2784.6636193328054, 54948.68715437438, 92989.91763658094, 54909.56277978033, 77395.18698734205, 81218.43719857799, 99340.83242454028, 61892.228111129385, 20758.96180158412, 93614.34058711617, 8597.028011398945, 30827.201858803164, 1059.8523808837078, 59625.644434998714, 86794.53589218202, 31320.934844687774, 68636.82172413362, 24046.362462650926, 10705.286466606922, 92832.2550983389, 76615.28179333457, 18949.195038916423, 58376.21834197897, 9775.255306747888, 89948.26154646187, 5507.094135535517, 33444.7062644532, 29161.759069249016, 87304.58855215403, 54608.71742736086, 50190.876471773685, 1865.847528314013, 63414.21952277509, 6605.798065115975, 97127.5325259587, 72509.82651617906, 81501.48958156048, 48012.011847663096, 84580.63453448018, 37478.0542358933, 19936.178990605193, 31451.679609154326, 76102.60760676042, 8583.653062978336, 20690.0505657167, 39193.4206049817, 75365.10988518054, 69798.40283827797, 16405.035558063817, 51039.191440384035, 1709.0265716449226, 44644.667353386336, 9293.781607792294, 30009.173052205464, 37835.44073790534, 31283.238388585367, 58296.6591629787, 61221.32152513299, 95878.8847278707, 57427.96681083043, 62596.27901996491, 34013.78607032733, 1308.6801400724269]} +{"id": 1263, "vector": [33056.65498671857, 40075.66299771288, 99587.42558748773, 28015.51998844366, 20567.19698534597, 74897.42113393145, 90744.60506522474, 49875.617228079886, 37400.39455428075, 21975.961229111028, 99039.29264120925, 24830.02774395503, 21812.88264124508, 66188.84910533654, 69179.51573465728, 20778.234101269856, 2178.6869705373692, 38945.6337299763, 18842.67544112238, 90060.7303247126, 46282.73476669259, 43473.039245438784, 33128.579939229705, 54705.21166465561, 34621.93108309761, 10131.511332862165, 39845.71631704409, 43814.80161152998, 61184.84059232344, 86549.09445693092, 3815.2045635267195, 65820.49983139765, 76675.24840069248, 82876.11579710137, 34157.07253753122, 50382.35850425997, 45695.55994912485, 93804.41938500304, 20040.19205328198, 21429.971994827312, 96281.35456024016, 97468.86872093577, 59416.18117278925, 67538.86162920513, 90643.90684573432, 49297.59337647286, 9548.414418500739, 16690.758439568788, 52409.97946032322, 68045.44959701583, 62960.15520671192, 40082.847787462946, 18626.376266666644, 30860.25289388008, 34994.334502032296, 7385.509915657318, 7058.796012419332, 10223.137442816333, 73457.89061035425, 86287.34285585841, 88685.6656474065, 88627.9958585168, 73132.67995450988, 79832.04345075306, 43222.64147965531, 48967.6753414589, 5292.547557595284, 39678.14318499311, 6681.973041268807, 8145.104518306601, 70870.55698635471, 21115.243994181466, 43932.999876478185, 80521.13452689258, 81852.05958158372, 60655.311611791694, 77361.78741032623, 67761.93700901204, 26323.874955791605, 68517.73160009865, 5607.28256760259, 60101.18096839203, 20610.616782737045, 21535.749078393572, 76967.5499146854, 27093.186527275182, 67882.27657795357, 77252.97308328054, 83653.15124245652, 80452.75284701205, 86667.27496379495, 30095.341822349474, 63700.06157511211, 18985.327336609724, 1241.7008460662514, 64685.602967094004, 82377.35660133815, 99834.11549450191, 73660.02353183042, 79805.83979833542, 46907.03016414456, 44182.547574609685, 58675.998975929775, 32978.83854998774, 9559.920462201731, 10792.735358052663, 9751.054552000627, 32138.673907367356, 22018.868637933396, 45853.782655937845, 6320.400000190918, 54516.07506228603, 79565.58242832392, 43389.95704335147, 7160.882654560119, 33456.1728251622, 96512.09732647569, 20620.39640322627, 82861.70931516061, 77125.96382528084, 36461.81847245952, 71477.85661383365, 87724.2643620942, 2009.1142296863484, 28329.041177519954, 69561.0976979405, 16266.761554473664, 54053.74590825355]} +{"id": 1866, "vector": [70595.18066653682, 44991.745173040996, 22317.99423434838, 40801.61348749529, 90734.20812569081, 4383.66200525091, 20501.58568249346, 96241.63596058953, 33744.82436433749, 86401.27884339487, 60044.18952704776, 60997.414034681584, 73981.00979261636, 76155.57799501272, 44822.81619427595, 84085.85959850991, 57572.83219210172, 24442.129514984244, 23122.70665232208, 12936.498201629543, 6809.002004646291, 23218.180566587354, 94103.15936045477, 51786.90615446221, 81700.96392684914, 98026.88602487283, 21467.66459176943, 5554.6030202769825, 47532.96123039481, 66578.63113047993, 74969.3656244531, 99995.21844875948, 55349.040120945116, 58862.116485291874, 54054.40206336871, 96515.6420007393, 58163.682319993146, 23218.07240740984, 58814.67946380264, 91389.42621012223, 49123.6432576995, 81790.97916019506, 43540.2367363292, 64853.58923297346, 83945.94662107038, 68966.02147283878, 33171.00087590195, 9685.517322147929, 96825.23886851568, 95769.72900227655, 84821.11143581195, 45460.20136748376, 16233.28897040579, 96328.52875933738, 77126.51143636959, 58951.87103412087, 74540.12598998369, 97287.15872777808, 38541.147352967164, 75965.04088292114, 89459.61132975487, 82535.43055631417, 17250.374267222767, 12546.62959933982, 60270.64395736242, 33521.40625746935, 1660.1472735803923, 20344.15769085617, 83489.49178462113, 43622.40531006055, 59556.67216724986, 50453.7743039912, 47100.06733596399, 37213.389832557055, 15663.697777746755, 58069.225457053944, 85909.017550923, 76669.00076135917, 54298.90898863473, 32732.606398792563, 69771.91169327275, 67761.55832159612, 21632.935286340817, 28347.022604586447, 84891.58881382752, 68466.8154105783, 39593.221398648224, 70721.50022033234, 23332.79892064194, 69996.92022867512, 15594.239954790966, 7437.657501503503, 506.1297612048654, 96571.64539687097, 1019.743064712686, 82053.13839166403, 4472.520527839652, 96628.8156362738, 38467.565030359096, 84706.90441696829, 5701.157244337774, 94455.09765326836, 94184.60088127953, 7219.171986483663, 49499.50121005771, 49511.93048658751, 13615.450619866831, 76220.84204984135, 47804.213369435565, 68934.9251501306, 97585.97417971176, 52299.804030388965, 4330.7705846116205, 68072.17041188202, 46413.40522663905, 80549.41822993802, 66172.77023781363, 54864.08991201763, 16574.13987439437, 6577.071991465499, 98639.03824629354, 88691.76101448489, 14723.426542240759, 51657.30270303388, 22225.162935132004, 70599.37154800481, 74430.2781988855, 16243.578669596925]} +{"id": 1732, "vector": [69157.15384834615, 32874.387197411656, 31003.38712047943, 80199.0364485089, 69798.40526819088, 13353.284370215202, 60126.91668109863, 96856.69734921472, 47177.69284594682, 45830.08279730092, 7497.470862304589, 73857.503654691, 47098.10533613815, 56501.873541196226, 67391.37672488893, 24632.72576510076, 885.8950597145143, 49257.646185471036, 65849.76692849674, 67744.61732217272, 55132.76082046364, 11339.196223685898, 26074.502513016483, 44196.88797942597, 9394.409068379151, 63082.11494539956, 59837.97499871354, 79362.22269394864, 79799.27327625231, 23914.3555863083, 69083.69195185014, 97695.5783810198, 70007.85418450204, 3747.864574799464, 79047.42266812306, 13802.290176207365, 20472.15734306711, 3365.708257947797, 29904.677558589345, 63156.514692641416, 59712.22753388461, 98431.58519638871, 13250.530193611565, 8226.027157028348, 9263.16078402779, 49543.04832994285, 7561.80001328699, 77454.48732545038, 1448.7920934632136, 45385.9785050573, 70676.13258209523, 72941.08064955761, 20436.392700265293, 35573.36582750411, 68153.85886077952, 13974.448211692414, 38054.47981012574, 63023.90473049965, 65251.33987932541, 22781.800171628576, 9000.905849677165, 85808.54059213727, 56512.60064094785, 67712.22692447492, 97218.4035104472, 49244.39287374471, 23764.984405549672, 2782.3745321296833, 50002.21770041522, 91596.3986777334, 85466.4206202868, 18512.605128967476, 89693.3405533107, 54717.966654471296, 64733.457826664075, 17260.38857326948, 9171.666121658183, 69696.25856063649, 38207.32844959571, 64892.02267758527, 56298.86584483753, 86124.0108676556, 76817.00526061194, 97330.98102330431, 80399.23135804952, 72225.42810848379, 121.09969332482207, 71971.34956396009, 74411.02304550866, 20311.516298620867, 29515.52979652873, 9668.775806027863, 297.70118161601687, 37919.57693346345, 46865.391517593926, 86897.7400112282, 82915.34084449436, 93905.8409762789, 12754.8668672117, 88791.43421763397, 37866.669892537444, 34451.2364125977, 15607.628804420327, 93968.82086228594, 21761.67523692809, 13117.08282560925, 64083.164613753564, 37493.1782272533, 66996.80261804858, 67595.50431111315, 13111.191682503353, 76943.82385805108, 5575.644713088012, 77316.6720091948, 65348.35840428938, 83227.2955371228, 66478.7860137351, 49586.006067576, 88132.62208034407, 42985.59537702394, 79171.27346936232, 21521.16014530834, 80957.88485656411, 88328.49278749661, 47966.16649874415, 20153.177215487172, 88208.29054767817, 77202.54728289672]} +{"id": 442, "vector": [80044.15396723058, 14705.439060364379, 34757.69654883235, 16241.880033353884, 97412.26786910131, 17189.211542638448, 25729.795896056152, 39233.385726635795, 2075.425773702666, 91408.49721737574, 49998.32870798112, 41151.27381808798, 81679.70830034612, 82215.94898484852, 98444.69832499661, 22962.611534332857, 13817.921513893327, 38030.81400674862, 25143.99850033213, 77232.915390829, 51212.039289624736, 82121.37639726046, 75318.8636271355, 28310.50780139658, 74917.78796496581, 41466.62956612641, 34401.05382896359, 60047.74103922558, 64516.815048670214, 33927.17045714193, 29626.14446258125, 98364.90792607916, 19427.60427360989, 34453.33384751215, 75633.51434903388, 80758.79659968328, 85581.81035623084, 97094.79038217221, 68566.57990268721, 62659.20488211884, 74096.45899550454, 23016.508244110777, 95660.7490813683, 15716.229953694594, 79564.53770856238, 85267.23244483628, 91412.45192132835, 53666.42119301247, 46391.515895758515, 95758.0636877091, 606.5724830168317, 21336.87818537222, 67489.13213506436, 89709.95365610573, 22879.747752944448, 34724.351407758346, 14867.890500769721, 68291.60407586778, 97618.92708035882, 83586.97819706168, 11584.957426880716, 52637.712803488255, 96072.64724938167, 4981.940915039074, 78420.9658653102, 86283.21125737405, 98741.04193464346, 97291.78147167472, 99337.89671225465, 96939.94692820542, 51059.471092165506, 97398.95025753412, 3072.112372316005, 59184.27853895171, 9078.699769824128, 71094.10526326526, 39243.2582842104, 47886.11658845573, 19055.37496523474, 55243.96300271763, 67388.30191639838, 5270.548816069998, 23500.80116641695, 34768.490253961696, 29674.369118865863, 50638.06011327073, 41929.957762154, 61743.00219823742, 25981.61449581351, 17307.372966256728, 53076.44438005821, 80362.33667821938, 18065.31436417882, 86906.85745513871, 30943.1247233245, 5302.146731749546, 56201.778797219056, 3828.289245775196, 73059.94845631896, 61025.71325998628, 39095.093336744205, 46357.74855042201, 16506.146359853425, 43582.04103631481, 15871.378937046144, 27124.797116109843, 72634.47332243045, 70959.01436793723, 16072.982461013618, 1852.8455629747255, 43444.17281350175, 78702.36733509281, 78711.95403657443, 5254.473561942141, 15180.621826943609, 28150.855862536428, 49985.39625901386, 61398.60576005298, 31673.40893031858, 85157.22293565249, 21243.32830521789, 18584.385552541415, 33148.00036436302, 42135.432335923964, 87003.25518269796, 65930.81582985337, 58964.30395606014, 63124.537484673434]} +{"id": 755, "vector": [33592.12851539425, 35077.65668552792, 24531.137486677413, 36688.14797427165, 60114.031615451146, 2134.9170169445374, 9506.681559321329, 81183.95674489297, 8857.643589787102, 85461.29682969437, 54991.31431354479, 12430.060856002745, 14717.423176157574, 76557.54520876559, 92211.00251877586, 52890.996022047795, 99833.52280553637, 30324.230928377216, 84424.46022579804, 23244.474347427204, 87780.57580995621, 73415.18222591383, 90686.98597829761, 93540.86214031396, 17787.201084479333, 76702.71785729256, 90230.70448513224, 1504.2316162071634, 43527.79092361533, 32349.306808218636, 82912.36076868426, 45133.10210475745, 10340.19211079844, 69976.04095715702, 42004.96343867992, 55478.71824817141, 92242.00540742584, 58072.42293810875, 55598.99062794965, 109.68336236636401, 37650.75622167406, 17843.885920755787, 56820.83655751488, 72736.5834939312, 12909.388717618642, 70729.91902522273, 25356.987840835453, 55942.048910374186, 33315.155002874475, 39267.88056707634, 72688.78376109703, 61217.400320552064, 36452.45501920842, 75497.86063173086, 66735.62002509985, 75552.44967986032, 32250.18822151148, 46461.1664831028, 77954.09393234468, 59891.01729534248, 35562.75428746616, 17595.373652965405, 90925.40186251818, 9518.84381483128, 53689.429926235855, 38724.101965358605, 13429.36885155297, 46931.3894815118, 16981.38021487352, 2584.280390748661, 11112.462181252702, 49423.36751933536, 80765.65431937795, 92000.41928776805, 16930.09242898782, 46182.61419620117, 39687.85132434267, 53693.067800716395, 44695.03548860622, 52879.17105816079, 33546.893297336275, 66028.9542267747, 78658.12714861952, 16989.302456735742, 75791.06524898033, 60745.08619887941, 74992.6093796163, 94510.55912869894, 72579.60347652182, 51792.65132597801, 80777.38937784215, 64250.12955336914, 94357.53871168046, 70898.67557293917, 37827.72933054024, 61265.52666182357, 11311.89913085341, 84358.19918759486, 17528.65418730278, 70660.31458538071, 40894.51381922623, 29718.762436471334, 9591.62637211508, 14099.57822995509, 50232.656133225595, 68662.99460256095, 75232.35730020673, 2475.156093217767, 54874.685969017235, 56098.94844372205, 77199.47958984428, 35892.801965697174, 65291.42737452929, 52071.00123905087, 36771.035684048635, 6953.508552008625, 68442.30071095043, 91002.93127166595, 91104.65490392326, 1099.1690981830836, 16920.711592132386, 14231.260039650673, 82556.94851491143, 55604.64290921905, 89550.85626621213, 87723.00301792633, 67199.26665106408, 27412.707951143268]} +{"id": 1716, "vector": [12988.42905485591, 69375.75769530323, 35499.99521922164, 70765.05258625734, 5518.49055112803, 30184.567392644047, 54419.3250928186, 29928.513168826143, 58254.17793981691, 55368.4289802856, 85561.01541991769, 84156.87492221118, 59118.38526911991, 89521.38596168546, 88751.15071483416, 94625.9516206619, 77667.037702314, 92092.6295537231, 32599.69129860192, 96579.48036122584, 86282.27727296442, 78005.22208764718, 69401.89819670154, 87504.20844254429, 98893.74331424186, 74301.80555242221, 55157.14053403943, 47123.40124426579, 73782.98732759924, 74584.17513772516, 59282.431297491756, 6075.781791823276, 7558.832216167255, 84665.37386789233, 9245.461827621904, 95361.82672712576, 19962.802867873175, 61038.30533062422, 98080.2763543739, 2281.4914989805925, 420.1195081236064, 31133.544858097506, 49643.894508405705, 56041.916106807366, 36840.36705666657, 95659.46599368409, 19293.85823212383, 81544.22889173971, 15641.761057785186, 84746.49028522981, 43198.52261081988, 97861.15925305919, 30950.49148383857, 51331.533612409454, 42458.98212933121, 32792.00912220414, 98618.14695191989, 96842.5317839132, 56666.85369001471, 76629.02747503875, 33505.43184628496, 34290.51778720968, 88084.39665753659, 78754.89077917048, 49741.40629893545, 59222.18921818944, 84676.4837322538, 31008.575848468856, 46368.17341069118, 63884.77211028313, 57006.27077637077, 85793.35232275478, 12494.552108818734, 83034.68133042383, 74454.66373538409, 51123.094765197544, 93340.90632594831, 96551.63783198447, 68178.0419443452, 66283.50726245886, 64427.47952844676, 39058.56707420471, 55106.09122313582, 90391.23451780967, 63660.26302120941, 85009.83702179439, 7248.246251763524, 62621.31264537058, 43223.23714970768, 84977.73656279201, 60947.390264017966, 11182.122476902923, 89526.71933868893, 22755.82207856258, 86311.62629738046, 8549.281730470604, 99842.77742238181, 42437.905454209365, 10548.573542587203, 88933.97432061748, 83749.35314790405, 27724.829455618637, 8085.937406263121, 23152.25963861144, 12604.679339954782, 9228.496367181404, 27113.294306155665, 60379.73045473203, 79490.71795632441, 10328.299481900705, 53993.96276997018, 80247.90126817305, 34499.77393147725, 76076.0935112553, 43319.59013400378, 72029.28771997218, 20881.933012259502, 66983.93040467733, 92701.9711689702, 24955.032046934877, 39995.32371078516, 2669.1602951010586, 63194.237790250765, 74053.10235887414, 1689.1498798095372, 28446.825142151945, 28618.818063464045, 39190.87221679649]} +{"id": 1222, "vector": [48228.78990198572, 89743.14088295396, 22522.947147875282, 24683.857476545967, 62424.46829028987, 83218.32471645692, 27807.973033514187, 77710.01154516549, 71463.6193908186, 64679.77510515003, 50707.731456835034, 67199.90120437906, 25979.296791932327, 55701.335540714645, 66717.30991316473, 91724.92995498345, 65643.12947982001, 11991.020447489764, 4272.146775009622, 66491.8493557621, 57409.90172629549, 50993.20902648058, 84526.15584563145, 86194.9611335693, 51320.15184006457, 29075.81264377527, 77975.5397201409, 64112.38290093424, 157.43414694441293, 55843.048044967945, 78500.11626287099, 29138.46799520381, 33510.148943404885, 83346.20046814728, 79688.82524033265, 764.4363354483374, 32958.741082670414, 19776.896443660407, 1701.85943388107, 16764.25815864394, 18997.56499470655, 14452.67809662244, 49662.4365862471, 61547.96771336583, 93318.84685337912, 90311.0342610535, 36172.619995379726, 83921.89032348203, 66738.46959052903, 91826.26381279695, 46999.6925498076, 92323.8276188042, 16838.6025013735, 72148.03168228848, 23562.770612146433, 72679.27555820836, 28970.01035850869, 78290.83793378447, 76700.17768799327, 82846.50346332144, 98211.48421813978, 17149.633598605727, 54464.38585349055, 55235.47937016764, 5815.909689765619, 73820.86744369725, 27992.23627068914, 97686.19519295478, 17071.566732883213, 97710.85435348943, 64535.36107054684, 97581.132133003, 2661.460796512105, 18439.866175949894, 57894.453245763856, 27612.59292629221, 57967.32409527889, 48305.47508436346, 72726.36087773688, 40672.034398114156, 47905.09471427543, 89483.58549667298, 48888.42518496448, 37239.92099690815, 38375.83564354141, 54721.90333143474, 96258.91315445775, 31293.550005072357, 20444.108068624457, 93981.24073479736, 5881.170089708942, 27084.80013855693, 83927.50049969669, 14798.91960706239, 42772.28650351258, 89231.65215397924, 62479.78503160596, 40538.19660228072, 76427.33996449097, 62455.14202223519, 58145.092194329074, 1904.7438676716877, 32193.33429986907, 77463.80840558687, 98776.58899365464, 55520.583855875404, 31353.206311941405, 77275.33190474393, 96802.65971759129, 24975.687765956365, 54473.49607939652, 23036.6059427909, 84411.37049563864, 67211.48296600948, 12977.59329564051, 40396.086630942984, 49359.10359492238, 18941.043117270052, 47304.890126678554, 33454.23041838056, 59763.36609092211, 88646.45930773226, 18704.45316828814, 76536.61903115499, 14065.276916545656, 37832.32834307168, 57152.765709731444, 72710.2036259138]} +{"id": 285, "vector": [25887.069333018873, 46234.79201996246, 20969.079069277774, 56433.51066216155, 37053.29124407601, 71065.38439070724, 53776.01900123538, 23298.952126552053, 17417.36683553372, 89946.69995052826, 58727.96581539264, 5525.318576448912, 65685.47239286717, 2767.061860231701, 6525.749098311062, 33207.220982981555, 93531.02953974508, 13675.367324411047, 10422.878511784005, 79972.35002998149, 76145.31512819088, 50833.422606697466, 56144.42520982996, 9466.776118220332, 29739.849218551608, 57805.580219807365, 76555.19234499028, 9658.62713828155, 62455.39400471557, 37985.498083298866, 1522.1469694372436, 26184.11617026536, 37274.7429482817, 58567.25103736612, 98397.39201727192, 58284.28650398037, 16956.9439454654, 50735.12057152863, 84557.94941391546, 13934.376838794871, 3375.928726539379, 89598.71550556106, 4531.761251615185, 26331.7869193588, 12968.03671749015, 540.1006347228399, 18526.936516267368, 18369.762095849772, 59526.7758178167, 67917.40493349147, 74526.68792814596, 48428.13121430718, 55167.3172221631, 17617.29429571629, 72757.81204836536, 45077.657471595136, 44822.08307937451, 42549.20565125946, 15131.047105291085, 55341.0116676201, 54821.48543729012, 79714.64948657418, 30415.981477194877, 64249.62144415629, 84686.1692216939, 70858.82694550003, 90689.1457806976, 20512.009144391173, 20577.960504699622, 97189.54575853162, 48850.18307959726, 44424.193150764615, 68432.65938856518, 85270.77993873025, 98398.66340885132, 72841.5420244927, 44732.39371635578, 20205.33007755342, 65191.12154722069, 51960.90074115075, 46575.96820557684, 31579.755265279262, 73518.77861959115, 80102.124709338, 90441.60882985636, 79980.05311664747, 56003.63243852077, 58656.569351837505, 48257.769907484006, 17894.28064071851, 30718.605603216052, 69516.99876278771, 39756.500331525436, 51051.056750125405, 95665.13659274134, 16477.31393361056, 36749.63887093614, 29529.288817349065, 10607.25643736371, 43973.980173565484, 34519.69500390953, 63918.23379856303, 27729.597123746018, 48638.50247322607, 12014.015908684118, 64400.419751198664, 74698.18568155817, 17474.14462624335, 97509.6551483208, 67697.9670713674, 85945.07349255797, 85424.95446178401, 32556.62674245523, 68123.93718067904, 36038.124615772824, 30054.265488126275, 81170.71622338367, 83085.49613583261, 98745.93615112135, 87402.97407216698, 27230.185271943654, 17460.77131216971, 51036.87063120145, 66650.022268679, 7657.299966285158, 61767.469072752814, 98486.47398194189, 15771.50870819377]} +{"id": 219, "vector": [30824.43154158315, 99655.6439978695, 83951.73593770037, 11938.829952289942, 22027.784860835854, 29898.458572313833, 77725.96380698074, 21056.50399179667, 43574.70646181783, 97851.93839369735, 73016.8020459745, 82106.29056502877, 17160.905669393876, 31202.676563128407, 48314.2052263836, 87244.35282149249, 34107.59745110463, 364.3734102150464, 92396.95869778031, 46603.59229523303, 18041.210209582525, 22916.820979874232, 96321.27009585475, 65114.330111091935, 82107.06921526298, 14722.869908169201, 17.720611941263176, 14403.709647227059, 57157.2910294058, 48043.86281577846, 60172.64687048182, 16696.5673524061, 59132.07576502886, 5047.2826918737, 95607.7779469759, 40955.926696504095, 15684.52252178012, 89477.70964930086, 68651.18433786498, 41947.37345861765, 75300.45359092466, 15711.955822946866, 68421.7730189301, 18283.313819381165, 69701.0492476347, 62848.20716838951, 93776.6691990262, 25902.428335257886, 97127.4427650725, 62849.96549263655, 70793.34815146016, 82488.83193541336, 58156.676544897775, 89445.36327944575, 5883.1675790559475, 57452.0122523379, 77012.73787555823, 94796.41011950938, 50798.29798369515, 83694.79573471463, 91884.11397204822, 95812.08857646331, 53066.649077341965, 35680.348460337606, 33887.54127097112, 53858.639498031705, 45903.14455762468, 30895.48184100941, 48023.38853515521, 76020.77855679316, 76454.03514166392, 64204.345084239714, 82368.88356217435, 70955.74328894491, 39358.22135018191, 13081.406722448684, 73000.25565076625, 63093.15421052515, 72283.94565672615, 40977.67012594282, 68292.7698249465, 57660.66042854938, 75986.8801771345, 71137.41142125771, 22938.693937556487, 55299.420422036485, 28764.503328396862, 53062.56368570316, 92393.0061660942, 1101.8218135232516, 9983.05696069275, 79451.63293835503, 52443.742988774415, 65811.45093616586, 89560.08514011811, 75685.5023246093, 67836.79451151473, 68430.39973830845, 1360.455967817853, 48988.912289860775, 21324.09525152369, 34754.21282678618, 80726.4849815297, 67289.48841836871, 21936.111890817367, 77085.17301828397, 36176.07508095606, 18309.988305407154, 35161.323099430774, 46563.709790801346, 18855.464279215128, 97676.52452556304, 32601.45598938462, 55448.4454405671, 35051.88938345277, 13886.666434397066, 2310.2517635697927, 55079.68209928335, 94342.43779063495, 1370.7858139190844, 81321.83303983697, 34514.50044007376, 26230.512261592732, 36011.93147965105, 90840.66675560093, 36014.7350605293, 31447.32033978326, 841.5954571999839]} +{"id": 201, "vector": [43377.13654031563, 54530.270897951785, 15646.135494519609, 69630.25844883962, 56978.21543888922, 53247.835036332355, 7240.628061968379, 41586.117066570405, 96410.69013546503, 57546.41567691501, 23179.573294269463, 74639.27185281432, 57163.12970619981, 35581.87854207191, 51111.04390826599, 79558.13380176385, 13856.105801753993, 23868.735068043756, 50220.62355295515, 28034.16384755839, 3984.794518943524, 38719.786846615745, 10339.350349540266, 14338.968063486513, 66322.78070642661, 31830.035588396287, 39823.89700326795, 14421.519035534448, 7422.222890495389, 15842.815344467053, 58522.12565447502, 1276.7723107383833, 68128.41979582245, 57801.57477840797, 66525.64972045891, 50191.32940017953, 85141.87580436599, 43590.93998670565, 67171.47115117387, 97894.70340209488, 32438.24851317627, 27914.229507274802, 42496.975066631334, 93602.40145880525, 9757.49968818902, 14857.177546191613, 48227.38265060821, 78453.60026059451, 74013.11829656528, 33034.56309504477, 72187.00691247564, 85102.22931441774, 32992.38967408743, 28595.462863105247, 70329.5303999188, 63048.479691541856, 62365.93500566881, 86287.84401142258, 2578.076385461714, 61696.616907762495, 40460.156504574785, 97948.60811881599, 49430.71984245611, 93763.32474917365, 36516.591282076784, 31411.21147235334, 98080.6450956893, 5642.159174073747, 87894.8820277063, 19589.6216450709, 13249.73489943545, 8825.6591198126, 34616.74722714681, 25583.814943581372, 8352.44618094122, 7667.1907409337155, 31559.097195908125, 79119.25227199166, 53750.25347795306, 21211.03585146211, 89097.82726138626, 44827.20138558164, 92188.53120947475, 25989.911558341282, 52975.68165911949, 11344.632890579209, 50712.08053323991, 8673.697384741907, 22392.53308737483, 33053.97867069608, 19660.717222420066, 25369.503152606456, 76430.0426778585, 36665.6116804969, 42081.91340783352, 62344.1858643783, 39951.38268927166, 92502.01470203059, 64681.08972175101, 71312.28858320715, 42976.86733755889, 53016.01203614697, 66923.65529669578, 20508.06601784756, 34901.24141826838, 72272.18904595105, 5939.6568538284455, 49065.166156086445, 57583.88388053167, 29516.769262287224, 45345.47017036589, 36031.13073681076, 3098.9581588825145, 31584.112029462685, 42109.23679204476, 21715.793468547028, 50196.93576407923, 93824.03571881802, 34505.50336897525, 70932.17059512684, 44724.73546147394, 7795.578660410707, 95262.93466179134, 26603.957455805128, 801.0524600233659, 98542.96282635684, 44698.4899093421, 23788.120213405462]} +{"id": 850, "vector": [56527.89290700572, 20492.426746510308, 30390.939546269223, 84464.20994542602, 66802.28789614476, 30428.239576628857, 39255.5538045312, 94390.00470947154, 81320.5755300911, 52755.49536341618, 52551.22365359134, 91082.24923312187, 51361.23976714124, 41327.3256137611, 50127.760900611254, 27690.312311118247, 54763.40137723299, 65924.3212417456, 57692.89356749072, 89990.62034849277, 50316.678516434564, 28602.08510089045, 58222.1919757141, 40895.3227365573, 82315.47863850575, 68631.38859287748, 65072.94317104679, 6646.502382247521, 40783.1192356081, 9486.048300729344, 85584.25969613834, 68827.85016873118, 25705.409261034605, 46356.371880310384, 38421.08776029622, 63988.28554507697, 74132.11558897955, 71521.84424171264, 22508.313784261914, 99127.77519998974, 86462.83702346674, 36866.79871802097, 62876.68755330311, 63941.972667396585, 19212.344742263354, 58923.49458615188, 4260.996412740814, 40845.767175641915, 80253.65068462967, 68463.01908993341, 90417.3806023171, 23003.545589241192, 67509.16620531394, 88106.75832231384, 46577.145572093905, 51323.26504959607, 67820.66125629656, 87045.71284880993, 57365.807808030135, 43631.68424991289, 4197.862114532036, 55600.89582328362, 91931.68341475136, 3727.297231147142, 14058.7603509709, 30213.433102973286, 56723.28116422737, 80642.78174493709, 5586.453420797499, 47274.07043425623, 22925.06005299997, 4773.795982874462, 39378.04018605139, 28727.38793337578, 88521.3120981401, 63388.354082029306, 603.8623200470882, 74112.68842226134, 77797.97696219618, 12843.228949055063, 23774.992100577587, 42526.6383446803, 26238.64497028122, 11789.370616884498, 36522.10839340022, 33117.05837100895, 10227.749536056985, 41720.55150835418, 24952.828242189327, 56355.49019676698, 38300.067413673314, 54623.05649560735, 64905.11595979148, 7787.99005890497, 17031.903696418914, 59433.21557639768, 19163.316452521794, 45587.032608784015, 16609.228003154207, 57374.963087597396, 25175.709925347244, 45615.48671805149, 69311.91121738015, 74449.78976762871, 38878.17626353435, 54605.721772773395, 18832.673332102157, 78543.82536462841, 37306.70543931698, 65449.810734956984, 34017.11902877384, 29296.602027787045, 96855.3500227124, 88840.2923759981, 43956.7299082643, 15509.631723347073, 17212.544326524516, 16279.185425606112, 31365.129790227296, 24091.33757086066, 26875.492380029307, 91835.00467572719, 57543.192514899245, 88506.31158511362, 65641.10055745547, 26566.88336283606, 87539.06158443575, 79021.3148916296]} +{"id": 820, "vector": [56725.75865775729, 76538.3346976301, 53730.1850587382, 46460.00058365742, 91070.4973201876, 16363.97584134185, 43409.31800555555, 18062.430833042064, 49496.910803923034, 38480.46947434185, 11980.203970460201, 91989.18075612198, 45350.43747691159, 58347.44600084606, 62185.761624120154, 10627.987193555022, 21904.690438299156, 78216.18469316611, 67952.64424265813, 30410.239059664178, 88787.59392082687, 57014.32628807723, 66652.65367618948, 87088.43478119932, 82892.69536901567, 1351.0918253896941, 30741.55989726828, 7128.944169603657, 3991.5997242591093, 26605.360651953837, 52843.40572509056, 88233.80411871073, 61144.324779044946, 2847.848017571297, 71461.334379747, 81059.49802281747, 20248.800431488868, 38807.0673712255, 98634.42302768923, 75049.5554544247, 80754.9253780335, 92079.08616504526, 77682.2991519211, 64349.673370603254, 54340.55974836599, 85488.95516320253, 91545.30219200051, 26769.46857865682, 5794.1046397419595, 11309.595754276757, 95581.86633531487, 32682.844347692953, 86733.57235080955, 5561.048592635653, 39334.208047552136, 54104.46450859825, 84760.39239223574, 42582.29071309211, 20197.591485683795, 91787.520813456, 31236.82379409115, 28495.320242818667, 11661.239897379804, 33043.417198252966, 52920.456344954015, 59878.23630753276, 53258.20612390174, 57496.42127212101, 33042.29348891562, 70998.57992040888, 32861.7063760314, 72048.88475353488, 303.8218315218444, 74015.44021495704, 6845.2901936628, 87099.81057571275, 77099.8744591876, 2812.698357020127, 27519.42975836038, 61461.40357158821, 25779.393974331866, 11971.684727512144, 68635.26713950052, 75265.76257140703, 95601.02623583493, 31786.334261761218, 86495.41523776474, 42924.372124427755, 12680.915574838291, 25494.28570038518, 87623.38943440195, 46844.357793356176, 46003.15266601172, 48696.949848046155, 67813.63536284138, 51917.932205870646, 30598.913148417796, 25536.25520997086, 79772.65468371376, 27402.673642610363, 5663.943069697863, 1185.12041677451, 36261.250284436464, 92454.88468962695, 48864.582892871746, 11552.987706627217, 17226.539775408124, 22764.74506971937, 49567.85884560012, 85865.90534842787, 40664.82917921746, 42005.79832773578, 94243.93111764212, 80837.53477901687, 18196.53601123634, 66727.70708950669, 11419.756968102645, 59686.258748376116, 78619.04752252971, 78669.26803225995, 12882.06353090462, 18802.9692105924, 37949.722195927796, 57829.94124705496, 58800.16081515258, 18560.29575509098, 79337.81440729827, 71742.31438533774]} +{"id": 1398, "vector": [52301.71632212208, 74113.02934625135, 17292.449741896577, 96848.70808212878, 17036.894854297345, 66866.94961809802, 78221.28249991279, 58782.03895151346, 19517.48799117097, 38535.838099302164, 7010.24361051813, 93628.16445416758, 33157.07059041755, 38797.90457416539, 90173.46950669895, 3658.3198455500287, 30733.23866621228, 22606.710194734213, 67606.83055323828, 1115.053236650898, 18825.13740507774, 84014.97639340954, 38639.95229085279, 21681.878969949907, 59962.42512831023, 8144.3402057568055, 41936.70028902348, 67992.04456275892, 74687.91366342477, 60034.452082960976, 723.4345910375195, 9959.12483767103, 29935.271392171337, 75339.0063781194, 33400.52420987697, 31124.506046915034, 9882.033856635564, 39461.77524093547, 94361.63177088743, 99737.03098787567, 33314.189684921934, 13947.925041994424, 10331.02799450033, 84511.59108945083, 42975.456825251946, 59274.00974839816, 86482.36403895759, 48611.079058678464, 67291.93541383058, 14156.73136390363, 51203.771674402546, 66079.2644801153, 66421.40161533585, 35649.15961017703, 58789.18280379792, 40361.65604274858, 42033.40645337589, 50298.79616369854, 6543.309505238737, 33505.155383443205, 44185.87335318283, 56393.607262683574, 67279.14343963268, 7173.990161753285, 75106.44266161852, 9315.208772711681, 93079.011635411, 43874.62188353528, 83298.8270201535, 45406.23255706758, 26817.66627105181, 88728.51803028013, 17588.830681771393, 53334.71491085281, 21522.893629681337, 73102.88379515779, 628.0972473872604, 55202.53710607198, 42932.85409465423, 92405.65691971972, 64958.543515599595, 82165.52348511587, 65969.27581993288, 63505.13391561316, 96238.08288605232, 70034.7462398404, 35382.880240006074, 79233.85931697783, 3206.159136476283, 65028.80835788836, 59242.147037695555, 57745.46855663064, 67207.51218169632, 21837.32734826581, 85559.41929969974, 90129.1390975837, 81635.39239845079, 10707.515589313387, 35125.413851321915, 15363.610038844678, 58492.76075748964, 58613.37513366409, 52370.80625744775, 12534.578839403142, 41433.81606162124, 86445.15606654658, 4118.441218887791, 86173.49040369468, 82392.99546606642, 58209.530110667605, 4992.909186731808, 59115.83621519896, 90531.33350281448, 356.0448040118791, 38436.32874526813, 5765.793242350947, 17229.27155296683, 4862.777008699648, 44574.13704654263, 74024.74453834623, 52694.595699309386, 65547.88713686625, 90300.13446569999, 314.3903506125922, 27148.216041631855, 64496.44550440312, 81261.7374524335, 5403.913804114624]} +{"id": 1427, "vector": [18527.89684719838, 8866.900777975761, 49193.268295798574, 84555.8647252511, 32704.8203871508, 26277.727057089738, 45420.594711541686, 88671.51121945135, 2170.808909064781, 86133.01071462575, 76247.21489607028, 61518.03252420464, 81063.44472703566, 6726.211508865298, 51565.83929525994, 83350.15546947898, 35088.11558795469, 1949.6661545768568, 69122.5053246302, 74317.33563536871, 12933.398537794383, 85644.25727146, 78383.76608254494, 62585.933251951275, 23050.440557164286, 82915.64354649081, 70706.15151278762, 44869.1299071998, 55021.711075012034, 53264.31966219346, 49404.88816662334, 69949.70668222081, 40172.0753823232, 8334.883459148568, 98349.93293110939, 8386.487455606439, 90401.67723384772, 74752.78568629455, 71566.23755943388, 37174.2685340637, 10226.415137501677, 63333.97252139847, 91733.48324821223, 45348.153608107714, 82352.98263236196, 89903.09249380058, 14405.508376388421, 77983.50011308592, 57596.632425785734, 88287.84710301827, 76088.59967477949, 553.8572262737906, 49724.95981481837, 24023.08490087386, 24423.93761971481, 45590.05705047452, 82102.40397187861, 24691.73281787207, 71596.18792641352, 7231.634208848137, 23015.608030814717, 19629.263240573502, 21035.652294598327, 14648.311202843744, 73309.21172701633, 74939.6624969541, 79886.81303300401, 12730.450183772246, 31065.257281513383, 9561.430664581361, 45041.32726618288, 50033.89699729175, 62905.134641013305, 62551.5077691571, 77138.76861724084, 97154.17848881053, 30442.139505607356, 75479.35829212434, 30438.358722651883, 61550.14061079248, 86728.91462863547, 95267.268708371, 16970.04316138647, 39530.94204186697, 13548.566392987894, 44602.432200806754, 87322.25920515224, 243.78408392710105, 26688.68868680979, 52431.54964904177, 83555.26727577804, 81582.27360903748, 51357.86427217157, 65370.95735010468, 37788.02910412509, 2648.1886169636427, 27632.630283928895, 80845.57942984914, 21669.39825257824, 31259.567210160887, 496.3333009793458, 46410.764369110635, 59338.44449599497, 34763.08698548521, 67347.69513355898, 53268.956522096756, 37470.07156108248, 44895.47600981666, 8163.366992013832, 48443.4790675649, 83855.15679228037, 10647.848735389132, 39187.5928067552, 10733.896432857271, 26160.304304097994, 18987.78121657004, 60590.9215685591, 96648.79415082089, 35094.28578278231, 36862.62899557231, 1640.0775185664852, 45452.093353958866, 81106.16564075009, 98945.61426530914, 4652.460551002757, 84716.16126925712, 63308.78069475477, 7545.388650811436]} +{"id": 483, "vector": [86926.77848135513, 22331.87441265858, 46781.16783112176, 38554.19821929271, 3168.9395113112087, 55592.73300998476, 50902.89060251448, 83301.47747620674, 64859.08749817384, 84423.69145936829, 82939.76616887421, 13690.655819438924, 96913.00942724399, 37799.09670552847, 391.9536573271709, 85930.25071183576, 10210.570508381612, 54629.93249491591, 55554.472491136876, 56574.71471940435, 25498.86329846628, 8848.647978367451, 6049.864685320927, 42547.798616636486, 1787.0702509631299, 64145.14190906174, 2411.5865875510067, 4673.683508297033, 40829.85044236665, 95015.87678677971, 87798.85824208707, 21508.24274482821, 78489.27563175069, 39695.73464026603, 59816.10321795138, 85294.69305754753, 5521.559825083977, 38400.62037426506, 30251.20684557402, 72086.65673504405, 39878.99650141078, 81401.7216735966, 38869.72954623037, 71502.0783312723, 77171.80255114847, 21997.84419689441, 69097.28658445185, 65643.63678647495, 65654.3187148678, 52002.13038619204, 98599.38118715408, 33045.85429538337, 39954.91236185833, 16433.953967813042, 73314.35693495646, 32501.426075958094, 49687.55966322897, 20756.143809386707, 53162.044985685134, 30396.812745093215, 10313.114177085125, 28348.8226575366, 70476.73293651841, 81264.61410340854, 54171.52515725796, 78986.86449077478, 13616.62435238583, 17795.375578607698, 85222.75997603957, 2379.8311327156707, 95936.6152535907, 92230.77902355471, 29231.168091517702, 45527.188775408686, 4445.16889172234, 78250.05409855465, 19567.930928741927, 90354.94010585637, 90812.43803161662, 71468.23965448362, 58381.96533171468, 15103.509031856722, 14505.154229380025, 36678.51972055046, 19692.92209660065, 18979.53816130188, 81349.19830891002, 40942.4635193767, 67595.62646977956, 6340.28279865625, 57336.50192774515, 20429.689146435616, 44359.661035679885, 32453.551071177644, 71597.12113265316, 78054.13128756738, 96474.5325978897, 4898.032586803592, 2618.320601760449, 12675.337688762467, 17428.802731563388, 38993.806315447575, 98786.55756701733, 46497.76965579021, 46050.9313923432, 95747.46068149213, 6765.280522350381, 59461.47060014217, 52891.52919775032, 19401.260635645045, 54251.25349013697, 41520.43155452204, 99260.40743222604, 23974.31475808308, 15917.920889196768, 82906.74882940741, 8136.987968013565, 84326.7367644462, 14573.669425201308, 1463.0384191771095, 1129.9144710682385, 68425.31937216708, 90859.09890434502, 72274.73980452394, 49612.41813403035, 24114.097253909407, 1737.5909715175885, 65664.16063454848]} +{"id": 1197, "vector": [3008.5811555938703, 13108.1063453802, 44580.187908263746, 5179.72694536083, 21626.964231671896, 63763.23335880483, 95477.47224702538, 45420.11573843881, 64781.13716036162, 19060.2840418996, 4553.968425009502, 38283.342167458424, 44205.544437781806, 25981.43932628085, 44260.030163817784, 24627.586767978948, 10514.983300211288, 89983.87421769797, 53774.06881209988, 37957.74164870367, 37630.08338469087, 64823.117796158534, 73153.84904709298, 39484.2695495508, 23218.549198835415, 62567.87062946042, 79438.16679624522, 28381.01778554668, 14113.857398120223, 83702.93575374325, 84775.94625493635, 94579.53241504417, 93614.10687107305, 37896.75159508943, 27700.5498150541, 24052.784600661715, 83788.74725079408, 89135.30726562263, 33136.55124558261, 35636.930734098794, 52801.672070251705, 45760.44772370851, 7637.430494432373, 15533.352172693949, 37247.4988819111, 28801.79799039223, 91289.63366653369, 91597.01913233551, 69322.29393193456, 24432.42100413454, 85485.1427291047, 35012.43904456051, 6172.691107822792, 41022.396207834136, 469.38771859978965, 43880.26555772194, 27356.837107548625, 62206.04553029594, 38820.72475422268, 95503.52725148798, 41248.61035975209, 39326.17379182081, 94406.44992938789, 25256.05475180257, 5805.8178712643185, 58770.79947936148, 51520.36928587722, 50524.87263597079, 86597.63020898677, 68359.41063369015, 87819.46193799286, 67245.33817488689, 85932.7963630443, 57056.96986897856, 76292.46105910258, 55013.895629258834, 81129.3280852389, 87347.94543222239, 47551.263128121325, 39741.297910630055, 24363.884888463726, 85664.93404847886, 60316.98401536303, 31819.463754963028, 79940.52809849753, 5743.307471953074, 83368.28496385842, 61288.83705272866, 26531.463301193613, 96714.06288891521, 37393.35530815473, 3479.067528130486, 33264.47516767258, 96887.45406580987, 28580.442779596917, 12285.343512696167, 99032.3971494507, 29960.23816269364, 95473.1484922871, 35250.03260698351, 82777.62362394156, 28158.192379589196, 56268.3262711017, 39654.26775586372, 35762.59360761503, 55983.536578036445, 97322.43573889865, 17922.621990165677, 27657.951662427015, 42169.92142052532, 73757.3879190339, 25162.03229184548, 35785.00468384622, 32348.101733611278, 8885.759531016269, 31707.974450181508, 35315.223510404445, 67260.44597597858, 44291.57844875199, 7174.88545720657, 96667.21518679618, 28543.58941447497, 92765.55180501271, 5406.68452704417, 88015.51997987584, 26282.44567175034, 3081.331319617331, 56400.63403183701]} +{"id": 1766, "vector": [53569.352235622515, 77479.29339415369, 43995.022787370704, 35535.6743710554, 13701.280070697963, 32738.794293156214, 11700.981624210515, 56768.92516083181, 11705.993401849835, 15487.753311887842, 89967.28236488854, 79561.34469858317, 62059.401237415426, 29909.71544900224, 75325.24635891247, 77604.87896943926, 44287.659805013966, 18473.183669578062, 64928.69218212951, 61868.9139532469, 34208.35389363595, 28096.278457130673, 24966.297326639175, 69208.6610462102, 20358.319882568197, 71520.96769496362, 64653.554242843136, 84132.66668441189, 76227.19469359909, 98341.43792408278, 68530.27502376765, 43874.12021428232, 26630.793584575706, 75123.77763636816, 8659.290088220172, 1163.5592611164202, 73959.12483874394, 8838.76688037315, 43522.02978880908, 1294.397738812081, 81842.61383020021, 81894.58709207053, 3012.951048923418, 54628.485128360546, 86080.08075165766, 29852.496269278006, 54481.01509270553, 52185.88453878562, 19938.426965830815, 25397.331469074634, 25076.573410451652, 72755.46219312244, 84818.20773377412, 41497.16014860051, 53045.93887608836, 77843.60351985559, 19447.78485926999, 56739.19730825644, 34648.086835672475, 62792.57176577355, 67149.45615959902, 95519.41563672076, 59233.419425755994, 63339.31607466687, 36547.39205043912, 8927.029576756562, 87426.082883357, 64462.778834297715, 18080.01085255333, 41851.19901923492, 18915.599143051677, 71520.04307975525, 97815.27801243323, 83018.33313567702, 18603.843666018383, 70492.82831697026, 29381.560210937685, 27456.170284757354, 23767.393114345716, 51677.29907139702, 38828.439858166195, 84072.78604977633, 56220.7697843683, 86653.44429649066, 52110.40968295631, 18352.788188112525, 92812.00587156165, 36864.96673790358, 43107.216444258855, 18262.232494418517, 5320.493138024452, 72674.30421541008, 56929.23649837084, 94432.77627036876, 21438.201376506204, 76328.91877572317, 25648.6013383181, 85287.94940594901, 71939.98527364948, 23459.940790514498, 84614.71691413138, 3702.0908730871715, 28384.704268907924, 2344.431709439654, 94877.85547456492, 58051.238263635074, 62895.40124195561, 80883.62017368399, 95494.22076103304, 27820.728146562746, 67544.42885637333, 27169.90194194421, 6980.594203379198, 95352.59470139192, 87140.52555627511, 23581.390464102915, 48858.03131044928, 73347.97531773859, 43897.46602913673, 9190.960911665614, 52498.704645955775, 61622.406299530565, 62650.0984130597, 47064.54032611599, 55670.980101802605, 16363.009717333021, 56909.588635695916, 22212.13280848229]} +{"id": 1355, "vector": [12177.315125119103, 21355.902959719086, 13610.729128796584, 51385.74926639371, 20861.145214108103, 6424.0925384555285, 47999.18164017188, 54740.39578782746, 71594.84041943657, 18047.67933506466, 59159.061028534874, 88174.8239397826, 40250.55684971799, 96212.3204830173, 23963.59717700385, 90051.24069569568, 98139.38536740352, 63626.28161808719, 15098.31168097705, 61963.016447419606, 96394.80890592706, 42547.77548496654, 16536.602431121028, 94513.05079031891, 92756.4968954502, 65457.68393714645, 23801.233867187155, 93847.24665069701, 28270.071784862495, 73665.9199271057, 25940.922237025265, 41430.51528428984, 64205.31454375118, 98428.5212084871, 55045.079539232676, 88864.8670119222, 3331.612727740252, 8064.83892697375, 93488.30861214033, 59032.63057213949, 25573.863723274913, 16267.359611426024, 74577.45619520925, 89428.77048960776, 66222.77155422534, 68021.4730171457, 77459.487029463, 52947.64148147365, 64799.73995925229, 26839.731513091225, 61646.444198892415, 45033.456331894115, 91341.14642366121, 57270.457968303, 43001.918285954554, 34089.55840697225, 13438.259993217927, 89888.60816764507, 22349.316134826193, 6303.111230534464, 30134.283035882814, 81422.76047871374, 30639.89194029767, 61108.1778413834, 79006.32713687225, 28443.91389034011, 33637.106977460826, 82388.91786953935, 75974.12364617345, 55396.176560188884, 57780.30177003456, 8799.52549153552, 79257.54769685621, 60435.852073445596, 6403.050591071246, 6254.721840852529, 1947.6000633809276, 6107.36187705484, 85391.55437509366, 13383.617167969831, 22096.852850527604, 81792.87109718066, 31392.366248503746, 39521.39873533513, 53355.22017091797, 7220.787245775484, 2322.498954630914, 63743.30908317761, 63268.0937879298, 28514.762776024392, 49236.85163878385, 61787.43553805418, 95466.95484133308, 16292.777387337575, 38382.77148500776, 830.6346569768031, 35281.98772084005, 4898.473756242239, 74542.01938346525, 43690.63485405805, 51360.62193081598, 82800.91149346309, 6437.860207251111, 46324.50483158251, 28372.897964935208, 6732.676160845119, 81718.24589593339, 94017.39898110187, 99238.81061617837, 21482.42553854177, 53569.68004029839, 82850.94428969009, 7149.062886412627, 75705.3280064201, 18809.350279063798, 87093.610017936, 31727.7262871421, 35102.719255924996, 65770.3554261391, 71239.87624330157, 10578.01688297576, 98505.78747125194, 11452.787754867333, 79054.65200054689, 64488.4271573861, 56800.55620040379, 90102.3494947781, 40534.039656138724]} +{"id": 1386, "vector": [97073.20462633818, 11577.935516253357, 60002.642319108636, 83375.85778957917, 7555.181699731084, 27980.428222597508, 97262.51360491081, 89676.37110741923, 90141.48445981882, 48087.56405342866, 60737.59712146588, 17511.882814260647, 99899.17202016612, 58810.16962281118, 73842.25630594441, 12526.070328676831, 86299.54831947733, 90280.60104569374, 19169.58987035393, 955.0762583895978, 59070.07918124364, 52758.24689669551, 79368.50708670978, 25279.20453699628, 69223.43513525573, 23935.884341508507, 69026.71750881991, 38093.893930002756, 26883.712365497635, 75814.01523910025, 90079.73026459507, 55033.36595504446, 71762.64734216439, 86848.30703712221, 59813.565504664555, 89604.77490295612, 38005.16987970495, 56788.41717479914, 91797.09572160685, 60306.759113904176, 62754.588692304314, 76350.28441470233, 47624.65774625558, 94009.17176801075, 75580.49891014704, 18545.161329828363, 75736.03870515415, 60325.50526032402, 98119.85573640528, 81178.64633000264, 18476.95652672434, 27551.995131797925, 89927.18257811396, 99830.28190768132, 42960.49841443473, 47419.16387531655, 31171.554908176837, 46015.22185343474, 10026.02308554138, 58113.84721108182, 5864.7902522118375, 89037.60054375714, 91817.38332430477, 61984.26253830807, 90804.16743619292, 5042.85630924981, 10617.365203275853, 6485.471497468875, 23119.755577950884, 3273.540233568939, 11733.326293349466, 41578.47536638627, 1046.5371362370356, 72130.82510722909, 90008.89293803636, 76029.67643485514, 94798.0656811244, 17005.086010592728, 97804.34697019981, 31728.188287738, 92590.87951567856, 80955.46081324895, 13204.608235831405, 4109.702648410096, 1271.440294011783, 81659.58302314716, 96066.55582218265, 62986.837698434705, 58588.92780945112, 8215.594903040635, 38310.327905240105, 86212.07331742195, 53345.73971687749, 16727.757075718288, 58994.07925909274, 7294.091405387071, 6576.682892549857, 76889.18150843852, 98684.51538687256, 14186.293515380832, 65609.50205371516, 19040.255679746864, 33353.22465136651, 13289.111583107317, 33110.85803945348, 25810.65832627313, 41626.31764901443, 79135.07962582658, 40147.411133925816, 15088.330954212692, 94714.71281054583, 57103.21819540513, 8382.171425722607, 42588.62545536314, 99384.77845521412, 3841.7531876846265, 74392.7210000533, 31041.584945347335, 89587.10912687659, 96723.58645974944, 86140.23976047517, 38630.65462363187, 63592.68085360194, 71852.61062432798, 11561.567135624795, 25612.373955079736, 28584.37067744838, 49161.814486555464]} +{"id": 480, "vector": [33167.70631916268, 90304.65718766065, 13035.706087205079, 40124.90746474855, 62847.171664869275, 89586.97903064496, 18402.534120483328, 46935.509280615595, 26928.884651178185, 99110.65919986929, 31026.922622141352, 95961.57875854513, 60983.137426236186, 68851.67950734482, 89266.16480512638, 40459.54153197776, 43092.8387256594, 82383.37944320285, 44491.32769137683, 81412.69255616915, 32098.671308253335, 93595.93743988372, 26017.790587163337, 50682.59377251313, 13864.884336922423, 12220.093524692189, 54849.20323631175, 3646.6785789652345, 41640.45928749194, 9168.544197505356, 15695.309423057091, 42414.484040945725, 48459.288785978, 26062.71880567911, 676.8593896266827, 29622.6882167685, 66559.83522054939, 89756.9445799128, 53834.00465175483, 53897.76957349091, 20402.777481681733, 52853.36621783211, 95232.87092612358, 92297.37825309724, 38141.98126163105, 20212.047182417813, 19749.331553871973, 92489.15118212698, 33054.18829180398, 81926.13849396867, 73001.69496286739, 85296.52210957001, 19489.050097296644, 24830.781511356403, 60998.30658186521, 36507.31600965223, 55321.07098004965, 37632.84152315257, 52884.58979956159, 9456.023513289658, 66427.32107977137, 84340.04388183609, 84751.66316108695, 45674.575328188184, 42302.55799171715, 20742.880778625793, 17856.80651827535, 38615.886532842516, 90672.19919906059, 68953.84936803402, 8837.137263861883, 15005.276892097063, 27811.085379961776, 92289.98224037941, 14915.877571172043, 44208.71171708408, 64625.34842969164, 60333.94023020968, 16357.343339161678, 15199.909202486117, 96896.64339199972, 23641.872458319525, 80611.51101475871, 65379.39918082659, 19187.82532387071, 7705.425943951949, 63281.00697630169, 57928.65264953659, 53281.588167450434, 68759.141846628, 78387.32247381216, 63242.57555074602, 47797.1815505344, 67860.20752406504, 12557.278491528312, 46594.79671046263, 47267.92411167219, 42540.22669125962, 55502.60573955085, 34974.97072977919, 25566.058023230664, 39301.431703815026, 31438.359678999816, 80003.00770288566, 92891.5874569952, 22853.88354154433, 91977.12457073147, 65613.14518396469, 71426.15673993595, 1668.8123394758447, 54151.60506322186, 76966.91862548923, 5825.097061664442, 66050.75259449217, 30375.66212659214, 55608.875301568296, 50393.57028381086, 46692.214689583125, 76771.42050146348, 20619.51553733561, 77961.686387402, 25236.648490211777, 85838.47854914407, 99882.88756091094, 55862.32161672118, 66062.27646235205, 57058.214439952615, 72071.39825900346]} +{"id": 43, "vector": [18698.772181921253, 5237.963730484474, 98171.38321296092, 76914.30653675263, 86350.99062721845, 34776.84809526639, 337.28643339402396, 46997.01803252416, 96070.24079826531, 61203.1816825451, 89628.2689695853, 7371.256836454121, 7254.310637195538, 71974.28809699748, 91179.97891237277, 81193.19086218034, 20348.117993081327, 79625.9163098983, 93897.90053879967, 59721.62832864649, 64890.84833366906, 51299.4853259346, 82725.92235947029, 16066.90890058402, 16215.31978969739, 6752.404088832387, 87813.23256370073, 94171.58572583516, 88973.66085573776, 92125.54187311772, 19470.74043597361, 36793.17346183293, 49513.65314827742, 58250.1227072822, 10274.658409370119, 44923.406670475764, 82730.22327664433, 57807.89419904797, 55505.74229121654, 39493.14827209396, 52927.30970199573, 64418.14124670805, 98548.47263968733, 60355.135794046386, 50460.505233603544, 43796.97489077297, 97704.97274059744, 66482.45698161119, 75182.53056542049, 7762.145220351924, 50659.49561227564, 36436.499082636634, 35263.22194821876, 83037.8981186251, 86165.29024618848, 85623.5520200693, 64084.62853670477, 40908.99062075397, 65176.65200088435, 69473.95384986911, 13744.243515764609, 31942.552047605466, 97338.51334649647, 17392.913077929374, 33984.56928225671, 48826.55597032482, 83747.4163041184, 76152.12845770954, 17871.239694214903, 60354.17468516689, 27633.252907414386, 87623.06894605498, 65235.82706712665, 20509.37290433046, 80555.41332908308, 63299.86613537474, 96166.9532645669, 54681.13919917284, 71507.62824433541, 1447.9469179962678, 74341.62716489255, 3561.5314358815153, 6710.601246864756, 64978.70373875818, 26598.386202058744, 17581.15414135135, 1704.8982908334676, 34239.03575044077, 78560.79085539687, 80836.67596448213, 81243.08061030971, 26263.543903325248, 21895.066276817975, 30133.181830529942, 34630.143704765345, 30210.776697407815, 36248.753796548204, 95260.51473500782, 61262.90276396902, 90849.29009049565, 53867.99883371981, 71904.11967608504, 74911.80330222582, 81623.34338725128, 82792.56998835005, 44758.58518760353, 98209.04414406455, 23171.80919576094, 87150.89475734628, 40834.53171909612, 70236.74398805582, 11276.7698029413, 13538.343725201174, 10078.42973345423, 14831.512735673925, 1004.8787478010856, 47165.41698193993, 30887.224362340727, 54293.997853088185, 65673.01349685929, 80046.96440980444, 94977.70137539342, 73554.23474031634, 26979.330664115285, 38563.753879412325, 91322.62784122712, 4037.7359484248145, 24422.793960160205]} +{"id": 1338, "vector": [50494.63890354861, 2845.5746115010115, 5692.729729362766, 23794.501267291424, 9728.958797619358, 84998.8401066901, 61919.972347707655, 62102.50586798737, 70093.90764526861, 11781.378272890108, 90149.6483976027, 34620.217758547675, 84376.49657788301, 45743.110841462905, 34899.77903199594, 94589.99114038188, 78325.06638775843, 97407.47914737715, 88306.1042290798, 54093.365446366806, 1256.9637336688365, 65864.96531362331, 98122.21542765237, 4083.3321312352286, 97096.4949540575, 96374.87420707071, 48284.47910337481, 82869.25173238471, 90886.20594697741, 16778.007962132146, 65338.16762826688, 8910.49531087148, 54732.96471733427, 61214.944681162444, 41461.98848629381, 63799.85369229047, 54419.024418948844, 79419.03702492775, 54109.2353043285, 93832.59609284949, 79191.19379068632, 47267.43252497666, 11675.517902037025, 3612.5199819368127, 4910.102581108111, 52415.006845718824, 53732.15923548758, 18368.288925695164, 42909.74759192975, 38635.345101275445, 9621.709479273966, 42854.15674834099, 62442.98282017444, 84219.60308526385, 19159.310369855877, 80641.23595713607, 86434.1920478949, 80074.40472803678, 14044.494430535315, 90679.343373068, 34145.97321884304, 22983.68359109193, 63446.74471761924, 73832.3438912174, 12612.92790263222, 6196.209704223122, 94548.40302802896, 13425.19688310362, 9811.196518386878, 10261.641324938508, 64972.59851882501, 31464.13440739123, 94502.11855290763, 40637.69056185027, 46600.049048517736, 69850.13247638199, 86171.23576465188, 52519.742654267655, 64545.44652217163, 61343.554833351045, 72887.51750382384, 89911.53197216583, 11485.317585519517, 46530.72068834327, 22102.306444080798, 62452.47973428881, 3253.405379283336, 87258.83810235308, 92310.52105459811, 76412.95764858315, 67603.5600253058, 40488.12559080371, 59446.807051129515, 85806.11411608706, 10540.958114142817, 68170.94777755649, 56519.07358157273, 11869.21914593968, 10485.46954781865, 96984.03796107491, 92905.68423977797, 66068.9235276524, 85101.7550652178, 15727.535788843328, 61672.3747169168, 404.1045944809585, 8318.69806345108, 40277.49498490706, 4420.971756423497, 47812.53535013278, 52936.23912632879, 822.8578355754412, 20871.509817342514, 67660.25388561905, 44884.37164262484, 63049.4257872635, 15241.722087742615, 16869.221285104297, 64343.31219837897, 33365.62834396807, 40450.22769386666, 6033.596499241445, 80754.6678458222, 80797.25945083195, 74956.5545186849, 60618.57926740912, 73045.95548707971, 93038.89455431774]} +{"id": 2009, "vector": [60507.91592265229, 74879.85107819893, 72937.36940533969, 9034.127469336694, 15239.927192705238, 6054.495803265536, 35126.11869566542, 49574.00142068321, 10300.54962630762, 42734.554696542014, 23177.41244304129, 33087.65194248033, 81924.35125762998, 43929.940348409094, 7177.7805682190165, 75855.58472087739, 99694.32731768127, 46498.28470470337, 11133.323334806588, 57852.981198496964, 13762.662529491932, 2787.898544299028, 32908.24592175833, 89579.33543736822, 25742.295717592457, 50234.41018090802, 6920.075885267341, 98311.1294743961, 22672.243868277554, 65457.971314890514, 17790.554555246752, 61136.386821170905, 81246.26282104164, 83092.07870750847, 66436.37250053929, 77433.5685561383, 21550.42876538893, 42150.415586873, 48103.3712030489, 53505.38153788765, 87793.57209248249, 56658.446291334854, 90025.4793941304, 9134.148058579673, 92182.46371446425, 72603.68363213724, 393.89595943616354, 58213.11232070364, 54631.8635330307, 97105.65190331852, 81221.49869097068, 38244.62208301284, 89412.13234145325, 69152.86591737957, 20553.862696969794, 63176.25413460568, 61832.95589539286, 43786.26900171878, 17412.062442146736, 32781.05286773523, 87191.34191704418, 585.9903205190586, 69441.96273740391, 91357.10795599339, 71321.48780772484, 40778.309921828724, 4055.041232386913, 81276.61207498248, 37680.84257409023, 48129.48883052009, 70749.41932157463, 53345.41411340941, 85641.12574010305, 37284.782076013566, 42641.49252123457, 28045.025156918968, 77252.8480515243, 96683.84620178535, 2351.9667072504635, 50461.83742379119, 5149.541243377443, 19717.590585566315, 26707.20293227329, 91498.41170055191, 52818.044877011074, 69864.67075400142, 4390.0567256852655, 61895.24885534961, 55657.88482360611, 49831.516402266716, 39624.97283092236, 30317.16145583673, 76888.1754176905, 65974.06283075489, 27276.820850354234, 35177.99272835111, 48053.89178662095, 86054.86043948791, 5626.344868614042, 11602.409477803, 94017.67316469226, 89938.21529472676, 29033.773821604693, 88663.53104546687, 30832.369100855205, 35513.06870526736, 91838.36888344276, 89669.34585791238, 59967.571536529686, 19315.44136105845, 94211.348899365, 96680.62249154095, 3226.9692727624697, 89767.99186089633, 74443.78661388166, 48116.6657041857, 53501.12324325163, 68763.55456657524, 68018.16405782994, 1143.3261160169739, 55280.90096082853, 20222.692972602774, 87933.9064556954, 49708.033055239684, 24576.865495913404, 80082.71801612231, 61493.60081068629, 96069.53086996016]} +{"id": 1085, "vector": [72037.27124217538, 4893.925612179273, 31841.260848838072, 47737.829695764725, 1709.8401476951296, 38600.742516009675, 85062.62049323553, 66416.65048407491, 88999.11729006625, 60425.62903271973, 80662.44828811938, 37960.354977115094, 59272.95303516163, 7637.145810866752, 88053.59791580589, 94530.18626333539, 92619.38732014438, 63906.70391964275, 90256.4123389455, 80092.61140515217, 17119.79409062465, 87961.72669592404, 70376.9622238387, 64727.43018562239, 43137.73095518287, 6006.074367963543, 95827.7437853501, 98394.94507198717, 28022.87949585648, 13453.209950110844, 64434.29409093606, 18839.824202548338, 60050.060743934286, 55776.12318622628, 9425.269050255347, 30830.61773478204, 56826.64658747934, 16456.09615198891, 63380.597050676944, 81310.81820455875, 41888.67068230736, 97324.11741630848, 39131.66657274004, 92527.3937942546, 48723.84252695321, 41122.83643447683, 89626.9320154953, 73740.59079086519, 78914.17929530126, 79462.95406932561, 59310.21178788174, 3100.601331403219, 89458.46340016232, 76053.6677082992, 5075.453236630734, 92412.83254481715, 19303.228534268026, 60654.91684602963, 94919.99872949939, 26212.32785360568, 28947.142785906643, 98101.73505663054, 86706.7726551653, 56312.58124167325, 19237.72605155194, 15220.611290384268, 46873.044580214315, 8053.440682735602, 59980.99938893705, 3685.881752989717, 31150.888901744787, 12518.606011233102, 52428.64020234973, 70453.78792583197, 45791.32616517171, 14196.096659246148, 95692.6195630752, 30264.60758824113, 95831.8877833261, 43249.671842119475, 26939.646317231734, 17925.767992175202, 35393.65586431742, 66278.96252459002, 49580.67996175174, 67902.15209918625, 41167.37326332632, 95901.95323062892, 29939.884685488527, 29622.208710262265, 38739.4894213516, 49076.79578709916, 83687.70523380858, 71514.75467811161, 9652.958892514374, 45533.18859108582, 85983.7251643924, 34165.85463272166, 57466.57409608045, 17868.067372352503, 63638.70662853841, 12728.862018700947, 60407.70877131196, 86907.82866573233, 20240.65170740006, 79393.74663770777, 58909.191736860244, 12944.348333708644, 39296.34930417222, 68817.05763008367, 44785.26696062535, 60548.423146104935, 37190.62755054432, 11645.12824892765, 19060.17522178677, 2572.21776153812, 78516.86638196225, 99845.03493507078, 72444.04048795048, 80139.33180163168, 91620.95737530374, 39450.14287340325, 16354.475031137505, 21750.26355972145, 11616.13826146185, 47393.73314055416, 18543.56744507455, 75103.76424712194]} +{"id": 142, "vector": [83576.4247603512, 13401.519920039273, 23015.217411318954, 52745.16239758704, 71194.05094406028, 33994.0113935006, 32907.34096903151, 25081.75108446198, 51199.56708583693, 52036.02721061013, 60534.86925801633, 75809.06021637973, 57769.410180624116, 58220.68155664659, 97570.40665104333, 99137.47002663917, 62705.4071275682, 76440.29405670128, 24746.807950269424, 88162.90756490253, 15742.966118459733, 71823.05440258431, 17809.492236052283, 39624.967922322794, 25599.03158542304, 32340.778250268253, 82967.27615581253, 53988.20098846613, 4390.053335251975, 30373.870408232284, 88847.18644551671, 25507.115332745954, 91285.97317203795, 43936.97173691362, 6136.754241114039, 26038.599908376757, 29085.3702759593, 4399.351485023617, 34029.021921504354, 72414.34469617026, 51110.94317046359, 99027.74647512483, 88741.87947716474, 65368.547772473095, 35719.67343765948, 16088.37617108798, 7690.9335486674445, 38223.008228932646, 91517.89898130554, 44415.28382261859, 90507.36078877345, 19644.12306197445, 87698.43547023828, 84863.31768327628, 85800.21405546088, 79263.73102310968, 82085.85649176459, 46721.30833576653, 61493.70679877123, 17938.604128074032, 31178.405961163837, 7533.1768352732915, 75827.07548601084, 45762.730518317505, 83903.32102654921, 86918.86887946576, 67186.9236408468, 57445.6213200747, 31733.94257347173, 2649.120260324067, 45459.48912502217, 49449.353312538246, 61174.09932926523, 22357.390832091263, 10728.971207840954, 5417.6557741623, 51682.31335179224, 46080.36811028879, 18138.29293306497, 56453.57942479645, 39649.23772663306, 55160.66917778342, 43350.95928743979, 79542.18226491041, 18893.20718468427, 26431.00817974715, 37164.353134469544, 72665.63100297464, 36094.562409887556, 23074.122332733627, 86151.47106869394, 24492.95866250518, 98184.9585582468, 46127.71126618307, 35217.22822348602, 40134.53750759584, 53993.36302539829, 96762.18507361757, 10565.859060228933, 92783.339633596, 25506.644829667246, 39768.39477158172, 45192.540513435844, 98777.4627703667, 45995.95065096266, 86481.00064126955, 35971.03483714456, 32464.594748339183, 31064.289518292433, 28506.3947982526, 55585.36838141831, 66740.46960653522, 46900.78872748781, 72229.31308777837, 29788.663503487624, 48376.88939034853, 65172.53065962708, 96737.11610486038, 73315.43785638938, 43573.50364681735, 24456.218911668813, 60986.37048545908, 2082.184435311485, 74425.96588635149, 79082.94674035489, 67516.80605525817, 5767.616942089205, 73175.70352659826]} +{"id": 893, "vector": [93421.93597321556, 35002.71787601721, 2957.976962586184, 38898.380049442036, 53477.153795599654, 43318.78398913239, 30900.32333997491, 83072.43782846676, 90971.5554758863, 62066.71213585668, 93936.69541891747, 63995.582891684164, 35735.551674496324, 92840.84999018256, 41598.52932162732, 56376.003942287265, 88512.05901328719, 66658.12641786349, 99251.51020131659, 42988.27487591242, 40272.56499955095, 29465.819149645555, 10389.478891368432, 84613.32699465968, 31997.582911026977, 86870.48445239071, 13695.914945557997, 64859.01415981653, 39064.21207193914, 44972.67370219433, 38872.88612759177, 65354.218812596366, 49619.48775666268, 10197.883683078424, 98169.35069139792, 81729.20093188333, 75820.81509279886, 18234.48876147611, 84075.19909633452, 38891.59397737089, 2546.458947967145, 86588.2223180868, 38989.57781182641, 21292.200854904942, 26643.081857804194, 41097.984054502966, 66244.30794705963, 30674.61232023021, 64292.418176087616, 3774.6406184327006, 26186.419666090864, 97052.51578536553, 76007.18623330693, 73755.07165787056, 84037.32837212272, 51371.96169083129, 21976.071419554366, 45203.196612715634, 49241.10373224797, 95128.89694454905, 22110.770714441896, 22434.73436593947, 98847.4777219111, 68550.71971428978, 55764.35163246826, 78882.12459130527, 63140.95680935417, 50831.80868926967, 63398.183742575435, 4795.411006663564, 15858.277710356317, 25024.209101082994, 7418.762042296978, 74425.18012399247, 33017.78058188508, 70332.48408192965, 22542.96777877146, 5606.467571146645, 40954.595641243905, 17532.586463531108, 31444.04591365527, 98762.59136592304, 52061.663216468725, 23476.542448029548, 24545.766254457958, 18425.02464106278, 43620.78036037969, 26889.118971822336, 89159.09409834997, 69568.93591118616, 54432.50854988574, 98105.15083979143, 45120.04015386558, 68668.18209010058, 89710.55538830296, 88668.75920317203, 95803.82066431384, 18656.71741617504, 90053.862934552, 64799.36956301309, 29632.9888562495, 1702.5789462480677, 85413.04535654391, 65445.86846527399, 78723.20353992959, 64160.76560406065, 43122.61018714529, 89564.05128533336, 45917.35638322768, 88388.76284034905, 11229.143546615995, 8762.182462151202, 31855.887596338085, 26171.045445598527, 11033.48749150207, 56448.71167021602, 74214.33136886421, 77596.07281150627, 89874.14258734032, 20033.157465855, 14061.388462807778, 35216.48287202819, 27127.76309890379, 59149.665235834305, 8038.1038113957475, 78658.75327172248, 7887.356364111675, 42031.229581829066]} +{"id": 58, "vector": [38693.837476686655, 53701.367663230325, 26333.591121043475, 95527.77974879244, 14902.560547508592, 62631.431289747976, 29859.386441148094, 33591.784595819365, 33271.25126449388, 66161.8941162348, 70014.1615079488, 72002.04361355724, 22260.018459944396, 59691.57663470063, 90035.26439880088, 2788.97406901486, 78671.51850578554, 97984.87982921881, 31608.45997767089, 74573.83539597862, 6831.828866592182, 44769.836580072384, 48836.08545239141, 10282.91577161442, 3699.039575227403, 21668.397340998414, 63271.25193904435, 68096.04259576355, 28422.744630677844, 77082.71294043149, 84101.42551093249, 43364.0253587985, 41408.40726529846, 67259.40908859555, 97035.28885475737, 90187.20840842837, 95260.23287331048, 3979.44884853999, 96786.65767371733, 39696.30819814261, 57324.843043951354, 99377.40545089611, 50581.24206555292, 45948.46496835419, 93077.70302016902, 40166.26194291356, 23144.854714355857, 12771.92954276598, 54266.96956491558, 27490.280787721567, 435.72982431761, 41392.38874088003, 90817.37669280116, 12158.788290815724, 41314.26191044899, 78417.68345122704, 46239.6355551402, 95320.25702095854, 32089.082869933503, 17358.297193578554, 25394.602030786053, 22692.414595119437, 23326.055075565677, 77194.27810037218, 31802.845043527006, 96415.82489876522, 36603.24397869042, 35852.860700410536, 95497.8399920529, 13223.85210713971, 35116.51847869054, 5291.664407281038, 32526.227430360144, 94454.84411835097, 71282.60281259469, 9814.667489890982, 42938.835187517674, 40594.08318243576, 45531.49448491347, 7709.579114260235, 2792.928110437298, 78370.81727389505, 13162.061292194916, 15685.013561879457, 14962.75464968564, 95976.87187558638, 95456.21941018967, 37015.14825681925, 49781.68763376314, 70558.74469727234, 33744.573680566995, 65159.343925804395, 13825.93740682928, 26502.323812334504, 86482.27981431877, 75526.14046191511, 70192.17218592856, 25549.732192283027, 13272.532750361699, 61216.12158902604, 52763.07440601281, 59102.922809655145, 42790.97585695481, 1516.319327241078, 88327.1612023932, 85979.37164067815, 84343.00625786417, 84449.66226327202, 40520.551306914764, 93964.81921187932, 54279.4418390441, 45462.000665172556, 5304.799113418301, 75617.18961523598, 20060.406906395066, 76928.6584853512, 66167.7107176726, 71971.10681183144, 69096.83648805211, 59477.73232348757, 16741.293091280597, 17050.497113106954, 84894.49206064852, 15852.873688563574, 16984.16602210021, 62844.4159845553, 40398.0623802038, 11670.281452227782]} +{"id": 1357, "vector": [30621.892662127582, 72355.44432135588, 74021.90938995514, 85332.2666151682, 60282.079739155015, 95907.10423391622, 83020.37365273469, 73900.97665545215, 81582.59384943383, 39773.52176712522, 80041.19235659433, 81490.9512219116, 26974.758076857386, 5806.28841435259, 32276.331109531988, 7986.4023475295535, 24155.622973463243, 87633.38973896188, 63269.34017204695, 19632.094867153904, 71442.75514339146, 31338.55953372292, 16543.079207290568, 3849.9952918873205, 29098.969175150323, 96281.39065484311, 93147.91552012757, 6004.358153577793, 87801.92185791323, 79952.33231762446, 79665.8688962601, 74400.50228926593, 13738.278115745272, 44329.85175092098, 57449.777095993406, 17364.88682606592, 27465.908164405282, 96569.88249170227, 93987.46514582132, 58613.38609842052, 31854.007942145214, 58300.19806210298, 37141.139465035325, 5093.80154525263, 47566.159664035426, 49274.60149096008, 55875.241569797785, 98201.28961306947, 42699.21092434406, 6320.495515199087, 17383.401008566278, 67557.75791822837, 15878.952411233071, 3233.8237089734043, 22398.765294779976, 10681.14096415963, 62921.202184794354, 97931.47691695651, 85456.88083518908, 3663.970948672912, 70860.33331079953, 47828.10069909909, 64839.987824154145, 41357.288556653184, 33898.23754589699, 95706.64261507004, 54817.8927519125, 16984.830719454512, 17558.22632802324, 60543.72839159404, 2407.3381675677074, 29961.241144232543, 32863.81463405135, 59841.89128511323, 22800.875220411242, 52618.91147416001, 78519.1621776405, 37409.21559187963, 49000.61750518398, 61295.95524108121, 74258.44369116591, 43493.59494292871, 78462.85661206755, 47629.67091875446, 13938.984463878567, 5546.952491806245, 68620.84106824579, 60201.67335731582, 41874.039294292066, 20573.14387857425, 59575.19479044709, 2128.3320655061557, 24071.91247545938, 64814.059633526434, 24284.840069528847, 44569.29568939776, 2093.863682760133, 89848.2339165208, 37310.78636724049, 29127.85173219443, 16859.371076649564, 95938.15019972518, 85482.94153403414, 43066.40250838434, 31082.947052473155, 10127.111952912172, 4331.672843702406, 7268.689664011318, 22263.713352111037, 30086.63633493761, 73444.4762359541, 20242.45148349333, 47724.63968890146, 90987.91852446378, 61978.05251831506, 86530.82242566539, 45001.533784935724, 91162.00214149726, 62807.96856888858, 2072.6053234280453, 64144.27146013992, 11685.972527345057, 52038.346416207816, 84246.87841265606, 70699.95041339133, 11762.89470111933, 70792.32077914609, 21849.81886815691]} +{"id": 2033, "vector": [77220.59598024392, 40522.09854977955, 46825.49354093603, 69098.99892126267, 80415.40935090599, 94591.17553815682, 31294.268917358237, 65966.17934861172, 65565.9579560898, 61200.17692484099, 55943.187175584375, 36166.91045728097, 37342.742743803334, 27051.038238619396, 91811.38785768888, 14109.491931192264, 96627.62649610855, 68571.67579553966, 49124.3409453382, 35554.02218300221, 88845.54961593439, 93809.34914322109, 95508.08111260219, 83233.70507784598, 32831.812298634424, 87150.37968750027, 12918.927208027942, 39622.7287512399, 15973.86186503359, 98440.88999670389, 24644.573745197828, 22778.887350992016, 55069.96957079408, 71485.75855727539, 20555.705815515346, 97863.75900093747, 17491.2650248211, 8614.766034111044, 25102.85542104056, 39763.92356302465, 95308.99765856555, 69935.86464633454, 3914.013528045068, 45077.539148938864, 69368.1423930479, 51204.805802217954, 82356.77682338207, 73094.52380671531, 81579.52401723017, 1502.864053267483, 71071.43185084563, 57534.301695659604, 21466.22376398998, 60840.25714541463, 65252.20521412821, 79956.41277705623, 40994.457437538644, 32286.731669658286, 52006.397923893564, 90218.1047782874, 81860.2629535855, 22609.024936330636, 72871.53903392519, 6188.046006518511, 59700.445869426236, 10562.681810511776, 26337.05458182054, 66485.58376759995, 90783.23829736386, 9946.110020106225, 26101.906501243076, 89147.6806209144, 66503.45762742116, 50158.80318050341, 57105.18835946458, 82124.16323738889, 7265.759086319723, 60926.91825944601, 98934.72247149696, 78684.90063157091, 78708.58795532181, 18682.760524340138, 11058.760812081291, 74967.90453653777, 65581.55052642494, 97677.3721609364, 80893.79686566534, 73121.01897990561, 12999.79066385284, 14170.960774185693, 73039.39322610882, 26735.72087634798, 94780.23373278991, 93888.07877127013, 62667.371097848045, 37615.892518050176, 50687.94807099071, 63160.993380544605, 16553.15491815922, 57365.97890999781, 36349.800386049494, 36585.90142070011, 8517.28752052363, 32954.64413082712, 99875.89231584943, 61585.54666472156, 95872.47400696421, 21673.50856915963, 16112.62001245579, 73421.7986096251, 88208.26143407905, 62494.25997849053, 54112.20803354297, 10418.754965957189, 71041.98692146184, 57608.755321217555, 10801.416187221757, 62684.49106758779, 96260.89730542076, 85433.09890568367, 79508.51574604257, 27590.079144312596, 33368.89184132325, 86509.75118648696, 31982.3101473095, 60521.481781391696, 47436.10940091475, 46079.71778323239]} +{"id": 1574, "vector": [25480.661697205764, 27956.656982841134, 14742.156786730964, 68525.5214307843, 27558.096242517793, 35088.538834101404, 25047.18431377463, 75322.35706853402, 2614.390784857612, 82409.14512785799, 87434.64830441258, 56456.71879995803, 86385.6006209432, 8485.682960700036, 55002.482360611924, 74100.63753314634, 8456.269617532818, 87627.79398697772, 63601.733264912094, 41117.89848163456, 24402.11152630126, 81882.27047443316, 74759.24118254124, 78918.47569750123, 17070.73967290569, 55779.012676133854, 43880.20166373915, 27879.268636550227, 11396.48151392071, 69628.70956707787, 91156.38061895985, 78429.9113482419, 6915.049835199216, 4969.940578348808, 22479.39313205627, 35990.61030732734, 41502.368720296945, 3316.244949712266, 23999.71694637515, 55792.89739577131, 47917.273333547215, 79767.50074736583, 92151.2758816275, 8812.941248477635, 13446.655628185412, 22941.27695808138, 83315.75080935557, 11671.182281302217, 39102.28917690906, 13273.101715295987, 25824.320871107364, 92419.05312014904, 40053.15507145698, 12642.051755180162, 10230.340485620582, 66641.46019922802, 12278.871829357451, 22038.10780414531, 48280.35990230433, 19795.10981290734, 6918.13229737539, 17082.23182884714, 38906.21856564157, 31426.81469727616, 47583.68278787829, 51427.308070056286, 93329.58407419668, 65946.46160408857, 57003.85112830506, 65637.2475577148, 66417.77650215746, 9769.474705094804, 55807.080808361774, 92320.37333300586, 78858.88714710226, 72714.92306774041, 38628.67069287932, 64605.88346198861, 44243.44160253334, 89069.157121465, 46256.3428667627, 27000.27758237614, 34503.96306196742, 30859.76497874834, 97363.15026529603, 23950.977412674256, 51175.18303243452, 12961.749704491733, 36564.709313856314, 78934.68166227883, 25601.11878525273, 41365.28791207563, 70553.71384886079, 63864.521996749456, 98796.64139943363, 7930.979962149109, 91912.47277727356, 63272.339047587266, 58146.75884890125, 45347.06110733898, 19015.399612287132, 17962.072939508478, 31631.117406883313, 42064.47455185873, 29372.73886537687, 29954.851194418, 5859.428827788948, 20728.068509727495, 33811.56896268565, 12401.442147975638, 78227.92941550106, 81292.92956199734, 52512.07525169921, 93041.88475113973, 17226.83171580982, 14399.604007520129, 42231.82952739571, 85946.9961147034, 6965.166600151484, 74491.64863741564, 40845.15463563556, 65420.118589296006, 62136.61864264499, 86647.01910744845, 26396.103415715268, 19584.80824654394, 11360.176992391713, 87600.86161481228]} +{"id": 574, "vector": [53249.21514853769, 75726.56542191138, 81521.28608626948, 2182.5255855415617, 8207.95953144351, 80130.68077001988, 3129.6703249644174, 33727.26219845109, 36033.41782284526, 10714.736523732516, 71939.19573461227, 56503.10959172623, 95358.85945211766, 74977.54788387011, 75177.45937016315, 45664.69787043533, 97076.92897957156, 41821.5077457466, 99344.939344258, 52724.92618460776, 27414.921972433327, 70249.46894974634, 8235.3792289229, 61438.820722391174, 61000.569028322236, 81213.68591938342, 12825.94809944062, 18335.36971235693, 86280.26052100651, 13264.06351728413, 81103.79095143125, 30439.194361023438, 52624.651792395845, 96858.79877598218, 10799.998788750665, 91544.12851965029, 1242.5652839735512, 26580.74080016982, 110.41092869078506, 15589.235072666474, 64039.17492777287, 29044.765029073584, 36560.45999513394, 60263.455082426495, 93981.43884151234, 12717.814085205859, 93013.20811431465, 65178.947698813725, 42213.02717740949, 7184.935330696562, 60368.236909223386, 68520.90133470105, 38142.49204606847, 3699.280778785463, 47435.19325601927, 90045.1485142838, 78416.21232367543, 54086.881215006884, 81445.05334586625, 28715.34170376295, 12907.054118674532, 42355.200402895876, 14424.658089970222, 43355.27427546888, 8160.510809005795, 89266.40863934394, 33722.389830488166, 67882.95079209864, 17669.715002974328, 18109.14409156925, 58434.30303438935, 32918.39543558044, 2594.9344361538283, 10134.99408897447, 82622.3254801644, 72497.22264839799, 63722.531575556226, 35502.197440997006, 89021.63908326211, 64835.50576186411, 77960.63606523241, 5651.463932549916, 6058.429355184824, 98546.5947118228, 12471.716850118231, 68969.88695475174, 29129.375162164673, 99289.99034926038, 6147.556667391751, 2587.1908698833936, 56173.849140602695, 53209.9835775239, 9432.448921581004, 82476.60011111034, 53254.76157932406, 77195.0741880424, 94521.11346091634, 59383.35153008439, 75571.82078666046, 92818.67103151034, 96811.30065435132, 40007.91827158212, 94982.21884618253, 47856.38816960437, 92814.61389003198, 24228.14974926112, 10896.90617466752, 94507.68273369767, 34012.92383081545, 13108.56787047101, 24658.960078901117, 49247.09620688872, 78479.24114339935, 59428.4735628446, 52439.937357383926, 60112.75963403154, 30510.195234316663, 27441.063233903096, 25821.14113540781, 34330.54399461486, 34458.6949128963, 3752.6079963057346, 18849.821042492542, 80825.17861866721, 9734.336545147393, 40541.83403283064, 80518.7101777954, 83050.48167113395]} +{"id": 1199, "vector": [58696.50191571084, 84235.01395310988, 15305.035410920531, 7455.515185775796, 16858.45582345181, 19441.032127100345, 29316.889170075876, 2024.3059691045562, 64345.6327587666, 40174.76420795768, 28429.947995874016, 26847.90407331765, 73474.17740768389, 35946.29052019973, 56669.62897727674, 5435.868073529693, 85469.04884536423, 65266.83899755285, 16201.104349821117, 32879.89042643592, 19577.125772993397, 17720.0162752845, 1311.751243321746, 49347.9769057196, 47051.48137196337, 14659.36001507675, 28618.841145379025, 14615.389872016238, 6128.512293345034, 41705.091731211374, 70293.49918481355, 95992.23924089801, 53416.03067767731, 87128.53081967044, 76772.6455533123, 72226.0068393488, 77895.93584299705, 67115.85661904495, 61669.70473439904, 45049.00073926164, 68965.24457588697, 43610.39480930808, 8398.731640669666, 62618.825732698344, 78190.10528796291, 60331.443027172725, 49237.083601440376, 25120.13089910199, 66425.77649767893, 58121.626508826215, 79278.82036393369, 38669.28915755349, 21827.51817142442, 59944.93707288251, 25325.387903721374, 72356.02844194313, 98891.87408547271, 95047.16343774644, 52012.547394306195, 24534.718843317827, 55080.507465632436, 85957.25735947912, 31776.618312621995, 16855.485163962003, 68323.9450519606, 25834.75410134214, 90924.46371099, 10874.70514404625, 31841.741706763405, 34334.5455267299, 30090.99193486604, 52574.06875497649, 17151.041294604118, 24871.64133028871, 53981.02381970162, 64040.63656786191, 12172.848469221053, 5241.573959031898, 90379.77665223657, 619.7631543910842, 87995.2860351859, 70912.05675403829, 34161.95964856465, 48172.38879045709, 49275.530016371515, 66315.37937589065, 49862.83761689781, 69724.5217939727, 56625.18205183866, 24779.723367395523, 85539.57870011302, 19420.442263594305, 82384.87355965102, 95960.05667257377, 82467.36875864201, 5981.213148381604, 63045.39242455055, 37222.625521153706, 26330.26359888403, 36001.582084272, 22215.471745749182, 35000.529642376045, 82266.98473926811, 6580.820191208503, 5272.69129247917, 70874.28374802457, 22242.35330422484, 88266.9213704697, 44013.129197516224, 93343.39149245486, 64083.75421188807, 15416.897991354239, 5281.908539330416, 48875.89216376853, 41238.49679646069, 44530.83690304894, 48558.855989014446, 39707.58352138915, 33792.597419095335, 47881.019920832914, 17411.702022595644, 3053.062519862615, 97473.05498897602, 58877.65832372245, 40725.10189438211, 25329.05568368318, 73825.11728077536, 47253.75698382447]} +{"id": 1748, "vector": [28100.502330955213, 64131.07864199353, 21254.630426121625, 19358.504664674314, 36379.366642105604, 15243.18162272571, 41405.07243113829, 51711.19436959654, 5127.3472978825785, 99711.44818234522, 96448.10592563909, 92431.3742030827, 40351.04987028603, 64378.39359388928, 54631.10532749001, 96313.11148444515, 62578.702535213546, 29187.422132016207, 98043.69866562026, 13297.582661448403, 90933.32559361639, 10927.80488466234, 84927.69239131598, 58854.7877818674, 19409.059443895927, 30916.664248529003, 32968.92590181366, 69028.72175490705, 63905.90428689876, 67203.26282261978, 78807.95385267309, 29562.880138047065, 61192.78876817906, 51169.45482936872, 17250.375323705113, 45450.63350307519, 66238.71241140083, 2365.1717271643056, 1542.315279771167, 90364.29837950018, 39151.4086494011, 64920.82623495471, 12336.92739615353, 15035.28168659991, 71465.42625156195, 3282.1749914393795, 56020.186504999714, 67132.31005485676, 28173.013787998934, 74364.29814712849, 4143.454923491352, 43951.17491535931, 58266.79740288484, 13319.514965454004, 46719.43264035051, 85469.30882223134, 45943.63470673964, 71418.70030996071, 69312.2848110976, 78800.08405083361, 45202.247305778044, 24558.882365309477, 25451.878700053498, 14950.063310613148, 45453.281846482365, 95178.2947494993, 73082.29003698193, 49018.6750032574, 87368.3305814694, 81417.72208447452, 88443.81472273715, 28512.969516795773, 32600.28921259307, 43252.75854381811, 35121.28415721769, 2535.063456312092, 27837.179394744704, 82145.00802245688, 88819.40620063787, 46561.650580695794, 31632.20624898263, 22714.715077575533, 14071.811988590576, 95773.08153311921, 83117.05704126862, 23391.62333288738, 40235.039114003055, 32238.10280116656, 86632.55082861365, 35325.679085496246, 73138.1225397974, 32503.326945062185, 77637.13135429689, 11736.798429738827, 66208.33844862222, 35141.87098500493, 70165.23384795742, 28144.38699746724, 61981.628214031145, 3866.3918511344764, 92347.66640533258, 93922.5333745351, 49725.73495967617, 58813.23580337515, 27461.45668683092, 81341.20230882007, 18813.16371881042, 15198.319922657543, 6590.697482521768, 99452.17341684425, 24105.126743362125, 57148.47786235799, 10479.658991280305, 85544.5146877148, 92525.36393534814, 43887.07041373311, 18912.050889174738, 8549.988134797393, 93083.41454389122, 3080.4661815136415, 41300.556193459845, 64634.70427684138, 77604.81656539426, 65092.75734752455, 51333.24758451776, 34751.65827671427, 3188.276544574653, 95155.70978180128]} +{"id": 1773, "vector": [83938.42847879947, 57494.14306510597, 4593.5252921277315, 89544.37798081838, 25785.10720551256, 91083.85387854172, 26598.636176249867, 5353.845343955232, 25885.846795775124, 34390.811497554416, 50208.98744627479, 99144.7440925696, 67617.32267794403, 4614.257708254166, 76164.28872066294, 66162.55325936184, 26944.887207582047, 10399.759542293285, 95192.32288784177, 63287.53446159936, 23413.23089143389, 67273.53201922163, 41355.018841293044, 41183.9752793279, 82501.63523720422, 40272.379901599386, 86983.43400357968, 58855.12062386802, 87377.41416596143, 38297.33645955824, 87798.1950323256, 74529.17325746764, 21810.776603378567, 79129.24701030758, 60369.33560410427, 29824.646291525514, 89279.13258381111, 86660.323327477, 68857.41062764563, 2917.1209822004607, 92896.27320191663, 41413.88536903349, 38773.52607044143, 18233.40019404971, 75048.99913675201, 27044.913255664706, 1599.6350698589845, 45446.537578388445, 23061.023761790013, 2289.501028162488, 58958.7546118837, 12725.176043939835, 63792.70304300546, 2092.5577337365776, 40829.46004413832, 71918.80584369472, 55432.784737317495, 90033.33929129565, 73981.65108803238, 37939.452520054874, 77465.88303015623, 42336.855447743736, 15511.385208031448, 92569.85066890901, 10024.457149115806, 66327.39849740974, 60640.50968359747, 51307.83114734628, 99589.38283219145, 56230.659123766134, 84659.44321312681, 50044.58354770831, 2227.365437438855, 72258.60368864327, 53430.19280534636, 762.2120192980697, 74459.42309916424, 42920.82326843616, 97309.06628101386, 6993.179318265885, 56864.19207469806, 23460.468914478493, 82887.6437830733, 11048.967169740497, 48824.53084356368, 53402.087800408015, 86345.73491288658, 74700.19750623, 21639.540568760916, 13709.478511949857, 49479.91366671821, 39110.344650051244, 33650.078212587076, 53750.45449372436, 99722.35328474095, 9778.968670842603, 94237.55694226621, 76197.94655717247, 83466.78295590635, 77315.49826149625, 66163.377066659, 97566.20434485751, 35823.556949697566, 7615.507614838657, 93386.71671306349, 24445.8169227043, 66789.60247425137, 34482.95289321744, 80571.70253897081, 9390.500861706485, 21679.023599871594, 55253.325541976614, 88444.10737488461, 67986.76362391646, 55929.533619295216, 4694.59414990342, 16725.913384071777, 88497.73225026033, 1579.09998468867, 50432.26175655624, 40964.91576070939, 6328.214373499807, 75685.15637856578, 30493.23264507755, 96875.70182762248, 43540.617498957, 4907.517280189289, 14682.062329567503]} +{"id": 933, "vector": [30500.86408059941, 33881.57857817543, 29229.005253932362, 80411.63953384782, 4299.3509326163085, 75784.03915531564, 10612.979126560784, 31569.01946956988, 30559.974100351938, 80557.12279909059, 10510.52979492394, 23787.964791136852, 26927.228653578404, 77370.95756304482, 35800.05303599074, 51007.29023063041, 91416.07392405761, 79031.5701589843, 65349.09870362123, 53581.83918056444, 80700.17245754291, 85555.30354341277, 39995.663981057616, 3729.7454505927362, 47765.44868939654, 55993.08979208435, 63909.559401496306, 11173.797346516767, 93753.82169262583, 11640.412080752305, 72196.30165022238, 85585.16902180677, 6277.202891396972, 85723.17078046082, 30823.96772105157, 25163.293469340842, 80972.78068417814, 36458.178220640555, 68169.54869498005, 35902.447540093366, 31972.500500031063, 22093.512305628006, 54856.98346483153, 32563.10741629905, 67259.25464030949, 41833.71016242682, 89535.51785631802, 55003.55840966068, 27847.639994881312, 48399.86982080159, 30369.610634105236, 51307.73916793373, 90352.07744536827, 29401.655827038674, 61895.32947822065, 72783.53195763107, 14228.000238012095, 24885.965540023626, 95117.14873815441, 84561.04216557549, 11317.987526314955, 55291.096430173595, 82532.65680611313, 30099.5750326694, 75046.19114289836, 51916.7374254219, 40541.56701355619, 22944.126235412255, 15291.98775577989, 76788.51490523784, 34704.65224255593, 90032.93829495915, 1457.7679495073492, 38569.86032427806, 9064.76949403372, 74856.75487990733, 90850.61849249255, 88275.31641743638, 57041.69824966926, 559.1137743719999, 1791.563662660567, 59702.53634619651, 79795.36754507842, 714.272489813017, 19659.339072076265, 35333.904508969, 56946.862948584676, 24851.74248701583, 63783.968227911326, 64205.495280420255, 57997.85287375478, 22206.52366747027, 22971.485311021468, 53952.79161694648, 55789.4857153171, 46489.546649970434, 46371.81779172148, 67901.19929481317, 17569.98052318318, 81159.78643176376, 99545.34092848348, 43879.08699611388, 90256.49095526514, 96820.82061297951, 6904.187392985439, 97910.60661490695, 59556.18095824804, 96307.83716605362, 74939.67019630414, 99296.56477588759, 44302.49681710734, 10089.18916504985, 83083.47295317514, 92112.33439715808, 62542.02101816444, 87361.86658398189, 55121.43614493813, 91266.20480732628, 52446.003846097934, 30565.923962731955, 31193.946847328778, 59445.67341544815, 99334.04442814205, 13157.92260750418, 62845.31314704592, 21755.25168248785, 37686.27115914315, 43857.355966993564]} +{"id": 862, "vector": [89578.55673242379, 92537.43892483372, 35177.97631286198, 40215.40288073021, 18623.663284308943, 27488.839077235873, 49898.710467612975, 9845.05615967658, 92810.38694206896, 99131.1788668763, 61070.59800953283, 10585.419678281327, 27311.92629769781, 15056.398104086365, 66967.30692285292, 21261.539580916087, 38826.71624389879, 89099.66933848719, 34191.613550109, 80783.41263574417, 81347.50363518267, 52898.012918328684, 12621.54608190249, 89103.71709953602, 19617.21489033934, 20241.45649004786, 99638.35822778447, 99586.99236720573, 64377.678910866, 93647.53936858768, 42722.434346556925, 80157.78283239184, 90021.09439215434, 30475.124806549124, 14532.947890552217, 4241.047092218486, 12970.73075237093, 10269.00320566776, 53845.19254658174, 25687.405342953283, 98910.97247398774, 88155.84402853518, 50674.991553475455, 11065.044622040099, 10763.465408337614, 17816.007896466912, 14864.41212100198, 22734.858030371586, 37571.96892249054, 87393.41101262177, 57757.25882537254, 92760.62899401734, 60342.69980407339, 94621.07178606608, 45595.651873308874, 48614.27421981431, 69792.99002455943, 79320.90824994042, 55824.146546298616, 63227.124144595495, 81105.61510301757, 26925.254028939926, 35670.80888535471, 69601.21618441655, 57725.30610059346, 82458.22852161786, 44209.878480809064, 2276.906164434223, 29390.989923577115, 48764.66431702328, 94709.84897723781, 36137.05617021894, 53961.57980236487, 38531.91321133609, 2180.1701739301184, 53033.19446747599, 15011.084548114706, 99050.05910848413, 85517.0087699494, 36178.76204037399, 68426.95130860966, 73202.3109287877, 7799.715264370566, 92480.68497132251, 75244.48555238567, 93631.5019865274, 3495.7801319436376, 35938.2971593811, 48302.44122303726, 47003.80802975339, 19824.71935385257, 56087.61721641757, 57878.95792588601, 85763.968297446, 84784.84280105031, 43824.572865320224, 90343.30591944979, 18688.457695841753, 86100.71032327041, 71698.20524429284, 7302.964352408103, 9908.858891944616, 42077.8196569792, 42291.229093895454, 21912.69156521951, 70873.19097480718, 37470.20496394624, 38435.03961965728, 13905.819610647608, 15315.763739551614, 16371.668327356736, 22586.227360029086, 63075.424076356125, 97390.765047265, 29601.45738675245, 30562.353889188966, 29905.135590756283, 48142.3795867003, 66084.09213781366, 71521.04439414437, 97009.57643038774, 60482.92129464583, 80082.1914852313, 7275.921596923052, 5891.243340772689, 73396.86766961821, 14020.633054798082, 31789.470500854834]} +{"id": 1723, "vector": [59654.42142416623, 46329.20186042223, 25457.23655346016, 98084.52781149297, 56474.93760143752, 42489.916972793886, 4355.153961538316, 86463.49282612599, 4455.857416126263, 53666.682264078525, 96455.35921939476, 2280.665818513505, 9330.060714033229, 78079.09462853587, 88578.64668089992, 10840.385515995065, 42807.85848913455, 67488.56410113451, 33020.08150670399, 88442.46143106256, 63937.9981353485, 70278.65616991899, 47101.569363708084, 73198.14449217795, 80396.1667906942, 20378.2142534509, 42793.97143406036, 95529.79918112348, 88731.85326503709, 82684.13671108668, 89998.04199234702, 94371.55417726589, 74047.37654156839, 39123.002402732935, 59606.37329832063, 52260.14989362097, 60330.752656662546, 59413.12743489451, 40309.86869186693, 67298.29993485092, 11423.056375868478, 3497.2291791074085, 77008.69955129763, 38621.49591597358, 65367.42275023032, 29363.527651957942, 46793.94325630045, 79201.44656891769, 77955.62943761593, 7747.942882854298, 78351.47564489758, 54752.80166810036, 62100.10003486208, 86431.73928740941, 20341.777376102, 48669.278737809786, 64093.26811615742, 44260.95093174408, 58934.51405640558, 75835.58991770678, 87964.13628289942, 12502.326885532311, 73225.33572817262, 81315.72921518939, 289.7025436526923, 41839.23350639952, 45242.57509884459, 41931.51300085951, 51851.563871521146, 88647.85333932916, 96744.69593397842, 75362.33758388653, 77495.36439099646, 25214.913053785458, 8312.638101743853, 47850.87080620255, 27248.175591688152, 77156.09790873004, 39766.3551822121, 72566.52643085894, 23908.02496267994, 75637.9066730375, 12192.85459587024, 40299.201426726264, 50179.38713879891, 57154.97804990415, 66914.39148710622, 78506.53960677529, 2749.39875124256, 35304.653363334204, 86152.86107601786, 61916.63145767664, 19501.578315263734, 4468.9379693501505, 95875.04591470711, 34950.670483117705, 27246.11024899254, 55945.94389171989, 69464.24116279552, 46108.54890708684, 4392.7536757650705, 58739.887451243776, 24194.74456082956, 64178.67615313666, 52522.45478253028, 82798.87767794743, 43193.203281006696, 45339.964471418185, 1251.7380695915458, 96756.741413062, 54838.89916985392, 36718.523192403176, 4949.851605746114, 72209.89291981165, 52288.79659763953, 59647.209347973454, 34792.51449130002, 53182.420710752456, 64300.82044155078, 61092.24966307527, 19998.080313060218, 46189.758029355646, 34656.21934835538, 91395.12867650461, 55019.530517305626, 98711.75222514666, 16116.014735478357, 49532.785035540575]} +{"id": 1684, "vector": [8313.98130980271, 12462.130432607355, 63861.59458551896, 10643.405521421568, 1637.2838469272954, 99523.13935222164, 79179.14035545457, 42025.76674407593, 73744.49078047555, 42228.89897215445, 96286.63717815561, 69997.96597914245, 65641.83594393787, 59643.4658065635, 4768.888504623059, 63563.64289004538, 34401.6985893299, 54119.06908534585, 52175.21224718752, 24772.152339090626, 13110.104135700783, 9161.101595396092, 26805.318748945683, 70403.99453475181, 77200.28616669269, 83196.89089577865, 56393.94821615645, 10371.94520274365, 40363.69149267655, 51794.00458110307, 74294.19032893056, 41998.34841680789, 45390.90030707995, 96899.95459707634, 3284.904906510411, 57719.794933028235, 94747.99544310915, 36482.20752050545, 20776.69998074786, 96858.48171069236, 25193.061930431195, 96105.25537090599, 27911.38862505168, 5760.378473620564, 46866.96457936617, 40038.017851901895, 4447.57946972284, 32995.43290885041, 9923.216399039813, 38779.62842602916, 14529.02339820048, 77865.58526215231, 78021.55093956097, 2456.5935346127167, 50177.63559042345, 31788.575483103643, 84351.4722831332, 99991.37179131557, 87062.44916887784, 72474.72221612235, 15659.385713694774, 15022.201306724759, 70650.74905625478, 61023.33247504872, 70856.97610912319, 14122.985109342057, 62294.44476190827, 55892.6169417461, 91672.06652152045, 96209.61932283953, 70666.9658647047, 9912.713284149466, 256.064927143107, 17879.694598994432, 59779.820286674134, 34995.931711216544, 48330.60015645174, 13431.768878814664, 38389.88055625113, 98678.1224555174, 94865.82703325673, 74622.59271692377, 73370.00431834067, 76378.90140914517, 93932.89396866906, 77803.65544958478, 6323.0781489959645, 56394.966897544284, 4980.02602275881, 56452.656785069164, 40897.98683722816, 60473.83945911496, 72268.88314715805, 99706.70154402065, 39316.88033135153, 85204.60402187397, 23103.27499154503, 96628.83372604937, 50131.39326559773, 50481.106874636505, 98881.84685862742, 32509.984572097328, 79621.64469620676, 44287.68535516984, 28465.436851895654, 8472.692730802379, 58237.030238355524, 14869.840036273852, 88858.57848032076, 32880.14665141159, 80945.99164556486, 90533.25075571035, 11504.455556485493, 80105.53041167479, 48777.627378537145, 18162.045565506414, 82919.92208583209, 91841.57495653167, 28251.550089107168, 76095.28062535089, 28646.309591630394, 25845.06556291556, 66577.35633611924, 50436.927393788224, 70527.80483745628, 13036.193211634662, 44903.61643753928, 5755.015347668424]} +{"id": 165, "vector": [25363.109912099157, 10944.188665203203, 31145.822665754753, 26874.646531929935, 64839.204052699715, 44286.90059873391, 82077.33239954473, 834.5269772571995, 93625.05097130155, 81737.32004525396, 58976.92543400817, 30610.83217535139, 14540.268584815853, 17492.475159076537, 9181.200711735815, 41257.43593609822, 45890.67070826564, 61.60606363977416, 89975.12190386499, 22144.481855193364, 49144.91400849968, 60556.01768369308, 92600.57443471932, 12922.8971513647, 99935.78113031169, 90499.050327803, 71708.02947108613, 72473.36719534751, 91787.96364141138, 51903.29628647661, 65646.30688030912, 12334.595300332663, 40483.59221217417, 11263.018211814557, 98108.66082732857, 56948.68204328176, 14035.45317010505, 89757.44294463538, 13458.694511632662, 30668.323472709213, 29089.33170919763, 75679.35998771086, 82549.3916906038, 84188.85690338597, 8003.071117958582, 4054.4696416762836, 14055.915878537284, 33459.45634059191, 25725.887412220338, 97021.0837006988, 22024.651101969317, 21625.516994488436, 6551.350766120989, 26286.58231515193, 73777.52063648359, 68888.84051337226, 65009.026567986395, 61494.4835607903, 23171.741941852986, 32803.98179708711, 67215.52260260114, 47977.54148717505, 6820.687778537626, 45091.90076686568, 32797.40388279234, 86204.43233668375, 17612.151401492847, 54612.40750993047, 17712.273389440335, 53121.453001682916, 99137.0623679544, 13996.764893521107, 71603.30979011329, 82161.19132594999, 85892.3757625895, 35893.326584094335, 56590.49661649638, 82066.6602982666, 95453.1052682397, 94731.25523424834, 94669.53985638646, 54756.57815010731, 60774.88314806507, 46845.31716945378, 84635.57578565615, 44459.58593647258, 91690.82691572208, 52666.194695183054, 72783.72234338631, 47687.636877064484, 39057.00326805892, 47417.368865567856, 76308.45313139653, 89394.81992555184, 77600.60474752286, 3781.1416966598845, 89384.13997965769, 94469.62233276607, 13754.112748120795, 48246.69545050075, 30210.636420013005, 95050.63539283452, 90993.34860141361, 66926.62679184749, 59443.60015780084, 84847.69757303198, 75218.15337525633, 89223.67735927047, 63077.146414302486, 12576.329904940787, 69838.38554425686, 86328.19148025059, 19235.968156800252, 21361.830355611688, 16807.195520682948, 83657.28551840693, 25097.649989701986, 32957.66658761462, 52781.38936093647, 69147.53114732355, 53914.02344548425, 28229.37089177898, 40193.463699115004, 32731.37452439148, 52907.608348907044, 81976.97435972509, 92100.81375844698, 93812.66898601332]} +{"id": 622, "vector": [62639.117367334664, 44865.41911969815, 64244.704148347555, 90527.23870154914, 95271.62511363666, 13398.995667273706, 94244.30913727228, 52856.79498535924, 26790.786064407555, 87250.41564721643, 73731.2620856745, 72497.6804605674, 48755.799046393935, 46435.7457898246, 49265.722413158495, 39038.65483787718, 74906.27047927055, 32938.98599516843, 28566.710074334478, 51658.18257738306, 10723.930883089482, 8903.679307777424, 6387.460369234199, 47521.3565671633, 65555.22433962337, 40985.93798835843, 26625.450958848938, 58901.9895683181, 69914.0525474744, 42182.054894461864, 42977.12422793407, 87392.68518994408, 36760.63822796738, 7897.72954156136, 99931.8137612781, 40836.918676725174, 71161.18470832308, 30613.871073998467, 13703.417599448409, 39467.79897563323, 69850.87219538605, 95930.30324026817, 36151.89634339781, 13790.53539473717, 95250.14692737878, 36285.85773398911, 65867.38656643125, 81988.71153053097, 26672.617128938247, 1641.7336039899144, 87831.29680589907, 29174.968970171267, 812.1543319337787, 42623.48534746362, 29105.25296418922, 65504.24279700466, 3223.428055740696, 75987.4448046439, 72503.3436341299, 34485.255867396714, 51688.10307343354, 36714.69030333428, 43699.06780126133, 82545.45811392082, 86327.69240346621, 56453.17930400677, 68966.94334720662, 14994.580044674489, 54711.358032374505, 57602.56684006957, 90241.12929205348, 79572.12164540961, 88283.11178938806, 2220.8319034809133, 76735.48110636891, 49969.73603799331, 78621.1939128371, 65276.574558430126, 12239.170106868636, 41790.61574964336, 55091.13687930224, 40033.48887515476, 17703.79130057913, 46326.95781589827, 75476.0979631043, 61685.27244008235, 29453.95358934648, 43851.81089209943, 35451.23759220986, 12189.929688780054, 94514.51353043839, 26841.603665669245, 88421.73055530137, 36436.91063729188, 5385.164631640549, 54438.38122744159, 56498.553096794654, 20902.508480760396, 37698.42188569277, 19495.215419786506, 80301.42934528021, 17126.723096710273, 82387.23636390165, 59014.238667165984, 90274.79567362233, 40004.18928078723, 16610.95637971901, 63969.127667414, 35588.86547875152, 18965.08574889618, 29655.545208257518, 47836.35794748225, 47695.98529009893, 65062.462868984396, 16127.570348059084, 3904.2931703965887, 48428.117063528356, 77494.35787873567, 99063.4151953, 58508.67364465151, 65441.59300926756, 91855.23536781261, 99624.70743591516, 94395.74110235219, 15029.583889111098, 44048.73194740233, 43042.96901705611, 78818.03363946754]} +{"id": 869, "vector": [66187.33007588911, 3375.517272438033, 4229.0769166952405, 99748.35129613499, 18848.81892152611, 95367.20254978888, 81825.89244126156, 92432.97660481592, 71465.49870170694, 1577.5957462728418, 25161.397078729366, 63022.43889754578, 53001.583776931184, 7530.223458512331, 78407.76512607948, 3991.891035077977, 11484.386173164241, 97636.53503192092, 59500.64059216186, 32394.675357951484, 27546.99001284646, 21041.855096351537, 8003.444587186393, 28828.364640628846, 52138.00987981288, 38975.75655565183, 51406.48104879193, 17994.08408712565, 77008.72949137905, 46621.54862884218, 38493.00814147531, 42145.104786662436, 58903.75727514678, 80134.35522466828, 21607.34383744094, 78068.8920754233, 48477.816434000124, 95113.39531956328, 63685.85083165984, 74907.2297210381, 19555.92222849666, 53615.70836829763, 80996.22052631807, 19685.084756149463, 87538.15218070056, 52627.82248441444, 20423.86992292278, 49682.130842952276, 42405.14433439352, 74608.21842917378, 46799.776914122645, 24192.73034830499, 8797.700148960852, 74172.88934859233, 17557.853795578205, 76843.35148140762, 6241.736561433641, 51666.129351090574, 66995.84871962474, 65645.89418706151, 9477.744497586715, 18891.121371888454, 65475.94171792238, 12294.978484406416, 63691.5512246189, 80693.27188889548, 49683.160545018836, 96924.35106133734, 66500.88450791468, 61528.59213280576, 18798.084457199406, 27430.48169454465, 2852.2626452510135, 96475.05102683548, 99740.09185427547, 23310.06189900272, 21839.851502856945, 73573.83888776474, 98156.47621538794, 17072.71744171385, 45952.8138444071, 91156.7124730696, 79682.55825131622, 4612.259712096489, 22246.22061643651, 78077.65816346118, 92877.27581007368, 74895.29231309873, 87426.77579603426, 64235.865374494824, 77178.89624442179, 8992.240521265505, 33749.27135513507, 44462.544930931035, 76496.00536219568, 25560.586487017223, 45663.130915831316, 15316.661392075714, 26194.195413348643, 71276.89746688849, 48922.29747200954, 91143.85301102915, 55935.20759885695, 36498.71182971556, 44988.54287372875, 9958.99084290689, 56576.52900276588, 87310.49370688523, 56732.37052908152, 67820.57103999333, 43863.075111390695, 40579.64838964181, 89696.56004384004, 11508.477894758951, 57292.73104451728, 66114.46027969065, 8704.59712322149, 35407.884670072155, 43082.383142915714, 81310.20337797524, 52407.23879464282, 24402.972714606607, 2956.3347877323176, 28230.336129951407, 42544.78416128309, 94465.02678794313, 34005.230485306914, 80167.29620400869]} +{"id": 48, "vector": [71176.60570545869, 60645.71345903989, 24565.402242394528, 28973.2439474197, 87658.31984532395, 29799.907639336376, 43807.90852493186, 72555.82980236794, 13952.291117376359, 57625.64639543579, 18233.24426704872, 32962.082772313595, 32356.510445376753, 53297.803875854945, 24121.132428506065, 28899.631440334506, 7090.995637828057, 43951.17577050032, 51289.517823960516, 20266.18985669687, 1024.0000769642932, 26829.100832349384, 59444.05499285801, 87768.76479686196, 96951.11234819794, 65826.53076562425, 35299.04057141988, 29669.711708163417, 8652.018833401899, 52578.2546212514, 9770.291652356533, 89092.31707930173, 56711.60915913744, 81248.29111446731, 59020.2375385965, 91934.61005823905, 67916.20974305422, 94871.41870898742, 91762.6292196927, 82038.39164000175, 27696.164259322777, 84194.48995391907, 68692.88443374276, 59127.3298045207, 65070.610591140445, 8403.42196583046, 34117.42298209372, 55574.48511654185, 81094.46845983583, 11137.432587760642, 62934.977134841465, 36105.27136754149, 6227.937613872514, 52185.59923225102, 10417.977853762706, 82350.64656002958, 17744.885048040393, 7658.126888721639, 60657.41728658138, 21331.9340692111, 63011.40667269213, 55936.95337805005, 75508.12440453154, 5466.334889199032, 56830.92834518089, 10121.98872102792, 18367.900329856544, 70761.20130597847, 80265.1177643913, 58677.52801937453, 50071.354381310084, 41054.94688861798, 87815.94887075643, 77446.75987246892, 12461.49716293683, 25355.49689938924, 48481.40780498679, 87145.07554349462, 4614.546080153892, 68151.31107559183, 50545.734022676224, 12610.629728109569, 90320.99637383202, 27005.238806652877, 72204.49812936036, 13007.82879963719, 71335.74171742756, 58063.06487907257, 19989.11185327379, 76306.78599581795, 28434.353102395093, 35881.72584225055, 85785.23271659788, 65639.30113792593, 39369.57415082602, 73166.41897476844, 84778.04447251494, 18917.613765587183, 15730.311975192657, 38477.09465853868, 77565.61801144877, 2573.563584397798, 78497.4228381806, 28478.907154251552, 4987.464260505037, 46942.98235576726, 13643.62965526843, 27319.95012375531, 19022.478803192877, 133.8101086770016, 94288.38207917842, 77609.24907300113, 63482.15635875113, 99757.53719376927, 9085.731976764933, 84740.52397260434, 27466.4607480652, 83460.46871290468, 93270.28764843522, 82079.49681361885, 47686.652317791166, 31189.305415244984, 10962.357620814666, 72633.18087976996, 66937.14696932191, 12547.92653542387, 51685.18053903683, 604.6503153622984]} +{"id": 1193, "vector": [74044.94564501438, 54483.43365427825, 39954.096461980094, 77078.81103285862, 82752.85578441156, 75268.78483555304, 80313.94996992011, 56558.56702081782, 61377.64675885917, 38001.66206593527, 84971.46122724349, 68740.70204256917, 35794.489902108595, 67880.96282199615, 69568.21344732954, 14112.810203090643, 46405.93369812819, 21827.39620987946, 35958.37778233, 83414.47228649956, 80449.698152902, 82892.68727338954, 56709.39916050697, 96960.35327875477, 69783.84596235822, 32427.91315140874, 40478.017191113715, 15326.271614574027, 74122.82464144919, 73438.35850185937, 90078.00658596463, 40028.0210260357, 51031.47417856872, 78945.03362905534, 84649.23002629043, 61032.79178131626, 69058.71178692926, 70426.37278940572, 84221.57458504513, 74254.45875869112, 25348.03531399251, 58869.47726400793, 33129.13649559845, 32596.099571194416, 28495.32213466156, 50030.97546900902, 81441.24013180862, 55526.208804315545, 58865.63415410715, 3228.377497350965, 19549.908872219323, 38227.374640261776, 66357.51736270243, 71236.8290787339, 33100.878767350165, 19252.233255224437, 86685.26604657895, 15826.32563941353, 22884.58378406577, 88269.24644075791, 6608.365886209022, 91043.03926826976, 28021.686454026018, 91799.28297131938, 43851.54563621887, 20318.586675057017, 9003.480948279019, 62369.270315141366, 14438.56748829665, 82427.82054032503, 5193.301654637139, 93728.50606740543, 94652.92284763037, 81422.11674021464, 288.80960782581286, 66516.5657785268, 10226.542655792104, 10248.76027723014, 8680.152765790905, 61166.87397332972, 54905.33779061674, 35381.49844054016, 62816.90381343645, 89064.83571011385, 84292.47229320607, 80109.001379099, 95324.41861380273, 22500.432609012554, 32889.33646253588, 15127.530636558895, 21770.193349340105, 89794.31212166489, 79516.68964999587, 60955.00055445041, 30706.251430651355, 54216.263344421786, 48791.95302776982, 5580.684200820707, 84774.81634630429, 73100.29580981778, 61996.08277463915, 7890.46219059093, 36192.42229385444, 38828.81875854861, 12832.406959854403, 69470.75625894756, 62953.93155363444, 95320.38827244744, 56883.68059153395, 70134.94924110036, 59894.24216700531, 80230.52785386969, 26712.95694053547, 5233.735828140285, 58155.6750745686, 49229.76634201469, 87458.5047842799, 45040.371940395686, 21003.82966121107, 49265.000737278875, 74490.77261265961, 10360.99170305832, 8442.858389021902, 74185.75841420236, 98422.16316396657, 85780.07101636882, 96167.58373823561, 70165.14986277232]} +{"id": 2000, "vector": [23216.371681738114, 85634.54787893004, 17118.14162922062, 29699.36592967466, 80449.04589932125, 96157.7407195183, 87874.68830709116, 72287.96265380333, 28162.03305883592, 95465.68885992712, 59487.90483908543, 23493.402119151364, 72606.16708033935, 90113.65592371026, 69773.48928448335, 83402.96649484208, 31404.881768946703, 88291.56598754521, 78485.86068898664, 40677.249841588826, 96751.76050649726, 56461.498478411544, 74478.9097017016, 80990.22654148545, 99808.69006127135, 74869.64360369447, 57843.94723859482, 21044.35376026487, 89581.88114572121, 76865.50699452007, 74189.0776471397, 72210.73367471457, 84936.42658490199, 80386.02987985796, 95439.56221391723, 83980.46309399525, 35756.30878065222, 72202.69247758205, 98148.58727792194, 13326.626496482175, 5239.1602001022, 70214.74351964773, 40103.31275638022, 17446.483280727487, 15369.970103031383, 58928.973475616775, 73159.16649784488, 20611.591224147385, 87439.51957056993, 47813.39727147228, 6237.222777107687, 49529.92175590339, 26567.770900404685, 47783.504342987304, 14576.407143026048, 45000.27422187983, 74362.1205710572, 36876.23762784151, 24297.48721425956, 20966.01404357631, 81285.02585876551, 28563.975862533054, 43175.78598554855, 99854.36689582212, 30767.665527752986, 41257.41189584525, 96198.54760847539, 88131.35647047524, 39190.32366570119, 97244.44726429862, 16434.484756331967, 48435.415196173846, 13157.851357611682, 89136.99441094011, 22841.553045940822, 6345.239244511924, 99902.01158933139, 16642.22007602685, 45093.83701080241, 42518.55699993447, 98369.43299765835, 29747.52735339864, 18924.913545115473, 34614.4889831268, 24883.524235950972, 49824.33783007644, 86024.26609169797, 36781.39389836621, 48271.172810557626, 87266.35486526025, 44538.18580895316, 71866.30918554905, 49926.5840408776, 57213.570369918845, 93055.69171371045, 52516.06392381387, 79093.03908064545, 61176.48873454535, 71303.98569539825, 67296.08257893624, 96458.0807983443, 78939.19337103129, 70808.26880000826, 9482.663014746584, 4803.8726377392595, 17156.841452909997, 28996.203801407682, 52573.556206467285, 7969.053916997415, 69783.25269746984, 40921.045590209185, 34846.685649608975, 23906.09642829088, 21393.009400408737, 49078.12242370036, 52232.59616348562, 87933.18402911825, 67513.84913157123, 32960.80105474998, 45628.51639664752, 47611.86702297613, 20210.3289462924, 12062.194392975922, 54025.4952651571, 75268.61809966757, 353.36747893187635, 40451.37161542103, 94006.07928955372]} +{"id": 1682, "vector": [19128.179807721357, 39011.295463871065, 15746.637051182044, 71458.30771666227, 67902.19219904278, 57632.384254908575, 54208.848777798725, 10129.101571573141, 15830.556597124656, 52669.7808956587, 48338.36377869832, 91172.54290653726, 76919.87984054234, 97819.39622639844, 50855.233761713236, 98874.3174566661, 9266.850691357231, 71508.42862176432, 68688.62463425548, 2066.4265411930783, 72567.41751088246, 49092.88462277636, 36858.482803266845, 18804.06824910561, 85105.00033336972, 58297.94777255158, 81640.18182664797, 13194.62232011861, 49719.923601340975, 18932.86042967367, 15070.859513637091, 31748.22608779063, 49843.901942485594, 24267.31304594497, 98064.54488666402, 96106.69680004219, 28525.22278247427, 16056.570441711283, 28547.287421753277, 93607.68967330802, 99155.65849509876, 57727.72143442777, 94838.60170835088, 48085.87390264586, 2092.60800502139, 10528.508379548828, 5860.7010463400775, 98604.00449382934, 53855.25087672587, 76634.5944037846, 1799.033360867952, 34439.6173714218, 58596.72465778179, 42092.306190835596, 87429.9206164012, 59492.81612222225, 30955.977362885147, 5694.490297161136, 68426.99736633355, 32068.65834525211, 71909.3109263388, 23698.819901993684, 77162.87808973952, 25838.143421978744, 27113.791771803288, 79303.08824291547, 45331.94180435486, 91282.16098245785, 68237.19807159757, 58089.040706092506, 6904.302704222032, 83275.61247854582, 33131.73441368492, 49751.37592644282, 48969.46814847036, 33856.72544213354, 8278.96440856487, 53050.95574099663, 21742.556060711016, 30824.42523449439, 84818.82833125924, 34107.83948119427, 39850.28227266775, 18000.92187517328, 50059.24113640101, 53085.9628345295, 29158.1593319159, 57668.08417458727, 20994.380582126658, 38749.390372336886, 12451.817270321497, 10915.105405064252, 88972.37664337676, 62022.74711341501, 82982.09092797409, 93705.29637612935, 35079.927334092106, 87293.49823918611, 1456.773356393337, 37438.94170628153, 78165.99534083952, 86494.26856723442, 29703.15265028176, 8815.893151431064, 94421.70088199576, 40013.4617129082, 25165.72776648848, 14723.169915873024, 70157.32007444401, 91530.01146740658, 31960.203815650966, 26198.88841945506, 67823.37140503104, 88806.40451364109, 36286.58966229676, 58482.889383985705, 10959.313101951851, 26803.009188838478, 26185.774288745477, 40560.06877723093, 38547.24028300045, 20942.161386551415, 87691.1385361561, 41362.20660146936, 86688.78963289277, 93032.93523365122, 61544.81903562533, 86786.58393050786]} +{"id": 1195, "vector": [49274.42377981043, 84727.16005981348, 81890.21090041166, 25225.838507784927, 83066.16620157138, 99728.53193952166, 48737.55621257715, 42398.91878513697, 38360.91235051939, 68989.1846272638, 74012.2448228811, 77081.92746954276, 70367.48596400987, 72329.40919658725, 54908.501360738584, 20795.51825751338, 36350.96837471109, 52784.97895466809, 8266.62682783188, 66708.54861391202, 41060.71669528487, 11200.583230024908, 4716.42212346759, 20210.4713845765, 18547.546902786584, 53016.72940284213, 64802.82726776183, 33983.35463171005, 97725.41015305548, 9257.45492899538, 13523.818422020817, 67762.42798696345, 27576.04716653135, 74247.99641264073, 63193.88061427127, 63661.094452902835, 15259.660678147657, 94919.98651567414, 38863.22992244744, 15632.111744523181, 31136.07011404409, 88906.38276826442, 70834.25301372212, 84098.53389253287, 88838.27878433383, 50964.496021560655, 26944.433025998183, 60259.82068337835, 49883.96582625598, 36206.798651602316, 81915.02846826443, 29160.31427085174, 47429.704493975165, 60632.57879017231, 82478.29203729064, 95335.03662409818, 92723.38126698518, 53337.42778035675, 85050.88906253576, 57657.05504950047, 28150.63543752501, 9819.274985756876, 50352.84527069022, 49303.509593986535, 57914.94433692154, 57037.4987291285, 32560.561616619256, 23360.234721470508, 67760.66920555444, 94237.29744571836, 75745.59288387813, 7087.202434195871, 56479.976714842895, 30213.787676163818, 36596.05468107235, 59399.201250744336, 5455.545840041931, 96750.66682615574, 32268.401657052404, 68235.47290704557, 619.052365107553, 83884.87829322154, 53667.392653311574, 47174.6633489191, 9365.235269124783, 23851.353758727535, 50915.62608482704, 33023.26339512317, 87700.68449420518, 35671.03364808283, 25365.06017145701, 99240.99405864808, 34131.21310378479, 60107.01417015082, 90256.41698157086, 70041.4809995131, 13485.756932206694, 69385.97747959706, 75669.73521879145, 14152.855358580297, 67303.84389170945, 9470.682716094236, 76378.09463186181, 25983.003132912752, 82893.78147966172, 58631.67528085694, 23509.90937433858, 12011.240008376977, 32475.61713234094, 19385.03696769457, 80040.49661275423, 50324.490655714435, 84367.07529805762, 39064.494212873105, 6561.790276167978, 79229.05300173498, 22.125538172212966, 57692.598385764606, 98649.81067719244, 35486.22175368621, 50521.228148182396, 45875.26248217203, 99591.5378020553, 72293.77796154986, 12366.606090885645, 3796.899078564042, 89420.55610275039, 94171.21985625527]} +{"id": 152, "vector": [70610.83448294688, 16135.104532117084, 47134.395033901965, 9289.514264365727, 64168.603531639434, 49908.02510416331, 46336.07585855435, 75947.66942785658, 72043.89666904026, 21459.823955304902, 55084.18102589304, 73720.00936624451, 39959.3002616781, 24165.00558090856, 269.5324343181427, 51642.056006398496, 8174.986658980732, 66230.94426162235, 49406.74627916859, 42302.353466471286, 70255.78155589459, 67363.03085006567, 95941.63235917171, 77746.24048892599, 67936.34740781586, 39777.16114342652, 38557.674819967404, 29217.735390854283, 91482.70204647302, 91959.80758060793, 54511.10580505338, 85529.53668058917, 79075.22047888784, 25713.810954960823, 82423.01995609017, 63677.64416765209, 25438.952717531727, 82673.54954588915, 17255.084439533275, 1711.3615074484057, 60255.887431756215, 31608.623077883458, 37532.26955040525, 92215.50445905887, 90157.44129507482, 33354.94286097657, 65718.30104728589, 28329.22636403915, 33023.77002367849, 20651.322084462554, 81743.59551081523, 89205.92416682077, 89530.08170369537, 76434.83462420963, 80287.25271107403, 57019.04493532265, 34149.45855619456, 23702.962183835796, 95851.83054008374, 35874.71940372453, 57418.80997005928, 91711.22752088722, 51649.49096740405, 42375.152114401826, 9653.402928671174, 93720.322525967, 10519.809400222568, 79196.02368507402, 82554.77081180818, 12909.806260726331, 63346.90119090344, 28139.089238840897, 69383.44529231956, 74466.44409825353, 87111.64140354638, 14492.562720694068, 34877.31031528534, 68175.84948026722, 18810.966964093634, 39060.45095011412, 42489.26473442411, 67313.77889274785, 17215.59073151262, 74092.86064652604, 63510.56757876052, 92097.78442787136, 3762.516078666267, 86889.43733379593, 57772.90877428817, 44316.66931276063, 80425.68839640894, 53195.309642335706, 78669.6939874394, 88685.12797078645, 66650.7042703992, 11568.577452020967, 45802.677034810804, 37817.75765812907, 96386.36868374344, 30195.177260488435, 82411.02195829042, 77621.06049634976, 6071.679331883628, 51206.21366573955, 1609.706847703296, 19823.76395227531, 56033.77582386867, 34714.93643483692, 94538.21733154614, 22115.802555563114, 88037.0184022597, 39349.60157887352, 83683.39575293304, 33179.35011130623, 92690.22115782666, 23059.22359081385, 84141.86200261772, 88533.94097308924, 25517.16677474456, 73045.23003607019, 18802.7812044547, 84463.07213929691, 81703.7918745988, 30121.037748308998, 25944.339242011836, 32112.962904936227, 78902.83345589013, 66486.3554713923]} +{"id": 945, "vector": [11998.543639623516, 22286.330019859422, 33750.627456346185, 79747.75367301979, 15091.968864224436, 87865.26961073995, 5464.401913002414, 69257.42305876306, 8494.698925252585, 51724.078147968314, 52700.55125660338, 58921.74775892674, 78803.68633027106, 8067.163606856875, 81683.949919794, 98473.2837934674, 57362.78017028658, 83246.7119992194, 87131.3335102701, 75563.91358902995, 68167.05053507623, 12826.848070396501, 96534.33508702244, 61054.97528325039, 79186.76593034806, 78892.26020737186, 47106.91529635738, 69548.18114493146, 97282.26352030627, 31000.23154421301, 50268.962424554884, 31948.638925156934, 11919.1916954406, 99756.96612008542, 76507.6156093163, 63327.36914548453, 6334.068322472808, 4995.100567932531, 42235.63910461132, 20219.915562080172, 71253.91709597722, 60215.06574787095, 43138.88116542759, 26948.81353895239, 32909.81404730587, 91475.17787321087, 38968.723180347806, 23167.023966147215, 4499.085894727561, 7840.421916494877, 5969.891432924756, 74511.35571758832, 80527.07522904876, 35012.54795373555, 16153.740698112162, 16941.73375555399, 70263.51429122183, 18607.168951844033, 59724.60561874916, 75168.7817872172, 68289.23605895463, 17667.14715207215, 69606.81958485527, 94098.20119634135, 24589.914041903725, 86784.5519927774, 21478.923071605717, 88792.94971484765, 19047.02776086199, 72849.67963139254, 96490.80876099916, 37987.24532768991, 8296.251210123906, 37023.31336719198, 34436.87411460294, 81055.2318836326, 63129.43584123843, 6293.207611020324, 55690.55501150655, 54560.21950779933, 84240.87021624258, 62723.62240757756, 31940.994131993615, 96926.4135617417, 55546.87099592226, 79637.88140662773, 18028.05073037227, 6668.318661382289, 71484.62591900022, 35629.37903519954, 24034.22489749718, 16609.172605725074, 69897.18947731036, 92469.05283315622, 81184.53623828657, 22619.539231841958, 55269.6485381083, 7291.635361130367, 78168.66776620074, 82433.53078617813, 87205.17548693578, 75058.49253261415, 30976.449855421462, 88613.97351618846, 90031.58921844202, 1208.2581439137764, 33831.26821213409, 9250.250863623965, 71305.32634653238, 27218.735418807617, 2575.325436454634, 50328.61828218847, 41790.084362511916, 17186.1287041863, 1653.8638535802174, 71444.51922624274, 36864.23570917244, 82566.43992342944, 54101.37056380643, 32512.622213853858, 89825.95118857603, 58053.10713023893, 24912.437514703455, 42603.60257211315, 45419.28126691859, 31845.122776778946, 53148.91424976399, 91973.71113004076]} +{"id": 1201, "vector": [96917.26860625354, 25173.229136029495, 14494.795682498041, 63491.42128171193, 34866.17357177195, 71412.28254224769, 59595.824251892904, 48043.50692477313, 10022.592737679992, 32495.466957352102, 37778.0088005534, 10438.470822569312, 24113.016337066838, 45380.19960088096, 33283.2099442837, 69242.02485547429, 38336.644201964176, 7314.00957759859, 52731.94758945181, 58126.6190050794, 28251.892688381588, 49225.562585679225, 50644.85452805491, 39629.94961637519, 4258.115649035732, 67751.83347118282, 82884.51396112119, 67887.30722867878, 75130.72911716161, 12461.663913426291, 20987.82882454957, 41123.282787563585, 42851.81182984023, 44413.46497005263, 85751.64235657819, 23512.883097723647, 29565.776747853823, 67505.68030639162, 16222.15827484872, 84996.62597522877, 2699.341260868882, 23019.207228427385, 5760.4563176396505, 82112.21889145434, 88902.23390391695, 55747.325750923985, 6309.0626826065145, 97287.67467243124, 16005.029611839072, 96473.58873962687, 13537.69914182329, 33169.82368972562, 87674.86309701669, 69653.51778555893, 85868.62396722392, 44301.80073265382, 67997.85601855189, 9346.92612084278, 59739.44377066706, 67544.86071213862, 89364.68402446673, 86552.64634253566, 3501.5844411314333, 75746.803357604, 70280.29278328137, 88256.59050285566, 70917.54917382648, 3712.962532048414, 83349.81190409104, 91524.47953461553, 26785.274242246127, 93097.42490574726, 59632.89911435018, 95084.71621803757, 67120.91292141931, 88627.4994932646, 6834.73691364741, 61888.35699092198, 30510.2555119089, 61180.16842513383, 79478.12312207097, 71457.40042180712, 61314.52663492233, 9277.486469402675, 6667.812718016963, 32426.30897918357, 28374.02575981537, 34101.21408259284, 69725.24095290912, 4214.543147535421, 8568.86902688132, 17618.839096866355, 34536.75653396407, 82665.08351778412, 50591.56280714895, 70844.35103034243, 95960.12821008376, 30862.15319064769, 89962.21101599943, 16417.61527564035, 78657.55979122565, 5300.6927660890215, 31846.78223675631, 92204.64041471633, 75325.24530378, 45461.56882753808, 92577.09228387236, 25530.023264688796, 3050.5570560329766, 8007.748654179903, 26515.018176370377, 41722.57523689311, 18574.7812227847, 7089.429908543166, 78142.24002303682, 9849.362066065914, 98201.33551780452, 54008.84962650795, 56624.787146372226, 91334.63164791146, 10696.554954314652, 79409.34606227594, 13109.577849806054, 3647.8719004159334, 44565.28052770119, 83946.75610711396, 75820.68528389146, 56959.80615544262]} +{"id": 595, "vector": [54987.00754318382, 82871.92461800067, 49605.678274782236, 9472.634492426312, 35187.49661641139, 72035.10095936072, 24747.491405660872, 12735.911724504367, 33681.88859218346, 83165.88048060235, 84968.47516143351, 31206.698296107337, 22846.36894550015, 54687.342083137846, 46238.760552323496, 89810.95406283965, 8708.042497000468, 57223.19110474611, 60582.15608975188, 48946.855969661665, 29479.56863128386, 54249.01684889119, 22140.505033038382, 25360.026853655327, 75487.47105184536, 68310.73854612181, 9887.38115329495, 40364.86265702049, 26254.677290404583, 8786.875236023916, 43034.72674142401, 22.281642670918256, 25782.557232501134, 57515.11422670273, 6800.549746880513, 883.5218035045389, 70525.96164710882, 17092.487999063567, 67421.85290198457, 71204.86449880172, 58160.76330091292, 56679.8408379111, 18987.928793526764, 26130.782540729568, 45953.75612950858, 62465.0979250269, 15370.124717124334, 915.832109945991, 19377.560289418703, 4661.251203050687, 62386.52140126512, 37372.61906518014, 26690.127397317032, 83844.40963230775, 40327.397666010125, 13826.54096963245, 1936.4941399795277, 18298.199729593944, 58488.973976144276, 46097.53100449227, 21055.281221281908, 1727.7352125562695, 40279.203446562664, 69343.49121783336, 86704.07959378486, 96617.52330949728, 31894.63386347209, 16000.051429474204, 74452.2481717666, 44590.66945605615, 64371.00233780134, 7399.499292082901, 30480.355803769522, 42394.89904661671, 37355.30980423716, 3089.271440452013, 39391.13307697488, 32454.948985461153, 62274.08155687998, 23623.56423805555, 88399.94020357713, 82448.70028938357, 88215.10493799533, 42560.16810586161, 66978.84931330004, 22571.839104286904, 88735.2252947245, 90024.37616511736, 58794.24155651253, 91481.23755625525, 30008.47810671722, 86180.95673151784, 58164.21235176786, 20581.99207133796, 59857.10133322434, 5036.300701402263, 80678.781459318, 75464.4278789615, 13060.486412644135, 7282.258397866459, 20682.092174734345, 35520.73028403829, 1265.8716247779146, 64416.23933273544, 4227.893614726763, 28581.40513877878, 30488.999666334916, 67382.88115853637, 70626.5070587425, 67538.11651203102, 88531.61651218733, 82267.5154065979, 84700.1999435565, 59473.25295558352, 36255.81509573325, 28702.270738757918, 78803.9381504054, 95963.30622184227, 76637.09830556126, 81532.38358416189, 72360.98068992565, 75570.62608364939, 81.21231889921532, 99011.95113553392, 84789.27106916675, 28091.645304195146, 94629.42397050952, 41275.64679192223]} +{"id": 1728, "vector": [189.06667470079964, 54899.77068793492, 22006.625660501268, 81292.42325746673, 9260.261428020322, 87308.24753067165, 91562.41483904788, 55245.65896096838, 95036.84347914757, 3660.64024580679, 22115.749933803807, 632.9191916137233, 20897.631129643447, 30275.368630008736, 24316.074989297253, 29590.999939117235, 16228.039725208177, 55098.61052764299, 90150.11384344086, 54358.063000914466, 52950.306458708044, 59783.37446432027, 97772.87585438469, 87734.7190467182, 34702.53698796036, 52127.41253303255, 66011.75100932892, 92008.2877707952, 33742.2129448361, 77130.29016496203, 47869.432036855775, 7073.610268937791, 63138.16480426322, 29442.070791696806, 65713.67781186024, 32695.284969277338, 1857.7002620396631, 16387.29042724133, 62943.06181364625, 46718.767463505574, 53188.55252416522, 62585.734730438446, 73921.22069352513, 31829.086113799087, 51094.01164450836, 66594.46186356162, 3291.9116745708134, 97365.27897421543, 44653.77125217407, 95432.23847722585, 40825.53616438761, 57611.31747652849, 97663.57906845167, 72247.29060372953, 25756.4355539355, 93656.98803581663, 6789.207100358197, 61979.50860171356, 62137.71983572003, 55429.410290246014, 80515.04046943893, 16436.98042208852, 7450.27020256539, 2726.182355239204, 79809.20089245484, 83122.44537822947, 73563.04300103644, 1905.6558553118164, 37767.5383131819, 53413.6545208864, 88081.74586516115, 48307.90877350638, 90144.69976768077, 57932.48766110573, 72549.10729877306, 83529.14328057204, 14523.084710798339, 15678.66037405493, 22890.472678327824, 50680.55369136921, 3509.9668964714724, 35001.861473518016, 74890.32888400136, 30073.064939094085, 29048.941930856752, 11436.693047602654, 76182.61162159589, 56493.975664397534, 15918.265391160647, 17996.91472686168, 57620.54692811895, 77568.2831114235, 68595.34978547494, 5889.032349503964, 98999.59625141554, 69427.26724220367, 30198.521431614034, 90133.08129358267, 19375.531273363176, 73741.3711132297, 92616.24319079921, 88844.91198852196, 9503.79966784296, 88477.21140432057, 33743.866097254984, 17694.200279414352, 47440.871012536976, 9115.95341736, 51045.24733899283, 10722.261621687157, 76472.45968792989, 42053.59406181231, 26354.281650086585, 52551.27143543826, 66041.50611160761, 28059.02662342753, 34196.65701838083, 64095.286570128905, 49424.413150028035, 21984.6393939382, 52742.01138019015, 10472.332877387413, 26395.78291897371, 40235.471595472874, 66875.14516074631, 89107.62425618416, 39597.73890187286, 6176.618071723828]} +{"id": 193, "vector": [85920.2904755355, 81349.3181250909, 5199.452524462767, 30345.753885231796, 96639.732116179, 90513.11317216304, 67278.85317265819, 12178.737468366086, 39027.35266333963, 95779.80060378926, 72209.97634587898, 9224.097188441172, 76090.2592766176, 91213.39932630098, 63779.20530157005, 11854.376030160096, 15270.878599704107, 56584.08204628922, 8262.274110506229, 71424.97344415465, 29316.63487945412, 82124.60730378762, 98405.92051637918, 11397.932565996916, 55310.96624909252, 12352.24776979772, 66821.12749822992, 49589.363209592266, 81467.21945221299, 53508.87269704304, 20658.652065550177, 17023.49100700857, 44360.87828344644, 72121.10927246456, 20687.380055433525, 65377.73226064764, 8097.724138313544, 38136.87828177526, 69593.40866487312, 75541.33346060055, 14885.472595401417, 66267.62626451901, 59020.01881809977, 29494.30656327675, 60724.57330702604, 49386.9364230299, 86888.90600742082, 79113.99589795266, 82973.81465134941, 99134.64315057479, 50986.854864482215, 19305.730850180324, 41534.22377947846, 35389.0255439704, 26325.188830199142, 8035.973582449185, 22839.223850774117, 67428.75950763903, 76759.75668959874, 4330.99984031482, 75610.46742646201, 64928.12031589258, 2488.029129037572, 45545.18492395264, 31357.17586773341, 8675.037263833885, 3589.0050535599903, 82037.62941190641, 43647.32645598343, 80355.58629246172, 34242.62339387366, 22222.513656477495, 24151.70227088932, 25911.301308884493, 97344.76898434706, 14798.214233768926, 25886.03715140313, 8506.10620212443, 34165.36680671659, 93122.25245049523, 23908.83527154256, 56588.25011879021, 45165.410073516556, 62157.78265277209, 24279.677524279065, 52005.70879579124, 72500.33532995956, 19533.821328030932, 54510.416538030426, 32750.045722787723, 18068.07079217787, 71021.74681267136, 33550.99275808725, 18207.941035450258, 85703.11666449656, 22624.469472346755, 67612.47030448512, 63857.05657922793, 96257.1273525872, 86233.69369198094, 33470.68497406599, 91288.41446418311, 72311.00384605957, 10836.327895231301, 14351.835775262334, 89043.88684786491, 83587.82280735523, 81236.52855721049, 8698.148714196552, 55208.71518104296, 93004.48299364686, 61725.28485566819, 31997.576776936377, 21455.757939339946, 12278.805270019078, 76183.61114893593, 80206.17007029767, 23905.072195736644, 79142.20798002675, 22054.96095901346, 94267.32456592562, 63257.65145963188, 14243.850505281975, 62699.50245962914, 86769.26942975621, 32472.80111963706, 12520.030000272342, 59558.82589138747]} +{"id": 478, "vector": [35856.38167284935, 93660.03898026451, 44661.5633069831, 74124.55336235889, 10067.967645399478, 86975.4211213013, 5379.988937721958, 8210.045500670305, 34402.023850358666, 58192.32180216344, 85526.23873603258, 59292.34498696053, 74073.78904872182, 8505.467886497687, 34215.53220168866, 36752.07493930571, 19659.909926313623, 40691.969106189405, 35148.5676440837, 45449.95290768328, 25988.261430557868, 67185.14496637158, 64856.19037891943, 42164.65152618808, 63038.7870986305, 35765.65048197138, 35948.08799097623, 20857.169593028924, 82423.61974277394, 31770.3775952327, 43946.760917016625, 67129.47325064518, 41681.9970321141, 63316.305731322755, 53039.37341938073, 21561.28121029577, 82913.61516178121, 7026.366566638831, 17696.1122827628, 79842.76720381652, 61294.39798822193, 37636.7633681282, 36725.10756821645, 23592.443503085226, 22570.990627707066, 7945.060433378281, 93870.1208692562, 62131.00531858149, 74561.49385117255, 52333.77777264024, 94492.13192171056, 13223.527653441657, 26749.37160627794, 81914.41514136698, 46169.54222573282, 89269.93784966499, 31027.28999580413, 67322.8634511672, 86789.2251261271, 98337.48720379824, 65342.794522006196, 17860.15419820217, 68276.24440172306, 37648.11146490721, 23795.41879603946, 88685.3775572644, 2739.787448651765, 40619.50660624841, 79787.16547917335, 26128.02493799453, 87964.94304303428, 51253.83230917973, 63943.761278429614, 96092.75298784567, 29744.30896316884, 39187.04294128369, 28351.817678245396, 34511.2054181202, 34451.369261577114, 80551.11569366818, 47301.93992347133, 52791.40136619916, 24633.547204534854, 21512.860922693864, 51418.07954786637, 98679.52797354358, 67034.34363814717, 52376.88874706674, 60424.07285574969, 22526.49797890116, 76149.97405245645, 49828.12569888796, 66807.60558640976, 24669.442675906605, 68242.00218525331, 86922.31486024344, 99794.05388585232, 52682.15858308073, 2951.589529472476, 25588.24688370883, 30696.170864338135, 24272.14615357739, 23402.755627780625, 60077.018898287904, 12873.076816170347, 40165.391881662836, 97538.997771301, 99806.89517837054, 50663.52152861285, 68438.92910788865, 12555.638411768023, 36832.46606251249, 32659.28312971128, 55047.30041799751, 30329.681628118255, 26147.8861413048, 99207.09841425372, 33030.26739145133, 40353.87822124636, 47033.156469197966, 35862.40165530788, 6605.7592587061345, 90233.91099262261, 65617.67838259813, 82355.98211328262, 34311.384758409644, 37933.7189272263, 75829.73864023823]} +{"id": 568, "vector": [77595.58332513584, 23323.501490704148, 7276.843761890151, 51007.05687106305, 19927.137170967955, 26543.54035285451, 43045.34396299947, 40608.34445053596, 6373.596306823348, 57826.82112263339, 32700.7402864528, 14352.7110764194, 25025.03888457792, 15256.271555584988, 54538.32077237025, 65625.97806047784, 46828.082593731735, 44231.721157689506, 39954.84188328638, 9898.772526384148, 33258.6499686616, 79313.18282330439, 3889.033418650145, 4338.62305155841, 93569.96680673941, 45812.85099744521, 81295.01982324292, 41716.788180913645, 74781.99362165123, 98386.25296998942, 76837.15019169186, 45655.660923725074, 19532.141242772715, 13299.944889871218, 24771.77470344324, 73496.21115014974, 21247.878566756728, 20332.157379488668, 14832.567965770704, 29120.924127751612, 87618.85117241475, 2250.3843757078366, 81843.32269611562, 16368.65972346494, 32223.082378921885, 4476.336443473961, 35636.458740167545, 65372.45399503302, 84216.4072204354, 79483.14323895781, 89336.53394762364, 96255.53645251584, 40902.03114872062, 60179.60864077999, 79897.28331725571, 91261.02695411263, 28139.869376705363, 65831.10675603802, 73069.35058397253, 32316.01094064821, 20595.98621818092, 33737.72803715813, 37692.1281908718, 83306.10985381008, 40356.51776554771, 74763.74920216123, 36870.17421457937, 20368.37716432356, 84212.4804155681, 17378.93306848134, 46266.556330971056, 48054.60659566466, 80610.78788077118, 99015.58561872705, 35697.278601221784, 2128.2462195858943, 18993.659217173932, 8970.649362757033, 79063.55777998418, 44866.92400537368, 65757.71064742746, 96515.33683024913, 18804.150771951732, 74395.1220968284, 95470.37502793287, 31112.7104758145, 4018.8740591610594, 33747.549595091856, 16766.83394332177, 30288.82280923303, 41205.79164498225, 39038.79903123215, 35562.57052073515, 8930.425888382442, 13551.065786366767, 57435.67986188766, 53258.411489310296, 96214.02977274507, 87194.10198596986, 12304.036899410741, 21991.701202408574, 31628.561513438835, 76485.09285142159, 10650.465465255855, 64355.1360363538, 18148.516071839636, 47536.04802416471, 89685.44879346083, 21286.534737416652, 8399.886920079647, 41531.2256981217, 47580.732485387925, 88281.99518308837, 27840.951749226937, 29617.643107016935, 56583.72854279111, 99879.01347440259, 30916.404299723286, 43543.13035128048, 19938.747224223807, 40662.19423093861, 98619.46197793675, 73462.35242648164, 93554.89587757322, 63362.0139127471, 88158.17740358783, 3148.777051716167, 9558.69147851518]} +{"id": 761, "vector": [28008.97908855493, 40993.712761092946, 51844.06835646817, 75987.95721027355, 71424.21001245544, 72035.10644636946, 43169.133009077144, 1421.1688079367013, 60281.30852196612, 90512.15554370792, 67574.55287971403, 62310.7108192845, 7419.6061724230985, 28412.465415644063, 9123.472935821863, 85751.49310033933, 58632.8984663633, 23276.041496921684, 88130.18435074469, 8259.265983471998, 28252.02775572805, 8092.569818071404, 54228.08712515199, 51755.16349774698, 58822.25968340565, 25166.367023677616, 40480.45722135972, 74310.49522818536, 41972.77444799242, 49510.849701948224, 57296.25591898872, 70208.06826639964, 16836.777590392794, 90278.19054619082, 12517.85272472099, 49142.64925001193, 97354.2482962416, 15321.211247759058, 34741.89122108383, 98055.8089545362, 63259.21730909799, 35909.30617273194, 88821.68031611727, 91206.1387567206, 17098.919531767242, 9808.714946827491, 84844.38449542402, 14896.61921929607, 75619.40275966359, 99848.0980485903, 26978.29228605295, 63809.197183998614, 31290.62117728386, 10482.972465320705, 11476.449868321548, 17382.492376646784, 4071.086234947374, 47699.82227122309, 94333.42463241304, 65202.03234024737, 62674.24193586907, 30716.69104707648, 75108.86886464508, 67176.36765061223, 61478.07806113823, 81975.44976053057, 1822.6650160813485, 5134.093318100663, 26927.371080489727, 1108.4446921341696, 16714.996936544456, 29135.521648159625, 65496.08964774809, 16164.987822325316, 34366.998708871855, 94264.76970856624, 96692.7009113378, 52392.70730160793, 20897.906840150194, 25190.50727520433, 2945.1369780379655, 72764.31799741094, 28284.40515741096, 37237.29440663374, 72038.50630246536, 54743.22716015955, 33500.20421196529, 20727.33098431354, 99208.12221487235, 65243.877581355424, 6355.869937856928, 33994.20683235348, 89154.7621787709, 93007.63146788478, 44122.91175088993, 18723.523845663116, 64977.96013145238, 9308.372896599692, 61485.42423553402, 76856.88935543352, 27910.04423179345, 36158.461090977034, 88228.51076824873, 99280.63341413344, 51299.84819829008, 48766.90081910986, 56084.42014396272, 40879.063537436785, 10682.447876282296, 38283.99766062187, 95855.08809617437, 27231.627027099235, 80705.2202724843, 1792.555588176359, 26589.138650588462, 32276.811779661162, 12780.678994936412, 822.227203944903, 69258.67721865133, 24284.642650189548, 99658.93019752235, 6875.63374611353, 5111.639821422132, 17171.672000678074, 23089.10771496544, 95954.32513991802, 49970.60603748928, 45397.987226296944]} +{"id": 1950, "vector": [97877.01287240381, 64892.70964940772, 65236.83563027874, 64566.50271315879, 96487.44290401257, 42709.67446773451, 57084.76694213677, 3786.4128245913007, 74213.12663733594, 58355.43763769251, 71294.73589496363, 84810.46495824987, 2071.488918537112, 27676.558453840316, 23310.379642777832, 43547.29773906677, 13445.046844734354, 56281.07154281381, 24628.83107920919, 85809.40053984629, 60305.37689149066, 40257.249156629005, 98937.44474229858, 64271.59531525367, 91146.1361000056, 73517.93208133533, 58082.43146969791, 51839.80198415794, 21148.286699452, 62885.27597617283, 16682.105407114555, 16042.028915180606, 13137.83093841755, 83305.49354655486, 72395.95959612788, 96912.84831111274, 88489.97788318868, 43321.00715076548, 72762.9414735261, 11449.50752685876, 94732.53230591207, 58150.256556381384, 53147.799059034616, 59846.97234396495, 72858.95015898588, 51196.79663737518, 81864.25670063215, 20535.683889795386, 56640.114109493945, 42603.2072358318, 78326.25778079177, 54449.77742234183, 72284.78772652059, 80492.95820978314, 26545.31173480751, 64676.87794444546, 43791.18650815327, 72362.26786485213, 96058.82293451492, 32723.842306445094, 69055.8308851362, 41267.7951503174, 30290.875723195255, 54598.94917304353, 85852.64892552554, 1176.4635517908407, 7882.443131679351, 18898.74948326906, 11843.822278911286, 32187.896862777965, 36312.396550387086, 84244.66104297971, 83124.52130309484, 59469.37557987372, 38115.138621360835, 8888.341874781081, 33066.21132780267, 2612.6675471280782, 84797.8985692528, 35121.06025240499, 33099.40683615193, 7218.237707825626, 91675.55433702018, 31440.627943322517, 6008.579102503442, 93398.31096635264, 86500.0856559294, 54377.69770921446, 52758.01391925918, 99378.79716538414, 31899.27551167837, 14797.55361011027, 62044.004870801575, 80565.13519182635, 54861.7423759468, 80039.65057248484, 83844.79929820994, 56566.65048788782, 90283.88259371463, 18715.937711787632, 14435.122307518577, 68101.71554757371, 29599.28070621157, 39152.878693860235, 67397.62385706068, 69577.18521089568, 32858.77288441341, 48975.5287901605, 81038.26209908408, 24484.06326002248, 14740.597180784087, 91477.24600467939, 48805.194306687095, 78719.17255091485, 39255.39308999735, 77986.71058748916, 94035.15164653149, 52104.579331335044, 99255.64336156678, 75570.66068692907, 64610.984205771514, 94610.77536701298, 97171.48003973294, 71285.22751752629, 19376.976433577252, 37182.316328898036, 35231.751194456054, 17484.989618078052]} +{"id": 901, "vector": [45122.66841926328, 7930.29186207479, 72306.06193715661, 80693.67524198574, 18219.504254150954, 78725.58966281856, 84523.6920927372, 82097.46409712141, 60551.9672225386, 63459.24719134721, 75441.20306926958, 30619.006282617156, 59527.72978175941, 75040.23031965394, 4402.99858477109, 87109.0450593376, 65378.640772163075, 6569.781467586011, 28016.32023304895, 76892.86430602432, 65695.51677174396, 19311.329920252574, 59868.77190973097, 99344.59855453044, 3726.738936815044, 34382.19686415001, 64616.363070294836, 15123.518754016695, 50923.104854172176, 94708.5342727183, 69146.86070251638, 23752.106905124238, 59646.060246606656, 60193.089022160515, 56167.85624828623, 18348.453496231, 86296.5653921785, 53687.87368349672, 86245.02019889042, 12683.868152942823, 11278.83282618093, 238.1324285997244, 79718.36713743416, 18446.32517038941, 42269.50526825314, 19124.814072967, 5604.905436279295, 64753.07970474037, 37863.015115487906, 50336.15681955331, 36634.46803716076, 24074.48645908714, 49770.13350095044, 97275.2578490065, 92542.7843594047, 91243.2821636799, 7061.251667847768, 15368.424038674144, 92060.73485963487, 1367.6939003784748, 11359.368536385939, 3656.1196078572334, 48984.58359961557, 37245.12749527886, 53319.701067779126, 72095.84202754227, 54856.32931898889, 52892.02439846099, 26440.948798278107, 3651.121831454063, 526.3679047194158, 21676.809825044696, 76330.4447557816, 81742.35522236254, 6987.892772884763, 61100.32804474832, 7838.553603227949, 28756.344017675663, 64622.76155325651, 57205.685337283554, 17839.433061270236, 96880.00344304976, 21035.02859139086, 7970.155469733986, 88488.48267306694, 57489.2742001906, 87144.82706136964, 64614.45232948594, 65857.43908992455, 5451.104736697421, 7659.496699471213, 1898.6763264413619, 44912.54547553659, 61524.37434431035, 70969.04654488486, 19456.887549531064, 69029.07562513453, 33002.35374100054, 80573.83124104417, 13875.471285854146, 61288.21048834233, 81227.84750295771, 56484.82838232785, 30430.79494920995, 77912.83686479788, 59368.05456258514, 78814.14435602016, 33217.19023428642, 17026.335654605384, 88028.94683786898, 26524.92086923841, 38244.52998485022, 7182.39174318267, 28853.533984484202, 95108.05456478351, 19017.10228133222, 78917.36754110536, 85513.45999608959, 70590.40581370762, 52775.663483756776, 38087.2091420225, 49826.836553203815, 98919.11931685108, 80732.27874150984, 15543.105023206139, 78271.85226920625, 8395.66965260119, 89499.83798827123]} +{"id": 910, "vector": [82731.67549590416, 40186.5223605896, 63665.65519552027, 95811.08159957948, 23263.116140997576, 36291.94226850326, 1464.8173551017662, 9020.189296625269, 99675.68694791231, 96222.39240662569, 65451.16278398707, 65242.21388749506, 76171.54538470217, 89258.67534397056, 21981.87097615314, 26224.900135421238, 36989.976946355084, 25327.320661370002, 30511.257347225473, 36087.988363772114, 16886.255437016294, 34193.9266095127, 53330.66916793806, 14544.813581130411, 22645.688305392363, 85708.7768059555, 9282.624131466833, 75022.51749858794, 66970.28862931233, 27575.864335741473, 39065.095778387295, 72376.39235078356, 43925.2622566861, 28098.647685079803, 12668.320410362654, 34130.153135584485, 51876.00558221619, 64102.17447179847, 88328.8158740428, 3302.282050360561, 46190.05618370623, 68995.44796328014, 88034.62698650012, 30780.204118038735, 21561.53870930547, 16040.928851956449, 6737.166059722655, 31920.9540842471, 3463.473454732291, 80891.5567421857, 86367.80532400122, 54221.986366361285, 47280.6106497696, 631.7132813999926, 94374.40457642652, 58216.75217368779, 56072.685418922774, 40802.00957776361, 19175.168807184262, 29247.85815260027, 4803.508647458277, 55257.33729016744, 6448.926806348009, 35850.46973349428, 60054.66637156615, 64579.917094995864, 64179.54700501604, 56698.02460617819, 98298.09175430867, 33202.993694173354, 22462.930136385028, 60361.80224803792, 56431.26822194977, 13125.279463823303, 69994.77667955511, 45425.84304851087, 84761.56357912699, 43100.52792447542, 91506.44236592918, 37123.616278968344, 8325.640318473448, 64249.15411098722, 83492.8667105859, 33904.09618942964, 69052.57171774056, 8749.358052030831, 34294.09304461043, 62486.86706998103, 59430.546894860636, 53140.84272392322, 17908.495432285064, 63542.66040563177, 83216.86254158533, 48579.7084114801, 89781.26065555852, 6216.161846044343, 1600.0783117219419, 63870.97807773955, 64775.60134714716, 28865.194276136142, 24790.59782887809, 12831.045880560177, 84255.35060432045, 8296.150816555304, 4272.352884659625, 7247.734456162025, 49859.793985224474, 36003.137926240204, 75094.52222705718, 90608.96415378469, 56207.10348666084, 79947.80394926717, 68405.15161797374, 13212.869985376496, 18633.774048512787, 43691.25085718695, 15827.083938566244, 97352.8672981267, 11269.9730638386, 1546.906315947827, 71593.28669028461, 63082.40244521775, 39090.79201162993, 4688.376462582167, 81130.28472675414, 49976.52941224171, 98565.0344947036, 78598.63993578203]} +{"id": 1576, "vector": [94973.10341603919, 30097.997701169566, 43187.94301815779, 63012.215007609164, 90720.54849432436, 76978.6837718007, 88336.05627979936, 77259.89846205314, 33439.17188407454, 81298.74744618409, 65293.709542075805, 41319.928396639494, 80126.87250684945, 22314.414931186344, 59484.32567586478, 85412.93426691159, 5699.076787699919, 32987.540309136355, 63070.817158626116, 84395.32453631371, 31005.19395481657, 78218.6230081567, 26159.05475209478, 85052.97233943192, 11642.458680322565, 78865.6549038815, 56204.97991553735, 43375.13262158461, 47657.32138451252, 99283.37447193859, 89203.21571425661, 93195.41715862197, 80850.44210519476, 47002.30415710151, 73650.86897142172, 28925.3323332875, 98024.65772912974, 85077.56620520928, 62785.92145633718, 28484.566338497196, 86934.21043265339, 24259.353554160843, 48117.63822921238, 80390.57233547972, 8411.59528517208, 68674.65951145456, 57106.25704037473, 24392.89028315772, 69321.30288685385, 91988.31334032207, 58760.735318789935, 92060.6646421386, 61319.749340662835, 26709.381316165436, 67288.51412957585, 85143.69097889781, 55733.95142340037, 60774.65131177735, 66594.41784420174, 17352.344880738692, 39178.42688154359, 38506.270395090534, 69778.2304853128, 70132.66378035118, 78884.90358333288, 8287.303739514906, 97127.91953383942, 11796.681576328372, 65557.17203633655, 96870.70274018575, 47556.3190335668, 14357.594328866264, 40818.47570028454, 65205.44999739657, 37235.871576478006, 19666.725845199573, 8968.638247757144, 21789.427789218928, 97538.13057532173, 45515.31253408203, 24606.558312495617, 44795.98021759518, 78889.67452940012, 10318.944138982155, 36668.34952934598, 71897.28812492419, 14125.277323665952, 71059.70116466055, 45320.63686199245, 32592.192129998675, 55349.44851769543, 13948.87875612112, 99730.70992010146, 10647.058882623462, 72553.0013548111, 63510.12327644048, 41863.55101428656, 28486.178639701964, 31006.35414285401, 91965.73127422557, 6234.206629756111, 48095.50299696612, 88565.55901508147, 97497.63820129196, 73467.56889136751, 22348.761140596063, 71258.31851818413, 20471.072367608467, 39991.67373704813, 52162.931329343286, 29325.87326796503, 72855.33044199583, 5331.0802882798325, 39729.89471420536, 41107.32069560694, 61044.48971285224, 72404.88091831621, 83445.30370085403, 96433.835503403, 67163.51067287092, 1208.5126776110067, 82871.99326066766, 67404.30444422754, 4143.738139870768, 7561.4273867694255, 42759.67929926453, 12745.282359820365, 98416.16717828975]} +{"id": 1997, "vector": [28942.234235511765, 56105.62928122142, 69084.92439456165, 81343.45661374407, 21780.842242468534, 20319.563099398674, 4125.327397708012, 60054.44092126414, 582.4520601029892, 20461.655238742936, 99039.55710062134, 73380.10410922905, 39680.40537882965, 10841.126789637823, 84713.41647709894, 1963.2026988001371, 96078.95796971789, 47106.04372123992, 15597.004612771792, 21886.956338917542, 76643.25645955707, 56311.08978682419, 75357.44517723312, 47104.96202537192, 69024.46817916894, 11713.092551841697, 98746.96197449979, 59051.50265153621, 47096.69703815373, 45151.975374985195, 78716.65338721467, 79794.43625709938, 82759.66691882207, 79979.16259458593, 58842.70118875752, 45657.70170970886, 34821.511552006035, 55163.94391149444, 33283.40937513277, 18376.318597720863, 30564.68419299465, 93677.34456267882, 99944.00344659365, 299.2956871868402, 73701.67835859208, 85622.82623509673, 63555.537016168295, 22785.83258385236, 64125.64164774558, 75126.77877345745, 49857.284223341856, 15605.992550868797, 20306.723846109675, 12715.185297718434, 83110.82350509605, 24421.390391233934, 5421.618325931221, 42377.101367482974, 3242.268954547278, 92518.75080310278, 94998.16090934836, 40561.250881730935, 58169.50440141512, 85197.87900960453, 58237.26962743934, 76112.79187295397, 33054.17187688187, 14805.436962923368, 58267.6232595613, 31293.261291460105, 69206.6610412754, 47483.91648152348, 95714.73704775891, 12720.32281529617, 10418.281177717581, 95623.55369807578, 76307.8553548588, 64952.29271476446, 59463.93739549199, 48572.76046678008, 1558.1014629360234, 54661.512428819704, 47972.88182661027, 80269.32622305985, 54156.72713230344, 56029.35477075387, 90032.42235473997, 62215.38603547291, 73881.68473195296, 6603.170951459181, 45850.5872396049, 53748.4508972778, 57725.54295903639, 22798.87253323187, 51256.25284519309, 21440.310524913308, 35775.241765460174, 31997.98783951523, 83350.41389710041, 59629.76175463207, 12083.051229226783, 92757.1561565752, 1790.7741119530417, 65966.85366408854, 83431.72510320564, 70528.86791243433, 38168.8555498369, 33323.44848583982, 22451.344805711284, 49613.874156538615, 88683.43314844511, 27078.948199841034, 32641.811810792842, 6750.749594125127, 91253.8530377946, 13913.620156139383, 44199.64037988606, 24290.776150287642, 24238.784336241726, 11295.461268529405, 49063.93306374119, 5739.114986354021, 14076.194716608825, 92055.64616577607, 72281.537949964, 64720.55364876528, 76323.59637481102, 89232.76243075689]} +{"id": 569, "vector": [68657.10559200695, 15242.217705284156, 87483.36948559088, 9139.455717263778, 58385.78050142356, 90388.47488992017, 59854.374819016084, 12406.393184198505, 95851.52460703443, 97669.01169925176, 40077.32729506654, 11122.505219618795, 37988.068767727666, 96601.7872972492, 47051.08119565003, 19995.890966029052, 4202.641653538908, 39126.64178773723, 20696.90859441099, 67560.24675211337, 69766.79310078168, 8033.224282818607, 82070.54516285987, 33190.710909530164, 56551.73755169944, 19218.769718734973, 2208.9571446520795, 22195.853759201058, 98180.85668987039, 45075.033340897775, 14846.440381567805, 58312.13498883696, 33315.490010975846, 66885.5745660767, 65239.624335024215, 1836.8702718098207, 289.6880025991755, 16709.699158835134, 98638.6041465571, 81613.03901209729, 74214.88263522857, 91167.76464308522, 7577.534106858652, 97672.5734917272, 90569.08656603686, 41687.35042502223, 21426.301085848077, 19416.286970775065, 85034.72809595207, 29096.386707097932, 45403.61948150822, 2705.4759704873477, 74651.3003709098, 56195.95440879276, 12279.079018310023, 99897.08349027864, 30917.495342226277, 1851.1659985871786, 31263.84969518896, 15056.771715464824, 95413.01793719822, 69092.94179559953, 51103.29467624101, 72669.63341641803, 86588.25434514027, 38066.01886318701, 82544.77508653277, 66217.30248439287, 77794.52767545106, 30191.639930441474, 68569.26083713121, 19486.906548664752, 93941.23230644772, 17785.709614229094, 58233.22095912563, 93193.3181610245, 90252.33138020989, 75493.32926123186, 54285.66922413946, 49559.93795965846, 40333.771553049206, 33869.251985561525, 80741.89442233079, 65895.34653712991, 50453.202200884065, 49800.65945445687, 40147.6920857753, 18879.38616884679, 39312.896592316705, 14391.755605729317, 90025.6739340418, 63405.08619635002, 61431.513474933505, 62000.65646555206, 34832.53969401734, 37566.17107328091, 57446.01252104793, 77529.90790723998, 68549.45351406661, 1024.6077466420256, 95456.98831722277, 78639.43627527558, 79695.69675262949, 36630.73479650686, 23689.889022056064, 75935.2765362345, 80573.45755158344, 98184.83605171347, 57053.543090228486, 42248.42306032944, 78868.6425777348, 58702.29454079223, 27530.795356778515, 51838.19331593657, 35002.90396879958, 31514.7175096532, 16985.027426848752, 1331.256099281608, 58873.355703178466, 70352.13672708055, 60032.263817635234, 8842.311961680705, 78652.65632528582, 13714.287138015934, 96782.17874823727, 73739.14767105677, 89425.3761460074, 83387.40353512413]} +{"id": 1765, "vector": [97620.12089774825, 18263.172733146195, 97957.51364014148, 94287.82058278073, 49808.04572148107, 72159.59610507412, 76855.12072689901, 89098.49229901229, 50502.02042503403, 84399.86001298283, 36116.854259066844, 8747.770784207976, 64580.42858210491, 28490.267477523124, 48739.077421605136, 2669.348342290112, 87656.59092082802, 7562.459098139873, 87072.56584241026, 16742.716293217585, 30829.30442427334, 75529.30522444473, 98766.91227242546, 8492.073539082023, 93432.88417893501, 1475.8607558686742, 26048.974142205538, 79132.13724315925, 2048.110036654749, 46167.925909381, 38306.88612502502, 216.9302708416976, 47826.63602183196, 36026.75758090678, 59579.25731529257, 80409.1543455884, 96918.22170726053, 88106.74876221555, 769.7052064108201, 49947.816523937196, 1436.9655084633725, 76393.0211216517, 53149.01772709586, 85006.78323612222, 61153.92873690205, 59813.14211632016, 32882.034522120826, 29514.409989020118, 49385.696426070346, 73321.8722689169, 49165.54336631951, 34717.71520812186, 2294.764650885306, 59573.1628541033, 40226.560699951355, 21374.129145374256, 19813.869488665503, 15937.277270051254, 41847.554532452734, 49875.5825115034, 63489.868325291754, 49469.67823570423, 89505.05072806812, 77634.53072094264, 67147.52056165025, 56318.20601408746, 6553.344608986933, 7711.4538284218, 38201.507860808335, 75196.07935559098, 48571.31835854777, 18900.150457905864, 2752.1353044628218, 42159.58561424025, 15936.159392360538, 59999.331183451606, 82331.65964579117, 46166.020710354875, 87544.10522903786, 61688.56932016511, 31569.66949878578, 83615.40262062082, 96634.03769349545, 47284.45293379726, 7929.572480530611, 84050.5538690351, 47034.00771484616, 56500.11135450366, 7899.4177099315775, 44506.96550513686, 51653.88315368677, 72481.35199025886, 626.4700200643847, 17027.12868619972, 6352.2504145663515, 91028.56511157233, 43154.37735304332, 51979.74320319606, 38421.972424191, 12166.033605102411, 25805.509340015964, 1058.1595839300517, 52886.21817819455, 9906.370472645365, 59668.38259420385, 12597.469025061304, 53104.29479292015, 22286.223583457242, 63611.24798292713, 50574.4483899481, 15680.77137668643, 59830.03624495699, 84687.96238516032, 84258.63538432072, 15443.758453489121, 3032.1001189167587, 86697.74715774813, 84684.35780987423, 35399.155217055886, 86805.73917168594, 70085.25371878031, 73421.56140499399, 56316.69231330979, 20463.78606749373, 56143.022197445534, 74769.77114361613, 33203.38234306379, 41259.101301487055]} +{"id": 492, "vector": [22535.18640096106, 59557.781540978904, 46637.639573183296, 28024.418315331368, 25115.385699079274, 9992.056690579277, 91205.21332282765, 79224.20516563625, 91952.38739224372, 2222.3532798969936, 70039.89295679792, 22040.889072593174, 45289.347292151164, 49615.38885438313, 81098.7415208594, 37803.70141431918, 34848.08583580926, 85138.01491493914, 47816.111839965146, 84134.08887015251, 54657.63319914129, 73614.19662604418, 50931.00500979461, 33669.41776555372, 57703.34995861909, 68306.0961018966, 75901.54267546801, 56249.378338167255, 16091.68222188192, 22993.86260767088, 73319.64252103507, 74058.9459272883, 94736.18789580786, 21868.46212924578, 47959.25307557807, 28299.919968823473, 76990.03527027805, 87805.74499762125, 40793.46830047909, 22894.497063150655, 91299.11854249449, 47848.9207348174, 97973.92243549047, 72872.69927789783, 1248.3336952780478, 75715.84050736995, 93307.26226689921, 34956.632139144305, 9109.05044570658, 39789.14436665133, 58368.842416706, 62177.59914501964, 64207.58341376518, 43611.789463998604, 62188.76659576866, 72556.66130348868, 95357.25836684536, 23057.59822405804, 46744.541075313384, 64818.88242612182, 14748.46944267385, 13829.011950014357, 86675.87728152996, 45924.6085294195, 34673.24242904598, 34307.760121769745, 85824.55364258889, 71218.89093542517, 80762.32808561767, 43968.29797132566, 30475.835079418433, 3068.2055065314185, 42849.782944452316, 4622.804198664976, 2507.8755952737565, 20825.864103046588, 58528.34827791097, 28086.31246132901, 61745.52269751289, 2546.881818201485, 62011.439089550935, 73622.63612738521, 61941.41551071972, 4082.0004359493623, 38634.03140890694, 42856.8977093027, 61265.16014264269, 94505.84673633818, 79531.29580339178, 31272.203882842576, 66030.0149637018, 25722.767881039643, 3930.642112765781, 18251.980045299253, 9689.645342153552, 90623.32381734987, 78509.43016047499, 60024.83893331302, 55946.8997155568, 49150.04611841955, 75539.94522556989, 27027.070073173832, 45432.18988179514, 86158.92910329264, 86041.91873136908, 55062.46596285003, 59157.13061559728, 72405.4467893202, 18022.309074788343, 64683.95448136356, 75656.30871464111, 32294.7382252294, 93785.88384525533, 14217.06273716512, 97535.2519046189, 34166.032700841584, 74465.75456268294, 90670.09151845222, 54715.59643878546, 28616.297288371094, 13412.38134874556, 80712.95627169259, 91894.0012404513, 98144.31389997818, 47425.983410060915, 12291.076381190069, 1678.2778731403214, 35937.57816765656]} +{"id": 710, "vector": [12574.732494013608, 6994.421647059013, 29044.918219210223, 19268.15956334662, 15235.458821390503, 37354.7323380028, 99898.43841945066, 32427.551285983012, 25705.074690960406, 92090.7500826892, 98157.74117932115, 13942.081917665528, 61955.52755938823, 91547.00438662499, 46981.13207003117, 61388.89484002648, 42634.613410876955, 25532.004313308287, 70978.9873533964, 24745.475788760894, 12514.606256428362, 76902.12016576438, 78530.53122789001, 27097.29033474799, 38867.62071726121, 31996.44991454217, 37069.836179489044, 60558.66734703429, 30656.540393767064, 12018.576625901245, 89602.67833460755, 17683.086133688965, 65105.52802077495, 51767.04749015767, 61011.369331617, 74612.77319374654, 31993.57553619887, 96668.88683013563, 64888.073771994306, 20911.434461109868, 5841.68279596956, 47955.6670706256, 90610.18734463077, 63520.68330042172, 33748.5646925747, 94985.858249553, 32611.92865668565, 69543.34195160672, 84290.61186757694, 95171.39272014075, 78372.0586321842, 18541.77235918132, 77768.16532531871, 31921.14327300679, 32878.15329674646, 82600.65604491017, 43281.37624407443, 51595.243403628236, 24270.121129586652, 75617.05214922801, 50898.91836385482, 69014.14597913195, 95956.93309672618, 13542.221061428972, 91917.50654034837, 20177.60884852512, 34473.41023577013, 47208.92839424412, 79536.96219748088, 75253.41534461964, 79827.45346072818, 48346.15345666239, 72335.01858554925, 35737.72321370757, 84925.03300886223, 66440.0399515194, 58672.32250950457, 52997.11706789742, 71778.35633206021, 14651.616440296788, 63369.14810369256, 74854.23772941966, 51515.66242118889, 17334.82026282376, 66255.99627268642, 66810.35285495348, 35565.60899524297, 40071.451086014444, 70035.86614208017, 53627.66952608396, 75122.7507108614, 50278.00098815183, 34709.066737673296, 99003.46710431356, 27878.09280044434, 81414.57082174554, 35495.26201212423, 78973.54269813151, 18587.574907235517, 21516.397878188476, 30071.706268725608, 69219.90070171744, 21006.19331355428, 48582.102480389134, 52791.97668446886, 89188.99928742727, 14124.152957066304, 90169.87216489925, 46410.51540332736, 72276.51619740661, 98724.09051396039, 41906.62934433783, 2569.504273605794, 30974.953569848418, 97502.48402665062, 86676.36214764147, 50825.717847362386, 12081.078980921433, 3610.226979981512, 75239.62173046664, 36484.57433807305, 66023.8043840368, 75936.46159085528, 83644.58665769109, 47260.35844939066, 91187.18543162057, 36703.17092618758, 47668.89587316059]} +{"id": 758, "vector": [87574.99960249732, 96770.05580394436, 99630.94020278854, 28612.269838351956, 64974.37471698213, 84587.02418877714, 39735.79527743309, 83157.78534112035, 64802.197957318145, 64770.42612045, 33645.37482561948, 19568.104588708113, 91166.0124004855, 12998.809147498525, 48797.79997900059, 74729.8066526219, 64980.9999568029, 17508.695574797097, 38615.41569103263, 66566.0402449265, 31834.41099830986, 82716.67727091355, 10589.395833893212, 28822.87905063199, 21534.24270503318, 3526.592327159672, 58274.415361214495, 47163.3626442864, 64302.17855022547, 90821.25513671704, 90462.63270866466, 65820.92123533315, 85727.07207520172, 95711.4892604871, 87494.21948188459, 80989.86393509807, 4490.108852398212, 92562.6224321772, 83519.28359888965, 16267.920337218045, 69717.74685576082, 91757.14878515303, 12846.861660954279, 40117.836846410224, 25814.959942278514, 54037.11654376321, 28283.999998813415, 79255.13954608233, 4334.6393763196265, 16926.1360591765, 32268.83123417167, 77172.5792169128, 83318.44062799003, 36133.68953532821, 6121.287622952897, 85748.94198477523, 16178.827319338729, 30507.65227746155, 83089.75353155627, 89176.77365831593, 42295.86062407642, 34722.201377645564, 77758.29822530909, 75822.23734678177, 96351.34958590327, 13282.56750258352, 31223.02029618863, 51175.64661741946, 15783.790751504168, 29790.56204310122, 8997.628353554943, 1114.5095039356233, 27619.543638216805, 60352.968170391585, 85491.40540479106, 90084.29237022575, 76838.94434523431, 26106.35474475407, 50282.962455975656, 33249.28174284387, 67060.85893361652, 18680.759631647194, 21401.67737215689, 99368.54145253455, 99285.41389637318, 31661.083706461333, 1370.4082461744483, 92093.42103019527, 40618.53857604947, 33877.303328446505, 88602.73197922876, 75692.60802201822, 56613.52372693653, 39185.14625240223, 34283.68019437645, 10069.368898540253, 91446.50056087055, 94821.63196152048, 16480.684734693306, 59960.76981280956, 12875.860076206724, 21820.795468696462, 32401.904401905023, 4014.3718747435055, 31522.09273428781, 8444.176977713847, 40826.60386215379, 41369.80401862871, 67563.8137305367, 50904.05516058911, 98811.74804432559, 22625.786645795688, 48495.14245283097, 79500.56323119273, 63329.65970080784, 58390.480446364134, 74495.153260117, 17190.817376891733, 36515.05883668892, 83538.29484401288, 90151.37155535871, 10084.367310206444, 66709.67392071715, 71286.93794929149, 25758.150572931958, 71358.79202407684, 19194.713579494284, 77364.2165231798]} +{"id": 1377, "vector": [50838.51501262237, 52636.54734534567, 75065.92268372135, 7220.3287989809905, 63787.775859359, 75002.80426840096, 85890.76836321407, 67221.96273873562, 83014.95735807497, 55726.185818420236, 77227.65524483417, 36688.24319199776, 25111.63960326519, 78393.93143026334, 21819.842489584484, 83838.09846154778, 59835.400526132995, 69436.53219329186, 63487.69930130741, 70269.82987880727, 77721.6962896664, 50704.26032305449, 43758.77028358197, 21833.931741141554, 2259.0036516325295, 50248.216524789146, 86694.78230560363, 358.9155350600448, 44687.016092454454, 30398.613231357896, 31594.46252586313, 784.6716317389202, 42281.06113979696, 10162.38711587255, 60818.7402842246, 63478.187843575164, 54579.31720053377, 21755.085162310726, 34115.583134195425, 29728.21355675198, 2898.733890458316, 38061.19569684574, 98372.48952832089, 17424.68317504331, 14828.0812199736, 41465.547006055116, 54847.094406344186, 340.76194479700786, 39794.03434704078, 49555.55777387966, 1406.6491971976736, 92482.05135225345, 74036.63588783513, 64731.49597921294, 66145.42296175103, 58378.41006849721, 57128.99796804045, 42739.00812387215, 90803.96745683992, 43666.83287929777, 40605.33371968392, 49079.75145370958, 83354.97419762112, 55199.1293014088, 67028.95194729712, 85452.82570297254, 62021.28437131901, 42863.46312826711, 72594.59690572036, 76302.32949805287, 56226.671286064135, 42727.06324045972, 37009.79487325107, 38845.12151159745, 8878.12854159321, 30702.02757170064, 22562.65712557901, 880.9871804242864, 84661.73111238786, 16307.427757010306, 12886.604346656039, 82754.85145454928, 46887.52448829967, 63137.628745202026, 28437.73420698451, 31996.71040321325, 54943.371398188465, 79633.29208771714, 68466.8222757852, 31248.517008184852, 26047.044957438382, 12327.127816181983, 34667.64069084549, 89135.23967449159, 94407.3480656959, 26615.835690692315, 86371.37073549702, 89812.09872384372, 27078.307323496465, 86961.62701272694, 45738.705103051725, 8290.656793236196, 62742.30713881683, 15891.836300665253, 98599.63002551782, 92241.7140787175, 98406.89469705289, 46315.34418856732, 78765.6354894934, 50718.02059105556, 39397.16937306648, 84846.61744724814, 2718.84008135459, 95520.42134122142, 12716.809662822849, 62187.782190814556, 70456.24204080799, 23446.787089745038, 9993.598089214394, 28859.619402133485, 81759.60348384971, 70575.12959276747, 82366.63058011534, 88606.83076241208, 49575.341341515144, 71183.4390648217, 75121.85413969276, 88783.87177791947]} +{"id": 1225, "vector": [43185.75275143667, 39194.27150552277, 40096.64053655128, 1671.4176257271229, 15752.690911760037, 76649.92247496298, 41881.49466020461, 52922.626495834746, 27447.117390008225, 53949.22594032781, 50905.53564409974, 28072.01592706756, 81032.29817542034, 57203.77902710696, 98250.67389630908, 54918.38963830304, 52986.692288982675, 25967.502067457117, 60652.16174239335, 84522.28430513605, 6295.8827016555415, 35022.64801236647, 72090.4867004562, 81893.75749777541, 77526.4868872433, 22776.04147402036, 34681.44446912664, 67838.90249954992, 93200.00902860548, 38313.53490208603, 49850.07215290058, 8256.988387571308, 62484.26713065192, 84438.94609934652, 1252.8990145351027, 18771.745510133474, 48429.8457357104, 17575.395495344237, 28626.857845504215, 13624.985273065171, 50654.38968297988, 21322.079320097466, 66837.7158802414, 22145.13666290324, 85867.24832711706, 4563.567613283137, 85417.63785201334, 15529.537211275247, 27529.0155807142, 75273.57864570344, 91707.58223494394, 56320.81125881975, 33764.16028405047, 83431.68723974809, 81270.1217409526, 16664.82808526567, 98988.17782046886, 46947.06061927921, 18661.421782578036, 54994.36412580589, 38872.95738535606, 45726.62976242623, 15596.09095474196, 34848.56487149314, 89545.82654228453, 55777.89727101745, 69247.64218613894, 10931.27281971339, 92822.30364405332, 30274.344405181087, 13103.101098513658, 9527.103044681984, 8937.432625911157, 80086.51179057459, 60106.179734013385, 91131.28582700998, 58260.17115744332, 83642.56091542113, 87234.06800910579, 61979.22160857743, 34212.62947640174, 15840.794824423221, 39725.644067271416, 20357.75402557045, 96173.06227359302, 96124.8316591958, 46419.76585697163, 20318.641470796385, 62481.64961348654, 2067.2382726388937, 89865.08596232324, 36377.60697093737, 7093.299403149988, 97502.78762286401, 94075.53074305334, 4101.744775042815, 58991.750415140785, 86193.25413086617, 89359.57141963045, 38252.878472170196, 86742.18242656333, 13006.359313593119, 98296.14823033668, 45932.026419561, 31067.770406026117, 14142.3422244625, 97922.30732074911, 92958.92803681582, 16480.66997213471, 56832.42732357334, 28307.563587072982, 60858.511694485554, 57330.51227735435, 74063.42517343602, 902.5616998428853, 61674.88125326062, 32779.84087750975, 31912.8489300329, 33910.2258836081, 60783.41567170629, 71094.90926708769, 98848.50873203714, 72124.50344049097, 73976.74073524226, 76686.56425068127, 51975.095063345034, 32309.322822613663, 58304.04957380007]} +{"id": 828, "vector": [78640.73773662836, 88131.21013438495, 75200.38425832734, 33513.65836541972, 17567.81621957377, 16044.303485445776, 68899.05338565148, 5014.360805751683, 44055.530039194244, 20639.151411833267, 89075.3425777803, 39399.03907019524, 25618.09375952977, 5091.3693315191795, 91762.66249875199, 84000.48802317807, 24750.01302264055, 77892.91097990728, 14772.350903467424, 31590.53083525768, 79264.21052275314, 29325.910338012516, 92751.0812986718, 64131.587102837264, 31754.854566752445, 54792.78831810166, 57092.85844687605, 14561.072629477467, 98438.64937745607, 44803.000240667105, 44121.12394258019, 80729.42788290753, 51373.34119292856, 8514.552935467413, 51059.5949326432, 29557.23264062057, 20130.86178843445, 22814.930632903917, 73557.9740574886, 73641.29202138122, 18742.432928508068, 1460.1793192854261, 48239.03380938991, 48779.46290800099, 31762.199562657755, 16358.02499816784, 38374.25700701085, 99534.24226573014, 91735.93453595012, 70217.97965157463, 59631.05525603254, 96411.36709008829, 75066.28881771625, 91944.48773808611, 51272.36434790198, 86073.43963760426, 65764.66271326684, 5865.712309842142, 26400.444753838827, 70501.63817770156, 51766.285373732004, 53449.237458077834, 92000.66005369584, 91891.94364739051, 84537.46100953541, 9995.453174018565, 97680.6996355392, 24035.8697965808, 91073.8677210271, 61496.35954156399, 75420.21884661654, 2823.6171033482747, 97958.51557544162, 29341.49090659255, 79937.15947684801, 13953.515156771968, 27350.038385849697, 67336.57610392362, 57876.097631802004, 59494.249645939344, 6586.529904665251, 10953.28295650324, 17692.678987362542, 34208.1364021268, 43914.7344696987, 21863.344151936537, 8990.58506542042, 46219.34227853104, 69914.27678686245, 7473.203877122803, 52461.9527056855, 73050.2935625345, 5334.698111981395, 3722.34235561828, 2867.5702994000794, 58270.62201340376, 58996.38960636766, 12075.066498289367, 6242.838890294511, 65237.01568586929, 63295.481067868954, 54148.5400737323, 91778.38537674744, 66222.75756210616, 82991.1210210869, 25174.797385054724, 39173.986213754295, 59270.13493663292, 22746.03885100578, 58388.922203378635, 82848.96581431081, 66611.4377009708, 42047.52112670249, 42598.357943654395, 17631.971293925242, 37034.7645502503, 83637.72937101698, 51131.94960053341, 95692.82520074768, 35603.78815748582, 26373.091077568857, 69588.8904001174, 57271.48152540694, 48839.92252892681, 80797.94493064457, 37764.99418437706, 47825.1877900709, 6767.167814337838]} +{"id": 1403, "vector": [55877.98202547935, 10655.271852036296, 93940.9422818658, 41958.54207106522, 62585.50200511844, 15649.299598091759, 65910.98301727531, 96274.32500647506, 26377.40968982256, 50519.185912270834, 10154.665115940665, 797.1303759972459, 40380.46086766724, 22326.66006444447, 23639.245592284242, 37224.954784657646, 26696.074280058834, 60421.82546758334, 18516.73818425654, 49525.37780745458, 88292.95733588166, 27851.565704046432, 98014.11414305495, 79430.49337733061, 22099.96941609398, 88517.74655736044, 9103.581611316147, 63231.90617687573, 81702.79141618723, 54159.80437086396, 57871.89889577402, 52348.807883303714, 26689.186945578636, 5395.804230469692, 49234.24227131072, 37029.94942791463, 54855.603830071566, 36420.86150826538, 90718.91292690158, 50549.58035788275, 69165.78148401639, 18523.87612355724, 62785.48894205289, 65443.07288811443, 31081.813173436334, 36346.616346411676, 198.64401926683595, 75817.44875088922, 72313.76778306016, 87715.81027601917, 40954.12335690952, 48878.1995414527, 4886.565260866049, 75907.31796026937, 55403.17691586068, 47062.42797367085, 69823.59390370455, 83739.89180106587, 74817.9009562433, 1814.701479080183, 50313.06885113952, 51209.03985183927, 25608.72215222877, 40833.42580207726, 43655.82238235539, 13518.84383620623, 19731.133494866983, 84633.98594165694, 12564.952198507306, 43769.91233954493, 58085.6806934832, 41938.895571692, 33643.419801651195, 48315.393193143405, 56487.9513131709, 36377.552867341, 54910.02840942358, 11968.212385641707, 88098.38728273354, 77755.66175375831, 33624.33443221836, 74420.49227870032, 32866.21743421575, 4483.55256188675, 88309.57293846384, 8837.333050300966, 84282.98300905307, 51133.844646680205, 91102.0842452968, 19607.16006067319, 56677.99357167345, 72712.53508391471, 51189.465278751166, 95268.24121744587, 44104.36933246835, 56286.9092817742, 87480.18638976211, 87281.97533348211, 53946.496989620675, 9614.208657544465, 5733.5692563490475, 46783.19728196639, 17036.884929812313, 20414.846836367695, 61291.77064677984, 95998.94528732992, 57634.532392613306, 1588.2148091917015, 20326.796233494082, 82535.72965155124, 44044.98152817939, 7592.397845766163, 32921.48178420714, 66097.41614513565, 24112.997736419173, 85595.95257153052, 7243.247660921759, 89358.70277590121, 15020.361189378862, 80316.35751222695, 89485.2969223223, 39121.36468441653, 45000.285251396264, 75122.85521647069, 40291.682676574426, 11819.860162987883, 39477.97678723618, 23142.427509050343]} +{"id": 1727, "vector": [43585.60079432209, 96593.4486123582, 45562.15628926818, 82963.77846138428, 33856.002191338186, 24192.648827554818, 55428.33077627698, 96871.45331482757, 90615.53089791513, 70496.10168513977, 56546.76491452315, 44381.00215856556, 96905.3596994747, 92449.7641924875, 47493.1709648619, 37950.060475246784, 8016.180587161714, 92933.68004184452, 91421.36585262782, 44485.98674222724, 72279.93601501103, 78437.19189968886, 7093.086759110134, 61487.127905788904, 86961.10935210675, 18924.281357969187, 76792.05619748768, 54100.89365353351, 13315.338376242125, 87355.33101927437, 87307.83974632353, 62605.20274714673, 32941.664151444784, 19867.875932606483, 5759.553319114774, 9221.92927439931, 56992.408862918055, 75992.51275321197, 34384.89493110115, 71502.07940709894, 84778.7003368723, 16234.83849366164, 6807.476485594854, 74982.95739240904, 51173.37736682852, 8632.816133861643, 14344.660185715207, 15037.382769595819, 44826.343210227824, 24783.03515179273, 73542.36180932522, 76906.50378390259, 55531.400328102085, 63856.63360600302, 72024.71033438387, 352.9803577499213, 11984.93243779759, 38205.17773960908, 83496.62202845949, 98938.42138537642, 15659.580198185053, 60191.975830279, 76949.21500695897, 66774.53616428684, 52164.683333523586, 89953.63117551823, 70190.68634891199, 62732.99261458446, 81342.14853576338, 96401.91186163465, 16162.87912912443, 17112.464786148983, 36863.347622305024, 57654.69563873238, 81012.5127106101, 77972.43314661898, 33514.78769549021, 36988.09200999557, 81176.03418526439, 88661.5309728396, 84188.93618549978, 52260.22632651777, 3956.5641921947004, 68644.57962099892, 14310.226428313566, 5486.135112017753, 68560.975382615, 20115.596084844034, 94645.64423990689, 74057.59974918971, 28848.755287509488, 89679.41277166069, 60637.023695542026, 15599.845134632751, 36989.45384174903, 40472.35155321056, 76580.07256360275, 56517.6599676834, 68262.45628674625, 98913.52997750865, 34201.60761324197, 17290.79881906671, 23256.0578838369, 28784.582683270866, 89242.02559945514, 40370.024609764674, 89818.56207580614, 29028.39792863615, 88597.0259782596, 40268.53657949103, 3545.666533667102, 94280.07741511184, 82810.16231872521, 4221.238611195022, 21723.289372426112, 6271.271841906933, 35745.28651831595, 55529.58054010175, 16696.661488614605, 4390.963034579376, 8404.914717984035, 68027.37097585507, 42618.79047846424, 97737.91405371921, 57796.43933901457, 92632.05501204127, 56485.187617709256, 86468.58744613742]} +{"id": 1964, "vector": [48525.16354691162, 85660.7981739264, 8579.936785576125, 18892.987922186345, 67088.16696807672, 85101.61260509831, 20556.834756584096, 77975.37856142731, 51662.1225632645, 59842.35399682933, 26327.172629931993, 11030.713126647008, 77875.04333655458, 19970.324215867786, 83466.1129440489, 18724.583141261362, 57902.1186064139, 40758.465229304886, 23283.072733775058, 34560.732049517486, 64429.71495371961, 66772.7955791439, 87258.68135667189, 11937.02017312409, 54968.5835060185, 55768.38370948899, 50920.69572432676, 5853.670085886398, 93738.50189760757, 62061.69268061015, 37427.626980136716, 98982.45018822853, 84497.74836373827, 86419.60135036807, 11455.705762502477, 30639.845392614727, 74636.21116133143, 58964.30896956532, 94334.95470418844, 5655.842596961446, 20741.351220290293, 23562.086226728763, 15356.587882115857, 50518.37267732653, 78493.80256030854, 92940.68508204269, 4844.133597057687, 34773.56536885303, 28146.61539415043, 56134.70677550563, 96381.03987059768, 7343.236925100416, 62934.45245321117, 62849.700837311975, 10560.447881592749, 39793.06865633951, 87394.30202666881, 79038.1896551404, 52265.52195160933, 8965.661900114297, 23931.345597688436, 31927.846302705886, 88975.34724085777, 74517.17639898916, 80755.97455887176, 37110.515407614905, 78037.2671604786, 52116.70436208071, 23478.134235453774, 40043.17054340588, 54960.89224947559, 64944.65639852192, 21262.136583605905, 91646.43539504972, 90189.4649774933, 2238.0779761147382, 19465.351903376348, 97286.17546171829, 68667.25131135229, 35291.35745859037, 62917.00293622878, 14655.779057425732, 45365.639616638495, 41463.39077503315, 81415.36526197041, 16264.726036696708, 44976.25089386761, 27959.300774387808, 76552.15538952267, 7675.643615326288, 70441.26961720035, 85518.22934930807, 82326.96928632758, 60497.090110112964, 70588.05864483798, 53791.44687807116, 21835.468003004877, 8159.743014934306, 54141.59256596075, 74814.84235804468, 46510.80604916645, 72012.27655163914, 19895.57940460781, 31597.090263301543, 42805.8564379532, 11487.321206795397, 70171.97922708573, 7138.497648354658, 76094.33867546347, 87669.7708319219, 31100.112342077213, 27.64598349923597, 34008.06329543036, 36777.800683416295, 58900.08833032054, 80342.71302116671, 87175.82789458742, 82443.47054190372, 8951.619578116588, 42357.866307783945, 26635.943556591523, 5011.063095173818, 84976.04608479116, 59461.27731735862, 93267.94485337216, 45867.7730577138, 48717.690567286976, 78628.00495097178]} +{"id": 1863, "vector": [18457.21708825704, 88707.51744441505, 31893.42495022497, 93425.08166274364, 87673.05490733331, 99032.53394688382, 5931.497215121584, 97066.07656655571, 99865.0538955925, 95602.33837659347, 59829.908235601084, 6257.381715061305, 58806.71083058128, 55976.877304837944, 36610.65165885969, 67358.4429995764, 31572.180382721494, 7077.363407983017, 70525.91528157324, 84354.67784463409, 77098.37594778415, 82840.4581463825, 21711.793854729498, 68380.71104461819, 49556.756723270024, 93349.07177740484, 78226.3202339312, 44577.942811452864, 91054.738139429, 48571.22708819801, 46007.81809032859, 69303.68901438695, 79586.30048530754, 98278.69330050609, 19017.397446720828, 56362.999820028635, 2073.0850810723346, 39398.71632987071, 3495.0739273089894, 52740.10254141725, 93274.59001701597, 38889.63622287197, 16668.42029618675, 70443.49114149157, 80983.7790473333, 60432.484146917355, 88414.0211968374, 22386.42908695696, 90851.7665565003, 2935.356580701931, 15941.071635397608, 70198.57380496524, 61361.60225261045, 44445.02136622395, 22264.846580438756, 37285.260409744995, 59571.12858138252, 82943.03937683784, 77448.48524737688, 4145.555220234421, 92413.00552565207, 40089.35346410604, 65071.688618776214, 27345.410752482647, 52628.5420927407, 60270.589553731224, 80369.45929637816, 84315.78303041804, 473.93329496353556, 10240.44535751809, 13263.079542931933, 71651.13647950155, 32992.96723963264, 42870.93484286745, 44690.05363800254, 88572.46139505318, 86840.86427642888, 82945.09836078442, 77353.51147552159, 12381.99428380684, 75329.16425157605, 17691.191487349035, 69159.17407453796, 66234.88310935628, 53397.64786136327, 75559.93947207628, 14580.48627624372, 34528.342152704085, 89376.89958947724, 61101.767579860054, 56210.281719007486, 63617.217618136514, 62375.379390102426, 66743.36098368639, 54419.04906591568, 37472.56813901444, 59083.17579115313, 46139.08508303268, 16247.414870775623, 37602.36866879107, 35811.00307264766, 22138.049731897558, 35309.863072712644, 93926.65715924045, 17110.91465798805, 38951.527487404994, 82588.4897070662, 23710.957370358887, 31886.15299411052, 52248.74636734608, 98082.70595990193, 4357.052667385119, 72010.21745387293, 36452.82085292715, 91565.86868649535, 45380.527726016284, 60658.95996981874, 60661.33186412238, 18194.049558467574, 68111.32214322822, 40281.82961976916, 87233.82756461468, 14670.524433720122, 34759.26022884329, 70205.20174656578, 47402.12763819644, 86510.16252256563, 75912.36469774442]} +{"id": 319, "vector": [5276.516757490579, 20305.694649498007, 19464.938369853713, 8657.38352924449, 45623.06295747178, 33874.914561778125, 89344.82953900626, 13049.782905138485, 91064.54508924205, 10938.903782658104, 97024.1730944543, 47306.717671479804, 62306.71532118378, 85301.15938800419, 92095.7175365549, 20854.35586605088, 84371.21474686515, 62542.022668525555, 91997.80102616474, 54868.154877765395, 5471.390763192707, 56417.76626016842, 39384.98940621401, 83893.06666243915, 99599.8668315432, 64121.44639220458, 66826.44688267137, 61763.4275059598, 82567.19135090931, 52227.52193273717, 91536.638190432, 90656.0817457044, 82569.3026184606, 44876.78926570082, 70970.29088479528, 4387.082925522401, 74301.06236957198, 88923.54827834106, 25151.52172038403, 83239.28338583084, 5408.692326099141, 94295.58282232504, 62366.23497477969, 87238.03892209983, 94937.34730764508, 48260.95412727113, 32248.88078736249, 33089.57506420799, 69464.65435118793, 48708.584173126255, 33930.27957614333, 28032.567128663122, 20809.96924754147, 8845.70866146257, 56107.0992013916, 47178.186605122515, 85450.70640179086, 45238.541750636694, 83168.7011056088, 49815.070823600974, 86331.56119396641, 70298.51323450981, 13466.747527626932, 26418.861066008347, 56893.25586226186, 71059.06786537507, 54474.91307780889, 18524.424734875243, 14789.993156609715, 57916.15905009126, 48478.47479265467, 51386.88423062111, 47391.019154650035, 78791.25857319981, 58393.87289121591, 57689.731681340425, 98406.47704724033, 89417.54425626896, 61556.81693932622, 53297.93605316237, 69100.77061466557, 48772.18036495196, 93148.55590319095, 78927.21053497426, 32985.86724629603, 92838.64694430465, 70069.91287946588, 8924.325215466522, 77446.10141559032, 59168.885437026365, 44270.20789367584, 56362.495037244254, 39359.09762679448, 57251.89350870215, 3063.1758253968133, 9605.779575397233, 57691.26161442513, 25240.95755055221, 731.0436563959244, 60024.31733885004, 14586.423046141106, 52085.6369502854, 68506.65532699955, 9530.48148512966, 59139.64762866526, 29451.116109451537, 79852.78841360599, 28010.185545050346, 60689.611441390625, 38390.680856997904, 43768.12426945598, 94048.77996952971, 71032.0287971679, 48541.549971963235, 96370.57492291264, 80452.75290924904, 24685.073058960406, 69332.97581696442, 33700.7162451907, 20587.396261130143, 96575.6993252608, 52084.103444859684, 47227.26204981747, 4405.199281194738, 5430.567132142317, 29961.198168532486, 79010.12489985464, 53498.17400846685]} +{"id": 872, "vector": [9764.698002454941, 25659.559766153907, 1510.2877252279768, 58440.07881493225, 99260.1082394177, 3586.647994890757, 25442.554745052505, 98209.27639595231, 18455.027380156575, 23176.287955099408, 31585.023693980074, 7360.719698398244, 28595.96827408092, 22824.70456322917, 14099.33553031887, 71163.2825886203, 32876.93187931125, 92179.50434752423, 67839.25043847872, 9506.156422476286, 85197.71662000872, 52896.00579809128, 10855.332256504458, 88130.08627581286, 7758.473326119997, 28620.494070524728, 55135.760191671005, 52759.88502990514, 67392.14703493519, 99257.27709453911, 92168.42923388172, 24679.3457442393, 90121.08436191862, 20659.230693848873, 8981.321389375973, 73979.47050484057, 5772.709948449239, 7087.515940222644, 22355.921126846024, 90995.04805891353, 85301.49333047243, 62142.95760701858, 83645.02246951446, 40693.33927851746, 58701.368287964404, 33013.83023816412, 48952.27743183719, 73587.86998897148, 4321.886176109224, 19410.244200066463, 63149.86564946411, 17555.081471428424, 34417.5093913251, 21839.534351542057, 38104.249046203506, 36852.948096246415, 92000.61097748239, 27329.34706238104, 78768.22146259995, 42121.65168327393, 22913.04538018658, 56552.01631387346, 5618.236879801597, 76644.92980883378, 39492.94304016879, 81499.1192334878, 81960.14208705163, 30436.488449373057, 92445.32912414071, 59339.1344786871, 41042.727178429464, 21915.35317606995, 46699.53373902894, 67338.71566889485, 44737.374611348925, 65516.454026218016, 92761.01285793322, 38602.15866676181, 10387.252374866019, 76318.49675804091, 84646.04531564303, 84101.30712274532, 22172.70738183197, 57377.525986227396, 56431.91901552499, 71682.01901738066, 94564.03819981668, 48886.99140152618, 76826.43476398519, 56454.25039349627, 40756.911150695974, 82435.81208307821, 13686.383811522306, 73591.32614802953, 22927.004183953613, 76999.1108231932, 62035.26342320014, 1267.7870805366308, 88063.62945434074, 94424.97286916316, 6905.540384820841, 51712.46430751852, 31560.373339479596, 10116.218925803989, 19742.035602959797, 39944.65824308403, 78905.60688879689, 18156.13594284864, 98204.20542505606, 86649.43542266742, 38315.97662868438, 58561.41070423231, 29053.124604479963, 9055.11709547795, 74051.8619365221, 45096.617555886434, 13011.580063995609, 10068.719874349263, 83098.75133388973, 54425.73822348309, 82643.34909749744, 79270.84859584487, 76897.48117055031, 81767.60261736842, 60120.51425524255, 99859.55503917529, 69477.79791179516, 45747.54515613081]} +{"id": 750, "vector": [5320.642442751644, 96367.38546126778, 71756.43776196045, 33479.24277758444, 11404.048723851745, 44888.78814245153, 20615.127492207863, 24115.588502716444, 88745.77090289128, 23951.471617596333, 2387.2055031600835, 63131.44871064111, 90105.917787923, 9313.873827606389, 81277.75903664176, 67610.18899053066, 19336.14595699221, 85052.7519981514, 84630.69226768307, 39088.86162925749, 33257.53697769446, 10664.861376056146, 88815.18051284728, 7877.651954524889, 27283.29722965048, 80245.65921387378, 96259.50255524757, 65522.069698003426, 1282.3718592038902, 48634.9936992898, 28979.75993332653, 94251.75464557498, 68599.89929143217, 70135.14974859892, 64130.35957988806, 36451.19497946613, 29598.50595283574, 2584.2014762023414, 21808.44002507747, 81380.3758959356, 25520.843307646002, 46783.581451921804, 40641.92325558915, 49130.800888939964, 13681.455728222103, 33049.722796145965, 18416.931686469117, 46803.540968955196, 86578.65133052233, 64347.65537163504, 23006.236751056564, 24746.064742892748, 26297.987340015294, 69771.52115120066, 33409.399954243854, 81623.50742189198, 68187.35368564907, 44338.11786141646, 81392.30990086647, 25013.64387986733, 98106.90574365843, 49610.28290327301, 42725.04135993348, 2303.6433312191075, 33045.42731824826, 41231.57359048529, 61192.20780591382, 22153.43037311409, 40986.56599434215, 31194.717363726453, 57764.70479222929, 5370.672907519747, 15533.273028385707, 74471.39049706233, 44150.44722949956, 98124.39200107133, 92079.72234721403, 98544.70575448047, 63692.55049984429, 54329.49356909249, 16931.04281754042, 86441.91820895496, 99805.31136722505, 70839.34061228251, 15857.104020290979, 55837.6276948412, 55837.38682359357, 97520.16591295108, 19257.018711555185, 11279.761286673007, 93522.7952618522, 20805.066083294078, 32335.268400802288, 55232.3397242374, 15500.197980748288, 96175.89706585456, 37494.82590219104, 30159.498114967053, 91840.95036307407, 46045.18495333521, 4788.63312651524, 89604.87831941695, 26413.3481519787, 96790.07107287202, 96752.54528529936, 19089.474031273756, 5064.312801415183, 53646.5647048162, 33773.588120998844, 35556.119754923224, 89045.72023651132, 960.7504872732875, 32071.765890115323, 45063.852283839704, 57910.105131819226, 51345.73885994629, 78705.77924073744, 82236.8183079183, 56241.18004555183, 64603.40846301117, 69054.73403039803, 17335.5612210769, 85130.6884942049, 43373.14971131747, 39469.99714546566, 1072.1730848807565, 6891.052214467164, 79758.7337359285]} +{"id": 158, "vector": [42541.96104318328, 62451.44105464331, 25424.54868258558, 18343.763397717405, 16215.104755797916, 95680.84729647705, 8473.306089054278, 29746.189494365215, 18010.336375576775, 69396.27920133705, 76963.812617273, 45885.251769357885, 92714.90763075573, 67021.3983982774, 11752.808050199914, 7385.1978020759825, 54525.344286974345, 90193.66455930591, 55280.03563122564, 95884.29526539703, 94913.32493103572, 72189.64924013692, 74545.58595573025, 9022.629162739615, 42579.57781254644, 2667.805386874389, 46306.33654126658, 97718.40358020447, 42768.829050739776, 56223.16330800554, 89945.58527052525, 57561.31326441071, 80158.50853515674, 41384.75168794805, 27578.241399813385, 20905.140840498614, 47716.88649407434, 60314.64445938454, 62492.37124936425, 9111.26445204803, 69.0803018261743, 42865.59988614578, 42171.529580913484, 34115.49053274747, 96630.66393522592, 92093.46749718938, 49936.738768572795, 30990.97369137702, 42936.69837862568, 55459.06401203469, 12601.323241967644, 24272.755171961922, 88465.54510828962, 75484.84510763836, 97907.1884746006, 12282.575968423482, 71318.91784836448, 43455.795377645576, 269.939962566268, 34923.398745881015, 29706.83293161991, 90120.22580645207, 40248.78044317165, 52907.55572567591, 34666.238884020284, 17590.021058216975, 61960.14958284185, 1943.6249829217434, 22393.599187410106, 17413.70918864482, 5486.632423964166, 79861.82157500855, 92850.19210334003, 583.8595385741053, 10360.878098941317, 96144.75112672042, 98767.97752413191, 72479.15850006393, 85337.89776693693, 56046.45580301902, 72156.09160686652, 65272.080986999616, 52097.690147923524, 46694.66291699486, 96516.86568588165, 18836.901864377654, 24522.151759191525, 18875.98185026117, 19132.852981591674, 63499.40967900557, 447.2204290346782, 41608.18149652819, 46396.64274266314, 47363.772480911146, 83686.32125989413, 75666.65035948118, 83295.56324347576, 67140.6762742234, 19118.577261517956, 11892.487945507468, 98196.45909780334, 45351.52094926902, 54239.59273083366, 77543.09062090497, 3396.7726547731836, 69010.23960307142, 87296.11957324443, 91536.57397521754, 2954.3431120541363, 57828.89416765197, 69263.06525189232, 9066.756862565706, 11269.245933693273, 96376.19397162169, 32514.8754019293, 87362.79496790268, 3295.716363258694, 90456.1330333955, 37253.379026157876, 54064.476879244176, 48519.47377578189, 34219.47562146702, 38196.191622204, 42055.22635686575, 65316.029731489514, 3030.885485594892, 73812.2143457977, 51963.09027540904]} +{"id": 1419, "vector": [36963.573476969956, 86435.65561951141, 47993.566620938735, 94295.8224857704, 47597.43838689983, 30769.73906902858, 1409.3387974830284, 15429.74884874193, 81235.4962742053, 87132.42562063264, 31038.21813855806, 53447.87560702847, 98070.52876055553, 61656.615478224165, 56251.91508162978, 65291.598674428365, 83806.17000108497, 56954.28108090172, 87263.98523241883, 53288.81837197449, 43307.889297111324, 66361.75993953196, 87998.27205990459, 67908.9748508863, 59610.14903458614, 86976.82382633742, 24357.34349237365, 61345.57630962312, 92757.9434451446, 37250.02848126019, 1704.7717526015304, 8892.244674754724, 84225.64221027457, 12715.709315397871, 3873.8240873684495, 21936.14573958037, 26329.432077427016, 72055.25594320438, 6890.722466718291, 13569.762111683414, 48572.77111340309, 74311.84358944812, 57787.56104307947, 42291.795830302326, 29437.75814441093, 94422.25979619836, 87765.68356144814, 68395.43603762584, 78023.74986871838, 39161.72521523339, 21404.80517545228, 93249.82327436299, 4741.616402791504, 82927.17089022546, 45806.44888141877, 69700.55622165538, 96512.64199506893, 516.4686639914539, 60622.69212253098, 60865.35038936195, 27878.468496855934, 23152.372283014, 94338.32011297105, 60589.9509557666, 83366.51360343206, 20601.118718271704, 94730.61194468194, 80215.50893178332, 14130.770156778217, 35180.4543289168, 22067.177423517736, 33751.74497413195, 34239.04025679347, 6209.8677613429245, 33718.401791518656, 59949.84476751069, 49549.56444325854, 72106.27982940912, 27903.963289605694, 69823.45795029607, 33122.24760917606, 67377.32190497917, 99999.26717563462, 55323.00143287782, 95228.56387928521, 43971.57009267412, 50556.697147884544, 15969.229275667496, 20198.826541706407, 8548.420451794203, 86920.88526758907, 28657.80821839232, 73203.07531796595, 74018.806364063, 14638.185826728057, 94716.36881194338, 24467.93360160794, 38552.00297485282, 87901.11334158666, 22224.435660922194, 18227.42697169577, 831.3509655685558, 83446.87457068004, 40867.4716587863, 74130.02938372048, 23241.86560621364, 68885.23135382406, 41213.526780395194, 19507.8932276622, 62329.66300573418, 64055.02043787542, 65013.399760116976, 39295.516214762385, 31719.788937786085, 52985.23046788402, 16119.216140009152, 38254.53201108786, 61275.17999026627, 16837.030007585006, 99096.97076105158, 24037.523675213, 90729.29626824675, 22926.30894642974, 63814.58165516814, 87738.57426703362, 48830.722326803145, 79392.16843352914, 77738.56664165221]} +{"id": 861, "vector": [81190.14109806532, 55038.466690863694, 28120.840300133088, 44977.88804335955, 59696.78676657114, 48920.74264342258, 58120.460561912034, 24836.079657689294, 83320.91782219478, 6633.605899142658, 66075.56527474495, 54865.886657308205, 89506.86301022132, 66466.00152592674, 50984.20947259369, 91174.30867777774, 50637.925585207675, 37366.95337012591, 76135.020299713, 43394.279225375154, 73345.29278636846, 19631.38983962338, 17249.702609263273, 23742.38583634446, 59399.976790746536, 80331.02725528632, 10350.052985694103, 87519.54540136323, 62224.658809406865, 73949.2900179622, 28910.225866758454, 33545.676194104126, 30505.68145470568, 64650.56075118466, 7251.386990538633, 65119.49306167398, 61675.230220150304, 77119.80629566315, 79718.84234330653, 57248.3404622932, 96807.67656277295, 12538.798453048528, 18030.462360989874, 6986.808974934255, 87197.75487963848, 20471.08868077662, 62863.96843925254, 38828.376309232226, 15995.546478723489, 84132.29717439007, 71931.12104884526, 61918.35309404484, 74954.2333555647, 99153.96966687714, 3848.942203755745, 12385.158590458046, 51111.55385366834, 22967.200820090227, 6883.155902836224, 23236.58663427205, 43893.04578979066, 66685.87084539916, 73242.93709206283, 43170.812608258515, 34339.31366275871, 70366.0874054052, 2578.664738760017, 54451.583899060075, 98169.20261573498, 18349.5285923063, 34519.80820837098, 27474.111136929612, 16460.30218549962, 78238.82206722713, 51472.614903920956, 43786.62854844796, 36569.34710800947, 99991.05017575693, 37348.22834442792, 53929.843314675796, 87331.43886396027, 11628.761814889676, 88210.92919683443, 5277.567297853103, 74777.03537084657, 99996.01886403048, 8006.082902327083, 82810.93975377336, 35169.10112828575, 48670.71827636799, 66998.82875059717, 48609.654392284974, 96013.29172484706, 12986.27564823015, 67867.71612453989, 52463.47825983412, 78872.92174337423, 40871.652362939625, 15762.093839772351, 24347.052861577402, 79470.38322432137, 40597.61047165347, 61439.8133408149, 96279.43025812572, 69912.79655795415, 55153.78600676567, 94798.41921729945, 55107.61449316709, 86596.48764252196, 29784.62292056232, 50595.740532024545, 63455.709040145426, 74016.24185964237, 44541.61424401284, 24362.429711078672, 76638.34264794565, 75110.94971088514, 45390.18846075814, 46789.05130598885, 46433.91451992231, 40867.35831235434, 73873.4270726959, 69531.5633301245, 28659.58560727885, 80307.36763938406, 88366.54450485266, 313.93968848180134, 18897.46760066635]} +{"id": 954, "vector": [58092.82722222846, 58651.73825934758, 58517.53293067979, 99185.58943699267, 33177.4893692122, 53712.47973329536, 68050.9284549627, 98521.4806304775, 21539.55816494156, 43591.32917197358, 80832.84855942152, 57321.6113313913, 46982.409736997244, 79168.48808452695, 51824.572824319235, 72956.25125355947, 84084.46879385108, 66584.68514510957, 73343.14352828631, 50577.37002379743, 99649.20011992301, 18612.379680423506, 34440.05834333106, 25181.02800614068, 55215.270914893146, 53840.06508355089, 84163.03904223099, 13327.947682062802, 60069.87794884996, 43418.922657648116, 49336.542844065414, 93314.4022594313, 9802.421244879, 53336.294429042675, 53427.17796245719, 96497.16846759601, 56351.07791592605, 86735.83680889306, 31448.12079269095, 98943.25240828919, 92789.16374035482, 99162.1244959899, 32906.291257955934, 6778.696210114344, 75628.51536050001, 57253.5666721526, 53951.13793736457, 56008.362991499875, 78895.6750705591, 53855.29928165622, 96417.55225768096, 67792.73640733173, 24731.971667427122, 86982.15878444686, 64462.98754954391, 76978.11744026914, 12493.33755123635, 47614.59743858957, 79126.9219865273, 9145.99347657975, 90114.78933160423, 10372.973776629058, 33804.22525904512, 71670.72898874847, 93447.9827081762, 52936.77338408308, 23503.52876143761, 97622.50817746905, 98468.31776962923, 85472.26544533367, 40003.4788583875, 27466.132082530992, 13149.142957591486, 95859.03329154053, 16989.355500689697, 33950.54389567591, 63016.585142381635, 83120.72219732939, 50626.53255639706, 8922.838591086767, 31987.5375093162, 38239.66950835628, 16896.218317627467, 33293.32271652807, 68462.42061424095, 54435.55364928446, 60254.131699298356, 20200.296098724302, 21767.264818911648, 20182.273633131565, 83440.06967189057, 51873.4641434894, 4758.177512268513, 27568.587607031415, 97951.38611180843, 96940.40169745804, 67285.97370590067, 39826.468198463815, 99874.27301899498, 11237.603838810972, 56814.78173601133, 45442.539858247044, 41587.81515182456, 36469.796083460235, 89133.37760892979, 9044.05417748334, 95821.96452019828, 91333.1822776926, 61371.95938076174, 8319.89169147076, 24179.94219637569, 67565.95505757711, 85963.70375342721, 17777.540720780005, 11595.419235141435, 48525.261975574394, 99748.92916435852, 15123.686838008054, 9533.818789575422, 82869.90260048755, 96661.99512413552, 49467.19616560635, 32003.764157626334, 59725.416869943714, 71366.92171868013, 1046.8083704338494, 65969.20574034874, 17125.625718524963]} +{"id": 51, "vector": [50226.420704096185, 72235.24119758108, 78324.91183376573, 46193.0717383065, 7861.394887908368, 32976.35221994048, 29308.520146684135, 37373.43528984966, 98958.36240465217, 76204.12264918676, 31831.537815909527, 32114.846958562404, 23631.62804186296, 25770.875875845122, 93027.03528201125, 37934.35123347003, 59081.19056952047, 31862.544253403656, 14021.96627656308, 95823.44031972282, 98957.46961566654, 59563.219966959405, 18824.660023246608, 20534.380207936654, 78712.1265277396, 38371.193964410886, 1651.6436000892586, 32735.888537219005, 65330.47884716206, 46492.043668350394, 41699.21932062467, 90967.60905694656, 35310.7453831628, 37510.85400858052, 16826.79555793105, 58455.66385980704, 71028.56115501226, 26304.119603700914, 29153.646172979108, 99964.5212223508, 89195.2017652433, 22694.60978997091, 54836.56164539042, 33953.46091979042, 62565.92710593007, 13176.594382006278, 65321.85463479008, 17360.17743687297, 97776.85828119921, 26922.828546202592, 47782.600059596305, 75683.52321747396, 40642.44434696048, 56472.535145049755, 30033.230692986668, 18102.25963274411, 7407.7407224267035, 55894.76581940815, 72062.03673452145, 47301.02737477821, 88030.8342186165, 28452.333357772473, 55610.537448158124, 79113.60368210245, 94030.6161643656, 28763.8832636827, 63218.28498223166, 74006.89183018153, 78341.2018599674, 53934.97912672213, 71128.09186419156, 58723.74985806601, 81984.18671795895, 61256.73880802103, 62188.50978870073, 4656.137007948957, 87558.87894453712, 4266.717719851076, 50904.38549762313, 99854.6292045094, 17805.81434251084, 34156.62493755682, 16289.797853827647, 55005.79530428624, 58191.62565281572, 81505.2440337237, 15965.031971646526, 36377.96209389678, 74207.35790543468, 47051.04309537663, 82950.91784573128, 48526.311272255, 82683.07279620078, 18320.942746978486, 44347.25109913646, 5157.0809602213385, 37717.21427016015, 3597.819500352373, 10635.145668797852, 4013.3903490540247, 88384.62823478963, 6605.074953067613, 47645.489825320954, 45241.992624880586, 8938.194898860418, 6953.369993831393, 30816.99526015582, 20008.355084050734, 91089.30201792557, 82300.60331149724, 68970.48346297677, 22105.734439648826, 84584.5770013023, 36130.43345334578, 33362.24391432349, 7565.03339247594, 75920.36053632003, 81666.83092772038, 65210.59852402891, 95.56810787993975, 60767.05444645673, 55140.70030992714, 87000.36788627438, 40018.759859656464, 41528.01680997262, 79934.79966933944, 43934.2394276316, 15358.310925541884]} +{"id": 1984, "vector": [58268.29457755686, 91458.92006721914, 42309.74352104718, 5862.90148970815, 66589.25947583203, 15403.659044862372, 20421.311141512455, 28770.05854006076, 59115.958069500164, 1933.782026205677, 23564.03273365608, 6228.022981530601, 98916.85930718739, 11518.514117399181, 45491.3746526418, 54365.193517675616, 21453.32986888444, 33128.264503076236, 93098.83105142864, 76833.23070050422, 63053.12864058755, 3138.4281790869163, 40745.995815376744, 99048.11978000234, 10004.828062745475, 16667.54771709713, 93824.81038905917, 92015.36652872335, 5594.518914758617, 93078.52263036673, 49840.8818812381, 9094.866045354032, 82985.76899885376, 77670.72903889186, 32726.748865589572, 61147.21754716993, 26283.855806529642, 22739.11360870564, 99027.78699931056, 21934.482210627804, 46737.88545017913, 25155.28447443689, 1269.4326954757807, 91830.69690749336, 68574.11750201452, 28070.058119687623, 8950.451161211548, 46128.260370760734, 84680.95973216223, 21707.753325371104, 17504.59239036212, 57259.42215047956, 93082.07255788849, 35450.35040591016, 15569.043656232561, 47048.498152202366, 98315.45352298483, 32210.574512722178, 99550.67964061322, 74654.33622006098, 6520.240665170607, 16799.426310656505, 78687.2268380343, 48382.59616855381, 74694.94782159678, 67106.75680448557, 48969.36230836443, 1830.153418400371, 88527.54526094864, 27297.98163410072, 43307.24831806449, 46394.92854938341, 37498.14140181964, 24836.30114527243, 142.39912171178926, 22949.863660645962, 37069.384455287945, 63671.930387337525, 31185.54112424169, 11079.786308342644, 82892.07468603576, 91647.37793955355, 65994.56603899438, 15087.696145453756, 85921.0359263985, 44176.4563432649, 3202.672421279029, 42322.93211581777, 84667.489335182, 32989.179953553125, 39682.98655419721, 79165.21399532742, 49922.495542643395, 96334.51703283687, 36545.15809847924, 72562.14228418579, 33289.03118286128, 81066.40988891109, 73560.51223080959, 20980.412720586195, 61022.637121471715, 68225.13759937738, 85201.00580349921, 97673.93438881323, 87432.11464272319, 5013.187330228108, 72064.42751041037, 20595.685917906536, 11042.52438092611, 45237.239950544936, 44370.489235561596, 58366.19764908075, 48838.75973065431, 98961.61218955465, 1518.812243781953, 88428.01251077291, 74821.8771593123, 55981.77356032666, 95945.14000912242, 89670.67620338195, 61120.7885151263, 97613.27693677992, 99230.53806404876, 17021.230183349413, 57466.6004733312, 83584.24875571081, 36770.596999594665, 48216.38199783325]} +{"id": 279, "vector": [33939.35381254847, 78067.39193725307, 36632.04215536887, 77802.94434971243, 89775.96989757637, 6863.290604634031, 8099.4098200021235, 31438.326646174653, 1849.5994830827644, 52695.82952688772, 18036.016809593846, 24106.800162593223, 97535.80298689172, 98733.83084378488, 78250.81244580724, 10088.376235582886, 86354.23102944584, 16907.628701658774, 91665.44671404913, 94609.47526866931, 36591.69249564397, 81496.10302842222, 20214.982734513454, 94356.1451178036, 32577.330666426296, 83136.10604415776, 21022.27440092903, 76153.70807607054, 18509.9028462974, 30932.47633185848, 52467.31922384202, 73197.99939638161, 36392.63237150121, 19919.891514877974, 74447.21329710656, 42775.020033138375, 94904.78460001807, 13884.163779837078, 42491.44583295622, 98427.69846521226, 79663.53038517236, 8054.739983683856, 77583.32505429453, 75348.65749841994, 16573.76441583235, 39101.93956644835, 7839.950193191503, 3798.5158933380503, 49782.797000090795, 38647.39946469863, 2875.999385448247, 24125.462360297235, 56331.22344889847, 88992.25783889455, 4912.2111143783395, 26693.70536694453, 18058.10238857214, 95819.11656374557, 28431.427447696078, 39240.382008171095, 31107.20328653557, 42015.45983309527, 38023.89468412081, 19436.990596897085, 67511.30609175959, 9363.672773552933, 47785.50554038901, 52242.716693771006, 15843.634213808577, 88443.90321168618, 46211.00927032508, 42649.304076499306, 83497.67283646183, 43772.78505871264, 21035.53861632499, 88914.98871107212, 60647.52393727877, 40431.108501115195, 19150.828776943596, 67041.13408837735, 23577.104312652773, 36804.82965733808, 7406.613536642115, 39957.51277240112, 72205.33311960488, 28789.78786240969, 96117.14944603255, 74846.8278965883, 20530.16542716646, 33565.468664063905, 62107.73698493307, 90687.38586079906, 24767.922392437336, 5821.581264072406, 42310.310396758585, 72591.46007601453, 53289.656989940995, 69517.70538509608, 7686.63292880668, 25277.86923749089, 57602.312181576934, 88271.63621346625, 37947.608653416464, 17007.530040882324, 4202.919547724094, 54318.73013686101, 17480.102195271673, 90577.18588793455, 23922.31867900834, 10093.965018993546, 76430.57971862682, 96698.46524180584, 30201.85292315688, 47426.41457013991, 39170.8557992978, 32492.57782902434, 62932.96022408954, 86268.38768117223, 38839.291620442586, 91238.72875463452, 5431.911776133235, 15197.382093681, 44292.81978314594, 92350.3658964941, 35730.24873116464, 85652.61220731681, 99445.02850590611, 23338.975778985226]} +{"id": 486, "vector": [85167.1032858942, 9499.44138537534, 26733.780085682036, 35925.69424659728, 19643.05660413359, 60168.22069055931, 75538.79621578334, 65583.97170363585, 77250.7351032326, 5534.661800449891, 14114.14422719507, 87517.8135467041, 69575.42200201088, 65038.93621805575, 98186.43653136856, 89927.48522490701, 95513.99040521086, 37097.60928125448, 13138.903577681016, 72678.8923074118, 49436.888992008564, 88552.50450262203, 65692.63679127349, 16783.225255450452, 49360.995830388434, 4925.164499340884, 43961.800278514194, 63055.31204791853, 73280.90601743624, 90274.65672998411, 51746.49626665234, 10183.546892001172, 17048.685647212904, 45404.319050594066, 11501.152217018895, 33836.65737986966, 91188.27920606293, 81712.15952898064, 68739.2809759374, 95188.80590578303, 15614.947381340604, 18102.54241691396, 39838.395748715826, 92301.31455603565, 38858.62646724403, 4274.92368788982, 57423.495190919195, 3475.82245047785, 62069.429384031835, 93386.62322617987, 15239.537106206102, 65141.176611975505, 66377.46619683325, 78541.25782548691, 17336.04599319668, 9474.746811084444, 39545.97464188626, 85918.15088147896, 22274.184905266313, 34138.640172178755, 7183.125157406367, 27.62579289974365, 75628.10284300231, 9168.644180722385, 22328.904399172257, 20369.507447906264, 71974.82431919032, 53609.27070719044, 46095.370305442186, 56354.077350900814, 9526.147473169, 97150.1850070963, 77024.28627480073, 19674.672472171907, 90235.87208994312, 14590.008698435175, 32016.525340210355, 20972.051461650277, 6818.070812103893, 96478.51638447402, 25861.511570597086, 28352.00959562135, 32940.58444061649, 25619.64354645092, 33872.37423566738, 75840.74871402633, 51795.68356879337, 7137.198379845233, 69989.15415663352, 71798.0240027705, 66299.99926508023, 73868.75958106705, 89839.16061195814, 90602.01625014805, 55476.291145167175, 83306.60813637289, 59418.44442739521, 96154.5637926592, 46319.91487415661, 51779.78318365078, 73180.45836325211, 18965.54905701009, 55388.29292623105, 69448.72800335537, 34513.62586159411, 9998.088021906693, 14137.866859050719, 97616.74382235574, 37874.86455455155, 34473.29343296991, 64306.583998718546, 75702.22200919334, 57607.57317033283, 80459.02337712958, 73602.29227996284, 91544.03167150404, 69127.47203517136, 22749.031051915823, 72636.93755486439, 90721.54852223399, 69418.04607202925, 55467.3211903566, 90202.15994272372, 64835.757200491506, 68271.54302997632, 96865.79041239757, 41552.08378560215, 44765.493520351054]} +{"id": 229, "vector": [67529.16411211806, 87996.1031650467, 34496.65225192027, 89174.23238725912, 43849.619045570995, 31893.472878606153, 70252.29419801211, 43791.26994086832, 66594.51691491617, 11038.512826801105, 433.44733653393195, 51670.51882350666, 90953.33816912698, 78886.14128515248, 36043.88833026586, 77134.67414277952, 35076.65438371257, 87885.32034327791, 83870.37246577672, 29967.84627064668, 10495.593974390405, 38321.512493398666, 41580.361593008085, 56845.34695832938, 62476.14195769472, 40670.46271704099, 72123.8203649433, 3546.433445235808, 17324.452036860606, 96570.93535758807, 52725.654134798286, 48010.56891257867, 12180.589359280502, 32981.18675295254, 96578.0788969448, 80521.6713110934, 96036.64890079206, 5485.369798994299, 4785.845059390248, 39974.9630096783, 8867.428376560527, 9843.441804597718, 24083.052010499894, 1596.2092287367336, 20658.733796060525, 12727.565771081472, 69687.70824727663, 72408.88747557979, 55120.68150631125, 71375.83291195893, 91195.94668784317, 20782.06872474082, 41551.418959087714, 93868.23494457385, 60434.998354931755, 22764.684725639516, 27864.682068584214, 3189.356990807013, 39070.64741388392, 6967.258513130081, 65858.28781455905, 33204.56997942864, 64056.9441681365, 57465.78288928667, 86619.09191870702, 41124.23866556072, 92708.6475956378, 76376.02716154422, 12020.939549933151, 59232.22585183696, 50846.796856263885, 35530.45004861704, 65099.14070312256, 51670.325690902064, 84827.8442608166, 89439.5361016688, 679.3613131627518, 14648.21130203332, 1843.8837599727865, 20498.30977758298, 53685.62695941452, 3715.6321112245873, 72346.80809256103, 42032.0452719895, 29440.54421927802, 31444.114371435295, 33305.274780321146, 38396.504745128776, 80018.76344629619, 79451.66251288401, 23642.77954824402, 91313.21856164213, 68773.38624786498, 92580.27549496101, 57094.94867870988, 14070.854910059948, 48375.47738268002, 20773.233435062833, 84577.92882435437, 59699.733973307346, 98493.79935275149, 415.0625844813849, 38974.35921898919, 729.1583046760475, 11034.87298370318, 16732.91616594096, 44726.60061021788, 18572.807343095978, 9472.99796767318, 2672.4314880036573, 54952.10862368736, 44083.14035501283, 80702.0882259989, 10686.965683146587, 75264.5777006215, 31048.856516620337, 18053.923783527305, 40994.453368639304, 98632.8345751868, 74891.69417200064, 77763.17194025868, 8193.848310676289, 8563.515928060151, 12671.974547795695, 47887.121746331795, 14289.997893818463, 32938.689998178306, 15672.199905448648]} +{"id": 1698, "vector": [40601.43041985652, 99489.01931968605, 28774.660373760584, 64066.020765577545, 61280.162886890954, 47813.602721912976, 3785.9327286101197, 33217.96271035661, 21422.535612860116, 49347.4720089471, 31321.56563058531, 72207.46815330866, 21936.245412140965, 4039.209316664505, 23274.740122954805, 93079.3887346752, 61143.424598625796, 49525.564355370356, 42099.857244442166, 44973.984454437144, 35930.37932208939, 82726.08941178833, 68522.44064963353, 52735.13863545881, 5524.016996321401, 98439.06526768967, 69664.8207205391, 60191.674400116404, 89924.29056105865, 11654.830410786642, 9425.869640833218, 2237.1020555775444, 17792.155272903066, 96688.93088606493, 25088.345694838175, 83351.2487269926, 89946.86629905336, 54215.03766373531, 25426.4059679727, 84937.34022570646, 66834.94800581022, 70568.57876071992, 40460.55835696893, 67783.37074793616, 171.43926134344235, 35685.95134074328, 50097.12616065412, 58808.68845290846, 94409.43487812982, 34056.70575951162, 11411.710754410176, 72564.87074855878, 28702.520215860328, 18638.78382933123, 3718.318626599804, 25713.086995374946, 10836.109159031581, 74333.9046009058, 49935.13927057767, 370.83371489009534, 14080.030535025357, 92671.20244550226, 44957.64112252654, 91826.224863888, 1263.6461915337293, 56080.79034513862, 38791.067083455775, 49393.5490007551, 87392.5158186855, 54532.54078942539, 77599.35218533486, 61489.32256846198, 94035.68822324328, 63071.781505979285, 52958.13050325028, 91199.0205710908, 26547.064285726185, 84003.13055965611, 56914.022939366834, 10586.088421660012, 57843.67340949713, 70065.41595889938, 8999.373696357705, 2547.3455012937184, 40347.93340750323, 50356.026979515555, 38817.28216984709, 64339.54726245889, 44343.07850648799, 80353.25401588771, 96386.20327376053, 76780.8288724555, 94050.10801711878, 4499.213228249654, 33468.920746103395, 61478.25866266975, 84109.33121861164, 60441.0208239628, 60109.847070825206, 7031.400813251065, 1918.4604949288419, 12516.596213892817, 45752.547983271616, 66594.17280239378, 46915.783920551046, 61137.21285572485, 90952.02419555721, 17149.175160610986, 4906.374495112375, 88585.68200315534, 96132.81172715344, 44245.41829172157, 2779.147462319198, 96260.62106860452, 73803.16189206362, 97252.78544741575, 36405.70286581046, 17847.695354582625, 65653.97449542594, 93297.25329552428, 72775.74294437825, 88990.78693319393, 16206.552435499078, 63566.70941794834, 61555.90810202095, 27164.709876008543, 56908.38943868216, 75191.95067953458]} +{"id": 757, "vector": [62732.47293478673, 94876.56421587837, 35954.19280554771, 73778.16159365742, 20948.698913590426, 61515.27178245251, 62513.34237539979, 54737.67486629483, 75942.36597310993, 48657.918484613874, 66259.09037808109, 93375.38613130964, 55632.64749510234, 15354.397959702892, 12426.559691960749, 70122.38345242856, 44416.405112596876, 58993.08496536323, 69978.655198004, 69884.48713654914, 80156.35657931476, 25086.51419795833, 14705.333077739857, 76448.09818794523, 84748.85548229225, 54189.70931853698, 41730.501888340455, 7034.064782888916, 30487.847686168847, 73710.74364845335, 98240.76586982726, 66503.27526378594, 76945.50686395557, 50356.11451798931, 79400.15142717205, 16124.752867832214, 48445.64536637327, 59619.6441803864, 11196.900151267964, 40150.22857653423, 7036.81045301886, 84717.56080772752, 85717.50975427117, 76074.29236001823, 69755.31474955991, 24956.104155834026, 61988.524724864226, 9888.381660348821, 38396.11417639557, 17606.91880371419, 23460.57806811813, 47494.13726413692, 39575.329337686184, 34522.86445521555, 52368.899861640326, 68151.39016692467, 22455.75340137954, 17860.306446122133, 38606.819821664765, 36015.87978991431, 77215.52523185046, 85894.83408584424, 7317.181983424659, 9190.979356445383, 62850.8955667315, 19229.695639142297, 14919.033426540873, 65209.477728722464, 71238.42766472728, 65219.970291594735, 69512.86973502087, 19139.724770688328, 88701.38939797452, 17484.28864034319, 92732.30920185763, 6736.120160230063, 17454.398867345753, 27167.900561537273, 96154.44980579928, 29844.930023331028, 23349.01770325316, 71688.26524381213, 87490.1572420361, 32785.36649628366, 91564.1124355718, 6560.571569671924, 97717.88310168602, 35094.005009296125, 54469.197560070315, 68016.82000168937, 5042.402000015489, 33966.9854236896, 75852.0494731344, 84632.85819754581, 89931.37957598516, 22676.340989790446, 45472.823437697676, 37628.41634450099, 61439.926556802784, 11457.943540851546, 10880.249753044713, 8751.8948175896, 60999.53712946727, 17304.51709093015, 67260.84326430885, 84775.76967215506, 96421.85826458388, 85213.97861029612, 59085.14115068042, 93762.56055511381, 66957.31947490822, 86922.54831913531, 57934.22170155984, 78544.22254296845, 28817.499886185673, 60672.795936635695, 21266.841647394573, 86645.08607126234, 6953.389763723605, 4966.073030557439, 70182.62587393171, 57085.42055277679, 40083.37890365902, 27992.072976472828, 68918.37605277421, 28782.562477872918, 25687.731099187262, 1650.9311547610816]} +{"id": 1242, "vector": [57564.93639582287, 7132.988127635865, 82048.33686158143, 37870.266448613875, 27556.075113386993, 42918.321851631445, 52965.6153582371, 53606.392350576185, 6972.428240614837, 26465.41616264284, 50308.62836960595, 85456.90308287523, 20960.385161815255, 68471.39391295255, 23559.76263321705, 16711.087698023573, 27000.11957831089, 90900.58000398609, 43607.5919239513, 86076.42558459374, 34524.11683028246, 34076.86555928669, 38574.302765697976, 75510.44713003846, 95282.10519693454, 96221.14146405314, 97043.53421236019, 25570.981037452133, 45237.55196175242, 92505.62845423922, 13065.415053947792, 69754.74960223291, 45015.61847022004, 51410.24159004947, 68373.2017171515, 68231.00691958994, 37689.31257643222, 63540.25383678198, 7102.920985805949, 69628.41006739941, 68884.26609951869, 10774.677128507337, 3151.977888920732, 1552.350318325879, 40501.20171958782, 34902.37641845868, 35208.93651236032, 88200.92120507617, 48485.73823374545, 84045.1865760987, 49054.06528928548, 65242.292454508, 42671.21026239282, 9377.16064510481, 49581.64408495581, 33608.15746553598, 37978.48524312644, 95227.12862549731, 23365.682127725708, 86463.14495250066, 32812.14199814173, 30202.441914906998, 70877.88591592455, 63910.91474698953, 665.7188984136365, 91596.57853833352, 60815.52489836798, 95369.99491448619, 89898.78352024732, 49391.511961601784, 13030.507201430031, 97569.30009346339, 53258.995981581, 81192.64670504238, 95016.65880923571, 18723.425327259058, 79416.23740201745, 78391.6615960378, 44106.361325000864, 10274.690144739063, 94612.30469451164, 4501.509892266886, 12704.403598658886, 86975.80847955657, 6214.990598451886, 76671.10455790794, 70781.37429970619, 96970.243192991, 7818.171553752695, 11626.403351607716, 65616.8446830268, 9616.879884487273, 98302.20508700366, 74306.71396613447, 57584.02659514574, 44450.75875858018, 29997.58913823479, 60596.76315236202, 74389.31887812186, 44797.96558597145, 61651.576004935305, 69603.99360841425, 96876.97484160111, 32718.683768925493, 36182.239208859966, 77140.50779982883, 35378.97282316277, 48614.09069300846, 57728.693404553, 15687.107007030954, 37500.493117939885, 84669.81947219935, 25849.967356173085, 70490.84483398686, 9855.78276020549, 35098.87279961284, 29810.184181338962, 51667.396286298776, 17830.809408751724, 71955.20028112053, 62217.819254908616, 9065.09611671723, 97503.62218369068, 76462.84645164103, 28228.94523400008, 97777.07808559865, 74729.59521630687, 67364.49780054623]} +{"id": 1232, "vector": [80157.62309006295, 57323.44647969355, 91760.13735758049, 37519.86150417391, 2146.7808745384677, 85037.93008795595, 96844.7123292831, 84908.62370105603, 63960.256888214986, 77010.10825316358, 16663.55829170487, 12837.943824791553, 35383.154724787004, 99721.48218521506, 52660.20953600713, 82477.96938110125, 38839.23646542151, 46163.11157822782, 53699.97416478791, 99384.92217245049, 20563.405473815586, 2512.6619647633406, 91625.63475654469, 26916.573933847754, 1270.0361942720174, 67850.74282954836, 20355.5954057958, 98309.21841782714, 26028.99463911322, 3244.5522588864574, 91094.7617568243, 53785.845897178006, 73090.24320414239, 88379.46911753406, 66118.18645833789, 60805.847386808324, 6326.165401396877, 69649.41954633154, 14503.87694230093, 81849.62318033108, 10696.99552920964, 69098.8488052131, 62646.18529447025, 46765.623222646114, 21878.94610446779, 62258.45716053093, 98701.16670789367, 70479.37529666843, 9869.773665489123, 92548.09280257853, 48937.825040252705, 98729.15228201634, 68347.7350033656, 41936.438045342526, 99484.05480130814, 53862.02513481724, 55908.68060046544, 12947.412554831695, 56519.67584276266, 40536.177600155235, 87.71579632977921, 38602.11068787943, 88438.77929950226, 80844.1817627537, 20728.928299091454, 29705.026341843986, 94181.07395024145, 68106.50491831415, 7650.862628217492, 17592.01727979378, 20168.013053636947, 50860.70071270015, 78184.77554824906, 64717.68463495419, 33257.794941476146, 44522.87672241942, 70675.84627661922, 20247.793013357328, 26595.865112696694, 12199.245362420663, 53423.49217876034, 51133.64143887441, 51893.0360199261, 33581.968271518715, 625.2574444856052, 87878.27710918701, 31917.286290700154, 40178.35269348944, 12368.836119221205, 16625.0977167541, 42237.596665279816, 90424.48958266267, 35930.981770065584, 50800.77201950573, 76912.05441711372, 93493.1382010829, 48167.51318365655, 60502.17618816992, 60826.11282745437, 56050.120252816305, 24123.009337458945, 49231.89989476077, 11563.742840893387, 6979.852795447372, 89424.68839637208, 89800.56595433707, 92110.23502428696, 10925.661758627713, 39153.0705122054, 63034.35899719982, 31052.95874287234, 90169.62631521089, 10480.642536502704, 93243.26469674174, 78169.16297760008, 12810.155478756835, 41922.55060466394, 5709.199726220826, 37676.88628367368, 33854.655152455845, 11686.717215321718, 64084.21238889771, 76676.77761093184, 34255.9381757753, 5870.634830969401, 33243.169243640725, 57215.38926691298, 21827.014738717044]} +{"id": 845, "vector": [77243.12847436883, 52766.994107220846, 28782.23646937792, 88319.87612753903, 39995.37839578415, 90578.63938306269, 33965.65324750847, 32956.70739609475, 37673.83151825954, 52908.11815602719, 89527.72525480244, 47348.63791591737, 21852.060248425954, 50929.71414798456, 12994.546387875416, 68909.45943741014, 9245.286315190471, 83172.53432469025, 57232.23967819033, 12371.012413280469, 2789.8385831719997, 62203.098288542744, 98161.41717105886, 12205.941665069642, 51110.44750251383, 84280.32374155484, 68336.5110088976, 20520.975330256373, 88762.03605714109, 42941.709353617596, 52578.4395923718, 58245.975037465425, 52633.22641662135, 57728.24596944905, 85831.96354775463, 46406.674894896874, 58641.36772578733, 32476.84373241242, 39627.85809830658, 794.0439012953493, 44875.69916800814, 47886.25307823621, 75918.32404820548, 69464.01518084558, 64407.532979463664, 19063.054453494144, 40417.35782091836, 66613.77615629834, 51763.58253494394, 51898.58178550205, 7208.842315348651, 61678.3505517042, 22482.425758392477, 8551.166490490425, 14836.623043546104, 27306.49794363207, 83949.076654817, 487.21609105610855, 80864.6118055092, 51583.19701312275, 1813.5427942663541, 25661.124143756297, 99184.96417087785, 92388.30533134856, 58177.11255143405, 89590.9070216457, 14137.55511438376, 39234.53730946883, 32549.85329145833, 37076.250711702276, 67445.87572759893, 67671.58507158853, 56353.64629055022, 71906.54788888185, 33679.75560551991, 58908.48946682323, 86013.8614872434, 95949.0571764533, 11448.665213406894, 89630.6188262901, 26918.912637420177, 5649.769373537628, 663.6027214611539, 42192.35290698831, 56969.83134730299, 58148.94227051106, 69843.92645961292, 78710.05361260305, 90983.5551682564, 71638.04170961415, 79456.62572426084, 87370.84426523779, 83821.12135456738, 15108.525063782774, 15533.20702554516, 89006.89085553325, 14432.38186132959, 9302.852621175827, 6905.554116338863, 93926.95850347234, 6712.011462550716, 84184.2196838842, 6334.735744800301, 58144.71403823269, 150.18929695607142, 35985.68254527973, 39966.9045668238, 75214.22616718455, 28476.5820694309, 35459.67939868977, 42130.685270663715, 19608.480096057912, 67621.15800653258, 48618.64731719097, 72317.98091977433, 3350.9783308535825, 86053.12593864792, 60099.82411829414, 75527.33248570407, 91672.17773629075, 94767.79336581957, 48977.00137949434, 70152.34765067992, 89983.99436274948, 62751.72306149828, 19452.586034867458, 36798.71136776018, 56899.05078000744]} +{"id": 211, "vector": [62298.95363967931, 94473.56592898154, 29303.575205263456, 80741.71806477927, 55955.91957185362, 26712.778673536886, 78488.2089544756, 16946.71246690812, 93842.62972533121, 44028.71480836189, 92817.60654608688, 5409.777011389639, 73716.12043297692, 95603.33319450443, 49775.47834038428, 16150.033154111443, 13441.951672744035, 24075.798487103995, 17653.15046006055, 98372.67730050419, 59822.49724820401, 89243.82693336229, 60283.66124585336, 76249.9451341845, 76979.71570228746, 48468.71494065615, 37749.482012650646, 57609.68781989552, 25055.890865099373, 96368.05249753084, 28203.88128593011, 76086.476065251, 73411.44524445475, 53220.37807266023, 7260.111460592844, 69280.69325159214, 38922.8796648841, 57663.29389998477, 24890.116891955848, 8384.553372057813, 33463.69739136772, 4446.807937699627, 73253.68681484889, 85329.40585371504, 64576.78238050138, 93270.70613755463, 12152.179144329612, 92414.17044208244, 55524.72723176738, 92751.63181116214, 23586.68762220513, 80730.34030339454, 61911.36608770691, 50373.747418298175, 62375.397256932105, 99021.85769244043, 90536.97542269186, 70000.95827529351, 596.2617331978337, 79799.42464586714, 6617.609766703847, 33816.913242591276, 94661.311522776, 67620.9820064206, 92484.85206823792, 80409.2148933741, 89702.98609480435, 31947.421646301656, 55029.48898943286, 81203.58549669149, 59874.58387628015, 36903.585618095305, 9643.306787296702, 84877.63702676995, 61493.11110791401, 17486.683253251333, 26477.79201109436, 93838.16551698116, 89608.05127861303, 67885.68893868134, 35308.80542460846, 21456.485860801942, 86980.20306466141, 20038.753646035202, 59217.59808376762, 9970.638397201437, 30244.604792711394, 59642.4560895909, 11650.28366192905, 27507.2790039545, 49086.13319301735, 58441.4532215567, 53388.018913293614, 49127.474418782156, 72192.72417830341, 16224.521239258871, 76466.88086381748, 2893.826213990647, 37685.3755854824, 39495.542560600894, 1628.3366265055, 37911.26673316227, 47208.00020710001, 82985.61079201484, 62631.76079900512, 17473.04519354251, 84516.09965250091, 60115.500743414086, 3065.9277782578156, 87841.35760372903, 83010.84271633031, 66146.16772454868, 18369.63153299921, 24653.235130511675, 4936.548519555084, 19030.56697368437, 26236.31134958553, 70493.46483326306, 39781.96797199891, 71057.23470046527, 98590.43547005665, 60571.49119227445, 59630.77734826677, 135.18255755521346, 84370.95052341552, 54530.22889662803, 38754.18353753892, 65584.77391085423]} +{"id": 677, "vector": [43988.2580515059, 1281.2758288312077, 22493.284961680125, 71727.22731225814, 23614.22262348709, 97181.70577442028, 70767.89585807716, 57760.21306663381, 96159.49355150883, 36307.8140361915, 25219.013782549137, 9140.478553301313, 73212.06400417704, 18258.64355538267, 50053.49251075104, 76195.64473304496, 41247.44648673461, 69566.6220809815, 79378.40010763398, 78020.97446221753, 71656.90814826917, 41714.396954955126, 11111.203042696881, 79745.49700823994, 30563.29616943345, 37967.922173463834, 47848.661914055614, 92349.13410035505, 63056.559240018585, 14773.896804589325, 9650.297432550802, 30180.904733882817, 12045.19830029267, 39474.38469885256, 55026.14797300081, 33731.48282594605, 42304.767915292294, 41478.670005601794, 78951.86750680969, 75928.14869284669, 61019.25423143716, 89209.52416580537, 45573.44854987323, 52107.250356187695, 64851.60321211534, 13702.098071418322, 26859.130148428634, 7999.88993842905, 48505.143290770226, 55846.53318183095, 45677.65458452532, 88356.54940218285, 35550.75948857745, 88379.9682679865, 17755.75162841022, 60884.33158408926, 45752.21251546737, 7796.635306008426, 55601.287942088195, 71484.45247722455, 40388.770382686824, 34901.79039577302, 31015.25237665935, 47441.39864332639, 34878.891853148816, 93154.97322079926, 49771.45385412591, 28651.388629807272, 44039.3177247529, 13473.557850824813, 7803.000325613841, 96388.70875234669, 61928.56978615589, 2667.5361696924747, 73398.64130341721, 27709.567515070645, 35821.11873372329, 59702.32808919047, 81529.2207523442, 43682.80878545391, 87718.54452803655, 12614.957069553113, 82362.1928796643, 17912.750079190566, 11220.199919581542, 61782.05151784797, 30575.54601623007, 79970.25906672192, 96972.55275875914, 66839.60337537329, 68565.17974227894, 89687.14998068684, 31544.261867664714, 60096.906009184495, 4613.352280016914, 45916.684906976414, 59146.77079740839, 74315.22715890207, 37498.20205331431, 72141.59109477738, 79449.11640411138, 17532.03993163005, 15665.385614637229, 28064.376003705936, 88737.79085125655, 93241.53882908629, 93529.86972830485, 78034.46626335038, 5684.372983009167, 95559.8811447251, 94355.85668141463, 44646.52449300667, 34256.05376452594, 4171.924059798848, 61816.50078906439, 12249.86808162749, 87560.30242105437, 41634.807798324044, 27314.620594576256, 41078.435894666385, 64298.7824942985, 44643.60999746616, 19270.108105288287, 42466.18324828199, 99951.58137109007, 83197.76736160138, 82228.28326040195, 71647.18819351673]} +{"id": 1255, "vector": [86123.14688294812, 83786.9843013692, 91196.97094788187, 94623.96066800776, 1385.6457533254595, 2053.7576162896867, 41470.90551279609, 73554.35028795565, 66931.52961382669, 15231.169304961279, 47030.467161578985, 70555.26404465658, 45462.80760970244, 62150.50228203383, 63998.0200832845, 93819.50676550486, 85697.77581396216, 43132.861535764234, 79562.19315520512, 46244.61322183493, 41819.596452709964, 35463.22903886051, 56403.74398667805, 55132.7715157174, 6971.918688909462, 56006.81849885997, 12235.02845416914, 1059.1487952563705, 55773.75941400017, 25258.489413388117, 52755.14411657011, 2505.4998646309223, 5736.873923146623, 82827.56813684964, 15561.89496164887, 71060.27336971226, 18208.334399348347, 94967.00022121008, 79714.82465975019, 68402.53405183183, 53784.730059275906, 97643.32939239853, 35248.05508871452, 53568.69946617051, 13570.790726113159, 39618.25938698999, 93726.73782310609, 37695.91396510124, 37291.73267394107, 1677.1359603074854, 55544.94917910019, 41682.806808269735, 30147.908168637794, 46748.47359233738, 71080.67875968169, 98705.15097766664, 66912.73575821397, 50393.18650228435, 87439.6383430821, 79875.1933261452, 95328.8740526554, 41453.372188849506, 20702.22955197023, 53219.38080947323, 97415.33226496591, 50672.21194887973, 90014.67839573228, 85236.76841582436, 77747.84543657691, 87246.43935977663, 80077.26823891432, 88889.53746802338, 52491.197377195764, 24432.879255371976, 40513.90756168282, 53742.25864184203, 18977.38069524081, 95864.3983430058, 23136.375755166995, 23882.75940477389, 13769.348965129979, 28481.936719791443, 78671.2194162191, 53242.754061457745, 83277.04231037483, 86659.34773415573, 1930.1338526926393, 20767.938698568432, 8723.953526377525, 93719.98704602031, 4104.632500418148, 4102.887896099472, 47074.01783938203, 32340.323808979843, 58528.3624374963, 11351.348630024693, 72186.38379340961, 39776.950492484284, 54970.37963436222, 8007.635590889262, 40395.823977563916, 5150.342747417769, 87242.96973179089, 71496.1965689255, 85596.37416716698, 75858.02755056528, 76606.11605412944, 77988.55486226744, 48395.91245591837, 17888.640201389193, 9390.922724758055, 47131.23454678245, 93047.03971086862, 49226.54852866687, 39792.930819113535, 38336.301013857854, 85988.47775394435, 75430.91471000662, 74325.51175180169, 68908.67211603814, 58541.12861851464, 20745.805837314514, 72408.03925159498, 71602.46349451733, 7827.618482766341, 79489.35738048854, 56217.61748208164, 63079.89567501021]} +{"id": 1336, "vector": [82344.8804238614, 17381.451335021804, 90235.16241035049, 74340.07546894516, 80414.50875013537, 80432.59542769546, 58669.99165705736, 77044.29572966025, 4520.511330105659, 69530.42969402943, 78819.18975686232, 72247.83804228308, 66003.15522625447, 3976.6866673396794, 19733.750891480828, 37072.81940727152, 22848.580575654443, 18978.79761457395, 75728.72354625809, 33051.588706496426, 35549.04754878845, 87879.53354440593, 78516.33175191816, 63360.03726017811, 67551.05234499832, 53019.29724706683, 30582.118418542104, 91857.01722318021, 17254.121856538695, 77379.96951407225, 53294.82160121259, 11416.969294788481, 87975.10922431345, 94261.05886706628, 95651.70694383007, 25411.57577189366, 25619.397883540507, 82913.85090793663, 95083.66388684667, 92007.63959110757, 85784.03368607449, 54308.81648308751, 89494.23014316977, 27421.937141760067, 9617.44261108074, 96787.65615578779, 35920.65551139107, 39102.29415159659, 64230.911726834696, 87661.81265912224, 50857.63061950337, 72450.0956569864, 4805.085143695243, 42672.79145527269, 1956.2197736710752, 40384.71363508286, 93195.5867737132, 64150.19167538559, 33235.169681857624, 92740.62274402066, 52714.51061583031, 76686.19920712325, 91998.97423582198, 82686.31062960514, 67674.22696394922, 61682.85326359535, 12493.475354198492, 73916.76305027356, 9074.31862583924, 73872.20360941337, 42814.80001430463, 278.78435156772906, 16402.076030615044, 24524.849988840902, 99928.41593091786, 21790.24946775736, 23662.360999808807, 15621.592463822553, 86342.28395628586, 82277.20550527769, 51624.55242531792, 57434.514771791255, 22895.494467718967, 76781.67029370595, 96954.89154742418, 39247.35585884, 78313.73340878593, 87309.83346358643, 51071.664081577524, 324.95915977037714, 50275.60764312409, 23435.10847892143, 59886.721899430115, 26103.02339807201, 56979.197924307446, 23048.126700382465, 46797.07470927186, 78089.27972675745, 33684.1749465446, 8726.154489715022, 75815.20383776087, 98732.32759503975, 57162.82654170278, 93258.95114885192, 66456.27083253088, 75083.25049868449, 99707.64728784388, 75640.10304901731, 37452.310327160085, 63026.62100300236, 27840.473171854406, 45668.94762196899, 69741.41293550082, 7175.730596501883, 40283.1178914247, 38977.965022230856, 22785.065581503906, 76217.69200032584, 55641.67357795931, 16256.435253576306, 8780.413844880852, 21810.951175927683, 26822.13114065468, 61902.62241271145, 81672.62790157124, 95691.20649691513, 73332.36320851496, 31677.438501088352]} +{"id": 1347, "vector": [54322.29414207401, 4194.283256825426, 65442.99720711654, 52716.83058845951, 69747.90742366425, 45412.25455965867, 52447.918232128555, 33699.693671695575, 18578.090047682992, 35161.260352614176, 7598.634171771934, 21278.975953498448, 50180.89844189314, 85343.16558319094, 43333.41493984502, 23744.40155520483, 41341.78648498836, 66634.66752932407, 2920.788651660744, 21583.66644343742, 89035.14365610316, 96122.02073554107, 76087.69408233691, 86353.5838781327, 34925.04878006542, 72433.09653539142, 63688.352167974925, 84938.72196646151, 57659.03533445741, 70793.27672538992, 81062.03362881896, 39795.51797526385, 6455.5974343876605, 76355.49359548544, 21500.192982443066, 80367.67331065398, 18275.745868028138, 99685.05859400125, 40942.94687211426, 36796.276149610676, 42802.31752823369, 57959.5999413257, 11266.913534315825, 79470.5034828703, 77969.98867013426, 15092.927113787702, 89442.0180124382, 41424.04630728146, 29172.536710160624, 37074.12439505542, 24927.126930710376, 43419.77834359465, 59064.17804674418, 59903.179282155914, 29730.24880420254, 23207.410259912274, 34219.78792500738, 23162.33309691027, 52350.627055763485, 2877.998162968576, 73223.29913746138, 43315.002778365386, 75539.72841768808, 91661.06053622942, 46194.02547978847, 44869.60353072928, 82534.43559091144, 18379.261974117144, 42828.71922205719, 75193.16682746219, 83524.12375975003, 92490.81007200171, 57535.56928884417, 27018.41645573151, 1044.8730652820038, 30241.025020869318, 68392.09511339321, 99590.12984966919, 39869.10586096034, 10687.888763983377, 90623.3005480993, 39288.03455005595, 79961.37524657765, 19991.736010230434, 9366.111102531528, 80274.89948017092, 90442.70179080301, 82447.6624843916, 53135.764373378035, 60093.43418815985, 14514.248350991533, 27485.148539083315, 3722.0984954764335, 50038.46977067645, 91864.04908771621, 91963.74427515893, 7993.582891303952, 26174.123424807007, 52626.10617916099, 10378.374840764249, 75247.48238582745, 79483.29089110662, 7531.931970800088, 85216.86968223148, 19060.175844139314, 33860.17582514359, 79425.49757363542, 22272.085370513018, 61905.55462989705, 66654.55084628772, 37153.23559076603, 59553.92287458927, 96147.68202592847, 67515.26055442113, 32074.97456745072, 78153.57381310638, 68563.8479767861, 54614.009533140925, 21458.042872773352, 86033.94198013649, 71647.03077033642, 37849.750120569704, 7209.17457054846, 86608.41242966296, 6142.416866188661, 57344.605242694604, 58959.05682707897, 57824.00221912059]} +{"id": 2032, "vector": [99699.2622400434, 39360.019510418984, 21270.43341201735, 35715.34316252455, 49572.64246786719, 63019.43353991866, 63728.5043178751, 46525.84790484596, 69084.85215421185, 95538.99113496297, 79408.17134989197, 55262.9084108361, 42466.47897496105, 37777.6753833892, 34583.85662403159, 12972.710196925975, 11893.961532190988, 38987.78752373049, 43355.13017418755, 90505.61012976465, 69018.26308820554, 344.6425489021632, 62051.51741851769, 11821.610008530648, 15046.426490580567, 54853.22472704235, 62752.79392484826, 84527.53701868301, 83959.67840839131, 65212.940714884135, 28655.061557758043, 13718.775039061515, 53356.14168252773, 70823.36716538458, 43956.254784143435, 39128.967640401104, 54998.20515517538, 12149.353089558423, 47871.780685083744, 8594.029812432535, 21662.75141513728, 66194.69125856945, 82465.52450113522, 43695.31667396653, 28337.197505023094, 5043.6601767801185, 6008.478565811571, 94139.10512106915, 82455.11290211423, 33715.53685201896, 60090.55494181268, 16544.11596797071, 52514.180273309976, 56894.59927262374, 87009.28598869589, 26961.99026604945, 89645.94528897191, 7483.856615244799, 70610.83198724285, 40522.64668916782, 78353.78481395959, 70872.49337952964, 87068.49139560398, 21526.84313309201, 58894.17026252012, 99172.14345207483, 18731.187625523595, 71388.17415205936, 34299.71842130706, 81483.05385450924, 11443.35575799711, 62592.11416012195, 37686.708068425665, 50694.11188076738, 25358.601796938907, 34307.327874735085, 55060.23782539156, 60185.2054524503, 23700.26412195191, 63458.6501281888, 25041.61917658232, 11680.262371850924, 70826.43676745011, 74883.56331606356, 80884.9178986432, 36212.352963957404, 98571.7294122413, 24780.08777407321, 23622.364933335815, 10477.43339187569, 889.3617842940627, 60472.74064869518, 11858.706317047574, 60920.99440653745, 62264.12687490582, 23385.17041447611, 59404.02184032546, 92507.32608418874, 93898.64506112377, 47837.47598889436, 1703.5374422869843, 57464.0125642281, 48275.28471757399, 81879.79092247118, 544.7359652468454, 357.6069856162434, 43114.519628224516, 83429.96438067047, 15888.138348018576, 62336.75198516959, 87238.90660435846, 35820.71142956606, 83207.36359836989, 84754.68141682279, 63885.91477326957, 1313.9928432602476, 60367.428394523784, 32525.515302873133, 21640.21029536789, 63180.961597416965, 39532.51679804168, 28500.926158259932, 20812.632203741632, 59007.32585158857, 46744.54270957098, 8072.646987246224, 58760.24113542195, 99509.86164703983]} +{"id": 287, "vector": [34041.14340536662, 81221.95787209395, 24640.38120488101, 68603.54753992317, 88297.1454349754, 19980.546100058815, 80995.64224065184, 7205.37255219802, 79596.63959713418, 49712.8794565051, 82008.7443168842, 42940.573355858556, 38028.50198612426, 1194.6300988491143, 40339.50122133405, 99432.38508635732, 33978.93405073873, 1849.6609691995402, 18319.527327527176, 36117.818833285375, 4874.728030429554, 36175.582863566415, 90560.0233464311, 21141.158989992946, 94527.2686204894, 8814.25356046549, 47917.495795981835, 39456.98577237337, 50013.251503282976, 25553.706181178226, 74805.14983095294, 9961.165615484824, 2862.2006774885353, 77712.14823227383, 80316.09841883833, 37702.25323800891, 61845.2544694811, 5818.808433991152, 34770.352288630645, 15094.671957877603, 28003.862329377647, 94263.0028143577, 69209.06176839802, 83593.04013554327, 20910.821453981287, 38380.61128389792, 62566.66506746188, 98416.10583340922, 86203.29352591645, 34326.64796627341, 98570.86386518557, 71781.94168689862, 96166.85562868541, 56179.846066856066, 65825.27156890219, 70698.81545237167, 91802.51386453978, 74679.06896800482, 57985.78373147494, 58478.89409763273, 88891.1187508201, 22164.419763636277, 49392.732415489685, 26092.61048209195, 92212.12556279672, 57079.89203332109, 10980.06034082205, 49384.21815647592, 11271.239372557784, 14587.1695787117, 50503.53329993846, 85515.30659791591, 13944.029413620063, 62024.15006087972, 96205.89473295558, 39274.55601106003, 16349.959754266807, 23759.35429245295, 61994.00756752865, 99262.21551121888, 85386.62419978341, 60236.20626015961, 625.7371418070479, 91153.67377934163, 10400.506179896396, 13290.008185178414, 82180.69580026317, 79913.45019481058, 94444.85299246367, 41273.1712305008, 29539.840727476963, 72895.07909255417, 25192.90547324694, 26074.241070258653, 67601.87428045561, 83564.83872677953, 10296.56112156785, 52697.283208686684, 54974.79749340088, 77481.57063958536, 3630.9187666685893, 7519.51668987122, 64959.838027172176, 88440.28101374255, 49984.205096243495, 32603.219981044927, 19304.299894697073, 36602.186190672735, 85219.50430984287, 80587.33459372458, 11692.572657821798, 78936.89023981294, 11091.368984932815, 32965.86940347952, 51388.20026047136, 46015.697738698116, 73129.2769901356, 82302.88696813381, 99704.21697403034, 15934.642625431017, 62548.59461567026, 90979.27875124804, 75119.50497470486, 59965.99154251354, 12473.716075940489, 28955.774373801403, 35721.47898154479, 28086.609415486284]} +{"id": 699, "vector": [59280.86834594566, 36855.43390113331, 82440.0583599553, 55765.17592176615, 39709.442457939884, 56762.2186324603, 75881.20625669314, 94314.29993513433, 66872.27190164257, 77128.54107166453, 93492.12708911282, 27032.5731598553, 34499.322013255594, 38126.841997530144, 80797.20052842631, 31812.227665050763, 61193.81522805931, 35136.48260163409, 80222.74850590788, 93300.2782931871, 73941.5875833422, 7541.186724362769, 69709.63342305232, 78309.15899387364, 61788.99267779047, 63930.44439447491, 91143.8626450597, 56790.0154909334, 24318.87362734133, 32544.763358158347, 72198.2319447401, 55694.27076625156, 98272.19305983873, 19897.7949111672, 89070.88316348306, 74200.83762913052, 83948.42487612167, 51052.8830967383, 82308.47550632452, 21656.158603354936, 77436.17077937511, 634.481661544195, 20976.920127457233, 84337.64768026449, 13703.533783979816, 235.40178338400386, 71882.13934907225, 46482.37947653075, 59594.79313232189, 97595.63412308587, 65268.93256495535, 37531.97821435182, 5878.33552624304, 34260.07847037315, 28664.015051003244, 90813.97093225468, 28193.123388781405, 9261.50614577822, 15625.156775117299, 86053.18415348344, 62834.928328503345, 44470.04333131065, 91634.46597994363, 15360.42512202387, 251.70804475440622, 20599.613769369575, 59052.380271546135, 56198.22114329204, 92803.2876769894, 38285.33953597237, 516.9569297574506, 52186.67025467838, 215.35538693060462, 18247.09991111736, 1794.7286698369758, 78583.9385032695, 42666.45466448225, 35000.25773942737, 61536.99723458318, 35141.3161021022, 64695.402845532975, 7816.441943699104, 56098.493020399874, 47728.85873138374, 83308.08796121503, 14036.610627345746, 67808.36488645064, 66083.73807540319, 47290.24137096223, 11570.130056808548, 55381.00567724548, 54721.084596673485, 17935.644646335426, 97634.07864013755, 9486.092891496146, 39624.87737417121, 7272.5709008559015, 82358.46142182623, 30522.009510952863, 88334.43410513141, 46373.243703471955, 33817.57757184968, 57246.7577244854, 37811.85625111725, 96037.51486102285, 47950.34818476518, 70086.2881310034, 68839.05657568086, 14829.744366218289, 3149.1945829875576, 70372.20673018243, 91752.39246881932, 31971.309478186882, 70093.1320407189, 76778.39606250511, 28438.465616747333, 60308.90034985651, 69277.86988236605, 71655.15886787383, 59555.36814476856, 18474.28564999538, 36982.700512335774, 25245.08548959702, 32034.909281634504, 57695.76239365352, 37696.12686786745, 76876.4304588674, 18617.367231235494]} +{"id": 2022, "vector": [67737.0637919375, 75818.18403639467, 39536.46832355405, 40226.489473366935, 99347.762672433, 91304.97490735022, 54953.00608424638, 62377.7884416831, 58095.416391764855, 62878.6381980691, 31103.809993195664, 10095.414251697131, 26495.73291296592, 17180.086002070173, 61433.50407101267, 4394.646526285484, 20008.064494712442, 50825.346511340584, 80117.8508283384, 34014.40560774213, 9608.49531272845, 15213.457317949842, 18575.569015197703, 57291.260691185096, 99830.55034914924, 54913.3960737433, 88313.20034460361, 23147.96027651552, 57083.661832759295, 52767.005817899524, 64665.432298743355, 14248.035621579269, 81219.20954979875, 15594.189638460564, 65173.146176449794, 16442.931452272693, 85436.18725255413, 72147.1191355343, 18531.58271214831, 70458.91754050486, 85874.46264944905, 9765.347767573185, 93337.03701966527, 88076.4368116559, 35357.77933521258, 90260.1612758099, 98081.34158452108, 55765.28051529682, 19346.948881501856, 64559.22404040257, 94221.72113015884, 83409.67077615003, 52475.032704420446, 7383.132064335951, 83127.71486681172, 34277.82348424907, 71133.73192527419, 45686.12999513545, 32945.96568120518, 78777.51371279357, 7080.444490845195, 77414.42230342799, 53693.42997769258, 84065.85801648111, 47550.34951670154, 22885.197744114415, 61225.713277010844, 35799.01739973267, 9539.05212607542, 10676.733033305052, 92787.56819219657, 72468.06426069383, 33965.33916702874, 41452.58066962758, 95004.64350024764, 35155.34904193043, 91081.35013874443, 8407.077425539144, 53964.77649457901, 42423.12063189872, 61928.66196279895, 87931.80756144745, 69350.39925092748, 44378.45977406319, 26647.683071388496, 15751.899590932084, 76011.64144171296, 34108.06552435503, 52287.27603661969, 39536.787934515596, 86389.92831370125, 17239.037486886245, 43953.88529501861, 20953.86673294041, 49399.3502104807, 56596.844564513645, 74664.19822709082, 94326.63813236458, 20700.054273894995, 85957.40401924054, 54963.60753626542, 10200.28618708968, 53500.98089488757, 25398.933213312215, 135.0631584171369, 35234.3677517227, 11288.229336443022, 13480.28091562844, 46344.15679194006, 90983.60501684865, 4748.679390505128, 21680.96811606961, 18370.396543101742, 69812.21000571945, 43137.07673353284, 5467.512396984475, 34674.1517158675, 8357.117473592234, 45655.75084289734, 87249.46327633852, 35340.330605197836, 80623.77809343311, 81072.10808483047, 39009.54500906876, 88684.40874923026, 30978.547459095353, 56972.3286246032, 61366.7840022719]} +{"id": 848, "vector": [83810.4988104661, 13250.285672594608, 53290.21152840261, 66327.24126505549, 58420.79921153183, 60094.856547546005, 97634.4579525315, 89508.65513181606, 97928.17780326496, 84404.90112868318, 9817.523793221428, 82895.31476077139, 42895.62337886181, 98300.02452645762, 50969.40028619302, 77036.12468853542, 4998.4586710764115, 58352.15010808542, 94563.19434270385, 38913.312525074296, 45448.04988372948, 20912.34188923179, 70646.93889715233, 78589.5785021261, 98264.40305243061, 41908.02608404486, 79720.58060752955, 27426.78120187989, 60280.49467358423, 35137.05009616693, 88835.0091604385, 16950.052133710535, 81773.71161374137, 87132.69428738324, 92572.08641072015, 7781.529370167861, 88129.98573761567, 67271.94740772186, 31220.070457908587, 54565.990252577554, 31113.588871970354, 57987.400624579925, 20837.328009581524, 28712.269846638304, 86337.53964422515, 36336.92873228379, 60524.33937041352, 76659.7465827282, 38955.2725823559, 68826.81633056648, 31344.41647624763, 82280.41613256471, 73290.77558000326, 38985.07809864096, 91936.51220237448, 34050.44357235078, 15670.640251089262, 95178.31850926552, 79985.20813305391, 69028.88503752508, 30861.35559024785, 98384.62173491616, 90763.86397116631, 83125.36339115584, 65883.37598276617, 99761.10592402049, 72596.7969367867, 86639.96704882872, 36818.58074427989, 89700.220508073, 30209.15540368103, 82459.68395941265, 16526.143689542394, 90230.38726215488, 13198.20196123659, 19041.151449091787, 15023.00855994011, 60127.85118811636, 5591.452585952506, 65229.424768653276, 10173.896434471064, 22104.68765899528, 28109.099551573945, 52909.989876603715, 32412.492225216338, 51970.89950275595, 84435.25785068469, 15136.99919741558, 6024.856428208803, 79921.30633809179, 56298.73146382032, 3434.5173520638705, 14796.462812892187, 54002.057493269094, 12842.113935676358, 32540.58427226575, 12937.889708631545, 8612.807917249687, 71768.65828987349, 26090.248483336876, 12674.581926096884, 11271.989171811947, 26295.11643344219, 96993.96652719546, 9119.942089980914, 57105.06816190939, 26741.337424080146, 68878.53849599081, 42052.26549261397, 56578.068349608526, 23365.10013948363, 14748.171071342731, 38029.25047626678, 60345.693261544955, 61628.58843756739, 8891.704435990589, 50509.192191665374, 50231.63182085372, 82090.12927842986, 39989.56208006775, 93288.6348036018, 20012.79150694766, 42866.67265595412, 23052.900065016012, 71659.32487330366, 5518.166158099325, 88383.76714468078, 67121.47977325474]} +{"id": 236, "vector": [88239.54511999147, 37357.85676661851, 24597.437484913797, 71150.48327231136, 74927.70959143151, 90724.65366932807, 41199.495754719406, 51838.07628913384, 42071.9912561274, 60800.63929331167, 83526.68046180351, 58751.83362858266, 62325.089475710614, 93743.53268317295, 48364.37955990837, 87988.26233650577, 20083.527436385295, 94033.902230985, 11062.015485776155, 33703.235468840576, 46647.258916582345, 58895.3983861331, 60190.631043110174, 23238.046653267964, 20293.162694341827, 70614.11845328347, 32505.98874607302, 98551.37979516586, 65172.82363363583, 72566.86744644301, 72978.87894985675, 52548.90411986916, 8649.198633843158, 699.3334368749049, 52269.62336001597, 70089.35069024161, 57232.21459416038, 47392.74561324363, 50676.6367135736, 23483.829645980513, 6604.297634891909, 10.796015277769744, 97401.8609127974, 60315.86397240115, 13015.263411323753, 80093.68307352773, 96770.4191697451, 65561.288669312, 79169.15742967486, 5656.916930979505, 68577.15425282778, 1201.2529464259903, 58579.7250359953, 67949.78523370413, 74394.48048298123, 53357.30195832971, 71246.96265916705, 27141.660749721297, 77636.06926121318, 50919.20781803094, 11968.152286842203, 37184.02731830054, 31351.945336631405, 79325.4849334386, 98818.61842553552, 89047.33106375397, 68882.33772738358, 4808.319609486478, 16015.671515587737, 9510.462816948095, 60291.111936734975, 54703.86798347921, 44699.47880601738, 36468.88161636167, 3593.423886973979, 41077.31904976294, 48767.04141268716, 89295.01095088787, 65127.51883345528, 24213.165737201758, 44003.0057997119, 6236.6720997779175, 8965.222620735913, 58588.5011155052, 79347.27126871952, 3797.462628224857, 4191.696415172852, 38611.135392813376, 67576.2654969583, 83978.8965534554, 17854.06141813618, 79546.44419581606, 39352.68294228802, 76167.32179796854, 16952.278060927107, 39028.07637759978, 70748.96395089375, 1483.58408334619, 31446.02893222831, 82962.83352003108, 76522.67883971272, 90046.33036595582, 42326.90815615675, 39879.023767417144, 82041.67836014404, 2924.616755585585, 33496.62088908798, 5135.317100407199, 17612.661024607467, 39728.26204226919, 45374.01696188666, 73144.34700984975, 96614.07945215446, 80806.83434648029, 92473.46687830778, 18809.939935081944, 14306.043745163732, 31914.97623179079, 3539.126179460361, 16077.070329367381, 517.1596395347477, 38161.91230540274, 23746.84555558174, 91782.08029877307, 43935.703844917836, 64640.54301227747, 18160.107030984218, 63582.92083674052]} +{"id": 249, "vector": [22064.126689359055, 81216.57159299523, 9864.37110027154, 22965.59639928546, 21711.997159924966, 14238.570528263861, 97368.6960007601, 46911.19900559849, 53802.36541973882, 68206.8305233328, 43718.35372572255, 55966.23880512604, 51054.69062174162, 18245.9707810552, 48149.89546903323, 90025.45703060401, 52537.75789230538, 53939.940199049175, 88394.94936943307, 73292.53558248632, 71571.85672545964, 20048.33544122542, 20485.133779060983, 33314.21263537382, 69619.57344235359, 5875.961473424729, 8260.968193252338, 64967.73498576952, 5724.263992817214, 46960.16999128142, 642.9389286266795, 41459.33006019845, 94894.10922487626, 74272.61941495826, 67547.26483103714, 33955.48074837202, 14090.74688792329, 2193.347692736669, 42851.47339780606, 38577.343898468185, 8195.724944251271, 50936.65204784457, 60899.711017162204, 17765.379204125642, 47147.331484175695, 84928.80888449268, 21281.22851624594, 50006.26256446662, 93894.37598918345, 56643.87505006706, 98226.93038459968, 23797.561978491387, 21776.063945944778, 57079.44424877478, 36007.1486732213, 38931.40571648048, 2119.170926322711, 2896.594266179453, 98298.07432700018, 37762.41663717271, 77768.83830168586, 56437.90878275826, 3397.0578385362883, 72008.25695326486, 14900.13510668403, 47060.317426777845, 19274.619126964077, 66720.74836568125, 92788.6520021845, 73105.46066926859, 53268.70365597997, 40717.201809203805, 92891.56933744623, 1456.4972550833888, 21056.631396921555, 59255.86141290099, 95394.75768108758, 31241.792249553513, 3075.0964431770276, 1599.0161348649724, 54008.414944142, 53173.72087129681, 73522.12836323207, 54026.812076927235, 27549.192381808807, 695.2040060907439, 90135.00584235268, 59310.24653321865, 55636.43082336698, 40691.99171513834, 14535.816730326213, 30901.841596483515, 82897.31395158137, 8949.58593977292, 94367.01322354426, 31873.18680035669, 64974.53946769441, 76977.26192399314, 99519.38428587938, 94752.7586482538, 49271.07171558572, 15030.9513971444, 99470.23895457404, 40824.97165903467, 54536.10449936795, 57727.36070275872, 80720.58678604742, 9505.423966484594, 19872.211026906494, 52909.98061420751, 51474.6595830267, 68842.09961071823, 9986.16855485548, 74392.03653297666, 77627.44936209121, 29801.199293244274, 90635.56727127044, 38090.772248154215, 8723.065588719748, 19415.916897183895, 93298.01367445433, 57072.49617569306, 78645.21754797052, 70048.81269710422, 74086.44337933922, 53972.134091281885, 21878.22001255151, 86239.4055591838]} +{"id": 256, "vector": [91266.62797384374, 49743.6371212859, 94476.91522723189, 63354.78323366866, 79272.79653550823, 36096.151950721876, 21556.475716200675, 64358.764321126735, 12372.662815980451, 35141.43783211794, 9888.31221913714, 89933.32131179234, 45233.84101837739, 67682.61528228965, 59982.237325857444, 83301.49562825021, 57639.22618643787, 13147.760321301204, 4279.463141438977, 65823.05398834567, 95431.75538014766, 72321.72224879859, 22946.909846887065, 46464.34050273346, 56550.4485185911, 61924.67571015179, 84411.28586007161, 57996.49508185938, 96069.56961487407, 26907.793388385326, 96904.7914811838, 87215.91834852185, 14734.844209454977, 5677.372714431317, 82024.0210580493, 70230.23193852337, 77478.76963101847, 34129.39343185002, 71098.64668627127, 14480.427060735523, 6402.52416532765, 43845.78041992463, 99331.84027876891, 63502.81566981738, 86983.31838179132, 1542.2834229312455, 11848.540527783536, 10231.96976593107, 92616.12376742304, 22669.085334164607, 66630.47504212573, 38947.94560621513, 83385.54252779395, 87787.61507255917, 33677.55809941159, 17497.97405536534, 27723.239642163855, 19436.378608391424, 70638.84798925051, 14394.129628602026, 41623.17834158112, 30005.568199491907, 58591.2518969541, 18095.436764492755, 96874.12541757232, 35939.725400520576, 22571.349750587455, 20496.064198782093, 2083.3840114194313, 44267.81331357391, 39087.48574411469, 63650.771637036996, 30564.96290641334, 56313.676229598386, 35156.25848282768, 42242.68036294284, 73563.94334098516, 18158.23482870803, 43862.22584133509, 26876.82693487493, 57845.94632364502, 87763.77844352993, 7172.256867871763, 5498.351270896018, 49588.55481806225, 6414.510895265491, 10939.581780297325, 65236.641077813074, 60873.50140165726, 65911.39765591534, 79764.83263641197, 46340.36429676981, 22573.32524015552, 54678.733310630036, 1648.7987952892413, 78127.24412120639, 18105.218631486343, 81092.23827362667, 87395.83611865276, 97687.25306871659, 27109.54752410508, 61568.674615364595, 75125.62853038107, 99456.94704953792, 15837.561792866418, 87841.34058840192, 11568.688218601565, 24399.67617277128, 330.00321793278385, 56833.748838953325, 41646.789057489965, 16156.012783725604, 34611.37830851021, 16005.658222998964, 2936.233593989135, 32185.591724037844, 49906.9958954906, 94973.38837549176, 87622.98850085516, 12773.341851235044, 80257.96417868586, 89739.71774707886, 77402.01849181495, 18115.247239369448, 82280.31121722313, 55223.57343961334, 7021.585880187142, 53551.97223844791]} +{"id": 592, "vector": [1326.7365535787335, 67702.46974615306, 43166.977793596176, 34852.12351818753, 38349.5176564377, 58362.899584285056, 23959.043196393435, 44337.82416113902, 81591.46350533985, 16477.999096832264, 1115.5407573532173, 96727.50566424867, 83073.23044903223, 48658.43590498854, 40776.081899262615, 49405.07242662867, 70283.25489849696, 44359.06523736443, 64393.35055589195, 38173.156903245734, 79744.59034014212, 81676.77168583582, 17480.25667100529, 51558.668717991764, 18360.26199542864, 743.403205363724, 61789.46782551421, 24446.52088070749, 30615.847047207622, 33615.844005197556, 51720.530928918815, 93150.06356753156, 16546.897354488523, 69430.86303562636, 14375.22942817423, 3397.657830339962, 34419.036370347996, 86635.59069831316, 87081.20970744816, 52935.639823023586, 46009.80354720472, 87510.87387846877, 23612.31245891462, 41658.51976457832, 86447.131744824, 35923.06628617742, 70973.51876267858, 74751.36234719449, 79658.50647222811, 53877.01287551113, 32893.352245436625, 50415.16916012414, 26974.490631598535, 43839.00566334129, 97099.67385738483, 87568.54996485943, 1115.9775178041343, 93195.62040199636, 19659.64924078716, 34893.95238954265, 77028.50803596665, 37311.46762668212, 24227.266003782723, 74316.74436441234, 42695.0453074021, 94972.25338110335, 81264.01166380552, 97821.08716827397, 12132.85112000484, 42001.76879079441, 58640.25846599965, 83385.17035959971, 86699.64339389424, 14155.682424690753, 58030.07828272311, 57399.68463062267, 83003.8124192007, 99544.98315040719, 33907.925968864816, 44779.61070486202, 61115.882700486465, 38560.85939749041, 22127.868584413958, 90058.99033824504, 72067.56376728875, 21033.94862787603, 50729.630335600465, 5536.970809067421, 63002.889193928844, 76885.45417079012, 75840.14637637236, 8395.972626432236, 45171.967819175894, 99303.27657016767, 43946.282882639185, 85829.95782545935, 58808.352764626296, 50196.35772406035, 68428.60005478177, 55247.81937023094, 54382.13460023204, 95081.76064513726, 24625.689227367555, 17577.554257196214, 37628.24315629603, 12242.956817919205, 54951.305794766224, 27606.360423923983, 77854.84813952685, 45622.98651601309, 29494.126994362414, 56236.795325868625, 13293.41880544943, 43707.763714030545, 69216.9081505827, 22884.955631599823, 56948.19243347676, 90377.88489974044, 75212.65308155047, 78816.46231665494, 92977.02135713708, 44634.97732156112, 20739.864963967604, 93875.73012290505, 52414.80210941203, 75354.85127159876, 76407.70353159685, 4406.987264695527]} +{"id": 242, "vector": [58855.50708195508, 41518.10794139307, 61570.08961379098, 88981.968142928, 31498.697816742606, 18261.31863802134, 28312.95348212505, 52067.20236339438, 84746.94832208032, 65091.014506713866, 8390.624215444364, 15598.14062435495, 55768.450794822464, 86613.03349466222, 99829.37927923012, 50237.6064253557, 79949.45689074145, 77666.03471050176, 73478.40417795113, 28273.59267896461, 89818.3163480463, 41738.94860341015, 89773.60132866938, 74777.5521512717, 75092.38860390178, 16396.832953615893, 86627.77910518892, 9886.907114788024, 90899.69232307971, 56585.63741589032, 51015.51271908783, 38481.26683441995, 99722.44577848286, 4798.60775242954, 96702.50918712949, 58502.76299769276, 63435.082451036214, 24491.865011066882, 75918.82704973707, 65756.68015690595, 78991.17490223178, 70788.45621914441, 89956.35940924319, 97749.52210792103, 78067.35489890307, 83838.20237936421, 9321.909327270128, 2419.217211407798, 77042.04483091107, 54061.40964933918, 74556.12175990104, 51797.72364261041, 8342.829991794453, 39542.88681720115, 57913.80909961986, 30373.25677104049, 59875.78347078022, 5800.879881400034, 65623.47581412668, 92444.74881883754, 26342.01517138709, 64853.63126280983, 34583.55367848218, 41952.268475222896, 57501.05620578622, 72076.16865729763, 56078.702906547005, 24490.43148427552, 7869.722469396212, 20922.169762825328, 13768.289084895758, 85036.3613473907, 7149.556702676052, 20985.924636792784, 7978.582355361308, 49979.48819992058, 10452.625592378017, 34916.88634866321, 88248.61174987754, 2562.5304483912646, 61388.104687801395, 25758.22479883736, 29811.853631835584, 76110.92697868163, 88926.87492457818, 39509.57079376664, 37548.4540688034, 2421.4859150841207, 8561.957956143995, 19662.463175054934, 96784.69006815023, 62538.15513069218, 17484.51367212549, 22859.059933030367, 34996.98582893555, 98019.11367719369, 74061.24847482692, 77076.69297324319, 46850.46412106685, 86764.79467051738, 73179.78581679091, 97431.44464893229, 44572.248540367924, 96416.31557361968, 80333.43353088858, 48059.190383799265, 92988.20225137161, 994.664639875531, 40230.18751892145, 34598.27036804531, 77463.51089121598, 18178.991251945143, 45043.91550384149, 68815.46477513759, 8135.308979370293, 65680.61216126716, 7059.375223715269, 48326.190485154955, 20158.441523603142, 60732.48935474978, 99047.3561209194, 54331.17788670387, 39989.389499390585, 3333.0591502746197, 50562.62014886938, 58849.341008958756, 13913.323081885343, 49507.59497468572]} +{"id": 1196, "vector": [45340.198397720975, 55321.38857965928, 32384.93812937233, 51004.291343829835, 37920.22822791332, 849.3235253046016, 30523.136102542492, 98262.3631439315, 74198.736265321, 26462.12730814156, 15830.868137028387, 526.5673177162089, 80243.99364320107, 51796.63168959976, 7793.818729058377, 82751.69699027881, 43741.288792871936, 60362.4364038887, 12674.13820846729, 77918.77143284408, 54775.62443511388, 70343.95518685646, 93114.22918637222, 19777.8618726567, 77323.90202417006, 38013.26181698226, 24262.50347088743, 28087.54003288656, 59051.358088559056, 1821.0923010788638, 13607.188153929128, 48550.424297364436, 37122.33766126599, 72113.09851408337, 26198.72914313417, 59768.81967999756, 63412.84530493032, 88280.83295344206, 67887.66102209172, 33057.72182660156, 89675.28747105594, 67501.63362065316, 38320.84324847177, 561.330177198005, 68133.18505474711, 74412.14726200084, 9394.649328969806, 39357.236097625006, 577.3843137562129, 69958.39140057431, 15913.175564838044, 95507.35521059505, 82169.6902417081, 28068.49617876942, 12085.350862475309, 99281.88948967119, 94769.4861016518, 95687.21421625558, 22440.5727291602, 27928.345594239923, 82684.59310879737, 99127.4035529778, 24547.456231235697, 60746.62430543881, 63574.56429905452, 37047.4725709419, 54386.45702228069, 52984.06961859676, 55214.85457870864, 94026.96272538876, 490.66689683743283, 97666.34295155828, 57130.8841730606, 15532.703986761031, 10578.68704414775, 44782.784873918805, 64859.00559555654, 95040.57187568732, 63699.16254920755, 52474.4036218553, 10395.812700989782, 68422.54637139525, 30949.06569200979, 51145.77006921117, 74550.02125896508, 59459.91043439595, 61826.35899386792, 27057.682737109266, 16763.836620360496, 25998.644631056068, 93184.12581769237, 59616.58296268102, 57204.62475321333, 80695.64665836503, 43899.04245307341, 64639.03254567115, 16545.222004259318, 45809.76426584384, 63298.8531170413, 91469.04615934212, 13558.806109542753, 82680.27601004235, 12827.453546942957, 9893.962535239898, 19235.686946969054, 13372.428691578352, 68873.31434621922, 88615.02426581454, 36674.57285407963, 59043.58680695859, 7114.922187393869, 73121.02389954623, 54862.066491743375, 57528.381690489, 53593.96449014052, 23711.537450090826, 78894.70550946, 12740.764281338468, 5640.874184165845, 5614.157008243448, 12180.899506058251, 21209.31375185896, 90.47209066844752, 2967.804910714722, 24380.756219640552, 31239.286455570815, 72493.12030904433, 5943.89874187391]} +{"id": 1998, "vector": [84522.53878084951, 94974.84248636017, 7903.513009933749, 88256.59885744714, 11953.255463477508, 16458.305438505082, 51440.2568049404, 58151.142342635874, 40258.0758949422, 98780.94077001698, 88622.08799167494, 39274.38533729165, 90813.12758367739, 11132.72046907876, 71119.4211895782, 27443.307847644104, 90066.09654461141, 3697.2716255445625, 44865.29296679806, 24364.876690190707, 52226.15998773479, 40079.091078389596, 65806.0669561809, 64444.34736842885, 81864.1705347398, 27591.41414422506, 58855.6562151612, 42570.97182665002, 47200.200607997875, 6992.119268253816, 64489.08547939784, 98344.48605065164, 71159.34235378237, 62700.4449426267, 84954.61999150823, 508.99482762524207, 35844.42753262099, 82693.96317962528, 10162.008252437527, 596.8678423856621, 47646.85516146952, 29425.85613043599, 50466.92914030787, 30602.8471950627, 86138.7271195277, 21913.285743889486, 32709.139564147925, 27611.976216601597, 26125.595096653942, 42822.29958609944, 23930.321730283176, 97995.60782924375, 59100.427084760886, 66540.1985215495, 57396.11837006952, 47775.09699379952, 3657.9629018544056, 48680.81016847352, 10893.353280542295, 73691.37703444422, 85322.94259150914, 78308.22545281441, 51499.48165067988, 25564.622454521068, 68519.9532094062, 88219.29407803624, 70995.0406243905, 32377.276703222215, 6216.016006414448, 22581.22408839395, 28792.79565817128, 27456.291197135106, 24070.60411960249, 90347.86997863828, 39869.9908367184, 21075.488363621575, 17091.04563243412, 75992.92774572506, 80610.87200780216, 12799.346158884562, 83546.88769317218, 48997.67069458547, 50072.856820965935, 33326.703172953166, 80574.48766719454, 25754.430257270244, 24357.333674875503, 51899.31505011792, 28590.871878090784, 26447.808185522015, 79387.56453624334, 99267.05406936974, 86751.0792153683, 37889.79131277662, 50656.052323140575, 91653.59578312296, 66372.2294269873, 51947.998306420996, 86228.95829154763, 30362.811421433555, 31219.129944701952, 32736.452132670413, 1270.8387366310526, 97069.76792787945, 43597.799053318195, 1366.4376727920312, 944.2792592228577, 36879.12922067902, 8490.652594632053, 26380.864949707415, 39786.73307757311, 57440.061192133595, 32200.50100821037, 66729.63976216436, 9450.590394443936, 68053.6487299407, 53907.49809587126, 4287.815678420426, 81899.32139393943, 7133.289466300507, 58234.17825692802, 17708.113605802235, 72485.00677159904, 39025.26556013023, 85191.70572417184, 62809.60523050348, 69878.95151655239, 38135.67938147048]} +{"id": 2034, "vector": [78439.95531364476, 79361.37966430541, 58568.698124122246, 34652.645523436884, 50533.591836551524, 24636.472485002538, 36861.94319207798, 38469.688449577145, 53634.807209756706, 95936.88117361163, 19206.74702414915, 34499.34630477165, 90282.91447502456, 69054.631717043, 22543.567601835024, 85978.21418046181, 58400.12374191373, 15280.398421420694, 64431.464980935714, 43712.176080114004, 7944.497157409458, 13072.623682289375, 3644.9126959082646, 74926.17608590417, 90098.09650183051, 10746.789518143807, 64768.47563069017, 11240.464404662642, 53535.72138468624, 34518.40927980949, 3122.5714946641724, 74952.71504907346, 68003.77658833958, 90776.03857740753, 51733.1166634866, 95369.48694233636, 4705.301544700868, 13744.257264585147, 9356.049864827342, 57136.0398851904, 62615.39669561406, 95058.46647551736, 34802.219451550794, 61676.06260246179, 44583.26422339283, 38678.831948400984, 2579.6654313677723, 90682.04224331745, 89258.28760059604, 56244.52108603566, 52607.196872034554, 84662.40403824652, 16582.638215051225, 13763.972009593994, 57214.67035377965, 66836.17270372884, 75842.67064344232, 30616.958709977727, 46513.41423303529, 67816.22109324817, 25323.865742580598, 70531.99502716216, 87696.01269089904, 35891.83187743705, 51850.97262201399, 63720.848973698354, 3018.1029354944267, 51952.975326704, 83857.59313717917, 50156.1486281046, 99302.22129575773, 97861.2609953586, 11524.732718611485, 31541.639566252023, 15879.33525352333, 31041.53096125131, 77959.41505898967, 23362.406888936315, 39936.2132077427, 11664.238245098157, 36505.652551686864, 61838.6699607836, 73246.40603443321, 99591.49452062244, 72790.75391505373, 73038.8758998247, 85799.32462963849, 4514.780014977071, 9602.598949250163, 36468.48758803483, 73634.81089514463, 43507.04545053321, 48326.56770126943, 15993.828833233658, 32141.05595900859, 45052.11889729862, 86531.95553670714, 83557.15481383074, 12615.135920194598, 91982.07912776632, 85706.87175581112, 80368.30968956766, 81211.47477553405, 60962.88744831657, 51728.96289098994, 17364.28073370163, 82828.82668951269, 30212.09958141544, 64149.63345841275, 35373.776636084374, 44457.61971570937, 63288.187191525416, 68564.01323187652, 62893.550030508515, 27560.94755778157, 96388.2571251397, 88498.8393248744, 67955.98054728642, 87977.90821359205, 60858.62477239772, 25931.149539034202, 30204.934451705478, 75071.6393489814, 2224.0464943553475, 23682.705888760636, 46023.57018382044, 49045.6720618187, 51988.39152307093]} +{"id": 587, "vector": [27420.933580109886, 6850.123548794573, 13740.518461322505, 7005.703656283668, 90046.62202368205, 60218.742533031014, 11454.104809059585, 56088.05106019701, 6585.139934606043, 12223.173034784151, 36631.67619725401, 94797.42472777468, 56244.94394173536, 61391.36155935279, 15163.44528626694, 15286.40137295264, 52336.030291645475, 95264.52807812368, 72652.55537296156, 38493.78160261403, 48995.417189288535, 46716.49917225046, 91688.63327693263, 17633.958120329964, 92344.7323463427, 12405.552164353805, 593.8796165923188, 96209.36978173809, 71827.63947008025, 36464.724893042374, 97602.04303722786, 45500.03844776217, 70479.55719462807, 86289.39920805521, 47973.29473668909, 15077.75567251053, 53852.80825695702, 63337.122940291236, 66265.63170119635, 89741.60785236297, 83115.83778422496, 64482.50261348317, 97297.87599547874, 71017.93996098984, 64228.68381284238, 33888.333677434464, 51273.169825032535, 87139.91045648574, 92436.08294077286, 16291.880997772756, 4655.709783074669, 50842.699253408966, 45565.21346455602, 44981.07854215111, 48154.328602485795, 32514.18964741586, 3963.6639392010898, 83617.20542079666, 88706.07885151965, 43593.34896274244, 22451.354105024002, 70277.11389519644, 96030.54098186504, 31034.29293317038, 58273.44982767295, 92986.487846685, 32876.47845700583, 48423.88393610998, 51877.764389942095, 35630.62356549696, 37956.4435642297, 82494.48327363378, 45957.226335950785, 24949.501912352844, 40199.280862065854, 29800.229916473643, 44890.69725220843, 92547.69868529729, 32363.82519411677, 78879.90133693507, 90856.0384761407, 96457.31091497003, 79531.80167310902, 30800.415018523363, 61492.25555704549, 30590.683819448637, 24708.826526471916, 64816.84085236838, 73024.7843589543, 70273.16880599993, 14903.320628007577, 2272.5538608206252, 82019.55407674695, 36104.18341019096, 80753.63814126006, 78427.22443423107, 22093.374367903496, 17252.476960747408, 25280.625934217715, 95830.63417226463, 65802.45178733964, 60483.32463478191, 78659.5901816421, 75593.15209775261, 63065.39359832537, 90234.28411837183, 56396.98951243886, 78822.68940854068, 32125.287561874316, 85674.04959125022, 27785.03767085908, 97424.20444152928, 99121.73946060192, 12846.827819048922, 52767.46214682853, 42960.54134374897, 87001.76985664417, 33732.902825430065, 3871.162336534806, 98017.95181992147, 49885.64795281828, 41390.35629291544, 81101.24872507027, 58359.38437583803, 86377.35214743871, 23544.747473875228, 97029.14570168883, 10744.354373673925]} +{"id": 902, "vector": [6645.030529003171, 97451.22192330142, 74075.11775466867, 17255.93749785882, 73580.48661617897, 51070.429770931136, 76990.30657108147, 6679.277046263576, 13652.582983308348, 70500.85739046542, 86943.75713806493, 33081.49735963257, 43458.580124967724, 27168.2823720763, 67898.60942842154, 99094.9979439942, 39424.48578620975, 31302.954429276688, 17693.068134180245, 6640.365452286723, 45401.23465048101, 66148.61861664332, 3108.801488836055, 28033.296822101827, 53609.48810579755, 36409.659665376734, 61965.61912995997, 13478.78762111696, 55237.98824070705, 48930.47703617165, 15209.526990634458, 65232.622242238846, 17477.517897988826, 80771.47027783965, 39295.747331011786, 64010.99799252076, 63033.208768549295, 67891.86636565713, 46560.45777207322, 14162.551421070446, 44299.86285605237, 91709.02388625665, 87805.06439049315, 6909.557447243786, 64905.73329742505, 25952.309489556235, 42836.396781421514, 2929.204923388373, 1515.8937366598236, 4701.990715875048, 19317.760097694547, 77996.87711990772, 85734.93749321236, 39661.3867337308, 75588.40666755466, 86212.59281934515, 30431.23524563912, 95788.34926292233, 20480.58776485433, 4813.331011502775, 96738.83616858737, 9251.564233730924, 78674.50830918347, 64800.091587501396, 61409.683618271774, 43337.44811972821, 59729.26740642728, 25168.95035638288, 21564.93508371421, 74864.19595711469, 25987.982127334653, 3486.494195235623, 42988.79920401899, 18753.490788152973, 2538.052088449139, 25030.52965899377, 30293.207366766885, 25885.351643410228, 70292.42167146168, 64221.92752353858, 55827.754471481516, 60319.06682791955, 10301.783150664634, 41942.5565720652, 52075.643904554, 26688.62077579085, 41911.167820599425, 80007.6037291348, 37889.901399092196, 66939.70534910167, 74085.07633032449, 94707.59468517828, 50448.18525784742, 28366.854120383512, 10295.833064649041, 47254.19782676191, 40498.566619667865, 9728.989530526289, 8212.249081542044, 39094.350655382485, 64194.47894860039, 12729.261881447874, 32430.496023273292, 98405.8590339849, 920.9032274123397, 69898.45703044912, 69310.03301544562, 96528.08846865453, 21616.674846290207, 95661.3510484603, 48648.933351816515, 41571.50498622145, 68777.33451341347, 71977.79467706378, 64915.40368756105, 32389.839780618324, 34946.19404257281, 88636.6733825497, 3671.314488675148, 28546.949564689083, 40071.29119387843, 19767.059535146214, 92797.74665850724, 33833.73888467709, 18665.776240241816, 77009.77575740167, 22406.246561051878, 59886.01721539099]} +{"id": 580, "vector": [71953.89232036553, 84926.80214286063, 91846.67443508633, 49161.01689931422, 14034.350030466292, 63827.38475526825, 70663.590538511, 38775.94173336659, 76698.0499129719, 24344.64250864392, 37631.50826527869, 1963.174506512122, 46331.254654925106, 97289.33995809765, 81281.05736704275, 98815.4098394816, 33388.998143441095, 73576.0496075585, 10918.681433571875, 38593.05176685568, 62867.12105145018, 99780.49929842909, 53808.4215471112, 55375.83837170881, 55371.554980037086, 33113.105579681825, 57115.13944554321, 61370.46051348081, 78588.735121122, 24035.867042307502, 32122.923616675926, 25996.379057051767, 87975.9954959112, 75249.01381185678, 53537.28707657879, 7050.026128934639, 67533.39051238482, 76878.89510573422, 8821.166548107818, 96804.07192945304, 55539.618050936755, 24707.86049803313, 97397.65013651122, 18814.156361317735, 75194.21395972473, 45146.96934728407, 86637.45304832202, 92647.77267554561, 40018.82196396292, 75623.29355296922, 79852.45351102902, 58454.65977955468, 50729.62509049881, 92870.42518054436, 52426.63935660864, 12926.081215402097, 59807.46177204286, 45099.6926440045, 35017.083548074246, 5731.24617141777, 50290.71951852883, 45924.27051807443, 40317.847368159695, 62026.92110207807, 68684.0686934663, 68190.40539687494, 70200.20749023688, 4327.113754038881, 62212.50764242752, 49335.22923483376, 56296.80175553693, 59475.51138664261, 77214.05458975026, 77046.26558237741, 88342.62395761057, 72172.82287267811, 26396.757762933532, 75480.54796871998, 81789.08652330672, 2785.783982073542, 62888.219336265836, 97524.83642191192, 14423.056950806302, 1728.665991354228, 69929.10131566519, 46922.246457222594, 91684.29243430564, 22126.503962749135, 81005.77950783339, 30018.04666533654, 50015.190220086704, 26570.938719238315, 89755.72290200494, 16333.524660097632, 24808.772055812056, 95746.24352672088, 35558.53636148321, 58769.367735426116, 26671.308371415525, 58811.58564001722, 30527.351779926725, 23012.6948728149, 57223.003454914215, 18499.986912508328, 79091.59709821356, 68825.95853609615, 73104.4978448939, 90014.53573211393, 40116.36676860722, 96861.57558294835, 14774.078570012283, 3339.7228151571203, 22917.858123881397, 52111.7997932196, 19599.875022113134, 42721.62140736854, 26017.315440241506, 33923.60451232597, 23741.50690241046, 27317.899812847125, 11543.80892959228, 9503.545302243, 54072.54828575767, 67982.86570976951, 61663.39050221101, 18917.884427978603, 53247.23649750701, 19970.772676605942]} +{"id": 1853, "vector": [72913.64042735289, 46759.854630035734, 92761.93151370478, 51040.000116427786, 24938.752338247938, 42412.80161149511, 6743.316229117846, 75763.04642135561, 10409.082207607235, 38325.456246437796, 41309.49133923619, 3279.6762748069973, 22664.01283024598, 87359.31316609122, 57722.91430764404, 66412.23000023625, 18858.04252381741, 89435.20637841527, 70976.93681725473, 93741.2661939623, 69026.35871249918, 43168.566780472036, 81553.37245806979, 55243.050361164016, 18194.614281571652, 49999.96156649474, 83512.82516019643, 20490.343466771, 67273.76235136064, 67955.13663538407, 57441.97244762186, 14716.087257858479, 58330.52988714492, 66189.69344530794, 27288.891636500623, 46751.86118146075, 36934.02361670517, 63019.12671920526, 88014.79749281776, 10168.554736649283, 57884.426625609485, 50547.86746857964, 44840.85805325464, 52617.73973392324, 41089.32916968051, 4067.0681282258925, 76119.22951425651, 97693.04263942807, 84423.8789048596, 8759.616651471446, 5311.19502061903, 35998.96316320821, 43809.329965697376, 54780.176217738175, 32825.173265242745, 3652.6323678422013, 65542.39503231637, 44799.26934125774, 8244.575444990533, 71342.10404512702, 8024.222542198545, 59875.40695165322, 87752.5878903349, 64260.05159795869, 1999.8502874749802, 27250.465691454883, 94189.58698514246, 59514.63039981485, 93663.84193465047, 97032.87975640716, 77189.67050689882, 57901.590687416974, 22223.687302521343, 63092.957615675994, 31901.714229916335, 45244.63690331456, 18098.590821566373, 62290.139400575026, 80026.8284273666, 81698.53629027394, 57523.96077153903, 85845.90999733676, 56639.12601764912, 13019.113398903715, 92445.93163896938, 89856.4496371856, 14354.276539883393, 25229.449371157043, 55746.60953818315, 85161.00757702078, 50648.68927345858, 16520.452743043457, 47233.891774444426, 17084.377914786954, 89686.67457690925, 3771.868618126617, 18983.736048630893, 68771.24189087412, 60301.637751179645, 21491.908296438534, 67347.53964559008, 49220.585024655586, 39909.45562161252, 42609.95018904979, 4947.056140879425, 49767.85728428328, 4662.762867879922, 1250.7536668747155, 82909.80756719298, 87854.25531720498, 67301.35696510965, 45402.05929355571, 13471.577840512016, 97805.27434198705, 26432.718532962095, 32796.41001618062, 11038.71310157728, 53547.52579135903, 66760.11903896517, 10658.61615363769, 34382.76552933283, 4102.4818844161755, 44665.42266681196, 11264.563472320677, 13757.993470814634, 62656.26525712323, 49462.01275826449, 60160.676623027335]} +{"id": 783, "vector": [75247.37479673854, 49259.85872999864, 64000.33979863029, 74562.48423158652, 97416.51409129902, 61644.12772728657, 21721.116424257114, 92818.09418884187, 13045.383973208101, 8896.549747802717, 2091.3680722640993, 91569.51056275846, 92836.85916313955, 70166.90422986679, 58828.23051038639, 98366.80228392831, 93543.807357339, 73997.15286096872, 96164.22181325727, 17690.83297975077, 11016.167584651015, 96872.96006404376, 84184.87669022704, 79330.96263839435, 46936.86484283105, 72918.1753177679, 83199.34032255823, 47661.627481017444, 11355.804371330902, 63607.852833868914, 95421.87502311627, 99537.39028941515, 60119.42103484551, 68475.56127966313, 24369.45761172863, 47382.06569235564, 59578.716800487666, 86048.77237875419, 81054.84772367903, 68920.99149509757, 82565.2989463067, 56053.044810240135, 8658.422558502421, 78153.64934693831, 83647.01210045234, 51653.900392465104, 75487.7532655627, 77963.05063317012, 49412.69188716558, 78705.06606973158, 8468.052298819972, 41511.15256362011, 27228.9979360102, 8096.309129831658, 8777.968927189617, 80753.71877596733, 91106.13150323613, 12262.232453382072, 31771.540702408653, 89517.11890710743, 42748.7766593859, 677.681406237185, 28500.070339553775, 42948.89194060511, 85192.52816346702, 10425.674416782038, 80157.91900003233, 92861.97590048896, 61502.03685495925, 29532.050569985124, 26146.421989426926, 17306.858356145094, 67060.51298019083, 19882.283062688257, 98739.22283649596, 46974.63838754837, 107.22205831623643, 76840.46791782096, 32645.21427718613, 54650.008994020915, 93184.47220598905, 3499.2294729320083, 43807.126568910026, 33965.86393441606, 60985.78749029271, 63331.55025517863, 51224.32524267842, 26112.576338255534, 62842.81088416277, 51122.87833377578, 55268.57652371198, 8349.762450267317, 81595.98170180302, 45233.44156479857, 88836.62614396805, 66019.33592616806, 85699.8653130195, 17471.13091404605, 55776.852492535254, 50555.21756284913, 35818.68603661494, 43167.19352186347, 73466.91300143127, 59945.98562915314, 56604.91197134296, 52151.116970586234, 52568.404478615485, 42808.301979142874, 94600.75949118024, 71910.21954991808, 21122.77380813593, 81634.27166618344, 8376.273293112357, 90428.08972734613, 67882.42029507428, 6426.520097191579, 92203.49189446849, 49322.02327446586, 77645.752176018, 18984.22817062044, 44672.20278628237, 55898.859185350746, 87095.43736353921, 40103.10001033361, 1334.7488874061119, 71716.39213953291, 57583.11855449754, 25346.47250906099]} +{"id": 1250, "vector": [65970.96152856105, 66325.17960658856, 42689.19415070825, 44204.182215162466, 27868.15090750292, 73331.3914561371, 79788.38469649634, 57197.93285750886, 35018.55108162897, 82843.22571289976, 47794.60814674386, 44618.96039120123, 23419.62118886338, 30927.871343244828, 78736.84951895328, 468.59944285232433, 92883.96615424022, 28355.333220259094, 72761.62518173996, 15674.605146864662, 85353.74183663426, 10943.323648847736, 67906.15755177927, 98136.05041862217, 74269.72730062691, 24329.5553769882, 77578.9882565956, 57121.9965280924, 19211.182278596538, 16474.127017781637, 80117.4463117902, 93973.1785655709, 649.3403081744065, 57723.64834492227, 22162.30009743121, 80169.00985566186, 15979.247911932003, 82211.45282815388, 74598.79682700658, 53907.7245394101, 48164.13463687449, 44581.0761978621, 2318.2641510708368, 45080.50565299451, 51546.318209597775, 13373.876711472043, 27831.871614860094, 3653.5216368009383, 43262.803971643116, 10479.199555990026, 85424.41027419028, 84697.43617448786, 63210.388782477035, 50358.342946735436, 4355.779670157877, 6896.767710916884, 25811.167927105973, 88291.30083666848, 57466.63924358813, 26311.94075410156, 63934.088659701585, 25692.967824842694, 49276.94056661292, 9105.947611616915, 63896.68256268731, 99191.47988808701, 23602.925987790644, 54891.17358686351, 21116.53142578236, 45395.61366707675, 34380.94023364741, 63282.57455752524, 5221.15948429277, 42453.93071160686, 96586.47986058908, 35268.111871518726, 48743.52236526735, 22180.797831780095, 37983.65795817299, 64326.5843027606, 37368.5977679344, 29496.78452768978, 48561.23486303482, 92869.3696709386, 58374.59834617771, 58263.5028742639, 84985.16664759349, 77221.47628751979, 71082.70644860796, 91667.39880978627, 78026.59943857994, 64550.13639669173, 62326.802114651655, 1200.3092989197994, 81968.74740308657, 4530.907081872116, 68574.8378808237, 74490.97856639956, 80409.66680737362, 20818.61815206427, 72891.66147680684, 78875.12041189092, 11851.430485073888, 87598.2401454028, 47502.681556521224, 90559.60384001023, 7681.492587144378, 69149.03650951164, 58055.892446349346, 24020.505408882997, 3432.258045405534, 92298.3811529177, 31590.30445208094, 62570.34318314803, 94661.00521736777, 10559.124466778469, 82105.20705363747, 8496.554098110943, 77764.76195092502, 17433.71151306634, 81545.5854286658, 83957.94996880373, 74159.04832234714, 29632.19802380328, 14498.189866519373, 81908.49256599104, 70085.66012528526, 88457.25731459311]} +{"id": 680, "vector": [969.0416298341776, 77960.13503365862, 5809.107765358112, 77692.82088741333, 88676.73111290458, 25910.688852484887, 79263.36979654105, 41960.66957364193, 21432.151658962695, 54857.8765306636, 43445.0725974719, 20688.626708437307, 4356.777010809865, 49432.827186732175, 18795.837271819615, 92245.82823136378, 57355.391771569644, 4285.274791481986, 67799.3953756604, 64251.29667049959, 32894.85147312934, 54061.56078425411, 95274.34607049628, 84057.53385155452, 94153.38207214205, 79723.62867465681, 28414.05051327922, 89881.43617878806, 15219.009514663307, 90529.13825634164, 79718.80993937074, 70350.49932372039, 13991.866321905689, 45583.07896662411, 28048.266287509006, 96457.80810873971, 56159.90273284171, 34553.9562157491, 70861.12805971502, 15192.139076368438, 91517.0378511223, 94661.28318914742, 96943.01713302759, 64709.0295805224, 41059.16536409814, 88406.74575494963, 29816.920252162083, 99742.72086994468, 51513.67987817936, 8722.325639128825, 19808.493986412646, 56185.70231123994, 90179.14458401658, 37147.46399956639, 81834.22082454638, 89872.26031797445, 65920.54558036776, 16631.58913882704, 70875.554840099, 81840.90899493896, 86576.70852676971, 70066.0347824932, 16867.571319159968, 2159.925946770469, 85626.90746891781, 9604.702182420287, 48173.2830378224, 66654.68356507634, 79483.34290334788, 45371.502450648186, 15846.321917960193, 25677.891226543758, 87864.06329576117, 74092.10922711753, 7012.833672037522, 30495.101823406367, 19113.2209809333, 47321.77484791765, 30628.788276336738, 62306.22434269307, 55796.81040611412, 64593.184105707354, 17523.33051736802, 11075.940843097898, 77152.14691631605, 2174.6757914780314, 53085.41214308814, 99082.66350559605, 63156.70368103571, 62595.3016653028, 22925.177322375544, 22208.743295210985, 64108.37002067291, 72081.73322008015, 17682.568691016477, 48405.501307051425, 93806.46265829509, 73881.19606754024, 83563.86779545904, 19266.2605943011, 29531.664136136136, 26121.52241869171, 98178.79572736293, 11816.314324336452, 1503.6537978374565, 65337.633804816585, 36416.40534606916, 51719.72934552315, 91868.81116744502, 1800.822576540795, 8873.028029608398, 89486.81439717465, 95985.66001261129, 61448.83640350951, 30407.74110380311, 17072.928521469545, 29396.511983771656, 44142.90707141117, 53412.58879788201, 25913.08307512743, 37184.80371636042, 4250.88759077521, 398.99430464617194, 29569.569437639708, 11028.644469921444, 49036.19938512925, 99790.75991504104, 19146.749027449594]} +{"id": 905, "vector": [97591.22468029225, 63006.52030235976, 35400.290488123865, 95696.75808046648, 72573.27079487768, 57397.26830891995, 85142.96761608808, 17194.587096469306, 37969.044790440676, 72799.17181174441, 26166.879656252717, 51996.7171874811, 8960.690968298357, 64314.95672855456, 64847.13975688522, 52106.56501183177, 75141.29108491902, 35805.81698710911, 52064.797311103175, 81251.88202776738, 27589.66830124766, 56326.24213062729, 28384.129723422124, 97078.85383925047, 98247.81910016641, 31856.815353166166, 35811.25342264827, 62072.35551584815, 48981.93716290357, 56715.28996307884, 18628.653399315077, 47476.987662292726, 19826.74933318187, 75806.91457681963, 6749.753682843718, 46043.74555576946, 34279.4940062088, 69140.39599760574, 33312.69735663127, 14036.004920803523, 54874.744218536805, 17580.943812609752, 32865.72481897292, 39954.88663866389, 40405.10987025785, 145.17455346101383, 89645.51666010203, 9240.91486963604, 37141.798890183505, 40705.71400288686, 30362.870775374475, 4066.557400927073, 2510.1611565633953, 54129.849531524946, 38585.04868191923, 92784.79275743249, 57779.87222348631, 35637.365886582214, 30244.31850442998, 6520.799799055088, 75714.51578335022, 90604.52550516225, 328.9175345853712, 46603.92793932322, 72495.23683470074, 85677.84583947685, 38686.138750814745, 11989.820837344778, 87526.7581198136, 45685.48506701204, 93798.26116916238, 46949.81681361901, 41019.10167081773, 79096.53236405805, 78448.60868086874, 6958.050186179232, 98192.86269746207, 30970.463006042904, 1835.4711528576483, 74966.28257948026, 16290.391213601608, 83186.72062481883, 2769.068185323986, 65465.761916379386, 96612.42098627384, 80197.9055800064, 93641.5873528892, 2568.5332168360887, 28288.1629786215, 96582.46165919893, 85396.6028313319, 33330.488042530036, 18369.095960934235, 66898.10326996047, 13192.63062326773, 35201.33109617004, 4881.024678116741, 33923.82049239464, 91679.79748319188, 91313.32805991582, 18796.96594404855, 32861.0783981784, 95760.12111490245, 49004.01426009897, 54161.744198836706, 88862.04611218133, 86319.632763445, 68271.04376888018, 92379.30140888793, 61996.84912961421, 95074.84537772088, 98339.55943722978, 85493.17519014592, 53452.233191350984, 18180.76353723287, 68434.27676750468, 80278.34078718306, 9180.367647899202, 71865.01811380246, 25724.59986583966, 37157.506080552135, 84159.23386327944, 73564.89760785125, 28430.91249206231, 83505.77598779261, 27413.003860803343, 98377.34624653732, 87104.61094166344]} +{"id": 894, "vector": [7501.3726214686385, 76502.27674378677, 12804.59932152448, 8663.772203258568, 54087.14912090144, 79178.57064326908, 5739.1354337428365, 21837.774505232275, 58632.680539907204, 59330.714012731245, 18890.659132790122, 39994.54301102957, 58757.99169434363, 67791.92811796865, 71084.57835182574, 12350.468002759497, 87344.58242145792, 58740.96807555544, 2809.589394350065, 81607.5177410179, 47687.63922869831, 92330.65531740981, 37901.89330408874, 11630.625216117996, 2057.494920940195, 6194.08514268035, 64294.18441952619, 84006.4341792108, 293.1972982149089, 41771.69265896997, 54953.4267084299, 11675.277587121212, 20730.796633530535, 55236.0798597739, 49683.899125516975, 32996.17774864577, 22360.081537332342, 98337.37884496059, 15983.672116075431, 35080.15914083206, 10612.540674535654, 26662.517370975012, 61418.61391031398, 99930.12669708999, 11348.017732989645, 22104.324922306805, 35273.108870682554, 90048.62093942316, 98213.29364223909, 82582.5576050284, 22615.177831406952, 83294.4688999762, 67166.91934124954, 90245.3061119446, 14025.340796598084, 13470.400535481042, 5888.4241003777315, 74938.2235979575, 48513.03574113908, 25771.363422491722, 12226.125872083227, 73271.2415553839, 60282.450806317924, 26928.75517686034, 69959.39558683235, 62275.78994409031, 70432.48214870394, 40988.39838656704, 33413.154280130904, 74128.25035741858, 10079.352475399828, 71435.83293154117, 22478.568811152043, 78998.39421062778, 25274.39310341819, 11944.894117685533, 49666.624649677164, 28672.81516106569, 31552.554159988867, 69502.23353661493, 261.3477762520189, 95206.0642610112, 84861.71642844846, 17957.129147701067, 40497.84851254601, 43667.26867248143, 72576.05260818153, 53774.45394882413, 46894.98655873148, 57768.77213682039, 23313.66755726867, 83070.61763580826, 29539.933406680797, 13617.714561657667, 8836.79089913092, 62929.05465252645, 29864.704796157483, 2788.5500544694187, 37431.72270428217, 57827.959182844745, 71054.23996919408, 12677.391748778622, 73584.90813711584, 77439.89572691162, 86312.40608258676, 44653.510230159656, 14794.073479127901, 52358.779643969574, 20290.391478553192, 23091.479202827715, 96520.97270635412, 89400.63158539351, 2824.2924034624693, 45163.1105553474, 36783.43090509846, 18729.106079206493, 82837.09534890996, 55882.627679709076, 80226.34670957917, 11692.680761389684, 27047.395191660118, 59357.47045088412, 73382.91465491356, 79578.4023563239, 89348.16881071331, 53922.80272580402, 91778.58883056721, 57573.70604849031]} +{"id": 1420, "vector": [17425.052997957137, 87629.72067373864, 7924.119162872345, 90819.08932019336, 66543.77209433645, 51818.78441150967, 87419.67913406913, 96718.07363331763, 93384.06392357129, 18251.259274032538, 52286.794171781, 48090.07090032559, 22932.193570335956, 72823.57556795957, 11332.544258293752, 75939.92086214684, 46146.353694581296, 97059.13636796676, 84594.37793312366, 36010.82417977893, 15660.602319075768, 99635.51396410297, 44195.19267835962, 95077.98180501004, 30795.012569008086, 60392.2243852204, 99322.69888289073, 28253.281741328574, 88351.0951776032, 75709.34791939532, 20455.200255594118, 46194.04687426162, 57650.50117935644, 75419.00242936739, 64242.97746697656, 72193.15224036103, 61064.11348378134, 28551.331757715903, 71009.28533276831, 65235.21915176195, 53534.75166288185, 23350.136466362237, 12744.085066718259, 21388.834494120023, 47059.68681083984, 69207.61588369211, 56407.96629522393, 9287.94310614992, 99868.94930617821, 93788.70233654344, 51423.10465254566, 50848.79353672042, 12933.504626403557, 75036.73840827758, 62430.86254431637, 99357.25277340232, 17761.79907886838, 13034.2671480558, 14950.563315854126, 21288.23285654372, 31454.465557402476, 94386.863078085, 36184.555900619205, 97353.35474681614, 38517.66017935192, 48739.59762621079, 90303.06089608074, 63829.15275092332, 6789.740606873129, 61808.578347482144, 61967.13744133207, 45529.07211010492, 18280.229048651876, 15348.461242770805, 22597.942857412523, 45199.91755820683, 31691.040950007133, 59778.28754366483, 96793.06395359345, 77936.78933486006, 40928.70906245054, 66840.23552475023, 61561.024337705625, 7768.934663940363, 91604.53887544041, 39758.18815963307, 92525.605446744, 47962.16467289466, 51958.047665710714, 68267.63550855564, 34806.36931423174, 85065.95276025434, 54725.39879759031, 64731.48935667077, 57113.154765844076, 74744.45704028753, 87547.12962545526, 38551.61788804594, 93103.09177174665, 41795.51523085447, 97256.38835673433, 23482.399739443328, 56022.1724458264, 55183.21978873221, 34713.63241319721, 16209.875606952563, 43461.49430106635, 38132.657314503136, 81929.97188745445, 97660.74807493255, 79131.13262481506, 72531.64153351229, 86662.21236262022, 84464.06611456414, 822.3184542163864, 56853.88181602345, 87309.48146738879, 32901.51143551657, 28042.56141251963, 81454.59598996758, 98917.4169312303, 62359.59562742409, 50412.116646948525, 48441.24012880466, 32141.710281118874, 62938.155148536745, 33958.22345044953, 6732.000684243433]} +{"id": 443, "vector": [64889.02952124424, 80405.15006137513, 90306.65758989389, 90086.90901135695, 35794.63096255496, 40213.99025180562, 41643.763156709734, 73650.13403800996, 95604.11596676178, 46861.452358743474, 74660.9481059498, 55561.79336068188, 89298.95048578622, 930.5611797997826, 35359.32165437071, 34218.1452429369, 34336.845573180544, 83511.9799170778, 20478.149658666032, 87708.45644294425, 95722.66259933851, 85771.82736503147, 86977.67583881995, 53830.545274464734, 16085.253899461972, 6253.715429446261, 16061.784018429282, 21584.85845160232, 54747.365351692635, 70187.52041329813, 2821.8218818281125, 82920.23611541995, 77751.37684358371, 49109.16083552789, 6123.759620164248, 87989.72563108968, 58281.16483334443, 51480.15305700284, 16409.08517164372, 30709.47080432046, 13785.415819777125, 82411.90841674674, 60873.58542414227, 1229.7699392232353, 31189.039243351544, 94151.90696455116, 10251.64196860755, 7442.929170623746, 8955.127561166842, 88062.58093846437, 87754.75259125845, 50476.11957082335, 73868.506814074, 42716.5790850769, 22042.99408232104, 62751.72447358621, 27898.37481736345, 75395.20915991797, 33181.51252801872, 85923.08440645524, 14548.84645784622, 80906.02088505554, 63216.9859536275, 12557.875306203192, 89040.578184146, 65193.05387219015, 63155.82261543672, 58796.732808544126, 87855.68129654705, 87744.53062601514, 7584.708380610383, 9072.987415044508, 92768.45036600134, 27808.819563803034, 93392.09942565409, 57182.62814266577, 7452.346813384603, 8679.872915643582, 23240.211812908816, 23573.725296816483, 50171.332547406644, 35696.13660510415, 96095.88199943534, 98763.69666068502, 34591.5035473563, 39087.71583211339, 86347.11008477108, 82738.49664347983, 68289.18229156765, 34525.80558525179, 31892.124809530644, 40981.9235974015, 59985.82867876625, 33555.73586267802, 8039.2109410351795, 7610.4408350346375, 25586.182872599704, 666.3362908021276, 3853.4636935810386, 33011.51156418271, 34931.94491626267, 45663.498709107385, 16232.45606604612, 2508.8042306777634, 32525.133020719164, 7231.2046803373105, 64334.67541196031, 8877.925549234433, 40751.01760538617, 25651.186664820092, 16848.46816930261, 52903.675583315824, 80923.5096828972, 42044.3407474296, 35056.99214682439, 29124.3780797204, 37597.985667514076, 4807.968021371134, 77534.69430558091, 66362.49960980401, 70824.54699248193, 21755.27537130555, 30274.57233579027, 47442.76547876238, 69726.11302583035, 94043.13927145512, 14593.30039081216, 21786.01450916663]} +{"id": 609, "vector": [79856.69137560147, 5209.3582850640805, 26456.630936595917, 50683.505708035445, 75527.85251002444, 91734.98348602594, 62848.5441488882, 99529.93276096835, 19643.130314180802, 26044.513629639587, 76197.0090930849, 94990.04196386486, 71643.46648888737, 73507.304869625, 58863.01885712017, 79590.66003813448, 78183.3566651967, 95463.82715717831, 53341.54379547458, 42348.17244730441, 55578.628842057355, 68214.83622636183, 58785.69001944476, 98324.61329830691, 13335.330532025324, 27243.69386387, 37629.42585621725, 46805.39287860219, 96783.5854506754, 95523.56660786901, 47246.169499969656, 29145.64616532099, 42225.103032881714, 61071.217673330655, 85641.52326243269, 68343.63757428517, 75730.06329208858, 43160.370244846155, 61719.4445703108, 83929.37698828729, 47417.83240724379, 14225.283667162148, 33520.085354249815, 98450.42423083386, 19381.41228254112, 9369.357359801334, 34267.600378731215, 49216.13819849154, 513.5152016415234, 76410.54917090006, 50378.86564820906, 13607.880733426136, 25775.78165438503, 65112.655140412055, 60238.379571901525, 90063.25934000332, 25580.794325617408, 64995.40970521085, 23606.35755303011, 92146.62701073564, 48417.561644044705, 78640.72434466492, 43278.83658791447, 3497.7103033273506, 65101.372964259375, 789.7159549618227, 23130.866111994754, 93622.94441878704, 1985.0016056910501, 45664.0734904143, 50565.88517130107, 30049.25650070114, 12679.203156390595, 40752.139626458214, 25559.623785399042, 14731.921656253555, 29942.461838307023, 79188.62484807802, 15991.76528533094, 60497.926923900915, 40811.10826250992, 1814.7162334824052, 63155.096538773, 18995.247376115633, 737.7312675338254, 41345.52539377499, 16640.11942662106, 84727.84364310463, 87560.28863127352, 89796.33732710379, 49488.80259039508, 46926.38383388242, 87677.29124887688, 89617.93588187873, 17556.735621488595, 94088.42656991557, 10852.464089742698, 66627.48673531019, 5798.410879160165, 83247.74048843258, 1253.794739747449, 65192.9533109189, 67781.53292080984, 11824.466763347973, 21126.998831292178, 81240.63239902954, 46664.188226578415, 66032.62623186104, 52541.58683995517, 53797.43184286636, 42494.70238825026, 42846.8428229271, 30084.472095474335, 16375.924306854917, 80307.64549257468, 56683.325201321124, 59199.97437193866, 50110.60960948873, 12544.357126808025, 73537.15395724507, 26504.059250646038, 81716.31768788013, 28537.998390959372, 33426.56341787785, 78681.6982387035, 80732.59488071802, 15845.077966458754, 41897.12824986115]} +{"id": 216, "vector": [60981.3169888594, 88997.59414490924, 34745.61639352865, 99598.53263208902, 87104.72530048818, 57888.89886975963, 83834.96486465156, 78984.94958621004, 1919.9879552622922, 80526.17703158883, 66191.1282484658, 76938.99535971144, 79740.92923642074, 6972.005463175723, 6215.430560201052, 54004.4198315558, 22339.986442413672, 4523.297198790155, 4893.652258726811, 20562.817213941787, 39280.28421170614, 70215.2395259301, 75502.37249483833, 44207.96959729742, 91895.174614122, 93119.08709310358, 37120.77861933334, 34322.89444984511, 68956.91639216758, 31496.10924622406, 80219.73430595647, 43213.92873136092, 24819.24827779276, 62736.14137919259, 32195.20509444279, 98898.18145391738, 17214.48131710961, 89287.61148505069, 36190.77881198719, 84848.75767932007, 293.67968661651037, 37496.69265876197, 8875.433963662537, 75595.66544444227, 99456.4116177131, 95517.43135534566, 52569.9308955567, 94099.82533255576, 86525.27692725093, 99854.79066543425, 12303.084190403546, 75301.69577790875, 60189.38685447111, 79320.92113618598, 22666.538235474552, 69791.11847056894, 47964.410654629275, 84444.02131452669, 47203.30488322549, 58329.54796773303, 48299.66782062237, 29473.008666184465, 23963.566054964504, 47851.76210185711, 57290.488732900965, 94604.5346532118, 64799.50613276624, 19675.06510017315, 58877.60159611478, 48508.12370324901, 10445.262717974501, 82512.48774493586, 64987.5184224584, 86572.76305645987, 33317.041217549435, 82518.06399981953, 28825.53112833519, 37700.739820293325, 30674.869409732975, 39507.10615451191, 35284.88125190841, 17604.475219008196, 98400.79324055008, 84097.53281873258, 23584.80646322968, 4559.18465026115, 51575.996963637095, 30163.801076977736, 57149.37560690632, 82025.12418419182, 9839.465639044187, 67291.20701720599, 33132.348109243714, 91843.58447011212, 30178.793305617237, 41619.09070586967, 23329.434851641585, 44146.30373407891, 71878.18533329389, 55422.56978763876, 89753.17856148482, 75980.59664912894, 15509.292340057878, 66265.59458507317, 15453.558216850737, 17729.586412019904, 78828.39634851561, 58173.26348886987, 53027.7075294551, 99101.36262377413, 23142.864929952288, 30359.09558958796, 32952.10624577888, 61365.935638016665, 75372.9277156211, 14832.627208788463, 87287.33442142706, 43665.37651846132, 27686.194058216308, 29261.47257214575, 25424.240541291787, 98726.42281960013, 68554.87834590483, 40720.442621749906, 45921.53728192183, 23957.283502186387, 71532.16697424342, 99406.15793505097]} +{"id": 498, "vector": [68001.96415707387, 54628.80777731989, 62349.77870878497, 59080.83825353223, 10069.09110086377, 91416.30935073344, 67174.80098336922, 13761.533920533575, 52102.16695479196, 74983.34867633619, 59013.25432458932, 25241.499946276646, 99922.96130808476, 76278.11076703721, 95782.94285341284, 22056.68491684676, 65217.76159959099, 9295.49719880972, 95185.73834988564, 55249.45951467932, 63640.061114872595, 10154.555164933476, 80142.94125526343, 1019.638695246483, 87531.67797444722, 44795.47326932043, 68713.0588327077, 27387.448420912286, 14680.498798753339, 48596.70352991568, 78033.63825210588, 64136.5609541059, 27855.764114700898, 9182.172381586373, 69743.42641881909, 38178.403121029914, 50535.79832904265, 83430.57595338815, 93296.54627024874, 39674.59991079125, 83089.30514746044, 30747.762931071844, 48850.50584233727, 11726.410302712953, 13619.02699823686, 73476.031333642, 19083.994143823078, 4973.581644661785, 26632.67131049455, 42099.97234602145, 26105.289743311965, 14866.6850624392, 64335.747264049096, 77366.20204838268, 13265.070383044542, 66911.1770279522, 38047.99598948737, 62754.95618346571, 86205.5991754466, 76884.66636236287, 98224.20868130797, 66161.4357346618, 12607.710875571222, 2879.09076531927, 94754.57864952927, 76467.73469636307, 82818.38230520935, 77840.45435453275, 97128.31991404576, 90942.36182248154, 30436.271661691506, 32671.257525897003, 30994.40368921673, 6603.262781215702, 26067.89474247153, 2093.550851726156, 55867.97959985961, 78858.43860845079, 94594.81583353071, 95335.73261929685, 83169.17891382196, 69135.98761576002, 2166.9123415528848, 11370.341096481263, 60673.71226360426, 72289.78852886402, 67843.95904057857, 95077.42400576196, 76105.43171823911, 10186.406942941461, 59068.57593416222, 76618.83192748588, 49833.204462458634, 62658.9726587148, 11602.18174394021, 97821.62014217975, 39458.08500094524, 62481.42097924233, 70324.70880806725, 99403.6148055906, 34111.52066827727, 90530.82996757136, 51913.852972442844, 66489.158622569, 82163.11147082003, 53951.447878213774, 5106.887870524956, 41571.161239075685, 39253.5091658058, 10375.128380162845, 52093.69478882835, 72199.94373766243, 76621.01580345094, 99235.46556369244, 8801.149309659484, 6917.878547550471, 56896.96856177574, 87313.72544979994, 59734.991859937756, 26707.34250352975, 63311.086573037624, 23434.41986408219, 4459.600726913004, 17271.63076249043, 63537.55829886674, 59271.60421251334, 87215.53536019319, 64238.53171969184]} +{"id": 1744, "vector": [15408.541443495817, 63430.27994787388, 67596.94187322806, 47160.74082888845, 95520.45771257707, 66265.42085313491, 62157.637789441025, 96048.26995581792, 30620.95972028339, 54386.525989442736, 87476.2512232778, 24572.09958560699, 34190.69934889741, 64170.97473201302, 54899.61007004309, 12608.070400902072, 85721.53362590431, 49651.78035718212, 660.4637764376808, 55474.63709621091, 72524.38605174559, 37121.21858116232, 61485.478995005215, 1669.2779451821127, 53563.15476689261, 90003.55745684507, 65830.25859270513, 12853.111188897126, 6099.237966847703, 76768.4347834514, 85746.96950964312, 1374.3063749353523, 35271.822238737, 76660.3419139314, 31583.021025517068, 26392.9024447467, 98486.45645664496, 77743.07618385601, 77838.29864309497, 45082.34990001751, 83273.55203855509, 11492.004318724059, 49783.70838482157, 31785.112040017117, 84287.43749803572, 88692.42525850031, 52979.78035084879, 79372.58028462241, 12179.09173132633, 34039.10297427532, 36934.22492560492, 96394.02536293415, 69649.23071203732, 88271.80769954961, 46284.84373979962, 81319.75479419588, 84800.30293029561, 84587.70796583264, 78572.2298602827, 77842.54022304402, 44576.76919969524, 44225.64877221096, 19132.18219738284, 43726.85943566152, 24141.582259443774, 89977.87591868332, 71678.62483151266, 9639.059157013797, 62835.22225415796, 91634.60460871593, 98429.45254353972, 46699.58089281106, 43514.87894257992, 75709.05614892725, 25623.741874069005, 27741.786233296873, 35820.51693652078, 90166.57434399152, 78575.63126163064, 9980.330846415087, 49908.27931420141, 26742.12067839651, 22473.522437266445, 26633.55377736919, 53279.518008306215, 93510.15234430676, 92641.3508607486, 79825.28788488854, 78300.07104930235, 54610.39208692747, 31043.446637528716, 84320.48791393882, 46219.7392374056, 91068.56123836251, 96576.58728566267, 6659.7513104313275, 244.77674213819745, 7110.305557700647, 21797.185751564506, 30837.14704340772, 95706.21070898006, 20383.551176806912, 42042.932872750374, 21791.91022386413, 83341.54066728037, 55319.58267354326, 73761.21435283055, 45515.57658381813, 54058.96959457415, 67352.08768130078, 94897.56816096685, 68897.48860731472, 59451.70712607606, 10771.532766473701, 62966.12136049275, 2131.6391997383444, 75868.14238232111, 1200.7090171313184, 22472.274580128327, 21516.844303834958, 44764.525300839756, 36274.59701040805, 6736.541547249985, 54956.80522220683, 13973.668295534859, 40556.608585013186, 76210.82610974685, 42644.79748626091]} +{"id": 1709, "vector": [6786.880696107844, 91317.16137492083, 4831.050505426259, 25955.445729768624, 92755.54095372724, 80856.54262327861, 59353.145176269565, 97295.78938006562, 99820.81710811594, 26163.40771656234, 84182.99813720054, 74535.69528708005, 72427.74839831289, 15772.207650217673, 71710.23935755502, 79212.38990846554, 48836.74041996487, 62593.08640816793, 42454.2789729127, 14559.038002313086, 87630.4874639622, 58892.493196524476, 4624.755370194811, 76732.36897487704, 59947.88956924042, 51410.81080509814, 90366.02368359172, 31254.870719131446, 95583.41744703932, 63065.04262084518, 9723.893668253659, 25609.179603949793, 23102.61850018095, 38903.40125130055, 23747.316923432503, 73677.57675544992, 24956.046868127403, 38118.204362465025, 32989.90540108792, 29120.235661764436, 96176.88124119278, 61040.3464328624, 78449.88087589195, 70790.43305245496, 94008.73318904602, 50061.17896624925, 37705.44972164852, 45526.84994582836, 51668.985219924856, 46216.82918871239, 45959.57639775668, 14403.847684340588, 7671.863449427874, 13868.974991017412, 16732.760198964035, 75716.00747315609, 62685.257885156454, 8744.23586557439, 47555.174650345354, 1366.3152854086147, 30945.251590149357, 36216.954831216666, 15460.281608202087, 98614.60023819841, 60738.86820832225, 12062.84426811458, 46095.275077955775, 1190.9317433699607, 79938.07864556655, 28076.49310164263, 6045.468777246021, 28155.828835674634, 22876.508899529646, 65402.91763483384, 99910.61113872127, 50956.554363200405, 92867.77186178407, 2507.7748105807427, 52876.446820271805, 6585.154454511077, 45933.32848364215, 25379.49462440595, 72839.57459647454, 92701.33400343603, 55157.818701828655, 46527.05864021664, 73848.7781086273, 7141.494433524953, 18375.514942122318, 34546.373411220855, 86247.74566177189, 2503.0042100144033, 24205.802215698324, 50586.98030353972, 95530.29301373997, 40802.94924752902, 46389.043632604844, 99486.88249929092, 59930.40447643381, 61160.79180379094, 34953.17654514085, 85565.32297298302, 15777.240075535947, 14695.045537052043, 64962.10085460471, 78050.6798194691, 56119.73385829927, 77953.8318296058, 90456.88490023835, 1936.2790439111377, 80788.07459926797, 29291.206853426687, 9200.942606604845, 42796.84597066722, 69942.61978352694, 36514.24073295141, 7650.016357402389, 93402.3417942043, 73771.8653737431, 94002.98551779253, 86331.61307731974, 95700.08269880693, 82359.31199313187, 52238.30537877375, 27613.410535737905, 25433.968403778705, 89612.54576591475, 56615.90945248531]} +{"id": 172, "vector": [61784.996356810116, 14009.822342157318, 22661.42708295784, 75076.77198017058, 15172.047385512255, 96728.40059326212, 80932.94988105599, 57716.93928002459, 77197.54735310678, 35550.46692720652, 56580.42895886508, 86682.44400104653, 57974.280681143486, 60073.30866445328, 95017.91438151807, 35941.48719072844, 6469.339402445362, 56195.99509568918, 93662.15995479796, 5763.545641455159, 77633.44319451092, 21190.918050831853, 36026.10257786435, 83384.6905434313, 67799.70632879515, 66659.7548549224, 86782.45220031531, 84777.09410129362, 18558.987837815544, 30372.35831025513, 79120.90450619679, 58525.977437834656, 79894.77548599799, 12721.853176602028, 66380.14797770121, 25383.3470824116, 54779.25606986159, 77243.3759023008, 2020.944226994148, 12824.92328475835, 55290.93915341745, 61874.93985226596, 30774.80554378279, 50600.40316897928, 30388.10867504861, 47460.247324049786, 86810.20196174613, 8978.091662621368, 52244.545757317705, 90389.32899701835, 52994.44924471897, 84429.85066148255, 94144.24101602605, 12258.965067909621, 57442.69424278277, 64211.671587492405, 10732.501554510633, 32993.2121595471, 59079.24182377525, 4587.250237016072, 43612.06032029704, 54266.00931525567, 68121.96563332401, 33079.04021939178, 64684.60500905334, 85107.36921586932, 271.57796289337676, 44444.96430913377, 78493.1118373946, 18137.067474855874, 32002.92660708376, 57889.4691045587, 12546.362689674117, 23239.083868626876, 29444.01053919431, 69441.93959645562, 76753.80922796208, 88558.97416117345, 58342.00392597703, 53437.611825938926, 65587.59369752278, 95784.56597620159, 15044.861386878916, 62563.26409288855, 58483.759346977015, 55640.324316737446, 28391.893383725786, 14424.469653389693, 47578.90373208208, 39310.256284229436, 86362.6929917226, 21513.09246997399, 41005.496964194535, 47641.916797395876, 2098.646839932239, 96801.73957132979, 8018.686305323675, 25970.367062931753, 3103.4323982891965, 62278.265008876064, 68782.08268084288, 10556.903435924369, 49637.47335376078, 8208.506426782415, 38973.11635550008, 92797.26248651896, 24147.392278807867, 56723.262792831694, 42162.340915118824, 47483.071555897994, 32527.076672844058, 32810.91600203269, 58360.78386485435, 35161.213827950654, 80790.45461838589, 188.12462566977217, 33472.65968239145, 52483.21704520654, 82913.96066950203, 62742.64632399406, 12428.829653275587, 9734.679097466382, 37753.81999197044, 84711.81314876869, 47139.702888982225, 39686.84271215091, 69934.224199981, 43736.317331819075]} +{"id": 465, "vector": [53490.62893552301, 55138.83323982056, 3532.6606321086083, 66149.59193951405, 12253.802058847063, 77293.27762119813, 5376.003835309851, 883.8455459787897, 49458.43918385682, 58151.61753591228, 45155.7984830159, 78442.89323175803, 73441.65714276569, 6661.558215766661, 67906.68451962616, 81023.64111855696, 53824.130731850375, 74652.27213227686, 84022.50002507008, 14743.14153971037, 40798.346427832366, 11706.77874598156, 14594.253234627353, 99396.42302294637, 34000.80447976818, 24209.22301803804, 50521.677269073916, 4083.1294898856286, 65879.26444526983, 25010.725110350373, 6762.666852281962, 31756.990409291753, 52430.649851962655, 3190.943677525959, 10724.818399734791, 21796.832952971246, 66428.01799169111, 75192.49056987558, 38032.828195363676, 87602.6400678361, 42802.27324522643, 71071.26141478374, 11452.598674995617, 95884.74001713768, 45272.48854007825, 58260.799470619975, 5700.316513133197, 83363.65836561701, 26644.47127602808, 82181.72281538926, 39259.3316156999, 49680.40016094852, 88939.00206439942, 73909.24626263513, 29879.507962862328, 82313.8245275701, 18232.554859141204, 10009.898745300572, 56451.51272585075, 90510.50242715584, 92846.32385877355, 67439.6610252948, 74121.29908608289, 24497.620300859213, 16771.09197291431, 46521.68444035511, 62245.090044578224, 69614.55374054445, 95746.39045917899, 40117.93395599302, 51326.81013981859, 52930.98463474966, 27270.529860572246, 30283.919344768183, 7525.897017489203, 82960.0020866962, 70405.4939350997, 51978.086580077645, 17629.86358162464, 4030.9584555181186, 76864.8901900246, 64337.97315401514, 8076.683265870255, 92.3168830973431, 99087.66316530555, 74963.41190413573, 19140.351128823517, 45718.37557242665, 92487.15976542771, 25981.83983446448, 77983.47885781553, 78196.01539633032, 82867.81921683607, 61612.483728022926, 37974.993211750094, 5037.235223197822, 88601.1269946166, 13586.599037437352, 89644.93020867913, 36477.20739607936, 74555.08860722907, 2830.769861349636, 42746.36786615958, 32697.927511949852, 12042.328262540259, 44998.63059861213, 69725.27922932798, 26932.148098830043, 82815.59748232606, 34937.55318571508, 72068.91879525608, 31168.406273954984, 8821.316493499653, 47821.00476906578, 49815.80531895069, 7876.272655438354, 20582.75827857766, 58949.00714954613, 63785.15395459198, 98109.71483586526, 87600.72364793821, 3035.236098863181, 27095.561692769275, 84211.00205919269, 76602.73868092755, 74305.61564826513, 35684.15175622627, 34241.23123085311]} +{"id": 1981, "vector": [89336.57745293638, 96387.95151416039, 47380.39415366414, 11659.835671212304, 43668.890921595106, 30537.173164691, 71191.12090893404, 45741.16157399645, 81808.33844774924, 64338.96820082998, 33491.20808526997, 56240.03548318087, 50305.67457182664, 73292.46063687062, 35371.996290355026, 35519.51163519446, 80058.77605758247, 77177.76249610435, 11551.812328703969, 79211.31064581052, 9148.645473321281, 62414.253522867155, 90905.22602376262, 63527.08468979845, 38076.327334764785, 72999.70741768809, 78932.34956690013, 73888.03236011318, 8859.93534998618, 61221.78832086311, 9703.085314429316, 16166.520834836807, 74444.83201672518, 47811.53081645134, 61099.65051173093, 98232.17498241576, 28660.06557485504, 79081.40459484844, 59683.88369433687, 2714.6243474219123, 75406.5813793594, 18589.528030568137, 72178.89945366999, 62569.43402368596, 72040.85101901132, 71213.0435896306, 1378.4532985360308, 1466.4393058317794, 26545.748271441127, 96515.5673285052, 4905.375195642525, 78223.28820905853, 63300.133878637076, 46961.12563586777, 89950.46506595581, 57898.57105443929, 36898.024554154304, 78064.70058528046, 36451.97864040205, 93981.18055761055, 92710.19487006073, 28959.942364986167, 65349.47877207389, 67721.66412962464, 61573.03079415458, 36298.843072014766, 12707.842012446768, 3408.3475083956814, 47685.32500244617, 92088.59343437178, 99700.33685524615, 37839.40107320537, 21179.120074333412, 94631.76076289946, 71689.35594737981, 38745.28352022303, 23614.762105673304, 48512.76670295388, 65306.77855647697, 84994.8314991613, 48206.20348258074, 71024.02294136338, 68195.0475711828, 93451.35218899166, 91889.43009331338, 80318.45638269624, 85962.89087231256, 68378.95275635434, 7214.627036883015, 78270.80479531424, 23813.184299468794, 36230.89013032531, 61771.18722319129, 11375.877343558006, 17157.219803350086, 67289.04319096736, 21345.594478847874, 94652.92197135885, 49109.60720239497, 35030.408024129836, 2771.1771141867957, 69121.12470281647, 24540.38624550825, 18432.32536114111, 97653.76600298985, 42444.66269568919, 33639.34677442922, 8389.554940090027, 79260.13922119245, 96385.63026006344, 42041.97090545686, 15089.55121922969, 59379.15845064252, 27224.543997331475, 71841.37582901155, 7189.559945220003, 39048.39086611712, 70646.71954677711, 53889.676701637414, 95041.75351292035, 67916.36827786618, 71471.18909363491, 89532.14765826933, 23366.676277256905, 27527.023896993363, 52908.18935090814, 31925.236130100533, 31960.432899481293]} +{"id": 281, "vector": [20974.29411011613, 88820.14592669763, 59716.43452637936, 75561.22590958251, 63899.09045069456, 51230.206830432355, 74829.26538490235, 95691.7282453273, 7297.353525866457, 88723.72250707747, 28924.764697511928, 72747.0888481377, 56023.66448711882, 33806.65087845127, 76464.74502054846, 57096.122214691, 7289.80934013187, 52991.32693493623, 33067.18080867045, 7287.433337374938, 28552.71215367694, 68094.3529705401, 42766.28580622092, 62996.737686653934, 27752.046908939366, 60320.62325041, 16604.867711243067, 74923.2070462169, 55693.71562273494, 19749.56233637174, 14638.098352674, 7011.424665364152, 79985.67071531326, 82010.87998980068, 18878.045102842367, 60492.24057779363, 34743.156715436766, 31784.87914422584, 12173.12851811192, 30333.843510995408, 10518.92777734551, 77502.23717553169, 24016.261660277305, 29477.609051989995, 56566.32972663062, 48779.9884187757, 85709.51485077232, 9804.628499059108, 28352.040996128602, 91924.68484855136, 88042.45415165159, 45502.722275399996, 7187.747437180358, 83584.88172314738, 9088.590551855557, 67760.7555616385, 34621.200671648614, 8344.612857429156, 65090.8789820018, 79353.22450316916, 36932.12694015613, 39116.972710091635, 45058.558593428344, 90018.37730787433, 70125.55175958527, 79273.12308439401, 81616.58856286012, 20593.77581666425, 51150.65918738842, 17721.192930263584, 40262.91898474795, 5268.750404121947, 33.163557358451, 77667.71733861402, 87818.5336411328, 30959.583962523597, 16297.73309978726, 2995.089070478818, 73894.30368564314, 79821.41272571945, 90717.9191116647, 90408.40967008777, 7260.644624844126, 63886.262184220825, 90651.97098288413, 73185.55718387966, 41413.47393579201, 81683.45152612869, 83500.39390665226, 22333.2777798803, 16490.56451555082, 1328.168204726321, 91455.83861304866, 28738.81355671065, 65289.11817083277, 20690.507006440985, 91402.39136322918, 58097.57284544614, 14767.724604160836, 31395.809025783485, 95422.64299115368, 2789.9341959879866, 10810.017023145934, 21690.202343206944, 57650.55962613515, 64549.83243318658, 32162.466796278277, 6835.03471185899, 34419.740031227106, 46370.624341305134, 41164.69604063644, 90595.85906557081, 13534.08258223091, 37409.79019406424, 60126.05072087478, 2827.321981469444, 85064.63829639093, 24251.094115602944, 81906.5249235889, 54612.230039770126, 82645.89664502848, 23417.817429516253, 56629.44399078259, 23141.053757700312, 32388.293163160753, 79413.77288878142, 55094.37734269736, 9571.24989985131]} +{"id": 1351, "vector": [15029.778414176832, 82468.43246101675, 30208.439792697638, 11686.22711841758, 11233.850669839529, 53189.80615832113, 90147.67944264201, 69591.26436253476, 82723.56797129275, 10918.52211378318, 89055.2309293291, 33189.639340812195, 7204.133806120028, 74224.23666602848, 58370.38414933477, 16598.828034041024, 22530.216223867126, 7402.726334881948, 56830.82457571945, 43771.548175052456, 9285.801499336532, 22853.966505820157, 30810.546993902364, 4010.4684536712675, 93909.1998871844, 27640.820405502596, 21111.324145076116, 40178.30259904079, 26106.775403119587, 9157.57864979766, 19046.327029677977, 93394.04526369223, 73190.96254419633, 82841.47567374718, 57175.30550619192, 24428.627138726577, 20755.80501192611, 98998.63624471029, 2543.8789050643563, 44754.856783868534, 36946.85201995729, 65377.260289323654, 25374.13703368635, 88844.03638461132, 81935.25277039202, 73779.37593870799, 23720.99953420268, 26233.853543379082, 89763.03302041098, 6075.252520867302, 75403.92268235605, 3768.8712330045228, 34931.550745693195, 1580.947157602286, 62193.40936505521, 45787.13899698067, 13995.949498928518, 59371.5544353198, 70101.8700844649, 21690.93829642449, 33161.4421605518, 17949.599104400626, 66523.81243579273, 41239.65197594526, 33409.09483487683, 56303.96533537306, 91259.36804978382, 74009.15878699905, 20583.674324974323, 60076.51225420545, 61789.653038632794, 25895.287937906174, 94458.36833167051, 58957.1756474242, 37771.646582376685, 68667.8312595406, 35811.8615901156, 52593.94283437289, 52649.91874102036, 77639.44897324523, 49741.02429556587, 37918.549627452536, 86081.28379579243, 28588.009655962876, 81481.71225164164, 59444.129352411815, 6724.1281922979515, 59235.08739658922, 66411.63687093418, 24983.103198344914, 78263.34687330184, 11487.837489665342, 46075.53412600824, 40875.19093193696, 7836.715430691299, 25778.834868796897, 17801.728428247752, 51859.846779487416, 84447.56391001442, 63656.47610210954, 20926.74097054207, 82754.9346575487, 99613.81032327536, 31871.0884342717, 28678.680383559295, 30887.996183314757, 90731.66019819555, 74851.51886743672, 91304.38104813523, 68299.96914935864, 26074.67974412225, 54602.37802183401, 82208.57835146139, 33959.75854239675, 87059.81032108978, 40678.62163396104, 20295.635694694825, 49547.4652686916, 85774.30242344858, 78649.0578297864, 11663.70796976508, 94011.04353692527, 54965.67382111563, 74930.14786943095, 72414.13126335543, 75488.86091682181, 26739.03258380097, 24637.5480277054]} +{"id": 1871, "vector": [95224.50057771776, 70222.20156594315, 35513.35150017534, 74137.51258497385, 5269.144890106248, 11873.530769559815, 28040.127960460493, 98061.59286560213, 15912.785182576716, 29070.97285818856, 56619.65735911918, 65542.05148820743, 95270.85109104995, 33812.9688414458, 41098.52663783211, 35105.23064322394, 30897.099406840556, 29382.398328418414, 49586.904764331666, 61215.18852385956, 26370.9898999429, 68346.42358391077, 86372.28912775432, 80743.98477982898, 99136.27909887901, 30108.452260097096, 93769.70779018122, 66765.1399657923, 58566.64281257864, 31397.541077700265, 47903.71051898238, 79991.95635938495, 95732.70676986905, 6824.9248913413885, 35258.07749718173, 49172.769812201186, 78495.30649053122, 36742.64330803858, 52644.44124850114, 62531.54437764715, 84696.08951842536, 88602.2237270944, 72017.00418381064, 51780.200176145896, 11632.18838871012, 55983.79569796022, 29159.70709827007, 5528.73388287729, 93101.24829108042, 43884.68873382979, 81119.24324224574, 84084.93071414145, 44190.78892566753, 24229.079981666502, 12663.919267508394, 47224.2292283473, 78289.50309098343, 70105.01107454926, 24490.650702317318, 75259.10024821086, 1773.8452214635104, 5550.13727797996, 43454.538264969044, 65142.03463165814, 89548.30305777719, 12826.37440437222, 64423.80185706642, 9134.949048584673, 32940.93667213646, 24100.258515779173, 18064.5644987931, 85010.75918790775, 83983.69525824109, 66276.81967279939, 67647.06592719276, 39468.62011967494, 74643.38940073104, 10268.67064605267, 92085.13409792945, 24058.69285472123, 47134.498770333536, 86296.08076101526, 31040.528967258575, 37622.698301671066, 85170.67117758967, 10054.129354819674, 68018.17098634856, 53165.65832431568, 99326.6924629814, 67047.30628810007, 45141.617783967595, 65347.900767524334, 4449.696883152543, 53536.326582712136, 30411.705425300428, 78091.84371346682, 28052.372370509172, 52732.14382404138, 42254.68460278896, 24242.11338408888, 8832.119977279395, 50449.4010145744, 42763.06830731855, 50474.731845135335, 40085.53248509698, 57212.02480020699, 91239.83106737754, 99627.55313213822, 8034.935907818342, 87284.68715338375, 71591.57496315904, 27763.705366729508, 28190.78941672388, 5262.73748133218, 52970.10530687464, 56775.136478468616, 40124.222441031176, 51500.25743197797, 88391.8297790318, 71141.62574352714, 16756.999739347, 79343.62572638119, 27039.010951395227, 32935.84786233241, 40755.6066419628, 90414.38034356905, 74727.04835204498, 31327.919002511917]} +{"id": 1421, "vector": [43751.54090365844, 7876.386543814729, 66163.18597836324, 19663.168505022677, 62875.242983285694, 69047.15371169506, 2111.452647885614, 19895.16083606263, 43734.35611846386, 8154.464916836235, 83638.91581979822, 56160.574143812766, 15569.69177492007, 66114.70393271469, 82222.41430609136, 20667.785478638034, 27522.657624892756, 95824.61111800015, 76446.67462404164, 24192.38563178242, 34779.9966826752, 56581.49476991217, 38870.46808748327, 17965.73376072478, 34062.960184984695, 36061.97248207687, 45812.74629369263, 56566.17978177868, 28376.98511927178, 58424.148453287606, 55887.86214232646, 85132.14394771718, 34752.45738903425, 43650.60214341778, 73389.82900499838, 55781.69603684433, 93220.59939099237, 42566.57169783558, 33890.558841176964, 7491.859908333787, 81862.10184867853, 45633.44384386341, 39392.11485862465, 60622.180514416665, 10511.417754184282, 26243.28143182695, 46840.249916480636, 97892.83075458824, 94530.77339542955, 90757.55386123616, 29559.008979888113, 99072.63157845469, 48517.92227573071, 19440.709182767612, 7567.690982428788, 53389.91977923084, 80046.0634677196, 53664.47341252781, 75898.72562405982, 61376.55474979955, 92999.3360330744, 60011.35636228595, 73347.81130431379, 50382.172693590685, 75503.12111260135, 9515.855985608347, 91372.23113611322, 1245.1702481173666, 92539.36620650452, 86794.99714656241, 30151.737016937765, 83373.90663581893, 19527.737586058447, 41567.70731004946, 7310.7507448715305, 13523.240982177309, 47928.34838260579, 49740.67227126096, 96739.77663974951, 41071.986340684205, 61625.383465700324, 9078.502012605171, 4809.421169937534, 36844.06251463438, 3452.06352068429, 88853.1210046223, 8584.322884442498, 86592.89894884817, 27922.25568728538, 48841.06362705388, 49488.18121503652, 89280.88962848972, 45527.30087685934, 71323.54409307193, 80654.98036984907, 76199.14389107582, 10413.679837753032, 52807.36122416605, 81552.42459162266, 25272.591200869134, 86078.45446307333, 73721.26497465464, 77363.22671435679, 69284.57962597096, 22662.631488486808, 60592.38266075849, 80085.93605577492, 34811.834019678245, 57345.476804691534, 36965.730976263956, 43914.14051244169, 83540.09469398868, 71285.18722057677, 8137.221291499808, 54917.88493450852, 97726.29840176126, 67866.55662492925, 86098.21707584673, 3184.42643445358, 59005.72866313653, 47030.18807230701, 60429.851906074095, 43162.56956935953, 23926.725741665778, 67011.20927618511, 27267.156555513593, 22533.926697876726, 69443.07276079996]} +{"id": 464, "vector": [26907.173892180548, 14918.32238126436, 82075.8307389608, 18028.568973276226, 51214.347625245195, 97633.71118775045, 57430.88066339306, 21674.306148259948, 46307.71197627014, 21358.732381473477, 19897.13528694298, 31949.349572892217, 50227.87621260513, 77792.50588693148, 4688.459585367577, 72003.82648129658, 39987.27867885412, 93480.29974813758, 67619.91793721652, 44928.78766945717, 20777.419230440497, 30975.591375812273, 83319.0912962475, 95724.96589794861, 10831.045120264293, 95930.56929224094, 44004.00294071746, 81255.6894381826, 67077.01821504431, 22063.804572713085, 45265.84896811098, 37841.84986414359, 23893.02480132026, 69596.61108059675, 77409.70482741216, 87606.67754321436, 7333.746236342209, 60422.8786379722, 74848.17643116116, 63063.11385332522, 76822.61602390249, 30298.67242693993, 25065.810389458908, 75253.98461733981, 92738.68273835609, 37849.55299060163, 4837.671684849154, 96103.12862020185, 35895.336561478405, 17928.853256301212, 62450.14074864741, 43000.580692086354, 74200.63913910174, 71678.4317808378, 59748.26555675855, 69442.50103312032, 15619.52395159487, 83413.9760033486, 86094.89109758027, 5163.227287751648, 12146.582507532166, 7438.421038002519, 65137.84415589878, 53088.456528667026, 67817.01264688619, 92386.51741758476, 34177.75788109857, 89023.89030831274, 70696.86887962595, 98050.75511436099, 70756.26617117535, 40836.303575286336, 54029.31260033844, 23834.114303406728, 58626.31000187471, 63171.41045139024, 49713.99640553381, 41461.75352532899, 84660.01069639194, 95519.85294320776, 1451.4159918394332, 60947.83823026473, 76594.39771624796, 7070.254091323269, 49920.258400974184, 11327.354905464048, 27992.45047334854, 19689.19772233465, 98123.1750712153, 62712.94533151204, 18993.81015368963, 91848.08534213541, 92885.52878542762, 46355.72883903863, 74110.67337819301, 36517.93514288997, 94808.97397105687, 81098.0766876231, 99415.38817560086, 76306.16742316831, 87790.51745951336, 35256.75759555827, 72688.65310824366, 71787.66264829926, 1505.747215394626, 21885.907223622515, 88553.16639564905, 69886.82707386428, 1275.1326838399302, 90064.83970581106, 51209.36642842392, 18598.00625584842, 67901.87056930817, 79413.5399607039, 76483.93560196485, 62439.248664618244, 47261.26167122451, 5062.329073169048, 38395.313465543215, 42739.8635804553, 34693.371298051745, 15545.929387245373, 11435.510665109472, 27089.917015286657, 88836.74925176818, 14558.363084846736, 93952.36487467441, 77132.42438074692]} +{"id": 1756, "vector": [94672.18479406367, 4671.382023809467, 18554.51032383676, 82291.78880656266, 50179.59712315537, 72742.56649649756, 82537.38108440771, 11914.889880813062, 18498.618905563093, 5050.443141626914, 46711.64779493209, 55675.777934295744, 54874.35587987034, 86507.33919000275, 77930.2189820408, 33928.39942302074, 73852.67693119068, 37968.2376557803, 44882.13422656201, 13600.727868939144, 12500.035935598664, 87733.90725954834, 10309.58304763222, 79426.86102498705, 52835.37506666449, 13841.68222373845, 11285.221763081965, 758.0681074730977, 14278.7081130476, 24642.163786816094, 81311.55839371702, 21013.049670035387, 59803.70543522926, 66348.8365738146, 3699.8208563526314, 7519.742810926333, 74063.03216306689, 47026.22551386889, 34990.251621375966, 21807.47020859518, 67807.50159350374, 33670.96620776376, 48729.314970698775, 70142.18240928178, 86126.85636884319, 74556.9715300409, 82833.83238060492, 35755.22948215726, 93814.79093392368, 80208.90921482713, 90497.74153182369, 96880.86095316477, 89508.00584323495, 71202.4316261386, 95677.1865536467, 76089.88462478398, 97087.5701910613, 69701.80781772699, 18972.57402339434, 82477.56243234724, 96677.60035786603, 40875.677912083374, 70372.66021573594, 82210.62728721311, 73664.93180658892, 33455.88399902073, 39991.56076134938, 24584.966112289218, 43265.713454388424, 93068.38695854304, 89935.47631970207, 62486.43772955084, 7744.704950499115, 24763.80988059962, 46630.01193235561, 69206.41802741142, 55175.53500722674, 59189.11027240983, 87082.83967197957, 50232.21344285181, 13518.001238544419, 37164.86259401303, 57657.19986120742, 85637.97312343991, 26352.771884849768, 93204.07456951737, 31083.911006297505, 16632.86373320274, 56229.522027289095, 90516.14239931064, 54329.96162189037, 46483.48110580904, 6685.151799138278, 8101.182973646848, 61031.49163517021, 43100.992349931075, 92805.60586173034, 21618.33109050284, 22655.301725868037, 64314.64457110128, 70039.64108458164, 76587.50386722975, 18845.50571908641, 19527.43568433738, 26511.02985620831, 34800.24158239573, 31699.762795628827, 99684.7324896671, 8056.704292204975, 14963.880096723715, 88330.86309281108, 43114.602692169305, 99008.76993708979, 79557.57072161914, 24352.628957530476, 14874.289589554468, 74428.04742252927, 40552.3030030732, 48116.46340362114, 34433.09060294066, 33530.226820787415, 51746.58348882181, 37988.92501043798, 70264.1893799825, 85689.79910086148, 57276.05543138621, 92846.25158682092, 98096.62671529515]} +{"id": 149, "vector": [113.26444493534594, 17327.123022407188, 57796.83251494251, 37741.83291457401, 87408.06321741261, 81367.31881817603, 95510.36023945716, 5097.086917044835, 99574.3671184644, 12212.040792357071, 8611.967347810345, 11694.768152935798, 88841.01055317305, 50135.08290538532, 84719.9411830078, 38877.66658712077, 65091.83000012654, 24072.47764008864, 77912.93095931265, 79712.75117223668, 68945.36824487722, 44796.1999205896, 54655.21160965417, 69162.96368779265, 97542.37964037454, 94112.01726521042, 72573.67512870979, 55725.537018921525, 21880.827577873606, 7967.21439194632, 99887.9349551529, 16172.649143222461, 61513.76406559501, 4436.047024380252, 23627.879602845227, 36948.03367105433, 57938.65165383465, 10021.143438473511, 9815.327471867118, 27786.039283986076, 57007.87028930575, 56432.16876826838, 5547.287359923536, 81693.38658616175, 42953.08726135603, 25688.03510029132, 3262.7205197268695, 664.4952480252098, 15733.507690789216, 19035.404096023023, 21072.331420800085, 76761.56505333017, 70052.96793215764, 51558.27821244222, 96278.79308651896, 88678.10983432883, 55501.3160779787, 85501.5667329138, 73797.80920499974, 43919.89109753922, 17569.36365396, 17880.911499422436, 53970.25300662674, 30998.4039287321, 10863.40604866075, 52660.286806037715, 33111.81950344036, 92542.63926426598, 29272.62382421313, 83588.75417170249, 63802.358247614044, 93446.81620154146, 7733.4250939744, 60580.921611381724, 78396.11879617951, 3308.5416117017653, 17849.425255808772, 93361.0518045963, 43771.60588992045, 73098.36352132932, 71522.22400898815, 73065.5654388556, 38128.668295396616, 22716.08168854905, 80372.5355177683, 6863.168414076781, 41401.41849820291, 79869.31822464964, 17269.297059152577, 5180.309522219884, 4284.414578341689, 89187.74079333834, 85804.0181878097, 15841.248282094812, 85257.13549681805, 36647.38867291692, 62197.417083484004, 54436.68778207636, 64572.15885530593, 4442.3320039528535, 23489.635383668163, 27300.98963091323, 45971.35336782079, 33824.84226864915, 6540.658186614667, 19239.986175180933, 63052.637954066246, 25864.474803091318, 52194.11807746329, 2439.993797821782, 70140.67414856184, 63021.704460633875, 55827.854862080065, 77853.44802048756, 66238.52406808731, 2405.3971873510436, 80816.96659034671, 30944.678210929476, 46004.7680334948, 59372.0954377965, 23312.564744178442, 35602.16072616149, 4594.234621582538, 72725.0975507399, 27890.615664605335, 90850.19456017968, 78069.75112372954, 92556.3087137176]} +{"id": 317, "vector": [97644.84508668057, 42830.60550148346, 49781.70195829857, 70184.18630100536, 74535.59170999259, 2216.764251662162, 38691.191190616606, 13808.66335703329, 69800.58107275942, 4496.02221167692, 79683.41361151786, 85382.2520572529, 16847.177256367628, 69969.19574048037, 21522.10617742658, 27141.554984948, 10204.167853337209, 54181.7072922444, 64073.576144816114, 88738.65485404109, 39424.883764069375, 49393.39448732416, 39956.770979095214, 66744.59499415137, 14874.708184051455, 18165.85719318029, 19988.27187566652, 76247.46432780488, 33899.01174328176, 54173.145209177645, 42069.69958465086, 96298.24134149168, 82884.19922611627, 50941.83009754194, 36195.84768678332, 33097.1737864468, 63605.03602235995, 19816.70715135214, 22758.320160072297, 56677.80410666838, 94444.04915859102, 57434.17110041316, 57146.28542927653, 22764.03467109046, 18788.43228698278, 93904.19134034781, 39730.0583167013, 97565.14262732906, 86536.05820374428, 81366.21268689416, 26941.240628244934, 34080.39898350502, 53395.701467930005, 9853.96999564918, 37297.410758646954, 47184.68365089948, 49534.32456114275, 83536.16705250634, 62260.397729145065, 29803.333759471006, 52285.77927801441, 23891.799252075496, 55117.064008551344, 32289.493677198356, 61231.17608006539, 14465.884808542161, 80142.76156824073, 26367.85051841063, 58000.04199147113, 73006.17130584725, 10481.481760357936, 70451.43641489687, 38394.58918399935, 63155.71954025197, 53556.441608886154, 33481.27185434008, 98327.94139227766, 89516.19242118223, 393.81457561152854, 94541.43469978789, 21151.37125708101, 29346.433642525306, 36979.081539743005, 95814.48392809517, 94817.32933606974, 10615.38714277912, 71327.42836389318, 78270.99915373985, 32357.54068831582, 39126.57620627258, 46641.870708476665, 29254.779511875306, 31625.735941106625, 20067.148341554297, 81244.40259939873, 31942.432530266342, 94632.02642283625, 79542.76719819402, 64289.94462225482, 70296.06184810949, 4623.553173771522, 93765.2362908201, 17367.857893013716, 33323.52705429566, 42706.60302356859, 59911.27141484539, 15831.90175751511, 87057.72222082173, 97151.57802811111, 5393.569769768447, 48964.720680300445, 19713.879222526575, 73098.67456266805, 59937.18579599184, 90875.19650780024, 6372.503690856491, 50796.50640193198, 63511.97888014689, 95598.78654326862, 84062.50512248969, 49888.07576535545, 83934.78273839841, 52021.26775691273, 16413.266123158144, 91471.0455504279, 67097.29079469734, 81615.30014234326, 11331.748247313311]} +{"id": 1260, "vector": [25806.51769909731, 78504.61944153733, 8935.614611788356, 25070.329945559068, 53159.38521366224, 25910.574867353454, 51538.79907240156, 56715.41795499706, 93281.91446742734, 36604.95145393697, 24850.234703508857, 15889.883190124188, 14630.491923763511, 45042.97830800139, 25203.205177923493, 67753.02325941129, 51493.6875064248, 97427.85530821414, 62306.66468746997, 54381.758643855035, 36066.528361839744, 33543.11128918162, 51915.425979315856, 67777.93518552008, 50057.897559556506, 64669.5047196705, 83421.3796834648, 98665.58574201987, 32119.915402926636, 98615.61523066461, 57450.03349712489, 49291.16179216164, 23237.03291354463, 9678.237034852922, 98071.2841232139, 7006.5449882732555, 49337.22195859492, 99996.92222513066, 41700.22677665971, 17561.644748204864, 39987.931619079, 30375.70776931463, 87800.58697378032, 74122.41491555324, 63060.4532861505, 76696.99916483273, 18399.89597524504, 47602.66661253341, 12684.908778134251, 94523.08260412555, 88753.88617498965, 64807.32614266544, 22022.056973547766, 50780.01261517362, 86333.39409443729, 61462.206328962355, 99299.35928730032, 42803.30423348709, 60857.020737814935, 97999.09813843307, 26581.918176409214, 30980.366422126302, 30414.260930906014, 64987.500199322065, 22765.81343349279, 88698.1885078461, 24784.747882620617, 75783.82958416954, 51792.701865942305, 98372.99600890349, 69004.87589359406, 41672.75744240268, 23597.25052657028, 35608.567498902565, 84290.85678981198, 63240.05092446056, 32300.82972638629, 48024.90426564643, 39585.469391087834, 66309.53191376521, 47901.409733909626, 85625.45135451778, 76154.75697887332, 1120.2051074927333, 52873.88019620543, 40669.77935631406, 23744.915864122697, 60680.627268118944, 35779.37916727326, 70271.76886428166, 56716.70399497607, 40844.29481641505, 35061.330624668386, 41781.00513789691, 56975.10484819303, 49820.20099904171, 34937.851405797825, 21327.218154564966, 17006.615327176823, 47042.02034168213, 85306.71087647315, 53757.040903311514, 53219.25205479393, 36355.86246863175, 68916.61943166886, 9220.372766426088, 85640.58229649907, 94195.98463836176, 23205.009906010855, 38800.21549092793, 43704.53986542794, 93721.58203242297, 25735.199246631313, 84685.15302312838, 79086.35430854348, 63542.51943138656, 59789.35559036076, 6748.994393900986, 7660.945065735325, 84963.62659197536, 1343.3138898104935, 26642.21177293594, 94929.07705269576, 1893.259619970078, 70796.71490821663, 86273.29908220393, 57099.28709953122, 40221.5450252847]} +{"id": 226, "vector": [63629.79513227711, 79592.82687426213, 5830.920230833647, 46421.77811561946, 48630.612078320926, 98691.64717152064, 86320.03597825876, 22271.04640464097, 39804.57216837723, 27063.646649088114, 90022.59754892818, 69167.55803148665, 85072.68668726753, 7989.567883462046, 32634.209214453003, 27212.561442619644, 22332.54841281518, 51215.46071185579, 49683.11737332349, 90778.37769897496, 82679.44508773855, 91550.02822045884, 46998.496475063155, 89020.09038306093, 501.2953382536378, 38199.927037395035, 47972.39461152246, 79766.84349542341, 87498.22768885827, 55089.412298691845, 72305.79712418113, 99075.09922862268, 80545.68597919379, 24164.914709020548, 78060.41010063812, 75555.24904437795, 16556.347943975037, 34534.58072982334, 22134.405030742633, 41977.44961552666, 77526.15137468153, 78448.57676367127, 79786.64623695727, 73429.34643834656, 49298.04656070928, 38654.98296981265, 10800.357303231789, 31538.61042567311, 36330.75433956017, 66001.14964468664, 93563.95325887024, 74966.23436617739, 22937.188546975627, 56347.840680262154, 96176.72079456804, 47274.68729209491, 15111.568723634538, 86822.81987638611, 69362.14782698805, 2101.7094744359865, 45599.978086936855, 78941.18774382648, 52786.1848541202, 89286.10397370855, 79346.30394159218, 25255.982385681153, 46789.36361352328, 89592.97595106538, 18224.114487144026, 49730.87490318805, 72738.03687703668, 32010.05247520241, 52526.507533194344, 31536.91535574609, 79357.87440238817, 97026.49318030944, 95506.68197709827, 27348.075409586436, 50201.24249023782, 50714.52045890297, 1991.2006716855, 51793.80507714077, 32301.621311384377, 65598.95283508951, 61847.11423581179, 65663.0319103456, 52443.075959420195, 85582.85699337664, 4002.1518279525603, 32590.388594946886, 54860.53568427651, 78851.97362547438, 88342.34515282643, 23183.748921953338, 28645.464181962565, 97798.82927376429, 21612.323496773522, 31994.48926246594, 85365.49575041243, 85636.73616904631, 92565.79383664922, 66075.5450588104, 51309.76755843807, 96493.80174167216, 95525.2833355917, 2911.3887636881495, 67748.88954870708, 2362.554233887071, 51019.97991696062, 33781.73068304162, 51265.37110411022, 8743.5414736579, 74787.83500370472, 82078.77052998483, 23024.45363231862, 45144.531149205504, 81593.99008373666, 65743.49683562818, 40401.73382762712, 6484.34861431968, 59256.061336600855, 97120.62202011001, 7852.8106232420905, 19778.63182168187, 51335.53580700758, 65600.35327043431, 13384.672625060668, 35492.40386743478]} +{"id": 844, "vector": [48443.98372463813, 40242.303864161266, 43384.85153432726, 17601.899036015267, 11316.604876849002, 46659.22968890315, 77005.9789951877, 26343.050140123236, 39542.01167709741, 79445.53694106187, 78932.90606488875, 35983.34344331468, 5013.383134774729, 9341.89197105354, 79052.02334421402, 13638.39696338739, 32238.05804463692, 49603.16309203634, 44731.37598900057, 91499.1671645382, 58604.66921010847, 2486.9306926092904, 23004.072638517002, 60874.9604942583, 45311.67639121756, 50203.25139359327, 64438.709699101295, 39708.2469985265, 42006.27910033177, 98641.20318175308, 85505.023167727, 50973.26861156175, 4503.0634889957955, 67151.33507658968, 27545.370620408903, 42230.43958731517, 1046.7765337948908, 79405.01544617, 48726.022975394786, 48332.19643770383, 94313.18277822166, 10066.612622185256, 87580.56668183501, 64162.81552661124, 88689.80761318073, 39284.76740231992, 55942.17245351487, 21743.60532496844, 59751.67266711736, 68244.82039316285, 83996.3392330694, 18590.27304478651, 71081.12966202117, 45480.24319396325, 41759.806839906145, 50861.53974676105, 68281.91499916972, 93499.3859174263, 3126.9085518303987, 57341.48444968795, 44787.96298730692, 92454.87462918459, 57803.39783762394, 29335.06697398591, 83030.31054936223, 15132.698430238977, 81973.20210213788, 81839.46604437764, 4424.75564132222, 47807.9660178452, 71404.89264519933, 19378.67200434531, 1294.2442142866662, 49559.161382770886, 1199.7058302862795, 71633.35493310126, 34637.399602291895, 56610.25443418325, 87638.75291733342, 70526.6951825544, 46743.58745660343, 90810.15710196042, 6740.65004564115, 29395.85882208181, 66064.63737294875, 76746.30140828717, 36977.302777798, 79802.34458008854, 80168.70137414134, 30520.704123374242, 55693.94145776943, 30726.308896582654, 42771.16726397843, 45779.62448870106, 1043.176941463564, 89174.36225041529, 53510.205589749705, 69414.51707313341, 84147.91775269683, 49630.82272383831, 75245.13602730638, 76580.86831269818, 51197.122470611976, 81693.09516650719, 67536.85622648233, 3873.2084391725816, 46790.01090003156, 21101.323723238485, 1721.8950037927393, 55468.50761954042, 11946.716464376705, 34429.21801758387, 53046.80867493635, 10286.891369960349, 48944.49903861562, 1449.9026366380785, 97088.88798183658, 40376.16058052418, 16425.63627462226, 73360.51577635533, 9409.0798832972, 71058.2294360316, 15006.395515314185, 70293.60822988093, 93767.87760095166, 3711.8958411687, 52973.61941007935, 50807.52008109265]} +{"id": 604, "vector": [62958.45378218763, 44650.53962410588, 71510.99137776314, 13367.23547333627, 77436.35562507506, 59032.148231112835, 49463.47388617118, 51729.39336718344, 88788.48209794446, 74599.18487172808, 84794.43986532597, 72743.16231678579, 11499.37478936226, 50588.26721973153, 71139.70089145697, 22348.83312242971, 17495.72230083547, 73171.27392480719, 73548.17977595635, 36880.65783680432, 67959.26657414781, 33281.71941495216, 37259.37759163172, 96745.46104600203, 92410.4507860226, 54017.22848538854, 632.2818020375598, 88307.75830501992, 94809.11321996614, 22724.581417470923, 6699.694590254923, 98218.69320162498, 47566.675983584195, 40266.2560840239, 90939.7322823485, 95173.73900331261, 45527.43553262964, 82075.27662136428, 56409.33045557687, 58134.8406008047, 33573.84743674848, 39401.523242361305, 3642.833318863159, 60048.70706741585, 27702.421628028686, 33981.323235489515, 67595.09813667338, 90939.40769113453, 90481.05006533056, 29980.259933664565, 34843.61995061021, 4444.464293005379, 71708.87344856712, 86264.61832652338, 76303.39691846183, 35203.77722413737, 62002.09295127621, 57232.25607610021, 23674.687004146468, 24428.02408035951, 24324.66391108524, 15375.606986421719, 37674.27097025681, 62421.762641165515, 51995.47222939716, 90891.59895835849, 15366.787510451852, 51647.223857152, 27209.942850297564, 39117.725996562345, 92559.39942345142, 4660.388725903997, 62726.54062207713, 67833.25995823501, 45203.45159715374, 50542.528367205756, 28736.341170103675, 38454.49837663637, 61313.92674366074, 76796.425729677, 91895.65829161613, 74600.3834246604, 93491.5199290836, 9106.83636666556, 1851.6340822958277, 90229.1198916359, 62674.37178515861, 85088.95545166328, 3694.573586163108, 9360.383126217697, 78480.20199661299, 1059.0540474218035, 71173.026529287, 13199.557722028676, 6061.439356385367, 72115.77673590872, 17763.72214379567, 52244.8925511112, 29586.51653746016, 85500.46539166495, 21955.953285758966, 97870.60393846019, 61053.67402383942, 14186.792440724861, 11294.733821303338, 94107.20556195162, 71256.74280201056, 66271.52683713625, 48776.22165435473, 14514.137181672171, 37127.290977984994, 18883.01314089972, 38617.36371915818, 66553.05800051683, 83150.19223873422, 11289.045092096827, 2314.4669555138385, 12632.438424713111, 73209.07553607057, 42054.20869887715, 61199.63239446686, 35163.75554857167, 62154.87630973452, 54374.653702612675, 69664.69702554797, 48775.86972441643, 49494.49523806184, 46271.85331421373]} +{"id": 600, "vector": [93527.62765398045, 27834.64919593801, 35201.688833819004, 36064.870188132336, 98955.66547592521, 32168.0509576722, 15874.155438924108, 53536.086204492814, 72647.95325006204, 29262.812039965804, 4965.66490315139, 14677.418880428793, 6568.376688667321, 18550.51544172156, 69566.31307268875, 88402.47978833351, 57316.622710175856, 14805.582194282608, 66698.37177475532, 59730.63202537352, 49232.93226873644, 85508.16261551798, 31948.91464931683, 74314.97215600107, 91940.00525371807, 28527.27445849702, 7247.956491692109, 26531.221549593243, 18556.95675193263, 41968.29889425705, 11044.257607771402, 79273.60718898983, 76743.60072031728, 28285.926530582885, 54194.81615796892, 23123.31990474409, 30479.286282340756, 60473.06336542446, 90569.88462153923, 21280.663238111043, 20599.707242562337, 77172.4895483321, 59027.173096431805, 78392.51135829564, 90524.01201224557, 8150.8894025782765, 1667.8467748082903, 51740.66577915158, 11888.699694135907, 74140.44712529829, 34100.238332713896, 84405.09496179187, 23742.52116752732, 82104.4660438914, 75104.79411793912, 27171.623881787167, 72555.6713044053, 92924.05745444803, 8036.304652324833, 48507.21375198106, 66894.04425768624, 49495.58390622977, 84410.13735394624, 89001.87817576998, 63318.773042871115, 94728.42474034341, 40428.22259988537, 56914.44790803417, 44171.49335884369, 25908.849200567674, 1939.5886998838873, 82785.51365513793, 38761.25897007937, 35779.45653561788, 33011.741929783835, 22338.23124530029, 68491.73997453434, 54088.89348242426, 40345.522097693174, 3321.4141525228724, 77114.18402817614, 48703.7012595487, 61410.02113645401, 91629.9618121284, 42593.00073133262, 51242.79470281102, 62124.146481028896, 34142.309698820216, 1659.259548357961, 37341.46826547301, 56533.13411742527, 18669.22481706583, 27549.290292515172, 74957.79294281577, 42951.76415045369, 7013.215677399109, 5569.827256679294, 48809.999548687454, 39195.04846409848, 88198.55336609708, 22132.829619828288, 29145.302454012966, 16079.106417997491, 9750.742754621266, 13355.614252379055, 14682.395032968143, 9444.328151313219, 46975.88434340089, 74467.40825111684, 99835.51247214254, 4387.8276544366045, 49119.61928499261, 50542.265017957136, 2488.566026841843, 13873.828371831243, 82987.5189277853, 57727.368958167346, 57219.39976186108, 85944.29035602341, 33882.803444320496, 77661.84020001696, 21155.36034609543, 38805.150007922784, 31431.105592146414, 43768.696452503995, 50176.49446159984, 54765.93037400127, 12731.194528942291]} +{"id": 1485, "vector": [33373.37874218188, 39963.77058367099, 4975.993741251028, 20728.67057677119, 33820.41797358444, 41847.9302320405, 24958.211172207066, 11653.805597422574, 69113.42329049534, 30903.424569606086, 59229.243247251754, 82171.37391836688, 26141.071972882124, 75848.97322682242, 78683.26823007806, 45487.08363969097, 93826.49430145086, 87892.39870559626, 79590.18560284133, 75957.64857147094, 40120.98354854412, 21420.202896193154, 88197.72892350341, 60199.96739488329, 52838.00392340331, 7427.8700814528365, 56534.81871176222, 50533.25011375386, 55028.50745345187, 53055.87281272742, 18820.283731818865, 79325.97414774331, 78882.2886431144, 57591.12740242169, 68000.69387718993, 9704.083368691108, 54785.385603604554, 37364.24049848008, 98642.57234300085, 53528.006211216874, 22814.24462577729, 75945.19734065319, 4132.5564935414395, 58691.20442392023, 47365.71607478655, 71042.0661908634, 20337.938792504752, 71088.63253061239, 74062.71813226111, 48753.65794702129, 62920.263712088396, 17578.747161352694, 52917.040737420684, 2966.6039960348776, 28808.18062035969, 87942.49895883721, 97435.50776394401, 93921.97113144364, 62368.84684136639, 18044.875238327695, 95141.13834812846, 84568.70801605275, 81948.93083931041, 4970.825179138527, 6144.301333637114, 15617.832546912912, 60675.106243996765, 70314.42000278384, 4245.1113116300785, 69325.93837753651, 65625.21797924505, 4863.094345756813, 78001.35453477917, 80189.70040156403, 52801.069350779995, 82369.38232662318, 47420.12261144908, 8907.611214125833, 91300.95957792144, 13984.391153138266, 29941.470503709188, 37664.743721070525, 3646.7143051830853, 8533.35153096021, 94664.98644983856, 46175.06969037395, 37474.68776779025, 76944.94139771361, 23539.601768675187, 69913.64486614167, 56047.97885619484, 16693.62108110055, 52835.190342901595, 16960.335390530203, 6980.582876971275, 7321.4407240059345, 22409.460207365493, 43999.37463333802, 27246.702756183684, 79703.50924834028, 64069.13847300275, 74989.86578326164, 52119.66309266671, 90964.98504178021, 10846.587366337046, 71647.0359318761, 55002.29347768116, 99478.94890867347, 42970.55438321927, 65098.37558039704, 99989.61425871709, 42948.41145500088, 26385.32582495903, 33042.6140757908, 5639.502938703534, 16598.25462117397, 13948.555350527691, 83567.97100281999, 85377.84018312524, 87285.52860475448, 16024.91946275041, 35128.1890684461, 54746.58324068575, 37134.114530853425, 38414.25878590787, 22198.197763425775, 53851.27426911582, 53898.96476818443]} +{"id": 154, "vector": [44064.19407056876, 38254.376865525766, 23216.658138921986, 3723.1057253103163, 69784.57661788694, 77048.85203665955, 367.13561376974235, 22366.452771500324, 32918.860574391576, 92524.21551125482, 89343.59469021943, 98488.90991936905, 70366.83489916974, 40110.27981852579, 79715.57593692587, 58912.68061161048, 24068.795346762363, 60300.87659299199, 88376.58444802638, 94456.99330406431, 77732.57995189933, 618.5557956375098, 6680.696147526144, 71129.52162893779, 42172.3943868202, 24194.984929714148, 32607.664812290615, 7958.632077890515, 69146.45489868104, 37771.2002726157, 84212.56679514302, 71546.6346235945, 56621.03050365025, 97323.6661982038, 67227.42636023357, 5938.160757350164, 32081.640101036755, 27163.07574823508, 25412.613884860526, 92761.21685268937, 62636.93051578413, 70998.16011855328, 91595.12165947954, 79334.15981000732, 4097.837838239137, 90193.58387510543, 85628.52764732628, 84841.44157730785, 78590.01417304328, 51207.80422747281, 70565.64151686677, 32118.975765998402, 40347.70630126236, 55716.44676402872, 76434.1028602827, 70294.87625853364, 64314.54637730402, 29207.08024086447, 75465.97062245854, 61567.09620919927, 64757.16877360844, 2918.6977652509217, 1254.0305598705447, 35682.70959997835, 63212.958489605684, 78936.98832774446, 46111.367681065276, 63768.77068814111, 91082.05435647568, 55773.15894242015, 1615.3225326669628, 59624.854125939484, 9419.141444388279, 26557.582192911334, 91388.83726139154, 94533.70590282399, 39268.476801435194, 81874.88654198105, 40857.81547643751, 7880.262743054178, 95812.77052591121, 72639.72707947926, 18730.151523955697, 70928.59231000164, 88194.37373267057, 90958.33571389952, 28793.050208423974, 24968.088313681914, 90264.69990572598, 72651.38426790164, 86141.67004949502, 34438.348223088855, 7185.638865044586, 69625.36143464831, 65834.47222182657, 68615.85273924022, 20202.791257937435, 56335.737212627, 19261.46853721099, 73093.66967949596, 43317.22170230088, 81275.27321370789, 9436.062102015285, 91023.92554306157, 73623.14601272879, 89881.88327019112, 4455.547717428599, 32284.31903258936, 85310.02069032086, 22823.910139312433, 53019.78906818373, 33038.74908222819, 63854.13258613446, 23773.257759571243, 40799.12283494761, 92483.47600800352, 22429.964320094965, 81667.7823375447, 70700.12636438521, 60295.18890918109, 55468.91718586699, 34903.181678290806, 93494.01558203381, 35879.34747290367, 2698.866958635293, 15068.429270180783, 80464.47993467125, 22211.365321209953]} +{"id": 2027, "vector": [71604.5432420498, 7702.292443710157, 13807.098473449974, 3706.935239640796, 83713.31093986373, 59011.89625153124, 18799.831090359443, 52010.97253162359, 83622.95005942526, 84089.07880072006, 81095.9397060405, 77703.34410450842, 49517.06180043927, 38626.72766510219, 45745.09764530049, 30862.214896296126, 63336.93659172938, 71610.75292849353, 48962.185532365496, 11208.798960300759, 67399.07749098416, 77891.73868206718, 68678.27060049713, 69441.89120963105, 92340.24007547513, 18270.907697174287, 10327.055502137027, 16841.950834944786, 63194.835392037705, 57600.90992494532, 15567.801582403961, 58058.71106753325, 98015.39176177423, 84416.47342515487, 28228.254877590687, 29310.376604101428, 10595.53966668456, 28815.212964526127, 29686.794401225103, 72805.6722625821, 44142.531630546066, 54494.32248088799, 41726.08164898893, 54807.48726310141, 34337.20337230735, 40100.91749534378, 80196.8444939866, 70581.91839272964, 46767.71706005686, 23713.621656940762, 74986.93246266793, 70930.09648062148, 73590.52808328372, 62739.86368812999, 58743.02156362883, 32889.704340189826, 7723.674922799928, 98251.11896506371, 50108.39775341695, 38082.98419103835, 2973.1475168843444, 23070.99907849285, 62974.01266262508, 58280.55546506522, 63135.28132207657, 72560.52040974601, 11533.403142517396, 89839.4892727444, 21342.785150020816, 75988.19414095832, 10854.79851002702, 47737.850075140996, 20459.21686086787, 69260.07054593542, 55396.1988178435, 38189.73146933965, 52624.648594004255, 50563.4250428076, 4636.232745737412, 64830.203034204846, 46869.18266464407, 1522.1736632870964, 81931.00750482481, 35047.994844621, 7561.899081114554, 29892.90823495706, 95473.83048915007, 56248.628157136714, 77348.3900536276, 57419.19026087463, 57753.17190881019, 33202.48588009408, 41555.36083742703, 25486.059685261276, 65298.05333406878, 5627.12571002556, 8164.5644128992, 33384.02612708801, 48574.86552435109, 10262.965761433241, 51201.6095759636, 83642.62192309531, 23517.72290645232, 54438.5703377392, 60042.268367727826, 47313.279155612174, 69748.45104565113, 32417.84312475112, 38929.57054291567, 96516.13287762151, 97567.71351595006, 75376.18207271512, 41505.6414318128, 45591.78963511907, 53397.85487091401, 29392.279101813536, 81752.16243253944, 53296.88493262827, 53206.65314559001, 80950.73408116534, 32377.997898887203, 31621.536550658213, 82198.90356633333, 22800.84816602137, 6038.45480945947, 93065.95511653372, 41340.97718829434, 34359.81437500102]} +{"id": 223, "vector": [70432.88293259946, 10052.229805177736, 78302.4830997041, 82289.97813696883, 71773.75454121974, 58015.80782179734, 82202.93743258016, 23749.321562659377, 47452.80062711785, 98719.97271106241, 68276.49652791022, 37760.22098333802, 24086.41358919271, 37561.20882106587, 53064.47326943332, 95652.99010702387, 9924.108624667393, 71282.67460968684, 75189.20707449179, 75720.83963779069, 80100.83892176326, 4811.638009582531, 13167.039006827397, 39311.823491562456, 43849.15838939542, 56701.02651169744, 90066.98513812036, 38096.62071917509, 68520.59725699913, 39949.44196836228, 21659.056210661263, 8438.16420040664, 95798.95041166406, 73587.68047674092, 98663.4829881471, 42088.392285855494, 98476.71215351063, 72055.0133879011, 1211.1452509054986, 64437.09140145667, 82702.3320831189, 34983.147579966644, 26614.834838069244, 30097.090423792273, 85188.80095663795, 73635.66714210236, 86266.58906062417, 31752.188554433815, 82229.46666554056, 46757.885373196164, 8764.290279302655, 28644.38052013911, 50028.55903395097, 69228.67693973736, 55335.712368347464, 66613.14773669698, 56589.329717373774, 5138.645005380759, 30132.98906484897, 7699.672006718628, 19654.535803842344, 24082.684487462182, 44524.93822774297, 96509.75365747313, 80558.2387654507, 32831.63951738051, 11740.714494390548, 2973.3524859267945, 37481.20264033607, 64198.08049325554, 85679.05362648125, 68676.73774069815, 56913.08744864714, 40870.20830311248, 68166.16387260477, 27329.19720081205, 44646.63810614587, 17195.071015404017, 60988.0418279094, 89122.71249501088, 85108.97966685165, 51070.64142236644, 61881.42346760727, 58756.24553220948, 42890.17386289126, 99739.41044457704, 46107.87911121268, 79019.98679638121, 49746.290401178594, 25308.324388891855, 88067.18069616471, 66595.36326987871, 29054.75024364986, 83791.86220892757, 35453.01193132887, 63118.02815927694, 33940.75302332604, 97305.95773124495, 1079.5401127930647, 39818.671403935725, 68099.68336369027, 76629.47777897248, 54077.03301961121, 15760.83808930746, 67589.38124778992, 39809.9508175215, 90749.78565088604, 16792.485223627373, 48881.22604118885, 16862.87151688235, 18624.82032871836, 68203.95731848764, 97200.69142292235, 66329.9722048202, 46564.55452027275, 93479.96358641714, 49875.652468532695, 59072.029351795594, 59083.55236843863, 95019.71140613484, 23079.137379339376, 53984.829809205716, 66910.6034808251, 45702.907994195186, 7717.658407612882, 47995.474115414785, 74516.9065738066, 36665.18823395934]} +{"id": 737, "vector": [83355.99707364419, 33025.402333711485, 1954.1120723499007, 32288.244058172568, 73288.06555989914, 88228.0137574613, 91480.97310800449, 22280.530566027246, 20889.95549028434, 11217.947923745385, 4168.539591905118, 76703.66758348109, 44546.19224337467, 82542.59558457999, 23503.41765155077, 8419.168618407424, 70769.87343366466, 47769.745700697094, 12199.308673192167, 31287.561927587125, 48225.83053219791, 50722.96224605507, 86703.0070272327, 86406.04635296628, 55627.13026785481, 82083.54981505139, 11776.462944327159, 64350.549328340065, 35308.213285089274, 98622.58582411046, 14100.962516855265, 88750.57794814248, 38244.076491522406, 51503.561731837755, 72673.56562814026, 7303.010233142692, 92202.6141775774, 32694.09825400923, 7887.41885747799, 79933.26804438517, 63944.5361676382, 83981.46791471903, 37781.26533658145, 11971.289990793344, 83589.32645886674, 28259.740999416328, 87626.91523343617, 97783.57970805837, 14550.875154756615, 38260.85574846657, 18067.4255914129, 37602.08274781539, 86715.35883945915, 55436.89097320137, 9793.882143347088, 19346.140267710754, 84317.65536751936, 52296.256911801385, 13115.009477667438, 62364.011537224316, 47282.726010660714, 12235.336507000471, 74370.71815347917, 40275.31678477858, 94201.96089347967, 97928.38046268864, 82228.217668814, 32597.891473428896, 34523.53958083334, 40659.65544685075, 72949.94072022373, 5101.197054372419, 48659.331308779554, 35231.971819967766, 62980.00704986498, 9200.39040053492, 25380.937183940154, 53872.55188088596, 9264.468026091488, 41342.865771006545, 92966.85355726603, 67753.69956630196, 7285.002242412164, 65864.5091087595, 91263.35781534563, 94969.80077854633, 76418.94380717154, 77515.28578262287, 53192.85502488694, 49885.41340218107, 76692.38833894771, 76064.12523130314, 61114.255649418716, 27053.18174915282, 83152.69631190565, 6613.193577517984, 44728.46679312308, 51121.67497948588, 38411.97551078318, 32818.37430998472, 89117.8020383622, 22841.639440838924, 47295.96970198889, 44966.569904972006, 21712.507580600315, 81039.09256357717, 78272.26208347753, 65929.90913148156, 45773.03764430602, 60372.07276121888, 46040.154878785856, 13418.523856941789, 35116.427523622, 66212.91670020849, 72109.68504321884, 75177.14691317227, 15146.840125077542, 53206.43212890717, 45147.456099575065, 52979.11798846664, 81852.76498971645, 94669.7718076845, 31398.183708434026, 23267.796853909684, 3800.8154697754317, 16805.001168108513, 1168.4744580101492, 69847.48729395242]} +{"id": 584, "vector": [40121.9878224526, 62322.42462029363, 72857.77639925957, 7379.189870511149, 595.7716607936093, 82421.64100545872, 63109.698695104664, 60978.86226964676, 36032.07626685788, 78587.23274248473, 2571.1094582483106, 26452.618674525496, 5622.714120254491, 91808.61137586071, 58326.78722223141, 18058.67941560374, 95495.5999863636, 3881.7846395905462, 94834.23513079227, 57391.16674898859, 59637.08007795957, 75863.16982006037, 77023.0840792632, 93155.03423422443, 63758.022224613065, 3232.4206567637457, 26647.241245987883, 57531.50897433919, 51330.252288159674, 36751.93876365145, 54141.41784440277, 43733.94167065592, 84456.52283434669, 94938.55034957924, 5410.179191148046, 42359.80428661766, 1437.8721011339192, 47278.21785114579, 23327.98719970053, 68516.3375138982, 22838.777149673573, 7868.945865511523, 96253.30548797603, 25504.313794564594, 90732.5446483794, 42591.91421502467, 6678.457435595497, 84702.75684903828, 50650.7987658413, 14575.916559787005, 97189.25914755375, 26310.598858005775, 49845.27425179809, 13538.967167157323, 78297.24559129684, 86336.94325029314, 60540.005239259466, 29195.656057863882, 82358.41503980002, 30051.248702765708, 87008.78625555987, 44975.59467597788, 13162.867592249571, 7560.019094184978, 48989.54641030351, 41363.56045275929, 23691.1230951379, 90918.30496427979, 19217.863493524223, 87621.84358832148, 51966.257494070014, 16857.311391988027, 74910.43699647141, 18023.54445617057, 70353.58942356188, 9472.80526531823, 17100.915294254228, 82533.60139816014, 57737.66728446055, 89329.73809881638, 2849.6636636561325, 69675.94696130148, 70724.78212293945, 65327.77848656235, 40044.74344649358, 94373.5464616399, 37990.68871633455, 11146.69243273898, 14993.581336283734, 49934.8646670076, 4106.679498491594, 58034.14618918554, 43798.3263882912, 58273.380492595, 55370.808707033815, 38141.520302212, 93293.12982922, 25095.574743140016, 82987.47353227994, 55180.70476226814, 15493.189172938348, 27163.255752755955, 70524.71090607242, 22811.32508665592, 73259.213298638, 87496.37261964864, 86626.70896176697, 50304.59687417209, 52287.47984245016, 6382.89656613451, 49866.08331948789, 80001.57274749273, 76505.69930777828, 50750.35867690816, 73533.3791811026, 43673.03751722576, 92350.61179559246, 24893.64141971634, 40913.1641857199, 77559.80546845823, 45861.57142224869, 12888.143310694111, 15928.808187405974, 48223.512068699456, 71668.93177744394, 60855.71991026038, 52183.984470366384, 64068.66727586292]} +{"id": 855, "vector": [29830.953260875147, 40703.94026121269, 60818.53247335747, 55760.773892227386, 31479.17391326176, 41527.80078597432, 18403.373856698345, 26029.213827166142, 26583.044981881652, 41268.134313694805, 16558.622355707485, 755.9436801764185, 30740.89905881068, 80869.78129286102, 6907.174761350454, 97039.45480290178, 6976.467102579331, 94295.36326394153, 82529.55362216299, 52997.15774020682, 2615.823675307949, 6218.326481696723, 95006.41537442036, 87272.09419146927, 60933.35758947922, 37217.327005829415, 90189.1369105504, 71170.87712735176, 74031.52980675253, 76009.0809877112, 14917.635798519068, 1345.6023918621108, 3031.8520440917096, 73719.62532950987, 25380.114835439716, 72708.71966979082, 852.4075657245222, 11216.831580659436, 9993.39846997418, 29278.297137066933, 34704.419642387606, 89282.81918347267, 79612.10482521661, 90074.65163802008, 2903.798290012927, 45640.73622912782, 41045.56046438405, 741.4167583519982, 64142.22131584565, 27151.737064099092, 2272.0828720120044, 58287.429568939755, 73287.35568406593, 19986.318189052887, 40335.274619909454, 47922.58209086612, 7654.431667506867, 87220.42100863799, 52540.31921394008, 69289.53315270258, 87756.62663960822, 12619.047110026027, 85356.89097129642, 53026.04085680116, 40837.87496527942, 69238.74908580538, 12535.076148008806, 12874.86781511611, 73485.23979848358, 57608.384415712324, 26801.42172747667, 93576.84421652161, 9443.066221464302, 39562.726372024845, 68991.11377073695, 19661.059917205836, 96766.26737399028, 5694.828917808514, 79859.27795883433, 64191.33491425573, 23438.052796821186, 44098.549766137854, 74648.59413136444, 85730.8187728541, 95728.53697741414, 58653.0594324731, 44531.47919567323, 90604.65654350433, 73935.21530713813, 21520.930757002698, 97200.1966139151, 15762.443343734589, 39661.10721489441, 83853.7935272671, 79816.2503300349, 60654.52198760889, 99166.01644606914, 10718.928146651808, 34778.86349267908, 17473.8249957235, 36351.45169484099, 42122.6258832639, 46402.10019901058, 14517.029755804622, 56424.81719214486, 64340.328184557395, 70428.78155134307, 27113.83551743225, 90058.3980136146, 72331.45137780305, 52205.08254861246, 18586.098177200605, 73207.62295617822, 76193.96539339345, 93967.94992166416, 46942.6937354229, 61097.03459870198, 20842.551145248057, 50791.293880826895, 49730.1359386578, 96524.10339682492, 57912.45997616782, 44741.151642723984, 15757.6621623164, 61125.60928396391, 11260.124967512264, 17372.688098031696, 70355.01901463939]} +{"id": 781, "vector": [90615.8795172585, 90327.39269844323, 86994.92594505938, 94382.9264576557, 53048.22586525487, 21710.830037984175, 23654.080904697195, 60995.84501373806, 68050.33658811734, 30168.170119216942, 25206.472152797633, 18169.585664318933, 59467.435684622316, 5570.383191936057, 68141.41389616195, 64565.657415101865, 26200.158958891163, 33134.28933403818, 63432.88554909765, 31460.655884465126, 86251.97879538393, 79362.96194059095, 6365.327166653822, 87028.0973045657, 59761.97330053049, 9961.208772210606, 39542.194124302885, 72162.93002732152, 21581.703224414672, 18089.527044240138, 35442.412037892514, 57891.78664068616, 3273.161034232974, 55793.55838330681, 88346.88753152665, 66334.77234838833, 45076.01799861006, 11768.415103318597, 83924.95673966539, 44848.89214723897, 32273.175007085276, 8904.621835360449, 56150.66246971219, 26849.63195758694, 92373.52112163253, 44490.73070260342, 32696.795596510132, 85624.90889615435, 43669.13128357755, 48575.15130349004, 22170.785400487614, 3324.2684446461167, 73048.63563131685, 39419.12705380635, 4784.84533598218, 65622.48525172942, 94685.98836163743, 45433.20725630944, 23283.38866611832, 2007.6128622949607, 48183.08192144358, 34261.841265245894, 65288.70763144771, 4570.243541940755, 91969.13551436712, 95964.98638508668, 48620.929597616916, 52783.83429918465, 64541.94623339876, 53220.17247232209, 18285.03729775942, 5842.735838438362, 33352.24356525465, 1376.2675092101495, 76119.09074747968, 11409.243651322531, 94685.34261388234, 21827.168881218317, 67623.77050463013, 93178.16642267423, 28962.958348431745, 87749.96128696238, 35032.30328978758, 75094.3404684692, 73924.64271952354, 49425.20173968147, 30436.25528861099, 30327.68878648625, 55186.28664892936, 52624.82110611507, 78034.62166565836, 12366.101067289459, 53399.27825567263, 39419.873021196494, 44546.08474906775, 22261.98222160841, 31369.503871001525, 93937.59168957113, 91389.69852266062, 36034.08605355115, 3269.970453774862, 62662.46673629606, 57861.44368697956, 86240.53896516349, 8344.975252648745, 94994.35924635328, 97228.33755216582, 2801.8795973724255, 22645.240682025134, 52217.80689285949, 46196.986488583905, 38550.52899294065, 66619.10416477309, 74255.86266227326, 3975.821247590161, 87700.771206479, 10335.924883446212, 80557.1829235696, 71428.50784031072, 44456.0111234133, 65147.4330952106, 48778.89473823558, 64721.37264284013, 71381.40603049034, 82621.80046418765, 57828.293989767546, 45035.79441429737, 9261.371027427624]} +{"id": 772, "vector": [44581.33747576979, 81207.59194584661, 22874.579832425356, 49196.41354202385, 82492.80538626092, 40950.492263605585, 63251.391044239914, 48898.81590243189, 81082.57660495568, 72561.86027554511, 88555.90271134558, 22770.841007848765, 28848.066381411296, 65440.55149007968, 19026.03164983426, 5924.948059093327, 25891.658357239212, 8667.160962837128, 49877.03748424579, 45717.1556943717, 27085.67000357436, 10747.100241161466, 58996.82408903776, 53304.96370621011, 97843.74346937267, 2752.617204342234, 10852.44051477715, 26604.771139762863, 15954.393333450778, 69582.57035123055, 46918.85341774113, 32584.34169857468, 73740.37037592323, 38607.362489782296, 78100.87427409197, 82954.04919591337, 76449.90652750217, 71072.72007305843, 88187.24192985751, 11121.204707449573, 53632.48117905871, 27228.739728068907, 48705.10244252467, 74503.48185884334, 60954.385590383456, 42625.8905611197, 89958.3285330168, 3328.200557894723, 56270.974189585795, 47106.800952022575, 50924.45950037256, 39923.782495561114, 47871.11607669143, 47061.53390748471, 16128.580572890549, 83575.49415959074, 3117.0320040097454, 1170.7245448666747, 1304.2730858799368, 14904.021648794553, 77539.79282182157, 71786.65869000072, 9673.064703208567, 51582.66822875031, 85480.28174281694, 16783.844449645967, 44330.54427286937, 31117.58680487562, 96264.99433522308, 11278.411749643814, 21588.60837992532, 20236.092988958764, 98662.29064488334, 36783.89455668417, 74279.98553590407, 11204.051293149409, 54640.89718945198, 45029.09981070955, 86098.00905377799, 989.6217191508683, 15661.28889997167, 61512.41219075742, 88406.15885651813, 39306.663681139784, 77660.88563864154, 56100.29328993487, 74306.839828576, 99203.22138217304, 85767.76454820597, 64740.640815710656, 10876.666423376446, 25488.73343472243, 16177.226351997331, 64143.41567481707, 90014.7818497069, 50769.10201545245, 98496.67609077289, 47235.46238695633, 19445.028531747466, 47486.91912206445, 24718.10295929051, 94455.04799969174, 17368.9807370201, 68426.03676414557, 23636.8784519807, 98819.56621072472, 24213.315464910946, 79379.05012592376, 37979.52756115407, 43878.18370820897, 89689.04826488487, 78312.32951653676, 99923.34133669881, 15713.447139048563, 91352.7191960977, 63728.56150772003, 7720.146968399266, 76261.59152258023, 59928.66569549844, 77543.79968920675, 43925.89467244741, 84233.5167414484, 22142.727651246787, 93564.82203260782, 67159.31787639893, 56449.462484157, 16246.713299377425, 66154.50498562596]} +{"id": 241, "vector": [62696.09140492381, 95080.21115137247, 68986.55929391779, 67621.21043100476, 67325.493492822, 23672.881846035965, 36601.82093672658, 7348.554148364306, 97285.69768745021, 20647.05818935554, 74496.72782772036, 289.7291546005487, 46760.79028591037, 22336.603147833546, 40104.05414350396, 7177.234983977465, 87191.94098636479, 99986.74148262323, 90043.47550062307, 57152.967144122835, 13689.02929219411, 60207.29591385367, 7029.467656156918, 5710.051827480267, 67426.39431158378, 90415.15570932781, 71126.0938481886, 4623.896700819452, 54244.99316025626, 49631.6488174168, 11488.677643950185, 44818.6441107992, 64750.31993519589, 86390.19828554105, 72365.52264009605, 21154.307594376132, 78709.53031633241, 85430.77740766219, 32613.30215839623, 36251.49800242341, 28894.47473385124, 49586.92279097073, 13682.075068860511, 50034.00899964327, 6887.449757416175, 19797.46335360367, 86542.48964526273, 45584.554777205754, 7991.239504046254, 22055.4595587515, 18067.646457784613, 38468.15835764761, 17678.941019588092, 32378.10780708228, 48353.02192524835, 30572.7846183983, 20150.99997945181, 79331.55346152348, 76599.6185185287, 98111.41172982506, 5034.467377776908, 11761.8893703468, 36074.41673710946, 69171.43867743194, 88767.33383391681, 35143.98553018855, 52630.29331932028, 21656.528959678333, 68106.38410331351, 23024.628710855377, 71394.16979820166, 45803.246890561124, 88662.86551054208, 65206.23293288612, 75625.02096962636, 19038.935126804667, 50795.207786946376, 84777.43073259611, 24148.936452403013, 34171.044930944074, 83267.0310404786, 61449.65354292123, 29407.51186437931, 89861.2449918711, 36685.90421531819, 5887.439274548856, 50718.78747150309, 15790.497008176519, 11307.06393660178, 76457.39304245052, 86670.59004524327, 41523.548164811975, 79973.69821604229, 74126.40288599652, 26573.98901845399, 87815.0404170677, 68566.21346231438, 52670.240352513705, 17343.61550872636, 31426.848741205315, 31814.199957206125, 30171.02008211, 39085.84095671462, 63688.5462719081, 65622.35748620458, 8753.719933479542, 32824.582879784626, 98379.82249475451, 5803.85151990932, 20590.038500765186, 13153.537504295087, 45241.56868761533, 24990.22543962228, 44061.16014636138, 32408.459568179638, 7917.8267067860015, 15977.340908479498, 21640.15676289238, 70678.42289775841, 10860.057833543346, 22807.031624666528, 56739.73315337516, 35433.95651375516, 81320.62558828214, 54512.98371236406, 45736.76311764432, 97545.98358104657, 18323.78022468738]} +{"id": 1409, "vector": [30118.385328433196, 12129.600256362772, 46435.94752477473, 59834.44973084368, 80726.58126181609, 88961.93910690538, 92863.9901039253, 57912.000501727925, 75627.50834623439, 86612.483293579, 83315.96446512506, 58638.37877079867, 98792.2364577043, 66129.83017363127, 98291.50974632736, 11292.642096691718, 69381.08803705816, 93202.50568321803, 695.5449972506966, 51631.823048722705, 55713.04719585766, 37399.11843317808, 31566.005420156995, 68486.55963133501, 47252.05399045447, 46909.52915270215, 87376.22673183282, 11715.039643592461, 36555.75329367948, 85791.51712675366, 1580.2244226870932, 37378.974898917, 56475.92062554794, 33846.98726997714, 34797.913843392715, 66618.70954773697, 42636.5438215661, 59376.35021035277, 40827.25674139532, 29752.053350879916, 76867.97804325717, 742.0322634521415, 2250.2839188169087, 62231.957471946494, 65047.10565656294, 16386.83228732153, 27477.28551650368, 71997.95315744933, 57356.53463323764, 75635.41348014426, 48002.2674982715, 45101.36713141887, 23204.064360094588, 90926.59633184524, 91605.54860270431, 94508.9052346842, 60170.427442885586, 82404.75291221167, 63731.53970043471, 54558.66166566102, 86624.438439128, 18252.576378279984, 71623.0790290819, 95295.2087955159, 75052.96015769502, 8259.164366260218, 12499.87248190375, 57343.60305318066, 24550.758255046636, 65830.18892081188, 82343.59098358889, 49591.482542197016, 53770.45656148286, 40799.31430200217, 7807.681045271453, 33317.24906153163, 35331.778446386044, 44841.07183137761, 96228.90587111826, 18686.182672970386, 77132.51063528266, 48782.068995947644, 78551.19187978414, 21783.965006421102, 20188.26799674971, 18527.38592312093, 92239.5552565876, 64479.193273670244, 66085.3160933704, 84363.49936729315, 2362.4682300601776, 29213.89236390285, 46909.194537249765, 72402.28711206588, 32268.138991386695, 38955.717785523215, 82736.39379516133, 49079.908794878305, 12461.611492822578, 17055.97737982345, 56541.442932032995, 70673.94299703339, 72880.35513042597, 58147.91034993191, 92936.9726759487, 9429.339293980287, 41267.46815887561, 14904.780159432317, 23109.875668285495, 12802.184092454905, 4046.233209908723, 6677.185450883027, 42952.68263191507, 17472.47278567433, 31020.66039726611, 20121.590347605932, 13714.601304999185, 75471.25600223504, 50113.58991350101, 31318.22657749206, 42180.673143241074, 92758.24994023485, 4768.687626328916, 77062.02432025701, 14296.199751053186, 19375.8856132073, 12130.967138029559, 10094.116279013122]} +{"id": 1425, "vector": [65208.39118963998, 82467.22153071925, 61673.98928683271, 16872.302213849154, 99873.65078908301, 83956.42492947396, 65493.74437090982, 3445.4007836089427, 36569.55365307118, 20570.95444502033, 48816.53240294472, 51658.193897716366, 32192.70955344634, 92692.60918024358, 49562.45949229888, 25306.62859817825, 26266.626334103814, 75512.96420033861, 98094.75058630899, 91079.66029057684, 2840.9615160546855, 49242.520538825505, 96987.51993299463, 87320.5140783439, 36149.25230763051, 51702.02204589645, 54141.29656656919, 93466.37508117569, 44131.97979781867, 9236.632877819973, 90025.98824160328, 36406.181901049116, 92718.0945136898, 84252.71200238347, 10408.95558928402, 14378.768960777965, 88181.38439803345, 66744.32548980643, 54086.780228509655, 25849.57090976825, 82782.17978562016, 64016.36190131045, 95257.36614396566, 51183.35834876014, 70631.81711732053, 16234.54942569218, 94183.58623409162, 61091.028383742596, 13344.70108796586, 54056.1026790399, 43634.24940891467, 40504.61056655787, 31444.589924468513, 3729.9344494379993, 54300.01796526948, 8730.02939215387, 72924.62893489834, 12611.366887537411, 86160.82167606492, 11053.496566660437, 7254.612937421656, 73585.89946092216, 66872.66025403311, 50132.3737471428, 83412.01143640127, 77819.56869627166, 28600.169484775695, 51967.064564310116, 16605.33406360194, 86015.29862582734, 78543.71299332839, 18139.692967879317, 24672.468458932173, 65677.50643279904, 53369.85035532652, 22205.174735800105, 46046.90296450722, 43014.90047340376, 92755.92375365384, 32362.387573362383, 3025.5353733096845, 53248.397852258924, 74893.57480853179, 38200.53469741801, 76603.3205858985, 77447.56909026112, 57056.92069280737, 15454.508609474171, 26693.759803209105, 59273.604670347326, 77030.31637526533, 50928.88313929686, 911.0820013583432, 94405.83156679381, 11430.531237025054, 44391.028151557424, 67832.510551087, 65065.56516105066, 10493.693722976572, 53552.14686580323, 55510.690054421444, 48226.111208618204, 3639.212933853153, 11032.505504432633, 97277.41166941205, 76742.44198065836, 29168.947045936333, 98312.36926208416, 24537.96948557677, 55532.67262974223, 24639.347941599088, 87304.1442013918, 77358.9226614849, 52130.755721602916, 21500.892192025654, 6389.790290576325, 75228.69405051423, 85595.88941649838, 35865.202390757324, 36705.33423050874, 70531.89561716949, 54378.41771673504, 71080.75562538992, 46747.41111222777, 93355.52122948423, 61463.53257378011, 50373.90414578452, 48778.99322785416]} +{"id": 684, "vector": [72825.98899921135, 91276.1137058355, 75460.83228757838, 38845.25757044237, 67068.10807069144, 3805.685998507824, 5909.174622773794, 6326.883214236878, 9029.064364786089, 16236.261119397444, 33937.283016539164, 93843.84905996415, 44460.36835214853, 77510.30162075, 61408.58173212349, 75795.84026881318, 72925.29331017433, 3597.3085589844354, 96757.56572403395, 65445.30471534254, 41728.05306888735, 11495.706190767429, 98184.60195507375, 64575.596636387476, 88265.33819123101, 92151.9045833731, 33381.14113638378, 35923.09519671563, 91888.09764070803, 34292.58655578424, 79418.73153756555, 26332.964010333348, 83328.84184190258, 34657.96807295612, 55199.48682746598, 7812.128845347676, 68788.10514475702, 76143.96900377153, 65466.981159411254, 20393.12365709043, 50934.0562780315, 11480.032317637868, 84460.13963435809, 7031.629143999585, 40765.780349328714, 93678.70738469066, 68004.53542213445, 62869.77307304381, 43086.095317255094, 34133.44258754787, 83551.4963936618, 40063.05314748937, 65834.78106165306, 86476.21578039376, 45216.535493366, 57529.95892960379, 9116.985515737064, 5617.827037784551, 50278.987347734335, 23178.029377921437, 39271.387672618854, 18329.458533553967, 14958.62083875611, 11652.280248144563, 32162.896614419788, 70542.82531670481, 77587.28853272622, 40032.870325749005, 27897.28584111073, 94519.08478303133, 20750.743030623176, 91217.77055740208, 98727.36937768334, 31601.03788499625, 18882.32237888051, 95066.63731324257, 59746.44056340558, 89224.03983700021, 41964.785891875246, 3831.4895075703625, 56335.60205784754, 38230.410129932876, 8488.421251189926, 99413.78843047378, 10989.01602962048, 29169.07565014074, 38041.86940468734, 26692.561301188223, 42410.3855080918, 67827.41151858187, 98762.18613349425, 23942.517785077587, 58945.15686221956, 70592.04788586142, 99277.98553747745, 11601.09060951705, 85919.3597864148, 74430.32139819054, 38405.97346671034, 21750.59049662029, 77733.00767131684, 23864.163503569802, 79354.84507879055, 33318.11570006335, 39186.464284362155, 85554.69226452583, 15970.365291038292, 3502.100799964758, 2789.163229115166, 90075.11239933711, 62832.70585448292, 61392.01396643643, 5763.100369594831, 81349.45462025698, 27973.073959909223, 39590.165611636665, 82581.6543637772, 81828.20854832753, 60735.672249106254, 6271.446160377958, 20126.029306846183, 97831.4274786261, 62001.826876543186, 25418.76613809193, 30298.290147080377, 97773.93517620007, 68030.91629676046, 92148.87054990031]} +{"id": 666, "vector": [82374.82602833115, 5834.999034193389, 7800.208658722662, 26465.042128401663, 80022.86419609735, 55819.51339690271, 54575.03818461073, 28263.92290713027, 93689.55297632542, 40863.79684358995, 17570.77952298649, 26164.88879347679, 42240.31312861055, 6181.049439828667, 40337.73007467363, 48807.83825337934, 49744.82615969241, 38850.93702233544, 47223.03170266286, 8275.001156952989, 43483.153435058455, 16771.2793282222, 42385.222985152606, 24328.90320665161, 83046.70141723107, 18355.44452864538, 72290.20686765476, 52272.37537301645, 18665.925542212826, 10910.344925330817, 64545.38412347039, 68932.30281737653, 36599.14807343939, 44356.7426034579, 54811.56319054187, 92007.0745244108, 11652.747839088739, 27636.598582842296, 32875.14226317929, 29788.274009343408, 9748.211671716666, 34046.39993012905, 76209.4540456494, 16752.330971290143, 67822.24345032401, 99370.95383399555, 28940.138685619666, 67457.02191505414, 45491.71186827955, 93188.2652337874, 630.6009189090412, 9338.164184346664, 86124.48869520024, 57809.59421654954, 84387.34969664674, 47986.8165975403, 96642.49856115312, 9945.93781909947, 80439.51360209119, 47235.17413219285, 75060.27585194822, 84621.07927440539, 88339.44695201305, 38099.961181287435, 59987.152539889554, 91258.36726474315, 6834.214254259752, 87681.99448256702, 6665.181249584984, 91570.27344342753, 39820.362132106304, 77611.98807022808, 37032.29398223965, 587.0181011066511, 49070.44726239647, 20512.36756346393, 91785.76193318954, 39113.157651705755, 1538.2680498978395, 62726.91274889197, 6754.023032923406, 68978.57620864674, 37573.40581492489, 2124.9043328812054, 18450.32734896479, 23783.709726578938, 6653.125238223867, 76667.20944643804, 17071.859119818233, 50570.259162462804, 60096.35369324158, 64392.60911025089, 11739.378126029198, 91587.24405653507, 3706.868765913185, 7122.365285096921, 32128.758208337305, 52265.38395691094, 51446.76068792696, 92254.22824322953, 49371.27832180498, 23962.153548495313, 53279.6646600388, 99139.79921173926, 2211.7892629527237, 25637.385052466678, 7718.063101573025, 76878.82090212336, 94537.78189719278, 20934.547258900893, 66937.31715414808, 18644.556118198972, 41720.507113872365, 34621.17474526869, 42840.508791114895, 23242.910643636304, 78581.10264249609, 79266.81781728931, 93770.61228527213, 82207.87827457441, 13644.114761952986, 65975.73206455517, 88714.47307385864, 12333.107123800513, 69161.61099082587, 40649.841810688005, 46343.807070059316, 77992.74100960008]} +{"id": 709, "vector": [67504.56625448784, 70621.30567163009, 2.562770661795888, 81894.20354567765, 53207.20150537668, 19183.918939719748, 97382.19532042857, 56407.33236033881, 29358.00694243017, 17074.362226422636, 57993.95777075045, 35826.100748193734, 63674.16721544493, 77940.9228190986, 11204.798011779194, 57273.15024089955, 84610.30377690251, 80042.26606625775, 77650.37941096681, 16754.763053209677, 45969.5500813477, 85153.72652132738, 46204.79114543983, 62866.496885317145, 43006.088955631974, 53568.798163958374, 28708.991861036404, 69959.93138042802, 41929.52841633083, 53133.822026308866, 16870.04486048582, 72293.61447744635, 20372.615548645357, 29954.375971553716, 96884.57638448995, 52167.402618457505, 75725.3935376217, 70377.21451650836, 38080.832136521436, 26513.09675705068, 36093.97633183199, 10014.210747258323, 94048.59477245959, 22655.89978740231, 16779.46548381333, 99567.30833231425, 24738.5441522501, 79500.33821202163, 38976.35172474727, 79834.12823454647, 81251.22483583436, 2456.9790704996763, 14179.59713461211, 71650.25464932235, 17449.99948038747, 9130.971458917958, 85538.73270914507, 11075.574085979246, 56895.51426285947, 76524.67435488418, 90178.18682192582, 15694.732973504222, 27221.962383920007, 25148.406124896883, 57844.99390463336, 22755.92477907352, 42871.718058409446, 7865.0245222212425, 97324.72734271201, 44130.21504107604, 41119.37841972087, 34581.37491642582, 91690.65229331469, 31472.88605666212, 41612.20485723306, 35498.89251894094, 16779.92237252297, 14158.122319319533, 11390.630489058185, 69673.44087442367, 94857.07194858171, 43820.38875824078, 80052.14106596325, 95929.99452245409, 32825.38773813294, 65908.55161287126, 89892.67859359464, 21069.46946334448, 14047.482758930984, 35397.650249598344, 29900.517400458048, 5849.283745381984, 42091.933928537925, 7515.652726740207, 2735.7088974855446, 43424.14439809088, 94092.66903535342, 22036.796524390556, 73161.13170370473, 9717.521458746025, 81375.84226966565, 24250.953062809454, 64271.61702787262, 73102.53059548655, 66410.24106605907, 40286.759262496285, 5026.423555766757, 18810.222069554882, 12687.529654827435, 4166.525223173667, 9520.432268578816, 24903.878556006097, 64765.083008898924, 19410.564941510856, 84193.15589489549, 22265.613840909802, 78256.17967700296, 2317.477809899271, 28001.051282514934, 95020.18982058846, 93934.79133948544, 50079.90595216183, 58033.033646243435, 5560.311455106282, 9445.742398447854, 71325.10690120842, 9372.569709634381, 54482.96158004362]} +{"id": 1700, "vector": [90896.7913191616, 36010.58989385544, 99340.76479767842, 66187.63888299365, 56502.000202072275, 44787.96956326867, 3897.737590321837, 28146.651366527953, 12049.973789936319, 38057.90472837826, 32258.59592364979, 4384.288870621389, 88677.63734413033, 39841.80321957934, 3717.407028456632, 41677.64577536234, 46236.64826391459, 33409.37379660743, 34155.43571120109, 33062.321862336554, 70586.60253244045, 19201.088941262322, 56836.599829019266, 65800.21705828636, 41515.672712320804, 53673.49163964426, 58597.98355433082, 52728.127757265254, 30131.80122899568, 13845.551949668721, 3608.2910579953164, 80503.46582395196, 6535.301360617263, 1261.3507876537144, 69807.34767936548, 68840.64994270694, 20743.99924637701, 93203.79807859994, 67281.74039657309, 90990.2860131706, 92844.78192039249, 22784.23958944922, 24351.631640349202, 54594.43674377039, 64438.28731179125, 85277.6294357453, 26947.07728501988, 61858.81457684588, 52791.12634885053, 422.5917816156333, 13029.731701259561, 83105.27354371581, 99106.00865881609, 33237.38883509376, 1280.2751228156328, 11950.35374282074, 38926.25536332769, 81794.69895159069, 29393.572091938313, 7325.341761891768, 27357.261731447623, 44625.04194094651, 36603.36658747963, 13423.417891575307, 49134.56100920796, 15995.03451170905, 41401.92833162958, 73872.30437734524, 96879.836137524, 73721.50702128622, 41741.697023463894, 24545.019310527394, 14999.537120626826, 6764.8558281133255, 46426.543675548004, 4018.904807124102, 79411.97814022042, 3839.163133501766, 6154.840474048673, 61255.66302658789, 54044.499316213434, 84146.82237832528, 32350.192652840204, 35957.65943933652, 82584.10512619848, 47851.19765548358, 84063.1977129169, 70119.3870078738, 44042.3778497857, 86976.74737116671, 10973.501311693744, 89980.53046015266, 78030.52739930929, 39938.34619211027, 45567.4962956958, 73253.75093700856, 30589.5474561091, 18964.160464839853, 8088.799990758044, 81654.36950000172, 91781.3225314049, 75948.88973790583, 13434.483278399312, 48937.41360916615, 5254.027229953395, 74079.48317160405, 34662.53916618247, 99684.11294613821, 58046.34034264946, 11147.068272284178, 61989.23997531759, 29759.61999600042, 56896.885494728485, 25856.941047185945, 62949.82184412283, 81283.92547182395, 50222.069050363585, 77190.08565440538, 52521.0554510812, 57836.011007419365, 89696.6415148173, 8668.527017851602, 41429.11197313324, 66536.90649892125, 10585.316288088963, 12316.095343126231, 59221.64958548136, 9511.485411977616]} +{"id": 1955, "vector": [40037.47030750566, 48366.83454358938, 51507.824879739426, 93202.46173055177, 60293.83416395507, 22496.80398163687, 94452.88436241311, 61337.38137329193, 96794.00415735703, 52769.098100726944, 19347.81749371517, 34892.754764882215, 28277.067270992982, 21226.177654698353, 85294.94175371982, 19561.646413722854, 60024.79391939079, 86062.2144338985, 49947.61598438257, 92599.13725942056, 77595.89232947853, 15655.42786452323, 56841.19810584614, 66885.50473828186, 66462.43734503053, 59343.51314457583, 84706.68126227733, 79200.9468975346, 96171.72911049817, 46122.30854354358, 93008.59332937529, 26049.778059489592, 12985.577500188372, 6312.119796145998, 57172.71067632849, 57319.92473105572, 13991.354828189329, 42335.83183349484, 50649.77578008587, 7627.749182864074, 74252.98321353682, 17934.751173640794, 6156.8439992438, 67116.52437388396, 17283.260785359733, 42431.38130532333, 59080.230681218105, 49536.13751257154, 41464.39201008976, 66114.07428049218, 6725.372245178818, 4962.764868810299, 52296.00732256219, 72131.27148055787, 52493.26278412567, 42632.27373293937, 66224.78243455976, 62894.15112039922, 2059.938101877179, 21553.24821259923, 71240.0289353958, 87189.65953073857, 96047.99181190418, 38945.999595983914, 15320.249023223354, 45783.365821173495, 45608.10491741536, 55504.14735089151, 614.2694655957515, 21382.048287719557, 72876.12346094228, 61047.31981601038, 44651.682634237965, 22984.847896271254, 59883.149500999796, 54855.82751280852, 73339.89079888024, 60832.44812943512, 81381.3096864049, 16052.334850592088, 17227.96282872481, 43690.23977980938, 24658.8403473309, 86177.02387766965, 35821.369071837114, 9606.606135679664, 77600.90198286649, 57465.44211396243, 66687.09138659317, 25799.589023738168, 79752.03673429054, 65589.21610622753, 21085.101568247377, 34760.31003742584, 54718.086683772046, 48691.45401581169, 9963.07681778792, 73622.71157721063, 76484.80736093089, 75268.10919248332, 45209.480201477425, 20751.05246073825, 60240.42986733342, 32800.880761804096, 28262.23444644559, 8667.32915299383, 52840.477905650485, 20165.6870237078, 91932.17428925082, 85514.23145850851, 64136.902321182446, 89060.68558793976, 9331.316212919472, 19166.55807065296, 83764.74423990448, 9566.607475490662, 64672.146073880045, 20535.53812899175, 25362.694132243003, 84124.61948652436, 43507.949440384364, 95909.83803077831, 45013.89938017484, 86216.20700242644, 35797.61186037702, 53638.335395232694, 92745.99967218313, 24391.895570513534]} +{"id": 1368, "vector": [5190.940852115411, 97135.3527608582, 69270.79897448853, 18907.250807597386, 22137.835844784193, 48960.98902871554, 29770.060446448264, 36702.66690978729, 17990.222789338706, 84034.9804733935, 44661.57931897744, 62286.640485522126, 91778.88016693166, 76536.98615459325, 69020.05494900742, 19615.87511533719, 1744.7412473081126, 37452.67888159172, 73279.55659220932, 42980.48962937373, 58328.57225063816, 33418.2215599208, 14453.512177706862, 68098.87544295243, 85958.0622884685, 97144.11206406841, 42040.64523187284, 86677.18022294351, 76295.35375047683, 25866.816633722567, 11487.925475973216, 82112.58923350617, 67677.28344573533, 20963.660722589928, 41584.88873140702, 15960.87160712696, 39720.25589620496, 34669.32970727912, 15138.441805555269, 50242.62500803386, 40811.82119787289, 3264.8760323209513, 27913.188973244585, 59266.01389601366, 16255.437348978186, 45659.68503301363, 49037.55843465008, 70400.99336859003, 96323.77940952237, 30384.077745473005, 17208.64393940539, 15346.17300849851, 24379.092054359066, 74493.02286292065, 67215.48822838628, 37768.554975678446, 91236.82902032306, 14409.223248568003, 65728.88349873424, 73927.67927299556, 28414.582945189904, 33287.81722812691, 7458.983729112456, 10085.242991921683, 94261.10342722316, 93813.06355595784, 72104.61397713466, 74912.5606832133, 83741.23998467633, 3353.2488636701996, 96304.7165423088, 91564.31490200959, 25905.893908641585, 9072.145098426165, 11076.245361984184, 55412.90553354689, 28640.120652656497, 4008.274336181983, 69260.20464779763, 40125.06403518159, 66043.98134367516, 89734.17341061929, 79182.73940503436, 18259.886896765485, 43337.575760237276, 30020.598016423985, 84632.4030755068, 97752.17474716448, 9496.88036720886, 13743.610208382639, 3035.0351244054164, 17060.530799982775, 69106.42149338547, 54194.82186275937, 12254.190032877144, 54434.79301438029, 35456.73434938789, 40425.43092670071, 80144.73180925667, 56971.58632169219, 37796.18514701568, 88166.43814052785, 33316.14332786707, 31933.59164539319, 44707.82998730345, 55219.69974818094, 75433.45765792046, 24559.152268701455, 1040.942834660774, 44462.084225344464, 58813.5020479715, 42742.37979204802, 80329.48761461227, 97741.72733872184, 24193.451967622048, 88562.32619434148, 8448.298691621758, 26226.424495774216, 49271.2518961736, 34155.34660718303, 8238.059136564347, 46219.39130674298, 91686.75321060327, 53051.64671328353, 90547.08101538446, 25133.63164443918, 80941.12001190505, 47066.88809114338]} +{"id": 671, "vector": [83517.15668914959, 66172.48853879594, 14663.554368702093, 12150.919532037153, 21367.892159601488, 13774.436850538441, 87202.445616221, 44083.02706553888, 58706.318191265906, 8686.307066755295, 41089.1151244468, 16120.057641562713, 25221.963292824235, 26388.88649122174, 1210.5365884052533, 44748.89932520653, 64920.842381734714, 80668.28323412337, 83585.88189476298, 39726.46253643529, 10360.85681181984, 46226.129175119066, 4974.303268137836, 26910.36248222767, 99969.49841242765, 87571.3383091439, 7965.644374754633, 93774.28127013707, 65322.00125537734, 75197.81110832968, 7905.375387321811, 97460.2479500734, 81478.73681996093, 7289.411595021056, 6637.7185384766, 43465.62382689483, 98779.99282538304, 48339.007940036136, 80962.84094381088, 3943.8288163436973, 79464.52214988632, 40486.68790228708, 43143.6099063306, 60886.582744427906, 82037.66308666203, 83125.75254124953, 20176.518815404444, 16963.155075464965, 13309.48905248034, 98829.49312289043, 2817.1772476445644, 20281.62891864018, 63407.08856821677, 27848.56564435243, 52737.59906103825, 19437.02243241182, 44881.36384322013, 84724.63779782927, 56699.35971926318, 8181.886518218795, 9386.914924503686, 90821.9433189295, 19169.750428790965, 17875.3284144585, 49273.99732048396, 63992.08583429543, 8081.094089712604, 55114.72271768951, 4165.203648809002, 53686.713632361716, 3156.6031965674424, 54292.437941685734, 8726.527666509077, 72535.81869721125, 35783.632245922694, 32964.09443329224, 48484.60405891307, 25600.386005238397, 88260.37516178883, 45767.83763880621, 76022.19554422762, 10795.921193702507, 44836.613919358184, 34258.09155588373, 29274.529215804192, 35945.01652048052, 42154.411981921, 63869.70843410855, 33188.02273762594, 79241.56308960884, 20018.511006823748, 1487.895429431596, 18368.131262951614, 74262.89626750704, 11240.67270356497, 36604.98652571851, 47827.56197490214, 29574.65771181754, 40233.0797353262, 1605.727610124108, 43873.11642224647, 40189.37682281156, 27442.112379603568, 10044.06478528288, 6868.321679871625, 84825.1581787181, 74997.63795185911, 37156.57076236137, 82418.29270903612, 35688.80732327016, 70814.75952934954, 19524.213860154483, 31346.354595137404, 96891.55900328427, 65222.689048225104, 24678.092602086843, 95188.99045420707, 13955.843165666749, 52081.029341318565, 55725.3550419449, 90225.4189494041, 57289.161272849175, 94780.22135947717, 62489.06474097564, 78171.59189177492, 64034.98040627125, 84116.60305705002, 3668.710795163033]} +{"id": 1965, "vector": [26944.898591860077, 51757.3609425491, 39514.365081042, 82331.60671767968, 91978.63346142793, 44176.38045677026, 51347.6146769343, 27277.392088966844, 47234.53164685351, 42421.84620797782, 2300.4130032253433, 73942.4012293125, 61859.02156848082, 9000.533163029322, 26525.3159743876, 4807.486359796598, 32583.53689956709, 69354.20286763691, 19770.595187675965, 95437.3601681114, 64620.516897490554, 505.35396538491375, 42762.68969978038, 62110.98566627104, 76582.35348272733, 73306.09523554826, 20530.174919488487, 6581.389651074387, 37016.93243325872, 65706.23857676543, 23785.88320496573, 9445.90224281504, 87779.87811243474, 34874.2527035754, 63148.256552667845, 57751.59684782915, 1833.4234295609208, 78865.69358248956, 1021.5786564813034, 85888.43246560983, 26484.42006342805, 80457.78979695817, 57612.72314893946, 69741.0620156726, 88836.30230300428, 35338.50732095165, 80760.8899593557, 48516.20416137614, 42411.87367942824, 68430.12931431782, 71271.75423347994, 26838.446294936635, 30126.208743985826, 64139.61340137777, 80836.02956368972, 63456.02553052004, 58564.26413424652, 24803.8420238321, 18746.330752975526, 92661.27346020288, 22954.134666753147, 80213.34591126246, 52794.2035401365, 28185.706896021322, 71940.3327458685, 84759.11623384058, 23158.623655646337, 72854.75089287257, 45518.87533709454, 89183.66417123738, 3641.615237042317, 78482.70488481282, 92348.20539727175, 17653.24721865804, 72477.14293167533, 20968.603466984092, 94941.01123479413, 83037.76317311502, 69411.36533722267, 92776.05545481697, 35431.261944451755, 2400.2546490833442, 6817.924637454442, 35209.35927605628, 61475.07110302696, 54232.570010340954, 81458.54829806647, 62540.40989962787, 79603.27097070334, 81966.1225880808, 36112.249754629214, 88451.34639399005, 55519.24877100159, 7299.762804111942, 27774.09934634768, 87179.49642947785, 19298.74850389811, 97742.45345243743, 27575.29291228058, 40138.79314463079, 79405.07006737891, 38253.537993756494, 23698.674244818474, 92918.89687518663, 51192.175682731024, 47445.8959477737, 85068.00787898563, 61569.32418691397, 56087.39263889386, 18954.713147658542, 35988.39701348641, 36371.19756841253, 33317.19974570957, 68897.72428100382, 78285.96191991934, 37821.91437539427, 55855.355005275575, 7856.6019842763235, 30226.45460435529, 89206.13533860643, 12144.363230686806, 80451.88119756703, 33022.63923380474, 69892.81520351075, 30193.088744312357, 17358.451912806715, 32467.887893497584, 52987.08385054445]} +{"id": 1339, "vector": [15341.721163330047, 96946.25250214672, 55410.57905592259, 14955.348775142475, 93087.99076114401, 66722.35502414471, 20140.506325703824, 7481.383972651323, 57629.88125145799, 89969.85493851127, 13138.59347661951, 86190.48596623074, 88515.25520891101, 3445.265020816468, 76475.24798919422, 9242.173637580952, 34103.16094960203, 41216.439381818214, 57477.639244478894, 67028.4722490839, 37948.346319119875, 59652.39518554833, 48802.965646774566, 96017.14436591667, 8043.953065140575, 36524.79233082482, 43298.17186645203, 15465.070685303306, 24336.935649106374, 74587.88834442041, 59395.7387497983, 8235.272480167976, 66960.8992202325, 21248.418811078616, 99426.699111559, 57770.34650820949, 96365.75722036287, 27968.260869195405, 97633.84327209856, 13489.226208739457, 45432.769755967805, 25647.240610690304, 43805.37661122319, 11053.434071403733, 65827.1959920311, 65909.36342298976, 65035.12970898601, 6725.899223134291, 19514.2687100321, 82144.73451625083, 59637.05607871045, 57443.43765296347, 34348.90434358955, 40455.50118897906, 40592.21488069208, 50615.26914765322, 9259.26792391173, 67622.19857434859, 81407.628717034, 82801.7222130891, 52017.6293158076, 40732.070733038054, 71736.93684679254, 85504.85222645801, 13253.86657496812, 15477.750165169668, 10394.96655934018, 94600.52718183998, 57992.95518541122, 10953.168496258158, 98126.48034060451, 76177.03249616042, 18636.82628696357, 1421.500563338096, 23058.07039975184, 74721.9871048612, 97014.57666958461, 92731.76362882901, 90558.59414669208, 8244.327415486852, 48733.27398520824, 36060.730582692566, 36334.879232606254, 37812.50312888682, 3194.8431283358227, 56095.68857708296, 19309.14240490208, 25162.886245942784, 68352.95983164279, 95546.46244197855, 19832.172337145927, 63697.01765655323, 71101.12895546609, 70286.14429074347, 12099.340923798352, 86062.43896249031, 74292.92603837381, 21490.3905384772, 15360.045042202297, 53713.73125205761, 2859.9953509272245, 2260.4518152573137, 38304.395577551964, 6167.615014637029, 28683.796946602157, 24006.862793123062, 80609.20899068858, 48291.42796668635, 695.8330670291391, 99788.03051585409, 81109.14566279108, 66163.0284779674, 92098.25011728809, 85252.49599877329, 63695.53387172667, 72974.87783744275, 7705.159975682674, 72116.96154808291, 68512.05258026085, 50863.667456063944, 91057.80731558584, 54455.09900378136, 91412.12173192111, 15501.926171988056, 84331.68272227356, 14954.397390092534, 23043.86157692375, 40780.031578695496]} +{"id": 763, "vector": [28213.866565927125, 39553.37703650037, 65916.69567688019, 71987.50977394755, 44971.18473466956, 15386.140412525361, 95424.36391733859, 23331.829996821307, 90565.78144140771, 59570.5551129084, 97980.24881066721, 53374.04907545879, 70585.54018075754, 60436.25699865419, 91242.04007755654, 67730.88775069252, 71242.13681582708, 8806.918831228682, 77263.9191314017, 81518.76998898077, 68538.0611470265, 21662.092036706083, 8476.571057599402, 8690.316994010705, 92032.51162874165, 57387.67942732522, 99454.98237428155, 98346.0172038538, 9366.328383771417, 25978.77160095633, 98606.86939113382, 13204.175950186136, 80965.30900840428, 20511.971698089583, 66818.9876517593, 39646.170356395196, 61538.00711711437, 85161.89022004356, 40239.60274314779, 48448.33180345982, 44378.67151183003, 5593.285469442922, 34085.589033270335, 47686.07218416987, 96484.62433153491, 1531.085935951604, 77921.53916199191, 81109.00891102215, 42178.23669117725, 20475.762538473442, 63696.99589312247, 53158.49434049361, 80621.51568280568, 21492.580448347577, 97231.01939475881, 60386.88818456836, 30468.978891599352, 79684.40946272714, 70252.96324214431, 83712.52924631322, 97611.39270339631, 51137.912990110366, 34971.380931103944, 67999.56363355555, 22366.84420791716, 88286.52011469645, 20351.274572886847, 46161.0478729686, 82813.75556301961, 98601.8330707154, 33663.452026732586, 7823.951895582837, 79422.69180077432, 58335.97809012416, 33542.731313444616, 8528.736360243494, 97894.9990076894, 54642.83396259118, 13662.677227918019, 91726.6370330407, 55469.67687960943, 58915.642306624686, 34708.407973177, 20156.258078776613, 38656.99122386401, 66508.36199621153, 4612.958255220135, 48818.70840371376, 73889.29996178046, 84775.50514467139, 4629.390714068948, 59971.083878147234, 4234.228781951743, 65795.73815697696, 54006.25916078383, 51560.194039643204, 10515.932879105172, 92063.65735895655, 76779.54532269725, 10023.40675159309, 41649.79530435294, 67813.26050761963, 60432.322381438906, 82324.87063803978, 12614.947113585551, 68670.22146826332, 22060.946481546907, 83405.74467198517, 37169.807988002656, 55774.916350537984, 44437.002613028664, 51121.086191850416, 33001.806786085785, 96876.17162233368, 5185.891558081579, 85267.87504899221, 47684.538256820364, 63294.38740635815, 56134.49779323828, 61787.351337961685, 41595.686764545135, 74695.09483578055, 74838.38497671415, 78951.0179590195, 21478.95796660727, 11085.238416138554, 59499.76323533428, 31286.141996227103]} +{"id": 589, "vector": [31890.363766394094, 90099.43962127522, 38197.57363165367, 26072.38701717496, 47106.75124004456, 6418.934671346244, 85521.40199249514, 32313.478390625172, 8692.599744632656, 93508.04094892908, 69444.72922978293, 42215.2854727203, 61602.54268968853, 60198.38680630034, 11490.173105265456, 29374.78339438718, 64928.272083960284, 69938.5193584434, 40099.22706639743, 59660.28805021617, 82523.61361524767, 10537.340558538222, 84798.25617905713, 82746.276618839, 81169.38241807825, 99986.2351441868, 54263.82574461671, 82286.86551139898, 20694.318364822695, 24953.65664433884, 45415.74860155943, 93714.72521441402, 77679.2985048523, 21578.016844722693, 46051.17090120112, 5245.723823453863, 22113.854069406512, 25038.046418152982, 15462.510804641726, 79375.75398621733, 5797.209934412328, 8132.584057697889, 7848.840596754192, 40346.44907659984, 47203.030983579316, 45248.202350569445, 69707.2034856079, 55817.18827828523, 80789.50540987613, 83610.11354795331, 54043.967070333885, 22903.389633916173, 75014.26746583293, 34203.56651203711, 63671.63528348715, 59272.477731490195, 16983.423689577405, 60871.780991871674, 48268.825161668115, 33498.39473917743, 68673.26643022912, 57066.36856022123, 2897.721329868963, 63081.6073350546, 9355.96887885677, 84937.39752931573, 49957.978330095, 72721.362976308, 9549.90830863105, 37980.18981245749, 30705.75675011803, 13940.753403191908, 55931.88487432191, 22148.37739091814, 41305.795331268055, 57927.59423475205, 75013.40874872723, 46503.224855886794, 33834.1031238557, 27119.055936050096, 15985.025496148053, 91419.69760149415, 34577.95213418833, 21801.314736103563, 76001.59435345187, 98180.89251922506, 56516.45986964308, 29969.18539577712, 33459.59177346584, 52667.256791701366, 95823.75050061791, 62447.93344018426, 12.048651312090985, 99956.66520315764, 13266.577972814686, 69747.08852497395, 24320.662970311525, 85863.36697296426, 61131.544996026685, 95280.01002210073, 20363.75354961303, 82839.95863458028, 75229.14384907656, 71959.70788350509, 26219.481884816487, 34211.79490091669, 26678.651495474205, 85479.46802941248, 31439.536274152146, 45957.29664262945, 48578.868402390086, 48343.1862358549, 73602.16325970854, 43136.49423390451, 45638.642342763706, 43172.356605812834, 45696.1156612383, 46063.92349364108, 87852.77970257815, 3149.975785801895, 88421.22857716108, 70513.26154857458, 2945.5015173790234, 84939.15311771035, 75069.08140322455, 46010.95098810984, 77202.45894783227, 71728.54694386314]} +{"id": 908, "vector": [33851.27119032494, 6920.529721128388, 99715.60592497497, 27192.367599668043, 96570.82528323082, 23657.135850022936, 31059.250737885457, 56698.3197362312, 25167.318966799347, 5743.7520905045285, 21098.20164616775, 69587.13780166174, 30697.927682994497, 32379.050351234993, 62207.82658295656, 38929.63783530914, 72837.88294058446, 97570.61798041525, 66245.15982267083, 33528.01409571959, 58825.8353077488, 26541.128123716484, 27457.792786617953, 37935.34109767429, 79883.35252795693, 41188.14122623827, 846.4374180241485, 70737.98116744222, 88838.09920402602, 91012.1996239341, 48607.258398369115, 5204.875327032232, 74859.73629184229, 75133.10154123322, 81668.157858702, 4965.609516976788, 46110.499319835064, 527.1175643319359, 58689.77507275342, 82410.98270912103, 85941.9565371559, 91009.91655764027, 91195.2489879394, 18651.454676127865, 52993.57328977969, 50495.43062636514, 97147.56634329555, 70946.44651544471, 73376.12184265246, 28487.59029562703, 4674.04375791809, 95975.74870696252, 76122.2304758665, 80416.17183956217, 85815.31383919992, 5683.011026442364, 49688.3996336123, 30340.091939299462, 87333.65364004031, 59096.503081411975, 15521.720392288551, 68621.94676289923, 27464.67836692903, 46780.906186139626, 86577.58059221478, 51927.59051740926, 2465.9511595138906, 90884.03647130945, 82115.85969142748, 65316.71459619797, 31315.632326707844, 58552.89588765809, 52851.077523139844, 18512.49683036582, 71358.03469145813, 90882.28387023481, 78087.79624749864, 60932.58003148743, 97151.582695696, 32294.58663711374, 26025.070752218617, 36085.00363989814, 31224.473057911127, 93811.17891355026, 36702.45989041018, 78133.10845949709, 68941.44091845793, 72496.68034979822, 4684.720777688378, 37413.29842536871, 96995.96022664687, 16925.98059959678, 95865.96352194459, 24430.738492088945, 12066.539752779881, 23584.72795289388, 7437.440261151285, 22477.6843221151, 44010.79366334154, 9445.523056902815, 95013.08784496888, 77408.05566840702, 48945.62611743355, 83722.0611166284, 54728.70964346675, 99978.64668651379, 71425.91918959354, 26909.62946572569, 93057.792105522, 36273.7458658565, 73422.37959172782, 43301.45052461011, 67040.06928711453, 94043.77752961229, 80848.05811156028, 14270.554248811284, 89238.94856973276, 22991.446405839823, 26918.713815714236, 58055.16066559071, 35560.91181384715, 56191.29188253314, 32816.805811041195, 95259.17635205375, 365.71738698800704, 48555.486017964846, 76281.71672477241, 29549.51844480237]} +{"id": 878, "vector": [54811.045903131126, 27434.228254819103, 99021.31685675826, 99683.8467289416, 62311.89644723135, 58686.5894410857, 84126.56942555467, 91498.18386414321, 98287.06963700576, 11567.9348860805, 65953.25522313222, 43054.69609827962, 25733.270509578888, 38398.74948268796, 92024.61842642915, 68273.56150656236, 85610.07060267926, 6141.683789460651, 86091.01714090172, 84138.27457734384, 1620.6477068433035, 63249.87514023226, 80231.4594361204, 18357.13054729431, 4134.083308741931, 8338.046148477773, 76711.5003639576, 14007.550825734672, 46781.240586180764, 46068.096267685185, 57303.79013361926, 81605.72346365692, 57297.18682194359, 78327.74150930227, 26198.916019747863, 90626.96473442344, 88696.95657349986, 81932.35856685771, 12811.636249175917, 55699.827298013515, 88644.94319654028, 57756.187433333915, 80462.32524092574, 84597.05509376117, 37006.11788899397, 3352.30138993603, 22753.814855658282, 67983.55711676452, 54276.45017327499, 39995.290797677815, 75286.28329962428, 66883.14229573015, 41095.65724860458, 77377.78692271697, 91588.22339499812, 51194.58351115945, 3188.2343124015056, 42551.410897322225, 58631.702220932624, 38956.30073318805, 19677.004913806704, 28122.958155007203, 97721.99688583935, 47606.19682646606, 35599.00976379532, 92794.08669095149, 90199.8286829288, 57013.15124660692, 17096.615536698755, 22831.21010439747, 12472.673388458954, 80654.234966816, 54608.02663518527, 89880.04115586316, 66801.60587536848, 1059.9139955482783, 79744.89329318973, 89905.47067037589, 74345.28628014188, 30219.797250222047, 40844.53278477844, 9284.290555270924, 65205.90769894947, 10254.56329596931, 92813.79711891765, 78096.7774318661, 14757.794733119455, 14334.335456826031, 76791.44715289006, 36798.64052159867, 87902.69613138687, 49565.621138648385, 27729.287258994416, 18759.6985075153, 60976.61858636728, 91687.32242029489, 50385.88762714661, 15722.282017056023, 62968.635628075164, 33498.520435206156, 813.7973580183866, 32060.539276247237, 89913.77968521642, 60850.17136962573, 28486.058004768598, 55387.80587236671, 83543.63815453532, 84132.44681623564, 45193.8614725043, 66489.91272430761, 57617.75326753746, 93710.45252100128, 52620.966836889835, 18930.87693663146, 93032.24857111415, 62825.5013182506, 14212.421076000302, 98886.01539122837, 44508.538904371766, 58783.51809791491, 55642.93748573734, 31374.880899800915, 48540.08103292997, 3459.547598316504, 52796.94241575137, 18694.40348694894, 98848.16786618413, 12910.667626482087]} +{"id": 1228, "vector": [38567.50882731842, 98886.59566809874, 6157.537597433105, 41004.72212568724, 83472.48830431885, 89112.79070891392, 82886.5741564465, 54191.78670613749, 84019.66647133172, 53036.96325780284, 79096.38479278015, 36034.178672653215, 13031.535417203955, 42169.15412637311, 3463.972229461032, 35695.56368690237, 66016.26606495588, 91465.72618467492, 90472.65989293772, 36626.78334114981, 81204.73530513716, 65558.66692505701, 33806.253796422425, 69060.98633022617, 14080.651344238737, 21015.638138715076, 96413.39836794107, 90216.88089769153, 68584.24850172621, 27192.01612793274, 7384.040157381688, 81237.47233795575, 13848.42401710743, 72149.42053827706, 11357.902409266208, 85223.84154265704, 16468.914694912117, 66150.4040424482, 91718.70936109642, 8782.221323901873, 18145.368856772824, 55889.35202203586, 35298.00848333299, 67309.79624383987, 15994.73228633711, 43883.88411914527, 96714.12059317032, 44526.52102405652, 85806.8243000228, 27445.203506170103, 30715.739518496175, 9210.317841068238, 75956.38946453799, 90160.61603341758, 97068.877889541, 60804.52236358007, 87132.86495390986, 66011.30943343716, 65522.102260681844, 29057.97450630049, 33846.43309601599, 41493.865113535314, 8210.090321359497, 12216.168773243418, 81577.56756584349, 94558.75381959103, 13338.246430849766, 13928.998217723343, 20010.06588087516, 85108.36785261652, 46814.06461071746, 71831.02320104727, 28013.337763508982, 97745.3272968838, 94449.6657798815, 41858.73696753579, 66364.27158665135, 50616.33079056855, 21395.12161765046, 63368.986975546824, 16260.527389715118, 23994.256030285156, 10283.255593656282, 74696.72694687404, 18812.45861227432, 8542.453265902328, 73960.75703700588, 94575.06092483083, 49235.03602711042, 70996.62908875405, 47357.40211270948, 21460.64117926221, 71366.14108718466, 69640.83866298373, 38056.35885431115, 73195.17022413213, 41459.76613519847, 65165.79084726329, 81046.04436375177, 67804.48368573227, 42121.77364407048, 5136.8609187605, 48320.6404751531, 23571.975114565612, 22283.948569015254, 26615.84952564582, 16087.805157553603, 3758.5359686798993, 8625.205596722519, 6202.105283844195, 81583.8149639291, 85787.53086271077, 95786.82313744705, 29808.805450275388, 46951.40667048656, 34456.34105770269, 17236.715338679754, 39413.378432009195, 29248.55467972344, 29757.722053934798, 52296.01308681209, 61016.75421792672, 72683.25292450447, 76069.03872831096, 44660.297476821994, 8979.463870509451, 37520.797661709534, 27245.441915317937]} +{"id": 1758, "vector": [30602.829922491914, 80425.0785711012, 30219.77148418893, 48893.99474316556, 33077.868692155476, 70151.9385338555, 52879.10425007518, 59099.42482981473, 90019.27299053082, 45491.167217158116, 98444.56629752094, 54573.97946594713, 83639.26200302025, 56883.065396596, 65125.75219205562, 79339.858611938, 17695.679906645524, 49582.63492278141, 51911.561307549295, 25947.25516359103, 46291.167731001326, 43286.89318514488, 31864.86624481507, 48063.522844105864, 50616.39310640016, 27740.813260521347, 78587.88089330077, 81741.49637322614, 74420.96241507518, 78439.26810045203, 81532.10760077043, 46197.80563723306, 8023.959859654639, 6458.870546782969, 15395.91001431131, 22871.331521691198, 18438.22006344974, 87542.1427420896, 25187.813718363785, 39733.39719691674, 37852.09957509196, 85865.96978036578, 89962.37834597651, 47397.08771052626, 22091.751127167958, 65429.67145928366, 44257.68759779346, 15662.618761450487, 3728.565789738991, 56732.93825213331, 14401.02313474928, 46071.445855966864, 45196.489690655195, 62501.8550288145, 43410.602065952284, 44390.66265252342, 3785.541280369875, 70658.32395952527, 26361.46091889885, 70374.09214131055, 14028.439272122805, 35131.78013878674, 75844.65043257408, 94523.09945655447, 55688.49396771625, 46550.62097457677, 98107.93138918902, 795.0557928746659, 95355.71975850439, 66673.18855851241, 95341.61845885798, 42752.22246113356, 90373.99758616358, 93898.48585546942, 22375.61245894405, 92014.72917294677, 20872.7041910169, 6529.835681778584, 44291.24358122675, 52431.842214921, 43880.611351406194, 36158.33856726414, 35861.69406609568, 96788.4681258423, 58615.835298867794, 82305.72113605704, 93939.26689239051, 33835.40184550683, 9189.546697466178, 67864.27414986245, 70593.09937274155, 27607.540907765506, 26156.101331179303, 76810.10504749355, 85439.62493458195, 45284.19862378473, 42204.64925986114, 30142.49843252609, 91821.79813148343, 38012.17804244012, 17625.523288392797, 98359.07849142728, 53257.906428831506, 92309.74322082041, 62541.569605455385, 48267.713666601965, 25606.712858823277, 65898.70806653817, 30008.050967070143, 54337.1780982283, 27751.779647282092, 62574.89556050969, 4942.111190857868, 47461.21094813704, 96727.3330874788, 87998.388246676, 75662.29929823775, 26783.92154716034, 34447.987122821934, 55586.95151737394, 71493.53123946587, 23445.62945771633, 24318.22779680335, 5202.979036748246, 36224.56476609085, 73794.129395116, 92431.6514323749, 14922.922899597257]} +{"id": 1999, "vector": [94060.99321111766, 24965.114063559824, 22748.18469068205, 93779.61339312892, 86316.21316048752, 60542.49037896229, 58520.74935506078, 31882.91687013769, 19656.50818868452, 78385.97756691024, 85717.4037667635, 82607.03899183287, 50820.548977626044, 93927.28674160122, 27075.419591816728, 53699.91883288148, 53046.1740992895, 65191.43628674995, 23199.858174547862, 74005.42172347239, 56701.59584720139, 43301.58392070197, 51200.24123313706, 38804.618908577104, 15907.962248480779, 44728.86586281379, 35286.959498742355, 30899.62021138183, 99974.90100963494, 42711.6224858181, 94820.07672619088, 36751.703789496714, 13870.644375012498, 60834.50053780254, 5218.310212301669, 38361.495216104355, 38032.168038432035, 44362.18298781176, 85829.97457152882, 90008.10827542758, 34930.54679036367, 3049.555918199065, 24941.873683874204, 53002.61945664726, 28162.17688834687, 7537.3038938737145, 70674.1224487831, 91803.49643586925, 23393.828291191687, 68219.2446225387, 31656.225266859015, 51447.53872399873, 71451.06454302769, 7378.327320276323, 83967.43233059002, 18346.948841056244, 15399.086028742093, 88800.03689659003, 15890.918000834208, 72169.39385506904, 79463.20735192692, 37616.461164469205, 84677.43478686741, 33404.34227628244, 16851.446449499454, 24657.15329505793, 63155.84888913212, 38322.125448079336, 55944.05892177905, 2510.627576526292, 23233.138897296456, 38577.59772817782, 99676.99485304102, 23780.78861803956, 37146.7861516206, 22369.35784403492, 42345.06157883508, 86794.95674668487, 14694.914942907179, 47635.22025045103, 14203.040040389447, 13599.329366295788, 45155.7042526146, 41826.37253330707, 46709.458516728664, 23859.47025865689, 65322.1859913765, 86091.43169791375, 96260.51871160627, 18277.060796606103, 20695.318544335372, 95020.64505622092, 40619.23584092084, 15648.162656143239, 48222.64062749805, 94987.24431620586, 39620.456816121085, 11922.966055262985, 91692.95663463285, 51117.695139122734, 51673.757090140185, 73396.53899479177, 88664.3204232933, 29932.038688482953, 18545.994378115673, 23390.791412211885, 58107.48500018591, 81376.42459356786, 61698.70908322482, 23895.821289127005, 11925.435278501629, 57599.535176936544, 10993.303615362604, 16645.561913580863, 58713.71357402225, 50834.845572462815, 16188.49701838523, 41007.30953935102, 29128.521845303425, 29147.743227672618, 51682.305880613334, 24117.78351219448, 4536.1883848106245, 72755.04112701958, 39905.24857589516, 23112.187536602258, 83487.6941650048, 85192.1844989409]} +{"id": 826, "vector": [44497.26762798955, 16431.44030626663, 42221.18545470253, 81710.04669372803, 35986.12768360302, 40392.52469931117, 85769.02727169366, 19199.680922577812, 87997.20582487535, 32991.78997471996, 53640.97856273878, 93847.32564233002, 2026.3851542657974, 8769.76751534152, 45629.84000443969, 38869.55955886516, 79411.98674616955, 17674.991173157607, 97005.22814953022, 15014.147272333055, 36383.46299747752, 43986.05617743537, 67140.51191155557, 48885.43604166452, 89202.32004084947, 5111.147417814277, 15317.721072471324, 83460.17766631665, 1272.5519167510636, 81845.76597112931, 80385.97492830225, 81817.24205052153, 68022.37620372139, 20287.666494693414, 33651.195174058266, 90158.1145818068, 79778.14153611481, 41033.61067187211, 26384.806594151178, 24592.372319194423, 98350.7140914485, 25323.585217928623, 86893.36441353474, 44885.244100636715, 32157.435397491452, 1304.396530015317, 94406.92679292706, 37010.4456328863, 40600.85768364021, 99453.79855620276, 46536.96296002407, 85597.95167865843, 47902.09652696675, 62115.312830563205, 86668.5003899853, 33805.64417286053, 69399.15397444357, 32278.733701748406, 28030.620331177168, 92913.32874534653, 35669.458049990644, 81454.43902824563, 94540.29748444547, 51014.15923752616, 56197.936735583484, 65577.94720070332, 68992.6482602087, 2864.0454653999445, 23017.40472404472, 7705.166981901768, 81421.3646071129, 75391.97059680065, 69133.56508991057, 22042.6959737966, 5978.913211983927, 78311.71436361353, 95895.97977807828, 73901.88962608704, 42600.77696910908, 70860.11307601069, 22624.397909626925, 84720.5886648809, 25131.925524742437, 10053.883669202834, 25855.553854819602, 48034.34070135086, 70168.06871553404, 21675.40073881371, 74692.11496207882, 41972.57989726989, 49719.67189568483, 50659.963816234755, 19767.509696865083, 5352.76378206857, 28678.54615396389, 2817.028207686556, 80550.95252273756, 34445.37288366951, 69691.90688494619, 3391.4495838022085, 60577.364817790665, 6129.534561110239, 4450.272687409362, 30757.169573258147, 81710.56782647537, 24620.804124788887, 84692.62972849421, 31782.024238645547, 50784.42056433937, 76901.07401337067, 12630.649050724962, 92647.37482698276, 2914.598222205855, 95968.83316764818, 97564.91864191776, 19925.797882301267, 4192.524162040711, 90569.27913761722, 29220.364189052572, 80132.19366508558, 22740.769437321673, 9340.527301598666, 96536.75146476981, 69489.16422295533, 89829.5372652524, 45746.1521215957, 90028.98554532441, 14152.091942453482]} +{"id": 1097, "vector": [51439.72083276991, 56821.798652648446, 68451.49835969957, 56180.83665113469, 54591.68113606983, 5316.709153256338, 8266.286060412842, 72142.76686962399, 18332.283892090894, 52950.86572652882, 97337.09578828888, 78836.96157613993, 40869.58324641406, 35155.670102691685, 45758.67181381239, 27834.005922948567, 64229.87703029612, 57291.825226916626, 65665.20324210684, 16836.385101453612, 86887.94642893213, 3106.9552726451952, 6173.875376063976, 23488.271035639296, 97849.01495338255, 64348.269725830076, 21192.800086414165, 26911.053147856856, 66719.86759858805, 16453.45040324746, 2204.3895554841874, 49715.20852669606, 71807.29571733408, 91613.11984329922, 44526.28164925052, 88578.65680544838, 70414.60380177162, 64940.27002636297, 40464.99277256001, 49700.65655299849, 58486.75960420868, 75261.7940260857, 34020.493921966256, 22066.499249234796, 94463.6480171243, 16280.478591852554, 24013.3803152504, 35311.51870116357, 4310.587736789262, 47247.2829292374, 13152.6260343787, 60394.58689857846, 69242.80105469444, 37792.58699737112, 93007.29201254969, 9541.22848971186, 59396.80644245713, 39960.46917534031, 9113.649559671378, 9090.93087408036, 12901.740556683893, 58610.092172512006, 38534.435604256745, 65796.47405058092, 21853.445116815816, 18569.385645814786, 29182.75014862107, 80288.42554636055, 93460.87931466663, 44031.888543049645, 90817.5809990093, 87087.50780684334, 86133.394098585, 50499.62237300141, 51087.90758533801, 3888.183231585085, 38832.43047745477, 85441.91421995989, 18358.533460541115, 74156.81677577089, 54477.06382444947, 30694.864296157233, 18881.484961066388, 39400.94258737513, 45226.84625054641, 21667.104614574007, 40855.203774227564, 90530.41463310955, 93483.11552359321, 24055.721342992707, 56900.34793485609, 62682.77931719073, 71421.09443529209, 25587.2169466536, 42084.25135378051, 4861.396734349599, 46403.038196528825, 16353.60146202004, 78439.85552267374, 79944.41068080386, 33440.298593213425, 95259.36976545287, 72982.61072671245, 6043.667620562787, 41545.25517622877, 47315.87169859671, 68871.11021723005, 17325.179566788283, 78459.14501919792, 29367.13056412993, 46018.79272492034, 80211.0498756352, 13000.234428145941, 76195.97945990729, 27343.177856362276, 83342.7450076815, 13083.704304319854, 90751.48872203518, 19002.741913756672, 2273.3258136223067, 69914.33784199005, 97761.58684064054, 39186.63416177415, 81100.05993135808, 92000.87735547424, 76979.41531548649, 51968.705821008145, 18468.738139803452]} +{"id": 1961, "vector": [45209.41188989427, 15011.968164052647, 90646.82843644927, 77272.2702065905, 40368.3738392817, 7861.613710005666, 6829.537698358712, 70346.45134359985, 5170.266831126746, 60020.33979270951, 15128.93624805859, 22302.08116365533, 78336.87483485698, 45656.762296314, 71596.05916752409, 84181.52083094225, 6551.3090810950025, 90654.56684505103, 13729.321377015734, 33166.72255855112, 74322.9159918006, 58444.83607644722, 61103.11484482597, 56319.46412248876, 88474.44025614008, 58988.11106131588, 25731.753686573633, 92649.00202812132, 70598.17383710106, 91323.3180616532, 7514.846233343964, 45998.9479735977, 87858.90910878703, 61757.00436339401, 87873.29293637074, 66787.7364992594, 37629.23222232137, 83266.12921870382, 96416.80626297437, 23701.441244027897, 27148.682770494815, 56322.71177092343, 1179.2128680706005, 83690.18590063434, 13724.776983638487, 63374.86001535782, 51626.57744254999, 52475.139013390995, 79167.5978497002, 59298.3242078359, 54653.565405841444, 33742.953775711656, 29315.863785987385, 78253.57588727535, 67685.88432761698, 66028.02735683406, 90428.13656246728, 26252.08936838517, 77702.70674550082, 51213.34822515167, 85045.05699468154, 64817.1575849219, 52067.3450153584, 66881.75466181574, 18984.39209063806, 32553.63054165653, 90337.12110926726, 76831.7235817217, 67014.66067153068, 45492.739797732116, 55758.45771342856, 28471.5853660523, 30090.47261299005, 79292.89094409338, 64776.569777617966, 98052.90849873687, 12185.781730361155, 49771.44394920078, 13541.683902372426, 23539.201110740116, 10640.36091797027, 39972.68752170884, 28698.860993848495, 29959.08649934792, 35151.44354559484, 90209.11103361739, 71288.11576481532, 35315.50485734162, 13127.495432770631, 48770.56966117767, 2090.56071336019, 35188.03508860839, 53654.02611111367, 87395.72684903201, 85046.8259685089, 261.1305556572852, 65065.419249050494, 59291.653641435325, 15412.416619723968, 79603.38431453376, 69944.42015902993, 48958.7907725447, 9457.522042648858, 13957.941479137182, 62695.34423209583, 48201.49277194804, 95451.4863099125, 33907.79417790164, 18678.182502631378, 39518.1640868138, 97339.13500880681, 31154.22932428614, 65769.5468466647, 57573.8955508928, 86400.43022634526, 56004.77055062672, 78910.81284978356, 96702.48674557719, 24796.93661735014, 91384.85104073375, 8212.147503093414, 69834.14212229093, 66947.70619965703, 51369.32878297197, 45839.84925622001, 14677.544248544993, 96303.66642376625, 34514.18629659968]} +{"id": 438, "vector": [28426.2907339083, 93602.60278274748, 28959.485749498348, 3738.063406236325, 29294.402563017164, 71179.13233886675, 4263.269980771233, 29580.5191085717, 70397.95609455895, 65034.28359673288, 55155.42201565821, 37907.9828243467, 6796.042174594708, 72432.30725610658, 92538.04723918367, 5928.031456370375, 39530.38121283572, 12171.33714833738, 41937.85456377449, 4338.588367242413, 96226.03396898633, 97019.66010809905, 53725.97911822171, 15736.376431021992, 31388.81658822157, 61880.334827457016, 18815.49957811286, 70557.52514291169, 36525.13103976004, 18387.637594133088, 10645.512617732755, 2008.380340819771, 41089.05145299806, 26210.050173632582, 64974.44980453963, 80026.64248610097, 4962.356744898367, 51011.91986714303, 88041.54972038417, 19128.207279175745, 97209.92199519629, 45920.653928209365, 15787.301756485662, 8910.697112992282, 33794.21530260814, 27588.508539743918, 21580.664719625755, 38731.312124407224, 62220.978665524286, 75679.18710600039, 4573.170370440216, 93321.9978456233, 38068.87233878424, 31192.627559416553, 90766.07364434861, 64624.58131489469, 10379.491446950384, 97318.94531226082, 15929.86614803371, 94655.73233765761, 25841.459622142316, 69915.55314916417, 1656.3914964291437, 65487.601611684564, 5319.099845978847, 80648.39257609825, 6125.114101503348, 22693.20460687938, 89325.28409076206, 2460.0146802075897, 92503.41658828227, 36487.06734217411, 67769.40911560597, 7656.428257654257, 56999.26949014904, 50359.04557704739, 65169.63442461082, 29958.486797445727, 34779.96918462528, 53295.05702863321, 67785.60032088275, 55537.90983999458, 36642.19560684412, 47026.22931082043, 5381.529624704662, 41901.26634465104, 12303.858978532211, 93537.36997320519, 56380.14426858592, 2783.9843271804666, 20480.137286855992, 58868.20327528888, 91650.57128623957, 78711.60112375242, 6770.359991366515, 91848.80701343197, 8645.397663550302, 18921.796977923, 29718.129079953713, 75634.45006209702, 18099.45928535147, 23116.02785432678, 25024.8426924339, 40799.78862166963, 82796.966804094, 46830.00851911172, 28435.24078463948, 54913.414667615834, 82912.90922452456, 37604.48871906104, 19280.734312355595, 15305.07092586082, 57984.88116435558, 53872.49572889521, 92298.42851405767, 91744.27937569257, 15333.970612019964, 24802.350730794886, 76723.30584739226, 59549.06891988907, 62931.81621738091, 90560.06292606523, 67214.38699291482, 20251.990918713935, 26147.58028711802, 54989.08626093929, 35267.173214741546, 2227.504162411709]} +{"id": 2111, "vector": [5953.531106157928, 52649.42693507867, 8089.976706779278, 87157.46325681388, 74188.76727830482, 67006.82754148247, 3812.4647805905965, 95185.26096122773, 2175.903509678967, 15315.57821226015, 29153.002359624603, 61791.86504932177, 26602.23073365605, 6932.967507928589, 3010.778580399354, 56830.217104512914, 73800.06745566089, 98529.5099145769, 58946.14138167911, 60435.773053107456, 20301.111684304575, 17931.853831866785, 2828.208393519116, 37270.92999442309, 20723.527101914773, 93774.27990061988, 19751.601384882943, 2969.8437645257814, 45397.397058163755, 96327.00559731692, 61321.12209776439, 40772.77697983589, 40269.81427270032, 47743.941037439174, 82090.48840622429, 96839.4831991019, 73304.35927341024, 55615.18094231257, 82727.13032552264, 36988.02019940781, 36373.39174808547, 90893.30736274981, 25391.196768651203, 10023.902954763076, 54850.70901503988, 81298.3177137528, 16116.700895524727, 4506.541410187903, 92577.074665615, 62775.796603806986, 52890.009307578315, 22830.22374925433, 97135.77509577107, 90475.35543253076, 40632.61781482833, 20047.53855020487, 31533.26238439942, 65021.677304100165, 40399.903036509335, 52851.49052024317, 91544.8932974817, 96143.25804393, 52471.410546401945, 91267.34924030685, 92679.56307687376, 97775.37591890963, 36044.92510873486, 55168.0145495394, 96319.58064945637, 25166.510925995157, 27128.335309100494, 42143.5352846672, 80625.23863102216, 5768.873406430708, 27364.510129448317, 42586.005047476894, 69063.45175589739, 25170.430860730696, 9427.182715323945, 77769.30196665537, 61367.96312783202, 72628.5156540765, 73836.98644384087, 75789.66750372728, 5717.911646477302, 38110.04934442117, 16832.555863392605, 18801.284755406923, 20390.84851893924, 30303.940390596017, 30952.70969812128, 45721.94150499991, 58447.060101025774, 37921.83422208111, 88048.48696757006, 57452.56908715218, 11157.857775448809, 99655.00442556344, 94778.62938182893, 15105.737935170082, 66219.89899277338, 88419.18771577097, 13684.537084744985, 46917.84199789237, 11222.303636818653, 63906.70732959906, 30864.399684516196, 96844.5813036727, 7103.440006788153, 20082.993079761723, 87301.94751033545, 46329.33412234237, 34339.81764620506, 43351.798568066646, 93719.22883506275, 43267.616200345015, 34152.19945653396, 60684.117490955556, 70735.3331644716, 22454.390750298615, 29226.11899373665, 80373.45240847532, 18249.514487893757, 80225.87172944174, 85368.22816834056, 39940.347554197586, 25952.536417305848, 96929.09495377947]} +{"id": 192, "vector": [59526.63564468636, 88876.50919244268, 44088.003394294305, 41433.01770451899, 14259.658636051665, 56788.964720253534, 74075.21410722386, 72412.72120763981, 65358.03269373235, 59212.560168236916, 97917.79908438555, 10203.487891687324, 51037.53699386521, 79300.46038744769, 58939.705336481195, 4807.992082260482, 3877.0172460330987, 67208.13056543007, 13720.226100915977, 8142.9989756971245, 62897.09069478018, 68255.28056239268, 86178.11107266734, 69835.17682320315, 41292.85323529898, 38533.49117118108, 76141.22932702967, 75943.18196879578, 89529.1118633165, 58310.45556435721, 54167.60428302765, 25080.693409431842, 18247.139816012015, 82718.58231341177, 9003.485346722195, 45781.323115935615, 85835.42661183084, 66410.6343497107, 78980.22444153177, 97823.9813433288, 23771.642578792318, 73198.35731860429, 51879.69821395259, 52699.76057766741, 67384.5540326244, 17352.794064966194, 66389.37049898383, 84969.1895516185, 88904.94936976531, 50548.2591782578, 48250.44529708514, 61814.16384342622, 71476.56872516908, 52379.60345555888, 24854.607698816788, 2459.443311322862, 58349.87400344278, 22054.916849976526, 49113.21051278353, 31655.46125766362, 49073.152401823885, 30775.74776452444, 83744.2675001623, 90056.00973218806, 38462.18722726462, 12915.212290667987, 44665.61574419789, 35100.13638127452, 63888.27296619772, 25819.9297144176, 28455.277041209203, 16667.116303781015, 22488.844604243153, 65295.28394360405, 57032.0318922957, 17611.70562514791, 95814.20253647825, 51794.21141692424, 14472.02691554963, 14953.642927443077, 27109.901063260422, 39563.93639081805, 55299.52252383259, 72321.74801450975, 20695.573095408603, 49043.60915926016, 99901.23793701516, 98608.9547928552, 38522.197810007776, 63533.26769959284, 36764.64667103726, 8251.235846879323, 86046.27894804954, 29556.03697216601, 13744.381085617053, 67149.57707721004, 5603.920268172591, 45042.29479417244, 8225.127445374137, 95627.79675568477, 77291.61673642251, 64110.29689327296, 8661.955273156875, 26328.571079394012, 68884.70703085832, 73958.57517924732, 71515.3030736513, 4813.643402636858, 27155.54221659414, 4226.5323273097065, 23090.182317999763, 51413.522473674544, 5237.16515444489, 19494.456456239805, 87890.24351648554, 54011.61589309552, 89369.77219682549, 54473.01450225731, 72175.49018821992, 8546.153876255814, 1645.920502239051, 14210.595478678079, 28760.301454937653, 26753.62215893937, 61815.81898828065, 31888.408736494643, 82797.91804151054, 72750.05157738109]} +{"id": 1944, "vector": [44460.410147958006, 13992.05776096294, 66433.75596991686, 93326.5386650315, 94402.41853797183, 33226.65862203473, 54559.299313192314, 91971.49342734367, 25665.46035562256, 84728.58752792484, 97704.0121565765, 78763.07140919469, 32877.353677878615, 10343.88449350081, 6902.112908819436, 84750.1129973942, 96746.13374280038, 10634.383903539312, 22697.899684095835, 46179.330394744444, 13827.6933828975, 26806.40522931067, 52383.83587310299, 40682.438968856186, 78962.40880828023, 45581.8224079466, 59808.53035518113, 27901.88162466678, 86124.38490524274, 62970.762418975515, 15284.072160418838, 77989.4302923292, 55973.52216899691, 57978.53360089172, 31979.479325546024, 2789.2827943598377, 76179.85040735384, 48875.34237462711, 48089.51067183471, 9786.056121964715, 19861.680143982874, 18708.022128101486, 25322.36780812428, 33023.72452721642, 60700.47001320834, 77483.17350777535, 36744.12994892423, 15927.75357340962, 56688.14125194146, 24420.016427733826, 7075.329243708639, 51848.60085237501, 7575.709120681673, 32927.590404437455, 57424.86967064181, 87356.0319863701, 76565.82290411936, 51416.769148087835, 76625.74314532816, 60328.24246244976, 71747.30009388895, 6390.704204480236, 13172.213206135775, 27524.065012447874, 50284.21502663122, 99477.95942265747, 62508.22832732012, 97277.80297586076, 65021.22180893407, 7547.832381250696, 83543.90430465911, 89116.46116501016, 81846.52148298456, 46291.777273683154, 85662.46665248608, 94559.79585325142, 48065.526354748, 71697.84760645035, 8306.1353328463, 24185.765048122186, 7208.7874114524, 69981.9706469125, 97718.54060155887, 41599.59493244448, 37791.53106141727, 34557.13841030158, 38227.81317110434, 73200.26540744248, 71218.54663421483, 60962.04854750582, 10758.818311874596, 96353.89789818744, 51696.453886521755, 6685.2353470514545, 8496.026648713783, 47990.341758931434, 66625.22817134058, 66020.72506509059, 76541.62186667239, 74735.71023307602, 28390.220611608187, 28496.780492617625, 39036.81948535439, 66945.81946102796, 65083.24752928073, 39379.153114981025, 43564.83631948405, 44778.1298396627, 15183.173980218979, 32009.567174123444, 94272.84658453698, 19857.92948080286, 83805.91170445034, 17599.967180059793, 79107.28585579684, 34550.073398557404, 87592.35911447692, 26106.712354174764, 43210.26034781698, 1195.1661734354357, 92962.64006545344, 67091.8046474012, 82886.64966732709, 17715.176742594474, 62078.58435004887, 53965.59521247015, 60056.85657965927, 67921.98705400563]} +{"id": 1221, "vector": [15833.59260477153, 15530.950194341574, 6633.337561565389, 6640.2287671946115, 44110.97319707905, 62155.30614679429, 85706.24530157894, 58974.36057395501, 25172.659810217447, 21195.92636655131, 12802.237238719415, 16324.557881714185, 3152.9817726166143, 48623.568001814165, 16867.184254868418, 86849.10729216841, 81112.970497397, 23217.269776135265, 3785.8449258424744, 40482.142295117206, 86953.41450270596, 84246.34031849317, 84783.81347460403, 55443.854102464466, 77817.2760315736, 78272.19201697122, 49013.593017799685, 92992.86096112258, 4104.806141771244, 48708.932127393426, 46746.67086330997, 47678.87536770899, 40026.132959772935, 27659.160666956217, 20078.66312114547, 12029.693245631379, 92290.74712799232, 25693.786798143203, 99958.4872555837, 99913.22449274914, 53975.47767875449, 31398.324229461916, 39430.23054978726, 30696.13047707931, 25472.901865136046, 5463.119236960579, 24133.674556373917, 25713.109097102493, 387.7888356565107, 70389.73203487885, 82823.05131323633, 52862.80589870165, 96577.65021083783, 59419.9843396959, 40033.57605006097, 68890.97495269452, 38029.40064527878, 34969.71164173271, 74483.33870611092, 55096.2178690553, 75237.60984679777, 60989.006642928056, 11962.460026620725, 16508.743472131904, 43771.26999593699, 9520.136741650564, 84881.33898780037, 93055.00891212453, 64825.8395659823, 89531.82192177954, 68354.78753993326, 60339.86386441827, 79721.48145009183, 43046.5614837806, 98572.35517103934, 68130.87248635733, 67509.2586784275, 19431.24591063986, 86502.91170426189, 75090.50463923397, 73918.6209312705, 81972.36876451943, 46568.43099572617, 78037.25652289634, 23108.44459265191, 92291.07258037034, 27073.24123802175, 28821.549455043736, 50925.30753292744, 549.3513682388374, 69641.6633752401, 46435.55660165983, 12827.611815362872, 86125.53524524975, 54754.58937328165, 2633.467313715787, 71730.36975284993, 97567.68675813545, 97778.61105669088, 11435.260605778285, 7591.872561166846, 36827.477822899345, 90239.40209542855, 3407.7471604360367, 21035.043466099847, 35673.760986675705, 40005.2567193375, 87944.04953914203, 96965.24080560861, 51844.30605698098, 79858.21086856812, 75450.76618918905, 829.9210101527121, 8224.435525333529, 56993.79934510662, 38682.435724161114, 81175.35908399004, 15249.47690538413, 46185.22284201235, 55080.64378764326, 81268.40484604568, 52661.498560351014, 33160.2897684537, 62377.10566242213, 27215.240410160703, 85896.18208705042, 74839.8899186043, 29844.89457520818]} +{"id": 947, "vector": [59129.26569859606, 47900.55287718199, 43735.12826313787, 20024.62101471212, 53129.88874033841, 16409.540282871083, 96753.25564026997, 56939.394308406365, 81506.71880648525, 3831.252331819224, 39514.216408753666, 49717.43763486777, 78739.79791487457, 49918.93854013386, 22748.068891940064, 30494.352999735074, 42104.33137653581, 69957.29393926014, 9979.210552451712, 12171.356154682644, 72383.29782726198, 89409.10253923647, 66632.23350105176, 14639.430563550715, 78534.26285840855, 92439.26639287113, 3105.7721931697756, 97264.08183980775, 98609.07551985829, 50477.907863535554, 13414.588412187233, 36065.98550800031, 37574.651997001994, 68604.99633907498, 48909.82258276455, 94397.70770640414, 75966.93219660607, 41908.3797794736, 71556.74858168022, 952.840952741596, 64086.5911034072, 21499.426856459224, 24182.59847478438, 52444.39399291262, 28858.12705208085, 76359.81393235055, 48670.47337758128, 11377.031182416553, 94597.7548317248, 15582.310627284058, 87996.45490888825, 59188.80758552195, 64135.35348536002, 80368.87617287107, 35827.983963421604, 30611.55087762848, 33356.075236738514, 93067.20481773112, 36978.520479274965, 57786.909448211154, 64904.67040611022, 75087.45047312546, 18962.622692894805, 32689.723395364847, 89201.45964043197, 39785.850184280236, 42927.09944258759, 52586.23534224297, 8509.313567817966, 76950.22438921456, 95446.31558297035, 41867.97778817972, 91823.40802972556, 31152.43451116372, 53679.90159151601, 12891.424681299757, 79720.1965373377, 1070.7027072538012, 3667.998156563712, 62367.11450145337, 58018.221903910075, 38863.119511311415, 70738.32974366182, 19314.66910242089, 42905.444602462594, 49717.652976084726, 16801.134024662868, 43577.76530402463, 3608.1950510807314, 36172.88510617444, 36110.03552547274, 71342.25208089077, 36942.95300208055, 74204.2261079686, 91458.71746012894, 52613.335328256915, 17236.69379054512, 24139.032385502534, 4730.02962077097, 51955.16196533642, 84440.14459334413, 66916.75104100292, 25612.108384080846, 78057.97330607008, 29270.858018262825, 11756.445264043281, 96696.0307629922, 61245.38045353254, 94301.98596811257, 37105.40269890852, 19428.705397144287, 9285.980034563701, 71918.1520525454, 4398.314864684283, 72946.69375666168, 56596.57708729691, 1098.2874008441445, 16355.420284155598, 62689.149650387146, 38352.84561925372, 1407.6441893442259, 38416.19092390014, 70394.17311589593, 64989.134914847666, 21942.105686056522, 24697.204067903145, 85548.10943331242, 68112.82279800474]} +{"id": 1697, "vector": [72514.09645926002, 66150.9588095762, 88416.2034775404, 49147.382990855935, 57109.78548305279, 72996.0478932932, 44606.67827676691, 46903.3848188659, 34587.66394851075, 12611.244129831544, 51687.54073376999, 64768.95300230556, 43891.64397418695, 11094.190486232792, 35472.714678081844, 11797.218652604824, 65369.65469052356, 26493.536918627047, 61083.74817030389, 43531.03685876081, 49338.612118139645, 362.44721334485195, 49677.777571869534, 68377.31801122754, 95694.10982896049, 68526.6371389814, 64158.4618702646, 45102.836458485304, 62727.48972495388, 62813.45426993394, 53273.47533957464, 58712.241731220696, 10914.801498462479, 54390.55702044842, 22738.15782772791, 26121.633091512685, 59577.36078064314, 82283.71523624091, 96187.59889146096, 6491.601219443844, 50693.757319814315, 94329.81497868463, 54909.89001003499, 48070.56259392982, 56622.10781902266, 64018.75662172225, 66539.54068072654, 92005.57623257903, 7929.205379401971, 87502.73669567284, 71804.29941734091, 70583.4862822475, 6213.096496031622, 11345.909066915272, 68578.94043799724, 69657.90925944419, 62080.06787096271, 80592.9978055102, 23615.074952232262, 17902.781414721692, 7912.368615696363, 90139.49131954185, 5856.6838455336165, 71223.12783583209, 32565.284179340713, 70613.18907949417, 18318.377079432026, 22398.023905143684, 95618.20198498345, 23257.835284675766, 75652.53048629474, 85305.1878496525, 61183.95373533297, 13540.158666495572, 7658.851136270373, 13078.231994434864, 97807.85491645291, 24517.991466034273, 19584.10359120245, 36911.19150765818, 86859.28512839424, 1472.9750536129682, 54554.13944036792, 90099.64769928505, 19135.512151389823, 20894.699734648668, 88470.52405518945, 76753.16033362804, 14444.811200966245, 19910.395412836668, 47004.041508220915, 42801.57253085127, 7924.147937038306, 5751.738565821795, 21543.644853827915, 18868.851324891588, 32834.812039803175, 25234.345521680345, 7141.271877134769, 63466.04466789249, 98109.13023914098, 32405.184011651058, 12017.746441046007, 4697.23416859803, 73788.79467673412, 57036.14286671279, 78610.88314989003, 69352.38807122456, 46051.1996384706, 88987.7118316213, 22159.252641427207, 27561.57088210356, 14029.545424008205, 31714.31062288327, 48654.3791084304, 64070.266708263654, 68670.96789571804, 68734.2070758134, 70326.60046659749, 72406.01534344358, 54141.692533715526, 38121.78371044878, 46364.58909197381, 52418.179692742386, 33472.47422543472, 30566.52201894633, 22649.217511934083, 93910.71618178528]} +{"id": 1987, "vector": [25218.183983389332, 44533.985744842474, 6322.190556237139, 73331.88160441715, 97251.53360899823, 25205.58305870614, 42204.7100768402, 94242.52230536296, 39070.58191102725, 83324.12744884881, 56338.64359027033, 47706.752061599065, 22929.92060397646, 24877.64281278323, 60720.831373666915, 57778.608607970105, 66274.96421431156, 83405.28111061781, 97600.81704887688, 59350.2398164319, 33972.039448864474, 79667.36748423369, 45534.781370795165, 15927.337514777595, 33200.414354767636, 42522.10977989211, 40247.1119525019, 61721.5031386745, 48090.57053694146, 74885.43546139363, 6745.275043641519, 39283.967676455846, 51647.35652569839, 61592.829147982164, 72062.39183075106, 604.3402143218412, 83580.9884923049, 29081.78709279329, 73643.91956464351, 9337.98211329553, 63049.66669668882, 30452.62916189856, 24909.589439319847, 7369.866086536292, 11823.699675607502, 6492.871742286544, 62879.12771518164, 21894.686605828607, 76034.22852004095, 74390.70642023972, 70492.78101855464, 67249.12022656042, 66036.49653843236, 16214.252734345702, 96700.29456505382, 93711.70261128379, 83665.3965120715, 51392.845648763185, 4028.350206798159, 87737.21275319895, 99215.71003310612, 96034.20900422228, 78054.84341829886, 20824.16242023213, 88941.73616406468, 69514.4064186487, 14620.809244888955, 23335.609184657325, 39955.55913958865, 36140.91173335944, 91666.57022040259, 81152.02536962903, 59706.574149600856, 17729.182687571098, 63615.64765352953, 62466.509971798325, 52304.051170741615, 29124.66128777127, 50148.90134039164, 46313.03680226754, 84537.05525582013, 80583.78146917644, 92882.39453308407, 34515.099246502825, 98511.50613841288, 72934.62416478484, 48287.41603715383, 48102.664117501445, 11915.80815505453, 1682.7715588909055, 60631.940230159285, 50237.29418918958, 11578.786453206192, 96746.08088976379, 85260.12132009538, 31627.467964945165, 95548.7322975171, 45999.32204280226, 72205.19236538338, 58457.60251028107, 38785.58673968273, 2397.853364345159, 4762.345306375959, 3277.1478389043373, 5676.831326671272, 78825.80826902147, 11997.301152859463, 87266.56445404606, 19393.377184287398, 26655.76757819379, 18368.29219819599, 92607.64678810052, 95872.04332467803, 20225.856168424973, 92307.4421516095, 86325.90104538272, 73439.36674887994, 14904.413699358543, 28881.16500736205, 49828.111314286725, 25156.301088160115, 73124.27079843258, 12422.399790473748, 15679.736315648062, 97612.72498393313, 94173.02377505547, 42208.392202771305, 26034.628716956897]} +{"id": 1745, "vector": [20957.974144810854, 92116.97665519625, 94025.03225858475, 31575.579867022007, 14411.762139876926, 93500.59691663091, 24256.03147209704, 54643.479776662105, 8201.02873602513, 82644.82421995784, 17835.948120263543, 44650.71002497571, 77545.93410925285, 19020.349679041716, 28682.336735716606, 35569.12521893467, 20232.107041453695, 37958.40196749559, 88160.21204771702, 41094.04729085313, 25572.97358355236, 51143.300872098676, 95594.47139203, 53252.90752840278, 14015.148530956323, 61974.25165529581, 22960.133693572083, 50028.937690359264, 9653.862769632937, 88060.20585582935, 38551.83531021257, 8116.38591555408, 73638.00428649849, 97989.92797838557, 30289.278273509924, 20213.039645346344, 69352.43758297079, 55426.71320151068, 83705.16031805397, 63043.706505934235, 26011.00474734266, 10537.088690920471, 17785.002399205674, 62234.54616916821, 92272.68483043686, 69490.08670911806, 23118.926554068643, 26062.730830587243, 74107.04067963209, 25580.806930753297, 2037.4047868864652, 40781.298513399364, 8015.73343831099, 37985.94636253212, 35172.28202335262, 75875.05091503306, 89220.51831817986, 83179.8234381259, 90747.54271193694, 72412.04851864485, 31513.398259012403, 69160.09324243873, 56588.36405175037, 55515.30391571023, 58426.04462363058, 40166.801250977755, 87838.20724056607, 95260.90708308644, 38083.39137710568, 69676.25429263571, 18091.386019302303, 60529.90255597361, 5824.75176914421, 66897.44979408382, 14562.437031347576, 70732.2375497594, 41887.39526471844, 66057.39374496492, 63310.89795544467, 96377.30617716977, 94512.16768279087, 41598.543427294375, 4937.586151023276, 1537.394290082883, 28427.09971635731, 33598.57542062605, 8619.01633246982, 19592.574566227526, 32754.359861569337, 10463.687636276076, 83430.04419209597, 47514.626228528454, 62885.98598232802, 49488.96174511622, 12237.03146102526, 98005.1870881559, 90059.85987727843, 27604.320427006747, 47049.49366305496, 72874.388268292, 34089.064020978774, 16276.68459556676, 81307.34125201571, 31023.52483906846, 82268.17961895144, 10614.692590375242, 97679.89091551062, 82336.05005838856, 43537.96854960688, 48750.42111607286, 63596.56963245445, 79112.03933147232, 27948.427004520592, 44161.920974212524, 50738.113638474206, 4912.7824928108075, 95998.33843355438, 94697.92230076538, 46591.76334994246, 68666.59076455928, 89937.41348554814, 21850.089269149186, 25004.449186672362, 52960.34462243521, 71181.2036443921, 15133.814410361534, 50476.17488418439, 11841.098466074829]} +{"id": 1869, "vector": [83484.0397356178, 81366.70343314171, 81999.30160603175, 15936.343819095844, 20356.687626598035, 72460.83221016987, 31657.68982277326, 14151.444065852615, 88778.29821887387, 17809.342362353233, 95496.56998437815, 12513.76455938269, 17257.026573409683, 76232.75090035136, 42419.61524082234, 92468.30354454856, 377.68608691430836, 18652.251830399113, 67782.82973718006, 3471.417410456179, 27806.06498630814, 41830.71739888271, 65295.25770412084, 922.027272357473, 34545.48984047517, 88714.85449981882, 82072.71373750435, 46350.04905917197, 52241.2566283633, 35856.67810860982, 57124.29216288143, 1179.0662822160568, 44343.00586715365, 79937.96379088785, 80058.88327955596, 92522.85575310903, 37369.62514646041, 92731.27609168828, 27143.74748949644, 34119.26609059254, 92093.39974159478, 10823.778826910979, 13680.796660433381, 5889.178256497873, 12223.484209009926, 924.8777389050167, 20272.799990352287, 62077.3902434822, 87643.97134696465, 71796.38525190536, 75433.92622863984, 69147.68699687315, 49938.06675428316, 45471.46054120388, 3792.347604236168, 28349.17459831204, 40373.04708998466, 58739.18801837995, 90005.45141463586, 45519.384609011715, 50710.50765666358, 11562.584143291953, 53475.26735678661, 71778.84196100594, 94679.79864616357, 65143.35402332417, 43310.84519108549, 29429.535671234364, 33153.86327677039, 64709.52079414889, 48568.97011197259, 84051.07364104778, 9299.391609015329, 26931.39850787989, 57439.43854850216, 8954.06619237994, 16897.56659996593, 99743.01485560044, 37583.18949027465, 49807.73559996465, 33220.28754056776, 60573.90895028984, 32438.092231449667, 76524.21552057669, 78659.10091149177, 75792.33335074381, 97547.98167770989, 36654.18079449646, 80890.54413393553, 55795.68325209093, 53221.7280879322, 24466.535228302288, 35153.919684257606, 96184.00831151033, 84583.21484205103, 35569.67679339007, 87158.36282622998, 97028.88600971992, 75659.74886306578, 28610.63701100309, 96716.20628300232, 6552.4319294035795, 10264.490800712289, 96360.5141368877, 11177.344544183388, 11034.916564040075, 38715.78896935228, 80429.28899598686, 88164.75675069857, 18343.24381685021, 11413.224811221511, 45039.87057380287, 51130.306133378836, 90193.80209115824, 44356.70542412085, 54081.61271912929, 71896.14350118005, 87098.50926571536, 56234.612759386306, 74468.53407836036, 88769.75851954703, 57382.209432900134, 46038.50790314678, 41528.760647333926, 5656.049844833877, 57586.84876478658, 78667.12459967023, 98361.17180821404]} +{"id": 1694, "vector": [59978.76007866765, 36588.66492433199, 96681.7972590182, 98428.68375386627, 83277.8251613714, 35583.47538686636, 28345.179504762385, 19441.443750281407, 55447.08915794192, 6355.197161588521, 36720.3859066092, 85530.83034262064, 38043.68413050775, 86206.2798293676, 80119.48952404509, 94415.65591691989, 40509.460777861284, 33024.657007275295, 11994.92445888859, 39610.015581196036, 20522.234040147847, 81190.24143262235, 54451.42025269044, 64842.06144121488, 83514.61579313231, 42074.30561942859, 92940.20190094737, 68238.61760221672, 16799.555915930854, 60420.520325489066, 62337.862039155625, 28527.695957924327, 24224.367530742296, 81301.21248780203, 7586.025061103008, 83123.22032757734, 27334.247522684473, 57557.663711644745, 38859.83614924021, 25415.543118422178, 19339.804642267256, 94441.43471089237, 22255.862606533872, 34704.43186925523, 87162.24893421764, 90552.25501584912, 56972.33579856124, 94119.1129608738, 61776.43709947318, 25299.444068737143, 55196.196220435326, 81158.90179905185, 47732.57370751469, 46352.54232820591, 20499.02094114865, 51627.77574291696, 93548.33100737032, 98503.58604949621, 66957.80452081404, 81056.68324290783, 16455.21102141948, 67440.51704762666, 75037.23437314513, 5737.242947944954, 9920.302898765709, 13032.789727964057, 39077.06528428402, 32495.375107593016, 52624.13457222476, 72035.30339103035, 56810.996810750155, 56797.49122987737, 15975.726283282942, 55989.929543742765, 27645.374605520446, 80813.5750455781, 45772.461684805436, 24836.44077640742, 49695.619433561325, 89286.78415106332, 17275.966916791942, 94549.72769137968, 22608.043043075442, 8410.903298735406, 29983.695420845346, 67997.58845741338, 78611.0892327005, 7483.867237957087, 10593.684796823965, 91800.68465322003, 83621.82293394725, 88003.23374292352, 64224.62907456558, 80235.62084352068, 25716.80089370828, 47304.81941436793, 45141.777218343326, 81352.00866847111, 50735.279328101846, 18467.874487629353, 8002.6867083913485, 1257.0619924249238, 56313.84988993204, 26493.515204719984, 5224.7177122427765, 2174.0410459839877, 65141.2213365674, 90079.95207489445, 52967.0944534171, 42098.202203801535, 53148.62957267165, 6648.20475266632, 53121.62902032545, 35942.92126563044, 31332.370984024616, 93036.17702035097, 70501.31785746028, 65962.91376541644, 74941.66164055072, 55709.91529886507, 63913.327038302516, 3455.1719693969085, 2495.891859211974, 98561.41768559552, 55334.660599594885, 78578.88498770089, 1034.0880144403152, 20462.548653077618]} +{"id": 613, "vector": [79306.88537302626, 74756.32116356048, 85614.23857484151, 46514.170628979045, 91125.23489408084, 60471.77118302023, 66907.00536819932, 18766.2051444587, 47947.78750185079, 83895.32416958024, 36130.71320626945, 79520.0791205481, 9586.56497214838, 94746.74012737101, 42604.438172360016, 67725.70521602732, 14184.842788702568, 713.0845785077166, 80671.79900664657, 43021.53508546672, 71354.19984092872, 2609.6251316430007, 89423.65297210646, 90483.78951242974, 64482.110878754276, 31792.746796370808, 81275.41402110535, 10939.562516316637, 24413.51531289656, 32007.25760053076, 61038.38620568304, 64545.339563882175, 64236.73519588586, 82630.5101911573, 33289.28711281542, 40672.368715740406, 34659.270805959284, 17025.029542028202, 46983.267865297516, 5093.143707667691, 43680.157350547175, 84116.8737665146, 59738.17886043699, 14163.10568061655, 15538.290955421386, 7150.913764295985, 26850.591035155823, 21859.099101657288, 18363.06871604736, 32724.161868973035, 97212.56736287616, 74367.24463979801, 34884.0112933366, 21964.744034677507, 65345.90183303305, 25310.524677634818, 89146.36928162056, 10327.880576302872, 37260.91657814878, 50659.55142834113, 569.8952131398927, 5196.654068202378, 89100.2762447255, 98369.72067255966, 27528.947518465964, 16576.380925876143, 71751.28325711944, 13500.359015349806, 33860.11761840155, 54814.61961989394, 34857.026918528565, 31397.098846186578, 93783.7342262328, 22152.05404400363, 93169.14521452227, 31697.40929999738, 78686.89316108706, 89767.49678754712, 27304.4246933285, 12029.977774470968, 43054.2576718418, 17078.17913015437, 53432.29976567616, 29746.33444620315, 98516.75795661575, 94939.44524802695, 92613.91416429693, 79189.69007947619, 50398.09169586894, 99202.99403817872, 55648.07073173098, 64986.69712638028, 3067.912742465473, 81942.67239105998, 72499.2576379937, 34495.52880982196, 26859.894402336715, 59299.1553276876, 66159.72398745081, 37505.74501691577, 63198.66213559823, 84322.68048431711, 86249.42746300703, 87531.86106023469, 57598.89802129194, 91664.32119783637, 6657.808847378455, 48036.6586500495, 97311.19899359813, 6557.548150786607, 29704.373159003826, 14463.40463030813, 55527.55262362706, 54097.12443177121, 41532.848188574426, 66629.50752867937, 54037.48757186897, 25477.066508236156, 63707.03496991617, 62163.69105200883, 95953.80665270476, 54639.86426744137, 60328.203439817094, 85316.05035205388, 35227.92514573882, 26547.459160682498, 91354.95190092079, 85054.08468938434]} +{"id": 1707, "vector": [20484.731578015824, 95143.28469802084, 10604.16156650168, 48944.626054916575, 52213.10692812123, 91611.9853862139, 12217.483272204843, 28976.56914344816, 61224.3614901684, 31699.734376747936, 42482.23785669794, 72222.70644821503, 68269.78445091764, 17185.90915631557, 72613.94312515359, 30631.599740209214, 76011.04851048737, 81893.60908273427, 69127.58999015218, 61141.062265492474, 20812.218257526238, 30298.40181713619, 31778.452873228314, 30099.499805593954, 74744.69877766113, 42515.66692921053, 49198.656420993, 19357.739883973503, 50632.93369417353, 53886.658350110694, 2511.644526273904, 35206.925507160384, 27605.220147842436, 44696.87146816562, 94352.48392851683, 61857.25614873798, 11766.76050487514, 58445.88818628801, 55608.986447367846, 51950.06671599197, 89627.71713551585, 36856.09614870839, 27532.486314056914, 75556.13018514629, 35904.72468873088, 44603.31459955302, 30452.915115584456, 71286.36752848777, 13804.927172463955, 57695.043393648804, 84138.74994309893, 711.8178014107857, 28454.578238216898, 6462.270863295883, 30837.765634904936, 90615.28809696487, 1149.361052431741, 39637.64898456238, 23181.86337831404, 14312.509389985627, 95592.44221238818, 13212.788660354823, 88074.71639711224, 12205.009257867583, 98108.98304870164, 56558.15677262398, 52840.68633761951, 41443.13478896977, 42059.20155909135, 95800.66779617079, 43250.99761172959, 55183.98208457276, 94276.1072980718, 56898.85752166261, 91799.57051125885, 45600.56543219827, 33793.546097050064, 75883.81254489784, 10408.61326584941, 88939.2355308462, 35094.41277817089, 92332.4088368672, 63200.18982740361, 4715.499884263907, 42052.132923572404, 17237.695619237635, 69266.94287167021, 85502.15153969554, 88286.17128873424, 7774.059908005726, 82724.77534829403, 9911.615643755733, 71117.9219917815, 98294.70431206415, 64873.77864818282, 302.54496597984513, 42419.15319846986, 40568.29436719387, 7929.095074935711, 1718.8620743604831, 986.5769928488977, 31768.992330350764, 40046.354302824795, 22023.996678085667, 77693.83185735154, 37171.640602264175, 75338.00001254259, 64527.816571310956, 5716.096935064996, 91310.58017378752, 89566.23045181092, 72159.28353278333, 54202.42869211594, 79558.31730057523, 94810.72568851012, 41830.548522585945, 55348.63339117434, 16236.477584932263, 42063.151827917674, 53856.369073146605, 89716.8536753936, 78075.17369275259, 89440.10136487019, 64452.046019683796, 85511.02657016451, 82368.00745694534, 99917.98688306504, 11059.517546523623]} +{"id": 1431, "vector": [11463.596134972198, 32431.086571774336, 21492.540831259343, 12887.953697084331, 86618.31319431204, 92277.43344467635, 14375.14694642561, 32254.776315538013, 75204.38817279066, 90961.21609740441, 15744.001857182433, 1731.2996128774682, 58311.16489284277, 21360.366892414062, 31143.0938499771, 2572.8380735127066, 5030.459144775145, 79603.49877305856, 2056.0638484727, 74023.27216997951, 54915.43661031155, 63154.41492352928, 17917.47594502626, 34480.64121545824, 73473.81963677134, 10111.404996097417, 93307.3977184641, 23173.423331894028, 58771.57668774608, 52886.05340630656, 13460.679725101265, 72351.32309673465, 71867.6490700007, 69423.76712369734, 63257.640570803094, 78913.79978177321, 19309.830243787375, 47785.82532988459, 42423.776353313915, 40692.88692501903, 8264.584735016655, 48649.51183606374, 10026.554706117984, 45308.39110764411, 39597.558679788846, 26840.774966125715, 9655.88798801128, 65759.65047218614, 14451.32259306363, 6568.3247194267215, 67022.84526489003, 30579.944582616714, 1003.3909299864252, 51517.717225820925, 29843.758005811884, 33213.83350960302, 33085.63879798751, 6479.986410720784, 43374.126057808615, 15474.812648932068, 4897.823066072992, 67537.72491853224, 58705.65416713166, 91333.37435688922, 49033.01682277412, 86897.12081884258, 89198.54453759582, 9862.691124165445, 61071.46253910339, 41535.02700434355, 5323.101968138455, 68150.88714515978, 31523.20187698182, 3222.930990167394, 37646.6359253374, 15811.301014135204, 12044.700099750306, 25657.649634415626, 98790.03539663972, 53486.999989853975, 63441.99443668475, 92373.22621074814, 87130.29519881427, 1157.8312885626162, 36001.81264336428, 31598.30771259411, 60285.877727732615, 13038.912770593524, 17485.371726646692, 56144.36710676814, 98744.1239482195, 7562.081390386544, 92610.21843923886, 29156.723466277756, 34625.824059496634, 49422.30975380798, 64909.726513711445, 82492.8590777332, 46349.7404534361, 44651.08537566362, 53048.12228936456, 52154.384506639675, 65753.93217258368, 97076.25241416803, 35689.60073776979, 75672.3167538244, 45450.33347407139, 58055.381427469096, 60324.16466456595, 36490.4770210816, 12310.657983514617, 87775.15619097442, 8160.62992214317, 5177.145638300373, 83370.61752475725, 22250.38213888284, 75668.94466750718, 42971.043708513724, 34809.52528989064, 93988.35791914466, 55274.28036310035, 27262.346959894778, 30744.64230483791, 39039.90262937962, 29437.61129746638, 85211.04568368256, 86478.7987982236, 51744.05516259247]} +{"id": 1246, "vector": [66763.42274253746, 93826.56458666257, 16557.914386741122, 50793.14188177001, 2275.924460189105, 71087.4571199518, 43897.95420729346, 11951.499486206652, 69404.75654899668, 44406.29484799439, 72113.14198337468, 17790.093125747575, 61607.63730552713, 61826.16098514308, 40441.504285685114, 86894.6562976246, 59182.419508766194, 71731.01717229579, 11224.706149814012, 35564.59696121166, 30740.962406142135, 65495.10746304845, 55825.43029609972, 91522.7232812876, 62996.178487768375, 7078.109092015395, 6221.350393121649, 37268.590563881764, 29075.128358861213, 86335.75233983334, 61811.44506582126, 77853.34594832998, 77303.81594000038, 27338.487524738142, 83392.96094531688, 79925.02340460973, 62356.12840768212, 96457.65137075372, 36763.04120355866, 14662.339085474974, 65422.28483170975, 50079.94655720473, 18634.198612536322, 69322.10075860503, 72209.21307346324, 55621.15268549729, 47646.9485060743, 81118.4692765361, 75468.81754447617, 87483.67899291212, 54225.63325931446, 23756.198692081452, 62355.70677113752, 54443.989815768, 43819.78824955652, 38277.963532380054, 80824.9705938591, 63121.01456349544, 80913.38662356095, 5127.938337759963, 39487.3600398192, 91514.94605409823, 20841.654766872263, 15009.664683996849, 64696.249927915786, 72472.8664811589, 65596.09217116695, 10.862703547764863, 81771.01268842029, 38130.78125683709, 59762.49588846949, 12342.84498397048, 67101.38854969764, 51227.8356302869, 33373.12690554203, 99261.6700751911, 54683.946984349896, 8034.771144224884, 91316.30416429423, 66070.0007290439, 92226.38165858708, 16345.801831766827, 78153.79903859994, 87251.60774317017, 83142.20368176271, 65464.11783663368, 55144.906459033635, 86500.3273121916, 29536.628287386557, 29665.295556081273, 69007.59588489747, 73896.66489692069, 5456.526101319048, 91463.5380182077, 61510.546778403885, 22370.998233556715, 15405.056464974665, 35042.598504159294, 12931.877373017263, 46383.98388358213, 59465.952470925375, 9666.752852131367, 62941.677936411375, 37064.53742444481, 7277.409980115124, 88212.14329794825, 78359.24375013911, 49065.70900824229, 80419.65610873196, 47314.787806844535, 62452.75872270757, 91404.34007076889, 24799.14544457371, 47372.657282452565, 63833.66788646096, 57661.183947827056, 30569.761637539817, 76328.82688568109, 36639.9236543656, 32281.70270863675, 45275.335896698576, 27890.619934786322, 99187.2525644242, 13906.776407640098, 54121.25740625375, 78742.0383945388, 48989.09518551435, 52791.340160217136]} +{"id": 603, "vector": [42516.79897713413, 47674.78675792043, 93119.20566921151, 48854.52490757539, 18655.581107570797, 63403.883953235054, 68690.954506418, 14900.482284748894, 99518.40133129491, 75186.87568249965, 1226.5192976050444, 26839.398907170063, 91039.07652059944, 84719.98777616571, 10563.12870727295, 36264.5482367445, 39346.921485430765, 18501.20213878771, 76830.31185212223, 20067.176750163395, 13064.112969202968, 74187.271293471, 3970.830434676864, 61275.27987445568, 54895.71642021756, 46506.14155335618, 38434.433708183176, 76556.54119584282, 2265.0368454745662, 16078.822141927229, 97198.97462110044, 22424.202278039596, 47869.77581594645, 9546.895969914725, 73135.64935322892, 24310.105471132225, 89734.69204929899, 63644.69035462923, 32282.082788167256, 86137.36252168044, 3909.5799196895873, 7626.750217889933, 48675.849190266206, 7781.787761634873, 23766.773003105012, 97787.78371028812, 79489.68615364895, 85106.47566987667, 71188.40970387819, 38330.484428586475, 23983.863439401608, 7921.306758875579, 55519.85987861652, 99480.10160515837, 88614.3299717024, 75154.70184417914, 76956.23748564078, 75539.20251842756, 28082.84259922439, 96388.70461512088, 4025.0111674237087, 15484.893435894819, 43740.858406670035, 99921.27291805865, 89798.08622967808, 45572.65290090713, 56377.560717917106, 84091.51148302676, 7802.377552925477, 16100.956005864231, 25833.063230194963, 41830.194709743926, 1210.0972139886924, 50592.357569883716, 5506.757425483899, 15507.597159564668, 35503.97299391814, 35705.05160799772, 60638.50889081928, 95133.39264175009, 19125.782250504795, 27526.492567072193, 6464.367552898709, 88247.28873129073, 78147.01899736458, 35184.563035054605, 4323.165342209334, 98207.4162173444, 88180.65965050616, 29551.894995734187, 46551.804107432756, 4010.950353967535, 30965.020489935636, 28845.760110743602, 45884.73522382688, 11458.671979732017, 56296.72416895133, 10942.047699086621, 88575.18822327812, 17701.807325003272, 28346.705552193453, 7236.217800125622, 11026.63153204545, 24840.73405845597, 58233.882443638875, 48095.39448383475, 73600.32552989169, 56445.37679885339, 55006.00969709902, 35825.84146678036, 99834.24255041825, 38683.69280241682, 33370.340674669395, 22681.633709707196, 89254.58958471577, 87721.91007321679, 94976.98949822442, 89086.9810537438, 94495.96394597214, 17451.41742098566, 35292.606025787674, 43282.70438342904, 92190.34566114322, 93747.46289676089, 61353.073484281296, 83025.8095918606, 49264.46173660301, 60487.54787020278]} +{"id": 155, "vector": [97871.73568473246, 61702.67914809489, 79787.59358363946, 79259.3348882096, 96706.9678697143, 99413.36983463429, 4643.811195915992, 2100.601393535939, 72538.09249853718, 60679.79810812355, 42813.96558595436, 71014.07042464126, 9393.323481365911, 80401.17073259485, 97986.15723241238, 32254.019938015954, 47626.737935255725, 26031.024582914695, 61264.91880914289, 53568.100669383464, 76508.57118358176, 94626.09124153794, 38490.13783232914, 59911.59598005652, 63115.31366833149, 85230.8239120391, 55321.13161379627, 95816.61346014083, 88213.30022472276, 81383.64128227027, 3839.232975882001, 16633.440731869687, 11872.256561625638, 69059.24158395115, 44908.51713729094, 92309.09540196368, 8598.484211574552, 94121.3179941739, 78446.29874195904, 2501.614744231673, 32656.31979967709, 74694.31805787314, 5008.086273941303, 42737.19343220373, 31143.15637998244, 21809.51730327655, 79347.02512974036, 98281.73340800662, 43046.480000207885, 99640.97849571674, 29969.39337629346, 94384.9399467154, 39718.10590771352, 64751.63708398244, 54971.94494837762, 32428.943691395907, 55073.04146754721, 65699.92440086286, 2064.7304758587, 79762.9570020751, 33627.52671815368, 94682.48112185312, 89774.40911145907, 56328.385151632654, 45916.69702668492, 38981.60228064801, 14050.938990989247, 55490.50831554365, 74513.79798522005, 28751.386403529756, 64145.95157799959, 75461.56650738952, 32028.743130059887, 7875.308315114537, 28092.09463437591, 64325.867000058366, 20192.557225275763, 62228.51401825168, 21970.075551967373, 78312.24899321476, 17689.834525659055, 893.77883211883, 84931.58968759653, 12684.998727393571, 50280.665386100285, 79090.21677533297, 92223.58610413934, 45746.13003177324, 8329.667355460957, 43991.602741710856, 15427.689240583632, 75432.04484026296, 69016.15686863042, 21723.876480480907, 63332.35395637572, 45877.675892623825, 86424.92448528169, 55454.06881206522, 55183.96088715643, 8888.702601618992, 75854.75430033723, 10537.161776842097, 164.32477535343048, 76027.25851691418, 48259.913324178706, 4084.7974149225806, 89310.05973397288, 85611.3787830132, 91520.67977443084, 24235.997160013812, 15465.104587870248, 8745.647845004345, 87244.44486229618, 42986.771842124406, 77485.47544332322, 86476.52045180497, 83140.06475327192, 75742.15444144924, 44023.00530557628, 25568.570475375098, 5487.9599033913155, 68052.62972397644, 21093.168418217367, 28884.992081244232, 55977.267309688104, 29300.91605467475, 45002.79836742227, 59698.393357631045]} +{"id": 1238, "vector": [88120.49896351594, 37467.0772054307, 27186.53262560571, 48876.428179962306, 28255.43579017301, 75950.1336608456, 90093.8156911021, 21053.410916846136, 98598.74994996001, 729.0444696569365, 75897.93760940878, 75731.26388429746, 47752.18916286651, 8909.80971013381, 58266.00920909906, 70623.3585661822, 47706.501526176056, 59012.14066565851, 71799.59523897318, 10947.710684480493, 91002.19732822737, 75584.81954132406, 21916.5679937653, 95524.49019395887, 8675.212354415973, 9885.210305934412, 71749.30656716815, 73777.62153693837, 93889.00925469291, 72885.67250115007, 18331.814512221277, 23236.855004284163, 23775.068546002796, 17461.547665125876, 18575.431485106397, 61276.8917510866, 81265.73676778494, 1297.3785792773972, 41955.80536798622, 5231.222577036965, 48449.002881207096, 52082.89690917197, 51991.78658122967, 68408.17307301647, 58716.544138132835, 63757.24558234019, 61047.168726385, 27543.705873638126, 66315.35089330294, 64273.33736381119, 88997.01996965091, 21751.609581283472, 78931.21153670335, 12890.311958465207, 26078.855964396953, 80691.26153495284, 6361.477193250076, 32349.5658328226, 89409.42562472522, 58877.00605962279, 18234.98533344007, 21056.941404311525, 9326.334633737753, 58776.21056830003, 25379.10920580516, 91783.32304071022, 56200.286301168504, 52559.42145541564, 65146.52297878083, 34771.67131520017, 8800.858106319609, 77145.31969189896, 52297.32972443498, 69940.90616544939, 15653.84835886937, 315.7266166294859, 46032.35335527086, 85153.20091357436, 72945.81949146457, 55371.9905899352, 79417.86486916557, 59334.97142425512, 30953.505148962933, 84401.05016326428, 14712.826721743466, 54244.00043466111, 22192.99589276347, 39417.21951797364, 48160.18511899986, 28041.881681749135, 82342.38053299025, 54829.72474784811, 26029.983038265847, 81547.31388142607, 1610.0166057158915, 39914.19331652105, 28264.051746562614, 61010.07685435742, 87630.17257337745, 85017.61099536752, 51258.19103062471, 13334.448948668343, 61102.53151789158, 26634.18946978944, 62861.811406906796, 54477.90249914934, 54022.00980208541, 53207.63424643339, 76959.14630940842, 54745.32458069923, 55999.2775156506, 31773.724441626728, 50348.098294399766, 94659.3866547992, 51237.92643906635, 81076.56132732613, 52049.628748262214, 86440.85201824464, 95294.04172967121, 89499.68983504751, 56272.37556660446, 49809.32117080066, 82297.99405939432, 40677.69730725922, 23898.49536917582, 62845.53522731376, 42424.552445344656, 91117.20874562123]} +{"id": 1402, "vector": [83381.19674382302, 18256.841992919894, 6679.6132447217715, 31773.501271371497, 56716.271663308245, 71018.98731982116, 5676.285767059652, 43655.23094260524, 39489.05708369803, 36194.38435732114, 19511.886472202255, 4415.268652992455, 74951.12734252798, 1255.265076264145, 89999.50791634523, 30092.57795799275, 34337.147044476915, 24839.19980896241, 63383.260684956724, 13992.091180994914, 3108.9156077414805, 3185.458266521679, 17343.503266786032, 10196.945288363502, 41164.38486418745, 96211.898770093, 94410.73079767656, 39060.20705276121, 3542.5903996467277, 73429.51082585327, 50481.271583234746, 38643.35232447417, 76230.19836635231, 84328.85387285464, 67929.78606240689, 94495.6209128059, 5949.026763850984, 70489.63140263561, 11351.720313053293, 22191.585730790364, 1628.784741202205, 37767.00446268336, 10567.745019305574, 22516.60607936873, 64435.15731649082, 42314.08169972137, 93555.48089587977, 23612.082001256418, 42341.3326603721, 35325.34829506293, 70859.27193754283, 94319.5082318345, 91750.87680898, 41654.2666658933, 414.5223891250715, 10504.289936031808, 99345.5591485863, 3527.478878848378, 83061.74913150615, 78921.37978992249, 31341.536027455597, 92852.85857205774, 86037.31458635833, 22428.17959437815, 60543.04808968772, 14749.889064501664, 93477.10782192301, 65528.82604364132, 12004.27783922271, 87972.56442969064, 62583.5271646026, 67945.33485295762, 76927.0114679673, 55786.10377910763, 35092.76553861762, 92711.4715884762, 99694.0464321763, 28855.923981702013, 69305.05222306299, 56273.908218937395, 84527.28458851205, 76090.44260499925, 8288.044328548549, 33118.84597206084, 26001.302526760515, 17660.286592677545, 66483.93753974357, 10516.291455463734, 19980.14204860189, 98229.9975694373, 7011.452495475357, 36091.66256196779, 74396.03406787268, 55136.28562390948, 76687.47075132959, 54561.710804753515, 92925.96955858685, 53062.55305638265, 89157.756123209, 40065.06180842542, 47752.902657963234, 56735.94314363577, 5720.351465332929, 30693.50556109025, 27611.74578933242, 64601.94764597441, 46715.65544224097, 80637.97394884605, 73262.50765518264, 54126.77597074929, 3586.5281702013194, 26155.92365849685, 88367.16539833634, 55778.41257048135, 40589.10473648443, 21220.450748148312, 37763.96482527886, 99415.96930518132, 79704.12931236673, 84178.53738115114, 46116.63760671247, 48488.204310792, 41761.64903570856, 40675.648702377235, 77390.86738271159, 12628.509613173166, 59613.72333082712, 54892.581640972945]} +{"id": 1974, "vector": [19613.271566187952, 222.22051393302468, 36844.94722695293, 59698.991950623524, 98049.19776442563, 42838.923572687294, 69470.0949648301, 77779.16993559986, 76748.83955170427, 52876.77649393162, 5195.684512119547, 66722.16904241519, 41564.34676329871, 58344.59949088831, 64218.1811759455, 37818.06912557478, 2392.4397717254633, 61643.77584129943, 48432.89377145124, 96868.69078653854, 29650.948001676446, 27717.802028089478, 62271.25671705933, 56926.86533453219, 91998.9787321492, 10666.670144160384, 69237.36726040533, 13607.052958864275, 51400.455246773636, 85855.45838714729, 95453.6869728615, 31971.958442320745, 48577.05669838466, 37490.909210656995, 69494.98407957907, 32174.29301515967, 70322.19877266613, 6007.589933504664, 10025.951821748535, 17909.42618710376, 60380.80063248381, 35931.6432029523, 83929.45125114155, 3535.674926925747, 92827.1233987732, 27242.667900141736, 30923.22878406393, 51075.94537674143, 86488.64153372507, 65680.73912123046, 41327.95954725367, 62265.03735096612, 64393.73758673895, 5151.89232149067, 19827.264999168216, 27524.225916671952, 35477.53803421681, 69588.5432683953, 78877.71422569004, 24082.791709038942, 26235.54799218185, 28857.50631220547, 62867.79071606823, 49535.556938349946, 38273.886927688975, 3114.790624656394, 18687.639783540766, 59404.10582136772, 93537.73448839164, 28018.61585702603, 16227.540323246503, 70261.94425007285, 43597.494339353194, 99308.89675454561, 98196.25411130815, 54712.22133065995, 90508.65430725229, 5426.8085131042535, 15603.772567864293, 67111.31746182301, 26452.478151222236, 66124.34352913812, 71788.13320243622, 62397.955373038574, 26562.141720241238, 99583.45002700039, 29511.790036640872, 3971.6703728825632, 67947.122598814, 74658.20674165062, 19049.99615168017, 3542.277772173974, 74446.71946555194, 43856.345820024944, 10819.839750942372, 60933.95010363684, 26348.684765510898, 23303.66177891844, 15984.829376835729, 48201.813150061644, 43495.16328707017, 35897.139597910034, 29248.868150849794, 23169.108928497462, 44693.19374724439, 60776.01048746809, 80789.16557543528, 8387.70626324341, 73022.28814223493, 65653.59727825542, 86059.11429774607, 80531.74643520333, 69987.92230306214, 27972.838380947294, 17066.017748673257, 71814.50818124143, 73599.96228975452, 4947.514059580504, 66256.56705333693, 69698.3463044179, 22872.312709283317, 92744.05080118103, 80802.25221945955, 89712.21096050752, 96454.87528232647, 52488.95884100391, 13452.382952200003, 40692.36604718597]} +{"id": 956, "vector": [24625.96210108969, 92049.8285111191, 82984.43830526972, 4657.99086641886, 28005.159643704224, 83234.317651289, 89858.86873173271, 50494.65056678547, 21641.871972757453, 25949.3407322796, 35373.21161141123, 30260.03944345119, 38904.38760661844, 75945.0625934267, 14847.993467239263, 45967.859678860914, 48087.44711450554, 92425.22444528773, 41607.711456235964, 53452.4026991947, 58710.8363685262, 68537.15527591732, 98991.03699894274, 42731.77403611745, 78324.74430368307, 46995.91375022121, 68428.1768499234, 9436.068740692783, 20220.593707984215, 10434.55798322931, 67631.05537107373, 42367.05528138413, 40458.23581593705, 65488.35145834698, 46706.436658347404, 28396.39834625407, 8333.926648684153, 9615.396156232404, 53611.677710000775, 158.07355562499347, 82705.21821973438, 74818.40657857513, 25777.83910016168, 47685.5811111176, 54180.77245018009, 24285.45447634096, 33189.86867997154, 22274.835904349366, 83250.3813738854, 50488.92796095589, 52689.31488691755, 96723.28670213002, 62068.26815967729, 57934.93555335275, 80908.63134299447, 99620.1841779192, 34252.68438934704, 58421.758041899244, 9088.108065526201, 26615.97996953353, 87504.87005546421, 99876.37682350553, 35915.27193819102, 93128.56995917067, 81346.05192409882, 27925.194222998463, 752.6842934291178, 565.0520180482732, 56437.68004740329, 12471.878822712046, 89856.18515949775, 30235.7829627828, 44509.212981687066, 86553.62945642759, 26958.40197270021, 60434.78058147771, 10969.445541411793, 37865.553284036236, 97471.64368148726, 1445.803661431566, 47988.1935337662, 86162.03367705962, 16353.78576160298, 84435.29208917757, 70355.22387068019, 6988.011087273849, 12498.089611678632, 17272.396079451857, 62512.310311307294, 17610.451514117496, 80779.18212225626, 60238.08381318453, 57457.58564558957, 92959.01661626161, 37055.21266519757, 38269.64791191231, 78039.53187434984, 10554.871702926117, 1333.2979673486457, 43345.06613944842, 39179.80513916438, 79746.08848585135, 11882.077520402856, 86594.07410736126, 57963.082900773, 64808.661591515556, 24796.133292458555, 42250.68817267188, 28287.61901901663, 28492.93284413228, 16120.450460125357, 16349.54516967575, 61828.55280267313, 21741.925204938616, 52089.78746358304, 45127.578429832196, 13477.438830444733, 17475.86108983016, 73399.71264182034, 97548.70646102107, 68271.76856760863, 51085.97492643484, 65498.29485501487, 69556.73272688717, 17695.43414143644, 57191.824739246214, 4661.910338788899, 51280.47725512259]} +{"id": 1580, "vector": [4900.221243898916, 31867.001541750462, 43479.75053695215, 55804.38855483013, 81959.81226609992, 72120.31856850472, 57261.54073208592, 1299.1836204994888, 58923.699565620416, 98402.67803724749, 95097.97541252337, 48069.837307079644, 52302.87628586066, 95534.90849043409, 28395.52226121903, 43272.2595919018, 87758.74416099372, 44631.6813063933, 5717.754349048543, 50678.44941475772, 89027.87983781548, 77467.70883672893, 37462.55092129974, 80832.8844821108, 59879.10684258395, 44349.12688515903, 93743.0433726093, 74361.1796159389, 24771.47679734628, 37306.32730615114, 60884.41557913198, 51748.64067273522, 75928.13496600483, 26554.631532137908, 59454.54907471509, 80110.79298170914, 58610.58545294775, 20170.533836872906, 30414.300072307764, 63773.82729414999, 31461.86018794811, 18313.814757450087, 43284.42006802325, 90801.82275368104, 36393.32351208041, 33263.98019368741, 1320.7254188447814, 12608.382893096159, 78502.33213270681, 59414.87848189875, 44076.758297874694, 14668.10155208902, 15213.67653194462, 68070.97581450704, 16450.093898538653, 2724.976179015559, 35807.759846015244, 95845.64863845934, 5938.21483149648, 16381.442435362791, 82154.4790981485, 80139.48875229289, 18553.227972075827, 66764.41546827325, 94354.98943044011, 83556.63279066882, 76981.24912063318, 14986.247664472841, 32265.290796046487, 68909.94927325136, 47696.37048258252, 25347.97987134505, 66532.07050162343, 57440.21968999956, 11469.79166780714, 96411.18062058817, 8486.868182286622, 34127.12765011114, 62329.91460131913, 76748.24107661204, 65761.8475482105, 9524.263373050668, 13418.408409941207, 31350.647531310326, 22463.824990962035, 26422.624482946943, 31617.688119000253, 83995.06600605647, 26057.253043441586, 84058.27667646622, 93042.02398395984, 5615.145144581013, 65141.295069525884, 40835.075439618515, 14275.370165096512, 93448.79154813773, 92624.8376498695, 3429.0436374353917, 21081.50667993426, 4494.3030775307925, 44467.73755235155, 27463.86178753184, 50666.66743672231, 35457.23729922713, 63882.2060510474, 81763.98568692272, 30564.21367262332, 54535.81104465241, 17081.55158354624, 74263.01088897519, 587.1203796415703, 40054.37046880829, 20963.240054299527, 31502.34678723961, 30573.33529504972, 59291.40258863234, 21054.610768325787, 31846.542280083966, 27661.95755268458, 31714.527229262003, 95610.26834160282, 99757.4076572232, 88216.4432803398, 68287.2941498873, 34686.51754458388, 87538.60951698883, 88743.26962022371, 98983.11402854009]} +{"id": 159, "vector": [14599.941244308622, 62054.48270389806, 89618.6348993627, 30104.362469735766, 46921.70557897394, 23576.060914936446, 14824.215308076527, 37882.09178640146, 27804.621672025944, 24125.28220464313, 51201.673195476826, 15249.692024847294, 35052.94566905728, 1777.60995294024, 5281.122098608038, 77392.12785083013, 38092.47501555357, 13930.02359316564, 93082.0110400783, 49855.0856514408, 86606.82444521907, 97382.64521485975, 56394.76740567411, 11563.934233297701, 6216.098359576205, 23553.842621190644, 82494.64588154956, 8450.04543559087, 42150.88291209571, 5454.636925248146, 80350.35510908281, 30927.75939248966, 26616.29626631151, 45338.75808388953, 11080.726433094267, 43024.13307220102, 45804.51459129068, 76172.48171351926, 97183.70660265832, 18831.32537935801, 49529.26077726491, 96871.91425181284, 67350.43285684199, 75219.26339577517, 37065.16328648488, 20857.51197875333, 85481.09114234686, 69192.62256981681, 82311.4475912218, 84204.1667812248, 57802.57584556878, 75768.243344832, 38094.32184158058, 59951.56904110619, 19412.02334613836, 20941.461125578455, 72184.82576665975, 24609.453608839005, 36796.62146385867, 32359.847151291033, 49834.41182209846, 71096.4967082131, 207.8083528356478, 42585.55215971012, 97646.28841747143, 90200.97915694895, 31392.115872162918, 77519.86139909991, 86823.325256492, 39396.85220878501, 60757.65894832971, 62113.251478225895, 85028.87090680719, 24662.683228741233, 69023.21563721541, 96999.60241553873, 94052.77608751669, 54624.689751678736, 4726.403510331634, 519.7023036711435, 54581.82037088227, 83466.890518889, 70374.27367175726, 31069.006880398298, 94968.49625322204, 89984.94617981547, 67706.04291609905, 16898.00530221398, 53163.83662267475, 63358.04287442412, 74441.55142414263, 27093.18087532686, 69504.60113675802, 87006.11528680543, 96222.52185010743, 22901.405160013277, 73304.88384755823, 37072.699651808834, 64655.15854327172, 66264.79333819961, 81007.49931263509, 60488.74613593246, 33238.16998252412, 60947.82585932301, 98185.40238683626, 69221.94919765848, 49010.21446588597, 9334.254702195432, 26767.511666514598, 27749.01345546923, 41640.408542092046, 54002.89220857962, 27940.465445211405, 81041.01761624636, 40172.814140035305, 51421.19586658289, 97708.0989260156, 92927.30361839946, 57918.25112391409, 88941.55454897229, 95925.72661546669, 26182.50559855624, 34259.3661036622, 8800.533862395876, 27696.003022364424, 89144.02059674702, 91537.94114998983, 180.33736019764657]} +{"id": 1963, "vector": [66434.53982028898, 54102.7908005248, 10333.958842376678, 95373.634125378, 12209.196153170298, 1985.9637051226175, 32763.960077047006, 59477.34709568777, 64349.47505423157, 84678.92373107237, 13623.041514670953, 10542.615139221745, 24899.818201273403, 6541.390595004104, 15884.425730043627, 93664.55618710277, 58646.73523658972, 66030.3768360916, 29361.996806297862, 57817.397788074595, 33827.38136399897, 83525.11907504496, 70096.22670341177, 63073.01298690167, 46008.53817225404, 97929.24231053867, 76713.91836518847, 99341.97172796966, 71210.87303278747, 65119.38258505215, 87506.19251905414, 94635.33903072368, 78532.13898448065, 44028.61362574256, 23218.589834466795, 34482.14775015412, 61336.986619348165, 89103.02147101013, 99374.18601335073, 58269.57881792841, 6697.114552496497, 76862.19721240042, 79617.85001627695, 26031.83945289097, 22319.30166697921, 30265.971373873956, 89805.16629637296, 2662.571695466054, 63194.241673706616, 68525.78635568327, 41987.137690706, 1174.2983028454778, 14952.698951711829, 99593.18916648903, 97484.80590313474, 86340.20503934637, 24247.64773001261, 28181.33855032796, 44215.63396729635, 48121.872976362465, 32516.158333752323, 73340.46787585913, 29577.78738982303, 11663.32823074907, 74866.41384669687, 71242.99181685061, 27687.49333933712, 95627.76972788462, 89464.26322084606, 26289.37235000586, 86927.77314045418, 3855.7914543601137, 90723.45937990834, 33831.82657154558, 93784.61319178992, 19460.894660401074, 242.94212343977105, 27350.094011366287, 85725.61698445122, 26700.934138827946, 30405.3419500653, 76996.54748070067, 40781.90452734762, 9621.251635692506, 30802.080666805985, 69956.59640616833, 46313.2486529404, 78043.92907930097, 70602.77358503708, 49784.77660263283, 67646.71669689773, 78860.68120315716, 30281.175142523633, 36985.72914372661, 87386.98393167501, 31502.085610453047, 63264.68326274072, 61400.62317386786, 93454.48111003648, 61929.980577243936, 56972.14798758985, 93106.23058760166, 10197.439676642984, 54063.13797869179, 661.8741185380195, 35786.15323369695, 43802.57054328232, 95570.42152349341, 25265.3260562547, 52565.86351471163, 53501.77134393617, 76561.74329685613, 39029.430977249955, 84640.63331665244, 28684.790591596044, 88230.24900658675, 27030.23432548972, 6371.961363218969, 99614.21335232387, 39225.4782614051, 74581.13382006822, 72669.67728280342, 61643.548430098985, 74607.91633510609, 73994.77015479654, 6673.22829624647, 84929.93671181121, 96665.64281934928]} +{"id": 930, "vector": [11175.815027791725, 37412.98837132974, 66584.80670847907, 77247.47766267275, 72777.9941160845, 85193.8475724288, 23893.215682736336, 26367.124847104627, 5896.202715411536, 23285.54386095123, 44629.903987099315, 39771.416393412015, 79988.36651419397, 5661.795219894783, 88392.59019877207, 51480.742340653815, 71654.30348425532, 46335.6832679343, 75551.47248670648, 557.8841162421688, 49289.16394422711, 70094.32958532023, 30981.93319287673, 67759.40938758355, 5464.880600777655, 94656.56776916378, 54016.629315695034, 6263.243024776355, 21221.44206103227, 90851.66537585697, 74921.37279025203, 81609.0196458961, 52529.90715178726, 2879.8602239691995, 88360.85915770177, 38128.708129989485, 32200.598855204575, 94633.07798066588, 46861.06800180372, 13128.32334528965, 81508.79689105292, 66460.41174629827, 26537.04044381333, 11387.676596289997, 79017.66106372191, 15959.881514105135, 23704.470741350404, 38036.08071703338, 23804.38260494414, 63804.310195507984, 68222.04366762655, 56077.672400602416, 71296.79217020498, 96043.20513690436, 37487.12304424905, 26781.704231973512, 97106.67001756528, 47823.02831159274, 84981.40064608038, 56528.26803291735, 71577.80724435522, 59768.37796202751, 57731.88505541073, 99400.69006864811, 63065.0186930368, 50054.7489473458, 55517.64455625228, 66720.11210684889, 32688.465686247735, 67525.63691931166, 6330.281629487755, 47291.274189010815, 60371.60696275324, 75002.60593418907, 61919.47975633573, 42680.64790629695, 19988.753840251207, 92624.37823922302, 19945.13535489668, 24655.082181918464, 21655.78981392965, 23890.813955234826, 62076.90436792954, 69230.81709466326, 93968.9170956915, 53430.14944057693, 74760.76240924552, 37475.628573865346, 64871.70397732983, 72556.63124714138, 43235.72573880656, 43102.146582883215, 61380.300209433, 22176.821506808275, 17900.15786337893, 93236.83741597028, 65313.25265256157, 82748.04118135503, 73318.51196446177, 19078.27635744963, 10064.948118272243, 26501.18214462145, 58978.92844990671, 41239.61074837252, 83401.15352167108, 40656.06466947078, 60931.981235066865, 50.90658734288045, 68988.62398577337, 72117.67743654396, 39675.5125963853, 93797.1114248953, 31944.411105691594, 9060.988327349594, 73498.75591492101, 44982.65622643006, 98154.72005785121, 91287.46446455317, 79273.49550267885, 28403.106791598588, 20193.85472286992, 86047.85229585772, 69864.68488233449, 65460.277586115924, 39307.02526358313, 88949.71472837229, 92588.86126669416, 60178.62472285852]} +{"id": 1407, "vector": [79965.70910379339, 35357.89930495753, 93335.36738970799, 67056.47823854549, 31250.71475305683, 42826.80351318671, 3985.6239375791633, 28369.14074207909, 7913.106041540885, 48999.27909453633, 43202.11485442176, 93425.16412986795, 98576.59159846012, 48302.496125047146, 92637.50703739353, 11707.692692917726, 66276.87625147244, 77280.94227910793, 76153.49023619322, 59655.89877657712, 8672.868192865002, 32482.592823311974, 22744.3508458114, 71604.04348673568, 56107.33119329178, 64244.56049139007, 85285.83282394204, 59904.3223262804, 98094.08129420379, 78685.41917813933, 73197.33356527326, 85226.19422889357, 11838.637787871176, 18016.14585041389, 92733.32073095044, 83737.8387657631, 15044.606532001559, 7869.417492248265, 63555.534574927355, 27010.866604071583, 19246.344210816336, 60949.49814089394, 69887.2229579662, 51730.00115953866, 12089.078663729846, 34876.77828964081, 35198.0156038223, 91426.44864022553, 24690.993264950645, 20238.93092382901, 19819.20034387853, 2450.651552648442, 82599.01364116848, 32538.27264600614, 2237.5181347103144, 6577.305021276314, 52579.417971293355, 84752.2563365694, 82240.60049580087, 67795.36305652747, 67391.88687330391, 27552.96988517668, 74424.91827032618, 25795.952435229217, 79360.2844820146, 36691.903965765916, 25992.231878387684, 67525.5243804806, 16282.829825956434, 96067.35115393701, 77409.84875603639, 66242.77824272799, 73694.72746283845, 3990.3896678848196, 23593.190029587364, 12065.983061619856, 6425.710767642323, 39502.40555097289, 48836.49147446214, 42456.44109567816, 79005.96855180465, 47460.16584577001, 86952.43854798058, 74242.56699926459, 63407.15939181154, 34575.7419087379, 62640.05387802672, 58230.242845184985, 48056.65714765013, 56150.718124361265, 75731.17917910746, 13315.52542820752, 37967.64188591939, 88977.47263331951, 19904.748191709266, 48799.12868862356, 86226.84204466922, 70000.99423301923, 3853.686766369435, 30193.50857065506, 95768.17563092482, 95504.13441025272, 79900.54691571793, 25098.583547366437, 98087.39229627111, 41007.67923998267, 47695.22877221919, 23832.074970215122, 26037.85180283661, 21437.011736460565, 47806.613777328144, 23540.90986632623, 29625.79439402887, 76656.88099696158, 87283.77252704723, 56517.51249583126, 649.555594382667, 41438.2115091376, 43738.20132549093, 14042.854670434568, 74689.56970103977, 38550.72117650228, 1740.9215321352533, 57521.46150532711, 99644.90485023812, 67989.7885656046, 24287.035352519026, 96068.16706234997]} +{"id": 1430, "vector": [3065.22802344974, 57414.17635743934, 27107.253046550482, 30108.687586057982, 28670.229177881167, 13182.857486708965, 64728.580681269734, 33761.49441156127, 88901.96925745378, 97405.58712273878, 27158.89797464174, 42100.10651428158, 74969.38096251234, 6512.1545454087545, 52577.73357723979, 20740.020275415194, 40451.56827241665, 39874.919537747475, 92681.33990937019, 80350.53379330052, 11124.8777422151, 14641.338039955765, 35252.00257171205, 89910.07098485006, 24991.96793090943, 99035.66286946948, 47183.33194770983, 70870.19993924703, 96671.74372438285, 89296.91116556883, 9138.732335702049, 84735.39874242643, 97286.3483893504, 27927.430791996554, 20292.55301783639, 24797.335653641796, 97037.91516514504, 59416.14925426204, 46217.951500569565, 22103.9629179835, 1898.0678408057283, 67353.45700314686, 14045.270957186029, 88795.99256773767, 17765.825324702346, 17952.665611841356, 13969.600974300234, 50634.75140983179, 46739.62443106348, 43963.0605781856, 69181.20161914056, 14192.2722810086, 86703.17138016454, 62604.0873847846, 8721.129381570636, 53287.803338334605, 1266.5241556789385, 95931.4260452759, 37416.86844853979, 29456.727071529986, 78853.0110338653, 58632.23683052402, 68045.338437789, 59516.587781534945, 3076.052876374924, 22231.657116747494, 24380.85542711734, 65661.73887830932, 42498.313517013084, 22553.04714205428, 98402.45959448027, 37133.4551797957, 50471.04818799711, 97310.59044724265, 21923.80169851297, 81122.03975972609, 89105.99360546345, 22775.55782592867, 7323.305401432467, 36178.12932322835, 22299.08471353821, 54282.958928869266, 84583.43498840644, 71570.500472122, 92631.4711139404, 75759.16516733733, 99841.64229738776, 47049.934066416135, 23812.860577453375, 7636.2253033233965, 70271.10914140908, 99782.48443437407, 95626.88039394132, 78468.31295157802, 46835.17657104672, 70108.01658954579, 8050.504735054498, 3374.8763421375006, 28827.025359812098, 79253.43884213887, 39.846232411122614, 9056.143892239521, 5527.131067824109, 81117.02554880356, 10674.536024377745, 96770.25415763419, 86206.22312832075, 9606.867286163257, 36.856001415941364, 80488.80419528007, 69307.53743066243, 54652.746495781044, 7205.701441451595, 19097.22883886874, 76568.59759751435, 76235.5910216842, 3347.852406507379, 45227.84532290987, 2189.230018799393, 81074.81153864757, 90925.31526989283, 66840.74308052342, 1641.1619114242383, 18170.528450190504, 62166.105781893064, 92568.16927883412, 46906.91107561991, 60806.48189618002]} +{"id": 852, "vector": [3050.524039228508, 79148.59369328858, 90545.76406483985, 10552.912056512143, 23393.105312301443, 69313.1555601266, 26065.746004892077, 29089.912821920494, 78371.62894264981, 47034.236705979136, 56269.97933934276, 45301.61979588001, 98212.38686218849, 33212.148275937594, 80417.92476465991, 11261.654507406005, 60044.414816882265, 61454.9027678329, 20766.113848307486, 25765.941926513737, 17573.243367797164, 4131.26845548778, 32066.390421676682, 77348.75123007112, 51252.1443513112, 99256.11311843705, 27088.870772876904, 73800.25034021103, 93982.67960736623, 8711.191763560433, 88580.02325044727, 63915.33758019492, 96019.74141811221, 99180.74786975638, 58515.73854233707, 70565.22030937581, 10550.865986366342, 23636.95427877035, 76061.30074589275, 26975.64662816091, 29998.119340420904, 36652.35686921724, 11796.386959883854, 10200.05473762251, 40323.29494493427, 80440.34735520468, 705.5675318847653, 26435.789050197567, 8690.753867157586, 82169.52993894862, 39912.55007145173, 70453.56917623983, 27419.528221281776, 77558.06678257116, 83293.41083378125, 49478.64649803916, 9450.776904493074, 88722.34613728635, 59644.89136442309, 6252.967167978418, 19443.37309704163, 98797.83625455873, 74423.93406773615, 18328.25554244859, 69446.55426592329, 96113.92571349506, 73898.49742509847, 2635.7508076776503, 85769.99933282286, 97274.13770354103, 42123.44027848394, 94845.1669565059, 8309.024809788412, 93199.64871581914, 79569.33065635522, 56534.866349319665, 7682.04776326572, 5625.293167302303, 1810.3094368614682, 48452.680922469925, 48866.45872093109, 6142.597554340879, 52114.707405506466, 52696.04718122537, 48935.36398803997, 83254.16518657292, 34186.31635365757, 76057.35649405066, 97232.96155973818, 83779.33885461104, 88356.184532757, 25049.10551907057, 59443.33953237113, 44396.13479343847, 88389.59524705584, 34401.21359432417, 72424.78571253798, 53167.490364739766, 53042.814043054066, 8994.933208394274, 31002.359527250377, 97501.64320350761, 30375.166004680977, 90286.38594885236, 31063.38656197919, 61230.60150858439, 47389.31687485928, 60061.552973177735, 61833.126150578624, 23504.2713673944, 48992.48267555751, 55740.795988985745, 24453.73238915407, 4653.848794031923, 44155.02351192203, 72989.9775547504, 41653.585511673366, 42794.53961865636, 66694.31712238141, 42337.46825336353, 78769.54990613683, 24517.34546910247, 65537.50239086998, 27228.35800805088, 93927.6824375785, 97603.97655156537, 27570.089262995447, 18721.916382739466]} +{"id": 686, "vector": [82277.21548581602, 52305.18746931109, 19513.460657586624, 29593.47849040692, 27042.459166331435, 55398.703490227184, 79660.68530048386, 81048.26465976296, 98251.85159095097, 38943.31143843972, 6152.978813604648, 68289.44822911436, 43855.66771515359, 29780.509169551482, 28949.95948913531, 36619.11247321924, 33202.54988802979, 7740.321926435212, 62016.60509615017, 72261.13146550987, 68325.00662739716, 57238.75890985456, 70946.27993898166, 65819.2898669125, 57877.674054324816, 25336.41233825634, 23783.24603797756, 23675.31432674541, 53619.1549141999, 83650.0028062722, 5285.913528522612, 82691.9413358233, 90144.1787815382, 20360.099861431358, 71605.93659211339, 92911.30327440132, 97831.85018129568, 49901.75560944956, 3807.4508901412173, 21730.88637352123, 14323.352620644448, 47743.098546056164, 11025.323102253249, 6538.251273829076, 43475.884298967874, 74639.22457864136, 6585.61611258438, 85670.09690991174, 76908.4468807307, 93382.71833848502, 83865.63067880207, 26300.382963948377, 92413.95981023701, 79883.1532418484, 85617.88051889332, 24054.541114497984, 4700.83307299366, 50144.326864773, 727.1765359777582, 36458.04214046382, 87913.17494354259, 73032.16429184955, 68679.06076923836, 83501.44318304259, 47903.07166245275, 78971.75011993058, 87535.35413602008, 50859.554490200244, 31245.34014676613, 19243.200928349448, 84680.1361546541, 32242.339797946395, 71463.55923914844, 61324.042936851605, 28190.836847833078, 30171.193133240417, 89677.64985128435, 649.42978723046, 58109.0921426234, 25061.380251031296, 14050.124319410352, 93721.20265363637, 40771.9836847738, 41163.35484343898, 17881.538910362982, 3040.549336282017, 76956.16287997739, 79145.13337193005, 91724.06362417525, 39363.31111034924, 66247.43365212242, 76016.15646844369, 56940.12041892385, 18558.475247069382, 82645.23441623976, 58358.62676647992, 32485.375058805042, 50994.08850990036, 76142.5500829117, 23712.692977534978, 45625.03742985263, 46893.93791332226, 35735.00308346678, 19994.96876279958, 54228.57081267422, 17210.090612055217, 58786.586758519, 62661.99566674736, 40477.467993226644, 92815.20887678441, 31258.690779777942, 28665.977556804424, 56944.92412637818, 57705.26971607998, 19106.80837163733, 32400.876819856905, 71736.42193141642, 60176.88404418262, 69565.83226496427, 72358.27518914893, 13797.225858396678, 29572.18396446617, 62836.443059557176, 3722.983615129216, 26537.578748292377, 70964.77529065941, 34573.032847436545, 9992.620813987629]} +{"id": 1194, "vector": [92338.07864539574, 90391.5415391432, 70589.65671171431, 61154.93092354063, 75880.08734546961, 70229.4453669624, 95837.35134083874, 23251.832235190515, 91535.97547420164, 45243.78365893271, 30311.119833350975, 57328.156725989735, 64251.98267231976, 52966.113415086234, 79088.39222651589, 98837.78254252349, 35892.594504150286, 17780.29078944715, 19228.262948274423, 82124.09372191355, 99001.1392141668, 85902.19867830201, 70902.33800919318, 2324.7092415878524, 5183.0787809499125, 84395.80692244082, 48696.56710694609, 96391.72109136863, 58152.80844757257, 75923.26580781642, 82057.68808005002, 1153.1247731305316, 31495.47924284408, 20766.38977356149, 35264.37666606145, 98935.66335223943, 40626.12095244461, 175.6121782752862, 74192.90928545214, 25094.51397263077, 8155.33186139622, 59607.62762161562, 48920.23681673988, 7263.533170831415, 508.5969429518444, 30787.563143102183, 75977.19640720554, 45735.313927141084, 58531.282758957655, 70667.02232275401, 63285.41860543085, 2201.711519816463, 23747.16539890416, 49195.01418384988, 26982.4162288138, 66459.14102954943, 85864.81442534711, 1671.9813909528702, 510.88391483895543, 82765.03998322389, 27120.70385210703, 85105.22704334409, 49278.31108151233, 17085.520084160977, 60304.40317217951, 79587.37287284459, 83244.83121031952, 31992.1765432428, 25549.000283840374, 91520.94157103563, 17860.362605810944, 97958.49182319372, 96937.60943194781, 50597.65349472866, 5877.928659132936, 61877.52505918271, 41634.38617556672, 51307.874624939745, 38883.11011756976, 5309.570678819698, 66233.13556940973, 63525.95080004075, 76744.89077695162, 62897.16757589258, 30920.452539046084, 66289.54922134134, 18459.305528889225, 15839.359464254954, 91382.05668654064, 24349.96187920778, 37261.367179508845, 57248.204347075036, 79289.04161817447, 34262.727615664226, 42231.02060867088, 4108.433360174568, 14313.186066587425, 30056.08666733168, 62054.614091033574, 34168.57858851836, 40025.058407410455, 22426.143767804042, 11106.821200548644, 27068.05478792321, 15343.387100316762, 16684.20065789421, 96823.89805031229, 66981.20715832774, 51483.32433963602, 19389.603224876162, 55173.90502520201, 15243.913443925994, 59143.14876524758, 67356.19286791653, 19516.60008254993, 36054.63766498751, 17429.900242121043, 99238.07606812063, 77303.85279263149, 20876.60196190212, 74661.59579420635, 8501.883041417368, 19724.474195417974, 57500.053570098484, 34233.10357535012, 91874.02683846808, 96191.87463764813, 60200.265231402336]} +{"id": 1693, "vector": [95914.39017213255, 63816.73599513496, 96037.17527661579, 53985.36784204613, 95453.1580442509, 3967.919539917131, 64731.23393121274, 98181.82472301702, 98062.40044231464, 5362.265018141598, 20023.712227191103, 47289.00741526942, 71702.31195422953, 66624.96986240587, 5421.2167880184015, 18984.468873933693, 59253.73313360782, 62295.832649303695, 63626.27610725126, 68707.68656076537, 56637.69729174583, 39556.85237031895, 59480.96969234434, 1700.3684842968437, 59964.7931352703, 1323.5456075645357, 3980.7963304145487, 20348.910135057817, 7029.053422559717, 83629.97884838484, 38914.034290382115, 12023.029681636632, 15463.726441219782, 32800.33249465427, 57823.67363391993, 85844.70598111588, 39428.74605524472, 78391.15902447278, 25427.262192996204, 3644.8275791358897, 79978.22671697554, 62885.901497605315, 27715.30850557292, 45225.19603251611, 7926.046227778749, 75393.34198886361, 67457.11006859422, 31109.056954657855, 72014.20765581635, 53937.1540074878, 87359.04893545625, 217.70397931595298, 36854.448700831344, 97277.34694407487, 24004.31755352689, 69202.84709713631, 17164.44563750077, 17648.41118940099, 86169.00000465487, 57970.09373290419, 93221.82077385862, 87930.19500104572, 30054.90444184573, 87382.26551180573, 17950.007623618047, 29456.71718100502, 4363.206529510378, 71617.90730840381, 38492.90613703383, 60901.167483485675, 39904.29532969719, 46332.68452738909, 19361.220709376437, 32433.86241681093, 16526.32000181332, 1602.3594736108703, 30252.902663868063, 19292.266264575275, 3631.4755490830853, 61820.6350500456, 87402.78336429468, 41391.92525651219, 33932.8083679463, 68555.93952872811, 19940.505996953685, 51373.72438221941, 86331.81708235321, 79654.34478524364, 65004.342381690214, 29003.396257950142, 36738.82316760747, 63755.09378026636, 83006.62840458647, 72145.57187768922, 62848.91078194055, 81309.74072026326, 80948.44218764956, 9125.237584570534, 63484.9525332784, 22040.452890922603, 22142.19350477189, 88041.5056259879, 36114.177012203974, 13612.828104576412, 84625.54147560809, 10570.030741775094, 2387.2765240886883, 59680.672924248065, 32427.26571596509, 45825.96028779279, 26937.38400398439, 31925.83136211318, 89095.92816061861, 11181.460073266659, 97861.13477798724, 24283.34858619854, 75887.14363885474, 28443.235938749876, 70932.68746382678, 14486.531449232387, 61704.595738448894, 63920.53671301265, 77993.17880294497, 12039.786654250684, 36377.97172991985, 57851.21959003815, 46531.125804432326, 34142.16533158085]} +{"id": 40, "vector": [30642.145100671427, 91141.09579900831, 56972.80265319516, 83268.23684750882, 68701.60018094484, 23145.976449845562, 66747.92558807807, 8358.5228248232, 67024.52950817501, 88495.82313358912, 11020.190922865802, 11986.646061133588, 16555.664602612007, 99413.88981953161, 93838.89834619695, 76112.0850194553, 40997.46653010928, 75956.15110659433, 13240.617259825427, 98170.73048775908, 27656.235566190957, 94460.15539224871, 48357.744099608724, 44372.30361493877, 30334.520534849464, 12046.87335302772, 99701.97958856632, 50916.33963942237, 31970.906453663796, 78024.53995372046, 57395.250703409736, 46659.26946398826, 61287.998648531975, 86108.91706157837, 15428.92195776001, 17493.166220121315, 38907.36843545378, 33033.74535234057, 55220.99780499783, 158.05892727647208, 17609.279367758747, 84652.15405630259, 5533.738745695749, 5292.869510973131, 69439.17365585903, 91639.84685669756, 86574.0936697916, 90198.21858438628, 29538.647362821168, 70739.82862054741, 77760.49656846155, 78217.23430821969, 95388.85232971038, 89707.9589453983, 71948.90210947765, 45485.20315516269, 71619.21009622587, 16699.00050459954, 8090.8286994544105, 917.6791329412848, 39890.965129806966, 94822.6549107008, 52121.49051296383, 93775.02927389165, 34541.788982394486, 13332.582739197574, 21196.454555079436, 59390.40262193317, 50544.018001698874, 34207.845312649086, 76243.84240931747, 63969.0821972412, 50430.692664981034, 32480.329315523824, 62631.18405919431, 79702.8711728206, 70662.9107039869, 7044.774001653753, 10847.235421359148, 77706.37515931588, 95229.64254986298, 11458.521332963834, 27605.779285797693, 26823.26166159711, 22468.484291917266, 65319.10946931475, 76937.2441378189, 62550.43280754565, 27246.88392857706, 93729.09345908371, 858.8288293133961, 5044.214593929974, 96293.96987445412, 22395.32017073779, 93475.25020451134, 81047.6777930968, 50682.789837978446, 80418.01072023672, 41991.819515554555, 24731.504536997563, 19457.966557728367, 74422.51400829449, 79584.53267090663, 83224.6050335603, 51488.39507996301, 91087.32030346441, 41470.84864742157, 27929.438706494537, 75432.29775434766, 10995.275780811331, 69269.83229695621, 60132.37825974287, 87035.624920994, 16689.008493175515, 15509.1467736163, 87379.43097157165, 85746.5780058049, 51730.8540895256, 13702.321369367088, 42490.93117675545, 61294.57274666316, 24470.68182821448, 16251.648689154696, 61504.14818507913, 56316.66413024515, 43354.60665011328, 76012.29861568984, 99723.60488950259]} +{"id": 1986, "vector": [83721.40694785625, 72436.90956421921, 38410.9734715114, 74846.04836969938, 54884.6926072389, 71400.25486507535, 52770.98896380147, 28713.88518294692, 55462.21310964592, 190.34924246656982, 3907.831992995725, 30439.195615379354, 45492.05307583085, 77603.17644339231, 54108.096841871244, 93377.42209561486, 84211.83403611061, 5406.045062661802, 48909.51376636028, 52010.76184014964, 15213.460000317824, 54741.814916017625, 47502.326280615314, 61744.169097435784, 70698.39500210508, 37606.895980402085, 35962.7944694383, 90791.86215797778, 87627.62205330851, 84045.00055859002, 52707.56751255232, 29985.12826882699, 89202.18766308652, 36822.896818381836, 77526.2107607309, 22249.320015511286, 82760.12978473888, 65179.06831467748, 19305.34070721297, 86116.02049948453, 28591.695351808467, 77843.03458894676, 87372.74601364881, 82241.43774511144, 85856.48592230253, 84234.37631300479, 83366.79734421024, 76067.27482743046, 36142.93856457539, 11905.514815513752, 94667.23263183064, 19655.861397256867, 60987.88939555164, 67309.95266636915, 86961.25113264225, 88092.56128847989, 55097.764462722786, 48001.07467561172, 37228.10808997207, 33812.29826642796, 19294.837708652067, 76443.62612939115, 45336.334348346005, 3513.841011751062, 21699.923453318315, 47017.178520588466, 890.2856023958128, 94323.76305135369, 1451.53683485989, 64175.00414474144, 16952.901963192002, 16103.570252781728, 39607.922387738094, 96715.45356801424, 75.18598236861962, 25660.537776204295, 77504.36269069057, 19393.9846767337, 95631.8295150105, 25908.293216449063, 20160.88822314047, 45345.50652547448, 96721.96549230361, 98500.61783335828, 68188.46268010134, 35556.501254073155, 26418.695582028307, 63526.30690708008, 47405.01910311856, 99935.80978474871, 70736.51124022715, 20955.948395435153, 57221.77515609783, 65279.966204962446, 43686.265587264454, 33067.06266312444, 59076.72772829607, 82476.89886462358, 72713.75240009007, 46664.634547526395, 11535.231887439213, 71594.3760891432, 69774.86580323578, 44409.79189932679, 27115.638363747963, 78420.28142767548, 40050.59644986991, 34429.977242388646, 70535.13285354008, 19845.447365020496, 12761.117126639321, 23790.335649300476, 90245.9762947766, 3503.4126033623215, 9967.004217244712, 51147.12493905669, 56675.76791329083, 90798.07452806026, 79669.96574627142, 29017.304685579536, 90965.4526722287, 31766.37805921506, 76244.30646048147, 24105.994504765116, 56550.38485023483, 2351.1940832967703, 85313.73303359142, 14053.832090783691]} +{"id": 614, "vector": [45204.38710101863, 33646.67537316112, 59956.7026017879, 80703.98028724447, 80737.59841503261, 14077.96511089744, 33887.66547723392, 97032.56086913896, 90835.99066798059, 82307.88383593339, 33229.76758514745, 61163.61120165397, 36061.30128145559, 63998.6203033187, 55489.234410163735, 37018.45337642798, 15932.771554841453, 50845.81970185038, 3473.235943211261, 88172.57865244921, 2024.6239820630562, 8683.594805138595, 14680.00964940327, 29175.700872274258, 17499.682646662306, 26380.53106828927, 51591.85927318631, 79825.70198029981, 32460.29086346289, 79513.83574336344, 96118.83898994546, 33250.87802464993, 19443.967385131124, 62611.12819162933, 61391.025081932785, 61065.91854341057, 51724.7490370161, 95823.01740222417, 5899.350442068929, 18338.941733766846, 83916.376072358, 31100.692244326434, 3941.157840576759, 72497.43423241995, 7760.0218247417915, 11019.511650905755, 30812.941646475832, 87528.34596907794, 91195.8143187862, 31695.691589310492, 53036.71912652369, 8743.536724800006, 7603.786297547133, 92359.11715052334, 27257.104890728846, 55839.5583099351, 11987.271095521468, 70695.60378718733, 68487.36875282635, 20499.91645507677, 41535.28773826617, 33690.690207445296, 60658.6339177528, 52325.929867603874, 3930.175292255189, 47881.37131043374, 62468.07631142413, 42909.200827193, 10328.622585821857, 48963.37194571921, 98273.60638885482, 90461.52071653698, 7284.4963744553515, 82363.42034798348, 21923.537724525522, 9330.480671382968, 26671.400456096984, 33003.31884473179, 53022.13512389566, 33139.10131364073, 48291.04972104005, 89343.66768869781, 42837.009832310636, 93245.07277931913, 35307.707924788345, 64546.83094775182, 37081.35144675055, 69080.60625125347, 81116.76189438831, 98988.83043377826, 15755.914858027343, 7819.967178956411, 86421.54720624717, 58953.46517590208, 74649.31056724094, 11848.523758404672, 74113.32985855172, 57915.29400254649, 45147.14323822684, 17284.01362580476, 36908.452436410465, 67223.79850370996, 73497.29993439236, 46960.83271969362, 58304.78504251405, 82425.18283711844, 86819.31768684763, 4341.828146035864, 42027.64139916918, 36236.443596519355, 68939.85672730739, 81713.37636024371, 44085.179557339834, 545.9897114425183, 75626.45121359546, 4883.6554144568, 65085.10599214915, 85313.91803991118, 66183.59975983972, 74852.95428564274, 84508.10832198098, 13834.664869268332, 91093.52969867954, 49682.740222741675, 33073.348612156115, 16394.56607147676, 22592.741827021233, 96186.48744758949]} +{"id": 696, "vector": [99274.6417264243, 36483.021173026056, 45876.64274311104, 67422.18621936755, 36002.17387230971, 50592.270367992576, 17764.86288604574, 59682.20479360725, 82058.65453191413, 99660.39141308337, 95826.64262066767, 72259.34524969658, 43001.33225697397, 61487.312250996154, 31807.572977652577, 26299.547571766645, 29541.282248944957, 12949.141136492914, 13987.4899053964, 84096.01824566082, 22509.638886074503, 26438.3851389311, 38858.129317614184, 60116.137716282734, 98620.60930592108, 58383.381580475965, 67663.38071145197, 10810.436153754532, 23502.509939723626, 79842.3124641431, 2480.652622278756, 52656.45478575093, 31496.751877261388, 6164.426271155421, 74510.26738875161, 92060.26682193097, 54265.39123536819, 48268.07251197343, 1908.2505137974936, 29844.427347593737, 81703.24416442147, 53579.37855301212, 14415.883249054317, 15849.78392097084, 7556.384789506498, 83306.08642461759, 64900.99721983074, 98756.94118527866, 30399.094115331125, 88807.31881190294, 23653.491281902363, 57746.34246507545, 42137.488985408745, 65861.22777944281, 79672.01330133442, 81243.88556820346, 61633.022218531405, 8011.434707486653, 68108.40731187949, 38228.1928789354, 80950.11064771423, 25590.525237567952, 99399.02837273915, 22244.686751112862, 66438.28108357222, 91782.72281301324, 37868.64300230985, 28873.76857073608, 7210.293884160601, 11060.478608969415, 18810.92242138358, 5466.656514944679, 39887.28032194081, 89683.63968033927, 77232.46667930466, 20132.320317656504, 18216.90308791243, 52521.19502860158, 99526.30122736948, 52939.509440625465, 96458.04019934131, 55448.17893011531, 20592.667018709475, 45407.94642407333, 3106.9279195853337, 2867.5158703789116, 6256.469550379429, 9421.804720069316, 23701.15163023472, 21118.349465627918, 16842.63087413784, 80547.78906529996, 46053.93501165556, 8801.101228143592, 53387.434447801454, 86358.29257838691, 63501.30561617918, 80015.44166723189, 21931.046773780494, 54676.70220143445, 39642.51416999465, 68643.95375023944, 22754.932467246337, 16083.986909916093, 5104.990701969825, 95503.02743373785, 53782.41280151972, 41225.63641218924, 16723.78311270294, 80899.817486354, 83904.0237543612, 26703.954067825096, 39076.95463326857, 30398.804928130685, 64641.80034122717, 2659.081374848549, 84343.39748345016, 56361.83092259671, 81494.68408660863, 62818.109606666105, 36887.71869739482, 66304.3892509202, 88941.76874992899, 13188.47933202073, 74911.22651815852, 30482.30650661469, 67631.46638375755, 98209.96136673611]} +{"id": 472, "vector": [99371.95741911468, 91236.76989522966, 71649.85085154686, 2599.0461540259635, 59721.70948073382, 61872.80919487908, 72346.70406077437, 28986.111781136104, 4488.95691279555, 73379.38463078845, 92498.4263371847, 95144.00039776305, 33163.229859954234, 87321.3370461832, 27942.730156320828, 13920.645421453404, 21340.039676678312, 3233.8331562931535, 10858.063874268653, 11640.904808000963, 82683.14558521046, 59902.42286339248, 76325.55276751431, 68133.80847880473, 45005.6946829426, 58857.34219181936, 39136.89406786745, 77271.80787548934, 57059.73417097517, 76491.96751854867, 73937.70818733516, 39842.37285820885, 17649.977092599245, 54443.7916253233, 42348.252607594586, 25760.776453541344, 49131.721737992564, 66932.47608878605, 50047.12384724743, 28186.15884384028, 40263.32545688929, 4580.407943347997, 13700.770789934902, 33004.79359693293, 20568.448272045014, 16492.00053613187, 6613.014594868105, 82059.49486229684, 9963.606362404787, 47919.80591204461, 79286.99013298165, 16841.47242236592, 92239.84480512938, 25594.405391715358, 87535.36841603382, 70171.40474818034, 10944.642269735194, 62442.31538381964, 32274.387085329272, 75622.35446823479, 29767.44726167335, 32093.326763143326, 63835.77287072051, 28366.60751858866, 95842.83961956657, 89373.31804432857, 78815.88917788026, 50670.53350882944, 72694.02038823852, 62833.799451505714, 63011.07698930441, 70353.90850730195, 23126.347400370272, 81008.86803764284, 21174.069239465774, 70913.47027531496, 19316.106105448405, 40548.54800689306, 83895.28313962868, 55509.61428029505, 59734.87681445182, 63776.65366064116, 3293.7382297272497, 8598.601174427711, 1054.6320791873454, 82554.153642842, 28943.75562919741, 77768.91113755037, 34968.92455138181, 55869.66422546625, 56987.27960513982, 58109.41475859514, 18361.45890094406, 40961.15437737902, 6795.408376877898, 95933.73964945471, 10722.289719582977, 46887.93272713266, 38491.02767538054, 43021.824715671966, 56779.1870203959, 86863.82533465198, 30770.18398861784, 69021.44017585352, 46473.66623076451, 39077.90941321737, 82109.50024051276, 46180.567818668584, 28329.91675337162, 17524.715329573683, 54187.24384449546, 3447.3057668230613, 39767.01472233645, 25160.78105390518, 3907.192478386301, 40344.112254174535, 75482.62562882659, 95202.74467034041, 20519.509763338017, 20246.09661906901, 2489.158513262257, 11092.891244332093, 26594.025809281207, 24730.03151673211, 40356.43810345046, 64677.897731438046, 87925.31872380803, 36913.817905247415]} +{"id": 473, "vector": [82415.77458526344, 10300.319388125656, 23948.019481332107, 70967.05740621054, 53205.520307359046, 58591.40110763852, 87553.73705960323, 34478.31565983063, 73772.53632933328, 16464.904387011313, 85894.52453388544, 49497.418234872035, 78964.27032272228, 20474.205451316764, 32213.474539983832, 8139.438506677543, 40654.83471484111, 23204.720029770386, 49324.68373941143, 72722.51237825224, 58778.6293349245, 94462.38568453326, 3426.2739532114783, 12733.069577205792, 80859.19252756242, 5995.192731374111, 34852.16766901905, 47446.16512302242, 2139.0125050767538, 95963.76516863474, 87963.51797190259, 67036.38948688918, 86328.46994165293, 29392.756552841194, 58664.5713039338, 77820.05195503082, 7458.163140249907, 14520.80242656345, 52456.38652208777, 54540.832536819595, 7072.276823876533, 79043.63301117429, 693.4465145900015, 62199.87789541108, 18193.53978003463, 78827.40878421628, 40610.870387461604, 97620.1604292276, 57150.49193565449, 80894.5777100147, 86908.29609188219, 89555.022311497, 44790.47142362733, 16075.552459057973, 78511.96366588214, 37870.665609130796, 66471.09925201103, 33641.58040022907, 55351.38182236948, 11713.237638398832, 2918.6228604552666, 19992.973230877964, 30907.87058136323, 37577.51468065098, 9402.345686221102, 43331.87235480398, 81270.95305407426, 29833.09918161746, 44366.79568449132, 91606.12091536917, 20156.973440602455, 99277.93660571736, 91083.68409817938, 48589.38139071566, 99281.02652820271, 53573.049186377444, 43003.96216758534, 35252.866994695556, 54163.519327451635, 70167.53760466453, 9199.994418890166, 99804.98436116187, 64465.02793992058, 81164.36021189798, 22423.661290241882, 18314.59156948403, 62011.4535534669, 7475.96515123623, 24729.265342510032, 85373.37370260182, 16024.769643913594, 45881.90971802907, 64324.22964383596, 99602.17488853498, 30956.854063206916, 33395.08559897709, 68492.23219221829, 35671.31162770402, 54564.794935593374, 71323.5140959951, 39663.03940677431, 27178.10671519143, 6784.02952281334, 64157.73167860408, 5308.62144880353, 41515.59690056217, 20870.755759691838, 50393.59425262853, 30145.104211037666, 191.48183939182718, 13085.966372353409, 92419.08144566296, 92964.48113185685, 39355.23556630558, 92736.6266263817, 10913.382332429555, 73720.70073332507, 84929.15805625239, 63764.88158884759, 74944.09142920014, 92869.1946866423, 10039.153962341707, 46975.27305049687, 97740.8704081005, 98901.72831722381, 12836.63833464036, 19062.149228635906, 22173.75910957132]} +{"id": 953, "vector": [77428.8166464561, 40057.95033640705, 84434.64207702561, 58600.1022086038, 35911.208618644974, 23715.337670402314, 11976.133191230765, 31740.34182537915, 14544.234616795437, 53281.18115002781, 46153.41833536216, 6319.963018610886, 47389.94626443117, 14967.706564273141, 30230.045955117625, 35509.204973022046, 14668.873065202948, 10250.98852751576, 90299.62823859353, 82177.2614187722, 70427.1980490375, 34363.213625051256, 99986.2567232488, 45015.45717350094, 12280.782389328659, 19220.72425702347, 32096.537837019412, 41846.12647413771, 61420.85496093932, 24132.70623415925, 37772.28660809874, 38402.71169915755, 77250.46888355994, 38951.28030644911, 83922.96991342911, 50514.28197088756, 33235.722348176314, 46870.89972630553, 78001.68343222378, 30503.160955956733, 98615.21919400648, 3902.650784518025, 96471.29922822076, 59373.675225170875, 89192.93367011767, 2458.6931371221585, 68007.5154717109, 38148.53034606499, 6435.364164719582, 38828.98740389025, 34029.67704794009, 44657.052458799226, 28196.148976161283, 44114.00219218431, 27718.08475415859, 30952.271063061908, 41403.67167695739, 54928.47315961239, 34225.36576178995, 1411.2540712451937, 12193.844710980895, 31924.913728987693, 36027.36511525526, 83455.7593712, 43026.54735840073, 27516.977237851283, 79038.50702009475, 93515.6492469835, 84126.84604050509, 11120.256507144422, 96387.26578763417, 76672.8488800914, 74229.65277957775, 19441.94456678664, 87250.17162348748, 8880.282099571934, 56832.66520064656, 24252.271717487005, 49917.69439643643, 65216.2258544842, 48323.74565436912, 5138.237775467891, 41770.82563475444, 26762.577004023435, 28535.90553667682, 29574.935318219188, 25328.29803853357, 32756.836193481842, 65830.15126837525, 24240.273352365493, 90508.06969973232, 64318.53340103837, 28661.688428571942, 22595.974526692466, 62961.79089122661, 59011.32274919634, 11287.86558249686, 35893.13587002437, 84950.25787117262, 29541.983624888657, 77082.2446959094, 42455.53423839732, 30658.584081669072, 42532.65888598342, 34371.208815897146, 25702.034711795586, 11465.653495882732, 45767.776262901025, 66772.38420420153, 10950.083874480542, 36293.02424469074, 95519.30417275736, 39913.68290398507, 11673.63739349, 25960.579191521403, 14269.28685114468, 71066.03251420916, 19800.606376451447, 28836.89415213527, 9900.423460678609, 42159.87956310058, 57455.964926584216, 31528.67987229153, 86450.41266577688, 24040.73197598825, 68395.63721887203, 16053.85149536792, 21145.89206116576]} +{"id": 890, "vector": [4096.445899333124, 10589.155495883851, 4789.178596668342, 14697.18422949563, 55743.85169286934, 85402.32082312566, 35856.43770066144, 63740.66726414814, 29321.66497387204, 19944.326024706395, 8855.715219807715, 76206.29255221871, 9763.926544817514, 17745.796135541725, 73420.14229668316, 91605.43422381, 23727.114397007666, 24077.880410475318, 43826.25738113569, 89037.1645816303, 84360.10469506863, 20914.616572965315, 25610.454776362923, 24021.001227696546, 93567.59381820266, 88888.39122062194, 9470.539679005286, 82010.06121177108, 3108.0476002380287, 57531.21461667381, 66719.66373824794, 87610.64371181995, 18489.873496958586, 28040.37030367489, 55736.465400293455, 61228.63036249354, 88548.3859967614, 65045.11890370373, 11272.367409795781, 31160.85006424405, 83714.21508053697, 15091.921022939403, 46264.326887805255, 96845.88367093031, 86726.41807124754, 72526.71988411581, 73545.06314136177, 71340.58337048469, 628.5954600789067, 3475.451838532884, 36204.44324648499, 91042.71320938948, 59700.83930031639, 12028.683518498228, 76730.74259194772, 57555.109810033886, 95447.78375703166, 40728.10989289589, 6898.986367917248, 92585.58310890199, 8330.914238114761, 53579.6487212099, 22280.027072717156, 52175.18107863276, 38063.725886273336, 98503.31602837327, 42994.33740032392, 72606.42634671305, 94039.86333706, 54849.820223981114, 88587.45824430732, 18390.501192920816, 85767.87963718091, 9709.992250665233, 21366.854207113895, 30856.000073367995, 77271.13911440318, 39342.63368341332, 55505.60054134742, 89325.5595870578, 99723.10547756808, 81028.84338314475, 8583.363983547542, 37192.16315599561, 83551.15616821637, 49300.021533793384, 90320.7315395606, 92622.72731433914, 29750.142407086656, 18223.875957136992, 29296.721605946143, 21288.158451578387, 17563.229859344498, 71554.65612529755, 36259.41106691635, 66262.1037866398, 35470.39160331207, 15766.79943183613, 37231.31383509183, 78509.89781970899, 79087.87200521203, 29569.48077993059, 64288.81243416326, 39244.26424071196, 35033.792420124686, 86452.14491862805, 71928.59701878195, 70582.48862785751, 49267.56390349582, 7615.817873635122, 68524.2664128648, 26734.33942275947, 84022.3822099119, 8350.962557749897, 60936.50950321696, 27697.569415227652, 4681.45194042433, 2532.7770242461247, 83639.10494281218, 36321.90690129245, 23427.2426590309, 39397.30677268035, 28502.647831760896, 1697.6326287377885, 11723.240338005848, 23074.4821969208, 31536.705397018395, 35384.33165822345]} +{"id": 1259, "vector": [10351.31604517937, 36181.366534278946, 41775.55892333138, 26639.733480763727, 32651.896983251627, 95284.67740490819, 7893.018654517936, 9457.39986511639, 67650.20826980376, 30323.022179255975, 37892.45542032407, 60811.7831132928, 92553.71911394867, 93017.67392920748, 53335.95685677991, 81910.34862374583, 63154.78379525272, 23901.16811850119, 5724.646764894781, 12461.072509698955, 85745.06652045966, 88015.69839602901, 91380.92430044716, 24324.796649126678, 2496.6396636741383, 79328.24347512583, 79747.39912812518, 40342.12665496898, 87959.65300687701, 79861.35201605882, 63299.28618474553, 86747.32078350996, 76229.16725992633, 20061.14940709227, 49894.7347771685, 89649.37062645626, 54521.61062345443, 77946.18301245794, 32890.292292069804, 9621.09663079348, 99593.19777436624, 92628.8365621545, 75877.38975383648, 86786.97756002273, 96688.46887868393, 91748.18939173232, 5006.130712356138, 98958.9049938207, 533.6789188592927, 59201.43787789393, 6019.7281620683825, 18094.47245954393, 667.3537789526063, 13290.97190133488, 95709.00721496326, 43482.826238863956, 28724.168191730216, 20436.317622846047, 74439.26869388367, 31268.1778284303, 33872.30439910873, 23322.365435139756, 79272.92552676176, 51347.71171818664, 16161.451647138758, 35220.15206349639, 90252.72926153976, 89103.9279237912, 78062.13205235796, 42223.93888171011, 67580.01985473554, 20472.76044635997, 70962.61513994286, 71763.90229536002, 53074.632517269856, 34041.81229141562, 23276.36290140347, 55786.375441039934, 30982.940784046354, 83903.73791485476, 77097.9326702363, 9187.332080467004, 83175.64393338934, 14138.975051282687, 46111.791101126175, 36371.54019353471, 33282.47068242005, 75199.40538050722, 70413.24283280064, 42802.69167110762, 36627.56174808237, 27545.472954989014, 87679.93167126841, 94168.92340840476, 98663.77107948967, 23275.015486653116, 11550.423248180452, 66952.64475278239, 35524.76597846386, 27802.520117566222, 32778.154336641244, 86535.72986281393, 31311.737129715133, 75843.78741470598, 31350.188807404535, 52023.13395925327, 68717.95859493062, 93029.00025985467, 78594.53837829082, 7109.926017390789, 78029.18658803219, 8344.244412054048, 44791.17356119081, 15320.902617100406, 25080.102703965225, 99106.48243140354, 66070.8472140506, 95479.97796055957, 66298.60244166815, 74414.47109967675, 51839.0943819382, 31080.954080994485, 66953.51884034273, 16555.350900296296, 35036.08285605587, 5558.26047233241, 80453.27081863282, 18156.29730614412]} +{"id": 1757, "vector": [96779.53864192034, 23107.43499476544, 26049.174672512974, 88185.0917875635, 53889.17034213373, 99089.19978071048, 28061.662066660563, 44921.66121072416, 6497.790944727688, 79707.91003374275, 47919.3270934154, 87817.17206814961, 52590.3988979576, 64595.42107204684, 16274.304477732792, 17472.31169548754, 28424.001920900842, 75931.47240122872, 38981.773734705195, 3637.710131337435, 69498.42232246598, 94281.98830898402, 80414.75576710545, 1656.1173812325912, 22778.150607771564, 43711.53416893668, 51706.03345597217, 51763.93018586203, 39759.24091858101, 64617.70986033343, 16812.12380523933, 54613.390146757076, 93404.94796639531, 33335.67646456973, 49983.41240841705, 18017.572406169667, 49958.340027085134, 67539.08331369511, 26550.31647759203, 96415.8952314451, 80131.84623399474, 68389.32680160938, 87429.7009021667, 28705.932381732502, 48963.92583443267, 54137.21172719902, 11806.188097695924, 45288.02012237882, 16106.794880140895, 3851.663776753023, 50544.33474540814, 2555.9367297277367, 59095.62607842963, 87899.98538805851, 42008.73594903425, 38127.893348483565, 60929.279778436285, 91776.87609218067, 33194.2183091693, 22351.625100947138, 55347.35690137724, 36170.38020660218, 63661.518584402176, 71455.7499418169, 88603.40841479985, 37906.76388476266, 92258.89915346213, 34829.9542761789, 97305.96430045289, 71131.04569509781, 45135.809467896535, 7388.584791838626, 3.1603347251607516, 55687.02598835077, 84783.1994945541, 92866.18883619104, 67194.82391163286, 93836.9208035577, 27705.09839738582, 7347.891148519392, 93505.3390032274, 63596.05830617731, 5157.421876554246, 131.06962629844122, 71455.85586341038, 2754.684840910615, 96384.68914384086, 78970.41464837102, 85532.5064667309, 40791.75560808822, 96272.24376519685, 5158.793162069553, 14171.007384597046, 51180.04923445028, 99724.48925706596, 1467.300794356774, 8172.753655079235, 19078.961376599946, 17161.95179260319, 98810.06006212898, 52239.5652374157, 44899.29570057085, 93953.90080408978, 59479.182870934346, 10096.460883103908, 83831.02760627429, 70889.24145169837, 47502.0087449531, 81362.6593035139, 27381.28780194439, 59477.050378942346, 1132.8646077705007, 27809.728999486983, 89910.88446980166, 13235.183705577925, 35825.85279994591, 29297.38243147284, 94157.83189744713, 75381.66140843343, 71354.37784915803, 56615.41584113289, 88756.20690387688, 69929.04019657552, 65105.07925479906, 90625.70716446642, 86788.959124768, 88986.41870608537, 2631.7093935842026]} +{"id": 1375, "vector": [36386.646878831474, 51886.0260521203, 6395.3556809367765, 9142.616896394296, 88254.61433183764, 70372.72281096218, 84440.85806337217, 74732.89945340056, 20020.581041020792, 11720.024818495422, 16446.316894543445, 26337.30891921201, 89224.64120539234, 64134.52168597241, 34862.84827137435, 20136.48588744441, 88328.91601165356, 16356.719936255626, 70174.98816350271, 13271.994181161428, 70499.85204191465, 52187.144971695896, 4774.151220971013, 32845.50629350843, 90000.354379164, 38443.26673782913, 19014.347703112388, 13387.988005231222, 70149.27710637728, 25290.020342702668, 89882.45646058096, 38559.86620280812, 96289.30378315006, 65453.84158224461, 16793.682757396968, 84431.50827402656, 16887.85609029153, 63822.91640932652, 88335.05432325546, 73680.29261453975, 68212.51356162883, 47906.9745295027, 82423.3266780868, 33115.89059056016, 36124.76171921938, 60457.307286306816, 68777.00967664963, 71211.23521351564, 24031.054042434807, 50219.71252746539, 21069.084490659585, 99506.73478004661, 71529.52377637554, 54016.38380564119, 56093.31621256247, 9968.770467365674, 24821.97344753233, 40501.138052653914, 17325.55070137285, 19609.362857222535, 30450.326381411287, 1705.2345218395938, 3516.0190939667714, 74333.15296611185, 52212.163360607534, 15603.327565664782, 52981.27662103673, 93467.5938886448, 61324.23527445239, 62940.34158950451, 78031.66769050669, 13765.771555015182, 59778.71622140655, 56832.45112422878, 94510.87467725042, 10324.143104476092, 51544.107708553114, 9371.668266782208, 42629.54755369129, 28189.47675591238, 13374.927023527449, 56760.91026219513, 51226.29896655756, 27424.59722531776, 70093.34373310686, 70881.15093524718, 3531.736712129474, 22967.15994962596, 94467.35481350098, 6020.934176155435, 58146.61426214345, 61155.945246256306, 44159.55026798502, 24800.250238919376, 45524.274817752375, 68794.0594445727, 42877.584040480346, 87377.42525460594, 77686.11267990568, 61781.11476583813, 62770.5274987615, 92838.01113851985, 57615.168455352425, 6422.892606326381, 39300.09344283122, 70762.17897496864, 93293.24095818869, 33300.28672891417, 59812.3511943094, 85144.71519909082, 32113.6497636375, 62651.56918179973, 13582.865141073375, 88230.38032956349, 92216.20356355111, 33282.90116476089, 13084.034898557384, 11893.808380429582, 59048.43335767869, 97469.46718892586, 59018.15460747175, 25521.836722309854, 95106.54546495518, 65660.15915534155, 57932.85862022713, 49366.50589812831, 90284.8300933526, 73153.30450996601]} +{"id": 1358, "vector": [82239.97743123995, 9989.703376814417, 89297.18938165739, 67154.39791559498, 93074.63945024536, 17839.924619818637, 48939.342348624225, 74613.27226577517, 47115.40865857384, 99766.55507096308, 92134.48585858564, 43125.4442324157, 88324.89959163625, 18332.08458051684, 97186.18098972454, 18456.07383178698, 58876.17136027714, 64669.845277706794, 38836.2339316791, 3092.7961213168433, 80046.55042147033, 83402.90175767345, 30612.13840356558, 28772.889179123395, 7392.464332016413, 93916.95416086374, 87915.47042497192, 46864.657167068646, 1663.624803340691, 52933.28179573348, 48303.64697661449, 91825.04797208084, 12022.739666302006, 40127.143090991325, 80337.49955848143, 9262.8443253303, 35502.193731635045, 71855.67084676733, 79306.40122831328, 7460.143999993318, 43334.531695042744, 213.38078692911822, 29830.318290040457, 52068.30830932483, 80763.04241759935, 37578.71404819965, 38879.08504058278, 59142.13875406138, 33137.06941774088, 8996.782322387253, 77623.30610254114, 49292.15089762318, 49831.47493781709, 82113.77410390822, 13950.691236404322, 81537.71536405072, 85986.62920616733, 69283.17370355049, 33089.078021149064, 90913.38860766731, 78848.29407471765, 13593.46517715525, 3379.997988688166, 81163.32650650191, 26821.81626726531, 72344.61498324873, 40595.419890907746, 54224.6411280866, 20400.79579680295, 45406.350468250064, 89600.01889681509, 41110.76138727184, 4471.951404745245, 64937.10651486082, 58566.89094533921, 89758.71115079499, 44970.98501488659, 5956.225804435533, 88679.81936675821, 66142.61586899431, 23877.68359347967, 27224.49857347824, 11177.478721650492, 18656.569510781228, 38115.99858921133, 41192.39133050708, 78616.91190336103, 93463.98581701115, 74409.26999375134, 44804.11626792474, 29983.842870391432, 18035.857284921265, 39449.562580290876, 57278.95018280792, 15986.780753539399, 68911.03424065009, 60656.143489821225, 50639.14996084548, 96046.04384380803, 5748.212173722367, 93338.77321177004, 7456.841681202742, 43647.61139138052, 62573.542623991954, 41542.61308592713, 76205.44672905313, 41151.12474078857, 89776.70807193115, 84862.9670954105, 57220.76959313634, 17048.230930790498, 57299.012542179065, 69983.37175700569, 37009.46572303195, 9117.680658161831, 4318.405621574706, 14068.00502723472, 52163.79421789096, 91629.19586028667, 90476.999859685, 40484.646860131616, 70319.45188098897, 25773.649272150113, 89755.76668276305, 29893.533922021943, 24419.196667160002, 59168.22343734896, 89604.8307578909]} +{"id": 180, "vector": [12057.760533631934, 54378.79717926605, 24978.273196535138, 41008.135038901084, 50122.33906888449, 2903.6763054752337, 84710.74341316002, 33729.897435223174, 97039.5306254621, 87670.04451418137, 77739.7774371345, 36840.88992684888, 54604.03415444653, 21368.54223382597, 97570.15507540348, 43173.749045826415, 31827.244925191844, 67005.35724303147, 52121.526486482406, 24793.83553502488, 43379.9715548915, 19986.91299483466, 7194.844063095152, 56916.00228340415, 61892.85438811236, 15838.394752875905, 60242.29055294472, 58098.473360155855, 36625.34191818878, 62673.713216886186, 61846.514241044584, 36115.66537588148, 81171.51563281853, 28418.20362313625, 60585.9131133877, 73433.99340752163, 39401.03310021195, 80935.06780444813, 23538.247995730777, 10119.598307556455, 6733.8382314620085, 55935.01950555059, 91109.17783689096, 74036.77337366807, 83875.37085981599, 23220.92244849946, 62310.565759326564, 42766.30128057902, 27840.11854416074, 68774.33640543878, 31100.962488306173, 28933.341935060707, 76181.47807640306, 92143.7596679156, 84047.48497278047, 66605.2095459352, 10114.496940275141, 19212.98405858276, 14039.09984868068, 3632.0126430411824, 83854.62908327984, 44668.963185267865, 47878.36751680851, 52661.41240550347, 11126.081103856555, 23719.557126041214, 99283.73979198106, 16264.890224774297, 16535.955686465542, 41849.996439636176, 11458.465576289424, 77185.82316928665, 87519.55915013822, 65215.48686614902, 81891.55924739808, 4633.989514914183, 44312.34009289492, 12842.115085061878, 49453.864543246615, 56302.15516346909, 58247.1554800797, 9386.720662210524, 14014.637299095955, 13678.583843222159, 35303.72007715764, 83181.10403315796, 37811.70979494137, 9974.967188671535, 36073.13047313582, 97030.97673025935, 31952.83502412314, 62136.858950746355, 52927.08054848292, 57481.186452485344, 20474.631991468883, 2687.364822590399, 28023.74119846146, 31052.00480881316, 70681.84457592998, 55016.009441227776, 93548.76037937331, 45501.131138676195, 94374.37336242871, 36082.977253041674, 69189.82545853882, 76558.46538626996, 14141.18594754059, 76383.39830015684, 88876.74320808241, 29734.508833950156, 89250.21343823106, 85909.10072701379, 93066.59163337533, 67719.64822617239, 36862.83615714311, 88861.03434404306, 15418.747793050414, 55300.129499071714, 84467.72183279235, 20427.145435577888, 4579.709837139833, 65508.38222163754, 51251.63651467386, 46509.071234947805, 90632.92728457041, 75371.26499996865, 76977.78594856082, 84545.74755572726]} +{"id": 742, "vector": [22760.900592483657, 73249.51717604988, 34956.78703810567, 70652.75931766647, 34643.96714123171, 11743.327923981806, 31709.91639030972, 72626.20348505674, 69997.37672768769, 86397.3925707431, 58799.101329319936, 24345.611328454863, 73094.19042966237, 25582.88036446108, 90863.82581810773, 90405.99688667865, 69439.92753378008, 48781.2068161821, 40982.67350226931, 71710.20818675571, 56985.341178472714, 27046.489097236547, 57769.99701391312, 49563.59939556669, 25746.399615818304, 24811.162485574812, 1410.8368687296036, 67853.65252247933, 27349.270656762936, 27781.528963641988, 71517.75163654686, 21339.85419669793, 89664.41906510202, 70116.36512643837, 99650.1845224778, 84794.35960438513, 53324.69881032938, 46532.7881574877, 95698.20840926674, 10322.525985096208, 28318.094595504506, 59533.97777265297, 5756.621886993973, 34548.32312743434, 39365.69026211617, 71344.21789224967, 68997.96606638725, 39477.54826573866, 30256.349528410898, 19714.498105941024, 59882.5703960567, 17386.318037560333, 80264.46529056544, 49241.27072173621, 95674.35348790576, 47256.074596113074, 51058.556016367016, 84632.5286705935, 92824.44517057536, 28336.246104991413, 39771.800041834234, 30961.997459767408, 4168.255209577365, 33638.85937241462, 40804.89782557972, 68328.26993608008, 80777.83836943451, 88593.96972731379, 38881.978764474065, 1710.071720468076, 92957.24092470713, 28105.3532946573, 81347.81672409855, 97704.48595025051, 96431.69107967576, 54515.49418140383, 8567.959317829554, 17539.22100366584, 52741.851852281055, 40305.47531352425, 1622.430021501786, 34373.48387346312, 75899.14374278554, 54228.37293209347, 80219.96230732351, 52081.189319284946, 21943.171783467886, 85826.27880774127, 22115.843945361758, 76941.48036751947, 89120.53855462388, 28609.055509108526, 82150.57982744412, 37790.37417546538, 26136.358705098493, 6557.858935217587, 20753.007112452793, 6424.506266732455, 55736.87187291433, 60945.58014273782, 73261.79395487581, 38132.234925913275, 73751.77196680295, 68723.28911196522, 83602.19074181795, 48730.591372805684, 49835.32037199287, 96416.5640797904, 936.4477218316903, 24737.241220122563, 21932.05397117969, 16205.032991642398, 88234.08391438978, 62416.45171034401, 19289.688070469114, 8332.297962289325, 99788.59258304474, 29488.6547382894, 23354.14775996867, 47777.8977227099, 17016.624836466355, 2189.5732646592614, 17661.188358176638, 69406.50271939547, 6318.0616539274315, 25472.52977556561, 14943.281162294319, 61427.81038515697]} +{"id": 2031, "vector": [39827.933779937404, 15978.78509298747, 54325.06405185241, 88156.29152696759, 2498.6869000913516, 38575.128742269684, 8791.754371861649, 51078.60365695341, 69912.93689775949, 80971.04916995172, 47233.07066481085, 28555.12939693091, 99139.76349532114, 87465.51868353103, 44478.27427732369, 50057.9095081941, 707.907510459993, 7279.099153514779, 84489.84576785992, 26651.38020038407, 81437.56425986908, 31269.56641659088, 60663.848426680466, 92261.65377804059, 39160.15943974679, 42184.875868285795, 2922.7790545097055, 83266.70489919647, 60684.695368965855, 27976.05205303978, 26526.091493472748, 71870.13116435413, 78678.0356231503, 65194.46899758069, 36773.304599334835, 16632.83180678069, 86973.49567458645, 58790.0730906144, 79727.43591801779, 73964.33838298738, 75590.66515437132, 87619.91454023992, 83993.17991749017, 84746.6043470563, 17142.666802271946, 80626.9749678799, 74758.70028956753, 41978.174874144825, 91913.14088588578, 53272.3863070428, 95627.90495748671, 4625.156090135296, 74303.87757397766, 2533.30097878971, 35933.97819019441, 10156.489059876616, 40788.997588674225, 19052.87013446022, 25557.47326087253, 76611.25229849386, 51619.34735352109, 8127.951935762745, 23814.942404092133, 17418.72543333741, 18874.736674865188, 7493.546490079061, 10732.4140701615, 22748.781952291873, 27609.955576027554, 64852.801089668246, 52783.999814426395, 17994.117419232392, 5887.361712724515, 18951.39982286852, 23271.060372215292, 55046.0184043176, 12276.761610745334, 65493.98004537422, 90429.97798026663, 48661.374572460605, 22619.603476990414, 99062.39501950864, 72795.48470301826, 94508.35911578994, 24956.612682144598, 56862.15257644315, 35844.881115202035, 661.4538399510761, 46060.90875916161, 20986.598000033096, 60271.03263440157, 46149.06209629579, 49535.81187759972, 20099.00657179472, 17596.383829253402, 36433.155773179315, 61856.07631103961, 4432.1453593059505, 33875.227537434715, 3968.9891847429103, 81508.8249475295, 82376.83593242649, 14216.191523829792, 83996.19980542093, 99331.19280847788, 74270.8326646353, 64804.182528469486, 11850.05282848175, 96974.83751670938, 23554.110791420557, 2751.2348634885943, 27600.8336824758, 60072.412035200774, 82822.16940958875, 79444.29633111057, 7855.416481612465, 97406.33199979596, 4410.376935231142, 13386.64841585765, 17885.50092942802, 41023.05767920914, 96866.50663123313, 11719.172278108335, 53436.31259862219, 71304.34194190918, 3829.0359948670916, 4107.457346385168, 85314.87551006826]} +{"id": 575, "vector": [36504.05545428259, 72246.51206461781, 98301.77012360012, 65414.16394475831, 54413.51934288689, 60043.878604958845, 44982.250114923336, 32572.571015429552, 17261.067493411698, 91055.81078654146, 72876.68757633163, 38372.53652706672, 58429.322529735386, 16464.9444311591, 29657.994989853953, 70696.7913405106, 61380.56234626945, 98598.35115026712, 49279.85018368972, 48584.27753504934, 2459.6637243906926, 25750.426578472252, 84660.95023122693, 80486.21317113821, 18240.842788205457, 72837.89539726573, 6083.092207218632, 32903.2260514258, 35424.260075395476, 27847.24342054511, 78689.41843994246, 11660.706170207402, 71127.05613460607, 27352.969694583295, 4227.0536049671455, 74025.65341220432, 27620.936389919036, 43703.30546883713, 5705.785468715341, 63265.73479789569, 54603.119962022894, 5423.472223090786, 41888.52229497044, 44643.185483910056, 1127.3313184691046, 94119.17512263758, 84450.14154621739, 8513.989088934182, 93109.29284618926, 44393.443643197985, 26264.827364595887, 11460.186108974434, 62002.988599767625, 54864.875342085514, 45047.49301479332, 38207.47569711159, 68312.60565205973, 65848.75424088881, 73093.51101663156, 87449.02973213926, 13799.353019179394, 54163.94528494108, 8454.44492763996, 69332.05618234797, 31387.927915372493, 55304.33284708279, 67375.32730319243, 34680.3726290399, 54230.50276646434, 32807.04471648047, 19975.337426330854, 36025.07435147532, 58677.018494410324, 41022.84519070581, 25813.74033007463, 61723.883168888126, 97548.59679025602, 45439.38737775417, 79185.34675378285, 48652.11510753994, 88249.63049302598, 87060.64325900824, 29813.44761156851, 7156.994203737232, 13695.56126155087, 30016.874223183677, 15730.607331429213, 82802.61102179775, 12402.385591278531, 88142.79579720122, 38377.806467777766, 81900.44928974379, 37775.41843624602, 41138.40586955988, 14723.233614190112, 33202.191405698824, 48708.82638148573, 76248.02048280957, 47962.77411849977, 73075.47993048505, 45687.68035988092, 42651.940819154355, 81568.40272342144, 46289.20645416884, 20658.01412519297, 36771.84211360649, 97609.13332113666, 54792.105861522046, 38241.846780226275, 32824.234952067134, 6785.99189312954, 17454.080199297427, 37209.992978054, 99411.52937931378, 49867.335234384554, 86867.99593426636, 49338.556563437785, 25597.47569643115, 87765.74431238044, 31310.1835114039, 38229.40512625276, 6098.620898558515, 26945.25439729514, 84228.32159464095, 31770.793656869668, 34348.798607116325, 54209.114867454446, 16737.361367674108]} +{"id": 2005, "vector": [57639.74098185951, 90263.945205901, 7105.676254547843, 39248.40981217972, 29085.294357675863, 59682.490011109294, 74264.79188321566, 51006.43428260966, 89917.95183953896, 55434.69693435252, 31530.586528160966, 85442.75335754656, 3084.786653354499, 14245.160034098802, 87072.46276431167, 15317.945831989387, 22302.989013228125, 74474.30921469911, 7815.274209270718, 49275.50284202249, 59131.33394179503, 66831.79697429696, 24506.72620159702, 9368.700780379879, 16228.199738058169, 53949.35184475996, 6980.758572977741, 94188.1873743209, 76747.07748821171, 8278.617259922283, 42980.759994130756, 69694.51347878907, 56941.731689731, 49650.31388983265, 12984.532855893072, 66532.43958346128, 27589.671654995094, 25048.39943334014, 37386.11406161535, 17626.09076337148, 52618.001844226834, 79421.36026937678, 31921.767350083184, 94162.23041179225, 30094.55961548372, 82004.26949257082, 78332.8256623091, 37735.75211593932, 87310.06899984225, 24427.40800094465, 40813.1232656404, 54137.25020324884, 67236.5307164714, 65971.67895651583, 2608.2817802886684, 76555.25972142494, 93587.03761945672, 91903.18532526014, 2698.8463108294723, 45973.862437335236, 64344.008269357946, 47593.76693265899, 29555.84142910351, 3749.9631678752476, 1392.7265554911305, 41711.2553140882, 33221.74337149001, 18286.07229292363, 8572.034999480471, 97123.51963734103, 73816.34073923896, 35559.1641982364, 80820.62010953041, 64650.897507237314, 61578.05296766768, 11121.599342629652, 67635.61853503894, 88081.08253860979, 60556.833935712704, 43282.69414175363, 64442.331014631534, 11182.251815832444, 82236.17792826123, 13642.340458465696, 30592.789824833068, 4566.290515737536, 23242.130963737774, 77969.72172690148, 94371.13290293154, 8779.395056638861, 74341.76617559603, 63355.42717985793, 91715.8301708376, 16532.77475402879, 14440.109986373129, 59823.64783561047, 49483.207943979614, 93235.38054470769, 7899.104754408104, 29793.053422389083, 22649.892481216284, 62094.91697038837, 28647.595426871976, 5005.284664890919, 54516.41783127278, 20664.254154940278, 16616.24854830752, 40700.80086017346, 73097.6868939116, 12744.931056570218, 63534.86219775094, 72380.05551613685, 16310.914713362012, 74545.51995087146, 12707.89607772561, 11585.917865132778, 27861.289439056258, 17879.145399714693, 76406.22776127295, 13706.703726390202, 48892.64907107246, 39221.416294601244, 58668.864352203454, 20766.53827669005, 39007.50892512822, 24339.605324553115, 53222.44334420111, 14934.73824992635]} +{"id": 784, "vector": [63320.09327533462, 38192.81022666801, 64675.73515471835, 78593.94614066175, 57955.301278770865, 14854.262548689801, 63667.87662272214, 45389.5411061473, 96970.64221739852, 8715.589521128275, 84489.8948078934, 9060.937528561697, 60292.109622259195, 61911.852776811116, 97893.61764208764, 21116.85577174802, 83389.48541266457, 65535.748534688595, 48352.87371612294, 46657.7074035375, 90187.71962546669, 26293.55654135127, 75070.26148481453, 94147.65319860009, 18544.333779745015, 82580.18327703621, 6335.515451421669, 12016.026325587693, 39155.66689867553, 60527.72684281065, 96121.98902210883, 31776.18901909648, 60870.861972161896, 6510.149727895709, 78144.48237878505, 86203.5377531841, 10316.868864869877, 62863.94296530735, 78649.14737909602, 7674.1794685839195, 36647.76729252933, 60018.64871408496, 87059.10353468162, 49659.50302773228, 96340.27772614574, 26087.861758766485, 29106.12554821519, 65599.58085618114, 35973.813638092666, 44948.01439796993, 13183.849865500984, 46992.82858558963, 50848.20959755092, 7040.8434028547235, 81541.71919668397, 99330.25296075712, 11634.012623083756, 96035.10112606238, 80134.90293391976, 55399.50807523506, 2341.7794018666837, 57912.344209546914, 54380.07152378488, 84072.76164774694, 25501.790866140873, 91628.00119102575, 44727.54540325968, 45956.7138895833, 52634.10818540819, 49637.49429385023, 81450.19381527699, 76003.64632243197, 55102.27711456793, 19557.554754411987, 87555.2836435722, 29441.32416725396, 56389.30078151078, 64897.62372815885, 60997.5761371788, 91206.68087247985, 17907.17387526577, 44006.35674495782, 71676.28122235627, 45824.64997440541, 22518.340050652132, 30897.599753488004, 23981.345862038404, 44535.46850124248, 42563.30151780685, 27078.54751325305, 91678.42871847794, 93197.53439194922, 96648.02898802055, 66305.0552101817, 12967.63210058849, 68240.17612909732, 91749.75316304603, 67433.39170312932, 23949.118961140026, 16518.13674108725, 63864.35854550749, 28274.11276519264, 79150.34601327137, 20382.04558489607, 78029.47565437939, 79825.50616166952, 30841.65299637692, 39651.26190046253, 12888.084594224158, 34544.66938785044, 51035.19525482424, 41266.25578975044, 76874.87734343362, 10710.770753350873, 70478.94560757645, 3899.7442706609654, 82399.96802572886, 20427.74020343825, 13985.729464964757, 23292.00017805586, 36375.90542248919, 14016.821544891645, 9346.86062957305, 58666.053987920066, 45519.9848623467, 79784.31707094643, 42165.03455124602, 25784.693063341736]} +{"id": 1739, "vector": [73775.20571367328, 81899.95654451792, 76348.86197968188, 37039.72323765832, 10448.979926245727, 477.99614285662926, 41964.76981799883, 27025.09736353751, 71960.09647893823, 99355.48328901194, 3282.21728334821, 52041.70077718939, 14.659688111606783, 33223.08181511697, 89485.93052901317, 13192.490515262367, 99464.93492977024, 18042.736672024883, 95103.56337098133, 74614.26747072066, 71393.03456528735, 75382.3514694282, 15539.747142168426, 63864.20661635218, 51055.503725023045, 82580.20039762445, 13809.34882010818, 15329.654575812112, 30449.427471327006, 92403.66934728278, 15092.067758568095, 92583.46803861635, 18188.1234149861, 43068.409289774434, 86156.29502225327, 84733.70921094193, 77658.37675182866, 70308.46546080762, 23969.026266506822, 60351.723297113815, 93265.003999931, 60468.68062500949, 44920.837655770294, 10182.296062884332, 33595.926781092225, 26553.395367609144, 9506.657697152865, 56707.173870291226, 70330.65024395539, 36037.248030223964, 43545.287431644894, 44883.7189100372, 18529.195234020524, 54807.32630787352, 69796.87334518667, 24831.953918918138, 97381.30587492265, 19409.794299057616, 98837.15523281487, 80197.31043850994, 71448.46760940713, 67621.4648878945, 52166.56416311058, 32547.92875501432, 99787.28074865474, 26946.24386527159, 27481.3191773399, 23442.308223025488, 98748.06810278683, 11754.868029099385, 52288.1879919186, 73799.89670038834, 1766.0191102251256, 43659.86587733136, 3802.9886356488964, 22760.482963650862, 16086.736254381272, 69122.1219406238, 74355.91780045658, 2220.168772897779, 1617.9525672929574, 55254.91200785959, 96656.3518473751, 23324.04248081028, 75631.95710994038, 30232.66358802328, 64420.10429266539, 49322.966348561895, 21693.042043382316, 75985.80365870641, 48725.80618965374, 93072.9776248013, 34369.3609864643, 46901.10219033774, 87217.96826792437, 6257.952214271722, 93235.11191562154, 31205.822027230603, 79022.96441907865, 20183.966318578816, 79486.09668684319, 34811.049784102455, 28533.276597145006, 78034.34626880492, 47948.63929981951, 27692.81411602077, 1859.1221664429502, 22735.866420448936, 99884.0497511144, 21536.586209439112, 84024.515239247, 38698.69985141062, 83881.99529101576, 73771.69319515573, 80775.56781258724, 43530.23206046, 71228.01255721091, 134.9517869777439, 95978.66228606297, 14985.340705967708, 22402.176613382962, 17246.72521620767, 82280.92840219477, 53114.768265590326, 46684.38821890203, 84002.61483476535, 71229.25120154025, 51401.30301928899]} +{"id": 577, "vector": [10088.528704413158, 95534.69092059985, 29365.928429882715, 16498.226261068783, 48575.684895366125, 17829.620131404445, 16140.443810586325, 71691.41400976952, 82014.53370377782, 91352.09022738355, 93065.64466164123, 22507.08451908846, 32014.72625641191, 28046.634292544746, 58467.12507727547, 45622.108301517335, 21319.64784566527, 47618.27180800384, 21119.189805132966, 60984.82560728136, 4999.257435861548, 73234.34907432116, 22837.051982794408, 80155.5148944503, 66365.06578965273, 27974.527089227773, 5715.7066490732595, 44795.289793490956, 89794.12580736047, 33554.03735658446, 56062.518962755144, 61805.078719884135, 51819.34268205941, 81917.0335639941, 84917.3764005722, 52091.70492157365, 8696.680288774427, 53346.24662066143, 64807.67620758798, 58558.35196891371, 89737.86785489222, 34245.82127410425, 62624.055516332046, 32557.606949978523, 44525.23613677089, 12411.912171342965, 88663.8208375136, 5116.7913997675505, 38001.34001441581, 65982.32052221733, 86589.92270401167, 91061.91907384372, 63653.68121261259, 72532.98987913642, 83913.40460905999, 78973.2167776973, 32601.87068829683, 36068.02212387488, 11165.949609251202, 85095.63021476925, 60639.24745441321, 92773.71841712312, 14787.261270962681, 42394.01856811624, 76198.69909194847, 18865.855675825816, 38124.425761928374, 59599.9126489074, 27126.755031418215, 68976.88707892224, 39234.73095547641, 21401.22200811687, 77199.01747088657, 90238.01385978296, 8880.649403439178, 66019.92038058751, 85167.18497706357, 80812.37135444362, 44304.101264310855, 50822.622668656826, 37159.43339000568, 27797.810352537188, 38160.04034355406, 17661.16416327468, 36197.6463528457, 98117.4424501853, 65994.92224189788, 60287.636007515466, 34786.3360758631, 87452.71575792234, 96881.5044401619, 7223.2058622994755, 30957.823526509987, 52950.97427188393, 35720.179766044916, 30790.321321978743, 57047.52315853494, 22096.60360707285, 54682.06268881287, 51685.504265420124, 53703.96843694258, 73612.08642175778, 90628.68594731092, 60945.69910010352, 69192.81660205073, 72177.39030176468, 61026.78215201623, 55389.41683340583, 15446.83577943562, 31023.298623206254, 41557.10480419449, 78263.84135110893, 54070.09311122429, 47156.30809567965, 82061.09467746681, 72809.78369140912, 16440.935526212306, 22555.302516537147, 32711.241244630484, 2369.0826678429366, 84552.09676045965, 72617.51635829262, 42039.52928310869, 40520.94725342207, 37265.315528954154, 80381.09029385654, 48326.94545758595, 36161.430466968115]} +{"id": 1867, "vector": [91483.52877794103, 30079.099080630534, 37811.26629805186, 25370.345389217542, 26984.760201141034, 72567.25382559195, 31962.12637533288, 76896.15021866551, 6803.408457437244, 38740.06351513285, 91324.66250728146, 65506.12099662329, 67145.06108134238, 72040.19974793847, 73373.43704349552, 55383.28719478356, 1415.8101783078746, 30896.408437120237, 63506.11482210869, 62298.96957235659, 72848.33317444142, 60925.27388591614, 11292.046168582216, 72373.68939360052, 37459.02951561966, 39242.03664385988, 87885.607578046, 45935.591806618904, 57720.92823879352, 140.46260796480857, 21085.411396822252, 18555.83845497417, 9363.842184590887, 51524.011922399106, 25323.432205095607, 23661.679616362595, 68372.09101814244, 78407.43141614282, 39764.9543625221, 1034.839804409049, 16912.881692223058, 46014.812894705094, 4606.310193430152, 43932.44117259524, 24619.339251696536, 898.6572198463216, 58225.11576419701, 17894.815695217203, 89545.56695703459, 70397.78846311905, 85774.42891336056, 78645.03590433918, 98763.75111520199, 70569.53325695162, 32535.12837257012, 54396.894118433745, 16350.38493490747, 79915.2728796753, 28955.128202791868, 3486.7520511809857, 23352.295884324427, 90959.37841866363, 85581.16624789411, 50938.01978278212, 92079.44837802049, 95838.60420953353, 67947.46987434349, 86311.97627272646, 25082.63916812041, 56551.200273972434, 90844.74802758932, 51438.3300284149, 25087.16362010168, 66580.79934524167, 33802.29670436795, 78685.2415284718, 60131.26460997086, 27922.000993306996, 42169.86863225972, 88573.01563613863, 21249.13494854388, 67913.75877545134, 22049.24662769275, 1177.1732884244045, 42151.6131958544, 89332.43750010936, 1673.019601830472, 41586.01851715056, 80670.1167598968, 90201.59311584715, 30785.251111731093, 46674.245539926465, 18040.751212811214, 57005.27539651158, 81890.41036516397, 69834.5736698933, 33097.859263136364, 59333.16433091363, 98146.74919091041, 52862.19712129524, 34584.476697541046, 90345.65767557846, 79031.36969783028, 90127.1262347741, 86140.86514578956, 12802.805537561335, 32576.581209537326, 47569.487344304864, 29766.092888251504, 61517.4396735049, 89201.29625837512, 74225.85170972146, 56916.0751687109, 34907.614591231904, 61544.76654135457, 31683.04022008105, 53534.34294009431, 81948.6183191603, 69142.02528761461, 18486.77990786688, 80311.0187731026, 35424.88464711324, 205.5879575066699, 34164.82619784453, 83559.67618202769, 84826.395561613, 98648.98558505568, 14815.611661945393]} +{"id": 1395, "vector": [17055.03914609895, 64193.83616655183, 26695.9054944528, 95218.8060099019, 3788.188295710648, 32180.27013837873, 11681.836544492275, 21767.64345023804, 29431.957112422202, 49001.08830466749, 20011.331489790384, 80095.23979710236, 18013.02083890571, 2154.93136555196, 1770.9406891679214, 73692.67889690195, 91582.55357459915, 92518.62899743907, 97867.60229157344, 16919.112509581657, 94526.09726383639, 18120.43408900217, 37927.18664399534, 27126.007028159627, 33674.98065403142, 94870.24878345018, 98317.58359924857, 68022.39914623869, 2069.1203719234386, 31188.02289003706, 3971.602875665403, 77894.95151578869, 18291.73072535618, 46742.1957735278, 99674.57623279915, 74655.34019536957, 45001.56559685253, 51311.11462611956, 58545.961287605984, 36961.38275956019, 75982.71789510465, 24576.06236439367, 45149.23797377899, 39136.62501836484, 82749.10140406186, 9915.476132759815, 23695.86534283622, 52385.510350907985, 49887.743906215306, 34097.55989888036, 14275.032888298445, 28632.69748097691, 58139.21272391597, 91059.79704423399, 16774.698801633258, 37740.46055803899, 93765.66745346152, 58698.42411225376, 35587.47218064791, 58553.07162406449, 78171.15625436627, 6218.417805642806, 71242.30690168357, 32638.528767102325, 17603.699739877622, 89856.67415658709, 70696.24907260523, 8888.431083699266, 70367.03418121285, 14185.217008648011, 21125.699582667636, 93880.01058876372, 85871.89521514028, 9235.654703785867, 24168.85291685096, 98687.54533432845, 43645.41927325056, 35265.23680726946, 36299.430308278104, 65019.38616014675, 34807.651186881114, 5518.933923777314, 78930.95570542727, 74295.43913584173, 21299.259440478723, 90411.007681664, 962.8017526737365, 55552.624585310004, 89439.40572435973, 96400.40867550079, 14769.985905626814, 78373.13915590111, 90348.03122309578, 93841.41358365792, 18358.60604838461, 71408.44207618467, 7683.887392024225, 3331.485527001965, 56143.67135115882, 85572.68818307885, 48775.391798188226, 40255.44344049015, 77705.72917019822, 85691.13929621276, 39473.48472562034, 5925.329935950607, 99739.94688583472, 6978.7540727690775, 26207.68785108274, 59887.293971381216, 70467.56996544074, 64474.0784240928, 65540.07474851763, 11048.688660398564, 59585.53926927296, 78676.37283399835, 96802.63805931102, 78317.0159079723, 480.6348564861951, 8771.783933183619, 19001.278317468496, 98689.94945244963, 15828.563157062757, 54222.50054392507, 55670.11218370332, 69977.95740732511, 17522.455841100505, 14037.725292157154]} +{"id": 1203, "vector": [13444.856693392427, 57978.50983197348, 45608.08985151286, 11659.530982003796, 42086.502972495684, 32705.560704597603, 77036.28358964002, 12153.560918591833, 20518.286299364874, 9103.940897539076, 15305.477652826361, 96778.69735282264, 95374.60571755783, 85632.1220995465, 9622.900513177436, 44921.60360937464, 93299.23377776387, 24785.850838374714, 40261.984847665764, 53240.30860634214, 26764.936201814216, 9551.928849369351, 82728.31318568476, 70611.98238722043, 7296.607387857712, 87846.13096194838, 72046.57172340972, 70109.51591387703, 80835.5815429202, 57308.685243396976, 81466.8693354481, 29593.28861362166, 94169.21261594696, 98988.2260446177, 44418.44273516705, 56755.521970367154, 39679.188937397594, 41884.969408910576, 27668.786794402946, 38223.13253742898, 55229.35668697938, 57250.25691301876, 30554.7420526173, 79168.3849102166, 76480.75490345874, 56322.82180787054, 50420.773570627774, 26493.32135753243, 64563.38042459235, 30328.529085996637, 18103.487144684517, 84857.49639436338, 18370.225362180594, 95949.55600111069, 26770.122111006454, 3376.9500990270094, 3398.5490727934107, 21476.693174899785, 39899.8034230576, 9363.628782171862, 62743.39454631852, 96480.62218200378, 85494.78384599247, 56595.973622014026, 98202.81697600917, 41170.03195130606, 83041.62492357586, 28704.40115645544, 68632.16380213303, 8655.56230711908, 49361.560323504316, 75802.54561137794, 42952.00937200528, 68318.08106588286, 38050.783135971215, 63982.503628067, 71480.53255490908, 25513.792718356177, 93011.66956177664, 46926.97977500817, 44117.71440720954, 64852.406673085185, 51960.736215751735, 43149.779000402654, 26928.65895611034, 9068.536217307077, 74604.446239478, 77276.61331674666, 24746.924204407962, 57148.61756387735, 27095.045121149575, 2522.8406113041, 75831.17466743858, 8116.847766950397, 94242.8442343867, 47487.2218621211, 68656.61267317811, 39268.41753516075, 10591.718640306524, 19332.231626931938, 12844.918218290803, 90655.76573587257, 49915.72137442959, 33803.81504930793, 45455.85932075154, 76983.40445143006, 52487.56935724917, 60011.08380493587, 66386.6376732624, 7575.978898774005, 65828.14173833911, 19856.028935131555, 36307.50756875989, 82540.31080665771, 5440.5568222232305, 87514.7423047218, 16498.00606824251, 74978.78323301167, 34664.550691094024, 50053.99943887241, 46349.45693023018, 2810.174279887956, 60909.52608219255, 78271.7294097596, 92394.6089072859, 3127.3199190629543, 72297.79661234033, 86971.52346877212]} +{"id": 712, "vector": [34205.061103764136, 4442.268272411654, 60126.776141738934, 74284.44945202368, 94339.98769225893, 87339.99309022067, 71127.86544196762, 76400.37096171133, 83476.23829636424, 25669.92109638532, 38470.253699419955, 51674.69380213727, 87997.62823155783, 6467.640122753027, 12489.158636346076, 46684.87954325944, 28162.377269467943, 17886.437525948095, 34789.24551918362, 32540.747971272067, 26155.302430706794, 22954.647908507, 24711.536977989366, 85688.87980017427, 74447.04087885568, 21880.53443021, 44070.949041791995, 69680.9617975773, 19296.584843048724, 95265.54604916627, 4491.113858420481, 6479.462640021472, 80235.5564509678, 47483.19727173593, 44377.52279228525, 27438.409905205197, 94406.58284787579, 1664.9033666016112, 17041.619347527725, 55003.53357372604, 33505.61688314456, 86074.66291396075, 49088.77462685426, 4053.3287492546256, 85323.41059853243, 90380.88674509389, 48348.66323172318, 43317.20523316096, 97269.62578645075, 99228.79473217666, 8110.659446728341, 58678.94241398778, 38113.45044660006, 8259.07482380398, 78174.113368394, 1710.3999149981064, 80456.8759916223, 33290.36577026203, 56914.87544239039, 2058.5099700186847, 32652.1094334127, 89237.89881470021, 24768.73044345075, 23724.86378568711, 37536.911211280436, 89061.18511716412, 49708.40421738382, 2953.9171364666217, 71760.4186172653, 91755.42092474565, 97431.12189828797, 51590.726229109416, 45623.0784117904, 66369.39029127914, 94368.0256913689, 19404.99983166939, 64953.28947681322, 5496.955373468771, 56077.564702724725, 88038.44943724829, 94471.25692578143, 28447.10856053071, 11539.455793875131, 46308.01977483503, 7691.049098844626, 91615.71816992387, 13116.938558311176, 43855.691039742196, 77792.24528666126, 1800.5248644636263, 51171.89689357267, 44950.86383553596, 18580.669433963638, 59686.01883413265, 91012.52940543492, 23587.68470039475, 3148.5635069310124, 6749.101640313559, 32649.901525105084, 68711.29503683466, 79579.70248550645, 56735.516664878305, 62074.86155136408, 82563.24263707451, 9506.318612404795, 84314.4063283142, 71143.39751306729, 46808.26140555926, 43434.42543954979, 11546.256237549656, 43346.531439070466, 24198.37094919325, 74420.38008749457, 44576.844180169515, 14956.403682062202, 3059.0746255489407, 38571.613720515874, 72634.23771788503, 67469.23905989228, 99746.42287366219, 2969.4802277640542, 974.7536127076462, 3432.83432660495, 87746.12586131308, 24989.08511082687, 266.0052051161266, 81846.29417853797, 11049.074956577288]} +{"id": 602, "vector": [67039.20699849055, 70306.11721305258, 38603.37435921269, 51988.431820719816, 82140.47867800615, 12157.827669448252, 61271.57202478931, 71777.87851569695, 97146.76245516371, 64053.14666366543, 94738.57635300871, 50298.087145943726, 69309.18783560376, 38939.31338107918, 97901.7792866868, 16189.361145419745, 93048.83125615222, 15377.777162102146, 68323.7274781532, 74158.24902584693, 63755.43642425262, 3534.828294132808, 44489.28577330272, 64014.81944396596, 76329.10706880168, 59020.231172042906, 70344.95254175043, 87689.10433424197, 99681.61808454651, 29467.164457294824, 28781.516677952, 92317.13634407688, 61805.519231061844, 84791.07605054369, 25438.74457601876, 2979.0347562678844, 71207.95664994855, 32112.12605033654, 55406.568179132955, 82776.53133951587, 44651.02487283888, 74261.5314402755, 7687.417165561938, 7485.9538914192835, 12196.862666177132, 89732.72193294247, 86043.57940130787, 98494.48941515324, 38195.0935741882, 92990.33538862353, 9322.104766642204, 18560.057519759976, 23943.23330609198, 58037.35526780555, 88462.6431850562, 33858.302376904096, 32151.992701235988, 82685.2015929778, 40668.88577651742, 66200.2997420419, 64270.383722336344, 15402.611561147784, 88568.8571063243, 24116.753038725637, 25644.06550617386, 93116.9176540171, 23898.18529749842, 41366.416620562486, 72510.64452091695, 59977.14169674896, 60079.47155057243, 17996.835899544483, 81793.05958654605, 17126.86733753195, 42714.38202601635, 6286.632206233556, 80236.36528794591, 97763.05923901619, 4451.778564048392, 31093.521118900324, 13995.216140408062, 14537.873647436727, 20444.756149227884, 15387.341766735608, 17103.074173155754, 16259.13078926663, 31697.1745838997, 89873.97490650514, 60712.60809625785, 90699.03337315322, 80543.10359671069, 2483.931164070452, 26205.37771620215, 51776.69512818419, 75969.82612794038, 44283.31473981842, 23673.392997350897, 51225.54984183062, 41196.31985057691, 59202.029885860466, 88120.3214826812, 629.3392254228536, 58048.06264123336, 18640.578270862818, 73214.11433935368, 250.60510762336952, 29965.450394675554, 80371.69533098882, 75667.25019647952, 19985.474732593833, 56953.32756640613, 56927.275451461435, 79911.30768972736, 78182.45336241259, 57861.97358763716, 38673.20260336354, 36372.22991444039, 30526.09397814885, 32765.716179400693, 53418.6728092087, 76662.53452179386, 54535.51305842567, 92828.46730846529, 78005.67992923026, 61074.63412145681, 80419.78692462675, 41546.74681288346, 10001.925827435765]} +{"id": 1772, "vector": [33499.198551340225, 86393.56390843893, 3679.2688728933376, 4131.27050763954, 41995.155025062944, 40773.18586978036, 44005.936330888784, 98906.40138102103, 63424.69525521551, 77416.5346270024, 51387.54063824389, 2743.2160999037005, 56607.25358293802, 50263.60897247909, 53543.38144195477, 32513.94393244943, 54287.323589028005, 58637.345770729546, 59938.9397578009, 52217.96151139008, 10716.611345592786, 64537.06643638968, 86724.58218013599, 9666.9449223311, 30440.76935275338, 90127.27535319494, 44030.188352336874, 55346.02803723512, 88317.7426361115, 81972.71463291733, 6624.564405158007, 40585.77586833596, 76579.09090130313, 59378.93485935071, 53312.8327449064, 2807.383374630501, 85389.29579555316, 13767.830184891995, 81708.140287387, 32277.397624973135, 40212.879538038884, 38455.33528916324, 34389.5046491531, 24339.396790195933, 65479.4671601244, 27874.30495013439, 9450.46169519389, 75288.77891342019, 60408.27339949428, 78754.67249026264, 68191.05820782846, 70991.11336662815, 73226.3750951429, 33806.92941819298, 90306.07234101555, 85989.88644279593, 19114.96785733244, 32609.822499819507, 40389.664556296375, 92218.194282207, 71574.64617471008, 57454.850868752284, 30799.32399802293, 83678.14717921932, 19533.200293710583, 68320.34219275581, 89869.02944030544, 66237.39646963062, 72222.32697221392, 32010.25477752373, 50784.835475361215, 15303.843886906687, 9516.751601434604, 71765.85567950808, 82980.69044242502, 46521.79230940482, 50535.80253227108, 70229.97612666045, 31035.503817397457, 71702.30427391338, 15183.032897922056, 92015.75308668822, 42214.99247183441, 59433.5867737188, 21059.139024306405, 60630.91140078476, 5221.08694074116, 87930.95863372102, 30324.03367874672, 78849.77797009573, 76650.73442045748, 41860.060344165395, 88851.00779056909, 14525.723260192724, 80293.13663906889, 68014.5500380836, 26771.32106975626, 9554.06299329088, 50668.10388109453, 28694.018669895217, 3101.0099881598862, 95238.57082315904, 18589.79512902166, 76229.79472444524, 38777.69696005004, 23176.95448229351, 35105.78314886975, 22797.414986034335, 76367.06162215, 80052.53989972046, 22852.34783082294, 55527.46264893436, 77956.70720141496, 15642.931238160018, 86196.634203814, 65190.27407435286, 29895.932058148766, 65555.61492661807, 48939.968872070225, 54887.36359424646, 92805.85378991925, 61004.600585615335, 91546.28905802817, 88652.18535516584, 73603.35069581633, 45192.904281832954, 78353.4040704565, 307.2619039516833]} +{"id": 1082, "vector": [51957.62124374412, 31233.113246841014, 88033.20376446117, 42986.703268906225, 26098.75196153347, 93767.651015297, 73141.81455971734, 77801.14440093256, 91972.95797046208, 90273.65101916637, 33621.605197917335, 28034.843113665775, 94944.04443588396, 34240.67101542922, 92760.21265446869, 44667.04337358357, 8022.6625277589055, 40141.77963051058, 66539.84890136222, 68876.62374803815, 38272.26539114144, 8382.376100468637, 51200.34716134645, 36628.9150673789, 70560.89701105095, 92170.58523770119, 84220.45645058845, 49639.77721597602, 79094.40604161986, 8346.22043131662, 17760.74698573912, 22529.40419918087, 57336.01924850975, 13192.851165470765, 70334.57929950711, 85234.31104296364, 6495.301038436219, 95920.60666486205, 37664.911428138825, 46100.52234637433, 41322.79476088978, 64696.65833313122, 17364.47616836654, 38838.22061956109, 850.8203592241581, 92882.0280582805, 41279.59163963691, 69540.86799876323, 86959.0588456187, 79725.84744207334, 50543.269988516025, 39263.34335705269, 83857.67466709699, 46492.16871803664, 16907.756171330213, 73116.31737009007, 19649.053713530993, 1494.7682590314648, 25541.874719317282, 5690.976783073309, 96211.413622978, 67332.74808334118, 95481.80555555336, 30619.54176265036, 62565.429746084475, 45257.01340582945, 87330.22272889687, 37729.86235244371, 11498.209622356591, 93156.70927538787, 85034.63564097874, 90955.41086344063, 24493.048454521915, 85186.89577716033, 80748.88568985413, 78844.84249932475, 63990.6124209858, 7207.376078100026, 40800.95368897244, 57106.23372391371, 78317.68151915516, 92749.30634680059, 66816.23449007123, 56883.203375320736, 79488.03463697773, 60534.874822745944, 87811.4611248265, 92694.8783177226, 14381.042962305812, 20348.607132405017, 31910.185543437587, 37906.18764056901, 84747.43688218617, 67610.5128905346, 51632.24687256309, 20144.883919086053, 73967.44275956413, 60883.42349103497, 3205.183554290036, 23798.253397140536, 97470.9734935357, 43972.06752213683, 59858.9841249461, 77120.58259599203, 66781.9825181455, 54560.27208333922, 68056.12673800343, 18037.433040053995, 47521.831004259315, 54802.66823485669, 50095.45694255704, 92092.73233952603, 45963.31706414102, 92154.38628394132, 91169.70589750596, 73536.40331887774, 39287.287990874676, 13347.864340776328, 47388.50760006711, 67368.94408529406, 40635.29378949079, 67173.68850605472, 62256.71690072111, 82536.90723122719, 43123.712676202165, 45693.29771454242, 80113.96242107586, 96729.30577475554]} +{"id": 944, "vector": [39421.68470210543, 7481.956996067463, 62239.96293316396, 18448.443907480883, 36525.88861763862, 14218.967597049548, 33521.41823495334, 66779.13221301253, 72633.23510113108, 77776.91293980676, 60274.66200860967, 46871.53196951217, 94284.51168824613, 18710.033978865336, 516.9458176017549, 82122.74085755652, 45424.399955419736, 5181.342325595695, 97805.55488102381, 19015.243302289044, 81215.3626971646, 39963.5907171905, 37705.82099537952, 87697.80930610844, 68112.91391904539, 26584.307538960293, 29364.466754051467, 89699.25925540045, 28181.622346500335, 6795.081569931738, 77409.18237080185, 23042.395857351315, 46278.63640967799, 48787.63400142691, 66264.90868837415, 76701.71500269181, 47167.5754731947, 36205.15376483805, 75691.40433216018, 28913.73579060922, 62239.37718334713, 17391.711100498575, 66597.477839, 49828.0866608938, 84807.28178667613, 5317.202844714486, 66176.98860642119, 90847.27468557595, 54253.58907002423, 95917.69334547833, 56149.940671624965, 5205.154095236398, 47368.59372285723, 15783.241036559015, 11097.228401053295, 4058.912651795976, 24722.31193924608, 67691.2461891955, 25249.7910222649, 80270.45392987128, 9857.840297617027, 21392.43666548367, 86598.63251772907, 43207.415722957485, 25617.50623976944, 37926.37818744255, 8375.642771543335, 73865.67914483006, 14570.685077924883, 28461.175879324906, 53396.607428985786, 59044.18279056356, 44778.61012793851, 26965.137440462207, 45852.73297679762, 42282.3706631957, 59716.59531135829, 37823.539646850324, 95404.70600215341, 48674.996123297395, 16483.247944291867, 93284.50319598286, 9906.733094360654, 67068.42839526803, 90817.95521182411, 42618.97051016528, 85736.33345808538, 63674.555029339244, 56278.52397158385, 13569.282239835256, 24920.151736571217, 74272.91272253796, 54983.299733037136, 3387.8201594177535, 11810.870123840245, 71472.91970958673, 20190.796031932157, 4138.928432191014, 36238.376570496024, 35256.72534556915, 34485.88472443177, 52383.48321387412, 28674.045583267172, 66976.99963715656, 56352.88305389927, 91970.21976894264, 17361.675326619476, 55767.53759764876, 77238.15928657528, 9500.935246702058, 17359.680855201277, 7967.825136295326, 12845.174100073997, 71683.0791688315, 33793.5056734714, 95836.87119819093, 19257.102902731982, 74276.49644360803, 36788.76481084291, 94268.82560285869, 16372.904281643152, 54374.83296373469, 51454.841262786686, 89091.49718000411, 42270.98149911891, 80043.20173514659, 25033.047865161083, 19635.317652828344]} +{"id": 167, "vector": [66158.67226561754, 92911.62873367673, 72810.48652215672, 42620.963459792394, 65411.81472324269, 26494.18500312627, 19006.865711146693, 59306.11453599527, 13706.21652396623, 59936.05159898201, 94283.02700554352, 45475.41562069708, 57533.65951822823, 66550.4748968152, 99649.38118446368, 47898.10509550806, 38149.637633636135, 68257.30035668625, 82327.30601051876, 67450.20119025497, 31739.435552515162, 23693.34304301588, 5299.559079723815, 92560.17293711004, 58254.38760383411, 97314.39476788293, 49584.79953292059, 61033.53597903325, 65102.74049458485, 77426.2560282146, 12868.084362172549, 28283.091649130256, 56729.66203107542, 6942.1069477851315, 21706.75425526608, 29850.82795635612, 47857.01376459173, 95768.26602997712, 86341.39227163931, 60948.69403364548, 10572.534593677796, 77659.88326141544, 51525.27414684706, 70119.91858996793, 60120.67283108492, 173.5651940258265, 74043.7536967055, 94503.23725709942, 46113.440497170974, 3436.8520524309497, 38024.40394350421, 48167.060981899325, 23792.59188977315, 50061.35645113238, 37632.41447962366, 28957.529344213584, 61975.964814768195, 21314.21704024823, 9749.173506903975, 4633.318569794586, 96169.00622438152, 93572.10966458291, 82222.98466080822, 3108.0021708308705, 46643.068339680285, 21262.532060071295, 91973.25182680928, 49790.556640147275, 83296.74101468298, 86425.97283646975, 42901.87451685343, 95934.27857115488, 75367.47491719424, 65699.68038140229, 27035.778553750013, 99672.5752775247, 99284.6019683964, 67201.22606419258, 56801.87896820427, 69366.52342506363, 56018.744293433374, 78063.37001187193, 56410.22883191005, 95969.00824559774, 15646.954011822201, 82499.9234766693, 11672.660261179457, 5918.686465571798, 20953.6682991978, 69564.09715298968, 11095.9787847968, 14029.24479983082, 47594.268632297506, 88975.00472347188, 79542.66707079059, 91934.92699989016, 46969.45764024401, 74582.4497683912, 74105.06210467668, 55478.73415775629, 75031.2106316469, 25337.292331541063, 99280.65398768763, 15779.58201024131, 25158.99315183614, 99876.81449770856, 57258.296129292765, 92694.19716714746, 32414.74547348151, 90977.03350245435, 38648.15778625569, 26286.426695378017, 94387.23806064112, 26858.657991085045, 52481.784661623795, 44603.25726208897, 56919.7670544868, 39839.17432734063, 1643.941942290983, 2784.247051985367, 49371.33048406198, 53861.06078021014, 53981.65207878792, 55630.10198365851, 34130.17994722444, 58548.879149937115, 68408.42909779729, 47361.796562235824]} +{"id": 189, "vector": [91215.06714965215, 54785.26494732813, 34679.054478650076, 58415.4771393069, 90921.22439075392, 79174.85635376361, 62606.09429833486, 28049.382973515712, 60807.17726823751, 66031.32143144062, 40862.66853869147, 77713.30204850255, 96335.23059113453, 38400.6139916487, 76957.5105889031, 14082.408890806508, 5709.118956673309, 49857.82271743569, 68692.60874587337, 93819.50845809486, 2798.3136722646964, 24800.520443690344, 15207.557621014168, 82340.42699711044, 35201.5488028153, 5246.280917682689, 72923.70665809714, 50479.03040668154, 85782.460399119, 31198.672017175355, 92551.55377700219, 60992.347072225595, 37839.78488880995, 77750.48586911878, 8435.896449110436, 18105.95612618714, 26199.55101904682, 71797.23277018486, 71154.917795469, 75345.44772013319, 27694.986655823504, 46314.26384797618, 48990.77181763411, 54452.130754982616, 78177.20638867436, 20290.12117248924, 75861.14019090655, 14952.082220764973, 54625.673904221505, 97747.54801244094, 26636.51453019118, 56435.34590443452, 50425.34836022249, 36908.829679605245, 50321.8672970203, 99844.08938966699, 80192.56553958474, 24425.3011064411, 1272.0431938068932, 83006.75702124847, 30796.648369979506, 48249.9575242713, 83977.12015599046, 25698.64360808627, 88921.59412542316, 95891.69244507368, 42266.57945473363, 12447.468121571337, 78717.63856221097, 10943.945671472766, 25458.11660043492, 11930.167143799787, 86358.10082656263, 33081.71216582107, 95180.48927471334, 8847.357831279523, 8795.521638937453, 3646.155782774385, 27983.535410379558, 63578.39890921145, 85479.25925152138, 98393.15513611326, 66294.12196021376, 57166.618981463136, 90188.2088789463, 60134.996320901824, 81357.56808459805, 30539.156716390993, 21935.24302836587, 20339.164028026047, 10124.428277941499, 83194.07786820238, 24813.572048853493, 64981.13409794614, 1783.8218973069586, 64746.43083870935, 85810.46073160635, 45556.527695692195, 47765.452673928135, 1912.2180316355486, 87705.4659874286, 70293.83384495955, 17709.896820333193, 82937.13610243896, 20299.853040882175, 91915.04795812372, 45295.44319415252, 86791.61546933147, 49678.28694199566, 75928.56187818642, 52994.15885256709, 15449.781406059248, 7868.743892292351, 85138.22705597228, 73565.64871720098, 29497.232939557514, 78195.2140618182, 35113.15135879703, 17754.10380293829, 22968.791950292823, 31012.88484633413, 24062.817820791082, 83209.23430963328, 9857.142667536657, 38976.88740976541, 42197.2772019476, 18862.95756336265, 98336.51310095514]} +{"id": 839, "vector": [17922.764138779323, 78678.79777241783, 72127.63916129807, 52438.08025059663, 87594.93738025105, 8638.5278163568, 73556.85646924218, 57562.751871106775, 54008.54555192169, 47876.94240417896, 2965.9536099581605, 51408.48037316761, 19635.737413000243, 67848.72746763934, 99372.56568356347, 11245.60856968282, 9025.656592936693, 85461.52845958162, 44001.31151780437, 73865.46667819584, 17618.59368914961, 62576.084427360234, 99767.94170250656, 8259.323548902032, 20143.516797230564, 80194.2803136956, 89660.59261450083, 92700.50719185648, 39590.845289297846, 90593.88119961972, 1375.4731978441348, 51841.82195415533, 73683.6866626386, 92859.80880964044, 97409.61247976721, 37594.137186928165, 21917.688268853984, 41217.14023815993, 56939.10909826977, 93445.6930656125, 20489.520961463815, 22866.454448520522, 47947.0905823293, 64357.77405529106, 7325.278853239092, 93465.3914634453, 88232.32736008184, 70385.72495565076, 82299.34744444929, 71433.78214288496, 90495.8116306566, 31144.913819860565, 68469.00593091772, 73088.46726611635, 13656.59648053822, 96182.20600167064, 86269.24253745312, 90637.47982402181, 39564.0888576083, 30173.31211433889, 96339.79563744177, 74331.26008096206, 96754.820695938, 57305.667561247465, 73503.01531160361, 13575.605510179912, 47165.76619760163, 53632.866040991976, 52188.95759920921, 27122.820097251864, 63282.02402522073, 38122.739640551175, 35404.51876902784, 59915.40708374635, 71354.94033625828, 79101.74508376958, 79344.06060819604, 73648.75924852333, 1668.362411768265, 46514.41939481082, 51120.3149258017, 30671.44042846386, 39668.139220773, 77988.19650518331, 29642.42986301596, 36304.21617178609, 71006.77446695064, 6553.427469497541, 53944.35130658225, 92393.13339997329, 35127.01000464647, 95799.1621975216, 56062.28590749245, 45682.85083271997, 33033.922534305304, 57658.436255928726, 6920.6820161752285, 41428.948195210854, 86928.83683653252, 12838.230431387254, 26273.45732422277, 37137.36490079713, 360.76251193437605, 86082.39048287344, 31548.47395758189, 99676.08537553414, 86925.17680768849, 94542.53939827325, 58515.805478322436, 162.49643395260804, 39975.44606765775, 57794.8345353605, 69198.22092534522, 52236.2276806798, 17632.991737975466, 56079.39276072787, 52690.15821590869, 17047.30428301322, 16402.859743360565, 47420.467359511276, 48948.61247254951, 78044.96895112522, 17316.868164787535, 81766.01745223912, 5927.447741629632, 44494.07529316578, 68490.62514703521, 87894.55263246903]} +{"id": 1705, "vector": [1353.4931145712803, 66436.35533383336, 92662.57224132483, 92020.54888549598, 53856.805516394576, 9390.232176157155, 66931.97719213968, 37510.335858952334, 7014.466200856451, 62854.627973382594, 13419.243410236848, 38783.84997954762, 9306.53488511951, 31495.21653930729, 53302.10092892681, 57995.407632644215, 15911.506029806644, 85689.97543450475, 10540.340335152332, 11506.618313202676, 77616.01268848474, 42234.47622174662, 70713.26257879865, 55711.86720390921, 26886.340330486837, 13498.946723709681, 59870.920444026386, 58882.9234444087, 29772.223644065078, 98599.8254921131, 57395.2351429863, 72306.6281245347, 97282.0110967631, 24077.617052978352, 18469.915160234195, 26953.366867605855, 78797.53822675208, 60359.75860908176, 33979.777250168154, 64469.00778732982, 38023.129102772735, 57435.6195454566, 4998.720569020698, 76548.22672533247, 16296.719530617087, 18972.65057221288, 28546.642266199317, 74431.19292919897, 25779.314867045454, 30506.440584807704, 56346.32020373037, 93822.82776155973, 78444.60609523517, 5348.840118931242, 72729.92164452728, 6675.706975104734, 36080.036237078086, 68242.44462453229, 82905.88696430791, 40403.92066313086, 40229.69185828097, 3306.824676786524, 39139.101173762625, 80717.32610715633, 34092.61882044598, 68801.9963758421, 87154.10411217892, 39198.70425094111, 44821.506161940524, 21294.137728988717, 20486.483046141646, 2180.8187558810087, 99936.15218044215, 70273.69853762841, 49984.36396954611, 5338.648422560399, 68242.4949942538, 56721.123489019155, 23235.306902999375, 82033.69667432529, 58805.730304708224, 62651.42785828977, 49330.81217338198, 66222.89797618706, 66573.0162186051, 7230.039087351014, 52146.62896426733, 49350.58996767151, 49476.224689376315, 79989.94635298, 22643.376341984924, 32827.34284331337, 85003.9952106345, 91090.29937138387, 78926.68008131217, 9831.35020341588, 22101.100435058408, 53336.789827473876, 8834.984354130338, 9460.203484423746, 84465.55947472507, 63288.48807263873, 67624.50962867627, 49266.77411840359, 96297.54729235261, 40212.629960554936, 19898.542720594804, 7870.211848964536, 67741.65396705084, 4560.943317916666, 31705.251513781484, 96529.50920899773, 87334.10439337067, 43221.465616894784, 98564.54864231752, 14724.687645781309, 5397.853274242215, 59181.50126726167, 95401.26951967529, 1648.6937041807037, 46234.94026257813, 70292.8226659879, 64830.582533479916, 37046.53078764914, 88040.84430263836, 95620.81550642785, 62947.628209317896, 84902.04487557948]} +{"id": 741, "vector": [74455.88710948799, 63704.940724237444, 82578.24713564008, 88367.87316582013, 94911.07514178763, 80773.50440806206, 30807.20906635742, 95243.56695339094, 52234.263839546365, 93583.41923213236, 54834.13528985047, 89333.15398689195, 91410.01703862411, 99553.42504230925, 43716.77090403216, 68037.20310895579, 20520.871842920984, 57247.85308676385, 62816.94162036318, 95879.39926037712, 42435.43511578082, 79040.62245216391, 40547.200778577695, 46578.88828863083, 46136.1600034527, 11455.506098544754, 59811.650242184325, 49636.1848614007, 19072.26759174808, 58633.26333272953, 66221.98678120985, 65854.32285394584, 50510.87898201948, 42927.35687881978, 84068.78120360995, 48372.828717829456, 94123.3854681187, 80800.35831937076, 51351.6194822517, 90189.4399890761, 77175.15913170531, 17119.322620346666, 15324.493720537546, 33294.50859834594, 20524.54531497584, 1679.7205922803316, 70823.7666743207, 59508.58240000602, 24119.716550838722, 34937.103833118344, 53303.86019734312, 26141.01773231664, 4168.060161624909, 81304.64321834133, 25152.44645873986, 92022.5112343672, 51177.63194356434, 31828.699563906946, 79613.37686069254, 98680.20243031332, 90090.59152522916, 55499.932045132984, 5269.44963635817, 71015.53771263946, 57195.99907564278, 26202.49487917533, 65257.143755288926, 92270.88838017937, 37135.71614709258, 33638.49271769344, 17818.949040829, 92750.33737182505, 36724.77061062554, 26310.431708460557, 90331.2137604376, 98628.38799721461, 45848.539672582745, 9017.067579223192, 76167.97272991223, 44623.59811406357, 81350.831894508, 33189.482147837094, 2871.22033736813, 69204.88009492405, 87365.42469297053, 95777.5036931258, 60661.532379269025, 22209.05113938495, 72351.25339297464, 49285.579038848125, 91859.01036458084, 67780.1755829337, 85351.48592661614, 75966.33071564636, 53474.48379341631, 57667.42743686052, 10794.747140114514, 45090.19372244398, 85608.49064823585, 37896.51936101892, 96383.85603128707, 38005.49008382873, 11293.019834342254, 40410.138584146735, 87690.06736657812, 65651.72776614541, 15604.026959150364, 20626.038391711776, 46737.381802080854, 16068.658783441759, 96706.49421756747, 71289.71978919624, 22624.994021893697, 69117.15114005476, 90909.96273793637, 51257.61863963365, 95573.15660140688, 44516.68478086024, 52347.92177186389, 67849.04258624074, 74794.92513401387, 32366.216153724192, 19560.766169880084, 14391.529625856103, 16375.426903215684, 27299.400493170568, 56645.889727560796, 477.832964160696]} +{"id": 1989, "vector": [68298.23914258125, 94344.42943514287, 68329.40151830376, 44817.72148446899, 67803.97515963844, 93714.5582557539, 4360.899201522939, 17574.448077183748, 57044.934132167036, 83554.04372017941, 88187.92450085629, 62710.58647051885, 26377.407406254293, 51618.31527066495, 48491.54915983206, 90910.47876008495, 65225.450197830825, 45729.30088270993, 28535.614437250046, 12347.550026943189, 75339.49161705376, 60746.60072342258, 36472.70753322395, 88820.30229211098, 38477.91652777873, 15915.050767696592, 95431.5068831192, 13628.844428358356, 64802.08250551529, 72789.32257332077, 82106.52031087688, 54623.338192114665, 79730.73098048569, 50787.20549346556, 89036.71939940774, 41069.474045555675, 92408.7580162689, 44673.176600942075, 37926.04895244488, 40220.82682775976, 6445.105489805481, 30915.979470544607, 92886.90385577822, 50566.76038849601, 69813.60135884273, 25559.877683295806, 49864.699646877656, 48557.302081900445, 33564.85360760193, 63424.553957681164, 37314.32938136883, 18156.95299490886, 6401.2235784259965, 68392.99989695477, 11122.7471353587, 45348.38205747195, 35709.95792934125, 81433.90800240083, 57603.621619690304, 38690.86626836689, 41645.34256040732, 15091.15635324355, 722.0372861731162, 46362.02068162172, 30642.188297970773, 37742.35865864214, 2503.3477836070792, 9717.437078609848, 92794.603860914, 38376.19344452673, 49732.55064064728, 95603.74370359635, 35068.04588520188, 76525.22124099758, 56688.3283911934, 61396.58645802085, 57244.729986234066, 82074.45652125389, 77978.32589790793, 68958.71328161033, 41148.098225461385, 74031.01254821436, 66325.63961315215, 21796.369804574733, 36928.32126259663, 78781.7504982363, 57004.98283097526, 15296.215403280066, 4992.7751292914045, 75338.83065345068, 48090.651057814146, 78674.28637718261, 66969.16842675705, 40193.46159372928, 27596.425023073436, 19324.768311367425, 61567.00211449164, 16299.270785860565, 39106.52290246922, 77493.0443791808, 5513.829432832063, 51230.18515176165, 10709.476578089383, 64373.1784783386, 50827.73443129998, 86714.85018809803, 68991.68742500796, 57565.354785725285, 5955.528555358225, 88478.24672198077, 8307.417766434311, 53034.676912192735, 604.2436037114074, 47627.92722513583, 65121.50487544714, 7598.889302797185, 21152.618000671762, 92790.69582981698, 97074.69164851717, 28192.22052652117, 93360.2446424188, 64537.48631095173, 91131.67232808933, 36226.82494567452, 9480.911089140776, 92179.10923785696, 34525.4236529096, 47081.77776611947]} +{"id": 1393, "vector": [23918.36807817549, 29391.86789141486, 40084.472108194146, 77505.41165905847, 38333.28510151893, 7583.5853775481855, 27006.465675640567, 14446.889412930963, 20282.62804247629, 28417.20189234397, 91384.11935408598, 35461.80067482184, 55453.71435686786, 74929.88450929591, 17989.40181704224, 40265.779242824996, 80896.40113940276, 68340.56392124548, 1694.5755209269553, 4968.779740176233, 9670.974463732884, 28074.06558352916, 24198.928355728156, 10062.470357131493, 67745.80425356628, 31385.43284297408, 12759.495925079356, 84559.68949372534, 37654.621857653445, 12172.07694775826, 75436.76155783409, 39998.880252148105, 37472.11797677464, 97248.35629773393, 72402.17117857288, 39942.41524932122, 83879.21574572395, 58984.555332733544, 72757.1462617077, 98565.82003195475, 3852.569384388427, 66155.40857710165, 30995.094315506012, 36839.37144120302, 27539.0709660854, 61391.3557951426, 90996.82869743246, 38429.60529821788, 85537.67969789883, 4572.921718218825, 31028.315654091755, 59198.06051498894, 46461.92900969677, 89935.10277108705, 15817.763162567711, 37637.80713033903, 11089.79157102542, 23185.14196706252, 69443.53234528101, 32766.941356921685, 59268.70794364943, 71865.28759029563, 6103.436937144613, 32620.55555270351, 23955.560525212295, 99537.9194788963, 20707.13955216783, 50250.20624081945, 20904.200849613175, 93030.52823318611, 19400.139823850892, 19590.607995437127, 43610.9219243358, 55343.803530975325, 86642.91932416473, 60380.528938494, 26067.62340818999, 25904.596516096746, 52571.902483979335, 45267.30067288438, 66054.2612721446, 4975.263100439509, 58692.460253324316, 81052.08309241317, 99442.94061915751, 92.20759165421421, 47270.887489429435, 23357.90057113326, 18251.677174329463, 59687.17397761558, 85624.58785205589, 48049.08005582567, 759.0365646726882, 1888.7779736836085, 19393.86733979922, 96579.76574133574, 70029.48613660596, 91081.77704313511, 8290.293317674901, 99945.40544091085, 35426.411035506746, 47742.04915098146, 24352.765210017023, 16061.876459525103, 37944.17793561006, 78106.03287961606, 38922.43192241416, 85991.81030684926, 13126.837922461555, 58434.01254398329, 373.93834323744545, 88648.19642065225, 59100.1852694653, 19407.42674016106, 38040.18816973205, 25111.567496231546, 58483.548618257206, 41404.74281945058, 82340.95049541561, 73494.69879195372, 81808.17178900556, 11651.122797297054, 12953.796139839713, 6386.890466239736, 11090.513502360922, 45009.20555874268, 73718.70418371576, 34902.34656013492]} +{"id": 1741, "vector": [55118.86467440291, 62184.422873549105, 89395.0517143046, 63863.95136097987, 31014.628160385484, 21231.166464965732, 74664.3304929392, 79591.37330145607, 88264.01522736896, 65340.634386365804, 61712.92331202889, 79683.66544066013, 53155.596839937905, 30824.479782142676, 86332.91812484097, 69552.38719758883, 50289.83819547742, 61754.94111759885, 43011.746106379964, 43070.90241613827, 50263.413448652886, 73846.50615319048, 38651.92312212505, 4981.715630546746, 5047.2812916335315, 99373.92641218871, 41089.877601207714, 40118.41303008541, 82293.4413240782, 23282.67532291123, 39111.00903056036, 13932.427783387902, 26691.865161682148, 82477.675637346, 87620.87178133051, 91672.42091906039, 89458.3973360016, 39840.697367419176, 93397.12908709148, 24484.931801050014, 89797.6289756641, 3020.549615615542, 91163.27989590708, 32928.766244294486, 7597.870656319761, 96733.86337023138, 44409.31990252326, 38177.412719897286, 3603.4193044854333, 64116.35624267009, 3362.78793717677, 50641.334416188554, 41604.033428686016, 25349.76567147883, 2964.3535961498715, 7306.99893622282, 82659.4186196966, 94327.60855767349, 12499.710976757617, 48063.544354519516, 38149.102871394156, 44301.826372939046, 44779.96454239706, 85588.07321004996, 91434.35507261178, 24577.22484298467, 21329.450806055338, 25653.025653978544, 49196.45925046635, 57479.50599605307, 40054.380079485185, 94976.18815283761, 54404.23865945222, 48248.9425318534, 24529.010642977733, 45884.928930294576, 44622.33487537302, 81167.03042856604, 78675.09453483092, 92031.9371577997, 39291.41009502641, 67854.35328720033, 22415.975104999274, 21696.133479094282, 69770.23120191637, 27770.22388203714, 47175.581798633924, 42416.69384845701, 1717.5278155248552, 5485.387131277886, 63738.91449979044, 22094.12481188496, 54992.33158814701, 95525.08943468193, 18354.481160186486, 70038.08701840224, 29087.155006566445, 99100.20079014837, 15557.896020422013, 26965.9617602261, 9162.177035387409, 16393.21244669213, 68079.5982217834, 3976.761188284905, 76180.62326008196, 35889.892388454195, 16421.01491051976, 31084.84716744362, 33673.642133184, 21079.78524315901, 39457.39795471443, 29184.054490712762, 11491.134569142114, 61806.47573643191, 71456.5843936266, 18293.8936508653, 87394.48454348632, 84889.71546463807, 87217.87456529103, 40379.93870111146, 52213.53995845275, 76450.71019004013, 52152.891426978174, 47909.35031504591, 83668.30405908034, 25602.90748265831, 75358.04742191262, 58325.72028844305]} +{"id": 475, "vector": [62465.39214993567, 88572.3986055589, 20861.509736325403, 87585.84345759438, 31376.521226991583, 38910.9070979524, 30899.65614908946, 63609.82019958541, 38852.80673067384, 82613.01348018207, 32471.47811089469, 72198.25320012783, 44869.58826423544, 36073.781174216216, 17945.069782920174, 9620.608182062606, 25215.7689501609, 8344.286419805047, 43850.93011887584, 64428.03306266925, 25298.162391992006, 49770.10826534871, 1402.1217137806352, 8117.633604835639, 54891.586697996885, 78312.48482320615, 97533.58789546872, 88438.28999592697, 26003.539242345043, 85975.37619382402, 83186.86933190169, 63469.93310187633, 10711.1199305712, 20707.247059735022, 74920.59660744682, 26760.96833615157, 74690.82360804497, 30814.608664349442, 33301.38921183082, 37762.14197700788, 93820.053014801, 80595.27988756963, 41815.477264689696, 51574.35143550904, 17060.341919925126, 79467.91914646568, 68554.00714997711, 80294.62431040272, 3351.6394478895318, 43875.83491425153, 77.73872032758034, 15429.648813064556, 38851.634416849665, 36428.4516387213, 89364.32926638743, 38999.648896551975, 71489.67257102406, 38684.16032645294, 75487.70282224733, 48599.452229831106, 64388.92938121217, 93410.48561350284, 91279.09340732344, 84733.8916988375, 45921.41750625801, 32399.30307431518, 31997.039421039954, 68505.93283164065, 62964.4018393094, 46622.12434543712, 13455.24904798392, 4934.944752608916, 39362.94700748118, 60419.86000566893, 38974.58118919399, 61162.93252404426, 54133.104990027605, 99391.65739625927, 44920.16849989085, 15868.410339679984, 55180.558654352666, 6724.972939940355, 85363.01511818047, 95285.03197142376, 10164.21953346257, 84823.81202610773, 68714.40288143892, 69685.30704969921, 87018.39205400016, 65875.81057328913, 90275.30464621198, 98055.67907157032, 53652.225598905825, 89859.07866525573, 61660.642461012314, 57026.24332668792, 15479.661867070026, 83764.65500082629, 31429.960248677537, 42420.059487790226, 33907.090438493695, 76612.772299551, 13503.714543739197, 80118.39375369193, 35622.92324397883, 6949.528091135693, 30812.507638789, 51476.978550943095, 39301.36016101948, 3173.9522203447336, 48662.630328122905, 36677.45842073196, 12745.832147296698, 86977.34344795569, 40622.90143518516, 14476.357404829643, 21309.14425256313, 31093.2925456489, 61111.76221013689, 71696.72897241275, 11608.146339527304, 16311.509770163857, 69980.48812864641, 11858.395544332823, 78736.35683097944, 97523.17979614668, 55298.27393893208, 21942.671112928736]} +{"id": 488, "vector": [93159.92605455898, 40094.33259525046, 80053.02854722238, 33642.96767703444, 86799.28251155138, 70290.23789197819, 58316.50051122175, 34828.14637342656, 30287.03190886972, 28453.222479846518, 78793.54395473773, 44219.7820400039, 11627.54090501511, 70901.89341099914, 80879.36921990084, 39598.68642881733, 49766.711357166394, 41573.057688775625, 23427.248154034787, 48749.88391951793, 14997.08859267087, 62508.83662956849, 98945.60592809695, 1545.940484081376, 1799.7633378572298, 77020.33048975441, 2389.2941387395726, 63233.33579065211, 65028.98945794806, 41994.51245981634, 28567.92747418636, 14731.851210103054, 96908.33480418782, 51340.135643833906, 48274.20658119461, 89801.92296843337, 39923.523900640575, 39478.8526051982, 76948.10232155037, 73721.97051093861, 29100.331930589175, 37481.49019470956, 96887.60606950498, 35284.55986049097, 80233.93363638611, 54307.7528679444, 50735.08915053773, 11692.541679882961, 75571.2746413711, 16926.217262704824, 82309.56665549728, 36002.526309524364, 83016.23142889146, 84939.60311451391, 24030.994258835803, 82400.4275082401, 34786.357988686344, 90659.12286459317, 4168.619178442556, 71081.09053364738, 90119.60623146032, 94249.59858287327, 93735.19267044259, 51052.477905300744, 113.28737440841064, 10191.938440247073, 85457.9707509273, 37489.27322549298, 43413.63909413139, 25768.458761128088, 73060.43558335483, 12201.73977392618, 287.0529497982388, 51574.41022386543, 35192.09672544433, 96111.32445686475, 59475.28593942731, 28733.66993669344, 5846.992207142787, 57613.71485239355, 27806.644692496073, 18151.145875834696, 81632.36290330866, 79925.51812567186, 36740.89835038573, 53751.193261222405, 64431.21445289343, 13860.852165767945, 39240.00315722223, 10640.999598960998, 90671.25143183905, 83149.44735837511, 76765.9063123111, 70160.38014367675, 37609.52222645417, 91969.91646198141, 11789.693226163534, 56834.16379077314, 6632.110441807648, 89319.42363455324, 74975.66022373135, 23126.214339382223, 40397.30862423654, 63775.14807411993, 33667.52646302613, 91846.59417594108, 41781.39268872001, 34344.44980463739, 82850.13234646953, 50947.38901609971, 68406.43755659097, 39669.192104370166, 69622.11746801047, 55143.741387181566, 96485.7551948587, 14877.635409216273, 74003.221005763, 75357.77781946251, 87174.81443333863, 23174.30323371482, 67386.65644110227, 12949.768101627356, 89251.4198614369, 59919.64297680952, 41299.83998315336, 8409.768072351786, 49779.683454305115, 63249.29088799498]} +{"id": 54, "vector": [33188.11260079596, 39348.807334234436, 22755.854065938496, 76939.03470227572, 74263.35237100777, 55574.554405075505, 64825.7056957952, 48343.98496957768, 41064.363645220605, 13160.436919052976, 13069.435706503251, 8588.691576293295, 74967.84490883074, 68365.2248066024, 33256.956398168666, 15526.839848295393, 49097.1583236701, 93573.83619243986, 70085.00192389448, 2580.2922263924775, 35198.56803118942, 6040.527386749228, 21003.334347281645, 30433.572848132473, 25183.923432870346, 53484.36779126169, 35743.89981256174, 8815.8574890441, 32493.591009876276, 80212.5469796876, 14370.517118636539, 89336.58950212471, 70421.76446683839, 31873.33618551407, 94287.78077617456, 8995.850700503637, 67912.76256842673, 90245.00101025232, 81188.56264111112, 54602.38107726706, 57847.314692955144, 39731.288485524004, 51292.208118881055, 75522.84715782717, 64026.4414038118, 60425.98350972558, 34729.74411474836, 74700.26671535833, 69134.81253897828, 71237.11604490055, 61511.217451583114, 53756.03987010278, 54033.96566592919, 28854.521241416398, 81623.55231480647, 71708.75591964879, 68872.18408105407, 47275.73030182238, 94836.17828986149, 45731.033319783906, 12422.091374173295, 65923.57541069348, 59054.4167082577, 90125.24904264591, 81978.13713207263, 66381.75270775193, 88581.7839905876, 27400.626476912883, 5625.034140246099, 32303.42235169362, 88898.06356303053, 43802.72820996251, 66968.52337488337, 2951.6855818302656, 8415.480327563508, 15216.03767819516, 29006.566308658053, 87277.6807466943, 81387.78605258345, 25872.276986700148, 50992.35351553051, 26601.24230604094, 59975.026795726524, 97054.7918892623, 85227.56349760287, 56618.145783363136, 35897.85308773033, 67997.44843929823, 31368.35017386178, 73828.84266557865, 29780.38231281753, 47437.675131825476, 35476.93588355124, 85343.00860075651, 94051.85290760131, 76172.7892737787, 88196.3403531661, 19620.33860450283, 51897.77781219364, 55778.826791635074, 42648.16190774254, 97886.992099562, 27793.20157631159, 6280.547293658256, 72390.27430994024, 85862.03957723771, 4503.013901378916, 93895.8330833097, 31969.69375614391, 56664.425770055095, 69988.52720618536, 23430.188612000624, 23770.42485543914, 7484.130866076266, 4598.38008406116, 66971.06458778352, 75681.75403341872, 24298.390315523568, 29714.35825496719, 47656.940202291546, 94001.4170447211, 21262.430537466815, 16319.491254378949, 11624.336743609698, 23211.545146511613, 8992.247705205558, 75144.5177507603, 34012.02306953668]} +{"id": 1718, "vector": [17226.383590479312, 81216.22992706491, 84312.12981729134, 78552.61090318394, 59298.968800665345, 50776.770498808975, 95663.06039208712, 23527.101222329704, 71724.73352527346, 34460.905470609956, 34464.03622142342, 93183.63169499065, 52942.63217379508, 54558.61086716478, 41462.21220351718, 33091.526676400776, 57326.66025935658, 46822.369267984606, 32911.23748890533, 8007.120424929149, 17949.84997833726, 64327.126617790134, 70617.94552524174, 23025.93363275175, 53965.62987884625, 30815.28675543206, 2822.24580042334, 8092.505401037831, 14120.80239958149, 53162.35560742337, 18456.736563687926, 26020.545810088013, 18339.835016696958, 97929.30203205877, 54254.97616269318, 39737.16928260174, 69871.74872895601, 10396.360036542406, 4905.923115577782, 37469.06052260806, 92463.35392979033, 69093.73203338074, 95034.77852235611, 1833.0363969167984, 82124.6828115879, 83512.76045479639, 4674.771182717585, 89964.02102196796, 45238.835859323844, 75610.68518873207, 8234.756014890354, 95105.51047605589, 10560.233711839173, 88770.77261821693, 43431.6700739026, 56954.19509952408, 49642.616621688816, 14921.19377877047, 32749.817450342223, 16665.560226177746, 63681.771252300336, 16430.79037451963, 17066.03705345705, 80344.88902663144, 70683.52464549421, 80207.15858468624, 64754.32897321557, 21397.181668056364, 32018.194656347045, 76088.74328239624, 12772.249690250781, 69079.37012658647, 44293.74651947464, 45853.616422755214, 79705.318314861, 61044.64110475014, 91472.51741062485, 7171.986665583241, 1052.9830166042918, 21539.243537826445, 24083.785065175856, 72166.46476814176, 19186.477452448693, 10958.03463570697, 4474.242303945252, 47855.6761220504, 45282.66319957518, 82707.7330685531, 89130.48779076526, 49899.8178043567, 43747.57181123507, 85687.13285083321, 96054.67601545888, 86817.71797163364, 27840.902652041055, 44000.136730786566, 74817.12960142571, 15196.319686778392, 99558.76574190227, 96767.384433019, 5949.308593266145, 75562.70618261381, 74350.86185303207, 58367.53890692915, 71518.19936547063, 55204.74993073189, 24373.413659906128, 97001.36678489526, 80847.00085895714, 69825.16267057163, 815.003841472195, 70297.3470570638, 51853.65909971033, 25512.92416129082, 42265.378060079114, 39671.57626680968, 77709.31353107258, 74568.05844626341, 95200.31401181038, 5003.104460906238, 78527.04597007173, 34908.13157688369, 25189.403850882565, 91858.90895665644, 52520.07773222668, 15325.988307605265, 26191.689081775105, 88543.98948794838]} +{"id": 1952, "vector": [22161.49089386491, 1303.5568973601253, 26670.6748958213, 39272.015848237475, 33651.84229235487, 9684.57257663554, 1278.277628787572, 22094.805651724902, 74073.55655730251, 13219.680061605677, 70332.33029412318, 55144.81355016886, 71635.17851441978, 59681.22332704398, 88874.87472283155, 86903.20142724735, 83176.34813165941, 52255.31455441177, 96943.33451473342, 18348.803808657365, 84036.7833848627, 67785.37473260531, 15835.890729709667, 95829.5746859146, 28604.008891816058, 21646.56428312389, 129.91625952795127, 90567.875278877, 42463.95193578805, 60179.68648883739, 22144.206969799307, 11968.281634114552, 19750.38783873594, 17407.90877447207, 78437.80408605382, 33266.047270582174, 71618.957320316, 28001.776456021442, 10444.166693599554, 99613.38810602426, 40303.66809601804, 91855.7633651442, 37132.476408476025, 91082.38066142936, 96167.95977138616, 41805.79448185029, 19614.91714849727, 56046.82169701584, 24850.862356771842, 53393.98607575285, 54651.96777292445, 59225.24380388414, 43723.721860101396, 80950.52363889568, 50959.295774769766, 42014.69468754039, 48902.830003438954, 51813.72228712707, 81815.90512022657, 20885.011001928666, 49230.8890535326, 58546.555543250586, 94282.81564202692, 91363.6699440785, 36150.365604972634, 91756.32230267735, 76405.62199930295, 47374.9024994565, 86198.04886128104, 86549.011775612, 5363.946747381654, 70494.36382439181, 87348.34828452735, 51102.578901407534, 55698.197099490375, 14994.323718008518, 21013.475912047506, 26776.020042875214, 3742.9237195173882, 51450.679126297415, 761.0063527605848, 37416.4582192417, 73522.75758525147, 88011.48845212733, 8386.650816352203, 29586.250755854493, 75950.15281693192, 17003.82070097817, 97738.99560474673, 29497.815614285537, 56727.89527414896, 11112.55945631594, 32900.60501008327, 26290.924159437658, 84063.8658500314, 81483.292884663, 80120.12468664958, 72948.63483084032, 75493.1620096863, 33402.67885912637, 87687.313643006, 38034.73330783256, 37935.29684527352, 25793.805789460865, 3723.931519488588, 18348.633803319535, 49235.30424793365, 90835.4760374279, 96692.2035413176, 25391.341248388533, 41477.9306053089, 38976.336932439146, 30994.600793272242, 22410.61783072592, 43753.09889454889, 83512.4361428638, 78609.86016155858, 90143.65138692438, 40139.49370569053, 45151.7662701857, 66495.81614649211, 13743.416778948691, 42340.06280448386, 78843.79200342808, 2527.0216297422653, 67611.25706473106, 3288.2138431538865, 12555.567873502127]} +{"id": 834, "vector": [32005.27361107548, 94235.58010005155, 90796.70394303354, 57180.95087585439, 49188.99585968484, 7601.323611727584, 68969.9928035745, 75181.58708450606, 74250.84658943138, 12484.853308728827, 49932.65455600916, 4033.4603112525792, 92573.84075905253, 12553.891737411604, 72058.22968671143, 23941.726055264167, 51899.860951230905, 54598.48326786303, 13428.694943578246, 41908.92533815469, 4981.0807798638, 50037.02430665408, 79031.708363573, 11794.611792692178, 32381.891477728997, 81099.40355766594, 81593.23109785606, 11231.43556525763, 53810.39437926629, 13483.850468652903, 99969.84918808773, 85283.66860436917, 73310.83275275795, 24109.440920849178, 14355.472831365212, 58806.01230762177, 16356.728424394796, 96415.33504972239, 26394.87104717454, 71457.34080297234, 9917.647417608188, 80686.9257477829, 34986.46141353701, 26788.957753945775, 27221.938944718837, 35583.826357466045, 92353.6687096575, 15496.606539212344, 75659.97329201469, 24330.347541147403, 7596.1769320795565, 94190.8795914438, 75729.16976891366, 48220.48372068077, 75383.41869298747, 97028.60185652919, 93714.4830148726, 13699.922505240458, 67293.37494988405, 91552.56194751407, 35068.30558843531, 71521.64210000538, 63765.4700530098, 91142.58721538364, 57598.59145635291, 3898.133172479701, 31012.519736442566, 98115.08754559171, 29423.711413088793, 21888.157432906908, 91629.51712252887, 23184.10933856775, 41823.43762637152, 62774.15679705225, 13794.396794238639, 3236.3572360181615, 80814.25346049927, 45047.98526469931, 7628.774353862855, 25387.413880213127, 12393.12071562194, 45242.62696367037, 75352.26367412646, 42440.38094374479, 18984.711630847072, 62020.52611071516, 11178.018428514424, 40539.59926937084, 65731.76649537445, 5276.9664074184375, 28972.463067097266, 73663.0498206582, 15909.504039941157, 13240.464544829367, 37867.72242019101, 75890.5122432929, 18780.69069361753, 33288.73789787904, 97527.25976157007, 16232.04136231028, 53190.117643900114, 831.4585350044812, 94276.67906054045, 97844.09010524391, 94164.1666713738, 97658.62665423607, 18258.516367856748, 13133.77149472228, 46104.1714156783, 61361.968187728235, 53798.5568793634, 1649.0062574705867, 14.136954626187137, 33025.049893980016, 1225.296249022456, 14863.948350879653, 68315.99626929892, 33081.78535290933, 97158.8481536415, 61128.548214905546, 34177.60437855704, 72227.55007543036, 64405.64777543269, 74740.00667657236, 65476.16906623757, 55526.87319754694, 55510.10357719345, 93018.42897224714]} +{"id": 435, "vector": [77738.24409768607, 35658.14320503384, 5302.26871851005, 28516.070233803413, 88615.73308281567, 62301.7985605201, 14753.327087241374, 4058.217001973963, 96746.24296549495, 49655.219798420505, 8516.519407703272, 76298.6965505596, 72103.11798732037, 88181.69655652388, 92321.94269393518, 58702.491912472935, 24284.88287979195, 73415.82873796669, 2871.7656475372055, 22953.80478661293, 71769.61342334138, 81208.85527805636, 89659.50931253354, 59590.13083464474, 85144.0686103363, 44197.55302466245, 5378.418429990106, 82345.32069575677, 89329.39474925511, 40649.15488767309, 1257.762456244449, 23698.72207754661, 91229.47625506812, 9718.02484021279, 13842.84055591587, 26361.072070484304, 12680.477715437799, 62109.06546409639, 32646.93997713892, 36967.62066278475, 96223.10772060051, 2504.992652755833, 76293.68402656712, 68170.78754008112, 74253.20353764747, 58044.880518532984, 55153.27368104618, 13781.328975871866, 6237.290549681052, 24244.469787516497, 65115.38035220581, 53593.71418441089, 40275.39422566891, 51641.35778503949, 59300.257886713945, 78229.86690352003, 32819.114207516344, 12040.11573642737, 42803.13869828204, 29272.680321151223, 12370.046765599353, 16328.274185933467, 52065.59634193628, 91661.97212765248, 97628.43230046159, 31960.733359116854, 88754.73918784854, 93147.96624714685, 29828.261594998872, 44862.59062134439, 37977.14281486949, 84561.6390035446, 5094.91641574028, 10639.674673513055, 44779.597943360124, 12682.018290752605, 81008.68765489441, 21209.937896494823, 53785.93081336073, 92655.04488946323, 99839.27123754757, 50572.35407696931, 21221.22159218983, 90513.07514269774, 73109.12641264562, 44090.570615434255, 58725.8046169551, 24782.08362770967, 58036.26004407884, 60303.92267632954, 85708.586795629, 47894.568382482474, 17019.372546425428, 58219.73332456748, 37266.5608615934, 49312.168313540074, 70089.1622504343, 1775.8901012851936, 47317.352990924286, 91421.85190035093, 15783.100271391859, 37206.31269155285, 64874.58185320718, 41935.66175292295, 48718.860718938995, 29814.342446374998, 11233.624787290208, 73738.92961189084, 5314.412251233059, 57892.65056559198, 77126.19122077848, 27880.930959947724, 43725.134656763854, 74574.46265838646, 7157.9315757341, 87952.49254828326, 67922.96658798492, 17683.758053387, 51471.63492461253, 3551.147665803156, 27507.046137860823, 88426.49333604574, 58122.231381056685, 77818.79726034729, 59634.24094305838, 22888.61749612674, 97798.4493379665, 43092.83059931999]} +{"id": 596, "vector": [67687.27629733995, 86453.30441400848, 12897.48902153226, 20452.474434969547, 34993.98319996327, 34587.772496986814, 88733.59084930125, 5157.003177721475, 74340.00841546002, 13606.167945513225, 22010.914356048263, 92391.4234890674, 90505.19077021994, 47679.58641613366, 98708.6740697412, 34956.140186277175, 53834.90692012399, 80793.29307784988, 39615.46427741669, 20221.653710723698, 46915.519764004945, 37639.339483903976, 91003.76433275074, 16475.443650961275, 75634.619709969, 74189.60153086681, 84592.16702450428, 92898.47237757426, 8727.783548089928, 97232.0092899635, 51985.335163129275, 13971.121244235119, 20461.47673988603, 92450.71038434267, 49500.971375415706, 4740.553456667607, 95957.30284260254, 32507.616200999244, 30901.37319681535, 52521.832309531994, 47066.08918352925, 92488.45751783559, 45691.24724552914, 21593.548439676768, 29425.682898837567, 64333.430427314684, 84465.84635753154, 5866.691779796218, 95610.72098981615, 49876.25767627188, 21803.866919459393, 45231.4570518111, 45842.742352840636, 63607.81497389192, 74895.45593627892, 77065.85690932092, 76834.53183278005, 19940.122204933108, 64639.47282465199, 56528.526690724226, 14634.390128170704, 96887.68332140386, 31986.671584730055, 86900.35543681776, 26778.15421361397, 44272.86251289672, 76502.96155740865, 37978.58630683606, 22082.273425741627, 14108.76876460455, 25245.229778144752, 48556.79826445001, 40472.0216206375, 58145.14928613892, 20698.658673125246, 95852.4863443644, 23733.138785423656, 32650.146651107658, 13963.929166685795, 87367.85435871362, 33977.69572568374, 78885.39615268153, 38944.93602947521, 48404.66146203707, 91613.67802338766, 4634.454066483618, 57671.54314458984, 91004.81146657113, 85424.75278866943, 52641.32113134975, 45055.23906054169, 24571.994771285033, 51212.520040142874, 21025.690302725543, 82113.33042609088, 10526.759552137111, 35366.1162983658, 28696.182355168166, 2991.4284471957762, 13154.737477713885, 50411.25917198046, 11021.207973463665, 84861.18667644738, 53555.86240470425, 94426.2924486273, 34269.257013390095, 89740.9725351523, 72190.7515024008, 95812.54605352048, 60576.83092723525, 37882.94313073986, 22191.763655367726, 1308.4426852634556, 5848.351129933871, 79828.4242799868, 47866.248946864274, 38087.36368714719, 13974.721830206316, 52211.136446681594, 81600.36766155752, 32170.307742876914, 21750.942946626074, 43545.88538065275, 71157.48236096535, 22456.31954614601, 26046.994246514376, 13379.366735072173, 1044.0103645460886]} +{"id": 375, "vector": [97109.37101517936, 94124.26981781832, 6536.619425377766, 40997.66255243041, 72600.88339539188, 78645.17986072588, 7540.470439724189, 49376.12773598643, 65160.07543448074, 23740.00934673465, 14882.899687979212, 94844.887345646, 65711.76324757977, 17099.173369054555, 89238.85796526278, 8396.509963781096, 8693.438654396723, 47437.12148750507, 78918.99770453466, 6765.313854879751, 22928.698931411018, 94518.42513550966, 14128.832418834481, 73706.28903979696, 11174.838487799621, 39150.79981858352, 56864.4531327114, 35636.99280491197, 8759.227805905779, 96498.39059759241, 67402.939767545, 46498.33567895702, 8969.402518868852, 6762.223801890166, 49097.30486015197, 67343.04326329663, 90052.85403395616, 96057.59711854381, 47055.31346451472, 29841.288854520444, 9842.760226220704, 93173.53166664997, 29509.78779280111, 8942.969449780847, 93922.24897536791, 88133.14711034708, 73000.41851928411, 42125.31853811609, 29685.327708493835, 44223.85644112987, 64547.85093934619, 80216.83192695735, 22481.941089158976, 16978.611046548376, 66633.79056164503, 3545.985400628393, 60298.402016134314, 54351.804734037956, 6603.059167492153, 23421.099507190036, 87723.29188357318, 43563.647637258204, 68477.61652856866, 38663.42443326594, 33183.44648814021, 86709.03900770492, 86367.87362991575, 22049.722951656902, 20050.275077656555, 57917.054507802, 41961.03981079955, 68753.17748016378, 19234.62442729448, 84140.66341768223, 25365.48770669996, 40730.93988864631, 11794.3639669433, 3243.483391158708, 50794.911913886834, 99518.94461069994, 53495.8098924067, 13632.043009245497, 49339.483720808304, 52822.74184036358, 41525.4277639521, 90713.51719058145, 53595.60761021721, 40960.47352035771, 60699.78644183043, 11058.427967445306, 62022.86789939873, 62336.15284460219, 18481.701204661527, 21207.924815218426, 23040.71078651756, 55668.30296488439, 37384.532257240244, 56291.39277220162, 67579.08355003147, 72430.03816492515, 72072.53189094402, 72277.85918920801, 69758.92215413216, 83403.90348669079, 54786.682831753074, 95320.32067273124, 82155.81319057665, 79562.83651611008, 95560.49356996636, 91444.0653146155, 29775.325572846345, 60741.51616109748, 71022.28279047577, 4778.557727173172, 71280.62640422581, 88040.39649682466, 36652.65068494953, 73419.8161930116, 14257.414090724107, 68790.9579259045, 98716.07662631264, 70056.99565979268, 71710.72356882431, 50636.737250496786, 4662.9572670730095, 60295.891732164106, 37154.7580299998, 33765.91639814661]} +{"id": 151, "vector": [9434.18614477226, 22215.83320277205, 54073.24459423166, 80596.64104080346, 99645.26163410422, 61574.94637608703, 78493.14000759907, 56907.919707422705, 10136.009600784822, 47935.61182934602, 44838.98697647125, 82058.77838142012, 49570.27102456386, 47156.79281016394, 78415.01439653011, 75664.47755907298, 3158.3847041333656, 22850.43445356185, 26302.438918755277, 76740.19949949649, 58131.80810180182, 64185.56660938105, 56202.25113953446, 12655.590034534493, 61174.349193260736, 17964.263172751536, 44919.53349139143, 7340.75396292857, 43322.78346030537, 34813.33180287097, 18126.058071422325, 32731.309302468613, 42192.70672921558, 47700.340591260574, 43258.294479568394, 60268.11446012039, 65935.1129415956, 60415.03995670025, 92671.18137410397, 78982.34244543656, 70348.10657775078, 26830.678704379872, 2069.3942474857117, 11131.690417458396, 38898.8309969739, 24546.128834103354, 78522.4038910748, 3677.3806653339957, 88923.91095007118, 7405.075964653318, 75753.80062982938, 59489.13016835249, 75638.43783252033, 16429.827612069468, 56562.81093360155, 21424.817027888777, 29729.627740880427, 20227.735505738743, 39237.55172900621, 39132.000064482774, 31897.51074774657, 57710.85554156776, 78522.89294342391, 19234.040629049643, 76548.0877416502, 946.5758425507343, 55233.57864133949, 96673.8192791991, 37134.79872099952, 3497.858607908044, 85415.02884892187, 28917.12969126491, 13584.740451591015, 99017.67134801163, 48376.86939346962, 24675.299338948185, 44997.256063316956, 19127.025482703863, 55636.35228036632, 99789.17541492057, 98013.39167435654, 8036.294742572137, 27146.07053569721, 97981.7861338949, 19806.08867299517, 75183.65262218307, 57936.0641651342, 9662.668659616136, 26538.263801293095, 26170.101741410133, 58188.49868158983, 88195.80539522941, 86323.95318484568, 28816.33850089348, 29015.394375863725, 61063.26358379249, 76690.33720030327, 85114.49105174126, 96305.35222242014, 63432.54499199404, 78608.24517176012, 26621.555776308036, 69325.96573272889, 60912.06809084204, 6867.857069317884, 73980.19258476129, 48545.072583960544, 66059.5650335604, 65303.88404442665, 18268.980986636598, 92548.13808532165, 45189.70581071435, 40538.18911046678, 42167.769562137495, 79742.62118501122, 29590.136421702828, 79227.71652272926, 76331.03562171634, 73301.75221697906, 72331.90667332955, 11248.008893278271, 90372.77884348699, 2364.7946118605655, 90624.9706990859, 45906.96402430586, 86559.94980418707, 26107.565778886565, 705.158885754209]} +{"id": 567, "vector": [86847.74290977042, 50587.91094708118, 63182.91818700028, 56616.27583728466, 83017.87227908277, 71257.48855262475, 68509.00402741815, 18640.22793704806, 54911.89168561427, 46447.119442021736, 17797.40383073025, 72011.43460939957, 52691.05501803933, 20023.118056785017, 63296.534806892836, 77201.96781998161, 99755.02712499145, 34297.06811875715, 12108.168682058373, 28764.718281182155, 18397.33369546146, 97122.20175347272, 35750.661324327, 94807.45316478475, 63834.78945214263, 50315.39675835651, 81231.7438869105, 72016.84025435943, 25805.70420666193, 2765.767330902014, 72725.03710907613, 97328.49326696465, 9658.5340724616, 11182.192421743997, 48186.91981706046, 70396.11694936179, 61347.609526754495, 37719.20920163777, 24386.054924609045, 90771.36112447169, 58190.60405500711, 32865.97646709365, 53129.84186944937, 66804.8378604483, 39445.17203673418, 45408.783620111084, 95426.27680074365, 45083.83883104733, 99723.4312406222, 5741.567764804567, 46961.44173021538, 36996.26938527901, 90984.59649643226, 77166.96582085727, 14211.181799032269, 74364.73299207572, 98053.57847929222, 63033.89957250989, 75217.98171614073, 19136.541603289203, 45851.34396123602, 16651.953868484416, 58501.39385006939, 22836.032287239374, 40875.002692726615, 11665.82895875251, 12907.195346679568, 6773.2773898910855, 24325.47011355951, 98542.81319572996, 14415.532295246858, 56661.63008131505, 25027.813269347742, 99905.7130879819, 25136.323304838435, 90868.41533618896, 66918.82050081603, 52595.15578823722, 10798.759474734243, 16952.79534857269, 72263.75092792686, 32596.974811035605, 59977.53496050851, 43429.01750843474, 78644.03403655547, 14367.060279758814, 40075.8384074886, 64920.10553263258, 55917.77332045125, 2393.342755994077, 73044.97011886851, 5091.663795758827, 26685.561521409494, 90474.80389612976, 9245.689422103398, 57759.88981406968, 4818.961478215267, 13237.3182327436, 67027.66009868446, 2412.0018883683756, 88723.25165557067, 24314.23003388542, 73141.96208276687, 37236.11675632299, 66729.92337180022, 5780.8841659843565, 73684.37167924624, 25729.294472743848, 8317.339603553186, 33753.82562522623, 54830.42244160945, 66127.45071634265, 24167.3808448998, 17236.06552165713, 75382.57169145264, 56213.90433438237, 98172.98194361717, 91743.56159249994, 4590.179479071899, 6079.874172676092, 17030.463072551793, 43066.91619325027, 15243.005343955207, 22473.326163661044, 94524.10944728438, 45422.269364840526, 49392.07227456378, 57488.06092579904]} +{"id": 1337, "vector": [6872.041752733038, 90422.70117785885, 34776.418595822535, 56837.72537553375, 65258.46540155235, 5822.406588495932, 93216.23729005935, 85706.30137775744, 69198.28163325384, 25268.08790379047, 89553.66221586491, 50741.46812848129, 50855.792451251145, 34092.21471120046, 18124.40497907628, 69985.35826106528, 20362.071211318413, 66287.60008543321, 17978.592399546833, 31686.514372057583, 36490.01808300428, 19886.769256135107, 14251.95921861001, 42288.458009383576, 66909.75207736353, 58464.40528231172, 18725.8586892461, 95263.42968614861, 86450.30082391185, 54995.767989830005, 57064.162682400696, 36109.08522735405, 25979.968773573026, 7911.17076510931, 26900.0817539224, 20172.282420737687, 60545.0368170679, 21141.166799304356, 49960.20225005835, 72454.54416613652, 9420.600304599337, 44975.89897641974, 73984.18438362353, 45137.461889712285, 1362.209270599768, 41493.437065706086, 26487.85270688533, 19732.705629976277, 50130.226310396174, 795.5843143512053, 72033.85108850707, 18432.528167351782, 26186.627095870295, 85185.61039845274, 80884.55290436772, 96081.87004907375, 80330.60263298557, 85002.68865038807, 13355.4044733331, 86351.1794680337, 93390.55338371587, 86449.7820281597, 22414.73613467496, 48585.19731333867, 42782.21286378079, 23424.479350517315, 63269.13037708062, 21449.437351302393, 75118.07347019426, 12484.656211815249, 38807.09651420932, 29715.726059343593, 47648.307299409535, 73359.33875646458, 56942.5075493057, 2937.014941782856, 48842.879603120295, 69262.7269461602, 10614.586854319985, 39775.53105171816, 34015.57643440061, 91938.62597554238, 52431.03878097511, 75825.65459928843, 22112.967106151293, 15706.91583142223, 31546.181812948802, 28810.163209131566, 99153.92077989301, 190.3224392447056, 2269.700970481603, 78769.51876711808, 45640.52303682848, 70463.48253695563, 59384.48852089976, 25127.397414278097, 85458.71255642208, 36283.144466947, 31684.888195134463, 10611.111489039371, 17548.570921200157, 22757.644267574517, 21222.67822628415, 63548.28852509344, 17062.723689324055, 17828.835362503458, 40186.74499733849, 76634.17486000611, 30534.86323262975, 39511.08958477659, 38178.33091259679, 95258.54716822154, 88630.64629723913, 20194.619695352278, 62574.65260790866, 31561.89538005033, 1822.4843116817158, 54217.308219656814, 91815.83858333682, 83690.05586089789, 83600.16425984706, 21635.535934402917, 43339.99342400977, 26656.989876395743, 77437.11401788346, 41359.835091176334, 25412.95362578635, 44709.89985528668]} +{"id": 1391, "vector": [28125.65073149752, 60787.21707703178, 17564.41374269464, 66574.93669212097, 37555.72245623216, 75381.83948411657, 98078.58935420218, 3724.43544764377, 28921.9607754016, 17013.443407114548, 77921.48313560468, 27488.00119595779, 11542.741104804998, 9628.400662561799, 47303.88717671723, 37498.81367081279, 1675.4328048814382, 11383.491064517448, 9848.91367714561, 4198.69796287422, 25939.973165348383, 25706.22222496798, 44841.770215865006, 34232.870679355, 46330.740526008194, 59593.21425032183, 16023.987546187978, 20993.069336968063, 83598.76053681446, 45625.83953918743, 9383.996889347412, 79075.30965140912, 46972.60334035583, 88324.35506296068, 71278.47466929007, 53750.06061690263, 58048.192224226594, 99476.11930983467, 97412.70162993224, 38944.90666983984, 47405.77169222815, 91962.64556887251, 84951.69216561201, 71202.4058108053, 24104.07980249756, 91559.0201021889, 5164.587582443059, 41265.2833757703, 72136.26957314034, 23454.211919673253, 31835.97357786191, 12591.176180038732, 27014.59205238812, 37539.56398309755, 98233.81245384044, 79221.86588400742, 35740.37234308825, 45188.6128479263, 11745.502760820991, 35430.583982104945, 24255.326223496966, 531.2854407019785, 55725.78176408075, 71617.08541273892, 50040.47197089364, 14688.062299706373, 69192.2483670153, 29756.222425335876, 56565.16810773259, 94544.76872017536, 30766.7068875764, 67332.90561119144, 24736.56409054692, 53433.2307724745, 57061.884732900435, 74986.53941310852, 22388.22930607932, 61737.043814134086, 31537.390712103163, 7732.61540974397, 67035.84284236844, 12044.938445196285, 61270.90984035599, 82871.03357757855, 95773.00157778634, 5034.532786189349, 34721.59244527524, 52141.14408293096, 4097.035862861498, 94879.79951360934, 8723.028636942186, 76297.51824196982, 31492.517796170883, 35460.78270839359, 35432.77235709793, 27865.407306565903, 78789.47184142533, 72550.44051411566, 84765.74366557378, 38634.15626620645, 67995.44692245324, 96273.38063955403, 56652.98295371689, 4487.194633109948, 19637.291064812533, 3152.558170430164, 30154.72836458777, 83329.28307498294, 53222.015419594296, 22376.429356337492, 10400.064220383343, 42606.01950310581, 44019.14216107666, 87586.2453974757, 96416.09721835601, 708.5120371095766, 65283.62996255203, 5620.2701287574655, 48564.68514726647, 15551.04400589864, 13520.27936130099, 61618.77827455383, 65814.51897466049, 16042.619602067265, 27032.64983593574, 37792.66660016427, 92064.5311535846, 39320.292996272845]} +{"id": 368, "vector": [34663.70468440678, 50330.062745115945, 66044.50375850452, 95096.85053595882, 84631.70936107167, 95332.04880972295, 43667.294306377094, 26077.061574852913, 84348.0469563763, 22027.618582822517, 56949.394653289106, 50054.32665480933, 65317.993798142736, 35684.9379168389, 17298.151233138982, 10860.39558796552, 60836.26769827836, 46431.76083877248, 69950.64843932477, 7083.882689506293, 74827.32821020756, 6934.34935307099, 88172.47869626488, 77491.25550746772, 22824.52969888974, 89957.02013840116, 31618.412253164595, 76243.6123725512, 9769.166436727028, 80904.18539255619, 28461.33811549805, 73861.78615678073, 5512.9078413641455, 49149.99987537778, 68721.96137424468, 26612.365775077018, 46715.36296030153, 86370.49943393088, 50591.670850460316, 91452.77422399262, 61617.89472402166, 19376.940684916877, 16414.456420572144, 53293.431589538784, 30154.143320091986, 29688.216150918102, 96190.24325144732, 36679.02837537957, 70352.51056129127, 82709.92122979069, 3167.5854670765057, 60338.764864755554, 16291.355055699441, 95031.53365981461, 74449.25529248851, 47613.66738295283, 24344.13254785299, 85875.76299022559, 59557.535464977875, 55201.567476308366, 74725.33079308955, 86613.29442835123, 25899.88467740969, 92138.67654863189, 78387.54110228947, 94150.74053770708, 39256.392905978886, 16042.364085363048, 3168.1440361087443, 73945.87440244989, 12241.98597152818, 31780.94315590586, 35450.63558561255, 46992.76415725439, 20959.894880178665, 85918.16808093069, 31598.90118673453, 4939.090104357624, 81035.00667375994, 92989.5996239057, 81322.28323658745, 48872.3730736557, 1028.7495170154036, 27481.51995939605, 56935.91256420098, 49164.65263810347, 12861.302991294366, 49693.99701059876, 26522.768283650676, 15629.781636511065, 4170.944835418122, 49432.682941405015, 83728.11386329016, 67607.63719225509, 39779.0265759777, 27560.42722808376, 74417.92125843656, 20207.081748412835, 32486.34433341182, 97770.27169408002, 41122.792390711475, 51794.54570744816, 34293.139965967566, 47284.9449248333, 8029.115983911895, 33096.52840432162, 41900.72905530217, 92729.20357252684, 23943.845244511165, 50435.51301194873, 2716.8232777685985, 96336.29575052646, 32232.061798395218, 93308.83194687426, 3082.782736304646, 13466.138273434148, 42630.73879006915, 53687.216172850916, 44694.07612738794, 95860.72912912283, 55491.51235917136, 96759.12036306666, 19536.232422625344, 76431.22870402368, 90996.85403553446, 90677.27184887578, 85860.37721896307, 21551.632732260696]} +{"id": 477, "vector": [33493.816464294265, 58635.31844936587, 44117.683318812116, 9548.7652549205, 66533.4229962926, 57889.12944230099, 38131.1781036795, 23732.761855082364, 31893.669129041315, 76086.69222740486, 53105.89365576396, 31440.88771320348, 15419.423376721497, 62075.42287415772, 24010.034739848674, 18332.052362867067, 1717.8980739647964, 58424.19417693828, 7547.382033431671, 4189.019748959288, 81862.50211965136, 65886.74540611196, 41082.688977744816, 72337.19103569587, 30188.438127403162, 19209.97538755399, 56059.73135229154, 77239.37087134903, 22901.374593358414, 1369.580054217323, 81558.15001378683, 71476.40320773816, 24678.85014316006, 55688.085843979075, 32897.14224593532, 49567.00128849923, 62753.71517257142, 62287.152326536125, 20663.68219937588, 73662.17745664732, 34918.964027217145, 6662.962460092592, 81639.74086629551, 63137.79121769365, 39783.07883893488, 623.4279291155165, 27719.22042554418, 43813.62803513724, 8046.366423071693, 17531.7859487776, 79786.80697144404, 97153.73420317976, 96352.29567189462, 89669.37300999698, 87712.95644430166, 46874.266305004276, 72877.2423112341, 91203.1035517269, 90700.86890424744, 44320.02392172579, 89401.94982504472, 90233.56142515973, 36122.7001992159, 98275.49183670928, 35910.33348515532, 52345.277831773186, 46637.77830568668, 50237.98030711069, 38145.883933386736, 25520.405948455726, 6756.09396243465, 93907.46992282421, 98313.5477400325, 87090.31160533598, 24024.08522439422, 56400.46933517787, 54959.842284105376, 24347.78550910034, 56669.89011639674, 6562.5708899068495, 24094.275976047564, 33978.490071099965, 82855.47153519091, 64194.434517203816, 53776.112136345175, 25405.077767146235, 35062.140084769664, 56463.11813469317, 62717.22528778303, 71980.12644631002, 98648.1261256521, 36796.67880274311, 76202.42680997841, 18652.798509473512, 33439.69223959189, 64318.315987040194, 19347.147812446143, 30924.39244273344, 54871.111920003736, 47118.01768323855, 64047.7573742618, 45502.118791822264, 13295.136202749747, 76697.07878037987, 93676.6848445563, 7842.946543075457, 36481.013452112034, 78186.03282024454, 726.158277297051, 70912.86298507368, 15402.53397217184, 47276.46356820342, 51710.89461189358, 70612.90060855636, 59062.23037212898, 36618.41523687038, 51536.14089711107, 29242.661473451935, 38320.35238844282, 53567.55250606782, 74717.1470023926, 92563.37881514676, 76985.10852404086, 94466.40807643438, 46460.788588362135, 31856.733134481165, 12600.554705670109, 56852.53663559957]} +{"id": 1862, "vector": [75218.47255183151, 41788.71259187329, 54734.982791642076, 16520.46054679045, 8092.758431990544, 66082.58118216855, 55236.60878810811, 76331.22583551222, 77350.47757018669, 10634.991979016584, 79885.62018637302, 91872.78387582403, 46806.244090354565, 26562.769623431424, 55695.29629558163, 11600.738182320625, 86104.23374709662, 86618.17353793229, 74651.26614191422, 13172.055841863017, 23317.775928760475, 46134.23026871846, 81167.45486666523, 805.0583601430627, 54084.0409016532, 86695.15349248676, 73511.83596440867, 14442.981042645186, 9014.836409267813, 98118.00886904291, 22815.817381730085, 68546.79932614892, 81607.12019889557, 2484.897646005746, 61489.38668937747, 23032.901462237187, 58944.99513312106, 993.4943929910345, 98979.67098130083, 34075.34353967158, 39610.504341426524, 14651.163967699298, 43287.5514363788, 20579.814219313907, 43862.49714538603, 20226.858934499138, 2383.1770655077357, 60562.738516898266, 32996.9482427935, 21418.450683505675, 37460.94719490609, 90826.23367838789, 57495.98185716737, 91717.33219046328, 83880.73926873817, 36032.02264695524, 54895.3636122898, 18849.669457481323, 98779.95079326522, 80043.26074262222, 10068.64633526655, 89681.66040831244, 24540.55579592619, 20327.905150903258, 59892.23810812068, 66562.39251520202, 43872.26220628598, 42398.04167870913, 32827.76533090981, 22574.761947718234, 53710.440646309675, 17049.27307957236, 78725.3920507083, 89843.13297435694, 29070.731590023322, 31139.451479273685, 21902.623987047587, 21279.247390940913, 48208.37956788368, 14936.571162706758, 63110.28765286797, 63793.28595717686, 57208.64382158831, 58339.46325115306, 39883.79726091448, 68227.2192484493, 35603.9733726254, 85242.53869716097, 70714.27975660404, 67347.14944836879, 35574.883125059285, 9532.445552116098, 61479.788653581345, 62766.3347398913, 67113.44203791714, 81117.43981296504, 1285.3230182279462, 15827.47641998543, 28952.211113832636, 92413.77380236697, 33075.85455692483, 65167.69553254942, 35094.166383776705, 27576.047591558796, 10133.91606008387, 24109.421262361473, 86402.00846752468, 54854.47892955413, 82566.94382600664, 3497.2115559380954, 48634.359823618164, 70993.15457875415, 65761.02263551396, 59464.36326196354, 75919.92506202718, 88472.89357857482, 727.2877270195432, 54620.239651522694, 90769.16633308561, 21969.134366776543, 63272.43006694945, 41154.14601794837, 66320.13751013209, 57059.86650747518, 40714.38175609756, 1141.705569566065, 90597.8295764532, 79681.56837539173]} +{"id": 1417, "vector": [38905.79172129425, 44612.31037446554, 8931.937854598982, 75473.7494090633, 2543.211057410455, 62853.01912685824, 76779.95671683284, 22037.51122089217, 98476.12904289967, 40.24571355552009, 45107.20816042199, 57411.289595348346, 84100.49350144509, 59109.95458363835, 76076.84080537906, 13750.337588425544, 96185.41415373534, 79458.55738758319, 54448.123857621365, 20239.457706427165, 18207.011907229797, 92457.4755274557, 808.5346697664808, 35095.52698156081, 90646.60002074165, 21871.946007821098, 27693.600457485656, 50246.316579447215, 12635.775474454025, 20136.647743733505, 28552.434335618727, 18779.478181867747, 65413.50568897919, 91266.4915733382, 71560.14659793896, 77032.0335587959, 84762.71989536117, 7872.678917063392, 74743.23696310323, 97801.58923460021, 17403.122163593478, 78735.27766603525, 81005.81097242719, 17763.395132822414, 55316.06617032579, 40110.14579331678, 16077.54048252823, 86148.5470027799, 84120.45421379492, 18769.85859515544, 84931.84556567001, 6060.372321904939, 88069.3476485669, 14073.762844563276, 72217.46526716152, 95215.32073974638, 50825.36222940214, 72815.332831833, 1323.5619321754432, 81918.67636683956, 29666.23890669259, 40681.791637747665, 74290.75944474975, 14397.026622209252, 62652.672393144545, 88611.79897650267, 3097.5787538488507, 13769.044563730347, 938.630113565464, 93967.91916568359, 23824.714014041583, 5056.798511881011, 39000.638969468346, 52652.668983961645, 9380.51712702237, 39943.147120199006, 73608.65974031406, 13560.408776788525, 41516.623097334435, 81073.46293326927, 9667.489583786215, 56247.46876544237, 2576.599882397956, 93972.97246858776, 20694.665711448655, 87245.72854917453, 5863.881800377979, 20924.146097061857, 24422.659891713618, 37293.01810673937, 33683.146761460106, 58766.831162682356, 36429.76276100083, 54030.999437347, 70667.4654905896, 2600.8528915882234, 86788.06792945511, 33963.884371278975, 71801.19706812457, 39453.298866693665, 62382.045643479265, 24125.530876557943, 79311.60729128131, 84869.91367172371, 9419.743077335374, 91547.03814983623, 64865.52407005651, 55595.70540387186, 74229.39987438188, 55200.05796656322, 32856.47478608642, 66471.19187231961, 14269.423903181443, 4745.221151934087, 44465.39146422467, 52554.65695270111, 95281.26619573863, 94900.90041040405, 16959.169761731762, 59988.345817081754, 31564.28228920062, 14736.443352311968, 48415.65853035175, 57927.229224863906, 21447.302328239926, 54266.62023169417, 59935.99470801588, 14056.73210271977]} +{"id": 1762, "vector": [97863.53477520974, 80436.35658847688, 68210.06242201512, 31084.056073417145, 35168.071360706024, 53852.587133280875, 61806.96129677815, 37772.39939578724, 30508.183533656953, 13967.33562099861, 18492.591638330137, 2004.8603309059665, 55050.920264361346, 34349.57891462426, 20999.56677045173, 94867.64896081669, 46884.29330507785, 89802.78136364634, 24043.846132865987, 85513.05033384016, 62613.77845427268, 36506.50510073407, 29178.505745663086, 89475.87895867127, 65291.19955472546, 78440.80172680822, 45640.924153507614, 20084.70660863584, 63895.78624728965, 84978.19079086884, 68737.43806413325, 2104.1939740075954, 96545.24767969381, 37397.34179754982, 63467.116316580315, 65352.894922778614, 11066.230136296874, 93962.75941145944, 22560.885282419396, 64622.92719832357, 88286.28629528676, 76579.3131723087, 68617.86518493529, 92759.07425975795, 98945.00571439273, 80139.54094243581, 69216.95222799416, 67993.29886284226, 44503.009528724324, 20706.832573304102, 97330.8350115539, 80306.80175113412, 14059.165372514237, 92689.57687884542, 41830.74180137288, 33842.06494089331, 23929.671474419443, 80439.15108444853, 57011.47770419909, 92889.90735149806, 95213.84315584885, 27351.00404957972, 23260.260709754923, 36218.85089604353, 89377.12266964348, 10708.857976732877, 45740.53061055793, 6737.841203088701, 25858.611532838717, 3687.818582911051, 96261.6939763001, 36402.58729778538, 90333.66740628057, 56321.7218677958, 99679.73740617033, 3708.9549712264193, 58175.68343310213, 1193.0179895841552, 26251.548087111474, 4206.3676053784475, 72767.58421419156, 82711.61955893932, 37518.99200655529, 82311.20833503349, 32069.01588730987, 46850.59047967305, 30293.381431020603, 315.69367527782833, 62660.91647078571, 33406.242205906245, 87001.30893010573, 7938.359180885235, 96986.06005851693, 91560.59006249535, 15257.999220695006, 83179.75855187204, 81778.16209680818, 31470.883389359584, 52600.98725289725, 5739.716186147825, 71375.53151335074, 55164.871146801765, 11669.044121002382, 85467.00588701638, 47099.649727484095, 89528.78717620377, 15800.784287672399, 24830.21693530809, 43913.823451295495, 77074.89727388059, 74930.14591237032, 22801.45079878725, 84771.37423807496, 18430.699731425415, 3596.141344779591, 86689.37818855207, 6257.546667845982, 25098.95208045927, 4063.4584829784326, 37648.97277765525, 37926.248374282535, 3674.569967372643, 1618.28152257113, 15438.774763806163, 71580.29681118668, 2028.9233008486951, 33908.270789260874, 66373.26038196974]} +{"id": 1696, "vector": [82999.2253342901, 81423.68356604617, 63505.1716930981, 58451.9740569215, 84678.98764604534, 61269.543143178205, 99466.00557235767, 36827.762673387, 25630.54736572602, 23088.930361967552, 49884.09008397419, 31163.02889713457, 75869.74238930918, 850.4461723140566, 35815.29777329902, 89260.58795960153, 25040.726034409, 18294.75609087151, 94815.05094712319, 18143.103946703155, 33502.57013899456, 22865.63380694885, 43687.68191990797, 75565.49742825907, 99915.12691042708, 28250.90256473467, 26077.235661326216, 43082.765432311644, 86352.17312547006, 28470.578335382335, 89875.53283035631, 63802.67614289755, 65830.04592677345, 61047.88400868657, 77825.14363987878, 43463.68349760957, 21736.09205270647, 39939.553439768686, 39913.43418929781, 13991.144712263536, 73072.78739231148, 63187.451814332984, 14267.345899924989, 32134.01523110736, 9342.797455840746, 27330.368273610027, 23389.31389713581, 43345.743980583095, 25690.239197948227, 58322.43931853292, 80062.8202166646, 13873.67099551251, 66031.05558951505, 24172.17487243897, 43941.80926408596, 72664.93074711863, 31721.37122574896, 94152.15948632188, 38470.625572782614, 14282.39550282111, 5836.071478291693, 3535.1235442098837, 86407.09363745917, 13415.57829470067, 95973.83742239566, 29579.95578228637, 64026.00197532828, 914.3054399799233, 70322.18517207631, 24561.604998673647, 18932.079232681575, 94146.52000127175, 57329.66864486271, 95777.7874245548, 15863.24216663454, 16100.724808759149, 22523.086402298366, 56006.823849005246, 9093.192300680697, 77512.51248912534, 91291.83929807643, 55384.35492710909, 61222.8092956024, 70348.72041164925, 65616.53130944415, 76835.15859551996, 73605.88427506956, 1186.0140230491202, 62735.76057952457, 34246.51238095257, 19566.464410809847, 99682.15620799817, 56222.23055354071, 57727.22545610833, 30001.27262930542, 88869.42867032213, 46903.097095576806, 57663.57500841472, 98518.02198124885, 12137.870331050648, 76783.42730435324, 10472.071564448383, 21188.339082120332, 1503.8894682772864, 21823.81509662027, 36538.9107526052, 34490.39961510639, 42796.57862698514, 62451.59936346086, 37333.05192975172, 7153.566799164101, 29691.563134539556, 20128.288414797025, 3137.4235837462793, 9982.648774159663, 5995.325979786825, 82394.73036444811, 47541.92312578511, 38141.261978032024, 28363.814360636887, 31242.99698694233, 17320.451862369246, 90599.35991901158, 48352.90838587227, 50926.971956635505, 95373.63869202284, 25876.971664549386, 38071.70565622887]} +{"id": 740, "vector": [11780.639796923931, 85302.39891278809, 30748.08749901673, 35088.73518132962, 15921.115073121828, 34664.60034458446, 58397.35993803246, 14894.283491870208, 33870.05914156069, 80243.9713863899, 25931.944796704363, 70387.90756125122, 77451.97137223496, 77039.48692819932, 1791.1620083947532, 98765.12646233654, 15649.345558950856, 60220.45687296514, 84824.15070834172, 9218.456733936342, 91984.03179303689, 79530.64773677988, 3819.960843785841, 23944.604109977874, 64597.56264576043, 52887.40057172914, 11139.764000930652, 76568.92706497968, 94710.46096963847, 28716.157119216412, 19450.81093269696, 30715.433919568546, 91227.34903773865, 22569.38991600129, 10998.670590146698, 2520.069425552207, 21787.26357008578, 79860.09267865018, 40998.86127279998, 10149.50777850977, 6762.926053135332, 96412.95652192782, 76854.75215320417, 69476.46096277304, 3583.7198507638336, 25810.823532122275, 52126.23967883041, 54778.71073480501, 64650.61025027617, 71313.37484017777, 65780.13076220173, 11501.86707054096, 56677.50930096143, 74154.98455087823, 53611.72732347571, 74391.27539699362, 7843.351473608118, 43765.7410282388, 28946.765881137304, 30791.947847436528, 42063.3464632106, 86954.32186286163, 69318.24620997258, 62394.05372749931, 20881.839720929453, 5808.087830516806, 94346.07640581044, 44752.823139119355, 28577.216531314607, 87762.8581770437, 38245.60185059297, 20423.531441180578, 54125.462382527956, 84267.53100169735, 28179.564121428102, 48287.47929489644, 19242.074593704085, 46065.08427442809, 83180.17905111522, 25710.861131266694, 27021.940368807762, 67129.87048060389, 20440.208060622634, 54422.89930129601, 25155.89021989838, 68320.84401815571, 51789.10980447875, 60946.58546369975, 11456.526461376503, 27223.619546216814, 35493.11416059654, 38169.029611840575, 62999.85023541678, 22381.303145624865, 36249.83194212651, 66801.2242430972, 38333.306536190394, 16874.486096526864, 33290.237895094855, 70001.17057787499, 20346.748072041242, 87255.10256055524, 31233.524545398785, 86176.76152380476, 82126.79597715862, 46022.41837135146, 90307.46225570171, 52499.303953377064, 34994.40732212405, 33187.051393949885, 34982.26212075363, 98062.07218076043, 90869.9102281793, 32031.16942088542, 33047.90181687509, 56827.753001159595, 51602.17268284535, 92491.96033948971, 5103.4122214268955, 83507.26511123357, 99532.16755959776, 11717.358951674196, 55901.17519049434, 21950.45825487608, 96968.4249450821, 56655.195072684575, 44909.75476266299, 3060.0685719860744]} +{"id": 240, "vector": [76197.22104037955, 19974.03633089825, 90197.1866162834, 6182.709217714711, 1632.8422427618227, 18220.755077748054, 66121.71972434733, 15527.358392498769, 11772.366616925367, 58877.0987534597, 2006.3566293370027, 38345.32198923006, 13517.454484091595, 8918.511687776576, 80329.2593611001, 3544.57048064718, 54222.34731518291, 74813.8364419617, 97967.6873862005, 29538.46321048943, 25097.66812878038, 47810.48713629074, 73140.22505239118, 7910.375653350788, 57000.547189260506, 65484.09764441752, 31400.382023737227, 9042.77753702295, 2230.8765293099, 64770.92576166677, 35163.18170357425, 99839.72687288742, 78980.29770103391, 33352.91288341694, 71550.44953473899, 12639.835444468528, 88421.40297715725, 43727.06816736227, 96655.55453083546, 33321.25909917104, 88869.1559998977, 49697.19276108806, 47301.04152794923, 8017.815009033025, 15006.480538603484, 18477.330320861296, 98540.43413002828, 72675.5364108474, 85822.67302595617, 39536.826478283816, 27777.93643907589, 63285.627349398485, 61519.647345554906, 46302.36418890202, 19692.257547385383, 81913.98105712478, 2697.6807271072876, 4164.897123388034, 99359.30835275441, 98134.41903144743, 57566.13707408773, 87187.31674483752, 41578.55526029053, 16199.680400692107, 75047.21689615837, 3278.34092516015, 88041.90534404037, 80096.06597964521, 78643.6087041432, 73371.60178164356, 12811.062989642407, 17458.21236500854, 92031.7239747588, 99142.57030893063, 72769.52353304873, 99279.18066336193, 70284.25429439318, 21146.661545121624, 46611.713350220365, 43780.49544819542, 92302.0830551504, 51105.20339174213, 82981.92146438404, 68010.48137610195, 59196.87344537432, 8589.46974147381, 75657.4765627347, 34703.51932009868, 5094.293186821664, 27645.602948991822, 32624.57827606945, 20862.816818256437, 8661.422728396983, 73051.71988426801, 51284.27396786967, 30598.314220989665, 17756.322534251532, 9683.380857102953, 41486.147019608885, 61578.92869940764, 15046.042467057187, 70506.59246356785, 42094.753783314874, 35293.29056561465, 99427.03514771519, 37384.16491545575, 91104.21156131568, 27803.278880238104, 75917.30616892377, 11536.048657287478, 9731.019710596689, 45049.30717558159, 5900.500414222898, 36868.51465596459, 65909.03789808108, 66764.77118560711, 13839.178337888246, 80128.72154689161, 7491.721892634062, 6676.943718790829, 57441.234193771896, 91787.0243286296, 56522.34686210717, 33387.78836942702, 84805.34378098356, 31681.674903913416, 51902.264098039974, 26322.327553800285]} +{"id": 1870, "vector": [92256.526098922, 89994.68020795305, 66205.75198823371, 31403.187684446886, 93113.36699057605, 89498.69972107261, 6374.860547516559, 54357.89407049445, 92140.06311220698, 5400.608336618695, 72590.53979978529, 7483.833200676892, 37101.613183021465, 19864.543678810885, 89911.74486620653, 72598.32044645642, 19977.182766163347, 88441.14710277194, 3400.523420061363, 39815.65900253026, 77883.09528877125, 74653.66550397228, 27467.75881460549, 45776.05514642165, 21297.435447531254, 75261.31789627431, 24371.260720412392, 61630.48934390953, 17789.443948869244, 57073.69184580472, 89559.76702233947, 21756.964921051967, 8051.343851339321, 30831.840668663168, 40355.385292639075, 55510.64911312116, 91421.28645231861, 71887.86531911793, 99646.89299840653, 50126.76541533816, 65788.65157337065, 44686.80778618562, 48910.696995206825, 59929.6313425556, 46437.06629430496, 44254.48774250269, 48657.20183260827, 56156.630527339556, 78065.85520720607, 62452.183238257065, 53410.82941006953, 48137.78089612897, 23404.52897991707, 88732.49600384278, 15434.635313018929, 50227.10112031713, 31111.262243121873, 81827.92541682921, 18758.50613620538, 4211.376040172876, 71104.60203038443, 11676.820385117047, 38370.81255073109, 97507.00223610565, 95191.13461380762, 60647.574562479334, 80260.12795778722, 28147.99968980176, 91824.99010493013, 39345.42650144417, 74004.52361502803, 15088.058161600504, 50506.27760399905, 58164.19807233096, 66929.63995053452, 33582.86684762236, 70842.41108569897, 86233.3221107257, 84402.41481969155, 3982.431257389718, 7709.541667533248, 73485.01567842148, 84554.20689515032, 27908.491846338246, 46954.46871205269, 35030.25387188653, 77759.86807343626, 46333.47305983703, 94483.22041634862, 92787.09517304812, 73912.89570381113, 58549.548073844526, 64063.684382077205, 98226.97327842811, 14675.591376461694, 64532.958030730304, 59340.11222552615, 5408.678948922218, 56203.15666282449, 13539.176515174035, 92587.46601031763, 88020.60650192696, 91535.04437313776, 64128.59135924536, 35058.27279273538, 92855.8045821042, 10286.006283279958, 72261.68562463892, 74823.26369854767, 18385.803002886238, 3025.2911263547453, 65202.0961832935, 7642.200619095507, 8113.680070491436, 76811.753190793, 93717.32438219071, 49380.017141357435, 72531.7308725551, 62159.043759428445, 85302.50144526041, 71827.40762633547, 59727.28913654906, 7428.843206498581, 84664.8999868937, 61417.96562282521, 88851.40222665627, 38660.0861686928, 68764.86950087837]} +{"id": 53, "vector": [46907.320379753444, 43048.92033621819, 77934.72326100018, 58724.06896399409, 53079.54512087821, 3407.2082314049344, 28852.910535034294, 93495.22190415049, 14015.335012530839, 87521.18697784208, 98155.95006321867, 14838.214402210591, 30790.815987151178, 49514.40291483378, 13938.2810430137, 85655.5239115404, 56762.87787058543, 79975.96982967207, 30409.500001608492, 46348.20746692228, 1094.82195838152, 99457.6858937902, 70190.97029753175, 72951.16609167037, 10295.693595143484, 12021.750424070276, 50540.162388934696, 14534.878254538864, 54267.052722169996, 28879.88888179003, 98639.71281420336, 46557.54141467887, 52573.94168384167, 10178.167087152679, 80640.46789290207, 40257.91539702409, 75845.35262441258, 19562.99398131164, 87323.72198656114, 48648.64984814202, 6552.223027473514, 15666.623846833316, 24238.087905009077, 15827.964260105564, 54651.952891142, 947.828167055298, 89886.248736894, 17110.077863668826, 69189.8605753158, 58221.715525366824, 72518.47918436307, 65889.3253345688, 53976.79615842981, 14000.957272056392, 71703.63096359673, 3229.869896847892, 86530.75029093721, 7994.57110434375, 45526.807071305855, 14147.206877796492, 54584.58619635307, 75644.12467689642, 59741.88842962231, 49826.0170146241, 31383.47339454789, 49820.02936666717, 69406.17461536416, 15103.515323518946, 33514.55311843158, 47056.9814135077, 98846.80475672775, 91077.38253320835, 29257.736248650846, 55678.9945290055, 51176.36938844328, 55796.75179579937, 75922.35318688529, 57612.24541300908, 93536.63443005741, 5623.348436072528, 8191.12240144535, 91906.11573532088, 36005.7401645448, 29760.002348778748, 2957.2860624803575, 8807.794152217752, 71879.66827327917, 30198.87658533338, 55410.18191906754, 44168.64549085102, 6988.349425191797, 67406.31594609248, 91928.79875458521, 13533.413622691636, 58073.2784641808, 45087.15646741057, 82489.64413270168, 6778.317996696182, 3270.622320911609, 88809.28404814914, 39620.142830277924, 51763.72282249765, 27146.99301097433, 21929.759864388478, 26298.509510741387, 10942.359002193725, 67525.28897991932, 73912.82790709773, 48525.501106693235, 54371.594398661095, 81155.7033777292, 72833.42568675322, 42938.378933454536, 25921.344697847802, 47844.65656652621, 68406.42752192386, 49061.28757903324, 32807.991780546785, 7572.22672247001, 12658.357016355216, 21516.07214324388, 66595.41548856038, 32963.63070142937, 56739.790296847605, 59505.607507983106, 63856.34684832785, 46438.43876157695, 40457.20098204203]} +{"id": 466, "vector": [88534.0094170058, 33098.851923615424, 57324.644627164445, 10222.629706595599, 69240.74408507692, 42935.31268121147, 40705.86779174077, 89943.46353779358, 1394.5811678274467, 79193.08136824348, 9131.047532606563, 16213.712458755235, 40338.87002576517, 91410.84531882043, 83027.42658590635, 49382.60892622958, 14369.600601417877, 40874.69085909803, 10529.606370970445, 11713.23233408692, 27746.03386869856, 21331.062266116263, 15969.519301579405, 61959.04888589266, 28132.558111350703, 12230.91776088363, 72998.83657522555, 18177.335136281315, 73554.95716831739, 28713.240121742532, 36316.19347870232, 80368.46840348245, 2715.7949363406988, 5730.8267532176105, 91665.98281992722, 4037.1607505437337, 85674.89079819123, 57894.067838247865, 49307.59415168422, 85873.0920545104, 85038.63030506339, 87181.63072812381, 53440.3385090788, 45560.08354364054, 32044.201325610044, 3545.387269613165, 96647.58798681223, 45757.650050729746, 93213.09742931096, 90820.96558636588, 73371.28855414913, 98477.49186673573, 87261.32811021812, 34982.73415435905, 64766.45216429884, 47494.713940158676, 49169.581314343566, 747.0837129981201, 49935.015678297365, 32313.322350317987, 80439.88566109519, 23382.886531379943, 22213.850402881173, 72631.53320106084, 67224.40426648043, 11085.763536073679, 76133.23620719233, 94811.4486088658, 97004.94808603499, 89851.56230306026, 5895.331509189472, 15100.124171843809, 826.5818199629016, 8087.977885301356, 60900.09723996761, 10738.963496950239, 20026.103167270936, 5858.9427749668575, 51057.24850852159, 85815.66982096374, 98776.52112569087, 81836.52782626856, 22260.447657726312, 76743.59267626569, 82727.75259359795, 41128.69820334606, 62780.19309513682, 76091.8548390743, 9211.359324962099, 76248.89720126275, 62925.074767785794, 65950.1176105935, 63404.471988557496, 76972.90816978866, 94688.57190704641, 69713.49096241269, 26693.130979016623, 1154.1163067459781, 71551.44955137395, 48434.647248093024, 5434.223200113098, 3684.839802180817, 23557.883316405947, 40482.04682371786, 25617.470940659583, 58444.42857738265, 82695.97042009664, 18811.98896715903, 60999.5686841331, 22973.5301609306, 52503.73829200238, 6198.064140327886, 26257.959617031178, 6903.152837401105, 94516.0379712763, 49802.002788187616, 42086.062200212735, 98872.1028801112, 76539.87077267224, 32816.01990693146, 60924.11996963514, 94669.94821602458, 58319.11622044006, 27832.568033728978, 15425.085104019887, 63437.2609985712, 53542.33667079923, 50804.3073331654]} +{"id": 1080, "vector": [59443.044251421474, 88891.76452223567, 63832.209328114055, 42223.80922171159, 35761.34999462096, 26674.60167202873, 78679.34489187413, 21285.726648780877, 59366.84266165469, 59075.59219000287, 11245.173524966012, 90271.98044654471, 46752.24906268868, 90582.00026127395, 68620.76196781089, 12607.162782669413, 62160.51993713878, 27711.302790063197, 40409.30770808196, 3336.4334592410282, 89895.1792042751, 36952.37784715947, 21874.519107966717, 57796.682414766896, 85721.3524517574, 21827.239500499607, 26082.082834048113, 19027.463533147704, 84318.7630550262, 92763.64646296206, 59366.629135940464, 20406.72173345689, 23931.407112470915, 38958.29045193086, 73591.8597674345, 65871.86775676603, 90089.30472443471, 68440.59591846692, 55098.8544552253, 78046.27886398196, 54888.47303156154, 87622.87858231128, 81286.76799528606, 35085.96649386503, 7467.240320403012, 63767.6032854338, 60097.60679083054, 73316.51878667467, 49035.20999576443, 40037.0422025158, 41930.89679863145, 21509.51834166198, 75792.08128420154, 47442.420489346136, 95810.36535613747, 93724.60800542562, 98415.520654735, 11868.01612309717, 44299.084378442436, 8667.62235977232, 95069.31925263345, 39651.10803963475, 60404.592709438584, 23234.18296335874, 55880.30137788483, 81883.31920811147, 59348.46475782172, 45525.74235714473, 19828.85040287987, 92655.10650050959, 85367.02766894473, 75891.5009674914, 13678.564934593318, 5584.314899005538, 85868.09618439469, 52070.13317278173, 82034.1870484742, 29082.232956342745, 91225.81114773366, 45341.38703394837, 44627.75661901469, 98786.92576087025, 31861.942817197243, 7820.256469483433, 86495.30904630247, 42817.26615794122, 12350.262721756344, 8514.32563121879, 46105.198910730796, 89922.73483798771, 23854.76579848499, 79700.91567885384, 60930.21141884316, 11892.290368413038, 6773.344142241266, 15892.923471209408, 97023.44152785702, 36219.98069356138, 32897.47463747994, 72635.06220219041, 33371.852846541326, 17504.901582062193, 86635.19554591706, 11051.063040597664, 78721.96000810416, 76141.84874758692, 79989.39553162518, 61806.62229744808, 88241.7032191519, 5534.481701486548, 66664.38991574723, 4972.94251494298, 62939.546556139605, 18676.08222107744, 83201.97256168547, 47782.541209914394, 26955.3122977649, 81030.12822420102, 55861.73316282645, 33793.860530098755, 94916.5694770368, 562.1123727439614, 80488.71816299499, 40204.32648833046, 30583.25998923991, 84113.90677580047, 98068.96498893964, 43176.723332021495]} +{"id": 2013, "vector": [92376.77571520864, 2228.5450903493875, 51483.96995049908, 73990.57613047108, 13685.78984932154, 8839.089498766629, 77104.40624740077, 77206.98996818098, 71441.38620422967, 80857.7250592934, 47646.93493920371, 58898.36192038148, 10003.154691019756, 19236.65878007528, 63954.27761568375, 29630.733944437714, 89558.30524471441, 78274.04607624629, 63649.36799507167, 97535.87317310637, 29874.27486687373, 64303.36176496929, 92322.47992132805, 87443.64331902155, 22409.342042854307, 25473.76478320965, 59105.18576867892, 16213.201469861826, 1461.5073184583882, 92205.09415040626, 73627.25103027027, 88117.0569920565, 57465.55942213473, 36227.37243872735, 41343.09569186444, 85532.50001686283, 24601.41181508777, 95194.01716454385, 44568.73354062717, 55885.87294260172, 94045.46051069694, 2040.6020372464307, 90072.56564334301, 76888.98311348495, 877.1102890203331, 57285.691749653764, 89282.50555371234, 42914.49631202084, 60730.518841238205, 72988.75963772558, 41481.328783436256, 91551.32636784112, 88662.38062451922, 25632.022387082863, 56246.098432615545, 9061.338174448496, 16786.95941024335, 91688.10669742638, 84873.39857735537, 24829.14369544188, 17820.203349297924, 79858.43556065005, 48926.653629904446, 88777.68510054637, 34141.97571625165, 85925.47222583473, 28448.26319317738, 65230.86186229108, 80481.98290015932, 11275.076940111296, 54713.466980918354, 65130.67916676994, 75233.6921618888, 27829.00562110786, 77082.23370937708, 11826.699653785545, 24447.372606172434, 54843.154455874275, 70914.68655873007, 81753.24717427332, 90264.56600578636, 69823.10498227927, 69502.67334838033, 28037.733146501163, 33961.12479485893, 77542.12822359269, 2693.6987137767596, 65741.3679854645, 39324.07898182207, 60386.91972648452, 26329.8609893896, 80458.74205677489, 90968.61516625634, 7006.815324566196, 95693.27231400598, 57079.9965701835, 99949.17582080254, 32551.00411315439, 6469.816277176455, 41201.176728554135, 69952.57597864838, 35637.31847621846, 94304.37746624167, 39781.85122335782, 8716.606716106267, 1999.2518133623794, 17889.90204309696, 86833.53832911, 48407.44059491271, 33234.44664161184, 50125.60023355082, 79449.01236315916, 11333.971074694016, 12006.467612032735, 81047.93156033417, 44451.654352153346, 22585.099520494045, 22081.017788999645, 74074.58175729748, 94617.94400636773, 43962.22086527737, 68335.69670449384, 11577.14896857477, 11075.612804118662, 27968.954639239906, 69281.97439698405, 57825.3670460714, 76331.59695670063]} +{"id": 676, "vector": [60372.51683621153, 48499.744992747204, 74603.90109728131, 96807.49976280554, 39477.066879303246, 91961.96472574712, 54873.56141593675, 69415.52268298375, 65855.31460604355, 65992.30738523559, 16395.255064506597, 8809.026837531375, 17726.699124892144, 26319.686892293616, 59525.91848481924, 33001.77038630745, 78725.61081649929, 6345.743359343559, 11902.353243837815, 32433.49436505999, 71406.58934594145, 62323.91444200094, 73266.2248208056, 89489.85161582849, 94530.66087294597, 7403.862919689008, 15513.103952648966, 60364.34202246087, 30110.79602053014, 63554.51339854614, 92015.41878452989, 68245.13186201679, 57980.79758056697, 92048.27947768319, 20598.172278374073, 17023.903074350656, 11514.847947080709, 45056.40721892456, 51335.68262353253, 45319.14422230392, 10641.419002415863, 67950.95949092775, 65046.93496704388, 47957.94693029694, 77196.82511518447, 13417.443323972899, 57114.40219411899, 8096.789083677447, 58106.73645127409, 56777.80318029767, 68776.43446761694, 79486.13022818523, 83647.41058526668, 63330.60235318531, 68566.76766076646, 16895.449487517933, 18049.875400462646, 53666.91154751143, 62756.54339590188, 51791.9902294928, 99867.5572678826, 66050.91547641913, 35171.52046641389, 6096.89931634162, 87279.99648946273, 82007.61988413216, 95198.65073498405, 32245.521275259147, 19439.623336244083, 13018.802779157013, 50878.21310624804, 63707.05093602548, 17400.732766677596, 95759.9955267546, 20981.482266045605, 3163.3975250905723, 54601.749522549784, 93433.72045273328, 36098.82674146594, 32590.436928026746, 4260.448956957286, 1687.8057053129503, 54396.4334153372, 51893.254218231, 30075.122618817208, 30074.00675517773, 23691.591123196755, 93865.14768649808, 57923.25388098941, 37644.41547388959, 4708.765263254433, 71954.09285791281, 21582.690907482494, 95107.8745715629, 94856.721415066, 821.757737453721, 60955.40815311251, 71415.24752439382, 73816.39590086309, 96225.71874322231, 69247.12961044173, 99070.73362266866, 58391.37794941639, 6982.261139868629, 15200.383061467493, 46636.01870400496, 85776.89088503072, 35736.476623026305, 63896.71099322179, 67536.98263138079, 27682.353340535014, 39138.76255444942, 55.8794774072946, 96334.6027262452, 13236.648534665896, 22341.34325928748, 37015.349954723075, 60792.80524480688, 87734.3436398505, 61829.00374206741, 24879.645411172045, 58240.781371977915, 29938.325552540802, 92026.22107232158, 91796.45110364031, 94027.12118731422, 47972.80699181885, 64097.86643288101]} +{"id": 1865, "vector": [82508.91077035024, 41924.170701741845, 42227.84320539338, 48160.13718185217, 6673.576107615509, 33916.2166357903, 59166.022881871264, 54862.21273960299, 45535.44538395647, 10854.732848522819, 90260.59309829577, 29041.2948368599, 5729.287982501685, 91926.32221768273, 49254.10958415592, 49909.825435219966, 15817.843298691792, 37929.351412574644, 34634.977455063934, 42185.0856972102, 69371.42324429851, 13004.23314763921, 85703.78075954993, 67192.86923794169, 88921.39046543796, 57264.77945065329, 18448.9352347877, 4829.671277222758, 99441.34167940005, 70911.9133100242, 44712.50766374597, 25781.243539491174, 18191.022911609965, 18900.59535430927, 36384.04388297744, 62994.69410852461, 88807.99470609394, 28482.672179193858, 22189.156944512633, 63262.65554221525, 9705.72624242848, 75857.18270540748, 77464.91123132987, 81341.27265002654, 46931.84321659768, 10602.874475079194, 73712.0454005444, 19253.63070000402, 65350.039063973745, 4105.504223194534, 99387.15265608412, 21630.837973041016, 42961.811443890816, 27929.661403163973, 80144.86112917107, 53423.683662782874, 41290.736150791054, 99788.85262618387, 75673.94085097122, 15159.121098119533, 89602.79523685858, 25275.190016716344, 29334.306592245706, 88474.23403542934, 86434.42029878103, 95505.42614063667, 84727.52553327456, 16843.17395435123, 92543.52415837295, 29470.130335880083, 81561.2130196068, 34063.074299565444, 18410.141747027985, 51283.98982683898, 49395.95488179007, 75450.8175191177, 80998.32323610023, 90870.01282196681, 1809.3821470755668, 75122.6608045184, 54658.49667425109, 15228.166192973991, 75113.25344650653, 18056.957767687654, 92815.02937164356, 97604.91391817163, 49629.4967567907, 64850.387634866456, 1934.9540846243297, 88038.71591142955, 21798.05084826054, 58619.444971077624, 67171.98889887409, 67742.63856981347, 98399.54489045196, 98568.46877465695, 43975.75621466272, 52967.67910129906, 13308.589294268158, 78659.52528373465, 40275.62663305927, 50831.637281646435, 14112.537724442896, 83.68004149068575, 75164.4228229243, 64847.87905970259, 72524.29532380404, 83027.03612648536, 96978.02555305071, 31328.458341567766, 98653.71336953792, 62949.384095535366, 48657.83878254939, 917.1788010261084, 47233.47066501733, 67565.46004300688, 9565.973753870783, 64888.792034167454, 68832.64087050979, 62448.869362529826, 40552.261749447796, 55242.94653931995, 30061.11623717843, 65708.6610423335, 44634.91626874642, 31327.97258806036, 98524.83966468163, 37638.71861478727]} +{"id": 881, "vector": [90651.1794279439, 59886.56718201067, 72162.12459480266, 94635.30745420193, 13531.676216677624, 47941.713985850365, 95141.09386274897, 60740.70291786139, 97709.59548472398, 53811.945988562635, 56823.736849532346, 51504.43195743721, 10345.255133717179, 99801.11234874037, 4790.821107501286, 12581.156523282421, 17797.768194235676, 29915.354052535447, 5522.057323733776, 74345.0087570369, 54506.63138010373, 5204.326868934717, 36898.22260199148, 30494.284101969537, 71307.40976218217, 83383.6438374897, 69358.28355852878, 52170.69289140645, 92594.94323777092, 80922.34423692753, 60197.87011538145, 85327.3905141977, 24973.7288364553, 6650.024903731333, 39346.67182201736, 16651.97500507545, 60420.019802611336, 78947.32049715305, 16214.672942694053, 77685.10347299438, 66417.76065671246, 18247.218222596395, 64104.442135878984, 84447.88825508868, 51304.02185275047, 47006.05214279099, 97912.11677740935, 94011.87407472891, 93389.11636194008, 98932.9331667245, 10945.89391551457, 85103.78614693975, 82689.56829070736, 6882.733456595913, 50013.75095652993, 59985.535388060496, 86604.35657872561, 26018.26567804144, 69492.36569549047, 18280.02479862455, 98047.98538310823, 18561.62858010998, 16888.382638656396, 15080.164141617259, 13559.765574253468, 63429.478358705135, 90311.09105084924, 35793.796128153976, 53660.95676281992, 87360.62508215362, 4498.185282509548, 53883.206551305964, 77695.80055875583, 16097.760047809796, 38988.464317533675, 58686.65534844794, 28841.121222311038, 1120.540434534656, 36681.03866542714, 86349.71887558403, 3529.5563038770906, 3656.771624466759, 34799.771981965656, 39613.37274616552, 79611.78625074266, 57839.632014133145, 32116.333201729154, 80199.72875233914, 88990.7711288239, 26406.878650703282, 22343.522008932083, 17361.02402763675, 4964.044280377, 20535.44118760805, 91242.2092334385, 26525.660393746475, 40828.66333783827, 12607.83710043264, 91991.77286109138, 75704.93913085053, 91030.84131323462, 37234.6919143298, 79984.36475721428, 70496.68636207313, 62281.45681452117, 20359.799992626682, 66202.90684238286, 21910.7022071275, 5152.204407686445, 15035.140249477974, 75786.14461240945, 49916.99067585272, 75914.76734542972, 20894.997477433753, 85641.8192941873, 23220.595908057294, 68267.86942182582, 77291.62212758965, 89821.22673859562, 7300.620597701945, 80357.18003988077, 6015.575228583037, 19146.072704573093, 89440.09056738875, 35140.32334312487, 51362.73171335356, 87268.99386164635, 11618.664197212813]} +{"id": 1094, "vector": [36165.417553008374, 23338.65917427792, 41720.31114653897, 51793.227183470844, 30348.527368244737, 78165.65531275146, 52365.258546865225, 21374.893945366235, 24442.68587458803, 42845.49608070255, 65209.80136176633, 88148.54839355222, 73859.99792946766, 96497.81028103469, 61352.888210797144, 84960.63502759626, 14428.078699941781, 25176.41226332664, 77372.08537312498, 16022.617641595805, 42517.27982522786, 62636.78865196941, 45826.6137910475, 2608.9472964258343, 9660.490645028296, 42830.33709627695, 71034.68314634249, 63721.60154350387, 62923.747493877876, 61042.14794560298, 69178.97550035316, 12266.164174805594, 16490.439858987804, 92599.1591685722, 18865.699102604616, 41567.995338815876, 39810.134212869045, 49786.60217561762, 83879.45559187759, 4523.921292787525, 52220.24747650116, 36926.56933848264, 5820.5280399744815, 28209.240371172516, 83099.07577750405, 11176.700358856651, 7804.824914339026, 32585.16485988485, 98002.25157610532, 43444.31828093832, 80401.70564706447, 15348.555296651111, 61945.782017774756, 15100.887415703834, 95328.05317155756, 21430.93816598225, 19639.62355242277, 55330.202527399284, 74015.66995082279, 60268.51316196912, 98370.1186774398, 15313.924699932324, 95128.63057278428, 96668.04130803034, 78211.52298874504, 29641.913370212304, 88646.31656259745, 39928.08462517134, 12386.18739357452, 26990.949375347296, 40589.996968363164, 75201.14330316766, 77145.30374873382, 74582.65103836328, 85619.39152177706, 62442.82668652503, 5348.426693050113, 58492.50238986377, 37673.75784810292, 91785.60244326823, 37738.66809488048, 66476.18227632195, 89191.84370790355, 53048.21459538869, 65797.30395186233, 60603.08256738835, 99293.75397185724, 5602.547148743109, 48494.3367584783, 73508.87722466962, 88243.87120424374, 32441.83047275472, 80674.46070388278, 58634.117452418854, 75886.16835416104, 75911.02274350704, 61698.18771613661, 18407.12522715392, 41292.2273471861, 49802.587086893545, 46374.96753097019, 44065.31444734567, 90452.01008974982, 76593.49824130883, 73860.02212282721, 85137.10396345008, 68568.07437959684, 75025.93071262771, 25826.198996793824, 39955.086168242626, 58008.38921518211, 80520.8618076431, 77067.91808046444, 10562.271892017017, 10387.732052309317, 99879.20697731692, 50120.21244416167, 16458.327110273884, 62290.08062804327, 18304.256178876643, 22534.251325540754, 50624.47118274166, 17122.290667365745, 1718.981261910546, 7284.724262304698, 19621.712009269544, 93856.65016888893, 65093.182981668055]} +{"id": 1381, "vector": [34540.58446455013, 90314.55130161723, 5149.480828791053, 41798.55998876221, 96421.32511545747, 59408.63144719738, 34172.09732417544, 72006.65858318341, 70906.72097420448, 68987.41557073212, 89366.86871312333, 27271.692967600946, 55050.48855852443, 1278.7197392543414, 248.20039349436885, 32105.02148857779, 56467.401468890224, 83214.67852820741, 74252.98982036694, 68558.14588055016, 77949.47697115534, 32773.08499088461, 27940.611137461903, 46989.70654408994, 98545.93512039098, 33244.000168952705, 89506.38339735084, 45568.55253518416, 57843.90875102201, 10080.552151887456, 53078.174276654565, 36728.37636848345, 39456.38157488302, 87064.10811901624, 9458.015886124904, 30017.67624438144, 61159.486927076425, 46237.46270680792, 98004.41249184511, 61769.62453210391, 15715.59918019022, 56531.04452386335, 9689.346180548553, 87024.39616550585, 25590.17954544942, 71034.19209930523, 5126.294399256381, 77352.76403027613, 40278.74324481338, 40649.29895171856, 56036.136283219275, 30147.557732022902, 85182.14170492614, 48030.36068094731, 71079.53813122363, 26542.012829721873, 25769.636696380083, 12685.671943639243, 84634.1062265205, 49932.42494666272, 51394.587445517456, 60792.648463623336, 92906.88914727568, 17089.89581320456, 59251.659567319395, 488.09040518047465, 8660.3191038235, 50432.94021401586, 21968.436666555903, 54814.49710541425, 95146.86017167877, 58957.99838425919, 11861.284768113212, 54564.217671503975, 49922.955477722164, 34060.26270844696, 18792.728324838947, 48984.02553831848, 46951.03767061021, 95449.16554360298, 60323.30117068658, 99899.23362671462, 12429.003081612866, 42766.55907744056, 33293.074785906705, 51936.166995199565, 97485.24609665717, 46043.882106317425, 25421.85267315019, 75153.24810013216, 96029.91725154857, 66811.255777723, 29460.741904296163, 49348.61666828749, 61233.70980951699, 83653.03145554704, 91713.2152732406, 23488.416736620875, 3480.7069017206095, 25568.149565106934, 50513.920883665494, 90340.56456610562, 4078.226368000015, 17840.861341059765, 20797.05384361935, 41968.74713621087, 67011.06727944475, 92915.13759432813, 25489.60005694113, 51629.55521282071, 45088.003474833175, 18822.38443788091, 25225.15331217111, 15843.388508719747, 1507.756918313319, 29453.57065514591, 68194.41566443697, 70145.65063606927, 95070.85470360883, 75417.47573699424, 48625.068939578916, 10192.966663055302, 86950.51941851378, 35033.0830086155, 24812.899124641473, 32647.09768890477, 8843.139045233029, 68464.30714097763]} +{"id": 153, "vector": [89342.78815473629, 22189.077244344637, 23786.90308011786, 25577.409412543373, 45825.28944117742, 37110.42636176138, 50799.17874465229, 98509.93559830342, 67474.02728889564, 78689.68426396283, 4084.084907304597, 5754.842090146583, 58265.76872711958, 81661.98567842689, 40566.17018638501, 69384.12456490028, 84634.16863212328, 87280.76541656653, 10317.965676326536, 41895.21259172105, 75864.95066893351, 3852.7998102097617, 26013.208439570346, 50648.09430546071, 83979.77793129919, 59821.19729262295, 91340.49254982478, 58126.57258520704, 42912.41971359529, 553.9187679179047, 12976.734751377073, 55807.438221560566, 31278.689695517205, 68223.00813274067, 99381.70867428501, 95223.52660888921, 69873.26527475826, 47526.63546156679, 85009.50934264738, 89056.18893193755, 7806.047644726122, 80200.11571082914, 26029.543456070893, 81047.72913215072, 60332.77060224702, 36156.65702371549, 36988.10199644793, 27944.262161520495, 82415.55313010984, 95219.6996476425, 93432.7623234111, 61502.327872924914, 87303.33843155032, 11300.832411260842, 22284.76648389992, 77039.43462430866, 26184.822024029952, 63671.49178488064, 55905.716578180334, 67053.0930197408, 63287.24932601495, 89476.99116188615, 10629.034045455743, 4056.914359487851, 37778.17435345907, 43150.154960376276, 92920.67290339628, 85738.02778837482, 97347.87161984987, 77435.44363260559, 2713.833015758138, 21076.040246338245, 10450.467177896739, 25783.53781123075, 49688.04771148455, 7112.891192149351, 43226.88151232513, 33370.06072476673, 97666.2801997189, 48162.900481145385, 77633.23420999666, 18560.443026741734, 71543.71892176823, 51025.17757289118, 62911.03200968634, 26223.6208233655, 89219.20083915582, 66272.30545675458, 50147.40269095066, 15353.205270028458, 13531.061571544622, 74716.48396483478, 17285.05886613819, 8758.395012054532, 67845.98124108555, 7639.267022753993, 69403.51394238068, 98247.15857326276, 32997.097519036215, 85274.85317648368, 9827.372106029563, 57294.53714505211, 39785.02003263137, 29343.835084153558, 65530.62618632699, 27417.1105345296, 88176.11207974602, 97721.2039360138, 62520.54792603677, 74227.41188224094, 77497.70959293857, 41074.14554711384, 32656.990388803464, 32107.793381996096, 30363.20252602962, 48467.12033488981, 5761.270295191556, 90294.22345720309, 44033.29740405236, 1348.5425121545825, 32325.82608142056, 45194.14286169777, 19430.170190145946, 45107.06368823908, 88003.2043218438, 77030.3760456422, 79310.79628315268, 17752.66534751837]} +{"id": 1374, "vector": [82674.12862999088, 19598.683773776458, 22696.79869388256, 26264.862616395378, 14457.285859606107, 99347.29448510274, 81970.18212261348, 28955.77953359636, 55723.66345979053, 48661.29327821171, 52877.44442140021, 37460.372412590346, 14118.95259386079, 29239.24705420008, 17210.440543657, 22908.38938574563, 85146.35298318382, 23971.793935584872, 27138.281324443724, 19695.73077242517, 95258.07019768721, 99485.25262824987, 8310.159571232556, 12864.601889534033, 73383.61312663117, 27037.608228900477, 54544.91290077449, 67861.95480976977, 31389.680915456753, 19211.609509196092, 31486.811922387904, 49293.8205092637, 85661.94522749762, 71771.05871692994, 50441.39451395682, 35213.86747452314, 45353.20665971382, 1857.9356540799897, 13515.804652542207, 4752.954080752847, 2043.830829121507, 53585.16822979755, 79510.59822850752, 25137.678490146176, 88812.92640510872, 58334.067443534695, 34748.003180991385, 10554.051815061937, 11271.164563570092, 94002.48406249945, 4880.099371534752, 48027.93560061168, 93531.15772563925, 79128.64583058712, 31043.327814810295, 21690.71488521962, 40429.079627640305, 4433.549327317787, 53190.97916005676, 90373.03430355886, 85792.30784428406, 14061.940110094773, 50680.85181900829, 82074.94192829444, 12983.245669906184, 42834.438557346046, 52323.39000481443, 6507.513132964505, 46554.586462076506, 94848.13340065544, 78369.04567007223, 37264.61921922635, 43068.28142736425, 96860.0608736224, 43725.355587760176, 24837.72965609592, 52098.83320812053, 97782.67269479275, 33533.927877552705, 72335.12986427578, 24002.660784027386, 59063.06927824906, 19603.32262725376, 43916.3781527153, 34440.72728067743, 15061.950566586669, 71287.84731847115, 26575.04025920647, 74993.89738595534, 75234.26800307326, 67161.16724234643, 5323.509871829835, 48530.671899343084, 63616.70206872715, 98080.48379532108, 55498.23439418234, 47722.40291904367, 19079.376289360385, 95510.68012151957, 33670.20598999576, 78706.45470143197, 88137.22204161795, 22516.21638899255, 26479.83524046744, 1736.3503726161578, 6467.427805433234, 27449.393936442568, 73161.22028725935, 97107.43302926235, 19810.675118520583, 40520.61962706899, 65728.40262401421, 15656.419255806099, 35964.065799502474, 4679.3758408352205, 23182.215964807674, 69968.5330456628, 36256.044219023206, 27912.428454542827, 9991.244611068994, 67527.54698885615, 15023.429178793956, 48092.979108830936, 60213.239798810944, 87650.02378467041, 84847.30968573851, 38894.85991557973, 17203.854356554704]} +{"id": 289, "vector": [21748.821587921662, 38121.85695892581, 94413.14107165307, 62079.880791285424, 80183.55799256127, 26299.847470031345, 20278.836847989533, 7602.347134930609, 92781.28038259564, 13213.404877414747, 45977.939176102576, 24490.428505122076, 36093.94872922791, 49003.41500956078, 5399.383483586628, 39789.15737621134, 70262.71411010851, 64679.977631337635, 46765.64147269372, 80335.05343190863, 85872.38529647792, 46980.86753585647, 26330.946555229206, 23588.646828422134, 65758.41431157668, 80743.37850021828, 18071.1037783582, 35044.561718653174, 43996.77337015875, 84010.12474014396, 64993.44809900584, 99378.46081487516, 31624.042226580874, 38672.49655669153, 28832.299416238948, 77866.96389280431, 35103.40698700932, 40115.75206436133, 27691.588405333812, 58499.02230695865, 12723.391447562415, 2952.396236209387, 64071.9420799707, 76030.41515443797, 2859.27109010905, 43027.383469426604, 22654.98477909692, 27610.54035373538, 4642.856449839361, 44140.95365261753, 89527.93806812025, 22233.59604461035, 20103.709668874835, 49215.589480978946, 21188.4994047559, 206.12239053833247, 23792.024080297735, 74619.2150774025, 29508.194351679096, 39903.63447358955, 48190.623271746, 45235.40256904234, 31626.220127452452, 63044.84860354451, 81711.39935670097, 70449.51844258531, 51539.76253062172, 13592.740980714723, 36874.81094354512, 21871.643981621648, 90028.02164388895, 12842.739980332462, 21501.05514878896, 64666.246806489755, 84728.25332660621, 76245.04225336907, 63834.20592414363, 52301.46102206595, 12597.046174022842, 774.1015719353705, 43918.95584938183, 31072.84553435187, 60365.312713744235, 20809.008126896177, 88221.52965240573, 60690.38635790584, 68017.770077289, 52843.75441772191, 43178.81181648664, 79973.43393257489, 7331.488498315252, 3888.5880704474407, 83459.88133826718, 97035.50739991266, 16092.548921872973, 63778.13674205336, 79329.03912198191, 36051.50692959338, 55039.575930058294, 60216.93844022641, 11267.094346678241, 34621.15370763451, 87692.84236276474, 47635.809416147466, 55218.39522745834, 66635.25969972221, 4721.337486322541, 54661.867782529516, 49299.859250683134, 83924.7238860449, 99820.61205893777, 69387.53722642992, 67142.56540727918, 74262.04275948554, 88070.38724284818, 55661.83741844457, 30502.37111191162, 29211.055455689795, 32020.19232454547, 22021.11457085809, 17083.443105293038, 97561.92728592953, 7595.28715076665, 76247.93389456898, 16584.339299685624, 7123.112890096306, 56313.61036280891, 53082.40020878269]} +{"id": 1202, "vector": [6654.572462365049, 84223.14331402579, 91411.42379669403, 80313.32218904325, 8466.591390879408, 96028.00691609512, 70937.14695290802, 4418.53293942992, 89384.06813260555, 14653.338742383938, 95432.92649602589, 56858.05780145662, 59358.56957380278, 39612.16095255763, 8195.543688513884, 79534.40006381545, 24002.892770252405, 38560.66767104292, 14802.771689297877, 26160.92110560736, 75384.8449721585, 9707.166774785337, 67416.41212305895, 82299.52779554923, 45481.88981609949, 20711.375710567016, 79775.6417513251, 33184.67897203833, 8556.29339532935, 44421.69413653567, 26392.67401412898, 7820.982549508782, 50940.320883651235, 74519.20143813416, 87991.84153115014, 13175.317236744355, 88647.3639239627, 63440.273236633846, 71879.74309412952, 62362.87934789987, 82684.98111071344, 37976.32802016222, 64339.38587248588, 72538.58683248953, 50357.02191591759, 19650.822222790088, 18834.284079265228, 85695.68595879174, 89215.1249634958, 36023.40303609964, 45516.18646991997, 75053.83381806868, 76349.49608060483, 87032.11870375353, 39174.16214203481, 46994.99825317336, 84204.11397281896, 96947.21353151237, 8847.601025330909, 43452.82638096666, 47122.99246517219, 56741.10378411847, 77995.34228003061, 41809.54680301686, 51223.69536568766, 16079.803265287073, 76487.93963934401, 71304.85059285232, 95598.3975369031, 42924.883447212014, 23446.643093504994, 86937.73158013706, 12825.941954421372, 38241.5906712515, 59461.09065271198, 78068.15282273291, 63435.0104572463, 40258.09726465071, 32610.03300031462, 10831.159283480996, 83949.92408343805, 8471.34600687769, 7780.118640003541, 25486.88565297641, 99630.16719708683, 24568.744588330705, 80550.40318772315, 54176.660885551086, 71301.86985829663, 46395.611635544934, 75755.05139239905, 36522.20234132982, 45996.30737504956, 42658.50560340719, 47240.826136813004, 22251.508613192218, 13643.890703888595, 30583.997235615567, 95029.34930667683, 6844.620967978188, 4813.887421765406, 93826.73773741994, 45473.31514698334, 56604.84047069453, 84538.62159057164, 58726.939532428456, 95838.27943409035, 7794.324898151294, 41003.44773475649, 88938.54737995607, 51157.71088552464, 15685.905849497862, 132.5581582239388, 78798.32202539775, 10426.92794643203, 12943.826702212702, 76935.40537119198, 35063.55925948341, 27023.863603475773, 31432.234661618586, 91780.34662016679, 50081.36565751805, 12826.838757055359, 38546.32006928042, 90853.34961852842, 86865.39948258911, 43871.4685645434, 63917.61499768481]} +{"id": 197, "vector": [2612.5277105782607, 9081.603665405148, 1525.2819088772185, 13472.135513263294, 33993.026711238694, 72326.23959864251, 13966.546290059878, 89345.4997421195, 82615.35967648437, 91896.73423735377, 34641.15180554141, 35694.58087567721, 52647.916162213136, 65246.951815610555, 77951.91446148242, 45312.915175610855, 82758.03684092737, 14126.48120316854, 77314.75258267313, 73117.10152702082, 59272.768449396084, 7192.970539383093, 28539.027924737893, 62906.269112231814, 26568.507225976744, 14389.972950600271, 89066.91473273502, 43068.282174889515, 43683.99899046614, 86916.57534786925, 39075.89023916885, 50330.45674421153, 8445.006258734411, 32215.86564660398, 52311.94953809509, 76579.8085657723, 83998.97418549402, 80066.99070352242, 63694.26735130591, 5284.622038014241, 59182.507322964186, 76396.14582592018, 39107.747810520734, 67088.23340544858, 52641.2263427691, 23524.354354692732, 20490.770349133614, 23728.245726429686, 93299.48420382269, 71417.0143298715, 11246.482086756549, 59816.116671723576, 75194.6407651806, 2642.2681232782243, 4957.339098394209, 64822.13146093866, 20138.11326217497, 2170.834280038969, 89570.6606824598, 71112.5065586469, 15967.951137662272, 21028.41592943523, 79992.26613924894, 27060.15077215126, 72360.29113305194, 39769.597352222256, 12639.755936317688, 92217.85349595148, 81205.34347354175, 70123.08014829217, 8555.532734158443, 75540.88377644774, 5329.999156668608, 73272.95760946005, 92492.25354012591, 89650.19848842591, 11155.601122695301, 85200.57624040509, 56435.55020747394, 81440.40664126947, 42422.9678745933, 97231.55324364472, 9038.069492302047, 42727.846317132986, 12308.916398913561, 1164.6987181517688, 1452.9004987607896, 21627.631763608944, 96506.6622275635, 39122.049138401206, 82273.93859220312, 84087.77331324272, 97959.26823732922, 22018.104042752595, 69443.97880274396, 86126.59175586553, 43056.7013385649, 39378.78277576366, 80953.42943314355, 62732.15848710965, 23128.026248843624, 6362.828628201278, 27393.5816211054, 82303.11255569904, 1070.9581530908063, 88606.95271599223, 42732.889572755994, 12486.288434566017, 89210.37948927714, 69305.2359192592, 83834.15924119124, 44435.32534250216, 45253.06005939904, 26480.441781933383, 87904.96720371585, 23975.919821637646, 48842.910987784795, 87610.11211930026, 10481.419821983496, 21703.3219098725, 64505.84982907753, 50147.89346726942, 24590.20129945172, 78002.79085812328, 30026.672496712235, 31819.794522151413, 54.9734556280046, 19782.794497820032]} +{"id": 1362, "vector": [77426.5916734358, 60308.392610345814, 55931.3367241696, 37615.044516951544, 62414.61281613165, 83398.53879346717, 60663.31699908375, 9002.19107578364, 93503.00927798086, 18246.874987556737, 94414.96458360254, 30293.693758408957, 32215.362585062634, 52354.02793613253, 79784.88630475603, 7666.320914433533, 93783.30739124413, 83597.00982139657, 82825.32714349334, 68279.66443382246, 55234.77903659526, 8257.861894197804, 70776.63789885821, 55959.04539119495, 96314.54181104481, 31992.00999352768, 56764.31583113554, 44969.23332704704, 60316.07340583891, 65384.74577188572, 54965.90736535795, 85755.67104528355, 31933.5269851309, 48887.91432142465, 29799.717081095332, 87707.0441078436, 32320.11721227216, 45252.04926953419, 27119.90704211158, 46333.566490159115, 47884.96064952451, 58674.04535481344, 1521.3655477661403, 36468.021678062316, 97267.68601767704, 12298.508797875362, 99469.80751018667, 82988.81283415633, 80179.70159193501, 88138.49350622586, 33497.3574899579, 7351.560741973406, 98659.08192998405, 15439.981244961298, 56004.95193903649, 16381.036211646582, 24674.2652064074, 70701.92100588232, 63189.35302905257, 18654.03450046492, 56467.39754536099, 16473.813919058623, 12633.497844408814, 6147.038985063913, 50974.8748800029, 99052.31814802223, 21192.71144226761, 7401.232694242721, 576.236218839854, 84916.02625637146, 82705.87304693424, 80995.72809798401, 51515.15324472988, 2144.3196065145667, 29983.43868481046, 84143.90316902369, 47757.01485153847, 77504.68097619526, 91701.8221043175, 99323.20424369168, 37373.16898241082, 90174.88490779426, 56819.01874860096, 71616.30273296779, 49342.85407420362, 31279.86990099446, 77264.07153383028, 99506.71918727312, 4560.183616642388, 61242.55090253746, 51241.92281713542, 41323.19854684222, 58650.47650667874, 69528.78686365829, 2907.045851944334, 345.32822262666184, 1119.5637961601835, 55498.141722382854, 97804.06622884348, 69969.78397806184, 85935.26561827736, 71570.72518091153, 53325.05607629171, 41433.28885369917, 21687.736297781634, 81474.36304153048, 73756.01267925483, 55856.55512114893, 43322.956184758885, 60698.06441231273, 37394.07566389404, 14512.547739394311, 10852.858618318029, 12160.645648718626, 55028.67602112399, 48177.830356991144, 28630.378728733853, 28203.539485372963, 63805.08295222415, 67305.92499881721, 12354.010641353918, 10261.74380356607, 94445.22391615863, 61184.68476160459, 97270.41676357335, 82364.52840189393, 85636.97594733346, 16433.095969235645]} +{"id": 214, "vector": [83947.31504250855, 54636.65808107774, 94825.99082561942, 22683.80408499765, 53701.71917572002, 79824.70004659818, 10907.422166180137, 81816.45949948762, 45236.67713427868, 69153.2295514394, 57134.894648646, 41943.05623903449, 45608.86804252503, 72331.7506722726, 1056.7781437469614, 82199.06846270649, 91284.14218021407, 67682.17134464893, 10577.332589774236, 76899.50166827302, 15676.623206936036, 78076.77211440179, 4252.311407520293, 79457.1421903114, 12440.295608823915, 6994.760800038913, 96285.67956600933, 45151.71975986102, 68549.5413457788, 7006.04717967972, 92867.53831450429, 69862.90503288877, 749.4999493604437, 59584.39865282934, 15615.205793578834, 50428.27488814645, 86847.90927364418, 41718.94381964876, 15786.145330315727, 69339.9325142037, 26222.637996500885, 97605.92731119896, 80861.05509556748, 76984.10002984904, 81966.1812242974, 47236.37963898582, 31318.474047001167, 91055.21660166013, 79500.55615325311, 8329.133143977797, 33977.20958999288, 30865.931371500334, 14517.944977721465, 97297.61447782871, 34489.71280041596, 86421.0512627555, 11809.57983699481, 46950.5435160002, 85769.18504903327, 78992.60009422744, 33689.34547425863, 37923.90744043972, 37158.37958143389, 69094.52397355533, 63109.34524945381, 95148.50549705976, 43930.82965354666, 78352.09352216283, 96336.46581638536, 68875.33065634906, 47073.88975938771, 46662.075748405274, 62104.7911026497, 94969.12629586118, 16079.150443386314, 80076.43760304392, 18863.49618406785, 68106.63499633563, 91879.08723417146, 50469.474605039475, 54840.159799945264, 56798.790161899815, 33305.34903050959, 5433.791287212275, 87872.64067576359, 39093.05318881625, 77726.13970274338, 47474.69757712822, 42174.927676338535, 11845.887149305001, 41853.41730736033, 31220.586926425287, 80487.78911780458, 85366.41074682916, 83130.39527772725, 60659.57365632804, 52339.33852411764, 96339.96909456381, 80971.631757217, 67408.36966558515, 88263.38959430273, 86055.11491743906, 32777.89080894071, 5936.866787690132, 18842.40961371927, 86025.39809064496, 46456.748403191785, 47620.460108248575, 31811.4964963123, 87484.11824017957, 98281.90132638435, 12428.923667994562, 40325.4455008123, 51853.78227696579, 59507.672772519385, 57804.29174967833, 31508.92774946431, 26058.19185117736, 58726.16794522464, 69248.46493347207, 72462.08967716989, 43335.504965388835, 68581.55147827967, 43522.2415997747, 96619.09145334936, 92839.29049291577, 21293.67083252909, 94453.77514969227]} +{"id": 1949, "vector": [85748.51128224848, 90675.97403476217, 73998.40365428987, 59318.98056438102, 94698.81605485537, 13069.304810042182, 16037.508679482526, 48767.2363088875, 70680.1264824477, 48249.23767341572, 22663.58286160346, 49907.020471854004, 71372.87680335736, 33429.51203253945, 92385.40979819404, 41864.300903261865, 93224.81772609816, 93089.67513533881, 71034.17632291753, 51603.43640772063, 42305.04856166557, 70171.5119600044, 376.2463653932957, 68886.49045591342, 52364.29408099897, 16029.785147322873, 33882.42714393736, 97367.64010339843, 51515.43210594343, 42193.07246187012, 87122.42722067567, 96486.78530093278, 73525.08122756811, 46262.580512800945, 86541.60648930163, 64176.02244584578, 49953.45435254409, 75095.1836157075, 85406.24546609123, 45082.02811047068, 2833.8597005153733, 37452.62210704885, 27586.31672267029, 3537.542689602613, 23669.577863371393, 63957.90907537, 78776.60861677917, 33638.08436393439, 63667.068801209825, 92600.44498029216, 85329.33915306286, 58100.69589418114, 31649.824285250637, 58910.84155529637, 58903.90159435676, 72868.38954407965, 79241.95615732021, 27307.561774292342, 14900.587781133034, 17124.63817293537, 75531.76070969664, 49861.372355315005, 10841.9114346887, 3832.136987684631, 35166.00968192099, 20883.833024490195, 54170.2207372542, 88302.95925460546, 47996.79066400235, 55406.95426897995, 13017.211960353336, 56428.04122133231, 60324.52823425567, 31726.119081538618, 51813.96747009062, 66828.52593585213, 32864.36895680784, 46331.94382437675, 8237.727486798163, 92498.95627406637, 20742.494738826168, 78319.31068925519, 73661.99642715773, 89680.38373741803, 26641.494675828937, 71881.86779300816, 61286.33708173414, 56394.81820370027, 65643.45077061835, 36770.20148853559, 73454.38308264785, 34609.756884887676, 77975.78876623801, 10229.407647530086, 77400.98483034664, 59716.440641483074, 67327.49471075133, 66132.28783114889, 27654.187028570854, 69649.76504719806, 36864.281990036885, 97845.52501860689, 45210.58618900954, 32421.712737954487, 87212.23422202388, 54315.99634832735, 32598.866975011042, 31917.02395562166, 86765.04133373934, 34760.178766600715, 47002.89870737713, 53166.27997053917, 68796.22480395297, 16687.398372155683, 64134.48613482049, 11493.947890724454, 43446.628498885075, 55673.98569629518, 12681.341618533215, 66491.45810970468, 66511.44766696771, 39075.779519762575, 95975.90122330433, 69937.35872529229, 90648.6074638249, 21878.4014431196, 6171.570309519902, 14437.42819111593]} +{"id": 1493, "vector": [184.4889405484107, 74910.18132903347, 52186.18963990454, 69108.31519310686, 19374.4578606975, 14081.637918720136, 10808.415217220458, 32615.042027293563, 94790.32678023094, 39531.59584824144, 29634.65374920532, 9085.606798639523, 93280.84849446025, 17688.420993950116, 18418.13845337804, 65884.02854287802, 67409.92775263933, 20793.496099706845, 91134.64901056858, 18432.908296752827, 37607.69894516411, 77318.41991755065, 28874.687881208207, 20327.675980656913, 93082.79545677284, 70812.35494697408, 75133.95290997034, 15954.501021586499, 76305.71369228582, 46889.968246784454, 3790.7611439715215, 8655.752639698965, 75759.04165401045, 83030.23854824378, 88397.29105859762, 69303.62432624897, 41216.47088681779, 66154.43982658589, 7025.93745294976, 4708.299124668791, 37549.209806981366, 273.24559685933946, 75597.56248073629, 64544.75493262276, 94864.72196479881, 48399.63098129125, 72136.12041515854, 71826.763717098, 83581.17317327605, 34513.890990828724, 55019.25433252472, 32003.72586531891, 50873.83033300959, 61539.8317203393, 53579.78694474576, 60823.96090411464, 64124.985890784155, 74778.44689475947, 84261.45352204806, 85748.75230101938, 21559.511794635968, 64350.37313215449, 87273.16000408506, 64675.49237371566, 4414.205399076898, 99332.40341207312, 68506.77990912371, 40945.88623674352, 3872.62278311189, 34877.5299899691, 87750.10411827575, 7380.201846239498, 55403.44396639223, 804.5182402447448, 33060.471541762774, 40542.608407313506, 97370.38531646195, 8084.560392294693, 86796.3415003003, 79987.65960311679, 9905.254315513012, 5086.338617321906, 46197.31423066108, 31395.741858641635, 35635.06885262052, 97137.72772131172, 37589.6843411209, 14759.643937088174, 64500.986066229416, 7864.146292158413, 28983.819043335312, 16437.034079782876, 23358.071646716784, 42036.97864853576, 68800.59500429929, 4610.907101849204, 88213.24777665264, 16582.508670469786, 16722.428737787097, 68375.9923038123, 67210.72609700216, 61738.68169743708, 91950.63461202498, 67486.66152301978, 79443.9712859782, 23218.71180434445, 73802.19965353428, 75364.06111201663, 87031.04161032337, 58420.2947568256, 58276.26428123468, 96724.1388962548, 10654.280427369711, 70551.08444823982, 30646.38571049464, 32948.34093335014, 75248.7742311432, 43101.17451988149, 95479.70220072371, 72914.52028782143, 82954.09005048408, 54067.94205118066, 14153.553989791733, 6258.394315301141, 13770.380452529884, 82012.81278410707, 42757.72402478093, 21390.174491322156]} +{"id": 1721, "vector": [114.85942996855015, 13613.28326052793, 95693.23951888103, 83996.52927035581, 90145.86943435397, 16515.531964142505, 51690.62509267591, 16226.211346908192, 48288.64582940856, 91482.49508395653, 55919.77281586046, 97580.25560150863, 15876.835970765445, 91009.1267823584, 78337.99412602116, 73680.58082200249, 6284.142796826886, 20163.174285048724, 56379.94762278087, 32015.615059713764, 59388.37540590351, 78120.11228387001, 7307.724631513634, 13687.822525150239, 83747.82766051631, 48984.28053201453, 34610.15321866312, 39275.04738481955, 41847.88138627499, 69997.60440277826, 36583.9439970707, 81564.34400002622, 55211.862613637495, 76268.74462829434, 63134.57458703746, 48405.911441259734, 55916.19429112319, 8376.506501822023, 74478.3193225106, 96854.74639392261, 22693.313074945985, 58262.38004971485, 40846.322241777445, 14323.163281182284, 49884.97519136378, 86842.93797274529, 10437.048932394511, 47315.20637873878, 5214.777247655067, 92379.35166823867, 80591.90351601412, 54113.91866231607, 99925.47948505102, 25732.771880841377, 91074.3703871216, 79840.4087972133, 21603.658464070984, 3952.425082658406, 77806.80372265408, 44462.72426326476, 43224.92724000908, 14917.144844447117, 73224.28584059347, 71795.62110665462, 69500.95030891628, 87907.09055259834, 29255.92824146025, 3087.4566979566366, 62325.539868064385, 88891.40043445217, 63557.600673096924, 23251.82418610413, 88035.24762534648, 97334.43934832142, 91263.17029434159, 99556.762243612, 94982.38386893005, 42310.90217691588, 46517.69371316271, 66126.11432927867, 85992.21672304523, 59763.049492414975, 41572.091174708694, 51528.70121896539, 81740.27080343125, 63538.43841997707, 10304.170023486526, 46555.39265897418, 51270.86013223286, 67410.57510460881, 32765.059687318397, 27266.232052670002, 10567.529186356584, 15608.775102129635, 77138.8660356331, 25061.429241964637, 94087.38219903636, 29862.27080896354, 89078.12291095054, 20130.299261652784, 58692.11043042794, 93033.50290144839, 62221.58219359276, 12250.586070781821, 23667.519882113218, 985.9179124669826, 28381.568636151333, 91287.37232162013, 63454.06251116918, 11912.599063105945, 78173.84540482165, 32605.62829642003, 8284.658139121071, 12778.248135394577, 44984.81236464104, 70819.26051736713, 71618.07726417722, 33961.435358598246, 40951.34349508183, 18222.131421868216, 26449.74545937673, 35820.25644828756, 72557.9348889381, 85160.12981549073, 13537.236869394153, 23652.73181081592, 89513.90068596056, 78492.43686519434]} +{"id": 1692, "vector": [63569.89437316573, 20026.199298137402, 77175.40804545248, 70404.82590934186, 74586.53456591879, 55180.11834641851, 87300.58297128344, 36570.89422133425, 28605.021881477023, 88667.96874336562, 18240.67642810252, 71050.04868559568, 55217.510876468565, 95608.70732088918, 14384.095515947338, 19869.36929552301, 84416.6921220615, 6670.952461521684, 95350.8319572603, 48897.84244829385, 8668.965296395836, 55897.04708603148, 19372.491409629343, 1038.6161672629958, 65508.29749009474, 79162.64959709137, 6613.633398106777, 96994.68706272838, 79169.77447215961, 55794.72250097239, 60257.56639137697, 81523.02212409842, 40247.3750131307, 3470.2677359482072, 34358.962072088485, 77603.2291160519, 14515.244506503777, 77863.03965465752, 27977.41387218149, 51479.278479382265, 89181.0573491718, 24627.556708165666, 86967.5639945201, 5167.942992481667, 16668.622325903183, 63997.23144408276, 88192.88894530632, 89007.28421517185, 67863.1737285223, 50195.24864435417, 16321.365402310961, 34840.65869870679, 19806.713604358916, 16890.08250116465, 95403.93024788429, 55260.10678287906, 55430.493630779, 79597.98000309205, 81855.25441482302, 9997.52626167213, 46592.36024193372, 46882.069944143426, 8833.305679336501, 86771.21718637359, 75178.37941197543, 69782.2769327769, 55123.47457511823, 61363.18098906272, 77951.95238769238, 65130.869581994964, 97778.41535672033, 73066.50294205901, 18794.378592635774, 44625.53353894118, 24249.704796481397, 47137.86815787618, 64158.55250224991, 12977.424883488076, 77349.61249175195, 62033.22812112867, 76131.12635459154, 57898.4741723081, 67760.44783016425, 75321.08332655062, 27940.0393038612, 64761.911928534886, 91538.3211765926, 88178.31927441512, 55707.85375380476, 38404.87436247747, 46530.778923321515, 7280.68310003066, 39013.37616465696, 14542.089696146888, 58917.28717894901, 75837.85516384541, 61481.58160396537, 59140.72005415767, 82744.71364210073, 14513.877350294746, 88211.79875938721, 15757.365142701963, 55717.85110651027, 61210.58307582898, 24424.17119697723, 98326.63310607354, 94574.13427861767, 3004.6912950508763, 35529.339084277344, 32407.707567644717, 92412.62260362075, 7372.201472325801, 92754.00868694698, 42136.70064906887, 6217.076656139253, 72379.28876355445, 54506.64170936177, 39820.913624887355, 18086.42192525005, 99489.62231644553, 840.19986751519, 4265.764205279654, 86825.9082191683, 98605.61407821061, 5250.699271470682, 16973.491745371015, 19299.26240162119, 53408.42972920391]} +{"id": 938, "vector": [49342.3492329095, 86280.77954799, 55511.414129314464, 50476.87017712191, 433.4892857627892, 16273.517622275634, 46189.58321739653, 30632.604000519805, 8737.10330258306, 23222.85662542385, 39865.86639424039, 25090.22464643732, 16453.093443347643, 8521.18118183436, 49725.75052238265, 29926.35314654892, 39850.280587873975, 9017.124020847355, 3067.1723875497214, 20030.510708663984, 11491.666664980083, 89397.96924670612, 51486.850052778886, 79270.47527406669, 78973.15064829018, 375.0001994332952, 60938.88877114263, 32444.20999998526, 53439.05445166313, 22345.93230137184, 76432.21424088567, 52230.8889390141, 74219.213126968, 79638.29748704267, 74245.0382043792, 24239.819577724185, 10912.003573071705, 42396.47848638608, 40752.21799104898, 99826.91544898543, 70891.9139099605, 33102.86543607409, 84588.3355576733, 12980.343161333185, 44442.00022405209, 75232.31119046344, 48744.68742454041, 27138.282871351683, 60413.43484270003, 5756.202853825387, 89741.29373975378, 64999.89489495934, 5428.6920769773815, 48133.789536203476, 63913.37566899509, 9033.230509379508, 6668.574052644305, 48776.78910999267, 33654.3529757125, 15930.377601640645, 37671.68847507827, 24813.48776889055, 51729.14651928817, 60896.23920097282, 48704.514400679574, 53060.19236519908, 10008.2650805974, 48375.14040814921, 89452.29023952117, 47555.144025373505, 69020.3335284041, 61188.978124988666, 99952.3086423106, 57180.54854472303, 58746.09266947911, 32093.365207832016, 66395.06350092156, 66715.66806549497, 97205.93821325815, 44360.56386276072, 23551.260878279023, 23891.231950105386, 57050.66870538147, 6130.920188290756, 99035.50392961537, 29199.194360006884, 83111.09011670777, 81707.99509676578, 8431.634088019913, 57937.02842520152, 53706.370795773626, 49663.824290803015, 7923.696058688479, 55678.97451949281, 83428.6114174265, 49473.52095456783, 11917.398517791566, 56357.048775892945, 89943.1739246729, 50625.253507922076, 33644.55287461543, 22516.25028653158, 66398.46490139888, 87541.55943830029, 63144.00839469988, 51990.86279532188, 96987.27675443084, 49807.91230540681, 76043.24772328249, 26835.59871108753, 79953.53230855531, 41939.906966451716, 31873.20859193855, 11168.61926006617, 73292.74912195686, 11156.089582870265, 72000.64896503955, 31851.47967871137, 64513.81000158004, 70470.86048447735, 66759.20158628329, 35036.19567145151, 8397.990948752898, 71604.76993848108, 78760.55016355346, 9676.552860214482, 79052.6808300292, 11459.938084574984]} +{"id": 777, "vector": [73733.46357201692, 99149.80643829785, 68265.27929367336, 43170.25696722867, 32377.86314655824, 12360.191135119403, 58085.715858654854, 56602.938971146374, 93450.20462474137, 76840.50519065716, 29004.57522904808, 27205.807001373505, 7277.315291351394, 86238.31849071625, 63715.860349505834, 67184.98452747677, 81642.7062834912, 36026.923068927965, 29239.89951160736, 74904.73282165948, 72376.20962550018, 92338.32582756471, 88075.82899687396, 69030.35723337738, 40581.799186439406, 47076.651634977505, 27310.432169254782, 48289.26558770591, 58451.9231472325, 95868.26784487147, 28544.98658475253, 87832.44720843041, 56315.0914945363, 87636.57485896078, 72268.71218260999, 26638.192096253144, 58955.69176563702, 743.106939982674, 85815.25485164855, 63592.67682464285, 3176.8801474090114, 30035.36211954555, 43491.89352983767, 51359.855460717685, 98045.5987817252, 39563.0571975186, 87227.18241330633, 66507.22406529635, 56613.56181123894, 22890.198270799454, 4372.635612285291, 24384.126722176003, 80183.64087101136, 34928.05590300497, 63036.73638270205, 38237.39551299081, 83067.05389050044, 93504.22850317124, 31365.738111861974, 2020.4928496906582, 31199.03970081276, 64354.987886157665, 13804.861585952389, 9458.360724666614, 52141.93275964411, 120.3554567195786, 93611.39944771191, 49867.28927379902, 95053.92005754501, 72336.03230689502, 60231.86628538542, 71490.46676703637, 59033.41644244507, 40152.70666016604, 80502.35294678125, 46135.86833377996, 52993.58408064572, 58150.06478520817, 19513.362485107446, 57994.23446064345, 72195.83864464032, 55943.47127131067, 3552.6483631551087, 42505.45582347459, 48637.3152443537, 7821.863361826864, 52636.46546796711, 48164.014314197855, 99925.0192847375, 17782.972060861135, 11224.653358596914, 94306.24344178376, 24025.88063723251, 59468.20932031362, 26792.632718923403, 3377.240988810648, 94618.61539457647, 2630.177308792159, 1277.1054950155624, 32713.505459378823, 4302.192682988281, 15522.98736777512, 84339.4938709237, 5602.983474153511, 67419.53538139963, 61357.231888913, 16342.448887237726, 22900.233056831254, 921.0114986322848, 28745.330634517653, 48186.875830844445, 57864.37603308662, 34000.35410474647, 81058.88980666426, 15421.887073469064, 54647.82040975712, 33947.41982029075, 78195.49801436064, 43998.87434159206, 18153.111011450495, 10918.001253984377, 60783.58363895199, 48834.26493940486, 81543.63653594417, 47981.732500906626, 21264.094185837013, 36784.820744902456, 50441.64999640296]} +{"id": 1266, "vector": [37014.3217397787, 97510.32368060756, 47507.83420896637, 5322.009849664122, 69630.54215693565, 27978.671896408734, 82429.18429172829, 47165.978576531554, 45676.9872540569, 73428.43932046543, 43292.7159464365, 67831.91905940408, 81039.2054157682, 94919.23817836077, 3370.6088633400855, 14084.343706602176, 49474.707266090234, 89269.61023631315, 11312.40530266655, 1722.3623418517798, 54308.62953539267, 40280.31349011602, 899.5237989747662, 93628.7256570453, 36463.12123771274, 9175.65416338696, 584.9098378615913, 12908.397360888657, 89298.4642310567, 11986.086653190509, 54275.974947908886, 82070.63044513337, 84485.95544709798, 71900.84741494776, 63909.31010038129, 84922.60884069538, 6470.899096194538, 44694.99439917559, 63786.13966633199, 74054.78265412572, 21080.409549319047, 23215.446295606234, 50821.015455171844, 18737.07884860537, 73490.00247817242, 36075.44302214325, 2724.193390363594, 21302.315596568933, 76091.55053942712, 3131.8543135178234, 84135.54901345095, 92196.75384688796, 57854.03063058879, 22481.443403973688, 7870.972011478671, 85167.98553710725, 39700.94092111172, 73760.33568100576, 1936.852807902234, 57822.47676724689, 3272.0957382518236, 82239.3481390777, 30874.717084236614, 55803.76094438691, 23497.27112400013, 27689.405671265733, 74658.90423894662, 71563.63378835075, 89289.01228017584, 74271.0389828219, 25349.130526070672, 58540.962151028885, 41671.03775543115, 38694.684814362736, 3151.4052709713637, 4205.7134906229085, 66246.22612838351, 31322.097128095593, 14284.722974780228, 7993.384519930691, 44649.64948883231, 8274.358088164458, 56067.09431221246, 38346.087354630195, 88092.89048979494, 98483.03455696272, 29978.67922455314, 2883.86499083475, 97133.84034693813, 39979.92728529785, 9900.766970322062, 58133.151526619054, 7445.206168213081, 63513.794558392925, 72086.53808630924, 21430.815639745237, 93299.2691705202, 55890.46951077982, 55154.50219142321, 84123.876548098, 66511.75133648959, 97060.75293382259, 81943.84987806145, 35934.21043019177, 2624.484953271666, 98428.32446002105, 10824.972635516027, 54694.66063757415, 32417.890480363876, 36599.41278371017, 91615.98745190314, 75549.94324369493, 81691.48997199655, 45376.31021920876, 84659.91712777421, 42349.02295623909, 40327.33213775165, 77758.3317176452, 44981.6976490727, 27804.2375329134, 69436.50001200478, 39344.223754327126, 17574.057926288744, 91023.36708403984, 19504.67970778791, 55024.33113326185, 32117.899431001384, 17675.29903787529]} +{"id": 863, "vector": [57067.02875946464, 39329.4853556877, 46511.15346532047, 38791.60345525494, 63164.113819845224, 20841.209183321098, 73574.4175134493, 76914.61450801829, 95435.30268516898, 10749.531345635032, 11003.3834305715, 42163.12880459845, 58077.202595335606, 25894.045356350194, 32957.26692252835, 50902.684904650094, 39323.13031642204, 7129.226766158736, 62655.46263251724, 77798.94806780088, 39997.308186357026, 39294.293136875014, 70819.05848970117, 71831.73028568161, 69725.23864761979, 8260.203486983542, 52926.420731450205, 18846.180777115907, 23094.299648230222, 67457.81030340878, 51749.8517414548, 11503.04804916188, 77101.61074499824, 78962.54267449802, 30213.479001208398, 64949.661763508004, 26671.040257403754, 33942.95082876746, 19809.725197041495, 54664.78576239514, 22477.802995431062, 6552.381256622431, 76904.26661575897, 88082.52446570272, 15793.169437383847, 31882.372671899462, 81984.05509415558, 74283.54429676772, 34759.04259352176, 17882.558052016928, 52132.83002518999, 38345.4562505144, 98107.48405552506, 14007.358665504898, 21727.344589865384, 3049.5267754041324, 61839.352494304236, 48858.94029776402, 61660.80147706686, 27847.79383159518, 33902.0015048812, 92307.52024907501, 1309.432274954214, 96281.91736612968, 64992.26344542865, 96638.59789655427, 79023.49046064845, 83868.16427160482, 53412.491173212606, 99204.90628333803, 30600.344385817134, 95067.1954705426, 3313.28389778226, 80504.61042668641, 77011.24417095899, 48576.08408699253, 55580.54240107113, 59328.802796221535, 29228.86051720005, 55900.55683704668, 94734.70596100966, 2917.134562935342, 53615.97617099777, 76040.70470828678, 18997.115548655176, 4417.16056764726, 27099.048596527096, 22029.72941532799, 28223.819186615452, 14954.915907712695, 10207.491109281109, 35858.14531928945, 46011.43898402318, 97515.34303086704, 33318.09762249949, 13010.429912086807, 21425.45545435085, 695.5889975495455, 93458.08734809424, 78708.25520606927, 17110.935346797174, 36671.46917729091, 78974.89350127048, 45516.16089584565, 17655.962434270245, 19223.841866776103, 542.9571853631354, 99329.15429279371, 41313.74426788493, 28573.50404440143, 82208.4723011144, 14873.783574509236, 70123.83738942003, 63453.4910689353, 55287.9618844335, 65911.93951969322, 15911.453956329513, 83894.02283267894, 67752.72804679397, 29310.82875563682, 48028.57544166752, 2282.71980526763, 66752.34680964348, 31464.372344848536, 84198.95653196544, 64717.669180086756, 49965.410524822626, 1071.6496932218101]} +{"id": 1497, "vector": [82566.72939028162, 41862.78529933631, 35492.04120586995, 21367.437430006452, 31229.382949583352, 1572.1216212755796, 24043.95355054556, 31752.69507140047, 43961.726393999365, 65581.7590606792, 18277.73784627851, 30679.30707423452, 49710.19566094943, 26702.39925858189, 23371.06979656135, 97015.40804187657, 84403.01018588858, 24485.49599065648, 87006.91265808404, 65870.48733659102, 92972.60223743731, 26329.738032168916, 25091.075443118705, 92521.33525168043, 51579.96501735222, 42065.98997852442, 21465.67781350979, 54022.16075437388, 76900.5070632409, 41980.47272038512, 29303.40191097871, 85448.68059245565, 9701.007407874251, 94341.56219988583, 96586.65363210814, 95335.33824829741, 27177.517257411542, 94251.73801519578, 14539.350611221147, 74075.72203409897, 94768.29908333934, 36597.76351308325, 47469.16051251332, 62347.42440805312, 21336.554682054688, 22730.10400335347, 44669.37380133722, 65852.13159389261, 762.191021389813, 1740.7819984210482, 85854.74334762368, 93208.01105172145, 569.131966835612, 56409.05079902158, 73253.06557060627, 72871.18557665365, 14095.070275568178, 34169.870573484084, 78909.02986438587, 50119.95836361284, 31450.32403180439, 65140.2174888178, 230.56205029581278, 45819.10133461329, 18712.32359537164, 60539.91803590645, 29603.778992974305, 43006.608775701, 59021.951015956656, 99092.70937933552, 38885.45542769504, 75232.00860937282, 67007.71955591456, 38505.222954466255, 12104.507499520945, 35146.492154055006, 2762.0084915045973, 33421.34400146023, 61485.64893825289, 87965.60656866232, 3416.942997964334, 53779.2576194581, 62877.69180874629, 67920.82321546078, 33580.29310524522, 79979.57709282388, 48165.122314834094, 46598.225025652166, 74855.7513162168, 67204.38006109514, 45573.50621774161, 84940.72895685567, 9825.519525455627, 44178.334381163506, 24865.932257398836, 19281.501232711074, 64436.38773167544, 69279.0181321358, 59213.75909563812, 40171.55753333679, 87582.13169853117, 71209.60269314636, 58574.97488725919, 36635.903035511496, 17892.141772386993, 87235.603797546, 8343.737348100822, 76504.88107832924, 78705.24122273376, 35523.69368680405, 3540.6928714968753, 58329.62883667942, 37912.46991405597, 22777.171964930963, 44585.158830094006, 59048.732389113415, 11677.117843712793, 42387.31848358306, 39223.696033278524, 46255.94899331653, 75510.86171202036, 64935.22061671293, 70752.58121538693, 57090.564141990886, 5218.486980079562, 85915.88462958708, 13747.776061369954, 23793.12096538019]} +{"id": 1733, "vector": [34037.42736545257, 10128.76961210638, 69325.82529628075, 27356.181078796348, 92655.63876625929, 27788.069696136463, 77403.28578723964, 54139.63121832712, 30842.928863783636, 65329.51850401388, 34245.867600022706, 20982.206857898756, 26347.75274760761, 2049.2207858878064, 65826.88340808863, 63444.99070811804, 16721.357138313542, 87781.68005459645, 43452.310375064815, 33266.314866026536, 54276.32670487215, 80349.77809117398, 4357.013053221071, 77308.32603748322, 75144.45387855424, 38769.18262248707, 72429.3697454604, 54147.5898216756, 16884.221665769648, 61965.27083954633, 92807.28962569006, 49155.03686254852, 3131.5461747319473, 37439.530824722526, 27993.368203393908, 84199.0207321376, 33042.27621786703, 7636.159320436953, 14968.737484144789, 89513.26437945444, 13367.886302355004, 9559.009171690424, 23677.86533760039, 88700.69285010951, 11096.984373700914, 30892.55218674043, 15302.911174419376, 88428.8215119529, 55843.37582743477, 50196.524026706815, 39286.467661486844, 88745.95374164017, 20318.93544323017, 67317.35909643625, 47889.394634961754, 40979.11195603633, 23400.35278200768, 34364.83235903552, 63034.690995849196, 59567.72956647033, 30579.27212934336, 58412.691795597624, 94978.4705367766, 41908.5181373762, 35380.711852960136, 6899.544017494774, 51620.19705376674, 77992.24392889392, 25490.855673626455, 92780.68719292688, 29426.696649918715, 91076.33822647455, 17598.339519429417, 48303.388889853195, 3634.299983993494, 85983.38350850406, 19719.607662542738, 9655.971545511089, 32087.878754941936, 52085.295966000696, 26156.0618412473, 83851.07324049562, 38113.434505109755, 29178.543399152666, 99306.00096915006, 55608.181452238285, 98207.87465657026, 97233.38267562969, 39656.848052388406, 88817.33739397125, 40850.38129378315, 25295.607903992, 38561.59567942596, 4317.757158334523, 45894.42870180906, 37483.31772290696, 46009.71292962255, 87506.74631875471, 28968.45370868957, 96100.19011408446, 57465.51042624436, 3058.849572755806, 44536.11188897127, 38223.92773688281, 119.19851423589645, 24189.045962611544, 68075.86944151932, 87502.84257568418, 54812.34101218929, 48204.042156743424, 10502.171307535113, 7048.335077264789, 91593.41723453844, 37317.268618641145, 86974.77695687386, 52559.80322508805, 21758.320198820402, 15012.95499880595, 60357.87752449157, 42225.06853450339, 5455.758535481247, 32547.62476278591, 89858.24360635495, 54534.47133685493, 98716.96777309703, 9992.653161562392, 6992.35824217389, 62219.04696536653]} +{"id": 682, "vector": [17816.49053789499, 56530.47938020744, 71476.93591835607, 83006.03263955476, 47582.97679249659, 60221.27042200045, 11982.261925717663, 5203.899730301742, 36350.82600515049, 14056.935364650037, 17478.25403236496, 63751.010309543, 63509.9394053956, 91085.13330803798, 22089.064050686113, 54102.062554955824, 69704.04774508342, 98079.123753231, 16643.546391156717, 22078.04483540631, 66958.49548199475, 45096.690174750845, 62117.80803027623, 61590.281208725675, 38586.62052004258, 8203.826922130953, 37964.56708418186, 19157.80369540392, 68498.90702971737, 62691.610083697415, 86488.37685724064, 16536.490798272975, 45000.76408399395, 43883.06763253067, 94816.41138099022, 53295.72812236805, 91743.67142191433, 67965.29292961115, 82955.00212317044, 86686.11203396184, 39224.90589468577, 83614.95712563195, 68227.05621127176, 66519.6412163083, 32625.49573121252, 75383.67567204169, 80744.70402696393, 51947.422354198156, 78608.7326242362, 1081.2736945026936, 24260.82764661859, 67249.07274333198, 23816.29647396031, 50958.87881955435, 7240.680948767197, 33188.90674508645, 56116.81364730341, 56395.891840121025, 74333.34598324144, 65573.13354798136, 72670.4632305798, 42984.91861905249, 76975.8215548612, 6181.829225280932, 25808.2953601108, 21079.61449253808, 62479.64477450276, 86748.70388631788, 78726.57612604662, 18138.18594177766, 76918.13802699407, 20805.849248199047, 49237.228988475676, 79075.98906456325, 98519.39388400388, 93385.1092538674, 11207.26527476954, 70492.04628121085, 39153.31783358004, 5782.2292841657745, 59182.00640031891, 5658.908839982258, 68701.81272549396, 66583.62498779657, 3850.8927796900184, 38338.71624419817, 74934.01967996011, 49980.82499424742, 40890.87218254943, 33935.05501992794, 56074.207654886544, 96291.33007680229, 21479.659947318873, 65258.87976385973, 15497.757750050734, 67620.59566197028, 12515.019921823377, 9603.998951460691, 46467.08860797963, 43091.05011897738, 9274.527624299311, 98910.22399274906, 52860.613820140745, 79392.43302249495, 82870.65003787879, 65879.07930862806, 40546.056517039964, 19235.833500716835, 47943.68324840431, 29195.95100555484, 13375.814671719378, 28280.93487105985, 96296.7203814591, 57781.44855884445, 77986.35251269827, 88879.61605821265, 51014.035292913315, 14017.555103019242, 7252.1448605165515, 36377.86419805824, 54700.87562903323, 5007.122462701086, 15740.54250824094, 35994.98730600763, 66935.57082995275, 60570.87403717969, 74912.8876904559, 39546.77138976922]} +{"id": 2024, "vector": [99014.10635244694, 88449.60137818663, 83515.15599984037, 33928.499499359335, 55436.61255742051, 98717.32436549617, 16856.724627355692, 93133.85593975744, 81562.90889532382, 16772.378920277686, 23262.36931510399, 40963.984681086804, 94163.39552878299, 30745.0136838374, 25231.452982241597, 51162.49326333267, 9022.32857849028, 21720.596330270626, 31131.21030089768, 64278.37196343341, 64149.1847934762, 86265.33755083084, 57617.40142628573, 18221.62175242331, 62687.78446980108, 95086.6039634084, 70529.59720994261, 13139.600356431712, 94614.76737997185, 99972.83567147379, 46072.79601994956, 20214.708042454564, 41796.81094962927, 74396.20854904101, 4726.603208838931, 93918.93898328504, 31212.85062486322, 59153.04653485088, 48267.20349913973, 44516.59500456719, 6908.905744463878, 36211.94643667182, 66567.45856075156, 58529.83699528925, 77780.86873036453, 24957.79726118529, 81042.7793588572, 95868.01627247562, 90945.39621304422, 51112.28025808217, 9279.005895592041, 26488.48171828422, 42389.80306415946, 50960.3326749141, 73662.98219291466, 94855.1445684057, 97608.99837538318, 1943.0220439611512, 91311.67298080343, 64049.51636350915, 18791.562272747607, 10416.416973759513, 49353.92109271275, 44045.70363780622, 84641.4814016145, 39929.604010033305, 94870.56852816853, 99575.36441578742, 80188.77200246225, 67431.65965246734, 33988.47278502487, 96654.76934888714, 99980.33456943235, 89369.03139638869, 16564.015669839282, 79830.77919669416, 15007.823553564736, 10344.944969406755, 77131.48593600384, 22022.765694138114, 43327.02245558754, 78137.22137681341, 33079.04213268164, 10217.061255528004, 47927.98250659712, 18334.65647782525, 9001.554620503815, 43031.62628963776, 33829.6338683791, 30282.689276771314, 85279.97697855163, 90963.07498595606, 87178.41608215169, 38067.384055019545, 52416.03960343334, 80643.69721593044, 26070.360177993567, 70678.01739138312, 27985.469476734324, 52582.7423674691, 22419.3786747106, 68585.30382043298, 88075.91974357601, 49293.70099159828, 25408.19912330038, 51283.41803285189, 14191.261219198326, 27599.857122544745, 15976.975786798565, 57673.34714564535, 12254.099356479186, 24627.89310047133, 96357.96190601267, 84910.21120226191, 84898.11045185913, 30704.594297844935, 88014.81592091951, 3017.7369831816404, 98119.78500671366, 77218.98490958064, 48881.14801592178, 88911.99691673914, 74556.44311323432, 24497.958232491434, 69799.69431615583, 40455.798002434516, 70717.3311449575, 8539.948869568314]} +{"id": 773, "vector": [65324.12702779603, 78415.92418146956, 60279.73813454545, 830.9910713041435, 48281.21343187887, 80011.12848134509, 23855.480191527924, 34389.961233725655, 11168.796552626669, 44429.97163315959, 79771.18271011619, 58826.836606629004, 54850.234821118946, 87163.74737059575, 33879.49009514181, 86062.24708643658, 31640.57007169666, 26814.06492460795, 19794.371616747096, 49638.23677396279, 23953.580638600524, 99087.1591037838, 53619.9554536341, 36914.681086528944, 16897.698624325807, 79622.32664104362, 52788.78034910917, 42226.56612580347, 64644.7727122875, 15727.439468370707, 98622.52990181488, 12743.883688005919, 28598.276625836515, 79720.93330897509, 76593.33531483015, 82248.98159871275, 94990.11563331222, 24623.506454457078, 71103.30900065719, 3004.8647175115793, 62197.00161966671, 45531.01362824198, 22256.090085571966, 82951.96292552268, 97403.08109970947, 72690.07805434286, 44124.04024305415, 10406.89112119555, 898.3732087171403, 14549.848456201398, 6337.868591714513, 21694.94568510867, 76598.06742754692, 92683.40577853638, 13652.52207026405, 44677.97615964996, 51681.05936795942, 67528.55067410987, 73928.79908326788, 15618.793699203603, 67778.21774658597, 23255.735203040917, 46066.138207294985, 28895.91917519302, 88465.84614498245, 63583.86823655806, 70815.06311779865, 25974.874577148275, 49198.44726535605, 20399.17411909167, 36244.878061456795, 53428.59169008784, 35124.27771803076, 54448.37520882593, 73570.07803689154, 91119.82304581991, 83210.26062585901, 88137.29062691671, 65295.69380484874, 15279.433928022612, 88395.10804987702, 82479.3984732643, 3872.2571339223964, 82988.18886280537, 33110.4949601161, 94885.70325666442, 92891.21904498125, 3311.3521485716446, 82123.33189905908, 89841.09852346838, 75149.40774294198, 19101.292689302496, 45765.31541833952, 41172.99420151019, 46261.67172155146, 36343.86241228435, 69388.5965549148, 72676.54396612098, 25852.302213645205, 22467.658267260005, 92280.80933434202, 13504.828333045527, 90228.34823062722, 75994.7458081978, 12340.950194291112, 5944.607214740616, 67581.85063684975, 16840.92604953882, 71538.98380115909, 7667.86585514313, 64611.2031300484, 60329.59676543613, 47758.144391951355, 74535.54844079082, 4179.354601285723, 34322.45247482361, 9545.529019753973, 11790.668542632731, 76098.56680993496, 12517.93454722092, 31627.297381906705, 63026.2951317428, 22518.204138016572, 97507.57482840639, 72047.14086530948, 25125.48087695813, 15362.790319654174, 4882.874402574411]} +{"id": 451, "vector": [62753.031528732674, 14745.628646426367, 74395.27491580532, 52572.19304650974, 48430.8637329213, 7232.33913703889, 58245.91602912955, 75077.46941122852, 32654.824038671624, 20782.853638891473, 18796.291045048154, 1806.9723585849906, 82508.825246023, 73300.51043553925, 6779.486297442827, 76499.6543672372, 2107.915069720978, 99873.61042614035, 16854.877874445963, 91106.29165968674, 14182.566156200215, 73509.78733585893, 43462.20069242547, 16650.187073648725, 36920.176243065085, 6203.770780781959, 73196.06914429968, 90535.0785939966, 32775.459921294314, 59045.01320383602, 93001.66260496699, 10259.00110429253, 86593.03921089794, 33021.139459946244, 61741.277377323924, 47120.138812365396, 93675.21622254244, 93903.94349854672, 9504.609277230536, 7719.251901669666, 59207.08329589535, 36470.49201193114, 28836.433676430395, 78513.81374784464, 1688.7767730618841, 85415.75958471003, 8472.537693980054, 23002.428793880346, 88594.98231379647, 50730.767440850264, 42078.991771302164, 38177.05164395055, 8464.713318674521, 83834.80480398178, 30785.87775003968, 71083.28489563354, 90242.93872913758, 46361.39952375955, 13127.199423765935, 42893.26532381419, 82753.97088208853, 39328.231505187316, 83385.89970724269, 55000.54843454335, 13364.632114731123, 22720.247266382055, 78148.72153491495, 59427.79823985821, 48217.980326029894, 58038.5286868916, 71142.66168944749, 55758.23970326058, 66859.1608573545, 85188.53190550095, 31457.887540147545, 4487.2725383091065, 14234.551747929314, 69636.29061991, 94122.2441611828, 57039.000577277424, 91372.62711607988, 56793.58036069405, 38547.96548792256, 7166.148472825529, 22417.564140719558, 13727.629979173316, 41094.11287672583, 11556.98046671092, 68168.86231160325, 4841.450623618149, 57721.26909109855, 86791.78299188743, 97776.73777692854, 34741.555100142694, 19652.213356372617, 63856.07854320485, 26524.862503892145, 52099.284809765886, 40116.023187596606, 42779.61498769023, 62927.42416972792, 91512.46888472536, 16705.354924799452, 92949.32185482037, 45682.91735781661, 68183.60009933184, 29515.812047428113, 22442.21825890994, 59383.20664156887, 61867.847979670674, 72743.25484770807, 35117.332429106995, 17447.24994489936, 49793.26086514757, 44887.40106976745, 51863.45610079695, 29766.976456927972, 12135.598147107085, 87800.15908027052, 60526.66082585306, 67067.43935835698, 85814.79899090203, 35437.12901561621, 8981.367423373544, 7635.204947782137, 59966.61196239433, 93506.89371287779, 68695.02991016855]} +{"id": 430, "vector": [34053.94089643734, 70743.12819544427, 38378.216175991605, 49644.23736735728, 50266.19495149478, 99418.07396240714, 96779.67039615136, 66008.10357402093, 36756.102698110495, 9107.433573477052, 61723.53603395457, 62832.605691122466, 52269.69559771007, 91752.61428774413, 17574.366553601696, 36799.17724351168, 77593.59957867581, 74710.78241467934, 41719.40178555218, 26819.47792570327, 93417.7826082975, 55512.67431563746, 76366.93634270172, 36364.102828550924, 95047.48154478193, 46340.547425633515, 34384.610773356915, 25266.53998422862, 93207.06390027821, 2847.114274971374, 81747.80252002245, 87325.2994958018, 4446.500368069584, 76056.50521537376, 43612.857378182205, 97046.49677545148, 60760.21684458754, 8228.303717598306, 52074.939652214845, 31972.538624305012, 47086.858888576986, 2298.3075266407973, 42490.512974228026, 42483.2149286221, 63126.85733089229, 15266.571102567195, 10198.042608113878, 23527.232523742725, 45902.11121053189, 99898.29563815088, 87678.62666175811, 3357.49406954059, 65273.46783157395, 51060.61101376391, 14847.148305758628, 50778.43238205526, 78436.06746816714, 33687.47677001269, 54233.59307268503, 32247.79452380593, 51592.688329186654, 81456.6402834434, 20082.94373274585, 93545.59725210161, 15310.82829652628, 57128.37373252629, 98313.40450673796, 67107.48030647024, 51257.76958682072, 72272.27581840986, 22341.601944623013, 81279.87874477767, 56239.897958656926, 91335.93455105771, 34196.325832338945, 96188.53704163263, 12998.79344073368, 14325.467353405775, 91941.71551466905, 75324.08738779642, 15836.681321567692, 95907.14689000763, 60170.51066503942, 17205.791061807464, 68676.73403945033, 85082.708974595, 52755.99327372525, 95630.87474271264, 3447.1479370654756, 87643.00558456153, 1844.321561701312, 93528.01829283165, 14079.296156565002, 88817.24876642131, 68251.22506456527, 60261.78204854787, 9030.357279999756, 96707.79854681352, 79947.35969970442, 82407.69341147035, 32885.11697516585, 31179.984191587206, 66190.39492672967, 46488.91349473641, 6939.390264013578, 66339.95927592108, 55248.93038777202, 89026.27311467078, 11777.015461104245, 82746.26766001595, 50596.98617306218, 67123.28120944345, 82021.90055950175, 72606.95559061822, 69808.9224920335, 39836.53269769747, 71622.24223496245, 34227.98898550046, 74689.40685392667, 35583.23396272279, 88451.02919081927, 90665.38241225177, 41560.582479829, 10292.62994697625, 11070.29894362337, 63721.49611235258, 74934.09335055925, 93658.87261209813]} +{"id": 904, "vector": [2977.1713317407466, 59157.460816964405, 80956.18598530066, 70089.66732084128, 81076.64973541923, 74194.30031731508, 98702.92866440894, 31552.61317453215, 64853.509930010056, 71523.45031477764, 72972.11673484292, 55071.17064697823, 22138.10388269576, 97940.24631916967, 57448.604932298884, 71352.954151714, 15101.548349484929, 59601.28523695126, 93724.1331822196, 79899.37221642882, 44648.21785232943, 84697.11686387105, 58130.14751772647, 98206.1294388394, 54955.00332110609, 52117.91463000611, 9699.257449429888, 67232.66759261535, 2699.5710456792945, 22045.459341333408, 82206.63990344729, 4117.992504805601, 59845.715274536604, 36749.03007491127, 21506.51660066547, 51925.36219566112, 33864.522626856175, 65406.12885392711, 51701.33292970255, 66979.20052676047, 47511.11488916575, 28336.18552278886, 87929.06211452499, 63149.387020635906, 37413.622332204344, 76428.88618897721, 63883.943326215696, 51784.9846668653, 85840.92671187628, 19959.131251201612, 73182.50696198414, 15976.824727533613, 2768.72223814445, 60586.529298609385, 15306.731561066366, 63087.29425048332, 62055.45076916629, 53525.060743315, 76407.83046547667, 29027.876453935976, 86273.95619979087, 64342.53796278503, 5329.827769358675, 15780.475248922254, 16053.170235033209, 77045.31376443256, 9955.256385645473, 14953.186004269659, 4633.058454990591, 23384.25564654444, 85037.21960700338, 67278.4385830967, 15408.404828904377, 97330.20167109995, 15023.298633816696, 98609.9998031907, 44732.1178145685, 76620.77325899983, 89430.87716804886, 73778.71925475438, 70100.3698476771, 91315.8278660283, 73658.0471165638, 1751.2795819601745, 49621.17954486668, 52208.628956096756, 19157.611354561475, 68094.50013707358, 12078.923510470884, 16359.064242279765, 74583.23504029468, 25044.17927121393, 97852.98726759136, 63753.555897023914, 69072.18749348144, 54124.44000638275, 13369.332297957659, 38031.9446455929, 39821.6304734332, 8209.115365986974, 21209.52669140962, 58965.757728855715, 97779.98777254093, 48561.320494401014, 4642.252507739697, 46168.31447624757, 92365.99547286461, 41086.315595673, 4808.228551177973, 46585.05977856764, 39648.94228809441, 93216.84679078196, 48071.44982019352, 30757.268747351874, 43406.91894491425, 26388.394343726908, 5494.449731195738, 61703.21678270212, 82011.81709277055, 65633.83216599603, 10809.884689562765, 71159.81433991094, 26017.867750476198, 52307.822862926245, 99891.05068437515, 50080.50119535019, 22147.132439537865, 90106.32242987613]} +{"id": 588, "vector": [73250.50063686266, 82346.18079322555, 1374.6444656215328, 79296.22118804605, 29150.76887608533, 68281.86534560569, 40417.812565129716, 92954.03251998754, 79374.03089375704, 39734.53645923331, 65348.41197555829, 88980.51864470128, 92403.87424361212, 15805.988276967199, 24356.797941228924, 78759.62399196732, 78609.73864069169, 47924.86139704981, 59359.02689782706, 40286.028705862656, 47938.21872726452, 46435.77351137275, 10655.257866507705, 65329.66443338688, 34592.480214772426, 2420.8332548164567, 62536.17099663355, 9524.037841625155, 207.02503955319918, 81139.51170314323, 71668.77414791429, 67408.24503034944, 68397.93846855308, 12155.927494399943, 84142.34785619732, 50159.55326282152, 75677.38893489134, 64129.75889494688, 13251.802051966477, 12991.656557587716, 74074.6275765534, 67549.79305049745, 29778.59954744301, 42797.21807085525, 68397.92111822889, 46420.745518312135, 79001.06951384664, 41012.58588509611, 88117.58732183486, 27341.56020167029, 71163.55972746569, 26496.20044928114, 68932.69540377744, 3418.760446538394, 35643.87272291242, 73376.72195769634, 8898.955795103291, 89303.88894129875, 5133.758273910138, 33978.11442249061, 67690.68098190398, 15609.441431036154, 68483.24645056292, 96040.46471062183, 20791.470941264644, 58216.404222014804, 76999.78016080377, 52509.492187006465, 12950.385039724044, 18676.237550249563, 25805.40738982796, 92716.72434808924, 27602.49571806206, 75398.27608049232, 49720.858215300665, 81244.62836371863, 85315.6784370568, 68674.32069214461, 73960.49007686698, 13505.505250243732, 62901.27189191694, 73420.49045022775, 58273.87511842107, 10787.765258553984, 17154.81955330469, 20755.09273980315, 55023.813214798276, 78777.63355258043, 31375.493207428062, 62207.362670182876, 19493.011389101222, 72240.04249519693, 87793.48356266048, 22321.38660273362, 73251.0468143858, 1492.0407615120146, 69834.51554431123, 34651.7103792358, 38318.66312994573, 66291.39722776657, 28675.35904156875, 79792.86014602924, 63101.433140240384, 66157.66329021139, 77941.49392590241, 42479.78133273787, 56931.86733595501, 7068.0677614148535, 17768.856860069725, 69943.63988152245, 56614.826765400176, 38399.30770679586, 61358.587115628, 26699.14413924901, 24447.407345072093, 22216.957887673983, 98954.42052574037, 40768.4353934891, 91674.2676131594, 32478.827167946056, 51113.34036924566, 77875.64316642791, 91706.41457069982, 27912.16872535309, 75553.68007589548, 80113.15991102367, 52025.87968507868, 43452.8666688036]} +{"id": 55, "vector": [38159.25041728316, 85983.2312253954, 14711.997970645318, 18415.88762910895, 77862.6240204097, 75455.28640574173, 38384.8760529652, 7187.932175739853, 88270.26735102375, 33376.17233640856, 54931.65124115904, 56141.902089380965, 54133.63143402305, 42722.40988457253, 52862.854643743914, 23460.231492503714, 23916.48741558405, 54527.511650062384, 8725.924028249066, 30113.82399744077, 55261.687626221435, 44766.29850623108, 56714.44402804675, 44030.150800005955, 33102.513488763194, 38410.677946345204, 20082.18770546525, 73150.92317603089, 11358.759319482082, 98622.33056540648, 86037.98019767986, 92326.37528450483, 1528.4308230500087, 61689.189841297375, 89718.50582451398, 99556.59857238157, 77651.99982238656, 21337.27895565508, 69801.38755542481, 32978.711560143114, 27584.408388048487, 37974.083800162174, 23006.626871841418, 8011.029976162276, 17961.985977491844, 99797.65171508305, 23659.028669293526, 52440.47715422639, 33391.67034970994, 80506.43904356992, 44971.144871474986, 33469.240744446215, 76862.06500913476, 28590.59560770518, 33277.2798707125, 40790.92073290945, 99389.96007836166, 72369.26122646457, 49424.05864868467, 49472.37494659771, 45070.542971747076, 15999.87252908025, 26819.99726596078, 15898.887709639454, 27627.593091063663, 67384.12252137868, 63409.4191279962, 93469.64813744267, 47590.293292896866, 28647.793942984, 30897.75692689156, 14758.287844617978, 18155.971968855745, 33846.73238285978, 21976.43410147786, 96049.91897552708, 12059.19213473815, 96377.51472671542, 31290.198769992596, 92510.35287369088, 13602.438158986508, 72947.01299431699, 90912.32018457311, 74616.13777400415, 40724.17888797235, 23648.363362607717, 34088.650151074784, 41879.887813254165, 57386.258510363376, 25460.28156642346, 31176.874018841903, 47499.72516176289, 8178.304066172437, 91245.04737311721, 90103.60956918498, 37222.02477320441, 74157.61686234154, 75253.44711795388, 71970.98399403699, 63260.68051663375, 45576.140838505664, 26622.463927972607, 63114.43447664553, 77418.16770858828, 25049.202240866343, 18639.823551070058, 54108.80443248752, 87971.79328144454, 35617.30764580876, 54103.39075617009, 37352.74613311307, 36075.255993012586, 54643.79209641769, 33137.21174816412, 29881.243782521637, 59376.604619891026, 51382.492098882016, 20344.088336629575, 41832.6384792345, 39261.45873097228, 41087.97506669207, 68765.22189818314, 98148.98026015949, 6619.569984205043, 11213.866610066536, 73306.52007712875, 29554.149181230972, 71202.57658916747]} +{"id": 891, "vector": [71434.40829564533, 23023.93646116473, 72917.11876851004, 21219.694941936086, 72626.42709512392, 30193.310829170172, 76452.2207646882, 39936.605937077664, 83714.52254083056, 95288.68938060213, 14292.12793579161, 79260.5657527625, 55568.44738037893, 2233.6181082662997, 15067.729575729483, 47456.39574778262, 43654.58812040954, 81514.51156334978, 38404.02432137811, 49506.25022887712, 94486.35318978822, 90350.64015164752, 24428.805378829988, 80960.18844963211, 16945.74285024586, 37102.51593698648, 35088.57843948669, 73085.46912056356, 18509.743703290816, 61397.5319271533, 16238.527523081148, 79815.15933045105, 53628.66866082747, 57957.51666375402, 82915.31044561676, 64391.73213478054, 49688.301119144344, 41378.059717429904, 27189.524888860728, 61095.315531390625, 14572.482317020485, 29011.93027923823, 79936.88091142308, 81955.77493302175, 47420.84456477177, 94710.34029404011, 92517.76233038673, 84998.81143423374, 6268.194404368976, 88920.77297993172, 31778.377442131765, 70589.32726957451, 96155.5758186211, 2228.419889048927, 52458.61376306232, 87440.1305186956, 30365.46039335173, 43196.4950204409, 5247.379311195233, 9794.678232444265, 56306.48323180587, 97036.97898556532, 93843.05497047157, 27515.87458308371, 59164.687399123104, 99832.02546182676, 36822.56201802676, 93264.1715449165, 80281.87312125742, 89371.41549227659, 49352.24445771492, 35497.57230655302, 80721.60561256173, 40994.711960458895, 14459.756833548654, 66367.03937834318, 13531.498188265534, 89997.66550506908, 46248.7297438906, 60096.759976462636, 60786.80295934299, 14251.18978703771, 34639.45081514518, 56586.39693809146, 36028.62309091368, 20588.32533176157, 82047.51729172793, 71881.00269736636, 15520.72803529244, 65119.74383114383, 4848.85006917064, 15820.320253907028, 87118.350805366, 27982.077901812852, 39693.97192380624, 1168.2613305332711, 48492.81256483035, 54885.35612670993, 23570.266915430748, 36481.51393569186, 51286.5608387611, 19068.92920945873, 34446.16677918834, 75848.48510619448, 25504.904857865706, 78362.78816524606, 87798.40992870722, 1895.3102307523895, 9021.416017804251, 4128.83283427935, 43229.13776912103, 26472.940129230905, 86302.04604303472, 87379.57585391699, 22833.268084320836, 42709.98112215023, 9925.696135264661, 62389.49133031135, 86475.88828078532, 91141.21054931662, 42785.48008145863, 27913.803070790498, 54468.21338377622, 12874.380342818758, 22304.05410722973, 67242.4506016634, 46803.91460828402, 7387.535639455545]} +{"id": 1573, "vector": [57755.600407460275, 96740.72730115529, 85491.36158999316, 45878.86712307573, 64293.36408959444, 12106.223365764035, 32930.30689284064, 67303.24978992571, 8466.2748935473, 61741.02132749084, 27987.01156387192, 42606.23558845641, 67195.32843747825, 84106.66748583961, 75100.39936671649, 2981.339153345175, 16579.897796070265, 62933.38386723909, 16308.29673989851, 46571.18756716432, 71406.5720076727, 43535.1130207385, 53787.94776161494, 89285.50860304265, 35829.90368662401, 38833.88556413985, 77541.17050093395, 87532.1651797086, 67315.5225680342, 79216.72023106607, 51970.93937639585, 69325.38622171484, 68325.59082406186, 52228.09985266901, 25951.002038082148, 80355.51865268561, 63569.37911969104, 45287.74924373659, 89291.79181304896, 88884.5161464829, 3993.655416228814, 75448.42400058346, 97012.27602983013, 76669.17762863672, 21355.85122593945, 40657.172766870244, 8038.636464144566, 51595.14564928227, 52196.55382212768, 38733.97984963033, 58478.04757887876, 10268.012614863508, 4694.342652871952, 60123.41241477097, 30775.764088273638, 59486.48920291093, 47583.10246018287, 30937.86217478676, 23345.074151458844, 25432.502504997112, 85170.52409487551, 36349.428795592255, 64325.90039396891, 10860.95864504475, 20736.388490637604, 39322.154929503406, 27364.710832110395, 63618.501806332206, 53091.70259694982, 69833.66898543014, 90434.89405219568, 25940.715964065796, 11056.79398580618, 56971.3794213853, 73122.9820445214, 99373.41735471974, 53858.92776761195, 85769.50212779979, 55197.74857236578, 44657.88832099503, 47450.27900731632, 84597.92926220677, 11509.518008252606, 8188.7355757753185, 8336.303564636282, 174.49205546493653, 63647.39579501799, 68968.48220445977, 70523.27663484169, 89463.64780645246, 29693.119438872105, 13659.58759406104, 23530.41487392853, 40889.15986224601, 64768.87016825205, 71451.3247375141, 65594.57869349464, 79663.86457854442, 80821.35715320657, 94791.5751854567, 55075.77426018334, 24399.963118504842, 22666.366964717676, 88058.68895148575, 12798.203451097268, 56895.41038279047, 43041.383285437376, 41197.00170045797, 71264.29621293242, 18114.576960869745, 51725.21558057204, 85753.68795653783, 73406.86761787967, 54544.701226745696, 5383.191324790859, 2481.6719589821237, 47509.83184707468, 19238.388041602462, 70087.92497323894, 70650.5372403023, 25972.19702894692, 98825.1872754291, 16329.45537410303, 32628.665112052546, 71126.09788926049, 6802.040103312712, 35900.61914932029, 7874.6275477430845]} +{"id": 496, "vector": [57564.07909939245, 8923.424405547887, 68307.16408511487, 60144.80261177564, 40508.17931639494, 24359.669559716513, 12003.59787213715, 27058.744277592104, 31014.5781322273, 35024.75304841878, 9181.917476158429, 42572.63441480181, 47647.68932452543, 9210.464887572201, 74548.30398118298, 4556.50545514762, 58054.32506754085, 73203.006605647, 89074.31585734148, 70411.4161238209, 91614.11131663005, 99224.10405799556, 9138.11143548403, 91039.26580784968, 91375.88059108796, 29559.14134154697, 34587.11326516136, 97741.1640678308, 28364.62490046193, 26379.816829072523, 11178.185775755734, 63087.697602008455, 54730.12687965117, 50897.255955077926, 36895.94350883824, 5530.747047895046, 81150.92004239032, 99393.8165761763, 35876.50179943613, 94579.1642925411, 32328.058871583387, 46110.288102394814, 39567.30601314056, 33190.61518061529, 25087.57153786083, 23382.769955540418, 50256.65618997052, 70396.8871841216, 67525.25562143678, 7581.104041953823, 66670.88885787199, 16544.143302322278, 46310.69598667288, 46279.738610382585, 56458.81152622568, 19715.776215087742, 81806.12807159066, 17734.52362166039, 79282.67630888968, 92565.35582893758, 43986.91382954776, 20820.864617874267, 45038.120926484815, 49894.63086275767, 9070.140912438184, 77172.54885909667, 68786.83378412291, 43096.98021164702, 21850.61558698158, 46428.00099357113, 56950.367336762676, 6659.819712629478, 88039.24345176271, 86500.4279748947, 74391.37984409196, 22238.548425893325, 99142.69300334548, 58675.140297329395, 80485.15348953934, 47202.99131151382, 91476.17341094151, 75458.435671613, 7525.602699029388, 46009.262674208105, 37798.694771793365, 74674.47944813622, 95826.87941092569, 26297.22606820705, 71076.07242980189, 10488.90557364509, 28967.603615658878, 60808.59533130103, 39514.450316777096, 88220.30167007825, 10824.058855280184, 85854.73725243923, 73157.19083927808, 73370.10561446368, 92318.96687905845, 40025.16536030983, 59040.67971356273, 54780.335694378125, 46746.57449790548, 86298.42341737406, 17744.77821717103, 60965.409856346756, 35604.08733063033, 89094.3577365732, 92112.65574912073, 75667.05314600261, 22577.95629753553, 71700.58927969504, 86983.78236159935, 61313.21231617799, 32735.421403078835, 17035.73323321238, 43634.342380901806, 27190.877460392927, 93397.52820194626, 92217.87893667893, 75287.28115489041, 23409.6485082081, 68952.45254845789, 19619.07689670017, 52280.785273622954, 57709.360253848965, 29458.281572928667, 83323.84365284686]} +{"id": 825, "vector": [92379.34132782405, 78886.43095184071, 74786.67397222029, 19634.686938008595, 86323.04991730303, 36612.28556880933, 66330.06555926571, 24418.800229193483, 48524.83823815442, 89132.25288184582, 34603.91088536014, 14065.149895338214, 25848.374209864178, 65272.3131228515, 31531.46641265263, 39790.53180673834, 87249.5035971943, 79655.05889382832, 31074.02660412555, 15558.708927164733, 13121.771282504269, 23023.599893563285, 24455.6009776284, 65861.81768035408, 54770.48346381106, 47040.97832002051, 16844.74751965518, 88036.64262013904, 64008.040122507846, 59525.77284590123, 74832.94495225539, 77562.58176244398, 88162.18335493295, 73197.88644368039, 61880.149458080894, 69553.43840622951, 48861.08807141663, 3358.2365260984125, 81464.84943173784, 92913.63007547206, 32072.621273288893, 17707.85890351484, 25899.04853790923, 74167.16118580884, 41401.307939002094, 48978.08464924387, 67905.82741670428, 87995.2190469613, 52598.653296660894, 81822.33331349949, 24241.78405155185, 93943.70802202709, 5529.058569299284, 1179.1434170132397, 19345.06166354293, 72130.65654753898, 33972.68576155924, 18922.50977698794, 69701.68573897087, 60579.49719414363, 11723.425667292076, 76485.0396702764, 65550.820414792, 50278.0808704406, 37794.68901017391, 21740.866180163644, 28302.066605704924, 76251.18832836956, 1742.5359243523953, 52000.44645699917, 86025.10625226705, 46116.40805854113, 6836.823303164907, 8600.31005310522, 16966.690747133372, 89289.26858722832, 73243.82163949347, 3443.346534165581, 9560.980367377191, 38516.69376459593, 41832.45465584127, 19381.56432616862, 62089.14595348847, 26395.32235165787, 146.291491970052, 3651.7047453163864, 9990.913400202839, 1070.3695114349543, 79498.3152800745, 94296.86904434145, 34274.49545647459, 5415.736501843393, 12016.517440011576, 91075.26669149172, 47280.99769652292, 75681.05570971013, 35444.86654839111, 20768.672803331756, 92824.33415656972, 6890.826308785636, 88432.96331148625, 88279.33862759144, 92367.41275760013, 27435.06275450446, 14561.349791099221, 92012.38794702021, 86960.49241859975, 86106.86867184457, 94520.25859711118, 21406.73140464011, 76100.62166402306, 79301.94971108083, 84489.81505958617, 35707.783387811265, 8792.426428556455, 63214.04594265687, 94123.85909151225, 16723.095280803856, 21642.062616936397, 7141.013737048163, 61340.36630957815, 69880.9494788826, 37117.19333124872, 91817.26222996201, 46543.98212958292, 24952.739918689436, 90627.69319373202, 92255.4035666498]} +{"id": 889, "vector": [52068.53245940699, 31602.140401549204, 29624.651991224073, 88882.27450562658, 5384.034979756336, 70564.33334020503, 82737.1601512921, 51289.450109582416, 41461.14335092791, 32177.499526046027, 36665.396746089864, 22752.008595389707, 956.8104341803663, 89959.09037766104, 99645.97550615291, 85794.44676145322, 46652.39818825896, 19511.670089303945, 70815.66408344549, 99991.73158710242, 55570.727164636024, 72944.15749593009, 41631.62425862838, 49794.45401945041, 64897.402287402794, 91430.63905346078, 57942.33187845197, 54214.91960767292, 23619.012910991278, 53065.364432155606, 20020.85958181865, 97462.96967998294, 13733.306307624294, 60017.087854985526, 38294.140190633676, 86536.89587608395, 95277.79747573945, 28673.85904938917, 74777.00163326121, 50497.042275218206, 70709.30303620291, 10026.454185590894, 57507.31202639523, 40565.963361509435, 59366.39596576202, 30298.46103026408, 21257.84566475295, 31323.268836132036, 19197.24354171397, 76823.32107287449, 72671.17187525697, 88099.57839927834, 78508.1076249436, 20918.569913247444, 88309.36480197156, 87068.69025556327, 49023.64117517201, 41445.51434044656, 40894.55025366745, 50920.04132665485, 35508.24205176652, 72513.35176532973, 10148.383195496346, 12114.254027147208, 38978.31187785603, 44800.923022315284, 24597.161855027614, 16044.329379530764, 50711.91746902329, 92768.44164231616, 52936.56251678665, 25498.805065439967, 34280.55669825498, 16348.392936646484, 74594.88293301064, 17292.796168060344, 9532.472295588834, 35279.67315347941, 90327.93121948016, 97401.14751697105, 3928.9414674798295, 16488.371324756434, 75468.99661440683, 5752.0417327739715, 35132.00404848886, 57846.77495065519, 76709.12104338771, 33898.81842588447, 85778.52818055086, 7484.128974639548, 30633.327917443865, 69559.95025399607, 5283.248914958949, 75590.81918480848, 2585.680294584025, 2710.353496243101, 89442.21772011957, 372.9056234511474, 22410.538820071168, 96653.18659182823, 37683.978121533255, 68527.17210506978, 67494.95475276948, 84778.44393904883, 24075.838341413302, 89960.80463886328, 40556.47392169421, 79588.10277674392, 75913.84713018275, 20188.371267798542, 90572.78592254437, 14145.884596214431, 16148.117040772835, 73134.42910675873, 5786.875185345086, 11920.534574878406, 20165.008977191577, 4600.587066526873, 63498.80215952152, 13080.816935536443, 63020.44619486745, 60401.316542530905, 91970.25442929045, 74410.91980489779, 59545.55131649893, 62274.30103734537, 45763.88976827361, 31237.00322959706]} +{"id": 756, "vector": [89240.75843329295, 64247.5269951873, 62777.54118956064, 80107.59220490015, 44873.23163072191, 74212.49841744234, 53790.40841810758, 47711.755304806604, 84803.0763230268, 98419.78973488214, 48455.234032091976, 98184.98960853777, 45043.60425839278, 46610.46535521558, 35134.01360835632, 74751.63677798539, 35421.54563724101, 92763.69098024703, 42064.33954663842, 41899.6767283495, 13400.0153516444, 40690.14108563582, 1849.1023333740752, 69833.22113457703, 43048.64028956703, 17918.375901236115, 8097.889016802229, 42661.20696830542, 67135.91872019034, 91664.44316877307, 31273.110780378964, 27712.99717699195, 61497.78361139371, 78463.6877163771, 9006.229326418703, 25956.44049343897, 84653.81352261378, 56613.77326175066, 39091.46318253928, 65457.61831514039, 39176.82684941726, 30350.081835394714, 21246.017791403836, 94066.61442106233, 59589.30037854093, 16.661154508057496, 4299.268379613752, 72129.11665429396, 35353.55920300555, 94832.78507473168, 71017.17044074963, 23908.531880430208, 66438.85435233988, 89744.45681238848, 82648.06618347885, 17100.416162813082, 99133.71899311498, 94576.68765299914, 34195.28649564747, 92961.28093782997, 24855.617532811913, 88103.84260443793, 37238.02895642877, 15833.490843323783, 84705.34477183105, 45598.70565028524, 85891.13874758779, 53462.49966612584, 67338.4512927614, 67159.08920442741, 63019.80165278319, 66477.35250991078, 32530.17837334804, 25901.742550636576, 15847.529145971528, 26133.736626421778, 91705.63573417268, 35764.18780547679, 65175.68566663574, 38579.93429867402, 45086.2633562201, 64720.31291889627, 93122.35126592469, 45142.747011759035, 97487.79301272081, 41046.31045501779, 97570.66995784579, 66807.40397158581, 24778.386331336176, 81880.23974968493, 99166.2199067974, 11591.169057502348, 18860.901923354333, 13252.149366131694, 8538.246243539994, 31315.88957555387, 26056.339020433596, 66286.5368075025, 16878.852061823203, 68438.0580321407, 6143.431670740262, 65106.34827071998, 82986.6676218904, 22295.42391736542, 32765.640074551415, 30695.386767106214, 923.0582270340682, 55251.11157746514, 76226.39788720699, 51695.96593899371, 47456.12539654726, 38746.10267048119, 2148.3294210163704, 5838.687306972734, 94252.60797364218, 66798.6248906775, 46023.719518050806, 42177.11516928569, 61830.746177279296, 37790.13197118341, 38626.06112193084, 69795.986106401, 33912.072682874095, 92485.01283381389, 53014.61818112053, 36293.56647502806, 4547.528008128621, 92971.40374674313]} +{"id": 1858, "vector": [93449.6810560467, 35317.12488910913, 47243.687956765534, 96983.97061830237, 21637.561899714776, 5454.4960566680675, 56245.954886131934, 10485.814938410987, 47249.127074063224, 85509.62452003827, 12461.568948129508, 72623.6709401138, 14251.162324434075, 73203.79336519547, 40446.21761517463, 4484.278974766931, 87958.33075745871, 73034.49119094253, 52759.334552191525, 28774.59251862994, 50424.64251365276, 80214.34436284237, 31474.487593782498, 10783.600131433568, 64216.96736400668, 53355.12980029278, 14354.426628164285, 97307.12903102425, 32955.3799427842, 25070.84756030061, 23007.757358323444, 10409.710271141548, 7291.168681743632, 75198.94586516249, 80823.31816841307, 81497.76853314281, 23209.287816426437, 19642.844532605042, 16431.578670767376, 26419.392448104627, 13591.773856624079, 99480.91843217639, 67089.26953732966, 14655.711625494516, 76575.49006318506, 91020.092489053, 54156.52739681012, 89572.44098563347, 95683.73575577514, 19721.692963605365, 66376.279435733, 96868.33948734, 61392.85835829793, 21789.51947471951, 70033.84475254407, 40101.244593277304, 30870.02797980929, 96896.6202340675, 91425.77881943353, 9616.163135602317, 99049.08157434383, 7713.365595412803, 22095.99624960257, 76666.40387088865, 95864.37348361028, 48457.175119784704, 74231.13346766334, 35017.63637161873, 75867.48653006315, 37723.17468517688, 663.0904487144318, 42324.50323084752, 86501.37371924758, 84359.87356372914, 76971.11641465404, 36129.26722086669, 50576.33877097555, 48922.300091296725, 30039.454437002212, 92172.98005134951, 45577.751386353906, 9309.989813589658, 51629.097899826906, 15498.096497613067, 88113.132334337, 5375.751157992026, 18302.0998383031, 50928.83112020104, 19909.528575623237, 78863.76661010171, 89033.37208905588, 99948.02331768918, 6320.672913578973, 78445.46384172102, 66793.58474415146, 48290.76728589554, 14332.884186700889, 96353.58323726607, 5075.7582690827085, 85587.33202819471, 52899.18423983926, 66775.96522020768, 30917.917034594742, 38879.92421180842, 61485.283158069535, 21609.996915158612, 77340.53137653557, 75185.58460315289, 56460.351779815595, 43035.0522070552, 67024.56737902744, 33837.92242003625, 18977.649939266237, 86835.67080912812, 3365.8468107157446, 10924.312895143918, 57995.31488061054, 41612.833586632805, 38215.79672354103, 63387.36812871641, 62680.28955645153, 83814.88863504629, 20683.807625603946, 96281.63627504489, 37718.44409873441, 20999.035049156002, 48435.06633059352, 68539.35586407458]} +{"id": 1217, "vector": [82198.6244056402, 30185.722011690097, 92332.6897543378, 56905.31935976379, 39742.65389669068, 12156.648423646366, 34801.89894955827, 42924.99642757053, 90000.57075295752, 43475.71672866958, 77989.87142400962, 77113.11275928111, 21703.17541249127, 72301.97886634401, 15079.908943012144, 2128.0584459360207, 66195.78874372723, 41711.984620352116, 51542.943005513174, 54746.22398659038, 142.04354559601563, 56316.943591697025, 65379.35270457601, 50346.59371381768, 26226.00564447223, 38975.859004595615, 44323.10896226693, 18788.75979357171, 42431.25373115027, 35732.878202981134, 79172.62896065513, 40057.68020578254, 50753.287331588435, 52571.82217234192, 34245.911555010454, 85943.08075052108, 33985.36530980843, 92580.75998937317, 28539.189859458937, 63698.11063946244, 16346.019739581674, 92931.76037606905, 30160.630093066353, 62646.166865045896, 40022.9780865484, 15456.474441475466, 64690.91385460382, 38133.40277409384, 43713.39605647787, 95612.37827463685, 11836.096464837865, 18841.0298956073, 61874.064657686424, 63453.80755193584, 15314.351572853391, 8981.916869646522, 84084.81434378061, 50104.8514100627, 15802.470982673743, 70859.12329167941, 10694.7214888091, 47051.01894481912, 55504.908193124604, 65562.36598141266, 98724.03696300088, 94616.98661483274, 14436.761155483335, 110.06890634163157, 9786.703041059174, 99714.72051103057, 9892.131163978924, 42934.24924335605, 92664.74776318522, 88218.25118452632, 84996.95085235764, 53342.64492550437, 2139.0623358403027, 4789.400940129729, 78402.54497682408, 20847.502420153873, 64016.85859161999, 90134.75922603285, 47241.62320887618, 78621.56046969196, 60878.2812592222, 53187.698352576095, 53145.23309379434, 39810.607231537484, 14768.4996605706, 91225.3964203242, 84327.16408402451, 57804.3040181095, 26193.68762974512, 32815.04100437598, 4247.538290859609, 52927.58303536429, 93023.91208123593, 58279.69002762764, 21295.426985753853, 26549.267853627967, 39983.02991053306, 28828.981500035778, 57678.65538465874, 66010.08051541794, 21529.83121552502, 64549.869205616305, 1757.6035664562228, 51325.468027187984, 52337.696081126116, 88122.19185078019, 89341.56089973185, 84243.7059626731, 64284.25201082287, 97043.1846146482, 4156.697602711468, 12301.732393950748, 41600.597343881105, 79388.54013223364, 59804.25280295729, 49262.47883356842, 90568.13595964687, 23977.29285204714, 74280.16094020211, 78932.70970437495, 88474.96891876297, 27860.750835894054, 63112.18676363647, 77620.17858731557]} +{"id": 320, "vector": [47255.84716759731, 50958.31597607002, 98278.89642636488, 6914.620378903324, 27729.993334049785, 50163.023315850405, 65508.96486797886, 67145.34136794659, 15613.74410834011, 3831.9741631269276, 97562.23863016083, 67840.01383268832, 56688.98965916175, 41268.268330469946, 14118.265398994201, 52818.88507464526, 83353.05199198767, 78636.10280406, 15311.267291183773, 91239.0992946151, 98066.76751240373, 10314.35797658734, 69973.18060895018, 16467.435736382362, 50765.62754237862, 67515.1359327428, 95942.20494339657, 54909.82021571475, 16745.44166891876, 93749.55927070249, 12577.341383099738, 30400.338328384867, 81913.66762208032, 31915.319430357446, 54358.886100562086, 49743.580976902456, 80566.7171937656, 39510.11487611573, 42957.077237780984, 13556.257240606428, 84904.6837120294, 50895.31640368276, 30608.78106731132, 47603.47087835548, 30140.160168467235, 89197.97025457046, 58414.55291444707, 71198.23576797366, 27566.443633898452, 86443.80885088236, 40912.55267549342, 41437.807478287104, 31968.16355747385, 88392.24094561815, 61295.50775662206, 5133.132237151383, 2370.1693169215, 85797.95884679272, 36142.57379370973, 52121.40271778498, 10949.607055093591, 24555.428425748836, 91821.20813596218, 67149.32140906282, 20778.749597188096, 99292.60343223874, 32925.21487568634, 1552.6707368870962, 60448.46190643268, 13789.407024496692, 60234.09326550361, 15910.719922365179, 58287.98327572014, 82956.20944271002, 34234.420404700104, 30913.173520614546, 7426.240359074132, 33127.72142660473, 51215.48618554168, 12044.864540669032, 7338.383758231426, 50274.58364780485, 85230.78152031556, 15204.508001092909, 27748.78738095394, 60019.86305579052, 63781.466412954826, 89130.64058298405, 47633.963215879405, 99915.53650098984, 33987.14497332484, 17036.642694265804, 14907.546082041223, 56388.79022584189, 63800.63014598888, 28652.085097787894, 74509.88324668186, 92273.10579785856, 65274.16027758817, 89546.44867273283, 62049.14598689454, 19917.784465817145, 81565.14222050676, 83334.85622684742, 74719.30713474382, 43140.47839004751, 24894.753494215372, 86055.78727135365, 43342.88260602118, 55920.077336136186, 73105.16729499993, 99898.80123131759, 33316.21145324862, 68131.68689014982, 87293.86348460434, 83891.02554684904, 29077.71616684022, 46810.48274755044, 54264.658025973346, 74341.20571254188, 14071.775012102373, 44717.321574665555, 67883.48803097551, 97877.13383142401, 45162.325178962856, 28894.30140030457, 26504.86392207847, 40072.50615161386]} +{"id": 439, "vector": [2621.7827928711636, 76291.96051859592, 61394.37055351169, 7921.411997281047, 43048.68035706499, 98065.80458566609, 98175.765349504, 96333.07197788447, 78889.17275233181, 7612.515583649182, 57887.872156425816, 39644.87258666955, 53371.410925932636, 63962.22923186844, 98867.89967064165, 59853.835626365806, 73315.09532563391, 90107.83854399752, 36971.55186235845, 99835.9247585256, 29146.58085951354, 60721.86398461826, 56123.30276751151, 54480.53865793925, 15427.179566056926, 74820.2565461301, 34451.91108457916, 79439.71745728016, 68202.91619566301, 67938.31665989234, 75843.66797326642, 90591.6181645202, 30505.65099836512, 21581.13099214537, 83943.36541878547, 85036.91863539917, 38640.356983635706, 87333.24712217438, 96262.00528217033, 22440.103524339327, 83458.16975736634, 68265.13970715074, 43526.17094055409, 26585.912057049245, 51332.34941790849, 96018.97752249544, 27336.21758900341, 25635.867823662116, 71920.16617052826, 36429.19813885127, 32977.614435871124, 70564.61960577582, 19438.83753407786, 54520.44436638138, 84879.38489354124, 97684.39891548804, 8812.689619667657, 76488.3992916194, 44376.124483854015, 12553.295981337054, 75090.1537057303, 94230.60242142667, 62700.4160318383, 33551.98356941309, 76553.28989686932, 93774.17508484276, 87426.18659082729, 72322.4548942641, 22532.058478170493, 95086.01643877097, 49869.76455997737, 67535.7935293932, 16850.253569397235, 15185.709617115928, 95893.84757666604, 19526.35208677477, 31764.383002579933, 92092.1577509891, 48614.308704756215, 9763.35811731791, 68480.34952854739, 97357.34114662597, 78479.39173398624, 73661.6316439046, 69255.499274073, 69985.70990086961, 22933.733505901742, 58523.5045369765, 50124.30272968579, 5291.940715813926, 60191.596233995195, 35758.52038254488, 81251.2442583754, 46599.950144550174, 88810.93315631284, 53887.64483410721, 21298.025666876452, 21189.524868730536, 19373.103699127703, 85458.6180385312, 33099.729375707335, 79205.22102973766, 41584.24240915246, 88157.83229103619, 35190.343218253074, 56272.56463749833, 93642.77801372358, 39732.69458411897, 13896.320437295972, 44845.81351084117, 62824.82235920791, 17181.619064256414, 10183.408606247202, 34319.155841237305, 74680.95169936224, 20885.288403885148, 53612.59620107717, 35009.97635331734, 13062.029019419153, 33094.946774710574, 98130.28583663734, 63461.24647166213, 12334.784126242826, 23174.372346735217, 32125.34358142115, 9025.216111185619, 91016.33207594436, 14560.12807337641]} +{"id": 1855, "vector": [40695.82995097486, 28738.351773406546, 76505.8131667328, 6790.874099903632, 15714.869357205818, 23065.881025769108, 66038.35995714819, 36862.20146441659, 48850.974136335004, 8350.815799669708, 59461.98008677815, 77279.63213692578, 72367.23218164477, 18599.048161208688, 18639.549253966838, 43381.4011506168, 40413.03566656503, 24506.604495947427, 68732.4530858657, 1315.309620698446, 3928.9708540961365, 15229.191242207851, 66186.33075126518, 20073.063768075895, 89473.16897092237, 68630.93280166775, 75893.36546330673, 71268.29717885786, 99662.63762253064, 25612.36275779387, 58960.57174783992, 74918.10304616575, 67684.02024337587, 441.94440555281875, 88442.03611767641, 69201.15605931169, 1602.0716263278168, 17869.689769085362, 81841.82241456282, 42930.045936795555, 5730.417542856636, 92757.94958001038, 44478.04645106097, 35920.61610045189, 83070.84791365651, 60429.60189302397, 91040.43414985396, 19846.13424390277, 95592.98649729553, 32323.624236707783, 19598.68389956365, 74222.21706368521, 4630.663747232611, 30286.06951291508, 63066.45778243171, 1324.4849850379214, 91079.53344896957, 41797.3953630857, 67021.80953647177, 36788.495334917556, 35860.862294093255, 69517.67765831717, 77322.75069937114, 28272.104849616397, 8008.969208426509, 36653.33719205882, 90.95728951350468, 62930.234661329785, 4212.850204453466, 58063.80361858372, 46652.9829959898, 73062.37836288355, 13364.886608173898, 27729.514441772506, 51056.43978362437, 31363.765815275292, 94767.46165815517, 75685.22959326977, 95153.13237416295, 38145.05486704933, 3500.456039672195, 64606.93983467675, 46718.57117748204, 93681.6142275268, 48999.24751874429, 44248.234870370914, 76688.9634269328, 68691.88585607137, 68221.12558135157, 7074.995382955285, 97050.86916350656, 36418.071135696795, 53959.40282174576, 14428.468754485513, 71320.64601933055, 74877.22963895276, 38724.43935427926, 83410.4317414703, 86306.18673850516, 75801.51838696652, 46373.88394098025, 91310.79898327238, 60468.03130815432, 6113.205902667085, 440.36419087349896, 77060.02874079009, 30891.971839278976, 34842.10156495387, 67293.62389498141, 98639.26613184247, 49205.027943349145, 23180.91893611611, 18701.89233860201, 18876.421044463354, 31342.17101967741, 61603.70497079599, 82529.34091450665, 4114.129137371902, 86614.38345694657, 66949.71272179305, 57574.9465571748, 15461.19935319743, 3685.802030552576, 14471.31155343201, 59153.44801018777, 21656.06008950679, 91874.12916328305, 73491.2794898042]} +{"id": 263, "vector": [11943.623414653115, 10681.173924281606, 67474.74220886445, 60049.07460725485, 1162.9162058551135, 75893.90671892822, 9401.230431010754, 59976.89962885844, 31856.516515841628, 79834.54562879962, 4057.701033602601, 64417.03512633476, 49324.175159174345, 1923.0107160274733, 26227.257271676597, 56302.1447944739, 82249.00122709095, 11829.019350148285, 32534.766615858945, 4699.3165833749235, 19457.174576034187, 95891.02156429025, 79175.37797627934, 44958.59582546092, 64389.68386428114, 24941.759228975458, 50855.66160750377, 21586.40657703853, 44380.4709941995, 28224.22786968206, 72219.14924362257, 36119.80499229658, 25541.542489863823, 7995.450305893548, 96857.38882658724, 31628.949425677278, 22852.84538324752, 40538.194189487534, 40110.65737501537, 4840.463829845298, 77671.89264444407, 23473.457802262878, 2370.4330859834254, 73518.3159045255, 65344.95092019367, 94685.91580761435, 39870.83865442271, 31986.59516534139, 65384.284875166435, 78332.43394837837, 75122.01486129856, 75228.48172965623, 32069.618240162923, 70542.80236784677, 52773.033524353144, 59137.54680468506, 73043.89137562092, 43237.99578164902, 90262.66583707544, 15604.596131138149, 67705.11037402804, 91687.90853462052, 42173.29349894041, 76996.8488893618, 85544.5898962656, 78246.33279145412, 42435.59923277083, 62315.138444410404, 17500.088679520464, 93286.36305339269, 18785.286150388114, 42620.12161606765, 1636.6673963485234, 63618.812791193304, 4456.360331141529, 71448.71379157089, 94390.67517171768, 22764.89169748156, 81398.26486524724, 11948.558706450196, 54106.94319749477, 87316.46433959386, 12238.342939394519, 57961.17987405603, 32524.488570570644, 71620.89148861654, 36700.35663091405, 68293.68908900225, 62843.130582567916, 95794.07699375374, 49176.29626235816, 30847.37236215649, 86421.84360176565, 56871.8625360602, 71798.62500875173, 11207.093349885066, 99280.25600937431, 97467.60786542627, 92462.20019101053, 82501.83387033963, 36391.26896741701, 32086.872309133498, 66406.62666402035, 53531.02852604783, 47129.89301495089, 1631.2959078009003, 68710.26180710313, 73661.45855018722, 8477.31464206527, 78519.9171776282, 30937.91148695859, 18466.752359325368, 96295.66839021328, 13615.42154501536, 64776.418076722606, 87564.51351777071, 73611.71360803231, 67510.57856273829, 74594.79363571013, 30076.38342039496, 77204.81827945904, 38063.42283191697, 49941.4843121349, 91222.57409871943, 11644.611630860769, 35940.44536296369, 81388.60950618071, 11896.63243684056]} +{"id": 1405, "vector": [32478.892286259976, 78175.6535200114, 65688.32974026947, 54313.58607716902, 22.152606565550848, 80342.53016078995, 64358.84192860417, 718.6297835381828, 35550.38225028616, 70983.773893167, 6396.733944571453, 42429.74851716242, 32391.010325087176, 23840.471310119672, 59215.29490048609, 45419.862186598184, 83459.8161664843, 63870.6427819155, 61842.00824135073, 29842.811739762143, 70168.78201400455, 73770.24981262535, 76957.50069367295, 74316.48533553295, 16878.722052885918, 35451.22289004222, 88411.83833861079, 57096.856926606124, 2686.6539832810176, 54111.21782294614, 350.5404903951681, 34280.13515082813, 12379.553309889712, 57219.03589870912, 20180.923965089758, 26428.11109434262, 59109.910901925345, 70476.52209742571, 60752.449897066974, 55047.178812179096, 34664.529059370456, 85755.71448254997, 81750.44296365116, 52431.55377889838, 84956.65387495318, 66657.18810869672, 42884.795509078955, 70073.51540643073, 82114.49649961964, 26030.572360974493, 53606.877877909654, 24640.205073007226, 57855.227577464786, 16905.725883102674, 76696.36722048889, 90890.74018576581, 43786.869798735825, 79008.28050096284, 76628.61337532166, 69222.71398064661, 48488.61714975127, 66304.98837565254, 66263.80165936441, 37346.15672098684, 28867.168136977696, 1000.6490613907571, 18833.146644408836, 37036.34641339032, 86327.91522424704, 55447.939170961916, 33473.86869543678, 83656.39414457764, 22788.709916376083, 98062.37661649911, 64091.206113251734, 47750.654597811896, 81045.04101163654, 77454.51665623211, 55297.95890706268, 63924.075511749325, 72915.57088327977, 89633.49849325973, 976.6687203314417, 29234.143344521213, 60748.12133935711, 50474.27464811606, 41557.20415820556, 31637.41353854659, 84411.00136214646, 64176.93830075852, 58918.186113775875, 65290.070368905384, 88735.42400790492, 81655.55894552408, 25052.280461343646, 48101.79624018145, 30396.6769526415, 79481.39635207472, 71525.84060457014, 48981.09974208879, 94196.68678161933, 254.157062547411, 71868.76664701084, 4573.9091755576, 39339.951403076666, 1594.3818061181391, 93663.26100424686, 84127.70149733635, 47965.24875009398, 69596.69732767381, 45970.52454534291, 69796.03227645151, 95657.1606630525, 95781.17463452964, 36956.22741570362, 55211.58495108274, 30633.856838672957, 92493.4230558624, 35831.38389299946, 4796.335001390206, 88462.73521930576, 16563.18931826899, 6036.201808803199, 57652.85385486357, 94559.20922775369, 9092.856334801147, 83892.26768249011, 225.3414894795913]} +{"id": 1096, "vector": [43788.02929963664, 69306.36259672107, 65375.15244901011, 24034.838298156104, 14061.866856689054, 64166.27705418446, 77996.27124443889, 15356.824893328157, 80051.61188693081, 26790.024173217396, 69755.90090944138, 17473.523615766262, 68506.25534645127, 89461.0789391565, 85586.48517768933, 1046.7095357400535, 42997.618155844895, 11919.70915606082, 66365.40888449384, 35677.475271383744, 43453.38986357953, 34312.14941058094, 62767.39281045117, 54402.62391961483, 6475.68654314985, 15132.18906004027, 88959.08537852383, 4663.267142058958, 11545.296408114958, 45694.455168111715, 75832.06017219191, 85772.12786695352, 47060.82459391013, 53439.50818770109, 75065.33409987763, 38533.53389509362, 31757.889164073506, 41559.48938365241, 35858.369099235664, 69613.95459261232, 72296.17960262914, 99628.20135902638, 18945.790788120397, 69714.49464129176, 29584.74392160939, 42272.87791301106, 18231.96221549821, 97143.07903042149, 96505.06350274297, 84350.28067762617, 43241.74244169619, 42200.2512990364, 27261.53315976011, 94452.49761258511, 83133.57933161178, 40152.03682961056, 61579.470620430664, 84803.22237236545, 84606.74581738042, 50884.895248512395, 66331.76287929367, 20779.073726076058, 55152.040098930775, 92312.13807713229, 15559.838105481094, 597.6740146693804, 62227.653325124134, 95750.4354681018, 39294.96936258312, 94117.96536317823, 16743.07527199732, 95410.24455612169, 28196.007064142203, 10620.087280932821, 84499.00822295665, 78169.37153152119, 93046.5769975171, 48564.25338658232, 12338.247062507224, 10297.82184358352, 52121.02755685396, 58587.81168384947, 81891.09762708907, 82197.13782699639, 27149.706336687108, 15027.279478255417, 22104.25496347086, 11723.848121710445, 22674.062040511635, 82804.7770619609, 88281.43983322084, 83975.07804848065, 7983.853283176645, 74472.08681453686, 31505.12190351349, 77779.50644923883, 41237.39126651839, 23293.01682295606, 32786.138018664344, 7592.4305453639045, 60230.72750803301, 49077.28560147555, 10636.231174773902, 48644.29238765412, 64572.78229820421, 67296.53166218214, 91845.31036146522, 60871.56675305273, 30421.24133027623, 90702.16330296792, 59524.28085870432, 88675.93960095428, 78271.54389176014, 63026.85884389635, 40910.10626780553, 79558.8421585935, 82360.50019349849, 51443.16179254385, 20278.861857669217, 74196.44039659046, 69240.15400230684, 47348.73388755004, 91727.03531467507, 81759.75833217616, 27980.814382588458, 316.91377731438706, 66400.69950985827, 63619.610746521226]} +{"id": 871, "vector": [46595.76218506289, 76923.28711767537, 69472.76200360355, 58462.16180441692, 52566.81305351469, 44193.23971785694, 62194.99401815304, 61023.01938234176, 4854.213932941964, 94676.81222932426, 89722.93807104357, 70541.10503766901, 64905.5953327349, 85678.12243368059, 45419.939678434304, 1287.650578519961, 82921.08390706549, 50338.42336303079, 25297.867471926736, 62609.567127830625, 19645.854426403297, 32615.541679099246, 66320.52175046613, 83535.8921976929, 2592.9255197791877, 17554.869514698425, 54592.10185153369, 2287.602915501141, 12085.124429668893, 152.4749584962626, 53177.568575156445, 35972.57206586806, 3500.8759091540887, 52870.03628305175, 27131.99428337716, 74881.81454695151, 74155.05509692195, 27679.919284821197, 35997.37750011971, 58654.067631799175, 93352.30055992641, 1856.6128674760485, 21380.735953184827, 1042.9676315497293, 57183.58964944741, 97249.24504455585, 62447.46872132981, 56899.330241478376, 4691.76555008316, 68914.29883642965, 21412.65369779959, 41585.43923657419, 21837.085947475964, 88132.51547312348, 18604.593481624554, 3027.655516795935, 40058.106278904146, 62433.8879801115, 80359.24647066445, 5164.915730234465, 2285.660515738752, 35446.72991919184, 83989.93329701923, 73651.80871329475, 27768.950263097937, 99649.69014389004, 29733.782612900406, 76598.58272516463, 40181.17986116126, 90764.387017674, 6072.11369893862, 82672.07298623609, 52442.38281740339, 85757.75090918256, 41648.34632922007, 21435.097210905707, 65400.93623360326, 57911.51670742375, 55281.202977393215, 59195.42152124999, 43717.841066504814, 72724.31294693494, 63448.76971771438, 96798.57849756401, 79899.27484878435, 69707.01172185483, 27633.060228266004, 21093.687305273324, 75082.08814622718, 68987.10101042919, 5993.4144121586705, 30073.581444943342, 93696.4507546369, 46772.016459990366, 12303.82296247341, 81769.02616733174, 17651.804421066852, 38748.80101345995, 68242.43403197602, 6437.340238686417, 85968.06370875702, 78357.3950807296, 36591.65568168561, 23529.25181810851, 2326.651487503706, 19114.910388651253, 17559.881228387807, 59724.67643684331, 67051.06913976479, 97753.09371886202, 1304.472956429037, 63195.97096753845, 14573.901416638058, 31642.798649120683, 96798.4862379626, 20990.092249008852, 92163.64703182383, 55452.47549295115, 93296.81004999223, 50320.09918730014, 60321.51503815841, 42864.01327556289, 96757.07279942915, 94017.00740449579, 97211.73477554056, 47405.94240704531, 63651.25454346664, 19934.995015905733]} +{"id": 688, "vector": [75787.35458973619, 75046.39776355997, 997.9132231258126, 26713.87174724281, 16155.275256784018, 17155.20410231942, 10453.931442603958, 41846.26522465141, 944.1579372020615, 95067.38440798045, 53835.57809868821, 19808.645189768824, 45213.60814097164, 28365.764359238576, 60630.5040410319, 98833.15716961448, 23316.377298653733, 51033.4746893696, 32912.64780662102, 75016.60181900016, 39499.573928936006, 56084.30585295967, 65202.88881393511, 84715.14235104843, 39890.97095958787, 4535.385654736812, 52839.361545204374, 56572.32381171459, 32376.945920567836, 13224.423071669777, 42647.67445308375, 35827.17984541014, 30909.703450637062, 93510.69675155434, 36058.516317482834, 47462.273277701395, 63822.267808177225, 23171.964154541758, 57349.920072259076, 93091.33088438422, 65817.91937831898, 76361.2109827239, 25212.507822633277, 14287.826562912942, 85784.60194431516, 55742.317614394444, 90998.92669932032, 63060.7093077413, 85155.75958588025, 30414.319858548522, 68523.89849902196, 81643.97504400129, 61547.84399724922, 23291.130149444507, 64256.595362184285, 68322.92665345423, 84030.19634834831, 9005.981725182222, 35845.39696154191, 91971.84308723973, 63171.67395494395, 47612.08529623128, 93816.02170393338, 25971.6919294843, 46951.443380699646, 26840.55313513529, 96259.47454288368, 53957.252794210486, 72545.27181674956, 40448.85356954952, 43873.18832891699, 62270.79029901994, 79902.42702711842, 79941.09464333588, 54903.176473260784, 73463.6201575564, 28251.987278922596, 58202.1701200684, 10403.57581587661, 66724.68598458297, 80312.17924340395, 22960.049293542626, 86991.85091658971, 63632.45476804485, 39715.18584106688, 89174.95691718324, 26559.573354473563, 29163.544890474514, 49267.93413751821, 53741.881734040035, 38248.94521567039, 77189.78898414898, 81967.27419923194, 99024.46755570211, 87310.65282116817, 52136.13482857329, 43187.370847605365, 1328.196696843309, 88031.89552223508, 77114.20186908844, 39469.93054868786, 44270.57959685132, 47221.69677513963, 32206.770968954213, 94021.82419673038, 80305.34362855549, 60845.42255385573, 78907.04267076636, 72.29172534073402, 4051.3532633340456, 24163.65059016129, 77217.65871024855, 84947.34209964814, 17208.681654996082, 42121.11100500053, 70820.56807992191, 82607.18351398897, 65653.38815994613, 57730.17646155737, 1868.210008666682, 59681.339600669606, 10911.497594347697, 10640.226483578452, 23795.479374251714, 46997.86299330072, 68511.80258645627, 25473.02165269786, 95803.21228154427]} +{"id": 246, "vector": [79980.09485734979, 32704.83558038255, 51762.68861805764, 52661.356687646185, 73773.53627246915, 9015.53666659618, 89647.8554884493, 2334.125152530153, 71689.01930427978, 72173.21641656997, 12128.16317265012, 19028.0890204347, 27594.684565928095, 88293.10449088702, 70174.66360013625, 9569.57775687841, 80938.61889007085, 20839.22542414489, 31891.61192064498, 98374.95220980284, 17335.608405236337, 32930.496922757804, 14665.095815062068, 17133.53170220391, 98035.68613094422, 22077.504995929554, 10114.78893853046, 84388.86044191774, 71024.23972273066, 96725.64536774675, 43865.33708397998, 6091.927334162006, 44153.06626271186, 55077.8663054138, 97477.47865879856, 7309.713742578261, 48049.42768900956, 5992.519837667076, 17539.724086340048, 58415.6138040892, 49576.85804199203, 10144.436312458427, 13214.041521566443, 10856.636568950395, 45327.35508369399, 39199.910598324204, 966.7554383285037, 37652.17441060966, 89795.31918121921, 73860.05751181104, 72545.9606801863, 51633.42177858525, 18759.44342095599, 34126.52645021575, 53197.14249592613, 74323.18270256331, 44385.913844128336, 13030.815962887244, 38546.08082562183, 79768.64649060127, 56261.74136737602, 37940.69474666061, 53410.013062481965, 42799.66019591551, 89146.84573616527, 81595.0986687062, 92743.15585730546, 62496.9210077699, 25265.39358251989, 72115.26717735917, 58447.699785960605, 35793.81223708309, 59389.486161577974, 39615.58976286717, 82539.90568080817, 18333.215949655358, 76534.89818713932, 95604.42054866519, 96433.39869422268, 50019.07353747499, 37886.47855178432, 79672.0040335211, 96593.65243386134, 35779.56028241891, 81246.98760558227, 62682.78193217922, 80192.07183741507, 21979.406689120817, 88639.25845671383, 62214.284921359606, 38348.07863423407, 76361.66688630127, 6187.664456918574, 75563.37137435938, 50874.60573248592, 70193.35380101275, 94510.41780573643, 96644.60347505202, 8672.413868714479, 78680.55288965612, 5112.452800006395, 69722.03056320791, 8276.059917548639, 38785.79729792688, 79526.04512583307, 24599.76011492413, 42132.117096736976, 5807.241051472911, 51933.903414068016, 81315.32315454393, 70588.83217322925, 94821.9395942024, 60278.546114361096, 46469.985292054116, 50522.87298553887, 26518.45170218703, 46680.2687483336, 40643.989117908066, 79184.14671106247, 26854.142674887582, 89628.75225370955, 94894.56973820232, 77825.68617627322, 44185.33736326149, 14251.866656117574, 4620.198590177582, 55605.922491179204, 52054.867955200825]} +{"id": 883, "vector": [59185.98264935488, 40258.98183897513, 65768.63315738503, 18260.758472473783, 20373.954015995034, 58640.72958430609, 69629.44477071974, 7844.964202010629, 13921.549692184088, 46546.61634128592, 48864.800058142966, 88197.82117252199, 87776.97048469407, 85723.88896890229, 99905.97805466336, 21295.81982880433, 86716.57883014911, 38415.9739791293, 94541.23719609168, 72841.15240352895, 59971.56961230693, 84685.44269252007, 42032.055547682445, 60457.62902840644, 29504.736235394168, 30325.891961236197, 69273.67271097438, 52129.551086522144, 35352.911641547194, 70659.21880052789, 53386.63570399995, 79204.74131744796, 39253.70938510075, 63773.56171147468, 58993.07979797176, 81199.62638261128, 30790.978695524464, 98776.20197551681, 9616.61210617467, 10750.39631740049, 77847.01524189068, 7762.857624867603, 74695.14287622529, 46203.71216945278, 79393.19062325309, 96689.84262004799, 91973.55426370646, 16053.262363480037, 66211.90302209885, 78847.86724388052, 90209.46971141107, 59297.561970261115, 99925.18148772545, 90387.33930782547, 4926.065098192967, 33983.687621530255, 14296.453002018328, 17864.86419225346, 93001.98424357631, 40643.399360304924, 10924.173588521058, 87332.8709264064, 95522.39756337216, 12154.520327552587, 48810.207957446946, 62218.07424364107, 99321.00696013837, 15183.88804405648, 31035.28999146119, 34090.983546904194, 79192.289036802, 39663.986913260895, 9278.593045646321, 27034.15382480805, 65914.32579726861, 33856.95944383169, 29325.31495998969, 1991.0340685833082, 17754.400789722447, 44975.45907426138, 42414.83050311323, 47366.17188648599, 31394.926242905396, 67889.2817502297, 21130.45876895414, 58584.50319566564, 31664.541628200892, 93194.72806268252, 48629.00174670246, 32021.52933049186, 15905.6363039909, 57482.13320059075, 41942.99164732823, 73492.43914779586, 20178.20669523278, 52954.86968359517, 57477.53773965956, 91017.12466044178, 76769.3512002486, 33354.015698490315, 56093.97910626206, 60827.982546211126, 69759.29243028084, 59155.43284577859, 7777.908772852194, 69673.15303651946, 22629.811116009336, 17438.472329929435, 78187.85172487216, 82226.22609365052, 44623.21729433362, 44851.7759173241, 80091.41618964945, 98004.81177033413, 39302.16467660708, 36487.90069664549, 59050.40355451061, 31950.541407012322, 27776.663758611263, 35229.47537017183, 61600.02234311584, 95111.10331634474, 20650.039124218965, 35846.79079700149, 81894.05659043397, 61982.856161323885, 2732.952867421623, 6544.722634881173]} +{"id": 931, "vector": [63909.82366229777, 51842.08355160951, 39423.488959010625, 83492.93407536323, 2068.8307333266653, 81838.23653595346, 21668.59237486156, 49718.618990796946, 78621.54793288742, 54917.05959714138, 87346.03313468752, 5127.9170687429705, 39744.9641364768, 45936.81501545234, 17688.626147840092, 54025.24470918542, 23751.456241700107, 15685.546117254335, 85506.17626598479, 93535.23160119265, 21609.561532738277, 75413.5028196029, 49238.9845239791, 34824.72841699996, 65587.0025747581, 32929.46643575955, 26581.71879111103, 5360.66194122764, 36015.494189992925, 73784.25232020905, 30392.66621536294, 83772.28995628639, 59292.63806194206, 39191.18282818577, 45302.791207202776, 84639.79443703046, 92383.05623524645, 19767.564333473187, 51883.58344079625, 26869.230405914324, 12239.003396045267, 2072.9167415626025, 86281.64182170444, 48329.954455174586, 43714.203419283745, 67118.13500500766, 92662.97746368512, 93519.43125007617, 88033.25951609323, 57269.91416368675, 60470.31482714873, 74012.58793151587, 59207.941344280065, 59780.87487058933, 57558.596809146045, 50933.44168299919, 67504.70368214954, 27337.331281900788, 65836.87061737372, 9013.628994960054, 77459.41244228325, 37388.82036591315, 43844.91419004822, 86180.96916970091, 42550.840515794254, 89012.82253621695, 13606.913303970092, 77910.53847051992, 86436.47546831699, 98972.97031457366, 23494.646216138106, 64295.49677994758, 48096.90863935756, 34736.7110962236, 62860.008211134664, 44054.538025479706, 55977.40092383976, 33402.860657954894, 82138.85679554995, 69879.4232015811, 37920.59553821795, 81383.05774514981, 76071.77466710315, 91285.83169794145, 17417.24018954345, 84310.25607963222, 29985.072457735805, 32665.599606287455, 61998.37620014923, 42142.18154609678, 60717.21837618236, 74772.49847907676, 34424.34153190621, 53588.81957269534, 57473.78460010393, 27486.11069451534, 96668.38034770975, 64840.74951180521, 90716.7394382739, 68208.09896144793, 32157.52176314983, 99816.29904323604, 37787.29836823311, 45489.56246898329, 2121.550641658232, 67237.90587545752, 63502.77879190239, 21543.90389267684, 20335.583177225548, 85901.88229361478, 45311.466282505164, 50592.77865128412, 71953.38235137338, 91287.64729189649, 69583.03374836108, 6613.443848335465, 33350.042855469816, 56896.649884562314, 79862.86593733374, 37678.21946607408, 71305.46252180882, 46031.22681472202, 86279.19950667635, 91188.37037831891, 75039.36531037213, 24882.51719190656, 8832.191272274325, 21886.369577347985]} +{"id": 886, "vector": [47881.56153793077, 11595.15861240359, 60351.479473584324, 90059.51436422314, 14962.698471650925, 1849.6350054121092, 10443.625557472125, 18597.024514008055, 92678.5464792297, 59693.84017509894, 29644.28680324668, 67947.54327902249, 62965.65540111776, 12610.655563779615, 47311.99853090525, 43552.99432796699, 45422.24894537406, 66499.50772312506, 77145.35958851046, 6601.835654154819, 47053.642667072105, 9952.545379648082, 11272.47099709835, 7476.8948282408655, 22107.613367457, 17045.687942153232, 44769.64698164084, 66954.20645753964, 69281.50438916728, 40658.2215229594, 30910.078079795556, 99346.1846267633, 40559.28959565582, 36982.5114480562, 56876.21369346146, 76993.55584983766, 14574.648549157342, 74267.07372704403, 1022.23658222097, 8524.810429367135, 97308.75483642168, 97786.52431669056, 70872.7284541273, 19789.453222746335, 40834.434137612974, 36175.72399061414, 55331.46308301243, 13438.427533613029, 26174.23387990159, 48178.60257080374, 9895.136441654107, 15472.26238097974, 61412.22009386672, 51197.56117856115, 43362.788142831996, 20788.39694012432, 33438.190574586326, 42155.09572076446, 53066.0646374714, 11883.25321839413, 84427.1417188772, 3080.6882147279666, 78049.37457029801, 21229.539146000443, 86902.88265393085, 10850.667525850044, 72647.74983716873, 66355.19116385032, 76573.7974833086, 16878.645422210437, 79933.44341821621, 7101.911792157667, 82078.48841107498, 40691.356943958766, 83638.89817827154, 72074.29411849174, 31704.667684006705, 51902.780547114366, 59270.967783578744, 14183.047832437935, 62299.91346139949, 62745.114993518844, 94575.56346072962, 93367.61945574378, 57015.976854307104, 30048.59546375881, 18058.72207203919, 61261.30035887848, 90456.30128628458, 96682.35907421139, 63362.583040577236, 69675.19562562872, 60731.5680442221, 63954.84855236771, 34557.480035717614, 67823.02904972872, 36378.80047188675, 37516.397009636305, 79551.83788614169, 56261.703288431556, 5485.986255708086, 28052.13495952579, 34818.24186715495, 29629.311279562564, 21483.218280923043, 25346.541652122367, 11120.83547520839, 79133.61029937274, 63696.43111377096, 23052.93761017204, 30267.025076848386, 34639.37123447509, 77968.38300288575, 10459.226857263171, 51701.73257365402, 88692.10745152016, 61797.27702012867, 73350.15008284592, 90462.69354133192, 58784.589005063026, 25376.989593443534, 48926.82301038464, 33203.6758026628, 8863.694522052356, 60140.426270357115, 20714.17024664981, 63316.00049438841, 44284.35007309838]} +{"id": 581, "vector": [11929.322835168032, 56816.693242807756, 40161.67748792609, 44529.176415358874, 77626.9969204732, 41644.48125352118, 60220.92141545747, 64706.16504091294, 89784.5209091889, 77765.77122605329, 90943.89399521273, 19244.894715160597, 65470.52459744165, 47410.29425059225, 42271.27971629007, 91218.25178536578, 23787.561535189605, 65964.56668361547, 66893.0519938704, 76962.2929806287, 43682.01204741146, 36879.08113939964, 34127.880200565676, 76864.38323658874, 72870.47378571247, 98109.51225372887, 39728.05728558566, 7418.657891590042, 22317.96094426929, 33756.87567423361, 13680.594317492434, 52943.23325032915, 46904.32325883327, 73230.35262103358, 45029.368387154, 69716.67639707583, 72927.74999514768, 53704.83609270042, 48031.267770563536, 58585.09007610285, 55651.79810879457, 8773.879032472387, 60827.01201671091, 89316.47134187206, 449.8006805997101, 68565.20747688074, 26825.922893403644, 40201.38848193294, 88141.39063326296, 29752.38465790958, 6610.573296371458, 81362.63411145807, 15995.280749622698, 23903.51989245104, 77990.57032255955, 61120.27655304163, 50563.99683139237, 97625.56398676096, 91692.36385194544, 5854.35776474763, 97186.94269615899, 77026.99696873369, 37855.87238855485, 80717.75931584276, 22645.3562070556, 53265.749523941166, 35595.42349968219, 46646.52873109427, 61890.6526963148, 73757.35713170674, 90549.68346962091, 32324.407098085772, 15639.733536231959, 70628.44435686692, 72999.48155311424, 89962.1149286289, 43550.662522318904, 74575.1016968149, 64222.79995741004, 61384.25390522467, 89300.85733257922, 64619.18769441843, 84898.41560203551, 57808.86852700956, 57256.4401340764, 58522.742844055894, 75495.62936497123, 83817.94370598659, 34645.54198556104, 95166.93229892729, 52375.79701583316, 88198.83571071128, 46029.80629333277, 44073.85371962702, 28252.979963437607, 53328.08026239486, 53134.29820196365, 88929.35347403624, 62304.12565387628, 95479.5961602131, 17955.97578229865, 54000.69220973482, 55254.35793244983, 63493.30685900156, 48158.77413012035, 80852.3861811213, 43881.30566776242, 75157.12927429007, 15019.861738810636, 94512.14908572994, 89582.64385748483, 69515.3697078792, 61256.2569864873, 27705.730919410522, 68047.51207310501, 45256.257351172404, 57002.75491545972, 57716.542703768806, 50696.670260857325, 87578.03154602244, 6531.922150529846, 1431.0232890812458, 5780.481956116323, 2561.4473587918574, 46908.48886532206, 12851.750453892351, 50243.22738869324, 12071.243722887026]} +{"id": 1223, "vector": [26110.617975714624, 51547.68992634822, 71467.63250303805, 33323.5197043613, 35569.4894663225, 76629.35413144076, 81804.32411975798, 30966.326519100894, 88759.2574115476, 91561.01390991472, 17545.08470813939, 50388.758501607386, 66118.98791927587, 91603.49696721451, 47648.14388825151, 2304.2115642854633, 46637.7083452026, 49113.97182107964, 22649.01127187501, 62016.48548607681, 15768.985752952336, 50564.811925561895, 292.56166054537624, 38132.52263027025, 38804.22480492767, 16051.97276129977, 57553.930084928405, 5024.262687844927, 43598.609169952564, 35940.41983899234, 65812.72884478369, 22987.002523218747, 60796.54801306224, 59488.04064248801, 41720.04852081057, 94565.49680861966, 21666.36904067082, 846.2510060232021, 87640.77157056596, 3909.578144995829, 14978.753390497845, 71397.40239613173, 9298.049344848825, 97755.58763704891, 88191.54699053799, 98594.54282307248, 58497.394818287874, 28419.38480527996, 66521.59783493789, 57622.69368060705, 64173.248688925865, 54958.97563955764, 96889.85152866905, 76639.29429080087, 12977.06061044135, 7460.463694642483, 13984.17750161942, 24604.324594014128, 78237.68942653587, 99174.24084438109, 40971.73558630711, 81942.33844399067, 11671.229840489006, 96930.72249077781, 70203.47490472884, 25820.058582485228, 89091.93446172061, 49681.707305019096, 95990.2613929009, 10412.693972651998, 24725.497093654958, 80944.87596064826, 96139.37933289216, 35576.650235686015, 74685.86334479079, 55964.30300986687, 85202.37260532446, 92471.81757359153, 23162.62128174389, 66156.13498466687, 16785.730773300565, 60766.754392946306, 15073.855376346102, 63957.988630589934, 17799.82455741207, 49854.65563293849, 11524.997550415872, 46489.80776456923, 83118.0161468038, 14357.265425040177, 9371.975850668756, 1749.4651717858467, 4950.424089279126, 84219.99976303193, 23478.44378992654, 87003.69947461752, 59630.78227476862, 75612.49011500862, 80279.9910832192, 57433.54460934511, 7685.099779417504, 12207.105317038291, 64306.359547690474, 18288.06737743476, 67090.62866263819, 95345.70587549524, 70924.04819283857, 34332.68928614982, 31785.739087642283, 45722.11871364512, 89716.1090858281, 82954.4707543541, 64237.803503951574, 70493.00015894507, 75014.5784465273, 75063.50025597106, 12603.7986303829, 48653.98424898827, 24896.94661384976, 92015.81259577982, 81466.02637485505, 31234.486428505148, 69722.91590811533, 84616.1569904671, 16408.821703553556, 2901.4171138598945, 55742.272744660615, 42880.55655172635]} +{"id": 892, "vector": [1994.689215648604, 62978.260252578, 23244.943593079835, 11619.687772000298, 65847.64117386944, 23265.404413042645, 35207.525223721335, 10430.297725873128, 87900.5973408705, 85510.24528522717, 18721.82979639566, 5609.50124188283, 5740.050274302466, 25889.03527545293, 22787.405138291895, 12257.778134081122, 41148.96107670825, 94879.51726928772, 57916.45731761659, 60429.11657670863, 51814.06013403551, 9987.663539449099, 11939.450581148114, 97733.1745145188, 62045.474219628806, 1070.1214088797096, 52964.55113840948, 78377.56811865086, 1723.3117156938426, 97870.45580959892, 48164.51989231083, 6738.475609420136, 68567.06162795467, 33096.0413740279, 29165.378209337534, 57027.881493183275, 33488.57633298562, 79882.71541910425, 36179.59207386801, 2725.3520648367726, 96136.82869517434, 6284.598616629744, 12462.653402597145, 36332.94754195197, 82328.91304904677, 38102.71597959727, 55895.88480145218, 35698.347835596986, 678.8368798153144, 42247.76248296215, 37343.61118851871, 65131.21662823361, 28894.08049721949, 75162.23223703165, 73466.99063498981, 63117.80080544279, 82114.01226014721, 13260.827863461755, 22708.981639242054, 74503.06559994337, 66883.79715299004, 75885.30103154057, 46141.1402111065, 588.4579628588682, 11958.702787712817, 54525.89435756634, 81243.03277377758, 57543.61859755505, 31851.424847893817, 59195.78459155443, 19810.373880678046, 58476.895818101846, 74254.70912695138, 82310.90507038988, 28302.5289988869, 32046.813218886404, 22813.097809443483, 14120.704706523413, 19942.794607723623, 73799.22847065824, 97500.02293128285, 24145.09731351514, 84740.35459549059, 62749.972079887986, 37758.972862137096, 71416.6378212626, 42504.816426617734, 69876.24699945685, 36190.18331109686, 51197.6306890382, 81093.4838268885, 33287.60726308308, 47252.58887484198, 24498.395660687722, 48835.29877840217, 32721.85389103891, 27388.432575120303, 32883.63990482498, 22028.54592440301, 72218.1980701772, 12631.554243197885, 99415.50645515177, 58073.25837637424, 20260.32753542247, 68559.69068303557, 81768.09334441376, 87021.27688331029, 94331.01312928373, 76148.59310665129, 85910.60762198736, 11533.2774883674, 85873.49949100931, 69898.99990269651, 51899.61107887081, 12089.00459198976, 25153.07633844106, 16941.78939731922, 12787.858705180943, 39278.05290171491, 27221.339215450167, 2428.241863392444, 95641.79338881282, 60274.98750336936, 22012.933486455444, 60313.32074496725, 44009.32328198087, 24002.49716636048, 82503.28889632321]} +{"id": 282, "vector": [35639.7272494601, 90711.98854539802, 16656.14369496412, 45073.058977777844, 38603.289728711985, 75247.58348049072, 19810.865643504738, 45914.97519325389, 98530.7467231639, 4552.151670284943, 53578.62532952984, 53292.504896560196, 88131.43722774542, 97208.40397571055, 84535.632563449, 24066.27957278651, 29333.95205250172, 87932.42687110907, 68711.81555529287, 1517.4512222939752, 47089.858520171, 24145.823457858784, 97158.6803694196, 73592.88552316636, 57894.74164284004, 74527.29818129596, 94046.79568485619, 55434.02794665861, 9115.70154763609, 44688.09955220633, 33059.72540724574, 12000.134506194505, 87893.200583151, 6335.32402795679, 48143.78697485251, 17194.145281658348, 36441.45055383988, 21511.529474930623, 19627.858699335564, 62914.683297860065, 50122.23024033544, 78417.2177101329, 41698.074049738956, 28921.43752202705, 70139.25532218584, 46875.14697877749, 86968.04884072063, 91266.15134475395, 47715.727380739816, 5601.511220089162, 1858.73490453885, 23326.77328105769, 62779.89348367133, 48137.38063158626, 13728.430431880633, 30762.325484545738, 49921.690625274474, 80268.10573793325, 67382.93932970252, 44550.072453111934, 75606.01812114898, 36191.109514068776, 75049.54317691778, 59500.98053215093, 90624.112642264, 70576.76234911967, 2531.215603699577, 85540.01469225358, 2134.235804684881, 32810.85468826417, 34091.438083465844, 32872.18773718683, 38489.58755100326, 50725.496954392445, 27820.786218087957, 85951.69374637405, 15324.803983811675, 45355.90821609462, 6614.702345933321, 97114.31945561573, 51884.196987233445, 58874.150609923556, 74407.78854054442, 31338.110559803346, 79828.13945243426, 48588.00517406756, 1019.1668340071791, 69843.18083034373, 55954.81954418726, 90625.15031052301, 80596.02222735119, 10304.095736276397, 36305.50358705909, 19505.580176756033, 42173.76004485601, 77933.99575500166, 69755.65712300484, 1022.4020658487709, 70252.76791464732, 74273.7805603717, 40558.917982697494, 49812.191199564724, 12305.405362843825, 4699.317010812331, 34184.48821132058, 95593.74135751021, 66167.2707790486, 23629.517172247182, 77602.72726432825, 39987.36431983428, 4167.824825080357, 20408.835156972225, 82340.28951173642, 58163.827129929145, 86944.5796192626, 26684.707837959133, 35594.23451417062, 45462.1621737765, 61635.08098731999, 93520.94984074649, 88281.67411937444, 16210.712882856536, 61984.25030914656, 19320.24269842294, 35881.03117683492, 89325.81464038504, 19700.736182403, 89877.56266622544]} +{"id": 1714, "vector": [62699.453636293765, 9727.399999528496, 25951.422436842407, 18357.68700349396, 96365.0657813437, 24719.51479800183, 95639.10946035656, 32542.568495425596, 38418.605674784325, 37182.06115424211, 56445.92698412746, 96634.97600512355, 50446.92021980716, 52485.13127000218, 39858.27127951251, 10793.359264093971, 15704.417120434711, 16189.272950304312, 9215.992441992526, 77161.21923845522, 52398.55815359833, 23661.244727191923, 64534.81856077468, 96232.80532254143, 15112.60086543228, 64138.106534617764, 47789.897494977886, 67229.010717062, 14383.881456292069, 32803.82810420328, 57415.04060928174, 24888.92542939255, 89737.91708488234, 7888.99887911454, 62017.263439352086, 36026.488199815365, 22887.0903108366, 87221.46000572755, 88418.1670764023, 57384.209298951086, 27628.327641628715, 43978.01640392371, 61444.55340835461, 44448.08159686163, 87270.17563120287, 22004.02480113256, 87541.39344375489, 7938.293743199476, 56364.20906924655, 99620.57038971655, 49257.82179338453, 7134.619824449528, 62345.07613108643, 29396.16917412757, 84795.93273509725, 87392.78994063997, 37059.72152825232, 64530.39591392057, 64005.89041165572, 96082.42239451472, 81344.05740484849, 17124.406578924067, 41010.391609506754, 67413.55171852636, 51302.45457075724, 27354.30039427861, 62874.1439336257, 68776.86080924273, 22648.078437561893, 30548.934789590974, 38652.51041521347, 1369.3684330895128, 64522.64543235076, 3116.401704137861, 72069.79596820498, 81981.84618529171, 71154.05567065353, 84922.67448952392, 66454.88854820262, 63990.36285142714, 73373.46324793913, 44022.3196159819, 62849.40715051524, 29167.750533856863, 36203.42670090711, 91648.98591013224, 3494.1302177897705, 88712.79938293794, 41119.65327357363, 93452.04887796423, 89415.175946007, 36473.11550847393, 21832.427788798424, 52484.71512926855, 25694.235052279557, 37466.18970556761, 85154.22634061928, 96493.44588584935, 46951.4852380783, 81990.01946238859, 82802.13772168635, 63792.49665728829, 18157.59307228577, 17486.75680722821, 76782.95807905874, 41319.175272100394, 26776.775697093013, 11899.618867153782, 13435.47693775692, 168.27040156651708, 66497.59238042064, 6340.609180343248, 85092.44948496376, 65649.59476814991, 58394.63942928986, 19700.832652514633, 78884.70707040366, 72957.05946223748, 8426.757086417414, 51726.50389189818, 16306.828990233502, 81290.88068195271, 37338.522929702456, 17802.270158591728, 42993.434041553934, 29366.488905759426, 25949.014403968686, 2695.612334297104]} +{"id": 1995, "vector": [81725.86283404581, 55037.27125274327, 49929.02835903762, 13082.124637860814, 59934.645978642366, 46905.545499242166, 76362.73549392029, 85133.18949150172, 20140.91012830228, 21046.474568816076, 65004.202049028536, 25978.445601977517, 61794.27350152563, 95200.61529885899, 48157.936972249496, 5388.307556387306, 56601.711857113434, 77836.8432539367, 10135.533646139449, 49509.498694632835, 27398.698225141503, 61780.022321252196, 98077.90499512365, 99677.42296418351, 64758.15583342063, 303.47475631082864, 99195.57220642241, 16478.826258801193, 67878.56304451407, 89933.37292013271, 33821.636550638745, 96946.57122525334, 96726.0484415057, 47591.829450196754, 51140.182107855035, 93915.60824968504, 86135.7589377866, 83150.55796962336, 99458.93300464479, 55045.99956265911, 40498.90116877963, 74759.67311028659, 26911.870596486508, 91566.39572640578, 12077.129239159623, 7332.226212339554, 41566.29492226983, 63389.70970614445, 12016.88867398295, 96397.43657190271, 47525.59736432871, 39384.1911049811, 16384.248590147276, 29349.711096325882, 44717.22487991079, 58395.15832538146, 54029.8562221308, 24843.01074567845, 69380.67711496154, 91723.428563065, 45476.29959778153, 72276.98451155904, 83480.7675663898, 62877.84224713241, 48591.00419902063, 87955.26541341757, 92526.18284601433, 54124.29574912114, 24567.837797587777, 45150.48671219605, 79.46460073944283, 42640.211118353174, 45177.2239599897, 82370.12714120511, 41349.103093529426, 48165.46670943874, 5807.490696585471, 96040.51919635004, 49787.625585649934, 52748.41374274465, 31352.47165564764, 79955.2099589705, 50306.28712723043, 44068.181349682935, 12696.668207948847, 70318.4443824753, 81658.2031169183, 97516.88358524135, 83168.27461419567, 25595.21421597082, 26944.291357260365, 79220.602757491, 73361.86202379085, 94435.95171353851, 43908.3568309382, 4231.036766542428, 4632.118709275901, 82217.43921974453, 3462.394875556474, 28217.029913858947, 65604.96044410452, 41007.55259888101, 57262.806086801946, 38557.56647665044, 91663.24292557375, 94419.94379635726, 91110.29194241216, 20432.897345979683, 26135.286997743046, 56654.176914109965, 9802.695043053489, 12297.730802580243, 1708.010222901346, 61287.186482262056, 43844.43608331089, 15535.180300296402, 48921.406987334514, 5971.593350265336, 67456.76757426102, 33376.851251292115, 96478.44779838769, 58207.88477809572, 83437.40734869304, 35483.973087348284, 67075.46112896656, 79513.23664087532, 3408.0646810808357, 11376.241226412765]} +{"id": 1954, "vector": [27169.094251881543, 48141.03854366772, 98268.92184583542, 59138.06187893307, 15175.14090334282, 75094.76170798329, 51642.347047332616, 97430.8642274513, 63873.3233896728, 14479.029845609226, 52314.69136460899, 95854.26021098008, 87590.99264082553, 63044.83944814736, 53429.32174306735, 67107.11047184658, 77657.92170870955, 29958.96988256381, 696.8095109372463, 82663.73989160878, 3585.5823656413822, 8082.001002463313, 34709.01862137323, 34925.54637895775, 59469.93880776439, 30850.324616571455, 89111.28625156944, 39343.196358861074, 47248.77444696351, 62213.64440792132, 83873.21555522246, 93445.76329098723, 49307.0494982779, 49371.657351892936, 11423.457845433593, 42117.48674466278, 97167.25332486315, 56945.82181284987, 77001.72734601451, 82711.99588885425, 76595.01455030155, 92345.13250831029, 9225.872112847223, 93620.04767328627, 93409.89795884833, 30419.48334542315, 42467.60051037839, 60017.76285503758, 31203.552104504095, 74275.5321067811, 91974.84985084538, 81480.88133644352, 93180.97070362083, 51656.33615175922, 92055.69555802394, 543.4712684102893, 66162.06005655999, 91867.63378753072, 23695.80261095291, 42041.224355097904, 85359.11015884252, 91804.96002528575, 51834.86898487126, 60322.99475908598, 11190.07358933819, 61040.03594981885, 49590.65426736176, 99970.83956045394, 63259.59221520566, 34599.91303559913, 81629.98370973322, 77126.67770901181, 67875.0109283693, 77210.56988095469, 51738.905781887304, 54986.88224886296, 27819.736683111696, 47090.666523134714, 85838.48335586285, 8561.921443208275, 70668.66726466476, 10834.708440117036, 80719.57877802881, 4879.378936811673, 12290.097864435002, 27446.796967231814, 96614.07889491881, 276.4641961911729, 43291.70189052558, 27545.75440660768, 86751.83477699602, 67311.68191665334, 79618.99735493962, 5189.029002828316, 20717.215291284883, 23753.025419242225, 29533.874138019968, 74501.6303113772, 37940.96357285541, 61176.81647387542, 61082.20465018488, 42383.87953447931, 48379.93776102657, 2583.1013616616992, 91399.05422016249, 76096.62443059063, 28834.257669322793, 17112.64814422575, 94499.12110546375, 21113.21236942134, 92881.79838782575, 23871.763287945392, 93057.18343914513, 84938.10463914901, 15681.524930687918, 54660.98437516889, 87684.46440487956, 1714.370048681524, 95606.77786992113, 85507.30611647235, 24318.60737027086, 73787.29227227431, 6968.273220946264, 44239.39393886671, 45756.632100585426, 36877.95709895848, 75187.5027240789, 86219.62930515922]} +{"id": 244, "vector": [16199.640950101546, 145.499876843036, 35754.76175304446, 58482.041263429775, 649.5910564940699, 95628.12457273746, 88750.98183068173, 58976.20658363252, 77863.87376421117, 68739.0756539733, 84975.98727248788, 13828.751350568225, 89371.329753423, 36431.49513107201, 66958.86025451761, 26273.782432693715, 54540.09427307547, 56161.725423686206, 25121.829087089154, 52325.083971158034, 16012.29263057251, 84982.65796487957, 34594.436887175114, 77187.54316164849, 69514.78702426434, 47765.85304314748, 90375.1904888745, 45174.7189256787, 66767.90293749602, 67196.26813475581, 45536.74661080943, 41879.87172075719, 3898.6811243597153, 77419.8089002684, 92034.94838502393, 4170.308721357618, 76075.75676429263, 42884.178183612195, 22060.585110368014, 43133.717135258456, 78905.87182750787, 63198.97245401369, 59298.512417396254, 42591.07661736375, 56152.357263296035, 14885.112355007002, 3342.2129939046476, 32683.38047104714, 10941.17426656459, 72734.01164706159, 52462.83567509286, 68004.7598655908, 22221.602067081625, 46624.74252621952, 47579.57638951669, 9159.546538718034, 49993.997395119826, 17817.55481184244, 17978.288588206513, 76293.98170083162, 70160.99475872301, 83849.58728050604, 45574.94987273837, 65706.76258667816, 96430.06007350788, 10873.164677435065, 40126.266054455926, 67937.42480941078, 20654.793315772025, 88461.87041089374, 9866.013620263193, 58478.07439774278, 61613.78742656709, 51494.66904470354, 61347.93160948743, 36258.45226423456, 11523.437365560874, 56323.50000400338, 86760.63033921787, 71650.28377637536, 33307.171327434204, 9827.693180850561, 32706.116553099484, 7394.933319182329, 99353.5737610168, 61230.71924734467, 62404.11491331344, 33242.020803129366, 73656.53325490585, 59898.68346517903, 56313.10283139266, 47532.45142271982, 78208.99014676889, 14677.41182959803, 49408.352311264025, 6489.45257102056, 9069.967375826227, 40481.786403766964, 60096.93759467592, 6493.206566008292, 86134.28649878842, 73151.02082246686, 37710.792049063624, 28676.42895732212, 15045.715466568166, 12870.866184181785, 44880.6535738866, 69250.4195674735, 21347.155090425294, 86393.60907852331, 51030.45557200088, 94805.15608099705, 39007.158424572655, 37477.43117551455, 57390.78343204864, 86050.029400095, 19112.266653574294, 66700.11227378236, 96213.86925073662, 35688.08574151168, 58138.09571726234, 49120.302405586066, 67973.7605668538, 26636.08124100344, 3135.8530541387486, 51069.101033420404, 4580.142897410245, 37043.609155858816]} +{"id": 503, "vector": [95661.56145118675, 7829.096971565985, 50215.99976517749, 30973.301766663684, 46489.148478754105, 60020.7693862387, 46987.14647639046, 9840.793094428445, 49027.160286978986, 45965.34153530126, 93175.49734239503, 37179.45610755324, 78264.10659848005, 12276.84794435777, 89870.96427951143, 73474.86575130827, 42912.72315669454, 38414.33734912625, 79273.44394922201, 9057.635500698314, 10930.842590068223, 71569.72606722557, 70217.72322539311, 70052.1113871636, 65871.0491986797, 57093.537568494714, 7976.681672185193, 84575.88345138845, 61715.33151997096, 9179.452511249963, 91732.0819788488, 33109.15143058153, 60995.127888834446, 31104.708007073878, 12104.651416772273, 11212.176208638502, 83618.15825343732, 25385.654543530978, 6335.616924152787, 7422.711732399401, 92081.33119078155, 82918.73628587199, 53207.62826077089, 51989.53297084036, 12483.622532406258, 65875.77976437887, 81444.3010100132, 69691.9321344817, 83657.76209474758, 27119.117961908145, 70802.88977181725, 70851.25409748328, 78950.34766737522, 4658.534370875244, 81037.90594868959, 24511.76465277608, 52650.407880684914, 2775.239471267266, 54986.60219144123, 54852.380531111456, 1883.9962732652093, 69248.78720596158, 1932.0180454902247, 82431.69335149853, 13914.672723656251, 66263.09073165114, 14442.733214067726, 7591.0280526460765, 191.83938193443328, 53598.314590966664, 41558.402220819415, 99064.25019259026, 1320.0949212618718, 69549.3821325445, 6078.254167658326, 65343.4903687213, 36633.00967574388, 76508.75153765567, 94036.02738628317, 32755.799742716263, 105.8028092961294, 18096.669975497138, 65510.47578856359, 66458.89985939559, 94566.95327742852, 98934.8261903366, 39688.207616473635, 66330.55397046827, 50241.01299450685, 42140.51616028225, 6611.409732594054, 65379.644752505876, 42470.20530488532, 26496.815784896367, 18273.7808152734, 52105.4908761142, 39293.0059905701, 79316.12603029882, 98857.30873807774, 55360.35325602007, 29035.735515459528, 24044.77931126372, 29720.48735794647, 24268.202540966522, 76646.23333899201, 27245.392695219172, 17657.46614608552, 4798.650963066963, 54643.19151846383, 47780.832985015586, 44621.01441397499, 22822.000544053666, 528.4849451575924, 70534.30387917825, 7353.059135883711, 26550.515838912914, 96053.98181432727, 93872.79599319633, 52137.79886384185, 46852.89372529359, 92093.07901085448, 1215.1606029114803, 30691.416754958354, 5421.328392780345, 25515.330722388386, 19832.00883286759, 96782.5149642884, 63254.63268682806]} +{"id": 1342, "vector": [69555.0747520157, 26346.63744649529, 60620.96817448406, 31838.497815561983, 47793.62220091741, 72425.04197850406, 22204.239504516176, 47128.11490632131, 8554.720249300752, 68702.93872128813, 71160.4061873783, 41319.67377242226, 4931.219980471003, 74988.03523361472, 23033.068902751773, 13244.375216867076, 41408.623034851924, 12327.803621412304, 20422.588244708473, 74731.4839593636, 66829.95615952655, 87624.23223634544, 8663.860304609749, 71169.49410535584, 42086.28514527129, 79801.3865171393, 295.443115787708, 11992.750150198417, 56930.15622128472, 47113.10310305711, 68978.68664788587, 64788.07289507771, 19080.849772655973, 66946.24838357817, 91549.67216655742, 61775.502044220186, 12237.32685592177, 68945.15627027054, 91912.86588323177, 21917.07506949877, 81322.40283084442, 33657.700578100616, 19404.970787990118, 74939.88685580708, 47298.168551858435, 56408.69521727236, 27322.502421891237, 67718.04992645814, 4932.268280879825, 24533.61544342353, 8913.280877077634, 2090.160786504169, 9166.76344818671, 3835.7122438418824, 94779.08761716228, 56632.225380642565, 18885.956403447777, 85788.88088726983, 1598.052128972538, 63054.507860137564, 50160.675426039124, 85306.49101218455, 13542.459816600705, 43170.54425209037, 45129.31356581156, 17792.45967199541, 46810.02613557377, 95748.0818653379, 88361.0510333845, 45265.13219741014, 67019.15943687115, 39511.892402981575, 42178.10182881684, 37218.05638625194, 30915.3130226539, 71324.01345887515, 45143.939361946395, 50051.018779251935, 81656.33095730536, 19250.312328518514, 68353.32055157775, 85323.38959759503, 48732.02807145217, 44137.93389189583, 37471.988312905014, 69663.9901222619, 46604.257352986635, 80789.73884090415, 78611.24656793683, 67325.20208475016, 33894.26627974834, 81079.4781891296, 89116.03920642493, 33839.48240501603, 99828.08570489484, 87837.97958750674, 4614.364930698778, 17622.55956277532, 52380.91791520887, 19385.699187176164, 72868.0712682318, 11697.954633221874, 60952.03444991122, 38507.949367911744, 55180.740784010806, 78389.01330923187, 41898.64637179859, 29829.31978235087, 20382.296303112056, 20798.721353760997, 25967.87000448757, 22405.07490412288, 19757.843980177393, 69651.04878223763, 4513.035977064639, 27936.005089650705, 5362.198324385681, 90758.49632605677, 24616.46624077688, 76007.02971112831, 17618.73428115438, 10273.093847473003, 25926.532610729315, 38324.99671773858, 57502.70363814626, 94266.90898406752, 4297.270626583105, 43530.63859937423]} +{"id": 900, "vector": [86475.95151681657, 15943.801795375522, 30329.58249766986, 63808.59658432947, 19654.243741578248, 17194.20899723161, 59392.34331484962, 333.29963066999204, 87970.28127250474, 1624.6723396102002, 62030.82899794662, 84225.46998455739, 11534.892131985875, 63445.6397346749, 92674.8515455118, 84014.7855098368, 90296.03525417107, 41318.35328714327, 57593.408947348944, 7816.645609457751, 11353.534161256928, 87261.5448474202, 45704.79414863792, 12420.437585180254, 29863.748387494426, 45967.48679718583, 4829.982174089142, 43952.086401446744, 75817.31516295664, 30070.293618507858, 3651.637393666174, 7077.673119222527, 34308.01925188403, 56415.34781336539, 64928.89664040764, 82922.09845957134, 23785.324684742416, 87653.60516104598, 2069.6856612559823, 27769.368599060173, 16853.394044541525, 81494.94473335147, 87257.47135502692, 65832.60138047484, 87680.00975592561, 13262.987484502797, 45690.68318310904, 7439.4114697756695, 80122.81300413504, 42759.17139623306, 89886.1210782575, 8187.435438356194, 71083.61697907533, 63816.27729767543, 6891.42899805063, 42905.46598186366, 64692.296852441345, 4442.80395877179, 90482.13646910503, 87064.86996805236, 54725.816248689385, 77482.34187659506, 93793.73736209526, 78248.18544508987, 8132.411806727136, 40454.153114295244, 20731.526684170753, 72390.16850550624, 28707.27262786741, 94535.982400775, 46677.717398080975, 14906.47652800282, 40116.48316739163, 70917.35319254364, 26993.373802292397, 77012.62811580575, 92716.29243880873, 81632.17062855544, 45255.27012137365, 10956.1058092153, 74003.38192624057, 320.2311077083353, 16897.10220778151, 51293.363714682506, 22164.99769611171, 45652.431591819775, 24981.573888047238, 59268.41359230421, 86516.43383418972, 13936.947625504903, 23869.16615639759, 92451.98333385684, 20442.0867937829, 24801.45283889523, 79358.398290535, 79886.78997423706, 61897.25564856156, 23167.002293956328, 37919.17992401862, 37031.426610547926, 83346.63431418694, 94458.89023998361, 82966.92210477366, 47169.99800672794, 86788.90918448457, 52111.53596190893, 64233.75293623159, 90133.23516425383, 11818.577064989366, 19888.586942659946, 26642.529652528658, 53995.80232280655, 51603.091860446526, 49236.75319837489, 73380.68419969236, 37010.7449960656, 8719.215831409421, 72020.84313542867, 99911.67900642415, 42104.46645157566, 69198.36095451174, 97442.86770184238, 78657.90094810283, 14686.39257247123, 14472.278405532368, 41306.485246238066, 83961.7270358324, 86934.40652481999]} +{"id": 875, "vector": [29721.424508531414, 41328.056174007535, 52223.88594999103, 57942.96520886823, 31821.816254651036, 27736.53911297416, 95441.08025308345, 64468.23900632458, 60956.08131645083, 26078.483240913418, 77525.40344918784, 42303.315594450796, 34800.99759848063, 99214.19949034863, 33837.55935296838, 79945.89261487297, 65515.94599225472, 79221.31909822744, 48553.878940374794, 96445.01311599663, 99776.80993363613, 99807.718657279, 58180.19798467618, 87257.6090714403, 45339.6973669565, 23910.235778009515, 83563.37963718164, 65950.20338313072, 4967.600913727332, 75844.67002016352, 31895.16511765401, 41009.56756626796, 80459.91571024036, 27280.85179905355, 61089.49274246712, 91325.41292238283, 4606.548644982134, 55113.58122687108, 58912.14723210505, 93861.30720039451, 83494.35971667044, 77668.66750409862, 9313.634054977581, 41447.35351854726, 71680.56691774316, 83435.9814565037, 37579.08266820797, 83813.93391283168, 70228.569776289, 3477.177549464816, 15136.435645315238, 67499.78568938568, 12258.37541686805, 49531.6129367544, 75243.60305184538, 88725.53923646326, 32616.412452233788, 1927.0152692926533, 64708.86492348924, 86275.12201737873, 81914.12303482788, 11703.471335283766, 22263.901517407357, 78529.43400452391, 24647.489125446897, 26787.460442232612, 66895.38315333052, 49835.656704786554, 61218.72112904746, 10250.07661075913, 31319.190203365823, 71695.93264614009, 32296.78453231892, 45096.727053154165, 84291.35581510802, 13759.926562033741, 56638.65101134644, 30603.31028090595, 99620.32034119134, 22900.35956053027, 98188.06871399283, 1879.024829967113, 88156.05283596688, 30488.018160964613, 19819.06683942327, 68697.44098918916, 76466.57689544733, 85616.06324649353, 79977.10582141422, 74911.42992372355, 43104.876326330384, 62682.478541348915, 93452.4865722184, 45279.6493993027, 35572.819000201205, 8510.810602441154, 63599.86376893737, 21051.91544158903, 36120.02047255829, 9520.63175854001, 1274.1670849032994, 41970.75968764905, 93074.55989898347, 9743.58195489672, 51731.90028126023, 78566.64784687574, 92801.2763828225, 88885.69372884562, 95372.17757018143, 92241.19117113433, 72380.26074902207, 97793.64927564914, 26720.493205989093, 26720.928824081526, 63015.428063280335, 5250.46592801961, 13496.798349834316, 32681.209168804402, 14571.520397756587, 69307.14528592098, 40851.58972377487, 95432.78593920905, 53500.132875151765, 7159.562930851271, 11206.655284083234, 92060.8645541098, 14567.998115629332, 79649.27601786611]} +{"id": 948, "vector": [96302.71323128183, 2801.685843842083, 28631.752321655014, 62861.053345223336, 59251.43559897017, 63111.83972411348, 49442.71842941658, 28835.666683147876, 58297.03738373395, 77617.35428827748, 5110.949868581039, 47693.65241742538, 31880.33039695357, 78979.71643606435, 57919.48570076728, 99123.23404768834, 7258.453494812078, 92078.39318788839, 93394.17422649842, 92565.6704520452, 471.59208373140916, 80318.32750436485, 80352.0952862759, 37673.67144076986, 95118.14888003432, 88736.4001543349, 18935.95045484587, 26454.289295351606, 42882.75359430208, 6660.782840934498, 39936.44661641732, 50555.51863576827, 17364.620960185563, 44461.45931910534, 76403.19397085547, 88387.48323076798, 37903.85062210658, 88530.00205863077, 3661.4312719829845, 84740.05781985051, 82269.21291759795, 12846.25323827252, 67120.32922419845, 84501.5354120466, 64993.39849574624, 8165.863865050116, 79988.53750035018, 48452.29934532441, 56866.9033082846, 26661.90686952078, 3208.1020969557117, 54934.835655820214, 44066.36178503051, 95828.92667607132, 21303.3861349066, 38778.36692398572, 6959.498238652484, 56448.26965213615, 73795.19429468262, 55454.77935624643, 44404.18166004103, 83208.92915395802, 71198.95148179123, 86621.51565598819, 23665.66829910881, 24033.935630479453, 25485.176575333568, 70497.4690419822, 33244.900696885314, 28181.38542826618, 87315.54472271737, 16207.250192504618, 25583.10926357449, 38348.84200423574, 35967.45594127854, 23204.13541043712, 43892.69921498052, 81319.73093608207, 20505.478002334166, 47761.89775215082, 93659.68573759515, 8726.046147296518, 58948.71977195105, 8965.000885923491, 98828.02098680609, 75907.24170145085, 47800.99914703486, 4871.588276123984, 11738.808837168424, 23854.608561339406, 62495.984527870976, 6766.338683042739, 21054.283646407435, 91250.13509522966, 79191.25814817243, 18370.81777031815, 63096.9893341258, 93790.21767727357, 79105.74501156296, 40458.676393346126, 25211.6417645992, 29295.992977845286, 76800.41054559992, 34383.58448628125, 18624.80015218827, 54127.12853348007, 31142.928852736506, 67588.66965588302, 47460.550154543176, 31206.92674000163, 5288.712449783539, 56583.60826326573, 92040.14563797644, 87576.69326562552, 75915.72983181528, 44110.72837342496, 15175.345747103263, 62202.061782377314, 76379.7043221993, 82142.494977185, 37140.65097978213, 27929.866124340762, 63761.20274887077, 79697.62769734973, 93726.45106218848, 12535.54066101512, 74175.17608010437, 24172.308118712128]} +{"id": 1240, "vector": [27621.773010881858, 66150.350369987, 38985.992470846555, 24151.430286658015, 24636.760388652045, 23327.693802182548, 26048.57506372149, 33452.45377028133, 44192.73614622162, 80576.71652049795, 58065.054502487765, 7747.290541823982, 28338.479044490326, 65811.3347915122, 8072.518834708042, 4438.133349167783, 563.5650858423991, 94615.56659068268, 1555.9732183947572, 14603.421590745125, 66233.20607226009, 75318.09210423645, 68674.64465637336, 72597.63739842433, 34128.48452523205, 70301.38675481078, 25267.01540660089, 5190.1688175762465, 53017.22023825209, 62391.03056699723, 9216.680490057572, 85059.82056583214, 20036.066156104047, 79167.54958488693, 51918.13188324862, 21110.237767861105, 54736.42387566546, 63406.76173176738, 19919.298830938948, 89372.25106777092, 30255.6527423952, 93060.90968252523, 78697.88721973155, 65845.18507894813, 54386.298746864006, 94052.3974064829, 18519.554456390175, 71067.71093017652, 65646.50085692185, 19057.90099501018, 60395.037070979284, 5900.283215517632, 30807.75589418583, 51464.22491397179, 3547.0781158719487, 3109.1275420859656, 37210.99065421285, 69404.93656065826, 87712.85069596967, 29980.156376800514, 47967.47034783095, 55616.05420920227, 14878.65109636105, 82816.74799402029, 29651.42645110904, 71238.09040732763, 8827.34032247926, 66438.05273806791, 16225.342425380562, 76315.88951608987, 81822.63020886444, 68632.52811232889, 16663.177114310456, 61683.534731750675, 47232.85350491807, 24171.671138081907, 25078.04508404439, 42043.297401397795, 39999.62631995613, 56716.21924176779, 36781.217505785055, 6351.548939966223, 82965.81838658251, 50310.62192864081, 41320.71204154497, 8438.534744459836, 19606.380256689292, 44243.97599970088, 33028.13859207791, 40066.85063211864, 99986.45088155537, 32828.19349317874, 89252.70227465217, 17928.00998321661, 84148.8068631584, 86370.9787203866, 65766.23693609414, 66680.2515829342, 36196.664981892325, 67345.05044333536, 96528.46929013405, 52860.61187145125, 87584.6822229494, 69089.16673145932, 80785.69665198262, 9709.11271185705, 66633.44557445166, 97866.01983821677, 41305.10154863004, 54950.60195519414, 99373.69609147581, 28989.713076721946, 41347.03083945932, 544.0266410991823, 44283.595944158136, 85634.71021701857, 70394.361765503, 12693.172875647773, 17981.10119577061, 7839.6724848282865, 27916.401775045706, 14045.101764285928, 7869.043304729817, 69746.20634850455, 99140.16891610103, 51965.460335105316, 8900.769539121644, 31310.797119977862]} +{"id": 481, "vector": [62849.07809427189, 79420.51560271117, 86574.30512342282, 99175.81882433491, 18353.667300611698, 63813.013482737624, 23654.277593236904, 1346.3194596981864, 7876.626541753828, 56418.69106879037, 60123.123612917974, 37268.26477097707, 27587.478618047535, 6111.115248531695, 73943.823735871, 44028.578479740674, 84439.24006418818, 14463.193473115798, 58642.33513810789, 47189.75513652124, 66666.72549753061, 57316.393277533214, 64379.38621606467, 43499.50416865446, 65504.54958145945, 94676.6295813698, 92492.39399574786, 54039.912657216795, 41920.18904924614, 21368.520683679115, 6077.627954658171, 15196.894116923531, 10994.335968015323, 44067.00626038505, 74960.26957191172, 38497.00469658632, 82546.0885633737, 48376.04715534452, 36042.40694404618, 41647.58511725668, 45058.20458656847, 282.8652017873967, 10217.700029138121, 73149.9931482034, 82814.65433669055, 92048.76199593006, 48709.957780662095, 52208.864607479634, 31538.03026498422, 22782.878480229596, 26471.26329991598, 61646.743497779644, 12669.043762338139, 1155.4602448002993, 16208.562518026536, 44333.10910009454, 52418.68628713143, 53361.25731599403, 27911.164444270897, 95122.41737280758, 23239.164629150644, 85385.62641801644, 72896.93794424922, 2634.35750139267, 60483.171430699724, 36925.34262374909, 76722.42641658064, 47496.66635477069, 43011.911224618685, 75397.59650661326, 98049.55395922971, 66044.88359414919, 84209.9082329997, 52284.63656925916, 16869.60828186225, 15207.644340734594, 40395.190122976244, 47674.26342948744, 53840.239717555436, 83498.05520540445, 60837.88733406773, 69416.79483075014, 19055.509171476104, 5737.151131046958, 61200.81607525152, 50680.169826699625, 51943.40964546483, 27436.249338342288, 11316.01733626435, 55289.527322695765, 76716.74156531245, 44473.77937780603, 41214.98661119817, 66023.05456544524, 18862.196336603232, 69771.87405164033, 31351.82997957073, 5974.911371937541, 22076.20456534124, 37632.18088174763, 68373.62420586972, 72426.00865256613, 60203.7570880386, 13962.238398558735, 11418.894468958186, 29686.288833892206, 2826.3555540899542, 11551.328543970629, 13099.328705543478, 87792.98097914817, 43747.80531907248, 15310.77202269293, 21613.82079244568, 81790.04340015393, 31211.441138567643, 55216.38836831635, 68764.21597157983, 23497.26503433509, 74654.40348852392, 79820.75832525847, 90678.12041638527, 97175.03672303898, 58946.33320693028, 73839.0481967559, 54087.490310869034, 92644.12098929028, 30338.3146688742, 33265.98984574889]} +{"id": 1253, "vector": [7601.143972982394, 11010.67560343606, 62729.25794292298, 445.77487059435094, 22807.210084641614, 53146.41898249687, 11361.997744919894, 41270.57241497646, 42310.20325176332, 11430.074873683727, 90684.73892618669, 67177.37233395068, 46999.33854503229, 16488.387522480974, 5852.744048993408, 1189.741541435807, 43600.4777372206, 9937.428750506982, 45376.958081538476, 6322.537198783162, 65504.12098568307, 23127.11332135574, 26032.969768934945, 79741.3550413614, 39677.58052037498, 68584.01908729564, 50350.875129517335, 20717.497158825227, 88502.40072725802, 58391.232682469985, 74296.40315462994, 29007.71084184618, 40805.916779203675, 50799.74090472722, 57667.047936858275, 69492.69135453082, 27264.810394396867, 39824.77702012772, 46891.12966340684, 40903.143085016025, 76839.62711689541, 82882.73464801656, 88347.84757029614, 777.3419551656114, 30436.99372657993, 93503.60912567955, 74379.97635809575, 583.0011612751651, 29049.679370444715, 65066.51046418207, 42569.750723182995, 63810.289119514506, 9868.651937424322, 49629.79877265094, 96469.68810432058, 27052.477971622702, 1816.6621003661842, 76102.51886362571, 59004.58453887376, 41325.62467284562, 93522.70724620098, 80725.88847420206, 1720.2311640532985, 43012.18123509579, 39743.639198569355, 13413.773436781496, 44689.21802275657, 54987.37893450072, 22191.24957091143, 17214.193881299554, 85667.20648100106, 17480.955569589663, 25106.690104897832, 80493.17581152286, 47248.41735913634, 72242.40700323269, 25716.04118295051, 21834.265417396437, 19622.74828636156, 28845.285144548172, 41207.78585065928, 11372.012324392377, 78546.0803143782, 44393.4359997156, 5372.17194991978, 14293.280889230708, 50867.09714693075, 77424.30426707749, 89822.89468033359, 79813.61150773156, 70361.78078176189, 19079.720295564086, 29533.50078536112, 87607.56211701715, 43064.24781002223, 49658.66217784079, 53321.400260504204, 49126.07443049531, 73332.58092796749, 43819.67133315092, 85311.02977883637, 36709.45808511231, 17571.284156900234, 54898.834583664066, 53442.61531673842, 38424.39717743063, 12549.534076379698, 53763.97250601234, 7469.333014172641, 56207.76630873129, 16572.48651118166, 10906.209067668193, 8957.4685377955, 76751.40967296743, 37268.36518096388, 39141.89562611813, 98972.96029008273, 31452.382099061448, 94732.43695325799, 7564.668451819045, 7519.219321252324, 11167.304160617708, 7936.026485990832, 24710.31418033883, 19255.38320229968, 22480.18981087254, 95282.47817750584, 88268.06336729725]} +{"id": 1210, "vector": [44756.871723584445, 97656.20267539921, 99518.2116727149, 6592.628075489404, 87879.5821005403, 45495.10147554495, 34414.60382156973, 51566.325085685225, 69034.09656553382, 82170.09845705445, 49906.805345048524, 67728.45281445094, 78191.20315143315, 75966.42580187587, 11367.313818252589, 32933.15812738888, 4119.93322474189, 9152.57868566055, 75087.79672479437, 14727.666994037458, 40106.14552962888, 94207.58282502081, 63619.07318674475, 19981.288739524272, 99763.32979086731, 4766.50598838183, 45474.419546859834, 56032.044223002806, 26439.032677648076, 45823.64557908371, 26870.154095438604, 584.7119864279992, 69876.4244991511, 11876.998633179059, 77887.28544197764, 791.694697098444, 97145.19141936497, 83305.6956420966, 90324.81117520566, 77255.98773685131, 60246.91869824537, 99005.85674477645, 3549.742165404768, 37882.82597927646, 93839.07779356265, 2354.192393321752, 61118.71352957282, 13554.487930875714, 91116.60036649491, 37915.766783969004, 90405.69625771043, 93009.85990891258, 45241.73374495988, 89313.6134104654, 28946.442765044656, 39347.71005054699, 35927.65973257306, 71888.22023461883, 1043.2083175856421, 24462.48555190521, 71889.54993383288, 77874.43204368358, 88624.19915130401, 27219.159749103273, 39577.35777780861, 90112.81380258164, 60466.92300913158, 17984.047196458232, 42419.88542845473, 83195.09005846742, 84821.0863080472, 41130.15780633364, 98609.07170755639, 53627.73250718155, 36259.417183830636, 74618.98858545095, 5883.00718825574, 66174.15753325286, 43400.79160632837, 75861.39369860671, 14639.022531253466, 29228.044125012075, 48597.84649541504, 82948.25944144088, 17690.182856823467, 3029.0883207543916, 23484.86169069336, 10481.867657574518, 18717.02826093322, 6537.847123984219, 23054.316203536586, 87104.43145765367, 79052.65536749779, 23602.105816405016, 15253.924806235642, 25049.621033878433, 46522.14135941388, 11901.772444627179, 23576.47915449659, 41062.005658401125, 29201.20596534129, 41644.89284020166, 30715.009352020294, 74015.8089361464, 56268.58493656997, 88769.7679235145, 60811.17212972638, 35633.00809516261, 24594.195604929893, 44440.127974068324, 3401.2285897681127, 10073.989787649462, 39181.660369733254, 61175.30361287561, 66571.33393025663, 21099.63984608443, 79059.34718712019, 28779.7857078418, 21779.476738394267, 38475.522949331644, 55493.805594418234, 3808.26509053509, 72584.07756616801, 238.51077487583305, 43544.84640041062, 78792.47266504445, 90634.3350065423, 57807.33493178339]} +{"id": 1872, "vector": [36266.555630333096, 19087.866955622623, 67174.50847701496, 20723.09147032906, 77644.88615555379, 24557.36125815502, 49193.095691059905, 35552.50599645754, 9053.598156322683, 87331.84636334484, 70132.4234497357, 75928.75114674482, 42572.61802544632, 9121.307027605275, 26739.013934728962, 10476.198544413463, 84438.75111203529, 80130.93882634144, 77015.03740779082, 70492.37425045249, 22752.126094547577, 33273.24830017328, 85.49157755465275, 39743.04006399956, 72605.12456662799, 71583.3688384321, 35094.389980699256, 58569.54468359384, 20102.80445895093, 94447.29471396413, 14174.676143123843, 67493.9828452329, 11918.637626086958, 74255.94888926407, 32682.7734899637, 92363.87797338911, 1758.724839566861, 9441.476180232012, 75215.10244770686, 67747.57171194798, 67382.35132784222, 41301.55295984587, 94277.34043686741, 42337.513802778194, 75845.33015228447, 20053.204693929903, 20655.64126816569, 26232.087203485433, 9876.625562920859, 50627.85822516667, 42887.22220026393, 51335.20914226083, 53049.118527531245, 33456.180214994856, 18133.984877257793, 41913.56241974711, 46065.74194703241, 29614.044576592813, 41977.25496338287, 5613.773691658342, 60770.97021327763, 96896.09894312515, 50941.764902144736, 40366.61828998058, 49505.49720078262, 75647.04158745988, 37791.97309902137, 35829.75033615041, 38656.897511148614, 76933.32232378685, 93864.64983596985, 16623.385255304278, 81539.57478421526, 30057.291826894405, 92390.13442342996, 75340.25196421973, 76921.9850148493, 61423.095004896255, 59653.59282921156, 94347.30006984623, 22737.362580800323, 53678.85765955009, 10301.708934020104, 38095.32684287682, 49616.92141333889, 93662.64160655116, 49043.20750519486, 27482.814871655504, 96106.19065794972, 24589.302294884717, 67955.1193349482, 63579.687408497535, 13927.412738861944, 49765.34264732913, 43319.34617558231, 68797.32252666926, 33569.08705342424, 34909.808153341946, 76570.96956576983, 81893.36395596314, 31150.039982173726, 6324.3371661235615, 8713.021667089104, 72182.21247960717, 41897.17040612363, 77860.97652149407, 73866.65127302856, 71452.01492691247, 18350.625629246864, 77302.63662965284, 12288.02354661992, 14398.759145813323, 33807.98752097499, 25766.855495166074, 15166.30332021952, 88495.71035344158, 74385.94977570757, 68674.54143359272, 18273.012555656842, 94818.2884573996, 52053.156545461396, 55932.42310409645, 98664.9507999063, 18069.804335574612, 69380.03646338664, 3317.6530866076305, 94337.12116088271, 75353.2125390193]} +{"id": 2017, "vector": [22769.095565620435, 70519.02122748039, 82667.23290175611, 13800.293456500478, 72134.90842204135, 65920.64056908265, 90564.80011394847, 34568.344752691526, 14057.891102270214, 92050.54010168614, 86265.82466264137, 55063.17110023605, 61074.54133335844, 65063.47031886914, 59873.8529528673, 42374.32635389533, 1824.269643318832, 22830.87652003869, 75850.31164304458, 60260.23033063255, 76975.2846206984, 89460.45859818028, 67492.0953163062, 98312.26901057591, 61050.81759298548, 50407.757139730245, 43967.76463380998, 71062.9342049337, 85828.375799274, 32848.16486839763, 12888.129875432385, 65249.028613679315, 21590.978001342286, 38499.733707687345, 36467.69491999415, 67933.77921049776, 4785.300217070454, 88468.14088396278, 80132.67794574038, 37842.919543649055, 95179.94754973578, 43482.07266197205, 91964.0914889828, 965.0494400322618, 78276.91337290002, 31286.38426107637, 90220.48105344722, 19425.922030774545, 884.461119934854, 52947.312158279005, 16841.91106340274, 55025.75732200047, 60883.77642615551, 92591.68250120549, 60691.76198194488, 41439.77227604918, 1717.891271553862, 20152.567501336904, 45097.696607019934, 13020.62533086179, 75540.34958117288, 31466.873365907555, 88249.9528830137, 66647.12937454338, 43085.11491141348, 47062.10912481702, 6000.7864953398275, 29474.459359657845, 34038.52765779646, 7075.187979665776, 84875.68268233107, 37535.241553835695, 34475.14095070025, 97687.55536803519, 8410.253401865286, 93626.9237843313, 82498.05031836004, 37278.95565607013, 63538.77347369148, 3236.3703591900994, 53476.11858742116, 75788.9105607336, 61474.20196592766, 33588.219094500026, 81409.28109005283, 87434.30532657908, 96815.06327208449, 49618.98354958737, 5087.964849466375, 77958.55542140166, 77442.5274077616, 3663.151042424695, 2100.8898075703496, 48312.650943678105, 99776.21471030066, 59477.16320124318, 15158.040054463261, 73563.65430360011, 64822.327748261036, 96373.30708408804, 80009.04489596398, 34577.593058459264, 76972.65997570247, 83068.91478303273, 85165.59967805288, 70155.81754943477, 36501.32117878129, 6162.6127881291295, 8956.396985143943, 13880.29458661687, 72209.65805564176, 79467.21273540866, 95105.4681977042, 55233.05855403212, 92237.38698505168, 81037.5534907802, 3453.340765632673, 23405.754184207726, 74434.93729036993, 59501.52475875552, 95705.74140604504, 37977.67043891081, 96618.3511600809, 50267.12073784201, 34953.539328201434, 7845.815473082429, 34326.84495923214, 86231.43407680132]} +{"id": 446, "vector": [40693.4740258999, 64413.54094670546, 87286.06226476195, 30845.412609478328, 37718.830876967724, 91671.2608780072, 55420.52657892951, 87069.68343034138, 18337.02402150841, 43429.041575573465, 17987.569709565167, 94572.99206533632, 52720.45080303172, 53423.35746427058, 86599.57587215787, 63045.03627102619, 43903.69171921056, 58587.569461114595, 63119.16366268218, 41029.95024330026, 89757.2580190791, 80094.16289125581, 78480.16349610496, 40275.01165671495, 20327.91046753013, 90217.05797604415, 69063.92135374337, 87003.45452370399, 58354.770986500924, 54062.52542322766, 88820.6856271614, 84455.69898009462, 83904.93707357424, 96403.10045747863, 56210.95615582082, 26110.90409289709, 9089.138588676715, 22406.502091334736, 13517.98038507841, 96883.94750872688, 58583.344310656496, 1124.1388454652167, 267.450928409374, 4666.361846132761, 71163.09931357802, 22850.90008044065, 7564.625140004433, 81698.70708926884, 2559.324791263129, 45819.52187370075, 22120.325288413344, 56295.543762339854, 16522.11688395351, 96182.03366170032, 74715.98383532315, 95487.49422201171, 12679.038472782233, 48904.0895259456, 77206.48737137161, 88028.96125366018, 47072.200492064876, 46056.371443370655, 43595.845288803604, 83862.12657921154, 20221.50115264554, 2594.4744134438147, 31864.260853333803, 65952.01864897626, 27587.898400421505, 36761.1961103672, 27833.788166003804, 79178.01007526164, 82362.25929651309, 41685.10125730325, 37954.175483834464, 12565.021175688074, 15291.50436160599, 85989.94604900171, 82984.86412653772, 35965.718320622684, 64268.388812196485, 38762.54261133226, 22242.32213337305, 4983.794635718708, 50300.04786856015, 5482.396604520767, 77425.39417903751, 9867.345109208414, 73783.20560350642, 41268.147904932404, 63653.58134367391, 62831.77501400396, 18956.621451369138, 82317.29313863775, 46443.40019638404, 70800.8283476375, 35577.32908979876, 38106.725011074406, 12517.45104682601, 97149.41424420533, 60995.229666851556, 62417.49839158122, 98016.78557019965, 16334.557705372632, 63736.34178765793, 99958.59730084013, 33229.79695806888, 51259.15024795188, 74249.61876797862, 7422.235996057247, 38095.619647933265, 48084.942612885316, 95172.27998494901, 95120.54829421865, 37436.265764005424, 59069.55825941663, 64363.65363472767, 74786.81422404971, 13155.049405755504, 62889.491470262416, 93015.102696649, 12296.618645917979, 70911.68823508435, 249.84531863802007, 43224.90869982636, 48714.86514046844, 80946.60232599163, 54119.40274704972]} +{"id": 1975, "vector": [18847.101547436087, 92333.06380946915, 71049.41110229996, 1450.334708909229, 29314.948161633114, 15537.672034317262, 85213.56534475728, 22587.586174139375, 66145.90776286746, 43414.79303805005, 21568.659872365115, 21185.172039386667, 85300.16630440809, 19095.038441849145, 6521.486047709901, 66.17508425119655, 860.6907185604529, 99715.20390959896, 61064.083688418614, 62498.52877164102, 69761.55321390464, 12868.457210968487, 82748.62096811502, 63874.249263764184, 52680.788817353576, 93525.06788976968, 54679.982956993736, 11842.883516867165, 53994.26816414957, 33941.05818414264, 2342.4803305565624, 93765.17737634164, 4900.479478452291, 58142.12857406003, 54394.75424882706, 529.4560539077397, 20303.45291545723, 58786.43562468934, 91677.92547660133, 58904.8876324221, 22412.808732002777, 82800.93233014371, 17865.468824897558, 75750.39444519783, 26087.674939123805, 38088.933520921186, 97153.88433211048, 4968.676379033632, 52591.22524723636, 15311.965073181067, 26548.421582945135, 98869.71767069891, 64792.37576939975, 75215.53097082651, 78989.46787815786, 74498.47967209181, 79281.44505283673, 29825.068089272965, 45072.881779707874, 89546.63571849129, 17332.969259583497, 46875.70330739186, 1071.913413857928, 69383.03917972522, 18381.614468083142, 26490.123172796175, 20172.86322835008, 68113.14566295546, 25629.538759668103, 51112.35607313671, 62376.1999266037, 69125.22365005921, 75420.94380342824, 10402.200198372624, 24219.414019000764, 9634.424665929642, 27017.072090184156, 81523.52406607267, 35963.4082257692, 54140.83163891692, 91320.32734296804, 18041.84543736338, 42165.73286017232, 3024.572124014724, 94166.48602371455, 97250.8287765826, 65190.33048322474, 22849.55459122461, 56095.097709418485, 96702.3959854342, 76033.96158805306, 11859.99708320682, 63859.77397197322, 45952.33844652914, 84300.79498298804, 50355.71704395685, 5444.296379917324, 95373.58671622128, 53850.15838057817, 32046.247276310547, 8498.079803644421, 82507.95436264935, 80648.52313968922, 76187.042469951, 96580.19819188473, 25969.231946180273, 83803.39009226233, 35228.39712697911, 12292.65674892881, 57104.071437177765, 59245.383185478015, 97948.81425820492, 88344.47581190546, 5002.757568906013, 15539.545270270282, 28776.352814630147, 46105.047291973344, 10568.79057854555, 93213.8805263092, 44512.89976007165, 23459.405544347544, 56081.721120785944, 78259.23214584161, 53893.16596719516, 86982.1976428858, 74148.66909762916, 40578.28982703656, 6844.083219936503]} +{"id": 366, "vector": [38615.06499000679, 38890.866092873, 78906.42139674217, 75183.63297075413, 72038.00823284687, 87813.00855423551, 19261.750351224706, 22280.636031192247, 57193.928211727296, 23434.42083937277, 13895.635395148198, 47657.411545266084, 53959.177261985904, 20419.231934083225, 37700.56942047595, 88464.31760045276, 87944.52238438049, 9323.635473746184, 19698.00731002669, 23452.518859144424, 1323.0888756038617, 78900.43676728284, 70784.01567479195, 14786.016297386062, 59301.49320156025, 73700.50028040842, 76828.81683355123, 79918.60598115528, 78267.33975400889, 62044.43498978275, 18304.101907427415, 50982.3319872587, 78913.38631487255, 19819.09835486978, 40988.019027236514, 48545.76690343401, 55383.75465929094, 40357.39115548446, 26344.384507726427, 84899.65320869618, 1299.9074606382944, 62288.252684736835, 12172.088796599135, 39023.424009856375, 49361.14606106921, 30323.1446104511, 49587.00921071844, 34166.94474843904, 10888.220212393362, 65777.68732658621, 16473.602419413946, 22811.57145388337, 56926.24131661789, 9404.596685352817, 28950.835473818493, 46128.10178905059, 58914.64726082232, 61639.214498415226, 22302.577013901093, 66462.46658116649, 42997.1140451212, 43807.505618669085, 9115.921823026807, 55150.59194659933, 47186.97298166127, 90933.24316181272, 77649.59671495466, 65932.98012877481, 8842.268485634242, 87452.72708653385, 66938.47984250302, 53303.7002006022, 68671.07767142775, 34300.76634247828, 10482.602943407594, 94287.63124983694, 39947.58690853023, 5350.534144398466, 66230.44555254004, 30246.25945853838, 94842.49092118777, 61672.5588813854, 71333.72411177878, 25395.402375711074, 29604.35873591446, 92171.88781834395, 77611.2424504206, 85838.7934100446, 12115.714005562295, 40438.26683882741, 32007.835043908162, 34371.27100968097, 26846.66559527955, 29128.06331568758, 6647.798655696913, 79976.06408049433, 72734.11103768322, 87368.373516197, 68207.64095296011, 69716.28219133828, 53028.86586489257, 50529.848141378134, 12903.206904517618, 93779.28719137536, 58447.405034147036, 20332.13755311165, 85572.30385369326, 34882.70723165276, 774.4895885519965, 40757.713807177795, 74968.24520596552, 42471.74074524156, 43275.509605948806, 82671.5106279027, 76801.933955662, 40636.97450151953, 11800.778388797284, 84178.8413299872, 2795.0943885879487, 3580.492616852382, 74765.27872727637, 78644.20416369294, 81553.35098174482, 70031.27402977562, 34912.58556713134, 29040.15767728949, 75103.49084127916, 41986.06398198275]} +{"id": 1769, "vector": [96205.9506378956, 35272.366126621724, 30913.821281981236, 67769.29232140032, 66372.54174407027, 9540.190587241692, 19885.768277773797, 20909.99978910023, 62478.5401751665, 74998.6528565879, 69795.9353984442, 94401.1014997239, 33060.7690431776, 28373.970388608406, 15323.400684993061, 14747.516765039725, 39046.77940960623, 65404.12527391901, 44720.838621867, 10140.055794854996, 99034.94908838168, 69615.66569579227, 68087.95411807761, 7666.114146221814, 39438.145745583795, 36350.05227230194, 22033.664500501116, 94444.33114705433, 42354.500300163774, 12966.290953759863, 86966.38105332317, 89363.65151729174, 74917.65403945137, 84858.61784706301, 40128.64977214876, 39206.435234920544, 52325.24701554845, 68846.20502214773, 75910.89901393467, 92135.63110622647, 17003.541869179684, 93818.06495320452, 66165.0431437457, 14832.837783482944, 57093.25982524991, 15322.062747342403, 79623.58600531584, 35263.865007728135, 96276.59122572264, 77225.12681545038, 31264.89949188144, 91056.30567313622, 3518.261312096993, 43265.732206100714, 74636.24817872187, 70790.59341434677, 84251.79024817148, 9262.700382633993, 66205.8063531496, 22885.508867611705, 4811.17254812129, 63793.051867542716, 33676.964950653906, 48616.91523784662, 95117.97621508881, 93003.12658042376, 97323.69140597488, 37956.753636817986, 2771.8799626842315, 81062.81765595792, 60633.08477428846, 35611.48489729061, 66444.55701961294, 87011.722044628, 50483.85974986123, 38632.14905036441, 11620.234578415268, 26498.781888963596, 63058.468600923756, 42414.31211426, 7583.381200355743, 3952.2006178984493, 89947.16561920318, 43288.165966630586, 9620.516869243567, 84186.27983223788, 59550.06454447451, 20701.788927323327, 46822.70414757213, 33260.34828699682, 11906.522742935476, 19303.62857970488, 30212.44143984926, 52450.515598716695, 65473.42209643576, 94848.94067152943, 28168.631749575845, 85999.81743351842, 1842.4867622576558, 63694.51375819792, 40560.12119993937, 30307.767634112413, 57474.56017223114, 53003.15180431876, 99832.54772209834, 68033.26140760143, 98954.61553484648, 80813.15679846407, 43354.254999851175, 28368.467965669795, 52769.40231275174, 30499.967303325793, 7938.743542327986, 13874.921547598995, 48661.556226625195, 95455.97010289879, 41485.95932002591, 98803.07840085833, 32498.471909979253, 76702.25518228477, 40694.90039391327, 19167.321816244, 13364.371283335153, 59814.29034001822, 733.3659845924601, 86616.44199636189, 59269.22248820318, 89131.13697702005]} +{"id": 1382, "vector": [75958.96834334605, 85238.8802814016, 95997.61995184545, 88759.3496319677, 31805.88103299814, 19834.623314254895, 67427.7857466505, 43374.53055087536, 39879.6290202524, 44876.78004975336, 95830.00545322629, 15737.764526232811, 51977.041509421986, 98957.71459587608, 36002.46491351169, 76413.44423227271, 50894.13565489191, 56841.46899120087, 86063.06977103988, 35626.158325326476, 26435.28840202657, 3124.244382004604, 42121.82050263216, 98225.3921737415, 87686.7116891206, 2175.683442095355, 45878.62723115337, 70338.36147633508, 14742.907097112611, 77226.1646916585, 57520.214371535214, 40010.67049789343, 23031.76824363904, 74636.05737985627, 18875.038151657518, 33374.82230986335, 62649.83125610272, 80069.03011198642, 92470.16310285621, 37519.02955080661, 33016.096847709654, 91167.14293203912, 20791.165120058264, 31433.70406859197, 98617.08015618146, 81106.22630876866, 52586.70147180597, 12320.382996077527, 68837.85376703192, 12403.8710421991, 50768.69895225346, 81850.28974399343, 13269.675332501152, 89830.48374672935, 42491.74871575895, 29693.923799437573, 41951.81269275784, 41307.48209297358, 97132.2844875329, 6384.32918262215, 65806.53247597608, 89372.1311695988, 85697.98205205648, 72426.39940587149, 63293.70367277708, 85672.24465768624, 96587.1235872273, 31054.187020717472, 63170.65379308719, 75673.1820802312, 57881.19321682645, 50849.0989360314, 74567.82203791446, 81557.79903611752, 65272.21963235289, 54538.65083011622, 79408.16087686935, 25776.415438522883, 51153.39581347137, 8940.490651359145, 90181.24310480674, 6362.845679762541, 46946.09040670329, 98540.64026776653, 71002.84974047722, 25730.82545509143, 8636.518014602412, 92617.23491928756, 17039.144492733405, 44001.387590269704, 91136.70697187551, 79202.43136304019, 28108.678941433274, 24910.350300851856, 57620.30251740295, 48093.2067101573, 25896.68593983184, 75968.80805602897, 33413.703722214785, 4387.138553454506, 61417.22226438367, 23476.90525451407, 74698.56054234203, 9502.404628539396, 60383.98023827796, 61677.321495262615, 19012.233006739578, 43161.423252536166, 34099.60597275828, 29324.00775776064, 61695.13167964518, 10713.356593641165, 644.2185313374594, 60020.489664804656, 93630.94508593151, 28594.83353821797, 62539.43605528215, 8865.319182412346, 17554.95075614889, 18339.53042774353, 48238.52336785301, 54832.85813596479, 29416.006162381735, 19371.13652180743, 34309.338957610926, 36874.00904792166, 32650.30346301848, 13435.014452219619]} +{"id": 707, "vector": [3959.9050845367524, 68542.05787616559, 5734.836584508141, 24543.684416587075, 75784.84445160606, 69074.24916998872, 82928.4739713123, 84328.22522033082, 32329.98940926213, 45129.99184645958, 65719.87862174465, 70493.43252439774, 24648.4784187364, 39767.74526435744, 19070.310762425936, 37445.650143574705, 33471.471652302695, 45460.321914540946, 71087.60851600845, 78278.69586649018, 79482.80681769122, 8134.162143337619, 16318.82593135997, 21936.992042734506, 50028.820823888156, 22010.77050057395, 77857.65660052709, 81928.26462124268, 15187.969016443658, 6955.926537492652, 84990.45838060537, 31804.38746655987, 9488.083145257177, 68064.93126754006, 13790.882009938921, 15624.646574217826, 98356.56945983245, 5655.4098647375595, 84480.39281065916, 98618.16201942296, 37804.59386077115, 12231.935162817475, 6905.938628670016, 53729.40102150853, 79827.01025180396, 56188.91314277126, 18202.703097225458, 66377.34312283312, 72519.45443139765, 4277.327647152151, 4606.50980670545, 49402.80758761667, 38867.468181569595, 68594.1328573987, 62555.13340055448, 43516.67332469364, 69871.33643821185, 5713.251037405332, 98034.07841349544, 93741.372348964, 62496.10431737394, 39943.97337960216, 94006.5482970248, 5910.3507488981295, 82409.64517770431, 10084.380480241172, 24870.486023444148, 84808.15054346458, 93825.48266012255, 50821.22259455177, 76668.92941187976, 5361.626134322839, 75918.61690556364, 73410.8581636128, 52952.154808320054, 96373.5296308244, 77240.08154508017, 23890.693965639686, 48872.942485830004, 55010.47243088042, 21623.608991503384, 82771.49609307555, 49033.817269114574, 97571.13024002264, 51227.36469776128, 31824.569826568706, 99442.78078206578, 46178.66917190121, 79677.536041179, 11639.978073278457, 93169.79012676711, 91543.5663535052, 4027.8036795725834, 81263.74299282204, 8238.586796466685, 59130.7050140706, 99430.40722381735, 57446.511203182505, 50737.10790293931, 29833.27545750717, 34067.708208507785, 84533.45681583317, 42223.21687964944, 39930.88556738713, 20141.443193854204, 1843.177306466681, 184.32302954949353, 39751.269096291544, 48055.692563406505, 85954.91728046475, 71324.51596000696, 32470.198898473478, 25288.894683157938, 47051.56450578377, 27127.787659557955, 8715.630423486342, 74854.41176778162, 68911.28366994401, 31866.40878896072, 36772.870242924524, 38682.96838762634, 68616.14598741957, 48491.52076191117, 36976.855137507395, 53214.4920654146, 94235.72461168798, 33535.81283960978, 32070.477939392516]} +{"id": 1258, "vector": [87086.43507458369, 81597.67588041075, 55028.30855257757, 11676.753087606317, 25721.53832013273, 86876.00379534454, 40055.0754079298, 6100.334418154851, 35796.14426000641, 56504.70706065573, 71065.54090788332, 10196.786601062879, 22968.379333998135, 17826.371564614696, 11179.387178014744, 4630.909124179894, 49231.23596191571, 91721.99969359103, 27927.7138950081, 29092.328002634928, 77417.58476989144, 1429.365580525932, 12087.167628728168, 71450.1069806403, 42697.84821979207, 12890.821049791357, 13179.875693820231, 96984.95917090541, 64878.867140670736, 6079.206102159817, 44820.15844017499, 4228.242081342825, 41114.28430947923, 71578.44147326135, 18499.994316394408, 83824.89833589141, 48279.74272560498, 36561.772486290014, 23702.82446027212, 72905.69071229537, 16214.155723538282, 42093.74304539389, 67487.30001856165, 66042.35018832903, 66027.36736326411, 82167.9603342497, 61933.928989587184, 86070.70652725936, 26264.882947215705, 37992.57883401159, 15600.697432248024, 22917.12544018408, 54778.58502087672, 62421.52298061775, 25297.6617425549, 10771.77893326533, 52538.381383825574, 45618.85553469108, 73029.27937411865, 55412.71993388547, 92085.07291893529, 43180.92003780675, 27437.637016184523, 64957.19957572711, 80060.15856698701, 16416.641268715215, 40402.02673818368, 26657.94549637116, 37202.56510319526, 81980.84951472975, 65229.79410986651, 65708.39661190205, 10312.622186238606, 14358.765023774766, 58764.209047260716, 68891.04082618048, 20807.23147875948, 14261.291392372166, 61305.20610964443, 26428.90031255891, 58099.60343817475, 20759.109816420085, 44859.997089518445, 88631.53099303192, 12930.549391451195, 80025.51353950564, 17913.066058122273, 45120.294014350045, 98008.38568082613, 32441.091462436878, 41683.89147452896, 90952.62733144096, 85393.36679299247, 39384.504754617534, 64323.51123870064, 35620.83737851627, 31340.399137559725, 39224.709258101175, 11067.925647600285, 73323.0761057578, 41393.07797852757, 78563.20571178754, 64054.83817227997, 67926.86687254348, 98892.69080368105, 9703.208741375247, 99050.21719932217, 29878.101447983587, 28863.015409658354, 99234.63154863556, 16215.48109133516, 41076.302411853896, 47412.92812545488, 61689.7897483702, 42624.986991426114, 1266.4270442324987, 94422.93952784163, 61801.524907741856, 7491.509684269826, 53758.10299640128, 82100.42407366156, 61030.797393270885, 69829.0635586362, 73600.16351086732, 44177.38360589605, 34192.54852341801, 61360.44949528387, 98407.55748260194]} +{"id": 840, "vector": [8611.630420819583, 85859.84959745895, 36094.15381868419, 21462.036794092288, 18730.465121229056, 5002.442649527894, 63339.43293204817, 96811.36444560293, 25184.780930927263, 18409.822911849984, 61019.78745750205, 5706.541257239072, 19825.872669200096, 10162.388192053506, 38915.76797920179, 94729.02095576783, 86776.53837080403, 51066.261048005836, 24831.88789069941, 69765.6229516329, 53110.11280503109, 86342.66794109215, 44634.08313070437, 31054.368080243632, 20219.00755865621, 24005.971516569145, 77500.87850688827, 73268.80125262051, 72462.37816418998, 50389.139894458065, 11461.752804608994, 75265.30243447232, 7642.996950314873, 68781.9459339569, 59610.35805826287, 85326.63524935837, 94748.79528394472, 90269.68257638824, 14827.785267605708, 54403.60401466623, 39007.51623527091, 95738.06587246215, 2479.8636364020576, 24393.231434160843, 70952.89928230015, 91195.2330824539, 38963.915535191096, 47020.61404489101, 80121.84512820035, 75689.13957382194, 55679.454524512315, 43673.88056826286, 19830.32981751086, 30938.222733997354, 59544.5852521597, 76157.92377544027, 59623.22767549761, 46099.52963257128, 12152.3234152492, 57957.068182477444, 99533.35257252956, 7485.296375544648, 97985.45572558214, 95762.1618110004, 33511.03717361955, 10685.722327064585, 75760.95724496289, 36635.78225319881, 34563.675673836646, 85877.35364226863, 61115.08859017181, 68424.59282821229, 80269.22573639892, 54227.042766682396, 48012.85594292568, 70954.5872336754, 4940.23524995737, 78433.7202456647, 1866.8630322326574, 59189.8889138756, 64435.58138086175, 57080.25832727284, 8989.649677717105, 25383.718200921136, 28989.776016606917, 37438.10732445178, 96029.42854985001, 95539.6432322889, 18780.733592413257, 11531.14492709113, 50533.15918361964, 18747.9448573876, 70134.69268097568, 7094.7751351424395, 62427.53051676349, 44077.797401677606, 69421.30092919269, 3246.611811580924, 96004.43176121586, 53206.669599616405, 80302.43148486069, 19449.80089536845, 47483.149626550694, 75303.18623776038, 88748.73049164453, 36364.516261109915, 16170.216865387587, 86049.51355924534, 92817.48349346334, 68915.75071632907, 85277.56670320765, 97536.6576034036, 55784.75517410182, 16539.98642327418, 67793.36051403795, 70316.96427788676, 49919.20123480652, 26563.966164662455, 58579.58581325103, 58347.37521307689, 67386.74749598444, 90170.30206457125, 40522.04748419611, 12370.7554878576, 48961.14780490526, 96206.57907187134, 74931.9872552398, 69112.33273866418]} +{"id": 471, "vector": [38847.81046114183, 43947.027396488316, 47638.456107521044, 3652.2568598466255, 10963.176436984568, 48105.231777773086, 73550.31010338952, 22662.856444232217, 3198.7063378023818, 50739.89919386681, 30659.590327844755, 5555.823847548269, 77893.97169883698, 7937.264878411254, 93676.03035788024, 16929.009308899946, 33494.754404879764, 47388.9773092278, 13018.308697307635, 13082.225604735375, 72071.27756936819, 56876.467216192694, 43054.46328570948, 91262.62689219705, 15197.492610325835, 42301.01893618414, 14886.589896523217, 50904.729546152135, 26880.793791820444, 19099.87856259373, 88298.00136995468, 28184.254227212423, 38807.33735160831, 35416.72398958971, 10848.67036041326, 48317.15575361845, 79524.73308615698, 93891.30390381982, 96587.99105144516, 31032.282942479327, 8277.939588983285, 36320.00790965534, 34257.52110610623, 93846.6424001844, 21142.485920912568, 88500.23633897584, 40814.40963894707, 77125.41249189475, 11868.916372949845, 97706.17183269748, 46580.07598414871, 75338.36925945213, 81853.52155727791, 60033.472969093404, 24976.581170825397, 78544.64638153175, 71698.13794286373, 6190.670562647616, 89727.4148723151, 74084.62273884023, 71493.44025620096, 29270.60948916105, 90881.74412330474, 93687.85073367985, 88946.85537917691, 11284.808181159156, 32255.845571596776, 141.94332211580064, 39844.667223174634, 46207.90491528658, 1803.8775736880398, 22573.022299463053, 52362.515831142155, 36155.42674252726, 92212.92450925893, 99135.33957221016, 11173.510944692554, 39596.58926496993, 667.8547569331395, 92451.6886818402, 72090.54696260147, 57742.368750294285, 35005.817625441734, 92282.68818499043, 56883.808786284506, 37051.56811209446, 34968.772831967966, 88055.97885470971, 52487.450223685584, 74917.44594200568, 53607.6010275898, 83289.66882039538, 76435.2483113248, 16492.712165624747, 74631.59340701844, 85569.8110584413, 1784.4291600430217, 30915.6797758677, 72507.83577256021, 3581.515672589086, 39669.200524608306, 4499.707240430428, 60957.12393572773, 83051.88868752177, 68010.01078858413, 29165.33110103299, 31176.071982676913, 3866.51869816339, 7171.926284172759, 91236.15334287677, 26266.371782884125, 25018.09474352392, 40889.79610999321, 28248.782950253724, 23501.85465782487, 58960.74715818043, 90864.81180991724, 73651.42947429542, 67773.02127459095, 57286.17415245676, 30710.16706049391, 35260.62735695292, 8335.35184660189, 10576.729513239292, 4514.278151051354, 72687.96131194239, 73808.54993060845, 48655.63632535333]} +{"id": 1685, "vector": [60711.99448312531, 75314.29768165172, 83275.42482901207, 49872.241014622385, 33968.6271103949, 55191.205399449216, 71780.54733412844, 56731.89533712404, 50777.18777225509, 35978.8728406868, 92215.0006661362, 55256.9277959139, 91234.62572608687, 30860.04803988446, 12357.795460456866, 68990.98971995812, 18839.398794145745, 78563.55605162872, 1234.5780997610411, 43934.70305977017, 11609.27402479004, 542.2691081011233, 86989.89636979328, 50702.905387921295, 23931.77366188638, 60577.900737242366, 84536.36649279513, 11797.288565249486, 30564.38972058931, 45674.09200696475, 29262.307751310367, 43966.495871207764, 6304.9890766286535, 91794.67996706565, 34231.18814764903, 27727.25519580185, 17635.61232049221, 32632.50728118642, 11442.843986069673, 67832.08999549353, 47083.12535715829, 46134.43586778474, 53588.43349998216, 59927.539748881456, 10498.325037120992, 79368.9892079895, 27576.397213863645, 27902.202420985002, 52335.38863505572, 67630.5702735073, 50901.44384706299, 49642.36152959961, 73171.26559089507, 19453.634976037003, 47421.98799376448, 34780.285471608855, 47208.86155421357, 84807.18606620519, 77928.23015431814, 37225.41298024516, 68129.0855727413, 1981.283019962632, 51920.072286054594, 79453.16075067197, 67081.69711045406, 86041.16685189967, 13725.605659072182, 88251.77189676961, 57405.36169536139, 67.55916645745374, 5139.326087633977, 91282.24473044818, 83393.22914205493, 45462.43278413364, 70333.34081798201, 67538.61164806764, 52809.483276297484, 63904.84258256279, 99010.77337739052, 51815.27934645294, 53001.40756189936, 49282.73925421794, 3882.8609571042725, 87879.97870333274, 83673.37643469102, 16938.73308708269, 83309.08678173323, 66869.96755312644, 29182.88004593911, 28606.30468317179, 41010.99993637656, 94365.53766448458, 6011.656450453129, 38904.07565177324, 7903.3993467489845, 31708.067882701864, 16434.1172084738, 72235.20632317917, 51132.64898248887, 84656.55532274743, 80415.88432865798, 55376.09283400516, 61093.97388144035, 1787.1147125444686, 28808.959671628465, 26395.533593250497, 6052.446620250617, 78217.00564336163, 67320.76107822343, 40804.53615323947, 14004.389633863668, 43394.06249702128, 7147.119321472839, 77617.65954320387, 10699.379321602342, 10986.781294440018, 67448.37768776961, 22619.605338445715, 42973.4044010959, 17635.96733470758, 68917.24795549942, 88014.3644985796, 97896.6863360741, 1161.514248721751, 56564.06412025485, 79143.69724674041, 20641.827455813243, 4649.28528119476]} +{"id": 230, "vector": [88178.78511940745, 40686.73862466018, 24269.030554483885, 13525.339656086566, 86590.18441453524, 91876.30506918536, 19713.30048716139, 83937.77542817591, 3358.988894383108, 68443.30967825197, 87138.1156757196, 66385.77529134607, 74166.09430504634, 85804.33653220595, 76879.9334614865, 73541.3822316036, 74959.20971293004, 95946.08212745748, 69537.01151513522, 62284.02491687492, 94218.7122260168, 83442.39291537843, 85572.60518957501, 89438.54092673787, 42210.969210882766, 37712.52448376898, 73721.46789276919, 9558.858535110492, 58019.78426233089, 94632.7561510139, 5949.006596165862, 56163.34847716945, 2187.8754822851843, 19083.5616320573, 81055.72594785473, 44744.52321188237, 31746.164133947586, 13444.422060295146, 91920.4630033545, 73694.64545789221, 61512.74398399709, 58805.75485995364, 21742.065758907203, 38679.07872454562, 53265.43358918002, 88365.36978287123, 59691.24410645337, 45509.823034402885, 71270.94205762778, 94680.16397140833, 62675.966850476216, 97746.42149558778, 47515.38717159136, 76427.81073945427, 87757.6112339916, 86046.36136435869, 42993.09877914481, 30473.40378686395, 13362.059519284785, 47772.33485763058, 4338.489160648973, 68783.33852856183, 91524.64535290247, 26642.502590470973, 48721.201519038725, 85380.45152246827, 9230.744747764486, 30778.41386829122, 57307.24629383364, 13756.456621173007, 81566.4414087624, 49359.66300046065, 87255.95360657382, 16046.163111687083, 75691.35062627896, 17310.3401776161, 74089.6958654791, 85861.06519756284, 55866.98449959572, 68800.17366668217, 57532.892861696906, 12314.121328116868, 58804.359400065565, 14353.505112553732, 20825.124335934852, 68110.81892820125, 26285.14680960148, 26105.819766846704, 42195.010723745865, 28302.611214429606, 22122.832831033513, 72534.0272643302, 78064.74439727243, 92703.36157576652, 60690.36970334588, 29949.004756557384, 28864.72940023771, 95378.05936560345, 36007.11608751809, 17962.596730005764, 50911.341756990056, 40150.719466915056, 28481.72490827825, 32336.636319530222, 87194.96180474332, 33864.088542826175, 51052.48677859718, 70405.53693869256, 66309.6737971424, 18739.608147336683, 69593.11146776834, 97380.31573911053, 32349.867290452992, 41331.26448586376, 14088.073024171232, 91672.2018935315, 61353.52516345265, 85408.35629776772, 96118.93113288605, 92561.30076269877, 83844.61998461976, 577.5208456638259, 3947.012696178831, 64704.88972335628, 67541.7817315591, 54653.12141558527, 61597.37693435619, 25517.86385142256]} +{"id": 1192, "vector": [70533.23638534665, 43424.54102202519, 34959.89845010815, 95969.67721878127, 24784.989271191283, 40817.066023970285, 83699.5763470948, 30999.64261410124, 79989.53766601885, 92333.04570642213, 99351.05974361364, 53214.67802490205, 25856.229399805074, 99012.26672126008, 22619.896030246866, 88484.10605888999, 69496.75987015225, 39322.09364756537, 22351.717402521586, 38828.38195324932, 77331.57761681419, 15117.646038316434, 90621.90960640582, 47417.308468442665, 68517.4988929869, 35332.33381054253, 84405.63474614921, 11197.136913814344, 88093.30737907936, 78055.64612134364, 28340.56344486047, 98158.93843518286, 95564.22884995004, 36685.487344839596, 44254.13773496313, 44882.787080269016, 17351.857088730427, 39258.866291103834, 79185.74497504279, 57118.66153597742, 10065.79343572469, 57436.60090655078, 52574.087184705444, 23729.842982274462, 61585.14418816408, 78154.16221804076, 82437.25633479648, 36282.360060870924, 37151.23292452147, 91909.12375209614, 9910.044143169584, 9756.01305995576, 25194.995606209093, 88962.5432042116, 12290.714864705054, 59376.8795153513, 22659.507089128696, 56858.484217851765, 37943.29653760801, 88037.0650318804, 64930.39367522349, 75985.20678800998, 35570.14861882102, 58231.21442946946, 72358.90786461842, 96604.03694785049, 15227.201668142465, 12272.316702727638, 26796.877020024913, 80929.91156538311, 96812.85151060949, 85374.04254096505, 25967.429881194937, 52151.76310395142, 11407.962160945484, 61380.93819546206, 51969.7578222461, 98790.65582180089, 40241.176488430494, 12441.12211449041, 8781.70045275296, 85021.76941798431, 53291.74922573066, 15646.578533209831, 99956.99402539563, 90102.32530484877, 74491.26113065981, 88640.85466639971, 25190.20427729677, 43752.14657322345, 70148.08592545766, 56552.035608565246, 88500.33822969935, 32996.774850499445, 68678.90056858756, 49145.1385498281, 45381.360309347205, 17961.876463724846, 89444.26576888685, 1033.6748939481, 10161.79446713853, 6539.321618044458, 32126.244781483725, 80736.09778976813, 96156.36864214907, 59993.23309819561, 55668.7878618726, 77628.95673504453, 54336.29647531692, 49704.729383237056, 13797.198706699986, 78442.88830839348, 48634.35869523525, 89823.50349762329, 46832.95520153794, 11847.893619495742, 19670.9686809309, 15365.954669331593, 84442.34031178718, 48065.31591562211, 87038.93103326822, 34709.9424406745, 70390.00704644185, 87487.96915934546, 69922.49641426456, 33473.6303698025, 84170.45386027408, 87566.24376080485]} +{"id": 234, "vector": [6994.278668658038, 79312.71370307359, 48516.308941651565, 39919.08069838058, 12720.476641440358, 37609.60282489704, 75939.20757387331, 31916.85098620375, 62679.63547294744, 85089.62561855404, 96962.13494868956, 81620.11010713359, 87143.68353892144, 56938.527498024094, 32722.938444730065, 52451.94562482913, 10824.399268861629, 99272.21652677335, 7827.728409401313, 89852.72266003251, 17597.81784547245, 30496.705913755017, 21455.11383588473, 96789.49401375752, 35032.947899692845, 78857.185390151, 83256.29750237714, 90443.71420826764, 24312.84640622776, 85692.37278749324, 71784.2063971462, 55699.92545973272, 21120.501549820492, 50713.734858973694, 68903.89932724333, 45494.34338595153, 47353.96063464357, 62054.59399276262, 57711.33867334736, 67446.59579314271, 81497.06157886924, 56145.773904730944, 76083.17242581662, 52842.39145412669, 18681.205823019554, 47905.04331454142, 90319.85302975749, 85360.81039908422, 76458.40976879232, 49417.06343463181, 35820.665753686066, 71545.62789435347, 60415.636554394805, 4573.347410538709, 36795.519268190925, 16158.171330565174, 35722.55898355259, 66864.63208180007, 26682.445254849285, 90164.40608646587, 81030.6793037463, 46228.670390279905, 59964.08343231924, 59649.88817326936, 87703.87713903758, 29294.941303708954, 57355.576587230185, 70358.46358320372, 74880.8953929873, 59199.64996802104, 1255.8416110559967, 20237.25992722496, 29328.82561622123, 53623.79626553097, 273.96425025052463, 74137.25358175652, 28288.757594226256, 68185.9990076889, 32347.749066175336, 87520.01900898894, 60529.946965650226, 30736.1459354194, 70728.79776407339, 48648.118026827105, 34895.62492956839, 34145.693407186176, 16319.736690374364, 45700.12742476495, 13259.84340653944, 30649.45035365786, 4524.699557308787, 96989.14057819989, 16479.900303738847, 19202.513274221554, 35316.69523578342, 13415.920196697207, 97457.66524483445, 14096.108761060288, 77304.37138487848, 8213.093164324748, 88909.85224739123, 39663.20431738032, 4034.019640181985, 50761.5355121051, 33934.81915006188, 99416.68904350951, 27120.19958469485, 93010.30082138765, 57699.74122469966, 65516.15531815615, 46435.12884280061, 67509.96341137211, 42775.39370081608, 19253.527742327493, 66488.79761676372, 4874.413818216283, 48113.52151591565, 60703.95145139744, 64282.87642004949, 47328.00620844718, 56931.42311603862, 89206.20414716277, 44660.7756590599, 52469.42025475588, 33112.030625716114, 16569.676584058678, 99693.57105745754, 61161.21869561432]} +{"id": 1706, "vector": [66015.16936156868, 72062.47059592835, 38935.457937288134, 64043.123469491184, 61426.38428123264, 17983.167315627168, 28780.128574730836, 6653.30596208199, 59224.81742990982, 78126.85157121469, 61885.71684648627, 14333.864365308846, 565.3835843224653, 50894.19847314727, 93492.61022362404, 86810.05144898422, 40699.44184049899, 52588.640982377125, 88191.04623293025, 82730.6040882469, 24608.24544426652, 27472.73614148771, 78842.67874928174, 70452.69628320812, 17141.535123180653, 60776.36139716756, 75739.25178461087, 66842.53224341164, 51316.65096035932, 44672.92197072974, 18179.30027383128, 57354.07628963278, 83104.78114416818, 94244.78779799736, 21893.177524709896, 41106.76500925574, 89459.78475630809, 91047.43379166993, 90514.96026843511, 70523.74837299206, 6075.378582153879, 22060.830298472956, 51675.00654869205, 91579.61443315735, 23514.90775313767, 48017.59372934768, 81228.64938784417, 9771.43462980996, 18458.762850758903, 10432.320550708451, 30888.65822094029, 50067.50889258449, 26760.154908898916, 79055.93638883406, 92918.18695321717, 36576.352513910206, 35189.279102713524, 10713.357945552294, 56392.18249460759, 93426.05783361809, 88995.42548846114, 22864.495447989473, 79807.23034408693, 68374.70485968063, 25777.603987271814, 15999.311934065763, 19687.341512350355, 84105.42540271748, 13204.170970657015, 31581.28340679811, 19430.875355627795, 85270.11867120636, 71401.77374622563, 72172.11599664326, 97818.84831370862, 34952.963981351495, 68026.12048195729, 19353.66441948968, 9913.314869104628, 93726.9117740856, 89721.66770737093, 81649.32900273666, 77802.07992014935, 24266.267140936892, 15938.931773787246, 95860.9970989503, 19492.365873390794, 48859.58554217512, 68707.10194219851, 58969.20200577419, 56580.07330716188, 11808.755444781893, 3766.986667336458, 36058.05918073149, 98612.50405667436, 39883.36498202372, 36578.405138985836, 85774.12650149985, 55505.51998352181, 13066.9894091893, 79046.89111473065, 15072.40315375743, 79058.65275285163, 15789.72935276074, 8551.043615464681, 93737.33084222206, 47861.47541478216, 76707.33708355331, 92454.10151331007, 62980.733169592066, 21054.07878327784, 36014.95143743607, 40653.92178907963, 18544.812283587155, 15885.300425193216, 99699.98444255748, 65415.22861201336, 96177.05369479474, 36323.475314955365, 91088.14498375978, 99355.5506172402, 23656.561495984162, 79471.02088630442, 47351.09220049154, 93407.93416516097, 68731.2488113207, 60942.4933448368, 46708.77795493615]} +{"id": 1262, "vector": [87950.99011779366, 72403.60652863765, 15378.761313076016, 62869.946881539596, 66965.6669859367, 68338.33155065497, 64831.130584651, 93025.40362233086, 88285.84828678962, 29023.26219480249, 22631.017277360686, 18807.980842198336, 10271.33778707976, 92218.05994950031, 48896.214900199244, 55169.42027643936, 25756.02814041267, 16602.343266831733, 11531.343386820381, 49361.0543881557, 57652.767654239855, 9136.39226156634, 48635.24623737607, 58957.984724439615, 77191.47251300068, 15291.42222524994, 16489.02152120445, 88005.10993033033, 13557.566038374347, 52502.35442679413, 74806.64968092568, 25605.85459702056, 24684.051668310192, 86673.9668208857, 3582.2076228957058, 17501.093405401567, 63218.33886296243, 68818.0725457964, 70596.60404169241, 11042.752951474933, 54048.21749658721, 54512.77331904894, 91099.32624994725, 42594.79785138266, 62307.5844405515, 89393.9295402833, 56628.068622647945, 51527.91443361425, 93225.94521395199, 35464.08911264652, 44271.87810508161, 81788.5192638244, 36591.71232002311, 14244.823847830057, 57395.96966872226, 36074.96315202521, 50240.420107595084, 21209.70794656031, 13155.009887572034, 79604.00118054044, 14988.873638936173, 18324.880581056725, 47025.80270628096, 31806.745851774453, 89340.66217284692, 94508.65850410251, 65005.468875810555, 77494.16473819515, 39226.012585632365, 1527.586119106228, 71069.76041428339, 2782.601017721531, 53209.419155249205, 64627.41549528988, 81891.64710855711, 3121.4647434826047, 6041.524270480003, 61698.88265762179, 91837.93266340031, 97450.4435263995, 49010.15215367346, 91558.1016807754, 32584.439409044742, 77420.31735295446, 96254.09463952704, 65205.99312012755, 94147.25676609085, 522.8157144386403, 47029.36462424597, 98409.35076132957, 94081.2991155374, 4043.579025158572, 41543.523929022784, 31807.377390408576, 58497.82979804767, 38679.77564106922, 98197.64133646814, 45871.660977364714, 31985.331797653595, 33738.53078011757, 25143.166826135344, 67637.30998410431, 91284.78395696389, 6304.838274104895, 2415.6848642340688, 44397.332974428275, 51780.17675068376, 4596.046542525855, 62420.69724688288, 43496.19230063041, 93709.58103497067, 43163.14577496314, 2538.5990143208414, 66524.75662532578, 41274.810069778054, 6561.483669929702, 84360.35934169582, 99970.90093049817, 10349.176598586419, 832.5278742928322, 4490.760755242718, 54730.98402082566, 11120.823159584192, 34138.73222191816, 15703.825220525436, 11795.815756938644, 18135.975734534517, 76583.40179546257]} +{"id": 896, "vector": [59450.86927151437, 70274.9405519863, 5134.866115303216, 40447.08255054337, 91108.25656534638, 38550.997838687814, 43138.79516198515, 62879.99033843883, 84386.97364655283, 83610.359162238, 85005.89598515136, 69653.3664122932, 42530.70073569623, 68726.8476295255, 53500.53137159265, 63498.163395532436, 2746.408931928124, 50456.56680663015, 51075.08720542074, 89831.60784734701, 14145.417056618948, 68612.09964429574, 23543.441085679195, 31998.651981549643, 75619.71599557987, 28294.46009193485, 4142.564439720353, 60828.432321007254, 10691.501696112071, 51306.82724451542, 64900.12984136921, 23321.908231855472, 68348.59111046352, 56515.76248786413, 27316.931437610616, 7864.641459136435, 47254.900190645996, 29935.04855167759, 24295.632962522963, 86333.41353252195, 21658.491914983348, 35583.60436260396, 51138.8711515266, 65301.24964032263, 17270.87012471251, 26920.76927345416, 1975.39851643026, 21553.825619751653, 64272.44943757855, 33874.99796197322, 12044.276340132586, 7865.137074239626, 59219.96654259611, 78993.29618001427, 15408.201225208839, 5182.9226520332395, 40934.089095478055, 78759.68949748926, 55032.8731549123, 50071.531601971474, 48408.638976144335, 32809.20480953232, 76591.7645566505, 132.0027730787765, 27175.296686898444, 55750.57849482186, 68047.16690700714, 97313.7594803272, 46511.57053453279, 86472.3707948476, 80522.56487437333, 8294.115599512119, 99809.17468280489, 50453.814159724556, 97540.5091191908, 27006.039145160023, 50721.24283095976, 12078.033858611525, 36384.373866138165, 5525.536542368858, 44731.947746750055, 38089.5686078964, 68794.81034912403, 57680.947058632504, 87731.82587484235, 64779.36192394151, 91532.44346843226, 85794.94411079376, 61447.27256387781, 43405.56464375955, 82193.73870878329, 44715.3932031994, 74117.27851090975, 42253.82697720256, 82456.90871261412, 54908.36406012769, 25626.588742807533, 58645.962466693556, 39164.03196132033, 20562.9078670323, 41001.63666941796, 36026.482221760925, 41683.19402927192, 74367.60496737438, 39021.822719485885, 4221.263900355332, 60968.588075998974, 42288.0382994706, 27672.854156053138, 41607.255563829734, 26974.401715348406, 93440.84595616153, 56529.97502568993, 88423.76604826807, 10904.343764006197, 21473.974817000908, 49886.6861347409, 6071.320164816984, 73432.6144890707, 12720.186918857167, 86972.45954027855, 98573.83321052974, 54908.17702343309, 91852.44275198426, 95501.55869291909, 81176.1396370073, 96376.79741414577, 43250.74391372009]} +{"id": 371, "vector": [3862.438602357154, 32868.18126262271, 70888.25100935761, 61211.09571569916, 93308.3123036127, 10865.103721832358, 85787.57968960529, 1049.563876764703, 59644.76150042762, 23892.601106967348, 11232.353234937065, 32106.060274330273, 32563.70194574859, 9247.827783230734, 99114.36630862237, 77349.2433608053, 45407.95651305771, 72943.53057616825, 19526.5416281458, 60691.74161911427, 63008.875199216505, 53198.240264599925, 18757.254714417493, 46071.318711394895, 30909.395548395423, 71752.05101564943, 95401.77289033435, 55964.42794851432, 68580.68473902866, 68452.8084396962, 58833.45772819164, 78068.5289497407, 14603.945506994343, 76022.38333928346, 15701.619073663975, 98446.54931046635, 53837.84227036129, 12048.686296168187, 68945.7982170874, 60072.05900918483, 31983.18420455205, 95423.3807741101, 57296.13850201874, 29158.02057207605, 98882.72693923728, 2982.2964164924915, 46986.600573927375, 59620.23626405056, 91366.87196708027, 79802.56178191504, 30891.965036332545, 76046.41793251621, 39440.145526610446, 76010.50216589446, 2191.386224290559, 23905.525395753026, 89604.90835047363, 50499.02278954961, 68262.07727836509, 70938.35902192844, 18015.52299365875, 62291.4530044691, 71192.98243682862, 29626.660159849816, 87514.92851227945, 65236.99480693749, 83878.3611834868, 14865.634738587098, 64883.95252296101, 51203.37548954104, 79878.93245250784, 85015.62010416028, 95858.92760686691, 84428.76113643919, 58812.628724039205, 61817.46345351387, 44485.900059545005, 86194.5615464921, 57324.673210036926, 60831.94326222818, 90231.82511672955, 87436.07152954812, 31561.12869407872, 35950.16422663189, 34269.31954333215, 90515.05342661579, 8999.074631157888, 81314.69971419337, 67160.77234759011, 51387.13944906436, 54132.22755172226, 54084.4808479091, 32029.779498649303, 70456.41871291981, 66054.31223209281, 97120.74383061581, 53174.016561334734, 69887.0249834131, 78691.63470440726, 79366.79049224437, 60996.089951960086, 77759.3185650059, 66230.10493332047, 26373.538213036096, 48213.436540658695, 36255.068649000554, 84258.81866999247, 98809.20195608216, 79634.30978041442, 15049.32618807897, 42959.08576369929, 16944.149300651246, 85623.6924400995, 58640.13391756251, 47969.20193297668, 37629.91668058746, 22662.33364960475, 32866.84446159156, 57157.61690393187, 69113.17829571448, 15017.323122399717, 95037.22618155605, 6656.700475444443, 16322.810418998313, 33868.04027083371, 82862.39099384028, 24393.12677623624, 25362.638387563795]} +{"id": 1992, "vector": [39767.88442211443, 56740.64964092497, 99234.73430791093, 48885.91767920225, 77357.055162995, 56275.51065217675, 27720.272947942838, 94350.08464420882, 16819.89561769398, 2704.318494123492, 86119.73475317913, 14213.008649405534, 56446.00550959975, 52034.79533423714, 17914.67442396767, 79396.64948559602, 27885.724840557636, 81223.83923759268, 7027.084865020894, 16038.612885744962, 9636.706721770171, 28505.939943493362, 49415.629516761976, 73741.6181020623, 54134.96833438161, 96816.61928601297, 46519.49935179645, 81984.51053661069, 49036.12088714197, 72492.30984661337, 42813.274734456674, 84071.09225327257, 51709.895706853116, 58623.040982149556, 52399.389681493725, 40880.49416973845, 7180.173750672625, 99299.8427265002, 84115.67262915164, 45637.99488160291, 65164.36436736189, 22468.324234110638, 77059.59034643763, 47964.21941354268, 96318.72751210246, 54820.22989294616, 61167.293665820456, 93048.95462674186, 42487.42771487596, 34060.172177151806, 64462.7136751458, 11983.019746212098, 795.182730803956, 58235.1811797863, 57217.39312683723, 79835.17378435555, 75610.64702026089, 54129.824682709805, 59790.31696998433, 12462.218059386121, 34263.36054847118, 95381.78320669143, 14024.217118928527, 75298.75093902682, 44381.27632470248, 32585.3682669053, 95592.0262305846, 2923.8339330700082, 80954.81318683816, 88420.69842982173, 21216.76161293492, 26151.106452006446, 36710.875128361695, 86960.7544830802, 24945.488303724094, 97032.0964611822, 26386.045185640007, 81319.9390591387, 12338.96562623714, 28464.89090536464, 24114.496003931275, 17848.829492249373, 45363.50220063092, 76916.25946183378, 89707.22113571432, 92099.14099153044, 71861.3864330627, 64279.122513341455, 98222.23634368643, 41042.21898282479, 3995.619909974257, 39842.59867223239, 817.8990635051009, 62161.132420481976, 16647.36299818501, 8155.08801927457, 88437.59029489859, 46970.92518204479, 51620.70883126979, 39670.86625834338, 3346.701026604981, 6189.622315483212, 95981.4524023924, 70613.7792397722, 25670.27568559542, 61340.056220651364, 32631.40779118532, 19834.47747220134, 16142.78076237291, 35478.44732172298, 731.1348690947627, 4379.288883092447, 11200.33346122732, 58393.31265707471, 46180.305548201526, 36776.86941584096, 87066.7789780979, 79883.2440718141, 51126.71277247483, 52502.08131273818, 4808.878600945932, 75787.23720928474, 87294.49941860126, 28765.014089922737, 43849.19612462561, 32379.896232369454, 70679.4426796428, 94955.74617231426]} +{"id": 1230, "vector": [62793.320615588476, 75928.46692143315, 51394.535777587334, 28194.09165956668, 58037.46238814852, 91003.46131301024, 78937.59837842423, 72575.43112527352, 57487.97858401543, 27101.352835243597, 57478.6451537327, 41251.87867914819, 4798.0359243699695, 76944.22263088367, 39976.79976975765, 3732.37533727, 95793.66269786957, 48192.36035737199, 1039.0569243184157, 94060.85005103041, 78216.98438516469, 40678.24819680392, 9123.829578108056, 68826.28310552755, 15395.620241287023, 64869.16179718607, 42629.27335757679, 13266.752052131536, 66457.64264374437, 52909.82101570902, 45700.05904204115, 86675.31173734508, 35530.49331771393, 5249.231362250317, 54572.76534439758, 67606.95241336989, 87660.88250101315, 27172.597762741712, 97595.34718767928, 10654.594049633137, 16860.229845648155, 73968.10949515771, 59141.13810901963, 20294.18700494615, 30009.549525776223, 29872.9694900694, 95379.21561204443, 31255.545628746484, 39911.67883730294, 36335.95503388437, 94936.0553171587, 83944.35064138567, 17040.16828405689, 40643.61582881193, 61038.74697139899, 82774.56688254444, 36522.650218928124, 3629.023379381935, 9764.300747702926, 9666.271517599178, 56021.38077790788, 33467.91117717469, 1495.8200206755423, 74885.30597089666, 91705.95817487258, 53979.90499511105, 70233.299559295, 75396.67161805618, 87728.69670334684, 53135.930508990445, 66356.26795175747, 18700.38158634534, 49230.438829765655, 50869.3850684452, 43483.77869642841, 17736.551505564792, 21259.082294640953, 56356.78420076056, 68519.15221764133, 86557.24498758269, 555.8990274669662, 65199.81770657052, 44834.87153024993, 96451.95683600096, 10085.976146589981, 56477.06057173387, 35823.95360947183, 65030.81367468362, 9719.79955845762, 48084.482697852196, 3492.4385603466226, 15707.599805685935, 26789.46895700608, 6221.409891032626, 99034.9456263521, 97735.33226284258, 3866.1952472669236, 35662.40208107953, 47434.095954704826, 91461.65805347088, 10955.117987228969, 56159.616315641106, 64229.529521587414, 54407.24881575516, 55940.0068823389, 97754.57563971526, 61579.28141368038, 71649.65640122793, 760.1216488377082, 3498.1654074055955, 10750.86381301662, 11817.833629276653, 33121.69081026159, 90314.99294961084, 69160.38190043179, 5486.688976141351, 59038.26058010814, 33399.812007808025, 27555.92870397442, 36462.82725835177, 25063.75304217153, 89375.67751156751, 63540.98526026092, 99489.53598357938, 56644.800546045815, 45559.29167399858, 22538.92215621751, 44122.3945678898]} +{"id": 452, "vector": [4981.699822087482, 95855.14395413127, 5313.461080119075, 81641.05229847827, 21425.160699817392, 57071.71139127184, 36343.76584829366, 87333.91698135526, 13515.586102122368, 49.17545902030085, 69047.44009658752, 56251.26450841096, 82358.67711682539, 92596.44286613874, 2623.825227285692, 60875.395134674916, 4192.94472969135, 1290.5946638666733, 99035.20445532526, 74654.78913733366, 51379.28300901041, 61248.90376556803, 9867.565028850988, 2405.4259263987187, 5395.117329490539, 66148.02443264954, 34611.998389287844, 768.5101286894036, 29410.90012388481, 15340.617571460813, 15870.494263183931, 55511.86037738501, 70663.51014548742, 22620.258601035894, 47996.17992467586, 87686.46762714452, 42531.14536264168, 38470.94157894112, 43318.43396893509, 58065.67691409012, 73576.16237627277, 16974.367352779518, 94133.82194149047, 39770.95606543396, 1315.2643990125946, 72224.51565658662, 18467.69232018024, 39339.15268452053, 83592.37605003336, 95634.77042454886, 82283.78653375745, 95598.71699971987, 76482.91699207705, 75816.11649640117, 50999.42247964999, 17165.74316167062, 21554.43626864122, 20929.797681587115, 73417.43786170598, 8838.804812833778, 11565.809428794637, 20600.501533290648, 10436.625150770951, 32736.76123746042, 79618.10568192556, 46154.33443434628, 90100.45534729297, 97972.46243859794, 19467.33114620377, 91558.76040579806, 84897.57275192742, 36923.660100679626, 14537.478004054727, 51920.1584206011, 63982.96061273612, 52251.90289211782, 35407.44833356606, 68446.28820909535, 25466.007308972672, 86390.06918018221, 90021.57356815363, 77964.32258386078, 85143.37761626126, 13453.283530733317, 1426.2798447538528, 95824.05814261077, 43227.38194863428, 45551.25684226317, 87560.33401196574, 42315.198630962084, 2748.776575017098, 99533.29973892482, 516.3521368199641, 38764.36591160174, 73092.44215184129, 34154.12724314157, 53760.82097246744, 29381.51178512627, 81534.795384161, 30660.760742592476, 81621.65388592835, 64841.71666950706, 98555.8333808418, 40462.259176243584, 42808.01408808914, 304.7926011569557, 95544.94492318155, 52954.47648209392, 35151.673422585714, 14628.435650968497, 31201.051883081167, 46038.936746099826, 29192.44309629492, 3237.8758469816303, 40488.240512704055, 91678.08688895422, 9117.102361742323, 87220.50754198676, 37920.89688005667, 36462.40722261389, 1253.163987678696, 60611.428446878235, 11311.068293649862, 77864.82599020319, 36081.179801057726, 46233.38182874541, 33123.98146798102, 5818.162585050568]} +{"id": 2110, "vector": [84315.79935766103, 92586.40025479125, 88691.05468118486, 94330.59548474816, 62175.74071080914, 64200.00154923488, 15196.5884659231, 90856.77808104796, 78647.98198869906, 40051.318816832725, 11104.964450138654, 24532.938399207917, 93762.51831380147, 62781.589503283816, 3995.7412523157655, 27671.685006435164, 50007.85391494642, 25496.785639107122, 48563.19904410522, 88023.05869633984, 17649.28004382601, 31922.808725617437, 25249.739396819128, 79742.17122843227, 56822.721246544475, 12886.684128645731, 91510.88679332563, 61828.116096145306, 27520.180191766707, 56511.2972912977, 10602.650092435262, 56937.47168314487, 28366.498143945075, 15016.281394277054, 160.13104582445203, 73591.50082515227, 55508.91729008644, 21727.84457482587, 43755.703992904506, 47738.49138086248, 53671.430706120605, 9988.5640566576, 31986.62366630771, 53561.4778755994, 24385.558824483465, 95665.97721292818, 96366.92328620446, 83364.64042767441, 51571.805382997336, 23084.0328925477, 20048.357980137378, 37465.99035674333, 85629.57240502136, 81250.45497445871, 3406.5986902860336, 9058.532654830764, 37985.57896629504, 54398.703666077716, 66467.2266641706, 32551.60557608634, 37431.18231999758, 21557.47502765094, 2553.3994854372468, 87948.52832526104, 53785.5670784242, 20873.500968559245, 26290.165978853285, 44016.8110093481, 23581.068235309023, 16253.464036362553, 73100.50546285106, 38225.25324750389, 25200.20670591694, 23435.28776701841, 51822.85996189886, 320.70071421788083, 17469.202432719576, 56776.79883519594, 18816.014809541393, 78518.18333214843, 40796.75927555083, 41971.32812046826, 2982.2206762209703, 54641.97408111576, 83202.39050516328, 52027.36782965255, 49147.67821081683, 30734.15108006626, 96045.73190326586, 6899.331709838397, 96704.91898628736, 79419.39759846621, 38666.09777519183, 29077.343495974084, 66328.38795149245, 48957.09744324103, 58142.88902526929, 62049.05666541458, 95918.89451864133, 9692.573865703813, 43109.74975093952, 14773.658971343295, 74670.53368572923, 63702.80387847158, 48190.92687020651, 48931.86662241601, 37919.404240159005, 72601.33638839133, 97048.75990361263, 57452.71633025551, 96989.77743815194, 8283.931665963551, 53965.08731653663, 88448.31810234964, 10093.503757225031, 70044.14596044541, 48316.81793338655, 52664.59017689814, 70578.58861335856, 2885.760976699181, 46499.99574093679, 70292.53139777012, 96338.6961563517, 9928.223387881308, 43449.88203365563, 98408.3107177116, 1740.7658968082605, 34613.05418451584]} +{"id": 1365, "vector": [32027.306141308454, 7834.630054672464, 21340.57845442324, 49974.398453227455, 82452.37300800021, 45145.298030009326, 25301.165012538153, 65844.19457893152, 81790.35756177282, 67817.57479357545, 77110.3572037414, 49238.87839040252, 11302.698585998205, 62769.5042375813, 25524.13090539821, 41402.59649965317, 99092.13319826571, 29584.041202091936, 75408.35833230382, 76360.04699481325, 58041.19273014183, 99770.11417975134, 17144.17693336563, 90914.96310490984, 52073.47932229314, 53047.693813143174, 75064.4787224214, 23291.802394520688, 61333.43913892272, 86106.20251469582, 71651.47127238841, 43411.65057433852, 50581.07584765248, 6091.554694811485, 58793.327127037606, 2375.2421387549625, 59667.71623808407, 98663.24388135878, 6615.276593059294, 39116.05147807425, 63326.52332611144, 78434.63946252898, 67763.08295068692, 81400.8676785852, 5110.707609252651, 37061.98613801295, 66242.70628894898, 48035.13154709175, 55381.85139692006, 57563.76107820879, 94295.162782766, 9647.285811193573, 56832.69278448214, 26764.527784414127, 82030.44874310847, 80246.4920616308, 62909.33511452495, 50174.37823551638, 21975.85140319225, 88367.5116119308, 64944.13025597433, 9241.378514514608, 75322.11949612192, 98422.78757784014, 64792.49015247104, 56694.26473680482, 77604.68571522116, 2047.2979161526239, 7058.753093563874, 15030.518340313449, 58906.057145953026, 42578.11914669879, 57260.87460416671, 44702.46687952897, 82657.33436764702, 83647.55077954863, 19133.58690296968, 45518.017341322506, 89623.55875823063, 63980.399561928294, 40673.000219728005, 93553.17309935638, 27929.296978203532, 10816.000276488181, 18041.858244005914, 6669.543679636991, 53306.413553395905, 82092.38269972974, 29711.530962380228, 19410.6027797607, 71527.51507595916, 957.1130035508469, 65817.84505970067, 29597.089791713348, 18492.429016344202, 63338.63734875449, 42139.200808674956, 32361.035607158028, 26235.862712990365, 53257.77222826908, 30422.433219274903, 52797.65982477443, 80890.2148680504, 52604.473762312286, 61939.56728391413, 40257.71902945252, 68665.65421533113, 59559.68534791176, 59191.96625404219, 36083.45428282852, 93987.47602802598, 14988.09901240028, 71724.95685436748, 4415.86645696257, 77545.37986198759, 35090.11518819374, 79141.70149436926, 35821.1760236901, 40528.83357513591, 66435.64836654541, 64690.80196053445, 82874.3108944039, 50885.98048215415, 12421.729233224343, 1551.1265123369756, 22254.28775232988, 23835.487159784607, 97074.93993517297]} +{"id": 474, "vector": [89719.08216273799, 41569.98530285773, 3145.437107144988, 89519.65443723247, 33143.17133559962, 35501.06623256466, 55020.415873579186, 66775.4387799709, 28360.280447882036, 37774.56987964041, 53405.51554587812, 43806.052905504854, 37974.0970095078, 32951.7401038609, 14264.407372300237, 87930.84715969676, 29473.57626881938, 85075.84079929732, 96131.99530561671, 50067.375495716915, 94019.66349692341, 94467.17193132151, 37011.19832888258, 79209.07938712787, 16522.626063431722, 32387.401448910812, 52263.01226501301, 12251.217831199312, 76245.66471569093, 28286.690027045326, 52750.778816587015, 77131.50281509575, 62442.230749947536, 55340.645320073636, 77068.66566810814, 38672.016205360684, 2560.3636603182545, 21118.126601233256, 46299.33851257951, 10096.792680943034, 40537.94046691257, 40723.59058558661, 96285.6290510944, 20515.534674196824, 8148.610329925609, 48602.14857328644, 1002.9249744209711, 39092.88547792193, 60018.02531974717, 63617.943036110235, 9924.493143535452, 7681.589781780563, 65681.37047977236, 31960.719436860953, 42110.45752436981, 43267.188731215945, 69473.66862059257, 68020.29046412489, 15953.613445591653, 60777.676832066594, 53545.46000363413, 96803.03632910986, 95221.19605187568, 29378.254798274138, 55637.471322318845, 42573.23881736987, 51035.825570778696, 48867.14537250787, 40935.99067683987, 64110.1815111289, 81674.03208032233, 79153.6541672361, 86755.20920318045, 88163.2462364084, 27597.24171150527, 24155.37425721124, 97322.8072075947, 45824.06426076024, 34191.98178795608, 34688.731615524914, 72014.25927195359, 19365.752257095282, 66989.36359201132, 56927.99859244054, 65014.19678858913, 97667.82770037008, 67375.02742338486, 12302.256059433337, 99126.8073677442, 50549.922590816706, 36966.79696168053, 9496.539718585951, 56117.93076377794, 7679.189001772402, 34527.75584492912, 57150.957282421725, 79253.76557662297, 6467.769521780964, 49253.94592499048, 13182.738171309238, 40237.06306374713, 5301.399579373189, 97182.78007658488, 52041.299517748805, 11781.707223162419, 16013.376115487099, 36150.88101363332, 99531.52986405989, 77706.35511338185, 58536.93778649572, 219.61257398367405, 61130.07041195773, 15374.205477957548, 5928.4623377571015, 26474.935321920668, 763.173676072948, 34.53112130684666, 11662.48166245799, 59911.470957617974, 90310.23647194596, 38175.8338246341, 77527.61384994413, 97191.14331547613, 42770.866861923474, 11697.659912165693, 41506.54760009825, 38162.320867497445, 58502.084064860384]} +{"id": 885, "vector": [37105.864979380334, 53699.08880675594, 75791.99063844, 55637.14170578651, 38425.08358067573, 57426.10818680902, 73762.71150470267, 41902.24219248625, 95404.24110690181, 50088.39516485173, 41516.684107007386, 43848.26920762982, 48977.48869125983, 3325.1469230263233, 64709.67091546993, 56198.70570572524, 8484.496560977928, 9085.517724035086, 16316.977906107588, 2115.4628410223086, 15358.177120376926, 32997.12966232168, 18096.935786612645, 91398.56210695674, 9435.786238535215, 70876.48594629085, 22700.628628041653, 79472.78395239897, 20781.787252923412, 32884.025828727434, 49190.5965217452, 14184.954263830095, 7637.7199030230995, 72841.38390131133, 56651.989779378506, 28902.893963460607, 74420.39143070475, 36218.474475359464, 84376.56279559855, 60067.291459615015, 74086.08413193663, 28453.63347423866, 40354.97982334494, 85601.00435845283, 91758.6610502074, 3138.647919727289, 50777.653307388835, 57338.51517514653, 94777.08950040837, 63520.005835492724, 11273.38433769628, 27138.185461762267, 6743.08131724446, 17510.886077414078, 47033.0191302636, 66339.71925180468, 23833.159365119005, 74326.47871222219, 15632.407452671981, 18001.534753521253, 36046.36582064907, 88625.54879041617, 95502.4180515416, 25294.237004131915, 61408.23760423362, 27370.94408493016, 22916.561500544307, 22574.59518592605, 32845.70581761859, 75060.61070947873, 51206.905234810605, 50577.39322350452, 76788.43281042035, 26758.864140597183, 30079.9456963194, 78697.26265362372, 54621.79880163248, 32640.295961491396, 14107.547404330468, 72996.37499651143, 72733.63852004214, 56199.083675334405, 32050.832788573513, 46525.904364390655, 47520.10071249521, 27228.697690494642, 13934.195684534523, 85829.06569968049, 87324.87027531194, 89126.15576313205, 7852.7684187887135, 79296.97519659137, 57913.88283111968, 6452.499795302291, 42253.18098990674, 46851.54057922939, 50126.33472583432, 44596.8202578988, 43840.8655847151, 55508.45882831562, 27662.105098049284, 67490.43928460551, 61313.73956783029, 71375.97616770954, 16432.40630121381, 38843.58221432398, 62983.59972811277, 17996.76006681069, 24894.733624557462, 69578.21860038572, 23538.058269317597, 32458.211836464212, 46240.42968520725, 21730.55013644597, 59174.783091751604, 6666.6535739858255, 823.6970049910864, 81076.4748889813, 52316.49133849753, 98755.70909851293, 58295.906875287605, 47918.37215355284, 26982.8736204635, 3284.4340463108024, 45062.84031786997, 53402.91865010085, 40543.281513442686, 74380.33893464779]} +{"id": 679, "vector": [86884.62091971526, 21377.344948871058, 6313.994427723457, 36257.069289892475, 79158.53676916497, 3648.268370449359, 65351.48046042746, 88288.38646838635, 93711.36742425569, 79126.90067345637, 37436.96530559631, 63193.38879766024, 94631.62001595428, 3540.238948132546, 85881.23365363048, 65720.94542845395, 59141.95977562821, 46633.813242912736, 69981.03134477563, 67164.16852048573, 43100.46889647619, 32150.432894990765, 13448.823476316107, 56305.702332488574, 73713.21589220522, 83215.34092610262, 69973.9646054894, 79352.02017415322, 93996.58154392468, 41818.48165472556, 44704.394869223965, 26195.47239620068, 50594.05844209861, 99063.60178027338, 18424.9023488812, 99613.84620450261, 86949.70529944147, 50064.40065004169, 28430.043467376632, 46389.61852339305, 93291.23416649953, 48665.10195873417, 72289.72970196824, 88623.9536184666, 42322.78741728076, 2100.0973022541225, 66292.33826107443, 41855.464413670285, 37586.91436804864, 76359.08760103394, 54235.78236492637, 70783.72777026951, 69386.41540174873, 10155.23547740217, 16901.72758936579, 77876.19606347936, 88766.10445052973, 27967.004277540607, 5987.297059874885, 61531.87513662601, 63370.988454676044, 90128.5695078569, 72484.45721781673, 7099.422431131108, 45997.24826383136, 62375.75508192227, 58924.5514796615, 11480.591584105892, 47665.38490907589, 38503.41154362116, 80126.74178802702, 83583.97246964405, 90066.2845201854, 48407.77504127617, 46494.54193812454, 91282.24967759007, 50486.76744243633, 35252.36876281504, 61097.33388660409, 7959.285372931191, 45985.019163957055, 99350.25910360488, 41353.07195179604, 28685.863372518306, 8353.248806128133, 279.8165137330333, 87147.91963936393, 51671.87767298661, 32622.04182050359, 76899.84524754116, 61927.24389890307, 71767.89342859152, 97698.76686601335, 38015.60679118265, 21972.76461698061, 31792.06958619143, 89576.17407545052, 53558.19867583214, 2914.970013080231, 85880.85976146882, 78252.04072173483, 10414.655018478703, 5967.705126699818, 41021.79322745872, 65256.5260289132, 25454.256825193144, 33971.19360955426, 57656.12951337926, 61761.21611572704, 56519.28783319934, 99948.7126856724, 59277.1173846801, 51514.08791787177, 59927.92711477864, 12210.622420505668, 55538.31346794163, 5100.398049815213, 99643.00335219473, 98358.14744580197, 9210.716115301631, 97747.46415083976, 9458.372154880446, 54805.009959237505, 25409.85078082695, 4729.92482066169, 73533.35257313256, 32425.110374055566, 28147.176062149592]} +{"id": 1422, "vector": [66548.4157523261, 44336.42862277167, 31565.02666271228, 48203.72339239581, 87799.0562287829, 94198.63377993075, 85439.02580358405, 34718.709840525684, 88945.14328715764, 9455.545148810628, 40970.427835456394, 68139.49795859493, 98964.25764010176, 36850.21435389622, 88225.60956043116, 85987.28432336626, 58501.99049043744, 52322.007669889026, 48.50330560052241, 73582.86824020131, 72940.54396005176, 63519.9195783044, 11160.21121779771, 98761.17072994393, 56840.30373604346, 65633.85744171304, 167.64995967349927, 23092.654161639926, 23529.913309078685, 34040.13449526936, 80758.64587546294, 86594.66639685129, 1414.229326691785, 81560.18475424188, 38300.989723823484, 24846.86716774015, 46373.89586922614, 49286.95730312841, 41468.94525241117, 89899.43724604332, 29911.720157805787, 91976.62764151869, 64929.43893302018, 88131.17207380499, 13070.354324089716, 95699.88166561903, 78338.95961728836, 59845.46215817093, 30079.826437346168, 63768.86099946821, 92788.44490153131, 73840.549485645, 80638.3015765535, 78779.85675185599, 25254.708076765553, 58979.373483394695, 86763.81989287797, 15593.056864601496, 4831.42291758788, 25497.762690975356, 83077.91678447468, 78882.77601410232, 73552.18517042395, 56153.5006312471, 30039.844088071433, 13907.384183653714, 74370.76481001044, 9480.754032358984, 28305.94640028552, 35722.48852413677, 92468.85646201481, 89502.24029126827, 7937.799709145321, 95841.51313622846, 77089.12650417464, 53669.0002137154, 1783.1647909424041, 58070.14570047331, 99130.0367665931, 74441.18309626712, 34125.373040020066, 35879.36647346968, 8081.622662525067, 64983.08825113415, 48642.873806158146, 34153.70780634351, 38449.158595379165, 34655.08100509118, 69486.21206326273, 19282.832628094406, 35074.48859319758, 4390.452635280695, 43124.94518786222, 53512.73659419379, 38269.51786007826, 2830.512604945723, 14238.890065244113, 1230.3377712269858, 21400.18424062975, 90291.77156345177, 71030.00999772262, 36094.96690159566, 90857.01221169216, 34566.8181819054, 35549.218904051806, 3568.9197715044306, 46826.53632666164, 61071.282975863236, 2919.4238465300227, 70921.54455777259, 96702.89913315607, 51281.46865163238, 51573.52851267673, 15265.983747050139, 47711.22540766437, 4138.9780391866025, 29450.89411278412, 83261.50821309777, 7225.701984980226, 39572.16375544728, 36873.8837029824, 49345.83738844489, 73291.78049169421, 53353.10870097509, 68426.77580983759, 24356.923574042066, 20480.275143937477, 26648.982017859013]} +{"id": 173, "vector": [48794.479933002345, 39923.22740816267, 70956.15827616946, 82097.18952201688, 41620.95439771628, 22506.811854478325, 44141.14102911848, 33890.26214059732, 50646.50113870773, 75276.70612434013, 46674.407058122844, 47701.90317051467, 51666.91642287429, 93143.45424276679, 64292.69835089484, 45121.10769648064, 23972.83578297502, 76778.3710977247, 88410.01948475726, 26410.42905782125, 67223.48314020359, 2745.187998298915, 34649.52029643884, 87637.99634759864, 91888.08298642033, 57100.722866824835, 75617.25540252101, 78233.43354976317, 70484.84073715736, 72861.51201063795, 95264.46107342541, 73315.13610393333, 53117.85836939567, 61418.00179265538, 37814.14386763087, 78313.62797203752, 9575.478292611073, 1898.7971368074952, 61100.04512356793, 48967.21935616174, 68824.31994394258, 92060.13883075504, 71257.98130830872, 87445.4121214964, 45194.81087548843, 18855.577682956737, 51435.72419854614, 24120.200241250634, 44151.27982547619, 22559.756246587203, 41594.86127053501, 97869.51809573709, 40738.36263361008, 23673.726031185306, 26182.397594955943, 14865.085257716992, 80935.68715319071, 96639.79315587488, 3152.5808081770633, 58429.133148732995, 42918.897265409905, 4538.183943441753, 19531.551481898456, 78627.9460684578, 32835.31319176747, 79298.80752716851, 95966.335525733, 24978.49530850803, 77843.57809130102, 60361.997886057026, 99108.3935244512, 14029.414893525993, 84988.13778753062, 51568.26486224995, 15263.85350445939, 13615.728724753773, 67171.68265782934, 72454.05564755491, 87483.42085182214, 74172.2995804115, 10214.313006847697, 38877.07294595567, 56518.590250775094, 88347.84816461448, 19217.653466040785, 32710.70759190875, 42389.09511033773, 15564.921165109246, 86871.79211030043, 56564.476615935775, 20734.427033995307, 92666.42197515401, 31349.752439989243, 70570.74861153336, 19674.87604145427, 57420.02616022456, 74438.80012364818, 7136.266341579589, 18841.892743396893, 24007.4566789609, 57787.43560297774, 99659.2712881185, 64715.453119356505, 2667.9782165923816, 62660.05088161842, 76390.73003305704, 58234.5181640532, 48641.640679149314, 13072.792594250359, 32392.278926274455, 50392.504660987004, 48256.85590398784, 37264.88477008164, 70933.19913373947, 15596.354563603087, 16449.36959653691, 85894.28500151975, 80137.44564071287, 45098.62958249177, 23513.297055560422, 4718.226893143728, 56704.235543556235, 8053.28460506094, 27173.751172876415, 31276.24481770418, 96620.51423164266, 63171.238068267245, 10581.403703224047]} +{"id": 1392, "vector": [50112.74806333894, 96758.15125070291, 98145.9798116596, 73667.53675320475, 74086.87927108846, 10105.629339910238, 73730.22066795446, 56054.5148376753, 11820.10505993477, 24075.329704855154, 20080.889017504076, 56840.24563031568, 5553.597935136057, 31721.694444865036, 79745.18996162275, 51347.24806223234, 32984.467042448094, 46991.28319391531, 8101.899236368348, 90626.95612179079, 39273.59073989514, 46493.99196827887, 65930.37611433839, 349.57031217175415, 73720.55104951731, 73789.58340074058, 52737.646212362844, 59242.81154270145, 75529.60214637764, 38205.808136684995, 34814.61392855187, 23320.40344395895, 62272.52365838431, 58577.651524481174, 50190.13672953893, 72253.70395693759, 31819.386771877224, 60328.516549231004, 5559.839425457536, 54005.666313563495, 71713.64598916192, 13712.376393000857, 61608.833310611655, 66866.81771986885, 75559.28778994919, 56998.687111942, 85703.39687068002, 58339.8400699073, 54208.20842803752, 77439.84055373246, 54649.269517853405, 11178.367426751867, 48497.651473844584, 61789.80556466691, 33277.43573485791, 44812.594716171596, 20001.65790128008, 34288.1358559658, 57459.95285197561, 54698.06062709247, 18583.23371151307, 67351.58492663929, 60495.01749557527, 16001.144164268377, 88701.21726823249, 32241.1475718272, 60402.18669960554, 42670.329109674334, 229.60107358046545, 58149.90051999539, 71164.00463654277, 36805.92594929666, 67890.71930925142, 40797.981618888676, 56303.229518697015, 5969.661248806335, 10649.61645183493, 94689.23550114286, 37485.30360241713, 37887.8084988591, 17819.490559532303, 39637.9382545193, 39971.75665780499, 22132.44641787746, 57462.05966152721, 89455.18630050615, 21915.299281709576, 95697.87208162704, 35067.380904171056, 49226.44635244231, 53173.99707626242, 44179.96961399069, 75357.84679276409, 95158.29482549932, 80272.97734565852, 84418.6642013316, 57139.40061088979, 18239.22580034223, 79252.27892463186, 12972.658535938075, 90786.58687336065, 75355.49304956826, 31426.138896151344, 45464.4162867368, 18302.197033462497, 67352.94967393472, 62597.145380142494, 68865.53197914471, 87842.00346548081, 92806.92174888469, 83901.89072746283, 67088.70761988442, 84559.8269455274, 51896.832765275736, 42177.624757216894, 45055.46086731905, 90537.44152677267, 45187.167868394565, 22876.515101917226, 12129.12035793714, 32923.38052618129, 25284.499723233177, 34904.65423660185, 28475.62486943106, 41843.258718301826, 47213.67298704443, 66164.78435743546, 46668.01630569485]} +{"id": 210, "vector": [22514.173617883716, 55382.34140270701, 99416.15187642215, 68742.78961331864, 49394.82556753904, 11434.391095166364, 1307.3509647822946, 54252.16087862567, 40707.04623862985, 9048.020606628237, 52160.74768746025, 16488.89528705493, 74828.92285479468, 48702.8952806526, 19666.457878419387, 9832.08394960038, 65272.93415309521, 80563.68016111813, 82378.67417829404, 3969.8057725817916, 58087.38038044463, 39864.73612504183, 22819.926768768793, 2952.9599428381957, 15949.056706154019, 81633.04868177252, 54922.87954444071, 27819.240672425083, 65582.6487182139, 44394.12682236319, 16186.079649983509, 13993.09259489726, 68991.50008797052, 73273.2299287588, 60313.97087493951, 58791.917531814586, 5599.286053336372, 22631.051164577275, 67796.43775265073, 44025.7359336504, 57103.67434056661, 66624.20801535367, 42773.54453741053, 74350.5020653144, 37972.80624502407, 83802.33995186021, 84355.51117376582, 13929.285839353412, 48963.84619807771, 71476.42774251178, 1488.4821700860916, 63804.643273694484, 41499.86400851503, 61785.762614438885, 64369.18735117375, 19726.43245510587, 78484.06311274246, 2277.3824489499384, 63505.255589259046, 6275.223621819004, 16439.715089943773, 51088.46140757689, 1549.5329365405098, 36636.933094797416, 61274.85807036382, 54113.24426369364, 85526.84744986113, 15325.023627289114, 39503.36712258841, 33008.96576185347, 1376.6438139174109, 8789.286398716778, 14214.967578210313, 93155.10468723738, 4310.428148494161, 44647.27726577378, 37043.218918440965, 54425.11266211734, 82627.94373905937, 6477.757185070654, 86788.94450538482, 22426.06319818218, 26301.957909252927, 17304.392682892365, 83203.40872639726, 30365.5366626278, 75673.38120509108, 99790.90084748178, 162.09062427640086, 62395.15283940983, 10082.102188886154, 47485.3358035598, 84721.07198732217, 99355.02445095798, 68789.41684446081, 68447.63129128749, 1474.865352520749, 66118.34599980264, 67017.76370743949, 57832.00207134392, 11096.680961337324, 74221.62740026318, 5537.882714068265, 25512.703049883345, 23744.922080674813, 32103.664547411514, 80216.91923856553, 87710.64725138509, 63415.78487567843, 7741.7732398802345, 10849.386776022486, 29031.358560551078, 32384.993413645625, 9435.838694192189, 55332.93403493117, 83815.49856549819, 97373.517331213, 4831.799257013847, 47297.78701601475, 77716.44612777526, 14512.62894144023, 25530.922588780002, 62965.605614816166, 64687.933473681536, 39541.01617862947, 90394.79289781538, 38960.75398847339, 58483.901321402984]} +{"id": 1722, "vector": [64697.036325657784, 74604.12620093806, 43273.03296652479, 64410.75150541291, 27665.062575641387, 11027.640003054152, 52402.14771453594, 11228.62841668788, 82816.24626716164, 37224.8111930859, 3529.888489611943, 21304.819647403318, 63294.50933784532, 70932.68301516153, 75529.86279197398, 33024.47665244887, 19559.239048855914, 62096.68837124115, 49068.218677173034, 69138.33040885637, 69368.26206277568, 91186.23292253591, 86056.99496017337, 47534.858023478046, 67222.78441870674, 54237.36095461729, 41541.384586825516, 53450.43838906369, 15917.330701118226, 63928.96202632734, 77889.85904437453, 98638.7321506253, 24810.495638390483, 49733.05200211452, 70888.65367999145, 67713.47037289666, 71541.02597498575, 70808.48194419181, 28093.46773735777, 8958.980641723268, 75966.69988864809, 68350.42573258757, 31890.17374309402, 52553.49497539633, 86395.2705846571, 32367.718579836146, 61637.53113826068, 77694.06061041041, 78089.35874849887, 28098.566998513306, 92734.45337168386, 59883.82294521083, 74453.47124349087, 72813.73009195241, 58427.37094771734, 14129.802022610449, 9303.821041648685, 46025.22542215967, 19144.112578799344, 12023.929655855547, 17924.75252953184, 50527.129110345035, 16311.704689361139, 17631.693517447857, 27407.456488005468, 29835.698586942104, 13479.631744761222, 72493.73936319292, 3163.6400995533, 94180.44233711236, 60754.40087873939, 89195.54167208388, 33165.38128737635, 10520.978442232454, 49077.179495773395, 87318.42432319508, 41744.811837824935, 60968.302618612135, 17465.99705189611, 58639.689531225195, 45850.98995184712, 83859.5443347018, 56195.290730821565, 63445.60956762908, 93136.73366192797, 24985.718455745053, 12988.82636537434, 23883.185536933317, 66981.1436859332, 74689.84847426694, 12919.709912670085, 43289.27125303854, 76313.2028627092, 43172.897721551584, 41260.06440801174, 9942.511431718038, 55950.664470058466, 10510.460649128028, 93020.67607123668, 67238.88285581207, 33554.06720480621, 44840.38937200532, 34910.094045971615, 56584.053370079964, 54362.469658793125, 32905.65833869986, 16078.50929305772, 56405.75581961075, 31918.791824302607, 54424.69157503599, 48036.27884535518, 15299.782633120984, 79426.98561763532, 15773.146722535836, 37378.17664357309, 86307.51363116618, 46855.67568023641, 27794.02395950643, 91085.10915688565, 85834.87368791824, 9560.733086301598, 42458.42743300232, 39430.16042395099, 48732.87325864728, 54157.06183172652, 45652.56750238504, 80635.95345020165, 69238.7317531981]} +{"id": 705, "vector": [68647.7105691868, 51163.7575101999, 97691.62534598753, 55072.91130361983, 33675.990958600225, 61387.90682988054, 68216.72501050244, 63988.13585119725, 26274.269558606335, 84642.37499649753, 42025.75186391818, 24272.739218735285, 86768.24313569142, 50421.54188537488, 36214.23408775833, 44477.07451074849, 62912.596251911746, 79280.24569427216, 87972.49532665436, 65641.2760794771, 93652.16814098686, 72024.64344596087, 95618.88828557143, 8958.272611730277, 59517.444158182, 84254.64665275445, 51367.893275586604, 86503.90256166235, 52105.851757056655, 10222.555215430219, 68374.08165476224, 22447.073240328697, 78534.20213498925, 31274.365893106293, 13395.146344268538, 13058.083807114574, 49968.452288255736, 86694.61332015363, 88809.49336704827, 67508.2265940923, 54608.896321402535, 12710.402686662957, 23714.322801416776, 14860.283303229317, 5710.089550574949, 18193.794037849355, 77019.16266135509, 32373.518065663186, 52333.874955847336, 773.3771098851761, 42349.4048857456, 18777.171199480315, 15478.0349000042, 47446.529006963035, 85822.21012586656, 96264.60419701718, 26309.7870372771, 38259.39377337214, 87446.37944292689, 25853.861618156283, 92520.89380058093, 85732.47945290398, 17267.717508279766, 27679.710149208615, 86411.60421881443, 74089.56011183868, 13127.001060541965, 42630.3025265216, 17882.976231869874, 94716.33363824249, 76140.79655676252, 69190.11198383763, 22158.89449507222, 67549.71704235599, 66943.56818565809, 22535.924312373732, 85689.56226659271, 5385.6212808479295, 20997.175414220514, 79340.12864692904, 61825.67722115122, 33848.4095711963, 27634.558645896035, 41506.29712983741, 34.14115498452386, 30497.828040431043, 61231.21435642066, 98704.6975502416, 79085.32082270613, 58180.431445067894, 76782.53861323961, 28894.254750904678, 74733.73673523481, 7916.719108578618, 36239.591143717975, 20645.6422201833, 42004.219934881716, 37667.54041320637, 15978.336143377292, 10941.463724003876, 56739.55440307482, 31205.30975700443, 20581.212120673386, 87265.10093029417, 55038.40842765707, 50474.102751564234, 54923.83137266009, 47657.18043910121, 81993.35631785337, 17087.89724224107, 20866.230660308214, 67056.68809868423, 91105.58025379546, 12993.65500745857, 22998.024863767652, 81263.76703047671, 84879.5802378551, 78633.35316749364, 11875.266392624384, 99173.36660684065, 5340.022747782847, 9954.917757645188, 82125.76053282138, 5950.456627753698, 31294.008755039627, 84961.12480548516, 62684.19678262535, 57266.01117727963]} +{"id": 687, "vector": [1040.6874186558834, 85157.22256554924, 35765.097822594886, 23275.199704672654, 23967.454024800718, 34276.9913147176, 77583.54485547719, 53277.00379586221, 81543.30460483691, 6417.414106043851, 42898.65447909167, 11358.739842316058, 34724.742011960916, 55231.92686760394, 96579.16276457363, 38844.59471735917, 76658.63420457291, 14461.258631462404, 26145.598614581377, 97491.71219303207, 90256.56598072166, 41652.88704682608, 89865.01801422931, 46707.257649270075, 96396.4417200056, 57281.50383849949, 84345.87449550701, 18695.746304137272, 52562.20776100592, 20764.1342655284, 83750.77696880358, 78039.91703745374, 21112.687012766586, 85051.80698949586, 39813.82572921094, 98014.78378054331, 70868.05181873906, 24588.73769753579, 16181.406422115995, 18960.300402324283, 25537.62865238133, 80519.59983583704, 12926.497295970728, 61838.08812445629, 20728.79922923101, 19017.64618359637, 93034.64908193557, 66496.95751798566, 77655.77574413364, 89828.10381302837, 20818.369914251645, 18906.057856726933, 54194.39931974144, 90941.3118456299, 54160.943879528444, 66852.02860535045, 20566.685430993613, 69424.05219513203, 13869.123656808435, 13356.783246359306, 4465.8951047405135, 31422.94623998666, 33293.77812394677, 49128.59441607026, 99108.70350728203, 10110.772668929769, 41232.76500806834, 16606.04582369857, 5537.250340504851, 91730.17340144671, 30251.191668734533, 82226.94582380101, 30498.123447535487, 69033.00662635443, 18455.790785863966, 67138.90473429303, 4285.148024607077, 48917.754936026184, 31176.78239428693, 4127.2197541203595, 33598.7446994892, 73303.96652068739, 43519.114056481965, 88347.5887520105, 8011.945542770472, 34123.23628316663, 12961.254185214866, 99155.86676775738, 9236.991228195346, 13564.005161941906, 26600.57902963856, 78375.78625395714, 8057.661857652498, 58528.62804928602, 96968.26552919703, 84391.30202274794, 54180.53191274247, 64864.36119184504, 69515.99302468736, 62336.022537125034, 18975.378956264532, 20945.670446009124, 71869.51057201756, 7550.651871956038, 54695.98167784289, 18662.674428563387, 16304.560430076497, 10788.061216155176, 71118.13448007322, 31075.717776299116, 31107.461633220733, 50077.22757272152, 7227.966371993266, 51452.26746671383, 16481.440027417706, 75856.82940810852, 65894.53069491057, 63778.10933417113, 22561.497593798053, 69344.97822619093, 7377.200445478127, 47383.7172230319, 86857.85554768765, 62991.586755895034, 4606.3887895914095, 48826.11658694652, 75939.59484270359, 16431.88591823892]} +{"id": 939, "vector": [47405.839890236224, 80426.46659211071, 48502.610369322676, 309.1195652289991, 37726.4849227243, 66370.57227531385, 65098.275573354156, 23524.69344521807, 95256.06480389556, 41221.40933422712, 14476.524181313522, 93261.93103509405, 74027.21971971252, 59742.10125613653, 34316.78175115849, 54236.88445891923, 18640.35391226334, 22653.73572271685, 530.7498302362212, 19123.36860019751, 4786.149870002254, 37630.359473203556, 59826.917652699995, 43198.64905500814, 85361.50820697036, 114.44090245400496, 36869.628669874684, 48844.64804591692, 24955.106721944474, 47014.002350522765, 86953.07234990085, 92072.80052457293, 54392.7319623767, 92655.19207853237, 14202.580116033781, 58380.067103101974, 27521.556516300105, 86624.77506938438, 71797.95475433198, 48387.65590986074, 82842.76032208833, 63547.28191658326, 5856.242024281344, 89878.86767472525, 15499.757656040802, 1539.9951915404442, 93378.67693457015, 62328.141091796664, 51100.679740825115, 11994.160720458536, 91023.41904045783, 368.3798159052065, 76320.67150085156, 22947.463936106316, 38215.75587594601, 49443.046671517, 44138.126017025315, 34324.12184080075, 51492.42833343995, 53089.96912600353, 24587.929975002044, 95983.8069740834, 10713.541491811384, 19560.113360042673, 85359.93258434988, 99258.75386572772, 52314.403249326526, 86594.11905011054, 4562.395184808021, 67177.16083756398, 7530.535423880525, 43286.6827997636, 48866.63877635416, 34015.03533642749, 22056.349836840727, 60125.41685976898, 50007.34946536773, 59464.063436153905, 50958.30507781922, 71112.13932881514, 92148.21957717247, 64858.23638807753, 65094.346102856194, 21332.794926561528, 62399.03501828047, 91844.95008310874, 93935.65089611721, 70613.95146093256, 50982.128723307986, 36749.218961053455, 14945.407370091823, 50751.30131366187, 91381.47599450736, 23981.759211640507, 4259.7813780077095, 12275.453183914631, 70844.06371341186, 55467.61111770247, 2116.022472372159, 93576.88450477488, 2553.0103739994092, 94107.31302699403, 97339.1411797705, 46533.421108459304, 73019.31988144062, 58201.58212453179, 78058.95945731037, 47185.64445100866, 11385.087059711086, 82068.2547107413, 67111.48517106131, 19243.022470738626, 79739.4796582503, 32152.527097964656, 91724.82863642993, 12572.315278689528, 31994.532190273785, 22978.201581148714, 35242.30682244421, 48014.83914306257, 78.74917215855426, 5408.157678653869, 33508.85696465942, 73212.31005532308, 85845.01133066295, 17492.161212431434, 62552.51434154897, 87417.47608268469]} +{"id": 1749, "vector": [46015.02920012193, 36173.47931839825, 93393.06264190777, 25694.15683759574, 64283.3509269581, 42993.08019699406, 8630.17149923042, 51265.111700034024, 9606.783882866255, 96321.90878026193, 58402.33223538648, 62322.49014225628, 56995.80841900126, 57920.971841220235, 35448.717782879954, 76391.4836595677, 10851.905314453248, 72327.37559161613, 86996.52043578474, 36700.37234686087, 28652.21563569974, 8911.279926222116, 99184.90603693656, 55695.366349817275, 74312.96918273007, 7341.53060532805, 17955.535328972117, 88453.01471693364, 74854.01473667398, 43393.491091291595, 90060.61517553909, 75154.8812503779, 13497.788243606223, 48644.62702761269, 35775.67137035495, 33055.88425265137, 10158.684616582825, 98080.16462640441, 70851.7816056159, 71852.91036226723, 24880.268605869915, 5032.090447248361, 26326.106305206853, 17196.160814496052, 9866.032906497347, 19972.405371239878, 97777.14933354534, 90904.53673371467, 58354.06025113574, 25836.032094369766, 9375.0859302706, 98210.20601763317, 58432.88944011208, 1654.381050913467, 34138.133100304214, 34071.40041802008, 55889.812448280594, 88970.95862945549, 87627.46598626047, 64329.67522047846, 94480.88555671473, 98304.14556650964, 6724.475891423786, 65819.96824346628, 40199.277324567796, 44250.097573097366, 56402.34143722727, 71784.69228360547, 58508.4182304613, 51987.638135828776, 54216.998757104506, 48746.35818280634, 88033.44242662002, 53277.49494859505, 40039.909115383496, 2433.4635484500322, 33799.93193825274, 64971.32798869619, 70163.76371174981, 38578.28541477684, 5847.223483457653, 47020.670891151785, 36201.191411992215, 67620.14492216628, 4964.564707340891, 86165.5458837631, 71859.92910212842, 53122.212673885326, 95424.30580771132, 97363.5056286413, 56590.57088116707, 83968.00558524726, 77333.34631252027, 88480.95262981932, 52897.00220191166, 2092.2667412313854, 14746.09793849081, 92360.5098312116, 64813.89622931097, 17002.16839220433, 93528.27866478516, 22360.58437861238, 59281.426991599554, 51534.87336765869, 72426.88133016796, 92978.77589668562, 40468.98880183627, 8073.964315582116, 9419.436910005097, 47766.5590573344, 53720.78159529079, 62662.96749824725, 31688.398550517883, 59098.02500467417, 86093.81080773904, 2738.718178574817, 12058.801669117225, 74724.51225770808, 68658.73616565303, 62681.38884751275, 25237.445251880243, 67974.35155983982, 28532.752159876363, 86111.34647597118, 85160.4795979367, 712.8356604495623, 64447.19304255335, 99342.94791022992]} +{"id": 1237, "vector": [6418.57637217077, 82807.70649386603, 28617.001302936394, 48213.788783481745, 73306.89721432596, 16226.993309854066, 25391.786351686453, 25884.30032538358, 71529.10380567973, 61092.32982840832, 1758.365055928357, 91792.20375615329, 57356.647572713795, 72201.1430074056, 52738.52606687753, 56780.93551797556, 65997.52109696147, 49800.31891640266, 8309.252101586118, 67847.55767871944, 32172.249285789432, 48391.16057072841, 86345.78457972733, 44190.98041476654, 64196.705543286356, 1902.7147587995175, 21900.059711158316, 9485.277608063048, 74918.33646251628, 76795.02527187263, 97551.26632424328, 38101.650741195815, 24775.890551409295, 97995.18022524806, 89931.67346859808, 5155.5335626163105, 78107.31644108704, 57636.54078994644, 29616.52847935301, 80296.18048326737, 53667.738164810464, 93311.47091800993, 39124.327074444554, 23301.549609760496, 36878.677409769865, 66282.9082095818, 46555.60545650637, 86158.27823711668, 44307.58112614084, 88155.15718202801, 86827.24642016583, 63753.6950864616, 36152.32021564606, 78653.25785197003, 2736.583136358106, 25306.672188486235, 21409.56597551039, 29372.535812706858, 13445.122778933837, 19120.23311010572, 41060.140502490605, 64633.28533509226, 45672.183313461945, 49401.17392752093, 40695.53936016939, 74574.70161711385, 58510.245300472496, 33705.84745662886, 54698.44057175226, 41815.72537102114, 46151.16328483982, 16504.363266991062, 48001.82836269156, 66998.10241990256, 61682.58830716172, 19316.831041692505, 42854.28338694368, 5158.243422095121, 53233.78515947725, 67834.27811976535, 19910.27037269938, 30507.354933064544, 4895.213564649814, 64348.856056534256, 15281.083295464005, 50306.23309078195, 76368.98466888044, 24113.596553911808, 96338.8322901619, 58284.77592449074, 61237.91620189728, 18840.863179865886, 15347.986394650316, 85951.19185358759, 9611.10064368509, 88662.25555489388, 48707.478747772235, 28214.293810539893, 52083.39943441064, 79844.8159083084, 87953.23451226216, 77972.60574526749, 48871.76141790785, 68168.80824342242, 73971.44981458299, 12496.490032859798, 93996.2652702404, 55304.02411242509, 41091.63512207531, 52685.88379924255, 34954.97574447255, 59162.744359208955, 17311.99923595663, 26497.523248942543, 54393.867706064135, 65526.79414351371, 9112.663504468499, 61384.828172825786, 64921.75512862316, 55758.86453263222, 91604.90484889923, 2750.0291203328197, 49571.15106413946, 59284.8690397123, 98574.79508932777, 40351.116564943775, 3697.8297353271405, 15894.243815798025]} +{"id": 847, "vector": [57827.71185353297, 16897.369624839754, 34268.22549655846, 18570.357050990806, 39454.84176042254, 94036.02218353128, 69998.62847230132, 17736.607215615186, 49934.73846574292, 70019.80069959415, 69617.49175701977, 78788.65073780701, 23730.954369157374, 56432.90016099516, 36750.44772758622, 27782.61662778794, 18746.396814736832, 19305.30584013931, 70939.31740148894, 62062.425992405, 9197.122096847854, 4031.8125094512493, 92621.27089596314, 65212.70461769606, 31095.55444853138, 44180.07942535924, 68347.93804278504, 61257.41201835363, 51112.53705702828, 6283.11323477716, 70475.06827263349, 60807.173424220804, 38021.57254268484, 89896.47758430298, 23242.290581857662, 35551.57725984639, 47224.8248989286, 87040.48012068347, 74094.66215945108, 2312.4608006267476, 40000.59353406321, 42650.511075776165, 43985.86486685068, 41724.00295065136, 84582.28644230928, 45784.722500260475, 39882.389873426284, 53787.64462198225, 68442.2339429499, 15651.10440524039, 19024.560908937925, 48108.97001351848, 88277.83786327379, 93617.76207449715, 53231.53761664804, 20149.345359609684, 13951.154455021175, 30018.425184963104, 70238.79379072883, 65537.18839830387, 30231.324141099492, 30907.673696123406, 90032.69211375593, 23643.597565642314, 77491.84483864855, 41971.2112831585, 73131.78545485702, 60392.208943052436, 37049.52279117202, 22993.64856138556, 94257.24879845961, 80537.60684066894, 60294.450451942175, 98413.4242660636, 84140.38042696957, 68699.35775789818, 47093.88572258755, 28813.425630180976, 4362.599829285052, 96992.183924252, 71647.73161828068, 21789.388275542733, 7230.167002152921, 45591.31324789941, 18948.72102928765, 82580.89174175788, 85741.369133577, 83851.29982796233, 72237.89319944377, 50820.27476062661, 48482.92410932229, 67581.38753061289, 73183.44892368316, 48182.2978036224, 23019.485908901337, 95721.36232133835, 17966.6067510009, 46738.07605351673, 40657.195173087566, 80399.1652923877, 67981.04823589849, 64287.15016776195, 79534.19395833298, 35988.95402565065, 39721.75996494581, 81564.09388711968, 27563.327776580038, 11829.025535609771, 91474.05482611744, 15526.114114255817, 74794.0829173527, 68222.89959164982, 88448.41934428694, 80981.65793003881, 2503.897813428413, 12685.456604156465, 63642.01110116043, 84895.6428592604, 59085.77380245955, 64531.417717499884, 8918.463471459037, 94194.21504625306, 26915.075048143157, 9788.145126167925, 78831.3604004072, 40822.21508892299, 49538.93703409724, 20946.645285246545]} +{"id": 1971, "vector": [80602.8477534776, 19485.403733979645, 15071.576651683594, 25296.5827248252, 98058.17019460717, 96694.47457451685, 47094.637899614114, 18589.070708691368, 55454.90223648629, 94634.9879064447, 89911.88723040422, 79310.11280419955, 62446.94514505292, 84602.1572586215, 2028.8735338086617, 10334.422324242587, 19868.02094436334, 292.40571421494946, 87775.05008574613, 39451.18147557687, 20548.181609496794, 19450.094040956457, 27890.50066532921, 70393.9558819816, 34142.007815664496, 88848.86237958395, 80851.02379268673, 16031.696652951156, 66957.75371311305, 75085.23584588809, 91823.69918215481, 44975.17724839066, 21556.227089293545, 9880.600324239242, 38069.622106604475, 45776.38811873746, 48952.39692854702, 78308.04124109518, 25778.119737455352, 87815.54874235834, 54626.148883222006, 57010.05923156428, 27903.572437831757, 26693.520821561313, 19675.836447861795, 19466.301054305703, 20121.940848724207, 41751.63278467584, 57031.49596124859, 15551.275798438513, 91734.6190227014, 23976.558006775816, 46224.77997787977, 48744.54295010393, 34354.816992449145, 29496.489459112407, 17744.836615486947, 19521.80453140926, 58719.46961186728, 68739.73474009546, 99118.73007006246, 92927.74249221182, 43620.87674396402, 28206.528756131123, 85970.88499060633, 72675.30330226046, 5614.8567386618, 49101.039257639655, 1623.3304079259715, 44341.026425171425, 44306.29021196456, 64660.459233372436, 81951.8412732283, 65050.98076122799, 89478.00481317667, 979.4546748404964, 69297.97912181952, 61185.02126002038, 85485.67243446404, 8744.24276212047, 24819.55286691647, 68620.64384440889, 12627.734405087953, 11444.685925619413, 26188.3003987878, 43030.88165085347, 95358.92777065819, 24093.48571637262, 29733.631918408977, 66437.5797202953, 94594.7784302056, 50447.91697882782, 65852.01385887658, 43017.96071470988, 57374.66479669571, 92376.50266725548, 41247.527996950885, 38181.63971570143, 72066.42684704757, 86881.28791125842, 65091.761507972624, 85244.85556916131, 43297.044656296755, 91195.1841716568, 44094.476375760285, 44774.07931321823, 11814.921212426088, 85725.65988558922, 60117.51275036407, 93977.57471003842, 79115.40148509157, 928.050299096439, 41996.2283512722, 13763.77432566449, 83875.31216099364, 61578.86207824902, 54756.74392027341, 57.267886391743076, 29669.218992156708, 72868.77066570768, 2710.6166555021737, 32467.946515260148, 14600.67855639341, 55372.14003957006, 33773.71979100062, 25502.070916562647, 28906.086685956667, 40188.53032361289]} +{"id": 460, "vector": [52066.951085615045, 50553.34621997336, 91982.89780991983, 26259.151900002387, 59719.97324857168, 71665.78926965651, 59287.318596533056, 93381.55716083209, 62253.83380570357, 89788.48874036, 75127.26952520946, 9733.88296686627, 41436.61516309674, 66895.68285068627, 92440.327303335, 768.7115537813538, 91587.28645699489, 36375.74020216049, 97832.04388803223, 8215.362304383156, 8175.399671341921, 55889.74256003781, 93690.14631165718, 1897.1386437800497, 21164.557617827308, 59376.815941445275, 44782.40316899852, 16484.056386972225, 25301.590247404714, 12025.371394919937, 76912.99963615142, 68722.68916016736, 59399.94968359298, 51811.99811734969, 10367.36932569905, 35193.42600895863, 78530.98563748309, 48116.84356713698, 78429.94231686204, 80912.24077538612, 65394.7573670931, 99651.1474764302, 35081.26241243645, 71988.55546821891, 71886.2227432161, 33989.89462222276, 90466.22160557336, 8730.67316418581, 28988.873536718616, 69617.87938773309, 32989.41803396343, 99548.78453017258, 35214.19290250775, 67613.89032159897, 58901.39902967626, 63069.28503774879, 10397.016373986722, 74422.60516907777, 19681.21154076714, 34914.47919385219, 76297.18278399811, 57181.44013738308, 67467.4207331988, 78190.38339194028, 29657.757395759952, 81848.76319178585, 90937.13952125172, 76419.17315601994, 94137.35611359523, 99826.207422148, 66377.35329597368, 85597.51727638909, 52176.395791594085, 92690.64531262123, 68132.11166824907, 95384.75144439752, 81488.26578857223, 8651.840706333747, 84068.01077963819, 87200.03937233899, 48540.75286350976, 81885.81538749475, 3646.58130779506, 81311.82855904594, 24910.13635820718, 97579.33723927179, 30320.163897868533, 59440.3091592939, 53701.02645582986, 92069.63229753893, 53091.036636354926, 71671.66342474472, 45468.461214586896, 91116.9786936713, 58940.38005590332, 60113.72223173883, 33537.20385999718, 68478.04478116657, 23965.586047189103, 43862.897544654865, 79120.83851925931, 63408.916293505215, 37024.34805713005, 80587.92935284048, 23029.75662658623, 55667.97856588618, 5686.754909344538, 35962.30976194109, 50913.04004088696, 41568.49391437421, 49317.1359527195, 61382.54118813019, 67176.65015433081, 88734.05761309418, 11416.942140507213, 23031.78328915676, 18920.28213110438, 74141.5774080211, 15474.918594310095, 40020.8764696967, 2187.7118413431317, 13879.594814284179, 15085.839341553197, 39020.884835818695, 28238.047636789022, 93947.9074410073, 31019.8443475296, 95581.21406041412]} +{"id": 482, "vector": [43302.886878733094, 7784.263804735514, 82588.03844897028, 11854.905637568969, 78295.74436562456, 77269.09233968682, 5669.982709033172, 91179.20006242239, 63581.02018361673, 67867.31046346886, 72771.70060464498, 45983.75179200572, 71395.9309340813, 91768.79927679467, 85850.67385205982, 53090.80315981799, 1760.9114638921897, 71555.7430385252, 52626.52510414899, 32090.63825469368, 92346.05824859558, 50683.522814246906, 27059.152199695225, 99218.41831254607, 43312.13317843115, 60743.23023109678, 33019.483846366005, 93417.76231340438, 21698.03451993516, 43100.82494393781, 36540.5492696052, 38838.0619239322, 91127.5666601751, 55322.80253726081, 40778.970102711806, 66932.0142637464, 15476.627210827699, 64756.129303032336, 16048.852655558943, 24550.490724977604, 99531.07248966224, 25864.319270259795, 75578.63298878516, 97236.40588378228, 61102.15315153098, 31540.357168684142, 91273.80073523278, 48019.179572076, 34509.94299518502, 5601.422432683356, 13420.1584047549, 70641.13091829127, 28335.473416986635, 92045.96451679312, 33998.05608969263, 20116.256351696247, 58574.530454293905, 66448.64280466073, 2965.6337529704533, 92896.37433113321, 63229.9887491873, 91755.69589493325, 2774.5457884058246, 35967.84404030842, 9400.621056132597, 41936.947816827655, 11136.69130387578, 60483.874786481865, 41165.4000852286, 68989.06739431513, 47429.17294635263, 12110.853551179778, 43973.90419166527, 43246.737730962785, 72295.63993784664, 72390.56507623862, 18383.940559375256, 49353.69930639073, 52256.26342975929, 55592.106799033594, 79382.23238232119, 37203.431695161205, 82172.32413789413, 10381.40342695142, 33902.603800346464, 53518.53304766322, 33619.7346041009, 76911.35230057381, 54748.9173545488, 55529.16513154735, 11963.247997910386, 66370.90423969622, 37842.86686800271, 5747.883208542348, 40673.001586069615, 56816.60643494374, 1681.222125607751, 68385.40924774251, 22102.75058919028, 74900.7656432602, 96231.79308865507, 50143.705040017274, 3804.428454862585, 75806.77072964827, 7276.348070446492, 41544.981554357175, 75428.97496764653, 82585.66052085852, 48602.22212038371, 5946.059040680307, 61078.44548963024, 1286.517064389503, 37320.444014559114, 95245.30070653354, 94444.13032256361, 47610.53528111427, 58236.98720093845, 7131.779906180957, 88349.58993644825, 47479.41999944254, 79031.3006319575, 85057.94557066582, 40445.58871776016, 21537.647943005355, 98081.22583363029, 26807.77298541306, 56262.43976839993, 51448.83984468599]} +{"id": 187, "vector": [86718.13064667689, 40142.68403950105, 52851.85046664737, 53553.10382063416, 71888.29437213967, 99337.17722941173, 61197.27410877853, 7778.275812332747, 78226.19799665378, 62486.42217882239, 38373.28759263637, 90277.92138059916, 85976.25393161716, 15396.947543476792, 4183.623199755848, 79195.91918615677, 18117.25451597863, 85240.19591862452, 25952.691181440292, 95225.20177969518, 95240.33779358854, 88762.002663874, 99890.01465601793, 39340.275283767834, 21217.19329632108, 73555.36043231648, 28250.268536258605, 35006.60771367605, 67908.66850711334, 27378.02031264375, 73839.2746875981, 97943.21051015906, 8478.710536883482, 896.8938012424155, 66593.82472720383, 35613.86176636083, 24.13123184165089, 58589.23844370917, 83025.9391992117, 52502.75436622899, 8485.87567465684, 43063.6588939978, 32959.9607684418, 22443.301850035736, 1474.0668783157053, 72104.19209358814, 81208.02686974865, 9849.278234062442, 80766.91714403864, 11915.805121104051, 7037.5055954456475, 94763.2889891833, 6174.941490731933, 2824.3105039916672, 55149.96437081201, 79630.96988974065, 85784.73964085647, 17274.146371765808, 13201.007766933537, 89982.16797370683, 40953.14026843671, 49196.774730293626, 73582.62430409473, 62177.79209584158, 69623.00536609637, 2543.697034827985, 42033.00095241272, 15663.186158966868, 17529.770156178103, 9230.897886829925, 35072.9576917266, 4718.35042276817, 55185.37061232455, 82846.92231595163, 32880.95291178961, 68923.91701984171, 20010.765362786766, 64335.771563852795, 11241.749081952423, 22243.89767303069, 75338.0669610893, 14429.702774163634, 65610.45290571799, 83724.13978222964, 11109.888664864176, 67436.03002348673, 90519.96490919185, 21176.375347042675, 86436.24517577671, 43401.91827115133, 37474.87301945652, 32188.87596921848, 93056.18686488822, 77527.80053221961, 56692.0559748788, 43400.89252216233, 99401.8056267342, 72811.70646621782, 1759.401321004217, 78751.51471521001, 69454.33565851154, 29357.532340144564, 11132.695687568483, 71959.99629972523, 3082.499407137362, 43997.99540750329, 19420.56233706163, 8717.598131112147, 71417.48096162466, 64110.529230264656, 31245.314903482078, 41110.262346449, 95418.42734541217, 85461.48056541439, 91022.90355076114, 65732.39397924511, 94273.55817738286, 31097.67737320516, 65619.60276039327, 25890.517836611114, 98678.82810742638, 35798.312219433195, 32527.932792156822, 52453.14157817729, 34929.232399598666, 31699.431572182977, 78539.52879033868, 61740.91375069728]} +{"id": 1737, "vector": [40984.55142683134, 5897.286818912584, 70515.45818210719, 70732.60873292228, 32943.32470903159, 15908.350971292084, 81639.04549990233, 95978.64047037368, 37836.4885124222, 2240.9741664200624, 40585.883501548116, 25730.59083745527, 1797.6047776866012, 35341.95511654373, 97857.21483904606, 94406.64272411825, 10824.550922198272, 79328.75303274166, 40969.88286921041, 58867.7635996077, 97800.72507277217, 12202.379075417814, 41905.94956561767, 28153.381145242107, 12849.941709474322, 91995.51812582719, 2060.9986115849633, 53925.33529669227, 96881.28274331511, 7726.373907308704, 78219.38426707563, 99945.82253950724, 87201.74710975194, 7819.29664169666, 48545.91838115484, 95241.66908736607, 48688.619447310055, 9636.137825993617, 23684.079338999887, 79341.52146018641, 59723.2935314156, 7972.055742116746, 41595.380233695854, 92924.99143448936, 10056.035787606399, 42536.83100579625, 38814.653976589194, 68688.89649389562, 18830.831104284163, 21280.930298342104, 96406.03498159708, 66502.17575192539, 64870.480642006456, 97410.94344932925, 43057.394522736904, 40544.19654933808, 33843.27818259311, 97367.45305013294, 49328.4134836826, 78982.98889039113, 92943.58495004418, 13764.185449355293, 33362.21717119967, 4201.790508122583, 2087.41531055433, 2501.187874848232, 83123.74333136754, 25173.254400028498, 32296.63252378847, 10835.728355873975, 16939.063105027573, 25410.071758576923, 5583.405120108176, 47315.102294787415, 57246.357181871186, 87545.74245573679, 78021.12378635076, 5065.963316227351, 21238.49603284539, 65865.664914312, 76441.09978832988, 94313.23389828186, 40091.20720137642, 73714.03340948392, 15531.990624439806, 80751.14741200725, 27809.030476635966, 12498.60227822619, 4082.9114220181427, 97827.82217236087, 10359.702598063825, 62587.32597333473, 36298.2360492735, 44962.52579227826, 31584.699718070508, 53140.62203479236, 90353.56423988941, 80659.64364302436, 81614.95508812615, 84827.09097234745, 81053.68943345212, 80895.95873886591, 9821.916403401065, 20051.946701025638, 94486.62331931712, 23006.20474748356, 89180.39870274595, 41609.27430570087, 74861.28110380398, 43445.543187265204, 73072.70906184534, 82832.77528387426, 65464.10244511154, 89306.53813902149, 88347.7634883515, 53152.786429852014, 94625.7618373531, 5418.682700947508, 43529.25968322524, 46928.64788552626, 58503.96839700251, 41373.81867482518, 52377.01070735617, 62030.12989865798, 48208.42744834981, 83403.8013973021, 33851.741849462305, 4496.769299429737]} +{"id": 494, "vector": [59884.55912211165, 18946.29469091471, 14775.943966431081, 28401.22517753876, 23637.75966522359, 33613.34876489669, 62098.415284580435, 62961.14225891996, 34995.30547713657, 9893.031280620267, 15456.693989392445, 57685.31481910172, 91240.18425157956, 81007.19578982773, 5774.02004920522, 47659.802022013995, 19337.98678983677, 93000.34349202595, 81125.47212263738, 49881.553392198766, 48737.10533986791, 37384.777513676185, 60009.728893605585, 75964.91543150731, 66660.85284994211, 56107.18423025879, 3357.1828220932853, 3104.6991223545374, 18674.968213362354, 98064.94106594972, 56368.60648104368, 32968.1427346467, 41622.1247301092, 49858.47016855183, 87242.29141241469, 1140.5167341499766, 50726.13924688196, 20900.382042680998, 43999.37321259042, 59756.84756170078, 87927.65068090257, 19471.247066681706, 73693.24942818197, 11911.269999267493, 13631.769365262458, 16464.280046174517, 37870.464699772245, 28391.878023913698, 93211.42158867298, 58131.72713684658, 61916.04151537999, 36008.24597034753, 71016.420567309, 24016.040765620473, 73628.69630822566, 4195.042095041701, 89661.90510626457, 96554.9313732213, 45582.21268368582, 11846.458553037965, 88491.88355894397, 57704.29890569151, 32728.792132843744, 2242.4406560391576, 88824.11755347141, 78062.53201347144, 82240.42947504928, 56308.48688551549, 58315.42899271277, 73097.47734654746, 20201.604863407407, 4823.230195531958, 38045.83413188032, 61342.09959407484, 43635.155510739154, 15519.04339767911, 30045.42470136198, 91710.46559645706, 66143.71387879454, 61705.51943837816, 14317.540017719277, 70679.15312077042, 70508.69338884273, 56345.90186345413, 62465.01643977984, 48710.53159448917, 13480.685396807125, 38930.27775022246, 81424.75197392395, 71772.2961241758, 65541.26830685922, 103.31340563042302, 15153.305452878341, 23375.12355326099, 82437.43320446393, 46547.11334349504, 20241.59375547765, 1303.4297443024423, 56216.548854677196, 4501.979993574212, 26196.16010421155, 63299.65111857957, 84336.18754449354, 40927.49376198163, 66790.02904292993, 61857.251446286755, 78378.18898391367, 70875.72508502583, 12731.085856286372, 92715.92409167011, 12203.541349304525, 35596.42421014947, 84468.28037268642, 25949.045092509692, 90390.03062137846, 44639.56272388585, 49708.87849833648, 62162.58302041859, 48824.35055993757, 98367.43778495798, 83378.06640762062, 18975.648530188384, 24883.768628604575, 12269.456116071697, 14116.895145863762, 24726.27197942061, 172.09455186508737, 72411.98322900951]} +{"id": 257, "vector": [410.58817294240544, 98007.97187262666, 85834.00452713988, 84422.54379512086, 55854.67921694074, 83512.64442841653, 29944.521285113125, 20625.309166892792, 46823.1198586099, 60771.60077811776, 61250.53050870732, 50625.52603800249, 66832.13344515846, 61114.70285393517, 84600.25291905737, 49311.22244813524, 56288.74827097189, 35419.4928731686, 52611.441258781975, 94724.42089387834, 49120.44750875246, 64801.96253986754, 99788.2380061444, 76045.72322058321, 72273.97299388588, 77160.42666409862, 8525.271978934024, 81834.61318848298, 75580.29748569355, 21798.57768900102, 70961.2076651842, 66270.48294925729, 53201.61256865606, 67869.63125240363, 50696.21541875916, 38215.56994614193, 47725.52436072696, 91246.41821120928, 13477.500553161059, 26320.311557277808, 52533.45504668412, 3327.1466350839664, 42575.250245753115, 42637.9436045476, 24713.48315460714, 40989.341223058465, 99856.21183249353, 8473.802206654978, 81150.37595541723, 38100.0155945304, 52458.473687310514, 67221.55496277609, 50939.2404551137, 70001.40494413637, 27636.746297193393, 55416.86896912571, 24277.123401315515, 25605.62810126934, 81107.54072967962, 17310.403721284263, 90402.48006671928, 48785.64102399732, 99823.71214090161, 10963.59893658524, 15074.221148231081, 39387.09329075858, 9624.037523566874, 73486.46850011952, 64972.523086508845, 14705.150749748253, 85752.02889093297, 47117.34679827435, 94119.68715349908, 81883.49513118035, 85830.59271346945, 8975.83624993753, 8746.83556628535, 34041.203788910614, 89202.01423905995, 50674.412813982846, 79403.28884765205, 58348.348570309805, 52332.683933458815, 97854.85918596401, 20118.53120458539, 77983.65569611448, 52982.309117733814, 62475.59479210071, 48767.926037139056, 35371.7562338846, 49075.915877833766, 49751.35360840618, 21845.602984264788, 40711.86667982442, 85266.38111632301, 5950.631840610221, 9984.30122978199, 40704.39165514995, 84104.41277934263, 22736.993299802754, 61869.21900889692, 86960.7772390967, 80575.40666340243, 63276.62495771986, 50028.91124929229, 44380.1890211866, 3115.810741283176, 27655.035273498983, 31071.50778188841, 74283.73907565823, 39695.038484388024, 82797.35251718458, 9178.219652481379, 3773.3865826945043, 24485.87288737083, 64232.35449734279, 87154.25706456418, 85409.23124530722, 98802.94480731504, 44015.25207089192, 29947.32794310945, 78578.19553800175, 26389.766131681503, 52299.24396535939, 42853.06834872407, 63397.65144528214, 99659.69337477454, 42632.80410785924]} +{"id": 1426, "vector": [71915.68168883231, 74159.58642013512, 88093.28120061872, 62194.66551918958, 51311.83616017142, 20080.37686566353, 15377.47016452139, 12344.508347803096, 17261.434563741197, 81454.7248248676, 2758.976766882282, 12648.639741171708, 46602.38268494578, 21875.98712882909, 80736.51164347123, 98983.7883149346, 21808.733347738096, 98266.71019813188, 34459.37642036716, 46267.38610059342, 26479.170617829695, 93194.16844552192, 26283.778835005967, 95651.11817265885, 55953.327119850714, 5885.51785264958, 87305.45321107327, 5752.204110817715, 32689.800168770456, 96840.8747806663, 83379.66055259597, 36272.37183258757, 72810.32275068529, 94083.15443954988, 46405.42642456482, 96794.75498495274, 62698.485081888124, 51677.9722157614, 34822.33708450049, 43871.90797331528, 6990.373625111568, 63072.67225604203, 73982.93894555568, 79354.18244870265, 84958.47310821846, 92359.21030850027, 10652.397658323975, 77969.07243590616, 13578.042652699607, 46339.786009324715, 86668.59518469719, 34164.76159629958, 34294.61069509832, 68281.17763300016, 64346.152103515305, 21585.88089043816, 61984.715084034426, 22323.385551705476, 1304.8882163660846, 68898.15957932489, 78415.64221292286, 113.28641738812539, 35103.78791257999, 2937.2981038661705, 72146.02488027282, 36744.44001868411, 12384.25782904775, 40693.43964398595, 92257.91960700255, 79546.02712461652, 24817.416991286256, 58160.820249785116, 93968.85246829028, 48285.88441555082, 42852.49166938603, 27618.519363154348, 20957.712328557587, 99507.78588076725, 20104.55732411659, 96715.75748376266, 86836.66943796707, 52470.92743147802, 99360.8862549221, 83473.04110451335, 8506.220488436122, 86013.13968323822, 12286.505980799157, 10502.669746539106, 90052.40720084203, 55658.18696480146, 59887.838885035024, 52004.66397974862, 7790.13028005725, 43802.37261766444, 50872.45109482279, 36062.98976795974, 11066.033338702553, 94728.55527541286, 24463.731230140475, 11063.404607825889, 54352.182981961414, 2914.0411865386404, 39627.939505242684, 83258.84902561642, 70859.00798372962, 23077.98968046868, 71384.95255199057, 86554.28062336426, 99004.19362776807, 83729.52476794917, 59966.71930447951, 2591.864899357166, 1554.4399548508502, 95088.47763029778, 96337.96130014156, 79131.06326424141, 7334.808357446387, 84232.95797409958, 7467.458393627502, 27445.027586393346, 343.53664219012313, 14920.535988074656, 90629.4718683196, 12691.559132097718, 34428.8195045654, 38068.31369898783, 21531.95003958177, 17565.850779998516]} +{"id": 2109, "vector": [58937.96481424085, 57194.35214714087, 40650.65869079654, 27766.02640902741, 53287.54484172131, 96937.88972499291, 7046.780213934922, 6365.615735279262, 13913.233964292725, 32774.498872901226, 3854.9874445366504, 45325.59373746685, 51844.10835962937, 89484.66206373216, 5602.1325848314655, 41943.92322484906, 86742.5211868281, 98029.22123616502, 57310.31320468405, 53165.09879925847, 23327.250226419637, 28541.93442077443, 66594.33099582541, 73914.85904702937, 84862.66745266425, 15052.337546682325, 51304.52643059432, 1748.0400765903182, 48455.360817058514, 78059.74846506116, 18959.00295278956, 31693.71397363392, 19671.92415350826, 33583.54240183887, 78107.8153902128, 64773.56968372322, 26267.695539599146, 74173.7681560242, 69383.235385215, 91945.65766604504, 27340.471266449094, 15752.942028051242, 20602.729986843828, 84500.60160611634, 98123.64148344638, 71763.05083212131, 90034.72678562823, 90888.92022658093, 27597.380271033955, 51762.58029112459, 10434.065962166971, 98706.63032196397, 93932.43407862665, 21328.058474485144, 19817.05278601251, 262.7816750042533, 89511.84304043493, 65894.74860786136, 32457.081958669565, 19529.50477678803, 14559.428092684622, 87877.40659299506, 96580.15275051986, 83905.2420166496, 85326.24568112912, 35392.1226200325, 55999.61700372595, 54318.014783346654, 5289.654325949711, 49678.155096581686, 42196.59577916343, 88751.48620369054, 42205.05299690963, 79991.30868047316, 72909.54464248926, 81449.3231550636, 58543.56529432, 240.00330577562679, 7413.653661960995, 68407.17817478087, 18341.951932177002, 7323.081047010549, 2285.790088493367, 91438.44946558915, 41856.53215881816, 92694.17180444526, 38516.34605449825, 22113.160444530065, 44062.53878950574, 11841.730039103537, 2544.554598501969, 40894.70725745904, 31299.96215133686, 51772.77805556354, 39835.28063077094, 8489.225424626435, 8207.571105599354, 72939.83749837847, 66244.71571195431, 80968.71340965643, 41152.19809702883, 68460.97951949397, 35439.56826799574, 37807.1899057143, 20281.75719430303, 88064.37865029302, 3507.1729462469725, 39313.32281785212, 93290.46779130817, 12543.745756034641, 44400.55007583364, 40780.856542207875, 34820.61711880329, 56200.60723438183, 95766.49856269982, 87375.72143615018, 94903.40672923428, 5012.73488459254, 85931.01633917821, 37637.800421478554, 9102.064990275294, 45436.51423717779, 61675.3678194513, 28348.91620343607, 55515.19515703952, 78745.87977216899, 13367.14557317491, 47974.74005232595]} +{"id": 169, "vector": [88201.32116772977, 81514.9316403656, 62145.8730620769, 75688.24456656295, 66358.79027736891, 33376.7525959565, 8211.134472545911, 31787.089562828198, 20343.45027012607, 68568.68655344345, 9053.557314948192, 18935.97373151529, 12838.121097612044, 44754.710161840885, 52167.90901789376, 20575.274755016548, 22411.544550299568, 1909.4795265185649, 45866.11279932555, 14257.774484488216, 84752.69652721142, 46525.462375826966, 44433.49771819591, 83548.64333230323, 59022.77796569748, 98388.471569333, 82353.0817035329, 26255.352782515463, 25601.48808924131, 54831.98396353894, 51897.63277283027, 13998.00898047563, 17635.060736816442, 52618.029541262644, 93564.8194625189, 71168.4504712647, 4501.492005383767, 51839.53725097866, 48228.9279337658, 48989.34189626454, 44356.96084867414, 26815.997770539736, 87295.32358409825, 83774.24316745707, 57209.88993091751, 53989.73268659224, 44176.424078430355, 20866.53068868355, 67798.84527702525, 47418.68281106605, 96120.18140046626, 87770.6740754781, 24187.389654739243, 59913.97322581765, 23604.909886365178, 37102.848766916795, 95875.6608674044, 99994.29847779259, 20575.936699708407, 57330.47852446106, 89665.46477800711, 67553.26545224803, 337.75517160613555, 34933.17105865237, 43615.38372556768, 92526.03314310634, 78021.68912875846, 32304.196656364715, 80507.39146067148, 4057.9396931981205, 76812.2318093469, 15676.35712915626, 81258.86744712584, 77826.24722191761, 91178.53026088403, 2257.6102925162213, 34624.5362949101, 7504.8631826567735, 71247.39292961963, 78104.27677187404, 49290.476294329834, 87069.39126142472, 93901.02488852228, 58294.85984079541, 3844.3869259165876, 74889.32219040727, 86915.68528349324, 56529.97174418715, 89583.45359191591, 90860.62584718288, 59691.16817286328, 20455.11559099097, 80595.70364318173, 1520.4140297179204, 11581.818992048598, 83889.85289137969, 98519.01676189066, 53046.66831515837, 29086.90168971746, 93714.08411373686, 52158.35398162557, 62209.929904202385, 68144.03348744487, 83340.90692790406, 76041.84401584184, 63641.67523853839, 50102.970922859844, 54920.40968245467, 8068.37338540749, 33803.394970185676, 15051.275141350907, 96934.04209297556, 89658.8909056588, 73831.52629597056, 14973.944638899538, 82592.14396571743, 50900.658270043255, 57127.03068537449, 7566.358769304759, 95034.7008451205, 68951.21809301228, 6814.3496277035865, 80262.63040998997, 63366.064773831386, 83203.71457137339, 49141.74291949682, 18440.789080737464, 61266.72381816203]} +{"id": 261, "vector": [10935.088596943877, 40719.10728518459, 39967.74124237677, 83411.79882747724, 2271.433908357867, 60892.13949048911, 69265.09308651675, 80711.53605450994, 72582.6382893769, 72563.88489534086, 24193.351418138063, 13486.809995468007, 9976.773137324835, 74917.83802439687, 13393.617392230894, 75584.04106891043, 16837.708277639762, 49827.61073989235, 66852.4532278819, 33311.55488692841, 28327.404303129544, 56867.939689999446, 96056.97754874946, 90996.71246466546, 52309.90052241505, 32829.49774463394, 12503.335569551233, 2782.78188219806, 54998.15610578761, 96916.66420079951, 92162.76046103612, 17204.50777391864, 25418.797758195833, 26313.617383593835, 39468.17504185888, 74442.18076921137, 99289.63735952564, 53922.13195592222, 22082.76722966729, 70313.14749448898, 55998.55015461345, 84842.11696901527, 64147.737049797426, 67271.02694103235, 49288.00631916613, 10498.970252633799, 69936.17847430822, 27744.834395973296, 29063.8871555573, 2187.915904126536, 45431.760044003524, 19844.4303742766, 5447.794142216245, 12250.631565528913, 49438.47737818205, 5347.060535650783, 8816.82296598364, 24063.83671413527, 89772.8857260522, 21920.295436362492, 65118.230608717764, 81773.45450151002, 76959.9426551533, 68824.52360810067, 13616.160989964832, 75201.35569830402, 24855.539845793628, 60385.3148489081, 59442.33076030116, 93723.01656885834, 13676.154606236036, 18348.543017175412, 96701.29401313915, 79900.91103511731, 96389.08702038295, 84580.38330157213, 43016.75805887023, 43759.46270220685, 26543.643059926537, 1560.7450842568983, 8735.58473122491, 94393.12575347415, 74515.60423175935, 44235.076159864606, 62358.364676941324, 33546.132107064, 15256.022633806377, 16578.103300600076, 26810.912584107584, 78690.64049212728, 2995.4658548725033, 82945.75450520418, 59502.76567892989, 96092.44465410337, 39322.869538170504, 23955.246368518758, 63350.36068305946, 13548.086968270423, 91506.94510655933, 90634.3184105608, 54867.97650344785, 21660.453633777943, 92064.66989492952, 80429.98242303618, 22640.068199637142, 61533.98050320812, 84583.19328213451, 27013.396665863365, 99751.01891976941, 35775.72943032298, 90850.68226727324, 95927.26978730143, 63973.22799962755, 16339.061413487232, 7092.218369993219, 4724.303877218305, 47932.19037639327, 69922.61334428686, 82082.80083590219, 41465.09699562663, 1542.1971189687356, 28016.47639891297, 54335.1336110885, 40533.605553147725, 48282.6102194052, 21287.49037342741, 17368.716925874694, 82872.88410523493]} +{"id": 880, "vector": [35623.37268146835, 40331.310933100925, 25267.547782407273, 90159.30544780455, 22241.67177310038, 21960.22446178082, 6723.470510041141, 5452.042918298316, 45887.428131603214, 60624.549716865, 39776.470893178775, 94798.62893529578, 9689.403381008698, 47278.555282726986, 67267.39410347232, 89967.40636130258, 30485.995808827392, 54203.073966521755, 18662.895590114615, 22763.482887041042, 52660.185379024195, 26534.662168009283, 91376.44262727037, 49526.06072633221, 89056.973985217, 70810.82627208145, 46222.96311759771, 4338.283905083018, 3285.713747363639, 33816.91076809261, 79620.51450371384, 93503.59481467739, 73615.08602144259, 49926.049631745125, 58020.73800022261, 23165.086035381166, 74184.85970740451, 11878.539755565254, 71821.54228921093, 78173.98377113562, 78112.6315021966, 99555.60898789908, 95385.3451885824, 30129.173970190324, 81811.1418991883, 15013.860279634338, 2352.708372025891, 17658.001413291757, 3707.637691808452, 73573.29880815564, 491.41921941260324, 92230.54341730339, 548.8895474020871, 54540.90659591081, 63485.61356921096, 63211.95086660736, 7883.984017210677, 47963.09469532968, 65414.38051253432, 39138.05886597174, 87188.92271981834, 28398.28916808079, 31333.239976719975, 97407.33114903235, 73469.72393118015, 32205.858584645808, 26806.530540273045, 92535.89740699469, 22885.933004953564, 75796.96157319065, 85912.56258486545, 99854.34007551866, 92778.25575154624, 43979.907809420816, 84999.55034218701, 39048.51289560378, 53711.4382875084, 35928.821903766984, 76570.51404327947, 89757.20447048685, 17306.815573002543, 86847.32236467698, 58861.20445350377, 45898.99512914393, 39430.08468950378, 19231.768717322262, 57908.4889054859, 72654.92583012141, 51439.770004592654, 35749.93158704016, 3788.553208758272, 92556.77975289752, 63587.57694041066, 74131.1829968026, 53246.44844992913, 87381.94325255214, 16954.796202243815, 5826.229716511743, 27128.138624400977, 95849.66268749315, 26370.019366179255, 10774.757209450747, 43831.78784440413, 41899.96235500198, 58050.63882064195, 92452.90765637162, 25888.207428386257, 9514.615798614945, 22312.257891692956, 55608.412093384686, 49919.690805167, 29345.472009012687, 23773.328611211007, 99576.42575425653, 74123.26604158278, 12398.745337254091, 39952.6792188314, 5892.000528300135, 79003.35297994672, 18760.292656648635, 75681.6099614796, 23133.1859563498, 56622.1081632799, 70002.89090359714, 92779.40838262768, 78497.41924628077, 74152.92218857683, 29928.7312255625]} +{"id": 1229, "vector": [13701.232737391978, 12109.9707848774, 84558.45467255039, 84390.01411509953, 58169.77948682924, 70921.90187525052, 32904.63199849638, 1737.889020694916, 16223.314287397972, 5139.188371135261, 43757.90764580897, 46553.34943073227, 77467.0865863698, 22370.996275348363, 4281.24751501302, 66936.04409393153, 49163.22035869435, 9494.352261513894, 44131.943631867354, 28598.495363505437, 40561.93526866847, 4149.060083016787, 49765.113639301606, 45423.580126580455, 60020.32709378497, 51201.47042005373, 30645.918106454108, 15265.91310397074, 42204.81866152328, 48156.93632103757, 33257.71777234341, 29832.406799201373, 27553.491061233526, 28497.75477277552, 92006.4059053217, 99964.30581149358, 9108.703196453605, 3452.951469060095, 16814.52301958034, 83433.51866733727, 86327.66550289239, 38629.298536241666, 74903.60932535141, 5651.2757479634465, 14098.982739081577, 18257.90770177067, 32305.59169764178, 49036.84728104072, 45070.018245519896, 57371.67367687034, 7738.304205352098, 1370.2237567531638, 78216.91100754212, 49254.050742444655, 73653.91903119624, 40297.50370504901, 19424.417004354178, 38268.315675965845, 97308.8084525319, 67052.5932920801, 13815.758568120096, 34073.20284028903, 18454.49065484337, 881.3616793842959, 31572.910400615638, 99529.34105556237, 87381.13216545199, 9236.998981414612, 28849.612333253095, 2482.9950121124434, 26542.59118005957, 88030.08042117821, 47630.264245191356, 59377.464160940486, 39389.215015416026, 38925.77897417077, 66102.11497702381, 40777.83794239781, 30516.017814816198, 90209.3549814444, 77274.82996084454, 19431.14194451534, 55147.93726089627, 13677.718991409516, 34468.51713798279, 45687.17774292967, 95517.86256430688, 65320.16868960361, 53876.70012611294, 2380.4916026534784, 82072.67206701366, 53905.93817384946, 22953.41805536856, 17606.735903838056, 7718.100436389108, 39554.35521478207, 50209.59958578353, 51899.185856743105, 69839.13769617831, 20223.26718796501, 10597.93136123659, 63820.42787111608, 10287.307273351653, 52496.18653541861, 15219.597806314943, 48415.980360525544, 99681.91275994378, 67531.56781789307, 98459.4740262545, 18998.911903935346, 20692.798432922034, 54269.087966466, 82841.83228296576, 71185.28167684017, 52427.18857452958, 95700.94371852533, 63142.90612840392, 75172.88628904257, 3380.149102655694, 23105.672899169705, 40479.72315216386, 5441.041981518335, 8176.473301715537, 67990.6185770268, 47719.73981918972, 26592.581243712164, 77524.66014874072, 82122.50520073567]} +{"id": 1352, "vector": [89764.72819918761, 55553.38020896821, 12700.845080881185, 53860.25336875373, 92942.68940142475, 29460.544365364716, 70468.38374984926, 32495.58951685506, 72991.93067920719, 79364.10871755648, 63424.66484250714, 18862.790638328297, 25097.970110343405, 66420.38633073398, 49486.30641505832, 71044.14584535109, 57096.2316356299, 71591.21444933796, 55086.024278903176, 47065.22448068621, 10855.704259287702, 13017.504787667678, 36938.42359704919, 3741.8773908938997, 18971.835619422916, 37318.872662559574, 61390.851669837444, 58130.868556630674, 55205.97623385833, 49255.04527038661, 26564.66814181597, 51333.233834511804, 95754.21525886611, 83098.85090999717, 78339.03339674615, 19087.62950650058, 22974.412149158285, 79566.74747430667, 58787.28249502632, 59482.43784118066, 43881.71730335163, 3768.1934149281783, 20177.60867619477, 1868.0269859076448, 8905.304227560451, 76305.27754424444, 28977.98580183155, 96871.74006892652, 58406.58163621824, 46799.91571860187, 46796.479303147586, 15674.731405695975, 67773.84750195645, 83831.23586200328, 44233.82384271478, 69691.0271509296, 9536.688993406884, 58360.02105770521, 12038.774983785372, 59197.308626988975, 93657.12318265496, 98764.90692734178, 22177.046501298926, 40696.500436433525, 71128.96902269065, 75518.4696270306, 81731.26824373174, 93259.4337050989, 38779.243377561164, 31456.971010605906, 23117.353327466517, 53108.21562861576, 79831.36160804321, 77477.81585365561, 47049.632783179826, 54592.002560329136, 44422.46779914842, 87111.90363445101, 65403.122319582566, 74460.98465848989, 93024.41320332016, 21410.825752427652, 38246.08718999476, 57417.812309617955, 43501.612375191035, 7271.565837605387, 39979.81234792557, 31520.979549827298, 77729.63422138226, 68150.94857543228, 65074.22841193929, 36613.022917141316, 92053.60973014064, 36663.038496690984, 60886.10302658588, 52143.050697878876, 69698.29650909596, 63066.359331810105, 51968.80514659916, 60689.41134147963, 1010.7767776962651, 42324.75116452888, 10486.48019337488, 69053.11770377899, 21470.35232228709, 55293.98002707872, 97461.54625747862, 6879.213235396475, 97026.19989807231, 371.07225250867515, 90984.22344249529, 7519.975592778261, 3381.3356882419375, 21099.830934699083, 73520.9580073288, 51453.48455574095, 75078.77733553751, 60127.08382259207, 45519.14631047373, 6187.722864362843, 60899.06704436815, 29173.333322591956, 66363.61218194256, 16824.159313702257, 76825.60096084992, 28219.157169214348, 4277.744758652868, 65757.7713336952]} +{"id": 1708, "vector": [43075.9273461237, 15643.949129023338, 57693.84861897329, 21833.605032907664, 44033.6972887583, 83995.71248514028, 46889.35033755978, 35271.44922999077, 45406.46020169512, 34768.50408031583, 5766.254239114188, 66974.88741162847, 31751.856298511673, 99029.77310171281, 49472.402985816574, 47835.84158385302, 91991.83086096066, 55067.071927470126, 28360.739146398973, 36512.5310417803, 4296.8814738371775, 73209.80777118276, 96623.42540736188, 60854.77672398496, 7542.305600846966, 25144.689599534653, 71966.88045048362, 51885.67562555817, 59642.20609648075, 85113.21048890424, 9155.252663719082, 78303.71580835257, 69787.67423729193, 94966.07611663724, 68618.30445871946, 2212.912906684117, 73655.33267177996, 6912.651951287574, 22647.04715082606, 75861.45527514647, 18196.19870509288, 29240.200964127016, 83431.62414819919, 21107.833075952487, 60837.7626160617, 13807.683253598401, 54682.53265445204, 51595.86942971567, 78219.0546510982, 51920.43881939428, 93822.45058851986, 62407.37834569138, 27862.131675592438, 57403.451677739846, 85013.9816535552, 22982.998807217937, 2730.291656586803, 87118.73227587741, 49429.213380813686, 79065.79706098417, 23935.47277289042, 93847.8332040942, 68496.83848150543, 39662.21954592054, 34290.14496909514, 95735.39640332566, 64357.599823590805, 55223.6119722466, 4533.804500536487, 78163.58972895653, 69192.8111961838, 60739.984783525404, 51637.36825979857, 85287.87589791033, 61289.96610462108, 19765.608851999426, 99648.71230471272, 91716.48905934159, 38074.26484510827, 31183.312986223344, 22802.445394446713, 25028.432742880057, 68688.37204669576, 71374.71104929411, 33581.09576692233, 96151.42659370783, 25906.7232155612, 61032.862953326614, 33606.17054641107, 34720.922768840624, 46380.44247270483, 45069.0754884517, 89005.13553894886, 77571.01586502348, 24018.20317082498, 81314.89052550701, 17563.240989437934, 53264.67535594753, 71670.3726200451, 43013.020576278774, 99926.78689703715, 66202.35986864212, 43769.5377411082, 42772.998802206894, 30491.088156080015, 75201.41937389267, 59400.989199360156, 92320.17493818697, 44594.25088189084, 77752.54603273336, 74314.73468487867, 11951.709390572529, 91639.94118033082, 56187.8121361716, 67639.57674921329, 10969.644840573756, 40530.75409091659, 36750.30342536857, 80411.9876317377, 76084.13645735996, 80643.90077302263, 45384.47724659483, 72231.86324131097, 43275.774035023554, 66111.69777740088, 76570.73502592601, 61057.42254547684, 4446.165866014584]} +{"id": 458, "vector": [33440.04812541059, 15467.647913840143, 17613.24888436774, 49744.683699272915, 40074.41703345972, 49749.108787366684, 44250.71508317566, 44776.482268408094, 94779.62887160637, 34652.15976834318, 36058.38207180322, 26953.106717028353, 66443.48109366014, 26164.953437160166, 32954.28731867287, 9295.85508824281, 62424.20354411997, 76973.12991086833, 51437.82068391256, 51701.434550309335, 68444.27517215362, 4260.751236370875, 40006.803519285684, 77027.66211319309, 59371.49533934269, 51927.19320989298, 20533.73178634661, 95852.84002486696, 24879.876842465797, 22175.188404707525, 36206.967732907644, 20058.953175076545, 62235.25711274493, 20578.757100273215, 61556.540936930134, 2132.63327244958, 41372.714173644796, 87385.49608011599, 42262.08736397262, 47586.51855439275, 50076.73069082327, 58513.19497542422, 7934.2017753118, 37013.68815814213, 7303.497147740401, 25008.337897326095, 52994.311117045014, 86552.52337837357, 3320.5154173197584, 76359.26248687436, 29128.90773762513, 8635.763106510252, 23312.910754805394, 66183.7461877787, 38981.68610043887, 96267.3945739151, 31201.211608227353, 63256.807162635785, 38133.61488234715, 61078.49477517256, 33244.73283149394, 66900.56177794216, 9939.615669871426, 80822.95270270245, 26463.16680719375, 1347.0190357719569, 45429.11403220861, 62861.6200710151, 59679.53027857176, 70044.83912599625, 63720.39409810095, 44124.271109621215, 67369.43147726549, 41973.355343570314, 72831.5753474007, 71327.69802078402, 65960.62774231964, 843.7793731619458, 53950.72792638298, 54294.768702864116, 98811.55702577159, 10441.063355613978, 76305.74319996331, 76407.25387116424, 22319.251010171127, 13418.007173409342, 43378.89844371283, 52253.19899090605, 51302.03510411249, 38487.584754902186, 17325.117629861943, 24995.428411994726, 23749.891050258575, 97745.34429048993, 95737.05171927236, 54467.374486646724, 19122.012394802547, 22845.646964290234, 80591.69153883304, 68492.4863524, 54187.024377622336, 32558.742540938478, 99095.23814243921, 54076.756925574686, 29038.431222018036, 60109.054615341316, 7876.031351452739, 84137.65839801809, 8599.59999982276, 62495.371249605494, 21731.784683479284, 41285.70219820204, 37961.48412462916, 94529.76274975126, 26459.27003049238, 28923.269213393443, 45505.63378717018, 12803.614264462638, 13149.675839869013, 45377.992050840796, 48310.65054340162, 93495.68611626413, 28447.600088403113, 92479.05341405216, 33178.11105659208, 48098.50720324259, 6097.632377187112, 84721.49699768917]} +{"id": 235, "vector": [84352.07186519714, 57309.6383436629, 70939.81180615029, 74021.97872590201, 75383.52894709623, 61706.05960081481, 52038.10205242274, 62047.600830275616, 52213.7616901923, 24041.000973614548, 37022.248397880256, 56397.89425500209, 79189.8749899738, 15678.8926169386, 19075.937896133364, 55140.898830446036, 59554.78784664357, 34958.326976836564, 98528.72247126889, 10745.520920380513, 34507.90953946876, 39831.002191006635, 80029.35384667655, 90973.49971644691, 11407.068300439927, 93584.01625952547, 31032.43955401962, 71475.005992183, 13040.671211534138, 35307.74643626242, 27455.734160730804, 87723.16837663605, 13074.136086672728, 97114.32282364373, 97029.04328425026, 45811.51936333523, 75676.87629358958, 69949.76831398791, 42449.19443772054, 52024.246567039925, 52080.28478999206, 40358.2718547599, 57367.56290033762, 79495.30598797035, 89279.7224512615, 57013.07839992642, 41239.44194185095, 74132.60994659891, 51218.0553008651, 62635.58478445827, 83690.470355561, 24387.632123747084, 85891.03469564697, 24004.021133067075, 58116.7129224768, 56184.41484330988, 7307.560450659123, 90232.59415185223, 17690.555319279432, 26612.44157627398, 99044.131443095, 23158.5795073316, 84671.56819459675, 44939.197531047634, 54114.748782080955, 49305.41973928699, 66916.44412404316, 80569.66876233337, 98685.39444690957, 55714.15833028093, 22134.78357626003, 65294.25733797245, 80200.46938537348, 46105.86075960624, 55966.65880949729, 42762.563440776714, 77033.05201276668, 24498.48803267456, 17115.27608861424, 83375.78769805146, 68691.89755162544, 49587.16515284221, 45113.7245197066, 61718.55125707818, 71454.98811976763, 75557.20758817207, 40072.50581127294, 31712.338780163394, 69363.37963067522, 15047.665479716754, 41009.18023732203, 18429.248034290056, 51814.872641356094, 84606.93672565109, 66079.67940340628, 37020.11955649901, 39593.967101713955, 86995.82154354731, 42841.19734203633, 41305.6825957241, 84383.45395956513, 81035.3553538914, 43080.94853953884, 51995.75798248634, 88750.5240452356, 45947.33792739417, 91514.63732091826, 99815.35242243359, 67714.38657187328, 69189.82786889332, 70255.020093972, 35952.13639472173, 146.32123182811708, 14444.99530775759, 59841.50827258059, 17145.135077168437, 73300.21715866515, 53995.51622721014, 85746.23821245242, 66799.85013568723, 92111.57973922421, 34369.67076616773, 40333.82632077833, 50582.61777069607, 26551.696369058107, 93514.95377547195, 69232.2747210488, 14046.302610718663]} +{"id": 183, "vector": [45680.52542061028, 90601.1644374573, 58972.793420335714, 49813.67911269824, 49851.478706361726, 86825.53252960266, 19295.92361646577, 33669.24547495209, 1817.4290643716229, 20473.043041932604, 93019.63883176488, 34249.04871885463, 85411.70428687155, 56301.2463449302, 92773.50771364613, 51171.18923752373, 94206.75993990515, 54206.209885643875, 41417.80634093085, 54941.286215231965, 37278.47722898541, 5569.564281952844, 78726.70725780682, 60912.774967139514, 78792.85725656332, 50958.267597869446, 14143.039055935513, 5025.433994092754, 59526.910672878716, 90777.80689581702, 66812.55987040664, 7635.653223973593, 51799.658864709985, 33646.937669264174, 94821.22590931821, 67238.29352337576, 77846.79466932645, 99377.70061908611, 16818.39819616169, 32531.919191740744, 39022.18245709565, 78550.1383566729, 57666.56814157271, 1844.098227785551, 34347.34349606517, 30092.24094341153, 383.12801316779945, 34910.803236302876, 88084.73562237373, 63968.64757617784, 76388.494578705, 90335.56436757848, 15154.550914911802, 53128.924801637746, 52509.42226694086, 17131.19107357758, 87740.32146549273, 2116.887521493682, 51940.08930397629, 95237.261089756, 15685.11972179889, 43351.05965826348, 15745.116977524909, 59970.70966771549, 20204.57573351766, 22719.81972402103, 60438.28496569815, 74191.25972105264, 2732.8154211510914, 88915.17356266556, 60321.77034160072, 39418.413823658935, 67343.95668820784, 90876.53384894154, 39402.26364248779, 84321.94722797652, 6894.462322034778, 51728.58069472827, 30525.597840259812, 58157.196116502106, 28642.237027566443, 28770.18537986491, 81726.32907977432, 21602.068492785565, 78852.15451730402, 6967.887442986431, 61575.21509995738, 65800.69356845807, 99481.42781778028, 7847.1254621791495, 61444.91344505918, 59604.93444466297, 40870.96085505251, 39544.8748546538, 20315.284915006003, 26533.152914740156, 76291.67217661686, 30453.23198706009, 85542.66563849911, 53452.537349981365, 69272.97546311181, 16798.727503493406, 46986.89653669316, 54968.6295213215, 19824.849695255776, 79455.31791461485, 75616.58024000078, 97960.16132293524, 40314.71631628642, 79591.7100775431, 63191.652679171784, 49221.92289112188, 84684.58121254532, 40110.64404574793, 28728.44698124698, 27375.142528918073, 87052.75223279232, 87731.25275568657, 71035.83194626519, 70290.39134808606, 76066.35803344, 13841.65885002957, 32195.511915268093, 53155.20665626555, 17110.10117920214, 43235.011924109924, 28897.427494189942, 76173.47841625614]} +{"id": 616, "vector": [70559.25486302607, 63092.048892336, 51959.93299073637, 39110.72399346809, 46865.450908430765, 51448.17136987956, 83859.10496604862, 61031.117520128966, 55628.31942252149, 26262.456336870342, 66643.8518922833, 6202.856559257697, 63917.09347150814, 45782.982995866594, 76360.26807930174, 4504.7529852070165, 57938.75187483919, 51184.9503002272, 24056.877762488893, 40590.82196849898, 19857.12226652149, 75730.15070350654, 64096.59913426298, 82674.46577337617, 2601.9310684731954, 15226.909207051254, 18650.78040873861, 70006.51096194077, 75328.55999506506, 11097.956664016028, 35798.6753892956, 27858.27638512799, 62206.18990802276, 18738.26579593112, 7015.448697254711, 55068.43290017813, 42505.205991482064, 31000.393246111336, 19796.580829808187, 68129.76329890304, 70493.18249446832, 11280.664370265902, 99248.90802082527, 43565.760929578, 6918.388544874066, 50016.700546131535, 50320.83848992779, 50009.072836130974, 65430.22560165217, 54920.7083474649, 52703.53365860114, 76249.16055180036, 96196.36014748603, 79145.94026805626, 99459.16587272646, 15483.12167814424, 10285.178876549173, 74403.56705817372, 77349.9640229226, 38093.508311279635, 1488.260729611779, 68714.60482635471, 62343.63718420282, 45486.491876929605, 64836.017371174035, 7966.166895801319, 78567.71932208167, 64916.82491456426, 27149.90561427166, 15796.234898264895, 46105.06422374368, 92887.34218700555, 83138.40128356712, 79663.72773927021, 22482.954040370107, 80066.71383268606, 51328.63710757161, 43847.27327111428, 25268.5837243782, 98607.39173716812, 26025.15250150507, 55922.32702269667, 37171.61440875689, 75331.12859952767, 26222.007781959488, 9596.037439734617, 63887.00479876182, 66942.05470466005, 69639.66723487768, 5751.911780090768, 59592.361581492514, 44893.82041811114, 70645.05156967742, 38320.10956858418, 92889.41868630672, 59881.03841458637, 8419.374819800607, 24898.92876370279, 54749.25641705591, 37736.54251268067, 64284.15119283931, 98227.58745861548, 13756.391059146677, 2610.2001219470594, 39400.92735853714, 8227.650499523332, 14747.354483918529, 35248.756062580236, 51283.1015197794, 24474.243639867465, 10957.162555352528, 12108.265749769664, 4410.41694616977, 31055.66594044591, 19048.712366363674, 9486.951018350775, 77662.76608731574, 83944.2196058387, 4264.310398222815, 48477.517666989836, 26006.573347314523, 96580.88143954294, 89656.65069460604, 88982.09009111459, 81245.43910309112, 57861.541109710946, 12597.416680870267, 67183.98659933597]} +{"id": 2014, "vector": [92891.4808710926, 69479.88549009565, 94103.65675937694, 86791.96140346919, 81520.23156166657, 83118.23533301867, 2145.1393660975928, 85281.48360915338, 65121.60678695283, 70916.64160699822, 43536.21218718385, 96352.71959559091, 22295.860787214217, 37043.81863765428, 72916.01201862886, 28972.558603961363, 4822.805505762295, 51175.36615171705, 71293.28082323965, 75148.87562789331, 75850.8777314811, 49074.79923230116, 28704.902951373322, 12472.106504005054, 88071.2547306514, 13672.002071677714, 69910.3100823397, 51790.80714479807, 38365.67320675655, 16361.823138074682, 58411.1539125196, 30678.695996189243, 67902.16460242016, 39310.27917446953, 71717.33960569053, 62525.288637391044, 32112.20498792886, 27452.203232918848, 68807.13574380487, 8732.967184089392, 79361.26305918144, 61756.960669297834, 38001.58527284217, 65348.86176759247, 99392.08760791183, 33505.92366305861, 6511.848487326321, 61372.22271908836, 17056.248675210052, 7443.7909305674175, 60632.98057046059, 29344.542201728185, 90326.19701211798, 53313.25966109157, 13788.698039861214, 42772.52966129907, 5716.254941234322, 64646.297655325856, 59964.37744780526, 96205.66407410981, 58160.20671413375, 45186.82430508183, 94199.35331332809, 96303.80895112721, 38327.547949992666, 60322.96442454668, 57026.15447053596, 74294.10093194207, 18609.08918826637, 99538.7246215382, 28501.28565346609, 51659.23087027045, 81126.32093415922, 5804.7599688973105, 20825.6794124151, 35510.495415970945, 19619.447416646497, 23525.743216115457, 17298.74388940783, 31894.209994225464, 83198.32980680195, 72957.82570183049, 80888.59854599719, 21307.760718576763, 91858.07432447792, 89895.16170838583, 79971.06159312307, 13283.736264164192, 71230.84656911911, 94123.9481843735, 28808.355541522556, 95724.57114861165, 71439.49276649315, 70546.92837055161, 16690.099372254976, 98624.77749366338, 78075.1428050505, 20895.98210317891, 34503.768483222084, 82707.13973236698, 51922.59833230003, 62651.213577776165, 14972.762550258512, 42443.185539307284, 7051.815063766775, 55382.2489291777, 61448.694521872836, 71691.42228177216, 99711.44802671892, 40100.707308230485, 81676.24573125919, 62054.646439764125, 91580.63724727827, 9186.333740505315, 13588.498623603196, 62865.374830161156, 18331.299957604697, 33875.951602196605, 2759.474196009826, 92528.22678189645, 53380.203037643034, 9728.160815131081, 54337.23630262631, 84437.19941734598, 53427.419328147975, 96827.16368394333, 95495.43818340052, 28393.52697458971]} +{"id": 689, "vector": [77828.22729792651, 72436.01428035198, 96321.63540013788, 15611.487645038535, 59301.7199210072, 94016.15274340633, 4178.978578387926, 85250.07956634359, 65860.45297972187, 49231.35496383613, 33324.67036935383, 13488.441106061655, 83074.84572730778, 41892.27443182565, 33796.96953152654, 76157.75735950153, 14201.391816825737, 57402.308972644976, 6003.907528518815, 40510.75029672447, 93141.81642131018, 86482.93457405418, 46789.212684794526, 10432.882606084493, 42701.6131236021, 92966.79317581032, 16883.654157322693, 5705.95378795824, 13791.107759189481, 81984.73482046173, 8694.748569342892, 94435.44144672765, 3182.755524666381, 62942.714303140834, 70202.37693719477, 2903.8660168693054, 90887.6320263423, 25945.35254719056, 95440.78013949428, 89406.34659683672, 2676.883075762171, 18781.961353781797, 23981.483707296513, 531.9671404838577, 92887.57241739653, 75915.17150562575, 18196.455973482418, 44341.644623516826, 61091.827889442284, 55121.10503803483, 60718.810354271605, 80584.11382888704, 30172.39324792331, 94881.20068095974, 65245.90228845344, 63469.691679303505, 16810.5374902342, 92939.48315377768, 31174.596363024033, 74622.70104249912, 13109.453631371049, 58521.53087240386, 22845.04027679889, 37945.570555628685, 78535.2847360372, 12592.866054919783, 7740.812394223018, 40776.40875251065, 89143.01523099779, 33012.704328403495, 97627.78874096084, 91038.68379271933, 44029.305669931404, 53883.4185758072, 14145.681696310408, 87049.55158631332, 26082.036760541516, 78433.2098565959, 60037.2958875624, 16847.10092778724, 43242.21678533625, 15833.451050970305, 34440.27868511722, 21811.245502571797, 12325.793605851664, 80920.3217332324, 56121.2463507067, 75961.91380325041, 95771.69871110369, 64934.405443367636, 18760.11157481492, 99227.77475563013, 7503.753344520314, 51851.91068987526, 93172.3919448528, 46477.822402509926, 7456.727016828823, 90033.98812662569, 64571.986391219696, 37204.83816842277, 94290.89989120353, 23156.088706072675, 85295.66283770982, 52366.61993006715, 53876.59198184144, 52978.497917218556, 95487.62501636927, 24767.47236368979, 70290.54496308437, 30998.60711236925, 93006.07423684449, 19010.863240723207, 75938.35954580108, 74266.81197453305, 8163.28925952593, 69640.73439791902, 90815.19511501871, 75897.48857981944, 29632.660268451717, 54678.06190362927, 8111.979845741513, 63300.83608525134, 58282.07635768942, 34929.62834129321, 43032.64253081909, 99682.73435102034, 6484.538608374124, 93781.32607912194]} +{"id": 597, "vector": [5910.358244339231, 24436.701408371984, 48314.849665947215, 86513.15565995491, 3711.102813833933, 94272.47686044668, 3950.0621395957337, 7317.639635650575, 51697.04988800114, 72749.92323659846, 16056.583319813644, 55448.87157825846, 37606.77869319506, 99168.15952296501, 27928.044028936118, 873.3833705630634, 94235.19815088976, 91070.56480629073, 24233.282527036914, 18477.417021601028, 34129.11665184354, 72676.9297260346, 6934.8002164220325, 14821.400831060804, 2385.0633230207195, 55216.91913818709, 57340.32324983104, 31759.91976110699, 18020.56317510099, 2110.3503698922555, 49665.492167682336, 59808.45926014482, 83247.32352092388, 57474.05669788418, 74729.18336047993, 29075.61971000161, 91885.8295833354, 38508.41139757442, 32091.96313069056, 11830.100351994932, 63174.62360005095, 48461.05928646194, 40266.24418072119, 75803.86382722178, 7803.6046193730945, 86588.26815103806, 54576.251261330224, 63696.85836776108, 24372.756853306055, 51391.33893647493, 52952.74214941036, 32178.797431694915, 17893.964278036812, 83529.15766922403, 8588.412485857643, 64444.891835030336, 29217.74873682136, 18487.451222910433, 17158.190420885476, 53481.06392612353, 17179.863064062505, 32712.901944584526, 36356.253914593806, 5524.933626707151, 6605.594256887659, 33342.622510298526, 54810.11709312802, 86355.96208674845, 5882.08355786054, 44775.09541923085, 66653.8954199557, 91722.15552796087, 25340.749111560403, 76761.76053285683, 60097.22592560762, 97449.2812164178, 10753.5010043057, 31187.22595938793, 2115.0710665881033, 53406.84129928024, 77591.30173728666, 83359.5896623644, 70354.83494147849, 92274.08312977033, 74250.65413034685, 49920.12408099066, 45374.52194651618, 11276.825114522882, 79086.02895135366, 3084.0528868520955, 23164.08638446611, 45909.885156854805, 97490.43942703234, 45877.18899495815, 78288.13068688322, 38262.776458607805, 86742.46048681025, 45483.997735496494, 28396.57349308167, 41062.12246777961, 14579.760185379942, 33487.78822534887, 37955.1099604578, 91867.75651967758, 56402.39448256482, 69568.04575454192, 5563.92870344361, 6445.383842302499, 15051.513203069344, 12723.368217804498, 30474.64535117881, 59816.33019008654, 12533.954131474567, 65057.12629291103, 5315.704817735645, 39233.218013517966, 83472.96317296052, 57057.13365143964, 58719.31334796967, 15818.073821229651, 62192.51548651774, 42583.09978839498, 93802.489913436, 24022.498237432334, 73413.24218858301, 57466.0032153413, 89276.72289555676, 27966.887649327255]} +{"id": 198, "vector": [13240.583202077949, 98636.92458534375, 95121.25989882495, 42832.92460762398, 95905.6683840708, 9319.169164947372, 38291.77990474346, 74377.25115415346, 57379.3595557357, 26665.508942457705, 33273.17026470558, 21943.22831964862, 81739.20923011417, 80952.06264806731, 3009.337316151739, 38754.65040451111, 79498.61025886898, 4265.131850524073, 29936.97507895857, 37221.10387524766, 19997.18330830216, 42972.33906637915, 62676.33041709818, 33934.342537072196, 19940.700835824133, 80114.39929091101, 29364.69442227454, 8231.784744858294, 6222.912754787402, 93132.72289197326, 79646.17716112421, 4884.994377292628, 34150.093191565415, 83272.39006851001, 1587.6785043092423, 44643.69054750587, 92879.94893852697, 15178.514590042947, 15490.312144584106, 86994.19250672318, 50688.67849293534, 14823.848225739788, 14327.760823142165, 75168.51544606769, 32109.1584738184, 92268.80962565124, 88720.86333318948, 77239.1260804802, 94258.19469681397, 3552.3174949014137, 85882.83718049055, 15052.189057486099, 86274.95001764908, 9622.273343555265, 14963.133352654679, 65706.68592348103, 93959.84020062449, 37663.702517745536, 39679.085778536304, 9140.505935765665, 19953.014358871045, 56308.434753102985, 14922.993732146982, 63373.0273866848, 58724.39839804264, 94079.907539957, 33608.94684112154, 81962.61255916517, 36776.79119280692, 87611.07364712076, 89494.5698206504, 47522.41366378166, 40683.66904261176, 34104.925585268786, 58518.33244174569, 94847.90596688642, 78032.23963516421, 71832.93620883285, 20729.820347389938, 27285.39193856876, 80893.64420310105, 66207.03777022423, 99458.56881836994, 2480.973230121186, 54009.544052966, 32502.96474634199, 45431.86336074324, 57107.11365081993, 2306.982082188558, 11895.77728572585, 65556.40482902633, 7264.955908279525, 4231.932981559871, 39816.050882792275, 48123.04692267404, 60650.40103285858, 81918.21284723672, 4544.324554615476, 36435.70328888696, 30680.970999495217, 90053.182333226, 68543.17927712828, 35862.45214887269, 98546.28419146885, 15820.066928563881, 49096.78369348861, 52031.665863649934, 41106.86472394504, 14931.603254349613, 24009.33533967836, 86994.0998179111, 4655.570863713055, 54594.206374805195, 54981.41125307538, 81559.27514564131, 47009.636750221114, 39431.72028961332, 65425.182794011795, 89106.97595438923, 52289.969167596806, 20935.821117810505, 90604.18795595017, 69087.48292737607, 6452.155956709482, 98000.16197632064, 95922.51034947357, 93054.65923710109, 90461.99436205429]} +{"id": 1703, "vector": [73333.00391741426, 36316.274454424565, 44576.05931483662, 61754.6175760304, 99503.01581737363, 71157.54364008973, 80580.81219835195, 24737.44968463342, 13976.472023029595, 37031.48518414551, 50392.98157708082, 85604.86785845913, 36150.575778733175, 73791.2350647618, 79194.33811908077, 86685.33569867654, 89409.22885128592, 32229.652290174094, 61447.76291110824, 8451.813265760933, 31758.82421096109, 83721.44519504749, 38169.937848012094, 45419.97370744998, 99384.83527567278, 40436.86647484473, 19066.856011861833, 94135.34042829068, 74022.55009518386, 67493.67905083306, 47138.64841941892, 5688.404327903973, 52084.19156183118, 79377.73595346665, 423.32778742336654, 83823.27978869315, 29690.543421102244, 15572.792444686735, 56575.72497203994, 21996.911445890644, 2006.9832187569082, 81017.3299308991, 20174.07293707666, 52782.64011435566, 49041.06388496276, 78264.93137068846, 58863.60140634968, 25215.99011403961, 82228.22638590446, 20334.984395121046, 95353.47594269144, 25939.033681199675, 21032.179651515547, 66746.44184473998, 8091.640742567596, 97682.6188729401, 39009.80575616106, 7356.239186194058, 78548.62974814115, 77264.37163725852, 54169.41595439111, 44220.01404166992, 59642.72925333978, 84660.15703107361, 78518.60382952445, 96561.48387834462, 41670.470422442144, 77144.54592091807, 55239.00804136949, 60421.32055128428, 69186.09239470177, 10054.608069242287, 87686.71989235021, 63725.513338389086, 13690.379982880619, 8337.96167582428, 9752.975602353343, 36429.04179151777, 86794.29287329593, 74357.95900697139, 11893.57008605144, 96121.9066091852, 47655.160175836885, 63787.67134588187, 40838.077629347434, 12636.116757434156, 20558.26423964251, 73284.87717520981, 46180.129407212975, 87480.06818624488, 23958.851989936848, 35979.00732445958, 90053.40563861764, 52201.550667176896, 98285.37079749265, 57185.77566558174, 98648.62775649548, 75806.53614031736, 69659.83426477312, 82340.12131723748, 46428.88750876552, 88451.50208037237, 86019.6633188885, 48565.52856343855, 14770.12615331942, 23787.17401895897, 63488.735774045235, 58399.557985359206, 96750.85534920973, 67693.92367936525, 97346.70851799105, 38857.155008036716, 51161.55814864576, 95605.70615991334, 68319.95968232083, 75864.671640058, 80571.14060666384, 32943.365728549914, 9495.110866325951, 74109.2164479482, 57281.24867898132, 47341.58440729303, 79665.23667588901, 51054.088505634485, 57581.49209415058, 69357.72949623606, 24134.579099957355, 36048.256930000374]} +{"id": 1750, "vector": [39788.96597198107, 97601.38580762701, 5978.499801060988, 84646.29096100606, 92014.8618748366, 48021.39335060861, 26878.41558193548, 92260.86958174543, 44783.73091355262, 39679.9238942264, 66785.6656399105, 70941.35070135006, 84938.02952608133, 93633.58106273573, 25297.156274411493, 23233.46995459088, 33132.76457154121, 11379.66056937325, 29827.00656045093, 32681.250279603348, 51421.73597025027, 81909.49816457235, 49212.80162303463, 45776.005484906076, 8914.255541543125, 75965.43949135496, 28876.70215817888, 86449.04064281177, 53457.50278669853, 65788.05374653722, 58705.91629941142, 54826.933628992556, 41540.0478338689, 58436.010256009926, 58382.7303786373, 12384.353582077134, 91764.39786100149, 4693.308153449416, 17568.807704670486, 78914.92806693142, 18068.02290006848, 98977.25054680133, 43814.19998244137, 29981.41310514432, 42797.36194583358, 77193.97002799781, 94954.26677810558, 65794.75090903514, 31936.737341642252, 56823.6910391099, 5019.522079563155, 71865.55821732579, 59360.61640006076, 65142.48389358037, 11427.269865277256, 85889.58043297719, 61291.64285500853, 90630.91008642227, 1443.9805077809065, 64879.0606659135, 50518.184181498684, 73867.73269927554, 98055.65417608031, 77509.97175449094, 29067.919896710613, 2099.8360173564156, 33225.577902668294, 74850.80524666405, 33266.93267098234, 10548.013440683435, 43688.52352311369, 89101.02395578263, 32796.602301407394, 19326.720351165182, 43648.86026396709, 79505.2373660963, 4053.8292039926982, 95619.04480668876, 10358.012648979076, 50533.64608167354, 66559.05450882863, 74590.92577739134, 99385.12946595829, 44773.93451420833, 26223.65434947079, 3176.3956612445463, 10160.624495999882, 28585.004419915793, 67045.23168937408, 23341.916541944895, 11263.042504411369, 23189.21575547649, 28452.521367833884, 26307.017503817766, 16585.138041255766, 70263.9975419727, 28895.172078751395, 42146.13117599263, 66489.28287388725, 63524.11549934919, 33520.25662596806, 87014.23242847508, 5830.7369539697065, 68301.84949830957, 99947.44679515183, 36731.48421560797, 8750.99967155425, 97892.34435639114, 29870.377100742317, 20590.239427686683, 53102.686084712936, 10550.308698876875, 53036.87492082466, 19280.114947240058, 98759.97229042609, 19846.292868110137, 19036.526599776782, 91667.58057731962, 70804.36910885193, 92347.73387619098, 86418.88548479919, 76795.25053052245, 23828.97357474394, 22852.11494808269, 7748.164354701714, 14070.460628029092, 44654.58246951437, 31784.989658140174]} +{"id": 619, "vector": [5879.361439803199, 69515.30312444999, 65479.293518807004, 33021.716836142026, 19697.599617569915, 48714.6299247158, 50713.18467957142, 74539.80940933852, 76918.83156043303, 56848.28979527886, 79658.9866084789, 93273.43462713096, 71230.20761090837, 56005.713489911155, 74292.27602503517, 84064.92255189162, 68562.51403172452, 21059.873007253336, 78454.12106945082, 22071.060459311208, 83711.19078663763, 95321.70211465303, 5685.363854261061, 16270.418788694786, 83333.69475708676, 52788.48664659307, 58204.552749961505, 21637.06444447534, 6028.844042160886, 64238.30540851839, 41116.91226091577, 34370.87573031362, 5440.144555676207, 93230.09455031237, 85969.95831866593, 36932.27580616335, 72304.27692541122, 32629.099941800356, 37506.492289072965, 64225.846752710226, 37614.34732194484, 64054.74662985865, 1199.1215876598792, 58941.78784015166, 73326.18317992572, 89934.00925649237, 20068.92820821279, 64405.40912284415, 36265.189225304195, 89126.67259156189, 82315.4284900733, 47715.711701813576, 71034.47807417433, 94560.48434382028, 44411.655457743924, 26828.377824407602, 50525.747762149345, 48732.65834079651, 6456.586003626497, 70683.1151800267, 56094.148300735746, 22997.522908584844, 59436.74037816743, 79844.96531674688, 93507.62993546456, 50014.47516948783, 35315.26944832892, 21656.817190632504, 20199.268976937867, 75076.98681649979, 87142.06540056407, 46960.861939843846, 48360.35763745048, 35892.51770875025, 81067.64563947654, 42330.22453289492, 87635.3416819672, 62921.00618197792, 69657.90759627047, 14964.99189099485, 97798.9895414217, 75572.23641812369, 47777.61913409145, 82922.62589389776, 87537.87265021681, 25966.326995702817, 88366.02256399492, 10653.75937668911, 10169.816018006017, 13054.33865122836, 83981.75516711995, 67983.96288848449, 54722.884813934834, 46574.15352767356, 41384.29328923301, 61807.88708893473, 79505.6105253132, 78678.22733918346, 89184.33264691726, 94158.5715429978, 93596.297641459, 26695.364808319588, 43486.53410580883, 59977.761183475406, 67846.7322710171, 63053.91698231575, 84858.82893125537, 80980.28557668519, 63586.88617283407, 88732.2892557652, 90353.37672120113, 75114.37552689409, 38519.94034610886, 47762.01337346514, 25500.900167410757, 25131.16236701948, 549.4701181472439, 83264.14107001502, 86664.47184403175, 74737.53094758303, 57435.57952130787, 76551.9110812074, 60478.60963426964, 80250.28103334167, 73924.68257853467, 27045.1625683509, 89889.06183034187, 55914.09098571116]} +{"id": 932, "vector": [40587.21159110447, 29108.81934826741, 70052.68415879426, 62762.38792337171, 29867.258856953493, 11296.84413229859, 86010.03297682722, 62933.54842938148, 49032.82320998403, 29680.25264893077, 28288.342881922414, 13205.261421375159, 2340.7277204151032, 55919.66827539693, 19753.031105678896, 41960.943579323815, 12723.021761429698, 8122.924400096876, 69820.94836797468, 85859.90183200393, 83909.2408964209, 90388.64385222206, 86774.81590084528, 26081.691828068764, 25390.616485381855, 77561.81041257716, 8294.09389023178, 65727.78301159889, 91225.95944273406, 65428.436354461504, 63401.72569697634, 16594.7632342483, 54771.84369485873, 16572.39681098629, 78031.71535761378, 78247.33560070823, 62906.87991524051, 26271.915779874587, 98216.68897753152, 11939.313503254634, 7976.25355311159, 71856.15660227698, 98787.22760300133, 21208.49259894638, 81830.85103392001, 37567.13274210764, 47725.30263313398, 72900.92167897343, 13918.399380718105, 63212.768589557076, 87023.37715214783, 90369.05188432746, 15775.64253938255, 57486.376686986194, 35140.74729028982, 69897.19963687693, 40694.60091787873, 28580.87192216169, 50989.26503520732, 16762.062341503326, 76988.99459112583, 62518.53651721186, 93766.84152555255, 16833.37173786943, 13074.780738207248, 19800.24428101671, 8097.003354122911, 89399.57019508875, 58024.08746613315, 94070.60503935446, 63166.66086986393, 21634.626905786736, 38119.579600284116, 13922.982100430925, 44319.588090205165, 21076.889161736377, 79735.93111265247, 8368.267703057996, 85215.38778884323, 95668.17117697987, 85620.84211353872, 67449.99227284569, 42178.51397640529, 65164.746298856204, 47717.329734539446, 46120.76637172187, 1458.728891779082, 19225.66025025305, 59514.7311785274, 92709.43364128232, 99670.90885756811, 70583.20787407328, 96781.94899578768, 7214.684615337086, 83243.91071531882, 31647.240143743828, 50872.36582855771, 55386.09886782997, 2944.907076115766, 35991.98690359855, 59623.55772614781, 70508.74555134741, 25585.38803149225, 13640.677167823524, 97916.28917392566, 86029.05001841624, 29350.412888447674, 13313.903411807027, 88256.2814164521, 10636.427707628016, 63260.47991467271, 53247.80755822169, 65127.343564955685, 37787.11363593964, 40252.04112693679, 19637.290558583776, 73707.77420002512, 38174.81069516895, 17213.236280100995, 70167.85158186694, 83613.19234523344, 11100.928174891389, 49639.272698065215, 35313.12737695777, 99462.19593978886, 43540.37412263008, 81785.394983104, 77910.55817245643]} +{"id": 177, "vector": [96852.3830220102, 84175.63313035849, 59818.2107978987, 63486.46633908982, 91188.89703665588, 85619.297424183, 53924.590269910754, 90455.63915563283, 25441.938263525375, 35581.33237998478, 74165.99605144403, 89319.27799103755, 28793.346529053775, 25282.89773013317, 56551.07436352824, 86012.62334657308, 82895.52021254241, 31179.37594840232, 7261.522442541546, 22386.078272321374, 20750.356085486434, 3698.2721170593422, 83387.29294481027, 21945.350132540643, 5545.800616398388, 72009.29089274888, 48908.064731493614, 19819.86536203568, 2252.5092291864503, 3376.324152864596, 32108.40174815216, 69757.10859936367, 84554.33584689666, 20018.083372219975, 72215.36911335567, 87181.05989871376, 69051.21208602689, 53515.01005198587, 10137.446358085956, 87809.02692367489, 51515.57009565197, 38891.685179098866, 66205.19377386819, 11216.99637509259, 26502.375557129755, 97748.60851496813, 21167.55150012667, 53376.10468665881, 58988.802526081454, 32060.022941303858, 60534.826126053464, 17270.108267558517, 31858.232868154413, 87914.9515127133, 3025.9603892255573, 68681.35164269607, 16233.070430581341, 18836.74146078915, 88723.39990445762, 10326.766826654055, 12173.096645430103, 85738.00673640102, 43020.55824163783, 43649.79504171912, 43531.45674757647, 5539.783699135181, 72354.6606982392, 51833.926893202755, 37979.85756992381, 86248.18447406356, 17568.390211580154, 88774.4699059953, 33427.13300773831, 77712.21752418383, 69568.04318397855, 92136.05762566601, 30293.724578284055, 1893.6454375068524, 6479.160880028534, 53710.479626675224, 59186.6317741363, 19105.79297544044, 20138.656439268256, 14362.221275280119, 32288.112367962585, 90104.71622480532, 58162.42517480066, 1507.5400227974046, 11481.992381819116, 64948.48812975629, 61139.0527980661, 88016.29407470816, 52694.63439827554, 27754.052252251084, 7913.157481630129, 96907.65240349318, 64737.805435784234, 34381.76528349706, 24303.80088987818, 95300.19058943106, 28945.341458196304, 85635.28837093631, 83012.04455880192, 50468.711729915674, 65208.11944562063, 93485.058142402, 79013.69451249971, 12138.993030613787, 70986.26431232248, 24105.146794782617, 5389.714053612782, 745.6679114860809, 80827.3326767572, 46920.93342435745, 75178.96869056209, 19780.958029626116, 97577.58449255323, 35943.10207260714, 86664.30768427654, 96526.60691587812, 60478.11554304162, 60157.94558090826, 23174.630997940014, 74135.54970542851, 27576.403263533146, 65166.488559041754, 58913.96853447977, 33898.37371710819]} +{"id": 268, "vector": [55915.40515260758, 85246.59732674983, 32568.997088793407, 90247.07460264786, 87829.1421117671, 39491.18717122041, 22277.524500327207, 36735.11699791847, 63374.11376017329, 36372.548899244735, 61547.111633244036, 56278.0586493246, 88050.56076471854, 95520.5039131101, 70315.43890735239, 57853.880775428246, 46343.78999775534, 91423.67186280045, 17194.27746483715, 27459.15301044185, 18309.85713670784, 49091.135396889775, 63040.83824478762, 430.11588193369033, 1718.2155179272017, 45351.866336973755, 14218.582825775882, 17414.32054793096, 65786.74756529722, 53.87461902479629, 69509.70450365929, 13696.647723587384, 7694.530889974727, 77914.41388204726, 86314.0385346281, 42663.994083690515, 69931.20706425184, 15513.624059275166, 32242.585975531678, 4068.8874662629382, 5041.175165430523, 32953.860310590564, 82103.59982906243, 49182.623077543096, 55194.07098326179, 89434.37455880226, 55489.57932804175, 54720.74416150974, 55670.84523538282, 58125.730155328085, 66499.77215805081, 27803.433413814928, 53391.88057396419, 94383.63861892348, 48732.73597615327, 45074.52751848134, 76225.66749873596, 3984.3812589442364, 55191.11205661582, 55312.158275644106, 58698.452040300275, 73112.80942059912, 65382.608438934716, 93586.56452581493, 41392.86525289535, 10641.149193441646, 35115.796195900264, 86462.90941556313, 72198.63358184906, 5777.2011400735955, 3032.0704785585795, 95459.85391605993, 92361.44270433988, 99557.35404341057, 37703.71687090219, 81449.32661540444, 11892.907286093623, 88629.50273740086, 32562.073515224467, 55619.77202136096, 82979.06000252278, 72265.10270369101, 28465.678156896956, 62334.345535911896, 69011.63120704495, 98186.1893135471, 99338.00929679273, 81522.49342652004, 64312.47466302319, 53480.68319465648, 11852.227685809536, 19189.35214412687, 22403.274246422723, 14598.260735987089, 82192.96984140573, 839.4676813085699, 76345.30542703906, 93642.31708006043, 70549.86163123201, 5957.927842259914, 5785.377487496723, 41182.58664877362, 76833.27489693214, 57678.17690177955, 41447.0170617836, 97087.25525489576, 38124.483829891324, 37857.988680430244, 47625.827961835355, 56017.76485071804, 15048.622253998856, 29933.28907726145, 18693.538770517403, 86765.06829749547, 88788.39866441688, 50984.98066767569, 42775.21598701347, 82357.96272114507, 88002.86675195045, 75017.60365227652, 45498.41245246616, 54761.14910148643, 45483.08127682666, 70336.05276438578, 22462.606542142803, 69646.344853017, 29091.570454680972, 23328.57626106333]} +{"id": 1873, "vector": [82897.78901731301, 57958.9350980753, 34156.824878412626, 54766.35524972118, 49671.996228257296, 15737.856725133992, 46355.054939931026, 45330.966976964635, 46648.96436427687, 45809.26333840349, 56044.05077276989, 90010.20539508451, 3206.512344080792, 3903.5771594879143, 54028.76135116551, 53851.8095546056, 95904.80018234973, 29492.82153032874, 13648.181751972443, 26063.915781118307, 41869.23500081864, 13906.568936633912, 88144.88120852194, 47724.67197600145, 11346.250394599854, 53610.05459094494, 99700.13687684147, 72932.92196788684, 14989.847221025788, 31026.282555572616, 91246.20405402752, 72369.95929539436, 81947.2291617573, 41163.02378387412, 29834.581180840523, 54388.45875259448, 63497.198316904316, 31645.701648158287, 92240.70219076076, 87792.74513730475, 16446.457978216487, 4626.614066631796, 44118.1351287185, 1445.3277152617684, 14258.783721044554, 38480.155214583225, 6652.96372983939, 71765.64780710063, 22320.9582335823, 4903.5717303057445, 73458.45071756467, 71489.61408789089, 9790.849558685033, 13703.514890476343, 60422.52110486276, 32087.91064734573, 96872.96462744556, 7307.138061807806, 54952.894408173226, 83776.2104303978, 72953.50918470728, 15462.385209641105, 9273.159402750398, 5193.7940240549315, 59269.15768813295, 50859.240468145996, 36585.78857706462, 73900.46555861046, 61621.83178844598, 15586.484266517864, 1459.3116755588942, 46892.03234627665, 38718.601860528455, 29538.700132707585, 75696.07122589547, 44543.56770425694, 77943.38464103136, 22607.164930594558, 31325.024995428364, 10062.550857384256, 69916.64205138254, 22931.10304179088, 3812.219497868674, 48064.07506731542, 77415.91755610259, 82505.429695883, 78566.48735395019, 22780.195066558328, 53489.46659672499, 94699.99476558802, 2744.5352322726712, 89567.6725359147, 49164.813651686134, 27562.686460438435, 58776.81944881009, 28531.177435397738, 20266.58196344715, 1732.1316015676769, 60665.20695308345, 93203.76819962544, 14238.815948982798, 49969.48199045718, 8630.305205272936, 30913.3137473227, 92542.8915747747, 32346.618457856446, 10671.536531213611, 35472.66071682029, 76201.9794723195, 18294.17981225888, 60569.391518714525, 16739.109702898335, 12677.770051878235, 90881.41334623328, 90932.995134081, 44619.41209103374, 79484.19441031292, 37362.51640835938, 22521.79926357908, 93494.85082110281, 96810.36577895143, 60100.04632273056, 84045.28851101876, 24075.505126725337, 57418.932171671186, 96869.91501319168, 71049.80805389104, 60246.35736888986]} +{"id": 1256, "vector": [29467.210876342888, 51168.840990520395, 96153.03370608069, 58626.8907765068, 30265.855109316388, 24407.598403532615, 96523.31723511456, 69486.18383674991, 97813.99091155655, 5554.95685642552, 86323.08790030704, 73150.96866474864, 92962.79517290358, 68924.27097050111, 54444.32998001466, 68928.47671664246, 71244.65792921613, 30176.969381576957, 99782.48691515902, 29240.462235430154, 38012.40071598548, 25973.509652017456, 51151.8631232785, 42961.73083570719, 3985.3527849519232, 34060.83372698902, 86032.30215755092, 55196.52015336416, 62273.047273509044, 16455.076287721793, 68427.29191742158, 41825.94560330145, 58792.41417059003, 48810.31497121002, 71860.54644387448, 32220.405995346922, 75422.6333049464, 82375.93857702763, 37607.4891444653, 49547.74163023071, 17772.159093671715, 9453.2989757399, 6719.298241499072, 33127.10205524192, 57744.75161716053, 85971.71534677682, 97896.82369984755, 709.3541682593795, 56517.393449604635, 91761.88238759676, 36408.70610353503, 99380.22045003343, 68405.76226288798, 21787.024713087343, 16270.070684809756, 38907.780659763994, 98514.09792781677, 61875.042859030815, 6547.942597809298, 94166.82533388687, 86224.22642149006, 82604.68993679679, 65244.073178182756, 44801.36432354114, 3915.554854323555, 38517.578425618274, 81643.30475168013, 18209.448546791475, 56045.81231832075, 69787.42924858724, 79708.33286464168, 6863.767592001646, 23431.987186651328, 5374.65774662852, 69416.46939042484, 20233.369987500206, 4628.446434479205, 3946.5851560300357, 4282.164019846402, 26542.627107544125, 77032.4191692877, 33827.815908722114, 80402.04888048914, 58588.7957468411, 97843.9634116694, 45159.06040893578, 2396.9765723903633, 82178.88955993399, 95365.63539410634, 62991.85622721286, 58938.391570995576, 97210.24881832919, 54009.45391415377, 3058.9302590664947, 95136.26924226448, 25793.609869098455, 40778.027255205016, 25850.279059556124, 62997.97715955143, 85169.91468333911, 7654.103485783925, 42338.66638336975, 92064.74698443, 57263.97345634775, 60322.69232952729, 54571.69310603723, 74590.72874325016, 83824.0549783568, 52893.40228340193, 9963.393167939006, 29336.301502413375, 43170.816417573275, 22393.482525561536, 57886.3790179397, 5524.180617600027, 62892.59226337054, 11331.766342000261, 77077.00570354829, 49099.213364240226, 72356.90980956968, 22619.734610137744, 37040.77754494993, 74698.13106034283, 2291.017627152847, 47793.527780593955, 48303.003129561985, 58431.613773549696, 94212.3463786768]} +{"id": 1343, "vector": [54202.74420412783, 761.4977696808634, 55908.82343131992, 34951.27946174297, 85343.40868093888, 84066.72649332036, 97355.56591282706, 19505.7068951638, 38939.89851915718, 55763.204493590645, 76515.32157541941, 29023.24843783306, 29260.27921715447, 29779.022341823867, 95333.33660594193, 85971.87166158282, 33243.46094893845, 75688.18900027033, 71988.48320631211, 95424.37658964317, 95306.50634522701, 88793.81920991137, 68819.43561406464, 95422.65148662146, 95376.83923209667, 24893.805241282716, 52972.42084782687, 58485.15938344643, 66776.04659377965, 31420.66787584561, 68639.79244813371, 40438.10115805914, 52598.999146569644, 44899.611443962276, 46542.100723396776, 22492.444331897223, 46467.781160164544, 18266.905768034892, 43740.02414695851, 48471.767906463305, 40852.3447953057, 5490.659029585221, 91389.9585936807, 28179.13092453129, 2047.3366453240938, 35656.475930281216, 28728.022681892828, 91525.33903727656, 78618.13195551524, 43971.12115094276, 41258.111503765, 36377.90305673068, 5574.7087367146505, 71436.54617984965, 52848.035652460014, 42894.21658303645, 12748.094997210314, 46004.69782437299, 86332.92363166019, 19706.874014205878, 5752.161576129355, 54683.88907769442, 28341.585631519607, 26243.871074752722, 10321.293665536357, 24459.56707002217, 2698.7464317625663, 29603.797643158814, 49233.65895134259, 18622.32580984322, 9491.046893465715, 17339.536548833144, 52404.48210970505, 19549.47420413283, 68014.13095872116, 84504.94808979672, 85294.04012086849, 99745.03418090346, 41185.51636208525, 81290.44274051115, 74586.58332693446, 92582.25758911957, 17429.866506997427, 44854.71527942372, 26420.365659887622, 73064.62548573698, 76230.72139195725, 60997.39929805516, 66125.48106379123, 47513.95812888167, 66078.19906798142, 82867.84194259818, 12291.593345823849, 30960.57663915044, 32644.313206136045, 79035.5171266656, 74396.34574631017, 23495.73831268751, 99229.48592555126, 10142.852868004571, 34345.16026928135, 98892.93435928297, 2394.503193615338, 46309.51105902871, 42762.470437928205, 86414.53207718162, 20802.236117104512, 66633.23881059508, 1900.9266437257534, 32690.275241913336, 18791.640705235714, 81923.7029721944, 56330.37031060905, 21763.686098924074, 52790.4074660708, 55241.07381830038, 19675.173295833592, 39036.23634164558, 64403.82693524066, 16058.214481791443, 47574.795051776084, 80071.97374591368, 56530.03098753021, 28208.68000478567, 43669.53736512983, 65532.2216254425, 43775.09855933015, 22177.48243347598]} +{"id": 765, "vector": [87616.65395612584, 69467.78017590713, 69022.5519235942, 90085.02740080305, 6948.846945949783, 17680.378468369352, 93103.1260195151, 94315.20953108817, 65639.8541446835, 18940.552312485637, 69265.32927972589, 47827.37157452952, 92070.02090037822, 90422.13283164967, 58043.4548926132, 14872.906951450015, 43024.97168998998, 54016.46599020014, 5078.304593052208, 81871.40032471006, 20355.251541284815, 75395.59904761544, 19144.9652586828, 64573.22920911833, 18885.085406932565, 20068.87607014035, 11979.435407904437, 27444.836246627703, 18498.878159056385, 99430.36397683069, 80133.98890028238, 88577.27478672269, 8690.664023417216, 25258.992486803712, 27202.73802895623, 5437.74103393736, 88045.48604954749, 63931.29513003968, 4611.046616643988, 43506.052274378315, 98307.91172381333, 46536.07310079866, 49658.686717393626, 35051.022610006075, 5667.916319264077, 35816.74981948426, 49077.861070909814, 64781.368594827545, 83220.63947318272, 50666.68684400833, 19612.889064628103, 78306.92992984591, 88856.83623159549, 5870.546111981934, 85242.256741, 85144.00792715167, 46225.25720276046, 34324.20534347742, 29219.644471920725, 75252.71732470003, 60965.46928771589, 82375.02275375945, 20776.3237499931, 59830.24289030504, 91400.60777917055, 64291.02719728224, 43034.84017023744, 10520.220028710237, 12772.97716888539, 16976.798003253578, 92943.60711732614, 31068.97784032895, 75655.36779339837, 2225.8324658339325, 98718.6904367075, 40621.993892362865, 97002.73708782445, 20477.8146803963, 19456.187958386574, 67936.361555079, 39336.088433989215, 49973.231617858684, 51228.98414990711, 81038.29078454037, 7356.210006454411, 43267.712647531545, 83323.44142043755, 68771.6118598946, 32438.4556559687, 66891.16158176353, 29133.034451067542, 43253.263679498654, 70594.44975284005, 7773.609489296462, 40977.1098019841, 7178.442810597175, 47248.1531869431, 41362.87491808229, 80574.29303469116, 7773.473283795218, 8423.151184038647, 9310.993973429415, 22663.250101324793, 13011.862118056106, 47432.36119906985, 47042.46054381056, 76205.7382157147, 45332.150048729694, 15043.586730444948, 82778.88353937745, 64833.776840615596, 99713.68297470738, 44595.91467569971, 28612.219090118175, 44512.886410500054, 56272.38302964782, 53110.805736620816, 40205.15347292486, 90718.38834217699, 52797.30622402728, 82718.6497996116, 24879.90062327148, 61669.477664319, 2471.3592667330086, 78777.77995679848, 89474.34050012604, 6634.035490964563, 69608.57922638037]} +{"id": 1349, "vector": [79608.02145265919, 23305.391161897314, 65104.09706693078, 24421.75961361345, 18997.878228351063, 42045.76879474573, 75148.66215213442, 27920.865475279566, 84597.67090424852, 24470.396226041703, 39252.87443882698, 63312.95410991704, 22219.94933177137, 36926.35447220005, 45675.18089713108, 70896.07462402894, 70734.79725979107, 33769.64327425965, 40589.89807577906, 19044.137997690046, 28296.586312680927, 80936.35186732696, 56294.67808172986, 22514.050310094026, 96697.3290160947, 10867.812112428155, 79820.30587759957, 59709.55569796491, 96424.44432867596, 85717.57747076632, 27622.446522894206, 89945.62865018578, 21101.457674626166, 21404.993708884558, 93868.41314433474, 80002.15061812288, 89003.63915144304, 41960.39205793597, 35297.625369124056, 45543.80471408231, 61093.88599739652, 34447.86639264262, 94560.73375341619, 61422.97820256083, 78880.03806126579, 78012.30428495206, 36445.22447158443, 64840.81761503074, 92020.46712570162, 4248.943242796355, 95102.48184261472, 13329.671211558269, 16331.114104607192, 11716.489854105861, 12476.479297269494, 12870.241604035926, 71114.80521552848, 60500.16327590364, 63566.58230956913, 17991.417515463847, 27059.039351085656, 62632.29579273571, 40249.67155538415, 78548.18362808181, 53994.14267762145, 49918.129207300524, 74592.80301857697, 27661.63228707795, 4132.402160563586, 43286.74968498052, 48627.89939516865, 65175.169940665124, 50581.58274570425, 70962.11915891735, 26406.89256051837, 57997.86633880021, 17245.223262404797, 29370.95627451878, 4765.643482997372, 78971.28538958738, 26598.715055563414, 65189.76901589066, 29705.16002222686, 28242.038234745138, 76029.18005559187, 19241.371668630734, 10754.902910833842, 58504.673004962046, 1880.66148616588, 60729.09352113865, 54510.76187584464, 37145.309474831534, 87057.94781511897, 78451.93618203314, 95138.50849160724, 89946.7592600665, 38595.58501288242, 889.5614195378099, 43373.623693441776, 39730.52048988328, 47477.78502352376, 12522.98108290888, 30384.90571158664, 8000.565570377494, 77529.45471059921, 67687.4276263664, 11161.426032297239, 30354.0138679931, 73085.96015952974, 2721.0471458956054, 38599.95812054513, 53151.224043068156, 8383.366118725855, 21317.98365745282, 25486.312047051986, 67896.96250711937, 61270.29243046891, 75441.35744631114, 52698.50435840118, 81695.25262589159, 97856.35636211162, 68954.71021667811, 63073.57681216399, 94705.77773379284, 94783.27381009166, 65367.70708192231, 89561.86844071696, 11641.56830997679]} +{"id": 1383, "vector": [44089.958205941526, 51634.38663217891, 88492.24921203262, 86522.95412684359, 78427.8703471099, 7556.7799957694915, 61954.45176526206, 58448.49895452575, 97681.65371671907, 77268.27441844424, 14910.454819513807, 62509.78645011634, 4375.7780637787455, 49460.61420233774, 85554.32649204388, 5681.135499829748, 66232.6877572326, 90821.87495966275, 61681.514609225494, 94180.83349015885, 94713.50345230216, 16737.281615971002, 5281.434297193965, 60787.9497290989, 8391.755551281954, 18670.999426123726, 76616.6668794927, 40445.71340688536, 14702.888200488007, 94330.25237055738, 99419.09517934926, 9722.443860693453, 46293.7495108454, 19820.577024158658, 85926.52818079959, 52123.92062316892, 17803.743996268608, 17487.58079080166, 22044.37692977882, 97065.92827131414, 14376.219115031896, 48996.38582115683, 14585.43106530753, 71363.68072577642, 14704.888903657331, 20852.57061917877, 96515.83597567223, 45955.238287088985, 92362.80046210744, 49350.63562455764, 50658.09816634859, 21284.61554101364, 39902.12497033458, 75652.94518925888, 65739.91092934097, 1874.2303234181911, 35743.29429694987, 45092.36196585743, 28322.34088464718, 24214.37525415886, 27440.294570591028, 17265.215629760445, 72108.06175791695, 37434.87045094123, 93461.56030048046, 3517.2799664102026, 59691.7830225445, 96200.79966161831, 1561.6705369619233, 44809.82613540528, 83786.4632689323, 45941.92654421291, 69580.97010951802, 48402.288063619715, 73889.70369951968, 55759.44958824797, 41816.221183191425, 9386.302996811346, 8947.669637784305, 71025.83426943464, 88137.73147667958, 99730.74723363452, 78857.1742583129, 13815.452629732683, 1934.3543334437218, 57698.3503740743, 18014.984248807454, 92813.53439893365, 82237.40982714348, 94063.33505980781, 70120.12140110666, 54242.35430368905, 92692.91803171294, 89367.4634622375, 78558.10886072909, 94926.03622950795, 73392.32221391823, 5061.507340724725, 49929.66440823993, 27373.820470024533, 4456.449596986856, 12466.989736307465, 92760.93572370848, 50418.361576618045, 758.0801046488949, 74731.54323496313, 47348.2587392642, 20915.731930063008, 46965.58404909431, 84793.01061072019, 78617.09873491703, 47483.651947010076, 37583.97871204505, 10454.337737524454, 75416.99108881954, 35463.642408639214, 75944.5736278895, 25127.179145167855, 92064.54591627452, 37787.45698481292, 27642.572056816938, 18421.58187841335, 78497.88904696926, 91555.1692827732, 23111.053821621543, 19922.004998834207, 12071.36174374367, 85723.11982639796]} +{"id": 487, "vector": [83330.4554100338, 98193.13244724454, 37086.38659216833, 12062.7620128121, 70227.3837048836, 74799.69267100084, 13591.568649508801, 56682.309426549684, 23780.82179012445, 80599.2384958067, 91709.17895775869, 8826.093346583762, 70866.42915602055, 51286.173906830154, 34529.67510407663, 24668.86274386867, 69492.18371310477, 21583.681218242822, 57275.366395441255, 3344.732205034462, 16655.652722850056, 87237.629565421, 53314.20011784012, 36857.387423622065, 64904.31542642846, 1220.92662586345, 42542.475034582116, 34238.89295764955, 50270.44351614743, 56414.188182551065, 78123.98960486022, 14746.37962410813, 14753.864108656422, 9667.032979159861, 3857.913676956648, 10325.171694069679, 40897.39602239689, 36687.767674848736, 30758.11923052958, 8044.418486175841, 12516.237815697228, 85533.77928920992, 24793.041975741915, 19565.45129508508, 22492.488341314387, 77849.16613622579, 19056.32172956876, 34309.47225174717, 15693.530246296672, 20999.334231234756, 67311.07621597014, 53004.241640004155, 73403.80078286513, 42098.08641209879, 5913.875834161353, 56154.114475511094, 9470.831179698203, 85591.19029471815, 3436.5227258520536, 4643.037825666385, 3076.641117585466, 13502.998935784128, 6513.20618905763, 65394.88627641146, 3938.1765987873905, 87568.22476947829, 81396.75391516439, 92096.61026921819, 92975.37427669755, 49972.0655442121, 47223.54287147515, 7394.790240858595, 40123.10930885626, 42531.03605317567, 93811.81137902982, 11156.28838087227, 71079.80933895905, 24172.576928570532, 71800.0877736513, 8386.433327516763, 82955.46595781685, 95967.72585432304, 43291.70299493813, 24862.817088199794, 91295.63205199264, 2792.841833764026, 16643.375640201673, 73138.22534878914, 43731.06611366769, 73749.97070985266, 19553.53066022324, 71035.3689015423, 81358.11962096789, 62771.30003622846, 58065.40851372627, 68415.2012470383, 61187.72859199398, 22061.037106240055, 45518.7920885607, 55474.80402385639, 42602.855236545955, 75582.7794750777, 68793.97062194723, 73811.9746515511, 19999.674081373752, 15524.042814593275, 17265.227382592664, 34827.38743241226, 97522.55827277785, 54669.51216489383, 94430.07519175051, 8859.633175488323, 60678.38492498724, 37325.85005488982, 87827.59171489438, 58642.1921391939, 59072.80141264275, 19727.078242288942, 57717.55967769248, 4524.96411636466, 67604.44202321015, 61790.656460112594, 82681.78244173239, 53517.01382718771, 5078.29483698653, 60277.32388255823, 64410.06952599679, 57045.0115301665]} +{"id": 381, "vector": [68272.89986779739, 15671.523575454583, 50539.534390252295, 9529.509969803674, 20233.02851951484, 8730.78036091446, 25676.384175652976, 88222.4177953724, 49022.14648810658, 38753.16792443595, 57531.39702680745, 22819.612135877465, 36774.30590262135, 81420.05440678814, 19341.547623154776, 94104.49966820072, 54834.49033286939, 43901.4522022275, 10309.790507107264, 82300.1488071864, 32899.07251211857, 97234.10217534222, 79030.48713647165, 62690.76256245064, 88889.97117837661, 97741.0186585949, 69457.87831601448, 22611.743507413463, 54881.99809754447, 93654.58250538354, 14345.018128277065, 713.0237024330688, 45647.48684999872, 88016.59291777769, 47361.947695743176, 93609.07416795925, 3649.7945323673566, 35987.32197141677, 4409.813584057986, 61955.16424230056, 56710.16836650753, 10893.920646560595, 65818.24022812428, 22068.53783103676, 50770.04187600007, 87817.97795903997, 72828.05462459258, 81910.3760988586, 17463.779774095667, 70766.66665920966, 23542.962360184905, 23987.957887418775, 74799.51107850537, 65817.80629481445, 50783.297459144575, 72384.11069484644, 61863.08271577679, 17188.878646235728, 92317.9838045785, 65808.19956250528, 95985.74327709257, 29352.65219951434, 47166.18742951789, 1564.9218885473658, 18655.473495134, 70582.4333368651, 4702.370216872986, 99811.36189064456, 62592.46807592149, 60736.365947902224, 10365.991453859491, 45364.219546469154, 18641.55417979011, 15121.343872362104, 67881.20721901598, 39288.06724955483, 58213.90373749251, 43538.043653714485, 1577.9695017533625, 40130.04022947443, 70358.54468588864, 77203.13425921282, 51899.05176829165, 5177.368102808999, 85781.37939306215, 80830.53694547013, 13299.42343573185, 45149.746945080704, 76504.9972154858, 56884.79530906645, 22893.68298982607, 20958.280868736114, 71610.68221609823, 48995.31667177917, 4399.891883232798, 91659.2983380649, 88425.7154031057, 10650.54791343667, 42135.249099966844, 75501.57115213516, 25671.474286934303, 52831.900615229824, 45026.46204259437, 50524.542051836375, 20619.423980551233, 4097.9974631501045, 24955.933082961434, 16548.822112060767, 59383.16357339926, 21908.81394128088, 60684.42988719589, 42161.441155904686, 45158.50299003392, 14620.795152947941, 79677.6255421765, 46734.47055117569, 55878.778251367636, 65815.22078454273, 12735.051584859713, 82828.39020149794, 2962.186844369241, 40570.419277280955, 71558.26130139564, 36442.65726758619, 89295.24532267285, 28382.69505519144, 61087.24273398663, 90947.35049772785]} +{"id": 821, "vector": [27637.73103024424, 8201.575858144904, 10240.372902579065, 95193.30484705833, 62096.52160453515, 68865.31814566899, 48378.43758984044, 46983.28835370633, 22446.046313126633, 7201.437319703863, 88215.70104160749, 95993.52484922747, 31344.497235098588, 80120.91692779415, 58845.59829295617, 47304.268977714404, 18155.148385470664, 17035.167583655562, 93713.40434356318, 66198.12430159735, 93032.33698865263, 48030.38168259163, 3458.6910141224057, 4001.016902599519, 13837.873871434636, 77894.60035983902, 28470.286631668307, 93898.78370294628, 7152.232015131422, 95951.7396849808, 21497.02309768592, 40256.30771941702, 93285.7924322393, 33534.404730351795, 3593.953406400574, 98254.04656731774, 55.77347327010429, 70462.358374526, 45098.70286917936, 68306.52719969956, 53109.2692347774, 92548.79746743361, 43435.33656261902, 21217.15125178464, 26021.02060485455, 19218.91718510328, 68071.03767921642, 50407.15436212668, 83113.33668738774, 4499.1340624761515, 9072.266986048417, 77356.04207697391, 94912.14620594477, 45631.48933076146, 34683.66959626907, 22193.163312267607, 76034.70860103106, 18087.594469028965, 34427.60473858783, 9838.701972021834, 40188.839408654734, 42.603495015836046, 1026.4518690074387, 47370.016172459895, 55891.59698300816, 7104.262961297137, 87925.04368742241, 80615.74226309122, 5970.403041173767, 18254.184762131197, 27150.026134530268, 97587.94070019951, 23764.677644663047, 82514.10762523118, 36680.70625140095, 1376.899156591871, 80427.69464897031, 75624.71788242005, 6380.738074042147, 26562.75392445564, 56729.24245910338, 23703.434736470474, 98480.6725896335, 11518.62251779755, 85704.67872945585, 29828.51016387337, 42520.24953574173, 40582.351709020644, 49623.07683049376, 90046.81209298548, 85848.87813223201, 53569.095585503776, 56116.73961486697, 72127.92578670912, 10915.465756839903, 93995.90458976121, 21627.655222399, 47140.55186051953, 55304.99428878239, 54413.47507398373, 93221.53658072407, 39626.96612897824, 36788.19713089351, 74259.19084336238, 64806.58071702265, 67211.15941021178, 71726.74481783126, 95804.546806305, 95565.94148666305, 13779.189697101712, 31366.660066595552, 58280.50635158255, 64733.468395751326, 28760.934792970773, 14938.196926433433, 76583.4855181276, 23647.736721167978, 95015.24463925714, 33722.869398113384, 90380.38255343866, 60575.429498265934, 47999.18931028704, 47597.69096389445, 65421.94833600582, 72.80185895669477, 30436.147636368725, 41299.44335486071, 39635.29095673027]} +{"id": 601, "vector": [66611.08977645947, 64522.929401921516, 2249.3563145882244, 89009.44780385353, 75048.14383934914, 38939.28046152448, 59146.536685461506, 93027.0447174121, 80095.4180896388, 7114.476775048528, 46847.99925669742, 68430.9428071315, 90989.23840147488, 36792.64834973676, 60805.9670161554, 1732.8215673997338, 48678.03988006667, 98300.52283298658, 69850.26457493758, 79020.9132986612, 29833.993028442364, 8379.17945478327, 54931.475290950504, 17128.22143308632, 36609.44681879269, 26356.763898851244, 78917.5847650767, 49645.000031602714, 35842.513768363606, 67963.67737111951, 6381.422455279228, 67701.8037907209, 17190.08759352625, 68084.5343012711, 6531.441990563191, 96408.12147532053, 37976.3833887955, 2238.607568064166, 86315.90678147684, 35285.29318974467, 75890.72701187969, 93002.23571632063, 26658.748317492307, 32239.41636506933, 83711.07088721659, 70273.92328783775, 81423.41848125377, 57922.35075172994, 16955.56712075965, 93367.86176707527, 40776.58218672878, 82553.81154101754, 69675.91178505437, 30417.753777495625, 3400.0418343981487, 39666.75797269833, 32425.474473798087, 49154.23594419167, 48457.4923824175, 16294.515655045361, 4470.167667378632, 6545.324427596422, 91793.81395819635, 59677.12006640875, 35534.9836255927, 69511.17150739463, 25960.688233322147, 44851.08604494653, 72793.73202044677, 50002.56271388321, 94540.80861609217, 7616.770009287555, 19712.585554407204, 7592.417137775476, 81287.67094241006, 40239.16231572123, 95710.54191489956, 85328.37178019754, 65642.11056017282, 47643.88774827673, 43815.64178146021, 5636.783267462198, 16987.418787641895, 31690.257424397903, 18457.297644546656, 24706.14162817445, 44586.38626057124, 11337.194946013806, 28275.661462686276, 77467.4667595492, 28931.327491227665, 71910.62622411663, 39989.842760624735, 58666.39260964739, 39784.64063601297, 66605.1813975627, 63184.99250567095, 98848.7595093352, 10068.958332347622, 74686.20049801578, 5484.710790420521, 73776.302067218, 98359.48023947616, 72370.18970429919, 11871.611710570462, 69675.0968760762, 70762.42613622859, 16815.854388127773, 1248.2788817661628, 86756.60676521303, 58038.90105174181, 10184.577975512742, 56790.97762946436, 19230.540968114852, 96212.02176401955, 55991.53302328672, 23658.267136318078, 23559.0627677034, 83019.48965099156, 54488.08841823953, 61666.03242837131, 27782.450791275605, 64450.33750979544, 47488.77927474277, 22465.647010418, 38615.310979194684, 85783.4338938985, 40561.44797581704]} +{"id": 748, "vector": [64087.26754017642, 37352.67255823788, 50131.82722083163, 52265.62277044916, 12571.548786155585, 67382.28745198305, 44216.57071516716, 40300.28667932856, 22278.67071285976, 4312.7860142977315, 36428.9483261945, 95915.66523624952, 64811.11754733471, 23535.544908231877, 42836.23617943499, 16053.714370969463, 64860.616049966644, 28966.2752574103, 14123.669145513119, 24525.695192784347, 66159.58004870926, 3080.0992616541125, 3880.5807369639565, 90806.83028380595, 65440.65395556994, 3984.709768939132, 51443.89874820956, 96231.37358235243, 72253.36263018986, 30751.890698153184, 64902.63267481638, 61784.33175975957, 14755.484203743374, 73270.20856840914, 39168.764526850406, 90887.25842014476, 36315.02304660713, 70668.92027697193, 85760.13943136239, 34307.354397328956, 68069.99191546226, 86569.07874725612, 76010.05786566161, 26471.28875261917, 96346.06210695264, 11890.710871931298, 13889.360015065278, 43048.50066570367, 89744.8242299064, 43084.37786484811, 5671.55459173897, 68315.14451278796, 49698.221463747475, 36567.16668590774, 7705.687981688714, 40758.88917723486, 21412.00511969481, 15165.517137225048, 71058.45672482533, 49256.66953309556, 62518.3515331703, 28859.69043091272, 21787.140584022512, 80548.58657493812, 79182.73816520946, 36713.71748632007, 60796.70253876468, 98116.5875514442, 93464.9649969821, 56976.522900802986, 26225.310892876852, 72913.6269594217, 7187.647355080384, 61896.9102665589, 74215.41136446952, 41064.60564584211, 57040.7396769016, 58251.87136653327, 44361.0635506963, 70498.03310740415, 79192.59539051887, 72812.64241220643, 38460.37380046746, 70148.0933816381, 91276.09810295438, 37712.90666873842, 41062.78097618519, 25128.74784685343, 18948.910485150984, 3410.44787907, 41285.77186925031, 16323.65943228179, 16266.381198353618, 19738.215349035214, 55499.73639351243, 69374.18327094124, 44674.49836141384, 43353.45528204204, 52195.87967317806, 5531.697465640884, 29567.860556908567, 39819.68342351403, 27512.75502657077, 98247.52617330114, 71804.32434153651, 60586.27074952173, 66013.46091087049, 12952.013556654263, 18540.72082278997, 58372.74243808579, 63568.223037106385, 13360.341612094351, 54261.986963136864, 70835.7784021514, 39721.382630404056, 52877.88877355869, 84151.16419858298, 15298.431362163057, 63657.276081323245, 36306.33337059711, 12117.010249023419, 85117.8214111748, 3543.06861721021, 54647.719347569466, 30435.65047892909, 91527.52309765325, 10108.182078275995, 28460.68855362652]} +{"id": 2018, "vector": [15174.13218521293, 15364.086176571023, 57963.949937563455, 19795.146236208515, 16375.372342059713, 37313.083909472574, 87185.51683875675, 70726.22723158597, 96880.98545516675, 71224.30915133098, 11649.677595386765, 48462.05419089069, 78899.38841196997, 92438.02780431147, 31730.941529153657, 44282.30067451624, 46838.035939719455, 11584.56232765801, 23804.420162070415, 91053.79223583995, 12679.4187868011, 37585.272292325746, 5958.033884364866, 6267.701670555514, 62480.51717056313, 94124.13224926483, 88457.69167021313, 18477.131288528435, 7919.507960295679, 13818.60206453669, 24781.21917311503, 10366.787534767342, 3431.811164791088, 36202.86954133882, 73640.62311788768, 70832.6696166591, 45180.85711742053, 68313.22317984983, 13070.798813404283, 53595.26730216238, 54092.72363900775, 60378.45331193532, 29024.59734185113, 73260.49498412988, 18616.58376830443, 32126.010973420747, 93277.15946794822, 91288.71563627578, 96715.15956417128, 51596.654743974555, 47287.733283633694, 98829.17503452241, 4478.650032642273, 31204.31269165982, 90550.08959901724, 4584.030104945625, 53968.60624630734, 75033.4553772776, 40634.871621760794, 93800.93481611887, 60634.817873005864, 70750.70491809443, 13977.711031341922, 1691.604927441459, 76254.66973825505, 48367.150159603865, 89083.16687066892, 34109.434039878906, 8084.788011569377, 40298.23834777494, 55774.56538192418, 57690.28942669138, 67342.95486219131, 29378.211973827318, 61747.61633920057, 78011.67084567007, 3983.1104912117053, 68821.68607804764, 19975.74602543396, 50367.00509556701, 67501.44860783184, 7327.998906431188, 28648.625212840394, 66369.71464647164, 89430.8157135057, 13857.071826439538, 31476.917663175573, 77439.0578069568, 66940.49925856352, 36630.4545549892, 76097.24724604409, 59124.74496175465, 57171.50851391969, 99048.70494786323, 82084.36405906049, 98350.38399419942, 92210.22273656879, 14483.658077567063, 98456.57440780743, 72961.91874204733, 90563.31502949767, 25917.244284353514, 95220.58426321988, 44999.69276403546, 44444.721267263376, 83837.86527680563, 50694.99295979739, 82958.2067576519, 73292.22308388555, 17897.68791609726, 56503.43933083771, 21567.378021143348, 49546.437683052565, 34866.37044731707, 38351.00917035977, 85989.60662822251, 31441.83647425908, 90096.58265676869, 25644.536884907244, 47022.52139348063, 50925.74654718175, 56874.769024786656, 72754.51554319063, 38141.26682669964, 70020.81106407364, 10334.89068136132, 61179.19284863946, 42168.84542140662]} +{"id": 1481, "vector": [4248.484317612156, 7036.261908838792, 45066.145058383154, 83812.95243022595, 66276.95542907515, 17272.14279980094, 49065.254144280065, 40941.71686122195, 20196.242536243324, 85576.55408480753, 50118.129103460706, 86378.885062174, 87269.40826172475, 24346.219576202733, 60884.57430079365, 37456.14207887347, 419.39349352929645, 75285.18925217298, 64306.38264418085, 434.78726871922777, 68423.1727808318, 71080.81184220285, 16758.792052489378, 35641.272226684254, 52986.69033773944, 72351.28642936696, 98947.04649704245, 49952.211709324234, 40108.99576781148, 17734.65439775006, 62157.54281132748, 3325.5630782847325, 7288.91186815297, 52817.27538166026, 41467.673454380216, 10424.66883730303, 37626.47913113326, 94025.29223462948, 37039.49449279185, 11721.244172660605, 70420.48746401332, 8200.374023439328, 1525.9758990936346, 38375.42480621786, 78736.4141705099, 69125.34500035674, 54954.5534508913, 55335.988782995126, 10820.962769404163, 87309.560310223, 25217.271690982292, 5377.908265983156, 33381.70835116253, 29017.602778837005, 38040.33371126297, 37302.300766946704, 39351.13472252775, 44918.2246329202, 95021.42787209479, 10346.715473950275, 84774.2396240082, 42848.20903029772, 17854.223073071684, 9503.680996319885, 22928.38929160942, 6258.054505412591, 55723.218940208884, 74533.35267222712, 62701.01682628148, 59413.83742561126, 14651.127404987474, 86864.07099977603, 15167.551343599162, 18970.82005343276, 66287.90009693558, 8207.554825191753, 40740.28741270007, 21020.300941495072, 9016.544454688157, 30927.903765180774, 3317.8624724747087, 15480.99102055781, 16185.721129533204, 14598.927229613933, 12321.030012466194, 72854.3448667599, 86979.56504890799, 38749.57302591885, 68358.13761733165, 13041.424360595289, 45302.11728227157, 23146.071273490244, 60633.118182197024, 8504.69066966213, 66544.80937822825, 29826.516513402014, 44046.69443782196, 59922.28451480874, 96931.40045580594, 74085.80755330138, 93651.68923551375, 13162.970081038893, 45659.84360304197, 51023.454226356946, 82735.03846168751, 53586.539485619585, 69051.59987876436, 9086.964628155869, 35159.07935649699, 81042.17817520347, 34360.247602351345, 96016.90074992456, 9318.488086121779, 57216.44622704293, 51185.34002825926, 73366.5619443446, 59743.52107080995, 72120.68943863049, 7613.418873595757, 36559.33090892991, 99044.63420079077, 10246.552825915134, 39583.31456641573, 40437.60076837294, 60797.025807738835, 1706.499957389107, 84700.5241766373, 7601.90815586912]} +{"id": 909, "vector": [61974.073116521264, 25340.791435623367, 8124.30658411779, 25066.24588609123, 9663.307864431414, 68784.1964359861, 57661.29308435336, 33836.76893365764, 62726.36640833177, 74612.02579007344, 70679.58711148213, 75161.561929219, 23298.804493173753, 9466.579231355521, 86121.33615696723, 67906.6457898877, 46042.30501522755, 49420.829823745524, 59587.93735944465, 94903.36785784067, 45560.23944375845, 48223.89568890187, 69691.38869173014, 29278.715421620294, 68505.66021057608, 71852.60076519837, 81862.11797064984, 15726.289138175576, 60483.81036765881, 11519.196759404682, 56995.192762075254, 26026.223107163547, 29207.518914558572, 66969.66647009354, 34116.051274388614, 97929.26128755751, 83187.60050376502, 20912.7821729017, 25164.830356762126, 67882.99178808014, 87468.8646678478, 64437.068392300076, 72992.40203752421, 31655.94993454499, 27008.386863205902, 66293.89433873756, 28747.587673137554, 95378.31359260828, 95463.73826998263, 63745.616475688614, 78680.57883464429, 39586.88891852148, 67443.87213331876, 31505.061230633513, 93538.39283649263, 4670.038604672477, 24033.07698096221, 186.80922493499173, 68041.95368984058, 69063.88271330702, 35712.70511946059, 77992.49897800839, 2242.298403862908, 45255.376575027905, 96613.11692694256, 57023.38069639759, 42386.19697233139, 43376.25892591718, 80669.71151251093, 46489.26807333086, 93861.88494772786, 36158.54779944133, 57519.578832478124, 90251.52270619855, 77467.43319867091, 86126.79493354206, 48025.506824316886, 70531.16355414533, 11893.640355559364, 6087.612252470442, 27858.22243980156, 75060.91799728645, 58579.05825673214, 68610.41510927308, 30352.265384975664, 69588.32849842838, 45371.22952666188, 35833.87129940763, 90563.66039592732, 430.3337382088546, 99263.10479451087, 86855.97282222877, 36070.37468730329, 72387.17438146888, 887.2961408724178, 83233.50639023092, 29959.357102663376, 21937.37655239314, 96197.16207643267, 17749.33160055563, 84063.99465951901, 20064.450492713848, 67446.9830054944, 48651.93594761078, 81025.18959365375, 34882.10530148005, 25456.849253621505, 52773.37170117442, 41493.9819353632, 39151.51027259639, 25303.179405596566, 29822.563682346583, 76245.99357644444, 76424.09917306843, 23324.326633980065, 15772.534570325248, 29523.626417522897, 66654.7333998762, 61255.79012477202, 36677.45918390868, 87969.95929050379, 82476.99818448062, 90929.95651993294, 39839.93619542766, 94485.02132475724, 45254.84229659636, 25816.390163218217, 92587.40844919671]} +{"id": 2008, "vector": [52067.10437314364, 927.3035386238471, 82022.55831390354, 96683.00224436591, 83821.56128784132, 73014.247701348, 58350.13093743765, 92835.35113082576, 71772.32659940048, 38478.84112211447, 94781.99071619811, 51821.08551159009, 44980.8074284775, 77657.6889379714, 66927.40983851561, 15169.000278243295, 30613.251718252588, 27145.608083736162, 46958.62520973808, 51462.561927939634, 97520.42574426308, 29771.937479790467, 55471.00204035992, 10891.23188884109, 23576.881733485643, 12484.890248329273, 87107.90912520848, 49922.31823882852, 14946.2798417478, 29390.149536189747, 67073.5382273128, 78639.54629671636, 52529.994805217764, 31883.90883684039, 49784.27428769604, 85316.9305293757, 25800.159262485, 88124.62011166116, 37566.82434389211, 89708.69017688988, 21991.21692612551, 18329.006704607607, 67351.19339404066, 31345.367808014656, 44804.861750222146, 54308.34405102763, 81751.90619818094, 12524.869818357365, 23894.38824591107, 72416.10083440573, 28218.959959563294, 84632.5213456529, 79145.221217602, 74510.26986915035, 58297.74302236714, 81984.6247948678, 80726.99852662356, 65124.552373135804, 33701.571083887815, 92532.86812713718, 93811.05702883792, 91733.96305658828, 53355.21093085679, 8051.250749723727, 79911.46054896782, 43381.59906549578, 10734.783563192495, 23602.661770691924, 87395.41608004646, 80401.89426063653, 855.0007633616241, 78346.131547472, 23453.85816000354, 59799.60270946581, 76780.91019520439, 96862.52166019456, 80868.24428117972, 32603.63775977545, 43116.702121393944, 19539.128756732993, 69754.27775235435, 54608.18248860918, 30922.117222574452, 57889.28496095389, 20570.661431354765, 52587.35328528596, 73605.97927594358, 15806.88624298634, 8160.770617918955, 41728.95559857887, 75772.76715995852, 57171.59097284445, 84189.57486460739, 85758.55450198629, 57383.23731620972, 57858.02688510828, 93953.84096054413, 67941.66360337586, 83722.13274786004, 29382.754193938643, 94486.80838248119, 23424.88012525795, 39607.45941387857, 64618.73802920083, 40689.73327279832, 16446.4828431945, 69067.22960850499, 49293.426923949126, 62038.844176762934, 13279.921925815008, 74087.73462655059, 66892.51213663415, 79862.13517300926, 35379.90342626909, 66496.24463036843, 69388.59847825712, 50835.32591854672, 83266.68404707895, 67898.26589084441, 86387.67276061264, 98178.37225271523, 42761.63706327245, 42926.03028799953, 16466.634906930398, 33363.68941191454, 90905.98017151318, 96052.47968259577, 20010.810928918356]} +{"id": 143, "vector": [41988.258336358296, 95207.68186064655, 64496.9685827097, 94645.07799983915, 19983.2701065746, 52578.321976129315, 54606.92564060119, 3217.553275642271, 28300.431108012737, 74540.54243981585, 57766.4995399922, 15711.2416871394, 47412.362469718646, 65825.13497766465, 65853.5931668807, 93611.67443312882, 32074.263002014268, 76308.24017948816, 57349.77885694116, 79002.92763923356, 77545.90802941262, 41669.206707287085, 17063.663835747102, 18344.08787492028, 12661.747508518594, 19683.61835525241, 22796.747471690505, 49791.42001360407, 63260.18397334978, 28699.04627595018, 82513.37724463518, 20917.751297515362, 5434.735737176055, 28615.793178921245, 17213.902711998373, 68548.35807408259, 35123.11179233768, 27129.87103932175, 23524.44595988569, 79530.33704673279, 18527.222160675927, 5294.904992356175, 17972.414433382, 40772.992519249085, 49298.56450779052, 69742.30929965073, 29090.4829768027, 51429.91428358017, 74355.93998888807, 82865.19689829876, 83381.03843493732, 63291.72825712842, 24682.67642283043, 60067.37778210849, 14160.191842003844, 99067.084951417, 90942.86989040766, 14921.616127517234, 84526.34261632431, 44086.41632155036, 27435.27767422137, 719.4012337427447, 16126.469633101226, 21147.370988452574, 41148.55507554501, 37109.95481636661, 46443.94747953898, 5084.315059473754, 76330.51711355134, 57841.77681470554, 17604.649527795325, 90917.78152038784, 61575.43131198329, 45373.3727008181, 47363.157441273506, 40962.69461397847, 82521.93835150759, 45379.88004748259, 16883.32105792495, 30975.76171097478, 91531.71238824392, 29973.507560023714, 39813.356400913624, 22122.974274236964, 83540.88373143197, 69386.74649987808, 57585.2658632276, 90568.73280729043, 6075.041580338947, 54445.48747716872, 85582.40358242202, 48032.19798659741, 75603.01328301146, 38154.03325969233, 47106.92497094752, 30698.020218908896, 54832.41668787884, 4392.433849347332, 49024.086723669854, 46648.96930388598, 45426.188068154006, 53049.14045056736, 27561.047117408543, 50228.280674794834, 42649.510914574494, 47795.520645159064, 19718.62664947217, 49625.57342420958, 40221.370350409336, 19157.262023258692, 95073.12369681298, 99708.28291459299, 37630.16642296579, 34411.45674904084, 11265.33944220165, 68675.18386143885, 74093.15829174257, 64719.09733172744, 61654.77819080033, 46096.27772670321, 77816.9320634861, 20841.535987838077, 56891.79662231216, 34235.57594084303, 76365.12312864709, 12496.1659832166, 44976.71259679768, 93218.1795081843]} +{"id": 693, "vector": [96961.35519449822, 38476.1649823312, 2134.431236119738, 87001.67252216615, 60245.12696611761, 2980.6393004732977, 33931.07174638723, 66000.11748706822, 18679.67409710687, 8229.683594399496, 76593.30752071232, 3879.4397601260357, 57965.34931162369, 92279.30066249517, 74465.58774867511, 70308.22210131539, 6688.875539958439, 89350.47153607763, 87650.69222133415, 30954.482849579967, 83803.01630037498, 58381.97360964665, 24317.054671188776, 1811.9788073508448, 81312.49392433942, 61527.888396701856, 22514.26793177248, 88871.95476143493, 49037.79860160622, 38018.684091742274, 24982.10325625787, 82262.899655425, 1384.9388088400526, 97330.93390445411, 16743.851408275023, 62766.008853076935, 70392.57179005501, 9160.71954601524, 53722.181395942906, 13784.987701682605, 96907.2087752339, 38284.06551439858, 88219.38386295707, 2043.272518794148, 18828.413265132105, 54351.0391914627, 25806.886598426172, 47572.14281316523, 85631.33869051588, 39860.69395139491, 18586.89221355715, 31122.62967026351, 12302.461366687123, 74143.90353625572, 36485.76807870631, 63210.16737602414, 37869.44294883642, 11377.965060634499, 20822.164561907863, 95037.11950053362, 60649.82339956735, 88011.02921954244, 19752.391387163825, 59836.33102716281, 18416.758243948596, 66149.34824649116, 9953.559067752294, 67250.5940178547, 19490.441697428174, 60895.63183831698, 1527.5793449520459, 36490.191855651836, 77170.67942703857, 2886.920690110206, 34553.27470648667, 16655.05807313593, 29853.42842866222, 20080.840876647053, 73305.86873101824, 24529.019716123545, 19798.652861234823, 11548.918265189279, 72534.08039307597, 31347.189502050733, 55058.54126093519, 76454.19170573789, 83192.70634924862, 33437.14887378101, 61422.834016478875, 56605.983815112246, 5876.367901472457, 70396.44929829845, 5488.257186386248, 61536.71728552716, 1808.1753162148218, 42092.214747223945, 55162.70815216016, 82744.3903158812, 95632.9125675055, 46492.12727505802, 54917.37166388206, 31715.409379370718, 96562.02090402016, 86765.5575787423, 26052.592348840964, 14181.57897544159, 46615.93969381283, 13619.719814071663, 44984.95764257205, 39851.704531302545, 19464.006056472106, 62918.10402517734, 55127.24734767209, 63376.14549221134, 22035.264112828834, 81273.76644207041, 44796.31265067384, 16258.459821901217, 23442.632352299242, 94284.59760207594, 97142.72204962315, 93923.55262888235, 9036.616993618829, 1527.010795288386, 97221.42054098005, 26146.756309931152, 5561.558467059313, 75183.15425421664]} +{"id": 1359, "vector": [71652.83689249781, 72700.54181154883, 3883.8348375070277, 37393.96381044392, 66987.87306605988, 14098.517696775614, 82571.73889564823, 10751.01441979519, 45659.26113230685, 40879.563074919635, 62924.12663249978, 63735.643979919834, 62698.41932632715, 28672.45543076, 70057.37143169649, 33938.475847422786, 65664.01710986112, 45059.506817339054, 35136.11838816909, 63028.77958755998, 13067.139740765977, 1538.7296426933926, 92866.50747250447, 91621.57988416398, 53808.77311991098, 29000.562482656976, 90227.01498388374, 73915.71856974986, 23121.088139930922, 89382.54539681604, 84446.5885207089, 68215.03908752416, 99182.55438115638, 96169.80667901426, 65950.18831421835, 28845.383613562393, 87259.18983503383, 95928.0913737595, 95400.94205024221, 5909.391507147155, 67329.86364654663, 79118.47111419715, 68300.68071299439, 56118.698787436704, 35763.5318118274, 78067.66909016085, 30842.250498474466, 7744.921406632432, 14706.705868763125, 97500.52431620595, 73959.54483070355, 41381.71655228946, 43748.83084463398, 92257.72068904276, 56775.58356443997, 5021.231268123516, 83285.41988100241, 19236.23006894366, 67485.2486149709, 3496.8390333472653, 19290.792601095265, 31048.577475617913, 66886.89607242246, 48419.63339544932, 89105.59203974521, 20929.95334185207, 68577.70544335373, 54620.81610900329, 48397.5843750999, 58094.34245496022, 54907.02629819229, 61381.88945431052, 30567.6852018243, 94751.52304973536, 18294.783859020514, 89580.55579760931, 63450.004459228374, 19407.099560227714, 814.5880585676224, 15801.856227311706, 50440.57516487179, 86107.0915302576, 88985.65986613087, 85545.90209978847, 38123.902762210826, 81570.22790686683, 5963.405320130777, 73636.49415194913, 96299.0701141951, 14868.615994888578, 14471.05331449452, 86865.43247593827, 93609.87540939935, 6510.919103369761, 99823.66141594676, 73688.54672254532, 95956.03979015732, 73191.45197368941, 41449.95601793659, 16137.747689624139, 75547.05850130835, 43882.18579220442, 36227.75563010214, 54358.29273925733, 43300.95748582643, 43391.1846997683, 52227.50617836438, 25169.544710595248, 61778.32706896449, 16417.874661871912, 28234.992925450886, 35336.48419811874, 55450.717101306735, 40735.87048144731, 61495.85144656053, 67064.44008092696, 35197.105778438556, 4874.607907759842, 67857.71435018606, 96021.95920439188, 49852.66009522154, 52310.878785960005, 81039.70469552821, 56762.977821225424, 63315.502145523074, 73450.85424682127, 20750.791227985177, 90246.35331518394]} +{"id": 685, "vector": [73955.21013373097, 68959.95001963324, 85002.0451611981, 64875.42424552557, 66539.06014838807, 35715.39019450949, 94248.3518773205, 43486.01659986282, 88931.6370275303, 6672.91981613286, 80864.41434229324, 88204.55410136863, 34035.128893331756, 70784.36745417601, 14485.371147477477, 72768.79238041824, 37096.105018969196, 9493.726497407763, 29131.412248736156, 60721.42876429012, 74028.74318741394, 19494.710257162416, 63721.52926101414, 9561.87467462769, 97797.67404172018, 78603.68135347997, 77326.51661272584, 70654.25808246098, 96971.60337262157, 88500.9613868021, 22179.028439598835, 25805.380073095985, 61819.76555723614, 38067.11135896816, 3608.8062855995863, 25706.969736415318, 71557.46296568836, 91655.8801916371, 71281.34008812298, 4020.082068712416, 12964.454817591486, 23850.467266072894, 70665.60508801458, 13243.205095009136, 73173.6304648412, 97695.19724031448, 43399.68652850009, 59149.27285474133, 62755.13798061658, 36125.671229306914, 54480.67507145752, 22935.173077121584, 16808.53045101789, 15346.716180581521, 96515.26560562765, 16539.570314560402, 58558.31730160331, 97766.80274843314, 65624.20217309962, 68038.81941950663, 22953.70813152885, 65834.00819287226, 41897.3631310389, 44556.89393275689, 59680.35620325418, 1719.005827941822, 50060.82423326875, 84999.94215036991, 78162.03514682098, 7339.927909715227, 50795.99670879928, 28960.28073154553, 45242.80641885775, 2501.380418524035, 68636.50652579758, 89004.72400897424, 86969.49588885819, 45975.36386728939, 37218.791341269556, 73963.00962579086, 90775.57874751005, 4243.958672624682, 92859.68460338224, 37260.4426303512, 30205.29190390526, 16287.801637855959, 87132.19919518832, 20482.656580816005, 19934.00715887693, 37224.72047751286, 15647.235323295827, 64214.53759570713, 83465.49433737218, 12301.008201398055, 52348.520710189936, 52538.925757576086, 45335.90573531494, 73489.17621895474, 82488.44426735028, 51471.75925686557, 58034.37252648913, 4292.226133318489, 40005.68861049073, 75176.79123093764, 27534.555540703066, 8288.04277827926, 59910.32643535858, 37938.6752789538, 95324.59638476456, 39888.78550928399, 54366.85872317313, 47946.24490599742, 77863.79682374212, 12264.945111682824, 86480.10206827163, 57438.04913506456, 86180.01216700271, 26034.92948476551, 39765.73939011726, 20071.50200244422, 11709.026618616736, 24125.729146400565, 85165.43494293664, 77840.23857546663, 91695.59024697146, 75204.231376787, 29023.41681857812, 25048.836256117647]} +{"id": 52, "vector": [62818.841691652684, 57620.406148073285, 4595.422017882233, 76220.89697698002, 45241.45573885088, 68863.69325575283, 85815.54333539741, 43867.286846933086, 52132.360145941624, 27227.097702336232, 76926.75338164721, 84130.5790542329, 24222.9266849732, 12469.676584480438, 2133.3475442108174, 98151.49632698722, 72721.70523479317, 27472.30012755869, 38897.1432234552, 73826.35787680604, 19284.197040689876, 50474.18214668989, 75850.46769888914, 56567.85900288234, 40993.01720959149, 82385.9732859447, 56093.12678008961, 46632.50719543799, 41753.40781547014, 40510.48304078267, 84975.97519030006, 85664.04561414603, 24334.47897593193, 95076.4260424577, 98229.37464020024, 56638.402715618264, 27637.86304969863, 24173.322503378113, 87911.29025953128, 84227.99356712753, 26564.437493180303, 94966.85724797366, 49246.78743317097, 95461.10460710779, 25301.156457403307, 86060.16929297117, 98108.06734314047, 47448.77557919262, 46040.77168357083, 37574.42122594481, 965.3311295478306, 38046.024680078524, 30519.918745404175, 1192.8836027972457, 29977.962256675804, 55498.78775930762, 15454.37735540146, 58152.16503104345, 21455.69299673885, 80126.14756291667, 88701.57305820502, 97755.69166611918, 10140.322839601367, 11795.816113712142, 16874.900236278423, 45442.28776615349, 18180.45363526769, 2862.003499443877, 59710.061697465986, 65174.15294617955, 23796.612838722587, 41347.85568555214, 92511.78553247738, 89623.6902801848, 82320.73368150248, 49930.39117699396, 80370.01113996454, 2483.920342354695, 26533.710048088953, 6131.020815976185, 56613.10477277706, 74434.94684422445, 2393.1441009535147, 27096.15530185279, 5367.704792876082, 29784.829919116706, 99639.88161216424, 60602.4039414917, 94462.15494532065, 3771.0234578311242, 53640.44019100166, 89909.63993691637, 99082.83297572518, 17983.254053235953, 69153.39482169517, 40510.66979847009, 95104.44354439824, 33958.61635734365, 12683.558575860388, 56496.549831452634, 18161.414101149887, 51603.73722117241, 52096.21504278934, 81473.72075967364, 12200.052395719275, 33587.22316974313, 20793.243471242164, 8157.149169465017, 20850.59060991985, 28886.08004977069, 29214.502951790855, 22319.0888655564, 84200.88208851099, 17968.4292571052, 90533.28413360499, 36926.42138659297, 33003.50721149853, 86748.71700588163, 46748.25603746472, 56685.32010201867, 8345.732370052328, 24069.018174133827, 4705.090333114747, 94620.02307060215, 65458.57837669945, 69389.77963405794, 6262.6306059545, 22241.79635447572]} +{"id": 491, "vector": [68587.84650061374, 97664.92738149279, 13933.620108901912, 13762.804531086536, 54189.085393774825, 53439.007394840046, 8836.635255495883, 25612.002946740653, 52852.52117354904, 1478.3400156730077, 27886.999790816702, 55895.3677378871, 54896.56125061237, 76136.43098921167, 3109.9770168318773, 42036.22826433681, 37750.374028438026, 16455.721156046155, 13129.5149197451, 8551.790245544733, 16285.617534240771, 46283.66699905007, 19601.47106044596, 18015.935514174762, 78084.8758383325, 41295.24400336522, 85113.67692495347, 65316.10539623538, 69935.59354123622, 21377.082999744413, 68892.23818940649, 25734.439509869244, 46128.02323323404, 26001.683646828744, 37989.7676004764, 94156.29906674124, 9180.732752181142, 29900.6847569128, 59589.33207896262, 30723.110122006135, 98949.30180359072, 95177.07040197197, 49209.209176746925, 43078.059652901866, 16043.244992884376, 8296.363755038672, 34053.98344614651, 70277.47950589063, 81529.30469588509, 61278.097280346214, 77454.95788463487, 16944.840874289657, 54.65211788115987, 81828.0430731211, 79517.8154626952, 76571.21313387806, 3116.584216401663, 63403.06608706181, 90975.5296303521, 30855.74056315582, 17818.88454651981, 5446.925627716392, 1096.137688952614, 85958.78347029151, 43299.59806023501, 48349.17392526471, 36709.21083610563, 97497.7667553615, 64807.89596981447, 41005.02261826522, 50524.92145114642, 67504.48817901414, 76534.65845051865, 44634.60090752556, 79077.27153426738, 29061.56046671077, 62815.323609181614, 82064.67820806749, 96869.43318338624, 54520.087451094856, 62988.021310666765, 15101.240374048108, 65075.3982464816, 6513.685222265475, 21231.44029523255, 47772.80529639895, 37438.635838109345, 92057.94249403417, 89925.0012919399, 21825.756469952885, 36477.60840976398, 84594.51402274285, 81712.39766340042, 59284.00432837027, 12279.342906494861, 9932.211871501562, 37095.01682417395, 19712.80485880004, 27585.92364849458, 81195.11993645987, 57311.41359601599, 91137.10496473237, 71328.79167863318, 69418.77417384824, 30477.625408856024, 75895.88343511775, 58271.13172917956, 12645.231457609685, 44773.165146546875, 2086.6322915996993, 19262.65556225456, 93689.30118315543, 74496.78715358983, 5879.450770654571, 68617.34854509735, 97472.22271503847, 8097.294244813879, 48348.914651624495, 40610.17071600797, 11612.696530070822, 83525.23083701158, 78752.70542893255, 79987.09367852063, 42577.263955142764, 76152.38568002841, 17095.067644017505, 41228.184619778054, 82744.62395311698]} +{"id": 260, "vector": [33028.983769277875, 89704.3923762624, 46670.81960220367, 5346.4737916200875, 81603.41596209054, 84019.27294914775, 57059.638193134895, 19342.830745980555, 35987.119546523696, 77030.47381619646, 7972.759336048829, 19074.36193083293, 64032.78459213331, 73602.1681887618, 56986.365211210934, 8899.247105189244, 27626.795128771253, 1096.6150706037015, 16328.892717679144, 8757.130039662763, 50385.042101720326, 83534.35801524743, 76348.84092223739, 91747.18859846407, 40117.93030639895, 3019.098372492657, 97008.83474800426, 30836.679959572666, 6574.591959916265, 32393.74781309996, 41604.50664659482, 79902.89801258486, 57090.637110999334, 34057.70138665939, 25891.478677783707, 52651.08562509904, 41632.82596888469, 74223.13341009688, 44016.95160136896, 63280.198935402484, 3026.2353341865887, 81615.66096248952, 17191.500816919404, 92756.44732734477, 25371.3029584404, 551.5694325516018, 79683.47669009557, 74816.81009574932, 61012.779965920505, 83163.83697777959, 12420.704034298224, 60079.03734399582, 27037.242954965634, 72508.43087125817, 93223.37516173006, 68453.27519083123, 14224.136282039035, 33314.38227029925, 49099.76136209777, 31689.418119019665, 87304.36329078364, 32496.061164999857, 75628.95936595231, 75545.40238540326, 46187.031171779745, 79608.39439353447, 15416.327948842689, 37670.987146611034, 70960.0966934595, 14644.661567257266, 56229.93005323587, 60089.3931810587, 63997.285460698935, 70011.29106869976, 18908.517277999883, 63597.326883674, 63268.94413374037, 64091.79466776978, 42697.06798096197, 45140.319065936674, 72789.26747828767, 41236.71745868485, 95710.5762692665, 74004.88062598732, 67701.68991864174, 11584.37055560534, 94817.05470558669, 7756.129579415605, 6862.045239969372, 54024.205560220114, 77941.03997318819, 63886.44289445418, 68714.51650030665, 44446.13297809911, 59747.67625524425, 36779.555101618666, 11868.1367632974, 93078.58264339676, 53299.1816378356, 22998.321444061232, 4676.07248902977, 16319.450953737014, 2357.149256201152, 99380.50005553228, 89663.15295815912, 97225.28315053179, 25867.397347472222, 61626.038095348835, 31703.1795443134, 98806.10181766251, 41578.568933190305, 1822.9059966760274, 54557.847165717, 31336.87813738636, 84646.37909499493, 8976.015364026412, 82588.41724620888, 3114.3825337293206, 67758.99837777307, 31013.02003266475, 24310.463034845354, 50137.892881355154, 19907.878935701974, 16464.25986465192, 60247.153432744904, 93743.78772284712, 59272.203799452385, 46229.74496708926]} +{"id": 759, "vector": [27168.481816837975, 96474.10779120761, 42797.831824866174, 35672.923781002384, 32515.57066893993, 87781.81515478189, 95102.81609344174, 5361.025269076591, 80931.7493658181, 46551.62084637636, 70428.52248287963, 85674.08292863917, 3008.5435596693187, 59298.200177056715, 44935.56029341036, 58244.93937876822, 22587.81902759809, 49565.1431268457, 74982.96398924898, 75479.15064512864, 57462.12524337002, 98506.58314189322, 31076.617237043447, 36429.25304430586, 66376.2121341783, 73060.06370785528, 81899.31610784645, 45457.086541506294, 74800.97000141963, 2085.77436671914, 44837.481051466755, 48995.72191501221, 55190.09045371288, 51986.6214967584, 46554.79447617147, 20787.431952330702, 97694.63448912963, 78155.90997031647, 66284.18145739057, 25700.001847327912, 46025.41183815233, 60928.52205835779, 75030.58489151248, 62180.84385017984, 43013.788208165824, 79508.03971782073, 34760.4085904415, 32516.501356071814, 97396.57604913544, 42362.900299691355, 97842.48552555658, 44296.4471931387, 50051.74189351196, 33249.732815589894, 35981.48975234274, 19960.40493172505, 13151.11073775589, 53390.27448320743, 7195.7328785620775, 75399.52320716932, 8264.28837893608, 92549.86868983878, 54569.22216644449, 25688.66433423981, 93772.51517138025, 75784.93018755138, 81863.19933516359, 11350.84252422034, 49192.480052534025, 23302.73184488697, 1446.8938035035683, 87862.75832046839, 47194.26839175897, 25989.927225484444, 65522.829297748656, 81894.56117100558, 57960.31503893929, 54749.771747482744, 96405.32136356819, 74755.27815941504, 89086.10454958897, 84532.10887413981, 85080.53493656722, 89944.91523134329, 49022.84085899194, 20109.552287434228, 71533.0985807251, 55444.57894920226, 76373.23012599058, 32452.342823040803, 40718.43353449447, 12830.558480062526, 12868.258072952944, 54027.36651990248, 35132.04917262268, 86490.12914058624, 62499.429442609755, 75666.82589791794, 97200.44459208766, 84853.85963021424, 97197.45001309099, 95576.10492316342, 67229.58009008176, 35570.04945130704, 8232.049745837212, 55144.97410755786, 54534.065100606356, 32782.4805051857, 80747.32519050849, 49726.59214794555, 2609.921367502388, 89919.41522103746, 96300.91009799702, 42514.57023566876, 8234.91900374581, 80261.25034614946, 93901.16752217525, 70664.60685567548, 9972.74099106713, 80123.9439563033, 79888.25957110948, 85949.1379023961, 3818.128363739881, 48165.4953377302, 52738.87374859815, 88417.79098160942, 11789.10605144018, 61054.69940919002]} +{"id": 1089, "vector": [86130.91648549108, 81024.37257026446, 77815.51790367422, 52478.29867094814, 86750.36345366803, 74368.20277517942, 45765.73159260313, 9354.430642901645, 69230.1923255963, 79771.29282778407, 22356.104124600362, 44143.36367501636, 91894.97455637525, 72054.49395447891, 63201.896196129695, 45504.42695870214, 67396.45231963876, 35231.773484948826, 31964.633785401962, 60073.08789829602, 79455.84049711312, 58030.34937164966, 90631.20483272306, 97252.66804981261, 55339.155726904275, 150.88779836760492, 69413.40413711405, 46224.789581121964, 91862.08927020484, 9156.166835594271, 93884.0767663799, 83912.36979852605, 85996.06583205175, 5433.748577417674, 22422.987675738324, 33400.87328930858, 49541.81012781428, 7245.870264697618, 88598.19604202574, 97263.11316354174, 86074.0473382008, 75466.56344025553, 59920.556642872434, 2347.196353912473, 12952.862814911792, 8187.197265052937, 35665.46622890102, 39822.94730613556, 25868.039842458846, 16423.906267002552, 50054.056144585455, 23143.883018227818, 50217.912955204854, 92212.84527476302, 68385.77695191724, 7259.779691844869, 46085.339997794195, 93678.50470241896, 43596.813767303756, 32852.946773135795, 81083.03697884225, 35771.85633656358, 33038.17244695211, 64365.404736859375, 72879.44586431238, 70308.04344300584, 18137.12004664525, 2961.6399474682753, 8358.56686098696, 1485.7140800868528, 88581.04715849202, 5815.502540311934, 34429.869482469774, 63668.51045473918, 73482.35485536668, 35545.62650099844, 58198.855231739166, 376.0552912491444, 79599.38558146938, 94311.94091901135, 5881.217093032276, 47454.077551285736, 85581.41754537186, 46657.02040168338, 91504.91828134326, 76162.52617998268, 96087.4405043844, 79970.91735208714, 7788.760819328144, 27514.038473835157, 74644.62235362946, 42596.330554580694, 94200.80925918813, 2571.7594050859384, 51621.99969874966, 64943.18626144467, 96693.95474373983, 94327.60039398666, 53197.70280672187, 42313.57536520281, 1996.206330568495, 95619.12130642221, 65176.768040773146, 38801.95901986287, 39797.99774397694, 2337.109021888795, 47794.549423300916, 16588.811297550954, 63589.12245796119, 63683.89502031949, 75687.90615311384, 83764.459118387, 41052.71813593022, 96641.76230208938, 76640.21138126317, 75582.97881762187, 86707.81869316669, 46963.85361521455, 93241.41221095237, 85219.06718493204, 82468.66834077444, 50467.83934259803, 59424.98907528355, 9507.015493918592, 5909.054568444938, 53939.971864856554, 8714.983237202667, 57899.776863514686]} +{"id": 1204, "vector": [83924.20806428374, 51268.70488143121, 54478.61000094318, 50442.03538277575, 98262.15502175268, 52748.49090923356, 67400.66046119742, 56668.09018496862, 50390.78203511128, 44000.169787634746, 57642.18183621245, 51563.09262848758, 94999.61591297275, 39488.69396630261, 47816.64888679237, 75292.2155283918, 90249.57323749925, 70800.57432595704, 32468.64015393046, 14307.10021711823, 36580.50897142636, 65571.09556391115, 47996.4919666889, 21066.334623365834, 94296.87679716102, 60308.81019492297, 6139.15684067462, 7967.297127870898, 76748.68481584842, 62722.34784452424, 15972.780716805557, 87407.56169155335, 89568.4788129304, 1827.1310518882756, 34975.28726661807, 34666.837098335265, 8904.197894665378, 79349.31527042491, 99320.532295227, 74889.6772028077, 24756.7418264367, 41193.00281424732, 90668.8132598149, 72184.86921652147, 73111.2878155883, 59227.971997345674, 25495.845412525097, 6176.601264458248, 59652.86454625767, 806.3206008688705, 43650.89370362241, 9090.414941513025, 78077.47090719081, 80644.85522360654, 59885.33818680729, 38460.392701807556, 52986.033331108105, 1071.4865777066818, 66373.6617842139, 71506.38298576773, 23646.325606255257, 20948.763788812263, 24369.31071301286, 39347.379319208034, 64367.99916474828, 53170.0154888798, 56065.45805423321, 13184.650376156014, 89825.28843442936, 91131.19045461959, 44594.443139460935, 3760.9291106153632, 29468.85791281888, 38873.03546417804, 89218.27423677809, 53645.61852635419, 73220.02960740043, 75870.73900229312, 80238.22919708242, 26297.16164674586, 93567.16234964784, 75281.24208732694, 47036.416920478405, 79747.10782387987, 24403.947269258562, 30380.527538944414, 26720.4536838874, 36649.515775447464, 94738.73070545022, 29912.78198098303, 90191.92655668236, 25638.34661786638, 39785.59181575591, 20582.783391897163, 46353.82336686775, 42509.71016672055, 20078.65895173223, 97836.77632457111, 70257.3732935414, 81841.43194860518, 68117.16828862687, 22780.510025343527, 28713.030199056677, 73521.71768871762, 65710.96082888448, 38624.16482324995, 89959.97573931974, 96836.52563319451, 13210.744808477692, 82494.61774209287, 99526.57866659219, 6606.085256193517, 65601.75233943663, 77760.76105243182, 82727.22861707455, 29896.252127334876, 94831.05596325181, 31631.191814029604, 19024.2944228188, 51167.23396184456, 68945.90117371478, 19611.292324548547, 27362.151923054047, 98705.72176988199, 10782.75962281594, 19186.37411099481, 55841.72861329502, 63059.57207762717]} +{"id": 2104, "vector": [16210.716700549488, 31317.293896644038, 4655.840483353246, 26958.673317425997, 6839.401866782069, 65902.50029820114, 44224.0373527148, 39265.92486977841, 89770.66515274871, 26045.75683293412, 64794.05852238027, 45490.47651264857, 48627.98900752024, 88639.1764395025, 50831.47436683733, 3104.026794872583, 82257.31142068564, 56244.588216036725, 35721.210840143525, 16615.78668761848, 33773.33990446979, 73379.06949921719, 52996.838751202726, 48496.50492871123, 49438.3595841073, 1744.8601425805398, 13578.274110698185, 72944.29268988736, 54283.846687948164, 92611.02414441132, 89608.37970506516, 43628.16801319577, 37384.74205950778, 34686.38485260261, 92455.10603993713, 75102.959517057, 64246.67008927983, 66621.50180928137, 57852.51719523867, 8938.988858202456, 6876.396844225729, 88227.85862027016, 30686.278104495934, 7683.725466466695, 22815.102735033797, 72046.92325026632, 99985.71132873553, 9141.044356311124, 74444.60523760058, 10473.433308774016, 55653.485600859734, 7131.179788701936, 9108.419322472517, 12862.910989404596, 63434.80194838578, 90793.76425892263, 72891.52520171546, 29710.587785497522, 38644.74467526591, 39556.27885313, 94681.25748239306, 39492.35650385643, 34545.866225651545, 41611.742775210194, 95727.07009982826, 45239.62693176684, 4729.310192614311, 62402.73309210731, 22296.0041806018, 2097.988163205511, 8819.317310799868, 14524.680513676612, 18846.846323868504, 44613.107955384236, 33048.94118714924, 12753.43973718065, 70351.55135487253, 80649.5765142151, 96156.43466717338, 5212.415458222686, 12231.057874058683, 7150.028520700446, 6745.646626711144, 39359.16556016717, 97120.79561509057, 92474.56460620403, 78043.8401956065, 59050.65491513655, 58614.836665759554, 8861.351773529357, 76632.90551920711, 59137.79327382124, 9110.617414661703, 90188.91620748938, 39304.88399118298, 77986.10073907246, 79663.93535469164, 36591.24139855224, 31034.906465173928, 54741.60658135193, 14509.581187928388, 68273.98860224913, 91637.99674616703, 25384.810003337356, 91286.21742108837, 76560.11934324556, 63989.217256970696, 11472.192459859509, 7900.472628582144, 73183.74056784481, 7439.670253030539, 56868.495395762766, 70380.97221193548, 56934.64155222174, 76107.29495615199, 19385.143693690454, 97381.80608858893, 61734.66608764487, 17651.31678872529, 15117.292185816055, 40029.771479284005, 3443.005870167015, 63067.282968000996, 43562.30955077166, 54585.04583462205, 49459.364754451184, 82326.65945528072, 66800.85716557538]} +{"id": 205, "vector": [91457.0076271612, 13287.31565262662, 85069.22871395313, 6860.236512748596, 48662.12369836648, 3123.632053252012, 56783.634961737014, 53273.614786512044, 40126.081176108564, 39019.27052868236, 56303.74178698496, 10166.916573774975, 44543.933658797796, 3119.5793479279146, 3820.6926345811466, 714.4908521484749, 78105.36261430384, 87489.40079955383, 57989.53684489454, 53876.53962529388, 30938.040742453122, 1266.750587930443, 85076.21302996062, 80169.39980678333, 41549.90563951554, 900.5981702684518, 20858.950069401206, 96252.01089934728, 32559.42733914681, 39988.10616467284, 38313.650804559686, 93324.12048684974, 74683.1168371323, 50877.478094817074, 35999.87370864885, 58156.54734706142, 18782.317345557432, 90105.77743025943, 52376.160958666995, 64413.160586239304, 30231.007457702475, 12819.953584433586, 19280.07569548932, 25996.065191985308, 20032.45319971305, 35733.003037298084, 86816.00557936112, 97662.6440717912, 25143.06799984154, 62859.629258399276, 14963.633959650557, 19206.646332891443, 81801.81046222983, 12230.890067334465, 76607.56742156055, 3724.7798450448568, 63533.75447803501, 84434.89954776427, 8631.371436137437, 85255.51650749329, 50425.74874350938, 37063.91923667619, 69818.86785053159, 47830.88995170394, 26188.746730652612, 44316.93290170878, 47820.070598595375, 1743.7688990374788, 37909.43568985518, 18001.860042212513, 30837.477955098802, 25922.327380783307, 62053.73989434421, 74187.38179962864, 83584.76974937624, 97907.48394923036, 74868.41742379824, 42464.220653236414, 74206.46487738156, 58494.28663445693, 60850.15531878407, 48068.41804421596, 68380.31342337409, 81729.81570298474, 91250.0158985259, 49448.56276368453, 19946.09081612303, 16126.096463291638, 11910.95278194655, 37220.77774328466, 16762.63549005903, 76417.98930238797, 38448.388891704424, 6392.208816672129, 47676.16654118082, 90444.82718146271, 93376.03946897628, 27950.506220650605, 15344.20398574503, 46842.36911655891, 10384.80095817761, 53697.77230524877, 49070.46582449762, 34351.00739599537, 56132.17721580014, 47015.102383547244, 1246.7416932781528, 7811.773792372523, 3142.454499667413, 1205.8041152177013, 89224.4311379696, 62863.87431339149, 46517.40175282648, 2136.1084963901876, 65694.4655732397, 56925.48802605239, 48635.34526635292, 27673.9261588146, 72514.69938481688, 59627.6577371535, 14005.010051216294, 38942.24312223432, 45490.948258286124, 39917.84348991725, 60765.141507703054, 52878.534971046945, 13354.47859846125, 81073.6607106404]} +{"id": 1412, "vector": [88375.58991246394, 62023.61203286904, 26327.06493680852, 1000.2028111024797, 55008.96313488534, 52580.13244234587, 16444.853049682708, 96358.68311379495, 44438.589442326724, 93425.54883557241, 36685.39905830688, 23920.039832816674, 27822.98982101582, 25890.203011857826, 97954.36053622879, 78626.53132036325, 88974.4316155384, 35696.59488332665, 57695.447377103425, 17538.704086662772, 17538.99799844635, 17679.436527160407, 59469.60163983957, 46597.80497421643, 45238.51151740576, 52106.84096249768, 65320.8657685255, 6281.922509133109, 25094.954532810832, 34082.54706961621, 98574.77698156302, 8037.96048058979, 12330.663426915056, 31397.51348443168, 79867.64010382842, 42852.61809137514, 45390.721615498, 15544.60669013058, 63295.8567022633, 20696.682244534113, 50531.59291506984, 60418.486176139275, 71499.13782238222, 77827.96390707912, 91821.8524000807, 70291.21560782217, 89232.59198518173, 3500.407473874645, 60087.70742680873, 9936.611902845249, 11960.594879440167, 56893.839913446776, 71857.4545514942, 43353.28002935515, 88376.42816200724, 48778.75954419164, 81348.57894071228, 59068.328303076676, 63383.7617029257, 18854.621154324945, 87330.46515824797, 96475.81721683864, 75030.88976646855, 64368.16495713058, 21601.864831364048, 35512.70327928483, 1352.018157051005, 2526.879803616977, 44984.060759531974, 70110.04566102127, 53565.92000753482, 24363.89396183414, 51113.00987654323, 29399.51142342363, 44356.16081285176, 66524.61227648915, 72025.2228569783, 33655.66705566635, 44455.1584024069, 47864.98162463254, 29373.259496506922, 27235.681542510316, 75486.20672372976, 31913.750594870326, 64479.10735640244, 94597.31784247515, 41488.62215286826, 53272.00709610249, 59210.36943807495, 80093.9224188932, 25227.738609134598, 52135.40732596681, 66517.55363798694, 91309.85709266005, 43738.72117292993, 96325.51176729047, 51938.676867027556, 94446.54432966329, 32600.342637867365, 36127.38396012717, 81959.45535364232, 75047.73061901407, 99442.37579113038, 60969.081042588434, 39984.007434318955, 71581.49527852904, 78270.06008563869, 21996.989386187626, 76659.10890777374, 5158.312985068325, 89868.87545538893, 40867.06488438813, 10071.822760152705, 97823.08345815833, 80832.43154572198, 9633.087547922803, 8589.62212534682, 39700.93651295517, 69267.26786753332, 40472.298544563266, 5373.0960537671235, 65914.69525879185, 54299.50095315663, 93555.9183016272, 45253.10350869448, 9886.124604324597, 288.74946499373164, 54820.849567637444]} +{"id": 1861, "vector": [96462.85121862277, 82934.99112101615, 46231.844422424896, 43248.65271892804, 26513.058441589234, 20718.76470044217, 75322.11760173872, 16661.054504158266, 66005.75215345572, 87133.96974896654, 40782.67320590137, 656.9745916019442, 2176.424164987745, 3890.6105863167895, 86070.31681117485, 19930.196811410005, 21487.692154155713, 53483.41251811595, 84357.22415837522, 41157.49401236681, 38282.36000009423, 55226.65336993449, 85999.28169632555, 6936.42100410602, 2668.1375661033126, 81676.78015688469, 16808.352723565844, 77911.42941150487, 12539.401518084393, 93026.20325031798, 31053.817822508678, 40621.332619965746, 9241.194921956463, 88607.60277674772, 51744.831077443, 37660.89782998663, 58860.15387587799, 8710.652884495663, 74083.73282787998, 72326.91995227477, 38427.19451335313, 95113.23656374487, 17400.797552254953, 3787.2877002492646, 42696.9329380411, 66563.7950937134, 61716.469261281236, 45857.054673727995, 3636.9498639870267, 23242.37016740701, 61149.038627941496, 37073.323985294024, 6629.524977706525, 14073.735667042753, 90071.34750775358, 67408.96422481984, 54741.81449137058, 13030.284778564872, 73208.23687129427, 80465.35278108377, 13914.613940445008, 85559.80347126495, 6165.57150820426, 61557.97475040062, 87053.60619839521, 72435.85421525278, 60940.773630453405, 61125.66554042772, 9042.9271063306, 58100.45142767004, 81890.11266855322, 26026.246369172222, 87762.44293144095, 18168.305506221295, 40903.40234473316, 36098.052677604595, 6102.51148424148, 46977.50757635574, 13393.899664290288, 10876.428333653921, 70565.41298521447, 57884.91982488518, 17242.37776319688, 2913.6292168729037, 57590.362748006664, 9334.122403629852, 58032.13132162522, 16146.82541700908, 85370.98296626253, 73579.1093519884, 92714.11920708239, 68495.24350167507, 62245.54321301336, 58296.10843250999, 78712.6411230752, 4970.060742042703, 49696.73786270046, 42615.89384877086, 40976.400035695326, 71024.24258211207, 58910.90482477385, 30092.638222605907, 5712.45320163144, 8304.947840100507, 81523.56474807505, 42998.79569993924, 9696.906038327292, 82434.21425433301, 64296.84451232493, 81522.76639276302, 52709.716076906356, 35069.27243611231, 22402.47639884484, 50042.36474292897, 80243.41149968158, 48797.07540231046, 76366.63659686148, 90758.80261936563, 47630.499233235445, 46509.631397376004, 45286.34751489341, 28511.135146301247, 70219.7361577386, 23541.835322329207, 99318.2542220641, 97201.89201092307, 12833.447875434646, 95072.80477803138]} +{"id": 1092, "vector": [80344.09979273086, 93259.61843556154, 58600.609640113835, 48670.21322828555, 91416.8929043074, 89387.73239829256, 32551.78996590158, 84056.73151612199, 691.8905306646072, 91252.91125223052, 99624.08883574049, 24144.832740803635, 92799.2092139535, 7663.256581576272, 47762.87860172086, 91671.32281292933, 43183.85520495621, 95028.51463951208, 92812.64197381875, 30845.5026681808, 35003.32990719052, 25527.680917144236, 67497.0347829577, 75887.96420997345, 45874.40746999552, 73672.24784382823, 87587.88689474418, 97763.20115637488, 176.00278041329798, 87096.84493770463, 76879.51498757767, 42271.860569637895, 40327.535279318436, 97652.37438569879, 39993.595530324936, 4380.473834610299, 51008.94866954757, 87938.65977269782, 57346.65062781292, 47790.44051299162, 71069.50991209512, 51332.94342897128, 81622.9029556451, 46097.056169326956, 40140.80603370224, 90177.61367545319, 33655.998131411136, 77893.06332632274, 9899.514340852056, 28653.593287079948, 19115.412526646604, 60355.14986568653, 44014.063032671744, 18052.740204504556, 27963.750453254142, 48732.01409456964, 92837.85420074851, 2925.7926614126186, 81432.37408049444, 17233.561561764876, 27047.397424494935, 33250.66272754996, 6813.232022290438, 18348.472426377193, 10946.226154181582, 13514.629391092803, 97083.53563064293, 94201.6661693165, 65401.81960439937, 18829.13157829803, 31403.52850103264, 18063.224364404916, 98352.6732267831, 72279.44364523905, 55835.25680626216, 40865.402922656736, 77463.75759433088, 32801.39308134572, 81138.32277634492, 314.79306346798853, 48305.13663368563, 25983.328022361762, 88250.21109475786, 27791.194679621258, 15341.782415473548, 15602.785792360151, 66534.99420455376, 98094.52558108358, 24822.36136112267, 78549.80567729066, 491.3804191922555, 37704.3474327533, 49458.09932286032, 83304.29011380789, 39969.06612301207, 68106.1897210166, 35802.733225074946, 22427.714994376, 23775.722500017604, 45904.32247696778, 48411.745008117636, 33223.30137794218, 76097.47081766756, 84345.91131040732, 35373.25989313703, 98851.88661905423, 50413.00740795158, 7198.337853960057, 43284.37244422938, 47188.18679757467, 74095.18454646575, 13545.970833932364, 89896.67420707384, 19399.921193930804, 55795.61363146968, 40665.41972714036, 94683.96812981399, 34898.08589896449, 62927.792112706746, 66946.87060464329, 55436.73568971006, 93698.83971556863, 98262.48459538398, 60535.433761302396, 46486.27195830941, 42398.36171031336, 45790.37833468015, 84973.37729303706]} +{"id": 1399, "vector": [18624.893115398867, 45122.03528188705, 91696.13471838737, 35825.55404559799, 76340.23347572592, 96329.7247482079, 29394.985495425564, 58097.3725407332, 44062.65793160875, 70430.25280343425, 10789.420157556573, 66030.25088958036, 53801.8470401466, 82324.74809503958, 86972.80333840895, 59838.31395160718, 47237.086064617004, 8194.687648649468, 18592.318767787576, 10155.05406012659, 77702.66698878464, 69617.77278648155, 93372.93494563401, 29305.770110207617, 61333.34403819849, 26498.665689506084, 68909.09288650233, 19990.396934961187, 16254.023308898946, 99593.2948674237, 97790.08572219196, 9407.833256499209, 16817.702819097834, 33377.691524671005, 65761.10913827884, 52805.53240925577, 73478.90575596446, 52592.21534877439, 86626.81123721543, 29201.589105938197, 9574.984855161505, 40259.079824557564, 46190.078942201784, 66367.21443754842, 34137.58180002916, 80593.34527480292, 21396.0138321037, 59722.16816194623, 91439.81934890636, 23131.30875907382, 71439.54263231295, 64437.26787953036, 10542.98323278543, 29259.25763823789, 49835.622382170266, 65509.36196052978, 2332.1036109979377, 28210.74719366242, 30588.016192798485, 69878.53569757214, 30646.232574457055, 64538.39983633006, 96799.50749704958, 54089.89615258739, 21894.391756947905, 46872.83033933404, 59780.96180600528, 45095.8574552868, 49621.660474030214, 83232.66371084671, 32847.98581687597, 59422.165093792275, 12596.904517408902, 68853.10938937125, 37091.92204902768, 66653.02873748961, 84930.10597544514, 36628.8786558592, 5405.841466764805, 28053.36199724735, 26801.96759194502, 3838.580385753576, 67302.10717823176, 32652.471436331143, 36315.461109940254, 66746.06050272373, 21473.537660569455, 21200.986185326965, 38492.745465876345, 87955.21371917019, 50974.91296305725, 14817.395857481031, 93925.27538815406, 59340.75372801052, 8491.473735830845, 16535.171917398216, 69971.88209352008, 11147.52019149734, 27146.320105788524, 36418.34723228447, 56752.12241749921, 62643.57369410286, 92453.59293589168, 52313.79449291493, 66863.21960854049, 62087.224359925465, 38657.63582504381, 72593.2992032808, 70289.08832258914, 76788.7988004539, 55983.445480470975, 97276.95466519309, 1615.1854529941945, 39535.27836202925, 34479.032494189945, 17028.255947326743, 16532.70225534651, 3030.2803979454397, 45867.851857329486, 61929.316169481164, 82178.57634917648, 46664.797161192306, 25107.819056777724, 9795.848417354946, 44580.3446664901, 71522.83732006191, 88842.87743610931, 1454.7142444682358]} +{"id": 39, "vector": [44678.01103247433, 60083.06412401678, 55416.62484102456, 72177.945041212, 73816.4989464496, 3088.8693882769157, 5604.078295937765, 43726.819510108464, 20697.786685307852, 46464.66399161869, 75428.68699520463, 8667.197208113786, 29354.588244487633, 10925.839412465155, 44911.63683728635, 81586.50603745457, 92601.53856989286, 57086.23443616394, 55025.041668753714, 66759.49407689962, 25366.646083492815, 82246.05702906735, 24843.27038846863, 33370.36960430351, 2719.714737208856, 38874.93794240614, 72956.13548714861, 84986.33819003584, 40685.685343986144, 48315.197247666954, 28470.187498429812, 61430.16056799664, 9095.927744967235, 7536.260183263099, 43372.89055058888, 83376.58570262762, 69713.4990099035, 64435.98902965138, 47437.59234284808, 52668.20289958779, 21852.51249876059, 51231.38785719685, 13639.70366513374, 6911.253876186296, 15693.616006039101, 71990.06445572965, 46468.76700784991, 53365.30459086227, 79963.64017860751, 40378.77008273644, 21351.48835558156, 35299.82921033767, 20895.868885525735, 36854.924685783, 64898.36533087957, 29237.479448868973, 70415.65618975078, 77197.35714335777, 58731.51724112516, 28515.16668241123, 67139.012952686, 11589.363362115424, 12511.550782682112, 51625.458749242, 76054.87476301695, 82719.78789380836, 35930.11760134177, 51286.18784811408, 84506.95569541112, 26596.994255690963, 45046.62102615108, 87886.91126183097, 58937.200128954966, 6156.740579859244, 68256.69867010147, 98955.3686278198, 64234.347614247265, 23444.12775483352, 21419.136541941076, 10835.632519805105, 76361.22662119228, 20519.657560997217, 416.5670027516777, 18259.708173250856, 17362.914087290083, 53073.928997826246, 79879.08203382995, 14960.48518877009, 83784.58485212174, 73442.59244017143, 34110.058363918164, 78977.70006102943, 86902.54926459324, 47391.433150352714, 34366.83979567702, 83738.21453051586, 51696.76847315785, 70370.44984814108, 13209.447604293955, 58605.69733445527, 12245.92235661014, 67240.0875669344, 30855.73374531524, 79306.91265071185, 70822.12686645571, 66431.05836729793, 33520.42105977111, 25255.14006065549, 47361.53715530553, 42306.15340195958, 40360.90163278282, 90626.8657446729, 30545.111816950164, 80344.17876452263, 85178.20099207468, 67735.84082599606, 93180.74471707127, 23692.961966842384, 23933.953413340416, 77517.22311418391, 18353.820860891134, 27917.930842059337, 14569.166555776414, 13700.148972092307, 13420.76054532998, 4704.623811576003, 20556.72395859047, 41308.83448953419]} +{"id": 1344, "vector": [77054.79282180766, 86455.13990455358, 12841.544131210858, 79241.21956095523, 45166.72087840735, 35339.28528281457, 93076.585133069, 9070.023845318754, 48371.24393467313, 5255.106662859, 73112.28203341491, 84585.38697804327, 30483.929369357487, 78396.52528352347, 61098.502693672526, 12003.208482179196, 93109.90313814596, 85643.97261457643, 70327.68091761156, 20891.764172483006, 96433.16618645652, 57894.83783537694, 47489.2840236259, 29427.297982633903, 34537.09405206863, 4786.580542454677, 4449.14405690302, 51027.97501276878, 90473.6146190062, 97103.9002227496, 71998.68027967155, 71201.68060366646, 38182.16694933422, 44537.69612603301, 93490.49179606262, 39955.96568821642, 61949.43330886605, 54565.46699783796, 30758.128021473753, 35653.208030266556, 2037.7192110464182, 44060.61674661783, 37023.800999957864, 12207.737287530528, 5823.80758243588, 30445.543389491235, 5544.748337655336, 4717.793654714675, 48698.80537309893, 89557.94448344847, 43828.974015611675, 31743.658624646, 49613.40431165944, 12675.860096821934, 97991.42480368196, 69493.1764477761, 99461.97616405143, 92865.48162426766, 81725.7576730374, 42101.29687068329, 62297.423975327714, 50590.7683820203, 11153.882857173669, 9882.16684862485, 93195.07201628883, 91917.24361265116, 4575.491241378249, 86581.89216637185, 84296.82178624776, 13685.14058634399, 81247.72228740108, 26292.34656821633, 1539.0647201939166, 31442.335900541406, 78081.89506297006, 3366.3206742794105, 93168.35683173078, 46588.60145528557, 91714.26825278708, 67696.56552387292, 92666.00606597115, 68199.06252923961, 23393.698344883273, 28798.9345348726, 81680.54994038145, 34747.4723330284, 58275.59788928644, 39488.461812759546, 52077.37057635773, 76082.03355088366, 54980.115337885625, 28948.954320364985, 47040.2353423119, 53732.08861994058, 22180.81785113749, 46338.00716954859, 42310.73207658462, 17406.819459090173, 20008.214792886425, 53204.35446701083, 38524.85127425328, 41574.244205688046, 7141.146226526606, 54032.355784698615, 98932.81692972408, 13446.859701257019, 67406.6289410507, 33793.280391866574, 40932.779344369315, 41360.23616948503, 6429.125553199777, 82149.64840022597, 59520.18909301462, 70235.87284544758, 23470.189542605014, 55482.43791103319, 53632.44418705169, 77834.20951830802, 51439.8592956038, 61369.074272685044, 72891.68869729646, 26012.92070476211, 80949.51088829323, 31921.755808480622, 55789.361339403134, 1595.6866510427049, 63515.734069999155, 56502.014365618634]} +{"id": 479, "vector": [96468.52297333449, 74044.48885507967, 64423.245965910355, 92553.34507327405, 62197.919266651545, 63470.41105238066, 1555.982408879697, 80623.65304744865, 85303.08586611747, 95007.24818850875, 58983.98872919008, 91003.57036922964, 61832.36652919941, 36131.264263026795, 78543.25729729285, 95385.79417306803, 66737.52149609545, 3623.7095500595838, 48980.09633376359, 22894.661990749344, 21470.324871108736, 53964.197658363955, 28487.947357988818, 18198.98740521607, 20941.85205724418, 3433.2932745718804, 5706.79259968404, 44023.25729890325, 46571.58005211438, 64498.719144080744, 28571.399765797767, 64972.002461459786, 72393.50042577366, 66140.40516072807, 70615.06960300286, 4054.8004917735693, 12424.091704950957, 49673.72139275298, 17448.137779250585, 63245.43019336235, 40144.260504438025, 59240.85430691817, 8982.273955698873, 19855.451051490913, 87163.27078244461, 82036.72593476057, 3502.888076911381, 57616.17744572981, 3415.287306207293, 29173.939344976097, 6508.958956087241, 31764.860824937026, 51238.743288404454, 23199.414251137663, 68629.32500696315, 47419.09071438619, 16287.398373643326, 37630.83553375467, 58908.75058407811, 39042.52487269499, 53642.283191746945, 23230.23822815523, 59109.07268062452, 14829.607344585927, 69886.17019835638, 58517.37737803582, 11148.991132889796, 26294.81627522461, 42868.42415266382, 6290.152453659925, 77075.83018490962, 49602.61447106725, 17610.827225452053, 18477.031055741732, 39423.33530237924, 16487.176673027894, 32337.61675850546, 81451.91830363877, 848.3477154349828, 96824.98689606976, 90292.75657265208, 31035.186582953043, 47348.446024304045, 27785.05811438845, 40690.63671990558, 4500.021031054713, 17140.61019107488, 86928.04361062917, 74780.59045265759, 86186.00522732823, 47821.10120462888, 30586.220269506026, 959.6397751516106, 59552.56900215658, 25470.71349214507, 93394.78468095695, 63326.14266972214, 51974.75349931782, 53087.23292824753, 62570.68028151619, 77732.17391674517, 72798.79760504123, 7263.587502595404, 4202.432920252363, 35490.29689223361, 75258.13330087367, 78352.81118240117, 29499.51339065562, 15505.421822946708, 96165.82232407994, 78753.72713033672, 71466.47566081557, 93562.7456802755, 70044.01481759081, 12728.567715501771, 4265.380741392555, 27135.23718947536, 76514.63266857123, 46509.07568814082, 28321.51276874395, 71340.59760407526, 27720.46428067092, 18669.21503582668, 34289.707389094445, 72367.24901162727, 45623.310918328185, 28679.632346882987, 5160.236370631399]} +{"id": 1356, "vector": [1984.844518124973, 50418.299409619496, 89377.25992762642, 8242.624212696037, 63881.606847857794, 67696.92181346494, 35629.51629300816, 48785.42399897491, 21473.00871287324, 7421.249191589274, 41789.62617875034, 8483.780185627942, 3200.4036828722437, 59321.572754250265, 63341.98530627129, 81002.53299499619, 66755.1878376003, 61824.76417682611, 58894.75145405844, 34080.74724305936, 82370.70131093604, 50291.206847822745, 17892.28195420256, 54976.402854962995, 13862.834514329825, 51520.089067239474, 51511.92155823699, 94225.86022559067, 42494.3581314804, 56017.67022631399, 43268.463274012145, 41481.72092169946, 41394.50878087233, 41303.75472511645, 64493.21411709421, 60953.71235428124, 56321.579726886965, 40189.020744951165, 12343.426186671424, 59394.5254966591, 77409.12820215539, 81199.8381509952, 13999.378048544042, 93428.41029054245, 96081.76877590762, 76861.59841825598, 14965.912526210568, 96138.2805844541, 1215.1803467916777, 5884.635590885512, 23370.321420804863, 96475.41182618591, 42097.06091395775, 43937.62020424241, 92385.30770276839, 39220.463495528755, 92338.87113867736, 57384.432809783895, 68155.57642647845, 32404.623674217703, 11213.699055302417, 11415.137324622083, 19493.100999368562, 44247.88388863276, 4418.513729552154, 98034.96920833793, 33991.678383704326, 96851.65296264025, 53699.11996474539, 74053.19187391606, 65424.37790701169, 99787.00801561726, 70261.96410612411, 39028.71028022281, 94073.55766740412, 40041.911712074165, 36224.79348927251, 55648.990696955225, 38950.35813961087, 17992.510599768986, 30990.627137168423, 39230.414548606685, 4151.21642943842, 38958.896032539946, 1430.7804644544954, 92556.16141018404, 21190.47621544118, 9920.441528003475, 20882.996149653187, 49208.359152828794, 64221.9202432358, 25953.753704769744, 89070.01516189579, 83523.58690463722, 44689.168284564716, 31394.730559135765, 64227.23829772049, 53977.58120444151, 10073.575205220543, 67854.01373515202, 60460.256660103005, 57657.195558987594, 21311.837054271364, 34646.460883771964, 96099.3340258864, 19159.954167791948, 18236.233266898682, 37052.20676819013, 24996.74338954142, 43497.855140437416, 13157.61689294318, 91630.53480566971, 17495.991489449647, 68934.53289675164, 72776.24293150945, 86471.66038524309, 17377.37456873374, 6479.4300135797785, 54672.8369435214, 57438.70543871633, 10094.893542376627, 87743.8300004883, 21758.30206445789, 54603.418743334274, 98048.99348897442, 99689.83478025829, 32116.675674798713, 4979.275816063478]} +{"id": 1956, "vector": [56115.617139732254, 28797.13846738464, 29547.459967372015, 94539.74941503406, 62733.39193926845, 91423.32587530588, 76801.06462544348, 44282.26351667313, 40469.69061891532, 65056.43616015699, 55626.88618673091, 40840.451672581745, 6487.564922338551, 13785.814669236363, 54565.01930533115, 28637.871225141687, 17361.344875014252, 27716.294508659612, 51935.18536797409, 23450.343043908528, 45936.25861168279, 83063.81258194517, 60352.93181159066, 40896.13532824549, 94016.40047150197, 40221.57129553284, 36274.512425737514, 70315.53984457918, 70014.9187739247, 66722.3157813671, 75956.95659243871, 35942.03804684629, 76071.55848340203, 71001.11725758697, 31841.691625346346, 3897.69408928754, 39710.71762398428, 95169.04009347831, 56101.08984443842, 27644.758582289698, 77681.55051968903, 50284.15301843834, 64295.61934975646, 74839.14671459282, 92364.12356092437, 3128.17223640498, 99155.3147381643, 98835.89755149573, 86284.84929862485, 54080.75054503735, 13646.492114939247, 20455.040186564533, 6904.879319022505, 34744.3995746047, 51958.80843020755, 8058.048019565833, 19778.83553094584, 3381.5829851169156, 68770.3492608715, 352.3264811967852, 85989.10376760628, 129.98596986799615, 82563.07351948005, 9396.862375461646, 73511.4997051614, 79296.55630400463, 2697.1466617775163, 86976.66786902882, 23407.658097051222, 12534.97797029074, 59063.10223950643, 52752.911230890124, 47076.161848345524, 69361.89579743613, 82280.50496314553, 90319.36258681564, 18996.66489167463, 82390.10847628357, 99488.44147760684, 95910.22573157812, 26332.734256694468, 10767.787098069004, 39780.33332463127, 10061.611613663268, 32379.027143531057, 69990.35485027004, 73653.61430282745, 33400.90016334243, 72790.05955430065, 78099.80725725055, 35578.76960011365, 86086.49072205993, 11489.356608337153, 36153.82805577949, 2232.899417985046, 66129.6734727279, 50470.60840489941, 94068.5998367767, 36639.29853651851, 30395.836414181565, 59391.446057147514, 35995.53878006886, 64396.1715451241, 5255.714074774942, 26727.1290476887, 21949.493250962103, 52235.91543731577, 92132.45223393636, 59191.111857281656, 38280.75845605621, 49035.54671597801, 56729.145765437446, 50387.56098950969, 70131.62208212815, 96041.82039747405, 26954.383636600054, 97853.41019759313, 59199.09394174987, 34337.042743633625, 39365.355220994556, 66600.14897050225, 42538.193036078665, 28736.830293120187, 73117.06060859202, 7003.892569173542, 28657.51538276745, 56435.61391217744, 1527.0266707223868]} +{"id": 594, "vector": [68128.57359335039, 27866.341202039734, 54807.65187531971, 59100.71732064403, 47166.38438277, 18862.057195819947, 7753.435412416421, 42380.66810018704, 90459.43908176084, 29116.161022786924, 8544.250893577344, 59443.24018018373, 92943.71875560589, 49613.86219868029, 21121.80271755585, 81734.37447153348, 98711.28818409679, 55107.24031031239, 43024.400751827874, 28406.37272027726, 16232.147762175064, 81765.19983782021, 50129.73964180636, 93391.97990865895, 5634.584449445912, 21747.562953930967, 68907.03740476788, 16890.946938678775, 85945.06514414545, 46087.936800647236, 25960.750551968693, 68388.08118416979, 96852.59195156119, 19479.02285294455, 54717.56941539654, 57373.87239613798, 24536.24622149917, 39836.041377023335, 88708.82011722583, 74437.60533873472, 2951.534444816795, 34012.86985338232, 84180.5132926153, 90440.97557032248, 9190.920146069591, 31592.34678271924, 14130.168509948937, 24510.23377087794, 36807.92898732399, 42211.69812307508, 55038.48735662188, 44632.495392765646, 73856.07972446432, 46160.40726370213, 61510.55421710561, 69976.51598791404, 74057.59500083931, 52172.83321061305, 87567.17132347007, 72057.31800208082, 14791.482841552994, 858.723684899032, 44214.64160544833, 69142.1442856255, 59795.6127259305, 70578.52280705268, 90916.47520518069, 84814.58586953746, 12456.120475575404, 99434.77919520512, 20242.100469954617, 6823.78778856072, 91399.59246564831, 5901.704928679952, 72551.20875964426, 3884.6226942609974, 4448.196137240146, 50039.39107517484, 9999.454502777306, 34556.11828990568, 59393.10393783358, 14827.077725306448, 27704.531777897868, 44419.08050931566, 14867.207897749835, 92050.24997464733, 96241.2399049975, 13505.264665269135, 8456.577637978346, 32769.291590412264, 52934.76013003271, 1656.718177553418, 23039.34639997197, 68002.0456660399, 10679.746577719818, 9720.326484944719, 16107.69378570478, 62424.98128119805, 61936.696621574425, 87963.12049457766, 96629.06180091933, 69832.64694522678, 86507.39585516858, 16242.230296670967, 83300.69400435167, 9694.225382812172, 50153.88768211253, 33359.88159896735, 74153.01177515878, 85837.07442649856, 81190.56500036169, 73489.98194548672, 34296.31319158211, 93257.30037785288, 30663.6805990673, 32512.126639556904, 38311.14206070505, 61122.95681176838, 16020.892448863044, 70174.49784546882, 83015.96231115566, 13427.943508097705, 58569.27598279128, 65599.40633487831, 82446.83891568016, 66985.37225920263, 46956.71295254376, 71696.17571978859]} +{"id": 1413, "vector": [2371.912561001488, 86072.690882943, 25605.393316850234, 56698.644320575586, 56129.37413938713, 66754.28957140722, 46391.57725550932, 92861.06543368928, 43913.9844935456, 29621.735626447888, 80064.14983572465, 34617.810157595144, 98466.7806076729, 37902.67375757619, 16414.770491007523, 54548.408160102015, 21397.967981603528, 38654.19854264699, 7480.6807520670145, 45593.36440079955, 65812.78655495423, 29927.289165820202, 81930.08280735459, 93192.4153170201, 89630.27913417129, 20032.35877932761, 49066.158573935056, 94790.32933997402, 55343.53513413057, 34893.59749641748, 74886.96245388668, 2062.430826561379, 32198.00626816649, 89559.4825490398, 80714.5573257814, 30519.05454833387, 78225.47694958016, 56782.60495280195, 6405.506600518118, 2702.6213851651382, 88791.00513649736, 76551.04533026111, 50466.17724649592, 34427.416192760174, 514.8683466604575, 30902.88562358663, 32453.225117832164, 96621.33208890159, 43840.69935859614, 32086.437890650755, 5161.755632953058, 14477.305715573064, 65876.50254110576, 98544.14128550657, 37537.42618347333, 3999.8640002124607, 10328.07241402972, 11588.220465227394, 54732.85051086784, 86387.69660888537, 4017.4176637183477, 92432.64552513057, 20509.55400533805, 98523.90627244025, 84238.22779360985, 44345.49501680142, 21383.86310847735, 5393.617869068601, 86049.82376025086, 85417.29982171171, 45883.916585245584, 44370.62295469412, 92306.42058673062, 50970.530021342194, 8596.853826338547, 4306.603174224688, 53353.20822496058, 29133.827630847896, 61002.57936148094, 82954.54103458021, 77847.21214963775, 14811.770401592783, 333.6723624588478, 58360.817306692195, 663.7648320914291, 91380.22513278996, 85729.5990285356, 53158.35141734728, 52954.977704560144, 57951.90281945089, 5991.890240829256, 82853.31192804236, 10384.562273010999, 23874.19681034606, 97417.93938844699, 94438.42184759538, 11737.48345757577, 53857.247262957695, 3264.9311575516203, 99631.24228470567, 19626.182386698663, 87589.82352803212, 92752.52557478688, 96426.15595995798, 41042.0899465365, 1656.786888220152, 84045.019852068, 35359.23619202287, 6324.262359364808, 95757.31621518357, 2030.7463535754966, 88296.81808253142, 46916.62813091515, 84807.61386404115, 23134.13360588825, 31836.82867970503, 29153.497859300005, 10162.780449511987, 48184.00334728974, 32314.49966262948, 88969.58492322596, 9781.379558581439, 70064.93672931193, 26929.727139925864, 54043.097326578834, 78089.83102390135, 28812.31484065091, 14476.802147489987]} +{"id": 2002, "vector": [42443.15956659702, 86422.56861560098, 42885.787583792575, 10180.804480207395, 81540.8504885496, 30445.823854981878, 42791.526510197786, 94803.86967362235, 93839.28070789234, 52807.934745321385, 86609.15441316139, 58730.5993873689, 3650.777786915216, 10534.718607219951, 67845.37229741368, 98353.76373430666, 45096.40470138028, 80242.51024895138, 32920.13663167535, 93602.00033765566, 65947.63795356988, 68547.27715798221, 34219.944758021, 37867.16119537962, 47893.49450915546, 82437.85265838557, 43018.25690652779, 69581.42423465801, 23187.19847442359, 11646.750029180053, 71197.35331812229, 43115.553521114656, 10859.031397615892, 81687.71691986492, 1955.940700064318, 65701.41800317231, 30098.107503482675, 3104.94689981492, 79525.66527916989, 11550.804891100219, 68403.82174630798, 95112.43384408404, 72250.46336559547, 77744.17893745407, 67306.63130812685, 67707.47250993397, 68377.43287578653, 24889.821957907123, 79271.12972782692, 43745.22651857181, 60289.73143917092, 23192.162366024837, 34833.22672137881, 37332.09363976572, 14808.32421643089, 41064.98098756265, 47802.31290797236, 73549.5406043555, 82785.66115401503, 77393.41273757319, 95071.0664699182, 4340.066601106729, 83126.86491064997, 3424.202347661798, 33118.34464423659, 28533.71659806475, 61971.99060231633, 67480.70123967524, 12227.878400305026, 29666.356191133546, 77367.90194701946, 35430.7548446844, 69062.41458121935, 78858.48267642347, 77309.92773709445, 30830.667902691388, 2006.430087229305, 25064.30999784409, 77388.48223347096, 90116.06099321904, 90430.6474898497, 7490.772617009056, 91968.48955176416, 99065.64220561722, 14745.606598013872, 21794.89004808509, 55683.67906956992, 29357.485688882156, 23081.51186859273, 92681.294805131, 10979.132563984616, 73939.38466751085, 75578.06268130305, 47599.797364970895, 64625.75789182948, 59520.05777365932, 48644.96321593137, 46942.42733727846, 48273.46524562967, 26362.060971460876, 26124.32852596034, 77418.77234150881, 69539.09392328243, 10255.704833547297, 3182.608203989712, 21030.65502178524, 98708.9819785001, 27140.946587662496, 61839.25838753021, 43294.50344639322, 85089.42277733929, 98822.86136678222, 56319.559933071985, 1338.5789684442261, 35613.64990060557, 5094.580475569454, 82529.39515076429, 39950.19114721871, 75605.67384333954, 47173.80444824888, 27103.169041659414, 47974.43159682932, 86717.0505928694, 5187.187876253963, 97704.6568693786, 7859.601336091359, 700.5423136704825, 65238.73757817559]} +{"id": 1215, "vector": [95822.84200285768, 1088.3646149490444, 59516.328975835306, 44392.07669161562, 97247.49241146393, 16885.519510990853, 32657.94220962439, 14803.92540672829, 53375.32211833166, 7347.247206875851, 16128.16073110147, 81077.71562409887, 59728.38251139626, 23485.40831694411, 56233.797675113376, 89654.68563628377, 28522.851951115448, 83545.94698602101, 66163.4678791194, 18788.26413521144, 36447.82558749604, 81953.40090853036, 43501.90743287955, 77711.46588982077, 4789.741798863256, 94775.86869268533, 17792.562535029967, 18558.151833767246, 64830.90898948951, 90175.06575273564, 59050.17372390677, 30619.471460964152, 48101.63681262597, 19615.627091507315, 22188.719999083707, 76604.76978988468, 47131.235600562904, 29146.67031474627, 98222.44098395728, 52931.31420793601, 94318.18123988099, 67812.49750649204, 24863.967144008857, 99474.6992901802, 7606.028965492673, 66651.17572858749, 68289.22228028288, 51384.55820595373, 3906.9161258533213, 45634.36689850581, 93693.36445455257, 503.28990265742226, 79062.17624478997, 74441.02412327133, 37367.11256907318, 75736.3751883128, 14327.482769925204, 75174.76755456218, 54994.87144825598, 43176.11109262488, 6108.853759273147, 24863.05009727384, 32137.27400013189, 74455.39399243434, 46432.79686273171, 2285.9290368482775, 21367.408574636225, 46936.23388788609, 63894.02314365554, 86087.11647910056, 57358.5774614255, 35202.409666959764, 3030.7448054305073, 88914.55889751855, 17988.188698012476, 27712.177655957228, 41373.11524880672, 69179.02772382573, 69999.08114939691, 35144.1358462348, 38010.91270098449, 39237.192559789415, 85619.5282405898, 21262.537469659503, 47360.22323751985, 35958.4958099974, 90005.22775942707, 55412.77707962964, 91833.21247736625, 77613.3813937277, 72854.46156091466, 38604.442003035365, 6418.223189024586, 77441.99535567652, 40393.236347843165, 78477.36274546759, 52289.70012305374, 72438.02204598261, 59648.479880172075, 705.094234520165, 69691.72444916633, 89527.19414224642, 20511.249921593568, 86692.32577868408, 29057.110965471387, 95949.65213968018, 7659.428334749485, 20026.777087466395, 13244.164943494341, 96730.45047207542, 58474.62968680601, 91404.05549113406, 27259.938620904777, 30226.911422037338, 26528.947069262987, 33226.96400150824, 84097.28087342475, 69350.79204960453, 85651.64887695342, 95840.65027135384, 22089.049729101353, 74128.31760339475, 86276.95897781814, 95608.07708444979, 78335.7209681823, 20406.87722820388, 57186.81712525461, 64592.73671378757]} +{"id": 1715, "vector": [23916.40072587178, 63932.68059787436, 50886.16379677482, 345.4266063843137, 72904.1252830037, 69922.66282857684, 50972.179460561085, 13920.574390406593, 56885.32293561135, 9091.471170316634, 29220.355438956103, 97470.86088204372, 73565.58391293797, 60974.55370272292, 49651.61816496305, 64487.240834696815, 47853.30580694196, 91665.95782909456, 45671.557528931706, 58260.825019652526, 80778.53748086953, 65158.05851490551, 9699.864558614647, 5143.522320355298, 62505.85884240264, 3468.968923863225, 41336.75163355087, 25820.991484984836, 72658.78511798843, 96337.0613474765, 96229.59077106787, 72995.74594231078, 5499.765554397706, 9538.483028554134, 37564.61467778333, 8763.353113932393, 90985.77508049025, 9817.110837171927, 26313.709170155686, 42954.13417053635, 42762.97854357126, 95532.67108114227, 85679.98776412003, 35188.52951062955, 76064.13817798707, 21792.646232729374, 66408.2917464229, 69762.19692332117, 986.518552309379, 78426.326809919, 32223.518888438874, 69675.05173050986, 19917.68567376779, 97685.04857986026, 949.2617729539021, 8318.74527666413, 97128.71263393812, 52960.97967154617, 29673.840620906165, 43222.901877095675, 82407.1641613855, 16477.84431247721, 42808.51136844596, 28829.12091937272, 2452.663479367867, 32773.62639466396, 82194.35792996567, 36396.04569425359, 28092.213745486082, 65616.06352219995, 34220.30343377943, 38758.05561310288, 38315.64465810531, 31270.478103624155, 76605.20487747701, 90024.68464052751, 45874.096626174956, 74504.41503349738, 57035.9950645495, 91458.48642255233, 72234.28414480668, 28562.359423262064, 57873.18489965449, 30316.247716225196, 42173.06715878276, 10475.15560408322, 21978.02175880138, 55899.99270667423, 85060.25854917178, 59692.631307002666, 78545.4321252885, 64319.05622858479, 79017.32811524316, 94082.02290166433, 51638.05755150318, 41048.7241050179, 84857.89561044575, 50777.09218011169, 61469.19570944729, 43018.44849290286, 35055.36505028182, 56410.63073391926, 57059.323858317875, 63671.17068333661, 77042.07608450478, 88948.26887652885, 13367.564415360477, 56281.7544753889, 56362.917640121545, 1901.4690878622841, 50972.30779964591, 33566.40378655516, 10488.012588652862, 38203.807975461954, 39672.457236620416, 43818.34319409767, 8283.019611709797, 78764.70152182636, 39472.19747520909, 10240.810365454956, 397.69638477205046, 60729.17568686837, 11238.029666550665, 95380.41292630571, 32561.6353668173, 64657.97298081432, 93085.73506957207, 35048.13389667148]} +{"id": 1346, "vector": [72819.33843765409, 73949.51864914403, 86008.75628433982, 67117.50038178805, 46995.8936139666, 60548.60904136184, 39310.2229041276, 39146.89313028564, 65696.69678896012, 22038.031902323284, 97874.44082005376, 16174.007332745743, 32366.45298955799, 32795.690154328586, 87201.03791925403, 38234.63815656172, 79230.1039269701, 86218.55092564547, 51223.813838148315, 42328.2614954686, 72793.9261783762, 97613.88812470053, 49820.48953307816, 15986.747970188098, 94880.68384873311, 769.4608615955945, 19869.90118270423, 74451.53616251984, 65918.73177493221, 64959.52722443107, 14895.577697166418, 75678.26966660666, 11361.719039593943, 50297.71408423546, 34871.15107978812, 94302.90081406465, 14554.048668125663, 21431.218429706143, 84695.67568097459, 40286.91573356587, 76987.054010074, 18.138635681874682, 75976.4368560954, 64794.90780146623, 1609.9510138782857, 84994.87101238698, 30403.579609157383, 44913.43609808025, 19797.909041324503, 44202.30362151984, 56368.22806252687, 49197.076940328465, 49345.8508049035, 38934.0605311675, 589.8500479883495, 63758.30016813275, 84275.21931511587, 95446.02924021608, 73659.43573255415, 72396.56269109134, 88768.47822993585, 6586.209326164693, 97174.58159828516, 7533.423615788181, 40923.81826650213, 27747.82019536699, 44944.386587939836, 92625.85401388173, 29543.74560641241, 59205.30382509437, 78380.35969067976, 54775.93547751601, 48600.39622120833, 46834.56661407499, 31970.907147875492, 26906.2889294471, 91620.47662233336, 44857.26909816524, 78302.90098708858, 25134.513807449486, 50889.11716132494, 23356.979674182134, 82404.51953917884, 49633.00522553391, 26912.932801505718, 14999.058273232124, 86246.90238413334, 80788.45450922025, 9664.300821398352, 84397.51986336349, 816.1723024285639, 23332.570005195263, 72908.70407859559, 7527.199044897426, 73109.69853880483, 4355.208282252776, 92420.42136115712, 25559.047053219274, 98996.21137543848, 46073.530300982515, 82818.70989684212, 41654.06997841431, 3960.842240773088, 27178.71838833833, 32935.67672854207, 11147.675888782238, 31916.679195914745, 66539.11247570827, 88669.92012018956, 33700.51903708544, 45083.744438468275, 29527.33952587415, 94194.61263656702, 70371.62563432408, 62452.41832227332, 38663.17834818057, 35720.865457631015, 36685.22932490711, 4120.9136650966375, 10536.441305954302, 72896.75972816092, 51919.144680207384, 76895.5168246797, 84128.74112855273, 7861.836206189232, 72478.40604872027, 40697.85310654277, 26241.46609907362]} +{"id": 1968, "vector": [34596.79611736997, 57403.159663551975, 18473.572823923812, 60763.90267031644, 31814.934028699183, 62803.302084715564, 62630.73887881285, 79922.47744761904, 92037.71305322628, 98330.30191632142, 38957.59897629602, 62093.67345683008, 59773.21376124196, 22667.627563733095, 92762.656372106, 2517.3348605676215, 74623.66314152058, 58625.458694928566, 20941.56560130115, 46918.69759986196, 23487.61781180437, 30133.697630307623, 36964.08700192895, 88283.00429491111, 48665.168394557004, 92559.49128588176, 14266.328716604137, 64345.195348014786, 13907.337089143379, 98912.23775544309, 32667.084725480556, 48034.520599311916, 49243.032631352624, 74497.506625118, 57188.78403929983, 88707.64828275914, 57896.94920510831, 96526.50388387933, 17732.672583293806, 9663.375662403407, 66374.17005331154, 49824.05202640228, 86967.9449330703, 3007.7613138790093, 53636.440200945704, 40747.10608654332, 19994.119579804283, 47611.70485844587, 40014.78213318268, 51206.24573397592, 84726.35765179986, 29318.207700524345, 83500.45235960656, 95390.0508028991, 27675.099700250707, 76995.98731860668, 39277.95248853189, 21817.452954952398, 14308.4953698406, 5684.470845963796, 99263.66996791176, 10046.45750836094, 72019.02662767607, 82080.06740610718, 22391.4294072992, 2487.277569098445, 20855.537532334, 76817.61889405016, 93008.7420103534, 11356.29924369529, 31929.330555761615, 54282.7325188992, 42910.67692347562, 16325.304849077971, 63159.82130668427, 78272.45200723442, 35965.4425740069, 89811.331314202, 8557.042532257941, 60054.051928832305, 62116.83754315363, 59172.08801948033, 87886.71027640224, 32321.76981818513, 32662.69821687485, 40069.51708142904, 76708.27611804927, 17361.599481148238, 40363.69050687396, 66496.8638666243, 89211.33247396906, 13495.801310204102, 74366.04360798871, 71331.32343144242, 85118.22035106755, 96051.39818324371, 34053.77111086686, 46293.28696109978, 43100.7746321068, 62365.677293379296, 81919.20090386386, 57919.036662458, 77109.28320834092, 99250.59969101788, 7503.139798863123, 23841.069454977725, 4020.1204794931054, 29698.17046717683, 24764.858624709374, 43319.8432274361, 47996.08710645435, 66842.23733051386, 96186.44095104434, 69803.15914278412, 29565.724801911685, 69164.77503074727, 90750.52845701281, 60518.95197102836, 51448.02705558274, 51318.05858390297, 67594.3581887676, 77581.71263933196, 67928.48340714749, 2798.6234072045813, 44522.483688345084, 6651.139099583869, 12259.253009090055, 13938.895226561155]} +{"id": 1247, "vector": [21358.021342528642, 86961.2565140737, 71685.21567731863, 83376.50857052067, 95782.6334839609, 76196.38448047236, 57450.44884529121, 56688.74198934136, 26559.43285001634, 65279.65806961801, 6108.274256788293, 52562.50596584092, 74213.99029627304, 21350.848987237703, 44583.28552868551, 15780.222433617751, 57433.44135040307, 45070.63274892412, 69887.20601036624, 14858.297759143736, 98224.09843340742, 89070.65358580588, 36013.367873716226, 2718.8119091695717, 42339.54609460788, 55784.53372343044, 65315.908767801026, 78564.17552117779, 25801.632620290347, 90178.30679652031, 72611.07735275409, 66174.08046473979, 54879.013423511235, 46894.80996150623, 17058.202167659896, 47185.95815895226, 20073.991819129467, 20686.994928114742, 50569.925349172016, 9055.643178540595, 81601.00446011891, 99139.94610260548, 33221.323691351434, 55610.40783562314, 85879.71058736932, 91556.95269110583, 26195.62744129421, 4711.263227966711, 24275.238333103556, 14318.216637007386, 96738.86499179235, 13882.824082119183, 53230.78742162475, 68170.33729540804, 84476.46493443086, 48213.655452743646, 97897.45347046496, 71517.1070959526, 14747.531752835708, 19933.92271301717, 38801.46851929519, 97908.08576979632, 44869.175906298, 33360.11487893311, 54000.00957859746, 65250.98936643471, 81936.25544344143, 70149.62674429358, 57893.42163394292, 21805.039980012574, 66169.259947082, 83139.27247213393, 3316.131632261188, 3665.9270193477587, 59464.71159232196, 17839.151434574607, 96051.95364499024, 68531.53869910089, 44349.90498015606, 12759.644927425117, 32322.68577953631, 68408.55746514433, 31308.946229562483, 46053.989315512525, 57578.19169685221, 27651.586612562118, 2874.4747392415393, 13873.891262120142, 68114.51194572114, 20088.65594724566, 47797.24678210225, 76922.87017754072, 23437.970111257844, 94763.17356329488, 60345.04761084582, 39614.85700353071, 49141.37876731018, 31284.433339757077, 30157.76796557921, 1615.5570345963488, 46264.12953322566, 33529.157677352094, 65238.26046670507, 46618.90892174582, 11283.57468404313, 24864.866354455993, 41986.89968507975, 73100.24397679591, 15382.2927305176, 81771.84905684, 59976.22241901776, 19732.025291303147, 75534.57009094825, 74495.85867003915, 54874.94594477191, 15040.878706138139, 84195.04858100625, 5234.7681755745225, 85215.22365908435, 39105.27336418865, 85178.93546361114, 96843.52842762641, 71160.05917767815, 43591.732070454214, 3635.6634653653573, 73772.62989369349, 96914.30244795758, 39692.627441794095]} +{"id": 35, "vector": [7280.058747177786, 59984.51448666249, 46675.423202411424, 41429.102734493186, 84102.80287095674, 54708.33917010276, 51979.590138445645, 87061.41095467117, 67569.77794143787, 27318.033557254705, 43314.60959356145, 49164.84625949884, 22341.517458351813, 59076.7559668188, 65067.49938929044, 82570.81685854576, 20588.811581086342, 50122.94957779443, 21670.38562364302, 29881.631828871436, 11043.597980717645, 56606.456049009124, 79296.9556284368, 69032.79806847151, 1240.2997388187841, 35442.74927264135, 28220.893456921258, 6400.402317776532, 42717.682846574775, 57656.65267996743, 9178.655217072184, 82673.20423152315, 35903.37178787476, 91513.19360771954, 4117.425849223222, 69048.35069728662, 73672.99857820987, 32918.6451637832, 41011.352382575475, 87076.68181663567, 76093.61226851397, 39787.646659187296, 26303.83423874737, 48272.56995015476, 34005.71207235687, 30296.442789819233, 46097.68971822476, 66820.48415459029, 1670.7103603365138, 21372.15641180563, 46577.70747516992, 72204.21056271544, 69759.07338338728, 25965.061774090624, 68472.02824087976, 12376.907309743512, 11521.371377411027, 58936.57114225935, 76045.17936947617, 92818.93548840129, 82617.10516408994, 84928.35443097986, 78892.75953705696, 20041.348601562426, 84497.70352597779, 65512.70354943447, 36935.78786806151, 71558.59145207792, 78156.45559859781, 35960.37937088629, 40372.84952958942, 93152.49670596202, 57089.87656902342, 23932.78106381446, 34077.537696699765, 31377.08742638612, 30810.785010757634, 66436.50484312116, 37131.635386078466, 48456.726388127725, 19442.959137281312, 4241.4431134803035, 11417.957623867103, 56947.19248319301, 87654.21185268534, 70695.93303159947, 8742.939782529236, 81545.90889607492, 53606.13080893843, 63021.47792540411, 71959.64398241254, 21691.883243743738, 87039.09644154325, 7713.096583642554, 94381.75775624113, 64285.97550908438, 68432.92590839691, 84321.3071866305, 81946.50980654506, 76240.533420309, 92789.57416152178, 5886.090124971277, 10484.44074511894, 59248.45059991931, 5105.003133510943, 53274.50839525049, 45180.3005770541, 43262.40611972038, 92996.64426013891, 23990.17375521455, 61684.17827098743, 31383.082810193675, 79068.6723691938, 60448.08331544933, 34869.363364974284, 15779.390661382886, 97522.93644888414, 9546.595056838036, 95825.67204855088, 91051.78328298389, 7136.104807595989, 82873.90359866062, 91685.99712167696, 23281.16695999284, 42636.907019721184, 23621.996463144635, 30834.512602477505, 14534.128154342829]} +{"id": 42, "vector": [85057.9632870594, 58124.37184028431, 96185.91497156667, 38548.60408135742, 62613.11882902445, 28233.608056897807, 52582.55981741734, 11493.357218059864, 57909.07852262719, 62391.764989546864, 93800.07488167315, 54748.788114753566, 59451.70958629619, 76837.59952946805, 68417.83715201425, 62698.740174970466, 48644.207089310374, 16379.683219707675, 41659.99955921405, 48773.99223068829, 16629.234596816954, 96489.9350222741, 44124.816750386875, 9550.080787153081, 34659.04693376548, 14389.959595800206, 35903.9872602397, 63640.002354547905, 91875.67859398972, 95488.43116244288, 52137.83684983585, 66412.96841408392, 4609.935317323666, 19481.622969582702, 76350.18669628089, 37933.49682890811, 87472.94556501479, 20124.508716248547, 2518.26131083972, 96920.01415304332, 32860.27668910177, 83632.6210974691, 29527.26692937859, 71304.22144277609, 3042.5758754136223, 4166.800556499772, 62167.339938551406, 40801.45794947921, 64622.10974192729, 27942.72734637551, 52696.46460182741, 24122.744244611604, 15653.617185747027, 64524.3373009018, 18416.97385072778, 82946.95620634867, 30853.237671012157, 90269.17151466674, 89843.59163051902, 80141.60644425964, 26658.78543108062, 30124.152739450015, 81720.20151033414, 28258.184177491476, 14492.705431068687, 20713.409227401615, 91598.21757114149, 28896.924289270708, 39245.959819930176, 24426.726951422774, 60932.92480639419, 3739.9076711840175, 45935.25467166707, 43551.39360076803, 44942.73042940597, 61182.59070689227, 17651.538720071036, 55299.77656361407, 51432.96791274384, 33849.69671394432, 95340.30603199152, 27568.357020433166, 60553.08017364122, 31866.537615971225, 88678.82984544915, 38734.72450981404, 5827.376299604725, 9289.418563849262, 74654.4072637229, 72451.79618889141, 20683.977969565247, 83773.88732028069, 64407.895981500886, 29323.72983325495, 32375.060865386407, 56778.85131296415, 42190.19643092502, 28767.946325532757, 91243.9540941081, 23142.050140392057, 85604.43077127376, 77445.74854742343, 54399.51262874095, 86463.75334353965, 61772.051345748005, 78129.4100191476, 87304.29783981477, 8897.884119014276, 83296.76447559599, 14867.458392672794, 71206.45567667928, 37027.481790197555, 91551.4967301337, 2681.948238513554, 56832.52114620226, 44001.012045427626, 86055.80973918238, 2074.3235442009177, 31209.456045974326, 43063.69143442905, 77758.99113596545, 59470.23813369777, 27894.99253224631, 79716.16601104409, 41551.54812776295, 75492.77702878878, 40870.36479552055, 80798.00904364491]} +{"id": 708, "vector": [8491.819868576611, 90521.98682757634, 44512.18809555628, 84148.81371653563, 74345.48164659095, 29728.305345067583, 57612.29579311617, 51835.76565626986, 83447.59186816722, 63375.46479379499, 51812.57059843375, 323.0476659188519, 17431.416810898038, 32090.008028771932, 733.3477697030411, 84945.76726879388, 20396.948379690028, 53463.72148151577, 4081.7965728578874, 4369.29690990181, 49358.178915462406, 24809.636146465076, 33151.986185389236, 58437.520274573806, 34445.359709492426, 39480.64341510217, 31137.675391754037, 1027.523546707576, 27755.400934303954, 47986.616458873934, 36397.4577223094, 91648.98952381365, 63589.388213670405, 18005.800447018995, 95496.0181695874, 7355.894095963422, 74155.84853137401, 35213.618605725096, 849.3559524425987, 12240.385599977177, 16880.54422197881, 73770.41082576453, 27627.902203082976, 8195.63615983736, 31728.825755161495, 62941.60691731515, 73589.63590032129, 92752.30209051685, 33311.93217501446, 80365.80516071676, 7038.608786087552, 58121.36765669099, 83609.29231056412, 72681.75648423078, 23383.295141774717, 77478.72891007928, 39852.60029626436, 63775.484458655104, 5205.662055762761, 25171.10492772344, 19686.617323362265, 36518.35069394988, 94096.75684895233, 10540.230653957584, 64314.61101907688, 23749.34952484181, 9653.173215476329, 56696.877508110934, 73383.56790055463, 54190.996950470784, 61909.51520217309, 37529.87727619492, 8658.545080059055, 5258.869957431045, 87673.28084398416, 46176.9957298632, 96872.9707803126, 44376.54938292319, 7747.205903961852, 89347.77759656747, 96178.50756029022, 74364.63689370565, 4291.978648376238, 45260.70524524587, 82106.94274737155, 42449.282448278005, 68676.739333986, 60515.322643670166, 21693.094331103126, 26150.314491298832, 55034.489316812316, 32114.90436135459, 21193.221609631164, 15130.211684353046, 81039.33558751721, 16247.722431898315, 67404.86160330413, 78292.00679346138, 2604.771087546798, 63202.791553703166, 18303.728989132473, 41999.416010234025, 70904.34952000363, 96816.33453536242, 64821.48320046013, 93660.81032289921, 51102.858389279885, 6276.166351349799, 96817.88439050771, 40746.75082269371, 5777.964809918468, 19873.719852015238, 77474.91253842652, 95904.02969521758, 20478.539321850298, 74072.6614422481, 6652.04224243432, 13791.517034582634, 33169.54041179714, 36346.882973942884, 82606.1997399486, 72799.86508360332, 31739.542139940957, 87445.75401537938, 8361.995793877386, 35425.162413968406, 91042.5628750034, 44717.52199158575]} +{"id": 570, "vector": [65119.7576538293, 28643.00896782691, 57428.155897733115, 35531.03840116865, 29361.17448919123, 26707.204259487906, 54234.948249454865, 97155.83141438509, 93985.4883566385, 4832.348773556771, 82172.07281854606, 75600.32283808867, 52402.731318921506, 47248.97613359761, 60861.593097095145, 65628.36262722984, 61418.20711427987, 22266.19531333498, 90227.25480250588, 58750.42270959749, 17969.785677047745, 77664.80961148153, 71807.26986415322, 92549.56513148808, 58530.06355845868, 96266.95129190844, 30182.255778078325, 81547.59294343754, 4830.7800002311715, 57197.01104070517, 95022.40286147392, 55602.50431407817, 44841.088366929915, 85422.33802731993, 44839.45580847519, 86198.73837322825, 59881.413223641524, 53893.82598681036, 35332.38826567705, 21006.99485273637, 98274.80524359643, 48506.37594819258, 35282.133033291626, 93103.03705520555, 73907.78556051708, 81435.81866353242, 17343.977045054537, 46187.2300336435, 61034.825749374446, 81343.68929956926, 10235.098214307392, 92418.28097940571, 12307.554812131639, 81541.1641585128, 24478.142364359745, 68873.86641551311, 86635.16799411568, 4967.259527540813, 11463.632026412097, 15335.375338649594, 3480.2016667130342, 16763.873408060714, 11086.087493417817, 68499.6243355528, 5452.745794270552, 10597.37761103986, 70728.81781588041, 77440.06024755287, 98191.54882138118, 22539.419070494314, 72908.74989959157, 61157.76195701291, 34437.51618276571, 1229.5698591654468, 51651.0663989376, 7342.009666858151, 50058.372901553, 78904.97861366648, 41528.0217232902, 19333.648452985253, 54317.98519315512, 79059.83672565929, 60272.77560652893, 66147.68360647824, 20196.654285821623, 58038.11170991632, 31153.843079448496, 28912.334468389443, 61524.21137981824, 84874.30262448777, 25985.1429579641, 66745.83891307526, 14632.097098528562, 30782.227941677054, 18621.086817324183, 68422.67597219386, 14146.208146895533, 1118.5863595771784, 70702.96077091205, 14414.806081176579, 4651.544725170053, 95161.19218078576, 48611.58516286318, 47660.132961131, 14220.583850567004, 76907.98939328686, 57268.30903175838, 98896.01781595459, 80957.97370297933, 88819.81681693338, 57390.941641980346, 11185.858004900683, 76096.06619446962, 77.76996854009788, 56246.548904730545, 6265.7773976509625, 84154.46419859171, 58614.53374850756, 55540.773314039696, 10137.56366077181, 70171.23358627348, 86769.64994743426, 55186.880366156154, 38533.304672314975, 12481.720248886752, 64693.95515050588, 56432.66189035209, 82087.78976980901]} +{"id": 618, "vector": [90692.8245375052, 22552.74163277249, 22413.717734651374, 94022.31015121494, 98167.41457565727, 55280.92609866866, 21580.00657291019, 11767.96272795071, 66961.07505088567, 87921.58388737105, 8163.352806990853, 41578.79915480455, 9205.394447456172, 22702.75175674702, 21914.227727306155, 70437.65373855295, 7253.264866239362, 12351.302644758722, 25535.140288285875, 99660.67146266198, 8964.766594654904, 18338.354420095515, 93108.11793634998, 46539.752419095996, 87726.5662082059, 10442.063890988806, 59765.741894597144, 69402.95017729679, 45744.64437988492, 54914.561568456564, 32893.501052203836, 81624.91083986286, 85551.42099038161, 8082.6251247600785, 60458.471293440874, 41724.91432668056, 67622.5798290844, 96065.06012222097, 37886.25774708096, 31475.45625143312, 10618.417615245367, 10220.729214631509, 36744.238001690974, 63607.28597392026, 60354.38333303068, 72309.577203229, 63408.68886821707, 49951.96649737993, 265.76313771181634, 9477.247816962741, 21109.41062939804, 74567.83051140537, 72029.17535104544, 99618.54103447407, 97272.79278566584, 96268.24758552539, 80434.14051197072, 4553.881864857356, 26477.375742464505, 5764.719510410144, 56462.494761209, 5759.839932690158, 38198.61996421556, 11850.813797280458, 24557.785951155554, 3514.303941500163, 88913.37318708099, 19116.212960113, 84351.84148449637, 42814.10257292213, 61199.21025530391, 56822.211019677816, 32666.743516503717, 48979.49501890286, 7017.879309276665, 45171.87081683847, 63691.68533776295, 1689.6234206926874, 87989.99973581261, 82965.59213306995, 13847.175188870386, 95096.26982931832, 22583.858252231657, 82426.62154020686, 58216.15916591168, 90988.23950747312, 21909.160184592136, 83708.0118414001, 77040.18295609158, 6533.108973318136, 29765.974056512277, 12437.462278247458, 94602.51248313487, 1469.9422866222278, 90774.29329176848, 55217.678402443074, 64301.872859866984, 10783.645711582201, 25258.05250753229, 18319.437881088008, 26278.357996628376, 47541.96832061441, 8348.9044764539, 60075.27081960534, 39698.57913539449, 82750.57828894728, 98446.46276997682, 64679.55662693972, 7549.08081425163, 9598.268505435304, 8631.869587248242, 81939.543136273, 65658.39646201636, 26331.25257995661, 83516.34176063932, 26627.470560322865, 38025.64779704859, 56638.840001352575, 59801.290442220765, 22543.46877725757, 70550.05510672226, 10722.81041010481, 67958.00231128208, 25450.679693699964, 18515.89352687235, 21543.267873684057, 94153.20429618876, 78142.58933595779]} +{"id": 849, "vector": [31739.784165997655, 70046.26842968729, 37092.339331884075, 63782.958416754576, 88893.51916631618, 8510.116903005937, 64368.2519764, 90260.343506506, 74213.99655054628, 51920.69588045795, 65495.975417165864, 90410.23212636658, 97286.09982555707, 12434.670127077985, 28605.077551214854, 45369.70216772177, 94360.04035588352, 22044.789313785783, 74550.6473877809, 44417.837646804415, 79945.39268483521, 43480.26254192896, 27997.052656969034, 78331.20218851992, 21498.445488089877, 2973.9201740292096, 91795.84065860574, 9767.771382471203, 53033.71430191176, 25236.717394225507, 28731.012284149492, 70203.50353096276, 25449.918693846463, 9198.320672003434, 79058.55728367684, 75122.30045631199, 44814.38100243552, 99408.93269587643, 41167.03225705619, 34465.58565887389, 81233.97536855964, 9768.24151660607, 64735.34635144696, 39200.32738540399, 38639.054360569666, 64633.59742644729, 37546.062847650755, 50880.59507472115, 66376.39747905932, 63558.546514671674, 48778.16813518386, 14537.438803166946, 95939.75004629561, 25559.784678743148, 11524.110103382323, 34446.06520699014, 67262.10369245344, 74486.24182853856, 65838.35012956511, 46367.97409005372, 5559.5171613020075, 30997.378553324874, 46122.43473422806, 1957.8013882806667, 56868.337321339124, 14673.953264503214, 71316.2646861371, 5613.0363817817215, 70768.3545655938, 94570.27732108385, 33888.802855518865, 31919.337798363966, 20375.106963166854, 64980.5181585683, 33019.56424419464, 30643.310510933687, 52737.896581326284, 42316.315754568466, 12107.5633344783, 27366.229440848954, 65294.01248013581, 70028.8888385707, 61445.30498804518, 72323.82679624761, 90875.91051229142, 16550.937552794665, 70531.70588581494, 86602.51936756841, 63976.97083631081, 61558.7766813848, 29752.36210523323, 55717.81435800065, 60139.49166137955, 35751.60822025155, 74442.84935796126, 45722.67183455777, 85753.04957049307, 62474.57352187361, 45598.15866160365, 68050.79876841263, 61316.85825009847, 35144.60364997931, 49061.13993576505, 53177.78754328944, 19184.532056495762, 93398.6705232954, 13777.431960198428, 53591.448716712934, 93820.847450221, 76869.8094260901, 30783.585509407152, 65373.02617039566, 49197.654064120725, 21125.381980799473, 59901.84965515427, 78318.83022112826, 30946.897110235237, 86235.37595996907, 58133.79066099731, 6921.231544859185, 28368.591470139436, 46271.20123622887, 37799.908669129254, 15125.313934071515, 79859.77993148568, 8155.788544173936, 90474.64753300838, 15664.55888259035]} +{"id": 695, "vector": [63559.54411803779, 28319.23740278177, 55951.83007653149, 73627.84150780948, 83947.41330671556, 68871.19958458582, 91934.98769271474, 75839.19945494183, 70584.21549663943, 53325.760462697035, 14177.044312749022, 4087.0814404179278, 72105.69288781256, 23617.396589731597, 19101.889208069035, 88224.11641669188, 32460.712189964204, 22027.74269707479, 28649.99863411425, 38923.21054950064, 36698.7297281881, 12147.250108870656, 55269.48103827968, 35062.70521142697, 87527.54876848341, 70581.06529058206, 57231.006785215824, 99771.81592578995, 93873.5686444838, 2109.587735151619, 34598.928238749446, 98558.2753939446, 64219.70914018176, 51388.111769160314, 81958.20614376923, 9176.703193117486, 83251.07962509849, 45996.611741212866, 74681.00585365061, 95261.97300017452, 11104.265258204305, 58559.24275102869, 85033.69165912051, 81240.90226635907, 22175.107623545966, 41457.20996567521, 76228.26223860645, 8960.175662478054, 90159.67953993833, 13263.59115399719, 40596.45307651513, 82827.84704665373, 13384.452819026194, 92801.97033587273, 20086.83643463609, 34544.72659818693, 35399.6613751776, 91471.0009904182, 29569.99526013191, 68919.44787826955, 39875.57588210747, 44255.39642933998, 73316.59539810535, 30100.27858211081, 43404.072794764834, 57788.274866891465, 19939.212968782736, 21351.46605699907, 29022.491259088212, 12860.205459797548, 73509.37288630137, 54690.7218625774, 25963.640753786844, 79680.9869991867, 49335.498226841366, 60080.325143410126, 56068.99048918592, 35547.12259623048, 69816.34056801695, 71094.27092911539, 11754.615002631486, 38252.78068874732, 42406.89257908635, 10049.411330745894, 23221.287483319218, 66792.06125056422, 95906.52150528847, 7012.587813994164, 81520.06420490531, 39084.647582829166, 82092.17736994327, 8922.763670376788, 40422.30108363219, 96319.31181741797, 12549.629304589493, 69548.35477281349, 99836.8591590351, 1032.9733230468373, 34956.779431399256, 39072.53429021159, 96611.142705446, 43552.88893488695, 58340.880181577995, 20497.379149174354, 95422.89302637914, 57868.16001182766, 47133.73812034163, 82301.32716684192, 17278.508967618734, 28779.97761659482, 67000.67569647485, 19753.72380730731, 23498.86000462328, 59874.491142690524, 62140.21920333717, 7598.427468654645, 78865.49325795987, 89956.11291410391, 32208.927941284826, 68073.78400330793, 45811.857411224686, 99832.87533322266, 98508.92372180325, 79551.98747175795, 8981.889912099872, 20028.06669844579, 42729.77615130261, 86432.1484343161]} +{"id": 816, "vector": [58689.38089635234, 81480.41343308901, 36188.01498731701, 96420.12388263678, 45746.952298098986, 44516.97675285395, 49121.77637898445, 38077.212866812435, 77765.25627108008, 89110.22956136311, 48468.092274589915, 16694.900662993183, 47691.39105051129, 84921.48710303966, 16251.810363291286, 62267.20501056926, 93640.58826187876, 8659.984535453435, 60240.89894037814, 68689.51210481129, 52238.143154681726, 12287.505178686853, 2620.6253086564548, 70373.47920636559, 91625.80529236721, 56280.217125865885, 80669.64819570765, 55362.77999989787, 65535.35753338993, 44342.55919151596, 91106.57604721485, 95479.97823694939, 40012.43334026912, 40617.59902926866, 34891.60401177112, 87733.6513049642, 37384.542481813696, 83791.10830294597, 81129.28808579536, 3426.3391648792553, 15343.568890185777, 81987.08057460445, 50808.016497812445, 46364.86199183016, 70063.89022889874, 43416.10040629723, 354.8660672895276, 194.3978512962241, 78873.46575749117, 7189.564630134349, 16547.82175327241, 91228.35349318104, 30309.27405005701, 97382.36036505332, 37000.11192939111, 63606.970061486514, 6746.793791697792, 92097.95362296469, 16915.945793729792, 45889.47667669384, 50777.829670006126, 81028.29741289794, 27843.368417463433, 94968.70835097437, 56098.070057679084, 5112.1523338664265, 62583.79089198481, 87081.1130337213, 57286.668828470276, 61598.53125420164, 99162.81387296368, 60024.96044052115, 77561.66789389621, 48866.958530038064, 29543.466705552102, 62953.13777580179, 9482.246051268396, 55460.31802919174, 26896.458329979345, 33146.705593056744, 43095.57822579545, 26215.80539520668, 6453.42870536284, 10313.178273267964, 71688.44051382276, 35994.450328029285, 87017.75685319783, 21187.355505453466, 3708.240802590035, 99388.0041434372, 32886.38352164185, 50422.23446507456, 96051.05772945461, 45868.615967485224, 83696.58996408677, 29146.51223160426, 98895.24720511143, 45807.13737449518, 24084.08188011385, 3824.7626257957636, 80095.52963358331, 94604.40963285069, 33421.51493724628, 49062.40014353207, 43281.18658068914, 24556.524318498618, 78083.34742549139, 4885.438283830479, 72904.97651958464, 73650.08000212874, 10615.226230767772, 4556.520117663743, 81744.95372306241, 97991.4938190012, 50581.29359262257, 79542.62472080423, 20297.879733309786, 53050.07576473072, 32273.75619711057, 84304.1987913632, 30700.663259512974, 39453.51332864919, 91074.82198294235, 50903.5333758389, 90100.878681568, 46832.454901999045, 17182.271881793808, 91853.91878263155]} +{"id": 176, "vector": [19556.531812603505, 71869.61722471463, 8769.13479425816, 23344.52245687706, 72503.550911123, 9561.413726067203, 22841.74817511693, 46080.84944628846, 31682.43363697565, 81932.80420297639, 76160.62344091266, 18036.134417762318, 72843.73480770177, 20083.949945592096, 70706.5244046653, 81364.17415274096, 77194.36508144574, 50336.95372071302, 41079.544776209965, 36657.00011842931, 93268.5287916818, 2596.3754619597103, 71973.0637776959, 85020.47268566072, 59130.05045254422, 87038.37898760929, 96194.17765999466, 56617.97278045484, 79174.13103017151, 8633.602800357843, 51464.165610430166, 93381.51667981096, 51139.998438439805, 5838.195286510761, 38590.639463933854, 10937.310201768336, 4444.250956295104, 65625.56342678168, 13175.405462404233, 37357.547687813305, 35043.11543072494, 55874.923384667105, 75594.34155530971, 39284.051506859105, 23070.71460246898, 96622.3532212585, 17368.508454605635, 14208.374421308667, 77909.63886813387, 94350.38800527553, 51371.41793731379, 58971.794419781654, 3672.4869407035567, 73418.03299147071, 85477.44971922606, 85007.61003720805, 93646.20351379835, 33176.39072516273, 72170.44005688597, 75522.43767638358, 38106.20232080506, 10714.322500048678, 3836.2483447193063, 74676.45593736145, 95112.42295835975, 25557.79452370164, 86932.1713652847, 19557.853459040976, 89330.64967719465, 49107.330023996285, 90399.23592308654, 22525.48451973656, 1815.0834336435385, 6722.286946089484, 63254.017365110936, 69219.71293017767, 95953.31691826416, 57234.30801300482, 24941.047509852233, 52186.910679690176, 1552.8781447696426, 36327.16873191717, 4963.968014078701, 30235.140714325837, 34758.02072375284, 21470.069465740704, 60572.663380820035, 65673.3422793111, 9890.791994139947, 30675.113057696668, 21652.171168595523, 90153.24209221813, 85164.64994924198, 22031.17293849729, 47769.8858852726, 98802.04618360709, 38399.11277521909, 67937.80007329477, 59110.497900491784, 9081.066425336403, 68613.79928732305, 24334.64388580977, 32124.16255569557, 91936.13412947945, 23749.115642384742, 86044.8289041737, 84615.5734586117, 8771.18677421962, 46536.04104358091, 26680.294648483516, 83057.49396630713, 91993.32732410792, 23422.768119873494, 44868.26335032268, 4035.275345663969, 83042.85096799392, 23139.987703075672, 88750.09559750439, 34455.82356134764, 60872.84830944001, 37117.745011492465, 18374.799290224197, 97728.59855584348, 48754.80359110773, 21590.751881207005, 50992.70919858868, 58279.55179144425, 88255.77559978151]} +{"id": 1482, "vector": [14263.345537489102, 99557.42676522194, 65018.4506051766, 74658.05786029618, 75167.17078984548, 78910.28556353015, 57378.75840278866, 71673.83908676723, 53256.4127168056, 45606.62782117397, 84137.6585337798, 43137.38103081055, 66159.73722661796, 48947.791710719466, 6631.545489695856, 55474.43074985463, 72036.51465201579, 82439.30290154502, 67408.60433743811, 45999.84318755583, 57844.85836796589, 11094.31094087, 3955.6936749298943, 47739.52560324396, 20984.9730803391, 25934.69922296373, 55893.05679894031, 80608.78992334454, 41677.916163280424, 2891.5258172834356, 32965.104579113555, 55382.64578857309, 50037.22021501934, 77585.77977457935, 4092.2214788507104, 15984.801116581804, 8772.45297364262, 74168.79557048582, 9056.508348371473, 9839.81769893537, 54486.762910584905, 68016.84908919795, 64482.1690830165, 48422.608454391535, 78311.10600228047, 8760.0678018382, 5311.560850506947, 73673.51078905699, 20528.822622497857, 45668.95128873768, 65204.20266672368, 49247.29242852678, 17118.43061392192, 49888.96070326152, 62935.85518528241, 10580.955912652978, 11359.200811548486, 17934.203759459644, 63626.193506208605, 77232.5857462588, 74093.95944891703, 12393.867406327498, 50045.72184843598, 43462.37497415588, 72510.61238846791, 24435.65972957822, 58943.61368967116, 95896.61981272965, 89922.47922880939, 697.5773562217569, 26605.599495032828, 17873.487017834253, 46309.50205284292, 32243.038453542773, 38059.741878976885, 41208.21000998415, 354.11166019684305, 31443.681792132284, 61886.780317538636, 0.42571568805716, 59766.82163642717, 50674.790946809066, 5413.463274611297, 61421.55807767401, 14393.3671812562, 79795.55707744158, 13990.072294998945, 50393.63077843117, 8303.925165109038, 28979.901248121652, 70655.8937719269, 6496.271832880385, 83372.45132425742, 7553.198540770812, 10325.484155310694, 40792.26321901545, 55115.77137324044, 76479.23941556738, 37721.1850624486, 43651.633013633495, 7990.211100788569, 83390.4759786654, 11208.399009523728, 96825.19613886758, 8858.76501793832, 61291.26761687326, 2469.7018619899814, 82868.0847645082, 51473.79082449787, 75315.36677713796, 13697.104047539688, 59329.49092713158, 33040.045255164296, 96752.23838413015, 2089.097234514403, 64811.005901518234, 89480.39786116726, 25016.225626263287, 17627.45883928026, 7583.441628278031, 55556.970394885306, 30160.856036129037, 91990.77234093212, 25888.145630655825, 5796.467251157012, 42168.187621177596, 95418.14438239526, 68293.63185753684]} +{"id": 1946, "vector": [57438.254099545484, 68491.30288224574, 38680.80146958608, 72739.08539076727, 26511.097362249737, 47439.36860292017, 81709.43003964453, 16251.182094019712, 52916.59487544593, 78756.4932514739, 47532.937402311814, 6448.858608194319, 12546.287678526813, 39810.478255381, 66396.52564203998, 52382.051798344895, 37127.74049125096, 72589.37202395151, 7693.9294660218875, 14605.47694599773, 38989.90557108457, 21811.604689338194, 86825.33112350099, 22710.082152631992, 88195.08208627002, 42876.213607815494, 74086.1079450859, 60649.32497049582, 90614.05917714215, 796.8606174062587, 16701.507643818768, 8385.891126601364, 66545.45829765119, 45929.79082814842, 84563.66299712715, 97556.55921553509, 55320.11112399986, 42056.10268408688, 59722.49598233614, 96325.85727129054, 11241.093440384897, 28827.323357631252, 45908.70153383128, 15394.27773014196, 14634.050424360368, 46521.97892512599, 30442.6948335089, 87204.56781797542, 49477.074300393266, 16667.43209915067, 29560.7134241044, 46446.65979468279, 51109.47610586832, 49441.43134757875, 37635.164750671815, 53021.983997005904, 23316.491522532113, 27755.32031184277, 58997.251558110474, 28601.1424267972, 78555.04388584802, 65055.08325817184, 43196.29381153676, 30403.363822455907, 21311.651750791872, 32302.51879112257, 99948.79434829947, 17012.300788379696, 86618.21146755839, 22069.508843759533, 29263.980835139646, 30935.978190254078, 58917.2414687246, 63019.54223547072, 64570.69487206473, 27128.150595221057, 86778.79510591377, 17091.210594990047, 26817.31902060149, 73468.76981968264, 11273.905192767652, 5031.8078596987225, 76765.46437943145, 89618.77691320947, 64637.64824601669, 96381.90438332119, 40218.243221975405, 97817.33217062266, 25053.789259754412, 4693.9092453635585, 37853.93629969092, 15244.238274639587, 43722.17670564402, 47852.82767823674, 79000.81859442807, 6436.15540505802, 82601.88738096897, 69230.45343892563, 78606.46420229648, 45716.09351838177, 54574.03938580454, 99865.80363551129, 37196.276589932706, 42522.483920830025, 29093.350284432716, 22180.782449211245, 29958.67593060758, 67652.48084771834, 78791.87545592015, 16799.679858121, 87385.6345440914, 14094.623300287512, 90618.61542749568, 28390.817142561587, 58314.264979577376, 95695.38641931242, 30535.93107339503, 70547.6264864617, 50895.629835966836, 52202.91512934513, 63225.230461132785, 95314.83964664344, 33566.18157458705, 44499.58098666018, 38264.742488611904, 72728.15809155426, 44188.877383320425, 91195.97224080226]} +{"id": 605, "vector": [1538.4069317389115, 9094.411908059974, 66061.31077719259, 38911.70792051603, 8586.556204713359, 3605.8816420550643, 55769.856132230685, 41454.0970217934, 8001.27460511435, 51243.70194283836, 51251.84315757314, 84669.63728949937, 27982.11622951653, 48890.753420033914, 45830.361256693795, 19897.537523693067, 4909.239872079196, 42966.34733786908, 26660.584663936083, 14844.20727283302, 54886.898551356244, 39878.37988606371, 40776.87012422594, 25951.347766281364, 64570.30012753827, 78615.26143563741, 59340.63445092052, 1551.3075757909721, 11361.707914945062, 17038.383525391422, 77382.59384024839, 72732.10596715228, 86099.78799273702, 81651.96615799695, 48799.70415501762, 16007.00665633883, 61115.61469135382, 96972.67663571713, 25034.562266464432, 26447.38548565031, 35028.81546699368, 89494.21800855488, 14614.263229300617, 95404.71312659852, 7704.242786076321, 38318.444226299485, 5120.640424765055, 95977.9953008135, 50389.867008469555, 28304.218161534678, 95299.2792700406, 42005.892927323206, 55983.23265776724, 16263.673831197568, 97397.20724991428, 72121.82302562459, 57313.83138672294, 19584.50251078293, 13298.155662648482, 59664.90894778654, 96296.9791688413, 28236.187307193573, 66419.3749555413, 3359.1199943257543, 80693.71008464872, 95175.08008979307, 78060.70937939745, 61154.38557530725, 33347.676448270766, 24160.613988894842, 69344.03915554192, 16367.511075873754, 55743.296702491505, 46741.406615954584, 93425.67607220479, 8490.583549486608, 53405.61950650375, 36957.16668816334, 82462.91075063366, 93574.23579513261, 55154.8155962622, 64777.2889663005, 48410.36446802727, 77115.99140607653, 56064.05290822064, 18231.718391291517, 81576.59932168634, 96359.39289302035, 41538.52926079943, 88596.39090915739, 51226.24559295016, 79453.14835711876, 14767.002537824392, 95111.36668290284, 67586.55794236419, 34867.13486576556, 5167.0616554728485, 12843.032398338728, 4090.8541668661537, 74000.11804014162, 61103.122438721824, 26910.466694404888, 9269.391228635037, 23564.228390097098, 47378.63401621504, 10993.701527255294, 5517.319494359518, 70393.23990936206, 78971.34677825742, 79833.3177598376, 10444.382950612995, 46039.4817978288, 18593.929463543303, 79370.7624975767, 94687.49755536376, 31449.716486415757, 77001.28260862568, 84138.04836217758, 71136.39917940358, 46930.221601255296, 74441.93613824296, 14606.422778094307, 53504.079032320806, 4695.847543315268, 36291.90956671968, 35346.67629167049, 36687.88641603068, 68403.38893584635]} +{"id": 44, "vector": [23274.5057990146, 87132.50526661437, 27166.444237827658, 23905.1679823704, 25279.946478407, 33166.96607139564, 63316.763852436954, 95490.5078466962, 775.1691603636646, 76057.52404780687, 32596.009418305093, 68685.60625584229, 46151.42555002849, 41177.068229462835, 96176.15176825061, 61082.22075264962, 5459.544654076331, 97143.53326103499, 79415.01211948443, 40736.060221111846, 86010.50609193523, 49676.95653966846, 96297.04030668348, 16062.724079295022, 93773.73950314087, 92482.09530244146, 94014.04910065004, 83538.09814870772, 91124.91231120793, 91098.97439585862, 68949.48474762934, 33166.60505899656, 75305.3420389082, 5965.706702706364, 98212.61420740302, 38495.021675773234, 61124.57801907786, 41889.30479130797, 49048.12206980072, 64554.44950744682, 75089.31697442895, 7570.563331287028, 51653.81256275653, 77633.10627801844, 67891.03694857677, 27205.185610356064, 28356.946899943327, 21713.120764298143, 33826.73398061443, 90474.92207655634, 31552.65940042551, 1826.4396446229014, 75583.33069727181, 62317.057230760984, 58859.235898718056, 64726.78025301617, 29131.21281766864, 95302.41582482976, 504.77035708890173, 25893.84658831979, 83438.94168321883, 9608.211681948864, 88446.18319529429, 36461.29591963587, 4562.283663924438, 10057.20348341178, 77565.60062970997, 58170.706787535266, 75659.5883408127, 24353.58756913605, 15271.359750398138, 10823.583499959645, 99985.45588744756, 88251.11801681778, 54135.97511235432, 58285.77556180391, 83469.0379917236, 46057.447325407455, 68809.50080277496, 86482.14385192307, 20322.90202126067, 99767.53218789469, 97567.09367786112, 2282.93817740427, 80151.73503751191, 88382.89452111976, 54224.787644318465, 99688.38909233069, 80107.2924658016, 23269.479314331333, 89512.7033654562, 84851.12750162235, 94312.45280914386, 27256.47049510057, 25295.203426336888, 34352.42598970843, 54276.362498721195, 9437.558468759988, 62907.371283687644, 16302.726861504436, 75016.5304270122, 54534.185172673446, 65618.85941819011, 93526.48174560427, 19139.479710710984, 78001.29313151038, 5515.373021583625, 75447.77497010445, 73064.84387427884, 65488.997205864085, 56349.36385810802, 5340.869109741841, 74361.36082399856, 37274.30315084912, 87303.77189577307, 18217.83639159521, 68645.25473625502, 79888.16571220035, 75810.2789560297, 27044.416692436112, 65420.556593619884, 49274.05687309445, 14283.295002891038, 81392.91280966082, 47556.20219056271, 29026.09008885637, 23020.59141936731, 50756.407185741184]} +{"id": 625, "vector": [28953.490538220718, 93583.82024112085, 59090.96049698307, 20957.029007159323, 29180.75604174928, 23028.68866243687, 69359.9663054781, 94498.65130120322, 23200.924113452882, 68906.37280344211, 64792.58281508967, 68781.89346771674, 74816.35700052882, 45567.625995228715, 29432.955379252824, 36296.1232241662, 81345.39970672106, 90358.90386998173, 29553.800548511143, 19234.125458173356, 56774.658199166006, 61077.35482662444, 99423.0727827111, 4335.2153668288265, 62170.77391419401, 29230.057869046555, 90382.8135614563, 63499.960795660794, 4496.241182304328, 52766.8836065033, 36148.43982088872, 76563.81190951835, 62559.39539120681, 37181.501364870375, 19399.769250022793, 97699.14417419751, 5129.659392130248, 78127.64684412858, 41359.63934896096, 16109.55592814961, 7480.958194417098, 27036.396491502557, 3583.2726441970995, 60759.34138192531, 91430.97771216344, 8279.603341676955, 45176.02563795354, 31637.933386845052, 7782.038588176765, 97478.5785381264, 86414.34913285304, 75471.75338680482, 14711.685201616221, 76484.3881999693, 79155.99547888101, 1989.6329378782052, 67110.8353089569, 43741.4751808724, 75437.78264155849, 92629.1355548861, 33024.47201365951, 37355.4249859148, 74816.35778267449, 89700.6682094903, 90628.7562872356, 4492.491708438262, 70489.24550096822, 48187.81460926185, 51256.33997973367, 85632.3016958397, 75339.57223393561, 32354.618818195315, 53096.68429226014, 38789.22756505291, 4655.398522384946, 15215.195504975509, 43238.095071951364, 90366.93677051301, 62950.21217122227, 4868.875909286608, 43931.33963758082, 64425.424340771555, 46080.72742222833, 3536.9891309812165, 23653.308677876306, 72551.78101876364, 81927.90808384957, 38585.15393692644, 68017.41406967632, 36915.14923667184, 35991.88563186034, 4065.339739948426, 148.71747273424995, 97151.99208889503, 80893.96321464307, 39177.89165960539, 46090.63700140932, 99708.9613061467, 29914.578632580004, 34736.22026682047, 97635.11470305263, 84907.9766840878, 40705.60814988764, 20876.532052998897, 89094.03227993158, 68908.06390616857, 52212.16372720118, 36604.884860400765, 84477.90685469861, 64333.29053288824, 6334.211536073398, 43839.12212526613, 67230.31486611863, 31592.93030594519, 99484.47204660712, 42380.68302930929, 59672.29003211747, 66517.74516793394, 62755.47315914808, 25091.421386349433, 75196.26012900285, 36799.25320899573, 30735.75405923683, 58182.675614694155, 48722.829500210406, 61073.68734684875, 91413.06659515342, 59651.090367846075]} +{"id": 615, "vector": [21703.63937823978, 49041.571389591365, 79068.20674532634, 89557.66883542134, 79983.47064463471, 13076.647833444566, 84011.91940612241, 28052.298615893313, 36503.09947406852, 53155.16893524865, 38537.82647865006, 37291.942493337796, 20313.331696868743, 60807.95026470349, 51444.728924112496, 15181.16107708305, 58850.43104939315, 80899.93521596478, 38890.61105916955, 70605.16860875669, 3698.7970498630784, 54530.26044407759, 14671.454515994375, 11346.675598081913, 75898.73275327666, 51639.09651395708, 41793.41674619337, 9236.643385706966, 21937.919354913305, 20727.14869466058, 38133.19193076919, 35427.94292211555, 70923.64483709841, 58934.09336675484, 64292.59230659331, 23960.903086815146, 62822.2516945404, 26181.65315944797, 89507.40270344149, 92529.44840289498, 63990.997810264875, 90303.58882780051, 32970.04735984474, 67589.0352825028, 41502.29866006065, 92993.43412425049, 61715.08355697963, 75474.77165910644, 61386.17336184923, 64804.655086542654, 62248.65874915998, 29067.207864222022, 40174.96352159016, 84362.50965246033, 67174.8378514848, 42631.96322629999, 49350.86345445666, 60368.24658883062, 95942.38662956517, 4394.126553690925, 70804.20840557554, 68.56934446695107, 90907.38020806859, 19086.639227059855, 27953.783375618324, 86521.72942460295, 91222.4433070072, 51161.46252979666, 19090.04466567028, 65063.8793298046, 2407.962002102004, 70085.56464867956, 87151.757810889, 75498.05255202102, 13424.556296841738, 82109.03833698809, 87806.67383778046, 7004.032824039464, 27718.628896813247, 1800.192220612573, 57438.68626018883, 49043.75467269267, 72682.83573022956, 58760.01438231693, 1881.0207990246909, 51670.638374289476, 41920.28409328007, 78785.55818227286, 46137.29529476872, 6432.405242777161, 61401.30774187287, 17544.688117597263, 39100.37700765732, 48434.247682616915, 61630.59439277996, 55395.69214362569, 7861.931673763589, 87318.74088071969, 96025.21733419863, 90831.74663363329, 93549.56071621674, 25274.235586405746, 86949.83848426932, 19212.43683516848, 65052.99233947335, 36933.518154127196, 24605.189397362294, 29791.33166271415, 35309.01819473847, 37441.82086718189, 21247.44470978377, 60503.43777077247, 44041.29126039321, 17644.94853876103, 80406.52545124886, 53526.04605829276, 83702.67652627414, 9981.066862000953, 69446.44084608016, 44868.68778145402, 12488.865759735434, 32488.774194728554, 34299.81510215148, 57732.368242600816, 97430.83905304407, 33434.70937805286, 50489.10686436038, 93940.0229757782]} +{"id": 1489, "vector": [83157.09234472027, 42583.01305662951, 62823.404821678174, 15162.375468013877, 45677.28277982111, 69740.00607790149, 24768.744923598217, 59229.68374181444, 78897.81816614242, 52742.0268788324, 93396.3707860349, 63758.42404623177, 77912.80967311039, 11462.774567701328, 12930.25753700695, 78336.97855967784, 35426.189475710125, 99509.54498601495, 39453.64365053964, 58016.63655864526, 37487.56224791521, 53458.782600790044, 46967.326588859, 53312.97798396929, 32477.06494931777, 47604.58943060425, 26339.73753114658, 19601.561120609167, 69364.09007257599, 71978.86982592096, 66036.85283273461, 98106.07533932778, 47304.16377899034, 66330.75506464441, 46944.14741935524, 69410.66285938233, 41159.45032033022, 27221.470317439456, 60084.059589726145, 86123.54193950782, 63391.97336892744, 61693.25030946841, 57415.92779039011, 55461.68325039261, 35471.59654877847, 61028.331318451346, 75145.04000659643, 86237.80823670281, 96772.41207691738, 46524.407468523146, 14434.114845255075, 12376.522536585366, 29448.567401971948, 23601.7937276904, 76296.50553362093, 59024.50818340538, 30389.67102755833, 55321.52493667142, 39030.98101175307, 48814.829995749555, 73089.4249544688, 27277.140244092836, 96354.60065649891, 90588.1878682443, 21652.774703925483, 40158.540116975906, 3222.9133599221304, 18339.647628855404, 86667.53871839821, 93271.49248229996, 38627.04615749919, 41013.0358443867, 10282.077965706616, 29217.034163221644, 78447.36858835655, 93822.57289654993, 85961.78468601547, 19252.428016624588, 99850.74445675702, 29259.865030054778, 83664.8358815003, 20504.845353673794, 49256.1143343097, 97831.29534844753, 89840.05100545078, 47724.907982197816, 35805.88798973479, 34640.68898718868, 49990.754695011, 63982.063705730274, 12242.21106717821, 41500.75560629498, 62841.097325747905, 2197.18658352821, 14818.536944257288, 39722.53092373484, 7378.264317178585, 15575.223953672568, 36674.45573949905, 3698.9392560844326, 70606.34608181166, 77955.00669486055, 92477.02166450348, 54038.08790824995, 8210.86744786732, 7159.9691488067665, 95370.00907274862, 23207.85472711825, 29113.67026817182, 76073.42544480905, 25530.557156880073, 72793.03332575226, 13130.946314840574, 5305.195872503587, 3980.553418799071, 92719.09494635042, 84755.41634307083, 54383.765858345025, 44141.717838629425, 96252.45099867083, 18807.26656820143, 93247.50725455463, 20731.972037655276, 24656.03011951907, 73381.76445382829, 99574.0370005198, 89089.09175188573, 25389.037314068242]} +{"id": 888, "vector": [92553.41170474564, 80780.97012923531, 52635.872978525025, 77337.15134716523, 84666.34366992519, 38099.73752121288, 4426.234709367472, 70214.85235926982, 939.470637292128, 61909.62673645709, 81202.51857721905, 94769.56813920102, 4746.218542508041, 41095.03810136159, 44114.23945991005, 43684.224657925195, 78515.09497824231, 28192.000241345184, 45858.41659638254, 86328.50489081981, 25970.5335322859, 69299.65099809073, 52369.67713160433, 14191.834408079229, 57431.24368544251, 76312.3973162123, 56550.92282196359, 45858.90394753378, 5175.1090553363465, 48606.49184227734, 9129.5893696969, 90854.01879984887, 86864.96258305639, 39534.71916282508, 44738.33701256181, 93113.6796875539, 53974.29400469016, 72837.88064885167, 70085.9849557367, 89103.5550080176, 78610.1182021874, 64883.77328497972, 49245.59715131314, 81920.33897287458, 68553.29302730452, 33021.709267847975, 70843.4572561597, 65198.00946908461, 55275.185545642555, 51357.93704355249, 54809.44864712154, 19445.826463685466, 3500.6984955215926, 66378.25511375866, 14042.191756482458, 58963.62813062096, 77718.89164021183, 51547.043408915684, 53350.41789694741, 53521.838963840164, 43778.01266443709, 14697.084846501495, 4826.67275998565, 38667.84599850191, 36835.982926970355, 60962.901115434484, 69297.32334304125, 80691.25343927043, 58503.6465886287, 74225.72136261649, 78753.87892542213, 95400.3862293972, 88205.17243003647, 50112.25680188821, 13966.532942283882, 30575.88721750748, 20428.130519702838, 49074.13995440756, 91409.69320222437, 48118.61952651037, 33392.53001036926, 90896.5935440291, 73472.84148443164, 59217.763179030735, 14098.453429910263, 93679.57736338103, 30449.52625596584, 88714.05418121684, 83591.03934014129, 45999.41159965042, 11766.20575059113, 27206.58381494101, 95509.49535891533, 83608.18107882213, 42096.43126339113, 57929.485762557386, 96616.25741047927, 14833.008550025239, 80136.30845601835, 60533.44549920886, 88800.01286086792, 71395.65182963187, 31940.4847559993, 91428.20051537508, 21805.868916003667, 17008.441462544877, 50639.11979942963, 7197.727686785993, 2448.4039732669994, 46997.73589036173, 85102.9199737266, 15652.543764220094, 14419.90422524816, 93724.74640220463, 46729.390781597314, 83800.16552016781, 117.00216959092957, 2624.6448345979557, 80477.34128323966, 22350.317680507702, 96687.63586326377, 76381.2445272786, 15458.62687313765, 9788.149728915641, 86280.01193532263, 20211.97368505827, 98436.74466923997, 69364.61931429661]} +{"id": 1854, "vector": [47654.509662967735, 23022.22299388803, 18248.52605163051, 94397.872440151, 95425.09109042244, 82197.88537697347, 58077.04699799926, 65639.53713910774, 73352.3102839626, 31024.58304984298, 98706.62321555817, 96036.43271018354, 28307.43302495036, 63336.10284165071, 52543.737425868996, 18887.612097645135, 42079.6022813522, 91484.78401388653, 6565.311725771983, 12262.029875416858, 99164.87353078721, 78602.06121894626, 49451.457835579175, 50567.94578099163, 24670.607455839177, 13660.032491612306, 50384.074672282775, 32632.682758437004, 29579.044231856456, 23827.813119086848, 50280.52443678439, 92116.9850514324, 57101.939184840754, 90137.08763787612, 20471.72605335541, 89330.23014853048, 89035.73827183191, 13160.41196836929, 61477.089589919444, 18505.14367947683, 73222.99023965721, 10866.02774699993, 14559.045155469486, 82588.9575992581, 56439.942237363575, 37530.11445870469, 86666.53877296967, 71233.82846500255, 66400.14369425013, 62486.964528725366, 49047.651552371775, 61512.388112971435, 45663.885624956005, 38854.9472580675, 99895.8190513929, 89229.3500249158, 96586.23738179861, 51247.61423605776, 20799.70322071728, 65533.455918047046, 52680.7040314219, 8298.430692661785, 29374.47646486824, 47052.868761705126, 87298.16392565829, 59070.20194130972, 49871.98502428294, 91552.04326926985, 14467.267061986044, 85521.64341709808, 34726.77868694176, 36801.46571611256, 5937.618157123048, 1453.4437830669656, 53785.177475644006, 70882.45946068638, 5428.1672539993715, 55504.195986018814, 99861.96278818375, 59390.6247905151, 91265.9975577695, 87495.33951774234, 51995.03017376902, 17972.63056833257, 39176.75795194087, 46475.69264752422, 93717.66098060452, 16088.016897020507, 69874.29440566797, 63636.7112544297, 9989.525159550172, 68424.99079566117, 58724.73469266072, 72235.16284160572, 16146.097446322172, 76098.92007892896, 9474.447948337805, 51686.68135286438, 76724.71589261644, 37945.051036628494, 92579.85191826776, 35994.44027075489, 94674.73378646614, 86491.63769075103, 29500.31348697213, 62938.07665180739, 91925.02987238714, 25915.891239777666, 75491.5580383929, 12570.499058087204, 60404.27072174284, 38809.66938586332, 60848.54221148931, 81272.84707174885, 59976.324964686515, 58401.629938069964, 97603.22706058696, 82704.18925635103, 14562.225204248147, 28200.006724658135, 86791.19065932724, 65154.15255529755, 70078.74309256603, 52538.73263519423, 92122.39492023339, 48075.55701187353, 52500.09440042778, 63195.59281475934]} +{"id": 224, "vector": [27646.232960590412, 41716.233493501866, 70590.25193487103, 66840.20476220401, 55523.57611613875, 71279.52567890748, 65436.89983556467, 38271.97378935424, 11710.73174147672, 59721.12378277984, 67801.9591843986, 61361.35855814054, 2778.7934891563837, 29226.405310255843, 62230.58547905804, 39689.7865931519, 53359.12633405794, 95039.163297446, 21623.40732763065, 80881.54630124105, 64526.771926399604, 43012.67037436588, 76848.37645903134, 65547.54871657756, 29931.945475033517, 19036.448457549206, 44507.614953756965, 48332.915932577205, 15503.684449582523, 57071.76311954497, 98896.132891838, 93465.6648683926, 1026.9533633218698, 25980.40832113706, 24859.02258714986, 38868.359648168946, 90142.92181359041, 7834.923023806339, 76530.82464129041, 27827.437499007345, 14129.238864105775, 82386.9072383822, 64277.02402771702, 56466.09310314575, 74689.54635445449, 82254.89114460941, 72929.86844550399, 58325.83095528236, 30131.806652966843, 35573.62499921307, 12157.51384188829, 17780.61531698646, 37036.84603526419, 91973.59138232947, 24863.318338141125, 17060.218383478023, 11137.477739222668, 80596.88259629115, 84048.69921216555, 1330.167873056487, 20502.274199540538, 34639.43995351685, 29051.802686076644, 83144.26610814682, 16947.877972114722, 58999.4659384927, 42555.00355826891, 83554.44571649878, 52412.76228582483, 68041.98070987289, 62727.715186729874, 53048.04558376768, 71968.71399351608, 11076.635071399232, 89254.62824069112, 42932.78915550589, 20415.50868250551, 20961.065548833547, 89451.27046630176, 88714.60614712977, 45689.61740189387, 71457.98041154469, 75298.02706681359, 62534.11989165766, 24768.834381863137, 48839.51740533911, 61543.26369385787, 26747.216234667816, 62851.123872459, 91573.78168477089, 35575.21106769712, 10545.785577846435, 13613.973568710924, 87583.59422400098, 10057.179357394185, 56231.29901234643, 53912.8874330136, 1644.0888205096483, 29716.180762730517, 26977.50839015769, 42669.684590061406, 98588.87098857557, 41786.64868406211, 41005.73166445972, 20326.514797499673, 75933.33594153047, 13552.52477127955, 3901.581758491823, 7356.376073320603, 53181.24714483561, 16673.635551842824, 47473.02929548506, 33968.34159953981, 8838.876818697572, 1530.4662095336873, 65499.67309334426, 57567.7778618043, 43238.35343696222, 35554.16154976202, 63878.05811287335, 14766.691457708092, 37935.29389159229, 16086.0605059856, 44435.29653168378, 17349.104048986952, 62679.79825377217, 61490.80003969649, 50445.684807092926]} +{"id": 1231, "vector": [34241.415517731846, 39997.07850029305, 83639.40516207433, 68868.33442079238, 7102.092075675337, 10515.227482279975, 32232.39099129761, 98916.75877442934, 87112.37198305875, 10151.198976796317, 97838.21092041416, 82936.84963347967, 12424.511560179486, 51443.677960632805, 20088.72743102299, 37332.74859218295, 77684.0476685179, 17499.167657516202, 40286.076367949885, 96855.05657014641, 93404.28602863908, 25103.348001252067, 87075.44006444214, 79902.17916859007, 13874.678901113703, 19830.3624167963, 86926.36397040209, 19578.99242411195, 42218.495623114315, 13175.934879578066, 27315.744161963506, 40183.92204584669, 52700.55703917004, 11061.119874560622, 5702.729999198286, 26754.032452782405, 71025.16026799256, 43097.27304823533, 45163.22547273929, 20445.22158541625, 10663.958243637317, 50933.99789807421, 65527.95587281303, 28332.944392718997, 94769.07794261315, 14057.237678793177, 26715.56578076164, 71139.40155476054, 27444.836203347346, 47985.91081901489, 50634.71656235343, 45764.99211087884, 62929.46962235049, 18891.181293527203, 83068.82019413369, 31988.96473342748, 75137.91525186285, 59859.38727606641, 44177.0101063382, 59274.92919677196, 86600.43634817793, 35378.81353170397, 72024.42470874009, 14109.037055598672, 54479.304380243855, 58728.315964912705, 46024.92816739347, 68712.17763942042, 14102.06699117198, 50584.24226970816, 75706.3780670947, 40589.50998883383, 91813.83087310428, 45975.14801666569, 1974.619614284656, 13832.359756848722, 15589.553130556766, 61255.63650531776, 22335.249437588955, 87810.06741009903, 9858.943566315293, 32250.812633786718, 9099.711805843757, 2308.632870736771, 21587.285168015525, 13258.229352604822, 63464.03421817171, 78825.45637245345, 76167.87386223923, 44381.30482832442, 77849.65097981572, 17787.341071311224, 91140.89825591589, 84966.60001572224, 88276.24770471187, 15376.020639027944, 14828.351948194297, 25594.808216070265, 7582.598478025638, 80566.12094820375, 71769.39465801948, 6936.862752014905, 17490.81900734709, 73975.01000146366, 52487.730950322264, 73968.8940876153, 20692.705514541944, 91663.54565238002, 8052.524827370822, 65487.03110080646, 20099.40659676227, 49377.27181952336, 73294.32388533988, 22097.70580275312, 50649.40991775869, 47084.1135069052, 48704.201342191125, 81467.41234798623, 94225.92457691114, 86003.9559209354, 45965.30618683985, 17860.37604644385, 61335.89399620045, 72952.63893719498, 78535.90972857755, 35692.48062476724, 91254.27362032271, 75233.3616816122]} +{"id": 1951, "vector": [15349.306793379103, 37288.26426647349, 24814.59366630783, 96556.90646055253, 2208.686732511067, 81984.77046471903, 85444.58012504295, 82862.13717226862, 90075.3934736115, 49603.01651949717, 98613.75685805401, 17156.481492867028, 79401.50821175697, 8189.430752525195, 3625.9821147166417, 89819.899709377, 19495.779807285995, 54222.130631656815, 83646.09175873591, 85769.70339150258, 44644.01627656237, 78151.82044636861, 22475.468889920936, 99030.14975518921, 64863.713582438766, 8825.67206026873, 97221.68271139698, 41813.8957042491, 9956.985924279015, 68006.4756900945, 73679.24345058527, 87747.97802417718, 75351.9858979063, 93922.70952463566, 2580.830968866288, 63264.90920558327, 3291.5904427590403, 5018.490688851251, 87335.58478960849, 19107.598813100292, 40406.63462304325, 91633.9010109128, 72429.309052135, 49659.96123543959, 28981.703241242674, 20431.407658795462, 97095.36374379572, 89483.21938728745, 1167.4842650744365, 72674.68067228801, 94153.67414510784, 58814.25882984255, 80527.81288200025, 81395.22641070592, 24149.595537077683, 89532.46590115973, 10070.929699738728, 6882.482559094449, 44160.39260670257, 53348.435327908075, 36953.61938337343, 35175.02885482593, 36809.39290835434, 62882.72618796017, 41395.03696771962, 70833.79795157342, 64696.907333758514, 21949.412620533138, 83551.61236764045, 47309.69944271134, 30205.815934292103, 83735.2840559562, 20334.009213727288, 48023.27823788567, 25806.79836452243, 98403.82793032216, 88825.33765632605, 85720.70036186554, 54151.47419694457, 42943.575092483996, 95379.01509956979, 95154.54669246283, 1608.8569167660883, 46780.54951841747, 88139.975063112, 54886.33078926361, 95468.98752071579, 69328.097987403, 59946.23967826972, 52899.22815187864, 6831.6862597747495, 82979.83929613259, 22234.043330276167, 92143.87109868281, 44562.43698580271, 27542.011932178368, 94075.29239511992, 10855.271765813635, 28716.96318369189, 56499.601174622185, 1031.9260374958117, 6151.510136885163, 23361.900911139433, 32117.269755632104, 40153.85744648674, 72719.76362898477, 41373.19484523088, 30106.80190760978, 71996.25176696021, 87681.91181091171, 16265.05246957174, 12647.519688910214, 77926.76552901659, 22395.398617944316, 26187.21203925095, 22278.372477333287, 58023.37079102543, 60342.308716799394, 49839.20770121207, 66779.75721063829, 69973.56009038011, 61883.41224305799, 60935.181312789085, 2017.0545208481938, 37873.853105429036, 24789.089831128862, 21594.561771804234, 25491.17667045988]} +{"id": 1360, "vector": [43263.74824946014, 82314.27177107628, 9740.311462709062, 61704.75854100241, 38598.70542583245, 76043.49818853002, 80660.33278275642, 11868.584140333449, 39567.44349593747, 20631.651871488586, 25110.098098254908, 71023.54342123483, 68316.13012705518, 88727.8783363111, 28249.482057201858, 75819.27583979108, 90551.67691713227, 2038.0994040681944, 88107.83974548898, 97831.5858836695, 87688.7360062137, 33748.47089680234, 19949.28351819666, 68219.08664369797, 19292.20875520672, 11920.778312782799, 69896.95121700788, 13335.961829370324, 47536.14808497995, 30840.492698577295, 5361.7229070135, 31697.48587273733, 16106.32632300586, 68570.11158638024, 16931.06767626834, 70810.32927555572, 99359.15872126895, 7585.633925914092, 22629.317887300036, 90127.41061838616, 17111.870281682295, 42261.46404228254, 70075.33119752357, 66241.86705848844, 67896.68967000907, 82572.12792378156, 96854.57078824798, 81237.80125735009, 74693.83798086822, 68738.45748024437, 68585.26887823857, 31941.68101448801, 66127.71265859401, 65812.55357513417, 28424.14232913705, 59210.625466813995, 50435.19312252618, 19011.543912833495, 25813.289281004014, 14122.61179321792, 93650.95800015633, 8782.112009306686, 3313.476995385156, 93949.3111584396, 64905.25833958195, 70277.99042488873, 13507.397629742813, 92469.73394964302, 47335.03532368372, 52794.21394719073, 66173.86397070508, 73500.17686750636, 16263.186077504666, 65927.05308841866, 46884.82804590367, 72739.59615758694, 77104.25986744647, 55506.98227220619, 29063.88152927074, 85830.18351471264, 92287.78037842145, 59129.68395939352, 52523.57534510503, 94746.82656378257, 19768.15557646281, 40095.20171580866, 84989.37108179263, 47691.95148029605, 52221.790089686925, 65260.08608407567, 22227.728866080055, 56037.5596672237, 97676.12850968454, 72474.39534553193, 72254.80547887294, 21257.33825539049, 81202.8178692531, 75429.99554734363, 66561.50660725533, 22245.27379568202, 56356.921421357394, 81217.11113700816, 60800.4631326916, 82121.46542601375, 88885.56671883607, 98992.31388474666, 28848.307399952235, 86838.72144449364, 17794.456726239405, 5964.318970770121, 35679.403255888676, 60774.12858404617, 87437.6047202608, 51824.41400819045, 31130.33310923794, 63459.89892122531, 33864.914579574586, 43909.33527332086, 55038.002765250836, 45995.69000834536, 62434.12440813523, 4333.685563921142, 67506.28501983931, 37291.858022153545, 19109.4665320628, 6441.295470916686, 48611.31138802831, 60369.17845508621]} +{"id": 1688, "vector": [13422.060960569948, 48772.693642348975, 48559.811293283936, 54110.77824939059, 92687.31690397549, 14421.319680946399, 220.78201624420313, 30707.05473319627, 38361.19247422429, 85350.11346280089, 85475.36074274113, 39449.81468285997, 90331.0480050079, 97820.1464277441, 258.31751722446006, 60345.10075044599, 43412.68410079064, 66624.05139599081, 14616.405570677925, 54516.87342482156, 28030.13516862143, 75256.22996894656, 1042.5932853830088, 16291.116626575098, 94806.87570770754, 7120.050948835743, 79894.23470130592, 1211.3332934576947, 22980.0757032371, 834.2933589443157, 3383.834805602182, 99186.94146181662, 94174.41872009142, 55628.851539802235, 82220.03940653162, 18037.36127796661, 49688.580159807614, 32933.986713027094, 8493.938192237383, 70685.9841476838, 58157.63338131239, 15266.026069946214, 96067.57062992522, 86943.08666804728, 56605.801672538546, 88013.32947377989, 7884.170866866236, 68696.78768834754, 44485.885583786876, 81550.55537390726, 75159.62918067636, 36826.13973176165, 61234.5881956616, 3528.6590406454766, 42069.319437437014, 66254.67486807558, 7499.048968315614, 76585.35676954224, 97452.80539855966, 51468.82139848102, 82133.87858965322, 85098.88028359973, 88985.65099387424, 93485.50849593288, 68836.37629754981, 28422.78596791443, 56298.40752253249, 30799.34619857668, 93641.86842402209, 97314.0119664833, 69230.40144461798, 85904.70788791013, 19069.454852849678, 36771.01315490944, 7378.883053447549, 52344.42017336426, 73775.19222341427, 62200.33423833754, 93395.58667940907, 99707.35422331754, 49430.24007707593, 28921.49140525625, 26215.113645751553, 83023.20577945854, 79718.33145776484, 58034.46697156071, 52341.82361072667, 83205.01817504212, 37214.076168310916, 87731.21475623765, 15672.516160188488, 90867.01643731141, 41490.575990829304, 34596.07397333099, 99234.5106212991, 21462.881818262114, 54667.292199940566, 1528.138880145813, 22635.200179952553, 67082.41385122077, 98901.3597542, 93800.85684394605, 3854.364936849497, 91858.54003281229, 40170.51789431379, 92204.58842171644, 33530.04780383741, 22429.124744716933, 44440.92549373473, 45962.52375737406, 36987.57088161832, 73199.31099245267, 37373.77933870245, 54857.596212832905, 26977.29512246264, 4764.379500495696, 21836.87411321402, 18275.218095030086, 31059.543306851912, 79312.66837013255, 25248.13172450191, 36912.22388877462, 40923.40625439217, 41978.105588878934, 37970.92383134169, 11170.297368006244, 78372.39950831477, 4051.3213150697447]} +{"id": 490, "vector": [22924.60054033013, 55919.483945645334, 80411.56561472148, 69416.96961067342, 53261.05980777065, 63343.222119715545, 49528.70849288769, 53088.04267857024, 73461.24434227607, 25846.065121546246, 1630.8400825185142, 37250.1005627498, 52063.60776967993, 6296.604876444711, 30039.717109590212, 56259.219159282045, 35919.5661849997, 77464.24958261727, 28245.556123368988, 79592.64639879894, 46700.57107859479, 21457.1753475986, 64216.656039011745, 95080.19324351335, 97343.71099757506, 13245.381244544507, 34357.84533408497, 90845.678188954, 75942.49044552997, 68000.43261640616, 8658.799731403999, 42620.2924134865, 9972.535497634404, 12192.101967291392, 81283.46385109934, 94158.31487338853, 11918.92963491481, 94542.02635012333, 95213.82902434254, 79697.45663746723, 84401.74655371462, 63810.70984001734, 27672.666933356006, 28753.133932955865, 82597.34570501413, 69186.96100593882, 68913.32673597938, 25620.83949322834, 13044.36522723693, 11523.433049608479, 87949.61798783847, 79116.25688263995, 5660.470421485764, 76466.3891131218, 91104.7036533313, 15945.960375803925, 33985.25943868488, 78716.07528788442, 52241.32508914756, 65193.87279541015, 37478.23771915567, 34005.48286780608, 7269.627964984315, 17185.1271206399, 15492.287443681786, 63886.604508264485, 23832.616704827848, 3853.6393218002063, 73088.56340142676, 65128.12215272721, 67273.87979113472, 5245.173335570452, 18336.303704547885, 32411.77374420969, 3178.348952313459, 50581.047209220174, 99259.16956975502, 43009.74252892642, 66619.86336561009, 78.23511409124073, 77484.88672563912, 57233.23738746843, 74022.02508717669, 22143.418424087802, 27221.841436600058, 11234.836657102209, 84091.88560405586, 87650.86227733728, 41138.20977327414, 18545.427287625636, 33484.37095148766, 47292.270747073904, 57870.65313994046, 73852.04408888475, 82350.96304111528, 86110.27036302468, 65513.74471065554, 54382.870315662665, 55268.085423185374, 30726.491460721063, 62723.04651562871, 29868.10803030373, 2882.9963814028024, 83313.55389611727, 1788.2109061637675, 99646.16871853759, 8960.691198245064, 69993.19171868177, 64042.250040617, 16778.787283625883, 98939.87615514817, 24226.270751033007, 40660.50017834235, 26376.986861393925, 87411.30941474417, 96135.81243278379, 90615.76072934162, 98271.29414501628, 48597.04546654411, 82972.86717260155, 39506.83086119964, 76408.65344143461, 22637.243840852418, 4058.002522368931, 10856.094362456437, 19402.388518444935, 78063.84522664019, 34050.33750261131]} +{"id": 1090, "vector": [82902.52665061742, 98714.26662448441, 70221.87010760553, 38785.57226479541, 45891.13799881675, 26115.742679530696, 72023.2593526178, 42832.3347811703, 89335.9224968305, 41084.341730914544, 43745.2354604561, 59441.976285169396, 52415.306733578924, 28541.27318716746, 768.7450643993233, 49890.75167353565, 35753.31806185742, 92326.41283210766, 3564.394736270393, 37784.98607034717, 55883.28132740354, 97675.74527057173, 43157.235870577446, 1599.453914146276, 99728.28025106403, 82845.67129405588, 86751.76097894763, 90951.62478152681, 52194.01058580071, 95526.84865785328, 16418.154019884434, 13396.398638501107, 9212.143916640925, 63627.3454997173, 19321.75298044829, 68049.68975933675, 35396.66058513427, 37083.00529813018, 10483.423518966072, 68342.87729540419, 81265.93460185027, 39258.99739369574, 13702.584151498188, 89598.08083532356, 75092.45977378968, 91516.59905864109, 35861.68179234037, 1729.645822597936, 65908.51065449462, 83578.16283355211, 56691.22921475852, 79177.86499561135, 8764.735623767228, 55004.67324212366, 64394.33984021553, 33681.646860198874, 32051.913037129874, 8046.897430230116, 70440.99666891638, 82381.13885872981, 65891.79566847642, 76002.42917815801, 87986.37514782413, 33223.44826656408, 60732.468226481025, 87169.83975008783, 55617.09639271369, 22288.72947554872, 46942.70268953219, 13551.049017799089, 29285.964028730737, 32410.067504397466, 73247.79275267136, 24114.273477652892, 17364.528568944814, 13627.6209845263, 85861.37312570662, 94496.71475098, 3589.745694813873, 5831.013463046353, 23425.722872480314, 82193.00806075436, 25909.38135750104, 91106.52079402126, 93834.6359272805, 42193.94868309335, 31780.83885230768, 72392.87773794333, 91879.14688672339, 81348.81250374121, 16105.751398693936, 59332.6239147015, 76441.73068596412, 93596.8855083435, 9120.477289104223, 65500.158642974704, 28378.26339905445, 75280.08524167414, 68311.5707853297, 47737.82841970406, 46765.56793492179, 68889.81654775447, 29751.25129767996, 55081.179417188505, 97363.49086915297, 77767.63758659844, 14301.213096983889, 77138.13792619135, 48189.03165714689, 1755.5677315615692, 41696.24925674991, 52760.16526741192, 99134.41327268914, 96154.39101136418, 91277.50607106557, 12569.7268173106, 51681.25154241018, 41085.64679405889, 22512.493586717752, 13077.332194960478, 66708.58846643077, 59969.71075951101, 39300.24504124017, 8839.401193780228, 4147.2674717235595, 42149.05857504722, 66326.91821284365, 69623.59541502393]} +{"id": 57, "vector": [98431.06346628416, 20770.467728913554, 1579.5571828954523, 88194.79954171936, 40492.63125414995, 97103.17857029308, 8236.211481603328, 37670.077851302056, 96785.3424774316, 5255.043449018948, 84754.95373337214, 76961.28968015271, 91094.61314048406, 96550.6725832148, 58782.869001072904, 49812.57698617079, 25001.067340020054, 34450.4553393302, 78805.02187582431, 64215.45499086216, 20567.846040814187, 14115.922380720158, 81140.03073304825, 83245.35972150728, 39722.045813518, 83632.93534987983, 71935.64159867991, 25801.968368973972, 48583.31919403717, 99259.96817522604, 58631.22746741316, 76380.21377250842, 57947.3986968961, 69246.97793355885, 89163.46770692304, 67802.99759683857, 58069.89895351303, 94457.53614037062, 97385.444202566, 47641.74657269503, 84365.36483153496, 75762.54778819649, 88668.2289356374, 55787.59140740502, 13217.457511579878, 20829.988406888722, 18541.761433439817, 85078.17496129981, 38461.56412336083, 17578.116272315146, 99209.28114532575, 145.0358241770422, 74817.90627577965, 53755.829827680835, 27190.06445987222, 18857.58648667677, 16583.59239736008, 90043.66895126806, 7645.8776348785395, 55433.05958799691, 48179.75122487099, 11441.679485983013, 99618.02358952347, 78058.57300760075, 78081.8603004183, 91608.85028945634, 36103.450974793006, 78708.7741608053, 92382.58078843883, 88457.89827587032, 79175.76210893372, 90888.93618027549, 23229.017411079378, 35839.75740235501, 49691.940569393, 36420.42737949592, 94422.63931649, 85282.70146790006, 65815.40915124072, 18357.96028146511, 61206.57104135016, 11588.574109175719, 25239.630006949454, 55703.58036478914, 81857.84737430733, 90761.42668650109, 31286.16433833824, 68801.4562712901, 93083.83549795025, 74542.50998677975, 68736.86686160776, 79541.53513806808, 67430.8231936553, 80826.44734032435, 89913.89555322586, 39380.17814528521, 91572.88345338962, 7948.064316284597, 45591.57045662135, 44236.489753685724, 48757.4292961538, 58093.48993066026, 19463.173588667283, 58498.63554800285, 1158.9185297039783, 60712.00726508764, 18988.365669142215, 2503.678684889932, 69060.21042945984, 30322.506635728896, 71808.03286990826, 47073.05018469289, 78509.39537260422, 93270.37338495476, 18191.280177496526, 83315.96937812578, 42426.366589252095, 11599.39517195846, 6972.657349864053, 79393.17660953497, 41630.775366607544, 77400.9258057247, 96954.6521043, 58658.61791567463, 53254.03265600882, 14765.856279096035, 60383.58292297125, 82730.41392624742]} +{"id": 838, "vector": [57032.104126368686, 41741.3552594339, 8150.6430449792115, 61421.61471814723, 12654.317826657412, 52454.21095605343, 16993.66177505488, 34432.34472010324, 86686.02953131564, 28295.24311179844, 66993.51931598558, 74035.62092777881, 38101.12267887857, 4075.655703700165, 94821.7459454036, 14913.815928408536, 82884.32315599972, 37659.361514856595, 70113.99851916412, 82746.17488865199, 57022.21764098811, 24525.911858487936, 7797.6958359301625, 77399.77010247174, 69291.42440835721, 77304.92037619719, 39470.68492495983, 63607.113688595426, 64689.833064248014, 5105.050301515957, 82770.69170442902, 81234.79592421713, 39324.73811520049, 3687.9623122186954, 56005.700830482216, 53341.53494399515, 79275.64452987154, 2877.163199759225, 76951.30351691932, 94314.4797521233, 74018.68862985392, 33120.179747054666, 14276.623724363879, 66057.9067716301, 84335.67755813923, 2072.4539668739108, 94751.19107652044, 39179.60066985004, 33877.091958310324, 44169.70362037824, 43238.072259405235, 91568.68318349724, 98120.24518949163, 58457.43208453709, 14049.473660122057, 58999.58809426238, 31596.83843354716, 97007.77717933929, 15473.002742728282, 72209.91712177274, 80456.28171610356, 12564.91839658157, 50642.132878848344, 55504.44913416003, 33453.87058813671, 66122.77873055398, 77272.3927364218, 22942.66613277548, 70026.65543104826, 12547.636851497546, 21579.314087050207, 97223.13442970786, 43106.70956759697, 49634.703995971744, 59076.170667802966, 87154.61366750757, 5902.9885787382, 67373.49001567782, 63011.67037316046, 33665.45493375311, 4806.087137502002, 32264.57910691797, 26864.485011265006, 81284.32893932937, 75594.29162590965, 90606.15912242024, 98854.45288828608, 2486.6709627775595, 23127.91498517599, 90166.3601650825, 43074.78283321465, 79491.21999849044, 20093.763977362054, 54790.54343010752, 94737.57315848948, 11228.490145079606, 40463.96791967315, 58587.595965366854, 22450.947598314964, 28492.05730083686, 89385.32317293264, 28202.516326301287, 54657.988919375166, 4234.930116093249, 19799.296257709808, 55700.99769245122, 71208.88629816666, 45058.03508379315, 86470.07487546984, 23307.90082918246, 39668.02979453753, 34640.7772678639, 42638.725470133235, 17314.56405370796, 63063.94456025433, 1554.201748148054, 23115.469860336678, 50602.88590635035, 92087.49411917444, 65168.69402524595, 40196.332167241024, 16204.261380249174, 36942.509638847674, 82146.12039197635, 61614.53057210856, 16086.632565247317, 95856.91585223678, 16326.7867580838]} +{"id": 499, "vector": [58963.68724249844, 62745.04253770698, 51431.66986110059, 81912.27997303038, 24700.797602480252, 27640.5746424398, 13325.951780073541, 39930.264409866875, 27155.333816828253, 32908.503399008194, 52994.43474983856, 64862.4224769788, 42198.80049916731, 81877.86510808888, 5040.4188839637045, 40338.12540595062, 22907.3763471915, 18008.296812453118, 56787.82286306141, 29881.731894519293, 18448.697920373823, 31195.598551586056, 96454.34490648333, 42991.30172197732, 20635.44780607851, 51328.44259270175, 15094.784987197996, 20421.84192228208, 83528.70174884208, 21660.567116215556, 10297.676058072258, 79650.82101787496, 19660.735470599844, 27466.13168990719, 70585.52952995765, 46104.277984778164, 31421.774102441468, 72734.55492729177, 15082.172757143542, 43549.00703053256, 88082.09329721516, 17826.387734549076, 84825.28692752527, 68870.4868251032, 75409.83134763561, 33340.053252304824, 76065.69979276943, 58854.66286133886, 55704.60505919109, 70859.14148907253, 22046.944796059743, 37305.00561374812, 59199.12981228102, 47405.94402650705, 48712.85745964017, 87172.3926735423, 51482.00076188531, 62066.467700493115, 71586.18107336268, 23024.534657574357, 59050.08757320609, 29554.983952505885, 87905.29863199819, 5772.3765740095305, 66992.23261201836, 63783.587302345804, 61143.14368376834, 97610.4608933276, 68221.22959130838, 78071.53528649319, 17840.747052741935, 20218.058816503613, 60827.74203123739, 87807.26546026704, 33152.67828032431, 56601.55937793026, 31914.494824233665, 82195.80221457621, 51205.07755233794, 7324.657556692216, 95298.13321550965, 76175.44960901124, 2705.785538460925, 85325.79652961648, 94733.29179171553, 15768.648613958725, 50082.98693336539, 99910.55271784545, 50327.582338936336, 73836.04516930526, 74320.77918348006, 77341.72432016887, 21720.723665906215, 41061.40770486195, 47669.40628961841, 18867.476807473293, 81085.23052935621, 47877.779528864026, 77637.0103982306, 97377.03369121531, 53067.0622897861, 25964.749919741604, 97947.47937861326, 46849.84620648963, 68259.44592684026, 26337.834504270486, 76765.8482173023, 31748.7351325914, 56374.51428309135, 49817.10852743188, 78912.98707223922, 24959.28961921412, 97030.13820140119, 51831.09994320012, 69041.45614593635, 79927.14228693038, 99973.99630668602, 67219.7430528328, 60355.96471487309, 90196.94654897237, 1297.539241513268, 58441.26031220187, 86892.65306741172, 71016.08307700185, 84711.45339881215, 33267.50040570009, 31336.9116992775, 54541.66086484304]} +{"id": 1993, "vector": [61018.29000293258, 56453.45653136841, 26430.548957135925, 23450.453020246965, 7596.450196494464, 93133.48911287588, 95673.46641020685, 36742.34548099118, 3169.6440753399834, 23146.085404760997, 41633.12233818626, 99314.51090395887, 75530.59669169347, 4080.955772086725, 26316.0176150407, 57533.7964017816, 59249.6109700523, 56222.7028938513, 29212.631266177024, 69167.73789373582, 36473.89573215108, 57855.82081218508, 22168.21388129101, 68257.80430168418, 15557.341465670437, 31722.734255985008, 34278.476167884764, 48862.66084645139, 46077.61024474998, 48333.958681416625, 58895.14485053037, 58720.25344379641, 64027.55453264092, 90420.75572305037, 10872.146257774484, 12809.191645496487, 63996.09255468821, 68751.84143997595, 45352.78640221648, 28997.35121419912, 60793.38897416395, 72227.83958721293, 51223.96621993077, 65145.748301185704, 5252.279346215661, 9678.074916238565, 85264.16575891079, 86544.69181838808, 55528.41693527285, 57516.842540084246, 5513.059569996892, 42326.59621589693, 80103.73729658422, 46186.24756887034, 79307.41283500844, 67307.291788143, 30567.969877296564, 41696.56727054461, 24761.634696684854, 19462.9787311818, 51135.4404048002, 30400.48307289387, 55787.39095500517, 36059.47022675913, 67337.8580061166, 96758.99152402487, 98600.34856088737, 24667.671943859114, 86491.41869689427, 92723.91865147417, 92939.1806657459, 70996.60577806363, 35610.42531777333, 59226.087010180694, 69799.92027675892, 27602.589840704815, 33778.26702184444, 92231.08017792065, 62626.02867400192, 85159.40802936298, 43876.326615491824, 88674.44626202478, 82016.36316311845, 2120.409049149885, 91145.69776480467, 72021.9169823505, 88671.0441769583, 96467.11801056746, 3295.0510256189336, 61945.69950995954, 2262.39563312568, 62772.15802550342, 26641.91965765469, 32905.93767913323, 59369.9239629394, 14004.174126696045, 52335.57703816083, 57569.118864221644, 66147.14465085542, 25641.202680745977, 31285.667776447703, 333.97307041980184, 82062.66473288492, 3825.592574022185, 12445.835966386765, 83648.06053638732, 52963.40729389084, 34600.28722503645, 6233.28153176903, 47003.92768132656, 27593.735779790295, 52368.81896037007, 29664.640959654276, 69678.15386839068, 60755.52892233941, 53702.79943670086, 37398.05576830524, 30688.258994891516, 82169.43526156856, 65330.73196301089, 21945.5870041644, 4940.455171769187, 68878.32502220348, 82414.83967916126, 82769.96731350129, 71641.56499678755, 91666.1701646931, 55931.66019312691]} +{"id": 1734, "vector": [79569.34860827767, 5511.180120467385, 2852.4822702116317, 81060.91774259946, 83903.47062230793, 97701.01138471898, 34990.89019184557, 61714.94205251728, 40010.52301457367, 24961.444377968157, 50829.998220259506, 80876.66287795645, 80364.61642228092, 1965.068941276482, 55963.37641429385, 12004.711940085655, 9139.054872099729, 79491.91276541841, 58927.10080247762, 82396.65031849634, 4128.60485514992, 739.4858429318418, 82132.77539382529, 83793.11080351792, 77381.47890632639, 85841.44395332056, 20383.50101743377, 50061.88405329246, 8079.539649726519, 16048.866251829786, 67279.50474975692, 69377.07425081971, 70165.15972719184, 63585.0493223875, 66113.37351940678, 80574.14085682984, 69385.61364683665, 18632.342944218417, 93669.89969128426, 31586.42334067243, 64385.27656526937, 35492.75179060275, 4631.161414449025, 2584.625172205113, 15579.63684259207, 77059.53961337151, 82626.25021826256, 83142.29152835309, 17557.234560453526, 95339.73007716457, 48697.046928688105, 64468.43595542798, 95150.56738414335, 47644.00235309794, 21767.27034477899, 17092.76079634078, 58881.61192166896, 70659.96383913164, 48872.99668152578, 1197.6156848584108, 57.25596550198997, 725.7021677807951, 1736.9421459583978, 58213.52120462695, 99213.69746624675, 29265.959002902353, 27277.051220155834, 42539.82110342993, 21233.19227513103, 80956.66110121696, 78444.85636161098, 43810.41538939772, 78429.46475220688, 60598.899405773824, 3507.702023019732, 5892.599623484362, 63259.56984439988, 42570.38422465369, 12376.581108587192, 66804.77204960921, 25465.555422204212, 40712.55901029139, 27943.162721554625, 43055.10535345966, 26085.73748426505, 60798.860733412184, 36744.28428466586, 23511.64921924478, 99753.25931908752, 87439.92099369229, 71116.24668726281, 59806.28047674944, 20407.99345829183, 52585.51953078945, 17808.156968989795, 79942.06183390222, 64875.757114644286, 80510.22136450284, 5105.150411602255, 24352.933236967845, 27433.04522842912, 8431.065185575848, 99495.69954767945, 33318.160720942826, 95485.01718765093, 53236.66181295389, 69628.0510871114, 87559.6839939993, 3603.3398648123384, 61132.81583735697, 13129.449385369408, 40381.83619671909, 32172.972998117853, 33131.345181076664, 84128.66852426116, 11051.106386964893, 46352.46419490984, 87028.23864009015, 72844.76662798064, 51354.51243484222, 98314.035482894, 20246.37164169927, 3302.874269479572, 16891.224986883903, 23524.89264492975, 23569.598334298615, 32068.097382154716, 76997.24316077393]} +{"id": 703, "vector": [99413.48789863237, 4667.851442816584, 39507.41141931702, 82168.64493043104, 80412.12405910327, 96147.39082404255, 29380.202423848812, 79830.84032051408, 56595.359019683965, 83709.59003235806, 99976.32235644072, 54292.54616014412, 91247.40843342767, 18343.299513799826, 8165.958678176832, 32624.65978023552, 97356.95652527953, 15154.27743021055, 1864.282768225023, 37114.4456968418, 18545.823983930433, 71270.39707749656, 50095.0536398914, 85346.52297355617, 41272.31797526719, 20668.285713736823, 20894.687320815432, 69891.7132588811, 3075.9793074940035, 84411.66517709193, 61882.830545986064, 87835.75415047999, 31385.99677230497, 12836.57479561141, 24823.654187377007, 34960.32790624323, 75615.99438593896, 46767.897075029075, 91923.11679979005, 38045.1147665553, 56728.16911157762, 46463.06427015124, 6182.614836709011, 16732.746493549887, 2906.1865971160673, 84050.72122794796, 14514.84275438224, 32756.516321464234, 69870.36954396301, 47041.822517658184, 36240.512386485556, 8269.38122264751, 38069.83207394462, 86982.86848294354, 7950.368554167852, 85316.76029803579, 97434.59087464071, 9794.038181572185, 83284.32935146385, 46302.919437199205, 8078.326916010681, 87265.32488919793, 49456.370416466445, 8767.81341638032, 88957.1969853331, 10364.720386488125, 86691.70159752663, 27671.434849670884, 41613.1420557737, 36861.27222551785, 45548.04370708199, 1760.8081407343045, 4336.266169082825, 55417.4376409238, 69910.51867769846, 35701.78473252087, 97054.00809341657, 84152.06999285378, 95106.91806782247, 93876.96647843682, 4089.8179911693087, 40975.017893907796, 78090.81706115916, 98199.88602658574, 31787.481529502616, 63253.32285610248, 43148.78700926202, 22360.66811818236, 24954.308289518824, 84516.54805018926, 37584.979065823456, 99647.36059943531, 50520.56211881946, 91399.090484957, 52940.49949404966, 27941.387718927468, 72141.62221176783, 157.67281263922018, 16124.779594482497, 84322.26454164015, 72253.59470078826, 80897.13953387071, 1704.2370258778749, 77372.98275922518, 75216.47889952532, 18975.66204351674, 73083.39181089081, 24392.9952226142, 91346.58664118918, 58479.831115281, 83297.77448583227, 27166.036589242005, 84168.36230287237, 22618.30495383699, 26141.96345567601, 32469.972794035308, 86989.38039188046, 2253.236300653405, 86070.78877535151, 12670.176389169495, 66990.58673507522, 24915.464443403267, 40779.109735062644, 59644.32835055185, 22905.32145097647, 51448.056065629346, 78916.09172699694, 82430.91334686814]} +{"id": 1236, "vector": [76408.75007503247, 75427.57378862172, 2574.810575163, 87260.354639193, 9974.51825696617, 60456.52785433855, 85743.59902556232, 86672.89784485492, 66382.05917183912, 78662.07018990736, 57333.960344237676, 12333.131237108886, 7961.528035854837, 28000.56620322623, 22625.781121328404, 8386.17222966207, 7774.171160195798, 27602.22889877888, 13227.799140323848, 41307.91130605618, 8097.568969872593, 96624.16787437293, 777.5873467138039, 13580.17620715809, 8120.048515111212, 36314.988167063864, 56192.63355783592, 4059.611622511039, 3545.8914922061326, 1360.5630952048919, 71738.58452145747, 47634.50201710678, 53512.43035732499, 42280.46768817855, 53242.180666523884, 36388.464762748816, 15605.715604173687, 82434.39138634315, 24860.115738273293, 65081.454164362294, 24963.638875225057, 22923.254142254635, 28015.610880245113, 35320.679188257716, 59086.58436975285, 82671.04012884475, 22590.38410421779, 81202.4546041762, 50682.62480984332, 25175.582384729812, 48476.605723242115, 15955.177703105239, 86190.4879901231, 54405.43370131321, 51313.65383683674, 3469.921032994461, 9142.139958210615, 78904.13637889735, 34987.4842547421, 9618.782362754353, 90015.9659720471, 53813.363568325076, 48432.450027521525, 23989.437552653515, 15064.48435895158, 28648.74162637132, 56977.50791467017, 24517.56836599458, 33530.311861799586, 94479.06190257074, 26388.27473087345, 72309.3584196705, 87942.15672279445, 77094.0254392746, 61613.96437700724, 81417.04360619203, 24769.838386645348, 61217.81058817748, 66780.4030147738, 4303.957196663843, 25084.004079130307, 74120.17286598135, 76225.55008800667, 96953.03438558258, 11719.682985862422, 50810.868432325995, 84436.9564230452, 99856.22047964303, 25428.23551208161, 56483.22482806404, 22945.02690676151, 88417.60952973006, 21680.036007877712, 30918.107688897344, 15700.915089138545, 28187.940912906874, 91902.88587154054, 64308.11202519004, 61973.995994454875, 58778.27645791314, 93493.13534358627, 53005.48482098933, 12957.14553879741, 2180.580654093789, 75241.48965998288, 6947.195439075249, 22399.187589816138, 94555.27689911127, 52481.96087972039, 19811.189011572606, 69914.68821900916, 64214.639484532585, 78523.9127522731, 71405.80300632596, 54519.373287503695, 65414.63991603579, 67071.72194229551, 1902.7898781847296, 14455.390645889165, 13005.067278139582, 84214.72695718736, 374.57093171764376, 9631.019195418678, 37344.606674678296, 71200.1868296696, 13762.091698318569, 44430.01671562117, 49594.78429171896]} +{"id": 50, "vector": [81617.58647064697, 18277.275363815836, 85326.55449726057, 29826.003026936432, 98445.05851842657, 494.70183942381675, 33867.153434597145, 30856.529972488013, 37885.15649857559, 4805.6852013734215, 71859.67314628293, 52764.14248027706, 92673.03486651748, 18490.106158897423, 47455.2571911813, 36669.74567900687, 56099.999108272736, 7064.952023133086, 71959.49036675139, 43622.06304333859, 28891.024940345065, 92770.61566233948, 33237.741492130146, 34846.087143059645, 36616.13816079371, 85608.63408059122, 43553.40589635782, 47607.72828202443, 68016.20394826298, 90739.78315872619, 30012.72014850055, 33409.187421380506, 38139.01120193799, 5395.133769766636, 59884.21818657089, 24077.80972078206, 1143.9950248563036, 57815.264372283804, 75517.65115661627, 77233.43656820049, 63739.6169009596, 34712.84558361869, 36981.77443784835, 25610.119618222838, 5064.615431460584, 63826.18308339778, 15939.327460110442, 9303.258966476735, 77028.5190379742, 18522.293102749954, 62049.14431890007, 88719.82340430986, 83525.76168700903, 62781.936218673465, 8936.657895017941, 54079.1644802429, 82368.80561937293, 35775.409494535925, 11599.83743736096, 51999.63123911834, 16290.453782935032, 38160.362103750624, 94551.03814642508, 27931.953331230343, 40758.83309521775, 34104.844699483736, 50450.94671674412, 31233.00992857848, 35456.49484467237, 84572.90364074081, 46965.156584762815, 90391.7940935423, 46994.912208620386, 66300.54604821562, 82460.58374369112, 49416.02858169938, 90193.20547369268, 46481.20386501976, 50419.92983573132, 31985.65960631511, 69498.94163518414, 41315.764618534246, 36239.76665846101, 7555.747726170425, 54020.705370833035, 53781.32746242537, 96900.73398602837, 13817.697472712409, 56991.95079863404, 92424.87885548832, 43293.687081990676, 56613.97465185532, 78953.78192633193, 7648.350307165141, 4355.5416482196515, 64525.57333147498, 73649.17495297134, 31514.399592615606, 70179.18350809271, 88682.98661589701, 3318.4411099858926, 22873.87315957754, 70753.03968960007, 69919.59147622647, 76168.24274053416, 79593.03898540493, 88248.36855509304, 98910.34966039131, 44987.92251617613, 98072.90393320535, 60250.4760666426, 16783.32493996223, 66304.78043643734, 10756.931587858964, 75302.23930544846, 90054.37905861113, 79453.09817322789, 27268.649435524072, 68630.45285039746, 83395.6247660148, 80660.28849064831, 26088.63780859624, 20110.490671159907, 91684.7854970027, 56263.07667722752, 83657.4451335971, 64631.47456456464, 12400.260485453196]} +{"id": 822, "vector": [77281.26783235806, 60720.693469546575, 91412.17842509804, 43760.55737219463, 48337.74598441493, 81303.17816536342, 79779.04691030963, 16454.528419044633, 92897.10319531376, 64974.01686577949, 62051.311388525144, 15818.660221762248, 32286.40359841515, 69428.71473627325, 99565.2045047039, 35464.66360445984, 84027.50049108463, 59458.474456206524, 91399.19691422039, 42919.776386615194, 63204.07758319061, 88223.01548535633, 85485.42961934497, 3434.9437032802466, 8177.356130256608, 39673.414793471726, 77695.47589508204, 69146.46134514514, 23417.328034526352, 21066.715072281404, 36169.35069620495, 72134.4396775131, 18179.778995881512, 37994.97242594557, 78049.15379881485, 52386.8355122745, 90891.08494791819, 76557.01240475297, 50498.68243203687, 46609.70579762747, 22701.636312941886, 1979.7534838820297, 7697.020115613773, 12088.990127199884, 45555.44834251384, 15355.439246295355, 99888.3511733435, 1586.2875273446764, 20994.152795114584, 28816.919763057424, 68601.68208837757, 93223.0579349898, 16176.11826551264, 93833.26217492386, 74765.97822246316, 96708.08547397217, 85897.26487924776, 24505.50059877622, 92742.36095157331, 60844.74789803259, 8271.09841831234, 94771.68926759415, 88230.98942415087, 43410.619607808585, 60735.20438349403, 34763.06393766001, 11135.111694675448, 28611.39983054104, 79109.07977783795, 99158.9916561544, 44388.132277443685, 84755.91295841879, 27582.90052472334, 21782.50081759192, 21834.5713605563, 90174.61556308303, 3852.008762117487, 45463.500076371754, 56222.70333660958, 6439.667051554731, 34540.86847738983, 47049.252644986926, 65545.1980817734, 771.0702031991423, 28297.14681022779, 19986.11815992387, 75329.50585533377, 28888.80703955481, 73103.80977830767, 87053.42468760276, 10144.39027247569, 59733.84368879427, 68443.85933574889, 9584.014053261968, 22104.088392410158, 14659.441950277918, 58499.368842877, 64090.93649539882, 66434.47871424904, 87225.29855179515, 6475.539636759664, 37836.62435908711, 52336.0781661771, 25597.185220064133, 86845.65097135863, 24600.93239026525, 83413.0443372558, 14245.458009848733, 31468.840138056454, 37050.02788141325, 16375.283886330639, 60833.84774990577, 6889.142836513096, 9839.03667852405, 3890.476766908457, 91544.10772743724, 35677.89731667175, 15242.024609734872, 9522.668832687741, 34883.25588038488, 13515.426743780801, 65217.8153984805, 82632.99419163089, 53596.938435073906, 97617.41060201441, 93906.91015249155, 68280.84050539286, 9000.97426549238]} +{"id": 221, "vector": [18985.698734253587, 77361.81455313697, 4275.904680598653, 24992.204846753564, 44267.60317273382, 42727.54135268804, 64441.25303345963, 85387.99364313694, 15894.531756082486, 44004.30604156155, 45442.8282689446, 97750.011018998, 84123.54544325196, 99605.66868088952, 66378.93295149194, 60167.397371164465, 33044.477200515255, 64659.65984851717, 69715.96677041774, 57728.204025316976, 2110.118946441486, 98398.95748045185, 34317.53113936269, 68499.93142864098, 96808.97130165368, 87437.91153388974, 19238.344167369214, 93772.77073510154, 57618.80553933714, 14325.985581320045, 3057.511045116068, 52677.072125260784, 85628.6198168884, 85264.45922012125, 7644.240014803317, 49416.779893528554, 78670.84294919518, 72968.32381022922, 28974.029748665853, 33149.70419714006, 89516.9857674766, 8141.004672448693, 41730.033097474625, 45350.97919719082, 18242.62175639847, 38116.81675940116, 64549.3935792976, 36096.75637908249, 19416.875612431006, 72430.38874113833, 66705.50392377343, 89974.62084139771, 45261.42830553929, 90780.69369730084, 25359.145748728286, 98501.44794359696, 1492.8134925816794, 88870.46607975937, 43926.58403281239, 37230.9528437594, 97124.7386774812, 52553.323427843425, 76604.12905768532, 94896.92906128262, 91305.17058834253, 83531.16394110925, 79161.90863390223, 13013.43039894507, 4686.472626232374, 92281.01790524319, 41515.48393319259, 50858.205348328, 2278.3308372291367, 80723.58302782217, 79302.81658581633, 13972.934834425943, 39735.01639909127, 43529.4474843088, 47890.68654030342, 57204.162504252374, 58408.912776034784, 62249.69207594425, 65525.07402210563, 50977.654630549776, 13804.917544506723, 10205.246630446996, 7718.184825002228, 68113.13822491263, 46450.2151648897, 60400.698979981215, 99119.43933219853, 16731.842455422495, 96545.95344178606, 50786.240400801784, 68176.93590538061, 36029.498520883164, 58702.851170537164, 96861.2915385343, 23676.575726409676, 93868.09086592816, 64129.685494540456, 29147.170916503062, 67518.87937230928, 84528.35578854592, 52675.99145559786, 81414.96265391965, 70934.51923504594, 45785.36301760137, 49149.33509224163, 50995.95380771238, 34894.640508666045, 94958.8292810381, 11197.636646735577, 72885.87545096483, 27824.41299727455, 39978.426520296984, 16319.456414982258, 35634.97025309271, 76429.29186492637, 38140.25660384876, 65318.275381974985, 14823.153880828899, 77286.46200028903, 14109.03566604983, 87951.86318585218, 4958.84803092459, 95622.22990374839, 60619.43973475089]} +{"id": 1254, "vector": [18317.757926953782, 80114.3601557377, 51685.47128231258, 62861.19709997416, 31625.135184300056, 32885.301573860146, 80736.29340461786, 98095.26970915959, 81092.35565384386, 66699.57055347918, 42669.9457172665, 66577.74164884399, 77081.04910362454, 6930.8870383794965, 77718.09596356774, 43424.2198470179, 77815.65444976985, 26401.792104321452, 20960.065842478205, 19041.143569050124, 49654.10196640601, 62272.07269381922, 82826.05461541389, 87173.54021443045, 76465.23318457509, 48552.99798928048, 33303.485542020964, 53874.68456397897, 15253.407060171887, 75996.60442820592, 53684.45334838398, 35116.327065531594, 25362.28318246716, 86830.85562462005, 88699.62572746191, 58278.213066108605, 262.6125367931054, 83426.15191064929, 43189.38001240784, 58073.972401368024, 60925.12638529807, 43965.34138982329, 72963.23261705316, 39924.46012709967, 56193.55054908171, 44868.04085225411, 39875.39026402475, 71031.52773305548, 2454.517648954213, 17147.059960188813, 96413.19966070153, 22347.403342170168, 32735.729996343176, 1593.5138216442547, 47421.5338709935, 23480.188175403782, 79228.93954153315, 15344.21136226648, 33608.87360199892, 47392.93631223904, 77848.28848267252, 69809.42400421483, 22932.05995410942, 94923.56630804668, 79187.8723349655, 42820.14947685555, 5420.963366914777, 91040.71771772942, 45283.88466114488, 28066.8550377005, 20558.195255244926, 62249.260234964364, 80736.7241396838, 51961.6971626781, 99174.53527338919, 5207.617916998308, 19557.48414594689, 60821.16681657152, 83.69950822290085, 63719.4748753177, 38541.55903286523, 30501.69639392347, 46102.02507706844, 77440.5479688493, 88342.40943899806, 14222.807951861338, 62906.72422387457, 62338.73312308359, 54007.12658656037, 37293.493410862044, 43385.32980905824, 92769.88498877443, 90210.33710644713, 67909.68748495403, 47696.33317935935, 59262.05494902892, 64296.52308472379, 55329.628222321226, 30398.285472958574, 42795.00736816292, 39077.017768579106, 86310.91919403106, 72277.3748223615, 2996.1159209021757, 63818.592505183755, 3460.681893702333, 63440.03088686181, 29065.226174680625, 94484.19570463248, 72819.58900039726, 30065.37186953161, 19960.79691311423, 49688.93828411121, 48202.83306031305, 78429.31835814193, 82254.83305927178, 61425.37418336398, 37532.504841944916, 34154.986626251106, 25421.00033503838, 34592.29588521215, 23550.143886227015, 28218.847066143215, 42921.78736272917, 91244.24909002325, 36427.48437184653, 74355.32306364214, 98818.79892419047]} +{"id": 877, "vector": [77166.08075780704, 26583.40414287352, 99165.76552316692, 74540.6308819017, 93736.0494763593, 26888.3878955547, 34808.69696337903, 29347.16057297555, 51540.89915877076, 8720.676111018665, 62676.17311443608, 41050.84568603986, 81381.41256078669, 43072.1655333385, 82921.11200038389, 931.5771439200815, 41138.45893122518, 40312.601303950214, 35353.82373336978, 57536.63158391348, 84966.48787825542, 63839.9090497362, 55791.33936262249, 24440.10653309199, 92473.41881333999, 97503.57918787352, 65736.18541897663, 85609.52540030162, 13735.677466790741, 84001.78830476225, 9011.893409009253, 26930.37305172873, 75209.48321611014, 44248.26271432302, 79350.52197182951, 70561.82375909614, 11249.727992833592, 33438.515110157095, 35174.54743028725, 57280.61541985458, 24931.097359078747, 79981.89633908872, 10905.284145426163, 56741.20310390587, 18943.023501618594, 87606.87753737085, 51441.747794906, 38740.25321133845, 88374.51862621366, 46117.86276494556, 76133.5672665839, 27736.59574754971, 7418.6595005980125, 39937.357756035664, 56921.03192598135, 67606.68996009447, 23100.817008957285, 20424.182601211614, 59906.247936699234, 48882.76123178017, 68623.37581011633, 85697.8422647942, 55709.77066868279, 49102.01998130983, 78682.76034255314, 77770.52437241092, 88566.1562960976, 58078.22024346139, 83925.72807074685, 53887.79472021077, 55534.55140762451, 10342.96024681277, 89562.11701263656, 68883.08922946814, 58647.22275538926, 65307.04940089912, 1043.9942239969469, 53639.226208831526, 99571.70322169963, 74570.04127395981, 46719.35989655892, 13545.335055238016, 57984.339701549856, 33072.804294873495, 28219.420377903993, 27114.34811544381, 82612.9896027613, 60758.86612292728, 16645.848797100505, 80184.96136379555, 68325.59960817258, 78009.21745428318, 18500.687711777053, 73184.50609231673, 28602.75941551934, 19940.95876900861, 84324.43280271378, 80200.7047374485, 58218.51239049589, 35975.871266922055, 17374.481043925705, 11662.560745653982, 64382.11068197189, 42594.021666264096, 84413.1829462427, 74225.93419649641, 15655.139836503151, 17729.152883731535, 55994.236360136376, 32951.12876691794, 32270.60866781354, 28427.21513800789, 47108.41453067143, 35757.5318831184, 37692.28823186167, 75847.97357533478, 35217.891256239694, 87751.08845212824, 59222.6323520152, 81209.05249977966, 87414.76216051329, 35988.813976928446, 32839.77438824793, 1611.1841752796895, 74315.63833804011, 93136.47425145582, 35304.38658306233, 61.045904123580016]} +{"id": 1982, "vector": [79782.84704945656, 41503.27677746798, 53366.423564777935, 68002.1278367019, 96492.65372651708, 7314.2146676659195, 73306.81032058108, 96748.77680943246, 39734.28615857009, 44513.417733037386, 59964.15805952768, 63168.19517861003, 8870.130738941929, 81396.43353895564, 69033.0605303113, 28522.92289716979, 76055.02060514942, 69023.79676155737, 31196.15862114451, 20514.88033335799, 57349.27442927432, 73934.60808114427, 47748.09157075675, 60988.27701701528, 73692.91593274048, 51264.27112035188, 35133.087444959245, 19679.77890398931, 51100.22946301617, 38128.29883469814, 33715.079757943764, 94304.2646947044, 21307.479465280965, 97210.0696407223, 39550.651893546106, 1822.572191111349, 3435.758304275971, 15410.027085307054, 32197.69298429004, 68197.66697821049, 36051.32897429862, 71352.28075634205, 28600.392995569855, 13193.074158838292, 25192.347907481195, 12502.303168728335, 19144.940361281882, 60141.36638356393, 87962.26885911303, 81818.57946009385, 62687.06247726249, 61666.971363484656, 93684.0615895059, 96274.70657598686, 98629.64889032384, 77575.5549268671, 20929.654307151824, 12137.036948424473, 55763.69394736618, 35864.631568669334, 96674.20994251284, 99795.47969269572, 68048.76161414426, 35866.85868027919, 41572.258037883126, 14952.638386982819, 86791.28331654477, 61107.330074403646, 56219.217077207606, 68624.15089505899, 10330.362588918906, 92185.12523317631, 19821.15592760585, 37248.219587228785, 23532.698857936175, 46458.892320339975, 19552.15532675787, 35387.99662755185, 87708.84965256478, 74613.74210905333, 57323.95658853532, 11190.1409202121, 73810.45634807377, 86161.73119439842, 91059.76474807339, 14614.752785177554, 45613.65453630737, 84955.71878868854, 47270.29319509821, 11638.659945625852, 15943.7039363299, 38110.21877948629, 10927.588761408591, 98970.55162556314, 51485.50766583164, 72972.47032516685, 94464.59397250321, 49162.95057570422, 79593.60954989793, 31642.033772736522, 55316.48411739477, 97431.56126830554, 91776.60149864857, 58703.52760695584, 66618.3299427299, 86252.88381133579, 7090.4974151208, 7027.170814192729, 20593.711451522133, 1038.2907089961036, 30510.525817122612, 25122.273255677064, 97510.33517683505, 62627.46888885732, 87539.0391573921, 36624.49919723146, 25574.465110180565, 24857.05127363336, 97953.83756047285, 81190.85316558168, 15635.496966305373, 70021.49986448827, 89879.40819590513, 2853.425050907299, 1204.2400070135461, 47360.096724334246, 42444.439287188696, 88287.26448659578]} +{"id": 1967, "vector": [31315.443989994972, 62049.097787282124, 49015.65744854589, 84886.34311694039, 39491.995370570665, 58343.56525050196, 3691.120413336535, 2958.4058081286057, 32912.86835850195, 15216.521994754406, 37587.32436071422, 18468.370613743202, 20023.467362645853, 64306.46072680015, 22618.67202059762, 25441.594103184838, 26152.334368047468, 86596.07616558284, 94164.93776139182, 18441.653378899315, 11954.63475431402, 81603.72064893915, 40180.4401386775, 17181.18215455976, 57648.35139627127, 33885.40682868676, 12802.767184805774, 73580.42771715672, 97520.95851456844, 51065.97599674889, 31981.243379893785, 30755.62860575762, 77782.26832211879, 60464.16340956664, 2869.462056137462, 66687.79698006167, 9408.13834308869, 4477.661657068233, 92442.7356218674, 71982.14075966153, 83183.00812332652, 11259.88711716146, 74645.99102129121, 23170.80303918673, 83755.5638217841, 32468.234104297633, 62075.81157966353, 34890.193011145784, 36798.77758447676, 94321.32721986949, 11937.69405879882, 25970.142447970633, 24042.33253597464, 44149.66415214213, 106.04270113806491, 79468.40643735073, 98694.78717911961, 37656.24240822847, 43810.98046523089, 31998.510422441763, 85837.1078977946, 7712.03445645372, 87414.97689469134, 73267.77313682646, 84637.73593297019, 75778.74724370804, 79877.76138394637, 93460.04829077529, 67237.96417391363, 80450.66896956897, 9906.518799534659, 6107.751367461589, 65337.45118652307, 11591.092462605302, 7028.783135207795, 53291.830545564124, 7028.785831179874, 84865.07004579272, 72751.0331247355, 71438.4955472617, 65812.78878990127, 81893.15428226552, 41357.757254698234, 30509.757329361477, 96597.7236800945, 32694.239450341178, 28500.79289761812, 11833.05493320146, 31214.457477386903, 35455.94058054722, 38776.67076012462, 94034.39167766263, 52815.255531579016, 397.79779130868496, 52870.587177997186, 25264.326807422243, 81345.30824573322, 91555.08409129092, 31918.424466281558, 31361.684532839918, 70698.81488628572, 74644.51130547812, 24253.36835810026, 87524.73711027902, 24927.45707218299, 30626.66284233857, 41151.330318317836, 45429.44792718216, 23272.971088021553, 89996.25791539197, 64743.10211667861, 41636.16036578851, 768.8703733463198, 14974.936227112234, 93006.60011559632, 12046.213894752855, 50402.89419187505, 28145.518591587537, 59287.904495266775, 9851.346664242255, 20675.45110251725, 82651.62995740106, 32771.931405831434, 30233.073014514222, 18744.414175168655, 86528.25373137764, 47100.25553065654, 60793.0974195614]} +{"id": 1081, "vector": [10879.604791929987, 13849.266754043489, 72833.26863483844, 44159.20844896392, 99862.55604647551, 70210.34371688333, 3439.931153258291, 73662.10966040973, 65307.8110331816, 48204.97939983785, 73984.11974100694, 66812.32975509111, 96716.82786910342, 20810.20487930485, 65247.73725525148, 98308.98618788943, 76040.00578955431, 90914.18758660679, 28013.207517588722, 12522.404238355733, 21054.222571134316, 39057.747742414715, 11701.755206917009, 19670.15206350129, 14401.114224598221, 17337.911980107558, 93916.59733955587, 93866.73055581687, 15115.64767050172, 26808.478219690824, 1531.8674597376457, 63593.167048867595, 13214.42529677077, 95604.21671180344, 579.1094573673106, 91145.84505051201, 75241.00537824785, 63118.43350092044, 43738.20392187858, 69322.30095188726, 5853.194960316221, 2178.474930241747, 82232.30901899945, 78093.96786224729, 74341.96909517478, 85932.19048404327, 73734.40404015552, 5383.377485399988, 92565.89813788708, 97800.38727162742, 20041.172599731726, 74824.57248066635, 83811.05868163705, 28814.14096526893, 62491.479486248914, 73903.88426321124, 64020.559650474606, 33798.936297154905, 70395.68731146028, 16775.821787140212, 50980.61282385942, 50718.321255294766, 49361.33106888858, 64770.16688195122, 96601.00674531245, 13081.805474655528, 9484.325390607084, 69992.53221933663, 3414.1245948231312, 19857.206813480454, 65506.20467989996, 67350.185618288, 75472.45865654299, 61678.757575266085, 87792.79359253928, 48123.22337955742, 99287.30080165254, 46172.06161373125, 73590.41856802383, 80964.76447224106, 36099.56799009717, 32603.20496062322, 95853.20541057727, 97051.74742010956, 89682.76542051721, 76667.82496721443, 89585.38520084046, 60179.01810848025, 48595.21756260076, 90305.60977334899, 50233.51241841279, 92662.32707364987, 18401.390663849616, 85846.59742505521, 7311.371678131695, 77390.80222962232, 48168.80773864053, 24425.797807536863, 98306.9775170619, 73752.31474072748, 33121.03469184471, 18768.02405465937, 94145.86240405132, 48777.05516211346, 54439.81970023257, 49656.79683191352, 84181.48722938716, 7119.927764151524, 36933.35722948545, 22943.944168715945, 83387.3496754844, 56015.01911031259, 27041.980630506423, 34916.67289565251, 90888.11919680687, 67837.3509126892, 9370.175880453668, 32826.73115804552, 14343.039942489044, 93913.81862036168, 12034.22161736375, 84611.95819971, 99526.93041966906, 78630.06731159936, 21988.200616802933, 12439.570471975914, 42097.519206215686, 83737.72331118773]} +{"id": 1390, "vector": [99616.47124607499, 49348.679418081774, 15314.861054646655, 50392.771568760516, 73692.13239782568, 17788.25633838279, 66988.14112912536, 16301.324487713142, 8678.219763287198, 58720.35449956218, 47091.001057427, 42084.0646798203, 16676.34261305919, 53740.03766516473, 89813.09474733431, 43652.154000792085, 55818.22969213094, 31299.49139983844, 71653.92999940511, 52299.62605536643, 37826.69766553945, 88869.04746723236, 1684.6374729064028, 40407.61941472176, 76452.6576555464, 93224.74464530913, 83542.35276845205, 59325.04757148074, 72663.47144580261, 26160.102409081308, 49819.23004298418, 50940.7447377884, 38584.511158681744, 57535.947375291675, 35124.28871331985, 17483.730014681798, 98895.36463929094, 81577.57540778503, 2171.2518870425133, 16370.982493686859, 93507.55849387818, 33304.41110519091, 8366.449082390203, 91453.13911087395, 78785.60182657451, 76957.3043542105, 22739.63461260449, 5034.396547837672, 72246.63190144462, 7423.885996755519, 7609.653391800164, 54124.10289219991, 62851.82746408986, 91096.1202684832, 65000.71277207359, 31746.01186171885, 67256.89418947803, 36644.3741733786, 40233.006305924566, 49167.918074628, 719.2565170587973, 44497.34111555277, 4534.551410844556, 510.15175233509734, 40799.004929322015, 24198.49592646024, 66584.50551171912, 96158.80206997883, 18102.60146442222, 84666.25129924824, 25267.681905926987, 45008.56755541499, 57227.52810987043, 35273.66870940601, 84247.27141064158, 20844.15881319248, 26179.73296120094, 66959.0306155325, 78098.2944509492, 72383.94644626298, 60476.344203259294, 21658.04066396151, 25808.712852798686, 86894.98206320789, 72817.61131542482, 75187.27227985016, 93159.90815367199, 2368.395874499374, 23259.77553823728, 93972.71507924543, 90650.61750505681, 67298.12185738541, 12010.424128877417, 61955.208311883616, 36861.32201645528, 59478.95965801563, 41305.70043753842, 57382.84861924987, 70452.7005307237, 25801.70209434203, 57192.88735635417, 41297.6096830865, 57334.052072302824, 63705.738681429735, 77495.49441418395, 37023.68842509137, 35017.531333331695, 69684.00764527272, 21238.088770324313, 10427.024464295353, 97700.90420675598, 3372.698048390066, 81498.70620035434, 75531.06322691942, 21004.942856289254, 49953.62576956087, 68629.34414401148, 40827.86548788401, 85977.15504197383, 41796.353631389706, 25905.117911608566, 80088.6797458749, 50002.65136113057, 56924.22299692072, 41187.10562901537, 2736.0496492547572, 82536.68915921512, 14990.755713677683]} +{"id": 269, "vector": [27726.940772416543, 23668.64122698227, 6409.205054566569, 64054.78375510407, 65699.28493058209, 28064.41102366598, 90223.49482723126, 5310.940918603058, 63477.220001110414, 12425.144523469322, 88863.86311510963, 24222.61980888102, 86245.48211701044, 11814.320509631982, 43768.72008081109, 55562.32212699913, 44702.58933729163, 89983.16502640699, 11229.726199470768, 88976.82083030006, 18481.60848139997, 75213.11500324734, 56542.392575579695, 5573.151079523653, 51880.14203743959, 5303.092541393162, 72996.92290127282, 88561.63442726694, 78577.4514982049, 45702.16876549794, 8724.267069400405, 21174.710247390027, 36766.88818421503, 39614.17730409394, 62766.68366386354, 88330.62875187413, 70950.56724506697, 64646.8939811928, 24271.869782675592, 8718.018040856801, 53264.53006458411, 25811.417917262912, 34049.34715633738, 50110.57587449988, 1942.1266762952305, 83140.71487918768, 37362.4507041013, 27566.486920931242, 50535.725279938684, 89656.52835122579, 43360.490639322525, 59029.655622915976, 25825.3169783218, 91335.32654267586, 79538.7311104147, 76178.24832062225, 18838.756660936186, 96422.4771537091, 53851.46744850695, 79255.15870232222, 98533.5280403485, 2461.191381062733, 4261.510224669085, 57996.795163262424, 54337.689483576876, 85459.5604938049, 13025.682147234063, 69318.26585042942, 45457.62191715391, 54259.8494809204, 5244.596200095264, 2236.927921849097, 50600.64136591088, 88028.33735407676, 20370.970138417455, 48503.32290282718, 32.58910656049885, 81906.8557337783, 63107.32641778887, 14009.2211930547, 65399.10353597161, 63573.0707758091, 21418.97284581332, 75120.13132091553, 50181.96963093941, 21681.648626381968, 9233.584405953887, 34680.7435003207, 27521.773630378033, 45806.11610781195, 46322.649149633, 70096.38144721268, 27360.560521220978, 98835.80570298406, 67551.90255218797, 51963.63303667279, 68764.23115925811, 56463.22048691086, 10911.615945952703, 24142.951893359987, 75320.34624099071, 85814.80046188076, 30068.643808164332, 26687.70227792907, 18175.904285222732, 74397.10720984492, 53845.951988216846, 42275.80857598233, 72345.39013740214, 27243.136120455958, 4717.973067151304, 38792.053196392604, 28243.414454982907, 29531.58330127551, 55097.126106295626, 63727.32970424598, 42766.49808306775, 10986.5816700355, 75613.78653157387, 45619.24306712937, 77094.23580251713, 53815.31775704883, 62484.472652214165, 50124.515308256334, 43980.08851770402, 84842.19282672588, 46087.00700840358, 12543.601885303935]} +{"id": 884, "vector": [65810.36043714982, 40926.35030003417, 77712.85779786887, 83899.0531078194, 72090.05435241603, 79733.23358428199, 69616.35419011256, 18651.130753609148, 51931.03085396247, 75450.39425791777, 56712.50789415476, 10809.942973860841, 50420.65693306823, 98564.67169333818, 26894.170989808696, 91750.60922295536, 37372.42546380093, 23116.763335202784, 22951.56616414109, 64374.43172234304, 95487.09704745421, 25971.99269465662, 17460.92322261994, 5601.121136410059, 82171.31844627361, 14117.901387544129, 79747.35584989576, 88117.79093446746, 99985.6928097177, 97295.93447856742, 3882.1504552569054, 70808.173465216, 26228.294880717716, 47706.83950136379, 75746.07287246034, 29252.926366473865, 71677.70749736746, 33236.39319692645, 13938.044870851252, 56628.859208258444, 57474.57415607284, 71647.32466087847, 52090.75418026598, 66066.17005616319, 32150.41915123269, 27044.161035059467, 64301.12940717351, 41802.33923996591, 69890.34489470896, 7375.585659542449, 70927.08139306722, 72638.57619775258, 35872.97422588403, 70000.50263591016, 93371.93087149116, 52791.75871564965, 29300.253470421623, 19517.50854527725, 44842.07871389493, 99694.45347180353, 60660.57864903176, 99476.96393509943, 63826.75948689461, 96056.95679518458, 95229.74829967659, 53120.862140850455, 61839.09528720036, 58325.6810545632, 48252.049278415674, 18012.24502063302, 79422.2814084364, 34626.5424955761, 61597.86770252983, 20590.783529230426, 73770.93050719237, 3883.6477558684246, 21017.20247625517, 80587.13759137374, 35503.71316754529, 34600.45089869146, 70588.75617206514, 98227.85920416738, 55291.464902390784, 95382.99188219341, 78442.9853175282, 86971.29600887976, 89050.02665614906, 63997.78774192123, 79074.67289752194, 35643.23403066045, 19933.29677226461, 15285.402687719108, 41995.73930958966, 2157.2200811170173, 4643.92867797746, 38161.74601851413, 57079.717858227756, 47875.07567151743, 44973.01971919587, 65918.743147246, 18366.918570509337, 92937.76199005467, 22722.703737081796, 9532.103189408259, 45755.83682877616, 62713.694927954464, 83596.75180165897, 29427.809309762542, 66980.70447900068, 67608.82075187276, 42080.628650474086, 47853.94098423052, 66689.6992438284, 66214.87360315066, 62040.047242659835, 19541.69100658907, 60610.60557404189, 76814.96767716654, 97344.49847382576, 11410.838705553184, 90572.28232083592, 88687.94532062563, 69341.06949859844, 22298.24139486407, 69272.45290933701, 34782.72636329339, 76448.19699308119, 40774.40733117305]} +{"id": 2003, "vector": [89302.86602818876, 55603.19385834676, 96244.83662811591, 79147.04830381076, 2602.735736605899, 50525.985562471134, 4511.311150148933, 81850.00147341397, 63954.849062412744, 92262.50757345158, 64863.05176735735, 678.817045678326, 86594.91002486116, 72391.8757669802, 52547.28875910002, 48295.36243052126, 1552.3241488998308, 97901.19373277582, 10933.488154162585, 28138.040620135675, 46550.71684245945, 14657.528650581331, 62442.995594068474, 14687.351484674049, 36500.375528697805, 63035.78364708783, 58934.71497798905, 21661.674864273962, 69041.64858005717, 86060.82084843678, 15986.493529803214, 69202.78088742417, 874.6133883406504, 71728.07532289132, 73059.80793865524, 14674.832157991403, 62859.641613194624, 86147.12644726151, 50236.76511510013, 23482.0270265926, 53933.91997469013, 28900.39918680403, 44770.11958499664, 61703.47919323856, 16130.769031812664, 41189.32660462086, 94395.26411059385, 23146.922145821536, 76732.89165287634, 53995.866449566274, 15349.497573774463, 11389.673576213954, 32107.193055803495, 38032.85233341194, 67216.5372622054, 88672.86300736488, 65100.41066057375, 92623.45129757456, 28919.264616189856, 30667.720386613262, 55902.28347654691, 56516.85533118556, 40110.40395972212, 12328.147346346917, 5226.364385414994, 33405.20944219817, 87247.5629918902, 82944.18591148437, 68402.73997358924, 5483.649644752775, 17042.49331134774, 6873.284313181683, 23400.350034203377, 36425.98581156604, 12036.595806765594, 4736.710241122355, 12517.525531389118, 65765.7559804651, 87754.76902212691, 93105.84559503097, 18454.173745556425, 1819.901354042941, 71949.68846365913, 11180.886294390491, 82171.61632075871, 68851.98122228116, 48845.63826261255, 5623.585643333029, 29024.741188569602, 73707.0111516333, 17843.621347582906, 11321.678831084004, 51084.597908690565, 7991.372667839014, 10000.670353301311, 93079.84077913803, 47366.99167360553, 14162.811775235528, 12314.547670472964, 55173.08533729132, 176.86650141582882, 33328.08761325632, 32829.21021865841, 35058.43585336897, 19831.414172749108, 56620.95843534646, 94685.22970562492, 2025.9024541395277, 59188.43058267565, 5865.147210534827, 41469.06401722882, 51392.56551997398, 38410.91894180094, 35773.12672777237, 43961.06875978954, 29636.395441546338, 51141.577676612535, 42773.61858095133, 9885.050238164673, 55653.25422058231, 18763.959228524007, 9475.650603771324, 24427.955674892764, 91435.3474748393, 60406.86370993249, 41598.503667982346, 23334.41055502329, 34751.94093782221]} +{"id": 865, "vector": [46980.72372229045, 52802.264980149885, 10901.39487477264, 50963.6673889593, 7532.620452769267, 55605.86409185715, 23902.413931534105, 76671.74288398869, 39699.4610918746, 55143.43063041031, 70948.30477282823, 59524.565150532304, 1476.5566973353073, 71658.76209069266, 15340.080159916337, 78814.1869463021, 36115.64259857527, 47822.88273602897, 41687.5204734759, 2366.81746879005, 79153.50634135975, 72938.33007134109, 99617.70401374162, 74263.68653602924, 5978.850651172862, 17680.569442746662, 12049.50075484701, 80652.88826715125, 44528.60015411563, 87176.49785481543, 51125.00315614191, 32236.284457115817, 46953.76717407084, 39883.24665550862, 71413.01094072466, 96005.28264890485, 43236.60104931436, 49235.09880521183, 99100.42239066871, 81190.77833906532, 11764.271197852617, 55225.12413829227, 36425.14294047886, 44922.35563458077, 62174.8488429468, 76648.35381592634, 81024.42622381568, 48151.42273902647, 74354.30953829232, 25965.959638248136, 11430.046561321316, 6684.065430898689, 33332.51404748249, 12281.357855451803, 82652.8014505802, 87510.47994466443, 88167.47360609534, 3980.307969664676, 59512.065119181556, 7002.9935537171295, 58140.37043285428, 16966.55659490469, 55695.00625558631, 12126.971399745224, 72003.22687113013, 38046.870451442504, 33163.613567909444, 34851.269696546326, 99435.84311419328, 50874.00448806335, 70889.495293687, 60433.963209821806, 79964.62476190855, 89251.9990559314, 36604.12255798853, 25625.82576790312, 28657.09618839488, 7004.509220507971, 18450.883849036072, 78832.79613978944, 84360.78247945744, 83329.11430488422, 84086.55806534653, 6840.811694017812, 80011.36686960458, 81629.58587918166, 19318.611852575017, 70405.40275697965, 10561.070379478942, 98755.92191987336, 19150.812614866096, 88914.96955312323, 5004.148383177043, 32573.490607366006, 26277.426669286662, 93481.02911578608, 91949.6738131718, 45122.89663224842, 1971.9544794047583, 1881.5815754909981, 42577.96753578597, 74664.5881202788, 50390.3491939103, 52025.876291134286, 65795.8692344952, 10885.543252660145, 93814.68328092596, 73767.0815469284, 68006.08213081438, 82014.45900605689, 41571.46453430996, 59994.4540684484, 71975.62610610212, 4396.048199607483, 37.40107741507792, 13875.01161549628, 1817.7839658708006, 73705.65266452452, 66783.82381965086, 57827.87876568441, 81412.76459260583, 71528.43064971811, 89768.81128993673, 15778.724176613512, 4023.844255360398, 60733.80960555239, 29923.282349492696, 14353.202445331903]} +{"id": 767, "vector": [97147.03025192696, 72492.74835191252, 28466.556087230387, 32129.187839537044, 65906.90292348979, 54542.12410733174, 97599.13624310387, 32045.8913515552, 21359.4859595336, 78133.33622825517, 20661.892832603557, 10651.289035007261, 23617.06801510791, 58841.86102142449, 50360.96578479285, 1037.9847855432113, 47106.45873548155, 64859.68481223228, 16252.549205507483, 59822.820644586485, 94006.13185706844, 54343.02148300381, 70507.86382856873, 3257.5192200273827, 37076.48581780193, 76905.9986559662, 93518.62725603385, 37602.08700209645, 94550.73373205433, 11022.07504367696, 78004.26069447288, 27293.44419055231, 69986.96933155243, 2885.9015522237464, 16034.576405301515, 14655.708317362192, 18326.758481934612, 79592.76447487251, 26948.411151531647, 55677.61683170944, 8886.520548477061, 55802.220431115915, 78805.36310209095, 79023.52598430186, 15061.217473474997, 73869.10042072227, 81779.67173310007, 38070.17200817857, 87110.15583112647, 27962.388890657174, 1249.864449105531, 8392.84483476308, 21498.542214240024, 47895.86209868162, 32652.87014984537, 89581.48566286398, 43503.58207328548, 3484.1622113550243, 98078.64361782197, 96572.15061846923, 71983.99796521728, 37643.42134834412, 50146.128799719736, 78982.91670944955, 76974.42927117387, 4079.7162929774754, 1005.7837790105273, 37562.955861137256, 48373.14171114543, 98677.99920066594, 24349.401304733987, 19437.45925818978, 23126.678895931374, 50721.93699136318, 99571.37935931949, 78739.626257883, 84877.74199519008, 6313.715008462428, 46298.098929968364, 67681.07966771368, 37632.46021124279, 5843.060183995963, 93652.95410811689, 67246.90500607243, 5442.306220307091, 40440.10947911466, 63462.68809315544, 72455.87549035896, 56120.81393036505, 53818.066606466666, 50873.02067708822, 89629.1769722895, 64729.87201644041, 66348.97663510394, 30805.024569985595, 20096.809206783917, 71203.35020342666, 52878.070725760765, 66463.23005238631, 22606.952983259933, 7105.585692637151, 676.2339263728601, 69775.65291817897, 48923.25410927192, 20327.525323241603, 25602.475370347423, 98424.36893041735, 83638.12670802846, 84476.89772113597, 4725.651093662586, 4284.105670531235, 15192.397849352723, 45569.526816108766, 43335.49970964896, 67670.11003265905, 17793.422292057836, 35798.25548301527, 96180.52160196709, 92807.50988522088, 75212.93703089988, 76374.69505157268, 68310.28482221098, 42242.7939810243, 17552.54947517163, 37111.04463652979, 30772.024329103897, 38004.05428133776, 21543.684016388932]} +{"id": 1410, "vector": [79868.27182849153, 90307.14850784668, 35374.43678150294, 68272.59646899099, 87122.58674566205, 64790.40929062906, 87201.77888918051, 43879.793765549126, 18465.76761974009, 93888.34143522888, 95800.16788762345, 33318.569979745815, 47486.050994415906, 96011.03978148005, 17738.134973477638, 40507.0452803373, 5905.6865327282385, 34516.73495918415, 44204.44461465868, 67049.51794442203, 7828.082572297934, 27315.11624221742, 81057.83926303688, 4691.000607964657, 25112.61528270433, 9739.404009535612, 68042.44587979537, 17792.70874815673, 52072.48786615682, 38891.020221082515, 81951.52268111281, 90277.94349888302, 84422.10871737481, 67649.83329223072, 21986.213585945137, 38443.01184882747, 37179.23421790772, 85195.54431471809, 43597.783462597916, 85096.00415004778, 67425.23466466673, 59390.62801510427, 34707.34423778101, 65131.3575819726, 56018.844699317095, 78263.70969608729, 32651.29365345195, 78559.35848104974, 8845.180460637437, 70887.35516813141, 37191.254880989065, 50249.308803894695, 92880.51440368447, 63548.62371055001, 99051.63458297194, 39261.181902027565, 97210.46920090132, 56840.90184231137, 4432.693414198951, 73243.1773237811, 51278.57081328965, 42245.789175272985, 16248.68805987576, 16404.927157657166, 4353.735844703799, 27811.29303672316, 16949.369941716806, 19116.299813796955, 34435.53274395269, 94545.67158446314, 64682.300626783195, 83454.41881227428, 29095.57297439046, 50214.53401823174, 50025.79613122611, 23390.286534393024, 86573.15088123163, 7953.144878660268, 68871.06724242354, 12881.515083804596, 8234.317238747191, 16278.54475956595, 6284.8019170780535, 57210.59277947046, 6878.377262269652, 97568.19752138645, 99112.00996027289, 27089.666341612705, 97802.45800706344, 69352.61127966314, 93284.92303902935, 88963.95470303972, 97579.29756649432, 3168.3436593447077, 5266.204429826027, 8878.144076565353, 47840.97816103552, 40157.46855875331, 46899.4796521925, 89445.5061215865, 77350.74010687388, 70411.63534802009, 29167.061099308445, 47344.334843907745, 70594.02165854964, 85397.96373362886, 44700.11408516428, 55228.654734178206, 25664.62195366471, 14785.433886436716, 54684.41442047246, 77955.3596811007, 45599.33917716238, 38678.53338410852, 36177.87378113373, 3180.018788968286, 14112.738136432645, 39673.80782302493, 72491.44377060092, 50070.914983103554, 26369.54675217168, 3155.0007573136086, 50800.912621131334, 10414.96518554179, 59877.12121131504, 75079.00509304692, 89060.7852025705, 60639.660351260194]} +{"id": 583, "vector": [50515.6776348614, 5551.548685210939, 56128.29849291141, 88315.18545153318, 26583.120544628648, 30448.909507416265, 76671.32944830843, 95547.95241090172, 72943.73215006289, 46935.95858225994, 82875.08203001221, 1236.2135645307858, 28144.974658726496, 98760.4321297013, 20568.932226111934, 53853.0100851239, 41270.710794235645, 96608.09402405606, 31878.492116941925, 71408.71664450203, 85657.79865756788, 43204.528327029526, 54124.32403581198, 62036.622133867306, 7648.774092934874, 70158.49348764386, 80897.99121409768, 42127.384660807074, 43074.92963870867, 38164.752135699055, 64855.80777114074, 25072.61402863601, 35068.899171099125, 11076.11005409337, 17441.335409456406, 54633.10202785247, 26838.443237000465, 67866.2068996452, 88675.45785426043, 96350.42375750307, 57616.766058602334, 823.1856659253411, 59831.041671929364, 59229.058437422944, 40294.665423674414, 3697.732420728417, 36101.9552101254, 16355.849793837806, 56257.56438063805, 23564.30964714299, 63670.34290109155, 6144.028201971063, 12060.806119748702, 40073.16646867749, 39055.52400621034, 4883.689708500317, 17544.585267718605, 17217.53226625352, 2557.548126937226, 41496.04050511583, 23310.723253244192, 88614.42266647272, 94070.87981745691, 4228.197998982586, 85464.9795409909, 77704.1022338911, 66102.31383633634, 6895.2641637822335, 69995.66097862687, 37994.55580252159, 22572.052763352414, 67288.33057286461, 36638.20746948714, 4932.135445004071, 36589.94353311046, 6879.55322736501, 60236.912575220034, 3951.753578812489, 38235.94811346484, 73253.46908224703, 73833.6933679968, 40030.321458203456, 1503.8575724556024, 55621.64732975, 47026.05231549944, 20681.908285995378, 81011.75842137795, 87563.51793485958, 43098.752384926054, 59191.828614144404, 71046.04322333992, 25180.493478214772, 42971.29311124296, 14600.084470871732, 29391.26081985025, 91236.28061507142, 10931.279165374819, 63722.262752658746, 60352.71857504475, 19584.89224322998, 99116.81243447722, 27269.114615971102, 70394.35089087243, 5695.498938786814, 85531.94798163947, 17785.235995380433, 52792.35794236389, 38642.78686258945, 65772.59652697372, 70833.3344157427, 57300.035144170855, 14401.247375583514, 15593.717243656047, 24792.48475563053, 66753.22407170534, 61908.83167218541, 24443.8600649339, 85340.50920806582, 84212.19432345827, 3770.7422415325764, 32107.78504806516, 54539.8291628091, 52681.389707839546, 71743.06678817706, 86716.26259748469, 97151.99115689898, 8456.149269295243, 64477.072092804025]} +{"id": 1962, "vector": [27664.133941432556, 71458.04570445188, 52862.53944067572, 2467.957660399278, 47739.601785904626, 71172.23408521418, 70535.80280077759, 81519.47550885564, 39679.23426235584, 3710.8515783191965, 75040.20502120808, 14382.509209237603, 40321.29112528491, 65976.5182883474, 61951.91922922565, 36492.52019728017, 90.62312490216718, 32476.971966015033, 52857.49616044518, 18427.59809786988, 78374.68985563872, 87007.31223969992, 89002.21655495866, 89578.90717604973, 70817.40788460837, 96430.77048065457, 62049.740836288904, 9592.30631334761, 16515.466910351704, 26016.834507374788, 18961.144017520604, 95136.00723428729, 38022.74146592409, 66866.37723081066, 85192.94109995906, 91862.26553969052, 67909.63185957793, 94053.79842771137, 67715.15001386811, 44397.96629394174, 44117.726746835215, 177.25964213859413, 19877.699176044738, 73073.01288647458, 46204.90355103573, 79082.60899183979, 44948.60996856636, 89002.28564229203, 89444.85157528109, 73272.61671669928, 51991.631991408816, 4573.303650408489, 89539.34991994877, 3986.2630976516657, 86205.32423094602, 67170.46477840008, 90891.39870282887, 97882.92642308517, 21763.543709775357, 53630.637661482484, 11038.209251447728, 59940.466381001956, 19726.936033194375, 60769.733703552396, 6809.73554842198, 72563.17512716683, 44724.81518676017, 74493.45047011878, 18034.674297156573, 78636.04009612216, 11431.821988526735, 15049.873193888885, 17241.132542899184, 77893.43870221745, 96101.87235449461, 64237.59486171766, 60985.450142592264, 25883.708065541865, 58454.58151496072, 24793.741750310626, 38879.3650939409, 72683.95412861937, 95485.05168519857, 23394.87176075169, 33842.80869766072, 43242.18367576789, 60893.01243380924, 90035.48383615629, 62785.43879043009, 57560.04016737437, 24946.15463749359, 96388.016311374, 52754.55411237845, 55882.47206193475, 8642.410481204355, 37342.220491054104, 60839.95395437908, 89554.58705224354, 19238.789078227746, 26204.7373585509, 66415.62354447476, 85593.34828007074, 44024.64525343238, 94118.4761944213, 71680.43833968086, 42095.028739945665, 22018.382681060113, 88109.51960275599, 36219.56665881038, 24431.531839375566, 94347.93912059286, 58350.33723093561, 76976.9868972398, 20952.34886266386, 79926.19913697153, 23439.829077121678, 66993.56078635568, 63136.939537282364, 92771.53428858006, 75902.48618383193, 4871.59060467186, 46541.104717302216, 88639.49915971766, 22315.25735715205, 13793.532059544033, 57592.872762759005, 31656.956239299227, 1296.4527133178417]} +{"id": 1406, "vector": [87475.74802394991, 32110.31432111554, 26417.57641041084, 27322.939298818983, 28877.095008804077, 73169.59997178071, 5830.6631986658085, 1993.944671385417, 18608.84019770015, 86803.05421734789, 17323.541748251126, 26661.685341652443, 44849.554652696366, 82643.28318908291, 6463.472536438319, 52428.82099573976, 22871.323563509748, 65582.25346260694, 26589.90339821228, 29455.945285662765, 15164.447806734916, 10134.592873290449, 14805.650733307508, 34585.72155443238, 15174.646884626163, 87672.76393039564, 4236.345037274425, 20633.163137277443, 8751.531144129232, 93202.53135509243, 25035.361136176525, 35089.88913404902, 85907.92721099264, 52521.8889269338, 91382.22789247126, 40261.611505075845, 98040.06643308396, 32098.69355082815, 23638.202503751414, 84145.8944004266, 19399.952544431, 52973.298639917586, 12307.22176242741, 48244.64269040446, 87240.90128295505, 66183.72342192344, 24531.535070912658, 20142.97176498736, 26118.133008783927, 91239.42905528663, 47314.9601559802, 19248.65308076281, 74328.01881569452, 44584.66783335822, 49628.86078715627, 25230.795775212777, 89122.58490140612, 63552.43990510893, 96966.01699407207, 95468.81279607421, 41014.265982032084, 72284.84189352518, 52339.29404011265, 91.63152350917247, 93679.13546608729, 70362.48086438405, 60837.7359239183, 61946.19000011844, 50404.60233922046, 46971.237904580776, 52955.12919901651, 95525.26220277001, 12983.237531753644, 84949.56851364198, 90831.71257790166, 16044.114235538376, 67982.13745750657, 86906.41178236163, 47335.489101631676, 40488.93999742036, 77893.75833180374, 71380.18686177877, 49631.59922039181, 37228.804563385085, 2514.813338626998, 77376.30957895255, 59947.187219287145, 27474.547217806034, 7825.244962027722, 70963.37431759063, 6565.497605171944, 17271.373158624716, 75931.63997350291, 61113.06495310752, 48582.60535769338, 19371.074149095035, 86662.67196936622, 66743.51038660212, 26394.405422203672, 50334.18808554355, 85617.90892942126, 62496.51649364002, 82409.84022534144, 50980.716807690515, 24407.80788161996, 38195.88354571094, 93012.81092350793, 95065.01914350108, 75950.22459489202, 2772.14369845723, 15209.34446689749, 85489.06960519157, 78493.70664795396, 35778.16912450944, 24094.318359125522, 38792.55124389459, 413.4244859513081, 83127.73847261313, 3108.627437989453, 2996.9630562968086, 37330.220540161565, 58755.9121941085, 99663.0902040559, 48636.61783014858, 69516.6356850118, 78414.57657283432, 3771.2020249997204, 78748.84369803696]} +{"id": 617, "vector": [52985.16278763966, 89978.85436870312, 91306.47941176711, 48180.24457032963, 88376.01490522355, 76706.04704763532, 40602.634009560235, 60437.85798484062, 54242.86837669477, 74181.79510467217, 51900.32832064374, 10045.010923341335, 138.01803575640426, 83888.77748840873, 76877.40516448411, 58830.02765181215, 52549.111946827776, 60921.39686921596, 28082.371233111237, 40530.15128830453, 19896.196810603094, 14825.80358326796, 56700.50373637382, 29797.05197404261, 93685.89267007612, 95882.99398189811, 68300.20990782327, 81518.50794059769, 85702.47788665336, 58820.81755901889, 14849.873333867481, 65724.32190269239, 13980.715595129566, 65442.385204746555, 74014.72462304728, 28715.18551778497, 50981.311313390455, 83528.45606312435, 53422.6947493348, 6048.748386700309, 63665.26697423508, 1818.1498702337651, 18322.323674824358, 26473.896739847947, 10742.412169889836, 32683.692435107347, 69417.21622917635, 92812.34776242459, 73176.82506806757, 27363.478190019418, 26112.134787305986, 74620.81407647688, 85949.33002954468, 39129.35071389786, 40655.05708607323, 13170.45160563346, 65498.86904179757, 56371.75231182586, 38758.43990564879, 61836.70234397548, 60368.707399891406, 27029.052779564612, 78294.51650123505, 16916.525955608387, 52423.781396832026, 33845.01473526832, 89938.4649872244, 19120.136712655134, 41379.86367966919, 4068.6669097257154, 63003.954478732725, 22873.416151520676, 19207.49117436752, 16114.265613503898, 98282.15413878289, 60948.097824311095, 44213.44162761654, 94092.72116727667, 80481.80410315454, 46706.35993904175, 71808.81120690572, 44277.51635778977, 27282.529862562875, 5985.103826590843, 3649.17422424732, 9531.558780548621, 95417.77823271534, 54048.69499178383, 27885.294197645937, 54899.34401814006, 97205.21832323752, 74745.13030554532, 79092.03865509952, 98334.09986030369, 73437.43413036318, 72184.82485464014, 6089.378249174426, 35404.94796211588, 39076.04460154762, 96298.09929332795, 64904.80058469254, 24416.498314818215, 65018.63051918654, 16377.707794321206, 1404.8093694611907, 33937.065283071664, 3211.9932555864207, 25613.403189526318, 44453.00843906241, 49136.239180378914, 14334.383461244293, 10225.3992125243, 8750.97145534397, 34038.67713093982, 37906.99588359548, 94673.40172864153, 67692.88189010826, 17113.194019318613, 27122.83725974679, 33268.90739533887, 75974.7015210091, 28914.23684923399, 12468.970585281702, 77307.07516956935, 34601.50790457095, 61050.080330682846, 53161.56055663567, 19893.996592971664]} +{"id": 1483, "vector": [70172.29874633947, 36081.07826814273, 51934.18032198864, 54320.17680796864, 76993.09063664306, 246.65602782677487, 23250.268921349314, 8169.658102853583, 43401.9062700309, 94640.622469623, 10942.852798835445, 91489.54366173278, 7675.60561266517, 75678.87436607141, 68642.88713348913, 98573.01277491772, 71002.66856919847, 75741.02703548245, 37388.57028301997, 27974.155036692373, 39175.660401603716, 67213.35511745281, 57243.38786138806, 21550.54998396667, 17955.83850680078, 84380.03441639201, 56348.360938853904, 81551.47748247575, 32408.777630373064, 92558.65218370398, 86959.51668806703, 23025.426074087052, 96490.78973720188, 94561.68703526894, 39693.11444749792, 93179.8881138881, 76064.04472916538, 53917.42959719017, 96774.8604729497, 93186.73924708054, 99505.99658996698, 58195.87602709778, 84210.01232817073, 61136.14866039582, 59524.55179260443, 83116.30682991447, 84522.66600491392, 38452.39792838694, 61424.12466539062, 26040.857787451678, 14039.282601517933, 99147.11128631962, 49250.413168481464, 77266.90098323338, 25623.639197812685, 11287.790015529587, 4639.487124590436, 28805.473296061646, 18599.249264026097, 50199.893516826654, 17117.88985527761, 7358.227110280469, 64038.20904765161, 64525.67321880662, 95686.77762462225, 98481.93106569529, 8391.478811987685, 64663.32053390418, 38805.38180941372, 48914.35836409077, 57407.94829257633, 8023.513808598759, 59814.56718839177, 9667.485700562462, 30429.87240982443, 93463.03606362223, 72046.0999079847, 32434.2672890808, 30096.648860132147, 99954.09264483345, 504.62725048857624, 83201.86586268686, 45487.038558503154, 34860.553223059884, 22721.43740393404, 31073.486715051367, 99464.47164510314, 62671.81346311213, 66870.91732517933, 21970.278914024733, 87614.91461144567, 11369.440564073475, 87210.37047992602, 10700.213532752889, 77968.8887964508, 85040.43712574229, 78875.04808969137, 96428.60159946267, 47113.31572912954, 52468.31780130179, 8804.796803430625, 24612.143146817787, 63695.70564979314, 9817.959927433118, 75527.15844004392, 16120.488570965374, 76859.24537514313, 983.241281516578, 29994.68123895509, 37427.879752328285, 69825.02560747194, 23307.867462073107, 23033.76629783643, 21052.329203031084, 43952.89933623552, 95623.5975978725, 90207.76630536345, 98471.3232108087, 27675.007384468365, 37360.96494134351, 50072.05166208847, 17751.34038722902, 27276.617298678364, 71551.36741817341, 36796.52572813261, 958.3027203052375, 66747.2640183814, 94047.7635681214]} +{"id": 624, "vector": [81565.23281049496, 37146.278079866635, 44828.64411374184, 27832.52959269734, 7401.429058402054, 75164.28126075759, 42940.175588264065, 36123.84221371946, 25737.061246706173, 83734.55052002687, 29980.83240646827, 62394.39899215481, 71707.64372295565, 11667.27818188561, 53993.97714522261, 29934.357588610794, 80157.90568789243, 7045.556074195935, 35526.261158523375, 76860.21765308156, 37642.35863087092, 47455.75033266803, 11519.4074604133, 35473.28928964628, 8602.649407574814, 95965.55764627627, 98593.62906140376, 13717.384358279849, 56694.97545690311, 29885.13662153558, 48233.860288016025, 12181.343446381188, 24490.867585514687, 55607.96860503152, 25340.695763972297, 31910.826179386742, 86143.27145092143, 44092.48538672132, 3830.6041479577834, 34764.7598783117, 68152.54276387287, 86454.81757442113, 60181.11019860387, 17826.282347709064, 8939.608310922575, 29551.4383942754, 12405.36828495491, 28549.93010708392, 8360.276464280558, 95704.59168166952, 48783.20027790661, 14663.30772613571, 17606.548951161836, 22405.268704929236, 81919.28044015277, 65597.8952157946, 45701.76328083405, 7831.478693323368, 46066.06459567899, 42369.52753987662, 68929.18764708388, 85417.13728646412, 50479.0425327641, 64252.053868768635, 49511.757871459326, 53606.83156510986, 80183.07031525088, 28327.702329907035, 77345.37339240128, 91584.36914091201, 65884.84014975607, 23942.7720913452, 44042.445367596185, 34236.93785216685, 4250.038484411866, 25993.435039474243, 14390.540195702317, 81690.2412942469, 35627.159586335256, 16543.21579723006, 2221.4970200714724, 41068.378603276964, 76127.51428150141, 44870.81286082748, 43095.17040156812, 16810.276456743923, 54319.478480258935, 6616.22194960827, 99975.74715963549, 29618.99449629877, 55865.26097694025, 13303.5087625906, 10446.778488311915, 9011.821593017145, 28063.67872370259, 28132.49940847077, 95037.57579846565, 51702.40439005882, 79813.37505611907, 83961.32207356473, 23374.86702294125, 9868.820270523915, 50240.430552022044, 48166.75600603257, 94200.31779457486, 79495.05273680834, 46032.67105715296, 64473.921640621644, 38208.63740592313, 4893.738811857474, 49628.19300606267, 44977.946354052336, 44449.57981818236, 8694.60900313972, 63685.610866448915, 5822.378288468144, 47477.403251186835, 69608.0233414944, 83278.52784393987, 67835.7725345831, 42312.223598747456, 70381.6653065863, 32551.94087806592, 22718.322488312893, 89856.38536741532, 61414.422719930786, 36668.099617152526, 67159.66760980745]} +{"id": 928, "vector": [71499.24237792172, 47543.36645213494, 99716.0091500911, 93387.77042767675, 38524.67236944068, 16539.562289952504, 85585.08789183668, 42561.118639477514, 3347.837290915512, 34164.23502839385, 25173.11609920857, 73938.86193274181, 30190.294976701327, 31747.30383488392, 94633.44913620134, 83253.46230292789, 56424.275840917646, 65365.61132192127, 34229.343625491194, 576.529105960466, 41470.53304479883, 45870.899059163385, 9117.254774076178, 54806.69623018877, 26615.710306271634, 81360.35824665822, 83600.11706188622, 83600.32316874136, 21845.714988810338, 76009.3585916663, 22125.71203150715, 42104.841793924534, 47535.63270086979, 54564.75966624431, 15806.537502857487, 56137.661593499455, 80416.21809724014, 61300.53484818155, 6623.510344678219, 59892.95759324018, 95405.28097265106, 7431.107066068698, 65090.75290688612, 14010.616574550328, 1282.2373336433434, 56364.42023511089, 47372.18202044215, 99588.49551515937, 4838.708318653806, 91873.76021204438, 93738.9313316393, 28616.695265924318, 25725.079003690633, 81769.00805283785, 74436.21764309025, 85240.73130790087, 55520.45260956797, 82337.20900046024, 38384.731651129354, 8847.01434957217, 4094.5383187459215, 10876.88632405569, 90830.76655895644, 75210.1928281099, 52693.57682052181, 18689.819920283357, 26397.28669646514, 52146.28717576571, 70411.53644277235, 48031.715522265884, 62037.73723548682, 89109.01745669142, 16087.575153137057, 49757.179752209704, 19552.105225002626, 82062.84596816219, 34540.84527846829, 5126.189490691935, 76092.8177041351, 12781.638216982405, 17303.453333162655, 99184.54026959516, 49970.4921776043, 54067.34577114791, 12436.74882600021, 41075.23935079161, 19807.58845576267, 89597.4127543251, 44133.771443015125, 16977.16810321749, 90710.117149843, 55151.41730281789, 52908.20346152002, 35527.2815300311, 5708.884075147546, 50738.760486484236, 63391.04712226606, 94193.1260442116, 5340.890624122796, 15967.650598882166, 95650.50590034865, 16848.467401956368, 40904.802431373944, 31115.956874597083, 84971.45010985367, 3950.8197326647787, 68382.8008574779, 4300.880464429113, 76150.53634762246, 56730.84644272912, 63204.4594934743, 70191.35508755394, 26670.72167606368, 35230.396181537006, 10844.186621758688, 50392.44939853797, 85947.13120231702, 95474.66475531037, 11271.795495921822, 7046.567179047103, 30251.767276776744, 82700.29932100476, 75522.52742456486, 12453.65462260717, 38100.61693079849, 66710.97114207782, 35354.81274814887, 48443.72746259266]} +{"id": 778, "vector": [31448.634384060537, 56358.01095545216, 81528.52749108186, 54359.27818393513, 29035.55674292324, 74888.11439266086, 22267.922723634158, 96146.62357650707, 45406.32965968037, 19133.307203256478, 60228.87066816006, 43897.3598202357, 28343.15683465254, 89164.3745042483, 57858.90594773393, 78590.2937769949, 62683.20292098985, 13542.086765268636, 74125.25276897452, 33050.9313773122, 73624.31591278392, 27609.98267612861, 5857.868828927293, 65213.65008122384, 4281.975298037988, 51886.253115298096, 54841.30955001333, 29159.330601909838, 3717.2320242847, 36481.450128527926, 34210.422478155546, 24679.539143342554, 51047.25221000634, 29247.38600554575, 79226.86799466221, 91877.76212155055, 73830.49892132264, 8456.480305096802, 14484.157949129307, 97238.5826941392, 67875.42029792698, 14120.603395683273, 56349.20167642982, 84133.6203514679, 30178.397602908168, 709.1613029821864, 96066.10543183661, 72896.14946494723, 42920.41807341682, 21830.28067256022, 78087.91459329493, 4394.089010558133, 44238.80385811988, 34457.26790052619, 91106.74716381007, 65744.59002952182, 31688.953329040814, 44880.582597719666, 91676.10716543956, 10016.650938948058, 37441.20439565471, 43155.44106585137, 58278.68174363497, 87709.60696191032, 85922.22295405637, 24064.11331869629, 40022.22012915354, 51244.77951776298, 11010.368651566816, 26906.84583038703, 98402.88201357731, 69190.32300214189, 71599.8905526467, 91620.53284120694, 29195.37565895536, 74785.50700469427, 62312.30432273273, 36615.23173345355, 57419.89960986037, 76645.05244036615, 27398.663861633355, 98118.64591367003, 36160.27333094779, 13577.049374819495, 11894.069491659542, 96542.03794233425, 31627.26062235971, 34992.95220949866, 81262.20724774014, 39262.68196268965, 60434.19943750609, 61219.992349067965, 57622.80061909373, 63542.01703124081, 73993.79225159234, 67568.19388468297, 28804.182430940175, 30871.29044155462, 41389.453157821445, 7532.804208895638, 92029.98297162635, 15633.438932043486, 93697.90621610828, 5440.347277753277, 99393.37304640633, 99973.8178218898, 59204.325653139225, 61834.93554866284, 45553.42767967795, 39483.53521789363, 29644.554672061862, 83201.94586428351, 61586.95309965803, 25143.308670121354, 55538.76730182597, 45951.94504258953, 86445.74837918703, 93262.07419212931, 5868.4375957730235, 50282.098762967405, 49620.918080871365, 48684.49880694337, 83670.95589025237, 82674.77132836434, 53930.72261513665, 54532.517479443755, 14609.656753647949, 54236.725427873076]} +{"id": 1387, "vector": [7602.338699972766, 57546.862913871264, 59188.62743701078, 97132.496328588, 1183.6519578971693, 58216.15515257793, 58998.917485884005, 45060.36830372136, 25359.944481137132, 85200.03051374803, 71593.65535892456, 31070.380814716304, 58173.51181305625, 14001.62629816597, 57983.05906179203, 71527.46847381101, 4016.993376769984, 94979.05349139018, 60190.84342486637, 76410.55479859002, 97737.42564088535, 15160.070088672639, 25284.39430831836, 50260.114219951836, 3729.135393210814, 22097.066711895764, 97299.41100700678, 9118.666385164199, 3592.3217487877414, 59729.272163551264, 38129.98450177515, 22380.442868160346, 62104.79753980353, 54445.508220121344, 47534.081397551905, 89018.17724812195, 91570.72639320533, 77931.08930468111, 54371.020436188825, 86136.97053909517, 5995.877584158127, 40937.42809318181, 27966.451171862638, 28724.599993556443, 64784.65294442901, 94590.78459637372, 14911.826467343104, 67533.9687729156, 31704.73762608711, 94804.5839758696, 41883.830179006894, 44056.44013587768, 16992.932261325932, 77638.42598343488, 73621.46988974372, 52866.014397739644, 16741.046745779007, 21951.37090526239, 34526.06914485337, 29529.498132378907, 31477.461218289372, 55215.32987026089, 95060.80826338504, 42360.14609356068, 92005.87430737144, 92947.93258191725, 83958.80648677234, 34516.52444167077, 67070.15355630762, 56705.056734306614, 80081.42876883919, 14296.8379723589, 22756.781177237717, 65426.911684006125, 47850.204528352224, 76119.75590467725, 56915.698550050394, 18209.5453374267, 85691.55288469262, 49664.32690653215, 40466.5450219442, 72467.13073125792, 20279.442015527347, 11630.117023456376, 72667.48657321176, 20478.428101693426, 11205.019088713376, 31632.284255601317, 84806.3465820144, 83812.64173152515, 48867.334216439274, 69810.56691491415, 1044.0049221329796, 18452.21550243554, 10893.82804082557, 49450.81823464261, 42924.01196405555, 77265.88410718062, 92564.72656546012, 42627.62635847001, 28481.64240312504, 7906.253819538611, 54120.63186388222, 70964.8017511986, 86365.74585552409, 79359.9465818674, 45175.83189708799, 16222.253169032385, 15456.44831311117, 54801.431142657006, 16653.047236528295, 48924.99040627224, 69553.53264240254, 21235.168274905624, 82102.12746976562, 21856.240377369275, 86326.52425889349, 82527.798297413, 71714.31983979861, 70602.45569688253, 45159.968844048846, 84341.04934957183, 72194.73605382553, 97732.6091457486, 17393.096635923168, 42608.0102796527, 18015.505055663038, 51489.00572428523]} +{"id": 851, "vector": [97169.0485204218, 79503.83485344241, 39430.20036985102, 30229.528302356925, 669.4370574929164, 27153.604167439193, 12322.90703024227, 70672.15613592116, 56744.97586835265, 32559.27021214954, 97576.0259795931, 13560.791982874443, 39643.131843306335, 36848.21205654789, 93481.44516827402, 66334.66739588018, 23825.706489164222, 84077.44187087197, 51504.729200460475, 78475.46138772146, 62191.125448821775, 75713.20555849206, 14298.809654913446, 63662.28511668184, 33258.79442794185, 14768.13420206523, 82791.15096603174, 80236.14586340086, 71002.75785819977, 52152.48305921989, 25218.66713307368, 1563.4461130094924, 6464.189088491678, 20122.324580940996, 25770.77811420856, 4612.327345520129, 48529.61179712687, 75271.5565554975, 83556.25747158377, 96707.69673623909, 43299.61051371949, 64171.38732008283, 42752.73665399875, 24569.840008301835, 50221.48188975124, 73490.62152539539, 8392.9272842439, 88930.03652654315, 41965.4499338582, 50033.518706975374, 91942.97758352757, 91957.10099719361, 39179.25523880025, 3984.913633230869, 98873.77998128849, 44549.796908853634, 49894.35152165973, 69510.5414740374, 79747.00417578085, 8975.834706799646, 28493.45222539318, 76766.44561005068, 28539.216845808645, 5841.044235650583, 95802.6398324511, 46544.10388664727, 10972.98410747608, 7792.484286325152, 3679.7351742126507, 92969.18135339947, 63310.22652692108, 11720.378777196827, 30556.237463050362, 6862.891816536687, 62788.51792257947, 97486.02459601415, 49670.625999104726, 8528.515909388223, 52420.33316475828, 22857.674259233696, 67800.92424484498, 54126.19551223842, 63648.02037630193, 79934.68059421523, 27367.963395029237, 24832.73157995467, 17570.13583953022, 68282.32612899668, 69699.60440498994, 6815.6459025594195, 78920.35691371506, 50233.31215792229, 64887.69312175433, 30746.198426857496, 43661.92787614262, 64200.298186382155, 83625.49322989708, 20495.529138809732, 32160.32203270509, 99098.3801903902, 71828.68740378242, 25792.285025019602, 79058.70856671035, 1389.9386233068967, 89882.85873786878, 64993.83447501932, 1191.6586943060859, 8174.994610546515, 29577.784802811446, 1068.8617316209736, 79531.00163247343, 22539.270303052883, 39353.23027004994, 19767.406150391464, 53964.20253958151, 68675.3025667295, 47103.80325370804, 7601.991590606872, 62459.78487598682, 40996.57793210598, 76627.99356635672, 29641.258019198423, 53491.71180821828, 68002.48096452173, 44261.998462869145, 78938.2049390134, 58737.96530159071, 446.3476388374632]} +{"id": 1341, "vector": [29155.039824468986, 84820.0852496271, 82609.32393060558, 56670.46206360256, 95168.0154795555, 22628.708350943838, 54478.512112346245, 91033.42930335604, 99039.61686370111, 69995.57826722451, 50494.85155614612, 35202.53585706264, 44868.288870655226, 88433.84052041112, 65686.8615301902, 88548.79799848175, 71107.96963009323, 72244.97650495873, 3977.0570251639547, 22795.08232897246, 68044.16305414538, 26185.568894044518, 79097.60619848609, 97002.09070882326, 35491.45768367056, 48874.096482710214, 25115.632124172727, 58209.80594206174, 55700.40258456425, 24484.85812418103, 56420.56746505759, 631.3083913843443, 45392.8442534966, 73519.72615403557, 77713.66091512064, 4419.996543893012, 14061.402597846905, 67755.99030955053, 34382.59145916943, 99567.17611826657, 2301.952232229354, 61261.073497947604, 68062.51239242795, 79286.3578883213, 37525.502743111414, 82639.82265897062, 5269.961707083681, 97492.73889019412, 73859.4333066196, 78306.82584356004, 6343.3406097033185, 19121.11416985249, 607.8344561255444, 33278.78502850739, 15049.626452220722, 85530.3081986768, 7634.535337234372, 58102.358017829945, 42983.32437409624, 6214.129225175436, 71731.3554789517, 53356.1134711256, 60383.874441766064, 22538.85528342413, 29741.483822883496, 19665.95637174715, 31488.76064395042, 92750.14819757121, 20144.202142474154, 21303.43207632257, 54465.532769521116, 11802.922117614467, 89878.86513584583, 36152.542502733144, 12870.406726746198, 59147.59915738008, 32604.757072724045, 59089.182527889294, 54656.822126201885, 85297.86516267047, 14979.020267714115, 25908.230547950207, 11764.616817225015, 64060.56007165361, 72169.71684773051, 84041.70173305979, 70479.52270869754, 3780.8739257211732, 14714.396535129094, 9851.66786660996, 9330.374043004542, 40652.1800690508, 64758.00236265562, 16763.914496551024, 56862.53664285714, 3797.0514374390473, 71492.30940125504, 33538.05190188868, 75843.24758552846, 51389.343366002846, 94540.62584677071, 46875.618244763005, 42720.428852731115, 49995.04856967209, 22081.20802006619, 39926.02236191117, 9768.633397380177, 79743.49911067492, 58826.1172781928, 4174.301807077319, 57635.75764605335, 1565.3703863527492, 73502.22988560889, 31053.055394341678, 26889.826140452933, 75184.24619656998, 47827.68905731075, 70845.73017479596, 35919.84094551327, 32686.827514172634, 95345.84513529015, 64581.16695449179, 9293.563686828376, 84996.7127158742, 45473.4812670859, 85399.88457925274, 96238.70235464006, 15755.716625045336]} +{"id": 698, "vector": [51349.966903760134, 74852.00888477324, 59612.80854819333, 31153.133614998875, 99562.57999917271, 66409.47942497584, 91170.99023103357, 48469.86345844963, 45213.39589655211, 22147.799729998995, 94912.37153900933, 87732.41699518482, 18091.466309233052, 21411.770976860844, 25453.527517188457, 60674.74320946051, 41672.27319487575, 7737.45798585378, 36168.31174771292, 80685.77180715186, 57020.261351469904, 78805.9552168608, 81588.82804168784, 37527.025419717975, 82884.10565939253, 65688.4044584611, 67446.79192753076, 61411.87531847801, 25534.81640708721, 26957.131407799785, 77218.65732909276, 76973.44570975495, 62159.7212414922, 82145.14842001843, 84508.29031833484, 98079.2785517784, 27744.25376111509, 12871.473652090415, 54704.019266477466, 51547.038130283, 20838.953486305745, 12293.211421424245, 95683.30253439896, 7526.406792342411, 69567.66516922292, 63430.668128261656, 60186.40606664515, 31176.253620859505, 97546.61722190029, 85614.41446319486, 62453.508904759045, 83288.89552424088, 41461.36683174645, 70294.67866232888, 90289.29130882956, 90264.76851121573, 10121.553875681433, 56672.71168611568, 19850.653517578132, 9502.12386516166, 35034.01550934717, 69116.52362412374, 2251.7600608704115, 88431.87403987907, 35177.49710764866, 43908.58245625409, 73885.21459124764, 35988.637748624686, 30257.760000691724, 86075.83422211489, 76988.88993269236, 1985.871585393728, 66511.58906460187, 11162.89022931617, 3738.301362353358, 97285.53761224277, 6048.948708284518, 55073.25196011736, 52609.45361184203, 1851.9550988876254, 14421.021763444352, 75411.8180782942, 33105.054101710506, 61860.67817921052, 51482.09427802995, 71250.13689875754, 85325.71776338964, 96246.31987529148, 84679.75997890515, 92819.20860051927, 58889.83988428734, 12000.761419553419, 75567.69012918587, 64403.77166244645, 27770.017423922298, 86845.88742892553, 607.9437622793615, 90083.5361586104, 35836.56574160018, 8246.022995636782, 31240.493739593712, 20786.593956010023, 39383.55507560539, 81086.61511265214, 96084.24812961239, 90994.04788579905, 44709.236732299265, 94742.86110938236, 98377.11734279693, 37294.273267599376, 38467.50180309922, 98502.7369098073, 56262.89829097886, 44131.12808111656, 87378.54403894888, 96536.54187564252, 52049.78285823804, 64223.37931274949, 63963.67979328612, 47721.24079223185, 54457.90854536215, 32198.662752937613, 18027.221324667797, 86893.27421649877, 68896.37396215876, 68207.43275686077, 75735.87734222252, 46532.01065204414]} +{"id": 701, "vector": [93646.5066557068, 64334.76444036984, 82500.45930584663, 44922.080511526205, 74799.99213697127, 78585.02459025032, 67825.42937814807, 78818.74839555398, 67079.64678301344, 76639.48042931598, 61147.99848180379, 54813.99755276861, 68062.23391793101, 69146.54916017877, 18221.765880815932, 37871.095836207445, 44994.529087585164, 87762.94886249954, 24474.71519830652, 78816.45312669902, 70424.98236287774, 54675.30598250906, 28087.56358779174, 79203.87496759238, 8697.26122677883, 688.1370893288108, 52172.61655319807, 52121.40363001454, 84533.69179606249, 33399.91926327737, 37701.6855407531, 66818.03487749667, 22758.068612135685, 39225.22132454598, 1048.8759436474204, 72751.24882836966, 82311.64198894723, 97988.34772700761, 80135.56174161103, 44026.61731747654, 31283.223413227814, 82589.71459315995, 94931.77022861104, 67104.00621470512, 83861.27126187347, 51069.62445879415, 74473.81380889009, 58184.52635606961, 80812.12471855758, 17026.720143566632, 33652.955864262825, 68029.61436028715, 67433.8125579508, 57993.21478270788, 77602.90744861601, 93749.01209441836, 26493.392684069062, 17282.971553680403, 31147.407435660512, 38932.46577672532, 26872.84830334903, 99201.6282100637, 58112.59367422672, 41107.2251148109, 57630.22392444336, 38267.0604975346, 57060.15974851277, 69062.98509326782, 60427.84383407513, 20580.75830592322, 22250.551594564826, 61011.87530491206, 7676.138786708786, 70671.78057160287, 24355.419860439964, 8734.93973243653, 44688.9667370555, 108.88692917547793, 42135.261090777574, 93223.31326568956, 87977.97982589215, 47972.7399430296, 84953.12740175915, 77457.46498993957, 83774.5507598681, 92966.21418601293, 40455.341296567494, 75046.44768730528, 8222.447183393855, 28318.694024615575, 33652.39899944177, 20185.21370140014, 40304.41271742727, 32036.33056349121, 35145.613997546076, 48565.23123930174, 51869.5972099569, 18940.579558957972, 522.4465822177838, 70448.992930085, 9921.125812537002, 65666.19337906038, 69716.4335419115, 52022.0281255292, 64900.02468441986, 98014.68640222638, 55449.201165094564, 61677.84259030228, 14270.574552605276, 83502.04730599206, 7517.755532822079, 10185.075207185268, 70888.73073585932, 76612.70316672187, 67636.86264343502, 79157.75525537066, 21837.538422081838, 44539.16213477391, 56048.272857527045, 24778.840940475966, 94098.95017449597, 9705.448851264031, 95240.04469085985, 45902.494981624906, 51331.71518256093, 4203.287199231565, 16371.331147222112, 41914.42389154196]} +{"id": 611, "vector": [84882.44409911725, 76907.33132412336, 78804.45355924289, 10441.009598189976, 72318.8610209454, 42072.58239038071, 45184.82265571088, 79633.60070255757, 89309.13443828264, 29131.991640303157, 58285.15518744041, 94758.4808829579, 18443.489904476428, 58746.21097371261, 50474.457358751315, 91315.55526580353, 29769.313127079477, 77869.27125257275, 30543.14852420802, 37825.27208547339, 63519.16749828524, 417.0813993206734, 81331.79294786774, 55723.00128065081, 83694.33066927602, 50707.479228798315, 44314.141313042135, 29565.521342611435, 12761.158297179176, 97538.8069446551, 53113.13723972615, 90637.34324039178, 80514.411391644, 82789.17184771982, 59059.15550651112, 39176.78886252431, 62390.995417881066, 96391.84433543542, 60352.23092441447, 92252.08579451808, 39921.2548512496, 83007.03992223997, 18026.983408052743, 58186.7639274043, 98419.4378981834, 63805.81972483852, 4816.674217799555, 47467.51792761344, 40927.775826629186, 53214.927694240476, 4662.826709421431, 85066.39554555863, 35048.31704362133, 8878.274607390767, 71089.88178039482, 27734.107581529253, 90683.28110515857, 65292.041150419325, 49060.28446915087, 67201.86517731009, 29265.89727822867, 89816.85806649123, 71371.33937353524, 79434.99493383216, 94403.3944144226, 76526.57903807516, 90052.3238147783, 83675.18726217425, 38784.69559059938, 34569.307494413646, 88651.30516612291, 8996.131251225226, 6275.485583272622, 84762.1418927919, 76650.8354950964, 72921.49080366177, 3877.3519354748933, 56970.25139869833, 3963.911092303718, 40331.51690965064, 4869.258657051622, 5011.771700191847, 64216.634727168086, 34802.233463950346, 54841.86445601452, 43648.583823546505, 13112.294746059593, 94997.3564726209, 39856.13933978489, 31858.88525731193, 75004.91668761619, 1905.8887871568952, 47216.58687235798, 30517.10943833579, 36382.79476960772, 77250.47820373617, 12679.89349355202, 80214.4128822345, 14313.967875113898, 7385.224427549786, 26260.468948340764, 26718.92318667234, 60995.92756552366, 85897.66074894642, 90922.63148274264, 12351.768567373178, 93064.38483296972, 89639.22116200863, 1762.221739860026, 47262.7768603605, 43590.43163311047, 4988.964238015503, 95802.31683953306, 79620.00514778981, 62742.03811477501, 34853.173590450526, 22638.512885979977, 65638.95781108989, 11416.571797996967, 46751.411236736305, 42957.52455338856, 79847.35055118048, 58789.20746742871, 52954.10436246979, 66828.80390275009, 49077.99327236378, 79599.52513446644, 40236.2538597718]} +{"id": 1966, "vector": [81747.65561526743, 91974.3833960357, 59115.19091164066, 28008.700752046643, 79663.62815597162, 76158.85518538547, 10375.36578746543, 31769.121503973198, 97038.15691241423, 16707.71901808954, 4691.218524385255, 94097.78053576499, 81448.86191490512, 57569.3743072734, 22614.843648367343, 80649.72629938714, 68159.07711917417, 1784.9621544096194, 79741.64246935373, 12436.233127436624, 35526.432101026396, 83797.27588166998, 5013.754493897315, 29519.57865886262, 61938.85653219825, 53964.25666455057, 81060.80894847837, 8192.409333664575, 27864.767737490693, 40720.09668923976, 86185.82042172436, 14733.392988235772, 65953.68662672036, 1589.4776028096746, 65515.104361671394, 48897.17402150634, 62415.14443025553, 70184.79244743638, 26001.17210431109, 69033.52035592475, 85596.31587660544, 99976.1456224039, 6420.5837472819385, 60527.56718506773, 83265.0396775291, 78896.27540704605, 83186.52250981846, 6207.044737482382, 80324.13071623637, 45411.83227474641, 58239.98985201674, 58529.36400122196, 23485.92604617964, 63117.400156235395, 42222.90454252915, 89980.14835708698, 81615.28221261124, 57528.59831422632, 24838.7483706621, 28468.540041818578, 75594.04385229082, 2308.123302081111, 31157.71144622317, 46127.939968792176, 27654.32062731944, 25074.832357301722, 12610.42539683267, 27742.129667532023, 77012.0370211825, 66003.76711636405, 35633.81542375442, 13201.47993259091, 42073.256482867815, 49722.26360541857, 96263.9273882431, 26603.529229148724, 3194.6496860975726, 68170.82722340798, 50928.43900300333, 52562.383683952175, 90572.822471093, 15459.0012301422, 59261.07864573833, 58887.65478095953, 45184.85039995812, 80970.10262851283, 28571.505910673222, 32021.090941598784, 34070.41015740488, 90043.15409808533, 6591.346833386702, 94207.2451553462, 82191.30074633031, 73839.26734026583, 80136.15849876494, 78569.72510199939, 7279.658037573733, 75284.00066069003, 9650.065902968952, 71600.68784529965, 80792.83720996279, 54240.61882263639, 44331.291525760505, 33426.9806889344, 48093.46516147038, 4190.513301434317, 32991.476414373734, 95744.85553097309, 80065.47183346507, 47512.85740597918, 6006.906893722419, 86845.52487565462, 90060.82025133073, 80819.0957626552, 86288.30487079451, 18169.485676842934, 6465.862278716095, 67329.76895139934, 75071.21210241153, 41052.92159097853, 21736.927812252492, 65134.87099349039, 59583.09391201171, 43721.70010390327, 65619.16092627626, 73664.68177762086, 50086.843958872574, 83828.08497840933]} +{"id": 1753, "vector": [18508.14213639943, 23842.705241595886, 91433.71422196174, 77186.84567432782, 66811.48067261223, 9931.1566420752, 93118.3668834668, 86646.85506281498, 78222.78248235528, 95536.48305049773, 79642.91348854275, 67584.70813221243, 4089.412585680152, 70099.91639174573, 91560.25968200614, 26542.277991984964, 60622.35293044124, 32139.420799077732, 80234.19558737906, 28949.64197602611, 60353.96130120909, 66946.48182463524, 36180.788477255024, 32835.240558679834, 82641.72999071049, 46108.031562278986, 35651.56511226546, 43901.68229805367, 48719.13286050764, 74022.4948024651, 26583.35202934561, 95669.12747078972, 40667.147795122415, 21367.004291253754, 95596.880362562, 84563.31776116166, 96302.48828818102, 17259.911064511834, 39789.15929846001, 52062.28689652549, 70331.28075076503, 87607.78822693355, 41054.43064513471, 52012.89545429405, 17757.587521326044, 94801.94426214762, 15528.76942885455, 72468.85887829965, 1029.5106702054356, 15017.40106581394, 22847.78825555833, 30820.659466100275, 75144.88071832921, 40753.68978703332, 65105.235212526255, 85630.66532902495, 28893.261238561296, 75600.47950870363, 17238.59101737749, 84229.03510135399, 3694.3827830636765, 79921.0055967039, 5808.845643019423, 35547.102327290195, 23031.092289862954, 4268.78656570977, 88457.01681795018, 9991.448703948736, 17705.694041136954, 96574.61158538066, 12911.613222210482, 70918.90352575669, 26141.113383634063, 15281.678298921275, 8709.566688989045, 63101.417752228095, 62267.62272371205, 56457.85994821747, 34313.288878532156, 98305.97993332798, 17477.010937626692, 65719.12254138108, 6672.653241780325, 20840.734913491655, 5242.794391524874, 70221.09989820115, 24280.18438275651, 87738.48574524416, 43336.35888393949, 34958.39738795373, 58309.19325798649, 79381.30150191365, 10548.584569866836, 4770.186718091507, 2276.7508362332655, 79801.01164035777, 84072.65312413404, 13116.107167553615, 90048.15820251452, 47913.07792201622, 42584.4281349038, 2975.092579410965, 50904.599235447764, 79066.02690640162, 64036.66897437601, 63014.7209491565, 5583.421323416859, 79815.46303827423, 84668.00106555934, 213.35841104760388, 83736.2241420029, 8176.48951232699, 59838.03644978595, 74619.42524625661, 52110.558263273844, 74578.01967566481, 99697.85095730191, 21786.72211346525, 8389.15700570788, 71808.92232392632, 75280.00867761765, 53964.70775587885, 95397.35440779474, 11290.464498201092, 20870.859986340784, 53925.484666713484, 96224.47676075509, 31260.50164974995]} +{"id": 868, "vector": [58651.85375393109, 56575.2062527634, 87535.59383505784, 72298.95898545183, 66707.30828670326, 55484.29592073708, 29822.451911127613, 2182.2302926419848, 69335.40283051097, 64064.287986690375, 51248.16873009149, 78909.9634187406, 80612.16333804476, 9052.01431362661, 16549.58447147079, 19728.636983267123, 1632.522484063992, 32105.116480476758, 87344.24624506508, 93219.75292281572, 19026.339365353928, 34670.17925555056, 6479.056647001302, 43975.798527597966, 76250.54803395635, 62976.99921599132, 54461.067302029755, 82753.00515914605, 54465.52706963814, 53495.83872526662, 24321.818913526215, 26750.876386226762, 20335.97080494327, 23947.383378565413, 7643.724433298893, 51590.16384907601, 63088.19918149845, 24902.565185172785, 60890.9660572714, 34033.48021645618, 49882.5407881005, 54312.209434598815, 6500.608081945014, 15004.882984915646, 79730.6688766232, 36274.47546817073, 49852.13702813826, 66471.74144712939, 46844.47709033761, 28103.657453344546, 62506.85952052107, 34396.790978764206, 33880.696266865685, 6295.766252199708, 39850.537161177024, 8348.887440978404, 98096.38451376201, 39583.913756460264, 36079.87897638981, 7110.581370615021, 51970.43780642674, 89200.70253107995, 54534.29477996885, 13024.065060896917, 51882.105934816325, 31775.395624120785, 64543.894616646656, 84551.18278667127, 65866.15757733422, 41189.95723690894, 77731.79907041021, 68145.4861621404, 58961.177279238276, 71864.70774401797, 29551.801466735793, 46652.65452267445, 24724.441947692212, 52906.35763676091, 79464.72930895525, 77448.99916232527, 48071.449501310104, 70154.82440938223, 9693.728391914268, 78204.65452175555, 9603.66505342226, 44450.556638693495, 83103.24090683933, 6632.26928288172, 14344.26017977749, 47861.74032564512, 34678.2829995669, 8288.259945195075, 32910.0475973883, 24999.329429659258, 73522.80515557237, 97677.1230492847, 48453.051683025325, 41376.67626237364, 28508.193989272324, 89883.22783327111, 73716.3482363365, 18277.166956705594, 50110.00234413697, 5635.493421345394, 15775.085513179754, 38093.001231051436, 9618.177896057166, 95628.35222231912, 44110.29862803618, 60210.87238071354, 19989.003459305477, 69559.05270125275, 25840.137813535803, 89238.50518271413, 67459.27583027996, 46139.512328343655, 69057.17751781245, 18904.382889341498, 44686.18082026643, 64218.66095536796, 34640.35225464802, 65747.34742487958, 55643.788433868765, 39318.41330993817, 3722.23714882145, 40061.483430386725, 93472.33236291651, 96756.79181447302]} +{"id": 762, "vector": [369.40168366420556, 4554.440785034597, 28507.65177113109, 97172.58997603196, 97316.55197387225, 5765.859321912226, 78988.22994192642, 80213.36874003969, 27281.68622170338, 10948.57395823874, 84901.98153908533, 4238.784311658228, 67617.1061107939, 5570.103642999457, 46430.06466182246, 82756.94584858553, 48707.04247717068, 23187.166892129586, 38823.03783899901, 85914.55636324083, 7522.182554437051, 65656.83099051853, 3359.2091503063857, 1989.1842561667095, 49445.98781414359, 46413.23802992465, 72952.88442283476, 13675.176631005437, 37913.97018742964, 63937.761539851344, 57923.349763914935, 33811.949052965654, 45536.4361603707, 72064.53443667927, 71166.95654667294, 18424.893597052327, 98635.07037844627, 87422.75220615037, 64840.37952177819, 88772.591417089, 5593.312105848958, 93680.24494455704, 90329.02365792736, 9712.33025740409, 78301.72577213151, 37427.99718284739, 89837.95201669475, 48799.04296275731, 15878.965439983427, 55190.35893308173, 31231.970465504754, 13938.905927318978, 20592.701555089297, 19440.366308151068, 43508.587321625455, 74866.30899455318, 2487.396710403855, 47246.14185119801, 99629.47424505716, 57709.307406890395, 31147.837294407887, 58084.05509418827, 14207.15189727173, 34465.90906926631, 34825.037253833754, 7937.218039446326, 53068.29539287914, 13065.279145444041, 11715.377551710182, 22187.70286570747, 52232.726292263476, 4062.8315563743354, 5589.630566394443, 46657.834332573315, 47245.69922268542, 59262.29785843306, 73584.41055639891, 50048.52274619577, 75089.21141639851, 58236.06349073751, 6024.4151273255, 60696.80965784636, 51600.16853792695, 33130.90217801479, 17860.52292686341, 98880.20957807961, 56378.866619439286, 67065.23211580078, 4009.3066636512685, 47286.552408132666, 43233.23907922194, 36075.49966111997, 18066.504375676184, 43216.40799350716, 39099.01949922803, 89936.27946640938, 84224.90055306997, 88237.36891241555, 16255.292439368508, 69954.70347050368, 93292.08457000286, 84968.55556418137, 62080.10539291642, 51906.88350479981, 90071.24600523853, 79698.10784189812, 23273.19324420205, 44614.37405431927, 17072.10597232599, 17662.479577199618, 26624.77906625792, 19739.107352648065, 68628.01192458138, 23434.7703868968, 92558.43713483462, 98790.18109486802, 22211.915082545973, 83476.16897774125, 31015.31804836356, 75889.72515615907, 41459.768614949535, 46612.90919518691, 33794.450119050765, 3901.870253764683, 84319.6558371999, 45792.874127059324, 23622.978590710176, 84643.53561613393]} +{"id": 280, "vector": [96350.4867757109, 71894.59402796815, 87647.79678547711, 30634.77833462578, 90887.25798041606, 61678.577031865534, 50583.33987241435, 88869.54471757356, 97466.99832145406, 42414.54691716745, 66780.21178863288, 58620.39958709721, 84415.07645538429, 81454.63634606493, 20536.892959910358, 36169.97818643709, 19257.946911529798, 25260.200618638395, 50554.77141602899, 45335.57686938183, 40190.62301618573, 62993.46411466964, 4587.2806465931835, 43258.47775776956, 14543.416240594532, 71767.83003254628, 92969.32574589261, 6800.6067577330405, 75453.87038971689, 5255.386441361409, 35964.18927697391, 94912.3516828697, 25038.631136648437, 1830.0340972073116, 61482.4029200806, 40520.94898216103, 43186.71485445152, 29821.573578046777, 82875.17123117263, 81976.8469141705, 62533.67843890518, 89911.38647547677, 58376.39531307012, 59077.37085985022, 10094.376826538997, 12477.315975688063, 73432.29064674914, 15008.49110602297, 96285.22273971153, 77935.38474106451, 64470.26055031945, 59058.540737260366, 72824.3033238336, 44085.838605698635, 74350.81343232999, 34052.50667640917, 6149.997372118909, 13329.458729688615, 21106.20091958404, 97029.68269117617, 468.8193049510514, 11817.74816729395, 36892.20710897451, 42436.07312652663, 25163.627711921865, 82839.90231317857, 90260.42380943592, 1260.694716712063, 59741.63745291253, 28185.88961727447, 9336.080122961675, 18183.97580242741, 91312.2587660937, 38677.03756527625, 3128.113774226049, 6849.24202642182, 66104.08845038785, 29046.97063454943, 64650.78082102731, 7560.8817514476905, 72277.49949099976, 85697.07035656463, 64977.80440112772, 24556.512598055426, 69718.54988559027, 42708.40971627774, 12919.806032687498, 82750.46425406366, 62156.65967269947, 66471.86322855267, 43278.251929736434, 31430.47551218737, 42291.317445614106, 90984.71103190645, 83272.42527034535, 8224.041531302251, 28840.736711047142, 42173.04832629062, 71986.95429387988, 91772.81734610705, 77356.63832980586, 49937.40518288213, 27549.70021228208, 5496.9771702438, 5732.134642337627, 59954.39901035119, 73485.39460336538, 15596.717037264241, 22842.774224736895, 87809.79914549527, 49772.97793028143, 9707.460904626398, 99244.75417794539, 23341.467331455668, 11153.858307723263, 91775.64755094833, 49351.87802561989, 29576.400897940337, 37281.09894724808, 37818.611020960125, 97039.62087168462, 933.4476464775698, 45268.03934949853, 6454.814969238876, 23453.929217650217, 96834.2708329074, 360.6739713420626, 4654.298156523895]} +{"id": 493, "vector": [48662.21795704157, 45333.48176145437, 93157.76334576814, 87593.86821528795, 1436.3835587969943, 28941.273641669763, 47517.11043972155, 96655.90715214376, 82563.16847455878, 66819.42483589733, 99048.96027292454, 91139.24337732306, 99527.15266904583, 8099.758433111603, 67202.38806295396, 27769.359080038226, 15000.098682819118, 60643.029133997894, 99358.1612684158, 71658.51161036002, 61557.82210653615, 16047.636252390252, 35854.21658267012, 59477.33365380977, 21032.257423541323, 59777.51261449656, 40361.89423849694, 5908.309713189264, 77902.71498890761, 71478.4740044917, 93321.0589282798, 69104.66564659952, 62697.94258797859, 88021.39303572729, 97845.10469558921, 2739.116369664374, 79721.70214528085, 5311.117828885803, 53905.8995851195, 20717.316307665, 14002.288370697914, 24690.740196557683, 74713.4296263692, 94508.46361154607, 40167.76633005242, 92707.28688738278, 94406.49577643513, 58983.978084070885, 25525.985557260577, 89607.10648001515, 94906.77459460699, 34882.12585851782, 2790.785595581369, 45099.11450205305, 78031.47861565836, 36251.849749692046, 35595.238080265444, 72262.68406554729, 60666.85257221376, 15674.129186884533, 20230.811587289954, 43025.20984660533, 65543.89906317375, 72949.02184928535, 17572.472938393523, 23202.11813659372, 52875.25971638278, 93062.43744752153, 75893.80979345366, 12202.443338057601, 35861.431962358256, 89326.06449035896, 98574.94012142297, 83243.49744986436, 53072.90494149045, 61010.267946415406, 22244.652292317434, 84898.74300985427, 27227.39745343783, 62397.00243240123, 87604.13792498235, 4295.152699730187, 41557.94040772399, 21344.75371491564, 28201.517609750957, 63509.22980197077, 70204.69628691487, 43266.16359290719, 48510.04598330444, 44481.145180613756, 51556.463118067106, 84356.10022979447, 18259.78456357713, 57235.018800583224, 47501.69869444395, 9841.680960604415, 69903.67842843017, 47355.645957486726, 40845.10675780099, 262.34147799620854, 75551.35039346559, 4570.85790083126, 10136.123934650077, 57379.471597739685, 3253.395390436453, 80084.3889817628, 71981.61835778801, 17518.7688110507, 65454.25507132555, 166.82372246472622, 55736.62944836857, 19459.44377611566, 62121.09885979365, 30055.19788925176, 65769.52703020822, 63606.5622419729, 28654.933124383853, 90971.8557473228, 35147.81863821045, 41147.00949419513, 70932.44883200487, 60429.57067718825, 6874.665042150652, 74078.47333844306, 71534.2939896173, 44647.836499749086, 15751.191722709202, 48805.30811500384]} +{"id": 1370, "vector": [69971.35518357808, 41389.055583246234, 34363.83444683323, 69124.30247346831, 41081.66896539669, 63745.66439722203, 26347.694534999067, 63468.06593160713, 32811.0211724021, 54832.330272642015, 38236.10258420662, 42569.88071268386, 79250.06144988305, 12682.427813792385, 96552.75755235575, 37728.205010396356, 74080.79145923155, 33149.274096935245, 37762.05055800397, 82471.56033235735, 56633.61630253673, 2762.6137308979446, 86878.79074858801, 93803.64060703125, 5444.54910707749, 93372.52394150871, 14472.85952175471, 21878.16421094937, 92511.73212782922, 524.6166202803826, 16696.819560441232, 46884.67135225079, 58942.24938285541, 68070.90199538517, 38375.538164650934, 89903.09861406783, 19535.89673875633, 62994.44193040307, 57319.24314736602, 38689.68721537164, 10649.161436165155, 19325.93065020308, 1459.6827634254294, 3764.484904625276, 44443.11157358728, 20588.916214715315, 41834.13133210074, 22679.86729277045, 69196.52006662873, 28777.155135602305, 50467.76696182954, 69292.76950413847, 83849.57202954635, 45480.07218267626, 44319.66152169341, 3242.811676139312, 39355.37168006634, 29951.7011299856, 99800.92447389546, 85633.27743203989, 6830.214959892566, 48096.28801696795, 6795.107323208471, 15673.390796754395, 62284.98527360051, 96838.40669926473, 95289.41454788703, 7970.339193321241, 30774.078219081613, 38645.130815953176, 43839.26807661827, 72035.40298478105, 37483.421911607984, 62137.601852530934, 83740.33689961612, 30372.29426800433, 98366.47603705994, 25882.35464051869, 29725.2236126054, 32344.55474246667, 96470.55484325378, 11811.134065529715, 5243.498980298, 2331.747276803897, 72266.02339865391, 47175.18714469543, 7083.308422036905, 3013.9214859209783, 81541.12512649132, 37600.44628955315, 43672.76650747194, 34920.216998171025, 49604.116132596144, 47790.62303351671, 47537.25510153927, 80426.76420081589, 4547.98423895606, 18144.61136376233, 70469.69731809416, 5974.799620074899, 3429.8822292941677, 45020.3786840684, 75603.71336709127, 92361.18947244486, 27345.508851137645, 48572.86147220825, 24527.746642285474, 76423.49049683941, 39447.16854112952, 74075.70412313662, 58510.16713407147, 96879.76314156693, 98850.56128899551, 20674.047465220625, 8604.999486279563, 95164.94401124014, 14216.138327492134, 9791.805780223483, 77046.30559435237, 79511.10619920563, 54997.70448893002, 1358.0742453775051, 59075.20711676129, 41330.841318822655, 23792.93042536216, 50841.193471965686, 35920.052902640695, 37200.686569836274]} +{"id": 1701, "vector": [23291.079760361823, 56780.80876667302, 7603.544857342948, 37472.68197893172, 2874.491098000509, 13841.837070781927, 57029.530896004675, 15649.661654903968, 27880.23422120799, 7144.259258207519, 41350.56853639285, 64332.208742530886, 40289.3917139744, 43726.981655261254, 49774.41338638433, 1532.5223356974948, 60613.55877793493, 56259.45547710691, 16813.079181497993, 41259.69172508545, 24913.549366957923, 63846.117847300244, 16574.28114890751, 83589.72369221746, 91504.3158092723, 47504.425095434344, 46604.731105799605, 21602.149347812792, 50246.760203421705, 13630.767593404946, 3031.225252903935, 66458.809048225, 71321.32496194591, 72484.77819571282, 43807.79754283557, 44017.14831641123, 67210.98454915317, 66593.95791619316, 5397.647454099463, 69037.7365142649, 23344.91151430651, 25814.707929936965, 51386.68838214966, 49357.95102335175, 29532.068776950717, 80522.13742631482, 40186.79276532313, 89556.82519376969, 16883.90724234876, 66356.07811371623, 6045.26546727866, 89428.98984050639, 76835.90800330241, 14496.828181034349, 4171.876560893628, 49935.221428993995, 72015.70927439253, 25616.29298813799, 70910.68352370207, 42900.10110223684, 81683.26979780714, 48052.86941814464, 60905.07037031047, 32664.870648867873, 56534.06801721283, 32082.051520000943, 44502.7985107833, 4317.52859215635, 27291.78255924555, 56798.37955258873, 58074.03127415609, 63612.600678750554, 23305.098641513243, 88394.97344143364, 48144.44031883097, 2784.4956918379958, 37345.935001141894, 57052.09211993895, 92543.80445500575, 89962.90748691323, 30331.28165048501, 9374.235882727255, 80315.73867378234, 71065.60785573033, 14496.913221120978, 79041.19119489581, 72037.11443422415, 55774.56559893993, 12718.326485533993, 7583.097965119278, 64939.961513098155, 45870.51935973182, 62515.90021677583, 35643.167025822164, 41199.42784404327, 13821.667648852532, 14001.075616759017, 85517.95789759896, 84303.80896558928, 51733.730955812614, 36577.213729353316, 94452.56449348679, 94053.80366772668, 28338.162205953842, 86163.01727155445, 81382.25959814449, 50009.55002003895, 11253.067620083846, 81875.69913056747, 27195.173698970786, 16026.993453457972, 60362.32459907468, 70504.11966242721, 29489.250769148177, 71163.49048588713, 11425.622567232962, 80672.05933626839, 91647.62621750582, 18359.06536872013, 86920.83836107331, 42401.99337715962, 80680.83405666833, 91204.07251972449, 52671.03492944575, 2974.2268995704535, 86828.68651687325, 71184.68702944079, 22421.501818979985]} +{"id": 1372, "vector": [28891.20670716305, 64092.01988263539, 86825.20225760613, 43096.854258808395, 42950.73594241912, 4161.977450411414, 72534.5738331371, 70142.1213157883, 56118.434211661486, 99734.31867573975, 97856.67406602697, 54524.81000597415, 14459.509072518318, 30030.40304285326, 26114.147797232177, 22769.31314261823, 15011.888912620552, 54318.45368520537, 34757.030449558355, 34572.02796119709, 38611.49235772495, 49841.74985820799, 63609.17965003943, 71745.05533991144, 38311.95587286754, 34952.2093438391, 63375.99561887231, 81209.82534886434, 3964.6967872719597, 65377.488066074984, 6617.87911159919, 68100.05072873658, 41227.043234464974, 69442.51634363651, 86691.01113711462, 60803.52546480489, 97183.31186098272, 95607.24843867672, 46519.675376853156, 22664.036319797808, 34075.78365184737, 42509.53660301702, 83636.48996409282, 38002.23313502724, 52844.78309285239, 60074.536188683, 96336.03522841063, 74649.95368635796, 69984.9908774689, 67381.59365668317, 16802.24762204011, 19074.474793650053, 83348.48645160801, 39206.22980320856, 42202.85798710347, 4651.996068460007, 23432.964026494985, 78615.3762781739, 18887.66550792155, 49550.443772641986, 75251.49738184288, 53238.014842362136, 73351.85127224623, 40385.18092426885, 19486.618265580146, 34461.57850396946, 19868.729422497967, 55933.26468273616, 10599.55973657557, 11765.493130035255, 62554.7949485887, 74825.07138568755, 70254.1423513827, 92002.46611600317, 7376.840758690584, 73783.28671942952, 30966.0757092168, 16399.57878794166, 96232.22503275861, 87070.56185011863, 89719.85790824286, 44168.651371826476, 91402.81148375821, 98920.44355061262, 13541.602872147529, 45821.588555642644, 4746.409848755006, 40565.411204731594, 7625.352291453225, 95217.5546196152, 73231.83503335546, 27543.15770234532, 14210.953307673568, 33004.439113903405, 33212.3721574195, 24787.78190321488, 87013.3516193403, 86239.89665572903, 32124.889435487003, 80212.89464222222, 11000.590986371672, 97513.04505219823, 10758.739684031416, 36402.930103180806, 57304.06901873648, 85706.32905293877, 51871.21983684139, 40648.8882837206, 51295.92984587202, 53878.0308138587, 52090.39334762168, 98438.85287319233, 49379.531295127555, 10433.647595910323, 26384.60707077287, 34657.101912573504, 6262.964632115298, 62586.191891497845, 5292.311862851185, 17768.443184824868, 4079.3336758161768, 3741.9678445729023, 62574.91130506463, 21942.479321701016, 1514.8216946163307, 54301.16787545629, 17479.2279235539, 60512.20714932642]} +{"id": 579, "vector": [32641.533450110226, 4003.515287036785, 49930.98401786869, 90908.71291317669, 73025.53561434883, 81147.46645816945, 68473.8714260829, 96602.60416624857, 74715.36612987079, 39872.283956119114, 83680.21111116522, 36350.77220133079, 10988.993974611316, 37604.76974169837, 64283.676138109106, 53704.611991028374, 53113.38350600067, 39132.335315861244, 79688.05139011504, 82742.74453401961, 57327.165345132205, 94288.94070335203, 27815.675188459798, 94018.51684386718, 52637.87654163491, 26063.747217396904, 16581.72025313408, 69209.54135242711, 89731.26255581893, 36135.14946677486, 19565.57503877885, 71910.79813475802, 70488.26368766122, 41787.76560076966, 30665.75396363269, 7587.499929776464, 53959.570276946724, 71031.57247250061, 72805.45572390525, 16365.32434988438, 26008.130094190452, 67401.70678913905, 25054.737335446887, 8384.933805223072, 43914.034212684586, 60723.096040446246, 6024.041143121328, 90399.05098782855, 43093.34862532891, 6748.926518603226, 26387.005566574408, 7465.242834036412, 46460.798494470044, 8851.411291571865, 60792.72100308154, 79479.78358711104, 62075.64467040063, 85419.68943695624, 69477.86936069498, 122.31459734919348, 69980.35152225594, 44142.328550307895, 13841.866495478638, 23016.677147506325, 74850.80764246175, 68231.1540903824, 88022.99598622747, 74742.41784885002, 60823.45546472282, 215.40028840331127, 86058.74962996243, 51944.34975443282, 82713.20991217437, 76542.89924504477, 23313.719700050682, 26317.463898968508, 79980.27130992194, 45091.084142931744, 78326.1158120373, 31975.068469867118, 32032.525020730496, 63531.58121038559, 31352.49090509419, 48450.026849675254, 90170.05306332391, 80315.19423168163, 84763.02134502746, 97568.6467145464, 47350.30030130218, 9672.575571109799, 50006.43216607243, 9861.00181986993, 78401.80300077057, 27920.730986380615, 28476.926050815688, 79129.50217084501, 12811.210181886845, 28682.409831734858, 9990.912279088481, 20691.559041170825, 11932.628788906308, 81276.57160699542, 65470.86889163212, 13573.543016121881, 69653.45329789481, 82566.81555662285, 25960.77869194706, 35287.21456522035, 68323.00726270118, 70263.4699642319, 10832.252821503575, 82409.72067893264, 93192.5410652122, 33178.33596439224, 18107.601869955124, 43801.87027421893, 1594.4558217759752, 35058.94922637026, 44534.08293457073, 81880.47506605342, 40526.021444230886, 16551.453166461437, 50758.75156184237, 55126.119587513414, 47534.27525894649, 41693.983659982216, 37579.34004107209, 51617.42793511215]} +{"id": 1345, "vector": [7751.739417901837, 42771.96997149334, 85764.74846804442, 79208.90681120422, 87781.78326514956, 18527.46039437674, 27776.51006008274, 36843.17506321356, 59646.409713676396, 67269.7942095246, 64000.97011381426, 47811.39331612325, 13162.289626603962, 24938.98504141846, 27083.571604638146, 1931.4320955289065, 66696.21161938348, 96493.42049867348, 78701.9466266394, 82150.00146213626, 73316.70766362105, 54982.54269129546, 72980.88854902158, 62956.11542879875, 10979.43607996309, 1432.5446523064666, 2888.1047955264917, 17237.58816565166, 51828.69004999203, 52200.41045028311, 63544.0790013091, 83942.64807041959, 25103.50107164667, 86457.20086877455, 62334.332858501904, 90145.67679610323, 50439.18319642019, 33493.84132517891, 27169.943747107616, 28912.92743804991, 53008.70105197011, 61901.511156641805, 64946.39369938596, 52493.38622612193, 69679.07801350836, 4755.452891149259, 6309.952111969475, 32231.730002450087, 88409.64475829487, 40687.52368996348, 64660.00690473847, 85432.73348392491, 22529.126924470165, 16141.976668701285, 38608.621101906596, 91768.63128414348, 2349.747152947279, 34593.19654199463, 39157.195593050434, 91279.56067368374, 87804.54718852327, 32277.98716441059, 32148.59853265568, 24378.20858439612, 24483.798198779805, 62681.772318636366, 81587.74746067486, 92642.25700322981, 27500.932421317837, 11462.518634561126, 48213.5813986525, 16305.518343533999, 2456.806094088637, 38458.27493010451, 70559.70489437642, 7724.417571539821, 62532.06770570412, 95696.8589616339, 2630.0341809004426, 83704.76502729813, 4148.956295886052, 92066.45309462442, 60308.57728538381, 32063.8001483202, 37559.332063676855, 37485.15862913365, 79279.83661918694, 91284.24834762514, 41946.58986631788, 13625.63967063023, 8151.102521548248, 6831.339071680809, 89117.78536863794, 71530.48335565187, 94985.48007466688, 61755.802917072935, 29836.36375695281, 2044.8215860962905, 23220.437846369434, 30688.85545083151, 85747.32588580408, 11066.922336306829, 6684.945010730503, 20453.855234985673, 31080.453283476174, 67404.23516495831, 28295.683349906907, 43391.65651818176, 40482.99546297459, 42527.74996422944, 81447.89262892528, 42313.68371628548, 26992.09660813803, 68284.0134094919, 13997.895121590687, 22021.26668169422, 49676.54179884471, 89061.01545347042, 4379.480683732362, 4731.493384579589, 71901.81694304221, 8.045305025927707, 47072.79554387776, 63436.935409180216, 70575.18267070511, 90777.17346990062, 79416.70978190606, 60286.278960608586]} +{"id": 780, "vector": [51265.852825585134, 2413.9587501826763, 8859.47344515341, 69079.10670279364, 27853.59246183775, 78064.34889293778, 24764.67270792322, 76636.72885824136, 88775.48800151031, 13195.05558892553, 66639.43516121931, 27454.342857853164, 67876.97077674836, 10373.482519321475, 73860.28624077926, 30850.659798739256, 32997.71333839131, 64559.873797041146, 26463.98611767533, 69561.95783905618, 16379.158065848698, 59858.26185424399, 28658.848655754577, 36055.026563448155, 39096.113031715984, 6389.387016697768, 51565.232529441426, 32036.77478299205, 88537.87559178271, 92763.65556591761, 59759.1850475817, 95093.10759966182, 86256.25308380791, 58015.795530338495, 17225.043729509638, 66435.76832134112, 26321.653284143853, 38385.68131186069, 69842.39297402307, 20112.624569201842, 81187.17937575099, 78652.38173917148, 18571.33002126291, 35322.665187409075, 80661.2821165446, 55534.75138566304, 53855.85337317348, 8011.617396226278, 40417.747481110666, 48432.31951079913, 23949.352586789453, 65040.81351904281, 47707.18136627922, 42873.50179971112, 68150.07103469945, 1743.6688557316838, 8759.598801237378, 17964.925520193796, 51500.54272921054, 57845.766390886965, 23014.18996211384, 39961.50361201183, 89424.44792292426, 96325.82839013357, 24969.662918924852, 35122.560219219544, 99740.40813140413, 23798.595982966242, 18140.408257048803, 17231.298874724587, 39095.525063274625, 37227.318861981454, 30839.777294758365, 65450.94149774385, 11625.020747628356, 90063.45310171113, 30852.912790601604, 41012.569822866055, 77705.3256914948, 93086.36148202182, 98611.5617551992, 26791.09506170172, 62525.175563263554, 59345.693294088494, 2388.3612714764467, 10923.846505788504, 39355.10839349816, 38621.01365557635, 93581.26486305533, 27388.735747133076, 19574.277310257592, 22307.48681340449, 67698.40203088224, 36046.84921580004, 40623.272086124605, 84822.58473606098, 13805.376022313643, 32154.38380481046, 2720.283236491761, 46689.62133816542, 69550.9191346947, 19902.700747600054, 12873.410560299204, 32547.0289718242, 26256.933085495348, 1497.525925344645, 12422.123325952738, 72727.5174526233, 53217.53787851319, 14726.600362458776, 76984.2516752687, 98169.31751809054, 21146.63634900128, 55146.03327051184, 81.03922132312036, 32916.72720934902, 80094.8442722334, 14315.470471533254, 73443.30501053706, 26238.670512063924, 66762.57814456092, 81079.05400338784, 15113.308423698369, 27388.611105886008, 7956.511179122439, 74658.69716437299, 4120.779288693366, 40347.60676053998]} +{"id": 856, "vector": [48191.029831227075, 22237.38407888407, 71532.47850716577, 46177.040239660404, 11629.752471634347, 59864.67123114462, 63351.838720255124, 95127.21050677085, 16254.035554060974, 51361.85030053285, 98661.86879267276, 84174.16486700712, 40795.08077961493, 26388.59144728183, 15409.284404063983, 65616.10677078283, 51904.35886517802, 43380.71903865221, 24387.95053274131, 36417.776274065174, 14757.367035969848, 87503.46781215508, 44432.33374733943, 15156.521350160978, 3351.873264010796, 28622.33342922219, 38980.18949701526, 26030.65445238496, 94430.84998274724, 86020.33569079131, 86017.26731603102, 22041.66452230284, 91866.0398721474, 38483.834304456446, 26811.366893044087, 53433.3933941889, 54043.21973804699, 77578.35582482557, 74002.77655032836, 3781.6641221693812, 48524.054000640026, 86891.00196651406, 31491.303813793937, 95074.92271198798, 96956.39937741906, 51332.73438684577, 27020.415641179996, 35553.56666486171, 90505.79576305494, 54037.40915164625, 2843.861756118016, 16641.862105656757, 29130.651183385435, 22532.617575360837, 52546.58755332359, 89501.5751385882, 70830.57885473473, 72616.5566483112, 17514.120001813073, 28616.749112420493, 96489.18336900463, 68591.54922099273, 82336.8692744431, 71065.69598543229, 3733.090054587951, 50055.86463427979, 28003.83542156636, 13580.664802097486, 37256.368666326045, 13320.463848250552, 47214.85836737041, 69884.4978734048, 64130.52936809015, 51826.403228223695, 87245.03362018625, 85461.88205308997, 86826.71573780885, 5006.885535020011, 41314.14385673918, 25133.991700096347, 98821.03498722958, 90062.69447061012, 7820.12873804594, 83723.43303853068, 48913.83303813285, 62669.89567220195, 13532.143897010863, 15174.420116394405, 86096.5107705998, 45473.026535049125, 48729.59634117147, 70029.85784149688, 33840.93867461496, 84400.19806550806, 1970.9082480864338, 46812.257422427254, 58878.18444508022, 97544.37039118327, 93902.9902032184, 14240.251379224845, 48739.69137052432, 41426.516408254676, 26355.898651598574, 67407.18378716196, 94413.36009812, 37383.89535811237, 82751.79523607579, 21654.566520986216, 59903.607014213434, 74763.03773017378, 49028.61120860239, 51938.05628670094, 69643.38217918288, 65995.02487241258, 46363.82483820029, 11034.21993902204, 32531.294192576064, 98173.92888302937, 95432.0048662217, 85372.12689276866, 415.79266611663223, 66213.89734866997, 5571.445182642098, 32146.57910270072, 13715.594204809844, 43450.91089500327, 19374.857299065952, 69217.87313699079]} +{"id": 286, "vector": [76467.92648089962, 83404.95139684198, 67783.62264376569, 38607.538202456184, 37727.0704762997, 69905.59550219798, 71170.01698759517, 46509.46387196304, 71918.67733909878, 30325.47240256228, 82611.5321184824, 86081.84268515177, 47427.019720591226, 53576.30521838082, 41630.220897160245, 69419.9427113611, 15199.418171639756, 97455.66171684599, 36327.71988534743, 36890.99065942434, 77674.39140103415, 24892.27115225122, 61822.72104839988, 2927.4916033870336, 55755.600249347845, 46159.18268347199, 65001.780387321516, 19741.420078871808, 50055.781709478695, 94986.41325845559, 19536.76793310717, 33255.03868355971, 44170.22195490036, 1251.868911169618, 79922.84656215421, 96655.65088981157, 40006.53752609987, 65474.73613482251, 83423.22859000522, 26850.817175617736, 15048.484936937111, 76332.6376906777, 44154.46181048627, 24585.087967254938, 13109.975599932277, 24833.15151938291, 83583.15993252923, 8284.239370516843, 23767.85641493414, 2191.236290294074, 86914.2054917216, 94790.89092306989, 32160.83417881943, 95019.99086678741, 66308.61438404178, 21953.638038854162, 54798.57407516843, 73784.18033254184, 35541.33220193339, 17523.559775158683, 55849.42789727008, 71456.11638192639, 35477.627605737514, 77219.66566594878, 90158.5993791838, 22296.891258387965, 1664.811224542473, 50107.81017047743, 80658.15805335334, 29269.98530464784, 41908.50933024761, 94638.03978861465, 39908.24653740895, 96911.03883603179, 54271.10534332424, 19839.72867086178, 13960.429991426981, 18710.941541483207, 18648.391374280327, 63735.43271579036, 38798.413853875834, 24226.478213187464, 77024.84046464346, 29717.98443823368, 52994.33040431634, 65361.030541503664, 70371.11494872441, 71861.70532400931, 58468.21056939735, 9629.267361655524, 18272.00039119129, 24668.049921888767, 77767.3621060287, 45396.91951066462, 69251.94886643978, 20030.55617289464, 81064.21048975711, 68133.10606106294, 14720.614801031328, 30501.8503118619, 63056.33695915174, 27761.431044925233, 7100.269016712624, 43838.779793158166, 21267.32857225493, 57408.60992892385, 69857.04672942306, 50691.96380740343, 72614.20927261298, 79987.79889481424, 71986.59048858323, 74029.73450403441, 10638.8089859082, 10627.387050065972, 45765.91588378596, 89686.2253378677, 62983.17612639045, 45905.64631583644, 3530.927392783989, 75188.07627334833, 13757.368436895833, 42797.10147048388, 33657.985495854846, 49889.9274969007, 75837.64967749287, 57452.45557254439, 46838.54293782994, 22287.413012976696]} +{"id": 942, "vector": [76.36494267625471, 32042.293397157286, 55441.310192397075, 69234.67296256305, 27875.981506525306, 90910.96512616247, 96360.97764252826, 24996.661803964926, 89977.31948095895, 68944.27142850134, 73274.80719703209, 39343.156080539586, 22340.994811652658, 63189.81390930661, 51115.974730007765, 31590.625433765672, 91233.0380168933, 51172.16173021676, 75477.90189471992, 89498.6429150711, 40773.135698883125, 81379.07074014278, 2324.5398576732136, 45519.15532353098, 6309.781545184834, 80259.33453413485, 74416.15822419188, 45614.30557851276, 54023.26983578213, 54338.25267969314, 81198.54532350144, 91415.47748215536, 43205.240671168554, 47655.18199777361, 63583.376716954845, 31502.11680256445, 15434.351960269365, 98802.11369310759, 68647.96973418733, 62978.74929441341, 77083.09256206661, 93834.48627574362, 93911.1383635071, 28831.35348109711, 64731.5597903005, 91996.29644767223, 2500.492094025386, 65102.91136892044, 10164.992164654252, 20862.618481206664, 66488.86056987518, 16258.889083488071, 83895.4596913132, 87664.20549636369, 98110.80525011376, 64750.52081302984, 50216.3785490907, 70392.91466640228, 84027.48875014117, 83047.19805895108, 55827.945775899316, 28597.364844278494, 77987.8358955826, 84468.65720854636, 72865.86292583209, 8840.00198049021, 37522.028931752604, 20858.201115734697, 93503.55384401852, 57667.65786379928, 546.7497683864675, 645.5304030621667, 76120.32888944093, 4992.839383405212, 10665.097518161116, 94565.1987805937, 30298.201606762555, 72236.78888014903, 81446.26872920807, 89927.6903641047, 48987.31185886728, 56439.85972200341, 50453.2557926984, 78982.77818324197, 51382.027333393045, 31704.867725190943, 78136.79561028954, 14973.74728048796, 56705.40538330051, 42164.61391280274, 12035.961486922886, 27345.936651818236, 2559.5007569425675, 66262.6927846836, 15001.4058343376, 12045.413079436828, 71106.95541192932, 66382.21925739113, 27804.807084425753, 20958.8969061822, 13830.822484872751, 61120.66934645634, 60212.931994721876, 3702.2635624171407, 45570.71323787778, 26398.52275880664, 22569.084406802485, 79599.46400089144, 25544.332191076744, 28749.922419012575, 66010.95887562391, 42005.22731380119, 31850.373568015166, 85539.76683069838, 60113.57418838694, 58738.4420271605, 38664.60970332484, 9094.702263290234, 75168.12761594272, 56378.26162677024, 50318.08064808761, 51275.74373299106, 74553.46026736371, 8928.052726837477, 81772.03091994305, 9602.20489876139, 89126.54132723897, 35618.80872796523]} +{"id": 818, "vector": [85605.90384904224, 59147.62538102247, 2060.9891878608287, 45754.40669712365, 18032.728339906433, 41722.84349006461, 39260.84130829865, 16583.209199729154, 64782.93778802174, 4837.413990453399, 49302.33969369704, 57335.749989928, 52072.667646312155, 30568.386576974415, 91844.96751040255, 36961.94654583465, 46294.19496877067, 73009.22525592225, 69145.55644806409, 63346.64072586682, 61583.33569528601, 68370.872387562, 21832.764163123764, 57077.648859656474, 58743.60157230203, 48913.13769755343, 50488.53440069599, 51965.90988083116, 67679.59931528992, 76872.24412808445, 45238.68797645912, 48017.58464846333, 14106.604498608354, 21805.44299801742, 57395.82528566749, 82406.80742568022, 91213.95074806164, 97217.71860509642, 85365.63578655232, 47007.71335499906, 82349.68946731437, 6037.486944868753, 2826.2990312032944, 35223.58064921117, 22449.14082417122, 7168.998067818311, 58379.17335979853, 55418.044983297696, 15651.940573872304, 32438.08943757245, 7990.680713837395, 79263.0843853312, 85598.3093174308, 71863.82547813283, 76783.97184142417, 17174.821554978847, 86940.13638114814, 51310.93438343367, 25953.67176826534, 60221.36210849613, 7129.413141913777, 23602.090686007294, 51659.69507955017, 68075.73882559422, 99211.63978573905, 52406.73672272789, 90490.82577792802, 52985.05294272682, 51939.38489343275, 10787.532510165032, 25563.34321953749, 40519.620740270344, 14846.125365838125, 82373.419971543, 88413.12853198081, 94505.91554992848, 52946.096846293614, 72205.6420741786, 13394.944592419733, 32320.729771135415, 64245.26474767668, 18294.503114751526, 27570.49221643595, 83107.45945673839, 77692.0344561851, 33462.8585203897, 16060.330214575602, 77043.5439379094, 64726.70219343706, 2321.593187973614, 48529.786438513, 60837.644524558586, 59198.49307025966, 66547.96140057078, 56123.074598329, 31408.632828123184, 17611.765812515536, 41033.10316299627, 47340.74991832493, 66636.11906667399, 83147.16152090691, 54803.2282562021, 71888.47759215695, 91677.97265114552, 4672.379457902242, 26604.428945832304, 30975.103946081595, 11657.908462237021, 58857.53677602533, 79039.78881216436, 6065.741047299455, 31761.969004568437, 16200.893633830427, 955.8609559627284, 87052.10858348549, 76268.68939052083, 93916.94348699284, 88739.84460882067, 92980.8820399183, 56750.14466094417, 40381.75729592006, 65480.29274154237, 31178.9962916578, 2710.429644433654, 91776.82923604222, 86750.13953141533, 72244.14380151039, 77905.52718878594]} +{"id": 194, "vector": [72771.05178786012, 39623.90539741151, 5739.8659627253655, 56474.44719650231, 31144.190489872948, 85970.5021289384, 38400.180456723734, 23437.58855185576, 61293.31036282165, 19195.414733093232, 23352.912305105656, 53884.090513601026, 94992.2659515343, 14423.841800945847, 41193.66995358349, 19175.976832073993, 64835.994772026905, 23802.35566244049, 9383.38875459791, 73927.36781532213, 10507.591060509469, 78130.90933299152, 22073.589133907124, 91061.92678032347, 31984.097885803785, 8988.433457316369, 28949.50547407318, 86415.9075582138, 89500.68250760599, 79838.03356354935, 83248.61696288131, 21196.473176672116, 34767.85130144733, 64048.455129808724, 64331.430704450955, 26577.990737700187, 20684.925670768927, 50672.00952665212, 80651.38593781076, 23474.359587494033, 46385.08790255319, 26467.231986136263, 55452.11225935731, 78413.12245579688, 53000.07065095472, 90149.48920017922, 37750.682933144184, 9619.754830306614, 35508.370619907306, 18071.606026875175, 46348.20816140809, 30885.387555846533, 76528.3271283637, 28730.537854015936, 79326.47078726231, 46246.39823048714, 36144.93313988637, 85412.99038964779, 26144.57377548255, 58895.94596749252, 41996.34321422806, 48209.870126148504, 44023.390948958404, 19125.678841241435, 70002.51461897054, 69189.26397918815, 48323.76924303923, 47829.565266683036, 8849.548269894114, 33427.431294607726, 73588.36051806499, 50026.72223247445, 40192.350269711395, 68481.6944485657, 80688.4283671126, 78008.52188391391, 1660.0986241534056, 58939.18827970541, 2630.1596002401893, 44862.66115712686, 58222.998796028776, 84665.7937910145, 9512.930466381596, 23801.984313940706, 6277.448874331771, 77952.4426312539, 4541.624260830579, 15645.241782467567, 43914.114679255166, 67474.16377895889, 94456.32815060254, 36878.90901493146, 81520.5721333001, 63589.89664163205, 31082.176113128113, 72305.24888744761, 54176.6254159491, 76486.90689483476, 5229.429393249907, 24145.492913356735, 62926.95272326893, 55003.97030045499, 51527.96516675834, 34556.98068047321, 87076.11067785516, 79806.3756065291, 1665.808278827019, 44481.84412304746, 94734.36851183114, 79476.62758169505, 16516.36141364612, 42943.88393777723, 75056.00011009366, 78484.72177427936, 44169.66647077073, 75068.05314305666, 93132.74040980946, 51555.44795846534, 33000.7141393325, 96639.7886475965, 92002.33061385201, 23602.015942222653, 39724.5016340311, 90644.74353152222, 97555.61191904402, 61583.8129665518, 17699.842408724307, 64722.1724991922]} +{"id": 578, "vector": [85177.24288818958, 55390.787090099104, 89877.10522816847, 93161.01145660508, 19638.720711176637, 80669.37364213831, 70913.75432006316, 97187.9283233313, 10807.077232199947, 73329.01907144593, 74491.1471643137, 47799.00193070623, 21453.350580061913, 83538.82999449297, 35201.45586524024, 19670.907806925363, 38735.474106497226, 42252.766898087226, 29585.47476587061, 39357.07031322202, 61516.073473132106, 77358.10454903406, 50122.489928778814, 20282.206071886932, 82114.03777083711, 1259.3876816060722, 88815.87227149881, 18274.940165700937, 24379.760265951, 32071.89082087075, 30040.760373453857, 2527.148893868092, 52147.84444389533, 7467.5291834835125, 70317.21257480147, 70917.68550188551, 19230.016737142807, 76615.5127856033, 8245.520724915023, 92519.52185700648, 3957.807888196907, 11072.013723548413, 1659.2476328448956, 45436.16594000719, 92882.13912365388, 77630.58671911054, 45649.249295549846, 7637.834852637515, 88245.8819219524, 20866.62982180194, 52300.40720679421, 71858.70748904077, 8056.287086672875, 98036.37984151475, 96464.1298185665, 89159.30233824735, 60051.18109845502, 27212.479899234455, 58385.13257929427, 83396.13344254553, 69762.18141447312, 12768.643676834845, 29235.25069257521, 38759.54338966002, 46656.7528256972, 44558.38442087223, 96835.21895857768, 18830.25430028743, 31998.810110838484, 40116.80606168112, 14489.653923687185, 38406.73544133202, 54131.431882815916, 48679.706601622565, 64431.283725838875, 55119.72694590042, 40067.79146004212, 32470.591696574425, 93718.9158944949, 3028.6532039177528, 55979.08216619088, 70955.82142534279, 98311.11085861486, 68986.68388103036, 56796.290318298416, 61649.75806158388, 93745.826084528, 63114.278462763185, 22548.756765149792, 48695.46290482368, 96467.46394360115, 28802.509711726343, 4699.170774165895, 18090.043155688752, 44237.92376275154, 94024.29355240865, 20195.999774553642, 44029.09334613497, 10891.771312499599, 56211.10368551663, 61300.796961307555, 59215.60075831127, 97718.85720609555, 79712.3762252115, 24816.02167515965, 38254.75145844353, 9111.327754047126, 72295.77492293564, 19220.039244618194, 29876.977495939605, 82254.53148183397, 90401.32030620507, 98629.98957288303, 2176.0619500247767, 41784.03423106687, 49260.217973116414, 31544.238771450197, 45416.447331299845, 30990.49664436482, 36856.825804391534, 74369.5931638103, 89791.51059831613, 65251.063576949695, 6171.618397972245, 26720.855522408772, 61720.27799330608, 23157.82714000254, 75880.72928191359]} +{"id": 247, "vector": [88189.86269803306, 50865.90717768049, 40769.860016438666, 14733.536691058236, 76138.40868912912, 22329.408420711196, 35640.87403344087, 52523.231024612745, 78492.37951277272, 8916.729953273218, 81325.00539031922, 70932.3561241936, 42057.07545483806, 87726.95884755264, 29521.153856958248, 99219.4106592519, 64671.84093327272, 38349.3277097776, 38579.39237593644, 92512.95719358236, 1731.2231259858702, 95383.03762661683, 69451.62870164981, 68181.32667232107, 37027.07384239978, 27662.69647217712, 6717.07254623527, 11558.639534618054, 25100.4887016503, 46836.92515437914, 46435.8382024521, 78845.0489719111, 59491.63686363238, 71181.16037625342, 97439.72290778268, 76164.13341793933, 27705.523859928537, 71989.57963245455, 65933.68411395073, 5266.369707270058, 98767.43173687535, 34641.1515300287, 31899.999866172337, 41305.86374897366, 31713.99224562964, 76752.49001572057, 84779.00578212492, 29423.06268541226, 61593.17142606442, 94955.19038781105, 96757.43088866226, 86854.9667304318, 33654.7243851618, 17580.712056843175, 58211.17220264623, 63291.980379069915, 66407.01328594483, 16913.486829430603, 69405.23310533592, 7212.884756715199, 53436.824569016426, 76157.99581583758, 89514.42395320292, 61034.30815816443, 84727.513454346, 85053.02907151415, 78208.2092360016, 33809.81893328282, 42210.47005194437, 90662.91557028648, 93808.5217876048, 79704.96847062014, 9577.032469266178, 67015.4060971488, 16962.967744839163, 86832.18587957064, 96356.90460742584, 12259.78478098575, 47887.08664089573, 18998.608274842954, 39724.75244452479, 69784.95328904912, 52322.36670648336, 28569.26142937859, 98686.3522198871, 34373.87659753876, 71137.724805902, 2498.999754947595, 83191.18375383518, 72019.36586672773, 30892.913742113316, 28994.068755862536, 54261.55204857025, 50358.520133463666, 73469.07165395665, 94696.85929675837, 14000.480523015724, 57572.42560988471, 28755.195912769082, 63350.93799879252, 45865.17707994894, 28957.56980181762, 19595.71317498193, 25728.85904637924, 64266.75431380019, 58515.69232491064, 20611.27221583009, 17872.179551523805, 27853.57795479496, 1997.3732563269286, 74321.95674202348, 46505.25552459968, 14152.465116951329, 70518.97686319241, 86861.91802686099, 55059.25869098294, 69734.44444700408, 41721.657937784716, 85215.54882488657, 96801.90113209908, 6288.544044522304, 36261.957463453466, 63315.447794969645, 15654.665861847372, 44505.10179625333, 49714.53222603162, 31620.779587533787, 30073.60039503928]} +{"id": 2011, "vector": [38021.07570510669, 21864.35935860368, 41577.57237314883, 97437.02423782184, 98635.45766975454, 55545.96134342468, 51171.12134177853, 69771.10754023939, 37861.11199582934, 68643.00780074662, 27028.31342556964, 93370.21438796647, 79425.54810211645, 74426.2831457258, 52846.1096419931, 50247.99862160766, 56193.38778771568, 55259.366657505816, 92857.65708931061, 3483.876203795444, 78775.38912631308, 12673.059749610627, 99267.46839491237, 42775.78580644722, 39498.34907497062, 62663.34012122058, 75766.80934444933, 81488.7423358783, 36235.53900896477, 63656.10344436836, 64816.70605440504, 24718.213675874646, 20137.76707686794, 69579.27127650457, 69826.14644658382, 42393.16739398088, 15158.398090692093, 92876.08527767233, 30415.225077272113, 66739.22248367537, 2806.625519216288, 72728.45828347509, 99037.00766344181, 13722.452755448756, 87561.25407021146, 37326.51780345373, 67968.4074894938, 22.601331330396324, 99583.62505405703, 4144.706782383311, 67382.95502536671, 4712.818385607631, 55641.31601195913, 15176.834757579927, 11770.966559929551, 69112.94212257257, 76331.87556240277, 19743.327774189023, 57881.60093168707, 60780.14894090974, 20673.02187827007, 13891.938704996575, 39004.31331837856, 35584.073324288736, 7013.530760357734, 20050.549255744732, 76004.59069493666, 23441.822885008114, 38160.97932572209, 6576.759196262027, 46788.24570274348, 13375.43663703118, 27151.002934215874, 63999.01771010229, 46243.58651069906, 72705.20769905043, 12130.431278937282, 54134.384393657434, 44352.666171759534, 17799.694626533346, 77910.0112430924, 57553.59908500125, 54641.4636558473, 74035.86744080893, 41425.363457973515, 36515.20370519743, 25444.756463190486, 52087.31846902694, 87605.34098832817, 45095.529034698935, 47352.84168973203, 6280.984684078395, 77428.26020733775, 78767.47914502531, 37897.67197236321, 65008.41936713164, 68378.1547874137, 61866.39170359377, 69391.6875559036, 85209.45492604455, 30523.06821155545, 24780.530918501874, 26406.571804812775, 83807.83037612555, 96981.68714832391, 86490.92639622264, 92301.57479977973, 92364.233384578, 38072.73918602469, 66155.72012964988, 4154.3653889672405, 8040.218334308991, 94813.30473110442, 43534.26673422801, 25305.071007019364, 26840.990590361303, 44314.74187373757, 41425.33725294127, 11094.726464977111, 67389.56846967818, 25411.73657525634, 99431.96392483423, 71989.69779133631, 36875.291978313704, 35752.889637155306, 48858.14487457566, 27708.567032686882, 89192.01620966202]} +{"id": 1494, "vector": [90162.47009243719, 43090.70006637536, 67105.22996744761, 43567.02025859821, 35001.397099192545, 94507.68937302237, 57362.595632387056, 64611.664572177695, 8895.885026703709, 2325.049348057928, 45670.75246018603, 92739.88385936663, 33577.897113071245, 90789.75046190916, 94319.84077505226, 97446.17380627977, 43408.065477631084, 74724.92707631207, 21491.458606852044, 25358.93553631463, 57131.454950780826, 45153.83615067971, 90568.78226797986, 30982.37799051887, 32311.39129844075, 54286.27767475871, 39867.60093107167, 24004.321035739173, 29751.58429828565, 82828.02903031289, 39636.80679483517, 7683.332930588327, 31166.386355058516, 34293.61641821367, 84568.88139900706, 48448.95353255278, 45162.31878771473, 32426.331846501842, 21621.601237401555, 41956.05207300275, 68532.62964673866, 95958.99283370539, 66752.29320866763, 55133.55963958473, 31656.120516377283, 74227.65311415773, 72558.37312701331, 21325.348815760713, 59259.336616111905, 47908.871489303485, 56039.50217856557, 95046.30067365606, 38335.45411002007, 71038.75639138221, 43495.27119455717, 56195.687625874016, 60166.83838468652, 80843.44112946227, 62069.83432546855, 77729.97956900064, 68970.31022092045, 1750.970543936281, 47105.314854757606, 38009.262943856724, 5048.475365752747, 61366.40452470168, 16242.485448186639, 83680.5467833949, 74217.21062249994, 26789.142739775096, 48854.44891169639, 3825.970986239791, 71755.69082009036, 95339.34727214849, 15063.56391531679, 40254.19247391893, 64935.30182795227, 23142.545210589215, 13121.534900040166, 75912.0921883663, 12851.680305093781, 2200.7585050011435, 56883.81885418583, 46231.81116254336, 63536.713801792874, 18930.953539871687, 36285.19599133302, 83701.26560104189, 49513.60878732182, 17737.69051000185, 85465.3998898174, 46430.18365437064, 61664.80649804051, 95098.34659691398, 37664.841103773746, 71945.08029160328, 70826.35404987288, 23007.395344445325, 62484.68429503345, 5884.54490120881, 38380.03537663259, 17305.610824833773, 25398.270652168132, 42913.27347845135, 95348.45890599204, 81294.65334967278, 13761.587832574884, 79494.74248206484, 90670.85585508069, 12009.761038119816, 77291.94836636879, 40642.77079225271, 18597.193877498554, 40598.21173710648, 8545.883589591553, 22547.03023200525, 20768.82492764499, 58093.77806740308, 75359.41991179831, 29946.19689737792, 36166.84512443167, 72288.51950329081, 45694.7473259887, 55216.25832716082, 51600.178044700915, 42470.6289710718, 56308.57427914696, 22115.594849832254]} +{"id": 626, "vector": [83648.29262864408, 82524.66830246976, 8660.284638648764, 81003.22564345185, 49374.755318931886, 31725.952897428157, 6689.9564235541775, 35534.15180493705, 78635.76940696374, 49350.342684795156, 20860.917127122015, 39152.17907785206, 91918.07759418638, 93131.64851781045, 14744.029872539842, 58369.39501357361, 69554.68263139419, 91165.33199317753, 78787.14452062927, 93386.94122846918, 34403.754186455735, 17250.570150061383, 56069.10042481151, 41163.753051575644, 60842.5508015552, 62629.409947664586, 46108.671661434586, 29325.694663486545, 22241.075683867828, 25086.071939502297, 22495.746982244447, 84852.6347909276, 9513.369501746827, 4288.72451908483, 65898.26561286514, 28369.694645724387, 39148.27420915085, 58523.231762997486, 93596.13515572515, 23201.76247961121, 84626.80730768641, 8447.95628513849, 2860.082034203948, 48882.70545132456, 29876.269899129293, 34304.241084281086, 41403.13629739991, 65346.26127362634, 80685.31243546869, 85269.22206688394, 56686.74015205134, 72039.37637258045, 98296.5203666917, 91110.48249233804, 39347.38911056699, 51241.53461676243, 65211.39487726365, 30251.100890705107, 5538.492603878886, 15493.569884338476, 37977.29284888559, 52121.61755833506, 65009.93717206276, 4749.002575240735, 4885.94642637622, 87038.7929187635, 31015.685441271013, 5291.899934629085, 54269.665220406685, 58116.61646878056, 68389.93399226971, 32443.057843236867, 98600.2637898895, 90908.46504156535, 62056.20980558404, 85323.54799324363, 64757.849095794816, 80102.2888667647, 40832.3132104479, 80276.04993436554, 6245.262955555652, 50818.9955181699, 48186.26954291786, 49547.327878646596, 55162.33361957961, 8125.5309611362845, 78144.71985431865, 57210.128717513784, 92045.73903203031, 45839.52938061758, 70322.07921844203, 2781.680986529145, 68671.53511842819, 18106.262249805626, 72455.38662561859, 18932.24073646995, 83804.31576543901, 10937.527031218886, 6687.056033607785, 14027.910824410617, 30684.134607006607, 41944.70072410169, 78735.45997644789, 67888.33295440681, 36065.71295411573, 91193.87111030317, 32855.081322648846, 78066.60393719359, 72381.1308928691, 76837.54766761519, 5707.147688437819, 72818.7649122146, 55676.00185574907, 61566.725379760435, 92014.02164760642, 81071.23626875089, 91769.69947972878, 27759.42490742118, 68121.40520893481, 18638.91279010603, 66991.19275802893, 60933.34569283731, 23096.068758574274, 13445.245474762212, 91352.62898272036, 80753.5543390725, 20929.02799869627, 46537.12703005525]} +{"id": 1404, "vector": [17548.241426181343, 1509.2913741673542, 18981.556328566796, 99727.80982450963, 25608.722751203517, 54965.723392052256, 52990.69100721142, 34631.00970686168, 90496.47135037142, 87350.6158437515, 4630.939277010959, 68620.148124143, 76770.02068713539, 79162.41925075698, 79651.7128774516, 71343.48023064648, 84740.723080335, 8924.784105866434, 99496.62545398861, 47158.74521736886, 92011.3171091434, 60635.259240592335, 53888.57207003094, 64658.91092310039, 43221.32464097151, 60586.76572314662, 33529.753221933315, 63587.04974370109, 44001.86884357556, 7232.716252700177, 81211.45328725572, 24544.84400790683, 3317.775200956763, 60497.14778040366, 71988.89934990779, 80003.78375073709, 24432.332558959835, 3878.1294316534654, 41732.13156771859, 57297.329561982035, 28626.49903658222, 24311.495214818602, 67594.61784576252, 65500.02866757477, 3091.658990823043, 98987.2804989639, 68266.45207397305, 47062.18215573314, 77907.87381200737, 94096.83336471266, 88931.06151578056, 33665.231017656624, 96559.28471375475, 71204.13590422492, 87009.16964320566, 299.8004709071078, 47753.35164465486, 17271.372480605372, 60531.22430909139, 85581.08376200826, 5089.046736696479, 88880.14944017444, 60751.9694456374, 41253.82693452987, 85390.99811588175, 53325.9184609852, 93859.47001443506, 36446.287403748036, 55179.04376082167, 51708.812202788955, 21519.588822851878, 42827.27873933828, 87245.60110935506, 77824.47352653777, 73958.33664165018, 82056.72648775976, 78854.881752444, 29958.321868815463, 72207.7238125507, 38994.259030064284, 48049.95471461293, 58283.73965411221, 12679.014152183188, 18861.43924286544, 65898.75443297236, 39723.053051503455, 87897.25714408518, 83090.93840109339, 11745.165748694664, 25851.025492793633, 81700.20991211834, 50818.84317951926, 63033.357722448294, 62324.73643406064, 8630.568149112205, 64424.49210218154, 38673.50220637784, 43226.163313962075, 40791.6530002444, 99991.2723626727, 39966.107185798995, 68193.53120487202, 26713.484145867616, 75448.1237438348, 31039.147001264842, 21178.397524385397, 54379.98903228125, 5390.373687310579, 37914.95926949135, 15260.478655210607, 19818.521805726865, 69540.70799343486, 20344.885865708486, 93454.59319207257, 35227.916290486784, 61614.36641185979, 47064.39045185252, 75204.96311086035, 56614.49808946449, 35685.603694737656, 82994.22092207163, 39744.69609064887, 58171.44042554053, 66108.21313269537, 91632.44675396461, 21781.440986615562, 12470.937224291301, 6593.557938725936]} +{"id": 237, "vector": [73919.94439106098, 69662.30707017897, 60372.32887079239, 75250.47513728598, 88416.38849801633, 17109.48689228523, 56307.576077055535, 5335.958440419597, 86148.94235906057, 68541.97895583695, 3011.753585380217, 35364.68479753565, 82309.02493366, 66301.04373633453, 89773.26318501704, 61659.77155817668, 93334.97640089499, 91418.9476591198, 74997.99476962333, 72275.00208456142, 79723.75313755791, 95462.31317246384, 51706.28040260622, 24547.07297076093, 15443.458937722953, 3291.01368102962, 44862.053904858454, 49079.53962532411, 89030.60307129213, 36921.029560588504, 54682.34768863855, 40362.472193005204, 9071.658003127459, 94268.88801459128, 98949.71556528332, 17207.94584587767, 56961.08299038612, 73309.84174221748, 32324.648737178275, 88835.98115844137, 25183.768786970617, 83543.04736776854, 2057.2078373636837, 28622.577546805805, 54468.02694561679, 57224.92457192787, 96807.8377638566, 16106.883769400238, 71181.90352346371, 29710.364234083074, 77098.15676927681, 51070.869535409445, 35378.51901038169, 51249.533001142234, 81515.44422794727, 56084.34656734964, 21332.471380637053, 91777.89814711994, 49562.92870480516, 75117.57595005329, 731.3907904065942, 46517.8434427615, 20234.232215452896, 84095.41760546922, 13159.663624233986, 27560.550623301606, 79153.4758251094, 46768.24435727974, 19672.16563588574, 43901.09882796477, 79094.32214335886, 51357.06745786064, 92608.08052265135, 47502.94278993341, 11232.405232872945, 51043.451704907595, 13485.489572642384, 33545.31195328081, 23675.146112830535, 70864.35293080096, 81976.68348706339, 10769.11831561429, 67655.48723690587, 32134.89435145962, 3602.242143464973, 47684.07520886756, 30532.21704872523, 9194.48547799212, 64154.24748116926, 51521.59344430185, 1502.379303706891, 35981.28707412975, 25642.467477511767, 35427.163304185684, 72077.40306258357, 98262.60998856832, 80322.91832329147, 35028.72603019699, 74147.93852594156, 59707.82094050954, 20472.74986004802, 95309.9254455275, 64760.23118774632, 8355.70846504623, 96253.12513730342, 20974.53273976406, 27007.471723089493, 75954.80936602692, 87972.88936079401, 93143.04644202645, 39647.84446209958, 58193.59315041781, 33743.7176782734, 51031.61476581882, 23537.239411792198, 28940.494377993396, 9735.595902763484, 49403.94203150899, 38270.540214883644, 92867.18670591239, 96863.98492056377, 65141.18845582417, 70345.71965822935, 85941.54389300001, 15617.843571085876, 86182.27460286215, 71583.98068073145, 35541.892841338]} +{"id": 2107, "vector": [80896.80205453323, 19206.372688471372, 79322.11385269948, 87670.11134176787, 6296.991812886144, 29092.75572933946, 24363.116219460968, 56410.21214438823, 4183.651866848625, 11845.35014518443, 79386.38181364203, 25680.050523028363, 78728.62577262055, 38093.78733210741, 72630.75711876016, 64098.91371009759, 33994.59879298233, 75949.2573007724, 40312.203676480196, 74605.92354546054, 34065.60182538939, 57820.05161827354, 41175.12259027795, 42537.94326658273, 9977.67867749162, 34688.335209877994, 40407.4143465517, 4643.980545491578, 37515.238457569445, 39981.8018704533, 46798.58262116979, 79175.2356941822, 49985.5617399104, 4718.179721080384, 83106.71980520523, 96538.72506090693, 86554.69644063742, 42179.24481021257, 13009.711531069024, 70687.23432881806, 91384.16304906605, 91276.02660483819, 34016.62401439024, 88404.86042336797, 63924.15012020022, 97723.71234829052, 85126.32103256816, 86499.43793083589, 39673.17709056047, 29765.696071340186, 73470.35001653327, 54728.60278250487, 64540.5099825675, 59085.45739099415, 39694.82780353016, 44680.78316939223, 66593.19911887369, 90837.14935788614, 31604.53615613341, 47543.31763407604, 38577.673217313815, 38261.81282044536, 75557.24361592671, 44022.332015480584, 37167.51450940129, 93705.25653807225, 14207.899437364913, 3420.920214464873, 25334.24065486638, 37635.36220262562, 69747.47492607715, 34911.57196968324, 10701.939714446307, 43875.42281293544, 13741.711184333293, 39417.35397548066, 42002.0083569794, 97528.17335993049, 78702.88296544597, 47861.41449344288, 87210.57528978695, 93549.1150363141, 77953.36167335678, 73002.20965175783, 85010.45831190851, 97631.3785524777, 57537.49062668637, 56826.061629416334, 90594.68892513758, 95734.51736045236, 10961.6583663804, 62952.88086459543, 29621.439141048257, 67328.64635766123, 59408.92606885062, 83529.00452550208, 5133.151880602493, 65312.79520820636, 39233.075684331765, 95153.80798784267, 29580.237851933554, 35798.851304461765, 73010.17074374252, 87018.1024885223, 61472.80445404889, 52961.575650899176, 56682.12028682774, 70642.10156482327, 78768.97996898963, 83780.12278588806, 15051.88227274129, 23924.299175015796, 42043.08528293104, 94205.14778290094, 24237.701838256708, 52945.17434826652, 78502.55224828361, 77109.8231648886, 16775.504121626196, 58272.673113555684, 36512.98581532954, 94592.4588388364, 21022.02278325881, 28589.990825454657, 11559.575775671594, 68154.35768836469, 97534.41732334814, 26591.110996528787]} +{"id": 713, "vector": [44291.54212329881, 43415.588818855285, 85664.07690352997, 60179.76961798422, 99685.73010816798, 23812.19720827763, 76314.84707328666, 70152.20773818932, 54409.078111773655, 13071.720112330187, 2228.161239169246, 58441.47386594441, 89526.49401288238, 81109.77292950718, 11865.248326762801, 50745.594986202734, 42200.39824346849, 29785.227578689843, 6971.266136951371, 88727.93668689238, 51364.32960830729, 59861.26174538745, 7783.830566362104, 77532.81319039325, 97974.81847728534, 73071.0415443199, 52552.02041227626, 75772.98615029262, 6152.740416121449, 27004.11755247988, 81338.82154912654, 15464.951604247857, 4772.657167129779, 2744.9246973780373, 53330.980305463425, 9422.516666579295, 16232.256821176294, 75747.38824490325, 4642.2816735788965, 28277.84084095306, 76432.84830330385, 41333.74764812321, 48444.62400278397, 91850.1527676025, 92313.51639875215, 24357.57758936562, 65513.9131606584, 50263.35445332246, 51721.14584660117, 51302.28129671809, 56805.96734954775, 52660.09379225909, 97003.89885679923, 36818.06044301164, 90248.62079162011, 81485.75196011098, 13013.480699863356, 63088.58334362604, 68878.45765712607, 741.8741872853963, 64392.2415755397, 73813.21353392609, 34705.382101237905, 56014.154633490645, 23282.008227460905, 87825.29440392341, 36652.80928763127, 37609.60202053507, 88645.40664871463, 18194.589384600335, 50798.46286121221, 61883.761288474314, 72594.68317306238, 26953.68203752979, 56861.17167213129, 58604.47953590109, 54441.53577202028, 53379.366880072695, 85200.15007973285, 61508.196045578, 95888.40911116953, 29366.194503745814, 60214.73177988199, 58133.732887797276, 57355.29303411624, 24432.12161281799, 23506.291598744756, 93341.47625617613, 44347.92393688086, 63779.03998414995, 11805.211442194885, 87139.89925166013, 86558.268872307, 74623.21087574646, 2807.0722295093774, 72459.85652820136, 51036.69946715462, 23371.647777298454, 37593.83262876081, 60208.579607395186, 25305.333934020502, 23316.5833723567, 27405.833354524566, 10115.231133547697, 44186.61932330866, 84376.91312236885, 81500.03346520357, 90434.82907984307, 98829.6448146, 12481.814549942526, 90156.14261939982, 40467.892280690845, 42622.36132022109, 65037.53282559982, 50563.86019186808, 55194.559470487606, 52590.37845566278, 26365.83077101681, 36947.88319545301, 81735.191358315, 64294.364843795396, 94594.12805465588, 29181.040226107136, 40002.3484481886, 50492.12147388786, 85189.7657242497, 17242.161630232076, 4397.08694819928]} +{"id": 858, "vector": [98627.94680550213, 28893.66361793624, 25829.521363757813, 35724.93924408322, 1299.420331979362, 52118.71583043614, 34966.17827220802, 87949.03099700954, 53233.08118380582, 52097.02319405884, 53182.59756122597, 22872.98022212565, 86771.73243442414, 76570.88406636122, 93117.64506653776, 17944.39203096704, 3565.4936220653854, 85771.3752792823, 21699.414062104348, 73917.55627110928, 97539.49779393166, 91753.16056670444, 18179.53922663559, 968.7608975873618, 18008.546766900803, 15628.248950169977, 38856.13028533717, 54822.03791092421, 88611.20881889523, 34487.35315476867, 29349.712632711922, 72546.16096645944, 16462.350052927675, 6713.238885641759, 99648.12645026112, 94004.07879924128, 32568.403572838688, 28941.31589332416, 51202.987239918264, 32774.96169770462, 12869.543731123113, 6151.166790220653, 79115.4888304405, 66441.40035216814, 53675.26668848196, 95239.8017449443, 46589.04138718798, 44026.820851590346, 86348.68380810565, 63543.53058327596, 28473.262778290074, 39256.65235228274, 97557.50276114795, 35711.17830997767, 91560.47760589475, 60821.5543951955, 2995.292198526034, 29768.20691481178, 45360.69478810263, 21427.83219714348, 85122.95600406725, 79859.1886662266, 22642.547741017337, 79672.51768874525, 38200.77467161485, 42832.69350807156, 96148.10581663427, 22720.793042890673, 42307.01791641497, 14913.076044211515, 51262.1880014066, 32501.11674403393, 5645.351951608913, 82083.42542402218, 28489.018063710526, 64202.48695493377, 53296.99051737577, 98575.52172774376, 37141.37906113688, 95584.42249233474, 5227.731946477187, 88442.75966302144, 69560.60564538612, 65689.31851404217, 17152.932662903862, 37921.49935812999, 89189.52670214666, 40866.85582045516, 45201.420318212186, 37434.83494609675, 22924.833443002946, 38963.83049767185, 41773.08993440021, 48991.34722055955, 63425.34159778332, 58958.67922187683, 40010.533765276254, 45843.24488454419, 56287.91197220227, 29902.62525312175, 92451.76831070018, 41483.54401401535, 69596.37740916187, 90317.64108549824, 89945.91983343464, 91446.65392493847, 65105.72768798432, 51321.66851853741, 57797.53338954794, 6566.953378312967, 89368.17176807552, 59508.323344861135, 3186.008951694408, 59497.5067072895, 98352.51699135631, 94333.51202949269, 3382.6665900279827, 36970.52021117789, 48242.01068553335, 36717.84303270279, 56756.06472075539, 13197.832162581102, 6005.72911572319, 58227.18215579486, 74710.024454586, 41239.42553003015, 3511.8566817351702, 93992.00294851656]} +{"id": 212, "vector": [83603.541944037, 2596.705963783774, 17847.69274664687, 30972.157755088305, 12388.785959712312, 16705.76787582968, 33443.959786844265, 97411.94312852804, 74595.30222815974, 98107.77718154134, 65586.17429232805, 99041.42164231358, 54429.30532316588, 30429.130892235557, 98614.24292980648, 24294.627621168995, 733.0957971145247, 37600.53220931734, 47160.92909988855, 75922.86225124075, 15646.710556499676, 19279.95746762787, 53269.28208998038, 56255.34611597827, 64759.583606683336, 61046.13452414246, 85825.8906796634, 61972.28596171045, 50513.0302292118, 24933.39681826133, 34786.29336924851, 32507.49241345724, 4510.904180709596, 92785.38032778681, 61771.39451368881, 87530.12022138534, 48378.76738077248, 51819.60303449436, 26180.732237535463, 91904.49737378945, 42795.90970390469, 38693.3954329176, 58921.34364214115, 93340.11175326578, 58877.83925134549, 49222.395388585246, 47610.07532296372, 55167.726561557574, 26494.55719299124, 10558.784649065223, 53500.432600067885, 12273.914535682106, 45690.21738785441, 54184.52928038438, 27302.03087661004, 12792.414760256754, 93124.96830510246, 4393.45569262104, 62487.69528072998, 1957.8884911447215, 67315.19050288788, 21525.482683201524, 62883.443931975846, 85943.24088657256, 55582.69516650043, 27923.52736777498, 63759.234588026724, 87167.63068041258, 31213.34759687191, 4435.657366048129, 82514.37286198273, 86898.38269492328, 66154.26429081788, 2218.6255887569837, 8881.99732634467, 83106.49313082223, 94172.852497507, 72789.5072879043, 72138.74153746809, 37758.40781240929, 30784.072148903186, 19821.984309720097, 38939.79568883969, 1122.2235871438113, 40581.94199994576, 80526.6298582258, 35599.56591149498, 26960.641074668667, 12699.661215718339, 46459.09253158849, 40369.65158650742, 22843.147263330942, 29595.634884911513, 76555.49497394043, 29201.729494393945, 6979.9656627635095, 94721.46866600578, 60261.95565443142, 30474.195297571794, 4849.169251401997, 86071.2476289096, 88661.54313258082, 84149.45129101373, 5687.820111695163, 38958.33498120186, 45747.71475751173, 77225.82140240773, 32923.333915386924, 16388.795272296586, 75466.56534725578, 15833.333989134191, 44489.093856313186, 74034.91413748461, 85354.58078731374, 47840.64020329112, 47102.49535071962, 96567.2683527981, 88254.84755947656, 25260.265463031028, 89366.71989683609, 50495.096342409284, 16409.4467436494, 35469.217751208045, 3855.1889282934626, 36798.673921534486, 640.4839514970839, 4245.012013873872, 7201.125016061349]} +{"id": 157, "vector": [93308.39045352835, 68008.76112749487, 74193.31292470336, 87212.16204229453, 33133.32866749545, 13144.440145927605, 44815.81257599374, 96280.98657051529, 42027.80552470431, 64654.72882341033, 13958.518081782056, 63192.16217794295, 32578.05230322248, 41263.44588613076, 74525.41279946353, 42992.652860622, 78152.62107300754, 36498.193243210095, 79431.9601137671, 38132.15900801023, 5471.433604270593, 71131.89799032455, 58849.53330014432, 78445.5128632425, 71188.97096393639, 46754.654057985914, 13450.44377363035, 43852.89460577005, 5944.61004663327, 61851.15329531393, 90382.71761219316, 98663.75966699555, 15154.450108219264, 4927.889918423267, 87746.97787906833, 30945.464643879706, 52224.03899542094, 48827.50547818323, 28925.856718658273, 9518.089191137435, 672.6966473024398, 91431.82596300814, 44670.40477468552, 12146.342009908161, 70499.12430541869, 41111.646051696414, 34304.293387133366, 72076.61796524003, 33322.57071794008, 70073.71875944856, 59331.03615349824, 10865.946091190892, 21757.804910729737, 17631.2008922599, 18269.49684137861, 33218.32880910672, 94542.6398783896, 59640.73576248019, 14600.310239060755, 94015.3714586955, 90657.03929422951, 76538.93856657909, 49937.737937392376, 54136.45661363842, 33487.857110866484, 60382.1133152094, 48033.84045108935, 8321.329486462713, 98139.4987508832, 4349.544608162281, 97266.22983446065, 37647.05077668413, 81938.33274348949, 75267.53221814688, 3130.098947629567, 97068.7415635573, 31596.60813767442, 54112.74994428885, 85752.72338868829, 63361.96868392156, 59802.41435117751, 79752.24126633837, 60878.94152306693, 96589.00518397646, 4996.458401996729, 71964.69790487537, 71511.55632986831, 20274.04812452467, 46306.08866968595, 30401.39421665361, 74690.08701868643, 39887.88187630039, 57607.926897645455, 95440.99278914362, 98816.98044161458, 89250.77300565082, 44240.15053717093, 6019.272088108208, 34871.19768581039, 55708.290840082234, 55761.99495799841, 68170.79808469923, 39473.71383489012, 21807.471995805507, 19506.695228836423, 95709.28709180813, 31319.798741261005, 50318.920932280205, 29752.84481800684, 92005.18540359646, 7700.046662567639, 51480.336115262995, 7177.870145989307, 30593.26836528974, 42879.77629004126, 93629.26916098013, 70375.09239502622, 11248.183842386294, 70594.31506785787, 74327.42716556256, 94752.5536253243, 7050.1087351168335, 39267.910713576384, 68952.11137061733, 63299.01926001968, 1443.0640595821221, 51918.84053664643, 65871.06059769908]} +{"id": 321, "vector": [48824.7695432157, 70608.87405339789, 65342.371539960455, 86638.07364594504, 37212.778636362666, 69595.155557298, 55609.61631084793, 43305.916582865044, 90097.74716336049, 27477.071895778314, 41713.90681907711, 17266.56147539719, 97418.20657258076, 81225.5991830366, 27551.01676019246, 22405.936239137747, 10568.982200504306, 17741.083770846322, 90128.89121413397, 5394.257414771164, 21289.659794077364, 11684.735613669784, 29620.55508705067, 12618.116095102205, 67149.33381819606, 58564.032259057974, 22788.11444758725, 92520.82779770391, 90033.56530150917, 99103.9030779216, 69001.71163893292, 86741.61210118362, 89891.94657874812, 15252.814910228075, 38082.924961670185, 26087.688318326553, 89251.06583811209, 53785.41539103232, 58496.53731554798, 22893.277562542826, 80139.29740839744, 94323.06949100214, 45289.89882408765, 40368.91321472436, 91767.22080758098, 21665.459842131062, 97523.0595422195, 61741.4389559909, 676.5336061146598, 32199.717403339357, 27149.95100081532, 11722.756284213487, 32922.958988413986, 94480.2975896781, 91361.11891641066, 29863.58808640399, 90101.97140093191, 13928.966864742753, 59356.61339095359, 24718.482097193682, 63903.468215958615, 84669.09869919234, 81541.09662373702, 63020.632343293335, 12897.53476766532, 87119.94148236414, 1876.1457033944716, 70883.86977744657, 98768.45085521364, 45892.58231104454, 15461.035332366491, 82808.96904081527, 54280.11948559711, 51710.64544104325, 36854.15953514614, 99578.33371433063, 95612.9705275362, 52619.0697421307, 73565.52860103645, 72101.05310564392, 62696.088565805905, 24987.298811526394, 221.08887938744326, 25435.375216112876, 34885.53508417444, 83130.7587376095, 2116.5508046644964, 54260.92694826472, 73524.62729199264, 354.4828307305292, 62818.58788462075, 39408.96070428463, 16358.356825056386, 30013.38477853035, 135.90947492000404, 69725.83999336581, 44605.542915120954, 50188.27158732251, 88234.69407313655, 58220.72354606864, 93908.31574340796, 49209.74075114045, 75334.3709796997, 59379.94895039756, 62083.8050017946, 62960.747965946095, 37463.711778954625, 92218.1171490147, 93834.54860312631, 59142.97689511454, 28756.119504668022, 54661.23867777995, 83070.94753981054, 42497.89360528833, 46891.76844713847, 17290.04726725477, 17030.76161915037, 89557.19724384745, 42759.63289724178, 81527.89496554673, 41292.60021222384, 75015.59416774362, 70428.47321083238, 80017.32913121559, 83285.28737397594, 93598.58198223893, 40355.3261765051, 9581.91777287123]} +{"id": 1212, "vector": [3163.3621199715135, 16303.166930218782, 26747.557484513396, 17606.91406477095, 7193.724692384684, 31997.68048767463, 62106.93805491776, 68593.46631797605, 18840.364149499666, 73408.80746497422, 73428.51873123372, 4020.0270432569796, 62341.84486497776, 38510.48157582439, 72324.88259465547, 26790.63250677609, 34450.30947436681, 70292.91427190888, 10805.60636569221, 53466.813708410846, 10146.351442921243, 22588.471067947692, 50637.30739413374, 10902.725335946061, 70035.37871953708, 12443.43486913757, 35223.55421239921, 14972.993016087854, 35481.47788309777, 5541.046853594245, 7088.3384034765795, 89412.40899569672, 4902.139006037664, 67021.18621952592, 91047.75685869459, 60972.562812549135, 76916.22787465987, 85521.7327021943, 56267.4799444482, 43183.72775266288, 89608.90018043523, 76484.22644462956, 86754.9029760275, 22170.45419902094, 22661.788132577574, 2438.188100948602, 58617.710843955196, 66038.69679407444, 9985.626421896965, 38160.61849138749, 36208.19120483277, 81062.28180298039, 16694.028763923863, 32873.119122464144, 37860.05444320128, 51668.90433229745, 42896.59423300012, 53777.195531735466, 91631.18095845246, 47008.541165975534, 96215.16564797996, 10977.783394682172, 9207.354863040917, 21110.960316093162, 59259.66073624417, 41647.40772871809, 39979.79353090558, 45275.108185467536, 64459.30409486009, 70266.81389147764, 84540.75000943242, 62522.13219481514, 65700.501329336, 70470.7683468324, 4662.173818561233, 42411.07776979665, 65726.64772568241, 95661.52189200552, 44882.51147759151, 61889.22817291058, 552.4027542260445, 99389.84549328056, 90586.3763180918, 57773.10668556934, 67788.74328240335, 28439.088676073254, 41926.64045164628, 863.1162949806037, 22133.807954636664, 83977.54631231242, 39908.34836469216, 63255.572881279324, 98036.95018889071, 64386.068652667294, 64683.727489078825, 858.2080350769905, 90492.83335486494, 98373.79349315856, 46509.83617743737, 33164.43023786149, 22451.862669653954, 10837.157216035597, 13794.976846717833, 39405.80367836977, 94813.08343103743, 13546.343859282439, 28490.128856367348, 10622.98369512812, 64823.29689342545, 21635.518417428324, 21304.687160679194, 95951.8607473476, 58756.4703450201, 12853.0597658851, 65338.03753900507, 37597.30194433963, 48652.16290024806, 79122.7333579485, 27735.844201861237, 98532.73864744682, 72120.72773938748, 91209.74264933617, 25905.51895864014, 52864.08881400134, 48875.744524729445, 87119.17964703981, 59443.936269625345, 77470.46361110079]} +{"id": 179, "vector": [50994.39113426144, 28061.143761254083, 32437.788126613355, 16635.44603437319, 68779.29132303348, 68206.25853665233, 63001.29935043633, 61424.38293523272, 34155.910117601685, 41030.257973861786, 38044.82686896435, 10818.694992322919, 11430.42072684649, 45790.273449057706, 56837.452453876635, 65584.03653885468, 56716.039494082426, 92714.92819483497, 91289.34203199406, 49629.31047988288, 4082.2778757506285, 8847.778479570434, 98397.24427419691, 78441.76537059441, 43371.95035309144, 76297.35310208278, 11052.529446222536, 96095.30210469566, 88363.66537128437, 81415.64277636322, 49065.09586768909, 69035.44423217273, 67902.42775428071, 17352.281537594437, 16104.215190051475, 47852.87057475498, 74813.1660612959, 42044.48628301151, 16959.049733711974, 80457.01474701295, 89160.02500057117, 10397.599077079522, 94506.08992722834, 25395.888252626064, 17115.05595112496, 73148.14171921053, 35467.57230436789, 56878.13319409827, 17084.095251015708, 2521.460854275426, 33786.91389514264, 99633.44803190422, 46882.49015972431, 32998.85925254391, 77582.68936227661, 4059.4226527512656, 70884.38130804675, 52112.718925132525, 93695.97901447346, 31506.914395290707, 14642.448532501618, 82809.26810985802, 6158.963485595114, 93928.5155982253, 41370.964783244, 34003.84950225359, 6791.587113551523, 51865.77201769144, 9564.60169807547, 38277.54903461619, 1650.2884212753477, 44026.41503067154, 26224.65212850763, 48959.4709288488, 87280.50544699968, 81706.85353561526, 92538.94485773447, 86273.74448925296, 37762.75408559277, 62615.84047397629, 10175.20707450269, 82436.39989502514, 17739.26785728288, 73750.96643360966, 66501.05712898026, 89307.61345044858, 13732.944128761172, 84869.5651963237, 48546.2261290876, 30772.454698736685, 63175.3643324091, 54038.44677855341, 56947.10062313298, 93843.89295454464, 36666.966970798254, 74270.61920990271, 41402.687501882116, 28475.295888268214, 75705.66947710459, 87658.90823301054, 9044.076323173356, 57281.11470208442, 54873.202563319515, 16008.00553189644, 78775.87672615697, 2260.1559107527146, 80516.88194308546, 49657.856094022565, 46850.093555735184, 55512.53647682659, 74651.73471237508, 1721.9345545893882, 46224.341326453854, 15335.53911272828, 95386.5451111278, 35638.48688100141, 685.2605334955797, 46317.536405180836, 46456.946934921914, 45016.41944937579, 22573.78139286298, 22078.258192693524, 87005.12248403272, 59513.88643069511, 6101.041952026998, 84468.98356648264, 35970.83028776768, 79716.1618419001]} +{"id": 238, "vector": [9649.412746931273, 17694.41782658778, 19154.867565461598, 26528.062759049066, 9170.414507722668, 34754.67843750867, 52159.972397876285, 35803.24918244757, 88368.99274516937, 88936.40461993139, 24418.853970359112, 91942.66188488202, 20200.879891489574, 12947.762151532472, 23495.760272925527, 76699.20902380794, 67166.32880868464, 59558.64603263451, 42423.782965524224, 27182.954550716986, 46238.61246601004, 83709.32476702168, 50007.67270013756, 35324.44516863484, 34320.389406300645, 26181.571737995146, 25920.596083750792, 995.5315764095141, 29974.12349326569, 58980.623344916916, 79576.65419125037, 64065.19546169501, 2349.5317954447924, 85905.08298689389, 77111.90025744558, 4135.1298687439585, 8567.843899283867, 73492.04467958263, 20111.90039727815, 13229.640521917874, 36565.65118450183, 93941.2780260362, 37300.378643449905, 85326.1034864955, 73013.86645343505, 28268.706079920736, 55841.45611127779, 34788.21952233382, 54434.60911309146, 36539.27450735769, 57274.17134666829, 93484.15798513824, 42363.29451247348, 10390.349729057469, 21791.311687443016, 76855.81486568073, 77227.13016342313, 92417.59595634868, 59064.852954644906, 6079.647595675841, 46406.470598026724, 57113.03264174223, 80616.62439469877, 944.4947167519091, 78403.06215475086, 95043.3485961178, 41076.95872683063, 6765.351438329037, 41402.00000554588, 13746.366390578158, 91051.79966796636, 5137.182768070603, 97044.49657164673, 86909.0159013223, 40825.62740516189, 24180.487064427813, 58649.56483346461, 4593.589024240919, 23480.862759065436, 85246.62162931233, 6898.152658509161, 30790.790851335427, 86072.36118334523, 39377.47598487444, 61544.805679381534, 96543.74097429808, 76405.0555710795, 94496.40954059715, 57627.22905062433, 94790.21030725836, 65895.51245862221, 72980.72665771972, 40088.198440394604, 89523.28877675322, 78644.30790539458, 17637.658617633566, 89794.64439824449, 17853.41147066989, 22735.157238347903, 4018.026151401433, 42414.10180604169, 28165.990132808372, 40557.46979048773, 19721.33797171368, 84302.77573515868, 5447.698497449937, 87793.64895614324, 68624.56478312006, 72434.69350355915, 86228.90902948921, 48269.93320193355, 43100.98115844333, 70138.53911066322, 94572.64465602244, 91258.5553146277, 94361.77693759411, 87320.57678631801, 32121.412219500555, 81041.05053956159, 55430.80304842398, 72425.17494136418, 91875.04502960604, 43631.04467514448, 62984.18800254949, 36032.649442827584, 65948.04371773625, 60707.854554689846, 46322.94689699844]} +{"id": 1978, "vector": [19827.965985514616, 73315.83883726106, 45185.36363917438, 15142.604285723794, 1720.308724460673, 64265.950209377595, 65602.82831835034, 40109.4796966351, 92217.54863960927, 92274.97817318814, 29504.622779351896, 67578.24788987523, 48867.59951994876, 61964.54806014453, 99142.09221030597, 47659.955775993236, 97142.37742851382, 34401.58222823783, 58780.22338084024, 62953.09399461869, 36074.58945715226, 57754.53074769312, 13815.4514856622, 52526.808030298664, 40107.76976469598, 98679.41487018263, 15693.54207746877, 77356.0398792298, 45545.3266151012, 51244.18421508372, 55038.831779325956, 68791.93216838881, 73030.58726279675, 94262.90888731, 16609.71806164131, 42312.07231630659, 70930.96959204695, 96874.29453945433, 72257.11690909171, 36963.36418727825, 65454.8348169325, 42377.5642244176, 4003.5354956767133, 50576.5609081703, 99341.7271484092, 36343.15973202108, 65381.297151121944, 18171.847552726173, 98625.59593820116, 27720.910559087584, 7080.153489683538, 84751.39117212342, 80124.49614503703, 22299.366885555693, 47457.96606001077, 30220.82849141441, 27100.094147624597, 54134.67896402693, 39992.6166870425, 25220.645583121626, 48899.481233743114, 76345.77533087638, 64931.624161763946, 18740.788853676047, 81877.30218708553, 76628.64934940654, 97248.91097649222, 29135.73395415573, 48465.332817792936, 98552.05977838024, 48154.29330032266, 43664.46352729135, 98810.19312830576, 38622.600765133764, 64821.739038466076, 62311.41643096227, 63068.5005148186, 55678.91162001158, 56927.40963319478, 89532.60141590149, 66307.50600832734, 13142.66802093983, 25951.847531672967, 70738.83473163968, 43.607498266495796, 24213.672403219367, 67290.26925817046, 61839.474071593715, 15241.153561551624, 42957.53079547495, 21470.61480116106, 62122.24194765675, 14990.78914589258, 56842.74547881214, 988.711806339948, 34712.45056882894, 99601.61685312664, 91372.71064755622, 52241.83752524, 91906.29326871902, 7557.70333115936, 98986.90240546384, 32314.128417268505, 38988.02960005357, 70280.74744116973, 17248.32262085557, 17019.738571611088, 76038.62014854528, 43763.47173369729, 23945.652562403673, 56591.494228067415, 52947.397239247206, 6123.2720177548, 52736.17617232681, 54704.75879312482, 23659.002758695467, 43178.331791118995, 94533.47274470441, 84848.79275996884, 62187.62160705183, 84140.35231970363, 10268.868704096101, 41155.706405998506, 2404.4446606957927, 57137.131614815706, 55436.498347107336, 67639.4244467887, 80848.55252442056]} +{"id": 1423, "vector": [78214.91975157787, 12211.754064772307, 45469.93449427629, 63660.25535455204, 18874.072157160204, 34343.21639931769, 97276.43720089221, 15922.785430857899, 76799.50641711654, 80640.72161544045, 87767.8818159286, 44071.40794036709, 89066.29772149431, 42198.56949012023, 60959.86852962487, 67958.36218029277, 50727.609794873904, 82997.99896211119, 43989.123339444515, 84880.81834615344, 28280.674441358857, 75078.12130682114, 21621.550573727844, 74448.1212066208, 16116.808607455214, 90575.10588341844, 95143.85702065595, 71290.44063695034, 95679.39401223535, 71557.39236909125, 30210.39081385214, 34900.23062371666, 9769.317674189404, 42342.69316407698, 7724.011186883972, 51141.9442351341, 2819.839542499758, 16483.572997069372, 18707.477102347468, 23798.344532358216, 62098.26585238801, 20759.160623557404, 8656.750423269621, 3365.891390326925, 23952.77328498826, 13857.363416143575, 47489.11148939997, 85512.90147699499, 1546.23957758796, 31259.257060697255, 54064.822770593615, 31917.770032105607, 3921.1640710721363, 49476.884768386866, 45468.679223406405, 85852.13198830053, 59750.20350610457, 69120.66877881832, 98307.55601465073, 35227.27446054291, 6137.1787607138685, 16508.94609864204, 92599.33496853447, 31542.736965821237, 92339.42271745783, 13840.239638111618, 1258.6549295386162, 19725.080935835158, 18388.908752343315, 38491.03617106767, 78976.73257618054, 42641.52948043829, 91446.56804427062, 74738.28948514113, 40348.457006050805, 85083.51571215862, 61909.4620703002, 58899.18729998923, 1127.2678044557226, 64657.10100735538, 60835.080826914425, 66863.31424485499, 38402.192900059265, 52128.37325550399, 27639.860216333855, 77645.45386131394, 40923.31449143535, 26215.786710873424, 53097.618717574005, 49841.13135100904, 44743.73409843049, 43046.82643142619, 38674.49454726991, 73386.03716349766, 73942.52480661699, 34140.404638125045, 79023.2789977693, 64313.99612408065, 66193.89141545292, 87639.42099486303, 37076.040565790456, 60520.82991079221, 31594.215652955514, 31675.736835541524, 70710.85042892372, 59799.49302505833, 35328.179390376725, 66200.77419975537, 96140.1195505578, 97391.96261295116, 47757.89967760031, 38528.487371713214, 48139.492095383976, 9866.910602753054, 44638.450248792535, 79562.36877644765, 40059.00719297705, 26449.2604895055, 61742.74854319003, 11726.877093323106, 44928.98725680606, 55660.470417518605, 9778.414220899002, 40865.01756168159, 29854.931979017598, 93746.55859055581, 60195.4558448896, 21324.705800613552]} +{"id": 1492, "vector": [1396.652658337494, 417.5715852063111, 90610.72989501982, 47303.5190109537, 97451.5825517569, 17111.925432164167, 73812.19433880283, 29388.71560944718, 25771.809803163236, 62891.386744897114, 1633.0078440643492, 50973.350538661834, 27949.697043238873, 48224.29358951456, 11698.897930628893, 21588.637515341947, 34763.45482663235, 4989.968533574795, 22327.791870359604, 3395.8634952678144, 44571.45090316992, 87895.12014649625, 6427.243107334069, 86968.19383169348, 83313.40591432278, 43150.17245019701, 63659.376643045936, 34384.843270722384, 89812.17304084181, 44637.943359488156, 74117.64983706112, 95019.00457210695, 57492.146513756416, 21314.893509415233, 51629.53247280615, 86841.45636097249, 91724.3287120605, 99137.04021976043, 14649.711761659346, 89054.47506839776, 41766.340898496426, 70999.99879150705, 95949.77501998337, 10845.762853330387, 5237.030018795796, 79848.56270043344, 37205.55557235783, 73201.16749859476, 7722.636202760836, 25013.779877156696, 25255.81229547731, 9620.60276253679, 84317.89988487301, 46206.90739230476, 90485.83841086026, 61081.16666831346, 53472.886350103, 21618.110864971062, 32383.82782139634, 54998.66328619395, 17943.714819795852, 75932.01172095671, 42786.30214665358, 45168.04125337265, 66164.61471526917, 8494.36100574169, 85719.41253642322, 78393.50164013155, 96982.80880854776, 9334.56590632552, 64004.536907508766, 79040.15006230543, 24230.30953787948, 7822.692982690493, 67393.2393227523, 28673.613806043653, 21797.29068204681, 78573.55774854602, 53043.69028523348, 26796.460994700243, 94682.65150251906, 62699.097277441375, 79197.29834085499, 99746.41386553954, 3986.5097248449333, 64028.06705938207, 93373.0193950644, 99596.13399661762, 99076.16113594836, 44993.195802811955, 29277.137733824442, 53490.46082505816, 62601.74774360755, 86580.55643257365, 67924.78142057208, 66489.92646071984, 64273.16111480996, 24012.965719510903, 83781.94423467698, 11154.90170183977, 13619.826870282614, 5366.6309717313125, 6843.658053832991, 44394.44664073009, 12454.192875440196, 40268.24573457247, 1658.0842036354793, 88493.91753025907, 80020.71557113717, 85542.69951795242, 77720.28019034436, 88382.45775156948, 15693.77939988037, 61685.11425303693, 18622.31819668315, 56314.92274014756, 8459.870744385411, 52167.32785512155, 72570.3259014497, 34621.91657536785, 63099.71219724238, 67186.00773586037, 70.9143103103993, 28517.930679789006, 22447.166390956896, 65777.2865757276, 33041.06877558971, 55149.366405328496]} +{"id": 1875, "vector": [11887.829611476298, 24328.149400055743, 17287.04512727809, 68724.09521475853, 40631.39646132806, 28874.04219204305, 61706.62143239142, 88338.32211785161, 51820.540832230356, 61237.03545097248, 40088.20069864646, 98994.38571036751, 9552.186313571054, 75223.38655158035, 81068.01336707553, 89691.05079766396, 63335.60774012361, 72283.595132801, 22642.080128828336, 68733.75308778055, 55240.85876393502, 12247.60679406779, 88953.9549675416, 57047.055553891856, 39.59268084205991, 57662.21510296643, 97953.21832324546, 46705.92643001074, 88825.17917556337, 7079.163344385331, 41021.95428574097, 76410.91427360168, 93416.14530870538, 22236.240778555795, 41985.95043242141, 92079.44280441375, 18854.182192168577, 13671.798473819385, 30882.731176436806, 80665.67754868926, 72840.10097325976, 7865.10219567903, 39972.79499518306, 33618.28254269604, 43916.284904146196, 30367.325517255173, 40027.294765280654, 53324.16370498624, 89126.28471037636, 18490.925926018586, 81352.29039353, 18581.55766637043, 84277.92829075451, 36943.39267052309, 61502.10577041793, 34509.643856059316, 35030.0357490006, 24903.66689288401, 86638.45694658683, 76013.23504872156, 90090.60278209619, 94866.487783181, 92642.14579189236, 17664.566637789325, 64224.579012765425, 7874.126604546328, 11685.7140588017, 84007.53320719038, 40357.309855203326, 83086.5144523681, 24666.17182397791, 15864.082786603407, 22431.184251293846, 71032.4276692392, 96217.89729244211, 26319.15974409963, 49226.74336946717, 11713.65787302846, 45549.53730711278, 26074.431048352144, 62486.19392931972, 79151.87429311538, 20060.051527235122, 9855.199161983119, 56970.95277574673, 48474.74722392887, 35004.11746144385, 49119.1491526108, 64349.232138869236, 95062.39882957812, 71116.00617149707, 29662.805561825455, 72359.8499047274, 20900.844987481338, 22113.20874893643, 33022.737621763146, 12273.074676989982, 58913.53416874833, 33724.009865184235, 9492.129022693163, 33812.7721512688, 39472.65252225267, 65226.38680460888, 35169.33348287535, 37521.35305380343, 14659.640043499166, 82049.59137671882, 85710.09710137047, 71919.36843337251, 21399.489931738302, 60356.32414137373, 19456.677477659756, 50046.47071350232, 28425.822021963188, 3789.9607694671713, 47692.45844510165, 97910.26535609948, 65166.455625531526, 78000.5381728814, 76321.49241812421, 32117.899735915456, 79186.77264816269, 37346.9759414932, 84782.24913047726, 26261.398782120537, 36265.15142540606, 64249.34578261092, 77627.24494816316]} +{"id": 669, "vector": [69989.54250980962, 8618.837731627771, 23055.459721020143, 43569.19284137186, 77822.74933516179, 8349.619526039709, 28351.998745307115, 75060.20275355101, 28672.53497230956, 67821.00162786621, 10503.402390018702, 85313.24313667616, 93396.24418165814, 7033.639297999972, 3791.531261246506, 60374.792503276796, 69217.63537061644, 5450.2341638963835, 70653.86255368563, 26179.577892257745, 49227.09305988453, 21336.37842725815, 38037.807383498264, 34031.30596631289, 94228.43074636447, 41250.55553077098, 13930.62488255049, 41097.7749462811, 21528.930606841266, 43158.13143339559, 76167.39866379177, 83082.0753243466, 50100.72325533569, 15121.807609723104, 35566.55019663536, 71902.72583337735, 84774.33374702839, 47289.64858681642, 17982.500070016937, 2131.4892149728926, 98779.74442678025, 18621.115522407483, 85576.68951139036, 43687.66516480015, 1074.7748235728993, 28908.684722380873, 35613.99479732541, 98532.3700009777, 10034.813080964523, 86003.01970312206, 15273.866621829113, 60207.235767133, 70935.1024401894, 71221.31477878115, 49162.822912055904, 93577.94458275127, 22293.707774277493, 49195.93791699062, 39740.44724046656, 58000.38828286116, 14670.501332952068, 61674.35761826836, 19565.790813140982, 14902.032429557876, 63749.507292820075, 10950.936462641535, 24421.62093667034, 45952.600757094384, 83297.99528678507, 47139.39563610085, 69240.30868767537, 98702.12722663257, 97222.61206175426, 28761.81639403739, 77384.19537018338, 15996.08165510691, 22651.308106417502, 40645.06675724634, 2524.8281980009724, 49903.4321219325, 25969.928995670198, 79817.5549574589, 17381.96607193534, 31098.516323670676, 16220.344238908525, 23173.38037014637, 62876.38131290676, 39377.91441188134, 59237.90735675882, 11172.963922427803, 47877.85974689215, 76735.92740318237, 19199.032414531724, 53436.41820326611, 2653.4102820887083, 9485.432573940954, 40804.235994702634, 68913.13329236257, 2347.5503131587416, 47234.124507563036, 46867.69553564347, 83681.89383402908, 35025.756455349, 49639.28100823496, 20380.597479392425, 24629.10636543505, 39977.39501164842, 55843.138456536486, 25358.75689859154, 98909.79919398869, 64916.08840787343, 46659.59911032783, 72360.0438638642, 60692.76198105834, 9278.405543377321, 34856.3574922379, 35782.56450347596, 71806.34920786436, 49321.3201973974, 26634.878150833043, 32690.462199000936, 59255.228551846696, 60570.30857977399, 69191.30860443879, 2858.9519032676926, 66795.17073015845, 48521.6980895702, 34351.356546269795]} +{"id": 497, "vector": [37364.866201803685, 56174.626416952124, 65859.3330950773, 4646.15611041036, 67065.10537202861, 726.5973108551926, 53446.34849106741, 78956.8990294708, 59698.864407710236, 25596.89097719645, 37475.188670862444, 57711.94592431945, 42671.13618388958, 43660.520157125335, 28155.909471681294, 14977.11215821812, 51456.322002269604, 56593.898278272536, 60476.13025864634, 21525.951538541776, 15066.227659233777, 42550.99287455415, 39502.874729018666, 50282.49246107822, 26983.346082209082, 13348.850539742274, 43443.37090818237, 36910.136172117745, 3575.098021285128, 17133.068906962777, 62615.6783220435, 79695.53749108453, 5686.312046256226, 44772.90307629071, 44651.184162478145, 51289.60615101389, 56779.65376956496, 52835.74143511136, 20077.824135316656, 99631.72190435445, 1254.7837095609138, 87105.923643786, 77020.46577421193, 23239.455341888137, 44237.15175908194, 36210.26829504933, 71474.25196579238, 11734.21965951611, 72443.92316080324, 58954.20282589242, 89620.35284071273, 93416.04778083193, 16744.790784118955, 93456.18922039882, 74176.32311651744, 71916.06336724985, 42498.850419476395, 27343.621517391115, 88243.26025453332, 39212.622380125395, 60024.933160994464, 8412.239047222869, 86602.76102657428, 72093.74837106747, 20538.718687331326, 36999.12190014603, 48409.37546388617, 82175.93369817446, 34413.33136885973, 50992.55403606243, 95694.68505841325, 70009.179871138, 40067.95749242512, 46508.58830607859, 96855.91661919595, 28323.194181891355, 19100.97975935511, 262.07929740088565, 99660.47553187251, 56831.28924016455, 12534.080843166495, 24284.72300680715, 44070.84008666698, 53064.62316974272, 31456.95000129225, 69985.40928339843, 93116.11956927061, 16585.22811779424, 33789.26868132131, 69902.6604284431, 97398.24137860886, 77155.01906362284, 12371.335268417193, 75488.59089332542, 81277.90054543651, 81489.81896076935, 60818.50818205665, 33585.883742798585, 44835.602308209935, 65573.13531954304, 70613.5159865812, 31866.238765886577, 44028.53863270454, 45934.41180373948, 88485.02850466996, 4505.48775588403, 27056.479617570174, 93079.20712906448, 38726.64440778042, 37345.92377135637, 60417.60385152807, 70478.86824017738, 256.77976357275645, 23787.366027806645, 58691.6293209539, 16793.139229521814, 35632.812424256386, 7184.295351047876, 79009.15730743698, 14054.040695022984, 59526.62408785539, 95203.31001324313, 68677.93203914963, 4960.466653640649, 42934.74642157599, 78743.79393011588, 59442.40511943666, 58279.26176867826]} +{"id": 620, "vector": [44534.90088992258, 61266.2181511782, 83474.67764920027, 52352.328040364606, 2619.9297976572943, 68352.40580406661, 11492.570643523759, 86364.22724440598, 53661.012856238085, 21311.957144297055, 12134.599637206711, 2835.8395565488136, 55846.95527485995, 81886.20045017608, 59603.30884850914, 64945.051416885944, 55075.46232301847, 94231.57840277375, 57625.87519424651, 56939.50557027274, 38096.12575665875, 67186.5367736568, 72681.68078205618, 1705.246965674434, 69257.55330110028, 73581.51946480974, 14988.301109601398, 19322.30955822952, 87132.81344688388, 87048.23318548665, 22621.141742140706, 54578.46440849983, 36644.896391404945, 72858.42201485539, 47796.03582289309, 25970.31204814454, 67671.41589010195, 8946.501563405962, 6136.9793518360475, 20745.07446880015, 84943.06165744289, 59351.719769979936, 29104.846329859534, 45148.43892434083, 12154.039828329365, 65435.765824059075, 95564.98995224683, 53970.753129113094, 69986.39995051609, 30440.26626873888, 73130.08871727448, 14133.439026085725, 71416.71434287686, 13889.603967596375, 46287.965323996716, 15144.605206155315, 36487.87870410943, 36748.00964380635, 76287.5766966182, 46148.97248547052, 74942.72910413276, 85831.26979683111, 79764.99367981179, 68534.26473896489, 53250.80683213984, 83688.84903631448, 30575.502379712292, 94843.30207871749, 34299.083229273245, 88902.21816235999, 23460.267117085863, 57210.94687731263, 7526.846071658133, 81944.03189706689, 6621.71134990569, 47705.923866496, 39649.9064844729, 45666.8747256013, 48434.367987619844, 35137.82190971252, 24478.327904399564, 29843.09807712412, 69750.46485039276, 60575.33861967122, 34756.25042911451, 65994.52403582945, 99151.32243316129, 2916.650296190071, 19997.016845844286, 59023.56946303419, 28862.629158254793, 69781.4928569319, 58513.11712069019, 42910.57018335858, 19607.205626075218, 45610.839737772854, 69930.19206527619, 44220.16610756441, 84254.49808644157, 41736.97823460186, 28187.369609559977, 2948.9469956243465, 8930.019549940049, 35641.867772433725, 80624.25158671464, 3612.087365816108, 26369.26377126998, 51035.74059625983, 48397.18973345139, 73116.12152883083, 44937.10975169836, 86114.53502384532, 79505.28520808344, 52092.894856153005, 82179.12646709543, 8057.356647962055, 61156.716480122275, 14353.437269238322, 82937.00458777715, 25922.309356466267, 23629.658549808595, 47734.093871504425, 51196.33701846983, 40892.21839374276, 97608.39200116618, 98766.72631949502, 51162.48128950196, 78718.47385299759]} +{"id": 711, "vector": [34698.632253604235, 3070.1660640511273, 9822.19846493264, 5810.264027034873, 12283.59786539911, 47391.61949358679, 1862.1103058814924, 53073.185276948054, 23851.63448080002, 65678.34619213338, 11639.953033671669, 84197.98616048701, 16175.000163985265, 9965.047291963036, 27326.27882719809, 51700.1550614167, 54801.70583874459, 34254.205588668316, 44666.61188806592, 52814.924258714156, 82895.09158866254, 99425.48658095731, 44798.97239348796, 77028.45161244774, 54202.66664593388, 27159.416860304253, 58272.62834360892, 43674.86835897091, 9396.02774043219, 53393.97678014316, 55037.312544402084, 51408.22373621955, 9552.907580558933, 44982.04670580577, 58574.50940542551, 71855.64794420458, 14205.903246704598, 16046.470814584634, 61978.29363678557, 81043.3325041377, 15323.530265678299, 27763.523371168318, 51931.486891638546, 67865.58528027525, 68592.54202550021, 82244.72879387203, 93097.42889349714, 61441.201646965616, 2494.697374089283, 10073.625516377771, 30229.94963169312, 78350.0982487399, 5929.240991588503, 97799.1843628036, 43459.231921547726, 98506.89303971032, 94643.17520998849, 24980.281778947578, 14221.142827843036, 65646.87177919887, 53148.36188156607, 41285.24814744803, 84228.56281489563, 70855.40104998658, 58144.24012717258, 6120.034941329422, 65198.684044725196, 96097.54055727953, 48511.64887287725, 95890.42227968681, 53791.83959126016, 73881.13530735113, 59143.579115744025, 38360.758607566946, 85229.1726048077, 19853.746180428876, 32890.653436275876, 8778.668826825875, 30113.429304042715, 39688.46641750456, 2043.6089744720887, 52138.912328443264, 16636.63960341961, 59030.65988791201, 17595.395171179916, 78653.71563315963, 53007.0105053882, 26582.195626412053, 27728.443755557873, 32468.33310898829, 1906.8382482762436, 73168.65865761232, 9486.396096319739, 35001.17613692976, 83819.75415976405, 71730.0880913019, 24264.67924563389, 33905.52195846429, 64497.15098250259, 89212.98545124376, 33757.54443304373, 84045.89188205836, 34247.28999336223, 78936.80678403626, 29248.838434097957, 42302.73572089292, 53491.29918213281, 86726.28230509126, 93224.73640771497, 21438.54283120139, 73428.93098583528, 50696.831028119974, 10851.23831158349, 89969.34030936516, 92983.89763638737, 13417.250368158839, 43662.26430254945, 67127.5066175707, 33140.53082986288, 51678.86954014281, 34464.459698206716, 30978.95186639362, 83401.675985556, 79619.48360016152, 37004.9379526575, 69269.01680234182, 7970.008853392307, 17603.828106086337]} +{"id": 1378, "vector": [46580.74984488747, 17029.208499547087, 73748.86365731263, 34015.848625866216, 8148.039212343428, 74126.18764159975, 21618.50295765402, 92675.03373205198, 64015.56997737414, 99383.04428278231, 787.181255492575, 51783.87406271455, 56026.3460466518, 95120.29153653626, 21807.484151365563, 23544.786160121013, 63072.343191099164, 12464.771789262375, 51567.210889957416, 89433.43934613162, 17954.632419914786, 88938.33089017501, 17524.94936993447, 31501.485547287757, 48855.871350675625, 96444.00215410737, 69098.03124785205, 48325.81546597625, 77899.49415226642, 43879.57104081599, 18197.028283544325, 29987.691744810953, 28907.458182367674, 74804.38095917983, 76699.39933033397, 90346.01180557808, 74174.9667155028, 40325.43391117696, 59.33372676765192, 38538.611716985004, 314.15000042774864, 55291.02779937292, 43893.35871620161, 23593.83031046798, 53670.44571586991, 65185.9679315291, 63250.90486751049, 79609.430092518, 43105.98083526321, 47838.84263567859, 14601.794415725577, 68533.0379767417, 42529.77305950166, 84161.68141508831, 88329.42541251073, 8387.354139986059, 89932.65530073155, 96611.9251305613, 26196.818784612897, 62556.899626273866, 65870.46723410212, 67863.550114088, 68899.0101666216, 47004.696886522324, 76869.25798094933, 71594.6689708973, 87601.17113053257, 69408.7802663672, 41246.23336270907, 46386.54018672169, 64093.269699316304, 67167.79490890949, 74323.20149886819, 91922.29445637364, 87143.49336921619, 33811.8809268811, 78420.2137585506, 90734.0807633159, 63985.39137512696, 71607.55655513858, 10163.410209830392, 42277.04494493806, 31365.360228262773, 77178.04062182312, 65437.567416961996, 66565.34407692624, 19015.294033479768, 56005.63013330769, 48533.87899581605, 2437.9130026477847, 52322.24726702005, 46366.347379280494, 98358.01887519022, 58535.406732627016, 41929.02250024052, 99822.13697292567, 18745.376097645916, 1696.120949208535, 14135.551640128919, 7748.806122489415, 61127.43886589987, 17191.41641045445, 77993.28911007964, 10721.726786691599, 27887.64131602355, 75015.4074689113, 31829.51521998296, 31216.969825558048, 30097.948333124725, 3153.006236434552, 58219.21576657118, 37795.86606076247, 3189.3097592474937, 94644.59757347639, 26709.64334565946, 74662.36619941074, 53570.50804411701, 89870.43321511109, 3686.0881410333836, 67139.17417100603, 41648.26706181746, 84905.84047177041, 7216.691053068658, 71450.15716320316, 80992.46107690452, 14751.509123673944, 88271.5606362226, 17768.95597472754]} +{"id": 1710, "vector": [92706.97182672205, 8255.13258689572, 12868.285322099648, 59738.74558018398, 56295.90182977935, 6312.579056209933, 20447.71575856954, 97944.22201594184, 78018.54050381036, 7928.134282177013, 75921.99162576522, 12240.190797997986, 13534.944147153705, 29173.916824301337, 51906.226513960886, 29094.540845895113, 45589.59177374231, 41199.6078700432, 21306.217924103064, 4580.832427601677, 90949.2625811364, 71729.66693462951, 80854.41444329087, 79860.67349272605, 10470.17016185986, 86149.58189636719, 52894.581419592025, 82730.51233418308, 95322.80890778251, 73661.03561078149, 21741.694315313285, 77673.02622140152, 49657.46550223128, 33820.01385977577, 90334.43966519773, 66583.1494780478, 38352.0645712893, 1751.206944264727, 40575.85164031492, 95143.07247185017, 29873.730017262802, 62870.49210427116, 48498.679704084156, 22076.36565114971, 14054.761630849443, 87710.13428529828, 61259.377142021454, 6556.471546684317, 43137.47802935393, 12771.291034650634, 46790.61117852493, 30632.411030003437, 30609.12398566804, 20589.74035976926, 98669.97151560018, 14033.029547852017, 5510.513274686535, 12583.78804800997, 64159.57617885384, 68397.65608943257, 24389.936234448174, 60341.867505393646, 1179.676754498038, 80869.4485148561, 70079.33317628507, 49931.24155068899, 42020.56819758686, 99331.54662359884, 68276.33382660708, 40585.39466057965, 63376.86941031924, 77856.05151617597, 94595.79199273024, 54549.28498799073, 31857.724387394413, 57479.82607952765, 52712.883438454795, 52851.8488425164, 53673.065494258575, 4921.654776484718, 65780.90770057487, 10776.613043601912, 30144.666387559715, 78312.3586122167, 22567.78026476951, 49916.24721781892, 40545.32128716466, 44155.70618377241, 28245.876527520108, 40902.33546883104, 1139.3763337039343, 85252.46695130582, 2115.5284950255427, 75712.25801476945, 83622.44074960821, 29257.01822033918, 9629.025839140004, 64453.321378458306, 72493.73038490316, 37936.984141109184, 39189.54718194411, 53558.91579489925, 60116.35736706177, 8570.365540443525, 10653.425131979133, 26734.158720771906, 79212.96997445106, 80006.14694696601, 82101.904223528, 56280.69383104123, 96802.45737405731, 83382.75309021676, 89163.80062882492, 67457.24598506067, 9425.278351174082, 85237.38091240167, 33681.09947413804, 997.617317680044, 81491.61368449025, 87236.07713632974, 85773.26851874296, 12135.70236170669, 45081.45913314361, 83833.61428549165, 29444.703499293257, 72445.83105711432, 49824.6796748752, 77554.24378392562]} +{"id": 215, "vector": [97724.65415668432, 14356.55272123949, 3993.0640650411297, 35391.865706239245, 8244.98539286761, 32873.44700680624, 8012.145265160709, 20409.86221191713, 95236.73815786433, 22954.02523001463, 28216.847885619867, 7834.379934165936, 67573.18917474234, 52795.363872178634, 1597.878644671602, 89580.95784216897, 32663.70419780642, 83251.38536023212, 31480.766939418838, 6893.377833890435, 83355.5070620505, 20788.94084942533, 28816.670429569203, 64780.04288967515, 64247.90396407273, 72856.54143274718, 63202.47334437804, 41547.98530480337, 50472.54629988276, 30613.82164354224, 31957.166480511147, 45174.945225395866, 53994.065697896054, 62625.601466352186, 35171.43698808647, 4462.964469897246, 39208.19573294504, 50240.49654491647, 86057.02701923881, 73839.7490837036, 34670.38459572089, 87052.3996223552, 33235.36038356098, 93378.4697454531, 71827.3141757477, 64184.17471695569, 83486.62761340679, 65993.7253253653, 96579.38640584244, 82455.09403415173, 71234.59459147321, 72608.64300874385, 38638.256241505675, 86488.94770557193, 9336.925485265323, 63959.89510139133, 25606.56180409967, 3975.066294775087, 57324.48220491588, 85176.98287995238, 60516.705155886084, 59090.032384532446, 69996.19213825406, 40763.473985651886, 51240.64169200273, 90833.22621436726, 52808.27226478434, 35525.193371425245, 64797.98406133824, 66278.29785754354, 52395.14805082205, 96538.11545418837, 78840.15457979757, 42648.2996561977, 56550.416137482294, 62558.195952145456, 87856.01349530717, 45401.827437778265, 75723.50274924411, 93086.73788717526, 13877.689863594, 60517.14672864998, 830.02101471773, 83661.69762109606, 27589.05319751449, 41711.56971349522, 24496.346273249048, 52803.492974731045, 52721.81215308609, 29164.648587141073, 38931.88895606354, 72026.46974097793, 76821.75276728664, 92010.06730045524, 50615.693083938204, 73366.45085075863, 75611.4150859461, 89590.43444771432, 66410.87186205969, 46647.75693370633, 64955.7425730943, 99727.20926012915, 81449.85515870579, 87887.58789836049, 99418.5183890491, 9609.544841162176, 93047.3700268245, 45774.45768879405, 35671.824256459935, 17188.296961742166, 72.93776161340082, 10400.509729825491, 89727.43820825336, 45571.52356914893, 32235.041133546616, 50373.1990916265, 92789.27680515654, 80461.79012933609, 30356.550344129584, 76767.79299849326, 52484.65346077728, 49593.26254977713, 81691.34850778444, 45357.61453024715, 35268.219393309904, 1660.7693047098194, 32606.778539597002, 92360.09835009994]} +{"id": 191, "vector": [21938.296693970984, 27641.0645927728, 94919.54248448655, 42407.19591341866, 80495.7616619754, 39530.712392945265, 20227.28600213436, 59685.707807630206, 93695.20684458014, 43600.390192925486, 20039.492635245493, 61304.406109077914, 62225.55418857535, 50943.53693270208, 48507.76553512165, 90553.19264288296, 97951.1076136879, 6115.157296858598, 51045.37480900416, 55600.23741167626, 72474.28628406845, 42019.02968215834, 96733.08870565267, 39664.44134404507, 73699.77666880372, 14075.735798437938, 5971.817817747138, 41842.36324725409, 66392.13359009322, 43048.78057522705, 98480.61356510391, 81341.35082149004, 58519.26777833681, 64491.33008026483, 66594.35944408677, 40934.983531963095, 91228.05019588927, 65363.61763489076, 90048.41249840501, 5677.955758373043, 2930.8764761112548, 12024.90527358817, 6421.411690054469, 11380.65509600824, 49288.18929794794, 21050.997399333937, 84882.03966569659, 34525.08811052839, 98372.58867875078, 22680.49311135566, 78569.32468849071, 7509.637781077016, 84381.18837013203, 12512.89862255015, 74950.68478996443, 36039.664580257246, 42632.55783791732, 1868.7571440633267, 57948.214693430855, 10399.72142248593, 49174.955635592174, 57447.380095970846, 11699.811615136003, 14089.150929007765, 43133.31488697079, 92904.21393561801, 22519.03502492061, 43463.15577577204, 30198.06491892868, 14352.411668249266, 97721.49709598132, 53989.34434429763, 62952.23846232755, 31216.678862474622, 59203.99235411192, 17836.079047789786, 44109.729204487936, 80009.75511664124, 84864.7498563863, 97085.74481472919, 90338.21489560067, 13564.68203004234, 27172.02622499999, 97692.57466252315, 64170.26688934367, 58973.762443347776, 21499.288500453782, 16031.355175364759, 37329.22316911954, 26351.358610356467, 82147.4712553513, 44020.97167605391, 10310.154248459714, 68416.75107492931, 38617.808550557405, 62470.49073163701, 40586.25099713241, 10079.147885440865, 63883.90531177888, 67894.19567740681, 43515.865022808466, 95563.8116183028, 6300.944769876382, 77506.15000523918, 52232.12344333155, 35234.586613216445, 97299.45012660307, 19996.042424368254, 55900.508635265374, 71642.55071644676, 49051.58700570848, 86912.63394271297, 87351.45326743134, 30050.590023712386, 28064.090171667278, 1690.2393076205003, 83255.54378077765, 64927.44530860285, 64623.00573449616, 33046.1993918668, 80993.54474686044, 34872.52374210198, 34063.19423793821, 81142.46376330369, 50764.82021969005, 23613.094033842874, 76466.75244000638, 85011.42543916624]} +{"id": 1994, "vector": [62995.29597499739, 47031.430282623245, 49978.94941431676, 76863.2467677845, 85493.88530012988, 49796.260979189676, 31341.24895987481, 92642.34633993609, 42102.64089050872, 82892.45947936614, 1390.626260941308, 80241.86479649592, 15549.978679142207, 17846.59447320256, 70064.40797579853, 92764.29400212308, 61781.90366627575, 13558.052360314676, 17740.72630670004, 85626.1991523425, 81255.37061823676, 49075.180924869586, 35757.51931313068, 90930.31230247674, 96574.68132128885, 29567.630922226206, 96454.80775700268, 48940.564806620976, 19333.159866172322, 8715.898050742544, 58190.25087351032, 3133.3202842998785, 39668.928943609186, 60226.30070444432, 82489.90360678955, 4680.718172931098, 87139.60863316024, 2792.513326393875, 4654.474477102232, 25774.197446006507, 76127.51570593892, 97967.60393597183, 65728.67653090155, 38089.8126570717, 37331.528168176286, 17024.324494962195, 26451.387614668052, 40595.21213478686, 50714.25205767749, 13967.73540893953, 44641.72467148259, 35168.108327959155, 10970.87678506723, 9379.129086114246, 85518.59427914943, 71109.59196567592, 59133.44538889604, 31035.615286283024, 24529.083135490437, 38416.11578756662, 65067.48765536778, 3318.7715838132294, 50499.480369951874, 27261.887581830702, 24031.64962250156, 55110.02094423889, 95774.05171518544, 8358.804494922833, 39100.38719772564, 40226.51149277861, 59410.11337212702, 84951.98125217728, 50423.03986267368, 21198.510155290107, 27889.74033294741, 14498.707877749239, 79242.49879226412, 35670.02105661578, 21386.251819756253, 44621.96844299714, 32856.42478663852, 56542.74552010968, 89798.07966602898, 36884.97512397154, 94357.64491925022, 79869.70983814418, 18723.141774280437, 54800.910805850064, 91799.72079760264, 98760.39801176522, 2609.0032364858293, 43683.67015795982, 66749.77461553192, 21562.36144934377, 79561.68276655035, 19520.03160413274, 40622.543383775046, 68960.38563024145, 95094.79831351247, 17614.867720334492, 11097.686050565359, 35946.57196941516, 82251.52333699683, 40495.198448681556, 48309.41661062341, 57482.95145744135, 43039.36801508773, 2706.9689944834277, 48147.78474360433, 21698.76106617288, 43794.58600167349, 29865.809765502603, 1264.060029063685, 39658.43977504685, 22526.41560816826, 51942.97749151487, 64365.516161501466, 75281.82002013132, 10365.508471020035, 73427.14192072669, 18266.24676039943, 85188.0511184318, 48825.286061247665, 64835.333328270484, 71663.01033982911, 34566.254955447585, 50077.79964250974, 76886.96818971237]} +{"id": 227, "vector": [89178.7105949131, 59052.44257131098, 78255.651338544, 6207.955481792304, 91288.24832417844, 59441.08844147611, 84439.05549854053, 44254.10120822643, 65200.2264939187, 30482.517766076868, 89800.28270687196, 26281.431007804535, 36029.538553208906, 28205.481820949874, 20641.0893899006, 39002.9856091155, 46092.704919193464, 25562.446041906616, 71735.66110141716, 10125.108679581062, 13921.906862529398, 83094.86925498978, 97342.48349426418, 90931.5266997293, 23612.723157200257, 27038.883489373922, 57181.94914150867, 66705.00978252264, 60267.82608482773, 8702.951952714744, 55219.702086244346, 84507.89910861348, 59793.169138791316, 97958.34441137614, 18911.793607781823, 14067.249308219698, 12847.201610847436, 64078.57337544636, 41312.73424095584, 90956.83702866797, 54328.14594270068, 92823.93113014058, 90429.57417840631, 59866.75199466662, 63381.73494126956, 35304.87328548597, 91687.84428120754, 34407.22466892354, 4502.233031751235, 93158.63002101042, 84803.69092297368, 80175.80868813983, 61852.303787378856, 89138.46259945232, 74033.7783785823, 13223.805880534444, 82237.39447285402, 97785.51700267226, 99632.00994276282, 39018.6574973513, 9399.635859000398, 517.6972367069154, 25251.31070515757, 8437.257437586299, 47642.26644142073, 43331.823945441975, 37403.64728184208, 22461.479380171502, 4589.317476832622, 52768.76090265945, 19422.109731152672, 27295.34511117736, 75996.11626171251, 91447.34064839767, 39387.8201989043, 4087.5786134371037, 48670.93800360455, 15415.869755628697, 86409.70973663611, 82262.22156946495, 12498.72436374666, 4063.5689362267804, 4666.721990252476, 13974.132974026421, 28748.500409752232, 47416.94364472475, 87373.66056445526, 33655.499213125826, 89488.2968773263, 75934.79605292135, 72228.71525201971, 87046.09521359151, 19225.855857854956, 66720.45974950545, 51794.65215950794, 3530.914557764253, 32793.446526990636, 22405.528240911564, 1567.554656010217, 7932.416687401534, 44877.36994830963, 58661.9234236094, 62674.391640754824, 96558.69692781127, 21143.00409652139, 85469.57124662043, 10778.112568471477, 91864.55150455963, 42906.81476324435, 69322.29843164774, 61065.326310112774, 17989.909199976482, 33722.161197623056, 60735.676672024, 33811.5520959824, 45014.3573421841, 67391.63696593467, 85395.90180246059, 46142.45995447912, 85299.49117250783, 68216.2931776029, 98302.72061271002, 36629.94138475112, 29099.332485379593, 79293.8348353465, 40109.836309456856, 41519.54191610986, 22944.394010267788]} +{"id": 610, "vector": [63968.80683934203, 81956.17525437348, 29492.439099826563, 8089.242411467112, 87569.44917424611, 16536.384983617114, 89601.37683966926, 66304.98581762877, 33226.381257883244, 8382.253818884677, 32258.507606126786, 15565.407652776841, 51271.23705156438, 41354.65627779655, 21388.43779644061, 31116.385448621477, 22018.21918786233, 89573.49722837671, 78328.65618260321, 15146.953283336496, 93002.22583653682, 97133.17625001282, 24648.857771521838, 43537.62226757665, 2156.0965801908806, 18909.20944409843, 1846.9933301815345, 25773.98451245456, 39787.51681753434, 76464.3826827976, 37541.587424184356, 64718.025646361355, 40886.83789122342, 16424.34440987789, 74296.49590522365, 76805.7217145085, 65081.04547227719, 33873.93633809349, 1940.4006648441664, 81815.28163661774, 5262.570198850258, 49387.8163173936, 13167.298686363216, 21965.591675942233, 55123.51177038529, 72145.31134448251, 87534.76268765883, 53519.611539939084, 91231.47282902636, 17704.91631771046, 91187.52009886007, 20967.356645464908, 77881.27841813357, 92342.98130743597, 7697.333865622624, 51605.95545388843, 59613.00563821897, 98412.34782905596, 13329.717949281694, 97141.90518480017, 94304.52235504378, 9349.625113491722, 9853.462020900572, 62863.36731758633, 2430.2886136738143, 24712.008502316752, 37328.46254543999, 87051.3965678333, 77585.4550566706, 77057.63012948242, 64646.969918110844, 38111.319104173155, 18099.34342153817, 94330.12098442465, 90751.39450648744, 43266.401462603135, 73582.70085529388, 17941.14992746645, 87476.43369946376, 17725.250024681714, 11238.129420600995, 81237.53157811951, 46022.964286689305, 29907.941594137854, 5076.317020638121, 26004.474790301156, 46803.03822043177, 7473.081082696531, 71661.70415950152, 45206.658798971825, 65907.85602526774, 68842.4718472345, 49593.95702065468, 7559.074965181856, 16088.752796808514, 48477.91090394293, 63653.351952187164, 53111.809286349395, 14773.324421171807, 78081.43616280917, 10286.434880079876, 71051.1012575513, 62823.50426790124, 57438.244051518406, 92400.89041247257, 55159.51783410239, 40547.18969772347, 50995.244888384914, 80075.06029607527, 42731.75902468336, 86353.43117242125, 74321.81495404211, 74380.32401142114, 50338.827012730326, 17883.527974413093, 42393.25595324549, 61836.62578828788, 94104.95478190496, 87190.37278062303, 61780.407681525816, 54324.79137687972, 8176.754901583261, 45844.32726109074, 80627.99323365901, 7789.268943568472, 13918.925923052228, 8205.573233800113, 91706.97205361638]} +{"id": 202, "vector": [8889.216934508348, 93005.51628782411, 69448.33828415404, 91823.41223749888, 73466.33505220506, 31551.094554684245, 38213.37555302126, 88035.61884549663, 11143.782231920186, 59340.81880001082, 70797.07484919227, 59904.9039441663, 27226.62702507822, 86525.29364755919, 30074.84354554084, 81755.41722166826, 13819.794945282281, 50889.0468907795, 24928.537210594226, 70365.73352526383, 75549.45023603001, 46901.07593156434, 51244.16678533571, 81888.52444181584, 94019.6097026798, 39106.229451474785, 79131.02845482824, 78538.8381069703, 39931.436200333614, 10458.409856332795, 62726.21867064892, 87214.76953620191, 24380.007228587365, 24029.880921792577, 89281.4373852549, 90361.20040795807, 88570.07933242913, 47675.37323906699, 36553.07798009735, 10292.983173843428, 72565.08888878647, 62876.54509134251, 93800.81843245224, 4576.412414295627, 60678.04297989538, 4976.672256230197, 58039.14277123527, 25158.541265875832, 88254.6159038019, 62397.32611840843, 86526.4719550453, 27391.747109713604, 89258.15346063073, 46247.90567991153, 51101.675455712866, 72932.11238330028, 53220.577804928995, 14057.90866628922, 40259.52497078824, 80799.97840920063, 20552.800202757815, 43178.78590021722, 52340.38055788076, 4570.977771695561, 44820.59683622478, 60173.07792013956, 8725.709650912628, 33051.89171499737, 17643.732718910578, 35013.578794588815, 3542.7786234957794, 61745.84195201981, 77654.87734970106, 55891.13660464351, 94092.49150178347, 59509.414269943736, 79177.31255248295, 11257.36758583371, 83112.83566925659, 12749.942657722035, 80753.13700509586, 68037.9542894574, 6367.915488226461, 39567.49770751404, 759.9319989170849, 63831.23989121788, 31560.317145084828, 19585.566452034676, 24885.56139115956, 28039.641182627307, 49965.32694238425, 47821.72797111034, 46633.887540655094, 37964.16165591282, 42706.10584432318, 10363.175620938991, 14886.507573281871, 70048.78992721258, 94202.53933634766, 20214.057313591027, 46787.991575403954, 50779.04315605218, 54119.654489463246, 31559.358408297732, 75230.64561842402, 70271.93378798124, 51787.2278323349, 53214.41887141328, 366.7862671499988, 77438.03012042439, 78219.34889590032, 15327.691989554338, 34560.09905676292, 79152.45907180307, 67654.73500365079, 63581.97400708253, 82018.76094676001, 37600.846061896074, 20230.1482137768, 38607.04717949842, 73044.6769930952, 10518.917141782058, 20822.316748256984, 58449.45465881009, 97976.20100196575, 27953.183759317613, 24065.58248257962, 13696.2217795783]} +{"id": 233, "vector": [41279.910023810684, 42097.63828739248, 4778.972327351338, 76909.2679297949, 81229.42256203116, 93123.70896163343, 93602.1926161233, 56187.386556451136, 94541.33555074486, 56297.584888425066, 34140.342817734505, 16951.897650599723, 16106.5084828373, 64993.62714576182, 87204.39961341655, 26226.04450866991, 89499.54835563568, 31080.90637344747, 72242.45767423908, 70136.98459540005, 44623.09821493043, 56334.292940541694, 46801.37248084275, 56698.11286843551, 10467.409852299392, 48555.492568610694, 38814.31814159011, 41020.851617369466, 87961.05643665156, 17589.216107242533, 20890.6051410527, 12452.778237442253, 5575.44390904241, 24308.94090064152, 12795.20813877285, 45693.047842516986, 44538.19655416691, 36369.29749302572, 75051.85955995737, 74213.10973050154, 79970.66549341289, 22961.829975053395, 42984.08254403231, 33429.7605015622, 72658.18798127265, 84888.29497292735, 62243.26442783134, 32362.702324900038, 74683.19262417496, 93988.95056992701, 99934.01819980233, 58492.383170082416, 51736.41606236281, 75410.2377467154, 58696.36864319728, 20731.828548157915, 94572.68632943105, 19859.62778114697, 99727.2247635628, 6740.796364884171, 97873.84255211246, 11651.311123391917, 55640.20568811673, 74083.02973630845, 19489.723763308342, 24716.948698841934, 59430.69324814201, 78100.57704500372, 51070.05516448714, 23142.06568188436, 47240.58173116799, 27451.73979509905, 55269.06982844215, 52528.00375900478, 79150.53972442696, 58688.21749719214, 45698.2673225999, 82321.5489949317, 24793.47459293295, 59177.968380980354, 71225.14907324096, 29543.80169715688, 89555.92894320984, 7525.211373717844, 26475.258807540435, 39048.20572511131, 32567.67777021703, 69822.28505355195, 73030.19534295105, 20884.317132408414, 99160.90529824315, 30448.533317147398, 19763.613672674506, 35538.612415692616, 33196.86356965258, 23509.815672206692, 57731.508827080455, 32196.874980374658, 16944.29135767559, 71502.35406200704, 6425.101058672644, 7844.203873680755, 52344.87967124414, 18819.345195562266, 13348.872801365453, 51308.12783950961, 32793.1545358219, 22709.644343824275, 1983.1518982034058, 44032.65367956521, 60188.37663762122, 40290.927104995244, 19896.995535943217, 16851.985985743333, 12802.243538705648, 64502.72309535698, 95987.25805638722, 69631.81972874404, 53673.04531379893, 89527.63363166856, 59292.41892527292, 60539.93607098405, 4770.350416647817, 30933.597510236777, 11508.595392123121, 43667.417062485605, 19499.760771854202, 55293.445037983554]} +{"id": 1234, "vector": [54343.35477576142, 72795.32958573765, 53615.152403547785, 6483.1083681738955, 23613.789551345355, 94097.40864587211, 66286.87664500836, 32672.431258018096, 13427.533582889517, 14449.853898514586, 21287.461263970563, 15191.531800183966, 9823.595857626155, 31228.39741390805, 41436.72110048396, 27340.426926896867, 2056.339805274787, 29898.15926056264, 37914.415195853355, 36117.72782295768, 52063.71089375679, 37466.162878998955, 39350.84837984163, 30122.98346102462, 68889.58930460083, 70335.94297846657, 61334.13406818291, 44246.898856463566, 80547.88293405781, 71621.45139424212, 90844.08768469638, 67447.45860868445, 21192.27961475627, 56302.37651882902, 19456.865322425663, 31069.71458091732, 74330.46467302689, 89820.99109920528, 26228.666602991423, 69268.40559688683, 32793.89486013656, 70145.77830980904, 78233.78615621492, 77672.3126074879, 88440.26403723564, 17025.432937773898, 13677.833965274243, 50828.38384608994, 42895.97089963968, 87543.94275265878, 31353.490011466, 45601.11834834787, 90865.13862837444, 37059.839781237904, 41293.259241808075, 88457.69705612221, 32119.603103395842, 47493.721537609126, 14541.920092104821, 32824.4137269333, 72309.18681844728, 70515.08412075872, 45084.979901656596, 30498.55767009847, 80764.77044681838, 26684.60660844736, 8139.62714892279, 95130.45969075218, 98992.53567943843, 86491.47455667813, 47472.24725205042, 99766.03999487124, 81237.98790482493, 5235.071166066019, 85205.73563221132, 13441.261832314343, 10951.259684285886, 57854.27609397182, 92736.03370187001, 11168.946540656032, 11229.475251896525, 54298.810891498775, 25510.691355906467, 609.5058360852668, 40016.35441915031, 50734.6246594042, 68616.4099119748, 86210.63827428134, 76671.93249919522, 5346.332725672442, 25920.935663110522, 97180.66147833606, 79524.05540748122, 18794.409945866064, 15490.19552643316, 20676.226762617945, 18622.983052135845, 2519.1494512378677, 16975.25463156787, 12939.57454728264, 1890.7680236946444, 56615.93314158444, 73489.53579493245, 78273.99085084401, 42932.289772957454, 53109.29481035892, 33539.28775869276, 46332.76890444224, 75858.91507228381, 16388.503290472025, 44820.77491498179, 1887.9034326489964, 59194.964917250756, 79340.27887207852, 74639.78430072044, 68008.93461832435, 69388.31183444608, 32184.985246168828, 1203.9477008974343, 86422.31969929258, 3124.282626253216, 88363.14961158832, 44344.162860323544, 63516.75051300295, 68782.87742473792, 10930.668962609347, 85039.45284433042, 87724.57888869997]} +{"id": 1969, "vector": [6021.827673339663, 9083.528679346753, 62733.484229429036, 94008.51372626827, 91828.80816313832, 18789.92652179938, 5701.52920855993, 69039.95174898942, 77359.52037132447, 50755.780114854475, 6763.521621856438, 12444.228369354048, 6872.753524472641, 18833.108835491486, 11886.268942621648, 91655.89175218697, 28564.671606404478, 99761.57019157504, 72339.31635419717, 84325.42605796638, 8508.78716749449, 61971.13437047999, 76872.77660045122, 21385.321641585564, 96087.75953640944, 89251.63588127373, 99416.1987065155, 26132.15071232444, 5332.400215813826, 78857.12569164293, 11858.24604603799, 64433.213524658095, 38669.335110322965, 20665.499966096868, 63222.60697002155, 36074.27965642923, 87571.22489930742, 29186.928801195278, 58143.97911773309, 74758.68896238391, 59493.25306435856, 7389.005114964453, 81640.21130381152, 11515.940435593397, 61150.83184679881, 45244.77582808756, 21934.53372173768, 99161.30492289209, 51861.511281846026, 92171.80593622947, 12958.9009005321, 38471.60521408502, 59026.482544739825, 81586.22279046457, 62901.77320788407, 80262.92732215043, 62713.16747037991, 11421.2955136202, 85894.86440560999, 33728.07899398631, 86609.15232877436, 49590.228992646815, 58077.595037153005, 82993.63567839366, 99250.3624293396, 14344.74815176312, 12041.85422145061, 11131.265447522099, 95456.25739494443, 16319.693155998772, 84987.43233642759, 45130.231458742484, 38963.99338817812, 15069.77877580269, 51856.90667850409, 71424.18226171931, 68252.79111742569, 97421.9959161271, 92892.12929386293, 38914.92015885466, 22352.245019931805, 69210.23688063434, 52795.84471661164, 75513.36783700666, 99915.42591670848, 48688.25819108532, 9478.764418913388, 31494.613967737485, 23676.033381253936, 19530.857365998567, 42471.26729197537, 83244.53769125388, 1699.0919419056504, 15305.638966369506, 56176.32616530811, 59606.013584739034, 48742.4145966899, 34297.132607497384, 5154.854677503584, 31528.43226475792, 87280.51252861254, 62667.1042755771, 59168.72716012127, 42585.116322294634, 26593.692960402946, 11317.935438439352, 6993.571197445803, 75366.82829607888, 28745.075230709972, 61021.47395212964, 5919.334825468392, 791.170049107448, 62503.463504829524, 23219.615681080642, 10208.622952490265, 93668.20825451969, 29684.93394896088, 83817.93202880789, 65559.47899515477, 39941.444233576054, 92584.21072986742, 68284.87154333747, 5048.400946348442, 18019.905357388587, 20730.47183125095, 17118.604999335075, 98373.43419199184, 47908.956009079986]} +{"id": 895, "vector": [20224.006454241906, 33766.43167504913, 10381.746438978202, 98845.7195642245, 83733.51204346135, 18130.79004081538, 12335.426968367046, 17785.452182129357, 96553.53311812435, 12505.184807962665, 20543.06673944668, 73764.63176675046, 26890.390879124938, 9463.719753340627, 84330.33939781082, 17543.3967491747, 49423.599180160236, 62176.354804448405, 84274.9556686216, 28019.281092253055, 26601.1279449877, 49404.38798470554, 63706.36000372149, 40711.947353133815, 20014.492069813416, 28471.73180681559, 98610.28743680617, 64256.92794046291, 83620.85038175256, 44488.985358345686, 4527.601996389497, 48941.249065949385, 99430.1073160311, 2783.001393508255, 80631.46603806452, 54466.9358400895, 27893.360168479696, 72870.54732167254, 3315.899779465814, 20687.447703255424, 23827.059016299947, 59241.442457021265, 19397.23488563525, 83327.48377070892, 62415.689606381355, 59538.023852454484, 99917.47994295583, 86.55732099622782, 72847.71963754077, 93665.87919335708, 85424.97585114914, 46653.460883809516, 91035.99209890177, 83241.35186566693, 64626.88902970931, 80271.33910113024, 42488.13550404549, 200.96954313656434, 75377.38223010385, 59381.31372391127, 69446.36103317114, 23639.56504125524, 62705.28450293825, 18494.869861534135, 70488.36495306027, 52186.8549580897, 48961.385053065176, 89256.00844873792, 26636.39659089616, 71573.48208513047, 41980.98759588772, 92472.1343426943, 91756.39491329614, 82006.49512343906, 94700.2864960774, 93697.4190976019, 93722.29585900133, 65256.23301299376, 36554.16465616882, 8749.362295676567, 28048.472738166587, 45022.77017814461, 63070.11676296107, 36367.70393976596, 73934.47362094735, 1019.854650551888, 42722.45147599996, 28518.315198639564, 17760.93947490903, 34418.114537832735, 22945.08799660974, 32415.20802195803, 67870.06690189047, 88440.91331266059, 23760.617078053747, 99146.49730751861, 26530.533706488633, 74612.40636756284, 29572.18779437819, 90510.95614396897, 75337.89538776778, 86293.12949814263, 55798.17342051975, 85836.00070809278, 4114.5689726447945, 89631.33574700235, 60271.5889057444, 39086.27758366008, 99972.89214242889, 22186.88837207763, 13405.809554444759, 33040.006635775564, 44348.430586943636, 52840.11469489294, 2305.937489909715, 47457.6480242191, 89938.65520096912, 51350.99147068087, 27903.5874834052, 3555.809863051418, 48345.05095495277, 70889.5254506136, 80921.8438857106, 9885.343706949257, 23454.345515371013, 59440.25422003096, 53596.548206645064, 84422.61763269559]} +{"id": 469, "vector": [17704.485619238385, 37789.792276272674, 73510.77752681273, 23260.985915706733, 48257.54304998293, 62973.79417814752, 41693.756863286115, 6017.251282982239, 75963.89592109031, 11985.900046653807, 87268.91380785676, 67033.297265871, 77164.86278565404, 98077.68316149974, 75021.19870070506, 42720.88700142722, 41242.295712518295, 86620.21741010777, 41477.146827409604, 39402.94438163845, 42295.32103793223, 51277.345184679216, 5282.470326238542, 72144.04682499463, 69098.98582315611, 62538.24449541554, 63786.74302146161, 18733.52076460284, 76036.91685594401, 59118.22444493162, 5230.269013714672, 54512.55117674941, 77639.59929089554, 51087.71017886548, 92243.51366058543, 32611.638290796673, 97440.05984596074, 72311.99502232933, 72330.4206289443, 56834.87320369656, 13246.475451639682, 79311.77157792318, 56239.65318825864, 66929.72431412077, 25303.348772974576, 95042.88941243687, 66282.91635333245, 65278.27346010907, 1447.3693257620491, 31534.894668899706, 5492.605321910704, 72515.19007870542, 82916.75045974695, 33401.88647376579, 22513.89867081861, 54737.75622635328, 875.0604721989852, 6842.866197941222, 20335.404449386264, 32278.262355876508, 72957.42841081848, 22098.67786169164, 66105.73322734488, 76500.63752011833, 40120.19880215505, 54067.45279230204, 38033.248552579345, 58786.7241508135, 3323.6416471855046, 70136.59858460583, 28887.561931827222, 39644.31219764657, 14408.099733241053, 12897.741270565222, 62565.91215583105, 48963.96653039463, 21239.588853828183, 11051.546675148216, 56621.93558419115, 14000.73697300782, 85363.54274386141, 98224.28642525984, 31166.609442842153, 57803.33008812599, 78597.08533923661, 61980.82230662421, 17091.7442196175, 93454.65345681514, 18949.64551023822, 83201.71769967278, 21018.5220225386, 49375.93888273184, 28327.853020251237, 76261.28412205263, 24895.883920065164, 21865.3676144026, 68761.05120251606, 95164.51477899018, 95085.78744393287, 49647.51261157988, 82483.78265610199, 12062.949502256037, 64758.26824345233, 18858.50757327938, 40472.5977591011, 72754.34430736215, 13601.843004626402, 9921.40860902846, 97595.73759532097, 39427.429213874624, 99277.64117086075, 82100.75043879997, 70988.46533870923, 36947.99476624954, 33004.98480915173, 74139.30655780641, 86025.24381471735, 9562.440490174828, 86040.02468880313, 57.13335355151949, 54867.306395398205, 32126.174578848222, 67922.02631964842, 53850.0900468879, 1861.8944710221963, 94293.18390690736, 96473.82900910382, 93109.264866419]} +{"id": 837, "vector": [23322.476572965512, 75318.04355770786, 57722.81735854665, 5810.499972768046, 89305.08328350204, 42657.792843970376, 5099.323397615219, 61614.45008251167, 89574.70173377811, 29461.050432927084, 84949.5989651119, 12836.598879250427, 25172.613665579756, 45938.1178216695, 18415.982849614265, 73437.9544523725, 45514.17158371108, 84038.15943577801, 38774.72262799251, 43226.551755605025, 9787.458783007785, 53841.28699990434, 81765.22602846548, 90522.36200136787, 55320.527189287546, 98920.80798438303, 2022.21209057859, 50153.60150401492, 58556.55150961285, 5423.773749422944, 89530.9061054112, 42293.743700291954, 28590.144393487913, 59015.94138481338, 41107.052762344734, 12782.702496086396, 23400.818423107194, 89409.18180699217, 66147.9894393338, 29527.968307764608, 49791.41301656556, 65412.260663940244, 8792.989393866812, 24587.542449803, 33072.78758367458, 90659.80976838281, 21896.303368271176, 62000.9850558885, 1899.9572976029278, 50942.62189731459, 22978.60112936243, 42624.01458156582, 28327.35178334851, 46744.483885056274, 16647.79559742908, 81718.32261635616, 84127.99632105623, 46545.79950633233, 58977.17234142186, 123.80602863318879, 83757.01289744275, 74428.57448765732, 53867.19832557229, 36087.367817308266, 42723.39088926662, 25322.147991801525, 88051.30854429508, 59700.4193074289, 77502.45350923987, 45804.01133140283, 34608.56860662862, 76777.50219884489, 70178.97525502538, 4660.507205608955, 15668.103906279463, 9752.383970662448, 28683.091666318083, 59014.496394696136, 67306.39653644004, 74039.57705568773, 94663.61314012637, 86550.8668932318, 33158.080979109174, 51680.26324644083, 81645.07845686287, 43451.04585414331, 79651.49621669867, 3548.863634913324, 16905.394644784854, 62845.321922118426, 35695.872615364955, 39682.19102423368, 18142.87632094267, 24385.101634280414, 1673.7906248407098, 69874.68349643744, 75268.20127595574, 78013.328394453, 24870.941884825483, 16624.193394304122, 20818.026658509636, 12496.124911845896, 50458.56846619446, 24232.008703564068, 80189.97974412121, 52999.894822993556, 58923.31941548367, 51400.631038207415, 47109.89022221679, 13634.338893958597, 60782.989199237905, 69846.13052802485, 31421.03124901796, 92622.37171285266, 77164.4608257932, 7663.179902800554, 38909.45497448143, 22005.964552795722, 69756.57943931056, 44940.57847806184, 25234.76862521803, 82276.51055695786, 25615.4337750545, 13135.498192453388, 88760.74079747322, 10032.688501038134, 99936.11340841479, 10133.780459333597]} +{"id": 1856, "vector": [71231.28962183004, 7817.911645600883, 83768.97186563516, 83078.75572637518, 16535.343043621608, 93800.90568012466, 82743.42617209487, 45841.46836069894, 31125.768633966887, 50239.95492280805, 59154.07528886969, 98856.94304728016, 52458.39908386487, 34636.90512081724, 63765.10661523562, 87079.39665959946, 51374.57810811966, 48720.98176696517, 26364.527723412157, 57864.2841539099, 47699.63048180026, 28193.549430520103, 66883.03249071847, 16590.139738159003, 2325.2958944820266, 17450.66743207986, 51055.47039675448, 98223.51995869029, 91066.38747234999, 34439.05438983762, 93488.93309554603, 58901.43023863929, 45014.78535395561, 82688.79777618907, 71919.23798410503, 34749.03744972708, 2278.7846680874745, 53406.114992038754, 30905.438346839754, 42400.92081524614, 98108.1650757372, 78016.18043063357, 90276.06304942707, 60073.29252350969, 25122.56950697789, 66117.98245169542, 78739.25968741729, 9114.44252429109, 91374.30254173206, 66136.71430164909, 24499.333018199377, 7828.239649635738, 32085.692735819604, 4535.819094707461, 30206.566125362744, 79469.21579563826, 25535.35292879876, 8143.831788063949, 8339.59385092925, 52588.84674447112, 8341.588751753981, 24226.19350150228, 17157.579408422607, 38427.32483984904, 524.4272272702966, 66744.68963726368, 63369.495144649656, 76845.82027865144, 28633.037016300234, 1601.0349990207008, 24015.363641604235, 78351.0235596806, 19458.678716727285, 84036.40537104348, 64887.44486729117, 95003.97461451728, 39866.96230704635, 7005.604379994435, 5213.9969029450285, 42045.757206832124, 67708.8477545155, 70187.63866468742, 12220.594535511786, 2141.061249221943, 77992.4736885811, 55500.19905911808, 85319.23790250397, 40604.68355316229, 60575.547672914654, 79498.88060265928, 30687.090705887476, 47654.773125311825, 33401.81228548198, 77008.48532014142, 64712.74701333022, 61165.257275631535, 31895.220878698994, 91690.42090555864, 77808.54270200012, 56572.70815026614, 10326.962621848335, 80885.13686599347, 5805.974387363255, 6822.466387745463, 16458.273139127443, 24017.283159145463, 81376.26632774799, 9682.350278866847, 82371.6909490029, 30751.345073427972, 30787.3365344379, 76110.80134534457, 38166.62499320856, 5796.93884041903, 37862.681443745096, 35354.75828300577, 29405.76405598976, 9419.769572315618, 82328.57617302529, 15973.68542623605, 86076.16955906645, 99086.45566760887, 96232.58501494092, 29908.243159658865, 63761.79004351724, 17358.706502895417, 26980.577426727003, 18539.792410724855]} +{"id": 835, "vector": [5468.279455492653, 26000.776387119007, 53103.96946913375, 88485.69910752932, 42776.12744777381, 32392.364767329895, 37754.088405669594, 11857.580736497765, 95440.0207238368, 14636.404583568885, 70607.35072204874, 30007.845460524262, 22860.4732772445, 39175.19439518236, 26501.921264107077, 9426.886950945667, 56324.406710497366, 91968.81409327731, 87294.11789193357, 31810.50315297648, 85489.97798698909, 53046.637929980534, 31276.640325733406, 98410.78321220202, 46348.811514318964, 38290.264363809954, 65729.76416838009, 56816.767656732016, 32894.82122399617, 86517.79299239183, 93395.36301657112, 90862.33909249175, 58121.09967902654, 26437.20800135939, 4720.075234834964, 52850.183139201814, 25955.234661667793, 15628.289897658598, 88351.24746646678, 20046.017645970238, 83547.52604696728, 30162.140060770336, 81041.16184257473, 41544.09211202836, 58453.968806752186, 10915.138993896257, 50453.19569341885, 1626.8747475792322, 36403.94853580163, 22674.50869389763, 6979.547486844795, 31459.13176354339, 19075.059449550125, 62447.00330275023, 56877.65931676313, 24073.589817761498, 51498.72671107795, 9825.382372848811, 88600.56114452392, 83321.69748029269, 62376.520509096066, 33664.40552152271, 27692.18075730757, 15682.572017263108, 43806.02671246692, 79287.43932189394, 36747.294807138074, 72076.03114322692, 38725.64156748592, 22331.492192068094, 75224.9537632682, 53242.760459559824, 959.8331750235944, 57791.07744494154, 52226.41280196545, 2634.4615995242402, 26443.138091509456, 77682.27442768702, 70357.09207254456, 22863.445141584125, 63870.46424509939, 35387.227658794654, 86849.64906317896, 79462.02652727523, 7173.1335231639, 78208.84947619014, 49746.43246896073, 6916.690253270785, 29635.722838559584, 27823.407824433634, 89373.70140782665, 8522.984660849741, 77498.18651095584, 11443.890457472062, 98533.91381921188, 25357.056729694195, 73191.34621184174, 34706.67787875969, 48200.719170327364, 97518.4430946017, 13683.946576683604, 28593.410389733486, 47559.856197390705, 76996.66997237138, 87590.93247918136, 11675.902551146512, 71542.94865790768, 12139.460468628049, 15054.761765892654, 61335.70510835546, 35172.563828493796, 55570.42733119126, 58234.100101392636, 69276.63091994596, 76479.19262219727, 95527.46116810199, 97660.42648333637, 47874.49381225109, 21222.93520115601, 46575.82292254618, 853.2583850544895, 15728.496754819422, 46376.5948461473, 27486.534214452273, 72764.73954122778, 30134.460245648588, 86827.54907332986, 9520.182682794088]} +{"id": 218, "vector": [90409.83388671675, 25605.100031286733, 78451.54401915318, 47533.71554588752, 72431.65473729136, 19946.53780112058, 48111.52947613411, 75015.09334996581, 77309.30900839333, 27921.539594202204, 9450.178209595384, 20288.392176466707, 91347.10703140472, 75917.50778821844, 17081.60801266918, 39269.22259528244, 52878.83141186239, 43730.6927809763, 64150.04616729165, 61043.71899121652, 7911.613865481182, 30800.065942214904, 75617.77619476621, 86380.06967777091, 81937.30439803192, 90772.7691131664, 59805.23788487537, 89887.32292513881, 40065.79282629553, 81083.95483381509, 3756.2355664881643, 7193.5920026142685, 46959.41743395224, 41799.918979584305, 27404.571443619276, 30594.59498675946, 26278.617669738767, 67479.56216000681, 51946.76098857856, 62087.26028474183, 69424.83287452464, 21695.659335716853, 26824.70363289574, 9598.378156956622, 82896.81154233674, 50444.48828903081, 24363.027243518332, 51891.41668565671, 22153.736717171603, 85081.42818559674, 15698.912050957591, 15950.39802309416, 61289.13967105999, 60426.26871650313, 20530.387955174247, 50431.04360192392, 35477.952736482024, 56664.91742173778, 54620.71353657989, 33288.58620895235, 96069.31995101493, 43843.09385224252, 95285.96217438609, 9949.238612925205, 78224.6070847263, 69363.49599316168, 80158.93791326023, 96460.84614269005, 26712.844780094358, 54041.97192626745, 59397.68707384757, 29671.271873351547, 93773.21943125567, 78728.43310010145, 82463.0206636544, 39459.75570232827, 23298.439927087733, 61764.24400175655, 75989.78503767627, 74283.36992538645, 72861.41415217202, 2211.0596449432805, 40097.195806295786, 77076.03644908608, 9655.538670592745, 21056.48300255343, 22037.551665690313, 62256.8000332083, 22463.125064792523, 90950.97439110825, 76677.85116736668, 53020.75835711938, 30554.39297048923, 4131.684417009063, 69114.07252179385, 4931.286075800812, 24016.843393543943, 58926.84403511722, 30861.024079093648, 7954.9898518036625, 3398.306949546448, 63913.11461291367, 87616.56984041249, 77267.79128026788, 92632.48019653028, 79970.65574090394, 35246.48118609835, 71045.01613026962, 19029.519424041686, 52699.52896209631, 95969.18704986702, 1325.9018795987677, 82023.78812102429, 23763.738382425126, 8936.056103557177, 45236.17067055731, 52096.744867046094, 63317.176529176475, 21778.21576673975, 6225.901199612782, 57785.3414056115, 36197.36822427968, 93338.85336986535, 29471.22476905375, 39200.58693016234, 24224.409966057825, 43897.50598077667, 73468.33468371182]} +{"id": 378, "vector": [64517.77359071109, 16559.893646266453, 41820.77790246006, 46422.05451386139, 80663.42076761465, 54669.72728544678, 46763.65529393489, 86454.12244342022, 1284.747146444909, 53876.16651943023, 18402.204979359693, 19209.64417737918, 42122.61657174082, 51271.49916381527, 33265.86319733031, 26716.067225983163, 56887.92626303066, 90275.76743712946, 11816.25049113778, 34384.78451042426, 6661.805050603697, 54977.24530373415, 33203.137453265364, 13441.999965409523, 75754.87904665392, 7012.093105596595, 43681.820538001026, 41418.04374877519, 80465.85215864182, 73857.25317305411, 10660.022786710953, 17001.39624349839, 4519.269849973928, 17911.06852861852, 28083.098129235063, 2739.190045738771, 21210.294044589118, 63067.626439383326, 63720.01015015142, 99492.70992279943, 23797.71879033935, 70614.52888876224, 21558.244035827967, 3804.8011109648837, 7681.525364537567, 16926.508883964776, 66814.6675482168, 95771.77379266529, 99973.27196281565, 34602.4916800107, 70408.08976313466, 26373.070823204915, 42543.99712496441, 71006.2989315396, 58072.47337761406, 17574.124250780253, 59023.82333148022, 28762.127327831222, 31561.111359763217, 52371.265157001515, 8146.6409377149885, 56136.00494370159, 89995.2064000276, 51957.11866696145, 16201.909964801685, 87494.21824560613, 49887.29631487526, 84567.1932382148, 76522.8288581649, 15861.230126174187, 4815.446137129342, 77911.6088394098, 45676.67094203188, 75254.12803195539, 41103.28512039081, 51206.47055583961, 95056.36210176178, 23452.988041516586, 73942.11694024796, 33105.12152702746, 98190.12356612216, 13045.434584011773, 24494.485870963745, 34532.62356244109, 21861.63216892668, 48070.37874579555, 36432.39067205269, 9654.280592610576, 87956.96594539384, 22928.967498980703, 59819.14075700493, 37019.94808904724, 71063.19585174702, 88253.46064802902, 68366.03634003458, 6248.729136918063, 36576.69807845827, 20309.011635595718, 71575.01958440535, 86884.87116165643, 96501.70611934457, 45621.826286910895, 91330.45522909386, 21941.10221460691, 30536.901429802154, 17723.10732815673, 12762.987437544427, 33998.54965446896, 8806.742816152546, 58345.74738518896, 24341.173950646957, 67879.58567760688, 99282.83989789846, 9467.497298774786, 41885.10916198489, 26625.359527180502, 13622.289318055458, 93974.67172052272, 88796.94644974424, 8033.057266517507, 46752.322078901656, 16857.31246633887, 9409.832034159537, 46537.56739068907, 23820.370813648708, 99924.77299587087, 50802.99393383748, 65903.40956229155]} +{"id": 259, "vector": [77220.877203854, 72511.4663710996, 29894.327536798082, 28994.118413427437, 813.942671400092, 51722.445898568534, 82417.58765656497, 54830.81347661675, 16357.649663833785, 76440.43008350137, 52509.90614597657, 36642.91128553443, 53415.87563385322, 38717.98124756314, 77517.55163023579, 33151.29788343687, 91193.9809214009, 74181.935201057, 50986.12415086979, 7011.927852390642, 81545.58065890284, 82318.73000691047, 30609.20728189408, 52464.953465619736, 81963.78394715303, 14757.031751465143, 7690.3920600930205, 92025.7644375983, 23846.400916319188, 29163.18894222233, 36492.435726547854, 34744.33390732645, 22133.49823480465, 54890.065707350375, 23542.58584011828, 32597.352365423492, 55755.70838220445, 45780.28580992498, 50380.17723086677, 37803.337124240796, 87524.6566985859, 61428.1827782197, 5557.749015888558, 82139.9887533014, 28787.108542460093, 7127.020252703164, 57625.0963661307, 23973.09197730566, 13237.759970617803, 58623.08749669499, 1548.2768205837272, 573.7458517514505, 10127.731519703553, 2554.476253768223, 56786.67001789364, 32412.815166543198, 92364.3159050994, 79968.54218652591, 70987.83906637436, 58627.191807844705, 1922.1013479183146, 10313.207564392502, 89663.85439481656, 5136.227582397157, 32543.333021755672, 87699.96894697766, 87287.36727268649, 15711.622303948325, 83119.66725801535, 30964.401772162488, 66119.24097914404, 70980.0347358713, 74061.31883117418, 48076.510841295276, 21674.53532419026, 95229.83332370826, 3022.725578092189, 50781.56016717449, 75563.22510589211, 57367.65609797651, 62040.85946319655, 31529.955591311744, 78526.62358924675, 43003.78886553127, 25640.240494498234, 94645.33679729358, 41903.85497587798, 18127.365320029097, 98636.6409003194, 83897.49995467514, 69678.11258586994, 81862.38546058728, 27309.897499014834, 34295.84201184268, 91323.68467575625, 22666.788812126226, 96103.59995499135, 5656.140760463113, 61318.40508317433, 72616.18606483456, 86743.89549297997, 70140.69730591936, 27053.04803189128, 87703.58598165572, 57912.74154316802, 90418.85775284655, 33778.350298937454, 97815.80276869332, 11148.041828767342, 43973.23328999678, 78457.73679302834, 12686.461094722279, 35926.990061852826, 53398.93499224202, 18503.010076349914, 96372.94389486311, 17907.752569101376, 39964.79862208359, 3983.420002924498, 18058.861731174213, 40433.52406868481, 67564.59042739043, 46229.51296739647, 47406.78452074387, 11150.087997318802, 91610.59377473674, 27479.15782474768, 86065.64789086874]} +{"id": 148, "vector": [56737.00682606419, 5963.485805624547, 46481.21054121609, 64191.61602584069, 51277.77894092735, 44335.20299680812, 10995.221164437951, 53944.20738907869, 80945.55578574738, 75368.38673953663, 4691.867393709636, 30563.792491728393, 76965.10443389809, 66492.66685551706, 40344.13219621978, 67249.8808874405, 59199.07478806292, 56754.48630219133, 29772.217144659397, 97135.11786139218, 34465.45751652736, 63603.96438752984, 7609.422072525851, 32482.401942515393, 70323.72044267507, 42493.49957564023, 51193.910071071645, 81286.24363159512, 22586.13092320615, 22344.66083915089, 16088.369718846785, 90809.2656601393, 22029.831259852504, 7747.293365002872, 61465.97007340268, 77445.30598657178, 42606.614087042806, 25509.19273860821, 49981.51175894761, 36513.98110277875, 38977.16785529866, 55543.33795631865, 25671.680571710942, 60821.37475055204, 8424.475440876655, 79856.19695479787, 63583.1532091692, 83678.91323986971, 23763.846613478156, 74852.05804094707, 15170.513652590933, 93632.98722403767, 39958.50959847883, 44521.12771543133, 21876.24170071304, 32279.518408919772, 59785.48326467416, 46196.580221531294, 77339.95234027957, 23069.058698708457, 21491.190134920467, 60950.572359630416, 49839.60537565423, 82515.0422501804, 95094.06410249391, 92149.31764074869, 29436.10988851928, 51170.73141740803, 12082.73673992627, 96945.66014892588, 97940.7644985023, 93157.17282243932, 91394.97498710112, 59628.31053308787, 73956.9494003435, 14733.401684013692, 20432.42568033321, 27860.775578111698, 38973.79150391528, 75836.66956905769, 69928.19677537183, 56468.91804960718, 77535.43261237498, 39481.14462367001, 99751.70334691169, 15433.900943891398, 53498.26636229876, 23037.808868672695, 99899.4126364532, 54258.873984352766, 98945.21763698936, 9995.766659377658, 94426.27009439353, 201.3726747201572, 38624.02768625996, 44658.458933591784, 54936.214621370906, 46081.968379063685, 14027.112306577905, 37303.67251693175, 3822.3075925507, 65638.0237964116, 80620.59156560768, 29808.074447083465, 54951.87070443488, 545.0124347140539, 74459.60964682233, 58119.866211374814, 8504.415026508905, 71405.81706340522, 17126.91209031827, 648.7735411696339, 99321.94186984537, 13376.362389184715, 92959.67424616792, 30227.150137245164, 26504.92090123976, 4802.394698523238, 58867.05997329893, 61288.842183723136, 94492.21382730233, 29752.00651655412, 64476.458000812905, 74642.01794807923, 98638.80673651827, 75782.46955732924, 2637.4122195668815, 76896.590309143]} +{"id": 1768, "vector": [32006.213485580716, 31394.17258861399, 5462.010435665432, 28575.607277854808, 28870.8778097297, 96370.19789624613, 24930.22956368106, 14105.213186912091, 67999.34462884437, 75059.90822071162, 54810.66228532022, 2764.0414680630274, 62160.13096329825, 27667.47483572939, 97270.89312086995, 25554.159851596614, 74905.14619828083, 94875.90810451828, 67430.57954590395, 16788.30912108762, 64370.56662613505, 72688.49474358567, 21740.88230165826, 27536.704522906573, 7288.895969090859, 84923.14153442501, 31595.74770760587, 37522.17455372035, 76530.33739315932, 34353.9453520717, 98645.32349883999, 74504.30114888068, 76194.16526281969, 38784.00568496228, 21220.899206573962, 77858.27578071064, 30121.84765842334, 92117.13447678242, 65831.93672505544, 81614.872378199, 93369.1215281208, 63087.02601045825, 76459.19042438026, 4335.319028754514, 44884.82788631596, 81813.63990765803, 25677.98058658135, 33893.236719478904, 74259.96499524012, 59113.59185521501, 21388.192654227933, 47217.15905409904, 5074.949725056033, 13413.779204329201, 79715.75476765256, 91090.64783888306, 7666.58048719836, 15878.696807128079, 23255.175412847573, 76009.32915321304, 86610.28036697788, 61046.528653753296, 53991.985965792985, 53332.48683878321, 61321.88145159703, 17875.239466900493, 81730.76856030441, 35916.414409167795, 46292.69552949862, 49673.835406936516, 76847.96165930924, 56945.41880649917, 5116.032598902065, 80034.14815907487, 59899.14965125029, 5789.5701380887285, 84173.26820553245, 61310.21338176528, 9319.954067868663, 25657.789270063146, 59784.16889308715, 41568.83728172067, 36661.330052581994, 88028.38228692421, 3744.9301615739337, 37321.25643642302, 24183.48794472005, 61784.1457417031, 29915.464972340968, 30682.442911730766, 64161.44306751417, 48214.85594030492, 66297.5332333746, 57809.82830736293, 90749.03748666303, 13951.92130509555, 96647.98091794823, 95288.51036973963, 53978.86486664886, 24897.963032154825, 78080.5287490547, 25597.102413438468, 54916.75546986904, 74794.36511753395, 35934.02589705784, 25347.169866909604, 71850.36947885872, 89003.669505229, 52733.61649817945, 99767.30338113412, 66.79495423882464, 83383.38401053914, 7191.003385719497, 65548.00536183339, 20137.73793860375, 69226.48218569998, 66430.30348942068, 31056.35404792495, 40613.39970218077, 63490.959469938556, 99903.97989121325, 75764.85643438876, 23934.401122730655, 22710.301129354702, 37162.94639631833, 63795.876678566034, 16059.874811611851, 59470.441952497575]} +{"id": 264, "vector": [17826.85557720096, 31231.98988632292, 45377.90548114449, 927.1439646565849, 35707.10122175279, 53787.066409711915, 78688.06687827641, 91385.82364369709, 51851.89929852504, 94732.17981396783, 58394.02774524907, 66033.73070381237, 53786.72394395903, 65118.672858010854, 24536.220988599867, 25313.650541636758, 61415.7524739675, 10293.231760500632, 73448.74442227882, 76143.71816748692, 20986.493443572017, 49170.00797189755, 72305.12516765256, 36183.36550654263, 94541.50106586474, 23935.276998322406, 44400.282264695365, 91559.9965592014, 25007.6245040583, 83206.54386222355, 93775.65704609297, 36879.77625668949, 31525.84696551891, 66163.28734047544, 44152.41066398917, 32654.874681271194, 74230.95974008681, 45490.77584960706, 91699.36810727461, 23706.045742859606, 6111.285312853487, 74928.61960941402, 7151.892918096148, 84737.2069223645, 29129.322367037246, 40411.75146561876, 22237.6283192548, 85635.98007542251, 98228.05039503558, 83938.60607706849, 62832.599793684196, 96739.64240174826, 27155.269450694264, 45898.15118601993, 56376.89063176001, 77040.52495670784, 80517.04962980344, 55321.46344969412, 30100.763146642996, 89302.04438641503, 8237.169724867388, 87255.90477135283, 12216.266900711293, 69355.58732372541, 99994.0024907227, 47423.457738589444, 59981.22059853173, 72706.70331509445, 98771.06727026426, 16273.831722335563, 77006.71388181107, 36034.24233751748, 34202.75867407015, 1687.75028910505, 3108.1052217955876, 17351.89644187606, 79479.95695838651, 4218.065317595243, 66236.16330707773, 32071.69657122755, 29919.791112165418, 42215.509620837875, 73822.15125144793, 24873.472707928755, 10393.30233578818, 32469.373210780715, 66456.6371027509, 42659.58336665922, 5525.468227946461, 42041.714580308086, 35595.592303588564, 21225.076412914066, 31778.883646786806, 23295.797737442892, 99345.50790241593, 53879.48400030514, 19019.020313381487, 65954.24068428871, 38787.77644350844, 26571.762273778986, 71161.97124790397, 4545.809881620244, 52043.07273750721, 74921.64008536203, 5328.779762599523, 15124.014574133216, 10548.276629984455, 9815.03293744318, 12129.370078612012, 34719.74685602086, 58111.609143541806, 1532.1927718921713, 28905.14182053672, 77268.77563340269, 10804.251427519963, 45840.99770008849, 72356.24436424192, 58038.43737864002, 35341.72462453544, 35106.26887050895, 53788.30806745305, 73102.9353435308, 22749.94054709949, 24728.115352399305, 40076.98171968509, 53741.28052758472, 61119.60496675477, 40715.443768162564]} +{"id": 623, "vector": [83062.86468031969, 82788.59217344629, 68630.55043733766, 22347.36631463965, 97010.79702075972, 15645.753019170194, 97887.6716199544, 82618.26312374098, 68209.02969315524, 75596.76124867234, 87595.3127214022, 93366.92350356998, 68873.19985925159, 81974.83071916258, 94041.14311962404, 15531.931428911017, 27747.38885700366, 39100.099381982836, 83707.7614476576, 54057.07327754053, 11563.066496779895, 89921.79739163448, 68110.57619691522, 38554.701433129216, 57289.52196155753, 68234.43275131092, 30712.057942280368, 31645.688813128527, 51223.75739532151, 35051.55753684276, 45609.58487202994, 35499.203188108186, 19462.749882764463, 56075.85759652829, 37207.32934259016, 24629.87438615457, 27397.57744435175, 29011.783120963864, 67189.1931800887, 1133.4293459700827, 90648.88398297498, 14837.330300184003, 92178.2022873454, 4081.9741519985796, 57797.50813133258, 91517.7756136822, 57910.20071221632, 74169.5591752546, 68356.70626840403, 69698.01525855107, 11907.192608990757, 52479.3759885422, 8508.989782596132, 96250.67503458359, 58434.39470230277, 18276.109723301604, 58875.44601212591, 71192.23830750183, 86973.17124689551, 72329.77664615668, 44245.665942068066, 69179.49886959005, 72956.26670809496, 81060.234003428, 49985.897577411844, 56219.00923809465, 95837.62760340757, 67490.74162975667, 56984.715492357296, 67546.9715293473, 32331.800689307212, 37646.03621408932, 49579.07590962115, 56851.93230710551, 37802.262423116896, 70008.4039673007, 94861.99522435975, 77635.66455182881, 91918.76860262084, 65132.87501916487, 77790.13019303698, 95985.24576022376, 68032.25415416721, 6676.090969730486, 23735.15070622566, 30340.07442066964, 99401.63710455495, 59024.75825792027, 73871.46787082344, 51533.9407291287, 58157.32221061822, 3119.3473955647423, 85906.71036641105, 16634.718537293567, 62911.4410055994, 68948.64891695279, 20145.458199741373, 97314.65159830998, 84464.39684501194, 4439.833017029638, 26112.218059010618, 61381.17255025705, 87822.89717411035, 86799.77146074896, 2378.035985315152, 48323.19950235272, 77759.0788971082, 86860.38238732031, 8885.021084444345, 909.325027781016, 75742.01013820084, 92907.68003167427, 46541.370826686856, 50208.69651187026, 33411.27809495314, 39079.09000017755, 7494.324504489069, 28615.12718344683, 8128.39151127831, 35056.92986127589, 40680.7456024146, 2502.304174121728, 26491.413555381037, 62800.30848342487, 87196.41823359547, 69827.17035034084, 47330.29549311542, 17043.71445208023]} +{"id": 1973, "vector": [62130.45812060618, 82545.1841038795, 37975.8766058874, 19809.216945945107, 3341.618882331454, 50947.32700105751, 16771.552442541128, 18653.236486915026, 75051.4150748477, 97920.43791078548, 4910.876498519623, 64163.737962405554, 87572.71853797876, 77017.70549948703, 57162.05422813547, 49177.16266892639, 85111.1091744662, 14404.452195326756, 83903.05856210971, 29132.62503930781, 21995.183648562743, 68372.15571221069, 74719.17142879045, 67305.48771271136, 82594.73514884136, 79297.63295745374, 72867.64874539495, 6653.794195108265, 6246.430548880444, 90467.70528600166, 37151.89126903384, 2365.645475089351, 39539.611056039845, 73243.1132707739, 7374.304982253288, 83518.27292045561, 29711.75646278327, 23780.544504969035, 2372.2401552195406, 25039.68284709319, 7313.081941449895, 29498.008785417274, 24440.13447270319, 27055.97732138958, 24036.23269462193, 5251.984151661959, 60061.0303057318, 12894.304417698888, 91404.30790535294, 50854.03316472386, 84859.27102813195, 91543.97313348624, 53300.121487333854, 67243.90952422566, 8650.768195879065, 69867.0639395213, 11472.913116793105, 11519.324539535026, 9732.47590386065, 37792.33680558236, 13656.050596497493, 80997.22556829096, 52765.64992729784, 24328.774164803544, 18718.196281369383, 52699.28591321165, 92999.28955204654, 17586.816123874494, 3817.0827331699074, 51496.653370720676, 39837.9551443923, 9526.3679291264, 19808.171526132202, 34941.253401227026, 44969.573753277116, 36963.54552741788, 26398.028830147035, 18628.722625767135, 6043.411958066269, 80122.29786029925, 8528.995306380983, 84417.67534758218, 37034.79589318419, 78396.49898623457, 58481.60217327246, 44709.6626815862, 1532.9304185583092, 67810.06124287283, 50256.622576469555, 99303.53841989396, 72660.35032827484, 16880.609046828788, 33165.40994997756, 41879.65228532745, 33120.382633156994, 66411.06850058795, 64721.06812435483, 1258.2792648590635, 22517.326605138744, 95444.14001547697, 82722.64885062508, 39756.2845096053, 21668.269690148445, 87423.11121368044, 97054.22242323187, 91617.20564533802, 27144.207429000046, 83807.35473724417, 34978.921127797934, 57943.81077866587, 91805.43259508688, 17197.16025588216, 8185.077837454069, 56562.65671996273, 77335.11231055064, 64115.72049138543, 5801.012039207465, 79629.99976891832, 21135.048078993914, 65011.69449215211, 88815.91832213622, 76773.4883216553, 14744.422646985555, 48224.034087686596, 48143.073213947595, 9164.846101355895, 77669.95297660548, 65901.53282524429]} +{"id": 1367, "vector": [71651.67296177922, 29531.224148913614, 11175.54898129205, 87623.80207883293, 18392.12187678385, 73551.81730563902, 1697.5448718923426, 62731.43511492889, 79784.72545466281, 7642.279041925038, 80071.41791947043, 2166.92717005087, 10040.168056845954, 86527.32431783767, 76346.0624477045, 18011.530363847673, 26120.71418752766, 75218.3253868654, 7771.3974886489905, 29267.64068410871, 22985.461604197066, 79848.06380274246, 95778.06172487185, 37021.18877727608, 61355.50387922784, 59679.11364574529, 92823.75092991552, 83131.09567663167, 72978.83882339438, 13245.636107710945, 65615.78630527937, 9739.497299266153, 14655.445775053111, 59339.973143395575, 28447.46824826191, 23779.80573694073, 58125.521009040924, 56890.91572578805, 53059.91062968478, 79081.89537732568, 39662.37075300697, 3946.377945166446, 21003.436913765407, 26963.76724777908, 65531.69173705463, 11916.545674580226, 25277.881240149523, 43679.48108124452, 36656.34871892907, 90125.57057056602, 6861.996995873532, 75986.32312927597, 74021.52707107412, 58905.19141584825, 2654.2569377651757, 29961.229573855908, 422.8869637330934, 63232.13906170594, 99539.9952290467, 21946.92840485989, 19397.063334097264, 88893.51146375449, 98586.860544195, 95272.99105623456, 35836.66098407902, 36276.22829640712, 34126.470836270026, 17806.6820704978, 48040.62264970994, 58714.145587002684, 77193.53681770265, 32235.591256640462, 95544.57977331009, 53.517296934813, 81151.83758324586, 51468.779020129055, 96295.65016475439, 16147.474391943317, 75730.69001144898, 20586.820544989336, 10346.47039908465, 25039.836133786608, 81266.76230984453, 8519.342450936983, 4202.407398357833, 6414.954553874141, 63153.83196304892, 6412.854030835524, 38420.691723619784, 50873.29394717466, 14424.354433766273, 74905.70725193649, 99419.25597065844, 95999.53339577296, 70771.18994674351, 34369.425674230115, 39390.307815439795, 46961.914011024375, 91960.6085291667, 24953.27283922216, 5193.57623212815, 80393.530483365, 41375.33704209301, 12956.489670869509, 37095.99950257727, 87566.91664404764, 21258.691682798704, 28422.034741254676, 39536.94100631184, 28887.848543619843, 22694.134927455765, 64311.60654742328, 12297.04773111948, 62529.02892825383, 14778.05503290356, 76895.6336785828, 69919.02005146364, 73010.86638664047, 49288.3984403933, 23093.837295066645, 33641.583730276725, 15020.26546352776, 84872.6203822837, 27921.268745767957, 84687.36132881307, 11463.47124748367, 50604.35899189616, 56108.820273351455]} +{"id": 612, "vector": [12010.155679664647, 31862.356534738647, 19562.284389396857, 62240.1234168372, 63639.85620799511, 85748.26876989473, 69923.88992149812, 59668.20633903173, 43216.89039564793, 65683.6789552878, 54723.11158845047, 15472.476897860888, 35652.65344476607, 78667.73852852682, 99079.61516252656, 80176.0535994968, 95353.05954940767, 63302.996550582146, 15816.872831931638, 44896.92246472118, 96802.53443731762, 78287.31666943697, 7672.057518443531, 99433.64139837353, 53169.78313624689, 75016.29419774236, 24026.37283787945, 41378.10847435246, 20126.410763708813, 14627.599097301358, 19619.91177634973, 23415.64422056819, 35767.549640424855, 70716.11608646745, 19399.373193210133, 38512.949406375796, 16470.40133668599, 39415.142170670224, 86221.3690089283, 65114.17675540392, 32042.94181644297, 17281.845343137982, 1500.5457882283247, 6534.2611899804615, 14517.761763272296, 87272.20151263602, 16460.53565508372, 44702.96492285841, 49756.53072218099, 49553.72467676381, 84149.83288481808, 1369.6418222446382, 28484.044218879844, 419.00748122211115, 74098.2873887813, 95048.72929766825, 95587.5969146287, 10042.025146288457, 12141.290208393852, 22011.816301240895, 98221.26084755518, 37404.06834301998, 34402.38782526161, 63633.61115776492, 11138.950132146298, 72980.84286646606, 89460.37097660247, 60388.99671252599, 81576.39475940148, 79588.99034758096, 28123.94799778004, 38541.67735596208, 15240.674614356352, 70385.4525443714, 92717.09046179456, 43218.4640906736, 73151.424884812, 18964.720338621788, 46623.56947295544, 46913.14777630462, 20950.266611397717, 97365.06269469432, 49119.29748666309, 90325.91753959106, 62906.848957030816, 44766.91440509507, 64041.43420769929, 81500.2994771805, 27949.229980480006, 95085.17094657078, 56821.949151087036, 88751.75315038407, 58975.61526883269, 20729.941572815947, 31798.642758287355, 30995.101011386094, 8349.80578468666, 17085.0838025668, 76463.27305222389, 99126.92896824019, 96750.96146717145, 52228.057349860646, 99298.29568786571, 4353.541195221255, 79639.3024048937, 9765.071750887344, 75944.78425861939, 97780.95502252318, 1177.0433888268262, 16185.023596988978, 15491.781910059888, 82124.7391599446, 83688.0388929343, 58009.42572432218, 31234.934462032903, 98193.53002651218, 60251.613447573414, 52795.612136114614, 19145.863308171753, 27847.91327070999, 9661.36170980303, 58475.28813602922, 84643.84834907179, 3996.235248751534, 47890.80048958083, 20232.393509972924, 20390.826811620344, 48269.762564157136]} +{"id": 1972, "vector": [3546.2139554963246, 93395.55954892602, 16086.203161313539, 70883.67534987061, 38062.96364326968, 49672.92069744155, 76591.73286877642, 5089.923628998216, 4933.5086320901555, 48271.78142790122, 25667.968909236348, 4283.66643204543, 16247.67695611753, 42791.69207641018, 91500.4224529051, 89803.91581786927, 18062.9859729344, 13348.722761128829, 90494.28783071181, 33586.90984762818, 21729.109968244287, 29144.723441045706, 46887.47922933196, 69494.52815389146, 1703.5836045729557, 93750.06840153663, 14897.789033081677, 36091.05346787921, 40529.16884332882, 79505.87935727714, 33425.79703182845, 26296.608129416276, 76210.58599574566, 17811.46701344307, 70794.44253462185, 43301.22711471889, 53440.78002056132, 44130.07710538613, 64718.43921098027, 30116.055490437033, 90002.76614698513, 40833.43676000903, 93314.71334202433, 75367.02535794023, 10933.067964162512, 80301.2125053825, 14928.516691194294, 46864.676463985365, 88423.94370689131, 87451.31551286773, 10718.302621804054, 95846.83838384652, 49022.75210843125, 77461.83923622107, 841.386662213528, 14745.394114065113, 57737.969375410634, 83415.69655497199, 11053.482450147734, 48535.69819961261, 32746.119999865696, 30326.91529295187, 67249.44109780983, 41033.57865623286, 84315.20333039522, 34123.20113149624, 32923.30981443629, 17005.578653867327, 68488.77711317071, 58015.07349219957, 9338.319675437779, 96077.03539661733, 24267.29957022298, 6776.837449506934, 96287.15558164872, 70036.6260851642, 53585.604245750976, 47999.59062839287, 72564.06446367514, 24887.008070292093, 25858.73021898807, 65233.717676341796, 13968.547984136103, 86670.2944824957, 51743.75912144289, 43713.51294042819, 81497.01520056523, 15405.660307902113, 35677.55002504432, 6432.898131643805, 32435.278196856976, 75528.30299203034, 91913.48966173783, 29267.87345396362, 51156.23532538753, 32956.329721103284, 35869.73837838836, 99010.7736582725, 72006.5916463065, 98541.82360534617, 94957.29233619462, 14938.627124039072, 48092.06103431418, 33019.688171774505, 96908.63884725324, 35903.847488807114, 9700.768050015018, 68812.20029243107, 74224.92935211086, 68441.08212910313, 73675.0465577829, 30918.527485688875, 22031.942561895656, 16785.184752837944, 84349.44397881061, 48814.57117433045, 75721.35538673382, 61686.25694936708, 41926.96709704824, 20279.911778301506, 80713.87304463085, 75192.36763774569, 5172.886620911665, 61534.06406403659, 80502.52559293246, 77305.4339927702, 98959.6505981917, 10484.919748286504]} +{"id": 484, "vector": [49754.977818834144, 13081.620947773898, 79685.02500974246, 39905.44392451659, 53281.44853477008, 39048.760552365544, 38598.056281995196, 48008.39084371944, 825.2424899361399, 37439.567211528556, 56978.85156953285, 60870.08677523657, 33017.10440636348, 46002.99518841905, 24470.832331468027, 4465.171723230222, 73495.21647189835, 28573.768996702966, 83362.4004680179, 49330.889820297074, 16829.74447149955, 28441.0347309787, 62239.16784096634, 28642.10675091279, 10691.383348057216, 60761.23716660687, 81168.21911252513, 83249.03077087167, 67110.45907337019, 78346.64822765112, 89891.73887392445, 32110.17000149775, 76910.07501327686, 36697.65912407855, 61743.962122812925, 92196.64864565252, 73019.18214402322, 87237.21288477205, 70349.89255576166, 89010.26400711376, 87021.66602787198, 26844.291113536754, 62021.398513865286, 43085.11138974166, 98751.8583785268, 76011.17233673754, 17431.694835288457, 53895.09778259395, 45855.86667059942, 57178.17659131268, 16171.860918111324, 69390.1020130885, 75412.54276823551, 94845.37335447311, 83706.37462972564, 45051.72181690509, 9218.962893290849, 71557.5592216605, 40638.86016622006, 35859.511674592235, 60103.00095960884, 35522.90226155469, 81852.88133457495, 79399.0840061249, 76924.26156728469, 34319.65007338202, 9337.99059258451, 18625.896061415137, 95370.82719291425, 66920.62311785345, 46429.667820246315, 5192.707580045575, 46939.91265766788, 14132.635322156728, 66701.58890306916, 2584.7270951658174, 56478.05865063463, 50175.58510131623, 47727.98439605546, 5630.184362143853, 10136.431838422055, 72584.07883610835, 40419.195776025306, 5490.186127978313, 54528.334558625866, 59115.47214288692, 71925.8027867407, 75595.42556559296, 75629.27298667394, 59055.352748895275, 66974.55031919354, 45265.29368101099, 4835.644731493582, 73679.67522520776, 6764.387749766088, 75664.92857273246, 98659.68599172453, 39484.44991039179, 91085.1119759872, 32963.71438228607, 41140.16668384445, 34379.60560183862, 96658.62282531627, 76536.15789012318, 78777.1498793094, 50435.125096946, 72343.00554128595, 5964.106219435039, 63519.62939609041, 90088.5550257987, 99283.67872565288, 57152.32773637847, 57599.973175251354, 20008.467224709315, 61329.26057555822, 85870.05606290232, 26187.69834360959, 35221.753180524596, 34243.3555648706, 7328.083912951544, 84879.59777261467, 15356.778844064012, 71234.74927364079, 53748.94348782538, 34996.9924030082, 75104.78606539743, 33850.90907487257, 13012.577597604159]} +{"id": 372, "vector": [91433.26599670094, 13211.117819244178, 1691.307802386488, 60832.110684408835, 25720.472393540007, 58643.1955347543, 54843.913045242196, 82026.60445503809, 13218.725682346721, 57887.33199954761, 28595.28013443553, 25977.218384873537, 54145.948111015976, 10075.19493132557, 19965.64863692204, 22235.09380918923, 99765.467068348, 80099.94509335345, 93623.59772743027, 15929.51949221375, 2709.700722267172, 91349.91790520135, 1642.2349406148396, 16593.874125441045, 29049.88985322989, 28101.5850416655, 20533.86813362411, 30541.272549873265, 43638.72037033095, 27083.093683212024, 49350.450703439805, 24111.65614759705, 34919.66948709909, 82741.33774458837, 45963.69831872779, 84729.73203860893, 99605.43455044723, 96299.30748971157, 45906.876984410315, 58602.48454722256, 29326.890423998753, 39800.85525937624, 94784.08478475943, 98206.50279281601, 37924.20953931319, 73939.6912726458, 52336.495105702255, 85136.50809038975, 66902.39561532687, 12759.622536982351, 91810.45741577528, 3526.942295107072, 8785.713136009432, 16358.800981100452, 59927.318065992484, 43863.29444212909, 54444.393631107334, 55670.75839342994, 35712.52725962393, 18739.148092223677, 56704.36689209966, 82769.09208081414, 47480.000393442664, 28814.158413006717, 78198.2180379962, 71820.47241066214, 52778.710286366535, 74257.00427710795, 99773.49593768174, 28311.148018678145, 31888.436208058323, 49190.159733516935, 46951.09729054946, 94511.59306382638, 52323.31905684805, 24735.073674337138, 78453.87940338202, 43054.585187988705, 67355.01515660217, 83696.32178245451, 95666.03181223883, 49473.783942044436, 79426.50762252248, 80342.04522744345, 67830.79978963574, 46933.07935518538, 37335.569286530204, 73472.8362329794, 37699.14047685988, 640.9372695531767, 19975.358802565468, 82414.78044638677, 39736.168528837734, 42937.84183215237, 94388.08203742548, 97843.04118794373, 52499.895799729566, 29382.884217456816, 99376.18922575653, 81459.39421401553, 31206.112035410504, 80572.38631250254, 64286.45959766002, 21654.492001942926, 32955.14184463993, 27027.392628698864, 88438.60850072389, 38640.69035794757, 94743.0740148994, 44472.47944302044, 20936.07266904155, 50234.38171502573, 38428.829379367235, 51401.29717411946, 8405.373533142623, 21682.567777453853, 64760.008148608875, 66203.027980002, 80794.74868064167, 27345.1390361765, 94174.1510667108, 60688.013855405945, 91099.26327105955, 7482.272805850088, 22630.51118346132, 39.736701485437735, 83267.30723284112, 24563.85299906716]} +{"id": 1868, "vector": [49651.6337069608, 94848.93827409954, 79107.87027438032, 50475.17748750759, 96409.77851810353, 56206.16261586911, 3329.619607452361, 20540.00942866453, 77898.11630551526, 18750.95004896161, 61628.72036628637, 92218.90113789089, 94891.57526176363, 88095.3260539967, 10650.499324973962, 21956.354834527792, 44870.81771132458, 4257.374309966655, 37871.275274192674, 36616.1901744127, 89132.37551255137, 83774.61070430765, 62805.101891160586, 51650.402304905365, 1721.9463141069923, 75424.52593742408, 17547.952832347746, 27116.415514593005, 37232.89268949178, 1154.409454411176, 89370.27479154587, 92325.76284147984, 37172.155541488115, 13773.173687143537, 73099.00704808338, 20993.506253700034, 84632.4966656301, 57575.392089210545, 26840.784312388267, 89122.3498346747, 50537.00182508156, 64062.75470861135, 48188.78900662883, 46056.80837970162, 65121.89075968002, 92182.01118002363, 89466.53837335762, 53682.46693392886, 34767.957437099714, 97671.83519116216, 68912.9200859563, 7021.185953174847, 78852.24874955529, 89774.82072667583, 99964.26360915111, 50873.13511650931, 35459.56184734358, 51982.08660385608, 10285.780035982727, 49940.65952741201, 14695.719257873585, 10921.018517317338, 59864.82402604796, 35325.457700897525, 97930.36705675775, 68276.31818131394, 7469.041356472428, 84693.31387082447, 60303.787294123314, 15529.218479147023, 16144.48160841654, 44569.94653934155, 33874.525529916464, 9651.686733153087, 42205.47918529749, 20853.208252881283, 99560.38900272358, 14482.730835403112, 45982.29450104025, 8113.075378247103, 21429.261755345462, 27930.743573635784, 2719.794253297414, 14292.821673462031, 72181.53502724672, 4734.400853343168, 20193.08997678302, 69304.13079781146, 10584.302503291232, 86623.43631075225, 84243.40177273497, 12482.304038705639, 67494.93463916543, 47339.512163144784, 93131.48903800405, 40053.72555120134, 68173.81949097865, 64429.9054992789, 55260.18237521888, 64246.348275799704, 67945.75180158082, 26464.920153602245, 61284.31682480732, 144.42147025579555, 35939.635277521054, 88115.4904949085, 91908.99969068718, 23995.861244354488, 44655.7140473625, 91629.41481721547, 82004.19385811353, 93594.98340484386, 60255.93148789678, 99163.88345171468, 24808.18759466006, 96998.30829742532, 21803.288722636404, 5512.696101452741, 66670.47027580891, 94083.47503056629, 80482.15063422263, 9379.582336135329, 2324.659602022727, 56036.889141100735, 59303.13734435795, 4380.2631438196295, 15511.798220071827, 11674.49508243792]} +{"id": 1241, "vector": [89712.18433041449, 9434.791192334013, 27729.312894098133, 92904.62190695544, 69946.70110380457, 27249.084172958792, 7299.717412693718, 44436.40864818856, 16689.91292998061, 55665.7328409289, 59352.60468728157, 81861.26333930872, 20437.121565461402, 9774.530449387665, 72211.76220003719, 29794.55434493079, 5360.4178684582785, 14617.826273024748, 85073.3952083606, 62610.47561465444, 90923.76252818183, 8474.74091318774, 94099.8337846553, 77777.57624627612, 96263.4216509917, 7161.040449152912, 92524.51773094949, 5074.439575102707, 74628.23338368317, 68952.99295875114, 17639.201011982088, 83933.33085959528, 45502.22737338281, 92725.1242010722, 73702.60452330679, 24625.515449641323, 63467.73835122421, 82785.47121957631, 42871.44343847012, 11556.102720893257, 38542.420011118484, 47594.19939030192, 79342.72000437502, 93579.0139318998, 84314.73305296226, 19752.304590553882, 69753.51117894969, 71338.02043281253, 13213.308210116515, 44595.76724703417, 89226.59641374262, 45031.20696447965, 99331.73029675386, 97682.91823167284, 88687.5420810164, 68582.97839826772, 48080.03975052657, 16234.187009255174, 58345.17680352544, 96527.19069399036, 16215.780260707746, 5517.75954218815, 91882.0063253081, 57186.36960402297, 7781.201885963163, 54070.40554535404, 16165.056018275514, 98992.01266144894, 57031.15552120166, 52833.91173771978, 13237.872222461645, 54560.04870300808, 74606.66542108027, 6932.838594764301, 6718.766611960758, 78716.00103492718, 408.24960996974636, 11067.850674627356, 13341.868526479117, 90109.42141492595, 39513.194321742005, 83492.97810739334, 94201.14076907506, 10410.407040292746, 65475.649320695906, 91125.78177547503, 82038.14201284992, 49984.59146798669, 93541.5535455851, 74461.59331400732, 5677.436329839092, 32456.384216385715, 40505.736007970016, 85169.83754028229, 27155.856594647932, 5138.187016177442, 3775.0431040613375, 51672.83741798836, 45180.738267618326, 46969.897623655364, 7218.270641926328, 94453.40366296622, 94710.77642526818, 8220.302212134611, 90164.31198716455, 76454.00694797507, 23536.280988431135, 13405.030969763155, 27630.497924037467, 44546.952918377225, 32805.14878259521, 43801.18165912437, 15218.69351302112, 89704.54371726561, 76757.28679185714, 46115.4812068645, 19933.941460268423, 65173.70667360602, 29636.960670772238, 20220.045096238893, 48480.45323072051, 27219.306008203203, 28530.4946322403, 14346.992729248275, 38299.16600027847, 70616.36618149349, 58733.10781286467, 11926.750877316961]} +{"id": 462, "vector": [88436.47232906352, 94218.36935192939, 60146.623553245096, 48970.989716866774, 1954.8964325369234, 36371.41614261266, 25201.239400837607, 62976.20993011509, 27275.411941329476, 45469.60972848344, 50781.38595332301, 50949.017219169735, 32828.43284893731, 1279.3438282942548, 72836.74180598602, 79368.63350652994, 55177.7468743218, 22369.518187060832, 85351.47277585004, 76009.88641068769, 97063.71497384309, 19774.252074809396, 61631.664487465845, 12263.847665437277, 97295.57722323376, 85230.54513146346, 79496.55614263582, 29259.277079497213, 44853.2796835363, 92538.43538316143, 63508.55755134733, 53998.9142690835, 78854.12762632033, 17108.807813975767, 53794.605261945486, 188.3606875487831, 16661.833632107893, 54217.66767151357, 4019.6568741828864, 41267.7953509447, 54095.2179481189, 12242.350848155958, 73823.15336801611, 174.41027200761906, 68199.13534151088, 504.9250273493011, 1659.6774393932922, 89466.50169135362, 85062.73004660232, 37777.39123273957, 39399.14462015299, 47791.00443188567, 46297.86456240832, 79989.86793497819, 68498.03822966886, 94690.46170080999, 91697.81306303608, 53758.12405872976, 2422.3504516896146, 64212.9252781411, 25488.64775254972, 65922.08382453457, 64675.47551792254, 19961.47041251197, 76598.94196492658, 84643.79065724314, 70431.00543612492, 64577.890082985425, 97689.10017649955, 24100.545367075843, 46198.42393393653, 95608.59489561239, 7543.921367393169, 77569.43150370075, 82173.65684360728, 8420.13179837845, 58930.79152153817, 20046.806893545956, 14545.732746173146, 71271.14514531243, 45876.416131874656, 60325.40880528565, 62019.546206307605, 42956.4389898781, 78627.92313146891, 82427.65280079067, 99157.15126843652, 67524.556853865, 23811.1534624315, 12725.60251646816, 9790.519171123102, 6896.540468757439, 23964.126544933395, 50104.35613143576, 83513.95447844893, 66350.73281432144, 35463.567385923045, 78435.11176320558, 20019.74455616279, 31050.959760805687, 55783.77693905924, 9467.289231452192, 34258.13227704514, 82844.87835176814, 13347.478193004714, 88729.9751908716, 4588.1824800323875, 6838.187571723264, 62230.587919554324, 45030.09184473653, 99182.18914119042, 48758.130329073814, 65157.9978126928, 93044.12639658577, 22691.5744392845, 20663.69237636173, 8735.693935139088, 8301.879849361094, 31375.08454645026, 24526.677138162868, 54106.805809339385, 4533.551958902726, 85601.7353104183, 44436.07097282875, 18898.343757915416, 45838.13666203479, 84332.9187402544, 75067.4907722543]} +{"id": 222, "vector": [30988.25391317168, 90169.79692953483, 37500.20846085623, 59328.088279780444, 87410.11710983007, 32906.09636869351, 3868.3860206932595, 70213.49242446755, 84037.13261121261, 29262.77683852252, 26142.015138440056, 61917.40394807197, 38330.47044156127, 78073.24829559769, 83640.07201369118, 5588.5881980983095, 42746.08434059613, 97426.78918797914, 35799.476908377284, 42522.96376640661, 37990.79409203022, 24280.294101040854, 17921.970449340406, 93784.94349663124, 81554.58558449501, 98960.14886621936, 52227.14955313838, 89643.94681759013, 37356.61286731011, 31544.3344856266, 57611.76918159583, 31933.887979909814, 45828.37651709617, 7176.786209363783, 50146.370352402024, 15811.765315789216, 36076.426616929166, 3279.8782306124763, 83167.09457321136, 23260.392810903508, 11493.143535222995, 91992.53804997398, 89819.90092056303, 36119.62510573516, 56123.717223924454, 90366.83040495073, 32750.49686187089, 2009.0048363578082, 88886.22400543862, 346.8750014990918, 36194.76326799482, 29549.789857697593, 7276.721066040836, 3284.955752308094, 90164.92404692985, 12288.050120312211, 80203.28162971538, 39483.41994784083, 4581.591853912748, 13143.32100288239, 83267.8976459316, 53156.88306384677, 20396.32485199342, 94459.76254080934, 76373.32506571167, 71379.28465749322, 55149.756496354305, 36831.65164291353, 50795.73897095072, 28381.273660631745, 64804.10461890189, 57994.82480116056, 74115.77564586823, 46443.27283211388, 59720.578688748246, 43780.08086080466, 89513.19100552071, 94089.68092900021, 90551.78258210333, 61109.5234449013, 11478.323884720732, 73132.2117099543, 38191.68046726237, 79150.21873510565, 74460.72850468704, 54956.249760697974, 96067.28766766922, 97358.83896774879, 84431.7949943242, 90969.66758107342, 73911.10286370946, 60353.25316189172, 76783.7822664324, 95466.52960659919, 19315.587379401644, 95170.46861389857, 48291.73293256917, 27578.40520817534, 23983.247093537208, 40715.584661782246, 90605.65154898567, 66046.97851613245, 59810.46187893756, 87439.57667162704, 62413.22850444181, 38669.029527634804, 98052.06494162371, 60362.072530173275, 76175.99948926117, 80656.00680176911, 7402.750381349721, 36537.23670990581, 21855.828337089257, 36955.6261290938, 16966.378133606853, 96570.20718604334, 2526.5080382182446, 17463.916174575865, 54075.82296441399, 83536.78879477637, 5384.092279384223, 91701.83785811791, 41226.83041830366, 53558.26130031923, 60790.775633533645, 62701.72716087091, 79084.08815238335, 72985.00502907294]} +{"id": 760, "vector": [97533.28591533657, 97689.74196222353, 56678.53635854424, 22339.03466475642, 72837.45433130227, 3996.1152333553905, 86361.14095594326, 28328.94261887354, 69473.15077977764, 18142.43587679595, 14917.25494125612, 74919.27499099149, 98379.68969400827, 94951.96183969035, 56792.87519315597, 37190.18991089908, 6450.465160584373, 24880.61462989153, 94015.53279138131, 17477.588556043043, 84004.64098891224, 69497.80980901452, 28094.81836845209, 36160.21349939978, 36063.678677929514, 40842.985338292456, 21714.54800415239, 34885.24348787312, 36446.16371052077, 50667.548905657466, 78951.00494175316, 48245.90989691984, 64567.52711473519, 93432.64388440127, 878.8299229355978, 28102.74426946362, 41807.048870963845, 32767.478424212302, 71260.2524755628, 93647.71693869396, 38354.24575809528, 82284.9416281857, 98731.51006858135, 49057.50967863733, 36238.68998583162, 63428.13819175197, 94594.76868634432, 67736.08017212259, 62232.76919974898, 87267.34782272896, 15190.367103239589, 15547.48053983619, 82558.77140954712, 38355.02863807264, 44220.602133959255, 29575.377569373275, 19185.599048883574, 52349.57836020631, 90020.72761521068, 7539.435177578035, 91159.27572152461, 12687.943371140387, 1271.0511829304162, 4655.223478866877, 81864.04155445095, 47203.335083505226, 32002.049689298794, 9660.739882579704, 42519.77795794527, 37153.86797829314, 67346.40478981809, 19735.555104717172, 75845.50216292785, 86221.86852953667, 18982.83873696314, 80540.5287376303, 89197.13020238244, 26308.813478721637, 82318.98609871272, 61139.4896071355, 41237.18189166842, 43257.06141110225, 4588.992309771756, 98481.68272023596, 67555.01877092446, 85160.75466648591, 39838.99023622844, 15821.127846263716, 15333.559599070744, 96694.33725397267, 53087.77700665897, 51866.31111799145, 6393.993622174443, 10786.26738070103, 35391.96233737751, 89347.74918137491, 97322.96372878204, 99555.59978440162, 35507.108997097515, 4595.853484940882, 84991.56904997802, 10134.130607810132, 14653.192019184069, 22393.4669086137, 89091.00007032823, 74687.08289802838, 10369.167276365255, 34968.91360665708, 75191.53579584403, 62619.15451064233, 52544.645456562175, 29207.05257230414, 986.3874178877907, 73551.73700525309, 16701.552863473422, 88742.7784254377, 32849.697165491765, 15123.393552693731, 12209.329160355053, 45967.08543485063, 69746.51576525447, 91510.8756115592, 96751.93904955193, 94218.92515216142, 50046.725664535865, 71879.09226886912, 50804.61019871566, 1634.8788827225792]} +{"id": 907, "vector": [32831.84805891142, 96636.89221237434, 39079.39951927555, 1367.9150115402329, 6055.256062313974, 38290.3043345458, 83596.35596137012, 55071.096999698886, 97238.07072860502, 68666.3714689061, 19278.185320577544, 97874.36309196662, 8040.078430369768, 99357.49914745598, 22921.975634521485, 54987.201404416155, 89797.59459724382, 93635.171953735, 67650.85817112964, 23670.05517185634, 83657.09782768214, 68378.65108172096, 28151.069150269224, 29342.20166186976, 22459.03806282733, 69257.30984155825, 60614.1541567787, 83246.30775573538, 39999.79300041364, 65697.56962666934, 99705.79477440674, 74163.06692485556, 40870.77735348499, 18858.839418762174, 8627.87769548775, 72850.72369680903, 66006.88718114905, 36414.673087644755, 19713.53461910752, 49594.83152563084, 11089.127218355045, 19591.280994436846, 82144.89954528256, 93119.68959703959, 93918.47207891775, 12933.030170578008, 32466.899052679353, 64479.31886079889, 49755.19410640812, 79936.92491398536, 35870.65083743677, 72421.20964717773, 670.6286944974215, 67286.12686549874, 11688.64937022216, 76627.48094492598, 15596.059767483905, 19331.71087543962, 17100.102704898047, 84052.42321568487, 98524.77046813567, 85899.36337852027, 47572.38281934404, 65148.76850024831, 3882.594487753099, 8531.442371452735, 6298.332310615929, 36167.91425125848, 41819.31507231144, 12814.028632395535, 41848.81320919195, 41411.30991090813, 65964.4518778955, 49935.820963984836, 30497.760541816264, 21003.765478963265, 72838.64761264887, 91433.43388737728, 87395.58652050044, 32511.96448095883, 21924.588390873458, 4249.474712440649, 27737.82046012947, 315.902764750553, 15766.394721991317, 49908.142229583194, 58041.68493279813, 238.05225151932729, 11143.868449877715, 99459.65578409102, 73818.38419557891, 499.25425744972165, 15046.704365659169, 41165.49169494446, 70927.67260060005, 10582.725003094583, 36438.03132731579, 63750.14536853253, 61577.32084177048, 50507.93400265779, 99559.17640998567, 95367.90332474007, 71628.64035148134, 39265.88472339717, 77314.83166488528, 10046.02671244358, 39255.77829360808, 29791.157593676566, 84035.53910881972, 21831.97585423189, 30780.168482151494, 49861.156820640754, 10314.608075702125, 51857.18795829316, 93826.25427558065, 34593.41867795841, 81469.42261725693, 94549.44544162438, 40426.11286692511, 85164.18603689922, 99306.5923254279, 69777.80125360757, 22250.482420624918, 2236.1982063180185, 14675.825873649928, 61462.02169478116, 87465.33901204144, 79205.6217706251]} +{"id": 1261, "vector": [49835.504729338274, 687.2054968888098, 5355.37634465959, 66162.17120798587, 31151.031162980104, 1417.0547864198802, 7014.29322725241, 40491.78287310583, 52199.607358771624, 20059.77357313523, 58178.475903937, 28627.9304476624, 27528.28159395664, 8302.088018175058, 50198.07888022316, 96097.57604140809, 23216.552458188766, 38656.18325859522, 59299.410628892394, 55441.07852641602, 16853.166447517055, 38855.20074797468, 88536.21970758637, 11617.450805606255, 37755.23242618333, 41801.7643210152, 34532.086444386136, 64305.7177386491, 3565.2492342170826, 18753.708759996145, 41464.008134695396, 6542.693224803664, 29087.36099394764, 24920.081044743027, 6305.307266989568, 83753.82153227789, 28388.508748592645, 67545.30893782341, 3770.3402591232616, 17943.151278655056, 17727.8546313259, 71133.71958209682, 50710.33174335063, 88711.6785454553, 72098.66180641654, 13480.972355578846, 20296.941798522537, 36984.059488761035, 88900.31857708446, 118.81932782207284, 49169.56400828316, 84662.47288724637, 20557.667159471748, 79499.42607078872, 65806.93641657212, 6584.089092582257, 8669.983263145043, 76607.20200213947, 21227.356299040057, 87752.9500561298, 39659.35920617369, 91068.46888271632, 93519.24155081974, 1871.973163580498, 17507.354524355855, 13259.246338780562, 79302.76813159634, 56791.75119323394, 23941.296270498446, 41129.10772910244, 68804.10107270215, 70883.20344856012, 97630.12083581608, 2591.1593359263074, 93530.83768632583, 79567.4653268231, 7485.192438224053, 4906.709368828766, 84476.61988194693, 65128.89095107251, 39667.87549734052, 32160.4916934207, 39893.93478097228, 27841.295614642502, 32867.14543525453, 36033.85220129976, 6997.422210900994, 88097.44876148143, 12917.966261385294, 20661.77459318592, 68472.81255600361, 77893.29188983908, 7149.862389441142, 64865.56907731979, 69385.1345381679, 83123.68443064191, 49492.97501314124, 39720.26389486503, 65267.20209951121, 35766.38371189774, 49774.727507341624, 9860.15641721163, 28143.928126707186, 27638.021569916247, 45416.62391825648, 9996.74325019555, 776.0286656024018, 12139.404412442313, 54431.7297363432, 72611.37387823076, 86347.46894788744, 156.93790473475522, 61726.81070669507, 22916.799777274744, 66581.73432952503, 11745.27402319624, 208.8271769235428, 34735.348647687635, 23000.3336645475, 64556.60554194415, 5173.337032674197, 8752.460768020608, 28753.739924529385, 7181.2081276980025, 5744.573645865636, 44499.92971678075, 288.7454558296088, 14664.130613667281]} +{"id": 449, "vector": [8305.609211884246, 54717.318042742794, 45619.75837660015, 9990.22663085567, 70457.00148603332, 78054.56975752677, 17765.19596353089, 68172.72849428015, 88880.13350812961, 51592.07229017564, 36514.48157262346, 95853.70947155151, 87880.81180077005, 57641.666899528675, 41036.427556617404, 63883.69883981043, 59863.191948068205, 11202.820720945294, 70206.32952752634, 54124.44949141646, 31336.262851813102, 41941.43316382504, 98527.89904028464, 92541.92062193509, 75431.31081434734, 19044.745221901383, 50204.4843020801, 45241.46732322609, 21755.338342257357, 32008.064015653392, 30673.60279966247, 75539.8296320953, 7672.235042691422, 64942.97941354489, 49772.427539169104, 89145.91972016077, 89055.31189944093, 39130.15234240822, 17007.115982767453, 79432.81228012938, 59298.113835260134, 40888.08246521695, 48024.94678454827, 39370.48331480322, 13463.346191795556, 43792.64820408783, 17594.781895023236, 4628.396498357745, 67321.49826458731, 2422.510310352943, 31838.867913176062, 84039.29148611194, 11951.544964204119, 91415.84331270216, 43725.38082774264, 35523.25327203208, 38171.60151550578, 75669.38977614304, 63459.694211679605, 55638.02968046684, 86825.7518338873, 22336.243534339304, 27472.276649696923, 11984.975645623708, 12768.637360400791, 75956.23324825041, 7060.005318173668, 37723.805451624336, 35410.36009521792, 79870.81370823343, 52771.78980521796, 51113.10951519381, 27965.28678756891, 23066.69433095817, 4685.849050877944, 14596.539357660266, 31791.696075680764, 62641.10672894453, 4676.089065930289, 10380.928178164806, 22556.005076113404, 1670.859254601742, 78116.29558931796, 83533.81058321169, 98924.07906289355, 23748.799169488866, 26981.67447762001, 12469.36128421543, 51999.98085587777, 99916.34698902648, 12274.738228192151, 17505.907939225828, 22236.678091492333, 29215.39114706847, 99167.11954267764, 86119.43530417503, 59576.92968294235, 37717.40459441328, 17089.62221619311, 53885.78161337204, 21834.42998621945, 85389.17473167957, 95335.96731359982, 82784.05170430116, 2600.406027865798, 37543.25848207836, 30918.906915099153, 11391.646114634092, 99698.98846839397, 47438.32195132497, 92444.14103230972, 15457.048143719654, 47560.80754904204, 69134.58956837644, 93958.75429616607, 95096.23319808897, 90820.49596530775, 38983.23737709717, 4637.442206207665, 40417.27574107321, 52183.05718590828, 59804.629698529236, 62285.58799511122, 21337.236743447218, 60311.24885971345, 16847.544019062054, 1595.650797011383, 13834.515822593097]} +{"id": 144, "vector": [35809.135042283124, 46274.130286084255, 35924.768376693406, 17933.37183065672, 38739.88306997078, 54654.407129886895, 18987.180337263766, 61527.9549436821, 50300.21778417977, 84538.47223333042, 72362.93931804052, 93019.17791708166, 26687.84526207706, 94183.4939769703, 35657.03929888085, 65166.971866577784, 30317.281975054, 72818.48307918395, 25013.826105310232, 93932.68126547075, 25219.169562488176, 33070.922608345, 36971.64935845245, 3608.6955084779215, 97996.8827707817, 82066.57997302122, 60549.257157782624, 28672.627981410413, 74742.41350578192, 71662.39139809115, 81415.8288865073, 47479.23664024379, 9073.301088542119, 83438.74926644158, 49982.26395259965, 5709.776630012553, 13213.230690497669, 41501.099457098346, 44593.62370997691, 8780.480035930905, 89372.40715907273, 362.0019913146644, 54919.70538736067, 19996.763248760384, 51528.83055815347, 91663.27734101859, 20240.596550221835, 65398.214072322255, 98514.17639853856, 89967.9297728347, 96273.41932117946, 39516.70366719967, 25820.558732279762, 82853.38796790289, 8904.437299204126, 30964.98436178976, 89640.52724774968, 43443.32737571553, 77870.8491857638, 59402.88016040035, 5194.1758176361045, 43446.197557192936, 48467.21635752878, 84749.27931513602, 88752.31693155572, 2420.535088697884, 84993.54027783584, 51699.35844128245, 45028.83883769888, 51092.42470863359, 68701.10986414345, 37870.80663399148, 8737.210779717308, 22985.45758337649, 12345.58333901482, 58707.10969594631, 58052.67952423009, 29282.31057000289, 12309.813470518206, 71657.00810983761, 51986.64725334642, 58069.633816172995, 37422.2843749084, 51521.06718201084, 48669.370790279776, 95668.09684886875, 99150.99869275598, 52708.39106788455, 87046.85224203786, 12153.188618337996, 85379.77707977429, 50995.544181479694, 61269.885403054934, 17126.188744610616, 18456.810331866825, 97177.10279981527, 72164.53958553498, 30254.706871189064, 84923.92790840936, 5659.6701068728935, 17487.2585808152, 5503.482498447021, 89032.23825534263, 9876.905129237723, 66151.3742307533, 61297.08072222162, 95744.0889643421, 8696.18090526183, 15104.07721589111, 52844.17260388127, 83096.57029400696, 81134.87558648028, 48626.79242448694, 2339.2351020309898, 5821.074094816836, 5822.185624691157, 94864.94000274081, 70144.34675161062, 47677.81641647032, 37297.0191817774, 62740.645828626606, 91849.3876536309, 29226.509192001693, 63536.01462150803, 94224.49511347106, 28222.37509080223, 6624.2660174508865, 2575.4603514954842]} +{"id": 1205, "vector": [76626.21657042863, 96550.71713463234, 51940.71433187126, 74901.7954566461, 44375.10428913025, 20538.79865090673, 79404.95795420939, 4507.0792906581555, 22480.38989218796, 41592.59883858368, 75785.62968994207, 87197.94023148157, 50283.912448408606, 43731.540695900316, 32031.00406788254, 21652.077478451516, 28936.74829360695, 36140.99306875382, 72065.29480665253, 40522.50440712648, 79452.41249702474, 94998.55298683714, 4650.323011338942, 34355.464121130906, 9531.386178347124, 89505.14992487757, 86585.06605765226, 43313.18663508813, 68293.61163310599, 68113.06964262469, 91340.79559141478, 80.49874418213365, 47906.31986753165, 63492.68992266629, 46192.07186060443, 95256.44291590675, 58552.42104659406, 41632.984335963076, 83426.27467268181, 48039.3022514098, 90103.69687737543, 48082.37622023552, 78588.69102688461, 49735.04619247997, 72982.04293440086, 46049.16444212914, 66963.43998018168, 38391.885004098214, 54086.18389113466, 34794.90249645958, 50310.16452056135, 29453.145008199077, 47730.4299766566, 78759.49100935043, 13391.341990741434, 22446.068325694814, 23538.60783530857, 55107.4351277127, 78973.0365867004, 83903.69571606671, 15472.008846002216, 11884.642077541863, 58255.91559642639, 76920.9577748138, 36269.294895866886, 56986.5172989257, 65682.69499344943, 2398.0792414185826, 70174.00843095628, 63700.59061707627, 70339.85451613116, 56963.03934814632, 46979.0561984727, 33274.45516442118, 93742.51803155168, 57618.04516896699, 1451.4430173777182, 64792.53176730911, 48290.34828249955, 90889.11490292755, 56869.048060541994, 72722.03690108431, 39589.7049188202, 96614.53256602412, 18892.77795123181, 62470.033756314246, 98920.79729930981, 93305.54185605353, 92397.76108567776, 48869.07420888651, 38278.7119574098, 68515.52848749781, 58054.352054369905, 82875.10616897576, 70444.10003092859, 63325.545126232086, 18951.93058403537, 78739.01395802609, 6213.51968634194, 95828.49805673797, 79335.21080357002, 86757.58520335447, 7823.037963905, 31479.29075663638, 4382.756484471107, 22003.80254022136, 95513.50529058323, 22535.465749376937, 93819.21481062165, 13194.4813136337, 70357.65070668694, 20386.633382331755, 42755.64975542582, 38783.48992697295, 27226.845146181044, 94735.88318966223, 33203.28387867679, 3740.1438562103763, 44519.910414040045, 22130.364477136787, 31532.74579376646, 31273.153437360947, 54202.97637849182, 66104.08141254437, 56896.35114466084, 97012.8926201077, 75143.8154724218, 62577.23565548705]} +{"id": 715, "vector": [47508.90058709286, 44613.12143971522, 18918.1177256287, 31719.86212831286, 31246.124549175936, 54922.598458269254, 83168.4102970702, 39767.21334854413, 91651.70831962184, 23090.99288690566, 76406.27776153553, 42041.58844324164, 30870.995717922277, 1599.969585659944, 24265.892355757078, 26947.96782365583, 68499.31721571375, 92146.38590607395, 38130.50262248503, 17829.06990944879, 44906.49812417713, 72246.81056104331, 63501.067086507675, 31390.4894712324, 69840.34038947803, 16190.092539299894, 87090.94257483867, 86027.96516993653, 70539.10322025343, 55877.4147824813, 57027.88655293116, 19006.702028675572, 92126.09085366288, 29523.386491616566, 40320.268703042675, 79738.41509659105, 45118.28485392107, 58208.184355902435, 42696.96646298081, 58821.44790947314, 32171.031149814233, 70437.79791484331, 69283.84328384441, 22454.598668354563, 86955.6882846896, 54695.838452420576, 82164.64287901709, 76071.35849401986, 29827.8633052801, 10899.952519536837, 30373.89922668743, 18869.267103637245, 89639.97449954889, 29620.9107231891, 63614.39829869733, 23250.175702930654, 91714.67024613022, 89365.79066371823, 21414.64801687125, 78189.3234862405, 71462.0851130027, 3201.998613775725, 5851.372753442963, 4512.734746312619, 56302.45065924002, 86402.48871379625, 60016.48793341187, 80383.9568736477, 81539.00823626175, 69718.37205103881, 18291.49090471731, 11441.02607479318, 75379.23243466763, 93623.01434723851, 9671.754805323362, 34317.42291500709, 598.3231216828311, 69943.73549995793, 21097.469014652193, 22644.950919214247, 11084.08107961889, 33966.14557696271, 27642.23480021174, 38161.62477858037, 2701.5840829801573, 27852.728853043183, 84037.94024759886, 6918.407688987105, 20439.6410622229, 39655.0297336486, 80349.93140066204, 35613.557760756376, 93544.5315881534, 70134.8389778199, 59361.51599479983, 97427.04665203483, 48321.633550023325, 55609.567643971626, 71190.20622740699, 14813.786747472346, 22973.24667229025, 10833.116846680025, 94257.25443192867, 78952.23967172139, 45628.27142469949, 18205.883909053977, 64296.01399901544, 57622.395240463644, 51339.170725836106, 90453.37426545049, 40045.08707820822, 58013.17826690699, 80698.79920208917, 29206.76429488067, 18068.92256652043, 56146.76702693325, 19381.27487629622, 15349.506407836345, 60188.3010163442, 42197.37724720436, 85427.96071246202, 35617.17937495839, 52080.89842499105, 60360.29924847295, 38393.80476081902, 23442.823867871575, 73524.42237148229, 20943.904681661395]} +{"id": 843, "vector": [47722.81230370038, 59100.3866235697, 58180.311839778384, 71735.7480660164, 93567.81682865506, 24565.388697290968, 18319.04998727194, 58143.082351872865, 99125.11708681988, 19947.05058847437, 97957.92786957494, 84293.93810172097, 94609.29088257848, 56556.04779024813, 23131.44342701966, 19408.604960860066, 58877.33799180673, 98052.51401806322, 69492.03434420054, 19796.32575091348, 89204.79718275533, 15480.16134081751, 60034.29190536111, 3231.1025892818648, 21894.067363394977, 45423.527827025864, 9258.033360689376, 42164.12908260987, 19409.903316144373, 938.4754817945651, 39271.151319423494, 45526.55345008688, 44520.729747685225, 83722.2252507662, 61463.852517604675, 51793.94286952427, 33203.753016407645, 53937.23830563144, 24460.579107470905, 89122.08552607519, 9685.475704725399, 57915.94615567479, 54582.414307564766, 22738.618443708623, 21977.73957192508, 23015.256095837976, 77365.01676504084, 26763.2919055535, 23678.54133954438, 47487.568274330515, 42208.46014040013, 95849.91960532215, 38531.82129252412, 72897.22564178696, 35341.22427756864, 27833.76962567683, 7553.755098887927, 86590.86996145004, 62398.903779552886, 44611.971076440015, 52020.341279833396, 38474.26371620686, 4667.023421113358, 32283.87134240317, 52659.19452614766, 98808.34568907038, 36099.14375105358, 64036.6581297288, 1466.8936080788342, 55978.17491542974, 88935.22161162371, 24807.82895510103, 72468.21541116487, 71527.58370296162, 92237.79050984816, 85092.34445009367, 77027.59770216275, 57991.025788185216, 92022.73174088931, 89793.26936415145, 26294.269896533817, 52793.028312859744, 26935.144105233754, 88115.95290214822, 28985.650636281214, 56976.8820396376, 10532.819202768473, 87831.10913120037, 94993.70832676133, 62693.49011108703, 56421.4308580444, 52535.99604794802, 26342.950939449194, 15934.719680776632, 41342.925707130664, 17018.080326078765, 82511.35936495758, 75833.55233423247, 75756.87062786297, 19882.61628956317, 40613.76907068193, 54940.913013639605, 62156.61820505921, 47505.08221867917, 82783.55633515885, 37989.51820058789, 71386.93579781172, 29320.939998585825, 58428.332272366875, 87124.79486502858, 41978.761002042185, 41467.85706472152, 90979.03851248213, 12924.805427037667, 71460.92683568945, 15082.781382769117, 43226.263058334756, 20422.387585173517, 26244.054468936527, 68017.30424975809, 13052.472770545242, 74863.60126717892, 27560.260597225526, 97573.39701996934, 50410.28351589447, 42064.13872752871, 90130.92583978588, 17146.850885036103]} +{"id": 454, "vector": [84322.99132183549, 74303.93429858616, 26282.949081289044, 106.61831536759037, 43116.816080186996, 43743.2046980763, 86268.72785281355, 78838.59367887753, 47664.9581889589, 56214.94762905464, 21006.82882053084, 98365.18160442807, 6470.95861129473, 27862.491171050162, 33310.685041532415, 83661.80429097211, 55018.4642486713, 57923.55635089886, 26922.459378960295, 78117.5166469012, 28346.58551843048, 36418.81671315901, 64079.459071757025, 15367.493788159958, 36099.81450561195, 20514.03390834954, 43007.06280767701, 92480.03583606001, 52239.89196894401, 95723.51987111325, 65567.98449043313, 97557.47720870601, 86795.41517742761, 40009.28663150619, 35955.98394095658, 1955.0774334919586, 50578.28081882668, 29188.36888374039, 6281.507736634606, 2170.2509736289066, 41115.123446511185, 21417.606862025874, 99417.73694998217, 75710.00143451367, 848.2024480089057, 56280.55217190267, 16325.562112440772, 72093.03756261489, 1879.1374688471474, 53795.510454213945, 59838.21142205028, 43785.43989360707, 42878.45735496187, 59898.77921788974, 60648.21635981228, 33557.31857548372, 49423.86927991361, 11327.387256127553, 62437.492937988114, 4548.398599937032, 26600.286299053234, 95816.18076593222, 43120.503129237964, 31536.39123009696, 55270.058869449946, 2527.4196852049126, 94083.77133098271, 38698.7840254956, 11526.432599002945, 38365.91792047932, 95666.92776624745, 66040.03788911931, 30868.205365739508, 42278.784063335996, 25910.48587795478, 99478.6351483482, 92959.40818538223, 78213.24293928976, 72018.69946865288, 56760.75824261297, 44225.201323840316, 19324.282914629108, 14702.023647602358, 94951.37671084862, 69675.74396870482, 5347.708691455388, 200.6007066863824, 86889.39480393029, 75709.55106107079, 28757.71794904949, 22812.314128874943, 25812.650364383004, 66298.23514216552, 33456.21244385424, 53169.43790083694, 92043.19684384251, 27051.143516979027, 58250.2030241474, 7971.321338614778, 40436.21106294505, 77918.3279442999, 9424.176906307392, 33241.98071999359, 52817.787854149065, 83770.35288842111, 26509.6653093176, 73724.15173074529, 68576.07356543283, 61274.72146062516, 35793.91881158216, 61489.252280588014, 20988.066952068573, 4891.878543592354, 84132.59743098209, 17100.072146483548, 45569.68407707732, 94216.0791784218, 43334.04454544838, 26533.681166681465, 21625.917436547337, 72302.57724216119, 59025.94378099021, 18162.29523787276, 34979.24150922171, 83229.37778066826, 3724.37350993714, 44607.1847498707, 46138.52426903995]} +{"id": 857, "vector": [67432.77697444157, 78835.03051968408, 74870.83524766479, 83770.04177258837, 47030.050730810646, 801.4632201118976, 34393.910633581945, 54074.57731243237, 99405.45401157091, 61516.412695442654, 90050.75111847666, 87045.432527033, 269.09050153749445, 32046.811112980457, 47680.08418984063, 21724.365610266726, 25878.525561193677, 2236.6364276514505, 48284.055359068734, 16569.11530537629, 40120.98136030122, 1910.4025095018962, 27846.163350147766, 20455.014200233712, 75197.13255408478, 63684.87136862933, 10849.375781777515, 21876.300910490852, 85738.72482959276, 65166.26389785075, 57137.29905580348, 65985.87834361896, 83458.87936882209, 4803.0246988763565, 1101.5765310463444, 55.59854366697925, 17471.435065512353, 10607.267974065837, 62995.5434967428, 73699.96537885127, 88740.33385123734, 3416.809834536694, 65312.48684636789, 26059.29686951809, 52993.204505258116, 21970.76120930609, 67514.46213197499, 24263.47932948846, 29251.903492034926, 67558.05660612063, 12317.15483887632, 76528.30622256565, 73917.91463313495, 2300.366814457766, 54923.21030116988, 8498.565268984059, 48125.24351566375, 82630.58769260981, 34681.04553287964, 10633.751066025465, 39723.95266948006, 90169.95446040295, 7632.5365495081505, 12871.851301545812, 98113.5513934215, 68093.20239434548, 1223.8274323252197, 8421.906771556798, 40215.50076735233, 29536.064837796162, 70718.9519487419, 94560.68750399756, 70144.87664916067, 20668.80225846741, 78357.77309530028, 76826.69895729952, 42010.822836792635, 88123.80282338618, 98923.50201956289, 66738.44871155825, 51881.289393178675, 57491.51103736759, 4252.022866298832, 62200.349339085056, 76121.86754151751, 67798.18077624495, 62918.7057354018, 54915.627969794565, 83955.8026528365, 31386.09073194014, 81265.70102765012, 21519.70736870219, 7653.955910132382, 87289.1481359519, 1880.6203285517076, 98734.2413669986, 67737.28600209924, 43711.05718923811, 70649.82680729742, 80556.53242576937, 45927.44531583612, 31586.473774569335, 98029.45331978933, 2997.1768260518947, 73779.53648743652, 97783.96557698786, 46333.31155019154, 12330.353145032224, 27419.84754571093, 62339.27421167459, 44896.09601072988, 84962.2317336148, 88343.561164329, 95235.30099618403, 35988.75670630709, 23639.552303639288, 83911.9630200726, 70038.0393126469, 71805.14347106461, 50093.68125482343, 99517.86043146517, 47785.97479446874, 27538.55885648443, 85996.23663723798, 46424.74091934034, 9452.719875707671, 9878.0670454943, 35848.563662606495]} +{"id": 1712, "vector": [25989.117981048348, 32036.555506894292, 12732.011478715798, 29335.75668324192, 42098.13899738031, 21677.334014469307, 4813.389283047753, 25095.524692273975, 1308.7779966332703, 10156.502157533543, 86332.09752373365, 61127.303875829195, 41061.851861944255, 57623.24947205273, 9816.535164120654, 97365.54582421642, 21165.17256135654, 84423.43529679884, 2548.2904515512605, 13881.187079464053, 14205.928097861643, 44090.49723163204, 26974.747935307176, 64610.26523675919, 46283.303053999545, 76298.55734963503, 76004.02669620002, 52121.681946770834, 85514.52580654685, 60113.198609856736, 53603.159158327195, 79748.2250244708, 36563.413247444856, 47955.6511902259, 74414.5641639046, 90217.9516707196, 58148.33362920117, 72199.8515325684, 71165.88424073772, 85528.64461900346, 40470.84524772044, 58959.16724346747, 90554.27656123819, 48729.18613314228, 81788.43851891955, 83589.06470578404, 73987.17152130655, 78743.62324385685, 18061.611953979296, 99473.36279437834, 17131.263024086697, 67623.17595003426, 98071.39196468156, 14233.605467871725, 17549.340222621955, 90399.52035306243, 94652.11316761986, 20839.568540124375, 18843.562915164424, 51199.3784808946, 24572.205641447264, 5546.811683940234, 89031.79355857719, 38200.31638767376, 59834.99667375782, 37573.696659611975, 80318.6581955855, 32547.420643572754, 8770.732016464899, 33493.266430902266, 18828.95786683918, 30896.948154116442, 36951.14881720097, 55365.40765085231, 61209.739579315334, 10848.115204584441, 89674.69492688045, 40122.54198348192, 76128.18375136366, 39387.80554557755, 37125.31460461934, 26258.759511735043, 8795.836070334428, 65975.64032640545, 55609.73786863554, 58882.48717309301, 64668.62137648356, 8136.38611240367, 47925.77883413977, 60306.238818839454, 85153.98301062606, 88067.20717288824, 92032.87793421706, 43021.62342385989, 16639.35214807524, 92957.23181215965, 20130.316785267954, 69673.87903011461, 82815.00294089061, 98257.42713733742, 88818.91295511815, 6349.251796350896, 15354.106154851943, 19860.0852044292, 25690.361655166438, 5480.175058804371, 76612.81958995806, 88442.47885762055, 12585.16340495347, 51145.372672986814, 2480.791538688276, 23037.142392290556, 42303.19775608929, 18255.25105248703, 3533.5033728968624, 29562.5441501721, 66283.65587483035, 37142.13903486033, 25495.67548503162, 75023.87795562425, 69855.12572316281, 97352.77899214442, 53560.79642798708, 21965.757062410783, 14365.188420108854, 41822.59820363907, 85521.10132247317, 17650.548361935358]} +{"id": 184, "vector": [57083.995905352895, 41847.9615032909, 51776.14528919662, 5833.563250470397, 88932.3711759158, 89258.36424182385, 37043.197091759495, 58678.962592673844, 54774.384460201065, 27932.516094449944, 2223.4575462226626, 60264.565683538676, 32001.89393152525, 82861.07714000135, 45673.25461339401, 81471.03869175912, 43588.02820304999, 98741.58741253344, 9405.521707404674, 74649.91502433739, 18056.03304825125, 8853.305659590505, 1532.311748722115, 18096.959017307367, 79460.03123174676, 60160.2130513148, 55230.78572299646, 60921.825393411746, 20075.86830912853, 3241.0045513961095, 20426.009324703897, 67259.86297052607, 59766.46762191527, 7873.145652030955, 51529.48172536754, 88643.52621720919, 6090.829261849295, 82653.68198311374, 48831.77004965774, 40865.16744011128, 84217.09020842313, 61294.658779460646, 26198.842112766764, 61228.167605970455, 31184.850827046517, 80215.67337552339, 81348.66416272389, 3632.3351686803453, 70537.17864532936, 96498.17107980466, 80704.3889332872, 23206.82854060989, 20588.228267251783, 357.3737216665318, 20180.92744821515, 30581.257374155222, 74468.69519100568, 94129.00517661544, 76855.5679108876, 89655.7002567325, 8997.634499181706, 23878.771814295542, 28350.94381881147, 32503.665396080716, 5661.667191217579, 55980.95691181865, 72550.62255779085, 20679.141764345954, 89744.47116468137, 68184.71070570635, 88589.6396429494, 93357.72042704665, 16779.470828335852, 82491.70218501611, 57910.80375073558, 32006.517122586054, 16945.52403165419, 91117.63922035557, 61059.32547131983, 70471.45764839728, 12521.431508890135, 28336.77332612069, 95077.62191178802, 26399.256535479064, 68801.18117518339, 51391.43009091999, 77876.85340981485, 16742.828771189434, 66226.48719392708, 94797.01995390766, 75317.99277770828, 57109.66883247641, 89926.6648957027, 45575.823265703984, 44663.11855553436, 47374.11412165985, 5124.068464834297, 83610.78043018443, 15641.97932092246, 96187.52896084334, 21563.701710311045, 65038.162252615315, 41762.013052303584, 69773.21616257106, 13915.855167819236, 30983.681968228215, 84223.33282177603, 40851.61022871287, 68090.94105503234, 24841.71231821636, 63907.03547729523, 426.7013312569956, 39163.73307632655, 60368.674748512494, 15136.94241435254, 2632.12074148329, 42685.83750095545, 79507.19439926653, 93685.01316380003, 20186.786500861843, 82933.22053266437, 57007.07460373436, 63096.358578578715, 79687.15965488326, 18132.503032641267, 47636.97819423526, 21088.642066914254, 25179.375412997317]} +{"id": 181, "vector": [64465.62635440076, 95652.92347485265, 17372.70345280202, 98991.89318790648, 87282.71253451263, 13756.926715572481, 82446.8474600116, 9100.582556365343, 72132.92930224525, 96907.81770574216, 8931.802577445358, 65965.01792489893, 7436.914375722514, 16062.848179495226, 39829.9923720961, 47809.16938430643, 35468.3807909033, 2079.363964870573, 97118.47883371828, 35723.64935676421, 75088.66872070447, 45454.37351998184, 20010.584728441383, 17129.839514629675, 51059.32961394221, 71378.75796088802, 23695.988822785275, 34776.03757100275, 99554.53928129196, 56214.93223772489, 17879.266362758484, 85092.85616211101, 37672.26929241151, 91866.85616272742, 1742.8379032057073, 95420.67808376474, 70613.91967758026, 52305.816721418174, 94067.6615523124, 65941.2938650058, 58779.979431733984, 25519.977587398913, 37688.351023881136, 85922.60895357752, 95055.92497012665, 55640.83298257876, 19382.207311432576, 48885.240709565216, 56903.18171465546, 67360.51239756015, 88293.13991248899, 69851.72918676275, 83004.68845847709, 88912.56046924688, 86897.36042900199, 31212.43591299474, 54577.966010748925, 20936.177560235104, 90183.21085023877, 5164.108820129454, 30346.35548146738, 86396.15156601601, 30603.40526973737, 72246.82025351844, 5282.583035637988, 74898.50922558081, 4550.880265353585, 38201.1668118295, 35897.89003189711, 96401.29081952284, 14260.682046072981, 33142.790133507435, 59977.765021555526, 59423.6186733348, 54931.80109592166, 54633.75085335552, 8890.213286788407, 47029.950333108914, 86143.58179352328, 23587.645641149335, 5073.993762152251, 18777.174603592273, 78227.97510443979, 75069.04617778993, 91792.20451616183, 24960.474956046484, 78648.10283405584, 42890.2557178348, 86052.90847076249, 17193.607106963795, 10243.973169366971, 26160.230238022752, 9800.933174864467, 98008.74972034886, 28480.877949322003, 48063.311413892174, 19899.517224876407, 64073.1594271878, 46681.690542018514, 16077.908448377542, 24344.908834919544, 62958.564620990495, 41839.095172052956, 87883.66964647704, 32304.042774198195, 95221.9962020438, 59550.293222303364, 89026.33758908283, 29647.849349242162, 70218.83980188906, 62102.73057766714, 55059.139917152075, 42412.75315907677, 62898.52355285826, 83532.95711260923, 7951.89265062487, 57248.8875577095, 65652.83610287293, 67899.31980172778, 50181.06853230917, 87701.34649530693, 20051.00579447936, 10905.288104741529, 98061.41781396928, 84072.48749536877, 1152.6138669965435, 61947.68287062303, 11605.491048450622]} +{"id": 46, "vector": [89244.10394730131, 30024.737964729666, 42877.97217367206, 10501.02539160036, 58701.075712084916, 6488.621026325059, 62915.742507022456, 84904.54647192461, 73776.21611011916, 49848.44173236879, 46041.11341119964, 27684.024679982434, 17230.835109310683, 98533.97350264913, 88805.68872847276, 40047.6288808552, 12210.891421862623, 9887.776833372585, 86735.2813053166, 284.1075711144558, 82625.1977520983, 97007.42669824757, 11693.610932354837, 46801.68229456715, 20944.549910207475, 569.3360224224953, 87983.44140941258, 18068.03930610674, 8147.475155990347, 1898.490376787554, 36089.76600685633, 63384.08554407738, 99220.4532287103, 2690.4374435985146, 20919.526231525164, 40363.038588712305, 75237.20153568759, 24604.364769411924, 56929.7450527898, 97454.428230553, 91183.35479827315, 8652.05721442327, 5454.23872344134, 96638.57649621503, 56330.56824364872, 34384.221738942375, 90110.8525541454, 54936.151865120984, 92836.92577854634, 3448.9088832277635, 25368.420272105042, 28543.281681472832, 80310.81841906364, 78234.23109971349, 62431.0629368609, 49356.9194501809, 32401.25628320992, 99872.52297117715, 92494.03752100423, 23437.43323795563, 52261.5360696427, 41102.35157991352, 98352.59212918827, 50099.246413502704, 49623.28885246977, 13185.825680450447, 5353.663764020311, 83813.56855085258, 67022.1133803507, 45203.76180632205, 41511.90236819648, 70354.14844994846, 21615.02899077923, 41432.76829615321, 23540.22009482917, 51758.75257333548, 22996.291501057243, 13417.312094876388, 79701.15873847871, 93252.6364225494, 21905.359040018648, 3009.6741300312924, 16937.53383935894, 59033.26290596902, 44117.13236002887, 32088.750585633665, 70147.55257007752, 68972.49076470772, 29012.165079618644, 20250.660398562493, 69625.44390625553, 63782.526476289626, 74320.39779821054, 90539.40832385307, 51846.12810788214, 42285.38863385388, 74927.50795898621, 64573.2458115115, 82679.15148673169, 29493.47603550492, 69400.04123911283, 65493.30834839567, 12653.134205184435, 41656.356705046106, 65228.358576093226, 52933.1064464026, 54775.138422958036, 22706.095782363645, 23071.442702838995, 17273.20757937335, 3725.647911659746, 10570.014332901766, 56695.967013924266, 39065.811846020784, 60208.46106107342, 96652.29173306868, 88769.15889237197, 48971.5239506502, 69296.25861327662, 95473.54643245935, 88711.56278683389, 35136.48443404689, 10736.716129565782, 20199.31358986251, 1491.4898880678318, 71635.42561616172, 73673.90082002731, 69301.16374626264]} +{"id": 897, "vector": [55138.32600267063, 91298.54039300932, 88700.24296627687, 37903.62271413579, 27923.903361783065, 45194.92970977475, 50568.33177840877, 24556.580357623392, 93021.8003262423, 15938.209532569957, 3749.4795035351913, 85126.36988280044, 34404.86488568005, 29061.312578905407, 46809.303017249156, 94628.83446133391, 10977.635037366817, 18228.419545403794, 97861.88100700441, 24188.228277688562, 46098.624145046204, 95018.29008498746, 24049.28062175745, 10544.899851536582, 31425.1424877079, 37358.0909563241, 7063.548530519448, 79810.44544462829, 92227.1607723229, 3202.915037832177, 36138.93967151266, 67008.672228034, 23530.255819430447, 27593.804879707906, 38353.346559629885, 75107.81383664164, 61846.972597672764, 17340.307208996164, 19394.99722221031, 91781.50039941302, 79028.54105666297, 11509.325985631114, 11037.856338527607, 96297.49001481394, 34178.65323950309, 35564.24615463405, 53731.39674923328, 32891.04599832805, 58997.79784374477, 29309.835244582893, 78529.52335333214, 82302.19132159566, 76410.77819732454, 60489.482912905965, 89196.78834400137, 92650.90188881796, 66387.22439472307, 6371.785686508302, 94947.78606631575, 63241.00674309532, 73176.9844361008, 14088.548428415927, 40758.749099376146, 76570.00978165441, 29450.35427599749, 64828.0152420653, 25728.148726198775, 67112.27134764522, 86384.80437553379, 96821.48929287172, 50569.25950227744, 8353.31997327039, 51679.70812213162, 75448.72713659173, 29538.0618265631, 49336.42961314462, 4439.414871028346, 94298.0292562792, 10068.08471651144, 44098.74552899239, 29996.066038530178, 88422.22076568787, 39414.233300636544, 35502.38197898472, 3302.6969290847032, 22013.06268041665, 33295.227626542466, 58233.45508185378, 16953.20045259777, 91124.87821076016, 73344.07228080943, 4089.7329701748085, 36295.44713660224, 72347.68547868944, 77381.7361327351, 9195.97453857951, 17184.43020332735, 2898.1903038567957, 12117.799396782902, 80239.75654777452, 31924.463173184136, 1524.771387859858, 71187.76216177804, 42823.78688111165, 78598.5178719053, 56970.758548130005, 75444.46652086229, 84495.64115595077, 35684.62846003751, 97185.90007739817, 84870.22677612382, 70.42176914009434, 36794.699533134786, 5017.921927767011, 93057.9940571605, 20188.93792708567, 80295.85365905643, 98145.13374147993, 76496.46391936873, 17419.367673927165, 41153.25052798239, 56777.72315691082, 19539.17794241239, 43433.28401342236, 2976.9667182903304, 17254.494977393286, 22944.31431802858, 52980.844971475184]} +{"id": 1206, "vector": [25284.975379896758, 16703.943300628453, 62625.456638653865, 41098.05639692059, 22279.299348655037, 16157.296547875965, 22928.10217133201, 55109.77357876161, 45949.815808283456, 96170.11305825463, 16980.818372070527, 33690.49952801414, 21636.642621067593, 70805.09944971358, 61110.613417348526, 65929.89343950717, 86648.88333213005, 36965.44155079865, 48073.62722837926, 72710.57396662873, 36524.79296137385, 81416.40673176553, 72451.46105431157, 36775.038094742275, 25273.33498592188, 9489.903875372329, 95842.63861108848, 8618.63934064082, 79951.2385665022, 86117.0048664779, 63546.557574962724, 13316.485282241785, 63803.52275338491, 3357.9071652392377, 57788.3354998597, 39405.090122142574, 62221.203925027505, 13765.767585467736, 95127.66673908522, 23461.039428331977, 56244.02699513288, 33518.06414864271, 48066.24764605547, 91335.52732291791, 10622.307059811197, 60495.534140697804, 9058.454986385612, 30858.636825662357, 78034.6892426487, 74472.9769450905, 30107.034226987307, 21635.819439536397, 8923.289254709809, 27467.10163250621, 28453.942060040692, 55332.10482110635, 96798.36099929869, 16330.753304031165, 99525.25972466478, 153.5650055000004, 59133.80005142309, 91866.95143382321, 52146.31794453731, 68034.50377500507, 49749.57972235985, 32513.59467918782, 69988.50923168361, 42225.72316047895, 69301.49369240606, 35473.23527838786, 18086.030797912455, 31771.161208358946, 63628.60988405474, 14799.432075892539, 32003.42844247377, 12346.31441497116, 97620.47794317825, 20113.35910469796, 57540.2435899354, 95113.08226383873, 58768.56884816518, 63219.020553523216, 39327.421999824364, 60228.398207967795, 94200.65267913514, 25158.36833729278, 17486.74167855675, 10841.32608152767, 43854.449063006905, 64459.675608941456, 14674.763833467397, 77841.85554225247, 76812.07582264328, 97735.86598106346, 91629.81090309698, 65287.99286060506, 3961.92037181583, 52022.55765278923, 47399.956553844626, 89018.33381607583, 53415.94165652802, 97579.13738357868, 50169.53604138633, 67275.75668719164, 92127.40011314575, 88945.84341556164, 98159.88685897748, 24023.34884611973, 4527.2222343114345, 21648.446695190316, 45336.64216346733, 81197.4843370104, 81597.09147132565, 43377.90915430924, 8813.481960657122, 44904.9139121001, 12751.590041622074, 7887.669808111719, 15873.433405425763, 75120.77945975735, 28664.762948426436, 24492.108580884254, 58897.67217967061, 62726.5960140305, 45586.51792411892, 70474.72393273705, 13183.766982882727, 18413.836624141055]} +{"id": 1953, "vector": [69230.12916778636, 44362.32571474029, 85643.6127773149, 4560.896847391916, 33178.639740637096, 13635.83805307803, 23258.597319678032, 42328.80906674091, 85315.27503788307, 56897.88329409459, 29620.701345718437, 6059.588654736325, 67959.92945539302, 79034.78585765151, 80293.05312494517, 66625.69442801118, 32453.992993008218, 26550.01093966164, 2262.469121992583, 57201.72716866443, 52569.84723465451, 52122.26170711758, 16646.871564364596, 38842.78914965844, 75635.8639969647, 68365.71056512515, 5670.613027315241, 38239.184042065324, 66484.33806182696, 19173.24887451588, 14844.542014138762, 76539.2845420122, 32178.57443283385, 58945.233839987355, 38865.604878266815, 71458.87516630329, 83737.845274292, 52656.92551858576, 75179.68595100801, 65908.17312385947, 76288.17752561861, 93567.35132566546, 27962.555696052183, 62193.57607246434, 95993.61387638604, 98083.86034662554, 56431.23664452724, 64613.04255473963, 92700.5932549833, 88975.8194573601, 59025.94064575831, 89715.50087416219, 33513.248678761534, 52911.98079885545, 33995.39935735764, 37226.59765924199, 26578.288221393952, 10704.148244206346, 10898.162013465939, 31697.398123653165, 75300.05687770853, 63468.09826530535, 42402.94563285787, 79828.9321416944, 8958.586592443684, 73719.24312731044, 15140.072164842555, 85596.37965227406, 25657.220282529324, 23408.899261832838, 57341.45269256351, 846.175215407552, 56166.26878965889, 40553.15263406678, 28062.65759374673, 9789.53640233049, 97598.46190243545, 27926.52665070693, 24606.232208926314, 70916.68807586246, 62162.464572888675, 62654.208765729694, 33961.454210230855, 46044.405162126786, 89695.29262324159, 8239.830678431637, 24921.261239629766, 60389.73729343238, 18776.71711617629, 17591.73089741002, 34984.709378420055, 56372.21868543695, 71295.63151506273, 80539.01716508972, 66444.98018559339, 34856.29989545926, 25326.093200585354, 30580.312092344586, 66626.34865996578, 97586.07239646395, 3824.922192229374, 80321.07349546683, 89255.15393648214, 15844.157531846547, 30070.24335638685, 8720.94766476057, 54422.49905516821, 24331.94931741368, 23336.671939169497, 69767.88787713181, 98442.71498629362, 2038.7097124879294, 74328.42020184276, 94273.74593695403, 8893.584502350304, 90236.83610804506, 38828.08442275296, 2446.007042981191, 56972.21741039643, 30356.186161249167, 12276.864226611073, 8759.709102955727, 2409.043074103001, 32691.99380170411, 28157.27962206378, 20323.056356491674, 32931.21843707245, 78518.90840555451]} +{"id": 887, "vector": [59707.3724819369, 9106.464228421162, 61839.73982155857, 6919.44113515075, 1900.6145863392665, 24412.95863885621, 97120.22417240703, 99286.27902512293, 6887.813739162574, 94804.44166243705, 20108.439235570397, 7653.641268117773, 87251.03479819196, 9348.223568311387, 98420.18360228387, 568.8635509412566, 70923.15486155862, 66855.61111914119, 83102.05337757066, 18223.73248602811, 54297.31146823314, 51106.45075198791, 30773.651466699357, 49220.135149905516, 3956.8235248470305, 64728.762854837805, 8237.931288162992, 35321.972064430185, 12936.623931855773, 84299.55056933599, 58192.14459839318, 12659.999936131384, 54718.27241297588, 94784.08431468421, 85395.13268613616, 97391.75757371596, 39115.451361398525, 81800.76983483137, 15588.541577862146, 52638.49728888248, 90209.82746278295, 66450.08528141682, 40033.88094259885, 22555.477518913824, 62153.159133531066, 98139.88044264859, 31433.317202574708, 20494.931917373237, 98088.60091450156, 77462.46898346282, 73675.47272720645, 73927.49414009592, 27316.86716206675, 78730.14306654107, 43571.04568649604, 59338.56852873034, 71904.42941525596, 13618.585695714813, 85758.45559517673, 66538.24835297602, 13913.100149218626, 11921.845269245012, 45887.78572913811, 76972.75215296235, 18085.319197051642, 20716.464459477636, 78502.1481064933, 67372.77788428945, 76443.99739005414, 6448.986009648216, 21317.4079607959, 46448.43963395592, 20869.1609752791, 93729.83119495242, 58029.74099889301, 70286.71758947552, 45615.23118505322, 94428.6363689829, 71885.97826323666, 22615.365108619513, 75143.59030361383, 671.9374038924086, 35896.03261940427, 88020.35622241396, 46312.781348331366, 25782.737844579064, 39659.451802242285, 33307.69466053589, 56401.10723657564, 23362.656656646617, 88278.10413496538, 3422.0140148042533, 59166.67572952051, 57391.49339769569, 16956.498674090748, 15711.413214554072, 21323.76299578801, 21100.03637517882, 19175.306939698632, 35369.08922090586, 88888.54383796675, 74143.97077451953, 45795.03314709743, 82654.81835837115, 4107.377362775877, 2187.2932538596456, 83879.86853092545, 55431.75265971876, 58412.25022151156, 94309.3523980173, 50376.42485957965, 6076.837776178734, 77798.283102094, 8212.076582486017, 41570.33994549589, 40038.84939641305, 95061.35784415025, 79161.86397867922, 62749.38538469871, 11114.798671351457, 44574.01023179827, 27814.492385236223, 21885.401013754734, 23930.925460494913, 14567.628234927954, 49704.18384979305, 56520.001659104004, 94065.85916814038]} +{"id": 1190, "vector": [75534.47738487177, 34155.15861852408, 367.30520150547454, 2443.4687109590845, 5966.577818141861, 38848.266269855514, 8036.16253341286, 89898.12124135054, 1759.725808447965, 8161.694031090594, 20453.855079709072, 12182.810794610854, 73495.9924368787, 36889.54696167905, 85610.12961092846, 41538.52784625687, 30310.930425127714, 5158.323421936461, 66902.7323401817, 53647.652585894735, 68782.46141752631, 12826.56704859505, 15363.205194236461, 78279.46872411897, 14825.145538967432, 21009.351039748602, 89077.9840785379, 96997.07173121486, 77827.73945262507, 66875.47291334922, 78945.14944722671, 8981.915762372295, 54944.074046606904, 58277.28269611441, 46915.54244932249, 30398.61211338787, 33225.212358471865, 71536.55441656985, 91788.86889561996, 91485.00211064226, 14703.512113876215, 29498.474006266817, 65324.54748589334, 19219.295816124195, 94109.65125752277, 99842.25639171479, 64052.74178529473, 24532.415436002055, 83746.4594406891, 53834.421214368114, 38105.77874861378, 66154.70944088296, 87309.6109840594, 81356.11484471135, 85387.05607732416, 78939.94027519095, 6262.317630097802, 55511.96752909304, 34099.57996309201, 21111.126669174762, 46348.09138852939, 71567.94780973153, 75539.16964172247, 14696.922770015697, 97365.34798107031, 44132.73186894894, 24785.90199415701, 36863.01432392579, 41425.52887192235, 29939.160136613486, 91557.67594170201, 76159.32059224673, 40426.63721218662, 50785.669660078194, 16892.65361626081, 75477.3032324645, 85588.09616161385, 31509.134852363153, 16727.70565599223, 81825.3177807502, 60688.99621002357, 11010.180627908605, 25731.85798926171, 56366.83827944655, 34468.32503559134, 31775.977757753004, 97330.85123417576, 61767.93816463885, 80457.1294971464, 31329.23645072553, 79694.72457084683, 85091.61571820923, 77327.22131297542, 36306.646063519256, 26554.142141489367, 210.45646125980034, 60614.405722618336, 25937.73262987722, 39911.34282127834, 1366.1476703611063, 23670.422759002107, 48862.893886131445, 29086.00740006404, 51873.289166453455, 31150.860369893442, 52478.3904010437, 61658.036618648504, 2693.0100108053857, 748.5227124262361, 23081.521852433863, 95908.40583976795, 96307.92373970368, 93653.25797191919, 7699.940885648227, 25693.325399682988, 81519.21103526928, 41876.56288391745, 22826.659178054586, 95669.89338103855, 72634.46084114144, 17428.58678892699, 24958.83968919964, 27587.496107005903, 56773.69425216491, 92676.6910955944, 84892.70880584542, 36912.895645225806, 32461.531043937484]} +{"id": 1487, "vector": [77235.81592214612, 32723.770591525492, 47425.68559049756, 58082.8060789902, 64009.74164010305, 74955.25254006594, 33009.65727564977, 69290.01665683347, 882.6980197218614, 86749.6064666994, 21306.93091774468, 38945.58280033078, 78510.93456418309, 29931.734994931612, 97432.18744695082, 69381.88512979785, 66293.84059091764, 22603.106232587645, 59446.74005942775, 80991.39020528985, 88472.51668434254, 89881.52257818064, 80117.74894573084, 51633.5459299211, 85212.26926305977, 30423.139280297783, 16416.593527955814, 1720.7686867301786, 84878.03713160264, 57645.78426789507, 83161.20053942077, 46469.31266040804, 14300.23329044755, 39452.15771353774, 35295.58874762281, 25534.948707122963, 54516.74158801138, 28711.373501403283, 7016.6422671054215, 52749.2032869683, 78555.59489619354, 13238.361768888319, 33816.91432657514, 29172.294070233762, 20585.966172031378, 85262.24698870936, 14130.18933910085, 43204.27423905473, 75939.85940512345, 85113.4658407709, 14798.123980318145, 96519.91677768038, 20415.7379365459, 20016.968194881556, 4062.181900497597, 97750.2288545697, 33312.196119913984, 71351.74833033548, 79211.00816828327, 18652.331554525725, 56403.80286660204, 37776.66378988701, 50206.57082369478, 47723.944201119564, 35402.53076987614, 39551.53959498238, 8226.490717335833, 41366.14926923612, 34433.818312238654, 46727.20278801171, 29309.631114491574, 9687.451328010522, 65529.34799196005, 78934.79114790032, 63766.06257086051, 63622.34770023258, 91153.28017425528, 38427.478313601205, 72572.69211301024, 5868.542962367574, 40333.50667707657, 89393.03914935187, 75341.56843235524, 64565.6497354577, 24391.223482906964, 12039.895830093139, 23676.971049826367, 90977.6732006842, 11873.217662370573, 37384.55845175372, 90289.10049564956, 70294.78514987946, 30949.644262340826, 65126.57056568668, 16898.38977868423, 23478.278318722045, 14755.25872452632, 88631.73150753522, 15619.091293941168, 68409.96086733388, 98734.01237873331, 2328.266820343394, 57555.15921570235, 15404.518869277761, 92494.77374157021, 80810.36796854757, 47825.00862812087, 506.4647573550629, 90221.43778557204, 99390.29596432955, 70332.65938047762, 88751.49446802038, 55314.66886677589, 22651.34964444977, 70404.55716126125, 46672.023021459216, 45536.035969137, 16544.279280802242, 58168.78449235948, 37517.723727296274, 26673.544358079358, 71356.86924916395, 50035.681128574506, 54820.32858652381, 81865.9586860544, 10887.167291395983, 58671.55131902764, 61821.17496545955]} +{"id": 1767, "vector": [834.7673106421261, 8033.9524934752935, 89069.66938248268, 26591.858968404493, 94959.0375033018, 5001.337622641889, 4279.308594444775, 96843.75135469795, 64617.372246668936, 96762.50949332847, 58905.97386536348, 37134.095049347896, 91511.22077176133, 90764.0529423591, 15402.303448045417, 4787.983714555755, 45480.1269480647, 20093.786267101066, 31798.517583979035, 2893.7767847735386, 80141.11732537202, 94523.79524868468, 15263.02520227647, 84474.15682636769, 28205.74120124951, 63568.46933804622, 58923.9443048937, 42719.01097496925, 74699.41936410908, 43148.382303511666, 21337.917631451175, 9339.227403824169, 24831.034612376647, 56273.20782103158, 61226.485368018846, 57652.27462757908, 58543.8622886687, 71398.45690233362, 70857.98237381235, 61707.24034495316, 3252.148297903135, 36220.85631655102, 34835.46509120787, 20100.39052111876, 78189.10706265828, 651.9068529062366, 95921.48321015059, 86542.00664567736, 58464.0010878139, 4200.765897345038, 68302.32734903072, 68606.79664085389, 61980.73898209426, 45919.17405392753, 87906.50319777642, 85807.26077744458, 32531.535124297352, 72755.9347022226, 70393.66775365722, 14592.559571959684, 86367.58388412115, 79552.49618736182, 96416.37865485171, 61975.95602917434, 7705.519906457248, 59812.199372565046, 80901.90408807828, 44405.2536398407, 40756.58426892429, 13168.485021359433, 92435.15875152861, 94216.62296287953, 58152.262788033935, 96422.79363991771, 85390.3098781543, 58859.81775905415, 73714.27273026988, 27729.629098592424, 16757.532694118894, 57092.547588284695, 44873.9423739249, 16866.96668252645, 50282.67391538445, 71381.32192905784, 27557.679632581854, 15482.29742516961, 17910.66200279371, 39257.607006284, 70078.59889756687, 16659.579861477116, 29937.192453118143, 77463.03440670457, 99243.24160282883, 89564.43970157423, 50333.93269373695, 10161.782774685546, 27371.408942916063, 15400.638724985694, 94552.82946482976, 32972.13703898363, 81308.23234816175, 55251.14295265128, 38702.126488941016, 24075.657429487674, 26271.836972085017, 80457.35964113324, 17496.095928968203, 35668.5972428469, 9980.552899874461, 92188.37817612762, 19484.331590422622, 95057.82243531871, 33611.856105819985, 48439.07281643207, 17586.084426016325, 21.790108760899596, 2124.7764199613493, 81141.519069495, 96926.71366984851, 17637.54048064389, 54270.86461579189, 34510.20302194335, 93804.65321945777, 64763.08074305774, 6393.557448048415, 39336.32523189656, 81878.26491709687, 81121.68444191161]} +{"id": 586, "vector": [84320.50821455737, 23524.14350344485, 42796.80249751635, 26392.880877161795, 78380.31739496668, 11193.049889331098, 9197.656452359071, 99735.16881547112, 16301.375991552903, 90883.37631368921, 29303.36412734632, 72218.27349007282, 10412.139903940753, 97990.14603498913, 70298.08510516252, 76100.82100756554, 43172.63768372134, 4405.540221083859, 74955.65293821707, 56370.037137464176, 52522.919814131, 20527.980273909496, 8645.670500875458, 39751.411020983694, 45037.19010473691, 62581.48953415762, 30595.223460951827, 57498.646294566344, 70258.08347909176, 6730.922466610756, 96484.29493542807, 51302.74906835276, 77742.65773089668, 1194.5710712667924, 98912.20390337416, 860.0304624526989, 59803.04665461534, 61842.28345669819, 96765.06581742124, 6813.56162525657, 83049.89669963879, 97368.09630195377, 25321.98095681644, 94489.93912537115, 86035.53701128428, 85585.21996335841, 10274.688731197102, 63581.79494057373, 40705.84131112087, 26331.34557634995, 96621.52223000325, 34650.225306270855, 26230.856601811793, 37641.832923859176, 54526.91515234507, 34614.678806306, 91962.40478020893, 1087.4311329384989, 52432.35501383382, 94687.83195694744, 55610.3183309633, 46570.75688544296, 91041.26642801813, 92311.82395519454, 50257.18988244495, 75086.01297543298, 70836.46222886971, 46886.64060650405, 4525.935118982694, 93339.56785719123, 74619.21373918137, 13527.719153012951, 84710.31816787722, 24628.870146530156, 10584.717915037401, 80018.68854107348, 65706.19158964757, 9452.103180298733, 61517.82943572389, 21608.061863043447, 53462.89017641648, 51990.35221614807, 26329.154908989516, 3693.799514841212, 29471.98845392124, 80708.78255148107, 5852.279074452349, 75521.82105829113, 77980.17141896769, 30705.02962231837, 14875.454619328266, 21943.101151366194, 43319.731741375086, 97541.65359025083, 16354.637084117574, 76379.10142746996, 81328.46277570668, 33198.34062084076, 38776.725549723844, 98750.16633694954, 72187.52497149524, 23876.677892813626, 13365.37575265857, 33183.44358079983, 24040.328314695314, 36077.200051034706, 11422.321098681332, 9203.394783181462, 30545.531802903148, 94551.72756896359, 51618.7391418031, 68334.9607553841, 19162.771620677875, 65343.811507251135, 9226.518081720948, 69499.60675961545, 22356.691816907827, 30038.137531926433, 19228.971026360152, 84585.09935852671, 93253.40450727099, 78749.74655100467, 81021.52815966531, 20865.761031474594, 73146.25178320738, 77602.13895747233, 99111.68879971873, 1540.7412201669745]} +{"id": 1087, "vector": [17544.91905832273, 15441.759592288141, 40470.13132037526, 75007.21530862106, 64706.238038957454, 96918.52151094613, 64809.11916849481, 34366.54944525404, 92743.0873819417, 56005.640003296896, 72068.50488961181, 69867.9858537854, 23191.93466304652, 49525.78412292714, 85393.43111093948, 19482.412359938106, 84634.6687421182, 6497.329843895416, 12495.293467322044, 66425.1236477642, 27127.78360682069, 57223.63845988777, 91623.2403511652, 32929.9651711626, 22920.503454259655, 32065.50501819274, 75887.48313518392, 48724.73515871773, 36265.67733494477, 8874.584416823684, 93109.4354430675, 78712.6153649427, 35954.33641968286, 210.8421576363262, 89852.65070826287, 71808.74666059967, 54372.26557842425, 58983.49301785792, 95287.40535608327, 82856.05255324017, 1221.1715545976776, 11095.438424274873, 29080.991962694603, 83356.73332081627, 37682.90051150481, 69522.67869729396, 79145.34058941454, 33638.52580248478, 71260.36279641768, 91469.9575841744, 59640.36051654363, 31941.631434977924, 89538.63583140667, 7819.486407699894, 25002.711623303785, 46990.23383612754, 20795.181182634493, 62231.51355447842, 88581.82430548707, 99720.73379759591, 94158.64615253666, 60672.44795072159, 98487.02767340474, 88145.98717331157, 71584.18859591136, 14494.796549053424, 34512.32839287998, 46262.85547939468, 28402.092462385808, 28731.276852578503, 5556.075372564906, 74311.34541948879, 67843.25083171112, 68353.4427907269, 39335.310165278424, 98475.23739750798, 81001.38495146335, 34338.16714320064, 47748.40554237147, 17376.23096669961, 8216.387533963687, 13753.368321177339, 49027.450086311655, 70348.75131145734, 70613.9968399931, 60821.20965371928, 95367.53403420132, 19574.76073463209, 50463.028934120266, 325.9761619972101, 60902.40371254251, 89745.24452716345, 60464.17189179942, 40332.098003720464, 74845.91998857741, 71801.93436984251, 65612.40700931214, 59754.21324728989, 93005.9234970421, 86506.3176942845, 83255.43909974911, 71600.93796683768, 21979.622284491972, 51195.17859928067, 67880.24262852392, 22088.058449814696, 77524.55820363002, 65526.10511198763, 53161.24576889483, 85857.18584198799, 74109.24387133363, 25730.72682621944, 72207.5219743714, 87361.04509956318, 96580.39494382315, 62128.67591733554, 73615.46509226163, 46417.54818719564, 28102.763771376925, 67005.83426802685, 71903.18990687655, 87422.29729520054, 22343.641296536276, 34871.70072710551, 22914.870675628274, 4033.9447880408775, 78597.60085902027, 82567.86498590538]} +{"id": 823, "vector": [12162.944955281053, 10721.604630756987, 4700.241739902677, 90940.70245564712, 99032.56685307497, 61237.429667811906, 49475.34207112867, 38699.835103655365, 31719.043542551826, 47199.80584345053, 37559.152930112235, 26259.55794839412, 12950.485752964025, 34531.15456464271, 21902.195578890947, 9692.01179109458, 23339.036874844198, 54127.51251847121, 7620.034050151814, 7975.368376162862, 34261.868237488125, 97456.4143689891, 13916.010583311721, 69967.83513471724, 40361.66502180892, 84480.06502754035, 56812.42240262554, 93292.3967792701, 8922.001200529583, 89174.6379041419, 44751.8332516388, 41744.50450973076, 43477.91924013862, 95104.98502781708, 66086.44885536032, 85674.43390982556, 38951.9629176688, 44080.04376121655, 37126.87459806069, 22583.393358695525, 39505.718936559286, 5538.769575386848, 15420.947818138431, 9109.99770541645, 9147.978244169886, 50442.491169691384, 69998.74820007136, 91281.428811098, 67825.78319218225, 56591.26324517291, 2181.1655078408985, 6930.866375491929, 89484.48612565052, 79695.00908086533, 64954.859029454106, 75675.94101638098, 51367.8914761556, 96276.21239365624, 51446.228229254055, 99016.96684023783, 72405.40723070661, 87391.34614492569, 76001.46086362767, 16379.183670157694, 64902.89758700138, 36499.075650697756, 62861.56255259005, 19283.72181164625, 12079.095378958304, 86199.92556423666, 14989.871686417899, 37820.724136128425, 11414.207271162613, 81061.81703982536, 57612.75345784209, 6713.935073690524, 45808.223362926336, 45035.727711665655, 25542.70900269494, 22636.879718984103, 72067.32670208406, 35417.879986019594, 81881.22876770508, 8330.511605893975, 18521.428856115395, 7217.434825540914, 93160.45382560026, 18195.933667507157, 42694.18881488495, 89155.66314711966, 5227.881849223181, 20087.970612282614, 29203.356219150635, 89024.1724271887, 35662.988126053955, 49608.499798243975, 35425.44756661567, 63486.5506548974, 64590.09278468964, 92419.84611069354, 99708.00749703437, 86910.19146709009, 24619.619487745826, 38023.70841568495, 42299.53519325896, 90654.32684620311, 21383.263904359297, 92377.0495906546, 46966.87845561592, 27495.495753818577, 57447.87830063238, 80654.12469259273, 16314.887039539672, 62969.70400019406, 93875.66666065152, 91745.23507283542, 98640.83778989215, 94683.21903002486, 97058.04366364442, 12722.27421293267, 69800.9812910599, 15042.36919943872, 10500.23573698219, 82599.83584929044, 73888.110377489, 85069.3190650356, 93718.03435949793, 53441.813019438414]} +{"id": 1363, "vector": [27208.12577940983, 92072.2343773239, 38948.11463759251, 78407.24248353738, 72235.79800619956, 46008.37332339805, 6061.286984602742, 63242.35668975666, 83182.74767121905, 6737.326707881397, 19669.83620289553, 19352.327245035573, 5980.645132138718, 36564.07215457237, 97338.82075399668, 6467.808107891471, 87952.34949097953, 43651.69293685064, 50710.04868831246, 24984.537333761757, 86137.41303358295, 15934.871157905483, 27268.97097523494, 5149.324502379804, 15076.773878037597, 94841.74851682632, 63177.43660186943, 87559.3301600584, 98623.80638898436, 97767.19612759957, 5952.527662498097, 8266.495746729197, 52211.219048446066, 82102.14072109548, 4544.060899962288, 8963.29955606594, 23414.800032764793, 4240.026654291495, 14054.382658958131, 95383.39434395739, 72842.7096815871, 35633.491608684046, 21188.655451561655, 46874.6214530481, 73048.4332044055, 22091.050218038443, 99777.46171557752, 6418.572681390955, 98594.25104842443, 80801.4369296466, 25386.863962220817, 57235.27252052203, 67280.22821907891, 76259.26113639079, 16279.992424295331, 74189.8286479717, 53799.78380089836, 50909.66897444088, 58216.48591837419, 27678.77126247239, 50705.56376790475, 47538.152135033095, 64287.119289963346, 22836.33258709765, 91568.88430384597, 61654.17843168948, 39758.459749237205, 49591.88468119259, 41058.57108950292, 60402.516863084, 7357.647630740427, 58712.51866107235, 21048.503351032512, 72441.53570177505, 37318.73646331788, 71672.01743493031, 6083.593250882924, 67898.20905416115, 87493.52529698076, 20938.424151125557, 95027.8054756054, 23151.086696445756, 2003.8507641943258, 9871.37302775407, 62354.31706138308, 39399.03411231216, 22347.61068510388, 16377.309438308484, 4463.233443009684, 99704.68721151598, 74578.16806738968, 36237.294071929435, 64801.875246475785, 35798.51332892294, 92518.44407553322, 68478.83111556918, 59028.59641674256, 77292.93896788129, 84286.27426228262, 31706.686655308946, 47421.318673392685, 30094.950910770047, 61068.50784796807, 89288.05469628217, 36509.615438381996, 59965.91052036175, 41365.55862695999, 19469.970443594753, 57912.43891102018, 63484.96016795948, 5957.666834100562, 12715.888213851978, 23985.019106649554, 34062.95635508869, 22335.576052096527, 26932.131776348557, 39751.9226317823, 46048.82494816615, 78196.18450999193, 42905.39012988501, 51594.33848092714, 71469.38123927267, 55916.60795422447, 88645.36834428503, 65579.30334769082, 93254.25799123038, 64266.56388403631, 54777.712607646354]} +{"id": 1717, "vector": [63321.099817847906, 19815.23755074397, 29442.169817959897, 82513.51160147434, 62650.34499767078, 75991.0777675871, 33909.645104392235, 21890.500965442337, 15860.05526705967, 55081.23364674902, 80523.6565640532, 65458.7006949374, 71962.60065562082, 93771.73542588009, 11096.950850943022, 8918.634220662236, 90394.52204913343, 79108.37257770424, 46032.65852597111, 15167.559348619352, 27930.65239941307, 61603.70959822412, 69609.10987807732, 4503.578154466814, 34912.48064496482, 80501.05277181459, 15011.03812852773, 87508.34722965788, 37533.82688993778, 48276.47872271348, 35398.75562085304, 56558.595505377016, 56847.52515186329, 59329.134974644694, 6907.422181350764, 22162.785761713443, 87727.91540234601, 10353.238049741132, 74222.96828246623, 53103.547887556124, 44684.358573389436, 77317.41595546906, 47394.37341584221, 91339.44023263108, 31777.42110662003, 17375.923883890944, 32016.568509750887, 76080.98683641206, 10885.950539594247, 17756.75738621153, 12790.39344051891, 78224.60819712465, 68533.08881384034, 29092.8718356928, 64071.49258018335, 51344.17204273715, 50798.140051602706, 68737.24754129202, 18603.495651518366, 91187.09706926814, 35406.535477108344, 25993.171275784487, 98431.26612431539, 96409.05515982879, 87817.40040967609, 77327.34667365701, 79073.71928533955, 57494.02583343592, 23781.194555950136, 94258.43246835095, 52473.38066541376, 98135.02800373199, 68377.92190897491, 21674.86676630378, 42455.599330428915, 1528.492599373399, 62915.95088090075, 79255.20439222101, 17850.013117821185, 58902.76014827912, 31330.146774382316, 76390.60455758894, 12906.181245240301, 60222.4529467715, 98252.12133332448, 34806.56566182684, 19074.30031593046, 31298.56698492709, 69162.28624361397, 59923.44513781379, 64200.085096847346, 36575.80789253634, 59309.726618663924, 57873.21824860543, 66388.59950288241, 76311.42740125982, 81186.09830563921, 87906.28079012925, 68186.8889480663, 62947.40667871213, 12241.360783575283, 33605.80124561176, 52769.592033696164, 99972.61452055245, 55449.01751435736, 30449.23146924424, 61250.438145009655, 55379.44255675295, 6744.878764990991, 83194.01866421527, 25257.506104529715, 60902.91557483677, 19509.944781868406, 60584.15434724723, 9503.555407555552, 3151.750151694277, 21213.61864275191, 18183.14781739875, 12930.488232622383, 31255.9829838012, 75895.99996966476, 59153.79448518175, 33853.97761440201, 84641.48721548016, 4319.21273574366, 28365.96814896175, 46643.50230460853, 42676.599956677775]} +{"id": 141, "vector": [70496.23762070748, 7763.178031669915, 72682.19004038375, 68579.48283559285, 76073.43738541196, 49573.49106076419, 27554.698809432888, 48114.334840253905, 65261.183249582136, 89530.44663419656, 27265.220794747493, 49820.774861881415, 7445.6583720657445, 79433.32016234793, 78958.51235987108, 81342.11179929797, 11182.265508513023, 20178.32530744592, 63086.23800017118, 65889.77781304289, 74538.30364025301, 73583.3290440338, 60394.23624492517, 1325.0320177852327, 21533.84650173469, 16071.374409482363, 26332.219442448622, 74458.9800701093, 93460.01548442681, 6697.581960571019, 64119.80280169499, 50674.08178469635, 91577.2225395562, 4283.781390441432, 31957.2617992996, 46706.56471896213, 51107.07523194066, 78412.31160892521, 80902.33601197552, 36525.66657654691, 15771.767557471716, 86145.52606279936, 47968.70660697178, 87485.57707461964, 42048.691688551975, 58372.01573900369, 15711.371751723369, 75544.20118803169, 32681.613027810043, 80421.95426196483, 44588.148015445615, 9006.237122997307, 44112.79392472385, 62644.3216982523, 46045.18921220483, 79809.52388784254, 16550.01048468838, 65666.90714163374, 45351.893613456705, 48322.8476467063, 15965.582450937676, 34166.58702532555, 70275.13621532242, 44255.01739373643, 8349.14164656063, 57081.90130438762, 11923.359286576595, 9777.64385306299, 93569.42324391365, 83352.68469581487, 6235.403675649176, 40509.90337036522, 8536.873376448628, 58344.26766940179, 93009.7598391896, 28908.41858885964, 88637.76912307143, 71046.36660889405, 85167.3481186624, 48084.740979565664, 88939.61296015214, 7036.381516498469, 93472.26096895536, 32781.87852815569, 90491.85825105174, 1088.3842848185666, 51752.857087160155, 69710.1323178252, 5646.480241561314, 39869.61965428231, 99350.61025308412, 30817.70054816223, 74891.62027974232, 87867.6537639999, 50468.90462014631, 38409.214242313195, 38946.30949525352, 44691.08013166122, 42243.79794164662, 27251.451294938277, 78428.35470117867, 55518.947679285004, 20208.3765064894, 93171.21404983506, 8939.94090506679, 25998.78372884289, 61864.765847414346, 45977.83207890301, 68331.98619681045, 34471.009174927836, 31021.897378644124, 19671.418316442014, 20944.16548209054, 19115.24498841508, 12212.427674880422, 54507.11234901009, 60871.26681714224, 21694.048542932596, 55669.90131069762, 85505.83779975842, 94790.56738286374, 5786.984243002847, 96843.32592938463, 40110.74104735068, 78683.42708037783, 41494.95236692231, 52724.109231940754, 84265.678454412]} +{"id": 2025, "vector": [51713.6377533822, 29035.668158793225, 7258.2316071885925, 42803.86850432177, 88041.64641179133, 85521.61796485135, 6242.3255764536, 80458.5462461377, 43218.663914897945, 59317.741073522935, 2107.3299306452964, 24881.768941681425, 19355.464261182155, 14644.143783720132, 52318.111395073, 89387.66280668967, 57575.69486427883, 20365.305313505633, 31157.631937906994, 65663.94949016182, 29582.98725329168, 51776.954761636865, 67927.38845538838, 69111.02698104175, 76225.68351526815, 69648.46738300491, 6127.435711293916, 77960.05586827213, 48005.41497200376, 88158.38667203282, 86277.31606795074, 81369.28414729962, 64825.66637051901, 47157.64108920242, 73430.63224348745, 79427.88621425859, 38208.29837337085, 11295.008780465289, 99762.16207590957, 23581.310883705464, 55756.65048778834, 98378.7653299877, 70062.55216263648, 4869.0566172106655, 93282.27069438525, 40715.356679080825, 34377.23218299104, 62765.56069319814, 72996.39584536641, 84411.92347981934, 37792.610022627225, 75086.21681205145, 51907.3673646496, 4477.283842797497, 32051.768053926422, 17862.414795500394, 88847.31087980277, 36270.76167647095, 97632.26253121087, 12703.747984585789, 52965.631629263786, 87860.71081918453, 50512.023310218545, 27670.10777805624, 3812.241416394324, 44657.46257217531, 20411.93340947868, 84350.90688775228, 41987.31613508071, 15943.582603463958, 56766.20501795194, 78932.3571781233, 87290.53532349755, 50954.71125684858, 23614.398692306604, 34940.78251011273, 18280.25358924913, 689.2844430618661, 98337.42452320739, 50277.129717238255, 54745.05423487288, 71272.1451455029, 89096.0390459674, 38603.725500164284, 64988.968597003615, 85952.77547692644, 39861.39549137519, 69674.04710386347, 70768.45113030291, 91903.14086621175, 41316.880406420154, 26222.137211565354, 70233.53804400994, 22280.128434857117, 50721.7408882693, 42900.89574472833, 9237.73819061493, 92015.96713294463, 15526.924470087799, 7738.762863235771, 34338.718275753374, 77978.5989530569, 58523.98048041488, 87855.79292336392, 66429.95028450395, 21977.089567550178, 65454.40633409843, 24880.212661371104, 69666.8411897523, 5654.8500358677975, 84077.13722690057, 56001.61855904016, 21244.329323347756, 10910.366018886409, 15486.609726120349, 95466.5463083956, 94065.54640013946, 97022.23428269284, 96834.5650196729, 65999.9234674524, 2829.7646520755948, 8928.509637840043, 68635.37041589266, 3867.556776982983, 17586.264588217637, 28137.915616193997, 5055.156858810706, 45215.30877669669]} +{"id": 272, "vector": [6997.316967210587, 45824.89076384605, 12726.606168591426, 76141.56977596389, 33097.29968841628, 77780.93504284383, 73471.64913948336, 92026.15025819339, 5497.888123239525, 63579.60192053768, 9040.64577740713, 64832.485692308896, 50250.34445215541, 69239.14952739917, 73245.96560087445, 70951.61393822642, 27472.00825484808, 25859.166766593466, 4777.001483211585, 71461.02971333289, 83409.61050272874, 37137.317522240155, 58490.31023949171, 47756.32146605401, 75522.57760390898, 41321.124662068956, 38564.47874014833, 10187.097057862815, 25249.691523048812, 36358.30016028758, 43180.16589806134, 59721.46150999045, 35953.638810455945, 84610.78033038946, 36701.805784858196, 45280.64709951547, 65188.77526849471, 61070.438095467674, 29361.39003145557, 34314.795844045184, 5535.688907144021, 8954.269129288494, 78604.99777399436, 13977.891257381592, 91459.3929547903, 81002.65886515722, 5060.487868162133, 21710.50824172872, 18815.302983975125, 98334.4651407557, 33196.56400687581, 34405.9954765358, 59234.250617685604, 5933.611135478201, 25820.4921717082, 42958.122168488364, 96166.4066648535, 1511.2667541750336, 88112.54134616897, 45158.92920215125, 89259.29939737073, 61419.03390083842, 25586.20167346689, 94362.58289355735, 69021.48300948353, 24908.619977960232, 22973.41817377816, 900.4208734841535, 88292.83533167918, 9598.955661088048, 12720.069235919518, 31507.092552759987, 25505.459441003786, 21945.24250513664, 16246.01464173302, 14520.449574110695, 65182.2942117132, 25552.940235796483, 27791.681845862182, 12021.490007068525, 35266.96817033804, 77905.02110841517, 39705.67802553145, 46589.29295011467, 75880.04300029555, 16732.262209519224, 1728.7554809262583, 59592.120431917836, 96070.94028449309, 78039.36658743229, 54530.084959097214, 90378.42593308943, 64031.4438338178, 29483.538213752247, 4194.5601378259535, 51533.78402386, 97667.25676354868, 50197.77933542537, 60624.81807332264, 97971.05456712481, 52671.164908660365, 29136.120713788438, 71574.29510415636, 71419.60511192557, 16355.57473021596, 57236.58928750033, 72655.47181237294, 69842.82656376228, 94820.75505622191, 16673.714273306417, 5708.929371358562, 1973.1046922777052, 63969.519307782175, 33765.94764424455, 56764.485627074755, 13906.439129597537, 10515.235829062187, 8532.616424862494, 89097.82766102464, 94065.92420389489, 2149.840853264229, 60923.574487458, 76838.55895079034, 78368.94978504807, 39030.51148071621, 1631.573911254125, 90268.85460601795, 62989.278915244206]} +{"id": 1959, "vector": [66073.3311888053, 2546.075458177166, 93236.68504208472, 56667.18912071999, 30248.254023747824, 28411.368049618803, 39900.68730478401, 302.996264153188, 97878.1739862483, 62695.70880903815, 21868.304185133915, 27144.64458783332, 46461.62492107989, 7945.339961070242, 68355.90960057409, 11050.864429229234, 99457.68659781775, 63279.29878978205, 11656.421741627453, 14739.036029894192, 88978.74858785569, 5688.767928779692, 45075.914175957056, 87739.21691441024, 62206.19136354848, 14148.540124193587, 36036.208527559, 29539.990862134637, 48457.78253928983, 20151.447955356503, 69409.16872909361, 4178.000145624161, 71533.87348972344, 54476.80667221251, 85309.80837724426, 18293.41516529195, 51892.56416650271, 1589.544444519153, 70186.9230313556, 36455.58548410431, 1612.59469932713, 59161.73346567705, 63793.11403116218, 25343.087063864543, 61764.21576420834, 2339.8114206556797, 51162.96252424093, 34653.967081477545, 54429.17835561558, 61640.53019114826, 43951.05907720651, 24833.319687352374, 97904.9581547272, 77762.66415890856, 72523.59897044754, 6323.032299496534, 58404.277839610695, 32096.644932137508, 16202.344124919333, 20953.8018629834, 68736.38645725342, 78918.35783083217, 85707.56166624601, 93277.8939678454, 16828.992693485325, 69089.91154614615, 51763.11235958596, 28496.109607918617, 88679.86211775464, 91388.57541480169, 96365.51455603204, 70606.2439404768, 45549.61973255933, 82000.59048101991, 55991.251980927016, 15705.19692739384, 62578.121956323295, 30416.012886523004, 81155.14052465279, 29850.856109080338, 6406.083724719058, 32696.98494332275, 81415.12362998254, 33239.27003909405, 30729.603236724866, 64085.050064396135, 98492.21026624412, 98824.1828055529, 54641.313720670536, 87873.92371283214, 99187.85177271313, 20203.56259881221, 72850.77341053622, 70378.71625183568, 94251.44037189333, 93286.57952128343, 85890.24953050088, 68655.72855861524, 79530.93352115336, 46141.657667918465, 26462.431215116223, 45795.421404089684, 8334.515263087373, 43458.63104157284, 15756.569828627642, 92958.93247061175, 49717.09815268217, 70056.37988280595, 28240.179087611006, 4263.46271001995, 61916.42047857996, 35220.640810889534, 65833.69828727041, 33240.160270175365, 67021.22914561855, 66835.81700401736, 43924.69246578471, 24199.62010644774, 43868.75431649181, 53913.607736193844, 80543.78355604107, 3282.81793161308, 64555.60485386206, 97702.27200613268, 64896.08084884158, 69671.59128014382, 47801.612739367694, 10032.83625581175]} +{"id": 1947, "vector": [62282.02592566113, 35638.328345931855, 71783.55634989662, 71629.39286569, 90294.93882437135, 40842.75712423035, 90284.21665668544, 90187.19152794726, 71204.93351188263, 9799.141419115975, 19502.393669620276, 77003.07152912328, 15327.946791487202, 21692.07692237688, 77365.40526153713, 93187.55410972217, 54234.71534477317, 77998.17952632284, 51876.00013548955, 90173.92329980846, 80501.7791009122, 70959.31196356381, 49115.33243417313, 39498.090865535494, 49842.99487881745, 80165.88953353491, 76506.69646234282, 20911.04104752062, 55411.93639110142, 72276.93191191061, 51115.34345075967, 77256.38341942099, 55825.979219001274, 52761.919782362755, 62405.30406476159, 26483.773995002113, 5039.696053966347, 87429.15266649575, 89027.57706162454, 92669.81328051358, 48102.82574184679, 1231.9091177997677, 39352.270861989324, 97350.2950010848, 12876.390320886689, 79216.23144284559, 76344.7744973813, 78559.98263728549, 72206.3932998139, 67220.3726005732, 37789.48014907804, 13684.945447027452, 87215.14779110519, 13294.457177017039, 53910.451604182555, 92544.8297247515, 68923.06952179386, 67834.28203751869, 83034.83389550568, 2408.0206804021896, 15435.726704845887, 45107.47241713656, 73445.25083819196, 39306.9443673783, 79470.34739336974, 49622.168510591444, 51034.78447487632, 38185.338492965006, 40766.27758685694, 4594.450901519309, 36253.44725522285, 67069.6507969608, 23534.311558635367, 96783.3132562093, 50289.21622950404, 40544.15295995749, 49252.04512970091, 39736.95342441168, 5996.8561531514415, 46285.31383960875, 50063.18299055436, 1997.4466553952852, 95104.92370938414, 69349.08428048363, 54949.35981274849, 69513.55166774071, 53893.44918112373, 40603.35276681717, 9634.377337078704, 79271.901590396, 41247.11910500628, 10339.687430330447, 365.77104531628765, 70845.41852393605, 15546.620539721323, 9127.961884505576, 16003.976380213315, 4060.992955047815, 82686.74673872377, 94533.07439152942, 7747.960036769452, 86121.80742173704, 67943.57713709102, 74809.73695691425, 73142.36942622073, 86666.99454833081, 63481.700824099644, 84615.37773841314, 96596.38045529845, 54575.57203917783, 85099.11681351461, 13152.758800158737, 24038.133008990324, 20683.450531703427, 60741.39434133392, 38576.83863531459, 11556.39783396676, 18245.251449098443, 29629.609568713367, 15362.62221505651, 8394.350259479188, 82284.52356642512, 96598.79668612004, 48580.88524008198, 21216.949578189924, 52517.33276300159, 25200.0297273528, 15635.409341252549]} +{"id": 1699, "vector": [65908.98511917319, 42490.46915948661, 182.63190572921363, 72658.36474885198, 5683.120363582372, 65070.01938912171, 89558.31186360538, 44732.64738583171, 41925.96395518794, 34154.96334915632, 60368.33467094751, 32418.727907848923, 66868.7880695048, 20162.41197284986, 72162.31303793543, 97234.83578776923, 39198.94119725026, 42725.19196584439, 91861.522580857, 80107.2017860211, 43710.76889216572, 593.7354890381607, 70658.30331121365, 27016.461561950735, 61035.18091661656, 47931.4195427814, 31253.771627040227, 42988.88197012762, 30591.80510567272, 2420.6628388617646, 36943.63529573625, 6947.933968173669, 23457.819108663138, 28351.50117804336, 46033.85798833276, 57207.575377205, 19532.898318395586, 2672.598438051088, 59152.90046710101, 622.2153759657845, 57271.510876065644, 38543.33838121655, 29886.748626183602, 64456.36120024044, 78409.41510421564, 53294.468361785584, 4660.334791859244, 79843.96841030219, 52644.06741016493, 27375.786011659355, 12595.949321266242, 92386.66996570934, 36341.925129456475, 62497.24776345648, 31466.47785994392, 65975.94418482704, 90335.8522520209, 53213.00007824218, 39517.240254323515, 43948.30400043839, 95969.38741752834, 24624.95293269673, 77010.73325043186, 19604.558942584237, 71046.30876914985, 13067.32155402429, 55011.33048004194, 67472.27884664637, 69576.45745291708, 91152.65948388247, 83941.68488385492, 40777.886523695604, 89945.42600580337, 63786.207302674455, 71676.42146311939, 18023.318620977745, 25611.669572385297, 94957.66626425224, 96044.96145502213, 48317.50681448534, 16009.466084640833, 41977.726534271664, 75132.21957666625, 83926.75274545273, 34920.46235631598, 51945.75109990772, 58529.97343058603, 43373.2461652858, 2305.9861233512556, 87088.46509320232, 43047.53375120536, 9272.42859566454, 57028.85773035499, 75600.97929274227, 1702.4657360846684, 50480.25149371428, 49640.30830991475, 30120.023269321093, 68525.81322228014, 72201.9916410731, 58337.46947664607, 41452.10648830177, 74526.68700778202, 82467.40410471102, 41414.26175259487, 76502.94769943511, 51953.1125808333, 43962.89923033254, 66918.33690650652, 43386.4828561676, 21164.116649391708, 32593.577922168293, 85621.7193878007, 14025.060071560612, 21835.995014123666, 47963.85146662895, 58789.68832409913, 7932.546641451377, 23347.232124128557, 97816.15648585791, 31518.21971619301, 95411.8346703938, 6229.8760367663135, 57322.42908966827, 9588.134275449867, 15082.22742331139, 96367.47644495196, 19114.7526535431]} +{"id": 250, "vector": [50549.08235511158, 73015.8460909989, 31428.450794021435, 41667.31895318209, 31074.61608450566, 25050.236919328527, 15623.589219651823, 66080.70799024477, 52189.5677877087, 21402.06452423361, 70956.77111166438, 26628.955569810885, 61589.39884666846, 18373.501231177135, 17367.244296138906, 76089.9264115886, 42211.97441673388, 27607.926238989345, 54734.241484522165, 4692.87145220415, 98725.91188586666, 10302.282991884182, 2719.57735373014, 87585.69104568375, 14007.30444494217, 77991.88208961056, 89344.09617988367, 91630.03261260231, 62567.014091138204, 98898.6035356629, 9399.202559039166, 64625.415189782296, 96079.01541408015, 74260.30620589374, 91844.79606882203, 44975.392366584536, 36142.60729175718, 64171.10199382822, 6121.399654202631, 53289.48059177722, 7397.87065757842, 89073.55250610429, 54845.623581594016, 34605.54331816238, 50904.65964397216, 7074.794900425119, 89907.96673480567, 14994.919651928029, 11261.706300342534, 73302.69519620274, 73899.65734363897, 35176.32150638984, 94194.22034877686, 54075.07687750274, 84087.76631177208, 39202.682338317005, 49760.760797521405, 49380.950506781286, 10954.614625992754, 80659.63825238647, 6534.090366675038, 42301.52364705318, 19202.90502354164, 24450.866320041852, 90346.37398009431, 71084.48482580888, 98918.01005136626, 17974.654823918136, 37061.688025590425, 61543.52121974792, 86690.97662517008, 9403.018954125819, 32748.507535908222, 68755.08108548347, 52046.5136724886, 88904.33547698658, 67649.31516237516, 27749.967646862915, 36785.51020368626, 54468.87472969031, 24338.086163494543, 30289.17874013869, 28215.364748665615, 99822.31816665926, 43430.420292507166, 77804.01273867198, 15563.78565902382, 58384.36563094566, 60259.193909326204, 914.1372117396851, 59195.69977049269, 18546.115750505698, 28895.107657028573, 88011.33857898555, 89466.629127691, 56464.63763143248, 21316.811744417108, 86190.7701831787, 56787.195298778424, 25771.864871816706, 9601.674541998307, 58950.36398325502, 38978.443713646535, 99024.39237666428, 7889.4103747845, 95360.71288355492, 88735.80888681111, 30440.166853425653, 55012.12338133258, 43262.09187711303, 50217.08840963032, 32857.32834202704, 58505.873481970004, 30292.908018044072, 76848.36690365271, 57606.85691265088, 15271.028196250236, 23811.990182343667, 8965.592263261724, 52234.08476052009, 36475.56679283281, 20724.064532258235, 20199.878882879173, 73002.36720881895, 8974.029365699387, 62160.89732698022, 19486.32150327053, 45176.66025039671]} +{"id": 1730, "vector": [73848.40321845523, 38930.31471595588, 10421.381418290764, 49855.11606411901, 9763.195722588269, 68894.61360803833, 32721.062085325582, 4091.8148309978974, 87657.83400014821, 76299.76864907343, 65606.63556279925, 69918.54770741293, 13207.114624898142, 32393.582744235406, 71353.59537586977, 90257.78026986135, 19849.579795461057, 15647.241128450529, 76179.69175027557, 4901.038510052303, 33443.407029596485, 40620.41634290714, 12542.12914479671, 67747.96670942871, 41684.77029239521, 20370.811719915728, 31543.69017716705, 98877.31633480296, 84347.94334157767, 62142.80821699997, 23049.02639989502, 78136.22262377915, 23901.60717768155, 50014.04529144633, 73646.48335852355, 40650.40381531734, 60512.58405053883, 63056.305307105424, 53918.05144587005, 85728.95379826381, 12253.098816264262, 29935.355696368602, 73143.08785769175, 52236.365168482116, 98172.45619145106, 32808.60693154533, 87549.03161844573, 50217.17056026754, 4282.4015735481225, 13705.101228075267, 86061.32033879224, 58591.75335575335, 33947.33361485139, 5893.48853940882, 77143.79490759525, 521.8257112216685, 48761.89723848929, 18802.094007583557, 28083.814708249, 75482.19581399344, 73317.11458616961, 46401.76137002059, 48264.045387484155, 76489.81441949337, 65809.5887832898, 13390.369496289612, 27985.69908405315, 8741.228024518532, 35029.76398838282, 57208.45364751578, 1077.8456777905453, 7342.00226857723, 64618.71932534674, 16211.468402407814, 74408.47628464534, 99878.71694065508, 50135.70687627753, 28134.45805345217, 13092.531386049566, 19140.848937158105, 82280.73648016131, 83648.59717547659, 69741.32909946748, 27755.452501429136, 14944.469677609573, 54543.88469788557, 6314.848282334818, 3761.5377969817064, 15584.345473886297, 98275.62972361676, 74890.32691573279, 55769.7838012058, 78584.77393670315, 15963.861366431065, 27910.81461473145, 25689.26024233198, 80337.67213504361, 60010.72943953933, 40229.66818418273, 6052.699931998551, 16797.59207786781, 59298.502536204745, 60299.16275921514, 93558.91995275633, 49047.89156010861, 43226.97043671109, 75034.35701496773, 65351.043943331846, 33073.478068771, 56119.95958311997, 89591.62801059, 53263.18440859272, 38180.51650570643, 16721.578885084422, 83597.67105553181, 79634.43711896536, 27110.823328948598, 42538.93172746822, 51639.578387068585, 4591.9630996054675, 61680.74798198965, 53983.7212430746, 77970.53366841325, 53601.64827291252, 56555.24885911759, 58154.924856005906, 39702.614564447416, 67730.73163649855]} +{"id": 1198, "vector": [69842.9378274169, 63080.80673086619, 71703.92628132126, 33169.493154619326, 74590.95037355943, 17286.130004514576, 18988.656365669864, 95533.45802182394, 92059.31356332959, 88564.15319763887, 49826.29227089803, 43602.390451776904, 31714.263060573656, 74556.85164641074, 22833.42241832672, 58665.38200783219, 89438.31890473541, 53838.63969200451, 46973.61807412453, 92068.28181999213, 34170.127526453376, 72801.30597839535, 25806.454611712783, 96003.58424723214, 39825.582765174775, 28478.840281983965, 65127.237815018736, 68873.09501289375, 38118.03530426393, 11397.757344769965, 57235.32786086777, 92487.21648922657, 62365.31148988531, 59213.66090500912, 5338.215853534612, 6521.4732401691845, 44264.05500081968, 69461.77560658558, 57510.02900171752, 16069.092342414915, 86774.99431496959, 2503.543659509422, 28006.11848922685, 24405.83335817016, 67021.7990779295, 48010.33831355509, 18530.783921070913, 45083.79249684552, 97616.12791637813, 81631.29283344037, 60377.826237988585, 94408.52402039163, 85236.30175327821, 21766.9403528847, 5653.778322007852, 45333.61953631905, 37392.35061995781, 36444.059495984, 62427.37803544512, 86450.93239831307, 14696.570152726084, 74846.42571944607, 19732.936243011056, 95169.99718419633, 19789.342483095952, 21095.47984166755, 15095.837209083751, 98000.2633391523, 24837.430396422533, 9482.722979983093, 90591.68109156008, 81701.37256500138, 59214.678760630726, 37214.421470459965, 20868.648783652967, 36762.88165000254, 74065.7825021279, 87624.12169503805, 2456.328421759657, 34855.72133045036, 88632.66618036164, 48072.00586507851, 16637.007660460356, 69387.74416007104, 83513.56597121259, 819.3283423398756, 65011.83004832335, 47848.3012145659, 4105.544470220857, 60593.2593377264, 30838.504349643703, 49826.30330071881, 25796.079778778803, 93168.45977125021, 46824.83685143432, 11959.911822302249, 16309.591880640739, 61980.02229967379, 53863.8704264053, 61376.65758732271, 30483.310023193368, 48619.07123461917, 27334.58883737402, 34799.040834877735, 66419.00220677082, 83924.36496538582, 93940.99270966939, 55337.07049883202, 82674.671136394, 51081.38419138553, 57799.97104117516, 30854.017060453265, 94351.08562125199, 14822.319171970355, 36819.440983651184, 44048.6624859898, 68009.33543584384, 45056.23270718286, 55215.61590584389, 46151.36343721354, 36531.07672336242, 3043.009275311215, 731.8375662384469, 41700.93307100981, 15118.987752807967, 40751.964834464416, 85935.40732031527, 82533.82379572664]} +{"id": 49, "vector": [35147.824057305676, 49867.532991284046, 91481.44289251453, 94011.54186287014, 83828.76293996378, 81755.06439691626, 15069.85959942414, 97359.19731422502, 5757.570054227134, 70935.39561932029, 62722.1241710328, 10045.660264329548, 68502.83874245515, 60230.86401629519, 9306.473164029561, 28156.98751738209, 27769.013088208772, 99979.07837320995, 39845.485644008135, 74104.85142612699, 337.5258290623018, 71988.9922039699, 93588.94078367313, 61487.41563877032, 986.992194431835, 48025.34732980295, 32689.208895692045, 76356.1548546269, 7427.1491349540365, 30200.999241307123, 4754.606310200382, 98750.33365015588, 53923.16782936923, 28458.728431605894, 89604.56479516125, 29979.465935229422, 6335.359976479215, 93799.79544647517, 72320.65871294246, 73338.5702883414, 46159.20614842517, 5996.455556366153, 11605.399650565429, 25192.651073810546, 40389.09193008022, 35706.64606626383, 58240.28753858421, 11690.069195673235, 56262.714739573836, 47059.27385084152, 71938.81881474076, 30376.885602119495, 53896.55562656815, 941.1444671983404, 51246.80761479762, 75384.0066884659, 98435.66059901529, 33747.127710962675, 30991.994934038015, 91854.47748100705, 55420.93529664384, 91352.0956070505, 37051.142605985464, 69920.84425049512, 47244.378766447546, 58661.5199803169, 79315.83578719161, 5128.154713618216, 90248.37960230611, 6586.968131194748, 99575.33567975213, 19754.933473442492, 12252.900176179694, 63802.049560745654, 92390.84859003228, 4083.58851218924, 94196.3643769232, 27845.922063439244, 34244.54558062385, 1058.3697374019873, 58150.100738182686, 90271.3398856554, 39917.9579045798, 60371.65874824441, 43990.851852345004, 29690.970505465953, 52973.78216595621, 14203.911128236958, 22277.958644140883, 69004.55453085028, 9077.189909658955, 76623.92531775434, 89808.90463969651, 17627.746808645996, 53232.26044409213, 12922.470570680889, 48558.952475703576, 45079.221028834916, 94552.85353094262, 14064.895041582404, 6848.308196491493, 81093.56637975811, 6680.5950861820065, 25118.04268456256, 41602.82921185532, 75349.66788988824, 52109.45519661449, 68255.74494038941, 87321.2601434271, 79406.52670849023, 30860.760567569512, 47990.907951415575, 88215.63497515359, 44692.511555202655, 2528.0624518275617, 12736.809836169505, 91165.77286807362, 63593.98999477728, 34172.11599101371, 11062.391966475383, 41241.25619307381, 66851.84540594889, 86655.13284312031, 7480.283375677088, 87048.60149539389, 79142.11040950014, 61563.949913849414, 71789.14495603937]} +{"id": 277, "vector": [51322.706325351544, 14251.064517125456, 46606.92685209487, 50391.3107137817, 73940.82003044858, 14996.390805358917, 87785.64562583122, 91759.55727587604, 44839.17565969995, 16625.847364873323, 90856.77169904821, 24872.64861716826, 51779.33025899453, 43875.18785510747, 85188.25799576966, 24034.640788764926, 44811.71060533559, 6075.873871647031, 64814.92431928657, 70515.64975592779, 39491.09388193196, 33606.82921190531, 19787.709172256316, 45321.70716850507, 7532.604555772282, 34035.30881353096, 48343.40793856016, 16290.229177074967, 66029.01647902941, 53754.386280031395, 32538.153080299548, 77383.96885004248, 79723.1934269253, 33152.52957949174, 58977.699885908885, 62754.61216794321, 19649.34638646152, 24609.002668190016, 43062.15380683913, 69908.71900041567, 79102.87049559545, 13566.624500814283, 54069.405206288226, 39814.88248459159, 31409.087067168995, 8515.628921191243, 95206.17076046152, 765.2637916320871, 14328.846175561504, 3797.290919532226, 65456.783028292, 52771.19756976528, 3278.8269550104496, 21952.655222082572, 47139.15406080075, 46265.6568266273, 47344.28253735342, 82692.82842444748, 71411.20205241622, 66120.17460017659, 75179.11161470864, 90735.59954644012, 39508.73160455283, 30918.69116923378, 85181.1564463161, 44022.55803389406, 12445.61326598772, 77370.47850154615, 77483.3296793612, 48556.59288652029, 8529.859520603644, 11593.026012328999, 31230.360176931372, 58743.529173160736, 28748.876092902454, 86335.46784405697, 39594.95656574733, 36838.08946820532, 78418.41125571892, 6058.2600393315315, 32051.880762969464, 6404.487923309665, 74290.09247887459, 48232.832876304536, 51906.6082298302, 55709.31069362347, 78804.74804107832, 13618.284670733672, 41273.84337223541, 43194.32852723511, 34633.59854792397, 98375.35490946555, 81026.85307420378, 15670.893215128912, 75761.09904361336, 65645.36358438052, 47364.3183308868, 21331.192470242288, 39841.14873024866, 72001.69890782112, 57450.0906313572, 63456.286834860606, 2973.012666943087, 99581.49463418656, 90942.19283005016, 29005.60537597705, 19504.22195335827, 96009.55134870189, 41626.40335921757, 68918.69530061154, 64051.276200170534, 62713.18363282114, 8436.197920551003, 70209.23938462975, 19508.21851027621, 21333.290717853582, 639.8978914793352, 925.7075723372088, 44848.68239269133, 13167.976621472277, 38589.981974158734, 30521.820665824085, 98836.67063303248, 39081.29799600906, 42009.21038064938, 96676.49820708731, 15990.015459903705, 60597.70887948324]} +{"id": 2108, "vector": [73072.62508478401, 93710.27797041343, 6776.1584856181, 59472.38450334821, 73565.28059579455, 4912.199714090082, 62919.462481425624, 2554.7170010745467, 56992.7629133307, 45462.05566155042, 29028.228753214924, 47963.03821546225, 37362.806747586284, 2844.285387458423, 19633.424452450963, 46560.598172416954, 42560.10223871622, 65392.76638299143, 64228.45240046683, 83651.88607899989, 54787.65798113197, 84868.53199512674, 27766.718513992782, 79732.81459772751, 8245.469969850872, 24722.45668778398, 33775.86186494996, 11908.080795076514, 16645.31536978204, 83667.16108469709, 96438.99029609913, 42631.619430926905, 84061.71596854189, 23018.950865056453, 49180.88678725099, 47134.94578132916, 63010.974805176425, 39303.870619601636, 7344.289676814908, 16330.089835883999, 55076.89366303353, 52424.51171594994, 20727.64143494944, 16254.522631302925, 70166.28760927945, 27803.17949099518, 40267.56066957167, 52518.22979342683, 96614.55187718401, 83235.95078175137, 7688.097495857638, 97570.7875452794, 58190.14399732614, 19672.200089130933, 65785.96272227325, 20930.247913077783, 73277.15784258598, 45343.57459207122, 60037.69592852254, 43152.201216680405, 76329.08965961215, 35272.18022092484, 77903.65504984558, 19471.150312971662, 41165.65375051149, 31625.692931201145, 2004.9438472129143, 50250.708809912845, 37040.62499662434, 41429.34793403826, 99255.93111478124, 9709.322784272, 12089.708194743864, 2867.0666545314916, 62024.540528230056, 24678.29468318542, 44480.60923517446, 25713.921504022364, 94043.6699573262, 10441.880879252209, 68026.70620725976, 15476.19002957782, 92029.64204058274, 22571.82172071507, 60491.43615786131, 27540.20653972111, 18350.301147412785, 68115.90162530832, 30778.79606340559, 83644.51251585931, 44404.617788407864, 52786.71251952578, 72263.78321837036, 47443.237409210415, 48724.37083562283, 87560.99696194059, 37395.94438709296, 36165.11515941856, 84909.42752690887, 12725.325426385847, 6242.877223086052, 53866.95888536107, 83750.00244041487, 81155.68495421195, 97873.41218458729, 53039.92647531199, 96506.71894955823, 6250.00862242171, 49564.034142603676, 5139.530305381246, 34038.03442131732, 1882.488032652385, 59705.92945487356, 55028.19840852884, 15437.025290022499, 42010.228277103546, 97155.3735818856, 55655.327523685395, 54730.71721598603, 56191.29062437548, 97602.6196574308, 52195.03426055476, 11233.713124097532, 90173.67106071863, 15353.228000210684, 95032.38212447298, 87636.98764958128, 54472.991901787915]} +{"id": 846, "vector": [48455.31562269788, 39262.225931294684, 91028.09902675552, 96396.51372324438, 90669.77433081862, 91003.05589547857, 75389.17050823708, 74290.04861390783, 71859.63091497621, 45272.6944243501, 95672.09700127483, 16270.3709649195, 51741.88856519197, 49453.902450979534, 49247.42627485049, 82123.78416857419, 8324.912407466567, 58175.12437455471, 32527.464212483414, 16792.53432425415, 94519.84584601299, 25478.40998949562, 8544.589808477065, 57956.15917630323, 38153.27682693423, 14224.638465655282, 74034.00368603921, 68270.03873200655, 98643.65348931335, 45703.76034029664, 29747.359512455852, 26348.81752002316, 48140.62089399349, 31302.076875073115, 24533.769867022194, 18440.413899000785, 50868.05511263123, 19257.56305118317, 37363.92532315676, 81545.70170169917, 53267.49895828757, 24664.26452808802, 85360.93303341443, 57447.8329681405, 14941.349595207997, 36829.43337582297, 18519.120923338716, 36063.453575352054, 69631.08245644729, 16621.144104999854, 26200.052335667257, 10722.36425619354, 29429.87914512035, 56470.14702060741, 90289.36426788269, 12471.790621911594, 64395.10063257555, 90413.09747194086, 71962.20663987906, 67106.21416937717, 41493.32332607862, 14166.552823266731, 54642.60420985231, 22969.874953221824, 91257.16204987375, 33356.9980530092, 6776.472548155787, 15138.508931091987, 96190.69402091598, 40028.161528106975, 14121.920738839111, 81179.02464934612, 97945.91295056943, 60248.6592995126, 28292.17466891675, 54514.89802954661, 27755.129456289797, 85250.89297532878, 87492.36704269623, 19244.924315157663, 43684.784873922144, 19982.78146639536, 82526.08367119778, 54776.82515077877, 60466.76119249147, 33640.7410764718, 44348.64468033287, 94335.00930379449, 29618.05047245094, 17990.727282826647, 1970.327027908403, 97745.84106956447, 2488.9114925146005, 84952.12598248528, 96450.87410915141, 5347.864090252774, 17710.76917173663, 21328.87536009165, 15524.129126529973, 26232.994743588057, 97096.97634266924, 6206.746868511958, 58704.42721007818, 59051.48429636726, 16109.376819149767, 72123.43516494882, 77546.51866884544, 17673.462720016865, 53367.49819299017, 3660.309564950093, 85107.30417084612, 45919.53177596471, 51639.22254119483, 4217.625401210079, 22473.246830956505, 34879.0039951261, 18412.517785485295, 22091.59997147793, 81588.73443391529, 86115.76088495828, 66884.96214187887, 52409.763872515716, 66317.35480591799, 74073.71667245675, 60052.554804379375, 95043.07762081064, 8409.39300092025, 99923.37812933483]} +{"id": 1428, "vector": [17448.920643119058, 14684.824490336568, 69017.80699305156, 75227.89102760403, 45138.8231763049, 37212.15713829569, 93337.42334800371, 84275.19982245401, 92531.18924030192, 66802.05707587682, 36412.81102746087, 46433.32735324604, 36730.94078997712, 29417.784992090666, 39354.48779588625, 47243.07861345957, 43755.73513415149, 50235.52534559489, 97032.77276234707, 43909.95526326964, 76474.82812108463, 20913.03878754397, 67479.31729653217, 71030.96130476968, 98166.0565471283, 73102.52834294587, 44630.005820335115, 19904.85929274415, 72799.59957503904, 14930.926346659257, 26050.046113711735, 26314.205962157168, 25287.354446771704, 1677.0019740851683, 29633.930842652368, 19514.969362658176, 87552.82001430185, 77497.10217104049, 78539.15800124568, 55013.343384377426, 61241.38127772873, 52267.10180140518, 28784.588422128134, 20538.602759553083, 93141.29288687104, 76613.16571576644, 42821.27699585454, 55270.21398904201, 54143.978826130115, 99523.81998324116, 46483.16012463511, 48237.33341991335, 3911.6047463871873, 67727.76288424162, 10372.991493929207, 91285.49151839205, 92471.69249312698, 64152.50587045292, 4854.912297258862, 29766.25608494161, 36867.44697140608, 42370.11518463435, 81494.57588963627, 28628.455618380143, 30439.679932788833, 60849.736065807534, 81520.6202476738, 42231.55688547206, 88728.91839946057, 54507.824293082194, 75246.63784732365, 12232.418213531737, 11982.551676729436, 63429.50310376425, 44262.030311082315, 78488.36311814509, 59429.63550104595, 33969.63991530412, 85050.05943870156, 92713.7162881958, 84733.97196440698, 6789.385430231554, 52027.84439932986, 83560.32984401888, 41902.98717089067, 22410.489273844993, 80719.41066830726, 95935.76568490999, 6999.774012631188, 30537.784223965125, 77254.02535449226, 50013.16070861596, 60317.97315658354, 34640.30802356577, 85937.76956912402, 91720.92600187707, 71862.91935205016, 15017.40632208387, 30638.74777861362, 92900.68170623001, 8522.717782635169, 49906.25519282191, 26854.179690185854, 72597.61867831262, 78591.31302782307, 41271.26573622022, 46826.96719467031, 25256.83807510293, 34112.474711033756, 16186.733974645906, 76893.28494181487, 80191.20200430088, 29500.246119491734, 50323.10221608759, 29291.228595200835, 90056.42023828179, 22734.378244843178, 88746.7959138256, 99676.62877886363, 25176.977363995724, 11013.041258982581, 74244.7794208947, 80868.18814397529, 84353.35736646812, 65435.96481985253, 94384.26665974622, 97285.89330747406, 28435.945525923413]} +{"id": 1401, "vector": [42013.92553874913, 11031.30963545973, 22591.6065560268, 66449.74365859016, 79024.8445669453, 88927.16452711404, 9562.831727374143, 32756.38591427249, 67591.06182802364, 97483.29754404761, 15203.925873917779, 21800.41812766047, 31567.497236523835, 54364.79171760448, 17327.474080272652, 91125.55825891756, 1260.6974162082895, 98281.52348302669, 70569.19616250752, 55554.31133557188, 88881.70029625499, 59667.66652855113, 58429.98628990096, 28007.20184116112, 4459.527778894346, 1570.5670547738416, 54299.7797634742, 65344.11502670179, 54122.19394166724, 2790.5417104460817, 36774.2436803091, 25244.96326840351, 23196.125029029525, 98275.35963609163, 5423.233394007232, 9450.659996739718, 900.5031336822622, 84574.78079016691, 39719.920559490914, 24877.96320465926, 73296.08124408976, 5319.762379593407, 57600.79394626457, 39125.1775718899, 16455.199667778197, 3863.0873302042733, 27211.8533372919, 57796.41074733227, 75350.98597253901, 18488.3173805059, 44095.54470671195, 44867.85043860506, 99606.28875217031, 96419.84660362678, 16067.99748152391, 34176.01321688146, 89118.04213268236, 33239.43447031476, 4598.390875379832, 48254.82007043215, 7938.743679339544, 15244.13258140136, 26234.679311951404, 92839.81627969077, 9716.973308339682, 85668.53227289482, 93916.25616141716, 78339.68881895354, 16766.85447019505, 48212.12900955079, 21811.18273477357, 88760.343187503, 96546.02723995858, 11890.516316158672, 63677.51272871529, 76747.22392972252, 63592.4245649097, 59548.08030934836, 77176.61977677037, 49535.28654118076, 67141.23668693464, 15978.716875295295, 10757.64428973911, 72768.41419600589, 62920.9819865896, 64874.08483792824, 76787.77802205228, 5554.4292372259215, 27420.27248037626, 19467.27512041153, 1886.9921502132959, 10729.188741626116, 46407.79891357205, 80440.9827218383, 5786.90849442709, 29995.357742041197, 18333.71059297354, 17803.74258780303, 49848.4736293843, 40992.36297239287, 40404.8132493245, 18685.36092920179, 35126.543764097674, 84920.85330921259, 66777.24249092065, 47842.63042175575, 70633.35777382269, 26609.529629826877, 12161.454429655383, 49551.113830520924, 21341.924908453104, 88257.56697129563, 68734.29828114217, 65390.21184123321, 96437.0937541726, 84926.23020485773, 45766.30358858974, 38834.15533355894, 10715.820851928493, 54217.09984985148, 84009.65340519566, 71983.07874419416, 72938.12387248044, 60893.76970874775, 40399.20804262531, 62796.66228491286, 87517.74661344937, 90525.48094407428]} +{"id": 607, "vector": [51496.34279118935, 24262.43878839679, 27658.25629107326, 75651.93908225624, 38484.56034991034, 19720.073510325277, 43209.94775843061, 27102.347063676192, 77203.84778288915, 75604.87100780506, 53148.963967877695, 56513.330298481254, 35784.06391283944, 59799.011364942264, 48885.5896343159, 4434.434980058822, 50309.76861465213, 72790.2855788338, 57811.100006758934, 53136.08042907727, 35542.19070774688, 74413.00992005524, 21384.85736513477, 64535.19513161959, 72305.31105676727, 10613.580677679645, 41557.616883760325, 91891.07784457481, 59270.5611744594, 47350.31063797737, 26857.25133256127, 32235.444257307543, 41549.33059806071, 49327.4888856516, 16716.2485903684, 18613.05697033445, 59946.90153332639, 77603.18561601028, 85068.96248348485, 26208.393403243124, 55521.155018227284, 16328.074278751381, 75568.2404757337, 2820.4427746476244, 9062.731510286538, 69948.46219685598, 70341.12008577478, 33673.39084018244, 68105.73177877895, 78202.64405915263, 7048.510818160303, 9972.325598973697, 3716.136802564685, 47226.45583011292, 16400.667891560228, 3728.2075362864543, 75311.02176697468, 19318.954230047148, 25047.97363103931, 17440.78810095411, 30480.528748814984, 20658.53282161718, 35397.86578256262, 27576.006148589906, 34624.529480567726, 95366.77658271819, 33216.724182336264, 71723.4033721977, 69410.12197350261, 59383.409137323826, 42247.30557532498, 3084.937414501332, 11416.176708350968, 22725.648435138944, 67919.97901777511, 12341.739239464821, 40339.433682429604, 47361.33244721312, 29260.00080642266, 15664.454828252938, 76853.0959065133, 95382.73369425148, 39375.50877055632, 34640.27423873052, 60888.08681231869, 3596.9080763668226, 44624.2463151597, 19764.396982388764, 50764.10075643518, 39267.21981970743, 35039.70170706194, 51589.82294458927, 33525.145266888714, 91099.81683915657, 302.20157459533993, 35873.05010438946, 50647.94521057213, 47207.29346611807, 10399.078123794003, 31078.676195007793, 34042.503824404856, 55435.75139224094, 86294.07466779089, 6889.029667723267, 24454.50223218666, 39697.23751875311, 97204.334708796, 40655.58230642472, 67185.37966474418, 25420.944495528096, 16724.082101179218, 98180.17596183736, 51144.645773819306, 25345.765710082058, 36145.78890402047, 59764.302669993886, 64326.60056391639, 42098.08032268591, 65493.824577763226, 73561.30496009551, 32499.965549769462, 85452.82322476443, 27315.194124446563, 84591.23298873617, 42603.79996300381, 69933.83624073086, 41478.24575957969, 62656.47171611279]} +{"id": 1373, "vector": [7218.430146807031, 12604.029719034903, 81469.34474641467, 28184.76422837706, 8855.831303602168, 27779.258161716723, 14769.568037426561, 10684.228594400991, 76679.19334627286, 65963.6368846739, 4196.291683392317, 8593.852453327321, 40195.54633386049, 7146.0709775518, 11587.702805504374, 17307.147685322165, 42723.2065853762, 6570.598739992705, 50394.78278321239, 37377.26257330757, 51604.07108288707, 6186.990110724255, 88157.61771799574, 13784.510663673922, 11915.085128573988, 22543.75781862705, 76810.23697934962, 65888.53015774193, 69716.11182187166, 37499.536437178496, 60801.63954194589, 77065.69705177085, 62521.2702892927, 15662.351115754236, 13734.979136629954, 69801.29537851425, 5362.105221125846, 36144.30104051418, 50119.41730629965, 25070.076072289838, 87948.18860664013, 12377.000148703986, 16883.24690288742, 76320.2448924033, 25501.48573936882, 40701.14021500003, 55544.09557237928, 40509.04957121374, 89622.44077034152, 23543.378935859815, 30081.861049963478, 94757.46576246338, 56843.00292850024, 23823.91071493951, 61280.82387033793, 39247.62935204721, 59717.98962524798, 42923.80994400694, 4297.078271218169, 53597.85253648669, 43514.45165070903, 86466.3095237503, 82975.49607324591, 85796.5153881029, 66000.01302768724, 63309.51069532238, 64164.39980020321, 15508.725512275678, 47640.04880803947, 58144.795669585845, 8343.90230785549, 62104.89466945568, 69407.02266099819, 77945.17114861602, 17935.60645407487, 1843.8608379090615, 19475.406595741373, 22838.865347576033, 94252.09385014125, 67258.40168385887, 74895.04704595235, 29181.252898616007, 50109.6568971109, 36119.94510170617, 50838.84877406888, 49638.30163424663, 23899.36256082623, 54161.462471404884, 28786.61539848216, 13429.692958917183, 89091.31054759123, 52360.70405890835, 58471.714055054515, 22980.5927606039, 43620.40267244647, 39036.83558027784, 31102.54444866407, 50096.20146335998, 34404.38166662935, 67709.88919940274, 40501.30164282769, 52270.10490936179, 17133.74578671404, 28633.884097160244, 65319.04777471255, 27216.00942967093, 61204.452511343225, 42452.62353382386, 77915.8623707345, 15251.135493338797, 7395.106347471625, 83786.69332662904, 30831.762024543186, 19298.042097194168, 65030.05847232909, 23265.656558279636, 49235.92795109962, 26840.149405794855, 57134.45525207139, 19234.495279943432, 74469.63762727083, 28904.629204911012, 87089.02925737284, 70034.30623078973, 18397.333234549806, 16740.499988970914, 53758.01296114564, 17256.932136581916]} +{"id": 1414, "vector": [52310.150703281, 23951.191156464814, 71629.69092247158, 96043.5347229317, 86779.42200092744, 94217.1908183299, 94119.15636522225, 29804.121104270387, 5246.607429516182, 32824.78740269622, 20574.932664255386, 77697.83684355048, 96163.14876296112, 55488.43508251928, 13214.253226532492, 47983.21094820861, 28282.22114701675, 59548.20509617794, 26182.04212140621, 37195.768939236485, 43018.65576702277, 73616.50453905198, 4993.523739969152, 69172.80612704338, 44747.073109006094, 88432.66223919239, 53422.06767354265, 90171.25172869215, 77.24900134353786, 48774.1397928747, 17892.45733616449, 3899.706419677107, 22121.6624526555, 43802.70898143621, 1394.252388699835, 53683.338952297185, 69336.01280511032, 95111.48065838251, 17108.483304400666, 45266.870548926185, 77269.02792457442, 34291.70938565518, 73195.9822525565, 59074.54543907444, 64658.91757078607, 4855.893677698564, 5039.491574347643, 26886.904484550443, 51081.06036156017, 33086.889582781274, 74609.7287666919, 99293.99542618857, 48159.48567408475, 38662.44618111582, 96331.73183551087, 60259.9894374399, 46500.62137958978, 46858.99829674324, 1347.9332084887828, 53057.379846441785, 73817.11343139285, 82680.51410456994, 65800.37725499737, 87148.96098988829, 74176.1714016833, 37183.495762630824, 1813.7632950367188, 63390.34803056416, 27061.707254427314, 29448.30989638264, 90480.90527330857, 80302.72178870297, 43678.895659344664, 30143.522586767947, 13427.604097022084, 96113.8501268283, 98207.15445706992, 8336.98596550918, 53930.79145054971, 67627.63860527711, 57292.05937587114, 57898.12630841082, 2542.34615641693, 72904.40619571175, 85101.09314293723, 6670.95772809807, 63207.40474138384, 35611.69770984083, 20175.696586727197, 86307.47299957549, 66850.22906229687, 83934.00103354835, 75815.10343542765, 47163.69367673557, 35642.87507376987, 27832.04652102084, 22743.84165333494, 77315.1511304488, 38548.214417605406, 55025.30241940563, 37785.859584560436, 29035.060421393864, 13222.606740249354, 19219.801539608095, 93088.53284665945, 50338.94475106216, 5535.710958914264, 76079.17749542303, 5714.832137318937, 69437.62629088524, 33043.9071048201, 98994.0429162836, 97802.97198329755, 34462.57232241938, 80434.97057614205, 79460.51352640304, 6428.712532849401, 1358.0210536028314, 31297.882224581743, 22001.687324723363, 28074.455525164365, 52452.370414936, 16667.00059530388, 16560.04353162357, 24392.78878192922, 80920.45925822278, 73964.65838355313, 89066.0331359026]} +{"id": 898, "vector": [2053.422526531423, 55547.25258591441, 63170.561906233845, 76580.84800936066, 42841.46392390843, 61435.55368026628, 90532.33967727273, 23543.600651959252, 44287.20190506886, 29987.10219157278, 72729.33845955171, 40526.406109004274, 68946.86788761304, 79813.27867787643, 22795.637949238124, 36552.026834231314, 57883.60805728215, 95124.43678257216, 27757.358941030365, 31376.17378373887, 21375.054254826064, 14105.896286233788, 89344.21923440702, 3887.280061370635, 7621.157358769959, 6822.085428689573, 78758.8457875039, 32340.09641248371, 3228.0726749404075, 7334.023396424683, 17116.431184236648, 84541.51578295791, 59359.23655533043, 17103.431125976953, 93163.67993044491, 58804.57284850185, 641.2266563370572, 67173.5566605734, 39572.81172833359, 90474.04061411115, 26627.468260210764, 28908.739913870217, 12094.164773841387, 71920.92588615752, 43089.26307096144, 97040.45292535624, 65925.56789193914, 40827.35421061722, 58213.11175947159, 99501.89465973372, 46242.371418179726, 5820.226466128275, 37135.62665481513, 60148.08078068239, 58759.485906195165, 55854.149768851414, 63125.699879690466, 98609.77606312213, 10752.818740099712, 60350.11247578071, 29775.77116232636, 87182.51399361785, 17253.64526952621, 15130.89614900276, 6030.6921719388565, 55920.86231902293, 54670.363129448306, 7809.420953829782, 19455.35411082716, 64755.905228129486, 60557.599287383666, 81363.31253636786, 93748.04120352666, 10390.57332525911, 34681.17699854828, 63756.98226034005, 27895.561898935706, 6868.144071119764, 75169.37156771359, 50799.90933534733, 23389.450567616655, 22167.800149108407, 35967.74062916315, 29710.346959749244, 12159.737763384072, 53617.89936567646, 43095.788904909205, 24719.812131904127, 54476.8823061405, 30638.795654984464, 67007.68275574711, 30924.055823734365, 39046.84398916134, 77935.5949945571, 58843.047667523904, 74536.1615976776, 98564.19259683258, 24787.47130766763, 86658.35185932781, 98495.73313180335, 7513.0225008614, 89161.52168601216, 61677.51560131471, 38565.966279901484, 44342.468659391656, 9815.200091312005, 38470.24579228927, 84756.15401487153, 30868.420883516857, 33091.288204207805, 34649.06760747791, 75185.00646146084, 78325.64776051116, 67570.13990029036, 23208.990144099884, 36186.81094928316, 52957.6725004146, 52894.77867033387, 35790.25420084838, 43768.913252036975, 59373.28608802629, 17072.787848454172, 16113.182508717595, 45278.16251913651, 20824.800994483816, 79039.17822561086, 32291.153548710237, 80280.73486550304]} +{"id": 190, "vector": [13377.590693793562, 25448.250944536754, 98903.0632282085, 218.38823587086998, 12865.43066835485, 62419.45031072582, 32888.011953826535, 33567.23579047652, 46122.65773465545, 12933.75953422622, 17992.397519231396, 89123.82346929747, 21017.58917128902, 41955.66771616143, 59482.450654494576, 39975.91001345211, 54865.66886415212, 51752.52414983547, 79695.68904073829, 29974.871855798145, 48107.79674175971, 37345.41064921617, 86584.99577419854, 45676.15356759043, 60116.47874473787, 67388.10749955349, 66599.94221664686, 29253.59593024324, 83325.37470886971, 46318.07611458729, 97425.06420612313, 32093.304499078567, 19960.1686464018, 75674.75899825494, 62323.4623679782, 58011.83188513338, 82508.88119251838, 85941.35338041582, 7336.786457128186, 82712.24547081972, 87370.12466187985, 89964.86667189159, 75144.87694807214, 10338.97921235335, 3415.404252075149, 95015.8319374624, 45941.13200045184, 73496.43365066066, 14237.225964240895, 25477.23962089462, 80441.32983405475, 60012.787735365426, 366.0317649384903, 38851.380077694106, 36703.40892143505, 52272.561697937526, 7421.542425605954, 15641.193549606558, 83638.78708251065, 88034.53874281266, 16830.53739256525, 51117.15661839117, 3887.660513372426, 29908.259060634635, 77484.01791062875, 66632.93892893301, 90233.75327747963, 83137.84293559726, 42682.741025625524, 68450.11418448285, 90383.3756128628, 83439.07913778585, 42091.63007224473, 91174.36204990043, 27128.415116415494, 93440.54589432747, 64353.325413513965, 1984.096247059164, 41796.39107626502, 22936.72151659456, 42399.21444455542, 91679.27418589043, 1468.6201581392222, 31607.734849605196, 5908.290603351729, 83973.9220140007, 83688.1052419833, 69689.75761862472, 67124.71693974131, 47300.69677968233, 58261.16399991787, 32551.544210186923, 74824.25280488917, 70173.16578215329, 85872.94683841955, 22765.65190518218, 948.2883699877153, 22089.507714967516, 57861.49943913638, 83389.78043060306, 82547.28110893797, 92354.76985497487, 47780.7312727688, 76199.42161512627, 61121.886067281084, 67060.38237133397, 10899.920560167875, 84979.82298792955, 87301.45929817417, 11081.912195140376, 68729.98234622357, 37909.49725912608, 80581.3930535351, 50689.27444444889, 69979.18396462176, 78880.8590923388, 24074.98132478152, 46385.23918025144, 88550.54977125335, 23489.89692276503, 95257.36544819573, 96337.5192071925, 51513.7332221295, 13585.789776424606, 31531.993541553027, 85888.81002617304, 27012.089433074914, 98642.3030036171]} +{"id": 195, "vector": [60306.03237320676, 71840.39709858362, 53242.45080800408, 90330.60500953464, 90549.60295391112, 62259.08804555409, 94823.24993818316, 17400.93436418544, 64554.46827678004, 58187.5424814437, 81695.80319572258, 537.7110163532927, 81944.70569788698, 90862.20707067783, 75252.43445759892, 31510.67967566645, 97352.9199316358, 74217.79634108933, 97436.75276153102, 27596.396125940915, 84422.3272234784, 45956.57696605042, 70156.57647635892, 23022.69698248217, 52090.10013023605, 60532.93944996495, 44848.22323954794, 74372.90460662967, 66327.97790821102, 28444.435330255546, 53449.02784388643, 61185.533419361695, 4848.815477794721, 76305.3832152868, 65560.32254621784, 1041.8464282189955, 12703.619180121328, 6159.087789729545, 95728.43941495595, 35926.704955775815, 35864.0979527174, 64677.86017636403, 14922.71682427896, 4377.6677238709035, 74087.56300131133, 41426.0236160858, 98587.57061105709, 19195.702939313796, 17318.33703182487, 67801.68567467578, 82013.06674274248, 56267.97600080044, 76138.57946400382, 44363.574840959875, 43561.671323184324, 4837.019596534631, 22801.901883568953, 91602.55533849722, 3517.0161555751365, 18665.28338282203, 96291.2922949029, 89491.44006599607, 47606.41363068486, 3535.7771244762694, 24768.765753544918, 9833.911663198269, 92068.79219101346, 98366.0210207515, 54469.748266457704, 37262.58328726932, 97556.39477374426, 54110.65534921673, 13914.188700133634, 40793.07518537728, 60897.53544739993, 16595.599087475333, 66733.71127615488, 6334.827838951662, 65142.37300908863, 1841.456302006006, 25851.506152865, 98629.98329587479, 11677.83138626991, 63257.069387406074, 87813.7470347693, 85895.86688449726, 15842.940648562986, 75502.70013130583, 61484.73840876436, 12119.599774440738, 39239.48886469011, 60192.56535516943, 74244.04569791311, 50092.83035189478, 41418.67088153446, 88829.07120247843, 29095.495934257375, 62941.486458593245, 30583.18350089404, 65045.63116506431, 84353.11129897377, 61241.27802771312, 52194.4339070036, 45362.38927797469, 70447.5265036163, 11589.012386547425, 47997.6099068974, 9189.001709602484, 86061.66772815127, 66884.63964499414, 19676.55093407514, 75549.06008061577, 63012.6914089979, 29210.960767794557, 52678.14593995342, 59193.001641577284, 57958.77374340953, 33795.82787908892, 78650.18197655011, 30583.825167220934, 90520.76213363439, 57301.0388197081, 48708.48051320348, 96455.55623424337, 8043.060440854865, 41878.19553820184, 48083.46413591782, 23812.863188836196]} +{"id": 866, "vector": [78921.49748667153, 44237.332077350744, 40544.247258382406, 89287.4450763056, 47326.589132550165, 48239.68615156505, 23592.614800111154, 28957.395587380197, 5207.418316838508, 96100.65086609493, 82245.8515804704, 65236.15389593225, 63741.070997542345, 71041.61169187131, 2742.979916477728, 59585.59239362118, 64795.77842000857, 554.9123255022437, 87842.84793967189, 88765.67055696019, 53145.36291667951, 80088.09040078668, 19121.108847785985, 81964.38672005151, 96675.97238929385, 64395.82499686408, 4655.726447269748, 79353.00746531831, 47689.02586330815, 59962.96769496691, 27295.59491863538, 87404.41581178182, 15819.272379957405, 63979.16020122841, 90456.78389916338, 83915.76681557138, 72471.74814858474, 59071.0565016345, 28933.09939538833, 63454.854118078765, 30081.088826226864, 9042.783952716793, 7889.511876522614, 7245.77862336141, 95149.04964334819, 15466.063257205353, 97492.25051454779, 17569.36238645371, 52851.29091380098, 80229.8181349642, 69657.96002756504, 33500.609367208664, 3251.9794111294577, 70909.6802724902, 91548.94874896422, 15003.118593740694, 30138.82727008338, 39407.27275664876, 25902.247406157232, 9162.488474578067, 34787.9289165675, 18285.97400288684, 72502.63948899222, 10104.623226323594, 27705.54540064678, 82634.42882848099, 7698.203725765207, 83553.67262051482, 40155.68512410245, 13511.299318611469, 28284.66059721121, 58290.964728718354, 3702.8110715879748, 870.8231094168739, 99864.70943378846, 96902.57029318353, 23170.737744974835, 9860.415284465174, 55700.965596752416, 40811.975839634164, 22655.352176491517, 43160.290600231, 56383.36998576312, 98571.70737501135, 97633.75714563763, 1497.7036277960653, 93835.40362629479, 3165.887177131943, 20549.480163276534, 73308.68140588311, 66159.54208323176, 91065.46126032095, 68151.38377389556, 543.184788641804, 19790.029357052663, 49091.22948589321, 74840.20924450652, 68167.9464420007, 89612.88875527092, 23080.849197781972, 17866.224286322507, 19415.98187991723, 8121.189261713502, 61495.374826497275, 874.8620830289489, 12921.925257107192, 99005.07098435867, 77238.52547718542, 80947.5967586166, 67418.77802129008, 63474.17149789304, 64116.495392944686, 65038.70193471767, 36100.02054013046, 3373.1229127240804, 79860.36174262805, 99240.44430927135, 16552.949561290287, 93760.26452591817, 66935.89356107601, 39419.31873620093, 61457.307524002106, 33811.85849269869, 8269.036498630456, 60132.92862805073, 959.277904167688, 87440.9359896249, 86676.61154604027]} +{"id": 1249, "vector": [65551.02582816823, 65305.95478055258, 62373.18028362013, 46209.420692578606, 20475.47894320353, 76350.64174696367, 34631.44711800024, 44074.47582400945, 51913.251440283384, 34675.26672613095, 58734.78519790836, 70244.42804590435, 45036.76187065565, 87664.70645960272, 84761.40706815278, 14028.658420034933, 44841.47003455429, 72761.15312035632, 21731.44220382679, 65466.46769336202, 39139.75480581997, 24732.54456436449, 8375.004922585806, 48472.222429199406, 76660.05142617047, 18965.500955250158, 59223.25507707877, 85031.11582276803, 82694.16857594403, 46282.98377126688, 37675.971069385516, 24737.26665959517, 7848.907614124933, 42.1166136388984, 5218.392951035433, 90104.62830380698, 15541.675451390014, 76637.3928049535, 1618.4610390903908, 35989.03400037442, 85235.1876925696, 52549.85345118367, 9018.928913550151, 25161.557776612786, 46082.39020445819, 35838.45890296913, 13827.112633489969, 3732.298992268068, 10011.98812868781, 39618.50494007342, 94007.45003623828, 87243.85684667267, 13770.833712295394, 68500.50971657707, 64790.3287836858, 6872.021342722567, 29494.373381221205, 46457.63883675945, 98070.04078241443, 38868.43779187703, 82021.65264557392, 11491.325794473318, 68424.38185583141, 1725.5428263668994, 29648.086973910416, 58529.67274098762, 86243.36107799596, 34327.657555434256, 14509.586390928385, 47340.489810976316, 7242.51800237441, 49215.0588225284, 30780.019233488474, 61710.36222187071, 76908.84000794544, 90740.78663351282, 22011.001405871844, 80015.47623348734, 12509.335298336111, 57807.314379986565, 91596.6515866803, 57044.47672436662, 33412.907847961505, 37786.17133395924, 70255.39068238297, 82931.21157300756, 45733.99985917872, 95816.6517931904, 10048.529144475637, 36713.767471885585, 33927.096799380095, 45391.242018579214, 13097.522300475017, 97251.58906085645, 47670.80305701899, 32338.849063271813, 89177.52712799833, 61365.33703287954, 32474.740224424593, 32546.337302903306, 48998.7420907552, 23190.002423700607, 53888.9936508672, 37966.306501765335, 75352.07696247852, 96784.86116184307, 6755.405324781627, 61532.99978892249, 34904.051961404024, 29541.485667298795, 99032.45080406235, 62498.678512221784, 76985.35206026418, 1945.061866507869, 10999.181772236454, 59930.34931899443, 61559.02918545709, 49076.17632774601, 15652.768019556985, 36806.09100892114, 12022.595331259066, 47654.124230860274, 17021.301998263196, 73518.1689934994, 67740.098152297, 3412.5282784364995, 18130.271573560243, 35035.048117810984]} +{"id": 590, "vector": [42910.85263778592, 21848.269965810807, 91464.9098040276, 72678.96827388214, 18664.283027606376, 76977.4425890895, 82024.34790726835, 29647.61125425882, 41181.249652949184, 41353.292482084034, 37078.70330168355, 22921.06317984648, 94385.77472984795, 37043.75613041463, 703.5098878300449, 92016.33518742047, 45423.8616587587, 40029.287299502714, 85776.06255414952, 20999.003698424578, 7530.919252321, 82678.65996421286, 3872.5269337105296, 12206.073572174702, 36210.287738342304, 31693.872515160692, 98198.03758857062, 87568.80529579551, 42445.28533257032, 19614.359756313104, 36166.8800675481, 25193.654127016827, 55411.46305508293, 77421.3364490991, 98571.4657267318, 53984.27747487585, 30791.146746546605, 56479.800011881445, 83406.56192136058, 17783.302564538473, 85005.37442608875, 73157.45810362336, 49122.82065151741, 30408.549641927995, 94477.6570537773, 68533.85079070475, 70483.94887529119, 6040.881864073577, 10390.640516526639, 34465.736483734836, 98273.5659107734, 75227.77654655531, 54349.80096187019, 68724.03438922866, 19639.67168826235, 48249.56975146861, 22277.4006143466, 96620.644019781, 9306.217297161224, 65927.54325418043, 23751.401544188855, 73872.36535776075, 82070.06352279459, 17290.414991092017, 4405.349247894929, 1962.654429283428, 99909.85620578035, 8009.267340635828, 92496.9430715673, 80818.76090797516, 50516.393707637464, 97019.66598214938, 39189.846805501285, 10456.541890403092, 58187.591588274234, 20169.753567792326, 36531.292608504984, 12450.126995422572, 27165.56976773514, 12539.963382961805, 65210.24775391676, 19858.976849691946, 16268.261032431075, 15281.83648316831, 2352.788763892122, 83638.0376311848, 80583.49365000414, 53999.17106427328, 84933.26897521091, 64688.216570097655, 90590.6087916786, 88886.6850974849, 90770.00869349985, 70706.92552784344, 76045.79124571246, 81053.02654721402, 45270.60135096751, 7937.565057448681, 24285.165327792725, 63876.48108374865, 15055.888165177756, 9677.373771054275, 10453.151504460167, 60057.45562427338, 15287.017389977842, 81908.85594696514, 30446.54523213062, 52625.259017973745, 56585.2206757225, 4698.7045138978, 55685.08250456101, 98105.51810389085, 35393.52337537968, 76267.8575801918, 13837.893437773819, 32658.009185142546, 2821.7412748499537, 34408.49627014011, 66180.30587465032, 95719.06790056077, 73635.1532421779, 82286.17624481833, 20500.2969233421, 75696.85640015212, 72240.96697770765, 21627.393845190036, 8418.993894670135, 60033.510269561615]} +{"id": 1086, "vector": [79414.11432325549, 35367.614724828854, 40059.41197107364, 10191.70216507832, 57547.38891558154, 72054.74390925348, 87544.85202190769, 25897.608904334134, 68995.40187587668, 71977.50756748611, 14878.365601160947, 18100.13089861334, 20159.422079939915, 4080.4199540710683, 88783.93476402907, 37500.14350700266, 640.1086628859854, 94096.82214728632, 77467.30250805661, 41211.557895538965, 17854.80276661725, 82626.97122319182, 94111.18974454317, 27313.505035432827, 97239.59150768703, 94133.51467063985, 78181.26747664362, 51114.71541871908, 47812.31276876859, 40448.37341034695, 63843.93838240464, 59296.44422338085, 2762.6443799932067, 70110.12965512204, 9102.509857556828, 37034.73033198631, 72100.75428854612, 29452.867491190173, 18848.419124910386, 77392.62829131902, 18892.52954570969, 41779.04276264009, 39975.468004480594, 33861.87602967441, 29772.77099301704, 76290.27643903489, 13661.835330412208, 75244.5122635658, 92836.50347646282, 30229.392561874247, 13758.290874465274, 99386.58754915847, 99726.86611421754, 55137.54811463486, 79346.11868147537, 14466.30141931048, 27134.616594613846, 34473.80858360929, 79064.03291252167, 2612.382160889781, 70852.4457206314, 38387.05020808788, 24460.482603421362, 70301.9621637443, 31634.25903695264, 30426.536414655402, 38776.782815783605, 84186.10108860681, 31810.400046625942, 46407.851353983046, 62661.8404464688, 26174.105644289313, 8965.145919898354, 92802.00730879037, 68473.11197144058, 52232.78916903327, 69265.03250431357, 98632.62975770513, 56403.341586464376, 72020.74927069954, 38058.05483134305, 13163.490682780775, 20074.508038673368, 31829.69300769545, 97352.11098543083, 54128.64123179526, 37780.82572855113, 22252.69329975351, 44350.77562344309, 4576.003657639505, 26079.90859081489, 93360.00595913365, 19156.47096321409, 18369.656747592722, 89682.02570032724, 64297.58721980756, 66969.19820033779, 20042.74130977284, 44536.36430418964, 55929.74544032569, 11202.943573938073, 23706.56484986786, 82617.40174597921, 53318.79009176379, 25141.745543601413, 35303.29454471002, 67754.03421550391, 80392.46773404792, 14114.187573466086, 61581.74357159689, 37431.10464881527, 73955.8099588088, 60358.49427570622, 57147.76609616598, 697.1401050659742, 72229.72114146597, 67926.66621233552, 7797.314231135566, 5925.556744399363, 44135.16412872477, 20375.124575587368, 17391.837300028557, 62618.86467153298, 31004.830102614178, 94167.75170707972, 43223.146045117624, 72362.39041815943, 44841.729321251514]} +{"id": 1088, "vector": [19926.286301493936, 94654.34213833879, 23257.08660378917, 50997.09062262764, 77016.91107578004, 38208.80464693635, 45985.0384950469, 67670.15707913262, 56924.29167892359, 59975.3450122836, 76857.1035475689, 26651.435364142297, 92545.99292334031, 34066.964986592095, 31448.247381540274, 62874.3621083806, 94403.4919443501, 52509.542662542284, 18743.61054706184, 97660.27108364798, 91831.21189293942, 4168.004261663572, 98493.90483565404, 65723.9475009701, 89265.35384654469, 6638.235857219798, 95633.11348039056, 53446.9890793838, 37352.48089545353, 21832.95503231655, 55321.169358525054, 8256.960669042968, 33182.80625107356, 46315.549736484194, 14435.293884768353, 3427.9291291884006, 73437.68548508998, 14799.856029158798, 41550.15322662712, 87480.63395206127, 29137.333842729797, 26265.18085136893, 74336.24278882, 18287.628589914162, 45469.69340392302, 97439.73966278542, 9612.597967904658, 4918.636774955987, 74397.7058800828, 87029.37211031365, 29293.39441409844, 29905.488254238022, 6923.779534776176, 38156.40856400798, 70599.42215010771, 57989.31986379916, 89811.50986318757, 78017.59674711138, 76591.70673599353, 59699.46209297043, 66461.44512840826, 91031.29351775366, 52221.15963108855, 73087.05328009052, 12468.142698249818, 12290.472378208384, 78630.36867183718, 27912.420542499305, 84066.36763721741, 56549.618605144, 81657.49411442768, 26380.03091919525, 39930.72877168079, 81992.48419518827, 78652.85646178492, 60473.480236201474, 39035.563112636584, 11623.74985531831, 31002.97162376592, 12306.982451935866, 69109.34061221275, 56736.339288209405, 27218.333062532583, 49749.030271004056, 90232.41815300145, 21848.478773166746, 79525.510090249, 3221.6449130772594, 55264.55109798083, 48569.73486232667, 16629.54421257372, 78733.19491753973, 1103.3009467687039, 8552.171559704424, 50609.981902052226, 38969.20485568522, 34392.281884997996, 21350.389054860374, 72511.63921625575, 96383.14723399094, 4751.732107082785, 53849.39020901867, 15452.56684230093, 21257.411943186577, 99022.58505739215, 62712.55271635261, 58329.6384719625, 30644.46452016203, 76471.6211022795, 33468.03293551718, 55971.23171981889, 9587.948893432353, 72333.22245448765, 9480.438156584603, 63795.06802670114, 32956.27189558994, 17943.858863046946, 72249.68741330637, 81333.48028402883, 77694.04846411955, 144.05244009829676, 80073.20378482032, 54509.83283222295, 53213.02517018525, 91689.22944278075, 84387.29985595266, 55273.453323212394, 78268.5610166554]} +{"id": 853, "vector": [55065.4944618459, 71682.9617138658, 39435.07263852629, 75516.03401611482, 15072.030308222362, 86380.90743703919, 68040.05708129807, 20942.199945344175, 89135.3976301375, 90714.78514061142, 61455.36514589473, 40910.1924418178, 66600.66720498772, 42939.78570316903, 72446.97157016904, 90072.74464161386, 24036.360063762673, 22993.875749596147, 88683.13104852186, 93693.70209387886, 69032.1591645854, 27568.9834935942, 88841.75525168654, 2584.0148666719356, 85597.06095537513, 52580.05240318543, 94447.63242806013, 43105.06803846097, 36650.051917541416, 49146.139980277345, 35771.233973331786, 10838.75739514366, 5752.257438189745, 91810.03191594723, 22269.569395492937, 2047.7829626253663, 77156.31369132503, 2564.130030962786, 90830.90146492721, 65109.309682826344, 40082.01349199417, 96451.61330361325, 64166.98045511801, 53719.15615788091, 72434.37794034176, 3846.619512684446, 63551.77466442465, 46023.14039924553, 98525.99183446435, 42074.61911157287, 93502.41833799036, 52613.51075034146, 73324.2401261817, 81941.67499479202, 86431.40854626504, 33763.86105713713, 84090.8716237826, 32325.93129744151, 14059.476921776404, 55638.649773655656, 26814.199574789487, 85679.2808352092, 14406.445671726287, 70357.52477023697, 59708.75963065147, 4943.69318772876, 95514.61849304347, 92980.92427273683, 51031.44633317791, 40072.45738545223, 88313.99100068242, 62289.61278298077, 76344.48327382068, 59481.24716925356, 40936.71543462865, 49303.836824944156, 42383.77809792759, 55686.098058150346, 43639.325763684974, 1203.1601989461738, 5016.226541634561, 34028.2135312903, 10568.782881563831, 80068.16018550792, 2663.3393652115255, 75897.30480846323, 44335.041521943385, 24719.622900375492, 65188.67295562442, 20271.87977959086, 47228.10165830492, 10978.670022407867, 29638.00520411203, 9886.256099160784, 50759.48573435215, 65344.541208864684, 91807.80535255844, 89460.07727531712, 54635.62511725385, 94957.35989619161, 22446.494922680016, 95166.83812068657, 76837.38159451746, 438.77901055932034, 86143.16853072002, 68659.59391822183, 27801.80837079674, 67879.61648891638, 95887.83087236754, 74154.35036226411, 99869.83761575494, 36909.29921595714, 73717.49211263513, 85229.8605766319, 2462.036984860438, 77179.16750920162, 55750.17245296824, 39563.76430992129, 28153.570925200987, 21403.602697147162, 78970.71972259402, 3870.8451384697582, 9507.632087018514, 55569.766310562416, 10930.69275187618, 98449.51412890997, 35257.06923412182, 29998.90630469979]} +{"id": 290, "vector": [80772.01536635491, 81638.19031275366, 99922.78575151881, 75685.75093775232, 62993.70706421082, 52430.78697261586, 60573.58445612395, 81493.59597704533, 29941.88973865999, 39559.93559319802, 16534.93765217241, 81301.68766250541, 58955.91199021946, 61453.77212860459, 41702.83769938412, 78852.1840810207, 33660.62779866994, 60244.716365534376, 46807.87096839674, 8779.408318258686, 31764.18562828749, 97487.90194092589, 76001.83829567273, 14091.329090115456, 83992.91169518423, 81126.01200312364, 5535.831105317168, 63795.76032750124, 16225.732368028888, 30897.745147787948, 84595.11603820746, 93131.95919418124, 42364.85506698818, 91523.00965187077, 60622.564894950025, 57869.90655359034, 52965.72624256084, 94144.47842361008, 22338.49238026955, 43167.89391082308, 46282.89042077063, 93080.70632435205, 68468.30567224097, 12920.684989283338, 40899.70047849258, 16622.751936912005, 34318.126111637495, 38585.42532569045, 69517.19515175058, 13290.440689244388, 19181.812567451074, 25378.055857242787, 5202.666637261233, 19192.257096523925, 71575.28737946993, 86985.47127240802, 46907.287430916374, 88738.56504248464, 67543.20253447858, 26169.98221891125, 99754.64742272088, 88991.757074803, 17676.282596565117, 18303.641809657478, 34595.417561801965, 42754.719679664624, 89340.76269415281, 37385.17195602248, 81241.89909509507, 68011.37289061934, 87802.77886328305, 69559.45372687005, 2064.838861883178, 43343.691306691726, 10317.977562579428, 55048.47577272922, 77243.34069521795, 43160.49059098134, 34200.412472028744, 73588.11283944345, 95952.45812349154, 79807.2601362099, 41903.600001749706, 51431.19431571887, 57136.95984329722, 9383.354867731996, 33025.44279269622, 77969.1828761828, 36832.5577148339, 54560.677991407894, 76613.6715919724, 96726.45847213, 4587.903284980799, 36942.84834889454, 40764.6793835171, 73431.01429172077, 32612.23411190396, 37075.30856395018, 3159.8846819646533, 27901.472427235185, 73690.21002472978, 98236.9771287349, 11290.061227322467, 3880.454345799278, 59607.68291797882, 65865.61675813187, 23902.24630228345, 60325.2480130948, 58713.71042333845, 2276.4638962211147, 14697.788850956738, 98356.05627906426, 32818.742613488794, 87022.12519427044, 14514.657112960138, 53749.7773802514, 90795.99656873217, 32686.348704819568, 75170.33515892956, 10969.450593338237, 30217.384636305498, 98240.48162756892, 85342.69258595555, 7772.766047365165, 31923.4847278571, 26427.512060675108, 90889.43928823542, 97528.36935520548]} +{"id": 1400, "vector": [11933.574304635242, 36472.49654189326, 33348.013237546504, 72552.41439390766, 44851.13778745741, 91128.61540768265, 71126.02746513774, 10678.850831394426, 82549.27554509562, 25342.203770194094, 84732.11352261223, 26000.7487114276, 4933.546946927791, 33703.01060057629, 99338.69664484206, 75913.80290505798, 49493.86716467218, 83008.9353642145, 94366.45789660142, 86177.03338150714, 90894.03805401681, 20417.05691523461, 97104.31633369162, 16778.492129959188, 44913.74741420371, 79776.32494414237, 14166.73139310677, 3472.952728120704, 21078.58142619665, 41641.41804872967, 67680.93182811765, 15803.585666644549, 60829.679388157165, 55541.44417952308, 62730.86114704275, 64211.67575862563, 69406.0439392554, 26456.928228598954, 31397.856693207304, 47672.52884724316, 69876.51490675389, 31619.97875258361, 5294.229513448112, 41336.06362383766, 91486.1671524692, 18432.36676883445, 21173.24554839991, 24091.554705398867, 41220.160327854406, 49861.2424223484, 71622.20222390587, 59271.17937683302, 60601.8707412525, 94234.72751692712, 22817.87961554048, 81646.3709234062, 21495.148897979467, 30281.57435949057, 24879.494735895736, 21503.687390007864, 14127.710963893493, 71432.54023723638, 14839.112136783882, 62970.29881965269, 86168.07161190177, 60196.87372181375, 88368.105112653, 63324.36768197006, 83218.91994915107, 39357.34324905974, 90827.523515588, 70432.84717386703, 13026.335803519107, 52539.35703265029, 89326.9762371341, 44552.82038268084, 68689.47997552963, 14018.177647761331, 57220.27480889402, 12339.06649502582, 30319.00622391429, 29393.64600704448, 85229.2011883223, 57895.54711889391, 2347.639661615641, 1521.9620882756412, 74661.93173581322, 52969.22816885812, 58710.94538561894, 77887.09703679054, 57089.572499376765, 34264.50014131992, 88954.87991509016, 70819.41038859606, 81363.97125935166, 16941.25973872055, 22425.916061875163, 32247.11529121903, 10267.520997827372, 32169.38891603215, 17733.971229251732, 49843.9415394123, 87327.49292491486, 46669.865228785864, 78715.17260883379, 23899.728802804733, 77971.78895391355, 50459.600634849165, 86348.22012967321, 86408.81049620945, 29676.11266957676, 85921.45754039881, 71505.30424893391, 36756.6714789612, 9421.397432296675, 53887.75712483228, 31443.390825332073, 12009.20779306588, 22664.559710712863, 36763.86131698884, 75965.1136562535, 89858.09321523414, 9906.33518481392, 17584.36068493664, 9685.973676498494, 49657.41101624192, 99794.7589281941, 55097.16328433603]} +{"id": 1726, "vector": [76795.7028315272, 63823.974073820034, 2150.8247180434983, 28232.802953086244, 41131.43635850435, 96432.83689913613, 33164.947249757606, 45910.31394270589, 65270.33378812571, 49235.53186085961, 16311.85613112205, 88186.4325034894, 44788.22983073885, 14408.83706937859, 86446.61464318755, 41777.78131281543, 59502.52244508869, 11887.286617014835, 1476.3594562491655, 58903.70064123882, 50003.05146658845, 82723.75446632401, 82508.30258863974, 43217.636651152505, 11426.252044254903, 79146.27449231711, 82608.78139576115, 32396.415314450733, 72449.1400453095, 6995.617597359472, 40274.49935159736, 67200.2933463156, 57126.23496836237, 75103.87127330023, 93972.763130421, 62867.661242810966, 32768.652505488084, 36589.632682817166, 23105.371726383462, 75006.9849449473, 95951.1691166369, 80938.5808168599, 18285.046546744477, 72571.93422314299, 34355.16189424127, 13241.953101161153, 92525.5303197532, 88746.75349465164, 42216.46520234681, 34173.55388706743, 47970.264184292166, 54378.08175745868, 31615.36274941107, 68641.79739785037, 97975.10121424215, 20226.4979472387, 302.15898546599453, 40457.77724281418, 52516.40857260429, 96690.89342782485, 94390.89483606993, 19640.634906979925, 21046.307753252157, 8592.371964658918, 49913.34449496685, 76494.87233727777, 79245.8236091784, 84556.25435824586, 74026.32477765287, 68321.89931307103, 15565.304018510295, 70856.7402904442, 41140.55146429677, 50515.91287317688, 17713.988148383574, 65274.80314463579, 52482.30390418235, 78581.13829593046, 49445.54266705621, 52188.32562696779, 53660.69787293594, 93221.62911385637, 12937.176862439603, 61119.55199082803, 33080.59902040119, 93689.97574125086, 84290.16414326215, 43951.84464632513, 22114.7079714487, 34689.74512984629, 60920.501883967685, 34593.08054959628, 11510.265291436983, 6705.678994156916, 44026.185574454066, 46470.498098152726, 76368.60680270192, 40881.040190877335, 68493.17674476965, 49265.379062329375, 19449.670570165912, 64594.37336698927, 40279.78069641177, 38727.14777971108, 7530.31752618718, 86319.13154549489, 29029.664101347196, 59185.0567965194, 91457.39201844613, 9461.118913573551, 3225.132972947675, 15006.417269157968, 66022.26518054142, 61765.26728249982, 2081.9495311774626, 40748.41050511949, 47923.5260868219, 43968.74042711751, 51748.18530123561, 77302.39060728773, 9146.658907116578, 12953.89811844977, 35377.025247892816, 1879.1286363102788, 11219.791473452411, 81370.6161539104, 72497.82683641736, 12382.218358077691]} +{"id": 367, "vector": [89407.45658180641, 67126.9948528328, 26166.463689878983, 53535.7549527147, 31298.356782072402, 80656.42867893478, 75478.65050352285, 21620.113434278166, 20304.6646888841, 90077.7441754392, 93943.10456828715, 31033.24297760237, 40393.43767222413, 40834.77366486321, 73627.84398207006, 94403.35866531613, 22935.612890091284, 33569.08878084891, 11573.94099559349, 77505.0425344229, 61660.647007288186, 34952.84132941925, 94262.91282552354, 65313.75566687318, 70351.60262841971, 10724.223734392946, 49627.71333982098, 77189.45866682482, 35390.22243740668, 74488.88332695897, 58888.285219639925, 97716.32868346407, 96167.28746885005, 56382.23790657172, 1396.8860025313834, 81088.07347492762, 97419.76873978306, 16737.030073529113, 84387.52600955665, 23062.254574641327, 86665.65298780163, 70923.61153645476, 22835.263655010072, 47549.30928322509, 18559.377257549604, 36656.75851047555, 276.83642176332677, 72095.23096867531, 96343.50318441636, 12539.372659995928, 86060.86156082571, 78458.09048648774, 69485.66063969256, 54164.203928462484, 93077.04939394706, 67840.15782096436, 49683.01144771392, 6473.3726063297745, 51733.7969262311, 53426.86244313266, 63156.33456400057, 18647.13170343162, 72070.45382633107, 40046.78691215113, 75934.72482879007, 34403.98634052543, 15785.155860826395, 83108.14384082268, 58052.790494504115, 79303.94667270972, 17880.270807052868, 64694.74925299493, 39488.63730535864, 77752.65485180538, 63224.87415564853, 31167.960594422395, 38119.99315632212, 14253.619770939018, 63270.64604900915, 38114.54980479175, 93414.61955671741, 16995.769807940786, 8289.633426408083, 509.6908090842067, 2158.580760016815, 85519.28293233391, 7069.020655997127, 21968.020691776568, 22274.427614882698, 35651.85805657183, 14703.832990528275, 84310.82893006924, 89022.85376529401, 45413.79246795726, 23733.75399177503, 14611.564600409421, 3264.826739333471, 23263.40598633737, 16701.510272337528, 56004.22712046722, 52765.399715657135, 23338.313748731387, 20065.479878867565, 75483.33623482338, 24922.50431970705, 1962.3622031272769, 1682.1544690547973, 60102.22834698439, 93606.5841760544, 22423.372434841494, 1964.4991165493009, 36451.83133518599, 4320.360049809424, 74360.15003089678, 83562.95489968866, 22811.53457053502, 47332.7756304233, 55685.222602141796, 21179.100279400063, 27908.302982281984, 58082.0680949868, 87420.20857275928, 40338.10848512701, 90747.26303755889, 93408.71884719474, 69064.46466142446, 28527.482213775547, 17943.066449982736]} +{"id": 873, "vector": [96706.78593844501, 71302.71291106973, 64857.71065416421, 97108.97192618005, 82574.7255312811, 97101.05183309458, 15685.428554125536, 97798.60361220455, 43710.04911530737, 5598.989655120335, 78371.57995748587, 98763.55034282562, 22826.8188969274, 33870.03642132278, 36626.12480570994, 53024.87973311621, 96656.38313899892, 4602.128113708371, 77440.00040555265, 45122.80836778275, 64390.35609788362, 41347.73881886259, 4748.328969468674, 37977.978453052594, 90832.49913068904, 65228.2191103114, 44886.284415456255, 3526.2901072200757, 46189.572118932454, 35049.11766750905, 33932.19222388343, 92644.40678199664, 45937.251696122796, 22759.673225327006, 18424.502523879328, 2246.5019975863474, 83728.29874020084, 18345.055218955797, 62051.89885088589, 86339.33133845736, 1645.0249507491853, 44012.00699060732, 77447.07181340025, 71688.50078026869, 11823.030907824483, 75666.29869337074, 57834.24712846942, 66363.80770508376, 22719.878523366788, 99973.02929427905, 76661.78019208342, 75848.9815223038, 25002.558585318347, 11465.139412874503, 10736.7297220762, 58507.34158112798, 648.7499896730808, 74618.89673312475, 58154.02898535622, 17196.49018752475, 12266.693091349112, 32259.640648388577, 55578.603934223844, 36843.63523515718, 63451.7539333335, 59527.400145490705, 93525.0026509987, 25358.696207305766, 68847.2137137792, 24809.814800680473, 67725.99843855736, 2724.081767961617, 20022.03374360292, 92858.73245439869, 82279.08301584254, 15208.290116802704, 25054.46993038669, 53897.35873175697, 86196.55146234899, 36795.00016326904, 85462.91822643875, 20394.284793189876, 26756.846219352472, 29627.360678628665, 84014.13125020528, 13505.512544482955, 90714.21145051229, 71377.05298553515, 81223.0546687274, 48455.59654197931, 70738.74585557755, 11137.848887908585, 67197.73321660617, 72611.50198121602, 74200.17631646532, 48451.327347247134, 6370.772399911184, 83856.98044488898, 18154.978183224157, 148.3578232366889, 98016.22863306338, 71768.14244365509, 90706.15319363229, 81628.04696371019, 99657.19487209455, 60695.74589709338, 27422.915765576618, 42011.50981199401, 50057.190926235264, 31217.773787122984, 10288.158405834658, 16788.88581248398, 39689.22948471025, 85274.00287812564, 62962.28082857099, 18709.00684457346, 4137.15531470753, 31777.156348368306, 2699.9778355626836, 77514.01083229312, 56861.45541436857, 1437.7729018691166, 73153.12567550912, 40028.17153603017, 786.0406941601839, 25352.10831772281, 28675.754198740044, 84499.07731924146]} +{"id": 745, "vector": [38837.92956098898, 66814.44187852627, 53019.70774481149, 24082.5413486787, 31656.87985221557, 81679.08169280353, 92883.35500411724, 37730.514249642765, 14224.605148426828, 58721.68749407888, 27503.36112315359, 59210.67837040914, 53668.971271266775, 6365.271461558186, 35353.91002615934, 88239.13875781606, 36530.28463312352, 6777.979379570842, 41158.81151756052, 82201.03368311617, 84340.38425544006, 53223.66999486704, 76111.10596250264, 56574.4509602615, 8223.651204461257, 33617.085226040246, 65094.136545327674, 11238.212777430645, 63234.6826725637, 74093.16989916957, 61944.32340130604, 73196.96183396451, 45005.178728274666, 88341.74430465169, 3735.7033137023323, 2951.9689022552975, 52234.25525736777, 35962.34533658781, 35385.314776630235, 62185.1098932168, 90431.33075361927, 78076.4985919305, 91266.88939647253, 1878.5438694060463, 84302.26887418873, 78833.10278211578, 86990.14297175083, 90364.81460400828, 41143.39281019187, 19594.861816123765, 15614.585025334105, 48937.59837099945, 75938.10487821233, 20231.067564421268, 85888.8900416765, 63789.88537490326, 26875.609080475206, 43418.02489464734, 85779.78050947066, 41861.41055589138, 69062.89691917642, 48579.9933321416, 15158.69462179229, 81590.38745246915, 89379.66511202387, 48520.66192584006, 99981.32969999549, 2949.0367334258694, 85894.09414781921, 15873.583266048552, 60342.0907821413, 95309.68111316278, 11224.91519352129, 79707.33955924268, 5704.2038524369955, 46771.68908498912, 72099.74365518198, 39490.9687169538, 32701.653911983074, 39679.79981248931, 40878.43023689793, 84807.52072092313, 42419.78523312211, 55222.577611308, 81912.3121058406, 13333.79223175334, 99678.64718838612, 91717.90195310659, 4080.589519619593, 31237.549577828995, 52185.374648834826, 31930.430749353887, 31736.314021160284, 54792.56800844996, 90723.10551230298, 12881.760145024778, 31186.34690622585, 62059.70860671078, 90169.1661992788, 24109.21934059418, 8260.90190954163, 72800.85885872428, 34465.04547810878, 84419.14766260111, 57415.56413252214, 77335.33864421019, 25786.21078969492, 80826.78605617047, 13007.165430030087, 74551.92248753189, 21979.962089946992, 96973.47013603062, 3140.89376602118, 11297.895884336594, 75454.76451212676, 30201.163672793384, 70520.28523400017, 117.32594121847129, 6713.469080201507, 65694.19417294738, 2309.664298720082, 10693.486814864373, 50777.70331041227, 45248.9226856467, 74804.74961294915, 68322.97810659056, 51357.24913482666, 84146.35858113362]} +{"id": 266, "vector": [23573.934634791403, 72340.36301047864, 60990.945202041934, 78350.97521226911, 86025.35460721068, 18060.16337620925, 45445.618729524394, 33764.200850343805, 79040.37790336736, 38226.21799936487, 68139.29270306177, 8990.198553174556, 73440.2135197692, 70631.76590191029, 18432.191168004254, 6637.986194506762, 18874.707501817888, 31058.38570209599, 79432.45269603156, 83272.8398687441, 22082.614582868177, 4969.44318752891, 91857.2123182838, 576.005041642702, 46146.06441920247, 78787.48737903357, 65331.271287414114, 94998.9457203223, 60640.457494523456, 7333.305556429237, 21848.777263123196, 98078.78353599389, 61803.50239788288, 3503.9634139577893, 2611.4815970014747, 60642.146685703505, 8261.396418759181, 74054.5733067366, 60915.1270792674, 51041.73673627581, 77431.10263720156, 25161.7966920459, 2553.4352979807663, 29941.965384404233, 99612.29637304728, 51508.453371207506, 52391.60742712284, 25445.139064327115, 78213.17646566754, 98533.94602911266, 22825.189437155135, 90405.60070900888, 89517.1902714018, 36457.37715288542, 2857.8000726459995, 37186.91985552273, 88281.01479435882, 65886.68061314653, 6998.607463688389, 22392.619718576658, 42674.99609823295, 92594.39732384494, 72418.24633380953, 1200.4428013862034, 33534.34335709021, 13528.498959267443, 47086.82359207467, 81238.7878504529, 29229.587101555753, 37359.3595542037, 88972.33780049911, 89738.24934733419, 2175.202579099922, 46294.79746727767, 93381.83367130319, 34782.304141417, 63236.329250464, 99963.6889908465, 58169.77186503734, 15173.834820126742, 84639.79353495096, 40898.97545802737, 79243.52933107912, 75314.83971795772, 84988.1565461013, 58550.0714786704, 54522.55423814315, 91978.081261943, 44009.055492806416, 23611.16546745983, 71908.73541216223, 40115.37936066374, 2064.3319494762836, 79759.71981075809, 80590.03717966712, 84062.10761044033, 52416.305153499095, 21431.70787044425, 51613.18366023865, 29098.873073737897, 18873.45188354389, 11746.703880617826, 37632.87373046153, 8859.104887004498, 6414.106659697538, 33049.68464026905, 38204.356977902054, 52788.76403841639, 13596.294789028228, 6977.190702352965, 35314.55536524878, 26115.681614326902, 29069.42486589773, 86578.13107309573, 4483.320674156422, 34519.70333721437, 89068.91815968874, 78258.92650661082, 54788.87817377321, 52773.89072745805, 72601.37930242444, 12431.763910936434, 52010.38937223279, 4449.825874015345, 48097.58914039781, 95140.76009831384, 2999.89441208951, 32974.54029343579]} +{"id": 429, "vector": [65732.80682367657, 76586.84909802294, 33487.610016233186, 39196.62065891407, 62252.57215719772, 26075.832828224677, 4063.1103396289504, 57414.93451522618, 87744.85932814727, 30593.516085140083, 69604.08246233685, 6829.786899310852, 28315.98535536228, 28456.278983267446, 22910.694552641166, 85506.48257139807, 9958.145808690477, 83586.43878209175, 55942.32914539279, 18685.449052898693, 73872.89893324679, 63138.35387202018, 49613.35238678859, 98991.37445537541, 39907.23900957025, 90754.89655478792, 76751.19147423269, 60638.80668878488, 45529.42678968147, 13182.682676041346, 57819.73126483912, 81381.87101380616, 11128.706665631738, 7845.87544065789, 52964.10559050213, 93744.03130784366, 33430.51272331109, 90523.4304417712, 62543.702003589075, 76783.01655867437, 46097.14375208118, 76188.3280476484, 61228.424443805794, 29064.62737540453, 67527.29614421776, 32806.73905074095, 80713.1754463792, 35604.298787414635, 43473.33925829177, 58345.13271350211, 65556.04579332468, 94747.54886107346, 49720.105102476286, 35267.729273666504, 17496.08491567696, 52120.070264675065, 93868.00120084238, 93550.3220567655, 70361.70340899909, 47058.33093590105, 22101.972045913142, 29337.922480320944, 44721.447819233406, 58869.35678757353, 6200.802614058775, 55155.93974006734, 5296.1467411623335, 82084.07696353423, 6122.127963431023, 98248.3946788837, 59175.675144507426, 32175.484705609637, 60444.00407412225, 97397.36810753385, 28346.828419425652, 83002.93455107365, 6731.23229197089, 21810.992664281413, 24107.736285309333, 40102.86425093437, 86.37849338889447, 8563.600619305234, 14905.676295531712, 13936.326052690385, 47635.003443699155, 32851.1176235066, 94946.34371371055, 91258.22594368961, 19695.220994340634, 72849.49722656074, 98732.6125869262, 27537.237440796336, 25024.519792783452, 71977.68282297249, 81748.28970735458, 18051.88985226862, 82120.84484036564, 71763.21714248222, 27043.76056427352, 5968.392928107125, 14881.916226621084, 47713.2403171303, 95568.65971339858, 60835.02875623044, 32253.173327135064, 56629.103709464835, 98126.98103031653, 38873.80064381347, 74132.63307209856, 84499.86313142952, 42914.11687290603, 30358.368905546053, 62895.54814694832, 78227.77870609985, 2663.981033158469, 77130.8446114808, 32897.306028226056, 5312.844318923837, 81537.78966185692, 74215.43802089605, 72462.84449668437, 43113.431778812526, 80627.65228682019, 52099.46981394986, 75976.1038784266, 86217.06012633216, 83502.60080846402, 37405.224671934564]} +{"id": 1991, "vector": [96651.43562098876, 39084.78630092314, 19102.426501420767, 50158.94027388288, 1386.4930759369677, 54406.53598457998, 29061.161194570428, 35311.1325694341, 23629.874126636252, 54248.57296683517, 76662.8852481461, 6424.424717570288, 5044.995889541692, 18864.640702864675, 52458.90093868343, 60338.95531679657, 89110.4943421475, 5530.441876802616, 73816.97894503865, 66411.24556265923, 8920.959259359584, 56012.47122087104, 14078.297284658669, 74743.8696028807, 55881.59070932519, 67278.39037398885, 9731.16438010595, 31305.68898202083, 20207.775491359804, 67526.58494952644, 59011.8559171805, 55909.29760732715, 68496.77484107234, 57217.365694521184, 85180.3953882937, 68765.49557100647, 42960.42674106686, 51828.49870081363, 45055.60262314712, 68790.05849968904, 83237.79010759197, 19437.973973644384, 6572.845728443688, 91543.94687589601, 24435.694552649024, 70139.28885412346, 58404.32036927058, 67960.57657667945, 81483.95958838625, 85550.75801161659, 98639.43755305457, 72663.1445233081, 26467.856714798665, 81880.3250943181, 71231.92658072493, 79644.43685698556, 52011.36567229563, 75193.77575169387, 97018.94882967355, 63321.25885398477, 1050.2744468294577, 72142.2092033565, 51279.615556456636, 98792.31868904726, 16587.020294304268, 31977.41199210189, 86688.98711388298, 56550.92750904117, 57410.677850394866, 91201.84392318959, 19805.704772969002, 99018.76693923232, 78348.60842473937, 18258.29373210658, 67855.03495352833, 63413.96730612064, 17087.77383196576, 41121.47001992622, 86975.27259879194, 16433.125617265752, 4193.007872113263, 53951.86637752947, 38368.73316670479, 87309.68096630074, 40849.421962828266, 78177.3065671462, 85301.64502370017, 20799.80102977057, 89764.86002491931, 1171.2923851949886, 9390.082449442383, 72171.5049029729, 62113.81725418901, 58412.19970272783, 97079.16921116377, 91650.47437764378, 46330.512945660565, 97792.60372127248, 81832.38869936008, 70216.96710369986, 75049.3774497857, 76336.23100428925, 82664.03085107666, 92719.48998700742, 83014.35111526672, 69489.80708313039, 38889.42324895647, 44236.81538548384, 11920.584379343136, 60330.3003273735, 2735.414798572633, 79553.87100728003, 60959.789943221534, 70809.44314132564, 31997.97205072421, 70073.04690456607, 44684.67953500731, 69117.61982589566, 37113.064143243304, 72323.86355931615, 77593.61766759332, 72432.32870626566, 47011.9091796248, 56110.19070420559, 86655.76718554064, 24613.578404552914, 61221.21061640161, 33279.94078487036]} +{"id": 379, "vector": [51605.389259792355, 10196.810839751048, 70959.63936455928, 37413.23907186563, 20724.700037168266, 56964.72869515052, 25029.65011801235, 50795.377341334315, 17704.305507905694, 83075.82690697622, 86582.44286440157, 50190.27997393111, 80214.55222160472, 54386.68126524389, 97684.06290443098, 33139.82824891043, 4485.757685697001, 75164.97932657889, 3385.4655302870574, 70684.58317230122, 80704.39239167103, 32757.11541214731, 45709.967107245066, 78064.23316208078, 55253.26069975912, 58001.99481747957, 77472.4461232703, 46208.78657630878, 36403.181972274804, 44930.28040745356, 36834.26953650204, 10225.686715271453, 99221.58913695137, 24673.836246849743, 85604.11518409612, 2020.0926686810617, 16542.731155277113, 65892.0820684969, 8504.468357917538, 50390.32415420509, 25979.893792413135, 39923.16354527046, 943.2250269952202, 18493.91258698446, 97829.02828044882, 6416.406164132793, 51737.84092136311, 43441.09783625899, 89635.14158985938, 62632.32314681008, 88195.27232448562, 1487.968310791421, 9162.822179534525, 18669.770812070787, 18951.819462938267, 59724.0685756754, 9956.996747668023, 8577.602559657782, 7107.80956229784, 64369.09279294897, 26132.208013455573, 9925.739554275537, 57169.496273007826, 80297.17380500076, 38576.13947311837, 59400.582076027305, 49879.18210923552, 21229.823210316845, 60196.336158372644, 7423.333349034445, 88927.45151987493, 22968.78475987999, 14556.358405693893, 81156.41003815082, 68002.81363923711, 54677.5185035299, 85585.41723833239, 80335.01192840398, 5632.207665607447, 20631.019911431402, 23958.594301948022, 33984.28240846171, 64316.97689582842, 23197.475119284994, 99897.97950942574, 41924.291822519655, 97531.16808969394, 60146.695448056256, 14180.726165628987, 58764.918952826185, 3567.897944076337, 71871.03650658298, 1481.2217969308538, 48978.458944790706, 79478.48904874346, 81128.051637342, 71156.95242298179, 8533.651259220787, 58402.64117982541, 63252.16552688934, 39561.553696668496, 77162.26532038827, 99258.47168235756, 56889.706074920134, 78228.0973171879, 32327.16149652217, 55204.041908309366, 78364.55796714257, 79125.44010948157, 62603.60327374238, 49523.31058391586, 7818.968210753829, 66136.52072419143, 16781.83256641942, 40280.50210620127, 33539.09538311737, 49339.28517339767, 84.49032632238396, 38644.999447055525, 66600.14315623198, 71426.26040136945, 44350.36283037075, 25007.783440481213, 79640.74613985469, 12.980954605257278, 50679.238391504245, 36335.62211076138, 19712.554878788203]} +{"id": 437, "vector": [50992.504663903026, 25156.510363176156, 42295.09963414081, 37560.58329709752, 66129.38685987088, 56976.871443968965, 1075.62225758856, 72899.57718395135, 52893.507100860515, 18012.070194747575, 45485.65746287088, 8464.539763121049, 89580.70089117554, 72967.98440102428, 12634.00682682425, 59556.93843695144, 49084.30338327055, 6792.720723258639, 93534.11577142548, 66657.30629839552, 75521.21637056502, 34817.87588416706, 4838.984474694641, 94263.89437936012, 55238.4977295614, 77852.8242815822, 46727.30223966522, 31086.371664659153, 66666.0224967921, 55989.05794702559, 96253.88803027147, 49636.66856029576, 90800.45556918596, 73359.03055927661, 39866.428029460585, 95221.62989811006, 63060.72346961934, 57206.261650792134, 8281.014834668098, 61045.01176052674, 67355.8435536594, 36969.200460274784, 74909.14190735895, 71761.00118161064, 70973.85317671478, 46175.388607818735, 93261.98125907833, 63271.9565095092, 91499.7974180078, 19483.642988230353, 21447.25314615711, 38097.16048453296, 93992.24671370815, 35800.907644803236, 87663.40735506969, 67571.92544968547, 9962.389834010477, 25684.98188201759, 92945.43068489435, 10662.308982033564, 19971.38552046004, 48702.60339616832, 19402.326833841045, 26433.22946562523, 21106.255694943677, 19977.544833526717, 67488.01500427388, 69704.97330237075, 96035.44433029504, 31594.60921949314, 5868.576112033841, 32912.68850493698, 71354.06773596823, 23049.56114779092, 33073.629469750034, 54028.986529739384, 58127.485457006325, 60413.211091055644, 3997.808879366438, 63567.955331753765, 95331.29292316953, 87159.77964537696, 25553.6006960579, 78713.03649495344, 51442.80242483124, 46005.85396620562, 24545.847654330675, 37343.873114570604, 1406.5386052236618, 2913.9153992530532, 74715.54575089022, 35082.16482277932, 119.70265424066096, 21139.921089516432, 10919.207733672964, 57477.428936672826, 45845.58223251928, 11797.46061158261, 80091.28519775736, 99847.12875361784, 27903.945923871655, 16823.379218128888, 86607.16971794868, 88344.10863295842, 38942.139987646726, 25972.486560486217, 1461.0261676818227, 45849.18925598082, 36006.90830185555, 83731.05877468085, 334.30368868886393, 74417.3136846611, 661.9931583066174, 50701.794503610174, 56013.772490435455, 51081.104433606684, 44953.957184680694, 53595.10251666336, 67747.44521098454, 2508.027517965983, 16649.390264666774, 76947.60437271972, 77090.97352031193, 8770.39326012814, 57883.048425758934, 5102.591022214265, 27540.84420378905, 6725.331124844402]} +{"id": 161, "vector": [17410.720148925462, 77099.2242317752, 70698.71509818366, 77692.5960332828, 58437.60769035402, 19102.182632108743, 9055.28384769244, 97613.1699916661, 2238.4541778968314, 96756.29835208926, 90453.53609494488, 89904.5666815946, 79262.71203975522, 34115.387293409185, 98215.73728548177, 63930.702082294796, 83368.17775393529, 93333.77272999028, 98040.21140515478, 95989.00633573762, 70313.95956271731, 81510.43634631882, 7872.113776890955, 67350.05291082963, 50275.09700453187, 69685.6864105898, 25612.72338495443, 81170.18901934191, 21428.752185927104, 64041.628597617426, 90237.42944334431, 49995.72791687892, 52702.89490565701, 36707.72524201583, 93485.4445963595, 50131.395548950975, 57776.62228545379, 5020.038377724379, 42243.22321015784, 63129.71466029984, 64778.21518907456, 88153.53830294902, 64565.604498244764, 82551.32264916378, 4132.763328743694, 56530.51463935063, 88687.97399508768, 6082.350298108808, 96155.94475908561, 56608.70045456194, 77516.03485516689, 24634.80169834006, 52428.0254764509, 91321.91995536843, 15802.139452150044, 55120.65055430698, 69270.7962775688, 32390.29828790414, 97989.2740447598, 36590.70946184573, 78269.78544424316, 50132.334501701735, 33839.82988472992, 97543.64033028072, 18633.604969065265, 33372.075007050684, 42955.69547614137, 79663.18084866974, 76687.94957192117, 80509.85908800543, 22231.210562966287, 84930.89168754347, 87581.50043147765, 68129.83439309328, 64787.48402992624, 54498.94277091072, 57878.29777828426, 11319.729650772859, 85046.0468575581, 19409.542255746128, 28149.159500677255, 85165.68327089939, 19734.06885234623, 13643.466269043314, 81357.58034420443, 59448.266668910765, 34586.83863627632, 19373.32787401156, 62415.50830227045, 9727.579007907372, 58156.06984786437, 17666.04987507543, 87571.92310698614, 92770.89997002807, 82304.77595192936, 73527.68750141555, 9536.67344359651, 15376.56025921893, 83300.15708779635, 27769.6156020516, 36646.57039980492, 32270.84724597182, 78508.57088430165, 15592.490444756557, 42747.35505594069, 57138.80067664979, 33260.48966642244, 6123.249637706241, 22033.270403671555, 45990.72721401093, 35768.93355105096, 49934.19113463617, 97724.42831002113, 39500.897139309454, 55196.87068948526, 71987.46759843153, 40576.762085460614, 45624.98393894767, 75613.04828226884, 37155.05317734192, 83775.69536151104, 32081.806523386014, 19068.197973460476, 43468.813746460466, 49360.034987516236, 73869.97182950184, 90757.9298735903, 77129.94057164472]} +{"id": 833, "vector": [25315.951996168373, 11816.144708216181, 90523.3944597384, 17791.09712590764, 61569.2826480749, 62463.46428875464, 94676.2950496543, 42933.95107425921, 30446.364871990372, 99974.90546162834, 57107.0061923633, 68335.77993117165, 21045.258522764365, 43135.440980252315, 69817.80051538414, 23536.4780052274, 60462.34610742235, 30812.653245139874, 55510.55661047289, 65799.09405287147, 19576.266030724288, 93227.68762373649, 67085.45020776149, 86860.02929657638, 82664.10506724784, 14258.996607983832, 38190.295888864755, 80311.40825971593, 82689.85724616231, 6806.946569715166, 6910.378815777318, 67730.93901863878, 81879.56404154579, 14435.113287600954, 2235.3403315174546, 86629.36330167361, 80603.48211267724, 83709.68289143895, 1269.0066726070204, 55749.147577220625, 87424.70360386085, 75153.18921893775, 23898.686636016155, 96942.81055921793, 13165.215087680115, 9386.89373978725, 62611.873371900314, 37744.734987454045, 91410.20266092083, 12476.637630704601, 97573.23123088716, 9143.428248951825, 57136.33510719321, 74267.14368584035, 59966.312926010345, 273.25967227438406, 16032.656142907232, 5104.606110350085, 86411.91839356445, 95499.7326022923, 84750.79507352757, 49362.47869719665, 11520.503876777522, 45609.31654299508, 27575.366787630097, 97267.64560897033, 11974.722384230063, 60542.93141855919, 4385.867851424497, 8263.695174361917, 76886.32698854423, 34360.09390081707, 29103.46870781927, 96824.79341530227, 4782.858038319093, 7174.811186570684, 41540.210303961954, 12110.277365550692, 29781.216611360516, 54592.3943689633, 59498.70236146985, 52614.12459317236, 32321.5992368734, 50782.74217270935, 71876.3616711199, 61618.53595146195, 70623.3829837989, 8297.90956129548, 880.4020691216441, 77024.30307139194, 51639.26788138245, 98054.98815828327, 566.4037380767151, 54077.16738491321, 3052.054390951986, 87167.72054096658, 50832.241755502735, 45274.52099525857, 26901.72895512829, 50446.78207456379, 61822.18760845802, 96519.4582501811, 57501.11244621181, 32682.98148066492, 83387.40815855129, 47880.42523272401, 51645.25612003165, 16167.460666328981, 97127.8858904358, 2951.617755499081, 81317.764003705, 13694.284305610316, 50097.83001952014, 91774.86801119574, 59147.69322596629, 91680.54259658494, 73494.40334897964, 48039.92903240263, 73400.80753950484, 93887.90639470099, 27597.893750232895, 81942.17608365281, 99721.54981875145, 17494.81172030999, 55681.863598206146, 35803.279293496314, 54793.21490864364, 90441.50959060609]} +{"id": 276, "vector": [74182.25093108274, 3421.0776287141866, 51402.295468437274, 59132.1215748222, 72783.25463761527, 7557.576340091843, 28047.383655871872, 53253.988212549484, 40636.220347921124, 54843.06525523745, 36969.01732792462, 69984.47121375895, 44380.8505351973, 84097.63203310048, 23150.43172043665, 94515.28070710509, 53290.86579502237, 26863.14498923751, 25389.328299054214, 19120.941172875628, 23122.268992698282, 83060.47892204084, 1348.4020687415389, 57175.546977229904, 14417.255780128735, 5597.516800835045, 17206.847058483123, 22451.322103499748, 99283.50823901223, 72243.07657715683, 74698.0335462794, 15708.681820590242, 50233.988956428744, 61426.809883713344, 48781.487422157144, 92070.6494230634, 84951.58751025276, 63420.74137454244, 96301.14138533796, 1749.6085012369101, 62493.28626592575, 49993.79890938672, 25543.16754147289, 69726.12399779764, 45561.8548786404, 59129.80942447572, 48413.77247841975, 86390.79925567852, 80206.95127219503, 5945.02698507825, 38749.123162706026, 85467.64084429141, 69260.68115070734, 96416.67973744978, 5339.933948821185, 68408.0543717248, 77441.34934105435, 5068.2961355382195, 79284.77333564589, 82571.93705105987, 15670.0236702775, 49695.97453537443, 5720.887951675202, 6195.765332887071, 85251.5996118346, 6609.922246728294, 14367.57925903187, 94580.84830482338, 17838.575636320187, 16867.269771770676, 24802.43287358053, 57224.75216518205, 70989.99706312288, 48187.58014210077, 25768.611151585807, 21558.846012289512, 10381.011226926417, 3522.34681719823, 21247.304897425067, 31660.435094468998, 45229.29099689153, 97148.20316789746, 1344.3704925948264, 20891.671046864347, 64336.94981791182, 63717.6124887524, 976.8165598482481, 39175.32003497566, 14241.36264752065, 27795.14297989356, 75126.39794249422, 27427.854194274903, 62849.45388429402, 1296.4384364943514, 80341.76718900935, 74641.2071068208, 34168.817138324994, 46208.56528602687, 96728.78990604292, 84440.31286844607, 59771.39442615981, 36820.67505013563, 16930.88667983027, 26339.853945308532, 36307.899747365635, 29091.362085856497, 27323.019514380554, 8070.70997951489, 17852.079662275366, 81021.65001751053, 29225.06013609045, 64122.437709424375, 1703.718915860164, 13976.49635967142, 44968.80881799143, 10635.241505129388, 41266.21669032983, 71921.43246132342, 73275.86448467908, 32816.512239113494, 28013.521850689736, 48662.63903028396, 40549.1992142885, 5243.018326262028, 31854.860602710545, 66514.21663841787, 83516.82603653806, 46503.24503713842]} +{"id": 599, "vector": [39655.77102719962, 23149.767625856133, 33058.47597930258, 21190.57346196336, 96413.74498773756, 41983.64382089806, 62263.10053275946, 31512.104057810775, 31086.03899900637, 85020.13021709849, 39843.625971052046, 73307.68275479894, 89531.04539660417, 85032.2352059019, 12580.40743422465, 41004.629759503776, 99165.65282239176, 25160.651773752106, 29190.625107610802, 72748.97777617573, 94947.14350254176, 75283.23102629109, 37093.51480909731, 80124.7159167926, 59265.83063345079, 66521.58946688611, 85720.82376059494, 26713.42932291162, 75306.61819387671, 74204.39326952162, 28853.461789686342, 18722.533118599426, 9255.989660517062, 98991.14254292325, 11370.169158193356, 49707.346255174125, 56406.96223533771, 72930.50223466854, 7395.394271534761, 30612.342109750578, 70123.67566617035, 63805.251684792776, 1773.1341739555928, 47270.15835507957, 85304.87620718873, 97167.40599082735, 81009.06555151835, 86524.82019387477, 96615.68329175557, 30338.43043388308, 10486.287509381786, 10882.034863668532, 42786.365762245514, 87809.33041416238, 51983.012484883504, 50413.65918585172, 36951.61698119147, 32691.666287762135, 43573.00538485724, 52102.0022280176, 53167.54052223963, 8280.755929819328, 49345.450110150516, 29375.531195850413, 28759.44820496158, 47889.45572352172, 21739.94588614857, 42149.49410837319, 16180.620105208809, 65577.54178854352, 2452.259435786941, 97681.18631365808, 96025.4511449572, 58881.681595556955, 42683.93847161296, 41283.824671043236, 5672.881446906886, 67785.45658002711, 91664.38995757439, 29682.6063326748, 46459.492064845566, 81802.79987798071, 52823.547949017404, 54505.67569936813, 26138.999710444587, 67873.93943225114, 8617.23053926472, 89480.41436881103, 71197.89024895291, 24040.91195858081, 44475.464998706324, 74797.97614049552, 22519.01311209089, 983.5293625676145, 32369.432970208578, 98382.92871116562, 60036.8387152536, 27557.621861523374, 72259.13823335206, 13926.418423363928, 370.51390207309964, 12811.49020904686, 26453.257446349486, 21940.521553947678, 85734.74339759327, 41871.5694390643, 70755.11757699103, 37402.685527900205, 56640.58607927478, 68228.66209843256, 68431.05105319808, 48107.4176074492, 87485.14180239615, 66333.79610788345, 2353.7239178806344, 88103.18934692317, 83645.99004955325, 6424.382877850743, 65201.80044484231, 44630.775239160794, 47466.91163281869, 70372.1880725621, 58622.15780338538, 6867.692440540185, 16464.469522875115, 21503.48661981345, 66481.96581874683, 67606.75078380032]} +{"id": 467, "vector": [31101.423773507497, 45632.66637561909, 54452.597365397596, 96139.41353379982, 94040.60404327347, 66504.01060252491, 73799.28122678609, 89031.0091633331, 83622.84140133743, 139.65696895525957, 4459.982064897883, 14578.16500137179, 29475.92168067773, 7902.563475952995, 42668.83161544539, 73993.93322573445, 33746.51843835973, 77631.6995711227, 90104.67554914813, 45408.70511576883, 35982.18976699681, 32467.615194265654, 31009.95982586181, 5394.032566152218, 48256.837029104514, 6355.717358127644, 71823.83221459606, 98383.30760710636, 16136.82203672252, 67052.2288538433, 36282.02025317004, 19137.73160518637, 19802.05409589928, 70510.79298419468, 94378.37377284946, 77128.40252391897, 38829.64541474598, 97714.03804752782, 74226.3368029483, 95289.19791766835, 625.6321324063952, 73845.38893244024, 5374.321507531876, 31100.72978720606, 47220.03541691231, 33840.98818057589, 64439.646876154344, 93423.83225836995, 81662.87333331125, 75723.17910915945, 30430.294725077267, 92925.58836125657, 24169.58963421554, 89297.74035087565, 10221.672255091884, 69288.95499646958, 49069.42967305632, 5414.062845210709, 40140.4285189439, 56064.35290667808, 4029.5076890009996, 21936.665888995798, 56176.8117160734, 89833.5896366163, 96771.42510133867, 7602.040232432328, 15350.689293343456, 38757.44171435136, 78630.08108407822, 52972.63692667449, 16462.432477808332, 71768.80329540503, 12126.51081168018, 37877.598941547476, 85629.2566190092, 59648.629508587816, 84822.07841422084, 14528.580538294778, 27892.935003041064, 16555.555965693668, 82213.41506962056, 87865.38291378814, 86981.85228554574, 55961.5072319394, 68668.05662591373, 20877.837298845992, 17881.65051977133, 34547.245137444515, 18309.803418402116, 97055.23904248791, 87927.10814958382, 16027.322568442227, 88918.31208713562, 30648.159910284823, 97939.62868589202, 5007.659804608644, 23298.3944973431, 60595.114370406634, 86268.67270327685, 90784.06010133159, 75634.05326144426, 97142.75404734882, 15374.15402205411, 27890.90555192616, 66100.84751286817, 51949.1229956399, 93395.39745148104, 13233.568096019842, 87825.81270084665, 20878.089564408398, 90278.37657273933, 67896.7024395442, 5241.9774304380935, 97193.7132342657, 11811.324354688546, 4781.535772675372, 85214.34831680908, 39502.37707609858, 59017.94099743546, 2185.523730026573, 90584.64447537686, 19225.23106607036, 17646.474598832883, 11870.454658662833, 27712.345937450522, 56202.09924790919, 88307.5279651005, 831.6534089392813]} +{"id": 700, "vector": [95978.61248663117, 4210.705259179482, 52410.894220391034, 8956.423590818196, 45593.75519898408, 43629.85628438901, 50648.466773747845, 76621.66832110043, 46620.631228128754, 59615.90256690111, 95723.47185380977, 80431.72618921178, 14904.024605586097, 62178.744848250135, 6247.149383693562, 32058.48675285069, 45414.614785739635, 63250.95080897139, 87024.55902363299, 54607.56433243927, 50907.5522074661, 46276.66794081163, 2724.560035482304, 51870.52593971455, 7989.5120085493045, 21791.520141160603, 149.29248092490076, 58414.058635620146, 52977.058043083736, 48723.068178056215, 47000.73990948286, 45552.027852816216, 70989.78381424393, 64080.16004618322, 30325.234737732186, 77849.12843627538, 16612.180030330426, 13653.399859857807, 63565.22214827743, 2262.1082121978666, 52594.709953392616, 68507.63593488999, 21289.75198728165, 56637.64447405466, 96418.39230412696, 99431.20554781769, 31655.990446731706, 12193.96139162513, 39738.671107090115, 86463.87669509264, 53160.82759801837, 6084.791965976699, 25308.780432228406, 80614.39085751922, 37532.66638819732, 50468.18149662811, 21348.69242399825, 6720.930233012578, 19317.67510774871, 71451.48706147728, 48058.62039970924, 69673.79699988372, 40882.83264563, 6222.076634954455, 31902.462091121943, 61275.09260448303, 517.0172011508379, 88399.52541678211, 88399.55950978676, 59464.71446334552, 26546.087554444443, 72972.1325167188, 16240.61345485609, 89687.42605698777, 62140.894894789446, 41702.97857887345, 81336.66294793651, 76907.76181354214, 31463.835128425744, 79542.97544737346, 28047.04435653499, 3193.8653124075845, 83046.13277854414, 42229.90402554335, 51.85741095591334, 1524.7718863463988, 48292.64684290745, 49478.74637782213, 68139.48351848288, 96498.92576391298, 74347.15479490538, 75008.38370825046, 44657.08663429202, 39389.48758731729, 40964.848601481805, 46785.51820432185, 81623.37897294566, 12954.721781955037, 70142.71858868678, 59389.54330529961, 83462.6643008641, 88503.8620788691, 9076.56212283282, 40507.58104752754, 43462.268220102116, 92723.05264885256, 83288.93112998766, 71358.40229085468, 6942.863119885934, 50985.84660114675, 44806.73949363817, 97789.37461934364, 65364.185764224356, 6055.673874780731, 56762.03512858512, 95965.48691872103, 60849.216293073725, 20838.513187545115, 62053.52833411003, 37055.72143859523, 48192.26308142968, 65645.48407282148, 10535.718930273018, 72732.90157559312, 76785.48384107946, 14557.574987131227, 87012.99250367508, 91427.47193410907]} +{"id": 2006, "vector": [28375.984567947897, 90379.92844032437, 56005.42629930611, 24454.951430349247, 85411.92380982841, 84297.74273697163, 90574.88547837806, 71259.15045704054, 93696.88919072083, 14709.2248043871, 84172.46925149573, 51800.2679271488, 5914.554144058481, 23265.28705579387, 61929.79274081235, 64339.571282909215, 30629.492051838635, 42107.61754048272, 51937.02439688548, 66449.5062417298, 57852.69716264495, 45701.196811054455, 38994.56366387024, 16695.646302443834, 44521.340407898046, 35204.565688195355, 97623.64318811234, 76983.83569657632, 68044.52144890865, 82744.02270156874, 37800.830355541584, 26297.28663471399, 95015.16625544504, 64293.20760313899, 11236.03787950862, 59827.69956778031, 97006.00545309315, 64848.311731599104, 76946.48359404107, 33617.30304395024, 89332.81056878802, 36789.13402829582, 51731.212459525486, 70552.98789304505, 92133.82794600505, 24507.523201788674, 52031.58133561307, 77271.41880044204, 54580.32245975271, 16056.97414631263, 89136.77165202422, 21515.403486677485, 9499.580996824154, 18510.62659000897, 46618.02041143983, 76055.97578803188, 76039.23454405555, 72235.97236776313, 34452.68318626498, 92101.8418819809, 36610.06782678129, 70894.44373981681, 28336.520081483242, 44784.03901674952, 79189.87623586725, 34707.69967861303, 33067.56969678035, 74898.25527381824, 9005.936415700244, 45386.67947776095, 47767.43492235803, 52863.997407215655, 17062.736986825777, 82468.67645083253, 75145.27777159013, 20812.07042629718, 30968.548792975038, 57926.66508433581, 928.2326987846478, 62025.499012019936, 2851.7369507788158, 94868.18286067621, 12492.059732586846, 32791.65087946555, 34063.512546351005, 10143.440786642455, 27120.89662915246, 26300.839644143336, 57995.01355678219, 14737.84294820325, 8562.692091745372, 91128.43603351261, 59212.271960599195, 80717.77648507355, 92414.34920850847, 74331.09454173993, 54679.346557737306, 9166.469914248688, 26735.226666164024, 9415.278079346168, 82926.17789625222, 66118.0938290161, 53680.21374126826, 63316.05362598936, 16598.715147351373, 47317.338009730935, 72620.85915572086, 25027.43852225937, 6037.673312158398, 68395.01820101716, 15600.491446324839, 74368.66107840906, 23449.75192429777, 80947.19243055132, 69215.10979827569, 73310.21482886733, 72752.92146211279, 91478.5974437304, 22714.621700104708, 22199.461456104764, 99710.6715223058, 2238.53767894977, 11529.070489367987, 8072.768191416136, 3087.8049862079206, 88367.62002629307, 4516.163039763266, 97901.76209447006]} +{"id": 283, "vector": [9643.893538216253, 59047.75859376957, 87517.3019916277, 1905.7027043503872, 11161.49268858435, 75126.90519961779, 25638.918289929403, 80853.33343627895, 27568.814507916795, 95620.71123082854, 88309.70936844866, 30128.48374464504, 29578.483277918476, 48877.288762592565, 65245.66558110668, 42794.00708230781, 65371.34684106996, 74628.37883631258, 61288.189970561856, 34447.60176006392, 24475.652628162792, 88132.17079970933, 26189.19219569461, 16146.714933741312, 27193.790605314138, 79800.60746403522, 74848.52636232949, 46891.09983979218, 47667.13225179068, 55207.27935707901, 40539.94873922697, 98640.16568165093, 79121.63047950178, 93532.87761939292, 1703.995273741521, 97671.29552848681, 73353.62683899552, 58918.84561890938, 15978.363400507578, 98678.95493095987, 53161.136861276435, 39778.062782446665, 32896.288507608515, 79903.28930465711, 90293.76655774095, 58755.35762817281, 1213.3678020382788, 51056.291939113464, 38144.82229163321, 9769.113935449313, 51581.776582152226, 48796.56347856851, 30694.412689420682, 14426.19141536523, 47844.185190818156, 47919.38958933626, 36981.72613845469, 22223.206081951197, 38792.084237558585, 23165.568290047435, 30352.019358517413, 30273.79551159457, 10041.714395709578, 86453.29734467143, 94001.2165704186, 91102.72003346546, 48980.548158684614, 10526.651298908662, 28059.70626966021, 47316.11701665852, 82440.53816471319, 99125.8537939847, 95944.01018344604, 97234.60168431968, 53766.57957555402, 17909.05263238175, 23186.488253137082, 93574.20998624475, 39483.91198789987, 97445.69148632193, 1399.9709365488554, 61362.80198601488, 75749.28713512729, 32116.08035933857, 98852.4721951831, 2684.9082914277233, 77301.01129486434, 37265.24017558675, 92579.67563175615, 26825.082112414224, 491.34773938516173, 25681.54766455354, 49805.47336447484, 85284.27439276973, 39184.99046938886, 88857.70907675877, 25133.914322666762, 84839.9650218429, 61621.23762814031, 2944.409433752038, 20620.444382226022, 74674.92456584315, 99996.99917049405, 35563.2951673147, 50076.64033213528, 91013.27437134858, 50509.528513833444, 84763.27909252817, 11241.088104421904, 50431.34594989229, 13428.33623771954, 34508.91979119506, 14909.615801222299, 58674.80292169923, 54719.63302305577, 10773.443901977287, 88186.3427548878, 19216.849889868714, 73678.42383454344, 97823.69733784146, 6860.426923162644, 29684.847711388273, 42195.56778158823, 96738.6752568796, 80611.3760269239, 77886.58989205629, 85933.49002449699, 61389.08714827005]} +{"id": 678, "vector": [52525.896093410716, 7671.3463917927775, 89035.62012767015, 25749.55232762347, 5192.7853283201375, 42878.48504840353, 49275.140924984975, 69971.49768783651, 20456.588330526392, 40606.54649231179, 94586.77381251738, 23253.9101610141, 73430.155540973, 80565.37625382247, 8612.135890540007, 16870.810052650166, 45981.83824346092, 40167.08052331972, 84696.35132232585, 5851.783136765265, 58473.32047173954, 83292.83908349344, 23165.215167940765, 12731.801726533331, 70079.63931385623, 319.4789237156548, 12233.80054323817, 22683.651771241308, 15354.956692413201, 61362.63758771245, 95934.60937635493, 41856.799585196604, 53816.23932658761, 92792.73850563316, 69929.75103924975, 23357.105590445106, 47232.38975543568, 8952.848655621037, 37289.79139146717, 9138.257906488867, 27049.163829090005, 51275.50223097732, 62413.04753372172, 79658.0609476015, 20450.89423729509, 87701.0004740028, 13441.581551605408, 78615.66359412929, 20927.865386766698, 11823.763151995703, 59241.97277286235, 73793.48952714882, 55159.32602843898, 52460.78495057809, 73270.74632984221, 43386.24874538906, 80711.85495128095, 68675.63702670774, 72449.23299214568, 7266.759043767556, 75160.69785492525, 89686.3003225389, 56381.774357294125, 95421.5852052406, 71575.46143217027, 14263.16575280765, 53066.8061372232, 95529.13716149701, 95526.1938558753, 96279.58034815494, 26063.34155970039, 20683.816871737436, 75749.85572187798, 2693.4061841961343, 18631.510788230033, 23349.24305613417, 51554.41300905414, 20107.796701663927, 90666.25253954041, 31405.250969796583, 83016.6262986476, 24427.89093122575, 46005.570802572336, 6111.370062334898, 42057.703057805804, 75724.82900437007, 17567.905067914737, 99748.39115949567, 71545.20814103648, 54048.733590431555, 10189.410947701572, 74535.81734746983, 97215.46749911898, 91978.28729637763, 15411.304904602663, 64863.46080286665, 65497.55538042399, 84254.03588303651, 8995.212448904, 95618.45491819903, 81738.76621191944, 93262.29636206044, 22660.63989105834, 97029.51342752865, 76957.31932305379, 20711.309785882313, 20759.996053667175, 14023.1783723608, 81029.54101570992, 26517.440642412283, 5467.512133403785, 70022.00282407868, 30614.65489808807, 2362.127485051979, 56596.59510828704, 15240.309601375713, 32765.328481496268, 69538.98225019687, 65105.6665445695, 36442.3120693245, 80193.20162096537, 97161.61342216554, 46090.02087102018, 62638.58666284853, 25581.063743343548, 34119.56829142944, 39417.243063703856, 61878.355723331246]} +{"id": 1495, "vector": [76569.91099506289, 16032.508233680686, 51820.781160212195, 69998.44378575437, 36605.66292917697, 56172.50853033897, 6343.688969407579, 93486.6970869915, 77276.2641219042, 96441.93260082696, 94138.23954249256, 37115.483936267316, 22587.518726188348, 15725.46935669884, 82229.90399964337, 72676.76281384383, 94371.91253436917, 12203.018450382586, 3976.5778919749528, 32268.9667119347, 4198.34955992221, 44515.984913367945, 91875.1678343372, 23169.189996583507, 77085.03218510553, 20222.728727072816, 8479.989046458846, 83941.9256286877, 36005.71875230554, 83820.95553771527, 51381.16843831395, 87495.27387768921, 22223.707960020954, 9183.308960225744, 69803.15351285641, 12174.449127639531, 64544.42493471779, 47281.74390435983, 73438.05070232674, 38234.695085467305, 14138.527085950569, 44279.62952666937, 57837.80160943154, 54553.451613778656, 56315.0663724685, 84877.27363861713, 56874.10025332211, 76359.74250119827, 74517.90557907546, 15148.906053497712, 92885.3137698423, 4109.560411745861, 21016.105584924193, 29202.811363334567, 75011.33087916646, 6825.7223032221145, 94935.44937489608, 939.8037278270155, 37733.651571789764, 14248.38499804969, 5775.6175560121355, 94551.92184904114, 44208.1697537835, 55086.82193419208, 43409.889474212745, 4166.850156449064, 96867.27622742178, 81748.84284678086, 34205.805953357936, 47828.06032251178, 92716.49225270869, 63962.71682251549, 14515.817707259605, 72468.83561233214, 55125.84787290206, 27015.68582778321, 60437.437937728166, 74923.78296153466, 53092.20151967167, 52638.84531063091, 30178.757008849312, 51523.22062071304, 59160.925408994925, 19452.449292623787, 48369.42878681183, 78224.11135127823, 27310.966219512433, 62267.89257056717, 53761.319636228254, 4207.511916078077, 87440.90461434021, 51478.24090817147, 96614.56717156548, 10387.731259433198, 84928.96858918539, 94739.7334978625, 76268.33965244966, 77801.04358216973, 68716.05476786214, 83619.82774927124, 44176.48200507096, 84501.04067903668, 903.1401661453109, 45768.81620682471, 6848.270535004574, 87478.98335324638, 15154.54911262566, 95303.72321983673, 94605.15343646704, 34360.15658148385, 8716.560683832186, 274.97573642141225, 48524.51137591989, 75932.23409961845, 80172.59782611196, 97102.86534062595, 48856.171175945405, 54275.41566991335, 82285.4478942716, 12152.089710065917, 2969.754943078362, 41596.98704715459, 5720.92370820424, 28761.95272418828, 63231.223122453724, 53027.715603553304, 55108.12341850893, 30129.1882273376]} +{"id": 1719, "vector": [14446.545608496852, 99175.92671432618, 38701.23148481902, 51014.54153959851, 76778.06907456192, 41583.58329574696, 77586.03908081519, 39640.68362506342, 38066.23976862803, 17851.261961987973, 69.13341060079681, 49226.64235776032, 60909.858660940154, 95230.28927217724, 14560.963326117471, 35252.08779924446, 24970.525060176184, 22974.360604250633, 70112.4121624971, 48667.45375273512, 68015.76653789009, 44718.43167567283, 76107.80531351671, 50744.438559242786, 40174.65787258179, 59440.92543090501, 30175.436865648353, 89010.87630513073, 2991.0951189044745, 91843.92548938713, 732.0742332969533, 67549.58561534811, 92441.53482061949, 26405.622713208486, 75585.98331742203, 84128.15799784791, 87447.70211882443, 55753.450004363156, 52108.85004716287, 45447.864838800764, 8931.854682631334, 25406.28964585798, 38730.328133211566, 56112.74555068061, 61393.47301228626, 10208.09511586993, 52613.68223039159, 50517.05977192946, 29699.073478288017, 844.6154100799963, 57408.83880308584, 90872.17509602265, 53867.0598266184, 67664.3719514141, 57872.46026445181, 2150.962170219206, 52054.48465941392, 86767.22330106991, 36137.31390934533, 55312.623736275345, 72143.78125976217, 65541.23640528994, 65366.77822015467, 71638.63942930661, 83561.54861100727, 75144.73097235849, 62131.982930458915, 21601.678995202077, 72426.07948473946, 90645.9620906745, 40359.071722829845, 26843.70545280935, 45962.51504172234, 18905.586010841478, 75766.01549738206, 24193.662520145954, 46810.55532729951, 68703.2605541338, 69116.37070312545, 63719.05734240035, 59067.29549922673, 98165.80994117775, 36319.87442176876, 67030.28632963159, 43709.05100481751, 31546.5551234411, 19214.060594985727, 16408.702363246353, 4004.030991406915, 26435.523874909482, 45039.931384742646, 64484.341816338645, 86181.31452606988, 44679.1197377491, 55639.720305326155, 54815.28820983779, 65521.39676693293, 44844.207044757226, 84706.05243395928, 44832.89478334918, 80644.63566037612, 98219.33771018639, 92355.51881464859, 39354.14601716952, 91139.84045827591, 58584.12617078763, 4119.198792016954, 96856.05879494583, 61732.775858637026, 41522.52529094318, 24200.324823771913, 37620.875616680205, 64161.56856600943, 26123.348955588986, 68057.43218862251, 28058.34513573778, 95958.23974147081, 80662.04169512689, 8945.202349043857, 48750.30303361524, 95176.92395401807, 60436.028607037886, 64366.4058841044, 34826.0008309433, 30868.072352985055, 81720.86784675471, 62424.51229512581, 10814.9889209458]} +{"id": 1380, "vector": [68538.29128266133, 22747.459190327936, 80477.12872107026, 59081.17934693463, 86051.58381514312, 23759.56864219897, 57423.34694386176, 26033.126738314448, 56066.98548675497, 1822.049050200336, 34666.41300323387, 97890.16120007464, 59865.029770066394, 35091.27543835059, 70892.91786765645, 62730.22948029912, 96102.90218847922, 43920.945496087304, 56293.414926709294, 80893.96949005255, 71858.15086650965, 62313.33072097007, 26596.065130461542, 72122.46400021734, 99232.8033520474, 84891.93588364615, 75642.7812081865, 85597.6936174652, 89880.37755006572, 89282.64109794433, 59575.42473990005, 42983.22146465084, 94467.34951352162, 13650.401120889266, 73669.0347686975, 49042.85784891059, 92529.24224533833, 91679.91843794803, 81273.94925586799, 71753.63082754966, 71873.58750145973, 23522.04025998298, 43856.129900084175, 1777.1896102447893, 66255.50715010057, 69063.17373127905, 72281.9669456531, 20853.130504930894, 56573.24033811039, 30941.005726012183, 63648.90129897803, 37366.015259791566, 83669.10348487894, 50350.349668902796, 9995.952345920856, 83622.42509740412, 35665.021873943406, 47503.2821186665, 8462.988725988207, 89781.13458672584, 12328.858515626762, 46959.19832544772, 48815.778455128864, 28384.136616988453, 13520.930468338933, 66857.8005639905, 5512.641154287301, 41279.19293318013, 36213.157324273736, 61283.45149527119, 72380.32154391939, 91214.94871763227, 11727.666613564048, 29229.884026998876, 99572.75488901544, 15750.939594718084, 73681.22320893621, 63940.73739932084, 15436.40982624418, 84258.83555269949, 35558.49787780856, 93667.02986143058, 13326.933787372907, 16015.259380645575, 45503.40313432192, 36014.77287862397, 76800.43656954946, 18577.454827687034, 39292.78009364936, 4493.042187287111, 5064.931770286929, 21219.210807244082, 61430.673899451416, 42330.47369810401, 605.6776509722139, 48909.457955609934, 29485.89892749903, 66077.91938078846, 99359.7870040501, 96156.04938963591, 46459.35556210864, 8129.363560891323, 26631.865684657296, 13321.991529226018, 16352.683662996325, 35810.55463643632, 58971.01445131464, 17630.581997167315, 3926.434910971821, 7964.63481978702, 5349.760382139046, 73689.14747864325, 38154.02212100618, 17154.127672622955, 1681.0756219039158, 34159.262040761925, 18580.912572959718, 54075.82203000403, 80455.0714359831, 42222.167379409024, 13625.79378471569, 67763.3609160915, 54140.852670652464, 25761.249888133563, 3652.272469787088, 97465.72480607772, 55028.77924462264, 7204.556609170742]} +{"id": 859, "vector": [34767.46945557594, 99727.79867878238, 9935.811640395965, 14321.255407697043, 89807.53893158691, 94846.81058768563, 32248.117633293772, 81702.91848537208, 61748.142112584006, 47031.911364098676, 81547.87485104265, 73629.23687430078, 14601.510584713651, 5035.160121593541, 97498.18262292662, 76882.6449055839, 66549.20083844464, 10373.643767768992, 36506.694189219656, 7584.864565106764, 1426.8446162468028, 73465.72877368382, 99315.5878445534, 56767.969710008794, 92543.62401454334, 79820.90119687126, 11567.890307422513, 31278.16645782645, 91685.05870337397, 34485.11420886777, 34052.28199879369, 19128.65610650982, 55804.90209576528, 76560.58928588414, 67834.22757380422, 86814.5894337634, 29048.35285984504, 60083.877372901705, 60604.26634573016, 11100.054549343851, 54617.932261469214, 67913.7248454668, 21064.284484849893, 72346.62431848497, 946.7273727597592, 61741.21629104564, 83581.74033632387, 14809.018994989132, 67714.31096701055, 86175.44476310628, 5176.898294636023, 52377.83293738818, 16440.305267088595, 29576.67937199545, 43528.901841709645, 49100.252357520716, 83732.6390184506, 71757.25466416721, 50637.1006607852, 70166.76148991939, 82958.70469788759, 7071.874205804729, 68343.6762000352, 75893.50855253745, 56039.71850902376, 89713.42546340195, 1340.846096394066, 5706.879193950743, 65711.6455839014, 28356.745555073336, 21694.035688507818, 24665.27998651544, 94841.02750502728, 37623.062875509706, 50234.04050696316, 20157.132890671248, 72433.2313993931, 32571.184886863535, 89162.38184983138, 2235.75764801317, 70455.09629572366, 8838.903100326479, 48042.438519271804, 5477.8974488472795, 41527.186149394845, 13163.972670828229, 76903.96277125784, 91203.53405384898, 95017.16147846296, 72247.73706750154, 44461.00627627011, 33167.52151869268, 68849.86925088629, 64684.97891317624, 7993.204990399106, 60930.37765584148, 6820.292564806607, 37975.91103472529, 76941.35900071103, 58526.847729817746, 25470.33696729929, 32229.91632525595, 45091.9969370298, 36292.9541762938, 57783.71721806247, 12261.719523175374, 16944.01992961283, 98587.49497079435, 74683.03624181582, 32586.942641956663, 64071.58175658438, 25725.957713966363, 60481.37034355009, 65448.99711440659, 58961.72869021047, 39664.193616414734, 45169.7766448848, 81091.73230173152, 41461.82414424045, 99256.77131464599, 31044.698131508674, 62668.32793532362, 32473.739308204375, 39459.78874492052, 64711.01219998735, 93404.57533675525, 75256.365104717, 83706.22451326178]} +{"id": 864, "vector": [65955.34603281386, 89873.84167112042, 8340.697087288574, 65309.70177519479, 8730.41917944779, 64047.01442483082, 31699.156859104947, 62160.005549156864, 67625.98066643252, 73434.63713779136, 32051.336260232056, 32914.95873992686, 40051.31793172507, 8720.66915533537, 81762.73186049196, 89064.75067298702, 64399.778490772616, 96740.21491356575, 4345.844309732827, 42764.61142005753, 47491.36190005753, 33052.16830569325, 21025.95641869849, 78293.2648757314, 28443.923441878906, 47165.962723675955, 41024.96070783422, 4330.5968912553335, 25089.95574297107, 71181.80742655526, 71334.16831212168, 37570.57855229073, 77165.03421654532, 22489.582459562705, 64459.97150494492, 90616.46418684645, 35449.48417030353, 99904.79756356971, 19152.971862709488, 49927.724913164995, 30945.730000083306, 22096.771152865735, 4359.181748432006, 95602.99064292124, 85893.6990945649, 45839.155153747226, 61477.209853844375, 52626.534505121504, 35919.121189322366, 51284.39809756028, 26425.986311690285, 60872.61770932636, 86861.24827371984, 41366.64098568522, 11637.966572153968, 7941.5022979218875, 79570.04155021282, 83951.484422263, 13929.678574470083, 32446.298387566076, 66495.92192607983, 85223.71434473185, 42833.0657180797, 54494.11421722403, 83248.6521662102, 71578.40234723444, 31356.448786376834, 87177.16913531846, 72336.66611338299, 4613.136420849706, 16561.418104101744, 7268.747098771766, 99064.78411613002, 28247.83989576636, 71207.76262398697, 423.29806886297615, 68162.33927373725, 2104.173788092645, 62205.04078924586, 94015.69076489046, 85976.34798627117, 28949.161074955486, 30849.73361164858, 28513.850762140213, 46920.74607142949, 60213.0011855477, 67186.21313735074, 24210.402140536236, 13625.750513389046, 62177.66444319613, 2684.233537010805, 62388.62675185398, 71245.52261717939, 2643.6564959278708, 94347.98478539508, 58012.828962334876, 69320.53814361256, 2165.3227484120575, 81691.47365625012, 89636.77289360772, 31702.73496895809, 69891.81047447764, 45871.55816708622, 39908.52478828669, 90932.08157031013, 95641.27253830459, 94083.66337613195, 6643.422808064858, 82762.55998418544, 62677.62494598257, 95960.62697001197, 89417.20221075918, 68004.94625618534, 72110.70348399274, 40143.0516625012, 36811.38088370327, 5323.223153164991, 37102.152828119106, 82012.91984279822, 81802.47721531102, 20828.999364443756, 97679.03591100765, 17257.375528604258, 47850.346838430814, 47127.509070830274, 90255.47355444486, 27981.92673298714, 28273.98646958088]} +{"id": 1084, "vector": [20200.98576812904, 92419.29822383796, 68667.8727448156, 70074.48316836891, 65289.757453107464, 21157.807151223664, 88898.1558582897, 75218.63208799892, 46174.88487196648, 9688.762491977199, 53477.345576838554, 44586.861219865146, 80643.87865713162, 33320.8731952719, 54826.62343062092, 81833.49960035752, 66572.00795804503, 73080.27796109508, 41285.43014279225, 60353.339490640625, 29899.42706904387, 7247.117316833007, 92716.37208091785, 83281.47482054471, 3625.622152793495, 24420.479129820083, 15493.53716261066, 2922.119112982646, 8249.108202516985, 89557.0512556998, 92666.20153955514, 88525.9036621667, 84608.64897624278, 26656.467275859854, 72799.75844718434, 86735.87546897835, 43208.151361497214, 86274.83057322275, 60731.409915828335, 63719.05360595832, 18758.56865097981, 52928.60102418831, 1145.2653956901981, 32375.9349008516, 23339.52280414321, 23956.84244330253, 87783.41673471728, 5253.1106695376975, 71402.5382714104, 80817.55228601543, 7841.16553776425, 8963.793146922339, 44871.96054162069, 91861.04822270687, 15255.448056036725, 40911.534195405475, 79432.86520498709, 1493.8071301321077, 39459.45855615898, 57792.198845095634, 73251.47452777311, 18977.115930792697, 22446.783731870146, 39425.90420058555, 56443.628584772465, 90976.52602120336, 89189.6343631789, 60636.987583606264, 66514.9462264046, 95532.81829919518, 86720.30118817432, 49338.346254970354, 83452.2916636316, 70235.89001292872, 777.2383005676397, 66848.56788086252, 96762.47548783408, 26304.901023061953, 68295.20883115761, 87793.45514350614, 63196.034328482885, 34167.41295913305, 52147.17704933991, 36151.85842051904, 79519.41254463347, 75085.55063571554, 49046.00826408708, 19891.934906878527, 40258.72580701393, 64950.05866119776, 7073.618897427947, 43608.45631131445, 80988.87369547908, 14519.718018667638, 24113.14313630425, 34253.972665295805, 52496.620883844625, 67329.67129473286, 4406.359567161444, 20829.052149530446, 99430.20457264858, 11284.408898809572, 79023.93556393264, 11547.109378624176, 23184.164129482044, 10094.175807492078, 31018.931293472073, 53335.6039112019, 199.85566412694666, 51922.513085539125, 85397.88942681703, 45457.030674428475, 22731.954983781066, 54881.80354634885, 59331.59821288054, 19283.160978488366, 15783.464814230763, 96070.56366327064, 56391.1040327809, 35462.53025548236, 16776.4341216322, 51455.262036692686, 44494.15692066293, 39019.604153660344, 38074.707428661306, 33761.100413764114, 81313.84788802842, 91011.23827451332]} +{"id": 1385, "vector": [5949.188651354553, 73275.61260535251, 78955.931765294, 14294.040223699456, 70653.569205389, 54521.384243540284, 56920.80281278138, 51875.97843186251, 77771.93532184746, 50157.68855735271, 13895.019157424427, 22309.178272101228, 14195.55017195535, 97476.34836375168, 80656.3714950247, 47069.42650170412, 325.61538431103986, 98014.77901565268, 19714.27350232844, 27349.148459768458, 32781.95922841104, 51620.473799154934, 34302.78815081746, 52061.899269525245, 49143.62063871111, 87492.98619669917, 17358.026080576517, 68821.18766049092, 54717.78191973489, 7437.7042163163205, 90842.55328053936, 64795.296288450074, 68576.09964063975, 65920.21050442586, 97787.66687579999, 11290.261974266246, 34609.246080196186, 27997.637644371152, 78617.56360264936, 81491.65538377025, 71173.91634683023, 89973.7159048984, 52564.19677714169, 99372.94332587633, 3727.3665483393725, 62739.20834444911, 82227.12262180682, 89117.76454298996, 82667.7971189037, 75894.62872097421, 85072.61008156338, 97582.87642361323, 94807.08684115777, 52967.02861666006, 35673.31152636323, 71092.1303021353, 639.3322136766689, 14658.014059173296, 12739.834527159766, 30833.42480539808, 54678.48102039865, 26525.36069546736, 68722.07521216666, 68050.7360627654, 33046.99507186566, 13137.358852216496, 57504.562126684665, 55107.72020533394, 53991.59736161797, 98362.8012616259, 38680.30360987021, 12843.297735030557, 13675.47703299843, 10899.573527745477, 34307.698726464885, 71894.21147395558, 77837.47545777976, 50330.581828466115, 67909.66742664055, 56508.265048712194, 72506.6041305289, 61019.37440525025, 49322.00426497225, 84556.14016177991, 47918.818760065675, 32744.88072349101, 64542.56206845904, 13106.377316546914, 68243.09332743699, 94854.0542447022, 24867.39729352322, 340.24305354631855, 48476.58772217668, 8361.651539163473, 57348.806472291246, 19051.5195721907, 58409.180018730476, 53429.17170121, 6629.814769053854, 13151.519918468292, 71063.38663068728, 65989.56813595004, 20110.44514169419, 15073.720562287008, 23190.52362192585, 94018.33902119078, 66340.48057400274, 22271.102257670616, 21229.56916404255, 55974.197421147146, 98908.60799647195, 73451.35747076284, 24371.084360789297, 23324.31829682413, 12968.071741743714, 18888.437278654867, 11968.679192894726, 13779.034960928582, 32853.4386569442, 17938.45448778577, 52511.361457067396, 4667.745867672746, 69980.7207848087, 53874.935142190014, 47420.77244938679, 57223.815001635914, 28688.733661141752, 93255.57074719247]} +{"id": 1384, "vector": [17060.37236184925, 96828.38602073515, 97410.17382332291, 59214.22727593434, 75371.31784754412, 97052.22557452084, 9554.396194612624, 18886.435957029735, 76777.94164830902, 21493.154940848737, 63268.5469097513, 63614.5892239279, 10334.755682120722, 21116.305195224206, 96678.30738062297, 67634.81151284711, 26220.160848373365, 7100.212915935811, 51360.16308421579, 62735.219335062146, 30367.48060239195, 44273.628945487, 42604.88880697968, 21296.313152923973, 55896.08708166164, 62966.960903896106, 45921.2911639475, 91426.5344949067, 31130.10095616051, 97045.8117310338, 80805.14515360352, 59380.15748357212, 82762.48488762388, 81472.74726722106, 94287.16001847199, 12012.534058337942, 45015.71283356197, 30126.981833513822, 67751.54382691812, 10808.5692576393, 79970.5693750186, 24142.27980021186, 92134.68462982462, 39708.35444861074, 31482.992959286614, 53254.70268609246, 87323.52576926598, 76521.34267493663, 5107.577669016206, 60583.59058722994, 8570.927449047416, 48578.48028793609, 84925.3383275512, 41785.50564775645, 16925.209760343583, 40492.81991479446, 28578.443052831626, 47248.51098179964, 7225.137264930859, 72613.39289418785, 30389.533875864705, 79623.3195756842, 82662.19119564329, 945.6720767094806, 67259.18227047646, 6717.220123277933, 62959.98157237776, 10456.506211220174, 63876.81877665241, 86051.19058637119, 22687.745143949865, 88869.4520854228, 98793.55147719142, 94335.82181856947, 79176.29989515078, 96393.74914715368, 25913.44036943194, 46464.7747496726, 45362.74554146563, 96534.23557947202, 44329.43889791232, 95098.12804404125, 13204.663709831233, 96872.46375709266, 77939.57211177972, 30596.733310805124, 16299.945173135055, 98957.90045643903, 79368.10085787598, 55239.758072697165, 15666.042790912605, 77208.64848872699, 7260.0459762234395, 23714.65126953852, 46466.60425590413, 45479.37488820049, 65309.6049854487, 11844.820503666597, 16695.31826234144, 65782.7957037622, 59301.337147891565, 28066.59955702242, 9453.74342395926, 83063.4355961752, 70799.50046115226, 86965.01397067093, 19538.073230637732, 28120.080689557304, 39642.560955955894, 30925.166849757912, 20378.465949247737, 14005.996276597265, 84948.29453122921, 98972.08796717288, 27617.3891857849, 47143.93425495837, 52498.70647782585, 726.9236374358612, 82039.33673460281, 88183.77761167871, 16023.550211608783, 45342.50696992334, 12683.249562933408, 50739.159019260915, 9958.462437001703, 28706.49431965563, 95461.46156641713, 73199.42914376102]} +{"id": 1211, "vector": [1865.948379839244, 67827.41262014586, 73124.32236757243, 26996.043447869666, 87291.88052774026, 78772.65811888399, 14830.745761852804, 91226.47026434958, 26957.752192863372, 81870.66791158612, 11772.477177746088, 23910.55479600176, 84788.40880003544, 29851.874058439742, 96016.11876137332, 48256.423096071434, 1152.9390900073545, 77141.418632585, 54879.09908715471, 93387.08748787173, 16143.87473653135, 6780.234602381485, 45208.994702421114, 77431.1295555812, 82771.4508693558, 52769.12944496698, 21304.809845771477, 25005.96741376212, 2254.4758369914875, 80518.81825311981, 87913.63240015807, 24431.56595514747, 19241.447042817374, 56737.61544586413, 95911.70435774596, 34038.43638607915, 15452.127055479548, 62001.41183888318, 89772.74621399019, 3992.153713779356, 43808.50482616159, 23029.677320522846, 24464.781700929972, 17960.58341206782, 63893.4687627011, 82964.27330333613, 87536.48008129276, 10377.473812923821, 59755.383669474184, 68483.33150068329, 67505.73265197054, 65989.74766403357, 3399.566405464616, 31599.365056172766, 8247.582433079448, 41294.56369710488, 15306.958027006778, 36091.21188093848, 18613.75648833554, 47818.500481239, 67292.61996489459, 6561.782553680307, 74422.60363380032, 42939.158583187535, 93850.65563398716, 34713.56305181118, 28642.557730510543, 69671.00739679934, 67457.34363452267, 36135.550863649405, 16120.766415210985, 63942.044395155885, 16785.212949606153, 44496.38393360021, 46939.470376196594, 93422.88461907617, 65412.71283307324, 36131.3717479668, 61192.5208695207, 86092.46690076776, 57488.51537853424, 97938.50118458696, 67979.11555621239, 52732.44816451986, 97983.03570474924, 26667.181219676593, 29552.26379733913, 63016.05369543593, 70523.75320797431, 55254.3027019263, 74213.51812294073, 53673.63494527849, 13494.93759328425, 13900.000743589202, 63514.74533773468, 55199.46523590556, 26967.908353721792, 61412.49008720619, 26203.920673640412, 59711.16206422019, 1758.6228661494063, 20057.376203251253, 20617.602904409825, 38209.44836919349, 19465.75147373649, 39123.47429519582, 62561.0649541275, 78621.48478387631, 92929.51166983144, 79945.95093814677, 28432.799164770928, 89988.4606590722, 46221.823788941365, 49425.72160915496, 60983.86310783944, 16818.741822815053, 73100.53555605115, 92502.41610547148, 98165.05830049104, 40135.01728088167, 96049.30180551515, 7674.166166906482, 36474.002175399786, 35554.868546237914, 18286.030956437917, 29181.11398720572, 64320.181429725744, 8296.50040390012]} +{"id": 1265, "vector": [18551.52840330171, 2659.69302562582, 7052.7947029134675, 33512.0600525738, 85668.1358375876, 9774.040740782086, 62693.22946453167, 12314.562111446114, 14759.698194598424, 7611.133458894515, 18361.34657384727, 68910.12373019003, 50880.351300080365, 66225.57701190336, 42248.980203734754, 43890.08785227526, 74025.32658405633, 95665.85575110867, 37225.53275984462, 99019.1138613067, 3436.6732812279865, 71396.56228479015, 7950.70381291223, 26071.886896662723, 59494.32293851139, 62566.08350357603, 36687.876024210396, 62790.308583246515, 89431.71038961559, 2831.87080798315, 49024.26458170962, 54346.94995733208, 60475.691050242494, 61280.61373093797, 52202.42635565602, 27632.29776025561, 23237.83569427048, 88576.36230288736, 63941.37999339044, 53758.645381324444, 68268.20386609895, 75780.41310937706, 14458.982948708177, 19798.72555161436, 38961.84070974169, 17684.327252881314, 22243.031032786886, 16527.68446592955, 10390.897384293696, 44748.431937131536, 85328.89796244595, 50927.99312730679, 14057.800652006414, 42402.25355076123, 21512.37920449285, 34356.81769322958, 42957.08718919313, 73409.04745768006, 63830.33123219246, 29631.34612257151, 15557.805589024743, 61911.73963046015, 18264.819083793558, 29171.687737645636, 41481.35823167312, 90731.45519522228, 57139.20600878327, 13789.114026091687, 12349.777878072488, 30429.075518829417, 84140.55075280332, 63572.71922727041, 38995.157402353056, 11030.602651811761, 43342.38391502446, 59459.18579868118, 96516.49513198397, 72918.21222951656, 50189.54978544847, 46261.26436677566, 15325.329280381084, 67311.1239032152, 20558.14620476504, 13430.346895270073, 27330.678293472432, 51079.52273587809, 17890.409180232935, 84684.53097559192, 61854.98498841836, 33032.66738294004, 10189.075705809903, 60894.78399450207, 24989.619033963605, 79435.78704755325, 24286.0443954923, 68375.24800709408, 54655.070347661494, 17501.435997221037, 94370.40330915916, 50641.02538405634, 98445.31433838018, 59138.75594603285, 73880.85679748471, 19957.60766658469, 52652.8241081844, 62514.10515942192, 6984.856292453412, 5428.066118617625, 34412.02546896855, 61000.49769363736, 66495.22060091043, 33256.1024469109, 8218.779995578918, 58181.87649377343, 47402.30248920176, 6951.187932588398, 66529.21091780727, 70154.71510506963, 18405.19524117924, 55525.1234687522, 92695.21478119667, 50524.49771629042, 33797.29716609325, 26714.831605422973, 49090.15416691549, 74313.65996995555, 19678.836004033317, 25905.36272958822]} +{"id": 867, "vector": [10382.220385930896, 52291.10029452492, 87371.14356765567, 74015.2024099997, 1694.3984535774614, 12596.40340517999, 73678.33058626867, 3590.601156544804, 43978.98543482225, 71018.29659198057, 97494.0045522639, 70609.62180262757, 916.1409102619089, 95322.2373242783, 17792.126250925998, 35701.823391362566, 7575.215897007537, 49931.05729834413, 87905.69114951625, 8533.177941980597, 37998.63478150509, 98374.79338680983, 79484.13742886507, 67196.49566760835, 2061.358262197155, 19908.56602018811, 8256.037080361579, 2972.7072007730726, 39998.92114953551, 10686.964200410865, 56913.096792458025, 36809.60411489064, 51189.8067632231, 5432.694320918774, 69611.0572894159, 73892.05891238949, 66430.00121707628, 96886.31776418132, 4234.0089935438145, 5122.571170652867, 6533.301874779784, 67222.3131866077, 99275.89706449276, 86331.43111171824, 38903.50909958445, 26481.175864538585, 55706.46776075698, 94807.12185901648, 37014.92182231517, 22111.154852202162, 43874.25812570638, 65371.618248375096, 82853.16803681682, 26935.426580879284, 99496.38788772536, 50343.55647434746, 62983.32601317624, 72479.14036101042, 24984.58849903844, 72718.69708070051, 72886.97866528381, 57708.1763786295, 55800.012477299635, 90608.42693775093, 86094.4713035987, 46086.748550099364, 7939.466687318486, 52307.08901156463, 62702.336586498175, 87759.04922632521, 28713.960388758864, 27137.25979957814, 84996.94382466159, 54424.619484051174, 23278.73852319018, 89386.78464633443, 67735.17690879414, 84740.308116706, 5861.25946094247, 7557.268659664252, 90040.42229345803, 13016.685311385678, 6761.22416170123, 69307.97574512202, 7356.2549707289145, 4277.821958014949, 28812.30131816852, 80059.60516239774, 28537.39536987596, 60115.909814591585, 47709.99674461065, 95764.87850228821, 44238.93088509888, 6742.544690553731, 69841.20930535966, 31264.888061277295, 50262.23188295984, 81952.91354930862, 68968.29172404256, 10402.698728916004, 49172.858009033196, 67276.22935398362, 66013.60265950052, 50648.68326174016, 82072.5681259459, 47643.261467518816, 7149.3075101460745, 74919.48575203256, 43132.73592732876, 95196.71166278044, 56763.68095700667, 30887.87865454389, 70474.14202644705, 17443.80258055479, 18912.3468082673, 82489.53533970534, 5444.991780306041, 59445.40910922078, 47539.93171456988, 41408.46459643278, 35039.298609621546, 55070.04371777139, 7003.41962115062, 20466.383229285722, 32316.175113786692, 44379.26014625733, 64176.590312397755, 68841.50467977063]} +{"id": 1498, "vector": [7100.3276697763185, 22551.25682928223, 21029.370741725295, 70868.1535292378, 9428.460536329252, 80742.81392145673, 68457.38381091398, 53419.107452605786, 2698.9958638951152, 61451.91965719564, 85802.68751221188, 91798.29702359159, 95555.75399490389, 43380.249535762305, 14090.372600147837, 87043.20867243603, 99835.92534447118, 85843.66155737253, 40412.91227287019, 33906.79813156284, 64797.65320908315, 20608.48409732935, 30600.82847792519, 98971.0743029527, 41920.811837927926, 83135.53844588906, 71144.75258431552, 68114.62147503596, 38594.826082808184, 32427.061851002392, 89087.39015892612, 61054.66901495789, 32096.188152093917, 63912.78925706263, 73383.20566921863, 62125.7107901897, 81872.60919131736, 75590.8022869488, 93522.44528648333, 37249.73007224367, 17900.750212576255, 18126.87763852555, 98049.76449089355, 24576.2151416174, 49014.90707875973, 59963.272772803335, 43941.79202354696, 79884.9883020943, 38452.04062845229, 34775.95444034868, 16257.256388810238, 92691.89526128751, 51998.21652843228, 32839.984112104205, 97999.55819348148, 25315.28702928898, 59617.9424413511, 76442.9836963734, 67520.73919945853, 35297.84441287172, 2092.6638530225027, 77759.07160998428, 5292.379221351617, 95988.06831492326, 81503.86875069609, 18853.709492726666, 65574.28858608374, 20665.839897713224, 56127.88599404041, 83311.68350402647, 55429.24166821415, 59901.42402270432, 45347.56811241265, 15949.213689177212, 57791.427378589244, 86642.77682408963, 8840.01977374671, 80255.82649409576, 12657.90287528642, 52851.40551858056, 27039.992227656774, 49887.935074211775, 19180.856335857465, 71627.66679323402, 28312.069271960372, 8192.023979437712, 4550.483413786055, 89888.8463726671, 46355.92998835025, 5379.7327636238015, 99022.77465767479, 50053.26527458033, 67787.51364542845, 97423.908095433, 69915.61233861133, 86115.51689478988, 45680.20770838852, 52929.165585802075, 96150.22292746765, 66597.40851351149, 9103.790650536337, 57403.01581257098, 66465.07951234854, 9222.59156044808, 83964.33724596913, 37636.64177986693, 92913.06331332518, 53034.98664270847, 36070.61534027499, 67473.05871543709, 65602.26416606463, 34811.94016477624, 90326.27602173704, 40596.91892685426, 30129.499752530985, 54384.36289491021, 84065.98976047922, 39066.84561945584, 29913.412601912838, 38045.995397158935, 53977.25556793621, 26901.444098058513, 72399.33254500313, 33078.19164362885, 92479.41553811496, 97764.24939839756, 47069.107611410174, 14903.792509357272]} +{"id": 1743, "vector": [54917.388823475056, 88367.26391783515, 18515.796834414454, 76775.98164508253, 39366.3375609257, 68593.92287112286, 96693.8497512909, 76490.03000953692, 38482.47155129361, 53114.79316474327, 29212.471484577472, 40991.121588414135, 63164.035208712325, 58718.84268838794, 29545.629506869453, 16373.471071786782, 36081.715284432416, 62066.0458928027, 82165.43181142166, 34745.17461377622, 94542.17904621501, 72019.93630932797, 25916.90783634847, 52197.0722695992, 8748.742950184464, 14057.589602839116, 2677.962612237217, 20456.714212995175, 18019.246694469526, 71310.53965031914, 29700.56730284526, 25216.814403661658, 23505.87514448448, 70667.3517858552, 16989.25922887857, 99140.78167610134, 47876.1220724529, 18914.226092802888, 90824.3856442248, 781.6174925931008, 98692.21640518217, 53123.549119310344, 62620.463083339404, 25308.911041015912, 64411.56403986588, 43598.78767903924, 37404.32309479859, 10050.935259920967, 53631.97275497383, 88438.90626373875, 9378.167557097628, 68090.72411895033, 24825.176011696447, 62831.57989654102, 59368.849881667826, 39960.376454523335, 11720.890977493758, 10028.894079916528, 77681.54495082193, 5424.77026303122, 50352.55866369892, 37951.41959311712, 8585.206470273466, 7849.9528069287835, 77195.909016439, 12585.037952423905, 81127.00671055516, 84610.39666636538, 14717.783996386868, 40220.07373927674, 59053.69311882864, 80683.7358465036, 27219.218801915813, 53131.66614280827, 46633.75065823313, 24821.26792810324, 92.41824065098214, 73291.64030655881, 67414.06360670929, 71543.28598408178, 1016.4227073394704, 48869.469191032586, 43876.953042483656, 32816.57007847547, 18577.691243443172, 32180.54415328041, 36425.31646931702, 88671.57128903203, 11394.67738573281, 31160.885160607533, 26841.104062294664, 47905.75525150529, 63451.37881868841, 15072.004997643251, 235.72512681176994, 58867.18268120389, 36602.33440127124, 55838.80726389196, 12485.903528022613, 72590.73837515367, 49651.53386307532, 38269.8345631051, 22287.44987200122, 17029.678583266283, 68719.46821209678, 7440.21151056441, 18130.215910855473, 72053.45041025375, 54724.50223092491, 83749.51616805868, 17181.777067698444, 82172.0244869711, 11274.23559808859, 67330.96272935368, 7877.533609612475, 365.78934288514773, 41033.53917552561, 89407.18929050243, 25713.984854895632, 37304.530966637416, 45864.50664905779, 57727.77443041276, 68219.3050181991, 65905.9836498247, 49856.88042320902, 23117.870868355305, 57693.549749456295, 43435.582904035444]} +{"id": 1257, "vector": [77948.00819755277, 48049.26234898686, 94944.7849141049, 49220.83414839108, 10304.67197703041, 45402.66152687037, 42786.88048858605, 85270.39451469167, 64896.86992796611, 33794.259577647565, 44238.43680085486, 22945.605246222745, 54204.02765000563, 98576.93726688827, 33216.49756874736, 41039.14516849353, 56822.95345775271, 643.675609533756, 61151.11066324954, 60212.32348565688, 1066.972062439442, 88767.82019932059, 74632.05497694811, 21439.578248057656, 91937.7290798339, 71449.5977391398, 29688.752266218165, 37408.60043672617, 37890.01424311158, 23594.100177553122, 73817.83031585676, 74923.39214411321, 12333.480659887353, 4526.658507419878, 93152.52466502452, 11132.709030393627, 66977.54060245918, 25818.073169872623, 96270.14191299009, 88189.80606995532, 86110.8412796625, 88191.42215409064, 67688.6325771519, 96540.31069059017, 95516.59194216896, 28200.831814676785, 80493.60426352742, 89045.1949236902, 13002.703626508961, 13256.404466371441, 57508.21378944483, 15286.826390443453, 17870.993002960044, 52106.664493363256, 82733.03943025324, 25376.22027672767, 44015.01431277938, 68003.10748161793, 36990.14499206434, 47288.72318500116, 75436.96830408947, 68769.1665222313, 5842.96643991663, 83392.36689901933, 45678.59048580588, 49450.49754477545, 97507.53634185338, 23329.12934512679, 96184.24897862389, 53296.41536242654, 26497.536895778838, 10519.4977183257, 80145.9147563904, 23436.78358772788, 6961.75297615097, 32979.12809178024, 70232.02128258048, 49416.238824938926, 91985.17521910921, 90657.77244265385, 98715.07720472003, 64933.504927383176, 44932.21888056107, 54103.47936738614, 57158.68225782117, 10240.300414167557, 85322.08374907669, 33076.10366582635, 50386.19070689193, 41283.15704952896, 25064.478775040832, 93201.84937816105, 10519.600893773595, 63493.81812411893, 8289.680441395076, 35991.766185162334, 97587.493604617, 84279.78889081547, 53586.80537096422, 2768.044676396553, 31904.823118180182, 91146.04715167651, 4058.0678036997942, 91114.46081523703, 49202.368001460236, 43436.954833173324, 92891.32339276781, 91898.40455398824, 76061.6781224608, 70717.82119367193, 60948.6962144384, 20622.559802305917, 79710.20258392852, 81383.38216728707, 11673.47129399574, 4487.645793551776, 29409.028437638906, 57037.02650440989, 87299.19767834569, 36243.710346921485, 490.98897062880286, 36070.93877386912, 31398.434613142057, 2694.5295262241207, 30217.598894532915, 49501.20155651834, 1969.4584182660235, 70276.98083750824]} +{"id": 2001, "vector": [59523.37921591393, 54395.60785987871, 28728.582396299553, 40222.65217027595, 53832.38551536754, 9331.804390766507, 64437.08036609821, 85484.65141233729, 69034.20641781022, 26056.237442876427, 24668.735940364593, 79028.76538158623, 72514.92022039325, 22382.957118706003, 45725.32528117907, 27151.725839226136, 3094.1820552010445, 15184.797931243265, 26319.053821962167, 73577.8535857409, 41321.85714321919, 95565.84391496149, 99813.27797153074, 33404.816293990334, 17490.313023717907, 44933.96409292869, 36938.38786656613, 23622.385634905975, 64034.80059861734, 63642.06297727405, 26141.896638796214, 38968.76769856107, 16415.55788088752, 65267.96466514202, 525.4200090969218, 42651.38626891424, 83750.08556688139, 84511.50073113132, 54925.71966208573, 77233.51461891852, 3086.772952338923, 2831.9199738301436, 827.2154758375616, 45060.84487564903, 48946.64670343494, 51103.71725793739, 17053.533586037473, 704.9700625591182, 64913.742134477536, 61557.08657216561, 60755.014054245505, 94663.0145534865, 8361.76273593171, 83925.38900707608, 28104.01178015527, 92284.87609143999, 51279.796635697705, 12981.453217993654, 59872.85401388357, 92466.69914037886, 57918.67324461979, 26785.683952659634, 27944.41512258613, 35119.231186796664, 97835.06946904416, 31350.084948916778, 95913.42508418244, 55837.28847694258, 40478.82282755627, 87680.56085804256, 66973.00876412085, 80710.73672463272, 19481.383159171804, 19467.061028689768, 92721.42427830782, 76243.50927994013, 27201.954288663375, 76380.89084628264, 87377.51126700139, 36799.55543471673, 98901.28548096048, 16851.197801959715, 47569.89972448493, 98487.53058362246, 45726.68091615002, 9162.423117412132, 80715.24847357777, 48842.720164396844, 94950.7856741748, 95304.75646032015, 45516.55736118228, 52649.293442328904, 54238.75451727511, 21955.28048561304, 64028.55523659792, 43941.749533058224, 50034.548903144794, 397.2620997650833, 47839.06905247035, 66909.67438315833, 36023.66057989629, 78620.29848500309, 45303.963428045114, 80366.955891065, 4454.740887057785, 67493.25997673604, 29697.5463609498, 98459.17691658276, 21513.704491829045, 50626.82033962167, 61855.39374295511, 91722.46158553947, 24358.923410244704, 95628.8401861342, 28249.114328956883, 59381.43647248805, 67634.78445190156, 32053.364293370723, 34027.140441252144, 48969.30740326756, 40091.23714974684, 51632.12314019984, 5482.994442371003, 26732.491272387815, 59719.465481654566, 92414.43718263903, 13461.625708428171, 13072.332535459564]} +{"id": 608, "vector": [62235.54138861024, 28684.279188108518, 28288.493941058667, 42111.099027857905, 4558.526138441177, 89669.07399396058, 36188.52874887518, 56402.7676291983, 79771.91026949215, 9639.164739571637, 57629.919666477515, 95343.36820283244, 26568.939296039094, 73363.91198137679, 63790.898336717684, 41336.854864997775, 42999.598222085966, 77959.99304490836, 96739.25355926032, 1685.492781448139, 82595.48519790133, 14251.363131659145, 26859.684965548226, 5935.365468194486, 2190.0324356884826, 40644.88809050021, 8393.203410352278, 26799.79248427057, 89410.66247040764, 61161.81130777111, 53894.32475837265, 26153.161891781696, 40416.90035055505, 4843.343863489802, 22555.203458195618, 68805.98495610773, 91254.57355571933, 36927.83303196868, 5399.9087550633985, 14183.189093070747, 10663.990560172231, 97943.46054493127, 36811.03668249144, 86910.06674754595, 51955.46834282606, 99033.90009848673, 7710.486152966967, 86127.42906846352, 75845.4426016443, 25557.1062453833, 99657.23615040272, 81156.34737555774, 91921.21229121652, 81457.61011341476, 51016.44563854212, 6534.2193096806, 59046.65218453884, 87045.71730715211, 56059.03016784824, 45948.1790132355, 45940.455582637944, 27106.632379179606, 84133.0789115584, 99545.983153674, 36835.63366992772, 67627.45855409192, 23127.21156791858, 79961.51512619393, 15448.180451396609, 38599.78267453275, 13904.1483093846, 22153.071126756273, 1282.7759514410063, 28168.864962512664, 29980.44982327722, 75580.89219354406, 60053.38031679277, 47596.969945203346, 46855.94717530275, 57254.999386081596, 15498.250623589882, 58067.49774689443, 4039.0272084050107, 59697.38668381453, 96506.39861086229, 8516.602631247517, 82292.53811124107, 92146.99437604222, 74861.14648926647, 44888.31049505877, 43291.32541296433, 43235.94937124503, 40321.630442630085, 63136.03924669598, 62056.59775510134, 86594.67916125225, 89835.88430174958, 43992.100113821194, 20127.493718924496, 16722.51554807804, 37120.94650471905, 23371.074464762143, 9096.67424504087, 61787.50884506744, 18811.855696288305, 14781.129475725607, 6587.084756501194, 65420.463333153086, 94692.28816918776, 8548.748122083305, 90830.9336439349, 24416.5406104107, 93434.59117278382, 6671.017050906925, 27229.30261287941, 6401.87570064028, 92042.70804915241, 58872.327220159525, 86362.94699179532, 2514.763018013788, 26532.99506936566, 288.58282454889974, 47914.94685088516, 52756.96508522235, 61985.728637378576, 27210.45450729589, 95117.42690721204, 49387.35038698605]} +{"id": 501, "vector": [22109.2013584983, 16716.10616959934, 49846.272637847185, 61908.044683068096, 19096.51305004578, 51786.521972427865, 81382.85761132902, 726.3561887042135, 89220.50112270338, 58584.31112390384, 23481.10297920293, 41526.1128158384, 292.9670988981892, 67997.35651170854, 39147.882675666544, 35065.47951049405, 32346.00397603371, 99851.71007818729, 38330.45057545289, 88047.1916701476, 72507.88487252458, 96354.77424191158, 74280.46156766986, 19579.19348756938, 3551.598705773662, 78591.07305403436, 45830.780186787604, 45742.927629593476, 14689.602527820743, 56748.932962147235, 30690.93969386284, 73004.43460545191, 44278.97170477112, 70784.77510943498, 52064.0000976551, 55005.678396384224, 9889.727313796891, 59226.579158046334, 84100.33724978207, 50307.09378494476, 89200.58981279617, 58433.89455289015, 69896.69748902365, 63846.89315302793, 18675.886586126646, 82291.78189760588, 43844.83815084685, 51764.619098393785, 69269.63471850073, 60189.23794608303, 53011.68632604026, 5478.629100890397, 80140.56641739204, 91944.51257535831, 56669.84698866933, 26787.666625793583, 36255.58622275906, 56294.39976663093, 60765.38196891165, 43319.82652221842, 54204.147721508336, 42729.92268895893, 14906.999301371381, 99994.70091942183, 93218.7760625057, 58298.87500946978, 87351.47359140028, 40321.97655221508, 85086.14420581567, 85738.5916345764, 76972.96599413293, 68221.61403448356, 49086.09637938935, 29374.701972313465, 77410.78315930095, 11516.896323504145, 19.43761946557876, 67224.23860958415, 85759.87195214049, 75589.21829484649, 47085.20387707316, 9356.649427048136, 20279.003882289493, 85142.33461928516, 4136.03606510069, 50908.02599599371, 91122.00863152012, 6970.951055823893, 28120.476986362264, 11080.448611455618, 46970.45580867241, 49887.30226432818, 10261.267114671968, 74063.0449308721, 5707.804900550717, 97799.90803392493, 53664.78353272793, 37260.50253144255, 54748.88027710943, 39020.62131833913, 97784.93759845967, 52274.039335488946, 77708.75562134209, 76450.6592894624, 5501.796292627725, 69994.48126432253, 90437.43734425797, 14096.34020772198, 51379.8051810047, 74573.5308376055, 43443.09165168299, 25878.98670723777, 76958.10688831394, 65358.6661710584, 32728.875120031687, 83445.0568368887, 39378.543525329536, 88326.36884548728, 6222.817739879716, 92609.67930138859, 75382.90650202634, 15347.236281501142, 92580.50524101117, 32537.98097319429, 61880.12141325402, 74666.9731812824, 4633.954427065334, 41144.11731981124]} +{"id": 832, "vector": [72603.4858181977, 91405.61111834514, 58635.59166422604, 83209.20412900353, 61367.28222087984, 14867.777819587569, 38043.493337376065, 16831.98471533576, 29373.391173965036, 91189.70216686772, 54904.214837644206, 81178.6485809915, 81911.7552973517, 46146.5743988682, 55558.32559944568, 76609.0223888244, 32737.05540486326, 94727.33663130132, 92064.07003171447, 47429.182971085116, 34725.07796618802, 80529.36978707726, 50478.50138754768, 63021.99691253299, 75048.95975695342, 82583.0186055351, 44224.727739765345, 21825.038415155228, 42285.61854096804, 34816.32470103679, 87477.258384322, 40185.26773148654, 63950.66357433808, 49887.60659101642, 58825.78595230992, 28253.60185520659, 15354.819206918457, 67968.6492004955, 71321.10720444002, 80071.62428530207, 20651.951466115905, 38654.193749480815, 2459.9221504751913, 38840.849810142696, 90514.6229679074, 55890.345577214684, 89634.78494384435, 66490.23315338006, 66121.93470698183, 48896.54128707661, 23734.294919682365, 43329.97503633896, 55386.69245354195, 95856.45777422175, 72778.65967277299, 3756.7337797179357, 80462.3870147064, 24583.64420788044, 456.4386416177846, 74094.37430613699, 17488.907985600934, 67210.46639744146, 5315.240288961442, 32367.57520843333, 71281.84351681938, 24133.351038151017, 22720.591852184203, 29963.883412639403, 50437.398118602185, 76433.27487221887, 50658.51341595226, 52276.03760348963, 46691.23561425058, 17022.139377908054, 19549.268272960617, 45943.924679550626, 63097.7534131854, 77635.59728645388, 57362.988923728866, 98316.54933008773, 16919.622658924673, 57803.817668204094, 61127.02117617526, 44022.07662321681, 4684.128503535834, 92603.7865048578, 83668.58168776252, 46744.33541189805, 38130.0139803214, 90084.38332874601, 88888.34716527943, 66839.68432700603, 69217.32709447724, 12646.899496865693, 93806.21512132799, 58471.74716317679, 87025.0969625147, 74802.96425620305, 46988.713599101626, 36192.71044431211, 87496.96428466296, 41354.726554988665, 80261.48419469396, 54214.623002798835, 61769.94334828374, 57047.305363590436, 17618.65379310775, 47764.942264713485, 32527.563839863695, 46291.02971902451, 13688.645240217378, 36771.65235165174, 48069.614175013594, 15957.557192275528, 13401.535791658669, 25071.402762791262, 38376.1888561364, 72515.32861483155, 11456.811801156808, 1157.3886024023316, 7623.562534385231, 19054.60300110492, 94057.06681805305, 17316.510376334583, 70445.22070148522, 1505.801596775369, 68809.78795550612, 80371.10732142488]} +{"id": 448, "vector": [90441.93289278728, 61987.78164273261, 98454.04885240432, 43034.59957615165, 99894.94111725464, 19708.44445487885, 93923.44873355218, 43277.143742585155, 19009.92575596788, 64279.623913747055, 80614.53205712682, 93682.90883989673, 9758.722781627039, 72810.72657506385, 15653.515241619021, 75715.96778391144, 91795.1086569907, 86512.41303796123, 37291.27521412923, 29088.875614622, 15452.878345369247, 75442.30411764972, 16922.5777275548, 90930.22415996809, 47927.81785571844, 38925.986750318545, 31367.36820531383, 52579.405881292805, 95640.4742686451, 58277.713746611815, 11474.502107202967, 53694.7861763022, 67117.59765067714, 16108.359008231753, 30145.892794832052, 26336.336675866423, 60715.451178875745, 1431.5648478625765, 16278.62706155323, 14581.46912043008, 72813.1279627842, 49784.922037736826, 63155.32471732968, 224.319572466547, 43476.30085864074, 67461.28578516071, 67183.42347522642, 90315.94345299732, 73355.31830844714, 87602.82108313103, 24966.34562894108, 16978.8966749324, 17843.61984435654, 13397.3640338387, 99196.09387566053, 61548.07661071238, 58997.20587426126, 19900.61961131032, 88161.54151360432, 65348.686765342114, 88031.53235787952, 59378.39312570639, 866.9963726380403, 8164.085563078172, 627.7709164028589, 62161.98427755724, 13977.843209118246, 99181.17667732178, 23750.074577971158, 66850.40694183453, 41056.329030108485, 73239.94577403915, 7815.461755410047, 31844.05620098878, 76585.95836310614, 84459.81153165881, 71383.03646819349, 61031.70634240381, 80097.40503394224, 64237.46204244972, 64731.04866079631, 63980.126220774866, 93290.98227031973, 80597.70359608985, 75800.31846137617, 25981.758668579536, 30941.89816982431, 57175.27106034733, 79667.19354811378, 89509.63403631333, 72713.23074263406, 12164.657533358237, 46170.81631538994, 39735.792313007434, 13957.012802796287, 50415.34138084437, 3107.5681148266863, 33114.93205974081, 20272.0085680423, 54.49743585227429, 26565.736551150752, 86882.67874734131, 38594.24812213249, 86751.98317727144, 99943.59436219317, 28547.144899502953, 96002.3589895672, 26870.62477446338, 97719.2274133863, 53679.53543601004, 58054.35354516546, 74180.18314640746, 53264.78783997836, 75897.65999485708, 5595.0611146602605, 37383.20033272029, 91941.75717528332, 94555.07776005365, 91861.35626632496, 85495.46880235833, 19519.129086666642, 71351.81952033033, 97187.91812024312, 96481.7583968201, 55535.08634211777, 39557.631881559544, 12068.975209855003, 35660.39821625672]} +{"id": 911, "vector": [24050.261527519735, 75846.89158357326, 53593.126839602, 600.3342875480544, 59831.16736696717, 88158.72388418106, 2550.970660083185, 61143.535910223254, 57415.11768701313, 80182.88230807624, 93835.78323904089, 41988.10344067753, 99838.03192821745, 11274.693982127259, 48109.06102915795, 88773.6824917209, 92690.06931752124, 52910.61552999584, 67836.1446918842, 25518.199644683602, 8428.217648224334, 7011.199553646219, 42015.14309540372, 42111.463639067835, 29529.44280162736, 57337.52631289394, 76344.5610120606, 69951.9991713559, 76450.0601833859, 28809.59804003629, 10374.844386921923, 27257.736651938158, 98452.04651267343, 8020.615937038722, 83801.69864845368, 46462.69743082097, 41323.581760030014, 52124.11742360716, 14049.268464617482, 55492.527175447, 23997.946375082967, 31843.09973613456, 95381.84578066715, 69985.3356601341, 48533.209594352666, 71450.42460732294, 49962.849557954694, 4660.810135497084, 64497.31172889147, 88183.60680003697, 16831.806724043196, 84437.29027966927, 19407.11341290724, 95679.70467490428, 81395.84277815458, 27136.74544670096, 36442.671139281556, 25052.38114329521, 38086.17330149205, 76113.93515325527, 81684.65359620494, 45612.48717167139, 89506.06721928144, 18158.440987440383, 68314.759736686, 99875.31926922537, 46448.60081837556, 33808.89318122915, 48125.18435502038, 56621.35144768049, 70658.19814527505, 34392.232734118414, 14393.724344340264, 20988.453491494696, 28136.255415558975, 33395.01828244644, 15506.879229068094, 46038.40249631253, 51036.43689765234, 35537.2326074896, 43733.74061523865, 59258.63437486426, 70221.65688745624, 68233.60106978336, 42756.06361337282, 6281.708829805543, 19880.856378069755, 98584.38618584507, 73122.72007045794, 62239.5814707731, 27009.64385612631, 67207.0452504999, 47101.42229981887, 99652.63390059299, 89736.41430199612, 90847.79375124267, 15097.594364678935, 26970.29554416216, 59325.08308583897, 37742.2142604373, 60090.67277511848, 35715.79718983875, 10828.42552841734, 39578.800549099025, 37722.93408809998, 59175.10602913094, 61884.269032097116, 90194.04827538606, 5860.219515683962, 52288.18249114858, 90270.75691426374, 34091.60121801973, 14773.16175472545, 43968.57392096773, 87655.72160450324, 44689.76841620829, 68033.55710732313, 62251.22529131767, 2640.783844102823, 98368.49345707687, 1223.6682362804063, 96355.12310317945, 81858.88589829174, 85730.38156935165, 93221.47348551232, 93019.11666096988, 59857.36127685354, 85936.71688863591]} +{"id": 1983, "vector": [64478.69407310053, 69282.59780894115, 55897.230030726234, 49511.61722945956, 90789.08510599361, 31625.000284521997, 76856.69675934342, 88462.02727871228, 93888.54401852976, 73118.01296670845, 67937.97454193521, 4781.970132423474, 66932.94486748337, 66001.09468920807, 3241.348005918865, 27798.876247264692, 21194.079377092035, 33765.434436936404, 28152.41584005026, 18396.29007975294, 12604.667744715236, 34719.11131552835, 10263.227099459527, 40829.18669041051, 73115.38403327244, 35779.35018625381, 50456.5556890855, 26380.428241898568, 28687.94655869579, 66071.7146824722, 25787.50323562943, 61926.653482124006, 19709.426599868984, 73381.46702926783, 68105.97173824767, 38722.93973601034, 93786.05448905699, 46646.28810676862, 94460.91340904363, 91612.81049626504, 39487.79077474217, 64709.37333393412, 27475.509876448144, 93084.58245933436, 8953.793738309112, 10586.818157713196, 96130.38786922667, 11037.092495096345, 8086.481419846259, 34468.40650674287, 72809.26766017578, 59290.90988872146, 56119.03499897407, 54018.337822355425, 27083.033663221646, 91104.60821354786, 42818.88547165118, 950.4581420381796, 14841.724530545696, 1629.979935848025, 37081.521977708864, 11852.406367290147, 13962.709427579533, 16446.623780599555, 65417.614509082814, 90264.40764820192, 24707.465412480724, 19006.38905066889, 69355.8591840329, 67507.7306347715, 8056.5215908549035, 49171.15487669568, 1859.7323301203783, 68621.18712851082, 99807.7611399468, 17647.096681967887, 60951.80542093712, 12946.54806639366, 11206.57050097702, 77705.22692106677, 4441.068048386776, 60669.17546479916, 45383.919453001574, 61018.64795173905, 34377.52643757172, 11651.138188673782, 9595.00513870133, 54072.23847346711, 51655.794621571484, 18572.6450687179, 31915.677577089584, 51624.37186560835, 58665.19947078807, 199.95723209418338, 23608.575033122725, 47878.803785058764, 87651.36002384455, 6875.085485426491, 81210.90634503725, 90171.33725758606, 77268.45826489729, 15292.119367768397, 52898.51248822089, 56337.137213365815, 71832.54964185525, 43889.555235161395, 59059.44871685594, 45776.786760752664, 583.0931456588751, 95476.36970890376, 25766.53842275114, 89009.10509251745, 55371.235693880335, 38917.60713272719, 82346.96102945066, 53347.16044888883, 76210.99122019655, 44425.884979785624, 86883.92100402183, 37393.49924948571, 81568.17849602202, 57428.08136300549, 74200.78696534708, 18547.21249807918, 13813.697426949135, 94195.35455452664, 11672.853910367332, 16426.678836062492]} +{"id": 903, "vector": [76381.3006064883, 56795.35205457208, 98730.82762721703, 93760.91981428929, 91084.76646054188, 70799.74078311957, 57719.642469626655, 70243.57386369472, 29171.578572655955, 2496.9416454666216, 89959.87955433446, 38836.110028336145, 23663.37055091543, 5463.693141850379, 18576.408436740843, 32258.651926337025, 18423.609517726956, 21040.10722808638, 53065.172495074155, 24871.55248656394, 68734.67583016088, 79290.28528186532, 40117.13179067801, 81104.33496256756, 11982.354554053454, 44348.82998823425, 44438.80192151548, 70333.33247759784, 35342.63195004862, 58297.35425759951, 60227.629046576134, 22105.216034572084, 33904.19110642267, 4476.819008543842, 24844.405207937558, 39254.792063688496, 37848.3402231474, 11403.743850692605, 40287.81920617357, 13829.984106223714, 40178.617005330365, 10573.781680322514, 31312.20565777648, 24528.62051595267, 26376.685749115826, 96096.61088635915, 68494.58273727335, 21053.177356501783, 19816.31955372073, 85691.90845139023, 67677.40044888988, 2892.4275486588713, 10903.754307390212, 26681.63768371711, 87249.21040727853, 7225.184637859516, 84395.50444151666, 54480.40776711478, 34531.427112627345, 23237.812843982763, 89628.28059418689, 40653.082751357615, 3522.444587414997, 63954.88618239027, 24053.31157124635, 34696.60527422999, 25055.72768487504, 82287.62903948569, 57541.9980961158, 19295.509878706995, 63914.596158896355, 62505.494740578346, 27193.588679908775, 36868.58556613957, 8324.510772232563, 35853.40267401046, 87142.18287968938, 58093.69056456821, 35998.485560623405, 60986.82744220646, 66994.63456347519, 58768.84855705576, 46724.56247203827, 27804.48429979426, 64505.16648071829, 15662.600251378, 80444.66831915038, 97423.2834561242, 89641.63650793486, 40788.15341923827, 44377.797923967366, 15702.320094851973, 95296.40970557083, 8583.54789663015, 10590.678589875557, 19611.25460368144, 47362.81336392697, 63630.38872242317, 44538.76916813968, 62202.46187875101, 58304.960714870416, 83512.55318386003, 97664.16884628488, 15564.833105603582, 93647.81231441673, 70535.56747184719, 14967.021752995213, 56679.76516594063, 57035.67207462, 37178.22442709906, 8688.221277675979, 89985.59285085664, 70747.8289668822, 45592.33859903779, 16451.91596104568, 50791.293648356484, 836.7634119658795, 59258.43023664468, 45584.41530482623, 40661.426177498506, 27548.768315144134, 91512.63753018517, 45305.40153394823, 17341.0604997723, 83489.31564554232, 82147.31845281295, 19455.15186878718, 74151.92998713724]} +{"id": 937, "vector": [99301.43133751485, 28063.390488304507, 16111.594455442935, 34364.77278720802, 12845.011588061905, 96688.84904784543, 34119.77056897066, 61867.901565518965, 61517.480277443035, 44504.66410993702, 13633.63605820428, 97993.49747717955, 83214.87954123516, 87689.85701741127, 81259.2982041227, 25859.13403903446, 53323.49566701543, 17552.157796460444, 72137.0315048865, 21152.74160632087, 52712.862723682505, 26195.004010023615, 81351.10880696749, 74691.66541565555, 11655.13204277282, 80528.81059560284, 18942.7673943113, 91213.7667079237, 702.9245670145401, 32717.123917445122, 12213.330555214076, 33276.168160415444, 74386.00032987622, 4783.495035963614, 1109.45753368048, 36640.81871086482, 30936.18611088239, 62619.00591493656, 44157.09982678307, 52706.42178018438, 44859.13685374934, 91064.84649893212, 35036.258737018434, 45745.28582285267, 10428.604767194705, 58379.82926266284, 54404.476890342405, 79138.83878143026, 13369.98599165351, 7790.942627034136, 84906.5912473331, 53606.4077629535, 19496.581611076657, 58364.63199197652, 12395.66921145825, 78601.4105902319, 92203.89031455101, 76931.23949087452, 6988.688122639664, 16292.820944790976, 5043.079054294452, 44322.28380769132, 46811.09349449756, 84628.03239179925, 59743.8517720182, 87963.81816834908, 9434.26750616293, 62620.22981583052, 11516.598920855115, 53814.9209545343, 17663.763401189615, 94767.00139866477, 34591.21848622082, 79704.59753453534, 17029.4958190634, 73899.02824229609, 5412.504520648309, 18016.66004486302, 27919.546456367916, 36104.75152574624, 95390.71597534648, 87697.23820116425, 26889.78667564409, 17102.649395035318, 75219.94200740822, 67221.74267626014, 62650.35064706765, 50124.130637200804, 31342.75262211287, 42126.34372807974, 48700.27609471376, 76490.03899209498, 24092.06268697577, 92969.54669577508, 21839.3720856505, 55854.33578285749, 94906.13126537064, 7627.606340639759, 37172.38167828689, 33088.736383526455, 58092.56997889736, 87918.80600317383, 8223.953747684343, 53389.7406169863, 5065.162413076973, 36340.488685076365, 78802.09210023028, 74595.1043434217, 886.709330739277, 39937.947801219234, 47583.30588435663, 17786.481870499083, 74792.53192907029, 54236.41920562068, 80315.34868152179, 85182.01648491628, 97801.03435671992, 77495.48865285622, 48579.804900550036, 17794.16122530819, 21186.94206001356, 66185.18930949472, 44260.40373294139, 26897.600510466345, 79947.7560321159, 15794.443074073493, 43005.42859347563, 97726.01064500622]} +{"id": 1214, "vector": [34220.66344388187, 59691.388562700966, 79824.91074514597, 10902.30886392637, 44211.2275079255, 51308.31984520976, 40041.40110008246, 30331.23980924446, 89549.71066353894, 23288.6291429128, 7541.325746210703, 79761.82935875464, 51871.81157960522, 11306.655913918528, 83917.05977746897, 1690.1103913388838, 56970.420442672454, 95768.06988310018, 70579.75300145902, 81124.38712943773, 71016.06562675616, 69884.67467439541, 38554.408162098494, 2990.4457480700585, 43507.44042861748, 71861.52380164409, 53285.97298074073, 54470.89114155077, 11787.339333040281, 25342.355075388055, 54164.942307945064, 75012.34244416471, 68923.6105104521, 7834.372848613902, 6954.2919507996185, 95890.88062447734, 11449.928358245998, 11345.032486575425, 43773.61621973041, 52748.04786859585, 44661.78435257151, 47215.70041813714, 7165.594210683279, 67714.50208995934, 55241.9356402559, 27657.785178567072, 90979.53022881376, 79927.23838699715, 83382.49561582762, 34462.43650274823, 6358.249559816609, 4960.291921328508, 74944.7509315906, 93391.3944603919, 48547.75516334889, 46203.79849267429, 7495.048880216637, 30934.157173647436, 92391.43043244338, 54971.24153919321, 59537.77990419912, 54508.8649327246, 82559.84197639953, 39410.0833701646, 43600.1332177675, 62017.92824403619, 69508.64500240298, 81460.24987051736, 71192.30708922065, 95374.31643239589, 81759.04462496666, 5934.475873768819, 54213.75449542106, 98327.45208959721, 18571.581229656887, 25224.422990240502, 32485.34339037038, 23432.81007044169, 61772.884116452224, 93820.73140586948, 49624.804792256095, 40431.88444539293, 73926.3061862751, 64296.40553847045, 87411.12614838887, 59928.52000182122, 52457.349308719844, 69857.80193430163, 71185.67848423129, 54468.97179174743, 5200.768099744302, 84263.47040483484, 9542.940902518392, 23300.48156987008, 26283.820677991152, 36882.86068724383, 24097.036613269916, 24307.43912805676, 50544.16975030842, 61351.50440544779, 48284.11300309359, 85861.0054104723, 1150.5379461371424, 44606.15854523844, 77095.03980564863, 73677.1189591111, 56257.83680195901, 58940.24976060198, 35906.352783928894, 69191.25207022308, 19876.325298032072, 19623.94366355109, 17985.96073597264, 54926.65704833457, 15514.484599647581, 49827.36900867728, 16206.372730217123, 23589.191475657834, 74656.76650520432, 75715.25802694788, 21840.852072292706, 70979.4390697128, 20444.164135289655, 26392.644068008663, 83488.32754973165, 86520.90577119624, 62481.94910802689, 82306.7085707068]} +{"id": 254, "vector": [29632.995466315548, 63528.19351993827, 69302.8515697144, 69567.00657726321, 22059.82075845745, 8366.177721530843, 36317.34811438364, 36604.31955245444, 9138.682215312721, 14409.352717349777, 78679.4396137722, 88212.74292777454, 92933.09943904649, 15148.597045278211, 21228.467439877775, 1950.9658851360246, 94477.21032975608, 31932.02576767439, 16413.439111045504, 25788.8851710952, 5910.13870803444, 86091.96496561536, 34293.97892328089, 3230.7477553194076, 73475.07921257261, 70195.93005355421, 80521.1636591346, 263.9155829989748, 63637.18792565587, 66476.60560333799, 31641.79099127624, 32673.88741698788, 45561.363842481675, 6560.012698062256, 96607.18371071109, 24972.43257016757, 4558.806099047485, 46575.54184705974, 83455.38937842975, 56038.63210589617, 78117.24306831895, 22432.454597451444, 419.160748355929, 71220.3173541965, 99009.47244776115, 92728.75342651716, 57330.69775946763, 36829.07327736217, 23540.110296595118, 91919.42023404741, 10912.29276316319, 12757.86005048236, 76729.34669498914, 28086.615816021676, 17298.042569753015, 77463.87464313212, 90990.94585542298, 71194.18208590716, 12886.112690330852, 54524.62192234538, 67255.70386531718, 57990.37231672345, 79953.95847034278, 25554.92785232636, 74728.9809466839, 24523.025106503803, 46035.594668160884, 58602.21944280931, 47312.74451847581, 84466.12900118755, 74155.23601131784, 84152.52096967393, 99609.57980147637, 79472.49775801708, 40669.34395570303, 4349.571369121197, 35447.78651202337, 61622.77542693285, 35165.523217228765, 59481.392649001886, 6444.38482518801, 16923.5208853324, 75301.4667247928, 88150.95092921094, 44700.969220812614, 50402.24739101418, 16813.292073690667, 8184.581852285977, 70903.40938454206, 75747.24965543924, 24931.264038772082, 23309.943248143773, 90380.12792597403, 49429.88229916817, 77479.54303068343, 66499.66377698885, 70023.97789491316, 36627.2374819123, 99353.07762332809, 9294.991875174275, 66913.10548425985, 38428.51166520582, 43392.25805351836, 24101.12834840573, 52805.60196904795, 23726.826465331407, 45013.03921647807, 99558.6056920456, 60123.561241496296, 25703.855480021586, 2305.343762370071, 91120.15866045868, 54408.79332061007, 83329.47433274404, 19743.15087854003, 37803.632606216845, 98807.80820301757, 13529.452942391552, 89506.04638851857, 95588.95444330049, 65638.2423262325, 27890.00559792869, 19750.82411053861, 70816.29732433843, 2513.1799004450727, 37885.70847264652, 47996.4949786529, 8937.454787474342]} +{"id": 1746, "vector": [30658.302482962506, 19708.437505994392, 97668.849665993, 12389.943439708939, 70613.55337938969, 58157.55033701644, 47555.595243357166, 82768.74078678545, 2585.308272087128, 64419.66647111525, 30595.277828652655, 82598.88023642905, 52721.90830811187, 97870.89025425029, 21911.677783620642, 49147.80185638162, 68476.04343535544, 22309.289085413307, 19623.498556731534, 53806.983832944155, 98590.8439269711, 20309.522111263468, 62741.29132497548, 52482.569978400104, 51539.29949312561, 97141.24863712813, 97277.54977735289, 82178.07978460846, 4778.044748563692, 52648.03423292036, 86747.24948532115, 43370.3591974807, 7935.47027661754, 26092.151512213237, 95430.4054903658, 88528.04153551122, 67485.10425478476, 95848.30595777837, 24679.028196339303, 57564.088346876, 10491.25228026715, 84676.96773033903, 42995.40815699477, 52419.4717943543, 88594.06958475815, 45959.42677838673, 25885.39827238545, 94952.25399524486, 14430.702633738545, 58652.34941460137, 99485.15465341679, 10150.915599917565, 69472.40208837073, 58393.4677526958, 81269.00695152605, 9016.961107405386, 59704.17823433746, 82265.70684930528, 93295.41275646968, 60603.24807768235, 65939.16874145957, 62146.41643228406, 41948.566968597435, 85764.63558799677, 21216.467435626142, 12307.575276697624, 68992.20000674459, 84647.7069403344, 95370.57557939482, 23721.498721017342, 68086.09490782989, 78203.2761431761, 87102.85204112437, 73163.46521379171, 7255.791802905098, 48733.631707191314, 86465.1272390508, 51511.334475676704, 51492.66449206291, 5210.9551037538895, 18456.62203838181, 6460.912464686586, 13292.453459344266, 48835.955333543956, 58332.562245904875, 75666.48526567721, 54985.31152271308, 10310.907741576824, 37189.72805807215, 16981.46556696447, 94257.8562022336, 81404.76137978649, 63292.75963140137, 13853.104966814788, 33311.06181500606, 50871.9028688712, 29744.228032839583, 92321.08640225207, 66212.80692747782, 19684.475998823968, 33188.68747297436, 73632.07950849502, 5472.625253504349, 80267.09959519477, 70036.18562412387, 2565.6916092575143, 51167.28631229335, 70702.63031800359, 78183.43198433034, 41464.282626780594, 28548.42200225777, 26160.838723352885, 11277.171784214746, 64509.130936277084, 84466.92915882614, 32835.63208810213, 85480.91060931154, 56470.99866644123, 3769.958134414819, 27388.98027677501, 73740.19730876482, 79067.7123087363, 64131.21850362159, 76547.15581947187, 33688.45255642552, 40747.303287488256, 5107.333290927208, 85114.70973466145]} +{"id": 433, "vector": [59402.046213776885, 58419.21675596865, 89839.7955226798, 13470.3401679377, 39612.262025345066, 90248.70798477209, 2093.964410094107, 82701.69332497053, 31315.583904812604, 67388.27134647116, 82254.8779071863, 16574.90043695622, 80484.11311095771, 31812.73051983431, 84657.00771273898, 15052.770216816747, 26880.247793965627, 16909.476904333966, 42368.11061583916, 75488.41263506147, 88439.50186864407, 61856.87611102726, 65458.19351498656, 43556.11817910282, 62942.53489624051, 15929.06608549205, 53910.4153740261, 59426.151350017084, 14254.921260014675, 40902.99487011184, 52043.66159996495, 84759.26944724705, 67590.05642852154, 36425.027286442746, 65948.74697807158, 35022.710578103666, 80864.84209283645, 89389.62566906466, 48708.63744778609, 8361.813460713418, 49353.58386290909, 16141.824715204944, 36772.166135855914, 55061.360184284, 8249.498790126476, 31903.1833846665, 28412.387372565918, 59690.18692027269, 367.80694257702027, 46929.802165907706, 49580.39432517984, 28919.606107738593, 18578.510204854305, 42151.58235341648, 64207.819298977345, 64702.125151023436, 27393.93903868309, 71434.97402973834, 20935.408779951547, 78263.00031914462, 50658.70708468001, 91747.21717193456, 81377.93915777968, 57378.74351067217, 29446.782529634853, 76283.16207397381, 95227.52350015452, 49525.576481375385, 449.9608621741147, 87143.47353231844, 45195.42825165984, 6948.360363831974, 70302.37298018995, 55541.98031714784, 98393.09492524849, 7751.985117704485, 82480.49406960803, 86963.47895314114, 81872.80944526884, 21249.0377994784, 6384.52085635981, 72091.70233030671, 75682.83360834283, 17633.967418108467, 17732.709237201037, 94623.14111189409, 11136.308435254694, 93331.83514632452, 47708.413524696916, 67306.13891969743, 58523.200489526986, 54705.12024286145, 75978.52615183979, 47796.25463352828, 88027.73013341118, 49184.501922972726, 93643.53902815224, 85554.74306654609, 36911.623010594405, 49656.57415568084, 35254.65499070899, 47488.70297696695, 13811.451994208068, 83853.58486217388, 53375.69111782412, 48557.750715329006, 1797.2258484060033, 40983.92524918199, 37793.23054434262, 8915.985443674646, 50993.844766666276, 24440.221133394625, 35457.17227669935, 43030.09343383471, 17179.872009255847, 80934.05086390887, 11581.572958503528, 72427.7315191492, 37550.54531161523, 82215.05354496265, 48334.108172495435, 168.9420364648586, 90297.64030927938, 3281.9533232710805, 886.2395556697544, 56423.95133538173, 26763.313890746365, 87848.54567510975]} +{"id": 267, "vector": [49771.102829818294, 53438.22093486682, 87857.37554985986, 92982.50276977483, 75746.11417929818, 65291.37034421075, 11578.216559407507, 40779.01852771263, 13179.040222269045, 79778.41210478787, 72798.67812595295, 61401.80637741221, 8289.479940006273, 66579.96032444802, 49767.951556995286, 45792.452753637415, 51815.78819411426, 30885.810417951863, 57654.95612553013, 60317.22587789709, 1804.6977153183375, 96734.82818240825, 32634.968228033922, 41497.26331546671, 3756.8253907279804, 47359.99042532258, 89025.29657867679, 49197.051501506605, 27619.499121095392, 92087.85632506294, 58187.41038898674, 14766.329368525421, 30267.749934453303, 95254.14951865766, 65838.54160222581, 2120.047013931925, 67687.0391171073, 7756.757604439668, 33606.846111917665, 67352.57344314974, 80270.4437908658, 53069.171508004845, 28054.147276536267, 16727.162833143684, 90394.91660188018, 53601.98080005253, 57153.371081382706, 25304.91205576577, 53358.1773170023, 89500.13306863495, 84113.40026134228, 78935.9760470589, 58649.83636147234, 21974.9592874557, 51438.05250131348, 55189.80756623311, 36671.025438440185, 83795.08124410454, 90143.56630383209, 18969.668616863244, 73540.22327081524, 35621.20591116927, 93769.89540748789, 55431.021689179026, 14177.013349323886, 57037.82939761375, 11710.063541704541, 18971.073856060528, 36280.38069733781, 6994.475809246259, 50257.640651609334, 22277.370833736608, 51053.21001155929, 98775.48182190541, 43753.179936979526, 71929.29635796911, 78693.71189094002, 41007.57359337707, 21621.919025143234, 63101.16090920622, 37407.56024903791, 45809.84903145595, 26395.554062737025, 90751.80735843253, 15017.490255718014, 90257.77405061269, 92365.09755660058, 9138.190747526663, 8722.957636717132, 16126.66103864281, 85005.45344202245, 34030.22186621757, 12861.019743346047, 63167.72478366155, 23920.707582940136, 75989.90609011847, 55414.274943774224, 97829.46746261042, 1100.611015791453, 2974.3145066113752, 55308.263417320566, 45069.582539582734, 54332.29140503384, 47239.34835443495, 49724.66039285585, 82630.80852694297, 4260.59455105019, 79442.51796477483, 41060.57641385, 55558.05500038068, 95904.9736538006, 53187.04323838493, 86789.21048968784, 94841.93062760076, 97063.73181144964, 83760.61121916879, 44082.721078604256, 17238.926205300777, 99306.46723195347, 90978.11398971082, 65784.62715327278, 55539.584936521256, 24921.218273732295, 87134.80180452311, 66899.99029592208, 46279.7610584153, 61003.32436909269, 14397.133717875544]} +{"id": 1857, "vector": [32749.17611595868, 45960.095774762085, 91488.62747848596, 98173.9344025078, 85080.46755393698, 45646.37394563655, 13710.206504148147, 5891.317601659851, 64713.66840462635, 74360.79935904218, 64216.63345261686, 23169.924645639105, 78094.5818976991, 75484.21842537103, 96068.32574201627, 59183.49786435291, 50997.90525043198, 7788.003818631894, 50262.51316906667, 22769.089606741, 89968.6026503941, 47304.24241322759, 14395.47616944613, 51915.080801792414, 46053.98467605833, 33769.62441142035, 33233.04291339004, 58303.20760691169, 58933.99088464969, 40961.04329065582, 59191.09292843891, 92073.63066952386, 44003.350401505246, 72274.99642464872, 31500.125927641155, 25605.510346975425, 43977.585364750994, 90880.6315940782, 13738.792572202985, 91686.50857079473, 20024.32697231652, 47939.6294224689, 40338.25103018833, 93183.13099034542, 87821.33166596189, 75915.39604078249, 1567.7146292763955, 76638.93193884725, 74379.68670110956, 72798.79304000332, 64945.25980483569, 78417.84782885737, 41795.44524592152, 43381.95656866246, 80184.83439546547, 97093.75614322115, 58651.44071415125, 85495.23153176154, 56963.626065836834, 32262.459911946884, 31710.60770014693, 44214.833329553694, 90006.65357366712, 28149.323819889592, 90279.99070195637, 96638.8706102417, 50207.12574648349, 86315.98191237035, 44089.32784191786, 88920.24144473865, 35911.51451601937, 36321.08798226783, 40383.31152261559, 23008.57722043935, 74166.08624396448, 36834.693598018974, 45726.471380620205, 47963.4162905049, 31012.052641604492, 66563.43399516758, 87236.54947049645, 44736.57218667466, 53358.211542704026, 84862.6675167663, 13857.351058360178, 2742.56750989732, 12686.220644112578, 52390.58888213088, 87052.34549407876, 67365.33253011625, 89940.97257365509, 3268.972794217062, 6781.625887434184, 61818.8973049704, 20968.53454950832, 37963.895066861165, 84424.87591317754, 50769.186058339714, 17887.08447393318, 10392.21939028947, 24101.48062941696, 36631.07402239361, 18276.237534935135, 32892.80763177205, 69219.70584747467, 58757.52378073892, 88262.85791467276, 21802.453475959803, 31853.407636131902, 9746.423810752924, 86059.77400266525, 69772.177827498, 41752.38808686004, 43406.24737943931, 1826.9170036387727, 85124.87117733287, 74928.82946284996, 39048.77019950363, 4903.900034044462, 5774.156163647426, 96718.16391285732, 20323.799476136617, 96466.25110467794, 38952.19371309283, 70298.16335732244, 64531.49772778803, 25063.904345004106, 28753.693883059626]} +{"id": 1376, "vector": [58121.99398810761, 9956.98935492514, 20671.091466287195, 35544.693308463524, 43718.205094936224, 75625.74112315572, 45215.14404433369, 30219.08395621764, 27737.206955700487, 25009.97232152814, 31332.283496654723, 16664.86873810169, 73484.3310895197, 72951.13596624814, 70671.77340392087, 14632.46474109896, 2055.650110309737, 39611.79340567218, 79804.65362051605, 58172.90253107999, 23534.424382877583, 54351.055770065446, 252.24877069643935, 9448.384522652408, 4666.153051301058, 41382.20924213032, 14035.316036557022, 38640.75548974082, 80577.49056495691, 51667.3058069687, 46960.81569679731, 48192.15647214038, 24466.189595342257, 8553.505139866224, 11405.362295962896, 32096.88938903733, 53449.90131117142, 65782.02319434677, 13905.820552051951, 15660.905366139577, 1799.5662763335197, 39136.05359481057, 40519.199532606406, 51714.36274318245, 21732.907022893534, 79823.3512182314, 10178.94248924568, 74327.43048912728, 18117.777692931624, 49936.09785714419, 77149.2626705237, 36366.03943729297, 97265.1635866425, 46954.00434005745, 78892.2502567159, 37793.01679563972, 411.0945925674292, 91085.6600558742, 98.3456930651161, 48988.95344986745, 63959.754870845594, 77154.38809358185, 55491.32860992116, 91821.961170736, 2253.41808828049, 35095.384939784344, 29165.921715929377, 93231.2651931908, 71836.05464883959, 91989.58968519146, 69215.62440287105, 39336.32017835235, 7393.541810392146, 84120.3644803524, 30062.518532103077, 19066.099039644778, 24713.99338967829, 13802.461355520956, 95981.48856062809, 17156.750010429554, 46916.41779912056, 1446.4362488142867, 58591.955257222384, 91892.18846983118, 37837.29552223779, 60704.849146283894, 58387.99141081822, 87022.8926871327, 38828.715654465785, 5733.333149239461, 74700.98989640202, 12737.078425951066, 3342.134818277509, 58247.125648711284, 62213.67555103573, 49340.76479700095, 68778.91569936661, 17513.47929804953, 76711.0332316944, 11161.389527040044, 21712.66537482989, 73260.57000966233, 70407.93356666509, 35692.28890222952, 99323.66563614245, 61517.377836876396, 18571.69627382288, 69885.1862124153, 11091.717993091488, 48198.85875934019, 60621.8226254465, 75162.59885081422, 12434.226843960927, 75039.78234065471, 90704.50712777593, 97169.43373233435, 26088.94887398291, 51733.91132703873, 52082.1717404396, 39582.63987426613, 3962.4298423838745, 19661.842332576485, 99353.95363112587, 10039.66684316706, 66285.34724647322, 61799.99226453684, 1457.6088334435467, 4484.221566386026]} +{"id": 2026, "vector": [42648.86398885916, 52813.613942936754, 15900.121160044822, 48345.72986740749, 76085.09815815015, 32260.078554669813, 59699.43908448131, 20534.282352172784, 69973.00958579195, 14485.745201797128, 59660.67774755175, 4436.098312665759, 77317.08661498298, 43870.38732604978, 39650.86114077977, 49182.13742877752, 99280.2265875563, 30583.82195182926, 13344.734339597153, 24705.52994406632, 55092.19632574022, 10064.852472001252, 50168.08624805582, 80809.04135790815, 3430.910958937039, 4826.017415707284, 46630.659912652736, 24574.531944863655, 20290.97507566362, 6773.013572500253, 65107.12988518822, 46823.15832353407, 51214.603130348776, 93665.30626936311, 91413.39035691816, 47405.73310313836, 13435.924413043433, 34414.06979839301, 95189.36882626617, 31476.681304988695, 70128.06806339438, 96049.59102776911, 74901.51306413775, 3440.7248865274355, 18137.169440555022, 10742.23523958826, 62178.77335545139, 63511.99457174697, 47853.06262838177, 86793.90757938969, 65637.5193514395, 24034.85298685254, 81835.88346655855, 90730.16107719287, 24796.59737327423, 37353.82023532203, 23401.722128139503, 1365.7592623240732, 73810.41916819534, 30161.17931100184, 65044.81915657335, 5098.068205586182, 97827.45772135034, 74647.33735726091, 39742.46334697329, 21762.766889575603, 3214.054504808728, 68520.83040309526, 30962.44936378959, 84737.87357462272, 30252.81658348885, 95775.95554317125, 8068.703284451606, 45463.39464695932, 50189.755404788164, 66006.70064053395, 11143.525042746705, 6375.726762620182, 34253.22973408328, 89902.21195141255, 69806.26379802104, 89352.4025767866, 31806.300178974965, 28100.928779853297, 14791.114884547318, 6806.0121523131675, 32397.674551736745, 26545.416995614236, 8042.294152273621, 4266.970748697873, 45189.40663589057, 6373.274573293042, 51305.472266988625, 90570.84444729262, 79159.76221300862, 23115.952821085593, 80117.24799843898, 50069.49291194122, 48083.22046273981, 43908.36015900921, 31229.027521800013, 17946.07218668678, 11939.448931373276, 78268.88151651731, 79416.9797088405, 49719.45032922449, 48874.76027364341, 46403.28711848259, 35847.17618445319, 85607.26457666977, 73848.96178024524, 20466.881073810328, 54494.06406053697, 17103.24076198696, 26682.269886363018, 34360.36033688058, 94707.6547765193, 85693.46395103382, 86364.56109313901, 28559.312496867286, 15768.606400134644, 44635.97706914048, 56725.322551281235, 61263.22201058244, 16873.696738656185, 63386.79867623374, 81931.35141688822, 62459.620889108046]} +{"id": 252, "vector": [83152.57704285008, 59011.23655799492, 63525.50507278814, 44779.97551250967, 21795.741969526603, 13562.681616519578, 56432.0388179929, 7904.816544021509, 61353.497799019286, 8532.685873764012, 2495.1780373551615, 98279.18995454532, 70757.00249641981, 30092.60341343718, 85925.91161467733, 52194.88715616476, 60739.77982032812, 95033.49687460241, 50568.71697769985, 62974.1201882364, 85207.25841164927, 38859.217780170984, 69629.00270212516, 76708.54825316159, 50584.157913913696, 14112.665393070412, 65431.81927961347, 80459.34463873506, 78916.38210512472, 76222.40588368266, 55929.47208486291, 75516.42218515021, 81434.30594028866, 17575.79431813666, 16189.074623333578, 27287.242406819136, 58921.328608302094, 17164.831383848777, 67865.4651597767, 80991.14564396079, 17730.820286173188, 82043.32631453083, 3910.7620221121574, 74555.925382785, 69638.5794501554, 19249.28757243882, 55203.604486408505, 3780.456752569472, 54299.59453017149, 28602.399757740815, 99114.15939965444, 35577.409342625586, 88891.69314005374, 72683.45608224387, 4824.752839755031, 29814.622435442827, 66701.03778134889, 18502.4200246669, 77713.47025243621, 57621.355282957666, 2470.7635568514406, 61551.6924679141, 10867.637237481775, 83443.6843119871, 34811.98631654613, 90940.33418120684, 98853.9420820894, 31652.15423936525, 29846.519207102196, 18748.85046699356, 17988.24535488642, 98973.82257529396, 88302.55256219061, 92631.0578400164, 61420.523178510564, 59436.91315860329, 15086.796525567548, 7329.737468081909, 68020.78144513667, 77261.34428828454, 62928.26275645548, 8940.082264144323, 90027.93331249959, 3933.6510641573286, 94876.86468463043, 28720.66820252782, 57992.099733331524, 27456.955931601256, 50697.26895707168, 71862.65355022196, 13143.422864829468, 71540.3604031469, 56626.83092034223, 4827.068043300397, 32008.521105136246, 6657.650999138765, 12030.575862826876, 64063.1127984454, 10192.406657037202, 57699.57799777059, 84341.69147395407, 55673.207998875354, 15458.014103519368, 48265.4859904672, 46132.72601527605, 807.9335826421508, 32992.91021985106, 90247.70645705055, 19519.61692766945, 55662.63864204088, 8085.968344675143, 70650.81001841513, 16348.86660164332, 76508.95188156162, 33417.237436144605, 28132.77406519752, 74311.94662588782, 43955.16577648264, 40455.46186742154, 84186.81815383682, 39347.775615152634, 90177.7406515993, 97592.55242099153, 67641.79588338631, 67397.02344000473, 35523.823645212186, 41387.19742583251, 58095.87875189469]} +{"id": 171, "vector": [4992.757600383735, 85736.38326276603, 1704.8765740181216, 7907.5960204207595, 42076.27221604241, 1053.2087910105624, 2225.0840877330093, 10432.892376270176, 16373.71850421484, 56772.91468601747, 82635.18975798086, 7587.3763150865025, 78812.61951826765, 61277.389830250126, 62766.76093994032, 12633.47694143766, 97894.43380894035, 30643.362701990394, 82546.21397614207, 49839.38028364977, 41489.38765393761, 51991.4515488053, 97314.10721553542, 11889.457136801573, 92286.58242891391, 45476.29815341989, 77738.9748152269, 29910.46871796522, 53428.76144700251, 94642.8540336934, 82379.86362399324, 95515.63464931771, 14272.182132874656, 87288.05135868158, 28804.41707229542, 94324.11684849937, 89274.83390250195, 89372.8878572945, 10511.273001386024, 57998.75439329683, 54701.90036390471, 21991.20835814802, 11252.432367990506, 36686.697790171485, 31404.13706269262, 14704.874356583809, 72634.06519823079, 82287.16959375917, 1992.2257971871638, 90723.8208723137, 21669.484287127627, 52274.505993543855, 66328.13016377235, 60151.335023049534, 83214.1105169968, 29020.10891849015, 24182.482755840207, 8846.911660323465, 94678.25983849644, 76506.58867176351, 67084.71934822392, 53383.97041252549, 36181.52561174737, 12926.825859278224, 86596.88179299085, 52342.35138111629, 11603.977469406967, 16810.49027120404, 11051.498396884308, 13807.796304037023, 57928.8522705111, 23708.6692951035, 3276.5328148421813, 54763.49376535843, 39074.234417977495, 62914.44604320632, 78400.23719484714, 38505.09934726427, 98619.55022354264, 1896.037939213713, 64904.22013232133, 55332.073336513524, 18960.081514688798, 4852.175536750314, 34088.83328502408, 31400.52594051995, 36554.782409842555, 79668.78756877022, 21275.136703911623, 62382.36042468396, 31612.717016120474, 92937.74017765706, 34151.86020246529, 48284.77237707764, 85873.98471543434, 5744.189205959382, 34439.743252595414, 8954.645165245334, 79693.28851661397, 86367.11967063739, 24412.21857872945, 79306.413236217, 84837.70444956866, 3230.952749187943, 15414.711664640769, 4926.978405657689, 6443.628904691701, 39627.43302592633, 97982.25556046973, 87563.50425571438, 99170.95611827947, 59521.42527855118, 13430.726651106263, 71672.18170479788, 3203.7168986363886, 28310.40733347714, 86052.08544212379, 69209.59937572914, 97184.44724986082, 25500.07569852002, 63537.79652495386, 8934.648608184947, 86222.09818647268, 59927.51195636967, 69840.79762474308, 72909.4989769798, 68599.03025257797, 5680.691500036783]} +{"id": 185, "vector": [49063.123619966296, 77034.2225008298, 91725.19996481668, 15492.33398201344, 27828.5534411687, 56279.77662377273, 97612.8547901077, 71745.09519403974, 35693.18421979268, 44199.11881605143, 57486.55105739532, 54747.156968369636, 27193.125087483837, 50417.83029348903, 13756.58843540104, 66851.12817684509, 47913.33735476952, 72541.14253507882, 145.95413567076722, 27165.152342731268, 83242.81155892578, 10305.347953541877, 65920.00693561538, 10001.321432086785, 12020.962704217041, 61008.92685978956, 56062.267387811014, 55999.167952158066, 5591.343104102675, 87644.29744965644, 75645.43439028149, 31243.46337065148, 24140.00945064517, 73061.649619722, 73932.00186210456, 7013.9605690315075, 32240.162944524396, 70571.53499063397, 55681.96613818122, 46233.43988660632, 92325.9983256044, 53246.3893609441, 6186.126196144625, 21495.72875251704, 61158.07266673058, 53719.64119452814, 92687.80212410568, 36754.50627313792, 39294.48361365104, 78523.16857944611, 94157.79092341167, 48430.41679616779, 68785.82349775487, 79064.61261695175, 97048.34837404924, 62623.04110514659, 43999.88378289698, 54385.64689406917, 4631.883857133168, 49714.34418981521, 76675.23924828072, 82441.78338744007, 71624.69695952507, 71042.25102590989, 82251.64734703222, 29503.18726644967, 24245.25072458249, 51078.84706317202, 84762.89382127271, 20881.859303303063, 35112.68064704405, 9331.113881736264, 89276.28222152793, 62797.99252836782, 60712.73200008703, 27516.278791069115, 97287.34167632836, 32386.848721021997, 31873.078040010805, 29410.323545182004, 32874.20452623586, 37102.44086439689, 21469.425276748476, 20887.303154929858, 5002.57320771117, 67683.55656450185, 70553.38171463738, 53076.05726315886, 95177.87185790423, 58022.16800792538, 31168.00392372838, 70790.11300576269, 16359.359646042738, 20512.286895105135, 36419.87636648011, 44597.382220254156, 34064.559792228545, 93406.28706012787, 88752.9760162578, 72388.82736137792, 50972.30163446381, 83040.38726860378, 1941.355365917452, 65744.68933630158, 85153.35884070754, 6946.3839430564885, 30722.41565050182, 70612.6433393226, 86500.47352463081, 77990.6082681376, 84695.9065382814, 33490.584868876795, 8660.06582977582, 23878.422018466015, 5041.567273756986, 52977.55140292637, 66980.03654543089, 48127.913539071444, 60426.4845801177, 788.2567077948321, 4220.707429983994, 70267.3463533453, 74064.87941866883, 3549.815407075696, 44754.80717424611, 61396.53443313612, 54597.939865418266, 63606.27419737631]} +{"id": 690, "vector": [98650.79868011104, 88478.14271040492, 85186.19691827447, 22129.90188313062, 37515.42645567908, 51135.56297127419, 78196.4367483309, 97108.41376000887, 17258.170032044552, 20236.513076271258, 98716.66043391786, 5241.81581237676, 74628.43997240372, 20663.679553960334, 39321.327794926256, 38855.543161121655, 51046.66724928161, 15246.394387801321, 30513.924737687616, 89974.95955902505, 86944.95853195523, 944.933512363788, 99500.92401255129, 14100.531680726968, 59352.995854554734, 79462.21547491822, 51365.06663167798, 77302.15269019497, 63156.30812399428, 46965.90957466105, 35245.01424412794, 7400.460339815762, 38588.40711784384, 97457.81370230611, 28026.98897618151, 38177.7643027701, 35246.81388802529, 5948.840743331729, 30884.794310768106, 2730.921467158154, 30499.13131882238, 89602.67351280285, 17729.531687341616, 82836.60491460006, 42610.55106557434, 66777.7716538818, 48538.175199278485, 98476.8248604536, 63862.24218858786, 8319.030709756104, 13008.541097785042, 5863.990143048037, 31247.83698977742, 55238.86250825201, 89954.85344137515, 34998.276003712635, 9568.691665877272, 71395.21792521716, 69951.17266359097, 35985.76721520565, 71282.99307689421, 29710.68455020628, 82339.60438367892, 88512.25101849697, 92556.31953637683, 44538.290753218134, 71235.13105269129, 91291.52323999895, 2357.7087208608295, 22274.370591030558, 2461.5842819916134, 86300.3486718761, 71930.10916119104, 96295.41641717534, 9908.592648669046, 25192.495274446403, 41777.93625243534, 68510.90100001464, 34326.225369441476, 95637.19641356987, 44273.47297957217, 65890.84059365955, 78213.6365078199, 73885.99731975334, 60689.17809709352, 69626.7336877477, 15539.318328246954, 27057.77123136823, 88545.5757168385, 15349.488856681914, 35421.4053194853, 20077.96834028266, 20673.382222326887, 91481.8624032419, 80138.74926992232, 40749.91320811974, 96235.17466871526, 97428.94054767434, 89315.72689919887, 49197.28491131523, 11966.074770548863, 89654.82730377062, 38893.45695999894, 59318.42438399862, 21094.974810165324, 41985.062939093084, 22828.068503572507, 81967.061856221, 55221.116598142515, 94939.27253006585, 73065.8717773774, 93289.44444537691, 54461.669490512475, 61131.17233275934, 16304.05326777793, 78920.48572489092, 33857.404452363386, 7026.433865526582, 24085.10053508799, 70078.88056209644, 42702.3413794178, 30719.603647222793, 396.72100996839265, 99267.97118328582, 87900.80821424496, 18531.795806553797, 28232.495618867935, 29754.02771679957]} +{"id": 860, "vector": [49581.77891439375, 47285.409428816114, 72755.91502621147, 76396.74478664216, 85467.90572840998, 31381.850828725142, 44329.43629420213, 98729.74027447132, 28156.793458833617, 13428.401946896707, 99151.21953310387, 3922.3114100233047, 86627.0527647772, 86053.40794598391, 46462.89348205075, 89216.74656789398, 77110.63597666072, 21295.405419062874, 25535.33122395273, 3221.171543736767, 35695.853422092136, 3968.9255143670944, 88091.82374287763, 18743.186989917584, 91645.5658708454, 5967.554417930787, 79900.49397222599, 82894.13420630022, 68147.14112258641, 89365.34575184579, 35181.90932704397, 18936.791164879996, 18015.229865052086, 48477.56319800519, 44399.43122105926, 70399.23999921507, 89949.63742516057, 98155.74139606755, 34251.222400594226, 48460.30181990822, 41279.19800415057, 32683.455168182198, 15847.359806642591, 65743.92717797241, 93861.51416753807, 26824.623775795884, 28416.012491300968, 49263.9702021683, 93513.77096174024, 10110.159672796803, 23958.825386779183, 16354.160864267385, 50702.10140907898, 60565.200113007486, 38041.71797958142, 37174.01311827701, 51875.570722260985, 76103.25598807017, 39671.104153944834, 26759.252811449795, 55509.081072428766, 55828.03660724759, 18149.102376000537, 61266.53590014292, 26396.956368754963, 78306.95050816888, 14017.89642713962, 92025.08610974724, 53691.3554237929, 59740.77860518381, 32149.430777333866, 64643.87228885711, 29118.724467559885, 87599.15652317902, 90737.57697475045, 53168.73982567002, 74827.8113250452, 73119.41074652079, 52475.631152390626, 45907.20235658983, 62608.00829663169, 49625.798994871264, 29343.438365481434, 95032.26939722296, 40666.49076472508, 87112.4451054674, 80475.2873661288, 53171.12381976458, 15682.642751144649, 40508.44138774808, 33911.37635316437, 88573.9111468707, 37815.75685587176, 68924.88502899869, 54151.22632901836, 12922.77863060094, 79029.33046993469, 13906.69518184443, 95047.42193156635, 80504.87529654615, 41010.11452849489, 47100.70143506068, 89335.34290258882, 43477.37740281674, 43266.675234166374, 71892.79690499489, 87382.74677696486, 99890.74620703327, 13073.17231805477, 59197.03589040568, 61199.36365019753, 75273.67459434616, 21153.327082073258, 1806.1532901885723, 17621.42061710349, 854.9892784795787, 45539.3884169082, 16677.981599691004, 12026.099342623153, 67712.10882624867, 94685.46127413528, 64264.12013002015, 84659.25509379776, 87758.42488860288, 97414.9193879909, 62673.329357384966, 4392.77789517486, 62571.423124969915]} +{"id": 950, "vector": [41615.080745869, 5407.754500658058, 67505.1061266728, 52003.4351192582, 61733.290422864266, 20856.87869250834, 70302.93121808521, 6803.026410621049, 63311.296273631146, 3662.7086961510513, 32836.950478650695, 23564.203055354017, 86908.44176312003, 34638.62041692062, 311.5054382623383, 83999.6824035517, 61671.063695218705, 54957.478938994966, 58560.48502492496, 98848.61591422789, 74567.57537273898, 20592.27509264998, 97860.1529457657, 65232.68566601225, 61931.1467530911, 31804.815537921237, 96372.47851521423, 88501.91649065058, 14493.495888820251, 9021.939353680453, 19294.831535749134, 70193.01591480755, 83974.69896841451, 22512.29972912829, 57214.56686809876, 52028.204930113665, 11574.448083199151, 8105.302229405109, 2393.2486770168125, 82712.93696140543, 60046.993202816666, 76431.43471384828, 525.2876514868832, 62364.59549842579, 27646.141012355518, 35337.7956114595, 30129.747346731438, 25211.027410599472, 73569.39136083897, 11733.19374634817, 5470.4637403231945, 67637.96816106096, 43258.78206358067, 50969.96749028493, 37434.50345845136, 28108.78705379817, 10486.292888152315, 99950.07308407356, 22058.77838816733, 48102.580678167236, 35198.707652873825, 75936.84987996219, 69923.95977964782, 26254.270172718807, 19691.514484815863, 62711.092279581295, 39806.7089062696, 88496.95311889923, 70195.49129677293, 70091.80127713048, 4000.7995292306696, 90971.10980710533, 55393.34943160187, 43764.19775644419, 60821.21883115763, 25551.218072177096, 67166.3491056623, 77647.27148436238, 43462.41369385606, 58846.13959874274, 14926.14995871233, 76561.73721083117, 67848.72497012503, 26262.76434029534, 77557.84425526601, 14099.399452543605, 87093.41496559494, 8474.185467475592, 40933.62010869288, 45700.67400426767, 16993.524655037574, 35652.01352111318, 28978.63421103545, 84497.89331429385, 59695.2679308292, 76444.49512279977, 52366.950940689516, 13360.154755170084, 77031.26737611636, 96778.14421991569, 49406.26274476316, 76417.52690712154, 91860.82770959211, 61761.63968745217, 99308.67811911114, 24890.735220353698, 70254.01363541746, 77429.06496081052, 14328.571472595497, 7954.744853760853, 46278.302237517244, 16524.895318224597, 7804.176088085635, 99535.1175078921, 66286.25658213496, 30919.78872154385, 27410.45597594293, 16017.618841917725, 77651.08604118129, 88987.63747422077, 63914.98119751596, 65144.557605160255, 90335.84396259817, 61909.6952086301, 69772.64252111295, 81447.88510611693, 83503.9942598998, 36587.26210036687]} +{"id": 842, "vector": [83057.69440886941, 60724.4847171275, 70805.32919745056, 64242.5630560188, 54142.28524742608, 10363.339353657875, 56734.34325423321, 85167.36337969599, 46448.05239819233, 22103.33058235362, 79409.5217309706, 39723.82127314041, 55831.183930385385, 68390.07448940759, 8170.48521028605, 68558.25939595183, 96647.9920127703, 36220.546484652084, 14487.322585622731, 16989.683896867315, 42328.66660460821, 80305.46516531978, 17222.383224294957, 55500.65653250263, 80947.60962317254, 2705.413964647174, 59925.97120157077, 2164.9073821532807, 5207.993286490265, 83243.38144580922, 24059.771628641458, 55109.82860129789, 34860.08178569064, 68472.34735089411, 79478.46621051326, 48807.80927377988, 57819.0356497728, 67230.91538752502, 7587.622568877284, 12412.650305801388, 81324.93778778921, 41513.66539623338, 85323.8926113459, 91976.10972925181, 74210.00597848164, 22907.414176276965, 15380.381141744525, 65936.1503171738, 48200.04702878066, 91319.37459438905, 98837.19454845377, 92565.11791320595, 3844.976557459601, 24691.024239897208, 84493.54274058143, 55954.12363047118, 40611.01427372276, 36903.17566102346, 9435.95931982193, 27823.44589423249, 94798.68561974159, 1814.9360365727007, 63462.727822651774, 8153.598474585211, 84476.49023106798, 99458.27398955272, 33932.00224331923, 48321.18109650677, 41553.11755811344, 271.3220197087307, 94588.23510989404, 75619.79945436545, 83509.6951125697, 86773.9032688558, 22156.50738437661, 70556.17660483782, 10915.298080996694, 72584.38756529099, 95277.21144513409, 17646.665532030158, 97264.34769134028, 38214.92402251634, 20500.24716455622, 55196.008449453046, 14152.473576966097, 54712.64536626709, 28956.308884051265, 39323.346247866495, 1282.710408462806, 15567.905060902365, 33293.81882358204, 15266.19090918333, 96378.77601517548, 72414.74604907093, 36169.86077502739, 98140.93688628945, 79977.92309503187, 57090.004016128296, 91344.42543934913, 47705.236507179674, 71926.0631161719, 21272.649853753323, 49389.85967303499, 60534.84663294979, 2637.5631307069325, 29503.128273640243, 88202.00721433805, 37195.8322079873, 79428.44241582829, 10196.374911696505, 85578.26086309012, 53002.93529960531, 96985.61617729074, 86720.71134930389, 34302.43746371651, 58544.31066293647, 13638.953215934269, 71178.1345025423, 31659.889786867734, 33766.20977935918, 91870.79301769949, 61984.54789100073, 84822.77694808741, 15606.353972107623, 89867.83219805702, 29881.27886687032, 43097.07883701281, 2545.920933545676]} +{"id": 1397, "vector": [12897.78923062791, 26971.762921967933, 42400.66365473425, 95740.77486918333, 58197.97966520126, 15343.902702982614, 35265.82933259977, 59373.17541527579, 34581.00988810099, 1660.5792224660966, 84632.69327518508, 93212.71030311927, 8339.715373390987, 42449.10768415613, 79554.35410697524, 1005.4156978287265, 52280.328851751656, 22056.69191342311, 24948.469456399158, 75882.87653261557, 82977.91768510612, 80420.78320653569, 82181.94654766534, 77145.83247801187, 87167.02015963275, 26418.11191036184, 14423.066482308834, 82250.30530603045, 2885.5359262771717, 90317.07022684104, 37790.37650269844, 97661.07329392931, 77648.34151401345, 53575.695167924794, 80058.6851006459, 59775.89286811703, 4287.3525445826235, 94168.74089056466, 76392.93920013992, 21107.511147986523, 19204.63450956611, 1626.3605706844376, 22889.432087147154, 40464.10210709559, 91011.27378662268, 81121.44383363488, 60353.16182534613, 4812.774177636569, 38694.794552578845, 47982.538910894735, 16870.502174420555, 71615.21334085132, 76190.21774233576, 27608.816627159093, 46719.40976475816, 28719.999263072514, 77756.64689194964, 79812.92057489885, 20972.26888132322, 25938.10066093343, 29902.148836801756, 86187.87852244133, 70546.85149484019, 5486.263768009092, 26619.285342085775, 68416.06158330097, 65547.67514080448, 37621.78534455636, 68352.21390544875, 70422.52942313033, 7668.085523275247, 85716.90623610446, 99065.34697868428, 55373.614402690604, 25139.080287531724, 2828.296631559113, 33332.87556842187, 34490.643429835356, 86323.0312494201, 92316.61254741225, 28550.85423726135, 62446.36282730845, 38683.96297768084, 37577.92064388562, 2814.4624286375097, 21252.354197137967, 48026.43726684794, 61567.48430404807, 58814.1401336268, 27742.130404032894, 4206.777433455833, 57954.00114966688, 47557.01304491269, 88138.4662649376, 4425.524326826913, 66031.87065773112, 25323.834509537824, 44924.578369696836, 42592.75445588655, 34273.66279130416, 46074.0284447504, 66692.95645086754, 49676.68117188077, 53416.33287260568, 11503.89508768067, 56495.20258281222, 87971.61215183264, 58785.43906083361, 68588.60726185142, 65560.18893816473, 12206.19312767609, 22293.561478886913, 23048.681021409022, 49151.46318457555, 88010.57889994446, 68377.61840177144, 31608.543683207292, 46122.92475405104, 67534.66748120521, 65781.32173804086, 96231.33034457022, 4152.594006599142, 40044.90979950887, 31829.418324536095, 38664.370593079875, 55604.04267305731, 23062.641253678008, 86222.25784524532]} +{"id": 2020, "vector": [76524.07673414133, 8409.102170218463, 15311.612496734206, 29683.960266373353, 41319.257474102575, 1158.5670190174978, 74580.36848013397, 36551.44610176367, 38286.98964675023, 62611.811130675575, 89983.0135978453, 94798.79043664757, 47165.04829898366, 88458.03993992241, 64963.61718395307, 53928.11137621604, 89024.14605402644, 21325.799545026668, 13194.00734795021, 64048.95067414425, 22470.183549910693, 55193.86360609611, 19985.536704418795, 39650.19843730148, 99911.11512511961, 7682.510657454955, 50704.07023816636, 33026.51743127467, 58100.79369449006, 18544.43476985689, 35105.048836583905, 3479.012044413088, 54521.77581149208, 34252.56760842391, 79357.753380614, 21139.666533320855, 91542.33888438815, 6260.35450270801, 89455.6861876457, 57512.674586711386, 13096.522336735283, 37467.779938127154, 5296.637601406529, 9208.776976156041, 63021.38818090835, 19584.48651378677, 67169.38058406768, 13423.192626045931, 67342.7599433004, 50377.85061443749, 81233.44021348083, 27834.48114742313, 31850.036070166676, 68426.79376369611, 14396.296183383962, 66333.86595623798, 85061.51471989424, 3959.9630387675666, 92073.86547164455, 96868.73505009341, 62290.88777714548, 71657.60672298737, 41490.76634146802, 14211.48837447328, 16544.485499630857, 64125.682517247, 14339.037938945332, 8595.55676699264, 98443.15195362622, 39785.287546000516, 18274.19896513619, 52698.40721286613, 71972.39968752862, 90056.07245578196, 15782.864054171741, 41426.36838842818, 80305.40058164358, 22014.293308186494, 4985.970747420932, 74428.30104501559, 83533.41450137855, 59012.85477773347, 4520.592396552003, 66364.50227350388, 74060.48085478644, 14719.226261275653, 9630.233049145208, 21286.55658749998, 98712.55117468552, 42195.948585483304, 59869.0208935544, 65010.11401443834, 72005.8095004072, 43539.56399148981, 13928.367224389782, 42512.30754015559, 16292.272519841112, 29688.298177033957, 26537.139139574083, 95873.16977998831, 42456.09578525216, 34135.27355288846, 47442.60782928487, 93646.94767909494, 59515.310729090445, 36349.68807473547, 71798.8298346869, 95960.75514084373, 15379.943327489165, 54349.79211475048, 90816.12604193324, 75108.72315996344, 84104.00108801702, 18452.927303628207, 88270.29137782169, 32305.54413975163, 6092.613768954824, 67779.86525940728, 3132.345996208885, 95673.08598987009, 65547.21991791291, 82174.8785263689, 82258.1635379419, 7466.98363610051, 21961.117582400326, 46125.79260630739, 9164.697622981355, 17417.604957734813]} +{"id": 951, "vector": [72540.24367568133, 19966.00366002754, 32071.470983355808, 76580.2961689042, 40926.25163725738, 34309.451921180786, 78357.45740631613, 9366.82713842789, 28111.474759731693, 85505.45703629218, 28836.92512008985, 30295.987569001325, 23016.042100446488, 37447.3256951514, 74980.06750372122, 10113.704584825733, 57736.424970191765, 63334.623964035665, 66800.81362520333, 61393.565471912334, 3862.478227121424, 35639.54420439034, 30950.863531835592, 86043.07741739064, 5371.028065967343, 61367.14624740472, 65812.55150804354, 89146.49549902354, 15094.66221498169, 97235.07622998561, 39.63929078074635, 65752.21771188789, 24138.712050692302, 83653.1660382145, 38585.411563573965, 69638.86379299304, 24973.863593291157, 86643.01960058998, 42995.61751904096, 32871.23417386541, 68435.02498296516, 21327.180241515274, 65473.2388245041, 50594.148747195366, 95339.77737846196, 53001.86987546217, 4174.317028309282, 81620.8058960938, 97971.24038617034, 15380.781112525243, 90832.00005385886, 21801.038679874975, 14711.116673074643, 74054.08651209153, 54768.702285410574, 77418.1952629963, 70845.7424080635, 34373.81572917128, 71729.23617400747, 3495.8850258440034, 127.30494365829071, 12362.569569019788, 2939.1962892323263, 425.38658067434733, 86052.85359088748, 86159.5746787923, 39414.29615435392, 4284.227454649181, 41335.36932268492, 31674.629985973403, 15692.287170711095, 25837.65571817618, 77287.09475523034, 74565.45470647469, 64688.61886209974, 26454.247821328547, 5765.34447039756, 81580.17981847863, 99167.62879798423, 37748.292473847876, 13094.885500479724, 83436.00668888047, 53344.75892036412, 6955.233962076913, 61929.25404357913, 87922.56706872497, 50797.4683511973, 67778.7976004522, 54051.710308420676, 57993.072810561986, 15293.231822799247, 72987.55055580966, 57027.46551972101, 24740.174764939515, 76945.60778031372, 40190.21206412184, 36066.71658257232, 97892.29212334844, 77348.01795544493, 41769.96518334405, 33918.2190265549, 7741.73731088863, 13902.036037440934, 6853.012776176049, 93503.41568168055, 37361.15240706119, 40225.62057549437, 32031.14608899048, 81106.42539074809, 75919.9361966255, 15017.082768259039, 55140.38489207269, 98537.57062712079, 65842.44243001218, 22384.289184164398, 27744.35386497275, 44406.48933296839, 61331.25878206436, 18468.855172171374, 73764.4604404478, 68797.83305728713, 30593.82103065751, 99322.58527182846, 28725.465535971227, 85429.41889850206, 85539.21797318709, 10976.437653071924, 79785.20850069408]} +{"id": 150, "vector": [21388.495905050288, 88006.21391960552, 85233.28096179274, 99825.82734240255, 90165.22524131376, 7031.460678101475, 99364.09927658865, 64721.639965014896, 68751.68959513417, 4651.1178796685645, 21997.218806619843, 79653.65497594072, 42001.908586311976, 6398.636302113825, 71499.74536411396, 66808.01752659267, 9031.422757084396, 88392.10149643847, 94511.31490077048, 79156.2636705839, 53815.16588642239, 19939.832684944548, 22544.823367450896, 59561.18806715731, 95493.67441230807, 98937.89719356204, 81747.09739551773, 64089.16844997183, 94118.32374631906, 56049.304482956955, 62129.83661082027, 19704.53046308287, 15571.838880753363, 19892.107987285613, 37026.05166310251, 55925.28853242196, 57875.932880483524, 8873.567825590146, 39037.58703860418, 41067.12908358087, 14229.155320733555, 35974.8581122615, 77424.00076566012, 89421.38282146181, 57629.83462469742, 41980.26027167531, 77733.91082462322, 40180.670182865906, 20456.44167264128, 69404.25453439835, 80173.67509177576, 71388.10160490744, 84199.90401558463, 89608.47545154067, 35097.362017310275, 25445.956427447258, 60358.52663126956, 27281.831479342934, 14825.844495829355, 55909.78267740091, 32544.929718731175, 67515.02206051293, 3710.8023843688275, 94166.03161296775, 59900.85518197057, 74368.8648617187, 68354.87011062159, 63051.86466003107, 62202.41167538692, 88626.85517368485, 88978.37485724667, 8725.87166288984, 9255.586512081849, 88267.40884123223, 52397.341557955566, 73825.75186613825, 8582.331659738651, 25250.75752637781, 66552.5704593822, 99820.84732891081, 1489.2093407055772, 6362.067914991177, 53625.47124037659, 89785.38572949401, 79291.19764382699, 78362.09165275158, 67589.56729156802, 8329.38025381912, 17956.676327013465, 35542.13738747233, 23384.599201938738, 12578.055699230717, 62075.723097575494, 75648.18773806555, 9464.867275989896, 20429.780991632797, 83014.773839681, 5797.840520998365, 91583.05867725008, 55737.40918198308, 8258.742568314581, 34821.47889441163, 2765.881308137874, 49832.47376529892, 8216.945781085295, 85987.61984170656, 3870.2781268150966, 1379.5925154554634, 9067.275769803606, 39119.74795282287, 80129.77233529183, 91165.23246497482, 85055.34259122694, 33278.22481969012, 20170.000587023907, 74299.29566464569, 18343.689173309318, 76309.64205574813, 32943.11594968872, 80653.37041091365, 95654.18098978604, 49462.265807536445, 46583.2087025719, 26563.046615587915, 64097.6512748354, 53750.652391164025, 69927.26294632108, 20552.802017741145]} +{"id": 704, "vector": [97030.09030594151, 6518.417342818239, 51398.55490902507, 57926.21784974298, 59749.02424692266, 4157.5913172728015, 94355.68218260919, 46137.55612321969, 82309.44233197357, 47485.83743584397, 6371.09487060995, 7441.907822584304, 84120.57865584668, 86120.84906526249, 53167.35753307589, 64538.691290286995, 10459.78829094638, 27962.37081996921, 9341.15971974826, 97226.92505637987, 34714.02152155748, 14152.891504219779, 81559.89593677495, 91285.55928995757, 53141.08846398171, 70217.28514793297, 96648.8615889766, 1130.7509939660122, 68757.48415607134, 88111.54276539235, 17092.055531699203, 87137.2579108476, 45475.3727643471, 89201.97976242617, 29914.157559965737, 56522.586476458615, 40322.1720292974, 18818.602256966322, 32673.58790815752, 37457.571205556516, 68974.32778793186, 31780.174011585703, 53934.62456839505, 35772.62242432971, 964.3614166689241, 40845.49474758613, 43169.70430440995, 18280.843955553737, 62799.58264394316, 67168.22414223659, 72374.93754541701, 75513.69530746392, 90973.92431681258, 71079.35051727502, 13887.610394433248, 91817.12898478819, 14582.134224164434, 94134.02153204719, 79909.428259739, 31996.40490832081, 12085.45352596483, 61040.176313063355, 50334.04589169931, 21848.135773884114, 42485.15715192643, 655.3064314764656, 87085.6733133463, 97029.44635271575, 72007.70999455698, 80319.18903329733, 66937.34552880334, 90637.86324971534, 24017.62765431531, 96887.4336854971, 68369.92151099356, 10120.225718664766, 21644.512708679333, 79605.84165451559, 63526.12041736333, 42499.66551794014, 79014.30207277044, 24253.166258683323, 16063.558377987198, 22938.844806908965, 66054.29681663902, 51825.6971824335, 45567.74865387109, 23666.464797603836, 54447.98002532932, 12765.839857400319, 41448.65620018438, 81726.84065162785, 52249.29721404594, 54512.23728265933, 78969.12138054062, 32747.62550150648, 35916.567614114145, 18700.895699533226, 59776.15621604978, 1557.1437508270724, 85080.48776283194, 49267.42579471665, 1602.1352933991207, 24001.865709548143, 3497.1383994771177, 77261.001718159, 53060.678302375505, 88178.1255415641, 25708.911402352096, 45701.65691196041, 5541.224423902103, 20329.534546749674, 72597.95492662098, 18713.43696951976, 72499.03252017383, 2138.3970733250603, 83717.73707030006, 79996.14979570222, 86933.42811753739, 27369.28144811096, 47396.51788192606, 7570.3130135941765, 1595.773215938523, 99338.18074240064, 73581.64211123515, 44942.55437071564, 47180.90735765306, 9539.826353729319]} +{"id": 766, "vector": [59544.31130782468, 99703.45062992019, 91170.14814182506, 45835.33079980138, 56362.387501273006, 19673.55116194267, 96857.81018162704, 85299.67536364296, 98689.29555822916, 21476.661723215253, 22822.50133361523, 13585.581591366557, 84628.81536468543, 1037.8299402341584, 22076.492391043266, 70753.31785902978, 45069.08813165663, 44906.38165963651, 51490.05600546493, 20485.67307464325, 48399.321181611755, 91113.18133350612, 28959.742220500793, 47674.41907543906, 62801.648404931184, 44567.891764257925, 14773.42535154137, 36178.849125378256, 40932.22012772138, 48355.138452673884, 68414.33979637787, 44930.5252392589, 43115.86700053003, 64641.296833014196, 43346.630650508225, 94396.26116686559, 49365.43776515651, 22138.11860566539, 17854.91338962504, 31183.84293426645, 43256.41549461229, 88112.68369286704, 51010.3015822596, 18073.372097368956, 22914.23900188393, 88127.42335840504, 1286.4679427931126, 15317.669399609778, 21467.660949014957, 3875.157677108243, 75509.48876235013, 73603.94629373305, 47480.62596711609, 10277.560353635585, 87463.53112399664, 76107.48179579455, 86744.11459999828, 6174.632265702651, 93132.22749975091, 4002.9165025340153, 28207.91797139409, 38988.48855412075, 60727.62406222091, 5874.373105832231, 52104.17895313581, 40069.316437713656, 1022.7872271203919, 91932.3367548556, 83690.49495119076, 59488.1945566719, 11713.829071455539, 44506.01990316836, 99859.22317527857, 72678.33693830481, 53837.09528177056, 37326.30444284552, 52784.4725878283, 30727.964620804905, 51840.02519207009, 50876.92828502494, 77570.48489404778, 21542.782082395308, 90323.55793327923, 88163.02599776, 36988.50397295467, 5814.168136745801, 88485.27229595579, 50879.08426150562, 40405.30863135695, 36845.885363035966, 59131.40975458305, 15191.381120242031, 21817.220143519866, 62764.31771978532, 10365.350896418578, 15704.623715300315, 78066.99736946431, 88265.17396563124, 39121.00626249685, 97579.20650181753, 64288.399794515426, 52169.64787386869, 53377.69392944932, 19764.630339892665, 79800.92446825342, 38178.619057976415, 86477.6026341738, 81906.60619153916, 13835.83042282136, 2824.8573322444236, 32575.61488071461, 24872.35450836175, 86177.57417958108, 81490.69573814694, 31156.60852151657, 38530.392672969916, 86968.68254442289, 68931.32944404487, 11038.269039374714, 70633.28728724454, 30308.91301191352, 28189.665438323318, 32992.99257439896, 50364.74543113613, 22325.52777842992, 50227.23695303465, 82002.63277350114, 38464.586778536024]} +{"id": 1083, "vector": [46544.88467885909, 73274.8142094474, 46336.677449901974, 10907.2042435018, 94577.45569512394, 29165.572449292253, 15353.457422707306, 20809.40968117716, 32129.40321543787, 62304.01580491819, 44134.11553185485, 14333.356144317933, 81234.87533901827, 64258.76304344024, 16778.3802600882, 85907.96612727994, 70726.02558318422, 38958.04024598234, 60869.539119805726, 85814.28856793825, 4340.76992595912, 34895.928805112984, 34425.58251592431, 90059.06594225042, 16378.576050840966, 27017.14687319905, 99956.60887077168, 94941.3715995872, 48674.714475853034, 4212.278081905574, 86921.37443362067, 98982.7247066606, 12419.569045387047, 33675.12270340143, 80260.03428250346, 85991.60774074036, 37594.81051427277, 96080.53270663494, 74800.25997148364, 64849.14792086866, 75653.7916369619, 43971.02056870841, 29537.829699429418, 5640.065224080604, 9802.962973149399, 68541.62078338423, 97600.44028422862, 81786.17400286556, 84250.93006767641, 90611.95779498025, 25075.28737685064, 5701.569965084441, 39394.70056521588, 14143.31355994265, 23756.73221161322, 10012.886263663135, 41220.24495946224, 61596.61019437157, 60216.04992856083, 40280.67386115165, 9475.70556382482, 98071.45897178043, 85787.38446947762, 31419.61952568394, 49658.46134834845, 36721.31411708457, 5109.114918399482, 1987.9071943137094, 36383.20499269488, 73642.59049355732, 69.66605081407673, 24216.06509296096, 32856.104827541174, 91130.75918324277, 96071.99835086994, 65567.74886574573, 42.621994187741485, 44533.88562554703, 63628.04405091455, 97683.94339906969, 23197.167200597713, 12692.096076154958, 84816.88607072156, 65928.31975919513, 54946.622716675774, 16911.553044506512, 9519.812684073759, 20886.014009111477, 67685.35634423896, 50697.69495275138, 508.6754232154211, 23554.372167121317, 44873.30233447132, 4869.528308701166, 83093.68453516823, 75550.29822493618, 96526.19203924376, 99831.21868583708, 95593.8318562903, 90783.86307556981, 43834.327949265884, 48193.48266136866, 544.7029512091306, 8101.746858490377, 69766.54172121482, 8149.708820432067, 26479.98385751298, 66887.81533830955, 74210.6123492349, 27613.015004411034, 95882.62684337277, 23241.547100032934, 34130.26403130827, 520.8901680903156, 9707.883948584094, 9071.655090240904, 90933.57883138831, 97768.4129114149, 79494.40403837893, 423.6873654816953, 94207.80623119016, 88602.23608923286, 15892.47562121302, 13178.57723537803, 30471.50318513263, 25705.114560473972, 63917.25818783993, 31375.89203546345]} +{"id": 1738, "vector": [77996.0951817715, 80644.0661817461, 35595.35804607011, 68977.8440598105, 2381.1641104962587, 22245.06927733355, 32779.52545090993, 13808.492479612689, 58288.43584830807, 74619.51071555741, 76793.35553514076, 8279.911042224054, 77153.65394270667, 31732.304006811773, 84095.29667228236, 56887.58322653736, 61481.644254951716, 46809.85326110048, 19642.85070104328, 13919.009025536954, 64247.1073382047, 53137.28158733652, 50681.313089531366, 48267.787733471334, 25979.070000363834, 62345.78996952796, 16514.958221835263, 14104.965532539514, 1403.1964687905795, 39493.85794148404, 95033.86916042912, 43614.681070271035, 87827.88443815475, 94772.74935988037, 77448.30559120745, 46090.74085904587, 29582.68638122851, 38607.108278335, 44980.71731962556, 45318.00157415942, 64432.366626784875, 67009.64086677629, 4873.942158320721, 19119.727995136604, 94821.21229335453, 4226.393900949632, 42699.17702784309, 30450.57679284584, 17872.01168217546, 61566.93959283655, 89969.01997619882, 32248.44611008544, 41836.66217754367, 88009.15907522947, 63571.75630440657, 11463.516039934873, 79126.20348855603, 25811.081343546204, 16249.386103318331, 12012.570896830288, 36367.222599105364, 79699.41446991425, 42355.56716980245, 91827.73008037322, 5319.860834783463, 35753.362596894076, 13834.628194779163, 56882.138701986805, 86503.7502047918, 31565.304627351863, 53805.89732406993, 24954.8407061296, 30795.341516600238, 74891.64253310436, 99683.47381031155, 78866.83457637439, 76573.37333996668, 6286.01232794298, 74665.0811733668, 84277.93581939091, 26481.481644758576, 56741.86933125948, 26908.233975398667, 30303.24380573317, 66753.4313307669, 73077.84918953925, 52255.227963720776, 83693.8667773128, 87585.5497295214, 34200.42762527456, 2776.2074030862727, 13773.019866718128, 20116.913070012866, 24307.18109001325, 5317.493105765736, 72738.75216114322, 22196.419073591045, 66609.29062239247, 10514.103176277134, 28538.44223499319, 81301.744717852, 35669.89724680535, 17625.786461603366, 19685.606888139195, 20827.493699129827, 61823.87219714381, 70227.40346188951, 98383.27813784388, 35630.87043846277, 2427.016357931333, 65373.49938681898, 99193.74834851763, 96852.7078126053, 20610.62246170423, 7866.993026674196, 78187.10897364229, 26750.97663580943, 31530.84518098197, 34250.769676531665, 96325.62366690065, 68163.78256988051, 93598.55953468347, 9093.612658966744, 11712.34560735962, 48466.46649965227, 26137.019745041067, 74419.55827622426, 84698.48476590178]} +{"id": 434, "vector": [24638.742575763783, 38967.939451771825, 92647.38503847933, 3724.7577963291924, 4329.982370261565, 88464.49308946461, 14832.839453668312, 91382.09867702203, 69257.32451397636, 1476.3964826647657, 97089.44505238072, 72315.4362105057, 71668.11518828534, 65437.07125460049, 67829.22738265326, 11672.58476705647, 4502.5070714930225, 39855.661748509854, 6393.827899795734, 41337.56962765117, 68411.91368123614, 59938.522263273466, 82156.52543405334, 6180.125069570852, 63799.293204682304, 25000.516753873282, 25556.598234955953, 44725.86883598465, 32445.569821517496, 99026.92893621688, 94051.92530816198, 98186.59588168816, 33049.16799686425, 11371.675379938573, 83772.17547778532, 52126.694833329624, 64375.51803166603, 52246.78277786272, 98410.73361912827, 92640.43535455098, 582.2481869324437, 97854.65716933814, 69585.11503016479, 3024.9190542263495, 17311.47372501227, 51497.77222314932, 1909.9202152456974, 88130.77355030482, 85113.68368523773, 56532.368218486816, 67498.4350653643, 37542.40694527331, 10927.513417764623, 98121.1038696103, 18465.544433964707, 31114.664300127894, 14376.177388220245, 95903.0782924719, 20221.107319815856, 46121.69784682323, 24444.65177364189, 11384.439597138697, 31194.263544585032, 5989.899059726822, 13528.495216397685, 49924.08836009579, 89886.20759230375, 9683.748643215073, 54191.83159674687, 57383.781475105134, 28062.206505994858, 92168.75452870972, 75374.70399240033, 43130.326044656205, 40451.83311188517, 85357.39554798482, 72512.72339607748, 59855.59921950061, 30627.522952491094, 74782.24117805599, 42285.1565937002, 72933.38562558514, 5982.721371932642, 71658.91090408017, 41850.65472997255, 54034.27520079598, 26813.322092812432, 92973.22601162562, 23312.181974370073, 1114.7828844769058, 21280.109141474768, 16547.230905112432, 57491.680015215796, 28813.015124588605, 91594.80342022197, 15933.962071803588, 37055.27833902385, 92990.7146643228, 87201.83299284976, 24868.473026870764, 96250.87932174582, 84947.63121408223, 60409.16138258625, 92269.19399972854, 57505.200972736784, 7063.462769964468, 77287.60723388304, 64415.932727694, 334.29598191618834, 42758.657839933025, 84989.84969929376, 63261.943460894276, 9873.424742438165, 58063.94125843896, 79432.59295583579, 69814.77073953675, 23740.246917566234, 34973.932196320166, 74521.96927422505, 50824.60901906125, 13295.56498279405, 96664.08567577033, 49896.705394073615, 56232.25580244271, 37344.001335986846, 27049.324680720933, 98811.54651746959, 6716.9127445276035]} +{"id": 1486, "vector": [8148.809395264256, 69341.75110329466, 7326.3571289156835, 90925.52780221746, 61259.13565004167, 70284.76608650076, 27363.12283742063, 96493.33490286836, 96847.57449026416, 93173.32975257745, 21786.300307372032, 11803.374223790897, 19603.58123862236, 85956.67340963995, 76999.2245149929, 37294.80416205513, 99907.22136679653, 74814.32357608428, 56694.10641187344, 2796.8805053420365, 15081.962152448914, 37880.454723668714, 84747.69212881623, 91226.39141099951, 89175.36911480283, 73195.69657948476, 60686.8209747751, 37489.78310419793, 91101.2106560921, 52968.464173227316, 16306.275582127006, 92047.31630549047, 56855.8137399415, 22194.46948468975, 46997.27578375198, 84582.01857984038, 83855.17208515509, 92681.57987575555, 77553.69839762221, 62725.74843133121, 46929.560898435084, 5402.480162030088, 5733.4861957265, 76478.9571631808, 73037.64189017637, 69976.00910406445, 67858.00138090718, 88016.35919405913, 77044.93296171527, 55393.28531160971, 15616.706102967459, 28072.418560609723, 83595.17079242598, 97471.1519954616, 78669.64354755018, 55836.50935370864, 14285.93609616644, 44572.359402601134, 78833.54009185114, 63146.82155947172, 79540.91729659527, 21145.925474702977, 67594.59383008721, 79066.46885079944, 37311.62802691459, 15330.134274100848, 42837.3611658802, 19132.2069326171, 50671.15535099496, 85686.84915079891, 97244.41745787353, 77512.95953490144, 8236.012675513093, 51993.52855380259, 22361.785613764783, 33624.80886728075, 98935.16808707095, 83941.58316611173, 98127.32180294931, 74848.76325721187, 40045.76601187013, 63900.049387357016, 99339.4435173382, 55431.59651401899, 678.7019519792436, 5999.842774520658, 97739.56568392222, 33627.24805507117, 57761.85041460904, 57229.327347670245, 58413.90177568854, 96391.1672803421, 24665.06393966683, 76370.15886231871, 84728.50347629447, 24480.481466855443, 5868.052119625399, 11167.244146301004, 54815.28400592699, 35382.025608390635, 43739.07280254861, 33767.250169285835, 11338.373378880839, 8143.703517482637, 88214.54392584151, 15229.670600044865, 67876.70326528655, 19020.96460477173, 7611.7252553518865, 82103.89266324507, 65325.512278924594, 44419.81292729636, 13662.056261828526, 57146.62403529651, 31505.28854474839, 47122.86573758655, 32510.423419460087, 87435.25635886223, 89361.51764537745, 29140.545509098036, 14346.806297005798, 28464.135688420767, 83734.69179087353, 58476.71103967527, 31796.028469856497, 69904.91780682017, 21631.154399497853, 68473.6304798474]} +{"id": 2021, "vector": [30642.031679798343, 70434.49094255308, 44068.47518397603, 26388.528880160466, 25530.48602485073, 74274.82273114628, 39901.92234687596, 48210.12540590868, 38363.358455858775, 98866.78468285354, 50025.1862430788, 30843.871323793137, 29323.022135605457, 99047.163527321, 19585.48771822356, 84972.11805329543, 57737.3618638696, 45836.67106608034, 81591.94410719934, 40949.89819822478, 34004.9683343768, 43390.227689156556, 30239.8013883748, 50530.97523131737, 41608.77021219679, 31520.119710782757, 90964.8925026296, 60944.691750998456, 52049.780028095505, 55851.22761838436, 6085.4241324934665, 17881.44337069075, 79448.90915274993, 15010.95410038389, 46124.02323785818, 78767.09985610067, 39092.61639177991, 57213.323897525894, 22533.313473281545, 89901.89864490127, 31135.303962582995, 49788.46960199518, 48383.505207735885, 27182.215429158052, 95479.64076234394, 95318.06378916292, 3978.9016795683365, 9736.194560287737, 26364.803466410725, 49912.43287651694, 75576.01857064098, 82605.70983397136, 14861.238696701328, 82194.9538984798, 10972.238326711504, 57880.597715332435, 83278.1200010312, 68257.79665494805, 36480.46002396986, 85268.93958897718, 98193.76111782838, 20688.356485899574, 79336.82631860639, 11944.782581000012, 91340.80685006545, 41221.04706562728, 38493.38791073804, 97214.29788143784, 42146.34419046511, 26597.62542574233, 97025.65825764857, 12072.483182183934, 1818.4215078612897, 55911.62207490225, 75918.41169212217, 86276.01990749953, 26775.74803723215, 31066.211561173885, 13030.735226351131, 4558.402920643434, 72327.62994628366, 7740.437107617659, 32375.77180366512, 83020.89398674537, 67953.00574429806, 14857.358255787523, 51178.61906647842, 14938.243197871227, 4980.418917771123, 50678.35890169412, 75817.45758989693, 51947.94742285213, 5041.765542381749, 69396.99884965173, 37367.114549738224, 69933.91208596769, 67598.90637126139, 76534.67401134207, 50263.731123282705, 72516.24879972759, 47022.326130164925, 82521.97148448408, 74881.19754951644, 80943.83996744949, 56462.12771169257, 32925.62801330919, 54097.08694351438, 11538.978741383766, 34029.39511459663, 45081.07777878227, 53584.46132462222, 62832.61412296133, 16433.633193548747, 96556.40012357854, 26127.410340352, 80455.36902500252, 92991.17923062686, 89782.83093139772, 99434.1091238322, 97022.14804843682, 30813.948235147425, 34205.19520499551, 82234.70069419807, 84457.23616045057, 65700.79786051477, 90266.85956705583, 51145.98933200719, 18079.956300792433]} +{"id": 952, "vector": [47017.19744923089, 86226.75796166918, 67287.00773873452, 9248.50809269735, 42184.60162794536, 94297.18545941697, 15914.712645544727, 25088.28493900719, 71135.22603509967, 20102.057281128462, 9236.047582811512, 2393.071086415521, 9992.953153053364, 41263.347504232006, 34823.619594860014, 27269.02047396309, 38744.79073674394, 56449.34809859521, 87734.1916765607, 57766.63276736135, 5365.227036302856, 29532.653932012097, 12754.33760845459, 19678.911530438236, 94652.18200063899, 10351.826500226214, 65354.93012773118, 286.30044486229747, 20122.394443787372, 1738.955640487816, 91943.31907664935, 5699.9121546191245, 91521.99330816334, 79125.59641229938, 87116.95719641574, 22003.40394968441, 5019.31702052858, 79920.21199263292, 99520.2219906769, 94784.58894208417, 67512.95422644053, 87904.81249234232, 84723.20281030769, 42504.85169073472, 9844.290076997186, 95167.98166778157, 10208.316346252477, 18822.598940709056, 66405.51136652777, 63104.17465419167, 5021.648503376319, 67953.61983454479, 67463.31788884805, 54309.02021532981, 68392.12143023463, 69276.09682040299, 75823.79504913594, 68218.81101728647, 8009.638471250191, 21861.69934399602, 42244.03932088912, 32758.12407239277, 48682.79991011709, 98719.51203138592, 55815.962433482826, 85570.55178499666, 54329.3343706128, 35305.47385916831, 6583.104813051755, 44829.49886428411, 25268.983576175087, 99127.83735575799, 32420.985613700137, 57108.38133844135, 92034.67036589075, 60229.57269302217, 81422.87488801125, 40937.92913798318, 44232.76233743313, 33937.656118173785, 55326.55702092964, 76662.02806646093, 18819.38843149056, 35669.46802462893, 77129.08416691121, 56984.11213499166, 7306.353826082479, 70866.34980171706, 1811.6003546294946, 68199.6231800444, 56113.11795027578, 73479.55230677262, 96951.49889421034, 58221.72528017482, 85385.40603777191, 34547.311917974235, 79261.24993807025, 22104.157822848058, 75288.33476986687, 98107.41686724183, 48494.93474125207, 11135.008993822616, 34602.80787711812, 92962.71824826568, 47632.291663833574, 93555.66121139425, 7968.255742072083, 12676.158421644446, 23922.18456198839, 86750.63086525255, 44494.48302677661, 57138.39877424127, 51395.33263365161, 2116.7561824498525, 27054.44731368877, 15963.454206022854, 49502.0592370776, 95178.00086183014, 25282.786536011237, 48525.801704581376, 71418.19729674401, 82851.80135732857, 37411.89477775311, 29601.905776831903, 40641.96192094977, 73092.85384645441, 80713.7581279065, 43884.0854627416]} +{"id": 441, "vector": [33279.14973461219, 57807.25975824883, 58706.37541092675, 83214.25858844144, 92319.80323883954, 64625.26487319385, 53314.080324166534, 57012.123188620644, 53276.68664071189, 32676.897962856056, 10598.419642081126, 45999.12620244743, 76064.77213592805, 98346.20008574752, 28044.81872632537, 61790.68888707886, 62779.58262645497, 38747.659712709334, 60731.55889871212, 37792.77822367204, 73709.16621697649, 60575.144703859165, 75605.82294563003, 5537.374645021764, 75677.63705278818, 25881.283204170944, 86135.049072508, 41922.948899970055, 64066.74560604034, 6700.292333188529, 92922.61441850687, 5173.257386693286, 73350.65214567445, 96127.15851371111, 34904.80486997115, 17406.019682659535, 36995.877294096754, 90410.06177105055, 36705.939370903485, 47429.72303655211, 97171.40559253012, 3563.8212801527015, 62622.86123925296, 75244.8106988183, 79458.6522297595, 82286.0677583944, 2139.610287944771, 29964.084703729954, 46174.416568015164, 82428.04398677123, 3679.6358511008043, 99957.97937183733, 51984.93520730165, 37696.923703395936, 52763.78133223011, 18553.725642783204, 22765.516167258294, 80616.83729305991, 78025.08007355679, 21686.463035302706, 6396.270836928264, 86407.9362761379, 35648.997783113955, 69756.63308276278, 36854.121170969156, 60222.9630194282, 51621.41374845821, 17902.90112310323, 43506.527936765626, 38868.030331092465, 2093.3212170033635, 19504.178655759595, 35979.172136034555, 742.067106915556, 20059.51828672319, 57119.24535979335, 68574.85770516226, 8314.642759175527, 89366.93726126122, 77556.68564444693, 96418.70007767093, 28574.904239189524, 23445.418683152653, 51956.89688379008, 93898.95722221007, 85709.32547995611, 82704.50351206647, 29421.16714462916, 93419.85703432088, 95091.61712039226, 89113.58726607227, 21512.41591323324, 49813.92746209214, 90069.72267328673, 56481.65333624357, 79200.69816735029, 31351.00028119303, 64863.52780812512, 92433.40874613314, 3608.881903158323, 48268.104063550796, 88604.79962345942, 9120.79630988243, 96809.78782102345, 49027.42168750273, 63352.705079461004, 23350.877991787544, 82669.19953646728, 91332.820914765, 85165.6305989284, 9472.530365065679, 40015.01169255289, 8366.775676603233, 80937.06118889082, 48177.409084092484, 1568.1007332703168, 64285.94168507893, 43406.616344164606, 32217.59709613534, 25234.952650538922, 67035.05880581195, 77968.79469960055, 74669.37790417162, 69953.22814391283, 61712.640197985194, 10815.147786179912, 15788.72104251221, 4776.846800994683]} +{"id": 1754, "vector": [79469.62728072748, 35390.971163628994, 93675.97783320928, 68439.19673344723, 55035.438640823355, 56681.77645213813, 79372.56822840929, 47618.17602121181, 80550.18887000563, 45787.53922504223, 73313.3784955416, 58169.127908726805, 99090.96784334519, 13091.561929211226, 6234.403513744879, 29150.175032989002, 35448.98703742819, 26702.48161222355, 1729.1494682077134, 19476.290126580698, 43748.872377305495, 5031.79181263137, 93415.05478073956, 59416.04582185117, 11319.005476948007, 85750.48785732909, 45643.51285713381, 40497.61655208254, 71509.96682556659, 65204.207687960115, 86264.0334576915, 29348.115351464898, 98890.67361558005, 18603.407848367526, 60211.01554023335, 32107.01366887487, 11170.368566752919, 10906.658980724682, 37592.367536225, 87497.78840172464, 61525.81139377881, 46054.588407821575, 8705.682722432406, 32287.460655517796, 29543.38443926757, 30452.65353405585, 12667.215086607908, 58753.339035281504, 7036.016341224505, 29722.229512591435, 52838.98899187681, 98795.04475380271, 9749.411976044808, 12386.708336186659, 28135.673222244473, 96758.96523655923, 95050.80665716672, 86263.32717998771, 15589.744637740665, 56571.76452218366, 81308.27393468734, 33586.079179819615, 10599.821913200103, 1789.169642168975, 44614.71058224159, 62561.15206744762, 90087.62427807233, 6702.823894464638, 13688.370994721754, 63751.45545131392, 39760.19486315566, 91940.2207836051, 10067.772578562695, 63065.26200600141, 57701.667391620314, 68533.77506868726, 70157.80142274396, 53023.25543260177, 13735.847348208474, 51420.949269404795, 28294.82747970091, 43103.82556986552, 93710.8690296709, 51130.05845252913, 62810.62999109294, 72747.78873165185, 44855.467901858894, 15307.411483326372, 11455.388632152364, 95253.74232181694, 28295.00451452869, 31077.32247597861, 5560.229317482801, 15470.055626909585, 18760.52851262776, 16625.578217961767, 21964.596552828276, 42552.59893585039, 10593.02442684542, 38238.247127015755, 77282.91656979322, 58043.20622992112, 48436.99480906283, 18783.716355208213, 85819.93115400012, 38549.916711531405, 23010.339263427115, 40372.339624328604, 90956.55515382219, 98808.22887411583, 93911.67328254205, 99121.62091160532, 36313.51689374782, 87962.43247619637, 42382.73848584634, 26659.216324960165, 88037.45077712073, 38780.76688361898, 75501.92406863952, 3565.845854982752, 85210.13516122085, 56572.629848670986, 93711.87984430516, 58412.263171167055, 994.4538036439243, 37526.078675728604, 27368.02466073861, 25092.919449792895]} +{"id": 1970, "vector": [99968.31924679058, 25760.622892862873, 42572.62303818558, 32483.269025909478, 21192.62120711145, 39214.13572223082, 68108.05415546948, 81763.46524408078, 21954.96524937147, 61225.208664781014, 92094.36558063654, 78884.67447088785, 50102.52460117087, 80263.28354828966, 90972.76834833379, 21317.151509268184, 878.3241431766786, 95963.69992457193, 63770.46536215496, 49291.25918608853, 32652.66686823034, 66427.90820739829, 83498.11982168273, 42283.79104553827, 12037.628162144221, 1394.544563040856, 69631.53589956355, 62569.468454756905, 92039.9786608642, 51778.84944064739, 83636.90417999285, 35649.21477101937, 32525.66813090515, 41259.03658039948, 56620.17045613783, 98527.51870948826, 60040.810997683344, 70654.61737250887, 67125.55047807869, 59.04548984670966, 6568.050470890152, 86384.3208802959, 79657.1455425013, 1098.4549704995407, 42680.91858172097, 5172.877885355753, 62886.35112711706, 99967.6439905226, 67975.28538673231, 34825.52687116676, 67921.54507837279, 61981.2347714949, 93986.9857807206, 1970.2267205813916, 97293.52265149262, 16500.19685492231, 50333.73610090504, 55697.579862897874, 9248.580038806575, 23086.203040584118, 738.1275783400864, 71473.46974835815, 16978.542596988667, 52362.34178563894, 95379.22776496697, 29468.43937462772, 60078.932773419925, 71997.72436927326, 648.1694135966886, 34040.409540957706, 34238.339167637554, 72545.18874037896, 92825.53731264772, 59308.6557060706, 85480.4786661191, 26865.16919928921, 71489.68674268312, 89690.65050893874, 18123.049731512976, 12076.51698911244, 9777.668548580188, 90107.94981916028, 78720.66862676489, 44431.16431976916, 47392.59858325249, 94798.71457500357, 40296.46942125716, 43965.22023659756, 97078.94831184299, 92076.90541488171, 58364.72095195011, 56796.72907685089, 419.8972783439614, 1039.2023778209802, 6363.845448341943, 50083.21901678009, 84722.14282451311, 82211.25381493429, 6768.079748544586, 71588.47803396663, 50848.913017937004, 72596.10223639883, 82437.41835037955, 68762.69087705102, 26840.99239032591, 42521.9061349454, 25223.996056203945, 51241.52098485634, 33568.179173171906, 82672.79764137097, 44565.87593573595, 66457.74107261437, 65938.15112435263, 32158.21056282796, 57724.891485826156, 99013.54380246668, 95080.4477135505, 11485.588078279641, 19785.463107710388, 4011.356375917441, 64120.207827200495, 14299.608328562286, 68948.0236323607, 67359.90091632922, 11706.827930015985, 22299.754827085628, 63443.196846807194, 68658.03517770866]} +{"id": 827, "vector": [2329.5274008403035, 59699.61232131632, 90796.7466437154, 51550.572510644684, 5715.063205503956, 83620.46079523541, 38023.181363560776, 99860.51333722817, 95737.3522025843, 79044.39531383602, 6190.937306379729, 81213.18466556206, 81643.08312879367, 53349.13206808809, 10373.714509174215, 72331.74873858044, 10511.52168301296, 13316.578644370613, 37388.737797882364, 48107.34470548689, 89022.01052019541, 68367.43982822189, 22694.61548310581, 12302.220229720917, 9530.809848123034, 73169.64783571466, 82176.95734423322, 60195.26493966475, 6909.7387174474225, 83409.8762851221, 23354.351814290087, 99196.94494343741, 46638.603617608045, 60123.45241063848, 96615.41999554972, 39850.91745006494, 72290.0978454811, 14876.612457052463, 88873.34799771571, 12243.760096083966, 49497.99489914109, 9567.685835737482, 21254.601935050865, 44643.57823297793, 11257.595441482881, 26246.53740160844, 98039.71228820087, 9350.422579850392, 1476.3474333663207, 45741.57227819299, 6020.503514346198, 28798.704680004637, 99016.786432836, 95270.78302185159, 35139.10320803127, 73812.43955217679, 61560.69178911675, 32763.012123580258, 6953.029813533851, 78455.20500767423, 32458.351796651852, 17673.55419143266, 75768.55922006808, 73650.19843084108, 23644.26279877574, 55742.519079375255, 28005.35217235731, 74615.30712384252, 97256.63655613294, 11563.302641833117, 21327.952246188263, 42201.18331714039, 45691.94190852199, 76123.67449152972, 89240.01090048443, 42619.08271878417, 10245.693121487631, 5193.686546771525, 36322.546253602595, 83082.03457282096, 81270.99421003756, 38334.469941805226, 23185.232869193005, 20697.9649925508, 79079.7802602634, 5292.802172179167, 73114.50678207994, 60199.31758721181, 15715.121500232743, 18403.3561796053, 75632.51815675704, 96079.10703420252, 31036.78012596709, 99380.88175102559, 59730.80639592243, 80382.88844669756, 72594.04249173647, 70133.95544806117, 71613.0674453291, 72192.5256563594, 98484.4920634025, 4559.296516380018, 79400.78986396221, 69826.4534446639, 76776.06739983767, 96308.30823021648, 72172.48097640666, 15692.303513820982, 11775.898500099802, 83189.60942620343, 67201.15133769126, 40171.13719498293, 84619.02333120696, 21793.750133257083, 62829.23480621009, 94932.89100078431, 70592.11028242437, 70259.0323655425, 634.6869581826064, 67512.5026793506, 72156.8302012635, 8561.482089331284, 27802.708365863982, 31303.94604424355, 73178.67265307176, 11369.019057382257, 59949.6164937657, 85699.77075928736]} +{"id": 841, "vector": [5849.875137314309, 18953.515421376054, 42388.53051726765, 42121.269683662875, 62106.43734464876, 91789.45334210164, 31914.87992808416, 66671.15869411753, 54148.39968434484, 93317.77543212987, 96210.33651537637, 71957.79940408071, 21518.875058849364, 83939.64721933955, 38381.56891808462, 31803.51014571218, 84233.58715661897, 99776.93084522821, 91103.84818220421, 74535.48250479292, 43887.31754379527, 16842.035621661133, 18199.918078178434, 93198.58529066519, 76010.27071925405, 1714.1359374456488, 64349.08337536407, 96819.87086722246, 65691.70907328822, 5068.946643580341, 36292.92626751518, 60903.76433696193, 48618.80925569543, 37657.11536404331, 12374.542731397054, 75617.72355148153, 67866.72982157295, 93059.60141354833, 44526.913097186094, 29730.79940235228, 8195.658324749844, 69790.82408503981, 89680.70893782441, 36807.7593575525, 76999.6864614808, 16569.479457308367, 54223.04164364602, 55011.79578229971, 29306.72633642465, 24386.238764015212, 37775.984545556916, 74610.32434164497, 3538.2432109153815, 40827.62775820324, 15818.641561021695, 67371.86277518161, 71343.84802820561, 13324.502448060293, 60546.00998194581, 35422.935293809474, 98059.16828695306, 56623.35613082072, 38477.144516450426, 8574.71406279864, 43434.271577951054, 17650.35303425583, 90743.27165112921, 73539.10603065955, 46855.61544063669, 11626.210055781172, 88258.82685806362, 47209.825110356505, 36391.39268242723, 17850.54350587375, 67541.10491046812, 96964.89726315944, 91436.01899602536, 45299.63694433094, 39265.75647075975, 59042.21624284882, 93276.90902845109, 21719.928066584936, 59456.01218712816, 4080.9549237021515, 49094.98268160743, 43237.6835574414, 32510.686983982352, 83829.80142400115, 30645.154417596386, 50114.431506035275, 94839.56012279901, 10180.063136690087, 66662.43206490402, 28071.655507246684, 52216.26394273106, 12346.100001536275, 40945.84099712706, 39412.07802200053, 12951.64437586721, 79095.69407304685, 14776.856839754038, 33645.873275004844, 83799.4702583455, 12559.694385351006, 90327.66998277734, 94869.12960509639, 18265.20251921674, 92283.09609556939, 32794.853352638675, 48625.55607691521, 57931.349916842155, 20760.88916157568, 2939.552878387619, 44465.49806173252, 87224.9950808009, 22241.881372849635, 99495.27420360094, 97014.00949597308, 81918.76244757113, 47787.22084520953, 39288.67125377924, 24205.105340660317, 33777.72550200083, 4844.158858283965, 80784.80959515057, 22853.262513601967, 56804.00647086482, 66837.21086064857]} +{"id": 836, "vector": [89300.64060580322, 57456.93705866094, 63957.762849420804, 49382.54507290665, 21478.020876918858, 67825.63764668921, 84859.82310379825, 83612.8466814625, 43514.250046407076, 35531.9638479018, 18944.453529214345, 94801.33701130941, 43155.3897410113, 42648.33136411538, 41491.02862604561, 78135.97619440392, 39461.63014664561, 56357.249954977284, 11783.95761490295, 30510.528364392398, 45667.462585631525, 20489.459828949784, 61721.88900730562, 5331.105648145618, 35575.474596545755, 47315.91553464205, 47391.66549729975, 82677.77724063885, 63217.6069167117, 24688.530706942514, 34236.67813527266, 13888.726653363303, 85136.37073675061, 4499.458193900919, 98811.19855151889, 67833.80174567187, 74090.13408331432, 99661.15262832062, 31613.541912442255, 76739.34667777302, 53170.577792874865, 72615.68758687958, 51825.38735260502, 66429.77820969683, 50942.9798640169, 44915.53209505754, 37068.55180335489, 3738.7809231798296, 11926.677987461931, 72544.24061966424, 77054.66886254026, 52186.20331105207, 48449.03124354377, 77318.43021445845, 87187.53848538767, 23037.393644515116, 33433.792926982576, 65295.499167906026, 6409.64601418973, 41261.749739627914, 78998.46844300049, 50641.98747740313, 95935.88820774508, 47371.668859960126, 95050.66153771429, 6554.485088291851, 43428.15488261718, 8280.907763539435, 54865.98686262175, 7699.448478233783, 61808.51304171923, 75223.81178682072, 55467.35971113632, 4825.918708958332, 50025.146490355364, 55388.58117258117, 38503.24605282627, 30231.45125038712, 619.0598165366712, 41018.58737769076, 43499.2383115817, 87235.18615188867, 76646.69566662205, 53781.53028675231, 81294.52483905792, 3958.061298178317, 47175.62969401633, 73227.10150718143, 1384.5900601403448, 30644.12052241321, 47899.34086515516, 48752.554169712035, 67869.1628452765, 60203.20726375921, 69607.8394580641, 51273.36154552444, 27993.22281885389, 55146.72906817264, 59868.81144361293, 93415.27992308067, 71895.95544666571, 28062.71078698843, 42022.26760099412, 86342.77034648054, 69301.7017738145, 43244.063617163156, 26427.610614836205, 62504.217655522785, 20052.634103960165, 42288.6915259683, 39064.09747949815, 28439.455952778913, 5255.74602403186, 96469.91335094151, 36138.89360773731, 36114.07468563496, 7399.870126714525, 94213.08065300375, 60038.020165676564, 83983.39023844374, 12538.20428923217, 93481.420360265, 94544.44214422962, 99896.74065935992, 43222.99783878794, 1688.876029717634, 45005.879128935674, 14481.951796383019]} +{"id": 1350, "vector": [2545.30688986786, 43928.06863442607, 92606.38516934922, 17695.723090983338, 46262.75132788853, 12249.910809724806, 43573.540786882615, 22345.522731414978, 82376.48407116896, 98436.66615855653, 49334.12532164029, 4669.044224464614, 53781.904217358555, 65511.77023948972, 90158.93130542146, 99108.43092107614, 22010.4862104363, 5632.169279826194, 7040.862528558611, 28670.294076514725, 24325.021385863965, 59891.572277682084, 95249.37068798317, 14153.527450486481, 74830.44306468677, 5077.569764408674, 37812.60660735921, 65196.65791264729, 56207.96088585125, 45632.401383537, 7750.665841781767, 81981.41131719381, 19671.854064518968, 95138.45134890318, 51416.36476547401, 31036.102856309244, 61594.89122576628, 28071.316582390828, 7047.7818192527075, 897.3127757724519, 59591.371230440614, 3743.433956138298, 86179.24304644155, 11728.911258506847, 69786.53156616463, 77679.51394744667, 61619.597868871046, 21775.920525302474, 47504.86531635872, 30235.34774042235, 53255.1316318083, 90491.89116342734, 24747.5825337233, 71026.68882750718, 46528.68416258855, 23117.748590565134, 77470.14872906181, 61685.56344510116, 52103.349573543834, 44520.91716769101, 6551.415371291158, 83056.55588555455, 24632.729927455788, 93936.52579310024, 34860.45535259846, 67628.80970327037, 85286.9477561156, 34837.18399869313, 67345.2349153672, 63507.87914126365, 75428.42569997732, 65986.36104324907, 59380.45506776644, 83712.00881444915, 59008.421489708875, 44565.17737656481, 60036.39482336499, 5070.44673192052, 62629.40050494006, 40999.60524932248, 75272.09175015653, 43340.86591753681, 80527.64021672717, 50021.041775639795, 17794.73965409819, 80165.44220493681, 10218.792353193363, 82142.7979469315, 53197.20574814643, 95371.63605643429, 76494.85095936118, 28082.629275023875, 24675.67018065162, 5060.540277020109, 87509.58165213441, 98997.3380833333, 82330.95973760763, 816.490077380172, 79864.04826331786, 99995.05070015436, 87302.59462544833, 47271.859831331276, 54019.235859230444, 19754.935137864304, 66273.43801296351, 34627.68731352034, 43813.98712882199, 89488.82838768723, 93023.69248253564, 75516.0635541249, 64199.69174446435, 52622.44746237286, 33884.94003530589, 71872.17979054144, 95542.83911838957, 41815.51694336592, 43606.78619482798, 8890.367296780032, 70912.03024630825, 89583.765481088, 98694.1977703384, 9104.641576474649, 93129.15294611498, 61759.66577936814, 54457.1242317173, 21222.918503381592, 24088.791125161868, 73411.81232951056]} +{"id": 255, "vector": [72281.89632454133, 72525.56457699595, 63222.89309045221, 28449.29159135098, 83071.815960989, 75936.75499955044, 46636.47915515543, 99756.69636852636, 34260.47584653636, 21976.320221947466, 83207.75033023002, 33874.12277655098, 48644.59306904048, 8453.360536262511, 48215.58206152077, 6762.374436337815, 55363.46726442912, 69274.00211548484, 2764.524182763839, 26365.75313000442, 322.7218671986609, 54156.514037890876, 58343.754686525, 65601.19546906017, 70135.58925871999, 80106.63778895109, 37610.11864508969, 8461.739404215108, 88189.4045212079, 93134.45833164942, 50195.495812067835, 73554.86090778171, 92169.12331574419, 84055.03208738966, 68319.7159859788, 99831.32899751818, 45659.411557958396, 40718.229754413456, 47153.89184125564, 49888.82054866667, 38080.73340059991, 42228.60774133354, 6614.5554403225005, 68945.62641354265, 96707.47420498048, 14260.549545392409, 38263.4616097022, 68104.10450132802, 12328.555393402685, 75097.41348760898, 69164.7420167559, 86229.51888177638, 28655.114422246297, 61010.181278513366, 35477.682858323744, 73570.08889819217, 70301.05414715504, 16948.827518864586, 21355.758675967863, 98445.51236748323, 234.3786769439249, 66716.67305409731, 45841.470622989786, 54818.89073890549, 60167.70731041914, 40017.11377396294, 54980.89033156893, 27131.87492569018, 38333.68919116836, 43998.46988774528, 24316.078997798086, 70906.50826056309, 98964.12706535094, 84153.23550766973, 46427.00656869061, 23707.86620353371, 22114.45725069284, 51727.06777869934, 95656.83374761885, 62685.60318743532, 28472.741141321923, 6315.973175753697, 69546.46844257756, 52408.3513121068, 11893.140129241032, 77759.6691922777, 76691.4438661782, 25677.857579815878, 39300.659692305526, 54577.14911968888, 38813.523375900295, 24810.893449910433, 97856.03280173276, 23008.586818705888, 37635.5353586721, 38585.78851900618, 82361.92579330664, 94335.29511702433, 21009.52662600429, 16915.054340855528, 80042.45142445517, 94042.56750264374, 22178.48131795358, 22533.994689359184, 91656.74875220335, 44299.56977597779, 51003.75870092716, 86075.3913545466, 69803.14471566997, 82097.1341116841, 26554.294240973508, 8581.934332680496, 98692.63766656311, 22473.919443337254, 40006.33454423662, 69798.57388944706, 62577.47110018481, 39789.166989165715, 83568.22811990799, 56031.09378015004, 94300.6887024549, 12839.572106446383, 82111.18187569024, 23283.69605615337, 74163.27668222609, 43496.167912021636, 66908.80720882375, 87540.99766937991]} +{"id": 771, "vector": [11067.408852975403, 26890.90909073898, 9907.722386867224, 59342.21959403577, 29562.08420565535, 56737.85833997603, 10625.867133936561, 1736.2850758326665, 46392.9245294969, 74116.0863942978, 65246.34112848683, 82362.04346051709, 53369.134225470574, 92595.03416957174, 63842.75467950534, 94277.25148512857, 88405.21791979451, 97674.05955261533, 59841.72789626024, 94710.47924047093, 39565.59209183296, 31930.789951458162, 30170.657276599977, 92293.10304557432, 32058.966723657155, 6034.213128067856, 70515.05206867101, 6484.252027078852, 4793.288747686586, 70012.61358930111, 57759.3673383379, 9333.406517356801, 10529.437633800564, 94334.4220768046, 4954.757715870206, 82303.55132043017, 11951.359799600692, 21840.213129684118, 45216.16402256854, 54856.15115913533, 37691.63947736637, 2331.327188219956, 29299.458083382237, 7891.085361214967, 24765.17286066684, 14.126498465694137, 45372.66129671865, 41271.93843444803, 46115.001124681374, 55571.73073342988, 464.0851275860447, 47555.853627156444, 81936.19945509189, 75308.26184583268, 36484.59828120942, 44744.62492640834, 27801.280993877986, 32619.43608500435, 21709.16791993449, 54599.73399293806, 62104.36271452362, 27078.339256331972, 85336.19145092448, 62837.978402459026, 14286.167898523772, 79409.32956041182, 92124.74018122179, 43632.49356962656, 97382.03467039138, 27381.65029563484, 29481.427112767255, 68024.44328560507, 29676.786080786544, 66760.60559366237, 33069.35504105525, 30813.678890083007, 90187.70983763838, 33021.78971071352, 34767.370479405436, 94059.50478717894, 94498.94858235076, 45610.92906433177, 46017.22201296371, 4859.408232291673, 3129.8957573065554, 73781.40079984623, 27009.04619804232, 77424.8588997864, 88024.36541503342, 34916.16911600316, 46844.07816991694, 1898.6636637402078, 15998.625255438092, 89766.18482928444, 29558.262652336274, 37839.93445414304, 34910.364041425215, 38040.05320067611, 43310.33869372663, 2218.7548413221903, 33458.34331714625, 5926.219442785241, 2524.085633837281, 33967.41359170395, 10378.565182704546, 29083.34250845025, 88612.14949724032, 73476.42644842729, 44375.79490004534, 74439.10257956054, 13315.908130054233, 53379.768284144135, 80431.10150727733, 98281.6211909862, 94853.00771834355, 24056.067231386845, 91487.44750764602, 29219.3183263203, 10868.414850756548, 66910.7904977937, 3612.1505558716294, 89165.32028653465, 10231.831832785698, 19370.754757487397, 68260.60776812378, 85812.73408290053, 25560.16145408033, 19061.63541435203]} +{"id": 2004, "vector": [75553.07161326273, 34478.87321033984, 30565.22095911838, 46710.19968621705, 58476.59044097767, 59051.23447349466, 22621.503093057127, 92174.47664401279, 8496.895984918063, 20493.54088695521, 1493.1893510890748, 22498.960291007486, 18627.99409158481, 71751.77217783309, 32768.858041094296, 95433.10985340568, 61632.18913530675, 13313.953379366605, 69161.71135274008, 6502.772504861187, 94302.6919004098, 91274.79603616598, 52407.290250054706, 83894.4140311249, 38869.27285150071, 10275.257705138774, 50309.71226142871, 34630.453524099226, 38189.28721273789, 72501.58361208958, 11343.43691097819, 29943.63403104674, 47653.52325896811, 11376.264867127738, 52238.03689891119, 24862.014158803235, 20222.77199727893, 62666.653655255825, 7037.512023665194, 73783.96102589036, 24873.081123116237, 90112.19237158682, 95722.4113796523, 94275.25467353968, 83215.37989556944, 81740.61176552593, 11180.599574217642, 74227.16474399742, 15464.256349397987, 99486.33192794665, 20768.19857236787, 93014.16053955178, 38198.67428155934, 32022.897375503002, 72578.496406841, 56084.78475462741, 37903.67073967637, 86455.09801259225, 42853.18629468717, 64258.30383934301, 70209.97994127803, 54713.30591652869, 46416.38454392273, 72634.98275576947, 92358.89106732719, 53103.87760324874, 95413.56511073066, 32629.995736118322, 22403.980689499826, 92037.50588564586, 46563.78852436396, 35207.884258108235, 3436.5472994551947, 20235.409179551923, 45004.01698886053, 4761.3188390594805, 23925.315431970062, 46227.84642108133, 55273.22179846772, 17558.752318292314, 78577.80002312188, 17080.260007877067, 81122.20061796055, 18816.003000958703, 13762.077863172895, 19502.95755401973, 30488.7041115689, 2345.5700195320173, 90088.77793096846, 24291.492935410188, 67288.5081416587, 13435.83557582122, 53695.58622043936, 48695.3185199204, 11249.555519144262, 12568.091910101242, 31765.82692148279, 48002.669950537354, 81501.08433524011, 23495.00827996136, 84098.07475374606, 61259.77683328023, 94024.22948697822, 35544.67813693488, 73806.63605339183, 43809.499197996905, 11155.125448255421, 87301.5132202103, 26651.018040959607, 39023.84051918608, 97290.21698459407, 76403.42621918494, 94671.86704125065, 95831.9458106951, 97899.01426345373, 49931.30841624074, 71246.96412810781, 49491.227945792125, 94655.24524461875, 78726.63314811962, 69121.93313542416, 57128.581342214144, 35183.42651767411, 4783.582849087397, 58676.87027967179, 13018.016098737695, 92595.1725693356, 64910.62398792833]} +{"id": 1755, "vector": [11052.046504426195, 23893.61021522065, 45459.67697534038, 42497.53466427313, 10186.193337123606, 63273.691167037105, 82842.43229200636, 95070.62399629988, 92339.92986202477, 67699.61277626755, 31.3084570454758, 90812.49831040332, 10494.872566072045, 11580.993331417478, 99103.14853764311, 93022.5760763005, 84083.9654895174, 54004.529482071186, 52361.20455642449, 94469.70859793856, 79225.2445130526, 17447.090261965615, 87715.07371350708, 91307.09225496881, 74514.0754864689, 7262.559788178313, 12321.532500038746, 93389.68624347712, 3974.6036033207142, 66957.77926141757, 58437.966939803264, 88878.94204678397, 43437.04021384183, 77593.80076285337, 4940.626538486647, 79909.30753965597, 41038.53694648657, 64602.04694579769, 21561.453512316762, 80388.06633668231, 35284.05793456582, 17709.825135935065, 32590.504659627393, 83068.11268051255, 44707.74388389591, 79105.93245480629, 54865.48347260599, 7007.996175023945, 96970.3258278208, 94094.23355779983, 43762.76785020373, 44698.09287126015, 5659.782123770951, 99990.90216842237, 79101.55956167188, 57274.341679381876, 9672.776063639976, 56177.299388376, 41856.84560341875, 18193.132023043203, 58332.53630572569, 89749.46713523682, 24587.841568591328, 52184.46920684727, 13390.14908951497, 29165.312121100797, 6674.700902370245, 6566.247163958538, 77109.46762198761, 59981.523491059765, 60555.19443105403, 21014.418235615805, 10358.741928731151, 24308.287179042585, 58252.452797880585, 9919.904955751057, 66055.30622182619, 76026.16288580642, 116.75606876242472, 94555.70436462386, 52315.20856078351, 37726.2667042818, 54684.167410974005, 56244.12558983911, 68392.72713933248, 18098.4613737034, 14417.433336503638, 50615.43039216575, 72388.84435257349, 9137.48862700312, 66487.69029106745, 19263.741796016475, 92941.27275945281, 57301.216816212196, 80087.87201473977, 33358.32983180669, 32490.494172025374, 22490.55481467651, 80416.00710252034, 47537.47181600242, 98434.80984937561, 43286.27893816146, 44723.04514961775, 61030.02165484096, 4560.147167263084, 21640.885732084447, 86246.42296198089, 6685.88394915226, 24563.271220344497, 57392.86598265878, 58932.31821716597, 86852.56932425882, 27484.349850476687, 24906.596227819355, 87317.45702594123, 99198.05972097265, 16468.345650084106, 79996.22429995162, 53976.15012441177, 35131.98232812075, 62345.66052077906, 43372.264037095185, 18631.07229838901, 77074.07861220847, 38525.90972099782, 99272.94703532351, 9036.710182175722, 78069.25650946239]} +{"id": 672, "vector": [25364.391985712842, 52922.348933665344, 90028.2166129547, 88815.64561652696, 99686.54246669516, 46834.91641537265, 7232.889038918255, 91223.10638088732, 95710.4384178087, 81634.99159979932, 20317.02269602881, 83884.71845692208, 45252.912894150235, 37249.462166425285, 49567.95292014503, 4547.440269257708, 55975.20561956968, 70176.95565113246, 16606.64434010313, 45262.52850723602, 66365.25386940487, 18696.231981698853, 68504.46396010036, 8695.410710499962, 63026.72603915629, 69171.93682602301, 16400.862421192796, 63265.065907914744, 78263.77072448241, 84400.54956628574, 66123.21411437502, 29770.77080863334, 37198.42334802903, 51695.03511933137, 27670.6921721946, 83150.67485206551, 45608.311095160745, 30585.925039606587, 24184.374162236945, 53981.733858897605, 75298.50018835437, 96477.11923986195, 26129.221979844442, 93932.25041321707, 91149.13286268107, 75647.23414028322, 61102.7320897875, 30512.56848547469, 2648.855300215336, 34607.03543219785, 66354.78799379447, 61274.1601640199, 28527.516409687436, 81385.84732029658, 94769.75680321662, 47759.11014570039, 17268.176628494435, 84979.7953733157, 41848.63903622518, 66502.82421203604, 73298.12428336007, 2410.707744986729, 57216.297059250275, 24988.583404700705, 984.283619295645, 72375.56056993168, 66544.45885842798, 7096.075063288731, 31286.387391885273, 86465.06513340145, 85555.01255036826, 8373.541669494878, 58537.7240860096, 61084.09149048114, 47996.60997104242, 33141.33434762134, 60399.32489547749, 34704.63927292575, 52618.59189047912, 91081.45743842388, 53398.2053278532, 88282.33911327788, 53157.992142636256, 43878.82039447724, 95607.71883635777, 52112.66724517889, 52521.457166975895, 62870.563897459775, 6758.744957642671, 31934.365769150707, 11271.665297687505, 94448.96278878363, 23685.04697589384, 70742.06344030997, 44392.46645820386, 12268.683907076105, 98566.22466493894, 73482.69510555954, 90883.36527635496, 26472.14935228621, 64928.06310858742, 66363.06662251084, 23722.102376572162, 17205.15146983419, 58439.88204642483, 81647.2604203493, 7284.399978766187, 38118.36493661015, 57920.016804312734, 19933.310742255682, 28820.074593246547, 38905.87929688511, 54127.51790358391, 82030.87820465937, 9776.030219489463, 6845.208838374739, 63070.074004251175, 55050.00196034044, 94394.02035608552, 89364.23759589967, 72264.54819936273, 86956.6308351394, 68910.53365229268, 5659.639088340363, 79120.28089013853, 54679.10795673569, 78457.88963749204, 43930.16153672899]} +{"id": 738, "vector": [49983.38133791061, 14828.811319063729, 2062.669113540183, 8838.13617403394, 17409.834451260765, 31933.42457977685, 26446.09184220038, 7036.225210945557, 21115.628188518833, 81609.92666049003, 55429.958786526026, 40513.48292451552, 15069.42429368181, 54618.81037225205, 5883.188020335284, 59899.40802067311, 76310.11238963666, 49182.169671453725, 96914.21810798711, 3724.7991153811054, 74128.44006542636, 92194.12156338601, 71776.44830180467, 12836.844941945246, 69217.5775679554, 59656.692655822655, 84465.89562467672, 21533.866789829204, 61440.620994122066, 33260.82932295995, 54604.88369303781, 45887.04110826737, 78500.03774713952, 97128.53300645141, 52723.48105046364, 49538.89757422249, 69179.6844359009, 80713.65882246436, 65163.90820375605, 88913.64037041311, 82084.0072099031, 3151.967624136409, 25185.467861366153, 73602.5053287439, 97874.87957711863, 93615.68892292079, 47041.783597288566, 53747.0690795125, 50777.2963026461, 15541.970004490002, 42698.991132011324, 32792.11423044391, 21908.88547961449, 98354.71141011642, 13992.063446136637, 27130.418135731583, 62943.74880937272, 6531.947294388374, 79471.20780685681, 70602.75494129158, 48574.30872608261, 53430.91760007197, 94594.03921829075, 27390.158376467265, 29952.006589047374, 90703.23417685824, 67907.08847593018, 50594.04890681838, 25539.504123244405, 89029.04555598859, 22605.50655404796, 53336.684448329484, 90249.79947949755, 20313.714242976333, 66525.1477201545, 60736.92441303263, 3063.130228206268, 83356.97220034234, 15154.696054001026, 3815.695149649667, 58003.70225093807, 48239.3741845909, 60515.89103599679, 62070.79471018429, 84146.68951160098, 50054.02644579215, 51928.72936078994, 10390.362394336928, 93650.20797553918, 57421.694229070665, 81738.8767885871, 65633.8401804195, 82941.1363991866, 92469.59220016545, 16394.770077786412, 18261.818139754647, 4195.918465374093, 13226.958735195838, 81417.29293685498, 19269.552853902427, 7917.434981255722, 25767.313503077483, 77193.53297897101, 24327.943282439468, 38510.94645985408, 3695.5703651005647, 52116.28564272912, 58149.2490194084, 91340.20255905963, 5679.917849216187, 26422.851745940166, 72717.71826212545, 73807.74382545816, 43713.918387505735, 64496.19334553439, 84443.38468461549, 44099.8005822985, 97804.35703785798, 65083.45759518852, 48371.5126334255, 41748.35876054724, 41365.514514424685, 13479.196308662034, 32451.140901529317, 94886.05593756883, 86009.47604877708, 34558.44830953773, 30144.920241273165]} +{"id": 2023, "vector": [63323.237185763755, 39283.44491342186, 44415.43787103223, 46984.41512819165, 86829.00818153044, 34356.559910731354, 74108.3965269157, 6593.745652904004, 59625.319884976, 57140.7025830849, 37905.07929006527, 99499.7126080601, 60490.797638498836, 45216.878313376794, 88555.65580586925, 46242.78114335367, 16992.997022927957, 14528.106351883363, 29881.513128421368, 36994.15131891959, 45310.3593896656, 3614.2252291179734, 75609.69316370692, 5119.595474473837, 42487.01039578064, 44774.361330539505, 65118.70622140953, 99914.06714866075, 2586.287838080681, 53496.58072772167, 29641.13565095614, 77009.99123773341, 24493.73937454894, 38941.42814156163, 46227.069536585885, 95764.57526272773, 78426.15979421153, 29868.07099788048, 26054.556790001927, 62076.321426589384, 62754.77821783702, 47689.0533246842, 58809.67551128887, 33395.948842685095, 64057.68704596411, 4682.151304646342, 42575.54125895593, 54108.62871556307, 44592.87402188684, 59195.90424910469, 43743.44031235266, 25287.19528242844, 20776.293875348973, 60449.11289172828, 1682.5864041647765, 41301.58330863308, 69173.88029303802, 61395.53196239746, 1115.3596591281346, 47871.6530331447, 94890.13224979983, 2407.7687485246856, 98004.1293047692, 75301.35192825874, 33455.83476273538, 52582.28213349965, 22036.460367692624, 56508.290897055726, 32211.996534077945, 77877.62809381777, 66826.97268761373, 85150.87823989391, 16220.716357640518, 18796.42656267656, 54396.316012325784, 8614.9866549868, 91117.90702225192, 38154.92642886494, 8456.27319569614, 65898.1987230469, 43425.82906741474, 43906.594522592866, 28680.059402895862, 42656.68377490428, 28579.727146427325, 24485.07577508342, 57678.346650006366, 71316.13418760356, 42631.84841398777, 40772.01184398179, 87033.15604036013, 76694.11747416026, 95891.82683485709, 80881.68947951484, 72361.01194352037, 30182.62800173219, 872.0624887316841, 17062.306307147333, 52394.07664087095, 9884.005952521667, 22185.82607829793, 5513.287533573197, 6174.632225957688, 78301.41623052908, 66730.91672163816, 59325.23233878135, 9366.954550879569, 44470.265695097136, 21907.171083600828, 22971.650322010482, 16886.45657821799, 86977.12203742545, 50012.79418286857, 16228.244461652352, 80678.94226685012, 86380.7720220789, 26590.28608806675, 68081.54025338225, 69437.31330941337, 71811.89489937502, 32772.134339297445, 82171.41539182603, 789.1032337374094, 38341.23388151871, 47814.124074861, 39850.89297911014, 63017.706454619096, 65281.56734721005]} +{"id": 714, "vector": [721.4175708493253, 5591.23955309726, 44380.91971577205, 1013.8836226041615, 51622.466056341334, 43214.82206286148, 36895.106039876024, 65809.32758821537, 40132.66844174861, 26844.518816450534, 48558.92506737421, 7760.290398283376, 4304.275029419257, 24577.395800641276, 38262.81585332819, 91153.76183129428, 26156.16267604901, 79551.7335203002, 15614.289959391059, 50793.31579479827, 25516.311894123166, 58778.01196758542, 88390.0796937794, 97629.92587513194, 56709.088861007906, 2784.555482836726, 51322.70750415699, 12658.157028455664, 94237.19012023688, 12145.587106576706, 91830.85230847243, 46415.86796767559, 7327.800362852666, 11236.53241742092, 89183.3047944495, 38830.154906806456, 12861.440176206306, 18614.19123148549, 79880.50464352629, 49909.924928252156, 22157.202507687958, 61291.89585708238, 35586.58104927387, 23441.450720488632, 66151.9901439507, 160.98589260372665, 46337.34721936924, 66984.94906193945, 44825.94222997382, 712.706145242159, 45260.48574524793, 736.2403887323566, 2244.3312571326924, 82041.03684838602, 86433.15811681475, 48370.231680746045, 97513.82994846007, 78253.45904443532, 35043.25068558307, 23214.113696624372, 43350.88272422899, 50449.06111565451, 53990.3497103374, 35070.819201581784, 96316.10917940308, 24409.100307936536, 46388.374853294765, 66769.68544129707, 202.0278594356717, 24022.41738881017, 24170.229801234356, 40233.596976141336, 27736.183872242036, 28317.24687853915, 75887.49217034827, 39106.66234582105, 29170.264366237352, 52210.78266621209, 54571.18341760595, 11515.576418440187, 23971.49450082453, 99737.13728661397, 19199.023387204372, 93110.38294621494, 17690.27714741026, 10623.830178370652, 71938.60035431219, 16362.00044636763, 59337.143998208965, 23051.199608711937, 652.3129642429071, 24379.987387001478, 31548.57330132468, 10127.507052688334, 93018.28102515974, 94364.95007071605, 42673.290820919196, 8787.437145275722, 66743.96351725492, 91645.08169796833, 47876.738419668494, 52603.730023940334, 43609.5542474349, 52606.41561985917, 40552.24456423766, 75540.18857899509, 6188.461758937447, 28577.77888737634, 98595.41496645643, 17014.965302841058, 13464.28389592772, 98729.90841978547, 81290.22243409278, 9899.737082125359, 45547.84326145514, 81863.84726549353, 8275.095894857575, 89953.83187154627, 25368.878342260825, 18140.3438190874, 43147.582713734075, 14518.76879602868, 63032.43707968396, 13459.713558791342, 99221.903234143, 70770.60347504259, 66434.76848829225, 20835.064448921257]} +{"id": 265, "vector": [77900.35954681096, 21669.889756492623, 76490.98548810075, 608.7887533276381, 92870.40822547002, 12603.643457111702, 13906.020718197231, 53095.222051130906, 92857.87341989811, 8836.533951077252, 1869.3853090722357, 86414.11168371368, 88084.91904710323, 61207.8402826864, 10789.31982066652, 78593.74776359249, 51874.79436216823, 86207.65660383669, 57898.25390662029, 8374.783260923146, 25233.34843437074, 43775.647734556966, 72582.33797627388, 86550.88357044564, 17260.01480312108, 81553.99643348635, 1970.408165327786, 53158.79701622062, 30143.68850245377, 34836.698263214894, 59485.21741082081, 1603.7658037129643, 61779.378285747014, 72437.1629931997, 87529.97229469821, 22009.72082592757, 63936.75182803045, 12829.186008945759, 91593.20418699189, 76648.86482228695, 20941.288841053796, 3479.953886692777, 17879.274600828045, 330.80625569712475, 65297.124124411734, 47295.35396282442, 94348.50907381666, 85350.83661596585, 30995.99233196463, 51765.209050062156, 14513.229587499844, 39784.71835974188, 12939.721484193922, 24582.34506142608, 71854.93589225746, 4410.931838251242, 14848.324522672607, 12612.77872170562, 99719.61735671936, 83077.3061847358, 95015.36932206592, 50906.727452639745, 60290.679396637606, 48257.62070998775, 44243.45769170359, 56656.15348707601, 8716.572998290883, 87390.6221102908, 18683.168034512655, 81926.00413743798, 43519.665766867554, 49500.01534918316, 14308.521598450452, 48318.88673295621, 13989.333759931154, 80282.54187557001, 33592.80812780038, 74419.08103501209, 98941.95718030972, 4087.3425441914524, 31128.957380815624, 14853.62539811681, 3644.9228396168933, 74975.17773738307, 11510.477538789431, 65540.54134941501, 45442.6510166677, 83736.34109499404, 61306.51316860996, 27745.999756339213, 31214.501148784046, 40022.86949008661, 77712.54411700257, 91882.29694774044, 2496.2267232981008, 19277.050677073414, 57313.08881527604, 74721.66021794679, 25408.658893688083, 30688.751966230255, 51578.755630096384, 12923.823547140157, 34922.73875794132, 72363.2664672512, 53541.0381492469, 29010.13337701899, 80459.64286124652, 3402.3603192644414, 74585.35195377166, 1853.7399442318913, 35122.78354774268, 37119.50048217226, 87980.99341968652, 49879.84510808593, 60709.02273116906, 62653.63503787974, 78447.93684178451, 91903.07571777349, 60275.38214603676, 94390.94209176651, 68527.39618071345, 13948.995362769401, 65970.03861989414, 2959.062934871315, 2153.9425301797423, 23305.991210011533, 27698.05334746843, 15607.190165983364]} +{"id": 1408, "vector": [35219.05473651853, 41428.97622302334, 86077.73677608071, 88998.662825926, 80605.60481601243, 383.9439639611975, 35394.35766883655, 73.6515484801803, 12294.518024354495, 89970.77173947025, 74789.19247204825, 16061.056755999436, 4008.166962786841, 96856.35700505799, 2001.387022183454, 90204.35260546823, 5211.1663066841675, 72466.82275667573, 87850.08676197467, 39404.078009774916, 50195.064279627055, 27833.122822739242, 28796.558116302695, 71980.72543481234, 10220.028784626356, 1159.9725119656123, 62356.30789659631, 75347.78869638611, 18063.146997756165, 33700.856096534095, 26638.379542865066, 48376.977798732136, 56366.85889538285, 25524.342162357218, 62828.232118384716, 39237.902747831075, 91595.95286246773, 51107.74522082632, 97936.17059088961, 34879.33181976325, 9756.956089161584, 78256.44640937822, 59466.691887327535, 92441.78553026411, 51518.981769280515, 63531.003236195415, 30549.382510400723, 47074.67796987725, 85073.91191485038, 93515.27661506836, 96504.0744582334, 67675.721891274, 18397.418503373152, 10284.503422274838, 32341.08341794497, 76007.0347018339, 15151.02775659788, 15429.700103701172, 53766.03201078843, 25349.148511067466, 42887.56393141046, 61219.41828448011, 29859.029293182448, 50103.49015474675, 93763.61204326345, 16898.632759924825, 77898.47131453348, 98250.7958622207, 33881.845954639946, 90584.9518405265, 5235.690227421119, 60046.88616908498, 99216.53598552, 6324.913440832313, 22526.98022582933, 96954.31182181374, 1542.0754178228635, 12726.080800472306, 38996.809707370805, 9900.06928863857, 63497.372939858775, 23136.483563910802, 4294.582838550476, 78304.0485967978, 7956.783950920377, 70235.03463979426, 93356.15110394442, 76440.23402920313, 40056.27151385497, 91707.56693281465, 13075.08545295193, 3411.0122012269485, 41315.141412538156, 60564.968993855684, 69391.79211804498, 43668.74334506376, 41389.08292170959, 87864.05856225562, 41966.486621621676, 86448.94174067395, 4462.253179494024, 10136.844165174296, 45931.66981336032, 78685.02434223656, 68103.67224119722, 92329.5129680638, 80937.11982463679, 23472.465836211486, 46233.30342167254, 6691.78777222923, 17990.486013970796, 49842.430310628275, 48754.1842680582, 82633.02846576615, 82922.36140781251, 87594.64029585868, 19633.93536680673, 15746.87987395459, 81471.04750329562, 59655.467903189296, 54173.37706854734, 50127.60943312534, 9362.269358373598, 34742.29420902463, 57583.73177922969, 1719.0964255319298, 62056.85207359849, 66424.93401723147]} +{"id": 146, "vector": [36082.64068319637, 7248.5918926547765, 57989.494938050615, 10435.402849906306, 85466.63397223086, 24705.663567745683, 31006.22983926299, 69559.9353854253, 22998.04878015386, 78853.47648582746, 77102.28155924763, 84823.79152826116, 16184.227475596124, 74335.65834444563, 94748.77259926274, 53554.76810689704, 6874.005238802716, 61522.42923921284, 68030.04784012969, 87826.6167874149, 76340.07946228204, 51207.62979244534, 82331.73225059797, 98706.67752106847, 31015.23033811997, 34867.122619639114, 96109.91221645777, 38314.03603225577, 26353.458395219244, 68102.89281925805, 71740.33907856049, 74709.37515762215, 73611.62613971224, 51602.2578137715, 55767.59985684875, 96713.30276783601, 56019.701575533, 83160.21349487212, 72490.7076133855, 50425.2650958398, 22868.81787823816, 89627.30843504418, 86488.29918822605, 88315.82836653739, 52017.8979183737, 4861.209818723866, 48789.69557642754, 66913.316992283, 593.4691785170476, 14511.886472238379, 80223.91853200727, 37757.596477837316, 85227.71452800787, 13160.4881855911, 17057.354293466065, 97770.66992890918, 32072.710176906337, 57730.10341811598, 90920.03253881782, 24006.215891711756, 41827.672404097626, 30397.58146657462, 26406.561492554094, 9476.893106015683, 5383.699635820316, 76037.52664304605, 57070.51950640598, 43674.90791432356, 88215.82360716553, 12510.758480578921, 80743.53801851455, 11205.169808645443, 35752.43808313705, 65572.40684596192, 96280.40820878623, 39063.01591882718, 57719.82131237521, 89441.85159260736, 69374.2827600384, 16251.181424380957, 11175.24268968384, 97856.64051673576, 18408.144743188313, 47029.01339954098, 49162.932472170876, 49707.56920463094, 33627.99299615083, 3774.8381523191265, 5638.12723946272, 20636.540496521116, 62287.38112548573, 93162.4193940579, 96670.00617055407, 32468.68700961796, 78400.33104272393, 36527.52231671205, 40845.17973839482, 25123.1032882819, 21092.89511232787, 96459.64151020886, 70916.0055979851, 1941.3707065977114, 94247.90606261567, 85316.72772696013, 89503.84332269858, 98838.43634835827, 48054.26477217448, 93165.23772613498, 15177.197695827905, 49655.74517469444, 35191.629234403044, 38437.16777642796, 48566.48028516462, 67215.89751871204, 94628.04921757548, 5649.066440409578, 12158.674497465949, 97771.11081971062, 30666.85929904457, 85361.16685158633, 90257.60858499688, 75663.22421332111, 6837.0691266825825, 80030.35869668412, 63011.741566499644, 66830.76917524482, 19752.37863290583, 91717.9280984985]} +{"id": 598, "vector": [56386.671199300254, 57357.479534197875, 92119.09747324085, 98988.29805239609, 51636.09901222408, 84178.53635641291, 94812.12610525242, 87132.45090262001, 3936.4130378178097, 48107.67915988313, 652.12723057414, 82569.92215946785, 30028.417985833945, 1814.8311143066476, 243.16024851168683, 27341.913868385414, 42448.7159263089, 4269.18215252734, 33810.23154030705, 21382.06352870834, 94188.56660436865, 67358.77814650214, 61223.520897402675, 41863.32024452245, 61735.08805063492, 79685.99737387178, 91222.34397453559, 464.38843289291486, 26870.032336101678, 93918.71318137569, 79962.96825648856, 45812.336895733264, 59004.09363594591, 93793.04709722898, 71841.12210825349, 23108.974197367304, 39045.392298402796, 1730.7456547359657, 54431.15036496443, 48966.452648056016, 35427.049722644966, 69475.59028536321, 57321.35243207741, 67063.61667819475, 79171.02499316269, 52939.31826682526, 32927.48402243971, 86839.77619469677, 62113.95688762301, 15096.679129192504, 41089.63783462243, 79090.38591613376, 5515.801322508917, 11253.849925065762, 21584.211522160567, 22340.842254820236, 78733.66120710628, 14942.735177866383, 2936.64382597969, 16458.21275318272, 74156.86050553473, 62458.6296649544, 62693.466317051505, 56944.844866356216, 35334.963598601855, 61612.999430876946, 64122.395558338954, 71542.05672930514, 92357.08027440227, 24627.972196147086, 33650.13445375913, 54082.153138103306, 56576.09322399124, 43174.447969620465, 1783.9887756106232, 82849.24784350391, 4050.4483790134827, 36632.4067089918, 1766.8792929436684, 54598.98025046223, 55779.81579853984, 94819.86131678241, 30899.821187185582, 61406.77157655674, 43055.89077258455, 33601.21215632508, 95926.20403833085, 87168.17492884902, 80586.3543740974, 84668.20182239277, 69967.6658387876, 92358.55586923659, 38110.16205533123, 3207.491164530174, 14618.155424637602, 23230.60976499761, 23894.060435437383, 28987.9543111631, 43067.44281262187, 11473.962698427209, 59367.631928074916, 72748.08782170917, 8737.35480693154, 91317.82264020863, 63239.32949062896, 60411.835398914525, 88041.32374514293, 45167.96223017228, 96875.87308411668, 43529.386714481276, 98902.04649162901, 10225.309580413645, 79403.35924210846, 44528.08276139635, 72213.17580857873, 19710.812775455055, 91575.64785145031, 27825.0675371872, 11947.67209111407, 6154.6870227515265, 32352.062172598395, 51547.33429530532, 58713.293154076186, 69958.07010521859, 87101.20280318063, 81425.30301818388, 99586.8083839923, 55344.40401673506]} +{"id": 593, "vector": [12246.510036405733, 39979.763092284185, 49090.00335101324, 70333.10852199503, 90601.6306390362, 85385.21524170526, 15547.987825470776, 25081.690735259876, 31199.144862664398, 22102.77511412838, 30988.539718398923, 20276.68900098264, 38460.38168311035, 17804.733566069208, 97066.00386045742, 22160.820285570713, 59406.67820433647, 87450.27003474406, 29385.555387698125, 40135.26578905584, 77470.05391721304, 92905.88599011922, 90968.74650742783, 16776.031874869957, 29753.242837419548, 20831.883388576578, 52394.981736440925, 53849.497829102635, 90542.8222832303, 90446.37297856457, 40735.459140500585, 90838.3296396896, 21241.539206721573, 21400.66290090321, 35941.971797332626, 76892.20071081385, 51042.05402504972, 58083.15020798707, 78654.1548981125, 27559.002193322867, 14230.029215514372, 52999.33921849092, 120.19193975609798, 5007.479695272754, 50242.73969756762, 77631.90193460888, 85336.52824472728, 29693.352653095397, 36455.95988669179, 4237.894863053093, 40988.31368920977, 14601.674140818233, 928.3176405904592, 93187.74811792687, 92897.59969191313, 37718.55524146227, 54359.08637225623, 8725.916880145978, 97351.36988619372, 38278.747542605626, 63703.22246380328, 24207.619235743718, 20221.640988870993, 61655.55001449572, 40598.83515638356, 84432.5924447554, 75055.98350709322, 8572.364580439451, 88173.70427460589, 70903.23422074073, 74531.31320045734, 52793.293637711446, 83269.92361033616, 65382.485902894, 22896.46236603505, 36549.985149177846, 37740.539240614344, 63007.2710667408, 63578.91780481564, 82981.85832431319, 95263.99169020544, 42978.654209203945, 59527.0783761765, 75290.16726663017, 55695.03003934563, 87766.72630922457, 48175.72135008026, 61257.159149282306, 55817.50089860921, 47571.03699146304, 88167.84572683688, 61152.93171877885, 92717.13433878258, 70099.55870631426, 26280.945294806002, 80219.7128546373, 30251.52110086201, 30172.564327037875, 79795.52840388409, 65685.36579460888, 82431.42880097995, 17817.165806799207, 58695.08251377304, 86018.73590980869, 24738.72888957076, 50014.69403686819, 33169.178799060406, 61249.02128837465, 42316.17676726539, 35079.626009829626, 48608.323777731224, 60539.878557406104, 79559.00821031573, 49757.397448316486, 48783.07497468731, 67378.5680996562, 62.07803778394138, 43040.246363862454, 30294.370500477762, 93105.90890323662, 46962.37005470626, 59751.79232351911, 42098.35510881865, 61174.67863562618, 3640.74091569645, 82901.88497814616, 10990.595573596496, 84790.13426354858]} +{"id": 929, "vector": [46493.01898686915, 79391.11302362592, 51661.51631992715, 91734.09607825604, 58753.602583214495, 58620.044237065806, 70006.14246768855, 68114.1635188529, 96253.83401640404, 97021.45085877257, 61178.76740471885, 58462.443904228334, 21511.11306986102, 42937.829333810594, 7495.390311771577, 36186.01398669311, 84304.9252858319, 78583.22404521305, 49879.38015166194, 33943.59796316253, 37362.90522685103, 90510.56292898873, 50512.39649918912, 20543.334756153352, 22119.435541563136, 98733.2470451284, 45020.54902652678, 25451.718962574156, 98997.10982238784, 47658.27182155824, 8583.651027013306, 50310.66082860595, 86485.44939354248, 11804.49085132116, 53073.99962215528, 97620.29665003307, 85127.25685556431, 65033.576654941106, 86664.91864548493, 27998.55399312312, 25146.628597247378, 88018.07534637554, 26729.66626133809, 75542.74070946669, 4547.159334949546, 34787.20652569485, 38835.7312894293, 75373.42032746776, 17837.855493508046, 3257.580538750371, 9970.18438624061, 79925.99154580839, 74187.43180759512, 14479.623970426037, 8739.793668721684, 336.08961882060083, 43927.29769130612, 11229.12330276764, 90690.92359456066, 28878.347280221205, 47010.04624188828, 64865.46311484812, 65014.21134226966, 96452.56154312784, 8282.54448185186, 32818.69647401504, 11383.474811917027, 91068.99054817743, 95098.5383537159, 56454.64205441126, 8995.461329398735, 20838.40538748971, 26187.00654377256, 20896.763377943116, 78749.58689703976, 44943.676824667236, 19102.951909112242, 56794.350333348695, 6869.168441460727, 10165.155764868172, 25109.234393196533, 23218.75388007062, 84399.33770418125, 86926.53742190248, 44941.36991268716, 40446.01944877806, 78213.80886586213, 47119.98314380243, 40064.489566123186, 52741.90579492117, 61879.918068404186, 36369.57837297672, 78342.69096088725, 29973.170167806817, 60277.28961599075, 19037.81441499973, 94241.93971892908, 85344.17299386747, 10679.35923946609, 14168.514860214254, 79518.07309281024, 66855.99642177147, 90323.72839372943, 66203.9951247108, 42247.802340803384, 29998.506186023144, 31826.889246903113, 65547.4977878964, 51891.39684690354, 49835.40150337495, 9424.747547617595, 68674.56330176607, 40087.39526426708, 15239.301460289445, 8246.327602494996, 19745.055872364836, 62199.46146492544, 89464.94108162937, 3531.432392652778, 56814.303800353686, 4263.197401473917, 77306.65614749334, 53727.49990365345, 82165.29211208543, 23365.444144501023, 93953.54966686867, 14773.403474346103, 67821.77042815076]} +{"id": 37, "vector": [98989.61578497729, 60591.678364911524, 48144.1636567828, 33908.71413369881, 65854.99073974071, 91614.98975834061, 47884.97041920709, 86648.6417012352, 90966.9468044291, 40715.07889166661, 56255.52629502778, 95932.0740362713, 16209.13335534262, 21639.598620768185, 26606.621037030985, 49630.03344181014, 55379.74677710129, 78145.92004166341, 36515.27585715835, 66028.02958718107, 24841.623602118147, 2206.150398649398, 48907.06847154364, 11312.33434388611, 38798.86640651593, 96911.31740169639, 91455.29522544809, 30179.111766005197, 95757.407852919, 9753.375878227866, 98076.6953565427, 19465.073196932648, 41863.86515176026, 78761.20470427917, 43779.90818842951, 46044.831625461324, 78284.2661510005, 5656.106769658609, 45743.72536579715, 51724.80493313099, 93033.20248781855, 88475.81571959406, 31660.584181274553, 61527.58443842391, 33614.421825697704, 54043.3722515426, 80075.06779360834, 81084.36061718327, 89188.15515458786, 87762.28770852448, 90603.76769021069, 73481.42494340077, 54827.764552662185, 28860.106074156345, 71615.07492125026, 36580.079003026796, 99071.28267096718, 50418.280761687194, 81933.00810933292, 23645.959778437296, 49203.07675154267, 4781.553499737168, 58482.2683362756, 48993.09886555517, 99884.16014970912, 13434.453130919133, 12947.376012475386, 43597.85549547921, 75727.47788749172, 66597.25439360474, 36771.424126343634, 46435.52319296902, 79102.33792021767, 69375.9989836466, 51564.30190320312, 81728.99674916134, 63833.30921119721, 31434.94532030805, 65342.46555370861, 60449.73070952363, 89609.98707496129, 47355.938417163954, 7338.100626688182, 8086.009408595673, 64058.921551225736, 97501.96713084322, 32723.27622528044, 90469.47178479176, 79877.79698735519, 74875.96236544922, 59878.60205083591, 46877.378668464495, 80651.56630754801, 29134.52670547574, 2717.108819257119, 15172.629035387008, 62783.98139110788, 90262.34149493875, 35987.36846753565, 52578.38798564687, 26508.422954549027, 18102.961011283303, 98876.88156250879, 92322.96987831156, 40225.8613118564, 10649.011592402736, 30741.14511429784, 14943.035074142175, 1515.92334197862, 82688.85329413333, 50575.101483338854, 95094.5698105198, 78766.33850704494, 16960.545458281882, 68124.11315088774, 59360.51579555, 54229.48065665586, 63483.96545368297, 67432.7921405494, 74706.75354243294, 94417.23508493867, 36648.97436411416, 94276.30680067654, 17707.679966547206, 91646.52926822084, 9436.868211210514, 27208.952134260177, 66077.20813152163]} +{"id": 1364, "vector": [58193.9366922324, 59650.13296842957, 19957.320259663826, 15634.525855113956, 1437.6379857529043, 22767.360451320063, 13002.450212744876, 85667.7085197819, 42384.58503570456, 70408.73492117057, 96890.76609415356, 74889.32727883129, 84505.6571085141, 20064.87867218485, 48057.10907510905, 998.41076740117, 94131.56571555586, 89953.09391672532, 58061.48439950841, 74289.88829635463, 15701.462139564659, 59752.46400215863, 2553.961897515711, 63595.89472879671, 3798.1028676103756, 89349.33180201164, 91245.00114535051, 29110.17676908646, 59200.46540596418, 77984.6966600541, 49843.807962537845, 39472.95130246057, 9575.61143509511, 25739.990590604324, 8623.815107879329, 50904.5748099376, 23834.939526932896, 93756.27074586232, 97202.31306538991, 31375.091337195572, 98280.33667495339, 3978.8426184438677, 63580.223813273995, 87028.13835609255, 43980.387000931856, 20426.2960643132, 6.337541033785143, 17555.705677371814, 3654.390675805708, 80821.73971328547, 57197.71245203504, 80336.64004852007, 39577.33316022671, 1007.1013131840667, 60359.09986106172, 4568.46573551809, 15034.629405227484, 61184.61652223112, 11420.62584981146, 84769.53797823186, 7695.661109125107, 8156.434393435697, 34623.553074846044, 33047.32338977605, 96717.84178573728, 14315.487612746292, 25879.66305895506, 33538.16721781825, 70890.39264970335, 10929.903577879651, 56892.21123845457, 58405.5101850752, 27110.785956112006, 77437.56957329146, 70991.29131850881, 94249.44867915982, 75579.12699128965, 13522.78511333751, 58268.85898339742, 28706.254792621145, 37948.66237954165, 33944.39516610555, 22144.714884701232, 81045.0907644337, 96068.76196478364, 44646.47239804928, 40597.54699575457, 90461.1004968589, 88322.83040403175, 53753.68504431698, 25896.59741107623, 10281.540320642735, 50862.37326594254, 71435.23916542651, 22211.04966652453, 77189.30706109609, 39939.467788111935, 53303.59853935084, 26487.458650232355, 90696.85646814057, 13125.930060997414, 63203.99437449563, 33895.68340419312, 57900.18575504562, 81962.18096910557, 13476.733024948873, 92754.64436103807, 18158.5325264627, 91328.74141105758, 52678.12829888739, 40671.90673822987, 31309.05735151832, 19153.365449691173, 83127.71238336495, 51920.74773971974, 48488.795369377636, 98599.9381825462, 43775.41906869245, 19722.06145711496, 18445.214402242473, 91268.23590115659, 25118.923483496892, 53448.95400438865, 70110.03702626833, 39962.30930537231, 51990.38590596831, 60798.9251040554, 38764.42651053712]} +{"id": 200, "vector": [13978.218387055153, 45379.77543481807, 18856.87437772977, 98133.10223163177, 25657.393263340076, 7957.543907190024, 94292.78492071632, 82497.4248297265, 9149.705217536663, 47504.45480323692, 91092.86767247072, 96141.46988959769, 96501.43761718507, 96113.00097519736, 85204.79710014247, 12704.350836394784, 68468.09938293394, 82888.57056862018, 5669.369842711558, 86356.6640579374, 58076.01643436624, 6327.4931440389755, 6715.524045854427, 46556.899651487925, 53310.531175683085, 63906.43289985551, 61515.06861207575, 60337.33592694609, 95339.2145812181, 44677.66055683331, 90697.4725465767, 45720.66091112866, 75860.86031649083, 82541.38456349661, 33383.861850292975, 42998.29870521712, 50217.592426463394, 62873.939521410284, 32400.98745237128, 17403.053341450115, 79315.95798645445, 84302.92918165604, 46408.756622622896, 40193.156991454845, 96124.22104161218, 67697.20840375658, 16851.470369095023, 72536.63893681581, 52873.51988071416, 71170.52968705256, 5642.383004864715, 29155.16895734175, 45988.770408376, 17466.929160678603, 7648.715017368535, 11689.31462890408, 62308.576153055306, 3659.97887115066, 33293.80011105284, 99021.30020088084, 86351.63355708976, 21877.46862643487, 30128.450711530684, 22494.96758309617, 62264.48225725568, 38367.56070628216, 20932.69405054545, 26137.03091548024, 23990.38340479307, 77360.63122590043, 42093.45333850384, 21130.074511938157, 32976.865761463705, 87595.66316866067, 29903.33173660532, 99595.18651620767, 46924.79572724465, 48851.86941440278, 70248.35376722224, 26541.814938274587, 26182.437147887373, 35144.68562018122, 27565.217940599905, 30178.282389261858, 59090.31056295504, 93118.341955773, 91938.05566700944, 60993.09026932023, 3797.5525537292997, 13575.793300311068, 28142.860814087922, 38099.3237713182, 34623.03726940059, 75557.23075060648, 61414.43190705893, 1536.4777432703702, 38919.318288216156, 97717.97417170044, 4324.7450368977925, 95860.49346385908, 6241.495121103946, 70255.42731911078, 81492.92930662942, 6198.781165130218, 93523.6048844977, 70319.62813439737, 10201.521548747061, 27271.692748841004, 8927.275732701046, 15202.616090543053, 78606.88447312107, 33611.27212465795, 78646.58117624886, 58794.75342167524, 69815.18658040272, 94350.62832427106, 42331.421092657496, 68139.40908119494, 54937.677329114886, 80516.60612992833, 38028.582734044736, 55364.75973030326, 95205.83686298768, 48994.8860525392, 81641.0880604245, 32614.44661577233, 64214.35900173339, 8259.634763331369]} +{"id": 1681, "vector": [73124.49325113781, 24069.057029140207, 40929.734453016456, 28972.353197528853, 20493.017156898175, 22.45370836228, 18387.493468148285, 88554.38352813454, 3071.8021398141727, 51965.57185142734, 92098.81847299472, 79413.27837731996, 60300.45484279157, 65687.86204601647, 42462.60254668155, 31136.059623561763, 69624.27651711754, 54811.41614319435, 73366.51275444182, 98827.58475968921, 7016.734338007646, 31269.537141542824, 86728.87569961704, 78741.06462668958, 2423.0509946576494, 83427.83555796593, 57825.05325372376, 22236.75229005603, 33440.29052226126, 18140.94600006424, 80176.8032406251, 36771.88237441911, 32375.650615452632, 64844.5353748914, 22833.854264138077, 81338.16078937167, 99633.48425826279, 1942.642975025266, 96163.78374189354, 86736.8356823781, 63135.612198376475, 33094.836973831036, 70201.45737075071, 47532.23317347824, 12340.904303371226, 41393.53980130895, 47896.64025256159, 42326.415293655016, 64488.71943537505, 29522.034267009843, 4916.883717103337, 82503.57322664263, 42484.80074567075, 67925.39048452914, 56103.18630032716, 57625.517096412346, 72510.4462824983, 10259.04476446239, 96084.60495857903, 50001.81017694366, 56115.72439595135, 25626.383548721264, 37863.14784151823, 92411.03099810371, 7184.525951211141, 43196.99182309934, 23258.502281810077, 76757.47250873412, 60417.791774644844, 7362.931333696254, 11635.807995465586, 23206.35090633706, 74958.50740515185, 56716.488273594856, 74133.99172242802, 92611.13042330596, 96509.4103362959, 54686.937526588365, 85540.73329450822, 49280.79049078241, 15691.731462091862, 92855.01436614848, 30085.883590524987, 51569.62092417653, 58630.73449739944, 63632.31962652337, 55329.870983879104, 21702.15480899891, 95505.42845067341, 96363.55851146395, 55407.24313507299, 42577.20885090789, 65672.55151263757, 69517.03685435714, 84475.46372802001, 83318.15624927312, 31317.650435378906, 80752.08391963525, 16728.5202975071, 25694.874475250417, 21217.230800325626, 46066.656374252554, 92943.94668320492, 77432.5136688849, 53472.92373896841, 92344.02019124765, 43170.014840172655, 65189.459512940506, 70320.62626292193, 17557.956119550887, 74671.3168981646, 15328.488798419827, 20176.807959092213, 94985.84250344096, 83781.26722523864, 11088.183843304656, 10895.485249652236, 93546.72938010358, 48311.30377231152, 83298.63529192006, 95857.77835267776, 95257.88249267032, 52846.60027817117, 79991.07670485445, 62512.64547505113, 91640.60601750002, 36748.843006057454, 59008.83897935569]} +{"id": 830, "vector": [5633.015566135302, 90045.64409580325, 82636.71864190065, 53731.83796708397, 45239.995021776536, 57015.573446963994, 27913.332003019208, 4104.5462533255695, 65445.96039670085, 32622.483471348696, 39725.62541259037, 95412.77860747432, 58567.61737686109, 13147.5710805815, 5855.994346747207, 5519.8847982324105, 22332.198623999942, 85895.30293107363, 62028.15032448096, 62408.55189132457, 33318.536929637274, 18770.0813725842, 6437.189908265273, 21678.475758641947, 22039.450886429146, 71266.40108998437, 47141.785269492306, 7467.483298473643, 95091.54184534142, 85350.98485430056, 11998.372179286454, 6468.729820667696, 54444.67535596646, 82027.12148007222, 7330.395202369, 11858.551290665653, 37611.04948536061, 15773.518675334908, 76330.30659086375, 65701.62706636303, 76373.76745828018, 95368.92628783469, 64290.253078405156, 77772.11202404276, 85003.25188225927, 5006.064818831868, 79076.59853706651, 55529.82331458255, 12366.290007270187, 87536.11135276052, 11557.52414508504, 50865.05469881741, 23804.086143239, 65371.823928385675, 43386.61209400765, 42853.88750529991, 77347.8886273459, 93860.71667384902, 68245.6618789413, 13231.428924133827, 92358.50281820928, 35743.87101422826, 14411.496308978167, 17159.64886052267, 11200.619730389095, 68945.66583817669, 79435.7862216832, 55724.554613131746, 23027.293398265858, 97203.74340512026, 87676.36895232444, 68575.27702841138, 45358.36488451982, 6393.6560716150725, 72067.26598109353, 926.4594206612409, 11499.127251004214, 78454.98652946118, 48182.18165855064, 50642.690138613725, 4918.622291911279, 21299.088500414608, 26024.44541052418, 86659.99863816075, 55434.440767795066, 93211.940822103, 75715.22535502937, 65259.726967809984, 48355.14726796578, 79118.94408881164, 92341.02933440017, 63603.50912682539, 68314.04695692597, 87687.63689115716, 87554.23517020885, 60420.907108499465, 6849.525921851796, 59339.933741944464, 44716.04275308843, 98826.6998506507, 63117.03575400257, 80971.19737816454, 55817.90388072437, 79309.90976773766, 50342.871288030874, 49783.05761800221, 67785.90658593652, 51725.00502013426, 21074.792567525157, 50211.33690193508, 85785.61553718182, 19170.809481893524, 34322.77189336152, 44854.255794979246, 38039.12325244987, 64426.961175979755, 66329.66312646863, 48783.84883453536, 992.2150643561167, 89858.36175048913, 75077.32659692453, 86730.64809486805, 39891.05728495808, 3046.8811400986783, 10139.465956110516, 27543.89361145225, 83753.33592483569, 21725.20972117249]} +{"id": 175, "vector": [3982.3839904068127, 98502.79891211903, 14067.544458509341, 37934.084878034955, 73122.31252226653, 5767.215278164505, 27145.493185439485, 52653.57863536186, 50844.4384816389, 74970.47289510927, 26500.31385726509, 7547.1780000579965, 43179.254319264146, 25842.14860768914, 21017.531437644044, 28579.802661860453, 91892.97484284647, 69965.06885447976, 74176.12301136921, 2031.9158619054422, 25203.576031507426, 31213.740776860723, 45534.382768720236, 74843.0405622935, 93538.9670541524, 67868.92709748299, 65600.26540418962, 25040.363733076378, 36973.43561087824, 66376.35014406999, 23524.212565480662, 48466.851518929696, 93950.80075387943, 74190.99128162292, 79259.12032724539, 361.3673133072259, 95871.90452616017, 4874.6008917850195, 89823.5344774021, 44946.07919394876, 34117.21565869109, 55146.02443655272, 2647.764857368573, 52156.82406965779, 67455.33983503749, 6491.051405365189, 6012.88988417813, 65768.01427881073, 41311.786906824746, 64969.15831221982, 18254.725736419354, 12271.98259792337, 81115.21236836942, 84424.73979964509, 16568.72257213311, 65832.94009674115, 28820.668570725506, 84781.97984074295, 58273.932322329994, 17671.809524940818, 12258.806949651358, 66215.30519481465, 14023.080385185893, 5224.826367859104, 58705.105867426784, 10307.574325347558, 48972.80903410505, 23737.67740150051, 79212.18955005864, 54925.41539425812, 1700.4690266526402, 22473.92334482402, 50386.67387936111, 77976.45874312009, 2408.862790113475, 80091.74786174533, 68377.20729337158, 80429.67883678425, 94329.81143117997, 54314.102408532126, 15639.282593310034, 50187.87550747591, 630.9914212023871, 8069.9904504699, 66037.06006692682, 53257.50565492686, 92322.55481626285, 98481.09672429688, 14808.809500993946, 42533.10311948784, 72491.12608178363, 51271.12908650597, 3570.1232643977733, 13820.799915373816, 2938.1159417858794, 19002.389123895235, 33294.96436450795, 94808.16846604018, 25974.282348460485, 37411.19325711426, 74024.51593590394, 59788.501740398715, 72196.35078908068, 92706.07336707374, 41606.56820915663, 89995.34972215164, 94893.89810020085, 96889.46629579249, 73777.74663576722, 70197.68686105407, 14922.518778598604, 33506.85971954083, 21248.51153360412, 10466.892246538684, 64615.1018965584, 11236.532880831219, 18403.66271882512, 93307.92855801847, 14448.153401737207, 89347.54503467839, 94858.62410136305, 1829.5169516089893, 29766.769584577945, 2662.9985309702843, 67268.49929222318, 55458.20928809495, 16260.079568735919, 80527.08243367224]} +{"id": 2015, "vector": [69219.5234616024, 71781.40273143952, 85121.59587082248, 24065.7358840106, 32856.31777607814, 44532.624412477526, 94481.5496698559, 78745.94210164827, 78079.644771808, 61246.5795728235, 60084.53379437737, 76272.79685977542, 25880.522966713437, 16636.808315437367, 12951.548273540991, 74833.13427472267, 9350.147209046678, 23105.85355876211, 47435.78446879215, 56229.70622294683, 78082.11450806119, 65986.3860991575, 75182.1446503879, 98934.33384517583, 4982.141710022281, 9309.67122459233, 86220.15249253671, 19382.72401182415, 70309.76321733666, 49450.72983191383, 28208.657730905506, 14320.799837548348, 79065.19262221767, 88451.18776643238, 25517.89929038698, 3696.822287745627, 83665.8238891648, 57583.23511535822, 42543.31008521145, 23151.989478767144, 61805.00697643968, 19019.346190895216, 42791.2193429007, 71516.02885131715, 68166.14106914707, 91822.6875850344, 15794.93796473781, 27087.442898323377, 94266.351932344, 14905.516424588815, 52496.393343624724, 8295.066633280147, 77312.0486267619, 96463.23287300002, 62206.273353234355, 45491.730207837725, 38009.35868035774, 60342.84292789896, 31748.10290465382, 62425.259414262146, 58455.469758158484, 818.0960657486191, 93524.96091774463, 14905.48579592108, 727.4036857530585, 25758.425302828815, 3292.8026836759104, 80385.93818992992, 56428.07886103206, 82916.99160839179, 99719.06030667004, 79872.81357620511, 48990.070513407925, 16478.351530987802, 95976.34358978561, 67996.9631096632, 73201.49480383348, 99547.02913585678, 16696.411973923696, 91843.11659352727, 6564.60658564828, 58370.21627671831, 57376.77439102791, 41674.865525209716, 18758.339517720335, 73134.35503316122, 2803.795133294229, 28532.555363450694, 89803.3269337464, 38560.087718490635, 37985.59406329692, 49056.7160005251, 84027.64501274454, 33941.48443955055, 40047.452497170285, 1642.2802683925709, 46737.76290231627, 82578.98822562906, 6844.565728161156, 24277.43311078182, 12564.117417103304, 27957.614507972605, 25153.312008944096, 32859.25566533141, 15470.049245368678, 73935.2648304479, 40837.769211779254, 58830.43647112937, 2816.8558742242444, 8149.207213284227, 59235.41099625327, 1760.1596538361796, 65622.0096715267, 9884.68660016546, 64572.54792733909, 48174.57152810737, 92504.65931660392, 88389.88523023366, 14772.096305185767, 96787.29770265454, 74835.8136191805, 65911.814973987, 26410.79558733269, 36429.717072879386, 64301.9137674542, 73959.93074229953, 83602.87668958968, 69944.41162539362]} +{"id": 573, "vector": [3829.930915957802, 5077.446766488203, 16239.268858522182, 78491.8355946346, 2179.7652880151563, 7950.847716708143, 33735.215797308214, 98387.38012853405, 68232.45876984499, 45548.754153392656, 46300.72523550018, 81320.95378925694, 115.07064077014294, 16426.727231275283, 40744.195536849125, 71366.17094396811, 2227.0180251573456, 28576.327851798, 60131.670134865075, 20394.82832556775, 28041.942404977483, 6233.806004680198, 4880.952197126553, 78005.26708592725, 69560.4607750407, 52740.16280374482, 44924.3386723581, 51828.99984442859, 77465.25739760751, 35321.645728227006, 42629.39226654925, 64478.48487698876, 29433.401570590133, 79390.90744102182, 12114.985265798729, 48588.66070849268, 32579.311041172397, 59616.218413008806, 37228.90563153728, 41808.0743247789, 75897.66674657304, 81279.8880893508, 61422.878253191615, 27531.125110914712, 23519.891514178016, 2213.0477608189426, 1905.1908603950674, 76747.9755748405, 88874.56387426432, 55935.14338252621, 81916.61798096371, 60784.5208602096, 96515.61563911843, 4972.618318614696, 7319.3425052346165, 17350.040422916092, 10906.334864772682, 92669.38107233465, 32654.32800104937, 93564.78179918605, 44212.063553892855, 53718.41815591447, 9701.06477829069, 36983.941694801426, 41384.13566153898, 850.09591683336, 42479.43990671581, 8185.074781149427, 44839.62684282216, 33398.65209158055, 30133.20821700142, 86964.1535946142, 14162.17329996089, 3742.336485059838, 30085.4048478306, 99811.66203854921, 88627.0390274559, 56417.17673469245, 84783.68007627036, 2663.5497200208856, 51532.85524681253, 96127.6675818168, 4055.896713740048, 65189.795241383785, 26078.240456061376, 5501.688861779897, 31571.11526265597, 49875.46675887714, 61868.95225667799, 15683.11273775187, 2575.827703866318, 12835.501110117231, 20886.968502724856, 35284.60124275868, 19769.223785735292, 66344.35183954945, 88530.20218588586, 46098.991721207385, 72734.83667744942, 17963.041830635495, 85157.40083347548, 22523.929584899826, 39913.135120858715, 66050.12305182661, 28222.424554553018, 64701.68223772149, 3235.3968107182272, 43905.995110585296, 7906.206811570571, 64837.87343185079, 51715.450083251504, 54703.65232450178, 54259.206222197274, 83389.22516064995, 12186.10338985423, 88763.11926754785, 37405.89047556841, 27549.933720225774, 21581.310536419074, 57397.00916287508, 26156.055366362907, 72209.75385808501, 11731.9477591597, 97957.612646846, 93443.69744765856, 46532.529606216674, 18026.27487433891, 41587.8044081646]} +{"id": 899, "vector": [14610.301549264626, 17092.99778367105, 30528.14577206019, 13667.312783375684, 64706.75346181808, 19673.587964218965, 52647.31383749799, 15043.200018129033, 27155.43069080062, 807.7862653310897, 68097.01225360278, 16691.04014151005, 1915.3503229869195, 47289.39068229056, 57693.579994852975, 33788.41542993305, 59164.463228831155, 94344.99993670301, 43663.646048591996, 81246.2679641503, 26337.04845150846, 95501.59839093852, 66901.8692471896, 30047.47300178826, 14575.710557010923, 55292.27403291669, 89309.19995708701, 34852.145960406226, 53818.45706759327, 30040.492908446835, 49456.743465636966, 70099.5307741254, 95644.68230423998, 81925.2584614586, 50257.74426075643, 79389.07828825343, 24204.13771014578, 64010.21961293764, 64143.31355733823, 6762.060972014061, 3801.429254563993, 81139.31628429037, 70919.73546971777, 19659.246123554043, 11787.781898397432, 23025.6217648927, 30443.33743457357, 68368.22656596507, 28175.712106370676, 11336.617852369534, 47454.146630211515, 73087.40189532797, 68197.64927771989, 87382.16997413855, 42938.91482755727, 11593.685680916122, 6904.004667578567, 26448.383336901537, 73324.71106628794, 25808.326061466556, 89692.56440877926, 32789.70614013225, 1247.7831491513602, 43804.96639334341, 95933.98435817564, 61206.83282459751, 30037.897341122978, 11755.032675658606, 84364.27596551366, 27072.203602924794, 74390.23702881501, 24096.47532900808, 395.4234075297047, 79581.76117963731, 85641.936081137, 79809.75435687277, 33266.6425717138, 70057.66218202608, 20598.54272210595, 29076.511718811314, 74582.53599176185, 54984.444591908396, 37556.58325727986, 1924.5276599267668, 7243.111260043367, 44167.85368853289, 54569.64359719946, 81779.54441267227, 82705.85005197974, 20706.929487655467, 93413.48593316354, 95015.58269337239, 23793.223144327756, 88235.20240868989, 48775.81969778062, 36709.24772546127, 14771.572710809156, 49405.96821929225, 5560.397458110078, 19432.671153886004, 49095.096092135434, 71864.14574932793, 38302.89972865231, 57435.11287671739, 20133.94089709738, 15929.917607517818, 32180.263720814695, 88048.69563530126, 87569.8268551001, 49061.73259380289, 68140.87711126365, 57641.58764492696, 97690.24205725716, 1489.9665888330803, 9781.44139579733, 53741.895632083426, 87415.38428642512, 14238.794423613199, 96137.43852549352, 7116.682877866842, 91492.10076097843, 10510.609579461838, 56092.42164669203, 55146.80024336541, 64696.621901807375, 31505.508169716413, 2981.4169188015603, 71671.44838864534]} +{"id": 572, "vector": [41615.1474467926, 32614.78749962845, 49613.185276371194, 39573.72589399062, 82511.80245843391, 45647.09277279593, 43451.089051153016, 99372.45641234613, 23843.254441986537, 68294.98210136191, 21855.10122870472, 82009.90725791047, 10257.88771478423, 63740.20667012163, 52284.843416382944, 99161.1448823989, 87649.53539186595, 45626.133017653956, 9366.293849717787, 15790.604820122357, 11393.097524013485, 44369.60094804814, 58183.17312128779, 38454.205947481765, 84527.61621511573, 4630.691231364825, 97475.2402553174, 52176.3084354961, 82882.60115281302, 54901.54988591049, 93987.72581794333, 31651.0680688981, 89908.76370855595, 58260.53265851962, 60210.21379331109, 1940.2915559989299, 1213.4803076310652, 74808.13547277886, 46159.10688276201, 21010.813355485425, 74527.34516874014, 78248.10436062972, 51325.02542592845, 73448.1631272319, 97198.8426207936, 52160.27601795501, 2305.2263712585286, 12518.945308896356, 36151.70513434759, 84380.1060417132, 5567.127828265217, 53877.714790377686, 56956.85613004677, 68686.85615844218, 50769.11786323756, 76734.18970891806, 4284.347552966184, 15399.043801746737, 47278.08183596751, 41162.132615122246, 65742.35882499984, 2654.204271963767, 28227.383399249018, 42877.44840774446, 31207.886412921758, 58618.62668557217, 25413.537148971587, 26924.732787243065, 56010.0959082631, 58790.56659210307, 47776.62401590715, 40910.63865652975, 54999.53082589593, 65772.8244824983, 3528.0232302718373, 4850.56956130937, 77728.73649568985, 18501.155336168627, 93771.37538359169, 71808.95718281412, 53432.68021007359, 31912.41075505883, 81512.12990431082, 39330.255055121444, 85417.05209440495, 57432.32387127148, 74860.96271451113, 58681.53179350164, 6480.741263925238, 10198.355154975536, 3900.6304391911217, 89477.10587305305, 88450.29839439149, 55288.4897661301, 99843.21344620998, 57979.065352725935, 848.6384326501728, 2460.7811479558927, 49773.80306867667, 69232.99834204846, 83114.70761312544, 69622.20506941264, 84516.52989250152, 28414.75267775373, 67197.90777878238, 9321.635654245309, 16885.795063594865, 88776.25535949974, 32079.149150311147, 52296.02208469789, 28423.450168613683, 8853.28328804934, 30454.526067832954, 4184.57450857983, 74059.97672711655, 67770.86539942346, 73856.03496436683, 88212.30237458802, 76144.20076697133, 39508.731105165454, 43691.15837561994, 51979.91137562731, 6115.699878204384, 28938.189265711746, 68457.76243928431, 71657.06603413537, 64552.49742006608, 74557.66423058276]} +{"id": 1219, "vector": [38598.047152103754, 1297.7436763284445, 59919.44001251858, 29398.089330689658, 19125.64817356428, 84875.37931977202, 9740.58639373151, 59265.0645247056, 27070.223277749115, 88340.48337307374, 411.6345052786907, 57475.37147567724, 60518.873438796225, 49669.63662389029, 51955.52970576595, 95805.94286082948, 27232.624076365253, 82208.97124795246, 84177.00656805317, 16637.20321214002, 17104.65395623303, 60415.33370837642, 46521.118827915634, 25247.03390170945, 54439.23421039606, 13902.475807548031, 66073.51395930427, 55476.12887513016, 97463.08050899424, 35577.80141963779, 56800.65894787014, 37293.806062606294, 44642.005618069634, 56609.811757858595, 97507.18825169769, 14140.782437223643, 3512.337707376967, 1195.4720855936962, 96370.24065981348, 19089.49513064524, 88088.70281477674, 36947.7318182539, 23680.15261356723, 16600.115825033656, 70251.87125325683, 11150.190743650224, 28792.78159562704, 4224.896894555941, 57015.60655864634, 72452.40211173423, 44832.05037679123, 28726.16838817407, 74386.85345022696, 33271.54237677972, 77615.40582317578, 99061.4542136707, 67726.54342923095, 2601.3494394800273, 85369.10128987402, 82575.54550483549, 13926.630127905604, 7278.773927804849, 65315.49820211942, 57620.025549596685, 80493.32293354784, 53415.16559333067, 81243.89920658404, 93649.00820821479, 32836.19976528186, 12918.08007139229, 95216.99321608417, 12158.03777813631, 65952.90602230738, 98507.90326994624, 4816.142751352959, 20422.109500384166, 85474.54488026052, 32964.745179946985, 87645.81370275939, 52657.8008718464, 80588.90208463212, 99116.73907721577, 65582.37751553001, 28336.809136507734, 94372.91789157018, 62247.119524439084, 15029.662483138873, 25247.910403251582, 32284.32151658911, 41440.785261349025, 23664.852694906393, 40134.141111094046, 23136.704699068912, 27925.08191608316, 67093.81294528813, 94014.75121333869, 9653.528455394855, 11088.8896097746, 28010.257174328868, 98384.86369297084, 36657.700322060904, 93741.42362938462, 86142.06056171116, 7385.233734048524, 74913.96767632003, 66783.96899652235, 92203.40185806096, 55796.084005766024, 52315.94299793232, 24354.070638625646, 55241.1646613249, 66382.32571873488, 15885.35269451422, 66573.59111463868, 64590.253182978144, 36553.88653671118, 28111.9707453787, 69787.56186644334, 68668.3761870733, 89205.83655229646, 73441.33696145655, 54696.24922968731, 91848.97715816624, 27807.924855735022, 93069.28799209405, 4476.683888230604, 27646.664090951977, 89753.06516084287]} +{"id": 1415, "vector": [65698.86874697653, 34286.303818689135, 81559.40497665684, 16333.104237224361, 27234.034256143368, 69916.15719335529, 37288.14395291927, 49628.93986387227, 85304.0457904325, 29867.264160811756, 70341.84859855854, 92230.55099842504, 74486.73664543846, 16001.190346072957, 8653.70185303207, 45775.32499621767, 80530.35252059306, 16483.822173877496, 35279.21536697255, 17597.053621029514, 56774.039281215126, 16159.426901887042, 65140.90950948472, 89573.72902247928, 90695.85157456594, 1563.3485587153496, 70909.35796953557, 73246.66361353415, 56443.62190063156, 71504.72709659339, 79306.67710883163, 33115.84291007036, 73346.51223328784, 41271.22610452052, 56818.07128280791, 61429.016282506855, 67.83363466037473, 97995.89750725655, 23353.775310608326, 68197.24818543342, 23518.523549057045, 85765.92824326926, 40293.69944552991, 41114.1334757287, 86963.54600582109, 49642.61363880274, 14123.586923865838, 67387.7306567843, 46845.5729271177, 72097.21850721547, 98741.71358615902, 13160.54966886594, 42141.80036858752, 31217.739423675095, 53365.00077640118, 10636.531995278441, 75915.249652673, 74536.85802732853, 92898.18788478516, 69646.21561289423, 48791.22470662731, 94656.23232939214, 44474.50304776825, 22945.296750240308, 18985.00283949134, 74654.35738923788, 70493.30336405318, 64248.72199105353, 58489.02787635817, 6637.299820368236, 47071.732167834954, 54054.45204319625, 95809.1371456003, 65829.61768768997, 56968.688659736865, 22922.086302537726, 19961.02569564866, 38354.94853290266, 13336.744168568115, 78400.13673573543, 55608.53950965688, 93072.28496261867, 4449.629346594675, 31326.98143842938, 13236.058849538635, 44256.99710197379, 14356.679385238845, 2775.3859651885373, 9986.107743370409, 99520.90515394465, 72895.915075982, 66053.77558367058, 87267.16359814267, 12786.548976742595, 88483.65313896611, 52257.88930862395, 16022.177567211482, 58966.475770788886, 95828.56064070624, 33914.41911676674, 11606.152220139254, 9340.598838920656, 63250.86396594982, 81701.69158982422, 96746.46949487567, 97222.53464740774, 45208.20148180728, 83738.58323664806, 2959.272497547716, 8994.271744812333, 15791.064728875004, 34111.00203203146, 85992.38515772775, 27390.402798016068, 55496.82510703323, 68923.92817724074, 58200.216579832406, 50570.83113374472, 19619.154895943204, 93204.77759331778, 88738.16939744883, 88247.73483919857, 85822.25436891524, 603.3318846458191, 86266.35648422242, 51370.20595016463, 20523.929714440703, 73885.19946126566]} +{"id": 743, "vector": [97067.40936104725, 98089.33041475361, 43836.09174714274, 15440.183526764495, 11804.793084769339, 46634.40946402196, 31919.02088176868, 11087.530502455267, 49260.78209624824, 35151.193404754886, 69265.96720261668, 82053.24825498513, 60244.66058893339, 55191.83963530713, 59017.345447065185, 54886.66571510231, 93030.45240036532, 82772.38471480178, 98011.99593596799, 32897.20021468463, 69055.48292600182, 55404.37471914669, 30123.35773206364, 39589.81249758674, 5398.162760062397, 87544.40138746287, 31795.692489905035, 58565.405424559256, 14807.526882202626, 577.4977500657918, 29194.16630158913, 69421.37735976516, 51291.578387407, 9112.461009711304, 23135.816914403273, 52670.00114027838, 8568.910467933943, 71906.59133511569, 94131.78058552612, 77995.75111252876, 86178.95260282131, 70166.99786105975, 89387.00537395307, 31867.906328769324, 31540.182538389505, 21796.767113854177, 63979.38529422372, 55293.49803922483, 26436.682319067993, 55408.520494695076, 68742.78448664394, 5223.231166985298, 94547.11845572555, 56402.01230672967, 12841.237585693332, 85185.17706068406, 2603.3242887857577, 11968.195999012354, 65568.8409079415, 91179.72626173164, 58461.34261907178, 43188.97272779358, 48160.3644645873, 14182.107479517892, 26959.568841417757, 97908.82526581791, 74777.21064260488, 87250.53221573876, 93941.37669764161, 67732.94520470139, 24620.39616175313, 57101.47406354221, 38552.91693324435, 7118.806543528511, 9215.727163586851, 91885.90143335299, 84747.84153673169, 62441.74636578038, 4222.37016123922, 58187.67345754681, 32888.78423401114, 18962.665184387217, 90985.96665719441, 3718.931695686445, 78935.52306704958, 91238.9233684516, 15210.43966076261, 57350.79334456219, 27668.644953974275, 47022.83074546387, 41249.654576714966, 87933.74449362193, 6907.008741877318, 85365.5782512309, 46933.08956934644, 13363.267623770136, 57882.32001897914, 17275.020731049593, 22031.483972619735, 55786.440323751616, 34136.6236731149, 84725.48989812571, 29930.009444986383, 92001.90414016976, 15203.03898392793, 83506.96874338776, 74931.10617946611, 33660.71247863015, 67992.49873521748, 59794.343109261914, 52426.91878581741, 21198.293777450104, 59441.30971515168, 85281.86062821349, 72130.90998222929, 25591.17173894685, 76685.32058483451, 66056.16997485909, 11697.194732363014, 60763.180551020356, 49103.38861597996, 70104.52203641956, 2471.2324146227706, 23105.766825532402, 85211.20180651838, 15597.920051612602, 11585.461222055272, 22663.66608347111]} +{"id": 380, "vector": [16963.740535522244, 6950.933031326034, 94537.520604267, 4888.476983650958, 93919.16026292047, 60294.6414018958, 9524.231728886423, 17844.2279195413, 34504.90321651915, 61404.630070462954, 42458.64271680344, 56641.30034471827, 37972.568611445044, 80871.10604146803, 87662.76591242464, 19297.67753284307, 33513.905364585975, 26668.75157374127, 39737.88940164934, 92645.83517094512, 14285.215377439154, 4957.599742871044, 50630.85801915398, 9521.56378801955, 82567.11946052489, 83.20585636442112, 2075.939948657257, 26842.096003151415, 49140.17262876613, 33861.90970929642, 61918.48054166373, 93361.26696433757, 46991.239656256745, 34569.37484978081, 63067.13143276377, 90190.43209928862, 48474.85189133477, 65148.88972059085, 76791.03666822928, 53859.31969945302, 49913.56538440021, 59660.72121991969, 21676.124873144476, 58144.14951829345, 3339.358565460382, 73395.17034978683, 37037.5190154701, 68642.59392059414, 76952.14284637329, 51976.17897478476, 45369.08054571692, 74789.32900291817, 77425.88828852538, 39265.48425640357, 42757.96905796143, 6556.683510331763, 95624.92823274876, 19099.857993775317, 78007.26916352041, 53319.44899869479, 17179.985346111214, 68270.90538610864, 263.6225961501815, 99696.72950297002, 68620.10323805914, 70360.02899640637, 9332.804565044462, 60260.571300055286, 74019.83868342599, 70587.72414208196, 97753.99841611447, 36360.76129057075, 95877.25703315878, 2880.4244005369405, 11974.53694128302, 41516.10027088983, 8743.040350987707, 11800.860592771245, 22189.25179142254, 77501.974418735, 94184.30419100364, 84160.21306380728, 9840.330589215951, 17466.218378794805, 95770.76908565513, 72661.01905669246, 99921.42961488775, 72243.82376796425, 70761.8191941448, 92389.05349599863, 91179.33753159779, 51950.46721631847, 66427.3230098066, 5037.264798789476, 57930.27054386314, 99579.42286755468, 97648.19325966065, 91495.42064804409, 31328.98924861608, 61614.43612288157, 91211.96622725301, 99457.10051738589, 29830.970626991893, 87179.70408075405, 99021.144581478, 272.3708697385252, 80102.12439157187, 10752.781011902745, 7444.677254332044, 50711.88663904427, 26659.732014550762, 47271.237676043565, 17160.330992564588, 2252.037103832283, 98852.58028302614, 16858.141825730898, 31022.75874628343, 91676.54109742484, 15750.044054204782, 14250.382501175618, 26067.834885172582, 16967.39353300436, 86924.29257553957, 882.7332233696583, 31458.74375937183, 31855.008746857893, 79207.12974388264, 59638.39575093949]} +{"id": 208, "vector": [57239.41496654201, 48808.22046102673, 60593.9299014856, 32038.78357369686, 86746.32034796293, 67610.58515245581, 46490.03500519513, 13541.006514699804, 21105.4987275733, 23870.136143242158, 76328.82904200755, 75843.22070744647, 32422.367811303044, 90334.39787891421, 8284.054155915976, 20657.14662713707, 72009.25022707722, 70921.5031229882, 87969.2797695276, 41201.45560184116, 16561.00992381775, 91807.16120878841, 99323.04825394467, 74708.92192804681, 21291.75822840339, 74681.60265434509, 1446.740863075502, 35863.5945381186, 57890.98077743054, 7411.140059233545, 27073.108046884943, 47731.06015133647, 18814.26651488378, 36687.35466305876, 9341.331768042672, 31748.1667777049, 51488.762453157186, 53523.21096066685, 29973.285316941136, 91914.09327262796, 67559.51691027993, 26475.378434309827, 76219.18871224712, 65753.99279543826, 5733.451378569676, 90377.74659754573, 86691.82625563882, 65014.032674168746, 38443.72277047246, 51947.20576760003, 99382.15545704737, 25.2096778212918, 92765.47142233879, 34236.684241528426, 87750.76000094914, 39620.27113672969, 9012.00229839727, 89298.16229971961, 29108.265916768938, 4500.899323177987, 34509.38825604799, 4258.771642794523, 49007.91994261574, 35683.01205342711, 50955.09715635369, 71142.29495444606, 88531.37144247872, 50014.10938366597, 74784.17558647145, 73228.87199067662, 79487.40842158294, 22013.199458245304, 98048.08728137868, 74895.91817355953, 9665.042856017104, 94614.63769954664, 21550.734459884403, 7453.445758490707, 87562.58149701731, 89142.40472013009, 41430.751858256095, 34802.39641345802, 81283.25109644841, 75137.28120157546, 20679.273161130663, 9968.141544446375, 16891.13095339867, 70417.11885600623, 52854.561468971726, 61399.34441438506, 73478.37424723025, 61658.22028773494, 61645.22412137115, 63171.81324981126, 96863.44605379368, 4253.017964736683, 80850.4146637684, 86912.81597881751, 68110.25266346236, 2264.5881652377684, 93671.42489347851, 71206.56103631537, 93962.58289951614, 96176.5863128069, 896.7015446426085, 88503.35865640269, 42633.50779995275, 5872.317085269163, 93711.07610818458, 64834.332926752395, 88464.62918187778, 38089.86308980734, 47220.99620593533, 97302.25361213811, 16873.14038458029, 53224.590454877696, 65813.6504698759, 12693.620303740161, 19038.446774400665, 11777.676960098293, 31389.89459959447, 88946.61446966176, 89674.58661323437, 94182.62407619772, 97491.33649569999, 69223.78212966499, 86570.4370490243, 6435.404694061875]} +{"id": 1335, "vector": [78788.05624169607, 84787.6535939966, 3077.429710923585, 35918.53010033866, 71968.72999403808, 91639.30613113786, 90293.62427433555, 85478.0026805648, 8388.167510768373, 39415.05540165813, 87119.75522446183, 28793.34283365884, 71250.60211547886, 36742.4214127049, 49662.050209516165, 59693.753210678144, 15923.42558093136, 57996.81501193824, 17385.37323883941, 80176.09759182534, 74844.39277669367, 19872.459200644298, 19383.28935667846, 27748.17383163185, 56748.93688668035, 36779.21263545057, 7561.372160672586, 20443.197859080632, 58064.926686600695, 97170.14889982359, 84349.84062525009, 29570.96129904976, 79976.81954293179, 18479.29044140407, 59464.692298096576, 37190.9477565573, 80052.6379772934, 4957.303038547456, 21490.86082399594, 93940.31102712962, 17790.72719589424, 38751.921337326276, 27152.805912490297, 34661.967279890574, 39062.47145050728, 91138.86956168903, 64041.38970823471, 94982.13047269169, 69834.12983369916, 8630.965378317756, 40553.875070436094, 10217.571798758972, 21299.636542669276, 97039.19845246561, 37345.13325945366, 71274.38492076252, 44945.53349472477, 56385.84000520208, 33637.50417868604, 70758.23256653464, 88505.11442942916, 51956.850938658026, 28071.37956158604, 13748.71271862873, 76566.06783981796, 5044.202464291802, 41421.890554982754, 46132.807155295006, 40482.39933563905, 28345.455868335666, 65534.80617826053, 93026.8369425719, 25621.999301323238, 74614.40048523727, 30755.109797057557, 60661.877699736375, 48635.35845226985, 12047.770096436383, 49639.70041880236, 99169.90143932788, 19914.171849754137, 64546.17495660059, 43245.80310632282, 38014.36073616168, 19718.802618196794, 78256.27354498263, 63898.29623042982, 56839.05475849818, 80684.43909770511, 61975.202197645085, 20104.22685823553, 44769.52011855306, 65805.23228408872, 67728.89423043613, 32645.909141190754, 28122.27145132079, 61741.91270273665, 57087.00656485424, 96031.17679657372, 50657.56131799115, 72904.56794525543, 61100.75009245789, 61395.1301606369, 82203.99755837803, 19843.46435445027, 5520.220916370277, 29278.172314068994, 97483.34399472123, 12520.414572064032, 68304.41507012924, 62840.99226695397, 45846.97739751613, 17457.585955036415, 20581.80984938163, 85358.37681888852, 90042.00524857687, 86449.1548686904, 482.89766916465027, 24919.401949033516, 82109.4375548335, 72373.30478026149, 63939.08188424393, 37191.41672261173, 30283.937795656002, 39369.322882172666, 74827.6048637143, 38984.81341629905, 31964.16599289592]} +{"id": 2019, "vector": [46965.88004055488, 22683.00941564577, 15330.691279749542, 43454.89223074111, 85582.86249819909, 10800.665635589945, 30276.747780223035, 94028.46673589986, 21609.468891122364, 39726.01962972019, 64807.81045630611, 30263.239320285607, 64732.829310069064, 54103.19861993591, 12249.795913035821, 38641.275394064076, 41734.16994071478, 50036.00441572026, 56550.742688623046, 42036.43681591207, 5415.416715223487, 68686.09815108615, 65902.05175383209, 72719.73109326289, 23864.261287309662, 60954.22370673611, 41902.47663586035, 93033.83020177955, 86216.45272038945, 25445.09972228448, 20671.69592901088, 86637.26590242285, 88986.24128215606, 88714.84674444719, 42833.45979212669, 10239.038349761264, 71232.87306534819, 26334.018332326035, 39512.56718472807, 90855.74742902053, 58535.47028955757, 3781.1716639311044, 31216.83900124124, 44897.2107699438, 23927.858455181173, 75136.72887777674, 74496.38881577119, 62927.791276843745, 83265.08394196192, 56911.6500505473, 24495.2757241379, 37611.44301368693, 69984.91176984896, 43114.09488561321, 22090.941653684804, 94847.34955080965, 94550.40463299333, 77542.9169470135, 86781.85985459892, 41859.845570417296, 90389.09806298354, 47531.072254040184, 34843.423349399025, 31344.730055581404, 75601.0616066957, 84228.07784364122, 91078.14953044611, 76937.03909513405, 34360.89555013904, 79060.12248298925, 21016.144788988, 42636.743513460264, 49095.26681124567, 94378.6861389442, 57670.216082539984, 28608.25262373704, 23024.77794939798, 1714.644112412045, 85263.73507694014, 17018.941902112172, 28278.319905499717, 70182.74882974372, 22255.880833660536, 57406.193630324364, 26484.65637674483, 75726.86274673071, 20142.988614815327, 9358.205110662188, 60729.93188936224, 24892.796464480994, 97962.23022710496, 3071.802954145708, 85696.44779858887, 26134.86709433883, 34865.33496838949, 30901.630321081884, 26836.72166602612, 97794.86719995829, 15147.359041709973, 73527.60490282299, 93829.71278055164, 57592.81941796857, 71430.34781319057, 16357.805509572798, 52044.13740698921, 77745.52587066425, 81129.22985677821, 6475.689282027397, 79760.84053150294, 77904.07679412821, 94182.45574864178, 66605.16935291722, 4766.425960306619, 85143.14743044531, 49920.196014782035, 32254.81116285125, 12481.592952037501, 2274.1810461972545, 74962.61489485523, 65872.10227013218, 7502.564812554912, 12251.668545198669, 46831.56797445934, 1730.2062613602786, 53972.697441328324, 37690.125656332595, 88747.22866981631, 68889.2218896723]} +{"id": 702, "vector": [19368.58398555611, 90066.18669889511, 53077.510054858656, 47900.61636051962, 16665.200995278905, 49442.464964034705, 95146.86659882941, 98301.30294279041, 1400.3826305144628, 39122.29267396479, 45925.58648554894, 77156.05730481372, 96467.36212298331, 34662.91523224744, 59064.61114107879, 6422.264293726421, 74891.56437377004, 99748.1147085257, 50785.58020785019, 72350.34831154569, 66886.05254535092, 35333.21043922798, 73263.35324393245, 65672.71664839078, 33008.68829364586, 29020.62822955783, 58041.8119721077, 61222.791527834, 99457.77770957882, 84743.40788529544, 56309.51571692674, 66812.28576558761, 21349.153489270044, 52200.68985002679, 90889.32488255715, 6425.368590919078, 75579.22625208713, 58891.301993705456, 4074.8024454978827, 10189.555641897252, 72441.5606611576, 8831.82266852378, 43650.87812624406, 64004.58687265505, 37428.253845213076, 93657.03158823967, 76074.2796312462, 91163.42451511804, 13520.130899057847, 56482.41211188789, 71247.6211408854, 26079.51055124508, 36746.28411748746, 58640.981705280734, 83430.36052186404, 32931.45942485073, 87990.64315618115, 11480.954022629752, 52139.75086499692, 50866.81287042171, 97188.77215272235, 50476.30994487863, 33397.726650938544, 11794.71358805545, 76879.1397657348, 88045.24909428561, 85558.77001119358, 42680.37624653093, 54362.61473935633, 72984.27498538618, 8808.308505816309, 23690.230213734696, 10038.643466030539, 80310.7136218609, 5254.538561764411, 8470.531733411579, 53929.19401983406, 20108.301741531497, 30658.26241394176, 84498.19204601766, 42113.92303681645, 91791.221067994, 34118.29681082176, 41057.808594894304, 18833.61323258681, 64250.714062593215, 14591.786364700522, 48722.68902363063, 70690.96574940329, 63005.78372887152, 68168.66732481007, 35880.69659802502, 46391.38222980164, 37249.58169579452, 14892.705185214583, 16489.520145769588, 16130.764656065121, 10309.794884941937, 45785.5667290457, 18239.656097226198, 9427.84715388436, 48594.16312884956, 39573.25486104185, 22915.950096769033, 7980.663339436256, 66817.01964883335, 86704.29870881469, 65216.089598055216, 76728.2615592044, 80987.18779805173, 14248.517711138586, 14364.172034107014, 40455.08345554274, 94476.13511414015, 58415.428801891656, 6007.172256628545, 44033.195368502944, 36551.41613035994, 30233.33159138428, 95579.13168139308, 55793.13193104568, 27282.719162588608, 69376.53272764817, 83394.16873324855, 22534.095654643916, 46105.27427991722, 29335.341421600282, 43765.395730915]} +{"id": 2016, "vector": [97009.51722497288, 61205.88222246527, 93745.56173728047, 66368.41032416429, 42288.7737448958, 25471.666451783658, 99930.08030016192, 23985.01761055144, 92860.13590673535, 79763.1037797182, 10187.28003732714, 87389.95113709461, 89872.14035686228, 2321.2213010328055, 581.8528676702517, 41393.76223575993, 81424.72798460978, 49616.921639065, 14570.593816798095, 68784.49255131478, 4336.568708172162, 51238.48582894718, 10921.92058062662, 51947.084293595566, 37501.61267913208, 96647.30555045904, 12523.651314296447, 5988.662694388758, 59867.58197392471, 3339.54029933039, 64081.28701967423, 65916.33744343059, 69272.66865140176, 1073.4418731573326, 54712.77963941208, 21782.581222381024, 88256.7597289087, 29245.641145077538, 85791.0360839624, 89764.96457455667, 74844.3664202293, 35669.754738044234, 30670.164508342103, 33497.80878740827, 18162.491398120717, 72742.65922663959, 95679.49172484459, 61775.24668407056, 48242.85723277822, 7505.875044567145, 3785.365249416306, 94838.2868807461, 58389.861780140694, 69868.00896309792, 99075.99841449711, 55289.630243274056, 78633.85293935616, 29402.97933688838, 739.3076035185908, 34054.87052185647, 81559.57811336973, 79082.44888258283, 44075.88541020998, 6462.533542380012, 8692.788942250085, 8087.469151788207, 9934.390139986026, 49389.91415025471, 73649.71545973672, 78190.17225995573, 1409.5925081336502, 99162.96374517292, 76476.76459800119, 85299.469399972, 38655.534259301996, 52891.716973485934, 35777.14995414878, 13249.141836529066, 21902.73074677547, 47624.62642506439, 57219.242251856616, 34207.43608231088, 45977.8213496121, 49610.917500783406, 62175.25459669434, 70298.42998755664, 92585.97306953586, 45747.636376233466, 98448.90070906718, 55324.28722717635, 82259.05591177358, 28187.206554301425, 61509.00704128323, 71797.35148301162, 59275.26530343276, 55056.020595300106, 33821.319231661895, 13681.974197718493, 12796.061400797344, 47510.33938846602, 21849.77259437685, 47659.89151107021, 40238.94000784797, 49085.34354291675, 73402.46275642522, 97631.52508205638, 91456.00313470214, 52816.09711845915, 38870.58519696794, 88617.24455271008, 15152.664783770675, 4101.881486676307, 68021.20226686979, 1454.235810890081, 98310.7485312147, 43545.62880905576, 36865.42115157211, 44036.61929531856, 83192.94189033855, 51756.819774099204, 69767.71446435682, 28666.813117878664, 46205.5946963021, 49447.42118983538, 48706.45818784733, 24860.9647589934, 8970.153572946238, 51651.009881965016]} +{"id": 1244, "vector": [17395.176490029007, 25188.574419960096, 76858.33696805722, 46991.98944391474, 26843.842484849156, 14034.325870492703, 98331.51355353204, 7832.504111082639, 48750.29382349032, 50961.78337713074, 25364.667267734585, 65396.27550952333, 53023.59086980528, 6183.087688835143, 27786.143321092237, 42599.341690581714, 1062.987182020858, 75844.45075878868, 25867.054819318037, 16957.03985553483, 8441.269006242313, 19397.999595064564, 33471.750057148034, 81871.97145293167, 29849.933097025503, 82604.59866650797, 20826.514835880826, 5309.674508217543, 74496.71248370287, 20066.282270132062, 2589.5851412912953, 50167.83118117535, 27669.915191357562, 84333.23090461567, 65257.3695780344, 7413.520329028933, 47047.3821232494, 22886.578337453368, 15190.842522522818, 85858.97265331433, 15123.904246928832, 83061.12833852686, 20586.79040275828, 21108.106909905156, 66450.77894301542, 29006.88513244275, 72748.28278869465, 93417.29186408495, 7864.1563825323055, 41423.88229708084, 45042.844651308, 83158.23785426936, 22424.56701015817, 15733.288803228317, 15406.204175035298, 99010.15447054316, 41058.79666888244, 62876.325530899034, 75436.97708925184, 67174.33516336835, 42423.53649211421, 77788.22685495397, 82063.1731478688, 92054.31298182103, 89809.57767293033, 80010.70771906257, 65187.966469126135, 91594.36728638857, 94459.86012284168, 23679.716366146775, 6570.029567605051, 12782.805176269718, 30188.854137988652, 81202.93523487635, 10464.84275514562, 7873.197292803436, 16299.153383516174, 97669.37874713885, 16818.958490809666, 61403.57014000577, 61668.44244604293, 91392.37264567339, 552.9538640996035, 34704.000757003254, 68856.44083803432, 90678.63009245998, 68531.02452409046, 11641.237775094593, 91032.19144426333, 89311.76906542858, 441.80377098074206, 34306.31841180923, 30061.97178337532, 69738.42266411611, 99450.3787992071, 98583.79578351406, 75615.89475741267, 29555.98950974606, 94879.04856508272, 7118.445612909984, 56511.39835955389, 22918.968874386846, 34933.31832151486, 77522.43810196663, 14697.43465248039, 56027.945152712666, 65995.54683133322, 82020.05663977456, 95256.92475363144, 30606.30294489717, 65443.52882603655, 79899.13894310144, 54548.74277496001, 23060.607068444693, 15651.914112106979, 84083.7294732407, 91545.3674357419, 18828.61127016836, 67064.14994722613, 47645.69647137795, 99428.91834737117, 90287.95891261515, 54062.03712248585, 34837.92339712677, 36164.9086641472, 2474.4757583177957, 39603.21610130579, 34660.05848385607]} +{"id": 941, "vector": [20484.890187572568, 1063.6634634539787, 17478.32691518115, 449.6368064242584, 15435.07432917952, 5130.004526999388, 6544.764862078278, 99689.19642826234, 30452.433729922555, 97817.26966482608, 33932.50964518044, 37666.61509389615, 72957.07340275825, 36475.008495693204, 95121.0968780101, 58803.02627997074, 7023.176497836803, 11541.765202194632, 41699.41645423644, 86393.91173087289, 80335.62575144239, 13509.810483310013, 76247.19126852496, 32423.43230038417, 11362.990892043912, 8431.43991878258, 27617.45593624525, 47002.22835677409, 24538.33298038801, 39741.64789918877, 21477.085552531404, 12120.880253789124, 20131.416003229686, 12503.519740509051, 24358.838930052385, 38858.95451826717, 93499.33532941624, 20067.801080806712, 59828.9925569763, 67269.38053196477, 52809.08782699591, 5784.105408209861, 65037.769025712165, 95749.61771184663, 75392.69176972097, 9669.336356642933, 92007.10191903333, 55067.16964058306, 69631.77426498431, 19039.47268984638, 26035.934406019976, 56660.09666010478, 16177.747627781448, 81459.97930292685, 91110.93997828061, 57548.23014037402, 18728.207984717228, 45631.29511184068, 53237.17627248751, 51191.773685584965, 86436.34894005182, 74449.12600092392, 22792.80855452317, 44281.02215339381, 28336.095387488047, 79195.5449874576, 81724.93330962796, 33838.79022125214, 11747.155241569108, 72337.32821944614, 62577.481606547, 10526.527161631238, 59075.55483145034, 99485.34526087389, 79619.46516058058, 52446.03265315246, 70134.93916241909, 36480.50827343493, 56783.81986246551, 65393.48875510662, 49431.98685939926, 40468.78056073258, 19312.430762955333, 12297.286440357224, 26687.493599943078, 62328.44795848242, 13301.209012713134, 62099.18617675042, 20647.80517727981, 1949.0097934683592, 95577.93360736068, 45118.38856980998, 74350.83583093729, 30573.81855302457, 98653.32349396838, 35953.20215527967, 2588.5050435757394, 80442.11626262436, 86326.02888180515, 53580.47358657509, 94317.16037954397, 82615.98127692123, 85804.9082937977, 66772.72907340708, 46044.18701694057, 38688.679236781856, 78943.71105926932, 28546.386363050824, 53547.81924945406, 19900.46096270195, 49050.064534259705, 64606.78530661741, 41998.07726580828, 25029.6116273918, 94591.7865291093, 91221.23259024754, 28413.18036707464, 75029.89245705964, 84050.87717586721, 33206.86807434554, 60314.89346547624, 69287.72449501981, 5186.272804008574, 31852.675696864295, 53245.1622241346, 33818.48614011326, 6493.766887357122, 64436.76276997161]} +{"id": 500, "vector": [83864.80537141228, 87438.95794805247, 40821.313076449595, 68706.08921941595, 80980.16599074355, 80274.44487490808, 25390.99295311583, 77908.2014767218, 13011.180345336015, 40922.504641247135, 12036.633349026604, 45437.58599742509, 75703.08827919578, 99595.23718766098, 21631.7607274919, 24531.965023222303, 82045.53756006365, 48059.70119540911, 38168.78293668221, 27433.475213594404, 32841.26393424993, 15168.294646000724, 43987.11592035167, 24188.801614753953, 93874.48313199026, 59914.23884834366, 49437.466383388804, 24483.983793702846, 1952.3110986073289, 43456.522498173836, 55536.67840981763, 67116.40301199257, 87432.75891628469, 75994.68455545242, 16593.944526869243, 53113.65045070929, 3824.9479982600087, 87413.25506912479, 68887.08994390588, 16041.12081413126, 9809.264713109456, 49372.712309799426, 47930.61749087916, 22879.500383901373, 90438.6186851172, 24378.33578826887, 69790.59259318044, 96254.15327537, 61586.21063937983, 2740.9476904756257, 82474.04805458608, 77553.15831390773, 92514.0899951092, 98344.43328917059, 60404.896351107825, 65082.72786068556, 89913.47467768694, 93690.24767581679, 13960.02793942812, 46164.42157223117, 56103.13339467024, 15242.881381968975, 87383.50518671935, 62431.507747793854, 13361.06567602724, 66765.1960903694, 66483.04230510857, 43083.56404858502, 93974.6854594296, 60955.50258689918, 87922.65771922609, 89914.62510889224, 7006.495769806653, 56259.04388021826, 45501.69622632825, 53801.62135920738, 43477.12837477171, 11747.60049096294, 47178.89915975045, 10944.169170610385, 97091.90037047002, 92104.95844064334, 39123.26518390645, 78209.45994551224, 49764.17674091947, 91949.82160659693, 35269.371628843626, 73527.68488718777, 56224.25387076847, 48504.93973136528, 75149.29407676146, 77328.38495977517, 82624.16966227246, 44350.342691940015, 60380.693296210164, 10913.681531519615, 5384.39518911179, 25128.482796566255, 92851.34757232705, 84818.99563598588, 83678.32440128234, 72956.77546478856, 47617.443081763355, 12703.165596680332, 46036.79158835277, 46411.17299598779, 33938.40139798574, 71640.40954062049, 88044.71489410136, 11909.596598418071, 88171.39891155144, 6184.562407898408, 79766.97535426423, 33074.72934539298, 84167.8127377822, 25736.019376865228, 78266.23206194451, 73571.578679299, 90764.9884078309, 32496.734612296164, 82621.58679928265, 39555.774515224584, 41158.571396153166, 54007.51653178251, 63018.015118601565, 22415.6609461042, 69946.19414335578, 5441.812635364329]} +{"id": 1763, "vector": [54525.84173374506, 69117.6434972707, 48858.57930976823, 81883.53634766937, 45906.270628240745, 32272.564701383944, 28562.201656962494, 21902.42506416271, 10811.649839282556, 24566.965175161517, 50653.92797010669, 73764.9182496511, 94428.69535717058, 29754.647188146933, 40091.81326013992, 40223.920159367764, 64982.38141089772, 13184.289338017863, 81396.77067073669, 72203.17004827497, 86385.91008166129, 60440.5792312254, 18438.842933661126, 97201.2159402271, 40251.0201393689, 48702.71547266956, 16588.824945077773, 15096.377789234893, 70756.54611697461, 3324.052498258845, 92826.75205284385, 24111.510061474495, 92759.99226065252, 99172.57011747107, 40676.296162073464, 5771.005654991101, 35736.906531278415, 78978.9688166137, 20814.481292947297, 49969.36211570535, 910.3350193443549, 8927.83528709038, 25544.236609047777, 73266.8028472121, 61743.140182597024, 35088.58928230083, 77842.96622833674, 80610.75779897766, 54040.50749507701, 98898.98842465863, 32461.46470639709, 37676.50479948821, 47749.280740122165, 37879.4322188586, 58378.87209282098, 66271.8337903463, 53533.26077492512, 60874.97283067278, 56599.48585314829, 70990.9140277911, 10686.874577067518, 73478.57976096662, 2113.3076183764365, 9935.461654185607, 57289.74994006739, 29071.349859663165, 75560.52866407121, 36983.95062249362, 44055.82348603312, 15622.587080092864, 57436.422634485585, 82734.55309106437, 90894.52025972061, 45579.73283418085, 34649.57388395867, 28710.63594664631, 55.56598809840496, 81169.57336552, 61184.88565894872, 49662.60706963056, 94953.73392707153, 38613.04613787641, 86535.73757665112, 27734.29129994872, 95622.63095624425, 49141.61020857584, 14328.12216100382, 99935.82252794501, 75365.23829544248, 93337.53435405814, 25276.090279219, 76024.28301939009, 82967.91623306155, 11420.940435668836, 74903.69901938437, 97115.10531736568, 56452.70042468403, 88864.76259871326, 65323.496741465184, 95173.82101293525, 70900.54948607054, 77924.94335763695, 42773.28768639027, 64786.96334132579, 2185.8760566571545, 48486.38298421632, 96071.60642290412, 16224.578474270213, 96451.0688582711, 12059.946292153978, 72894.08279378798, 43659.191401454336, 59614.6540521281, 52293.00888609245, 69506.28361093218, 96943.6942720557, 82803.12807815029, 41111.50794089541, 72893.63305630081, 37009.755155017556, 53862.33444406455, 72718.56886440067, 86151.24151137772, 45131.47302151432, 67443.34659123865, 38553.34179402357, 86229.78053061095, 16674.145203907075]} +{"id": 231, "vector": [90021.12802178819, 4729.603368969038, 9671.620675391236, 65392.21194608502, 93404.05963729518, 58470.98393273865, 72971.00880191415, 11403.598977621532, 67755.57482970301, 46847.64995403842, 64556.711654404186, 39372.57293049228, 87936.76134789531, 2283.8887611365653, 84212.2950962068, 99920.99148319436, 40364.50679410717, 86115.39645577608, 95050.85682340771, 21961.801988826624, 90448.28892682989, 82137.60036664903, 43607.804632948886, 14989.460444266779, 84674.66728350919, 44954.67661066136, 38566.90876032848, 56777.22511193245, 70633.19994994104, 57443.36489595445, 64167.348737085806, 94869.17077501463, 10727.37794861529, 84344.18266157979, 21572.76999005979, 78232.56627689187, 78297.58974649022, 79467.46880317663, 82942.18719903458, 24377.481943189207, 43374.76077169764, 34884.17187714155, 74841.24244788052, 64767.25492456825, 94110.4792170656, 34692.918990267004, 8796.005021247267, 70050.66322952048, 2982.228875122772, 90086.99230949601, 22586.83866823835, 91698.39439278671, 79706.26495894979, 5550.357718173115, 70589.51946516396, 21727.17191556043, 57985.83004825182, 70422.43827867863, 91492.16743908508, 82991.2176573733, 83847.72640210141, 29753.47626450223, 9948.505808797381, 53541.18471349133, 88237.84097112746, 13168.184124356963, 33383.11429817518, 50681.07178267967, 60487.363780272775, 69075.88306847379, 68109.0014433326, 19245.320290166102, 21171.584723545366, 73781.11776601902, 79435.21681859512, 44280.024326012404, 41197.737642071275, 77657.77233410045, 37500.4900504453, 89019.70906700597, 30833.262699701423, 52802.39983561431, 72823.29147569546, 60958.513916657874, 30539.723000006914, 542.3331518698449, 83034.99336558252, 6581.094936367104, 96277.14960378365, 50512.71120516606, 66763.01110709779, 42746.07532944905, 86635.80770292066, 16760.94708042948, 66278.85877797683, 82055.535203151, 24141.48060505117, 17881.36047926747, 25653.193334939282, 51141.75664419897, 49102.258678078346, 12707.799656038676, 50465.27803126909, 80036.05740574225, 46980.951089641974, 37354.040418976256, 31873.38315604108, 25244.19818029914, 93530.59886525858, 46546.33711834637, 87893.44787100826, 98279.71000445985, 36925.48551359942, 86748.61712787904, 1333.0425900358823, 62819.844767959985, 23592.863549125865, 49333.78589413002, 17558.535457096514, 12486.395092157787, 27855.600738911544, 11274.991201687744, 62920.443358111246, 75941.83951270909, 39526.52112093964, 36649.80179647058, 1848.4325832492932, 36572.06237371978]} +{"id": 769, "vector": [99476.36441619846, 95846.03425360809, 4888.321624875969, 16405.1704623274, 58510.29312383312, 14046.594890092456, 66377.01817214747, 58809.93498091652, 18891.41020080877, 52466.026534567834, 33916.23158912702, 63872.063151079026, 16987.233481237985, 16110.876199365854, 79978.5901195405, 9256.173031969895, 25898.168383641885, 66704.78610882565, 29724.931801766474, 13424.19991487972, 42996.19382255513, 60584.29905566245, 59544.22962625628, 30052.2167814682, 78778.36899338732, 83369.07056027021, 33266.52521705095, 49842.07141499643, 89120.44720807638, 27871.586200351263, 19344.750463305736, 17331.209119527335, 62688.990146817814, 45056.14040579149, 85169.99111123089, 44030.14574680108, 37241.550790630805, 57838.03706669621, 79390.60800624579, 26203.885877609056, 7091.205228145536, 54961.02302237858, 474.6862323561252, 62589.41967126065, 58559.06956589448, 56180.482773760275, 3686.063111743365, 69962.275412041, 94862.73088492284, 84902.6193456507, 19096.19628075542, 52555.714065106506, 79432.94111018786, 42188.421780383534, 70901.87840391116, 99849.49326581076, 95110.41221127246, 40592.62883919447, 20427.31131023019, 78717.35064403898, 39435.799399111456, 5940.611630146264, 70601.41815438011, 36959.06757483209, 41707.18141785551, 17890.61470302197, 21069.010853707237, 97368.55817766464, 10446.214091268546, 88190.81016529702, 61849.6379846064, 93333.52368990799, 73277.48233660829, 77949.87861516546, 82649.16910509182, 77235.39127021644, 48416.88407797141, 34658.358540138535, 38885.90063808207, 36225.09981389302, 37763.86332031948, 95646.18954562205, 57418.61335471302, 13347.901657164208, 75493.4979295757, 37000.652503430065, 65635.04159995011, 37050.46417758202, 20421.380940033683, 91815.15216103327, 12286.853596595038, 44882.10619061649, 59881.01143929111, 1225.6545225656778, 75603.17458395143, 55146.368392258526, 15702.666489815887, 79299.05424735717, 72569.96249497023, 50739.937212421326, 14240.525348149113, 99254.25080231068, 95260.0854745531, 3632.803878054136, 59060.367571640716, 69042.70059337073, 21258.75626077196, 20922.338639280526, 4136.9195340895, 58442.288461870754, 99065.7424383235, 1519.3591953274522, 56327.3581651945, 36361.02390145, 51351.668622467885, 79052.36231786807, 61169.709646654504, 24386.57641043317, 35227.877942038365, 69885.93806626285, 68713.5075009658, 36058.612773053144, 40398.40653500516, 35836.04584083049, 54457.867597725526, 95765.94187108437, 19214.525116400604, 20339.522160758395]} +{"id": 166, "vector": [36025.6090606534, 38128.46750515826, 23010.41700936898, 46443.74985729915, 4213.36228439787, 69074.33844548234, 64587.22769801415, 82005.18549327727, 83374.36905914739, 69367.5355558658, 75703.28970612277, 96106.22874924992, 9902.838701048822, 14743.837957641015, 91942.09882998258, 82906.30208517065, 15601.874538969263, 32895.62532617457, 70615.41259503995, 66422.87500692831, 88740.67066410376, 50195.42551348409, 40648.85855298309, 33542.74069145541, 72748.86391027913, 77846.21002808945, 38870.366168359236, 59307.25907016888, 95322.32377620914, 13425.536353326239, 91706.44012976545, 87424.62377438322, 46298.77298285309, 66040.47678569108, 81258.28433502097, 49819.801985465805, 2859.1735729854518, 73970.58183322838, 23361.32605091937, 48307.705769747045, 50746.43909791692, 54620.76728706555, 22069.30533589607, 24223.52431022643, 75640.26403487736, 56014.858681503945, 79615.70877601851, 71257.387908646, 10003.448548428983, 48238.06255202546, 61164.51791337343, 26607.564486752977, 47726.15872116319, 44111.87994768077, 46944.56231527503, 7893.6935099161, 71231.2848930638, 53872.02904992156, 35658.55306626677, 20236.160159547955, 75816.68588412393, 93033.10049588446, 28083.89372289215, 8451.748221640342, 2060.8809948641783, 12944.895390191003, 97255.55856161981, 97509.93279009279, 91017.3986770986, 94605.96974537782, 69251.67541565736, 77375.63228448962, 7623.96570888082, 32679.969065104342, 39419.08108776567, 69348.02423020329, 68333.83499972778, 47831.796252816726, 38988.62624527092, 91337.16176371787, 99250.4331517991, 15653.694578078559, 98436.09340739479, 74946.12448234738, 44059.33074526376, 75102.98690638698, 68447.54862657622, 29719.911286737733, 14798.714329107943, 70081.65953431341, 27362.200246477176, 58847.153706298974, 9426.916315812872, 32686.82771354767, 62178.514747677815, 21729.641872315286, 56859.87967696494, 88195.4060731694, 46285.89497403761, 27810.404171258673, 84523.4227498889, 62263.89931815391, 53983.58580128577, 72978.89579417012, 57773.522263200204, 10125.045677029331, 47259.86262449988, 41019.548086276045, 11344.218060228584, 12966.494642999938, 7371.973417062217, 11068.250624877473, 50251.65975464214, 4074.257956216809, 95965.23384332609, 42026.32229807142, 63716.59218651239, 35529.062702829164, 50272.674600056285, 92492.74719214368, 25613.120180672046, 85510.83803223728, 36651.11344719509, 86099.3572112877, 98841.00646960169, 80289.21909491743, 48984.811270283644, 65722.18868298028]} +{"id": 1988, "vector": [8752.353995214835, 33200.481693300666, 49113.46878680411, 31781.159256801715, 69251.59490994686, 47881.672711501655, 75838.05884351092, 54590.301527764794, 45701.74350423342, 66668.57183530601, 73825.7532753018, 40704.417140916696, 95634.70509011821, 95884.78498180595, 76909.694668384, 97650.06352418261, 96494.45330527106, 65242.90503686384, 9637.424000905925, 87119.36515958085, 36015.65194330879, 75292.90020823968, 31717.902331995585, 59656.574394988784, 49658.16116376858, 91750.2731960479, 24860.22520502682, 91011.0290117895, 69619.12958703439, 78969.5476734114, 91686.8614324554, 49709.285022536984, 62434.8221735037, 61536.88707019661, 54176.40147524251, 44600.91548950642, 13203.115803199127, 15060.628903988016, 90654.0900555328, 71287.28159515724, 38999.84145134243, 72613.86302170024, 65403.879352245975, 26720.73978289903, 83784.63576221373, 68371.657461061, 65777.49262847529, 29203.310155982475, 44958.48795902925, 86769.39365772554, 22767.544748024637, 19801.148131348644, 94761.79271191718, 78000.53537253804, 43934.480400547545, 51725.107651828686, 36848.96192585881, 47243.59457606114, 45776.29107428156, 48489.107389778575, 11194.086910419599, 49481.707219157965, 70022.75418276167, 69207.57166366318, 11775.208525602276, 81274.7740655887, 22315.738649237515, 17257.61177794212, 41582.55398759455, 86680.10200133645, 58238.97347069046, 68039.81717154967, 9822.790060482466, 44412.15179707188, 22532.891996763217, 25372.73814545504, 69107.86711749194, 10777.129848956101, 77561.3491694363, 53214.868653479985, 79706.45960291465, 45749.412322868935, 50499.63834414156, 83133.91131810899, 19328.20895454673, 10489.842083682688, 93081.43984545479, 86333.26923267214, 7721.308737675492, 57320.65479840809, 91895.67264467604, 94795.47732729137, 39492.54393153602, 15978.577199965905, 44092.33785587454, 17933.97330346771, 52987.35642412282, 58189.12958406134, 26848.05237615263, 52803.7168694995, 17805.398768460112, 37747.77321665744, 59901.08796032045, 80371.17843393488, 53159.791428007586, 38191.45236881667, 6100.477990660791, 62432.857746502545, 31096.29367657031, 58147.14932425309, 55511.77893995443, 87438.74809235858, 88821.57764106893, 61347.60513396916, 18072.931094809286, 39175.143602615426, 13376.110025130638, 68054.83987344912, 98874.06466244171, 17124.672426198984, 85950.64474882379, 39236.1310737887, 84601.25683377423, 64300.59216481538, 31660.329412344847, 2818.997323514083, 31653.060238365983, 87470.28217947282]} +{"id": 253, "vector": [64461.12669041973, 83930.52638711297, 90722.34999837863, 9038.470436614565, 36551.57998106028, 46766.803551742676, 17902.3935943739, 47827.79058071014, 70583.19257418776, 67037.22405019263, 94730.63484973962, 20649.438023489198, 67704.8738441533, 13174.002059340195, 52707.10053783535, 39096.69551464118, 20126.708445806475, 26979.309817381745, 56894.97813762588, 76271.21080257655, 54388.9872276184, 51393.037786774476, 57197.35014974395, 37081.788548489516, 59407.62260424615, 82694.4121287161, 27844.140736198697, 48251.982196328536, 56020.985790546874, 16165.379499114973, 50600.882945854355, 39775.32882918443, 41337.77165905107, 3406.519138709885, 77194.58034525925, 48851.22077938353, 53440.478606105666, 4875.31752170417, 72703.40153799336, 8625.26079917245, 60599.13407355496, 43091.13101021025, 68438.52328807973, 9987.346024562938, 24917.29983004861, 58647.89847236277, 26319.67930658059, 34978.07379089931, 15238.974739186184, 29677.007055833048, 69832.76363789692, 21807.044759814307, 34027.58547561896, 29140.189231822977, 72151.28028171194, 94070.43426601558, 26102.635940596607, 84604.61166886064, 17995.111052563472, 87954.46608639038, 65747.73163159857, 12389.447087047523, 17502.218031145778, 81359.61399258042, 1460.9838144792998, 92969.18840210717, 3797.1990543080315, 66917.28559076962, 29792.03118907213, 62657.59554165291, 2894.082910801088, 65509.54894981955, 43058.565399653315, 26434.577278345907, 5658.816474757744, 84600.92595924297, 24481.798806710707, 21482.10649540041, 44551.25163364797, 79995.72941453119, 18588.53708348318, 87655.47889075245, 14347.780767270191, 41135.81829216885, 3053.625078819866, 20011.705954688365, 31728.666209322597, 2175.706888012363, 37212.53805742563, 2301.499957513342, 80643.82617357066, 47640.99606555698, 39437.47157385218, 27080.626858085943, 55502.346855940166, 65163.24996856683, 24023.45952015994, 3849.890526941202, 49476.35148147139, 99161.06742279623, 11458.963802861112, 39061.339234761224, 69218.17740101255, 43920.1412154128, 63995.34395942753, 8991.735988672033, 48575.15956003517, 45139.50455845651, 97049.59740556542, 29624.875822798815, 89646.91635891647, 91874.63876275737, 7156.025908758356, 58794.05552070186, 34311.683603274076, 14633.537080296099, 3098.3823711023483, 26819.440439807673, 93328.12393787487, 61093.3161438683, 71696.46972862542, 88596.28619855746, 58295.447984773775, 39443.81399322143, 97514.726198085, 82790.82079829757, 39725.17849485341, 65107.445364495965]} +{"id": 1209, "vector": [92285.61053546626, 3052.354260801149, 18916.114619110678, 75819.9565989592, 57985.22012679357, 17982.873727606286, 3521.323916863095, 71411.0967201856, 85901.70880217604, 65882.68029232608, 95450.13940252275, 69640.05792278702, 85665.08353816232, 20686.111692034214, 34227.18395773143, 26445.321568445324, 80096.80181900274, 45573.673632237136, 9970.352803848737, 17676.873472518673, 32896.35997130698, 86392.79684087358, 60932.19497864921, 51675.47848282111, 39563.64578887019, 33604.89927359337, 16462.76053937922, 24374.46913112421, 94930.5553014853, 37906.73338409462, 72118.60379345638, 23649.031650212328, 84893.7080488111, 51605.21242484396, 95262.96179040287, 35062.49928790201, 53924.52699973791, 90488.92616528358, 51136.43701633711, 27610.611459111235, 26668.578250391085, 96312.98640755368, 89982.01975276397, 49951.79478188506, 35304.943588820046, 25777.19236767867, 45396.73135418213, 33578.53747582654, 63959.87564792303, 21697.34768030912, 17370.79500570221, 5927.877991002839, 53553.11330623835, 66319.71146902924, 19894.336916772638, 77737.49780274628, 7702.3934410491265, 82594.58964780686, 62494.31704899918, 69961.96189948666, 42530.27609517689, 92853.74271463399, 76103.67628013516, 30181.77617682345, 96665.85487948627, 67383.4635628107, 71863.46468685483, 82046.84160455714, 40080.45273990799, 88994.90355370544, 76919.20735655967, 7122.272815723895, 49020.34996924835, 475.8196655857083, 89649.68259751462, 44493.119772403734, 47860.562133314066, 6429.225515365811, 96209.39190998841, 84265.71416541288, 35141.38698149801, 83599.31609625311, 52451.79320236613, 78573.47146793362, 3605.131746166368, 80616.76059348253, 62996.102373983944, 15866.356282649096, 4050.6308916471335, 55844.707053179176, 16146.85546774486, 88676.24572481193, 17416.203288165365, 40646.09357899335, 62754.40754979609, 42081.83023493079, 78377.88855974934, 16888.926069836973, 46108.761092238994, 87169.0120665991, 24687.984895633454, 47380.40988131298, 9945.510432751125, 98919.07311793757, 98684.571625156, 51933.04795306315, 8129.357070917554, 86361.98238354806, 47597.222128543974, 95034.63851292328, 5235.6056431482775, 88463.02649069313, 21596.603257890878, 93045.15906403135, 97489.34096461556, 63256.58136097105, 53512.44790740602, 79006.58857124408, 36463.23222284029, 83423.20047581311, 72521.57689892835, 79441.93233414042, 44884.48194527682, 42676.05515803036, 62662.20856260749, 68146.81247052008, 43627.032807053765, 45241.86393739846]} +{"id": 2105, "vector": [75982.13313580272, 19352.497659210898, 6048.979577750401, 35029.12642218773, 9958.37083193225, 48048.5932317105, 66022.51648443736, 49674.84171314984, 60872.28560188698, 53037.349321432994, 75102.26781726032, 74592.38888254947, 9750.062524971847, 6979.843420094234, 65743.56137647308, 52350.66982921446, 87878.95060017362, 17373.326010874778, 45724.13911964607, 78099.07415347322, 92960.04559085458, 79928.9984049541, 44074.81843979088, 80423.26437347203, 8943.964088613599, 70772.18162800166, 81311.94014581593, 15783.044887208931, 55647.21058029907, 63776.81656826742, 60130.3648680093, 65830.05912573158, 43190.715636905334, 71792.32934675852, 66639.6449660216, 66320.03908536998, 22557.623342106148, 20536.971913665413, 31084.88413300271, 17563.96403594177, 48489.86772761732, 65834.81495914023, 2676.302910338735, 11205.61319667731, 20764.873787966044, 50622.84740970965, 46443.883470217974, 71206.81653658493, 29531.05135704366, 79801.67479125137, 68629.56038838181, 79288.94073115228, 98750.70776376639, 31276.804962898565, 24643.368588996495, 38633.16156246268, 32266.784694117334, 82451.2174184494, 98833.87694637285, 71969.91651428494, 32201.7528209094, 54996.81956503659, 59608.29088738787, 20855.417244306373, 89316.69488023534, 26730.06511025019, 46196.59946246965, 66849.11417547945, 33316.747490994734, 95117.67153363653, 67861.89984329695, 63664.34098203357, 1564.1607701188054, 40073.73612586894, 58348.88411065674, 91967.51384810616, 78205.47711886953, 558.0234118431671, 22944.028849631515, 13214.84513609552, 34336.334524343445, 27492.741362217064, 18734.243125143224, 79730.75411804329, 70258.82653468834, 4783.504851641595, 16956.448841162965, 48567.45308511644, 86729.78374201001, 75801.65632009822, 9514.703221167143, 57241.2874845766, 27404.735038314375, 110.79457828178096, 12828.079487177902, 34681.71587136054, 76599.86241553145, 67493.93426991002, 97992.15501792847, 3567.402445897361, 2122.518680666308, 81469.39790050473, 78249.00374973482, 98679.65188475499, 82838.76661054387, 77935.18055464103, 36604.55157677697, 40784.19300712618, 83805.14551678958, 25494.90682258624, 85611.2409866325, 43894.84469119921, 50490.67940587453, 48826.15774703869, 97694.81174842967, 70000.33573994692, 14586.270894290832, 67465.55357497282, 38076.84286235874, 97216.85749303599, 22296.433462101573, 88783.91244641131, 23634.16255409784, 18141.653207197807, 54609.05051166165, 23228.57244697333, 67828.19186197453, 28675.749398580298]} +{"id": 156, "vector": [97508.60852338656, 80127.77584559927, 14119.342297592451, 80472.42851383102, 73786.15333895199, 6594.253490603707, 84009.6381359923, 37090.21523739866, 67065.59872947528, 67493.83228619272, 34840.50077880202, 71614.68740566925, 88662.12620028212, 98330.62460655675, 74377.81656374143, 89617.16824103503, 97249.95154336301, 15464.780522614841, 58557.69296124411, 51304.73419839164, 28021.298423998163, 95255.92768039578, 23960.606964264163, 44943.05684977302, 47136.16241061778, 61683.799005408066, 78024.003725717, 1056.6459090995627, 69907.12265299626, 99363.64505076985, 17236.197412865808, 4946.583248398928, 30331.59117835784, 52919.61608817813, 82709.06665599652, 71913.2192648, 10137.647817655115, 30651.298052900846, 2016.2046090392516, 43503.451437798314, 2140.036197513251, 96108.63084286625, 35788.46197263728, 28799.265510903926, 36610.84271879128, 4976.59376120686, 14121.126779567317, 59125.719110687955, 75885.26297102805, 63357.98602033267, 13972.098832959511, 25093.981922305986, 54745.10617912325, 65261.697409795604, 35601.24664566745, 63056.367224926515, 83425.5543078066, 231.9024478202203, 40461.73910780158, 55977.54150755815, 8295.195805516809, 4366.918499810291, 64236.79837225221, 64531.93747302608, 81661.31778293285, 78773.74469978895, 88948.63583498415, 21539.654686602316, 79920.85629419994, 52927.89786069566, 61573.03390034914, 87920.58188250102, 45760.13973109392, 89623.44106900618, 4332.186820976902, 85366.79699598101, 50708.34290755519, 67858.92785311324, 72965.72749560182, 6452.069073538003, 37413.716871549375, 24600.626973642247, 23396.935877884185, 38780.69756768834, 33516.67232543896, 67663.11294252955, 83055.53084064068, 70953.24579629133, 83676.39839947966, 66331.47563494656, 87891.01105951778, 50877.48331125225, 89047.83719659103, 15303.750697751982, 50532.87207302818, 13592.251254358656, 48669.45187018263, 95864.48230645775, 75218.62081284568, 10453.829441132333, 92508.31250687315, 47212.42019801561, 46108.57294666415, 30942.08809977138, 877.3587660883097, 39451.363977026354, 72656.36093258818, 724.7322643935905, 82346.4359385235, 4026.181211522206, 62978.604012855365, 70019.61771940668, 90072.48149878619, 34858.37572846435, 14474.82520744856, 33789.860537907814, 12727.559211705851, 87195.16360855312, 52559.845391403185, 99674.72926050537, 4554.937113133217, 96018.46570683042, 57581.61352210478, 81163.37520841659, 34673.522842741026, 74949.54269360953, 11033.139354305777, 69865.5149888051]} +{"id": 1720, "vector": [83727.14749422227, 28963.615847881087, 82644.34196984718, 71253.2675826791, 69534.23626816987, 543.0762295311142, 20051.96628338618, 51842.943626000335, 86117.75362415082, 19319.99444339172, 4507.196153471083, 82726.75903918379, 94088.46017536949, 14376.997371754607, 41345.961599132905, 52502.13292370232, 80196.4388365855, 55029.05294610124, 29023.79937705122, 25518.604181214545, 92610.26977277655, 64380.357513112394, 18826.76135591801, 13194.837929565594, 56951.01304311382, 61394.39871328595, 24307.809394594406, 328.97846169737124, 19833.03422211766, 56196.927717395614, 81653.7277610002, 11576.15827883447, 81225.57937874868, 23998.99265792762, 27630.87743502549, 79216.47556609909, 27801.624848216856, 77906.38256526784, 8633.654606182461, 70174.08586674761, 14438.00220385697, 36741.13508893541, 27948.139225535928, 90545.10984700584, 72406.1744720919, 70577.39375011243, 6607.824828444719, 42805.62959730375, 78311.6517109074, 88800.51411068898, 47644.978047854405, 21185.38973089197, 43351.0251545566, 35142.73768919946, 37222.02323651509, 64966.994339313766, 47777.68304378278, 42624.75965722149, 27651.12385126368, 50260.26356976709, 68952.12364240379, 639.9307870803783, 40769.80121364732, 17944.54062888131, 5081.326230171324, 59573.70349236164, 43088.4399292305, 15290.44168468261, 88360.47224030188, 87463.17716079354, 49581.44291746619, 51291.235348411305, 98382.77488666601, 46429.17746529285, 11726.77378930027, 74444.23176739672, 88178.4759826733, 16901.313851091592, 25348.3223857193, 48418.82822292631, 63686.808720278685, 45776.81935276294, 95696.04282759076, 1305.5498540617273, 50270.20299889248, 29678.41078099259, 78284.41868654778, 50123.663545102914, 28570.24864651241, 82505.33171005678, 15612.975606814549, 98143.40390371457, 47981.04353787868, 29307.837122907433, 69651.54771536824, 4483.7865375653155, 40293.81030700575, 76920.55571238844, 26083.83185989821, 16265.18853367881, 94646.98614263452, 23544.814089689648, 42573.13216843347, 73265.91605431204, 72569.36479575929, 69418.37714447963, 2095.68968868965, 4962.572295289414, 84739.43061086883, 45092.01202566449, 79123.69734118863, 36894.92695726271, 35740.20146418718, 39602.6261207496, 60629.733932539144, 56945.000381651, 71699.22766763858, 93271.57839028462, 10977.782298714068, 10348.896723264334, 31564.14377526349, 9478.74359597276, 19769.404871652474, 25451.126456872065, 48423.46909695996, 40514.59344300229, 98812.641759476, 34047.14940765306]} +{"id": 436, "vector": [17847.52969509098, 52938.38365057365, 88176.6851008809, 36534.920269376504, 99368.30143557346, 78906.37494503941, 77411.21328423297, 42798.3880749483, 44052.171510779306, 96262.50738122161, 27760.208302098, 14693.561313521974, 21927.4601891848, 28983.581321443096, 79991.31444606594, 78522.76196194874, 84184.00193369882, 86023.48126114659, 18635.644443360823, 88151.31897528215, 9165.394417547912, 35467.0420867542, 51110.922957058225, 64156.72504698088, 45763.11105914934, 80055.89791679001, 1290.379590131674, 23315.934066357357, 31672.625651281905, 7350.85916160525, 24278.032295381003, 61462.72047196382, 54739.06605529759, 70855.77000790785, 47995.88998851463, 90395.79619617807, 585.6272305266308, 76927.85804885767, 39923.96205975074, 841.754862917421, 9476.382204256784, 12345.10877629319, 489.780649110616, 61437.89028780619, 63401.74769304277, 72680.36948861109, 91220.09343936267, 98292.73398251124, 54803.654655761115, 75774.73856014328, 42687.687674460794, 69332.03831272075, 22951.301784147403, 3628.697063748121, 96167.98966809992, 28149.151772303947, 75406.11138801812, 45055.17151161462, 43154.68513444786, 29438.943485659962, 23630.025139753987, 21431.5574242244, 47122.38332923523, 96683.0072513357, 66309.2566454707, 3.354998960558575, 91701.65729381642, 18289.724534012654, 96276.9598905076, 44454.64752166671, 61298.42087769977, 63778.05664284798, 41098.7336149068, 80895.82718138579, 52865.609219327904, 22263.103177751353, 6883.943223486588, 6772.595950804083, 24884.71380312751, 664.8577652427767, 53371.343347836046, 86465.13546007336, 47120.59288570882, 22964.310909121465, 97521.4253663139, 29460.315850718234, 39399.551848610514, 11763.769796630719, 24741.305179824456, 5282.796969745207, 20954.541226771275, 69888.15822652588, 68597.90865515881, 26399.39354745178, 72352.3727149548, 45163.12137612727, 56198.08277367849, 41639.84715453265, 99748.24696437479, 72093.97724546593, 68089.67547007238, 95965.09083903694, 20859.746640414767, 49267.8733997033, 78721.67041165958, 10215.660754909284, 22704.701838252895, 55377.41678228349, 4124.305224243996, 75591.15754025806, 19522.52054724234, 96643.90778648402, 9400.72467685651, 92734.39071918366, 64710.031345137344, 81583.06493886426, 61435.39140692784, 97031.46635295532, 77053.50193795902, 45855.0765090943, 78199.99034063893, 67475.35201595204, 61964.787791997456, 64866.923344700146, 40763.013321705264, 17331.45411845599, 84589.3099631648, 1927.2985767019968]} +{"id": 251, "vector": [68509.53158954208, 48231.020056589405, 28940.100932461566, 3676.9171749961747, 62418.00882067444, 1656.0072839922557, 47907.519175852554, 61494.38835656358, 61437.15938148372, 28096.22496204166, 28059.73771732354, 14816.185162195128, 47732.7272993635, 84160.71196746003, 57560.900862796974, 52393.6533589073, 43553.16620704619, 820.8355371745912, 48909.31865032808, 53395.42324285114, 56664.6531218945, 26424.642484435324, 86380.793761307, 85623.25358837523, 99775.6891459825, 40109.216669580106, 80986.91730819564, 50270.38206377371, 69847.03040219552, 93673.11445886639, 90047.16757072674, 82319.32310758869, 13902.441957398425, 10375.85355171371, 3993.2945564899014, 33920.1792279242, 74350.27038189396, 26729.683269357964, 71696.56232789789, 21981.287379672565, 14092.46873545703, 17692.47547165972, 5587.857914972927, 96943.41033949447, 92427.47648405223, 90458.61672348669, 75990.1676179478, 69442.8161430637, 81699.92538773446, 15571.311570505408, 62993.05775587748, 6961.730051082426, 38139.925602040414, 85513.27688109885, 32105.499201867515, 67417.74216490539, 94987.13695876689, 28695.107434224865, 7540.120235672399, 90005.89643422111, 7749.280233813028, 69008.3195604828, 53268.54886739184, 45265.66353949501, 44508.480178669284, 50064.0306505182, 2884.478326237738, 87338.92312940324, 39022.087972522466, 14048.508273891424, 76422.42532609243, 66044.46393598572, 7274.247832091074, 14641.437923197787, 6898.17923520667, 74405.87150440755, 65716.54005289119, 24950.84306315699, 87466.14293981672, 20196.6653011449, 58425.11513879779, 85868.01400691774, 73331.5108957941, 78863.35154474739, 6675.930642633277, 72992.55509758793, 98396.3366241933, 62772.98502078459, 91327.02101442656, 52985.60211460824, 36861.527511743894, 72326.01459344615, 99006.94806043392, 23047.31040284981, 67979.05664646243, 87364.98632834754, 19896.285106563017, 80923.70757035381, 59592.009972062035, 17814.09541830785, 62696.253054472705, 91221.27434752056, 85949.61557188845, 32039.791380862094, 54902.651662796, 68827.28409720257, 4839.344371916865, 31725.388152226074, 38319.788108877474, 67565.64182490627, 93593.3268731304, 19770.121379123175, 2965.235636354713, 9384.757518323982, 62218.5270706825, 92134.95782021494, 93524.1752785248, 93830.2104797632, 22474.476756246255, 44026.497148962626, 61671.404766309126, 1202.2251437796006, 90396.08060253324, 40721.3312743726, 74102.34341957024, 13227.499974630131, 1821.8856020207231, 13334.129402777995]} +{"id": 47, "vector": [41098.159625398286, 51818.561354755075, 33797.87670333868, 26671.835174375203, 42572.231447578124, 76022.12794687033, 93449.57672948057, 81395.9051781906, 49828.823185372996, 74337.86137186662, 28630.115310611236, 81687.44142059254, 51315.50566264264, 20432.964288637344, 67569.34588169449, 8183.295961412873, 73479.83333027987, 49167.752035836034, 59955.331329008855, 39082.57798050192, 94018.54349469904, 39726.232600669806, 87352.03760599365, 55456.893824710714, 78852.15343828681, 41478.223382405566, 97523.6870016115, 49466.10888915067, 62312.178897498685, 89252.35520019552, 47613.269986232786, 18748.69069722661, 28121.252885564107, 26916.710483248673, 33366.6777393338, 61406.633304693016, 17480.206986915393, 44241.29494374643, 64299.550657184176, 84882.04170049621, 85101.18147471116, 7184.312626036104, 7991.329963802096, 6368.789747915504, 50605.48735270909, 73003.2301413435, 98977.42498311037, 95657.33298939097, 99770.69731027095, 34426.4555806643, 28034.903240985153, 40881.029591764505, 95255.74481946259, 88384.14568451894, 97784.509686343, 86823.87196953387, 47350.89891241385, 56541.78569370662, 41816.19485058009, 47216.09787188345, 51546.81625117933, 87317.03455852885, 81677.20794600058, 64619.30688357743, 75329.9933122652, 22715.166163443646, 27034.916754303253, 6103.771795208446, 73965.57353693293, 10598.473610612313, 44064.707484726496, 91800.50495070047, 19058.41186646904, 78542.29066724521, 17589.63653274486, 78620.78413440583, 36523.95649915389, 26989.171239490915, 73681.84468576875, 80085.1694618645, 40122.19960103007, 25100.752743749665, 57592.21685434298, 97313.27762939005, 91855.26357021314, 36021.642086189844, 13451.518466124367, 28586.137330239835, 46573.38916385485, 76093.3572744252, 36474.75804868336, 37233.56666245251, 28893.41640294896, 28783.504251200065, 7802.105576881224, 6496.055586373927, 8140.808955252821, 68453.97728593051, 10167.10237081091, 2802.6972741463487, 82839.33033419187, 15615.453641579625, 16730.80093270045, 13054.518179124974, 8568.27160614193, 57194.125252296, 26566.016919524893, 14580.126851601872, 70298.57078913305, 51245.01445381939, 19352.908498837274, 55861.86224288716, 72807.17404114088, 13583.968202551743, 11556.803852523422, 9110.291282997106, 31913.836143036267, 48577.20128780761, 62569.28975075594, 60744.84294976468, 81579.78131850294, 94662.62339840815, 34130.39077029664, 25694.0944753158, 90132.83151394298, 9436.647052062752, 19597.61777585263, 13960.53778789147]} +{"id": 736, "vector": [10673.257772600708, 49198.4980428616, 31860.95804498339, 87982.07175103563, 21586.773016193827, 22134.774018465374, 45381.30725409054, 67541.85352473104, 18684.795637230967, 99737.48315691872, 75340.44136502755, 3903.0120031936067, 62003.53592101344, 27640.00629751695, 24029.941100828444, 42959.47211340111, 7012.195346640082, 87100.93706224926, 2809.4955801351684, 7198.058853989242, 90891.28818682686, 87345.3491504746, 1274.3393691830418, 50115.31890786024, 9794.576003258471, 46934.35718869771, 81854.66196396777, 39947.459371655634, 93290.85866873484, 67747.15646959029, 50879.222558857626, 84446.20689947854, 46227.99260098249, 56728.34665468425, 17511.20016629315, 24630.179828694752, 88117.65356816484, 71732.82853762306, 2524.4642890593805, 33171.38061280286, 78873.54315885332, 23311.40157790542, 94009.3092113048, 92780.60517454853, 57942.88047676724, 39889.255007155036, 48979.49061009835, 56417.63909895863, 94727.2478396306, 94064.38186700306, 65040.2047690028, 2628.934101454339, 25149.474729871967, 10486.256611423762, 60873.853741232095, 33800.01924616345, 80567.65468117026, 32557.143515311083, 61197.04053598245, 57615.340634313594, 27524.75104826486, 42549.08935970787, 41718.981554112725, 8166.686253844901, 91305.3566642168, 66499.07567071324, 30411.137364241215, 40434.96652738623, 83172.39859743006, 46711.92109323048, 14294.567797732905, 18455.086741194315, 97012.67474223087, 67574.56338881988, 53351.80698247429, 87994.16757056663, 68289.80597060648, 16312.425724179036, 96436.86279824453, 30954.444062616858, 53386.901835796576, 6986.218999888904, 51359.46962705424, 11417.932098876938, 72742.59386103168, 36176.341246765274, 68727.5598372024, 83487.99574325587, 75056.81034143436, 57100.87756273724, 13667.854325305074, 65617.45324876855, 64394.32510218709, 61094.83632561101, 84079.46528019736, 21372.960719180744, 84176.25845392434, 66757.33326492632, 21729.84530833737, 78819.99667026145, 25461.034310082152, 61354.579302437116, 62073.38098578466, 4344.938344607241, 21116.81896595331, 59998.26469841277, 29125.79715250806, 26559.301478544163, 4530.826658963804, 19019.53428126836, 66986.31791455636, 10569.800118397021, 88038.93471404126, 42023.00406045081, 45385.617967457634, 36576.33997833682, 33519.57612777205, 16561.79402224004, 74847.77132832863, 75508.77952444513, 63302.9019988866, 81892.24223849882, 28159.28106460742, 72870.12246964176, 20036.278971286934, 94740.36176476139, 26948.889912185416, 10951.986358714716]} +{"id": 1208, "vector": [92777.70476439172, 84032.25843004267, 54439.08113920964, 34842.40344039764, 32.791171270685204, 14823.856745073905, 44906.29740247046, 67114.50726117588, 6855.992072864803, 3784.5463220503084, 1464.2389101112929, 2761.7953958305907, 6240.546314743556, 66954.6348395992, 96443.18556927503, 81191.57528647664, 45320.42480884116, 14248.919665684578, 89911.64769241564, 10619.512129943143, 82016.6941308709, 42068.295915186995, 49502.34267154552, 76135.68359628353, 1678.8168548807826, 50168.26501228461, 23287.412450998392, 71930.48255334301, 6836.206316460647, 32556.189152195013, 46869.837655701784, 30082.12849575791, 89120.29195870015, 65995.05109845393, 10460.707573108952, 86610.23945316554, 51315.28827221381, 53075.945022044936, 5263.583670492711, 11000.156989392095, 63058.931115851956, 46786.36823966187, 22143.451679861515, 14447.482427405323, 93345.10694858438, 18678.24014968176, 96144.69330781017, 99713.8625691491, 47905.854927641056, 36955.2818437438, 15668.210475632526, 43193.85477996606, 88743.71087101754, 74810.49103650011, 95069.38043257168, 93141.36190613084, 40177.55010475287, 22292.095918619616, 13207.705414715987, 2605.404042508741, 79604.48797940394, 2963.725680953888, 96412.49796016324, 28636.88036781967, 50107.418528281334, 85045.77390494992, 81316.15165722193, 68021.8845056696, 87187.79664850613, 48396.82784253645, 831.7665974550747, 44952.87451825116, 83440.98707916842, 41378.01759163365, 2905.574892878338, 63995.153421552, 89950.38876294476, 92141.84859781186, 34272.70351300276, 26863.985785766432, 96508.52976265125, 40491.536902175714, 23574.39742616003, 36646.12661821119, 71069.81656389862, 52532.386465982694, 23669.274804087494, 17506.10061239248, 30146.818437358503, 12438.988981324006, 45354.2440233279, 99286.72076491363, 75554.11063535388, 8139.892251730529, 23767.081054414106, 8046.596744320255, 92410.24103952588, 4280.3987646515825, 88846.31539224644, 45934.1445185612, 61547.276847119116, 10271.28622912311, 45049.42660346402, 84750.10304096944, 57664.78237206315, 89252.2890307967, 57077.871727744, 78223.07861490703, 54201.57554405889, 77022.77633824492, 50853.36102421243, 94067.51672119964, 91148.97931763825, 38008.98484484853, 33324.94696260914, 4734.607352419662, 40759.69284042996, 79301.8989420138, 29018.857357014906, 6307.568265018593, 29024.411679886587, 38868.421253963104, 21220.421127438127, 76553.47237340733, 56161.47042815314, 92608.01705756104, 17075.087734097306, 48045.01096796287]} +{"id": 957, "vector": [73931.51929118953, 3112.156389127474, 4086.602344790091, 43222.06794673044, 24182.773846970627, 97356.1421574098, 93901.74175546935, 16931.353052810326, 13428.076561695323, 18717.652059590895, 80053.4865405, 88320.89693693153, 56491.33654619316, 56281.01158723614, 12719.759201818726, 52624.22384121254, 83519.19000421197, 38234.031319730544, 4013.9243336312693, 20999.45544224907, 12481.638340843814, 27333.436012410217, 56731.41234676866, 52104.70285560427, 6649.519386005898, 24831.056095674063, 13146.60252438924, 91241.04337035732, 5407.429932777863, 41238.70144242995, 93965.99934416819, 74953.42978153493, 66813.13816177791, 51809.30806214734, 79600.36875275905, 61283.34887369514, 2151.696377841239, 91705.99197804987, 90104.54534587725, 24547.853989372114, 51430.127084464824, 43694.123494354084, 45367.39626387312, 3179.6396907929457, 93625.22173927996, 21884.232145759197, 78016.81746175558, 21526.384382525164, 87692.11069214386, 27763.069145982776, 43040.56635838484, 48365.860064770364, 37585.58207347423, 86639.16318383896, 73198.42412897204, 19961.817473452193, 84985.41969772095, 65849.84070912759, 3886.473384779143, 71649.8874281177, 33905.19439104911, 63725.867296837, 28618.62527611967, 54171.23039952905, 28537.07921711074, 20661.264943937764, 46972.38801481337, 30066.322611071828, 6337.1724359910895, 44864.87231905042, 80828.16619540301, 59441.87868101803, 10281.75279024629, 31367.721864848674, 46460.81961294373, 38758.58894497686, 38491.53877310738, 45510.66241204728, 99449.34238026099, 72196.37351879598, 74588.82427290305, 45599.05434220541, 4136.646762857943, 15182.53636968887, 18574.310764933045, 2779.447672412849, 90298.15523101765, 26875.680709689554, 76694.7687564977, 64973.1775523584, 60137.11860543891, 46060.03748454826, 79979.20661783824, 93135.20130345713, 21310.26915673607, 25508.687750721514, 2335.331505260929, 94005.31435607576, 15127.87894423222, 53866.758919698754, 9467.152561629366, 28171.783146597052, 14873.461541796807, 12784.053997893885, 27209.60385991583, 14012.610652377378, 40419.53978064084, 7606.808405927756, 42253.79761722335, 55519.8663255015, 52029.53740643368, 31931.40243404449, 88837.60281109576, 98422.3809593589, 44148.15407317188, 91095.36807207215, 44314.034575882935, 19999.21844165463, 5585.474276221769, 77574.90295176115, 61758.904180671925, 61913.15386476411, 77762.07975225152, 59480.83778871822, 42656.71570052084, 54080.720281974645, 43183.295961102296, 29352.110595212987]} +{"id": 706, "vector": [59865.24814499479, 75546.15412815538, 67745.02571797092, 62285.52806077101, 37511.60920405343, 57455.47840889626, 12000.341758310518, 18065.18253985231, 60476.15843944897, 39859.329972275584, 27523.434652939526, 75921.51489928171, 66959.96841397653, 81185.16451507124, 5004.773827803421, 91897.08423591068, 4571.3399947207245, 9423.206572232923, 97713.14732065791, 67521.52619802025, 45573.27051799924, 98543.06578716818, 63556.34134635916, 33094.85626665892, 30786.530399611966, 59333.0303705611, 44361.12277377851, 56184.24229212499, 73445.55834713583, 78931.89550207922, 61047.133841370385, 15275.340636071332, 69162.88924704172, 63891.506221279036, 8871.833123447093, 31325.242881397862, 87814.25610640539, 91300.47196422679, 8840.636203757147, 93132.47253916445, 40791.693757297806, 38682.35701870495, 76270.49269249222, 59431.23789025498, 85984.52415752642, 82497.89623203535, 43550.70213162601, 72572.98201897829, 65879.04798874374, 8862.613032718513, 85830.97625790788, 28181.136429973507, 89624.82750756026, 24850.66622760366, 89483.99931424875, 83131.1671418576, 33605.03644858367, 23906.454947590682, 98737.86454421317, 45030.51673963938, 68268.97514533691, 91669.61378451515, 9086.800618305724, 87004.40573222187, 79613.72244456268, 14344.081886083404, 24191.063938618307, 39419.74635995209, 99806.02218557945, 67123.95272029798, 45763.93362527059, 98623.67605989378, 8521.015142212018, 99139.4227478669, 10511.852264243904, 20429.37277119402, 5171.992186169083, 39758.74516573018, 86487.16272480409, 98338.70148150416, 87434.37718293822, 58078.15131018711, 9660.373649725596, 22622.89826299293, 1739.318919009314, 58361.959674197074, 81101.00351276688, 95416.1041127079, 70817.13594489364, 54369.785459044004, 76574.30910143525, 37390.784833912585, 56882.25448295626, 30556.13372229141, 63836.245515124756, 89989.70784821917, 75353.35970847044, 24492.165608982796, 16856.35885014498, 75495.70788172842, 99119.71751422781, 13991.181326563374, 5024.390144467939, 45783.79710480224, 11956.075923703891, 95884.10662321556, 18826.520881359975, 63269.7307358433, 84093.5951657762, 98213.94585190558, 82826.71755950752, 90344.19980877536, 23655.70949107343, 99972.64860358209, 45396.51991018227, 54515.77293477709, 44743.222846478304, 10525.699858250404, 90798.52915878322, 45061.97266303955, 1472.8480007353494, 29116.517196380097, 9614.33505468854, 34462.841313473735, 59017.99711077243, 68525.56415915536, 17556.423207433127, 76897.84900161845]} +{"id": 1213, "vector": [4009.66092266245, 44643.020279379416, 552.0725846732244, 96835.85122060624, 5097.485684146131, 32111.432089512637, 23508.51739823393, 1093.6941571901282, 42520.69298313527, 45827.51721082303, 83385.97761881362, 33272.908899697286, 69907.51633214833, 53328.911437509705, 61850.67925931657, 76399.77135687732, 20253.513363437203, 96266.19672592393, 61932.657542521396, 2501.149032690719, 76921.2309167859, 71302.30293017092, 93142.90062134563, 3470.059744584042, 24561.183216446138, 82449.39909164874, 94998.47683889608, 37957.42048118428, 70980.60173380031, 48412.975384614074, 10995.81025055948, 34624.06743661254, 67759.43970549818, 29745.29491437412, 59598.92934470211, 41192.52320769281, 89936.66225572025, 13033.022600777444, 57494.504627460476, 32927.031462567415, 52440.71191734081, 9512.456775040479, 36890.91567455039, 54379.19554052834, 97747.85000411529, 42348.31786292376, 68699.38663004908, 44493.21876316328, 86445.2753354467, 71143.15427716193, 39602.548020741, 70407.16902952206, 36524.79232499717, 72705.8544201651, 13187.474634395425, 5466.868434647132, 4345.714083512075, 84824.32821287416, 24150.091625870427, 84836.45944244631, 40838.38767641096, 71232.8321877419, 99790.08434939747, 74867.87227799596, 43471.06427862071, 31049.49100706491, 5575.578971507622, 6201.019768260107, 72260.75179096292, 25197.79054384409, 75234.66736337326, 52137.55143717321, 31358.865949431747, 10703.850038021468, 61338.19825884034, 66775.29061258408, 86540.7308372466, 56932.31816877208, 19764.18831513116, 32407.697067734465, 42416.596738178436, 36573.68883939584, 68126.41573405868, 98227.67449563382, 14588.637616122624, 83091.60629344235, 62631.67158513945, 42399.459464643274, 1586.9907116450088, 18846.804931301664, 48185.07634798776, 97760.96641396212, 32925.724756611686, 5661.562474445669, 47181.770564487044, 25279.89054742351, 68513.91664846934, 13768.242686380872, 71989.7673954884, 18424.046571905907, 66133.98713502112, 57277.58548612832, 46536.07606218986, 4193.909279577501, 87647.69190553323, 25899.33284908511, 81893.00548686556, 44033.940234202164, 91396.45613423052, 69505.5777875403, 32767.62889295873, 98937.3636727077, 79204.04880432488, 34321.81614098717, 77298.08958275424, 20845.435562014525, 56291.891543035876, 79089.14509882158, 47614.03161682608, 57852.26821675298, 53842.92978434937, 74236.7270621964, 93222.10529224276, 38391.80027109428, 2002.0973268533583, 4440.058408533431, 61866.80304143189, 26463.097232679665]} +{"id": 879, "vector": [12440.730613570395, 42447.2649237546, 60160.266083931354, 32468.8061945833, 57278.59589418933, 14287.46552379373, 61631.07699023415, 22783.986556432512, 15082.903390928637, 85138.25780804435, 44483.36362754993, 28085.2319982971, 46809.66410618479, 82138.02988429434, 46670.94850739998, 85992.65920906169, 51608.3380413621, 44722.317459074875, 35487.361680962495, 26146.655972333578, 35919.61602813111, 85284.80356423208, 43787.47807496284, 92885.93027129523, 17396.466360420327, 31897.336953875198, 34190.11739145201, 17230.87384771238, 43316.19514457041, 28312.207096687725, 401.7206061643241, 38476.90272454598, 86712.01730177157, 50033.83717808431, 59896.26621449988, 11924.982416884866, 57498.37676826862, 33587.81244095568, 15041.105942796317, 76212.29996877952, 95032.05001576013, 69206.49670308907, 45963.744006694906, 18328.05295566058, 41350.60734096135, 15337.696881227925, 80844.69506344812, 72285.49847910566, 28859.325652908974, 10498.559444288569, 78340.31071596178, 25456.17184406368, 1804.273933654521, 50421.0046510681, 49355.05674967913, 42721.515673635826, 37645.37090598304, 96500.33675619378, 32712.140391542656, 46975.07257810444, 25657.65471719348, 49667.5908929331, 33514.35077051915, 93576.41185932604, 78032.50804550515, 14427.891740134059, 82060.04854507503, 43185.20292623611, 82525.11262704691, 53800.376673157734, 92212.26098382086, 72505.68355865483, 61284.65334259533, 15610.626003305617, 72154.14360459374, 9213.693048974481, 34585.10979314812, 51503.05174105223, 94203.31329695536, 5017.278135817948, 88869.0571208474, 58386.543537464, 85968.00739323492, 35415.795147597, 44332.0960395817, 52815.4833284875, 1880.8311268766697, 99030.07428458866, 66982.6374832118, 30842.14908783105, 94134.61327409653, 55250.57804113375, 98330.86048975872, 63066.04710932839, 57331.29940188917, 11455.758608417898, 56651.646221078, 89300.98175835621, 12729.07007920664, 31781.925444650984, 33139.510352527745, 35612.937428549754, 64232.27554763673, 25012.05391421856, 84780.57545373522, 9390.037545123885, 78231.33484617271, 7511.760506517684, 60678.47811534013, 26998.830140041795, 38818.61090438017, 97931.61464840053, 670.6276393592448, 61946.748450125, 34387.416353818575, 50274.2757284664, 49989.385273547705, 28304.615070978743, 98796.54152153958, 1044.7787541999153, 73025.08559237358, 59283.58394380084, 89974.96038714302, 42677.30811572951, 10231.633604418244, 17281.52170721766, 10051.321279583635, 64653.90916736674]} +{"id": 2012, "vector": [53266.29823915662, 67430.21405428334, 88944.49256944803, 72895.6032709078, 83214.39193061643, 54417.37879752888, 44552.427785396256, 42695.02344229331, 4985.4489608830545, 1307.4359695687265, 24856.30266662583, 23981.436781686294, 20754.74323044205, 29834.621942925532, 47897.153211847435, 25520.35424349509, 28447.3865607742, 56752.98256547372, 36032.36834172461, 15619.368426125302, 86319.38736010517, 60789.861775936006, 90083.70307264065, 45222.83159868762, 84292.05840308257, 52377.85546468143, 49855.53144461992, 23967.20391694167, 44591.11507698122, 58428.95476574267, 96452.61320196695, 78306.38614776816, 74220.09726414361, 28925.826109960428, 35575.03757840769, 86859.98897476948, 47821.001630142135, 21242.761788788266, 82700.1294254713, 67205.34612115701, 27920.562454582654, 66823.8619300919, 4758.035674899441, 8315.206670306385, 26554.79639384144, 24157.40289042899, 72980.56079332545, 80143.74390965617, 20996.01508870379, 84494.81196815292, 29148.64093501396, 21792.145603998393, 45043.45117262915, 71284.9739903915, 47958.384652726956, 95401.54324127651, 13736.081717551519, 37932.937663268815, 95229.58142213785, 82236.48304578033, 29968.500394509647, 75246.45760669737, 36978.406641573994, 22867.991816469446, 1557.5153737690362, 11862.10317244809, 52293.78166256779, 72845.89769019604, 36237.47655113986, 5955.768964476082, 64700.16262086797, 63306.563127642956, 29732.18492684181, 90633.53359500045, 39153.58537908308, 9706.40522212478, 96591.00364793098, 66312.44429750206, 74609.73455047909, 47182.99985608862, 35097.2864035976, 54802.13947165056, 69003.3909011115, 377.92413458389575, 69868.19667115368, 57210.769453026565, 81068.85316450591, 50629.01125581693, 59788.94059924812, 35254.21847943094, 13324.351817425006, 76586.47347592858, 82615.2225819626, 48997.843907460214, 23103.677099282926, 49427.9837884052, 88018.19479569043, 77626.6274635571, 40820.56595980216, 84220.62490732572, 469.157840311063, 11299.84794042479, 98788.1824360063, 87097.73411250947, 2652.928168251634, 75454.37816917653, 4725.571157751962, 14449.447511097991, 91562.45111808735, 35359.71564456736, 56861.8731279958, 72303.99568116598, 18597.72617957105, 30326.956664667305, 79717.5603846221, 2648.0741214607083, 93107.29447334763, 33583.791860852296, 70192.46066208882, 86632.91460474065, 28538.77253791567, 24319.826206831363, 79860.00813396407, 86203.0050527705, 49351.416397076595, 77771.23350148101, 79082.79904444366, 10892.408514458408]} +{"id": 1735, "vector": [82658.56496478384, 49508.88798450339, 97808.95537877004, 38584.572767737634, 65567.67828762588, 15444.789305447837, 83396.59911639593, 23946.625178462255, 28806.006383295236, 83883.05155571041, 8888.866578503319, 56936.780095989525, 65169.83495796431, 25522.553767756228, 78643.54589259894, 92355.00577805792, 44450.878289842476, 39454.87643032168, 97722.5707731191, 24719.837839192925, 41498.32019013536, 80415.24672004182, 73878.05770876762, 25379.020120057517, 55202.80779906575, 86104.64732615366, 66790.20308192521, 5516.081456274435, 41651.70517860339, 43472.200167254596, 35488.120059917295, 93325.08823324557, 87931.75467919787, 97670.54341856754, 66886.43093295932, 34580.63653793316, 46253.353611268656, 22730.96305814991, 13832.614677747923, 47311.93699565514, 5528.973693958228, 7836.569399714266, 37131.02570692352, 31011.154258398165, 28039.740387058955, 12345.12301088896, 63467.46319639598, 14442.817401398866, 60042.1785917855, 55823.46324423789, 96695.66684463674, 58261.4276014288, 76321.55279524934, 64276.124067796045, 46663.099891912454, 80332.20306728309, 75422.56576128113, 18049.890603876116, 64458.323284502694, 55338.77613763481, 88602.7109362546, 69538.88995016321, 93425.20826395751, 90615.75660311375, 34246.05027731147, 12041.706387238904, 64403.32373961456, 26198.931132840018, 7408.274396318259, 47243.01170350717, 13467.410488876441, 35174.03615433146, 81136.5068391849, 39029.60439865364, 51252.207526404134, 66167.19803685525, 43727.18642081527, 49450.59889350366, 67646.41018229305, 92724.08245358433, 22882.694409926753, 16171.961176962535, 60487.30503613246, 33635.349772748545, 46497.32175039784, 77363.75215259388, 25061.484549392222, 18140.59936490052, 93089.76848924746, 9405.436461088679, 92713.16841650377, 11682.963367811904, 5651.401773236275, 92336.13404702852, 22967.422041730024, 98146.32483931116, 16797.80000787213, 51789.298091353034, 4592.061427680316, 22716.539696948246, 1749.9460636172582, 62934.86271622174, 54179.8441013833, 60581.79445027867, 39062.22098298715, 11304.848268979506, 66895.92947005384, 64128.1844423971, 90976.56375204485, 54665.81322701544, 66812.20609970698, 71952.78916486062, 56012.31600353768, 63961.508023906455, 67737.82541411351, 1453.3694337088398, 13695.707591055805, 93291.02937949225, 86905.66514934924, 38252.758617307736, 61528.99036541864, 29401.08023356638, 22290.803770181654, 36740.31830432448, 36681.88584974565, 59728.720266391414, 94804.6791654143, 9644.102361126306]} +{"id": 461, "vector": [48455.63627419506, 3022.918340422198, 38419.02304537721, 21308.885854904038, 72350.57800875022, 49610.76622844718, 18725.923783724684, 24193.057149903085, 98381.54586451681, 14019.13456568068, 52287.00940502312, 23040.466268590586, 68395.33151589421, 10891.504754221793, 50134.90732480924, 4091.3928441351886, 954.8065105262738, 60965.437319809746, 24032.06440954635, 55499.61171315426, 81697.04276199818, 37845.93212463164, 47470.46337406656, 92922.6074432963, 10868.568273646917, 95492.6629085537, 92403.73936560593, 72341.71253793665, 30533.28822964504, 19415.38635808614, 27731.906571405896, 41319.85926608114, 42963.688911414596, 79939.66737449348, 34526.74258867247, 25833.562109051465, 72622.32694591017, 58804.35075207327, 85826.7173508938, 92336.49740470859, 32987.795850603754, 4263.997823187715, 44355.77231037803, 71973.04939333128, 79085.44057617984, 8055.490678522759, 56655.23077179545, 25363.840341093448, 8027.296444976084, 25789.46632937076, 69150.46486379052, 86107.38921690302, 26467.901874707233, 31578.093495338322, 6946.943489764412, 46461.302972158635, 33275.99331304676, 34769.34214924893, 34646.91353320891, 40406.55187252072, 2798.7362223676814, 96335.26636596669, 37979.66332936967, 5427.014812816499, 32994.83760629864, 79987.07127387085, 64065.14290307984, 35597.694284380545, 26899.120211225734, 27485.1621075032, 13189.030526945045, 22500.263942510035, 22283.663645516215, 26448.940024533884, 6652.441308014701, 23479.978612592546, 99393.88818770331, 35129.929128323434, 9659.868903541292, 6441.532565434593, 9556.58494427889, 26613.808594246402, 53551.5493709477, 75915.95248775954, 77260.86989975219, 52459.36695422914, 45926.31329790068, 60399.14429693905, 72946.8057333558, 26323.648851759095, 49264.15710863099, 71941.61069584233, 59860.7909604163, 26367.769815497453, 26041.18233949332, 47963.55048179936, 17853.668143204326, 61970.07343507837, 90774.30679925275, 90709.88470542051, 99755.62633573345, 20678.458538781706, 45104.838695367245, 26852.171178008633, 45474.79883177279, 96180.1704354458, 20300.34036342473, 62428.5702750746, 86853.63362323974, 59172.65388939432, 98238.29439709081, 88227.5631471327, 37384.36193003205, 55945.037572008696, 57621.90799057966, 44637.53191044107, 33559.04981167243, 86017.26203507451, 55585.069519335564, 17791.523674305066, 9922.288809009284, 27173.85363388738, 80387.38915235919, 19804.648448571315, 67394.60281192968, 12244.949680839323, 70269.3859518562, 3268.163876754837]} +{"id": 566, "vector": [11596.88124441569, 58604.4316609148, 43036.59189699744, 57257.412435743194, 62493.368456697026, 60602.91212928922, 69561.33741116563, 29160.458540183034, 72253.0411843612, 7594.42313988774, 19410.68988941712, 1233.317220454111, 60754.412395524756, 2329.5019691516263, 37414.52576373624, 74478.18673111242, 58740.5077970018, 42694.93200500999, 55077.649685881144, 16663.352374658014, 46444.809460100914, 50541.35132796616, 79732.35459659698, 43345.42521914408, 65004.06976245703, 71084.49228282108, 77139.21833996275, 84345.92082599206, 52082.07359558921, 63668.60572991212, 90000.94912690143, 43781.93757776383, 16561.8404465657, 91530.37442737728, 19059.450744151975, 60543.81160017017, 69621.1488647281, 69242.44275833234, 73314.17951112473, 37695.047538918305, 52700.43712781048, 44923.48788486166, 94557.95825941862, 63896.90103233809, 72174.6317906847, 41205.38779167997, 38675.62935588087, 25082.857490800758, 70512.28955345174, 53235.794008768426, 75835.04000460824, 24517.488356125883, 89447.39439000963, 55.6296457908223, 49425.323121677015, 898.6088428008077, 20563.54909420941, 97443.43609701726, 70945.04125447712, 54999.99902976509, 7081.911544751829, 74082.43401245764, 60806.29280180495, 96472.938286845, 86135.60473489686, 22307.45039904738, 44877.412584045436, 50894.28779063236, 63953.81439604011, 39511.012234093214, 47162.89954860358, 64238.7200235406, 98178.41198938093, 89006.7040152996, 57845.34087018643, 91362.26349876178, 62480.1954423988, 69359.12112106838, 12957.368650496404, 32002.82645039112, 6961.418678768528, 74685.45095302093, 5970.075701571232, 12945.928820473473, 85419.60739071069, 64069.87430798978, 36739.10862197313, 93464.76712557254, 79615.99377314643, 87960.3515554349, 89593.1016189603, 118.52051995119295, 92466.62830460671, 31089.26574349965, 88300.18095194388, 99520.3205841668, 75574.67492784567, 37080.82230101027, 96369.79238775632, 75084.00389900552, 8327.406194470566, 21800.631574025563, 1292.320067950925, 76133.85006668766, 47262.62427825285, 24481.945975925122, 20901.016401043005, 33554.74335959442, 32132.334659882534, 59821.37812347045, 8697.32536132839, 31795.427889711314, 8371.804641871016, 15379.68809900443, 28079.965265830087, 1161.1146070193424, 12540.759832481985, 42162.337004916924, 73013.08248577749, 9303.894632543941, 38398.32007386408, 29584.30865468239, 15792.956060381435, 10421.817027891644, 27809.546049841018, 87010.90040842684, 99402.33841808044, 89984.87654185711]} +{"id": 196, "vector": [7486.076259742369, 59730.10866165234, 14498.395502726713, 58272.580408999405, 14740.183758969939, 94339.2504881926, 6714.993173237249, 66881.1647387293, 51049.15061739402, 70530.86311831472, 86414.20516147828, 49114.73963377908, 52858.19300249028, 3164.980092104619, 69581.65009704488, 72444.28444845967, 51596.91137274766, 80805.1826222708, 54334.70823234202, 32480.331618048396, 24693.554153555575, 88415.89338368284, 35940.41774047362, 269.4920298730308, 32738.23108645607, 93792.6446375939, 5253.0490958121345, 58866.13292200746, 35057.99477072155, 883.7212865322352, 45220.25714476407, 64033.13274243948, 40024.53634969607, 28241.040047674505, 89092.61398101258, 39533.20935737914, 19567.91215634881, 16261.023370680028, 88546.14221674093, 52361.28329872455, 12141.277746056523, 83420.05029891082, 79544.84514679747, 97318.26617233582, 41778.58165255185, 67616.53342381497, 47041.75533387804, 18102.101821384487, 31044.130860488607, 80741.73164844414, 86846.57272850687, 11554.17331451477, 16822.59772308131, 25317.58488393171, 43147.201484948, 15947.250736609652, 98926.53103298703, 43495.52013547757, 50002.86516773147, 60770.45196377295, 16488.176116245657, 30442.869682357577, 65690.61058439006, 37539.501301124226, 31701.10246671146, 66250.39772040889, 44381.71288834412, 61432.15427932812, 23549.83230124674, 99029.05323966865, 15176.060275384873, 37674.834441168394, 15754.27726575972, 75510.89272066967, 66665.57125858351, 52534.70969655907, 3849.6986554791056, 41708.62784446908, 11223.865635053731, 58225.47848257905, 10718.980085107833, 46987.986784007, 3957.8640609523986, 18853.731811034835, 14272.341339366702, 14288.217990925079, 7581.716239564873, 78267.36450414528, 90207.89038626257, 10561.28360365015, 92643.52840528675, 44751.17401058991, 41177.874639527035, 31234.05728553964, 51762.0462258319, 40079.35872927552, 94339.98748709551, 6742.252559187412, 70269.16113101694, 56384.45815822387, 74742.08270478272, 69974.960502343, 32260.812038502594, 78197.48752502633, 9218.295704797463, 38593.65788414857, 7102.386021556539, 50503.23708336965, 72799.69526918598, 40568.85235716411, 26345.023155817158, 25042.46439658998, 89178.26456878214, 16638.6934051316, 66152.05677372168, 59300.04498572722, 56678.82950412707, 77564.34190226844, 38290.560615592796, 62669.68618321271, 74201.87541415627, 28022.24950513906, 59162.408353560546, 8183.840957520682, 17823.17787976213, 59750.17284395325, 39718.25332724532, 51448.824645970446]} +{"id": 188, "vector": [99574.9768052108, 97303.91986065244, 93639.137747502, 92883.43573487662, 67761.24186217126, 55373.24592527363, 58968.28284236737, 74724.65478060202, 21590.84439496419, 32529.552624282398, 71309.63793133266, 43605.725067680345, 72209.61971280168, 47784.64604707847, 94556.12265037071, 37191.148631679716, 51187.98761022883, 7086.062429059925, 19716.506823609958, 3875.2479334980603, 73185.99693007754, 73762.6908416201, 16485.084012646712, 34014.979619095495, 96412.11304573031, 20728.280699797364, 98042.39871559321, 60888.37972036897, 44079.66307273108, 24115.758759719552, 50044.33141680724, 39822.44337196903, 98988.11428347931, 47618.133684662665, 87670.50089130903, 87059.29390558899, 52481.98607201544, 64090.679900875904, 97917.2002003943, 40201.86059947418, 34886.71791353901, 92350.65886178536, 77578.90895802886, 82921.8227699599, 61777.53363294265, 14725.034256805713, 55694.223886831765, 99891.13120859589, 15344.995548045727, 24202.230230766465, 25515.762162143918, 36195.53485607724, 23484.783160865452, 82685.15879107936, 2045.5218143692, 30439.373143305747, 69947.39521787211, 40387.5806878917, 88110.41425006732, 21932.581735746004, 59690.95265886876, 3644.641695105755, 79642.75124558153, 52645.01844319063, 35483.7488016162, 34786.3694582638, 12733.686889057972, 45772.555321820444, 42791.2166712357, 37824.727986530226, 35849.6598513656, 79364.04312011927, 24985.53499001108, 81063.62938756388, 21451.25107490975, 72943.26830596775, 63530.37842973307, 45777.70686139455, 87122.48036213158, 86380.71000562624, 81432.74668159481, 58047.994344322404, 73225.32149108914, 6907.666455276296, 58528.57188445672, 7755.68722285408, 52229.09716640858, 50234.807779270144, 20646.655513770875, 6490.347041331656, 94054.3664624159, 11137.769299298317, 72543.97460520462, 14144.444064376316, 20420.164158662978, 70850.99035110894, 51519.332482911996, 2005.018557202598, 15673.996459854556, 94950.94937912881, 7256.478663692312, 50199.56946884269, 31885.53315637932, 81207.39781716726, 59079.58222419937, 60065.14589483355, 47016.320872041804, 60019.650112249336, 72546.48810620465, 74195.155797621, 15436.358785328008, 95830.792381495, 54996.53736423474, 3956.1179369055453, 30811.368289550766, 37851.641168976355, 98601.17484298916, 58991.84705713557, 48014.6419467675, 52461.67540542233, 73066.78804778232, 36527.892508302764, 31869.933906958526, 53917.450339070325, 41231.93672532223, 52648.4901151728, 53357.895396791064, 86870.42725029356]} +{"id": 1354, "vector": [7218.44969408516, 45159.64137611622, 79449.17415527834, 50170.16534109379, 86285.19469006502, 26777.498151936485, 25280.336442583863, 39643.83512173315, 99860.45246503214, 36091.237837461784, 54917.768518537356, 1650.4061946255865, 65219.621609772614, 77486.04307569818, 58310.34867028271, 24420.030944907423, 48296.01340408238, 49391.90505751416, 15440.540624434429, 59829.849080210515, 5178.30803818774, 8134.390013362314, 76208.96353252523, 53502.168701987306, 12889.211779258636, 76615.20650958143, 56363.40382130666, 6172.641397507028, 51913.89242775012, 10702.974246127194, 45700.76371018402, 11116.350035784384, 521.6260113209969, 81083.64675751077, 69662.89258910826, 61422.73137402793, 73160.9687723313, 68544.80182445336, 54172.16871717868, 57788.61419828261, 90548.60889362133, 87253.04169165526, 17735.18926852179, 32066.88367985696, 24125.13404289338, 350.3595984197583, 23519.099888864548, 32386.30393417162, 94482.3335898631, 71665.97475641839, 69441.71086229938, 34182.45584277552, 72494.0494202262, 5201.783561759865, 72832.81943332586, 88741.41467151727, 78050.76205688628, 30942.50106466331, 17891.29521384182, 32220.59908482825, 88646.96830591698, 27356.415474548357, 97783.65386599854, 19732.8089259297, 72764.40354005716, 98671.99492594211, 38943.34460787069, 19365.059734689337, 60816.407455487744, 82435.7461979648, 39223.26963918713, 7565.689293602929, 3050.395726468125, 43533.43629065762, 31865.795037091993, 44330.49355812432, 19970.181778209495, 63313.329297495744, 21183.630039937107, 99078.68892228579, 29168.83160267004, 4110.677435208509, 12755.173634569828, 94151.11912290502, 7593.88170095262, 64746.7165301733, 88980.56568004584, 50366.07867170007, 92236.68132693542, 8194.62709431209, 3074.0303165938785, 59127.30905343434, 62936.01073953592, 18099.620095124414, 35626.48897261447, 8325.378217098578, 99339.49600608932, 83768.5374211996, 56665.196537605414, 95762.03849673441, 13453.871659352923, 97947.777359366, 20468.050679470718, 26534.797383079844, 71710.87607181793, 5660.919799017849, 90064.39473562667, 32181.75451086439, 24007.69420735003, 41169.05612147055, 97516.7519238451, 66566.72236448884, 94779.67517653722, 46441.96623887108, 59356.7051479894, 78624.63377827132, 68606.0742333035, 82437.5423360513, 39067.75114130233, 30338.70748637655, 58098.46066554412, 48854.6029997795, 30411.414942571057, 75212.19447449679, 88758.84856059645, 30163.123604416644, 65257.38376454779, 62357.72391337029]} +{"id": 182, "vector": [90535.94986013269, 93686.6600360115, 13031.03572169254, 70305.95515837413, 93998.04004821314, 58101.49298677992, 18168.45012869869, 59117.37709959618, 40545.39306978567, 52357.00156032861, 66663.73724403774, 68129.6303223842, 38678.85414131566, 90829.56485575765, 67383.48150168883, 58595.57981724869, 73307.91262295454, 29149.446551663692, 32829.79499243195, 95004.25482724879, 65543.81977410697, 8002.472520365301, 42811.42122250573, 88091.11013124966, 12450.697112797172, 37993.94118478863, 29812.929403714737, 45306.82053562099, 17175.18269025463, 69219.88845117217, 83001.74997160764, 62727.23875467665, 83406.44314040749, 18751.595736744188, 25157.185778283296, 43112.42373748059, 75341.30438183881, 55879.90459118274, 69224.55369677836, 46708.457267175494, 41610.46444111383, 36351.31955209858, 20480.44329409695, 59604.563055413586, 99308.93616099856, 94558.56997391739, 11450.725186559297, 3050.442469819803, 79990.79099983515, 86598.81259941007, 39003.08532340311, 69382.7414769849, 65684.47854211248, 87839.58053918173, 35668.891889098166, 55999.57503073189, 92105.9977617917, 33477.9682565678, 39174.16594910399, 65149.594127497076, 33439.49896603885, 14363.6762761576, 51756.604234260114, 47339.26058878926, 55630.32269377715, 17034.08860845901, 92905.37948931046, 25558.02953192907, 56700.543626522434, 35544.33527285243, 22276.163950487848, 69875.3035979573, 85878.4815342405, 85897.46023315395, 87865.06144100851, 9358.370896803302, 57795.864746174295, 62486.99215294956, 41351.43956165863, 64153.57947210013, 28403.644599898882, 48691.05045609028, 58596.44103088026, 61419.12654026114, 22010.58557585963, 36996.82010790426, 66892.8650275325, 53745.374375539046, 17343.79360890229, 15201.763252299028, 47637.026177431195, 23487.78977238364, 28868.72630775269, 37235.41805830898, 8351.569386181134, 37622.93298955428, 13971.511672075532, 57374.94382876505, 34596.47762051852, 19272.849458991892, 56632.967762301436, 3764.696254040756, 85658.88141209513, 75141.36016120005, 26289.034763660435, 45104.66511896259, 77074.18972312317, 2060.4264238898163, 60270.36006832718, 53875.491215857415, 81475.61683550995, 49389.18178488589, 75280.80722410322, 83591.46846684856, 27959.278273276777, 64722.10012763486, 93309.28976410521, 9361.616777663461, 17735.372247929226, 47844.399078767405, 26799.253184239657, 98484.42075769149, 60429.27901990231, 78233.65696166435, 36423.560855439915, 44049.498065570755, 71256.47645401498, 80054.2520372678]} +{"id": 829, "vector": [89125.53130447157, 79985.39953798296, 52962.970177939016, 27744.86138038056, 42176.69005347309, 23691.395056526977, 78531.49457434738, 9086.573282314392, 66665.9857912967, 37261.277745678875, 27729.440374468562, 18594.57343822597, 78865.23078154991, 2994.758756631466, 25991.77200029307, 51972.77638965985, 16417.853611600363, 92592.46469354795, 7151.125854473084, 91690.61598626805, 97456.55679676245, 38702.99686174119, 3322.931611617397, 90661.27598780036, 56871.70072860289, 81866.45900516574, 62474.18022911611, 20307.472569228114, 5274.151250191483, 23324.14704557789, 42159.65298490501, 32989.877185863326, 28049.018135034297, 22404.26881681954, 20972.424395272847, 35435.23783105839, 20043.644621852753, 24438.747387715997, 52576.5234766183, 55138.09651443354, 42099.473343359474, 28684.106165995327, 85712.47074643531, 21783.61225491956, 92490.88061030758, 62019.20978121958, 58166.43987414644, 30725.63701491965, 1767.5588288860733, 6237.80987754563, 43974.3101608828, 69110.2088334122, 70996.7076747733, 1887.1728528856102, 55918.417762950434, 66591.39533101559, 96876.50756400659, 47018.97848018545, 97231.52026166837, 34383.79861014412, 93404.39170888181, 47060.99380693899, 71073.36183326057, 64306.256273000894, 52183.71262100834, 36424.002730788154, 84908.48679148992, 12834.382319651739, 8553.069717351025, 60999.46725503473, 86582.67946930471, 17340.340899106854, 96570.90447190822, 89971.15615829319, 89799.63880684818, 73511.61616607492, 34182.407796161875, 65854.31612625504, 78342.39025736503, 66289.07513796739, 9683.293683536753, 77151.68510224589, 62230.91760679613, 64174.82052453464, 41797.21957717908, 70004.90201303127, 28370.29787699299, 72492.17750535755, 48551.86074551483, 43790.44379161999, 91084.9274541588, 5478.763921897734, 58998.983426096805, 61026.96428657449, 51677.09329191419, 17843.584965371818, 34877.26761584865, 21692.904019138616, 6216.181467845417, 51020.07047748614, 45937.328342696484, 86314.06280280657, 56922.90854081656, 57448.440429213624, 28679.292961576597, 41048.747211652495, 61789.680487865495, 6713.670581764275, 95022.93393423119, 82131.01229194022, 55602.98279055275, 1961.911477207512, 40989.84072993415, 40711.20478653446, 61856.619392160326, 20662.280648656473, 12374.751531758166, 55743.15644659445, 86031.55776861419, 55516.677268249856, 91143.29257138123, 54192.77830480322, 79553.72560499546, 72241.36436396153, 51077.74570842766, 91667.7315194278, 67223.98747632977, 23815.176243153]} +{"id": 495, "vector": [86965.07397312914, 22573.50655608206, 96755.93505114084, 55288.52507676094, 60598.96414890415, 58108.321834535425, 36501.074391657894, 86399.80440513055, 19978.31241782695, 32419.61835097478, 69823.16272141977, 13459.617704880146, 37083.75674716312, 7110.457569491435, 22127.731504814597, 86495.31757980425, 90870.6568300283, 75922.09819706394, 95938.52379540062, 15589.722540104078, 27919.69808975433, 68003.5545203444, 56169.09442706656, 86866.86856814467, 93931.7056745447, 25192.825731432, 30141.457412042073, 30705.473650466185, 70761.0964271256, 14585.328270255783, 70305.57666554443, 86045.52425149069, 74928.1299103136, 26658.458480271995, 57933.666101058436, 15451.045556183673, 97535.8306988159, 88225.18695642208, 48800.6064325368, 33251.06268472613, 34485.31469621236, 85661.20377842362, 11694.996618028963, 22417.545524498528, 77075.19838256798, 97037.89205651115, 44942.387124523695, 57466.82692236654, 29934.16016113739, 69517.60510864585, 67109.16294093248, 18976.456810177344, 7285.396056639248, 21521.137696609738, 63019.57383454465, 36145.343828465615, 66431.76168623581, 77197.05189322364, 59401.47250445272, 75129.187615149, 64485.25478006646, 7666.547357938669, 73851.33340097252, 76309.51834345066, 79903.37534189744, 98169.51621356684, 89127.09673626933, 15323.868155141774, 9594.333303157366, 25002.78276217671, 64628.01148213997, 41886.46495362341, 78723.68348770046, 66060.0251131142, 91189.86081775579, 46886.96529785196, 73498.59479808493, 6858.210102092244, 97659.21070389202, 82627.86799085104, 93518.04778926299, 68695.41178698688, 93974.71134513775, 77120.08326803446, 35485.58893499552, 32596.47172909319, 70207.49965462637, 20906.50178184098, 74256.3391219777, 38902.980954428036, 96306.31560507797, 42626.070605591936, 24326.74761389577, 97593.63497398015, 45778.26028245547, 28581.175820266613, 11682.6121732567, 77536.63120710652, 95667.87872444552, 51614.43852703653, 91321.7006179771, 44990.03598828625, 8528.90277347358, 90973.90857894935, 15462.57373323554, 48061.63261055838, 54227.11155434449, 62993.42526660756, 81340.97489492704, 64574.85155239947, 37433.639822558296, 85620.9977456014, 95295.38385540621, 90871.54073228792, 51602.17069386166, 11653.190522975454, 44811.6805901913, 29034.82449989446, 4834.079151895842, 26361.334683388915, 8671.345530957853, 18912.07141103972, 19904.57057786209, 48529.53572792762, 76239.95867913179, 89126.32918964257, 25078.236905464448, 132.78110454877367]} +{"id": 1960, "vector": [86750.31564088534, 71829.43014866141, 84109.01419634416, 61005.52278735886, 22958.407266600057, 215.68956614440583, 22209.99651202167, 3052.047680197667, 52325.629737942756, 35636.43863276563, 22711.64101960863, 55347.741962478416, 34713.10841847215, 48263.03487755585, 44175.84469511753, 75929.60649677046, 5207.921520398362, 74515.26626570699, 40218.865140930495, 74203.81579222239, 17933.845493221812, 15802.121367549527, 44134.952552088725, 66675.10112606816, 93591.35856299117, 12214.338033474392, 38821.56342556049, 19724.113496177597, 10370.34310832875, 22767.979281227137, 33567.38612115482, 73944.27615284889, 23460.38488726573, 84819.66099287911, 97378.53137653918, 27053.11826856327, 99155.87642436518, 97155.80932180941, 97868.9329183252, 86781.4693554114, 21096.27656770656, 2053.0508992529726, 60435.030084446094, 17886.87084499505, 81764.78522040756, 82717.87333387203, 89963.95542404673, 96291.82097699, 83466.1628223472, 29918.712242104328, 32018.173681721993, 50552.50695529706, 96371.62689396211, 66907.84042750999, 69779.28658311232, 28043.053826234885, 51449.69101984598, 57250.09342911991, 66651.52705200382, 50596.74288870146, 2895.569475216464, 25323.089615143825, 90692.49531814436, 23373.135705162375, 54619.17802526632, 98696.00234771933, 27473.71901216633, 13020.517297776623, 33156.98350488494, 47370.04117250141, 99675.19084866026, 76763.35076343149, 65345.697856932886, 70771.6364787954, 95253.62032617694, 99453.26351001444, 62735.20399326152, 29679.351957435563, 92439.77018989003, 81695.18887337453, 945.4220891610521, 77040.41063953438, 19172.419381058782, 42679.850797344545, 96736.03205301495, 90253.03201160977, 42138.65697886441, 30769.551452297062, 85478.72354321134, 47659.64443133951, 47341.53887335459, 77321.57146552838, 51110.35587441861, 198.59314275557648, 24614.506879815202, 76565.06244892212, 45779.91687073617, 33954.54709708864, 25682.473604243572, 36227.52430279622, 70302.7363984674, 92276.92202949866, 94476.92093715646, 44190.27423703279, 99207.58962214881, 23853.834377679315, 61589.24675433709, 76068.06895033778, 43167.37950932676, 51412.16234651471, 41567.12007282216, 10654.458732906336, 69540.40475134569, 38610.55526515034, 44518.382056664326, 34352.75090123482, 31531.295905852396, 5676.768504285723, 4849.733212964346, 72362.11456413937, 98906.98260750774, 46522.84155645195, 86490.56902016688, 58810.09156070528, 70395.14175147076, 37413.52231437761, 92447.14071663693, 55142.11977006629]} +{"id": 1264, "vector": [25539.771744324757, 75146.68878830469, 31982.295882465995, 90951.55601591725, 49289.821153976976, 94246.25570804771, 498.7756611618876, 98596.59836176778, 15075.78195996393, 43266.88990523777, 13852.299952344716, 79834.31031943034, 28430.562162216633, 66035.93330185908, 37083.53534124645, 39693.56224947026, 88902.62742933296, 38176.24881716927, 26197.550501543155, 46019.39551262525, 44938.30047462196, 28627.82679763144, 6617.232950479057, 25813.608675546508, 24086.18208105936, 20267.13644146657, 60954.04852395109, 89593.27150816427, 86734.64372517925, 37328.91468069274, 95735.29209457705, 28043.15866339452, 40182.27281506862, 71066.91504697254, 41703.73900249382, 18303.940991773958, 86110.45970218125, 59264.71638199179, 55946.720342591085, 35861.54045852053, 44935.27973650845, 19731.03031503548, 62958.693587755806, 14270.201963793383, 57282.76291535067, 88510.39073153763, 5852.900089185564, 79920.87425298491, 73405.22509786545, 52551.06863611223, 93085.43489205618, 3005.9786983845706, 34658.063167409884, 22213.585299253326, 27837.39277000775, 93741.07683204461, 67283.04294151593, 81768.93706848573, 58376.60402570418, 48160.104440237104, 27157.877600781678, 10796.761229666774, 30874.21664719203, 67747.45684167476, 63444.310183127585, 36648.14473348502, 8994.821959880495, 80041.53126153866, 73808.96548920227, 39430.801615622724, 81505.52793605863, 58705.950176223545, 61370.569666405194, 54260.78513945093, 45352.76756179608, 86012.7747703237, 82044.52647738453, 27940.945729397637, 54921.902162582206, 26272.948471709777, 90748.92887222738, 15206.31840551867, 38618.21033831541, 80215.19396994008, 58812.031912997845, 5953.158053205587, 83699.5979667007, 72654.60951285344, 51124.70466927902, 31510.009512718596, 88850.68139224031, 35914.04126501333, 76302.60080477936, 5444.851216587687, 62458.33651191142, 39629.787424029964, 71077.03037979317, 4114.564691602917, 95293.68660958973, 38378.80991089604, 22197.07892974032, 46742.28654208272, 24369.49947969491, 73551.06782844347, 34018.01170317289, 17478.621212077218, 23688.471080112173, 26912.02709254652, 67995.4474554972, 5598.579184749164, 7673.930998933043, 82693.66020085073, 78253.32442534628, 73762.00127590004, 7949.691047957263, 6153.396089533192, 83380.2733865499, 81994.26979582294, 54370.84451476471, 74441.32095023777, 81231.08209115264, 25885.346049279866, 47754.30505034503, 61334.81365182127, 79149.96510508898, 16027.487298619957, 76492.34115348126, 14667.589692228888]} +{"id": 752, "vector": [21073.439468689416, 69930.30350741612, 29284.337268014184, 37442.46200592809, 78877.27309737189, 11278.492471774649, 3955.241970508372, 44634.675993917685, 12109.921910156052, 68086.22094730075, 60245.18043094201, 44167.33562323458, 10178.133792511735, 65003.57865909942, 71917.79678382588, 4308.6813282676385, 50971.526895985146, 30645.657422459983, 35606.24937920494, 38002.761163431234, 44544.0210289696, 46829.15995371354, 5531.336679874965, 3931.8359989779506, 36582.27487987622, 26011.591947233457, 3201.4493762638986, 16840.22778722599, 32465.714755371668, 27190.822054371056, 31102.709396009377, 28935.007979329042, 28223.281283119384, 73134.90039305744, 95491.41894283402, 23498.501014318983, 91887.44688626227, 91511.79227316457, 65215.14267349331, 70737.12096087215, 89137.07490776475, 14323.117157532772, 32550.8525779599, 96827.4186204387, 49078.11222148416, 9426.256670440225, 15564.8449608654, 67872.78236547168, 50817.53411612158, 65451.218864116556, 14668.362001982992, 8356.891723527682, 90170.88180029887, 68206.85468940728, 85970.02136342438, 78599.07042603605, 56957.02251602113, 62270.357446263195, 82694.76750603532, 52951.81601388614, 2828.8451134722113, 47565.47549941371, 9715.149595250738, 10267.564858233014, 53371.85091738465, 89973.1394441722, 35148.11156802272, 29589.687834405544, 31250.330682670836, 6890.306248472178, 2105.9730422050784, 36597.4059895206, 27915.664024874688, 86328.00316237162, 68413.94288383053, 74697.68937583503, 25443.1133254729, 30051.088193116404, 91113.0178573689, 51773.16194975755, 30394.517583528268, 80334.50952667963, 6234.331191226572, 46641.10533406887, 47771.57636901736, 75758.3387411727, 22432.450409445202, 30481.181704311122, 93681.00948118535, 43577.231053345946, 14263.622993820025, 32884.839423822385, 2334.7217183700363, 81340.56746712184, 67988.79016965926, 43605.523342070075, 37827.54155958033, 60808.85015655734, 6956.4985882946685, 55045.75043169757, 68907.5411886202, 87766.00042984328, 69117.73783757842, 99973.97221079831, 44384.35352451865, 49886.473545085464, 49330.88368174704, 48928.38039432754, 85333.63778434564, 98707.8094940157, 73985.5803737662, 28757.65177609114, 51015.13148555006, 88504.85655279805, 9011.33944546232, 74257.71567358279, 60145.74191960702, 17012.845516339083, 81993.96575913519, 10520.619764884786, 18659.094959849564, 66170.33590238, 37282.51783360711, 46425.03727821984, 37958.85699857357, 36582.68311238161, 32363.185834475673, 22697.41944634074]} +{"id": 925, "vector": [78402.8217901924, 9922.483242785485, 91562.54373595589, 48951.41637850975, 53713.49387894778, 13414.03518300508, 1561.915121463542, 4084.3567065897537, 63551.27487778426, 82765.06841007338, 36081.76793036531, 3257.101578359012, 62671.15239312453, 99579.99868514757, 84599.14217975234, 63298.09613995549, 89368.88033773992, 46425.70284250352, 44152.76695589061, 48316.71926148604, 58869.27729774155, 72478.526812123, 55081.59163385745, 41763.08506665217, 89999.51058469179, 48232.91439528704, 1653.7917267288037, 88670.68025451989, 78979.46202631555, 14040.36371117957, 14787.598836692028, 31505.120221047433, 16312.036213472069, 94399.37936093865, 9300.734487826678, 30980.18543546036, 44942.25183861949, 12486.944170453351, 393.1700157651319, 13271.999903770271, 80123.75135021197, 2483.7316453738767, 43493.73158303178, 25427.31734592344, 77651.14748937394, 71648.31505872935, 39109.14110550994, 70964.26714818394, 68955.19372216894, 35625.607875373564, 43139.18707503451, 59279.5392659599, 12445.307194714505, 5665.928644426111, 50283.58717065624, 79512.29540154841, 4268.523793342593, 83765.49516705367, 73508.22747712003, 41688.546805454906, 35521.23207375705, 14250.695764488275, 84112.58211713047, 9276.717076616847, 93695.67150637628, 59164.23524774243, 12698.577488238672, 24558.47258889412, 66491.55833363874, 74377.45543482726, 80855.65664552445, 15467.778660017795, 42484.50849402138, 68852.4575565995, 40939.25902550809, 44764.6136619117, 45524.062848226866, 72075.39638585613, 41073.10625547682, 11705.207627427084, 93709.97472267303, 49945.96735396688, 94655.14896452261, 63224.93481062917, 63883.24541494945, 84485.96857364119, 69966.5132305618, 47043.46943818144, 94521.29381664025, 6552.947245404827, 64466.44313376257, 13085.486559611336, 72592.95985365212, 15648.221337682078, 34544.72056178105, 78524.37080028481, 15488.148045365913, 48343.174293061405, 91997.3688316404, 74034.49010225714, 67264.4660144334, 62614.96814734244, 82969.23175754306, 38869.48283361018, 76729.5271133144, 86969.3419787947, 22870.39049648851, 42434.42662921431, 76429.2283093767, 19410.12712403062, 15193.147513063943, 22757.384360467626, 22335.720547354078, 51705.91372402307, 69722.6471849829, 97138.53051536881, 87704.38452911982, 82066.73310012376, 18345.295434851872, 68330.45719810265, 56659.22935803823, 4039.1047844658633, 14735.353838185949, 66030.05745881738, 19282.96683700528, 34012.056265024745, 91493.9750975555, 91292.79665916231]} +{"id": 1695, "vector": [78763.14455140058, 61729.994151041064, 73493.62431509778, 66457.10441592716, 17096.655161432984, 85059.26821251259, 64529.80130785546, 66122.20084879744, 49079.17754216535, 87724.79505999477, 54369.7983410606, 66832.99577903832, 76268.04245903554, 16238.110631068625, 11363.305461590478, 75489.15357715773, 75476.51515902519, 75226.86348353302, 55023.095265696575, 75802.78964271028, 85362.69528770483, 70253.86890791953, 35593.905746592915, 99609.73377180335, 97553.89289049146, 76239.0145788069, 12178.636334134151, 66396.03738393568, 16628.81451993551, 5449.571575797751, 93153.7699170525, 90976.18528077664, 36660.524702895025, 63514.16564833197, 45661.41205278078, 71704.46274559232, 57191.26947501353, 95929.56621731046, 77894.36545216867, 99709.2212786166, 4879.862128493962, 19013.942195519616, 57094.09538223666, 90002.40885494367, 79191.85237638533, 19694.79682758264, 49250.59876818002, 54665.405052598806, 58943.40050217163, 34737.23838225834, 959.822078398187, 19137.42569945538, 89691.5237913098, 65428.03916785954, 39397.56572451255, 3102.9135525077577, 2651.302018238977, 81602.0098029013, 81849.97537089862, 54221.08645770153, 23945.958586211735, 45585.99130156119, 31362.93687901601, 82094.29566026818, 86756.96139645332, 77073.50752134262, 32984.33821443897, 96236.32797566515, 94959.60210439203, 11769.509279844548, 69780.11844758311, 31122.482059383226, 57353.72014193102, 82490.59676640625, 81308.61654402033, 20176.476653768583, 858.2046207675997, 29910.86562087081, 73460.68678881014, 88040.93620785451, 14804.142701766697, 68645.2242753195, 98031.17644096901, 84895.52702405583, 99490.55444772617, 32071.818776586282, 38508.69994548529, 73117.77277861565, 46048.997435893856, 22599.28595569767, 80853.86736931218, 77955.86649775035, 67416.80129488379, 17799.084305829838, 11243.268977302712, 26836.852253900244, 42034.80289258549, 71669.49548618183, 49649.717600544885, 74331.11506003352, 38709.75850754759, 5426.540167612326, 45480.05174561058, 47139.65657106656, 53838.629083494096, 87431.85210304063, 19807.671245203863, 97434.02979304249, 38751.97215502939, 67428.11231609533, 94016.60411850591, 12733.02644733787, 25020.00670675737, 34841.44146496836, 37795.71616294841, 1073.3217305958776, 6829.915295518763, 74915.57158420277, 70519.25191413432, 72758.49724600955, 58441.38831396981, 75149.6395666768, 50187.4163927614, 19348.27602431496, 31483.131776689832, 5117.428878691666, 3340.309072258285, 44869.297686072576]} +{"id": 376, "vector": [52983.51439023816, 69748.90648126788, 91615.4221810193, 94895.43249546824, 43781.560287924345, 55476.627326650574, 15236.957564927834, 61574.37717047981, 75346.30785043663, 771.9673319448561, 63005.627514907195, 23045.41508239549, 75946.12518947525, 91987.40018649341, 37849.20359238939, 84432.35083212677, 9651.276995716318, 8171.322085442345, 70184.8385045819, 94822.70582095176, 5476.152095448628, 55060.75028045957, 70868.54133754507, 61237.61468981569, 98107.55183455607, 96031.52724200455, 24552.272577150514, 65228.41099293729, 90030.1993998546, 67066.50346767949, 24046.31271509081, 93019.54649728061, 46709.36400696531, 52523.808410636186, 18620.37727882926, 22038.658049153848, 39806.02928347566, 90133.61060866348, 34342.087342690065, 7239.960103539677, 13264.428228612513, 37547.86571143309, 10319.030933606655, 41661.618036446234, 22949.108308776355, 79427.34683573186, 79573.37755240001, 56356.05107643998, 19832.813329494402, 39299.32103997011, 14194.183066844702, 91723.53991254602, 84263.88174509617, 43589.440122230524, 6833.297437636698, 40562.216013006844, 18769.325515813794, 87846.38942793453, 79197.93509791148, 73632.28427330722, 29881.976039201076, 59396.71549972564, 39740.75461713117, 70755.73633834993, 7845.210739480734, 40060.58983988895, 57146.59795048751, 20058.269731743992, 48079.24018515034, 29386.83375657888, 68842.5368242201, 40863.03748298208, 15466.187032383526, 60992.11796403157, 57481.20602897867, 45411.23734314935, 54600.36155363767, 77021.24198295617, 25145.69995491831, 18569.65743086455, 38605.92102506545, 60734.308245372835, 44283.67805384579, 82910.83240963616, 16739.187942143395, 48337.724558171765, 94582.93014685022, 10998.785777261011, 19437.940057526102, 62517.76612575288, 3961.200484521965, 6119.161338341639, 78648.2644917305, 77944.25206632887, 2732.0703655019483, 90045.73538905033, 82530.41741916623, 3488.534089014861, 53952.37836065921, 67879.39697293336, 3832.949842326394, 86475.8116391073, 9421.060794583502, 32617.655946421542, 1084.262384557022, 78177.44918653024, 4074.659944002268, 55742.85934996335, 73501.75274542018, 58127.31680677832, 89802.27502765729, 88961.39468635996, 41925.90542556484, 32826.93503766743, 19276.237566539556, 31598.06129237933, 17221.201748312164, 86551.67794909487, 39593.42847296636, 69288.34025695211, 80743.208667074, 83332.80876570933, 36156.16247185043, 39759.139536980285, 59677.25142375631, 64248.01308758536, 18654.727703144657, 79116.30249232499]} +{"id": 1243, "vector": [14403.15030391237, 22105.433347351987, 3212.035705178973, 62176.539802949934, 40643.55015247414, 44018.31122614477, 61821.27239935427, 8083.927580864669, 61310.50749611964, 39378.07054614158, 99588.17898390355, 16540.98611924384, 3255.2689582739936, 18034.914793952437, 39029.03305657294, 7208.856569433875, 18411.452857754884, 37217.53695420249, 53929.451820809074, 34705.90033976787, 53185.467872658075, 85514.37977167816, 44867.30798380981, 92858.2398724861, 77013.58184697476, 76431.4526389879, 64994.68188296287, 76340.4957188616, 23739.636897575412, 49214.09113369751, 28181.479319927006, 96906.86985573039, 15725.081593406498, 64594.85642487342, 54424.97839152425, 6804.010304008945, 68018.83290557448, 72338.26246084987, 77785.32127529675, 92181.44946370958, 56881.594586283056, 45571.23100563183, 99206.97809508175, 87014.50204121071, 60876.45683379024, 41883.752369792135, 25085.749353573094, 78748.59922358219, 25462.02978546018, 40370.173352522135, 96210.5982159049, 21994.03541456966, 56124.51968618349, 12344.766752696112, 52692.895371644525, 18195.597806507856, 80481.30737728503, 69095.9880186341, 92092.4728880312, 70132.34398346137, 53839.12098965037, 35490.67699736289, 27635.956450586284, 99974.62210687842, 88565.5235885044, 70469.18232123299, 1015.7308292046973, 62658.07728237312, 33088.517186472745, 49502.868938828484, 33655.792826631325, 58966.784386851956, 92451.34169278826, 51408.31926975372, 34010.32694400895, 45323.90186410363, 67903.57232610113, 12528.406155652905, 74699.97687268036, 28703.538361484647, 53849.50531369304, 54949.27672305269, 99791.5668826693, 42836.11598816796, 17201.932149720644, 8877.912637551932, 45135.36068947567, 16903.104594164408, 48872.3569017809, 47083.46255261815, 55918.211324410426, 80990.84981695419, 17721.83599115409, 38263.66712286915, 883.9758309941769, 23548.890237860443, 77829.5426711654, 81770.72239366567, 95061.98244229014, 11891.025496132912, 28759.24153277768, 15375.054325799176, 41362.59947542804, 41625.67612351621, 76791.22323824263, 79067.72855553543, 43277.453339609005, 20432.964396390573, 75495.892656137, 82234.27380621014, 17741.98663101826, 18555.96200769164, 86876.49918506102, 61509.04493280448, 14679.552239807559, 82767.41151210816, 91340.74073870407, 57051.930677168326, 14552.430636887591, 3420.7184518191493, 52859.366679662424, 34755.388061638456, 11312.317855353836, 87887.63613073545, 41444.72630825992, 28822.426630739938, 16603.938066953582, 36832.70662877581]} +{"id": 691, "vector": [22978.879973216637, 15357.383097454536, 79281.77968720847, 93496.34863647055, 85380.87748938153, 67139.02592595536, 55078.931897844064, 67667.56179152103, 91994.25888264325, 65999.99423295085, 61476.77224570346, 49281.108906304835, 82860.24186563818, 91999.48620081187, 91999.93351687863, 65991.73209183941, 87182.05778720895, 6072.313504324667, 64349.356305692585, 28190.866422886684, 83400.8014650178, 24947.125817254688, 17213.20431379432, 80652.28333976971, 17343.018222071005, 25683.413835164614, 37010.09079793931, 41318.14409284507, 26173.14401199632, 69755.97993640286, 18468.226823228073, 52492.10579477773, 3082.063227960241, 60681.77856769674, 99410.31963474743, 53696.12871195684, 65927.78355975852, 44553.35114178707, 45729.97284743341, 80560.09408484692, 71245.14261981004, 65742.63171040687, 23528.90188412208, 51098.9452220747, 77701.77205950802, 52893.616424415246, 10535.54846749386, 32.78489638685045, 51167.35330603817, 44026.36456263199, 68577.63893490918, 21896.10106205161, 60051.65213021002, 83236.13327897368, 40436.61777230976, 67719.88730288672, 83459.98938895152, 65461.33155009131, 94393.74576488958, 51774.4777857453, 24154.057954967946, 24411.322629843013, 72717.66040745018, 41686.72265008223, 83273.46247058109, 89890.1979437877, 82696.11054385491, 93481.0843090645, 45296.5317795408, 23244.186902526442, 12836.900685696495, 80993.20452066459, 10637.737859205654, 46525.24133417197, 37037.0068664074, 15592.653986973648, 7807.398156008249, 13578.18444720037, 58387.65052926351, 79295.26865845849, 14806.116642466905, 33177.93814621877, 13307.544203537258, 14098.827896839572, 46218.03728218455, 98809.9510651164, 68667.36262801324, 24563.812154229614, 69958.00484414099, 40345.62338864447, 51450.990888564585, 19929.358719003132, 21399.017085714568, 76278.95037513036, 22695.0644775586, 53928.80699982442, 14666.040339915531, 97524.37352540159, 24488.446036316658, 96862.91171707495, 2968.892642741061, 47152.69933348938, 7664.260598786354, 80654.92252941041, 32799.6714614604, 5280.84685746214, 59413.65327893776, 75530.69469098462, 89800.3278863698, 30891.839385420673, 68370.8801167292, 52308.988354797235, 30620.779763160823, 221.05620700993978, 93289.07570385854, 64428.92213287517, 69636.61783308425, 4026.408506504542, 77699.7075293864, 85984.94064758727, 68776.01955117717, 70923.46990070304, 11703.5582666406, 93032.58995851791, 76157.2897841761, 63003.5945483353, 77483.81504810802, 15088.22334931299]} +{"id": 1724, "vector": [96560.95091502991, 56301.59274562364, 71169.40980561159, 18604.24349717632, 23860.51996028441, 74899.94787524598, 88900.41620441499, 90049.0398912815, 8461.122894413487, 50398.42309619068, 47418.5363282699, 66281.14324301401, 44134.94019275776, 33589.529243318895, 49334.253490404626, 6522.4972772922165, 91752.23613983889, 76949.10059603078, 14865.271254789037, 13806.294881753722, 86719.59513538267, 41884.78031361135, 64523.05345690895, 94904.57744993843, 83663.25511934856, 35227.39631681286, 2058.0079703831243, 84274.94667554453, 75663.31124031181, 62890.68800347468, 74958.69234964706, 70004.40900888323, 3081.9292802877762, 20572.982926706474, 50766.52859222802, 98840.7882896717, 46003.900122525796, 99580.23673335469, 34056.04431475052, 82268.67454247203, 57227.62179625926, 53730.40824480301, 18544.075435140316, 3523.72897889891, 31596.458549089923, 57398.5265904344, 34942.258359667576, 2231.1658543171075, 39198.66568736973, 37109.44232349589, 57262.31986449605, 65068.01905869876, 34327.957584776945, 87652.23693376686, 71197.53057629016, 42821.920850901915, 68100.82298923335, 90268.29647990549, 24841.2365258174, 58200.74517138582, 70970.89463164414, 36264.881084619716, 39230.77911826348, 43234.26257268049, 36765.827154495746, 81438.28235123954, 42234.5542902523, 99116.84350165857, 43266.799277954524, 35235.86753228731, 17302.989940545387, 83916.50585740538, 7901.066701380888, 50492.81880155859, 11353.145316376067, 84460.92801717418, 27691.903983595857, 30581.28645371533, 49157.081899965604, 87788.36233646823, 57954.46615309487, 48269.851127320166, 49977.229611720366, 55509.38311240285, 51167.32149090293, 41175.54060334737, 32644.963425153263, 10622.325586030402, 66201.92983552837, 72809.06628751002, 42772.165451652334, 29836.90347298693, 52287.381914507, 30778.667159781515, 16004.790837078543, 69219.94761072655, 44515.836321058865, 70962.35449602705, 80542.98054664151, 36629.56624483495, 66031.47097875807, 33242.960962751575, 37821.52890378314, 56350.215114645674, 47852.80860294781, 71289.39248750218, 15885.525099561237, 24457.45764364572, 85105.16151617799, 20674.630538682635, 62849.82423142976, 24424.234803990807, 66962.48803091387, 26711.276968858, 79692.63576334243, 44883.882193666104, 68891.81246823138, 14114.059812103342, 50504.98396128246, 76313.72118614386, 31689.221902824305, 75451.36406685531, 49433.96100032694, 45306.647509739305, 12709.160479561731, 33411.62166385415, 26837.887963816574, 82055.09991495236]} +{"id": 1980, "vector": [55773.1350826222, 33440.284261594155, 76425.12421670703, 91926.83941558341, 10751.22053765405, 7336.997536179102, 18883.04181848037, 23187.87733558141, 18590.7514736386, 58482.30829441988, 95095.11333676316, 8913.344345718888, 87074.32098744657, 50036.359685412455, 20557.829516416103, 60537.76187750565, 8722.760861810164, 51500.85004186815, 16883.72045542221, 49574.72045098318, 51282.00384849122, 97533.2947786958, 23953.329146126267, 29091.396821235106, 11511.506747398658, 40657.60355861981, 27384.52394543751, 72352.66065697443, 2799.7758414564864, 4548.998339768329, 31650.42834009474, 12865.482910364357, 15670.664382565481, 19568.738421926402, 73493.33839797325, 13853.68395391291, 78097.18837458978, 13451.637287019936, 41944.6281070972, 16803.735544087027, 82647.82464135259, 17976.78752882137, 20864.241763838763, 73438.12077378033, 42418.851180264486, 39119.79604940179, 30699.282283707973, 84161.11645221876, 53552.68927706376, 55132.8444340072, 42301.67655837721, 64580.17299411345, 32570.596888198965, 48924.19986326351, 89787.61188683331, 63413.751223754836, 14994.022049097744, 49584.48137303271, 16696.274326266048, 41859.51176606058, 643.7486691059923, 25837.765402760728, 28433.231924865744, 11924.789459663876, 85457.93068586476, 59678.9420891554, 81122.2442260931, 87306.58839722964, 57652.187965676814, 51989.11602035271, 73867.88549711561, 76824.5263193595, 59112.7524252394, 98340.49597403109, 66502.11456946288, 89427.50278990318, 23814.00982886429, 33269.756156754025, 45980.49422700292, 70059.14906551692, 79511.15498282542, 7976.402772501235, 71322.01938110402, 95957.40774900788, 43661.22306652796, 33678.327290777954, 80908.15801896776, 3940.3518028833905, 42210.67529961274, 30426.34235438766, 84352.1964776613, 26123.34291301658, 78553.80025374846, 94556.08690112812, 58401.472944673835, 39091.579768728116, 64946.656004569915, 88154.56191793788, 85287.51596963206, 73250.07546368471, 26851.775157187618, 56200.168870745445, 39750.114312922015, 22600.87292841173, 32186.17361065289, 58913.6653668908, 98203.7258960859, 41194.402242465345, 14273.860534011996, 77978.17662184597, 63196.430787467725, 6874.124300007945, 40248.49560644805, 71260.63650818459, 29532.103906500386, 34185.329767141884, 10124.049576703454, 82072.18232667017, 37933.432447244, 66656.30819405013, 80252.92828338098, 536.5414664860957, 13002.456526090178, 69835.44519227491, 12388.673491322867, 13956.929084207304, 3734.0837123822744, 59354.682706691434]} +{"id": 1690, "vector": [82376.1492512955, 95168.8875961675, 6457.772338523826, 18963.851683740908, 13541.128126476653, 68433.84739044314, 69317.92713279299, 53917.11723712869, 99793.49898514645, 4900.155581185161, 21790.525191360855, 7697.101855319855, 95185.36843362823, 41212.68769868268, 18405.302072905604, 4250.870194368206, 44916.22591526322, 16366.292615212808, 80315.94648697057, 21260.358445562044, 41739.60835454573, 86685.0766789074, 94850.72408398487, 82252.0803457803, 15887.32092938524, 48043.58400010233, 59116.274639933654, 49625.161147691986, 35644.87665465286, 84955.02369901614, 17352.958772404458, 26705.798410301897, 86783.58592613493, 1591.6328868019637, 83415.26400934892, 18803.961600866736, 47311.98178576951, 3173.5572705993254, 22763.718494258344, 33887.02766883928, 95291.96260757571, 58924.98282751459, 36414.437397825706, 51893.54476320641, 45090.50068304794, 65843.93949109098, 69857.07533146278, 3260.99734235209, 76425.38282939303, 46456.0465777534, 26823.214942707553, 13391.469700546055, 37059.133868621, 30687.21083223126, 36235.99751174077, 53838.960034567026, 75526.48807608089, 56396.31246580088, 64387.8692438512, 77937.61991233584, 83054.82795560057, 8936.248102243106, 76091.19891859485, 41158.76092987688, 38066.31060289665, 93323.09371966105, 7358.607528322469, 93300.25647203051, 39843.268071774575, 6882.45877063941, 80972.24480313245, 84752.37646003139, 25106.280749225694, 25126.948498774636, 46543.67336778901, 11040.425098445605, 36313.902712401905, 64148.42293646044, 92592.71935469864, 4878.556822963642, 64024.93946240335, 65478.96796481948, 55260.897322272795, 24860.02451080521, 77359.79852851917, 66608.4461536551, 17376.83572813764, 87268.06707517501, 20481.638019803362, 68007.5523699815, 1050.5465351254961, 17413.89070870897, 24117.985890411263, 90247.43350488856, 55938.684806522375, 8034.729800198082, 42860.36448549514, 46588.27365679996, 8702.528569183798, 92430.75341633716, 28722.941500850917, 60849.398179495016, 54757.324041226, 39191.451066403446, 11615.32389257709, 33675.98253202402, 28429.058229181257, 11809.042580627127, 95912.31276434925, 66599.90975295899, 98003.97257306092, 22682.553200079004, 72017.43425123843, 33174.16061763905, 52068.37780411087, 78403.37473765848, 68697.6967191147, 84644.43171099995, 70974.36330581162, 70815.4896236138, 50702.680436593226, 23601.184634381778, 21041.247997069702, 3550.5641356565643, 3278.1524008781757, 99987.12120596315, 20470.4255076288, 62110.50908556467]} +{"id": 817, "vector": [95456.65842292676, 46647.46334392917, 24591.66819650036, 96507.08257298217, 60505.00453858053, 47299.477578739155, 560.1084777459909, 63059.430959699734, 91242.00483096081, 40097.79991203501, 28235.657459182574, 93598.25876415453, 31362.47943422905, 98237.42631451819, 97050.2369090161, 17543.012230910903, 70790.9086035908, 88144.48028684368, 75869.12475082939, 48996.53473515176, 38006.77146766954, 29649.129750304583, 12756.614529528899, 84199.05247652064, 15723.220332391564, 23127.036244191135, 94628.40701182531, 44445.93860486481, 98805.2294182127, 46919.38706691187, 82221.01974028077, 75834.15222350853, 44605.13167480281, 71841.19448204816, 79092.7715317475, 27808.233983108654, 94234.2903753157, 32300.35371116624, 47167.51501236851, 19419.634234230885, 6661.365597247815, 45245.76751495813, 17920.979545955495, 65698.11438836758, 85126.87621015105, 48382.32446446058, 14396.729749243197, 56617.908801521786, 26460.272576194187, 29352.91704151303, 4388.743586388433, 79319.17341998544, 31125.605580234427, 25091.577345040394, 75055.57479987714, 44197.66428646076, 41464.77686106511, 62496.03045817399, 43573.525334121085, 47258.70719101209, 81234.98689592209, 9555.783347596458, 15712.992376183776, 1531.61146510008, 83820.97568207548, 45679.54313217122, 88925.11014907894, 54946.36750180471, 49155.61199259644, 63609.15878533708, 89615.40400503237, 30841.645745704038, 67981.25095640504, 12992.565404161827, 73121.815635045, 19174.207475881733, 8302.750997092246, 66082.4272081044, 49513.95515707821, 20365.225892016526, 67980.27906605009, 45545.44657512467, 48358.460210446705, 41146.8058514906, 70195.94186421829, 48050.58323082928, 21946.518015297857, 54608.058667855206, 46383.75964835444, 61419.204380241, 15003.055296013501, 71598.71466741922, 8043.577877449903, 59869.6810972523, 35950.164205229754, 37619.786166053134, 87792.7368695153, 22114.80250162039, 39006.20618648336, 13319.775909350074, 67807.1059575001, 15793.19155520682, 44290.27236401625, 87569.52390598247, 93631.79389313194, 7162.321641951874, 72848.58110099276, 37364.25460232981, 66853.22629496135, 73554.81687888197, 3179.0615678660374, 42060.60360654222, 32188.938203827543, 11855.48495033909, 16394.019095487678, 89434.41494302465, 52837.2184992144, 97843.4078416223, 43179.80499424419, 31625.268125897877, 38047.24021850374, 62916.71053176148, 74738.88167065437, 54807.731356047785, 25761.790762615343, 16090.523994134497, 64514.137757785815, 75711.01255419258]} +{"id": 1689, "vector": [4779.985457349922, 11890.112994757863, 60359.62250190571, 57742.31989525967, 15097.458354036542, 40072.36101341869, 16196.484742796036, 36198.076202786535, 89037.84250701989, 99570.43330583061, 43374.4740453884, 17187.51846608868, 20625.645648741865, 63996.71995736297, 33633.09626719859, 77896.1311680573, 86547.37319085962, 81257.11959847284, 20323.444026897185, 98991.23815804707, 51984.1923049705, 10613.126260064166, 55462.49308487818, 58777.453179239594, 78050.71431313171, 65098.31516947808, 505.519201265614, 37825.05105722489, 49583.1585249511, 36630.46245653029, 50102.083474682426, 8877.111259360914, 28118.462598553306, 91166.51046946438, 29796.590214809083, 72868.10784790717, 99725.80182103845, 62519.975457484135, 16231.674637257409, 59394.170361340584, 68051.04914850224, 12203.41938591839, 19192.13153918259, 70483.75916559422, 11311.588207654322, 48933.80853568908, 43852.26624444034, 29240.35408037907, 91629.10375261893, 72286.85899231596, 73753.65747644409, 6060.223832672728, 52039.40167542634, 13318.356446740952, 86196.94964919084, 87613.51030696742, 97016.48262939387, 1961.7478569003642, 44652.720704731284, 96445.2435285927, 78731.95487648212, 46124.29618271872, 4239.530723352569, 65575.13181623703, 62945.674394768146, 76559.36645716098, 58599.76739133196, 99436.62658978849, 54207.05949626557, 42980.9352713448, 77049.24373178078, 47488.241964938585, 11223.662195115669, 50471.332036138374, 70172.28090749866, 60596.999337289184, 59915.58927826594, 31352.39438769194, 39072.13488370116, 36244.37836240659, 84864.84608963295, 86803.22639259104, 52778.748685201026, 62001.65110768978, 3291.956448190736, 78493.71209793811, 87844.39095385393, 39515.55914312453, 76758.88728716667, 40615.31798710366, 74021.26158734514, 82257.79292690392, 13417.589399593577, 28451.096777689032, 51327.97638573532, 49427.72814545425, 96408.78086827994, 25334.24388297346, 42723.629044673675, 72798.10603759432, 6664.028274876943, 71685.42415158202, 72307.52536390818, 48092.18637692694, 66908.37405015362, 65298.41135404347, 75322.42815626108, 38141.83951088934, 96136.21150226198, 4057.1701387558833, 82694.7070937217, 97031.92064639878, 80108.04572280096, 97176.2633537659, 31093.796072586378, 50006.666068098115, 60323.758372146185, 1484.8154951194736, 10190.194178973155, 95274.98235485608, 4505.1910489864895, 94751.80858436413, 60220.33040813875, 69159.51932973252, 48566.88592021592, 66351.36764810531, 49014.26410487111, 20393.182749534255]} +{"id": 753, "vector": [32875.92478256415, 66886.68942199904, 83249.15522043461, 83396.79921115912, 78329.76242183258, 34678.89016980282, 26906.591389135236, 2760.1633620436437, 75859.84457392985, 80600.58515046425, 46721.44236933825, 7141.042554705013, 29926.807775841557, 44240.95559101942, 40597.282486222, 83592.69364719685, 14438.5975659152, 97761.12773916488, 30358.856127616596, 42027.00254926207, 80546.76979209008, 94785.92960570865, 35408.427001743235, 47923.76167059493, 93719.94147232667, 89677.25478914769, 81811.22953361459, 22338.671575930024, 95869.83215882392, 52919.521771267995, 51237.65950344631, 49272.688756171854, 61023.7782684927, 40153.76504055764, 79302.26709288248, 88181.52498343747, 51861.29222158899, 33337.19006147595, 89323.52347358232, 4511.483669641947, 69201.23262982213, 83630.2193175822, 42806.46984030846, 72257.94779257194, 85968.48736153371, 84060.7187644388, 52258.90995116136, 94595.46532889477, 64497.4052392845, 15438.150587588729, 60647.86952965881, 33303.04325800131, 42981.60173611979, 73039.23615061812, 13110.066262979735, 43508.52377304095, 32788.2008455322, 93330.98387863448, 25240.438885046544, 96077.66187714288, 66774.42503134104, 60175.23252787772, 35077.890399979915, 42497.89082332777, 53549.54773805951, 15530.044002206645, 47208.62144295591, 28666.186392799453, 58427.76924217352, 17828.76213853387, 63609.477823570516, 49722.278054643546, 90042.98262020241, 39919.75345552102, 9793.500601640537, 31090.762416889484, 81325.2352333485, 37360.49764823591, 64609.31209579239, 63976.08592712495, 41746.44844517269, 46383.299755195185, 45501.65030800289, 56373.10023859895, 3217.706917819063, 27760.811605704737, 9111.671375775642, 24808.02397324422, 94178.71712980582, 71347.99328657285, 93189.7245711404, 56595.09735355911, 81306.90135604293, 34630.88498194495, 67946.09926911246, 93641.88250166755, 77410.30521716666, 86756.47565616862, 68903.96488100498, 50664.325611233005, 53186.50566696194, 63455.45337048658, 82222.31210481179, 56131.42456393462, 97372.46207122551, 75171.59378351728, 53494.556894319445, 8148.540660061066, 75966.66074335066, 53009.10141567471, 6723.712802786897, 36363.418261649815, 52857.5550631314, 56059.73860456358, 75137.4308343062, 13848.75419028091, 36017.65306699635, 76747.19879245207, 68688.80576751805, 9744.746751428735, 34730.670792343946, 18029.739339671345, 39347.120215837174, 36506.98805585079, 33467.95414284891, 44857.3040827428, 24770.871107286297, 85989.76569802749]} +{"id": 1579, "vector": [28780.39063391318, 30617.724582191386, 53993.7072188228, 18284.920696942896, 60580.32334190598, 12330.241552664611, 80491.26914841786, 34723.638558853774, 85647.66694655501, 48253.936902380134, 80594.2612535669, 98661.1967691369, 83511.15474106128, 92327.80883727386, 96342.87349517662, 31719.590171904445, 45264.280436730754, 11865.908892982003, 85848.73686675586, 92758.16663314996, 63002.181229763475, 74682.77666963234, 86321.31987671345, 71022.57446830808, 68695.76990873256, 30887.26281557016, 73193.57893160415, 4354.388035222423, 20750.12887431258, 23825.259810137977, 77220.81795832163, 28326.78746450986, 8575.791229042972, 14094.266889099848, 72732.31244049812, 76668.21890016191, 67854.47725693983, 90130.04411499389, 60282.87015469999, 53088.976751906135, 74207.2898597693, 20857.410877947936, 90263.9329775764, 17420.189296031498, 62597.13969887393, 68335.4810640524, 7212.7248790257445, 80290.83178114766, 99731.72568367117, 12700.332821517235, 94223.98164276412, 82226.09935150456, 65194.65711974764, 46157.26916436601, 78736.14468975998, 25484.785657989585, 66269.2567279457, 35758.90560062005, 87837.69619125564, 92455.77097577481, 10170.840612235732, 14431.470050993044, 86907.3298346298, 3143.4805337524076, 91582.1554342394, 39793.78986577953, 6907.063793608481, 62213.57494877797, 44661.36423305111, 6237.019706737201, 51090.56459350709, 73008.24343851971, 76032.27395216594, 80804.82055919516, 84946.99958756556, 3240.160164077055, 38319.5337114342, 50438.1728831147, 86480.5185634736, 76132.69937088751, 36224.511983299024, 91805.78269541418, 22949.21180397471, 68548.77505151402, 37377.69100089732, 36004.30427485155, 81205.67842319963, 48452.131664339824, 65381.68491460933, 99975.0890054676, 37129.345961393265, 42470.47053645193, 5088.8653132210475, 91245.6329904799, 6701.689803177491, 3535.228527044665, 38319.69525576051, 22460.14816666709, 81436.02810890977, 40141.274977971574, 55612.15610266655, 3504.6574814897813, 14672.74533091395, 99569.4901228972, 16429.54361394997, 87122.16596848, 5697.458032337788, 27728.22667972851, 5238.173604694374, 81587.4406365771, 20579.320584512494, 92611.56036987294, 15217.139628362564, 69825.22730200957, 34286.40806037283, 80497.75621493068, 22732.077556755736, 16624.828351660904, 31092.856867839157, 25565.598647911323, 3450.1720790610093, 66249.12210089597, 70539.35927244484, 65764.13055235267, 71644.5503146153, 67099.92311291352, 65619.41831336054, 72710.97886920074]} +{"id": 2106, "vector": [89365.97610690338, 64757.73946830344, 24153.58610081163, 66659.66243545737, 87560.26666663382, 57535.66814327909, 66808.7773591618, 80616.20887350684, 30515.899577622742, 86177.09747376856, 8891.673745162365, 87252.67116577059, 84179.58377977097, 68420.1635727129, 83659.4140854452, 57874.68511050132, 15027.213712450572, 50691.09003602315, 68663.86233962816, 16239.362765406773, 89374.9266926974, 17993.654245077818, 1470.42282646338, 68649.90985200205, 76858.08253125793, 97065.01072491573, 81921.99947614875, 91347.25836855781, 8792.655840365682, 29141.189463174745, 70453.5428354785, 76830.31301049095, 82529.82720063362, 95746.22001137276, 72504.48220148565, 69303.84124957111, 24684.921497291743, 55951.138387478815, 3428.7900569534236, 81534.10314485562, 47040.09402182164, 15439.097630255439, 80680.13503513054, 61261.23884860395, 74777.97211354692, 3951.3771804132625, 5715.961414679927, 78607.27703978587, 79487.1889911069, 3458.573840926804, 47567.184348714465, 36135.207944416936, 55339.39220609455, 15428.264214698529, 40884.4591221448, 94984.99208878128, 22568.682508094338, 94249.60364507446, 86234.4025920297, 64306.008134619195, 40133.888490301, 39927.04854024007, 28690.074052951488, 7304.740619481798, 87901.33361003491, 62102.15997282015, 90921.301382764, 74450.58520324159, 80632.79499488162, 70776.95049931757, 22024.067260337866, 61955.127017408355, 81267.07185387787, 21325.711659938872, 13813.679962973336, 46649.64618150984, 26474.27214017346, 792.9982068696106, 98340.08239820017, 94641.82801662986, 49999.17137047245, 87547.71083980742, 68724.48351671005, 80421.60563703573, 43891.063258592934, 70599.71615706656, 93286.77374015193, 50733.78558848234, 50515.62085664526, 28504.675296398298, 53851.93450580376, 51326.36262078469, 23176.717236303, 27253.666595584204, 66953.3111812254, 89060.89824982999, 65124.688894556115, 54850.41305273337, 85511.79847395045, 26270.224427318335, 56076.25350742923, 31327.476297373523, 46489.48682080907, 41002.51600067334, 4346.083043750859, 2258.0588607662435, 92140.39655347216, 34491.3235441051, 74169.1218010526, 74081.54198936757, 53612.836355505075, 20691.264553137466, 33687.096167124655, 68671.88318369574, 99157.38100539053, 86955.50933243021, 89423.08037506844, 74224.58532817627, 91767.17554745986, 8674.228353816616, 59425.16048257286, 22258.539204852124, 20432.422004576623, 93211.11781716443, 30289.436650889613, 86523.11870073125, 69886.2931652694, 2696.1010570927256]} +{"id": 667, "vector": [67347.16401307448, 36700.704407256315, 55558.18424261916, 36859.3618902211, 1886.2138153619146, 4648.438157495116, 10883.010978616137, 64003.86871701379, 89260.1827069014, 85113.22545165096, 15631.889553664147, 81156.71988476445, 24814.279167572608, 93221.33309973747, 64858.95580080123, 34710.4963173266, 5130.858888443157, 15781.30507130282, 33636.266830697845, 37422.728903207026, 29527.428298376435, 78985.66010092168, 60232.55186510388, 56237.91744598791, 9642.34330207887, 80828.72834033679, 70220.13767012929, 83426.33605263372, 8656.453725610258, 2835.7613439996876, 26.1894576845334, 23355.557638636405, 6528.071111609157, 62394.390219796805, 9320.707956655406, 66643.60628645546, 53618.86918230407, 26339.07650978885, 23418.660796533575, 91930.75337757506, 5649.897656349923, 41555.13707948037, 4687.139322004785, 34506.92129402117, 38538.85264592068, 42894.89806407118, 8202.457213430036, 21655.612430111203, 105.48390198876767, 95913.16123815531, 22550.327910114633, 46382.50856955544, 98575.42502809431, 25971.578235942507, 84666.7101252146, 50793.7171506513, 77259.12915586702, 88822.04537312314, 91385.26032327475, 14405.327419700765, 23728.400725642074, 30547.739973100674, 47120.79509718001, 55657.34342893828, 11930.786841129615, 45842.899045820406, 3599.7412683895445, 53205.46813849212, 80911.41607392518, 47497.52684806523, 98519.76354208266, 39792.80106831669, 52198.08475580108, 91523.43013989639, 36701.687113246284, 59128.162980450004, 55377.8196028661, 25736.0930380909, 16532.69273746186, 86510.29135890286, 91345.41083819068, 55979.44904601624, 95150.58977257997, 65542.62096235022, 32420.787790435734, 5471.648441226828, 9923.242376360975, 39136.683136797365, 98683.47293006374, 37161.72316248243, 86184.87854785219, 10171.791425816067, 47140.24671934025, 78860.08363908889, 99622.07113478163, 48123.70588494197, 6332.198929884925, 49704.00976095218, 45369.84479995348, 3988.689725334782, 41856.19623928125, 75374.91319832436, 28876.580985003453, 86294.59245919848, 88321.37891631649, 357.5782768224234, 91964.74782749699, 53948.69547582252, 3285.8837439398326, 77076.98124326536, 14172.981820405861, 83918.04760167669, 13375.320430618609, 3398.350185578591, 87808.37759420958, 32388.55464524075, 8237.755152895032, 68261.60827106748, 50306.628196558864, 30046.560087193964, 11290.860933271806, 63225.01860815516, 1799.145978520511, 43720.03622333424, 81673.33922176536, 36929.62653366388, 82573.3351366262, 94984.4898589321]} +{"id": 1683, "vector": [56898.72897379705, 92235.87116213645, 42148.77378611861, 73129.91885354035, 95887.24277442253, 2219.1399208848716, 9411.384262426947, 90202.26766108819, 3333.9515793968876, 87818.38032375503, 735.5988482856457, 81694.64695945085, 37001.69266398365, 63180.81267919645, 36017.4201489459, 28618.925894613512, 71755.24776474849, 50420.20878752528, 30390.944721285417, 66898.65179616737, 81058.81285553296, 11210.686948454484, 7952.851613459977, 6146.087007892298, 81480.07466155053, 74261.47615978758, 65642.26101417509, 79987.58743905276, 75714.34778787642, 51040.36146911182, 25769.13600632641, 56400.06436388787, 59459.307197300106, 80766.98803034147, 24386.31346783526, 31511.44547531274, 91368.22814692717, 49865.132041894125, 3225.0912791181418, 50794.084603750685, 72564.80436030882, 12907.86344046897, 56617.340480447376, 59713.04352517277, 61454.307452465975, 77977.45233172122, 10304.832059082892, 83684.18964791328, 43547.19189837085, 7018.952775363352, 17731.455864601998, 61029.95690063093, 66255.16058500913, 73837.96730954412, 38965.24221781238, 62181.84600731835, 97570.17608860225, 6165.868327761826, 80886.12898661298, 17548.457084897105, 38694.076271006474, 87230.95710211528, 99304.5278425819, 16287.794472894657, 73039.4315967484, 49174.8769865321, 26015.152230656124, 49467.445200156864, 80278.70243956018, 24797.453234279088, 21619.500319602124, 28963.343891902605, 76378.37501246636, 61739.476213172886, 544.6694179670475, 2446.2110067152375, 26730.238697696164, 72713.02751544352, 58740.47522575615, 95636.56654489718, 35811.107392666694, 81877.80584154117, 23908.363967830493, 32217.60186171585, 115.70281195516507, 57223.156063329974, 15280.139454042108, 31277.419051223053, 75887.49755865839, 99323.07840534249, 47155.04418674473, 59343.326722736754, 45967.70788529106, 20035.047214179434, 62739.17628182269, 64772.36654968977, 97609.11022622262, 79380.20627189522, 61552.2163078437, 68784.43227901908, 61903.473260955936, 31851.83329965734, 81176.0666911839, 92352.58287724484, 13040.172368120018, 58764.32217812666, 53309.04493766956, 23408.143785724267, 53219.2925375876, 57816.06446180714, 88141.13622039753, 22271.720310070596, 64226.454717266235, 91054.50491211165, 19547.565781275578, 63070.20671234017, 15468.93742624016, 92450.33643286687, 23639.522809560796, 34942.11023842617, 42027.996925361454, 51246.65706308221, 89651.72243564803, 29740.969606114853, 65098.37214131631, 52477.68328971449, 75808.02098224593, 9578.43340566813]} +{"id": 1859, "vector": [83592.82077175305, 67741.7331025484, 59791.34923228849, 24335.324067638565, 8524.389989283254, 84767.98794894952, 86627.72920010563, 47116.50160886354, 50389.6046032242, 50049.55312064494, 39687.609001487624, 63606.794125968234, 76307.53623669606, 4046.7150894400893, 4645.072349609547, 96603.7989157825, 33802.50087391264, 65828.10340731734, 13186.045310297268, 33807.561116679586, 94241.72695598347, 24437.39662988974, 3735.074391529225, 12873.36802958543, 56057.274271250324, 45165.0870753033, 87218.5273926604, 30463.72278000844, 7181.297744614101, 88089.31520701695, 90734.61132287711, 55525.21849147162, 86243.15192125468, 63073.63527194843, 99950.20785121039, 40256.09407372257, 61515.55249654144, 37409.301511473524, 88409.11956252746, 86385.53295905425, 33146.15880452434, 34701.04161312464, 88153.41693580885, 27030.899732013157, 65978.76232314782, 21194.825281833706, 4771.048824491942, 36043.55241020486, 42355.15340490954, 25039.124755319564, 84280.15482793412, 37345.935718905675, 15452.999532936406, 63678.304592405024, 99962.20419050596, 91880.1550070965, 84138.3985057301, 4599.114810574256, 82559.25515026065, 81976.60931578345, 64575.684811903775, 17027.973287496003, 25937.24800981647, 35086.427999076266, 43961.63491901609, 21645.1004473579, 41155.84697333063, 36190.638369697284, 47749.854864124, 5885.366303287309, 30817.899957197547, 30063.77324766788, 23615.46772166001, 9742.310133989573, 11487.519974099014, 94761.75211609503, 34094.53366092733, 69686.22931125197, 3827.641188108488, 8184.820433620011, 34062.39806200159, 51993.79099905742, 83531.64774336918, 96865.26944223445, 15611.153832919721, 76868.60628900772, 60440.460880654515, 27402.022368684353, 98608.64855980281, 19651.88212851793, 84639.24407786269, 78763.9817177718, 85179.37945331002, 12405.909608122745, 2327.37329043724, 9372.174204071882, 50481.45162714279, 47359.811753515365, 88702.73504318795, 91636.00155795731, 57725.876459327905, 63772.99249769182, 82231.48104161909, 27435.867271439507, 61433.79542422638, 34916.66305549625, 44486.707979906634, 36982.47300735798, 92417.32163628675, 38941.1497178305, 22801.973937150964, 71507.62401282342, 95499.18899098746, 25789.917688834397, 54044.465810911934, 53633.819007490914, 17405.699355877725, 40781.611489939496, 28075.065539745305, 38708.147205623485, 22916.71688302368, 86828.62766050859, 77533.70091392145, 79413.5612202319, 96433.60776398731, 12947.719371345167, 94022.01249832772, 65819.74774979107]} +{"id": 1418, "vector": [37996.21604927514, 57361.753100352995, 36378.50062567459, 9635.48254290758, 40960.873702958, 16065.54612671336, 14843.52608659515, 11227.64887197506, 35225.445170406056, 31192.075839922218, 34048.83324565643, 8265.330147358973, 81018.29202841771, 6734.915132634145, 68.20033328932551, 65302.96179929019, 66889.84677446508, 35391.11521545849, 42544.763811084165, 73829.61179006573, 71103.28242191623, 46607.63834492205, 13877.70554918426, 65573.33383172995, 51956.14805863376, 79056.00915216954, 87608.63268304452, 94889.727148934, 96655.26267523882, 23972.031770123704, 89196.38778504865, 76744.35162161964, 98755.08067292502, 52715.4651098609, 41441.61450535783, 38315.769742767116, 99929.79879273602, 60946.687420305316, 52079.68550144102, 30396.809035016126, 69644.63456686439, 79380.266479976, 49497.339446528946, 99265.95466127327, 37911.62187802488, 18868.307521036877, 33850.43807264329, 63958.401624214835, 94797.13359211062, 77789.8333899047, 62919.59241087864, 88595.75014907363, 41601.641785541644, 40156.08326386628, 54275.484369077894, 29429.83761198238, 42746.66454602172, 16214.823254890898, 71218.91305031997, 77574.05311066126, 71816.0160241032, 21778.38308978368, 86266.67642314221, 59034.40519822378, 33152.497852256376, 33789.70543303122, 25796.135585135016, 30538.980337748235, 64566.71467277651, 1860.2293173253681, 88693.08524402614, 61888.10623678056, 17201.78646355621, 95198.52256067957, 26293.31147013686, 94257.17032698833, 51546.4274392048, 26906.488546419227, 75135.90022443736, 49914.92558573187, 69818.20054649655, 7959.608219262759, 31708.951192495548, 2452.0561740308035, 9940.984237434359, 6214.564742054041, 30642.57628804503, 60877.28276916761, 87626.03705677639, 82176.83807563536, 87517.78662161942, 72585.25736973913, 1297.6791937308496, 84124.8584261879, 53268.51119555, 34876.01209370524, 60867.13244691636, 81116.01744247698, 91857.81097780501, 85054.83793031563, 69099.0289811801, 99158.50742804875, 83824.14336379948, 14241.956982023295, 62156.31082354792, 80709.44349803893, 63854.06035214373, 89608.58035586945, 48012.03906180882, 65719.33795103987, 65767.141186227, 21010.136604957628, 84918.72248639275, 74449.27960301941, 63716.186321013854, 74568.53731287703, 28321.207948489613, 75224.8052433715, 91971.35777988388, 58664.249498532416, 28139.571824791055, 61038.93850234697, 50389.41866838399, 54057.76381515586, 34859.90855094953, 26880.582764172002, 35573.76350341075, 45113.8281968049]} +{"id": 1216, "vector": [69194.84155189493, 40807.4083719895, 41677.156789958135, 81761.85981591252, 31497.59621665221, 72997.51056293299, 75659.69862024573, 86417.38387164682, 17663.487121974864, 41167.406132470765, 18688.13205150195, 44135.943743245596, 89364.1955644526, 35124.69382540538, 59880.40078877317, 72620.2677358421, 65601.04178127898, 88411.13444969465, 14019.386550166857, 62381.483669557056, 45106.72286869304, 58253.49841820494, 83365.88718315726, 8230.982554904409, 6203.166345058142, 98456.65210487304, 61666.51511934871, 44036.56427220952, 88363.40036492437, 94668.2709085022, 40990.911629969596, 20935.518743382287, 35144.59488671722, 50514.03653145481, 14904.18601584007, 40443.713193969, 99172.93867137402, 22525.979052386734, 52740.12534229634, 19511.72809013464, 958.2875300607086, 34392.53643437328, 93425.55386753632, 39441.613074766625, 12364.739449226136, 98984.57364242541, 53287.544953800614, 86683.71040932414, 18581.33789452674, 13194.087579292136, 81885.91783583046, 87543.46040477925, 90540.58828418488, 78163.8328264669, 27190.53668790259, 70049.72558481802, 70500.26550024237, 46126.25635866553, 67543.84177341529, 37744.53167645756, 10376.979912558105, 8802.198679126372, 53213.27574100603, 18690.621915580254, 17675.046731853807, 25151.600032290546, 82411.30987419872, 78839.89258871732, 37162.73139513286, 26376.906858324757, 31137.53553694316, 99885.22014790504, 80462.23178638915, 60622.60253754865, 29363.636670476455, 28496.792628636667, 90272.72510722291, 57256.883901368004, 17170.987278693672, 71758.31474443438, 90734.30814133807, 16003.094070951229, 88891.509893459, 74726.22092491602, 23282.358575746064, 96678.70755753262, 24608.674641911144, 14532.307284489498, 25272.938811980785, 74206.1569300524, 99955.7277605204, 74156.63550101363, 45512.611630073705, 15017.519548210134, 3156.6949840008297, 32945.051656688964, 27000.451893103393, 32731.433757645922, 4827.244280522269, 49563.194123329034, 62155.01373103293, 71144.6202524842, 96189.47056044413, 9805.982532446122, 71291.08125160537, 76181.86333873663, 18524.582329617035, 17206.983162256372, 99247.90741711934, 23005.47515650615, 28885.147808875256, 25786.454170295958, 5334.864421178653, 39551.19645694049, 80800.08553421008, 75002.89036412447, 12150.454717646686, 63803.2218549315, 52750.02661675068, 1393.8793985844188, 43167.48448835809, 49414.62310999156, 89340.30864224379, 55274.43334495864, 5313.06479936352, 33445.27837419899, 38920.94005638757, 48227.93212257239]} +{"id": 1736, "vector": [2828.7439657016744, 94276.45967675553, 95195.64714235892, 93261.59700902623, 20304.459569945477, 79822.17000868938, 30975.108440341413, 52212.84613174247, 22763.82700751226, 77745.25705388469, 67916.36575262887, 66293.86498101357, 11028.828788958168, 52015.51985226044, 77179.58789545916, 61809.83342609223, 66693.5203542694, 52163.071977179934, 72786.58498469413, 73614.97661862733, 89367.67493075147, 90353.85072081845, 97873.65613231384, 78288.31244391078, 47288.652741943595, 86753.64497200576, 98415.08358509495, 68396.20680408612, 42700.46479800198, 41848.93796798851, 45678.40446614867, 44418.98071565098, 39477.64973143136, 89777.08888929033, 1162.2464422618716, 44826.28701711741, 70226.23341290129, 20480.10716938079, 36985.93210884117, 85867.4472515005, 14877.074247648114, 19180.097144062715, 27897.46769293041, 60922.71060826085, 91478.33859457723, 6973.047228021423, 60152.13857253641, 51850.62245203441, 21576.045362692843, 47812.47429331028, 43973.2360997642, 59366.753065535115, 74291.51192979954, 68523.42335539342, 75479.0528978435, 34605.16411895001, 47116.30557097596, 94593.70562550261, 46068.096031187364, 30781.76949090623, 52662.08875353278, 87274.96557627714, 83512.43201086046, 67068.67077458686, 14204.074651407838, 65524.59708016118, 62096.36187445426, 99685.62418836378, 60460.006823917654, 97470.66723770044, 50526.335035245625, 50043.84973420886, 26911.79254864832, 95356.59451989166, 98162.68249501243, 83047.05787893735, 11597.35730746464, 62433.479570389885, 20728.855321269613, 49381.35395552778, 76548.9296262816, 2674.447618834319, 45220.8063004074, 97784.38672850555, 90391.86052877628, 59474.8984568645, 98209.6955818858, 29776.42763166083, 61984.037691965386, 29460.784996402777, 51726.33382178031, 85324.73282623547, 67112.59570368829, 29778.46989259977, 96859.26214060637, 55766.430440700795, 76099.83169809051, 68117.43828223909, 68669.02015604956, 6999.364455554979, 28683.399302759128, 12346.988602019072, 54573.00277138679, 20853.345594785966, 6531.619064253003, 79159.98576523761, 80621.59434658241, 80255.25795925145, 24124.06712116687, 46846.00179162671, 54247.77148673171, 34661.569235361145, 67018.65892822124, 30953.106889698258, 86110.88283464557, 15863.151924561615, 98065.30728748177, 44890.63599933294, 48536.75316243393, 45690.94124688283, 96607.81491966148, 51227.082526950915, 5871.628058427925, 93382.19039246204, 34327.293974555825, 47412.075241198516, 80582.53874195754, 52504.22601000386]} +{"id": 163, "vector": [336.2727320592618, 46853.7785350821, 10819.8284490985, 96398.62996711228, 45357.32233848626, 51276.93390079156, 87670.88641313624, 8489.910170415182, 32766.341246071373, 6519.5249871958795, 64481.31352535691, 16820.942039953745, 91331.40094783256, 23496.913786117933, 85402.01374427341, 50529.769005139024, 95645.14457324184, 27935.954498368155, 84209.15185695654, 75631.37498057524, 79542.12323918784, 96921.12780445388, 13993.969056712629, 9109.167552379882, 48340.99082775013, 94607.20100717437, 99967.8208713853, 60404.89997510726, 79653.83788068284, 7400.422857718336, 75056.42301754242, 16318.43198428442, 95509.90060673442, 80100.26696608808, 19549.539317491115, 41765.53613464854, 55965.6584273311, 78540.92916750857, 261.1858843620696, 24107.918504210345, 75741.64107100978, 64359.77948993311, 75374.08779468034, 88219.14638185431, 21643.79931957272, 98377.16467301064, 58089.56447345328, 83077.93634541315, 70906.25106259984, 87949.81091074206, 79538.23916707114, 6067.718135586464, 8295.950968363686, 3861.486308863582, 28575.43123666041, 62254.92794516328, 53421.64596097367, 98860.22343447385, 31430.728037559176, 87308.23472045123, 1360.4549457636006, 69216.36578181556, 64677.55106360227, 96847.62955552014, 99859.72371707774, 59823.79161841783, 41802.0221461, 26852.669726410084, 89525.28693132516, 91911.7940925203, 76444.48144402546, 29523.812106332436, 46432.78787263819, 82238.13411435619, 11257.056762471906, 82359.454376251, 36788.024828897345, 43898.327553176976, 69591.51884007461, 67444.65239788062, 36880.76006915728, 84648.42483759575, 54602.25955436005, 3873.541382085011, 83666.42298069717, 5113.422890490826, 23454.42015810839, 36152.55503744746, 53621.93163276144, 27951.471598264143, 51726.42947605693, 2815.2755374064477, 68517.63564067658, 1782.11644956342, 10443.859460011661, 92880.32684913313, 61754.79287117578, 65471.490182601134, 62883.58251247079, 48321.134484435745, 64846.1251101415, 85639.5082990132, 32288.237103240324, 81508.0058358523, 99200.9361966822, 10002.543103361106, 41649.351236136936, 59852.373179442795, 49785.19007015705, 55167.77114334572, 75640.5096198887, 65246.777285244716, 87628.48475300084, 94995.58036584403, 60063.86232628163, 11844.790343848255, 97604.62924745421, 17032.777770890007, 37531.99316857926, 5220.125822682598, 86842.43745330331, 17420.397612051154, 77655.55718337564, 64841.62666372677, 98231.97672453469, 55788.1361414719, 11095.455667129383, 79728.23732476961]} +{"id": 949, "vector": [12439.504426221825, 23265.349833502813, 79600.66344048736, 44352.395189110684, 55998.23082183569, 5584.443494483205, 53005.02704296858, 44493.05579822629, 67830.2399541558, 52237.711221505226, 35718.94234880325, 75720.14920345167, 44207.84681672684, 22333.182219218674, 36179.67713998235, 5081.33005192919, 71921.21644840974, 38225.813597759174, 74057.46510056118, 33055.85050434741, 57061.25109044633, 40802.59550931409, 43284.37003061474, 51510.25836287391, 12082.692122336546, 3029.339707348655, 6556.992823988495, 31112.795321151476, 28135.695472239375, 25020.663520834285, 10022.795760365954, 92624.3987239303, 27632.9075171605, 41392.853658239284, 95036.76916404993, 8417.651100369461, 3937.9369925311435, 96528.19403164953, 64203.331243465946, 86045.06681154804, 85646.32382699876, 52294.410628229634, 78024.95421566343, 28392.38548909059, 88771.73046945047, 8105.243272106444, 35764.35518874906, 74874.87208912201, 44674.03335472186, 37892.85493344813, 77280.96724686734, 41258.556157550185, 28333.28180342066, 92848.94690604194, 54881.721817674676, 61382.00752738123, 70980.642777368, 77339.54085048377, 67482.22836577523, 36579.84331510554, 21915.188527307062, 9450.84012925117, 51792.5211216474, 12728.161658174975, 75980.18272599307, 88074.0082895394, 81642.3967462008, 93859.09383050217, 46942.861532530434, 64443.02422220314, 75938.2919221022, 93819.18992686029, 89157.20549739564, 75202.51987192716, 15659.369906113807, 70534.06336639733, 65986.67024612025, 1902.0780999736587, 46742.491172811504, 80527.71521872334, 46979.67923935724, 67285.54902371728, 18662.59565904709, 94698.06572600953, 67063.61582084058, 26625.236378331017, 32862.14937026376, 65640.02830075222, 73377.76235311397, 99155.97709466162, 18668.202748767802, 1031.7810291339047, 57515.70921161141, 19548.330708489415, 3598.788551340193, 10277.524709163345, 94002.08958460175, 8770.477391681352, 93533.62661449415, 38385.572073387986, 72682.74930541909, 8996.018609428413, 62467.46719514568, 23967.29908466846, 48144.51815501766, 22303.471956217003, 19034.946305695965, 31492.651622333546, 6299.035084169402, 43354.55228354287, 55807.95636749738, 66236.7383809539, 31526.277973398865, 51652.28967343645, 23398.463457039397, 66590.52158933149, 3338.217977235558, 108.85354356107158, 54754.286107620996, 10756.031806496869, 31611.754561937578, 6715.050005843326, 33679.06019482125, 51643.20442871804, 29169.93429631981, 44090.521104606705, 90896.3228364407, 38100.28668006957]} +{"id": 1945, "vector": [78587.28764776373, 25143.217693165465, 32441.870605920543, 5528.1399281916, 66100.65649215828, 99824.80970581331, 46919.93035969188, 35611.95629474926, 81451.56714557034, 71479.89347659673, 10211.082056796173, 47559.825145372255, 96518.34181599144, 97780.01356846163, 67549.97156502052, 82709.68535454584, 11167.736037161047, 68679.22660014383, 2678.1112045116574, 86642.73217187062, 38732.7413420489, 68689.85512367096, 3121.7678152679064, 7827.137238582549, 96726.22607951034, 80743.39960156447, 42545.54966152028, 92868.45374665623, 180.9433716448594, 51060.49312500981, 79917.47199844976, 49638.47030965507, 73800.4917882274, 88594.69084862298, 24373.153250092706, 96500.9449796937, 86314.23430124634, 23159.24775698345, 72664.3814786866, 76296.01081145438, 15055.58700365731, 29583.749211084632, 61792.31851316743, 97052.84718579432, 41758.02039170793, 22633.78292483471, 36250.45088962785, 6982.211379373737, 6310.894306040826, 68103.91541376634, 79514.85251722645, 48177.97145648082, 14846.218758710738, 66885.54614944782, 99424.76681072939, 36729.5791774339, 70200.45486959058, 60806.761543405904, 60007.17862961243, 80665.51990960081, 77848.40099422091, 50351.97336217842, 69153.95254275242, 59844.0791661995, 54430.54274792201, 31993.992974056007, 74.43821561972098, 22309.279367130806, 77773.060137871, 37640.65676613747, 19009.132748370306, 97023.30596642358, 68339.02123091841, 47918.87171511105, 30819.188496930037, 89553.85877823771, 20325.197924977456, 17703.278342345373, 74411.93790313503, 72979.5656356809, 60035.38435344947, 65675.32340158788, 18142.98315911751, 96066.76090197648, 63098.867924756094, 41918.03127954197, 26.148058118047324, 11412.674424257662, 12749.02885146365, 78303.9231840481, 30427.044506578404, 24658.853605964003, 24328.316074275146, 28194.36148723524, 51778.97082356196, 80649.30916825088, 43883.79364245889, 50650.24444454358, 79104.49571278728, 21394.33831356453, 23453.080752170972, 90306.1206630643, 93250.13087084168, 89324.94209739154, 51494.08929947769, 24279.751948246754, 91674.84988555385, 53831.91040520502, 91963.35046590352, 99061.59801790825, 94309.38191927127, 69770.83294660941, 94845.76127580574, 27033.324510822775, 57417.113950693085, 76663.98913683026, 94598.21079876403, 46012.803663810475, 14647.566608831086, 79430.53469522583, 69564.90128006125, 11420.149814750568, 86710.5986462809, 74941.70330551459, 4528.1511552997645, 73371.54370145831, 44680.58066173684, 98760.79809083289]} +{"id": 140, "vector": [67336.54287806875, 21921.46783653354, 60723.66629279801, 77653.85970845073, 40664.962409915854, 11277.941734476293, 64519.71627302724, 74695.86961095373, 47001.9863136104, 47276.48449563931, 39133.3620158127, 18190.60993314946, 56267.72041489935, 45022.788885564354, 82429.19524642585, 9658.502420243376, 25331.372278100596, 93350.31415945284, 543.6705427940059, 18748.285395039988, 19417.36230585239, 35417.60893029758, 69558.42364759212, 65985.60711775016, 75687.3671473813, 63705.464057926154, 86571.94429950177, 54467.42400291266, 41990.25303866367, 13268.86873458607, 82501.3660612269, 33864.38856661078, 30483.67523169194, 64832.79422719527, 56854.94072469557, 21074.747515047577, 35312.690174751006, 832.97333039839, 40687.44845112198, 7129.918895700594, 89615.87329907194, 41616.81437173777, 12313.81393017299, 57127.25085658112, 89649.2900484437, 73482.54129392967, 75883.30098782509, 65698.85566001365, 2723.7748201501154, 61906.93771695601, 90134.07169917462, 14303.798555934654, 94997.5183388283, 83181.4428143375, 15477.515787174012, 63656.17787474152, 16498.446128150303, 44972.886754551735, 56584.88833977259, 48888.71393121275, 49207.06605383586, 70857.80879733885, 89195.59071443172, 11655.77435248102, 91556.87908000458, 28930.55031825419, 73272.19052973531, 90576.03897312803, 1635.492450312326, 58771.2610564002, 50029.562084758014, 70666.44021367168, 17788.10440495111, 68101.29235042684, 93980.83279732129, 30238.200721887588, 8439.940913753386, 26298.167712603026, 71080.17184595861, 94539.67046979092, 20330.042157873286, 46505.06044201571, 23557.524006169417, 19325.47306498159, 38681.71942048974, 47444.68887990191, 88784.92840124172, 62719.45970089632, 33114.5556383326, 34549.64807482018, 98511.2769053621, 42435.50612272138, 62350.18894287664, 59773.03207221089, 58723.00573831412, 30717.187910430886, 66355.56651915015, 70524.22509866676, 19874.673831869517, 12225.620913542702, 80946.24207365993, 5102.935136234177, 56619.00016130172, 22815.069365092346, 92363.94469939868, 45936.47936456061, 53120.794558805326, 37906.760639078406, 22414.94132931874, 75558.8907842369, 31002.7766759439, 60958.76104775215, 84381.64508775793, 51720.61180503482, 6417.168276526808, 89935.93231667731, 85774.40696776149, 65121.39322270935, 85255.65986905771, 97430.65639268993, 89553.52585973694, 9512.012836910388, 49316.57396607097, 78393.23742424558, 92024.27003469656, 12938.59336269092, 5894.1746772411925, 7630.564704967213]} +{"id": 934, "vector": [26361.2617943887, 21562.286570118562, 45245.03598702022, 24927.133847336492, 65618.75074138094, 6082.890876503222, 15486.801895468905, 53286.59763559138, 59670.22230527708, 86072.71503945196, 78046.58142810289, 53815.30801821509, 21043.442462220606, 5238.983306319378, 53864.30753731549, 20087.535999359974, 4020.3618059796, 34217.390776662316, 83075.56268702149, 49232.350689476276, 44755.64714930744, 96336.76538063705, 94404.67990156883, 29991.826342406268, 5773.676805543926, 8585.838171324867, 34980.84936010212, 99454.24183710913, 51942.791808563525, 157.80515058084754, 94424.23738242757, 91070.0591459369, 60716.15601871978, 70050.75620490649, 69243.6317403701, 53055.12617496061, 78284.53274435186, 97871.9596865361, 50893.897605107355, 93406.71993979029, 86914.02550410423, 88769.13969877924, 82703.72957467189, 38431.430867441595, 71932.23559628171, 59273.75806343175, 3686.3971178826673, 5274.777165149491, 51989.2161747962, 46212.75290799226, 19129.553787973087, 90764.70006016454, 75944.0314077316, 4077.1843024358213, 18601.65327798624, 58456.8517999281, 90854.9395021249, 253.44811821523817, 23274.235973526436, 15107.96316441877, 71003.03821683102, 29572.707083905803, 14210.229817903786, 46773.15223347385, 26493.215579425654, 26556.966542886563, 51930.068857119324, 23179.55267799179, 18918.333199409553, 57344.13757860442, 18820.02620496973, 37018.70147327422, 29732.767738440456, 35768.584279731986, 75066.30457306512, 84895.0197695763, 80979.13605100838, 98797.83231883726, 81826.0843023058, 8823.152074323027, 99664.44065027525, 42548.774950115156, 8340.039799999688, 22739.940840641804, 4450.212703385403, 83881.55685597651, 52916.86254026644, 42800.890900492515, 26131.788040236424, 53331.071535051145, 10182.032124139883, 94742.08559049018, 21940.61029343659, 26767.264361191035, 81048.16500438115, 55578.56432893801, 42022.27121018356, 1281.344492101577, 19279.232132982084, 2168.6702618975205, 7750.023266224948, 12590.304424439535, 82565.24611133774, 4203.964603776778, 19059.115922289584, 15542.401619105462, 81377.5917644855, 98620.08866921988, 90489.32872450404, 99346.9078764714, 43960.561701574254, 74262.32716525297, 29065.29349865532, 26290.612775759593, 5973.282739605656, 3898.605742849337, 38296.62001021714, 34635.40577713597, 91435.96733418028, 47085.04272836964, 56576.27495869918, 86658.478450657, 15584.951227705302, 29793.398869270328, 79669.47542297115, 21590.001625455247, 99826.10406718325, 38605.4686286462]} +{"id": 168, "vector": [58967.21615711897, 36547.84392308719, 19225.903862387917, 66370.82569183031, 30806.193655460844, 83582.58125038836, 12546.44604170998, 85479.38843423208, 99629.5155382756, 19701.510010145008, 79488.35172539811, 78698.00536085141, 327.06555839493444, 87277.42092181044, 21585.531990283558, 34926.58131647694, 23677.863896225983, 17532.650507441682, 28121.209771776612, 57904.17112701686, 91552.54617431469, 70734.85102705135, 43685.443343451305, 78536.26313004649, 71061.26118717241, 20779.58399299703, 96842.43365435897, 12327.54859132481, 31010.738028640662, 47904.89760553428, 79103.45708760395, 27811.975353854647, 12277.632919898108, 69611.73423429685, 25795.841057731726, 17053.833409208808, 9925.134278551173, 31986.24942672347, 95496.98931675777, 17261.14161149307, 31889.185670320385, 70456.35196737677, 23037.845094046606, 98162.20681764289, 52323.050490525566, 75201.48566348037, 29234.213268850883, 14426.622972480707, 30430.705737382013, 2210.9144149624726, 35886.70176124735, 41809.274317673364, 95097.54573847883, 48255.33931713331, 20407.562890199104, 79852.02275277935, 30943.588274458256, 76096.13722193784, 10397.228887391697, 24501.27950757357, 2677.3719526654418, 7741.505624719381, 23532.444946538777, 31050.093903136054, 57372.80297688878, 17992.10691055475, 37778.31816811081, 11695.254736805904, 26716.132412128525, 35459.15494288705, 3055.611208936404, 60881.17389843437, 65200.71547331611, 64288.46710381618, 28865.866656804617, 75341.18276589042, 63851.441617083416, 90734.37083522925, 72629.54591863444, 24046.990461079677, 67255.56543822901, 77869.21267239502, 23767.031181339404, 60001.20334934821, 53089.43630497266, 3307.8597049920777, 22324.37514299659, 14843.650404047392, 7596.648667499717, 48661.90487476071, 45498.47133135002, 52097.120922498165, 60967.30807787436, 46465.51465864258, 16677.467165049242, 65117.79627663815, 41677.30446406166, 94550.5842788509, 30206.269967556043, 29914.604314865646, 68097.44337043223, 77257.95993806938, 60544.71342022871, 9368.70184863251, 43184.37901729886, 12954.161166342603, 34839.55224673334, 55451.77726896898, 61113.85057098514, 62041.05263532387, 33339.57051508061, 15030.44303393216, 98564.1977320135, 35229.3342283425, 47666.43967707187, 60498.42750345486, 33240.528510844306, 38967.96802596176, 20769.622817843592, 98226.39207897363, 23599.401723799296, 80802.87505413948, 39544.3238759558, 91936.40835892677, 60377.088999367676, 30832.175283257147, 29191.880328821262, 68342.0463473804]} +{"id": 245, "vector": [86052.16131660064, 8355.693180994906, 43086.71490561946, 56520.19515029162, 77002.53229376778, 64283.18282739265, 16860.882000580677, 30958.251909850685, 78662.99990280147, 80065.92487527899, 24100.741781995326, 7196.446288889724, 69126.52595588124, 92003.7160580872, 50618.75817705811, 12454.741537414615, 17408.082625539846, 81365.98902275095, 51848.44791699918, 98733.58575194124, 34837.03684951006, 48179.1263322093, 44164.587885521876, 73820.77497116681, 30694.023710224716, 75559.86368916457, 57975.845615090526, 9776.716637993377, 45169.32613213193, 67067.02748571258, 72818.96437736735, 31880.96613055903, 7421.224306286922, 27609.16719702172, 8323.328842697752, 74074.01310355384, 22786.81414238528, 17783.08769986876, 8360.482636402678, 80280.87012517675, 44288.53341762016, 90260.00132766871, 7071.245574482721, 69854.48883871426, 71690.52896000829, 13784.28588870838, 61281.06734041963, 16215.95965466951, 97632.54571240398, 69165.87561357429, 16682.487396868186, 18289.831964387904, 10349.365224972862, 98200.0497268201, 75926.97094100811, 55018.75479727806, 21823.28769522939, 49245.25569093356, 72429.81157396591, 33541.39092241813, 66658.24079197695, 47326.11956451984, 87233.44440138488, 60677.22814648302, 18187.641146539325, 21330.622499717756, 9857.98461899362, 69073.46073374299, 3444.58013590947, 59982.44512567523, 57953.34338340711, 47014.614417265766, 13058.603886250698, 66644.50361888383, 15640.765921497523, 46446.72379636349, 39294.18701869839, 70294.05070281145, 3886.4416327089257, 7557.15641677045, 49899.27031856979, 80524.47932867456, 60989.92043199597, 17718.032346387612, 87094.2982516542, 22917.730691545134, 64826.98836650897, 24919.885474604198, 88738.58711385766, 47662.33400632631, 74573.20397865516, 88306.54417801503, 63600.88493255851, 92317.56825272497, 41210.7299944482, 16785.614715088646, 18448.39473893116, 7767.858999579525, 9563.508411369881, 81009.57704228339, 47177.467384441974, 89820.43754505494, 774.6848964501618, 48473.336552782996, 40222.45681508715, 65648.9991939727, 14408.596907520998, 5373.581304709818, 45196.19582663824, 96274.25442006426, 43617.60986705979, 80723.97218641058, 83922.76342585744, 63526.05559981851, 55725.027023865194, 97229.82860650973, 77606.67344211478, 37499.24295700985, 8717.391573150713, 74737.52702509818, 50751.67771908128, 61940.95802876352, 82091.464879749, 99332.38239749441, 32279.237795755234, 91117.12630417557, 45296.28486009407, 69119.22070507682]} +{"id": 1759, "vector": [11374.788788535172, 87889.07628728781, 96980.16301260078, 26330.447426177605, 86995.41354392424, 5113.440772629352, 34314.91779451179, 21552.711199640285, 16830.025495702117, 14364.992278919754, 35411.16435305559, 38145.45515213424, 35793.6380145368, 78201.86419007469, 55853.341280204164, 56408.38147226455, 99957.83771825334, 11624.074651031191, 71509.08290437798, 61045.938639333166, 5774.70768922621, 63316.12003067978, 91467.36774363567, 23953.12091601166, 66657.1904389761, 10567.60692290113, 44353.216080399405, 60761.528881522776, 62411.27270094925, 89811.81124843797, 61209.047105669575, 83865.99629572842, 84231.40555722675, 45424.67853386142, 52527.099530101186, 46593.92653373049, 34615.00923713721, 58261.87148934875, 87499.87071544265, 37317.8328897207, 959.4288977817223, 18555.584746378983, 90352.9092260859, 96840.4685677599, 61542.0841848257, 71459.37276857482, 49823.25796685067, 68415.46602873965, 86777.23910371597, 77481.13174094447, 72450.89346286027, 60109.47659817673, 69016.38866929385, 99915.30544637363, 41016.44655940049, 54563.95166672532, 73737.23504015511, 97337.77866449035, 12440.06649333761, 68538.91039855806, 75813.55320344842, 24451.927933446237, 50934.95450926705, 74163.63612878378, 27067.19631467761, 53025.613866972264, 95511.66969373239, 21894.735149808832, 49717.86197371168, 10007.727381881105, 46436.06382200486, 44764.222073008954, 14961.508301667414, 28049.86121632461, 18861.531558492385, 72836.46029198296, 98084.33419390558, 6200.1639364941075, 89211.6199403193, 20207.686622259935, 15052.888962262856, 95492.5217108272, 52057.65540789519, 90211.8891923514, 15753.889817201705, 7535.124726846542, 82633.37775756721, 20777.25703504828, 73251.43956636157, 55248.18947823979, 13875.389075675837, 62759.21489125086, 67531.92750419561, 53260.4994729834, 41325.398195870555, 71282.15515444873, 79738.47795249228, 13340.472182662277, 22014.06475520611, 22489.845402842213, 51791.204996980865, 62389.75012198933, 81437.97089587497, 34.80581795414217, 47713.92013960248, 36205.85165140707, 61074.18795335949, 61202.61620992466, 74759.5330449913, 78832.22226020924, 94854.82978301575, 896.9101761360454, 52412.13523161473, 26403.051119836095, 3768.961009043104, 59575.8926995679, 83319.27025645645, 3406.741700557792, 3660.342244886017, 27102.82513573826, 15423.144904911556, 58139.512109509815, 49720.04951951137, 70139.93271453636, 16707.727181653598, 99018.73862653927, 36874.67707550225, 28312.215437387043]} +{"id": 943, "vector": [83378.70292633251, 23054.185408734505, 34096.42966923025, 50744.82010885307, 50494.4566095154, 45339.44491514613, 80361.8621106869, 47655.66083987637, 75458.47049194275, 54088.52703469479, 81431.86520558264, 42597.20206880383, 82568.65405559726, 29999.03384491487, 6481.074362707218, 71704.07351166673, 67582.59044075043, 80144.42743663456, 40254.185149742196, 50502.780815254635, 81331.96559753094, 79365.18061368901, 75948.31561857207, 34492.38494062695, 51761.81404571763, 41224.92176794361, 92706.17272213969, 43855.5531839294, 82.23068948588485, 7405.524933356666, 33306.488717168024, 45346.13175281077, 23391.719882471563, 16803.182868997446, 12548.542093416116, 36804.67671750016, 74027.20627777565, 99566.57383294181, 86898.2211504162, 29223.567167903762, 42761.575240437145, 84816.0163888945, 4889.71698606685, 92291.20849993223, 88561.87430839779, 22095.525461740763, 92134.87483221413, 75750.04100769766, 70882.88922184236, 33330.83588175462, 75504.18240151541, 24536.132819629842, 42640.21020556299, 61091.71537558034, 83911.78381649996, 70003.77603164424, 10328.59828035122, 62548.94007369898, 1892.9890221532796, 79081.98811932116, 9629.875214237349, 52005.442051878934, 30798.813286714598, 6563.605575176434, 96931.87936533746, 23941.37135085416, 7510.87758013036, 88520.41084144423, 25682.73151543301, 81286.16053215171, 95268.67121233286, 48896.03586930266, 28178.510653744735, 33513.8670716005, 37368.82550281392, 47429.07493073433, 55338.68476724458, 76668.18451198713, 17044.084500414257, 22996.422936996787, 40709.45208763162, 48320.76738228571, 68570.35445163587, 28487.407438314272, 40672.839782637646, 13671.394247072389, 23531.628907089696, 23068.32595635615, 18704.166633610384, 33590.416236998266, 90214.60932247824, 59915.116528830025, 26431.37847812026, 4628.584362280841, 69759.1291698433, 59182.93347825443, 57414.96478822642, 6785.112556180883, 4484.1887128143635, 94698.25401867971, 52105.90149944485, 81474.16658537969, 57263.24010103292, 6172.564905799282, 98671.2499959774, 99098.53099056745, 82391.12491558056, 77623.4998213191, 15893.42698879097, 69561.0332243577, 78751.91245695266, 81117.04071838675, 56116.62291684413, 14630.984620039333, 83627.36073883215, 65576.43745961225, 83864.29373720585, 41040.52498535751, 75770.52389510265, 82984.50873330391, 91945.2380693594, 9078.602731661245, 49163.28984789925, 93255.90763096936, 1206.2413009123918, 56132.29854275236, 33597.010980540406, 85.95426019492037]} +{"id": 278, "vector": [21645.492119475406, 1904.7191761015881, 25888.80350632654, 57567.81980756658, 32310.92474630316, 74642.04363988302, 77590.87083177728, 30788.999558492047, 65238.01181961417, 46007.34496400626, 97260.87329956176, 7748.523914418714, 20009.730598143437, 61094.78015182852, 81779.40668673578, 34591.93627376595, 91706.1825011705, 34103.464649092064, 88456.87170745638, 50724.69700961903, 64466.16098374003, 77783.75641236275, 84293.0987780197, 886.3324329769662, 36606.1598323168, 65321.74552242895, 79875.67488672718, 68504.87077062979, 26463.108133327663, 82201.61728773706, 33030.270816523036, 95353.48582871791, 30372.576319557153, 42683.30477934395, 10699.711516059884, 1621.1330013800707, 51358.209698456805, 27684.533764027197, 13616.022491686297, 96635.81979397502, 29052.637571213778, 18039.340919421666, 54586.42862045382, 98593.35651606768, 42619.63056994235, 28106.55254025204, 74614.86563564654, 28856.82579338693, 83503.64352737738, 10095.409768188101, 59235.39234292641, 77224.29297111435, 1545.7306969591023, 11106.676850757813, 63573.05745813491, 6246.299271694112, 66011.5240403561, 86504.52911716928, 30927.855239594217, 70989.62939143273, 72411.97729351437, 24647.87834299229, 76709.95736825033, 4084.7857643522166, 20296.984400274065, 90017.07851854737, 78984.944282941, 16024.932019215965, 68961.73545210245, 99935.89237099474, 57300.84878725499, 27975.746078767937, 77940.69374375089, 41904.74219029666, 49800.89610299686, 59705.05837186201, 69525.4771016659, 5755.639509009303, 60128.449903892004, 52302.551451570966, 49273.59277143116, 1861.2184994463933, 90235.50048772183, 83354.97617816181, 23141.491600049678, 3329.802302467089, 6732.573325791113, 2134.838105831216, 5872.369973109903, 88488.94331064996, 89483.00001206885, 67359.2803578497, 1282.6401239349304, 22603.53823016149, 9386.054770028806, 32368.539119877227, 81351.20741459614, 58241.906771570786, 63286.169453323906, 8611.18974797065, 12083.410824686724, 1639.3973796681616, 81714.00319225116, 16155.376917590802, 26247.437131223116, 40613.01602552693, 85573.49540276926, 9450.558025459666, 9441.483962640528, 34290.71641617598, 82981.68898576753, 3989.33016235935, 34489.98555672504, 97700.40286871632, 40362.90206508616, 87315.62197163262, 35992.65511572505, 32307.66244898382, 8855.91403508046, 59506.5234558121, 39516.084574761626, 59054.761434358596, 83036.39430069183, 37285.21312924215, 24420.138376816434, 76706.22014006483, 17665.00138292869, 97498.66129686519]} +{"id": 746, "vector": [9650.839694253833, 55064.36107324849, 85505.62005772337, 53900.39543587555, 82445.0138046029, 64993.79602773741, 31610.717667381483, 97109.35837152922, 2949.9378418500255, 78331.66792888303, 1638.18828327571, 9563.027002438528, 93684.29955464516, 35810.20418761993, 48488.51763898824, 52427.188416009405, 12130.265753969783, 12891.411161695498, 3519.3838512725397, 12016.70964559305, 40842.83323316646, 5293.482746696143, 8780.305498827202, 13025.704862768129, 6648.6171594023035, 60786.03112179177, 95133.37492516874, 12049.811185097758, 2608.685888524698, 14290.050308769409, 36037.74438124444, 28329.58599903107, 19685.107520600264, 93646.36570928142, 23440.692510337736, 81198.58635025313, 88787.41753959381, 10679.723299264298, 52683.32743430887, 64536.83772260228, 25777.18706545057, 1226.5781175026525, 83408.54891795706, 62967.824685276464, 91353.75660640078, 66093.34889725634, 86577.31401997425, 60313.60867222046, 12167.34453973728, 154.95634944339142, 25994.615294259103, 24739.738892311958, 2660.0547592042776, 76556.30125738746, 86095.29115044388, 84767.68073396823, 10642.978850986829, 56035.95047644447, 71716.03866995324, 8025.896803502464, 40353.20963701066, 80405.89499527216, 33012.959032198174, 34541.28835378248, 29875.05014963556, 38625.52345437724, 55156.302295913476, 42636.00638372632, 75278.3938483063, 51107.104251653145, 8529.150543800446, 82197.4451755957, 49849.01709639905, 44844.58255019665, 22090.216281433728, 79239.27662140963, 93227.17676501867, 49325.22201464492, 26662.834446762572, 9242.282521807832, 93538.22068835888, 498.2508039591105, 29933.177983218153, 47204.549975662856, 49223.6173992304, 59636.999260739496, 50072.21655236964, 15909.159401852869, 7420.224414607679, 64879.54927750468, 91115.35685755851, 54114.6692203136, 52907.373539600274, 18770.07005122312, 65637.48919087654, 77560.37848348715, 95427.16150586003, 80130.25537780255, 18614.739715871554, 93910.23326453702, 91034.28425724425, 79640.01077965704, 34993.58492845732, 17329.12289281866, 90845.6539314271, 33343.08066025431, 54359.20129353283, 35106.224762304184, 94099.41244687716, 34657.68880653527, 84651.27360953094, 19319.28273776171, 9479.663539074789, 68534.43386584107, 52173.798475357384, 87263.59601355113, 53282.59221101654, 84751.80356204172, 62916.753096878034, 10678.986592585537, 68239.32444491885, 34068.834392185374, 53562.7383729215, 57497.25392356033, 45411.69023934575, 16550.98416519736, 86944.23681489658, 76244.39722454347]} +{"id": 1252, "vector": [9331.51627845592, 82343.08054392121, 91925.5672326142, 77867.39910456415, 49898.21689825369, 71407.96163813402, 69559.61771757201, 99023.77416541595, 96760.64265396944, 9217.277676160063, 1337.243195869242, 11780.83275464199, 27108.04461013987, 47094.38108260375, 90423.23325439593, 11207.22998715592, 15680.87722397914, 143.50165518677295, 82806.61124977305, 90266.11770966316, 36543.13131107071, 6908.79297258733, 7944.0474454537725, 30693.38038681276, 75684.8887715996, 17687.282043426232, 621.7405418458943, 62028.65229897333, 29364.416582232177, 20404.030698184306, 77795.49808060563, 54937.244713372034, 76853.51362094708, 89969.10885440296, 18312.189395627996, 32155.876349939907, 73517.74655513107, 35784.021212475425, 30582.200636618683, 12982.074954130918, 42783.541201433596, 99001.85005590801, 18360.976407166472, 36970.16463130928, 79183.15601971539, 67021.0530814561, 51626.60264110086, 8310.794123175425, 74420.08901211238, 48979.60622389539, 24828.502351165327, 56395.88618031376, 28592.00978659976, 86952.9604601484, 58281.39799195309, 63275.089678176235, 1433.838211169791, 79627.43083849225, 3459.354196045239, 28544.22098922614, 62127.90710205222, 33143.58547881381, 51781.078422361956, 35073.797052306734, 14425.004681473409, 40458.91130827337, 89307.8295010513, 46744.52508202717, 16105.294362073064, 90463.59037510138, 82215.58482427959, 24247.478482153274, 58030.592022587356, 34992.95424984767, 83883.67524136668, 64466.72585557155, 72827.59802420778, 33292.28028442598, 38853.3708408912, 34849.66956732911, 17957.91908296165, 21786.770271367404, 94353.5010741559, 63491.327802612184, 43176.25537046298, 39431.46360327766, 34237.739256311375, 15483.39051042399, 32240.04527175156, 91020.69983534866, 79871.0099993497, 72140.96042318172, 75043.39370959684, 79293.38461575727, 43862.078774584144, 80226.45071008007, 94924.13636335314, 83946.5951424026, 94234.10495950215, 5941.87939437717, 99654.14765320036, 41462.07274005348, 64151.51570566257, 30024.605406596962, 87052.01385320484, 44539.12999243653, 44849.01104966735, 68554.0364369731, 62145.7218184908, 90651.92199122773, 90783.14777361543, 38488.28710996354, 82046.64786785911, 20513.32782920382, 11004.612647215517, 36634.76743522317, 24794.171128047237, 82559.92218896642, 57289.62466389549, 27308.267402448106, 44062.66628901645, 99568.09942747053, 34946.2435380256, 71534.32419516156, 32684.056991012356, 8362.949997929914, 23964.373408612206, 26109.547472108552]} +{"id": 1371, "vector": [52343.562033253445, 39359.554095338564, 49575.47434687668, 57875.481044907814, 86240.76606400378, 9333.149647027783, 47629.29241702923, 98892.46197937832, 39747.538215300985, 72102.37880838342, 22386.62446982401, 59213.4449412432, 30735.789668269543, 85231.03357840996, 41994.768212602285, 92801.99184695588, 40232.366876500804, 35993.77429568647, 1967.855322805745, 17727.19840758986, 89263.71446332177, 51339.45891588978, 10395.104974218239, 47782.7606199847, 78045.03997357265, 71654.144497534, 94639.24386061894, 15384.51414098242, 26542.1235901964, 52708.47494111358, 23199.340753705732, 25656.427721692577, 98706.97235916997, 35121.244463453426, 33180.16165500715, 33779.9817693013, 80482.7407412188, 14642.29046876575, 83873.41910090909, 92431.25809453789, 17708.16115029149, 57406.095057323284, 30969.5006112802, 39904.035895688125, 56585.31342028097, 22267.47389577185, 58180.87517255178, 54595.043951053005, 21991.27355013988, 53023.9723320074, 28058.229029365422, 70993.38506874377, 27166.23742045551, 88730.8872417469, 93185.66653657894, 16976.859949821366, 39625.0168094296, 40412.108226480814, 74564.02722966972, 12500.039583477785, 95596.894976687, 13919.073203696385, 9104.054410064065, 39080.939140144386, 5671.849313898591, 43613.39823841076, 88113.29859198732, 20631.841439795662, 58630.73636374355, 46513.21787836596, 4515.473370165535, 64377.84298400995, 79549.2358497412, 92159.32699658896, 82023.0390871065, 16763.42757226562, 38600.54073636756, 52561.63411304305, 45013.30093515793, 85294.69490019894, 68409.35337058893, 32516.395562319565, 64573.04357293443, 45639.34062698498, 65385.99252977776, 99663.54550635743, 27677.650722170387, 37861.42298360526, 63778.33919535856, 82173.51392567722, 59848.4621415232, 43530.266481799474, 34780.21739183386, 60806.189641395395, 66408.30161796194, 28310.254731636454, 22551.926856170478, 297.41069565597525, 77024.68104552529, 1140.3077134826024, 58605.55221598677, 28941.822429761833, 41128.65298398949, 4555.0234887027145, 48378.46586874629, 3799.6070851489817, 86179.89963073237, 57424.38525027279, 24652.663591763434, 35193.50737672306, 83237.28837219458, 211.0133046854812, 6799.906019095181, 64438.67007509943, 49442.73148480558, 72660.80816144726, 15797.252962844155, 21992.698076112614, 1044.2200132498747, 77085.29645757501, 26578.738358926723, 19914.4502859256, 24077.68098525447, 24732.45887212603, 27134.813963362692, 31075.134472922684, 39081.606469683706, 56857.099944080815]} +{"id": 1985, "vector": [20645.494547702114, 29603.6639548628, 97268.83399783997, 51986.75300578741, 43282.248637382916, 41370.963090024605, 50611.09142827017, 48485.62788770332, 18611.8489566227, 70173.7048045087, 3048.3130923380954, 37582.696333458975, 36172.38315074921, 57999.96993175007, 93540.39047456146, 81828.83434205677, 40784.52497061754, 10306.613851206426, 35001.127715983406, 55552.12787635475, 13735.400584587476, 81869.68543523544, 54107.884603427105, 69069.10651549812, 69629.8976285467, 22618.58698908915, 23922.233356755994, 11056.554774927152, 52429.67769509316, 59527.28240454091, 21492.774957959737, 40818.979489147234, 31435.946575650498, 74936.91904732576, 93294.22705138517, 74914.72999585138, 58920.8277850958, 59957.14423928002, 28282.60559827549, 53152.19240066218, 77882.25194113646, 73054.48343589374, 83762.30788658146, 8549.128305293641, 949.4532455944338, 17296.210164460346, 19588.29837812528, 30378.98369166816, 76668.60825871165, 71418.56560903652, 59683.34330554874, 12765.824066392339, 59937.13533094919, 6720.720892229437, 39145.48132744733, 85666.70099911385, 94326.81547575013, 72934.16581481477, 68571.51767622014, 67220.03747417923, 3337.806100615948, 7661.7479564727755, 46114.41203458848, 33121.66462939212, 42293.66682323736, 17484.147669769467, 18427.570424302896, 95687.09031593052, 66274.74658447367, 86277.76522189975, 12315.513658689402, 14256.472117429998, 58672.45438828924, 9998.65586709363, 75215.85222256534, 11124.16404753932, 75390.72167060804, 72293.90586526763, 20481.68232049481, 26565.47546900597, 61857.34539479519, 63456.45919577542, 92773.54347623458, 58988.24659126073, 55847.19734620166, 96817.50487508158, 69807.22074219791, 56112.18647341536, 40094.1279412222, 18494.96160398517, 405.70326010531676, 90874.83377140177, 12003.59904327366, 53215.47662134375, 70274.78852638394, 84607.10117421077, 45642.92103567464, 38251.53759577887, 99216.11018937328, 47615.564434600776, 24920.122686883617, 62467.27493797032, 19163.278650846794, 9097.379928212846, 91579.87114214526, 46452.98015972639, 9416.693502347762, 65137.058907238876, 53599.03329477821, 12837.646888294474, 34628.88367368262, 20804.908828168456, 37699.81451179756, 85855.14384598163, 41930.718819921385, 32849.81710499116, 22696.921700524363, 94009.32426534398, 49513.709970200645, 66504.3889457828, 56394.63692293073, 22223.87192039149, 52254.1608142545, 16162.284510198044, 89862.53540031271, 268.2334524856889, 61695.89733784153, 11358.78716951535]} +{"id": 692, "vector": [10585.782602637062, 10194.37754613739, 8749.529567192792, 47777.35673855149, 65030.515244909984, 29208.33864958777, 51788.491963126304, 53817.22237628076, 28901.572159303945, 80394.1459440534, 41510.51259368707, 503.61652689058235, 50086.242479926535, 30030.543908247364, 1250.3551477185404, 92346.06708803424, 6641.095056657753, 4597.115982698218, 78681.87975254045, 10066.579717515522, 29506.682659527538, 49687.555676161755, 40300.012641789006, 36703.561670885465, 62887.849133126205, 95288.58978583317, 17300.340168924657, 34645.052049562844, 69476.19206625894, 30204.52626210819, 76921.25328391974, 92141.36589208509, 71748.28989700985, 57145.68114442584, 86472.46090779989, 56206.82310689126, 94037.1260337289, 11416.627576597604, 68927.8975942547, 87611.70005043209, 40691.94548684121, 62081.28876867309, 79125.66621147442, 23527.86243548628, 9945.24633726488, 40119.5721326556, 97717.09859639575, 60443.90523000925, 26853.6744454171, 21612.66009499896, 25436.037192747674, 71148.67786453491, 80060.45462003315, 74458.60452177386, 23391.62663920442, 84093.8799123184, 87654.06367214482, 35057.67961403517, 41734.41223558052, 88111.22630912665, 24333.58566335837, 19050.291250883834, 38509.68393721335, 49803.35893982399, 4516.450550624162, 66154.41574086019, 16785.115629103577, 73740.5409995674, 32202.065878882768, 84960.03023619993, 27979.749605723904, 52511.68345450232, 60420.54095102025, 3173.765674840057, 44600.10136901441, 89374.17878840583, 27877.223129191785, 55205.62630959395, 7333.070076047643, 37907.20379875952, 64713.84826943497, 62905.09651932908, 24988.237294523307, 86904.09049177702, 97470.88606391563, 86897.50360171258, 19252.72273723828, 78689.0571059336, 57395.58840645056, 12709.211378802544, 61036.60867984375, 15966.552418579227, 16558.516033428117, 80679.756331308, 61743.33898843127, 80734.06800066635, 46780.04314433257, 35909.43281746377, 66684.09625646376, 38661.539453069585, 39736.27924647462, 88809.92584295408, 43362.01670728625, 45720.96479290515, 88602.69224408115, 71511.89420754641, 16773.000169252773, 42948.297013165335, 61661.55268536269, 72854.09137769224, 19684.714853192785, 52936.40833708682, 1931.0789463733213, 92125.34444873942, 44176.360137272466, 44273.431176600985, 23317.13264299321, 31304.997587893748, 373.796613646038, 25396.197094027106, 23493.391539295717, 7308.457920693134, 62181.35122626486, 31639.064284439322, 69660.6237268197, 11351.290818649895, 77243.08887792926, 75177.6640953183]} +{"id": 262, "vector": [53079.076087401125, 59696.892723433426, 72836.72919178185, 70672.79325700011, 30170.480604100547, 18965.061103944292, 39182.33192980576, 87412.43588836951, 71493.74462787247, 13192.568045929953, 5988.852930008504, 78835.03468871585, 63577.25197586511, 96241.38139098961, 80545.73541712044, 33102.92805899431, 41351.303458336864, 80785.11196242433, 10951.808629381265, 27175.626713862122, 96333.04156026397, 82353.8883190327, 28008.70360876586, 5223.271825061826, 54053.84435310959, 89104.83946393411, 98464.93319508119, 43005.162257241915, 26902.342478634266, 4305.257786469408, 54583.887715480974, 50776.286552889316, 50240.5515754585, 98471.52406793355, 36536.59558939819, 2439.2445375891803, 39239.54158870093, 82421.56928394327, 6202.337087906096, 92496.84983435723, 47459.65850435261, 61907.31244668383, 69478.30034420025, 86863.68139410698, 53036.82400176724, 87139.83232419011, 12552.578971908302, 98192.15769012638, 89888.28689291111, 63772.34242133383, 95303.31665559864, 29036.691418485294, 31442.046948356274, 54756.49382933272, 62476.19320527677, 32909.9690412732, 12482.274130941329, 48805.56589896689, 53116.08774838124, 49172.315784234364, 10655.487601463754, 66255.02976855039, 22614.755492726912, 11444.822621572448, 14864.05966775528, 38727.467699247856, 72579.80376818168, 51571.52987949216, 60976.51998278879, 50249.88970752582, 39178.648571698905, 71233.23218814352, 33991.47978381072, 9584.843534955346, 93502.14366302485, 73520.74468040647, 29304.430060206378, 76366.65464567624, 16498.817964514034, 15324.470749243435, 9592.439619325332, 31741.29394563381, 5384.713712146095, 46376.69980833605, 62829.53205613643, 24459.672974302805, 45528.13171285409, 62131.71792594169, 39033.49033742295, 92489.36853747543, 3733.8193782873395, 59997.910470795025, 4851.736671469053, 80941.77802801937, 40665.84696542684, 87494.58186043894, 71708.05324384518, 13525.806770863814, 60057.297541536456, 84103.78472952114, 9238.55847942121, 89541.84330443772, 26345.720724740597, 82920.76410131429, 90309.27474649658, 7993.073964665132, 48977.07921429615, 1119.1291340059806, 94390.82402971813, 49029.73454642066, 89954.08740725044, 59392.365554670636, 82695.97580570324, 29927.31141635595, 41114.56957460972, 36196.240574735304, 19001.340427515457, 31090.138226464638, 84606.3496269864, 11127.32207229159, 84350.06829351159, 13683.448903048811, 58447.29339764161, 8719.164410020274, 92947.99091166921, 23808.086219547364, 36034.87180019292, 13630.204609671593]} +{"id": 681, "vector": [22800.55498939556, 36308.970937259655, 60432.76748587808, 91935.847720056, 3044.4091494441604, 85240.57680936006, 73390.06992600216, 72369.91600316468, 98639.37951871387, 5477.508990876767, 31592.569445341134, 18090.601680086216, 68158.63169229303, 68614.74668132603, 6090.297092053298, 77445.41894822405, 91930.64093253619, 96209.59265028899, 16432.303143424186, 15540.56172647973, 21118.25250519179, 11966.88583707738, 99527.36909080016, 91119.51763974679, 81578.85270588553, 35741.997308159436, 72307.48967883509, 38823.38541619433, 91906.63266206742, 80398.15953698487, 69283.52983182932, 2797.268644478368, 79789.65482810666, 24656.02691312486, 65088.55798189151, 17094.326054131438, 71440.77985007425, 76531.21590145658, 47739.45292946498, 48241.14359753966, 9203.553463296277, 68337.11715254327, 12095.043597223343, 18712.688274704436, 71063.12307462387, 59944.85882483559, 28454.616092163575, 16139.502286970775, 15697.078594407287, 29600.508203954334, 32363.915798668786, 85989.67089664999, 83202.65705251337, 38781.25353174836, 34865.83591939398, 91700.35412535084, 28449.199802648283, 75410.89883542925, 63472.888040292884, 28912.607363202824, 48122.85806844458, 6257.858676906647, 86651.90297121227, 5354.582657963847, 14070.832356735453, 94320.5276062911, 62100.29995933376, 3374.0666671993713, 75350.624067902, 14963.0561566794, 49492.190808122985, 72238.6147853808, 32536.85961001348, 73167.89677602673, 88415.66781416198, 41519.632441489295, 27488.080028777083, 40224.18690442918, 43256.53119706383, 46988.54245206209, 4740.955969765059, 10027.344264522786, 90301.32849751771, 13584.576240360957, 5584.055897334883, 31142.37798944972, 6825.573391545759, 33122.80850296565, 60709.35016162287, 85383.78549134806, 99365.49586814016, 44905.47610508968, 82721.6995320329, 18201.77568135115, 56740.79407295293, 85554.89254001615, 83260.1746640264, 76848.9193071439, 31019.14343000247, 24137.28115073348, 91497.6284620721, 23673.196802243514, 79782.3924263707, 44077.252922198364, 93515.3350076352, 68387.21573354241, 73184.55946164433, 15289.203092242198, 45219.16201275684, 19759.40366870651, 37092.96668301929, 89861.03038868084, 64979.14359221008, 99727.39813989363, 64435.88375447079, 35180.734066440935, 80425.24503170737, 29133.075330855983, 34181.3241034421, 83845.87397192513, 53749.677918260386, 37803.00495518732, 89554.43887735024, 94565.85784093419, 41759.90526150502, 15029.51664990886, 83150.13901406937, 13102.342949569502]} +{"id": 432, "vector": [46382.99954408609, 26570.269532108916, 86275.55223705363, 42528.51522858373, 32813.42799974415, 74867.68014248037, 13577.604753238214, 84106.57317926367, 76374.50648534382, 90089.44470807789, 98137.56466036291, 36746.35355898373, 91106.48115835019, 92214.51734222885, 50705.972931690856, 71626.34067221367, 86953.71901707281, 97792.00846061578, 29568.924420581312, 59435.144900844985, 92308.37839206446, 38063.975000403174, 62035.24647146307, 98382.13148364177, 27771.35463420398, 65582.91112318754, 85870.21877347695, 5799.585458131795, 7706.028488385164, 48989.909834662736, 68321.5434730263, 44545.678684289895, 95035.66612934603, 54906.06698571143, 56707.4466981917, 84599.7848851493, 44122.08356707047, 51098.52223046504, 98978.85415363713, 26245.17939348765, 13703.087310839124, 78328.1378098802, 95274.93195914259, 95681.91295673768, 49398.86425060278, 90809.05179827852, 20054.455407909067, 87067.44216387594, 17065.54116918452, 91849.86233683319, 78904.69773261143, 83736.37053623334, 84120.4009595787, 13082.47876100025, 97838.91913213789, 76802.3938988786, 28968.439351037166, 33325.174648879976, 2005.7949939694674, 67926.93397958545, 23598.481137442795, 63825.47528411937, 92382.09255954457, 26126.58028561662, 84406.89228285501, 89643.52893960837, 23356.340268275377, 49787.49715644247, 56979.93138576325, 93437.67318541679, 20938.138574575383, 65980.66785405402, 80569.02670698035, 89593.17373554122, 84817.91636889074, 26804.903043817296, 90463.80780079632, 49510.34712424665, 65678.88119993563, 57064.2241618456, 2186.99061996821, 23773.35686386567, 21085.90469518684, 25564.140021418312, 69192.91183346606, 40669.704539958904, 90326.2539134437, 52276.31186124233, 37562.75586938299, 23381.903653992907, 81559.25511897822, 18606.338709749405, 82573.21248576952, 29380.640490848255, 38660.04681040832, 94846.27741650246, 36049.48200095036, 99971.77527069447, 98634.59526378805, 50269.43656353284, 26698.172762198723, 19749.551697109713, 23574.18597197213, 71349.60903042741, 47708.06682636584, 82287.77422231135, 79100.76261938237, 17237.040917875944, 29121.00816847416, 81426.27209125266, 80977.05625756338, 61302.44148155513, 85313.13182391672, 6290.982126834355, 32298.66404494872, 95641.9934128926, 98491.67487471351, 63398.231898056016, 57172.58862487028, 55950.037753661716, 44307.4850323937, 69124.476162278, 20225.783406651888, 1263.2102272144064, 68297.51960131012, 4223.165375175253, 98957.3937924112, 4130.15363131104]} +{"id": 204, "vector": [45278.410219388585, 48800.70137904327, 30984.15772454899, 75062.19791701644, 47990.60011262404, 83548.97735504547, 57429.22966568317, 33132.525640120555, 25574.593427214033, 39738.06668235881, 99345.24124554596, 44892.43235518773, 2644.2356221251353, 69429.5879558785, 99587.51917374879, 92944.36103582, 92684.65205625925, 41215.232577828865, 24125.958676026305, 51266.53173188884, 8494.652935857916, 93410.0341327588, 17454.76849304537, 77967.9250266545, 18472.077818049736, 30239.700932375068, 4111.746429853602, 58274.150170078254, 64775.529213506874, 94737.25297309288, 17828.212624057684, 38638.510892452185, 10360.740046057781, 78739.0521691063, 19999.26836166297, 10917.50067749675, 31478.92896305575, 20113.643391096113, 74822.57404889798, 44221.7130226639, 10498.787664782561, 50618.93411746525, 46153.586388991265, 28151.720693581294, 26591.68694508558, 99602.4054478617, 65937.15112823274, 67863.55614759281, 3985.7842408322354, 11100.636163299949, 83696.08475156639, 33750.1158920773, 24471.20771940525, 42657.65753000292, 72812.63699662595, 43.474624493189395, 66779.26285217251, 20094.11709463257, 77980.23695645237, 29297.06042174066, 44082.31568619506, 37903.42270373217, 21715.792827597434, 15864.852310463873, 2092.127586823511, 83719.48688712342, 68474.1270254576, 34851.13769526535, 73511.88434725396, 54718.753449768876, 94413.92965936905, 2032.945411429088, 3516.47199638494, 23277.636809421278, 62282.462592781616, 43472.6483166862, 24543.403947383304, 83879.44407403239, 84654.36391960389, 74827.27344948702, 41247.38335882182, 62081.208404396995, 98207.91410879337, 17180.86086878934, 83423.0603777329, 66688.64288693125, 38093.683143477465, 66297.53404395912, 60905.931933241955, 2992.689833300044, 58148.53036884357, 98141.91173078526, 31588.75478306563, 94403.31924055786, 53360.84558755099, 22002.263409437106, 16362.634128275266, 32430.42585375616, 91984.80589757016, 60239.55649242813, 66181.73828501638, 23997.804882144846, 20841.80397998079, 55839.62394420245, 69257.55619527666, 86892.36596249611, 75489.50211656527, 50613.890578269136, 49050.70874927991, 30904.873085029038, 6852.814697638088, 23961.97683810767, 52898.586866109785, 72683.26606637427, 58859.30036453141, 39284.675004450044, 4783.863982883818, 26326.420017172048, 45435.932313718884, 49318.26310906371, 9767.155882450985, 35299.478253413676, 75450.8472257975, 74610.37006062364, 86025.40641334333, 26092.311200916763, 32405.314086697334, 23248.245804013]} +{"id": 1771, "vector": [86788.97219338891, 59020.648623613626, 53984.17560073849, 50446.82057665596, 35339.0664483959, 43825.08388886735, 77691.30696051083, 59650.70369683333, 99861.52306265349, 68105.52473024714, 48997.75310745789, 19531.387874886463, 90268.08035802413, 35049.53022621179, 69892.77722604203, 19671.512131900636, 48573.050019530114, 42434.78813193555, 44666.840736229016, 15463.029769005998, 722.2530880796829, 2080.027039986798, 10103.664739309348, 48073.07312157845, 91462.9239066708, 14018.543582364462, 1421.7736975047912, 75954.53161374245, 15455.706439701833, 44314.019040124054, 7119.086605699054, 92203.97153829332, 6391.929373655103, 73951.2544780575, 19662.82369469684, 69451.05847829046, 70173.01107439224, 82348.26930853311, 20183.650894117323, 28788.54245011543, 72210.45035398187, 84848.24162047789, 48273.153973816305, 58315.581966640384, 71210.30825343869, 73767.81009761695, 48954.41749725965, 13160.396852987533, 14091.764436214282, 10236.587817189291, 52449.005848971894, 10457.544591343893, 86220.44776746284, 67580.46369684956, 38761.873204137475, 86111.90840310535, 99247.36169076526, 65059.64893272435, 85639.30802810284, 56692.81156849641, 52364.80729992121, 75448.75321921012, 40950.67608834331, 79061.63727478962, 12740.547478597286, 36565.67149866814, 89373.72237140246, 69975.55220030819, 1402.9986251573146, 81267.77642679939, 22622.069286610535, 906.8885760843636, 34018.466025318994, 44389.336204252875, 85560.3735561335, 97739.2780221626, 14227.27610550929, 76526.19537561356, 70344.07879008101, 16979.695945131578, 30278.15780127231, 32378.32776310775, 92020.78429518292, 72781.21566069395, 1968.0026792579963, 31716.451618014085, 98710.2803909178, 7490.3350878408155, 84729.38314775538, 54470.35930723659, 98897.17418347234, 43154.10306184886, 16100.281860918674, 74724.14031954999, 65921.71583388347, 50635.57320250756, 36704.57171092317, 96175.69093946162, 28626.503136839365, 75570.37624766593, 1016.683555482778, 91404.44801381647, 55318.44449690674, 50468.17760186433, 23659.79279478323, 18735.001624419358, 79352.40307116989, 33274.65181640454, 35601.017099943965, 93299.30879515228, 63744.50572232774, 2950.84698478999, 93457.66111145275, 15134.201479557141, 84578.7396773984, 30917.428512240196, 81041.78168619158, 7091.206282884921, 4454.8920421672465, 90008.38560442174, 67899.39702193362, 25369.833248625306, 43215.41242631115, 94114.78716535764, 45855.00000748375, 50867.63763152359, 14553.679739118974, 59327.955722208346]} +{"id": 946, "vector": [3220.205439182089, 32314.894444660724, 9708.924361691374, 51237.223735886226, 46733.588967019125, 95835.4746522402, 88054.77326758215, 76459.31218081969, 36698.231103730395, 93045.83788282961, 65946.92584262567, 16608.17617437299, 20503.592038301354, 2445.061509474078, 8697.430660748818, 23255.948333408327, 21766.435570132937, 78580.45873687985, 96244.41008272344, 87377.74857002504, 15364.859515586626, 81403.93718641679, 31926.875097176842, 87612.18592118124, 54530.12885166419, 35740.72925137341, 62121.560190077966, 13873.352810370898, 4895.46229541955, 77138.06377449687, 30223.94185844488, 3404.019682834747, 85223.74938692892, 77652.2298029818, 14846.260280333656, 43673.94871692999, 7854.518028467217, 21483.62950057142, 27462.255615580478, 48720.292836659, 44618.99657308539, 73311.70531496887, 25198.66081177612, 68431.73825787684, 88309.84779447655, 90127.27486221316, 39869.192985336136, 52340.947954872056, 43909.62104363035, 51805.483669905494, 32694.432062693544, 71491.14121221896, 21767.20596537115, 60635.827630894215, 14396.225343782931, 66946.21822441, 66590.10506160314, 8085.232106502205, 60562.82414948606, 90877.06520436158, 7277.515061727479, 89564.60325526778, 16860.11867398972, 42679.79828856432, 28511.808937088223, 98902.72584693437, 24474.91666276671, 5561.584570794942, 53023.668353011955, 50339.82545404396, 52454.23322429498, 73714.7969206306, 28948.28362010814, 51701.34608356637, 31558.91408895609, 50050.02276423591, 24303.686238502054, 37624.84336369218, 61297.79429167577, 34770.100031617025, 36341.976741294144, 53806.593029014584, 37908.85035363597, 3408.89596592735, 42000.64608639601, 29144.443259436448, 566.3563604596966, 45489.3442739941, 80218.72427512983, 63449.35095120366, 35518.88849082641, 88232.94165791642, 64089.48796316051, 29255.846785237947, 96173.73319809974, 11305.414315320617, 70371.4780614358, 5894.164959344861, 3868.572330126063, 58362.4789910621, 20045.313663592413, 64553.06916163648, 94107.48987138412, 31948.32734203741, 97000.74982045453, 43646.075619264215, 14803.840472729356, 73032.19106428148, 76880.04617372426, 42452.42154646695, 8638.626072409716, 26858.993824152643, 75592.33925358996, 54823.97989073169, 82669.86587478146, 41053.16591401762, 54852.23196993277, 95517.85629057928, 91779.14316940271, 34652.77812759435, 41076.71586607107, 27996.771709711542, 97680.20892470924, 50737.225929454224, 26700.910898960617, 93358.90305306416, 71937.3833110875, 8120.222111021647]} +{"id": 1220, "vector": [36309.08511299641, 48041.28030735126, 30349.79820506648, 30897.212289540486, 61420.543635545735, 70916.16311452577, 84778.9196398892, 69592.9446206461, 59249.38262322462, 75953.11108545156, 64382.830407102454, 8961.537865389979, 85083.03287381014, 86670.07536056063, 54333.828352601, 646.9710321167277, 71075.23019159377, 50321.885714008604, 68677.99876575734, 69401.9395264469, 70922.51834469632, 5287.075277600051, 81458.43338061581, 50149.59808604331, 9500.817666510153, 54311.59271356184, 38459.16075737368, 4994.368400626603, 44403.13804367682, 71152.6305037379, 99483.43449631365, 35712.62573805715, 23679.08606370488, 81258.06398711502, 35619.7218241115, 24270.39967383948, 9584.495559378693, 3049.4451473304316, 42206.146854014994, 80389.8254428096, 69298.90852717096, 87654.62983187127, 20678.788329576837, 18144.83584041826, 93754.32275381213, 56831.385501910714, 59568.29021711741, 89429.90939261975, 32348.928338913385, 1994.0141671637934, 77787.69245565061, 24031.770043104338, 67160.58993275376, 15113.622566769336, 32646.99925863065, 61813.32822306373, 52250.763691725486, 97858.52256617758, 71427.81266589563, 15367.326972816487, 72571.83616755376, 87158.20188301454, 29854.140601662817, 49249.97070074346, 21266.76239585692, 31101.503178902225, 54562.573222961175, 68002.58813196201, 6470.52770855755, 96283.44893641281, 53626.38566502015, 69751.19976498965, 10772.583352062104, 24583.147592696852, 2755.7219482895134, 47489.115650870495, 29541.82180009928, 56772.801277826045, 10479.29435813899, 20543.484315773487, 2410.8362242840853, 27608.516420553042, 53675.22489478145, 63585.03796595664, 47886.768486428686, 13657.652216033555, 94260.64099306936, 85957.08192217913, 31338.98767309925, 63038.14435177094, 68618.10014023287, 55965.77908293065, 14471.739295151476, 44329.82640262036, 73022.22021961294, 17972.390810746307, 62395.618685690926, 89934.44469373855, 30401.32456742015, 36511.17265464391, 85125.17751245435, 16192.785024691226, 18850.232861271143, 35839.18617242461, 28093.32907025466, 88502.9370069743, 37145.720943996464, 99384.36427971111, 14486.32925981177, 46653.46251151038, 22065.67715023986, 80204.3171101538, 35735.24702615885, 73973.67493554928, 35004.86831006537, 52140.05485786872, 33259.660651301114, 69383.1866901668, 57170.81595888419, 27584.691237915904, 78377.06130799673, 43692.1091335282, 47536.43306944859, 16266.756223087243, 44119.2091975817, 63160.349391110074, 37575.64733027533, 94326.58122975893]} +{"id": 882, "vector": [79919.2167525092, 75511.92030184159, 40204.30351331998, 9566.586606326255, 66608.84563149197, 70819.59187186397, 61748.05664671521, 25466.16819571229, 66341.74179584745, 16388.762385705068, 77015.77136714244, 36390.72654341537, 32872.64280971341, 60999.437848588466, 28603.38502459333, 77943.97514733025, 3973.9864424516045, 37781.928771978055, 67935.91043494834, 8867.14440328562, 29123.98426826538, 10189.280774109287, 7098.8553903526545, 23470.86209710395, 4324.190941432505, 34546.647621413176, 80011.83525450193, 63525.34643325064, 22356.591322323606, 95467.0953830917, 86102.71243254471, 31520.45195819314, 14597.050191698014, 8474.887171052114, 4794.535508679176, 62246.40616043211, 7318.448663155375, 87649.11611430018, 72993.02365382957, 43795.564337671945, 43576.625437649695, 41888.72583477714, 33151.83976524858, 65687.2543928588, 1763.3857562638245, 18938.106354831994, 57081.701072398006, 21452.89666396012, 90818.55458822299, 94093.90379799657, 34952.718992002294, 86043.75552867583, 7500.995985535641, 60072.867442806244, 16772.19739191289, 49718.66908275121, 14925.312550595516, 85952.51444090864, 80248.14666210994, 77333.35702501635, 29652.889510675173, 66208.73220342779, 60388.02119955924, 11300.706491263234, 14719.104109339132, 22558.524695019412, 1.371374321457175, 1407.812222706062, 95204.01614806964, 59650.64475782811, 69941.75002890688, 97063.1549074822, 45116.241684712666, 92958.74992737478, 13079.457596203138, 30121.375911919524, 23120.62562086621, 46657.59253125582, 38017.0033076695, 64303.69518574425, 93290.05582288493, 64955.280312950956, 37603.758390711206, 12093.705341502813, 20534.661994958893, 80149.00607949942, 51445.14073522396, 52113.50090334381, 1877.4179245500843, 38979.02523087648, 25331.674809956505, 29887.373823796992, 31750.916016575702, 25073.368220398097, 89260.01929018153, 90366.59637851722, 35871.10184168093, 63413.60301700928, 18173.331637416988, 84175.96949500382, 84724.09366182233, 90667.03899675522, 61946.356888212176, 82713.12585056276, 70489.13605885352, 58108.08638028917, 95674.19255230317, 62886.01687291666, 27878.162170827614, 57440.305786743615, 80373.80359352745, 5275.518330801144, 74400.51980068836, 18005.293755488074, 54099.93530318649, 45864.75422337372, 67837.18260456341, 69486.99795565702, 43240.91118294331, 5412.402400848515, 82626.66033964315, 69416.70929935171, 38556.41638069872, 77994.41859242268, 53835.87806634558, 2149.82469910705, 52446.56668289964, 40616.17383465903]} +{"id": 1578, "vector": [99479.64272493693, 9554.064431811172, 59783.18543569236, 12345.126606914058, 63520.13126499412, 81415.2623824191, 4072.6458577275303, 21496.138295354438, 75543.14972989698, 56773.96070556781, 96132.93809263686, 19670.127136827774, 38671.80801144013, 53789.593677752564, 11593.88807214351, 17574.970846125725, 9587.39885411204, 69382.12135152405, 45254.1537797229, 7399.993385527392, 92138.09861361938, 51453.84290214614, 73951.09765912923, 9668.233352060251, 3013.1651131449667, 61800.63339490097, 16007.987457406669, 44153.07833053828, 33261.70681871625, 18917.660736300935, 50593.318192886494, 57762.412892902415, 78508.52855543919, 51606.10703366567, 98912.82744471553, 89672.1879707277, 83725.78750623744, 55753.683348187835, 81985.36935071489, 35966.12513472055, 2699.428003029813, 3058.9996793836226, 26016.750985381033, 88951.80652175823, 42821.23835087592, 86779.30035023781, 15842.339151276274, 13709.769918220249, 86007.4285251395, 35267.570244755254, 46621.13329381059, 20004.072436340226, 97065.22651764395, 56049.07254969187, 55256.41277767365, 41382.49540740443, 77552.17886946286, 83259.57814799646, 71442.81909492635, 94292.93957014814, 40101.814231085475, 77099.28662972215, 57376.95730195663, 4256.496991710068, 13354.647117684648, 67966.05999624057, 86328.08318576777, 34661.06793043777, 70564.12587086752, 98496.83299493298, 29551.73173468162, 55133.24800792163, 11856.179918399446, 53154.8459896227, 38607.553772813786, 53987.689011196395, 60147.89042956112, 74316.06678132349, 81710.13971589279, 30272.373156233956, 84286.66431821286, 71883.83610178757, 51135.25601585476, 99080.65268790393, 17789.87992384723, 74963.05746236799, 23157.418663210916, 84674.1565666425, 63520.8333066718, 3632.31173125369, 83877.29505866134, 30607.50949810882, 22110.9767038255, 94210.06447319541, 49396.09485159623, 6717.685619170732, 99198.03913301232, 66709.51820266974, 34185.91138788799, 4720.642815265685, 4434.716616452417, 59521.59117976263, 20406.35027656609, 47759.22819256959, 57131.57453350137, 28659.860029611984, 31233.16830988625, 88341.98600999237, 19888.976347667376, 12433.937787628534, 87449.65895965384, 31995.320042665975, 66412.93731253064, 1033.0903358433675, 4821.0482305221185, 30048.131506038568, 78752.83893076025, 3862.692629832587, 59966.503808906105, 9005.656861029376, 25579.247938271266, 98752.03539920517, 69960.34687604442, 13127.876861769771, 30174.616007906374, 24924.162135068185, 76471.84388349351, 81154.80029744425]} +{"id": 1575, "vector": [18185.4149252057, 71251.56955242847, 48424.157929445544, 36426.283686196606, 24074.248323423773, 46973.43458175115, 99722.82818583689, 12946.172953812296, 20008.930917087488, 21594.46336560097, 83444.28846817605, 90530.13796563268, 39498.6988395076, 9781.421067310646, 2953.742302164897, 30353.136144950156, 53647.75086552345, 65364.63632875493, 65162.20875956131, 6773.876644761678, 44435.500021646236, 20881.889469659054, 81372.18871331564, 47068.317343245435, 12692.133176143827, 87298.92909130965, 76056.92912045745, 41735.31891497545, 68910.22997136247, 27702.64308438232, 4382.956885444478, 34088.94678095115, 4440.319745542421, 29082.093869091917, 2682.906494493942, 90235.89154335055, 96718.53124852867, 46719.89054388841, 9974.720409945116, 27443.253789703325, 85471.6217291986, 24099.879325251062, 65715.0695691337, 7547.5274902444435, 98575.82326311481, 45715.76532955526, 46850.53143043404, 73390.68980019433, 68777.98987292605, 37994.35362801977, 36665.61146805377, 96064.85388863327, 13838.675147465228, 78874.05805488786, 43017.57231949016, 63610.354513822334, 34421.80220506397, 73212.7644221382, 46657.93960348992, 66360.74483960512, 70271.15223211836, 93700.07643737282, 32929.731295107675, 24888.657000691983, 63414.19575824249, 44754.61945264939, 86599.48643380792, 11953.909604916778, 73538.32682015418, 6606.184737355303, 19653.868759637917, 43673.303141785225, 10442.88425861053, 27368.28649913674, 33418.57008867889, 8910.721256266163, 95395.12905456197, 55535.26880154667, 83185.28065892046, 97824.07627006949, 6070.894768992852, 26199.019464541463, 33912.98377753834, 20612.27842372494, 61543.192903812764, 22154.39301489556, 29866.897836231055, 51089.00935202735, 35418.71188434631, 96012.0475969277, 46919.504394224365, 80510.32450204357, 53950.94395878858, 90588.92321416263, 2999.7356393862697, 85849.58750468852, 87073.2037743723, 30158.602394038913, 30736.81600181429, 11477.997891751957, 73624.37474226157, 73448.03151836699, 42500.160818811695, 39073.96008965488, 88380.48504310376, 90494.2310754719, 36092.67655393359, 79267.54784872475, 36936.791602838246, 95278.62541495422, 90413.93811945818, 43705.63046567058, 58315.485712834925, 87130.56966752108, 78694.82440305877, 59855.41665768834, 68965.78027678726, 4345.266950547688, 23217.672302420033, 12039.362516628993, 25851.187184369694, 47434.72493649419, 55109.075905775084, 53444.453284509706, 98218.01606250618, 27251.82053730589, 6437.284640383689, 89688.74847716764]} +{"id": 1979, "vector": [45216.56546303309, 69462.78181622403, 35554.10619992353, 87720.71979522976, 19769.437709834514, 2639.4231883665743, 49170.304603468765, 65330.03602801492, 54671.85338007007, 99491.23069414408, 20296.19037717787, 73282.48271509183, 33145.44778911975, 85908.94913383643, 14549.101430679411, 51813.73214396515, 33244.36055063075, 11415.155178271918, 79043.90301582277, 16525.32022296922, 28212.453141524897, 89296.95676871885, 40550.39067159341, 70356.09642939732, 48936.105922448725, 66746.66767416467, 41270.30450856969, 61296.35743122039, 58051.104127951134, 71516.40335474299, 65482.898525765246, 75040.45316095793, 99776.62347512686, 84847.46232227466, 20918.81321019281, 71008.96998558917, 90543.1752391681, 92395.86390038078, 2043.1903617872615, 39400.14343387058, 11477.089322904056, 25810.651046714695, 73173.42460101159, 71950.04171124089, 49482.641039801754, 67967.92933019753, 25580.236049077754, 57043.14433505264, 75674.4058121496, 50341.3017858921, 6946.114255345137, 88487.13777062189, 84485.83705573068, 16382.910524092653, 17804.21931096361, 37264.50056559041, 52535.24266998963, 96927.00258262688, 15928.69767326771, 51909.56728302981, 60692.108057040874, 46767.3846390587, 51680.58430748852, 86603.9888298251, 44438.98034661039, 93992.118597579, 46986.405803067355, 74954.99410105398, 59931.03662820738, 69154.35715181075, 13803.222510697, 57975.98112066904, 86459.89169827537, 96035.48654448181, 68697.84232946509, 50709.16473200989, 65402.06854701559, 60404.70640540553, 41156.983762683674, 33672.818791043246, 23239.590316866597, 17556.322869085907, 90056.61727197577, 80418.54846062581, 6591.491090636848, 10422.947721326193, 42789.37141405417, 69962.99171995759, 40904.684139643054, 40721.397239549275, 16465.647589042608, 93131.47805885907, 28740.14266798981, 39141.85098745915, 92195.13218109733, 84282.77905197455, 481.03849086525986, 97148.8623852955, 76333.85542463166, 54014.470023967035, 68975.93431955988, 86009.46045436316, 70149.59965957123, 82568.99763669592, 40264.03454906483, 62963.809911317956, 39163.90734566453, 11545.120323603498, 44860.93369974531, 4035.635897998513, 43758.86890143137, 33388.027452800714, 61955.25705018093, 3031.720834783358, 33160.50919986398, 88381.88674884429, 10141.10474915858, 35659.38867021166, 89847.05156866123, 72744.33971248347, 66610.56399522925, 56320.407534346705, 57193.86528763966, 41376.01005691314, 71965.32728888051, 28683.492294103762, 8955.433924564748, 16656.888889879996]} +{"id": 217, "vector": [15234.889362418091, 78629.5592059779, 89731.47109652606, 83817.03438739404, 85151.12736808999, 90277.98794335524, 44142.181434213366, 40735.39327636214, 36441.72866443902, 96751.85157978829, 69305.56237810456, 92624.52411327175, 19470.156690996744, 96613.33328627468, 6018.822288580317, 75535.05579814785, 42430.56261990957, 208.82590615789454, 95283.48186046463, 89253.26918754088, 88385.82110960802, 58772.802335882116, 34798.67465099419, 5869.145855274971, 20956.959060361678, 14738.019025558913, 22.19184875580371, 61790.774499926934, 58342.289975478176, 59777.71164845128, 25754.76306768596, 5172.730624190413, 21583.96186552639, 35104.985860732915, 67930.98172197708, 32288.436965372213, 31196.725264064262, 35867.02974518587, 67027.47086217457, 25573.72411633385, 73359.97618691444, 41009.721515790174, 46794.12191458325, 89744.9736197951, 72140.59386933233, 73516.60255034929, 54298.44337718107, 22306.0778857878, 72021.75680098943, 77582.23552541372, 87384.34555869113, 67680.83492476938, 32231.28427359945, 719.9077352306405, 66327.69179312195, 67893.39681712814, 14097.18408858075, 43488.99941824498, 46366.2132766143, 31378.936669184677, 26816.751609971034, 89561.43809085095, 96511.31409914746, 80070.78200252533, 19483.215310436954, 951.1030433612566, 45187.035162760214, 82587.10628561115, 87528.14168724138, 2066.5015717307724, 659.660227118275, 21361.65675644096, 58126.19575828005, 94729.54128574803, 72840.59605864233, 28664.35405587546, 80374.51912522381, 57868.87762232482, 55695.506980372666, 86866.29954958976, 41768.15574828219, 28869.094629746094, 46380.34210633314, 40492.079966062876, 67277.07724942347, 9376.39441292375, 36020.00930848205, 86765.23006737677, 125.37851391112298, 14676.377152686138, 18262.739456210962, 38027.51586788899, 73076.28865185355, 2818.0701703201194, 21443.721260226033, 32245.257451170008, 35765.62291209935, 52196.767441941694, 69949.08802806919, 784.506465597401, 45822.443845728434, 92199.36770380787, 89543.29413878416, 88421.9690011878, 76408.2494652335, 58518.57022212273, 39068.819098234366, 71529.56933977242, 66997.7814817666, 363.75224850567764, 64869.840111189944, 57306.6199594858, 23015.79064643774, 54283.98535851332, 32004.412223749023, 43760.13186035796, 4113.95927398489, 97880.15273294829, 59106.904121561645, 2678.9795219488324, 75259.86713933258, 52997.53278173085, 12128.048269626146, 52809.01701310075, 61181.611325791986, 56174.04261388224, 44333.73255129554, 76589.62504193743]} +{"id": 1490, "vector": [58070.14498447154, 72725.82664079634, 69653.45592050934, 72138.02639898048, 52542.225361634446, 53076.796885954325, 42334.67387802733, 80212.96374146437, 79740.70485596903, 5430.460617440225, 67170.8715141898, 76475.14341716286, 18858.46346035008, 78939.26387911223, 36795.22794330168, 48092.57630033805, 20802.45646910258, 2389.161604881207, 34151.984886248676, 27868.736593625566, 10977.427472797273, 41316.68401754881, 16232.24727509045, 15155.411267412012, 46882.5448486005, 32922.95399750647, 46741.957686853, 63817.25167011183, 95899.65150731297, 14289.657846009684, 85157.97387712439, 95349.51833322294, 1275.5528109223978, 97391.92191242849, 44202.156762256396, 4131.627117482905, 63517.99580909904, 43593.85164650525, 53456.85185147666, 8741.776999510586, 46343.56001689923, 74774.42723620948, 80836.38893368108, 6005.707198486487, 13047.920811954162, 43737.46601801536, 55185.02932435485, 92943.83479425653, 83697.32652569827, 74111.08291279452, 35407.83591548176, 49126.7374309798, 80605.14200118431, 14885.154042809578, 57203.0813072223, 37009.57679411876, 91628.09929451795, 92968.31510543551, 57763.589298944775, 13622.820850473994, 73851.65016942659, 75253.51300908624, 27750.903684208595, 49434.76345589939, 25202.96762966813, 45624.7254473645, 69634.97169813704, 67662.81745020665, 63305.1282712559, 9665.481418138588, 74433.78399653321, 52820.64071490266, 2221.022437490794, 76797.72725913607, 49992.30550577907, 11089.971196030257, 61304.80030008537, 51296.734664019474, 21794.294542755633, 20311.775333488036, 14099.487205260575, 36281.70899939099, 27963.4204187859, 51807.440892640385, 65078.73730034131, 2691.335466178546, 75019.49902294033, 28180.562845071123, 32115.194593325126, 56347.601557489324, 5625.4337155007715, 27582.937057369905, 11018.052586824268, 6990.570363456894, 36817.43205324962, 47423.68935061998, 88547.29021957127, 28249.698062534502, 92764.06119834732, 58085.25585124519, 83406.97401547998, 89469.10553706306, 89590.17380396008, 75547.99841101673, 65389.20591191613, 15508.044670396348, 6576.2742923425985, 7708.0107981009505, 93441.32117306066, 72459.9781383333, 98693.01209278188, 38247.859395039915, 99612.89922351614, 53015.097907662064, 68828.97755225135, 65971.5647244218, 15580.391676510286, 63739.97745000117, 90235.89169550544, 97949.07344635016, 47416.15275154358, 97373.50550689951, 10039.477869812563, 63717.82532811008, 13062.851525506492, 24461.873709537096, 98521.21977482978, 94129.4604647456]} +{"id": 1429, "vector": [61216.210403781544, 54108.632732768914, 98823.9349661335, 78436.44000386437, 69745.62369828497, 26426.674425847254, 5358.185551113847, 24883.578418907593, 58448.543997870685, 27485.293269523103, 41046.10490904729, 38032.66169870948, 21033.190474578667, 80029.11890008695, 87870.12102935098, 73151.02919601585, 39705.2799646322, 62997.368125819106, 86349.54973209185, 18429.5653127193, 11097.800150960635, 94445.06748783593, 81725.92847350323, 1975.6676380668137, 62682.761399363764, 45643.880141903835, 40341.927379217435, 10778.353330915203, 10056.038484583407, 27648.300215312294, 10282.53325783327, 32187.45075333724, 2787.990001076801, 3707.395011577652, 37931.78624131743, 2715.27550305114, 43489.93098818202, 75801.56300329263, 50612.771275864296, 49825.52968487123, 24396.494525143575, 86020.45571671812, 71412.3299651119, 3119.25790805776, 29325.384909498454, 169.55734685457634, 96761.96673678418, 82081.59829162102, 98503.39533695222, 56255.158436128615, 75681.64542626213, 18393.799583331573, 3456.24198995228, 35796.8460677136, 6957.608134081083, 94064.92917445039, 84632.3638262087, 81105.4470884073, 51326.97158349493, 88794.21641274435, 31846.91966291069, 97282.78501536738, 84823.3098742419, 69878.09672067598, 25831.975818723142, 63587.13706799503, 53017.28669783469, 82719.21269562385, 35733.98968409376, 9742.329494443647, 96737.32747902392, 3720.725784984924, 45394.81458925495, 14973.257859448497, 56634.26248331385, 92040.73532271296, 1103.2045140305424, 33098.18760796091, 19676.257939448216, 8549.928447006938, 3283.4089267819277, 25038.478702728105, 78763.74164422983, 9628.588412456818, 22019.846542335687, 41617.99372818768, 90425.28648079858, 2431.3204147883184, 85282.616302124, 20209.15056176349, 24255.760389008054, 56968.34428205866, 39310.02560368481, 19380.888989593135, 21628.993025291886, 63798.891309903614, 31057.085287292306, 51741.8575830937, 72529.7797330534, 21441.9212625387, 35888.93147854127, 17239.866168595985, 50954.47120749249, 45383.58216232834, 78926.75690836838, 97358.50590692314, 8624.982125921988, 24777.967743412388, 66635.9537100121, 51827.94788337808, 99899.74733221222, 99044.93417442797, 6485.055688995755, 29457.22670204767, 58226.036214434265, 20211.860349255807, 82136.18105552971, 2718.8828279982813, 47680.67163460037, 65525.6263429315, 90904.61620298692, 41025.913757026225, 61294.57065722722, 77357.75418440536, 8847.786747864538, 28364.544903767495, 63469.76909574261, 98726.50262007615]} +{"id": 1725, "vector": [35579.09020128309, 41235.605532084584, 40264.80675901557, 24527.639271429725, 3669.5826614073444, 67428.55676238477, 12413.03278949616, 97155.81767473134, 21206.66057316808, 99195.18624490767, 77673.3609452823, 45246.12405805993, 19083.91699866444, 13593.044287836108, 37844.55399211049, 13135.277195252414, 59794.53833854617, 39548.420607337175, 51191.85343935024, 95157.193114032, 40219.252636759076, 15745.012768582788, 74951.33999099617, 70998.01274343503, 97303.79722154618, 94252.01814033171, 83405.25456971703, 78273.1866820481, 44613.70235395411, 33860.35365388024, 31787.12249750647, 81321.294399037, 78385.58571821533, 78602.49288130012, 80644.16297428632, 53391.464150816704, 35732.239358252686, 53570.3364838088, 32998.33349229444, 99112.04475159428, 36719.75887332187, 56248.6476170517, 66307.47152221845, 98110.30091069838, 40054.431085403354, 13936.291474869866, 74744.47494241457, 37118.94869188819, 43994.7794800329, 24634.143041414824, 54346.02040185629, 95460.56974314454, 78110.35928108876, 44747.670197673084, 54249.51900977398, 64580.11172642896, 73122.64753519424, 46160.77223649634, 91863.45579557655, 58364.376409628836, 24863.742318683548, 29262.11109900023, 52210.638547801835, 28695.992310181795, 52105.76705765425, 77087.18134788997, 27985.86438094849, 6790.653540591818, 81012.07983209904, 95123.16399002742, 26891.051044451975, 97345.51523013519, 74170.53953322634, 5500.654859458298, 34892.30143957322, 6999.983481068905, 33518.65580252937, 17790.21190529161, 25921.741434455915, 15900.823357015215, 93767.54276816183, 4757.589285156294, 35626.63981916932, 19097.1141811634, 48046.83789986867, 92030.20800056169, 51534.507037298295, 53823.00086383444, 11827.918457683296, 3077.517084945236, 33253.40198930581, 2528.177573942403, 83706.40245557501, 77288.80392754289, 8900.22493187017, 38134.323152013574, 52804.72220111998, 57533.27009398769, 54897.83022794095, 63524.13493544613, 39153.485633243305, 41362.449392929346, 31667.54100845741, 26078.711224489547, 715.2915420674976, 58464.70282840466, 5403.072562753664, 75476.49717604545, 88412.891032023, 38543.06974672006, 92029.53074778701, 5937.9815549606365, 92936.17537379658, 74238.47489467065, 65949.22386706011, 34391.4338581152, 54773.08141661961, 18747.3050107124, 47607.18124098552, 56830.660286668775, 23933.212004465575, 96648.63445738584, 43791.676005122325, 29301.769658232435, 8145.755894434603, 46277.37839074134, 57639.95252173962, 50023.63620722223]} +{"id": 1704, "vector": [98826.10101810936, 28938.811616497405, 68658.34235852894, 17272.51269884328, 78765.37674558922, 81100.257734264, 55209.68255278528, 61546.12966894384, 21676.09204309625, 87857.67187120103, 68167.22856519675, 34055.268162662236, 6027.772388201757, 3149.923411593869, 54883.82124186904, 30275.514255331414, 50701.29329804702, 87399.91283327689, 53234.01254950248, 61994.12252549296, 9526.399761304194, 69948.56960625085, 40690.33249272698, 48911.55449607691, 70252.85000662606, 61342.945440173826, 74672.2816699523, 36257.78862848643, 30809.706703309294, 54859.018824863524, 88802.09645412762, 80550.74820382023, 14695.73740228537, 46566.246974682, 22185.596360614567, 841.5613669579546, 88563.59008236797, 80596.04034063738, 2359.4590879918333, 36376.12909035199, 93827.23856770218, 76783.47731578529, 64581.00454740477, 44127.89923269685, 14713.859045961197, 22572.22165635997, 18811.92905741026, 56597.38212416002, 11560.978931074573, 19943.240828040554, 35163.11373889844, 7790.16481428082, 76868.02502435152, 38386.041710028796, 98940.32982392245, 38383.38503052286, 76466.76139254254, 56084.26819251079, 34372.721561042206, 20257.71793214137, 59575.37649437768, 47921.22360210712, 84224.67934764095, 91330.52140300594, 74706.92498997583, 52986.712667505184, 38904.21640508339, 90631.92984236081, 93978.41658117346, 30347.382345391117, 62917.16685519501, 77787.01866803522, 74007.62755418467, 24545.711749273723, 23305.714097680862, 67586.00728563327, 75089.04612942811, 39900.07980104887, 71994.77480647879, 85192.54496578885, 44724.745542563105, 40850.03737533744, 68331.40326829832, 24537.367589135294, 25629.860545152605, 38110.68059717651, 16269.056005358063, 95728.8159127646, 23027.38213313219, 80528.5247727283, 76306.32605440763, 87673.07594696176, 96464.14393279933, 3465.715537654812, 76172.15701469392, 74601.33219479871, 9997.740082420514, 14597.141424550475, 80866.71329717325, 79865.95628542866, 99106.12315930905, 12900.467300489016, 12201.845550304679, 69110.76777113647, 42813.92571118319, 95928.97442173402, 5093.660245417464, 11986.0850271996, 90966.56958357619, 72093.30709587851, 64218.677380309164, 87000.17236967142, 60378.70755072089, 91260.03006017825, 20170.652940591295, 3784.8667632227275, 34890.51665484242, 81784.9247169218, 3704.458792357523, 19034.33986502062, 57640.08708173848, 86312.89989403421, 1492.6381575624869, 12771.62046333652, 3868.284864751459, 80744.45232479299, 3437.9785054575173, 97882.68689138639]} +{"id": 694, "vector": [47206.92795246954, 43925.05040652525, 33485.959505474835, 12702.238403876887, 63582.08849328063, 34704.459876828485, 73594.87514120471, 46167.591012103636, 84567.53429308489, 42060.340418368534, 87049.6572777459, 24913.597324918057, 2300.721387183624, 33063.5667340544, 83234.47734571852, 77961.26270437846, 60692.425064715775, 13223.907875246643, 70995.41947446889, 26096.96753756251, 87442.65716166419, 35088.8761659128, 94554.82954267078, 76925.62633610617, 54365.53452472806, 79083.41971642054, 99439.10550591936, 43029.80807387252, 44466.822867427414, 81284.3271440947, 74916.88983139304, 50599.459659336666, 2047.5572947561304, 36716.78527689948, 22515.4819107946, 96788.94908847094, 93220.10146491809, 49270.60349622622, 99712.41193042147, 68955.56595541192, 76747.61275521545, 25345.82255294834, 98123.69819968438, 956.6388616455979, 22510.679889383413, 95307.6050558143, 15805.936572166402, 73357.80143309286, 74629.61273681627, 15709.985120289894, 30201.38321221789, 75619.02790807713, 76602.30263361498, 10181.166707312472, 54368.788680033176, 78251.98045686627, 98636.355729907, 62328.003633838656, 85803.98021682163, 54737.42151829826, 31699.84400397312, 55934.74585091115, 98604.73217979365, 10810.823028173432, 63082.15318340329, 31276.133310243127, 26028.197990547586, 65122.10981045719, 90160.99501766401, 75320.61368375509, 18420.97328383675, 62875.12564874198, 44928.54683595348, 89324.10968478494, 39681.29257033733, 46544.01396089144, 22543.5675640381, 31173.940588631354, 83618.93572881783, 92983.77838668438, 68742.44046104136, 36741.99861309156, 75879.11437084763, 17375.216069163536, 32868.83220308444, 60457.845567201395, 29001.76854860297, 63334.14820492875, 87190.45647260091, 56763.79322076074, 42227.30200980559, 71344.80220896944, 19586.644193878878, 80355.68730843128, 74718.1518894674, 53203.42921107586, 13702.262463377812, 55710.51456696611, 5832.507948600729, 10287.162083301082, 42843.113926772414, 70344.03746127019, 60778.75946767597, 44720.18875888653, 92085.42980299419, 499.08620197215424, 469.37773875691533, 67650.81392341264, 36271.66844436766, 84147.1163328676, 44590.12243289065, 51216.902307238364, 56475.13151080702, 44145.28933287307, 10731.288288372387, 27170.41784080829, 70091.8393213511, 46694.312719833666, 92316.6720068588, 20635.412125112805, 96707.54777626906, 70312.80457628137, 96049.00387783367, 23755.048163729454, 97126.44286474132, 86787.7667315542, 44858.56393639522, 83341.0046509502]} +{"id": 927, "vector": [68753.20265388952, 53193.20321952051, 33708.26001804099, 2417.6201815868326, 62619.81720380113, 63579.25696830134, 29200.918566991473, 93817.420309919, 8996.201573848584, 54940.86676999388, 58967.77409951027, 27162.263283468736, 59327.9422816011, 54208.00816160246, 33804.68246241838, 5752.414656110594, 57280.23145735118, 90283.04519049976, 52504.09494238726, 25480.812159838883, 74527.11105060678, 64095.47210782764, 18723.018235885458, 80774.72390629318, 78619.8130821741, 85210.62933412292, 49067.72195108415, 62828.495727423004, 78829.67428903427, 66949.28915511351, 69983.78989445824, 1492.3985261003802, 42323.888547568175, 51452.01814676613, 48630.175911426435, 57132.20710432225, 18457.701349170886, 23651.878060380528, 33817.00915458095, 29385.418543415122, 40213.82566874692, 11643.8516408509, 96376.3156693943, 79210.23364680346, 37346.65811947059, 58823.224395956444, 24354.43403607034, 60455.72914525439, 93135.70518881445, 24167.495931634276, 36621.18035048252, 22541.72490290638, 43437.168654277535, 18910.20813283161, 1370.8509485670372, 46319.5115943176, 25240.595722815207, 66531.1366797683, 89537.28320453953, 45709.62590938562, 75370.90676246419, 92851.04688950455, 6263.34766108978, 72153.28977477133, 93584.88559341342, 40797.46148868401, 4033.878700362781, 85558.1683689748, 25192.754780874682, 14340.777934814409, 1242.0959327362757, 25638.67436029006, 1767.7315389981297, 78881.86757585149, 84089.81989156773, 15798.067239809943, 40477.745937700805, 51985.05052019038, 93337.44567342909, 9274.159900576662, 9865.62919884063, 74126.79734856529, 12300.411802395838, 13003.55906912678, 69004.30938677918, 34075.154180374964, 27848.72670705951, 48106.451111543945, 90866.50797934324, 11912.02939265199, 78011.88898809071, 13995.439531289889, 54801.30426098945, 2073.8999908073706, 52610.00955527988, 75363.76450170975, 37893.934091753465, 55350.21801331169, 41760.78423590518, 89967.07782048947, 34466.21974369636, 89807.68186996221, 78762.095380282, 2277.645857698063, 20502.51374470986, 49325.57073221666, 86798.57084329776, 39979.44653499554, 50212.542817259484, 37973.826868499, 749.2729471204274, 33708.8423110539, 97847.13040790391, 18091.923202216174, 49370.588722331246, 67733.70800060342, 63994.578467802174, 72715.59170736276, 92042.0959338878, 29491.664643259595, 77209.12473850414, 86240.89836646178, 55834.07722779463, 92134.67215720742, 27991.60468762134, 82499.50239701718, 86898.62257481506, 99751.26812637592]} +{"id": 744, "vector": [28450.720019199947, 88718.2154678419, 63114.78099846753, 40526.79488666432, 75569.90623378157, 76351.85962940111, 16241.566976125476, 72017.04373965622, 15719.567070806052, 96457.08582873594, 62845.709608642916, 75028.0217309979, 31983.611130329402, 25899.681435635935, 79742.08004891577, 29216.14933234712, 67306.0920027188, 52195.83759423838, 88084.16863981471, 86957.78707036801, 83762.40171740527, 15966.784789609555, 33407.46121692415, 16635.11458915955, 39031.650678657126, 27803.01684761016, 76083.32774463555, 35511.40944382813, 24093.085617740497, 36070.50614153613, 66655.36009548495, 93578.95794402006, 39455.815516297065, 87679.92125633341, 81652.68312551377, 53295.621202157294, 43992.692854955894, 36408.85508944726, 24001.046753921208, 33271.98159110505, 75334.62813011029, 95065.3629520411, 45904.832278293994, 48228.91367845953, 29754.760602717033, 13034.741335009114, 243.41686431643694, 74300.40131305016, 34662.221843503685, 43774.9878031197, 53488.818585831534, 76428.58082197017, 6212.7918667646445, 86058.55110674228, 2310.343169551277, 69377.73814120427, 3690.62278333695, 41247.50712785718, 87826.99882789585, 40654.63520410397, 34626.51300338391, 9727.65659803121, 38498.65131531441, 35286.28026653995, 36109.72069490745, 38846.35721833564, 30939.246408153842, 11703.1584816067, 45701.178853574645, 99118.47895150109, 34888.62447105546, 27243.968311118617, 20911.92521016373, 39791.18277219434, 65329.46095371507, 19028.73236795276, 15032.204863096065, 24169.10624833103, 64609.438884603755, 3558.3050236170698, 58891.7304337952, 56531.10466112149, 99458.00822157993, 93507.01384126146, 71370.60763524604, 18163.156754025433, 25469.007793821806, 67269.7949195285, 33918.2371901929, 93728.08221240695, 72830.55448849418, 54624.49182752933, 65992.37303221179, 85711.18113834628, 16770.646513225485, 67498.12600375219, 87272.57828664436, 87740.13903374593, 15808.369850589675, 803.9279014114387, 28243.849351398243, 66260.37254983204, 32034.817771635517, 6568.815205614343, 80430.05677703262, 59319.63539672452, 89213.51228589515, 23033.219408541107, 6724.631473283749, 69825.62336442425, 30572.650136684188, 72508.17647358833, 41886.8420100275, 61431.01158197888, 31529.154949045856, 5787.125239762625, 28657.4713342162, 18350.843434985774, 27888.887887569115, 76470.07317073845, 67724.16806655192, 87417.54782490882, 99064.36662796557, 59041.97561658483, 84461.16932163532, 12031.928191681973, 86951.57072253454, 42153.55304872244]} +{"id": 1224, "vector": [73922.72924691995, 36602.316079062904, 34419.399117491135, 51605.68177592072, 11021.18347563782, 78279.41806008221, 30931.259963858793, 8789.315123798891, 60832.29859267394, 27174.81487955985, 41161.523074690886, 50066.26450651867, 73536.16451834506, 4307.262506763832, 60644.68734165644, 34791.54363042275, 19615.0164884509, 49438.56948877032, 59021.6679901281, 59354.71523273144, 38058.549942843434, 73114.79530530296, 76006.79860397188, 87146.32217926938, 99601.10232531354, 97429.79421871736, 89179.96772923588, 18866.643779905935, 97569.01105751951, 21164.645406220927, 25753.483809531153, 50839.68567237569, 94802.49397278288, 20948.634490830063, 631.7268459111625, 73039.43516399496, 59266.68779006465, 85775.62757458954, 87142.11936751811, 65010.10910136653, 41886.35734444882, 24219.10274379769, 9556.028213217527, 67045.62775216233, 30450.258699129663, 25447.95233381536, 15481.130949968214, 56735.27662337176, 7163.54345579594, 6239.9497221486745, 92873.09309697157, 59884.20598969957, 87851.36296011663, 36424.16039979027, 95106.85596557139, 18725.240511751705, 69912.17365626608, 21295.0888669721, 96472.61454854895, 18582.612510617768, 89337.02712954991, 89938.74518270473, 23183.01393028105, 74304.99195673018, 7767.267410951284, 89553.81816614777, 31376.679550188623, 2035.9681716786372, 43036.000900281644, 3869.91123789483, 53337.892253279795, 7960.073201934714, 21485.174262001918, 96140.64817455747, 35450.05481019857, 11108.294124137197, 83690.78092603698, 96595.31069771903, 12391.735863594477, 97410.60457785659, 55194.03089875554, 18962.430785367513, 70554.81351994719, 92873.25824943882, 14055.9753905113, 62939.092659187525, 35563.65371116182, 94915.86141797858, 29209.27516270183, 47521.42157423499, 42124.470780153, 61523.97230018248, 17422.934959945345, 2004.7178117774345, 60720.246226181494, 65932.6000828063, 27160.366182944108, 17446.685539904418, 30106.331052884307, 52402.30221131893, 57589.802749676324, 95272.95647346987, 57031.19035747842, 70425.81441995142, 62057.662669533485, 7166.420416818842, 17465.159599962233, 83987.22903138699, 77189.41695260387, 14856.050569236157, 93069.970345567, 93069.83973560903, 53600.94267198425, 34289.27962519832, 4099.26124369675, 4025.9352734758336, 53021.8473295633, 47851.898669670845, 15049.221801684309, 4544.15349654016, 63817.02196070573, 9928.338059216168, 47751.11038414864, 8414.846724254166, 67507.197117624, 12479.362444629005, 66371.86529262875, 13227.154699062827]} +{"id": 764, "vector": [62659.60075234859, 18507.596610595945, 13225.644730231368, 9460.865971183652, 29246.663545386808, 19911.90644764178, 49498.12298690982, 76720.84166245817, 10183.73974333563, 84009.45308626092, 75100.52067307036, 86869.60506586856, 92095.99160813447, 53580.24664280331, 92230.7862342158, 82878.49677051579, 6710.096054888292, 33704.25165682037, 43347.63713701806, 3377.858291840752, 58622.36816658458, 58632.207784472004, 52316.25154707902, 51873.469379916285, 51266.066997128255, 4507.7978168434065, 30898.23419607154, 57056.28868099098, 43939.38936406714, 51582.20999843224, 56250.282379333, 44605.91364029646, 56147.26766609792, 38100.013222780624, 61845.81014412691, 57524.307386926055, 50304.571533351904, 6117.931862073989, 97919.47929963189, 19903.135461894606, 63444.158289683706, 84568.47527055397, 45775.64259744482, 25656.523502594187, 88734.45667456397, 72429.46619878532, 36801.82561075009, 75929.7085127969, 48038.04064377164, 2368.628292651664, 67982.47913678386, 43981.24203764373, 3123.142109821697, 15466.154541311827, 92745.80458925782, 4099.129903032228, 21510.299270399057, 60441.35475801084, 20010.813166883356, 33083.3291798962, 7508.056935662577, 93379.53080909482, 5310.057613043517, 40936.394298402614, 59705.26961315412, 76917.40502230519, 37962.82533150285, 71928.17543364299, 60480.05696550466, 74578.60718964999, 61523.588902031464, 59331.22519450384, 95940.75503760239, 43301.19760712099, 58841.11795475281, 594.903505385691, 84494.60267779393, 21841.116033741204, 57946.477459889524, 37127.75708863241, 64048.72177245962, 99318.05119406237, 24162.520980427526, 32313.879831765345, 75109.86325911974, 52039.15035597345, 34110.60875955495, 78732.21750373612, 24615.420006715893, 41692.43857228013, 97206.78045930658, 51790.41470835292, 51528.134394014734, 95207.38079953195, 51429.26344077516, 33360.121911319475, 6606.629498400507, 56173.3370959986, 95349.8220562002, 87295.53818610538, 27444.250278048345, 44052.715475115125, 2054.5387688862716, 36607.86878211725, 21668.326780994073, 96376.28529629923, 8625.247442348516, 33898.484042948265, 64973.611604849546, 72489.75506045451, 89966.48005579646, 42762.58366973832, 85915.94217058184, 58444.26859924774, 16314.703794546793, 58301.9420616205, 35086.339729996675, 83467.67440819573, 91736.41597490643, 22785.636263386466, 8900.940995891959, 15745.350397429504, 7007.38928746606, 79047.25925331691, 72433.2980830033, 44684.13710680911, 18679.521545016098, 96378.03794060266]} +{"id": 1388, "vector": [61532.393693243794, 70294.20191390674, 10293.31036367589, 78771.72489260808, 73427.402285155, 3838.460301598501, 82471.75016543677, 49027.68632490644, 30756.940683667068, 21382.298834991896, 98885.86772431595, 93263.61312465406, 47364.25744061521, 40522.4600628738, 39925.499996880506, 37242.308310949134, 45074.67359777245, 76364.41716814302, 28408.194856756374, 94729.06617737543, 76644.49180684472, 87852.75775613467, 41789.58074749317, 36068.32737991001, 93091.96286078663, 16925.894727457737, 16309.707825981568, 21751.729455899625, 37629.335041433376, 25805.428730380965, 46190.538772886146, 49022.07129108913, 49750.58992812891, 91336.33737615652, 81351.80771641868, 92220.6224405247, 53592.17415560486, 4218.406575830003, 64683.88658013487, 95378.76351485986, 75375.46353171307, 57077.513232041834, 2703.214324401704, 76390.20136512582, 47728.42903856601, 51719.33646360781, 14921.370874388074, 70474.08083809346, 51578.48551541081, 45872.80251019322, 45978.402875375534, 82778.9321747374, 77246.56291104542, 86024.21494421344, 78496.62264602477, 84851.61495121832, 71009.91376485782, 53936.59176374097, 66452.1755729829, 73872.70921574946, 85217.85507738068, 76578.47127452027, 1906.3124886965154, 88915.0966401807, 6942.459987481042, 34855.65019741065, 64318.96762396691, 76292.04711493719, 14640.722392081729, 53994.323725122675, 51838.17778196291, 18896.75125025573, 34046.64289562414, 13202.67632938167, 72286.52364425917, 60130.08806109853, 71941.02866147163, 79072.26256756228, 45030.969894904956, 9137.863526480638, 35235.638257159255, 42388.43135103666, 79205.67070137497, 50237.342918655435, 41358.8005228394, 26660.629656255096, 3411.6565973019488, 28366.143196980564, 48390.573428105046, 51049.339953017414, 95420.94849855344, 64387.26814710092, 1490.4927442270678, 69173.69973750142, 41841.79234879369, 40097.455000650305, 58572.8769636461, 94762.43191089893, 9211.84898063303, 63654.897660241884, 8903.258860145468, 24853.087667672182, 82252.93896852242, 38755.13035938549, 50065.683417014305, 7462.983520758237, 51214.25384924228, 43620.17030860706, 90690.81868358835, 12158.512065248784, 44083.3070111761, 29965.911607396567, 3197.5490039993338, 42022.26308399885, 50551.38026368579, 11155.584194041912, 1748.2020898564542, 48967.430064362816, 73044.70094816553, 37084.352322243874, 17545.43626121644, 91591.96901688362, 28155.491079220807, 66270.28687914278, 99906.03071985873, 25120.804197364356, 35748.50356627558, 82447.01756020228]} +{"id": 147, "vector": [5639.626897254868, 56944.250374492796, 34078.84903920756, 63217.85019370434, 7448.3121818073705, 37924.508584736184, 29913.155927047676, 1286.6841478499703, 44468.4069192971, 12617.850600190372, 12303.900788402889, 18585.71848680419, 94383.46585790595, 49125.55037253749, 73338.17215624466, 35075.742529585244, 92500.56244182213, 14010.43469515516, 73434.91194824326, 62208.975584739965, 85825.9814325909, 47619.6418674388, 73986.81498565675, 82384.18225790984, 2536.1631589796875, 40965.37860303354, 30610.101415057477, 90371.22578915987, 26874.146650463947, 77022.56830774719, 72466.71444253258, 34550.459688711344, 2360.2856235604986, 10678.095632676965, 15040.084075879979, 95641.81585626747, 54006.127273615275, 56935.814544595756, 42261.27093535604, 15011.029634195449, 81449.72577853313, 23613.953463535086, 8732.56486564371, 72434.93290617478, 93284.63480632011, 95131.54531495349, 69718.27528069973, 39901.193679729855, 73943.30403183284, 57805.56041227154, 19604.611807275283, 6637.227551740354, 86841.59476219115, 1571.9576806394575, 62230.630184515336, 56030.284713404624, 10115.119192002076, 62405.78574961601, 94447.92492558046, 38141.598293296665, 5256.630163920139, 78008.00664482769, 9637.922551281752, 39768.94575078489, 40705.497012839354, 85767.26403576716, 39489.63551199469, 52879.97511276987, 97911.12642608421, 43041.0723288289, 27672.459837122664, 81073.34774331006, 86602.60570841136, 77167.77163535275, 21128.591221909777, 7553.331740389924, 27380.12382160634, 46427.02190464286, 28092.569587271377, 91979.63917610711, 62701.998781874754, 350.9978023109484, 24517.13783482662, 14310.984845338471, 53307.05638982842, 15985.320543704385, 58952.57070909366, 41457.90142970663, 38765.383975459554, 37398.976216823954, 75721.46449389211, 41645.572744063065, 81754.78437746174, 34534.5895032156, 13139.878441971054, 48337.68880919823, 33534.01949893055, 29349.889281025233, 56091.94699751151, 81939.7801243832, 1048.3963236659677, 27954.770264435247, 42742.586238690295, 47489.762150544244, 39171.20399107226, 30444.86049098015, 73517.54871625376, 73043.59098120751, 13180.15819395547, 78957.1968130577, 5712.070981647732, 48986.257577399825, 77407.71246936734, 15254.208098489353, 12835.215371767095, 95174.5771043069, 14490.357343459003, 97231.9350412876, 14529.470483040352, 25601.17597310155, 75340.92533253747, 20796.29506990628, 75364.80504691231, 31893.02799732846, 35240.79457705349, 4492.689253097826, 20524.872663083315, 3044.121064123195]} +{"id": 1957, "vector": [76870.24886368097, 25732.437186308478, 45070.95716951591, 99784.64605733231, 70392.65082477833, 95189.30796336687, 22168.184679624468, 19470.84108199193, 80043.87953927949, 94030.80204163204, 49606.290129238885, 58460.97407756997, 55243.043378814175, 33771.27229265126, 22092.997860016894, 58740.40050265038, 10800.697504304802, 15795.121007331647, 26912.60142222063, 98211.10011042735, 50020.51323179022, 29474.93715075651, 71155.29427890068, 76620.90533851204, 94604.55433878457, 40212.61489000548, 6833.250881358799, 45042.6348395795, 1935.6215963883483, 12254.736194535399, 5491.095945036539, 76057.93363680721, 50623.59898570135, 17513.84362880074, 81443.38312028666, 99888.28513283361, 35040.53170895156, 27408.436455722975, 83103.25446453432, 77244.53320960382, 65032.684078733124, 65719.84361876763, 83040.72296722732, 17429.564647854935, 11664.23461812357, 73432.39357443503, 84779.5204393201, 34829.80177049454, 99933.66628631433, 85838.35247758204, 81554.41545401403, 82061.16134889491, 76877.9929408798, 87628.74037323595, 90687.01448892005, 13527.436433835472, 91770.56930221879, 4808.556580802814, 77130.58685517957, 39266.75184760328, 80440.39911518185, 5623.183590624925, 23996.42920078865, 83179.0264330305, 26694.496764731968, 36491.87169887177, 64634.76934827116, 52790.47215488491, 56465.28495066408, 33.37440058976959, 34449.0473879982, 9280.720357427585, 52091.2130369281, 56704.39564381198, 17000.534905481345, 20293.162411312416, 83828.2659172446, 79046.16366085329, 86308.9648516217, 71561.60769865158, 72068.10831673372, 80857.11060963945, 65590.18514205533, 25627.774824986038, 67539.28763180424, 20425.780596351407, 53286.55942968717, 32996.48763770284, 25249.754244590906, 29840.530424367927, 6908.975737352463, 45242.48694889998, 14146.240714127167, 71114.76756242418, 31092.308950313163, 27686.668372022505, 11309.012444609401, 79986.0253080312, 43409.95918649924, 99816.4830207932, 16674.049931354028, 2241.49228229471, 56325.40256350764, 81270.27587433778, 56110.74851244251, 69858.45829660304, 19648.074414558403, 5090.887600028748, 55691.16010897695, 99893.0868875753, 73794.73886824593, 73226.28537971157, 37627.054804614854, 11160.443586095558, 61380.00548327796, 6202.569309893546, 32125.773388636848, 58169.20337319511, 32760.983377163888, 48925.57582712137, 85565.41483855722, 10670.436336383826, 75948.11006704223, 44154.33555144217, 99586.27505883516, 33800.50495339234, 97185.93251451751, 31329.862317872747]} +{"id": 1191, "vector": [19903.50963543773, 58345.560767653136, 72110.26272126305, 73746.02202130141, 25959.523557107477, 69196.97098291534, 81202.3744295385, 52742.93510108484, 48655.24634406119, 60210.82281584779, 39673.949520928916, 75184.77175467412, 67298.28166134942, 53423.912104828516, 8778.825516650002, 78365.57677160934, 17872.494025201646, 89489.0245606512, 39884.424619998, 30243.067863357497, 29350.582657566603, 97422.37837390331, 53796.64060009261, 34309.853032879626, 27868.547955593614, 58725.4038560303, 75141.30356116728, 41212.749807501044, 11332.445025241766, 28236.36595484994, 24738.14630005459, 6699.52786816157, 14598.074401457483, 24603.424961637753, 96837.57594897354, 49661.37693245253, 70478.20246010239, 67331.40999140752, 14911.877581451783, 45773.7612476779, 54132.80071756996, 93490.31157489336, 51962.98363786858, 71673.71849669357, 14674.53726447132, 50008.73694916138, 88090.89118645708, 21880.79071015636, 99844.95085750548, 46557.05705330308, 81710.0591646911, 66670.81206135747, 95858.99583149921, 19675.322319510025, 51777.06599784063, 67589.02824397835, 86954.8621332568, 14650.92558079547, 24681.299810109736, 33717.684081656094, 10759.182780903797, 71427.16670064378, 26366.474152650444, 99212.22347051346, 11062.295937914923, 92094.14310372084, 75957.06821116302, 95463.23201023036, 33710.17244992224, 31749.719778822517, 69215.51649622672, 37865.706398880204, 25005.748352639355, 87109.13896971889, 2700.254906611965, 27874.617299510417, 30940.089126383686, 24987.3431946384, 16478.773747832664, 66699.41405097293, 30808.07244268612, 19444.17844535381, 20642.031669095508, 64327.934197808136, 54993.11412817038, 94767.70756980278, 42681.78255065247, 77283.58772050946, 40891.49914065225, 26615.046188599066, 53941.15962344417, 83878.04605996705, 63640.59087152557, 96279.90749341535, 62066.45919637297, 91389.28313502252, 90261.1089158923, 70179.41123167273, 15305.673396106424, 55265.38204868558, 63943.358414548326, 24085.798835389905, 2072.7226922064856, 52756.29136104359, 76821.7395191862, 47731.32825250937, 29360.68768547101, 74155.76596093738, 69818.53995719935, 40521.460454938904, 7311.062538546243, 78949.58762183311, 28041.156403154633, 20037.446569909756, 31886.603686424387, 90466.92354846522, 18057.215233496183, 62221.956258673476, 82658.88925977176, 83898.0730990084, 20804.417889802353, 90865.91980287006, 59464.621792350255, 58801.75538660016, 48982.61775562815, 48480.220337293045, 40819.84852122847, 72662.38089236103]} +{"id": 248, "vector": [45763.064020902166, 50346.161988345986, 53575.41557134875, 12259.554832794318, 80402.31557320229, 37211.21004519148, 86109.48618921475, 31545.794421509898, 19351.132746931864, 46570.87877100099, 63811.358673735755, 94199.02920885207, 87402.03091171428, 37723.77555019419, 56970.06661327186, 50313.95182299837, 43519.99147749601, 38307.009114806344, 53230.86343592016, 97790.40757064107, 15251.832892764472, 43170.309243756325, 61162.78655123275, 9311.30908306368, 32886.96590655925, 31704.97977757124, 23676.01667812692, 95529.59975069927, 34281.328109607886, 20750.346583470557, 99941.94849358148, 11556.706320998534, 82601.57446023206, 81562.51434968894, 65913.41276672618, 41370.712620811, 68002.45074281718, 79181.28898475015, 64566.84602240021, 22614.360524668275, 76989.05147805513, 79160.18453372749, 36178.19771482659, 73850.98725870388, 58292.42940345336, 9969.124340562474, 540.7112252721613, 18169.807011651952, 32291.0132887967, 68066.71105287658, 40341.955000946495, 18681.911841864396, 83333.94972604487, 91645.28690600597, 61693.36120979626, 50208.14625842187, 34256.13087359402, 44218.502173527544, 19134.832275217796, 47245.76273222013, 12003.668752160524, 76137.22555419094, 40281.40029143833, 66879.5370585948, 75227.0417481126, 33331.523260518006, 11222.034862494635, 19731.59747824045, 3337.162116727077, 95679.39282896263, 42733.32679057535, 4909.017381398184, 6757.008598815261, 33711.40810663993, 81607.58146652342, 87178.80150989008, 21643.153049720644, 53558.803328910566, 17735.59614553476, 33549.23981777213, 92654.47845050432, 81453.4669091159, 96349.83133552506, 87633.21360437587, 82497.74578454277, 23215.84204682662, 79022.33184339768, 89804.07558068281, 37425.11245235952, 15232.638315483793, 37381.23925999564, 56847.21289077911, 1852.9514574431416, 2321.479755315492, 81020.17789934839, 60238.49753675777, 37296.88945759962, 60494.03888511099, 92974.98763222853, 66854.27413074314, 62779.24764846579, 88405.05599330112, 20557.993937887688, 73510.275345316, 28483.44339250508, 34397.847549773884, 60134.655550073294, 27367.154912879963, 38446.75611075009, 48269.42551226957, 32871.05880264379, 92286.2763224061, 78695.1574577259, 41865.123120151824, 50157.24691603561, 69929.99913682402, 99727.53580734582, 30328.09709257843, 46016.584037761175, 97275.75695631912, 37906.73171956847, 81879.14430170559, 29332.536199477534, 34715.81994084571, 98818.89817421733, 76078.62359003499, 23153.595532982163, 33459.90858472201]} +{"id": 502, "vector": [19129.8581721552, 45511.26475299948, 87980.27900187588, 81769.4563135387, 71612.76755296772, 87361.11483358426, 9307.717409894556, 69363.60474606916, 69914.4485544661, 21727.91310290949, 34641.400535237866, 98410.59290802875, 18851.119080437562, 76254.85420973928, 75705.36652956298, 685.4413567984441, 94579.58927365091, 73316.61601390991, 29439.886956953178, 94790.23000320236, 85877.80511046498, 68073.2942890773, 55318.71096330996, 61852.20585293909, 97708.23320491234, 44402.35563478856, 58300.60882013862, 57852.14580856994, 23274.727025908458, 94162.22367111419, 24341.31494136699, 21461.343453519166, 14846.369203222686, 74690.85731418345, 86549.3605322277, 32200.303821012, 27040.990904709648, 38755.25416041387, 80166.43434215883, 35509.5188041918, 27214.873844769205, 23416.84902401532, 1700.1584421922344, 88889.5819807063, 13382.983947050054, 4582.488231431836, 36181.083913530565, 14163.373705231663, 51329.76244422416, 52787.228634650055, 70255.6617686544, 83544.87426044534, 44979.55793855292, 71865.39323144728, 92531.09868622526, 2916.0132906047575, 17029.00491078264, 7570.400178295733, 65555.454733588, 37564.34139391275, 75992.44233940377, 13610.184643565948, 90516.62124768551, 41931.60160923065, 55659.08470324945, 89041.5914541536, 21533.423361742454, 62570.479422551194, 6450.210156149771, 31034.972131947812, 11278.920519855596, 99204.90056218689, 70539.48906389711, 92594.64121064669, 89478.11249640516, 7327.244605127647, 20541.18638020622, 84466.16671235074, 10566.43476094078, 63472.80622160326, 46152.49918014066, 24726.69871952724, 74178.48711061344, 35885.98707606412, 40455.0466845976, 95801.29660872232, 63623.839001155546, 5007.868040094421, 96419.86567737901, 81008.765404072, 29299.924209461202, 76104.7553663049, 93406.95905470062, 13519.039964045165, 63152.41780676283, 78581.45668117651, 79251.74989076467, 98418.84882153124, 7120.680086039921, 34798.776840168044, 11534.012519744874, 6265.621893629159, 35603.417058137864, 86475.77316054827, 70023.71435304648, 25694.189640475808, 20352.397097067143, 90276.09557535847, 3475.8956705561172, 68985.57408220576, 58218.76948786762, 23876.70985804642, 70787.42512923619, 41770.165216301466, 40121.02476746942, 54447.14693096566, 67173.90072369255, 72900.81536158087, 63541.67071240167, 51345.81768956957, 20704.348586682852, 91198.34099514264, 57541.97944532502, 22554.96303905852, 79268.1691843732, 85177.70711010715, 94407.81176583571, 69324.84637217912]} +{"id": 1577, "vector": [58992.588429536605, 48156.817690605, 79295.30698427565, 56335.72285074851, 53325.38884168901, 78283.12625085187, 15631.29695033928, 73431.5449079214, 68895.62476190194, 51525.04886569157, 39263.032112215165, 64301.68602211064, 17778.323437099585, 9691.400663666627, 60314.744270308365, 61588.925049595164, 50972.81145718916, 24331.559423293937, 42755.75847855169, 86260.66592405735, 49876.46174774575, 76051.66618655522, 16766.854600296545, 86924.0093478091, 20102.66663051645, 41513.48686822329, 73750.09929250754, 37289.592274786264, 75978.46913017456, 55113.80592980267, 47317.58352787284, 83579.67915574445, 96166.41739021958, 35199.06823721807, 27068.7013957849, 96856.54438770485, 9158.228184463802, 74691.55838742055, 12519.973284832786, 86202.1023826689, 99041.34727453708, 46829.148777824405, 98051.4980322194, 38672.8831772416, 41643.68037121773, 80224.2497971627, 72545.34222879249, 41716.64780384832, 91550.57614732684, 97600.05567973033, 49247.05042980475, 12986.72943485667, 32891.755031913795, 23044.88177863704, 82059.09963772674, 19908.028744533356, 48505.02682745109, 86319.32169408826, 91747.17676407755, 32076.341043898716, 35496.96130689338, 45919.97331250269, 71035.84655528178, 75401.49928802637, 14638.420708293997, 60190.363925819656, 30868.51870673465, 68959.88882125102, 20632.161528691206, 8290.089689695456, 81415.46785490905, 32247.402544552926, 24975.554089029916, 57721.95731946339, 58470.98018967638, 44954.023025739596, 85466.34111394861, 11481.4083247481, 82547.44421305298, 98431.72827846676, 66792.34595512933, 82625.24039571536, 90280.73078280996, 41013.52684336167, 31725.215525473628, 94910.66296058915, 28527.872014242672, 77345.1729101575, 31476.009194593047, 29223.445321025156, 62395.213819285374, 90821.54920223953, 48293.25030392478, 84548.10525342333, 6126.7813954025805, 81462.17633895321, 79901.56580386219, 24409.95166186197, 87978.81676282671, 9759.424692694296, 89831.5683568566, 91150.41171772005, 84026.72226886128, 72666.16302811197, 33406.22740494394, 37769.6795818568, 56638.22862487362, 54257.46081575582, 54858.22868224325, 69252.2357610981, 39048.048242755736, 23045.249852397985, 59384.38005761531, 83398.36548021022, 13125.647812420028, 67620.56698128724, 63865.3223930249, 73791.20038577823, 30539.862105258886, 2411.924121860609, 46545.29807565035, 64287.56891047329, 91383.41623315627, 84311.69093818443, 48874.44429332051, 34937.95709362882, 59729.26670749032, 13087.241466936593]} +{"id": 186, "vector": [56861.44338255042, 42908.92752126985, 3049.632183915463, 74250.44287551755, 28033.737685004635, 84871.10283338907, 26867.709905232306, 44896.072533008715, 27597.145232730345, 15803.762183800296, 98759.02207791887, 41788.444158073966, 3833.7571385122947, 60853.91752956532, 81414.00262650511, 15654.879713831104, 18069.133057477393, 79862.4528601709, 91859.05957177457, 89132.65168226216, 11710.49598895545, 88329.59771510513, 54807.72133412188, 86726.4602030865, 64764.77309148058, 87402.45319677093, 90010.08555745, 261.5855138190737, 83180.73510755213, 35402.819751178, 46950.00255406726, 85321.43854336064, 5237.981138322245, 12222.974186067459, 72681.20187663089, 95872.94253900606, 35768.410622411706, 36440.659073051276, 22875.72322312652, 22229.509974290897, 83707.73749160508, 3044.277747397206, 32311.84175119799, 66079.51851923102, 961.5272290057542, 75300.53373154716, 49644.410594533685, 64101.07838376501, 91285.6576082511, 52299.33292924912, 54950.21729929982, 83417.00166981827, 22317.958161530925, 17384.905312891096, 58602.91764948001, 57359.88688547037, 3403.196683306486, 55488.7698544226, 44118.74598651755, 15826.682709334695, 59018.001942330746, 23783.87067732185, 55758.58760383885, 35666.909212284634, 47127.36234895972, 86295.4049026411, 35403.087695851566, 21997.277884005383, 74381.41465670752, 38423.12408447014, 24010.603237213647, 81384.90271998751, 12303.026939103456, 97141.06406522018, 42027.23105978165, 7314.509666310964, 64840.176778060835, 72756.41903977765, 36624.55201393097, 31214.026354457048, 27050.680495196055, 10303.430520191492, 30047.979071575115, 52750.112804277436, 97239.15560545541, 97900.48512448356, 13603.715558874852, 62052.0223485664, 64459.358144737744, 34782.61877069653, 50028.89674358252, 59278.59671946485, 10814.630038092355, 13864.671354487757, 61967.2244727874, 84956.97290959355, 71938.36040749337, 24157.65576358745, 10116.178243901952, 65294.654651089266, 45730.932041166874, 62201.16636603281, 18545.527291993978, 79757.75651396318, 53012.6310302209, 51556.43411219758, 57660.59896793859, 24066.092394372907, 77852.46907260695, 93213.22380914974, 74922.6839762696, 8259.28749543713, 6164.179725196972, 42127.42800256058, 4192.456057090021, 46042.65811832131, 42412.22267147585, 32437.29995683312, 68052.99753192873, 42473.2185830316, 77135.86645329451, 37242.381868131226, 22580.6646412813, 5849.959101006585, 3967.636314887324, 34980.599007393655, 5933.626370451739, 49178.228346187905]} +{"id": 1976, "vector": [85604.11860198017, 83463.70339431295, 67296.68794117673, 76968.9104668735, 91417.44873349192, 14964.174803034248, 75669.32532313085, 35473.3381698109, 69110.15102115762, 61557.79580375118, 65657.35321684496, 58968.46557890335, 51429.18663073186, 90250.78161188298, 92897.7513185151, 7561.046984542286, 59960.99098550085, 18513.02433426121, 96752.33899279134, 18992.237111399736, 53477.75626797413, 11590.875855695092, 453.93491153278285, 39684.53415886153, 72764.86490492588, 59588.68038629307, 51857.451653908116, 10484.88314221231, 83173.17660730534, 28325.71755675717, 68683.74637289927, 63183.53053872238, 47656.854089901244, 48299.969230823735, 53862.27361996696, 7682.672761710718, 10221.433624767418, 37590.16836285014, 98262.43606696109, 8821.266889145674, 84413.52240376372, 54209.846161026675, 22409.40218916051, 79116.97575877729, 17650.168134249543, 55408.47741951775, 73902.9660563261, 28073.821793185714, 79913.26797683106, 68528.52626429012, 71898.36173038595, 92166.65570817339, 25308.128103718074, 39404.37367810142, 8837.847414108812, 45842.861487849965, 70195.51821008376, 11882.729991334296, 65341.36474451629, 54458.385339639695, 64703.94637797421, 64044.82651094408, 50706.25022968052, 59382.43549041634, 10906.109205293034, 65487.50435703533, 82275.32604331148, 87257.12972762393, 35212.954760906476, 27249.663111198617, 42955.86434956005, 81811.06634730594, 78987.2145664166, 38992.48245246741, 96775.39177888019, 36009.68982435229, 56356.582356037645, 61736.74821663742, 56449.64401963646, 8509.436376671676, 64205.32235406308, 10770.700769093488, 62099.42860592779, 43680.49665119657, 30656.632547420348, 78620.00083129841, 73468.1334321439, 90883.95621525777, 94647.2184912898, 38502.38827542151, 51114.905650231005, 19224.01999581268, 23386.633191188932, 10839.7342570003, 27523.662013471574, 14843.937981710076, 13427.047200128562, 47423.74059668272, 47141.351791144305, 31693.46215942812, 44311.094289700406, 9184.755022273783, 50981.90697247665, 21804.312935888116, 34895.60785464641, 56257.386475001746, 35981.893040525036, 86496.20248688864, 52195.67505230098, 22522.635101119282, 243.6719874788884, 1990.3711076564746, 92844.10402204223, 47117.4172966475, 47489.64212614647, 90828.31734848951, 33467.77319537405, 4995.418276523677, 14856.671934890775, 75619.3061272732, 76332.62299533999, 78406.17357544966, 56417.16235282884, 69067.04133429033, 45383.53432225822, 61159.403472099315, 69700.90278872554, 7228.747325372631]} +{"id": 935, "vector": [97979.09704791493, 75610.05828118179, 41628.66193867865, 58301.364038321226, 44693.9816931459, 1094.9019945799555, 94741.84170489406, 38980.114175318704, 64191.355811393456, 82432.13837016605, 32966.68342855981, 40904.91252498234, 7611.39764732941, 28405.151771614957, 63117.15720557288, 29939.388478059624, 46729.90828980736, 178.08736460024343, 68406.50664910435, 11432.311753644808, 80694.17010076785, 77454.53753240274, 67486.35700031536, 22721.455944076963, 15785.633541416977, 90847.73829157517, 87904.41122153745, 86467.89413074146, 51789.20249090445, 48976.50594812491, 11501.111877860116, 58474.43927270901, 17520.831838273552, 88084.42724497963, 35288.09694527621, 70555.86421913185, 32975.07312261686, 4497.686894308783, 52739.00762643015, 16418.71867902571, 87613.10133968733, 56679.77042152921, 36226.907126283855, 4871.07408954286, 20485.994520831686, 60615.48488302479, 41105.14050289672, 97275.6258992053, 16213.299949419923, 65778.20766841221, 71440.51642813021, 37319.48904768484, 80166.35634151433, 96258.11968316099, 60038.061761824625, 71222.98325876126, 25922.96872648945, 18026.43298897537, 59331.01751396002, 5488.890689741655, 84961.85249339446, 72552.33057472874, 49445.48664959786, 16883.435348752686, 13419.112043580084, 16974.269147189312, 7799.196422650356, 13326.044136580305, 12690.033435110137, 60765.58159721587, 99628.05080968275, 68029.31983389692, 58837.61959110533, 25795.99833249715, 35650.11773523632, 86301.43789142776, 19529.637126063302, 20992.398950056533, 30321.887157113557, 95577.38720882291, 3060.2514040916517, 81044.8977841857, 88164.96265962759, 67803.15939898913, 52320.504652312906, 23786.406901177194, 86167.17021378977, 76919.97865000773, 87250.22438066834, 54865.34030319429, 92814.77190109192, 94832.24513100855, 21140.07204195789, 54216.83140917465, 221.89746650208565, 31686.57399659084, 62184.731889509705, 64379.53982727965, 53756.78376029061, 57870.88216959956, 84287.97187885674, 12862.2738230329, 21852.127414832878, 65352.45196386602, 92965.71654638373, 68509.12109224263, 5082.122493378039, 79934.17556618324, 97312.41220095391, 64138.95005687079, 15875.469893566884, 76162.1055551551, 54411.840294258815, 82518.05731436139, 66309.42892263811, 19198.93483729991, 57060.44879142153, 11224.553189879383, 83809.6280506086, 7626.063572616715, 76383.41819724749, 11943.204721645074, 90914.70945207209, 21246.3757377792, 50143.18501771293, 88233.63047932505, 51781.797982851815, 19412.73935281128]} +{"id": 870, "vector": [24151.12802884589, 64807.037870595894, 66006.54502015973, 51169.10195712825, 29714.393646386226, 51704.146936376404, 90158.62570563269, 84942.75835108917, 42906.863961508425, 15988.161810216894, 49424.31913849835, 9863.299253131707, 68583.12226187314, 9159.769372297811, 80460.57055719542, 99054.23735550922, 89061.69265460224, 78347.22362719923, 80875.58811855585, 68054.67088925482, 49878.43986864572, 45592.906454624725, 55937.57178894698, 63189.936920832944, 33624.00266303687, 23590.690012012707, 69346.28220878387, 45669.71111336899, 72906.9488271177, 81934.5684301588, 47145.02062536491, 79099.62628233778, 87315.4619287734, 41914.37516483339, 3765.660566021678, 47343.19322313125, 89984.55058683084, 72843.53465803464, 20891.52013706205, 63596.14110401315, 25061.42261282117, 77423.03422571065, 76932.86916659394, 88477.06063862555, 19750.169276286335, 88154.74413323108, 31188.25988511157, 14905.575210810362, 30797.09448970499, 80184.28376321158, 2732.01441972345, 3517.3735347296843, 62646.9447772927, 92030.09532708424, 8246.698018216903, 7858.03681937477, 72894.81829181043, 82412.89637498681, 5530.156844347977, 13421.383500256046, 20147.472389170864, 2047.7062840721483, 91055.99958894901, 6007.013941083584, 7203.677608871695, 77218.46955516291, 39473.07352266421, 33809.12557128247, 36465.5413206, 9417.326884748367, 64551.36830757685, 43671.44855255589, 85086.52667536895, 19017.502890062566, 61968.25852811356, 3425.068838310852, 74338.40097539926, 45312.340304852696, 23990.003469646625, 82119.25497865729, 5123.554267305419, 26064.34251839951, 22324.883444150946, 48098.487878293585, 70510.31538421132, 86456.56897533682, 27375.670893048053, 19431.431527067067, 30139.153324882318, 87064.06023672866, 23434.021914255452, 52200.249484898864, 52320.92437664432, 19588.395483114306, 4949.745709532372, 62079.86745677424, 87364.43404263929, 63899.35520086045, 73034.3313369264, 1051.3153070120352, 76055.23291854553, 62876.29803646127, 56605.919832848085, 80530.88705447194, 31116.857446843795, 59521.34815772131, 1018.5220226488823, 75195.27350357789, 94336.93265637395, 43612.07244864662, 43795.818161768715, 59218.25612781994, 88959.39165937771, 11287.946106839741, 33960.81304358642, 9498.86504097731, 14741.210766976088, 14827.082428400496, 20035.249175839253, 6379.875305802685, 87844.52740258626, 16138.690793084443, 71681.48873641258, 75649.4808095152, 52442.05819991858, 89209.18039828792, 76230.94613944202, 421.2750324128711]} +{"id": 1411, "vector": [5314.766689157246, 24774.33722610307, 51493.65889231992, 36385.58220979925, 55638.64968817601, 56223.70599203306, 54691.368436978606, 88816.02330086214, 12893.096131869997, 75427.62609188507, 14639.137626188593, 92125.8078205107, 63084.22102607968, 20201.388862100044, 57340.3746400139, 86612.65785251264, 64673.11004320014, 47349.59594464617, 67174.84116425167, 58873.51041077539, 46700.86709673448, 21616.20294569768, 99320.60396819688, 30912.83642257553, 68117.27411042, 17590.17442775609, 97208.80875507141, 67566.62340630648, 43676.57155873491, 38508.73615355726, 37150.08289654459, 28317.762175940064, 61693.81482795429, 96029.39261222455, 32594.42557987562, 98602.36915782062, 50176.355724493296, 42699.16676210865, 47844.125905095854, 75737.29033667097, 20301.818570561758, 66256.5803999608, 94263.23527538404, 56715.88739579917, 43470.34004469231, 17275.98125901313, 71216.63895991225, 16149.864351954857, 4751.00769581972, 21633.902636430015, 87361.84684917057, 85866.09702795606, 42606.746745938784, 29221.71566858933, 19262.615245281755, 16394.978402374207, 32802.623706872546, 10886.120231415374, 31669.1361946676, 66870.66885268984, 50112.56360053824, 83157.06306584642, 84541.36796388321, 80512.11929226258, 59303.35306926176, 16346.580267434363, 11421.533002536544, 80695.19315688699, 80228.3100257721, 66191.9238014798, 14308.565579797994, 41542.17417370158, 97978.93515411473, 60080.68669149781, 67438.93310022388, 85955.589051918, 96809.99000935859, 90830.59245919889, 13556.010453926026, 16431.446272826146, 50830.96374390753, 65403.9494228952, 45104.71881620626, 47502.75062344594, 41304.30988543626, 74380.76411596575, 46928.395340059134, 6866.914903193322, 81395.05981003167, 70918.57677497182, 93913.55663724865, 41698.344046165235, 4257.248684778759, 97380.73993279526, 56218.4474370722, 14436.595186652368, 34900.677315647445, 64773.92011385916, 56899.65380926335, 84205.57620717937, 13507.366539677823, 25218.603318517286, 82614.2401607633, 67647.39312094991, 37037.00529830019, 42926.615851428694, 53465.85556864429, 68604.46428620821, 42398.364066729766, 74679.74062640767, 21908.784609336075, 73534.15746548372, 88304.10913359353, 58692.21423541599, 57647.092361033145, 22911.498961332156, 71095.46793468411, 31827.670323124767, 44850.12757632242, 67074.79814812075, 12619.035554178925, 8383.83246461022, 43220.982182265725, 65641.25740462143, 64194.211900806054, 72006.73356371674, 99766.26490711326, 39665.6644689776]} +{"id": 1742, "vector": [30351.831455976087, 95151.06258110671, 35077.25299667569, 7656.4983565884595, 26078.371054959094, 37265.30747114481, 6420.217426157004, 10631.537652310008, 32579.96661598963, 14902.354172982179, 52318.573065703065, 59758.72139824849, 35362.85521294391, 17475.97315378524, 29329.9827463258, 84329.20246453893, 63086.4023166921, 33006.464260229484, 81813.34080998851, 12943.53519308622, 78772.21249773115, 31243.28765607144, 34144.04590945114, 69982.12990291578, 10779.088403028869, 48315.287404063565, 57718.70855892009, 14064.682259663152, 227.8673637952644, 15397.737221729802, 77742.98061235728, 32632.753743774843, 98636.12554620177, 9949.146621391836, 69608.55152591511, 40388.65017105902, 97809.59217546649, 26144.44901111388, 73368.99732867473, 67173.26201945945, 30849.434810637387, 13949.4874803178, 27362.87199352797, 64269.73016966624, 29746.449319933334, 83326.66456085395, 20484.22466972838, 64554.45549481501, 88660.40361940557, 51544.27019005584, 85588.06375882112, 81156.94382348626, 26966.31895714926, 81797.47884231363, 83634.00067944273, 85686.25332581045, 80178.81365724062, 50835.00045973739, 30537.37134099792, 94731.80930219164, 13882.62770562907, 734.5202723820221, 98133.03345547874, 86150.69303052171, 1231.3175215297179, 92584.813841752, 72515.01318843925, 51324.86817837184, 98903.41148609802, 54945.540609647636, 89205.55711275726, 63643.8105185288, 36339.002428173226, 52741.493985706235, 99918.46426387315, 42734.819023416094, 57985.94385855672, 90731.25632650674, 92101.57930493151, 68363.21189131604, 77117.93113075932, 54303.81756542112, 48956.07070640997, 47450.26793333313, 99678.59734805622, 70918.45441683345, 82005.10627630922, 41083.61455124024, 17888.339412828347, 61564.942523249076, 8743.979620223263, 7533.157590727713, 13423.145990256102, 79649.3094601293, 96452.84707665603, 42756.27054834641, 97669.63250109772, 50303.38822026239, 34550.35510275063, 16769.59419942369, 13164.650315654613, 92328.13525458395, 77514.05321057876, 63758.823724735084, 98735.97339944511, 3599.3577089475816, 19564.6371815703, 52558.489379181294, 97971.0993528978, 46980.30106497438, 60263.84475127943, 32793.342541453305, 93509.28817205997, 73994.33141573615, 13290.455297896253, 72887.310337581, 32749.142587030445, 63483.08149147409, 29950.484696604428, 41117.438085414484, 28104.848662979464, 18854.34356761714, 81377.80711852486, 14285.047903594772, 16941.957955100617, 16330.711237956019, 74461.59374109501, 95793.99091939654]} +{"id": 207, "vector": [94990.60889589632, 70312.09933913952, 70973.72344549975, 66418.8189423319, 58738.74512693675, 4502.896765852649, 32247.975785326245, 85976.51650765473, 47473.19754240914, 37498.72884233205, 27246.132810858515, 39186.70292073659, 82776.31354264337, 57482.35779348348, 81494.4949642054, 92968.40749763178, 48106.92320042927, 80620.96095208185, 30301.68354150614, 70595.2711658179, 42976.21668102811, 66012.83756669157, 38676.946434355166, 43490.307167022525, 99817.90014542303, 8929.014779131161, 61855.889312391475, 52109.52045725306, 30933.110891063043, 82722.95177261432, 39699.827785031695, 47113.47643969478, 65779.09444125574, 13737.317265527148, 58135.48618155325, 74288.92277547749, 4611.579354140272, 44205.65208446381, 37871.05562803396, 65345.24688175808, 93800.04998466173, 24225.276445395593, 94495.85773629886, 87925.29312027022, 83399.52057312976, 19516.923823546374, 24769.773917187966, 20447.66468650695, 73760.22340553693, 70291.58893435632, 80212.040344786, 65718.94259390909, 7380.46384054768, 41286.4284230735, 50551.414883898724, 16106.307922593998, 694.0376578190577, 30397.224204099137, 9886.310431775813, 69232.05967763609, 26375.476667334675, 60050.27562655255, 75499.94943841199, 18310.098235958616, 65979.89071304009, 390.5521758360453, 83849.73295038746, 19830.85730704791, 83198.75627756248, 9831.141184121927, 56306.11408120756, 2247.8198645195844, 37172.67185220011, 45980.09266176982, 39855.57220624816, 81104.69012661664, 77956.99955114411, 85685.78182067805, 90332.52802923511, 75160.49918959363, 79881.49990259799, 52828.108309631214, 16823.38249928853, 9592.629962350253, 62490.94086292756, 71514.46382274407, 3891.7293515369656, 59084.154034272215, 92986.78437201711, 70792.57015654683, 42870.79552254182, 50988.80352080035, 362.25173170976177, 34383.28170774143, 21482.237349237876, 26481.137666545994, 92436.12953115288, 74262.8933888918, 88606.12493667426, 8466.654206382163, 59068.00591204523, 18437.523059937834, 51752.518488194866, 63332.38486018909, 41765.41890436996, 47492.07091507215, 7448.370961239648, 5131.522271624278, 92611.34751126809, 59870.32679731882, 7569.264423752742, 32916.426086645726, 68448.29220375634, 21717.616941452357, 46407.10872840269, 54308.53731361534, 33386.59473554002, 25544.89418428031, 70349.62775923392, 37505.351592490784, 72995.02440816272, 65545.06367415837, 97289.85505318282, 31235.857817280797, 83925.45942586329, 16672.12744903883, 70710.5522287887, 90540.8714374164]} +{"id": 1369, "vector": [64031.35322375559, 92940.26400633767, 16551.069159808594, 99485.06294257569, 97006.70106974208, 61014.72790292852, 30246.186666511778, 35674.73018399527, 58194.81884365676, 91858.30363676809, 34907.40777229311, 75930.6248113584, 49599.29468291799, 47984.04154820193, 23324.694195751927, 32982.45559693925, 20767.71692996432, 64429.766896027344, 21723.34064174445, 4465.338908459071, 39101.08550586475, 4771.166814529737, 68151.5076951949, 49917.49699450304, 85539.49929189461, 5433.918037810737, 93864.08222944773, 5303.466299141913, 33411.62196493709, 52845.74847064697, 14641.442742559053, 59274.860491403124, 52038.75860086489, 41409.40552450474, 65075.703111905634, 87352.18920858052, 65298.26740241539, 70098.57261861951, 58179.28948817581, 75648.03150159575, 55119.285740976135, 69717.0348870097, 17910.702618317286, 7372.069012469317, 75229.69105437695, 47841.7033476925, 76370.59011312854, 8125.371391494274, 51319.87068634265, 27564.04835133657, 88526.98550807679, 57167.33548979377, 84384.20054286625, 81174.41896901769, 71945.35503433456, 7664.186849842469, 84519.48329229381, 48580.68783838114, 81174.18800648027, 64711.937791516524, 35030.283143530774, 97752.95413245067, 3479.510048551393, 21943.09147291914, 3074.8428647646488, 19872.0363797926, 46330.97745897163, 774.5408359244421, 1802.8899451445257, 74161.76070871724, 10226.14135434744, 30196.49186110117, 71786.27919517917, 40677.18592862591, 47593.11212307173, 15862.537926999908, 77690.48173971598, 10257.935529811757, 52804.13336263864, 20450.033388370637, 82429.68337590768, 5287.525939065474, 92111.61241554287, 63074.2247693978, 38880.67517488274, 38258.52458417385, 2773.385044422638, 19927.15960533967, 29980.8452742239, 9534.676580317602, 2292.3689127439916, 42368.5702259802, 95386.18586812845, 17263.76484011629, 14450.071996230463, 8221.47799751406, 47812.887422863125, 84388.43915977688, 80653.92370383545, 93903.82861585909, 93369.19097053996, 84066.49952835485, 55031.70543020629, 12376.948086739381, 46810.335050362904, 62583.689268119066, 61964.54643934348, 65028.01028328062, 75812.44973074127, 43154.90333039026, 44347.96335772405, 79362.84776363151, 18883.38069628892, 78228.69945113438, 59182.897509300536, 92160.90211188915, 51898.07432579121, 97673.38180399404, 91130.18890554477, 93211.91918719135, 39577.93085017628, 91220.63937957108, 5501.320769303486, 98616.4549712409, 95688.61976780726, 94345.20862168266, 32367.482660951042, 55294.6776714094]} +{"id": 1416, "vector": [18693.61019625344, 14716.861022001027, 51249.52646155453, 14518.487325546836, 78538.40313483, 98793.71019144879, 93208.55207845928, 32369.74440731023, 90840.8034847334, 8070.440404435597, 28584.08005922526, 13812.850156813394, 85802.46379239473, 5572.543460385282, 19480.085088767395, 33744.446526849795, 74796.2537227988, 65990.96450923398, 75292.99124723123, 29793.43192700283, 12512.766711318069, 55972.42458362372, 82466.6844393439, 67230.14676162438, 93995.49126724055, 96794.29929802817, 18883.7552245382, 26214.228724900822, 25806.578977186946, 39535.83778313518, 90091.46201231335, 65531.32332537728, 89911.24015863502, 47740.691708582635, 89910.9113024358, 63289.79943969134, 28796.159788426, 70475.59204794442, 16298.935336047782, 42082.63348966956, 14987.471863113999, 5706.671929447593, 83867.84675945852, 83410.57915203414, 92591.21812334523, 72563.30214830466, 20991.946607382994, 71525.89990888513, 45662.08575918146, 23947.819380056113, 35147.64403350107, 91101.39065295254, 86035.69547557649, 11734.923617447357, 1147.0923011428158, 82228.65724512936, 28644.72194566665, 10345.925027639236, 78792.9665068326, 20560.76780076599, 76586.40828409744, 5377.614257293139, 60719.98897025731, 23562.016846707567, 57809.020708042844, 5993.214889445164, 56680.1690775107, 82095.39803830537, 63588.98086947499, 79795.2648951243, 94432.06722449619, 33973.792957679005, 30460.613756029874, 7329.954281250817, 64277.46220349297, 14580.88554775624, 79789.11314087668, 74178.47842716696, 43877.54854747921, 79679.22535122892, 2931.156965138004, 59919.76020822023, 58126.45541968007, 98913.95540025982, 5640.694669063307, 11796.370709995595, 15426.60105266568, 49584.74988552779, 3513.292892574671, 78471.38378954113, 37744.517229248384, 93479.81143378367, 65517.344772912766, 67902.74665144282, 57625.22124789151, 60230.87739101106, 2090.1138658895757, 35348.67629138979, 37955.823517878795, 45748.569521000936, 30364.48963224747, 41035.69201937729, 12839.412101026182, 46856.48732132021, 50972.35397016443, 32040.34786791412, 11618.344364254152, 91917.50931990948, 2412.6460360282763, 56796.83963038531, 99978.21670776495, 38776.7727450424, 64441.50344914941, 90197.15323739519, 47797.80108816616, 65592.84238665621, 75173.23317308091, 60362.711177654724, 42168.020335951725, 93761.92597092745, 97056.81566165965, 42990.25774565723, 10741.219759957921, 44578.54166326647, 46447.33348834086, 47205.47569467121, 66975.55262890844, 6628.647387460452]} +{"id": 1200, "vector": [85125.1607142566, 46843.03449828633, 50903.804272635825, 55648.364170583096, 75499.53866164152, 9693.204456042637, 35774.28075511612, 84523.57574096725, 49604.467873150716, 49099.576615342565, 518.8708787997487, 19076.76224176357, 92559.50756084667, 32043.55425365182, 94430.90627665048, 80038.74641876198, 55921.95361021516, 49390.94999187034, 16023.278730253354, 39130.27470327013, 8018.1278515570775, 35759.59998377578, 82789.074350659, 5644.358105303848, 12105.852414703011, 1528.0711606412533, 75823.4636779202, 80735.58275359904, 51411.065700454994, 7828.611252338935, 32645.51987460367, 16127.440915804258, 22715.28023730681, 20357.923289052447, 93220.12971560453, 11019.312490501798, 85668.2336254665, 58010.02208537992, 34181.58347565924, 63851.19989191599, 54178.5299129951, 21576.92864127213, 1582.136143638102, 51615.71525827722, 89391.57738587727, 98182.60038166108, 50066.87380991767, 85150.09103457986, 18053.50196532135, 52491.20047477405, 24523.7586122558, 70807.33550966975, 15041.105799898824, 32000.243894376003, 13973.45998266487, 7702.029930110055, 82221.95160693588, 42675.32495501659, 41196.82709247523, 50885.91879913835, 54486.05927109289, 85088.6796460817, 55278.698071827246, 87929.11168214091, 91320.46961394152, 6466.802996349575, 53570.547281525636, 97065.35708502088, 19791.69521667531, 89192.87750576716, 33988.47794997472, 48488.2423007246, 78848.29754037094, 40549.44249472598, 16682.91749760661, 88912.85756208873, 99061.31385216457, 72380.06742470952, 16198.273122973438, 18560.309083055938, 7078.806807921134, 59143.759278744525, 46034.82095454645, 77599.48993529988, 91574.84740181216, 47195.738036106326, 87951.43545858344, 50385.41954478447, 86061.8019332458, 83483.23998953898, 20206.177540642846, 20664.986964322918, 83417.22173954803, 40168.05828381204, 570.9005955669033, 60529.86193893491, 92577.95965247699, 28723.389299982784, 61255.95068186024, 87773.46244129576, 15497.536497235109, 19386.19081015922, 96187.90167784382, 48277.41924729092, 23314.058635944333, 51743.680149672226, 56198.205266375335, 72309.79797417138, 22769.230903191794, 44058.82736997452, 75488.66504834432, 6117.932405783577, 36609.324380133934, 91357.98613290119, 19837.939785658444, 60140.36613069311, 75381.1972208498, 14325.72220604117, 50082.40249955318, 94104.50376832929, 13247.223118356833, 4655.841113028203, 53747.50238894283, 30251.739198892636, 90600.98477043769, 81879.63841064014, 11346.307868932181, 52989.45625135203]} +{"id": 697, "vector": [4901.086070838801, 66435.33982271263, 17357.819288191935, 10865.142621193247, 47057.936452905094, 17819.267786217606, 33690.14166288576, 98871.2249929444, 3005.511758381296, 71304.2262785339, 46139.52763293712, 91120.9926750661, 30719.74531565016, 16123.097669225683, 39356.7298430315, 96065.90787216938, 96658.72990025935, 7429.700053483157, 67035.93072884351, 13515.518148426265, 18131.490364925718, 6127.597843264476, 56155.02179862983, 7652.663298280382, 50353.28287262941, 1771.465086838586, 99804.95962940017, 77061.99958512904, 97894.01140627268, 27971.66285708065, 628.9085631869251, 12462.371170398534, 13372.984889410322, 30456.73603526248, 30154.45795066316, 26891.97671268381, 25259.863650690484, 28580.27501000717, 14371.693788554874, 6467.735602374314, 97269.20018755605, 22314.79100632553, 83604.01621585328, 11441.775184842561, 54232.00378258842, 38959.58548952522, 24484.280236141254, 15320.387699225214, 70232.0843618891, 79471.52685640132, 54873.02072523942, 76300.99535193031, 24922.41126133775, 42697.21524683989, 78322.00794468072, 87049.25179064026, 5408.363301091102, 75251.62733211384, 76921.4812305247, 91384.78625235996, 85153.08052444387, 19759.857489338017, 79573.06744653679, 3879.568203486472, 85533.63594987508, 50621.756316176936, 58982.739493831024, 99927.49141504832, 82466.44319468053, 14769.803576132701, 9236.79604531944, 64727.50836096614, 72029.57945728822, 16572.424131423348, 73990.6459378353, 844.5088185687788, 67956.50238584902, 89829.71140812265, 13930.218012251438, 6377.581524243093, 60008.03981814617, 16921.91500522937, 57495.94578591565, 3305.261977260798, 18603.830346858296, 79384.39612720857, 84168.682845419, 24949.322204881886, 30686.211313892363, 58960.96672565759, 51334.76256159617, 72854.12081769724, 41877.034275525504, 64857.559353864446, 8911.82387638163, 19815.43371704305, 42251.90930099859, 30580.48895705482, 68801.54189020181, 60855.73469276565, 99270.57076915009, 58636.91020706751, 39918.76363213992, 55311.26589619581, 14740.56141182183, 31181.701998142074, 5205.643686121153, 75332.19997037196, 26345.708127730217, 51859.22846438543, 34507.77932940522, 87670.91316163285, 54453.353481745595, 90966.7126978082, 90581.49481415979, 926.433918611358, 57674.35398618899, 59070.93276262115, 96100.9142513424, 82109.78255236632, 65619.43346032934, 28115.8876709657, 5862.981980044923, 57338.404988046284, 95816.79232108661, 7711.0710965443195, 59814.267822780246, 55995.739573686464]} +{"id": 585, "vector": [45901.86969594419, 37832.91374915543, 28410.79447160001, 35379.20859793267, 37715.962738599694, 19892.144098957, 46136.07329064251, 94238.50068336677, 24069.184979689893, 7725.321852299582, 58969.16301484486, 20721.978437704936, 69955.86827088527, 79313.02143528579, 95936.50989340675, 75266.42782862597, 12573.489975767616, 25652.01018860126, 19462.86838644583, 5500.562656400542, 70011.26304029414, 5752.082502212142, 21717.2464114334, 26051.581417834546, 54061.62293355583, 57078.072594570185, 51532.044865193384, 95583.1365184488, 54161.766165485446, 92635.26529852043, 80273.75907416704, 35420.58841806988, 72816.73714425431, 74610.97590634094, 56664.01098623436, 57370.652203545724, 71864.23990109049, 13033.320553655747, 56170.387551897395, 62990.06231822229, 85500.26184306489, 36751.74855348082, 86560.52422580747, 30361.004574784656, 2500.2143218281426, 64506.91713780556, 46351.08557299447, 46824.68330539674, 69168.39628986607, 32621.422403675315, 93061.55421659283, 73496.05145665472, 64014.14832287059, 50579.04859231866, 91011.14200349586, 31209.196034289555, 9789.803199801683, 37371.94538019797, 24610.547903428782, 4880.338717872579, 19653.626434656668, 68162.82211366962, 9653.936232466376, 12961.515055186923, 23256.30814052555, 68489.2142016459, 84605.60282642962, 41203.36054564653, 45859.6046995146, 72308.20096140735, 4200.4768047891885, 28873.519789780945, 72772.026980634, 52353.324492525986, 48432.20389466456, 13069.595266909884, 72073.18359411962, 88120.90255218356, 41934.945617847865, 12996.522475401584, 44022.511986555946, 7450.628344552501, 82807.03994633263, 2214.307174505903, 17908.689685143363, 64757.83854747416, 3111.7383544516542, 78085.03242050351, 9376.578588311713, 86774.44567304372, 48240.03307025768, 97914.45606271872, 6880.485213700127, 12947.799523021886, 97439.06937616388, 56398.49249439804, 77054.38711733925, 82531.40427413616, 31802.597538488142, 79757.00729796325, 57339.8830238292, 73999.6966205039, 20728.56955796558, 46713.23648907876, 10073.016431450065, 65858.43172486339, 55269.312831647556, 96756.45742549145, 17503.37643273542, 14701.064254781548, 13488.075318167892, 65137.4060084281, 3308.732394914382, 41524.10506263468, 55525.45675910461, 75057.01662367895, 62284.75075307247, 36814.20817938876, 29253.64789567938, 56491.42838918383, 61565.26893070853, 19720.568631990278, 88257.42736276404, 20979.573199818147, 35947.885784365484, 32830.90525544088, 6990.725248987939, 80766.0167305759]} +{"id": 1361, "vector": [1855.738803768947, 94262.69228440752, 62792.1733982673, 18962.557556654603, 87907.39510138391, 72623.99170618856, 54578.61203969312, 78310.63788909305, 76014.27060182374, 11248.457986430494, 65141.496590080525, 10617.18791494981, 67397.61254415792, 98396.90560921405, 13663.733658291278, 95674.02659705437, 10477.484558576145, 45269.8819946463, 33202.86085574557, 77665.64141507541, 90034.53423009459, 28416.41734150261, 8246.4177149559, 98930.69019827402, 80797.37228209208, 95172.69526174039, 81142.97537060273, 2790.6143678540498, 29072.54107589876, 38047.71374176317, 6754.360054753028, 41833.71014071774, 43608.476520977056, 17880.448049547125, 62981.7916314852, 54578.0497905912, 17091.89478993378, 89459.9862637442, 49596.25978146122, 35894.286946516055, 90033.38802139739, 96834.72242084703, 85388.08030907957, 77193.49163489662, 39080.369971836335, 6869.373747404661, 73199.75227139193, 89456.82423562517, 37149.318060605685, 81071.06578265116, 62071.61756138785, 85979.2798698282, 71519.02594984265, 20800.121964260576, 64358.75393178295, 59604.50250715976, 51688.76745252843, 5125.539027101667, 15927.813535209889, 22073.433739794767, 27226.971262954612, 57232.08194254302, 70184.99902618979, 53413.32927716117, 15250.342418766139, 22151.797458446177, 51368.7334548983, 68436.02650122628, 73565.09580971302, 7336.142315533589, 7103.090081854435, 70944.11237738692, 2381.1844961267316, 41461.855327733145, 4564.387672497305, 56082.40715084656, 45262.275158820456, 60959.8144625482, 85326.75058368687, 23951.12847513896, 93436.86571722435, 43533.13293476394, 23277.670468007418, 12478.180367251902, 55357.92082191734, 88385.80175977471, 52626.96898146912, 94906.01596126321, 74536.90176989094, 38184.121631605194, 39325.536237449924, 49039.67846920682, 43406.237083064836, 70820.24384945974, 58743.44603917462, 6777.251280811914, 77227.422102291, 97310.18591014108, 78669.72468033183, 42996.70842581582, 82232.05165150508, 3437.445214437962, 423.46134354073195, 37116.16140393587, 97923.6482762706, 57944.521471259235, 96182.9147831795, 41748.66548006078, 33235.281345759584, 80619.5257460187, 85530.53200675061, 54469.41803366977, 18772.221180465753, 30443.149375658486, 67065.25390540076, 10893.692301017722, 3766.6558735518497, 38097.60472195206, 76627.50923789223, 74404.60815706571, 89073.32869333294, 71894.9202116849, 79371.81705724484, 38454.54176670456, 35121.56657119647, 96575.71771745005, 30233.568041017388, 21916.21627721254]} +{"id": 770, "vector": [76434.75042435841, 48989.569780841346, 81382.14993831718, 32925.17515625503, 37771.16572639587, 8652.800059527577, 84285.40882047305, 34861.673421937834, 94256.80022816526, 13695.703401315817, 91405.11251727698, 62731.25656803692, 47630.141054081374, 13857.24759861332, 82398.82405522886, 41702.0060301086, 48495.85578780351, 68181.65640642792, 27735.514766557135, 72581.64418059196, 67295.11527290006, 10747.193434993018, 13549.015374832208, 80871.65850580658, 95545.73723589476, 39230.153180141184, 87470.3659507005, 43642.31340819616, 43094.67097328584, 44397.38519706552, 15878.25693821726, 42124.15248773752, 15734.160916484074, 16889.365675930636, 3476.5612260893477, 33084.497317887486, 58412.14943307909, 75785.08228107354, 8160.916023298337, 79032.60417390837, 48756.44712354706, 87985.39314960084, 80546.31983657043, 64350.26168480662, 55657.59914659986, 24021.672607421606, 87617.9150903746, 99623.61665442538, 7594.993042529297, 9800.783233046574, 38525.715334720466, 55932.80481959149, 4641.698186352716, 56465.245739723345, 27011.047959468426, 96293.92236098276, 73041.21602865706, 70189.77645424323, 3419.5447830299377, 99370.97863928924, 8029.58880824679, 82267.09766300298, 59253.74001397339, 85320.97009174628, 82490.23997562351, 78454.3939207444, 128.07393981973948, 52848.34017648878, 32481.87661909211, 50612.83811916316, 55759.757514540965, 61080.680663226005, 20915.29047373043, 45078.618747158725, 41683.214843262016, 74796.38435113634, 90423.9735217241, 32442.76575961865, 81153.36146040096, 20777.465868370604, 3171.1066291143843, 56336.61247384179, 86592.48978545166, 91300.04884870621, 9847.147047326176, 62558.3392601871, 83190.39115599816, 2120.303583280847, 58952.50038318467, 27661.885263184504, 99503.65289778882, 79927.77019462133, 48116.03432430379, 47198.46302963408, 29339.741451141064, 62756.616002414914, 34307.0265727365, 61476.35405870856, 50212.69709154226, 4922.622277296207, 16688.96964871056, 83408.62170808858, 4301.395835656641, 21432.265239090688, 86901.25396499656, 3786.8544422125615, 6910.231685505874, 22983.37447867379, 24458.202771675064, 2139.723891418277, 62964.125784307966, 65784.23412810129, 53564.372223034705, 4368.517037216701, 96006.51260184737, 83403.2908623963, 90144.42347874452, 10071.358788018215, 67162.1531109719, 2852.0413795227296, 85908.77411978977, 12471.045034573248, 82055.21157880075, 84476.72952632568, 30936.137026433287, 68117.25669181121, 59038.450187072835, 13701.090504194346]} +{"id": 1702, "vector": [22366.09101516166, 89490.09599925838, 45939.57677853069, 20246.871198571327, 78521.40809181407, 95550.16533504041, 85876.17230690477, 45699.9716381735, 8710.996220333944, 70513.83557258631, 85758.79586753162, 4646.7935899598415, 86558.51562136802, 33546.393525673266, 2198.26701397835, 68074.72760441934, 8985.289371781779, 33546.68844293458, 11011.435458788788, 4046.39878704971, 27522.739508858707, 71979.2626202393, 77229.71316059255, 50963.49533116655, 80545.06783183153, 89122.79566065726, 77416.75693635934, 22492.642061752198, 53636.180447164596, 87676.0816855635, 84259.5085859956, 55788.32801486028, 58893.425633066465, 30682.326524321456, 39194.952701118214, 30583.021101600294, 52198.63148507721, 30291.885141929266, 12620.767105843266, 87507.52682255421, 99383.55913824022, 63030.24946318002, 20197.188519713105, 4509.7309017182115, 95447.7470533077, 24625.200435059458, 72138.43613902354, 61891.34686848833, 55638.99815529317, 80194.55482209905, 5812.653881833496, 86791.89248415477, 31485.386717242258, 64861.43414838804, 11791.849303629542, 91698.26256142274, 13376.900368476641, 52982.80855750597, 74880.09194854295, 35369.11870227726, 90175.87349821492, 74719.16715295825, 61844.951681922954, 82389.47816936298, 10643.25559975895, 81023.07223078239, 22284.604103730697, 52705.911512946055, 91849.80271366313, 14177.682423438786, 65820.01762723281, 63090.515396651135, 76981.0448877154, 80305.3624298258, 79140.22141554255, 86971.96016434369, 65339.59935044953, 42450.85911578112, 21225.49362490389, 21009.68241189396, 99136.49057117528, 33697.98133064917, 66554.02081449445, 27176.35064665971, 43406.32026309126, 88873.4607334685, 7392.175353218977, 73561.46026567022, 13985.117545817127, 26564.66466003008, 16666.977377000992, 50531.091987613894, 69211.29426571666, 21348.556480367075, 89348.93453334451, 21944.15380530448, 52882.888843020104, 14063.232987164798, 87435.68652547148, 38774.27105382426, 91684.76295822466, 87082.7224970525, 79989.11886883096, 17175.326195323592, 45285.30471998837, 50445.39666048994, 36561.99730387788, 65222.07307157528, 67916.64826060356, 27892.416016645315, 66754.53685643717, 80231.32907220241, 50787.12503375252, 2508.7573896072413, 33444.537821758844, 14766.7742523096, 55450.98676967476, 5845.586780421408, 48583.524611173016, 88663.3082950718, 94393.71512929384, 75084.4930761576, 74407.30901679386, 81929.42188213757, 32947.96467542569, 97116.06028294633, 13418.579353863592, 2912.6124367806037]} +{"id": 470, "vector": [18741.745659520217, 74883.56960144121, 95403.63623740972, 66064.19225309517, 80989.08598564476, 43775.64936302193, 94959.8587735472, 3150.797795611904, 1918.7992567062051, 37637.480550813925, 88446.07910886497, 6590.642750515441, 42977.31751960876, 8740.605408976799, 69476.8076837501, 48343.210292271055, 87143.20724849871, 71148.3021050325, 14282.784636018852, 69119.05251404009, 30639.78952013221, 34057.71721401628, 4775.266260595812, 21393.923718821552, 33777.16993442602, 72252.56291643383, 11207.301067976616, 81250.06099953326, 11543.300378646414, 51029.50790812947, 13091.478211843854, 95057.2190208213, 75727.91045606104, 32899.60753172959, 20653.12016326012, 68283.53893960796, 28914.836967768486, 89058.72488570496, 81102.05987153771, 54730.9102405561, 93380.73833476614, 90492.15321871934, 37935.816229338896, 54734.51915138708, 91078.85377667144, 15765.826568061104, 91257.69108537481, 81163.09409370378, 95901.63210276325, 87641.20678707631, 11615.327028243471, 61112.77715970521, 11554.483688895967, 22953.309886070216, 93980.52423990547, 25562.18269263928, 30826.7747691071, 53603.120099042426, 34599.865206748604, 28763.648835531974, 91301.63970729485, 36693.33949469958, 87085.84391724736, 35470.80975350318, 74526.97172230392, 61896.69488858366, 10678.132607112479, 10152.752780437957, 75908.1335980093, 42019.083740918824, 28342.7031974543, 76518.78971144896, 32931.11244765642, 64010.06155678309, 60531.66383130345, 48394.89991578808, 93992.51621438234, 31434.15573107783, 82058.86706188845, 4167.9246086753665, 93272.99229656311, 41395.39428957201, 14139.79237150721, 37015.47332592496, 11883.501067738289, 52673.12485771055, 34189.0510386563, 80529.04825551355, 1501.5112568799616, 61302.74888586087, 98183.91098310953, 59633.02562435413, 50624.82282406044, 22360.39756834981, 93502.67772598339, 8551.714808860734, 61468.47611956939, 51934.34864272572, 96898.0384700915, 16311.69233663511, 55037.75155194485, 65246.9143778278, 45401.23770735017, 84423.1115123673, 91217.89786440924, 45480.98699974813, 99516.45704895217, 30345.189307840716, 79579.24915887228, 26451.814006860342, 13070.781244419428, 84589.27363717643, 19425.52814281345, 87096.97254117987, 34828.35700259, 48601.77142075293, 62076.84394020748, 48555.05596250033, 58680.163622652035, 56228.274535931785, 85401.48232874894, 27030.782903501273, 91498.0061800999, 15518.826937935193, 90056.15170315781, 15181.376582634997, 11570.577518602688, 1877.8109300563783]} +{"id": 779, "vector": [17094.941892399253, 95657.85034641273, 71386.78976461678, 5288.975100479321, 90027.06340791252, 90346.25286713919, 39736.15177692327, 34696.7733806195, 95653.27893399535, 18255.186944538214, 81009.13026329794, 87743.81588778042, 27997.87536773062, 10952.606938052655, 46120.81516932338, 68820.18964929151, 64425.90350933158, 74628.7815838856, 19690.95444516401, 23964.412043744953, 64163.94363922856, 3710.963358755348, 91106.68716208277, 6157.8701568886345, 35576.66217043916, 67048.80024322476, 69494.25403815488, 64980.21858199955, 95013.54928803736, 22043.08835192561, 22992.427142804583, 23912.972729619774, 3066.2022561363856, 82706.10108358711, 34728.5759883192, 23266.422662079043, 35702.850451669154, 64116.04906182862, 5558.212668163209, 88361.48798044532, 786.1027038342461, 3025.5215797882483, 6359.364104521114, 74738.34579403076, 53576.64823743668, 21444.331524964455, 74162.86735427762, 74569.64521784244, 26959.402307819924, 73343.20005686709, 46824.63664048867, 22796.71505457763, 62684.71115893266, 15316.5320479286, 91227.78223914455, 83334.36154736314, 60594.56023292784, 24859.090054480694, 23009.9503139937, 44458.34071468813, 76895.0645153719, 92858.70729369018, 34328.01373508222, 15976.354832948959, 78550.99685164214, 60742.792278664536, 82548.04584228473, 30481.411004790458, 8574.016416087527, 57245.32901814344, 66434.1830416663, 50600.61083011844, 22958.44528733474, 96606.16832134538, 95923.90910936268, 75831.23894678762, 46764.803133055735, 53177.73545598401, 34885.057901010456, 3609.7267291157254, 50634.83680609432, 19772.646785388137, 31373.190034642706, 16061.626685612251, 51692.59724937332, 3464.537102906018, 19088.340465612062, 29341.44777826664, 82804.82110740198, 13445.908575217303, 69890.05047842188, 65477.85484555819, 72635.29441080667, 862.538373142796, 66351.02944169451, 32176.315245086807, 55636.87773119266, 25614.187168614655, 18168.667109650993, 64802.27074804232, 91389.22015872263, 35358.80991268338, 78415.46667473148, 95804.3313635596, 12106.387333587865, 62750.60453176435, 98607.32100762655, 3887.139080776292, 63422.73424242032, 30660.88016077263, 53119.10056141799, 33728.0491528194, 13556.307472148199, 34414.9328856897, 80017.89329247309, 4407.033491432199, 7712.572881398272, 68784.4802218895, 95679.77731309303, 17115.361068689283, 71517.6451628674, 73846.35341837417, 83208.50541964156, 86661.00960278841, 52431.01208104467, 78859.72518954096, 81568.51489213326, 48879.22043366114]} +{"id": 41, "vector": [94854.67285860547, 46205.16798748658, 70837.12220818877, 57828.1778171093, 10960.623880710562, 58765.97321173626, 82867.92873614687, 1412.8286402148915, 91942.33210240256, 22499.48506747127, 56516.416502255066, 84580.11361405166, 3727.561390367928, 86868.32832685197, 61790.1753229784, 52102.20452165006, 47659.39154381302, 61092.125275765175, 97323.33715321208, 69263.87689374533, 34862.7838335302, 77478.58689525897, 28983.713617236805, 83298.84171291377, 22940.853492148817, 74770.31845727375, 68826.21377060187, 7679.928516518253, 4780.97550417883, 39245.00475162889, 21248.324874890757, 56517.98633290607, 79062.69338782685, 41099.741502310804, 68374.38635085891, 71655.02823763459, 76840.1471348003, 94230.90289947379, 7044.525980075012, 78536.63556928077, 21799.76239050626, 9665.556926756291, 52233.56718073123, 64316.27843534054, 93125.17837682988, 52974.06026567473, 90075.47916586226, 42963.92654727673, 60584.398247332836, 49960.666603911915, 73515.33387714131, 147.46023171166024, 67868.27779420803, 52320.95793308053, 58798.17438667764, 80668.5963308748, 62863.979222347225, 77840.99328340159, 57009.92099411938, 68243.37006295378, 61436.058613092115, 7187.037485964587, 81987.32664175103, 3050.0894049669046, 36691.570084918305, 69466.99614666963, 8076.666533365528, 85932.89592133297, 49933.13901516827, 72126.29458857182, 78033.90906622498, 27208.396127051015, 6027.305081882095, 95186.69787947816, 51703.50309662357, 47157.2987904227, 74759.62017865446, 88703.22428318732, 4854.920753558756, 934.7267834239559, 74253.14922247233, 15675.25402951281, 19455.896591617526, 74970.50261024325, 59002.23055680596, 97824.61631818561, 12079.155854794299, 14276.131523698421, 83575.08352088733, 74737.47499876429, 98932.7977041659, 4259.232850887451, 16962.071204402386, 55556.32788173339, 73399.36129323905, 23409.99023227147, 8002.860126896816, 7320.117857262232, 69588.87033917084, 31386.822003142046, 64056.59337190417, 91883.79919843827, 36460.26591149014, 11710.74523009854, 3820.0053718824843, 15877.769489400362, 58542.48269877265, 20931.995435043627, 70640.54088089617, 89490.31437012088, 51925.77654925471, 64168.326216459085, 70224.95919208453, 28829.969128326004, 31840.45574418901, 74280.95877485514, 62871.60827933415, 46729.89555440469, 9516.736663180625, 50870.85300479708, 89416.88900280546, 9536.10503934399, 23238.319569860032, 20679.82592000649, 64564.923053323975, 70611.78232637144, 14568.185800563515, 88489.6104806902]} +{"id": 1207, "vector": [95274.64689471916, 94726.6312299184, 12014.183446666515, 55112.67406650309, 5523.953573940122, 5546.169524336908, 86395.77438811625, 22348.946993342044, 42689.29924612835, 53755.8650342002, 48804.43694719048, 62270.636160228896, 73228.39530240909, 81071.8401886427, 57259.82594864355, 73120.948133342, 71137.26258368904, 27655.579701687162, 93838.7515927486, 55990.31202266221, 49859.17963997639, 32360.290146808245, 92603.03800074567, 4579.43280872073, 17805.65288769612, 95584.8042729791, 8513.696994305652, 38548.696638936904, 14892.711335262198, 2494.0816344306118, 97556.83087161579, 75966.54022307883, 38237.90601095497, 7514.896009326466, 48255.66359170067, 55113.301541114786, 66464.98933081905, 12738.64824035671, 30990.278628447697, 25294.67469436738, 54893.785903287484, 79048.54244423818, 65574.94140874612, 93170.10177705782, 99396.40514981057, 44556.24678593942, 19195.249126184644, 61182.41663787268, 60845.41941967194, 33932.86896763204, 82432.44897082509, 7975.988612038265, 24475.5067168143, 62589.21780261111, 1029.627442671943, 63299.35184583788, 55771.64081496261, 92909.1362466704, 70684.37230076281, 32326.540197836584, 74167.92957380298, 77756.60791709684, 7429.312786437447, 32530.060776765225, 62781.21689593049, 44447.24538028527, 97930.03737570829, 48307.86969416259, 33359.16649940759, 77391.89450114718, 20333.612556451997, 82573.47151282315, 65643.08740483667, 486.82837145727075, 80011.48985475405, 822.1806104045992, 93932.4492200358, 73798.77299315215, 47911.75371231632, 71237.10503561154, 2784.996688803698, 47949.74147433484, 64487.3525929048, 40943.983136117975, 18632.67848186, 83740.83455319428, 35546.66861026789, 7502.674489261752, 38903.116873127095, 8975.59335665744, 53677.55259144485, 25010.705511595865, 88471.93199618655, 45442.78170518051, 45784.06735800405, 17925.454783328863, 68666.30725886975, 92526.35656312804, 20155.70642112875, 38296.55538136316, 44398.553177369926, 13053.81701254622, 43637.3471603764, 78577.20165932408, 6222.473239788284, 99160.35562580918, 60955.73399278087, 79574.67370830008, 16165.252412653075, 80102.55531491862, 11618.190682555274, 80505.95362062844, 47410.09543512782, 45774.93134443894, 760.9988461984685, 51493.082956035716, 20865.755725409163, 88213.65561497465, 23035.962267600185, 45727.255578675155, 57248.91391745147, 33552.43854247106, 16763.039268573622, 29230.738510013864, 24269.30336810823, 87159.48692771475, 78634.09833036816, 45802.51689446057]} +{"id": 1948, "vector": [76555.92577543446, 47408.80798236761, 69202.97197912131, 9395.064510291118, 4058.290500293205, 74665.07782349944, 17320.210400272717, 3830.0213679665185, 32002.200746894472, 73673.18134336182, 81235.47837391064, 98227.7302749774, 46136.85278700313, 86333.58915382317, 49405.27207442268, 62081.64410753611, 21474.214693617254, 9151.98598833249, 76334.40110189644, 50310.01649958959, 22093.44056047855, 29950.262278134564, 85420.16654579724, 47315.325737226645, 11312.928507052433, 822.1721134059168, 37161.004444982784, 73293.21673741397, 37722.22074543109, 66439.74353687912, 17263.80389317551, 5294.24101291176, 14262.486810907138, 88480.7246467968, 92959.45010842387, 42980.4569946917, 22166.767751362026, 97694.4227249317, 14879.034193434836, 88805.66307256863, 64252.9088732954, 93824.41979527856, 83157.7566281524, 42134.09088822159, 34773.69761909684, 8091.659238249438, 24254.943030957988, 19137.624894793014, 14057.068675554196, 22411.57237060537, 54569.15403122491, 77001.06163103417, 73665.700651011, 66034.09094174598, 63688.653129256, 43820.31599663707, 10463.727585094528, 56767.0726744733, 55201.809120280996, 12760.633307334047, 56009.01118008458, 82001.22457411258, 13338.463334169104, 57228.00748370244, 14814.836463037384, 77540.91105683729, 15038.525916423096, 37754.34619057837, 69981.32023241122, 46918.28681649921, 32973.75030569301, 21057.27431440383, 47779.92669895061, 41082.696976453124, 40752.94419928233, 87398.70633333933, 21410.81922440887, 72542.20210580272, 2115.918990151866, 38082.96886374686, 69500.48000834588, 63443.32579719611, 33980.437742812595, 82837.82222261057, 94482.1635306299, 20705.213433353143, 36743.40210478876, 60049.108870469914, 58856.466458919, 52593.03360040412, 24022.25549331116, 36281.64018274653, 72915.71460997887, 53538.13221715688, 8427.690991858828, 64698.61938996495, 24054.770313524445, 32470.949918670998, 54173.93429545906, 76473.81586003014, 94452.49727096292, 76094.8405749172, 18943.08841581972, 11258.722120530629, 10378.161594832934, 86162.01472923082, 39568.123597599435, 23990.183045746184, 10664.99296135094, 71560.35396564398, 71385.20829904816, 21416.39239385178, 24605.03342405116, 5118.62958210314, 90262.05392573097, 68605.51233669627, 1059.2330675094242, 85768.96833147672, 64269.498338744765, 78432.78755185593, 33025.33581226856, 4689.3982385131985, 54322.525692539646, 95840.8653555114, 26656.032443408363, 40565.988575814285, 52127.35378976368, 70560.42881538607]} +{"id": 1218, "vector": [1255.0250105440864, 41815.05804745118, 55462.66014946708, 63917.154062095375, 81980.54856523951, 28148.154031788876, 52631.66235101956, 10601.558979155101, 73663.46152124793, 82885.06854746537, 43628.05938790911, 19529.407400191834, 18502.853602422387, 28836.714851410215, 96537.58614671441, 39809.986326538, 94312.81096939692, 62927.93606149252, 88060.86172693099, 51442.86531574072, 49980.67611056896, 79832.7602756661, 94493.1549123777, 51270.58892334773, 12380.132773393549, 4603.828794365761, 6615.9955242477645, 82977.71061610406, 22775.440869009366, 30806.926843512385, 65543.57663386852, 48753.46782724632, 28582.029862442483, 10546.556590155953, 16245.704208006895, 79757.33016629964, 24756.681041033746, 30240.733388524077, 78724.48805297974, 23338.262302935942, 28289.02020658912, 18963.26548943652, 79522.26100071355, 81302.3949281893, 17748.805551862333, 44549.593051227486, 98194.44557073378, 10343.720695234893, 34734.42842417822, 18169.63270805537, 17158.85678730371, 54964.12476971553, 41795.19297844868, 89136.55205534892, 74541.41486478312, 93688.61426696187, 13511.60117304393, 3365.7860937298724, 79284.71885685415, 7430.209328230985, 13895.162016948092, 88570.90873901859, 15589.142263506106, 55255.10245720462, 78364.80348057448, 65286.39167880575, 76363.90078284786, 22859.820290192645, 9448.424525933118, 15511.586999378791, 48859.29119951025, 98432.03594364153, 22258.613995329113, 17975.65023583614, 48927.95346591414, 85422.28898000509, 33396.99208432835, 38239.87185541932, 99244.1128492942, 68425.88339916487, 32816.81894876493, 17579.373807731656, 63208.354148349434, 48972.00364067789, 21983.356738649018, 80212.458707341, 63080.08273519882, 80098.0220446311, 18744.005150401765, 46480.26093343702, 37506.89604030869, 46150.64946566985, 22587.54998405518, 47439.73707398278, 59754.297335433126, 26531.399087825215, 32841.8924596821, 79706.05421026342, 9536.505121191185, 20934.224219309726, 82253.22076519715, 22442.29517620072, 13899.641277027109, 23308.743765610594, 23350.586265289385, 97023.88629959775, 62182.93783971659, 8179.69226859957, 46819.38625748619, 80364.3566456233, 15294.039699557261, 13641.22254577771, 29783.173911354697, 79108.59385898155, 70876.04435571776, 43014.63225233973, 27425.855235575436, 47043.49667366483, 28979.816387487732, 49640.10026155256, 26384.82091043245, 55569.764056724816, 36745.67901543089, 6044.307355512624, 45047.9613489829, 12519.804739627416, 64513.53654470699, 90858.09023525828]} +{"id": 876, "vector": [44524.730996301965, 59020.71804622887, 32227.990762609348, 87416.35140937546, 31086.153517640414, 23787.618977940983, 45151.66373824654, 86938.71098488543, 8884.042203133491, 58905.50632144565, 35887.324694196366, 96193.40405914867, 87895.63150591095, 2522.2023451820232, 55444.27465920616, 45893.58807902552, 58574.440847582264, 13189.83639284138, 79637.99610499205, 19411.461490213078, 25536.666758722327, 17461.614559932925, 37955.84002217457, 81941.64998833706, 3628.2935091136337, 10199.936998119209, 99820.05327140543, 53571.43235832258, 93048.08461074256, 17553.260872898856, 25664.028982136733, 52187.537192258606, 64235.10834990586, 84351.62974517095, 10267.736962981788, 32305.310262776522, 34968.17656082586, 16300.446109670274, 70167.59967119267, 98317.18902510307, 22660.302856999493, 33033.42963569715, 6781.143099999354, 75539.07746924904, 77743.37787024051, 16401.821042875752, 19230.588033946195, 99151.1406471949, 41392.42422370851, 76584.53909163299, 80918.76099719292, 30797.520422046797, 87698.60893020166, 83986.72251864956, 17745.658158137954, 42749.40171513761, 30881.05586107902, 22864.26700991544, 14915.928731433425, 55476.436793600005, 96351.07617986057, 10439.291364202929, 95686.39553733708, 4798.705416818205, 23826.10503566249, 28104.524126352502, 4463.671432170036, 20860.079518697283, 4963.026660255532, 78455.40377617069, 74790.1404684398, 60390.15214626819, 80821.04024621971, 35718.1863148203, 86934.81412369925, 33152.640466150864, 93405.89606942807, 1351.0294731443273, 90831.46638681139, 10112.67019499572, 1450.8701485238528, 23680.692216115516, 16888.544369340274, 22085.245448694157, 95996.92556166755, 84431.27820276526, 57494.50537719457, 41461.88894605632, 972.7325913892026, 71530.0175104507, 82205.17109992963, 2275.1172067787315, 2675.371607781496, 16347.725211743858, 43926.50448107921, 97994.90659931231, 56639.047943452904, 35417.374603409815, 20903.017760094866, 85118.73021994662, 34470.84782954668, 36290.75036797783, 38261.74612676937, 54488.236204892826, 88870.14404679515, 77032.1295292745, 82014.77743800882, 15101.898159835702, 4326.33671719077, 34337.126234276504, 35873.90946496044, 26688.32509541008, 69710.68871289783, 99922.49731062273, 45678.10412972959, 60748.348810492884, 76104.0461041461, 83470.1973627124, 2528.8352675607252, 49573.593953995696, 26691.973808020484, 86952.42119911268, 89699.55113901524, 32634.108422089903, 26219.761679620467, 33308.77431177105, 28312.479393506484, 90042.3172812886]} +{"id": 1226, "vector": [7576.7408646918, 29796.555112113343, 29036.93116729673, 64599.33125583688, 12434.937233847098, 26569.962523551872, 96719.64404545903, 92770.5017523275, 58008.42457608692, 12755.172111854585, 48867.29805340904, 43166.2165864646, 29738.50229965773, 24132.670822896063, 2689.120832579306, 24076.307881322355, 42010.256231951425, 74147.43452453273, 14438.983479084332, 21075.725739819194, 70703.69235665357, 11399.981413604975, 63626.325978167966, 51561.214658955956, 60810.97168095932, 69738.00810104056, 17009.59428254207, 73865.74375829563, 81412.3158662824, 19077.04987143498, 97871.007906176, 20389.43014763781, 95764.3071984767, 22668.580905786086, 32083.00654401929, 74190.46242102218, 53265.86584219817, 57594.11540567573, 24190.91763177489, 99714.66469315006, 91995.85854419599, 2199.4955666310047, 3140.9182331649577, 82667.7471353023, 34675.283859103736, 9926.867914080118, 19085.071726723156, 66270.36845484872, 12214.735571124369, 40341.94829862758, 1107.5355343216575, 87985.88170814565, 39851.37488078456, 75624.70316956722, 22114.63500788432, 30294.475365378326, 73099.79145401706, 51996.068630303096, 49215.51323581945, 23749.552086296288, 15818.30674961653, 2463.4459498334318, 30111.257894787435, 7751.9344881019415, 67630.54670569969, 60470.16973963942, 53531.74037483126, 60562.66995530206, 74013.48194929943, 62369.08769940529, 28161.91683520134, 52921.324099500525, 44754.373948830194, 35588.28848478774, 60759.63623650063, 49700.72158859646, 4104.09886034212, 41718.41872772839, 90721.8123237796, 32416.107644474945, 95461.69419429165, 48932.62320498009, 35622.538017073246, 65503.36942728119, 37796.108805841635, 41720.65242911107, 85965.36757723527, 18970.2367953015, 43946.93990453458, 17217.47420619427, 49776.889994834724, 29028.208074265116, 81400.91032077745, 69045.80880634878, 58821.96938213452, 55650.943935940486, 82927.02458475955, 6047.111473720901, 48592.44580015878, 55742.20560366134, 68163.4946695955, 99992.75017514687, 44953.103714347264, 57556.852512565085, 69512.91730531737, 95765.1752261611, 52473.55877480102, 44346.91645475692, 73700.04227858021, 58982.40334288067, 21434.511775361563, 60066.242529439885, 74349.64390390267, 99144.3737020482, 18466.11180066069, 57612.70764996871, 37602.90763860028, 81504.1127437397, 72302.2025724186, 29197.50038213912, 39652.06346100015, 92894.01047733767, 15009.834698342118, 43359.32033198668, 72016.58059234364, 2856.8369207976784, 40863.42917100681, 21668.52101490182]} +{"id": 926, "vector": [65164.25094846699, 606.8557578818079, 65279.602929570916, 78050.1150584859, 73691.19488171043, 6418.271496054539, 97862.97832219317, 89854.19417292706, 65263.14941066037, 49445.34456899433, 89589.27802753136, 4199.112219460832, 39549.23953697601, 21590.29593140086, 75736.71381548382, 65939.39625633962, 53611.13605327744, 83404.03205938423, 54050.70697785539, 15413.992816086653, 30240.048452243573, 65122.023176644274, 28034.14613820111, 7178.950105977821, 29327.265446029938, 99553.49146271902, 64686.879459401855, 37516.39886846666, 42117.44550333628, 52034.15612207533, 60066.0566857333, 62244.08442649338, 656.8162405659383, 23648.579168637352, 88658.6934267779, 92880.67303512937, 28399.66672368598, 70537.23313648452, 4753.810611578479, 17367.024053729707, 58687.04802185223, 18944.470113669842, 49801.38128303435, 46916.56296789324, 53078.8506088379, 2334.6836729645215, 11048.916663523556, 43660.02885668151, 40765.56071672265, 5989.729972502211, 25783.354854951558, 86783.14893923586, 66242.82615543915, 38312.42908448079, 54410.03240784591, 13753.14729477961, 87364.92359726618, 45614.18155778075, 39841.99978034633, 61173.87611697174, 50890.45963792272, 31093.64999995886, 34616.37860580528, 53393.09388643873, 89284.63150279783, 43358.041211501986, 51752.32692426007, 98871.86160515598, 91120.99219498974, 45957.014906061646, 60518.83029797185, 67694.65395181405, 96235.64599369539, 36334.20130939988, 60291.16065530996, 48933.77796057094, 74345.82145593014, 88772.57365625357, 43314.89423360656, 80241.76977984825, 8332.78730264685, 4748.641969023526, 67603.7944253331, 57823.182893888195, 511.88946940239344, 73508.83373831856, 49812.691575571145, 3522.511592217925, 39288.06777966247, 45878.96015113251, 74438.39796475114, 95219.68514107676, 46731.21871232006, 29713.119425250457, 76389.15343725985, 36567.85975951287, 59225.128047564656, 15143.072083248322, 21900.545542746473, 64733.477244854555, 86365.80368966458, 41232.01369573879, 13919.526932836912, 58018.55047564798, 15244.384738015893, 92804.00013774252, 22310.697759914798, 96237.07759645168, 81445.98172384498, 77060.1872857355, 24255.228351073532, 54404.44009278373, 63348.51310274403, 50191.303606964066, 62629.497197064455, 63706.97873098522, 6392.047955858527, 56091.1213397397, 61004.39363920164, 57633.034678304684, 1228.0505093688387, 6844.411612679868, 81952.0852459812, 9340.529172285673, 40701.70496075415, 55259.74506955582, 42919.8753107384, 83005.71952238209]} +{"id": 940, "vector": [93783.66884445463, 70407.9951480181, 42875.51106052218, 66801.63987637068, 40887.56709167526, 81543.17677907656, 59100.004823675015, 37810.01046337034, 54605.52633815934, 62076.19939480775, 25789.96727653584, 70801.73462568774, 76880.90196252997, 82736.68074569634, 74454.14786972199, 91598.83731631176, 3659.877647808751, 40755.01925625655, 48144.01703243, 81565.70230226371, 53669.16503279751, 19647.333415414836, 46734.12869360489, 55491.896467677514, 4917.749729069709, 1876.2432676007522, 88207.78668262543, 92196.92391547536, 46883.6539154014, 97471.9460194799, 38875.27306207087, 61770.27328174926, 81730.08661840764, 61026.508111416064, 45390.85076515207, 96260.74382476757, 66939.8177964158, 42026.65572474423, 5370.971236975097, 50107.724974235425, 88219.1439602482, 77886.70385245918, 73092.86456906192, 96807.24650780993, 49459.73174696197, 45131.01656196661, 82244.11926308555, 75002.74992855577, 17937.186058881794, 87358.17280926584, 43534.898017955056, 17987.555740634918, 49684.375130027234, 65751.36615479979, 13592.840389523886, 86881.49489371364, 45019.86937305389, 94853.48113538674, 97659.84863146381, 40524.160757234255, 84020.70077562697, 21375.933499387334, 74830.79268265456, 42539.11450681059, 731.9281163361558, 94167.46333869577, 48518.24773894, 68523.69947784931, 67747.70578793719, 15739.108975612482, 52504.05519771535, 79177.05723095035, 77159.84354286916, 26825.15560624944, 42442.26077510553, 45478.89079571824, 88393.86020626199, 86629.56979157157, 84935.63201341734, 4964.450640430907, 98117.03549718023, 6020.888903472976, 13718.915527617837, 92099.43984872552, 83235.16027420401, 41877.0691353526, 70634.01910685854, 58511.47355387053, 19401.098801893833, 38168.50608015031, 68977.62986188935, 29189.605184319724, 74343.3980384224, 15884.47060184035, 12168.842393690704, 41017.85459168324, 14102.166900651058, 28615.733177174374, 99070.40261322372, 75486.38831520009, 7129.001999592466, 54964.23269270824, 10084.121391351242, 83522.9566940699, 23758.77847044302, 92498.45230852615, 57597.824491813524, 10325.516527001299, 77534.79139026234, 21859.448363033294, 60512.187985975565, 57936.57421217867, 98840.69437419137, 46536.46086215463, 75295.91168085694, 41450.92595792703, 44404.74796987135, 67665.10396891575, 52928.00860328384, 85832.42136134367, 24823.46963605344, 58572.51433646348, 88328.65984487822, 53909.83151392229, 20523.984869511358, 64798.46939095239, 3919.232953277718, 92336.39697507813]} +{"id": 936, "vector": [92279.2071005517, 93810.79349400794, 49119.99205961395, 74337.0269025168, 15505.779350211023, 31643.600466383592, 41374.98896997035, 15594.211196778817, 27727.3758002914, 36394.19824332655, 11718.311302028595, 6053.265635488847, 59771.281591297346, 19755.391759455786, 53074.03447271458, 21681.765668794862, 59063.79819641293, 2599.9545120933167, 62221.95587262355, 52292.35101144102, 12379.121717569096, 60391.56038739383, 46396.44791758349, 21309.66812724028, 4338.482420912848, 51062.447466354824, 92211.77078683252, 28345.72839580417, 45790.77841547111, 18253.955861284532, 53203.60192212754, 21948.260467244396, 79158.54241717923, 88760.05128679829, 56581.43654821876, 24844.820276211598, 65630.94191703567, 46601.25048140943, 60558.847062655615, 22643.771722294405, 37809.12827409146, 9162.383166335609, 64302.226388398034, 45331.62593640496, 51288.55024194086, 47630.67660545357, 28475.617520634267, 48041.011927390224, 89098.33882759507, 47708.39145751651, 66530.20448235612, 38511.59346071177, 13201.376874456582, 30129.172500183122, 46234.088963033406, 78516.82518949204, 73463.93392282655, 72649.05204210259, 50417.231519120134, 35276.03471338505, 86265.10358548404, 25650.65339642859, 93215.23251962275, 31653.47083652076, 24220.860517134824, 56538.33866590427, 92991.88720906326, 74941.79398849298, 37007.34754435992, 28110.70303733604, 73835.35840892134, 58426.517842465444, 80497.27771900252, 2481.0407009826176, 70490.3587282085, 97560.55672667618, 20197.136314116447, 59084.48284990774, 73998.7476049127, 48670.20447645791, 20715.519351091625, 65.32701894751413, 85733.61831999291, 38136.98991206272, 18259.4889923841, 89700.43516540364, 24538.924366444804, 16734.076733940517, 76750.72227090684, 51512.28932159112, 87658.39925977236, 16996.295493753845, 45287.24699934683, 62363.86068474591, 31736.830457337674, 96729.84090644206, 91825.3774342, 33048.083950848806, 16382.772027088333, 9914.1214813263, 17280.781983638717, 31634.101361099732, 38485.31087983564, 33233.454553666954, 80151.63050586412, 67310.20022040955, 71968.14232756027, 9852.677448324053, 94344.20347712688, 37938.85992252282, 17595.08893619022, 42653.511117121445, 59241.80100851628, 37446.215945712924, 21951.477702630305, 6313.598150217792, 4120.881309767388, 32128.21757362089, 13788.339478388656, 20797.2115795234, 41089.37197290543, 54178.30303902741, 84098.39499052944, 32265.813798979594, 65762.50031587115, 938.5075626878514, 39662.457840910305, 33624.48795507485]} +{"id": 445, "vector": [88788.8120564028, 89426.87958527954, 82146.1096295986, 34399.25620213372, 67697.06428663568, 28504.03585338741, 39862.489770649256, 9463.442369292163, 35004.41959759788, 63551.83014046153, 62855.44589757921, 42170.4352327264, 6168.16416740924, 39258.58447540509, 48555.49704756354, 49323.967241030994, 25658.75745078786, 56894.82619216065, 90452.55509601336, 7791.018522825621, 65581.10485408941, 22281.73265382658, 49744.20008284577, 43380.15206726853, 67520.76279666675, 64033.93004018771, 75860.45430658231, 57816.89866412656, 95473.31901806149, 45511.810839903, 33798.40783191439, 48916.30545583474, 16897.781119165113, 30937.37479313091, 20651.837263158013, 51318.77276881037, 41524.24223355731, 66758.65904794555, 2301.8230799449866, 80289.82181307282, 8113.123597014138, 6915.997241933047, 55061.29326984897, 88130.70028197081, 65588.87911951142, 93870.26612343128, 70873.39789782882, 24174.540274668023, 41134.38139498589, 65135.412256990996, 51788.4547265316, 33786.278728062716, 37731.52618120974, 33469.86885006315, 42441.54429065239, 95043.95043835959, 8254.299646726016, 98847.34415553792, 99264.43354480843, 26201.91244921023, 30228.57651181352, 46014.8717221196, 77784.50612888102, 94984.58135475057, 3215.044391366584, 35974.080009145226, 53346.164086045865, 1044.5375588052252, 96674.73479089285, 81190.787992326, 12550.677893487727, 22937.065306155324, 92541.09577331104, 56901.40474402148, 30521.999975658877, 15650.286144558811, 77846.75712574861, 52606.443455719884, 54820.5204522029, 21931.41039898938, 7464.017837403559, 28800.954114528144, 68508.59642680302, 42005.906535576374, 89408.31903344036, 27182.058267176424, 37398.92861477716, 10411.690636731108, 82037.32611538291, 45255.802670597965, 64447.73821700108, 37685.57022086149, 98945.01467138401, 17198.78700892593, 60940.58962360957, 72614.67226899839, 68987.80098680455, 94191.0347479074, 6862.544173843077, 50540.51249720082, 67070.9150264198, 55594.929076731845, 26385.18960479912, 1541.0084805761005, 33068.74336287912, 37290.24049612144, 14998.343683926263, 77871.72270486415, 14546.981555408045, 83857.68460827712, 74533.6662668242, 59886.09366605786, 61185.921186709566, 20797.03906591829, 89645.89290780888, 4741.582510222853, 19930.78477950254, 54576.206236409256, 54145.51034196684, 83857.18701910214, 93070.44889399256, 44074.03079522816, 3965.933455860826, 49404.976583048934, 54538.68729297983, 92990.8531062935, 82809.0511440419, 57773.03495167549]} +{"id": 591, "vector": [35655.160234041236, 70534.4671840067, 52244.89209258654, 2708.78194862878, 96280.32050662447, 70038.8977640118, 32717.14146717121, 5099.239258455179, 94957.46687468525, 9101.046952480441, 27125.814188240183, 14017.280370038054, 11610.77733075153, 71244.81097350597, 77566.38226989297, 47037.250364989006, 7571.992023453189, 6535.620945660358, 9634.764815342567, 36444.98399951416, 99136.69475745132, 69265.46311586423, 74546.97337784142, 74762.43100783703, 17923.23265047825, 9390.975685906977, 60707.598060546145, 15722.353397368704, 1034.6884651159849, 48287.102101412434, 52398.81218777192, 59562.580148725465, 93311.39755503133, 80924.1917989984, 49233.52229113248, 24.05203222765051, 68022.85679696701, 3640.787654947253, 35395.8191766419, 5768.98595981108, 10031.01063998344, 68530.75326751138, 25744.866290718117, 16963.386918432756, 18019.42262678129, 59167.44714075335, 73602.46727063038, 35738.645923583426, 2941.586007514163, 67301.42133832935, 68453.07559012607, 65114.18494919035, 2640.621126255127, 90914.01670149947, 58401.81120633483, 54269.390910097296, 10155.061479682692, 17088.43822993059, 4643.832960972327, 17890.33801367509, 15958.130397936788, 34926.851479021934, 7062.1241728241575, 16848.13975509395, 23275.328804440654, 23496.8963285751, 46383.15778279817, 36290.853883990305, 71914.38352572975, 90146.62665666491, 86944.46543613488, 81443.45096827808, 16545.500461514963, 90211.68840643209, 48782.12417928974, 78752.52643813289, 82751.25701978208, 31743.293658269777, 50854.060531476905, 64898.28507942888, 90513.13759591167, 26131.640522565423, 39281.27634737394, 96616.11234814441, 74868.21696965437, 10307.434191075692, 85148.6472153038, 71602.47088040746, 4685.4438327992075, 80745.18103666231, 592.6981802947373, 71368.71589950418, 79879.21620196948, 13941.60872829796, 27970.474549269864, 25479.935465504732, 17348.63153648365, 62011.95814325555, 55862.528232530494, 34583.289441428824, 22979.88978332691, 34885.209182474166, 77912.90093655392, 43402.54136093154, 82404.45345276345, 79613.96463380083, 26396.273205641086, 73886.48260619509, 34303.2012831372, 54205.513435169436, 67388.20779689013, 57865.43140149756, 35085.63247045964, 13536.664843966562, 12457.289700305373, 98591.37779059465, 82634.3760028237, 17142.936856590397, 47248.8528678367, 72276.85088841205, 49207.135705139146, 54365.40585015828, 47523.77769552641, 35130.89398875579, 82215.13792532124, 77277.0331327322, 18274.114729004177, 24255.30743809069]} +{"id": 874, "vector": [11128.044724983254, 24742.10317468275, 26587.910564877304, 64632.43897050516, 71385.06089367435, 94277.68510622469, 16628.830894928105, 69228.43707380732, 54824.04392911381, 48118.396235768305, 55506.9654090919, 33141.80094200611, 68982.10608947986, 35962.544827889986, 5588.866784055269, 52845.57195131863, 30980.98010295206, 23005.19829032739, 2105.8646364200627, 65214.81066317116, 95431.42327793995, 15165.67609084144, 39021.41013126346, 5425.525786949847, 99224.31888587966, 48424.94916705457, 24753.54121904173, 7867.722940568278, 23402.31451473116, 58610.89735278042, 42865.345801073876, 76223.2745302826, 98570.14577063739, 45861.42914329994, 61266.72778100168, 61978.679769558934, 90570.49998623125, 90011.13046626518, 92064.97508870263, 43072.464520002904, 94953.29841793112, 23843.83967656636, 77256.87517077621, 52475.12929618882, 7004.902612375275, 1453.625437025352, 15579.506376804942, 90081.12599165134, 89143.28422821796, 80949.33995585293, 40252.350532621975, 13549.527027161956, 91361.34513021776, 22915.01767250518, 40196.560628147774, 37320.31510323168, 84432.99376530432, 23429.91856515697, 61939.00249531468, 51640.532221352296, 38097.2769796476, 73142.75947985954, 64341.93365901766, 6217.978614267694, 60080.10561266196, 23679.416299462308, 15772.89305791203, 56665.54468647146, 61093.725761143345, 5761.535504946324, 52895.372961928224, 84920.05664763048, 71561.01987955382, 73943.13401971951, 41891.41015395104, 4989.364043716016, 44339.95901015948, 37039.96021393343, 37771.59011904181, 36382.59958517257, 38571.972099806095, 75552.02466962239, 7279.860521050497, 96932.15846569903, 41350.14480502009, 48022.78252481053, 80580.17334674574, 76200.91627751813, 29456.268660504393, 33038.03929538097, 82641.7700921256, 66652.53052382139, 74088.18511767845, 17026.074857798467, 61665.34151946312, 22176.06261391111, 70749.6123120782, 99545.69413913201, 86883.81632237411, 7604.100386539026, 73250.24894960265, 95695.50875550475, 65709.767370495, 38665.13001086015, 43811.356084203624, 87715.7730613923, 431.9487804619881, 59942.74611706047, 36404.241274537984, 685.9225675231339, 19005.80571669641, 93079.85437858614, 69772.13839338506, 43195.70924050562, 6507.242767979271, 53434.93943550798, 4413.576258515773, 4940.570797930044, 32183.914505427012, 22969.93008607923, 67591.24656709458, 88151.85817780372, 38693.70772861925, 42156.01786850547, 60892.92720255821, 16939.36259854678, 77377.66053919183, 52407.15034355537]} +{"id": 206, "vector": [71822.4986493735, 74922.45157315164, 87600.3506537045, 82115.37777517938, 60341.94341397727, 47155.45352778043, 83470.34648239007, 43184.65700474625, 36841.77599672973, 64885.75692922607, 79283.85810738566, 79682.09692388188, 83864.75783292361, 4935.279748703125, 85175.18092568089, 51348.81125033526, 5186.939124388346, 59707.37658623829, 49563.55028199329, 48899.45336442173, 45866.76805858114, 79566.24184545505, 41514.057320206986, 11817.175049834837, 30606.02975053812, 46813.59224838086, 95022.39057944079, 40621.696626839024, 70436.09297945292, 79348.59578936537, 47287.321013761466, 20486.573599869273, 96865.93012426585, 34848.37567475714, 7249.528416696705, 17455.667411668084, 26586.272158918124, 54097.00533692946, 32014.816696570404, 15942.206843056461, 77623.60224653914, 81395.24410877294, 5590.956697748028, 45903.27343316183, 48582.675745326844, 24358.79946076053, 6861.339854893756, 40738.31063089086, 96280.72745990251, 53907.50946289429, 77422.41281007735, 25844.66279603668, 4361.845713469615, 52790.630728443924, 2381.716243998588, 3341.3888094551926, 54346.82942724608, 91861.97344878364, 54259.389031691884, 58528.38122154824, 37953.115232505515, 87664.16572900393, 95880.5992593176, 60942.202103157884, 22629.827692835224, 94143.86989452937, 5303.606847876729, 58772.842842658094, 14707.454429948375, 595.1786465151488, 93422.96265335164, 78468.89122257975, 87911.58469637016, 99488.82509392187, 77852.72910966437, 87428.14957114647, 92669.86774661376, 62741.9479951382, 84465.33163193498, 7098.075994803965, 3971.761188624845, 92171.01685248883, 40855.40390436897, 36256.02253880984, 96921.71552389582, 48305.402136184406, 229.04755410628752, 72853.28496938488, 832.6060088962751, 55496.57753851666, 4170.741436928982, 69102.26382534268, 97407.1915157311, 82994.14681343704, 35956.486618353665, 81347.66265448707, 15854.136370459815, 19235.946246311174, 62162.498550710254, 8388.3362458679, 70449.45002965722, 22381.62213265248, 76836.56543288947, 11954.907158807482, 94237.18843670166, 53024.91401579798, 21572.412855534072, 2176.8803403150573, 70579.13927800457, 43093.90932292579, 79636.7854988786, 88148.65291506577, 74962.45673755785, 99055.90889003946, 95796.84305642513, 63927.32415543135, 8337.13670780739, 24030.901895903055, 63246.28735318501, 74746.6907608312, 58186.73414952138, 98094.33859317124, 58621.37164145984, 26308.033146779242, 5576.272678330852, 1032.7229205560063, 15460.945586752829, 18458.082028007473]} +{"id": 1499, "vector": [17320.993972118336, 71113.14169922953, 85578.5047804288, 59953.34449789379, 76823.88274105031, 2323.738356978944, 75155.98645517825, 86531.99657542538, 57679.18001152945, 78611.51075402311, 25134.377195796696, 51720.911919273225, 43902.05543307527, 98228.04778799116, 97753.55871811695, 22486.863562199534, 36401.45084459295, 89377.36613800273, 29969.38378039493, 33993.031018865084, 65877.53614969272, 83368.81377603533, 97363.84161043564, 5271.207426823221, 58483.971851884984, 59826.312469787066, 56396.85401473658, 43559.589515423504, 59160.085552420736, 77887.22807404787, 42324.19163427765, 1158.131218033731, 91764.4506534955, 58854.13013069969, 70070.66989389964, 50302.024828346606, 98276.34578596584, 63247.88934951219, 72510.08313279362, 71403.0807572054, 39158.19532870716, 77448.7421618806, 28714.71212193435, 80329.24208220496, 77192.8668700625, 673.8300984524659, 92877.43640882778, 23639.296178779445, 70177.47246279457, 99941.05705802907, 67805.76735142033, 53483.19000401043, 82785.35022834409, 85451.05988330887, 66566.71920286215, 56907.11888746049, 12857.101761763433, 91685.7405083294, 68758.50342489587, 7198.23034423992, 48100.275338026964, 42459.94117754007, 67811.32961953472, 68204.32596433446, 1877.3840381310358, 66090.38291962475, 48334.98025851633, 23665.120535407623, 39640.0276926267, 84946.57342661885, 12397.397160257306, 6574.967149324462, 36300.96570416069, 87780.34908859059, 6228.200303217357, 71224.27721177672, 9469.943046861095, 35861.758374953766, 15028.86853523383, 49574.664895954935, 34153.11349210623, 27278.938242263328, 51507.24628114067, 69245.1568725752, 83597.11483620423, 84826.41740456616, 72717.9769980636, 45923.88806034149, 48274.39073006015, 3766.0464490786303, 53234.76976306109, 94733.50234763091, 41711.726172155606, 63754.58357294934, 62434.375520359186, 58428.005611916444, 65442.21418653597, 64031.973637437775, 56401.01103634412, 4938.180851009166, 44364.36292255186, 72609.36905460144, 92468.06024850914, 56538.71562350674, 92867.91085527763, 30905.404151648185, 31.03165072917413, 53193.41405273389, 53817.5223718551, 55048.252775156456, 44834.15344939165, 83887.70703508325, 47784.8818351638, 81014.93700578572, 63670.86796877898, 38888.74702110026, 80302.58262816312, 88140.15923030808, 82554.60961692901, 13138.109075289294, 75563.13184546126, 58896.91988253496, 41300.01250088308, 9136.927964228193, 72586.60400442271, 26256.84126000135, 4439.368580827774, 4103.793023882085]} +{"id": 747, "vector": [3762.701097654486, 56046.996543297624, 84325.87146173543, 52697.132454011, 36051.68789944723, 71947.43765431964, 97403.71484902539, 74568.10844815304, 23013.23928084483, 87681.09766536037, 5818.30367480225, 41884.01283057727, 94191.92476475993, 4620.820157304462, 41380.3266226144, 36440.136354703856, 65798.3781750345, 26468.18820819541, 75954.27586294747, 56135.29667909849, 12911.348177984815, 48935.45391427319, 91205.64329040385, 51354.11009594954, 79977.91174161748, 13016.575497891048, 60190.147779229366, 62254.16948798187, 26870.621770950587, 59.01057828153755, 25148.07321650333, 77958.96055943528, 71070.94511156542, 88224.4697721596, 20296.772300489396, 60777.56469446334, 80796.11957689872, 55817.01294736853, 37710.89123976953, 86503.09235526859, 10509.308755375125, 538.1766091243855, 46364.01044794116, 39862.81828426708, 92098.94788972412, 79864.48931064837, 54346.78352167667, 60236.44036973237, 22661.2307866082, 3704.3673184750883, 60775.42788602474, 2212.632344751364, 59756.82543001079, 87819.35689260565, 23247.778023063558, 59736.50855067004, 1170.7376046350305, 2826.6816516052672, 98912.96432696153, 39560.083655984956, 31906.918944031848, 27423.64494082529, 19262.787042612916, 26518.71970014701, 64484.29961620174, 77807.51296197467, 36965.9582397147, 10602.64140751379, 31917.102899498994, 14055.684381753552, 34214.26852941145, 71582.0182747751, 12813.043244978116, 65204.34858494022, 50384.68685072295, 71010.82583328927, 48032.004899972766, 23359.575655616947, 90052.0049571143, 49727.127659119185, 63754.48683947411, 62155.53765724215, 76829.26735763228, 77016.84980970966, 78707.8233055367, 16361.113042998875, 88191.38759788492, 77613.14009400668, 70247.54608789226, 67154.82311628925, 33855.09612822782, 58612.40938390025, 89079.25461002381, 38641.71424084275, 34264.55622828145, 40460.21765357083, 71222.00583486637, 83551.4092822926, 10654.344014407125, 5772.489993563623, 70710.6497347869, 98727.84072962428, 88245.24834597463, 10162.534531566036, 57133.17562466526, 10446.998081019654, 71556.54411376802, 74637.05911035326, 66959.0316958538, 22465.34498675743, 48907.018583163794, 73515.89962624964, 57545.80808996502, 63740.323285384315, 75999.45785763365, 72530.7427598102, 57599.5757186175, 29967.12913221725, 82920.79976600334, 95996.49098995078, 77221.053444283, 44656.29808747933, 74427.03150979527, 24578.848060089855, 82686.17348844853, 83546.15365813991, 46260.39020786802, 65277.7864620828]} +{"id": 447, "vector": [39574.13499829076, 18587.160666015745, 75486.27519781352, 28251.89842709794, 52140.079116505556, 94948.37441472358, 45307.98735056813, 78573.44912999876, 90581.01916392538, 55093.67309360143, 10417.902892708818, 27233.461560730255, 33864.919144504755, 17690.559640158444, 64044.09191582208, 15491.206305615457, 56824.303701437, 61747.12626338597, 59931.926386441424, 34681.97802069441, 49652.47556879504, 53946.99372998792, 44029.229798885324, 56743.66813946349, 8110.184704850009, 37927.025445975734, 99610.08707499538, 15554.862248671241, 21729.27520931406, 31819.071284651745, 81560.11872125653, 89934.46824391697, 1023.4205104456385, 63508.34063566967, 84784.66501178335, 38984.224225078855, 92774.41041819422, 39607.26381248199, 58613.880302854406, 43381.867161339105, 3479.2410844116216, 18857.688032550814, 33058.18490151441, 82679.8036438543, 14598.339618205047, 76126.50809847134, 44789.5718017929, 26442.0860015419, 73135.48676395687, 19930.150420022906, 88424.02411760166, 48050.68426266842, 26028.22079845657, 14988.420417076464, 34605.08332452429, 22920.404369100965, 99739.08721481606, 40547.414503003965, 52138.97302388521, 84508.25050908676, 47627.73214160099, 41223.81187195918, 13806.181623169967, 98139.91776386647, 55529.8439570097, 31482.876798383495, 89849.7005431454, 54418.67377911519, 72735.52287495999, 98854.03733354244, 1966.626709254038, 50125.074289844764, 41457.18020660434, 97194.67477333856, 93758.77209371557, 1377.0583017017611, 4005.235844314292, 61494.20152979966, 83729.40819634231, 87753.22030117134, 54651.16057565286, 48859.48679486619, 68682.90945597696, 49953.19550234627, 21983.379101772138, 41121.76678397669, 63959.92199640046, 71047.03947712596, 74690.82079178678, 10664.011888928504, 46531.11767222573, 88355.24309669544, 50020.52585893195, 9832.403941072187, 20227.940858508777, 50009.90444846105, 81409.63482263051, 42561.16844006035, 60055.798972812794, 59924.00850686882, 52781.74471290094, 19777.15555265247, 36233.74533950014, 1373.7794915190893, 62889.86362648008, 24627.69858813686, 58628.02327625882, 7219.115064187198, 7663.863751343636, 15150.147894447096, 60854.50508944299, 46915.84726895686, 88796.72226985548, 7892.450428648212, 13578.449692315864, 1253.3367530377925, 92941.09985543297, 82819.13382763688, 79605.8573875982, 61453.90332214131, 43297.92052743944, 14327.092702141652, 30725.463442023847, 50281.91539338871, 77719.68951840134, 27513.465872429522, 26140.149580431138, 20724.624477155685]} +{"id": 160, "vector": [53134.40992967704, 5104.48283090289, 16494.136343816645, 10882.345975663033, 87296.4940877093, 39522.4700302763, 37381.57669061732, 2716.168383534212, 58944.40302454493, 44659.00919695672, 77317.78089073428, 2653.578830404546, 6705.394243129436, 7094.704229129212, 25642.578260038972, 85203.69336479928, 92594.32006543776, 74173.87121772332, 88434.76208958609, 88054.19100246596, 40625.06513983536, 91418.03599981578, 75715.76498328341, 34727.751428930584, 74399.6580781568, 83484.72298371428, 30085.55124634299, 29688.94134739606, 54603.53800848021, 98196.59603231918, 58934.10688061037, 85604.23366778447, 40173.7113695124, 76484.75101722093, 66066.07220099092, 53249.36480793911, 97725.39524317259, 85251.48696540587, 47469.11595439627, 12623.928067453782, 37291.08572497843, 75709.1790978483, 1102.4796607255305, 7761.197748239734, 13219.088911972454, 51758.33111669133, 81038.11136283462, 69244.23476392559, 66873.46601645157, 95697.97247890089, 91576.17887457274, 36295.82957781553, 78511.62039916009, 8749.569846242011, 79278.20194787956, 16218.746543007122, 94449.37187086594, 50426.303424884114, 61035.80251608884, 36603.825567462445, 79011.06706284946, 84971.79798586499, 68700.51313211533, 9767.467131597074, 6843.2624325515, 36383.35468112581, 60873.76904654971, 26476.490851280454, 1853.9305997675503, 75529.57370752133, 17062.18249154634, 22993.181005394214, 31693.74183550169, 80787.11633653325, 88625.18835273097, 91624.95959396497, 99289.68933064844, 58790.139020067254, 20449.1464436266, 51583.87314090226, 87382.95273053633, 52735.274590111105, 73072.39477986447, 1878.016516737624, 66623.32592829212, 2671.2057173819635, 88581.9464465877, 46593.13170722851, 55421.302682683774, 24735.055796349505, 62234.65185754336, 45660.0665459885, 91229.45988644547, 38304.48564301564, 26906.97301941418, 44426.160768053465, 13120.038205989193, 78846.22612809586, 80086.13114374658, 4976.823655980045, 34403.200993616556, 38848.23724504531, 93509.2807221128, 72749.17949846742, 77935.09252933564, 4490.287802009451, 59409.610592230274, 64993.38624623224, 20563.697724980502, 23301.537186762478, 12683.169222666935, 81241.48328699986, 12478.975669267078, 61473.52634094263, 15389.019377627832, 35508.30175562384, 98694.56321222837, 3294.780634437533, 99795.57674745294, 80510.17817383823, 45456.67963179017, 54405.00444378169, 66064.22169678361, 13679.417229969493, 56174.433457759784, 709.3396200819724, 61195.470310596, 95954.55623982828]} +{"id": 906, "vector": [88016.71976639904, 27436.788209511076, 58262.054484733395, 18007.2171138009, 89480.42216593922, 34568.99062789584, 82296.63038210938, 51845.939873858326, 12176.791823681866, 96986.05173423831, 11554.051268781996, 14162.962376568621, 80738.45297399996, 57756.77364175803, 84078.6029646576, 25637.344828856014, 47102.290415027404, 1865.0144414006475, 57688.68109838226, 31740.2295352752, 78040.84510049973, 49709.57653142528, 27412.06778672508, 42506.58772999164, 35160.077638062474, 25646.66062504314, 53887.419219430456, 60464.98244510881, 43558.643529278765, 48107.69435387988, 19039.07669901297, 61903.54651485769, 61093.85824324898, 65890.17646322622, 64065.73543481459, 19237.90314287117, 72579.21266584453, 93101.1856922398, 30422.303319182854, 19636.88704008406, 24024.404051012127, 60538.423742781924, 86498.59278136717, 27659.657476372733, 47270.68986789186, 63757.98045613885, 89029.3321117093, 24203.68365205592, 22876.306581296, 37322.552997990766, 12990.668267939542, 86714.34123692763, 78214.08130310308, 1167.7386966944202, 35308.16576222722, 45524.85975593834, 98963.54953708497, 89725.73826699691, 74789.96493807853, 54465.3314027964, 70475.02784725543, 44956.41185068053, 77771.55413038991, 3639.5879014409884, 38546.044008815814, 37834.44530212307, 73816.84505463716, 26785.771345443874, 66230.9286813626, 20187.422140224196, 11125.528553183673, 35650.34112962217, 76276.89553336934, 40373.970869579236, 13757.544588519111, 32139.29177650471, 50138.829154625586, 61791.43221977, 87584.51268499345, 76421.99371365047, 19706.498081837974, 89002.93439209815, 94836.261067778, 74589.83259979174, 13825.140214251363, 53153.03071536504, 91219.94335554274, 87803.51289789277, 48128.85257913605, 92570.11581650336, 15717.694874865962, 13166.446880293302, 96215.83757938076, 27809.914106294476, 30707.824627524562, 86703.42825047906, 97189.37042845902, 73537.99587992141, 91953.8829277647, 20206.743202202702, 34211.59569796438, 74962.90851786971, 6074.058121097281, 41442.11829396629, 1802.0200541030151, 85680.1703763383, 96126.80723581968, 70521.56166282795, 99855.2836914799, 25820.341501111176, 92442.86995964126, 54756.21707785023, 21995.496227117394, 85231.09667536472, 32573.31704361619, 83194.42926968184, 19172.691018805344, 76582.13124519776, 49595.237276284395, 33964.44345894376, 27530.971224819477, 78287.65903144176, 89838.26331435786, 32407.392000169788, 72698.78719883217, 6227.100874317714, 64906.27596707517, 94371.694120972]} +{"id": 374, "vector": [21342.352251083008, 3434.111323013156, 9440.403229025651, 47993.13385835934, 2412.858143656682, 2152.1578252565955, 90103.47905988521, 86544.10496236016, 29289.076506612855, 5626.355906486524, 44603.22423844231, 77657.15465415262, 731.5223941884885, 36311.95504222138, 83474.25396655567, 85669.10572778477, 4331.594529802274, 58381.71826756333, 83748.02199080569, 84865.85133492737, 79277.8408693184, 88001.2672188657, 33209.91374272825, 90021.15220055012, 20314.77835124098, 35147.47069144683, 79011.62813488286, 74154.66495538187, 63062.83454310192, 79341.49121827284, 41713.61425444762, 58223.11797734585, 43858.98291221479, 11331.496452150248, 69961.74622252981, 87603.3006778137, 8877.76773700577, 93331.44201651726, 97645.14134614877, 16208.209288654485, 82392.12526617397, 76777.82198161175, 16401.712245230203, 24134.020315891135, 13990.37596510463, 20642.44251625428, 24298.3840969918, 12986.430209335864, 42140.037163362045, 88243.16100438575, 71560.392314323, 42243.947953590636, 58111.84307139896, 48521.36107880114, 18693.02365506794, 72724.68268277094, 43582.09064036953, 60428.4292828313, 55537.59011682715, 13159.198152032559, 68804.07303068771, 98653.88185380846, 88671.39455873327, 35600.474729123365, 42855.628385076685, 50510.96447270305, 74156.52697369327, 85600.19499790133, 99798.61089308906, 5934.15394983946, 625.0823224115965, 44378.57949721562, 19661.763941177247, 68031.43852990134, 54367.99767386678, 70257.29044361717, 15682.302298143914, 42656.3108903811, 32587.34515663102, 28870.959638787175, 1821.184830137479, 71822.815483808, 86510.32440424565, 28576.82073660962, 98944.25901206635, 81474.73895477249, 55342.55030343313, 95177.23984125812, 24668.18897652845, 59698.5922127232, 50547.09009965889, 85018.53761269058, 11984.10512434992, 59521.78230063073, 12139.132989549094, 64476.30083834358, 2342.0365368268526, 59102.72063038062, 49474.69745637959, 91376.4085136766, 21791.95597178364, 34856.53075549384, 25282.987727323558, 65163.79735760261, 88684.43507271148, 12304.714338601741, 72909.01116604173, 60436.35439349265, 12405.460525419787, 15432.799509705808, 14463.215191680567, 53801.207425791974, 8139.774639557673, 84164.67619338619, 59141.53409629221, 98185.94655619356, 52245.30460706377, 37175.18550989236, 84827.21192851188, 39241.913029783806, 42522.91185713382, 41581.512608748795, 65678.73160709391, 50004.30125938399, 72343.82537079282, 88664.95134154621, 93178.55975240601, 25753.277305663723]} +{"id": 1389, "vector": [95699.15591180848, 94470.22212940533, 75551.58678693473, 96849.51213467661, 77943.8057569401, 46891.20803571188, 20534.68949641899, 78022.49562111431, 6681.575967735631, 43873.25457482249, 3497.4272373475833, 43214.973428682904, 79045.59232297876, 83980.16306292973, 44690.021500091156, 49063.20530922405, 17764.580082122415, 34824.15313525769, 54540.79436365455, 98824.30767446976, 16856.95743623348, 16607.372120820506, 71154.67428609238, 29179.774342412657, 56111.70762307255, 83946.43848946632, 60848.015785944284, 32794.71765408135, 60374.55138316928, 80641.83538697474, 4064.120322849929, 4220.430152094834, 46477.63420878301, 9933.264405064101, 40765.19629181132, 61083.58767139006, 3650.895258604647, 61377.65457756138, 98309.45695049381, 20641.138269796356, 66062.52312868892, 47481.203727621665, 28755.217203559445, 2111.162803050304, 79988.26536072057, 70822.53397149226, 73381.6242716318, 49303.5792208687, 33240.769110419955, 53341.2099306974, 11031.327589444718, 45.18272182405525, 49539.03368365158, 12996.139075059777, 10412.042327611782, 16468.643806212734, 39254.19962408222, 69180.93762540695, 84270.16602216239, 98496.01310816438, 15173.411187384978, 53539.69490884001, 33706.021114209005, 90952.48225104377, 92541.8853212381, 41486.386669214684, 72646.01725876705, 57759.45150256006, 49636.40424560582, 67724.4701821063, 14940.34908464864, 72940.89890424855, 17906.39470673778, 87831.98770716097, 87989.28289713392, 70579.37424205078, 93523.06395485027, 57984.658791295755, 27967.414934018896, 88776.25586718424, 39336.80138603045, 43776.57941590942, 74673.49798657035, 45670.07461911246, 7174.128951087544, 6375.45151438077, 64594.79781348212, 52145.07580307075, 98819.21463959308, 65884.9634067738, 19453.420639492568, 96903.2756218309, 69694.87885972786, 52657.08488428666, 85771.48815376798, 42132.73812416128, 65167.87825210235, 2261.024305978543, 54078.81337001882, 38147.68771571672, 37533.72403433567, 86056.54839408502, 86445.07417218098, 43742.94918559194, 96861.16991271594, 23686.68926097833, 59991.69648863374, 51920.87222640761, 54592.21948637525, 86680.64608939785, 47820.011303698484, 25315.742192818747, 48689.49168213235, 88502.9302052118, 51141.366838488546, 74012.56306786987, 2024.292327325461, 41654.80319141368, 16768.246057592096, 11545.151250257224, 1965.62390307784, 66682.49333582164, 3823.206787346434, 3314.677717801584, 45782.20584527372, 90202.90988641042, 67786.07792478887, 95878.28837722947]} +{"id": 824, "vector": [18195.25990166393, 61088.16132459695, 36860.111592498724, 12121.07392400591, 64047.87359760988, 48189.39143028651, 25593.480198583962, 70448.30790758657, 94892.18962016913, 31580.809806673482, 18317.745991844637, 86094.00069813321, 94197.31660124994, 21057.666227124682, 50802.01748193649, 13508.235233474508, 9547.0634180729, 42912.94488183771, 85495.10709157353, 68042.74775994723, 35206.15907705845, 25189.052769616505, 836.2738771703348, 91284.87616590124, 9528.080029335728, 94013.73400424325, 14929.447562164854, 43223.47821168198, 54919.32569964028, 39454.49087079288, 71319.03495883496, 48546.285074092906, 49209.15111038845, 5698.697162358979, 90283.36567197826, 34749.0412835495, 7605.853487692549, 18242.47840049076, 33071.21687369094, 69615.7069421537, 23486.47003908614, 18976.99966431977, 66906.40884835673, 97745.10077626845, 52754.676103017104, 7519.852699523877, 30596.9488879002, 29545.402294154766, 77102.99685975787, 39051.86961572811, 14190.56841941515, 92252.38128959364, 97840.06040202077, 59149.676600151324, 93703.88811489311, 21966.771765047353, 46641.54081304787, 39910.163601765635, 69101.87861873963, 61575.01623517301, 77593.05069990305, 41549.09104632063, 94353.52846203386, 38849.14681997641, 65300.839822694164, 5135.3598668806735, 72190.38274770795, 17127.854260922548, 57418.2842439082, 76329.65768372819, 88625.26688604934, 90082.86728241463, 31513.92891278624, 45039.883122081184, 42622.419650194875, 35787.96848818198, 16700.641965724317, 2833.4898513125318, 11306.254633515666, 32544.626850174962, 28233.929758934373, 27257.95861025193, 55151.5572535026, 61881.74216886455, 26374.604776656684, 69156.22843258733, 85779.14348455609, 61450.35710036482, 41014.80503399448, 15179.012453954876, 93025.82943871903, 44347.3669712756, 90524.81473906824, 69655.40898617206, 11136.77427667622, 45993.631695998316, 37307.81512672948, 52258.29471514106, 74399.27977633243, 1390.3886828637412, 71947.91527072941, 66543.52031114498, 91841.30259385551, 34362.166131293714, 24406.212229382218, 74756.17189538834, 90162.16447801246, 59593.89293611225, 41069.101707763286, 27734.555020059193, 77802.56657027412, 10385.432903463987, 77389.8299567893, 16412.34699337496, 56458.668294796335, 67570.42525714068, 70968.20881670175, 95611.69540355777, 43139.67736171391, 63908.98088337752, 99296.98986639091, 93062.31848746205, 2801.562468698715, 93030.69476975696, 82588.60361454694, 53519.495276176145, 82207.40995894131, 43850.71318499255]} +{"id": 232, "vector": [95597.91693309988, 29768.775591135778, 15352.133057276918, 64084.228315258675, 43096.893783320724, 19732.511631443238, 79676.65995382752, 20552.073491129842, 35846.23753695239, 93895.31901465383, 49771.61251898552, 40847.664192705415, 92856.12220119512, 58227.48292059233, 99406.48003079287, 46166.80500697664, 29913.58625605499, 96277.55422887053, 79375.89762257256, 68994.70772594732, 70933.39841732624, 20105.58245043782, 10278.827995096373, 51820.891252604495, 21147.326941950927, 8773.5462106997, 66504.38037161499, 40255.652427171895, 14496.279358005348, 39450.01480461393, 24979.521166072816, 39810.24342205835, 17507.618734468655, 96978.57273537517, 37514.89183211306, 72807.19065522196, 61906.65809409848, 9469.252535102212, 83805.42974782671, 5672.379747395195, 95129.28931877723, 36377.49786044824, 7376.923317282746, 62428.607045491124, 63621.955658486964, 9121.586635168433, 9610.82167750762, 60107.64482625807, 21384.670378843395, 18467.550809963894, 21834.80494262836, 95893.597468903, 91018.79951003173, 82520.65787130242, 12270.616500150532, 67973.15606601744, 89392.40047025094, 60047.91381240459, 40659.68953508432, 72286.06605221021, 34430.34219733367, 90524.92290629649, 30900.89733213727, 61304.32560837575, 53034.48141395337, 83920.38048192816, 39075.06002038465, 27066.106509551057, 8105.6963854478245, 50867.64309022045, 67640.40018946456, 71933.03479590235, 90345.825254323, 80478.63707718923, 45248.86202057225, 38050.9220197289, 65846.46443575727, 97257.26083264951, 2759.802033260017, 62094.9747352444, 72482.49786628083, 67585.97146722561, 6177.265094915274, 50817.2316752072, 98370.24251652203, 46174.540089710405, 99766.78166443818, 75600.04114610354, 8388.16634775199, 46337.605557851515, 57678.98825336172, 89497.70752271723, 90565.00650815878, 15498.74114251021, 96685.71389840955, 52816.721895216804, 24996.256829893617, 85474.47842931551, 47474.42872622374, 29054.954060561133, 16039.79162436815, 40066.37709269888, 28380.155173161125, 82977.57426081167, 36564.69534751547, 57550.694629175916, 56656.53433840783, 92024.74963587169, 41816.27381623356, 70163.78111827005, 24741.758830980933, 49028.34865879001, 77248.87496902636, 81658.81390504495, 33370.63665556061, 39288.69808211367, 58260.53283955126, 29822.618050979432, 82380.76356017773, 69951.72616797408, 3792.6161610307927, 15530.556512238447, 77339.99524844124, 1065.436998341207, 95857.76049131673, 80536.61414230897, 96129.07568761753, 60454.59959262829]} +{"id": 1990, "vector": [58357.68592846522, 8740.107817860555, 96269.85479490556, 20078.77642523882, 88222.86483255035, 97597.35099333599, 3378.194844110727, 89925.94134160539, 98514.4305573549, 95467.798668783, 3183.290305910058, 27353.915241052895, 44064.6183857366, 35776.75749250322, 32428.54438052124, 68286.30466653072, 90924.69866229799, 49631.669932891156, 61034.03553533102, 90977.6147005288, 20952.548938849526, 82062.85690610891, 58698.291580151395, 74869.6869098164, 24048.23211832152, 32066.52203960144, 21448.414575760955, 33541.509955124835, 39240.77959566048, 9065.97425679826, 3038.6904533846405, 52863.91017295799, 8204.522848010565, 98001.41929026168, 97914.65520576025, 50941.518772488045, 39579.42970190895, 80522.26486378671, 3467.141523960315, 76480.18642397315, 87275.80656705123, 10118.497989452257, 37030.21569970794, 3280.694999672351, 45687.698991550475, 39276.18534080266, 93746.16027717789, 93345.22836355021, 94577.89702334268, 68076.30093968743, 55239.956436991844, 43587.09168687989, 85357.9812851162, 59039.076876792715, 71022.31619725194, 64477.25680716666, 87833.9798068876, 66906.71092826403, 47541.12573482693, 81132.42815539223, 46413.16962281068, 59988.594276056465, 12105.741743850373, 22990.43387786951, 62481.9801204221, 50397.56169450881, 79827.23039906561, 6892.515219268258, 41284.19761078811, 32693.224142838495, 77612.55789765478, 62990.93000478757, 95359.4751864261, 19175.028741328017, 19547.198992510806, 76170.57286539306, 69120.46860214014, 79595.23339724082, 74583.06911670958, 76373.94820443646, 2282.628410501386, 69135.67401831699, 19276.502956206998, 36497.74848821455, 62286.714585231595, 9106.18311232222, 58535.422669190186, 45570.96850355995, 31415.933549647558, 876.1244182654449, 99109.89974329603, 62786.5783059963, 62427.49480202613, 49667.12236533405, 6772.163076171778, 97386.45943058371, 14191.340699171717, 12117.49283468242, 98044.58250614665, 75949.11062081525, 77074.83446263906, 83438.38775534547, 2397.4795931879457, 91535.17242765827, 74638.94588794488, 78376.64970065636, 88659.3875050988, 88350.9603532122, 47755.759291398914, 64276.150225128236, 93508.55000390514, 68774.45439180051, 24310.27864167582, 89472.46354778737, 21171.234742009594, 31739.804126712123, 67597.49754490657, 68874.75218143078, 90207.14490355014, 15228.267049218424, 2425.3353368482667, 63265.571197427074, 13449.702276657017, 22589.1472485906, 87180.29703586224, 44024.69013231962, 64262.17836939343, 80520.7059516127]} +{"id": 228, "vector": [69814.74423384662, 67506.74091635145, 685.4555163691822, 67169.62703056009, 41969.14058668315, 7668.412191035523, 74539.5395143709, 47045.652431215116, 77264.23209688977, 35003.41426972547, 66689.74427969338, 38758.30049555804, 30784.344285217736, 96770.67337702318, 81457.50932697431, 91290.76168809157, 84377.78424218806, 21027.058318314317, 5748.258046094401, 27727.76400071646, 22153.248422255743, 40511.3459471291, 92039.63197538449, 44183.77574903094, 7787.494399922279, 74066.62106324044, 89308.49219949808, 67664.74863881421, 33437.59089573105, 65272.9709646577, 40668.38605213541, 91107.1785707501, 23078.272252122566, 95442.57780909394, 15072.171620921792, 87443.36530172708, 57324.44622524023, 31585.86551574124, 65072.380700202535, 25715.3142318719, 49909.13113062843, 22088.90069613868, 20036.62258067429, 30128.848538543774, 83802.11677710361, 77367.09305137054, 42319.23186192857, 18573.23385412102, 61082.671839951865, 95418.34153236111, 53602.600012521485, 27639.560397938778, 71275.27855003846, 23623.199370359493, 49015.11081752871, 80376.96149738565, 8459.072149489333, 25094.860847242362, 35266.85430877331, 55416.58188603808, 80514.79658745496, 21701.723628256954, 46093.25122435336, 46419.81832272822, 18981.92101422278, 48925.51165025225, 26167.413969746845, 86301.26646566535, 68491.12958802219, 66770.7459755389, 50105.34811288489, 48045.31486552786, 1220.2330855680898, 55859.959542045544, 24256.07527100464, 79734.04987877744, 1951.6906193770667, 43582.374151037606, 45096.09687306678, 46938.06109075916, 47266.95243391863, 19217.551408564646, 33556.954231281896, 23412.34707679547, 97741.02933853671, 10004.573909033787, 44044.55631695744, 81121.09048055176, 83364.33768470968, 44361.6780771942, 63517.20302698214, 64991.66413362324, 22031.126451911976, 64564.82680942498, 62635.56563772523, 27637.326108946225, 45988.18044262214, 415.618519178973, 2593.840852392482, 8257.937351532373, 70096.75119544017, 41123.29339610686, 21573.875633468255, 7667.378007911297, 81103.65187138741, 43638.09902880804, 94233.96215527112, 91374.83184201019, 24478.769291433055, 41885.93398282963, 59089.19377315329, 36105.314354556525, 34212.87516027165, 15388.589387158547, 16512.502903994642, 13306.477207960166, 85235.61485696524, 8915.135779818938, 94324.46986715452, 34811.528661117954, 9662.879398185243, 65013.95124299454, 98759.46407107759, 69192.68284552387, 20699.869131069005, 23348.110289780154, 54665.602898757315, 8617.090241215898]} +{"id": 318, "vector": [29347.91970463725, 35095.71129114549, 57830.26686694096, 46812.35155090814, 66449.53131983508, 31399.60817075025, 28219.80956247575, 20747.39664096451, 6432.289643336231, 80923.53757941369, 42412.44827053009, 42502.070238127235, 55530.00379115878, 97374.63187068995, 73993.74502798793, 80551.41766393984, 29663.73410295683, 14642.161066262282, 71263.9861613949, 61532.16071983743, 64382.798815989074, 70276.71106555882, 40794.224522629665, 65208.95802746669, 38915.61839799257, 77879.54281415444, 53359.21145402524, 39386.402291349, 25880.969508030437, 84591.99575444583, 88564.3186108406, 9638.180530398387, 35719.1830410578, 5261.2823131715895, 53708.32729001362, 668.571137799745, 12976.69628851249, 43142.38754168309, 73391.1282268108, 48636.98494684844, 20074.950011670324, 60659.68989817796, 73508.7795870518, 86212.67053944485, 61685.63720234943, 62344.555931910276, 9248.174169107371, 24690.916270003625, 61319.66623151297, 76726.97183376827, 55300.729497558365, 77503.81497820019, 13675.080307264887, 46052.19349338463, 51083.20243438176, 94860.70030009538, 583.8559443046232, 94151.27197807084, 17454.941034561554, 62970.407864194836, 2672.0006285676945, 18093.438067473246, 47357.4613253622, 69417.60379952924, 54323.51377981883, 15581.80872631696, 97167.79794732945, 3053.7424209319374, 22958.534387869535, 4310.09662564219, 59300.42826629658, 52098.924034900774, 75835.35558053048, 36195.47401842786, 38036.71970964918, 55440.508746092535, 26636.96689500802, 28622.323477870526, 92158.0756683253, 69545.1424543752, 54149.167466920975, 37358.678470643834, 12292.441684878164, 21979.487988258305, 62953.55743094698, 72964.99830852277, 84918.3631171317, 87961.59523542931, 93449.97343315843, 62802.7074753597, 16182.46245431242, 80390.10495166574, 91576.23202647059, 54161.1739586962, 53361.216888055766, 68309.2413874676, 45336.62678400734, 80299.07546253527, 95128.86602117952, 68653.46690954831, 27334.43397840827, 46137.869665956874, 8.322177297104272, 5216.83016899005, 39431.43801466568, 92951.2571809046, 23752.667538533966, 51301.7405575742, 18872.22599308249, 7459.928817937555, 17768.19189329952, 94103.12487174149, 16909.255795030163, 41241.28925133781, 6199.791650528863, 11128.472783454601, 90713.70105766764, 94351.80223787246, 61033.75075192219, 40322.81689753154, 56383.20982779732, 58730.185825832436, 17145.891326624507, 62165.403526490096, 86498.0454924217, 65246.00492717335, 78390.83878525805, 26257.761464738414]} +{"id": 2007, "vector": [77692.16494266147, 77306.835059311, 5913.748919750372, 89117.14261990947, 77819.18699527581, 87780.89278747767, 60221.31196153261, 79287.19827517447, 15187.01379143732, 79917.5516142202, 1080.149336122327, 46460.50108537102, 30893.093786140467, 65676.82502512968, 10921.714742978495, 91476.73421069022, 41245.84017413105, 72230.06837499056, 99716.9883920113, 19405.952812787997, 54492.6929207191, 23864.645296447106, 23895.673912027483, 95858.58302288786, 66170.00002680876, 33538.71016725868, 95762.72089994978, 41494.3074223469, 52529.30171499209, 50467.07535023645, 6600.696557081087, 9938.899077034746, 8140.938946320786, 71392.42079923334, 76474.61645237534, 15950.255950875237, 88333.10869703705, 20417.66012745003, 15499.924291713785, 3055.904403013099, 77252.29829654862, 68036.47001663517, 76048.61691177377, 26549.045859718524, 28949.21410993606, 17487.436085264162, 59636.77087757165, 19749.293196825078, 66386.6944372991, 81049.5889053514, 85250.99035993291, 52727.14602363742, 34975.791585339175, 23505.269062243715, 78605.65349474459, 78207.6806172754, 50083.053238948436, 53420.881651425734, 43175.37766241253, 57462.01711652207, 98935.86049714076, 53776.691069051565, 4470.12455886977, 55248.587042766565, 37090.171167674634, 66376.19517446142, 34993.92586171228, 96612.68893244553, 16479.83183767242, 19496.385254897774, 63331.39775553741, 79761.56338298073, 54683.70462889639, 5508.719271723295, 24081.008663255023, 98370.11783359208, 35307.24273893997, 91556.99223626236, 12609.922535004036, 20049.95277245801, 12480.8170604921, 66274.50973437862, 41224.105325586854, 47438.55811052603, 99414.94188668419, 77643.30323917652, 58922.25635718527, 52271.211106386196, 32721.737456990384, 4582.192430113075, 29673.562357831117, 85157.81216736467, 29372.665015728384, 46737.42815281052, 13139.612968133397, 7495.5774800301915, 35287.094329426836, 24056.473478430307, 10060.755300204915, 18919.160489169684, 49872.01102369352, 30980.832975619844, 14959.671236564509, 79063.04497198974, 91817.02259694155, 8566.943594810238, 79420.17359905317, 37831.063064494876, 67293.14673786539, 5317.183967153771, 83065.562933449, 98072.38821928613, 63128.14154357752, 41401.98342849515, 86146.36907103703, 85805.8199639052, 12510.704039776465, 84134.28272460584, 20414.421209237855, 80769.55574429124, 26854.214937019227, 81139.78092372982, 54952.174063626546, 98225.04094093294, 68026.85430411492, 48508.16178656455, 84600.2023684345, 74645.48994897815]} +{"id": 1394, "vector": [23970.097128259393, 15777.749496082404, 80744.5965376259, 33233.052196623634, 54020.17657270439, 38397.14210234626, 21372.569283005505, 78094.88731794384, 36761.45953629936, 7676.52984389654, 55991.31194952709, 45261.04043131493, 86160.83713752935, 74441.33357724774, 76941.42515465936, 72151.12419693233, 69232.73797047834, 49384.04664734058, 40320.75249961039, 79530.65031623557, 86561.94300488221, 3033.3392494684585, 38578.967811701084, 17738.165200351865, 86072.86361020555, 63917.61274297051, 69622.41424113537, 9347.148567191954, 78901.68230442236, 5427.479983237371, 55582.19977652778, 41544.696624546654, 67543.68725471494, 14842.2839379592, 89421.46165756069, 40477.47185659468, 57629.012714360186, 38635.18290336313, 93202.31576349417, 94792.77252185144, 2224.6154674148142, 25829.929432306697, 64426.95298320248, 63934.2235392766, 32296.548004841086, 55191.51672777425, 80813.50663161758, 15786.264115342774, 1396.8167311615098, 91110.08659154235, 29583.32861835622, 22106.17615616801, 24622.489228163813, 56290.65192513464, 62990.592314264395, 69765.44726535423, 51300.85257944492, 61126.77322594473, 12534.214117276033, 20834.72818438711, 90624.29069494066, 51638.046084518974, 48785.66800205475, 62501.805277711566, 94196.77979413648, 11017.585312734802, 11021.264369262828, 45244.48425097215, 74384.17857781614, 2566.2108513796798, 91862.42965634387, 7353.553024535431, 7864.015860740792, 94783.72207253145, 85406.2174394704, 77940.1565900356, 35112.31594540272, 19932.79631120236, 95784.28727770048, 19844.75714827667, 76993.55054907144, 26560.0635010677, 63580.89825881306, 41540.23365791796, 78015.06137704669, 81501.60874397882, 10979.67114929438, 36846.567386710725, 7129.367355771288, 16008.303804494573, 41341.36800156403, 75594.55560578709, 49933.143461509935, 27447.623360534824, 1223.0126899968275, 84395.42251078025, 64454.62737866395, 45001.87984837649, 22753.64157269708, 5259.74341809723, 17804.008006831064, 59879.72018500471, 70301.33355832493, 83932.4189293768, 26322.48514543287, 91749.76323014016, 35008.62673404349, 37076.43357741806, 89502.05496029506, 70197.70641217519, 76254.71091426419, 2747.4121604219026, 1397.6941507503948, 85277.31149094646, 81306.86655387843, 78675.99873482798, 34400.203105876135, 15892.095096137782, 20307.93304415125, 5565.082351702966, 39791.89972589436, 20679.119120669708, 74212.07201643054, 42917.352871684656, 58534.911740617914, 89330.44884800317, 31785.167147122982, 18536.678352719748]} +{"id": 1424, "vector": [41142.03254292625, 38483.85083197605, 20190.896046970673, 1480.3427345807972, 7834.175950283473, 8754.554798737956, 92106.13603660956, 7453.991423263528, 5308.441147041732, 14010.268048055219, 93169.28439591378, 28529.445011078347, 44128.869479575114, 76714.63961902885, 72818.63847281864, 88395.89366040418, 37745.36679048928, 81559.1058603126, 60931.818278096674, 239.41308649634286, 74351.78764390651, 22497.905692682118, 61302.22513822941, 5382.103759714585, 65079.91865741983, 66630.71292169082, 96834.90026342476, 25497.7010006995, 30592.79028163997, 93699.80827895326, 33802.984242838706, 73045.31676077022, 69746.4801622551, 97155.21942431855, 4523.140054791875, 22100.74774634918, 54123.96844922352, 92015.74759609048, 71355.25530835713, 29611.598696456676, 50474.42105271087, 20445.35715135315, 78542.13783489614, 9107.17014703315, 49932.042395672135, 19947.870994413053, 55468.81774550836, 54837.91429503201, 98701.72503965316, 92129.17180639283, 84978.12797347065, 84447.55959122414, 62729.77270296686, 28587.916435648287, 80045.36534610181, 68501.1844892598, 23159.793759184166, 45521.74688492706, 35498.4540788401, 18477.14534884569, 1432.070333756008, 65174.96795007856, 54795.5079158989, 49949.771062017986, 11866.576297196307, 79658.59380951484, 14883.438589529685, 27611.55933173265, 51091.62624009075, 90471.20828685735, 43122.87614454275, 34726.25929796808, 83240.39245310008, 46775.106800582624, 73871.46805313023, 84320.78227353208, 77437.46958715728, 99537.03897097694, 63147.46577525314, 48563.42900058699, 31310.640780379108, 4888.762583071948, 80946.16537146286, 67699.51515307608, 9345.908607791265, 87925.15287290889, 14523.792917907696, 73100.87653953116, 22397.484836341795, 48549.215714693804, 37108.58689949583, 42800.48992036005, 30582.37753657038, 78047.33045677705, 47608.42535518182, 7183.245343581079, 37748.66872223227, 46994.938996411576, 2839.1243176769553, 21329.44253773201, 77458.20344362014, 6677.457614103178, 39767.4275177197, 16030.909037672569, 38908.541147633914, 90096.36346387543, 21174.85615501773, 86417.10182350283, 69914.0282026905, 82450.67057740751, 95630.17155319496, 85851.51750013801, 11009.588114440905, 47281.54389858981, 91770.05755059174, 42852.096729359044, 30371.396466091905, 69885.93420448968, 12026.191531234053, 64281.59979053198, 67901.43973930056, 40180.04425317773, 32765.26255061757, 47500.055677216165, 46170.54717402268, 40622.2943237632, 88080.72100413515, 25498.714204261585]} +{"id": 775, "vector": [47598.21542540879, 39626.52869151566, 52008.33965119738, 70727.52799226838, 13396.719370704246, 15325.235407491833, 78317.54029634013, 74752.1570528612, 40038.19755923067, 35514.81456208638, 20399.81712207637, 43561.33601209058, 49932.08278362352, 16895.15935521916, 27042.463585839017, 64677.1459711287, 88086.09790989269, 8515.797006190973, 30043.04110393129, 5148.3013232882295, 6466.694169762288, 99863.04995579257, 1618.8925552356404, 42468.34699095453, 88595.82047607678, 98500.83560647011, 96335.11892316682, 57847.551880383406, 81242.11471861637, 53207.921890829726, 17017.22185332365, 86397.99348803384, 43749.50449676959, 45795.07834516363, 42859.54736632112, 81971.33609670174, 7607.062854606305, 76197.78041493054, 2486.0858062993652, 10112.197714299542, 72905.34392280669, 10072.385027727149, 8621.599158486815, 78559.36750803115, 88998.69722614005, 88430.00713901779, 52878.862784970996, 68107.93669394968, 11676.967371242508, 79009.99173226474, 18699.524948227976, 33016.77181213661, 92674.11113662123, 18751.53812659961, 17354.50822336543, 32805.15237261231, 43008.57242130165, 54711.5541560517, 55058.78044027893, 28220.388820973276, 79847.13881209918, 33145.8521058449, 63899.12373609543, 11000.617738685669, 58971.32265593287, 47491.40422832825, 7731.915802969335, 76411.61523819056, 35418.090928834325, 142.79561327175117, 95771.57416142989, 11760.593333915936, 16968.309358046117, 5757.033432540071, 77640.56686870178, 97275.24209416361, 12620.762262005303, 79647.45490679394, 32562.96533593369, 19166.713303918335, 62544.41427660747, 72907.20918119993, 22704.16283932398, 41296.435906568775, 756.4912856604766, 56325.3961823309, 45156.62678359337, 66979.84439301952, 19783.726717080765, 96919.89522060195, 22713.351521537927, 151.94406516582103, 47105.8352328358, 61854.55212727782, 22792.43084894499, 13648.886930665205, 4590.483432059944, 56946.454024641134, 86576.73725728849, 74989.67086388325, 67702.16502379204, 37654.55470371104, 96961.6006690334, 30661.1861831245, 31346.63827344868, 3740.8217597995176, 63442.12414340245, 27817.99419584321, 43650.831448598015, 67142.14460112943, 55392.407047204186, 70013.62701612095, 28086.197495890065, 96183.18679436181, 75855.29267680152, 8333.295734704427, 33959.15818145317, 29489.517197463843, 47450.53214740067, 87804.37914992984, 12950.358761747382, 50164.83421373124, 10483.158060477337, 19327.37020435519, 36526.389133892786, 18514.881491923596, 88972.217019913, 52936.534125174505]} +{"id": 239, "vector": [12821.306000909848, 30335.58573270785, 56599.93895243325, 18993.181677958604, 9366.037725833608, 10111.92321937845, 4702.201847522314, 59329.73778448869, 27577.708830066505, 11413.263130704787, 60631.50003869061, 6911.503474375325, 18182.12720340082, 30676.58435933377, 26994.204202853944, 39454.430044204455, 72419.35422856736, 71973.4702534437, 63320.59399996992, 30357.429464894016, 55584.63728889717, 8899.453193975081, 92773.03606507389, 31400.669307609784, 80580.76577925675, 42163.95436705776, 86422.33538557844, 47614.38465560446, 80556.11363221948, 63424.06543361977, 24246.52662535317, 36658.91578403056, 89989.31520832371, 6639.6521822506775, 92039.29228176357, 40388.691329485046, 21447.559141935257, 96434.97301700922, 94178.40196568322, 43162.867372058514, 20628.196035594516, 43338.816575026736, 74199.46826981303, 72523.11309156376, 55892.72698688025, 34495.59853496229, 3605.9086488672397, 77327.97206209075, 10510.384162636432, 44561.04335001212, 44690.06491131382, 22128.95944759573, 70242.12324069405, 39028.26634403186, 57319.11725795129, 87655.76911342912, 50633.571883974815, 33201.09451770235, 96673.114596163, 64525.16183542092, 64051.271892690485, 54548.231168073835, 99820.13189902884, 27273.030432861022, 71922.69775844344, 20099.70504123867, 79361.92527521244, 46327.04222033131, 46149.25020090752, 59236.31637330249, 96838.68597116775, 60648.46878101847, 74006.29448290057, 31573.38441263199, 91672.73307617653, 34649.6773778479, 57529.89325893925, 35260.81695621607, 52476.594449936696, 83851.89673627957, 27477.81407950616, 66789.74802247089, 1825.979046810855, 26205.14232876201, 56354.36601042356, 56385.768945711534, 32734.945053206986, 96454.15503760845, 52768.05688802897, 53652.144593714635, 2161.6399621995465, 68039.30203296289, 97219.69689332652, 81673.46604680395, 18087.318217476124, 71392.60212386103, 1969.632637207086, 89429.15646256346, 62442.251099271794, 1014.1422114601118, 43491.728402985755, 93292.55742595166, 90222.98390464947, 1613.6110332662822, 4350.687929334729, 19962.159453874083, 20677.327217853293, 45387.52244969888, 70609.64364202633, 67647.65997048105, 36819.16553659478, 33412.512107090144, 29112.75637451306, 66856.93462214108, 11907.32916619912, 43747.06446198315, 27479.488827738187, 6165.238905098114, 18428.081021703725, 40833.10338817401, 87035.48817338885, 40991.72654906454, 2033.338075883473, 39760.00549716774, 66650.2145084263, 67102.11691117613, 92945.78094788552, 44085.33582346468]} +{"id": 440, "vector": [38701.29683041607, 90015.99188056382, 88483.73412310531, 2360.9009288554807, 59180.1931208578, 75773.00508767091, 85967.35376200962, 80115.43429964897, 41711.77792761862, 40581.58148482175, 75488.88477037406, 77914.58894164354, 12354.083058673405, 68508.67505163631, 50935.437388435326, 87623.93811612249, 16163.488860402696, 87927.86677075762, 40625.15969121463, 96998.3368899675, 72115.84595842694, 67839.24014420752, 24355.625297906892, 88774.70320307743, 12921.915542105444, 19808.021613402303, 61486.16946509395, 54729.80994131343, 4768.436139136989, 3163.219050828403, 70929.89275139435, 46548.836049488076, 46521.88176568637, 82167.7707653373, 43184.29634631034, 64679.0474710099, 87012.143219639, 11569.134861777553, 83522.14569957154, 86641.21720786855, 40022.134685776015, 23452.952509693892, 63301.544008956655, 29708.00865846365, 64328.502933207834, 55451.78721415309, 76655.83385887579, 93486.8128084629, 10247.011609080591, 68331.43532463642, 59956.30578888253, 29865.957715435456, 22594.941502389032, 99369.3063322215, 10378.978433863473, 43104.004577530904, 58034.12807034443, 195.60178682412888, 15373.524310822795, 4181.7989948951135, 59162.24155234018, 27364.274504693385, 61241.92325748033, 13110.032366032598, 82646.57798506308, 81160.70954652783, 40151.15006931973, 93949.73747593882, 29920.21876015376, 49862.33527323919, 95141.47936851, 54345.5800242431, 58941.6886973645, 4142.02070625711, 45866.42624057494, 59080.68283188108, 65025.3378866064, 80618.129041337, 77578.59624414275, 69093.79204043544, 23318.976461541406, 99915.2200900607, 72444.55587569675, 87080.34033522474, 54073.22710079846, 66842.60761587005, 30762.09317371169, 23264.64044013824, 8450.161391343203, 63492.82364478912, 96885.79423406081, 48394.71557037505, 75246.27660449538, 44788.535703082496, 31041.01371151653, 24249.391603189186, 4345.243809861987, 85628.70591041364, 2150.951245742372, 93292.56459101081, 3375.5420486283883, 53141.756574536215, 56912.07904569758, 27717.490574845084, 68612.9448873245, 46875.225522742716, 68378.30090179645, 43289.3315793449, 13376.71335749685, 65004.8311841795, 75307.56951993231, 54778.96634066396, 11309.64848854471, 11587.160002094244, 25117.71622931921, 32501.601870732433, 86956.78513738346, 16493.663715173436, 75698.03713095227, 23739.88133916265, 96954.10464402384, 82195.96787852596, 84014.19118436768, 22622.079800644868, 6542.81474974614, 3707.6846250085337, 30018.1408944009, 10127.558159245353]} +{"id": 288, "vector": [17701.25473901176, 16478.15188535564, 15530.03312588942, 79853.0338092488, 47078.17437316201, 79119.6146044207, 60451.039957930174, 30610.478512543203, 75413.6387218514, 95575.63215926118, 93193.63853180081, 50637.47572416505, 6557.461410607423, 83759.0779157722, 51585.16362982086, 7613.728420423893, 27106.254627057104, 14444.9281439603, 444.05536957637935, 58434.341813479674, 53980.80136081197, 24284.065237790266, 892.9865730692654, 3621.1946398831074, 13643.135451529786, 96585.15673740859, 57398.10938335349, 25481.355096116476, 67842.25867838021, 9885.747809614808, 45930.31435035501, 68877.23011308131, 36711.44289871808, 45387.79512191835, 62480.92370679606, 22666.34583528283, 64234.621340015365, 933.4278051916444, 25978.8015334948, 83842.53208697689, 27654.86058061477, 90080.45989764338, 9795.729809403741, 36774.537423101414, 27081.43019268442, 93534.83165855556, 90161.47514903302, 3976.169023853626, 16664.16363526887, 66034.17706966863, 43265.64895103963, 84708.69763020004, 51467.476506800595, 84726.31577870675, 35561.040226269535, 89853.47527425826, 92804.22265603737, 81798.48833131685, 61795.536687060696, 27374.49300518969, 65691.3424103759, 98539.82526689599, 58061.03832862408, 39577.909369203226, 36630.912852577, 24402.82929448676, 40777.66985967377, 99488.8677018002, 40329.07944110172, 66878.89352444927, 80186.38847578617, 48031.092142649126, 76167.5868661781, 56600.720312612706, 17367.99247561259, 73470.21670664797, 49090.15916038601, 75835.98446949506, 24032.1788819555, 11385.55017500379, 3161.1046143188883, 44648.66839723219, 47928.610250902704, 56053.92145571175, 21317.032358424804, 3057.7110646123783, 13486.463112629588, 76441.8046405399, 65047.25707119896, 38519.06543449774, 68625.34682652188, 22214.83079691313, 25821.5311581251, 3006.4970609997245, 11463.35961149162, 63097.167638874365, 58429.74409730175, 41495.90152967007, 42151.32492341722, 98649.3826579246, 80466.75375979068, 50326.062531132884, 85967.71943055117, 32165.47746687606, 47097.64242967328, 83001.55015036598, 43827.576377706355, 65626.55029280773, 58003.73044908948, 83161.95740474934, 4923.928207900163, 77983.87802378804, 15902.338841912011, 79125.3724358809, 3918.1071662670374, 17974.464200864404, 85009.98134196717, 77407.1479736469, 34035.86447986969, 99303.75118717956, 6463.700912942716, 15683.177336076526, 47151.19238323933, 35820.87892778334, 97569.63676232022, 96195.62546852611, 18418.904426471396, 4904.791696495814]} +{"id": 1340, "vector": [456.97956530277304, 96756.7482400255, 90013.38829658364, 54676.00624064556, 20911.801480231683, 80282.94897405225, 64480.72074826553, 12554.697007608196, 76644.24780163629, 96967.27262938773, 99971.3549631606, 31481.859220946717, 70568.18823516314, 51124.367641460325, 42521.02410312904, 32316.080742245125, 39384.05888091021, 18605.119271802894, 65462.35402901273, 64797.91287520844, 99737.13073127093, 7256.3809803197655, 94650.79011219366, 59799.37299726047, 68488.19357103868, 79185.6240739432, 197.82397485613723, 60578.031517548116, 69571.86141023238, 83979.09811050966, 57827.02102423652, 94009.85528616709, 50084.59912904545, 37456.45396263092, 57035.13261524761, 17707.55727505614, 2391.9817758382146, 69144.75655024985, 51705.6012481672, 68514.84214707761, 27213.00274323739, 33408.82364560301, 13517.16529237934, 2602.018130221162, 8493.916956982128, 34238.70719681702, 27687.374633802254, 45359.555497940804, 67782.45882774222, 61278.37915646756, 91879.94632099263, 19007.232821322796, 44700.649709326666, 65729.60052854124, 4690.7295776442925, 58628.3277973693, 1742.4517662700368, 6370.8072475376575, 80714.12032541851, 51468.02597922929, 70496.4392685055, 66187.06057614647, 60877.89669833252, 34657.92063686735, 87135.851716133, 34952.448265461986, 35570.0117150082, 30853.03919880801, 48694.11130866935, 72323.79879288885, 70929.2510980392, 53299.39115413811, 26095.966937792426, 52920.42852358154, 61157.84560316343, 66816.67823069208, 72235.13242539624, 97061.33198371409, 30359.084829278992, 9548.480220252843, 14481.036825932691, 75537.4437090897, 88701.89487360707, 77501.34592069438, 45983.67531246626, 80479.29395212227, 73825.10130489801, 97867.93287833041, 41270.257249590766, 50857.05163819731, 75669.97776482462, 93832.76253852379, 44926.779895757805, 49348.014656851905, 9737.587716017926, 2917.746659209852, 23307.13245338425, 68554.73493249378, 34946.99861827748, 14610.221128520729, 30465.881720939924, 603.1909842372895, 72158.16938018039, 9792.783120401271, 21953.84847478663, 57154.77574190833, 38082.5608035928, 47371.1484399352, 93423.05818595305, 72856.57081228828, 96593.32155503784, 19179.29361363383, 75026.56250052626, 74604.9509272431, 51962.88522100111, 92152.15430972133, 89425.27087725943, 7916.054908785164, 45898.556176666694, 71176.70836745106, 14884.175761327557, 58822.804073612555, 43601.213668660355, 21042.22622332943, 94970.47175866713, 44850.38487541445, 87498.75527617657, 9054.857478828659]} +{"id": 456, "vector": [40129.524278365025, 78482.83298640356, 55679.38932911549, 28000.09917710843, 12291.494004601078, 83370.25338910495, 86171.66548358717, 67704.05991113604, 73393.5743054033, 38403.970539678514, 63646.21727829168, 16352.16096782174, 75039.63255773498, 46137.568074324874, 79315.955934033, 75212.11667415917, 81533.27663719862, 46587.76118021911, 66619.56922082667, 52685.4955941198, 1113.0498090297424, 64676.14625481597, 94589.15888627831, 665.8338256698148, 62771.113233180346, 64794.5269437546, 69488.35226798986, 61721.75381838116, 21894.103106758455, 29761.89258150106, 22487.930857472682, 28477.021257878467, 78517.5265544316, 43834.27514818359, 45990.62810277545, 26141.84698087253, 96805.67779868213, 57006.65264474314, 17671.059497054164, 84050.36484287499, 56102.21625639542, 53337.70665790861, 48921.98561150797, 91856.93975239154, 49802.604402840865, 20992.57706092077, 49489.34285573133, 12890.172962299073, 4155.436145153902, 96375.7274964778, 78961.60000009436, 91485.51948353641, 47070.01284549609, 50221.96112836431, 53563.562165851974, 65208.658092725804, 86101.53093257497, 64521.631752715504, 67374.77371051606, 56533.12228907509, 7750.635850120447, 21854.76346078318, 38174.74350321542, 74167.76497766419, 79918.1514257377, 40270.85709955851, 92337.08957148343, 76558.57656832815, 65401.789544278276, 16449.953432572438, 68917.69192404505, 6785.209525579617, 34951.12568844823, 14774.540326305985, 65870.48463026709, 40983.93471799409, 44906.88261565795, 66188.70512248448, 44973.86209571537, 30338.533750657716, 96067.60262723178, 29137.181676261636, 74324.8059215498, 84934.45358946177, 61012.79246621312, 46292.0984095699, 50525.68694107222, 64627.44523184357, 88102.66899708861, 89535.61281558621, 67427.18881209775, 99320.59796886388, 64560.79254332261, 74548.51918215997, 78569.29450360485, 25864.787313324843, 30574.527474924762, 49280.822170186955, 84385.76677902552, 1928.1751439507943, 97970.26890877332, 66021.1869419266, 54661.79314836498, 35374.09855788031, 28582.832947226343, 6074.505563415122, 95698.7576320544, 42497.691748098245, 6066.498856480984, 67672.35712721349, 56017.10442067756, 30433.309349896852, 92919.09901100729, 43109.68735586501, 50851.862250893806, 2875.0254046221003, 5570.927500508438, 92836.43691593182, 85827.83910886131, 84808.94433355954, 66455.58312786951, 2648.9165753479706, 77993.26825323355, 71077.53750658905, 23562.577897297022, 54344.89285747225, 95435.41501245172, 30508.110469727435]} +{"id": 1496, "vector": [92911.85151236673, 33494.72333472146, 46998.89831357687, 18137.63122832761, 62268.56284669106, 4218.767466468232, 88239.39728284162, 6063.857729491584, 96074.8619644578, 53136.74276427021, 34520.71718444163, 88549.6710227546, 54208.467743041525, 28452.724201465528, 89375.08745693127, 39963.30025916569, 37491.879320766006, 23400.58167624882, 27219.133971805666, 12716.289910970669, 66655.88598085742, 36398.28067796996, 67388.50804176253, 91309.43552263269, 17411.860624256948, 84939.8488234544, 90865.0627030698, 68267.32288964454, 52105.33738416523, 37874.19765064176, 25706.248773313277, 72765.47158769354, 92109.94301659336, 83251.89256750743, 65736.78751851064, 11058.204597352073, 66123.44637572506, 82025.00475020793, 61426.4604553032, 82702.40983486973, 51015.08855690091, 12603.344437947939, 17482.977826176837, 17450.67364868551, 340.52936233872623, 13166.938820974949, 44121.896750924694, 94016.38996113656, 79656.51765175058, 86841.02939394917, 84632.40384378452, 29419.080063879675, 39747.031397316314, 73913.47411403472, 22551.497440185543, 59188.178248514974, 53114.00030025208, 5569.5514262597335, 16425.383502517376, 11494.103413227074, 16703.364163012913, 68071.76994125255, 44269.96447863808, 74838.07082845477, 14970.39515901889, 57644.487652944845, 28051.79632713517, 95829.35227350978, 16674.606287785442, 27453.3881622381, 72952.29969283118, 23183.90412147585, 78749.0128249863, 22154.627580596476, 878.6109329092628, 63652.4032940767, 95116.89986307338, 30994.748909837243, 94051.31436616713, 28585.765050688227, 60881.09453122919, 34664.68876906551, 33929.42967774607, 9247.170054879161, 60752.213685833, 65026.62347599318, 68611.52985811178, 6885.317377363432, 77284.08073843428, 71326.74440647342, 81502.20798299562, 3550.372636640986, 1321.1933094993024, 41627.30877903736, 91616.17223975169, 34780.71493202617, 70623.50004096313, 88248.9414247405, 87827.46416316547, 62124.26516704476, 51885.37865539339, 20662.286073420422, 54443.916889870045, 74749.50440858725, 38359.01137631814, 72008.6814408675, 90224.54038862861, 37714.36256896292, 16299.702698968422, 89930.81589693521, 35310.34824924072, 46100.38027881396, 47448.88778222622, 83367.40019009977, 32738.32884896699, 18846.533373169827, 37493.4446061469, 29700.149177503754, 91077.32846116989, 11467.73889064121, 54140.204061847406, 64804.82500971355, 68130.7111151304, 23253.420775372637, 79663.08647332145, 43435.40521870276, 96920.28834644484, 64223.1760616581]} +{"id": 673, "vector": [65777.92331520464, 19377.749860173353, 29779.111652150237, 6747.333324400773, 27464.868186270218, 35876.61308621874, 61155.31870726422, 95566.8509754895, 83330.38236001384, 96732.17848568625, 58704.47603589521, 16195.698815453352, 92582.93113792596, 19654.44612788203, 77990.3717221117, 27324.042371734104, 89052.29346719352, 87834.01866449177, 53510.428434681446, 79947.23981049542, 43137.80262506966, 24144.66831256109, 79183.50985209295, 49908.31641178701, 17369.46978170436, 12676.780530057575, 33955.144252245685, 46292.66628937639, 15010.270387652392, 22100.579992074876, 64616.29120561745, 45893.85777985023, 645.5258087964322, 43276.28172900436, 89567.01286108581, 42600.944913204876, 67412.94500355172, 9686.717501216292, 26030.937172969927, 93740.01433738133, 83949.79245984001, 49723.97560967259, 15978.915923277991, 56092.56330901401, 10960.648392538786, 20835.705080325282, 99794.73010002065, 41668.143307859864, 13536.256111584633, 51749.24373425489, 69066.44549531108, 7391.819265574684, 29114.094751908095, 217.19190329226868, 41801.779416235775, 74066.63135490227, 35979.10371486858, 93816.5300580762, 99179.70649638143, 94515.88577125223, 56731.659493221276, 96056.17811878082, 15056.32877015788, 76833.5059957876, 39534.20481897557, 45321.29754906096, 84065.61000121137, 71124.95304306083, 47552.246234301056, 22320.090943043015, 95492.13377609728, 65984.96865844044, 69986.09880638027, 61217.79772590088, 63496.09313456981, 17251.618635883104, 8504.95968542927, 94710.04003051511, 83205.75264449486, 22812.836852894547, 82046.39274614395, 98633.81368453307, 98088.06706311, 36412.58225616224, 44942.30696848732, 61902.839101909165, 91816.94529886641, 45619.88355836166, 16632.3939842162, 98881.49187523816, 55137.901788995194, 48657.79227314862, 26208.88519478556, 86275.98252101777, 8971.48947719305, 35485.97888891305, 58115.13947662944, 67011.6447286766, 7768.632828112843, 59461.97852193048, 66060.52056660528, 23034.898754621758, 51041.41848095735, 16696.973221009783, 22917.428767631263, 24503.77617491788, 72017.00710718294, 4689.928251417885, 16554.604878879552, 43684.09533223838, 2888.9335390022698, 19888.048848567898, 24367.56992524478, 21660.71283536739, 45858.89686347336, 25419.83913500465, 83687.75002325, 70574.78340289144, 20771.06520691535, 69022.38390382014, 26729.48591890667, 9564.60721518927, 74961.73814229785, 76714.97223196026, 21648.944284043893, 55077.684899289394, 37458.713071311686, 76916.6676743041]} +{"id": 854, "vector": [60526.16364560385, 102.7307976717684, 43605.454166265, 80569.8443752287, 70274.91411821092, 53763.616875702224, 81106.94668326956, 31988.18096320698, 75786.52437392797, 15114.682921118649, 76018.59198213847, 92663.05649589846, 31140.979969798165, 59947.17703675878, 28641.113950883446, 22323.87228360535, 58207.55438221445, 44872.86761346168, 1889.9458263258784, 88876.04880793436, 14613.328012849957, 20155.748369325134, 98230.6867802407, 36620.26671595111, 73410.28856258024, 85185.77813663294, 41973.53179919664, 7169.695925044483, 89555.78838629591, 99506.24273057962, 11247.853923513829, 19056.628751607284, 82428.04066036684, 81151.27217560833, 53219.999849385844, 60518.5920080956, 20057.461209163717, 80768.5678326593, 69935.07279869918, 10350.920972465039, 19051.75833182434, 41426.02847240988, 65847.99478634397, 75863.49274246645, 45583.826679209385, 43513.85767045313, 75690.95159086472, 10071.420974751554, 53865.258928421754, 99473.56164689307, 24067.156424605197, 20891.782107667677, 57348.66166455857, 81774.02891747511, 56374.32563603897, 81973.27645456846, 64380.57212771146, 52944.95800594045, 47904.24094892187, 20627.961654348615, 78830.88015207986, 38463.3203767491, 95666.68914913606, 77791.21770301291, 13035.543069480203, 54472.454929678315, 27769.075142522193, 47675.26927818621, 37396.04436590187, 52203.78341340672, 69560.16106266015, 21295.009972874024, 56161.83787121226, 12804.745950980545, 75382.15763416197, 49030.635335205516, 75489.0610338191, 48834.34939654783, 98793.2196680614, 24493.03738388896, 44692.25450174459, 73709.92396808634, 77430.3623699291, 39229.80029656592, 64463.21832548584, 89325.15986789601, 93883.18017106452, 65938.39208494774, 59934.03831863926, 67962.83312052775, 5139.760184818531, 75494.0354128629, 3347.038156218196, 74659.13664751763, 54151.181500905746, 22868.041250236438, 59939.60774631106, 82494.44835607959, 80569.66048518896, 70983.82228809786, 78334.37184147959, 34809.485236018445, 20211.478783807026, 77349.10352314282, 76001.18190719797, 62089.04308113809, 69580.97210387349, 37904.16634507638, 82610.4294537915, 93132.94787395865, 36939.27945588749, 95244.89829164058, 27605.063586928147, 36141.0745433085, 67936.68199461182, 76961.8717249222, 83349.44717281216, 67100.94254104435, 1226.3323326260456, 95251.56604116075, 17478.544403966924, 45841.75889856661, 43667.75903878214, 38072.949663823354, 39059.658333303814, 82304.52072264919, 99445.90139191809, 92726.86722698499]} +{"id": 1874, "vector": [6429.017498488821, 30781.96094886889, 52901.66363146327, 74724.70469445888, 82746.98788475755, 11721.477920302692, 55909.482618427006, 81127.75713411589, 49866.189185287556, 53737.159549782045, 79881.48176538697, 5919.287937903694, 81339.68178781646, 3278.460189523369, 11400.600875041944, 58298.600610739915, 63650.69571174512, 32282.842106702137, 22945.03700217835, 4538.9613176554985, 90819.59044444017, 1713.152396443729, 55052.96123476297, 438.508314611008, 17572.312872991202, 20090.201829225174, 68406.42281332979, 26466.946608973783, 73281.77378765243, 12278.499317819447, 2593.187595479918, 92570.99424013801, 74212.99099683533, 88857.18185924146, 24499.59783036626, 16577.873680372322, 80176.99615813994, 98834.86102466751, 59157.30537643482, 99889.84689098637, 95985.12789131685, 36972.70350854798, 37575.673506762665, 69814.5534452027, 1546.2852215227917, 27952.734293836733, 94800.90466839875, 52414.30655278907, 42691.44615208796, 42767.84454014719, 44166.100915429815, 71314.13322155613, 17125.735504665387, 16172.50004592241, 92319.75741556167, 26729.090705964696, 41821.52163377217, 71670.35487270927, 23983.92856918731, 13902.211121549835, 51439.492052844726, 3676.0014745831127, 5838.215515355006, 80951.528260016, 23122.254387763984, 70567.10385437036, 14130.170364663063, 79755.63606503057, 88456.63455282235, 75612.17267149525, 45686.47819943016, 84020.73858854537, 42087.21390343224, 99107.82482531975, 37417.29331073678, 84409.02716276444, 71721.81541602242, 34097.294481926125, 92892.70548773567, 20327.799519176893, 81283.1048320934, 72888.68502849397, 69917.39524164326, 81537.10653736735, 55528.16075969375, 64344.54910040739, 20586.83389236423, 9597.90298061579, 72301.26281489944, 36398.29319685403, 89168.29559148478, 58771.859351083134, 66715.1340256358, 55253.03220724037, 70050.79031965074, 75738.39068954502, 35995.94644222711, 10615.075182190138, 39983.8554410819, 59258.784746245365, 70810.04240642287, 91779.31585074407, 51285.95683578991, 97999.36517724476, 81072.25347678948, 97454.28591971897, 19135.464923870517, 66868.74276354093, 4674.55753397904, 24042.748746803245, 15485.532629858168, 21080.19281119352, 77750.02038971441, 83697.56132349593, 45799.11570135953, 23179.70217201132, 36833.74348150306, 83346.01353584357, 59530.91366355795, 427.9972812580635, 25999.133303934785, 67734.80534800998, 16124.446446290774, 80431.01630322919, 46924.21580814561, 57969.75972507478, 77774.57873918879, 93853.22671275186]} +{"id": 1227, "vector": [96126.7374551742, 55741.839915022974, 3297.4712250056327, 41355.85107256386, 28175.238118987334, 22381.253315463877, 20011.55111114724, 7778.02296957214, 35727.225343738035, 27237.91932885119, 67568.33915592398, 38465.76450699943, 96072.25472829232, 80240.29883352858, 58765.501718484506, 69668.54995114835, 50106.931666449236, 91471.12372398825, 48604.03460630781, 64366.673494108436, 77620.46432354089, 33772.34665225166, 16221.457335907364, 97759.27307153092, 84539.70950783147, 60142.31531554997, 71633.6729430621, 17180.1233130082, 64432.14282789565, 92610.5450675135, 40688.75976388475, 3385.871555461184, 34515.58562717727, 31414.791410556987, 59179.16250713795, 13677.365027892663, 19246.600079930253, 20792.623593953187, 13387.494739889295, 46784.69528965589, 45597.14190603045, 72493.37392677803, 67995.43221127297, 57847.78615502857, 57470.65205947864, 24987.05537437471, 8860.102611022035, 39844.70606373603, 27953.938837565416, 42851.9898607251, 85544.03195171652, 68578.46324947814, 46566.8091591949, 42556.140477389825, 94340.7477422858, 70519.23223319216, 74956.31296247718, 56037.050540962555, 85493.7512234658, 12195.839869999381, 95145.49754210576, 50349.994842900625, 42854.44993529659, 60124.014259285876, 18169.663617526654, 45374.74698742877, 26497.0781476193, 69453.9085044462, 2424.261325224364, 67639.65937830711, 36402.15661180538, 52249.26742892614, 74036.58330296667, 45810.86596109055, 96614.04508391395, 39573.61810166519, 41368.533044650576, 47880.48991917663, 52071.39052717771, 14986.345570476855, 49353.58642159118, 57482.03207332082, 4397.197227111927, 74322.64198490784, 3649.592139457947, 5477.621410817912, 55107.22063268682, 85510.458081821, 60991.56048094328, 74660.19772696873, 53049.55840274426, 5542.13598498492, 12795.149983858533, 93069.75085388547, 20409.449744130183, 92038.01999132782, 54623.07984472017, 61803.47084108051, 29306.045692680895, 39299.226839055314, 71485.44626175574, 82717.0096061326, 28227.150896811905, 40145.4287454465, 93233.53930119224, 92779.01193231484, 43957.9586861813, 92608.41313503648, 37560.281341125265, 46892.44052812055, 93272.10162005728, 7967.509524420735, 12128.95140276351, 37480.7852465838, 89905.77619341994, 85502.68869657817, 41090.18744528997, 43563.840962577706, 82473.17497299542, 75480.94020222379, 33595.72775262396, 72875.74111914482, 59136.04484649653, 2548.0862369195447, 52591.812881624224, 99470.09414380112, 22712.274059566473, 38923.901229575684]} +{"id": 1752, "vector": [68947.73413061202, 89658.1450054056, 18674.61762291466, 38355.97777240943, 41623.90718067633, 97079.85036071544, 15990.960472242255, 69457.38487478261, 71941.75428150078, 97113.05030626383, 2125.6198164714356, 15869.561338360516, 99550.40767219893, 54121.07719289416, 59100.12631774704, 84231.15258921903, 53413.16071941228, 36556.69315176488, 70860.23679186936, 6051.232038567678, 82617.31723317345, 46813.8048323128, 68322.02727895127, 33943.61889347504, 13878.941884211394, 92435.56090732999, 26636.113229759096, 88050.17037297145, 96002.89457816139, 28705.80206534399, 83838.88312292706, 44340.36531832446, 11683.802071098537, 66348.56349524861, 32479.119119324474, 58087.35394342444, 42873.93778156059, 7224.35043673958, 86254.57386610085, 1405.7130899946424, 43334.58885089809, 36072.829573286755, 18088.289856047657, 8146.170518573103, 70347.03482969695, 76660.20350155211, 87978.65148529106, 86760.57088460593, 9761.003163398285, 75525.57772608932, 21128.30549845943, 99742.60231042135, 94047.75601491686, 37448.066192887134, 6987.260590513578, 97566.6781415931, 25348.119560692772, 31876.302773685595, 65678.27458239258, 54263.97827871206, 32843.49367585056, 39494.17423590832, 5377.9403434993765, 91314.90133406752, 87951.76577854846, 40678.26514169904, 31897.882156433323, 82705.07158930748, 25022.279933188784, 83944.61468358822, 68170.23838578787, 4099.721643353449, 24632.97844930825, 80203.26286077927, 21570.61864616503, 8486.651397825928, 68127.9649126004, 7432.930806713811, 8284.829489573875, 81531.67368481941, 68851.83676120297, 49855.732593568544, 99376.81448460442, 63968.62060224278, 16325.329149970135, 42984.69166919149, 79258.05057757054, 77978.60434663764, 37047.412877399845, 78904.11611399693, 46835.99439734856, 87976.05610654903, 48094.68700493187, 87924.70322753505, 69798.32235017973, 9033.666293383325, 24798.715403065296, 97326.69680235599, 22620.067289713606, 91280.96796791276, 3656.091593436106, 29946.238482981014, 48081.08989725258, 40515.78729951348, 15902.086838310226, 26778.95488507094, 78716.07242910103, 62298.325988922275, 44733.894955731455, 72808.04524917607, 1821.493383068229, 82226.93817768744, 12570.190053581886, 94139.14333564893, 98457.90606308778, 26618.406398526484, 23838.645545520387, 28274.198823098995, 65274.76614728467, 96518.40321959528, 94340.09250081751, 87868.08716073146, 43758.256106009365, 53254.20700454862, 79348.9851342927, 71064.68652827111, 42863.803276822735, 94645.64047771215]} +{"id": 38, "vector": [45795.65178502033, 39289.317915421285, 93227.51812125005, 59387.449641669446, 23454.981784858297, 43213.61826247129, 15727.224520917249, 80587.12146379915, 81037.07266131375, 11855.34894791287, 29007.805924431883, 36077.93372184146, 5932.3723907582225, 72131.91731370286, 50409.37503894607, 55132.91246013962, 87830.4450684216, 15883.933425305675, 8144.9387494378025, 4152.745822347947, 54275.24410209661, 80643.14507801316, 86798.64028526862, 73779.87898648952, 16554.673473755676, 89040.54951996225, 47937.36133161814, 69042.68043160417, 52626.977709762716, 51410.58238693012, 7314.023208039933, 29578.774541729435, 72842.57394075283, 54456.19123480947, 51249.498251565994, 92447.7763986789, 83795.37591480564, 9463.881187863732, 21076.886975634712, 14194.919765434166, 48809.13464057743, 53324.42477053916, 28471.923009455313, 74673.69091849377, 62683.832122479034, 85441.81607220355, 92466.2780404803, 42123.20748965642, 610.8391430274751, 72354.52132032372, 27103.258656492024, 21132.114244379387, 58037.712696116636, 75344.73846369001, 16449.49794874825, 72858.04722868674, 72675.34698260094, 62405.97548535826, 91741.38571209728, 42053.93744395871, 49054.158582697295, 61850.23252686874, 61233.147237908735, 455.9619412564997, 73631.29628774211, 11556.105785937487, 46196.20713737008, 26492.52963196176, 80749.4359713904, 7681.4547864081815, 57976.918469344586, 79254.3499450145, 13325.046817564124, 59009.58877177937, 71428.79369250801, 77142.03104464633, 98888.40269211824, 63881.34647811351, 97491.24505820738, 30077.107926975023, 37443.96315429447, 10272.41913190774, 86366.93666967991, 3806.8116647704155, 54395.05236469904, 87103.27238280051, 30941.09784630963, 41220.624189523995, 18441.179458570357, 13079.64440263314, 30247.84276002166, 1596.5621194434675, 65286.5066879632, 94354.06379489237, 74185.17892399366, 80728.79631209512, 27610.245632704013, 25704.182663944575, 87441.2848787211, 36027.58099109591, 60653.924582845044, 84511.63421250202, 7189.1966878422145, 75569.94498993328, 97170.3589514782, 87437.8210043477, 100.0241580899064, 58547.598639065225, 25009.653909053563, 28724.235766892834, 326.18073772126, 39656.79245456871, 54848.98010497459, 41411.05671066551, 88295.74373831373, 68365.11843881165, 64615.304214180724, 19877.707261625765, 69927.58637658865, 38675.601635997045, 43757.66074498606, 60709.25029906659, 61834.52546082194, 25246.990765789236, 74336.4755170107, 25123.909303872184, 58426.03725492661, 490.91777522016764]} +{"id": 1751, "vector": [56519.91874790845, 8712.377954182448, 57348.94771652658, 39081.40144314653, 81931.54920636155, 81295.77435627245, 6872.182211463262, 88093.87409876987, 21518.35294494958, 4370.263223288684, 49674.76849035581, 94198.89054933394, 47416.57591483325, 3565.63500227195, 41196.578091619594, 65697.39343843775, 97159.32151726144, 52133.47357876894, 61800.58029292373, 74428.78436905694, 61099.0575314824, 15443.430458891893, 68147.63632682573, 88748.71013558679, 48888.186421208426, 46273.416959585, 94721.18156069599, 97657.60649346338, 15088.852007364161, 30483.606979292366, 29010.0945692882, 25811.447520119058, 60146.94017625397, 50890.16248507558, 87940.3877759771, 10935.58668341158, 73850.2795474404, 61911.238260268896, 72853.85247712042, 39642.062226078146, 24096.769699964825, 60383.52572301683, 47317.44704829295, 56834.983865307564, 17829.110322278895, 16390.256820838622, 93115.95672493597, 69934.05030387292, 1378.9539043171928, 40693.536618360384, 44146.94772338341, 59233.158854774134, 72512.94225467418, 96738.7518625875, 66560.50787261843, 14969.17321595661, 82295.5337138335, 45932.26144070629, 6125.410132114517, 39218.58631047368, 63947.45306879718, 34823.50979839853, 21633.419743853843, 7657.530466582363, 21113.23325920329, 9757.34151445321, 84727.50357827004, 61301.08015264119, 79589.47848804576, 91293.66350203186, 24244.452066981125, 96158.49237016744, 2840.2060863700117, 50155.70725830015, 14793.093719429151, 7925.755498965048, 20797.347713010127, 93431.45704506616, 66504.2772002276, 91795.16302156367, 20692.646950681024, 74648.71630333866, 84819.90729199792, 67039.1487452521, 38358.859430294775, 54517.98724306178, 27449.80652746295, 44628.43721253479, 80958.9920114359, 81231.00322617442, 77648.04127854077, 17958.372816171675, 77504.35228138466, 93910.1675468919, 67205.9210500337, 96877.59210236026, 22156.920002356663, 26258.644847961245, 67476.02134032686, 76112.18794437633, 12513.326829093952, 8930.23653116728, 51623.83277567878, 94445.72244219841, 32885.54894655835, 82324.61587488736, 71730.20218908727, 79309.03428403978, 62244.24576108382, 21401.51364542101, 58725.978028674705, 76655.06790021663, 18124.318665157978, 72997.59338119537, 8890.817984404275, 31673.028840170624, 74877.92633911267, 84072.59145074042, 31926.58965765758, 86443.01048178092, 83411.93424530998, 60776.27000504467, 26737.976531434826, 33740.11080508074, 97181.53738964371, 98670.61986445102, 17783.841283526013, 5642.698391025935]} +{"id": 489, "vector": [97762.34844815712, 9898.848227612856, 71547.17777861143, 30099.674346623506, 53126.59179573027, 1314.9334427460024, 64596.69816798745, 24223.83318459024, 26348.360424687835, 12550.020380772243, 54857.10331085045, 91219.49845878697, 42149.33137631865, 71715.4083616514, 29966.484590873697, 40345.91813765537, 58150.09509002651, 20245.028904902385, 12178.589916635774, 1074.1640666271146, 41128.09493015854, 62495.117944270794, 89898.24469291794, 12914.438518990057, 76977.53357117788, 26553.54001155211, 88939.70654720113, 79366.48806501231, 91470.71146657737, 91999.51708895256, 3016.9057494469744, 63016.58345416698, 59488.57872034729, 21348.342780444673, 90182.6934924586, 32665.347842936942, 50287.10796019738, 46809.397510265706, 49576.40592935839, 20443.861463530866, 73933.97079473738, 37123.00632564187, 68545.63165987331, 78193.02941146848, 91769.45700078033, 4890.2673221515315, 85899.07169216976, 93289.2347591674, 44528.72381842584, 66515.78802292475, 56353.437259526305, 22191.133208820025, 50433.52028371876, 62181.31929793073, 25002.5905921447, 61658.146940299295, 74905.18775285684, 16235.946091230424, 19077.41922077376, 12274.421827851378, 57218.5814706161, 40102.287648029145, 6254.664101492469, 10253.40233448333, 25676.838832203986, 14953.312354503656, 90636.27124591333, 64960.68283308705, 9914.785350659395, 72124.02913233367, 89611.38746699627, 36158.538244053554, 69502.1121302516, 20888.30646374138, 10509.616497268114, 89014.85150453837, 60577.648983942825, 80810.8513006988, 77889.53463659454, 75752.50824671832, 69566.95887103157, 27509.913633285232, 24506.871243531168, 49740.49094922758, 10083.60477549124, 79328.67874674624, 87047.8710274847, 22700.656338673652, 21377.87121649921, 54023.14249138097, 83337.50602654443, 47262.94897489156, 72508.6897856087, 85782.92931398853, 17242.18424501944, 22923.80566281589, 9313.414414572586, 72508.5240171605, 73153.07209703285, 24230.613569125748, 74357.45219592686, 51901.87913109945, 61325.92107589625, 62686.311487117506, 74751.07192544248, 36063.11182657843, 56614.68483133028, 50832.98948597322, 18091.746251473218, 27226.720630286614, 61234.388729857295, 56518.433374549895, 90274.46298276843, 77950.57726814745, 39034.97059671682, 41389.61167765777, 13531.743381635919, 2779.9827941885956, 79902.53591446026, 17097.202289081593, 82092.45827936895, 85424.54561237314, 67721.44257872521, 38835.406791995585, 1544.0955312298388, 39106.25544702124, 7967.312075559907, 41577.51967437693]} +{"id": 1740, "vector": [19889.459463697513, 30505.46478404036, 26579.678671590646, 66838.75795403057, 86414.63036136609, 64059.34757966185, 19280.784537287243, 8141.757271396832, 7732.726933216827, 32238.79494879188, 73724.25870876168, 72571.17004023494, 28075.963409130334, 26192.60462538956, 38209.4644739647, 66756.09909807905, 6565.353987621803, 43845.20524060025, 21773.374316724447, 138.08268746143827, 9099.615680894447, 99291.5769829556, 81026.39868238017, 77110.63197815021, 92696.36453079016, 59197.397847941094, 94025.59198285542, 88693.26313947036, 44836.075355510475, 14627.69116199406, 66673.42569106843, 36542.60516171439, 9235.632460684306, 49158.42883626455, 79925.06392727216, 93123.07281572463, 89594.40531659912, 63147.015317723555, 42946.14444400231, 59304.991426977474, 66300.4914078076, 21054.905476556196, 13235.467244518251, 15742.486828578662, 21828.79428665918, 27246.597273813255, 86316.26295474215, 3.183636999293249, 15254.408334857671, 17384.836973117413, 39208.70360656165, 9768.798537877166, 23456.135989102433, 46217.39536718712, 1177.039820711201, 5462.569460146172, 28011.13747263654, 55141.609464236564, 96406.46681568409, 53592.420293441064, 33202.026479996726, 84348.72613588396, 68165.28850200541, 5195.1314225240685, 81100.8226545485, 33144.519831976904, 60101.18323890075, 61360.5718591413, 83737.94896819227, 32622.449164544476, 61977.990716332984, 17175.54657236896, 44534.41047869616, 39214.281425774236, 96704.07316872936, 73692.15535629343, 57466.531072480495, 79932.38515210313, 24887.65620750655, 2119.7497947448073, 27741.04516872191, 12176.900648944478, 50687.60547055815, 59295.86926213295, 46001.96726968989, 22638.74720903477, 46156.03784212055, 35981.62707680107, 20280.605917613448, 10830.279869122895, 39955.62397327341, 98002.20929827, 49273.13415941717, 7091.8261344484645, 61887.58859054675, 67533.22442406528, 53846.53362512703, 59895.32713976727, 10828.993110615049, 31836.49755930302, 89971.59562719417, 19740.112704267853, 76897.73776344501, 72236.36272981555, 35577.01078975152, 92795.266624566, 64630.536553994134, 61980.36043203031, 18937.26900353433, 66695.55147802226, 75394.21875036399, 59809.32039843092, 2361.850539675803, 63208.43041416716, 68322.40635566686, 70726.53328018553, 97296.8935839611, 27295.910890763407, 41093.137768777444, 92077.88380364292, 67686.6182994248, 60884.84537941766, 71280.57337996212, 68322.23388443213, 66116.52533684253, 62090.674958834745, 51367.116453338866, 14446.108109127965]} +{"id": 674, "vector": [52192.25466679234, 42077.05778906279, 35170.412725896895, 30325.375099024, 5373.677704142899, 2836.226851893053, 9819.584329400743, 72941.75785531907, 74552.27856397268, 9776.256867167021, 87811.03371047774, 51674.663700634184, 42247.78142900736, 5759.59728367943, 97011.8035226858, 4989.554456789536, 35672.228060480615, 43032.61297778916, 27199.90146378587, 88541.11022130483, 80918.86441921856, 77058.69621174507, 62125.131510979525, 95071.29421190947, 29550.3636549592, 58424.40494124077, 4128.6937995186345, 53257.29213121893, 64020.175400885884, 27214.401350984517, 30443.950690057376, 94998.50884132869, 41047.286686510575, 15037.464487602747, 79504.79536365022, 81595.48629260025, 30989.405377878233, 5227.777782841647, 47679.69774461408, 49092.5564100066, 97657.37964257113, 34445.04781940645, 63672.90379784854, 44307.46243099573, 77592.51300594512, 65471.5064638007, 179.11106616775862, 49430.10115643868, 73315.45689447325, 75400.31382586538, 46411.07220705023, 43428.33650606962, 28705.227764224674, 1201.0191318820418, 13052.922934738464, 44276.747580704745, 19287.79710884616, 32652.886979836516, 33455.918164608025, 68144.3975273874, 16479.474293823794, 16944.59540834292, 75553.78278898077, 35011.43414407386, 56204.9148729373, 60720.926260575536, 50109.38134368602, 57435.849439819256, 96677.61550914602, 57462.60537942277, 91811.3575113373, 44456.51094153437, 76843.3171527563, 9926.09403862238, 61064.45588338033, 55439.46685508109, 54937.602100638425, 96084.538602011, 59660.21244248682, 44863.99862598186, 61955.15759838233, 41149.77363933507, 60312.6894969326, 68233.95539370099, 53934.229037978665, 55336.04085158265, 18986.168708528272, 74709.57495868334, 91229.61725161402, 14587.414713939961, 73751.25279863682, 72140.49476492307, 26964.93771078464, 63237.92905416326, 62397.109637676505, 2702.0377923478, 21395.005632672048, 11642.420571849, 51274.40254048463, 71288.76024623658, 81957.42452331044, 62322.577176463754, 62326.69458762774, 81069.32799136343, 33946.25580425027, 52753.819594972876, 9833.4045488632, 36797.15235573022, 760.9566015004842, 91025.46281151082, 28255.322928475314, 19583.340220822898, 30670.48259712095, 65257.58184199801, 60742.608109728244, 1116.8717834316387, 73628.7640582795, 51670.3428139627, 61482.073542744074, 24300.936317608444, 51509.50892415427, 57607.75057526967, 35929.63008904892, 66859.45594288176, 8093.238171714756, 79836.7573971981, 23623.67281852311, 19085.266071051556]} +{"id": 316, "vector": [78551.33354661577, 96269.88690866553, 72512.82889887856, 60636.83669956459, 54383.864648519775, 53990.49235455787, 93202.8286321022, 88146.64826945552, 79607.33075148432, 62623.73373426653, 37136.98970914462, 29336.148335834834, 29588.08958028969, 56189.73466213709, 1303.9617116262336, 89888.07731704565, 80950.15020972937, 32042.156072956695, 50384.97123043345, 6038.937694298174, 83343.08523446607, 98386.06016982802, 2055.6247831267083, 75295.53238298833, 59031.69241141289, 96113.36047412771, 12740.22696280126, 59645.232550081826, 62973.91010766491, 56967.207013748346, 16162.23578914736, 73053.2811683867, 26796.2907948344, 51363.53171056961, 26566.90512847426, 59602.92431542864, 56435.87579932782, 50386.02049543102, 42586.79529385096, 63399.090786500354, 50698.931151980876, 1524.5856528600998, 93083.6621063672, 44482.09340256406, 27215.503726689803, 97055.58836662467, 5819.995742641159, 99890.27758716336, 37738.88381914393, 10116.713048874293, 43851.400668639784, 88200.86109479287, 38312.6857283867, 50705.87654855747, 48420.47344970215, 49241.91671761226, 53364.696032987325, 48654.01251485288, 67944.70243018403, 14452.666982625527, 12786.679621293684, 17923.675106500537, 29752.83276441113, 98438.30812127152, 53791.0830250239, 97226.95018786828, 78060.83249968951, 97739.9758277714, 82666.11502334042, 69259.84513440896, 80731.11279311909, 42163.60754566982, 64241.027482667756, 42682.83839322751, 48941.245222922524, 29214.302373279344, 37346.03058757684, 33216.11295965755, 71180.30456935086, 29003.949715397714, 48863.174451856925, 68542.30979253078, 33715.21937944939, 88293.32102174395, 47515.14463230263, 19265.819860708245, 41101.22464880717, 14092.915963564312, 41976.52427049655, 49782.93755748695, 86212.59838064152, 12469.404958499641, 51088.376851762914, 84529.14585598123, 58109.580598480024, 85896.70547251144, 4906.201506809571, 45813.49362399473, 64176.050457219935, 60295.473509420124, 18791.759021390873, 6146.215754797579, 42866.15289901972, 49547.180070960174, 99845.92521757045, 11670.186523885395, 68690.02256134037, 54249.64798962177, 45499.72730525097, 88936.63287351892, 88583.3401617333, 47122.52719718039, 69254.78406077274, 27896.971725239207, 6404.060227844832, 72751.90299612537, 78911.28137173034, 33448.1917359008, 55384.07690762741, 55835.266378302025, 12433.312790584094, 96953.7944455067, 2321.777424761895, 52650.41387911121, 65227.333292009615, 73505.81807401094, 89393.86863419713, 14693.00972948393]} +{"id": 1687, "vector": [68047.3635505909, 3183.4598745197563, 38388.74975906572, 56197.28941031539, 10872.122629475589, 30402.279003684896, 20680.798355862673, 95005.45995070974, 38585.09546856719, 43401.76184504348, 40139.38016604035, 87811.07969289967, 95348.20548300153, 91992.12211067021, 38045.74359891445, 80742.96518141202, 64939.08330741502, 60538.45410574168, 3426.348069013718, 51502.93239874614, 30943.18626139748, 81862.19144808796, 51319.398317853134, 68724.37690788717, 60113.706599663885, 44276.66292956437, 12266.5357172565, 98251.76484232445, 37331.425725497116, 18837.29777939572, 40930.63308057274, 88657.58978097713, 80165.16470201453, 43993.507490438664, 52327.58677712033, 35035.80700470481, 82988.24800495525, 55632.97937912154, 10667.865243987451, 69033.78833469871, 47692.21951678565, 56714.95745869878, 1455.848673544924, 36360.68053826421, 33481.705971902986, 94965.82020604599, 58640.42607669136, 76632.0451463923, 50870.90027841287, 65717.3122192107, 70701.81282594014, 66728.0111540993, 8236.253278032635, 62428.20355129295, 96157.1584726477, 11163.584364949063, 32015.43752674876, 96192.72273722093, 67497.56622352301, 74411.04390437718, 28544.36666313287, 28894.281453239135, 78300.94015656473, 22139.71698291278, 64838.2197012418, 96696.37151072001, 68511.98779552565, 52474.827355144786, 70556.59436811358, 90404.22080953528, 12906.415452933406, 14161.168117439516, 53480.56646578598, 57157.861385601616, 89906.4865496493, 14371.93655955007, 37830.7102478824, 33307.46136811633, 34571.85175167059, 79773.19876001439, 75123.24554928255, 52766.80399761047, 87990.7615488353, 6094.469293166505, 22738.500312536813, 65827.33463948518, 64459.984348755774, 93946.80458420835, 3757.143515615835, 28993.70641295754, 69982.68228543454, 66952.165044655, 31891.633590204183, 11171.55697320903, 98893.99959104546, 75100.2876026281, 76288.04969885999, 41544.9784592923, 27965.858310006308, 32937.05528324743, 28669.699268422177, 21168.50725465884, 38338.853617718305, 42080.31992037166, 3461.6317967043897, 52257.87971142384, 13965.258180426665, 27840.69154966653, 28035.904706721005, 80631.81798795442, 40801.801810104975, 61932.50479475766, 3154.5982533793594, 24452.00862841449, 57909.826142026744, 86503.52627200112, 83460.90265807453, 69454.98705266752, 18220.372988974665, 36231.44089669508, 40282.63064195579, 17311.024245377048, 64792.683788296104, 58844.262733927455, 22284.5093245599, 84675.56931719118, 82466.69755962418, 99562.4812332098]} +{"id": 1251, "vector": [97748.51384217852, 78142.87611090313, 15266.019391093056, 28298.306070745195, 76642.06568223878, 39006.980494451134, 28920.140608369573, 94080.3069746531, 57012.92498167641, 69405.26913470337, 73468.53945234018, 14093.703538868729, 53484.69671977475, 53280.44834488023, 76327.64381016335, 77311.39722714959, 43125.198263645194, 25752.617937629806, 36353.45666869731, 10452.33847582614, 74637.0538024904, 84728.71402354375, 39773.51630139319, 8693.052796317446, 46690.0070034208, 81974.6280905571, 21758.28376872083, 88757.33688105232, 51313.98682733247, 53651.581480050714, 11354.305662152186, 22154.20245919004, 90588.75363536675, 42451.30693752287, 58170.84919907669, 99067.45904147445, 14982.672531735187, 45974.55398924008, 5054.377997127146, 11302.36905461719, 31207.707994091426, 4675.2088038442, 72400.91836224242, 24130.96205525368, 4867.189496762902, 94479.53208228757, 11621.16199284663, 62400.84891498563, 62525.69944910804, 47149.07211931344, 72159.2866761983, 80545.27779345962, 70543.25260761665, 97567.02132592075, 26140.076714925497, 57196.19450820532, 84418.8112532422, 76778.41731935696, 10282.96588941574, 21631.148836950488, 52350.74648909588, 73871.93550344835, 19269.8223452344, 83668.64089585903, 4608.924594776587, 54742.594576967786, 26709.109666791952, 4164.275360998304, 62007.91078790021, 42070.706541591964, 23107.264008293627, 4375.257627011231, 96166.06444985585, 96400.0056038095, 31288.790233311247, 2228.0995039193117, 68263.52756197125, 61838.336058036504, 86178.32858757944, 88025.97497973501, 78067.84113510494, 94027.4381067412, 14418.089585035887, 36343.40264723903, 53835.99128596398, 42556.00839997477, 52233.4498758251, 45759.64502450392, 38882.552660378, 15070.393262851478, 32182.597933588055, 2517.987619538342, 69989.62393994881, 26171.852440322917, 1959.0626928527643, 63675.98391911207, 38670.52588479756, 8648.263183171168, 21403.359502325926, 65604.09738521207, 27241.763934963725, 18737.429253539995, 92884.46267245138, 46831.07779349637, 74261.41775010015, 2607.2800819034514, 45415.09522179607, 74351.88049084046, 15135.46187394832, 55565.044314104605, 73879.02211564589, 51760.91859643921, 99493.73799802091, 37336.334584931166, 27715.834215878098, 90575.14329268063, 23120.005286905252, 53299.31429751412, 84668.79410194456, 16544.352064973213, 1253.988450510002, 97807.08269854168, 37665.56502753249, 19424.153374818787, 10840.278005662885, 17772.397347042435, 17771.445163240085, 18094.54767596059]} +{"id": 1691, "vector": [22166.05826942345, 16920.276850643935, 82496.11786202475, 20118.868945875078, 13937.333579534505, 12322.022928985154, 61863.659078441735, 22348.85086114747, 40419.3275299001, 78150.64105710865, 85544.55731355552, 80096.48104086178, 53361.36585401884, 40151.9876694405, 34760.522476588776, 34303.997761888386, 73336.83912217655, 87425.97442061576, 64643.10534682749, 40432.01907665166, 35705.36385731269, 58517.45281806146, 75299.13078116118, 32883.089772516214, 38471.15529998008, 49672.4356544724, 85894.60866733277, 83504.04719972248, 70626.51098233051, 46612.2479192747, 39314.484069525235, 92487.98211583532, 25320.350868484686, 57458.40391964895, 52082.594370663996, 7433.325007968728, 86429.42182985546, 29403.14216897525, 95747.49206030068, 25286.62528588468, 46785.42463318913, 88572.01691944696, 600.8040596099495, 95993.57586764963, 94128.10551589842, 34325.19954172537, 93908.3335997049, 75815.24248782507, 65208.65084301789, 60653.3976184057, 52170.77541370528, 47812.68720257618, 13564.560055161879, 95849.77149358443, 50722.82208501273, 43077.71414134346, 39616.64300748067, 56430.595149118955, 31971.673408049304, 78082.48137533375, 67994.24735533756, 41564.77036651899, 84777.85120951069, 7047.358892991085, 94014.13003310302, 27144.465454446177, 39067.95393804472, 32554.565850807427, 17356.23205653043, 92771.88422127222, 70541.98349447346, 27019.738942417105, 45592.954523097505, 87547.90773696404, 93428.13288015156, 49582.98725321436, 7972.637794776582, 87848.020389697, 1200.520344463485, 39172.34043969895, 59785.583532816876, 59518.83757681694, 84574.8999712969, 25422.0887193801, 81540.40708966556, 38308.42070148085, 78586.60765864726, 89762.21900209368, 54404.579131203034, 56147.15713370412, 52090.535872370914, 26278.203818556955, 14486.874469596423, 65421.85613801308, 43161.315488231136, 86141.09976942146, 47049.89073606477, 91734.68501890986, 42667.68913411445, 38181.01846488836, 24078.978599267066, 6656.749793330075, 75815.54151819486, 7667.329499033615, 28292.168108135083, 92605.7940840278, 1226.64190906282, 3608.7779779694683, 17800.031171590737, 43603.215258427896, 45527.186345189344, 33044.8135980068, 58551.38018991355, 13819.447651243245, 41940.51095374788, 20254.426698443294, 89847.89663297989, 5069.043834123743, 81039.67467922193, 7853.6209845259555, 94610.41356092707, 2691.772073529275, 24100.32570612757, 33251.90267507897, 71465.05631497584, 19009.706854633467, 40067.74291627198, 2290.8911090189554]} +{"id": 468, "vector": [75046.43756628035, 38536.12086391463, 30388.07339743792, 32871.19924077968, 1166.4450059912347, 17221.294014343413, 6529.747827686516, 82416.04979135712, 85559.480549961, 2851.545838924996, 28233.66525199209, 38293.85629140256, 82427.30028989108, 84845.10759508728, 391.69962231822007, 48841.797753081664, 22007.391544498867, 36386.93459131901, 47978.07893181115, 31637.005257375327, 77024.67227681466, 28588.804940850343, 6882.888140616905, 44498.438209600885, 62574.35935611485, 27140.99617310075, 16517.48144732842, 27606.701324994166, 34390.21167577483, 7634.390750696385, 86105.99159295876, 12466.40108120074, 32963.30037431413, 86900.56193478621, 45272.4145581046, 5185.7113777801, 14794.497445863586, 69026.4080488004, 74450.99131526223, 30470.53691020173, 19190.826362645887, 59477.385144994456, 89630.50432032764, 42742.38256781957, 24132.038703233284, 17346.21302902204, 64549.361988594545, 78924.4532043398, 28527.35580961362, 71065.08054661361, 50031.58029026408, 63641.16698903808, 62311.45847499372, 43277.844957438974, 5610.32129003185, 55851.71870742592, 59465.76462593844, 61823.46470441981, 35341.473788718235, 92083.11683575534, 21555.58826950429, 753.4426225518365, 83513.0468637567, 59512.82006350632, 86157.53566737335, 40863.42598802597, 70555.99833216437, 18577.248652153023, 61760.886598391575, 56775.893654005085, 72940.05406020093, 96598.98361561549, 98543.48756259911, 42361.997031888386, 18795.44375906601, 16000.650770125125, 25487.557385440185, 56836.23393485184, 23562.59566558031, 31460.566006599554, 45336.29638259711, 66086.74197701763, 48055.02337176274, 15291.837653603114, 91938.98251313009, 90679.75294230353, 9025.253247464016, 25875.37670684662, 10586.044164957399, 86292.29276534545, 64649.500462288146, 84671.88242570274, 21100.170356096016, 31301.942073586775, 64809.263467086486, 44476.70614797257, 42353.62263004022, 22789.665793830784, 6082.5959458087, 670.8905723897929, 7136.072045038555, 71981.51274030466, 74289.44383289415, 21388.93601801548, 24967.93335314743, 92271.56557164385, 94130.16255307135, 18885.983715416667, 22160.518712151177, 22367.607602479176, 95949.79411183127, 16774.072139988173, 39109.181750531665, 75433.37915154392, 37581.78133027016, 3792.094660246148, 18563.487114754986, 90754.11411324338, 51403.97368750412, 86826.66999756798, 47080.78907415103, 39652.22624865833, 33407.087043261075, 77490.18661741307, 22869.742484059017, 33768.77951781323, 51937.6979282396, 87244.88331781204]} +{"id": 178, "vector": [28179.17186368197, 42030.53232630354, 8093.226601984882, 44317.31598974039, 77659.57153354104, 67010.58715430406, 67021.99141153743, 83295.67620455511, 14555.790547680048, 1775.6983421655548, 23159.515371628226, 63229.09001941988, 41136.15942325516, 15937.01942500061, 87183.4998597404, 60666.65757378894, 67791.86785694456, 56802.22011830064, 61534.89318222354, 18153.332112996923, 27132.934920174546, 80043.51382380455, 32769.493890030564, 5977.924933461553, 6366.971318870574, 85481.10933984684, 43950.43097572366, 38932.08292756032, 60964.20371661812, 34823.49147084193, 33481.61386658638, 91456.73166192109, 65409.63450421025, 71305.21247842888, 77029.31686683919, 80003.7064063305, 33929.97203528287, 96770.91925348235, 60039.239469395514, 38704.71580069035, 43479.77079276818, 87387.3696460957, 2391.817635621385, 94991.80754047663, 99393.46063117958, 53196.94873440023, 88349.4054147366, 29676.07292304091, 72187.6740143747, 60666.45152758142, 82658.00221597959, 38373.11844156079, 98115.24301662011, 71981.04413731671, 79664.81938073633, 81243.70762980072, 79779.61941404613, 93388.36784648504, 49295.52232233795, 19720.801194407333, 31498.52677340883, 95121.71219829637, 87795.78962592456, 26275.16942195053, 99539.53275329404, 95723.55268546988, 15668.33974149614, 98878.93786023273, 69565.94625302785, 50332.325237674966, 73958.29928078158, 4033.7822576762774, 47922.792092272284, 56817.854126873346, 54619.275257410685, 59009.88199918664, 97601.69139427238, 38880.314084240796, 91299.65747500407, 79196.17717652388, 77424.93535565876, 91124.13646098743, 92481.36308569236, 23146.472838072885, 22062.082189795874, 28393.230074095398, 50297.938454285904, 88164.78508251611, 65489.843210374085, 28004.778332234826, 56224.46693262281, 98956.78138996677, 86153.94384109942, 14091.579301834501, 39467.61805929796, 70723.96112570495, 6788.804890044642, 4498.241378058409, 40850.4154431059, 25722.30536322684, 7185.634806341368, 45158.4181600047, 75158.333513677, 87035.5125226283, 88795.70547394692, 66315.5566137964, 98019.84085665259, 1527.693735259883, 8529.22820623747, 48022.82045738392, 31666.18944728441, 78025.64352379326, 77448.27437293404, 53685.459530256, 4604.256938736396, 11517.725711845205, 60180.17663766067, 11536.389820032811, 59957.71672885781, 73202.36811701288, 70127.73948053116, 5551.524955654863, 46415.00059952617, 4295.374233232096, 40108.79346322589, 49016.63202803672, 32391.28862940649, 6611.49462278191]} +{"id": 36, "vector": [20084.38442827034, 85908.75125987313, 61592.04774587349, 77514.41269206675, 29139.41907103301, 90535.5725095655, 71166.51790239612, 87967.41156165447, 30448.545541867778, 41008.2036822746, 75264.8729017518, 35480.24594857047, 83894.90053369309, 80982.02040278241, 88884.31419583561, 68991.47903497478, 43736.91052622409, 8389.092681308819, 36874.06271400553, 89729.75312561533, 86418.17041992935, 43022.70268835433, 91562.33936148463, 33803.01169334322, 43656.748614138705, 24059.427741737905, 91142.12601222415, 51731.17693340429, 6099.593712291163, 35970.38499890559, 25944.59859815821, 392.9542908311712, 76259.89814156578, 18088.341654961805, 52717.5695446981, 87503.09626245723, 17187.35735053075, 35776.27196819781, 98592.7237385488, 56462.358467739, 64119.4490473853, 48685.95580980235, 57565.38323404212, 90322.19410447708, 62612.760857921094, 19478.347935924023, 7175.286409892756, 52504.95136376334, 24581.883598158816, 873.9278669968487, 52400.166062110155, 79516.37641729943, 6196.935393481773, 3097.709734506893, 37375.81755138626, 81239.2063933369, 77764.30607823224, 40249.28735298842, 24491.137811577802, 91478.63559457456, 86544.18822382393, 27607.085439953993, 50036.63964514857, 67574.97757837456, 99212.15217488924, 24227.82007575032, 9876.80779150213, 78799.91413597798, 42095.71820318227, 10562.89917934432, 40348.540040266445, 1379.3836084648237, 82456.26107586361, 80361.00585860216, 84487.45953755811, 5978.513996902224, 19734.09595692346, 72488.25827973863, 3441.5431550965936, 87765.01258612679, 1368.2546854290156, 85678.91610065308, 77639.5298890088, 98200.66047971754, 50930.27279264086, 99360.44393702611, 51501.415730888955, 8628.362191634342, 51578.72899698847, 9890.711496188465, 9539.280335470856, 84864.51690877372, 32045.48867663669, 95601.94224980797, 34304.878568612585, 47139.2558316946, 979.7077737831339, 82896.20773108787, 41252.54921563438, 91864.89271656096, 80918.66110012961, 44888.70178731651, 16604.90689809946, 79757.08953765802, 53656.28666208785, 79205.58527156836, 65727.97295541604, 33348.2480850156, 19144.16452765839, 52676.309202524695, 21500.311336659972, 17465.587561673423, 62354.91894061465, 30342.672527915227, 55687.178545254166, 34385.1132401898, 35318.65058862219, 76212.3922823424, 79983.32479309864, 29798.825267215212, 53446.298384361035, 21228.881383382657, 50170.26546598043, 31852.437281447586, 90001.07461766929, 72249.03607646289, 18590.451451900914, 49387.010567711106]} +{"id": 370, "vector": [69320.09098465648, 98504.70099342735, 53726.32159549762, 16438.12089298665, 64857.69445970263, 3987.483231606237, 27385.390569715717, 60087.68231270177, 54989.06883072881, 95734.01490454227, 58217.191537336665, 71810.31769718186, 18309.44186836372, 65884.29617071834, 74625.74664603961, 56292.22073681102, 65757.79066193385, 79327.20094979087, 23012.535792631596, 57190.9067985325, 20673.421047877673, 3102.610198764355, 37887.053903625456, 36773.139083701535, 37212.30588637182, 52287.852759775276, 19415.05717593511, 65906.07943562295, 36672.196868640414, 72704.24412925336, 16800.209628568853, 17501.32700007585, 21623.600542941047, 45292.91594936528, 87379.88488518338, 3180.8381063288207, 85558.88933249358, 98615.44005144239, 78669.49075947069, 79802.83961878726, 45066.9694258842, 16336.691277088445, 99666.82949421009, 6531.230150652312, 63575.89847863734, 1702.2674986495124, 42053.69747262739, 57885.42338935966, 75363.38202709092, 81309.76526374923, 691.1670178508577, 55581.032713510205, 61083.610413527036, 34602.79252231151, 67299.2336807426, 49026.52433336245, 68343.52370998586, 93547.24351835455, 80380.98940239713, 22039.47681398026, 51690.29875115861, 29402.62983552292, 59377.5086433945, 231.6938506705113, 26855.805400454126, 87124.43373617656, 34650.54804994446, 31740.76152734574, 75545.77021299955, 33653.37280978766, 57473.57174167861, 33424.29078564462, 2002.5457772836153, 84244.7192125329, 42201.882591555965, 51172.52465785521, 10801.884122172889, 12545.448899863542, 9631.146864565788, 21916.11857912944, 41985.027637663276, 20325.81570075952, 61521.85776334176, 80061.85334806888, 47007.003715766405, 47747.5916085857, 33141.33040409608, 11767.162584437818, 20700.99907386993, 66702.91270435552, 60591.36598211042, 36828.30584290081, 44095.42868768897, 27933.240907953004, 62300.41780627752, 77486.19845804214, 13095.03167345687, 14563.764720977024, 51590.11251259625, 30478.5345177245, 1050.5829026448366, 11580.589172264321, 72580.26007425746, 30544.556243420375, 6964.445970442357, 38563.51248956133, 4623.8417011948195, 4907.744002159575, 66310.90722929606, 23689.686260471986, 71320.64705664897, 93086.6229668996, 13371.928208854455, 18203.92859813189, 14427.525904895167, 64004.112079705475, 73313.56239610832, 65174.632739381144, 91783.13600761739, 58379.86772097764, 79788.35682177488, 87735.5045828709, 84192.06952446766, 92302.5487137792, 84446.58115442403, 70118.384329762, 18862.822542950762, 97178.35716383928]} +{"id": 1095, "vector": [35106.39125960771, 92090.23507700271, 57908.80260789613, 11977.989809957968, 13372.88863835705, 92732.61659481842, 33602.432862719055, 76492.72445815684, 37209.39132289824, 70991.58005154668, 21781.250508164783, 94178.00167848362, 10184.076007561827, 67577.37037889527, 20179.903920992692, 86022.31156444567, 13852.913111547949, 44765.5923906259, 78892.28093459616, 11238.6961659537, 82366.09114912164, 44279.190768254, 50131.438673812045, 43562.647757373496, 73703.79527626577, 39736.4191629095, 66234.87261394056, 53747.05257764154, 97059.54447064485, 46889.3166325718, 36154.32773670604, 55854.199098034995, 69699.14112825661, 96864.19473288197, 11935.643214853508, 2723.513845887682, 55848.401179555076, 27814.76699133003, 82213.66612400897, 96362.22708428452, 1260.4976380309329, 6450.931163831119, 1625.0051737619176, 43696.6157774965, 21725.818780886795, 14549.779123356731, 39204.37738178348, 77372.38015935435, 93649.94174816246, 14464.58175796661, 12510.595728308093, 69560.55278596995, 73164.57617345364, 50230.92236686578, 70002.63129091758, 41123.4342098689, 77721.08087998901, 72243.4346969225, 31446.215510436858, 49557.01242160343, 51213.99231500695, 73266.51869130289, 83954.5803790298, 31362.921486243144, 14835.01628546582, 21574.84997526884, 8971.160294233505, 24995.513144802862, 69791.68754025083, 11612.27391629991, 50363.82091694212, 26608.411831509595, 67770.90221528673, 84272.81849320025, 9833.160884655945, 5438.403065039288, 49693.16110079558, 16495.322588721017, 38420.259285152366, 65969.2065892339, 55995.10368531283, 54599.65601350044, 39529.703229653715, 79442.46966931352, 28441.200870853445, 12360.426005416048, 81520.77622620943, 8346.704032537666, 93395.01326652465, 56834.71724573892, 92631.32032042813, 1157.9775780255707, 93693.61782641464, 7597.43870881765, 87746.2539525407, 92762.86638062763, 32463.03897934987, 67198.23183321534, 11715.510934024831, 14894.952039194775, 18719.72130188567, 61196.220101309525, 89278.74369392112, 33299.93487040974, 6042.442160375339, 92768.7615602551, 5444.881500765553, 58156.09270367428, 81598.86704627649, 56393.901636436996, 10307.201526714793, 94248.04314032687, 57044.086323413525, 3238.6920702138223, 8536.206055508366, 41404.703779049945, 95602.93576200739, 68474.72622318237, 97066.71917915481, 97548.5896947995, 90407.57826361754, 43798.691628358945, 1244.962569847019, 65630.58538309623, 98734.34093420896, 60753.7276264849, 93829.81383660674, 7974.662208893624]} +{"id": 1093, "vector": [38057.17028827687, 18163.353821055483, 62013.11826677142, 75034.92376574696, 29947.461313122105, 30310.851343420254, 9910.389264538777, 43357.09434780306, 80514.24260783668, 93809.39887976524, 94564.09973100899, 68790.47404396543, 54034.47059489643, 18031.3603253773, 46230.24432796073, 98175.28138012321, 80242.87221675877, 453.81181734545083, 1172.3888722688037, 37886.70605887761, 58779.721091100786, 40129.07242673829, 22430.36634687099, 24676.685008101784, 72063.3642040577, 89621.20844290555, 67919.6470359168, 57207.61888758136, 50722.29146783064, 91444.60698412852, 18025.933921631055, 23978.529014727046, 95321.52498750224, 29253.268810029087, 53411.28745434659, 82667.6286055601, 96153.47350030231, 75254.06754592882, 56512.679952009625, 98969.91642576215, 91639.01526260155, 54880.79018561609, 21049.136137243142, 53112.38790388062, 53844.00617368031, 43664.42474582243, 93064.02961971554, 82117.39553971733, 23755.417785234557, 9667.203767752897, 37968.06969406893, 96729.44869481308, 94151.29231607034, 34382.87656246718, 38082.43944797809, 36105.81912988705, 9487.698703089798, 41970.810140572314, 41332.969570021414, 39198.301478929265, 50973.48710383728, 98452.38441588305, 10276.402699854803, 7548.263507266651, 80085.0034107043, 51249.23934407144, 58206.10072830521, 11724.727345529396, 18705.31931147771, 83375.4209595337, 27928.225515154038, 94041.06274729293, 42537.02520865931, 267.23391980597586, 90306.2957432461, 56136.12583187063, 66064.60693817092, 69372.49385510801, 10507.380136655964, 9185.797054754397, 11804.970308397578, 62584.944196554534, 85571.0025310115, 82370.97923902754, 10715.266881832042, 92650.36714361145, 94553.32636772079, 38145.54129731217, 53854.88019305755, 1602.3369731687942, 79869.52041129133, 56246.200820361104, 93765.5057755192, 77225.3679343996, 68909.32923323345, 39428.631185072874, 82716.42690017674, 29791.27746165652, 44507.442415448975, 4218.941775984986, 84131.08867171024, 82661.20104947487, 6488.990444973142, 50155.390694546346, 80418.28167366557, 84798.6735100534, 92598.94109866513, 9716.231221330961, 21119.855195190008, 95528.1204488163, 42385.85330156722, 4573.892331243778, 48638.50202496024, 48427.15510634053, 27763.92668612998, 77538.5841080019, 44599.2074431428, 80330.94955731566, 59109.32542029683, 88881.57136383673, 30439.645931494753, 76193.283829026, 13881.059142940532, 24292.161368928566, 67425.64089624063, 95652.06159348071, 61952.61766803536, 78490.67405792639]} +{"id": 444, "vector": [30971.58984012619, 16323.430175806698, 38129.92510622294, 77349.83934897759, 1269.3043718768338, 36623.87206863825, 56026.664520251135, 72879.7029284308, 92538.44529448808, 94554.59119645126, 89597.12063171898, 22321.924433964592, 8602.961879187566, 20511.73320851134, 19767.81120989356, 14330.453934635101, 30067.028698328057, 35716.789904714795, 69042.77604506715, 18861.105919005662, 40320.08916498647, 24101.55243795058, 40650.396438553216, 7831.521224389837, 48359.12128937844, 13709.106529675708, 85428.09427306816, 65854.33859469778, 30021.079468720403, 11533.947526159238, 27549.54059425192, 76193.52120410324, 35210.81793402758, 70369.35696294323, 93264.9945418591, 2791.5120945625536, 15.696039497248293, 39186.50129729201, 25451.54091261215, 36661.93106640232, 65330.97232418287, 49672.02271195057, 52063.59062154894, 24693.80228065725, 55028.73907128097, 13646.666351057413, 6538.680562364308, 61188.156794443836, 53531.5801420702, 25650.198220596678, 54340.891214587304, 82262.89673536968, 82597.05351793293, 53900.28056195211, 24928.9270696895, 75747.11238654378, 80776.1917721387, 40033.80691581162, 54362.49560562191, 35989.96208547883, 76814.44560106426, 44743.22614487949, 5380.856771713694, 44977.028946257655, 20315.678813404793, 59622.964617426966, 714.8151344322229, 17519.844893069192, 40229.11650692641, 90541.18477755293, 1462.873486406868, 56137.53532740783, 86236.95693882946, 15591.237219881537, 24119.05012557025, 27801.170266942467, 38176.88352273186, 61102.287363113894, 14249.760053325777, 88980.0949805285, 49176.59038898826, 51387.75016465047, 70271.29018304501, 65547.06517766159, 501.68583960886747, 24228.14941852631, 57445.441790486315, 121.8824408861785, 96114.25355197648, 66030.4760068997, 30402.042796161066, 7261.848235945956, 11030.171238914565, 20622.048980817155, 6770.937297470814, 15629.644140324483, 57665.81009272067, 40022.600943635814, 16448.35410295725, 48121.034256069725, 2399.1717551112956, 12407.648226961588, 66779.81627646995, 71458.23160899236, 38088.621096515584, 37833.30994967986, 85437.27923739163, 47529.62091778792, 49568.539366388606, 14058.022519684133, 18709.646364325894, 39775.52321737221, 64158.51305082043, 46216.308816682315, 93304.58760304505, 37454.51213261714, 19524.242666396196, 90614.05469396933, 92475.14884841007, 37208.93367185378, 19506.744538296272, 42390.92613748668, 71885.50841385435, 11764.66220723298, 99645.43943986748, 59726.142170729494, 97301.19529450555, 62002.24763472746]} +{"id": 670, "vector": [72888.71490720312, 83637.47003990716, 65337.23935929314, 27071.491652375367, 21048.325764576668, 59670.74713561512, 45612.32535169478, 67353.92587501156, 1099.0251265114548, 49969.32381756931, 49686.677228257846, 28048.15410328537, 90710.50152501969, 16454.38773044593, 74471.71250315041, 76627.43418683676, 29442.540726097177, 54572.36297858293, 648.1249680518664, 41757.91989092751, 82829.07357962799, 82530.35046716568, 23224.57147510646, 80146.35765252158, 6508.822019520099, 83144.55699444344, 70219.19289679285, 78801.15133160738, 45346.01819570911, 14590.831411709005, 6068.098367942587, 97895.52992134361, 8042.910350444954, 9486.959618683777, 71580.9569391306, 26308.395014625617, 91221.5735691936, 75797.06025968387, 18475.018995178805, 19818.25342829533, 51372.3296308765, 50832.74906162736, 72406.90423071694, 74563.2989956673, 33646.779338461005, 32921.62571447856, 38552.983965168154, 9988.090428917962, 18308.196171969106, 66752.6874320731, 30597.37109799425, 85114.45294825766, 72917.63092308464, 85296.22976813524, 35893.83455967554, 235.27699137975145, 51589.29627605877, 50645.92732194874, 54868.67945099687, 58033.61579508155, 33357.99971910538, 13334.607100574602, 93028.86400422073, 44551.75631705879, 66125.5341828238, 41382.43667688403, 85949.97293828295, 24741.083022292576, 20762.09389269703, 93139.14795562289, 7795.840935576526, 99350.052774938, 69172.59099638493, 82596.38620872208, 12.374871165465873, 50985.53641295341, 54790.61125034537, 86343.20113917328, 23392.85630318364, 44769.343182892764, 39130.96201673539, 67166.60458298361, 22074.022417525186, 80719.2767996667, 92943.61436778572, 14664.147403176108, 72728.85501944386, 31747.940467337943, 81378.02049431833, 9298.7192522479, 4287.342567397401, 34861.61033492049, 17166.324343597462, 45841.28631902373, 48452.13674373095, 46968.08547747215, 97819.91688552784, 31693.212902500247, 38669.79471539578, 15525.073613057983, 77082.25846043308, 52322.187301779755, 32293.984612604065, 88311.47601228509, 86557.80518663673, 58083.524355843176, 21058.59354773565, 42267.346617371484, 44071.164487924776, 306.13040523798094, 47362.06958848682, 30713.27002525298, 11735.23055410075, 68978.72184982938, 14389.612508943472, 33960.25241580257, 37210.59120981225, 39689.92570240362, 45020.45679372836, 31964.890433064953, 55328.481081595135, 27055.280593486776, 45982.50035489204, 60789.6070560511, 14770.268501338813, 79300.41902948011, 56130.20214926623, 49681.937079550866]} +{"id": 831, "vector": [40632.67261659853, 43605.07145452677, 94094.97067477005, 25885.762703766224, 19353.075725847026, 19163.11599192705, 20331.256102283747, 6592.71547157314, 29982.385759855988, 44180.10656886648, 56640.50660602178, 428.6200063639778, 62379.79992431786, 28469.90466810796, 26509.744398673098, 20245.363554007756, 8724.16286492579, 19869.90414541877, 79547.43100937238, 37215.38903767066, 37256.463791411945, 25754.618518836814, 39097.261457752975, 89649.20728727801, 97913.76627370545, 3073.5811212688536, 49740.38990256383, 81359.4483256357, 26063.44209104422, 93728.40794521556, 70887.56588368973, 11274.6702267845, 68732.22203448619, 71470.83748679039, 40958.86975396144, 21744.879201582036, 42729.483724486636, 56559.535629946724, 58346.04727049083, 53751.460923783525, 75039.09694960905, 26396.138361749, 54548.58637231533, 97381.57611839463, 73479.72690623134, 98724.68968396074, 98917.79961055884, 85058.81815842472, 98595.49773398216, 11730.353760456868, 78950.13835007932, 76601.10726242256, 13240.096263841817, 87912.12041274527, 91883.95601087385, 64651.073858026175, 85850.8789134862, 81282.7874858391, 49315.45473769532, 2491.0021980111273, 40096.49119916167, 79681.43285037029, 38972.08281674031, 71287.58743293874, 74192.15552110084, 33584.21644526843, 95499.65433051776, 23169.431887625604, 32825.27590771773, 90901.95422480766, 42563.86182676587, 99630.1504462545, 14965.73520672866, 2938.806709059183, 75061.7310232739, 93087.25466850666, 96887.23876338881, 82289.0348358521, 91375.98696413032, 66587.72471536424, 22145.040598455635, 70747.60234094392, 54796.94534316561, 86489.65369210484, 18969.76433282418, 17634.900099685423, 47217.58090868895, 4167.474914001678, 34233.96486469591, 77317.59136142595, 66122.06525776949, 16321.663408366538, 85838.64906436097, 51828.56230245777, 66366.16024127293, 84178.1455866021, 52505.141415101905, 7659.4258932483835, 58004.2633643186, 29563.951938439426, 10474.766013072156, 59123.68405546686, 2495.048972513314, 94532.40172004406, 76535.82608936816, 77752.4295307398, 92263.94731561319, 16429.42749772892, 17948.1770755034, 45938.12136918751, 63814.41924580381, 94340.30528903649, 89050.95485842241, 87574.4989620897, 68677.07728894292, 53721.282576376, 97284.06518183424, 56268.058480066604, 66659.16613248059, 24545.06709421368, 79113.1262601805, 46273.611157624386, 25488.719362513988, 47064.65994011493, 33464.520562323116, 2778.5363583108124, 90854.64226146547, 76582.31656417053]} +{"id": 819, "vector": [32047.293466221327, 32454.66088084026, 57813.317166785506, 65614.31192623085, 29516.82479515646, 64296.41743061225, 55028.91896155173, 64466.9948641349, 50138.065896450156, 13841.246234672733, 63469.3722948699, 82306.26068363282, 85513.09456681697, 53414.22653124208, 90633.96665730955, 16854.188277367786, 57009.63024301541, 62676.901234986784, 21351.58473943456, 33763.18903646822, 4753.580691750059, 42957.20981662392, 58031.047424574164, 32955.04463086446, 17574.16215561456, 15990.535749436374, 75459.09501101448, 27216.566028862166, 88912.75898188894, 31344.444651767044, 76590.10860492544, 3920.988503011702, 43334.22806518862, 58457.25190426501, 95406.09627010023, 55897.65293178888, 75591.08045426005, 15350.540072189367, 30290.959893834068, 98182.7858605578, 57965.3370549781, 26990.013919107547, 59873.463642237766, 66667.7079520907, 30508.046925888168, 64509.81790847093, 5888.159670424841, 38869.822336452235, 13106.921005140992, 6525.37873619109, 4630.029900587329, 85413.14984330663, 69393.26201477098, 18135.384440954593, 26590.05566422011, 4629.459321026452, 54825.80158437788, 81121.14783045494, 29821.779062323672, 15761.56240352794, 80909.16456509747, 32067.271981012535, 60364.73545677522, 25507.344313658465, 38459.9587388613, 66441.99906924734, 49845.88950297986, 82515.41691130711, 87050.56447269149, 92684.24381901498, 33188.63745887505, 69100.03152677299, 50468.92110454955, 10501.93190476093, 611.6214418508071, 57659.27457323534, 10706.549409560052, 55585.118947989264, 55232.465637393565, 51832.486864449966, 5037.52527679594, 41143.86660851114, 43260.3203686577, 5700.054384446429, 98894.17531768307, 94058.92439303806, 81057.37513679761, 13867.138973446503, 55153.878065392746, 84791.80504124404, 7368.49400451971, 30534.035372743252, 64893.02732601233, 57064.95195697409, 95246.92418982199, 66528.94519611303, 31720.435048351927, 34411.398698543526, 38970.48518888341, 23870.117239810428, 8694.4227298148, 2044.3726066704082, 78488.69733360894, 36146.70992200339, 53927.11625095644, 46438.549087456195, 37742.11033826267, 30077.71029996397, 12975.948513480394, 11355.383753711201, 9882.23380249038, 82272.32715587827, 55999.066860813095, 59679.278939363634, 85414.26997466039, 61872.15657563179, 15162.730204657171, 65113.46451408766, 61601.98664285697, 22480.84282138847, 13849.51830785034, 28888.90533044851, 62776.50193114499, 59585.57535884722, 93642.00289374457, 57358.46647563455, 90086.72310892527, 99863.7914916999]} +{"id": 271, "vector": [15243.055724279697, 34822.93816461179, 16331.776663040599, 35378.03855929058, 14698.635349065293, 54056.264031632694, 59066.16592213898, 3163.1327623809048, 17938.866929174124, 89915.88157054689, 2844.6286335161, 35746.989326022675, 88335.87719744195, 39619.89173063203, 50800.72906258702, 10750.419361467722, 62614.74681250588, 67865.98855609774, 76202.35652112412, 27011.83969094445, 24371.95404605117, 38242.7103201476, 97606.55423171801, 53188.72723262091, 14668.836905107508, 4265.23055129554, 80105.26146848072, 44609.594233595475, 80024.38546797959, 29462.806732380875, 43110.69454108316, 60841.417318061394, 63729.254001898305, 51547.07834635393, 74449.87979665448, 55058.991378492625, 29785.58242526911, 34999.716194759756, 74935.8286867941, 55175.12555443579, 54654.64192007361, 86675.62118189338, 54621.78147148345, 6314.528778919793, 7349.4822445971495, 44745.53955543098, 38751.88917128286, 91657.91712069821, 31400.56532397928, 72319.5185091581, 30462.53303713594, 37830.10841625644, 45493.637475448864, 27684.113850786118, 52930.38232542049, 91792.20215707559, 26343.45359344039, 879.7899500975536, 5245.547608335921, 73296.59725468012, 77293.35659007975, 1393.400456706717, 68926.19026215273, 15192.65377475767, 20995.754269568122, 85704.56972276562, 58190.22548979147, 18091.851194873267, 86409.567805482, 38675.814427489975, 88202.97189538987, 24450.176731405947, 63914.97184206339, 91235.8758754596, 32910.241762167, 29492.399379569557, 22449.72962169902, 1294.5520833380076, 36519.263758642395, 28941.37205079721, 33879.16961587807, 27984.24380740977, 52067.517538863394, 51064.177948839184, 87829.95646736673, 68394.88809643008, 93513.43429187029, 72088.54629638721, 11667.702904344145, 41843.576740975965, 97589.19065049714, 24407.54763097083, 64515.517368810484, 8104.2348508349905, 28791.036675905634, 83636.89379619792, 81595.24693499996, 73128.84517775662, 79441.8348131328, 86146.59762972989, 34491.0104275297, 86645.74210273553, 94535.42802417302, 56743.18422784991, 2036.5440184575646, 42740.23187507922, 82855.36006511483, 97439.83438062234, 78772.61049680877, 80815.22886055858, 55491.04653313081, 77278.36290032393, 27033.78881926618, 44110.44195290175, 30877.15327541767, 53130.21727608044, 6111.033265524568, 71913.346449416, 84208.15163632066, 90339.3097365012, 47517.288576313666, 55292.36863646876, 98432.88131619112, 73177.41156514705, 96686.99559896064, 16272.259910221188, 7256.095555973218, 18311.10373053626]} +{"id": 485, "vector": [79164.56014036985, 53358.39149595378, 48318.248995684124, 78736.30831248134, 64255.96292878469, 79522.27627516467, 56557.249157404185, 2077.9459358238882, 285.58226534729505, 97267.26233391608, 68618.58201205939, 79959.90936116326, 14044.662508965677, 58863.50186092141, 57860.21567863459, 28790.223982040876, 57219.67541033682, 42724.6173707746, 74646.49631529527, 41582.63036739118, 68303.70411124008, 3301.087408221026, 38314.46552853863, 15369.741907565216, 39078.09599809747, 4740.285429941038, 71560.40902063088, 67081.6419282036, 61953.13973292362, 28599.569578284845, 50707.25224654041, 729.1772182965727, 84531.34896914421, 21074.640567819413, 26332.099653370646, 49379.26841054109, 17710.58294302107, 91854.7115057993, 65031.75946506399, 17470.701937054135, 92896.08903031364, 29565.128652322124, 82978.38223108271, 63021.22512662391, 44540.72665310066, 32418.587496876306, 58610.545512242395, 9152.053895987943, 31829.87354515967, 92385.4143624535, 6368.106840013965, 86811.72214719933, 29600.945999160966, 92999.39587156111, 13187.092987258542, 42147.74507529793, 67969.51739404375, 96920.26444218068, 48344.51607973064, 78875.74214451297, 51715.49803914981, 60820.05408128471, 14461.785310126274, 38698.053301573396, 55273.48683391442, 77002.97509231787, 39287.926926050386, 14589.281097277928, 61181.950975340085, 84037.51771977657, 76957.4807414984, 70979.03636986479, 61694.88407204551, 81297.78169349906, 12629.79431278135, 41134.16346046027, 38082.95125324313, 89004.18863628476, 21179.804362492494, 7742.072016334667, 17347.08733707159, 96055.8781097529, 29538.17471728449, 42226.67798073474, 3115.1360182289254, 46495.1512691663, 50410.31775013027, 74496.8567063823, 12087.98295742075, 77360.51296962672, 2499.771400183326, 5712.105906985421, 22821.396329381692, 891.7304789959446, 98328.52820963424, 50477.65451693844, 50460.81475560858, 64321.77397577259, 29856.259024483024, 31383.839498290377, 42016.957235330854, 6856.599863062041, 82397.31014356129, 84257.42418246204, 13243.731787199375, 77939.08258153271, 2590.0615555840113, 68552.37444345407, 61864.235377089586, 72435.37432174223, 44822.76574566583, 85026.98856102373, 42136.3243627521, 63427.01285194765, 95827.72905615931, 41521.09551192251, 61402.91216882993, 28552.757404228134, 54459.93233266196, 58182.320123236896, 23401.00418240545, 86252.66341668004, 38423.6995838618, 45598.39604230025, 10496.750427506007, 10410.71719140787, 2488.1782104772874, 55752.68843723033]} +{"id": 751, "vector": [82370.24190946088, 72894.30019451097, 68343.79820765412, 12108.369310818201, 73545.71949971694, 27179.039769436287, 15109.994480300325, 39891.281161191015, 55891.73652388034, 58435.55557556308, 88684.97233098876, 11960.583230226073, 89167.71072153356, 59136.28097955795, 41183.02467610818, 31585.338889199178, 12240.920654740472, 46499.99098908276, 36046.74966166887, 8432.543886022082, 80178.16050489995, 74579.19332030627, 36708.9284955232, 61967.495503058024, 2135.7606891651003, 24760.701948976926, 7462.6644013278565, 32309.883738628265, 51732.05249682398, 60252.556075774824, 2714.0589925753634, 71001.62610795735, 56744.011380645075, 85168.77759709614, 14716.350269932122, 1483.2185880497616, 54904.5201590867, 41035.688442515115, 28946.998525080282, 64908.14317854607, 12393.278490221726, 204.78588854584422, 63344.44076456814, 70047.1199749201, 49224.48646135288, 47525.44650897843, 97739.1647068764, 52973.18915105717, 6410.460786928696, 30360.887175696625, 24859.322483845535, 41689.55270120954, 72082.1613285535, 75885.34716136241, 2697.5412452186156, 6707.199429586774, 9801.769701091056, 69071.05815644853, 55234.2185681289, 73670.59161484492, 40904.89859840375, 57557.49949524571, 28578.766650252586, 40029.80717639153, 90651.83980718013, 40882.31156602822, 65376.387163766856, 85893.19332014622, 7022.1717012102445, 69577.64492591438, 17929.455554931516, 19859.609623072814, 99213.72184636463, 37705.21080835333, 66101.48473978702, 44888.530456009015, 63450.49975675604, 9035.135992765641, 8492.795810924013, 37911.2868486966, 79940.59227933551, 76287.7458907302, 72507.61323004909, 70255.70833497592, 95335.30259736713, 50418.17728513307, 64472.64298333418, 17746.40048124919, 41581.68480026005, 77103.10518995393, 31881.653607962544, 72832.28471182665, 2323.3544643898485, 5735.523615436045, 85981.78822564847, 44201.44149242402, 83909.22290193182, 69067.25169293351, 54367.6920410157, 84962.11760537901, 48342.29936782302, 86636.75909483588, 97944.71214973762, 95784.03227747667, 89221.75384802363, 37263.02543438283, 64673.91126494437, 44531.330168457884, 94413.78751179058, 94863.17403223454, 48846.20978780935, 58986.53023921886, 68686.3759475013, 43306.717074104505, 73954.82934813072, 12947.953157500237, 61903.56168053043, 43922.64885406305, 61553.73160501946, 44847.81091252258, 27147.408568819876, 83819.66853169176, 72336.87114649429, 16833.84029746493, 96623.54072032573, 51802.47903556162, 20653.869078704713, 45622.45003013522]} +{"id": 284, "vector": [29306.24972565996, 82861.34921714263, 71883.38548376856, 17161.317844019606, 41858.85079411788, 51199.836428295894, 48630.25930209664, 24769.15060276713, 38425.08386987903, 9278.414149925673, 14803.144202798812, 6183.099727788011, 40894.93926749888, 55128.66309402511, 61923.07662602632, 67982.12844329361, 5711.14677747534, 34399.0974709025, 94572.01097540883, 31982.436982709783, 56925.67711885035, 62396.62147235243, 29726.73077442194, 71986.8822321632, 99449.91791343442, 93240.82657668188, 59664.511981819065, 11115.285347774829, 10548.811618116872, 29587.16802860806, 22901.1271111062, 30483.397253818745, 76035.90274725949, 84548.07345191715, 95924.42290322931, 87377.9970315074, 34670.81481894389, 58761.64161739787, 26429.469034581245, 56591.61228923495, 53492.81320967866, 74906.24048572498, 38763.85972276523, 31028.45102022209, 4009.862283488352, 68865.7442689211, 1662.3024229200034, 8609.209679869013, 98351.97951385612, 44053.318399213116, 34405.62858689295, 40228.2774601749, 72088.34659156913, 74758.4486538535, 40428.97055049357, 42566.11639388076, 84197.40184405775, 64942.79777175623, 34982.86576435805, 95787.1786458104, 54226.864276270535, 91066.19481390192, 92546.80548967209, 34857.616967455586, 86103.49388169141, 96458.04039905751, 97709.54391092385, 64723.54523296241, 68887.6582029732, 39916.16287844778, 83570.98328920301, 15447.734169464866, 96503.8728429295, 22269.79925243463, 23904.746390225217, 30797.420389193765, 49585.87464985855, 26656.876016392194, 28492.64172874446, 90720.28430012485, 58770.16377991443, 71077.91684637628, 5430.50464895104, 19142.16913047403, 56731.847854664804, 9639.142496816245, 39719.255007956664, 99939.936054689, 87947.87988415104, 26781.723228902454, 14462.524748029848, 58546.51389226645, 76925.44973358091, 97886.46743765827, 12653.211261627861, 1948.4327475202235, 72443.13868316056, 7529.431974655176, 93960.45155914388, 33164.227302124105, 90755.07868772889, 54747.34702721287, 31914.504842064107, 99129.12568504651, 49209.83225855506, 38773.300215987896, 91682.53679978367, 4782.624475169417, 58248.09683198256, 71169.34430589038, 15445.089451458827, 44053.13937729408, 62201.45917292136, 77820.80117532526, 40352.61025833865, 72113.8468392626, 16750.148088453774, 19627.262265952915, 10497.298371858644, 47917.884710873426, 60382.16194798535, 94129.11110327055, 51884.22904169525, 10351.703104231157, 97923.21698103768, 22303.57330249736, 77752.02550796815, 2978.908498515509]} +{"id": 1353, "vector": [96984.75517890921, 36168.78561456134, 51764.602532313154, 41689.229390164626, 20032.188236302096, 85284.3079021805, 61413.49055629043, 39029.05408185199, 52608.335191127466, 63185.858271720615, 23912.69306017526, 32939.759774464226, 92621.83393651678, 65291.91249385047, 38028.44516372407, 26584.865959744795, 26118.039503596767, 36433.387963771434, 63474.31370693011, 28038.26352011338, 23300.64347879157, 55720.4009751797, 40968.97451788702, 2377.6537831563414, 7673.399650586299, 88988.54273470977, 5093.669055243255, 78647.54121383336, 1371.4461441177273, 69530.81312773318, 58828.67020205942, 99222.49446747842, 41270.461903324176, 59633.80287271359, 47708.1186195497, 55974.4826381476, 25099.06161090013, 80517.97027648495, 61822.51642847405, 7549.875589453747, 11675.09177819064, 88220.6592391333, 51184.866394646124, 32777.741300607144, 89324.49095675176, 90589.80761606603, 18299.775760951943, 24037.2610227063, 20603.628269784436, 96449.50097175567, 36217.62388976302, 82078.96541120642, 19620.393583538775, 63840.046513090856, 9.10612206024286, 81094.97813846379, 35356.67203453628, 92261.70800710037, 34170.42423998099, 57866.94961048175, 25888.32209805584, 6928.891097278045, 53848.63486621306, 18605.29546787998, 31102.008000444246, 49028.667885074254, 80017.19354869037, 53385.875158707444, 40073.77655756064, 80289.88135604654, 26709.09660529286, 20746.08391509751, 67578.16967531547, 83994.76532120659, 74724.93066965051, 24955.58498857361, 29549.692473410905, 81357.50548867357, 96673.43060651768, 383.95335184999715, 88134.76509085331, 60186.183494616496, 44232.39824140203, 79490.9507333731, 85289.69516489164, 5171.84247294451, 68851.81063590491, 57689.57755481208, 85868.40832940664, 33636.13860852465, 12647.436224969277, 5107.006757407839, 2926.7139577172306, 81164.11520918571, 56983.54485093222, 78133.42478815439, 27420.280249669493, 44718.90906417824, 97098.5364557662, 52540.22688095805, 18368.717825438187, 77965.13308558696, 32047.075993454142, 51518.932663154614, 23107.24905645589, 94959.95672719467, 76599.50640848435, 36500.55265205078, 98632.09618318654, 55385.77698469408, 28440.61803890917, 67830.60093270625, 15922.352153999831, 1527.9264383167513, 62013.6416036449, 44384.30073867799, 73108.98508035367, 23556.005886411556, 72018.83826676373, 69549.98710324288, 38456.001440228945, 69844.50836905396, 83492.7754428004, 66124.11081193194, 51100.3192065146, 25594.95896051338, 71540.7114388953, 16489.157961794397]} +{"id": 1713, "vector": [34539.15324123832, 54754.09227047302, 75906.34405920983, 71527.72747294231, 32363.049451350467, 29243.338581445012, 80804.78161730296, 83369.70119846163, 69929.98419022243, 97926.34032183589, 14230.461665483395, 87195.50803740081, 16803.50147330124, 93755.57542172015, 88662.6750469675, 70073.16674623577, 39264.36236515474, 57016.37950898696, 61571.52479372916, 94921.01033364591, 57148.19266228324, 52831.06718859685, 74434.43312310026, 88123.35663536278, 16415.605722706572, 98492.6040833003, 82151.13755654411, 9561.208598765936, 62460.1365154785, 8224.890919489491, 83457.40734888155, 54634.77633790775, 76830.14834079701, 96269.69418507392, 91665.96852605458, 78592.98122853419, 26243.230259877437, 49875.02716686774, 46910.852779982306, 73652.2254821485, 59295.59649264089, 59257.746746048964, 63218.03727000228, 30262.14634861425, 25522.456489109623, 84917.14883060115, 92248.79073630777, 80980.74863060722, 24661.291848901525, 32836.64098123847, 42592.21333780042, 29151.406967446124, 88869.31908004633, 80448.1370556953, 42795.411990708744, 31614.975325394444, 41385.07207871803, 8807.226845832673, 59048.8020021866, 19759.4412979767, 72897.46822674116, 83545.32187126247, 62568.42712132336, 16870.31427623499, 63933.33981001489, 45330.92939161218, 70842.0043758374, 98536.36311337833, 38958.94062762635, 98489.08074960778, 81120.35031107836, 67899.83028726395, 11061.846504633699, 99490.42808549282, 52739.561204647725, 83828.65283556884, 96750.30393358348, 93765.41316227941, 73860.0905637653, 8429.124024973344, 26234.487105170534, 92566.9740466702, 45510.739930495714, 18105.168721468977, 80666.18865462173, 41959.043547941335, 2691.988086803465, 27905.678924203305, 37663.191885587745, 33161.70659413498, 37703.136362688136, 7720.850423399573, 50132.27056427928, 76442.18240370281, 52811.01065632017, 98454.84424110154, 22461.792702857554, 29031.813438303954, 57865.07361555538, 29890.756561985752, 37729.833867908215, 74305.12351368512, 59687.76382080291, 8775.952193278625, 26385.959053902807, 70732.22875432565, 35560.9072505594, 1400.1981362467332, 88126.6168026914, 31149.52570042264, 33550.8773903384, 7725.702101356624, 7884.81208471905, 60819.3000400964, 52990.59910178386, 36260.66647508479, 25183.124066339213, 34823.46652288171, 59413.457695037076, 62677.95911937057, 89906.22464677435, 73616.3328795699, 94117.9656556192, 63608.01406422394, 5839.566231287763, 68431.26566716774, 72154.3097851882, 84284.436605102]} +{"id": 450, "vector": [85151.65085853335, 37401.2036391272, 68552.93438301323, 79909.28361459613, 61584.23623248007, 91691.78323679508, 29965.501580390974, 52177.49014828418, 16425.432445878076, 24116.601163533614, 20948.86027251479, 97960.86073653537, 70308.06397252758, 37863.56636884702, 12620.260110391278, 68447.36663386154, 21941.78330762866, 8028.3127184383575, 35632.36010606898, 36177.648994015035, 74773.15866184336, 33085.670853951364, 35576.26925984601, 23428.96395588382, 53151.371321600745, 23478.340935598062, 70151.15714222293, 32481.498480941907, 37233.58566818738, 21718.209185967586, 17963.988488097926, 43905.606050075075, 29976.380727835305, 20752.828149942325, 41317.90511830523, 73037.18289050092, 2016.280773706569, 53184.94127027684, 82778.31271961985, 41091.40049911805, 37490.19556661661, 3679.343674103286, 14038.41300296953, 26544.493407111848, 90663.94687053145, 11890.243363880249, 68289.95569837159, 30378.336083804592, 4082.3685830448376, 98512.76937110687, 73516.92701572235, 11294.233075544324, 60734.55265610255, 84157.89614727009, 43776.276092533684, 42079.580237640366, 26906.10144409389, 29347.39642597104, 60364.25659556753, 37546.641240735415, 24950.35937253298, 95837.76147158488, 96055.8488090441, 37808.529373423924, 23434.52594530161, 82091.08425480731, 48713.48908268066, 76900.09358555498, 93869.83338868857, 58313.46121987899, 94422.96871294042, 91655.02066887308, 89490.97764602475, 36265.96617816017, 68292.47021567829, 11258.01827841154, 58328.211094644575, 40711.36427197316, 58029.115402400246, 43640.53968335259, 76524.98721649274, 86527.65985461592, 55528.71776553794, 70907.51814872812, 58452.641477037185, 21983.026396773264, 77337.8391643424, 23091.579439364275, 50325.02792230636, 66870.20600222293, 84127.87862369532, 92389.99825390904, 88177.5347184161, 67457.7609796607, 28671.90405145349, 91841.12767204225, 71663.66414108856, 4613.440997665175, 44674.8342317862, 2818.722242770311, 11870.363821193108, 33054.93260460148, 13030.29678508063, 97019.7608118345, 35144.8164152095, 25223.986089959948, 97759.10884505039, 2511.500265079036, 63294.90732469873, 33749.61311030338, 48325.08744712987, 13847.913754044872, 89842.86335799495, 3838.249859768428, 59416.89604322207, 14929.829381769589, 22474.117803079975, 89917.45431607512, 69101.25360935739, 38240.90123842392, 95984.68335360056, 5939.653961713731, 14733.796121285981, 31950.906549624913, 98068.44234871783, 52085.00532098177, 80561.50211688949, 79850.65037451911]} +{"id": 243, "vector": [5244.58216161482, 89411.20648844457, 30222.458620536774, 93438.11768662136, 47208.32724912667, 99186.11145843896, 41500.15684008677, 90155.14482163035, 61955.48281941181, 57473.237115330965, 31824.81751852596, 56751.15231381426, 95436.39272292532, 54350.957979247716, 77877.12911658484, 20105.062602947, 75424.37895559931, 53705.26516887022, 65640.82376977747, 26695.050744658056, 8591.23574821522, 60260.34772428434, 56138.01735970605, 87450.32830547055, 34187.86492091406, 41925.25787926537, 96271.65250701603, 74611.71142894051, 95643.3297990225, 66967.27975417982, 19950.776112217718, 60186.852920994585, 21903.178575353155, 45258.88282748722, 35157.844631606364, 18989.98746849194, 15700.329464436834, 35310.159820265544, 85060.36621016028, 78663.50539661541, 48955.79164742232, 70889.8053643898, 93937.122309194, 36884.52498623678, 29773.813488823096, 88099.11242471712, 89996.08685949526, 34353.0511181555, 92602.0031024178, 87276.69402752322, 43657.17317472759, 19057.210582306372, 34206.092723870795, 47032.84778494078, 80297.53673137286, 44049.07023486925, 19495.41827654333, 72072.98894323513, 17415.42333375421, 96764.62184641355, 82080.82086255527, 25208.020000608987, 64586.89209218012, 2078.830820058608, 60748.094883290396, 15335.842257926146, 33109.43447504897, 52409.9300700769, 35728.51706409974, 45451.68067769941, 86109.62273439062, 13736.545583487014, 1817.1665663643143, 29749.11454757857, 63340.52430758426, 90589.54217916781, 64338.63321279365, 51806.0495376475, 26619.7976588599, 77368.24859092731, 56621.49795867346, 55604.55998107161, 38840.01893916022, 79847.28849085557, 89571.86264974206, 75624.67358975766, 19374.196029442013, 64660.087826462666, 66863.10177143238, 123.18804497251091, 35337.96580269695, 2586.0942668746943, 61870.27515429863, 79244.24699397231, 47274.480155114696, 24378.269347212223, 88448.85068061287, 38324.571611439926, 44283.905181204806, 33599.641020056784, 20636.843976130935, 82666.98957219138, 64429.48169022129, 64986.46183100913, 15443.144574532975, 9770.329163757408, 76037.98184462401, 35297.175262439916, 11657.815003859218, 47468.39837354843, 98473.9379160254, 13213.57545952101, 59792.61043085865, 26305.81456984331, 45785.77434918478, 10966.074445727525, 54291.27868786252, 63515.0174042221, 66719.35495411197, 19390.838831018686, 72035.28377352671, 69135.94861976954, 34857.14162867064, 56823.638438063004, 75789.55343051959, 40643.50122133413, 27973.530517843203, 40855.89264071992]} +{"id": 1484, "vector": [40468.69560341511, 50973.382096358764, 56495.470172048415, 84092.83712667848, 50506.179923666816, 11978.49629302883, 95495.13789994996, 58923.49863055363, 91889.71930211455, 3643.827181693693, 29686.878684292762, 88728.53005277325, 79113.38450288438, 32038.07142242103, 21809.16123611443, 57392.70360885526, 41839.08752767595, 28196.600685441288, 2335.174138132212, 12522.241560634351, 33069.54227775587, 79289.1744330791, 74042.93165927462, 56889.17329945472, 41578.599645995484, 36612.561858375026, 15537.959985152283, 13289.784492228175, 91109.87622768623, 52777.18014817012, 14483.31037260534, 1738.834844598791, 27201.296530421314, 19742.33673148703, 82891.93995495234, 46235.220349073905, 50969.806520143524, 334.83840614139115, 64374.08647696697, 75213.51072832292, 44674.03870518076, 64674.91440403618, 22312.873963065405, 20535.171795834052, 11230.61997210043, 7508.971826327138, 6244.346362486753, 72855.07062739371, 1051.977880465904, 59828.32890649238, 32693.90653784998, 19083.13307257299, 45416.57832262177, 82016.44253468502, 84682.65249406097, 56531.11341293204, 76278.85058869446, 58304.93618121196, 92554.34252895342, 88833.13915596779, 50067.42333983275, 54543.527743347986, 58514.7614569763, 42929.48819856092, 71940.80707854724, 76492.2256175833, 92680.11101804319, 3423.550158379485, 10353.817204357862, 63087.44584840204, 39964.48983565587, 3999.656280620989, 57398.902459792436, 78786.67087300602, 85588.13194046737, 15491.73946029716, 45568.75880893181, 4760.747994437853, 28994.475696391277, 49965.822632775715, 11706.771467685918, 14783.300554726486, 70681.06620616463, 20234.02642982243, 58991.70179911672, 94162.3405453591, 78501.97622314494, 17137.755788227416, 98028.67476132093, 26997.036085868054, 11909.587517849073, 66218.3038614563, 98309.88723711878, 10241.217547185344, 22661.026244027737, 95049.26698930358, 2781.9324211085795, 79465.58012386477, 65096.72121811637, 52155.135728562804, 42296.31771715281, 65665.12779194338, 47423.15945581692, 26519.356632845636, 62050.753414919556, 37183.31008957455, 37398.48126808678, 56325.23136322417, 11495.083326140231, 3104.807608289717, 62944.701059283245, 55612.730037736335, 97914.58784524155, 82255.97746147704, 71588.80052595345, 36067.5010454398, 1869.3904551205121, 57936.85625082138, 63092.040919354054, 63895.32833620963, 65859.48744695519, 28871.366053121194, 26045.634398474594, 64345.24712543882, 87267.55501513688, 37341.50782063042, 45738.724169699904, 25717.568324816442]} +{"id": 776, "vector": [20485.55541410536, 26723.267871631517, 81943.69948693766, 27174.657317008932, 92209.29469616372, 42232.14505506212, 20536.081971408672, 53745.65642636701, 98874.41805641478, 67493.3164290119, 82657.49061411589, 93910.93277100468, 50655.992320858786, 36834.17289452826, 71966.68279557007, 81285.08130771006, 10675.689316580294, 64693.445670941626, 25959.434425967553, 9543.082258309421, 9164.611253133991, 28534.93769096971, 80498.18693440435, 42939.7727892887, 67763.25744250906, 32204.164826556193, 54425.97897707483, 83586.90956150503, 44582.46395707238, 48705.191375782844, 32586.812603727845, 53100.710746551726, 69761.64661552024, 9221.890729054372, 16474.799146463225, 62760.61006920417, 68421.94881064091, 84503.97629486742, 10048.48641001701, 64071.11482971134, 92510.10547318144, 45077.51761839619, 42381.84122596388, 28496.366289670284, 322.7561294738335, 77655.97542740159, 92694.54346289732, 71433.21914123493, 40053.146164465616, 80017.31297904876, 76167.62421614956, 33905.662404050265, 76570.77842621402, 61109.563678758874, 76476.15401964329, 8046.262168645735, 4987.22913731926, 45953.66617117527, 26643.262745515094, 34079.7185465196, 23589.691019542246, 89294.55574648203, 62452.05085497421, 13972.498794040323, 53402.19459152612, 54187.61787504482, 22170.828238879436, 80447.40200045696, 23456.41872531654, 3695.3247570936765, 61892.664907392624, 87120.20519676502, 6857.135438568207, 40755.9786676282, 24374.397874793063, 76614.68756952374, 71185.10322154593, 8415.638855982355, 69701.77309703515, 54471.41623462518, 67134.79824782313, 23356.006698050558, 5181.208165954465, 48051.55933422413, 43464.27992387631, 73455.86984500589, 26822.487973271127, 45196.31790982963, 31738.895034222845, 88219.2912090691, 91931.57687472399, 66985.5857994755, 23341.723786021495, 85448.7147102676, 2071.504160897353, 31716.12877363219, 48217.544302538685, 92455.72699677975, 92433.6820098529, 44288.12651817648, 26034.149277011085, 32854.72649813426, 45890.35408076063, 43524.159485085904, 15103.799819774444, 90323.71528522554, 13870.17259071529, 41741.52481064629, 63007.792745543244, 65683.04833502177, 77096.80803951238, 64251.74137635954, 31317.537727478593, 74579.26306577565, 16964.405259403626, 37565.491473112976, 1016.3568386508448, 90065.52525268063, 99850.02834679751, 58292.389191377755, 83881.21256530398, 51084.682560351015, 71065.7236135378, 45448.50942052241, 85553.95165715918, 79610.32078712179, 48685.972240118346, 78069.814279927]} +{"id": 749, "vector": [82398.69524580654, 57108.31333768517, 15236.417747483889, 30066.473373237746, 55055.98069784809, 95383.9124754419, 16588.99324092967, 77550.68008679563, 61557.499042709744, 27939.108842973048, 39313.68813258904, 49623.38114981796, 46276.78353069826, 92262.87786316525, 25775.13207347798, 93881.9225393535, 30691.25651335468, 29400.37326252879, 25272.17633148333, 47672.31033484452, 77216.04576463642, 35629.821583875375, 22505.3887832288, 37426.79945922452, 52280.10762552075, 95216.5604729131, 43706.8108233867, 85526.03608492472, 35431.75994383896, 33333.72214329885, 50252.91789609846, 7068.694859879332, 85364.68248311809, 50868.9347462095, 74912.06721688586, 52149.54943914448, 13665.927905262153, 26907.252632754018, 57545.95369943484, 58994.310031848094, 69986.04981691242, 16580.093940340867, 17207.382375481728, 36903.07142001476, 72691.2851450563, 48696.24759633284, 33734.11953615535, 69561.49171616489, 63909.746092689355, 48880.137836181224, 71688.084759941, 65575.59054271113, 2695.215098612169, 44267.08754099239, 78192.55850185918, 71738.96179568098, 52046.873404834856, 45431.39420044341, 76771.477425586, 14745.661260566334, 62438.89570411556, 89737.82208350906, 93164.08924614891, 68520.0983018546, 95989.19622331367, 26320.811340244778, 37242.63966906486, 30766.8740441293, 49345.38198988488, 68335.00120745604, 52899.86162267515, 51172.05428521824, 68910.19259173486, 76031.93758103011, 25652.023778131006, 56256.36787652586, 56401.45176674641, 70562.42262924388, 23616.70574421747, 72544.42497180033, 7653.901265584317, 33364.12254161213, 70512.04672268663, 40152.01747893745, 13393.522645439392, 94601.86148240855, 41432.04306925351, 97540.93456230883, 22197.032337963796, 27295.46213461511, 3730.3103708547724, 37066.59077735419, 72863.0538778267, 89072.22651260767, 17386.867667794748, 96286.96535095948, 60978.96194230391, 27857.958724811473, 96874.32918024129, 85892.17739218086, 73625.28107063795, 91833.26878870706, 2053.890280587667, 75316.9274999342, 5146.964734143145, 19987.269788076035, 29004.582136147183, 70858.42109239934, 43252.6913110851, 33110.181508290225, 9332.515601353596, 98713.86966848935, 20972.06016003831, 40938.22577890706, 29600.267025499015, 96749.83592450467, 71313.34654151059, 10308.6158545307, 89631.84136628578, 37612.14942156186, 72983.31296712786, 85271.14467447397, 63889.45107982107, 54656.664598747986, 67524.62224968898, 85744.08979291284, 59813.95790777523, 17346.8389690693]} +{"id": 2112, "vector": [7656.692416777478, 18339.244047193704, 46368.29030129159, 33708.28061023933, 98212.81543618381, 8384.903600390136, 75619.62459722454, 91526.48321247917, 15746.75749632658, 85180.3813176473, 80015.55590417792, 86015.690218243, 84819.77508618175, 5341.191483771412, 69751.66544448395, 91651.8800967079, 34439.144419365904, 6658.515216420513, 36065.55760919374, 44448.34067521262, 68356.93898624688, 96622.91702635723, 93720.93379367093, 21053.17375473542, 47110.58509167265, 68070.14326411647, 52032.90202715546, 75594.5776927179, 61072.60745242255, 83695.91806581453, 71287.37685986659, 25235.187139486803, 57032.42050787364, 88058.5853631642, 22943.854792105754, 82587.09088628781, 65149.70944812772, 99205.69455053711, 55003.16474987288, 84014.69344065913, 92281.25180413644, 88634.3757261255, 13147.388740095623, 76935.72007855687, 79218.12888278987, 12745.408388635426, 30242.16060177628, 35385.87463228656, 14425.33865705684, 44271.97919819514, 22245.912840082983, 1360.1885371050337, 46370.7426488259, 8608.425850137603, 31551.489249086273, 26430.32983462661, 43429.02653130616, 47373.47837925797, 14450.980462667796, 1383.0155835551805, 70413.65181951382, 35192.52784961675, 65573.85584379197, 1750.4549376887835, 90447.74405576274, 21496.587071571816, 98570.62256654551, 21723.51158788678, 88735.77923400744, 80233.09245810173, 79201.52284896903, 16441.556531363276, 27403.14884841992, 70682.67166367384, 2549.092012283527, 77668.13932022238, 93302.9351925629, 57096.20537818769, 62538.1967758177, 11001.824481041678, 45302.02821237075, 91940.64243766521, 16520.240663743512, 6450.722247216645, 15677.68138968435, 42408.19018764007, 83933.94188228354, 14968.0714133814, 84932.98241081451, 82303.23398530265, 21802.553445026362, 30299.206142552703, 82093.95581828334, 5765.597533869204, 1878.586927218473, 52852.193634487754, 74173.70063755116, 28977.984730641194, 67957.38404067473, 4731.0537159855, 21095.63058222824, 51747.797347514694, 54155.90620652937, 97715.88795757412, 43967.11504877634, 59723.34890454961, 86551.84858528015, 45681.19891969844, 48270.26651755264, 8854.932081534505, 26838.814941366873, 75174.48119300335, 61532.715865992395, 46872.75551272455, 67373.37939165247, 64645.36970146423, 17663.184370954732, 28212.550083261078, 57286.0048857514, 51273.06212195307, 34143.43174385432, 5351.314119408446, 10105.94444063153, 89754.66514045218, 60864.99448003923, 52372.99326192673, 28272.88400243424, 39359.77825000632]} +{"id": 1977, "vector": [15272.550436462328, 9477.181400724678, 21677.618925428043, 69471.097920067, 46858.38056751464, 74135.44303667443, 54629.74853656687, 38357.70924041227, 92883.35393264795, 78096.96679213535, 33640.409865238784, 3963.946765819515, 1692.3702820845099, 92462.18918120734, 17654.119350216548, 93619.90673931935, 31596.595502002066, 66611.58831654376, 22605.106007619157, 94109.65260505782, 65063.95619850281, 55316.25937218486, 7214.043043185592, 37972.59190707118, 23044.51632949264, 57635.812888251436, 10345.18789608464, 26884.033572129174, 71471.64601349155, 47922.082783033715, 85753.53725641788, 10648.950306601102, 9828.952896787036, 95725.38821388553, 31896.123881738313, 69420.16927567696, 4354.666730145363, 65946.32150324568, 90387.2572452267, 75465.6240792457, 29250.393703566646, 60792.38150337658, 76171.34384650453, 2103.851342992347, 37757.72091695169, 11059.119794482942, 71591.9356485619, 31761.694383657403, 21975.227389829553, 50688.53403934117, 55934.78261220992, 34670.38169360248, 12370.45274695625, 73524.28490985361, 35166.992103809665, 45768.62276488389, 13154.88732085387, 15889.015361974656, 37575.7961266758, 5099.834874439502, 23073.824584448386, 6202.097394195926, 83243.32827676767, 56059.752434285394, 61797.51558072851, 80400.85150430669, 92290.91693377604, 194.04118440369268, 28825.63948245118, 60526.310471401404, 80654.45122423733, 70797.4792488649, 67564.9997915849, 98811.6611933498, 21589.24716548597, 15503.063986070942, 76027.73427245466, 61505.00607934697, 7811.86397493866, 68890.45867318759, 73783.48711140969, 45679.603581335934, 60542.50968542858, 10378.007074459527, 86782.09469552533, 32299.689204970084, 32112.83412872734, 50680.187458629625, 19704.43811306062, 69807.96937875022, 73239.97811391274, 64664.926979068, 15110.403093712566, 94227.81131059927, 33942.41447684771, 46751.71702741514, 20204.522428008175, 82652.52885096906, 58006.948486119436, 8668.354861835647, 89673.19535208176, 95264.60280331, 5438.488468130398, 96537.19723464981, 60095.20432493063, 73787.19288498512, 52532.87351228366, 39577.55317688207, 38288.035037795235, 89364.34095686034, 37372.98636465481, 404.7380669602085, 10903.602600714934, 27417.61571752598, 76493.28150480626, 83683.78397531172, 75306.45233182999, 53718.12821107799, 39669.74931209126, 48440.02102169278, 96720.19417559524, 90073.89154153084, 42008.52567899487, 44792.58284610755, 96314.6208424891, 10327.312164059898, 14991.656517677588, 59632.33991149008]} +{"id": 369, "vector": [34513.71662325294, 7707.363167664272, 37565.696562155485, 54579.94111603103, 16841.15757906519, 53627.31411609162, 88450.20012114815, 89779.89792443544, 24345.506671138715, 49504.536822102564, 2067.3319170849136, 29905.166692526742, 25115.709844400226, 33001.31382918198, 43792.43959897772, 81379.84412737604, 16903.44029859391, 19910.912690830562, 80195.10758415975, 13008.795171073729, 27665.75386123301, 519.3544370853842, 34814.487237377776, 86895.14413268985, 26450.364356930957, 76024.26391760734, 52630.45754522923, 19524.28514487584, 8787.835230217133, 18626.430391757043, 26100.312812879532, 40538.261115580346, 87154.75300497859, 72323.9055530844, 66393.47023306422, 16274.090946859966, 25552.91210689331, 87549.25316665514, 66618.43418308337, 91483.23007239873, 69269.44790172538, 74165.76410658236, 99956.37058892127, 50600.10057317204, 55491.20326396393, 71108.0775067562, 22144.919058833813, 42838.499190354836, 63623.62148972562, 32768.71984672256, 32171.29151631136, 82799.53815923043, 21709.032963621154, 53938.77730636232, 4438.1138291687885, 90694.17865847294, 39184.37660805648, 59169.3339643694, 60829.78131304814, 73337.70496676108, 31719.708407247195, 84876.02688634496, 85522.07942473199, 24412.781973932917, 70137.28079522125, 21884.2941579926, 68312.98620377957, 87083.90160916813, 60098.08440158617, 99742.28240216762, 94382.36605422906, 42531.12341929339, 72039.81549028546, 92000.58981522464, 71308.06605545578, 79892.2678156793, 56830.412900122894, 55617.29835609778, 97497.97153059731, 20217.760379633488, 7874.697778228113, 16012.131647965665, 77667.42373364151, 80017.50984106836, 72897.66365491025, 15117.992034179739, 57318.6241833251, 91676.60549054014, 88046.7065102615, 78394.58644371998, 34805.27730678124, 23121.138428886177, 85916.60539932486, 66473.23218330926, 69603.87474225114, 87934.68567999925, 48539.45153724066, 46158.42507362921, 74259.54071475157, 92288.43328086725, 73260.98400121191, 22649.562888983775, 45036.6198908534, 1626.8826420472114, 98635.41597955007, 86461.93790850771, 1436.7074179985061, 52033.57977341493, 95517.28260409922, 80755.5133350335, 97081.75308176046, 29029.92895042227, 48187.63206370429, 39261.5305466076, 16465.472738369892, 19861.050217313168, 63729.27898384785, 17046.778528558392, 26290.42004725093, 72095.120582212, 22765.667843617655, 86886.18373751252, 44092.51426255274, 83254.54767283727, 96162.10908813523, 81689.029924326, 85650.2004209312, 36634.24675760682]} +{"id": 774, "vector": [12909.965976995896, 48585.07945509586, 42733.2910411072, 11635.309971160845, 3795.6843326071075, 18749.088853923866, 60143.95048161323, 14409.057065482379, 39708.90893294329, 92901.71726204528, 17060.344482321343, 80801.07336091434, 4321.787400524247, 4364.869846037922, 31132.28339296379, 38850.84300169122, 95640.43670589277, 32692.626489655875, 78388.76544870604, 43157.06746223178, 47897.67494769921, 601.3745979468132, 37798.51283213083, 4281.335973256617, 23019.903327219203, 79339.82613716183, 83362.34131242987, 37379.89043784055, 77093.21392254921, 39178.840831434325, 21786.46392058583, 48600.22781217937, 61295.269383936204, 12774.537318144086, 62166.32109055505, 94945.5935794857, 67134.8005462706, 56028.5548204399, 92011.81252931371, 15469.223304492209, 74369.98510694597, 48379.789145870745, 38465.842581764366, 75532.19495933897, 34545.52146716972, 57885.35064300917, 1682.7762092329724, 99589.9023319177, 10036.249928370067, 28693.671634045968, 96231.5006002042, 1040.7518154827212, 74566.432581386, 52118.72688442294, 87705.58439795834, 97666.04695543776, 75664.07300454192, 85143.41685217812, 26122.62565253165, 53002.58298416552, 6277.751674480003, 17632.33447207695, 31800.289155126917, 2075.293921414878, 97156.24460962183, 15161.042801369273, 61044.444280740354, 92899.3108518401, 56983.825980928625, 80441.88421877177, 59567.15357325561, 45833.20766576332, 11914.672788947999, 74800.61358146681, 54200.57040401962, 71475.95614800068, 43472.15525339089, 40424.231211352, 26500.468054103465, 67975.0796252795, 50417.51948254882, 76924.36498063238, 46661.01029985472, 8970.606725688667, 18476.970398133908, 70223.25273414786, 55264.56587851196, 83141.8092554364, 45078.89503828951, 58931.560287982255, 52741.231026374866, 96487.6858265616, 89805.6046348319, 35898.45617045818, 57990.58583439291, 95699.54543583316, 31175.00138787357, 57017.75921944635, 32675.40929866838, 2689.7100967828046, 75849.74453584143, 96468.77258419836, 5120.905640302764, 17169.75314437641, 29608.014215998923, 8390.734915792475, 94150.13118142846, 27170.481202131392, 15994.739732893504, 28071.173985865717, 24957.056863302783, 40448.580902008405, 47923.69941647076, 94827.5639302703, 48690.24504645315, 54023.271079963844, 64069.79534860734, 99994.04607985844, 53032.90768466829, 33435.88412862546, 41808.88841110267, 19311.12106305085, 47960.91271857318, 39406.44440704757, 69498.09059176287, 26757.375234585325, 33409.74998821938, 75772.05971242044]} +{"id": 1864, "vector": [20206.05963738488, 86191.87164959809, 35906.613297183576, 60056.67435210175, 75708.60718947853, 17737.090459521798, 45403.57879797417, 9661.118370514221, 44565.44145812915, 75119.62485966146, 58595.923402034874, 75766.50926484137, 43858.72290077503, 95258.01515140793, 49947.863911936205, 27342.926643882616, 19680.950542982933, 15279.383517287137, 16916.432002529426, 46896.09068624897, 77461.31881000762, 35030.525068460614, 46864.481395243, 60819.24482632131, 32643.542604330956, 83977.02571100657, 97039.40305261106, 42584.90740642233, 19054.3973970132, 94677.84860022896, 58385.360125187835, 9146.025027688165, 33595.860950219416, 9554.551440461046, 72711.05799375956, 9295.397625093738, 8666.957138818765, 72578.20559260726, 97055.32192478361, 43401.40546714225, 77285.23995128872, 88707.82707762375, 15480.512614895315, 91025.36899325693, 89808.19302904954, 10355.806265839185, 60253.35742074056, 86321.15787783994, 49379.76322857762, 84401.87778008312, 15666.452742057902, 39968.35792913035, 84219.06198714406, 8933.477428783754, 88147.4990735358, 78376.0746149091, 30477.89764709953, 60128.11537037876, 81399.86843538353, 76077.01526282202, 70539.53305812884, 43071.95711852312, 99240.4829887659, 34760.94772885781, 64640.94971858767, 44312.3225316569, 57868.399485768095, 49799.15734615061, 8151.490645286197, 33104.5752585384, 68201.76187734201, 91432.2276159836, 82386.60281083948, 18291.13427426403, 87754.28722892063, 38398.93794319209, 32756.61124917556, 77909.09543275215, 10447.141226516254, 10539.258976631494, 22884.598176243686, 38637.44464881861, 9001.986410339157, 80686.358407982, 38618.110774443456, 44040.75193017231, 28564.894325592795, 73028.68086196657, 91976.32651588356, 104.7278264042495, 88556.10992809378, 38096.50853328816, 7948.485567191788, 31902.279476343265, 48696.61473758398, 14145.178305362615, 37476.090105284784, 39841.32537646769, 45627.57861134986, 94418.21003466743, 11641.398415098669, 46779.612068709284, 58065.709394538266, 20650.266445629317, 47590.90655849037, 79987.04925601342, 70474.36755974822, 78430.90948932622, 82472.77789370586, 45095.101387213144, 56665.20752321093, 47492.880413964245, 62294.15461148243, 21220.27895079328, 20109.949107408174, 50403.39299636899, 34496.273309313205, 34125.76621995813, 35991.68268962513, 29549.478243522874, 9850.633637402529, 85293.82094552902, 70415.42313748678, 76882.93325634015, 88870.97622618351, 87313.20256806749, 8007.674439188595, 41563.23211475057]} +{"id": 683, "vector": [12085.363508309232, 73967.65463652613, 80119.31782914267, 38206.64227964658, 44207.514792809954, 60248.0784030108, 32741.74256948953, 68026.22476055284, 40097.03440613051, 59730.30405979944, 94136.69636591428, 98850.12730831625, 86648.52901527227, 59569.87691433891, 2033.9470854714104, 35257.6545528047, 81724.9184999571, 31118.07628463964, 86360.01797689564, 66756.16226691466, 37116.18654692437, 92144.65385225657, 56761.306388722944, 97982.04505288004, 85459.96037944178, 10157.035584259032, 67299.72519276186, 30828.620827877694, 47620.86936978661, 20792.534948930996, 50889.75447861387, 47980.184205912, 99007.39848941231, 7209.987261798289, 37112.601509525935, 36002.40213244578, 16932.42434746265, 84820.01659723377, 51518.51526811338, 2174.701937358514, 27795.667739902565, 17092.885559111382, 56882.37484699714, 33170.290697591765, 80273.34566343096, 95336.44953929912, 45892.48099512433, 64470.240823497836, 23040.588553415266, 61418.35164309898, 13675.300326664308, 58010.967001625926, 54671.882278892444, 17943.912435779395, 85006.02965894705, 47136.24985911232, 62890.16408666683, 26147.347306274936, 26543.60302617398, 47324.596765392314, 17742.865070676016, 84255.92900168398, 15141.69143031655, 76984.90608410804, 76225.75209530616, 83303.86047506635, 29935.379574364328, 94350.18250765197, 39458.79984074435, 13486.521241518134, 74610.6891763545, 8061.510674471495, 7499.573365048073, 1315.9883253393011, 14135.634531888452, 66038.26566527187, 19042.28243216346, 91530.18258607597, 23725.802532779762, 46870.11049620574, 96629.74633861546, 86993.14283833337, 29769.548979338102, 44548.48294078957, 91926.05233891454, 72616.12252227718, 75001.77355488918, 82903.81117969827, 54424.40468644984, 79581.49828585587, 89854.09962992866, 65547.05721208238, 57343.0224969673, 95338.48962995729, 45910.835361206424, 207.38707419801773, 87909.58035585278, 50143.4014406164, 92557.96577664168, 7099.690844271234, 96239.79954314872, 61773.852374397444, 89853.9644982093, 29102.474708573765, 93716.86403382415, 17342.98396273476, 2861.749632355637, 74665.58571531712, 10203.28264818655, 29083.491326124957, 71061.91316946494, 80972.17727630847, 35948.98987142736, 25915.5138369919, 9617.011724794233, 68360.43524425967, 93749.34155282924, 65827.4891470687, 57383.96648197893, 59733.27973698749, 19368.21743206303, 92834.13486719297, 36556.400149259745, 74465.49068209504, 76617.69062351443, 32681.522755867943, 30405.949639797647, 15959.446827817781]} +{"id": 768, "vector": [4645.996983576295, 8261.207034203477, 383.14419388641994, 31159.10529945759, 24108.785963675706, 18608.7352958212, 60374.51063378905, 91813.42600022651, 44815.419824108816, 37512.19989833817, 13203.72490198889, 97827.96592204759, 68909.14268416037, 58642.79714286216, 68476.98377490387, 49745.87382241904, 45120.56715343696, 86117.78456721858, 98302.4291922881, 39050.12520321998, 33041.05518622994, 45456.29391286182, 76311.93978811051, 53066.56207997491, 20010.86997261904, 39561.94686458729, 89044.78845280675, 41095.534615593584, 90682.85272450006, 94088.65015697981, 14179.53439810905, 96435.79084601089, 62022.933342068274, 57565.59143009252, 43551.26040371152, 78820.82960838069, 13442.115465834082, 16676.063659555075, 16864.506549421887, 30494.277339325894, 16688.863446728854, 50663.4721736133, 34611.50143918698, 32508.718898282597, 91043.14541810374, 41732.5836899432, 43437.12068413333, 78291.09678631426, 2772.3297603084898, 82323.4107744834, 72118.56973147807, 59292.679981894755, 64611.24884785757, 46195.0901689399, 561.1983733121328, 23267.018992851587, 21917.381575167383, 9286.344331555263, 58527.79988946104, 47504.83288994608, 43151.26318834025, 6958.355879943878, 79841.38582040936, 94924.70417538649, 5762.574552485644, 95847.23191903431, 19218.813165001804, 55745.32329606439, 70392.80777645842, 40899.933167025, 98097.71573044457, 97351.07957358733, 47075.991873903244, 3056.586309212406, 84046.68842444227, 35849.76564786791, 24065.768427829626, 66935.31838393524, 78944.38722661643, 99240.62367885123, 79888.41601754735, 25092.594684375512, 73239.36148602537, 46926.074149315646, 41635.959233049565, 94182.01806422415, 18610.066967395724, 5626.036337929064, 13353.92290480023, 51197.05638680937, 52119.10047557103, 57403.32651997494, 8641.017095003623, 3591.7594346967176, 51115.357351580045, 9038.118199853629, 3026.459674318671, 95002.69239037206, 95778.38874739682, 34807.18454062046, 25610.206201820194, 84495.1742927013, 58963.11105224462, 3921.725616910299, 20307.08466584862, 80702.94353876429, 33223.03799789348, 10058.977406008751, 82545.62503417839, 81855.5609611526, 88176.87710545801, 76377.62179212498, 96018.82226680095, 13841.188852919462, 86791.06854049882, 89524.75777532234, 50174.95061736805, 38265.094570300505, 53543.66238943044, 38336.40851544916, 8448.543064430525, 45909.138149133665, 880.4761369157976, 62471.24824411513, 61039.85073031757, 89997.37478025623, 83390.70923354122, 91834.28129972132]} +{"id": 1686, "vector": [32187.155056633455, 25893.05505063777, 36594.50523371702, 35396.6266992036, 96178.4418755513, 29008.217771571533, 59365.11307263255, 73820.62707267716, 12012.218401393793, 10330.103272453583, 2896.781212465016, 31100.341492625128, 82579.93360250417, 43874.14021907957, 33529.16451592234, 506.7953549512727, 84883.41055764913, 42373.13457281498, 31292.910479564405, 44630.62660419255, 66671.73542179482, 77876.44858289974, 44360.54680519672, 99964.68214623134, 65999.3779144481, 76259.0193591877, 55289.90478292459, 60660.556081872055, 81533.17742927757, 36561.51817906699, 96051.63496697012, 58355.05731314433, 75322.21618318217, 20718.290282849604, 88357.85605560808, 18975.30337556691, 71637.79887948759, 340.66487266511956, 18461.0469476821, 8928.834121206297, 72106.02602861606, 25081.853222765127, 24907.830719940004, 78227.18190555288, 98884.6745490001, 33399.996400987366, 87803.77148277003, 31554.532423820416, 55925.66185333901, 69947.28891764766, 86828.87419616133, 70987.99551653692, 55290.99871669126, 3780.5079004682884, 76636.02185962057, 23975.507029122688, 70036.88799892756, 50430.57592245139, 63506.796051732206, 4432.577041361563, 85964.23713643677, 39638.100913153496, 36884.2023568386, 98722.02412623525, 65955.5937987086, 21100.06058229854, 52555.86759344119, 44120.4809467574, 82732.93523173689, 88534.6834820308, 80129.24164570715, 23856.062954782574, 61782.04580794848, 68237.83039433611, 56188.74915152788, 66622.95410631613, 52856.29066763161, 69591.87206813814, 81410.1780301082, 181.80322103120704, 15562.705164640955, 9025.76807471911, 4237.4534063054025, 16383.80681678201, 13871.401341783729, 78390.71915297571, 22314.977326602704, 57691.788579704786, 3907.26676648625, 72095.99115248337, 13174.48868232537, 24650.103744343298, 3183.845009950359, 13795.549076662195, 30392.084473155835, 71382.82195770323, 9127.652855149094, 65515.69122936171, 65462.52877311534, 59196.13523321859, 38035.11934039106, 92501.13224171617, 5121.096717719886, 55288.894734249036, 30883.21279095103, 89066.31544067644, 8249.719965705894, 6807.439910682733, 5698.195771578185, 58607.85780301271, 75884.88412992703, 49763.91289836816, 72423.96730029838, 79094.58538036128, 35583.10419331758, 45091.71131868642, 37783.91256666202, 84362.81893482881, 29244.496958161137, 33658.8456557552, 22210.693017104, 15857.719187674635, 31866.019938308422, 88566.45251220219, 72167.3197568894, 66761.4302517488, 81763.69962322853, 76488.19578571909]} +{"id": 203, "vector": [32497.94363302988, 82231.79471245664, 14834.338895454768, 87656.56395926212, 26804.413312693763, 86718.89373091175, 89495.82182048456, 58525.20658969117, 80250.8926612086, 36437.2380425677, 47994.969860210265, 1725.9137631815856, 20217.341234738327, 37130.068411360015, 41838.49681785202, 49307.970864107054, 14003.328691837025, 91040.11380483334, 86115.19134247288, 25511.62269255587, 83924.15619054312, 89046.9335491186, 31388.122302725074, 41553.54146952903, 3058.9614381626775, 64642.83772631968, 25355.76300112893, 25197.03785635048, 17812.491336448446, 44521.76395445073, 66609.79595507159, 69094.37703504377, 82467.13292433672, 17999.628357473695, 37948.935568880435, 87795.31814138232, 90684.92698185457, 11075.672519195034, 52288.56693025524, 88080.42651282041, 65542.18683221302, 77781.11481768482, 40012.808949227474, 9884.481506083865, 89426.66307461177, 55745.26327196792, 92692.87291602998, 24478.777353480917, 25142.21028614497, 54234.197979199205, 42068.05835653404, 77828.3339642914, 33635.1718817629, 48677.739201758275, 24304.175792182592, 74634.47883389013, 64421.766462067266, 85596.91713658329, 14019.054521134733, 40923.38364061796, 62046.95212293888, 70947.33945544348, 34841.021077092635, 76999.21743318894, 1421.242801541389, 51514.90546986841, 89570.05145342796, 8896.995847379507, 7427.201436440023, 33201.941100632896, 64278.73379817577, 69027.93649594244, 54062.832040637884, 31669.06099725122, 87864.04587175093, 36276.92899422085, 18016.48462790334, 51276.96646520099, 18353.57399843397, 571.9952149787156, 52932.80391062818, 18980.761779150336, 69582.73619047072, 28797.149263808366, 7754.627363682831, 13276.368697176033, 54400.88552549154, 46607.19806721054, 3474.963645938922, 2538.980244317546, 17626.00257748618, 60273.673909043835, 63332.919121565654, 71316.20160887038, 84179.92279930833, 33691.7295164728, 87840.85860825282, 90000.22746411442, 68205.45316965571, 36245.50126364736, 58288.460332936134, 15121.039750434673, 19098.938404901135, 84099.14623903098, 77308.34406346608, 70823.4815848961, 7760.178564647146, 77460.71322948465, 82387.37108825534, 45610.23944913894, 58451.02500985846, 37792.8643937657, 82650.1034336048, 24042.261882189763, 97351.6645139734, 39786.1936165497, 52773.302202066465, 54427.73404420258, 26911.166827550336, 49099.104521421345, 72043.65144333478, 77179.50748125078, 2565.259401716713, 80484.40074920129, 64171.37980741886, 43299.99554051297, 88701.35473085106, 97974.48741699751]} +{"id": 270, "vector": [88023.81611964913, 48165.17858516908, 64286.76891315029, 3430.680815525644, 5710.402515604706, 96590.78250354875, 72036.24270766687, 52116.84901849948, 74323.18116092414, 41249.137874920496, 49649.12097086807, 51453.60993345125, 96799.9742264109, 63899.43395177774, 57595.54159148179, 18721.29602504261, 43194.14120405921, 98592.66035653844, 34247.52280252736, 64059.11370281929, 78954.7639473885, 77946.34140200895, 99116.69228768541, 49479.69267308709, 30103.531817564988, 95672.90137740938, 20334.323863484995, 50628.51226962604, 80426.02726095254, 1349.2741898644024, 1185.9477471272694, 24811.947383005416, 83534.85958005827, 97800.15525363825, 39206.19189722241, 50595.147910252155, 17466.87735964968, 5534.316776598936, 65985.10963127388, 74885.4980313756, 17074.588120035573, 54741.21902015124, 86424.3463262398, 52593.67783956258, 60567.754641912754, 34306.54458930225, 9399.052416850061, 58962.58752354395, 43928.412743269386, 5947.702440532598, 3607.373369538236, 77974.03475730226, 37027.177950380516, 19920.716053559583, 27629.57103480971, 21307.234782528052, 57866.11082980776, 64606.944044737, 97096.82760634921, 23588.49497252321, 24357.007267761353, 23947.142552737423, 31368.616489564847, 46095.90570133614, 17362.620970260123, 69712.54435202347, 82028.96723983131, 83273.65267657625, 34940.51927804412, 80917.00857799432, 59209.87832959046, 48369.70110700073, 32890.82419162186, 69871.15686930709, 19885.232032814736, 88312.37862067617, 14322.58683682225, 62759.795991085266, 49226.14947801063, 40655.52704396786, 55469.00828748803, 88610.65018316686, 8013.028084540674, 37726.522149520715, 50718.99676133849, 99311.42214732355, 28765.236325759415, 44112.02512762842, 65641.56783243382, 8782.899439497938, 30117.645357875135, 87382.95612766239, 70842.06155961897, 41136.76356612288, 4337.250549063887, 37231.68788642711, 10327.330884864805, 47209.97229305923, 61307.20492149763, 41027.5442072371, 63298.467978074004, 4885.432223058583, 22641.78111452131, 83383.94895122426, 61580.663883085785, 78750.59355509815, 23601.539347029953, 43424.72911956134, 60813.29156393275, 63494.68680182276, 98293.32075511124, 18109.014863781325, 76772.6071911112, 3998.8773354176055, 54892.39846064854, 43732.60165906559, 10831.830318229608, 63882.98714042545, 8382.748515210204, 4863.463379229193, 40746.79925750056, 39767.52167557346, 29473.58145765576, 77778.06009731577, 5667.57373333241, 62951.22816143941, 87049.67117752228, 14063.396623674862]} +{"id": 668, "vector": [15785.15222440292, 84020.74975029725, 96774.51395829792, 29063.810537797817, 17892.955266198285, 50592.8244174716, 94913.03371410388, 67144.17887504671, 49934.29132996204, 51531.448298035124, 86328.18439569627, 83491.26134389691, 2113.2851529182985, 61519.55236659245, 14768.973200608592, 55723.844498360275, 40586.970942700354, 19865.076638296232, 36771.917554232656, 55347.37262945047, 23916.6228321678, 95084.54935295209, 61879.00103574113, 19376.22080739322, 7470.450853246624, 98334.60657902379, 82669.7351089661, 6150.797373839112, 35974.14561476765, 59014.10648102089, 94797.10521652624, 71887.45060145852, 63624.22551839498, 31488.021854764393, 6520.535866393207, 49259.626844120365, 97466.65067583707, 88191.35449179076, 86546.74788323695, 97381.41116245255, 30292.67067558933, 94444.36178510092, 22003.171215581107, 99982.69966665414, 93665.69483864134, 96139.27577153746, 4725.32104916632, 929.6698682066507, 86361.76018878388, 42517.806675301654, 91598.4385630984, 37033.205221304874, 11467.793356996935, 79367.94746631259, 2404.3336572050935, 7949.371666549787, 15325.6422545338, 72738.45246741803, 8598.038543560693, 16881.858179429597, 37619.38962249873, 22810.45114094177, 21155.53362999354, 78056.42797348664, 5880.536875458509, 77012.25907223292, 33635.91961792765, 55111.88066065031, 23690.066708941104, 86113.30043157144, 48059.389968637086, 86251.12320839764, 72423.0139975545, 24336.287997287054, 2086.244879391463, 85034.23718099164, 74723.32525367996, 45968.89348911279, 13098.01466264, 96085.22504529767, 11029.204930261016, 59129.893044495075, 27827.87364659003, 90058.04779111309, 68608.47097256972, 6120.174280380841, 54008.655161188144, 19494.48821376596, 95548.23760759938, 91491.59718157796, 3430.7519739152026, 16267.51690340832, 16559.447989167074, 54441.94956157116, 54701.2092237154, 81653.11704538277, 82001.2785341472, 30667.539927358124, 75224.13889915479, 91743.79091649089, 33399.18123836042, 6878.419680924475, 3823.425789998114, 29705.59668440449, 28388.906316595785, 15506.57791105512, 82307.28264600223, 29253.11255772477, 59649.47404827219, 44348.43229034858, 49751.3626787353, 59664.532770897815, 62593.76406009841, 38130.867542695094, 97045.67716696498, 99179.3097554896, 11226.426811366997, 71634.41031389919, 15755.119408280849, 93706.53857894731, 15466.226733915322, 13951.359936808394, 50998.40824773708, 95986.6543148241, 99640.12889254693, 89842.05346810573, 22784.46351324911, 47011.924483371935]} +{"id": 275, "vector": [53073.340089478705, 32556.017062196006, 49148.49151133113, 84922.20249448702, 55780.91476419882, 33386.70088503893, 94408.41367836541, 73459.4925982763, 35279.83038325815, 13869.038857700543, 68842.9857102633, 35899.573731484656, 75754.18778120364, 67635.3446596613, 30295.47787326734, 83630.69597191225, 17210.990188156782, 6945.314513782686, 56327.32177607066, 91891.2756288969, 521.251868556527, 2582.829243832785, 92546.30701419183, 97654.40169972168, 6784.8826213096845, 21869.929945996937, 97830.49097554212, 10058.681908497214, 37835.38368375763, 68165.73464317434, 54750.00451710597, 71681.36925896206, 28732.14222699665, 22033.10082639285, 37698.65070113302, 98310.89150221994, 10671.845498756316, 86098.40421475835, 76639.86799893357, 12686.025006072365, 37451.94894018849, 48448.29859395836, 34870.36074197937, 49688.59495969974, 34273.35315715546, 79043.90329175547, 81411.02087152482, 5558.784183055188, 32597.615958210834, 17817.704690111736, 56496.41140536356, 58162.808748508956, 56095.01387659389, 1885.424527238422, 6708.890745459961, 32232.17114142698, 72108.66653549012, 95735.16984190879, 4576.302259479492, 83910.36935010557, 23901.8419423755, 39083.648441810146, 27037.74785698053, 32543.99307688245, 23703.527018656077, 88073.11641895, 65923.05965548691, 34784.680661373626, 70437.61959714469, 68889.2798660633, 24614.88747055758, 32704.716216137396, 12323.524393331043, 16932.57455358367, 46235.347848238496, 30433.306627016864, 65766.71309340997, 9585.099689489596, 26945.527357314768, 91375.52037148266, 95094.35652794036, 10183.26388645755, 16396.621400376578, 21653.964266313164, 35973.52587582783, 9537.40820328135, 73333.26833644253, 94217.07927264397, 83224.12612035907, 63958.80778463735, 95080.02086344206, 73900.04409774642, 69000.28276058425, 99663.20201433671, 70164.65832993563, 26914.97681329107, 63964.25360886054, 38191.20395262549, 22718.312286995046, 91658.7625106156, 32187.63942345122, 33240.17369968514, 26910.17460231517, 86584.85894532032, 3188.3147110208456, 25915.91326164271, 78567.285996825, 23709.40468674264, 52432.645177294704, 45251.60703655215, 42662.50651713472, 11256.252865278982, 54588.60835112121, 96752.41157632724, 12473.94794247637, 38713.75392846734, 73399.75677317315, 10108.386222140198, 92547.58092980574, 72154.65720851596, 77006.52275350208, 98153.11246232936, 80337.12838046462, 70536.4374687935, 38812.12505841418, 2886.4259304286156, 51137.67030787583, 53547.90036838706]} +{"id": 274, "vector": [2834.6476735205074, 45917.28550501878, 78905.83067359874, 56832.35699509187, 79550.26742144098, 30008.69822657074, 47091.97990053053, 70188.12705233812, 2306.279430276592, 77810.10175857329, 86448.40914492309, 6524.9654032628505, 74065.25215118834, 20113.98136471141, 46117.7720550734, 82508.43786441641, 61955.951150530345, 49562.16284398972, 81872.330318858, 92015.6555252151, 82277.86219056525, 22277.387330559806, 18481.421089437743, 50914.30142144493, 27031.735843028226, 68176.16076383604, 31776.98224489416, 85153.15937061398, 3200.7313150857253, 3733.9497132935717, 80462.24734746966, 18056.453844313182, 68256.78901034825, 17320.101066184678, 30212.1213525521, 23639.50132971393, 10723.555178440569, 1399.9754308049116, 65270.473413229105, 32379.290772301338, 53550.23799294008, 36223.40972748818, 26248.57318003705, 81474.03325523186, 74926.9680384442, 35076.49450570542, 6671.778670266426, 72881.07079518095, 32753.71604802321, 73707.37341493016, 26132.84353852535, 25898.17368983167, 98806.40205302509, 77724.8002885214, 18467.821334793898, 35302.880415254964, 56855.9587249968, 72615.64619222425, 68653.52013920427, 43075.55690042154, 17530.77123581782, 31581.283724928067, 35215.77656438252, 66156.85817729922, 22305.617534640787, 29515.604468188783, 9307.260015918417, 72427.24205029219, 57466.09581800643, 5155.694681780276, 98512.41310285595, 63965.85991984548, 13283.03527525737, 42651.67777948339, 96953.22743350823, 5278.4830056604505, 34515.96124942471, 60809.065312942956, 96833.64501591468, 60534.4442509517, 38764.834024644224, 4452.83147369584, 56811.31326267817, 72159.26121041135, 69352.18538727348, 48652.52814524793, 20648.20280055082, 19727.62549376991, 6632.379804808541, 47937.65070490045, 56574.39406835184, 99137.14485424157, 55287.99415358854, 41745.78989193716, 91973.65143092336, 23696.84764758728, 49730.108046131936, 4814.801046738415, 22283.52561957989, 87813.02439018087, 24874.028978177943, 16284.697231470369, 16465.544041650217, 66255.0845916606, 27064.80833100927, 18539.62734211709, 85637.46741330276, 59628.24693817652, 97081.712243072, 1883.642444698652, 84668.05134730147, 29095.13244510468, 9434.689048263079, 68000.78880487346, 34860.110122563136, 55879.84833330696, 25216.277611899663, 73176.32525338126, 48176.78892136162, 66803.33640545892, 77478.67590883086, 32486.324159402113, 50445.667248724, 57228.17697534062, 81981.19325846872, 72425.04198734301, 55709.0658904265, 79956.29096910296]} +{"id": 1491, "vector": [10533.343710553234, 85116.88629896531, 95767.26414243843, 34624.11190498076, 72070.0536722211, 87648.58910687416, 37437.61524492024, 64463.64089245915, 90014.60748081226, 9691.3335818803, 66476.1220138203, 63889.47727506298, 65385.50585337144, 14408.072691298934, 67206.13681048952, 78288.4657198011, 66184.55758454528, 23251.683046610473, 14401.887180025697, 84287.68611083757, 84951.0654264614, 30164.17002170222, 7162.680565432866, 3392.7789025277157, 26461.104850944484, 9772.976884200702, 64575.120683306144, 9822.40663611309, 79704.72841225946, 18222.65861878577, 85404.12333863838, 94261.74374230923, 57092.32890271683, 75364.91851561674, 33052.78213024585, 99180.83624376243, 42625.78260052915, 44537.41395993489, 93291.58979226298, 9408.721373591588, 84735.81282650302, 61303.72135733983, 64653.57011288209, 43982.85733031688, 27627.02383741876, 20154.962833718095, 10160.503573247115, 13819.366026660296, 67043.97783618444, 639.6419103623318, 90467.82978606457, 48537.41350902412, 66632.48747688132, 18888.187882783513, 41024.693350854825, 16743.746618912213, 33864.31166432068, 36303.21926090684, 1650.2307536053086, 96562.32390455787, 27771.840377133994, 78147.74761598668, 6918.97535313013, 80952.49614584683, 43209.792898873064, 61793.760852133164, 88875.21586415505, 48897.118306812125, 4705.819925847243, 21987.85607971, 74537.50479191577, 96693.87452951109, 57790.477274917575, 93712.98844593149, 94222.89761813378, 29186.429439837902, 97974.78829771822, 14532.840413030646, 63624.691340073034, 92722.44966920538, 60975.72365633481, 41373.567934307684, 42703.28526787883, 24996.90972770612, 2571.8080993913372, 36792.08280559067, 83934.4389549677, 44186.16803148662, 96969.14068544137, 20002.7216017478, 9790.531855075867, 70902.96434979101, 18746.416599120286, 49662.12919520491, 91598.79993793531, 98780.05781830607, 21660.862957815807, 79514.42213049013, 74031.75090648312, 98379.08307983972, 5444.699980212476, 67527.56400066245, 95468.55507991996, 84293.22960952777, 92708.8157862616, 26351.444243925438, 76879.71674502497, 73407.82776411755, 3079.709326255986, 32638.43973601056, 34697.84999635337, 78369.39024182351, 84605.38002957233, 46710.77613604985, 98056.32328666806, 85822.18486556194, 47506.53607877804, 1418.7671031607786, 65977.3305280655, 60551.54367543216, 25848.202108683017, 30552.52077994606, 23926.915420578433, 12402.332297654306, 63680.996654723276, 71985.3705887135, 19579.776080227275, 79787.40295638652]} +{"id": 459, "vector": [88888.65532764555, 47409.1750629406, 2420.0860424372418, 42233.70331962424, 746.5517426573642, 16022.8929463614, 51240.98457935421, 34459.65059494036, 63773.80517631899, 81803.38361053795, 2543.6315193145488, 78660.80801241637, 42325.679901642645, 710.3954103711452, 88887.28734921545, 63490.134228295705, 53934.64665668346, 1484.118098815057, 62706.07770170671, 24009.583855860415, 46535.54518510835, 28969.476606906865, 79027.45550651851, 25763.63238370637, 34829.86989397088, 23146.83639114953, 67091.9304923734, 12710.9911988186, 64819.706281340375, 42304.65853240999, 81780.90746860702, 97603.52184125203, 68480.39513151106, 54936.85365024175, 33952.67310666397, 44843.597500532705, 55367.1139951142, 49415.3833052366, 50553.17444551825, 34572.395423667425, 60897.855606993326, 69460.33411536746, 91224.8489192725, 46419.67786102047, 29858.10028283291, 99022.98979626098, 8447.39926900916, 60013.79444550822, 14567.740617001878, 48530.061974034, 46797.76601876634, 43317.469977520195, 64695.93652554002, 35925.43092136834, 82855.15259231701, 38089.5132265941, 36759.19358901323, 40103.75523585182, 30097.914348528586, 69532.35791388394, 31114.429695691735, 3155.2892667532806, 51086.916765695, 97475.06591767663, 80687.8032632626, 75112.95573598426, 69602.3409257807, 89061.6008075813, 85745.88991431205, 53237.11186021156, 47091.36650379205, 59808.57278636896, 81738.82721486417, 24327.042367063423, 64670.09510556442, 43576.6758286682, 35266.767620323255, 30809.619115345067, 91665.51899970767, 44266.694366325755, 47745.28998641061, 19042.916939905517, 60947.51449933246, 29528.454295702166, 37185.88283356875, 93714.00079947744, 37880.544442237995, 62246.67815787963, 64359.549948805325, 67600.90680393303, 93968.968562578, 40325.77214360192, 74267.90419886021, 51651.642728156876, 78524.33644564498, 41901.69084730174, 89139.94087583333, 69535.42732036895, 44170.62078590431, 8122.2574364928305, 86814.31544459796, 2346.5672201494203, 13606.443276948188, 92739.08296915068, 87452.92883514431, 24354.035638709647, 4533.636418390319, 97540.97869897242, 13888.458114900293, 49811.97075549016, 4512.868650571178, 38134.03182374658, 19889.493181999605, 5947.264226329175, 62079.57652725511, 44690.41333645175, 56529.54926102909, 90349.21254698493, 73394.61448201803, 28730.188260933486, 95117.7620116601, 91167.89579399195, 14291.909808861425, 33586.855716746235, 64638.81286533558, 61675.03859251744, 31765.993246996284, 71155.31499806845]} +{"id": 2029, "vector": [30122.844157544372, 49612.027129524984, 7829.361249021571, 69282.0057918608, 15928.541465689961, 25366.7970687636, 97054.20998362477, 93053.75722074526, 8772.468844755333, 36083.43451793591, 42672.18084362216, 24042.963066669654, 15599.012429779514, 81631.25105261085, 19591.635723513322, 42368.32156571724, 43891.0596574118, 65970.13457722934, 91915.83218129762, 22157.610431388952, 61878.64082643829, 9673.341608179664, 49199.316116492766, 85437.3316319653, 22504.9840463149, 47940.48918134036, 67640.29621217612, 50305.59963581265, 46206.468638895545, 20791.62679740546, 99240.52832178767, 15981.042319135619, 86221.2143975578, 25750.42365749223, 70508.19983191112, 65196.14628112749, 46403.018092325365, 16440.37819672508, 13507.154057375548, 30885.161005614293, 10133.1346118838, 50887.83324697663, 1890.2376772044095, 36367.65328618082, 57087.2232105761, 15056.32068163728, 48268.99191163915, 20919.06366241314, 24826.110259183497, 54052.70802934196, 28611.23293216291, 22910.100610892157, 24818.12650041927, 50818.514847577244, 25738.21283355404, 80658.78003227395, 13426.622499783913, 80978.4170800079, 3764.4496210575194, 43722.10194019486, 9692.68083546968, 57994.572390975605, 10907.81552700808, 38527.21580352358, 56586.785813878836, 50107.79503972203, 14914.06355595395, 63234.397100019, 22378.29276241268, 71675.10913434, 47911.011617247204, 15931.183227577605, 35255.9594181423, 73760.76231466235, 20001.001348665315, 61447.83513571034, 92204.32940887571, 40598.64900261608, 41037.24255766533, 70225.134223953, 72650.53416703735, 68769.20818373119, 40538.16046753393, 37608.02755602384, 24328.18396646935, 8033.605150891854, 71691.19232266494, 66839.55088571404, 62019.7126643382, 72903.53522840742, 76839.12675691744, 8935.716984652687, 21980.503642616968, 34187.601533746296, 47213.12465539212, 238.9834690303516, 53514.18860478071, 38644.34401167189, 20526.971416232864, 35351.08970747342, 48551.63370099964, 36462.90662459104, 97434.76199583686, 66370.15881166288, 48711.82519591499, 5955.820678625745, 5434.564008257825, 74418.33227409933, 84593.65302460513, 34925.29964375042, 44655.87357802896, 38382.048940807676, 96332.65144318156, 71042.70991738547, 9748.653529210505, 19108.991145853095, 14269.491692459591, 2913.8591863396846, 14138.651971767435, 86424.75692145053, 71550.91968663021, 43357.8278657098, 57225.939905333675, 90645.89665678436, 84906.70291096113, 98291.8072058876, 36508.25970547513, 29863.804466799706]} +{"id": 754, "vector": [75570.856784484, 38763.76204415213, 76981.0157425574, 80190.57274392672, 76647.59794837797, 13648.543502030152, 99919.79333261546, 66141.90621704159, 17181.716811404225, 30927.479438737526, 16963.580713301064, 91097.9052816187, 35293.445922942, 60941.65658084686, 91334.5277764839, 97517.89437905718, 63902.32428468996, 827.8780559284438, 8794.14531679672, 86121.51250307655, 52360.18296710501, 23424.94924301295, 11903.718315514589, 76005.34380784603, 91218.53434068394, 90228.00411941113, 71946.36471667157, 49940.64257599574, 84200.01338342324, 3583.9417656200358, 87999.92346351602, 13862.226358729657, 59211.63988269294, 38189.5717459697, 52188.74806847529, 13274.301103042973, 51131.27592640091, 8657.894742915483, 66221.76794115987, 30693.109457209266, 38978.27651210969, 35606.02507830077, 81697.45915142032, 30672.795604280578, 22544.505138854143, 82619.75562586718, 22480.35977042422, 70220.26245770953, 64543.8526790045, 48919.41524488778, 44996.28891157008, 22669.461783358867, 796.472410239546, 58859.90600462828, 20232.188377588, 30885.68814849515, 37510.66364436808, 35384.54282815612, 82475.72963501973, 95017.35804628034, 63403.33504532023, 76913.11501289847, 17033.02817174822, 97438.67587433191, 923.8478133056449, 52220.25455897791, 66336.99852207272, 69078.33475400129, 98414.44293402624, 39949.21023160274, 74044.7352733595, 1964.9400618838354, 12636.715087253247, 16730.706041592402, 80398.9023300641, 47557.11788541267, 58037.75734660631, 65132.523950122464, 5027.308879628834, 46376.83802278244, 48662.87427672369, 38645.46953987341, 81901.87190534537, 25658.888632937284, 56614.561232857915, 57272.02534349639, 58850.0770551224, 92726.48273705965, 97340.07097086293, 44578.76396158609, 63188.426831380595, 17722.826092104304, 1726.7472474176038, 86995.67477091565, 45643.68515830901, 91456.89802312094, 33610.15366116469, 69911.0694454082, 99378.37635166585, 41858.84844587199, 58624.22654712392, 78975.39551516676, 64281.43701021166, 6170.049194810556, 67808.15079693178, 34849.058803905544, 32838.41296707452, 98474.73617450279, 48283.47520317358, 73872.34923971261, 31057.523885235427, 94291.6275285632, 92668.94411829728, 41049.107047343794, 97147.08539059997, 62780.14801113257, 61772.2625385896, 33450.375379320074, 93890.43786643536, 77976.39164960345, 81110.45493806092, 90433.31105776723, 18194.25188726067, 87868.69316179221, 73310.87070708907, 84597.1280756877, 25804.258111121868, 65148.903084644204]} +{"id": 1996, "vector": [73322.13899942895, 68165.81499567836, 98038.76445273147, 5055.5274944912635, 16916.088043119904, 13404.86243936726, 87612.38623928522, 19923.877275426814, 47766.394102453916, 41704.95231419671, 23575.250317799633, 19342.630994638144, 27983.306367317684, 38327.6236725001, 94774.27188390777, 93480.4131043918, 29656.88403787985, 43600.696115858176, 35359.983490295686, 93985.39345089627, 84990.99212140613, 56281.96494479814, 59738.35121997346, 94.15982185849714, 95458.16137088723, 77501.51982437518, 95678.55815610135, 86907.54441328862, 98410.32494560743, 38561.360248191224, 28411.435167668187, 39014.765509302684, 40488.233316013, 36589.91955060693, 6450.737106239501, 74498.82232063139, 99929.57698378715, 26311.536080134534, 52110.03755597092, 86189.09147740382, 34748.920907034306, 43111.21128674377, 23758.481010078693, 11563.005069537568, 78216.22964050501, 64737.90613890328, 64805.29167890562, 40388.47679543156, 85488.17889684843, 18378.63824892163, 16315.365487506439, 3659.1442994861277, 69242.44320759934, 16556.29160484994, 29996.099091735563, 10155.065764125271, 22401.422320060126, 58090.26969415776, 83907.2666420566, 78622.41069586707, 78961.91749392539, 6242.001313488588, 79957.5825078334, 14020.709081800076, 25316.60297834707, 83698.74271541458, 42820.54177018782, 84426.07097708677, 83235.38846930665, 93881.14752195425, 97820.56700039239, 62781.578767255356, 5323.435234143548, 3177.670670690036, 40434.446672869286, 18357.736593011996, 36626.11275299839, 26820.23150896451, 94000.20975049559, 19115.677892183216, 23645.252156931772, 76243.79880025818, 16800.833045618445, 19632.58845151581, 50362.13724998884, 55148.30387209951, 30729.53011268924, 52787.51210565128, 4589.342248907069, 68071.01865191558, 9518.477096192313, 81823.85123657093, 71456.9200340294, 75427.7602114593, 36639.6445904927, 37893.77998224602, 94160.82457160627, 39514.90494923655, 35342.58621516475, 49381.773383594176, 49297.72802513346, 74653.648479463, 95517.23127401136, 82909.15949239217, 30289.333572146636, 44615.71399525516, 26385.34426347572, 51143.366621331385, 31081.97812422696, 15308.464581317992, 51956.08333822466, 20123.49343516253, 93834.96463402457, 69707.78668984029, 99486.1903322636, 86743.53224168957, 61453.110927368725, 97692.7722414324, 94262.88645908693, 16516.525236754576, 61070.245146030065, 74275.42213049867, 35788.09047099602, 65999.81430809686, 21207.763125459376, 68346.19701065782, 21552.053862322562, 37931.59537178673]} +{"id": 209, "vector": [81234.70426436423, 74415.91434586147, 9106.904595529797, 50807.73050715377, 40431.41121498127, 1151.801856741419, 45601.600805498485, 95950.49275388243, 81267.63070009612, 50895.55939910616, 33748.75879717353, 37043.81474204651, 70624.13077088931, 12299.7604999176, 75309.75455474784, 212.87126653403155, 55886.297552078504, 89056.52404931227, 31915.142124739017, 81246.81542190918, 90888.50681266814, 15584.145367707792, 13794.248832028155, 21361.075323819477, 14035.89670156513, 70590.09509121985, 3009.9416903758724, 74948.73950364126, 24478.124496370358, 53982.35617145267, 2935.979637955077, 53558.71344955474, 94402.34113336435, 84584.17918270666, 95046.09821834283, 96109.85247529324, 44716.69835472397, 82267.3537105228, 52302.859877024275, 96785.58495827888, 18579.798613823838, 76298.84766038688, 89141.23225243115, 88472.11358502794, 42520.937995531116, 69161.8447193021, 63004.215365785545, 6896.254095528109, 57111.0545748034, 64387.94020797088, 34343.39586704736, 80368.53572588242, 6541.888357037562, 21222.614276543085, 33471.755671591796, 25885.69500143061, 13594.48768211582, 53480.2362590546, 31718.39371087153, 38711.71594985134, 70072.23555376448, 26658.436047395728, 85452.03142551269, 13343.341326613867, 35173.6766278402, 94483.4572296792, 54345.01402563733, 97684.2956045878, 55771.97543307176, 66172.1064902696, 78744.54119761387, 72892.43850313438, 39086.5802484342, 80403.84537651192, 13932.26256368294, 2578.2757756374617, 72381.79377769494, 12748.319297143362, 41086.61433635726, 7663.650893854346, 38881.28936640674, 92273.55278180841, 21888.190601728675, 71011.15169437033, 42388.28036446623, 60894.79210849405, 50288.94403901691, 16448.59556746774, 49447.536118138865, 84490.09166770495, 93957.8865591809, 87537.81132244956, 77832.70164554106, 47601.23675395299, 37353.58277637931, 60845.765563872024, 15774.774822432291, 47917.256357250815, 62427.37543912682, 34589.277535612484, 27036.24885079806, 17748.898212633525, 28202.818899106896, 8349.035542428963, 13175.077874819795, 15507.100541145213, 7874.1855309704015, 79340.24961065911, 27546.463560137523, 39397.74355498481, 84045.525396297, 98395.20232373575, 95417.11489803554, 52616.633707043635, 23222.990144279654, 97712.42569132258, 22771.545762968126, 47851.650228880106, 10461.666307790107, 81288.6390343039, 48651.405824976726, 7073.13373044951, 27125.381196627506, 59044.051368023946, 99263.7803014818, 56524.24864132582, 53294.46998305943, 72039.56758594321]} +{"id": 145, "vector": [52187.80993532055, 21090.90696356579, 64302.743151090835, 615.5733255605145, 25282.470292734506, 58552.068575879246, 30656.65875380177, 39749.8926338239, 16844.860611288226, 25533.993175731073, 60180.9817709779, 76749.37291001738, 10520.897117655803, 781.4632717616621, 25288.63403672569, 55485.54924398618, 70604.2882979632, 12242.71032769183, 93828.80144756736, 24267.544453560142, 68499.00338780347, 52237.62371647244, 12254.715816272099, 81130.48880037534, 71589.79501310013, 85788.40223526322, 57993.59884262258, 1766.6544291892405, 4763.114006908498, 83901.89149922582, 21869.26365946501, 55858.16938773651, 61096.75262682308, 93278.1237623912, 43471.13038115005, 48265.7542922548, 75805.3460178292, 72593.43778748334, 33843.02481092335, 97268.73309617226, 10084.238321426654, 96313.48416388294, 80222.25576525621, 46886.59823479678, 64395.533074273706, 3897.547997510531, 64745.44341039791, 63363.28420668609, 93157.73476406033, 31023.721361270418, 56287.089646372166, 86098.50564531291, 22449.938039286266, 58448.576370983275, 61425.52219836519, 35332.42629720962, 72215.04963939366, 87714.55451477435, 87924.31900026549, 24412.1067984149, 98098.06644395803, 56126.102020239865, 50649.96198631936, 14040.318142236485, 12394.893568908983, 37425.796838507384, 56453.86408379413, 37795.004301292945, 14301.239912776387, 81344.05327046984, 75877.11812771921, 47608.51627598917, 94272.20135844967, 37643.284825639654, 98031.94760538149, 70076.85425968573, 90036.7458409582, 13680.71184155326, 11148.246085636903, 57028.82651737874, 75702.84668349329, 17053.555701254474, 41288.56865739045, 36149.5719950342, 40842.81146026917, 12434.711992366132, 65265.120234404625, 86371.15854025244, 84832.77869820944, 8532.393044907616, 16793.576113208917, 95551.46081154144, 13188.870825625398, 84245.30377506428, 26842.374180606486, 7840.866703180849, 42755.16781679907, 63735.957083001835, 82325.63860465534, 8115.486804707805, 73414.90925029168, 21261.792584810733, 70062.0850754999, 13897.322346429186, 20089.18861319048, 55259.75482942204, 75012.24940885621, 1205.866940829048, 34231.70220421287, 98853.97994656615, 73642.56110247066, 59555.03251573092, 49961.68472839095, 27892.58469883189, 43178.37568175196, 82905.29331466091, 5135.981573195137, 89127.80861553623, 51311.65468892461, 51637.584701541564, 63499.38972452127, 49199.46269586307, 57888.334012470375, 85564.47720329491, 49730.14898869508, 14996.659307208416, 11295.68648971141, 6347.90768763267]} +{"id": 56, "vector": [25772.5774436112, 56678.44921796455, 58346.10542202921, 14915.149125039728, 63502.7994363353, 14551.341367706173, 59908.68670784786, 25931.082876302415, 1380.6974991989573, 63404.64632385517, 24792.991492609162, 86141.8962861094, 88988.04018810415, 37186.673245341604, 82533.02729616532, 1239.9959191227495, 42529.173982172644, 25484.90756196573, 42248.05135345477, 17369.608682760674, 35146.22671229484, 39442.626896567024, 18546.823539527035, 48147.064796670005, 99967.02813058914, 36730.17829860376, 81856.76698301228, 32582.74522538396, 59516.52967958189, 32920.59153859345, 75452.04461216087, 43279.564064450795, 16024.25987647833, 22860.34390459646, 13077.391268466465, 77118.14009428414, 24749.499217426695, 94159.94710879782, 59151.53186794576, 30506.22888298259, 70108.05905860044, 25942.045426628047, 10556.977634674824, 10440.21389545754, 64704.1241352773, 15590.398107466719, 63872.63604475152, 76353.46840214894, 53744.436390371084, 28947.621541368786, 15854.137435476134, 22226.51958872316, 1891.5435782687396, 19927.395520280545, 63616.47504857345, 1836.0705142415723, 57004.848878560246, 77746.02875511129, 94445.38781343983, 9020.174751657662, 2105.258904417573, 34636.05680207116, 78785.96471592668, 22282.030446066605, 65834.6813227994, 7828.225054745352, 77304.16432912216, 87960.00973571418, 52430.371922519436, 54461.34464215166, 18764.49892613029, 71725.60840115538, 52690.9487431256, 60378.78157759603, 99140.3691270424, 91791.96822180982, 72521.24383219097, 40323.806430884346, 50510.418042272, 93341.40386457666, 78258.87935641668, 30769.045371472093, 4800.067275413933, 23741.192485428575, 25810.93636558185, 84928.38202710003, 62140.67351376872, 8703.103570905236, 7020.31332715699, 80093.97812325387, 46909.470096060235, 2535.913421700731, 91860.10078463597, 4043.0021724600306, 38122.46347127756, 53374.80479289551, 70708.60137649311, 69912.59919255781, 75307.54343122647, 67513.1582434261, 9366.613377026133, 54133.62851719945, 10499.188520286518, 89086.151404733, 34441.21834587073, 35231.550961675675, 20109.670263115364, 60624.0740788068, 12854.89646458421, 68693.92214915235, 54645.68695103768, 1108.7460872661547, 19363.24367384964, 71531.0124021659, 5432.1478855617115, 53533.7377353899, 82060.87750399484, 58849.210800971145, 66523.20247308891, 56677.369913413575, 86882.12367420603, 23692.848274731514, 22038.52940869755, 79442.16335670919, 28696.92018144897, 56562.851561917836, 82212.28516854884, 94801.6758353349]} +{"id": 463, "vector": [10350.683758296009, 90634.21780358593, 59141.5454542529, 72383.39869777538, 65250.98210560855, 83917.40834144647, 58916.46819590567, 55969.84870336319, 63362.41024492068, 15166.377673680232, 80857.1380155647, 37135.753158456566, 9724.873879286522, 51024.769521341885, 48324.912009443564, 32170.93529563647, 70935.4783233682, 41955.093558943125, 80884.98926826894, 80413.89123014688, 79639.93993907733, 88245.82940139757, 6064.961045999217, 38655.0863311103, 51145.84951374621, 98583.35234711258, 85190.113044607, 43139.31766656061, 44339.547895052136, 89861.5116973441, 40373.6759764609, 1228.206271524601, 44402.399188441486, 79607.45031087109, 98628.49079871054, 51721.672978504204, 4299.176191876708, 19425.04381131227, 62909.21277068309, 97606.85082512574, 59375.872547270745, 43620.887761645456, 81650.3650362513, 89622.46068538324, 98984.74277754, 34786.81219242458, 45000.772195956684, 98635.82163008963, 31799.388581085474, 72787.16780988162, 46752.603776627046, 50233.042888006006, 94837.19822545456, 73452.27631255572, 47236.293099188886, 21151.42479894424, 779.4301866186237, 49087.556177019665, 4362.57331341835, 15967.208550627021, 46836.16124287347, 16252.086352346983, 28360.1616365868, 45642.53835635461, 7401.422093896137, 13751.579802043678, 79458.62582826836, 55803.70066616012, 23565.864651872005, 25987.855128084015, 46229.61891443962, 3922.1004427217986, 53110.62938750627, 95520.62304861416, 57813.89136338805, 72946.81392331458, 66974.81638508102, 18836.50564015237, 76611.6082758976, 79327.8150709982, 40555.42336322882, 31191.536137599585, 33246.57497969109, 37118.21186094112, 20861.744341807633, 25551.526135229662, 34591.311290416204, 42848.284727446764, 13966.39764850881, 50141.71760921901, 63734.42814219142, 94327.41809378217, 55091.952165234026, 34913.86697008927, 14324.63450722109, 20322.51302211754, 35506.92317411772, 19205.043754346916, 30683.190215721734, 36183.68303716721, 63708.197888319686, 98373.37824895837, 94348.27299482726, 57702.82079814647, 57709.88202661375, 73194.01958170837, 81779.72332159882, 35178.80475445064, 61731.67455641926, 22554.16241530994, 35337.773106801666, 31431.592717466905, 71942.96138511268, 47870.375427882114, 50940.04674340824, 30814.27628994947, 2400.973292510511, 11505.647338351155, 20847.824570153018, 92443.85893386668, 36257.476897292974, 96664.28048780134, 69305.1599694911, 60510.16421074083, 66142.06247399967, 53277.404631601064, 54558.70468037518, 85390.79798323146]} +{"id": 220, "vector": [84164.36908230344, 66056.74342840955, 690.7449521104447, 36187.827846243104, 22956.27930561671, 32.76800258894541, 39023.40571441632, 50339.804756549536, 37501.14467168897, 7207.501239754543, 89413.12774205422, 6956.384925656367, 47372.95121214073, 94753.78424309414, 92794.810149443, 45742.419834817905, 61318.764572120876, 11618.534756819465, 33789.92061860119, 82709.95652648062, 34510.019631734736, 2491.4916337940076, 75138.63368470865, 29307.55562286643, 69428.21407896018, 63103.7297818553, 68284.3160294104, 44632.68650418242, 38374.16828272, 26557.49799336988, 54923.23443588114, 81762.60946613249, 34944.004771251246, 49516.8340375173, 99178.94968316387, 98920.82591391608, 28184.14088634531, 89511.64849910983, 95083.22493721242, 23808.830383158453, 3280.462761360381, 6088.707250758019, 60812.64565013671, 45282.24399320442, 69491.77179023308, 96517.76072787712, 45631.213059953414, 47711.375437176364, 14578.08680404159, 1938.3015938489789, 97206.3951176967, 26094.203246761084, 24542.898727470718, 25649.626370860846, 35039.482698417676, 33039.49775440659, 48838.70525535068, 81383.30928463998, 39444.50125528103, 68621.38628091439, 57809.477677046365, 87828.07428464227, 93001.02562795067, 91631.6068667069, 25508.928154022426, 57036.08036339187, 71911.44836949566, 36952.43793766141, 54564.32517465584, 71734.88663567624, 96302.46558116526, 61045.2544433065, 42344.212028268244, 23550.623755210785, 87479.9496065505, 94553.87183850535, 7325.162285254371, 54788.08306397549, 67199.41013976435, 56056.453186232444, 93939.42532445953, 62030.82471307398, 93589.24142590926, 36848.24675914231, 10818.781851431559, 91633.91070891068, 76492.42903597807, 42728.05914010779, 53137.57071846423, 80039.29219940292, 96074.78083095, 27284.809504699693, 51911.312599280056, 34496.857040210845, 13613.535953581168, 93841.29704563784, 58771.47257577485, 16626.80950923884, 97612.64790843376, 48560.0518216948, 38631.15945003234, 37113.52570891995, 59939.01309261939, 98867.87769688864, 10426.925119012465, 82896.24929881578, 98384.48617304697, 18815.559880006203, 61989.632736150146, 7551.825723108185, 95151.31994214116, 287.6077132089816, 46444.67785136953, 72068.9068484014, 56426.45620142859, 44349.63646966422, 57455.3364435884, 77000.1614626908, 14823.50935021779, 71284.33433490366, 84051.48405392795, 14789.621072458714, 36869.31401351719, 5628.326090688185, 21147.68805232987, 46736.60289220705, 92880.43614488243, 29287.70870123918]} +{"id": 1731, "vector": [35956.235135835544, 43940.61555456666, 60736.04381931404, 12181.033172385236, 51922.00272985052, 24226.452011888443, 13313.632668128927, 31031.72811407329, 14028.230494188321, 87245.2997437268, 46565.293451285805, 67963.69285618467, 58721.43749176271, 48383.334660973276, 79684.406957312, 42426.99488177769, 37723.46766853615, 84860.29038698529, 68699.00975222308, 18556.112759382493, 24195.227676162933, 23670.308243244497, 29431.269929244132, 88033.7647416802, 56681.31965545779, 3691.8521434299746, 68522.20553479952, 15450.995714254468, 39700.19587182162, 62847.87879258272, 36448.643478438695, 56936.55346531315, 84323.22553443321, 48321.97888221602, 3478.38245082156, 53569.30975228752, 26967.05458072849, 58330.603982962515, 94598.94436945337, 45045.02226220625, 50004.89917009891, 85649.62793297949, 25976.778289196256, 62205.09176266591, 95234.05515884553, 22847.72121376841, 85412.61072016311, 82053.86541448416, 97401.409566586, 22151.007360749507, 6206.704542476759, 51544.14020875322, 71510.62596394429, 71673.21130885696, 77784.04660488295, 29590.30061729533, 81265.30379965561, 25067.469759354877, 48319.23719885679, 36039.366094270066, 52893.08609033838, 53429.85202980632, 64745.65322789274, 19063.104409457443, 50407.19815334067, 91608.54661496668, 50624.12400959373, 61900.83622779785, 6100.375944464087, 20548.681294792736, 69780.84488652344, 598.6156810147025, 95855.66237503152, 50278.200871338166, 74253.44689627417, 20309.240992035626, 57960.94156556386, 73115.86382659416, 22770.961633634968, 90290.6481024293, 40217.312267794536, 61080.55374454463, 70634.32577709832, 2666.6960245382065, 8062.6271713268525, 12091.190763787008, 48995.227350590154, 3679.8085042696216, 84207.60303742411, 56537.62272615167, 75222.18275169843, 44982.33022048309, 61318.292060177206, 92870.38346392852, 67317.9317661283, 3421.935073014781, 67288.75059094733, 54963.06622936828, 79369.0134910734, 98723.01507029621, 52392.051261566, 71304.81813138654, 84717.53006454435, 19049.011415390138, 46985.57015243683, 36948.58192289021, 99087.47791902468, 52763.77154042101, 62730.45695348084, 56310.776186885036, 50527.01957003057, 57127.84805127146, 2100.946783926105, 77806.27574656841, 72273.2364914955, 53207.10134687261, 93732.47134246744, 6994.2879143781165, 15761.557392203851, 55574.20065601295, 38597.09598344745, 35013.547289166025, 62366.198109285135, 6730.2391576083755, 45672.747932736565, 85373.73716759212, 57721.8905939159, 93249.633380978]} +{"id": 373, "vector": [29118.16961442524, 83428.58761746994, 67002.51245201637, 17927.618847378435, 94069.95448848697, 59009.065193669965, 701.164474761129, 51038.98090728043, 43160.53944708933, 45173.14397227585, 88383.20626551942, 43835.855338700414, 67309.60621375235, 57734.61777596785, 54181.77251776047, 93287.90413026774, 1336.1715501847505, 31901.613363112803, 14229.131769885551, 11926.047751606062, 65446.189287881265, 44354.05545012099, 46482.922749666766, 55249.63501325322, 90020.7649035581, 99244.77008117548, 78093.1307684078, 88926.03913032457, 30040.98066831369, 19831.6589112575, 84228.94252507201, 91747.24871710582, 74580.22288989503, 1670.1147553053852, 46049.705011609956, 38990.27841187256, 89615.79983815303, 80661.09300140908, 87184.173393582, 61267.3289574687, 31123.93748722879, 96.36269721257085, 17909.028316626485, 42169.22091840195, 65648.9559841492, 63336.068317041, 88179.51666468423, 75647.6982945589, 9125.181228307667, 74678.67626474067, 96588.12047801692, 35002.161756681715, 23354.146859901524, 1810.8143722688096, 68786.31253992156, 51881.88412063773, 86580.43732873302, 30007.37940969994, 401.9636346355715, 15911.162257769307, 94065.01342264442, 48492.394449215826, 82937.99800629854, 35092.2097434284, 92225.90358923686, 73240.84996015663, 45844.430569311604, 75060.4493111523, 56413.64597940952, 94397.94796072875, 50543.16108277582, 10969.70730584802, 48408.63911898841, 86282.00159837496, 11880.145578106827, 24960.60426578459, 14497.487326236336, 90548.40927797026, 10146.788856476263, 54837.10763611448, 7437.317233075713, 63565.44315971115, 91468.739748089, 9261.695993264186, 96222.85181074367, 26871.77097317772, 93334.50414389763, 90139.01805805387, 22269.27707099644, 39393.44696614049, 68614.42979805739, 77559.50934741454, 12099.780267442273, 29930.454434271523, 40838.13432145988, 14443.165993867224, 57633.741900562454, 70470.58459184977, 1038.5213399279446, 91181.35931447575, 61956.37174018804, 2982.1689538390037, 63604.02751573495, 64971.02303419134, 43994.66604053154, 46275.75663635842, 81612.38460684135, 24340.523699103054, 50624.2428632986, 44518.579171064084, 12477.283553324303, 58445.19397273623, 70910.3682846243, 49881.59036424876, 36935.84369193857, 81189.75647206591, 54109.61792197334, 96007.81487246913, 48843.75550009118, 50890.89030439002, 825.3136113302029, 31969.200246735763, 9879.0143152502, 10680.053615256767, 31371.34689290071, 8476.970094900915, 53846.93864412984, 66915.38429019919]} +{"id": 1729, "vector": [26578.138716902777, 80513.29936659485, 7397.515666183852, 10762.767012790198, 75047.9032156375, 42654.25164397447, 44787.59650076947, 25582.466700986504, 45799.24663128406, 44351.33800043012, 72769.329781446, 93196.88522689344, 33150.914585947525, 47877.44362903795, 96766.8324325102, 52342.809021467176, 7271.32582653306, 889.0330816257564, 91354.87799062562, 21879.168476114854, 19753.75336409788, 80980.50126300675, 88312.2512809601, 91552.3422710684, 31490.629287691518, 4194.346150314099, 22386.954342559096, 18124.21237795896, 98206.34795727384, 12633.56825988019, 29122.654209321485, 77905.1234456719, 10061.577149854083, 16678.34723538626, 57466.96973242983, 49754.82044822559, 64078.76634947937, 2269.443844780561, 24987.387439073784, 18065.070874626865, 97217.72799060919, 58342.6825284205, 75592.35436391554, 24896.473263301632, 52634.16661629031, 79230.87298044347, 92714.32087165072, 63953.9163218359, 4005.103462222559, 54028.01420681403, 9268.319299225735, 99680.35195480603, 37621.40799923721, 72793.61149441244, 62856.07382321337, 36804.36386928002, 64761.02744829598, 22242.31602528305, 86403.32237318797, 85246.59707928282, 23717.474126194127, 87275.91500595027, 27713.126735693673, 51873.315917921595, 99458.28079871584, 64762.52986990105, 85299.18677045824, 61930.60561144418, 26236.05507536828, 42594.9624317565, 97590.53708929202, 36689.75751795503, 96289.06869185735, 87963.18627624703, 41930.39819228194, 94926.46475513009, 12698.667629904847, 13742.379269458648, 23013.347871140733, 39231.951894026264, 25858.361536122844, 92646.54786232043, 78644.35382436887, 51729.894277101885, 2791.204445298434, 86146.72172669602, 4642.996309999991, 89426.9897512651, 24209.398813723026, 68419.04999688503, 94236.37027012931, 75183.85273450948, 3738.317015950998, 70744.20383063694, 24625.92359804722, 6493.572804378545, 21915.080165827207, 71848.30416058553, 43057.6800363662, 25966.858475723566, 92344.70315249666, 98185.7199380973, 24047.177238016404, 81377.22883959382, 75883.69451396404, 72804.2521528242, 49215.709786310166, 1391.2314613135025, 32428.99264359105, 45184.12726810146, 94289.27730560175, 74985.58898943664, 57468.68249401514, 46391.28231726656, 61345.61712561025, 99093.9062945486, 20579.223804007674, 45133.86804383016, 78286.71939153754, 50331.149295133946, 52483.34380889056, 63177.50225015773, 34937.85215016335, 3781.0344375321715, 96442.3586303011, 55577.894812703045, 82814.9815980897, 26096.216531199378]} +{"id": 1760, "vector": [95652.98321596542, 77006.79162986626, 92855.41540589326, 26803.03547011208, 87889.26156823778, 67851.19079222469, 46202.099282006835, 38869.50038204313, 1100.6798503771731, 83764.0540517673, 16617.885015193657, 2270.015513112955, 21799.24241263472, 78956.46437815763, 10768.480350212361, 31144.14686393989, 83857.45621706669, 42798.50929581025, 19357.84695603584, 30384.303009647807, 75885.96590725546, 54853.70963365803, 96471.83138373945, 82498.19345754287, 61966.12159280983, 25780.862944227745, 59041.9265151489, 31938.99041261625, 72540.59791191822, 31479.070111842833, 91410.11473019636, 53368.249304294135, 85475.13834181853, 60678.17513509776, 89265.451831662, 84014.68880903986, 87624.22354247641, 70810.03576210674, 62536.13956477625, 41345.43906534026, 60516.20249696684, 81350.65680126424, 74676.86920360278, 43248.64004752022, 15803.94823764516, 11093.250434211777, 47341.82803588623, 93034.7684111017, 85365.45731503097, 85392.68740000678, 69867.92035248122, 11490.254756082375, 85026.04802375023, 58364.04934761173, 17886.599769948465, 18942.284829344557, 20153.67413942485, 52285.904510876404, 63113.10537642987, 84506.86787984776, 66747.51392914145, 64335.19253766089, 82732.37433597242, 4834.6472895296365, 27604.841975927895, 42449.92566096359, 7263.714816328448, 95636.28800523252, 26817.16948323286, 49539.023276042746, 98329.61534799178, 55637.578307064075, 22265.011188650085, 99888.14064217648, 65207.20599224098, 51965.68474805671, 12267.102916448346, 26854.99202221967, 54044.73335657272, 61541.181532432835, 12199.66411216662, 654.453862816906, 69230.90396897559, 89240.18745128933, 84539.65592638259, 93811.29195504374, 36210.52287804002, 27963.290147269738, 53665.04752067514, 82014.12649640608, 96113.42796518003, 98941.13712136698, 49888.36202243445, 43119.26910658963, 60196.57743101634, 67045.66232149988, 46744.37934965076, 45708.53552094961, 85694.08348507316, 28706.005165438575, 65593.55690774463, 84422.77187597186, 38674.37038043906, 60129.56680903176, 88465.39544416609, 74133.51756802856, 19071.00642770899, 55056.752584153415, 77107.27779852747, 53877.21094402519, 14260.896577205851, 14336.323056218802, 80536.2101621562, 35952.87674203774, 59219.33105656816, 79672.41119708093, 9811.745383439018, 6680.615533318879, 95923.2039041639, 38006.426956538286, 876.0704102685369, 35871.51999602304, 86897.37409629197, 63625.36973014851, 70000.20517388465, 74848.88130011388, 2639.493382417146, 681.4264268199199]} +{"id": 258, "vector": [44620.176191612634, 10605.813863989644, 77946.54732957478, 57364.431334193745, 73581.05962021965, 58053.14654029916, 34343.2707463774, 58082.56249475755, 47121.33679097793, 33593.00201263458, 53829.003627005935, 83165.19722642259, 79447.22932594176, 73480.82400424298, 78999.18577785655, 60178.62313352214, 58473.9268806086, 50953.574728889595, 84096.85778438959, 71529.42285223841, 56357.75009979901, 48324.14731127562, 41304.38354768177, 85590.15879033845, 5796.633513349747, 5314.099467893918, 30566.46959425844, 11635.194194919774, 75276.94019875309, 4454.939929587931, 73403.09760996858, 18890.12262975206, 1641.6178529328085, 18104.684196978928, 42616.32098850396, 88802.31447744896, 87192.37460886173, 88282.2095201801, 85261.51944466507, 42826.43269633392, 55981.31442829563, 11822.30856918155, 18685.955849718182, 40855.35089363769, 87691.95279026212, 11287.441498564565, 66509.56807550784, 66981.09753480312, 81246.69722204347, 42220.19523220685, 86130.08152903055, 91816.64549302495, 89326.6040795162, 24515.59285717818, 31955.30459276803, 32397.963833551756, 64752.748885125155, 31765.308919126546, 19512.24423600305, 38337.52793684011, 6781.794696417532, 50210.70702362136, 69320.96109684564, 87416.08920118945, 54746.01003140685, 56467.590863036785, 15063.530123309089, 49733.86531493491, 11475.20834267446, 32576.84156303823, 92918.58252567897, 76752.88996107788, 78090.38261166002, 24164.507256734345, 97666.13519566158, 81273.30704271236, 56932.235452898974, 19710.368375902588, 76166.56040804173, 57484.97329721536, 41870.07451504622, 37323.45362742591, 32698.805322105418, 58500.20964743764, 71264.55386292622, 9477.925810898525, 50735.85014916826, 8253.134240472027, 33657.88126063267, 90593.95608361563, 82629.96677073845, 34058.993424629545, 51121.53512321468, 75682.72204424374, 90683.51748131232, 71987.64866339604, 94258.65245134672, 68586.37514685601, 64107.741363533125, 67647.98218741964, 10051.269466165657, 60804.81649483448, 13438.477301402907, 55182.95505114166, 9312.262252487491, 77316.30712048747, 74505.6385850407, 36021.82632846253, 91996.42810249566, 32783.80220740861, 42082.98573089354, 116.35452324216811, 44010.75257555601, 1710.155931977575, 50112.81657119022, 22062.35796957937, 75481.85732947962, 89238.29151409521, 72367.0430969245, 51117.42567561463, 57611.03139744091, 66351.46081437473, 47995.08053662449, 75966.63013616983, 19198.020599365507, 15908.65376677445, 82944.98254237177, 92055.0982082324]} +{"id": 1747, "vector": [35846.73528639307, 10092.297566998253, 56288.306538220124, 21428.26809344507, 58143.59941744525, 54720.31912452171, 14999.45162673758, 26507.523861720307, 88974.51512131165, 18855.875570837954, 3928.4650020032454, 82932.3276641598, 58521.748588070564, 48702.533780109705, 95706.84763890397, 90051.63913810125, 94454.94941772602, 16596.00038412974, 2175.5146837499906, 65984.75261931209, 12958.38338427211, 96842.4436604511, 87441.65676194741, 74866.31376752045, 7520.816258690899, 35747.46484692743, 30777.61304959601, 47734.12967303045, 45714.893566657054, 82857.23920477845, 81793.27723415304, 28124.18144710046, 3691.4625827744853, 87858.34315402916, 40646.11539988566, 16999.878291738623, 92374.31898039584, 78864.74401441518, 47820.32148479689, 66061.13174629236, 65706.71692591962, 42247.73401572711, 93795.84987728028, 11022.726073125767, 96197.31292801452, 15076.640204897263, 4071.5753525545574, 34882.339649032736, 31845.19853320842, 69444.30969747638, 60261.40281431444, 58605.4680423823, 70095.02501216017, 64762.11855923586, 36930.10080999448, 43445.172393918605, 72492.90127317251, 84302.75607017266, 47254.16446223585, 30581.736557776283, 71258.56384038129, 42251.32350728361, 38964.98445513617, 87109.53089968594, 99133.61901305946, 1312.9665886218665, 42874.96787593156, 21949.389161017683, 37569.8766691801, 64944.58544143742, 36646.90903371754, 8843.778386106016, 35881.017800244575, 56711.84534316543, 33108.637341903544, 86935.47284840664, 22099.59037779201, 19642.42705090775, 32395.83412374164, 85988.28696353921, 39934.87520812646, 96245.0787875388, 42522.73250847893, 23751.323421964986, 42478.95680848217, 8963.821078933797, 70153.78940974032, 13571.879794468445, 3809.8008114568984, 56.66596814090097, 22161.096295961357, 59742.42400791322, 63693.76546716889, 607.9547904443872, 71311.66028821794, 14439.037668588584, 37056.760868358375, 70691.01529557163, 18892.098519819378, 4487.894648713953, 99484.50686043312, 70992.43049797011, 27574.819688752395, 12689.274495557624, 59305.893709476564, 35475.495870071994, 41511.08407622462, 6837.705655529247, 33384.89834903193, 58978.451786567886, 72662.70526673793, 30970.77042040477, 64413.11912817437, 78575.57378518651, 94788.72422568462, 33851.95105467817, 72424.67179962786, 34902.53407656184, 87451.82658590529, 11821.3858118835, 84257.80124697469, 82849.75648248412, 49673.36294721667, 52224.51974373791, 28988.360944173906, 44720.75545234897, 19149.686234308083, 8446.714370949237]} +{"id": 1248, "vector": [88315.87305036721, 55600.63627779156, 51205.96746535751, 22584.444302886463, 56331.81884873507, 46239.58074424403, 94763.48045978941, 79502.82496997659, 94997.65147493832, 49232.269855853425, 37856.776873747, 12625.067525850087, 17528.367340654284, 68866.26338693946, 37730.24863137842, 69051.34116762801, 13949.90555165201, 39278.072505086704, 35755.00391240846, 89431.73808031555, 29177.694819622368, 52302.11605659403, 9786.021168825255, 49462.39314468326, 5537.1821307878545, 63036.63630738161, 68899.91392372551, 58539.78771262691, 9100.499592348011, 15493.475665983047, 38241.792037541425, 28192.604845492242, 98406.99176245314, 33503.55863601998, 85168.86782469485, 41442.80374106335, 36860.38375471834, 27699.64125949862, 17762.39168819925, 11335.619142034737, 42963.2325695244, 9201.00176245857, 20222.981915621353, 89061.08980446638, 11866.783407603054, 25585.651027007138, 23185.62702924997, 8907.208577865822, 72535.10814597824, 47846.773627042174, 29532.04934387721, 17495.54557268783, 14406.936431452788, 74519.96998470083, 4090.9231580657847, 14196.917004042974, 57860.55308700201, 2109.725142678953, 2278.172282441704, 73360.76729818656, 94223.12041064075, 91046.58002250786, 94379.99459860468, 50913.648500040996, 8060.51558683979, 16738.148660385134, 43627.59810366998, 3714.1348188279853, 78512.42050280876, 24684.108206255063, 15153.178408233736, 50367.50801091872, 73189.41699736095, 29052.759076025304, 31827.654628785862, 38410.63100255202, 76321.06569534341, 74772.98872820035, 33200.13774575852, 40742.962050780385, 17920.812421488074, 32920.82793185236, 37975.92481191338, 28645.38460169517, 47829.563075981365, 89481.4539748892, 56619.48547868435, 91209.84615155257, 8740.259096141523, 41956.881762162746, 46731.67053005776, 94813.86322122015, 85647.02173461528, 71193.88824296586, 20847.665589138887, 81010.88740001523, 61546.41382582136, 29210.87182440857, 47088.998798447625, 43668.15685199873, 73640.45944441571, 73185.7393480668, 93711.00119239648, 48939.761110427506, 76715.58929530231, 70480.18858404209, 86932.06736343454, 74071.36091391668, 44436.99886447356, 84815.4867091826, 72087.7720033392, 58677.05295252881, 43017.79260141093, 46641.22782449103, 5889.967488876679, 54341.77844808774, 29553.820431409706, 42322.20499647965, 85615.07369888958, 84120.75452884432, 26261.181937523557, 71323.93136770632, 34033.85107783917, 81751.70142223466, 43110.43013126036, 11500.49063551457, 16628.03664329664, 4227.441594568926]} +{"id": 431, "vector": [4208.392652384685, 12465.237887363544, 91238.31045894667, 16265.95531505538, 12852.395865637822, 4131.397918121593, 61060.00575771275, 33905.32102095533, 78442.35901643553, 70820.70039424596, 46878.490414049134, 40630.38722027371, 16648.963232636306, 50974.989376670266, 13144.493413745884, 43649.73693043135, 19697.87767076544, 21287.779812244924, 32394.744456925895, 91020.75461951163, 78690.44647489088, 74734.57344371542, 35791.942837279435, 60335.663818785404, 73574.62819206933, 15984.022689438405, 49561.20219326888, 30614.14982835804, 43464.58675884044, 99228.7952537888, 37038.02782705806, 3544.2629392265167, 92934.84464880815, 45970.66073253631, 19094.986711928475, 43336.331597945464, 75393.68862647029, 3553.8964763068725, 92770.7138482325, 99514.92711917256, 96328.2446629813, 64948.840609007755, 19603.838003223616, 16612.592748107192, 99443.30462855862, 3459.994487713958, 61624.44141944502, 28910.515654301773, 75431.95599025048, 24892.092409442335, 33873.44969438226, 47274.29852595992, 70679.82934667586, 54739.22590062777, 31209.254428354172, 28475.372322833802, 26281.9616238515, 69113.28036047483, 41241.82286502521, 24009.371120861255, 96118.33594382889, 67397.43444963443, 90886.3632865267, 97194.98073470694, 4123.807609287044, 99788.70124679821, 15871.24464192171, 72211.68238170432, 75528.7830251648, 20683.821711566387, 60560.5152587806, 86758.1145855541, 52203.130383062904, 51370.53145953952, 31482.033975521972, 29055.00721561157, 58980.497996349746, 20117.34280613685, 89504.5285199524, 27941.0961348874, 73382.1833151689, 85091.21245721605, 31427.674834312656, 24038.20001342921, 50168.090150519296, 26750.053354999414, 50930.809796854846, 13602.960979878553, 80570.47968574686, 5172.44620913736, 70797.05442450261, 11776.24786133794, 67381.83951341879, 92414.19406058377, 13542.989267212713, 26221.80497161174, 61711.12018220256, 94492.11856528945, 61406.776231076255, 69982.22532511945, 32712.407092193895, 46187.68247166006, 441.95124547875554, 85898.22850409578, 54735.94508173056, 53223.84952939283, 16169.19241540874, 51228.171157989185, 82649.72929578339, 31447.050953990973, 69562.53262615741, 60064.47447604405, 67718.37407776616, 92174.40216289925, 48530.850259089275, 93947.29080896552, 45439.86203930497, 41584.285294102374, 21050.14150118043, 84093.85760080132, 29090.08906805983, 25263.905518654585, 82735.70999226026, 73231.46856485948, 75816.52253956498, 61072.8867346065, 6912.169671080115, 3398.3818392861685]} +{"id": 576, "vector": [60519.408961353016, 75144.21576629589, 9687.38951198299, 48213.22860431795, 18016.48202723466, 82294.7968541907, 83064.35813904139, 6757.441634486127, 95131.94849817554, 48914.57882545763, 17151.015486353837, 84573.43364535227, 19737.4944985076, 91706.23317169042, 66264.38839474373, 13456.956665842668, 94810.86610374675, 99279.8330517725, 33559.45628647145, 14433.280601222265, 47269.101985448855, 11928.924451380086, 31419.61893621881, 61561.4542712673, 60937.913372451025, 585.5671732230228, 46782.745697387305, 11354.586346602791, 57143.792710690366, 37502.27627966167, 63870.19617576799, 93793.55962248189, 91352.79693782167, 49104.426977265255, 56381.66936205156, 75449.8441607389, 63044.70218264136, 51179.388796560146, 91490.50909536883, 60862.784605602494, 27312.77871600135, 69809.48713057977, 26598.351625522853, 86819.95783208989, 30565.193877902453, 50231.49485774605, 71760.03734431381, 99064.28200369944, 19889.783268306393, 37870.44267603832, 44727.759073602334, 19702.87575634233, 19472.504242435938, 35150.029290453276, 59840.444970014796, 95637.54825490771, 93453.4471930346, 97926.66063138399, 55160.64969588055, 1662.5175648294799, 78338.79601474793, 1615.49356966042, 66368.24865472209, 39832.4196392559, 40481.13258419, 37863.12560133685, 60979.58115182864, 10354.634323546541, 96967.88281228255, 24468.65104656334, 32222.001594238947, 94329.77047368426, 16640.557920827916, 91731.44519869998, 64405.831958128, 11985.436820708783, 10129.429600879103, 33393.73414890643, 57745.66625760659, 46076.4377545269, 9910.553055743976, 50224.87937552105, 285.06933619451, 43968.39958785474, 5739.313511757083, 28114.605097101452, 70967.98158789275, 73230.22816593779, 45697.48556064441, 81136.25835917922, 66761.83666695398, 52819.02424035545, 11498.292623799422, 82861.10818650648, 51363.912031680826, 13582.403153678602, 51473.67880267649, 25558.740626012343, 52552.887013447835, 21873.10209588571, 28761.62518625821, 65505.0306336521, 21401.7921543589, 66490.70693816316, 10636.131531274163, 9597.228843440986, 86579.02585465451, 71704.09400841883, 26986.721604750765, 41866.623888429065, 6154.835102594536, 70250.59877519486, 65021.978592191466, 70654.40947039258, 18680.288756746788, 92548.9248082787, 34982.013296979065, 19260.004571137124, 65690.56057378538, 77769.58851537551, 65539.75095492002, 50024.646983512175, 18697.71853926543, 20606.41774510331, 39892.993981028354, 59031.265404865866, 20119.053813418173, 49130.83948353483]} +{"id": 1091, "vector": [2860.3933634344658, 10920.411949954789, 80371.19038828718, 99759.11277024855, 96430.3099087451, 12826.317527176245, 54149.66942990759, 78515.59657130181, 72894.84192672525, 47108.46215856921, 78242.26913683559, 38459.53760973957, 9748.970208050412, 30273.440584649226, 69056.24720775413, 12892.836930456686, 83974.34934318956, 69498.66860072842, 92180.47075255019, 38510.50153322401, 54940.16207089229, 49911.70221282504, 88874.02572050397, 29850.206252798882, 55971.19586308803, 21593.970859725898, 45859.814887272456, 35357.76523752392, 50442.51007248712, 91605.33928280657, 8242.6515193327, 43560.42759253399, 80565.30549398327, 27970.788537781733, 44958.308692545776, 88351.05806208967, 51215.62481322885, 51652.291891077984, 37997.50902986342, 45237.24031531649, 10716.06223570658, 5253.148080793379, 9934.739374855228, 61954.37211893219, 12799.619790434646, 36718.39947121557, 59867.512826205064, 10466.946322343872, 5996.294742176356, 4510.213042464761, 74719.4552717106, 49301.16915965204, 46735.96245446835, 81565.49436726069, 97624.14749055088, 32964.893844173734, 90292.83774759265, 5637.578435432644, 29834.391913051495, 23799.14057577911, 32465.391803481714, 13314.087800513864, 53782.43499151882, 86191.88010130689, 65711.35957167113, 84791.83967615751, 82486.69632173635, 31012.214965942363, 39707.08223172859, 41540.90270349601, 94686.43867350587, 29417.409418162544, 76386.04896011476, 33665.35463183421, 84324.90749924976, 97801.46842862804, 32420.59322444679, 93010.42157183468, 12461.339324684628, 50019.89156583604, 73196.31059639489, 11278.218407030783, 7773.369795129037, 96258.3789975231, 14453.234260559433, 68982.4944064587, 11314.173534266203, 9426.663491185027, 66875.88919711411, 42402.218510128754, 3998.007858367381, 41120.281081487665, 17036.003146969448, 69662.91399770735, 88738.09580535653, 67186.94272038416, 39863.55331289648, 27612.33983025082, 17787.286390097357, 28123.435645062265, 58360.439477861524, 41628.51172544886, 26707.585980009007, 57348.29645716056, 74626.50049113527, 60953.2882142244, 24898.241182204074, 373.05035791566075, 87264.95783611071, 51101.40516168897, 3201.592111626261, 11911.87949616178, 23315.3026265699, 84683.24057020305, 87891.54657755379, 51450.491572965395, 26707.46642136008, 7350.552790199671, 88279.54924914462, 61355.57361084425, 28801.56902798249, 60587.66836252728, 28033.294368563445, 13964.552336333802, 87058.28955866878, 4689.475087556405, 5081.2821662184815, 66950.06284513093]} +{"id": 457, "vector": [87463.43654384231, 49099.93830320238, 7442.935477233059, 72329.27578385176, 7648.433266749943, 82176.4981848355, 57357.45783575643, 3509.882362379535, 99165.26226416092, 6322.2622931408905, 79681.54463977071, 2519.0159582906604, 57269.09183583746, 55667.249082937786, 38130.927581323835, 18112.77308794823, 28998.051113528967, 6410.839686374415, 91411.22930579666, 27480.918976655157, 87127.05700728233, 53974.94253963252, 99011.94802767447, 16439.39279510587, 55861.296817093564, 34485.21727483607, 84534.2672470003, 63182.23264756008, 45241.47419446438, 40014.231748353224, 69290.05325491159, 29561.15618018188, 17679.934251849307, 48700.04949933838, 80481.81245661965, 70885.98468500424, 40263.54221450778, 26623.97530865923, 25359.790735772647, 14006.100412470534, 90829.97008040744, 75889.8254450185, 20839.004472729383, 76102.0473035524, 78430.55607389998, 57027.50690762819, 18828.37713053498, 60727.814819672574, 63464.45111611624, 3209.2117412556463, 55450.31303785117, 13260.798778248938, 54958.28795955255, 22040.899804836412, 18309.53498138096, 90090.66415646963, 777.1964085902528, 91732.1308473799, 37055.31726110315, 58645.913269482364, 80371.17419990183, 18623.095906117916, 94772.29030810439, 95807.03754883847, 4096.450870806678, 6508.781375655559, 2612.5621499552885, 39842.43838365139, 56402.14340518661, 70290.71153124231, 63643.14834102926, 20494.961776796685, 66662.8866503094, 13570.214471719677, 440.396759865691, 19956.493915352614, 68546.07949749968, 13480.735794014221, 55088.044716790864, 78770.45476774716, 3687.096665227141, 33662.29402732079, 56392.25999746352, 64860.18911322674, 534.2366772207118, 35642.03654850654, 22203.285884357483, 58834.84389726858, 84667.34084189826, 51523.329720532005, 39397.664755861915, 87903.20315255609, 13371.440844095827, 9540.880318179135, 86121.70150145664, 30198.87887890281, 918.7163758073824, 67777.89368073932, 80049.55366244788, 10054.907711518945, 68968.60303927926, 2769.7549457242344, 34480.42983894081, 76215.036671713, 41576.55679514578, 96666.24211752426, 50595.48811189678, 93208.68515075103, 64486.240051168534, 66735.75339591276, 58868.91078890526, 92733.17313544147, 43372.85093065826, 28039.60486115804, 7908.0242233813715, 92291.71861986199, 55782.42227174437, 7011.940050095899, 31065.80638863684, 50157.40156984238, 33984.51505836189, 26063.748177541056, 21787.285160269752, 45677.13752351138, 5682.759510504831, 50477.44578611709, 98924.08822504754, 79011.58122287723]} +{"id": 1396, "vector": [60964.42304672106, 87122.06580797459, 19396.590686035863, 64454.54707836041, 55337.47313964139, 94758.2323164972, 8484.479178746851, 39873.06797727177, 85334.30568727333, 6820.803097977801, 75694.48294038002, 56076.81302588055, 86877.27776115527, 49150.487977367586, 84344.67390988892, 96761.29226926184, 6307.85940389127, 78965.89880237292, 34001.81079255984, 99743.53876976285, 96462.08958755732, 94936.17415132339, 83904.8190846978, 73797.7600484904, 93813.58831998953, 20864.931670778275, 17107.059196912232, 7139.92905924703, 27875.60599647607, 1374.245399558438, 10219.221026519464, 13339.508991765104, 38917.14986162777, 74078.87070918862, 52153.1384927511, 8283.631557669312, 46759.166399328176, 63384.60897797661, 9056.744655460025, 66730.5650249089, 69939.23492472786, 87559.43745529649, 86996.76213599127, 86546.04465733907, 74715.48209850905, 31593.18635356416, 73241.87998209905, 64873.772306523446, 3588.5070139245468, 9741.616296844113, 3254.130219217777, 14013.510886150492, 40613.54321607755, 14483.933329874677, 87989.07066676022, 60270.5134307726, 97043.2525524707, 71835.43535667763, 97782.98943277058, 31092.911752125106, 31563.496201039197, 65302.81720243688, 83808.07869985915, 96658.61652085977, 93666.43989404048, 75281.593890818, 44650.37510204657, 75482.94032928013, 68288.44436436628, 35075.91705188353, 75412.47513508184, 1957.7001125719917, 21620.332189696866, 14666.890997203274, 38794.22559186171, 64527.14503978522, 29077.533423747125, 12966.764945379184, 48100.4853100229, 6925.329089888443, 51888.49087133914, 30326.821994253118, 45156.99651529927, 24153.723447140397, 55307.57708745685, 17004.59410987283, 91697.3255591761, 5246.03371865453, 88776.13968568415, 46974.8407530975, 93980.69052489624, 84684.86675723252, 40955.70573346976, 21420.493571151932, 68707.03557004884, 25130.74198837891, 40670.23387006825, 50856.61916147453, 98344.39006812658, 70940.11103543136, 22906.399745485694, 24988.483819841178, 92828.18295496979, 80185.2322930618, 76759.02243559234, 55437.877091182294, 23763.36026787981, 59524.16752437328, 88298.49439979435, 11251.726756118009, 26393.182655713023, 43030.05922571712, 92159.74450855826, 77412.66382328139, 43717.033916518936, 83102.01177898825, 89795.57234294435, 9924.946689329661, 87031.22381502608, 99417.46079612362, 38931.16824775342, 48214.852134877394, 26562.189610940844, 92521.7295791014, 5586.77879808287, 12050.046080798316, 72387.80316818182, 71009.21124733698]} +{"id": 606, "vector": [81491.11894359854, 69257.5930312464, 52743.98848583579, 60605.531869700804, 28744.249257242926, 92258.05185395708, 53520.515052748284, 97053.59558923074, 46124.343156003575, 98683.11880249118, 65068.05197586031, 81033.69725019134, 75761.91716599144, 26315.001384706793, 72762.78963288208, 16240.959613599482, 36502.7312152692, 31840.670067533516, 29059.646907728897, 80973.08306930745, 99401.1276513742, 9718.779334112403, 22779.170357108123, 28630.72344986667, 75689.14498054226, 62370.10239152568, 22421.507363919467, 18726.871722595595, 70418.46076691175, 8804.385183545626, 73423.94053676615, 77322.8805529202, 81272.78931776623, 55981.31869228893, 56919.67213593917, 35650.657742850235, 92221.59784097169, 32853.56969758733, 69918.00743654373, 31515.22863159474, 72584.72653990374, 34324.283805255516, 83016.38860629562, 51861.48101317557, 29876.084316773442, 85751.27867698192, 693.6275939453784, 1730.4322151808105, 23847.727734451397, 10389.304870905991, 58166.06786162033, 67420.48757285708, 49320.5749327427, 56433.11902523132, 50550.44686418324, 95143.9410265758, 53596.83616662365, 3395.4405258374454, 92977.62768700212, 76527.43632628318, 55146.21668784275, 35970.73608161117, 9750.90755363428, 12446.895493359667, 96926.28217395188, 29535.01922795959, 52301.30324241559, 47284.49298599372, 82762.6288404786, 83441.99179194914, 29906.27337235556, 65899.16375001436, 64984.43688181476, 60540.167100196195, 73020.08178426806, 71739.52589722305, 46741.83060875218, 78242.02866229258, 91977.39943436059, 74825.37029023671, 76511.24813052363, 99025.6425495496, 57488.24077904408, 18424.348095062727, 12920.6656237565, 18363.879940734652, 72705.63109277355, 44851.41405995317, 18507.64985575264, 34564.958944887716, 66636.00052360597, 48268.55708155149, 88164.57871097211, 23293.920531106083, 5407.08773040699, 32134.23222282522, 64126.923156756435, 59173.16922914324, 73316.6046376848, 13806.583988977794, 74105.74036230486, 32505.665230629354, 68358.40101090109, 7071.092891011377, 74965.19740107788, 96478.1462506899, 40238.72514389105, 28094.9265713643, 72280.66661805572, 23385.58411825725, 15226.95211027516, 49434.94018739123, 39020.40086327623, 46902.321439891944, 31758.199256076845, 77195.4972418331, 29258.10532612987, 3953.822171573429, 44006.5565160721, 50887.33300702326, 65622.16914053174, 51742.64354269482, 2733.56718735589, 187.83131914297257, 27714.760440708098, 42757.532292151016, 91764.21587695219, 1049.478687262584]} +{"id": 2028, "vector": [62226.13680120925, 71656.9108134855, 65115.19951054019, 9077.033467719353, 21513.39017303602, 15999.862226062256, 77332.7264425172, 3960.4484827772458, 2251.281552776807, 22775.28163513809, 55504.68499766223, 252.88176047073563, 24387.415986856508, 71516.07884773581, 95029.01101022394, 16937.671559971724, 43022.840365077405, 18201.804744010908, 9262.826458186368, 99931.52049915958, 69989.84567628313, 52581.11231518158, 15177.496809067836, 71141.77817834535, 67222.2204111956, 81185.66956477275, 62130.25459377024, 59542.98291661315, 94496.76420878954, 19579.763747117227, 31875.82375730288, 26606.035690971174, 96433.96784591736, 82999.1792367282, 86519.4726049654, 78748.07348778719, 75970.65513378079, 314.77486738963336, 92325.48601018262, 28535.515644792387, 70223.51111178687, 52152.338313669075, 57383.666592015434, 62645.66134120611, 46740.6974647635, 41545.81862908273, 28313.200741308443, 89478.75732221353, 56405.93074030193, 85167.30726013453, 24874.67929838606, 36175.8450139017, 51976.26586792963, 66087.56645231506, 4362.520246313806, 32650.545064963877, 54033.67356355691, 19620.879540165624, 98048.19457241903, 34851.65046084449, 95839.68811388257, 30440.969871160694, 19499.704675017238, 98958.0295033669, 88075.81232252362, 25571.442664103084, 48872.692665470575, 18048.378524564923, 44514.90383139633, 99391.63169420767, 36892.52707462882, 93680.9840069258, 92323.46480617685, 80986.21704852495, 6541.0882394182245, 24781.165852859354, 21028.564051165966, 69388.22742349523, 44388.144717007024, 27460.287343052834, 16815.39256103215, 2290.3691005976334, 96144.42865699285, 95980.59942275366, 63081.709517722564, 503.275049999774, 82050.20957553729, 60644.489161166246, 89627.6846318805, 33921.68491997026, 38580.62766592208, 21605.545525031645, 24745.771814826712, 57124.3493632064, 11728.516210644113, 98631.32547468135, 58718.01796411662, 69417.36476358012, 2822.9640709785954, 75481.22623216079, 35949.43638694349, 53471.347548752754, 53338.60258161153, 38622.00130067025, 37837.66699192932, 52655.934958996775, 3015.3066935000084, 81363.13727533804, 62792.26268831645, 50846.07476586062, 21431.412391643968, 64882.51935238114, 57068.687862915016, 60849.80757830985, 26978.529767574964, 38320.23049760814, 11935.785450866888, 3613.909380312086, 14004.258142189341, 87012.53385885459, 95709.10068772902, 45516.05115021731, 64610.218076819736, 37426.83953845162, 14790.245086887455, 14413.899208944036, 28632.523898299234, 49545.43093552436]} +{"id": 1711, "vector": [56978.79424378206, 613.263270474429, 9607.667503070894, 52317.2087575663, 81844.61707685649, 61746.19781093243, 23696.41283843975, 37866.108795456574, 3805.9870266766716, 75928.65588101136, 83613.12056490769, 41443.70509788381, 44983.812087225204, 77839.62729076766, 92934.5151370889, 93905.63916714423, 92731.88821230207, 44279.467986964235, 34682.28278294804, 77781.70381688763, 58401.06266583951, 12887.757384863862, 28864.66488020687, 50668.185730335324, 98542.73045355498, 79833.73308751493, 55392.19278285641, 50612.204693561056, 95766.77507926997, 71769.57513644788, 75059.32896109002, 59534.7189838743, 67664.80415327934, 40610.68907964939, 99128.11788617611, 94832.51805969724, 77691.00970593204, 80346.31641411371, 92099.41131555433, 67526.77075738486, 10498.524274904676, 24187.075664992553, 60650.98474818117, 2131.1119343975784, 26257.096677691272, 8691.315594279937, 44636.84334635126, 76057.39942202228, 88667.17781318724, 34512.94051616145, 47273.5034404924, 33791.07578694597, 17865.110029274267, 24858.75284138215, 17828.6494287902, 57450.609549261935, 97283.07253081052, 4492.918709304128, 58987.69225186862, 63167.95837673193, 55292.63989607358, 37521.477764152485, 6188.77093590593, 70957.91040894, 61389.223525803915, 69978.54220385705, 70781.31772006636, 75918.11000187862, 34038.70715386516, 91302.73503414089, 3579.2129355754487, 80030.52574398629, 75753.00317129068, 13819.0626849367, 41827.72408725733, 58369.449189522114, 27070.029891219994, 9270.204832170404, 88474.29714232378, 28600.480197441313, 22390.21959845976, 78076.60460341937, 62771.797977586364, 30026.447645815348, 27968.284237216623, 6301.931198094057, 90066.77818469534, 53417.54135855405, 4568.335788733147, 55138.47133841129, 3741.806290613503, 6725.280072150763, 17583.169943587807, 23447.360129549554, 15571.171049451916, 7053.40227568938, 23918.95757087853, 30510.722623180976, 69533.72383411309, 48013.95761418996, 75474.35492390636, 36177.33523883585, 97077.37866619718, 23503.5582671256, 57136.481300617605, 48053.37549666254, 63629.642595612866, 56001.13766372957, 26053.389331215814, 89658.09087147632, 86291.57113214661, 17984.13779004956, 34705.2026389262, 14616.758113292793, 65131.900990931776, 6630.5816300546885, 79667.01471175606, 90175.56740146264, 29431.37557133373, 92385.69533189174, 49480.81366425801, 93891.21430329223, 77537.57095660912, 3248.3726435717062, 41143.33315340225, 5954.930688987292, 79485.4005557842, 33154.841362311774]} +{"id": 174, "vector": [36.724189532688634, 17757.315184291743, 59914.91820953318, 74717.48280716948, 96134.32626515032, 77670.5059628805, 90458.66704865285, 34481.45959539567, 80987.7872666867, 47984.49970758895, 4289.75928062757, 38071.06788739854, 85123.32034004462, 24433.84546324613, 86877.26963380286, 97022.50959282844, 4648.784914423021, 83109.76817595008, 81191.60557701142, 73779.99688718416, 22363.314786660238, 7853.963155096455, 24825.583551500717, 18314.88689089348, 89088.68804830893, 45382.087029285234, 47515.23064515831, 47542.04304907088, 2615.0653931914735, 32151.729795544736, 97994.87263851438, 37667.62398906045, 41115.66494217701, 35538.044921922396, 91185.51736597589, 57469.193656613694, 59792.80221007412, 70896.64565373606, 12462.765976785018, 69871.03664315496, 52076.243946334944, 34192.42052045489, 79246.23474083055, 40145.76059867169, 5389.38226107889, 37053.13772229486, 11594.029949964735, 1843.2153877593116, 29189.750494196986, 93058.79106739443, 21119.47199658234, 61202.46470949906, 72107.84627511015, 56165.99898103023, 57193.22452548189, 20021.59111249333, 46370.02674961826, 47066.76213426966, 83980.29147292746, 27296.162611405605, 32381.48284118917, 77002.25717478087, 50006.60333319875, 41345.57131497503, 67931.21943182503, 12462.134939186575, 13701.056983511262, 78070.76011992457, 97565.25040604168, 36314.56177703953, 27477.359893583474, 26655.038340039562, 23842.67549623965, 99994.03419505454, 64282.16009431675, 38057.7100750312, 40112.00857069079, 65223.43697063382, 32864.178341252045, 21808.496926656397, 74286.33786527687, 10326.834336346901, 72763.1717893753, 28638.718281968566, 26402.97762408307, 144.93142249858425, 34112.68230752211, 50886.866497477844, 939.8471824904608, 57203.88475854122, 6352.9355367187845, 14924.961216496402, 20000.97819733163, 5607.018491926175, 73193.85937245659, 2197.6705912030934, 25453.267411510762, 68559.41382173871, 58396.06489966959, 95097.25980267282, 14435.537744596306, 17479.59905527642, 68106.64297853177, 55159.73731037257, 89454.19876964927, 935.4469556884459, 89100.91107746091, 80692.21628516319, 45849.63829751124, 59319.82762022781, 26969.61380045646, 25111.33403108119, 88384.62828776379, 21968.955674763478, 92062.4038039432, 45782.85447799481, 40751.76482658829, 6639.460977379774, 67407.73548143405, 22309.797983134104, 36690.25777207629, 92697.58709505497, 10759.654637288451, 77368.63981237596, 87734.66874119427, 15554.686690255205, 27796.25085816577, 54378.16730799682]} +{"id": 1235, "vector": [73552.66092363816, 76094.30198609579, 69887.95070316846, 84737.50002826974, 60833.55112809049, 16699.370267840084, 96523.19031849787, 19217.70548026096, 31597.497740085833, 57880.12547477544, 11269.345548565769, 50079.79058409322, 44330.78207972039, 61449.35772594085, 10915.370131886726, 80006.82581751308, 27601.130691098064, 37657.428013450655, 44618.09010737821, 20803.863897445375, 57254.38960982774, 13306.2940356504, 58083.75860196046, 729.4488657685449, 6557.094220837034, 53070.12989901799, 84773.62125632878, 32826.43166386854, 73602.52481315605, 79446.8054964019, 31949.729408821026, 86593.60666645636, 22508.330960857736, 79859.9817643361, 28728.779787497428, 73228.8409767014, 25581.801226414424, 91833.94621092684, 31723.775873889816, 87673.0575196571, 37680.08160655669, 10571.528371839278, 51001.054126014045, 33208.472998742145, 43518.83462750591, 55073.66158548729, 20003.036321358937, 28485.939770669145, 86148.58177189673, 65303.488028474974, 3086.118708735963, 13944.271972590272, 78404.77389661047, 55037.180369347196, 8449.279500554507, 64610.60627863017, 75277.52985973198, 9083.318006774643, 42106.49965586282, 42472.062818374434, 44595.45468860181, 89645.16344445324, 23595.831161274127, 32077.65284852173, 60716.61510376458, 8754.929994913662, 59064.09396135462, 39804.96454754434, 46578.28779930127, 61334.827134204104, 37064.228218973796, 6759.121009512059, 45586.68004650536, 9848.084829057858, 36091.74697516988, 28671.8830480161, 66933.3107761668, 22767.664205151228, 40907.589383522034, 11712.924247178447, 73956.83008882619, 81309.44288359389, 24017.430119844263, 49872.651642342, 70928.98015926186, 98178.69319579475, 34674.39488576699, 48581.38951963973, 9529.024201286862, 1831.4583935248984, 28826.88241721295, 64128.5024550323, 3054.1697450833904, 98703.12713380111, 79693.1366705018, 25953.706796508224, 87557.11900027358, 47950.237948403585, 69936.76339108302, 89424.5050690133, 24695.45308152822, 87744.11088371798, 82488.71382713407, 38821.15804372576, 23178.616957339782, 41074.23193492336, 7772.917407751279, 37766.77520001856, 39228.109204414555, 92112.70049786712, 84596.41099451983, 18823.30328212052, 80414.04851010886, 81018.28571617721, 64237.56956817418, 20150.41678865209, 71721.37217410361, 7092.468223140036, 90489.53395759634, 46340.66879436979, 13872.214348859225, 7885.067128862455, 18041.61398201991, 70507.29459400217, 14121.563844421004, 38926.25401452227, 52635.80835391417, 53908.45159197275]} +{"id": 1348, "vector": [5614.426757358071, 62483.21538292719, 76515.38797550474, 54859.69261410153, 16153.539604236645, 47294.30559028688, 21472.00033779796, 29533.119737885805, 88958.32729022417, 43729.54834843146, 67712.13924596658, 63235.39466285937, 21098.73075071469, 74026.58631585841, 69059.77428195365, 84445.55279457738, 66445.9098517756, 87936.54470986358, 5924.060910347428, 95535.531391572, 13352.209828712746, 9862.932255954403, 56480.16900593006, 80145.43783244373, 84492.94215699424, 77318.70022397184, 97107.0499422649, 37683.71118726339, 71405.6923052901, 43034.89409138946, 25850.8596200247, 66463.10199928898, 19732.225554426586, 68051.02227201128, 89935.60253768718, 2643.5635658583424, 95556.3065329857, 40220.897248280686, 93630.40671013382, 55877.371569459945, 96741.00494019676, 27014.800389686243, 18828.47533530517, 47083.48612222392, 44093.08105766633, 21059.069584536504, 82669.72588743444, 94474.94216610085, 91081.72390519634, 95867.47693102401, 31994.46152502514, 71662.70560800981, 83883.81781918395, 57306.40775299294, 43237.047461090486, 80463.47718599356, 21177.038358904865, 74801.53228864077, 32646.59222655022, 74368.93538645598, 98750.68729630955, 10268.01630704145, 71839.71624367667, 89594.91449697179, 16282.457281854679, 55327.63848913323, 32845.14061171285, 63404.89075465407, 20922.108246067473, 35931.5801737328, 47878.544272261315, 64372.433500882544, 30450.005949727332, 32076.349691543837, 83962.496862951, 64970.13216952974, 81070.82095583851, 93827.52278513025, 50847.076339630396, 78325.93260216125, 5773.182408740807, 91997.59583827332, 7014.066407414043, 63131.835909938585, 56953.528193171034, 45330.29850819303, 67243.81020076295, 96190.50845240716, 79729.31967540403, 98394.37125770476, 62852.26053048597, 85331.41837659171, 65381.52896367352, 99200.07110594722, 94826.32381933309, 704.5051940887448, 96338.23954300981, 23629.783496427426, 3097.263808463402, 95254.56591999096, 12391.370584041882, 81343.87959821294, 41471.4870496218, 11921.067591759005, 62530.44162232142, 17979.172761479702, 67884.39468068324, 20070.896574347807, 61044.619370585075, 43326.27855699964, 73931.43535890954, 47081.71075510411, 69249.54045318223, 16759.99798790824, 12708.321834054559, 21288.283531037654, 95892.68498570062, 58083.236551953254, 36412.691987101905, 12208.740868182078, 78756.82269090442, 69364.92794700732, 3194.7800633226107, 45749.55276499334, 6994.3412828845085, 558.9591644597269, 10325.171537984668, 21312.746495607505]} +{"id": 1239, "vector": [25011.072415396095, 14705.461007845555, 93755.19097740189, 8247.306441407487, 27206.502649129205, 97454.6857311332, 70633.44857261586, 13397.061480474704, 47681.950973531806, 5871.253444043878, 90072.24702597233, 85928.6045198629, 86479.09553963762, 84822.92238869387, 86175.5947505315, 42114.86027233462, 27020.529698763774, 30084.543849188572, 83868.74618277994, 96977.6927636907, 25217.765806270952, 61445.810270315626, 25419.62984639927, 87786.39961839924, 24134.37855285433, 70125.58368944746, 12063.683727069852, 90444.73814151727, 5279.340346221661, 12588.494480684441, 65878.6678809587, 58894.29502371653, 68513.1743977192, 48721.55465830701, 35760.84794070337, 58348.38919756375, 85751.91220141776, 15636.887045104786, 61805.50001100811, 84969.11664243173, 2008.0294192668214, 21846.42375609126, 34235.2447671224, 9471.003888295803, 40187.660431457836, 43354.807052736556, 83659.09498394669, 30187.67215464342, 16619.404413966575, 95389.59196684825, 73510.45535227, 9826.571193870093, 80590.25487513107, 89307.56546412686, 24164.499011380158, 77216.06537844856, 78361.50613593284, 65990.7245923333, 53426.867939176416, 48828.00646437031, 83032.11695398777, 9234.936059299458, 65050.68512481078, 30317.002718138432, 30376.70687454689, 91254.7719750962, 59246.47904521815, 91574.38923922798, 37418.11134597716, 25837.23893841534, 17130.16852748116, 10026.43843938359, 9720.923295606266, 94276.0671949216, 56974.63972766891, 87686.43278148246, 45017.09246369121, 94603.1724195168, 42547.763867655674, 89713.91785398134, 24249.440740833816, 79440.71487752502, 78062.63442457259, 16046.486794774184, 28960.26094533848, 43025.58268733279, 6843.360846128999, 37892.38942812204, 12004.351093092957, 99344.94507163286, 24502.358794103817, 69123.11166867227, 64455.74217270735, 85720.25732674755, 84567.57674098483, 5943.001219900401, 59401.6606932352, 55490.30300162477, 71062.86085484165, 57634.385569875645, 49885.45968333365, 24864.975430584367, 88395.64167446441, 97127.50503982302, 61770.96790676642, 20361.34340481651, 65106.86644388235, 64709.90943982142, 47705.20837059302, 65439.91539963017, 25336.513261522177, 33807.717394426974, 79792.2210936595, 18400.83790204813, 68184.04637092129, 87400.98167049758, 25329.24735185599, 17174.21733501294, 21165.346543652995, 1960.4485646089408, 54476.34585446186, 4055.212924440943, 54003.37099708754, 7227.460407474829, 83069.96254302033, 10472.643766276802, 13823.818834138225, 45360.01127137479]} +{"id": 782, "vector": [70578.85669587934, 23913.237135456133, 87691.30086466973, 72295.75525259838, 91624.70838710714, 32557.271542147137, 30216.385093303823, 6081.5018389484885, 51054.29682945343, 30316.644836354157, 5623.442320500838, 84876.30255830615, 35840.26297277956, 66034.30471352508, 49086.46612439496, 72319.82067701612, 78976.4530328539, 81065.46176420388, 15242.647943971999, 73244.8224753086, 27896.732914751843, 70692.1704717538, 54766.61073341925, 89206.69752796339, 67660.24570701849, 64626.903724441356, 5102.710426584256, 36776.21046570759, 97254.57478579751, 82481.71471951953, 25763.66957940287, 23239.70104804446, 43462.34891097529, 58053.1986681191, 39140.354289751376, 23182.79049079467, 792.6867377667679, 33024.809301287096, 95794.99046655558, 75271.45057839333, 28498.261488869593, 6987.209313212817, 47043.03527909474, 98611.03409938703, 60876.680388152956, 79904.3088775168, 27737.092373675565, 70782.87886095508, 74836.07360355926, 53957.29119469328, 98301.08031703396, 97619.29790993598, 30175.272944472043, 26206.225224916303, 92015.43156846635, 4340.652005052226, 36915.50768100089, 693.9253015538882, 25877.089784202144, 74254.23859510971, 20642.252066807843, 91174.94064404766, 50301.62422980591, 56767.80988094448, 74894.87911201351, 82789.59433936547, 85344.47740740581, 22401.84579516683, 18043.995725193552, 21223.917789537307, 19948.767612445426, 54472.033120650995, 82657.56187618173, 16131.641298480781, 80871.11735394738, 22983.978023523323, 91830.76634678821, 48918.877871851386, 72298.66815394502, 47832.09724341877, 43275.69526340137, 61457.16468938486, 99215.76621576076, 83502.28443868119, 34975.49863505024, 22426.852676518138, 13242.202328102449, 80127.72362440286, 8014.044959599842, 3678.4687356951063, 51121.19039459308, 82087.8806579843, 82863.21822530752, 72093.67308711054, 49852.62317286776, 29386.19635141784, 13228.748792889777, 43598.57325227252, 75789.16500897653, 24495.673172478284, 36096.238051660504, 31908.806464095596, 73664.07766970378, 4306.342375197148, 13766.703611165909, 79369.38337533234, 39566.16204579061, 14870.322652435041, 25331.737151327692, 45257.51408023011, 71340.90968932009, 91812.4764113732, 9743.86559251611, 69885.58471235477, 7851.072176723406, 91557.19750132748, 8919.760342797967, 54967.91563312874, 85757.31760242539, 2297.792099800211, 75411.50972438414, 30827.876432581335, 13730.320694941967, 90520.5535326401, 58145.425203083556, 48930.57056805657, 37017.069196577635, 40557.412619473354]} +{"id": 162, "vector": [11485.393507491992, 13427.646713797003, 74825.89702629618, 16890.174781742993, 66827.86446877982, 18307.287112752412, 30888.28433734433, 96641.58783580105, 72393.89321570145, 80194.04127010147, 23271.795177859112, 70508.43937226679, 27611.89493103474, 86889.10115020801, 41718.79060674566, 84492.01906185797, 84646.84799434242, 56163.50563418038, 56276.2898100592, 40213.786622185464, 72649.36167661747, 24824.62714634398, 85586.35807513412, 84869.42896456229, 38595.81505282421, 68869.28147617003, 99924.03192225029, 66356.0220876814, 58481.53277565168, 74233.74295874916, 16821.327882694848, 50781.272472923454, 78812.04080941087, 83439.57449895807, 88537.40460665173, 79128.97993795066, 28299.210912350416, 39119.24229575074, 85335.78752104878, 8314.90412706104, 47509.94251439872, 20576.013723955966, 69120.73527586958, 97415.08411591919, 12671.174866416712, 95262.42943870123, 84944.02212531591, 47792.017890620285, 49888.799642238526, 17755.662346685673, 16397.267547822637, 6340.874301790566, 3893.8611360297414, 55930.119843432854, 45470.133288238416, 46054.191304250446, 50888.13863730864, 25514.823093477335, 39483.3973760112, 25615.863089292758, 28482.038952912237, 68773.2840856984, 58763.263160313574, 88311.37882734319, 15235.71278120821, 55200.015399973025, 77252.24255453717, 64811.52202178248, 39284.153768777396, 3589.069787922339, 83225.28170169568, 42030.092560229095, 32595.135139943588, 40999.256930155745, 97403.68659811579, 96912.85532737555, 71746.55000086626, 66853.69108129719, 85964.62454372506, 82563.99126252763, 21325.06338393071, 77566.52327807643, 68074.99314835254, 99303.36454247808, 86141.23729784385, 39132.12530089702, 8073.675778887413, 10083.642459150611, 55368.352251680175, 27715.52667592191, 33757.95607238138, 24034.706324169987, 60171.949357498546, 35593.25680617734, 3256.2071730293906, 88209.99513369416, 40948.425221535355, 99717.31915615124, 10238.539994993524, 76408.57227299215, 18201.58545402494, 22637.08042627456, 57567.39928702571, 21189.902367321345, 21008.90143532931, 96157.77254443076, 11130.23846847836, 54230.51529451777, 38667.65078436134, 38365.73450651525, 39050.17046279795, 89755.61972231066, 51256.778219571155, 20608.12062389231, 78229.75798702218, 27470.44755962137, 62103.3768833312, 74423.28127139767, 51241.317570911146, 37947.56468920611, 40136.610748009225, 51760.14260924559, 80530.22465119626, 75465.2499923132, 5587.940405750846, 21195.6194169061, 77320.13677472551, 99357.95568246143]} +{"id": 199, "vector": [85993.51714762485, 69793.5173891808, 11787.54282426977, 89873.25891577077, 6032.193912586336, 6534.908339743717, 44311.301739157796, 34402.01717370336, 46489.91647484319, 47775.73678795512, 17107.42892591577, 17586.6159669105, 11410.00036087685, 69023.21230115513, 158.64543252503972, 32256.01865445603, 34972.21720142772, 62550.86006540739, 18729.71536044864, 39716.25398071634, 74166.42103144364, 54990.79345788106, 58177.6143539154, 3324.0453385810056, 91705.13660824712, 99865.30148456499, 33449.68246993232, 96396.16123666032, 22541.980116527026, 96154.60377005834, 14901.42460118299, 26785.756499708546, 46159.144615984216, 90111.31747674975, 60837.14689946799, 89315.42078342673, 81074.03534459189, 77933.22779857871, 75718.28616841328, 64523.87213357539, 29444.489889315162, 17904.267073810566, 41750.642801319016, 10203.742740803113, 44938.87538023404, 82963.99005356125, 93753.35975502833, 58024.103251488465, 24946.72546311997, 8113.525659631404, 82712.49774153875, 99738.1488447527, 71055.39404356854, 57585.648244423945, 45070.41759695212, 81898.95115282341, 88552.92265115805, 6508.162952901075, 70500.64794627878, 24128.848315723306, 71915.44273833884, 15532.934053475312, 17911.74542305195, 56442.95017540618, 95446.66683697958, 81978.66448259895, 69653.50011293363, 31936.024929172614, 39052.72195502181, 40704.42479804396, 17338.783532406076, 56426.41292684143, 70958.181657695, 38730.882847168876, 51075.10572331133, 26052.982087897337, 77498.7760560492, 58400.00067568104, 72769.33800345106, 43440.82727043171, 25309.07316640535, 72874.60649579857, 50118.159226372096, 66605.03702435707, 84837.00444536265, 24774.15464152566, 63244.02810141403, 78707.176573838, 26750.552816171625, 14107.900970488652, 9309.553562396233, 84430.77669430911, 34898.26171263589, 58354.56530385705, 79879.78863383128, 12358.88987735444, 8787.141634500551, 37044.44963499559, 4926.489852503635, 84847.88019732894, 60017.14803095073, 59990.888970808744, 29172.27346866442, 58284.02739594881, 15450.27342962716, 37672.54212800413, 92.3320886334178, 12583.728955790375, 12385.784541355737, 73272.33261609996, 18002.86355612143, 35638.25472638944, 14792.677085927508, 9165.985401995902, 37230.274257016004, 36044.379562692455, 98225.36580184558, 44694.349088096154, 67055.90348878894, 72083.38999982584, 37606.728477745964, 46889.85006811496, 41633.28589048653, 28912.192657848056, 12218.905861430141, 5151.87110389399, 96349.49902973184, 75840.01695719665]} +{"id": 455, "vector": [51566.06430803903, 7978.925686686811, 11583.669383895034, 53221.04862355672, 19026.421889602054, 99167.62869555507, 65490.594850534464, 88969.44983386506, 74196.15694399286, 94275.1370084143, 13035.384711551767, 22932.155561993484, 48800.26768495938, 80975.5148093591, 35229.653957759554, 23433.78310771592, 64360.995791231115, 74758.78969181968, 39749.47261232707, 58728.17035697726, 98118.1767222915, 18427.479345557607, 81779.96311327131, 42747.73035865645, 39390.12188501877, 84828.82509401324, 18166.975936378305, 47574.75557759763, 51185.32294060958, 92392.69379783189, 3798.791792501488, 55838.52735592659, 59979.16924735947, 40856.173809133514, 35671.049289402436, 15997.938655506516, 58006.60162730757, 29504.091559730005, 71956.73373796309, 71735.05702226138, 30339.8294521798, 68681.08806273478, 51396.865348855594, 62291.23716415622, 36762.427146492075, 25384.51017413004, 95407.00423894393, 7058.352803969692, 53370.07244804458, 64173.23771115089, 57540.784633387135, 74255.49905704071, 30525.392799735928, 96733.48098558032, 74869.48153510163, 38034.83393946569, 93805.20652564614, 85100.09501591776, 59434.84956594104, 95814.21554753231, 21733.97129834209, 72591.26621055997, 34426.60734496874, 9636.036722353969, 75871.99688676122, 45279.446398186374, 87997.9073669138, 81180.61427102624, 61172.28558985015, 22712.710648058244, 45949.010308314544, 8018.645369228328, 1299.4299936897935, 31794.542361767497, 7371.031343196432, 95145.87871506454, 16316.466091178816, 12121.682983002835, 9892.059317195135, 25004.945431154203, 17071.61839387591, 37524.480692411686, 16815.337529232776, 4492.89370394148, 21451.66938278239, 26098.176641108916, 17935.506984565465, 70505.41035582009, 29180.664228437425, 3823.114252386961, 25583.174829208532, 37733.379717759686, 68383.03151761957, 12495.126277651558, 19107.945794374937, 86617.92513632988, 21266.748656774602, 64699.38892430054, 58426.89146507671, 67766.41726950339, 54225.03658762629, 90510.0584843323, 27763.638514484366, 33452.702200091844, 50841.37190662352, 46252.00855309916, 85075.55484284874, 22147.892774281474, 69286.20185276719, 84823.70138222195, 89130.92729788522, 79379.8321051171, 70372.7915915901, 7987.893615073738, 43370.35592170666, 79259.99059773314, 81059.59800799872, 96183.06876974733, 26696.546745295913, 89830.88592839468, 18166.526366005266, 9642.58786747415, 81773.38496411125, 48453.79742719468, 80569.91548001529, 98857.04450657607, 29105.00739022789, 54588.990078716735]} +{"id": 1488, "vector": [77547.75997381895, 45906.844393932755, 10827.787411404843, 83744.97139582055, 88387.79991468287, 41106.95982809466, 63311.69499446283, 16596.15783828874, 79122.89217614454, 26449.99323334711, 93481.01874839886, 72368.55218254778, 36823.6908614166, 4181.88040577887, 23368.805529613666, 90697.68219977569, 28527.763488743883, 28270.876825886193, 64669.282797584834, 78793.5807167853, 43884.09923097028, 60705.1905427718, 35971.574923077496, 14802.72762862731, 9727.765357450724, 38296.667885490366, 83898.15407832754, 34056.67231060177, 45741.97905281109, 97191.21799330559, 19509.028155827324, 43469.62473174992, 97967.74903378707, 48015.65380650096, 17538.762661809582, 30441.97503333944, 55160.83944013974, 20125.00800301159, 61895.27679113122, 65217.35016995901, 95697.20039631639, 32305.334439196355, 31702.091417820677, 79779.13715081054, 22699.6458864552, 68973.74450813456, 83884.88531243427, 43346.8578472108, 79939.18726765142, 623.8918588367337, 1943.794299230972, 40447.69060828033, 1162.5133682112266, 65775.90105760015, 1153.3474487446238, 83728.27823220717, 16002.067992251712, 19239.204523943376, 62079.777245198486, 37182.840463473985, 23627.100688465773, 98035.70064244553, 71256.14795012386, 65087.14863347004, 37385.20275846774, 32130.26707988794, 68650.03484868631, 29366.87753504681, 68787.78782658557, 84465.05221546101, 24236.367033910254, 53579.49462991869, 64724.062496779785, 77343.91213200548, 83357.49655517156, 37366.521520956296, 71219.00763468379, 53433.80591552061, 34172.423713371114, 83659.90404776657, 63578.21042235814, 38414.6769597287, 9289.417213325823, 8206.061155580901, 79721.17383840887, 68407.70614979132, 77940.21408960993, 7783.95891052831, 75874.46793858854, 44826.35863792847, 47973.94934683195, 54400.35461394723, 16828.5703516045, 53688.51517668093, 69331.01222870025, 31631.32447358653, 13666.933513778933, 47079.03713492412, 56032.717753739336, 44597.26848724197, 68032.28784693399, 26725.14187506253, 60530.53794682981, 22867.388497581742, 28594.164001592428, 60524.54768118325, 72193.8615223958, 20265.186981532424, 51154.27657420914, 22941.88195688097, 85668.49138933119, 68253.98130732539, 34101.21753783162, 1893.3547394206385, 20053.379842088158, 55000.48114121438, 13278.568237580823, 48513.90326657803, 77723.15454606638, 55098.79228514981, 78236.80947647012, 65967.58095217786, 95925.72539340894, 88561.25706975111, 94288.11902277007, 43771.66008121759, 91825.1114043054, 51279.505413102146]} +{"id": 453, "vector": [57993.93248696928, 86775.13293105549, 30293.806366172703, 29882.97420958288, 60910.572234431624, 28210.969991883805, 75173.48741044666, 99488.24854709955, 3094.8339809162894, 52017.60448612698, 9043.269714357293, 38781.185822533094, 2646.636342437592, 94669.69617969684, 27186.20530673089, 19424.844412440656, 99965.94779997425, 84549.97845878328, 79730.21891417996, 44808.11289275118, 42821.00362791091, 40226.41124086709, 33156.749647537734, 35499.01175619826, 85378.65661123152, 83274.01063414347, 83162.8262269464, 10480.500857883635, 55563.29891579392, 88320.95418325649, 42978.90977478613, 89105.0120135621, 60139.043960602015, 88784.5397518455, 4617.596283607805, 75668.73451703977, 16158.515177614507, 22806.90326151732, 22779.970695492313, 67801.2738025533, 63652.556256642754, 56193.427997271836, 48789.38065793909, 4575.782272737949, 54459.36445637254, 76941.62620335819, 59765.72272924067, 93059.69256738196, 21979.000807152217, 51469.74054261197, 50183.67569592058, 97823.98257160431, 19749.531037124325, 51.38943130231777, 15890.944975775812, 49843.38508934712, 772.8146357939503, 40694.927258834054, 74311.88459274457, 12076.56993005054, 65342.28687320786, 64373.69146797796, 74120.80158953916, 60003.43841712431, 48609.43715992624, 55619.39091766288, 90708.05827664919, 32217.468060191502, 45586.29912140555, 16006.187775069624, 29273.9050140682, 71198.62101563819, 5827.990565611673, 42298.55265429481, 992.209718228243, 54913.235214209446, 71994.95243494086, 66006.26436620507, 16398.141649573616, 18995.232902951087, 54384.19709578261, 51674.4554123464, 4906.824556197398, 77982.48898814202, 95736.67760660399, 41305.066402125754, 2324.191250135366, 55654.493277454465, 83439.91254332116, 25356.055542429433, 44874.0831842503, 39547.03462685786, 91886.67350731218, 8845.622090205241, 72784.0888666047, 5079.825580071051, 72037.11660121154, 16272.41188751496, 71854.2049159317, 49895.05628078178, 98961.03637964788, 88239.01375106111, 94597.49224824483, 25656.63755771027, 66978.80574106825, 23377.795088502196, 97668.04660851562, 63368.543246002315, 38598.009859696824, 44327.73931830437, 2171.3239156002096, 78966.43136702682, 89240.11576120593, 58876.74446197506, 88870.07592214028, 73909.10254986606, 12279.074699530835, 50887.04477316718, 84876.06993082957, 82092.42074206211, 84917.41731223097, 97533.64237882264, 73647.94488782852, 47778.02035969265, 94454.35216197239, 69525.24069545793, 94356.45176934027, 9870.8377750653]} +{"id": 170, "vector": [29737.252211459952, 50074.83084461397, 72654.82829624874, 94292.0450639273, 44886.177866205355, 50024.66844604606, 76500.86916779891, 54114.84602185158, 28324.369687858365, 90726.40741417915, 86360.6558242908, 21985.521393757514, 60494.648555119056, 65379.30051389017, 99149.3737348302, 95965.3634048438, 55771.67456953077, 14697.31313832745, 45479.11613058604, 33787.28866786708, 6369.486617385655, 8493.054251508991, 30939.223699042584, 55951.55716967824, 16528.535441638047, 33072.238043654215, 99351.92481771336, 92143.86316960417, 36297.34067848264, 30132.001211926905, 4850.958462454147, 18706.232500012255, 16323.094014713046, 16772.2195729324, 8345.5505310995, 97636.56136910181, 97444.59362788794, 7983.517273227503, 69482.92151556449, 36058.72767958904, 33549.43105078223, 78641.92460053113, 54706.897633794746, 80864.99990747073, 20505.382512749493, 59741.48042542693, 5646.874847912264, 3556.153332399714, 60226.57192409264, 17206.922046904394, 90831.28100692533, 79804.34676082735, 39729.20873784107, 40845.19173085176, 20491.871044888732, 19199.177793679668, 32661.22774866733, 12548.21769048423, 78971.64485100456, 69206.43975745706, 44038.98134256145, 73805.18535619437, 20104.54834008354, 187.97655023214955, 66159.02843340822, 42523.75132273856, 71505.50570722959, 79543.68839756039, 93587.4089474684, 54576.55238080536, 45773.41890502063, 6714.80697685829, 39462.2756204582, 60824.499097292384, 11409.835119910094, 72835.95372120281, 49262.21343096579, 16305.743285347618, 97855.03896462894, 56450.05950131822, 39499.70359914487, 37689.456190460434, 18016.92455714805, 3487.3394866110298, 30005.309523884705, 36197.52946339236, 72066.52962508038, 3171.485597755963, 88894.00865561068, 99280.00291051061, 1883.0949579163648, 96193.08259089715, 97752.94934749596, 14809.414744868187, 91006.97233144606, 5887.931049444306, 78731.0315340713, 43532.66647414779, 25006.860019416465, 42206.33476801444, 80702.50461036449, 38426.14690063531, 2539.1941987939927, 76568.46900915286, 17040.557804715816, 30232.399251903964, 87094.77236183264, 48875.24565822828, 75581.32321332113, 24056.085435030196, 38908.95864225909, 91489.33756374977, 1095.4730291993785, 99128.17729585216, 50560.22111393531, 33486.95836600909, 11668.775686222132, 12776.87910644919, 13076.80020078057, 709.9308163285966, 8109.805026858541, 89735.81539256757, 34456.66210760404, 64569.28836010686, 24907.728220877667, 8758.66896743418, 84008.66277010771, 41390.32825776656]} +{"id": 675, "vector": [10038.303183218977, 23135.44079437444, 62747.313113367345, 75232.75513742247, 48824.41047992154, 77494.96237336297, 90976.49365542053, 48303.789187416165, 78744.12447833511, 68916.47528552925, 82858.86646156482, 21157.01661587569, 61574.98447653555, 13635.506346290416, 59605.250820501795, 33020.34635441973, 31982.919642113706, 63565.19720728557, 57597.689632326386, 53779.11867314302, 97150.64818465925, 86915.83895884252, 66237.12593070292, 79719.16695729681, 92393.6956928766, 31885.818994017733, 55689.08021355658, 43207.898120869984, 22023.013993426364, 39542.69757449015, 99762.74159053488, 79405.96999762814, 5953.2617111485715, 33904.88457925196, 3055.0344796011595, 12720.679440458327, 34831.41747100661, 99906.47883142452, 69129.0014603982, 84595.89841191268, 83517.41569393207, 74573.28394849686, 85710.01434569598, 36557.43527056968, 80395.55447006332, 66085.89051084184, 14571.38717381542, 76956.14308607983, 29431.58874768427, 48987.02135025976, 79469.53631061575, 82269.44078714054, 91174.3378334526, 33402.13791025971, 53747.55012692668, 8956.980362467693, 20779.52695326366, 82321.88790498539, 69153.49217351864, 41669.838278122064, 62616.99078522129, 58964.66225348554, 16698.9072121952, 94388.46726332264, 63035.07173327085, 66537.19020466106, 15906.675173247875, 16608.282117323914, 94780.89222126645, 39846.82958508111, 50985.25909407315, 67762.43761920616, 89482.43699743463, 94543.65125844459, 24503.322725595124, 33261.01086820253, 82641.55634040301, 49622.81161867541, 49428.065128218754, 65342.881176184375, 6729.332765802743, 82636.08536197645, 66702.83252752917, 54544.233313616554, 1509.3809473216925, 11021.1484635227, 39063.23823653732, 63298.203803741206, 34316.572632665186, 9591.902738653169, 52622.57353800004, 57595.013283828135, 22736.557567908356, 13831.833957673167, 99574.38429000994, 80246.93332987413, 53418.0933162543, 41982.06696067427, 48890.50876429549, 9811.762909232435, 34763.49510801099, 73460.36836562213, 87838.36412630393, 5080.659566771084, 25746.51875886935, 29722.23987248218, 17975.164331089287, 6575.168289763844, 31493.118134101594, 77046.80300100464, 37683.616180801335, 83664.04302809334, 59486.75334274011, 91978.88567520186, 74752.75029594456, 91140.45801893566, 85220.55550071907, 7448.660371265969, 41246.56370188586, 17383.22499052618, 45190.075558118595, 58433.24471051876, 88029.9179806004, 21131.92628090781, 59848.52589689047, 32385.91706052971, 15005.016396367699, 14963.046974981786]} +{"id": 1770, "vector": [76583.91505897902, 83261.31307212632, 89283.32137808684, 19094.3324742713, 45126.71763905124, 72811.88030063188, 30871.924679093132, 99143.91353740633, 79708.8563504337, 16619.186422954936, 44940.3354316183, 80213.1140113909, 3962.9330101389805, 18085.427313657332, 93457.37608500817, 22871.435849726153, 95009.26870358805, 79833.24584849432, 31482.860732525864, 57594.32596589958, 50810.99758219545, 9710.278263712902, 72554.9209996637, 31728.4156159387, 5534.979070097778, 14348.31685066953, 83420.79359814373, 46773.798985580426, 39597.92287742848, 72925.55143382981, 162.00047042196354, 96119.95249331578, 7898.195928076535, 52241.87488083718, 89220.08009262902, 28214.390068258, 19801.11426350143, 15067.669500119351, 69750.54231959094, 57762.01619263026, 67040.79465025575, 98811.93366773993, 59698.26665414133, 51936.299732592364, 96362.88258818387, 450.8445273385009, 35495.692340656795, 67559.22688601696, 71476.137647335, 77493.36404462061, 62715.00322276779, 6529.071979542289, 19247.304176701753, 12207.851844180217, 59573.06927776942, 75683.66725140384, 25181.98858695566, 1137.9340595019328, 24038.801203785355, 98761.93192274426, 59321.11405817254, 59378.113375387, 49653.54206399758, 95535.7166214453, 94327.56231749083, 39782.93273395971, 8217.17805859532, 11503.42563575878, 5805.8021082283085, 43988.29255275293, 17663.214070708887, 86225.61133948505, 28735.148244367025, 93553.03559282169, 65736.0395118176, 50616.74275607089, 73886.24944617198, 79416.41790122277, 61278.86623566195, 89115.51581161268, 43367.649112780215, 20260.021331596578, 11873.433470784823, 72849.51363978088, 24997.20725361553, 30207.913872150115, 81429.07437145019, 59861.08184075469, 5060.671470845435, 29754.430607868366, 19385.725432577892, 20486.08004069932, 93657.01309830186, 12534.528623926888, 75736.8215810113, 98862.56923944793, 40446.66290949596, 45782.964708291685, 46970.959016476874, 49408.979700551004, 47175.49631357805, 69010.43560593316, 61745.40208532443, 1838.2793463798341, 20123.898716401367, 96894.64027392928, 74890.981618039, 75894.51802469019, 45644.76714157045, 84538.6046043062, 35136.31384560132, 37472.244858636215, 31023.49115728633, 66648.45647706206, 13293.02118044836, 59924.71843869156, 25020.325821607094, 15433.016694757207, 56360.1814095356, 94404.60933848792, 6114.745440688274, 71574.14635464858, 32839.95296089446, 50192.40927874222, 40797.92812888386, 29455.822852106085, 62028.394939314116, 96272.01245152557]} +{"id": 992, "vector": [16065.75850873606, 20182.529628063017, 14738.17125948662, 57752.67144648238, 71840.96013971018, 79346.87946967827, 84808.5738101287, 11332.411109204577, 90336.15247127583, 7120.251926838217, 87003.79925869578, 34275.2012562704, 6093.571101580908, 41891.97604296808, 51649.779461986865, 31072.956584347612, 16592.14185739004, 99121.75928025793, 2887.826622989742, 33308.96759669323, 92281.0437446866, 83556.48423767133, 79616.04944486532, 51720.45509424718, 3300.5661371276365, 77979.6108347934, 3268.1731334675555, 73657.53686921645, 63684.94305318386, 35681.01637229396, 60673.805135645, 97166.08075476595, 7154.935351528935, 72206.24578677675, 94072.49128601517, 28196.623588590173, 4452.9221339615015, 19540.268940618167, 30372.056668684367, 35416.68167118055, 437.06051513855425, 43065.12432877683, 42016.93660204351, 7724.457077043956, 14841.068267209057, 10166.154243429992, 11977.528799704063, 62797.958883192216, 50671.770411034144, 44983.87684261149, 7482.675039848375, 40806.63047251296, 72205.3245906133, 16821.400512632088, 99806.53022472459, 2150.1188389382087, 60193.18289803602, 39911.52356845834, 29770.811655623707, 13189.673988332595, 66027.53358614078, 41760.28584765872, 8678.89217458715, 79560.77023920644, 53759.98180441698, 79892.97278792936, 86920.66262787466, 42987.36180740229, 90164.97979082822, 58298.006959136226, 43910.54020584015, 67429.94153428373, 12607.36986327442, 63872.607479506194, 91081.20081553742, 91036.06208230086, 256.80872146172095, 99682.8818785952, 8871.312547177367, 42369.44641049327, 40100.84195480367, 60166.239505476326, 73641.64576779936, 50883.5044651124, 12436.435241260813, 64815.52046336454, 54216.77318140029, 76463.14283459232, 22470.17129395188, 45289.98386948247, 87601.20538656595, 58413.602151140076, 66785.5786676984, 49678.17862363354, 88140.61171786304, 60826.84229020432, 40813.20510693166, 99903.49615778505, 31201.064777743326, 71284.63931097268, 90222.36642875259, 62271.574114292525, 96949.0050837706, 23322.52006921475, 26632.950792866995, 64746.755251809685, 87146.44648903543, 52446.39775498051, 81594.73780571566, 38306.67335484696, 46257.23781450626, 43466.5783672874, 35171.964589662864, 54792.71240130685, 38411.5023367405, 98409.40561413024, 84289.7244603919, 37024.125021819455, 46201.024807313705, 92100.77809200763, 82913.3181519639, 71962.60122067925, 28353.092915700563, 9107.587601893296, 26977.15646916118, 57527.92580996994, 54827.514803918864, 11406.375397957469]} +{"id": 1366, "vector": [77103.15880427707, 37954.101492844806, 8897.416429993355, 82429.8928758287, 3256.5867931832913, 18743.143242522354, 84616.57433712785, 3383.4723156225045, 18608.364061180473, 21654.508858617137, 28830.25133678634, 68300.532890187, 22908.816850741077, 88671.7463595862, 50314.8014874292, 66420.79808531309, 92442.20295438737, 93648.27928425254, 48425.99904411798, 19339.406778689015, 28573.37849675897, 75469.94337633603, 23721.93839777008, 36140.033190388924, 72222.69796758163, 13629.550854007211, 64263.137578463145, 84361.15608835047, 8270.768701409814, 10407.722083331828, 43888.1802378373, 92177.5120859445, 7854.777569268445, 42523.50297773296, 5120.54462742646, 79140.62768601795, 96754.68776497105, 70986.323796649, 67354.24278718847, 63496.56248548742, 99010.0224126916, 42501.988664109806, 20567.498654439707, 91208.76593053053, 18176.963697482595, 84693.12164048935, 14158.22965690371, 21688.545634993563, 52672.9009824327, 67487.99557319419, 40904.70334877493, 94701.16182285873, 48267.35857293136, 4068.3415781585295, 67491.33039933852, 17130.866161774593, 50970.95900567229, 39062.4122733374, 95269.29813443913, 79707.14465234255, 20527.647496754544, 4805.933334371615, 55468.00159362418, 13745.847248581955, 59071.77313734148, 62651.21185216228, 840.4457823266598, 19609.54748171997, 3562.9183412547395, 99589.2757357921, 53492.966796566885, 80869.84868276613, 51190.93480147617, 25428.363961887124, 18503.68607545563, 70687.36387025399, 75776.75361038795, 85168.39236167468, 86861.34871586344, 14608.955299803472, 169.81827539653827, 36494.04866878756, 50054.00932880291, 13334.061030779198, 5441.329437947007, 82935.40134474891, 423.32385930272665, 86966.54955923962, 37228.422289540984, 36337.566443142554, 11335.664721234418, 90534.62068978228, 36606.22573312101, 40313.50015862124, 91031.29770341028, 27761.510385117483, 17774.799263954166, 15055.406077827593, 96358.52653388772, 55346.53672891554, 63386.82375613406, 63860.907217565575, 11190.698963924538, 1725.8897232484305, 5363.529160510483, 68325.09179014232, 83206.72922848113, 48358.898862990216, 9750.552858483452, 20845.708828031307, 8443.38829609267, 71698.96343489492, 39064.55152797575, 35339.01458602857, 75167.92756160145, 73605.7420090605, 59013.31126463494, 43241.07033999966, 15751.130421316806, 78741.06509961942, 15776.359549343111, 69880.55330005658, 64971.620427596274, 31255.918144775052, 8638.974442757486, 89480.22582976226, 45991.504258082125, 81696.69730501868]} From f41e44ac440112e390cf6ce709812f5f71688183 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Fri, 19 Jan 2024 15:01:29 -0800 Subject: [PATCH 181/416] Handle multi-vector in exact search scenario (#1399) (#1401) Signed-off-by: Heemin Kim (cherry picked from commit 8c982656b767088713b6daca40a3a85e6438fd6c) Co-authored-by: Heemin Kim --- .../opensearch/knn/index/query/KNNWeight.java | 71 ++++++++------- .../filtered/FilteredIdsKNNIterator.java | 74 +++++++++++++++ .../NestedFilteredIdsKNNIterator.java | 59 ++++++++++++ .../org/opensearch/knn/common/Constants.java | 11 +++ .../opensearch/knn/index/NestedSearchIT.java | 89 ++++++++++++++++++- .../knn/index/query/KNNWeightTests.java | 62 +++++++++++-- .../filtered/FilteredIdsKNNIteratorTests.java | 56 ++++++++++++ .../NestedFilteredIdsKNNIteratorTests.java | 62 +++++++++++++ .../org/opensearch/knn/KNNRestTestCase.java | 10 +++ 9 files changed, 449 insertions(+), 45 deletions(-) create mode 100644 src/main/java/org/opensearch/knn/index/query/filtered/FilteredIdsKNNIterator.java create mode 100644 src/main/java/org/opensearch/knn/index/query/filtered/NestedFilteredIdsKNNIterator.java create mode 100644 src/test/java/org/opensearch/knn/common/Constants.java create mode 100644 src/test/java/org/opensearch/knn/index/query/filtered/FilteredIdsKNNIteratorTests.java create mode 100644 src/test/java/org/opensearch/knn/index/query/filtered/NestedFilteredIdsKNNIteratorTests.java diff --git a/src/main/java/org/opensearch/knn/index/query/KNNWeight.java b/src/main/java/org/opensearch/knn/index/query/KNNWeight.java index 180ce1b31..afa7e93b0 100644 --- a/src/main/java/org/opensearch/knn/index/query/KNNWeight.java +++ b/src/main/java/org/opensearch/knn/index/query/KNNWeight.java @@ -9,42 +9,40 @@ import org.apache.commons.lang.StringUtils; import org.apache.lucene.index.BinaryDocValues; import org.apache.lucene.index.DocValues; +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.FilterLeafReader; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.SegmentReader; +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.search.Explanation; import org.apache.lucene.search.FilteredDocIdSetIterator; import org.apache.lucene.search.HitQueue; import org.apache.lucene.search.ScoreDoc; +import org.apache.lucene.search.Scorer; +import org.apache.lucene.search.Weight; +import org.apache.lucene.store.FSDirectory; +import org.apache.lucene.store.FilterDirectory; import org.apache.lucene.util.BitSet; import org.apache.lucene.util.BitSetIterator; import org.apache.lucene.util.Bits; -import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.DocIdSetBuilder; import org.apache.lucene.util.FixedBitSet; +import org.opensearch.common.io.PathUtils; import org.opensearch.knn.common.KNNConstants; import org.opensearch.knn.index.KNNSettings; import org.opensearch.knn.index.SpaceType; -import org.opensearch.knn.index.codec.util.KNNVectorSerializer; -import org.opensearch.knn.index.codec.util.KNNVectorSerializerFactory; -import org.opensearch.knn.jni.JNIService; import org.opensearch.knn.index.memory.NativeMemoryAllocation; import org.opensearch.knn.index.memory.NativeMemoryCacheManager; import org.opensearch.knn.index.memory.NativeMemoryEntryContext; import org.opensearch.knn.index.memory.NativeMemoryLoadStrategy; +import org.opensearch.knn.index.query.filtered.FilteredIdsKNNIterator; +import org.opensearch.knn.index.query.filtered.NestedFilteredIdsKNNIterator; import org.opensearch.knn.index.util.KNNEngine; -import org.apache.lucene.index.FieldInfo; -import org.apache.lucene.index.FilterLeafReader; -import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.index.SegmentReader; -import org.apache.lucene.search.DocIdSetIterator; -import org.apache.lucene.search.Explanation; -import org.apache.lucene.search.Scorer; -import org.apache.lucene.search.Weight; -import org.apache.lucene.store.FSDirectory; -import org.apache.lucene.store.FilterDirectory; -import org.apache.lucene.util.DocIdSetBuilder; -import org.opensearch.common.io.PathUtils; import org.opensearch.knn.indices.ModelDao; import org.opensearch.knn.indices.ModelMetadata; +import org.opensearch.knn.jni.JNIService; import org.opensearch.knn.plugin.stats.KNNCounter; -import java.io.ByteArrayInputStream; import java.io.IOException; import java.nio.file.Path; import java.util.Arrays; @@ -306,33 +304,23 @@ private Map doANNSearch(final LeafReaderContext context, final i } private Map doExactSearch(final LeafReaderContext leafReaderContext, final int[] filterIdsArray) throws IOException { - final SegmentReader reader = (SegmentReader) FilterLeafReader.unwrap(leafReaderContext.reader()); - final FieldInfo fieldInfo = reader.getFieldInfos().fieldInfo(knnQuery.getField()); - float[] queryVector = this.knnQuery.getQueryVector(); try { - final BinaryDocValues values = DocValues.getBinary(leafReaderContext.reader(), fieldInfo.getName()); - final SpaceType spaceType = getSpaceType(fieldInfo); // Creating min heap and init with MAX DocID and Score as -INF. final HitQueue queue = new HitQueue(this.knnQuery.getK(), true); ScoreDoc topDoc = queue.top(); final Map docToScore = new HashMap<>(); - for (int filterId : filterIdsArray) { - int docId = values.advance(filterId); - final BytesRef value = values.binaryValue(); - final ByteArrayInputStream byteStream = new ByteArrayInputStream(value.bytes, value.offset, value.length); - final KNNVectorSerializer vectorSerializer = KNNVectorSerializerFactory.getSerializerByStreamContent(byteStream); - final float[] vector = vectorSerializer.byteToFloatArray(byteStream); - // Calculates a similarity score between the two vectors with a specified function. Higher similarity - // scores correspond to closer vectors. - float score = spaceType.getVectorSimilarityFunction().compare(queryVector, vector); - if (score > topDoc.score) { - topDoc.score = score; + FilteredIdsKNNIterator iterator = getFilteredKNNIterator(leafReaderContext, filterIdsArray); + int docId; + while ((docId = iterator.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) { + if (iterator.score() > topDoc.score) { + topDoc.score = iterator.score(); topDoc.doc = docId; // As the HitQueue is min heap, updating top will bring the doc with -INF score or worst score we // have seen till now on top. topDoc = queue.updateTop(); } } + // If scores are negative we will remove them. // This is done, because there can be negative values in the Heap as we init the heap with Score as -INF. // If filterIds < k, the some values in heap can have a negative score. @@ -352,6 +340,23 @@ private Map doExactSearch(final LeafReaderContext leafReaderCont return Collections.emptyMap(); } + private FilteredIdsKNNIterator getFilteredKNNIterator(final LeafReaderContext leafReaderContext, final int[] filterIdsArray) + throws IOException { + final SegmentReader reader = (SegmentReader) FilterLeafReader.unwrap(leafReaderContext.reader()); + final FieldInfo fieldInfo = reader.getFieldInfos().fieldInfo(knnQuery.getField()); + final BinaryDocValues values = DocValues.getBinary(leafReaderContext.reader(), fieldInfo.getName()); + final SpaceType spaceType = getSpaceType(fieldInfo); + return knnQuery.getParentsFilter() == null + ? new FilteredIdsKNNIterator(filterIdsArray, knnQuery.getQueryVector(), values, spaceType) + : new NestedFilteredIdsKNNIterator( + filterIdsArray, + knnQuery.getQueryVector(), + values, + spaceType, + knnQuery.getParentsFilter().getBitSet(leafReaderContext) + ); + } + private Scorer convertSearchResponseToScorer(final Map docsToScore) throws IOException { final int maxDoc = Collections.max(docsToScore.keySet()) + 1; final DocIdSetBuilder docIdSetBuilder = new DocIdSetBuilder(maxDoc); diff --git a/src/main/java/org/opensearch/knn/index/query/filtered/FilteredIdsKNNIterator.java b/src/main/java/org/opensearch/knn/index/query/filtered/FilteredIdsKNNIterator.java new file mode 100644 index 000000000..a286829d5 --- /dev/null +++ b/src/main/java/org/opensearch/knn/index/query/filtered/FilteredIdsKNNIterator.java @@ -0,0 +1,74 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.query.filtered; + +import org.apache.lucene.index.BinaryDocValues; +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.util.BytesRef; +import org.opensearch.knn.index.SpaceType; +import org.opensearch.knn.index.codec.util.KNNVectorSerializer; +import org.opensearch.knn.index.codec.util.KNNVectorSerializerFactory; + +import java.io.ByteArrayInputStream; +import java.io.IOException; + +/** + * Inspired by DiversifyingChildrenFloatKnnVectorQuery in lucene + * https://github.com/apache/lucene/blob/7b8aece125aabff2823626d5b939abf4747f63a7/lucene/join/src/java/org/apache/lucene/search/join/DiversifyingChildrenFloatKnnVectorQuery.java#L162 + * + * The class is used in KNNWeight to score filtered KNN field by iterating filterIdsArray. + */ +public class FilteredIdsKNNIterator { + // Array of doc ids to iterate + protected final int[] filterIdsArray; + protected final float[] queryVector; + protected final BinaryDocValues binaryDocValues; + protected final SpaceType spaceType; + protected float currentScore = Float.NEGATIVE_INFINITY; + protected int currentPos = 0; + + public FilteredIdsKNNIterator( + final int[] filterIdsArray, + final float[] queryVector, + final BinaryDocValues binaryDocValues, + final SpaceType spaceType + ) { + this.filterIdsArray = filterIdsArray; + this.queryVector = queryVector; + this.binaryDocValues = binaryDocValues; + this.spaceType = spaceType; + } + + /** + * Advance to the next doc and update score value with score of the next doc. + * DocIdSetIterator.NO_MORE_DOCS is returned when there is no more docs + * + * @return next doc id + */ + public int nextDoc() throws IOException { + if (currentPos >= filterIdsArray.length) { + return DocIdSetIterator.NO_MORE_DOCS; + } + int docId = binaryDocValues.advance(filterIdsArray[currentPos]); + currentScore = computeScore(); + currentPos++; + return docId; + } + + public float score() { + return currentScore; + } + + protected float computeScore() throws IOException { + final BytesRef value = binaryDocValues.binaryValue(); + final ByteArrayInputStream byteStream = new ByteArrayInputStream(value.bytes, value.offset, value.length); + final KNNVectorSerializer vectorSerializer = KNNVectorSerializerFactory.getSerializerByStreamContent(byteStream); + final float[] vector = vectorSerializer.byteToFloatArray(byteStream); + // Calculates a similarity score between the two vectors with a specified function. Higher similarity + // scores correspond to closer vectors. + return spaceType.getVectorSimilarityFunction().compare(queryVector, vector); + } +} diff --git a/src/main/java/org/opensearch/knn/index/query/filtered/NestedFilteredIdsKNNIterator.java b/src/main/java/org/opensearch/knn/index/query/filtered/NestedFilteredIdsKNNIterator.java new file mode 100644 index 000000000..d2e4d3e25 --- /dev/null +++ b/src/main/java/org/opensearch/knn/index/query/filtered/NestedFilteredIdsKNNIterator.java @@ -0,0 +1,59 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.query.filtered; + +import org.apache.lucene.index.BinaryDocValues; +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.util.BitSet; +import org.opensearch.knn.index.SpaceType; + +import java.io.IOException; + +/** + * This iterator iterates filterIdsArray to score. However, it dedupe docs per each parent doc + * of which ID is set in parentBitSet and only return best child doc with the highest score. + */ +public class NestedFilteredIdsKNNIterator extends FilteredIdsKNNIterator { + private final BitSet parentBitSet; + + public NestedFilteredIdsKNNIterator( + final int[] filterIdsArray, + final float[] queryVector, + final BinaryDocValues values, + final SpaceType spaceType, + final BitSet parentBitSet + ) { + super(filterIdsArray, queryVector, values, spaceType); + this.parentBitSet = parentBitSet; + } + + /** + * Advance to the next best child doc per parent and update score with the best score among child docs from the parent. + * DocIdSetIterator.NO_MORE_DOCS is returned when there is no more docs + * + * @return next best child doc id + */ + @Override + public int nextDoc() throws IOException { + if (currentPos >= filterIdsArray.length) { + return DocIdSetIterator.NO_MORE_DOCS; + } + currentScore = Float.NEGATIVE_INFINITY; + int currentParent = parentBitSet.nextSetBit(filterIdsArray[currentPos]); + int bestChild = -1; + while (currentPos < filterIdsArray.length && filterIdsArray[currentPos] < currentParent) { + binaryDocValues.advance(filterIdsArray[currentPos]); + float score = computeScore(); + if (score > currentScore) { + bestChild = filterIdsArray[currentPos]; + currentScore = score; + } + currentPos++; + } + + return bestChild; + } +} diff --git a/src/test/java/org/opensearch/knn/common/Constants.java b/src/test/java/org/opensearch/knn/common/Constants.java new file mode 100644 index 000000000..2580d2c9c --- /dev/null +++ b/src/test/java/org/opensearch/knn/common/Constants.java @@ -0,0 +1,11 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.common; + +public class Constants { + public static final String FIELD_FILTER = "filter"; + public static final String FIELD_TERM = "term"; +} diff --git a/src/test/java/org/opensearch/knn/index/NestedSearchIT.java b/src/test/java/org/opensearch/knn/index/NestedSearchIT.java index df5256fb2..7a81e2f45 100644 --- a/src/test/java/org/opensearch/knn/index/NestedSearchIT.java +++ b/src/test/java/org/opensearch/knn/index/NestedSearchIT.java @@ -10,6 +10,7 @@ import org.junit.After; import org.opensearch.client.Request; import org.opensearch.client.Response; +import org.opensearch.common.settings.Settings; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.rest.RestStatus; import org.opensearch.core.xcontent.XContentBuilder; @@ -18,7 +19,10 @@ import org.opensearch.knn.index.util.KNNEngine; import java.io.IOException; +import java.util.List; +import static org.opensearch.knn.common.Constants.FIELD_FILTER; +import static org.opensearch.knn.common.Constants.FIELD_TERM; import static org.opensearch.knn.common.KNNConstants.DIMENSION; import static org.opensearch.knn.common.KNNConstants.K; import static org.opensearch.knn.common.KNNConstants.KNN; @@ -39,8 +43,11 @@ public class NestedSearchIT extends KNNRestTestCase { private static final String INDEX_NAME = "test-index-nested-search"; - private static final String FIELD_NAME_NESTED = "test-nested"; - private static final String FIELD_NAME_VECTOR = "test-vector"; + private static final String FIELD_NAME_NESTED = "test_nested"; + private static final String FIELD_NAME_VECTOR = "test_vector"; + private static final String FIELD_NAME_PARKING = "parking"; + private static final String FIELD_VALUE_TRUE = "true"; + private static final String FIELD_VALUE_FALSE = "false"; private static final String PROPERTIES_FIELD = "properties"; private static final int EF_CONSTRUCTION = 128; private static final int M = 16; @@ -98,13 +105,70 @@ public void testNestedSearchWithFaiss_whenKIsTwo_thenReturnTwoResults() { assertEquals(2, parseTotalSearchHits(entity)); } + /** + * { + * "query": { + * "nested": { + * "path": "test_nested", + * "query": { + * "knn": { + * "test_nested.test_vector": { + * "vector": [ + * 1, 1, 1 + * ], + * "k": 3, + * "filter": { + * "term": { + * "parking": "true" + * } + * } + * } + * } + * } + * } + * } + * } + * + */ + @SneakyThrows + public void testNestedSearchWithFaiss_whenDoingExactSearch_thenReturnCorrectResults() { + createKnnIndex(3, KNNEngine.FAISS.getName()); + + for (int i = 1; i < 4; i++) { + float value = (float) i; + String doc = NestedKnnDocBuilder.create(FIELD_NAME_NESTED) + .addVectors( + FIELD_NAME_VECTOR, + new Float[] { value, value, value }, + new Float[] { value, value, value }, + new Float[] { value, value, value } + ) + .addTopLevelField(FIELD_NAME_PARKING, i % 2 == 1 ? FIELD_VALUE_TRUE : FIELD_VALUE_FALSE) + .build(); + addKnnDoc(INDEX_NAME, String.valueOf(i), doc); + } + refreshIndex(INDEX_NAME); + + // Make it as an exact search by setting the threshold larger than size of filteredIds(6) + updateIndexSettings(INDEX_NAME, Settings.builder().put(KNNSettings.ADVANCED_FILTERED_EXACT_SEARCH_THRESHOLD, 100)); + + Float[] queryVector = { 3f, 3f, 3f }; + Response response = queryNestedField(INDEX_NAME, 3, queryVector, FIELD_NAME_PARKING, FIELD_VALUE_TRUE); + String entity = EntityUtils.toString(response.getEntity()); + List docIds = parseIds(entity); + assertEquals(2, docIds.size()); + assertEquals("3", docIds.get(0)); + assertEquals("1", docIds.get(1)); + assertEquals(2, parseTotalSearchHits(entity)); + } + /** * { * "properties": { - * "test-nested": { + * "test_nested": { * "type": "nested", * "properties": { - * "test-vector": { + * "test_vector": { * "type": "knn_vector", * "dimension": 3, * "method": { @@ -152,12 +216,29 @@ private void createKnnIndex(final int dimension, final String engine) throws Exc } private Response queryNestedField(final String index, final int k, final Object[] vector) throws IOException { + return queryNestedField(index, k, vector, null, null); + } + + private Response queryNestedField( + final String index, + final int k, + final Object[] vector, + final String filterName, + final String filterValue + ) throws IOException { XContentBuilder builder = XContentFactory.jsonBuilder().startObject().startObject(QUERY); builder.startObject(TYPE_NESTED); builder.field(PATH, FIELD_NAME_NESTED); builder.startObject(QUERY).startObject(KNN).startObject(FIELD_NAME_NESTED + "." + FIELD_NAME_VECTOR); builder.field(VECTOR, vector); builder.field(K, k); + if (filterName != null && filterValue != null) { + builder.startObject(FIELD_FILTER); + builder.startObject(FIELD_TERM); + builder.field(filterName, filterValue); + builder.endObject(); + builder.endObject(); + } builder.endObject().endObject().endObject().endObject().endObject().endObject(); Request request = new Request("POST", "/" + index + "/_search"); diff --git a/src/test/java/org/opensearch/knn/index/query/KNNWeightTests.java b/src/test/java/org/opensearch/knn/index/query/KNNWeightTests.java index fa0613921..f4b44a4f8 100644 --- a/src/test/java/org/opensearch/knn/index/query/KNNWeightTests.java +++ b/src/test/java/org/opensearch/knn/index/query/KNNWeightTests.java @@ -601,7 +601,53 @@ public void testANNWithFilterQuery_whenEmptyFilterIds_thenReturnEarly() { } @SneakyThrows - public void testANNWithParentsFilter_whenSet_thenBitSetIsPassedToJNI() { + public void testANNWithParentsFilter_whenExactSearch_thenSuccess() { + SegmentReader reader = getMockedSegmentReader(); + + final LeafReaderContext leafReaderContext = mock(LeafReaderContext.class); + when(leafReaderContext.reader()).thenReturn(reader); + + // We will have 0, 1 for filteredIds and 2 will be the parent id for both of them + final Scorer filterScorer = mock(Scorer.class); + when(filterScorer.iterator()).thenReturn(DocIdSetIterator.all(2)); + when(reader.maxDoc()).thenReturn(2); + + // Query vector is {1.8f, 2.4f}, therefore, second vector {1.9f, 2.5f} should be returned in a result + final List vectors = Arrays.asList(new float[] { 0.1f, 0.3f }, new float[] { 1.9f, 2.5f }); + final List byteRefs = vectors.stream() + .map(vector -> new BytesRef(new KNNVectorAsArraySerializer().floatToByteArray(vector))) + .collect(Collectors.toList()); + final BinaryDocValues binaryDocValues = mock(BinaryDocValues.class); + when(binaryDocValues.binaryValue()).thenReturn(byteRefs.get(0), byteRefs.get(1)); + when(binaryDocValues.advance(anyInt())).thenReturn(0, 1); + when(reader.getBinaryDocValues(FIELD_NAME)).thenReturn(binaryDocValues); + + // Parent ID 2 in bitset is 100 which is 4 + FixedBitSet parentIds = new FixedBitSet(new long[] { 4 }, 3); + BitSetProducer parentFilter = mock(BitSetProducer.class); + when(parentFilter.getBitSet(leafReaderContext)).thenReturn(parentIds); + + final Weight filterQueryWeight = mock(Weight.class); + when(filterQueryWeight.scorer(leafReaderContext)).thenReturn(filterScorer); + + final KNNQuery query = new KNNQuery(FIELD_NAME, QUERY_VECTOR, K, INDEX_NAME, FILTER_QUERY, parentFilter); + final KNNWeight knnWeight = new KNNWeight(query, 0.0f, filterQueryWeight); + + // Execute + final KNNScorer knnScorer = (KNNScorer) knnWeight.scorer(leafReaderContext); + + // Verify + final List expectedScores = vectors.stream() + .map(vector -> SpaceType.L2.getVectorSimilarityFunction().compare(QUERY_VECTOR, vector)) + .collect(Collectors.toList()); + final DocIdSetIterator docIdSetIterator = knnScorer.iterator(); + assertEquals(1, docIdSetIterator.nextDoc()); + assertEquals(expectedScores.get(1), knnScorer.score(), 0.01f); + assertEquals(NO_MORE_DOCS, docIdSetIterator.nextDoc()); + } + + @SneakyThrows + public void testANNWithParentsFilter_whenDoingANN_thenBitSetIsPassedToJNI() { SegmentReader reader = getMockedSegmentReader(); final LeafReaderContext leafReaderContext = mock(LeafReaderContext.class); when(leafReaderContext.reader()).thenReturn(reader); @@ -636,11 +682,7 @@ private SegmentReader getMockedSegmentReader() { when(reader.maxDoc()).thenReturn(1); // Prepare live docs - int liveDocId = 0; - final Bits liveDocsBits = mock(Bits.class); - when(liveDocsBits.get(liveDocId)).thenReturn(true); - when(liveDocsBits.length()).thenReturn(1); - when(reader.getLiveDocs()).thenReturn(liveDocsBits); + when(reader.getLiveDocs()).thenReturn(null); // Prepare directory final Path path = mock(Path.class); @@ -667,10 +709,14 @@ private SegmentReader getMockedSegmentReader() { final SegmentCommitInfo segmentCommitInfo = new SegmentCommitInfo(segmentInfo, 0, 0, 0, 0, 0, new byte[StringHelper.ID_LENGTH]); when(reader.getSegmentInfo()).thenReturn(segmentCommitInfo); - // Prepare fieldInfos - final Map attributesMap = ImmutableMap.of(KNN_ENGINE, KNNEngine.FAISS.getName()); + // Prepare fieldInfo + final Map attributesMap = ImmutableMap.of(KNN_ENGINE, KNNEngine.FAISS.getName(), SPACE_TYPE, SpaceType.L2.name()); final FieldInfo fieldInfo = mock(FieldInfo.class); when(fieldInfo.attributes()).thenReturn(attributesMap); + when(fieldInfo.getAttribute(SPACE_TYPE)).thenReturn(SpaceType.L2.name()); + when(fieldInfo.getName()).thenReturn(FIELD_NAME); + + // Prepare fieldInfos final FieldInfos fieldInfos = mock(FieldInfos.class); when(fieldInfos.fieldInfo(any())).thenReturn(fieldInfo); when(reader.getFieldInfos()).thenReturn(fieldInfos); diff --git a/src/test/java/org/opensearch/knn/index/query/filtered/FilteredIdsKNNIteratorTests.java b/src/test/java/org/opensearch/knn/index/query/filtered/FilteredIdsKNNIteratorTests.java new file mode 100644 index 000000000..cfb66662e --- /dev/null +++ b/src/test/java/org/opensearch/knn/index/query/filtered/FilteredIdsKNNIteratorTests.java @@ -0,0 +1,56 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.query.filtered; + +import lombok.SneakyThrows; +import org.apache.lucene.index.BinaryDocValues; +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.util.BytesRef; +import org.opensearch.knn.KNNTestCase; +import org.opensearch.knn.index.SpaceType; +import org.opensearch.knn.index.codec.util.KNNVectorAsArraySerializer; + +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class FilteredIdsKNNIteratorTests extends KNNTestCase { + @SneakyThrows + public void testNextDoc_whenCalled_IterateAllDocs() { + final SpaceType spaceType = SpaceType.L2; + final float[] queryVector = { 1.0f, 2.0f, 3.0f }; + final int[] filterIds = { 1, 2, 3 }; + final List dataVectors = Arrays.asList( + new float[] { 11.0f, 12.0f, 13.0f }, + new float[] { 14.0f, 15.0f, 16.0f }, + new float[] { 17.0f, 18.0f, 19.0f } + ); + final List expectedScores = dataVectors.stream() + .map(vector -> spaceType.getVectorSimilarityFunction().compare(queryVector, vector)) + .collect(Collectors.toList()); + + BinaryDocValues values = mock(BinaryDocValues.class); + final List byteRefs = dataVectors.stream() + .map(vector -> new BytesRef(new KNNVectorAsArraySerializer().floatToByteArray(vector))) + .collect(Collectors.toList()); + when(values.binaryValue()).thenReturn(byteRefs.get(0), byteRefs.get(1), byteRefs.get(2)); + + for (int id : filterIds) { + when(values.advance(id)).thenReturn(id); + } + + // Execute and verify + FilteredIdsKNNIterator iterator = new FilteredIdsKNNIterator(filterIds, queryVector, values, spaceType); + for (int i = 0; i < filterIds.length; i++) { + assertEquals(filterIds[i], iterator.nextDoc()); + assertEquals(expectedScores.get(i), (Float) iterator.score()); + } + assertEquals(DocIdSetIterator.NO_MORE_DOCS, iterator.nextDoc()); + } +} diff --git a/src/test/java/org/opensearch/knn/index/query/filtered/NestedFilteredIdsKNNIteratorTests.java b/src/test/java/org/opensearch/knn/index/query/filtered/NestedFilteredIdsKNNIteratorTests.java new file mode 100644 index 000000000..90d56fddc --- /dev/null +++ b/src/test/java/org/opensearch/knn/index/query/filtered/NestedFilteredIdsKNNIteratorTests.java @@ -0,0 +1,62 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.query.filtered; + +import junit.framework.TestCase; +import lombok.SneakyThrows; +import org.apache.lucene.index.BinaryDocValues; +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.util.BitSet; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.FixedBitSet; +import org.opensearch.knn.index.SpaceType; +import org.opensearch.knn.index.codec.util.KNNVectorAsArraySerializer; + +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class NestedFilteredIdsKNNIteratorTests extends TestCase { + @SneakyThrows + public void testNextDoc_whenIterate_ReturnBestChildDocsPerParent() { + final SpaceType spaceType = SpaceType.L2; + final float[] queryVector = { 1.0f, 2.0f, 3.0f }; + final int[] filterIds = { 0, 2, 3 }; + // Parent id for 0 -> 1 + // Parent id for 2, 3 -> 4 + // In bit representation, it is 10010. In long, it is 18. + final BitSet parentBitSet = new FixedBitSet(new long[] { 18 }, 5); + final List dataVectors = Arrays.asList( + new float[] { 11.0f, 12.0f, 13.0f }, + new float[] { 17.0f, 18.0f, 19.0f }, + new float[] { 14.0f, 15.0f, 16.0f } + ); + final List expectedScores = dataVectors.stream() + .map(vector -> spaceType.getVectorSimilarityFunction().compare(queryVector, vector)) + .collect(Collectors.toList()); + + BinaryDocValues values = mock(BinaryDocValues.class); + final List byteRefs = dataVectors.stream() + .map(vector -> new BytesRef(new KNNVectorAsArraySerializer().floatToByteArray(vector))) + .collect(Collectors.toList()); + when(values.binaryValue()).thenReturn(byteRefs.get(0), byteRefs.get(1), byteRefs.get(2)); + + for (int id : filterIds) { + when(values.advance(id)).thenReturn(id); + } + + // Execute and verify + NestedFilteredIdsKNNIterator iterator = new NestedFilteredIdsKNNIterator(filterIds, queryVector, values, spaceType, parentBitSet); + assertEquals(filterIds[0], iterator.nextDoc()); + assertEquals(expectedScores.get(0), iterator.score()); + assertEquals(filterIds[2], iterator.nextDoc()); + assertEquals(expectedScores.get(2), iterator.score()); + assertEquals(DocIdSetIterator.NO_MORE_DOCS, iterator.nextDoc()); + } +} diff --git a/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java b/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java index d32ce419d..5ac053fb3 100644 --- a/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java +++ b/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java @@ -694,6 +694,16 @@ protected int parseHits(String searchResponseBody) throws IOException { return ((List) responseMap.get("hits")).size(); } + protected List parseIds(String searchResponseBody) throws IOException { + @SuppressWarnings("unchecked") + List hits = (List) ((Map) createParser( + MediaTypeRegistry.getDefaultMediaType().xContent(), + searchResponseBody + ).map().get("hits")).get("hits"); + + return hits.stream().map(hit -> (String) ((Map) hit).get("_id")).collect(Collectors.toList()); + } + /** * Get the total number of graphs in the cache across all nodes */ From 4d9da8d6c4583961e66668679b5de04af280a743 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Mon, 22 Jan 2024 10:16:53 -0800 Subject: [PATCH 182/416] Few changes in integration test (#1395) (#1404) Signed-off-by: Heemin Kim (cherry picked from commit 6abec195426344aa06a5905d721fc1fb005e3257) Co-authored-by: Heemin Kim --- .../index/AdvancedFilteringUseCasesIT.java | 6 ++-- .../opensearch/knn/index/NestedSearchIT.java | 35 +++++++++---------- 2 files changed, 21 insertions(+), 20 deletions(-) diff --git a/src/test/java/org/opensearch/knn/index/AdvancedFilteringUseCasesIT.java b/src/test/java/org/opensearch/knn/index/AdvancedFilteringUseCasesIT.java index 9ce994a78..b559b5760 100644 --- a/src/test/java/org/opensearch/knn/index/AdvancedFilteringUseCasesIT.java +++ b/src/test/java/org/opensearch/knn/index/AdvancedFilteringUseCasesIT.java @@ -448,14 +448,16 @@ public void testFiltering_whenNonNestedKNNAndNestedFilterAndNonNestedFieldWithNe private void validateFilterSearch(final String query, final String engine) throws IOException { String response = EntityUtils.toString(performSearch(INDEX_NAME, query).getEntity()); // Validate number of documents returned as the expected number of documents - Assert.assertEquals("For engine " + engine + " : ", DOCUMENT_IN_RESPONSE, parseHits(response)); + Assert.assertEquals("For engine " + engine + ", hits: ", DOCUMENT_IN_RESPONSE, parseHits(response)); + Assert.assertEquals("For engine " + engine + ", totalSearchHits: ", k, parseTotalSearchHits(response)); if (KNNEngine.getEngine(engine) == KNNEngine.FAISS) { // Update the filter threshold to 0 to ensure that we are hitting ANN Search use case for FAISS updateIndexSettings(INDEX_NAME, Settings.builder().put(KNNSettings.ADVANCED_FILTERED_EXACT_SEARCH_THRESHOLD, 0)); response = EntityUtils.toString(performSearch(INDEX_NAME, query).getEntity()); // Validate number of documents returned as the expected number of documents - Assert.assertEquals("For engine " + engine + " with ANN search :", DOCUMENT_IN_RESPONSE, parseHits(response)); + Assert.assertEquals("For engine " + engine + ", hits with ANN search :", DOCUMENT_IN_RESPONSE, parseHits(response)); + Assert.assertEquals("For engine " + engine + ", totalSearchHits with ANN search :", k, parseTotalSearchHits(response)); } } diff --git a/src/test/java/org/opensearch/knn/index/NestedSearchIT.java b/src/test/java/org/opensearch/knn/index/NestedSearchIT.java index 7a81e2f45..e4d828a01 100644 --- a/src/test/java/org/opensearch/knn/index/NestedSearchIT.java +++ b/src/test/java/org/opensearch/knn/index/NestedSearchIT.java @@ -63,17 +63,16 @@ public final void cleanUp() { public void testNestedSearchWithLucene_whenKIsTwo_thenReturnTwoResults() { createKnnIndex(2, KNNEngine.LUCENE.getName()); - String doc1 = NestedKnnDocBuilder.create(FIELD_NAME_NESTED) - .addVectors(FIELD_NAME_VECTOR, new Float[] { 1f, 1f }, new Float[] { 1f, 1f }) - .build(); - addKnnDoc(INDEX_NAME, "1", doc1); - - String doc2 = NestedKnnDocBuilder.create(FIELD_NAME_NESTED) - .addVectors(FIELD_NAME_VECTOR, new Float[] { 2f, 2f }, new Float[] { 2f, 2f }) - .build(); - addKnnDoc(INDEX_NAME, "2", doc2); + int totalDocCount = 15; + for (int i = 0; i < totalDocCount; i++) { + String doc = NestedKnnDocBuilder.create(FIELD_NAME_NESTED) + .addVectors(FIELD_NAME_VECTOR, new Float[] { (float) i, (float) i }, new Float[] { (float) i, (float) i }) + .build(); + addKnnDoc(INDEX_NAME, String.valueOf(i), doc); + } refreshIndex(INDEX_NAME); + forceMergeKnnIndex(INDEX_NAME); Float[] queryVector = { 1f, 1f }; Response response = queryNestedField(INDEX_NAME, 2, queryVector); @@ -86,17 +85,16 @@ public void testNestedSearchWithLucene_whenKIsTwo_thenReturnTwoResults() { public void testNestedSearchWithFaiss_whenKIsTwo_thenReturnTwoResults() { createKnnIndex(2, KNNEngine.FAISS.getName()); - String doc1 = NestedKnnDocBuilder.create(FIELD_NAME_NESTED) - .addVectors(FIELD_NAME_VECTOR, new Float[] { 1f, 1f }, new Float[] { 1f, 1f }) - .build(); - addKnnDoc(INDEX_NAME, "1", doc1); - - String doc2 = NestedKnnDocBuilder.create(FIELD_NAME_NESTED) - .addVectors(FIELD_NAME_VECTOR, new Float[] { 2f, 2f }, new Float[] { 2f, 2f }) - .build(); - addKnnDoc(INDEX_NAME, "2", doc2); + int totalDocCount = 15; + for (int i = 0; i < totalDocCount; i++) { + String doc = NestedKnnDocBuilder.create(FIELD_NAME_NESTED) + .addVectors(FIELD_NAME_VECTOR, new Float[] { (float) i, (float) i }, new Float[] { (float) i, (float) i }) + .build(); + addKnnDoc(INDEX_NAME, String.valueOf(i), doc); + } refreshIndex(INDEX_NAME); + forceMergeKnnIndex(INDEX_NAME); Float[] queryVector = { 1f, 1f }; Response response = queryNestedField(INDEX_NAME, 2, queryVector); @@ -148,6 +146,7 @@ public void testNestedSearchWithFaiss_whenDoingExactSearch_thenReturnCorrectResu addKnnDoc(INDEX_NAME, String.valueOf(i), doc); } refreshIndex(INDEX_NAME); + forceMergeKnnIndex(INDEX_NAME); // Make it as an exact search by setting the threshold larger than size of filteredIds(6) updateIndexSettings(INDEX_NAME, Settings.builder().put(KNNSettings.ADVANCED_FILTERED_EXACT_SEARCH_THRESHOLD, 100)); From 8513952ee7217604c201281ff14d279e0671f380 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Tue, 23 Jan 2024 10:15:33 -0800 Subject: [PATCH 183/416] Fix KNNScorer to apply boost (#1403) (#1405) * apply boost Signed-off-by: panguixin * add change log Signed-off-by: panguixin --------- Signed-off-by: panguixin (cherry picked from commit fcbfef13668e773befd94579f41a8a82943ffda0) Co-authored-by: panguixin --- CHANGELOG.md | 1 + .../opensearch/knn/index/query/KNNScorer.java | 2 +- .../knn/index/query/KNNWeightTests.java | 35 +++++++++++-------- 3 files changed, 23 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 961ab62af..f57bc71fe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), * Allow nested knn field mapping when train model [#1318](https://github.com/opensearch-project/k-NN/pull/1318) * Properly designate model state for actively training models when nodes crash or leave cluster [#1317](https://github.com/opensearch-project/k-NN/pull/1317) * Fix script score queries not getting cached [#1367](https://github.com/opensearch-project/k-NN/pull/1367) +* Fix KNNScorer to apply boost [#1403](https://github.com/opensearch-project/k-NN/pull/1403) ### Infrastructure * Upgrade gradle to 8.4 [1289](https://github.com/opensearch-project/k-NN/pull/1289) * Refactor security testing to install from individual components [#1307](https://github.com/opensearch-project/k-NN/pull/1307) diff --git a/src/main/java/org/opensearch/knn/index/query/KNNScorer.java b/src/main/java/org/opensearch/knn/index/query/KNNScorer.java index 3e5c8fff6..02dc86e80 100644 --- a/src/main/java/org/opensearch/knn/index/query/KNNScorer.java +++ b/src/main/java/org/opensearch/knn/index/query/KNNScorer.java @@ -49,7 +49,7 @@ public float score() { assert docID() != DocIdSetIterator.NO_MORE_DOCS; Float score = scores.get(docID()); if (score == null) throw new RuntimeException("Null score for the docID: " + docID()); - return score; + return score * boost; } @Override diff --git a/src/test/java/org/opensearch/knn/index/query/KNNWeightTests.java b/src/test/java/org/opensearch/knn/index/query/KNNWeightTests.java index f4b44a4f8..5d49f052c 100644 --- a/src/test/java/org/opensearch/knn/index/query/KNNWeightTests.java +++ b/src/test/java/org/opensearch/knn/index/query/KNNWeightTests.java @@ -169,7 +169,8 @@ public void testQueryScoreForFaissWithModel() { when(modelDao.getMetadata(eq("modelId"))).thenReturn(modelMetadata); KNNWeight.initialize(modelDao); - final KNNWeight knnWeight = new KNNWeight(query, 0.0f); + final float boost = (float) randomDoubleBetween(0, 10, true); + final KNNWeight knnWeight = new KNNWeight(query, boost); final LeafReaderContext leafReaderContext = mock(LeafReaderContext.class); final SegmentReader reader = mock(SegmentReader.class); @@ -214,7 +215,7 @@ public void testQueryScoreForFaissWithModel() { final Map translatedScores = getTranslatedScores(scoreTranslator); for (int docId = docIdSetIterator.nextDoc(); docId != NO_MORE_DOCS; docId = docIdSetIterator.nextDoc()) { actualDocIds.add(docId); - assertEquals(translatedScores.get(docId), knnScorer.score(), 0.01f); + assertEquals(translatedScores.get(docId) * boost, knnScorer.score(), 0.01f); } assertEquals(docIdSetIterator.cost(), actualDocIds.size()); assertTrue(Comparators.isInOrder(actualDocIds, Comparator.naturalOrder())); @@ -364,7 +365,8 @@ public void testANNWithFilterQuery_whenDoingANN_thenSuccess() { // Just to make sure that we are not hitting the exact search condition when(filterScorer.iterator()).thenReturn(DocIdSetIterator.all(filterDocIds.length + 1)); - final KNNWeight knnWeight = new KNNWeight(query, 0.0f, filterQueryWeight); + final float boost = (float) randomDoubleBetween(0, 10, true); + final KNNWeight knnWeight = new KNNWeight(query, boost, filterQueryWeight); final FSDirectory directory = mock(FSDirectory.class); when(reader.directory()).thenReturn(directory); @@ -408,7 +410,7 @@ public void testANNWithFilterQuery_whenDoingANN_thenSuccess() { final Map translatedScores = getTranslatedScores(SpaceType.L2::scoreTranslation); for (int docId = docIdSetIterator.nextDoc(); docId != NO_MORE_DOCS; docId = docIdSetIterator.nextDoc()) { actualDocIds.add(docId); - assertEquals(translatedScores.get(docId), knnScorer.score(), 0.01f); + assertEquals(translatedScores.get(docId) * boost, knnScorer.score(), 0.01f); } assertEquals(docIdSetIterator.cost(), actualDocIds.size()); assertTrue(Comparators.isInOrder(actualDocIds, Comparator.naturalOrder())); @@ -433,7 +435,8 @@ public void testANNWithFilterQuery_whenExactSearch_thenSuccess() { when(reader.getLiveDocs()).thenReturn(liveDocsBits); when(liveDocsBits.get(filterDocId)).thenReturn(true); - final KNNWeight knnWeight = new KNNWeight(query, 0.0f, filterQueryWeight); + final float boost = (float) randomDoubleBetween(0, 10, true); + final KNNWeight knnWeight = new KNNWeight(query, boost, filterQueryWeight); final Map attributesMap = ImmutableMap.of(KNN_ENGINE, KNNEngine.FAISS.getName(), SPACE_TYPE, SpaceType.L2.name()); final FieldInfos fieldInfos = mock(FieldInfos.class); final FieldInfo fieldInfo = mock(FieldInfo.class); @@ -457,7 +460,7 @@ public void testANNWithFilterQuery_whenExactSearch_thenSuccess() { final List actualDocIds = new ArrayList<>(); for (int docId = docIdSetIterator.nextDoc(); docId != NO_MORE_DOCS; docId = docIdSetIterator.nextDoc()) { actualDocIds.add(docId); - assertEquals(EXACT_SEARCH_DOC_ID_TO_SCORES.get(docId), knnScorer.score(), 0.01f); + assertEquals(EXACT_SEARCH_DOC_ID_TO_SCORES.get(docId) * boost, knnScorer.score(), 0.01f); } assertEquals(docIdSetIterator.cost(), actualDocIds.size()); assertTrue(Comparators.isInOrder(actualDocIds, Comparator.naturalOrder())); @@ -483,7 +486,8 @@ public void testANNWithFilterQuery_whenExactSearchAndThresholdComputations_thenS when(reader.getLiveDocs()).thenReturn(liveDocsBits); when(liveDocsBits.get(filterDocId)).thenReturn(true); - final KNNWeight knnWeight = new KNNWeight(query, 0.0f, filterQueryWeight); + final float boost = (float) randomDoubleBetween(0, 10, true); + final KNNWeight knnWeight = new KNNWeight(query, boost, filterQueryWeight); final Map attributesMap = ImmutableMap.of(KNN_ENGINE, KNNEngine.FAISS.getName(), SPACE_TYPE, SpaceType.L2.name()); final FieldInfos fieldInfos = mock(FieldInfos.class); final FieldInfo fieldInfo = mock(FieldInfo.class); @@ -507,7 +511,7 @@ public void testANNWithFilterQuery_whenExactSearchAndThresholdComputations_thenS final List actualDocIds = new ArrayList<>(); for (int docId = docIdSetIterator.nextDoc(); docId != NO_MORE_DOCS; docId = docIdSetIterator.nextDoc()) { actualDocIds.add(docId); - assertEquals(EXACT_SEARCH_DOC_ID_TO_SCORES.get(docId), knnScorer.score(), 0.01f); + assertEquals(EXACT_SEARCH_DOC_ID_TO_SCORES.get(docId) * boost, knnScorer.score(), 0.01f); } assertEquals(docIdSetIterator.cost(), actualDocIds.size()); assertTrue(Comparators.isInOrder(actualDocIds, Comparator.naturalOrder())); @@ -543,7 +547,8 @@ public void testANNWithFilterQuery_whenExactSearchViaThresholdSetting_thenSucces final KNNQuery query = new KNNQuery(FIELD_NAME, QUERY_VECTOR, k, INDEX_NAME, FILTER_QUERY, null); - final KNNWeight knnWeight = new KNNWeight(query, 0.0f, filterQueryWeight); + final float boost = (float) randomDoubleBetween(0, 10, true); + final KNNWeight knnWeight = new KNNWeight(query, boost, filterQueryWeight); final Map attributesMap = ImmutableMap.of(KNN_ENGINE, KNNEngine.FAISS.getName(), SPACE_TYPE, SpaceType.L2.name()); final FieldInfos fieldInfos = mock(FieldInfos.class); final FieldInfo fieldInfo = mock(FieldInfo.class); @@ -567,7 +572,7 @@ public void testANNWithFilterQuery_whenExactSearchViaThresholdSetting_thenSucces final List actualDocIds = new ArrayList<>(); for (int docId = docIdSetIterator.nextDoc(); docId != NO_MORE_DOCS; docId = docIdSetIterator.nextDoc()) { actualDocIds.add(docId); - assertEquals(EXACT_SEARCH_DOC_ID_TO_SCORES.get(docId), knnScorer.score(), 0.01f); + assertEquals(EXACT_SEARCH_DOC_ID_TO_SCORES.get(docId) * boost, knnScorer.score(), 0.01f); } assertEquals(docIdSetIterator.cost(), actualDocIds.size()); assertTrue(Comparators.isInOrder(actualDocIds, Comparator.naturalOrder())); @@ -631,7 +636,8 @@ public void testANNWithParentsFilter_whenExactSearch_thenSuccess() { when(filterQueryWeight.scorer(leafReaderContext)).thenReturn(filterScorer); final KNNQuery query = new KNNQuery(FIELD_NAME, QUERY_VECTOR, K, INDEX_NAME, FILTER_QUERY, parentFilter); - final KNNWeight knnWeight = new KNNWeight(query, 0.0f, filterQueryWeight); + final float boost = (float) randomDoubleBetween(0, 10, true); + final KNNWeight knnWeight = new KNNWeight(query, boost, filterQueryWeight); // Execute final KNNScorer knnScorer = (KNNScorer) knnWeight.scorer(leafReaderContext); @@ -642,7 +648,7 @@ public void testANNWithParentsFilter_whenExactSearch_thenSuccess() { .collect(Collectors.toList()); final DocIdSetIterator docIdSetIterator = knnScorer.iterator(); assertEquals(1, docIdSetIterator.nextDoc()); - assertEquals(expectedScores.get(1), knnScorer.score(), 0.01f); + assertEquals(expectedScores.get(1) * boost, knnScorer.score(), 0.01f); assertEquals(NO_MORE_DOCS, docIdSetIterator.nextDoc()); } @@ -733,7 +739,8 @@ private void testQueryScore( .thenReturn(getKNNQueryResults()); final KNNQuery query = new KNNQuery(FIELD_NAME, QUERY_VECTOR, K, INDEX_NAME, (BitSetProducer) null); - final KNNWeight knnWeight = new KNNWeight(query, 0.0f); + final float boost = (float) randomDoubleBetween(0, 10, true); + final KNNWeight knnWeight = new KNNWeight(query, boost); final LeafReaderContext leafReaderContext = mock(LeafReaderContext.class); final SegmentReader reader = mock(SegmentReader.class); @@ -777,7 +784,7 @@ private void testQueryScore( final Map translatedScores = getTranslatedScores(scoreTranslator); for (int docId = docIdSetIterator.nextDoc(); docId != NO_MORE_DOCS; docId = docIdSetIterator.nextDoc()) { actualDocIds.add(docId); - assertEquals(translatedScores.get(docId), knnScorer.score(), 0.01f); + assertEquals(translatedScores.get(docId) * boost, knnScorer.score(), 0.01f); } assertEquals(docIdSetIterator.cost(), actualDocIds.size()); assertTrue(Comparators.isInOrder(actualDocIds, Comparator.naturalOrder())); From adb1f736830b05a9c0030ded5225973b3fe231ca Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Tue, 23 Jan 2024 12:04:17 -0800 Subject: [PATCH 184/416] Fix equals and hashCode methods for KNNQuery and KNNQueryBuilder (#1407) Signed-off-by: panguixin (cherry picked from commit 89fc2677da4cd6a9e918a8787a3fe8197c9acfec) --- CHANGELOG.md | 1 + .../org/opensearch/knn/index/query/KNNQuery.java | 12 +++++++++--- .../opensearch/knn/index/query/KNNQueryBuilder.java | 11 ++++++++--- .../knn/index/query/KNNQueryBuilderTests.java | 8 ++++---- 4 files changed, 22 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f57bc71fe..8291a4953 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), * Properly designate model state for actively training models when nodes crash or leave cluster [#1317](https://github.com/opensearch-project/k-NN/pull/1317) * Fix script score queries not getting cached [#1367](https://github.com/opensearch-project/k-NN/pull/1367) * Fix KNNScorer to apply boost [#1403](https://github.com/opensearch-project/k-NN/pull/1403) +* Fix equals and hashCode methods for KNNQuery and KNNQueryBuilder [#1397](https://github.com/opensearch-project/k-NN/pull/1397) ### Infrastructure * Upgrade gradle to 8.4 [1289](https://github.com/opensearch-project/k-NN/pull/1289) * Refactor security testing to install from individual components [#1307](https://github.com/opensearch-project/k-NN/pull/1307) diff --git a/src/main/java/org/opensearch/knn/index/query/KNNQuery.java b/src/main/java/org/opensearch/knn/index/query/KNNQuery.java index 74a289994..9c78b18a1 100644 --- a/src/main/java/org/opensearch/knn/index/query/KNNQuery.java +++ b/src/main/java/org/opensearch/knn/index/query/KNNQuery.java @@ -5,6 +5,8 @@ package org.opensearch.knn.index.query; +import java.util.Arrays; +import java.util.Objects; import lombok.Getter; import lombok.Setter; import org.apache.lucene.search.BooleanClause; @@ -127,7 +129,7 @@ public String toString(String field) { @Override public int hashCode() { - return field.hashCode() ^ queryVector.hashCode() ^ k; + return Objects.hash(field, Arrays.hashCode(queryVector), k, indexName, filterQuery); } @Override @@ -136,6 +138,10 @@ public boolean equals(Object other) { } private boolean equalsTo(KNNQuery other) { - return this.field.equals(other.getField()) && this.queryVector.equals(other.getQueryVector()) && this.k == other.getK(); + return Objects.equals(field, other.field) + && Arrays.equals(queryVector, other.queryVector) + && Objects.equals(k, other.k) + && Objects.equals(indexName, other.indexName) + && Objects.equals(filterQuery, other.filterQuery); } -}; +} diff --git a/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java b/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java index 0fa3835ac..4779101d4 100644 --- a/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java +++ b/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java @@ -5,6 +5,7 @@ package org.opensearch.knn.index.query; +import java.util.Arrays; import lombok.extern.log4j.Log4j2; import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.commons.lang.StringUtils; @@ -46,7 +47,7 @@ public class KNNQueryBuilder extends AbstractQueryBuilder { public static final ParseField K_FIELD = new ParseField("k"); public static final ParseField FILTER_FIELD = new ParseField("filter"); public static final ParseField IGNORE_UNMAPPED_FIELD = new ParseField("ignore_unmapped"); - public static int K_MAX = 10000; + public static final int K_MAX = 10000; /** * The name for the knn query */ @@ -376,12 +377,16 @@ private ModelMetadata getModelMetadataForField(KNNVectorFieldMapper.KNNVectorFie @Override protected boolean doEquals(KNNQueryBuilder other) { - return Objects.equals(fieldName, other.fieldName) && Objects.equals(vector, other.vector) && Objects.equals(k, other.k); + return Objects.equals(fieldName, other.fieldName) + && Arrays.equals(vector, other.vector) + && Objects.equals(k, other.k) + && Objects.equals(filter, other.filter) + && Objects.equals(ignoreUnmapped, other.ignoreUnmapped); } @Override protected int doHashCode() { - return Objects.hash(fieldName, vector, k); + return Objects.hash(fieldName, Arrays.hashCode(vector), k, filter, ignoreUnmapped); } @Override diff --git a/src/test/java/org/opensearch/knn/index/query/KNNQueryBuilderTests.java b/src/test/java/org/opensearch/knn/index/query/KNNQueryBuilderTests.java index 5e62abec2..8c045ea74 100644 --- a/src/test/java/org/opensearch/knn/index/query/KNNQueryBuilderTests.java +++ b/src/test/java/org/opensearch/knn/index/query/KNNQueryBuilderTests.java @@ -90,7 +90,7 @@ public void testEmptyVector() { expectThrows(IllegalArgumentException.class, () -> new KNNQueryBuilder(FIELD_NAME, queryVector1, K)); } - public void testFromXcontent() throws Exception { + public void testFromXContent() throws Exception { float[] queryVector = { 1.0f, 2.0f, 3.0f, 4.0f }; KNNQueryBuilder knnQueryBuilder = new KNNQueryBuilder(FIELD_NAME, queryVector, K); XContentBuilder builder = XContentFactory.jsonBuilder(); @@ -103,10 +103,10 @@ public void testFromXcontent() throws Exception { XContentParser contentParser = createParser(builder); contentParser.nextToken(); KNNQueryBuilder actualBuilder = KNNQueryBuilder.fromXContent(contentParser); - actualBuilder.equals(knnQueryBuilder); + assertEquals(knnQueryBuilder, actualBuilder); } - public void testFromXcontent_WithFilter() throws Exception { + public void testFromXContent_WithFilter() throws Exception { final ClusterService clusterService = mockClusterService(Version.CURRENT); final KNNClusterUtil knnClusterUtil = KNNClusterUtil.instance(); @@ -125,7 +125,7 @@ public void testFromXcontent_WithFilter() throws Exception { XContentParser contentParser = createParser(builder); contentParser.nextToken(); KNNQueryBuilder actualBuilder = KNNQueryBuilder.fromXContent(contentParser); - actualBuilder.equals(knnQueryBuilder); + assertEquals(knnQueryBuilder, actualBuilder); } public void testFromXcontent_WithFilter_UnsupportedClusterVersion() throws Exception { From f3d7d49a3dfe5c16a6d71dee04224fc095430255 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Thu, 25 Jan 2024 16:41:30 -0800 Subject: [PATCH 185/416] Skip delete .plugins-ml-config system index during integ test (#1419) (#1422) (cherry picked from commit 47728cef3d81f2fa6e306e57c2740395de7c89dd) Signed-off-by: Junqiu Lei Co-authored-by: Junqiu Lei --- .../java/org/opensearch/knn/ODFERestTestCase.java | 8 +++++++- src/testFixtures/java/org/opensearch/knn/TestUtils.java | 1 + 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/src/testFixtures/java/org/opensearch/knn/ODFERestTestCase.java b/src/testFixtures/java/org/opensearch/knn/ODFERestTestCase.java index c93e537fb..efdde63c5 100644 --- a/src/testFixtures/java/org/opensearch/knn/ODFERestTestCase.java +++ b/src/testFixtures/java/org/opensearch/knn/ODFERestTestCase.java @@ -45,6 +45,7 @@ import static org.opensearch.knn.TestUtils.KNN_BWC_PREFIX; import static org.opensearch.knn.TestUtils.OPENDISTRO_SECURITY; +import static org.opensearch.knn.TestUtils.ML_PLUGIN_SYSTEM_INDEX_PREFIX; import static org.opensearch.knn.TestUtils.OPENSEARCH_SYSTEM_INDEX_PREFIX; import static org.opensearch.knn.TestUtils.SECURITY_AUDITLOG_PREFIX; import static org.opensearch.knn.TestUtils.SKIP_DELETE_MODEL_INDEX; @@ -56,7 +57,12 @@ */ public abstract class ODFERestTestCase extends OpenSearchRestTestCase { - private final Set IMMUTABLE_INDEX_PREFIXES = Set.of(KNN_BWC_PREFIX, SECURITY_AUDITLOG_PREFIX, OPENSEARCH_SYSTEM_INDEX_PREFIX); + private final Set IMMUTABLE_INDEX_PREFIXES = Set.of( + KNN_BWC_PREFIX, + SECURITY_AUDITLOG_PREFIX, + OPENSEARCH_SYSTEM_INDEX_PREFIX, + ML_PLUGIN_SYSTEM_INDEX_PREFIX + ); protected boolean isHttps() { return Optional.ofNullable(System.getProperty("https")).map("true"::equalsIgnoreCase).orElse(false); diff --git a/src/testFixtures/java/org/opensearch/knn/TestUtils.java b/src/testFixtures/java/org/opensearch/knn/TestUtils.java index 61c0abe4f..1cd0ac3db 100644 --- a/src/testFixtures/java/org/opensearch/knn/TestUtils.java +++ b/src/testFixtures/java/org/opensearch/knn/TestUtils.java @@ -97,6 +97,7 @@ public class TestUtils { public static final String UPGRADED_CLUSTER = "upgraded_cluster"; public static final String SECURITY_AUDITLOG_PREFIX = "security-auditlog"; public static final String OPENSEARCH_SYSTEM_INDEX_PREFIX = ".opensearch"; + public static final String ML_PLUGIN_SYSTEM_INDEX_PREFIX = ".plugins-ml"; // Generating vectors using random function with a seed which makes these vectors standard and generate same vectors for each run. public static float[][] randomlyGenerateStandardVectors(int numVectors, int dimensions, int seed) { From 50c5256bd591249f577fd1f88d16c7ab7e699c96 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Thu, 25 Jan 2024 16:47:46 -0800 Subject: [PATCH 186/416] Remove default admin credential references in DEVELOPER_GUIDE (#1415) (#1420) * Remove default admin credentials Signed-off-by: Ryan Bogan (cherry picked from commit 0c000ade1f4a52d8035ff9e016410b2e34a9c2df) Co-authored-by: Ryan Bogan --- DEVELOPER_GUIDE.md | 10 +++------- build.gradle | 2 ++ 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index a62437a8b..964aeea6b 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -274,15 +274,12 @@ curl localhost:9200 Additionally, it is also possible to run a cluster with security enabled: ```shell script -./gradlew run -Dsecurity.enabled=true -Dhttps=true -Duser=admin -Dpassword=admin +./gradlew run -Dsecurity.enabled=true -Dhttps=true -Duser=admin -Dpassword= ``` -By default, if `-Dsecurity.enabled=true` is passed the following defaults will be used: `https=true`, `user=admin` and -`password=admin`. - Then, to access the cluster, we can run ```bash -curl https://localhost:9200 --insecure -u admin:admin +curl https://localhost:9200 --insecure -u admin: { "name" : "integTest-0", @@ -301,7 +298,6 @@ curl https://localhost:9200 --insecure -u admin:admin }, "tagline" : "The OpenSearch Project: https://opensearch.org/" } -``` ### Run Multi-node Cluster Locally @@ -331,7 +327,7 @@ Integration tests can be run with remote cluster. For that run the following com In case remote cluster is secured it's possible to pass username and password with the following command: ``` -./gradlew :integTestRemote -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername="integTest-0" -Dhttps=true -Duser=admin -Dpassword=admin +./gradlew :integTestRemote -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername="integTest-0" -Dhttps=true -Duser=admin -Dpassword= ``` ### Debugging diff --git a/build.gradle b/build.gradle index 2ed70a66e..462adb910 100644 --- a/build.gradle +++ b/build.gradle @@ -336,12 +336,14 @@ integTest { var is_https = System.getProperty("https") var user = System.getProperty("user") var password = System.getProperty("password") + if (System.getProperty("security.enabled") != null) { // If security is enabled, set is_https/user/password defaults is_https = is_https == null ? "true" : is_https user = user == null ? "admin" : user password = password == null ? "admin" : password } + systemProperty("https", is_https) systemProperty("user", user) systemProperty("password", password) From e4dda01acd40455bc74011b4abdf82e7a4216704 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Fri, 26 Jan 2024 13:12:36 -0800 Subject: [PATCH 187/416] Fix markdown formatting in developer guide (#1426) (#1427) Signed-off-by: Ryan Bogan (cherry picked from commit d5538a4e22b3687df3d771582f57f30dd325d45f) Co-authored-by: Ryan Bogan --- DEVELOPER_GUIDE.md | 1 + 1 file changed, 1 insertion(+) diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index 964aeea6b..073a8cbea 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -298,6 +298,7 @@ curl https://localhost:9200 --insecure -u admin: }, "tagline" : "The OpenSearch Project: https://opensearch.org/" } +``` ### Run Multi-node Cluster Locally From de9cde319ee78e773ec802dda771bd667464e909 Mon Sep 17 00:00:00 2001 From: John Mazanec Date: Fri, 26 Jan 2024 14:57:26 -0800 Subject: [PATCH 188/416] Refactor integ tests that access model index (#1423) (#1428) Refactors integration tests that directly access the model system index. End users should not be directly accessing the model system index. It is supposed to be an implementation detail. We have written restful integration tests that directly access the model system index in order to initialize the cluster state. However, we should not do this because users should not be able to interact with it through restful APIs That being said, some of this implementation detail leaks out into the interface. For instance, in k-NN stats we have a stat that is the model system index status. So, in order to test this, we do need direct access to the system index. Similarly, for search, we execute the search against the system index and directly return the results. This is probably a bug - but we still need to test it. Signed-off-by: John Mazanec (cherry picked from commit 2b963b4ae2ae06f0108e3c80e7b9d8785b9033b3) --- CHANGELOG.md | 1 + .../action/RestDeleteModelHandlerIT.java | 48 ++-------------- .../plugin/action/RestGetModelHandlerIT.java | 17 +++--- .../plugin/action/RestKNNStatsHandlerIT.java | 9 ++- .../action/RestSearchModelHandlerIT.java | 57 ++++++++----------- .../org/opensearch/knn/KNNRestTestCase.java | 20 ------- 6 files changed, 45 insertions(+), 107 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8291a4953..3f7907b31 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,6 +31,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Infrastructure * Upgrade gradle to 8.4 [1289](https://github.com/opensearch-project/k-NN/pull/1289) * Refactor security testing to install from individual components [#1307](https://github.com/opensearch-project/k-NN/pull/1307) +* Refactor integ tests that access model index [#1423](https://github.com/opensearch-project/k-NN/pull/1423) ### Documentation ### Maintenance * Update developer guide to include M1 Setup [#1222](https://github.com/opensearch-project/k-NN/pull/1222) diff --git a/src/test/java/org/opensearch/knn/plugin/action/RestDeleteModelHandlerIT.java b/src/test/java/org/opensearch/knn/plugin/action/RestDeleteModelHandlerIT.java index 21459612c..de78d2113 100644 --- a/src/test/java/org/opensearch/knn/plugin/action/RestDeleteModelHandlerIT.java +++ b/src/test/java/org/opensearch/knn/plugin/action/RestDeleteModelHandlerIT.java @@ -11,6 +11,7 @@ package org.opensearch.knn.plugin.action; +import lombok.SneakyThrows; import org.apache.http.util.EntityUtils; import org.opensearch.client.Request; import org.opensearch.client.Response; @@ -30,7 +31,6 @@ import static org.opensearch.knn.common.KNNConstants.KNN_ENGINE; import static org.opensearch.knn.common.KNNConstants.MODEL_ID; import static org.opensearch.knn.common.KNNConstants.MODELS; -import static org.opensearch.knn.common.KNNConstants.MODEL_INDEX_NAME; import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_SPACE_TYPE; import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_NLIST; import static org.opensearch.knn.common.KNNConstants.METHOD_ENCODER_PARAMETER; @@ -43,9 +43,8 @@ public class RestDeleteModelHandlerIT extends KNNRestTestCase { - public void testDeleteModelExists() throws Exception { - createModelSystemIndex(); - + @SneakyThrows + public void testDelete_whenModelExists_thenDeletionSucceeds() { String modelId = "test-model-id"; String trainingIndexName = "train-index"; String trainingFieldName = "train-field"; @@ -80,42 +79,8 @@ public void testDeleteModelExists() throws Exception { assertTrue(ex.getMessage().contains(modelId)); } - public void testDeleteTrainingModel() throws Exception { - createModelSystemIndex(); - - String modelId = "test-model-id"; - String trainingIndexName = "train-index"; - String trainingFieldName = "train-field"; - int dimension = 8; - String modelDescription = "dummy description"; - - createBasicKnnIndex(trainingIndexName, trainingFieldName, dimension); - // we do not wait for training to be completed - ingestDataAndTrainModel(modelId, trainingIndexName, trainingFieldName, dimension, modelDescription); - - Response getModelResponse = getModel(modelId, List.of()); - assertEquals(RestStatus.OK, RestStatus.fromCode(getModelResponse.getStatusLine().getStatusCode())); - - String responseBody = EntityUtils.toString(getModelResponse.getEntity()); - assertNotNull(responseBody); - - Map responseMap = createParser(XContentType.JSON.xContent(), responseBody).map(); - - assertEquals(modelId, responseMap.get(MODEL_ID)); - - String deleteModelRestURI = String.join("/", KNNPlugin.KNN_BASE_URI, MODELS, modelId); - Request deleteModelRequest = new Request("DELETE", deleteModelRestURI); - - ResponseException ex = expectThrows(ResponseException.class, () -> client().performRequest(deleteModelRequest)); - assertEquals(RestStatus.CONFLICT.getStatus(), ex.getResponse().getStatusLine().getStatusCode()); - - // need to wait for training operation as it's required for after test cleanup - assertTrainingSucceeds(modelId, NUM_OF_ATTEMPTS, DELAY_MILLI_SEC); - } - - public void testDeleteModelFailsInvalid() throws Exception { + public void testDelete_whenModelIDIsInvalid_thenFail() { String modelId = "invalid-model-id"; - createModelSystemIndex(); String restURI = String.join("/", KNNPlugin.KNN_BASE_URI, MODELS, modelId); Request request = new Request("DELETE", restURI); @@ -124,7 +89,8 @@ public void testDeleteModelFailsInvalid() throws Exception { } // Test Train Model -> Delete Model -> Train Model with same modelId - public void testTrainingDeletedModel() throws Exception { + @SneakyThrows + public void testTraining_whenModelHasBeenDeleted_thenSucceedTrainingModelWithSameID() { String modelId = "test-model-id1"; String trainingIndexName1 = "train-index-1"; String trainingIndexName2 = "train-index-2"; @@ -141,8 +107,6 @@ public void testTrainingDeletedModel() throws Exception { Response response = client().performRequest(request); assertEquals(request.getEndpoint() + ": failed", RestStatus.OK, RestStatus.fromCode(response.getStatusLine().getStatusCode())); - assertEquals(0, getDocCount(MODEL_INDEX_NAME)); - // Train Model again with same ModelId trainModel(modelId, trainingIndexName2, trainingFieldName, dimension); } diff --git a/src/test/java/org/opensearch/knn/plugin/action/RestGetModelHandlerIT.java b/src/test/java/org/opensearch/knn/plugin/action/RestGetModelHandlerIT.java index ea893021a..91402b444 100644 --- a/src/test/java/org/opensearch/knn/plugin/action/RestGetModelHandlerIT.java +++ b/src/test/java/org/opensearch/knn/plugin/action/RestGetModelHandlerIT.java @@ -12,6 +12,7 @@ package org.opensearch.knn.plugin.action; import joptsimple.internal.Strings; +import lombok.SneakyThrows; import org.apache.http.util.EntityUtils; import org.opensearch.client.Request; import org.opensearch.client.Response; @@ -21,7 +22,6 @@ import org.opensearch.knn.plugin.KNNPlugin; import org.opensearch.core.rest.RestStatus; -import java.io.IOException; import java.util.Arrays; import java.util.List; import java.util.Map; @@ -44,9 +44,8 @@ public class RestGetModelHandlerIT extends KNNRestTestCase { - public void testGetModelExists() throws Exception { - createModelSystemIndex(); - + @SneakyThrows + public void testGetModel_whenModelIdExists_thenSucceed() { String modelId = "test-model-id"; String trainingIndexName = "train-index"; String trainingFieldName = "train-field"; @@ -81,8 +80,8 @@ public void testGetModelExists() throws Exception { assertEquals(L2.getValue(), responseMap.get(METHOD_PARAMETER_SPACE_TYPE)); } - public void testGetModelExistsWithFilter() throws Exception { - createModelSystemIndex(); + @SneakyThrows + public void testGetModel_whenFilterApplied_thenReturnExpectedFields() { String modelId = "test-model-id"; String trainingIndexName = "train-index"; String trainingFieldName = "train-field"; @@ -118,8 +117,7 @@ public void testGetModelExistsWithFilter() throws Exception { assertFalse(responseMap.containsKey(MODEL_STATE)); } - public void testGetModelFailsInvalid() throws IOException { - createModelSystemIndex(); + public void testGetModel_whenModelIDIsInValid_thenFail() { String restURI = String.join("/", KNNPlugin.KNN_BASE_URI, MODELS, "invalid-model-id"); Request request = new Request("GET", restURI); @@ -127,8 +125,7 @@ public void testGetModelFailsInvalid() throws IOException { assertTrue(ex.getMessage().contains("\"invalid-model-id\"")); } - public void testGetModelFailsBlank() throws IOException { - createModelSystemIndex(); + public void testGetModel_whenIDIsBlank_thenFail() { String restURI = String.join("/", KNNPlugin.KNN_BASE_URI, MODELS, " "); Request request = new Request("GET", restURI); diff --git a/src/test/java/org/opensearch/knn/plugin/action/RestKNNStatsHandlerIT.java b/src/test/java/org/opensearch/knn/plugin/action/RestKNNStatsHandlerIT.java index 43f07dc0f..7d11f2e4a 100644 --- a/src/test/java/org/opensearch/knn/plugin/action/RestKNNStatsHandlerIT.java +++ b/src/test/java/org/opensearch/knn/plugin/action/RestKNNStatsHandlerIT.java @@ -338,8 +338,7 @@ public void testScriptStats_multipleShards() throws Exception { assertEquals(initialScriptQueryErrors + 2, (int) (nodeStats.get(0).get(StatNames.SCRIPT_QUERY_ERRORS.getName()))); } - public void testModelIndexHealthMetricsStats() throws IOException { - // Create request that filters only model index + public void testModelIndexHealthMetricsStats() throws Exception { String modelIndexStatusName = StatNames.MODEL_INDEX_STATUS.getName(); // index can be created in one of previous tests, and as we do not delete it each test the check below became optional if (!systemIndexExists(MODEL_INDEX_NAME)) { @@ -351,7 +350,11 @@ public void testModelIndexHealthMetricsStats() throws IOException { // Check that model health status is null since model index is not created to system yet assertNull(statsMap.get(StatNames.MODEL_INDEX_STATUS.getName())); - createModelSystemIndex(); + // Train a model so that the system index will get created + createBasicKnnIndex(TRAINING_INDEX, TRAINING_FIELD, DIMENSION); + bulkIngestRandomVectors(TRAINING_INDEX, TRAINING_FIELD, NUM_DOCS, DIMENSION); + trainKnnModel(TEST_MODEL_ID, TRAINING_INDEX, TRAINING_FIELD, DIMENSION, MODEL_DESCRIPTION); + validateModelCreated(TEST_MODEL_ID); } Response response = getKnnStats(Collections.emptyList(), Arrays.asList(modelIndexStatusName)); diff --git a/src/test/java/org/opensearch/knn/plugin/action/RestSearchModelHandlerIT.java b/src/test/java/org/opensearch/knn/plugin/action/RestSearchModelHandlerIT.java index 3aa6085d8..f4ab352ea 100644 --- a/src/test/java/org/opensearch/knn/plugin/action/RestSearchModelHandlerIT.java +++ b/src/test/java/org/opensearch/knn/plugin/action/RestSearchModelHandlerIT.java @@ -11,6 +11,7 @@ package org.opensearch.knn.plugin.action; +import lombok.SneakyThrows; import org.apache.http.util.EntityUtils; import org.opensearch.action.search.SearchResponse; import org.opensearch.client.Request; @@ -19,22 +20,18 @@ import org.opensearch.core.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; import org.opensearch.knn.KNNRestTestCase; -import org.opensearch.knn.index.SpaceType; -import org.opensearch.knn.index.util.KNNEngine; import org.opensearch.knn.indices.Model; -import org.opensearch.knn.indices.ModelMetadata; -import org.opensearch.knn.indices.ModelState; import org.opensearch.knn.plugin.KNNPlugin; import org.opensearch.core.rest.RestStatus; import org.opensearch.search.SearchHit; -import java.io.IOException; import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; import static org.opensearch.knn.common.KNNConstants.MODELS; +import static org.opensearch.knn.common.KNNConstants.MODEL_INDEX_NAME; import static org.opensearch.knn.common.KNNConstants.PARAM_SIZE; import static org.opensearch.knn.common.KNNConstants.SEARCH_MODEL_MAX_SIZE; import static org.opensearch.knn.common.KNNConstants.SEARCH_MODEL_MIN_SIZE; @@ -47,12 +44,7 @@ public class RestSearchModelHandlerIT extends KNNRestTestCase { - private ModelMetadata getModelMetadata() { - return new ModelMetadata(KNNEngine.DEFAULT, SpaceType.DEFAULT, 4, ModelState.CREATED, "2021-03-27", "test model", "", ""); - } - - public void testNotSupportedParams() throws IOException { - createModelSystemIndex(); + public void testSearch_whenUnSupportedParamsPassed_thenFail() { String restURI = String.join("/", KNNPlugin.KNN_BASE_URI, MODELS, "_search"); Map invalidParams = new HashMap<>(); invalidParams.put("index", "index-name"); @@ -61,27 +53,31 @@ public void testNotSupportedParams() throws IOException { expectThrows(ResponseException.class, () -> client().performRequest(request)); } - public void testNoModelExists() throws IOException { - createModelSystemIndex(); + @SneakyThrows + public void testSearch_whenNoModelExists_thenReturnEmptyResults() { + // Currently, if the model index exists, we will return empty hits. If it does not exist, we will + // throw an exception. This is somewhat of a bug considering that the model index is supposed to be + // an implementation detail abstracted away from the user. However, in order to test, we need to handle + // the 2 different scenarios String restURI = String.join("/", KNNPlugin.KNN_BASE_URI, MODELS, "_search"); Request request = new Request("GET", restURI); request.setJsonEntity("{\n" + " \"query\": {\n" + " \"match_all\": {}\n" + " }\n" + "}"); - - Response response = client().performRequest(request); - assertEquals(RestStatus.OK, RestStatus.fromCode(response.getStatusLine().getStatusCode())); - - String responseBody = EntityUtils.toString(response.getEntity()); - assertNotNull(responseBody); - - XContentParser parser = createParser(XContentType.JSON.xContent(), responseBody); - SearchResponse searchResponse = SearchResponse.fromXContent(parser); - assertNotNull(searchResponse); - assertEquals(searchResponse.getHits().getHits().length, 0); - + if (!systemIndexExists(MODEL_INDEX_NAME)) { + ResponseException ex = expectThrows(ResponseException.class, () -> client().performRequest(request)); + assertEquals(RestStatus.NOT_FOUND.getStatus(), ex.getResponse().getStatusLine().getStatusCode()); + } else { + Response response = client().performRequest(request); + assertEquals(RestStatus.OK, RestStatus.fromCode(response.getStatusLine().getStatusCode())); + String responseBody = EntityUtils.toString(response.getEntity()); + assertNotNull(responseBody); + XContentParser parser = createParser(XContentType.JSON.xContent(), responseBody); + SearchResponse searchResponse = SearchResponse.fromXContent(parser); + assertNotNull(searchResponse); + assertEquals(searchResponse.getHits().getHits().length, 0); + } } - public void testSizeValidationFailsInvalidSize() throws IOException { - createModelSystemIndex(); + public void testSearch_whenInvalidSizePassed_thenFail() { for (Integer invalidSize : Arrays.asList(SEARCH_MODEL_MIN_SIZE - 1, SEARCH_MODEL_MAX_SIZE + 1)) { String restURI = String.join("/", KNNPlugin.KNN_BASE_URI, MODELS, "_search?" + PARAM_SIZE + "=" + invalidSize); Request request = new Request("GET", restURI); @@ -101,8 +97,8 @@ public void testSizeValidationFailsInvalidSize() throws IOException { } - public void testSearchModelExists() throws Exception { - createModelSystemIndex(); + @SneakyThrows + public void testSearch_whenModelExists_thenSuccess() { String trainingIndex = "irrelevant-index"; String trainingFieldName = "train-field"; int dimension = 8; @@ -151,7 +147,6 @@ public void testSearchModelExists() throws Exception { } public void testSearchModelWithoutSource() throws Exception { - createModelSystemIndex(); String trainingIndex = "irrelevant-index"; String trainingFieldName = "train-field"; int dimension = 8; @@ -192,7 +187,6 @@ public void testSearchModelWithoutSource() throws Exception { } public void testSearchModelWithSourceFilteringIncludes() throws Exception { - createModelSystemIndex(); String trainingIndex = "irrelevant-index"; String trainingFieldName = "train-field"; int dimension = 8; @@ -244,7 +238,6 @@ public void testSearchModelWithSourceFilteringIncludes() throws Exception { } public void testSearchModelWithSourceFilteringExcludes() throws Exception { - createModelSystemIndex(); String trainingIndex = "irrelevant-index"; String trainingFieldName = "train-field"; int dimension = 8; diff --git a/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java b/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java index 5ac053fb3..7aa595faa 100644 --- a/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java +++ b/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java @@ -5,8 +5,6 @@ package org.opensearch.knn; -import com.google.common.base.Charsets; -import com.google.common.io.Resources; import com.google.common.primitives.Floats; import lombok.extern.log4j.Log4j2; import org.apache.commons.lang.StringUtils; @@ -23,7 +21,6 @@ import org.opensearch.knn.index.query.KNNQueryBuilder; import org.opensearch.knn.index.KNNSettings; import org.opensearch.knn.index.SpaceType; -import org.opensearch.knn.indices.ModelDao; import org.opensearch.knn.indices.ModelState; import org.opensearch.knn.plugin.KNNPlugin; import org.opensearch.knn.plugin.script.KNNScoringScriptEngine; @@ -52,7 +49,6 @@ import java.io.IOException; import java.io.InputStream; -import java.net.URL; import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; @@ -78,8 +74,6 @@ import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_NLIST; import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_SPACE_TYPE; import static org.opensearch.knn.common.KNNConstants.MODEL_DESCRIPTION; -import static org.opensearch.knn.common.KNNConstants.MODEL_INDEX_MAPPING_PATH; -import static org.opensearch.knn.common.KNNConstants.MODEL_INDEX_NAME; import static org.opensearch.knn.common.KNNConstants.MODEL_STATE; import static org.opensearch.knn.common.KNNConstants.TRAIN_FIELD_PARAMETER; import static org.opensearch.knn.common.KNNConstants.TRAIN_INDEX_PARAMETER; @@ -754,20 +748,6 @@ protected String getIndexSettingByName(String indexName, String settingName, boo } } - protected void createModelSystemIndex() throws IOException { - URL url = ModelDao.class.getClassLoader().getResource(MODEL_INDEX_MAPPING_PATH); - if (url == null) { - throw new IllegalStateException("Unable to retrieve mapping for \"" + MODEL_INDEX_NAME + "\""); - } - - String mapping = Resources.toString(url, Charsets.UTF_8); - mapping = mapping.substring(1, mapping.length() - 1); - - if (!systemIndexExists(MODEL_INDEX_NAME)) { - createIndex(MODEL_INDEX_NAME, Settings.builder().put("number_of_shards", 1).put("number_of_replicas", 0).build(), mapping); - } - } - /** * Clear cache *

From c64707cae8c7443ba623a8d5f86166ebaa6d1fbc Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Mon, 29 Jan 2024 16:09:10 -0800 Subject: [PATCH 189/416] Fix flaky model tests (#1429) (#1430) * Fix flaky model tests in k-NN Signed-off-by: Ryan Bogan * Remove * imports Signed-off-by: Ryan Bogan * Minor change Signed-off-by: Ryan Bogan * Add changelog entry Signed-off-by: Ryan Bogan --------- Signed-off-by: Ryan Bogan (cherry picked from commit 9e4251e919eccd77fe671f46773091f6026b7d4c) Co-authored-by: Ryan Bogan --- CHANGELOG.md | 1 + .../org/opensearch/knn/indices/ModelDaoTests.java | 14 ++++++++++++++ 2 files changed, 15 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3f7907b31..b3f172cbb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,6 +32,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), * Upgrade gradle to 8.4 [1289](https://github.com/opensearch-project/k-NN/pull/1289) * Refactor security testing to install from individual components [#1307](https://github.com/opensearch-project/k-NN/pull/1307) * Refactor integ tests that access model index [#1423](https://github.com/opensearch-project/k-NN/pull/1423) +* Fix flaky model tests [#1429](https://github.com/opensearch-project/k-NN/pull/1429) ### Documentation ### Maintenance * Update developer guide to include M1 Setup [#1222](https://github.com/opensearch-project/k-NN/pull/1222) diff --git a/src/test/java/org/opensearch/knn/indices/ModelDaoTests.java b/src/test/java/org/opensearch/knn/indices/ModelDaoTests.java index 1297dc184..2af8df953 100644 --- a/src/test/java/org/opensearch/knn/indices/ModelDaoTests.java +++ b/src/test/java/org/opensearch/knn/indices/ModelDaoTests.java @@ -14,9 +14,11 @@ import org.junit.AfterClass; import org.junit.Assert; import org.junit.BeforeClass; +import org.mockito.MockedStatic; import org.opensearch.ExceptionsHelper; import org.opensearch.ResourceAlreadyExistsException; import org.opensearch.ResourceNotFoundException; +import org.opensearch.cluster.ClusterChangedEvent; import org.opensearch.core.action.ActionListener; import org.opensearch.action.DocWriteResponse; import org.opensearch.action.StepListener; @@ -46,6 +48,7 @@ import org.opensearch.knn.plugin.transport.UpdateModelGraveyardAction; import org.opensearch.knn.plugin.transport.UpdateModelGraveyardRequest; import org.opensearch.core.rest.RestStatus; +import org.opensearch.knn.training.TrainingJobClusterStateListener; import java.io.IOException; import java.time.ZoneOffset; @@ -57,6 +60,10 @@ import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.mockStatic; import static org.opensearch.cluster.metadata.Metadata.builder; import static org.opensearch.knn.common.KNNConstants.DIMENSION; import static org.opensearch.knn.common.KNNConstants.KNN_ENGINE; @@ -73,15 +80,22 @@ public class ModelDaoTests extends KNNSingleNodeTestCase { private static ExecutorService modelGetterExecutor; private static final String FAILED = "failed"; + private static MockedStatic trainingJobClusterStateListenerMockedStatic; @BeforeClass public static void setup() { modelGetterExecutor = Executors.newSingleThreadExecutor(); + trainingJobClusterStateListenerMockedStatic = mockStatic(TrainingJobClusterStateListener.class); + final TrainingJobClusterStateListener trainingJobClusterStateListener = mock(TrainingJobClusterStateListener.class); + doNothing().when(trainingJobClusterStateListener).clusterChanged(any(ClusterChangedEvent.class)); + trainingJobClusterStateListenerMockedStatic.when(TrainingJobClusterStateListener::getInstance) + .thenReturn(trainingJobClusterStateListener); } @AfterClass public static void teardown() { modelGetterExecutor.shutdown(); + trainingJobClusterStateListenerMockedStatic.close(); } public void testCreate() throws IOException, InterruptedException { From 7a7f76d819eb88682267d5c20df0ddeb66032cff Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Wed, 31 Jan 2024 09:58:06 -0800 Subject: [PATCH 190/416] Bump aiohttp from 3.8.6 to 3.9.2 in /benchmarks/osb (#1436) Bumps [aiohttp](https://github.com/aio-libs/aiohttp) from 3.8.6 to 3.9.2. - [Release notes](https://github.com/aio-libs/aiohttp/releases) - [Changelog](https://github.com/aio-libs/aiohttp/blob/master/CHANGES.rst) - [Commits](https://github.com/aio-libs/aiohttp/compare/v3.8.6...v3.9.2) --- updated-dependencies: - dependency-name: aiohttp dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> (cherry picked from commit fe592f571bc0f5adf68e303842e6c5b99ec06836) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- benchmarks/osb/requirements.txt | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/benchmarks/osb/requirements.txt b/benchmarks/osb/requirements.txt index 7d7cfcf67..2da38cfaa 100644 --- a/benchmarks/osb/requirements.txt +++ b/benchmarks/osb/requirements.txt @@ -4,7 +4,7 @@ # # pip-compile # -aiohttp==3.8.6 +aiohttp==3.9.2 # via opensearch-py aiosignal==1.2.0 # via aiohttp @@ -20,8 +20,6 @@ certifi==2023.7.22 # via # opensearch-benchmark # opensearch-py -charset-normalizer==2.0.12 - # via aiohttp frozenlist==1.3.0 # via # aiohttp From 71b3c5357985951846b2c4838673da63198410d0 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Thu, 1 Feb 2024 18:27:15 -0800 Subject: [PATCH 191/416] Pass correct value on IDSelectorBitmap initialization (#1444) (#1446) Signed-off-by: Heemin Kim (cherry picked from commit 4c7a05566259c554568a6d744d450017f11ac1c7) Co-authored-by: Heemin Kim --- CHANGELOG.md | 1 + jni/src/faiss_wrapper.cpp | 4 +- jni/tests/faiss_wrapper_test.cpp | 68 ++++++++++++++++++++++++++++++++ 3 files changed, 71 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b3f172cbb..b0d366420 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,6 +28,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), * Fix script score queries not getting cached [#1367](https://github.com/opensearch-project/k-NN/pull/1367) * Fix KNNScorer to apply boost [#1403](https://github.com/opensearch-project/k-NN/pull/1403) * Fix equals and hashCode methods for KNNQuery and KNNQueryBuilder [#1397](https://github.com/opensearch-project/k-NN/pull/1397) +* Pass correct value on IDSelectorBitmap initialization [#1444](https://github.com/opensearch-project/k-NN/pull/1444) ### Infrastructure * Upgrade gradle to 8.4 [1289](https://github.com/opensearch-project/k-NN/pull/1289) * Refactor security testing to install from individual components [#1307](https://github.com/opensearch-project/k-NN/pull/1307) diff --git a/jni/src/faiss_wrapper.cpp b/jni/src/faiss_wrapper.cpp index 8e9deb07b..4609f3144 100644 --- a/jni/src/faiss_wrapper.cpp +++ b/jni/src/faiss_wrapper.cpp @@ -228,7 +228,7 @@ jobjectArray knn_jni::faiss_wrapper::QueryIndex_WithFilter(knn_jni::JNIUtilInter // create the filterSearch params if the filterIdsJ is not a null pointer if(filterIdsJ != nullptr) { int *filteredIdsArray = jniUtil->GetIntArrayElements(env, filterIdsJ, nullptr); - int filterIdsLength = env->GetArrayLength(filterIdsJ); + int filterIdsLength = jniUtil->GetJavaIntArrayLength(env, filterIdsJ); std::unique_ptr idSelector; FilterIdsSelectorType idSelectorType = getIdSelectorType(filteredIdsArray, filterIdsLength); // start with empty vectors for 2 different types of empty Selectors. We need define them here to avoid copying of data @@ -248,7 +248,7 @@ jobjectArray knn_jni::faiss_wrapper::QueryIndex_WithFilter(knn_jni::JNIUtilInter const int bitsetArraySize = (maxIdValue >> 3) + 1; bitmap.resize(bitsetArraySize, 0); buildFilterIdsBitMap(filteredIdsArray, filterIdsLength, bitmap.data()); - idSelector.reset(new faiss::IDSelectorBitmap(filterIdsLength, bitmap.data())); + idSelector.reset(new faiss::IDSelectorBitmap(bitsetArraySize, bitmap.data())); } faiss::SearchParameters *searchParameters; faiss::SearchParametersHNSW hnswParams; diff --git a/jni/tests/faiss_wrapper_test.cpp b/jni/tests/faiss_wrapper_test.cpp index 5fa5165bb..ed3ec880d 100644 --- a/jni/tests/faiss_wrapper_test.cpp +++ b/jni/tests/faiss_wrapper_test.cpp @@ -229,6 +229,74 @@ TEST(FaissQueryIndexTest, BasicAssertions) { } } +//Test for a bug reported in https://github.com/opensearch-project/k-NN/issues/1435 +TEST(FaissQueryIndexWithFilterTest1435, BasicAssertions) { + // Define the index data + faiss::idx_t numIds = 200; + std::vector ids; + std::vector vectors; + std::vector> queries; + + int dim = 16; + for (int64_t i = 1; i < numIds + 1; i++) { + std::vector query; + query.reserve(dim); + ids.push_back(i); + for (int j = 0; j < dim; j++) { + float vector = test_util::RandomFloat(-500.0, 500.0); + vectors.push_back(vector); + query.push_back(vector); + } + queries.push_back(query); + } + + std::vector filterIds; + for (int64_t i = 154; i < 163; i++) { + filterIds.push_back(i); + } + std::unordered_set filterIdSet(filterIds.begin(), filterIds.end()); + + faiss::MetricType metricType = faiss::METRIC_L2; + std::string method = "HNSW32,Flat"; + + // Create the index + std::unique_ptr createdIndex( + test_util::FaissCreateIndex(2, method, metricType)); + auto createdIndexWithData = + test_util::FaissAddData(createdIndex.get(), ids, vectors); + + // Setup jni + JNIEnv *jniEnv = nullptr; + NiceMock mockJNIUtil; + EXPECT_CALL(mockJNIUtil, + GetJavaIntArrayLength( + jniEnv, reinterpret_cast(&filterIds))) + .WillRepeatedly(Return(filterIds.size())); + + int k = 20; + for (auto query : queries) { + std::unique_ptr *>> results( + reinterpret_cast *> *>( + knn_jni::faiss_wrapper::QueryIndex_WithFilter( + &mockJNIUtil, jniEnv, + reinterpret_cast(&createdIndexWithData), + reinterpret_cast(&query), k, + reinterpret_cast(&filterIds), nullptr))); + + ASSERT_TRUE(results->size() <= filterIds.size()); + ASSERT_TRUE(results->size() > 0); + for (const auto& pairPtr : *results) { + auto it = filterIdSet.find(pairPtr->first); + ASSERT_NE(it, filterIdSet.end()); + } + + // Need to free up each result + for (auto it : *results.get()) { + delete it; + } + } +} + TEST(FaissQueryIndexWithParentFilterTest, BasicAssertions) { // Define the index data faiss::idx_t numIds = 100; From d5b42375c29315175fd4491e22abec13a166e5cf Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Mon, 5 Feb 2024 14:24:49 -0600 Subject: [PATCH 192/416] Update Build Script to build libraries with SIMD (#1451) (#1454) Signed-off-by: Naveen Tatikonda (cherry picked from commit 8eb0776e7232608a3938a3037f52acf6cb1e4510) Co-authored-by: Naveen Tatikonda --- scripts/build.sh | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/scripts/build.sh b/scripts/build.sh index 6a9adc256..6e0c85865 100644 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -148,6 +148,20 @@ cd $distributions zip -ur $zipPath lib cd $work_dir +if [ "$PLATFORM" != "windows" ]; then + echo "Building k-NN libraries after enabling SIMD" + ./gradlew :buildJniLib -Dsimd.enabled=true + mkdir $distributions/lib_simd + cp -v $ompPath $distributions/lib_simd + cp -v ./jni/release/${libPrefix}* $distributions/lib_simd + ls -l $distributions/lib_simd + + # Add lib_simd directory to the k-NN plugin zip + cd $distributions + zip -ur $zipPath lib_simd + cd $work_dir +fi + echo "COPY ${distributions}/*.zip" mkdir -p $OUTPUT/plugins cp -v ${distributions}/*.zip $OUTPUT/plugins From 0e1771150ee9b565fa0c9207f96b1335131aeac7 Mon Sep 17 00:00:00 2001 From: Naveen Tatikonda Date: Mon, 5 Feb 2024 14:26:45 -0600 Subject: [PATCH 193/416] Add support for Faiss SQFP16 Quantization and enable AVX2 Optimization (#1421) (#1448) * Add Support for Faiss SQFP16 and enable Faiss AVX2 Optimization * Add Patch Script to fix build on Linux CI * Disable AVX2 support on Windows * Add CHANGELOG * Update Faiss Submodule * Address Review Comments * Update UX Interface * Add Parameter to enable SIMD * Update DEVELOPER_GUIDE * Address Review Comments --------- Signed-off-by: Naveen Tatikonda --- .github/workflows/CI.yml | 23 +- .github/workflows/test_security.yml | 6 +- CHANGELOG.md | 1 + DEVELOPER_GUIDE.md | 18 + build.gradle | 5 +- jni/CMakeLists.txt | 20 +- jni/external/faiss | 2 +- ...-Custom-patch-to-support-sqfp16-neon.patch | 495 ++++++++++++++++++ ...ustom-patch-to-support-AVX2-Linux-CI.patch | 32 ++ jni/tests/faiss_wrapper_test.cpp | 53 ++ .../opensearch/knn/common/KNNConstants.java | 7 + .../org/opensearch/knn/index/Parameter.java | 43 ++ .../org/opensearch/knn/index/util/Faiss.java | 19 + .../org/opensearch/knn/index/FaissIT.java | 161 +++++- .../opensearch/knn/index/ParameterTests.java | 14 + .../opensearch/knn/index/util/FaissTests.java | 61 +++ .../opensearch/knn/jni/JNIServiceTests.java | 109 ++++ 17 files changed, 1058 insertions(+), 11 deletions(-) create mode 100644 jni/patches/faiss/0002-Custom-patch-to-support-sqfp16-neon.patch create mode 100644 jni/patches/faiss/0003-Custom-patch-to-support-AVX2-Linux-CI.patch diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 04ec15fd4..b689e1021 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -45,6 +45,10 @@ jobs: cd jni/external/faiss git apply --ignore-space-change --ignore-whitespace --3way ../../patches/faiss/0001-Custom-patch-to-support-multi-vector.patch rm ../../patches/faiss/0001-Custom-patch-to-support-multi-vector.patch + git apply --ignore-space-change --ignore-whitespace --3way ../../patches/faiss/0002-Custom-patch-to-support-sqfp16-neon.patch + rm ../../patches/faiss/0002-Custom-patch-to-support-sqfp16-neon.patch + git apply --ignore-space-change --ignore-whitespace --3way ../../patches/faiss/0003-Custom-patch-to-support-AVX2-Linux-CI.patch + rm ../../patches/faiss/0003-Custom-patch-to-support-AVX2-Linux-CI.patch working-directory: ${{ github.workspace }} - name: Setup Java ${{ matrix.java }} @@ -56,7 +60,15 @@ jobs: # switching the user, as OpenSearch cluster can only be started as root/Administrator on linux-deb/linux-rpm/windows-zip. run: | chown -R 1000:1000 `pwd` - su `id -un 1000` -c "whoami && java -version && ./gradlew build" + if lscpu | grep -i avx2 + then + echo "avx2 available on system" + su `id -un 1000` -c "whoami && java -version && ./gradlew build -Dsimd.enabled=true" + else + echo "avx2 not available on system" + su `id -un 1000` -c "whoami && java -version && ./gradlew build" + fi + - name: Upload Coverage Report uses: codecov/codecov-action@v1 @@ -88,7 +100,14 @@ jobs: - name: Run build run: | - ./gradlew build + if sysctl -n machdep.cpu.features machdep.cpu.leaf7_features | grep -i AVX2 + then + echo "avx2 available on system" + ./gradlew build -Dsimd.enabled=true + else + echo "avx2 not available on system" + ./gradlew build + fi Build-k-NN-Windows: strategy: diff --git a/.github/workflows/test_security.yml b/.github/workflows/test_security.yml index 783b4399c..cf83185f6 100644 --- a/.github/workflows/test_security.yml +++ b/.github/workflows/test_security.yml @@ -45,6 +45,10 @@ jobs: cd jni/external/faiss git apply --ignore-space-change --ignore-whitespace --3way ../../patches/faiss/0001-Custom-patch-to-support-multi-vector.patch rm ../../patches/faiss/0001-Custom-patch-to-support-multi-vector.patch + git apply --ignore-space-change --ignore-whitespace --3way ../../patches/faiss/0002-Custom-patch-to-support-sqfp16-neon.patch + rm ../../patches/faiss/0002-Custom-patch-to-support-sqfp16-neon.patch + git apply --ignore-space-change --ignore-whitespace --3way ../../patches/faiss/0003-Custom-patch-to-support-AVX2-Linux-CI.patch + rm ../../patches/faiss/0003-Custom-patch-to-support-AVX2-Linux-CI.patch working-directory: ${{ github.workspace }} - name: Setup Java ${{ matrix.java }} @@ -56,4 +60,4 @@ jobs: # switching the user, as OpenSearch cluster can only be started as root/Administrator on linux-deb/linux-rpm/windows-zip. run: | chown -R 1000:1000 `pwd` - su `id -un 1000` -c "whoami && java -version && ./gradlew integTest -Dsecurity.enabled=true" + su `id -un 1000` -c "whoami && java -version && ./gradlew integTest -Dsecurity.enabled=true -Dsimd.enabled=true" diff --git a/CHANGELOG.md b/CHANGELOG.md index b0d366420..4d5796d4c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Features * Add parent join support for lucene knn [#1182](https://github.com/opensearch-project/k-NN/pull/1182) * Add parent join support for faiss hnsw [#1398](https://github.com/opensearch-project/k-NN/pull/1398) +* Add Support for Faiss SQFP16 and enable Faiss AVX2 Optimization [#1421](https://github.com/opensearch-project/k-NN/pull/1421) ### Enhancements * Increase Lucene max dimension limit to 16,000 [#1346](https://github.com/opensearch-project/k-NN/pull/1346) * Tuned default values for ef_search and ef_construction for better indexing and search performance for vector search [#1353](https://github.com/opensearch-project/k-NN/pull/1353) diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index 073a8cbea..c232e75a9 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -11,6 +11,7 @@ - [Build](#build) - [JNI Library](#jni-library) - [JNI Library Artifacts](#jni-library-artifacts) + - [Enable SIMD Optimization](#enable-simd-optimization) - [Run OpenSearch k-NN](#run-opensearch-k-nn) - [Run Single-node Cluster Locally](#run-single-node-cluster-locally) - [Run Multi-node Cluster Locally](#run-multi-node-cluster-locally) @@ -236,6 +237,23 @@ If you want to make a custom patch on JNI library 3. Place the patch file under `jni/patches` 4. Make a change in `jni/CmakeLists.txt`, `.github/workflows/CI.yml` to apply the patch during build +### Enable SIMD Optimization +SIMD(Single Instruction/Multiple Data) Optimization can be enabled by setting this optional parameter `simd.enabled` to `true` which boosts the performance +by enabling `AVX2` on `x86 architecture` and `NEON` on `ARM64 architecture` while building the Faiss library. But to enable SIMD, the underlying processor +should support this (AVX2 or NEON). So, by default it is set to `false`. + +``` +# While building OpenSearch k-NN +./gradlew build -Dsimd.enabled=true + +# While running OpenSearch k-NN +./gradlew run -Dsimd.enabled=true + +# While building the JNI libraries +cd jni +cmake . -DSIMD_ENABLED=true +``` + ## Run OpenSearch k-NN ### Run Single-node Cluster Locally diff --git a/build.gradle b/build.gradle index 462adb910..d9eb0f8c8 100644 --- a/build.gradle +++ b/build.gradle @@ -17,6 +17,7 @@ buildscript { version_qualifier = System.getProperty("build.version_qualifier", "") opensearch_group = "org.opensearch" isSnapshot = "true" == System.getProperty("build.snapshot", "true") + simd_enabled = System.getProperty("simd.enabled", "false") version_tokens = opensearch_version.tokenize('-') opensearch_build = version_tokens[0] + '.0' @@ -297,10 +298,10 @@ task cmakeJniLib(type:Exec) { workingDir 'jni' if (Os.isFamily(Os.FAMILY_WINDOWS)) { dependsOn windowsPatches - commandLine 'cmake', '.', "-G", "Unix Makefiles", "-DKNN_PLUGIN_VERSION=${opensearch_version}", "-DBLAS_LIBRARIES=$rootDir\\src\\main\\resources\\windowsDependencies\\libopenblas.dll", "-DLAPACK_LIBRARIES=$rootDir\\src\\main\\resources\\windowsDependencies\\libopenblas.dll" + commandLine 'cmake', '.', "-G", "Unix Makefiles", "-DKNN_PLUGIN_VERSION=${opensearch_version}", "-DBLAS_LIBRARIES=$rootDir\\src\\main\\resources\\windowsDependencies\\libopenblas.dll", "-DLAPACK_LIBRARIES=$rootDir\\src\\main\\resources\\windowsDependencies\\libopenblas.dll", "-DSIMD_ENABLED=${simd_enabled}" } else { - commandLine 'cmake', '.', "-DKNN_PLUGIN_VERSION=${opensearch_version}" + commandLine 'cmake', '.', "-DKNN_PLUGIN_VERSION=${opensearch_version}", "-DSIMD_ENABLED=${simd_enabled}" } } diff --git a/jni/CMakeLists.txt b/jni/CMakeLists.txt index 04dca217c..901929fc3 100644 --- a/jni/CMakeLists.txt +++ b/jni/CMakeLists.txt @@ -111,7 +111,13 @@ endif () if (${CONFIG_FAISS} STREQUAL ON OR ${CONFIG_ALL} STREQUAL ON OR ${CONFIG_TEST} STREQUAL ON) set(BUILD_TESTING OFF) # Avoid building faiss tests set(BLA_STATIC ON) # Statically link BLAS - set(FAISS_OPT_LEVEL generic) # Keep optimization level generic + if(${CMAKE_SYSTEM_NAME} STREQUAL Windows OR ${CMAKE_SYSTEM_PROCESSOR} MATCHES "aarch64" OR NOT ${SIMD_ENABLED}) + set(FAISS_OPT_LEVEL generic) # Keep optimization level as generic on Windows OS as it is not supported due to MINGW64 compiler issue. Also, on aarch64 avx2 is not supported. + set(TARGET_LINK_FAISS_LIB faiss) + else() + set(FAISS_OPT_LEVEL avx2) # Keep optimization level as avx2 to improve performance on Linux and Mac. + set(TARGET_LINK_FAISS_LIB faiss_avx2) + endif() if (${CMAKE_SYSTEM_NAME} STREQUAL Darwin) if(CMAKE_C_COMPILER_ID MATCHES "Clang\$") @@ -143,12 +149,20 @@ if (${CONFIG_FAISS} STREQUAL ON OR ${CONFIG_ALL} STREQUAL ON OR ${CONFIG_TEST} S endif () # Check if patch exist, this is to skip git apply during CI build. See CI.yml with ubuntu. - find_path(PATCH_FILE NAMES 0001-Custom-patch-to-support-multi-vector.patch PATHS ${CMAKE_CURRENT_SOURCE_DIR}/patches/faiss NO_DEFAULT_PATH) + find_path(PATCH_FILE NAMES 0001-Custom-patch-to-support-multi-vector.patch 0002-Custom-patch-to-support-sqfp16-neon.patch PATHS ${CMAKE_CURRENT_SOURCE_DIR}/patches/faiss NO_DEFAULT_PATH) # If it exists, apply patches if (EXISTS ${PATCH_FILE}) message(STATUS "Applying custom patches.") execute_process(COMMAND git apply --ignore-space-change --ignore-whitespace --3way ${CMAKE_CURRENT_SOURCE_DIR}/patches/faiss/0001-Custom-patch-to-support-multi-vector.patch WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/external/faiss ERROR_VARIABLE ERROR_MSG RESULT_VARIABLE RESULT_CODE) + + # 0002-Custom-patch-to-support-sqfp16-neon.patch is a temporary patch to add NEON support to SQ. + # Once the commit conflict issues wrt to Multi vector are resolved, this patch can be removed by updating the faiss submodule with corresponding commit. + # Apply the patch if the OS is not Windows and Processor is aarch64. + if (NOT ${CMAKE_SYSTEM_NAME} STREQUAL Windows AND ${CMAKE_SYSTEM_PROCESSOR} MATCHES "aarch64" AND ${SIMD_ENABLED}) + execute_process(COMMAND git apply --ignore-space-change --ignore-whitespace --3way ${CMAKE_CURRENT_SOURCE_DIR}/patches/faiss/0002-Custom-patch-to-support-sqfp16-neon.patch WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/external/faiss ERROR_VARIABLE ERROR_MSG RESULT_VARIABLE RESULT_CODE) + endif() + if(RESULT_CODE) message(FATAL_ERROR "Failed to apply patch:\n${ERROR_MSG}") endif() @@ -165,7 +179,7 @@ if (${CONFIG_FAISS} STREQUAL ON OR ${CONFIG_ALL} STREQUAL ON OR ${CONFIG_TEST} S ${CMAKE_CURRENT_SOURCE_DIR}/src/knn_extension/faiss/utils/BitSet.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/knn_extension/faiss/MultiVectorResultCollector.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/knn_extension/faiss/MultiVectorResultCollectorFactory.cpp) - target_link_libraries(${TARGET_LIB_FAISS} faiss ${TARGET_LIB_COMMON} OpenMP::OpenMP_CXX) + target_link_libraries(${TARGET_LIB_FAISS} ${TARGET_LINK_FAISS_LIB} ${TARGET_LIB_COMMON} OpenMP::OpenMP_CXX) target_include_directories(${TARGET_LIB_FAISS} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include ${CMAKE_CURRENT_SOURCE_DIR}/include/knn_extension/faiss diff --git a/jni/external/faiss b/jni/external/faiss index 3219e3d12..0013c702f 160000 --- a/jni/external/faiss +++ b/jni/external/faiss @@ -1 +1 @@ -Subproject commit 3219e3d12e6fc36dfdfe17d4cf238ef70bf89568 +Subproject commit 0013c702f47bedbf6159ac356e61f378ccd12ac8 diff --git a/jni/patches/faiss/0002-Custom-patch-to-support-sqfp16-neon.patch b/jni/patches/faiss/0002-Custom-patch-to-support-sqfp16-neon.patch new file mode 100644 index 000000000..d743d0a97 --- /dev/null +++ b/jni/patches/faiss/0002-Custom-patch-to-support-sqfp16-neon.patch @@ -0,0 +1,495 @@ +diff --git a/faiss/CMakeLists.txt b/faiss/CMakeLists.txt +index db5133d6..22dac7cb 100644 +--- a/faiss/CMakeLists.txt ++++ b/faiss/CMakeLists.txt +@@ -189,6 +189,7 @@ set(FAISS_HEADERS + utils/extra_distances.h + utils/fp16-fp16c.h + utils/fp16-inl.h ++ utils/fp16-arm.h + utils/fp16.h + utils/hamming-inl.h + utils/hamming.h +diff --git a/faiss/impl/ScalarQuantizer.cpp b/faiss/impl/ScalarQuantizer.cpp +index fc7b28ef..07d77d56 100644 +--- a/faiss/impl/ScalarQuantizer.cpp ++++ b/faiss/impl/ScalarQuantizer.cpp +@@ -91,6 +91,20 @@ struct Codec8bit { + return _mm256_fmadd_ps(f8, one_255, half_one_255); + } + #endif ++ ++#ifdef __aarch64__ ++ static FAISS_ALWAYS_INLINE float32x4x2_t ++ decode_8_components(const uint8_t* code, int i) { ++ float32_t result[8] = {}; ++ for (size_t j = 0; j < 8; j++) { ++ result[j] = decode_component(code, i + j); ++ } ++ float32x4_t res1 = vld1q_f32(result); ++ float32x4_t res2 = vld1q_f32(result + 4); ++ float32x4x2_t res = vzipq_f32(res1, res2); ++ return vuzpq_f32(res.val[0], res.val[1]); ++ } ++#endif + }; + + struct Codec4bit { +@@ -129,6 +143,20 @@ struct Codec4bit { + return _mm256_mul_ps(f8, one_255); + } + #endif ++ ++#ifdef __aarch64__ ++ static FAISS_ALWAYS_INLINE float32x4x2_t ++ decode_8_components(const uint8_t* code, int i) { ++ float32_t result[8] = {}; ++ for (size_t j = 0; j < 8; j++) { ++ result[j] = decode_component(code, i + j); ++ } ++ float32x4_t res1 = vld1q_f32(result); ++ float32x4_t res2 = vld1q_f32(result + 4); ++ float32x4x2_t res = vzipq_f32(res1, res2); ++ return vuzpq_f32(res.val[0], res.val[1]); ++ } ++#endif + }; + + struct Codec6bit { +@@ -228,6 +256,20 @@ struct Codec6bit { + } + + #endif ++ ++#ifdef __aarch64__ ++ static FAISS_ALWAYS_INLINE float32x4x2_t ++ decode_8_components(const uint8_t* code, int i) { ++ float32_t result[8] = {}; ++ for (size_t j = 0; j < 8; j++) { ++ result[j] = decode_component(code, i + j); ++ } ++ float32x4_t res1 = vld1q_f32(result); ++ float32x4_t res2 = vld1q_f32(result + 4); ++ float32x4x2_t res = vzipq_f32(res1, res2); ++ return vuzpq_f32(res.val[0], res.val[1]); ++ } ++#endif + }; + + /******************************************************************* +@@ -293,6 +335,31 @@ struct QuantizerTemplate : QuantizerTemplate { + + #endif + ++#ifdef __aarch64__ ++ ++template ++struct QuantizerTemplate : QuantizerTemplate { ++ QuantizerTemplate(size_t d, const std::vector& trained) ++ : QuantizerTemplate(d, trained) {} ++ ++ FAISS_ALWAYS_INLINE float32x4x2_t ++ reconstruct_8_components(const uint8_t* code, int i) const { ++ float32x4x2_t xi = Codec::decode_8_components(code, i); ++ float32x4x2_t res = vzipq_f32( ++ vfmaq_f32( ++ vdupq_n_f32(this->vmin), ++ xi.val[0], ++ vdupq_n_f32(this->vdiff)), ++ vfmaq_f32( ++ vdupq_n_f32(this->vmin), ++ xi.val[1], ++ vdupq_n_f32(this->vdiff))); ++ return vuzpq_f32(res.val[0], res.val[1]); ++ } ++}; ++ ++#endif ++ + template + struct QuantizerTemplate : ScalarQuantizer::SQuantizer { + const size_t d; +@@ -350,6 +417,29 @@ struct QuantizerTemplate : QuantizerTemplate { + + #endif + ++#ifdef __aarch64__ ++ ++template ++struct QuantizerTemplate : QuantizerTemplate { ++ QuantizerTemplate(size_t d, const std::vector& trained) ++ : QuantizerTemplate(d, trained) {} ++ ++ FAISS_ALWAYS_INLINE float32x4x2_t ++ reconstruct_8_components(const uint8_t* code, int i) const { ++ float32x4x2_t xi = Codec::decode_8_components(code, i); ++ ++ float32x4x2_t vmin_8 = vld1q_f32_x2(this->vmin + i); ++ float32x4x2_t vdiff_8 = vld1q_f32_x2(this->vdiff + i); ++ ++ float32x4x2_t res = vzipq_f32( ++ vfmaq_f32(vmin_8.val[0], xi.val[0], vdiff_8.val[0]), ++ vfmaq_f32(vmin_8.val[1], xi.val[1], vdiff_8.val[1])); ++ return vuzpq_f32(res.val[0], res.val[1]); ++ } ++}; ++ ++#endif ++ + /******************************************************************* + * FP16 quantizer + *******************************************************************/ +@@ -397,6 +487,23 @@ struct QuantizerFP16<8> : QuantizerFP16<1> { + + #endif + ++#ifdef __aarch64__ ++ ++template <> ++struct QuantizerFP16<8> : QuantizerFP16<1> { ++ QuantizerFP16(size_t d, const std::vector& trained) ++ : QuantizerFP16<1>(d, trained) {} ++ ++ FAISS_ALWAYS_INLINE float32x4x2_t ++ reconstruct_8_components(const uint8_t* code, int i) const { ++ uint16x4x2_t codei = vld2_u16((const uint16_t*)(code + 2 * i)); ++ return vzipq_f32( ++ vcvt_f32_f16(vreinterpret_f16_u16(codei.val[0])), ++ vcvt_f32_f16(vreinterpret_f16_u16(codei.val[1]))); ++ } ++}; ++#endif ++ + /******************************************************************* + * 8bit_direct quantizer + *******************************************************************/ +@@ -446,6 +553,28 @@ struct Quantizer8bitDirect<8> : Quantizer8bitDirect<1> { + + #endif + ++#ifdef __aarch64__ ++ ++template <> ++struct Quantizer8bitDirect<8> : Quantizer8bitDirect<1> { ++ Quantizer8bitDirect(size_t d, const std::vector& trained) ++ : Quantizer8bitDirect<1>(d, trained) {} ++ ++ FAISS_ALWAYS_INLINE float32x4x2_t ++ reconstruct_8_components(const uint8_t* code, int i) const { ++ float32_t result[8] = {}; ++ for (size_t j = 0; j < 8; j++) { ++ result[j] = code[i + j]; ++ } ++ float32x4_t res1 = vld1q_f32(result); ++ float32x4_t res2 = vld1q_f32(result + 4); ++ float32x4x2_t res = vzipq_f32(res1, res2); ++ return vuzpq_f32(res.val[0], res.val[1]); ++ } ++}; ++ ++#endif ++ + template + ScalarQuantizer::SQuantizer* select_quantizer_1( + QuantizerType qtype, +@@ -728,6 +857,59 @@ struct SimilarityL2<8> { + + #endif + ++#ifdef __aarch64__ ++template <> ++struct SimilarityL2<8> { ++ static constexpr int simdwidth = 8; ++ static constexpr MetricType metric_type = METRIC_L2; ++ ++ const float *y, *yi; ++ explicit SimilarityL2(const float* y) : y(y) {} ++ float32x4x2_t accu8; ++ ++ FAISS_ALWAYS_INLINE void begin_8() { ++ accu8 = vzipq_f32(vdupq_n_f32(0.0f), vdupq_n_f32(0.0f)); ++ yi = y; ++ } ++ ++ FAISS_ALWAYS_INLINE void add_8_components(float32x4x2_t x) { ++ float32x4x2_t yiv = vld1q_f32_x2(yi); ++ yi += 8; ++ ++ float32x4_t sub0 = vsubq_f32(yiv.val[0], x.val[0]); ++ float32x4_t sub1 = vsubq_f32(yiv.val[1], x.val[1]); ++ ++ float32x4_t accu8_0 = vfmaq_f32(accu8.val[0], sub0, sub0); ++ float32x4_t accu8_1 = vfmaq_f32(accu8.val[1], sub1, sub1); ++ ++ float32x4x2_t accu8_temp = vzipq_f32(accu8_0, accu8_1); ++ accu8 = vuzpq_f32(accu8_temp.val[0], accu8_temp.val[1]); ++ } ++ ++ FAISS_ALWAYS_INLINE void add_8_components_2( ++ float32x4x2_t x, ++ float32x4x2_t y) { ++ float32x4_t sub0 = vsubq_f32(y.val[0], x.val[0]); ++ float32x4_t sub1 = vsubq_f32(y.val[1], x.val[1]); ++ ++ float32x4_t accu8_0 = vfmaq_f32(accu8.val[0], sub0, sub0); ++ float32x4_t accu8_1 = vfmaq_f32(accu8.val[1], sub1, sub1); ++ ++ float32x4x2_t accu8_temp = vzipq_f32(accu8_0, accu8_1); ++ accu8 = vuzpq_f32(accu8_temp.val[0], accu8_temp.val[1]); ++ } ++ ++ FAISS_ALWAYS_INLINE float result_8() { ++ float32x4_t sum_0 = vpaddq_f32(accu8.val[0], accu8.val[0]); ++ float32x4_t sum_1 = vpaddq_f32(accu8.val[1], accu8.val[1]); ++ ++ float32x4_t sum2_0 = vpaddq_f32(sum_0, sum_0); ++ float32x4_t sum2_1 = vpaddq_f32(sum_1, sum_1); ++ return vgetq_lane_f32(sum2_0, 0) + vgetq_lane_f32(sum2_1, 0); ++ } ++}; ++#endif ++ + template + struct SimilarityIP {}; + +@@ -801,6 +983,56 @@ struct SimilarityIP<8> { + }; + #endif + ++#ifdef __aarch64__ ++ ++template <> ++struct SimilarityIP<8> { ++ static constexpr int simdwidth = 8; ++ static constexpr MetricType metric_type = METRIC_INNER_PRODUCT; ++ ++ const float *y, *yi; ++ ++ explicit SimilarityIP(const float* y) : y(y) {} ++ float32x4x2_t accu8; ++ ++ FAISS_ALWAYS_INLINE void begin_8() { ++ accu8 = vzipq_f32(vdupq_n_f32(0.0f), vdupq_n_f32(0.0f)); ++ yi = y; ++ } ++ ++ FAISS_ALWAYS_INLINE void add_8_components(float32x4x2_t x) { ++ float32x4x2_t yiv = vld1q_f32_x2(yi); ++ yi += 8; ++ ++ float32x4_t accu8_0 = vfmaq_f32(accu8.val[0], yiv.val[0], x.val[0]); ++ float32x4_t accu8_1 = vfmaq_f32(accu8.val[1], yiv.val[1], x.val[1]); ++ float32x4x2_t accu8_temp = vzipq_f32(accu8_0, accu8_1); ++ accu8 = vuzpq_f32(accu8_temp.val[0], accu8_temp.val[1]); ++ } ++ ++ FAISS_ALWAYS_INLINE void add_8_components_2( ++ float32x4x2_t x1, ++ float32x4x2_t x2) { ++ float32x4_t accu8_0 = vfmaq_f32(accu8.val[0], x1.val[0], x2.val[0]); ++ float32x4_t accu8_1 = vfmaq_f32(accu8.val[1], x1.val[1], x2.val[1]); ++ float32x4x2_t accu8_temp = vzipq_f32(accu8_0, accu8_1); ++ accu8 = vuzpq_f32(accu8_temp.val[0], accu8_temp.val[1]); ++ } ++ ++ FAISS_ALWAYS_INLINE float result_8() { ++ float32x4x2_t sum_tmp = vzipq_f32( ++ vpaddq_f32(accu8.val[0], accu8.val[0]), ++ vpaddq_f32(accu8.val[1], accu8.val[1])); ++ float32x4x2_t sum = vuzpq_f32(sum_tmp.val[0], sum_tmp.val[1]); ++ float32x4x2_t sum2_tmp = vzipq_f32( ++ vpaddq_f32(sum.val[0], sum.val[0]), ++ vpaddq_f32(sum.val[1], sum.val[1])); ++ float32x4x2_t sum2 = vuzpq_f32(sum2_tmp.val[0], sum2_tmp.val[1]); ++ return vgetq_lane_f32(sum2.val[0], 0) + vgetq_lane_f32(sum2.val[1], 0); ++ } ++}; ++#endif ++ + /******************************************************************* + * DistanceComputer: combines a similarity and a quantizer to do + * code-to-vector or code-to-code comparisons +@@ -903,6 +1135,53 @@ struct DCTemplate : SQDistanceComputer { + + #endif + ++#ifdef __aarch64__ ++ ++template ++struct DCTemplate : SQDistanceComputer { ++ using Sim = Similarity; ++ ++ Quantizer quant; ++ ++ DCTemplate(size_t d, const std::vector& trained) ++ : quant(d, trained) {} ++ float compute_distance(const float* x, const uint8_t* code) const { ++ Similarity sim(x); ++ sim.begin_8(); ++ for (size_t i = 0; i < quant.d; i += 8) { ++ float32x4x2_t xi = quant.reconstruct_8_components(code, i); ++ sim.add_8_components(xi); ++ } ++ return sim.result_8(); ++ } ++ ++ float compute_code_distance(const uint8_t* code1, const uint8_t* code2) ++ const { ++ Similarity sim(nullptr); ++ sim.begin_8(); ++ for (size_t i = 0; i < quant.d; i += 8) { ++ float32x4x2_t x1 = quant.reconstruct_8_components(code1, i); ++ float32x4x2_t x2 = quant.reconstruct_8_components(code2, i); ++ sim.add_8_components_2(x1, x2); ++ } ++ return sim.result_8(); ++ } ++ ++ void set_query(const float* x) final { ++ q = x; ++ } ++ ++ float symmetric_dis(idx_t i, idx_t j) override { ++ return compute_code_distance( ++ codes + i * code_size, codes + j * code_size); ++ } ++ ++ float query_to_code(const uint8_t* code) const final { ++ return compute_distance(q, code); ++ } ++}; ++#endif ++ + /******************************************************************* + * DistanceComputerByte: computes distances in the integer domain + *******************************************************************/ +@@ -1019,6 +1298,54 @@ struct DistanceComputerByte : SQDistanceComputer { + + #endif + ++#ifdef __aarch64__ ++ ++template ++struct DistanceComputerByte : SQDistanceComputer { ++ using Sim = Similarity; ++ ++ int d; ++ std::vector tmp; ++ ++ DistanceComputerByte(int d, const std::vector&) : d(d), tmp(d) {} ++ ++ int compute_code_distance(const uint8_t* code1, const uint8_t* code2) ++ const { ++ int accu = 0; ++ for (int i = 0; i < d; i++) { ++ if (Sim::metric_type == METRIC_INNER_PRODUCT) { ++ accu += int(code1[i]) * code2[i]; ++ } else { ++ int diff = int(code1[i]) - code2[i]; ++ accu += diff * diff; ++ } ++ } ++ return accu; ++ } ++ ++ void set_query(const float* x) final { ++ for (int i = 0; i < d; i++) { ++ tmp[i] = int(x[i]); ++ } ++ } ++ ++ int compute_distance(const float* x, const uint8_t* code) { ++ set_query(x); ++ return compute_code_distance(tmp.data(), code); ++ } ++ ++ float symmetric_dis(idx_t i, idx_t j) override { ++ return compute_code_distance( ++ codes + i * code_size, codes + j * code_size); ++ } ++ ++ float query_to_code(const uint8_t* code) const final { ++ return compute_code_distance(tmp.data(), code); ++ } ++}; ++ ++#endif ++ + /******************************************************************* + * select_distance_computer: runtime selection of template + * specialization +@@ -1155,7 +1482,7 @@ void ScalarQuantizer::train(size_t n, const float* x) { + } + + ScalarQuantizer::SQuantizer* ScalarQuantizer::select_quantizer() const { +-#ifdef USE_F16C ++#if defined(USE_F16C) || defined(__aarch64__) + if (d % 8 == 0) { + return select_quantizer_1<8>(qtype, d, trained); + } else +@@ -1186,7 +1513,7 @@ void ScalarQuantizer::decode(const uint8_t* codes, float* x, size_t n) const { + SQDistanceComputer* ScalarQuantizer::get_distance_computer( + MetricType metric) const { + FAISS_THROW_IF_NOT(metric == METRIC_L2 || metric == METRIC_INNER_PRODUCT); +-#ifdef USE_F16C ++#if defined(USE_F16C) || defined(__aarch64__) + if (d % 8 == 0) { + if (metric == METRIC_L2) { + return select_distance_computer>(qtype, d, trained); +@@ -1522,7 +1849,7 @@ InvertedListScanner* ScalarQuantizer::select_InvertedListScanner( + bool store_pairs, + const IDSelector* sel, + bool by_residual) const { +-#ifdef USE_F16C ++#if defined(USE_F16C) || defined(__aarch64__) + if (d % 8 == 0) { + return sel0_InvertedListScanner<8>( + mt, this, quantizer, store_pairs, sel, by_residual); +diff --git a/faiss/utils/fp16-arm.h b/faiss/utils/fp16-arm.h +new file mode 100644 +index 00000000..79c885b0 +--- /dev/null ++++ b/faiss/utils/fp16-arm.h +@@ -0,0 +1,29 @@ ++/** ++ * Copyright (c) Facebook, Inc. and its affiliates. ++ * ++ * This source code is licensed under the MIT license found in the ++ * LICENSE file in the root directory of this source tree. ++ */ ++ ++#pragma once ++ ++#include ++#include ++ ++namespace faiss { ++ ++inline uint16_t encode_fp16(float x) { ++ float32x4_t fx4 = vdupq_n_f32(x); ++ float16x4_t f16x4 = vcvt_f16_f32(fx4); ++ uint16x4_t ui16x4 = vreinterpret_u16_f16(f16x4); ++ return vduph_lane_u16(ui16x4, 3); ++} ++ ++inline float decode_fp16(uint16_t x) { ++ uint16x4_t ui16x4 = vdup_n_u16(x); ++ float16x4_t f16x4 = vreinterpret_f16_u16(ui16x4); ++ float32x4_t fx4 = vcvt_f32_f16(f16x4); ++ return vdups_laneq_f32(fx4, 3); ++} ++ ++} // namespace faiss +diff --git a/faiss/utils/fp16.h b/faiss/utils/fp16.h +index 90691d8f..43e05dc3 100644 +--- a/faiss/utils/fp16.h ++++ b/faiss/utils/fp16.h +@@ -13,6 +13,8 @@ + + #if defined(__F16C__) + #include ++#elif defined(__aarch64__) ++#include + #else + #include + #endif diff --git a/jni/patches/faiss/0003-Custom-patch-to-support-AVX2-Linux-CI.patch b/jni/patches/faiss/0003-Custom-patch-to-support-AVX2-Linux-CI.patch new file mode 100644 index 000000000..22d50e66c --- /dev/null +++ b/jni/patches/faiss/0003-Custom-patch-to-support-AVX2-Linux-CI.patch @@ -0,0 +1,32 @@ +Temporarily replace the intrinsic '_mm_loadu_si64' with '_mm_loadl_epi64' until centOS7 is deprecated in our CI on Linux OS. +centOS7 only supports gcc version upto 8.x. But, the intrinsic '_mm_loadu_si64' requires gcc version of minimum 9.x. +So, replacing it with an equivalent intrinsic. + +diff --git a/faiss/impl/code_distance/code_distance-avx2.h b/faiss/impl/code_distance/code_distance-avx2.h +index 0aa1535b..6e4e5b55 100644 +--- a/faiss/impl/code_distance/code_distance-avx2.h ++++ b/faiss/impl/code_distance/code_distance-avx2.h +@@ -91,7 +91,7 @@ float inline distance_single_code_avx2_pqdecoder8_m8( + __m256 partialSum; + + // load 8 uint8 values +- const __m128i mm1 = _mm_loadu_si64((const __m128i_u*)code); ++ const __m128i mm1 = _mm_loadl_epi64((const __m128i_u*)code); + { + // convert uint8 values (low part of __m128i) to int32 + // values +@@ -199,10 +199,10 @@ inline void distance_four_codes_avx2_pqdecoder8_m8( + + // load 8 uint8 values + __m128i mm1[N]; +- mm1[0] = _mm_loadu_si64((const __m128i_u*)code0); +- mm1[1] = _mm_loadu_si64((const __m128i_u*)code1); +- mm1[2] = _mm_loadu_si64((const __m128i_u*)code2); +- mm1[3] = _mm_loadu_si64((const __m128i_u*)code3); ++ mm1[0] = _mm_loadl_epi64((const __m128i_u*)code0); ++ mm1[1] = _mm_loadl_epi64((const __m128i_u*)code1); ++ mm1[2] = _mm_loadl_epi64((const __m128i_u*)code2); ++ mm1[3] = _mm_loadl_epi64((const __m128i_u*)code3); + + for (intptr_t j = 0; j < N; j++) { + // convert uint8 values (low part of __m128i) to int32 diff --git a/jni/tests/faiss_wrapper_test.cpp b/jni/tests/faiss_wrapper_test.cpp index ed3ec880d..5afe09c22 100644 --- a/jni/tests/faiss_wrapper_test.cpp +++ b/jni/tests/faiss_wrapper_test.cpp @@ -17,6 +17,7 @@ #include "gtest/gtest.h" #include "jni_util.h" #include "test_util.h" +#include "faiss/IndexHNSW.h" using ::testing::NiceMock; using ::testing::Return; @@ -425,3 +426,55 @@ TEST(FaissTrainIndexTest, BasicAssertions) { // Confirm that training succeeded ASSERT_TRUE(trainedIndex->is_trained); } + +TEST(FaissCreateHnswSQfp16IndexTest, BasicAssertions) { + // Define the data + faiss::idx_t numIds = 200; + std::vector ids; + std::vector> vectors; + int dim = 2; + for (int64_t i = 0; i < numIds; ++i) { + ids.push_back(i); + + std::vector vect; + vect.reserve(dim); + for (int j = 0; j < dim; ++j) { + vect.push_back(test_util::RandomFloat(-500.0, 500.0)); + } + vectors.push_back(vect); + } + + std::string indexPath = test_util::RandomString(10, "tmp/", ".faiss"); + std::string spaceType = knn_jni::L2; + std::string index_description = "HNSW32,SQfp16"; + + std::unordered_map parametersMap; + parametersMap[knn_jni::SPACE_TYPE] = (jobject)&spaceType; + parametersMap[knn_jni::INDEX_DESCRIPTION] = (jobject)&index_description; + + // Set up jni + JNIEnv *jniEnv = nullptr; + NiceMock mockJNIUtil; + + EXPECT_CALL(mockJNIUtil, + GetJavaObjectArrayLength( + jniEnv, reinterpret_cast(&vectors))) + .WillRepeatedly(Return(vectors.size())); + + // Create the index + knn_jni::faiss_wrapper::CreateIndex( + &mockJNIUtil, jniEnv, reinterpret_cast(&ids), + reinterpret_cast(&vectors), (jstring)&indexPath, + (jobject)¶metersMap); + + // Make sure index can be loaded + std::unique_ptr index(test_util::FaissLoadIndex(indexPath)); + auto indexIDMap = dynamic_cast(index.get()); + + // Assert that Index is of type IndexHNSWSQ + ASSERT_NE(indexIDMap, nullptr); + ASSERT_NE(dynamic_cast(indexIDMap->index), nullptr); + + // Clean up + std::remove(indexPath.c_str()); +} diff --git a/src/main/java/org/opensearch/knn/common/KNNConstants.java b/src/main/java/org/opensearch/knn/common/KNNConstants.java index e4d59a00f..b8089d858 100644 --- a/src/main/java/org/opensearch/knn/common/KNNConstants.java +++ b/src/main/java/org/opensearch/knn/common/KNNConstants.java @@ -7,6 +7,8 @@ import org.opensearch.knn.index.VectorDataType; +import java.util.List; + public class KNNConstants { // shared across library constants public static final String DIMENSION = "dimension"; @@ -89,6 +91,11 @@ public class KNNConstants { public static final String FAISS_IVF_DESCRIPTION = "IVF"; public static final String FAISS_FLAT_DESCRIPTION = "Flat"; public static final String FAISS_PQ_DESCRIPTION = "PQ"; + public static final String ENCODER_SQ = "sq"; + public static final String FAISS_SQ_DESCRIPTION = "SQ"; + public static final String FAISS_SQ_TYPE = "type"; + public static final String FAISS_SQ_ENCODER_FP16 = "fp16"; + public static final List FAISS_SQ_ENCODER_TYPES = List.of(FAISS_SQ_ENCODER_FP16); // Parameter defaults/limits public static final Integer ENCODER_PARAMETER_PQ_CODE_COUNT_DEFAULT = 1; diff --git a/src/main/java/org/opensearch/knn/index/Parameter.java b/src/main/java/org/opensearch/knn/index/Parameter.java index 4d69e7838..bef5a33e9 100644 --- a/src/main/java/org/opensearch/knn/index/Parameter.java +++ b/src/main/java/org/opensearch/knn/index/Parameter.java @@ -95,6 +95,49 @@ public ValidationException validate(Object value) { } } + /** + * String method parameter + */ + public static class StringParameter extends Parameter { + + /** + * Constructor + * + * @param name of the parameter + * @param defaultValue value to assign if the parameter is not set + * @param validator used to validate the parameter value passed + */ + public StringParameter(String name, String defaultValue, Predicate validator) { + super(name, defaultValue, validator); + } + + /** + * Check if the value passed in is valid + * + * @param value to be checked + * @return ValidationException produced by validation errors; null if no validations errors. + */ + @Override + public ValidationException validate(Object value) { + ValidationException validationException = null; + if (!(value instanceof String)) { + validationException = new ValidationException(); + validationException.addValidationError( + String.format("Value not of type String for String " + "parameter \"%s\".", getName()) + ); + return validationException; + } + + if (!validator.test((String) value)) { + validationException = new ValidationException(); + validationException.addValidationError( + String.format("Parameter validation failed for String " + "parameter \"%s\".", getName()) + ); + } + return validationException; + } + } + /** * MethodContext parameter. Some methods require sub-methods in order to implement some kind of functionality. For * instance, faiss methods can contain an encoder along side the approximate nearest neighbor function to compress diff --git a/src/main/java/org/opensearch/knn/index/util/Faiss.java b/src/main/java/org/opensearch/knn/index/util/Faiss.java index 71eed404a..420288033 100644 --- a/src/main/java/org/opensearch/knn/index/util/Faiss.java +++ b/src/main/java/org/opensearch/knn/index/util/Faiss.java @@ -28,9 +28,14 @@ import static org.opensearch.knn.common.KNNConstants.ENCODER_PARAMETER_PQ_CODE_SIZE_DEFAULT; import static org.opensearch.knn.common.KNNConstants.ENCODER_PARAMETER_PQ_CODE_SIZE_LIMIT; import static org.opensearch.knn.common.KNNConstants.ENCODER_PARAMETER_PQ_M; +import static org.opensearch.knn.common.KNNConstants.ENCODER_SQ; import static org.opensearch.knn.common.KNNConstants.FAISS_HNSW_DESCRIPTION; import static org.opensearch.knn.common.KNNConstants.FAISS_IVF_DESCRIPTION; import static org.opensearch.knn.common.KNNConstants.FAISS_PQ_DESCRIPTION; +import static org.opensearch.knn.common.KNNConstants.FAISS_SQ_DESCRIPTION; +import static org.opensearch.knn.common.KNNConstants.FAISS_SQ_ENCODER_FP16; +import static org.opensearch.knn.common.KNNConstants.FAISS_SQ_ENCODER_TYPES; +import static org.opensearch.knn.common.KNNConstants.FAISS_SQ_TYPE; import static org.opensearch.knn.common.KNNConstants.METHOD_ENCODER_PARAMETER; import static org.opensearch.knn.common.KNNConstants.METHOD_HNSW; import static org.opensearch.knn.common.KNNConstants.METHOD_IVF; @@ -75,6 +80,20 @@ class Faiss extends NativeLibrary { methodComponentContext ).build()) ) + .build(), + ENCODER_SQ, + MethodComponent.Builder.builder(ENCODER_SQ) + .addParameter( + FAISS_SQ_TYPE, + new Parameter.StringParameter(FAISS_SQ_TYPE, FAISS_SQ_ENCODER_FP16, FAISS_SQ_ENCODER_TYPES::contains) + ) + .setMapGenerator( + ((methodComponent, methodComponentContext) -> MethodAsMapBuilder.builder( + FAISS_SQ_DESCRIPTION, + methodComponent, + methodComponentContext + ).addParameter(FAISS_SQ_TYPE, "", "").build()) + ) .build() ); diff --git a/src/test/java/org/opensearch/knn/index/FaissIT.java b/src/test/java/org/opensearch/knn/index/FaissIT.java index b373832d1..3096d1332 100644 --- a/src/test/java/org/opensearch/knn/index/FaissIT.java +++ b/src/test/java/org/opensearch/knn/index/FaissIT.java @@ -14,14 +14,14 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.primitives.Floats; -import org.apache.http.util.EntityUtils; import lombok.SneakyThrows; +import org.apache.http.util.EntityUtils; import org.junit.BeforeClass; import org.opensearch.client.Response; +import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.index.query.QueryBuilders; import org.opensearch.knn.KNNRestTestCase; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.knn.KNNResult; import org.opensearch.knn.TestUtils; import org.opensearch.knn.common.KNNConstants; @@ -34,15 +34,20 @@ import java.util.Arrays; import java.util.List; import java.util.Map; +import java.util.Random; import java.util.TreeMap; import java.util.stream.Collectors; import static org.opensearch.knn.common.KNNConstants.ENCODER_PARAMETER_PQ_M; import static org.opensearch.knn.common.KNNConstants.ENCODER_PQ; +import static org.opensearch.knn.common.KNNConstants.ENCODER_SQ; import static org.opensearch.knn.common.KNNConstants.FAISS_NAME; +import static org.opensearch.knn.common.KNNConstants.FAISS_SQ_ENCODER_FP16; +import static org.opensearch.knn.common.KNNConstants.FAISS_SQ_TYPE; import static org.opensearch.knn.common.KNNConstants.KNN_ENGINE; import static org.opensearch.knn.common.KNNConstants.METHOD_ENCODER_PARAMETER; import static org.opensearch.knn.common.KNNConstants.METHOD_HNSW; +import static org.opensearch.knn.common.KNNConstants.METHOD_IVF; import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_NLIST; import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_SPACE_TYPE; import static org.opensearch.knn.common.KNNConstants.MODEL_ID; @@ -267,6 +272,119 @@ public void testEndToEnd_whenMethodIsHNSWPQ_thenSucceed() { fail("Graphs are not getting evicted"); } + @SneakyThrows + public void testHNSWSQFP16_whenIndexedAndQueried_thenSucceed() { + String indexName = "test-index-hnsw-sqfp16"; + String fieldName = "test-field-hnsw-sqfp16"; + + KNNMethod hnswMethod = KNNEngine.FAISS.getMethod(KNNConstants.METHOD_HNSW); + SpaceType[] spaceTypes = { SpaceType.L2, SpaceType.INNER_PRODUCT }; + Random random = new Random(); + SpaceType spaceType = spaceTypes[random.nextInt(spaceTypes.length)]; + + List mValues = ImmutableList.of(16, 32, 64, 128); + List efConstructionValues = ImmutableList.of(16, 32, 64, 128); + List efSearchValues = ImmutableList.of(16, 32, 64, 128); + + int dimension = 128; + int numDocs = 100; + + // Create an index + XContentBuilder builder = XContentFactory.jsonBuilder() + .startObject() + .startObject("properties") + .startObject(fieldName) + .field("type", "knn_vector") + .field("dimension", dimension) + .startObject(KNNConstants.KNN_METHOD) + .field(KNNConstants.NAME, hnswMethod.getMethodComponent().getName()) + .field(KNNConstants.METHOD_PARAMETER_SPACE_TYPE, spaceType.getValue()) + .field(KNNConstants.KNN_ENGINE, KNNEngine.FAISS.getName()) + .startObject(KNNConstants.PARAMETERS) + .field(KNNConstants.METHOD_PARAMETER_M, mValues.get(random().nextInt(mValues.size()))) + .field(KNNConstants.METHOD_PARAMETER_EF_CONSTRUCTION, efConstructionValues.get(random().nextInt(efConstructionValues.size()))) + .field(KNNConstants.METHOD_PARAMETER_EF_SEARCH, efSearchValues.get(random().nextInt(efSearchValues.size()))) + .startObject(METHOD_ENCODER_PARAMETER) + .field(NAME, ENCODER_SQ) + .startObject(PARAMETERS) + .field(FAISS_SQ_TYPE, FAISS_SQ_ENCODER_FP16) + .endObject() + .endObject() + .endObject() + .endObject() + .endObject() + .endObject() + .endObject(); + + Map mappingMap = xContentBuilderToMap(builder); + String mapping = builder.toString(); + + createKnnIndex(indexName, mapping); + assertEquals(new TreeMap<>(mappingMap), new TreeMap<>(getIndexMappingAsMap(indexName))); + indexTestData(indexName, fieldName, dimension, numDocs); + queryTestData(indexName, fieldName, dimension, numDocs); + deleteKNNIndex(indexName); + validateGraphEviction(); + } + + @SneakyThrows + public void testIVFSQFP16_whenIndexedAndQueried_thenSucceed() { + + String modelId = "test-model-ivf-sqfp16"; + int dimension = 128; + int numDocs = 100; + + String trainingIndexName = "train-index-ivf-sqfp16"; + String trainingFieldName = "train-field-ivf-sqfp16"; + + // Add training data + createBasicKnnIndex(trainingIndexName, trainingFieldName, dimension); + int trainingDataCount = 200; + bulkIngestRandomVectors(trainingIndexName, trainingFieldName, trainingDataCount, dimension); + + XContentBuilder builder = XContentFactory.jsonBuilder() + .startObject() + .field(NAME, METHOD_IVF) + .field(KNN_ENGINE, FAISS_NAME) + .field(METHOD_PARAMETER_SPACE_TYPE, "l2") + .startObject(PARAMETERS) + .startObject(METHOD_ENCODER_PARAMETER) + .field(NAME, ENCODER_SQ) + .startObject(PARAMETERS) + .field(FAISS_SQ_TYPE, FAISS_SQ_ENCODER_FP16) + .endObject() + .endObject() + .endObject() + .endObject(); + Map method = xContentBuilderToMap(builder); + + trainModel(modelId, trainingIndexName, trainingFieldName, dimension, method, "faiss ivf sqfp16 test description"); + + // Make sure training succeeds after 30 seconds + assertTrainingSucceeds(modelId, 30, 1000); + + // Create knn index from model + String fieldName = "test-field-name-ivf-sqfp16"; + String indexName = "test-index-name-ivf-sqfp16"; + String indexMapping = XContentFactory.jsonBuilder() + .startObject() + .startObject("properties") + .startObject(fieldName) + .field("type", "knn_vector") + .field(MODEL_ID, modelId) + .endObject() + .endObject() + .endObject() + .toString(); + + createKnnIndex(indexName, getKNNDefaultIndexSettings(), indexMapping); + + indexTestData(indexName, fieldName, dimension, numDocs); + queryTestData(indexName, fieldName, dimension, numDocs); + deleteKNNIndex(indexName); + validateGraphEviction(); + } + @SneakyThrows public void testEndToEnd_whenMethodIsHNSWPQAndHyperParametersNotSet_thenSucceed() { String indexName = "test-index"; @@ -623,4 +741,43 @@ protected void setupKNNIndexForFilterQuery() throws Exception { refreshIndex(INDEX_NAME); } + + private void queryTestData(final String indexName, final String fieldName, final int dimension, final int numDocs) throws IOException { + float[] queryVector = new float[dimension]; + Arrays.fill(queryVector, (float) numDocs); + int k = 10; + + Response searchResponse = searchKNNIndex(indexName, new KNNQueryBuilder(fieldName, queryVector, k), k); + List results = parseSearchResponse(EntityUtils.toString(searchResponse.getEntity()), fieldName); + assertEquals(k, results.size()); + for (int i = 0; i < k; i++) { + assertEquals(numDocs - i - 1, Integer.parseInt(results.get(i).getDocId())); + } + } + + private void indexTestData(final String indexName, final String fieldName, final int dimension, final int numDocs) throws Exception { + for (int i = 0; i < numDocs; i++) { + float[] indexVector = new float[dimension]; + Arrays.fill(indexVector, (float) i); + addKnnDocWithAttributes(indexName, Integer.toString(i), fieldName, indexVector, ImmutableMap.of("rating", String.valueOf(i))); + } + + // Assert that all docs are ingested + refreshAllNonSystemIndices(); + assertEquals(numDocs, getDocCount(indexName)); + } + + private void validateGraphEviction() throws Exception { + // Search every 5 seconds 14 times to confirm graph gets evicted + int intervals = 14; + for (int i = 0; i < intervals; i++) { + if (getTotalGraphsInCache() == 0) { + return; + } + + Thread.sleep(5 * 1000); + } + + fail("Graphs are not getting evicted"); + } } diff --git a/src/test/java/org/opensearch/knn/index/ParameterTests.java b/src/test/java/org/opensearch/knn/index/ParameterTests.java index 4e7adfc8c..08decd592 100644 --- a/src/test/java/org/opensearch/knn/index/ParameterTests.java +++ b/src/test/java/org/opensearch/knn/index/ParameterTests.java @@ -15,6 +15,7 @@ import org.opensearch.knn.KNNTestCase; import org.opensearch.common.ValidationException; import org.opensearch.knn.index.Parameter.IntegerParameter; +import org.opensearch.knn.index.Parameter.StringParameter; import org.opensearch.knn.index.Parameter.MethodComponentContextParameter; import java.util.Map; @@ -51,6 +52,19 @@ public void testIntegerParameter_validate() { assertNull(parameter.validate(12)); } + public void testStringParameter_validate() { + final StringParameter parameter = new StringParameter("test_parameter", "default_value", v -> "test".equals(v)); + + // Invalid type + assertNotNull(parameter.validate(5)); + + // null + assertNotNull(parameter.validate(null)); + + // valid value + assertNull(parameter.validate("test")); + } + public void testMethodComponentContextParameter_validate() { String methodComponentName1 = "method-1"; String parameterKey1 = "parameter_key_1"; diff --git a/src/test/java/org/opensearch/knn/index/util/FaissTests.java b/src/test/java/org/opensearch/knn/index/util/FaissTests.java index 0e7bc6482..5dc348a29 100644 --- a/src/test/java/org/opensearch/knn/index/util/FaissTests.java +++ b/src/test/java/org/opensearch/knn/index/util/FaissTests.java @@ -5,6 +5,7 @@ package org.opensearch.knn.index.util; +import lombok.SneakyThrows; import org.opensearch.Version; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; @@ -22,7 +23,10 @@ import static org.opensearch.knn.common.KNNConstants.ENCODER_PARAMETER_PQ_CODE_SIZE; import static org.opensearch.knn.common.KNNConstants.ENCODER_PARAMETER_PQ_M; import static org.opensearch.knn.common.KNNConstants.ENCODER_PQ; +import static org.opensearch.knn.common.KNNConstants.ENCODER_SQ; import static org.opensearch.knn.common.KNNConstants.FAISS_NAME; +import static org.opensearch.knn.common.KNNConstants.FAISS_SQ_ENCODER_FP16; +import static org.opensearch.knn.common.KNNConstants.FAISS_SQ_TYPE; import static org.opensearch.knn.common.KNNConstants.INDEX_DESCRIPTION_PARAMETER; import static org.opensearch.knn.common.KNNConstants.KNN_ENGINE; import static org.opensearch.knn.common.KNNConstants.METHOD_ENCODER_PARAMETER; @@ -86,6 +90,35 @@ public void testGetMethodAsMap_whenMethodIsHNSWPQ_thenCreateCorrectIndexDescript assertEquals(expectedIndexDescription, map.get(INDEX_DESCRIPTION_PARAMETER)); } + @SneakyThrows + public void testGetMethodAsMap_whenMethodIsHNSWSQFP16_thenCreateCorrectIndexDescription() { + int hnswMParam = 65; + String expectedIndexDescription = String.format(Locale.ROOT, "HNSW%d,SQfp16", hnswMParam); + + XContentBuilder xContentBuilder = XContentFactory.jsonBuilder() + .startObject() + .field(NAME, METHOD_HNSW) + .field(KNN_ENGINE, FAISS_NAME) + .startObject(PARAMETERS) + .field(METHOD_PARAMETER_M, hnswMParam) + .startObject(METHOD_ENCODER_PARAMETER) + .field(NAME, ENCODER_SQ) + .startObject(PARAMETERS) + .field(FAISS_SQ_TYPE, FAISS_SQ_ENCODER_FP16) + .endObject() + .endObject() + .endObject() + .endObject(); + Map in = xContentBuilderToMap(xContentBuilder); + KNNMethodContext knnMethodContext = KNNMethodContext.parse(in); + knnMethodContext.getMethodComponentContext().setIndexVersion(Version.CURRENT); + + Map map = Faiss.INSTANCE.getMethodAsMap(knnMethodContext); + + assertTrue(map.containsKey(INDEX_DESCRIPTION_PARAMETER)); + assertEquals(expectedIndexDescription, map.get(INDEX_DESCRIPTION_PARAMETER)); + } + public void testGetMethodAsMap_whenMethodIsIVFFlat_thenCreateCorrectIndexDescription() throws IOException { int nlists = 88; String expectedIndexDescription = String.format(Locale.ROOT, "IVF%d,Flat", nlists); @@ -137,6 +170,34 @@ public void testGetMethodAsMap_whenMethodIsIVFPQ_thenCreateCorrectIndexDescripti assertEquals(expectedIndexDescription, map.get(INDEX_DESCRIPTION_PARAMETER)); } + @SneakyThrows + public void testGetMethodAsMap_whenMethodIsIVFSQFP16_thenCreateCorrectIndexDescription() { + int nlists = 88; + String expectedIndexDescription = String.format(Locale.ROOT, "IVF%d,SQfp16", nlists); + + XContentBuilder xContentBuilder = XContentFactory.jsonBuilder() + .startObject() + .field(NAME, METHOD_IVF) + .field(KNN_ENGINE, FAISS_NAME) + .startObject(PARAMETERS) + .field(METHOD_PARAMETER_NLIST, nlists) + .startObject(METHOD_ENCODER_PARAMETER) + .field(NAME, ENCODER_SQ) + .startObject(PARAMETERS) + .field(FAISS_SQ_TYPE, FAISS_SQ_ENCODER_FP16) + .endObject() + .endObject() + .endObject() + .endObject(); + Map in = xContentBuilderToMap(xContentBuilder); + KNNMethodContext knnMethodContext = KNNMethodContext.parse(in); + + Map map = Faiss.INSTANCE.getMethodAsMap(knnMethodContext); + + assertTrue(map.containsKey(INDEX_DESCRIPTION_PARAMETER)); + assertEquals(expectedIndexDescription, map.get(INDEX_DESCRIPTION_PARAMETER)); + } + public void testMethodAsMapBuilder() throws IOException { String methodName = "test-method"; String methodDescription = "test-description"; diff --git a/src/test/java/org/opensearch/knn/jni/JNIServiceTests.java b/src/test/java/org/opensearch/knn/jni/JNIServiceTests.java index 0a05c95a0..c2470ea47 100644 --- a/src/test/java/org/opensearch/knn/jni/JNIServiceTests.java +++ b/src/test/java/org/opensearch/knn/jni/JNIServiceTests.java @@ -13,6 +13,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import lombok.SneakyThrows; import org.junit.BeforeClass; import org.opensearch.Version; import org.opensearch.common.xcontent.XContentFactory; @@ -41,7 +42,10 @@ import static org.opensearch.knn.common.KNNConstants.ENCODER_PARAMETER_PQ_M; import static org.opensearch.knn.common.KNNConstants.ENCODER_PARAMETER_PQ_CODE_SIZE; import static org.opensearch.knn.common.KNNConstants.ENCODER_PQ; +import static org.opensearch.knn.common.KNNConstants.ENCODER_SQ; import static org.opensearch.knn.common.KNNConstants.FAISS_NAME; +import static org.opensearch.knn.common.KNNConstants.FAISS_SQ_ENCODER_FP16; +import static org.opensearch.knn.common.KNNConstants.FAISS_SQ_TYPE; import static org.opensearch.knn.common.KNNConstants.INDEX_DESCRIPTION_PARAMETER; import static org.opensearch.knn.common.KNNConstants.INDEX_THREAD_QTY; import static org.opensearch.knn.common.KNNConstants.KNN_ENGINE; @@ -466,6 +470,111 @@ public void testCreateIndex_faiss_invalid_invalidIndexDescription() throws IOExc ); } + @SneakyThrows + public void testCreateIndex_faiss_sqfp16_invalidIndexDescription() { + + int[] docIds = new int[] { 1, 2 }; + float[][] vectors = new float[][] { { 2, 3 }, { 3, 4 } }; + String sqfp16InvalidIndexDescription = "HNSW16,SQfp1655"; + + Path tmpFile = createTempFile(); + expectThrows( + Exception.class, + () -> JNIService.createIndex( + docIds, + vectors, + tmpFile.toAbsolutePath().toString(), + ImmutableMap.of( + INDEX_DESCRIPTION_PARAMETER, + sqfp16InvalidIndexDescription, + KNNConstants.SPACE_TYPE, + SpaceType.L2.getValue() + ), + FAISS_NAME + ) + ); + } + + @SneakyThrows + public void testLoadIndex_faiss_sqfp16_valid() { + + int[] docIds = new int[] { 1, 2 }; + float[][] vectors = new float[][] { { 2, 3 }, { 3, 4 } }; + String sqfp16IndexDescription = "HNSW16,SQfp16"; + + Path tmpFile = createTempFile(); + JNIService.createIndex( + docIds, + vectors, + tmpFile.toAbsolutePath().toString(), + ImmutableMap.of(INDEX_DESCRIPTION_PARAMETER, sqfp16IndexDescription, KNNConstants.SPACE_TYPE, SpaceType.L2.getValue()), + FAISS_NAME + ); + assertTrue(tmpFile.toFile().length() > 0); + + long pointer = JNIService.loadIndex(tmpFile.toAbsolutePath().toString(), Collections.emptyMap(), FAISS_NAME); + assertNotEquals(0, pointer); + } + + @SneakyThrows + public void testQueryIndex_faiss_sqfp16_valid() { + + String sqfp16IndexDescription = "HNSW16,SQfp16"; + int k = 10; + + Path tmpFile = createTempFile(); + JNIService.createIndex( + testData.indexData.docs, + testData.indexData.vectors, + tmpFile.toAbsolutePath().toString(), + ImmutableMap.of(INDEX_DESCRIPTION_PARAMETER, sqfp16IndexDescription, KNNConstants.SPACE_TYPE, SpaceType.L2.getValue()), + FAISS_NAME + ); + assertTrue(tmpFile.toFile().length() > 0); + + long pointer = JNIService.loadIndex(tmpFile.toAbsolutePath().toString(), Collections.emptyMap(), FAISS_NAME); + assertNotEquals(0, pointer); + + for (float[] query : testData.queries) { + KNNQueryResult[] results = JNIService.queryIndex(pointer, query, k, FAISS_NAME, null, null); + assertEquals(k, results.length); + } + + // Filter will result in no ids + for (float[] query : testData.queries) { + KNNQueryResult[] results = JNIService.queryIndex(pointer, query, k, FAISS_NAME, new int[] { 0 }, null); + assertEquals(0, results.length); + } + } + + @SneakyThrows + public void testTrain_whenConfigurationIsIVFSQFP16_thenSucceed() { + long trainPointer = transferVectors(10); + int ivfNlistParam = 16; + XContentBuilder xContentBuilder = XContentFactory.jsonBuilder() + .startObject() + .field(NAME, METHOD_IVF) + .field(KNN_ENGINE, FAISS_NAME) + .startObject(PARAMETERS) + .field(METHOD_PARAMETER_NLIST, ivfNlistParam) + .startObject(METHOD_ENCODER_PARAMETER) + .field(NAME, ENCODER_SQ) + .startObject(PARAMETERS) + .field(FAISS_SQ_TYPE, FAISS_SQ_ENCODER_FP16) + .endObject() + .endObject() + .endObject() + .endObject(); + Map in = xContentBuilderToMap(xContentBuilder); + KNNMethodContext knnMethodContext = KNNMethodContext.parse(in); + Map parameters = KNNEngine.FAISS.getMethodAsMap(knnMethodContext); + + byte[] faissIndex = JNIService.trainIndex(parameters, 128, trainPointer, FAISS_NAME); + + assertNotEquals(0, faissIndex.length); + JNIService.freeVectors(trainPointer); + } + public void testCreateIndex_faiss_invalid_invalidParameterType() throws IOException { int[] docIds = new int[] {}; From fc2c15496139cb30bc1bca12d081506cca6ea54d Mon Sep 17 00:00:00 2001 From: Naveen Tatikonda Date: Mon, 5 Feb 2024 15:28:27 -0600 Subject: [PATCH 194/416] Set Default value for SIMD_ENABLED in CMakeList (#1455) Signed-off-by: Naveen Tatikonda --- jni/CMakeLists.txt | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/jni/CMakeLists.txt b/jni/CMakeLists.txt index 901929fc3..0bec4b945 100644 --- a/jni/CMakeLists.txt +++ b/jni/CMakeLists.txt @@ -111,6 +111,11 @@ endif () if (${CONFIG_FAISS} STREQUAL ON OR ${CONFIG_ALL} STREQUAL ON OR ${CONFIG_TEST} STREQUAL ON) set(BUILD_TESTING OFF) # Avoid building faiss tests set(BLA_STATIC ON) # Statically link BLAS + + if(NOT SIMD_ENABLED) + set(SIMD_ENABLED false) # set default value as false if the argument is not set + endif() + if(${CMAKE_SYSTEM_NAME} STREQUAL Windows OR ${CMAKE_SYSTEM_PROCESSOR} MATCHES "aarch64" OR NOT ${SIMD_ENABLED}) set(FAISS_OPT_LEVEL generic) # Keep optimization level as generic on Windows OS as it is not supported due to MINGW64 compiler issue. Also, on aarch64 avx2 is not supported. set(TARGET_LINK_FAISS_LIB faiss) From 042de0d996af910b4fa274f39820c153c0567cf5 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Mon, 5 Feb 2024 15:03:51 -0800 Subject: [PATCH 195/416] Update spotless and eclipse dependencies (#1450) (#1456) * Update spotless and eclipse dependencies Signed-off-by: Ryan Bogan * Update dependencies for spotless and eclipse Signed-off-by: Ryan Bogan * Add Changelog Signed-off-by: Ryan Bogan * Add comment Signed-off-by: Ryan Bogan * Add resources force resolution for eclipse Signed-off-by: Ryan Bogan --------- Signed-off-by: Ryan Bogan (cherry picked from commit fceb8f8391b6fbeb7614cb2467548fa5da7ec743) Co-authored-by: Ryan Bogan --- CHANGELOG.md | 1 + build.gradle | 5 +++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4d5796d4c..321e47ab9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -42,4 +42,5 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), * Upgrade urllib to 1.26.18 [#1319](https://github.com/opensearch-project/k-NN/pull/1319) * Upgrade guava to 32.1.3 [#1319](https://github.com/opensearch-project/k-NN/pull/1319) * Bump lucene codec to 99 [#1383](https://github.com/opensearch-project/k-NN/pull/1383) +* Update spotless and eclipse dependencies [#1450](https://github.com/opensearch-project/k-NN/pull/1450) ### Refactoring diff --git a/build.gradle b/build.gradle index d9eb0f8c8..098c8294d 100644 --- a/build.gradle +++ b/build.gradle @@ -44,7 +44,8 @@ buildscript { classpath "${opensearch_group}.gradle:build-tools:${opensearch_version}" configurations.all { resolutionStrategy { - force("org.eclipse.platform:org.eclipse.core.runtime:3.29.0") // for spotless transitive dependency CVE (for 3.26.100) + force("org.eclipse.platform:org.eclipse.core.runtime:4.29.0") // CVE for < 4.29 + force("org.eclipse.platform:org.eclipse.core.resources:4.29.0") // CVE for < 4.29 } } } @@ -58,7 +59,7 @@ plugins { id 'java-library' id 'java-test-fixtures' id 'idea' - id "com.diffplug.spotless" version "6.20.0" apply false + id "com.diffplug.spotless" version "6.25.0" apply false id 'io.freefair.lombok' version '8.4' id "de.undercouch.download" version "5.3.0" } From b899ed0fd9cd739d191a182b847411ff5fbcd241 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Mon, 5 Feb 2024 19:47:28 -0600 Subject: [PATCH 196/416] Change eclipse version to latest (#1457) (#1458) Signed-off-by: Ryan Bogan (cherry picked from commit 48fcfa70c38b6829f4e675903ef35ed6720c9c25) Co-authored-by: Ryan Bogan --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 098c8294d..1bbed9444 100644 --- a/build.gradle +++ b/build.gradle @@ -45,7 +45,7 @@ buildscript { configurations.all { resolutionStrategy { force("org.eclipse.platform:org.eclipse.core.runtime:4.29.0") // CVE for < 4.29 - force("org.eclipse.platform:org.eclipse.core.resources:4.29.0") // CVE for < 4.29 + force("org.eclipse.platform:org.eclipse.core.resources:4.20.0") // CVE for < 4.20 } } } From 543834abdb3a76224b77e2edec1099af482e58c3 Mon Sep 17 00:00:00 2001 From: Naveen Tatikonda Date: Mon, 5 Feb 2024 19:51:26 -0600 Subject: [PATCH 197/416] Revert "Update Build Script to build libraries with SIMD (#1451) (#1454)" (#1460) This reverts commit d5b42375c29315175fd4491e22abec13a166e5cf. Signed-off-by: Naveen Tatikonda --- scripts/build.sh | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/scripts/build.sh b/scripts/build.sh index 6e0c85865..6a9adc256 100644 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -148,20 +148,6 @@ cd $distributions zip -ur $zipPath lib cd $work_dir -if [ "$PLATFORM" != "windows" ]; then - echo "Building k-NN libraries after enabling SIMD" - ./gradlew :buildJniLib -Dsimd.enabled=true - mkdir $distributions/lib_simd - cp -v $ompPath $distributions/lib_simd - cp -v ./jni/release/${libPrefix}* $distributions/lib_simd - ls -l $distributions/lib_simd - - # Add lib_simd directory to the k-NN plugin zip - cd $distributions - zip -ur $zipPath lib_simd - cd $work_dir -fi - echo "COPY ${distributions}/*.zip" mkdir -p $OUTPUT/plugins cp -v ${distributions}/*.zip $OUTPUT/plugins From 83a2d1e031582054dda70b205c68574d8d382cbf Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Wed, 7 Feb 2024 11:36:32 -0800 Subject: [PATCH 198/416] Add release notes for 2.12.0 release (#1463) (#1464) * Add release notes for 2.12.0 release Signed-off-by: Ryan Bogan * Update release-notes/opensearch-knn.release-notes-2.12.0.0.md Co-authored-by: Naveen Tatikonda Signed-off-by: Ryan Bogan --------- Signed-off-by: Ryan Bogan Co-authored-by: Naveen Tatikonda (cherry picked from commit 45437be729d1f92fbd45184a8b79125c445d96dc) Co-authored-by: Ryan Bogan --- CHANGELOG.md | 24 -------------- .../opensearch-knn.release-notes-2.12.0.0.md | 32 +++++++++++++++++++ 2 files changed, 32 insertions(+), 24 deletions(-) create mode 100644 release-notes/opensearch-knn.release-notes-2.12.0.0.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 321e47ab9..ffd99a595 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,33 +14,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ## [Unreleased 2.x](https://github.com/opensearch-project/k-NN/compare/2.12...2.x) ### Features -* Add parent join support for lucene knn [#1182](https://github.com/opensearch-project/k-NN/pull/1182) -* Add parent join support for faiss hnsw [#1398](https://github.com/opensearch-project/k-NN/pull/1398) -* Add Support for Faiss SQFP16 and enable Faiss AVX2 Optimization [#1421](https://github.com/opensearch-project/k-NN/pull/1421) ### Enhancements -* Increase Lucene max dimension limit to 16,000 [#1346](https://github.com/opensearch-project/k-NN/pull/1346) -* Tuned default values for ef_search and ef_construction for better indexing and search performance for vector search [#1353](https://github.com/opensearch-project/k-NN/pull/1353) -* Enabled Filtering on Nested Vector fields with top level filters [#1372](https://github.com/opensearch-project/k-NN/pull/1372) -* Throw proper exception to invalid k-NN query [#1380](https://github.com/opensearch-project/k-NN/pull/1380) ### Bug Fixes -* Fix use-after-free case on nmslib search path [#1305](https://github.com/opensearch-project/k-NN/pull/1305) -* Allow nested knn field mapping when train model [#1318](https://github.com/opensearch-project/k-NN/pull/1318) -* Properly designate model state for actively training models when nodes crash or leave cluster [#1317](https://github.com/opensearch-project/k-NN/pull/1317) -* Fix script score queries not getting cached [#1367](https://github.com/opensearch-project/k-NN/pull/1367) -* Fix KNNScorer to apply boost [#1403](https://github.com/opensearch-project/k-NN/pull/1403) -* Fix equals and hashCode methods for KNNQuery and KNNQueryBuilder [#1397](https://github.com/opensearch-project/k-NN/pull/1397) -* Pass correct value on IDSelectorBitmap initialization [#1444](https://github.com/opensearch-project/k-NN/pull/1444) ### Infrastructure -* Upgrade gradle to 8.4 [1289](https://github.com/opensearch-project/k-NN/pull/1289) -* Refactor security testing to install from individual components [#1307](https://github.com/opensearch-project/k-NN/pull/1307) -* Refactor integ tests that access model index [#1423](https://github.com/opensearch-project/k-NN/pull/1423) -* Fix flaky model tests [#1429](https://github.com/opensearch-project/k-NN/pull/1429) ### Documentation ### Maintenance -* Update developer guide to include M1 Setup [#1222](https://github.com/opensearch-project/k-NN/pull/1222) -* Upgrade urllib to 1.26.17 [#1278](https://github.com/opensearch-project/k-NN/pull/1278) -* Upgrade urllib to 1.26.18 [#1319](https://github.com/opensearch-project/k-NN/pull/1319) -* Upgrade guava to 32.1.3 [#1319](https://github.com/opensearch-project/k-NN/pull/1319) -* Bump lucene codec to 99 [#1383](https://github.com/opensearch-project/k-NN/pull/1383) -* Update spotless and eclipse dependencies [#1450](https://github.com/opensearch-project/k-NN/pull/1450) ### Refactoring diff --git a/release-notes/opensearch-knn.release-notes-2.12.0.0.md b/release-notes/opensearch-knn.release-notes-2.12.0.0.md new file mode 100644 index 000000000..0bc6be9ec --- /dev/null +++ b/release-notes/opensearch-knn.release-notes-2.12.0.0.md @@ -0,0 +1,32 @@ +## Version 2.12.0.0 Release Notes + +Compatible with OpenSearch 2.12.0 + +### Features +* Add parent join support for lucene knn [#1182](https://github.com/opensearch-project/k-NN/pull/1182) +* Add parent join support for faiss hnsw [#1398](https://github.com/opensearch-project/k-NN/pull/1398) +### Enhancements +* Increase Lucene max dimension limit to 16,000 [#1346](https://github.com/opensearch-project/k-NN/pull/1346) +* Tuned default values for ef_search and ef_construction for better indexing and search performance for vector search [#1353](https://github.com/opensearch-project/k-NN/pull/1353) +* Enabled Filtering on Nested Vector fields with top level filters [#1372](https://github.com/opensearch-project/k-NN/pull/1372) +* Throw proper exception to invalid k-NN query [#1380](https://github.com/opensearch-project/k-NN/pull/1380) +### Bug Fixes +* Fix use-after-free case on nmslib search path [#1305](https://github.com/opensearch-project/k-NN/pull/1305) +* Allow nested knn field mapping when train model [#1318](https://github.com/opensearch-project/k-NN/pull/1318) +* Properly designate model state for actively training models when nodes crash or leave cluster [#1317](https://github.com/opensearch-project/k-NN/pull/1317) +* Fix script score queries not getting cached [#1367](https://github.com/opensearch-project/k-NN/pull/1367) +* Fix KNNScorer to apply boost [#1403](https://github.com/opensearch-project/k-NN/pull/1403) +* Fix equals and hashCode methods for KNNQuery and KNNQueryBuilder [#1397](https://github.com/opensearch-project/k-NN/pull/1397) +* Pass correct value on IDSelectorBitmap initialization [#1444](https://github.com/opensearch-project/k-NN/pull/1444) +### Infrastructure +* Upgrade gradle to 8.4 [1289](https://github.com/opensearch-project/k-NN/pull/1289) +* Refactor security testing to install from individual components [#1307](https://github.com/opensearch-project/k-NN/pull/1307) +* Refactor integ tests that access model index [#1423](https://github.com/opensearch-project/k-NN/pull/1423) +* Fix flaky model tests [#1429](https://github.com/opensearch-project/k-NN/pull/1429) +### Maintenance +* Update developer guide to include M1 Setup [#1222](https://github.com/opensearch-project/k-NN/pull/1222) +* Upgrade urllib to 1.26.17 [#1278](https://github.com/opensearch-project/k-NN/pull/1278) +* Upgrade urllib to 1.26.18 [#1319](https://github.com/opensearch-project/k-NN/pull/1319) +* Upgrade guava to 32.1.3 [#1319](https://github.com/opensearch-project/k-NN/pull/1319) +* Bump lucene codec to 99 [#1383](https://github.com/opensearch-project/k-NN/pull/1383) +* Update spotless and eclipse dependencies [#1450](https://github.com/opensearch-project/k-NN/pull/1450) \ No newline at end of file From 29279dc067c49b8d64216cc678ec246d61834c3e Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Thu, 8 Feb 2024 09:51:41 -0800 Subject: [PATCH 199/416] Disable warning checks in k-NN test case (#1442) (#1471) * Disable warning checks in k-NN test case Signed-off-by: Junqiu Lei (cherry picked from commit 9e28957013cbecf98f219927938aa69ef5d49152) Co-authored-by: Junqiu Lei --- .../java/org/opensearch/knn/KNNTestCase.java | 7 +++++++ .../knn/index/KNNSettingsTests.java | 19 ------------------- 2 files changed, 7 insertions(+), 19 deletions(-) diff --git a/src/test/java/org/opensearch/knn/KNNTestCase.java b/src/test/java/org/opensearch/knn/KNNTestCase.java index 3f20981bd..565feb398 100644 --- a/src/test/java/org/opensearch/knn/KNNTestCase.java +++ b/src/test/java/org/opensearch/knn/KNNTestCase.java @@ -53,6 +53,13 @@ public void tearDown() throws Exception { openMocks.close(); } + @Override + protected boolean enableWarningsCheck() { + // Disable warnings check to avoid flaky tests, more details at: + // https://github.com/opensearch-project/k-NN/issues/1392 + return false; + } + public void resetState() { // Reset all of the counters for (KNNCounter knnCounter : KNNCounter.values()) { diff --git a/src/test/java/org/opensearch/knn/index/KNNSettingsTests.java b/src/test/java/org/opensearch/knn/index/KNNSettingsTests.java index 4292b1a4f..6b4751afa 100644 --- a/src/test/java/org/opensearch/knn/index/KNNSettingsTests.java +++ b/src/test/java/org/opensearch/knn/index/KNNSettingsTests.java @@ -51,7 +51,6 @@ public void testGetSettingValueFromConfig() { .getSettingValue(KNNSettings.KNN_MEMORY_CIRCUIT_BREAKER_LIMIT)).getKb(); mockNode.close(); assertEquals(expectedKNNCircuitBreakerLimit, actualKNNCircuitBreakerLimit); - assertWarnings(); } @SneakyThrows @@ -69,10 +68,6 @@ public void testGetSettingValueDefault() { actualKNNCircuitBreakerLimit ); - // set warning for deprecation of index.store.hybrid.mmap.extensions as expected temporarily, need to work on proper strategy of - // switching to new setting in core - // no-jdk distributions expected warning is a workaround for running tests locally - assertWarnings(); } @SneakyThrows @@ -87,7 +82,6 @@ public void testFilteredSearchAdvanceSetting_whenNoValuesProvidedByUsers_thenDef Integer filteredSearchThreshold = KNNSettings.getFilteredExactSearchThreshold(INDEX_NAME); mockNode.close(); assertEquals(KNNSettings.ADVANCED_FILTERED_EXACT_SEARCH_THRESHOLD_DEFAULT_VALUE, filteredSearchThreshold); - assertWarnings(); } @SneakyThrows @@ -133,7 +127,6 @@ public void testFilteredSearchAdvanceSetting_whenValuesProvidedByUsers_thenValid mockNode.close(); assertEquals(userDefinedThreshold, filteredSearchThreshold); assertEquals(userDefinedThresholdMinValue, filteredSearchThresholdMinValue); - assertWarnings(); } @SneakyThrows @@ -148,7 +141,6 @@ public void testGetEfSearch_whenNoValuesProvidedByUsers_thenDefaultSettingsUsed( Integer efSearchValue = KNNSettings.getEfSearchParam(INDEX_NAME); mockNode.close(); assertEquals(KNNSettings.INDEX_KNN_DEFAULT_ALGO_PARAM_EF_SEARCH, efSearchValue); - assertWarnings(); } @SneakyThrows @@ -168,7 +160,6 @@ public void testGetEfSearch_whenEFSearchValueSetByUser_thenReturnValue() { int efSearchValue = KNNSettings.getEfSearchParam(INDEX_NAME); mockNode.close(); assertEquals(userProvidedEfSearch, efSearchValue); - assertWarnings(); } private Node createMockNode(Map configSettings) throws IOException { @@ -199,14 +190,4 @@ private static Settings.Builder baseSettings() { .put(NetworkModule.TRANSPORT_TYPE_KEY, getTestTransportType()) .put(dataNode()); } - - private void assertWarnings() { - // set warning for deprecation of index.store.hybrid.mmap.extensions as expected temporarily, need to work on proper strategy of - // switching to new setting in core - // no-jdk distributions expected warning is a workaround for running tests locally - assertWarnings( - "[index.store.hybrid.mmap.extensions] setting was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation for the next major version.", - "no-jdk distributions that do not bundle a JDK are deprecated and will be removed in a future release" - ); - } } From 34e58bd79ffb127b7dada523a2699c053221ef3c Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Fri, 9 Feb 2024 15:01:44 -0800 Subject: [PATCH 200/416] Use valid data for sqfp16 test (#1474) (#1476) Signed-off-by: Heemin Kim (cherry picked from commit e2d5bc5e2412270dbd6f5173c73f14f9b84f7e1f) Co-authored-by: Heemin Kim --- .../opensearch/knn/jni/JNIServiceTests.java | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/src/test/java/org/opensearch/knn/jni/JNIServiceTests.java b/src/test/java/org/opensearch/knn/jni/JNIServiceTests.java index c2470ea47..7673ee463 100644 --- a/src/test/java/org/opensearch/knn/jni/JNIServiceTests.java +++ b/src/test/java/org/opensearch/knn/jni/JNIServiceTests.java @@ -525,7 +525,7 @@ public void testQueryIndex_faiss_sqfp16_valid() { Path tmpFile = createTempFile(); JNIService.createIndex( testData.indexData.docs, - testData.indexData.vectors, + truncateToFp16Range(testData.indexData.vectors), tmpFile.toAbsolutePath().toString(), ImmutableMap.of(INDEX_DESCRIPTION_PARAMETER, sqfp16IndexDescription, KNNConstants.SPACE_TYPE, SpaceType.L2.getValue()), FAISS_NAME @@ -547,6 +547,23 @@ public void testQueryIndex_faiss_sqfp16_valid() { } } + // If the value is outside of the fp16 range, then convert it to the fp16 minimum or maximum value + private float[][] truncateToFp16Range(final float[][] data) { + float[][] result = new float[data.length][data[0].length]; + for (int i = 0; i < data.length; i++) { + for (int j = 0; j < data[i].length; j++) { + float value = data[i][j]; + if (value < Float.MIN_VALUE || value > Float.MAX_VALUE) { + // If value is outside of the range, set it to the maximum or minimum value + result[i][j] = value < 0 ? -Float.MAX_VALUE : Float.MAX_VALUE; + } else { + result[i][j] = value; + } + } + } + return result; + } + @SneakyThrows public void testTrain_whenConfigurationIsIVFSQFP16_thenSucceed() { long trainPointer = transferVectors(10); From 5e8a1c624d3cf872284dcb5ea1f3f39fc1f5d5ce Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Fri, 9 Feb 2024 17:05:46 -0800 Subject: [PATCH 201/416] Fix a bug on sqfp16 test (#1477) (#1478) Signed-off-by: Heemin Kim (cherry picked from commit 06ccdbc955f4d2e9370479f4180182c2fadefa8a) Co-authored-by: Heemin Kim --- src/test/java/org/opensearch/knn/jni/JNIServiceTests.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/test/java/org/opensearch/knn/jni/JNIServiceTests.java b/src/test/java/org/opensearch/knn/jni/JNIServiceTests.java index 7673ee463..2afcff83d 100644 --- a/src/test/java/org/opensearch/knn/jni/JNIServiceTests.java +++ b/src/test/java/org/opensearch/knn/jni/JNIServiceTests.java @@ -57,7 +57,8 @@ import static org.opensearch.knn.common.KNNConstants.PARAMETERS; public class JNIServiceTests extends KNNTestCase { - + static final int FP16_MAX = 65504; + static final int FP16_MIN = -65504; static TestUtils.TestData testData; static TestUtils.TestData testDataNested; private String faissMethod = "HNSW32,Flat"; @@ -553,9 +554,9 @@ private float[][] truncateToFp16Range(final float[][] data) { for (int i = 0; i < data.length; i++) { for (int j = 0; j < data[i].length; j++) { float value = data[i][j]; - if (value < Float.MIN_VALUE || value > Float.MAX_VALUE) { + if (value < FP16_MIN || value > FP16_MAX) { // If value is outside of the range, set it to the maximum or minimum value - result[i][j] = value < 0 ? -Float.MAX_VALUE : Float.MAX_VALUE; + result[i][j] = value < 0 ? FP16_MIN : FP16_MAX; } else { result[i][j] = value; } From cfd38753d696c55b79cf3909b4a19e0f26a94cf9 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Tue, 13 Feb 2024 14:55:12 -0800 Subject: [PATCH 202/416] Update faiss commit and resolve conflict (#1443) (#1482) Signed-off-by: Heemin Kim (cherry picked from commit ced1ff2c6c03f1fc0e3a4191272d8402b05f3b96) Co-authored-by: Heemin Kim --- CHANGELOG.md | 1 + jni/CMakeLists.txt | 15 +- jni/external/faiss | 2 +- jni/include/faiss_util.h | 25 + .../faiss/MultiVectorResultCollector.h | 69 - .../faiss/MultiVectorResultCollectorFactory.h | 26 - .../knn_extension/faiss/utils/BitSet.h | 54 - jni/include/knn_extension/faiss/utils/Heap.h | 255 ---- ...Custom-patch-to-support-multi-vector.patch | 1129 ++++++++++++++--- jni/src/faiss_util.cpp | 23 + jni/src/faiss_wrapper.cpp | 39 +- .../faiss/MultiVectorResultCollector.cpp | 67 - .../MultiVectorResultCollectorFactory.cpp | 24 - jni/src/knn_extension/faiss/utils/BitSet.cpp | 47 - jni/tests/faiss_util_test.cpp | 28 + jni/tests/faiss_wrapper_test.cpp | 8 +- .../MultiVectorResultCollectorFactoryTest.cpp | 78 -- .../faiss/MultiVectorResultCollectorTest.cpp | 96 -- .../knn_extension/faiss/utils/BitSetTest.cpp | 52 - .../knn_extension/faiss/utils/HeapTest.cpp | 86 -- 20 files changed, 1041 insertions(+), 1083 deletions(-) create mode 100644 jni/include/faiss_util.h delete mode 100644 jni/include/knn_extension/faiss/MultiVectorResultCollector.h delete mode 100644 jni/include/knn_extension/faiss/MultiVectorResultCollectorFactory.h delete mode 100644 jni/include/knn_extension/faiss/utils/BitSet.h delete mode 100644 jni/include/knn_extension/faiss/utils/Heap.h create mode 100644 jni/src/faiss_util.cpp delete mode 100644 jni/src/knn_extension/faiss/MultiVectorResultCollector.cpp delete mode 100644 jni/src/knn_extension/faiss/MultiVectorResultCollectorFactory.cpp delete mode 100644 jni/src/knn_extension/faiss/utils/BitSet.cpp create mode 100644 jni/tests/faiss_util_test.cpp delete mode 100644 jni/tests/knn_extension/faiss/MultiVectorResultCollectorFactoryTest.cpp delete mode 100644 jni/tests/knn_extension/faiss/MultiVectorResultCollectorTest.cpp delete mode 100644 jni/tests/knn_extension/faiss/utils/BitSetTest.cpp delete mode 100644 jni/tests/knn_extension/faiss/utils/HeapTest.cpp diff --git a/CHANGELOG.md b/CHANGELOG.md index ffd99a595..b55634b53 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,4 +19,5 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Infrastructure ### Documentation ### Maintenance +* Bump faiss lib commit to 32f0e8cf92cd2275b60364517bb1cce67aa29a55 [#1443](https://github.com/opensearch-project/k-NN/pull/1443) ### Refactoring diff --git a/jni/CMakeLists.txt b/jni/CMakeLists.txt index 0bec4b945..0f0b58738 100644 --- a/jni/CMakeLists.txt +++ b/jni/CMakeLists.txt @@ -181,17 +181,15 @@ if (${CONFIG_FAISS} STREQUAL ON OR ${CONFIG_ALL} STREQUAL ON OR ${CONFIG_TEST} S ${TARGET_LIB_FAISS} SHARED ${CMAKE_CURRENT_SOURCE_DIR}/src/org_opensearch_knn_jni_FaissService.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/faiss_wrapper.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/src/knn_extension/faiss/utils/BitSet.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/src/knn_extension/faiss/MultiVectorResultCollector.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/src/knn_extension/faiss/MultiVectorResultCollectorFactory.cpp) + ${CMAKE_CURRENT_SOURCE_DIR}/src/faiss_util.cpp + ) target_link_libraries(${TARGET_LIB_FAISS} ${TARGET_LINK_FAISS_LIB} ${TARGET_LIB_COMMON} OpenMP::OpenMP_CXX) target_include_directories(${TARGET_LIB_FAISS} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include - ${CMAKE_CURRENT_SOURCE_DIR}/include/knn_extension/faiss - ${CMAKE_CURRENT_SOURCE_DIR}/include/knn_extension/faiss/utils $ENV{JAVA_HOME}/include $ENV{JAVA_HOME}/include/${JVM_OS_TYPE} - ${CMAKE_CURRENT_SOURCE_DIR}/external/faiss) + ${CMAKE_CURRENT_SOURCE_DIR}/external/faiss + ) set_target_properties(${TARGET_LIB_FAISS} PROPERTIES SUFFIX ${LIB_EXT}) set_target_properties(${TARGET_LIB_FAISS} PROPERTIES POSITION_INDEPENDENT_CODE ON) @@ -228,12 +226,9 @@ if ("${WIN32}" STREQUAL "") add_executable( jni_test tests/faiss_wrapper_test.cpp + tests/faiss_util_test.cpp tests/nmslib_wrapper_test.cpp tests/test_util.cpp - tests/knn_extension/faiss/utils/BitSetTest.cpp - tests/knn_extension/faiss/utils/HeapTest.cpp - tests/knn_extension/faiss/MultiVectorResultCollectorTest.cpp - tests/knn_extension/faiss/MultiVectorResultCollectorFactoryTest.cpp ) target_link_libraries( diff --git a/jni/external/faiss b/jni/external/faiss index 0013c702f..32f0e8cf9 160000 --- a/jni/external/faiss +++ b/jni/external/faiss @@ -1 +1 @@ -Subproject commit 0013c702f47bedbf6159ac356e61f378ccd12ac8 +Subproject commit 32f0e8cf92cd2275b60364517bb1cce67aa29a55 diff --git a/jni/include/faiss_util.h b/jni/include/faiss_util.h new file mode 100644 index 000000000..f23540aef --- /dev/null +++ b/jni/include/faiss_util.h @@ -0,0 +1,25 @@ +// SPDX-License-Identifier: Apache-2.0 +// +// The OpenSearch Contributors require contributions made to +// this file be licensed under the Apache-2.0 license or a +// compatible open source license. +// +// Modifications Copyright OpenSearch Contributors. See +// GitHub history for details. + +/** + * This file contains util methods which are free of JNI to be used in faiss_wrapper.cpp + */ + +#ifndef OPENSEARCH_KNN_FAISS_UTIL_H +#define OPENSEARCH_KNN_FAISS_UTIL_H + +#include "faiss/impl/IDGrouper.h" +#include + +namespace faiss_util { + std::unique_ptr buildIDGrouperBitmap(int *parentIdsArray, int parentIdsLength, std::vector* bitmap); +}; + + +#endif //OPENSEARCH_KNN_FAISS_UTIL_H diff --git a/jni/include/knn_extension/faiss/MultiVectorResultCollector.h b/jni/include/knn_extension/faiss/MultiVectorResultCollector.h deleted file mode 100644 index a11a278d9..000000000 --- a/jni/include/knn_extension/faiss/MultiVectorResultCollector.h +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -#pragma once - -#include -#include -#include "knn_extension/faiss/utils/BitSet.h" -#include - -namespace os_faiss { - -using idx_t = faiss::idx_t; -/** - * Implementation of ResultCollector to support multi vector - * - * Only supports HNSW algorithm - * - * Example: - * When there is two lucene document with two nested fields, the parent_bit_set value of 100100 is provided where - * parent doc ids are 2, and 5. Doc id for nested fields of parent document 2 are 0, and 1. Doc id for nested fields - * of parent document 5 are 3, and 4. For faiss, only nested fields are stored. Therefore corresponding doc ids for - * nested fields 0, 1, 3, 4 is 0, 1, 2, 3 in faiss. This mapping data is stored in id_map parameter. - * - * When collect method is called - * 1. It switches from faiss id to lucene id and look for its parent id. - * 2. See if the parent id already exist in heap using either parent_id_to_id or parent_id_to_index. - * 3. If it does not exist, add the parent id and distance value in the heap(bh_ids, bh_val) and update parent_id_to_id, and parent_id_to_index. - * 4. If it does exist, update the distance value(bh_val), parent_id_to_id, and parent_id_to_index. - * - * When post_process method is called - * 1. Convert lucene parent ID to faiss doc ID using parent_id_to_id - */ -struct MultiVectorResultCollector:faiss::ResultCollector { - // BitSet of lucene parent doc ID - const BitSet* parent_bit_set; - - // Mapping data from Faiss doc ID to Lucene doc ID - const std::vector* id_map; - - // Lucene parent doc ID to to Faiss doc ID - // Lucene parent doc ID to index in heap(bh_val, bh_ids) - std::unordered_map parent_id_to_id; - std::unordered_map parent_id_to_index; - MultiVectorResultCollector(const BitSet* parent_bit_set, const std::vector* id_map); - - /** - * - * @param k max size of bh_val, and bh_ids - * @param nres number of results in bh_val, and bh_ids - * @param bh_val binary heap storing values (For this case distance from query to result) - * @param bh_ids binary heap storing document IDs - * @param val a new value to add in bh_val - * @param ids a new doc id to add in bh_ids - */ - void collect( - int k, - int& nres, - float* bh_val, - int64_t* bh_ids, - float val, - int64_t ids) override; - void post_process(int64_t nres, int64_t* bh_ids) override; -}; - -} // namespace os_faiss - diff --git a/jni/include/knn_extension/faiss/MultiVectorResultCollectorFactory.h b/jni/include/knn_extension/faiss/MultiVectorResultCollectorFactory.h deleted file mode 100644 index 45c0338b3..000000000 --- a/jni/include/knn_extension/faiss/MultiVectorResultCollectorFactory.h +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -#pragma once - -#include -#include "knn_extension/faiss/utils/BitSet.h" - -namespace os_faiss { -/** - * Create MultiVectorResultCollector for single query request - * - * Creating new collector is required because MultiVectorResultCollector has instance variables - * which should be isolated for each query. - */ -struct MultiVectorResultCollectorFactory:faiss::ResultCollectorFactory { - BitSet* parent_bit_set; - - MultiVectorResultCollectorFactory(BitSet* parent_bit_set); - faiss::ResultCollector* new_collector() override; - void delete_collector(faiss::ResultCollector* resultCollector) override; -}; - -} // namespace os_faiss diff --git a/jni/include/knn_extension/faiss/utils/BitSet.h b/jni/include/knn_extension/faiss/utils/BitSet.h deleted file mode 100644 index 0c8079d37..000000000 --- a/jni/include/knn_extension/faiss/utils/BitSet.h +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -#pragma once - -#include -#include -#include - -using idx_t = faiss::idx_t; - -struct BitSet { - const int NO_MORE_DOCS = std::numeric_limits::max(); - /** - * Returns the index of the first set bit starting at the index specified. - * NO_MORE_DOCS is returned if there are no more set bits. - */ - virtual idx_t next_set_bit(idx_t index) const = 0; - virtual ~BitSet() = default; -}; - - -/** - * BitSet of fixed length (numBits), implemented using an array of unit64. - * See https://github.com/apache/lucene/blob/main/lucene/core/src/java/org/apache/lucene/util/FixedBitSet.java - * - * Here a block is 64 bit. However, for simplicity let's assume its size is 8 bits. - * Then, if have an array of 3, 7, and 10, it will be represented in bitmap as follow. - * [0] [1] - * bitmap: 10001000 00000100 - * - * for next_set_bit call with 4 - * 1. it looks for words[0] - * 2. words[0] >> 4 - * 3. count trailing zero of the result from step 2 which is 3 - * 4. return 4(current index) + 3(result from step 3) - */ -struct FixedBitSet : public BitSet { - // The number of bits in use - idx_t num_bits; - - // The exact number of longs needed to hold num_bits - size_t num_words; - - // Array of uint64_t holding the bits - // Using uint64_t to leverage function __builtin_ctzll which is defined in faiss/impl/platform_macros.h - uint64_t* words; - - FixedBitSet(const int* int_array, const int length); - idx_t next_set_bit(idx_t index) const; - ~FixedBitSet(); -}; diff --git a/jni/include/knn_extension/faiss/utils/Heap.h b/jni/include/knn_extension/faiss/utils/Heap.h deleted file mode 100644 index 2aa19da52..000000000 --- a/jni/include/knn_extension/faiss/utils/Heap.h +++ /dev/null @@ -1,255 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -#pragma once - -#include -#include -#include - -#include -#include -#include - -#include -#include -#include - -// Collection of heap operations with parent id to dedupe -namespace os_faiss { - -/** - * From start_index, it compare its value with parent node's and swap if needed. - * Continue until either there is no swap or it reaches the top node. - * - * @param bh_val binary heap storing values - * @param bh_ids binary heap storing parent ids - * @param val new value to add - * @param id new id to add - * @parent_id_to_id parent doc id to id mapping data, see MultiVectorResultCollector.h - * @parent_id_to_index parent doc id to index mapping data, see MultiVectorResultCollector.h - * @parent_id parent id of given id - * @start_index an index to start up-heap from in the binary heap(bh_val, and bh_ids) - */ -template -static inline void up_heap( - typename C::T* bh_val, - typename C::TI* bh_ids, - typename C::T val, - typename C::TI id, - std::unordered_map* parent_id_to_id, - std::unordered_map* parent_id_to_index, - typename C::TI parent_id, - size_t start_index) { - bh_val--; /* Use 1-based indexing for easier node->child translation */ - bh_ids--; - size_t i = start_index + 1, i_father; - - while (i > 1) { - i_father = i >> 1; - if (!C::cmp2(val, bh_val[i_father], parent_id, bh_ids[i_father])) { - /* the heap structure is ok */ - break; - } - bh_val[i] = bh_val[i_father]; - bh_ids[i] = bh_ids[i_father]; - (*parent_id_to_index)[bh_ids[i]] = i - 1; - i = i_father; - } - bh_val[i] = val; - bh_ids[i] = parent_id; - (*parent_id_to_id)[parent_id] = id; - (*parent_id_to_index)[parent_id] = i - 1; -} - -/** - * From start_index, it compare its value with child node's and swap if needed. - * Continue until either there is no swap or it reaches the leaf node. - * - * @param nres number of values in the binary heap(bh_val, and bh_ids) - * @param bh_val binary heap storing values - * @param bh_ids binary heap storing parent ids - * @param val new value to add - * @param id new id to add - * @parent_id_to_id parent doc id to id mapping data, see MultiVectorResultCollector.h - * @parent_id_to_index parent doc id to index mapping data, see MultiVectorResultCollector.h - * @parent_id parent id of given id - * @start_index an index to start up-heap from in the binary heap(bh_val, and bh_ids) - */ -template -static inline void down_heap( - int nres, - typename C::T* bh_val, - typename C::TI* bh_ids, - typename C::T val, - typename C::TI id, - std::unordered_map* parent_id_to_id, - std::unordered_map* parent_id_to_index, - typename C::TI parent_id, - size_t start_index) { - bh_val--; /* Use 1-based indexing for easier node->child translation */ - bh_ids--; - size_t i = start_index + 1, i1, i2; - - while (1) { - i1 = i << 1; - i2 = i1 + 1; - if (i1 > nres) { - break; - } - - // Note that C::cmp2() is a bool function answering - // `(a1 > b1) || ((a1 == b1) && (a2 > b2))` for max - // heap and same with the `<` sign for min heap. - if ((i2 == nres + 1) || - C::cmp2(bh_val[i1], bh_val[i2], bh_ids[i1], bh_ids[i2])) { - if (C::cmp2(val, bh_val[i1], parent_id, bh_ids[i1])) { - break; - } - bh_val[i] = bh_val[i1]; - bh_ids[i] = bh_ids[i1]; - (*parent_id_to_index)[bh_ids[i]] = i - 1; - i = i1; - } else { - if (C::cmp2(val, bh_val[i2], parent_id, bh_ids[i2])) { - break; - } - bh_val[i] = bh_val[i2]; - bh_ids[i] = bh_ids[i2]; - (*parent_id_to_index)[bh_ids[i]] = i - 1; - i = i2; - } - } - bh_val[i] = val; - bh_ids[i] = parent_id; - (*parent_id_to_id)[parent_id] = id; - (*parent_id_to_index)[parent_id] = i - 1; -} - -/** - * Push the value to the max heap - * As the heap contains only one value per group id, pushing a value of existing group id - * will break the data integrity. For existing group id, use maxheap_update instead. - * The parent_id should not exist in in bh_ids, parent_id_to_id, and parent_id_to_index. - * - * @param nres number of values in the binary heap(bh_val, and bh_ids) - * @param bh_val binary heap storing values - * @param bh_ids binary heap storing parent ids - * @param val new value to add - * @param id new id to add - * @parent_id_to_id parent doc id to id mapping data, see MultiVectorResultCollector.h - * @parent_id_to_index parent doc id to index mapping data, see MultiVectorResultCollector.h - * @parent_id parent id of given id - */ -template -inline void maxheap_push( - int nres, - T* bh_val, - int64_t* bh_ids, - T val, - int64_t id, - std::unordered_map* parent_id_to_id, - std::unordered_map* parent_id_to_index, - int64_t parent_id) { - - assert(parent_id_to_index->find(parent_id) == parent_id_to_index->end() && "parent id should not exist in the binary heap"); - - up_heap>( - bh_val, - bh_ids, - val, - id, - parent_id_to_id, - parent_id_to_index, - parent_id, - nres); -} - -/** - * Update the top node with given value - * The parent_id should not exist in in bh_ids, parent_id_to_id, and parent_id_to_index. - * - * @param nres number of values in the binary heap(bh_val, and bh_ids) - * @param bh_val binary heap storing values - * @param bh_ids binary heap storing parent ids - * @param val new value to add - * @param id new id to add - * @parent_id_to_id parent doc id to id mapping data, see MultiVectorResultCollector.h - * @parent_id_to_index parent doc id to index mapping data, see MultiVectorResultCollector.h - * @parent_id parent id of given id - */ -template -inline void maxheap_replace_top( - int nres, - T* bh_val, - int64_t* bh_ids, - T val, - int64_t id, - std::unordered_map* parent_id_to_id, - std::unordered_map* parent_id_to_index, - int64_t parent_id) { - - assert(parent_id_to_index->find(parent_id) == parent_id_to_index->end() && "parent id should not exist in the binary heap"); - - parent_id_to_id->erase(bh_ids[0]); - parent_id_to_index->erase(bh_ids[0]); - down_heap>( - nres, - bh_val, - bh_ids, - val, - id, - parent_id_to_id, - parent_id_to_index, - parent_id, - 0); -} - -/** - * Update value of the parent_id in the binary heap and id of the parent_id in parent_id_to_id - * The parent_id should exist in bh_ids, parent_id_to_id, and parent_id_to_index. - * - * @param nres number of values in the binary heap(bh_val, and bh_ids) - * @param bh_val binary heap storing values - * @param bh_ids binary heap storing parent ids - * @param val new value to update - * @param id new id to update - * @parent_id_to_id parent doc id to id mapping data, see MultiVectorResultCollector.h - * @parent_id_to_index parent doc id to index mapping data, see MultiVectorResultCollector.h - * @parent_id parent id of given id - */ -template -inline void maxheap_update( - int nres, - T* bh_val, - int64_t* bh_ids, - T val, - int64_t id, - std::unordered_map* parent_id_to_id, - std::unordered_map* parent_id_to_index, - int64_t parent_id) { - size_t target_index = parent_id_to_index->at(parent_id); - up_heap>( - bh_val, - bh_ids, - val, - id, - parent_id_to_id, - parent_id_to_index, - parent_id, - target_index); - down_heap>( - nres, - bh_val, - bh_ids, - val, - id, - parent_id_to_id, - parent_id_to_index, - parent_id, - target_index); -} - -} // namespace os_faiss diff --git a/jni/patches/faiss/0001-Custom-patch-to-support-multi-vector.patch b/jni/patches/faiss/0001-Custom-patch-to-support-multi-vector.patch index b07620c7e..a22e28130 100644 --- a/jni/patches/faiss/0001-Custom-patch-to-support-multi-vector.patch +++ b/jni/patches/faiss/0001-Custom-patch-to-support-multi-vector.patch @@ -1,90 +1,131 @@ -From baa7e23c54637d68adac45f09633939b402405a9 Mon Sep 17 00:00:00 2001 +From 0d1385959ddecabb2825957e48ff28ff0e8abf53 Mon Sep 17 00:00:00 2001 From: Heemin Kim -Date: Wed, 6 Dec 2023 16:33:52 -0800 -Subject: [PATCH] Custom patch to support multi-vector +Date: Tue, 30 Jan 2024 14:43:56 -0800 +Subject: [PATCH] Add IDGrouper for HNSW Signed-off-by: Heemin Kim --- - faiss/CMakeLists.txt | 2 + - faiss/Index.h | 6 ++- - faiss/IndexIDMap.cpp | 24 ++++++++++ - faiss/impl/HNSW.cpp | 27 +++++++---- - faiss/impl/ResultCollector.h | 74 +++++++++++++++++++++++++++++ - faiss/impl/ResultCollectorFactory.h | 33 +++++++++++++ - 6 files changed, 154 insertions(+), 12 deletions(-) - create mode 100644 faiss/impl/ResultCollector.h - create mode 100644 faiss/impl/ResultCollectorFactory.h + faiss/CMakeLists.txt | 3 + + faiss/Index.h | 8 +- + faiss/IndexHNSW.cpp | 13 ++- + faiss/IndexIDMap.cpp | 29 ++++++ + faiss/IndexIDMap.h | 22 +++++ + faiss/impl/HNSW.cpp | 10 +- + faiss/impl/IDGrouper.cpp | 51 ++++++++++ + faiss/impl/IDGrouper.h | 51 ++++++++++ + faiss/impl/ResultHandler.h | 187 ++++++++++++++++++++++++++++++++++++ + faiss/utils/GroupHeap.h | 182 +++++++++++++++++++++++++++++++++++ + tests/CMakeLists.txt | 2 + + tests/test_group_heap.cpp | 98 +++++++++++++++++++ + tests/test_id_grouper.cpp | 189 +++++++++++++++++++++++++++++++++++++ + 13 files changed, 838 insertions(+), 7 deletions(-) + create mode 100644 faiss/impl/IDGrouper.cpp + create mode 100644 faiss/impl/IDGrouper.h + create mode 100644 faiss/utils/GroupHeap.h + create mode 100644 tests/test_group_heap.cpp + create mode 100644 tests/test_id_grouper.cpp diff --git a/faiss/CMakeLists.txt b/faiss/CMakeLists.txt -index 27701586..af682a05 100644 +index a890a46f..137e68d4 100644 --- a/faiss/CMakeLists.txt +++ b/faiss/CMakeLists.txt -@@ -162,6 +162,8 @@ set(FAISS_HEADERS - impl/ProductQuantizer.h - impl/Quantizer.h - impl/ResidualQuantizer.h -+ impl/ResultCollector.h -+ impl/ResultCollectorFactory.h - impl/ResultHandler.h - impl/ScalarQuantizer.h - impl/ThreadedIndex-inl.h +@@ -54,6 +54,7 @@ set(FAISS_SRC + impl/AuxIndexStructures.cpp + impl/CodePacker.cpp + impl/IDSelector.cpp ++ impl/IDGrouper.cpp + impl/FaissException.cpp + impl/HNSW.cpp + impl/NSG.cpp +@@ -149,6 +150,7 @@ set(FAISS_HEADERS + impl/AuxIndexStructures.h + impl/CodePacker.h + impl/IDSelector.h ++ impl/IDGrouper.h + impl/DistanceComputer.h + impl/FaissAssert.h + impl/FaissException.h +@@ -183,6 +185,7 @@ set(FAISS_HEADERS + invlists/InvertedLists.h + invlists/InvertedListsIOHook.h + utils/AlignedTable.h ++ utils/GroupHeap.h + utils/Heap.h + utils/WorkerThread.h + utils/distances.h diff --git a/faiss/Index.h b/faiss/Index.h -index 4b4b302b..13eab0c0 100644 +index 4b4b302b..3b673d1e 100644 --- a/faiss/Index.h +++ b/faiss/Index.h -@@ -38,11 +38,12 @@ +@@ -38,9 +38,10 @@ namespace faiss { -/// Forward declarations see impl/AuxIndexStructures.h, impl/IDSelector.h and -/// impl/DistanceComputer.h -+/// Forward declarations see impl/AuxIndexStructures.h, impl/IDSelector.h, -+/// impl/DistanceComputer.h, and impl/ResultCollectorFactory.h ++/// Forward declarations see impl/AuxIndexStructures.h, impl/IDSelector.h ++/// ,impl/IDGrouper.h and impl/DistanceComputer.h struct IDSelector; ++struct IDGrouper; struct RangeSearchResult; struct DistanceComputer; -+struct ResultCollectorFactory; - /** Parent class for the optional search paramenters. - * -@@ -52,6 +53,7 @@ struct DistanceComputer; +@@ -52,6 +53,9 @@ struct DistanceComputer; struct SearchParameters { /// if non-null, only these IDs will be considered during search. IDSelector* sel = nullptr; -+ ResultCollectorFactory* col = nullptr; ++ /// if non-null, only best matched ID per group will be included in the ++ /// result. ++ IDGrouper* grp = nullptr; /// make sure we can dynamic_cast this virtual ~SearchParameters() {} }; +diff --git a/faiss/IndexHNSW.cpp b/faiss/IndexHNSW.cpp +index 9a67332d..a5e0fea0 100644 +--- a/faiss/IndexHNSW.cpp ++++ b/faiss/IndexHNSW.cpp +@@ -354,10 +354,17 @@ void IndexHNSW::search( + const SearchParameters* params_in) const { + FAISS_THROW_IF_NOT(k > 0); + +- using RH = HeapBlockResultHandler; +- RH bres(n, distances, labels, k); ++ if (params_in && params_in->grp) { ++ using RH = GroupedHeapBlockResultHandler; ++ RH bres(n, distances, labels, k, params_in->grp); + +- hnsw_search(this, n, x, bres, params_in); ++ hnsw_search(this, n, x, bres, params_in); ++ } else { ++ using RH = HeapBlockResultHandler; ++ RH bres(n, distances, labels, k); ++ ++ hnsw_search(this, n, x, bres, params_in); ++ } + + if (is_similarity_metric(this->metric_type)) { + // we need to revert the negated distances diff --git a/faiss/IndexIDMap.cpp b/faiss/IndexIDMap.cpp -index 7972bec9..0f82a17c 100644 +index e093bbda..e24365d5 100644 --- a/faiss/IndexIDMap.cpp +++ b/faiss/IndexIDMap.cpp -@@ -18,6 +18,7 @@ - #include - #include - #include -+#include - - namespace faiss { - -@@ -102,6 +103,24 @@ struct ScopedSelChange { +@@ -102,6 +102,23 @@ struct ScopedSelChange { } }; -+// RAII object to reset the id_map parameter in ResultCollectorFactory object -+// This object make sure to reset the id_map parameter in ResultCollectorFactory -+// once the program exist current method scope. -+struct ScopedColChange { -+ ResultCollectorFactory* collector_factory = nullptr; -+ void set( -+ ResultCollectorFactory* collector_factory, -+ const std::vector* id_map) { -+ this->collector_factory = collector_factory; -+ collector_factory->id_map = id_map; ++/// RAII object to reset the IDGrouper in the params object ++struct ScopedGrpChange { ++ SearchParameters* params = nullptr; ++ IDGrouper* old_grp = nullptr; ++ ++ void set(SearchParameters* params_2, IDGrouper* new_grp) { ++ this->params = params_2; ++ old_grp = params_2->grp; ++ params_2->grp = new_grp; + } -+ ~ScopedColChange() { -+ if (collector_factory) { -+ collector_factory->id_map = nullptr; ++ ~ScopedGrpChange() { ++ if (params) { ++ params->grp = old_grp; + } + } +}; @@ -92,97 +133,161 @@ index 7972bec9..0f82a17c 100644 } // namespace template -@@ -114,6 +133,7 @@ void IndexIDMapTemplate::search( +@@ -114,6 +131,8 @@ void IndexIDMapTemplate::search( const SearchParameters* params) const { IDSelectorTranslated this_idtrans(this->id_map, nullptr); ScopedSelChange sel_change; -+ ScopedColChange col_change; ++ IDGrouperTranslated this_idgrptrans(this->id_map, nullptr); ++ ScopedGrpChange grp_change; if (params && params->sel) { auto idtrans = dynamic_cast(params->sel); -@@ -131,6 +151,10 @@ void IndexIDMapTemplate::search( +@@ -131,6 +150,16 @@ void IndexIDMapTemplate::search( sel_change.set(params_non_const, &this_idtrans); } } + -+ if (params && params->col && !params->col->id_map) { -+ col_change.set(params->col, &this->id_map); ++ if (params && params->grp) { ++ auto idtrans = dynamic_cast(params->grp); ++ ++ if (!idtrans) { ++ auto params_non_const = const_cast(params); ++ this_idgrptrans.grp = params->grp; ++ grp_change.set(params_non_const, &this_idgrptrans); ++ } + } index->search(n, x, k, distances, labels, params); idx_t* li = labels; #pragma omp parallel for -diff --git a/faiss/impl/HNSW.cpp b/faiss/impl/HNSW.cpp -index 9fc201ea..5b5900d1 100644 ---- a/faiss/impl/HNSW.cpp -+++ b/faiss/impl/HNSW.cpp -@@ -14,6 +14,7 @@ - #include - #include - #include -+#include - #include +diff --git a/faiss/IndexIDMap.h b/faiss/IndexIDMap.h +index 2d164123..a68887bd 100644 +--- a/faiss/IndexIDMap.h ++++ b/faiss/IndexIDMap.h +@@ -9,6 +9,7 @@ - #include -@@ -530,6 +531,15 @@ int search_from_candidates( - int level, - int nres_in = 0, - const SearchParametersHNSW* params = nullptr) { -+ ResultCollectorFactory defaultFactory; -+ ResultCollectorFactory* collectorFactory; -+ if (params == nullptr || params->col == nullptr) { -+ collectorFactory = &defaultFactory; -+ } else { -+ collectorFactory = params->col; -+ } -+ ResultCollector* collector = collectorFactory->new_collector(); -+ - int nres = nres_in; - int ndis = 0; + #include + #include ++#include + #include -@@ -544,11 +554,7 @@ int search_from_candidates( - float d = candidates.dis[i]; - FAISS_ASSERT(v1 >= 0); - if (!sel || sel->is_member(v1)) { -- if (nres < k) { -- faiss::maxheap_push(++nres, D, I, d, v1); -- } else if (d < D[0]) { -- faiss::maxheap_replace_top(nres, D, I, d, v1); -- } -+ collector->collect(k, nres, D, I, d, v1); - } - vt.set(v1); + #include +@@ -124,4 +125,25 @@ struct IDSelectorTranslated : IDSelector { } -@@ -612,11 +618,7 @@ int search_from_candidates( + }; - auto add_to_heap = [&](const size_t idx, const float dis) { - if (!sel || sel->is_member(idx)) { -- if (nres < k) { -- faiss::maxheap_push(++nres, D, I, dis, idx); -- } else if (dis < D[0]) { -- faiss::maxheap_replace_top(nres, D, I, dis, idx); -- } -+ collector->collect(k, nres, D, I, dis, idx); - } - candidates.push(idx, dis); - }; -@@ -660,6 +662,11 @@ int search_from_candidates( - } ++// IDGrouper that translates the ids using an IDMap ++struct IDGrouperTranslated : IDGrouper { ++ const std::vector& id_map; ++ const IDGrouper* grp; ++ ++ IDGrouperTranslated( ++ const std::vector& id_map, ++ const IDGrouper* grp) ++ : id_map(id_map), grp(grp) {} ++ ++ IDGrouperTranslated(IndexBinaryIDMap& index_idmap, const IDGrouper* grp) ++ : id_map(index_idmap.id_map), grp(grp) {} ++ ++ IDGrouperTranslated(IndexIDMap& index_idmap, const IDGrouper* grp) ++ : id_map(index_idmap.id_map), grp(grp) {} ++ ++ idx_t get_group(idx_t id) const override { ++ return grp->get_group(id_map[id]); ++ } ++}; ++ + } // namespace faiss +diff --git a/faiss/impl/HNSW.cpp b/faiss/impl/HNSW.cpp +index fb4de678..b6f602a0 100644 +--- a/faiss/impl/HNSW.cpp ++++ b/faiss/impl/HNSW.cpp +@@ -110,8 +110,8 @@ void HNSW::print_neighbor_stats(int level) const { + level, + nb_neighbors(level)); + size_t tot_neigh = 0, tot_common = 0, tot_reciprocal = 0, n_node = 0; +-#pragma omp parallel for reduction(+: tot_neigh) reduction(+: tot_common) \ +- reduction(+: tot_reciprocal) reduction(+: n_node) ++#pragma omp parallel for reduction(+ : tot_neigh) reduction(+ : tot_common) \ ++ reduction(+ : tot_reciprocal) reduction(+ : n_node) + for (int i = 0; i < levels.size(); i++) { + if (levels[i] > level) { + n_node++; +@@ -804,6 +804,12 @@ int extract_k_from_ResultHandler(ResultHandler& res) { + if (auto hres = dynamic_cast(&res)) { + return hres->k; } ++ ++ if (auto hres = dynamic_cast< ++ GroupedHeapBlockResultHandler::SingleResultHandler*>(&res)) { ++ return hres->k; ++ } ++ + return 1; + } -+ // Completed collection of result. Run post processor. -+ collector->post_process(nres, I); -+ // Collector completed its task. Release all resource of the collector. -+ collectorFactory->delete_collector(collector); -+ - if (level == 0) { - stats.n1++; - if (candidates.size() == 0) { -diff --git a/faiss/impl/ResultCollector.h b/faiss/impl/ResultCollector.h +diff --git a/faiss/impl/IDGrouper.cpp b/faiss/impl/IDGrouper.cpp +new file mode 100644 +index 00000000..ca9f5fda +--- /dev/null ++++ b/faiss/impl/IDGrouper.cpp +@@ -0,0 +1,51 @@ ++/** ++ * Copyright (c) Facebook, Inc. and its affiliates. ++ * ++ * This source code is licensed under the MIT license found in the ++ * LICENSE file in the root directory of this source tree. ++ */ ++ ++#include ++#include ++#include ++ ++namespace faiss { ++ ++/*********************************************************************** ++ * IDGrouperBitmap ++ ***********************************************************************/ ++ ++IDGrouperBitmap::IDGrouperBitmap(size_t n, uint64_t* bitmap) ++ : n(n), bitmap(bitmap) {} ++ ++idx_t IDGrouperBitmap::get_group(idx_t id) const { ++ assert(id >= 0 && "id shouldn't be less than zero"); ++ assert(id < this->n * 64 && "is should be less than total number of bits"); ++ ++ idx_t index = id >> 6; // div by 64 ++ uint64_t block = this->bitmap[index] >> ++ (id & 63); // Equivalent of words[i] >> (index % 64) ++ // block is non zero after right shift, it means, next set bit is in current ++ // block The index of set bit is "given index" + "trailing zero in the right ++ // shifted word" ++ if (block != 0) { ++ return id + __builtin_ctzll(block); ++ } ++ ++ while (++index < this->n) { ++ block = this->bitmap[index]; ++ if (block != 0) { ++ return (index << 6) + __builtin_ctzll(block); ++ } ++ } ++ ++ return NO_MORE_DOCS; ++} ++ ++void IDGrouperBitmap::set_group(idx_t group_id) { ++ idx_t index = group_id >> 6; ++ this->bitmap[index] |= 1ULL ++ << (group_id & 63); // Equivalent of 1ULL << (value % 64) ++} ++ ++} // namespace faiss +diff --git a/faiss/impl/IDGrouper.h b/faiss/impl/IDGrouper.h new file mode 100644 -index 00000000..a0489fd6 +index 00000000..d56113d9 --- /dev/null -+++ b/faiss/impl/ResultCollector.h -@@ -0,0 +1,74 @@ ++++ b/faiss/impl/IDGrouper.h +@@ -0,0 +1,51 @@ +/** + * Copyright (c) Facebook, Inc. and its affiliates. + * @@ -192,77 +297,259 @@ index 00000000..a0489fd6 + +#pragma once + ++#include +#include +#include + +#include -+#include + -+/** -+ * ResultCollector is intended to define how to collect search result -+ * For each single search result, collect method will be called. -+ * After every results are collected, post_process method is called at the end. -+ */ ++/** IDGrouper is intended to define a group of vectors to include only ++ * the nearest vector of each group during search */ + +namespace faiss { + -+/** Encapsulates a set of ids to handle. */ -+struct ResultCollector { -+ /** -+ * For each result, collect method is called to store result -+ * @param k number of vectors to search -+ * @param nres number of results in queue -+ * @param bh_val search result, distances from query -+ * @param bh_ids search result, ids of vectors -+ * @param val distance from query for current vector -+ * @param ids id of current vector ++/** Encapsulates a group id of ids */ ++struct IDGrouper { ++ const idx_t NO_MORE_DOCS = std::numeric_limits::max(); ++ virtual idx_t get_group(idx_t id) const = 0; ++ virtual ~IDGrouper() {} ++}; ++ ++/** One bit per element. Constructed with a bitmap, size ceil(n / 8). ++ */ ++struct IDGrouperBitmap : IDGrouper { ++ // length of the bitmap array ++ size_t n; ++ ++ // Array of uint64_t holding the bits ++ // Using uint64_t to leverage function __builtin_ctzll which is defined in ++ // faiss/impl/platform_macros.h Group id of a given id is next set bit in ++ // the bitmap ++ uint64_t* bitmap; ++ ++ /** Construct with a binary mask ++ * ++ * @param n size of the bitmap array ++ * @param bitmap group id of a given id is next set bit in the bitmap + */ -+ virtual void collect( -+ int k, -+ int& nres, -+ float* bh_val, -+ idx_t* bh_ids, -+ float val, -+ idx_t ids) = 0; -+ -+ // This method is called after all result is collected -+ virtual void post_process(idx_t nres, idx_t* bh_ids) = 0; -+ virtual ~ResultCollector() {} ++ IDGrouperBitmap(size_t n, uint64_t* bitmap); ++ idx_t get_group(idx_t id) const final; ++ void set_group(idx_t group_id); ++ ~IDGrouperBitmap() override {} +}; + -+struct DefaultCollector : ResultCollector { -+ void collect( -+ int k, -+ int& nres, -+ float* bh_val, -+ idx_t* bh_ids, -+ float val, -+ idx_t ids) override { -+ if (nres < k) { -+ faiss::maxheap_push(++nres, bh_val, bh_ids, val, ids); -+ } else if (val < bh_val[0]) { -+ faiss::maxheap_replace_top(nres, bh_val, bh_ids, val, ids); ++} // namespace faiss +diff --git a/faiss/impl/ResultHandler.h b/faiss/impl/ResultHandler.h +index 270de8dc..2f7f3e7f 100644 +--- a/faiss/impl/ResultHandler.h ++++ b/faiss/impl/ResultHandler.h +@@ -12,6 +12,8 @@ + #pragma once + + #include ++#include ++#include + #include + #include + +@@ -265,6 +267,191 @@ struct HeapBlockResultHandler : BlockResultHandler { + } + }; + ++/***************************************************************** ++ * Heap based result handler with grouping ++ *****************************************************************/ ++ ++template ++struct GroupedHeapBlockResultHandler : BlockResultHandler { ++ using T = typename C::T; ++ using TI = typename C::TI; ++ using BlockResultHandler::i0; ++ using BlockResultHandler::i1; ++ ++ T* heap_dis_tab; ++ TI* heap_ids_tab; ++ int64_t k; // number of results to keep ++ ++ IDGrouper* id_grouper; ++ TI* heap_group_ids_tab; ++ std::unordered_map* group_id_to_index_in_heap_tab; ++ ++ GroupedHeapBlockResultHandler( ++ size_t nq, ++ T* heap_dis_tab, ++ TI* heap_ids_tab, ++ size_t k, ++ IDGrouper* id_grouper) ++ : BlockResultHandler(nq), ++ heap_dis_tab(heap_dis_tab), ++ heap_ids_tab(heap_ids_tab), ++ k(k), ++ id_grouper(id_grouper) {} ++ ++ /****************************************************** ++ * API for 1 result at a time (each SingleResultHandler is ++ * called from 1 thread) ++ */ ++ ++ struct SingleResultHandler : ResultHandler { ++ GroupedHeapBlockResultHandler& hr; ++ using ResultHandler::threshold; ++ size_t k; ++ ++ T* heap_dis; ++ TI* heap_ids; ++ TI* heap_group_ids; ++ std::unordered_map group_id_to_index_in_heap; ++ ++ explicit SingleResultHandler(GroupedHeapBlockResultHandler& hr) ++ : hr(hr), k(hr.k) {} ++ ++ /// begin results for query # i ++ void begin(size_t i) { ++ heap_dis = hr.heap_dis_tab + i * k; ++ heap_ids = hr.heap_ids_tab + i * k; ++ heap_heapify(k, heap_dis, heap_ids); ++ threshold = heap_dis[0]; ++ heap_group_ids = new TI[hr.k]; ++ for (size_t i = 0; i < hr.k; i++) { ++ heap_group_ids[i] = -1; ++ } + } ++ ++ /// add one result for query i ++ bool add_result(T dis, TI idx) final { ++ if (!C::cmp(threshold, dis)) { ++ return false; ++ } ++ ++ idx_t group_id = hr.id_grouper->get_group(idx); ++ typename std::unordered_map::const_iterator it_pos = ++ group_id_to_index_in_heap.find(group_id); ++ if (it_pos == group_id_to_index_in_heap.end()) { ++ group_heap_replace_top( ++ k, ++ heap_dis, ++ heap_ids, ++ heap_group_ids, ++ dis, ++ idx, ++ group_id, ++ &group_id_to_index_in_heap); ++ return true; ++ } else { ++ size_t pos = it_pos->second; ++ if (!C::cmp(heap_dis[pos], dis)) { ++ return false; ++ } ++ group_heap_replace_at( ++ pos, ++ k, ++ heap_dis, ++ heap_ids, ++ heap_group_ids, ++ dis, ++ idx, ++ group_id, ++ &group_id_to_index_in_heap); ++ return true; ++ } ++ } ++ ++ /// series of results for query i is done ++ void end() { ++ heap_reorder(k, heap_dis, heap_ids); ++ delete heap_group_ids; ++ } ++ }; ++ ++ /****************************************************** ++ * API for multiple results (called from 1 thread) ++ */ ++ ++ /// begin ++ void begin_multiple(size_t i0_2, size_t i1_2) final { ++ this->i0 = i0_2; ++ this->i1 = i1_2; ++ for (size_t i = i0; i < i1; i++) { ++ heap_heapify(k, heap_dis_tab + i * k, heap_ids_tab + i * k); ++ } ++ size_t size = (i1 - i0) * k; ++ heap_group_ids_tab = new TI[size]; ++ for (size_t i = 0; i < size; i++) { ++ heap_group_ids_tab[i] = -1; ++ } ++ group_id_to_index_in_heap_tab = ++ new std::unordered_map[i1 - i0]; + } + -+ // This method is called once all result is collected so that final post -+ // processing can be done For example, if the result is collected using -+ // group id, the group id can be converted back to its original id inside -+ // this method -+ void post_process(idx_t nres, idx_t* bh_ids) override { -+ // Do nothing ++ /// add results for query i0..i1 and j0..j1 ++ void add_results(size_t j0, size_t j1, const T* dis_tab) final { ++#pragma omp parallel for ++ for (int64_t i = i0; i < i1; i++) { ++ T* heap_dis = heap_dis_tab + i * k; ++ TI* heap_ids = heap_ids_tab + i * k; ++ const T* dis_tab_i = dis_tab + (j1 - j0) * (i - i0) - j0; ++ T thresh = heap_dis[0]; // NOLINT(*-use-default-none) ++ for (size_t j = j0; j < j1; j++) { ++ T dis = dis_tab_i[j]; ++ if (C::cmp(thresh, dis)) { ++ idx_t group_id = id_grouper->get_group(j); ++ typename std::unordered_map::const_iterator ++ it_pos = group_id_to_index_in_heap_tab[i - i0].find( ++ group_id); ++ if (it_pos == group_id_to_index_in_heap_tab[i - i0].end()) { ++ group_heap_replace_top( ++ k, ++ heap_dis, ++ heap_ids, ++ heap_group_ids_tab + ((i - i0) * k), ++ dis, ++ j, ++ group_id, ++ &group_id_to_index_in_heap_tab[i - i0]); ++ thresh = heap_dis[0]; ++ } else { ++ size_t pos = it_pos->first; ++ if (C::cmp(heap_dis[pos], dis)) { ++ group_heap_replace_at( ++ pos, ++ k, ++ heap_dis, ++ heap_ids, ++ heap_group_ids_tab + ((i - i0) * k), ++ dis, ++ j, ++ group_id, ++ &group_id_to_index_in_heap_tab[i - i0]); ++ thresh = heap_dis[0]; ++ } ++ } ++ } ++ } ++ } + } + -+ ~DefaultCollector() override {} ++ /// series of results for queries i0..i1 is done ++ void end_multiple() final { ++ // maybe parallel for ++ for (size_t i = i0; i < i1; i++) { ++ heap_reorder(k, heap_dis_tab + i * k, heap_ids_tab + i * k); ++ } ++ delete group_id_to_index_in_heap_tab; ++ delete heap_group_ids_tab; ++ } +}; + -+} // namespace faiss -diff --git a/faiss/impl/ResultCollectorFactory.h b/faiss/impl/ResultCollectorFactory.h + /***************************************************************** + * Reservoir result handler + * +diff --git a/faiss/utils/GroupHeap.h b/faiss/utils/GroupHeap.h new file mode 100644 -index 00000000..b460b20b +index 00000000..3b7078da --- /dev/null -+++ b/faiss/impl/ResultCollectorFactory.h -@@ -0,0 +1,33 @@ ++++ b/faiss/utils/GroupHeap.h +@@ -0,0 +1,182 @@ +/** + * Copyright (c) Facebook, Inc. and its affiliates. + * @@ -271,31 +558,493 @@ index 00000000..b460b20b + */ + +#pragma once -+#include ++ ++#include ++#include ++#include ++ ++#include ++#include ++#include ++ ++#include ++#include ++ ++#include ++#include ++ +namespace faiss { + -+/** ResultCollectorFactory to create a ResultCollector object */ -+struct ResultCollectorFactory { -+ DefaultCollector default_collector; -+ const std::vector* id_map = nullptr; ++/** ++ * From start_index, it compare its value with parent node's and swap if needed. ++ * Continue until either there is no swap or it reaches the top node. ++ */ ++template ++static inline void group_up_heap( ++ typename C::T* heap_dis, ++ typename C::TI* heap_ids, ++ typename C::TI* heap_group_ids, ++ std::unordered_map* group_id_to_index_in_heap, ++ size_t start_index) { ++ heap_dis--; /* Use 1-based indexing for easier node->child translation */ ++ heap_ids--; ++ heap_group_ids--; ++ size_t i = start_index + 1, i_father; ++ typename C::T target_dis = heap_dis[i]; ++ typename C::TI target_id = heap_ids[i]; ++ typename C::TI target_group_id = heap_group_ids[i]; + -+ // Create a new ResultCollector object -+ virtual ResultCollector* new_collector() { -+ return &default_collector; ++ while (i > 1) { ++ i_father = i >> 1; ++ if (!C::cmp2( ++ target_dis, ++ heap_dis[i_father], ++ target_id, ++ heap_ids[i_father])) { ++ /* the heap structure is ok */ ++ break; ++ } ++ heap_dis[i] = heap_dis[i_father]; ++ heap_ids[i] = heap_ids[i_father]; ++ heap_group_ids[i] = heap_group_ids[i_father]; ++ (*group_id_to_index_in_heap)[heap_group_ids[i]] = i - 1; ++ i = i_father; + } ++ heap_dis[i] = target_dis; ++ heap_ids[i] = target_id; ++ heap_group_ids[i] = target_group_id; ++ (*group_id_to_index_in_heap)[heap_group_ids[i]] = i - 1; ++} + -+ // For default case, the factory share single object and no need to delete -+ // the object. For other case, the factory can create a new object which -+ // need to be deleted later. We have deleteCollector method to handle both -+ // case as factory class knows how to release resource that it created -+ virtual void delete_collector(ResultCollector* collector) { -+ // Do nothing ++/** ++ * From start_index, it compare its value with child node's and swap if needed. ++ * Continue until either there is no swap or it reaches the leaf node. ++ */ ++template ++static inline void group_down_heap( ++ size_t k, ++ typename C::T* heap_dis, ++ typename C::TI* heap_ids, ++ typename C::TI* heap_group_ids, ++ std::unordered_map* group_id_to_index_in_heap, ++ size_t start_index) { ++ heap_dis--; /* Use 1-based indexing for easier node->child translation */ ++ heap_ids--; ++ heap_group_ids--; ++ size_t i = start_index + 1, i1, i2; ++ typename C::T target_dis = heap_dis[i]; ++ typename C::TI target_id = heap_ids[i]; ++ typename C::TI target_group_id = heap_group_ids[i]; ++ ++ while (1) { ++ i1 = i << 1; ++ i2 = i1 + 1; ++ if (i1 > k) { ++ break; ++ } ++ ++ // Note that C::cmp2() is a bool function answering ++ // `(a1 > b1) || ((a1 == b1) && (a2 > b2))` for max ++ // heap and same with the `<` sign for min heap. ++ if ((i2 == k + 1) || ++ C::cmp2(heap_dis[i1], heap_dis[i2], heap_ids[i1], heap_ids[i2])) { ++ if (C::cmp2(target_dis, heap_dis[i1], target_id, heap_ids[i1])) { ++ break; ++ } ++ heap_dis[i] = heap_dis[i1]; ++ heap_ids[i] = heap_ids[i1]; ++ heap_group_ids[i] = heap_group_ids[i1]; ++ (*group_id_to_index_in_heap)[heap_group_ids[i]] = i - 1; ++ i = i1; ++ } else { ++ if (C::cmp2(target_dis, heap_dis[i2], target_id, heap_ids[i2])) { ++ break; ++ } ++ heap_dis[i] = heap_dis[i2]; ++ heap_ids[i] = heap_ids[i2]; ++ heap_group_ids[i] = heap_group_ids[i2]; ++ (*group_id_to_index_in_heap)[heap_group_ids[i]] = i - 1; ++ i = i2; ++ } + } ++ heap_dis[i] = target_dis; ++ heap_ids[i] = target_id; ++ heap_group_ids[i] = target_group_id; ++ (*group_id_to_index_in_heap)[heap_group_ids[i]] = i - 1; ++} + -+ virtual ~ResultCollectorFactory() {} -+}; ++template ++static inline void group_heap_replace_top( ++ size_t k, ++ typename C::T* heap_dis, ++ typename C::TI* heap_ids, ++ typename C::TI* heap_group_ids, ++ typename C::T dis, ++ typename C::TI id, ++ typename C::TI group_id, ++ std::unordered_map* group_id_to_index_in_heap) { ++ assert(group_id_to_index_in_heap->find(group_id) == ++ group_id_to_index_in_heap->end() && ++ "group id should not exist in the binary heap"); ++ ++ group_id_to_index_in_heap->erase(heap_group_ids[0]); ++ heap_group_ids[0] = group_id; ++ heap_dis[0] = dis; ++ heap_ids[0] = id; ++ (*group_id_to_index_in_heap)[group_id] = 0; ++ group_down_heap( ++ k, ++ heap_dis, ++ heap_ids, ++ heap_group_ids, ++ group_id_to_index_in_heap, ++ 0); ++} ++ ++template ++static inline void group_heap_replace_at( ++ size_t pos, ++ size_t k, ++ typename C::T* heap_dis, ++ typename C::TI* heap_ids, ++ typename C::TI* heap_group_ids, ++ typename C::T dis, ++ typename C::TI id, ++ typename C::TI group_id, ++ std::unordered_map* group_id_to_index_in_heap) { ++ assert(group_id_to_index_in_heap->find(group_id) != ++ group_id_to_index_in_heap->end() && ++ "group id should exist in the binary heap"); ++ assert(group_id_to_index_in_heap->find(group_id)->second == pos && ++ "index of group id in the heap should be same as pos"); ++ ++ heap_dis[pos] = dis; ++ heap_ids[pos] = id; ++ group_up_heap( ++ heap_dis, heap_ids, heap_group_ids, group_id_to_index_in_heap, pos); ++ group_down_heap( ++ k, ++ heap_dis, ++ heap_ids, ++ heap_group_ids, ++ group_id_to_index_in_heap, ++ pos); ++} + +} // namespace faiss +\ No newline at end of file +diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt +index cc0a4f4c..96e19328 100644 +--- a/tests/CMakeLists.txt ++++ b/tests/CMakeLists.txt +@@ -26,6 +26,8 @@ set(FAISS_TEST_SRC + test_approx_topk.cpp + test_RCQ_cropping.cpp + test_distances_simd.cpp ++ test_id_grouper.cpp ++ test_group_heap.cpp + test_heap.cpp + test_code_distance.cpp + test_hnsw.cpp +diff --git a/tests/test_group_heap.cpp b/tests/test_group_heap.cpp +new file mode 100644 +index 00000000..0e8fe7a7 +--- /dev/null ++++ b/tests/test_group_heap.cpp +@@ -0,0 +1,98 @@ ++/** ++ * Copyright (c) Facebook, Inc. and its affiliates. ++ * ++ * This source code is licensed under the MIT license found in the ++ * LICENSE file in the root directory of this source tree. ++ */ ++#include ++#include ++#include ++#include ++ ++using namespace faiss; ++ ++TEST(GroupHeap, group_heap_replace_top) { ++ using C = CMax; ++ const int k = 100; ++ float binary_heap_values[k]; ++ int64_t binary_heap_ids[k]; ++ heap_heapify(k, binary_heap_values, binary_heap_ids); ++ int64_t binary_heap_group_ids[k]; ++ for (size_t i = 0; i < k; i++) { ++ binary_heap_group_ids[i] = -1; ++ } ++ std::unordered_map group_id_to_index_in_heap; ++ for (int i = 1000; i > 0; i--) { ++ group_heap_replace_top( ++ k, ++ binary_heap_values, ++ binary_heap_ids, ++ binary_heap_group_ids, ++ i * 10.0, ++ i, ++ i, ++ &group_id_to_index_in_heap); ++ } ++ ++ heap_reorder(k, binary_heap_values, binary_heap_ids); ++ ++ for (int i = 0; i < k; i++) { ++ ASSERT_EQ((i + 1) * 10.0, binary_heap_values[i]); ++ ASSERT_EQ(i + 1, binary_heap_ids[i]); ++ } ++} ++ ++TEST(GroupHeap, group_heap_replace_at) { ++ using C = CMax; ++ const int k = 10; ++ float binary_heap_values[k]; ++ int64_t binary_heap_ids[k]; ++ heap_heapify(k, binary_heap_values, binary_heap_ids); ++ int64_t binary_heap_group_ids[k]; ++ for (size_t i = 0; i < k; i++) { ++ binary_heap_group_ids[i] = -1; ++ } ++ std::unordered_map group_id_to_index_in_heap; ++ ++ std::unordered_map group_id_to_id; ++ for (int i = 1000; i > 0; i--) { ++ int64_t group_id = rand() % 100; ++ group_id_to_id[group_id] = i; ++ if (group_id_to_index_in_heap.find(group_id) == ++ group_id_to_index_in_heap.end()) { ++ group_heap_replace_top( ++ k, ++ binary_heap_values, ++ binary_heap_ids, ++ binary_heap_group_ids, ++ i * 10.0, ++ i, ++ group_id, ++ &group_id_to_index_in_heap); ++ } else { ++ group_heap_replace_at( ++ group_id_to_index_in_heap.at(group_id), ++ k, ++ binary_heap_values, ++ binary_heap_ids, ++ binary_heap_group_ids, ++ i * 10.0, ++ i, ++ group_id, ++ &group_id_to_index_in_heap); ++ } ++ } ++ ++ heap_reorder(k, binary_heap_values, binary_heap_ids); ++ ++ std::vector sorted_ids; ++ for (const auto& pair : group_id_to_id) { ++ sorted_ids.push_back(pair.second); ++ } ++ std::sort(sorted_ids.begin(), sorted_ids.end()); ++ ++ for (int i = 0; i < k && binary_heap_ids[i] != -1; i++) { ++ ASSERT_EQ(sorted_ids[i] * 10.0, binary_heap_values[i]); ++ ASSERT_EQ(sorted_ids[i], binary_heap_ids[i]); ++ } ++} +diff --git a/tests/test_id_grouper.cpp b/tests/test_id_grouper.cpp +new file mode 100644 +index 00000000..2aed5500 +--- /dev/null ++++ b/tests/test_id_grouper.cpp +@@ -0,0 +1,189 @@ ++/** ++ * Copyright (c) Facebook, Inc. and its affiliates. ++ * ++ * This source code is licensed under the MIT license found in the ++ * LICENSE file in the root directory of this source tree. ++ */ ++#include ++#include ++#include ++#include ++ ++#include ++#include ++#include ++#include ++#include ++ ++// 64-bit int ++using idx_t = faiss::idx_t; ++ ++using namespace faiss; ++ ++TEST(IdGrouper, get_group) { ++ uint64_t ids1[1] = {0b1000100010001000}; ++ IDGrouperBitmap bitmap(1, ids1); ++ ++ ASSERT_EQ(3, bitmap.get_group(0)); ++ ASSERT_EQ(3, bitmap.get_group(1)); ++ ASSERT_EQ(3, bitmap.get_group(2)); ++ ASSERT_EQ(3, bitmap.get_group(3)); ++ ASSERT_EQ(7, bitmap.get_group(4)); ++ ASSERT_EQ(7, bitmap.get_group(5)); ++ ASSERT_EQ(7, bitmap.get_group(6)); ++ ASSERT_EQ(7, bitmap.get_group(7)); ++ ASSERT_EQ(11, bitmap.get_group(8)); ++ ASSERT_EQ(11, bitmap.get_group(9)); ++ ASSERT_EQ(11, bitmap.get_group(10)); ++ ASSERT_EQ(11, bitmap.get_group(11)); ++ ASSERT_EQ(15, bitmap.get_group(12)); ++ ASSERT_EQ(15, bitmap.get_group(13)); ++ ASSERT_EQ(15, bitmap.get_group(14)); ++ ASSERT_EQ(15, bitmap.get_group(15)); ++ ASSERT_EQ(bitmap.NO_MORE_DOCS, bitmap.get_group(16)); ++} ++ ++TEST(IdGrouper, set_group) { ++ idx_t group_ids[] = {64, 127, 128, 1022}; ++ uint64_t ids[16] = {}; // 1023 / 64 + 1 ++ IDGrouperBitmap bitmap(16, ids); ++ ++ for (int i = 0; i < 4; i++) { ++ bitmap.set_group(group_ids[i]); ++ } ++ ++ int group_id_index = 0; ++ for (int i = 0; i <= group_ids[3]; i++) { ++ ASSERT_EQ(group_ids[group_id_index], bitmap.get_group(i)); ++ if (group_ids[group_id_index] == i) { ++ group_id_index++; ++ } ++ } ++ ASSERT_EQ(bitmap.NO_MORE_DOCS, bitmap.get_group(group_ids[3] + 1)); ++} ++ ++TEST(IdGrouper, bitmap_with_hnsw) { ++ int d = 1; // dimension ++ int nb = 10; // database size ++ ++ std::mt19937 rng; ++ std::uniform_real_distribution<> distrib; ++ ++ float* xb = new float[d * nb]; ++ ++ for (int i = 0; i < nb; i++) { ++ for (int j = 0; j < d; j++) ++ xb[d * i + j] = distrib(rng); ++ xb[d * i] += i / 1000.; ++ } ++ ++ uint64_t bitmap[1] = {}; ++ faiss::IDGrouperBitmap id_grouper(1, bitmap); ++ for (int i = 0; i < nb; i++) { ++ if (i % 2 == 1) { ++ id_grouper.set_group(i); ++ } ++ } ++ ++ int k = 10; ++ int m = 8; ++ faiss::Index* index = ++ new faiss::IndexHNSWFlat(d, m, faiss::MetricType::METRIC_L2); ++ index->add(nb, xb); // add vectors to the index ++ ++ // search ++ idx_t* I = new idx_t[k]; ++ float* D = new float[k]; ++ ++ auto pSearchParameters = new faiss::SearchParametersHNSW(); ++ pSearchParameters->grp = &id_grouper; ++ ++ index->search(1, xb, k, D, I, pSearchParameters); ++ ++ std::unordered_set group_ids; ++ ASSERT_EQ(0, I[0]); ++ ASSERT_EQ(0, D[0]); ++ group_ids.insert(id_grouper.get_group(I[0])); ++ for (int j = 1; j < 5; j++) { ++ ASSERT_NE(-1, I[j]); ++ ASSERT_NE(std::numeric_limits::max(), D[j]); ++ group_ids.insert(id_grouper.get_group(I[j])); ++ } ++ for (int j = 5; j < k; j++) { ++ ASSERT_EQ(-1, I[j]); ++ ASSERT_EQ(std::numeric_limits::max(), D[j]); ++ } ++ ASSERT_EQ(5, group_ids.size()); ++ ++ delete[] I; ++ delete[] D; ++ delete[] xb; ++} ++ ++TEST(IdGrouper, bitmap_with_hnswn_idmap) { ++ int d = 1; // dimension ++ int nb = 10; // database size ++ ++ std::mt19937 rng; ++ std::uniform_real_distribution<> distrib; ++ ++ float* xb = new float[d * nb]; ++ idx_t* xids = new idx_t[d * nb]; ++ ++ for (int i = 0; i < nb; i++) { ++ for (int j = 0; j < d; j++) ++ xb[d * i + j] = distrib(rng); ++ xb[d * i] += i / 1000.; ++ } ++ ++ uint64_t bitmap[1] = {}; ++ faiss::IDGrouperBitmap id_grouper(1, bitmap); ++ int num_grp = 0; ++ int grp_size = 2; ++ int id_in_grp = 0; ++ for (int i = 0; i < nb; i++) { ++ xids[i] = i + num_grp; ++ id_in_grp++; ++ if (id_in_grp == grp_size) { ++ id_grouper.set_group(i + num_grp + 1); ++ num_grp++; ++ id_in_grp = 0; ++ } ++ } ++ ++ int k = 10; ++ int m = 8; ++ faiss::Index* index = ++ new faiss::IndexHNSWFlat(d, m, faiss::MetricType::METRIC_L2); ++ faiss::IndexIDMap id_map = ++ faiss::IndexIDMap(index); // add vectors to the index ++ id_map.add_with_ids(nb, xb, xids); ++ ++ // search ++ idx_t* I = new idx_t[k]; ++ float* D = new float[k]; ++ ++ auto pSearchParameters = new faiss::SearchParametersHNSW(); ++ pSearchParameters->grp = &id_grouper; ++ ++ id_map.search(1, xb, k, D, I, pSearchParameters); ++ ++ std::unordered_set group_ids; ++ ASSERT_EQ(0, I[0]); ++ ASSERT_EQ(0, D[0]); ++ group_ids.insert(id_grouper.get_group(I[0])); ++ for (int j = 1; j < 5; j++) { ++ ASSERT_NE(-1, I[j]); ++ ASSERT_NE(std::numeric_limits::max(), D[j]); ++ group_ids.insert(id_grouper.get_group(I[j])); ++ } ++ for (int j = 5; j < k; j++) { ++ ASSERT_EQ(-1, I[j]); ++ ASSERT_EQ(std::numeric_limits::max(), D[j]); ++ } ++ ASSERT_EQ(5, group_ids.size()); ++ ++ delete[] I; ++ delete[] D; ++ delete[] xb; ++} -- 2.39.3 (Apple Git-145) diff --git a/jni/src/faiss_util.cpp b/jni/src/faiss_util.cpp new file mode 100644 index 000000000..c2abe7f26 --- /dev/null +++ b/jni/src/faiss_util.cpp @@ -0,0 +1,23 @@ +// SPDX-License-Identifier: Apache-2.0 +// +// The OpenSearch Contributors require contributions made to +// this file be licensed under the Apache-2.0 license or a +// compatible open source license. +// +// Modifications Copyright OpenSearch Contributors. See +// GitHub history for details. + +#include "faiss_util.h" +#include + +std::unique_ptr faiss_util::buildIDGrouperBitmap(int *parentIdsArray, int parentIdsLength, std::vector* bitmap) { + const int* maxValue = std::max_element(parentIdsArray, parentIdsArray + parentIdsLength); + int num_bits = *maxValue + 1; + int num_blocks = (num_bits >> 6) + 1; // div by 64 + bitmap->resize(num_blocks, 0); + std::unique_ptr idGrouper(new faiss::IDGrouperBitmap(num_blocks, bitmap->data())); + for (int i = 0; i < parentIdsLength; i++) { + idGrouper->set_group(parentIdsArray[i]); + } + return idGrouper; +} diff --git a/jni/src/faiss_wrapper.cpp b/jni/src/faiss_wrapper.cpp index 4609f3144..e88254b86 100644 --- a/jni/src/faiss_wrapper.cpp +++ b/jni/src/faiss_wrapper.cpp @@ -11,7 +11,7 @@ #include "jni_util.h" #include "faiss_wrapper.h" -#include "knn_extension/faiss/MultiVectorResultCollectorFactory.h" +#include "faiss_util.h" #include "faiss/impl/io.h" #include "faiss/index_factory.h" @@ -51,9 +51,7 @@ void convertFilterIdsToFaissIdType(const int* filterIds, int filterIdsLength, fa // Concerts the FilterIds to BitMap void buildFilterIdsBitMap(const int* filterIds, int filterIdsLength, uint8_t* bitsetVector); -os_faiss::MultiVectorResultCollectorFactory* buildResultCollectorFactory(knn_jni::JNIUtilInterface * jniUtil, JNIEnv *env, jintArray parentIdsJ); - -void releaseResultCollectorFactory(os_faiss::MultiVectorResultCollectorFactory* collectorFactory); +std::unique_ptr buildIDGrouperBitmap(knn_jni::JNIUtilInterface * jniUtil, JNIEnv *env, jintArray parentIdsJ, std::vector* bitmap); void knn_jni::faiss_wrapper::CreateIndex(knn_jni::JNIUtilInterface * jniUtil, JNIEnv * env, jintArray idsJ, jobjectArray vectorsJ, jstring indexPathJ, jobject parametersJ) { @@ -253,13 +251,18 @@ jobjectArray knn_jni::faiss_wrapper::QueryIndex_WithFilter(knn_jni::JNIUtilInter faiss::SearchParameters *searchParameters; faiss::SearchParametersHNSW hnswParams; faiss::SearchParametersIVF ivfParams; + std::unique_ptr idGrouper; + std::vector idGrouperBitmap; auto hnswReader = dynamic_cast(indexReader->index); if(hnswReader) { // Setting the ef_search value equal to what was provided during index creation. SearchParametersHNSW has a default // value of ef_search = 16 which will then be used. hnswParams.efSearch = hnswReader->hnsw.efSearch; hnswParams.sel = idSelector.get(); - hnswParams.col = buildResultCollectorFactory(jniUtil, env, parentIdsJ); + if (parentIdsJ != nullptr) { + idGrouper = buildIDGrouperBitmap(jniUtil, env, parentIdsJ, &idGrouperBitmap); + hnswParams.grp = idGrouper.get(); + } searchParameters = &hnswParams; } else { auto ivfReader = dynamic_cast(indexReader->index); @@ -274,30 +277,29 @@ jobjectArray knn_jni::faiss_wrapper::QueryIndex_WithFilter(knn_jni::JNIUtilInter } catch (...) { jniUtil->ReleaseFloatArrayElements(env, queryVectorJ, rawQueryvector, JNI_ABORT); jniUtil->ReleaseIntArrayElements(env, filterIdsJ, filteredIdsArray, JNI_ABORT); - releaseResultCollectorFactory(dynamic_cast(hnswParams.col)); throw; } jniUtil->ReleaseIntArrayElements(env, filterIdsJ, filteredIdsArray, JNI_ABORT); - releaseResultCollectorFactory(dynamic_cast(hnswParams.col)); } else { faiss::SearchParameters *searchParameters = nullptr; faiss::SearchParametersHNSW hnswParams; + std::unique_ptr idGrouper; + std::vector idGrouperBitmap; auto hnswReader = dynamic_cast(indexReader->index); if(hnswReader!= nullptr && parentIdsJ != nullptr) { // Setting the ef_search value equal to what was provided during index creation. SearchParametersHNSW has a default // value of ef_search = 16 which will then be used. hnswParams.efSearch = hnswReader->hnsw.efSearch; - hnswParams.col = buildResultCollectorFactory(jniUtil, env, parentIdsJ); + idGrouper = buildIDGrouperBitmap(jniUtil, env, parentIdsJ, &idGrouperBitmap); + hnswParams.grp = idGrouper.get(); searchParameters = &hnswParams; } try { indexReader->search(1, rawQueryvector, kJ, dis.data(), ids.data(), searchParameters); } catch (...) { jniUtil->ReleaseFloatArrayElements(env, queryVectorJ, rawQueryvector, JNI_ABORT); - releaseResultCollectorFactory(dynamic_cast(hnswParams.col)); throw; } - releaseResultCollectorFactory(dynamic_cast(hnswParams.col)); } jniUtil->ReleaseFloatArrayElements(env, queryVectorJ, rawQueryvector, JNI_ABORT); @@ -509,21 +511,10 @@ void buildFilterIdsBitMap(const int* filterIds, int filterIdsLength, uint8_t* bi } } -os_faiss::MultiVectorResultCollectorFactory* buildResultCollectorFactory(knn_jni::JNIUtilInterface * jniUtil, JNIEnv *env, jintArray parentIdsJ) { - if (parentIdsJ == nullptr) { - return nullptr; - } +std::unique_ptr buildIDGrouperBitmap(knn_jni::JNIUtilInterface * jniUtil, JNIEnv *env, jintArray parentIdsJ, std::vector* bitmap) { int *parentIdsArray = jniUtil->GetIntArrayElements(env, parentIdsJ, nullptr); int parentIdsLength = jniUtil->GetJavaIntArrayLength(env, parentIdsJ); - auto* parent_id_filter = new FixedBitSet(parentIdsArray, parentIdsLength); + std::unique_ptr idGrouper = faiss_util::buildIDGrouperBitmap(parentIdsArray, parentIdsLength, bitmap); jniUtil->ReleaseIntArrayElements(env, parentIdsJ, parentIdsArray, JNI_ABORT); - return new os_faiss::MultiVectorResultCollectorFactory(parent_id_filter); -} - -void releaseResultCollectorFactory(os_faiss::MultiVectorResultCollectorFactory* collectorFactory) { - if (collectorFactory == nullptr) { - return; - } - delete collectorFactory->parent_bit_set; - delete collectorFactory; + return idGrouper; } diff --git a/jni/src/knn_extension/faiss/MultiVectorResultCollector.cpp b/jni/src/knn_extension/faiss/MultiVectorResultCollector.cpp deleted file mode 100644 index a7564d3aa..000000000 --- a/jni/src/knn_extension/faiss/MultiVectorResultCollector.cpp +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -#include "MultiVectorResultCollector.h" -#include "knn_extension/faiss/utils/Heap.h" -#include "knn_extension/faiss/utils/BitSet.h" - -namespace os_faiss { - -using idx_t = faiss::idx_t; - -MultiVectorResultCollector::MultiVectorResultCollector(const BitSet* parent_bit_set, const std::vector* id_map) -: parent_bit_set(parent_bit_set), id_map(id_map) {} - -void MultiVectorResultCollector::collect( - int k, - int& nres, - float* bh_val, - int64_t* bh_ids, - float val, - int64_t ids) { - idx_t group_id = id_map ? parent_bit_set->next_set_bit(id_map->at(ids)) : parent_bit_set->next_set_bit(ids); - if (parent_id_to_index.find(group_id) == - parent_id_to_index.end()) { - if (nres < k) { - maxheap_push( - nres++, - bh_val, - bh_ids, - val, - ids, - &parent_id_to_id, - &parent_id_to_index, - group_id); - } else if (val < bh_val[0]) { - maxheap_replace_top( - nres, - bh_val, - bh_ids, - val, - ids, - &parent_id_to_id, - &parent_id_to_index, - group_id); - } - } else if (val < bh_val[parent_id_to_index.at(group_id)]) { - maxheap_update( - nres, - bh_val, - bh_ids, - val, - ids, - &parent_id_to_id, - &parent_id_to_index, - group_id); - } -} - -void MultiVectorResultCollector::post_process(int64_t nres, int64_t* bh_ids) { - for (size_t icnt = 0; icnt < nres; icnt++) { - bh_ids[icnt] = parent_id_to_id.at(bh_ids[icnt]); - } -} - -} // namespace os_faiss diff --git a/jni/src/knn_extension/faiss/MultiVectorResultCollectorFactory.cpp b/jni/src/knn_extension/faiss/MultiVectorResultCollectorFactory.cpp deleted file mode 100644 index f4c7c0656..000000000 --- a/jni/src/knn_extension/faiss/MultiVectorResultCollectorFactory.cpp +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -#include "MultiVectorResultCollectorFactory.h" -#include "MultiVectorResultCollector.h" - -namespace os_faiss { - -MultiVectorResultCollectorFactory::MultiVectorResultCollectorFactory(BitSet* parent_bit_set) - : parent_bit_set(parent_bit_set) {} - -// id_map is set in IndexIDMap.cpp of faiss library with custom patch -// https://github.com/opensearch-project/k-NN/blob/feature/multi-vector/jni/patches/faiss/0001-Custom-patch-to-support-multi-vector.patch#L109 -faiss::ResultCollector* MultiVectorResultCollectorFactory::new_collector() { - return new MultiVectorResultCollector(parent_bit_set, id_map); -} - -void MultiVectorResultCollectorFactory::delete_collector(faiss::ResultCollector* resultCollector) { - delete resultCollector; -} - -} // namespace os_faiss diff --git a/jni/src/knn_extension/faiss/utils/BitSet.cpp b/jni/src/knn_extension/faiss/utils/BitSet.cpp deleted file mode 100644 index 33e9470e0..000000000 --- a/jni/src/knn_extension/faiss/utils/BitSet.cpp +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -#include -#include -#include "BitSet.h" - -FixedBitSet::FixedBitSet(const int* int_array, const int length){ - assert(int_array && "int_array should not be null"); - const int* maxValue = std::max_element(int_array, int_array + length); - this->num_bits = *maxValue + 1; - this->num_words = (num_bits >> 6) + 1; // div by 64 - this->words = new uint64_t[this->num_words](); - for(int i = 0 ; i < length ; i ++) { - int value = int_array[i]; - int bitset_array_index = value >> 6; - this->words[bitset_array_index] |= 1ULL << (value & 63); // Equivalent of 1ULL << (value % 64) - } -} - -idx_t FixedBitSet::next_set_bit(idx_t index) const { - assert(index >= 0 && "index shouldn't be less than zero"); - assert(index < this->num_bits && "index should be less than total number of bits"); - - idx_t i = index >> 6; // div by 64 - uint64_t word = this->words[i] >> (index & 63); // Equivalent of words[i] >> (index % 64) - // word is non zero after right shift, it means, next set bit is in current word - // The index of set bit is "given index" + "trailing zero in the right shifted word" - if (word != 0) { - return index + __builtin_ctzll(word); - } - - while (++i < this->num_words) { - word = this->words[i]; - if (word != 0) { - return (i << 6) + __builtin_ctzll(word); - } - } - - return NO_MORE_DOCS; -} - -FixedBitSet::~FixedBitSet() { - delete this->words; -} diff --git a/jni/tests/faiss_util_test.cpp b/jni/tests/faiss_util_test.cpp new file mode 100644 index 000000000..d8b45d951 --- /dev/null +++ b/jni/tests/faiss_util_test.cpp @@ -0,0 +1,28 @@ +// SPDX-License-Identifier: Apache-2.0 +// +// The OpenSearch Contributors require contributions made to +// this file be licensed under the Apache-2.0 license or a +// compatible open source license. +// +// Modifications Copyright OpenSearch Contributors. See +// GitHub history for details. + +#include "faiss_util.h" + +#include + +#include "gtest/gtest.h" + +TEST(IDGrouperBitMapTest, BasicAssertions) { + int ids[] = {128, 1024}; + size_t length = sizeof(ids) / sizeof(ids[0]); + std::vector bitmap; + std::unique_ptr idGrouperBitmap = faiss_util::buildIDGrouperBitmap(ids, length, &bitmap); + int groupIndex = 0; + for (int i = 0; i <= ids[length - 1]; i++) { + if (i > ids[groupIndex]) { + groupIndex++; + } + ASSERT_EQ(ids[groupIndex], idGrouperBitmap->get_group(i)); + } +} diff --git a/jni/tests/faiss_wrapper_test.cpp b/jni/tests/faiss_wrapper_test.cpp index 5afe09c22..58daaee2b 100644 --- a/jni/tests/faiss_wrapper_test.cpp +++ b/jni/tests/faiss_wrapper_test.cpp @@ -205,7 +205,7 @@ TEST(FaissQueryIndexTest, BasicAssertions) { // Create the index std::unique_ptr createdIndex( - test_util::FaissCreateIndex(2, method, metricType)); + test_util::FaissCreateIndex(dim, method, metricType)); auto createdIndexWithData = test_util::FaissAddData(createdIndex.get(), ids, vectors); @@ -262,7 +262,7 @@ TEST(FaissQueryIndexWithFilterTest1435, BasicAssertions) { // Create the index std::unique_ptr createdIndex( - test_util::FaissCreateIndex(2, method, metricType)); + test_util::FaissCreateIndex(dim, method, metricType)); auto createdIndexWithData = test_util::FaissAddData(createdIndex.get(), ids, vectors); @@ -335,7 +335,7 @@ TEST(FaissQueryIndexWithParentFilterTest, BasicAssertions) { // Create the index std::unique_ptr createdIndex( - test_util::FaissCreateIndex(2, method, metricType)); + test_util::FaissCreateIndex(dim, method, metricType)); auto createdIndexWithData = test_util::FaissAddData(createdIndex.get(), ids, vectors); @@ -474,7 +474,7 @@ TEST(FaissCreateHnswSQfp16IndexTest, BasicAssertions) { // Assert that Index is of type IndexHNSWSQ ASSERT_NE(indexIDMap, nullptr); ASSERT_NE(dynamic_cast(indexIDMap->index), nullptr); - + // Clean up std::remove(indexPath.c_str()); } diff --git a/jni/tests/knn_extension/faiss/MultiVectorResultCollectorFactoryTest.cpp b/jni/tests/knn_extension/faiss/MultiVectorResultCollectorFactoryTest.cpp deleted file mode 100644 index 3177360bb..000000000 --- a/jni/tests/knn_extension/faiss/MultiVectorResultCollectorFactoryTest.cpp +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -#include "knn_extension/faiss/MultiVectorResultCollectorFactory.h" -#include "knn_extension/faiss/MultiVectorResultCollector.h" - -#include - -#include "gmock/gmock.h" -#include "gtest/gtest.h" -#include "jni_util.h" - -using ::testing::NiceMock; -using ::testing::Return; -using idx_t = faiss::idx_t; - - -TEST(MultiVectorResultCollectorFactoryTest, BasicAssertions) { - int parent_ids[1] = {1}; - FixedBitSet parent_id_filter(parent_ids, 1); - - std::unordered_map distance1; - distance1[0] = 10; - distance1[1] = 11; - - std::unordered_map distance2; - distance2[0] = 11; - distance2[1] = 10; - - os_faiss::MultiVectorResultCollectorFactory* rc_factory = new os_faiss::MultiVectorResultCollectorFactory(&parent_id_filter); - faiss::ResultCollector* rc1 = rc_factory->new_collector(); - faiss::ResultCollector* rc2 = rc_factory->new_collector(); - ASSERT_NE(rc1, rc2); - - int k = 1; - int nres1 = 0; - int nres2 = 0; - float* bh_val = new float[k * 2]; - int64_t* bh_ids = new int64_t[k * 2]; - // Verify two collector are thread safe each other. - // Simulate multi thread by interleaving collect methods of two ResultCollectors. - for (int i = 0; i < distance1.size(); i++) { - rc1->collect(k, nres1, bh_val, bh_ids, distance1.at(i), i); - rc2->collect(k, nres2, bh_val + k, bh_ids + k, distance2.at(i), i); - } - rc1->post_process(nres1, bh_ids); - rc2->post_process(nres2, bh_ids + k); - - ASSERT_EQ(0, bh_ids[0]); - ASSERT_EQ(1, bh_ids[1]); - - rc_factory->delete_collector(rc1); - rc_factory->delete_collector(rc2); - delete rc_factory; - delete[] bh_val; - delete[] bh_ids; -} - -// Verify that id_map is passed to collector -TEST(MultiVectorResultCollectorFactoryWithIdMapTest, BasicAssertions) { - int parent_ids[1] = {1}; - FixedBitSet parent_id_filter(parent_ids, 1); - std::vector id_map; - - os_faiss::MultiVectorResultCollectorFactory* rc_factory = new os_faiss::MultiVectorResultCollectorFactory(&parent_id_filter); - os_faiss::MultiVectorResultCollector* rc1 = dynamic_cast(rc_factory->new_collector()); - ASSERT_EQ(nullptr, rc1->id_map); - - rc_factory->id_map = &id_map; - os_faiss::MultiVectorResultCollector* rc2 = dynamic_cast(rc_factory->new_collector()); - ASSERT_EQ(&id_map, rc2->id_map); - - rc_factory->delete_collector(rc1); - rc_factory->delete_collector(rc2); - delete rc_factory; -} diff --git a/jni/tests/knn_extension/faiss/MultiVectorResultCollectorTest.cpp b/jni/tests/knn_extension/faiss/MultiVectorResultCollectorTest.cpp deleted file mode 100644 index 934d708dd..000000000 --- a/jni/tests/knn_extension/faiss/MultiVectorResultCollectorTest.cpp +++ /dev/null @@ -1,96 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -#include "knn_extension/faiss/MultiVectorResultCollector.h" - -#include - -#include "gmock/gmock.h" -#include "gtest/gtest.h" - -using ::testing::NiceMock; -using ::testing::Return; -using idx_t = faiss::idx_t; - - -TEST(MultiVectorResultCollectorTest, BasicAssertions) { - // Data - // Parent ID: 2, ID: 0, Distance: 10 - // Parent ID: 2, ID: 1, Distance: 11 - // Parent ID: 5, ID: 3, Distance: 12 - // Parent ID: 5, ID: 4, Distance: 13 - // After collector handing the data with k = 3, it should return data with id 0 and 2, one from each group. - // Parent bit set representation: 100100 - int parent_ids[2] = {2, 5}; - FixedBitSet parent_id_filter(parent_ids, 2); - - idx_t ids[] = {0, 1, 2, 3}; - float distances[] = {10, 11, 12, 13}; - - os_faiss::MultiVectorResultCollector* rc = new os_faiss::MultiVectorResultCollector(&parent_id_filter, nullptr); - int k = 3; - int nres = 0; - float* bh_val = new float[k]; - int64_t* bh_ids = new int64_t[k]; - for (int i = 0; i < 4; i++) { - rc->collect(k, nres, bh_val, bh_ids, distances[i], ids[i]); - } - - // Parent ID is stored before finalize - ASSERT_EQ(5, bh_ids[0]); - ASSERT_EQ(2, bh_ids[1]); - - rc->post_process(nres, bh_ids); - - // Parent ID is converted to ID after finalize - ASSERT_EQ(3, bh_ids[0]); - ASSERT_EQ(0, bh_ids[1]); - - delete rc; - delete[] bh_val; - delete[] bh_ids; -} - -TEST(MultiVectorResultCollectorWithIDMapTest, BasicAssertions) { - // Data - // Parent ID: 2, Lucene ID: 0, Faiss ID: 0, Distance: 10 - // Parent ID: 2, Lucene ID: 1, Faiss ID: 1, Distance: 11 - // Parent ID: 5, Lucene ID: 3, Faiss ID: 2, Distance: 12 - // Parent ID: 5, Lucene ID: 4, Faiss ID: 3, Distance: 13 - // After collector handing the data with k = 3, it should return data with id 0 and 2, one from each group. - - // Parent bit set representation with Lucene ID: 100100 - int parent_ids[2] = {2, 5}; - FixedBitSet parent_id_filter(parent_ids, 2); - - idx_t faiss_ids[] = {0, 1, 2, 3}; - float distances[] = {10, 11, 12, 13}; - - // Faiss IDs to Lucene ID mapping - std::vector id_map = {0, 1, 3, 4}; - - os_faiss::MultiVectorResultCollector* rc = new os_faiss::MultiVectorResultCollector(&parent_id_filter, &id_map); - int k = 3; - int nres = 0; - float* bh_val = new float[k]; - int64_t* bh_ids = new int64_t[k]; - for (int i = 0; i < 4; i++) { - rc->collect(k, nres, bh_val, bh_ids, distances[i], faiss_ids[i]); - } - - // Parent ID is stored before finalize - ASSERT_EQ(5, bh_ids[0]); - ASSERT_EQ(2, bh_ids[1]); - - rc->post_process(nres, bh_ids); - - // Parent ID is converted to Faiss ID after finalize - ASSERT_EQ(2, bh_ids[0]); - ASSERT_EQ(0, bh_ids[1]); - - delete rc; - delete[] bh_val; - delete[] bh_ids; -} diff --git a/jni/tests/knn_extension/faiss/utils/BitSetTest.cpp b/jni/tests/knn_extension/faiss/utils/BitSetTest.cpp deleted file mode 100644 index 96ad6b3c2..000000000 --- a/jni/tests/knn_extension/faiss/utils/BitSetTest.cpp +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -#include "knn_extension/faiss/utils/BitSet.h" - -#include - -#include "gmock/gmock.h" -#include "gtest/gtest.h" - -using ::testing::NiceMock; -using ::testing::Return; -using idx_t = faiss::idx_t; - -TEST(FixedBitSetTest, BasicAssertions) { - int ids1[4] = {3, 7, 11, 15}; - FixedBitSet single_block(ids1, 4); - - ASSERT_EQ(3, single_block.next_set_bit(0)); - ASSERT_EQ(3, single_block.next_set_bit(1)); - ASSERT_EQ(3, single_block.next_set_bit(2)); - ASSERT_EQ(3, single_block.next_set_bit(3)); - ASSERT_EQ(7, single_block.next_set_bit(4)); - ASSERT_EQ(7, single_block.next_set_bit(5)); - ASSERT_EQ(7, single_block.next_set_bit(6)); - ASSERT_EQ(7, single_block.next_set_bit(7)); - ASSERT_EQ(11, single_block.next_set_bit(8)); - ASSERT_EQ(11, single_block.next_set_bit(9)); - ASSERT_EQ(11, single_block.next_set_bit(10)); - ASSERT_EQ(11, single_block.next_set_bit(11)); - ASSERT_EQ(15, single_block.next_set_bit(12)); - ASSERT_EQ(15, single_block.next_set_bit(13)); - ASSERT_EQ(15, single_block.next_set_bit(14)); - ASSERT_EQ(15, single_block.next_set_bit(15)); - ASSERT_EQ(single_block.NO_MORE_DOCS, single_block.next_set_bit(16)); - - int ids2[5] = {64, 128, 127, 1024, 34565}; - int ids2_sorted[5]; - std::copy(ids2, ids2 + 5, ids2_sorted); - std::sort(ids2_sorted, ids2_sorted + 5); - FixedBitSet multi_blocks(ids2, 5); - int parent_index = 0; - for (int i = 0; i < ids2[4] + 1; i++) { - ASSERT_EQ(ids2_sorted[parent_index], multi_blocks.next_set_bit(i)); - if (ids2_sorted[parent_index] == i) { - parent_index++; - } - } - ASSERT_EQ(multi_blocks.NO_MORE_DOCS, multi_blocks.next_set_bit(ids2[4] + 1)); -} diff --git a/jni/tests/knn_extension/faiss/utils/HeapTest.cpp b/jni/tests/knn_extension/faiss/utils/HeapTest.cpp deleted file mode 100644 index 97a30babd..000000000 --- a/jni/tests/knn_extension/faiss/utils/HeapTest.cpp +++ /dev/null @@ -1,86 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -#include "knn_extension/faiss/utils/Heap.h" -#include "faiss/utils/Heap.h" - -#include "gmock/gmock.h" -#include "gtest/gtest.h" - -using ::testing::NiceMock; -using ::testing::Return; -using ::testing::ElementsAreArray; - -TEST(MaxHeapUpdateTest, BasicAssertions) { - const int k = 5; - int nres = 0; - float binary_heap_values[k]; - int64_t binary_heap_ids[k]; - float input_values[] = {1.1f, 2.1f, 3.1f, 4.1f, 5.1f}; - int64_t input_ids[] = {1, 2, 3, 4, 5}; - int64_t group_ids[] = {11, 22, 33, 44, 55}; - std::unordered_map group_id_to_id; - std::unordered_map group_id_to_index; - - // Push - for (int i = 0; i < k; i++) { - os_faiss::maxheap_push( - nres++, - binary_heap_values, - binary_heap_ids, - input_values[i], - input_ids[i], - &group_id_to_id, - &group_id_to_index, - group_ids[i]); - } - - // Verify heap data - // The top node in the max heap should be the one with max value(5.1f) - ASSERT_EQ(5.1f, binary_heap_values[0]); - ASSERT_EQ(55, binary_heap_ids[0]); - ASSERT_EQ(5, group_id_to_id.at(binary_heap_ids[0])); - - // Replace top - os_faiss::maxheap_replace_top( - nres, - binary_heap_values, - binary_heap_ids, - 0.1f, - 6, - &group_id_to_id, - &group_id_to_index, - 66); - - // Verify heap data - // Previous top value(5.1f) should have been removed and the next max value(4.1f) should be in the top node. - ASSERT_EQ(4.1f, binary_heap_values[0]); - ASSERT_EQ(44, binary_heap_ids[0]); - ASSERT_EQ(4, group_id_to_id.at(binary_heap_ids[0])); - - // Update - os_faiss::maxheap_update( - nres, - binary_heap_values, - binary_heap_ids, - 0.2f, - 7, - &group_id_to_id, - &group_id_to_index, - 33); - - // Verify heap data - // node id 3 with group id 33 should have been replaced by node id 7 with new value - ASSERT_EQ(7, group_id_to_id.at(33)); - - // Verify heap is in order - float expectedValues[] = {4.1f, 2.1f, 1.1f, 0.2f, 0.1f}; - int64_t expectedIds[] = {4, 2, 1, 7, 6}; - for (int i = 0; i < k; i++) { - ASSERT_EQ(expectedValues[i], binary_heap_values[0]); - ASSERT_EQ(expectedIds[i], group_id_to_id.at(binary_heap_ids[0])); - faiss::maxheap_pop(nres--, binary_heap_values, binary_heap_ids); - } -} From 6be0f0c46472e79804b8620351a78d25608f0d5c Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Mon, 19 Feb 2024 18:10:07 -0800 Subject: [PATCH 203/416] Remove unnecessary dataset (#1485) (#1486) Signed-off-by: Heemin Kim (cherry picked from commit ab9991eaf2e736f6ebe1bb22cb9cd9c192650782) Co-authored-by: Heemin Kim --- .../dataset/sift-128-euclidean-nested.hdf5 | Bin 74496 -> 0 bytes 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 benchmarks/perf-tool/dataset/sift-128-euclidean-nested.hdf5 diff --git a/benchmarks/perf-tool/dataset/sift-128-euclidean-nested.hdf5 b/benchmarks/perf-tool/dataset/sift-128-euclidean-nested.hdf5 deleted file mode 100644 index 4223d72810785f11ba801ac0fe819b0a72d0ada6..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 74496 zcmeFY2T&DF+bv2EK@cR01OX8cQBV+3VfR|(3?c#w0+JO_5haR%iWzgxVnWP{m=lU% zz#K7WF=xf>neThw_f&me^^-Fs(O?Vj%G-Lt2s`|0OdYxNXgPj7WqLshwd z9V#kv&Ez!yefrPWpT7^e_VWKO`k(o~E5!Ys{&iJOZt~yx3WdK;PdO*E4Sa`$32Qoxaom zXXO1=kd-_#>tAPCBjQtsNojmm)`--EY^nIK>;Kg5`d9XU`zplhap`(=Yz_xt!Fx%D=BGDE!r+%HK0Z zQC_<6*J1E?`gfA|pV|MzxxT@@J>}(^|Ihh6x%q$p2mb0UIk{@xe|r9ZUElTZ;tP8J zeZBsF*Sp{PzpwXC=b0(~ukZIy=|Ar+PwAhM|1;_TZd~>M+kXD@|Nd+K|Him#_xBSn z_|GT(e|B8`U+s^-mjStd`s1Ix;@|T~M;aFYTFm~Nd9~-S=dOC!p3RXb6qz>mjBx7i!LwogsAV(;0|VN#wrd~* zIwr5Eh2o<~q2TP!^DSG@N+plkA0ERfx|7R!?-#Ns0sc(vH<*JvZ^Py# z#++J_C<|#G!eFlt80ut?6o|AjMukBvejub#`iQ~ zXk`ZHY;)lrlT5alQ6(0BRTH_N4&z0UJcmqpFLExwf_%U|+0AJIjI9}r4jM+NmABzI zh333;`5x}Q^QX$^K9E1;LM6v25#RAUTGSkNX-Miq_tWJ#?>-y#j*TwE2DIbptGlow zZ73qvhEe->$kAbrT(L+i-kcF~=Qu7qSo9{B}^D z!nOuew^l>p;t{me?aC|lU3h%v3lXFl$8R6Huyf-|^mLeoG|z0RbgdQEo!ZjlaVjgL z$u8qRiPPWmu=%SS6ZHo1;z&bkZZhL*tJ5MgyAo+rM{~iGI=nlZz+oQwTzGplBMiJa zy2PHrecQ2Yss)md#L4`U(zqxnhmC8vO5O(A8d2-4$O`yO1IWR15MSHQSTA5+pj zXu7wUx^;RyS-M8#PnrywK`*{3ZHEF~LoQmi4o7=D$J!?f=rUq9mO35A`{lQwme&=J zE}OI8{5N7szsWe38qPs;y;ztSf_-l_S)b^^k9XcWx9psaLm{oWWQz-r1O#H|iZ{r8 zI@&BC+IY$1s&^xxoAfb+eY-{k_kJ|Hm(C){Q{UA z^I7<(1fpH_6IsDWdwzPD&b!+mVosk6&|6TB@(rOJJ?1+`pGd*lBtI^EKN6<%b8z`h zARi_>aG1F*Efohd?usto_K)JM8N+Ck+>YKyZp%KT_aIudz(uKi{iIfmt=C|mMthDt z+KH=GqNvn4jVXOoxYi+n{r`;M2N$4eQ7v4SoI+yN5J>rICt5Wcu=mv= zB6rXSF<^jxeQ>uKy^`m=&?s{5tlVof1zPGKA90aZtOd!Qh28ID9UP-!)dE{iW-e{wJ2LjB;^g z#4{K;(vHJUq4B#MP=9r4>Jhy4~9qPED6Uz9K6;@uE>*JLv2 zZ52#5XmHo@7{08NUH9gRN~TDZ0NJtj{|WxEtDGvaWZ?tk-I*r* z7Y}5#LoZ%bsz9*UMXdbZkJZNG{4Trx z%9po=EsvMmiwN0poEW1`GmCFxtByI3sw>0YD2*pu4;3SHqj1P-2nWVRa^B?@d=gr~ z<&|UMA*~&Ur4`^{rXYf z*Ht_XTPqCmD`i1tHdyhtE3b_RrG5D=SXw!=MN~Q`Z?R|DZ3C_?_d@CuWo}<@!@h?G z(Bow=f4sHEmVJS=dz^qW<%ignyc%ga-I;7Vn&U$*;Mdv$?rbU(3jX(G;>VEimt77xZGO-#dG4=>w6W%Wp&PWynv)oQLibqypmYCaFq1^yK>}m zcV6q5En4&}=J+qkuzy~Uep?*b-oFUHX3j*DsWsJS?2wf`{fR@{1t0j>h^5lpZ(Jl# zn^%VD+u4S;Uwqi!tPAyV35_Ywa3MZNR4L}s!lDZD=XBuaW5MAwqPg`HIL>ZAtR8h` z#+%WsHSiL#T8g}Ty#b|#ew+}x4GrH6**8v(rS(%0v8FvnH&0oz8H2IGp)hwZkv%%2z*%#PxF&c%GKZL>qH-!+ zmJH#t@wMo=(TD{ihbFdZOtg2QnR}y1s8(Xb=7l1Az+f7rgm8y>H<7x=mTJn;oZRah znkCIe*vuHNC`zQF{W)>Ta0VuRkH=Wg!PI;D4i6fJP<3oO29^Y))lV;OJ~tPMnFr7% zOM#y)RtlpnePMsanZ0}GGVy>bb2V(Kv_A%!+kG*x)nVDHh^t6BFdu717W2KwWSDNw z#vOAbj`hi53xhXchXOXM*#h;@_i%gag&8kPu}n3QuKj9obDo80RW+IwD=%YX;dBiB zQ->+LoM_q zOKF|Va@H&)-R;H)ixgOM`xcH|)S`BE9s2(u@4xvV(^+1?=l(?;)$=O$Y>46)4KMaO zn!=S;Eot;3kw@E)rf?oE)S5={WrsSHd=C&uGj2j*!5esL^oPP*J${yal(-+JypnFn zQGRL2PM(Ppv-R+uD8tVDyBM>jHv=MPi_l&Dxas+19KEZ}2RGJY!>hiWoi;|CaB0tx zhh|}>_CTHrQ4=Z$idpi#6`Rz`;JChnXjW0gU(Tf%k(N!Ld5Kha2;$^HP8?9&gF9R9 zLwYB&^70!Ly0*oH&!_Nboj?40nWNac89Vna#qN?o=4czjXGtb^n5A*Mw>fnO7gD~r z2YU7RjW&}8^6jE*Mh-uP*;D$npH4eo|LVnd1K;7N=N)M0_Tp|a9a)ifIBDyMX&C{0 z+{T{1HJTWwP$s&)3t{Wv2%b!A5XPt4^I_FEJfG;mY&8w+w(KHicX*DgVkrwQIXD)Vp zJtc4Yg`uj=zy!4G_G zaS(IT^0|F(G?)2|n}XO*Ev72 zzwLMER~yp$$!@V?c@kG_ca+9^2>%?tj;N6HFxDEtW?qGyWPSm2&Vds#PwX`4$ml`e zF?VA#YJKm*gBhdPB{3T%13joVNgt1+jhMQ26!zrhQr=?(@6Tz;jbqz##rjLaQzwgl z!85R@dI>D_&*Mpo9A!OL;D&WI=6*ec-IA|1c6c{zbsQ?1cU*!|7VpvYxC8yT6r-<= zNI{nK2YsW~Dl+BoqAkp|R^s)hzKj!c ztRAh$-=6(B=zcL(kLT0l+9)b5j)c{aFwUMC$mogo)IXuiS9{lp^Mg*{i*79zp4or{ z!-&Y~&!9Ijoa6hw#*8he5ZA39p~l0xMOxQqHPzvQ)+pW`@5U$vQ${U*hmyCJ7<}v_ z<}S%&p8?5Y*smD+xensIaBn&cOQfybQ8-29BEjN{i+ zjWDWtDO+6Ci75|t=;atKsNuw2$6jFk*SV4%Y=E-51=C9W7&7bwv_?O`ZaGWd88;2e zLEGSQ&x;Q3yU_RRd93lW=EG$@Xcj-1%8DIXwsHUtJ(G#SYNud$u@qL)97=7kjbUou zE>E+o5ZM(XJkCqDdXO4j!!o#d!68IEa%7cB3obb^9@C}%-g;RcQ%2`gMCuEb*ILrp zGi8$8PrO3btI+dp@aoOIZ=+>+! zHXf1D`S@A!=z2O|gys?#-ij?N{du#xJKxv^v8P;1o^IvLdq$4zxvn*@8FuFxF2>U( z(JT&Vhm3P4U{)Q5*+HH8D8?V=ZY?=>UJ_M)d2&!|4JPaO^OpZ?IPG36I_93lo8|+k zq?yk?_3~_$*^FL|9(0MlE>!!QaZaH(w{5xy<)xibY9z;)ULSDr&|cKb%@cROJEDDj z4ScpQM%EgCo~jlwZ?zFa&ic^x%TyF@+l7#~YKTq`M`C0L7Q2kZM=7R0IWLG6j}AiH zBUtE8{2(jp-kBA?ZxNi{3f*7(@xsnX_8XK)Y<_@4()^2kZi}^l9^-utc-=QpwyeP& zCnprs{FpKq>}bnz26ou5wM68MHRP|$v#?svh>n(hG3@km@z!IlSUl+_UisOua@JUU zcbW&yXWumA~{fIeb z?-5=%15f8Eq2&V?E-r5mwN(YUp3oVer$o`}k{)X$n-W!a8lAQ~&}+RRr?0d_-@_T4 zazw&h>_d4^PKW7te!%9X65sZ?i~YM7pkI;;=I}T6tAh7Zr-*qKFOd_MPOWQ7SfYMG z{Bk@2)!OzLzeS5DYhTM&MfSa-&t*cCqw+G$*ayhL)$} z#m4qmU?atw=6H<5y`?$fs=-qZ+3dCZo3NO2Pu6>V7DqlT!Ru-j9&2e0`wtFmYB|aI z)k1SPHY`T`?%ybKvg4I!zT!*UfiThN&Ff3Enf6eh3p0P9ZM+HmHCBl7TT+aXSq1N7 zgZXotKBqanh3VEoToKj`pUk^qd*f+ryYmuyUuB5C6Z2=p~ZlCxRUA5`*P9LYA0woX$IzqRk#X& zYLE2dx0>cO2utAzdwKqPZ_cG79c5cij28)=4q@?ESB&u(&65jdRMU>;>vnxOr7>Q( z<<5amO&sTKbl`HQ57=SR4w*$S!~}xiF;>2ns%e2_#RudK(D02Qp$YhhRplQAzPvyk$=*&(?8qkMttlwbHfneSpqr+#KyJY=? zoFMyrR_q(ymFE88R6BeXJ8Z$BTW<<&ol;rc>dUB2Z-@6qM*JQ15J`95U{vuej9;-s zw)A%ZTa@QwMS=_K^%sb9ty5(t-(6^-*IKgOO_=aBh2C+od_Eyh+&gp?yDw_equ2^g zir3+MMUN|DAHrsDN3OM5hx(iVE-un!S36JUX6tf*xjREX5kGIrp-*WRzEpjcFm`!5 zYM$yL`m%D=7wnHI;OA~*WfN*kMZ2s0xb>(q)E91s;cFL;3T;V`hqi40 z{3V=|>(IUNozR~CQQVk+2Mboo7<_Okrreq*i%sn=-re(KRC7lvrN;6_Lkdm%?tz(v zg&b3`#H+rJ+`Dj~p!fX$7y}8SL1?lLjR+zU=x`_&H9&lV!Vbysm(4*Gh5HJtraNHDh^~ z5`38)K=sF6c=)0@?KJ|}?1wxL7f!{Nc}BdwXgf|9R$_WYCeyWh^5W&@ur{z`Mp`t7 zkG+E67BSRLv*rE+seJr4n|H4I(k`?CT}oVKvXWI8@mUWQwR&tmra#Imb1-w&J%oMC zM`-U%4(#c~wpL-h)}+T-QT7}auFKi;pF*arLb=6B%zr8&@E<;}lus&vS^cHDZ}T z1~s|_(QBsU$L9EQ*7g({&j{nq^>c7OyEB5dJ9Az7I#JqS2ji15Tm@e~He3vcr-H|y zmBMk{5ISG^j&-dUqI;wR$DDA6l4&OUT^T3dd~QRf$~<28Fr~|yg?QK8PP`624SBg? zTvDRL8y?L#D%+pmAG(SD!)}XS!+&B%mJW}d2w;iERWZ2RBaz(Amby!aa9ijcq};oL z2TDmepsCA4e^fYQ?>RhSD&zGN*zm-j-BMMlwV)3(yiXw}XFD1Ux^b0~DYTlrXf`YX z>y}$^{>vh+b@`1g2DXfJ+=iK}vqY4Z1JWl2aKS}=CT(_R`AbWd1Zv>;($i>e;L08g zw?gGg2kK`uU`#@LM2K9(XY59gWeuphG@O$bti>wHU;gQH6`JLhqA+x?aC~mVh;vfB z`}zsyT<=P~4f1^PY&FgY2}Epe%d%4vj~CsMYGWe!@Y`y!H#VM*!-wCU}DoBVY{R+H}2D>&520f(;3D~ z$!T0OJA^-;e}VNaX+3(<9lER9@wmi4ym$saIcmsr_31Pk=gLUUwIbK-qv$%iuh`k4 z7XozFWBA$>);xFS>8RFh+PnpO?rg!&`u==;Es%3ua(GKunKya_^V2D9EHK(F8yI{- zT*&OgYeRlv{22wjS(?vbPU{gW{{X)mtfknr6N8Ulz|`fYV!8fIY`)QtImMcMJ@gks z%=+`pt9r>C=&gv(GY*LZ8wYn|$LVG{-R^{{I7gbJu6+n0Q=aA3u z%a$@Zerat?s~78_yx*5s%Y*1J-bwcG)-h~#uNOH(wfSLr8kbJZVCwddIPgx37qX3+ zsqzZ*uTI3cqH@4+6mOfBptElczQ=}BOMfK0Z4ReXsV}=}8Z&8hAcNOrik>qji-gQD zI&9WrPisf6*dHW&F-eOCyF(eSk`31yJ>HjiJMl7+XTNu)cLxK0+M7h*xsIIBs|0cF zGw8c=EE;aU!*Bl-E-R1Z>m6RqeLsjfMML=g;!@-&IItncgXR1C^WhCO?CT}Z&WcSk zmrX~-L!G7Qw``qwuA;z>{d2iyha%Ve?GXyuj(nL{Cp<>9=EFtGG^!elwomQE(16kG zI_WO%z8Z@*i=8<^>j|oQ>_+LdbgpnrWWqoNRx}M@_!b*>t_)`T@va=a>;^iY>x7V$ zV9uC%P4qaF#xCvWA=*rV_Q&kl+s2&cZmC@B-jO3SuE}&fjd92I6P9&6iDw6L_$MQt zvqoHm%4!9Ea__^DCWARpeI0ZUHDDk%A;>+I4@=e~sjLRYWtWB7*)-~E-Nd_gmeo-Pm{b=@`7;*m9IzEpak)bOW)M{+jwSbv9Vh-?1+~@coMM^m za%ZDGBMNo-yvmE+MFX5qHo^9;7cV6Jz^7;VoG0^Ok#or{?sXrpOB>8MHS$i?2rK@NHQ)Uh8`V zuTQT)xk@*#`BWhr^uZDRUk|6lML)K*d5(cyx8TLp!OTvJ;j{2>aFh7Z#<>G%+a`}! zrUbFiX&E0l1yIsRaqjecNHvS1^^ZB?cB|fa`plSmofPpsN(mM_h9fR)G|u(ADSr8_ z#qG5-p#IE^#zRe+EBRqfwGGg*t;Y6s&qdm<0=ih7f_D98{Mj7JVRICj{<n-fpO1*3WqmU`9v#kC$A{8zn2)$ReI5=(+{fz2@syMNoU>nxsAuHI zFZ0UqwaL&qa;zn$?}&l$D8|;|^IMfGsQ-CN9FqCksntvSCej=3&cbZ{}^*|A52x0xooywYb)c?OL=_KDwhkHucE6yDp? zih~|ZL_?b*Iz>0hD+h=Y3`@pRKbgY4>(60T(I6;g4xwZJ-YiOx zNxs1ovB9`C^+&abwKOfq_8H1ET6we@-%E&XPCWLm6~kQ_PO7vz_Z~yE4LN#N68HL@$IkKxIA7UUVK_~8gT^^x={g<#_e%4D{c%5e2aa;XWxu;sHV#JXnhspK5UKS3L%O7PRV{i#3m|MExIiUbK;X zo_8_O)=lQ#Tnny=y#v?gdDK0t%0)g;QQ0$?fh*dfYK|sTyu-Qt6`1Ntmd`nlOO{g1 zt!BvLjz`gIeqVNp5sY8cisqL`BJuN86xld&YOWHo^jQX?53|b1t__Y$ts&~R+!CrV)N%(I`Dt`~l z;ItuIVQtrlGY>0Z^!=ewTxZ3}d*=v`hnw(T=>b~wyoR4m2J}jZq5sPdXh^n`t-cfj z`*}9lw=s(WgTLV2MLQl|1!`rQV?w7exP8fCdP5G^tNC(CxC!pMjpoa~nH*sq#_o?5 zxIKIgRLm9esk|I#vc4jC&M>wMms zIa>r8^ZMv?{@Is+fD|X%9ME94We82(y}1)+{Q5?f^VH3dK13N_Go$g#cqkXU4rapB z7*Y9VKE4+e&~3tN%-oR0j?P-LPVaQF^Vn1*Sr+lGZ(Gjx%VVooZ(Me5=nj9?ew<*o z6#DY!nC0|Zn0332v26{x@zGmZQ$-M`K8#^~O$c?{Ji(CFCY0+MM>7W{PLSqmhDr)A z>wbZfP^4;*KlRJCIqQ6_n7(y7YV>-dg=8-Z_myGXT?&oIR3v)-8bd2w$2E-6gDiBz`Kxg-5Slf}Pt< zSUvG&lhZfZu*E7Ivo%N9MZ2?n>qwMK8U>YC?wH;Et;o_@Ay(WSi(j)PJbcJ)c-`A6 z>%00k4&F`UgraO-ZRUrg6{d9Wz8ACi+QVYMF{hoYL$NFo5qn=?_Z1Vy)m%ZlPt9q& zz6VE+lQH+66%BvXV0HEzoO~_$Ul-CuhsQb$th8ab%Mn}`6fJ4GzKi~z3&f@DS8y9B z>3<&2l+E$9M3>F?@pQ`?5hP(*Iq5D?2w#G8Z}fP3!8k;o=tsF9H)Xx_Kj2-1B?Got zF?W0tU3p7nI)N|c454&5oCA6XvHqJ8{Y<$=j8DfE&n;7_x@T)Dpu z*H_IG;}h@TxtX-Cn#Un(qcn~}wqeyZXDlh1iJ8@x@z!XqOUo-}OrCNE*~4NOQ*{oS z!8M|Baeu~3eG%{7hu!Vh+(=+pRPgc&(WNDz=R%kBY8MBl~tMRS_GX_R_pr#i zKx(ruqn*vMR@;C+4z=Q{pAPp79!=AcVT|uE5WU-};#JHz44@Iyx7xAOyjJ|Zb^?m` z^yIc(50IPo1|P3wvt~+XgzhlJh3PS@G#$-uU*l+5-Y%CUN`6L&fkzO`S{}k zOCT4Y&4>{;w6-0>aXWXSF6o7sw4xI8nml=Isya3oP81&xw&d*x6J)E~zk^o$-6)&* z3}Y+0N}Pa%S?4!lmAVELtB-+iB~7Z`K%NZQfYs4)bPtc`_wR=hcCsfPY3Jhd21DAF zTtP|6XM}5d(eHL!TDLQ0P2biG_Wgq=~W*;IZm_d)PtZ?H0>4{|un9gg=L9 zd5V?01P4|3p-Z#}n)8}xA1zs{3#xn9_9I8*NgN=xl% zUZV$(mTG)Dt_=;hEP>Cq42F$#WaISDcsigOMUUI#gMTPmJJn#yoL{m%D*l|_!-oA0 zcZ*3uBUyGi5^p~A;c%-C?Cqz|l*(lhow%=+ zC68uBvia9%FbP-S@Y%bOI&i0OF7e`k4?FSnNE)}b8;8|vf?$)KjH?A_#nT~{T=T75 zl)v7OXU$_NFZtqOlFoQh>n!R`>d!0dlvpg=jPW|n`CxuDXB*Xv;B*zPvD6jjr#o`& zmqhkb`i7w&lJLDWlK$JHnBTY=2ZMj$!z&LISUI4MSE15L7r);)L+?c)WA61uXT5iL zuN%Z3HSbZi_!2@DYti=WN(|mLQ}~{lfJ?I)usv=xlJ@CvTeF1-(e$Oo)FN639fr#b z2NsHShSqAc;gSii9eIswae)j`-zY?{-7s1{n6pX#PfYFF zf=(+78MIZChmS>xn)?fouX<8^e%piYRzuNp@(TU(7@5>{c7NPsaWwNnL<|16GGkl2; z6-xp|M|&k+Dplpi&-VD_cMGqryEAR0GSAYVN5a9rdr9vxETwGvqvK+AmguN5I;9F9|9}UM`@?vm0SDw2i(Lwmc5{X% zLVibxX3vN7_nk{EcH@=m$QLoqGO`tBvZc*lJ^^SZh#ke!a z-o)vh_hokTM(|*094G9tL26aCm^;Or#gC7Q6EB|1B95;Ru1zU;@@gASE;%4^gF0+) zs)G%qThpwA9X93N5u5j&LWq4Ol-{hu2a^Imy}S&usd=2<<}|iQxbGcPZElk=A*bR3 zzKtvt@oOS*ZM6*!+YMpZ{WDNHYJhJuM&PP>rR=A@5~d&QN0<(zU(ROe{+uAXjZ=o| zYdKyp@4=+ogSoJwH}AFy=9v8^tbZ~SDW?l~-rI}Ea`ULIVSw}J-Kml7&H>7iG&Ng@ zp|w3}x-yPMW$W?7>WFwUbPbB5?_p_t5Z|YoXL1r}BvI&^vlWTw9?0Pw%pEV(0KMDrA_l-mx=VKRzRDZ(4GsnE{8q zmhQFK1I`kEuID@%-o0zFx^9G|VUlnN#SS>AJ{s>YxzJ>N3r?Bco`sUnw)H?B6(lUX z&7heQznjCt)O5~tXo=fq&>ZZ+%^Zsl%$A+n&dr)~_9o|ng#axf2 zV);NbTH8#6dqp`;UK!4`S#q=tZNoKs-8pws5$|s-B#MqlB(Zy2Y##foqY)DU>f7+WfD~wrs%L^4fM{tzR z1hK-S1+HFFxRBx@q^+Chv{4QO*p8rA^Uv3n$2g%d!@@z5p>lQp19nB3| z3A}o3GfI24V$P)IoZmYWE^!%RSk4oqZPP;g$GMzZ(uDOBa;a=E3Az{F38T+NJh|~6 z^7Rj4Gmqm`_zS$1cyxzFWzxDMY5)4UG2Uq@Zg;3cr~V35zr9$@oa4@C$H&XEZidtU z`U#OYxIaq^t8rBOtVox%f|nXKX&a}&w_43<|63hX8Y1|>a47PnK3z8SJ63;DWcx$0 zJS{gB+bwrtQHcYM&4SsttQ4;*N8w0cL7ff{@UVCxeB5F=c}lei+b{$3ZXR}7yJ|Il z={aJ?Y!^maRN~L$&p5F3IBuV-#k&bl@bT1Hlq^i*>ZNU1Z|BVz?R4lFgfaSMFx1Cb zvkl&g>Ybm(R5K-(BzNJ|V=Blvo6Ambui)`~IsWXG#!;_U<7c!f#E0JEXv1Z=%|0S* zzct8|rgf3dc@iyS1e5f5>>XPp5+_Lea(Ycb zX;d6LD((?#5sB=)!jNeyUa+Y9< zoyApqQ0wZ#`7y=3+rA3T)CWtLcS{79crf+F6!?uc=8(dHOdT^9!JG1B_L?O)(Ort; zvVe$*g3*(_0(s9wFXmsgLtDfhdF1QIotjl?tUnjIiz?~<(M*L zYj?n`;Q`cb|JO=vXyGen4jIfbIYmr$RKUCP zIDWY)>ES2#gLb3DGis!A{LLtI``Upiy%Mo!T^8)CA~|b!1;P)G;~-Ppv1Y<*lb80`u6iWd3RQ_;iT7l|26tR*y8y1CFND<;1>V`#oADRg^6vcKxM*}ijEu5F z`qeUw4Ul7k{(7v`P=<+XEEOc3fRAlEo{4w>+3dc&p6e*>t?kOKl>>Qpk2@|?TE}}U z!TnewH6nM3!B14Uciwq8B^qFzPX;e9E^#)rlKf27U-+6k4wY(T*Tg9HF$!l>sU3V~ zNc-gcCyEvOtD&?inmgJj<89b3wDpOgh2;p|8ks_Eg`w=KFbfaYyD~61n~zFu*-V<&v$K7&ukTN$X3qb3t@nR>*O>!};xbJ}>W^ zi~&Onx!XUVkJgW%!yE?|t~PpwPTY7})3` z)=r)84b7kMqk0@$D-EUOaFOqYa7? zdip(Xe_0Bnsllvz6pC${cg$Bi5bl1P6uA7_uvyJ*KH* z^~v)noU}}~Qqrr&oztaH!4BA1`tU%2l~9oI4mbZb_!_thg_g<8TU;sIZnYWZ2REXm zn+NYdj>n_yW+-%4!aet&5NALBd(YpT9lxNudAls{{02n7^yak!U7mT=mN|bGpwd*1 zPX>0ONkTs^moWdQKKT}0qbgt~X%Fre6|;1R1uX_jai)?R2c>^S+q&iA zW7Q|ryqtq}GxfRoWh{plER(HWlgo_Tdi-@GfdTRhFlJ*YcYl70!H$8no;Hk~I{5N{ zx`aFR^Q7siUbKk23dhiwcrh$VazFY%z5tuNBBuMlg7JDkG?*udq_00Q zY`zjz3#9!}lGgUrxreasn#s~{HE1Z@fM>}&gzdE`Y5g8b%Pe{Ep)81!2o)O_45QBE zJJ3`fi^Z{nu`;S0Uz{7E-QEV%KKG&4TV?iK5zltpwqQe19JeO@LHCYTSkTd#m%kWs zr>;HApBzX0@tLqQ+^jW@FM$ zX^+T;0dy_1=KrARyaTCz|0rHogv=s)res7R+~=G`8WcrRl+lop6_G@P_FhUUN>h7% zwfEk8Ylt+p_xL@(|EsHeug~*-&ikC#Ss5K5BF4y!&h&YrNp@e~x_$Ya>8!#HHojDw zco~KdtI@^6krTv2JV-i+v=&;TqpmT+vnPvHPv!e-_YDqb)VOkGKI3e4cxG}}<}Ju# ztIdNst5Jzx*Z6S5xhBlgn}}(j!a4845Js>y)#oW=L~eUdbIZXbIm@Mon_{71v3QVj z1}0}!Y43IciKVT@jR2tePakYQtH*Dj%sBt?I0o7j3Lon=Sllv*bN5S~SNi&cWuENj zVyUnEI3Su%4`)aB5Ki0m2ve>NV)wN{n6lCyeFhKb=b$?5v@eJ1-atHUG=Z7vZwwEK zW0k)Jcf2UhWX&Zi?x)Rm<^)Tf3V<@k;<1+IU{<3n% z`85M+opc%pFXUZ(T&vcNa|j7{u+Xq@QHXE`@`9ogd$==&+=K23wji z;`%hn5OCz()45z|+LWEkPT^L;VceZ-1l#))k#e{NR3>?I@zkMQdN+ncHV@~2x*mMp z+?uMt;^As_5m6ao2tM757q6?rN@YCSMXIvz*(9HzH_nL>lQh|~m5F%QtWgpAU z%;&^asm!Q4iiQ~xOg^Lu`}b++yxyK8W5)Z|dPnoebq79M-;Q^mcH@z@&7_xqGA5a} z!{IJ>u+c*aAL2?-;+;*c`)&DKC7lt!g4q4zNX}^+su(M?a2*c0<4&K^yu8;6alv|g zdq{BEH#;70nyK(g=*8|J(A9M5RJGB)vs{65%aUUej4}{sMbPi5!fs%p2 z^y#X~2U4r*q_bOQQD0zMttsd4+k#`G+wy^CI2V4m=k$TkMWCyKGcp62H6@P5Nk?GQ zKaVPU&dfY|3u&4@bUS($TgO?-4CP|+v$`dxZ`9zVCx21c+XMB3U%~pUCi`s*#_Lms zd}GFDwui+F|CzpQKClN*6fO`k zkDGI+d`>Se*)1*w$eFS-lEW@8!#mp%Od1v{bY0hA={7UmU8l^OTiy6*h&r6Q_^}~! zJ8rIa$HmwGVDGR5Hs8Woe^{9X0ZJ@5>??g10jym$kh744fDgee4zQv6m^@BfcMgwF zbYV%E%(6Jn6!R`#7b~L_9DceBKZbNe<(6bj*w&X{3SAVVRxQS+ZV!dSBTL?2G@KWY zjYE+|bH2#w3{&Ny9Co7aPX+oiw%Q-K*fOJBdwp?zYvVG%ZG4CXZ(W8N~$#ub(EoLA8oSIlzwZ)XKM z&dCz*J#C>~=gmB&{U{tY62?tuqvrHkSdG{z3>0}BxvW@doX_A|eRtMujpSLKzC1iX zhdFMx=(6PuA`1Fo?w5QnH8baU*JOD%1!8cQBTyX>PMh*>Jm7L0eSR8p=15ImygC&I zr>2UswJ)K&Gl*VJFVIQyBig1+LdIX2;c6=Lbz%J((yLtIaoQf9vztOEbToGU(4um$ zXdZoS$gulo#hS?XzB2}!%8c0vrrCeS=v#Nh#Fu$2JWcB8Tk$~lI;^>)&Upj>LZiEp za4CC>kRvI0d$1#ZZXP0c&^o?AEhNj;T^!PCX zxmH7h`uYcIPbj~ zchr4F^ypYdYFtCt9+DNY!-eO{K4R|)xvS90=a}K2<$c|SD=Qyk!Luw}Egs7-9T)zP zo`D`?PvN`acwXIe4^1A^d7NMoe&LtM5;7XZ%bIl$!D`;Jkb5X zXj;{$@V@0YJg*o=9ZPk#-WEfhTDi-Oo(JotJ^0~mA(}m1ASUG>L326J&bp?;!eK-3 zY{+uqTDeTVkAwJKu?r)*yHnYF51M5E#X+51Ha^RsTKHRRoNde&QCDHz(TBg26FF+d zYfL;RcMcvhgT6hHK9gpm*X0NVYgdY*sd;F7_9@zF{6mJ!gW6BdVYh3)eNWuV;fZTt z?9yq9tWl3(f3r`?AU@KLrYNh z>L|YNQ{sSTomnV#-gCht#MYZ#eDAxRzzMe_7;ya`^3ITN_LpG%#}t|bnKP=BJs&7! z2IqA(X4jA7uR|)lAhQ%9*(O|3dmam$3};4e4`wDt(DaltgNMZOscKg?&XJkQV`)sX zw5N4eD%TZ&)#sz(R%FBx*KPQFQv%)A7mHt8CgINQKpySq!vjubSUBYicFT;n<36dA z)K{UU!#d1b{|^&>MB(^42VNR-0H#`YT)QBahy1$m^^^Db;^NA>b3;(5u7VLi61m6J zhLPTXFw&zn+s&_n>jMwo%rD2CynNa#9fn`#LRMRosCN z4o2MJyc-APz3JjI4frpq7Qc$(=%RfUdfG9*iT5fHq@%{#ZH>?jIe}*HO;GgKnen&% znYD5aUOo@zqe1x`+rvfXlhSB+L+0e#tBI7lAtK~0c(Yjk-v099=Mb3@k$1EWGdJUP zO*|Lek^h&0v(R?_ExbK4S6JCOqx4Tt_K4~tuJ+ssjlW~XXP;qgJNvRwZpMK}TlTMpYUJ!{J`6+CB$MZ~hdQE9W8Xt1Vmp zY0dlZ(z$l-YsGiH1Xf5ddR=KdRt)dJ4!*u(u&p5u$$dwqK_8hNY|dXNPeDz7ZdH0M zQQoIJt;Sez;?-DoEbYV&ooqN$w+&|u{)?)^iL|-B1O1-(FuUh)8b!6nx=#zmgdh8b z=3iTmT5=gRE#+<}wE(e>n%rSFfG4F7{DZ|mMBGzD^~cFRLmp3u?VCO%>UuTYYC%indiUyV5I# zCo(=EZ=K|oJ&57$*+ckM&xCQopI~q6$GeYW+2N=+7defk%KdbjWIPlhF)>uzuw95F z)1b7d3rfGKOFiQ#mbLa^+L)0TK2GlQjk-g*pEIL8Y(B?I_nNTF;7s1Ac!6JUhjQ}oWGGch590H#*c+J6K{|%;?U5@Pc2di~WXzfXWnN54 zqlkL(rfWJkgl-Y;l3nsP zGL?JUMoDkRNLoeCfur4MF5W#GJBsg$x%GC8wAzIuvu;DbpfzI#ZbFPL8SD`&+S~Tx zh%e#Hkh#y!4GZw~oijImK8aSz?~pvn3>%$QnfALaf4!Lrl0r!t(#LMzay)g{SaL&FI1Sdc=8MlU>=%&12?2h>rCSB;W0s+;S1XPTUxOjblW`;_ zh(}td@Th^zxP`WpGq$`BO>E7@E;8e~(oU2*eZs|)P1t{xC9f{pg7%bYO`&EM_o zJ|&GW-1g#QwDkW>nB|_N z%>p}1G2jIm+dZDzp--`}z5==3CZhkR<{Z_|p7_&_e_+5~gy zwKYG~gNwWL=RB$XT|Z#XumWYd*It0JQFmaVb{}`!$=B*Q97_({@n7|Lc2A7sfJ?hk zbHx*LcX_~JWeShQh4FdqU|MJDGG|i}dQX{+RUMl1WnL@Z9e)RJyUN}A+d(|#z8Lv0 z?urrV(qDh$1eBWnMtRX+(e<_o#}-XPpQajUdu26Zrh9O}ejhAbH;(oi&k!;CqP&McAshuaZ)xYE!c+dIC+vKKv=t>}pU*OV9~Gd~-n3b5?@HI&=wqJzx_ zJn@}~u+ufT7#+{nwsGY9Y^Lmef!m5jqOuG;{(KOpre4BFHFIuUN(SuCW5Cw&TrWSr zK93vGJM6ER5U?20)tb^T@Kfe7FXLD7Yb=pjkzrRHVIUbP+s3D(sa-2h`WS%GcFMeE zSR)Jv74U+lrrgb~RCGUO!kTB#6wwEo(_wf%-6dW@ZV>`gm!w8|Pq80D9^cA|>f-uNBlYuoq*2`{-71d|aC)XVDc^z3<^bs3ZwBX7K8CcZ*AnH1N z5u?{j&!*!HRJJhV;zhHtTWuiM@A!-qt>>7dHyhSR{CVE91s2-G(AHmzUe20aWMRwd zIR~+Jrys|~4dEN%P1mK{(EglcrfKH!y42h}v%9e*!Bw(=H0h}H1fO*W@sV;AJ$L!> zciSI|nnkgaarqb)m68{!bxHWVFUNK3zDV~B5uK~9VM&Q6vre4Dbp0~4c_+Q43zFns zH-%$Zi+|@!Q6>Ga+grBgu$eEg#-u+NA+fasX zl>7Co2l3l>6r1nu%fRM)pg(N~ruV2pf<*#`*dBsoAL-Am_vP@Y=f1C{zp2h4iM_4b z(K|nqOOIT`>ivyCt9#-{yLNce(uy0>jp%i8x;U9#ihq0EG3WSS;gOKb<8nW-M*7!x zWP~ZM%Und`)uV8czOVq_;Y|^ zBii;TVA9W64wlc}%FPxm3DZNC%=3;amEME$>-aV29V*f#)7f(};y<Xuv+Oq=vdx`uG%RyKK4?S8YxQ-fMiUpyp6AOKjNPBcO~9>0G0Y(zU6NW zU}6%>8iy&Oer^XE>m@SQ`R+l?oCPe!raul}^&i3cZiMeMql;^`)1(em+6XusTs zk9o;dTW!tXefMC6oCW7h>%}fz3}~n^A8+H_xxM8q1itCd`1!9E-_3bC#THF`i*2z48>2b`Peg!3Y>)oNDsd?iudRa0Z6*HIIR=OJlA|~wn$4qz)9_oJaBWkBc0)bj zckVij=3hb2%@dI@XgOSW`l2ByO1|zzyl!;o{S_bZc*Zp}i=Bz6(!pHkpUFSJ3^Dvn zGwN>{AvVo%BmYjon$DR~lW-6w! zr@Y(uVz1^tG?3nW?aUIm4j)KGwTbvR<3Ege7luLO2XgO}Xoh$j@cVzYQ0?E5i!&W~ zaJ$UHwAm|j9{Ma)5h7nK5bkG!S@!EM%D?_Vn9kED2!op;@?JJqwjLoA_>9 zBl#hxqBz<|YG>~}SXO?{H@?RZ8jbMh^8dU=f0N( zQ~EwQF7*5)L*ctAbu~tcmCao7Vbg!;^SA*9CL6@B_6|IO4On%06d#XGlV?I6+ivf} z68$ft$#M^tzkZ0~7ZEgj6d}DnR@ApxffaUx7;L0Ux2eW7zIRG|-mS`z4P}Tf31RDh z1^5)>NaOxeM^^Dc=*&@!jFY+f`-W_?s0Di;%j0cRS01pCS+`x}M1v?g`kCO^KwG+P zYQZ(@B-bU_9_`fBSf48QD9@yppnXWxeskfO;E!TLdNll{-gfSkAMW^{^R-rQ$HvO% zxYO>o%#sqr#Zl}Z)tOtXJK*ByoA7D=7>3=hVp3TedripY38f3Z_P_Gz8Q+cDzxRUP za3iKaRp51$mxHEEB^Q=LhEBi3@SqR?@kZce| zC}V2B;n=717<-@uBk%OU&xAJIaYVBERt#e6mzyzp`Y~kJIWiRknAU!=7-f~rH76=D zcAyU)Ta4jpujvT8*(jOShWK=JC4!yFjekzy#k>dTH?S#p;;HBrpU&xe`Skwt9lDK; zShwUin)Q1wj;C!CE6w{tDb|$hVg@#t#tEY6Fgu9b?N^Br;3-hjWo{zL32b8w*a!zJ{9PDCU8vif1y z@+)H5;Nv2wej83^=Ch#AL)>oAf^jFAe~XM3O|%vv-z$q&TV{yyfuAs?+MUO@N3;LE z>rm`=#;WXHmw|DBH&!=o^zAeMiy%A`A$CQz)Ls)oq3);)wceg-I z{`Xpmaid*nn&`_*F@>}bYJ+q83TSgMSZtPjd)=4j)IOWX4b3*fF|N0`Jhm-QM_Kah zWOW`r-H+Gi6(hD^AGSRp|Gk>a==VpRSr)qd6av)q(;_m~+^Ra_VLm3yIfQ+mQa z7htvNFrIem1*1u;vHom4Uq$Coc~>SE{gC&ctOUj$ZO<=v7og#t2G>|lz~)WDZA{BpS!M@bmFAC|PgI&No z>?;{!&rAQpDOup7vzb`YTaz|PD==@40iFdVFaX0CZKlIim25inS%NhdE_}CTBd}mA zx*>%*-dWtTR)fE*euxe7JgF()3-z6O9AzBG=u0bMx^sr;Jl&EbTY~BThBCQ#6()Uf zVd01XUYOgCwVSqM=AU3Ldm#DFGN1M)(2nI(cVfb_X8inftC%>oGoL>&;sePNyZNaF zdbB(sdb(HPPgGO--(QQ=on2_x*c??!Gw?IHrF?x0_W$I=PT@Kn8Pbzo7ev!yqueR` zbVc~c*%-I5B}+UK=~UZ_v$v|CZOTkkNoMca@%bG1xv%W=xQdX49}uJaR5&hnV}3@4 zXt7Z~)@;@y#iS6ucDLgiS4WONn8DV6?U-tn!kW+i^w#`@3!esa)JrGs*c!<%I`>80 zt|pAVqsNC*5Bl<>LHxRLPq;acpz4@&7?Y+W8C)xI@?1Eld-h|j-2FcrVa~PXZtOdE zq|6&dbNgZs+;MHoTXvoK&3U%mZ@R%L=_m#z+wo!NV;G>+5w1@zDhj4+ibvxlOLkUM z-Yb&K`M_6~1Z&YXMJiZK&{*hY1ln7CW=nk5r^ zT-Gav{)#~6ryNDf^JIp(NS17+D%Vuhp|F! z0oL`a7cT$Wvf@k}pTCObn88~xta%(Bd5+`&@AVkHPK~y0d$Y*@jyUA+A5ahlk(ZiQ4QZ zig~RH`SH2n>1QkP)XbZWDI4)LS09hRXo;y~X5yEv^j>LCM?>2NC^n>1`@}7DEE~xF zPg-K6{LUVJY{NBUKZ}2!t#EX}Vw?>q7qM?gaZOPRrf-qYvAP6aR@x=L+#Sk=((itJ zVZNNH(lPK`9(CjyqtWfCP|3fFhF|AJqx>$Zwut5BgM+YZcQdGZpT(i2o=m9q7p-@E zMOyJ$B;MPCb$Kzo+TuS<(VK~uZB`;IpbK|iFBVhf^C|sb9$QV(XHmvF+@K3?U(A@|EwTho_P!i`$#@d172gdUx@{o4+1> zW?%z@l!1KzUl~-M3=-aLx^nC`UuqqUqUM{XTz)c_h5Gjpb1|1kT8x3$>@F;_Ov3yb z)p+@M6pQM+aLeLJqWRH&JZ=%d^q8)k7WhNBZLJq+a^5n%+JQ49ujRqm0{RpN;G$$m zw_lhnf;Xihr0ErO`7ZCF!U?Zj>u{q|kDpazxN1)qamW0DIAdZ@gJD*vmi`+P#R#@B zY0a8-OEGEKP#T;YLys?UJl9O#N0owjf4H)o%{y|+${w7eA$jLNjN#iDkKFVa8sF&x z3rauhwpvWQmQT+dOWM4tQEZ$N$ZL-dVaq!Uz8(D+MSOt8k-rt|qatbE>>tKE>+wSR zX7QyU9ol6fl2J2NJhialw9SQF-_C$tj>WNmvL$o9ZwTXaZ!qkk+{Nu4uUHX29wWLo zqvhL`FfWbamNS9k{I7k8jZn}^^B}x_gyLw{9mR>-UOe1RGNsavDJ)j^k{MVv>Yub= zP!id8fEP2j`s2s6Dv6Z@Fh)#X3*&oB;P2IvvySM(dYm)6Z25%UI~;ts=pDl03JqFWCer%2 z8?0o4VTfA}%hOu$&SXDE$n4aH*dA;gqE4Imc=qWa^^5QMaR2&M(XdjRU31;}ah@&B zR}bZpUZ)gZpEqIPzp)(nNSp81`Eri!7o-)<$A?{=VSZbWePrHXM7H#LNWCsY_7y~4 zEQ6`kyy|YYV6noUn+p3etuct^*$SRdYKg$PN_;l&GR_UsF+_2YZ)`Ll+i_ zm_2?x{2C=)S%jDJ-1(B$-6u)vlH2OFAxSvgqN5HKttE(( zy1DZC#b`FAKaU2DXM4v;(Wdi!I4zfc(}IEgEx&YMGOMt0oCch0oA8y+1T5TPCfsW0 zBIsN-O4^2SVND3bHc4iLd=^Z#&f*)H>B~;|iedkBrB|RAPneZp*0~bVs$>QxZs{r( zU6~?QB?fcn>o+3oKpk!-r*p2hF2e)+aMR{Tu=?pB^{`%0>so=670t0JWdX)jO5gV% z9iPidC(z{DW!#@U7InL2-@xiZzIm`hQM^P4E%zmeyRl!fQu_An`Wf*0@|j{qNn1vz zj^@8%mtp&_jR-G&=sR>u4(|*J;locxqUY9;eEei8ma4_@Tj66|dVdS;Yqz4|_hc-a zq>6x-jfkt%v6bKWWcv{_(_dw15ye;qhExLyCLXPysRL0Y= z%T#E-UxxjiYejL%9C1r>;>@J>rrrG}+|up&+NCXfylh2-gbu7KDT1b-3SUI2QT@hz zgnn@1r5>Ab@oOvI4-!y%lF02UvsX=s`xcYu=)D&Fy_>N8vkZI-j%L8Tq3CqK4(r#gl$~wvjCSZN z(vm-6`>~PKYws)l6>7|t?2LdZs=O$(4~{clA?9y?sI?Tb2gZ_Jj&|h4bMufnRB}O- z6R3X8l{qp;H>PDF9=)H5nXM&1X>|`q?e>F)@|Mq&|2r+L9C6H{rAJi}02V z1<$*kq>;idpenOA&fCMEa>&8IaG2pS)1`ua%GMq~a-J2p4>I7xO~W|4VkthpmYk7*?rgtvsCX!QSEifI!)^Tl zPP*_OtxpcXPx;*Gl#|T1p^j)`t;{BMtI;qqn3raFa@OH6T1(yFWxeFy?Ytl^|I(*n zb{IasxCL9YJ>s6S{92sh7-EgEu2V`8?cg&m&=CE>{F*^Q@&NmM9zZx%L{2tC);-b^%Oj z{TQd8l;fH7Y*jt%%=2YZ=a(F1v!w56a@&thw#D7$w2Rx*Rz zj)Aj@HM_YEXVZi7XHNbH6WP!8$KH@$X0_1kTnnEW1y~qh#g?U?agiEOJ7XHF;7JXm!W~WEJ;U9cQ zBy33)!Q(Y~>)>6)A2ATYDJO9q&McE%LeDK5@LsZy<{0GDW2_%HUWtlr;@vv_eC&5qDOW)9dR^xg04G^l6O!eW%k(QIhN}mkAk19d0->=1`!#T{V z%VEnI%@j)&cciu%jQAzV$Zq}y$__(#v$gD#JgAG*!Vai7(Uu+qWY5mIcv}0-MyzDP zcntW0%}alXj^=W98(W01n&+q(Y{Lz|m!XYqHusE4#e%b*&vzap=a%&JXZT(h!nXbkFmpIXdMATAz>LmX3XCAorgypFYFmcHL+`bcOh{ zX&@tC??BD_ccO<{lw<*_N^Sc&7PL4cwS38%HSWeZr8uURORnfVS01U4l3bYMi23gj zPWB#)gKBlyxZw!+$zL+B?Zvfk8f;#<2`kDcz{*5s@IJO>R?=pq9bFBTwVOq@O_J}I zj2x!AXfoRH1k?stNng7z_cm+8T@gPKaxQ`gCD*rJEb-@kFs( z?U(5IxI(P(EMV*l$-n5Ez==QPY?@-idt=ST2E*2@cp1qBjps$P5w{d`J9)6`dJ&v2 zOQ!dd-y(AS0B&EF%Z^YJKJ$mm`FsvsKg$_#t`?6h8pnN!KCD+ChrfH|U3S|ZJdL)% zd82`RZQF$BrAMfx>l*Q*pEmQ)>N0Oul;~NT!()enxNzrSEEcw0dc}uh`{?qa>?*i_ zLH3z-k=YT+lxtpIgNS3vi1kM8Kg1` zZ~F`orUm=Oh&8P+ecm6K?ir0k|39)aW{v8Y8d5Q_hg5 zp5=3RwK4O8Wp-RA5JN1R@#KV3?5y#}p-0!y>Qa@smwf{L4lPH8x+YCyo1rF%AAE6;9PvG&+$pta`-&LGs69sN6>CnGbGbbX z>D@bp1-nP{r_VfbZbF6d(7c3=a@Kr#awP@?U4_;84=9)27iR{K;kPTh#e=#QC{J|Y zPW5Ht%J4wRTfKs97w#!~wRU6U{$PIhYs#+%vZt@P{6008jJ8EjeUme;z#(if64h@Z ztjvJhD=QS;bqv_)v%oN$DD2CcfTQ!Wc(#58@-mI+uWZ156SraI)Z;i*zZ^SPW>T+9 zGA7>`%_ADNa^JT?Z2x)#Qlf!xaFQq+(}AHPkf(F9d48SjvHAdvaJ~bpzs_8C%L&D5 zlX0Vy?DM<)7;jg~`>)|n)Xg!L{Zq0B?O=cO^^`qb?G0JJy(zB`JTBUVnJM(=jKrT8 zk_olppCVy!EbrcL!}-!-Xw@kjLDzfIElye9K_=kC?tF$n&0?~jG3PBgq$vGu zPqnhP{OKgw5K(r_|M@|7^Ss4NZ6!MIa%cPaQJf|Ha(5OQ@Z|yN`&$+cC$H~tGLp>J zE60)j+L1F|PKu>!^Rcp8GJxMTWn!<{h}znj57j*hf3Y{8zGgckB{ zUf5CUXPPgebZ;7hI-C%rUudJZc!|>2WB-kTdz1`PP&bOIpHxy8*q@PKu`|8n9>cT6igiFynb6j>mh@WI-Zx8vF8T zU4LdI>(IsBQbbPbPp47M_^!)v8lU=y-M_jp%p{1xp7({B^iWT4vH<#ne+c)gCRlAZ zj1P}Iz#WS_B699A@okR+cc%D=o%-Fm!hM|Nq9ya;LzzL)`3C!gT{(T4kx-I+$#YBE zGk=sTx3;yVx@7&Y{nrc=3;M9%uN9bna6T4xtjB=(=_qV*8LOTy6ZR&}Id;@VT(cg} zeMjb)~KLAvpaT!W+{2vSN-mFSi&hUj1jrwF^rXzpt9GGB+OA%g1rQ+gS9u zB7gt#1E~3o9P`GH?)S&B&E4Z@e%OhfE@vo)^qY$fbDOjLV<&!QBCuOeahvY<)ZXH(USbPas*p1-YD}1Sxnq)Cig7aJU#0-93;=fIe)6$H|Mf* zoi#Px+7Y8hi0!Kb(0F_a6p~*SIcy#5R*y$;y#aT>uw~;HT`FA(RU|?Bfwz9gpT$*B zydT0)^%%qy<O7v&R^D8YVT?lOu{`u}3zApaFut3< zdM(F94aM3m7 zf#G?)xn#NGeyTsrZ;g~)va67LZUwXqx^r7EZQeQ@$$Q6xnJ#l7K?iJErreA9mw&>~ zRF!&5&SFPW8Vq;((t2|MxW*D>9wXmn;w={xa#T0g1|3g^G_NZv7%xq5#)$w5$M<+X8iH7v%wmoHFJ zJCZN-QW!l;Re?e=} z!AWv)pB%)@XK9@4HCOSZ)h0N0-zpAI4q%(ZMTlwMlx1#n6gRcA6{`j=!~z=^1~W-! zcE_@MvA5!~vl-7w9)ZF4G_+Zz#4DC#SQeq+B==c`t;}e9~V*LA~_5mefVB0fxCMTW2W5w8V^2!iw<%p z)l`=o;^k-NH%Isu8^JQJldoS>EqcmMgFkUkaHL%i2HvQ|+0jN!oKoi7R(4C3YmA`7 z7b7$YR%2R&5*_UC3xhK;Xt*IW_z#hJ~x_9UF?Yddh$pM2go?#;@d>74a022Cfo{Qnun<{xudencyj-~*aPwxYdc*Omt4 z)2!EY#l)4iV(h$f+?Xl(nOkBdzd4X)L5E z?6mp%02ckQVb{Q=C{W5^;@7P(y4sEPj)Az>pv55zlc_)65eY8$74~Y0aGxN1isz?_ zn_Y9c?7Rid!+*i=>PHxLE5oLTDG2PWi9N?;WHW3Ef;@)^2oy|F;h~+Ws99WRL8-rq|Hph~P+@L7a0v ziN-Fm+>_Xz&2(i3bBivml#ifQ`e<(Y)}KMrbKWt@j-&gX6rU?E;qNCQ`;opVoKz-b zwof1%*2>NtX%%SrY{Fq7l8JA53ZolZ@pQZkZPLAANq<~AM?0WH~ctc9F8y7PG2djv*Wuw&3^ z{Qi@{OX{=mX68~1oH+wa^Dkg}@J0kYaN;ScPc6y1AnrL2g>dnp;c6`g_n3t2Z%Hh5 zlXLcm3;5C3jGHgI<3@5f;UJmdz8c!xBXt0q>UwB)aAo_!tvGME84NBh#RZRBV#w8I zJiSvgR^_agDCUUODmTT<=ewlF4y0ZPlKU2a4m~nK(JgK-Dz)}vw>XQnNpf%es|PmJ zNPRC{ekbmJMM-o5d*!{tn-fNCYW55vdb(`)wi%~5j-b8g2%grSg$C~w4j)@2rW@$- z$D;lCuW1~u4h%+li!WH;B!O$>ndLg&j;o9Y!7-yFPrvHP+3O3rruZQ=GwNV61F}Q- z5ZZf~!uIqfXmxoaLWg}2`E|0xF5(WJ*ent2&0C3UOop4}td9)rD_-A~`NP}Ycr;)c z95+g)hEFwo4F|IK&fe5sIUf~rC!RigJPW0kTqL>8QCeN-Io%EI#;y@v)uZ7u(wHx< z)!-`}*#BFZ^qGCf^^0*F_tBRn#_LdLR*$P1>Rdi$JBANv#qzB!MPIXa$h)W^wd)*q z_#8{CPVqFa>&}w-$Br8 zXC?|IgC(=#v$$%h#Q@Jpt_hQSdC7SC^23!GyX7uF{fEN1UldDA=b=>U##f4cg@)xp z6qLWkK+}F~=8?ryD(z_<<-oh{Z!p0x2(icfc*5h27Hz`)7g{tdOW@FR(u=u$iQN4dG4K9h7ETFa zVjjPJ?E?-lGF z?LnvZrI5bR|kZm++B~EMv-(I6V93wp`1FTv#4LI!pR3i71vD0QfsW#-P`q+^IQv_ zva*4h^mrIU$9mfe;E%b-&&szLhExB^W9oeMU3i#Cx zlHC9wVc}ke5ZTGMT;&il+bQwq^9uBTY`_^2Q^kJ~*3?R02iqI}Ywt^-dR)7BqmUso zrIaEhv*yzO-9Lm3X)uMPl!!#lp(y6c?vE#A+&|9js18Q$mFy8@Q&O@xEEL$_P#!DZ84nh-1MWL)1wjdJeM zoglHVvAQho*L_Pa14Zq;T3e`v(?N3j+z~^TT~pS)5Z^g6?$D%K4H0=)C(aC*(})LC zX!dq-@8xk*I{siQ-Ak#B&>LM5A#(i0Wj&E}X9MLl>V|l~k`(M{ghEcsXh@&}8gf}V z<5Gtnl>SOZW8R2+u3a%g^m&cFC-$Qv`{Ck@mU z*A10kH-}L6*^c!1Xf}1LZ-&KAf0y&Gv|#RF71;FaQ- z=|JJhC#iSy9`wFkecYTqn-E)<*TZk;r*&g_$0Q=+o9MGENGV8IJXR(Z=F9VfG6OOY_CX!F7>Q-Vl$9^}t9kFFOCOB8K>v z!o=X#czAmx3ci}cIM{-Qx9)(LbtSR7Q#c;Z5xrDz8{y;IX%sLrpW2klqJ^Eu)AsY* zmGosDFePUgx`}W4d}QBSI-W|rywsfO%VO+2GoFYemy*X zTNQ;)d*VZNU)mK^f=4pTsQu7}sA+bYKKoZgV)J;|Zs>qsE``y^xV-4E5KL=xt6&0&ewPJ< zDduE6UW*>^gDRL)*qUBQ|FiB=D_@taN zzo68pUG8L!_@QFY@$+-~ zw9%Z7%%4g*mS?EsyPae*tp>iXx=W9}#TooL$0>H8#Pnap9+Ro~Cj3(AbFU43Ts}>lhdWF6OkA*UUoku$G@Uj_ zTp{nRP4Oh*ciQFF0P6$A`Ov@)*zwH`J&e{;LJ?PVPA`j)I3G-j?u+ngnYtM>k`b2I z6!t!gb(jAr1CR6xO7hWEx_B!Y4#gYc#O!s-h~xDrx#ANF=w1aQ%Zan3O%lnbbF8TI zeu)yAR>NnP`83mPKh;mqqXyzEM}nx~^<>aJ3f`7V*S^)lQ0EyGW@z9Z=%1) z`)5i}(nzcq>)_Z11&aNkwm2SDMAQ{LPBrg~^JRsTQEf*Eg`Fw_gV;nG+v<)kYC~x( zFPpAJJ%|xC?K_aor;>P5x-=?<#o@epXWZPRYx zt44XjTAbG~?68i4_NR;1p>32tdm52%(PYt!&zy#qv?hDQZ{%(*=8*g9(6?RpXieF# zv}5x#YUTP|w>HrkCtti!_J<#&Q5E`7%3BZW-=Z((F0M#V<}Rl2)xH=v<~zlXIiL&Z zHI{0$DvjzNTH=GZIBT)Lu;*Kc8B|@I?KztMzMC zXW54iXBR`xr^&QxSuVABZB5@-i(KePm|46vjrFWS1?~nES!X&G%y7r0T}9~8g%-Gf zcde+wvPKDyM53(WedfGT%@Cm zZK#q!P3OzUp~0#VD9A62qdl#ueb2iT^Gw{`{+LNUn&c?f_Fky_ygUth zHbm5Yt3@N4E>!Z5_rl~2w`s$v08)~H>ZglQ^koC2in`v{7xhw1o{AcDZAa4~M{m^M z6iaQZc1MqMxyrafI^fE6IwilF7 zZWEQS%O;3=zrQN>XNsZp&0Dl)@S&G2cE6;b&&h%sZujhi+6z|PDG1-lwy8RssvsPXgU zMmP9v4X1f;#GZpuA2iwXT=6ZPOM#n))4AXZc@whwkT9Njhc)eOAGt8!Bvw@lznF@nL1QP(C(fz z(ZvEC|1gEk>H0K6RH_D!gL(sT+Klr4VL&3G_G}uziY4YbO4d&NW!Y$j-l(D{u znlqj5p0&s5=MmU6=mdG37rpT|i26&#tYMyEhsRCK5wSS{X_n&d+O$qsnfsj_T^pmD zr59SPD@xTbom3WlK2Dn-e5V65g3xqe9p$jDK3u=-BPDt;j@7D*4Pg((K3FSU>{=X! zVmeWlou@c2cS^}VY^1al-^n{a6|ocR0*JlYwKoP*ps1TN+qEO^WccD(vq(BF4m}SH zD2YSeYE#k1X-ZUGW9pFentIsg(Z^a{5gBHSPNAJJX6JIHZPOZfKd&mDhX1bkJt7<` zw3|K~tfk}67m<77Guq#;65LW~|2t75>DxC739?1+tOf|H z-VK}c#?j{(ZydX4fs{(sXtwSml^!PU?)S4t@QP35SR@SlMBRs%75<>$^{w&cwGYOR zx<`$xl*QvI;$B_aGsW9-G}&zRhv7|MT<#<8FTPqvX>C_2cA^eP)z?utw)hLhRvw9W z8y?X0mnW1C=@pQ;#R~fdR>sG@2g$2QA&fgWnlkNeal+^{6*}4rM{5cno!5$@A0AQ4 z)e(IhYGu)@`=ZX_@K2sTJ7sgw?Pm6aNvj}?Z<&CJ39IOCQ)7DQaF_h9_eQBj=fxe;nslm1 zH>@`-M2i?Vq zP_sVn)VI|=?p7aj9H!Iw-Z3=XxjZaxm&dddp=fZGaAc0?t7$L#SMD`LeLHV_8)A{oODnr`pu>d`G~qCTdo!)4$nI6-%((j!Kg5<6Ya|I5qqRBXz7z*XvfKwWF0vOU7i>sWkoEW zJiMt4ORfNu?82Da@s+3@UzyezyVJqbi7 z^JHP%%4mXFVy|-kq!DN=&Y>91uPx3%+lxKG<;t7+e(*lFo7`jjQoF{YFWHy-w6s*F zZq^`U@>|`LuBT_wxF=SW^-0wK9A}L(T}3ZjPf`xGnxP~)8Q`v|=+EfskI<!p?Sp>iVZ7ql1am<04_Wva|NbcUAn8i8qjPEh^S-f-|J3M0|GHl&j; zdXz9j|N7j*$)JlRwisv8 z+%?TmWKcMKBE|k`h1pahB^gii*Xbff&C9Rl#hs7mC&*#uH^r%W7+PF!0$bOn__jEk z?wJQ*O@-!Ya?6i=_a0JS+gal6UL#~30%mz6iZfpUI5(pNPIYh>=iR2zqTszsf=zcA z)$2@#4^Hczjtij?!<{ia3#e_<5%;R4Q0x~&8h6D)chWnMvRg+|v!%5#V8c!FCu0na zd`br0{V?@r0uElS3|gD4Ox<4w57!0Z57z@UZqOmBWl$d8&0Of~phV0Rb0ysD0*gQ| zbotGYEb63D$Qn;9xe$fvo~P)Hy$QVY3*l|554>$+ME(ED-`9}#+;mZH4y%bv%S656%@Zj)XdHb#=81{zN5JSyH@xdU6ca?x50|s`QKF3l zo`$Zc(2&`p=5rfVuF@0{W<4>k-vRo)Y$vz}dcxqh*<{_$9vj>;D56?@Jb2aves;q| zUB}Hd`|@dOBuj3Wi#(8^tBiqUFyQ^)hjv~+!6IWJ}Q3Kn^T@eGfcBLRd(3K zVyUe=#*A=A)%K!?>6HmoWBd)8KEDgn2xfgPQ|IJ$G9_JgwqM)2qj0LOW3}W~(tBUsndF2ZbnmDvH`YnU$30m1mIe)L^V` z;z+g5?4nOfO^iFch~C{3wU{?_!MPkeC8uix1c*CG<%adc_ZiXX_G=xw=mr$u5QN(= ztBV?t(WqGC7sV`e5?yY(m-4rX`-dOIna9}|sB4ejSk)twQksal6C#yaRx_1JPCIG& zj#g-Nc_3ML*P&t87&2S4kzR_r0zS6a>CKLoI4JJXEs7TXV@1!D-AzPa-1oImxkn0J zyeDd0Y}luy56Y&_mU*J)j5~anS67akI^boc-b!lRak{kB0LdTM&>vgwD77t$)4BQm zU_9RiU7F0KB`>@%`k}ac{Jj|rMeSVA7iDnqmcP#Wjt6B#1|wVCiOan-jsm~c(Cz4L zgWZ$b6Oqoi9a^5MA3dP#t!R(t;;h8xf>;c`UmCOWex--^=Th^eBeX330CibkSL~}6 zr5oZq$U)orx;9%x4V?vq!CU$%HSk&K8GVaJn)C!Y6r+mXg;2gl8!~$DN~v+%mBD74 z$;rkIom@&#t(M&|ARrJi#^Jag-WV3sF466+wGgv6%d?W>8pR^`Bh{PJ2p7wiM`v*^ zGirP((&@yScG+QSR@MZYa{i#v(-p)_ct~L{yr@h$56b(zNl~_xN2U2yDfx|*#l;c8sO2(Th!XN85~-OTw2v^PR-i?Sd&Z z!NeQS`{|GxGLddHv!^xWrZgDvk?yt-cM>YK#^{UUe8VlV4>>v(x?{lJh}zwGf^3yN0y%R93Cn9A%qUY z%AMsYK2(Q&4FTUy7d&&Etgvf!C_EqRRL*2aP}z(1@&4W-va9owuC4Bi6d!9mzj{~M z-l;MyL9{$Cjgn&GQT(6=#zZO|9uF41ye=w5n>nCR zSsPJ@uRPq%c2bhjD>|H+P8}l}((VjXnk#B%3~zgr-b}NF=>l_nD;$H+I7eLE8$tEZ zjTRSo!_Y&9NbLQX_Fo&06QWMjg29Hc_&5|E8RzN4n>?k}p-6F0Zy1K!deQU)6X@~S zlXTl+H`Ns9-s|;=$LQ9jXvoeWq(GgZT}hb9^~`-04JC+q+YfxB{him(DPG8jKPK&&lrDdgbJ< zOl4gDdvbmeAZnWz$5X>SN}gXK?E6+6Q_6iM!;F4d8lFS@z2d2pe|H=gXM3|PqR4a8 zc-`g;qQ>dOXSDF?NHn!?iYn(sjo*e15p6aMS85(5bJ6!TyJS}sEC@u0Bu^NxeN4go zUnpDWET)g09Z*%5LHEZe)AdeAXwNtU^0<;i&%_<7-QqmzxZ+-LJUC04Gi4DC5%rTy z{ie{_yACLOts^}gEoyYH4#LY3Ls0MJ3fiQz!#0bVboSmL)O)8x`>`u1&h!#xh1Y?R zt2y$;y!d6F6ZR;L;A=kwb$#cNLuOfwo->CY=iZ=A^&XK=rV}3azDZ@DHuOARcQ3UT zIXu(!rSiPX6?*ohG!ka_r=#NB?LB3Uu7qPC-kX1+XMW-w?wp!1ZGD9b|DH;2UxjZ) ziGF;#Jf+j;cv0)HG*%RyL;FmsC`LI!*pfa&cjsIyls7lQ5Z8RA#L#teQ2|Be>$C1o6?=@!_ay^Z7r9KUM6NJcu1)7HG?wV?Vvre zZBh5qBwD>79!-bYk_mNz(yloo-yNeg;q#f9ixmGI#uRqtmTutIp*UTkKXyBLV!_G1 zRIZ&k=XbUPGQ_#n7taeLDph>r3@D2EX2a-xjdQer;6CbXm8eu*)SOIa&Q{`AWYCG8 zH4#wU4{Kf;;Z@C(;@tmw>TEPgS2v@lf`KtgJ5d{?<%K}}=K4yvtbypq4ZkIqLj67tNcFVCA3cWPaR#uk=0wGBmoJ_e ziu$k<$CLB%D-`A56E2lRt=l&BaMHzuI(4zZmhAS(KNO4TA=OdD?m8X#G>Pi>7!2>! z04lS@367pN_>>cd>FatcA18!hS4uE03@(Xvwk>qWg6g2n&^0vu%TlW0c$Z~XaN7LF+?g-yX7Cl*e zInnfn;=Zv_Pvj@3Dl_cNabMhnH;o^L%+W3gn2?Olb|Z0T=@nZ0)fr>gR=|i{8_?J! zYVxE!S@#m>sl@HN^TV&}T;0UEuz}_#%4C?)qMy1u+K*nO_?!^6ko!BqZ2bzlFt0Y|e9og@21dyB>xbR<*XyP&6@4Fz*TeXS zMQBIgr{s2}F=_>b(%IAV>HDp&BQai_Q~My!gMY7hn|hzm zQM!g2(3~gll~Lo3VANdn@x0y?cUN?Q*WF%pYF|^VK0Qa-e73D}CTJwW{om2Kj2ycC zRn)z_dW*LIS`R0di~Y(nLos{16+Q7!q1t;UD&JauqpS_yNGcSG*70uexEx2iHbkK# z7m;W0D9kmlf?Fr2D)w$i>C8({(f_Rt_FN00T<765e)U#S8&LEs5qa&5Jq(sD~+&Cq1LxV(-A^>2>8kp&b~(@gZK6ul8g z+hOe7H_XgK>HJ;W*Yjg3vJ%0x!#0mR; zzDm>b8zLyQF$Pwu2iHR<=A6*^q7TVFFQ)#qy zp%q4W{7T_VKGRjR(Mrcj4)|seuk76@>NPgV)_rT3O4I(ROra}U;_mJw968VcNuoY8 zni-LWxSud9Z7^1i9EnVEX6VMRCCSsPAIi2UiMnUIqQt7on6bPHUZN^ZIZzlk9=c=Q zPSGnTd#ciKdnpWU(-z|@UZlt8t6<>!Ug&&jBZVgo!?4vZ$Y}15vkQID$S03VU2#Ok zhnCdY-%U5KaTSy_c2oSt8SI=9HL&Js2r87bLE8Kt;ylj+`Xkv0jYThFk0n;fJy!_5 zp5IdLjIWE&r)?2u*bZ&2d|*5*hx)ZN!oUt8C@8s+`VY-g_K3TR_V-3mdbX&UJhl)e z_Pjx%I$&w^Fg(pUP8BTsqRq@)`mlLCxt_kR zWOoijYcap~99x_U`roGxc1_`C*930HHz{v$QJnbl=iamN3R8@9bnp~5!}sGeV3J)Ob`CLo8Cs?tB|6A8Uvw1>MnR z_+9FnwU8D)83N}^(bPlKE-mq(EQ*aW!>J`n*kU2>ab63B*}|vv=KEthHT(iSit0^c z#@|p1n}pNom08rTN_+SR48!z!lgQ7kFdf~VD%Pt{$T-Fjzdmb)C$;)upV%*)Std@@ zDKNqCt7ZtOA-)4H^+Of=&x+Fa2^B1;MPI_JDxWs>#*j;|m8zeIW83wmx|+ReV`x_o z+T+%k))x`|;%sxNr+*JrjI*Q8PRZytz64V1Wza9h&(OxPIn;F6HG=4inYY&oO&^H< zu4%uL`Ek*oM(k}|zQ0TK94>=rjY1Idy)4297ss)ZT@~k>Uz8Ms3$*M)38a-gO_OtG z=zPQ;XR#AyDC^=7YG^o$+U;6I?Y>vRhv)qiml-9g%gE=-r*4A?jVq(pnM9a9D~@DQ z^U1iksJqinoI}e3qQ@avwLA z7Ju!G$fs+mXuYn;h)%|*1CuEG)Dk+eDFSCNZPiuF_)N#`v&g9VMsZ)MBl35aL24mS zWR6)a&eEBnMeXjG^!PEgz26z@BaTwHtb5Asef6l{n|SQm)(ItUh(3Vgn@IH+jc__E zoxa@&Q0{EdA^XZMGJjD6c54TcYsOs)ez;Veso73F4;4jvyE0hXwFS%@wm@j1H0tuL z0gYNT6mDmV;=-LU{Boxw9bVa;?%FoODc=y-?-b{790L$mtT_($v4C!4Em}F}iEdM2 z6Xc2BVQ&vvQJHOG4_xdQ7Pl9@bHseAtjBOTw-(HcPB_`s8^!v0!?$b&-M#moG&L(l_j-66M3f6A_tdde^l2B}ad!g@ zE#*!NX15^wwqNOSehihdjH0uL3SoTB>PY(>j_K`0-@eGUX!hn7y{uK6k}^u-UDcIz zEXEFd%DPeW=g}x0Y=r2{v&!?s{lvE(W2|~A?zM|`@#T`GU?sjmxsRzP&Y<_f;^`A9 z>{=mgJXsTS>a>N6embM>=a=1Ou*E<=SL}!$RY&02kfP!|(teuY7=~724>W$kSjERc)Z9IEjC5TZ!fIuE znC-O0Hn%3YX**GOJpCx0&K7gJ)>fjYjEK(`qITP!Ksfp@rfS8#vGi?CEZ7@Hx{B2? z*}V{w#N5kb(+%O}O;sjr>_e;W zED__s72ZYW()@`I7}qP64y1IW%vZsRx5j6F}lekC(@HtQKzWbJ$mb3 zjTTznQxbZYgU^>*w02@7=9jLG4yDBRmS+KYKfOFv4J=FHh6yP6xQI5)`Q&lqktgcv zgihAHq2xRl%Gxqo^jHXkTgeVsy4)6hHs7Zl(UrD$=e<-v#~MN2^>KWuGaTDnBPn7E zz3Q1PYj`4X&;KKS3{tuS8frDfJ#^dVv-}o3VtZ#gL z#9xqZ?*9!Zlf%*Nzv24CMnpva_0PuX~g9MJ>vcI@X?5lxee^%X7 zeK64Qr@}cq{kNVOj{Q${15JD|U4C-buVvT!u&B^7uCQrd$8ZTsSe0+%oWp>B{mm%fXmkB>H1}Fg}^QjE~`* zRrb{f1E$7s9uhxbeEbU+N&L$AQ`M)5BgVo{E_&_8e&r^i%UD=vH$6ZW4f*qIruHp{n_!u)mdL2XUwwrkm^t3_p5t)94%z&G^~ZLTBa7@=hlGoS|*lx0Mi{V&*t`fgzIHv2U1{}vZ%d0=H zF+L65ALiA>FWZgfaFF-`(`Eh1#u<(N)bYq&WnXzow%Ww{o@k1uBY@Ns6#80yIstlj%RvP_j z?565ZHh=uhEJAvSq`?Zvn0-04#uobvo(5QIL0TN z|FNDm@=D@}sd0QbN%&*EFkRU^k>zE-@=yigcw>AFC$k&N!JNwM!FpkN-6e8p;8cH+ ztwWhE%PX5dGd||;r`G|hVqa{mMmR4-K3eZ$CSKmgONCe=1qIys96_)&Y!<n|><+SCWl zJ`&3-i*x2rV-LwZjPn}nS+@RQ?)gcUhnOzI$^44nYUoP*h56%O$ox-6SLRpj2aL~A zZ$4l++4_zBp5@R#-(`I661vPE+xKVVT>rX^{a!Y2VmX*U**ML5X1~(E-e)*jTrxF= z({Eqavn(!Ua{O#Qm96t+`1FsLtY?}2Wcb`9^9_b$e6A8W)*nCVr_1=%>7-^{;CN$u z$oOMFV0(C|X-a+2tmBzK**u%&)xb&OgW;I&KiHSYX-WQLd>n5-?WYoa%)N#`i5#py zwukJyp9W4H_htM18aP#6HJxhg%lU;lmE|GE$8zY850*ogCuR6#-wkAN&guk^{ffC~ zyUF}d2IusXc5{%#nJ|dv)K;T}=r3 z_3Wkw9K&hos`wlvde-Pqh0{L|)5t5q$NFPGknM{y96!mv3-Mc~D_g&@y!;FO>wSik z*+ZijHGXCJUS>BLKIV`0qTlZ|ekzGK4PBK#)m=3Ch2dC#vVBaJm-XT-S?@C(^XD${ z0~uWzK89nuKRsSb^diewvV9e%%X*Q0A7p$iuai0<;&_w6{lmDR=BuC0yRvbC?ZI-$ z;+(b1c}O;2V|*;HZ2yJz$Czd76ozBJm(A}vK3ERf_{MOIPc|;d=*s5pGJLY{1{ydu z-eh@={ebl%+b?D~*0anW49ECna2k14{$%Sx4IKNQ>c;AW0n^p+*XYOlAAcKYzGJgJ zWb0oIU3Hw1>5rvkKak}~4S%Y9zin6B(R z5=+T&GP*K+vb@W1oOku>kLk+tB+JX3{_MO{Hcqo2Fh1FM#BW)SfB4SC{wx_UWqNUv z%_c^KP`;qP1o{ri86Pd5K!{jr{9^2+>KHjia$oL}_EFU#R3 zS+BCZ?5DE*9F~JQ(9a*k$?`MPWw@VR*Xp0IvAq8p?gYUrx#ZrQ$>22SNqwoca2RpDfGS%0i&*?LgpFDiek z8_4yi!pXh|Fh1r~Hco5g<@lA@LnDU@CtLSu;8gx(^LFNs{aJS2hq>oClC3K=a4KC{ z9LfAjwk~5im_L~xFdX}zY+cLpvLERGPRtU?&hs-qhLg?jnLmb;(dDQ(3R{9fZWPYH5Q{~XV4$$O3z4L6wEb|wZm*HgJQ>6A#Cr<1KQa!7$ z)d$V_A@=8=AAf&Duzq~9ePx!H_4fmt_WnL=gY72UFVVy~`-`e*=8yHldE+1QxH?~x ztzyGP(@M^2*{;BZsO#Ssbz5H275Yst=m;=xn!t znD4S2lJQc*pQ;zxdPb&a*>`N2US#`(EH7i0eb-?AG;r#Kob|%^INoIAC1YXz=|88% zzfjYVCO`9A#;2-T1IO{farqD9jA}R8zOoFTY+PkHjbBOh$8zwKn*KC$@LR?w+oxuG z@GoTZ4dzc{H+3Ln`?4Qs^rtGC;g~MN>0e(mf3i4Yx-743++x3HIb{2t%%29Ins`|b z=8FAFHlAzrr{)*gc{rAX@yYyN1E>1m&*oFvx>h5HiqE0Zj}b{KuL`GsACv8?f4`XZ zEZbMn$gAc*{rf6xUm1Vw2TWHTE;Vt)aI7KOI!eQzYG2uSDZ?j=Hw`|PS2ctBpqcM7 zf3p0d!N>glFt0{lhGTp`JAUY&-?Ker--($T`-^Pfg5!woCW|+ggXzl7N66r0`xXqx zbY<&R=8xr&RdbQw`(EQjnqm~7nA|NV{mlYO6NdHG3p zZ;|!Key^q%jb4~Pj!XUXPL@}8&W-W09QyM;O~r^cIpKVbg! z=L3zus1s{mJYrTmLc~)BV{vm+gzPyo^u( zc+MUn%LmLK>sg&s;*}PMu7xsHKVR8J*^ddX&sDV@MrvJPJ`>Fo@NtQ@9 z&afQp2eLSl@h3YstbtSYA{(a}j_JzqF@G$tY#&bJS1MiEc&Wk1{vw%&Y2Z|Q$i53{ z;8eP@_?7vo%%5d)$l^^#SN6SI1E$ZKjoklia{YAo;1u6y+7 z1D01duao(KY#yPJSJks@{ME>z$}8hf>Iaf}9ovKBQYMFnuF9WG4wjek$?~{HUbe3! zJ~aHPbY<%@8Go|<4b}_uC-Y|wK2^^$|0`rr#Gufh^)UbX&;Q@h!1S-5ni?Azi2sTIn*8-=5%K^2{7vD)eYj~_Ps;U7Q#zoD@EpYoNh3e`WHOBy(T|Bv_}HF(|Lh0QGd@1HZ`fe*&wsNO9nrT>WS9uOkZ1v^Ha{@n ze~*pm-!oBtq5bzbfhznzzxmrYeni#Zxc~k0Uk&`Lfqymde?kL4^2@(}|E Date: Fri, 23 Feb 2024 08:59:43 -0800 Subject: [PATCH 204/416] Sync benchmark folder from main (#1497) Signed-off-by: Heemin Kim --- benchmarks/osb/README.md | 485 +++++++++--------- benchmarks/osb/params/no-train-params.json | 2 + benchmarks/osb/params/train-params.json | 2 + benchmarks/osb/procedures/no-train-test.json | 10 + benchmarks/osb/procedures/train-test.json | 10 + .../perf-tool/add-parent-doc-id-to-dataset.py | 4 +- .../perf-tool/okpt/io/config/parsers/test.py | 2 + .../perf-tool/okpt/io/config/schemas/test.yml | 3 + .../perf-tool/okpt/test/steps/factory.py | 4 +- benchmarks/perf-tool/okpt/test/steps/steps.py | 44 +- .../filtering/relaxed-filter/index.json | 3 +- .../relaxed-filter/relaxed-filter-test.yml | 18 +- .../filtering/restrictive-filter/index.json | 3 +- .../restrictive-filter-test.yml | 15 +- .../release-configs/faiss-hnsw/index.json | 3 +- .../faiss-hnsw/nested/simple/index.json | 3 + .../release-configs/faiss-hnsw/test.yml | 15 +- .../release-configs/faiss-hnswpq/test.yml | 26 +- .../filtering/relaxed-filter/index.json | 17 + .../filtering/relaxed-filter/method-spec.json | 9 + .../relaxed-filter/relaxed-filter-spec.json | 42 ++ .../relaxed-filter/relaxed-filter-test.yml | 64 +++ .../relaxed-filter/train-index-spec.json | 16 + .../filtering/restrictive-filter/index.json | 17 + .../restrictive-filter/method-spec.json | 9 + .../restrictive-filter-spec.json | 44 ++ .../restrictive-filter-test.yml | 64 +++ .../restrictive-filter/train-index-spec.json | 16 + .../release-configs/faiss-ivf/test.yml | 28 +- .../release-configs/faiss-ivfpq/test.yml | 26 +- .../relaxed-filter/relaxed-filter-spec.json | 4 +- .../relaxed-filter/relaxed-filter-test.yml | 16 +- .../restrictive-filter-test.yml | 13 +- .../lucene-hnsw/nested/simple/index.json | 3 + .../nested/simple/simple-nested-test.yml | 2 +- .../release-configs/lucene-hnsw/test.yml | 18 +- .../release-configs/nmslib-hnsw/index.json | 2 +- .../release-configs/nmslib-hnsw/test.yml | 20 +- .../release-configs/run_all_tests.sh | 102 ++++ .../sample-configs/faiss-sift-ivf/test.yml | 2 + .../sample-configs/nmslib-sift-hnsw/test.yml | 2 + 41 files changed, 860 insertions(+), 328 deletions(-) create mode 100644 benchmarks/perf-tool/release-configs/faiss-ivf/filtering/relaxed-filter/index.json create mode 100644 benchmarks/perf-tool/release-configs/faiss-ivf/filtering/relaxed-filter/method-spec.json create mode 100644 benchmarks/perf-tool/release-configs/faiss-ivf/filtering/relaxed-filter/relaxed-filter-spec.json create mode 100644 benchmarks/perf-tool/release-configs/faiss-ivf/filtering/relaxed-filter/relaxed-filter-test.yml create mode 100644 benchmarks/perf-tool/release-configs/faiss-ivf/filtering/relaxed-filter/train-index-spec.json create mode 100644 benchmarks/perf-tool/release-configs/faiss-ivf/filtering/restrictive-filter/index.json create mode 100644 benchmarks/perf-tool/release-configs/faiss-ivf/filtering/restrictive-filter/method-spec.json create mode 100644 benchmarks/perf-tool/release-configs/faiss-ivf/filtering/restrictive-filter/restrictive-filter-spec.json create mode 100644 benchmarks/perf-tool/release-configs/faiss-ivf/filtering/restrictive-filter/restrictive-filter-test.yml create mode 100644 benchmarks/perf-tool/release-configs/faiss-ivf/filtering/restrictive-filter/train-index-spec.json create mode 100755 benchmarks/perf-tool/release-configs/run_all_tests.sh diff --git a/benchmarks/osb/README.md b/benchmarks/osb/README.md index 92272e20b..0f806a344 100644 --- a/benchmarks/osb/README.md +++ b/benchmarks/osb/README.md @@ -106,26 +106,28 @@ use an algorithm that requires training. #### Parameters -| Name | Description | -|-----------------------------------------|----------------------------------------------------------------------------------| -| target_index_name | Name of index to add vectors to | -| target_field_name | Name of field to add vectors to | -| target_index_body | Path to target index definition | -| target_index_primary_shards | Target index primary shards | -| target_index_replica_shards | Target index replica shards | -| target_index_dimension | Dimension of target index | -| target_index_space_type | Target index space type | -| target_index_bulk_size | Target index bulk size | -| target_index_bulk_index_data_set_format | Format of vector data set | -| target_index_bulk_index_data_set_path | Path to vector data set | -| target_index_bulk_index_clients | Clients to be used for bulk ingestion (must be divisor of data set size) | -| hnsw_ef_search | HNSW ef search parameter | -| hnsw_ef_construction | HNSW ef construction parameter | -| hnsw_m | HNSW m parameter | -| query_k | The number of neighbors to return for the search | -| query_clients | Number of clients to use for running queries | -| query_data_set_format | Format of vector data set for queries | -| query_data_set_path | Path to vector data set for queries | +| Name | Description | +|-----------------------------------------|--------------------------------------------------------------------------| +| target_index_name | Name of index to add vectors to | +| target_field_name | Name of field to add vectors to | +| target_index_body | Path to target index definition | +| target_index_primary_shards | Target index primary shards | +| target_index_replica_shards | Target index replica shards | +| target_index_dimension | Dimension of target index | +| target_index_space_type | Target index space type | +| target_index_bulk_size | Target index bulk size | +| target_index_bulk_index_data_set_format | Format of vector data set | +| target_index_bulk_index_data_set_path | Path to vector data set | +| target_index_bulk_index_clients | Clients to be used for bulk ingestion (must be divisor of data set size) | +| target_index_max_num_segments | Number of segments to merge target index down to before beginning search | +| target_index_force_merge_timeout | Timeout for of force merge requests in seconds | +| hnsw_ef_search | HNSW ef search parameter | +| hnsw_ef_construction | HNSW ef construction parameter | +| hnsw_m | HNSW m parameter | +| query_k | The number of neighbors to return for the search | +| query_clients | Number of clients to use for running queries | +| query_data_set_format | Format of vector data set for queries | +| query_data_set_path | Path to vector data set for queries | #### Metrics @@ -141,86 +143,90 @@ The result metrics of this procedure will look like: | Metric | Task | Value | Unit | |---------------------------------------------------------------:|------------------------:|------------:|-------:| -| Cumulative indexing time of primary shards | | 0.00173333 | min | -| Min cumulative indexing time across primary shards | | 0 | min | -| Median cumulative indexing time across primary shards | | 0 | min | -| Max cumulative indexing time across primary shards | | 0.000616667 | min | +| Cumulative indexing time of primary shards | | 1.82885 | min | +| Min cumulative indexing time across primary shards | | 0.4121 | min | +| Median cumulative indexing time across primary shards | | 0.559617 | min | +| Max cumulative indexing time across primary shards | | 0.857133 | min | | Cumulative indexing throttle time of primary shards | | 0 | min | | Min cumulative indexing throttle time across primary shards | | 0 | min | | Median cumulative indexing throttle time across primary shards | | 0 | min | | Max cumulative indexing throttle time across primary shards | | 0 | min | -| Cumulative merge time of primary shards | | 0 | min | -| Cumulative merge count of primary shards | | 0 | | -| Min cumulative merge time across primary shards | | 0 | min | -| Median cumulative merge time across primary shards | | 0 | min | -| Max cumulative merge time across primary shards | | 0 | min | +| Cumulative merge time of primary shards | | 5.89065 | min | +| Cumulative merge count of primary shards | | 3 | | +| Min cumulative merge time across primary shards | | 1.95945 | min | +| Median cumulative merge time across primary shards | | 1.96345 | min | +| Max cumulative merge time across primary shards | | 1.96775 | min | | Cumulative merge throttle time of primary shards | | 0 | min | | Min cumulative merge throttle time across primary shards | | 0 | min | | Median cumulative merge throttle time across primary shards | | 0 | min | | Max cumulative merge throttle time across primary shards | | 0 | min | -| Cumulative refresh time of primary shards | | 0.00271667 | min | -| Cumulative refresh count of primary shards | | 115 | | -| Min cumulative refresh time across primary shards | | 0 | min | -| Median cumulative refresh time across primary shards | | 0 | min | -| Max cumulative refresh time across primary shards | | 0.00135 | min | -| Cumulative flush time of primary shards | | 0 | min | -| Cumulative flush count of primary shards | | 43 | | -| Min cumulative flush time across primary shards | | 0 | min | -| Median cumulative flush time across primary shards | | 0 | min | -| Max cumulative flush time across primary shards | | 0 | min | -| Total Young Gen GC time | | 0.849 | s | -| Total Young Gen GC count | | 20 | | +| Cumulative refresh time of primary shards | | 8.52517 | min | +| Cumulative refresh count of primary shards | | 29 | | +| Min cumulative refresh time across primary shards | | 2.64265 | min | +| Median cumulative refresh time across primary shards | | 2.93913 | min | +| Max cumulative refresh time across primary shards | | 2.94338 | min | +| Cumulative flush time of primary shards | | 0.00221667 | min | +| Cumulative flush count of primary shards | | 3 | | +| Min cumulative flush time across primary shards | | 0.000733333 | min | +| Median cumulative flush time across primary shards | | 0.000733333 | min | +| Max cumulative flush time across primary shards | | 0.00075 | min | +| Total Young Gen GC time | | 0.318 | s | +| Total Young Gen GC count | | 2 | | | Total Old Gen GC time | | 0 | s | | Total Old Gen GC count | | 0 | | -| Store size | | 0.647921 | GB | -| Translog size | | 0.00247511 | GB | -| Heap used for segments | | 0.284451 | MB | -| Heap used for doc values | | 0.0872688 | MB | -| Heap used for terms | | 0.0714417 | MB | -| Heap used for norms | | 6.10352e-05 | MB | +| Store size | | 1.43566 | GB | +| Translog size | | 1.53668e-07 | GB | +| Heap used for segments | | 0.00410843 | MB | +| Heap used for doc values | | 0.000286102 | MB | +| Heap used for terms | | 0.00121307 | MB | +| Heap used for norms | | 0 | MB | | Heap used for points | | 0 | MB | -| Heap used for stored fields | | 0.125679 | MB | -| Segment count | | 257 | | -| Min Throughput | custom-vector-bulk | 18018.5 | docs/s | -| Mean Throughput | custom-vector-bulk | 18018.5 | docs/s | -| Median Throughput | custom-vector-bulk | 18018.5 | docs/s | -| Max Throughput | custom-vector-bulk | 18018.5 | docs/s | -| 50th percentile latency | custom-vector-bulk | 98.5565 | ms | -| 90th percentile latency | custom-vector-bulk | 100.033 | ms | -| 100th percentile latency | custom-vector-bulk | 103.792 | ms | -| 50th percentile service time | custom-vector-bulk | 98.5565 | ms | -| 90th percentile service time | custom-vector-bulk | 100.033 | ms | -| 100th percentile service time | custom-vector-bulk | 103.792 | ms | +| Heap used for stored fields | | 0.00260925 | MB | +| Segment count | | 3 | | +| Min Throughput | custom-vector-bulk | 38005.8 | docs/s | +| Mean Throughput | custom-vector-bulk | 44827.9 | docs/s | +| Median Throughput | custom-vector-bulk | 40507.2 | docs/s | +| Max Throughput | custom-vector-bulk | 88967.8 | docs/s | +| 50th percentile latency | custom-vector-bulk | 29.5857 | ms | +| 90th percentile latency | custom-vector-bulk | 49.0719 | ms | +| 99th percentile latency | custom-vector-bulk | 72.6138 | ms | +| 99.9th percentile latency | custom-vector-bulk | 279.826 | ms | +| 100th percentile latency | custom-vector-bulk | 15688 | ms | +| 50th percentile service time | custom-vector-bulk | 29.5857 | ms | +| 90th percentile service time | custom-vector-bulk | 49.0719 | ms | +| 99th percentile service time | custom-vector-bulk | 72.6138 | ms | +| 99.9th percentile service time | custom-vector-bulk | 279.826 | ms | +| 100th percentile service time | custom-vector-bulk | 15688 | ms | | error rate | custom-vector-bulk | 0 | % | -| Min Throughput | refresh-target-index | 76.22 | ops/s | -| Mean Throughput | refresh-target-index | 76.22 | ops/s | -| Median Throughput | refresh-target-index | 76.22 | ops/s | -| Max Throughput | refresh-target-index | 76.22 | ops/s | -| 100th percentile latency | refresh-target-index | 12.7619 | ms | -| 100th percentile service time | refresh-target-index | 12.7619 | ms | +| Min Throughput | refresh-target-index | 0.01 | ops/s | +| Mean Throughput | refresh-target-index | 0.01 | ops/s | +| Median Throughput | refresh-target-index | 0.01 | ops/s | +| Max Throughput | refresh-target-index | 0.01 | ops/s | +| 100th percentile latency | refresh-target-index | 176610 | ms | +| 100th percentile service time | refresh-target-index | 176610 | ms | | error rate | refresh-target-index | 0 | % | -| Min Throughput | knn-query-from-data-set | 1587.47 | ops/s | -| Mean Throughput | knn-query-from-data-set | 1649.97 | ops/s | -| Median Throughput | knn-query-from-data-set | 1661.79 | ops/s | -| Max Throughput | knn-query-from-data-set | 1677.06 | ops/s | -| 50th percentile latency | knn-query-from-data-set | 4.79125 | ms | -| 90th percentile latency | knn-query-from-data-set | 5.38 | ms | -| 99th percentile latency | knn-query-from-data-set | 46.8965 | ms | -| 99.9th percentile latency | knn-query-from-data-set | 58.2049 | ms | -| 99.99th percentile latency | knn-query-from-data-set | 59.6476 | ms | -| 100th percentile latency | knn-query-from-data-set | 60.9245 | ms | -| 50th percentile service time | knn-query-from-data-set | 4.79125 | ms | -| 90th percentile service time | knn-query-from-data-set | 5.38 | ms | -| 99th percentile service time | knn-query-from-data-set | 46.8965 | ms | -| 99.9th percentile service time | knn-query-from-data-set | 58.2049 | ms | -| 99.99th percentile service time | knn-query-from-data-set | 59.6476 | ms | -| 100th percentile service time | knn-query-from-data-set | 60.9245 | ms | +| Min Throughput | knn-query-from-data-set | 444.17 | ops/s | +| Mean Throughput | knn-query-from-data-set | 601.68 | ops/s | +| Median Throughput | knn-query-from-data-set | 621.19 | ops/s | +| Max Throughput | knn-query-from-data-set | 631.23 | ops/s | +| 50th percentile latency | knn-query-from-data-set | 14.7612 | ms | +| 90th percentile latency | knn-query-from-data-set | 20.6954 | ms | +| 99th percentile latency | knn-query-from-data-set | 27.7499 | ms | +| 99.9th percentile latency | knn-query-from-data-set | 41.3506 | ms | +| 99.99th percentile latency | knn-query-from-data-set | 162.391 | ms | +| 100th percentile latency | knn-query-from-data-set | 162.756 | ms | +| 50th percentile service time | knn-query-from-data-set | 14.7612 | ms | +| 90th percentile service time | knn-query-from-data-set | 20.6954 | ms | +| 99th percentile service time | knn-query-from-data-set | 27.7499 | ms | +| 99.9th percentile service time | knn-query-from-data-set | 41.3506 | ms | +| 99.99th percentile service time | knn-query-from-data-set | 162.391 | ms | +| 100th percentile service time | knn-query-from-data-set | 162.756 | ms | | error rate | knn-query-from-data-set | 0 | % | --------------------------------- -[INFO] SUCCESS (took 46 seconds) --------------------------------- +--------------------------------- +[INFO] SUCCESS (took 618 seconds) +--------------------------------- ``` ### Train Test @@ -243,41 +249,43 @@ algorithm that requires training. #### Parameters -| Name | Description | -|-----------------------------------------|----------------------------------------------------------------------------------| -| target_index_name | Name of index to add vectors to | -| target_field_name | Name of field to add vectors to | -| target_index_body | Path to target index definition | -| target_index_primary_shards | Target index primary shards | -| target_index_replica_shards | Target index replica shards | -| target_index_dimension | Dimension of target index | -| target_index_space_type | Target index space type | -| target_index_bulk_size | Target index bulk size | -| target_index_bulk_index_data_set_format | Format of vector data set for ingestion | -| target_index_bulk_index_data_set_path | Path to vector data set for ingestion | -| target_index_bulk_index_clients | Clients to be used for bulk ingestion (must be divisor of data set size) | -| ivf_nlists | IVF nlist parameter | -| ivf_nprobes | IVF nprobe parameter | -| pq_code_size | PQ code_size parameter | -| pq_m | PQ m parameter | -| train_model_method | Method to be used for model (ivf or ivfpq) | -| train_model_id | Model ID | -| train_index_name | Name of index to put training data into | -| train_field_name | Name of field to put training data into | -| train_index_body | Path to train index definition | -| train_search_size | Search size to use when pulling training data | -| train_timeout | Timeout to wait for training to finish | -| train_index_primary_shards | Train index primary shards | -| train_index_replica_shards | Train index replica shards | -| train_index_bulk_size | Train index bulk size | -| train_index_data_set_format | Format of vector data set for training | -| train_index_data_set_path | Path to vector data set for training | -| train_index_num_vectors | Number of vectors to use from vector data set for training | -| train_index_bulk_index_clients | Clients to be used for bulk ingestion (must be divisor of data set size) | -| query_k | The number of neighbors to return for the search | -| query_clients | Number of clients to use for running queries | -| query_data_set_format | Format of vector data set for queries | -| query_data_set_path | Path to vector data set for queries | +| Name | Description | +|-----------------------------------------|--------------------------------------------------------------------------| +| target_index_name | Name of index to add vectors to | +| target_field_name | Name of field to add vectors to | +| target_index_body | Path to target index definition | +| target_index_primary_shards | Target index primary shards | +| target_index_replica_shards | Target index replica shards | +| target_index_dimension | Dimension of target index | +| target_index_space_type | Target index space type | +| target_index_bulk_size | Target index bulk size | +| target_index_bulk_index_data_set_format | Format of vector data set for ingestion | +| target_index_bulk_index_data_set_path | Path to vector data set for ingestion | +| target_index_bulk_index_clients | Clients to be used for bulk ingestion (must be divisor of data set size) | +| target_index_max_num_segments | Number of segments to merge target index down to before beginning search | +| target_index_force_merge_timeout | Timeout for of force merge requests in seconds | +| ivf_nlists | IVF nlist parameter | +| ivf_nprobes | IVF nprobe parameter | +| pq_code_size | PQ code_size parameter | +| pq_m | PQ m parameter | +| train_model_method | Method to be used for model (ivf or ivfpq) | +| train_model_id | Model ID | +| train_index_name | Name of index to put training data into | +| train_field_name | Name of field to put training data into | +| train_index_body | Path to train index definition | +| train_search_size | Search size to use when pulling training data | +| train_timeout | Timeout to wait for training to finish | +| train_index_primary_shards | Train index primary shards | +| train_index_replica_shards | Train index replica shards | +| train_index_bulk_size | Train index bulk size | +| train_index_data_set_format | Format of vector data set for training | +| train_index_data_set_path | Path to vector data set for training | +| train_index_num_vectors | Number of vectors to use from vector data set for training | +| train_index_bulk_index_clients | Clients to be used for bulk ingestion (must be divisor of data set size) | +| query_k | The number of neighbors to return for the search | +| query_clients | Number of clients to use for running queries | +| query_data_set_format | Format of vector data set for queries | +| query_data_set_path | Path to vector data set for queries | #### Metrics @@ -286,131 +294,132 @@ The result metrics of this procedure will look like: ------------------------------------------------------ _______ __ _____ / ____(_)___ ____ _/ / / ___/_________ ________ - / /_ / / __ \/ __ `/ / \__ \/ ___/ __ \/ ___/ _ \ [63/1855] + / /_ / / __ \/ __ `/ / \__ \/ ___/ __ \/ ___/ _ \ / __/ / / / / / /_/ / / ___/ / /__/ /_/ / / / __/ /_/ /_/_/ /_/\__,_/_/ /____/\___/\____/_/ \___/ ------------------------------------------------------ -| Metric | Task | Value | Unit | -|---------------------------------------------------------------:|------------------------:|------------:|-----------------:| -| Cumulative indexing time of primary shards | | 2.92355 | min | -| Min cumulative indexing time across primary shards | | 0 | min | -| Median cumulative indexing time across primary shards | | 0.497817 | min | -| Max cumulative indexing time across primary shards | | 1.37717 | min | -| Cumulative indexing throttle time of primary shards | | 0 | min | -| Min cumulative indexing throttle time across primary shards | | 0 | min | -| Median cumulative indexing throttle time across primary shards | | 0 | min | -| Max cumulative indexing throttle time across primary shards | | 0 | min | -| Cumulative merge time of primary shards | | 1.34895 | min | -| Cumulative merge count of primary shards | | 39 | | -| Min cumulative merge time across primary shards | | 0 | min | -| Median cumulative merge time across primary shards | | 0.292033 | min | -| Max cumulative merge time across primary shards | | 0.6268 | min | -| Cumulative merge throttle time of primary shards | | 0.62845 | min | -| Min cumulative merge throttle time across primary shards | | 0 | min | -| Median cumulative merge throttle time across primary shards | | 0.155617 | min | -| Max cumulative merge throttle time across primary shards | | 0.290117 | min | -| Cumulative refresh time of primary shards | | 0.369433 | min | -| Cumulative refresh count of primary shards | | 96 | | -| Min cumulative refresh time across primary shards | | 0 | min | -| Median cumulative refresh time across primary shards | | 0.0903833 | min | -| Max cumulative refresh time across primary shards | | 0.10365 | min | -| Cumulative flush time of primary shards | | 0.0278667 | min | -| Cumulative flush count of primary shards | | 2 | | -| Min cumulative flush time across primary shards | | 0 | min | -| Median cumulative flush time across primary shards | | 0 | min | -| Max cumulative flush time across primary shards | | 0.0278667 | min | -| Total Young Gen GC time | | 13.106 | s | -| Total Young Gen GC count | | 263 | | -| Total Old Gen GC time | | 0 | s | -| Total Old Gen GC count | | 0 | | -| Store size | | 2.60183 | GB | -| Translog size | | 1.34787 | GB | -| Heap used for segments | | 0.0646248 | MB | -| Heap used for doc values | | 0.00899887 | MB | -| Heap used for terms | | 0.0203552 | MB | -| Heap used for norms | | 6.10352e-05 | MB | -| Heap used for points | | 0 | MB | -| Heap used for stored fields | | 0.0352097 | MB | -| Segment count | | 71 | | -| Min Throughput | delete-model | 10.55 | ops/s | -| Mean Throughput | delete-model | 10.55 | ops/s | -| Median Throughput | delete-model | 10.55 | ops/s | -| Max Throughput | delete-model | 10.55 | ops/s | -| 100th percentile latency | delete-model | 94.4726 | ms | -| 100th percentile service time | delete-model | 94.4726 | ms | -| error rate | delete-model | 0 | % | -| Min Throughput | train-vector-bulk | 44763.1 | docs/s | -| Mean Throughput | train-vector-bulk | 52022.4 | docs/s | -| Median Throughput | train-vector-bulk | 52564.8 | docs/s | -| Max Throughput | train-vector-bulk | 53833 | docs/s | -| 50th percentile latency | train-vector-bulk | 22.3364 | ms | -| 90th percentile latency | train-vector-bulk | 47.799 | ms | -| 99th percentile latency | train-vector-bulk | 195.954 | ms | -| 99.9th percentile latency | train-vector-bulk | 495.217 | ms | -| 100th percentile latency | train-vector-bulk | 663.48 | ms | -| 50th percentile service time | train-vector-bulk | 22.3364 | ms | -| 90th percentile service time | train-vector-bulk | 47.799 | ms | -| 99th percentile service time | train-vector-bulk | 195.954 | ms | -| 99.9th percentile service time | train-vector-bulk | 495.217 | ms | -| 100th percentile service time | train-vector-bulk | 663.48 | ms | -| error rate | train-vector-bulk | 0 | % | -| Min Throughput | refresh-train-index | 0.98 | ops/s | -| Mean Throughput | refresh-train-index | 0.98 | ops/s | -| Median Throughput | refresh-train-index | 0.98 | ops/s | -| Max Throughput | refresh-train-index | 0.98 | ops/s | -| 100th percentile latency | refresh-train-index | 1019.54 | ms | -| 100th percentile service time | refresh-train-index | 1019.54 | ms | -| error rate | refresh-train-index | 0 | % | -| Min Throughput | ivfpq-train-model | 0.01 | models_trained/s | -| Mean Throughput | ivfpq-train-model | 0.01 | models_trained/s | -| Median Throughput | ivfpq-train-model | 0.01 | models_trained/s | -| Max Throughput | ivfpq-train-model | 0.01 | models_trained/s | -| 100th percentile latency | ivfpq-train-model | 150952 | ms | -| 100th percentile service time | ivfpq-train-model | 150952 | ms | -| error rate | ivfpq-train-model | 0 | % | -| Min Throughput | custom-vector-bulk | 32367.4 | docs/s | -| Mean Throughput | custom-vector-bulk | 36027.5 | docs/s | -| Median Throughput | custom-vector-bulk | 35276.7 | docs/s | -| Max Throughput | custom-vector-bulk | 41095 | docs/s | -| 50th percentile latency | custom-vector-bulk | 22.2419 | ms | -| 90th percentile latency | custom-vector-bulk | 70.163 | ms | -| 99th percentile latency | custom-vector-bulk | 308.395 | ms | -| 99.9th percentile latency | custom-vector-bulk | 548.558 | ms | -| 100th percentile latency | custom-vector-bulk | 655.628 | ms | -| 50th percentile service time | custom-vector-bulk | 22.2419 | ms | -| 90th percentile service time | custom-vector-bulk | 70.163 | ms | -| 99th percentile service time | custom-vector-bulk | 308.395 | ms | -| 99.9th percentile service time | custom-vector-bulk | 548.558 | ms | -| 100th percentile service time | custom-vector-bulk | 655.628 | ms | -| error rate | custom-vector-bulk | 0 | % | -| Min Throughput | refresh-target-index | 0.23 | ops/s | -| Mean Throughput | refresh-target-index | 0.23 | ops/s | -| Median Throughput | refresh-target-index | 0.23 | ops/s | -| Max Throughput | refresh-target-index | 0.23 | ops/s | -| 100th percentile latency | refresh-target-index | 4331.17 | ms | -| 100th percentile service time | refresh-target-index | 4331.17 | ms | -| error rate | refresh-target-index | 0 | % | -| Min Throughput | knn-query-from-data-set | 455.19 | ops/s | -| Mean Throughput | knn-query-from-data-set | 511.74 | ops/s | -| Median Throughput | knn-query-from-data-set | 510.85 | ops/s | -| Max Throughput | knn-query-from-data-set | 570.07 | ops/s | -| 50th percentile latency | knn-query-from-data-set | 14.1626 | ms | -| 90th percentile latency | knn-query-from-data-set | 30.2389 | ms | -| 99th percentile latency | knn-query-from-data-set | 71.2793 | ms | -| 99.9th percentile latency | knn-query-from-data-set | 104.733 | ms | -| 99.99th percentile latency | knn-query-from-data-set | 127.298 | ms | -| 100th percentile latency | knn-query-from-data-set | 145.229 | ms | -| 50th percentile service time | knn-query-from-data-set | 14.1626 | ms | -| 90th percentile service time | knn-query-from-data-set | 30.2389 | ms | -| 99th percentile service time | knn-query-from-data-set | 71.2793 | ms | -| 99.9th percentile service time | knn-query-from-data-set | 104.733 | ms | -| 99.99th percentile service time | knn-query-from-data-set | 127.298 | ms | -| 100th percentile service time | knn-query-from-data-set | 145.229 | ms | -| error rate | knn-query-from-data-set | 0 | % | + +| Metric | Task | Value | Unit | +|---------------------------------------------------------------:|------------------------:|-----------:|-----------------:| +| Cumulative indexing time of primary shards | | 2.92382 | min | +| Min cumulative indexing time across primary shards | | 0.42245 | min | +| Median cumulative indexing time across primary shards | | 0.43395 | min | +| Max cumulative indexing time across primary shards | | 1.63347 | min | +| Cumulative indexing throttle time of primary shards | | 0 | min | +| Min cumulative indexing throttle time across primary shards | | 0 | min | +| Median cumulative indexing throttle time across primary shards | | 0 | min | +| Max cumulative indexing throttle time across primary shards | | 0 | min | +| Cumulative merge time of primary shards | | 1.36293 | min | +| Cumulative merge count of primary shards | | 20 | | +| Min cumulative merge time across primary shards | | 0.263283 | min | +| Median cumulative merge time across primary shards | | 0.291733 | min | +| Max cumulative merge time across primary shards | | 0.516183 | min | +| Cumulative merge throttle time of primary shards | | 0.701683 | min | +| Min cumulative merge throttle time across primary shards | | 0.163883 | min | +| Median cumulative merge throttle time across primary shards | | 0.175717 | min | +| Max cumulative merge throttle time across primary shards | | 0.186367 | min | +| Cumulative refresh time of primary shards | | 0.222217 | min | +| Cumulative refresh count of primary shards | | 67 | | +| Min cumulative refresh time across primary shards | | 0.03915 | min | +| Median cumulative refresh time across primary shards | | 0.039825 | min | +| Max cumulative refresh time across primary shards | | 0.103417 | min | +| Cumulative flush time of primary shards | | 0.0276833 | min | +| Cumulative flush count of primary shards | | 1 | | +| Min cumulative flush time across primary shards | | 0 | min | +| Median cumulative flush time across primary shards | | 0 | min | +| Max cumulative flush time across primary shards | | 0.0276833 | min | +| Total Young Gen GC time | | 0.074 | s | +| Total Young Gen GC count | | 8 | | +| Total Old Gen GC time | | 0 | s | +| Total Old Gen GC count | | 0 | | +| Store size | | 1.67839 | GB | +| Translog size | | 0.115145 | GB | +| Heap used for segments | | 0.0350914 | MB | +| Heap used for doc values | | 0.00771713 | MB | +| Heap used for terms | | 0.0101089 | MB | +| Heap used for norms | | 0 | MB | +| Heap used for points | | 0 | MB | +| Heap used for stored fields | | 0.0172653 | MB | +| Segment count | | 25 | | +| Min Throughput | delete-model | 25.45 | ops/s | +| Mean Throughput | delete-model | 25.45 | ops/s | +| Median Throughput | delete-model | 25.45 | ops/s | +| Max Throughput | delete-model | 25.45 | ops/s | +| 100th percentile latency | delete-model | 39.0409 | ms | +| 100th percentile service time | delete-model | 39.0409 | ms | +| error rate | delete-model | 0 | % | +| Min Throughput | train-vector-bulk | 49518.9 | docs/s | +| Mean Throughput | train-vector-bulk | 54418.8 | docs/s | +| Median Throughput | train-vector-bulk | 52984.2 | docs/s | +| Max Throughput | train-vector-bulk | 62118.3 | docs/s | +| 50th percentile latency | train-vector-bulk | 26.5293 | ms | +| 90th percentile latency | train-vector-bulk | 41.8212 | ms | +| 99th percentile latency | train-vector-bulk | 239.351 | ms | +| 99.9th percentile latency | train-vector-bulk | 348.507 | ms | +| 100th percentile latency | train-vector-bulk | 436.292 | ms | +| 50th percentile service time | train-vector-bulk | 26.5293 | ms | +| 90th percentile service time | train-vector-bulk | 41.8212 | ms | +| 99th percentile service time | train-vector-bulk | 239.351 | ms | +| 99.9th percentile service time | train-vector-bulk | 348.507 | ms | +| 100th percentile service time | train-vector-bulk | 436.292 | ms | +| error rate | train-vector-bulk | 0 | % | +| Min Throughput | refresh-train-index | 0.47 | ops/s | +| Mean Throughput | refresh-train-index | 0.47 | ops/s | +| Median Throughput | refresh-train-index | 0.47 | ops/s | +| Max Throughput | refresh-train-index | 0.47 | ops/s | +| 100th percentile latency | refresh-train-index | 2142.96 | ms | +| 100th percentile service time | refresh-train-index | 2142.96 | ms | +| error rate | refresh-train-index | 0 | % | +| Min Throughput | ivfpq-train-model | 0.01 | models_trained/s | +| Mean Throughput | ivfpq-train-model | 0.01 | models_trained/s | +| Median Throughput | ivfpq-train-model | 0.01 | models_trained/s | +| Max Throughput | ivfpq-train-model | 0.01 | models_trained/s | +| 100th percentile latency | ivfpq-train-model | 136563 | ms | +| 100th percentile service time | ivfpq-train-model | 136563 | ms | +| error rate | ivfpq-train-model | 0 | % | +| Min Throughput | custom-vector-bulk | 62384.8 | docs/s | +| Mean Throughput | custom-vector-bulk | 69035.2 | docs/s | +| Median Throughput | custom-vector-bulk | 68675.4 | docs/s | +| Max Throughput | custom-vector-bulk | 80713.4 | docs/s | +| 50th percentile latency | custom-vector-bulk | 18.7726 | ms | +| 90th percentile latency | custom-vector-bulk | 34.8881 | ms | +| 99th percentile latency | custom-vector-bulk | 150.435 | ms | +| 99.9th percentile latency | custom-vector-bulk | 296.862 | ms | +| 100th percentile latency | custom-vector-bulk | 344.394 | ms | +| 50th percentile service time | custom-vector-bulk | 18.7726 | ms | +| 90th percentile service time | custom-vector-bulk | 34.8881 | ms | +| 99th percentile service time | custom-vector-bulk | 150.435 | ms | +| 99.9th percentile service time | custom-vector-bulk | 296.862 | ms | +| 100th percentile service time | custom-vector-bulk | 344.394 | ms | +| error rate | custom-vector-bulk | 0 | % | +| Min Throughput | refresh-target-index | 28.32 | ops/s | +| Mean Throughput | refresh-target-index | 28.32 | ops/s | +| Median Throughput | refresh-target-index | 28.32 | ops/s | +| Max Throughput | refresh-target-index | 28.32 | ops/s | +| 100th percentile latency | refresh-target-index | 34.9811 | ms | +| 100th percentile service time | refresh-target-index | 34.9811 | ms | +| error rate | refresh-target-index | 0 | % | +| Min Throughput | knn-query-from-data-set | 0.9 | ops/s | +| Mean Throughput | knn-query-from-data-set | 453.84 | ops/s | +| Median Throughput | knn-query-from-data-set | 554.15 | ops/s | +| Max Throughput | knn-query-from-data-set | 681 | ops/s | +| 50th percentile latency | knn-query-from-data-set | 11.7174 | ms | +| 90th percentile latency | knn-query-from-data-set | 15.4445 | ms | +| 99th percentile latency | knn-query-from-data-set | 21.0682 | ms | +| 99.9th percentile latency | knn-query-from-data-set | 39.5414 | ms | +| 99.99th percentile latency | knn-query-from-data-set | 1116.33 | ms | +| 100th percentile latency | knn-query-from-data-set | 1116.66 | ms | +| 50th percentile service time | knn-query-from-data-set | 11.7174 | ms | +| 90th percentile service time | knn-query-from-data-set | 15.4445 | ms | +| 99th percentile service time | knn-query-from-data-set | 21.0682 | ms | +| 99.9th percentile service time | knn-query-from-data-set | 39.5414 | ms | +| 99.99th percentile service time | knn-query-from-data-set | 1116.33 | ms | +| 100th percentile service time | knn-query-from-data-set | 1116.66 | ms | +| error rate | knn-query-from-data-set | 0 | % | --------------------------------- -[INFO] SUCCESS (took 295 seconds) +[INFO] SUCCESS (took 281 seconds) --------------------------------- ``` diff --git a/benchmarks/osb/params/no-train-params.json b/benchmarks/osb/params/no-train-params.json index 64fe3c296..58e4197fd 100644 --- a/benchmarks/osb/params/no-train-params.json +++ b/benchmarks/osb/params/no-train-params.json @@ -10,6 +10,8 @@ "target_index_bulk_index_data_set_format": "hdf5", "target_index_bulk_index_data_set_path": "", "target_index_bulk_index_clients": 10, + "target_index_max_num_segments": 10, + "target_index_force_merge_timeout": 45.0, "hnsw_ef_search": 512, "hnsw_ef_construction": 512, "hnsw_m": 16, diff --git a/benchmarks/osb/params/train-params.json b/benchmarks/osb/params/train-params.json index 4c598d25b..f55ed4333 100644 --- a/benchmarks/osb/params/train-params.json +++ b/benchmarks/osb/params/train-params.json @@ -10,6 +10,8 @@ "target_index_bulk_index_data_set_format": "hdf5", "target_index_bulk_index_data_set_path": "", "target_index_bulk_index_clients": 10, + "target_index_max_num_segments": 10, + "target_index_force_merge_timeout": 45.0, "ivf_nlists": 10, "ivf_nprobes": 1, "pq_code_size": 8, diff --git a/benchmarks/osb/procedures/no-train-test.json b/benchmarks/osb/procedures/no-train-test.json index 03d72d6bd..01985b914 100644 --- a/benchmarks/osb/procedures/no-train-test.json +++ b/benchmarks/osb/procedures/no-train-test.json @@ -46,6 +46,16 @@ "retries": 100 } }, + { + "operation": { + "name": "force-merge", + "operation-type": "force-merge", + "request-timeout": {{ target_index_force_merge_timeout }}, + "index": "{{ target_index_name }}", + "mode": "polling", + "max-num-segments": {{ target_index_max_num_segments }} + } + }, { "operation": { "name": "knn-query-from-data-set", diff --git a/benchmarks/osb/procedures/train-test.json b/benchmarks/osb/procedures/train-test.json index 49930044a..ca26db0b0 100644 --- a/benchmarks/osb/procedures/train-test.json +++ b/benchmarks/osb/procedures/train-test.json @@ -100,6 +100,16 @@ "retries": 100 } }, + { + "operation": { + "name": "force-merge", + "operation-type": "force-merge", + "request-timeout": {{ target_index_force_merge_timeout }}, + "index": "{{ target_index_name }}", + "mode": "polling", + "max-num-segments": {{ target_index_max_num_segments }} + } + }, { "operation": { "name": "knn-query-from-data-set", diff --git a/benchmarks/perf-tool/add-parent-doc-id-to-dataset.py b/benchmarks/perf-tool/add-parent-doc-id-to-dataset.py index 54a40b281..a4acafd03 100644 --- a/benchmarks/perf-tool/add-parent-doc-id-to-dataset.py +++ b/benchmarks/perf-tool/add-parent-doc-id-to-dataset.py @@ -116,8 +116,8 @@ def run(self, source_path, target_path) -> None: possible_colors = ['red', 'green', 'yellow', 'blue', None] possible_tastes = ['sweet', 'salty', 'sour', 'bitter', None] max_age = 100 - min_field_size = 1000 - max_field_size = 10001 + min_field_size = 10 + max_field_size = 10 # Copy train and test data for key in in_file.keys(): diff --git a/benchmarks/perf-tool/okpt/io/config/parsers/test.py b/benchmarks/perf-tool/okpt/io/config/parsers/test.py index d0ef4c02f..c47e30ecc 100644 --- a/benchmarks/perf-tool/okpt/io/config/parsers/test.py +++ b/benchmarks/perf-tool/okpt/io/config/parsers/test.py @@ -24,6 +24,7 @@ class TestConfig: test_id: str endpoint: str port: int + timeout: int num_runs: int show_runs: bool setup: List[Step] @@ -67,6 +68,7 @@ def parse(self, file_obj: TextIOWrapper) -> TestConfig: test_config = TestConfig( endpoint=config_obj['endpoint'], port=config_obj['port'], + timeout=config_obj['timeout'], test_name=config_obj['test_name'], test_id=config_obj['test_id'], num_runs=config_obj['num_runs'], diff --git a/benchmarks/perf-tool/okpt/io/config/schemas/test.yml b/benchmarks/perf-tool/okpt/io/config/schemas/test.yml index 06b880cc7..4d5c21a15 100644 --- a/benchmarks/perf-tool/okpt/io/config/schemas/test.yml +++ b/benchmarks/perf-tool/okpt/io/config/schemas/test.yml @@ -12,6 +12,9 @@ endpoint: port: type: integer default: 9200 +timeout: + type: integer + default: 60 test_name: type: string test_id: diff --git a/benchmarks/perf-tool/okpt/test/steps/factory.py b/benchmarks/perf-tool/okpt/test/steps/factory.py index 625254df9..2033f2672 100644 --- a/benchmarks/perf-tool/okpt/test/steps/factory.py +++ b/benchmarks/perf-tool/okpt/test/steps/factory.py @@ -10,7 +10,7 @@ from okpt.test.steps.steps import CreateIndexStep, DisableRefreshStep, RefreshIndexStep, DeleteIndexStep, \ TrainModelStep, DeleteModelStep, ForceMergeStep, ClearCacheStep, IngestStep, IngestMultiFieldStep, \ - IngestNestedFieldStep, QueryStep, QueryWithFilterStep, QueryNestedFieldStep, GetStatsStep + IngestNestedFieldStep, QueryStep, QueryWithFilterStep, QueryNestedFieldStep, GetStatsStep, WarmupStep def create_step(step_config: StepConfig) -> Step: @@ -44,5 +44,7 @@ def create_step(step_config: StepConfig) -> Step: return ClearCacheStep(step_config) elif step_config.step_name == GetStatsStep.label: return GetStatsStep(step_config) + elif step_config.step_name == WarmupStep.label: + return WarmupStep(step_config) raise ConfigurationError(f'Invalid step {step_config.step_name}') diff --git a/benchmarks/perf-tool/okpt/test/steps/steps.py b/benchmarks/perf-tool/okpt/test/steps/steps.py index 75404e354..99b2728dc 100644 --- a/benchmarks/perf-tool/okpt/test/steps/steps.py +++ b/benchmarks/perf-tool/okpt/test/steps/steps.py @@ -38,8 +38,9 @@ def __init__(self, step_config: StepConfig): default_port = 9200 if self.endpoint == 'localhost' else 80 self.port = parse_int_param('port', step_config.config, step_config.implicit_config, default_port) + self.timeout = parse_int_param('timeout', step_config.config, {}, 60) self.opensearch = get_opensearch_client(str(self.endpoint), - int(self.port)) + int(self.port), int(self.timeout)) class CreateIndexStep(OpenSearchStep): @@ -163,6 +164,25 @@ def _get_measures(self) -> List[str]: return ['took'] +class WarmupStep(OpenSearchStep): + """See base class.""" + + label = 'warmup_operation' + + def __init__(self, step_config: StepConfig): + super().__init__(step_config) + self.index_name = parse_string_param('index_name', step_config.config, {}, + None) + + def _action(self): + """Performs warmup operation on an index.""" + warmup_operation(self.endpoint, self.port, self.index_name) + return {} + + def _get_measures(self) -> List[str]: + return ['took'] + + class TrainModelStep(OpenSearchStep): """See base class.""" @@ -739,9 +759,6 @@ def get_body(self, vec): } } - def get_exclude_fields(self): - return ['nested_field.' + self.field_name] - class GetStatsStep(OpenSearchStep): """See base class.""" @@ -841,6 +858,23 @@ def delete_model(endpoint, port, model_id): return response.json() +def warmup_operation(endpoint, port, index): + """ + Performs warmup operation on index to load native library files + of that index to reduce query latencies. + Args: + endpoint: Endpoint OpenSearch is running on + port: Port OpenSearch is running on + index: index name + Returns: + number of shards the plugin succeeded and failed to warm up. + """ + response = requests.get('http://' + endpoint + ':' + str(port) + + '/_plugins/_knn/warmup/' + index, + headers={'content-type': 'application/json'}) + return response.json() + + def get_opensearch_client(endpoint: str, port: int, timeout=60): """ Get an opensearch client from an endpoint and port @@ -947,7 +981,7 @@ def query_index(opensearch: OpenSearch, index_name: str, body: dict, def bulk_index(opensearch: OpenSearch, index_name: str, body: List): - return opensearch.bulk(index=index_name, body=body, timeout='5m') + return opensearch.bulk(index=index_name, body=body) def get_segment_stats(opensearch: OpenSearch, index_name: str): return opensearch.indices.segments(index=index_name) diff --git a/benchmarks/perf-tool/release-configs/faiss-hnsw/filtering/relaxed-filter/index.json b/benchmarks/perf-tool/release-configs/faiss-hnsw/filtering/relaxed-filter/index.json index b8f591176..7e8ddda8e 100644 --- a/benchmarks/perf-tool/release-configs/faiss-hnsw/filtering/relaxed-filter/index.json +++ b/benchmarks/perf-tool/release-configs/faiss-hnsw/filtering/relaxed-filter/index.json @@ -3,7 +3,8 @@ "index": { "knn": true, "number_of_shards": 24, - "number_of_replicas": 1 + "number_of_replicas": 1, + "knn.algo_param.ef_search": 100 } }, "mappings": { diff --git a/benchmarks/perf-tool/release-configs/faiss-hnsw/filtering/relaxed-filter/relaxed-filter-test.yml b/benchmarks/perf-tool/release-configs/faiss-hnsw/filtering/relaxed-filter/relaxed-filter-test.yml index 3445634b2..ba8850e1d 100644 --- a/benchmarks/perf-tool/release-configs/faiss-hnsw/filtering/relaxed-filter/relaxed-filter-test.yml +++ b/benchmarks/perf-tool/release-configs/faiss-hnsw/filtering/relaxed-filter/relaxed-filter-test.yml @@ -1,24 +1,30 @@ endpoint: [ENDPOINT] +port: [PORT] test_name: "Faiss HNSW Relaxed Filter Test" test_id: "Faiss HNSW Relaxed Filter Test" -num_runs: 10 +num_runs: 3 show_runs: false steps: - name: delete_index index_name: target_index - name: create_index index_name: target_index - index_spec: [INDEX_SPEC_PATH]/relaxed-filter/index.json + index_spec: release-configs/faiss-hnsw/filtering/relaxed-filter/index.json - name: ingest_multi_field index_name: target_index field_name: target_field bulk_size: 500 dataset_format: hdf5 - dataset_path: [DATASET_PATH]/sift-128-euclidean-with-attr.hdf5 + dataset_path: dataset/sift-128-euclidean-with-attr.hdf5 attributes_dataset_name: attributes attribute_spec: [ { name: 'color', type: 'str' }, { name: 'taste', type: 'str' }, { name: 'age', type: 'int' } ] - name: refresh_index index_name: target_index + - name: force_merge + index_name: target_index + max_num_segments: 1 + - name: warmup_operation + index_name: target_index - name: query_with_filter k: 100 r: 1 @@ -26,9 +32,9 @@ steps: index_name: target_index field_name: target_field dataset_format: hdf5 - dataset_path: [DATASET_PATH]/sift-128-euclidean-with-attr.hdf5 + dataset_path: dataset/sift-128-euclidean-with-attr.hdf5 neighbors_format: hdf5 - neighbors_path: [DATASET_PATH]/sift-128-euclidean-with-filters-updated.hdf5 + neighbors_path: dataset/sift-128-euclidean-with-relaxed-filters.hdf5 neighbors_dataset: neighbors_filter_5 - filter_spec: [INDEX_SPEC_PATH]/relaxed-filter-spec.json + filter_spec: release-configs/faiss-hnsw/filtering/relaxed-filter/relaxed-filter-spec.json filter_type: FILTER diff --git a/benchmarks/perf-tool/release-configs/faiss-hnsw/filtering/restrictive-filter/index.json b/benchmarks/perf-tool/release-configs/faiss-hnsw/filtering/restrictive-filter/index.json index b8f591176..7e8ddda8e 100644 --- a/benchmarks/perf-tool/release-configs/faiss-hnsw/filtering/restrictive-filter/index.json +++ b/benchmarks/perf-tool/release-configs/faiss-hnsw/filtering/restrictive-filter/index.json @@ -3,7 +3,8 @@ "index": { "knn": true, "number_of_shards": 24, - "number_of_replicas": 1 + "number_of_replicas": 1, + "knn.algo_param.ef_search": 100 } }, "mappings": { diff --git a/benchmarks/perf-tool/release-configs/faiss-hnsw/filtering/restrictive-filter/restrictive-filter-test.yml b/benchmarks/perf-tool/release-configs/faiss-hnsw/filtering/restrictive-filter/restrictive-filter-test.yml index bf02144ac..94f4073c7 100644 --- a/benchmarks/perf-tool/release-configs/faiss-hnsw/filtering/restrictive-filter/restrictive-filter-test.yml +++ b/benchmarks/perf-tool/release-configs/faiss-hnsw/filtering/restrictive-filter/restrictive-filter-test.yml @@ -1,20 +1,21 @@ endpoint: [ENDPOINT] +port: [PORT] test_name: "Faiss HNSW Restrictive Filter Test" test_id: "Faiss HNSW Restrictive Filter Test" -num_runs: 10 +num_runs: 3 show_runs: false steps: - name: delete_index index_name: target_index - name: create_index index_name: target_index - index_spec: [INDEX_SPEC_PATH]/index.json + index_spec: release-configs/faiss-hnsw/filtering/restrictive-filter/index.json - name: ingest_multi_field index_name: target_index field_name: target_field bulk_size: 500 dataset_format: hdf5 - dataset_path: [DATASET_PATH]/sift-128-euclidean-with-attr.hdf5 + dataset_path: dataset/sift-128-euclidean-with-attr.hdf5 attributes_dataset_name: attributes attribute_spec: [ { name: 'color', type: 'str' }, { name: 'taste', type: 'str' }, { name: 'age', type: 'int' } ] - name: refresh_index @@ -22,6 +23,8 @@ steps: - name: force_merge index_name: target_index max_num_segments: 1 + - name: warmup_operation + index_name: target_index - name: query_with_filter k: 100 r: 1 @@ -29,9 +32,9 @@ steps: index_name: target_index field_name: target_field dataset_format: hdf5 - dataset_path: [DATASET_PATH]/sift-128-euclidean-with-attr.hdf5 + dataset_path: dataset/sift-128-euclidean-with-attr.hdf5 neighbors_format: hdf5 - neighbors_path: [DATASET_PATH]/sift-128-euclidean-with-filters.hdf5 + neighbors_path: dataset/sift-128-euclidean-with-restrictive-filters.hdf5 neighbors_dataset: neighbors_filter_4 - filter_spec: [INDEX_SPEC_PATH]/restrictive-filter-spec.json + filter_spec: release-configs/faiss-hnsw/filtering/restrictive-filter/restrictive-filter-spec.json filter_type: FILTER diff --git a/benchmarks/perf-tool/release-configs/faiss-hnsw/index.json b/benchmarks/perf-tool/release-configs/faiss-hnsw/index.json index b8f591176..7e8ddda8e 100644 --- a/benchmarks/perf-tool/release-configs/faiss-hnsw/index.json +++ b/benchmarks/perf-tool/release-configs/faiss-hnsw/index.json @@ -3,7 +3,8 @@ "index": { "knn": true, "number_of_shards": 24, - "number_of_replicas": 1 + "number_of_replicas": 1, + "knn.algo_param.ef_search": 100 } }, "mappings": { diff --git a/benchmarks/perf-tool/release-configs/faiss-hnsw/nested/simple/index.json b/benchmarks/perf-tool/release-configs/faiss-hnsw/nested/simple/index.json index a982afc81..338ceb1f4 100644 --- a/benchmarks/perf-tool/release-configs/faiss-hnsw/nested/simple/index.json +++ b/benchmarks/perf-tool/release-configs/faiss-hnsw/nested/simple/index.json @@ -8,6 +8,9 @@ } }, "mappings": { + "_source": { + "excludes": ["nested_field"] + }, "properties": { "nested_field": { "type": "nested", diff --git a/benchmarks/perf-tool/release-configs/faiss-hnsw/test.yml b/benchmarks/perf-tool/release-configs/faiss-hnsw/test.yml index f3e976cf3..c4740acf5 100644 --- a/benchmarks/perf-tool/release-configs/faiss-hnsw/test.yml +++ b/benchmarks/perf-tool/release-configs/faiss-hnsw/test.yml @@ -1,25 +1,28 @@ -endpoint: localhost +endpoint: [ENDPOINT] +port: [PORT] test_name: "Faiss HNSW Test" test_id: "Faiss HNSW Test" -num_runs: 10 +num_runs: 3 show_runs: false steps: - name: delete_index index_name: target_index - name: create_index index_name: target_index - index_spec: /home/ec2-user/[PATH]/index.json + index_spec: release-configs/faiss-hnsw/index.json - name: ingest index_name: target_index field_name: target_field bulk_size: 500 dataset_format: hdf5 - dataset_path: [DATASET_PATH]/sift-128-euclidean.hdf5 + dataset_path: dataset/sift-128-euclidean.hdf5 - name: refresh_index index_name: target_index - name: force_merge index_name: target_index max_num_segments: 1 + - name: warmup_operation + index_name: target_index - name: query k: 100 r: 1 @@ -27,6 +30,6 @@ steps: index_name: target_index field_name: target_field dataset_format: hdf5 - dataset_path: [DATASET_PATH]/sift-128-euclidean.hdf5 + dataset_path: dataset/sift-128-euclidean.hdf5 neighbors_format: hdf5 - neighbors_path: [DATASET_PATH]/sift-128-euclidean.hdf5 + neighbors_path: dataset/sift-128-euclidean.hdf5 diff --git a/benchmarks/perf-tool/release-configs/faiss-hnswpq/test.yml b/benchmarks/perf-tool/release-configs/faiss-hnswpq/test.yml index dd88affc3..f573ede9c 100644 --- a/benchmarks/perf-tool/release-configs/faiss-hnswpq/test.yml +++ b/benchmarks/perf-tool/release-configs/faiss-hnswpq/test.yml @@ -1,20 +1,21 @@ endpoint: [ENDPOINT] -test_name: "index-workflow" -test_id: "index workflow" -num_runs: 10 +port: [PORT] +test_name: "Faiss HNSW PQ Test" +test_id: "Faiss HNSW PQ Test" +num_runs: 3 show_runs: false setup: - name: delete_index index_name: train_index - name: create_index index_name: train_index - index_spec: /home/ec2-user/[PATH]/train-index-spec.json + index_spec: release-configs/faiss-hnswpq/train-index-spec.json - name: ingest index_name: train_index field_name: train_field bulk_size: 500 dataset_format: hdf5 - dataset_path: /home/ec2-user/data/sift-128-euclidean.hdf5 + dataset_path: dataset/sift-128-euclidean.hdf5 doc_count: 50000 - name: refresh_index index_name: train_index @@ -28,19 +29,24 @@ steps: train_index: train_index train_field: train_field dimension: 128 - method_spec: /home/ec2-user/[PATH]/method-spec.json + method_spec: release-configs/faiss-hnswpq/method-spec.json max_training_vector_count: 50000 - name: create_index index_name: target_index - index_spec: /home/ec2-user/[PATH]/index.json + index_spec: release-configs/faiss-hnswpq/index.json - name: ingest index_name: target_index field_name: target_field bulk_size: 500 dataset_format: hdf5 - dataset_path: /home/ec2-user/data/sift-128-euclidean.hdf5 + dataset_path: dataset/sift-128-euclidean.hdf5 - name: refresh_index index_name: target_index + - name: force_merge + index_name: target_index + max_num_segments: 1 + - name: warmup_operation + index_name: target_index - name: query k: 100 r: 1 @@ -48,6 +54,6 @@ steps: index_name: target_index field_name: target_field dataset_format: hdf5 - dataset_path: /home/ec2-user/data/sift-128-euclidean.hdf5 + dataset_path: dataset/sift-128-euclidean.hdf5 neighbors_format: hdf5 - neighbors_path: /home/ec2-user/data/sift-128-euclidean.hdf5 + neighbors_path: dataset/sift-128-euclidean.hdf5 diff --git a/benchmarks/perf-tool/release-configs/faiss-ivf/filtering/relaxed-filter/index.json b/benchmarks/perf-tool/release-configs/faiss-ivf/filtering/relaxed-filter/index.json new file mode 100644 index 000000000..ade7fa377 --- /dev/null +++ b/benchmarks/perf-tool/release-configs/faiss-ivf/filtering/relaxed-filter/index.json @@ -0,0 +1,17 @@ +{ + "settings": { + "index": { + "knn": true, + "number_of_shards": 24, + "number_of_replicas": 1 + } + }, + "mappings": { + "properties": { + "target_field": { + "type": "knn_vector", + "model_id": "test-model" + } + } + } +} diff --git a/benchmarks/perf-tool/release-configs/faiss-ivf/filtering/relaxed-filter/method-spec.json b/benchmarks/perf-tool/release-configs/faiss-ivf/filtering/relaxed-filter/method-spec.json new file mode 100644 index 000000000..51ae89877 --- /dev/null +++ b/benchmarks/perf-tool/release-configs/faiss-ivf/filtering/relaxed-filter/method-spec.json @@ -0,0 +1,9 @@ +{ + "name":"ivf", + "engine":"faiss", + "space_type": "l2", + "parameters":{ + "nlist": 128, + "nprobes": 8 + } +} diff --git a/benchmarks/perf-tool/release-configs/faiss-ivf/filtering/relaxed-filter/relaxed-filter-spec.json b/benchmarks/perf-tool/release-configs/faiss-ivf/filtering/relaxed-filter/relaxed-filter-spec.json new file mode 100644 index 000000000..3e04d12c4 --- /dev/null +++ b/benchmarks/perf-tool/release-configs/faiss-ivf/filtering/relaxed-filter/relaxed-filter-spec.json @@ -0,0 +1,42 @@ +{ + "bool": + { + "should": + [ + { + "range": + { + "age": + { + "gte": 30, + "lte": 70 + } + } + }, + { + "term": + { + "color": "green" + } + }, + { + "term": + { + "color": "blue" + } + }, + { + "term": + { + "color": "yellow" + } + }, + { + "term": + { + "taste": "sweet" + } + } + ] + } +} diff --git a/benchmarks/perf-tool/release-configs/faiss-ivf/filtering/relaxed-filter/relaxed-filter-test.yml b/benchmarks/perf-tool/release-configs/faiss-ivf/filtering/relaxed-filter/relaxed-filter-test.yml new file mode 100644 index 000000000..adb25a04d --- /dev/null +++ b/benchmarks/perf-tool/release-configs/faiss-ivf/filtering/relaxed-filter/relaxed-filter-test.yml @@ -0,0 +1,64 @@ +endpoint: [ENDPOINT] +port: [PORT] +test_name: "Faiss IVF Relaxed Filter Test" +test_id: "Faiss IVF Relaxed Filter Test" +num_runs: 3 +show_runs: false +setup: + - name: delete_index + index_name: train_index + - name: create_index + index_name: train_index + index_spec: release-configs/faiss-ivf/filtering/relaxed-filter/train-index-spec.json + - name: ingest + index_name: train_index + field_name: train_field + bulk_size: 500 + dataset_format: hdf5 + dataset_path: dataset/sift-128-euclidean.hdf5 + doc_count: 50000 + - name: refresh_index + index_name: train_index +steps: + - name: delete_model + model_id: test-model + - name: delete_index + index_name: target_index + - name: train_model + model_id: test-model + train_index: train_index + train_field: train_field + dimension: 128 + method_spec: release-configs/faiss-ivf/filtering/relaxed-filter/method-spec.json + max_training_vector_count: 50000 + - name: create_index + index_name: target_index + index_spec: release-configs/faiss-ivf/filtering/relaxed-filter/index.json + - name: ingest_multi_field + index_name: target_index + field_name: target_field + bulk_size: 500 + dataset_format: hdf5 + dataset_path: dataset/sift-128-euclidean-with-attr.hdf5 + attributes_dataset_name: attributes + attribute_spec: [ { name: 'color', type: 'str' }, { name: 'taste', type: 'str' }, { name: 'age', type: 'int' } ] + - name: refresh_index + index_name: target_index + - name: force_merge + index_name: target_index + max_num_segments: 1 + - name: warmup_operation + index_name: target_index + - name: query_with_filter + k: 100 + r: 1 + calculate_recall: true + index_name: target_index + field_name: target_field + dataset_format: hdf5 + dataset_path: dataset/sift-128-euclidean-with-attr.hdf5 + neighbors_format: hdf5 + neighbors_path: dataset/sift-128-euclidean-with-relaxed-filters.hdf5 + neighbors_dataset: neighbors_filter_5 + filter_spec: release-configs/faiss-ivf/filtering/relaxed-filter/relaxed-filter-spec.json + filter_type: FILTER diff --git a/benchmarks/perf-tool/release-configs/faiss-ivf/filtering/relaxed-filter/train-index-spec.json b/benchmarks/perf-tool/release-configs/faiss-ivf/filtering/relaxed-filter/train-index-spec.json new file mode 100644 index 000000000..137fac9d8 --- /dev/null +++ b/benchmarks/perf-tool/release-configs/faiss-ivf/filtering/relaxed-filter/train-index-spec.json @@ -0,0 +1,16 @@ +{ + "settings": { + "index": { + "number_of_shards": 24, + "number_of_replicas": 1 + } + }, + "mappings": { + "properties": { + "train_field": { + "type": "knn_vector", + "dimension": 128 + } + } + } +} diff --git a/benchmarks/perf-tool/release-configs/faiss-ivf/filtering/restrictive-filter/index.json b/benchmarks/perf-tool/release-configs/faiss-ivf/filtering/restrictive-filter/index.json new file mode 100644 index 000000000..ade7fa377 --- /dev/null +++ b/benchmarks/perf-tool/release-configs/faiss-ivf/filtering/restrictive-filter/index.json @@ -0,0 +1,17 @@ +{ + "settings": { + "index": { + "knn": true, + "number_of_shards": 24, + "number_of_replicas": 1 + } + }, + "mappings": { + "properties": { + "target_field": { + "type": "knn_vector", + "model_id": "test-model" + } + } + } +} diff --git a/benchmarks/perf-tool/release-configs/faiss-ivf/filtering/restrictive-filter/method-spec.json b/benchmarks/perf-tool/release-configs/faiss-ivf/filtering/restrictive-filter/method-spec.json new file mode 100644 index 000000000..51ae89877 --- /dev/null +++ b/benchmarks/perf-tool/release-configs/faiss-ivf/filtering/restrictive-filter/method-spec.json @@ -0,0 +1,9 @@ +{ + "name":"ivf", + "engine":"faiss", + "space_type": "l2", + "parameters":{ + "nlist": 128, + "nprobes": 8 + } +} diff --git a/benchmarks/perf-tool/release-configs/faiss-ivf/filtering/restrictive-filter/restrictive-filter-spec.json b/benchmarks/perf-tool/release-configs/faiss-ivf/filtering/restrictive-filter/restrictive-filter-spec.json new file mode 100644 index 000000000..9e6356f1c --- /dev/null +++ b/benchmarks/perf-tool/release-configs/faiss-ivf/filtering/restrictive-filter/restrictive-filter-spec.json @@ -0,0 +1,44 @@ +{ + "bool": + { + "must": + [ + { + "range": + { + "age": + { + "gte": 30, + "lte": 60 + } + } + }, + { + "term": + { + "taste": "bitter" + } + }, + { + "bool": + { + "should": + [ + { + "term": + { + "color": "blue" + } + }, + { + "term": + { + "color": "green" + } + } + ] + } + } + ] + } +} \ No newline at end of file diff --git a/benchmarks/perf-tool/release-configs/faiss-ivf/filtering/restrictive-filter/restrictive-filter-test.yml b/benchmarks/perf-tool/release-configs/faiss-ivf/filtering/restrictive-filter/restrictive-filter-test.yml new file mode 100644 index 000000000..bad047eab --- /dev/null +++ b/benchmarks/perf-tool/release-configs/faiss-ivf/filtering/restrictive-filter/restrictive-filter-test.yml @@ -0,0 +1,64 @@ +endpoint: [ENDPOINT] +port: [PORT] +test_name: "Faiss IVF restrictive Filter Test" +test_id: "Faiss IVF restrictive Filter Test" +num_runs: 3 +show_runs: false +setup: + - name: delete_index + index_name: train_index + - name: create_index + index_name: train_index + index_spec: release-configs/faiss-ivf/filtering/restrictive-filter/train-index-spec.json + - name: ingest + index_name: train_index + field_name: train_field + bulk_size: 500 + dataset_format: hdf5 + dataset_path: dataset/sift-128-euclidean.hdf5 + doc_count: 50000 + - name: refresh_index + index_name: train_index +steps: + - name: delete_model + model_id: test-model + - name: delete_index + index_name: target_index + - name: train_model + model_id: test-model + train_index: train_index + train_field: train_field + dimension: 128 + method_spec: release-configs/faiss-ivf/filtering/restrictive-filter/method-spec.json + max_training_vector_count: 50000 + - name: create_index + index_name: target_index + index_spec: release-configs/faiss-ivf/filtering/restrictive-filter/index.json + - name: ingest_multi_field + index_name: target_index + field_name: target_field + bulk_size: 500 + dataset_format: hdf5 + dataset_path: dataset/sift-128-euclidean-with-attr.hdf5 + attributes_dataset_name: attributes + attribute_spec: [ { name: 'color', type: 'str' }, { name: 'taste', type: 'str' }, { name: 'age', type: 'int' } ] + - name: refresh_index + index_name: target_index + - name: force_merge + index_name: target_index + max_num_segments: 1 + - name: warmup_operation + index_name: target_index + - name: query_with_filter + k: 100 + r: 1 + calculate_recall: true + index_name: target_index + field_name: target_field + dataset_format: hdf5 + dataset_path: dataset/sift-128-euclidean-with-attr.hdf5 + neighbors_format: hdf5 + neighbors_path: dataset/sift-128-euclidean-with-restrictive-filters.hdf5 + neighbors_dataset: neighbors_filter_4 + filter_spec: release-configs/faiss-ivf/filtering/restrictive-filter/restrictive-filter-spec.json + filter_type: FILTER diff --git a/benchmarks/perf-tool/release-configs/faiss-ivf/filtering/restrictive-filter/train-index-spec.json b/benchmarks/perf-tool/release-configs/faiss-ivf/filtering/restrictive-filter/train-index-spec.json new file mode 100644 index 000000000..804a5707e --- /dev/null +++ b/benchmarks/perf-tool/release-configs/faiss-ivf/filtering/restrictive-filter/train-index-spec.json @@ -0,0 +1,16 @@ +{ + "settings": { + "index": { + "number_of_shards": 24, + "number_of_replicas": 0 + } + }, + "mappings": { + "properties": { + "train_field": { + "type": "knn_vector", + "dimension": 128 + } + } + } +} diff --git a/benchmarks/perf-tool/release-configs/faiss-ivf/test.yml b/benchmarks/perf-tool/release-configs/faiss-ivf/test.yml index ce6b78867..367c42594 100644 --- a/benchmarks/perf-tool/release-configs/faiss-ivf/test.yml +++ b/benchmarks/perf-tool/release-configs/faiss-ivf/test.yml @@ -1,20 +1,21 @@ -endpoint: [END-POINT] -test_name: "index-workflow" -test_id: "index workflow" -num_runs: 10 +endpoint: [ENDPOINT] +port: [PORT] +test_name: "Faiss IVF" +test_id: "Faiss IVF" +num_runs: 3 show_runs: false setup: - name: delete_index index_name: train_index - name: create_index index_name: train_index - index_spec: /home/ec2-user/[PATH]/train-index-spec.json + index_spec: release-configs/faiss-ivf/train-index-spec.json - name: ingest index_name: train_index field_name: train_field bulk_size: 500 dataset_format: hdf5 - dataset_path: /home/ec2-user/[PATH]/sift-128-euclidean.hdf5 + dataset_path: dataset/sift-128-euclidean.hdf5 doc_count: 50000 - name: refresh_index index_name: train_index @@ -28,19 +29,24 @@ steps: train_index: train_index train_field: train_field dimension: 128 - method_spec: /home/ec2-user/[PATH]/method-spec.json + method_spec: release-configs/faiss-ivf/method-spec.json max_training_vector_count: 50000 - name: create_index index_name: target_index - index_spec: /home/ec2-user/[PATH]/index.json + index_spec: release-configs/faiss-ivf/index.json - name: ingest index_name: target_index field_name: target_field bulk_size: 500 dataset_format: hdf5 - dataset_path: /home/ec2-user/data/sift-128-euclidean.hdf5 + dataset_path: dataset/sift-128-euclidean.hdf5 - name: refresh_index index_name: target_index + - name: force_merge + index_name: target_index + max_num_segments: 1 + - name: warmup_operation + index_name: target_index - name: query k: 100 r: 1 @@ -48,6 +54,6 @@ steps: index_name: target_index field_name: target_field dataset_format: hdf5 - dataset_path: /home/ec2-user/data/sift-128-euclidean.hdf5 + dataset_path: dataset/sift-128-euclidean.hdf5 neighbors_format: hdf5 - neighbors_path: /home/ec2-user/data/sift-128-euclidean.hdf5 + neighbors_path: dataset/sift-128-euclidean.hdf5 diff --git a/benchmarks/perf-tool/release-configs/faiss-ivfpq/test.yml b/benchmarks/perf-tool/release-configs/faiss-ivfpq/test.yml index dd88affc3..c3f63348b 100644 --- a/benchmarks/perf-tool/release-configs/faiss-ivfpq/test.yml +++ b/benchmarks/perf-tool/release-configs/faiss-ivfpq/test.yml @@ -1,20 +1,21 @@ endpoint: [ENDPOINT] -test_name: "index-workflow" -test_id: "index workflow" -num_runs: 10 +port: [PORT] +test_name: "Faiss IVF PQ Test" +test_id: "Faiss IVF PQ Test" +num_runs: 3 show_runs: false setup: - name: delete_index index_name: train_index - name: create_index index_name: train_index - index_spec: /home/ec2-user/[PATH]/train-index-spec.json + index_spec: release-configs/faiss-ivfpq/train-index-spec.json - name: ingest index_name: train_index field_name: train_field bulk_size: 500 dataset_format: hdf5 - dataset_path: /home/ec2-user/data/sift-128-euclidean.hdf5 + dataset_path: dataset/sift-128-euclidean.hdf5 doc_count: 50000 - name: refresh_index index_name: train_index @@ -28,19 +29,24 @@ steps: train_index: train_index train_field: train_field dimension: 128 - method_spec: /home/ec2-user/[PATH]/method-spec.json + method_spec: release-configs/faiss-ivfpq/method-spec.json max_training_vector_count: 50000 - name: create_index index_name: target_index - index_spec: /home/ec2-user/[PATH]/index.json + index_spec: release-configs/faiss-ivfpq/index.json - name: ingest index_name: target_index field_name: target_field bulk_size: 500 dataset_format: hdf5 - dataset_path: /home/ec2-user/data/sift-128-euclidean.hdf5 + dataset_path: dataset/sift-128-euclidean.hdf5 - name: refresh_index index_name: target_index + - name: force_merge + index_name: target_index + max_num_segments: 1 + - name: warmup_operation + index_name: target_index - name: query k: 100 r: 1 @@ -48,6 +54,6 @@ steps: index_name: target_index field_name: target_field dataset_format: hdf5 - dataset_path: /home/ec2-user/data/sift-128-euclidean.hdf5 + dataset_path: dataset/sift-128-euclidean.hdf5 neighbors_format: hdf5 - neighbors_path: /home/ec2-user/data/sift-128-euclidean.hdf5 + neighbors_path: dataset/sift-128-euclidean.hdf5 diff --git a/benchmarks/perf-tool/release-configs/lucene-hnsw/filtering/relaxed-filter/relaxed-filter-spec.json b/benchmarks/perf-tool/release-configs/lucene-hnsw/filtering/relaxed-filter/relaxed-filter-spec.json index fecde0392..3e04d12c4 100644 --- a/benchmarks/perf-tool/release-configs/lucene-hnsw/filtering/relaxed-filter/relaxed-filter-spec.json +++ b/benchmarks/perf-tool/release-configs/lucene-hnsw/filtering/relaxed-filter/relaxed-filter-spec.json @@ -34,9 +34,9 @@ { "term": { - "color": "sweet" + "taste": "sweet" } } ] } -} \ No newline at end of file +} diff --git a/benchmarks/perf-tool/release-configs/lucene-hnsw/filtering/relaxed-filter/relaxed-filter-test.yml b/benchmarks/perf-tool/release-configs/lucene-hnsw/filtering/relaxed-filter/relaxed-filter-test.yml index 62d032b00..3bbb99a0f 100644 --- a/benchmarks/perf-tool/release-configs/lucene-hnsw/filtering/relaxed-filter/relaxed-filter-test.yml +++ b/benchmarks/perf-tool/release-configs/lucene-hnsw/filtering/relaxed-filter/relaxed-filter-test.yml @@ -1,24 +1,28 @@ endpoint: [ENDPOINT] +port: [PORT] test_name: "Lucene HNSW Relaxed Filter Test" test_id: "Lucene HNSW Relaxed Filter Test" -num_runs: 10 +num_runs: 3 show_runs: false steps: - name: delete_index index_name: target_index - name: create_index index_name: target_index - index_spec: [INDEX_SPEC_PATH]/index.json + index_spec: release-configs/lucene-hnsw/filtering/relaxed-filter/index.json - name: ingest_multi_field index_name: target_index field_name: target_field bulk_size: 500 dataset_format: hdf5 - dataset_path: [DATASET_PATH]/sift-128-euclidean-with-attr.hdf5 + dataset_path: dataset/sift-128-euclidean-with-attr.hdf5 attributes_dataset_name: attributes attribute_spec: [ { name: 'color', type: 'str' }, { name: 'taste', type: 'str' }, { name: 'age', type: 'int' } ] - name: refresh_index index_name: target_index + - name: force_merge + index_name: target_index + max_num_segments: 1 - name: query_with_filter k: 100 r: 1 @@ -26,9 +30,9 @@ steps: index_name: target_index field_name: target_field dataset_format: hdf5 - dataset_path: [DATASET_PATH]/sift-128-euclidean-with-attr.hdf5 + dataset_path: dataset/sift-128-euclidean-with-attr.hdf5 neighbors_format: hdf5 - neighbors_path: [DATASET_PATH]/sift-128-euclidean-with-filters-updated.hdf5 + neighbors_path: dataset/sift-128-euclidean-with-relaxed-filters.hdf5 neighbors_dataset: neighbors_filter_5 - filter_spec: [INDEX_SPEC_PATH]/relaxed-filter-spec.json + filter_spec: release-configs/lucene-hnsw/filtering/relaxed-filter/relaxed-filter-spec.json filter_type: FILTER diff --git a/benchmarks/perf-tool/release-configs/lucene-hnsw/filtering/restrictive-filter/restrictive-filter-test.yml b/benchmarks/perf-tool/release-configs/lucene-hnsw/filtering/restrictive-filter/restrictive-filter-test.yml index d7f451a48..aa4c5193f 100644 --- a/benchmarks/perf-tool/release-configs/lucene-hnsw/filtering/restrictive-filter/restrictive-filter-test.yml +++ b/benchmarks/perf-tool/release-configs/lucene-hnsw/filtering/restrictive-filter/restrictive-filter-test.yml @@ -1,20 +1,21 @@ endpoint: [ENDPOINT] +port: [PORT] test_name: "Lucene HNSW Restrictive Filter Test" test_id: "Lucene HNSW Restrictive Filter Test" -num_runs: 10 +num_runs: 3 show_runs: false steps: - name: delete_index index_name: target_index - name: create_index index_name: target_index - index_spec: /home/ec2-user/k-NN/benchmarks/perf-tool/release-configs/lucene-hnsw/filtering/restrictive-filter/index.json + index_spec: release-configs/lucene-hnsw/filtering/restrictive-filter/index.json - name: ingest_multi_field index_name: target_index field_name: target_field bulk_size: 500 dataset_format: hdf5 - dataset_path: /home/ec2-user/k-NN/benchmarks/perf-tool/dataset/sift-128-euclidean-with-attr.hdf5 + dataset_path: dataset/sift-128-euclidean-with-attr.hdf5 attributes_dataset_name: attributes attribute_spec: [ { name: 'color', type: 'str' }, { name: 'taste', type: 'str' }, { name: 'age', type: 'int' } ] - name: refresh_index @@ -29,9 +30,9 @@ steps: index_name: target_index field_name: target_field dataset_format: hdf5 - dataset_path: /home/ec2-user/k-NN/benchmarks/perf-tool/dataset/sift-128-euclidean-with-attr.hdf5 + dataset_path: dataset/sift-128-euclidean-with-attr.hdf5 neighbors_format: hdf5 - neighbors_path: /home/ec2-user/k-NN/benchmarks/perf-tool/dataset/sift-128-euclidean-with-filters.hdf5 + neighbors_path: dataset/sift-128-euclidean-with-restrictive-filters.hdf5 neighbors_dataset: neighbors_filter_4 - filter_spec: /home/ec2-user/k-NN/benchmarks/perf-tool/release-configs/lucene-hnsw/filtering/restrictive-filter/restrictive-filter-spec.json + filter_spec: release-configs/lucene-hnsw/filtering/restrictive-filter/restrictive-filter-spec.json filter_type: FILTER diff --git a/benchmarks/perf-tool/release-configs/lucene-hnsw/nested/simple/index.json b/benchmarks/perf-tool/release-configs/lucene-hnsw/nested/simple/index.json index 8dc749c39..b41b51c77 100644 --- a/benchmarks/perf-tool/release-configs/lucene-hnsw/nested/simple/index.json +++ b/benchmarks/perf-tool/release-configs/lucene-hnsw/nested/simple/index.json @@ -7,6 +7,9 @@ } }, "mappings": { + "_source": { + "excludes": ["nested_field"] + }, "properties": { "nested_field": { "type": "nested", diff --git a/benchmarks/perf-tool/release-configs/lucene-hnsw/nested/simple/simple-nested-test.yml b/benchmarks/perf-tool/release-configs/lucene-hnsw/nested/simple/simple-nested-test.yml index cf1e4edc4..be825487a 100644 --- a/benchmarks/perf-tool/release-configs/lucene-hnsw/nested/simple/simple-nested-test.yml +++ b/benchmarks/perf-tool/release-configs/lucene-hnsw/nested/simple/simple-nested-test.yml @@ -9,7 +9,7 @@ steps: index_name: target_index - name: create_index index_name: target_index - index_spec: release-configs/faiss-hnsw/nested/simple/index.json + index_spec: release-configs/lucene-hnsw/nested/simple/index.json - name: ingest_nested_field index_name: target_index field_name: target_field diff --git a/benchmarks/perf-tool/release-configs/lucene-hnsw/test.yml b/benchmarks/perf-tool/release-configs/lucene-hnsw/test.yml index 96b991325..b253ee08e 100644 --- a/benchmarks/perf-tool/release-configs/lucene-hnsw/test.yml +++ b/benchmarks/perf-tool/release-configs/lucene-hnsw/test.yml @@ -1,22 +1,26 @@ endpoint: [ENDPOINT] -test_name: "index-workflow" -test_id: "Index workflow" -num_runs: 10 +port: [PORT] +test_name: "Lucene HNSW" +test_id: "Lucene HNSW" +num_runs: 3 show_runs: false steps: - name: delete_index index_name: target_index - name: create_index index_name: target_index - index_spec: /home/ec2-user/[PATH]/index.json + index_spec: release-configs/lucene-hnsw/index.json - name: ingest index_name: target_index field_name: target_field bulk_size: 500 dataset_format: hdf5 - dataset_path: /home/ec2-user/data/sift-128-euclidean.hdf5 + dataset_path: dataset/sift-128-euclidean.hdf5 - name: refresh_index index_name: target_index + - name: force_merge + index_name: target_index + max_num_segments: 1 - name: query k: 100 r: 1 @@ -24,6 +28,6 @@ steps: index_name: target_index field_name: target_field dataset_format: hdf5 - dataset_path: /home/ec2-user/data/sift-128-euclidean.hdf5 + dataset_path: dataset/sift-128-euclidean.hdf5 neighbors_format: hdf5 - neighbors_path: /home/ec2-user/data/sift-128-euclidean.hdf5 + neighbors_path: dataset/sift-128-euclidean.hdf5 diff --git a/benchmarks/perf-tool/release-configs/nmslib-hnsw/index.json b/benchmarks/perf-tool/release-configs/nmslib-hnsw/index.json index fa88192f6..eb714c5c8 100644 --- a/benchmarks/perf-tool/release-configs/nmslib-hnsw/index.json +++ b/benchmarks/perf-tool/release-configs/nmslib-hnsw/index.json @@ -4,7 +4,7 @@ "knn": true, "number_of_shards": 24, "number_of_replicas": 1, - "knn.algo_param.ef_search": 256 + "knn.algo_param.ef_search": 100 } }, "mappings": { diff --git a/benchmarks/perf-tool/release-configs/nmslib-hnsw/test.yml b/benchmarks/perf-tool/release-configs/nmslib-hnsw/test.yml index 96b991325..94ad9b131 100644 --- a/benchmarks/perf-tool/release-configs/nmslib-hnsw/test.yml +++ b/benchmarks/perf-tool/release-configs/nmslib-hnsw/test.yml @@ -1,22 +1,28 @@ endpoint: [ENDPOINT] -test_name: "index-workflow" -test_id: "Index workflow" -num_runs: 10 +port: [PORT] +test_name: "Nmslib HNSW Test" +test_id: "Nmslib HNSW Test" +num_runs: 3 show_runs: false steps: - name: delete_index index_name: target_index - name: create_index index_name: target_index - index_spec: /home/ec2-user/[PATH]/index.json + index_spec: release-configs/nmslib-hnsw/index.json - name: ingest index_name: target_index field_name: target_field bulk_size: 500 dataset_format: hdf5 - dataset_path: /home/ec2-user/data/sift-128-euclidean.hdf5 + dataset_path: dataset/sift-128-euclidean.hdf5 - name: refresh_index index_name: target_index + - name: force_merge + index_name: target_index + max_num_segments: 1 + - name: warmup_operation + index_name: target_index - name: query k: 100 r: 1 @@ -24,6 +30,6 @@ steps: index_name: target_index field_name: target_field dataset_format: hdf5 - dataset_path: /home/ec2-user/data/sift-128-euclidean.hdf5 + dataset_path: dataset/sift-128-euclidean.hdf5 neighbors_format: hdf5 - neighbors_path: /home/ec2-user/data/sift-128-euclidean.hdf5 + neighbors_path: dataset/sift-128-euclidean.hdf5 diff --git a/benchmarks/perf-tool/release-configs/run_all_tests.sh b/benchmarks/perf-tool/release-configs/run_all_tests.sh new file mode 100755 index 000000000..e65d5b5c4 --- /dev/null +++ b/benchmarks/perf-tool/release-configs/run_all_tests.sh @@ -0,0 +1,102 @@ +#!/bin/bash +set -e + +# Description: +# Run a performance test for release +# Dataset should be available in perf-tool/dataset before running this script +# +# Example: +# ./run-test.sh --endpoint localhost +# +# Usage: +# ./run-test.sh \ +# --endpoint +# --port 80 \ +# --num-runs 3 \ +# --outputs ~/outputs + +while [ "$1" != "" ]; do + case $1 in + -url | --endpoint ) shift + ENDPOINT=$1 + ;; + -p | --port ) shift + PORT=$1 + ;; + -n | --num-runs ) shift + NUM_RUNS=$1 + ;; + -o | --outputs ) shift + OUTPUTS=$1 + ;; + * ) echo "Unknown parameter" + echo $1 + exit 1 + ;; + esac + shift +done + +if [ ! -n "$ENDPOINT" ]; then + echo "--endpoint should be specified" + exit +fi + +if [ ! -n "$PORT" ]; then + PORT=80 + echo "--port is not specified. Using default values $PORT" +fi + +if [ ! -n "$NUM_RUNS" ]; then + NUM_RUNS=3 + echo "--num-runs is not specified. Using default values $NUM_RUNS" +fi + +if [ ! -n "$OUTPUTS" ]; then + OUTPUTS="$HOME/outputs" + echo "--outputs is not specified. Using default values $OUTPUTS" +fi + + +curl -X PUT "http://$ENDPOINT:$PORT/_cluster/settings?pretty" -H 'Content-Type: application/json' -d' +{ + "persistent" : { + "knn.algo_param.index_thread_qty" : 4 + } +} +' + +TESTS="./release-configs/faiss-hnsw/filtering/relaxed-filter/relaxed-filter-test.yml +./release-configs/faiss-hnsw/filtering/restrictive-filter/restrictive-filter-test.yml +./release-configs/faiss-hnsw/nested/simple/simple-nested-test.yml +./release-configs/faiss-hnsw/test.yml +./release-configs/faiss-hnswpq/test.yml +./release-configs/faiss-ivf/filtering/relaxed-filter/relaxed-filter-test.yml +./release-configs/faiss-ivf/filtering/restrictive-filter/restrictive-filter-test.yml +./release-configs/faiss-ivf/test.yml +./release-configs/faiss-ivfpq/test.yml +./release-configs/lucene-hnsw/filtering/relaxed-filter/relaxed-filter-test.yml +./release-configs/lucene-hnsw/filtering/restrictive-filter/restrictive-filter-test.yml +./release-configs/lucene-hnsw/nested/simple/simple-nested-test.yml +./release-configs/lucene-hnsw/test.yml +./release-configs/nmslib-hnsw/test.yml" + +if [ ! -d $OUTPUTS ] +then + mkdir $OUTPUTS +fi + +for TEST in $TESTS +do + ORG_FILE=$TEST + NEW_FILE="$ORG_FILE.tmp" + OUT_FILE=$(grep test_id $ORG_FILE | cut -d':' -f2 | sed -r 's/^ "|"$//g' | sed 's/ /_/g') + echo "cp $ORG_FILE $NEW_FILE" + cp $ORG_FILE $NEW_FILE + sed -i "/^endpoint:/c\endpoint: $ENDPOINT" $NEW_FILE + sed -i "/^port:/c\port: $PORT" $NEW_FILE + sed -i "/^num_runs:/c\num_runs: $NUM_RUNS" $NEW_FILE + python3 knn-perf-tool.py test $NEW_FILE $OUTPUTS/$OUT_FILE + #Sleep for 1 min to cool down cpu from the previous run + sleep 60 +done diff --git a/benchmarks/perf-tool/sample-configs/faiss-sift-ivf/test.yml b/benchmarks/perf-tool/sample-configs/faiss-sift-ivf/test.yml index c8fb42ec4..027ba8683 100644 --- a/benchmarks/perf-tool/sample-configs/faiss-sift-ivf/test.yml +++ b/benchmarks/perf-tool/sample-configs/faiss-sift-ivf/test.yml @@ -43,6 +43,8 @@ steps: - name: force_merge index_name: target_index max_num_segments: 10 + - name: warmup_operation + index_name: target_index - name: query k: 100 r: 1 diff --git a/benchmarks/perf-tool/sample-configs/nmslib-sift-hnsw/test.yml b/benchmarks/perf-tool/sample-configs/nmslib-sift-hnsw/test.yml index deea1ad47..6d96bf80c 100644 --- a/benchmarks/perf-tool/sample-configs/nmslib-sift-hnsw/test.yml +++ b/benchmarks/perf-tool/sample-configs/nmslib-sift-hnsw/test.yml @@ -21,6 +21,8 @@ steps: - name: force_merge index_name: target_index max_num_segments: 10 + - name: warmup_operation + index_name: target_index - name: query k: 100 r: 1 From 5d0ceaa75c5868785bbed640ddc19564716efee8 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Tue, 27 Feb 2024 15:54:33 -0600 Subject: [PATCH 205/416] Fix FieldInfo Parameters Mismatch (#1490) (#1492) Signed-off-by: Naveen Tatikonda (cherry picked from commit c6ac3db072affd85c49153bd59d82deaf1c62491) Co-authored-by: Naveen Tatikonda --- CHANGELOG.md | 1 + .../opensearch/knn/index/codec/KNNCodecTestUtil.java | 10 +++++++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b55634b53..93ad21dc9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,4 +20,5 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Documentation ### Maintenance * Bump faiss lib commit to 32f0e8cf92cd2275b60364517bb1cce67aa29a55 [#1443](https://github.com/opensearch-project/k-NN/pull/1443) +* Fix FieldInfo Parameters Mismatch [#1489](https://github.com/opensearch-project/k-NN/pull/1489) ### Refactoring diff --git a/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestUtil.java b/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestUtil.java index e602501cc..b1d9d9434 100644 --- a/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestUtil.java +++ b/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestUtil.java @@ -71,6 +71,7 @@ public static class FieldInfoBuilder { private int vectorDimension; private VectorSimilarityFunction vectorSimilarityFunction; private boolean softDeletes; + private boolean isParentField; public static FieldInfoBuilder builder(String fieldName) { return new FieldInfoBuilder(fieldName); @@ -92,6 +93,7 @@ private FieldInfoBuilder(String fieldName) { this.vectorDimension = 0; this.vectorSimilarityFunction = VectorSimilarityFunction.EUCLIDEAN; this.softDeletes = false; + this.isParentField = false; } public FieldInfoBuilder fieldNumber(int fieldNumber) { @@ -164,6 +166,11 @@ public FieldInfoBuilder softDeletes(boolean softDeletes) { return this; } + public FieldInfoBuilder isParentField(boolean isParentField) { + this.isParentField = isParentField; + return this; + } + public FieldInfo build() { return new FieldInfo( fieldName, @@ -181,7 +188,8 @@ public FieldInfo build() { vectorDimension, VectorEncoding.FLOAT32, vectorSimilarityFunction, - softDeletes + softDeletes, + isParentField ); } } From 513e496e0c9f0d5e12d70dca5c350f0fb6aea9b5 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Tue, 27 Feb 2024 14:32:32 -0800 Subject: [PATCH 206/416] Increment version to 2.13.0-SNAPSHOT (#1470) Signed-off-by: opensearch-ci-bot Co-authored-by: opensearch-ci-bot --- .../workflows/backwards_compatibility_tests_workflow.yml | 8 ++++---- build.gradle | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/backwards_compatibility_tests_workflow.yml b/.github/workflows/backwards_compatibility_tests_workflow.yml index ad90a79f3..8cda91f14 100644 --- a/.github/workflows/backwards_compatibility_tests_workflow.yml +++ b/.github/workflows/backwards_compatibility_tests_workflow.yml @@ -14,8 +14,8 @@ jobs: strategy: matrix: java: [ 11, 17 ] - bwc_version : [ "1.1.0", "1.2.4", "1.3.8", "2.0.1", "2.1.0", "2.2.1", "2.3.0", "2.4.1", "2.5.0", "2.6.0", "2.7.0", "2.8.0", "2.9.0", "2.10.0", "2.11.0" ] - opensearch_version : [ "2.12.0-SNAPSHOT" ] + bwc_version : [ "1.1.0", "1.2.4", "1.3.8", "2.0.1", "2.1.0", "2.2.1", "2.3.0", "2.4.1", "2.5.0", "2.6.0", "2.7.0", "2.8.0", "2.9.0", "2.10.0", "2.11.0", "2.12.0" ] + opensearch_version : [ "2.13.0-SNAPSHOT" ] name: k-NN Restart-Upgrade BWC Tests runs-on: ubuntu-latest @@ -46,8 +46,8 @@ jobs: strategy: matrix: java: [ 11, 17 ] - bwc_version: [ "1.3.8", "2.0.1", "2.1.0", "2.2.1", "2.3.0", "2.4.1", "2.5.0", "2.6.0", "2.7.0", "2.8.0", "2.9.0", "2.10.0", "2.11.0"] - opensearch_version: [ "2.12.0-SNAPSHOT" ] + bwc_version: [ "1.3.8", "2.0.1", "2.1.0", "2.2.1", "2.3.0", "2.4.1", "2.5.0", "2.6.0", "2.7.0", "2.8.0", "2.9.0", "2.10.0", "2.11.0", "2.12.0"] + opensearch_version: [ "2.13.0-SNAPSHOT" ] name: k-NN Rolling-Upgrade BWC Tests runs-on: ubuntu-latest diff --git a/build.gradle b/build.gradle index 1bbed9444..bcec35c65 100644 --- a/build.gradle +++ b/build.gradle @@ -13,7 +13,7 @@ buildscript { ext { // build.version_qualifier parameter applies to knn plugin artifacts only. OpenSearch version must be set // explicitly as 'opensearch.version' property, for instance opensearch.version=2.0.0-rc1-SNAPSHOT - opensearch_version = System.getProperty("opensearch.version", "2.12.0-SNAPSHOT") + opensearch_version = System.getProperty("opensearch.version", "2.13.0-SNAPSHOT") version_qualifier = System.getProperty("build.version_qualifier", "") opensearch_group = "org.opensearch" isSnapshot = "true" == System.getProperty("build.snapshot", "true") From d9bd869f462f720ad19c5fd72b7c9d4820a4ba99 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Mon, 4 Mar 2024 14:18:09 -0800 Subject: [PATCH 207/416] Optimize Faiss Query With Filters: Reduce iteration and memory for id filter (#1402) (#1504) * Optimize Faiss Query With Filters. Reduce iteration copy for docid set iterator Signed-off-by: luyuncheng * Optimize Faiss Query With Filters. Reduce iteration copy for docid set iterator. Use Bitmap And Batch to do id filter. and you sparse or fixed bitset do exact ANN search Signed-off-by: luyuncheng * Using int64_t instead of long type for GetLongArrayElements Signed-off-by: luyuncheng * Add IDSelectorJlongBitmap Signed-off-by: luyuncheng * 1. Add IDSelectorJlongBitmap and UT for it 2. Move FilterIdsSelectorType to a util class Signed-off-by: luyuncheng * 1. Add IDSelectorJlongBitmap and UT for it 2. Move FilterIdsSelectorType to a util class 3. Spotless apply Signed-off-by: luyuncheng * Rebase remote-tracking branch 'origin/main' into Filter Signed-off-by: luyuncheng * tidy Signed-off-by: luyuncheng * Add Changelog Signed-off-by: luyuncheng * fix javadoc tasks Signed-off-by: luyuncheng * fix bwc javadoc Signed-off-by: luyuncheng * UpdatedFilterIdsSelector Signed-off-by: luyuncheng * UpdatedFilterIdsSelector Signed-off-by: luyuncheng * Rebase faiss_wrapper.cpp Signed-off-by: luyuncheng * UpdatedFilterIdsSelector For description Select different FilterIdsSelectorType Signed-off-by: luyuncheng * UpdatedFilterIdsSelector For description Select different FilterIdsSelectorType Signed-off-by: luyuncheng * UpdatedFilterIdsSelector as Byte.SIZE Signed-off-by: luyuncheng * UpdatedFilterIdsSelector For comments Signed-off-by: luyuncheng --------- Signed-off-by: luyuncheng (cherry picked from commit 3eeb855b0e0434c8eda8ae45343775548c067824) Co-authored-by: luyuncheng --- CHANGELOG.md | 1 + jni/include/faiss_wrapper.h | 3 +- jni/include/jni_util.h | 9 ++ .../org_opensearch_knn_jni_FaissService.h | 2 +- jni/src/faiss_wrapper.cpp | 120 +++++------------- jni/src/jni_util.cpp | 26 ++++ .../org_opensearch_knn_jni_FaissService.cpp | 4 +- jni/tests/faiss_wrapper_test.cpp | 14 +- jni/tests/test_util.cpp | 34 +++++ jni/tests/test_util.h | 9 ++ .../knn/index/query/FilterIdsSelector.java | 107 ++++++++++++++++ .../opensearch/knn/index/query/KNNWeight.java | 51 ++++---- .../filtered/FilteredIdsKNNIterator.java | 22 ++-- .../NestedFilteredIdsKNNIterator.java | 16 ++- .../org/opensearch/knn/jni/FaissService.java | 3 +- .../org/opensearch/knn/jni/JNIService.java | 6 +- .../knn/index/codec/KNNCodecTestUtil.java | 2 +- .../memory/NativeMemoryLoadStrategyTests.java | 2 +- .../index/query/FilterIdsSelectorTests.java | 60 +++++++++ .../knn/index/query/KNNWeightTests.java | 29 +++-- .../filtered/FilteredIdsKNNIteratorTests.java | 5 +- .../NestedFilteredIdsKNNIteratorTests.java | 10 +- .../opensearch/knn/jni/JNIServiceTests.java | 25 ++-- 23 files changed, 401 insertions(+), 159 deletions(-) create mode 100644 src/main/java/org/opensearch/knn/index/query/FilterIdsSelector.java create mode 100644 src/test/java/org/opensearch/knn/index/query/FilterIdsSelectorTests.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 93ad21dc9..69b213ad8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ## [Unreleased 2.x](https://github.com/opensearch-project/k-NN/compare/2.12...2.x) ### Features ### Enhancements +* Optize Faiss Query With Filters: Reduce iteration and memory for id filter [#1402](https://github.com/opensearch-project/k-NN/pull/1402) ### Bug Fixes ### Infrastructure ### Documentation diff --git a/jni/include/faiss_wrapper.h b/jni/include/faiss_wrapper.h index 078526000..e99cdafb2 100644 --- a/jni/include/faiss_wrapper.h +++ b/jni/include/faiss_wrapper.h @@ -43,7 +43,8 @@ namespace knn_jni { // // Return an array of KNNQueryResults jobjectArray QueryIndex_WithFilter(knn_jni::JNIUtilInterface * jniUtil, JNIEnv * env, jlong indexPointerJ, - jfloatArray queryVectorJ, jint kJ, jintArray filterIdsJ, jintArray parentIdsJ); + jfloatArray queryVectorJ, jint kJ, jlongArray filterIdsJ, + jint filterIdsTypeJ, jintArray parentIdsJ); // Free the index located in memory at indexPointerJ void Free(jlong indexPointer); diff --git a/jni/include/jni_util.h b/jni/include/jni_util.h index b4dd44891..52b08a202 100644 --- a/jni/include/jni_util.h +++ b/jni/include/jni_util.h @@ -80,6 +80,8 @@ namespace knn_jni { virtual int GetJavaIntArrayLength(JNIEnv *env, jintArray arrayJ) = 0; + virtual int GetJavaLongArrayLength(JNIEnv *env, jlongArray arrayJ) = 0; + virtual int GetJavaBytesArrayLength(JNIEnv *env, jbyteArray arrayJ) = 0; virtual int GetJavaFloatArrayLength(JNIEnv *env, jfloatArray arrayJ) = 0; @@ -94,6 +96,8 @@ namespace knn_jni { virtual jint * GetIntArrayElements(JNIEnv *env, jintArray array, jboolean * isCopy) = 0; + virtual jlong * GetLongArrayElements(JNIEnv *env, jlongArray array, jboolean * isCopy) = 0; + virtual jobject GetObjectArrayElement(JNIEnv *env, jobjectArray array, jsize index) = 0; virtual jobject NewObject(JNIEnv *env, jclass clazz, jmethodID methodId, int id, float distance) = 0; @@ -108,6 +112,8 @@ namespace knn_jni { virtual void ReleaseIntArrayElements(JNIEnv *env, jintArray array, jint *elems, jint mode) = 0; + virtual void ReleaseLongArrayElements(JNIEnv *env, jlongArray array, jlong *elems, jint mode) = 0; + virtual void SetObjectArrayElement(JNIEnv *env, jobjectArray array, jsize index, jobject val) = 0; virtual void SetByteArrayRegion(JNIEnv *env, jbyteArray array, jsize start, jsize len, const jbyte * buf) = 0; @@ -139,6 +145,7 @@ namespace knn_jni { int GetInnerDimensionOf2dJavaFloatArray(JNIEnv *env, jobjectArray array2dJ); int GetJavaObjectArrayLength(JNIEnv *env, jobjectArray arrayJ); int GetJavaIntArrayLength(JNIEnv *env, jintArray arrayJ); + int GetJavaLongArrayLength(JNIEnv *env, jlongArray arrayJ); int GetJavaBytesArrayLength(JNIEnv *env, jbyteArray arrayJ); int GetJavaFloatArrayLength(JNIEnv *env, jfloatArray arrayJ); @@ -146,6 +153,7 @@ namespace knn_jni { jbyte * GetByteArrayElements(JNIEnv *env, jbyteArray array, jboolean * isCopy); jfloat * GetFloatArrayElements(JNIEnv *env, jfloatArray array, jboolean * isCopy); jint * GetIntArrayElements(JNIEnv *env, jintArray array, jboolean * isCopy); + jlong * GetLongArrayElements(JNIEnv *env, jlongArray array, jboolean * isCopy); jobject GetObjectArrayElement(JNIEnv *env, jobjectArray array, jsize index); jobject NewObject(JNIEnv *env, jclass clazz, jmethodID methodId, int id, float distance); jobjectArray NewObjectArray(JNIEnv *env, jsize len, jclass clazz, jobject init); @@ -153,6 +161,7 @@ namespace knn_jni { void ReleaseByteArrayElements(JNIEnv *env, jbyteArray array, jbyte *elems, int mode); void ReleaseFloatArrayElements(JNIEnv *env, jfloatArray array, jfloat *elems, int mode); void ReleaseIntArrayElements(JNIEnv *env, jintArray array, jint *elems, jint mode); + void ReleaseLongArrayElements(JNIEnv *env, jlongArray array, jlong *elems, jint mode); void SetObjectArrayElement(JNIEnv *env, jobjectArray array, jsize index, jobject val); void SetByteArrayRegion(JNIEnv *env, jbyteArray array, jsize start, jsize len, const jbyte * buf); diff --git a/jni/include/org_opensearch_knn_jni_FaissService.h b/jni/include/org_opensearch_knn_jni_FaissService.h index aefadcee4..3b649c227 100644 --- a/jni/include/org_opensearch_knn_jni_FaissService.h +++ b/jni/include/org_opensearch_knn_jni_FaissService.h @@ -56,7 +56,7 @@ JNIEXPORT jobjectArray JNICALL Java_org_opensearch_knn_jni_FaissService_queryInd * Signature: (J[FI[J)[Lorg/opensearch/knn/index/query/KNNQueryResult; */ JNIEXPORT jobjectArray JNICALL Java_org_opensearch_knn_jni_FaissService_queryIndexWithFilter - (JNIEnv *, jclass, jlong, jfloatArray, jint, jintArray, jintArray); + (JNIEnv *, jclass, jlong, jfloatArray, jint, jlongArray, jint, jintArray); /* * Class: org_opensearch_knn_jni_FaissService diff --git a/jni/src/faiss_wrapper.cpp b/jni/src/faiss_wrapper.cpp index e88254b86..43f1454b5 100644 --- a/jni/src/faiss_wrapper.cpp +++ b/jni/src/faiss_wrapper.cpp @@ -29,9 +29,32 @@ // Defines type of IDSelector enum FilterIdsSelectorType{ - BITMAP, BATCH + BITMAP = 0, BATCH = 1, }; +namespace faiss { +// Using jlong to do Bitmap selector, jlong[] equals to lucene FixedBitSet#bits +struct IDSelectorJlongBitmap : IDSelector { + size_t n; + const jlong* bitmap; + + /** Construct with a binary mask like Lucene FixedBitSet + * + * @param n size of the bitmap array + * @param bitmap id like Lucene FixedBitSet bits + */ + IDSelectorJlongBitmap(size_t n, const jlong* bitmap) : n(n), bitmap(bitmap) {}; + bool is_member(idx_t id) const final { + uint64_t index = id; + uint64_t i = index >> 6; // div 64 + if (i >= n ) { + return false; + } + return (bitmap[i] >> ( index & 63)) & 1L; + } + ~IDSelectorJlongBitmap() override {} +}; +} // Translate space type to faiss metric faiss::MetricType TranslateSpaceToMetric(const std::string& spaceType); @@ -42,9 +65,6 @@ void SetExtraParameters(knn_jni::JNIUtilInterface * jniUtil, JNIEnv *env, // Train an index with data provided void InternalTrainIndex(faiss::Index * index, faiss::idx_t n, const float* x); -// Helps to choose the right FilterIdsSelectorType for Faiss -FilterIdsSelectorType getIdSelectorType(const int* filterIds, int filterIdsLength); - // Converts the int FilterIds to Faiss ids type array. void convertFilterIdsToFaissIdType(const int* filterIds, int filterIdsLength, faiss::idx_t* convertedFilterIds); @@ -199,11 +219,12 @@ jlong knn_jni::faiss_wrapper::LoadIndex(knn_jni::JNIUtilInterface * jniUtil, JNI jobjectArray knn_jni::faiss_wrapper::QueryIndex(knn_jni::JNIUtilInterface * jniUtil, JNIEnv * env, jlong indexPointerJ, jfloatArray queryVectorJ, jint kJ, jintArray parentIdsJ) { - return knn_jni::faiss_wrapper::QueryIndex_WithFilter(jniUtil, env, indexPointerJ, queryVectorJ, kJ, nullptr, parentIdsJ); + return knn_jni::faiss_wrapper::QueryIndex_WithFilter(jniUtil, env, indexPointerJ, queryVectorJ, kJ, nullptr, 0, parentIdsJ); } jobjectArray knn_jni::faiss_wrapper::QueryIndex_WithFilter(knn_jni::JNIUtilInterface * jniUtil, JNIEnv * env, jlong indexPointerJ, - jfloatArray queryVectorJ, jint kJ, jintArray filterIdsJ, jintArray parentIdsJ) { + jfloatArray queryVectorJ, jint kJ, jlongArray filterIdsJ, jint filterIdsTypeJ, jintArray parentIdsJ) { + if (queryVectorJ == nullptr) { throw std::runtime_error("Query Vector cannot be null"); } @@ -225,28 +246,14 @@ jobjectArray knn_jni::faiss_wrapper::QueryIndex_WithFilter(knn_jni::JNIUtilInter omp_set_num_threads(1); // create the filterSearch params if the filterIdsJ is not a null pointer if(filterIdsJ != nullptr) { - int *filteredIdsArray = jniUtil->GetIntArrayElements(env, filterIdsJ, nullptr); - int filterIdsLength = jniUtil->GetJavaIntArrayLength(env, filterIdsJ); + jlong *filteredIdsArray = jniUtil->GetLongArrayElements(env, filterIdsJ, nullptr); + int filterIdsLength = jniUtil->GetJavaLongArrayLength(env, filterIdsJ); std::unique_ptr idSelector; - FilterIdsSelectorType idSelectorType = getIdSelectorType(filteredIdsArray, filterIdsLength); - // start with empty vectors for 2 different types of empty Selectors. We need define them here to avoid copying of data - // during the returns. We could have used pass by reference, but we choose pointers. Returning reference to local - // vector is also an option which can be efficient than copying during returns but it requires upto date C++ compilers. - // To avoid all those confusions, its better to work with pointers here. Ref: https://cplusplus.com/forum/general/56177/ - std::vector convertedIds; - std::vector bitmap; - // Choose a selector which suits best - if(idSelectorType == BATCH) { - convertedIds.resize(filterIdsLength); - convertFilterIdsToFaissIdType(filteredIdsArray, filterIdsLength, convertedIds.data()); - idSelector.reset(new faiss::IDSelectorBatch(convertedIds.size(), convertedIds.data())); + if(filterIdsTypeJ == BITMAP) { + idSelector.reset(new faiss::IDSelectorJlongBitmap(filterIdsLength, filteredIdsArray)); } else { - int maxIdValue = filteredIdsArray[filterIdsLength - 1]; - // >> 3 is equivalent to value / 8 - const int bitsetArraySize = (maxIdValue >> 3) + 1; - bitmap.resize(bitsetArraySize, 0); - buildFilterIdsBitMap(filteredIdsArray, filterIdsLength, bitmap.data()); - idSelector.reset(new faiss::IDSelectorBitmap(bitsetArraySize, bitmap.data())); + faiss::idx_t* batchIndices = reinterpret_cast(filteredIdsArray); + idSelector.reset(new faiss::IDSelectorBatch(filterIdsLength, batchIndices)); } faiss::SearchParameters *searchParameters; faiss::SearchParametersHNSW hnswParams; @@ -276,10 +283,10 @@ jobjectArray knn_jni::faiss_wrapper::QueryIndex_WithFilter(knn_jni::JNIUtilInter indexReader->search(1, rawQueryvector, kJ, dis.data(), ids.data(), searchParameters); } catch (...) { jniUtil->ReleaseFloatArrayElements(env, queryVectorJ, rawQueryvector, JNI_ABORT); - jniUtil->ReleaseIntArrayElements(env, filterIdsJ, filteredIdsArray, JNI_ABORT); + jniUtil->ReleaseLongArrayElements(env, filterIdsJ, filteredIdsArray, JNI_ABORT); throw; } - jniUtil->ReleaseIntArrayElements(env, filterIdsJ, filteredIdsArray, JNI_ABORT); + jniUtil->ReleaseLongArrayElements(env, filterIdsJ, filteredIdsArray, JNI_ABORT); } else { faiss::SearchParameters *searchParameters = nullptr; faiss::SearchParametersHNSW hnswParams; @@ -454,63 +461,6 @@ void InternalTrainIndex(faiss::Index * index, faiss::idx_t n, const float* x) { } } -/** - * This function takes a call on what ID Selector to use: - * https://github.com/facebookresearch/faiss/wiki/Setting-search-parameters-for-one-query#idselectorarray-idselectorbatch-and-idselectorbitmap - * - * class storage lookup construction(Opensearch + Faiss) - * IDSelectorArray O(k) O(k) O(2k) - * IDSelectorBatch O(k) O(1) O(2k) - * IDSelectorBitmap O(n/8) O(1) O(k) -> n is the max value of id in the index - * - * TODO: We need to ideally decide when we can take another hit of K iterations in latency. Some facts: - * an OpenSearch Index can have max segment size as 5GB which, which on a vector with dimension of 128 boils down to - * 7.5M vectors. - * Ref: https://opensearch.org/docs/latest/search-plugins/knn/knn-index/#hnsw-memory-estimation - * M = 16 - * Dimension = 128 - * (1.1 * ( 4 * 128 + 8 * 16) * 7500000)/(1024*1024*1024) ~ 4.9GB - * Ids are sequential in a Segment which means for IDSelectorBitmap total size if the max ID has value of 7.5M will be - * 7500000/(8*1024) = 915KBs in worst case. But with larger dimensions this worst case value will decrease. - * - * With 915KB how many ids can be represented as an array of 64-bit longs : 117,120 ids - * So iterating on 117k ids for 1 single pass is also time consuming. So, we are currently concluding to consider only size - * as factor. We need to improve on this. - * - * TODO: Best way is to implement a SparseBitSet in C++. This can be done by extending the IDSelector Interface of Faiss. - * - * @param filterIds - * @param filterIdsLength - * @return std::string - */ -FilterIdsSelectorType getIdSelectorType(const int* filterIds, int filterIdsLength) { - int maxIdValue = filterIds[filterIdsLength - 1]; - if(filterIdsLength * sizeof(faiss::idx_t) * 8 <= maxIdValue ) { - return BATCH; - } - return BITMAP; -} - -void convertFilterIdsToFaissIdType(const int* filterIds, int filterIdsLength, faiss::idx_t* convertedFilterIds) { - for (int i = 0; i < filterIdsLength; i++) { - convertedFilterIds[i] = filterIds[i]; - } -} - -void buildFilterIdsBitMap(const int* filterIds, int filterIdsLength, uint8_t* bitsetVector) { - /** - * Coming from Faiss IDSelectorBitmap::is_member function bitmap id will be selected - * iff id / 8 < n and bit number (i%8) of bitmap[floor(i / 8)] is 1. - */ - for(int i = 0 ; i < filterIdsLength ; i ++) { - int value = filterIds[i]; - // / , % are expensive operation. Hence, using BitShift operation as they are fast. - int bitsetArrayIndex = value >> 3 ; // is equivalent to value / 8 - // (value & 7) equivalent to value % 8 - bitsetVector[bitsetArrayIndex] = bitsetVector[bitsetArrayIndex] | (1 << (value & 7)); - } -} - std::unique_ptr buildIDGrouperBitmap(knn_jni::JNIUtilInterface * jniUtil, JNIEnv *env, jintArray parentIdsJ, std::vector* bitmap) { int *parentIdsArray = jniUtil->GetIntArrayElements(env, parentIdsJ, nullptr); int parentIdsLength = jniUtil->GetJavaIntArrayLength(env, parentIdsJ); diff --git a/jni/src/jni_util.cpp b/jni/src/jni_util.cpp index cb9270c22..a0c1d5733 100644 --- a/jni/src/jni_util.cpp +++ b/jni/src/jni_util.cpp @@ -319,6 +319,17 @@ int knn_jni::JNIUtil::GetJavaIntArrayLength(JNIEnv *env, jintArray arrayJ) { return length; } +int knn_jni::JNIUtil::GetJavaLongArrayLength(JNIEnv *env, jlongArray arrayJ) { + + if (arrayJ == nullptr) { + throw std::runtime_error("Array cannot be null"); + } + + int length = env->GetArrayLength(arrayJ); + this->HasExceptionInStack(env, "Unable to get array length"); + return length; +} + int knn_jni::JNIUtil::GetJavaBytesArrayLength(JNIEnv *env, jbyteArray arrayJ) { if (arrayJ == nullptr) { @@ -376,6 +387,17 @@ jint * knn_jni::JNIUtil::GetIntArrayElements(JNIEnv *env, jintArray array, jbool return intArray; } +jlong * knn_jni::JNIUtil::GetLongArrayElements(JNIEnv *env, jlongArray array, jboolean * isCopy) { + // Lets check for error here + jlong * longArray = env->GetLongArrayElements(array, isCopy); + if (longArray == nullptr) { + this->HasExceptionInStack(env, "Unable to get long array"); + throw std::runtime_error("Unable to get long array"); + } + + return longArray; +} + jobject knn_jni::JNIUtil::GetObjectArrayElement(JNIEnv *env, jobjectArray array, jsize index) { jobject object = env->GetObjectArrayElement(array, index); this->HasExceptionInStack(env, "Unable to get object"); @@ -424,6 +446,10 @@ void knn_jni::JNIUtil::ReleaseIntArrayElements(JNIEnv *env, jintArray array, jin env->ReleaseIntArrayElements(array, elems, mode); } +void knn_jni::JNIUtil::ReleaseLongArrayElements(JNIEnv *env, jlongArray array, jlong *elems, jint mode) { + env->ReleaseLongArrayElements(array, elems, mode); +} + void knn_jni::JNIUtil::SetObjectArrayElement(JNIEnv *env, jobjectArray array, jsize index, jobject val) { env->SetObjectArrayElement(array, index, val); this->HasExceptionInStack(env, "Unable to set object array element"); diff --git a/jni/src/org_opensearch_knn_jni_FaissService.cpp b/jni/src/org_opensearch_knn_jni_FaissService.cpp index a7b24fcab..e8e761ad7 100644 --- a/jni/src/org_opensearch_knn_jni_FaissService.cpp +++ b/jni/src/org_opensearch_knn_jni_FaissService.cpp @@ -89,10 +89,10 @@ JNIEXPORT jobjectArray JNICALL Java_org_opensearch_knn_jni_FaissService_queryInd } JNIEXPORT jobjectArray JNICALL Java_org_opensearch_knn_jni_FaissService_queryIndexWithFilter - (JNIEnv * env, jclass cls, jlong indexPointerJ, jfloatArray queryVectorJ, jint kJ, jintArray filteredIdsJ, jintArray parentIdsJ) { + (JNIEnv * env, jclass cls, jlong indexPointerJ, jfloatArray queryVectorJ, jint kJ, jlongArray filteredIdsJ, jint filterIdsTypeJ, jintArray parentIdsJ) { try { - return knn_jni::faiss_wrapper::QueryIndex_WithFilter(&jniUtil, env, indexPointerJ, queryVectorJ, kJ, filteredIdsJ, parentIdsJ); + return knn_jni::faiss_wrapper::QueryIndex_WithFilter(&jniUtil, env, indexPointerJ, queryVectorJ, kJ, filteredIdsJ, filterIdsTypeJ, parentIdsJ); } catch (...) { jniUtil.CatchCppExceptionAndThrowJava(env); } diff --git a/jni/tests/faiss_wrapper_test.cpp b/jni/tests/faiss_wrapper_test.cpp index 58daaee2b..4318e8ef9 100644 --- a/jni/tests/faiss_wrapper_test.cpp +++ b/jni/tests/faiss_wrapper_test.cpp @@ -251,9 +251,13 @@ TEST(FaissQueryIndexWithFilterTest1435, BasicAssertions) { queries.push_back(query); } - std::vector filterIds; + int num_bits = test_util::bits2words(164); + std::vector bitmap(num_bits,0); + std::vector filterIds; + for (int64_t i = 154; i < 163; i++) { filterIds.push_back(i); + test_util::setBitSet(i, bitmap.data(), bitmap.size()); } std::unordered_set filterIdSet(filterIds.begin(), filterIds.end()); @@ -270,9 +274,9 @@ TEST(FaissQueryIndexWithFilterTest1435, BasicAssertions) { JNIEnv *jniEnv = nullptr; NiceMock mockJNIUtil; EXPECT_CALL(mockJNIUtil, - GetJavaIntArrayLength( - jniEnv, reinterpret_cast(&filterIds))) - .WillRepeatedly(Return(filterIds.size())); + GetJavaLongArrayLength( + jniEnv, reinterpret_cast(&bitmap))) + .WillRepeatedly(Return(bitmap.size())); int k = 20; for (auto query : queries) { @@ -282,7 +286,7 @@ TEST(FaissQueryIndexWithFilterTest1435, BasicAssertions) { &mockJNIUtil, jniEnv, reinterpret_cast(&createdIndexWithData), reinterpret_cast(&query), k, - reinterpret_cast(&filterIds), nullptr))); + reinterpret_cast(&bitmap), 0, nullptr))); ASSERT_TRUE(results->size() <= filterIds.size()); ASSERT_TRUE(results->size() > 0); diff --git a/jni/tests/test_util.cpp b/jni/tests/test_util.cpp index a75886c51..3c6933d89 100644 --- a/jni/tests/test_util.cpp +++ b/jni/tests/test_util.cpp @@ -116,6 +116,15 @@ test_util::MockJNIUtil::MockJNIUtil() { reinterpret_cast *>(arrayJ)->data()); }); + + // arrayJ is re-interpreted as a std::vector * and then the data is + // re-interpreted as a jlong * + ON_CALL(*this, GetLongArrayElements) + .WillByDefault([this](JNIEnv *env, jlongArray arrayJ, jboolean *isCopy) { + return reinterpret_cast( + reinterpret_cast *>(arrayJ)->data()); + }); + // arrayJ is re-interpreted as a std::vector * and then the data is // re-interpreted as a jfloat * ON_CALL(*this, GetFloatArrayElements) @@ -146,6 +155,13 @@ test_util::MockJNIUtil::MockJNIUtil() { return reinterpret_cast *>(arrayJ)->size(); }); + // arrayJ is re-interpreted as a std::vector * and then the size is + // returned + ON_CALL(*this, GetJavaLongArrayLength) + .WillByDefault([this](JNIEnv *env, jlongArray arrayJ) { + return reinterpret_cast *>(arrayJ)->size(); + }); + // arrayJ is re-interpreted as a std::vector> * and then // the 'index' element is re-interpreted as a jobject ON_CALL(*this, GetObjectArrayElement) @@ -193,6 +209,11 @@ test_util::MockJNIUtil::MockJNIUtil() { .WillByDefault( [this](JNIEnv *env, jintArray array, jint *elems, int mode) {}); + // This function should not do anything meaningful in the unit tests + ON_CALL(*this, ReleaseLongArrayElements) + .WillByDefault( + [this](JNIEnv *env, jlongArray array, jlong *elems, int mode) {}); + // array is re-interpreted as a std::vector * and then the bytes from // buf are copied to it ON_CALL(*this, SetByteArrayRegion) @@ -347,3 +368,16 @@ float test_util::RandomFloat(float min, float max) { std::uniform_real_distribution distribution(min, max); return distribution(e1); } + +size_t test_util::bits2words(uint64_t numBits) { + return ((numBits - 1) >> 6) + 1; +} + +void test_util::setBitSet(uint64_t value, jlong* array, size_t size) { + uint64_t wordNum = value >> 6; + if (wordNum >= size ) { + return; + } + jlong bitmask = (1L << (value & 63)); + array[wordNum] |= bitmask; +} \ No newline at end of file diff --git a/jni/tests/test_util.h b/jni/tests/test_util.h index 6eac70fcf..a8ce6ca8f 100644 --- a/jni/tests/test_util.h +++ b/jni/tests/test_util.h @@ -64,9 +64,12 @@ namespace test_util { (JNIEnv * env, jobjectArray array2dJ)); MOCK_METHOD(jint*, GetIntArrayElements, (JNIEnv * env, jintArray array, jboolean* isCopy)); + MOCK_METHOD(jlong*, GetLongArrayElements, + (JNIEnv * env, jlongArray array, jboolean* isCopy)); MOCK_METHOD(int, GetJavaBytesArrayLength, (JNIEnv * env, jbyteArray arrayJ)); MOCK_METHOD(int, GetJavaFloatArrayLength, (JNIEnv * env, jfloatArray arrayJ)); MOCK_METHOD(int, GetJavaIntArrayLength, (JNIEnv * env, jintArray arrayJ)); + MOCK_METHOD(int, GetJavaLongArrayLength, (JNIEnv * env, jlongArray arrayJ)); MOCK_METHOD(int, GetJavaObjectArrayLength, (JNIEnv * env, jobjectArray arrayJ)); MOCK_METHOD(jobject, GetObjectArrayElement, @@ -86,6 +89,8 @@ namespace test_util { (JNIEnv * env, jfloatArray array, jfloat* elems, int mode)); MOCK_METHOD(void, ReleaseIntArrayElements, (JNIEnv * env, jintArray array, jint* elems, jint mode)); + MOCK_METHOD(void, ReleaseLongArrayElements, + (JNIEnv * env, jlongArray array, jlong* elems, jint mode)); MOCK_METHOD(void, SetByteArrayRegion, (JNIEnv * env, jbyteArray array, jsize start, jsize len, const jbyte* buf)); @@ -150,6 +155,10 @@ namespace test_util { float RandomFloat(float min, float max); + // returns the number of 64 bit words it would take to hold numBits + size_t bits2words(uint64_t numBits); + + void setBitSet(uint64_t value, jlong* array, size_t size); // ------------------------------------------------------------------------------- } // namespace test_util diff --git a/src/main/java/org/opensearch/knn/index/query/FilterIdsSelector.java b/src/main/java/org/opensearch/knn/index/query/FilterIdsSelector.java new file mode 100644 index 000000000..bf06e8c5e --- /dev/null +++ b/src/main/java/org/opensearch/knn/index/query/FilterIdsSelector.java @@ -0,0 +1,107 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.knn.index.query; + +import lombok.AllArgsConstructor; +import lombok.Getter; +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.util.BitSet; +import org.apache.lucene.util.BitSetIterator; +import org.apache.lucene.util.FixedBitSet; + +import java.io.IOException; + +/** + * Util Class for filter ids selector + */ +@AllArgsConstructor +@Getter +public class FilterIdsSelector { + + /** + * When do ann query with filters, there are two types: + * BitMap using FixedBitSet, BATCH using a long array stands for filter result docids. + */ + @AllArgsConstructor + @Getter + public enum FilterIdsSelectorType { + BITMAP(0), + BATCH(1); + + private final int value; + } + + long[] filterIds; + private FilterIdsSelectorType filterType; + + /** + * This function takes a call on what ID Selector to use: + * https://github.com/facebookresearch/faiss/wiki/Setting-search-parameters-for-one-query#idselectorarray-idselectorbatch-and-idselectorbitmap + * + * class storage lookup construction(Opensearch + Faiss) + * IDSelectorArray O(k) O(k) O(2k) + * IDSelectorBatch O(k) O(1) O(2k) + * IDSelectorBitmap O(n/8) O(1) O(k) n is the max value of id in the index + * + * TODO: We need to ideally decide when we can take another hit of K iterations in latency. Some facts: + * an OpenSearch Index can have max segment size as 5GB which, which on a vector with dimension of 128 boils down to + * 7.5M vectors. + * Ref: https://opensearch.org/docs/latest/search-plugins/knn/knn-index/#hnsw-memory-estimation + * M = 16 + * Dimension = 128 + * (1.1 * ( 4 * 128 + 8 * 16) * 7500000)/(1024*1024*1024) ~ 4.9GB + * Ids are sequential in a Segment which means for IDSelectorBitmap total size if the max ID has value of 7.5M will be + * 7500000/(8*1024) = 915KBs in worst case. But with larger dimensions this worst case value will decrease. + * + * With 915KB how many ids can be represented as an array of 64-bit longs : 117,120 ids + * So iterating on 117k ids for 1 single pass is also time consuming. So, we are currently concluding to consider only size + * as factor. We need to improve on this. + * + * Array Memory: Cardinality * Long.BYTES + * BitSet Memory: MaxId / Byte.SIZE + * When Array Memory less than or equal to BitSet Memory return FilterIdsSelectorType.BATCH + * Else return FilterIdsSelectorType.BITMAP; + * + * @param filterIdsBitSet Filter query result docs + * @param cardinality The number of bits that are set + * @return {@link FilterIdsSelector} + */ + public static FilterIdsSelector getFilterIdSelector(final BitSet filterIdsBitSet, final int cardinality) throws IOException { + long[] filterIds; + FilterIdsSelector.FilterIdsSelectorType filterType; + if (filterIdsBitSet instanceof FixedBitSet) { + /** + * When filterIds is dense filter, using fixed bitset + */ + filterIds = ((FixedBitSet) filterIdsBitSet).getBits(); + filterType = FilterIdsSelector.FilterIdsSelectorType.BITMAP; + } else if ((cardinality * Long.BYTES * Byte.SIZE) <= filterIdsBitSet.length()) { + /** + * When filterIds is sparse bitset, using ram usage to decide FilterIdsSelectorType + */ + BitSetIterator bitSetIterator = new BitSetIterator(filterIdsBitSet, cardinality); + filterIds = new long[cardinality]; + int idx = 0; + for (int docId = bitSetIterator.nextDoc(); docId != DocIdSetIterator.NO_MORE_DOCS; docId = bitSetIterator.nextDoc()) { + filterIds[idx++] = docId; + } + filterType = FilterIdsSelectorType.BATCH; + } else { + FixedBitSet fixedBitSet = new FixedBitSet(filterIdsBitSet.length()); + BitSetIterator sparseBitSetIterator = new BitSetIterator(filterIdsBitSet, cardinality); + fixedBitSet.or(sparseBitSetIterator); + filterIds = fixedBitSet.getBits(); + filterType = FilterIdsSelector.FilterIdsSelectorType.BITMAP; + } + return new FilterIdsSelector(filterIds, filterType); + } +} diff --git a/src/main/java/org/opensearch/knn/index/query/KNNWeight.java b/src/main/java/org/opensearch/knn/index/query/KNNWeight.java index afa7e93b0..5bd4e9359 100644 --- a/src/main/java/org/opensearch/knn/index/query/KNNWeight.java +++ b/src/main/java/org/opensearch/knn/index/query/KNNWeight.java @@ -100,11 +100,13 @@ public Explanation explain(LeafReaderContext context, int doc) { @Override public Scorer scorer(LeafReaderContext context) throws IOException { - final int[] filterIdsArray = getFilterIdsArray(context); + + final BitSet filterBitSet = getFilteredDocsBitSet(context); + int cardinality = filterBitSet.cardinality(); // We don't need to go to JNI layer if no documents are found which satisfy the filters // We should give this condition a deeper look that where it should be placed. For now I feel this is a good // place, - if (filterWeight != null && filterIdsArray.length == 0) { + if (filterWeight != null && cardinality == 0) { return KNNScorer.emptyScorer(this); } final Map docIdsToScoreMap = new HashMap<>(); @@ -114,22 +116,22 @@ public Scorer scorer(LeafReaderContext context) throws IOException { * . Hence, if filtered results are less than K and filter query is present we should shift to exact search. * This improves the recall. */ - if (filterWeight != null && canDoExactSearch(filterIdsArray.length)) { - docIdsToScoreMap.putAll(doExactSearch(context, filterIdsArray)); + if (filterWeight != null && canDoExactSearch(cardinality)) { + docIdsToScoreMap.putAll(doExactSearch(context, filterBitSet)); } else { - Map annResults = doANNSearch(context, filterIdsArray); + Map annResults = doANNSearch(context, filterBitSet, cardinality); if (annResults == null) { return null; } - if (canDoExactSearchAfterANNSearch(filterIdsArray.length, annResults.size())) { + if (canDoExactSearchAfterANNSearch(cardinality, annResults.size())) { log.debug( "Doing ExactSearch after doing ANNSearch as the number of documents returned are less than " + "K, even when we have more than K filtered Ids. K: {}, ANNResults: {}, filteredIdCount: {}", knnQuery.getK(), annResults.size(), - filterIdsArray.length + cardinality ); - annResults = doExactSearch(context, filterIdsArray); + annResults = doExactSearch(context, filterBitSet); } docIdsToScoreMap.putAll(annResults); } @@ -139,7 +141,11 @@ public Scorer scorer(LeafReaderContext context) throws IOException { return convertSearchResponseToScorer(docIdsToScoreMap); } - private BitSet getFilteredDocsBitSet(final LeafReaderContext ctx, final Weight filterWeight) throws IOException { + private BitSet getFilteredDocsBitSet(final LeafReaderContext ctx) throws IOException { + if (this.filterWeight == null) { + return new FixedBitSet(0); + } + final Bits liveDocs = ctx.reader().getLiveDocs(); final int maxDoc = ctx.reader().maxDoc(); @@ -166,13 +172,6 @@ protected boolean match(int doc) { return BitSet.of(filterIterator, maxDoc); } - private int[] getFilterIdsArray(final LeafReaderContext context) throws IOException { - if (filterWeight == null) { - return new int[0]; - } - return bitSetToIntArray(getFilteredDocsBitSet(context, this.filterWeight)); - } - private int[] getParentIdsArray(final LeafReaderContext context) throws IOException { if (knnQuery.getParentsFilter() == null) { return null; @@ -194,7 +193,8 @@ private int[] bitSetToIntArray(final BitSet bitSet) { return intArray; } - private Map doANNSearch(final LeafReaderContext context, final int[] filterIdsArray) throws IOException { + private Map doANNSearch(final LeafReaderContext context, final BitSet filterIdsBitSet, final int cardinality) + throws IOException { SegmentReader reader = (SegmentReader) FilterLeafReader.unwrap(context.reader()); String directory = ((FSDirectory) FilterDirectory.unwrap(reader.directory())).getDirectory().toString(); @@ -265,6 +265,10 @@ private Map doANNSearch(final LeafReaderContext context, final i throw new RuntimeException(e); } + // From cardinality select different filterIds type + FilterIdsSelector filterIdsSelector = FilterIdsSelector.getFilterIdSelector(filterIdsBitSet, cardinality); + long[] filterIds = filterIdsSelector.getFilterIds(); + FilterIdsSelector.FilterIdsSelectorType filterType = filterIdsSelector.getFilterType(); // Now that we have the allocation, we need to readLock it indexAllocation.readLock(); try { @@ -277,7 +281,8 @@ private Map doANNSearch(final LeafReaderContext context, final i knnQuery.getQueryVector(), knnQuery.getK(), knnEngine.getName(), - filterIdsArray, + filterIds, + filterType.getValue(), parentIds ); @@ -303,13 +308,13 @@ private Map doANNSearch(final LeafReaderContext context, final i .collect(Collectors.toMap(KNNQueryResult::getId, result -> knnEngine.score(result.getScore(), spaceType))); } - private Map doExactSearch(final LeafReaderContext leafReaderContext, final int[] filterIdsArray) throws IOException { + private Map doExactSearch(final LeafReaderContext leafReaderContext, final BitSet filterIdsBitSet) { try { // Creating min heap and init with MAX DocID and Score as -INF. final HitQueue queue = new HitQueue(this.knnQuery.getK(), true); ScoreDoc topDoc = queue.top(); final Map docToScore = new HashMap<>(); - FilteredIdsKNNIterator iterator = getFilteredKNNIterator(leafReaderContext, filterIdsArray); + FilteredIdsKNNIterator iterator = getFilteredKNNIterator(leafReaderContext, filterIdsBitSet); int docId; while ((docId = iterator.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) { if (iterator.score() > topDoc.score) { @@ -340,16 +345,16 @@ private Map doExactSearch(final LeafReaderContext leafReaderCont return Collections.emptyMap(); } - private FilteredIdsKNNIterator getFilteredKNNIterator(final LeafReaderContext leafReaderContext, final int[] filterIdsArray) + private FilteredIdsKNNIterator getFilteredKNNIterator(final LeafReaderContext leafReaderContext, final BitSet filterIdsBitSet) throws IOException { final SegmentReader reader = (SegmentReader) FilterLeafReader.unwrap(leafReaderContext.reader()); final FieldInfo fieldInfo = reader.getFieldInfos().fieldInfo(knnQuery.getField()); final BinaryDocValues values = DocValues.getBinary(leafReaderContext.reader(), fieldInfo.getName()); final SpaceType spaceType = getSpaceType(fieldInfo); return knnQuery.getParentsFilter() == null - ? new FilteredIdsKNNIterator(filterIdsArray, knnQuery.getQueryVector(), values, spaceType) + ? new FilteredIdsKNNIterator(filterIdsBitSet, knnQuery.getQueryVector(), values, spaceType) : new NestedFilteredIdsKNNIterator( - filterIdsArray, + filterIdsBitSet, knnQuery.getQueryVector(), values, spaceType, diff --git a/src/main/java/org/opensearch/knn/index/query/filtered/FilteredIdsKNNIterator.java b/src/main/java/org/opensearch/knn/index/query/filtered/FilteredIdsKNNIterator.java index a286829d5..a53cb8d60 100644 --- a/src/main/java/org/opensearch/knn/index/query/filtered/FilteredIdsKNNIterator.java +++ b/src/main/java/org/opensearch/knn/index/query/filtered/FilteredIdsKNNIterator.java @@ -7,6 +7,8 @@ import org.apache.lucene.index.BinaryDocValues; import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.util.BitSet; +import org.apache.lucene.util.BitSetIterator; import org.apache.lucene.util.BytesRef; import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.codec.util.KNNVectorSerializer; @@ -23,23 +25,26 @@ */ public class FilteredIdsKNNIterator { // Array of doc ids to iterate - protected final int[] filterIdsArray; + protected final BitSet filterIdsBitSet; + protected final BitSetIterator bitSetIterator; protected final float[] queryVector; protected final BinaryDocValues binaryDocValues; protected final SpaceType spaceType; protected float currentScore = Float.NEGATIVE_INFINITY; - protected int currentPos = 0; + protected int docId; public FilteredIdsKNNIterator( - final int[] filterIdsArray, + final BitSet filterIdsBitSet, final float[] queryVector, final BinaryDocValues binaryDocValues, final SpaceType spaceType ) { - this.filterIdsArray = filterIdsArray; + this.filterIdsBitSet = filterIdsBitSet; + this.bitSetIterator = new BitSetIterator(filterIdsBitSet, filterIdsBitSet.length()); this.queryVector = queryVector; this.binaryDocValues = binaryDocValues; this.spaceType = spaceType; + this.docId = bitSetIterator.nextDoc(); } /** @@ -49,13 +54,14 @@ public FilteredIdsKNNIterator( * @return next doc id */ public int nextDoc() throws IOException { - if (currentPos >= filterIdsArray.length) { + + if (docId == DocIdSetIterator.NO_MORE_DOCS) { return DocIdSetIterator.NO_MORE_DOCS; } - int docId = binaryDocValues.advance(filterIdsArray[currentPos]); + int doc = binaryDocValues.advance(docId); currentScore = computeScore(); - currentPos++; - return docId; + docId = bitSetIterator.nextDoc(); + return doc; } public float score() { diff --git a/src/main/java/org/opensearch/knn/index/query/filtered/NestedFilteredIdsKNNIterator.java b/src/main/java/org/opensearch/knn/index/query/filtered/NestedFilteredIdsKNNIterator.java index d2e4d3e25..9776ebbe9 100644 --- a/src/main/java/org/opensearch/knn/index/query/filtered/NestedFilteredIdsKNNIterator.java +++ b/src/main/java/org/opensearch/knn/index/query/filtered/NestedFilteredIdsKNNIterator.java @@ -20,7 +20,7 @@ public class NestedFilteredIdsKNNIterator extends FilteredIdsKNNIterator { private final BitSet parentBitSet; public NestedFilteredIdsKNNIterator( - final int[] filterIdsArray, + final BitSet filterIdsArray, final float[] queryVector, final BinaryDocValues values, final SpaceType spaceType, @@ -38,20 +38,22 @@ public NestedFilteredIdsKNNIterator( */ @Override public int nextDoc() throws IOException { - if (currentPos >= filterIdsArray.length) { + if (docId == DocIdSetIterator.NO_MORE_DOCS) { return DocIdSetIterator.NO_MORE_DOCS; } + currentScore = Float.NEGATIVE_INFINITY; - int currentParent = parentBitSet.nextSetBit(filterIdsArray[currentPos]); + int currentParent = parentBitSet.nextSetBit(docId); int bestChild = -1; - while (currentPos < filterIdsArray.length && filterIdsArray[currentPos] < currentParent) { - binaryDocValues.advance(filterIdsArray[currentPos]); + + while (docId != DocIdSetIterator.NO_MORE_DOCS && docId < currentParent) { + binaryDocValues.advance(docId); float score = computeScore(); if (score > currentScore) { - bestChild = filterIdsArray[currentPos]; + bestChild = docId; currentScore = score; } - currentPos++; + docId = bitSetIterator.nextDoc(); } return bestChild; diff --git a/src/main/java/org/opensearch/knn/jni/FaissService.java b/src/main/java/org/opensearch/knn/jni/FaissService.java index abf3e052a..f330352ec 100644 --- a/src/main/java/org/opensearch/knn/jni/FaissService.java +++ b/src/main/java/org/opensearch/knn/jni/FaissService.java @@ -104,7 +104,8 @@ public static native KNNQueryResult[] queryIndexWithFilter( long indexPointer, float[] queryVector, int k, - int[] filterIds, + long[] filterIds, + int filterIdsType, int[] parentIds ); diff --git a/src/main/java/org/opensearch/knn/jni/JNIService.java b/src/main/java/org/opensearch/knn/jni/JNIService.java index beef9f927..2835be23d 100644 --- a/src/main/java/org/opensearch/knn/jni/JNIService.java +++ b/src/main/java/org/opensearch/knn/jni/JNIService.java @@ -99,6 +99,7 @@ public static long loadIndex(String indexPath, Map parameters, S * @param k neighbors to be returned * @param engineName name of engine to query index * @param filteredIds array of ints on which should be used for search. + * @param filterIdsType how to filter ids: Batch or BitMap * @return KNNQueryResult array of k neighbors */ public static KNNQueryResult[] queryIndex( @@ -106,7 +107,8 @@ public static KNNQueryResult[] queryIndex( float[] queryVector, int k, String engineName, - int[] filteredIds, + long[] filteredIds, + int filterIdsType, int[] parentIds ) { if (KNNEngine.NMSLIB.getName().equals(engineName)) { @@ -119,7 +121,7 @@ public static KNNQueryResult[] queryIndex( // filterIds. FilterIds is coming as empty then its the case where we need to do search with Faiss engine // normally. if (ArrayUtils.isNotEmpty(filteredIds)) { - return FaissService.queryIndexWithFilter(indexPointer, queryVector, k, filteredIds, parentIds); + return FaissService.queryIndexWithFilter(indexPointer, queryVector, k, filteredIds, filterIdsType, parentIds); } return FaissService.queryIndex(indexPointer, queryVector, k, parentIds); } diff --git a/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestUtil.java b/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestUtil.java index b1d9d9434..472f05113 100644 --- a/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestUtil.java +++ b/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestUtil.java @@ -341,7 +341,7 @@ public static void assertLoadableByEngine( ); int k = 2; float[] queryVector = new float[dimension]; - KNNQueryResult[] results = JNIService.queryIndex(indexPtr, queryVector, k, knnEngine.getName(), null, null); + KNNQueryResult[] results = JNIService.queryIndex(indexPtr, queryVector, k, knnEngine.getName(), null, 0, null); assertTrue(results.length > 0); JNIService.free(indexPtr, knnEngine.getName()); } diff --git a/src/test/java/org/opensearch/knn/index/memory/NativeMemoryLoadStrategyTests.java b/src/test/java/org/opensearch/knn/index/memory/NativeMemoryLoadStrategyTests.java index 798c64a17..1f1088de1 100644 --- a/src/test/java/org/opensearch/knn/index/memory/NativeMemoryLoadStrategyTests.java +++ b/src/test/java/org/opensearch/knn/index/memory/NativeMemoryLoadStrategyTests.java @@ -74,7 +74,7 @@ public void testIndexLoadStrategy_load() throws IOException { // Confirm that the file was loaded by querying float[] query = new float[dimension]; Arrays.fill(query, numVectors + 1); - KNNQueryResult[] results = JNIService.queryIndex(indexAllocation.getMemoryAddress(), query, 2, knnEngine.getName(), null, null); + KNNQueryResult[] results = JNIService.queryIndex(indexAllocation.getMemoryAddress(), query, 2, knnEngine.getName(), null, 0, null); assertTrue(results.length > 0); } diff --git a/src/test/java/org/opensearch/knn/index/query/FilterIdsSelectorTests.java b/src/test/java/org/opensearch/knn/index/query/FilterIdsSelectorTests.java new file mode 100644 index 000000000..02b1553a3 --- /dev/null +++ b/src/test/java/org/opensearch/knn/index/query/FilterIdsSelectorTests.java @@ -0,0 +1,60 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.knn.index.query; + +import lombok.SneakyThrows; +import org.apache.lucene.util.BitSetIterator; +import org.apache.lucene.util.FixedBitSet; +import org.apache.lucene.util.SparseFixedBitSet; +import org.opensearch.knn.KNNTestCase; + +public class FilterIdsSelectorTests extends KNNTestCase { + + @SneakyThrows + public void testGetIdSelectorTypeWithFixedBitSet() { + FixedBitSet bits = new FixedBitSet(101); + for (int i = 1; i <= 100; i++) { + bits.set(i); + } + FilterIdsSelector idsSelector = FilterIdsSelector.getFilterIdSelector(bits, bits.cardinality()); + assertEquals(idsSelector.getFilterType(), FilterIdsSelector.FilterIdsSelectorType.BITMAP); + assertArrayEquals(bits.getBits(), idsSelector.filterIds); + } + + @SneakyThrows + public void testGetIdSelectorTypeWithSparseBitSetHigh() { + SparseFixedBitSet bits = new SparseFixedBitSet(101); + for (int i = 1; i <= 100; i++) { + bits.set(i); + } + FilterIdsSelector idsSelector = FilterIdsSelector.getFilterIdSelector(bits, bits.cardinality()); + assertEquals(idsSelector.getFilterType(), FilterIdsSelector.FilterIdsSelectorType.BITMAP); + FixedBitSet fixedBitSet = new FixedBitSet(bits.length()); + BitSetIterator sparseBitSetIterator = new BitSetIterator(bits, 101); + fixedBitSet.or(sparseBitSetIterator); + assertArrayEquals(fixedBitSet.getBits(), idsSelector.filterIds); + } + + @SneakyThrows + public void testGetIdSelectorTypeWithSparseBitSetLow() { + int maxDoc = (Integer.MAX_VALUE) / 2; + SparseFixedBitSet bits = new SparseFixedBitSet(maxDoc); + long array[] = new long[100]; + for (int i = maxDoc - 100, idx = 0; i < maxDoc; i++) { + bits.set(i); + array[idx++] = i; + } + FilterIdsSelector idsSelector = FilterIdsSelector.getFilterIdSelector(bits, bits.cardinality()); + assertEquals(idsSelector.getFilterType(), FilterIdsSelector.FilterIdsSelectorType.BATCH); + assertArrayEquals(array, idsSelector.filterIds); + } +} diff --git a/src/test/java/org/opensearch/knn/index/query/KNNWeightTests.java b/src/test/java/org/opensearch/knn/index/query/KNNWeightTests.java index 5d49f052c..49c7c7566 100644 --- a/src/test/java/org/opensearch/knn/index/query/KNNWeightTests.java +++ b/src/test/java/org/opensearch/knn/index/query/KNNWeightTests.java @@ -157,7 +157,7 @@ public void testQueryScoreForFaissWithModel() { SpaceType spaceType = SpaceType.L2; final Function scoreTranslator = spaceType::scoreTranslation; final String modelId = "modelId"; - jniServiceMockedStatic.when(() -> JNIService.queryIndex(anyLong(), any(), anyInt(), anyString(), any(), any())) + jniServiceMockedStatic.when(() -> JNIService.queryIndex(anyLong(), any(), anyInt(), anyString(), any(), anyInt(), any())) .thenReturn(getKNNQueryResults()); final KNNQuery query = new KNNQuery(FIELD_NAME, QUERY_VECTOR, K, INDEX_NAME, (BitSetProducer) null); @@ -300,7 +300,7 @@ public void testShardWithoutFiles() { @SneakyThrows public void testEmptyQueryResults() { final KNNQueryResult[] knnQueryResults = new KNNQueryResult[] {}; - jniServiceMockedStatic.when(() -> JNIService.queryIndex(anyLong(), any(), anyInt(), anyString(), any(), any())) + jniServiceMockedStatic.when(() -> JNIService.queryIndex(anyLong(), any(), anyInt(), anyString(), any(), anyInt(), any())) .thenReturn(knnQueryResults); final KNNQuery query = new KNNQuery(FIELD_NAME, QUERY_VECTOR, K, INDEX_NAME, (BitSetProducer) null); @@ -345,8 +345,13 @@ public void testEmptyQueryResults() { public void testANNWithFilterQuery_whenDoingANN_thenSuccess() { int k = 3; final int[] filterDocIds = new int[] { 0, 1, 2, 3, 4, 5 }; - jniServiceMockedStatic.when(() -> JNIService.queryIndex(anyLong(), any(), anyInt(), anyString(), eq(filterDocIds), any())) - .thenReturn(getFilteredKNNQueryResults()); + FixedBitSet filterBitSet = new FixedBitSet(filterDocIds.length); + for (int docId : filterDocIds) { + filterBitSet.set(docId); + } + jniServiceMockedStatic.when( + () -> JNIService.queryIndex(anyLong(), any(), anyInt(), anyString(), eq(filterBitSet.getBits()), anyInt(), any()) + ).thenReturn(getFilteredKNNQueryResults()); final LeafReaderContext leafReaderContext = mock(LeafReaderContext.class); final SegmentReader reader = mock(SegmentReader.class); final Bits liveDocsBits = mock(Bits.class); @@ -404,7 +409,10 @@ public void testANNWithFilterQuery_whenDoingANN_thenSuccess() { final DocIdSetIterator docIdSetIterator = knnScorer.iterator(); assertNotNull(docIdSetIterator); assertEquals(FILTERED_DOC_ID_TO_SCORES.size(), docIdSetIterator.cost()); - jniServiceMockedStatic.verify(() -> JNIService.queryIndex(anyLong(), any(), anyInt(), anyString(), eq(filterDocIds), any())); + + jniServiceMockedStatic.verify( + () -> JNIService.queryIndex(anyLong(), any(), anyInt(), anyString(), eq(filterBitSet.getBits()), anyInt(), any()) + ); final List actualDocIds = new ArrayList<>(); final Map translatedScores = getTranslatedScores(SpaceType.L2::scoreTranslation); @@ -669,14 +677,17 @@ public void testANNWithParentsFilter_whenDoingANN_thenBitSetIsPassedToJNI() { final KNNQuery query = new KNNQuery(FIELD_NAME, QUERY_VECTOR, 1, INDEX_NAME, null, bitSetProducer); final KNNWeight knnWeight = new KNNWeight(query, 0.0f, null); - jniServiceMockedStatic.when(() -> JNIService.queryIndex(anyLong(), any(), anyInt(), anyString(), any(), eq(parentsFilter))) - .thenReturn(getKNNQueryResults()); + jniServiceMockedStatic.when( + () -> JNIService.queryIndex(anyLong(), any(), anyInt(), anyString(), any(), anyInt(), eq(parentsFilter)) + ).thenReturn(getKNNQueryResults()); // Execute Scorer knnScorer = knnWeight.scorer(leafReaderContext); // Verify - jniServiceMockedStatic.verify(() -> JNIService.queryIndex(anyLong(), any(), anyInt(), anyString(), any(), eq(parentsFilter))); + jniServiceMockedStatic.verify( + () -> JNIService.queryIndex(anyLong(), any(), anyInt(), anyString(), any(), anyInt(), eq(parentsFilter)) + ); assertNotNull(knnScorer); final DocIdSetIterator docIdSetIterator = knnScorer.iterator(); assertNotNull(docIdSetIterator); @@ -735,7 +746,7 @@ private void testQueryScore( final Set segmentFiles, final Map fileAttributes ) throws IOException { - jniServiceMockedStatic.when(() -> JNIService.queryIndex(anyLong(), any(), anyInt(), anyString(), any(), any())) + jniServiceMockedStatic.when(() -> JNIService.queryIndex(anyLong(), any(), anyInt(), anyString(), any(), anyInt(), any())) .thenReturn(getKNNQueryResults()); final KNNQuery query = new KNNQuery(FIELD_NAME, QUERY_VECTOR, K, INDEX_NAME, (BitSetProducer) null); diff --git a/src/test/java/org/opensearch/knn/index/query/filtered/FilteredIdsKNNIteratorTests.java b/src/test/java/org/opensearch/knn/index/query/filtered/FilteredIdsKNNIteratorTests.java index cfb66662e..dce703050 100644 --- a/src/test/java/org/opensearch/knn/index/query/filtered/FilteredIdsKNNIteratorTests.java +++ b/src/test/java/org/opensearch/knn/index/query/filtered/FilteredIdsKNNIteratorTests.java @@ -9,6 +9,7 @@ import org.apache.lucene.index.BinaryDocValues; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.FixedBitSet; import org.opensearch.knn.KNNTestCase; import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.codec.util.KNNVectorAsArraySerializer; @@ -41,12 +42,14 @@ public void testNextDoc_whenCalled_IterateAllDocs() { .collect(Collectors.toList()); when(values.binaryValue()).thenReturn(byteRefs.get(0), byteRefs.get(1), byteRefs.get(2)); + FixedBitSet filterBitSet = new FixedBitSet(4); for (int id : filterIds) { when(values.advance(id)).thenReturn(id); + filterBitSet.set(id); } // Execute and verify - FilteredIdsKNNIterator iterator = new FilteredIdsKNNIterator(filterIds, queryVector, values, spaceType); + FilteredIdsKNNIterator iterator = new FilteredIdsKNNIterator(filterBitSet, queryVector, values, spaceType); for (int i = 0; i < filterIds.length; i++) { assertEquals(filterIds[i], iterator.nextDoc()); assertEquals(expectedScores.get(i), (Float) iterator.score()); diff --git a/src/test/java/org/opensearch/knn/index/query/filtered/NestedFilteredIdsKNNIteratorTests.java b/src/test/java/org/opensearch/knn/index/query/filtered/NestedFilteredIdsKNNIteratorTests.java index 90d56fddc..d732376ef 100644 --- a/src/test/java/org/opensearch/knn/index/query/filtered/NestedFilteredIdsKNNIteratorTests.java +++ b/src/test/java/org/opensearch/knn/index/query/filtered/NestedFilteredIdsKNNIteratorTests.java @@ -47,12 +47,20 @@ public void testNextDoc_whenIterate_ReturnBestChildDocsPerParent() { .collect(Collectors.toList()); when(values.binaryValue()).thenReturn(byteRefs.get(0), byteRefs.get(1), byteRefs.get(2)); + FixedBitSet filterBitSet = new FixedBitSet(4); for (int id : filterIds) { when(values.advance(id)).thenReturn(id); + filterBitSet.set(id); } // Execute and verify - NestedFilteredIdsKNNIterator iterator = new NestedFilteredIdsKNNIterator(filterIds, queryVector, values, spaceType, parentBitSet); + NestedFilteredIdsKNNIterator iterator = new NestedFilteredIdsKNNIterator( + filterBitSet, + queryVector, + values, + spaceType, + parentBitSet + ); assertEquals(filterIds[0], iterator.nextDoc()); assertEquals(expectedScores.get(0), iterator.score()); assertEquals(filterIds[2], iterator.nextDoc()); diff --git a/src/test/java/org/opensearch/knn/jni/JNIServiceTests.java b/src/test/java/org/opensearch/knn/jni/JNIServiceTests.java index 2afcff83d..8e3382ece 100644 --- a/src/test/java/org/opensearch/knn/jni/JNIServiceTests.java +++ b/src/test/java/org/opensearch/knn/jni/JNIServiceTests.java @@ -537,13 +537,13 @@ public void testQueryIndex_faiss_sqfp16_valid() { assertNotEquals(0, pointer); for (float[] query : testData.queries) { - KNNQueryResult[] results = JNIService.queryIndex(pointer, query, k, FAISS_NAME, null, null); + KNNQueryResult[] results = JNIService.queryIndex(pointer, query, k, FAISS_NAME, null, 0, null); assertEquals(k, results.length); } // Filter will result in no ids for (float[] query : testData.queries) { - KNNQueryResult[] results = JNIService.queryIndex(pointer, query, k, FAISS_NAME, new int[] { 0 }, null); + KNNQueryResult[] results = JNIService.queryIndex(pointer, query, k, FAISS_NAME, new long[] { 0 }, 0, null); assertEquals(0, results.length); } } @@ -726,12 +726,15 @@ public void testLoadIndex_faiss_valid() throws IOException { } public void testQueryIndex_invalidEngine() { - expectThrows(IllegalArgumentException.class, () -> JNIService.queryIndex(0L, new float[] {}, 0, "invalid" + "-engine", null, null)); + expectThrows( + IllegalArgumentException.class, + () -> JNIService.queryIndex(0L, new float[] {}, 0, "invalid" + "-engine", null, 0, null) + ); } public void testQueryIndex_nmslib_invalid_badPointer() { - expectThrows(Exception.class, () -> JNIService.queryIndex(0L, new float[] {}, 0, KNNEngine.NMSLIB.getName(), null, null)); + expectThrows(Exception.class, () -> JNIService.queryIndex(0L, new float[] {}, 0, KNNEngine.NMSLIB.getName(), null, 0, null)); } public void testQueryIndex_nmslib_invalid_nullQueryVector() throws IOException { @@ -754,7 +757,7 @@ public void testQueryIndex_nmslib_invalid_nullQueryVector() throws IOException { ); assertNotEquals(0, pointer); - expectThrows(Exception.class, () -> JNIService.queryIndex(pointer, null, 10, KNNEngine.NMSLIB.getName(), null, null)); + expectThrows(Exception.class, () -> JNIService.queryIndex(pointer, null, 10, KNNEngine.NMSLIB.getName(), null, 0, null)); } public void testQueryIndex_nmslib_valid() throws IOException { @@ -780,7 +783,7 @@ public void testQueryIndex_nmslib_valid() throws IOException { assertNotEquals(0, pointer); for (float[] query : testData.queries) { - KNNQueryResult[] results = JNIService.queryIndex(pointer, query, k, KNNEngine.NMSLIB.getName(), null, null); + KNNQueryResult[] results = JNIService.queryIndex(pointer, query, k, KNNEngine.NMSLIB.getName(), null, 0, null); assertEquals(k, results.length); } } @@ -788,7 +791,7 @@ public void testQueryIndex_nmslib_valid() throws IOException { public void testQueryIndex_faiss_invalid_badPointer() { - expectThrows(Exception.class, () -> JNIService.queryIndex(0L, new float[] {}, 0, FAISS_NAME, null, null)); + expectThrows(Exception.class, () -> JNIService.queryIndex(0L, new float[] {}, 0, FAISS_NAME, null, 0, null)); } public void testQueryIndex_faiss_invalid_nullQueryVector() throws IOException { @@ -807,7 +810,7 @@ public void testQueryIndex_faiss_invalid_nullQueryVector() throws IOException { long pointer = JNIService.loadIndex(tmpFile.toAbsolutePath().toString(), Collections.emptyMap(), FAISS_NAME); assertNotEquals(0, pointer); - expectThrows(Exception.class, () -> JNIService.queryIndex(pointer, null, 10, FAISS_NAME, null, null)); + expectThrows(Exception.class, () -> JNIService.queryIndex(pointer, null, 10, FAISS_NAME, null, 0, null)); } public void testQueryIndex_faiss_valid() throws IOException { @@ -836,13 +839,13 @@ public void testQueryIndex_faiss_valid() throws IOException { assertNotEquals(0, pointer); for (float[] query : testData.queries) { - KNNQueryResult[] results = JNIService.queryIndex(pointer, query, k, FAISS_NAME, null, null); + KNNQueryResult[] results = JNIService.queryIndex(pointer, query, k, FAISS_NAME, null, 0, null); assertEquals(k, results.length); } // Filter will result in no ids for (float[] query : testData.queries) { - KNNQueryResult[] results = JNIService.queryIndex(pointer, query, k, FAISS_NAME, new int[] { 0 }, null); + KNNQueryResult[] results = JNIService.queryIndex(pointer, query, k, FAISS_NAME, new long[] { 0 }, 0, null); assertEquals(0, results.length); } } @@ -877,7 +880,7 @@ public void testQueryIndex_faiss_parentIds() throws IOException { assertNotEquals(0, pointer); for (float[] query : testDataNested.queries) { - KNNQueryResult[] results = JNIService.queryIndex(pointer, query, k, FAISS_NAME, null, parentIds); + KNNQueryResult[] results = JNIService.queryIndex(pointer, query, k, FAISS_NAME, null, 0, parentIds); // Verify there is no more than one result from same parent Set parentIdSet = toParentIdSet(results, idToParentIdMap); assertEquals(results.length, parentIdSet.size()); From fd83164e7c975117f04d2d325f227209de3f191b Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Wed, 6 Mar 2024 13:36:16 -0800 Subject: [PATCH 208/416] Manually install zlib for win CI (#1514) Signed-off-by: John Mazanec (cherry picked from commit 231ad934e6b0f4e62ee0ca68bcb160534262d401) --- .github/workflows/CI.yml | 9 +++++++++ CHANGELOG.md | 1 + 2 files changed, 10 insertions(+) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index b689e1021..a585dd06d 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -139,6 +139,15 @@ jobs: Import-Module "$env:ChocolateyInstall\helpers\chocolateyProfile.psm1" refreshenv + - name: Install Zlib Using Scoop + run: | + echo "C:/Users/runneradmin/scoop/shims" >> $env:GITHUB_PATH + Import-Module "$env:ChocolateyInstall\helpers\chocolateyProfile.psm1" + refreshenv + scoop bucket add extras + scoop install zlib + regedit /s "C:\\Users\\runneradmin\\scoop\\apps\\zlib\\current\\register.reg" + - name: Download OpenBLAS run: | curl -L -O https://github.com/xianyi/OpenBLAS/releases/download/v0.3.21/OpenBLAS-0.3.21-x64.zip diff --git a/CHANGELOG.md b/CHANGELOG.md index 69b213ad8..6420f56b1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), * Optize Faiss Query With Filters: Reduce iteration and memory for id filter [#1402](https://github.com/opensearch-project/k-NN/pull/1402) ### Bug Fixes ### Infrastructure +* Manually install zlib for win CI [#1513](https://github.com/opensearch-project/k-NN/pull/1513) ### Documentation ### Maintenance * Bump faiss lib commit to 32f0e8cf92cd2275b60364517bb1cce67aa29a55 [#1443](https://github.com/opensearch-project/k-NN/pull/1443) From 9df8e205264e4c8bad5735c19c770737a317b324 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Wed, 6 Mar 2024 15:13:36 -0800 Subject: [PATCH 209/416] Upgrade faiss to 12b92e9 (#1516) Upgrades faiss to https://github.com/facebookresearch/faiss/commit/12b92e9fa5d8e8fb3da53c57af9ff007c826b1ee. Cleanup outdated patches. Signed-off-by: John Mazanec (cherry picked from commit 1303182971d26161e845aaf20e48a95b6e6a490a) --- .github/workflows/CI.yml | 6 +- .github/workflows/test_security.yml | 6 +- CHANGELOG.md | 1 + jni/CMakeLists.txt | 10 +- jni/external/faiss | 2 +- ...stom-patch-to-support-AVX2-Linux-CI.patch} | 0 ...-Custom-patch-to-support-sqfp16-neon.patch | 495 ------------------ .../org/opensearch/knn/index/util/Faiss.java | 3 + 8 files changed, 10 insertions(+), 513 deletions(-) rename jni/patches/faiss/{0003-Custom-patch-to-support-AVX2-Linux-CI.patch => 0002-Custom-patch-to-support-AVX2-Linux-CI.patch} (100%) delete mode 100644 jni/patches/faiss/0002-Custom-patch-to-support-sqfp16-neon.patch diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index a585dd06d..b52a37702 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -45,10 +45,8 @@ jobs: cd jni/external/faiss git apply --ignore-space-change --ignore-whitespace --3way ../../patches/faiss/0001-Custom-patch-to-support-multi-vector.patch rm ../../patches/faiss/0001-Custom-patch-to-support-multi-vector.patch - git apply --ignore-space-change --ignore-whitespace --3way ../../patches/faiss/0002-Custom-patch-to-support-sqfp16-neon.patch - rm ../../patches/faiss/0002-Custom-patch-to-support-sqfp16-neon.patch - git apply --ignore-space-change --ignore-whitespace --3way ../../patches/faiss/0003-Custom-patch-to-support-AVX2-Linux-CI.patch - rm ../../patches/faiss/0003-Custom-patch-to-support-AVX2-Linux-CI.patch + git apply --ignore-space-change --ignore-whitespace --3way ../../patches/faiss/0002-Custom-patch-to-support-AVX2-Linux-CI.patch + rm ../../patches/faiss/0002-Custom-patch-to-support-AVX2-Linux-CI.patch working-directory: ${{ github.workspace }} - name: Setup Java ${{ matrix.java }} diff --git a/.github/workflows/test_security.yml b/.github/workflows/test_security.yml index cf83185f6..643552512 100644 --- a/.github/workflows/test_security.yml +++ b/.github/workflows/test_security.yml @@ -45,10 +45,8 @@ jobs: cd jni/external/faiss git apply --ignore-space-change --ignore-whitespace --3way ../../patches/faiss/0001-Custom-patch-to-support-multi-vector.patch rm ../../patches/faiss/0001-Custom-patch-to-support-multi-vector.patch - git apply --ignore-space-change --ignore-whitespace --3way ../../patches/faiss/0002-Custom-patch-to-support-sqfp16-neon.patch - rm ../../patches/faiss/0002-Custom-patch-to-support-sqfp16-neon.patch - git apply --ignore-space-change --ignore-whitespace --3way ../../patches/faiss/0003-Custom-patch-to-support-AVX2-Linux-CI.patch - rm ../../patches/faiss/0003-Custom-patch-to-support-AVX2-Linux-CI.patch + git apply --ignore-space-change --ignore-whitespace --3way ../../patches/faiss/0002-Custom-patch-to-support-AVX2-Linux-CI.patch + rm ../../patches/faiss/0002-Custom-patch-to-support-AVX2-Linux-CI.patch working-directory: ${{ github.workspace }} - name: Setup Java ${{ matrix.java }} diff --git a/CHANGELOG.md b/CHANGELOG.md index 6420f56b1..65e3c8869 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,4 +23,5 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Maintenance * Bump faiss lib commit to 32f0e8cf92cd2275b60364517bb1cce67aa29a55 [#1443](https://github.com/opensearch-project/k-NN/pull/1443) * Fix FieldInfo Parameters Mismatch [#1489](https://github.com/opensearch-project/k-NN/pull/1489) +* Upgrade faiss to 12b92e9 [#1509](https://github.com/opensearch-project/k-NN/pull/1509) ### Refactoring diff --git a/jni/CMakeLists.txt b/jni/CMakeLists.txt index 0f0b58738..c06337338 100644 --- a/jni/CMakeLists.txt +++ b/jni/CMakeLists.txt @@ -154,20 +154,12 @@ if (${CONFIG_FAISS} STREQUAL ON OR ${CONFIG_ALL} STREQUAL ON OR ${CONFIG_TEST} S endif () # Check if patch exist, this is to skip git apply during CI build. See CI.yml with ubuntu. - find_path(PATCH_FILE NAMES 0001-Custom-patch-to-support-multi-vector.patch 0002-Custom-patch-to-support-sqfp16-neon.patch PATHS ${CMAKE_CURRENT_SOURCE_DIR}/patches/faiss NO_DEFAULT_PATH) + find_path(PATCH_FILE NAMES 0001-Custom-patch-to-support-multi-vector.patch PATHS ${CMAKE_CURRENT_SOURCE_DIR}/patches/faiss NO_DEFAULT_PATH) # If it exists, apply patches if (EXISTS ${PATCH_FILE}) message(STATUS "Applying custom patches.") execute_process(COMMAND git apply --ignore-space-change --ignore-whitespace --3way ${CMAKE_CURRENT_SOURCE_DIR}/patches/faiss/0001-Custom-patch-to-support-multi-vector.patch WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/external/faiss ERROR_VARIABLE ERROR_MSG RESULT_VARIABLE RESULT_CODE) - - # 0002-Custom-patch-to-support-sqfp16-neon.patch is a temporary patch to add NEON support to SQ. - # Once the commit conflict issues wrt to Multi vector are resolved, this patch can be removed by updating the faiss submodule with corresponding commit. - # Apply the patch if the OS is not Windows and Processor is aarch64. - if (NOT ${CMAKE_SYSTEM_NAME} STREQUAL Windows AND ${CMAKE_SYSTEM_PROCESSOR} MATCHES "aarch64" AND ${SIMD_ENABLED}) - execute_process(COMMAND git apply --ignore-space-change --ignore-whitespace --3way ${CMAKE_CURRENT_SOURCE_DIR}/patches/faiss/0002-Custom-patch-to-support-sqfp16-neon.patch WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/external/faiss ERROR_VARIABLE ERROR_MSG RESULT_VARIABLE RESULT_CODE) - endif() - if(RESULT_CODE) message(FATAL_ERROR "Failed to apply patch:\n${ERROR_MSG}") endif() diff --git a/jni/external/faiss b/jni/external/faiss index 32f0e8cf9..12b92e9fa 160000 --- a/jni/external/faiss +++ b/jni/external/faiss @@ -1 +1 @@ -Subproject commit 32f0e8cf92cd2275b60364517bb1cce67aa29a55 +Subproject commit 12b92e9fa5d8e8fb3da53c57af9ff007c826b1ee diff --git a/jni/patches/faiss/0003-Custom-patch-to-support-AVX2-Linux-CI.patch b/jni/patches/faiss/0002-Custom-patch-to-support-AVX2-Linux-CI.patch similarity index 100% rename from jni/patches/faiss/0003-Custom-patch-to-support-AVX2-Linux-CI.patch rename to jni/patches/faiss/0002-Custom-patch-to-support-AVX2-Linux-CI.patch diff --git a/jni/patches/faiss/0002-Custom-patch-to-support-sqfp16-neon.patch b/jni/patches/faiss/0002-Custom-patch-to-support-sqfp16-neon.patch deleted file mode 100644 index d743d0a97..000000000 --- a/jni/patches/faiss/0002-Custom-patch-to-support-sqfp16-neon.patch +++ /dev/null @@ -1,495 +0,0 @@ -diff --git a/faiss/CMakeLists.txt b/faiss/CMakeLists.txt -index db5133d6..22dac7cb 100644 ---- a/faiss/CMakeLists.txt -+++ b/faiss/CMakeLists.txt -@@ -189,6 +189,7 @@ set(FAISS_HEADERS - utils/extra_distances.h - utils/fp16-fp16c.h - utils/fp16-inl.h -+ utils/fp16-arm.h - utils/fp16.h - utils/hamming-inl.h - utils/hamming.h -diff --git a/faiss/impl/ScalarQuantizer.cpp b/faiss/impl/ScalarQuantizer.cpp -index fc7b28ef..07d77d56 100644 ---- a/faiss/impl/ScalarQuantizer.cpp -+++ b/faiss/impl/ScalarQuantizer.cpp -@@ -91,6 +91,20 @@ struct Codec8bit { - return _mm256_fmadd_ps(f8, one_255, half_one_255); - } - #endif -+ -+#ifdef __aarch64__ -+ static FAISS_ALWAYS_INLINE float32x4x2_t -+ decode_8_components(const uint8_t* code, int i) { -+ float32_t result[8] = {}; -+ for (size_t j = 0; j < 8; j++) { -+ result[j] = decode_component(code, i + j); -+ } -+ float32x4_t res1 = vld1q_f32(result); -+ float32x4_t res2 = vld1q_f32(result + 4); -+ float32x4x2_t res = vzipq_f32(res1, res2); -+ return vuzpq_f32(res.val[0], res.val[1]); -+ } -+#endif - }; - - struct Codec4bit { -@@ -129,6 +143,20 @@ struct Codec4bit { - return _mm256_mul_ps(f8, one_255); - } - #endif -+ -+#ifdef __aarch64__ -+ static FAISS_ALWAYS_INLINE float32x4x2_t -+ decode_8_components(const uint8_t* code, int i) { -+ float32_t result[8] = {}; -+ for (size_t j = 0; j < 8; j++) { -+ result[j] = decode_component(code, i + j); -+ } -+ float32x4_t res1 = vld1q_f32(result); -+ float32x4_t res2 = vld1q_f32(result + 4); -+ float32x4x2_t res = vzipq_f32(res1, res2); -+ return vuzpq_f32(res.val[0], res.val[1]); -+ } -+#endif - }; - - struct Codec6bit { -@@ -228,6 +256,20 @@ struct Codec6bit { - } - - #endif -+ -+#ifdef __aarch64__ -+ static FAISS_ALWAYS_INLINE float32x4x2_t -+ decode_8_components(const uint8_t* code, int i) { -+ float32_t result[8] = {}; -+ for (size_t j = 0; j < 8; j++) { -+ result[j] = decode_component(code, i + j); -+ } -+ float32x4_t res1 = vld1q_f32(result); -+ float32x4_t res2 = vld1q_f32(result + 4); -+ float32x4x2_t res = vzipq_f32(res1, res2); -+ return vuzpq_f32(res.val[0], res.val[1]); -+ } -+#endif - }; - - /******************************************************************* -@@ -293,6 +335,31 @@ struct QuantizerTemplate : QuantizerTemplate { - - #endif - -+#ifdef __aarch64__ -+ -+template -+struct QuantizerTemplate : QuantizerTemplate { -+ QuantizerTemplate(size_t d, const std::vector& trained) -+ : QuantizerTemplate(d, trained) {} -+ -+ FAISS_ALWAYS_INLINE float32x4x2_t -+ reconstruct_8_components(const uint8_t* code, int i) const { -+ float32x4x2_t xi = Codec::decode_8_components(code, i); -+ float32x4x2_t res = vzipq_f32( -+ vfmaq_f32( -+ vdupq_n_f32(this->vmin), -+ xi.val[0], -+ vdupq_n_f32(this->vdiff)), -+ vfmaq_f32( -+ vdupq_n_f32(this->vmin), -+ xi.val[1], -+ vdupq_n_f32(this->vdiff))); -+ return vuzpq_f32(res.val[0], res.val[1]); -+ } -+}; -+ -+#endif -+ - template - struct QuantizerTemplate : ScalarQuantizer::SQuantizer { - const size_t d; -@@ -350,6 +417,29 @@ struct QuantizerTemplate : QuantizerTemplate { - - #endif - -+#ifdef __aarch64__ -+ -+template -+struct QuantizerTemplate : QuantizerTemplate { -+ QuantizerTemplate(size_t d, const std::vector& trained) -+ : QuantizerTemplate(d, trained) {} -+ -+ FAISS_ALWAYS_INLINE float32x4x2_t -+ reconstruct_8_components(const uint8_t* code, int i) const { -+ float32x4x2_t xi = Codec::decode_8_components(code, i); -+ -+ float32x4x2_t vmin_8 = vld1q_f32_x2(this->vmin + i); -+ float32x4x2_t vdiff_8 = vld1q_f32_x2(this->vdiff + i); -+ -+ float32x4x2_t res = vzipq_f32( -+ vfmaq_f32(vmin_8.val[0], xi.val[0], vdiff_8.val[0]), -+ vfmaq_f32(vmin_8.val[1], xi.val[1], vdiff_8.val[1])); -+ return vuzpq_f32(res.val[0], res.val[1]); -+ } -+}; -+ -+#endif -+ - /******************************************************************* - * FP16 quantizer - *******************************************************************/ -@@ -397,6 +487,23 @@ struct QuantizerFP16<8> : QuantizerFP16<1> { - - #endif - -+#ifdef __aarch64__ -+ -+template <> -+struct QuantizerFP16<8> : QuantizerFP16<1> { -+ QuantizerFP16(size_t d, const std::vector& trained) -+ : QuantizerFP16<1>(d, trained) {} -+ -+ FAISS_ALWAYS_INLINE float32x4x2_t -+ reconstruct_8_components(const uint8_t* code, int i) const { -+ uint16x4x2_t codei = vld2_u16((const uint16_t*)(code + 2 * i)); -+ return vzipq_f32( -+ vcvt_f32_f16(vreinterpret_f16_u16(codei.val[0])), -+ vcvt_f32_f16(vreinterpret_f16_u16(codei.val[1]))); -+ } -+}; -+#endif -+ - /******************************************************************* - * 8bit_direct quantizer - *******************************************************************/ -@@ -446,6 +553,28 @@ struct Quantizer8bitDirect<8> : Quantizer8bitDirect<1> { - - #endif - -+#ifdef __aarch64__ -+ -+template <> -+struct Quantizer8bitDirect<8> : Quantizer8bitDirect<1> { -+ Quantizer8bitDirect(size_t d, const std::vector& trained) -+ : Quantizer8bitDirect<1>(d, trained) {} -+ -+ FAISS_ALWAYS_INLINE float32x4x2_t -+ reconstruct_8_components(const uint8_t* code, int i) const { -+ float32_t result[8] = {}; -+ for (size_t j = 0; j < 8; j++) { -+ result[j] = code[i + j]; -+ } -+ float32x4_t res1 = vld1q_f32(result); -+ float32x4_t res2 = vld1q_f32(result + 4); -+ float32x4x2_t res = vzipq_f32(res1, res2); -+ return vuzpq_f32(res.val[0], res.val[1]); -+ } -+}; -+ -+#endif -+ - template - ScalarQuantizer::SQuantizer* select_quantizer_1( - QuantizerType qtype, -@@ -728,6 +857,59 @@ struct SimilarityL2<8> { - - #endif - -+#ifdef __aarch64__ -+template <> -+struct SimilarityL2<8> { -+ static constexpr int simdwidth = 8; -+ static constexpr MetricType metric_type = METRIC_L2; -+ -+ const float *y, *yi; -+ explicit SimilarityL2(const float* y) : y(y) {} -+ float32x4x2_t accu8; -+ -+ FAISS_ALWAYS_INLINE void begin_8() { -+ accu8 = vzipq_f32(vdupq_n_f32(0.0f), vdupq_n_f32(0.0f)); -+ yi = y; -+ } -+ -+ FAISS_ALWAYS_INLINE void add_8_components(float32x4x2_t x) { -+ float32x4x2_t yiv = vld1q_f32_x2(yi); -+ yi += 8; -+ -+ float32x4_t sub0 = vsubq_f32(yiv.val[0], x.val[0]); -+ float32x4_t sub1 = vsubq_f32(yiv.val[1], x.val[1]); -+ -+ float32x4_t accu8_0 = vfmaq_f32(accu8.val[0], sub0, sub0); -+ float32x4_t accu8_1 = vfmaq_f32(accu8.val[1], sub1, sub1); -+ -+ float32x4x2_t accu8_temp = vzipq_f32(accu8_0, accu8_1); -+ accu8 = vuzpq_f32(accu8_temp.val[0], accu8_temp.val[1]); -+ } -+ -+ FAISS_ALWAYS_INLINE void add_8_components_2( -+ float32x4x2_t x, -+ float32x4x2_t y) { -+ float32x4_t sub0 = vsubq_f32(y.val[0], x.val[0]); -+ float32x4_t sub1 = vsubq_f32(y.val[1], x.val[1]); -+ -+ float32x4_t accu8_0 = vfmaq_f32(accu8.val[0], sub0, sub0); -+ float32x4_t accu8_1 = vfmaq_f32(accu8.val[1], sub1, sub1); -+ -+ float32x4x2_t accu8_temp = vzipq_f32(accu8_0, accu8_1); -+ accu8 = vuzpq_f32(accu8_temp.val[0], accu8_temp.val[1]); -+ } -+ -+ FAISS_ALWAYS_INLINE float result_8() { -+ float32x4_t sum_0 = vpaddq_f32(accu8.val[0], accu8.val[0]); -+ float32x4_t sum_1 = vpaddq_f32(accu8.val[1], accu8.val[1]); -+ -+ float32x4_t sum2_0 = vpaddq_f32(sum_0, sum_0); -+ float32x4_t sum2_1 = vpaddq_f32(sum_1, sum_1); -+ return vgetq_lane_f32(sum2_0, 0) + vgetq_lane_f32(sum2_1, 0); -+ } -+}; -+#endif -+ - template - struct SimilarityIP {}; - -@@ -801,6 +983,56 @@ struct SimilarityIP<8> { - }; - #endif - -+#ifdef __aarch64__ -+ -+template <> -+struct SimilarityIP<8> { -+ static constexpr int simdwidth = 8; -+ static constexpr MetricType metric_type = METRIC_INNER_PRODUCT; -+ -+ const float *y, *yi; -+ -+ explicit SimilarityIP(const float* y) : y(y) {} -+ float32x4x2_t accu8; -+ -+ FAISS_ALWAYS_INLINE void begin_8() { -+ accu8 = vzipq_f32(vdupq_n_f32(0.0f), vdupq_n_f32(0.0f)); -+ yi = y; -+ } -+ -+ FAISS_ALWAYS_INLINE void add_8_components(float32x4x2_t x) { -+ float32x4x2_t yiv = vld1q_f32_x2(yi); -+ yi += 8; -+ -+ float32x4_t accu8_0 = vfmaq_f32(accu8.val[0], yiv.val[0], x.val[0]); -+ float32x4_t accu8_1 = vfmaq_f32(accu8.val[1], yiv.val[1], x.val[1]); -+ float32x4x2_t accu8_temp = vzipq_f32(accu8_0, accu8_1); -+ accu8 = vuzpq_f32(accu8_temp.val[0], accu8_temp.val[1]); -+ } -+ -+ FAISS_ALWAYS_INLINE void add_8_components_2( -+ float32x4x2_t x1, -+ float32x4x2_t x2) { -+ float32x4_t accu8_0 = vfmaq_f32(accu8.val[0], x1.val[0], x2.val[0]); -+ float32x4_t accu8_1 = vfmaq_f32(accu8.val[1], x1.val[1], x2.val[1]); -+ float32x4x2_t accu8_temp = vzipq_f32(accu8_0, accu8_1); -+ accu8 = vuzpq_f32(accu8_temp.val[0], accu8_temp.val[1]); -+ } -+ -+ FAISS_ALWAYS_INLINE float result_8() { -+ float32x4x2_t sum_tmp = vzipq_f32( -+ vpaddq_f32(accu8.val[0], accu8.val[0]), -+ vpaddq_f32(accu8.val[1], accu8.val[1])); -+ float32x4x2_t sum = vuzpq_f32(sum_tmp.val[0], sum_tmp.val[1]); -+ float32x4x2_t sum2_tmp = vzipq_f32( -+ vpaddq_f32(sum.val[0], sum.val[0]), -+ vpaddq_f32(sum.val[1], sum.val[1])); -+ float32x4x2_t sum2 = vuzpq_f32(sum2_tmp.val[0], sum2_tmp.val[1]); -+ return vgetq_lane_f32(sum2.val[0], 0) + vgetq_lane_f32(sum2.val[1], 0); -+ } -+}; -+#endif -+ - /******************************************************************* - * DistanceComputer: combines a similarity and a quantizer to do - * code-to-vector or code-to-code comparisons -@@ -903,6 +1135,53 @@ struct DCTemplate : SQDistanceComputer { - - #endif - -+#ifdef __aarch64__ -+ -+template -+struct DCTemplate : SQDistanceComputer { -+ using Sim = Similarity; -+ -+ Quantizer quant; -+ -+ DCTemplate(size_t d, const std::vector& trained) -+ : quant(d, trained) {} -+ float compute_distance(const float* x, const uint8_t* code) const { -+ Similarity sim(x); -+ sim.begin_8(); -+ for (size_t i = 0; i < quant.d; i += 8) { -+ float32x4x2_t xi = quant.reconstruct_8_components(code, i); -+ sim.add_8_components(xi); -+ } -+ return sim.result_8(); -+ } -+ -+ float compute_code_distance(const uint8_t* code1, const uint8_t* code2) -+ const { -+ Similarity sim(nullptr); -+ sim.begin_8(); -+ for (size_t i = 0; i < quant.d; i += 8) { -+ float32x4x2_t x1 = quant.reconstruct_8_components(code1, i); -+ float32x4x2_t x2 = quant.reconstruct_8_components(code2, i); -+ sim.add_8_components_2(x1, x2); -+ } -+ return sim.result_8(); -+ } -+ -+ void set_query(const float* x) final { -+ q = x; -+ } -+ -+ float symmetric_dis(idx_t i, idx_t j) override { -+ return compute_code_distance( -+ codes + i * code_size, codes + j * code_size); -+ } -+ -+ float query_to_code(const uint8_t* code) const final { -+ return compute_distance(q, code); -+ } -+}; -+#endif -+ - /******************************************************************* - * DistanceComputerByte: computes distances in the integer domain - *******************************************************************/ -@@ -1019,6 +1298,54 @@ struct DistanceComputerByte : SQDistanceComputer { - - #endif - -+#ifdef __aarch64__ -+ -+template -+struct DistanceComputerByte : SQDistanceComputer { -+ using Sim = Similarity; -+ -+ int d; -+ std::vector tmp; -+ -+ DistanceComputerByte(int d, const std::vector&) : d(d), tmp(d) {} -+ -+ int compute_code_distance(const uint8_t* code1, const uint8_t* code2) -+ const { -+ int accu = 0; -+ for (int i = 0; i < d; i++) { -+ if (Sim::metric_type == METRIC_INNER_PRODUCT) { -+ accu += int(code1[i]) * code2[i]; -+ } else { -+ int diff = int(code1[i]) - code2[i]; -+ accu += diff * diff; -+ } -+ } -+ return accu; -+ } -+ -+ void set_query(const float* x) final { -+ for (int i = 0; i < d; i++) { -+ tmp[i] = int(x[i]); -+ } -+ } -+ -+ int compute_distance(const float* x, const uint8_t* code) { -+ set_query(x); -+ return compute_code_distance(tmp.data(), code); -+ } -+ -+ float symmetric_dis(idx_t i, idx_t j) override { -+ return compute_code_distance( -+ codes + i * code_size, codes + j * code_size); -+ } -+ -+ float query_to_code(const uint8_t* code) const final { -+ return compute_code_distance(tmp.data(), code); -+ } -+}; -+ -+#endif -+ - /******************************************************************* - * select_distance_computer: runtime selection of template - * specialization -@@ -1155,7 +1482,7 @@ void ScalarQuantizer::train(size_t n, const float* x) { - } - - ScalarQuantizer::SQuantizer* ScalarQuantizer::select_quantizer() const { --#ifdef USE_F16C -+#if defined(USE_F16C) || defined(__aarch64__) - if (d % 8 == 0) { - return select_quantizer_1<8>(qtype, d, trained); - } else -@@ -1186,7 +1513,7 @@ void ScalarQuantizer::decode(const uint8_t* codes, float* x, size_t n) const { - SQDistanceComputer* ScalarQuantizer::get_distance_computer( - MetricType metric) const { - FAISS_THROW_IF_NOT(metric == METRIC_L2 || metric == METRIC_INNER_PRODUCT); --#ifdef USE_F16C -+#if defined(USE_F16C) || defined(__aarch64__) - if (d % 8 == 0) { - if (metric == METRIC_L2) { - return select_distance_computer>(qtype, d, trained); -@@ -1522,7 +1849,7 @@ InvertedListScanner* ScalarQuantizer::select_InvertedListScanner( - bool store_pairs, - const IDSelector* sel, - bool by_residual) const { --#ifdef USE_F16C -+#if defined(USE_F16C) || defined(__aarch64__) - if (d % 8 == 0) { - return sel0_InvertedListScanner<8>( - mt, this, quantizer, store_pairs, sel, by_residual); -diff --git a/faiss/utils/fp16-arm.h b/faiss/utils/fp16-arm.h -new file mode 100644 -index 00000000..79c885b0 ---- /dev/null -+++ b/faiss/utils/fp16-arm.h -@@ -0,0 +1,29 @@ -+/** -+ * Copyright (c) Facebook, Inc. and its affiliates. -+ * -+ * This source code is licensed under the MIT license found in the -+ * LICENSE file in the root directory of this source tree. -+ */ -+ -+#pragma once -+ -+#include -+#include -+ -+namespace faiss { -+ -+inline uint16_t encode_fp16(float x) { -+ float32x4_t fx4 = vdupq_n_f32(x); -+ float16x4_t f16x4 = vcvt_f16_f32(fx4); -+ uint16x4_t ui16x4 = vreinterpret_u16_f16(f16x4); -+ return vduph_lane_u16(ui16x4, 3); -+} -+ -+inline float decode_fp16(uint16_t x) { -+ uint16x4_t ui16x4 = vdup_n_u16(x); -+ float16x4_t f16x4 = vreinterpret_f16_u16(ui16x4); -+ float32x4_t fx4 = vcvt_f32_f16(f16x4); -+ return vdups_laneq_f32(fx4, 3); -+} -+ -+} // namespace faiss -diff --git a/faiss/utils/fp16.h b/faiss/utils/fp16.h -index 90691d8f..43e05dc3 100644 ---- a/faiss/utils/fp16.h -+++ b/faiss/utils/fp16.h -@@ -13,6 +13,8 @@ - - #if defined(__F16C__) - #include -+#elif defined(__aarch64__) -+#include - #else - #include - #endif diff --git a/src/main/java/org/opensearch/knn/index/util/Faiss.java b/src/main/java/org/opensearch/knn/index/util/Faiss.java index 420288033..3b21488b9 100644 --- a/src/main/java/org/opensearch/knn/index/util/Faiss.java +++ b/src/main/java/org/opensearch/knn/index/util/Faiss.java @@ -56,6 +56,9 @@ */ class Faiss extends NativeLibrary { + // TODO: Current version is not really current version. Instead, it encodes information in the file name + // about the compatibility version the file is created with. In the future, we should refactor this so that it + // makes sense. See https://github.com/opensearch-project/k-NN/issues/1515 for more details. private final static String CURRENT_VERSION = "165"; // Map that overrides OpenSearch score translation by space type of scores returned by faiss From 0400f2c0ce03b7e5ce6723578cb608b0cc20fa71 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Wed, 6 Mar 2024 19:23:40 -0800 Subject: [PATCH 210/416] Disable sdc table for HNSWPQ read-only indices (#1519) Passes flag to disable sdc table for the HNSWPQ indices. This table is only used by HNSWPQ during graph creation to compare nodes already present in graph. When we call load index, the graph is read only. Hence, we wont be doing any ingestion and so the table can be disabled to save some memory. Along with this, added a unit test and a couple test helper methods for generating random data. Signed-off-by: John Mazanec (cherry picked from commit c9262f51aa2b6a4c9bfa81f52a99a2e1a162e119) --- CHANGELOG.md | 1 + jni/src/faiss_wrapper.cpp | 3 +- jni/tests/faiss_wrapper_test.cpp | 65 +++++++++++++++++++++----------- jni/tests/test_util.cpp | 16 ++++++++ jni/tests/test_util.h | 4 ++ 5 files changed, 65 insertions(+), 24 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 65e3c8869..75feb863f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Enhancements * Optize Faiss Query With Filters: Reduce iteration and memory for id filter [#1402](https://github.com/opensearch-project/k-NN/pull/1402) ### Bug Fixes +* Disable sdc table for HNSWPQ read-only indices [#1518](https://github.com/opensearch-project/k-NN/pull/1518) ### Infrastructure * Manually install zlib for win CI [#1513](https://github.com/opensearch-project/k-NN/pull/1513) ### Documentation diff --git a/jni/src/faiss_wrapper.cpp b/jni/src/faiss_wrapper.cpp index 43f1454b5..433507dff 100644 --- a/jni/src/faiss_wrapper.cpp +++ b/jni/src/faiss_wrapper.cpp @@ -213,7 +213,8 @@ jlong knn_jni::faiss_wrapper::LoadIndex(knn_jni::JNIUtilInterface * jniUtil, JNI } std::string indexPathCpp(jniUtil->ConvertJavaStringToCppString(env, indexPathJ)); - faiss::Index* indexReader = faiss::read_index(indexPathCpp.c_str(), faiss::IO_FLAG_READ_ONLY); + // Skipping IO_FLAG_PQ_SKIP_SDC_TABLE because the index is read only and the sdc table is only used during ingestion + faiss::Index* indexReader = faiss::read_index(indexPathCpp.c_str(), faiss::IO_FLAG_READ_ONLY | faiss::IO_FLAG_PQ_SKIP_SDC_TABLE); return (jlong) indexReader; } diff --git a/jni/tests/faiss_wrapper_test.cpp b/jni/tests/faiss_wrapper_test.cpp index 4318e8ef9..03ae38dfa 100644 --- a/jni/tests/faiss_wrapper_test.cpp +++ b/jni/tests/faiss_wrapper_test.cpp @@ -22,6 +22,9 @@ using ::testing::NiceMock; using ::testing::Return; +float randomDataMin = -500.0; +float randomDataMax = 500.0; + TEST(FaissCreateIndexTest, BasicAssertions) { // Define the data faiss::idx_t numIds = 200; @@ -124,15 +127,9 @@ TEST(FaissCreateIndexFromTemplateTest, BasicAssertions) { TEST(FaissLoadIndexTest, BasicAssertions) { // Define the data faiss::idx_t numIds = 100; - std::vector ids; - std::vector vectors; int dim = 2; - for (int64_t i = 0; i < numIds; i++) { - ids.push_back(i); - for (int j = 0; j < dim; j++) { - vectors.push_back(test_util::RandomFloat(-500.0, 500.0)); - } - } + std::vector ids = test_util::Range(numIds); + std::vector vectors = test_util::RandomVectors(dim, numIds, randomDataMin, randomDataMax); std::string indexPath = test_util::RandomString(10, "tmp/", ".faiss"); faiss::MetricType metricType = faiss::METRIC_L2; @@ -173,18 +170,46 @@ TEST(FaissLoadIndexTest, BasicAssertions) { std::remove(indexPath.c_str()); } +TEST(FaissLoadIndexTest, HNSWPQDisableSdcTable) { + // Check that when we load an HNSWPQ index, the sdc table is not present. + faiss::idx_t numIds = 256; + int dim = 2; + std::vector ids = test_util::Range(numIds); + std::vector vectors = test_util::RandomVectors(dim, numIds, randomDataMin, randomDataMax); + + std::string indexPath = test_util::RandomString(10, "tmp/", ".faiss"); + faiss::MetricType metricType = faiss::METRIC_L2; + std::string indexDescription = "HNSW16,PQ1x4"; + + std::unique_ptr faissIndex(test_util::FaissCreateIndex(dim, indexDescription, metricType)); + test_util::FaissTrainIndex(faissIndex.get(), numIds, vectors.data()); + auto faissIndexWithIDMap = test_util::FaissAddData(faissIndex.get(), ids, vectors); + test_util::FaissWriteIndex(&faissIndexWithIDMap, indexPath); + + // Setup jni + JNIEnv *jniEnv = nullptr; + NiceMock mockJNIUtil; + + std::unique_ptr loadedIndexPointer( + reinterpret_cast(knn_jni::faiss_wrapper::LoadIndex( + &mockJNIUtil, jniEnv, (jstring)&indexPath))); + + // Cast down until we get to the pq backed storage index and checke the size of the table + auto idMapIndex = dynamic_cast(loadedIndexPointer.get()); + ASSERT_NE(idMapIndex, nullptr); + auto hnswPQIndex = dynamic_cast(idMapIndex->index); + ASSERT_NE(hnswPQIndex, nullptr); + auto pqIndex = dynamic_cast(hnswPQIndex->storage); + ASSERT_NE(pqIndex, nullptr); + ASSERT_EQ(0, pqIndex->pq.sdc_table.size()); +} + TEST(FaissQueryIndexTest, BasicAssertions) { // Define the index data faiss::idx_t numIds = 100; - std::vector ids; - std::vector vectors; int dim = 16; - for (int64_t i = 0; i < numIds; i++) { - ids.push_back(i); - for (int j = 0; j < dim; j++) { - vectors.push_back(test_util::RandomFloat(-500.0, 500.0)); - } - } + std::vector ids = test_util::Range(numIds); + std::vector vectors = test_util::RandomVectors(dim, numIds, randomDataMin, randomDataMax); faiss::MetricType metricType = faiss::METRIC_L2; std::string method = "HNSW32,Flat"; @@ -405,13 +430,7 @@ TEST(FaissTrainIndexTest, BasicAssertions) { // Define training data int numTrainingVectors = 256; - std::vector trainingVectors; - - for (int i = 0; i < numTrainingVectors; ++i) { - for (int j = 0; j < dim; ++j) { - trainingVectors.push_back(test_util::RandomFloat(-500.0, 500.0)); - } - } + std::vector trainingVectors = test_util::RandomVectors(dim, numTrainingVectors, randomDataMin, randomDataMax); // Setup jni JNIEnv *jniEnv = nullptr; diff --git a/jni/tests/test_util.cpp b/jni/tests/test_util.cpp index 3c6933d89..89b19f9aa 100644 --- a/jni/tests/test_util.cpp +++ b/jni/tests/test_util.cpp @@ -369,6 +369,22 @@ float test_util::RandomFloat(float min, float max) { return distribution(e1); } +std::vector test_util::RandomVectors(int dim, int64_t numVectors, float min, float max) { + std::vector vectors(dim*numVectors); + for (int64_t i = 0; i < dim*numVectors; i++) { + vectors[i] = test_util::RandomFloat(min, max); + } + return vectors; +} + +std::vector test_util::Range(int64_t numElements) { + std::vector rangeVector(numElements); + for (int64_t i = 0; i < numElements; i++) { + rangeVector[i] = i; + } + return rangeVector; +} + size_t test_util::bits2words(uint64_t numBits) { return ((numBits - 1) >> 6) + 1; } diff --git a/jni/tests/test_util.h b/jni/tests/test_util.h index a8ce6ca8f..1e32ad3c3 100644 --- a/jni/tests/test_util.h +++ b/jni/tests/test_util.h @@ -155,6 +155,10 @@ namespace test_util { float RandomFloat(float min, float max); + std::vector RandomVectors(int dim, int64_t numVectors, float min, float max); + + std::vector Range(int64_t numElements); + // returns the number of 64 bit words it would take to hold numBits size_t bits2words(uint64_t numBits); From 65545a372afd91a6f0ac05183c33c20c1275ef65 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Thu, 7 Mar 2024 18:25:34 -0500 Subject: [PATCH 211/416] Raise gcc version requirement to 9.0.0 for SIMD Neon support on ARM64 (#1517) (#1523) Signed-off-by: Peter Zhu (cherry picked from commit 7a144b84f8af1e178054d34cd716ce139209f1b5) Co-authored-by: Peter Zhu --- scripts/build.sh | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/scripts/build.sh b/scripts/build.sh index 6a9adc256..95295e710 100644 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -105,13 +105,15 @@ if [ "$JAVA_HOME" = "" ]; then echo "SET JAVA_HOME=$JAVA_HOME" fi -# Ensure gcc version is above 4.9.0 and is not 8.3.1 for faiss 1.7.4+ compilation +# Ensure gcc version is above 4.9.0 and at least 9.0.0 for faiss 1.7.4+ / SIMD Neon support on ARM64 compilation # https://github.com/opensearch-project/k-NN/issues/975 +# https://github.com/opensearch-project/k-NN/issues/1138 +# https://github.com/opensearch-project/opensearch-build/issues/4386 GCC_VERSION=`gcc --version | head -n 1 | cut -d ' ' -f3` -GCC_REQUIRED_VERSION=4.9.0 +GCC_REQUIRED_VERSION=9.0.0 COMPARE_VERSION=`echo $GCC_REQUIRED_VERSION $GCC_VERSION | tr ' ' '\n' | sort -V | uniq | head -n 1` -if [ "$COMPARE_VERSION" != "$GCC_REQUIRED_VERSION" ] || [ "$GCC_VERSION" = "8.3.1" ]; then - echo "gcc version on this env is either older than $GCC_REQUIRED_VERSION, or equals 8.3.1, exit 1" +if [ "$COMPARE_VERSION" != "$GCC_REQUIRED_VERSION" ]; then + echo "gcc version on this env is older than $GCC_REQUIRED_VERSION, exit 1" exit 1 fi From 9a952fc5ddaacc62f16c25b2383292c56df2ac1d Mon Sep 17 00:00:00 2001 From: John Mazanec Date: Wed, 13 Mar 2024 10:52:54 -0700 Subject: [PATCH 212/416] Switch spacetype similarity function to MAXIMUM_INNER_PRODUCT (#1533) Signed-off-by: John Mazanec (cherry picked from commit 62fc890e068238bd63dac713d989c4e4272028ba) --- CHANGELOG.md | 1 + .../org/opensearch/knn/index/SpaceType.java | 2 +- .../org/opensearch/knn/index/FaissIT.java | 52 +++++++++++++++++++ .../opensearch/knn/index/SpaceTypeTests.java | 38 ++++++++++++++ 4 files changed, 92 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 75feb863f..f4711683f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), * Optize Faiss Query With Filters: Reduce iteration and memory for id filter [#1402](https://github.com/opensearch-project/k-NN/pull/1402) ### Bug Fixes * Disable sdc table for HNSWPQ read-only indices [#1518](https://github.com/opensearch-project/k-NN/pull/1518) +* Switch SpaceType.INNERPRODUCT's vector similarity function to MAXIMUM_INNER_PRODUCT [#1532](https://github.com/opensearch-project/k-NN/pull/1532) ### Infrastructure * Manually install zlib for win CI [#1513](https://github.com/opensearch-project/k-NN/pull/1513) ### Documentation diff --git a/src/main/java/org/opensearch/knn/index/SpaceType.java b/src/main/java/org/opensearch/knn/index/SpaceType.java index efa0f1be3..8a9559322 100644 --- a/src/main/java/org/opensearch/knn/index/SpaceType.java +++ b/src/main/java/org/opensearch/knn/index/SpaceType.java @@ -76,7 +76,7 @@ public float scoreTranslation(float rawScore) { @Override public VectorSimilarityFunction getVectorSimilarityFunction() { - return VectorSimilarityFunction.DOT_PRODUCT; + return VectorSimilarityFunction.MAXIMUM_INNER_PRODUCT; } }, HAMMING_BIT("hammingbit") { diff --git a/src/test/java/org/opensearch/knn/index/FaissIT.java b/src/test/java/org/opensearch/knn/index/FaissIT.java index 3096d1332..2f5990ce3 100644 --- a/src/test/java/org/opensearch/knn/index/FaissIT.java +++ b/src/test/java/org/opensearch/knn/index/FaissIT.java @@ -19,6 +19,7 @@ import org.junit.BeforeClass; import org.opensearch.client.Response; import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.common.settings.Settings; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.index.query.QueryBuilders; import org.opensearch.knn.KNNRestTestCase; @@ -711,6 +712,57 @@ public void testQueryWithFilter_withDifferentCombination_thenSuccess() { assertEquals(0, emptyKNNFilteredResultsFromResponse.size()); } + @SneakyThrows + public void testFiltering_whenUsingFaissExactSearchWithIP_thenMatchExpectedScore() { + XContentBuilder builder = XContentFactory.jsonBuilder() + .startObject() + .startObject("properties") + .startObject(FIELD_NAME) + .field("type", "knn_vector") + .field("dimension", 2) + .startObject(KNNConstants.KNN_METHOD) + .field(KNNConstants.NAME, KNNEngine.FAISS.getMethod(KNNConstants.METHOD_HNSW).getMethodComponent().getName()) + .field(KNNConstants.METHOD_PARAMETER_SPACE_TYPE, SpaceType.INNER_PRODUCT.getValue()) + .field(KNNConstants.KNN_ENGINE, KNNEngine.FAISS.getName()) + .endObject() + .endObject() + .endObject() + .endObject(); + final String mapping = builder.toString(); + createKnnIndex(INDEX_NAME, mapping); + + final List dataVectors = Arrays.asList(new Float[] { -2.0f, 2.0f }, new Float[] { 2.0f, -2.0f }); + final List ids = Arrays.asList(DOC_ID_1, DOC_ID_2); + + // Ingest all of the documents + for (int i = 0; i < dataVectors.size(); i++) { + addKnnDoc(INDEX_NAME, ids.get(i), FIELD_NAME, dataVectors.get(i)); + } + refreshIndex(INDEX_NAME); + + // Execute the search request with a match all query to ensure exact logic gets called + updateIndexSettings(INDEX_NAME, Settings.builder().put(KNNSettings.ADVANCED_FILTERED_EXACT_SEARCH_THRESHOLD, 1000)); + float[] queryVector = new float[] { -2.0f, 2.0f }; + int k = 2; + final Response response = searchKNNIndex( + INDEX_NAME, + new KNNQueryBuilder(FIELD_NAME, queryVector, k, QueryBuilders.matchAllQuery()), + k + ); + final String responseBody = EntityUtils.toString(response.getEntity()); + final List knnResults = parseSearchResponseScore(responseBody, FIELD_NAME); + + // Check that the expected scores are returned + final List expectedScores = Arrays.asList( + KNNEngine.FAISS.score(8.0f, SpaceType.INNER_PRODUCT), + KNNEngine.FAISS.score(-8.0f, SpaceType.INNER_PRODUCT) + ); + assertEquals(expectedScores.size(), knnResults.size()); + for (int i = 0; i < expectedScores.size(); i++) { + assertEquals(expectedScores.get(i), knnResults.get(i), 0.0000001); + } + } + protected void setupKNNIndexForFilterQuery() throws Exception { // Create Mappings XContentBuilder builder = XContentFactory.jsonBuilder() diff --git a/src/test/java/org/opensearch/knn/index/SpaceTypeTests.java b/src/test/java/org/opensearch/knn/index/SpaceTypeTests.java index 5df9ec6f5..9d4fbfc2b 100644 --- a/src/test/java/org/opensearch/knn/index/SpaceTypeTests.java +++ b/src/test/java/org/opensearch/knn/index/SpaceTypeTests.java @@ -13,6 +13,10 @@ import org.apache.lucene.index.VectorSimilarityFunction; import org.opensearch.knn.KNNTestCase; +import org.opensearch.knn.index.util.KNNEngine; + +import java.util.Arrays; +import java.util.List; public class SpaceTypeTests extends KNNTestCase { @@ -23,4 +27,38 @@ public void testGetVectorSimilarityFunction_l2() { public void testGetVectorSimilarityFunction_invalid() { expectThrows(UnsupportedOperationException.class, SpaceType.L1::getVectorSimilarityFunction); } + + public void testGetVectorSimilarityFunction_whenInnerproduct_thenConsistentWithScoreTranslation() { + /* + For the innerproduct space type, we expect that negative dot product scores will be transformed as follows: + if (negativeDotProduct >= 0) { + return 1 / (1 + negativeDotProduct); + } + return -negativeDotProduct + 1; + + Internally, Lucene uses scaleMaxInnerProductScore to scale the raw dot product into a proper lucene score. + See: + 1. https://github.com/apache/lucene/blob/releases/lucene/9.10.0/lucene/core/src/java/org/apache/lucene/util/VectorUtil.java#L195-L200 + 2. https://github.com/apache/lucene/blob/releases/lucene/9.10.0/lucene/core/src/java/org/apache/lucene/index/VectorSimilarityFunction.java#L90 + */ + final List dataVectors = Arrays.asList( + new float[] { 0.0f, 0.0f }, + new float[] { 0.25f, -0.25f }, + new float[] { 0.125f, -0.125f }, + new float[] { 25.0f, -25.0f }, + new float[] { -0.125f, 0.125f }, + new float[] { -0.25f, 0.25f }, + new float[] { -25.0f, 25.0f } + ); + float[] queryVector = new float[] { -2.0f, 2.0f }; + List dotProducts = List.of(0.0f, -1.0f, -0.5f, -100.0f, 0.5f, 1.0f, 100.0f); + + for (int i = 0; i < dataVectors.size(); i++) { + assertEquals( + KNNEngine.FAISS.score(dotProducts.get(i), SpaceType.INNER_PRODUCT), + SpaceType.INNER_PRODUCT.getVectorSimilarityFunction().compare(queryVector, dataVectors.get(i)), + 0.0000001 + ); + } + } } From 73074806288ebd27a0de66cea4b6244bd1017c52 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Wed, 13 Mar 2024 16:45:39 -0500 Subject: [PATCH 213/416] Detect AVX2 Dynamically on the System (#1503) (#1531) * Detect avx2 on system and dynamically load the libraries Signed-off-by: Naveen Tatikonda * Add AccessController Privilege to read cpuinfo Signed-off-by: Naveen Tatikonda * Add Disable AVX2 Override Flag Signed-off-by: Naveen Tatikonda * Update build script to build avx2 enbaled library Signed-off-by: Naveen Tatikonda * Add CHANGELOG Signed-off-by: Naveen Tatikonda * Address Review Comments Signed-off-by: Naveen Tatikonda * Add Unit Tests Signed-off-by: Naveen Tatikonda * Rename JNIUtils to PlatformUtils Signed-off-by: Naveen Tatikonda --------- Signed-off-by: Naveen Tatikonda (cherry picked from commit 2d30dd9f8ba44bc6aab61d9f523c99268ec42972) Co-authored-by: Naveen Tatikonda --- .github/workflows/CI.yml | 10 +- CHANGELOG.md | 1 + DEVELOPER_GUIDE.md | 4 +- build.gradle | 6 +- jni/CMakeLists.txt | 6 +- scripts/build.sh | 10 +- .../opensearch/knn/common/KNNConstants.java | 1 + .../org/opensearch/knn/index/KNNSettings.java | 15 ++- .../org/opensearch/knn/jni/FaissService.java | 13 +- .../org/opensearch/knn/jni/PlatformUtils.java | 83 ++++++++++++ .../plugin-metadata/plugin-security.policy | 3 + .../knn/index/KNNSettingsTests.java | 12 ++ .../opensearch/knn/jni/PlatformUtilTests.java | 127 ++++++++++++++++++ 13 files changed, 277 insertions(+), 14 deletions(-) create mode 100644 src/main/java/org/opensearch/knn/jni/PlatformUtils.java create mode 100644 src/test/java/org/opensearch/knn/jni/PlatformUtilTests.java diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index b52a37702..8c7011e1c 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -61,10 +61,10 @@ jobs: if lscpu | grep -i avx2 then echo "avx2 available on system" - su `id -un 1000` -c "whoami && java -version && ./gradlew build -Dsimd.enabled=true" + su `id -un 1000` -c "whoami && java -version && ./gradlew build" else echo "avx2 not available on system" - su `id -un 1000` -c "whoami && java -version && ./gradlew build" + su `id -un 1000` -c "whoami && java -version && ./gradlew build -Dsimd.enabled=false" fi @@ -101,10 +101,10 @@ jobs: if sysctl -n machdep.cpu.features machdep.cpu.leaf7_features | grep -i AVX2 then echo "avx2 available on system" - ./gradlew build -Dsimd.enabled=true + ./gradlew build else echo "avx2 not available on system" - ./gradlew build + ./gradlew build -Dsimd.enabled=false fi Build-k-NN-Windows: @@ -158,5 +158,5 @@ jobs: - name: Run build run: | - ./gradlew.bat build + ./gradlew.bat build -D'simd.enabled=false' diff --git a/CHANGELOG.md b/CHANGELOG.md index f4711683f..2ac0929df 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Features ### Enhancements * Optize Faiss Query With Filters: Reduce iteration and memory for id filter [#1402](https://github.com/opensearch-project/k-NN/pull/1402) +* Detect AVX2 Dynamically on the System [#1502](https://github.com/opensearch-project/k-NN/pull/1502) ### Bug Fixes * Disable sdc table for HNSWPQ read-only indices [#1518](https://github.com/opensearch-project/k-NN/pull/1518) * Switch SpaceType.INNERPRODUCT's vector similarity function to MAXIMUM_INNER_PRODUCT [#1532](https://github.com/opensearch-project/k-NN/pull/1532) diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index c232e75a9..ca5e7cc52 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -238,9 +238,9 @@ If you want to make a custom patch on JNI library 4. Make a change in `jni/CmakeLists.txt`, `.github/workflows/CI.yml` to apply the patch during build ### Enable SIMD Optimization -SIMD(Single Instruction/Multiple Data) Optimization can be enabled by setting this optional parameter `simd.enabled` to `true` which boosts the performance +SIMD(Single Instruction/Multiple Data) Optimization is enabled by default on Linux and Mac which boosts the performance by enabling `AVX2` on `x86 architecture` and `NEON` on `ARM64 architecture` while building the Faiss library. But to enable SIMD, the underlying processor -should support this (AVX2 or NEON). So, by default it is set to `false`. +should support this (AVX2 or NEON). It can be disabled by setting the parameter `simd.enabled` to `false`. As of now, it is not supported on Windows OS. ``` # While building OpenSearch k-NN diff --git a/build.gradle b/build.gradle index bcec35c65..ca26b4921 100644 --- a/build.gradle +++ b/build.gradle @@ -17,7 +17,7 @@ buildscript { version_qualifier = System.getProperty("build.version_qualifier", "") opensearch_group = "org.opensearch" isSnapshot = "true" == System.getProperty("build.snapshot", "true") - simd_enabled = System.getProperty("simd.enabled", "false") + simd_enabled = System.getProperty("simd.enabled", "true") version_tokens = opensearch_version.tokenize('-') opensearch_build = version_tokens[0] + '.0' @@ -287,6 +287,10 @@ dependencies { testImplementation group: 'org.objenesis', name: 'objenesis', version: '3.2' testImplementation group: 'net.bytebuddy', name: 'byte-buddy-agent', version: '1.14.7' testFixturesImplementation "org.opensearch:common-utils:${version}" + implementation 'com.github.oshi:oshi-core:6.4.13' + api "net.java.dev.jna:jna:5.13.0" + api "net.java.dev.jna:jna-platform:5.13.0" + implementation 'org.slf4j:slf4j-api:1.7.36' zipArchive group: 'org.opensearch.plugin', name:'opensearch-security', version: "${opensearch_build}" } diff --git a/jni/CMakeLists.txt b/jni/CMakeLists.txt index c06337338..30a77d095 100644 --- a/jni/CMakeLists.txt +++ b/jni/CMakeLists.txt @@ -112,8 +112,8 @@ if (${CONFIG_FAISS} STREQUAL ON OR ${CONFIG_ALL} STREQUAL ON OR ${CONFIG_TEST} S set(BUILD_TESTING OFF) # Avoid building faiss tests set(BLA_STATIC ON) # Statically link BLAS - if(NOT SIMD_ENABLED) - set(SIMD_ENABLED false) # set default value as false if the argument is not set + if(NOT DEFINED SIMD_ENABLED) + set(SIMD_ENABLED true) # set default value as true if the argument is not set endif() if(${CMAKE_SYSTEM_NAME} STREQUAL Windows OR ${CMAKE_SYSTEM_PROCESSOR} MATCHES "aarch64" OR NOT ${SIMD_ENABLED}) @@ -122,6 +122,7 @@ if (${CONFIG_FAISS} STREQUAL ON OR ${CONFIG_ALL} STREQUAL ON OR ${CONFIG_TEST} S else() set(FAISS_OPT_LEVEL avx2) # Keep optimization level as avx2 to improve performance on Linux and Mac. set(TARGET_LINK_FAISS_LIB faiss_avx2) + string(PREPEND LIB_EXT "_avx2") # Prepend "_avx2" to lib extension to create the library as "libopensearchknn_faiss_avx2.so" on linux and "libopensearchknn_faiss_avx2.jnilib" on mac endif() if (${CMAKE_SYSTEM_NAME} STREQUAL Darwin) @@ -160,6 +161,7 @@ if (${CONFIG_FAISS} STREQUAL ON OR ${CONFIG_ALL} STREQUAL ON OR ${CONFIG_TEST} S if (EXISTS ${PATCH_FILE}) message(STATUS "Applying custom patches.") execute_process(COMMAND git apply --ignore-space-change --ignore-whitespace --3way ${CMAKE_CURRENT_SOURCE_DIR}/patches/faiss/0001-Custom-patch-to-support-multi-vector.patch WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/external/faiss ERROR_VARIABLE ERROR_MSG RESULT_VARIABLE RESULT_CODE) + if(RESULT_CODE) message(FATAL_ERROR "Failed to apply patch:\n${ERROR_MSG}") endif() diff --git a/scripts/build.sh b/scripts/build.sh index 95295e710..b2cdef687 100644 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -119,8 +119,14 @@ fi # Build k-NN lib and plugin through gradle tasks cd $work_dir -# Gradle build is used here to replace gradle assemble due to build will also call cmake and make before generating jars -./gradlew build --no-daemon --refresh-dependencies -x integTest -DskipTests=true -Dopensearch.version=$VERSION -Dbuild.snapshot=$SNAPSHOT -Dbuild.version_qualifier=$QUALIFIER +./gradlew build --no-daemon --refresh-dependencies -x integTest -x test -Dopensearch.version=$VERSION -Dbuild.snapshot=$SNAPSHOT -Dbuild.version_qualifier=$QUALIFIER +./gradlew :buildJniLib -Dsimd.enabled=false + +if [ "$PLATFORM" != "windows" ] && [ "$ARCHITECTURE" = "x64" ]; then + echo "Building k-NN library after enabling AVX2" + ./gradlew :buildJniLib -Dsimd.enabled=true +fi + ./gradlew publishPluginZipPublicationToZipStagingRepository -Dopensearch.version=$VERSION -Dbuild.snapshot=$SNAPSHOT -Dbuild.version_qualifier=$QUALIFIER ./gradlew publishPluginZipPublicationToMavenLocal -Dbuild.snapshot=$SNAPSHOT -Dbuild.version_qualifier=$QUALIFIER -Dopensearch.version=$VERSION diff --git a/src/main/java/org/opensearch/knn/common/KNNConstants.java b/src/main/java/org/opensearch/knn/common/KNNConstants.java index b8089d858..fdec9d37c 100644 --- a/src/main/java/org/opensearch/knn/common/KNNConstants.java +++ b/src/main/java/org/opensearch/knn/common/KNNConstants.java @@ -113,6 +113,7 @@ public class KNNConstants { // Lib names private static final String JNI_LIBRARY_PREFIX = "opensearchknn_"; public static final String FAISS_JNI_LIBRARY_NAME = JNI_LIBRARY_PREFIX + FAISS_NAME; + public static final String FAISS_AVX2_JNI_LIBRARY_NAME = JNI_LIBRARY_PREFIX + FAISS_NAME + "_avx2"; public static final String NMSLIB_JNI_LIBRARY_NAME = JNI_LIBRARY_PREFIX + NMSLIB_NAME; // Filtered Search Constants diff --git a/src/main/java/org/opensearch/knn/index/KNNSettings.java b/src/main/java/org/opensearch/knn/index/KNNSettings.java index 81a8f6f2e..ba172bc30 100644 --- a/src/main/java/org/opensearch/knn/index/KNNSettings.java +++ b/src/main/java/org/opensearch/knn/index/KNNSettings.java @@ -76,6 +76,7 @@ public class KNNSettings { public static final String MODEL_INDEX_NUMBER_OF_REPLICAS = "knn.model.index.number_of_replicas"; public static final String MODEL_CACHE_SIZE_LIMIT = "knn.model.cache.size.limit"; public static final String ADVANCED_FILTERED_EXACT_SEARCH_THRESHOLD = "index.knn.advanced.filtered_exact_search_threshold"; + public static final String KNN_FAISS_AVX2_DISABLED = "knn.faiss.avx2.disabled"; /** * Default setting values @@ -230,6 +231,9 @@ public class KNNSettings { NodeScope, Dynamic ); + + public static final Setting KNN_FAISS_AVX2_DISABLED_SETTING = Setting.boolSetting(KNN_FAISS_AVX2_DISABLED, false, NodeScope); + /** * Dynamic settings */ @@ -339,6 +343,10 @@ private Setting getSetting(String key) { return ADVANCED_FILTERED_EXACT_SEARCH_THRESHOLD_SETTING; } + if (KNN_FAISS_AVX2_DISABLED.equals(key)) { + return KNN_FAISS_AVX2_DISABLED_SETTING; + } + throw new IllegalArgumentException("Cannot find setting by key [" + key + "]"); } @@ -355,7 +363,8 @@ public List> getSettings() { MODEL_INDEX_NUMBER_OF_SHARDS_SETTING, MODEL_INDEX_NUMBER_OF_REPLICAS_SETTING, MODEL_CACHE_SIZE_LIMIT_SETTING, - ADVANCED_FILTERED_EXACT_SEARCH_THRESHOLD_SETTING + ADVANCED_FILTERED_EXACT_SEARCH_THRESHOLD_SETTING, + KNN_FAISS_AVX2_DISABLED_SETTING ); return Stream.concat(settings.stream(), dynamicCacheSettings.values().stream()).collect(Collectors.toList()); } @@ -376,6 +385,10 @@ public static double getCircuitBreakerUnsetPercentage() { return KNNSettings.state().getSettingValue(KNNSettings.KNN_CIRCUIT_BREAKER_UNSET_PERCENTAGE); } + public static boolean isFaissAVX2Disabled() { + return KNNSettings.state().getSettingValue(KNNSettings.KNN_FAISS_AVX2_DISABLED); + } + public static Integer getFilteredExactSearchThreshold(final String indexName) { return KNNSettings.state().clusterService.state() .getMetadata() diff --git a/src/main/java/org/opensearch/knn/jni/FaissService.java b/src/main/java/org/opensearch/knn/jni/FaissService.java index f330352ec..a1a07547b 100644 --- a/src/main/java/org/opensearch/knn/jni/FaissService.java +++ b/src/main/java/org/opensearch/knn/jni/FaissService.java @@ -19,6 +19,9 @@ import java.security.PrivilegedAction; import java.util.Map; +import static org.opensearch.knn.index.KNNSettings.isFaissAVX2Disabled; +import static org.opensearch.knn.jni.PlatformUtils.isAVX2SupportedBySystem; + /** * Service to interact with faiss jni layer. Class dependencies should be minimal * @@ -31,7 +34,15 @@ class FaissService { static { AccessController.doPrivileged((PrivilegedAction) () -> { - System.loadLibrary(KNNConstants.FAISS_JNI_LIBRARY_NAME); + + // Even if the underlying system supports AVX2, users can override and disable it by using the + // 'knn.faiss.avx2.disabled' setting by setting it to true in the opensearch.yml configuration + if (!isFaissAVX2Disabled() && isAVX2SupportedBySystem()) { + System.loadLibrary(KNNConstants.FAISS_AVX2_JNI_LIBRARY_NAME); + } else { + System.loadLibrary(KNNConstants.FAISS_JNI_LIBRARY_NAME); + } + initLibrary(); KNNEngine.FAISS.setInitialized(true); return null; diff --git a/src/main/java/org/opensearch/knn/jni/PlatformUtils.java b/src/main/java/org/opensearch/knn/jni/PlatformUtils.java new file mode 100644 index 000000000..8a5549dec --- /dev/null +++ b/src/main/java/org/opensearch/knn/jni/PlatformUtils.java @@ -0,0 +1,83 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.knn.jni; + +import com.sun.jna.Platform; +import org.apache.commons.lang.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import oshi.util.platform.mac.SysctlUtil; + +import java.nio.file.Files; +import java.nio.file.Paths; +import java.security.AccessController; +import java.security.PrivilegedExceptionAction; +import java.util.Locale; + +public class PlatformUtils { + + private static final Logger logger = LogManager.getLogger(PlatformUtils.class); + + /** + * Verify if the underlying system supports AVX2 SIMD Optimization or not + * 1. If the architecture is not x86 return false. + * 2. If the operating system is not Mac or Linux return false(for example Windows). + * 3. If the operating system is macOS, use oshi library to verify if the cpu flags + * contains 'avx2' and return true if it exists else false. + * 4. If the operating system is linux, read the '/proc/cpuinfo' file path and verify if + * the flags contains 'avx2' and return true if it exists else false. + */ + public static boolean isAVX2SupportedBySystem() { + if (!Platform.isIntel()) { + return false; + } + + if (Platform.isMac()) { + + // sysctl or system control retrieves system info and allows processes with appropriate privileges + // to set system info. This system info contains the machine dependent cpu features that are supported by it. + // On MacOS, if the underlying processor supports AVX2 instruction set, it will be listed under the "leaf7" + // subset of instructions ("sysctl -a | grep machdep.cpu.leaf7_features"). + // https://developer.apple.com/library/archive/documentation/System/Conceptual/ManPages_iPhoneOS/man3/sysctl.3.html + try { + return AccessController.doPrivileged((PrivilegedExceptionAction) () -> { + String flags = SysctlUtil.sysctl("machdep.cpu.leaf7_features", "empty"); + return (flags.toLowerCase(Locale.ROOT)).contains("avx2"); + }); + } catch (Exception e) { + logger.error("[KNN] Error fetching cpu flags info. [{}]", e.getMessage(), e); + } + + } else if (Platform.isLinux()) { + + // The "/proc/cpuinfo" is a virtual file which identifies and provides the processor details used + // by system. This info contains "flags" for each processor which determines the qualities of that processor + // and it's ability to process different instruction sets like mmx, avx, avx2 and so on. + // https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/6/html/deployment_guide/s2-proc-cpuinfo + // Here, we are trying to read the details of all processors used by system and find if any of the processor + // supports AVX2 instructions. Pentium and Celeron are a couple of examples which doesn't support AVX2 + // https://ark.intel.com/content/www/us/en/ark/products/199285/intel-pentium-gold-g6600-processor-4m-cache-4-20-ghz.html + String fileName = "/proc/cpuinfo"; + try { + return AccessController.doPrivileged( + (PrivilegedExceptionAction) () -> (Boolean) Files.lines(Paths.get(fileName)) + .filter(s -> s.startsWith("flags")) + .anyMatch(s -> StringUtils.containsIgnoreCase(s, "avx2")) + ); + + } catch (Exception e) { + logger.error("[KNN] Error reading file [{}]. [{}]", fileName, e.getMessage(), e); + } + } + return false; + } +} diff --git a/src/main/plugin-metadata/plugin-security.policy b/src/main/plugin-metadata/plugin-security.policy index c7345edcd..91624613c 100644 --- a/src/main/plugin-metadata/plugin-security.policy +++ b/src/main/plugin-metadata/plugin-security.policy @@ -1,5 +1,8 @@ grant { permission java.lang.RuntimePermission "loadLibrary.opensearchknn_nmslib"; permission java.lang.RuntimePermission "loadLibrary.opensearchknn_faiss"; + permission java.lang.RuntimePermission "loadLibrary.opensearchknn_faiss_avx2"; permission java.net.SocketPermission "*", "connect,resolve"; + permission java.lang.RuntimePermission "accessDeclaredMembers"; + permission java.io.FilePermission "/proc/cpuinfo", "read"; }; diff --git a/src/test/java/org/opensearch/knn/index/KNNSettingsTests.java b/src/test/java/org/opensearch/knn/index/KNNSettingsTests.java index 6b4751afa..8008c547f 100644 --- a/src/test/java/org/opensearch/knn/index/KNNSettingsTests.java +++ b/src/test/java/org/opensearch/knn/index/KNNSettingsTests.java @@ -162,6 +162,18 @@ public void testGetEfSearch_whenEFSearchValueSetByUser_thenReturnValue() { assertEquals(userProvidedEfSearch, efSearchValue); } + @SneakyThrows + public void testGetFaissAVX2DisabledSettingValueFromConfig_enableSetting_thenValidateAndSucceed() { + boolean expectedKNNFaissAVX2Disabled = true; + Node mockNode = createMockNode(Map.of(KNNSettings.KNN_FAISS_AVX2_DISABLED, expectedKNNFaissAVX2Disabled)); + mockNode.start(); + ClusterService clusterService = mockNode.injector().getInstance(ClusterService.class); + KNNSettings.state().setClusterService(clusterService); + boolean actualKNNFaissAVX2Disabled = KNNSettings.state().getSettingValue(KNNSettings.KNN_FAISS_AVX2_DISABLED); + mockNode.close(); + assertEquals(expectedKNNFaissAVX2Disabled, actualKNNFaissAVX2Disabled); + } + private Node createMockNode(Map configSettings) throws IOException { Path configDir = createTempDir(); File configFile = configDir.resolve("opensearch.yml").toFile(); diff --git a/src/test/java/org/opensearch/knn/jni/PlatformUtilTests.java b/src/test/java/org/opensearch/knn/jni/PlatformUtilTests.java new file mode 100644 index 000000000..7816505de --- /dev/null +++ b/src/test/java/org/opensearch/knn/jni/PlatformUtilTests.java @@ -0,0 +1,127 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.knn.jni; + +import com.sun.jna.Platform; +import org.mockito.MockedStatic; +import org.opensearch.knn.KNNTestCase; +import oshi.util.platform.mac.SysctlUtil; + +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.stream.Stream; + +import static org.mockito.Mockito.mockStatic; +import static org.opensearch.knn.jni.PlatformUtils.isAVX2SupportedBySystem; + +public class PlatformUtilTests extends KNNTestCase { + public static final String MAC_CPU_FEATURES = "machdep.cpu.leaf7_features"; + public static final String LINUX_PROC_CPU_INFO = "/proc/cpuinfo"; + + public void testIsAVX2SupportedBySystem_platformIsNotIntel_returnsFalse() { + try (MockedStatic mockedPlatform = mockStatic(Platform.class)) { + mockedPlatform.when(Platform::isIntel).thenReturn(false); + assertFalse(isAVX2SupportedBySystem()); + } + } + + public void testIsAVX2SupportedBySystem_platformIsIntelWithOSAsWindows_returnsFalse() { + try (MockedStatic mockedPlatform = mockStatic(Platform.class)) { + mockedPlatform.when(Platform::isIntel).thenReturn(true); + mockedPlatform.when(Platform::isWindows).thenReturn(true); + assertFalse(isAVX2SupportedBySystem()); + } + } + + public void testIsAVX2SupportedBySystem_platformIsMac_returnsTrue() { + try (MockedStatic mockedPlatform = mockStatic(Platform.class)) { + mockedPlatform.when(Platform::isIntel).thenReturn(true); + mockedPlatform.when(Platform::isMac).thenReturn(true); + + try (MockedStatic mockedSysctlUtil = mockStatic(SysctlUtil.class)) { + mockedSysctlUtil.when(() -> SysctlUtil.sysctl(MAC_CPU_FEATURES, "empty")) + .thenReturn( + "RDWRFSGS TSC_THREAD_OFFSET SGX BMI1 AVX2 SMEP BMI2 ERMS INVPCID FPU_CSDS MPX RDSEED ADX SMAP CLFSOPT IPT SGXLC MDCLEAR TSXFA IBRS STIBP L1DF ACAPMSR SSBD" + ); + assertTrue(isAVX2SupportedBySystem()); + } + } + } + + public void testIsAVX2SupportedBySystem_platformIsMac_returnsFalse() { + try (MockedStatic mockedPlatform = mockStatic(Platform.class)) { + mockedPlatform.when(Platform::isIntel).thenReturn(true); + mockedPlatform.when(Platform::isMac).thenReturn(true); + + try (MockedStatic mockedSysctlUtil = mockStatic(SysctlUtil.class)) { + mockedSysctlUtil.when(() -> SysctlUtil.sysctl(MAC_CPU_FEATURES, "empty")).thenReturn("NO Flags"); + assertFalse(isAVX2SupportedBySystem()); + } + } + + } + + public void testIsAVX2SupportedBySystem_platformIsMac_throwsExceptionReturnsFalse() { + try (MockedStatic mockedPlatform = mockStatic(Platform.class)) { + mockedPlatform.when(Platform::isIntel).thenReturn(true); + mockedPlatform.when(Platform::isMac).thenReturn(true); + + try (MockedStatic mockedSysctlUtil = mockStatic(SysctlUtil.class)) { + mockedSysctlUtil.when(() -> SysctlUtil.sysctl(MAC_CPU_FEATURES, "empty")).thenThrow(RuntimeException.class); + assertFalse(isAVX2SupportedBySystem()); + } + } + + } + + public void testIsAVX2SupportedBySystem_platformIsLinux_returnsTrue() { + try (MockedStatic mockedPlatform = mockStatic(Platform.class)) { + mockedPlatform.when(Platform::isIntel).thenReturn(true); + mockedPlatform.when(Platform::isMac).thenReturn(false); + mockedPlatform.when(Platform::isLinux).thenReturn(true); + + try (MockedStatic mockedFiles = mockStatic(Files.class)) { + mockedFiles.when(() -> Files.lines(Paths.get(LINUX_PROC_CPU_INFO))).thenReturn(Stream.of("flags: AVX2", "dummy string")); + assertTrue(isAVX2SupportedBySystem()); + } + } + } + + public void testIsAVX2SupportedBySystem_platformIsLinux_returnsFalse() { + try (MockedStatic mockedPlatform = mockStatic(Platform.class)) { + mockedPlatform.when(Platform::isIntel).thenReturn(true); + mockedPlatform.when(Platform::isMac).thenReturn(false); + mockedPlatform.when(Platform::isLinux).thenReturn(true); + + try (MockedStatic mockedFiles = mockStatic(Files.class)) { + mockedFiles.when(() -> Files.lines(Paths.get(LINUX_PROC_CPU_INFO))).thenReturn(Stream.of("flags: ", "dummy string")); + assertFalse(isAVX2SupportedBySystem()); + } + } + + } + + public void testIsAVX2SupportedBySystem_platformIsLinux_throwsExceptionReturnsFalse() { + try (MockedStatic mockedPlatform = mockStatic(Platform.class)) { + mockedPlatform.when(Platform::isIntel).thenReturn(true); + mockedPlatform.when(Platform::isMac).thenReturn(false); + mockedPlatform.when(Platform::isLinux).thenReturn(true); + + try (MockedStatic mockedPaths = mockStatic(Paths.class)) { + mockedPaths.when(() -> Paths.get(LINUX_PROC_CPU_INFO)).thenThrow(RuntimeException.class); + assertFalse(isAVX2SupportedBySystem()); + } + } + + } + +} From e3e3aa12529b91da2c26cd7cdb831433966edeeb Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Wed, 13 Mar 2024 18:37:30 -0500 Subject: [PATCH 214/416] Bump jinja2 dependency from 2.11.3 to 3.1.3 (#1535) (#1536) Signed-off-by: Naveen Tatikonda (cherry picked from commit 089db16ac7c8e73e50dc36f8c0d95a3256a1cfa9) Co-authored-by: Naveen Tatikonda --- benchmarks/osb/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmarks/osb/requirements.txt b/benchmarks/osb/requirements.txt index 2da38cfaa..3d41012f2 100644 --- a/benchmarks/osb/requirements.txt +++ b/benchmarks/osb/requirements.txt @@ -38,7 +38,7 @@ ijson==2.6.1 # via opensearch-benchmark importlib-metadata==4.11.3 # via jsonschema -jinja2==2.11.3 +jinja2==3.1.3 # via opensearch-benchmark jsonschema==3.1.1 # via opensearch-benchmark From b61da1f5bf003bfebf3fc9210bd15349440c0b63 Mon Sep 17 00:00:00 2001 From: John Mazanec Date: Thu, 14 Mar 2024 11:12:37 -0700 Subject: [PATCH 215/416] Validate zero vector when using cosine metric (#1538) Ensure zero vector is not used when using functionality with cosine similarity metric. (cherry picked from commit b7bdda4f7d05b3e3e4248020ce01b2c888359a7e) Signed-off-by: panguixin --- CHANGELOG.md | 1 + .../knn/common/KNNValidationUtil.java | 83 +++++++++++++++++++ .../opensearch/knn/common/KNNVectorUtil.java | 45 ++++++++++ .../org/opensearch/knn/index/SpaceType.java | 39 +++++++++ .../index/mapper/KNNVectorFieldMapper.java | 80 +++++++++++------- .../mapper/KNNVectorFieldMapperUtil.java | 65 +-------------- .../knn/index/mapper/LuceneFieldMapper.java | 13 +-- .../knn/index/mapper/ModelFieldMapper.java | 2 +- .../knn/index/query/KNNQueryBuilder.java | 37 +++++---- .../knn/plugin/script/KNNScoringSpace.java | 4 +- .../plugin/script/KNNScoringSpaceUtil.java | 6 +- .../knn/plugin/script/KNNScoringUtil.java | 24 +++--- .../knn/common/KNNVectorUtilTests.java | 26 ++++++ .../opensearch/knn/index/OpenSearchIT.java | 31 +++++++ .../mapper/KNNVectorFieldMapperTests.java | 25 +++--- .../knn/index/query/KNNQueryBuilderTests.java | 48 +++++++++++ .../script/KNNScoringSpaceFactoryTests.java | 5 +- .../plugin/script/KNNScoringSpaceTests.java | 19 ++++- .../plugin/script/KNNScoringUtilTests.java | 50 +++++++++-- .../org/opensearch/knn/KNNRestTestCase.java | 23 +++++ 20 files changed, 472 insertions(+), 154 deletions(-) create mode 100644 src/main/java/org/opensearch/knn/common/KNNValidationUtil.java create mode 100644 src/main/java/org/opensearch/knn/common/KNNVectorUtil.java create mode 100644 src/test/java/org/opensearch/knn/common/KNNVectorUtilTests.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 2ac0929df..08f248690 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Enhancements * Optize Faiss Query With Filters: Reduce iteration and memory for id filter [#1402](https://github.com/opensearch-project/k-NN/pull/1402) * Detect AVX2 Dynamically on the System [#1502](https://github.com/opensearch-project/k-NN/pull/1502) +* Validate zero vector when using cosine metric [#1501](https://github.com/opensearch-project/k-NN/pull/1501) ### Bug Fixes * Disable sdc table for HNSWPQ read-only indices [#1518](https://github.com/opensearch-project/k-NN/pull/1518) * Switch SpaceType.INNERPRODUCT's vector similarity function to MAXIMUM_INNER_PRODUCT [#1532](https://github.com/opensearch-project/k-NN/pull/1532) diff --git a/src/main/java/org/opensearch/knn/common/KNNValidationUtil.java b/src/main/java/org/opensearch/knn/common/KNNValidationUtil.java new file mode 100644 index 000000000..ca8e1459a --- /dev/null +++ b/src/main/java/org/opensearch/knn/common/KNNValidationUtil.java @@ -0,0 +1,83 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.knn.common; + +import java.util.Locale; +import lombok.AccessLevel; +import lombok.NoArgsConstructor; +import org.opensearch.knn.index.VectorDataType; + +import static org.opensearch.knn.common.KNNConstants.VECTOR_DATA_TYPE_FIELD; + +@NoArgsConstructor(access = AccessLevel.PRIVATE) +public class KNNValidationUtil { + /** + * Validate the float vector value and throw exception if it is not a number or not in the finite range. + * + * @param value float vector value + */ + public static void validateFloatVectorValue(float value) { + if (Float.isNaN(value)) { + throw new IllegalArgumentException("KNN vector values cannot be NaN"); + } + + if (Float.isInfinite(value)) { + throw new IllegalArgumentException("KNN vector values cannot be infinity"); + } + } + + /** + * Validate the float vector value in the byte range if it is a finite number, + * with no decimal values and in the byte range of [-128 to 127]. If not throw IllegalArgumentException. + * + * @param value float value in byte range + */ + public static void validateByteVectorValue(float value) { + validateFloatVectorValue(value); + if (value % 1 != 0) { + throw new IllegalArgumentException( + String.format( + Locale.ROOT, + "[%s] field was set as [%s] in index mapping. But, KNN vector values are floats instead of byte integers", + VECTOR_DATA_TYPE_FIELD, + VectorDataType.BYTE.getValue() + ) + + ); + } + if ((int) value < Byte.MIN_VALUE || (int) value > Byte.MAX_VALUE) { + throw new IllegalArgumentException( + String.format( + Locale.ROOT, + "[%s] field was set as [%s] in index mapping. But, KNN vector values are not within in the byte range [%d, %d]", + VECTOR_DATA_TYPE_FIELD, + VectorDataType.BYTE.getValue(), + Byte.MIN_VALUE, + Byte.MAX_VALUE + ) + ); + } + } + + /** + * Validate if the given vector size matches with the dimension provided in mapping. + * + * @param dimension dimension of vector + * @param vectorSize size of the vector + */ + public static void validateVectorDimension(int dimension, int vectorSize) { + if (dimension != vectorSize) { + String errorMessage = String.format(Locale.ROOT, "Vector dimension mismatch. Expected: %d, Given: %d", dimension, vectorSize); + throw new IllegalArgumentException(errorMessage); + } + } +} diff --git a/src/main/java/org/opensearch/knn/common/KNNVectorUtil.java b/src/main/java/org/opensearch/knn/common/KNNVectorUtil.java new file mode 100644 index 000000000..fd9e5b6c2 --- /dev/null +++ b/src/main/java/org/opensearch/knn/common/KNNVectorUtil.java @@ -0,0 +1,45 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.common; + +import java.util.Objects; +import lombok.AccessLevel; +import lombok.NoArgsConstructor; + +@NoArgsConstructor(access = AccessLevel.PRIVATE) +public class KNNVectorUtil { + /** + * Check if all the elements of a given vector are zero + * + * @param vector the vector + * @return true if yes; otherwise false + */ + public static boolean isZeroVector(byte[] vector) { + Objects.requireNonNull(vector, "vector must not be null"); + for (byte e : vector) { + if (e != 0) { + return false; + } + } + return true; + } + + /** + * Check if all the elements of a given vector are zero + * + * @param vector the vector + * @return true if yes; otherwise false + */ + public static boolean isZeroVector(float[] vector) { + Objects.requireNonNull(vector, "vector must not be null"); + for (float e : vector) { + if (e != 0f) { + return false; + } + } + return true; + } +} diff --git a/src/main/java/org/opensearch/knn/index/SpaceType.java b/src/main/java/org/opensearch/knn/index/SpaceType.java index 8a9559322..50d8d352c 100644 --- a/src/main/java/org/opensearch/knn/index/SpaceType.java +++ b/src/main/java/org/opensearch/knn/index/SpaceType.java @@ -11,11 +11,14 @@ package org.opensearch.knn.index; +import java.util.Locale; import org.apache.lucene.index.VectorSimilarityFunction; import java.util.HashSet; import java.util.Set; +import static org.opensearch.knn.common.KNNVectorUtil.isZeroVector; + /** * Enum contains spaces supported for approximate nearest neighbor search in the k-NN plugin. Each engine's methods are * expected to support a subset of these spaces. Validation should be done in the jni layer and an exception should be @@ -44,6 +47,24 @@ public float scoreTranslation(float rawScore) { public VectorSimilarityFunction getVectorSimilarityFunction() { return VectorSimilarityFunction.COSINE; } + + @Override + public void validateVector(byte[] vector) { + if (isZeroVector(vector)) { + throw new IllegalArgumentException( + String.format(Locale.ROOT, "zero vector is not supported when space type is [%s]", getValue()) + ); + } + } + + @Override + public void validateVector(float[] vector) { + if (isZeroVector(vector)) { + throw new IllegalArgumentException( + String.format(Locale.ROOT, "zero vector is not supported when space type is [%s]", getValue()) + ); + } + } }, L1("l1") { @Override @@ -105,6 +126,24 @@ public VectorSimilarityFunction getVectorSimilarityFunction() { throw new UnsupportedOperationException(String.format("Space [%s] does not have a vector similarity function", getValue())); } + /** + * Validate if the given byte vector is supported by this space type + * + * @param vector the given vector + */ + public void validateVector(byte[] vector) { + // do nothing + } + + /** + * Validate if the given float vector is supported by this space type + * + * @param vector the given vector + */ + public void validateVector(float[] vector) { + // do nothing + } + /** * Get space type name in engine * diff --git a/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapper.java b/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapper.java index 5b427517b..2369a6937 100644 --- a/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapper.java +++ b/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapper.java @@ -5,22 +5,29 @@ package org.opensearch.knn.index.mapper; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Optional; +import java.util.function.Supplier; import lombok.Getter; import lombok.extern.log4j.Log4j2; -import org.opensearch.Version; -import org.opensearch.common.ValidationException; -import org.opensearch.knn.common.KNNConstants; - import org.apache.lucene.document.FieldType; import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.IndexOptions; import org.apache.lucene.search.DocValuesFieldExistsQuery; import org.apache.lucene.search.Query; +import org.opensearch.Version; import org.opensearch.common.Explicit; +import org.opensearch.common.Nullable; +import org.opensearch.common.ValidationException; +import org.opensearch.common.xcontent.support.XContentMapValues; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; -import org.opensearch.common.xcontent.support.XContentMapValues; import org.opensearch.index.fielddata.IndexFieldData; import org.opensearch.index.mapper.FieldMapper; import org.opensearch.index.mapper.MappedFieldType; @@ -32,9 +39,11 @@ import org.opensearch.index.mapper.ValueFetcher; import org.opensearch.index.query.QueryShardContext; import org.opensearch.index.query.QueryShardException; +import org.opensearch.knn.common.KNNConstants; import org.opensearch.knn.index.KNNMethodContext; import org.opensearch.knn.index.KNNSettings; import org.opensearch.knn.index.KNNVectorIndexFieldData; +import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.VectorField; import org.opensearch.knn.index.util.KNNEngine; @@ -42,25 +51,16 @@ import org.opensearch.search.aggregations.support.CoreValuesSourceType; import org.opensearch.search.lookup.SearchLookup; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.Optional; -import java.util.function.Supplier; - import static org.opensearch.knn.common.KNNConstants.DEFAULT_VECTOR_DATA_TYPE_FIELD; import static org.opensearch.knn.common.KNNConstants.KNN_METHOD; import static org.opensearch.knn.common.KNNConstants.VECTOR_DATA_TYPE_FIELD; +import static org.opensearch.knn.common.KNNValidationUtil.validateByteVectorValue; +import static org.opensearch.knn.common.KNNValidationUtil.validateFloatVectorValue; +import static org.opensearch.knn.common.KNNValidationUtil.validateVectorDimension; import static org.opensearch.knn.index.KNNSettings.KNN_INDEX; +import static org.opensearch.knn.index.mapper.KNNVectorFieldMapperUtil.addStoredFieldForVectorField; import static org.opensearch.knn.index.mapper.KNNVectorFieldMapperUtil.validateVectorDataTypeWithEngine; import static org.opensearch.knn.index.mapper.KNNVectorFieldMapperUtil.validateVectorDataTypeWithKnnIndexSetting; -import static org.opensearch.knn.index.mapper.KNNVectorFieldMapperUtil.addStoredFieldForVectorField; -import static org.opensearch.knn.index.mapper.KNNVectorFieldMapperUtil.validateByteVectorValue; -import static org.opensearch.knn.index.mapper.KNNVectorFieldMapperUtil.validateFloatVectorValue; -import static org.opensearch.knn.index.mapper.KNNVectorFieldMapperUtil.validateVectorDimension; /** * Field Mapper for KNN vector type. @@ -313,7 +313,13 @@ public KNNVectorFieldMapper build(BuilderContext context) { return new LegacyFieldMapper( name, - new KNNVectorFieldType(buildFullName(context), metaValue, dimension.getValue(), vectorDataType.getValue()), + new KNNVectorFieldType( + buildFullName(context), + metaValue, + dimension.getValue(), + vectorDataType.getValue(), + SpaceType.getSpace(spaceType) + ), multiFieldsBuilder, copyToBuilder, ignoreMalformed, @@ -384,17 +390,24 @@ public static class KNNVectorFieldType extends MappedFieldType { String modelId; KNNMethodContext knnMethodContext; VectorDataType vectorDataType; + SpaceType spaceType; - public KNNVectorFieldType(String name, Map meta, int dimension, VectorDataType vectorDataType) { - this(name, meta, dimension, null, null, vectorDataType); + public KNNVectorFieldType( + String name, + Map meta, + int dimension, + VectorDataType vectorDataType, + SpaceType spaceType + ) { + this(name, meta, dimension, null, null, vectorDataType, spaceType); } public KNNVectorFieldType(String name, Map meta, int dimension, KNNMethodContext knnMethodContext) { - this(name, meta, dimension, knnMethodContext, null, DEFAULT_VECTOR_DATA_TYPE_FIELD); + this(name, meta, dimension, knnMethodContext, null, DEFAULT_VECTOR_DATA_TYPE_FIELD, knnMethodContext.getSpaceType()); } public KNNVectorFieldType(String name, Map meta, int dimension, KNNMethodContext knnMethodContext, String modelId) { - this(name, meta, dimension, knnMethodContext, modelId, DEFAULT_VECTOR_DATA_TYPE_FIELD); + this(name, meta, dimension, knnMethodContext, modelId, DEFAULT_VECTOR_DATA_TYPE_FIELD, null); } public KNNVectorFieldType( @@ -404,22 +417,24 @@ public KNNVectorFieldType( KNNMethodContext knnMethodContext, VectorDataType vectorDataType ) { - this(name, meta, dimension, knnMethodContext, null, vectorDataType); + this(name, meta, dimension, knnMethodContext, null, vectorDataType, knnMethodContext.getSpaceType()); } public KNNVectorFieldType( String name, Map meta, int dimension, - KNNMethodContext knnMethodContext, - String modelId, - VectorDataType vectorDataType + @Nullable KNNMethodContext knnMethodContext, + @Nullable String modelId, + VectorDataType vectorDataType, + @Nullable SpaceType spaceType ) { super(name, false, false, true, TextSearchInfo.NONE, meta); this.dimension = dimension; this.modelId = modelId; this.knnMethodContext = knnMethodContext; this.vectorDataType = vectorDataType; + this.spaceType = spaceType; } @Override @@ -496,10 +511,10 @@ protected String contentType() { @Override protected void parseCreateField(ParseContext context) throws IOException { - parseCreateField(context, fieldType().getDimension()); + parseCreateField(context, fieldType().getDimension(), fieldType().getSpaceType()); } - protected void parseCreateField(ParseContext context, int dimension) throws IOException { + protected void parseCreateField(ParseContext context, int dimension, SpaceType spaceType) throws IOException { validateIfKNNPluginEnabled(); validateIfCircuitBreakerIsNotTriggered(); @@ -507,10 +522,11 @@ protected void parseCreateField(ParseContext context, int dimension) throws IOEx if (VectorDataType.BYTE == vectorDataType) { Optional bytesArrayOptional = getBytesFromContext(context, dimension); - if (!bytesArrayOptional.isPresent()) { + if (bytesArrayOptional.isEmpty()) { return; } final byte[] array = bytesArrayOptional.get(); + spaceType.validateVector(array); VectorField point = new VectorField(name(), array, fieldType); context.doc().add(point); @@ -518,12 +534,12 @@ protected void parseCreateField(ParseContext context, int dimension) throws IOEx } else if (VectorDataType.FLOAT == vectorDataType) { Optional floatsArrayOptional = getFloatsFromContext(context, dimension); - if (!floatsArrayOptional.isPresent()) { + if (floatsArrayOptional.isEmpty()) { return; } final float[] array = floatsArrayOptional.get(); + spaceType.validateVector(array); VectorField point = new VectorField(name(), array, fieldType); - context.doc().add(point); addStoredFieldForVectorField(context, fieldType, name(), point.toString()); } else { diff --git a/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperUtil.java b/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperUtil.java index bf331eeb3..b525b9dc6 100644 --- a/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperUtil.java +++ b/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperUtil.java @@ -11,6 +11,8 @@ package org.opensearch.knn.index.mapper; +import lombok.AccessLevel; +import lombok.NoArgsConstructor; import org.apache.lucene.document.FieldType; import org.apache.lucene.document.StoredField; import org.apache.lucene.index.DocValuesType; @@ -25,69 +27,8 @@ import static org.opensearch.knn.common.KNNConstants.LUCENE_NAME; import static org.opensearch.knn.common.KNNConstants.VECTOR_DATA_TYPE_FIELD; +@NoArgsConstructor(access = AccessLevel.PRIVATE) public class KNNVectorFieldMapperUtil { - /** - * Validate the float vector value and throw exception if it is not a number or not in the finite range. - * - * @param value float vector value - */ - public static void validateFloatVectorValue(float value) { - if (Float.isNaN(value)) { - throw new IllegalArgumentException("KNN vector values cannot be NaN"); - } - - if (Float.isInfinite(value)) { - throw new IllegalArgumentException("KNN vector values cannot be infinity"); - } - } - - /** - * Validate the float vector value in the byte range if it is a finite number, - * with no decimal values and in the byte range of [-128 to 127]. If not throw IllegalArgumentException. - * - * @param value float value in byte range - */ - public static void validateByteVectorValue(float value) { - validateFloatVectorValue(value); - if (value % 1 != 0) { - throw new IllegalArgumentException( - String.format( - Locale.ROOT, - "[%s] field was set as [%s] in index mapping. But, KNN vector values are floats instead of byte integers", - VECTOR_DATA_TYPE_FIELD, - VectorDataType.BYTE.getValue() - ) - - ); - } - if ((int) value < Byte.MIN_VALUE || (int) value > Byte.MAX_VALUE) { - throw new IllegalArgumentException( - String.format( - Locale.ROOT, - "[%s] field was set as [%s] in index mapping. But, KNN vector values are not within in the byte range [%d, %d]", - VECTOR_DATA_TYPE_FIELD, - VectorDataType.BYTE.getValue(), - Byte.MIN_VALUE, - Byte.MAX_VALUE - ) - ); - } - } - - /** - * Validate if the given vector size matches with the dimension provided in mapping. - * - * @param dimension dimension of vector - * @param vectorSize size of the vector - */ - public static void validateVectorDimension(int dimension, int vectorSize) { - if (dimension != vectorSize) { - String errorMessage = String.format(Locale.ROOT, "Vector dimension mismatch. Expected: %d, Given: %d", dimension, vectorSize); - throw new IllegalArgumentException(errorMessage); - } - - } - /** * Validates and throws exception if data_type field is set in the index mapping * using any VectorDataType (other than float, which is default) because other diff --git a/src/main/java/org/opensearch/knn/index/mapper/LuceneFieldMapper.java b/src/main/java/org/opensearch/knn/index/mapper/LuceneFieldMapper.java index 831a23f4b..81c7216bf 100644 --- a/src/main/java/org/opensearch/knn/index/mapper/LuceneFieldMapper.java +++ b/src/main/java/org/opensearch/knn/index/mapper/LuceneFieldMapper.java @@ -5,6 +5,9 @@ package org.opensearch.knn.index.mapper; +import java.io.IOException; +import java.util.Locale; +import java.util.Optional; import lombok.AllArgsConstructor; import lombok.Getter; import lombok.NonNull; @@ -15,14 +18,11 @@ import org.opensearch.common.Explicit; import org.opensearch.index.mapper.ParseContext; import org.opensearch.knn.index.KNNMethodContext; +import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.VectorField; import org.opensearch.knn.index.util.KNNEngine; -import java.io.IOException; -import java.util.Locale; -import java.util.Optional; - import static org.opensearch.knn.common.KNNConstants.VECTOR_DATA_TYPE_FIELD; import static org.opensearch.knn.index.mapper.KNNVectorFieldMapperUtil.addStoredFieldForVectorField; import static org.opensearch.knn.index.mapper.KNNVectorFieldMapperUtil.buildDocValuesFieldType; @@ -75,7 +75,7 @@ public class LuceneFieldMapper extends KNNVectorFieldMapper { } @Override - protected void parseCreateField(ParseContext context, int dimension) throws IOException { + protected void parseCreateField(ParseContext context, int dimension, SpaceType spaceType) throws IOException { validateIfKNNPluginEnabled(); validateIfCircuitBreakerIsNotTriggered(); @@ -86,6 +86,7 @@ protected void parseCreateField(ParseContext context, int dimension) throws IOEx return; } final byte[] array = bytesArrayOptional.get(); + spaceType.validateVector(array); KnnByteVectorField point = new KnnByteVectorField(name(), array, fieldType); context.doc().add(point); @@ -101,7 +102,7 @@ protected void parseCreateField(ParseContext context, int dimension) throws IOEx return; } final float[] array = floatsArrayOptional.get(); - + spaceType.validateVector(array); KnnVectorField point = new KnnVectorField(name(), array, fieldType); context.doc().add(point); diff --git a/src/main/java/org/opensearch/knn/index/mapper/ModelFieldMapper.java b/src/main/java/org/opensearch/knn/index/mapper/ModelFieldMapper.java index 2f138dba6..2367d7422 100644 --- a/src/main/java/org/opensearch/knn/index/mapper/ModelFieldMapper.java +++ b/src/main/java/org/opensearch/knn/index/mapper/ModelFieldMapper.java @@ -61,6 +61,6 @@ protected void parseCreateField(ParseContext context) throws IOException { ); } - parseCreateField(context, modelMetadata.getDimension()); + parseCreateField(context, modelMetadata.getDimension(), modelMetadata.getSpaceType()); } } diff --git a/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java b/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java index 4779101d4..7287ce3c6 100644 --- a/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java +++ b/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java @@ -5,19 +5,13 @@ package org.opensearch.knn.index.query; +import java.io.IOException; import java.util.Arrays; +import java.util.List; +import java.util.Objects; import lombok.extern.log4j.Log4j2; import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.commons.lang.StringUtils; -import org.opensearch.index.mapper.NumberFieldMapper; -import org.opensearch.index.query.QueryBuilder; -import org.opensearch.knn.index.KNNMethodContext; -import org.opensearch.knn.index.VectorDataType; -import org.opensearch.knn.index.mapper.KNNVectorFieldMapper; -import org.opensearch.knn.index.util.KNNEngine; -import org.opensearch.knn.indices.ModelDao; -import org.opensearch.knn.indices.ModelMetadata; -import org.opensearch.knn.plugin.stats.KNNCounter; import org.apache.lucene.search.Query; import org.opensearch.core.ParseField; import org.opensearch.core.common.ParsingException; @@ -26,15 +20,22 @@ import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.index.mapper.MappedFieldType; +import org.opensearch.index.mapper.NumberFieldMapper; import org.opensearch.index.query.AbstractQueryBuilder; +import org.opensearch.index.query.QueryBuilder; import org.opensearch.index.query.QueryShardContext; - -import java.io.IOException; -import java.util.List; -import java.util.Objects; +import org.opensearch.knn.index.KNNMethodContext; +import org.opensearch.knn.index.SpaceType; +import org.opensearch.knn.index.VectorDataType; +import org.opensearch.knn.index.mapper.KNNVectorFieldMapper; +import org.opensearch.knn.index.util.KNNEngine; +import org.opensearch.knn.indices.ModelDao; +import org.opensearch.knn.indices.ModelMetadata; +import org.opensearch.knn.plugin.stats.KNNCounter; import static org.opensearch.knn.index.IndexUtil.*; -import static org.opensearch.knn.index.mapper.KNNVectorFieldMapperUtil.validateByteVectorValue; +import static org.opensearch.knn.common.KNNValidationUtil.validateByteVectorValue; +import static org.opensearch.knn.index.IndexUtil.isClusterOnOrAfterMinRequiredVersion; /** * Helper class to build the KNN query @@ -314,12 +315,17 @@ protected Query doToQuery(QueryShardContext context) { KNNMethodContext knnMethodContext = knnVectorFieldType.getKnnMethodContext(); KNNEngine knnEngine = KNNEngine.DEFAULT; VectorDataType vectorDataType = knnVectorFieldType.getVectorDataType(); + SpaceType spaceType = knnVectorFieldType.getSpaceType(); if (fieldDimension == -1) { + if (spaceType != null) { + throw new IllegalStateException("Space type should be null when the field uses a model"); + } // If dimension is not set, the field uses a model and the information needs to be retrieved from there ModelMetadata modelMetadata = getModelMetadataForField(knnVectorFieldType); fieldDimension = modelMetadata.getDimension(); knnEngine = modelMetadata.getKnnEngine(); + spaceType = modelMetadata.getSpaceType(); } else if (knnMethodContext != null) { // If the dimension is set but the knnMethodContext is not then the field is using the legacy mapping knnEngine = knnMethodContext.getKnnEngine(); @@ -338,6 +344,9 @@ protected Query doToQuery(QueryShardContext context) { validateByteVectorValue(vector[i]); byteVector[i] = (byte) vector[i]; } + spaceType.validateVector(byteVector); + } else { + spaceType.validateVector(vector); } if (KNNEngine.getEnginesThatCreateCustomSegmentFiles().contains(knnEngine) diff --git a/src/main/java/org/opensearch/knn/plugin/script/KNNScoringSpace.java b/src/main/java/org/opensearch/knn/plugin/script/KNNScoringSpace.java index 0e4c9f815..5a8cdb036 100644 --- a/src/main/java/org/opensearch/knn/plugin/script/KNNScoringSpace.java +++ b/src/main/java/org/opensearch/knn/plugin/script/KNNScoringSpace.java @@ -6,6 +6,7 @@ package org.opensearch.knn.plugin.script; import org.apache.lucene.search.IndexSearcher; +import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.mapper.KNNVectorFieldMapper; import org.opensearch.knn.index.query.KNNWeight; import org.apache.lucene.index.LeafReaderContext; @@ -90,7 +91,7 @@ class CosineSimilarity implements KNNScoringSpace { */ public CosineSimilarity(Object query, MappedFieldType fieldType) { if (!isKNNVectorFieldType(fieldType)) { - throw new IllegalArgumentException("Incompatible field_type for cosine space. The field type must " + "be knn_vector."); + throw new IllegalArgumentException("Incompatible field_type for cosine space. The field type must be knn_vector."); } this.processedQuery = parseToFloatArray( @@ -98,6 +99,7 @@ public CosineSimilarity(Object query, MappedFieldType fieldType) { ((KNNVectorFieldMapper.KNNVectorFieldType) fieldType).getDimension(), ((KNNVectorFieldMapper.KNNVectorFieldType) fieldType).getVectorDataType() ); + SpaceType.COSINESIMIL.validateVector(processedQuery); float qVectorSquaredMagnitude = getVectorMagnitudeSquared(this.processedQuery); this.scoringMethod = (float[] q, float[] v) -> 1 + KNNScoringUtil.cosinesimilOptimized(q, v, qVectorSquaredMagnitude); } diff --git a/src/main/java/org/opensearch/knn/plugin/script/KNNScoringSpaceUtil.java b/src/main/java/org/opensearch/knn/plugin/script/KNNScoringSpaceUtil.java index 3ec1a9941..c482413fb 100644 --- a/src/main/java/org/opensearch/knn/plugin/script/KNNScoringSpaceUtil.java +++ b/src/main/java/org/opensearch/knn/plugin/script/KNNScoringSpaceUtil.java @@ -5,6 +5,7 @@ package org.opensearch.knn.plugin.script; +import java.util.List; import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.mapper.KNNVectorFieldMapper; import org.opensearch.knn.plugin.stats.KNNCounter; @@ -13,11 +14,10 @@ import org.opensearch.index.mapper.NumberFieldMapper; import java.math.BigInteger; -import java.util.ArrayList; import java.util.Base64; import static org.opensearch.index.mapper.NumberFieldMapper.NumberType.LONG; -import static org.opensearch.knn.index.mapper.KNNVectorFieldMapperUtil.validateByteVectorValue; +import static org.opensearch.knn.common.KNNValidationUtil.validateByteVectorValue; public class KNNScoringSpaceUtil { @@ -108,7 +108,7 @@ public static float[] parseToFloatArray(Object object, int expectedDimensions, V public static float[] convertVectorToPrimitive(Object vector, VectorDataType vectorDataType) { float[] primitiveVector = null; if (vector != null) { - final ArrayList tmp = (ArrayList) vector; + final List tmp = (List) vector; primitiveVector = new float[tmp.size()]; for (int i = 0; i < primitiveVector.length; i++) { float value = tmp.get(i).floatValue(); diff --git a/src/main/java/org/opensearch/knn/plugin/script/KNNScoringUtil.java b/src/main/java/org/opensearch/knn/plugin/script/KNNScoringUtil.java index 130c4d8e0..114499100 100644 --- a/src/main/java/org/opensearch/knn/plugin/script/KNNScoringUtil.java +++ b/src/main/java/org/opensearch/knn/plugin/script/KNNScoringUtil.java @@ -5,16 +5,16 @@ package org.opensearch.knn.plugin.script; -import org.opensearch.knn.index.KNNVectorScriptDocValues; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.opensearch.knn.index.VectorDataType; - import java.math.BigInteger; import java.util.List; import java.util.Objects; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.knn.index.KNNVectorScriptDocValues; +import org.opensearch.knn.index.SpaceType; +import org.opensearch.knn.index.VectorDataType; -import static org.opensearch.knn.index.mapper.KNNVectorFieldMapperUtil.validateByteVectorValue; +import static org.opensearch.knn.common.KNNValidationUtil.validateByteVectorValue; public class KNNScoringUtil { private static Logger logger = LogManager.getLogger(KNNScoringUtil.class); @@ -134,11 +134,9 @@ public static float cosinesimilOptimized(float[] queryVector, float[] inputVecto * @return cosine score */ public static float cosineSimilarity(List queryVector, KNNVectorScriptDocValues docValues, Number queryVectorMagnitude) { - return cosinesimilOptimized( - toFloat(queryVector, docValues.getVectorDataType()), - docValues.getValue(), - queryVectorMagnitude.floatValue() - ); + float[] inputVector = toFloat(queryVector, docValues.getVectorDataType()); + SpaceType.COSINESIMIL.validateVector(inputVector); + return cosinesimilOptimized(inputVector, docValues.getValue(), queryVectorMagnitude.floatValue()); } /** @@ -183,7 +181,9 @@ public static float cosinesimil(float[] queryVector, float[] inputVector) { * @return cosine score */ public static float cosineSimilarity(List queryVector, KNNVectorScriptDocValues docValues) { - return cosinesimil(toFloat(queryVector, docValues.getVectorDataType()), docValues.getValue()); + float[] inputVector = toFloat(queryVector, docValues.getVectorDataType()); + SpaceType.COSINESIMIL.validateVector(inputVector); + return cosinesimil(inputVector, docValues.getValue()); } /** diff --git a/src/test/java/org/opensearch/knn/common/KNNVectorUtilTests.java b/src/test/java/org/opensearch/knn/common/KNNVectorUtilTests.java new file mode 100644 index 000000000..457ea8c5b --- /dev/null +++ b/src/test/java/org/opensearch/knn/common/KNNVectorUtilTests.java @@ -0,0 +1,26 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.knn.common; + +import org.opensearch.knn.KNNTestCase; + +public class KNNVectorUtilTests extends KNNTestCase { + public void testByteZeroVector() { + assertTrue(KNNVectorUtil.isZeroVector(new byte[] { 0, 0, 0 })); + assertFalse(KNNVectorUtil.isZeroVector(new byte[] { 1, 1, 1 })); + } + + public void testFloatZeroVector() { + assertTrue(KNNVectorUtil.isZeroVector(new float[] { 0.0f, 0.0f, 0.0f })); + assertFalse(KNNVectorUtil.isZeroVector(new float[] { 1.0f, 1.0f, 1.0f })); + } +} diff --git a/src/test/java/org/opensearch/knn/index/OpenSearchIT.java b/src/test/java/org/opensearch/knn/index/OpenSearchIT.java index 35d59cbd9..98a1e1b71 100644 --- a/src/test/java/org/opensearch/knn/index/OpenSearchIT.java +++ b/src/test/java/org/opensearch/knn/index/OpenSearchIT.java @@ -13,6 +13,8 @@ import com.google.common.collect.ImmutableList; import com.google.common.primitives.Floats; +import java.util.Locale; +import lombok.SneakyThrows; import org.junit.BeforeClass; import org.opensearch.knn.KNNRestTestCase; import org.opensearch.knn.KNNResult; @@ -263,6 +265,35 @@ public void testIndexingVectorValidation_differentSizes() throws Exception { assertThat(EntityUtils.toString(ex.getResponse().getEntity()), containsString("Vector dimension mismatch. Expected: 4, Given: 5")); } + @SneakyThrows + public void testIndexingVectorValidation_zeroVector() { + Settings settings = Settings.builder().put(getKNNDefaultIndexSettings()).build(); + final boolean valid = randomBoolean(); + final String method = KNNConstants.METHOD_HNSW; + String engine; + String spaceType; + if (valid) { + engine = randomFrom(KNNEngine.values()).getName(); + spaceType = SpaceType.L2.getValue(); + } else { + engine = randomFrom(KNNConstants.LUCENE_NAME, KNNConstants.NMSLIB_NAME); + spaceType = SpaceType.COSINESIMIL.getValue(); + } + createKnnIndex(INDEX_NAME, settings, createKnnIndexMapping(FIELD_NAME, 4, method, engine, spaceType)); + Float[] zeroVector = { 0.0f, 0.0f, 0.0f, 0.0f }; + if (valid) { + addKnnDoc(INDEX_NAME, "1", FIELD_NAME, zeroVector); + } else { + ResponseException ex = expectThrows(ResponseException.class, () -> addKnnDoc(INDEX_NAME, "1", FIELD_NAME, zeroVector)); + assertTrue( + EntityUtils.toString(ex.getResponse().getEntity()) + .contains( + String.format(Locale.ROOT, "zero vector is not supported when space type is [%s]", SpaceType.COSINESIMIL.getValue()) + ) + ); + } + } + public void testVectorMappingValidation_noDimension() throws Exception { Settings settings = Settings.builder().put(getKNNDefaultIndexSettings()).build(); diff --git a/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java b/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java index 4a5db8c8f..278f90ba2 100644 --- a/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java +++ b/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java @@ -71,21 +71,21 @@ public class KNNVectorFieldMapperTests extends KNNTestCase { - private final static String TEST_FIELD_NAME = "test-field-name"; + private static final String TEST_FIELD_NAME = "test-field-name"; - private final static int TEST_DIMENSION = 17; + private static final int TEST_DIMENSION = 17; - private final static float TEST_VECTOR_VALUE = 1.5f; + private static final float TEST_VECTOR_VALUE = 1.5f; - private final static float[] TEST_VECTOR = createInitializedFloatArray(TEST_DIMENSION, TEST_VECTOR_VALUE); + private static final float[] TEST_VECTOR = createInitializedFloatArray(TEST_DIMENSION, TEST_VECTOR_VALUE); - private final static byte TEST_BYTE_VECTOR_VALUE = 10; - private final static byte[] TEST_BYTE_VECTOR = createInitializedByteArray(TEST_DIMENSION, TEST_BYTE_VECTOR_VALUE); + private static final byte TEST_BYTE_VECTOR_VALUE = 10; + private static final byte[] TEST_BYTE_VECTOR = createInitializedByteArray(TEST_DIMENSION, TEST_BYTE_VECTOR_VALUE); - private final static BytesRef TEST_VECTOR_BYTES_REF = new BytesRef( + private static final BytesRef TEST_VECTOR_BYTES_REF = new BytesRef( KNNVectorSerializerFactory.getDefaultSerializer().floatToByteArray(TEST_VECTOR) ); - private final static BytesRef TEST_BYTE_VECTOR_BYTES_REF = new BytesRef(TEST_BYTE_VECTOR); + private static final BytesRef TEST_BYTE_VECTOR_BYTES_REF = new BytesRef(TEST_BYTE_VECTOR); private static final String DIMENSION_FIELD_NAME = "dimension"; private static final String KNN_VECTOR_TYPE = "knn_vector"; private static final String TYPE_FIELD_NAME = "type"; @@ -757,8 +757,7 @@ public void testLuceneFieldMapper_parseCreateField_docValues_withFloats() { doReturn(Optional.of(TEST_VECTOR)).when(luceneFieldMapper).getFloatsFromContext(parseContext, TEST_DIMENSION); doNothing().when(luceneFieldMapper).validateIfCircuitBreakerIsNotTriggered(); doNothing().when(luceneFieldMapper).validateIfKNNPluginEnabled(); - - luceneFieldMapper.parseCreateField(parseContext, TEST_DIMENSION); + luceneFieldMapper.parseCreateField(parseContext, TEST_DIMENSION, luceneFieldMapper.fieldType().spaceType); // Document should have 2 fields: one for VectorField (binary doc values) and one for KnnVectorField List fields = document.getFields(); @@ -796,7 +795,7 @@ public void testLuceneFieldMapper_parseCreateField_docValues_withFloats() { doNothing().when(luceneFieldMapper).validateIfCircuitBreakerIsNotTriggered(); doNothing().when(luceneFieldMapper).validateIfKNNPluginEnabled(); - luceneFieldMapper.parseCreateField(parseContext, TEST_DIMENSION); + luceneFieldMapper.parseCreateField(parseContext, TEST_DIMENSION, luceneFieldMapper.fieldType().spaceType); // Document should have 1 field: one for KnnVectorField fields = document.getFields(); @@ -825,7 +824,7 @@ public void testLuceneFieldMapper_parseCreateField_docValues_withBytes() { doNothing().when(luceneFieldMapper).validateIfCircuitBreakerIsNotTriggered(); doNothing().when(luceneFieldMapper).validateIfKNNPluginEnabled(); - luceneFieldMapper.parseCreateField(parseContext, TEST_DIMENSION); + luceneFieldMapper.parseCreateField(parseContext, TEST_DIMENSION, luceneFieldMapper.fieldType().spaceType); // Document should have 2 fields: one for VectorField (binary doc values) and one for KnnByteVectorField List fields = document.getFields(); @@ -862,7 +861,7 @@ public void testLuceneFieldMapper_parseCreateField_docValues_withBytes() { doNothing().when(luceneFieldMapper).validateIfCircuitBreakerIsNotTriggered(); doNothing().when(luceneFieldMapper).validateIfKNNPluginEnabled(); - luceneFieldMapper.parseCreateField(parseContext, TEST_DIMENSION); + luceneFieldMapper.parseCreateField(parseContext, TEST_DIMENSION, luceneFieldMapper.fieldType().spaceType); // Document should have 1 field: one for KnnByteVectorField fields = document.getFields(); diff --git a/src/test/java/org/opensearch/knn/index/query/KNNQueryBuilderTests.java b/src/test/java/org/opensearch/knn/index/query/KNNQueryBuilderTests.java index 8c045ea74..987d44422 100644 --- a/src/test/java/org/opensearch/knn/index/query/KNNQueryBuilderTests.java +++ b/src/test/java/org/opensearch/knn/index/query/KNNQueryBuilderTests.java @@ -6,6 +6,7 @@ package org.opensearch.knn.index.query; import com.google.common.collect.ImmutableMap; +import java.util.Locale; import org.apache.lucene.search.KnnFloatVectorQuery; import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.Query; @@ -244,6 +245,7 @@ public void testDoToQuery_Normal() throws Exception { when(mockQueryShardContext.index()).thenReturn(dummyIndex); when(mockKNNVectorField.getDimension()).thenReturn(4); when(mockKNNVectorField.getVectorDataType()).thenReturn(VectorDataType.FLOAT); + when(mockKNNVectorField.getSpaceType()).thenReturn(SpaceType.L2); when(mockQueryShardContext.fieldMapper(anyString())).thenReturn(mockKNNVectorField); KNNQuery query = (KNNQuery) knnQueryBuilder.doToQuery(mockQueryShardContext); assertEquals(knnQueryBuilder.getK(), query.getK()); @@ -260,6 +262,7 @@ public void testDoToQuery_KnnQueryWithFilter() throws Exception { when(mockQueryShardContext.index()).thenReturn(dummyIndex); when(mockKNNVectorField.getDimension()).thenReturn(4); when(mockKNNVectorField.getVectorDataType()).thenReturn(VectorDataType.FLOAT); + when(mockKNNVectorField.getSpaceType()).thenReturn(SpaceType.L2); MethodComponentContext methodComponentContext = new MethodComponentContext(METHOD_HNSW, ImmutableMap.of()); KNNMethodContext knnMethodContext = new KNNMethodContext(KNNEngine.LUCENE, SpaceType.L2, methodComponentContext); when(mockKNNVectorField.getKnnMethodContext()).thenReturn(knnMethodContext); @@ -277,6 +280,7 @@ public void testDoToQuery_WhenknnQueryWithFilterAndFaissEngine_thenSuccess() { KNNVectorFieldMapper.KNNVectorFieldType mockKNNVectorField = mock(KNNVectorFieldMapper.KNNVectorFieldType.class); when(mockQueryShardContext.index()).thenReturn(dummyIndex); when(mockKNNVectorField.getDimension()).thenReturn(4); + when(mockKNNVectorField.getSpaceType()).thenReturn(SpaceType.L2); MethodComponentContext methodComponentContext = new MethodComponentContext(METHOD_HNSW, ImmutableMap.of()); KNNMethodContext knnMethodContext = new KNNMethodContext(KNNEngine.FAISS, SpaceType.L2, methodComponentContext); when(mockKNNVectorField.getKnnMethodContext()).thenReturn(knnMethodContext); @@ -294,6 +298,7 @@ public void testDoToQuery_whenknnQueryWithFilterAndNmsLibEngine_thenException() KNNVectorFieldMapper.KNNVectorFieldType mockKNNVectorField = mock(KNNVectorFieldMapper.KNNVectorFieldType.class); when(mockQueryShardContext.index()).thenReturn(dummyIndex); when(mockKNNVectorField.getDimension()).thenReturn(4); + when(mockKNNVectorField.getSpaceType()).thenReturn(SpaceType.L2); MethodComponentContext methodComponentContext = new MethodComponentContext(METHOD_HNSW, ImmutableMap.of()); KNNMethodContext knnMethodContext = new KNNMethodContext(KNNEngine.NMSLIB, SpaceType.L2, methodComponentContext); when(mockKNNVectorField.getKnnMethodContext()).thenReturn(knnMethodContext); @@ -320,6 +325,7 @@ public void testDoToQuery_FromModel() { ModelMetadata modelMetadata = mock(ModelMetadata.class); when(modelMetadata.getDimension()).thenReturn(4); when(modelMetadata.getKnnEngine()).thenReturn(KNNEngine.FAISS); + when(modelMetadata.getSpaceType()).thenReturn(SpaceType.COSINESIMIL); ModelDao modelDao = mock(ModelDao.class); when(modelDao.getMetadata(modelId)).thenReturn(modelMetadata); KNNQueryBuilder.initialize(modelDao); @@ -356,6 +362,48 @@ public void testDoToQuery_InvalidFieldType() throws IOException { expectThrows(IllegalArgumentException.class, () -> knnQueryBuilder.doToQuery(mockQueryShardContext)); } + public void testDoToQuery_InvalidZeroFloatVector() { + float[] queryVector = { 0.0f, 0.0f, 0.0f, 0.0f }; + KNNQueryBuilder knnQueryBuilder = new KNNQueryBuilder(FIELD_NAME, queryVector, K); + Index dummyIndex = new Index("dummy", "dummy"); + QueryShardContext mockQueryShardContext = mock(QueryShardContext.class); + KNNVectorFieldMapper.KNNVectorFieldType mockKNNVectorField = mock(KNNVectorFieldMapper.KNNVectorFieldType.class); + when(mockQueryShardContext.index()).thenReturn(dummyIndex); + when(mockKNNVectorField.getDimension()).thenReturn(4); + when(mockKNNVectorField.getVectorDataType()).thenReturn(VectorDataType.FLOAT); + when(mockKNNVectorField.getSpaceType()).thenReturn(SpaceType.COSINESIMIL); + when(mockQueryShardContext.fieldMapper(anyString())).thenReturn(mockKNNVectorField); + IllegalArgumentException exception = expectThrows( + IllegalArgumentException.class, + () -> knnQueryBuilder.doToQuery(mockQueryShardContext) + ); + assertEquals( + String.format(Locale.ROOT, "zero vector is not supported when space type is [%s]", SpaceType.COSINESIMIL.getValue()), + exception.getMessage() + ); + } + + public void testDoToQuery_InvalidZeroByteVector() { + float[] queryVector = { 0.0f, 0.0f, 0.0f, 0.0f }; + KNNQueryBuilder knnQueryBuilder = new KNNQueryBuilder(FIELD_NAME, queryVector, K); + Index dummyIndex = new Index("dummy", "dummy"); + QueryShardContext mockQueryShardContext = mock(QueryShardContext.class); + KNNVectorFieldMapper.KNNVectorFieldType mockKNNVectorField = mock(KNNVectorFieldMapper.KNNVectorFieldType.class); + when(mockQueryShardContext.index()).thenReturn(dummyIndex); + when(mockKNNVectorField.getDimension()).thenReturn(4); + when(mockKNNVectorField.getVectorDataType()).thenReturn(VectorDataType.BYTE); + when(mockKNNVectorField.getSpaceType()).thenReturn(SpaceType.COSINESIMIL); + when(mockQueryShardContext.fieldMapper(anyString())).thenReturn(mockKNNVectorField); + IllegalArgumentException exception = expectThrows( + IllegalArgumentException.class, + () -> knnQueryBuilder.doToQuery(mockQueryShardContext) + ); + assertEquals( + String.format(Locale.ROOT, "zero vector is not supported when space type is [%s]", SpaceType.COSINESIMIL.getValue()), + exception.getMessage() + ); + } + public void testSerialization() throws Exception { assertSerialization(Version.CURRENT, Optional.empty()); diff --git a/src/test/java/org/opensearch/knn/plugin/script/KNNScoringSpaceFactoryTests.java b/src/test/java/org/opensearch/knn/plugin/script/KNNScoringSpaceFactoryTests.java index 24bc74ff4..52bc22eff 100644 --- a/src/test/java/org/opensearch/knn/plugin/script/KNNScoringSpaceFactoryTests.java +++ b/src/test/java/org/opensearch/knn/plugin/script/KNNScoringSpaceFactoryTests.java @@ -10,20 +10,21 @@ import org.opensearch.knn.index.SpaceType; import org.opensearch.index.mapper.NumberFieldMapper; -import java.util.ArrayList; import java.util.List; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; public class KNNScoringSpaceFactoryTests extends KNNTestCase { public void testValidSpaces() { KNNVectorFieldMapper.KNNVectorFieldType knnVectorFieldType = mock(KNNVectorFieldMapper.KNNVectorFieldType.class); + when(knnVectorFieldType.getDimension()).thenReturn(3); NumberFieldMapper.NumberFieldType numberFieldType = new NumberFieldMapper.NumberFieldType( "field", NumberFieldMapper.NumberType.LONG ); - List floatQueryObject = new ArrayList<>(); + List floatQueryObject = List.of(1.0f, 1.0f, 1.0f); Long longQueryObject = 0L; assertTrue( diff --git a/src/test/java/org/opensearch/knn/plugin/script/KNNScoringSpaceTests.java b/src/test/java/org/opensearch/knn/plugin/script/KNNScoringSpaceTests.java index c80949b43..6b40f375c 100644 --- a/src/test/java/org/opensearch/knn/plugin/script/KNNScoringSpaceTests.java +++ b/src/test/java/org/opensearch/knn/plugin/script/KNNScoringSpaceTests.java @@ -5,8 +5,10 @@ package org.opensearch.knn.plugin.script; +import java.util.Locale; import org.opensearch.knn.KNNTestCase; import org.opensearch.knn.index.KNNMethodContext; +import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.mapper.KNNVectorFieldMapper; import org.opensearch.index.mapper.BinaryFieldMapper; import org.opensearch.index.mapper.NumberFieldMapper; @@ -59,11 +61,26 @@ public void testCosineSimilarity() { assertEquals(3F, cosineSimilarity.scoringMethod.apply(arrayFloat2, arrayFloat), 0.1F); + // invalid zero vector + final List queryZeroVector = List.of(0.0f, 0.0f, 0.0f); + IllegalArgumentException exception1 = expectThrows( + IllegalArgumentException.class, + () -> new KNNScoringSpace.CosineSimilarity(queryZeroVector, fieldType) + ); + assertEquals( + String.format(Locale.ROOT, "zero vector is not supported when space type is [%s]", SpaceType.COSINESIMIL.getValue()), + exception1.getMessage() + ); + NumberFieldMapper.NumberFieldType invalidFieldType = new NumberFieldMapper.NumberFieldType( "field", NumberFieldMapper.NumberType.INTEGER ); - expectThrows(IllegalArgumentException.class, () -> new KNNScoringSpace.CosineSimilarity(arrayListQueryObject, invalidFieldType)); + IllegalArgumentException exception2 = expectThrows( + IllegalArgumentException.class, + () -> new KNNScoringSpace.CosineSimilarity(arrayListQueryObject, invalidFieldType) + ); + assertEquals("Incompatible field_type for cosine space. The field type must be knn_vector.", exception2.getMessage()); } public void testInnerProdSimilarity() { diff --git a/src/test/java/org/opensearch/knn/plugin/script/KNNScoringUtilTests.java b/src/test/java/org/opensearch/knn/plugin/script/KNNScoringUtilTests.java index 4a2bb7254..8c43a4acf 100644 --- a/src/test/java/org/opensearch/knn/plugin/script/KNNScoringUtilTests.java +++ b/src/test/java/org/opensearch/knn/plugin/script/KNNScoringUtilTests.java @@ -5,8 +5,10 @@ package org.opensearch.knn.plugin.script; +import java.util.Locale; import org.opensearch.knn.KNNTestCase; import org.opensearch.knn.index.KNNVectorScriptDocValues; +import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.VectorField; import org.apache.lucene.tests.analysis.MockAnalyzer; @@ -22,17 +24,16 @@ import java.io.IOException; import java.math.BigInteger; -import java.util.ArrayList; import java.util.List; public class KNNScoringUtilTests extends KNNTestCase { private List getTestQueryVector() { - List queryVector = new ArrayList<>(); - queryVector.add(1.0f); - queryVector.add(1.0f); - queryVector.add(1.0f); - return queryVector; + return List.of(1.0f, 1.0f, 1.0f); + } + + private List getTestZeroVector() { + return List.of(0.0f, 0.0f, 0.0f); } public void testL2SquaredScoringFunction() { @@ -211,6 +212,24 @@ public void testScriptDocValuesFailsCosineSimilarity() throws IOException { dataset.close(); } + public void testZeroVectorFailsCosineSimilarity() throws IOException { + List queryVector = getTestZeroVector(); + TestKNNScriptDocValues dataset = new TestKNNScriptDocValues(); + dataset.createKNNVectorDocument(new float[] { 4.0f, 4.0f, 4.0f }, "test-index-field-name"); + KNNVectorScriptDocValues scriptDocValues = dataset.getScriptDocValues("test-index-field-name"); + scriptDocValues.setNextDocId(0); + + IllegalArgumentException exception = expectThrows( + IllegalArgumentException.class, + () -> KNNScoringUtil.cosineSimilarity(queryVector, scriptDocValues) + ); + assertEquals( + String.format(Locale.ROOT, "zero vector is not supported when space type is [%s]", SpaceType.COSINESIMIL.getValue()), + exception.getMessage() + ); + dataset.close(); + } + public void testCosineSimilarityOptimizedScoringFunction() throws IOException { List queryVector = getTestQueryVector(); TestKNNScriptDocValues dataset = new TestKNNScriptDocValues(); @@ -227,7 +246,24 @@ public void testScriptDocValuesFailsCosineSimilarityOptimized() throws IOExcepti TestKNNScriptDocValues dataset = new TestKNNScriptDocValues(); dataset.createKNNVectorDocument(new float[] { 4.0f, 4.0f, 4.0f }, "test-index-field-name"); KNNVectorScriptDocValues scriptDocValues = dataset.getScriptDocValues("test-index-field-name"); - expectThrows(IllegalStateException.class, () -> KNNScoringUtil.cosineSimilarity(queryVector, scriptDocValues, 3.0f)); + dataset.close(); + } + + public void testZeroVectorFailsCosineSimilarityOptimized() throws IOException { + List queryVector = getTestZeroVector(); + TestKNNScriptDocValues dataset = new TestKNNScriptDocValues(); + dataset.createKNNVectorDocument(new float[] { 4.0f, 4.0f, 4.0f }, "test-index-field-name"); + KNNVectorScriptDocValues scriptDocValues = dataset.getScriptDocValues("test-index-field-name"); + scriptDocValues.setNextDocId(0); + + IllegalArgumentException exception = expectThrows( + IllegalArgumentException.class, + () -> KNNScoringUtil.cosineSimilarity(queryVector, scriptDocValues, 3.0f) + ); + assertEquals( + String.format(Locale.ROOT, "zero vector is not supported when space type is [%s]", SpaceType.COSINESIMIL.getValue()), + exception.getMessage() + ); dataset.close(); } diff --git a/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java b/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java index 7aa595faa..c68ff6b8c 100644 --- a/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java +++ b/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java @@ -18,6 +18,7 @@ import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.index.query.MatchAllQueryBuilder; +import org.opensearch.knn.common.KNNConstants; import org.opensearch.knn.index.query.KNNQueryBuilder; import org.opensearch.knn.index.KNNSettings; import org.opensearch.knn.index.SpaceType; @@ -332,6 +333,28 @@ protected String createKnnIndexMapping(String fieldName, Integer dimensions, Str .toString(); } + /** + * Utility to create a Knn Index Mapping with specific algorithm, engine and spaceType + */ + protected String createKnnIndexMapping(String fieldName, Integer dimensions, String algoName, String knnEngine, String spaceType) + throws IOException { + return XContentFactory.jsonBuilder() + .startObject() + .startObject("properties") + .startObject(fieldName) + .field(KNNConstants.TYPE, KNNConstants.TYPE_KNN_VECTOR) + .field(KNNConstants.DIMENSION, dimensions.toString()) + .startObject(KNNConstants.KNN_METHOD) + .field(KNNConstants.NAME, algoName) + .field(KNNConstants.METHOD_PARAMETER_SPACE_TYPE, spaceType) + .field(KNNConstants.KNN_ENGINE, knnEngine) + .endObject() + .endObject() + .endObject() + .endObject() + .toString(); + } + /** * Utility to create a Knn Index Mapping with multiple k-NN fields */ From f4380e46d159d872299f767507e22717ba45f6fd Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Thu, 14 Mar 2024 15:35:46 -0700 Subject: [PATCH 216/416] Add patch to fix segfault in nmslib during ingestion (#1541) (#1542) Signed-off-by: John Mazanec (cherry picked from commit 2b0f5a33d8114c689f6dd73ebc3efb9e0dc6273c) Co-authored-by: John Mazanec --- .github/workflows/CI.yml | 3 ++ .github/workflows/test_security.yml | 3 ++ CHANGELOG.md | 1 + jni/CMakeLists.txt | 13 ++++++++ ...vel-during-add-from-enterpoint-level.patch | 31 +++++++++++++++++++ 5 files changed, 51 insertions(+) create mode 100644 jni/patches/nmslib/0001-Initialize-maxlevel-during-add-from-enterpoint-level.patch diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 8c7011e1c..f333f25fd 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -47,6 +47,9 @@ jobs: rm ../../patches/faiss/0001-Custom-patch-to-support-multi-vector.patch git apply --ignore-space-change --ignore-whitespace --3way ../../patches/faiss/0002-Custom-patch-to-support-AVX2-Linux-CI.patch rm ../../patches/faiss/0002-Custom-patch-to-support-AVX2-Linux-CI.patch + cd ../nmslib + git apply --ignore-space-change --ignore-whitespace --3way ../../patches/nmslib/0001-Initialize-maxlevel-during-add-from-enterpoint-level.patch + rm ../../patches/nmslib/0001-Initialize-maxlevel-during-add-from-enterpoint-level.patch working-directory: ${{ github.workspace }} - name: Setup Java ${{ matrix.java }} diff --git a/.github/workflows/test_security.yml b/.github/workflows/test_security.yml index 643552512..b2f6eb353 100644 --- a/.github/workflows/test_security.yml +++ b/.github/workflows/test_security.yml @@ -47,6 +47,9 @@ jobs: rm ../../patches/faiss/0001-Custom-patch-to-support-multi-vector.patch git apply --ignore-space-change --ignore-whitespace --3way ../../patches/faiss/0002-Custom-patch-to-support-AVX2-Linux-CI.patch rm ../../patches/faiss/0002-Custom-patch-to-support-AVX2-Linux-CI.patch + cd ../nmslib + git apply --ignore-space-change --ignore-whitespace --3way ../../patches/nmslib/0001-Initialize-maxlevel-during-add-from-enterpoint-level.patch + rm ../../patches/nmslib/0001-Initialize-maxlevel-during-add-from-enterpoint-level.patch working-directory: ${{ github.workspace }} - name: Setup Java ${{ matrix.java }} diff --git a/CHANGELOG.md b/CHANGELOG.md index 08f248690..51fdaa1f9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Bug Fixes * Disable sdc table for HNSWPQ read-only indices [#1518](https://github.com/opensearch-project/k-NN/pull/1518) * Switch SpaceType.INNERPRODUCT's vector similarity function to MAXIMUM_INNER_PRODUCT [#1532](https://github.com/opensearch-project/k-NN/pull/1532) +* Add patch to fix arm segfault in nmslib during ingestion [#1541](https://github.com/opensearch-project/k-NN/pull/1541) ### Infrastructure * Manually install zlib for win CI [#1513](https://github.com/opensearch-project/k-NN/pull/1513) ### Documentation diff --git a/jni/CMakeLists.txt b/jni/CMakeLists.txt index 30a77d095..cec6bf54f 100644 --- a/jni/CMakeLists.txt +++ b/jni/CMakeLists.txt @@ -87,6 +87,19 @@ if (${CONFIG_NMSLIB} STREQUAL ON OR ${CONFIG_ALL} STREQUAL ON OR ${CONFIG_TEST} execute_process(COMMAND git submodule update --init -- external/nmslib WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}) endif () + # Check if patch exist, this is to skip git apply during CI build. See CI.yml with ubuntu. + find_path(PATCH_FILE NAMES 0001-Initialize-maxlevel-during-add-from-enterpoint-level.patch PATHS ${CMAKE_CURRENT_SOURCE_DIR}/patches/nmslib NO_DEFAULT_PATH) + + # If it exists, apply patches + if (EXISTS ${PATCH_FILE}) + message(STATUS "Applying custom patches.") + execute_process(COMMAND git apply --ignore-space-change --ignore-whitespace --3way ${CMAKE_CURRENT_SOURCE_DIR}/patches/nmslib/0001-Initialize-maxlevel-during-add-from-enterpoint-level.patch WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/external/nmslib ERROR_VARIABLE ERROR_MSG RESULT_VARIABLE RESULT_CODE) + + if(RESULT_CODE) + message(FATAL_ERROR "Failed to apply patch:\n${ERROR_MSG}") + endif() + endif() + add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/external/nmslib/similarity_search) add_library(${TARGET_LIB_NMSLIB} SHARED ${CMAKE_CURRENT_SOURCE_DIR}/src/org_opensearch_knn_jni_NmslibService.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/nmslib_wrapper.cpp) diff --git a/jni/patches/nmslib/0001-Initialize-maxlevel-during-add-from-enterpoint-level.patch b/jni/patches/nmslib/0001-Initialize-maxlevel-during-add-from-enterpoint-level.patch new file mode 100644 index 000000000..a9d9381f9 --- /dev/null +++ b/jni/patches/nmslib/0001-Initialize-maxlevel-during-add-from-enterpoint-level.patch @@ -0,0 +1,31 @@ +From aa1ca485c0ab8b79dae1fb5c1567149c5f61b533 Mon Sep 17 00:00:00 2001 +From: John Mazanec +Date: Thu, 14 Mar 2024 12:22:06 -0700 +Subject: [PATCH] Initialize maxlevel during add from enterpoint->level + +Signed-off-by: John Mazanec +--- + similarity_search/src/method/hnsw.cc | 6 +++++- + 1 file changed, 5 insertions(+), 1 deletion(-) + +diff --git a/similarity_search/src/method/hnsw.cc b/similarity_search/src/method/hnsw.cc +index 35b372c..e9a725e 100644 +--- a/similarity_search/src/method/hnsw.cc ++++ b/similarity_search/src/method/hnsw.cc +@@ -542,8 +542,12 @@ namespace similarity { + + NewElement->init(curlevel, maxM_, maxM0_); + +- int maxlevelcopy = maxlevel_; ++ // Get the enterpoint at this moment and then use it to set the ++ // max level that is used. Copying maxlevel from this->maxlevel_ ++ // can lead to race conditions during concurrent insertion. See: ++ // https://github.com/nmslib/nmslib/issues/544 + HnswNode *ep = enterpoint_; ++ int maxlevelcopy = ep->level; + if (curlevel < maxlevelcopy) { + const Object *currObj = ep->getData(); + +-- +2.39.3 (Apple Git-146) + From a9f783bc4e9c128a5b2b9ccc926b9c9d0b8d861b Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Thu, 14 Mar 2024 18:20:50 -0500 Subject: [PATCH 217/416] Fix AppSec Findings CWE-22 (Path Traversal) and CWE-476 (Null Pointer Dereference) (#1528) (#1539) Signed-off-by: Naveen Tatikonda (cherry picked from commit bfcf7dca75691cbc55919d5c55c4efc0f4577e04) Co-authored-by: Naveen Tatikonda --- .../opensearch/knn/bwc/AbstractRollingUpgradeTestCase.java | 3 +++ src/main/java/org/opensearch/knn/indices/ModelDao.java | 3 +++ src/test/java/org/opensearch/knn/index/FaissIT.java | 3 +++ src/test/java/org/opensearch/knn/index/NmslibIT.java | 7 +++++-- src/test/java/org/opensearch/knn/index/OpenSearchIT.java | 7 +++++-- src/test/java/org/opensearch/knn/jni/JNIServiceTests.java | 3 +++ .../java/org/opensearch/knn/KNNRestTestCase.java | 7 ++++++- 7 files changed, 28 insertions(+), 5 deletions(-) diff --git a/qa/rolling-upgrade/src/test/java/org/opensearch/knn/bwc/AbstractRollingUpgradeTestCase.java b/qa/rolling-upgrade/src/test/java/org/opensearch/knn/bwc/AbstractRollingUpgradeTestCase.java index 9f54fe3ef..235e3e4df 100644 --- a/qa/rolling-upgrade/src/test/java/org/opensearch/knn/bwc/AbstractRollingUpgradeTestCase.java +++ b/qa/rolling-upgrade/src/test/java/org/opensearch/knn/bwc/AbstractRollingUpgradeTestCase.java @@ -71,6 +71,9 @@ public static ClusterType instance(String value) { } protected final ClusterType getClusterType() { + if (System.getProperty(BWCSUITE_CLUSTER) == null) { + throw new IllegalArgumentException(String.format("[%s] value is null", BWCSUITE_CLUSTER)); + } return ClusterType.instance(System.getProperty(BWCSUITE_CLUSTER)); } diff --git a/src/main/java/org/opensearch/knn/indices/ModelDao.java b/src/main/java/org/opensearch/knn/indices/ModelDao.java index 1d88c9a00..e1c30cc86 100644 --- a/src/main/java/org/opensearch/knn/indices/ModelDao.java +++ b/src/main/java/org/opensearch/knn/indices/ModelDao.java @@ -466,6 +466,9 @@ public ModelMetadata getMetadata(String modelId) { } private String getMapping() throws IOException { + if (ModelDao.class.getClassLoader() == null) { + throw new IllegalStateException("ClassLoader of ModelDao Class is null"); + } URL url = ModelDao.class.getClassLoader().getResource(MODEL_INDEX_MAPPING_PATH); if (url == null) { throw new IllegalStateException("Unable to retrieve mapping for \"" + MODEL_INDEX_NAME + "\""); diff --git a/src/test/java/org/opensearch/knn/index/FaissIT.java b/src/test/java/org/opensearch/knn/index/FaissIT.java index 2f5990ce3..0e05c1e3c 100644 --- a/src/test/java/org/opensearch/knn/index/FaissIT.java +++ b/src/test/java/org/opensearch/knn/index/FaissIT.java @@ -66,6 +66,9 @@ public class FaissIT extends KNNRestTestCase { @BeforeClass public static void setUpClass() throws IOException { + if (FaissIT.class.getClassLoader() == null) { + throw new IllegalStateException("ClassLoader of FaissIT Class is null"); + } URL testIndexVectors = FaissIT.class.getClassLoader().getResource("data/test_vectors_1000x128.json"); URL testQueries = FaissIT.class.getClassLoader().getResource("data/test_queries_100x128.csv"); assert testIndexVectors != null; diff --git a/src/test/java/org/opensearch/knn/index/NmslibIT.java b/src/test/java/org/opensearch/knn/index/NmslibIT.java index 9aae61298..cb15a0eb9 100644 --- a/src/test/java/org/opensearch/knn/index/NmslibIT.java +++ b/src/test/java/org/opensearch/knn/index/NmslibIT.java @@ -44,8 +44,11 @@ public class NmslibIT extends KNNRestTestCase { @BeforeClass public static void setUpClass() throws IOException { - URL testIndexVectors = FaissIT.class.getClassLoader().getResource("data/test_vectors_1000x128.json"); - URL testQueries = FaissIT.class.getClassLoader().getResource("data/test_queries_100x128.csv"); + if (NmslibIT.class.getClassLoader() == null) { + throw new IllegalStateException("ClassLoader of NmslibIT Class is null"); + } + URL testIndexVectors = NmslibIT.class.getClassLoader().getResource("data/test_vectors_1000x128.json"); + URL testQueries = NmslibIT.class.getClassLoader().getResource("data/test_queries_100x128.csv"); assert testIndexVectors != null; assert testQueries != null; testData = new TestUtils.TestData(testIndexVectors.getPath(), testQueries.getPath()); diff --git a/src/test/java/org/opensearch/knn/index/OpenSearchIT.java b/src/test/java/org/opensearch/knn/index/OpenSearchIT.java index 98a1e1b71..24dcf7b1c 100644 --- a/src/test/java/org/opensearch/knn/index/OpenSearchIT.java +++ b/src/test/java/org/opensearch/knn/index/OpenSearchIT.java @@ -49,8 +49,11 @@ public class OpenSearchIT extends KNNRestTestCase { @BeforeClass public static void setUpClass() throws IOException { - URL testIndexVectors = FaissIT.class.getClassLoader().getResource("data/test_vectors_1000x128.json"); - URL testQueries = FaissIT.class.getClassLoader().getResource("data/test_queries_100x128.csv"); + if (OpenSearchIT.class.getClassLoader() == null) { + throw new IllegalStateException("ClassLoader of OpenSearchIT Class is null"); + } + URL testIndexVectors = OpenSearchIT.class.getClassLoader().getResource("data/test_vectors_1000x128.json"); + URL testQueries = OpenSearchIT.class.getClassLoader().getResource("data/test_queries_100x128.csv"); assert testIndexVectors != null; assert testQueries != null; testData = new TestUtils.TestData(testIndexVectors.getPath(), testQueries.getPath()); diff --git a/src/test/java/org/opensearch/knn/jni/JNIServiceTests.java b/src/test/java/org/opensearch/knn/jni/JNIServiceTests.java index 8e3382ece..c20facf3a 100644 --- a/src/test/java/org/opensearch/knn/jni/JNIServiceTests.java +++ b/src/test/java/org/opensearch/knn/jni/JNIServiceTests.java @@ -65,6 +65,9 @@ public class JNIServiceTests extends KNNTestCase { @BeforeClass public static void setUpClass() throws IOException { + if (JNIServiceTests.class.getClassLoader() == null) { + throw new IllegalStateException("ClassLoader of JNIServiceTests Class is null"); + } URL testIndexVectors = JNIServiceTests.class.getClassLoader().getResource("data/test_vectors_1000x128.json"); URL testIndexVectorsNested = JNIServiceTests.class.getClassLoader().getResource("data/test_vectors_nested_1000x128.json"); URL testQueries = JNIServiceTests.class.getClassLoader().getResource("data/test_queries_100x128.csv"); diff --git a/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java b/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java index c68ff6b8c..2bfef125b 100644 --- a/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java +++ b/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java @@ -122,6 +122,11 @@ public static void dumpCoverage() throws IOException, MalformedObjectNameExcepti } String serverUrl = System.getProperty("jmx.serviceUrl"); + if (serverUrl == null) { + log.error("Failed to dump coverage because JMX Service URL is null"); + throw new IllegalArgumentException("JMX Service URL is null"); + } + try (JMXConnector connector = JMXConnectorFactory.connect(new JMXServiceURL(serverUrl))) { IProxy proxy = MBeanServerInvocationHandler.newProxyInstance( connector.getMBeanServerConnection(), @@ -130,7 +135,7 @@ public static void dumpCoverage() throws IOException, MalformedObjectNameExcepti false ); - Path path = Path.of(jacocoBuildPath, "integTest.exec"); + Path path = Path.of(Path.of(jacocoBuildPath, "integTest.exec").toFile().getCanonicalPath()); Files.write(path, proxy.getExecutionData(false)); } catch (Exception ex) { log.error("Failed to dump coverage: ", ex); From a4479e4ac332d4c9c1580add4c76e2e7988680f7 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Fri, 15 Mar 2024 11:25:51 -0500 Subject: [PATCH 218/416] Update k-NN build artifact script to enable SIMD on ARM for Faiss (#1543) (#1544) (cherry picked from commit 2959d0683aadf485158cbfa69f9405ab4d2b0841) Signed-off-by: Naveen Tatikonda Co-authored-by: Naveen Tatikonda --- CHANGELOG.md | 1 + scripts/build.sh | 4 +--- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 51fdaa1f9..c345fb1ce 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), * Add patch to fix arm segfault in nmslib during ingestion [#1541](https://github.com/opensearch-project/k-NN/pull/1541) ### Infrastructure * Manually install zlib for win CI [#1513](https://github.com/opensearch-project/k-NN/pull/1513) +* Update k-NN build artifact script to enable SIMD on ARM for Faiss [#1543](https://github.com/opensearch-project/k-NN/pull/1543) ### Documentation ### Maintenance * Bump faiss lib commit to 32f0e8cf92cd2275b60364517bb1cce67aa29a55 [#1443](https://github.com/opensearch-project/k-NN/pull/1443) diff --git a/scripts/build.sh b/scripts/build.sh index b2cdef687..700ab5731 100644 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -93,11 +93,9 @@ if [ "$ARCHITECTURE" = "x64" ]; then sed -i -e 's/-march=native/-march=x86-64/g' external/nmslib/similarity_search/CMakeLists.txt fi -# For arm, march=native is broken in centos 7. Manually override to lowest version of armv8. Also, disable simd in faiss -# file. This is broken on centos 7 as well. +# For arm, march=native is broken in centos 7. Manually override to lowest version of armv8. if [ "$ARCHITECTURE" = "arm64" ]; then sed -i -e 's/-march=native/-march=armv8-a/g' external/nmslib/similarity_search/CMakeLists.txt - sed -i -e 's/__aarch64__/__undefine_aarch64__/g' external/faiss/faiss/utils/distances_simd.cpp fi if [ "$JAVA_HOME" = "" ]; then From 06d395f5c0b97ebb3863da565b447bb90a0d4838 Mon Sep 17 00:00:00 2001 From: Ryan Bogan Date: Fri, 15 Mar 2024 14:17:28 -0700 Subject: [PATCH 219/416] [Backport 2.x] Persist model definition in model metadata (#1548) * Persist model definition in model metadata (#1527) * Add MethodComponentContext to ModelMetadata Signed-off-by: Ryan Bogan * Add changelog Signed-off-by: Ryan Bogan * Address PR Comments Signed-off-by: Ryan Bogan * Address PR Comments Signed-off-by: Ryan Bogan * Change fromString Signed-off-by: Ryan Bogan * Address PR Comments Signed-off-by: Ryan Bogan * Address PR Comments Signed-off-by: Ryan Bogan * Address PR Comments Signed-off-by: Ryan Bogan * Fix spotless Signed-off-by: Ryan Bogan --------- Signed-off-by: Ryan Bogan (cherry picked from commit 4734d88b8b12ff3edd676e3a1f337fc814c8932f) * Fix compile errors Signed-off-by: Ryan Bogan * Fix compile error Signed-off-by: Ryan Bogan * Fix imports Signed-off-by: Ryan Bogan --------- Signed-off-by: Ryan Bogan --- CHANGELOG.md | 1 + .../opensearch/knn/common/KNNConstants.java | 1 + .../org/opensearch/knn/index/IndexUtil.java | 3 + .../knn/index/MethodComponentContext.java | 202 +++++++++++ .../org/opensearch/knn/indices/ModelDao.java | 11 + .../opensearch/knn/indices/ModelMetadata.java | 140 ++++++-- .../opensearch/knn/training/TrainingJob.java | 3 +- src/main/resources/mappings/model-index.json | 6 + .../index/KNNCreateIndexFromModelTests.java | 3 +- .../index/MethodComponentContextTests.java | 69 ++++ .../KNN80DocValuesConsumerTests.java | 12 +- .../knn/index/codec/KNNCodecTestCase.java | 3 +- .../mapper/KNNVectorFieldMapperTests.java | 6 +- .../knn/indices/ModelCacheTests.java | 38 ++- .../opensearch/knn/indices/ModelDaoTests.java | 44 ++- .../knn/indices/ModelMetadataTests.java | 316 ++++++++++++++++-- .../opensearch/knn/indices/ModelTests.java | 43 ++- .../transport/GetModelResponseTests.java | 17 +- ...oveModelFromCacheTransportActionTests.java | 13 +- .../transport/TrainingModelRequestTests.java | 4 +- .../UpdateModelMetadataRequestTests.java | 10 +- ...dateModelMetadataTransportActionTests.java | 4 +- .../knn/training/TrainingJobTests.java | 6 +- 23 files changed, 834 insertions(+), 121 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c345fb1ce..a669bfb2f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), * Optize Faiss Query With Filters: Reduce iteration and memory for id filter [#1402](https://github.com/opensearch-project/k-NN/pull/1402) * Detect AVX2 Dynamically on the System [#1502](https://github.com/opensearch-project/k-NN/pull/1502) * Validate zero vector when using cosine metric [#1501](https://github.com/opensearch-project/k-NN/pull/1501) +* Persist model definition in model metadata [#1527] (https://github.com/opensearch-project/k-NN/pull/1527) ### Bug Fixes * Disable sdc table for HNSWPQ read-only indices [#1518](https://github.com/opensearch-project/k-NN/pull/1518) * Switch SpaceType.INNERPRODUCT's vector similarity function to MAXIMUM_INNER_PRODUCT [#1532](https://github.com/opensearch-project/k-NN/pull/1532) diff --git a/src/main/java/org/opensearch/knn/common/KNNConstants.java b/src/main/java/org/opensearch/knn/common/KNNConstants.java index fdec9d37c..314d7d4d1 100644 --- a/src/main/java/org/opensearch/knn/common/KNNConstants.java +++ b/src/main/java/org/opensearch/knn/common/KNNConstants.java @@ -48,6 +48,7 @@ public class KNNConstants { public static final String MODEL_DESCRIPTION = "description"; public static final String MODEL_ERROR = "error"; public static final String MODEL_NODE_ASSIGNMENT = "training_node_assignment"; + public static final String MODEL_METHOD_COMPONENT_CONTEXT = "model_definition"; public static final String PARAM_SIZE = "size"; public static final Integer SEARCH_MODEL_MIN_SIZE = 1; public static final Integer SEARCH_MODEL_MAX_SIZE = 1000; diff --git a/src/main/java/org/opensearch/knn/index/IndexUtil.java b/src/main/java/org/opensearch/knn/index/IndexUtil.java index a18c228bc..d7833be46 100644 --- a/src/main/java/org/opensearch/knn/index/IndexUtil.java +++ b/src/main/java/org/opensearch/knn/index/IndexUtil.java @@ -37,15 +37,18 @@ public class IndexUtil { public static final String MODEL_NODE_ASSIGNMENT_KEY = KNNConstants.MODEL_NODE_ASSIGNMENT; + public static final String MODEL_METHOD_COMPONENT_CONTEXT_KEY = KNNConstants.MODEL_METHOD_COMPONENT_CONTEXT; private static final Version MINIMAL_SUPPORTED_VERSION_FOR_LUCENE_HNSW_FILTER = Version.V_2_4_0; private static final Version MINIMAL_SUPPORTED_VERSION_FOR_IGNORE_UNMAPPED = Version.V_2_11_0; private static final Version MINIMAL_SUPPORTED_VERSION_FOR_MODEL_NODE_ASSIGNMENT = Version.V_2_12_0; + private static final Version MINIMAL_SUPPORTED_VERSION_FOR_MODEL_METHOD_COMPONENT_CONTEXT = Version.V_2_13_0; public static final Map minimalRequiredVersionMap = new HashMap() { { put("filter", MINIMAL_SUPPORTED_VERSION_FOR_LUCENE_HNSW_FILTER); put("ignore_unmapped", MINIMAL_SUPPORTED_VERSION_FOR_IGNORE_UNMAPPED); put(MODEL_NODE_ASSIGNMENT_KEY, MINIMAL_SUPPORTED_VERSION_FOR_MODEL_NODE_ASSIGNMENT); + put(MODEL_METHOD_COMPONENT_CONTEXT_KEY, MINIMAL_SUPPORTED_VERSION_FOR_MODEL_METHOD_COMPONENT_CONTEXT); } }; diff --git a/src/main/java/org/opensearch/knn/index/MethodComponentContext.java b/src/main/java/org/opensearch/knn/index/MethodComponentContext.java index 66952f448..b9fd56b72 100644 --- a/src/main/java/org/opensearch/knn/index/MethodComponentContext.java +++ b/src/main/java/org/opensearch/knn/index/MethodComponentContext.java @@ -11,24 +11,29 @@ package org.opensearch.knn.index; +import lombok.AllArgsConstructor; import lombok.Getter; import lombok.RequiredArgsConstructor; import lombok.Setter; +import org.apache.commons.lang.math.NumberUtils; import org.opensearch.Version; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; import org.opensearch.index.mapper.MapperParsingException; import java.io.IOException; import java.util.Collections; import java.util.HashMap; import java.util.Map; +import java.util.Objects; import java.util.stream.Collectors; import org.apache.commons.lang.builder.EqualsBuilder; import org.apache.commons.lang.builder.HashCodeBuilder; +import org.opensearch.knn.indices.ModelMetadata; import static org.opensearch.knn.common.KNNConstants.NAME; import static org.opensearch.knn.common.KNNConstants.PARAMETERS; @@ -41,6 +46,13 @@ @RequiredArgsConstructor public class MethodComponentContext implements ToXContentFragment, Writeable { + // EMPTY method component context can only occur if a model originated on a cluster before 2.13.0 and the cluster is then upgraded to + // 2.13.0 + public static final MethodComponentContext EMPTY = new MethodComponentContext("", Collections.emptyMap()); + + private static final String DELIMITER = ";"; + private static final String DELIMITER_PLACEHOLDER = "$%$"; + @Getter private final String name; private final Map parameters; @@ -161,6 +173,15 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws return builder; } + public static MethodComponentContext fromXContent(XContentParser xContentParser) throws IOException { + // If it is a fresh parser, move to the first token + if (xContentParser.currentToken() == null) { + xContentParser.nextToken(); + } + Map parsedMap = xContentParser.map(); + return MethodComponentContext.parse(parsedMap); + } + @Override public boolean equals(Object obj) { if (this == obj) return true; @@ -193,6 +214,187 @@ public Map getParameters() { return parameters; } + /** + * + * Provides a String representation of MethodComponentContext + * Sample return: + * {name=ivf;parameters=[nlist=4;type=fp16;encoder={name=sq;parameters=[nprobes=2;clip=false;]};]} + * + * @return string representation + */ + public String toClusterStateString() { + StringBuilder stringBuilder = new StringBuilder(); + stringBuilder.append("{name=").append(name).append(DELIMITER); + stringBuilder.append("parameters=["); + if (Objects.nonNull(parameters)) { + for (Map.Entry entry : parameters.entrySet()) { + stringBuilder.append(entry.getKey()).append("="); + Object objectValue = entry.getValue(); + String value; + if (objectValue instanceof MethodComponentContext) { + value = ((MethodComponentContext) objectValue).toClusterStateString(); + } else { + value = entry.getValue().toString(); + } + // Model Metadata uses a delimiter to split the input string in its fromString method + // https://github.com/opensearch-project/k-NN/blob/2.12/src/main/java/org/opensearch/knn/indices/ModelMetadata.java#L265 + // If any of the values in the method component context contain this delimiter, + // then the method will not work correctly. Therefore, we replace the delimiter with an uncommon + // sequence that is very unlikely to appear in the value itself. + // https://github.com/opensearch-project/k-NN/issues/1337 + value = value.replace(ModelMetadata.DELIMITER, DELIMITER_PLACEHOLDER); + stringBuilder.append(value).append(DELIMITER); + } + } + stringBuilder.append("]}"); + return stringBuilder.toString(); + } + + /** + * This method converts a string created by the toClusterStateString() method of MethodComponentContext + * to a MethodComponentContext object. + * + * @param in a string representation of MethodComponentContext + * @return a MethodComponentContext object + */ + public static MethodComponentContext fromClusterStateString(String in) { + String stringToParse = unwrapString(in, '{', '}'); + + // Parse name from string + String[] nameAndParameters = stringToParse.split(DELIMITER, 2); + checkExpectedArrayLength(nameAndParameters, 2); + String name = parseName(nameAndParameters[0]); + String parametersString = nameAndParameters[1]; + Map parameters = parseParameters(parametersString); + return new MethodComponentContext(name, parameters); + } + + private static String parseName(String candidateNameString) { + // Expecting candidateNameString to look like "name=ivf" + checkStringNotEmpty(candidateNameString); + String[] nameKeyAndValue = candidateNameString.split("="); + checkStringMatches(nameKeyAndValue[0], "name"); + if (nameKeyAndValue.length == 1) { + return ""; + } + checkExpectedArrayLength(nameKeyAndValue, 2); + return nameKeyAndValue[1]; + } + + private static Map parseParameters(String candidateParameterString) { + checkStringNotEmpty(candidateParameterString); + String[] parametersKeyAndValue = candidateParameterString.split("=", 2); + checkStringMatches(parametersKeyAndValue[0], "parameters"); + if (parametersKeyAndValue.length == 1) { + return Collections.emptyMap(); + } + checkExpectedArrayLength(parametersKeyAndValue, 2); + return parseParametersValue(parametersKeyAndValue[1]); + } + + private static Map parseParametersValue(String candidateParameterValueString) { + // Expected input is [nlist=4;type=fp16;encoder={name=sq;parameters=[nprobes=2;clip=false;]};] + checkStringNotEmpty(candidateParameterValueString); + candidateParameterValueString = unwrapString(candidateParameterValueString, '[', ']'); + Map parameters = new HashMap<>(); + while (!candidateParameterValueString.isEmpty()) { + String[] keyAndValueToParse = candidateParameterValueString.split("=", 2); + if (keyAndValueToParse.length == 1 && keyAndValueToParse[0].charAt(0) == ';') { + break; + } + String key = keyAndValueToParse[0]; + ValueAndRestToParse parsed = parseParameterValueAndRestToParse(keyAndValueToParse[1]); + parameters.put(key, parsed.getValue()); + candidateParameterValueString = parsed.getRestToParse(); + } + + return parameters; + } + + private static ValueAndRestToParse parseParameterValueAndRestToParse(String candidateParameterValueAndRestToParse) { + if (candidateParameterValueAndRestToParse.charAt(0) == '{') { + int endOfNestedMap = findClosingPosition(candidateParameterValueAndRestToParse, '{', '}'); + String nestedMethodContext = candidateParameterValueAndRestToParse.substring(0, endOfNestedMap + 1); + Object nestedParse = fromClusterStateString(nestedMethodContext); + String restToParse = candidateParameterValueAndRestToParse.substring(endOfNestedMap + 1); + return new ValueAndRestToParse(nestedParse, restToParse); + } + + String[] stringValueAndRestToParse = candidateParameterValueAndRestToParse.split(DELIMITER, 2); + String stringValue = stringValueAndRestToParse[0]; + Object value; + if (NumberUtils.isNumber(stringValue)) { + value = Integer.parseInt(stringValue); + } else if (stringValue.equals("true") || stringValue.equals("false")) { + value = Boolean.parseBoolean(stringValue); + } else { + stringValue = stringValue.replace(DELIMITER_PLACEHOLDER, ModelMetadata.DELIMITER); + value = stringValue; + } + + return new ValueAndRestToParse(value, stringValueAndRestToParse[1]); + } + + private static String unwrapString(String in, char expectedStart, char expectedEnd) { + if (in.length() < 2) { + throw new IllegalArgumentException("Invalid string."); + } + + if (in.charAt(0) != expectedStart || in.charAt(in.length() - 1) != expectedEnd) { + throw new IllegalArgumentException("Invalid string." + in); + } + return in.substring(1, in.length() - 1); + } + + private static int findClosingPosition(String in, char expectedStart, char expectedEnd) { + int nestedLevel = 0; + for (int i = 0; i < in.length(); i++) { + if (in.charAt(i) == expectedStart) { + nestedLevel++; + continue; + } + + if (in.charAt(i) == expectedEnd) { + nestedLevel--; + } + + if (nestedLevel == 0) { + return i; + } + } + + throw new IllegalArgumentException("Invalid string. No end to the nesting"); + } + + private static void checkStringNotEmpty(String string) { + if (string.isEmpty()) { + throw new IllegalArgumentException("Unable to parse MethodComponentContext"); + } + } + + private static void checkStringMatches(String string, String expected) { + if (!Objects.equals(string, expected)) { + throw new IllegalArgumentException("Unexpected key in MethodComponentContext. Expected 'name' or 'parameters'"); + } + } + + private static void checkExpectedArrayLength(String[] array, int expectedLength) { + if (null == array) { + throw new IllegalArgumentException("Error parsing MethodComponentContext. Array is null."); + } + + if (array.length != expectedLength) { + throw new IllegalArgumentException("Error parsing MethodComponentContext. Array is not expected length."); + } + } + + @AllArgsConstructor + @Getter + private static class ValueAndRestToParse { + private final Object value; + private final String restToParse; + } + @Override public void writeTo(StreamOutput out) throws IOException { out.writeString(this.name); diff --git a/src/main/java/org/opensearch/knn/indices/ModelDao.java b/src/main/java/org/opensearch/knn/indices/ModelDao.java index e1c30cc86..0c0f08545 100644 --- a/src/main/java/org/opensearch/knn/indices/ModelDao.java +++ b/src/main/java/org/opensearch/knn/indices/ModelDao.java @@ -43,10 +43,14 @@ import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.concurrent.ThreadContext; +import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.action.ActionListener; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.index.IndexNotFoundException; import org.opensearch.knn.common.KNNConstants; import org.opensearch.knn.common.exception.DeleteModelWhenInTrainStateException; +import org.opensearch.knn.index.MethodComponentContext; import org.opensearch.knn.plugin.transport.DeleteModelResponse; import org.opensearch.knn.plugin.transport.GetModelResponse; import org.opensearch.knn.plugin.transport.RemoveModelFromCacheAction; @@ -288,6 +292,13 @@ private void putInternal(Model model, ActionListener listener, Do put(KNNConstants.MODEL_DESCRIPTION, modelMetadata.getDescription()); put(KNNConstants.MODEL_ERROR, modelMetadata.getError()); put(KNNConstants.MODEL_NODE_ASSIGNMENT, modelMetadata.getNodeAssignment()); + + MethodComponentContext methodComponentContext = modelMetadata.getMethodComponentContext(); + if (!methodComponentContext.getName().isEmpty()) { + XContentBuilder builder = XContentFactory.jsonBuilder().startObject(); + builder = methodComponentContext.toXContent(builder, ToXContent.EMPTY_PARAMS).endObject(); + put(KNNConstants.MODEL_METHOD_COMPONENT_CONTEXT, builder.toString()); + } } }; diff --git a/src/main/java/org/opensearch/knn/indices/ModelMetadata.java b/src/main/java/org/opensearch/knn/indices/ModelMetadata.java index 04836f184..fa88c8416 100644 --- a/src/main/java/org/opensearch/knn/indices/ModelMetadata.java +++ b/src/main/java/org/opensearch/knn/indices/ModelMetadata.java @@ -14,13 +14,17 @@ import lombok.extern.log4j.Log4j2; import org.apache.commons.lang.builder.EqualsBuilder; import org.apache.commons.lang.builder.HashCodeBuilder; +import org.opensearch.common.xcontent.json.JsonXContent; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.io.stream.Writeable; +import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; import org.opensearch.knn.common.KNNConstants; import org.opensearch.knn.index.IndexUtil; +import org.opensearch.knn.index.MethodComponentContext; import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.util.KNNEngine; @@ -29,19 +33,12 @@ import java.util.Objects; import java.util.concurrent.atomic.AtomicReference; -import static org.opensearch.knn.common.KNNConstants.DIMENSION; -import static org.opensearch.knn.common.KNNConstants.KNN_ENGINE; -import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_SPACE_TYPE; -import static org.opensearch.knn.common.KNNConstants.MODEL_DESCRIPTION; -import static org.opensearch.knn.common.KNNConstants.MODEL_ERROR; -import static org.opensearch.knn.common.KNNConstants.MODEL_STATE; -import static org.opensearch.knn.common.KNNConstants.MODEL_TIMESTAMP; -import static org.opensearch.knn.common.KNNConstants.MODEL_NODE_ASSIGNMENT; +import static org.opensearch.core.xcontent.DeprecationHandler.IGNORE_DEPRECATIONS; @Log4j2 public class ModelMetadata implements Writeable, ToXContentObject { - private static final String DELIMITER = ","; + public static final String DELIMITER = ","; final private KNNEngine knnEngine; final private SpaceType spaceType; @@ -51,6 +48,7 @@ public class ModelMetadata implements Writeable, ToXContentObject { final private String timestamp; final private String description; final private String trainingNodeAssignment; + private MethodComponentContext methodComponentContext; private String error; /** @@ -76,6 +74,12 @@ public ModelMetadata(StreamInput in) throws IOException { } else { this.trainingNodeAssignment = ""; } + + if (IndexUtil.isVersionOnOrAfterMinRequiredVersion(in.getVersion(), IndexUtil.MODEL_METHOD_COMPONENT_CONTEXT_KEY)) { + this.methodComponentContext = new MethodComponentContext(in); + } else { + this.methodComponentContext = MethodComponentContext.EMPTY; + } } /** @@ -88,6 +92,8 @@ public ModelMetadata(StreamInput in) throws IOException { * @param timestamp timevalue when model was created * @param description information about the model * @param error error message associated with model + * @param trainingNodeAssignment node assignment for the model + * @param methodComponentContext method component context associated with model */ public ModelMetadata( KNNEngine knnEngine, @@ -97,7 +103,8 @@ public ModelMetadata( String timestamp, String description, String error, - String trainingNodeAssignment + String trainingNodeAssignment, + MethodComponentContext methodComponentContext ) { this.knnEngine = Objects.requireNonNull(knnEngine, "knnEngine must not be null"); this.spaceType = Objects.requireNonNull(spaceType, "spaceType must not be null"); @@ -118,6 +125,7 @@ public ModelMetadata( this.description = Objects.requireNonNull(description, "description must not be null"); this.error = Objects.requireNonNull(error, "error must not be null"); this.trainingNodeAssignment = Objects.requireNonNull(trainingNodeAssignment, "node assignment must not be null"); + this.methodComponentContext = Objects.requireNonNull(methodComponentContext, "method context must not be null"); } /** @@ -192,6 +200,15 @@ public String getNodeAssignment() { return trainingNodeAssignment; } + /** + * getter for model's method context + * + * @return knnMethodContext + */ + public MethodComponentContext getMethodComponentContext() { + return methodComponentContext; + } + /** * setter for model's state * @@ -221,7 +238,8 @@ public String toString() { timestamp, description, error, - trainingNodeAssignment + trainingNodeAssignment, + methodComponentContext.toClusterStateString() ); } @@ -252,6 +270,7 @@ public int hashCode() { .append(getTimestamp()) .append(getDescription()) .append(getError()) + .append(getMethodComponentContext()) .toHashCode(); } @@ -268,7 +287,9 @@ public static ModelMetadata fromString(String modelMetadataString) { // Because models can be created on older versions and the cluster can be upgraded after, // we need to accept model metadata arrays both with and without the training node assignment. if (modelMetadataArray.length == 7) { - log.debug("Model metadata array does not contain training node assignment. Assuming empty string."); + log.debug( + "Model metadata array does not contain training node assignment or method component context. Assuming empty string node assignment and empty method component context." + ); KNNEngine knnEngine = KNNEngine.getEngine(modelMetadataArray[0]); SpaceType spaceType = SpaceType.getSpace(modelMetadataArray[1]); int dimension = Integer.parseInt(modelMetadataArray[2]); @@ -276,9 +297,19 @@ public static ModelMetadata fromString(String modelMetadataString) { String timestamp = modelMetadataArray[4]; String description = modelMetadataArray[5]; String error = modelMetadataArray[6]; - return new ModelMetadata(knnEngine, spaceType, dimension, modelState, timestamp, description, error, ""); + return new ModelMetadata( + knnEngine, + spaceType, + dimension, + modelState, + timestamp, + description, + error, + "", + MethodComponentContext.EMPTY + ); } else if (modelMetadataArray.length == 8) { - log.debug("Model metadata contains training node assignment"); + log.debug("Model metadata contains training node assignment. Assuming empty method component context."); KNNEngine knnEngine = KNNEngine.getEngine(modelMetadataArray[0]); SpaceType spaceType = SpaceType.getSpace(modelMetadataArray[1]); int dimension = Integer.parseInt(modelMetadataArray[2]); @@ -287,11 +318,43 @@ public static ModelMetadata fromString(String modelMetadataString) { String description = modelMetadataArray[5]; String error = modelMetadataArray[6]; String trainingNodeAssignment = modelMetadataArray[7]; - return new ModelMetadata(knnEngine, spaceType, dimension, modelState, timestamp, description, error, trainingNodeAssignment); + return new ModelMetadata( + knnEngine, + spaceType, + dimension, + modelState, + timestamp, + description, + error, + trainingNodeAssignment, + MethodComponentContext.EMPTY + ); + } else if (modelMetadataArray.length == 9) { + log.debug("Model metadata contains training node assignment and method context"); + KNNEngine knnEngine = KNNEngine.getEngine(modelMetadataArray[0]); + SpaceType spaceType = SpaceType.getSpace(modelMetadataArray[1]); + int dimension = Integer.parseInt(modelMetadataArray[2]); + ModelState modelState = ModelState.getModelState(modelMetadataArray[3]); + String timestamp = modelMetadataArray[4]; + String description = modelMetadataArray[5]; + String error = modelMetadataArray[6]; + String trainingNodeAssignment = modelMetadataArray[7]; + MethodComponentContext methodComponentContext = MethodComponentContext.fromClusterStateString(modelMetadataArray[8]); + return new ModelMetadata( + knnEngine, + spaceType, + dimension, + modelState, + timestamp, + description, + error, + trainingNodeAssignment, + methodComponentContext + ); } else { throw new IllegalArgumentException( "Illegal format for model metadata. Must be of the form " - + "\",,,,,,\" or \",,,,,,,\"." + + "\",,,,,,\" or \",,,,,,,\" or \",,,,,,,,\"." ); } } @@ -321,11 +384,27 @@ public static ModelMetadata getMetadataFromSourceMap(final Map m Object description = modelSourceMap.get(KNNConstants.MODEL_DESCRIPTION); Object error = modelSourceMap.get(KNNConstants.MODEL_ERROR); Object trainingNodeAssignment = modelSourceMap.get(KNNConstants.MODEL_NODE_ASSIGNMENT); + Object methodComponentContext = modelSourceMap.get(KNNConstants.MODEL_METHOD_COMPONENT_CONTEXT); if (trainingNodeAssignment == null) { trainingNodeAssignment = ""; } + if (Objects.nonNull(methodComponentContext)) { + try { + XContentParser xContentParser = JsonXContent.jsonXContent.createParser( + NamedXContentRegistry.EMPTY, + IGNORE_DEPRECATIONS, + objectToString(methodComponentContext) + ); + methodComponentContext = MethodComponentContext.fromXContent(xContentParser); + } catch (IOException e) { + throw new IllegalArgumentException("Error parsing method component context"); + } + } else { + methodComponentContext = MethodComponentContext.EMPTY; + } + ModelMetadata modelMetadata = new ModelMetadata( KNNEngine.getEngine(objectToString(engine)), SpaceType.getSpace(objectToString(space)), @@ -334,7 +413,8 @@ public static ModelMetadata getMetadataFromSourceMap(final Map m objectToString(timestamp), objectToString(description), objectToString(error), - objectToString(trainingNodeAssignment) + objectToString(trainingNodeAssignment), + (MethodComponentContext) methodComponentContext ); return modelMetadata; } @@ -351,20 +431,28 @@ public void writeTo(StreamOutput out) throws IOException { if (IndexUtil.isVersionOnOrAfterMinRequiredVersion(out.getVersion(), IndexUtil.MODEL_NODE_ASSIGNMENT_KEY)) { out.writeString(getNodeAssignment()); } + if (IndexUtil.isVersionOnOrAfterMinRequiredVersion(out.getVersion(), IndexUtil.MODEL_METHOD_COMPONENT_CONTEXT_KEY)) { + getMethodComponentContext().writeTo(out); + } } @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.field(MODEL_STATE, getState().getName()); - builder.field(MODEL_TIMESTAMP, getTimestamp()); - builder.field(MODEL_DESCRIPTION, getDescription()); - builder.field(MODEL_ERROR, getError()); - - builder.field(METHOD_PARAMETER_SPACE_TYPE, getSpaceType().getValue()); - builder.field(DIMENSION, getDimension()); - builder.field(KNN_ENGINE, getKnnEngine().getName()); + builder.field(KNNConstants.MODEL_STATE, getState().getName()); + builder.field(KNNConstants.MODEL_TIMESTAMP, getTimestamp()); + builder.field(KNNConstants.MODEL_DESCRIPTION, getDescription()); + builder.field(KNNConstants.MODEL_ERROR, getError()); + + builder.field(KNNConstants.METHOD_PARAMETER_SPACE_TYPE, getSpaceType().getValue()); + builder.field(KNNConstants.DIMENSION, getDimension()); + builder.field(KNNConstants.KNN_ENGINE, getKnnEngine().getName()); if (IndexUtil.isClusterOnOrAfterMinRequiredVersion(IndexUtil.MODEL_NODE_ASSIGNMENT_KEY)) { - builder.field(MODEL_NODE_ASSIGNMENT, getNodeAssignment()); + builder.field(KNNConstants.MODEL_NODE_ASSIGNMENT, getNodeAssignment()); + } + if (IndexUtil.isClusterOnOrAfterMinRequiredVersion(IndexUtil.MODEL_METHOD_COMPONENT_CONTEXT_KEY)) { + builder.field(KNNConstants.MODEL_METHOD_COMPONENT_CONTEXT).startObject(); + getMethodComponentContext().toXContent(builder, params); + builder.endObject(); } return builder; } diff --git a/src/main/java/org/opensearch/knn/training/TrainingJob.java b/src/main/java/org/opensearch/knn/training/TrainingJob.java index 7b5404f6c..2c86082bb 100644 --- a/src/main/java/org/opensearch/knn/training/TrainingJob.java +++ b/src/main/java/org/opensearch/knn/training/TrainingJob.java @@ -83,7 +83,8 @@ public TrainingJob( ZonedDateTime.now(ZoneOffset.UTC).toString(), description, "", - nodeAssignment + nodeAssignment, + knnMethodContext.getMethodComponentContext() ), null, this.modelId diff --git a/src/main/resources/mappings/model-index.json b/src/main/resources/mappings/model-index.json index a8c7d6528..cd2a50839 100644 --- a/src/main/resources/mappings/model-index.json +++ b/src/main/resources/mappings/model-index.json @@ -26,6 +26,12 @@ }, "model_blob": { "type": "binary" + }, + "node_assignment": { + "type": "keyword" + }, + "method_component_context": { + "type": "keyword" } } } diff --git a/src/test/java/org/opensearch/knn/index/KNNCreateIndexFromModelTests.java b/src/test/java/org/opensearch/knn/index/KNNCreateIndexFromModelTests.java index 8fdc55766..710978928 100644 --- a/src/test/java/org/opensearch/knn/index/KNNCreateIndexFromModelTests.java +++ b/src/test/java/org/opensearch/knn/index/KNNCreateIndexFromModelTests.java @@ -62,7 +62,8 @@ public void testCreateIndexFromModel() throws IOException, InterruptedException ZonedDateTime.now(ZoneOffset.UTC).toString(), "", "", - "test-node" + "test-node", + MethodComponentContext.EMPTY ); Model model = new Model(modelMetadata, modelBlob, modelId); diff --git a/src/test/java/org/opensearch/knn/index/MethodComponentContextTests.java b/src/test/java/org/opensearch/knn/index/MethodComponentContextTests.java index cbbb872cf..5ce1a76ac 100644 --- a/src/test/java/org/opensearch/knn/index/MethodComponentContextTests.java +++ b/src/test/java/org/opensearch/knn/index/MethodComponentContextTests.java @@ -281,4 +281,73 @@ public void testHashCode() { assertNotEquals(methodContext1.hashCode(), methodContext4.hashCode()); assertEquals(methodContext4.hashCode(), methodContext5.hashCode()); } + + public void testToStringFromString() { + HashMap parameters3 = new HashMap() { + { + put("nlist", 4); + put("nprobes", 2); + } + }; + + HashMap parameters4 = new HashMap() { + { + put("nlist", 4); + put("type", "fp16"); + } + }; + + HashMap nestedParameters = new HashMap() { + { + put("nprobes", 2); + put("clip", false); + } + }; + HashMap parameters5 = new HashMap() { + { + put("nlist", 4); + put("type", "fp16"); + put("encoder", new MethodComponentContext("sq", nestedParameters)); + } + }; + + HashMap parameters6 = new HashMap() { + { + put("nlist", 4); + put("encoder", new MethodComponentContext("sq", nestedParameters)); + put("type", "fp16"); + } + }; + + MethodComponentContext methodComponentContext1 = MethodComponentContext.EMPTY; + MethodComponentContext methodComponentContext2 = new MethodComponentContext("ivf", null); + MethodComponentContext methodComponentContext3 = new MethodComponentContext("ivf", parameters3); + MethodComponentContext methodComponentContext4 = new MethodComponentContext("ivf", parameters4); + MethodComponentContext methodComponentContext5 = new MethodComponentContext("ivf", parameters5); + MethodComponentContext methodComponentContext6 = new MethodComponentContext("ivf", parameters6); + + String contextString1 = methodComponentContext1.toClusterStateString(); + String contextString2 = methodComponentContext2.toClusterStateString(); + String contextString3 = methodComponentContext3.toClusterStateString(); + String contextString4 = methodComponentContext4.toClusterStateString(); + String contextString5 = methodComponentContext5.toClusterStateString(); + String contextString6 = methodComponentContext6.toClusterStateString(); + + assertEquals("{name=;parameters=[]}", contextString1); + assertEquals("{name=ivf;parameters=[]}", contextString2); + + MethodComponentContext methodComponentContextFromString1 = MethodComponentContext.fromClusterStateString(contextString1); + MethodComponentContext methodComponentContextFromString2 = MethodComponentContext.fromClusterStateString(contextString2); + MethodComponentContext methodComponentContextFromString3 = MethodComponentContext.fromClusterStateString(contextString3); + MethodComponentContext methodComponentContextFromString4 = MethodComponentContext.fromClusterStateString(contextString4); + MethodComponentContext methodComponentContextFromString5 = MethodComponentContext.fromClusterStateString(contextString5); + MethodComponentContext methodComponentContextFromString6 = MethodComponentContext.fromClusterStateString(contextString6); + + assertEquals(methodComponentContext1, methodComponentContextFromString1); + assertEquals(new MethodComponentContext("ivf", Collections.emptyMap()), methodComponentContextFromString2); + assertEquals(methodComponentContext3, methodComponentContextFromString3); + assertEquals(methodComponentContext4, methodComponentContextFromString4); + assertEquals(methodComponentContext5, methodComponentContextFromString5); + assertEquals(methodComponentContext6, methodComponentContextFromString6); + } } diff --git a/src/test/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumerTests.java b/src/test/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumerTests.java index eeca1e5ed..7736652ce 100644 --- a/src/test/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumerTests.java +++ b/src/test/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumerTests.java @@ -343,7 +343,17 @@ public void testAddKNNBinaryField_fromModel_faiss() throws IOException, Executio byte[] modelBytes = JNIService.trainIndex(parameters, dimension, trainingPtr, knnEngine.getName()); Model model = new Model( - new ModelMetadata(knnEngine, spaceType, dimension, ModelState.CREATED, "timestamp", "Empty description", "", ""), + new ModelMetadata( + knnEngine, + spaceType, + dimension, + ModelState.CREATED, + "timestamp", + "Empty description", + "", + "", + MethodComponentContext.EMPTY + ), modelBytes, modelId ); diff --git a/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestCase.java b/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestCase.java index 42eb81759..8ab2641db 100644 --- a/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestCase.java +++ b/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestCase.java @@ -212,7 +212,8 @@ public void testBuildFromModelTemplate(Codec codec) throws IOException, Executio ZonedDateTime.now(ZoneOffset.UTC).toString(), "", "", - "" + "", + MethodComponentContext.EMPTY ); Model mockModel = new Model(modelMetadata1, modelBlob, modelId); diff --git a/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java b/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java index 278f90ba2..72dcac5c5 100644 --- a/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java +++ b/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java @@ -164,7 +164,8 @@ public void testBuilder_build_fromModel() { ZonedDateTime.now(ZoneOffset.UTC).toString(), "", "", - "" + "", + MethodComponentContext.EMPTY ); builder.modelId.setValue(modelId); Mapper.BuilderContext builderContext = new Mapper.BuilderContext(settings, new ContentPath()); @@ -691,7 +692,8 @@ public void testKNNVectorFieldMapper_merge_fromModel() throws IOException { ZonedDateTime.now(ZoneOffset.UTC).toString(), "", "", - "" + "", + MethodComponentContext.EMPTY ); when(mockModelDao.getMetadata(modelId)).thenReturn(mockModelMetadata); diff --git a/src/test/java/org/opensearch/knn/indices/ModelCacheTests.java b/src/test/java/org/opensearch/knn/indices/ModelCacheTests.java index 3146d898e..3a5255cd3 100644 --- a/src/test/java/org/opensearch/knn/indices/ModelCacheTests.java +++ b/src/test/java/org/opensearch/knn/indices/ModelCacheTests.java @@ -17,6 +17,7 @@ import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; import org.opensearch.knn.KNNTestCase; +import org.opensearch.knn.index.MethodComponentContext; import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.util.KNNEngine; @@ -43,7 +44,8 @@ public void testGet_normal() throws ExecutionException, InterruptedException { ZonedDateTime.now(ZoneOffset.UTC).toString(), "", "", - "" + "", + MethodComponentContext.EMPTY ), "hello".getBytes(), modelId @@ -79,7 +81,8 @@ public void testGet_modelDoesNotFitInCache() throws ExecutionException, Interrup ZonedDateTime.now(ZoneOffset.UTC).toString(), "", "", - "" + "", + MethodComponentContext.EMPTY ), new byte[BYTES_PER_KILOBYTES + 1], modelId @@ -136,7 +139,8 @@ public void testGetTotalWeight() throws ExecutionException, InterruptedException ZonedDateTime.now(ZoneOffset.UTC).toString(), "", "", - "" + "", + MethodComponentContext.EMPTY ), new byte[size1], modelId1 @@ -151,7 +155,8 @@ public void testGetTotalWeight() throws ExecutionException, InterruptedException ZonedDateTime.now(ZoneOffset.UTC).toString(), "", "", - "" + "", + MethodComponentContext.EMPTY ), new byte[size2], modelId2 @@ -194,7 +199,8 @@ public void testRemove_normal() throws ExecutionException, InterruptedException ZonedDateTime.now(ZoneOffset.UTC).toString(), "", "", - "" + "", + MethodComponentContext.EMPTY ), new byte[size1], modelId1 @@ -209,7 +215,9 @@ public void testRemove_normal() throws ExecutionException, InterruptedException ZonedDateTime.now(ZoneOffset.UTC).toString(), "", "", - "" + "", + MethodComponentContext.EMPTY + ), new byte[size2], modelId2 @@ -257,7 +265,8 @@ public void testRebuild_normal() throws ExecutionException, InterruptedException ZonedDateTime.now(ZoneOffset.UTC).toString(), "", "", - "" + "", + MethodComponentContext.EMPTY ), "hello".getBytes(), modelId @@ -302,7 +311,8 @@ public void testRebuild_afterSettingUpdate() throws ExecutionException, Interrup ZonedDateTime.now(ZoneOffset.UTC).toString(), "", "", - "" + "", + MethodComponentContext.EMPTY ), new byte[modelSize], modelId @@ -370,7 +380,8 @@ public void testContains() throws ExecutionException, InterruptedException { ZonedDateTime.now(ZoneOffset.UTC).toString(), "", "", - "" + "", + MethodComponentContext.EMPTY ), new byte[modelSize1], modelId1 @@ -411,7 +422,8 @@ public void testRemoveAll() throws ExecutionException, InterruptedException { ZonedDateTime.now(ZoneOffset.UTC).toString(), "", "", - "" + "", + MethodComponentContext.EMPTY ), new byte[modelSize1], modelId1 @@ -428,7 +440,8 @@ public void testRemoveAll() throws ExecutionException, InterruptedException { ZonedDateTime.now(ZoneOffset.UTC).toString(), "", "", - "" + "", + MethodComponentContext.EMPTY ), new byte[modelSize2], modelId2 @@ -473,7 +486,8 @@ public void testModelCacheEvictionDueToSize() throws ExecutionException, Interru ZonedDateTime.now(ZoneOffset.UTC).toString(), "", "", - "" + "", + MethodComponentContext.EMPTY ), new byte[BYTES_PER_KILOBYTES * 2], modelId diff --git a/src/test/java/org/opensearch/knn/indices/ModelDaoTests.java b/src/test/java/org/opensearch/knn/indices/ModelDaoTests.java index 2af8df953..ee2c77d1a 100644 --- a/src/test/java/org/opensearch/knn/indices/ModelDaoTests.java +++ b/src/test/java/org/opensearch/knn/indices/ModelDaoTests.java @@ -36,6 +36,7 @@ import org.opensearch.index.engine.VersionConflictEngineException; import org.opensearch.knn.KNNSingleNodeTestCase; import org.opensearch.knn.common.exception.DeleteModelWhenInTrainStateException; +import org.opensearch.knn.index.MethodComponentContext; import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.util.KNNEngine; import org.opensearch.knn.plugin.transport.DeleteModelResponse; @@ -54,6 +55,7 @@ import java.time.ZoneOffset; import java.time.ZonedDateTime; import java.util.Base64; +import java.util.Collections; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; @@ -151,7 +153,8 @@ public void testModelIndexHealth() throws InterruptedException, ExecutionExcepti ZonedDateTime.now(ZoneOffset.UTC).toString(), "", "", - "" + "", + MethodComponentContext.EMPTY ), modelBlob, modelId @@ -170,7 +173,8 @@ public void testModelIndexHealth() throws InterruptedException, ExecutionExcepti ZonedDateTime.now(ZoneOffset.UTC).toString(), "", "", - "" + "", + MethodComponentContext.EMPTY ), modelBlob, modelId @@ -197,7 +201,8 @@ public void testPut_withId() throws InterruptedException, IOException { ZonedDateTime.now(ZoneOffset.UTC).toString(), "", "", - "" + "", + new MethodComponentContext("test", Collections.emptyMap()) ), modelBlob, modelId @@ -257,7 +262,8 @@ public void testPut_withoutModel() throws InterruptedException, IOException { ZonedDateTime.now(ZoneOffset.UTC).toString(), "", "", - "" + "", + MethodComponentContext.EMPTY ), modelBlob, modelId @@ -318,7 +324,8 @@ public void testPut_invalid_badState() { ZonedDateTime.now(ZoneOffset.UTC).toString(), "", "", - "" + "", + MethodComponentContext.EMPTY ), modelBlob, "any-id" @@ -354,7 +361,8 @@ public void testUpdate() throws IOException, InterruptedException { ZonedDateTime.now(ZoneOffset.UTC).toString(), "", "", - "" + "", + MethodComponentContext.EMPTY ), null, modelId @@ -392,7 +400,8 @@ public void testUpdate() throws IOException, InterruptedException { ZonedDateTime.now(ZoneOffset.UTC).toString(), "", "", - "" + "", + MethodComponentContext.EMPTY ), modelBlob, modelId @@ -442,7 +451,8 @@ public void testGet() throws IOException, InterruptedException, ExecutionExcepti ZonedDateTime.now(ZoneOffset.UTC).toString(), "", "", - "" + "", + MethodComponentContext.EMPTY ), modelBlob, modelId @@ -460,7 +470,8 @@ public void testGet() throws IOException, InterruptedException, ExecutionExcepti ZonedDateTime.now(ZoneOffset.UTC).toString(), "", "", - "" + "", + MethodComponentContext.EMPTY ), null, modelId @@ -496,7 +507,8 @@ public void testGetMetadata() throws IOException, InterruptedException { ZonedDateTime.now(ZoneOffset.UTC).toString(), "", "", - "" + "", + MethodComponentContext.EMPTY ); Model model = new Model(modelMetadata, modelBlob, modelId); @@ -572,7 +584,8 @@ public void testDelete() throws IOException, InterruptedException { ZonedDateTime.now(ZoneOffset.UTC).toString(), "", "", - "" + "", + MethodComponentContext.EMPTY ), modelBlob, modelId @@ -605,7 +618,8 @@ public void testDelete() throws IOException, InterruptedException { ZonedDateTime.now(ZoneOffset.UTC).toString(), "", "", - "" + "", + MethodComponentContext.EMPTY ), modelBlob, modelId1 @@ -672,7 +686,8 @@ public void testDeleteModelInTrainingWithStepListeners() throws IOException, Exe ZonedDateTime.now(ZoneOffset.UTC).toString(), "", "", - "" + "", + MethodComponentContext.EMPTY ), modelBlob, modelId @@ -713,7 +728,8 @@ public void testDeleteWithStepListeners() throws IOException, InterruptedExcepti ZonedDateTime.now(ZoneOffset.UTC).toString(), "", "", - "" + "", + MethodComponentContext.EMPTY ), modelBlob, modelId diff --git a/src/test/java/org/opensearch/knn/indices/ModelMetadataTests.java b/src/test/java/org/opensearch/knn/indices/ModelMetadataTests.java index 219710308..da56a8421 100644 --- a/src/test/java/org/opensearch/knn/indices/ModelMetadataTests.java +++ b/src/test/java/org/opensearch/knn/indices/ModelMetadataTests.java @@ -12,8 +12,12 @@ package org.opensearch.knn.indices; import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.knn.KNNTestCase; import org.opensearch.knn.common.KNNConstants; +import org.opensearch.knn.index.MethodComponentContext; import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.util.KNNEngine; @@ -21,6 +25,7 @@ import java.time.ZoneId; import java.time.ZoneOffset; import java.time.ZonedDateTime; +import java.util.Collections; import java.util.HashMap; import java.util.Map; @@ -39,7 +44,8 @@ public void testStreams() throws IOException { ZonedDateTime.now(ZoneOffset.UTC).toString(), "", "", - "" + "", + MethodComponentContext.EMPTY ); BytesStreamOutput streamOutput = new BytesStreamOutput(); @@ -60,7 +66,8 @@ public void testGetKnnEngine() { ZonedDateTime.now(ZoneOffset.UTC).toString(), "", "", - "" + "", + MethodComponentContext.EMPTY ); assertEquals(knnEngine, modelMetadata.getKnnEngine()); @@ -76,7 +83,8 @@ public void testGetSpaceType() { ZonedDateTime.now(ZoneOffset.UTC).toString(), "", "", - "" + "", + MethodComponentContext.EMPTY ); assertEquals(spaceType, modelMetadata.getSpaceType()); @@ -92,7 +100,8 @@ public void testGetDimension() { ZonedDateTime.now(ZoneOffset.UTC).toString(), "", "", - "" + "", + MethodComponentContext.EMPTY ); assertEquals(dimension, modelMetadata.getDimension()); @@ -108,7 +117,8 @@ public void testGetState() { ZonedDateTime.now(ZoneOffset.UTC).toString(), "", "", - "" + "", + MethodComponentContext.EMPTY ); assertEquals(modelState, modelMetadata.getState()); @@ -116,7 +126,17 @@ public void testGetState() { public void testGetTimestamp() { String timeValue = ZonedDateTime.now(ZoneOffset.UTC).toString(); - ModelMetadata modelMetadata = new ModelMetadata(KNNEngine.DEFAULT, SpaceType.L2, 12, ModelState.CREATED, timeValue, "", "", ""); + ModelMetadata modelMetadata = new ModelMetadata( + KNNEngine.DEFAULT, + SpaceType.L2, + 12, + ModelState.CREATED, + timeValue, + "", + "", + "", + MethodComponentContext.EMPTY + ); assertEquals(timeValue, modelMetadata.getTimestamp()); } @@ -131,7 +151,8 @@ public void testDescription() { ZonedDateTime.now(ZoneOffset.UTC).toString(), description, "", - "" + "", + MethodComponentContext.EMPTY ); assertEquals(description, modelMetadata.getDescription()); @@ -147,7 +168,8 @@ public void testGetError() { ZonedDateTime.now(ZoneOffset.UTC).toString(), "", error, - "" + "", + MethodComponentContext.EMPTY ); assertEquals(error, modelMetadata.getError()); @@ -163,7 +185,8 @@ public void testSetState() { ZonedDateTime.now(ZoneOffset.UTC).toString(), "", "", - "" + "", + MethodComponentContext.EMPTY ); assertEquals(modelState, modelMetadata.getState()); @@ -183,7 +206,8 @@ public void testSetError() { ZonedDateTime.now(ZoneOffset.UTC).toString(), "", error, - "" + "", + MethodComponentContext.EMPTY ); assertEquals(error, modelMetadata.getError()); @@ -202,6 +226,7 @@ public void testToString() { String description = "test-description"; String error = "test-error"; String nodeAssignment = ""; + MethodComponentContext methodComponentContext = MethodComponentContext.EMPTY; String expected = knnEngine.getName() + "," @@ -217,7 +242,9 @@ public void testToString() { + "," + error + "," - + nodeAssignment; + + nodeAssignment + + "," + + methodComponentContext.toClusterStateString(); ModelMetadata modelMetadata = new ModelMetadata( knnEngine, @@ -227,7 +254,8 @@ public void testToString() { timestamp, description, error, - nodeAssignment + nodeAssignment, + MethodComponentContext.EMPTY ); assertEquals(expected, modelMetadata.toString()); @@ -238,14 +266,84 @@ public void testEquals() { String time1 = ZonedDateTime.now(ZoneOffset.UTC).toString(); String time2 = ZonedDateTime.of(2021, 9, 30, 12, 20, 45, 1, ZoneId.systemDefault()).toString(); - ModelMetadata modelMetadata1 = new ModelMetadata(KNNEngine.FAISS, SpaceType.L2, 128, ModelState.CREATED, time1, "", "", ""); - ModelMetadata modelMetadata2 = new ModelMetadata(KNNEngine.FAISS, SpaceType.L2, 128, ModelState.CREATED, time1, "", "", ""); + ModelMetadata modelMetadata1 = new ModelMetadata( + KNNEngine.FAISS, + SpaceType.L2, + 128, + ModelState.CREATED, + time1, + "", + "", + "", + MethodComponentContext.EMPTY + ); + ModelMetadata modelMetadata2 = new ModelMetadata( + KNNEngine.FAISS, + SpaceType.L2, + 128, + ModelState.CREATED, + time1, + "", + "", + "", + MethodComponentContext.EMPTY + ); - ModelMetadata modelMetadata3 = new ModelMetadata(KNNEngine.NMSLIB, SpaceType.L2, 128, ModelState.CREATED, time1, "", "", ""); - ModelMetadata modelMetadata4 = new ModelMetadata(KNNEngine.FAISS, SpaceType.L1, 128, ModelState.CREATED, time1, "", "", ""); - ModelMetadata modelMetadata5 = new ModelMetadata(KNNEngine.FAISS, SpaceType.L2, 129, ModelState.CREATED, time1, "", "", ""); - ModelMetadata modelMetadata6 = new ModelMetadata(KNNEngine.FAISS, SpaceType.L2, 128, ModelState.TRAINING, time1, "", "", ""); - ModelMetadata modelMetadata7 = new ModelMetadata(KNNEngine.FAISS, SpaceType.L2, 128, ModelState.CREATED, time2, "", "", ""); + ModelMetadata modelMetadata3 = new ModelMetadata( + KNNEngine.NMSLIB, + SpaceType.L2, + 128, + ModelState.CREATED, + time1, + "", + "", + "", + MethodComponentContext.EMPTY + ); + ModelMetadata modelMetadata4 = new ModelMetadata( + KNNEngine.FAISS, + SpaceType.L1, + 128, + ModelState.CREATED, + time1, + "", + "", + "", + MethodComponentContext.EMPTY + ); + ModelMetadata modelMetadata5 = new ModelMetadata( + KNNEngine.FAISS, + SpaceType.L2, + 129, + ModelState.CREATED, + time1, + "", + "", + "", + MethodComponentContext.EMPTY + ); + ModelMetadata modelMetadata6 = new ModelMetadata( + KNNEngine.FAISS, + SpaceType.L2, + 128, + ModelState.TRAINING, + time1, + "", + "", + "", + MethodComponentContext.EMPTY + ); + ModelMetadata modelMetadata7 = new ModelMetadata( + KNNEngine.FAISS, + SpaceType.L2, + 128, + ModelState.CREATED, + time2, + "", + "", + "", + MethodComponentContext.EMPTY + ); ModelMetadata modelMetadata8 = new ModelMetadata( KNNEngine.FAISS, SpaceType.L2, @@ -254,7 +352,8 @@ public void testEquals() { time1, "diff descript", "", - "" + "", + MethodComponentContext.EMPTY ); ModelMetadata modelMetadata9 = new ModelMetadata( KNNEngine.FAISS, @@ -264,7 +363,20 @@ public void testEquals() { time1, "", "diff error", - "" + "", + MethodComponentContext.EMPTY + ); + + ModelMetadata modelMetadata10 = new ModelMetadata( + KNNEngine.FAISS, + SpaceType.L2, + 128, + ModelState.CREATED, + time1, + "", + "", + "", + new MethodComponentContext("test", Collections.emptyMap()) ); assertEquals(modelMetadata1, modelMetadata1); @@ -285,14 +397,84 @@ public void testHashCode() { String time1 = ZonedDateTime.now(ZoneOffset.UTC).toString(); String time2 = ZonedDateTime.of(2021, 9, 30, 12, 20, 45, 1, ZoneId.systemDefault()).toString(); - ModelMetadata modelMetadata1 = new ModelMetadata(KNNEngine.FAISS, SpaceType.L2, 128, ModelState.CREATED, time1, "", "", ""); - ModelMetadata modelMetadata2 = new ModelMetadata(KNNEngine.FAISS, SpaceType.L2, 128, ModelState.CREATED, time1, "", "", ""); + ModelMetadata modelMetadata1 = new ModelMetadata( + KNNEngine.FAISS, + SpaceType.L2, + 128, + ModelState.CREATED, + time1, + "", + "", + "", + MethodComponentContext.EMPTY + ); + ModelMetadata modelMetadata2 = new ModelMetadata( + KNNEngine.FAISS, + SpaceType.L2, + 128, + ModelState.CREATED, + time1, + "", + "", + "", + MethodComponentContext.EMPTY + ); - ModelMetadata modelMetadata3 = new ModelMetadata(KNNEngine.NMSLIB, SpaceType.L2, 128, ModelState.CREATED, time1, "", "", ""); - ModelMetadata modelMetadata4 = new ModelMetadata(KNNEngine.FAISS, SpaceType.L1, 128, ModelState.CREATED, time1, "", "", ""); - ModelMetadata modelMetadata5 = new ModelMetadata(KNNEngine.FAISS, SpaceType.L2, 129, ModelState.CREATED, time1, "", "", ""); - ModelMetadata modelMetadata6 = new ModelMetadata(KNNEngine.FAISS, SpaceType.L2, 128, ModelState.TRAINING, time1, "", "", ""); - ModelMetadata modelMetadata7 = new ModelMetadata(KNNEngine.FAISS, SpaceType.L2, 128, ModelState.CREATED, time2, "", "", ""); + ModelMetadata modelMetadata3 = new ModelMetadata( + KNNEngine.NMSLIB, + SpaceType.L2, + 128, + ModelState.CREATED, + time1, + "", + "", + "", + MethodComponentContext.EMPTY + ); + ModelMetadata modelMetadata4 = new ModelMetadata( + KNNEngine.FAISS, + SpaceType.L1, + 128, + ModelState.CREATED, + time1, + "", + "", + "", + MethodComponentContext.EMPTY + ); + ModelMetadata modelMetadata5 = new ModelMetadata( + KNNEngine.FAISS, + SpaceType.L2, + 129, + ModelState.CREATED, + time1, + "", + "", + "", + MethodComponentContext.EMPTY + ); + ModelMetadata modelMetadata6 = new ModelMetadata( + KNNEngine.FAISS, + SpaceType.L2, + 128, + ModelState.TRAINING, + time1, + "", + "", + "", + MethodComponentContext.EMPTY + ); + ModelMetadata modelMetadata7 = new ModelMetadata( + KNNEngine.FAISS, + SpaceType.L2, + 128, + ModelState.CREATED, + time2, + "", + "", + "", + MethodComponentContext.EMPTY + ); ModelMetadata modelMetadata8 = new ModelMetadata( KNNEngine.FAISS, SpaceType.L2, @@ -301,7 +483,8 @@ public void testHashCode() { time1, "diff descript", "", - "" + "", + MethodComponentContext.EMPTY ); ModelMetadata modelMetadata9 = new ModelMetadata( KNNEngine.FAISS, @@ -311,20 +494,33 @@ public void testHashCode() { time1, "", "diff error", - "" + "", + MethodComponentContext.EMPTY + ); + + ModelMetadata modelMetadata10 = new ModelMetadata( + KNNEngine.FAISS, + SpaceType.L2, + 128, + ModelState.CREATED, + time1, + "", + "", + "", + new MethodComponentContext("test", Collections.emptyMap()) ); assertEquals(modelMetadata1.hashCode(), modelMetadata1.hashCode()); assertEquals(modelMetadata1.hashCode(), modelMetadata2.hashCode()); assertNotEquals(modelMetadata1.hashCode(), modelMetadata3.hashCode()); - assertNotEquals(modelMetadata1.hashCode(), modelMetadata3.hashCode()); assertNotEquals(modelMetadata1.hashCode(), modelMetadata4.hashCode()); assertNotEquals(modelMetadata1.hashCode(), modelMetadata5.hashCode()); assertNotEquals(modelMetadata1.hashCode(), modelMetadata6.hashCode()); assertNotEquals(modelMetadata1.hashCode(), modelMetadata7.hashCode()); assertNotEquals(modelMetadata1.hashCode(), modelMetadata8.hashCode()); assertNotEquals(modelMetadata1.hashCode(), modelMetadata9.hashCode()); + assertNotEquals(modelMetadata1.hashCode(), modelMetadata10.hashCode()); } public void testFromString() { @@ -336,6 +532,7 @@ public void testFromString() { String description = "test-description"; String error = "test-error"; String nodeAssignment = "test-node"; + MethodComponentContext methodComponentContext = MethodComponentContext.EMPTY; String stringRep1 = knnEngine.getName() + "," @@ -351,7 +548,9 @@ public void testFromString() { + "," + error + "," - + nodeAssignment; + + nodeAssignment + + "," + + methodComponentContext.toClusterStateString(); String stringRep2 = knnEngine.getName() + "," @@ -375,10 +574,21 @@ public void testFromString() { timestamp, description, error, - nodeAssignment + nodeAssignment, + MethodComponentContext.EMPTY ); - ModelMetadata expected2 = new ModelMetadata(knnEngine, spaceType, dimension, modelState, timestamp, description, error, ""); + ModelMetadata expected2 = new ModelMetadata( + knnEngine, + spaceType, + dimension, + modelState, + timestamp, + description, + error, + "", + MethodComponentContext.EMPTY + ); ModelMetadata fromString1 = ModelMetadata.fromString(stringRep1); ModelMetadata fromString2 = ModelMetadata.fromString(stringRep2); @@ -389,7 +599,7 @@ public void testFromString() { expectThrows(IllegalArgumentException.class, () -> ModelMetadata.fromString("invalid")); } - public void testFromResponseMap() { + public void testFromResponseMap() throws IOException { KNNEngine knnEngine = KNNEngine.DEFAULT; SpaceType spaceType = SpaceType.L2; int dimension = 128; @@ -398,6 +608,21 @@ public void testFromResponseMap() { String description = "test-description"; String error = "test-error"; String nodeAssignment = "test-node"; + Map nestedParameters = new HashMap() { + { + put("testNestedKey1", "testNestedString"); + put("testNestedKey2", 1); + } + }; + Map parameters = new HashMap<>() { + { + put("testKey1", "testString"); + put("testKey2", 0); + put("testKey3", new MethodComponentContext("ivf", nestedParameters)); + } + }; + MethodComponentContext methodComponentContext = new MethodComponentContext("hnsw", parameters); + MethodComponentContext emptyMethodComponentContext = MethodComponentContext.EMPTY; ModelMetadata expected = new ModelMetadata( knnEngine, @@ -407,9 +632,21 @@ public void testFromResponseMap() { timestamp, description, error, - nodeAssignment + nodeAssignment, + methodComponentContext + + ); + ModelMetadata expected2 = new ModelMetadata( + knnEngine, + spaceType, + dimension, + modelState, + timestamp, + description, + error, + "", + emptyMethodComponentContext ); - ModelMetadata expected2 = new ModelMetadata(knnEngine, spaceType, dimension, modelState, timestamp, description, error, ""); Map metadataAsMap = new HashMap<>(); metadataAsMap.put(KNNConstants.KNN_ENGINE, knnEngine.getName()); metadataAsMap.put(KNNConstants.METHOD_PARAMETER_SPACE_TYPE, spaceType.getValue()); @@ -420,10 +657,15 @@ public void testFromResponseMap() { metadataAsMap.put(KNNConstants.MODEL_ERROR, error); metadataAsMap.put(KNNConstants.MODEL_NODE_ASSIGNMENT, nodeAssignment); + XContentBuilder builder = XContentFactory.jsonBuilder().startObject(); + builder = methodComponentContext.toXContent(builder, ToXContent.EMPTY_PARAMS).endObject(); + metadataAsMap.put(KNNConstants.MODEL_METHOD_COMPONENT_CONTEXT, builder.toString()); + ModelMetadata fromMap = ModelMetadata.getMetadataFromSourceMap(metadataAsMap); assertEquals(expected, fromMap); metadataAsMap.put(KNNConstants.MODEL_NODE_ASSIGNMENT, null); + metadataAsMap.put(KNNConstants.MODEL_METHOD_COMPONENT_CONTEXT, null); assertEquals(expected2, fromMap); } diff --git a/src/test/java/org/opensearch/knn/indices/ModelTests.java b/src/test/java/org/opensearch/knn/indices/ModelTests.java index c015e8d62..13579acad 100644 --- a/src/test/java/org/opensearch/knn/indices/ModelTests.java +++ b/src/test/java/org/opensearch/knn/indices/ModelTests.java @@ -13,6 +13,7 @@ import org.opensearch.knn.KNNTestCase; import org.opensearch.knn.common.KNNConstants; +import org.opensearch.knn.index.MethodComponentContext; import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.util.KNNEngine; @@ -39,7 +40,8 @@ public void testInvalidConstructor() { ZonedDateTime.now(ZoneOffset.UTC).toString(), "", "", - "" + "", + MethodComponentContext.EMPTY ), null, "test-model" @@ -59,7 +61,8 @@ public void testInvalidDimension() { ZonedDateTime.now(ZoneOffset.UTC).toString(), "", "", - "" + "", + MethodComponentContext.EMPTY ), new byte[16], "test-model" @@ -76,7 +79,8 @@ public void testInvalidDimension() { ZonedDateTime.now(ZoneOffset.UTC).toString(), "", "", - "" + "", + MethodComponentContext.EMPTY ), new byte[16], "test-model" @@ -93,7 +97,8 @@ public void testInvalidDimension() { ZonedDateTime.now(ZoneOffset.UTC).toString(), "", "", - "" + "", + MethodComponentContext.EMPTY ), new byte[16], "test-model" @@ -111,7 +116,8 @@ public void testGetModelMetadata() { ZonedDateTime.now(ZoneOffset.UTC).toString(), "", "", - "" + "", + MethodComponentContext.EMPTY ); Model model = new Model(modelMetadata, new byte[16], "test-model"); assertEquals(modelMetadata, model.getModelMetadata()); @@ -128,7 +134,8 @@ public void testGetModelBlob() { ZonedDateTime.now(ZoneOffset.UTC).toString(), "", "", - "" + "", + MethodComponentContext.EMPTY ), modelBlob, "test-model" @@ -147,7 +154,8 @@ public void testGetLength() { ZonedDateTime.now(ZoneOffset.UTC).toString(), "", "", - "" + "", + MethodComponentContext.EMPTY ), new byte[size], "test-model" @@ -163,7 +171,8 @@ public void testGetLength() { ZonedDateTime.now(ZoneOffset.UTC).toString(), "", "", - "" + "", + MethodComponentContext.EMPTY ), null, "test-model" @@ -182,7 +191,8 @@ public void testSetModelBlob() { ZonedDateTime.now(ZoneOffset.UTC).toString(), "", "", - "" + "", + MethodComponentContext.EMPTY ), blob1, "test-model" @@ -199,17 +209,17 @@ public void testEquals() { String time = ZonedDateTime.now(ZoneOffset.UTC).toString(); Model model1 = new Model( - new ModelMetadata(KNNEngine.DEFAULT, SpaceType.L1, 2, ModelState.CREATED, time, "", "", ""), + new ModelMetadata(KNNEngine.DEFAULT, SpaceType.L1, 2, ModelState.CREATED, time, "", "", "", MethodComponentContext.EMPTY), new byte[16], "test-model-1" ); Model model2 = new Model( - new ModelMetadata(KNNEngine.DEFAULT, SpaceType.L1, 2, ModelState.CREATED, time, "", "", ""), + new ModelMetadata(KNNEngine.DEFAULT, SpaceType.L1, 2, ModelState.CREATED, time, "", "", "", MethodComponentContext.EMPTY), new byte[16], "test-model-1" ); Model model3 = new Model( - new ModelMetadata(KNNEngine.DEFAULT, SpaceType.L2, 2, ModelState.CREATED, time, "", "", ""), + new ModelMetadata(KNNEngine.DEFAULT, SpaceType.L2, 2, ModelState.CREATED, time, "", "", "", MethodComponentContext.EMPTY), new byte[16], "test-model-2" ); @@ -224,17 +234,17 @@ public void testHashCode() { String time = ZonedDateTime.now(ZoneOffset.UTC).toString(); Model model1 = new Model( - new ModelMetadata(KNNEngine.DEFAULT, SpaceType.L1, 2, ModelState.CREATED, time, "", "", ""), + new ModelMetadata(KNNEngine.DEFAULT, SpaceType.L1, 2, ModelState.CREATED, time, "", "", "", MethodComponentContext.EMPTY), new byte[16], "test-model-1" ); Model model2 = new Model( - new ModelMetadata(KNNEngine.DEFAULT, SpaceType.L1, 2, ModelState.CREATED, time, "", "", ""), + new ModelMetadata(KNNEngine.DEFAULT, SpaceType.L1, 2, ModelState.CREATED, time, "", "", "", MethodComponentContext.EMPTY), new byte[16], "test-model-1" ); Model model3 = new Model( - new ModelMetadata(KNNEngine.DEFAULT, SpaceType.L1, 2, ModelState.CREATED, time, "", "", ""), + new ModelMetadata(KNNEngine.DEFAULT, SpaceType.L1, 2, ModelState.CREATED, time, "", "", "", MethodComponentContext.EMPTY), new byte[16], "test-model-2" ); @@ -263,7 +273,8 @@ public void testModelFromSourceMap() { timestamp, description, error, - nodeAssignment + nodeAssignment, + MethodComponentContext.EMPTY ); Map modelAsMap = new HashMap<>(); modelAsMap.put(KNNConstants.MODEL_ID, modelID); diff --git a/src/test/java/org/opensearch/knn/plugin/transport/GetModelResponseTests.java b/src/test/java/org/opensearch/knn/plugin/transport/GetModelResponseTests.java index a54605d24..2a59874d5 100644 --- a/src/test/java/org/opensearch/knn/plugin/transport/GetModelResponseTests.java +++ b/src/test/java/org/opensearch/knn/plugin/transport/GetModelResponseTests.java @@ -19,6 +19,7 @@ import org.opensearch.common.xcontent.XContentType; import org.opensearch.knn.KNNTestCase; import org.opensearch.knn.index.KNNClusterUtil; +import org.opensearch.knn.index.MethodComponentContext; import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.util.KNNEngine; import org.opensearch.knn.indices.Model; @@ -34,7 +35,17 @@ public class GetModelResponseTests extends KNNTestCase { private ModelMetadata getModelMetadata(ModelState state) { - return new ModelMetadata(KNNEngine.DEFAULT, SpaceType.DEFAULT, 4, state, "2021-03-27 10:15:30 AM +05:30", "test model", "", ""); + return new ModelMetadata( + KNNEngine.DEFAULT, + SpaceType.DEFAULT, + 4, + state, + "2021-03-27 10:15:30 AM +05:30", + "test model", + "", + "", + MethodComponentContext.EMPTY + ); } public void testStreams() throws IOException { @@ -58,7 +69,7 @@ public void testXContent() throws IOException { Model model = new Model(getModelMetadata(ModelState.CREATED), testModelBlob, modelId); GetModelResponse getModelResponse = new GetModelResponse(model); String expectedResponseString = - "{\"model_id\":\"test-model\",\"model_blob\":\"aGVsbG8=\",\"state\":\"created\",\"timestamp\":\"2021-03-27 10:15:30 AM +05:30\",\"description\":\"test model\",\"error\":\"\",\"space_type\":\"l2\",\"dimension\":4,\"engine\":\"nmslib\",\"training_node_assignment\":\"\"}"; + "{\"model_id\":\"test-model\",\"model_blob\":\"aGVsbG8=\",\"state\":\"created\",\"timestamp\":\"2021-03-27 10:15:30 AM +05:30\",\"description\":\"test model\",\"error\":\"\",\"space_type\":\"l2\",\"dimension\":4,\"engine\":\"nmslib\",\"training_node_assignment\":\"\",\"model_definition\":{\"name\":\"\",\"parameters\":{}}}"; XContentBuilder xContentBuilder = MediaTypeRegistry.contentBuilder(XContentType.JSON); getModelResponse.toXContent(xContentBuilder, null); assertEquals(expectedResponseString, xContentBuilder.toString()); @@ -74,7 +85,7 @@ public void testXContentWithNoModelBlob() throws IOException { Model model = new Model(getModelMetadata(ModelState.FAILED), null, modelId); GetModelResponse getModelResponse = new GetModelResponse(model); String expectedResponseString = - "{\"model_id\":\"test-model\",\"model_blob\":\"\",\"state\":\"failed\",\"timestamp\":\"2021-03-27 10:15:30 AM +05:30\",\"description\":\"test model\",\"error\":\"\",\"space_type\":\"l2\",\"dimension\":4,\"engine\":\"nmslib\",\"training_node_assignment\":\"\"}"; + "{\"model_id\":\"test-model\",\"model_blob\":\"\",\"state\":\"failed\",\"timestamp\":\"2021-03-27 10:15:30 AM +05:30\",\"description\":\"test model\",\"error\":\"\",\"space_type\":\"l2\",\"dimension\":4,\"engine\":\"nmslib\",\"training_node_assignment\":\"\",\"model_definition\":{\"name\":\"\",\"parameters\":{}}}"; XContentBuilder xContentBuilder = MediaTypeRegistry.contentBuilder(XContentType.JSON); getModelResponse.toXContent(xContentBuilder, null); assertEquals(expectedResponseString, xContentBuilder.toString()); diff --git a/src/test/java/org/opensearch/knn/plugin/transport/RemoveModelFromCacheTransportActionTests.java b/src/test/java/org/opensearch/knn/plugin/transport/RemoveModelFromCacheTransportActionTests.java index 5d30f54bb..a2da83dad 100644 --- a/src/test/java/org/opensearch/knn/plugin/transport/RemoveModelFromCacheTransportActionTests.java +++ b/src/test/java/org/opensearch/knn/plugin/transport/RemoveModelFromCacheTransportActionTests.java @@ -17,6 +17,7 @@ import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; import org.opensearch.knn.KNNSingleNodeTestCase; +import org.opensearch.knn.index.MethodComponentContext; import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.util.KNNEngine; import org.opensearch.knn.indices.Model; @@ -68,7 +69,17 @@ public void testNodeOperation_modelInCache() throws ExecutionException, Interrup ModelDao modelDao = mock(ModelDao.class); String modelId = "test-model-id"; Model model = new Model( - new ModelMetadata(KNNEngine.DEFAULT, SpaceType.L2, 16, ModelState.CREATED, "timestamp", "description", "", ""), + new ModelMetadata( + KNNEngine.DEFAULT, + SpaceType.L2, + 16, + ModelState.CREATED, + "timestamp", + "description", + "", + "", + MethodComponentContext.EMPTY + ), new byte[128], modelId ); diff --git a/src/test/java/org/opensearch/knn/plugin/transport/TrainingModelRequestTests.java b/src/test/java/org/opensearch/knn/plugin/transport/TrainingModelRequestTests.java index 7465ccc58..bdae54cad 100644 --- a/src/test/java/org/opensearch/knn/plugin/transport/TrainingModelRequestTests.java +++ b/src/test/java/org/opensearch/knn/plugin/transport/TrainingModelRequestTests.java @@ -24,6 +24,7 @@ import org.opensearch.knn.KNNTestCase; import org.opensearch.knn.common.KNNConstants; import org.opensearch.knn.index.KNNMethodContext; +import org.opensearch.knn.index.MethodComponentContext; import org.opensearch.knn.index.mapper.KNNVectorFieldMapper; import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.util.KNNEngine; @@ -168,7 +169,8 @@ public void testValidation_invalid_modelIdAlreadyExists() { ZonedDateTime.now(ZoneOffset.UTC).toString(), "", "", - "" + "", + MethodComponentContext.EMPTY ); when(modelDao.getMetadata(modelId)).thenReturn(modelMetadata); diff --git a/src/test/java/org/opensearch/knn/plugin/transport/UpdateModelMetadataRequestTests.java b/src/test/java/org/opensearch/knn/plugin/transport/UpdateModelMetadataRequestTests.java index a41ca900a..3719d124a 100644 --- a/src/test/java/org/opensearch/knn/plugin/transport/UpdateModelMetadataRequestTests.java +++ b/src/test/java/org/opensearch/knn/plugin/transport/UpdateModelMetadataRequestTests.java @@ -13,6 +13,7 @@ import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.knn.KNNTestCase; +import org.opensearch.knn.index.MethodComponentContext; import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.util.KNNEngine; import org.opensearch.knn.indices.ModelMetadata; @@ -40,7 +41,8 @@ public void testStreams() throws IOException { ZonedDateTime.now(ZoneOffset.UTC).toString(), "", "", - "" + "", + MethodComponentContext.EMPTY ); UpdateModelMetadataRequest updateModelMetadataRequest = new UpdateModelMetadataRequest(modelId, isRemoveRequest, modelMetadata); @@ -64,7 +66,8 @@ public void testValidate() { ZonedDateTime.now(ZoneOffset.UTC).toString(), "", "", - "" + "", + MethodComponentContext.EMPTY ); UpdateModelMetadataRequest updateModelMetadataRequest1 = new UpdateModelMetadataRequest("test", true, null); @@ -103,7 +106,8 @@ public void testGetModelMetadata() { ZonedDateTime.now(ZoneOffset.UTC).toString(), "", "", - "" + "", + MethodComponentContext.EMPTY ); UpdateModelMetadataRequest updateModelMetadataRequest = new UpdateModelMetadataRequest("test", true, modelMetadata); diff --git a/src/test/java/org/opensearch/knn/plugin/transport/UpdateModelMetadataTransportActionTests.java b/src/test/java/org/opensearch/knn/plugin/transport/UpdateModelMetadataTransportActionTests.java index 11961f6f5..ab0e4f506 100644 --- a/src/test/java/org/opensearch/knn/plugin/transport/UpdateModelMetadataTransportActionTests.java +++ b/src/test/java/org/opensearch/knn/plugin/transport/UpdateModelMetadataTransportActionTests.java @@ -17,6 +17,7 @@ import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.knn.KNNSingleNodeTestCase; +import org.opensearch.knn.index.MethodComponentContext; import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.util.KNNEngine; import org.opensearch.knn.indices.ModelMetadata; @@ -66,7 +67,8 @@ public void testClusterManagerOperation() throws InterruptedException { ZonedDateTime.now(ZoneOffset.UTC).toString(), "", "", - "" + "", + MethodComponentContext.EMPTY ); // Get update transport action diff --git a/src/test/java/org/opensearch/knn/training/TrainingJobTests.java b/src/test/java/org/opensearch/knn/training/TrainingJobTests.java index 6b07b2dd2..9dc461b97 100644 --- a/src/test/java/org/opensearch/knn/training/TrainingJobTests.java +++ b/src/test/java/org/opensearch/knn/training/TrainingJobTests.java @@ -56,6 +56,7 @@ public void testGetModelId() { KNNMethodContext knnMethodContext = mock(KNNMethodContext.class); when(knnMethodContext.getKnnEngine()).thenReturn(KNNEngine.DEFAULT); when(knnMethodContext.getSpaceType()).thenReturn(SpaceType.DEFAULT); + when(knnMethodContext.getMethodComponentContext()).thenReturn(MethodComponentContext.EMPTY); TrainingJob trainingJob = new TrainingJob( modelId, @@ -78,10 +79,12 @@ public void testGetModel() { String description = "test description"; String error = ""; String nodeAssignment = "test-node"; + MethodComponentContext methodComponentContext = MethodComponentContext.EMPTY; KNNMethodContext knnMethodContext = mock(KNNMethodContext.class); when(knnMethodContext.getKnnEngine()).thenReturn(knnEngine); when(knnMethodContext.getSpaceType()).thenReturn(spaceType); + when(knnMethodContext.getMethodComponentContext()).thenReturn(methodComponentContext); String modelID = "test-model-id"; TrainingJob trainingJob = new TrainingJob( @@ -104,7 +107,8 @@ public void testGetModel() { trainingJob.getModel().getModelMetadata().getTimestamp(), description, error, - nodeAssignment + nodeAssignment, + MethodComponentContext.EMPTY ), null, modelID From 67b54c04b9001547e5d97974e70596bf00b10f4b Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Mon, 18 Mar 2024 08:25:04 -0700 Subject: [PATCH 220/416] Enabled Inner Product Space Type support for Lucene Engine (#1551) (#1555) Signed-off-by: Navneet Verma (cherry picked from commit eefc7d78aeb247c50e57a4ff1ce7ccbd54a23f35) Co-authored-by: Navneet Verma --- CHANGELOG.md | 1 + .../org/opensearch/knn/index/util/Lucene.java | 2 +- .../opensearch/knn/index/LuceneEngineIT.java | 81 ++++++++++++------- .../mapper/KNNVectorFieldMapperTests.java | 23 ------ .../knn/index/util/LuceneTests.java | 14 ++++ 5 files changed, 66 insertions(+), 55 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a669bfb2f..c5f52a431 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), * Detect AVX2 Dynamically on the System [#1502](https://github.com/opensearch-project/k-NN/pull/1502) * Validate zero vector when using cosine metric [#1501](https://github.com/opensearch-project/k-NN/pull/1501) * Persist model definition in model metadata [#1527] (https://github.com/opensearch-project/k-NN/pull/1527) +* Added Inner Product Space type support for Lucene Engine [#1551](https://github.com/opensearch-project/k-NN/pull/1551) ### Bug Fixes * Disable sdc table for HNSWPQ read-only indices [#1518](https://github.com/opensearch-project/k-NN/pull/1518) * Switch SpaceType.INNERPRODUCT's vector similarity function to MAXIMUM_INNER_PRODUCT [#1532](https://github.com/opensearch-project/k-NN/pull/1532) diff --git a/src/main/java/org/opensearch/knn/index/util/Lucene.java b/src/main/java/org/opensearch/knn/index/util/Lucene.java index bfa6cb040..63642ae2c 100644 --- a/src/main/java/org/opensearch/knn/index/util/Lucene.java +++ b/src/main/java/org/opensearch/knn/index/util/Lucene.java @@ -42,7 +42,7 @@ public class Lucene extends JVMLibrary { ) ) .build() - ).addSpaces(SpaceType.L2, SpaceType.COSINESIMIL).build() + ).addSpaces(SpaceType.L2, SpaceType.COSINESIMIL, SpaceType.INNER_PRODUCT).build() ); final static Lucene INSTANCE = new Lucene(METHODS, Version.LATEST.toString()); diff --git a/src/test/java/org/opensearch/knn/index/LuceneEngineIT.java b/src/test/java/org/opensearch/knn/index/LuceneEngineIT.java index 30ac6922a..083b5b370 100644 --- a/src/test/java/org/opensearch/knn/index/LuceneEngineIT.java +++ b/src/test/java/org/opensearch/knn/index/LuceneEngineIT.java @@ -12,8 +12,8 @@ import lombok.SneakyThrows; import org.apache.commons.lang.math.RandomUtils; import org.apache.lucene.index.VectorSimilarityFunction; +import org.apache.lucene.util.VectorUtil; import org.junit.After; -import org.opensearch.client.Request; import org.opensearch.client.Response; import org.opensearch.client.ResponseException; import org.opensearch.core.xcontent.XContentBuilder; @@ -80,36 +80,6 @@ public void testQuery_cosine() throws Exception { baseQueryTest(SpaceType.COSINESIMIL); } - public void testQuery_innerProduct_notSupported() throws Exception { - XContentBuilder builder = XContentFactory.jsonBuilder() - .startObject() - .startObject(PROPERTIES_FIELD_NAME) - .startObject(FIELD_NAME) - .field(TYPE_FIELD_NAME, KNN_VECTOR_TYPE) - .field(DIMENSION_FIELD_NAME, DIMENSION) - .startObject(KNNConstants.KNN_METHOD) - .field(KNNConstants.NAME, KNNEngine.LUCENE.getMethod(METHOD_HNSW).getMethodComponent().getName()) - .field(KNNConstants.METHOD_PARAMETER_SPACE_TYPE, SpaceType.INNER_PRODUCT.getValue()) - .field(KNNConstants.KNN_ENGINE, KNNEngine.LUCENE.getName()) - .startObject(KNNConstants.PARAMETERS) - .field(KNNConstants.METHOD_PARAMETER_M, M) - .field(KNNConstants.METHOD_PARAMETER_EF_CONSTRUCTION, EF_CONSTRUCTION) - .endObject() - .endObject() - .endObject() - .endObject() - .endObject(); - - String mapping = builder.toString(); - - createIndex(INDEX_NAME, getKNNDefaultIndexSettings()); - - Request request = new Request("PUT", "/" + INDEX_NAME + "/_mapping"); - request.setJsonEntity(mapping); - - expectThrows(ResponseException.class, () -> client().performRequest(request)); - } - public void testQuery_invalidVectorDimensionInQuery() throws Exception { createKnnIndexMappingWithLuceneEngine(DIMENSION, SpaceType.L2, VectorDataType.FLOAT); @@ -449,4 +419,53 @@ private void validateQueryResultsWithFilters( .containsAll(expectedDocIdsKLimitsFilterResult) ); } + + @SneakyThrows + public void test_whenUsingIP_thenSuccess() { + XContentBuilder builder = XContentFactory.jsonBuilder() + .startObject() + .startObject("properties") + .startObject(FIELD_NAME) + .field("type", "knn_vector") + .field("dimension", 2) + .startObject(KNNConstants.KNN_METHOD) + .field(KNNConstants.NAME, KNNEngine.LUCENE.getMethod(KNNConstants.METHOD_HNSW).getMethodComponent().getName()) + .field(KNNConstants.METHOD_PARAMETER_SPACE_TYPE, SpaceType.INNER_PRODUCT.getValue()) + .field(KNNConstants.KNN_ENGINE, KNNEngine.LUCENE) + .endObject() + .endObject() + .endObject() + .endObject(); + final String mapping = builder.toString(); + createKnnIndex(INDEX_NAME, mapping); + + final List dataVectors = Arrays.asList(new Float[] { -2.0f, 2.0f }, new Float[] { 2.0f, -2.0f }); + final List ids = Arrays.asList(DOC_ID, DOC_ID_2); + + // Ingest all the documents + for (int i = 0; i < dataVectors.size(); i++) { + addKnnDoc(INDEX_NAME, ids.get(i), FIELD_NAME, dataVectors.get(i)); + } + refreshIndex(INDEX_NAME); + + float[] queryVector = new float[] { -2.0f, 2.0f }; + int k = 2; + final Response response = searchKNNIndex( + INDEX_NAME, + new KNNQueryBuilder(FIELD_NAME, queryVector, k, QueryBuilders.matchAllQuery()), + k + ); + final String responseBody = EntityUtils.toString(response.getEntity()); + final List knnResults = parseSearchResponseScore(responseBody, FIELD_NAME); + + // Check that the expected scores are returned + final List expectedScores = Arrays.asList( + VectorUtil.scaleMaxInnerProductScore(8.0f), + VectorUtil.scaleMaxInnerProductScore(-8.0f) + ); + assertEquals(expectedScores.size(), knnResults.size()); + for (int i = 0; i < expectedScores.size(); i++) { + assertEquals(expectedScores.get(i), knnResults.get(i), 0.0000001); + } + } } diff --git a/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java b/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java index 72dcac5c5..824b61a9f 100644 --- a/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java +++ b/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java @@ -431,29 +431,6 @@ public void testTypeParser_parse_fromKnnMethodContext_invalidSpaceType() throws ValidationException.class, () -> typeParser.parse(fieldName, xContentBuilderToMap(xContentBuilderL1SpaceType), buildParserContext(indexName, settings)) ); - - XContentBuilder xContentBuilderInnerproductSpaceType = XContentFactory.jsonBuilder() - .startObject() - .field(TYPE_FIELD_NAME, KNN_VECTOR_TYPE) - .field(DIMENSION_FIELD_NAME, dimension) - .startObject(KNN_METHOD) - .field(NAME, METHOD_HNSW) - .field(METHOD_PARAMETER_SPACE_TYPE, SpaceType.INNER_PRODUCT.getValue()) - .field(KNN_ENGINE, LUCENE_NAME) - .startObject(PARAMETERS) - .field(METHOD_PARAMETER_EF_CONSTRUCTION, efConstruction) - .endObject() - .endObject() - .endObject(); - - expectThrows( - ValidationException.class, - () -> typeParser.parse( - fieldName, - xContentBuilderToMap(xContentBuilderInnerproductSpaceType), - buildParserContext(indexName, settings) - ) - ); } public void testTypeParser_parse_fromKnnMethodContext() throws IOException { diff --git a/src/test/java/org/opensearch/knn/index/util/LuceneTests.java b/src/test/java/org/opensearch/knn/index/util/LuceneTests.java index 38cacffa4..6de46b52d 100644 --- a/src/test/java/org/opensearch/knn/index/util/LuceneTests.java +++ b/src/test/java/org/opensearch/knn/index/util/LuceneTests.java @@ -82,6 +82,20 @@ public void testLucenHNSWMethod() throws IOException { in = xContentBuilderToMap(xContentBuilder); KNNMethodContext knnMethodContext4 = KNNMethodContext.parse(in); assertNotNull(luceneHNSW.validate(knnMethodContext4)); + + // Check INNER_PRODUCT is supported with Lucene Engine + xContentBuilder = XContentFactory.jsonBuilder() + .startObject() + .field(NAME, METHOD_HNSW) + .field(METHOD_PARAMETER_SPACE_TYPE, SpaceType.INNER_PRODUCT.getValue()) + .startObject(PARAMETERS) + .field(METHOD_PARAMETER_EF_CONSTRUCTION, efConstruction) + .field(METHOD_PARAMETER_M, m) + .endObject() + .endObject(); + in = xContentBuilderToMap(xContentBuilder); + KNNMethodContext knnMethodContext5 = KNNMethodContext.parse(in); + assertNull(luceneHNSW.validate(knnMethodContext5)); } public void testGetExtension() { From ae6faa8854c463a09e9c10be52f00030787ccc90 Mon Sep 17 00:00:00 2001 From: John Mazanec Date: Thu, 7 Mar 2024 10:28:02 -0800 Subject: [PATCH 221/416] Introduce faiss patch to share ivfpq precomp table (#1520) Introduces a patch for faiss library to share ivfpq table between faiss indices. Patch is built from: https://github.com/jmazanec15/faiss/pull/1/commits/c5ca07299b427dedafc738b98bd20f8f286f6783. In addition, modify different locations to apply patch. Signed-off-by: John Mazanec --- .github/workflows/CI.yml | 2 + .github/workflows/test_security.yml | 2 + jni/CMakeLists.txt | 4 +- ...ble-precomp-table-to-be-shared-ivfpq.patch | 512 ++++++++++++++++++ 4 files changed, 518 insertions(+), 2 deletions(-) create mode 100644 jni/patches/faiss/0003-Enable-precomp-table-to-be-shared-ivfpq.patch diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index f333f25fd..504aabada 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -47,6 +47,8 @@ jobs: rm ../../patches/faiss/0001-Custom-patch-to-support-multi-vector.patch git apply --ignore-space-change --ignore-whitespace --3way ../../patches/faiss/0002-Custom-patch-to-support-AVX2-Linux-CI.patch rm ../../patches/faiss/0002-Custom-patch-to-support-AVX2-Linux-CI.patch + git apply --ignore-space-change --ignore-whitespace --3way ../../patches/faiss/0003-Enable-precomp-table-to-be-shared-ivfpq.patch + rm ../../patches/faiss/0003-Enable-precomp-table-to-be-shared-ivfpq.patch cd ../nmslib git apply --ignore-space-change --ignore-whitespace --3way ../../patches/nmslib/0001-Initialize-maxlevel-during-add-from-enterpoint-level.patch rm ../../patches/nmslib/0001-Initialize-maxlevel-during-add-from-enterpoint-level.patch diff --git a/.github/workflows/test_security.yml b/.github/workflows/test_security.yml index b2f6eb353..f6eebced2 100644 --- a/.github/workflows/test_security.yml +++ b/.github/workflows/test_security.yml @@ -47,6 +47,8 @@ jobs: rm ../../patches/faiss/0001-Custom-patch-to-support-multi-vector.patch git apply --ignore-space-change --ignore-whitespace --3way ../../patches/faiss/0002-Custom-patch-to-support-AVX2-Linux-CI.patch rm ../../patches/faiss/0002-Custom-patch-to-support-AVX2-Linux-CI.patch + git apply --ignore-space-change --ignore-whitespace --3way ../../patches/faiss/0003-Enable-precomp-table-to-be-shared-ivfpq.patch + rm ../../patches/faiss/0003-Enable-precomp-table-to-be-shared-ivfpq.patch cd ../nmslib git apply --ignore-space-change --ignore-whitespace --3way ../../patches/nmslib/0001-Initialize-maxlevel-during-add-from-enterpoint-level.patch rm ../../patches/nmslib/0001-Initialize-maxlevel-during-add-from-enterpoint-level.patch diff --git a/jni/CMakeLists.txt b/jni/CMakeLists.txt index cec6bf54f..574a04c9d 100644 --- a/jni/CMakeLists.txt +++ b/jni/CMakeLists.txt @@ -168,13 +168,13 @@ if (${CONFIG_FAISS} STREQUAL ON OR ${CONFIG_ALL} STREQUAL ON OR ${CONFIG_TEST} S endif () # Check if patch exist, this is to skip git apply during CI build. See CI.yml with ubuntu. - find_path(PATCH_FILE NAMES 0001-Custom-patch-to-support-multi-vector.patch PATHS ${CMAKE_CURRENT_SOURCE_DIR}/patches/faiss NO_DEFAULT_PATH) + find_path(PATCH_FILE NAMES 0001-Custom-patch-to-support-multi-vector.patch 0003-Enable-precomp-table-to-be-shared-ivfpq.patch PATHS ${CMAKE_CURRENT_SOURCE_DIR}/patches/faiss NO_DEFAULT_PATH) # If it exists, apply patches if (EXISTS ${PATCH_FILE}) message(STATUS "Applying custom patches.") execute_process(COMMAND git apply --ignore-space-change --ignore-whitespace --3way ${CMAKE_CURRENT_SOURCE_DIR}/patches/faiss/0001-Custom-patch-to-support-multi-vector.patch WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/external/faiss ERROR_VARIABLE ERROR_MSG RESULT_VARIABLE RESULT_CODE) - + execute_process(COMMAND git apply --ignore-space-change --ignore-whitespace --3way ${CMAKE_CURRENT_SOURCE_DIR}/patches/faiss/0003-Enable-precomp-table-to-be-shared-ivfpq.patch WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/external/faiss ERROR_VARIABLE ERROR_MSG RESULT_VARIABLE RESULT_CODE) if(RESULT_CODE) message(FATAL_ERROR "Failed to apply patch:\n${ERROR_MSG}") endif() diff --git a/jni/patches/faiss/0003-Enable-precomp-table-to-be-shared-ivfpq.patch b/jni/patches/faiss/0003-Enable-precomp-table-to-be-shared-ivfpq.patch new file mode 100644 index 000000000..dfc5099aa --- /dev/null +++ b/jni/patches/faiss/0003-Enable-precomp-table-to-be-shared-ivfpq.patch @@ -0,0 +1,512 @@ +From c5ca07299b427dedafc738b98bd20f8f286f6783 Mon Sep 17 00:00:00 2001 +From: John Mazanec +Date: Wed, 21 Feb 2024 15:34:15 -0800 +Subject: [PATCH] Enable precomp table to be shared ivfpq + +Changes IVFPQ and IVFPQFastScan indices to be able to share the +precomputed table amongst other instances. Switches var to a pointer and +add necessary functions to set them correctly. + +Adds a tests to validate the behavior. + +Signed-off-by: John Mazanec +--- + faiss/IndexIVFPQ.cpp | 47 +++++++- + faiss/IndexIVFPQ.h | 16 ++- + faiss/IndexIVFPQFastScan.cpp | 47 ++++++-- + faiss/IndexIVFPQFastScan.h | 11 +- + tests/CMakeLists.txt | 1 + + tests/test_disable_pq_sdc_tables.cpp | 4 +- + tests/test_ivfpq_share_table.cpp | 173 +++++++++++++++++++++++++++ + 7 files changed, 284 insertions(+), 15 deletions(-) + create mode 100644 tests/test_ivfpq_share_table.cpp + +diff --git a/faiss/IndexIVFPQ.cpp b/faiss/IndexIVFPQ.cpp +index 0b7f4d05..07bc7e83 100644 +--- a/faiss/IndexIVFPQ.cpp ++++ b/faiss/IndexIVFPQ.cpp +@@ -59,6 +59,29 @@ IndexIVFPQ::IndexIVFPQ( + polysemous_training = nullptr; + do_polysemous_training = false; + polysemous_ht = 0; ++ precomputed_table = new AlignedTable(); ++ owns_precomputed_table = true; ++} ++ ++IndexIVFPQ::IndexIVFPQ(const IndexIVFPQ& orig) : IndexIVF(orig), pq(orig.pq) { ++ code_size = orig.pq.code_size; ++ invlists->code_size = code_size; ++ is_trained = orig.is_trained; ++ by_residual = orig.by_residual; ++ use_precomputed_table = orig.use_precomputed_table; ++ scan_table_threshold = orig.scan_table_threshold; ++ ++ polysemous_training = orig.polysemous_training; ++ do_polysemous_training = orig.do_polysemous_training; ++ polysemous_ht = orig.polysemous_ht; ++ precomputed_table = new AlignedTable(*orig.precomputed_table); ++ owns_precomputed_table = true; ++} ++ ++IndexIVFPQ::~IndexIVFPQ() { ++ if (owns_precomputed_table) { ++ delete precomputed_table; ++ } + } + + /**************************************************************** +@@ -466,11 +489,23 @@ void IndexIVFPQ::precompute_table() { + use_precomputed_table, + quantizer, + pq, +- precomputed_table, ++ *precomputed_table, + by_residual, + verbose); + } + ++void IndexIVFPQ::set_precomputed_table( ++ AlignedTable* _precompute_table, ++ int _use_precomputed_table) { ++ // Clean up old pre-computed table ++ if (owns_precomputed_table) { ++ delete precomputed_table; ++ } ++ owns_precomputed_table = false; ++ precomputed_table = _precompute_table; ++ use_precomputed_table = _use_precomputed_table; ++} ++ + namespace { + + #define TIC t0 = get_cycles() +@@ -650,7 +685,7 @@ struct QueryTables { + + fvec_madd( + pq.M * pq.ksub, +- ivfpq.precomputed_table.data() + key * pq.ksub * pq.M, ++ ivfpq.precomputed_table->data() + key * pq.ksub * pq.M, + -2.0, + sim_table_2, + sim_table); +@@ -679,7 +714,7 @@ struct QueryTables { + k >>= cpq.nbits; + + // get corresponding table +- const float* pc = ivfpq.precomputed_table.data() + ++ const float* pc = ivfpq.precomputed_table->data() + + (ki * pq.M + cm * Mf) * pq.ksub; + + if (polysemous_ht == 0) { +@@ -709,7 +744,7 @@ struct QueryTables { + dis0 = coarse_dis; + + const float* s = +- ivfpq.precomputed_table.data() + key * pq.ksub * pq.M; ++ ivfpq.precomputed_table->data() + key * pq.ksub * pq.M; + for (int m = 0; m < pq.M; m++) { + sim_table_ptrs[m] = s; + s += pq.ksub; +@@ -729,7 +764,7 @@ struct QueryTables { + int ki = k & ((uint64_t(1) << cpq.nbits) - 1); + k >>= cpq.nbits; + +- const float* pc = ivfpq.precomputed_table.data() + ++ const float* pc = ivfpq.precomputed_table->data() + + (ki * pq.M + cm * Mf) * pq.ksub; + + for (int m = m0; m < m0 + Mf; m++) { +@@ -1346,6 +1381,8 @@ IndexIVFPQ::IndexIVFPQ() { + do_polysemous_training = false; + polysemous_ht = 0; + polysemous_training = nullptr; ++ precomputed_table = new AlignedTable(); ++ owns_precomputed_table = true; + } + + struct CodeCmp { +diff --git a/faiss/IndexIVFPQ.h b/faiss/IndexIVFPQ.h +index d5d21da4..850bbe44 100644 +--- a/faiss/IndexIVFPQ.h ++++ b/faiss/IndexIVFPQ.h +@@ -48,7 +48,8 @@ struct IndexIVFPQ : IndexIVF { + + /// if use_precompute_table + /// size nlist * pq.M * pq.ksub +- AlignedTable precomputed_table; ++ bool owns_precomputed_table; ++ AlignedTable* precomputed_table; + + IndexIVFPQ( + Index* quantizer, +@@ -58,6 +59,10 @@ struct IndexIVFPQ : IndexIVF { + size_t nbits_per_idx, + MetricType metric = METRIC_L2); + ++ IndexIVFPQ(const IndexIVFPQ& orig); ++ ++ ~IndexIVFPQ(); ++ + void encode_vectors( + idx_t n, + const float* x, +@@ -139,6 +144,15 @@ struct IndexIVFPQ : IndexIVF { + /// build precomputed table + void precompute_table(); + ++ /** ++ * Initialize the precomputed table ++ * @param precompute_table ++ * @param _use_precomputed_table ++ */ ++ void set_precomputed_table( ++ AlignedTable* precompute_table, ++ int _use_precomputed_table); ++ + IndexIVFPQ(); + }; + +diff --git a/faiss/IndexIVFPQFastScan.cpp b/faiss/IndexIVFPQFastScan.cpp +index d069db13..09a335ff 100644 +--- a/faiss/IndexIVFPQFastScan.cpp ++++ b/faiss/IndexIVFPQFastScan.cpp +@@ -46,6 +46,8 @@ IndexIVFPQFastScan::IndexIVFPQFastScan( + : IndexIVFFastScan(quantizer, d, nlist, 0, metric), pq(d, M, nbits) { + by_residual = false; // set to false by default because it's faster + ++ precomputed_table = new AlignedTable(); ++ owns_precomputed_table = true; + init_fastscan(M, nbits, nlist, metric, bbs); + } + +@@ -53,6 +55,17 @@ IndexIVFPQFastScan::IndexIVFPQFastScan() { + by_residual = false; + bbs = 0; + M2 = 0; ++ precomputed_table = new AlignedTable(); ++ owns_precomputed_table = true; ++} ++ ++IndexIVFPQFastScan::IndexIVFPQFastScan(const IndexIVFPQFastScan& orig) ++ : IndexIVFFastScan(orig), pq(orig.pq) { ++ by_residual = orig.by_residual; ++ bbs = orig.bbs; ++ M2 = orig.M2; ++ precomputed_table = new AlignedTable(*orig.precomputed_table); ++ owns_precomputed_table = true; + } + + IndexIVFPQFastScan::IndexIVFPQFastScan(const IndexIVFPQ& orig, int bbs) +@@ -71,13 +84,15 @@ IndexIVFPQFastScan::IndexIVFPQFastScan(const IndexIVFPQ& orig, int bbs) + ntotal = orig.ntotal; + is_trained = orig.is_trained; + nprobe = orig.nprobe; ++ precomputed_table = new AlignedTable(); ++ owns_precomputed_table = true; + +- precomputed_table.resize(orig.precomputed_table.size()); ++ precomputed_table->resize(orig.precomputed_table->size()); + +- if (precomputed_table.nbytes() > 0) { +- memcpy(precomputed_table.get(), +- orig.precomputed_table.data(), +- precomputed_table.nbytes()); ++ if (precomputed_table->nbytes() > 0) { ++ memcpy(precomputed_table->get(), ++ orig.precomputed_table->data(), ++ precomputed_table->nbytes()); + } + + for (size_t i = 0; i < nlist; i++) { +@@ -102,6 +117,12 @@ IndexIVFPQFastScan::IndexIVFPQFastScan(const IndexIVFPQ& orig, int bbs) + orig_invlists = orig.invlists; + } + ++IndexIVFPQFastScan::~IndexIVFPQFastScan() { ++ if (owns_precomputed_table) { ++ delete precomputed_table; ++ } ++} ++ + /********************************************************* + * Training + *********************************************************/ +@@ -127,11 +148,23 @@ void IndexIVFPQFastScan::precompute_table() { + use_precomputed_table, + quantizer, + pq, +- precomputed_table, ++ *precomputed_table, + by_residual, + verbose); + } + ++void IndexIVFPQFastScan::set_precomputed_table( ++ AlignedTable* _precompute_table, ++ int _use_precomputed_table) { ++ // Clean up old pre-computed table ++ if (owns_precomputed_table) { ++ delete precomputed_table; ++ } ++ owns_precomputed_table = false; ++ precomputed_table = _precompute_table; ++ use_precomputed_table = _use_precomputed_table; ++} ++ + /********************************************************* + * Code management functions + *********************************************************/ +@@ -229,7 +262,7 @@ void IndexIVFPQFastScan::compute_LUT( + if (cij >= 0) { + fvec_madd_simd( + dim12, +- precomputed_table.get() + cij * dim12, ++ precomputed_table->get() + cij * dim12, + -2, + ip_table.get() + i * dim12, + tab); +diff --git a/faiss/IndexIVFPQFastScan.h b/faiss/IndexIVFPQFastScan.h +index 00dd2f11..91f35a6e 100644 +--- a/faiss/IndexIVFPQFastScan.h ++++ b/faiss/IndexIVFPQFastScan.h +@@ -38,7 +38,8 @@ struct IndexIVFPQFastScan : IndexIVFFastScan { + /// precomputed tables management + int use_precomputed_table = 0; + /// if use_precompute_table size (nlist, pq.M, pq.ksub) +- AlignedTable precomputed_table; ++ bool owns_precomputed_table; ++ AlignedTable* precomputed_table; + + IndexIVFPQFastScan( + Index* quantizer, +@@ -51,6 +52,10 @@ struct IndexIVFPQFastScan : IndexIVFFastScan { + + IndexIVFPQFastScan(); + ++ IndexIVFPQFastScan(const IndexIVFPQFastScan& orig); ++ ++ ~IndexIVFPQFastScan(); ++ + // built from an IndexIVFPQ + explicit IndexIVFPQFastScan(const IndexIVFPQ& orig, int bbs = 32); + +@@ -60,6 +65,10 @@ struct IndexIVFPQFastScan : IndexIVFFastScan { + + /// build precomputed table, possibly updating use_precomputed_table + void precompute_table(); ++ /// Pass in externally a precomputed ++ void set_precomputed_table( ++ AlignedTable* precompute_table, ++ int _use_precomputed_table); + + /// same as the regular IVFPQ encoder. The codes are not reorganized by + /// blocks a that point +diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt +index 9017edc5..0889bf72 100644 +--- a/tests/CMakeLists.txt ++++ b/tests/CMakeLists.txt +@@ -33,6 +33,7 @@ set(FAISS_TEST_SRC + test_partitioning.cpp + test_fastscan_perf.cpp + test_disable_pq_sdc_tables.cpp ++ test_ivfpq_share_table.cpp + ) + + add_executable(faiss_test ${FAISS_TEST_SRC}) +diff --git a/tests/test_disable_pq_sdc_tables.cpp b/tests/test_disable_pq_sdc_tables.cpp +index b211a5c4..a27973d5 100644 +--- a/tests/test_disable_pq_sdc_tables.cpp ++++ b/tests/test_disable_pq_sdc_tables.cpp +@@ -15,7 +15,9 @@ + #include "faiss/index_io.h" + #include "test_util.h" + +-pthread_mutex_t temp_file_mutex = PTHREAD_MUTEX_INITIALIZER; ++namespace { ++ pthread_mutex_t temp_file_mutex = PTHREAD_MUTEX_INITIALIZER; ++} + + TEST(IO, TestReadHNSWPQ_whenSDCDisabledFlagPassed_thenDisableSDCTable) { + Tempfilename index_filename(&temp_file_mutex, "/tmp/faiss_TestReadHNSWPQ"); +diff --git a/tests/test_ivfpq_share_table.cpp b/tests/test_ivfpq_share_table.cpp +new file mode 100644 +index 00000000..f827315d +--- /dev/null ++++ b/tests/test_ivfpq_share_table.cpp +@@ -0,0 +1,173 @@ ++/** ++ * Copyright (c) Facebook, Inc. and its affiliates. ++ * ++ * This source code is licensed under the MIT license found in the ++ * LICENSE file in the root directory of this source tree. ++ */ ++ ++#include ++ ++#include ++ ++#include "faiss/Index.h" ++#include "faiss/IndexHNSW.h" ++#include "faiss/IndexIVFPQFastScan.h" ++#include "faiss/index_factory.h" ++#include "faiss/index_io.h" ++#include "test_util.h" ++ ++namespace { ++ pthread_mutex_t temp_file_mutex = PTHREAD_MUTEX_INITIALIZER; ++} ++ ++std::vector generate_data( ++ int d, ++ int n, ++ std::default_random_engine rng, ++ std::uniform_real_distribution u) { ++ std::vector vectors(n * d); ++ for (size_t i = 0; i < n * d; i++) { ++ vectors[i] = u(rng); ++ } ++ return vectors; ++} ++ ++void assert_float_vectors_almost_equal( ++ std::vector a, ++ std::vector b) { ++ float margin = 0.000001; ++ ASSERT_EQ(a.size(), b.size()); ++ for (int i = 0; i < a.size(); i++) { ++ ASSERT_NEAR(a[i], b[i], margin); ++ } ++} ++ ++/// Test case test precomputed table sharing for IVFPQ indices. ++template /// T represents class cast to use for index ++void test_ivfpq_table_sharing( ++ const std::string& index_description, ++ const std::string& filename, ++ faiss::MetricType metric) { ++ // Setup the index: ++ // 1. Build an index ++ // 2. ingest random data ++ // 3. serialize to disk ++ int d = 32, n = 1000; ++ std::default_random_engine rng( ++ std::chrono::system_clock::now().time_since_epoch().count()); ++ std::uniform_real_distribution u(0, 100); ++ ++ std::vector index_vectors = generate_data(d, n, rng, u); ++ std::vector query_vectors = generate_data(d, n, rng, u); ++ ++ Tempfilename index_filename(&temp_file_mutex, filename); ++ { ++ std::unique_ptr index_writer( ++ faiss::index_factory(d, index_description.c_str(), metric)); ++ ++ index_writer->train(n, index_vectors.data()); ++ index_writer->add(n, index_vectors.data()); ++ faiss::write_index(index_writer.get(), index_filename.c_str()); ++ } ++ ++ // Load index from disk. Confirm that the sdc table is equal to 0 when ++ // disable sdc is set ++ std::unique_ptr> sharedAlignedTable( ++ new faiss::AlignedTable()); ++ int shared_use_precomputed_table = 0; ++ int k = 10; ++ std::vector distances_test_a(k * n); ++ std::vector labels_test_a(k * n); ++ { ++ std::vector distances_baseline(k * n); ++ std::vector labels_baseline(k * n); ++ ++ std::unique_ptr index_read_pq_table_enabled( ++ dynamic_cast(faiss::read_index( ++ index_filename.c_str(), faiss::IO_FLAG_READ_ONLY))); ++ std::unique_ptr index_read_pq_table_disabled( ++ dynamic_cast(faiss::read_index( ++ index_filename.c_str(), ++ faiss::IO_FLAG_READ_ONLY | ++ faiss::IO_FLAG_SKIP_PRECOMPUTE_TABLE))); ++ faiss::initialize_IVFPQ_precomputed_table( ++ shared_use_precomputed_table, ++ index_read_pq_table_disabled->quantizer, ++ index_read_pq_table_disabled->pq, ++ *sharedAlignedTable, ++ index_read_pq_table_disabled->by_residual, ++ index_read_pq_table_disabled->verbose); ++ index_read_pq_table_disabled->set_precomputed_table( ++ sharedAlignedTable.get(), shared_use_precomputed_table); ++ ++ ASSERT_TRUE(index_read_pq_table_enabled->owns_precomputed_table); ++ ASSERT_FALSE(index_read_pq_table_disabled->owns_precomputed_table); ++ index_read_pq_table_enabled->search( ++ n, ++ query_vectors.data(), ++ k, ++ distances_baseline.data(), ++ labels_baseline.data()); ++ index_read_pq_table_disabled->search( ++ n, ++ query_vectors.data(), ++ k, ++ distances_test_a.data(), ++ labels_test_a.data()); ++ ++ assert_float_vectors_almost_equal(distances_baseline, distances_test_a); ++ ASSERT_EQ(labels_baseline, labels_test_a); ++ } ++ ++ // The precomputed table should only be set for L2 metric type ++ if (metric == faiss::METRIC_L2) { ++ ASSERT_EQ(shared_use_precomputed_table, 1); ++ } else { ++ ASSERT_EQ(shared_use_precomputed_table, 0); ++ } ++ ++ // At this point, the original has gone out of scope, the destructor has ++ // been called. Confirm that initializing a new index from the table ++ // preserves the functionality. ++ { ++ std::vector distances_test_b(k * n); ++ std::vector labels_test_b(k * n); ++ ++ std::unique_ptr index_read_pq_table_disabled( ++ dynamic_cast(faiss::read_index( ++ index_filename.c_str(), ++ faiss::IO_FLAG_READ_ONLY | ++ faiss::IO_FLAG_SKIP_PRECOMPUTE_TABLE))); ++ index_read_pq_table_disabled->set_precomputed_table( ++ sharedAlignedTable.get(), shared_use_precomputed_table); ++ ASSERT_FALSE(index_read_pq_table_disabled->owns_precomputed_table); ++ index_read_pq_table_disabled->search( ++ n, ++ query_vectors.data(), ++ k, ++ distances_test_b.data(), ++ labels_test_b.data()); ++ assert_float_vectors_almost_equal(distances_test_a, distances_test_b); ++ ASSERT_EQ(labels_test_a, labels_test_b); ++ } ++} ++ ++TEST(TestIVFPQTableSharing, L2) { ++ test_ivfpq_table_sharing( ++ "IVF16,PQ8x4", "/tmp/ivfpql2", faiss::METRIC_L2); ++} ++ ++TEST(TestIVFPQTableSharing, IP) { ++ test_ivfpq_table_sharing( ++ "IVF16,PQ8x4", "/tmp/ivfpqip", faiss::METRIC_INNER_PRODUCT); ++} ++ ++TEST(TestIVFPQTableSharing, FastScanL2) { ++ test_ivfpq_table_sharing( ++ "IVF16,PQ8x4fsr", "/tmp/ivfpqfsl2", faiss::METRIC_L2); ++} ++ ++TEST(TestIVFPQTableSharing, FastScanIP) { ++ test_ivfpq_table_sharing( ++ "IVF16,PQ8x4fsr", "/tmp/ivfpqfsip", faiss::METRIC_INNER_PRODUCT); ++} +-- +2.39.3 (Apple Git-145) + From 35a4eb40fba9498a29fa9f546a2d5fae8347788e Mon Sep 17 00:00:00 2001 From: John Mazanec Date: Thu, 14 Mar 2024 15:13:48 -0700 Subject: [PATCH 222/416] Add ability to share IVFPQ-l2 state in JNI (#1529) Adds a set of JNI functions that allows IVFPQ-l2 dynamically precomputed tables to be shared amongst indices at load time that use the same model. In addition, refactors JNIService interface to take KNNEngine as param instead of as string In addition, add unit tests to confirm that the shared state operates as expected. Signed-off-by: John Mazanec --- jni/include/faiss_wrapper.h | 15 + .../org_opensearch_knn_jni_FaissService.h | 38 ++- jni/src/faiss_wrapper.cpp | 101 +++++- .../org_opensearch_knn_jni_FaissService.cpp | 42 +++ jni/tests/faiss_wrapper_test.cpp | 123 +++++++ .../KNN80Codec/KNN80DocValuesConsumer.java | 4 +- .../index/memory/NativeMemoryAllocation.java | 2 +- .../memory/NativeMemoryLoadStrategy.java | 2 +- .../opensearch/knn/index/query/KNNWeight.java | 2 +- .../org/opensearch/knn/jni/FaissService.java | 31 ++ .../org/opensearch/knn/jni/JNIService.java | 153 ++++++--- .../opensearch/knn/training/TrainingJob.java | 2 +- .../index/KNNCreateIndexFromModelTests.java | 2 +- .../KNN80DocValuesConsumerTests.java | 2 +- .../knn/index/codec/KNNCodecTestCase.java | 2 +- .../knn/index/codec/KNNCodecTestUtil.java | 10 +- .../memory/NativeMemoryAllocationTests.java | 4 +- .../memory/NativeMemoryLoadStrategyTests.java | 4 +- .../knn/index/query/KNNWeightTests.java | 19 +- .../opensearch/knn/jni/JNIServiceTests.java | 313 +++++++++++++----- .../knn/training/TrainingJobTests.java | 2 +- 21 files changed, 711 insertions(+), 162 deletions(-) diff --git a/jni/include/faiss_wrapper.h b/jni/include/faiss_wrapper.h index e99cdafb2..2629eea43 100644 --- a/jni/include/faiss_wrapper.h +++ b/jni/include/faiss_wrapper.h @@ -33,6 +33,18 @@ namespace knn_jni { // Return a pointer to the loaded index jlong LoadIndex(knn_jni::JNIUtilInterface * jniUtil, JNIEnv * env, jstring indexPathJ); + // Check if a loaded index requires shared state + bool IsSharedIndexStateRequired(jlong indexPointerJ); + + // Initializes the shared index state from an index. Note, this will not set the state for + // the index pointed to by indexPointerJ. To set it, SetSharedIndexState needs to be called. + // + // Return a pointer to the shared index state + jlong InitSharedIndexState(jlong indexPointerJ); + + // Sets the sharedIndexState for an index + void SetSharedIndexState(jlong indexPointerJ, jlong shareIndexStatePointerJ); + // Execute a query against the index located in memory at indexPointerJ. // // Return an array of KNNQueryResults @@ -49,6 +61,9 @@ namespace knn_jni { // Free the index located in memory at indexPointerJ void Free(jlong indexPointer); + // Free shared index state in memory at shareIndexStatePointerJ + void FreeSharedIndexState(jlong shareIndexStatePointerJ); + // Perform initilization operations for the library void InitLibrary(); diff --git a/jni/include/org_opensearch_knn_jni_FaissService.h b/jni/include/org_opensearch_knn_jni_FaissService.h index 3b649c227..64a858f84 100644 --- a/jni/include/org_opensearch_knn_jni_FaissService.h +++ b/jni/include/org_opensearch_knn_jni_FaissService.h @@ -42,18 +42,42 @@ JNIEXPORT void JNICALL Java_org_opensearch_knn_jni_FaissService_createIndexFromT JNIEXPORT jlong JNICALL Java_org_opensearch_knn_jni_FaissService_loadIndex (JNIEnv *, jclass, jstring); +/* + * Class: org_opensearch_knn_jni_FaissService + * Method: isSharedIndexStateRequired + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_org_opensearch_knn_jni_FaissService_isSharedIndexStateRequired + (JNIEnv *, jclass, jlong); + +/* + * Class: org_opensearch_knn_jni_FaissService + * Method: initSharedIndexState + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL Java_org_opensearch_knn_jni_FaissService_initSharedIndexState + (JNIEnv *, jclass, jlong); + +/* + * Class: org_opensearch_knn_jni_FaissService + * Method: setSharedIndexState + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL Java_org_opensearch_knn_jni_FaissService_setSharedIndexState + (JNIEnv *, jclass, jlong, jlong); + /* * Class: org_opensearch_knn_jni_FaissService * Method: queryIndex - * Signature: (J[FI)[Lorg/opensearch/knn/index/query/KNNQueryResult; + * Signature: (J[FI[I)[Lorg/opensearch/knn/index/query/KNNQueryResult; */ JNIEXPORT jobjectArray JNICALL Java_org_opensearch_knn_jni_FaissService_queryIndex (JNIEnv *, jclass, jlong, jfloatArray, jint, jintArray); /* * Class: org_opensearch_knn_jni_FaissService - * Method: queryIndex_WithFilter - * Signature: (J[FI[J)[Lorg/opensearch/knn/index/query/KNNQueryResult; + * Method: queryIndexWithFilter + * Signature: (J[FI[JI[I)[Lorg/opensearch/knn/index/query/KNNQueryResult; */ JNIEXPORT jobjectArray JNICALL Java_org_opensearch_knn_jni_FaissService_queryIndexWithFilter (JNIEnv *, jclass, jlong, jfloatArray, jint, jlongArray, jint, jintArray); @@ -66,6 +90,14 @@ JNIEXPORT jobjectArray JNICALL Java_org_opensearch_knn_jni_FaissService_queryInd JNIEXPORT void JNICALL Java_org_opensearch_knn_jni_FaissService_free (JNIEnv *, jclass, jlong); +/* + * Class: org_opensearch_knn_jni_FaissService + * Method: freeSharedIndexState + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_org_opensearch_knn_jni_FaissService_freeSharedIndexState + (JNIEnv *, jclass, jlong); + /* * Class: org_opensearch_knn_jni_FaissService * Method: initLibrary diff --git a/jni/src/faiss_wrapper.cpp b/jni/src/faiss_wrapper.cpp index 433507dff..a7075740e 100644 --- a/jni/src/faiss_wrapper.cpp +++ b/jni/src/faiss_wrapper.cpp @@ -21,6 +21,7 @@ #include "faiss/MetaIndexes.h" #include "faiss/Index.h" #include "faiss/impl/IDSelector.h" +#include "faiss/IndexIVFPQ.h" #include #include @@ -73,6 +74,13 @@ void buildFilterIdsBitMap(const int* filterIds, int filterIdsLength, uint8_t* bi std::unique_ptr buildIDGrouperBitmap(knn_jni::JNIUtilInterface * jniUtil, JNIEnv *env, jintArray parentIdsJ, std::vector* bitmap); +// Check if a loaded index is an IVFPQ index with l2 space type +bool isIndexIVFPQL2(faiss::Index * index); + +// Gets IVFPQ index from a faiss index. For faiss, we wrap the index in the type +// IndexIDMap which has member that will point to underlying index that stores the data +faiss::IndexIVFPQ * extractIVFPQIndex(faiss::Index * index); + void knn_jni::faiss_wrapper::CreateIndex(knn_jni::JNIUtilInterface * jniUtil, JNIEnv * env, jintArray idsJ, jobjectArray vectorsJ, jstring indexPathJ, jobject parametersJ) { @@ -214,10 +222,60 @@ jlong knn_jni::faiss_wrapper::LoadIndex(knn_jni::JNIUtilInterface * jniUtil, JNI std::string indexPathCpp(jniUtil->ConvertJavaStringToCppString(env, indexPathJ)); // Skipping IO_FLAG_PQ_SKIP_SDC_TABLE because the index is read only and the sdc table is only used during ingestion - faiss::Index* indexReader = faiss::read_index(indexPathCpp.c_str(), faiss::IO_FLAG_READ_ONLY | faiss::IO_FLAG_PQ_SKIP_SDC_TABLE); + // Skipping IO_PRECOMPUTE_TABLE because it is only needed for IVFPQ-l2 and it leads to high memory consumption if + // done for each segment. Instead, we will set it later on with `setSharedIndexState` + faiss::Index* indexReader = faiss::read_index(indexPathCpp.c_str(), faiss::IO_FLAG_READ_ONLY | faiss::IO_FLAG_PQ_SKIP_SDC_TABLE | faiss::IO_FLAG_SKIP_PRECOMPUTE_TABLE); return (jlong) indexReader; } +bool knn_jni::faiss_wrapper::IsSharedIndexStateRequired(jlong indexPointerJ) { + auto * index = reinterpret_cast(indexPointerJ); + return isIndexIVFPQL2(index); +} + +jlong knn_jni::faiss_wrapper::InitSharedIndexState(jlong indexPointerJ) { + auto * index = reinterpret_cast(indexPointerJ); + if (!isIndexIVFPQL2(index)) { + throw std::runtime_error("Unable to init shared index state from index. index is not of type IVFPQ-l2"); + } + + auto * indexIVFPQ = extractIVFPQIndex(index); + int use_precomputed_table = 0; + auto * sharedMemoryAddress = new faiss::AlignedTable(); + faiss::initialize_IVFPQ_precomputed_table( + use_precomputed_table, + indexIVFPQ->quantizer, + indexIVFPQ->pq, + *sharedMemoryAddress, + indexIVFPQ->by_residual, + indexIVFPQ->verbose); + return (jlong) sharedMemoryAddress; +} + +void knn_jni::faiss_wrapper::SetSharedIndexState(jlong indexPointerJ, jlong shareIndexStatePointerJ) { + auto * index = reinterpret_cast(indexPointerJ); + if (!isIndexIVFPQL2(index)) { + throw std::runtime_error("Unable to set shared index state from index. index is not of type IVFPQ-l2"); + } + auto * indexIVFPQ = extractIVFPQIndex(index); + + //TODO: Currently, the only shared state is that of the AlignedTable associated with + // IVFPQ-l2 index type (see https://github.com/opensearch-project/k-NN/issues/1507). In the future, + // this will be generalized and more information will be needed to determine the shared type. But, until then, + // this is fine. + auto *alignTable = reinterpret_cast*>(shareIndexStatePointerJ); + // In faiss, usePrecomputedTable can have a couple different values: + // -1 -> dont use the table + // 0 -> tell initialize_IVFPQ_precomputed_table to select the best value and change the value + // 1 -> default behavior + // 2 -> Index is of type "MultiIndexQuantizer" + // This index will be of type IndexIVFPQ always. We never create "MultiIndexQuantizer". So, the value we + // want is 1. + // (ref: https://github.com/facebookresearch/faiss/blob/v1.8.0/faiss/IndexIVFPQ.cpp#L383-L410) + int usePrecomputedTable = 1; + indexIVFPQ->set_precomputed_table(alignTable, usePrecomputedTable); +} + jobjectArray knn_jni::faiss_wrapper::QueryIndex(knn_jni::JNIUtilInterface * jniUtil, JNIEnv * env, jlong indexPointerJ, jfloatArray queryVectorJ, jint kJ, jintArray parentIdsJ) { return knn_jni::faiss_wrapper::QueryIndex_WithFilter(jniUtil, env, indexPointerJ, queryVectorJ, kJ, nullptr, 0, parentIdsJ); @@ -337,6 +395,15 @@ void knn_jni::faiss_wrapper::Free(jlong indexPointer) { delete indexWrapper; } +void knn_jni::faiss_wrapper::FreeSharedIndexState(jlong shareIndexStatePointerJ) { + //TODO: Currently, the only shared state is that of the AlignedTable associated with + // IVFPQ-l2 index type (see https://github.com/opensearch-project/k-NN/issues/1507). In the future, + // this will be generalized and more information will be needed to determine the shared type. But, until then, + // this is fine. + auto *alignTable = reinterpret_cast*>(shareIndexStatePointerJ); + delete alignTable; +} + void knn_jni::faiss_wrapper::InitLibrary() { //set thread 1 cause ES has Search thread //TODO make it different at search and write @@ -469,3 +536,35 @@ std::unique_ptr buildIDGrouperBitmap(knn_jni::JNIUtilInt jniUtil->ReleaseIntArrayElements(env, parentIdsJ, parentIdsArray, JNI_ABORT); return idGrouper; } + +bool isIndexIVFPQL2(faiss::Index * index) { + faiss::Index * candidateIndex = index; + // Unwrap the index if it is wrapped in IndexIDMap. Dynamic cast will "Safely converts pointers and references to + // classes up, down, and sideways along the inheritance hierarchy." It will return a nullptr if the + // cast fails. (ref: https://en.cppreference.com/w/cpp/language/dynamic_cast) + if (auto indexIDMap = dynamic_cast(index)) { + candidateIndex = indexIDMap->index; + } + + // Check if the index is of type IndexIVFPQ. If so, confirm its metric type is + // l2. + if (auto indexIVFPQ = dynamic_cast(candidateIndex)) { + return faiss::METRIC_L2 == indexIVFPQ->metric_type; + } + + return false; +} + +faiss::IndexIVFPQ * extractIVFPQIndex(faiss::Index * index) { + faiss::Index * candidateIndex = index; + if (auto indexIDMap = dynamic_cast(index)) { + candidateIndex = indexIDMap->index; + } + + faiss::IndexIVFPQ * indexIVFPQ; + if ((indexIVFPQ = dynamic_cast(candidateIndex))) { + return indexIVFPQ; + } + + throw std::runtime_error("Unable to extract IVFPQ index. IVFPQ index not present."); +} diff --git a/jni/src/org_opensearch_knn_jni_FaissService.cpp b/jni/src/org_opensearch_knn_jni_FaissService.cpp index e8e761ad7..c81f23a62 100644 --- a/jni/src/org_opensearch_knn_jni_FaissService.cpp +++ b/jni/src/org_opensearch_knn_jni_FaissService.cpp @@ -75,6 +75,38 @@ JNIEXPORT jlong JNICALL Java_org_opensearch_knn_jni_FaissService_loadIndex(JNIEn return NULL; } +JNIEXPORT jboolean JNICALL Java_org_opensearch_knn_jni_FaissService_isSharedIndexStateRequired + (JNIEnv * env, jclass cls, jlong indexPointerJ) +{ + try { + return knn_jni::faiss_wrapper::IsSharedIndexStateRequired(indexPointerJ); + } catch (...) { + jniUtil.CatchCppExceptionAndThrowJava(env); + } + return NULL; +} + +JNIEXPORT jlong JNICALL Java_org_opensearch_knn_jni_FaissService_initSharedIndexState + (JNIEnv * env, jclass cls, jlong indexPointerJ) +{ + try { + return knn_jni::faiss_wrapper::InitSharedIndexState(indexPointerJ); + } catch (...) { + jniUtil.CatchCppExceptionAndThrowJava(env); + } + return NULL; +} + +JNIEXPORT void JNICALL Java_org_opensearch_knn_jni_FaissService_setSharedIndexState + (JNIEnv * env, jclass cls, jlong indexPointerJ, jlong shareIndexStatePointerJ) +{ + try { + knn_jni::faiss_wrapper::SetSharedIndexState(indexPointerJ, shareIndexStatePointerJ); + } catch (...) { + jniUtil.CatchCppExceptionAndThrowJava(env); + } +} + JNIEXPORT jobjectArray JNICALL Java_org_opensearch_knn_jni_FaissService_queryIndex(JNIEnv * env, jclass cls, jlong indexPointerJ, jfloatArray queryVectorJ, jint kJ, jintArray parentIdsJ) @@ -109,6 +141,16 @@ JNIEXPORT void JNICALL Java_org_opensearch_knn_jni_FaissService_free(JNIEnv * en } } +JNIEXPORT void JNICALL Java_org_opensearch_knn_jni_FaissService_freeSharedIndexState + (JNIEnv * env, jclass cls, jlong shareIndexStatePointerJ) +{ + try { + knn_jni::faiss_wrapper::FreeSharedIndexState(shareIndexStatePointerJ); + } catch (...) { + jniUtil.CatchCppExceptionAndThrowJava(env); + } +} + JNIEXPORT void JNICALL Java_org_opensearch_knn_jni_FaissService_initLibrary(JNIEnv * env, jclass cls) { try { diff --git a/jni/tests/faiss_wrapper_test.cpp b/jni/tests/faiss_wrapper_test.cpp index 03ae38dfa..2b1684cfb 100644 --- a/jni/tests/faiss_wrapper_test.cpp +++ b/jni/tests/faiss_wrapper_test.cpp @@ -18,6 +18,7 @@ #include "jni_util.h" #include "test_util.h" #include "faiss/IndexHNSW.h" +#include "faiss/IndexIVFPQ.h" using ::testing::NiceMock; using ::testing::Return; @@ -204,6 +205,37 @@ TEST(FaissLoadIndexTest, HNSWPQDisableSdcTable) { ASSERT_EQ(0, pqIndex->pq.sdc_table.size()); } +TEST(FaissLoadIndexTest, IVFPQDisablePrecomputeTable) { + faiss::idx_t numIds = 256; + int dim = 2; + std::vector ids = test_util::Range(numIds); + std::vector vectors = test_util::RandomVectors(dim, numIds, randomDataMin, randomDataMax); + + std::string indexPath = test_util::RandomString(10, "tmp/", ".faiss"); + faiss::MetricType metricType = faiss::METRIC_L2; + std::string indexDescription = "IVF4,PQ1x4"; + + std::unique_ptr faissIndex(test_util::FaissCreateIndex(dim, indexDescription, metricType)); + test_util::FaissTrainIndex(faissIndex.get(), numIds, vectors.data()); + auto faissIndexWithIDMap = test_util::FaissAddData(faissIndex.get(), ids, vectors); + test_util::FaissWriteIndex(&faissIndexWithIDMap, indexPath); + + // Setup jni + JNIEnv *jniEnv = nullptr; + NiceMock mockJNIUtil; + + std::unique_ptr loadedIndexPointer( + reinterpret_cast(knn_jni::faiss_wrapper::LoadIndex( + &mockJNIUtil, jniEnv, (jstring)&indexPath))); + + // Cast down until we get to the ivfpq-l2 state + auto idMapIndex = dynamic_cast(loadedIndexPointer.get()); + ASSERT_NE(idMapIndex, nullptr); + auto ivfpqIndex = dynamic_cast(idMapIndex->index); + ASSERT_NE(ivfpqIndex, nullptr); + ASSERT_EQ(0, ivfpqIndex->precomputed_table->size()); +} + TEST(FaissQueryIndexTest, BasicAssertions) { // Define the index data faiss::idx_t numIds = 100; @@ -501,3 +533,94 @@ TEST(FaissCreateHnswSQfp16IndexTest, BasicAssertions) { // Clean up std::remove(indexPath.c_str()); } + +TEST(FaissIsSharedIndexStateRequired, BasicAssertions) { + int d = 128; + int hnswM = 16; + int ivfNlist = 4; + int pqM = 1; + int pqCodeSize = 8; + std::unique_ptr indexHNSWL2(new faiss::IndexHNSW(d, hnswM, faiss::METRIC_L2)); + std::unique_ptr indexIVFPQIP(new faiss::IndexIVFPQ( + new faiss::IndexFlat(d, faiss::METRIC_INNER_PRODUCT), + d, + ivfNlist, + pqM, + pqCodeSize, + faiss::METRIC_INNER_PRODUCT + )); + std::unique_ptr indexIVFPQL2(new faiss::IndexIVFPQ( + new faiss::IndexFlat(d, faiss::METRIC_L2), + d, + ivfNlist, + pqM, + pqCodeSize, + faiss::METRIC_L2 + )); + std::unique_ptr indexIDMapIVFPQL2(new faiss::IndexIDMap( + new faiss::IndexIVFPQ( + new faiss::IndexFlat(d, faiss::METRIC_L2), + d, + ivfNlist, + pqM, + pqCodeSize, + faiss::METRIC_L2 + ) + )); + std::unique_ptr indexIDMapIVFPQIP(new faiss::IndexIDMap( + new faiss::IndexIVFPQ( + new faiss::IndexFlat(d, faiss::METRIC_INNER_PRODUCT), + d, + ivfNlist, + pqM, + pqCodeSize, + faiss::METRIC_INNER_PRODUCT + ) + )); + jlong nullAddress = 0; + + ASSERT_FALSE(knn_jni::faiss_wrapper::IsSharedIndexStateRequired((jlong) indexHNSWL2.get())); + ASSERT_FALSE(knn_jni::faiss_wrapper::IsSharedIndexStateRequired((jlong) indexIVFPQIP.get())); + ASSERT_FALSE(knn_jni::faiss_wrapper::IsSharedIndexStateRequired((jlong) indexIDMapIVFPQIP.get())); + ASSERT_FALSE(knn_jni::faiss_wrapper::IsSharedIndexStateRequired((jlong) nullAddress)); + + ASSERT_TRUE(knn_jni::faiss_wrapper::IsSharedIndexStateRequired((jlong) indexIVFPQL2.get())); + ASSERT_TRUE(knn_jni::faiss_wrapper::IsSharedIndexStateRequired((jlong) indexIDMapIVFPQL2.get())); +} + +TEST(FaissInitAndSetSharedIndexState, BasicAssertions) { + faiss::idx_t numIds = 256; + int dim = 2; + std::vector ids = test_util::Range(numIds); + std::vector vectors = test_util::RandomVectors(dim, numIds, randomDataMin, randomDataMax); + + std::string indexPath = test_util::RandomString(10, "tmp/", ".faiss"); + faiss::MetricType metricType = faiss::METRIC_L2; + std::string indexDescription = "IVF4,PQ1x4"; + + std::unique_ptr faissIndex(test_util::FaissCreateIndex(dim, indexDescription, metricType)); + test_util::FaissTrainIndex(faissIndex.get(), numIds, vectors.data()); + auto faissIndexWithIDMap = test_util::FaissAddData(faissIndex.get(), ids, vectors); + test_util::FaissWriteIndex(&faissIndexWithIDMap, indexPath); + + // Setup jni + JNIEnv *jniEnv = nullptr; + NiceMock mockJNIUtil; + + std::unique_ptr loadedIndexPointer( + reinterpret_cast(knn_jni::faiss_wrapper::LoadIndex( + &mockJNIUtil, jniEnv, (jstring)&indexPath))); + + auto idMapIndex = dynamic_cast(loadedIndexPointer.get()); + ASSERT_NE(idMapIndex, nullptr); + auto ivfpqIndex = dynamic_cast(idMapIndex->index); + ASSERT_NE(ivfpqIndex, nullptr); + ASSERT_EQ(0, ivfpqIndex->precomputed_table->size()); + jlong sharedModelAddress = knn_jni::faiss_wrapper::InitSharedIndexState((jlong) loadedIndexPointer.get()); + ASSERT_EQ(0, ivfpqIndex->precomputed_table->size()); + knn_jni::faiss_wrapper::SetSharedIndexState((jlong) loadedIndexPointer.get(), sharedModelAddress); + ASSERT_EQ(sharedModelAddress, (jlong) ivfpqIndex->precomputed_table); + ASSERT_NE(0, ivfpqIndex->precomputed_table->size()); + ASSERT_EQ(1, ivfpqIndex->use_precomputed_table); + knn_jni::faiss_wrapper::FreeSharedIndexState(sharedModelAddress); +} diff --git a/src/main/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumer.java b/src/main/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumer.java index 901328766..804c4f7bb 100644 --- a/src/main/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumer.java +++ b/src/main/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumer.java @@ -184,7 +184,7 @@ private void createKNNIndexFromTemplate(byte[] model, KNNCodecUtil.Pair pair, KN KNNSettings.state().getSettingValue(KNNSettings.KNN_ALGO_PARAM_INDEX_THREAD_QTY) ); AccessController.doPrivileged((PrivilegedAction) () -> { - JNIService.createIndexFromTemplate(pair.docs, pair.vectors, indexPath, model, parameters, knnEngine.getName()); + JNIService.createIndexFromTemplate(pair.docs, pair.vectors, indexPath, model, parameters, knnEngine); return null; }); } @@ -223,7 +223,7 @@ private void createKNNIndexFromScratch(FieldInfo fieldInfo, KNNCodecUtil.Pair pa // Pass the path for the nms library to save the file AccessController.doPrivileged((PrivilegedAction) () -> { - JNIService.createIndex(pair.docs, pair.vectors, indexPath, parameters, knnEngine.getName()); + JNIService.createIndex(pair.docs, pair.vectors, indexPath, parameters, knnEngine); return null; }); } diff --git a/src/main/java/org/opensearch/knn/index/memory/NativeMemoryAllocation.java b/src/main/java/org/opensearch/knn/index/memory/NativeMemoryAllocation.java index 53852f129..416980759 100644 --- a/src/main/java/org/opensearch/knn/index/memory/NativeMemoryAllocation.java +++ b/src/main/java/org/opensearch/knn/index/memory/NativeMemoryAllocation.java @@ -143,7 +143,7 @@ private void cleanup() { // memoryAddress is sometimes initialized to 0. If this is ever the case, freeing will surely fail. if (memoryAddress != 0) { - JNIService.free(memoryAddress, knnEngine.getName()); + JNIService.free(memoryAddress, knnEngine); } } diff --git a/src/main/java/org/opensearch/knn/index/memory/NativeMemoryLoadStrategy.java b/src/main/java/org/opensearch/knn/index/memory/NativeMemoryLoadStrategy.java index 93da658d0..568cc892b 100644 --- a/src/main/java/org/opensearch/knn/index/memory/NativeMemoryLoadStrategy.java +++ b/src/main/java/org/opensearch/knn/index/memory/NativeMemoryLoadStrategy.java @@ -92,7 +92,7 @@ public NativeMemoryAllocation.IndexAllocation load(NativeMemoryEntryContext.Inde fileWatcher.init(); KNNEngine knnEngine = KNNEngine.getEngineNameFromPath(indexPath.toString()); - long memoryAddress = JNIService.loadIndex(indexPath.toString(), indexEntryContext.getParameters(), knnEngine.getName()); + long memoryAddress = JNIService.loadIndex(indexPath.toString(), indexEntryContext.getParameters(), knnEngine); final WatcherHandle watcherHandle = resourceWatcherService.add(fileWatcher); return new NativeMemoryAllocation.IndexAllocation( diff --git a/src/main/java/org/opensearch/knn/index/query/KNNWeight.java b/src/main/java/org/opensearch/knn/index/query/KNNWeight.java index 5bd4e9359..7e2fa19cc 100644 --- a/src/main/java/org/opensearch/knn/index/query/KNNWeight.java +++ b/src/main/java/org/opensearch/knn/index/query/KNNWeight.java @@ -280,7 +280,7 @@ private Map doANNSearch(final LeafReaderContext context, final B indexAllocation.getMemoryAddress(), knnQuery.getQueryVector(), knnQuery.getK(), - knnEngine.getName(), + knnEngine, filterIds, filterType.getValue(), parentIds diff --git a/src/main/java/org/opensearch/knn/jni/FaissService.java b/src/main/java/org/opensearch/knn/jni/FaissService.java index a1a07547b..0da6f54ef 100644 --- a/src/main/java/org/opensearch/knn/jni/FaissService.java +++ b/src/main/java/org/opensearch/knn/jni/FaissService.java @@ -84,6 +84,30 @@ public static native void createIndexFromTemplate( */ public static native long loadIndex(String indexPath); + /** + * Determine if index contains shared state. + * + * @param indexAddr address of index to be checked. + * @return true if index requires shared index state; false otherwise + */ + public static native boolean isSharedIndexStateRequired(long indexAddr); + + /** + * Initialize the shared state for an index + * + * @param indexAddr address of the index to initialize from + * @return Address of shared index state address + */ + public static native long initSharedIndexState(long indexAddr); + + /** + * Set the index state for an index + * + * @param indexAddr address of index to set state for + * @param shareIndexStateAddr address of shared state to be set + */ + public static native void setSharedIndexState(long indexAddr, long shareIndexStateAddr); + /** * Query an index without filter * @@ -125,6 +149,13 @@ public static native KNNQueryResult[] queryIndexWithFilter( */ public static native void free(long indexPointer); + /** + * Deallocate memory of the shared index state + * + * @param shareIndexStateAddr address of shared state + */ + public static native void freeSharedIndexState(long shareIndexStateAddr); + /** * Initialize library * diff --git a/src/main/java/org/opensearch/knn/jni/JNIService.java b/src/main/java/org/opensearch/knn/jni/JNIService.java index 2835be23d..555c2d6a6 100644 --- a/src/main/java/org/opensearch/knn/jni/JNIService.java +++ b/src/main/java/org/opensearch/knn/jni/JNIService.java @@ -25,35 +25,35 @@ public class JNIService { /** * Create an index for the native library * - * @param ids array of ids mapping to the data passed in - * @param data array of float arrays to be indexed - * @param indexPath path to save index file to + * @param ids array of ids mapping to the data passed in + * @param data array of float arrays to be indexed + * @param indexPath path to save index file to * @param parameters parameters to build index - * @param engineName name of engine to build index for + * @param knnEngine engine to build index for */ - public static void createIndex(int[] ids, float[][] data, String indexPath, Map parameters, String engineName) { - if (KNNEngine.NMSLIB.getName().equals(engineName)) { + public static void createIndex(int[] ids, float[][] data, String indexPath, Map parameters, KNNEngine knnEngine) { + if (KNNEngine.NMSLIB == knnEngine) { NmslibService.createIndex(ids, data, indexPath, parameters); return; } - if (KNNEngine.FAISS.getName().equals(engineName)) { + if (KNNEngine.FAISS == knnEngine) { FaissService.createIndex(ids, data, indexPath, parameters); return; } - throw new IllegalArgumentException("CreateIndex not supported for provided engine"); + throw new IllegalArgumentException(String.format("CreateIndex not supported for provided engine : %s", knnEngine.getName())); } /** * Create an index for the native library with a provided template index * - * @param ids array of ids mapping to the data passed in - * @param data array of float arrays to be indexed - * @param indexPath path to save index file to + * @param ids array of ids mapping to the data passed in + * @param data array of float arrays to be indexed + * @param indexPath path to save index file to * @param templateIndex empty template index - * @param parameters parameters to build index - * @param engineName name of engine to build index for + * @param parameters parameters to build index + * @param knnEngine engine to build index for */ public static void createIndexFromTemplate( int[] ids, @@ -61,44 +61,97 @@ public static void createIndexFromTemplate( String indexPath, byte[] templateIndex, Map parameters, - String engineName + KNNEngine knnEngine ) { - if (KNNEngine.FAISS.getName().equals(engineName)) { + if (KNNEngine.FAISS == knnEngine) { FaissService.createIndexFromTemplate(ids, data, indexPath, templateIndex, parameters); return; } - throw new IllegalArgumentException("CreateIndexFromTemplate not supported for provided engine"); + throw new IllegalArgumentException( + String.format("CreateIndexFromTemplate not supported for provided engine : %s", knnEngine.getName()) + ); } /** * Load an index into memory * - * @param indexPath path to index file + * @param indexPath path to index file * @param parameters parameters to be used when loading index - * @param engineName name of engine to load index + * @param knnEngine engine to load index * @return pointer to location in memory the index resides in */ - public static long loadIndex(String indexPath, Map parameters, String engineName) { - if (KNNEngine.NMSLIB.getName().equals(engineName)) { + public static long loadIndex(String indexPath, Map parameters, KNNEngine knnEngine) { + if (KNNEngine.NMSLIB == knnEngine) { return NmslibService.loadIndex(indexPath, parameters); } - if (KNNEngine.FAISS.getName().equals(engineName)) { + if (KNNEngine.FAISS == knnEngine) { return FaissService.loadIndex(indexPath); } - throw new IllegalArgumentException("LoadIndex not supported for provided engine"); + throw new IllegalArgumentException(String.format("LoadIndex not supported for provided engine : %s", knnEngine.getName())); + } + + /** + * Determine if index contains shared state. Currently, we cannot do this in the plugin because we do not store the + * model definition anywhere. Only faiss supports indices that have shared state. So for all other engines it will + * return false. + * + * @param indexAddr address of index to be checked. + * @param knnEngine engine + * @return true if index requires shared index state; false otherwise + */ + public static boolean isSharedIndexStateRequired(long indexAddr, KNNEngine knnEngine) { + if (KNNEngine.FAISS == knnEngine) { + return FaissService.isSharedIndexStateRequired(indexAddr); + } + + return false; + } + + /** + * Initialize the shared state for an index + * + * @param indexAddr address of the index to initialize from + * @param knnEngine engine + * @return Address of shared index state address + */ + public static long initSharedIndexState(long indexAddr, KNNEngine knnEngine) { + if (KNNEngine.FAISS == knnEngine) { + return FaissService.initSharedIndexState(indexAddr); + } + throw new IllegalArgumentException( + String.format("InitSharedIndexState not supported for provided engine : %s", knnEngine.getName()) + ); + } + + /** + * Set the index state for an index + * + * @param indexAddr address of index to set state for + * @param shareIndexStateAddr address of shared state to be set + * @param knnEngine engine + */ + public static void setSharedIndexState(long indexAddr, long shareIndexStateAddr, KNNEngine knnEngine) { + if (KNNEngine.FAISS == knnEngine) { + FaissService.setSharedIndexState(indexAddr, shareIndexStateAddr); + return; + } + + throw new IllegalArgumentException( + String.format("SetSharedIndexState not supported for provided engine : %s", knnEngine.getName()) + ); } /** * Query an index * - * @param indexPointer pointer to index in memory - * @param queryVector vector to be used for query - * @param k neighbors to be returned - * @param engineName name of engine to query index - * @param filteredIds array of ints on which should be used for search. + * @param indexPointer pointer to index in memory + * @param queryVector vector to be used for query + * @param k neighbors to be returned + * @param knnEngine engine to query index + * @param filteredIds array of ints on which should be used for search. * @param filterIdsType how to filter ids: Batch or BitMap * @return KNNQueryResult array of k neighbors */ @@ -106,16 +159,16 @@ public static KNNQueryResult[] queryIndex( long indexPointer, float[] queryVector, int k, - String engineName, + KNNEngine knnEngine, long[] filteredIds, int filterIdsType, int[] parentIds ) { - if (KNNEngine.NMSLIB.getName().equals(engineName)) { + if (KNNEngine.NMSLIB == knnEngine) { return NmslibService.queryIndex(indexPointer, queryVector, k); } - if (KNNEngine.FAISS.getName().equals(engineName)) { + if (KNNEngine.FAISS == knnEngine) { // This code assumes that if filteredIds == null / filteredIds.length == 0 if filter is specified then empty // k-NN results are already returned. Otherwise, it's a filter case and we need to run search with // filterIds. FilterIds is coming as empty then its the case where we need to do search with Faiss engine @@ -125,44 +178,60 @@ public static KNNQueryResult[] queryIndex( } return FaissService.queryIndex(indexPointer, queryVector, k, parentIds); } - throw new IllegalArgumentException("QueryIndex not supported for provided engine"); + throw new IllegalArgumentException(String.format("QueryIndex not supported for provided engine : %s", knnEngine.getName())); } /** * Free native memory pointer * * @param indexPointer location to be freed - * @param engineName engine to perform free + * @param knnEngine engine to perform free */ - public static void free(long indexPointer, String engineName) { - if (KNNEngine.NMSLIB.getName().equals(engineName)) { + public static void free(long indexPointer, KNNEngine knnEngine) { + if (KNNEngine.NMSLIB == knnEngine) { NmslibService.free(indexPointer); return; } - if (KNNEngine.FAISS.getName().equals(engineName)) { + if (KNNEngine.FAISS == knnEngine) { FaissService.free(indexPointer); return; } - throw new IllegalArgumentException("Free not supported for provided engine"); + throw new IllegalArgumentException(String.format("Free not supported for provided engine : %s", knnEngine.getName())); + } + + /** + * Deallocate memory of the shared index state + * + * @param shareIndexStateAddr address of shared state + * @param knnEngine engine + */ + public static void freeSharedIndexState(long shareIndexStateAddr, KNNEngine knnEngine) { + if (KNNEngine.FAISS == knnEngine) { + FaissService.freeSharedIndexState(shareIndexStateAddr); + return; + } + throw new IllegalArgumentException( + String.format("FreeSharedIndexState not supported for provided engine : %s", knnEngine.getName()) + ); } /** * Train an empty index * - * @param indexParameters parameters used to build index - * @param dimension dimension for the index + * @param indexParameters parameters used to build index + * @param dimension dimension for the index * @param trainVectorsPointer pointer to where training vectors are stored in native memory - * @param engineName engine to perform the training + * @param knnEngine engine to perform the training * @return bytes array of trained template index */ - public static byte[] trainIndex(Map indexParameters, int dimension, long trainVectorsPointer, String engineName) { - if (KNNEngine.FAISS.getName().equals(engineName)) { + public static byte[] trainIndex(Map indexParameters, int dimension, long trainVectorsPointer, KNNEngine knnEngine) { + if (KNNEngine.FAISS == knnEngine) { return FaissService.trainIndex(indexParameters, dimension, trainVectorsPointer); } - throw new IllegalArgumentException("TrainIndex not supported for provided engine"); + throw new IllegalArgumentException(String.format("TrainIndex not supported for provided engine : %s", knnEngine.getName())); } /** diff --git a/src/main/java/org/opensearch/knn/training/TrainingJob.java b/src/main/java/org/opensearch/knn/training/TrainingJob.java index 2c86082bb..aa2786c0a 100644 --- a/src/main/java/org/opensearch/knn/training/TrainingJob.java +++ b/src/main/java/org/opensearch/knn/training/TrainingJob.java @@ -186,7 +186,7 @@ public void run() { trainParameters, model.getModelMetadata().getDimension(), trainingDataAllocation.getMemoryAddress(), - model.getModelMetadata().getKnnEngine().getName() + model.getModelMetadata().getKnnEngine() ); // Once training finishes, update model diff --git a/src/test/java/org/opensearch/knn/index/KNNCreateIndexFromModelTests.java b/src/test/java/org/opensearch/knn/index/KNNCreateIndexFromModelTests.java index 710978928..11a8bdb15 100644 --- a/src/test/java/org/opensearch/knn/index/KNNCreateIndexFromModelTests.java +++ b/src/test/java/org/opensearch/knn/index/KNNCreateIndexFromModelTests.java @@ -50,7 +50,7 @@ public void testCreateIndexFromModel() throws IOException, InterruptedException ImmutableMap.of(INDEX_DESCRIPTION_PARAMETER, "Flat", SPACE_TYPE, spaceType.getValue()), dimension, vectorsPointer, - KNNEngine.FAISS.getName() + KNNEngine.FAISS ); // Setup model diff --git a/src/test/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumerTests.java b/src/test/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumerTests.java index 7736652ce..cdf3109a4 100644 --- a/src/test/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumerTests.java +++ b/src/test/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumerTests.java @@ -341,7 +341,7 @@ public void testAddKNNBinaryField_fromModel_faiss() throws IOException, Executio SpaceType.L2.getValue() ); - byte[] modelBytes = JNIService.trainIndex(parameters, dimension, trainingPtr, knnEngine.getName()); + byte[] modelBytes = JNIService.trainIndex(parameters, dimension, trainingPtr, knnEngine); Model model = new Model( new ModelMetadata( knnEngine, diff --git a/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestCase.java b/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestCase.java index 8ab2641db..b82bc85e0 100644 --- a/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestCase.java +++ b/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestCase.java @@ -197,7 +197,7 @@ public void testBuildFromModelTemplate(Codec codec) throws IOException, Executio ImmutableMap.of(INDEX_DESCRIPTION_PARAMETER, "Flat", SPACE_TYPE, spaceType.getValue()), dimension, vectorsPointer, - KNNEngine.FAISS.getName() + KNNEngine.FAISS ); // Setup model cache diff --git a/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestUtil.java b/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestUtil.java index 472f05113..c4d50ec27 100644 --- a/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestUtil.java +++ b/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestUtil.java @@ -334,16 +334,12 @@ public static void assertLoadableByEngine( ) { String filePath = Paths.get(((FSDirectory) (FilterDirectory.unwrap(state.directory))).getDirectory().toString(), fileName) .toString(); - long indexPtr = JNIService.loadIndex( - filePath, - Maps.newHashMap(ImmutableMap.of(SPACE_TYPE, spaceType.getValue())), - knnEngine.getName() - ); + long indexPtr = JNIService.loadIndex(filePath, Maps.newHashMap(ImmutableMap.of(SPACE_TYPE, spaceType.getValue())), knnEngine); int k = 2; float[] queryVector = new float[dimension]; - KNNQueryResult[] results = JNIService.queryIndex(indexPtr, queryVector, k, knnEngine.getName(), null, 0, null); + KNNQueryResult[] results = JNIService.queryIndex(indexPtr, queryVector, k, knnEngine, null, 0, null); assertTrue(results.length > 0); - JNIService.free(indexPtr, knnEngine.getName()); + JNIService.free(indexPtr, knnEngine); } public static float[][] getRandomVectors(int count, int dimension) { diff --git a/src/test/java/org/opensearch/knn/index/memory/NativeMemoryAllocationTests.java b/src/test/java/org/opensearch/knn/index/memory/NativeMemoryAllocationTests.java index fbd519163..fabffab46 100644 --- a/src/test/java/org/opensearch/knn/index/memory/NativeMemoryAllocationTests.java +++ b/src/test/java/org/opensearch/knn/index/memory/NativeMemoryAllocationTests.java @@ -52,10 +52,10 @@ public void testIndexAllocation_close() throws InterruptedException { Arrays.fill(vectors[i], 1f); } Map parameters = ImmutableMap.of(KNNConstants.SPACE_TYPE, SpaceType.DEFAULT.getValue()); - JNIService.createIndex(ids, vectors, path, parameters, knnEngine.getName()); + JNIService.createIndex(ids, vectors, path, parameters, knnEngine); // Load index into memory - long memoryAddress = JNIService.loadIndex(path, parameters, knnEngine.getName()); + long memoryAddress = JNIService.loadIndex(path, parameters, knnEngine); @SuppressWarnings("unchecked") WatcherHandle watcherHandle = (WatcherHandle) mock(WatcherHandle.class); diff --git a/src/test/java/org/opensearch/knn/index/memory/NativeMemoryLoadStrategyTests.java b/src/test/java/org/opensearch/knn/index/memory/NativeMemoryLoadStrategyTests.java index 1f1088de1..207b6373b 100644 --- a/src/test/java/org/opensearch/knn/index/memory/NativeMemoryLoadStrategyTests.java +++ b/src/test/java/org/opensearch/knn/index/memory/NativeMemoryLoadStrategyTests.java @@ -53,7 +53,7 @@ public void testIndexLoadStrategy_load() throws IOException { Arrays.fill(vectors[i], 1f); } Map parameters = ImmutableMap.of(KNNConstants.SPACE_TYPE, SpaceType.DEFAULT.getValue()); - JNIService.createIndex(ids, vectors, path, parameters, knnEngine.getName()); + JNIService.createIndex(ids, vectors, path, parameters, knnEngine); // Setup mock resource manager ResourceWatcherService resourceWatcherService = mock(ResourceWatcherService.class); @@ -74,7 +74,7 @@ public void testIndexLoadStrategy_load() throws IOException { // Confirm that the file was loaded by querying float[] query = new float[dimension]; Arrays.fill(query, numVectors + 1); - KNNQueryResult[] results = JNIService.queryIndex(indexAllocation.getMemoryAddress(), query, 2, knnEngine.getName(), null, 0, null); + KNNQueryResult[] results = JNIService.queryIndex(indexAllocation.getMemoryAddress(), query, 2, knnEngine, null, 0, null); assertTrue(results.length > 0); } diff --git a/src/test/java/org/opensearch/knn/index/query/KNNWeightTests.java b/src/test/java/org/opensearch/knn/index/query/KNNWeightTests.java index 49c7c7566..2e6895d0b 100644 --- a/src/test/java/org/opensearch/knn/index/query/KNNWeightTests.java +++ b/src/test/java/org/opensearch/knn/index/query/KNNWeightTests.java @@ -157,7 +157,7 @@ public void testQueryScoreForFaissWithModel() { SpaceType spaceType = SpaceType.L2; final Function scoreTranslator = spaceType::scoreTranslation; final String modelId = "modelId"; - jniServiceMockedStatic.when(() -> JNIService.queryIndex(anyLong(), any(), anyInt(), anyString(), any(), anyInt(), any())) + jniServiceMockedStatic.when(() -> JNIService.queryIndex(anyLong(), any(), anyInt(), any(), any(), anyInt(), any())) .thenReturn(getKNNQueryResults()); final KNNQuery query = new KNNQuery(FIELD_NAME, QUERY_VECTOR, K, INDEX_NAME, (BitSetProducer) null); @@ -300,7 +300,7 @@ public void testShardWithoutFiles() { @SneakyThrows public void testEmptyQueryResults() { final KNNQueryResult[] knnQueryResults = new KNNQueryResult[] {}; - jniServiceMockedStatic.when(() -> JNIService.queryIndex(anyLong(), any(), anyInt(), anyString(), any(), anyInt(), any())) + jniServiceMockedStatic.when(() -> JNIService.queryIndex(anyLong(), any(), anyInt(), any(), any(), anyInt(), any())) .thenReturn(knnQueryResults); final KNNQuery query = new KNNQuery(FIELD_NAME, QUERY_VECTOR, K, INDEX_NAME, (BitSetProducer) null); @@ -350,7 +350,7 @@ public void testANNWithFilterQuery_whenDoingANN_thenSuccess() { filterBitSet.set(docId); } jniServiceMockedStatic.when( - () -> JNIService.queryIndex(anyLong(), any(), anyInt(), anyString(), eq(filterBitSet.getBits()), anyInt(), any()) + () -> JNIService.queryIndex(anyLong(), any(), anyInt(), any(), eq(filterBitSet.getBits()), anyInt(), any()) ).thenReturn(getFilteredKNNQueryResults()); final LeafReaderContext leafReaderContext = mock(LeafReaderContext.class); final SegmentReader reader = mock(SegmentReader.class); @@ -411,7 +411,7 @@ public void testANNWithFilterQuery_whenDoingANN_thenSuccess() { assertEquals(FILTERED_DOC_ID_TO_SCORES.size(), docIdSetIterator.cost()); jniServiceMockedStatic.verify( - () -> JNIService.queryIndex(anyLong(), any(), anyInt(), anyString(), eq(filterBitSet.getBits()), anyInt(), any()) + () -> JNIService.queryIndex(anyLong(), any(), anyInt(), any(), eq(filterBitSet.getBits()), anyInt(), any()) ); final List actualDocIds = new ArrayList<>(); @@ -677,17 +677,14 @@ public void testANNWithParentsFilter_whenDoingANN_thenBitSetIsPassedToJNI() { final KNNQuery query = new KNNQuery(FIELD_NAME, QUERY_VECTOR, 1, INDEX_NAME, null, bitSetProducer); final KNNWeight knnWeight = new KNNWeight(query, 0.0f, null); - jniServiceMockedStatic.when( - () -> JNIService.queryIndex(anyLong(), any(), anyInt(), anyString(), any(), anyInt(), eq(parentsFilter)) - ).thenReturn(getKNNQueryResults()); + jniServiceMockedStatic.when(() -> JNIService.queryIndex(anyLong(), any(), anyInt(), any(), any(), anyInt(), eq(parentsFilter))) + .thenReturn(getKNNQueryResults()); // Execute Scorer knnScorer = knnWeight.scorer(leafReaderContext); // Verify - jniServiceMockedStatic.verify( - () -> JNIService.queryIndex(anyLong(), any(), anyInt(), anyString(), any(), anyInt(), eq(parentsFilter)) - ); + jniServiceMockedStatic.verify(() -> JNIService.queryIndex(anyLong(), any(), anyInt(), any(), any(), anyInt(), eq(parentsFilter))); assertNotNull(knnScorer); final DocIdSetIterator docIdSetIterator = knnScorer.iterator(); assertNotNull(docIdSetIterator); @@ -746,7 +743,7 @@ private void testQueryScore( final Set segmentFiles, final Map fileAttributes ) throws IOException { - jniServiceMockedStatic.when(() -> JNIService.queryIndex(anyLong(), any(), anyInt(), anyString(), any(), anyInt(), any())) + jniServiceMockedStatic.when(() -> JNIService.queryIndex(anyLong(), any(), anyInt(), any(), any(), anyInt(), any())) .thenReturn(getKNNQueryResults()); final KNNQuery query = new KNNQuery(FIELD_NAME, QUERY_VECTOR, K, INDEX_NAME, (BitSetProducer) null); diff --git a/src/test/java/org/opensearch/knn/jni/JNIServiceTests.java b/src/test/java/org/opensearch/knn/jni/JNIServiceTests.java index c20facf3a..36a3d93be 100644 --- a/src/test/java/org/opensearch/knn/jni/JNIServiceTests.java +++ b/src/test/java/org/opensearch/knn/jni/JNIServiceTests.java @@ -86,7 +86,7 @@ public void testCreateIndex_invalid_engineNotSupported() { new float[][] {}, "test", ImmutableMap.of(KNNConstants.SPACE_TYPE, SpaceType.L2.getValue()), - "invalid-engine" + KNNEngine.LUCENE ) ); } @@ -113,7 +113,7 @@ public void testCreateIndex_nmslib_invalid_noSpaceType() { testData.indexData.vectors, "something", Collections.emptyMap(), - KNNEngine.NMSLIB.getName() + KNNEngine.NMSLIB ) ); } @@ -131,7 +131,7 @@ public void testCreateIndex_nmslib_invalid_vectorDocIDMismatch() throws IOExcept vectors1, tmpFile1.toAbsolutePath().toString(), ImmutableMap.of(KNNConstants.SPACE_TYPE, SpaceType.L2.getValue()), - KNNEngine.NMSLIB.getName() + KNNEngine.NMSLIB ) ); @@ -145,7 +145,7 @@ public void testCreateIndex_nmslib_invalid_vectorDocIDMismatch() throws IOExcept vectors2, tmpFile2.toAbsolutePath().toString(), ImmutableMap.of(KNNConstants.SPACE_TYPE, SpaceType.L2.getValue()), - KNNEngine.NMSLIB.getName() + KNNEngine.NMSLIB ) ); } @@ -163,7 +163,7 @@ public void testCreateIndex_nmslib_invalid_nullArgument() throws IOException { vectors, tmpFile.toAbsolutePath().toString(), ImmutableMap.of(KNNConstants.SPACE_TYPE, SpaceType.L2.getValue()), - KNNEngine.NMSLIB.getName() + KNNEngine.NMSLIB ) ); @@ -174,7 +174,7 @@ public void testCreateIndex_nmslib_invalid_nullArgument() throws IOException { null, tmpFile.toAbsolutePath().toString(), ImmutableMap.of(KNNConstants.SPACE_TYPE, SpaceType.L2.getValue()), - KNNEngine.NMSLIB.getName() + KNNEngine.NMSLIB ) ); @@ -185,13 +185,13 @@ public void testCreateIndex_nmslib_invalid_nullArgument() throws IOException { vectors, null, ImmutableMap.of(KNNConstants.SPACE_TYPE, SpaceType.L2.getValue()), - KNNEngine.NMSLIB.getName() + KNNEngine.NMSLIB ) ); expectThrows( Exception.class, - () -> JNIService.createIndex(docIds, vectors, tmpFile.toAbsolutePath().toString(), null, KNNEngine.NMSLIB.getName()) + () -> JNIService.createIndex(docIds, vectors, tmpFile.toAbsolutePath().toString(), null, KNNEngine.NMSLIB) ); expectThrows( @@ -219,7 +219,7 @@ public void testCreateIndex_nmslib_invalid_badSpace() throws IOException { vectors, tmpFile.toAbsolutePath().toString(), ImmutableMap.of(KNNConstants.SPACE_TYPE, "invalid"), - KNNEngine.NMSLIB.getName() + KNNEngine.NMSLIB ) ); } @@ -237,7 +237,7 @@ public void testCreateIndex_nmslib_invalid_inconsistentDimensions() throws IOExc vectors, tmpFile.toAbsolutePath().toString(), ImmutableMap.of(KNNConstants.SPACE_TYPE, SpaceType.L2.getValue()), - KNNEngine.NMSLIB.getName() + KNNEngine.NMSLIB ) ); } @@ -261,7 +261,7 @@ public void testCreateIndex_nmslib_invalid_badParameterType() throws IOException vectors, tmpFile.toAbsolutePath().toString(), ImmutableMap.of(KNNConstants.SPACE_TYPE, SpaceType.L2.getValue(), KNNConstants.PARAMETERS, parametersMap), - KNNEngine.NMSLIB.getName() + KNNEngine.NMSLIB ) ); } @@ -276,7 +276,7 @@ public void testCreateIndex_nmslib_valid() throws IOException { testData.indexData.vectors, tmpFile.toAbsolutePath().toString(), ImmutableMap.of(KNNConstants.SPACE_TYPE, spaceType.getValue()), - KNNEngine.NMSLIB.getName() + KNNEngine.NMSLIB ); assertTrue(tmpFile.toFile().length() > 0); @@ -294,7 +294,7 @@ public void testCreateIndex_nmslib_valid() throws IOException { KNNConstants.METHOD_PARAMETER_M, 12 ), - KNNEngine.NMSLIB.getName() + KNNEngine.NMSLIB ); assertTrue(tmpFile.toFile().length() > 0); } @@ -312,7 +312,7 @@ public void testCreateIndex_faiss_invalid_noSpaceType() { vectors, "something", ImmutableMap.of(INDEX_DESCRIPTION_PARAMETER, faissMethod), - FAISS_NAME + KNNEngine.FAISS ) ); } @@ -330,7 +330,7 @@ public void testCreateIndex_faiss_invalid_vectorDocIDMismatch() throws IOExcepti vectors1, tmpFile1.toAbsolutePath().toString(), ImmutableMap.of(INDEX_DESCRIPTION_PARAMETER, faissMethod, KNNConstants.SPACE_TYPE, SpaceType.L2.getValue()), - FAISS_NAME + KNNEngine.FAISS ) ); @@ -344,7 +344,7 @@ public void testCreateIndex_faiss_invalid_vectorDocIDMismatch() throws IOExcepti vectors2, tmpFile2.toAbsolutePath().toString(), ImmutableMap.of(INDEX_DESCRIPTION_PARAMETER, faissMethod, KNNConstants.SPACE_TYPE, SpaceType.L2.getValue()), - FAISS_NAME + KNNEngine.FAISS ) ); } @@ -362,7 +362,7 @@ public void testCreateIndex_faiss_invalid_null() throws IOException { vectors, tmpFile.toAbsolutePath().toString(), ImmutableMap.of(INDEX_DESCRIPTION_PARAMETER, faissMethod, KNNConstants.SPACE_TYPE, SpaceType.L2.getValue()), - FAISS_NAME + KNNEngine.FAISS ) ); @@ -373,7 +373,7 @@ public void testCreateIndex_faiss_invalid_null() throws IOException { null, tmpFile.toAbsolutePath().toString(), ImmutableMap.of(INDEX_DESCRIPTION_PARAMETER, faissMethod, KNNConstants.SPACE_TYPE, SpaceType.L2.getValue()), - FAISS_NAME + KNNEngine.FAISS ) ); @@ -384,11 +384,14 @@ public void testCreateIndex_faiss_invalid_null() throws IOException { vectors, null, ImmutableMap.of(INDEX_DESCRIPTION_PARAMETER, faissMethod, KNNConstants.SPACE_TYPE, SpaceType.L2.getValue()), - FAISS_NAME + KNNEngine.FAISS ) ); - expectThrows(Exception.class, () -> JNIService.createIndex(docIds, vectors, tmpFile.toAbsolutePath().toString(), null, FAISS_NAME)); + expectThrows( + Exception.class, + () -> JNIService.createIndex(docIds, vectors, tmpFile.toAbsolutePath().toString(), null, KNNEngine.FAISS) + ); expectThrows( Exception.class, @@ -415,7 +418,7 @@ public void testCreateIndex_faiss_invalid_invalidSpace() throws IOException { vectors, tmpFile.toAbsolutePath().toString(), ImmutableMap.of(INDEX_DESCRIPTION_PARAMETER, faissMethod, KNNConstants.SPACE_TYPE, "invalid"), - FAISS_NAME + KNNEngine.FAISS ) ); } @@ -433,7 +436,7 @@ public void testCreateIndex_faiss_invalid_inconsistentDimensions() throws IOExce vectors, tmpFile.toAbsolutePath().toString(), ImmutableMap.of(INDEX_DESCRIPTION_PARAMETER, faissMethod, KNNConstants.SPACE_TYPE, SpaceType.L2.getValue()), - FAISS_NAME + KNNEngine.FAISS ) ); } @@ -451,7 +454,7 @@ public void testCreateIndex_faiss_invalid_noIndexDescription() throws IOExceptio vectors, tmpFile.toAbsolutePath().toString(), ImmutableMap.of(KNNConstants.SPACE_TYPE, SpaceType.L2.getValue()), - FAISS_NAME + KNNEngine.FAISS ) ); } @@ -469,7 +472,7 @@ public void testCreateIndex_faiss_invalid_invalidIndexDescription() throws IOExc vectors, tmpFile.toAbsolutePath().toString(), ImmutableMap.of(INDEX_DESCRIPTION_PARAMETER, "invalid", KNNConstants.SPACE_TYPE, SpaceType.L2.getValue()), - FAISS_NAME + KNNEngine.FAISS ) ); } @@ -494,7 +497,7 @@ public void testCreateIndex_faiss_sqfp16_invalidIndexDescription() { KNNConstants.SPACE_TYPE, SpaceType.L2.getValue() ), - FAISS_NAME + KNNEngine.FAISS ) ); } @@ -512,11 +515,11 @@ public void testLoadIndex_faiss_sqfp16_valid() { vectors, tmpFile.toAbsolutePath().toString(), ImmutableMap.of(INDEX_DESCRIPTION_PARAMETER, sqfp16IndexDescription, KNNConstants.SPACE_TYPE, SpaceType.L2.getValue()), - FAISS_NAME + KNNEngine.FAISS ); assertTrue(tmpFile.toFile().length() > 0); - long pointer = JNIService.loadIndex(tmpFile.toAbsolutePath().toString(), Collections.emptyMap(), FAISS_NAME); + long pointer = JNIService.loadIndex(tmpFile.toAbsolutePath().toString(), Collections.emptyMap(), KNNEngine.FAISS); assertNotEquals(0, pointer); } @@ -532,21 +535,21 @@ public void testQueryIndex_faiss_sqfp16_valid() { truncateToFp16Range(testData.indexData.vectors), tmpFile.toAbsolutePath().toString(), ImmutableMap.of(INDEX_DESCRIPTION_PARAMETER, sqfp16IndexDescription, KNNConstants.SPACE_TYPE, SpaceType.L2.getValue()), - FAISS_NAME + KNNEngine.FAISS ); assertTrue(tmpFile.toFile().length() > 0); - long pointer = JNIService.loadIndex(tmpFile.toAbsolutePath().toString(), Collections.emptyMap(), FAISS_NAME); + long pointer = JNIService.loadIndex(tmpFile.toAbsolutePath().toString(), Collections.emptyMap(), KNNEngine.FAISS); assertNotEquals(0, pointer); for (float[] query : testData.queries) { - KNNQueryResult[] results = JNIService.queryIndex(pointer, query, k, FAISS_NAME, null, 0, null); + KNNQueryResult[] results = JNIService.queryIndex(pointer, query, k, KNNEngine.FAISS, null, 0, null); assertEquals(k, results.length); } // Filter will result in no ids for (float[] query : testData.queries) { - KNNQueryResult[] results = JNIService.queryIndex(pointer, query, k, FAISS_NAME, new long[] { 0 }, 0, null); + KNNQueryResult[] results = JNIService.queryIndex(pointer, query, k, KNNEngine.FAISS, new long[] { 0 }, 0, null); assertEquals(0, results.length); } } @@ -590,7 +593,7 @@ public void testTrain_whenConfigurationIsIVFSQFP16_thenSucceed() { KNNMethodContext knnMethodContext = KNNMethodContext.parse(in); Map parameters = KNNEngine.FAISS.getMethodAsMap(knnMethodContext); - byte[] faissIndex = JNIService.trainIndex(parameters, 128, trainPointer, FAISS_NAME); + byte[] faissIndex = JNIService.trainIndex(parameters, 128, trainPointer, KNNEngine.FAISS); assertNotEquals(0, faissIndex.length); JNIService.freeVectors(trainPointer); @@ -616,7 +619,7 @@ public void testCreateIndex_faiss_invalid_invalidParameterType() throws IOExcept KNNConstants.PARAMETERS, ImmutableMap.of(KNNConstants.METHOD_PARAMETER_NPROBES, "14") ), - FAISS_NAME + KNNEngine.FAISS ) ); @@ -634,7 +637,7 @@ public void testCreateIndex_faiss_valid() throws IOException { testData.indexData.vectors, tmpFile1.toAbsolutePath().toString(), ImmutableMap.of(INDEX_DESCRIPTION_PARAMETER, method, KNNConstants.SPACE_TYPE, spaceType.getValue()), - FAISS_NAME + KNNEngine.FAISS ); assertTrue(tmpFile1.toFile().length() > 0); } @@ -642,28 +645,24 @@ public void testCreateIndex_faiss_valid() throws IOException { } public void testLoadIndex_invalidEngine() { - expectThrows(IllegalArgumentException.class, () -> JNIService.loadIndex("test", Collections.emptyMap(), "invalid-engine")); + expectThrows(IllegalArgumentException.class, () -> JNIService.loadIndex("test", Collections.emptyMap(), KNNEngine.LUCENE)); } public void testLoadIndex_nmslib_invalid_badSpaceType() { expectThrows( Exception.class, - () -> JNIService.loadIndex("test", ImmutableMap.of(KNNConstants.SPACE_TYPE, "invalid"), KNNEngine.NMSLIB.getName()) + () -> JNIService.loadIndex("test", ImmutableMap.of(KNNConstants.SPACE_TYPE, "invalid"), KNNEngine.NMSLIB) ); } public void testLoadIndex_nmslib_invalid_noSpaceType() { - expectThrows(Exception.class, () -> JNIService.loadIndex("test", Collections.emptyMap(), KNNEngine.NMSLIB.getName())); + expectThrows(Exception.class, () -> JNIService.loadIndex("test", Collections.emptyMap(), KNNEngine.NMSLIB)); } public void testLoadIndex_nmslib_invalid_fileDoesNotExist() { expectThrows( Exception.class, - () -> JNIService.loadIndex( - "invalid", - ImmutableMap.of(KNNConstants.SPACE_TYPE, SpaceType.L2.getValue()), - KNNEngine.NMSLIB.getName() - ) + () -> JNIService.loadIndex("invalid", ImmutableMap.of(KNNConstants.SPACE_TYPE, SpaceType.L2.getValue()), KNNEngine.NMSLIB) ); } @@ -674,7 +673,7 @@ public void testLoadIndex_nmslib_invalid_badFile() throws IOException { () -> JNIService.loadIndex( tmpFile.toAbsolutePath().toString(), ImmutableMap.of(KNNConstants.SPACE_TYPE, SpaceType.L2.getValue()), - KNNEngine.NMSLIB.getName() + KNNEngine.NMSLIB ) ); } @@ -688,27 +687,30 @@ public void testLoadIndex_nmslib_valid() throws IOException { testData.indexData.vectors, tmpFile.toAbsolutePath().toString(), ImmutableMap.of(KNNConstants.SPACE_TYPE, SpaceType.L2.getValue()), - KNNEngine.NMSLIB.getName() + KNNEngine.NMSLIB ); assertTrue(tmpFile.toFile().length() > 0); long pointer = JNIService.loadIndex( tmpFile.toAbsolutePath().toString(), ImmutableMap.of(KNNConstants.SPACE_TYPE, SpaceType.L2.getValue()), - KNNEngine.NMSLIB.getName() + KNNEngine.NMSLIB ); assertNotEquals(0, pointer); } public void testLoadIndex_faiss_invalid_fileDoesNotExist() { - expectThrows(Exception.class, () -> JNIService.loadIndex("invalid", Collections.emptyMap(), FAISS_NAME)); + expectThrows(Exception.class, () -> JNIService.loadIndex("invalid", Collections.emptyMap(), KNNEngine.FAISS)); } public void testLoadIndex_faiss_invalid_badFile() throws IOException { Path tmpFile = createTempFile(); - expectThrows(Exception.class, () -> JNIService.loadIndex(tmpFile.toAbsolutePath().toString(), Collections.emptyMap(), FAISS_NAME)); + expectThrows( + Exception.class, + () -> JNIService.loadIndex(tmpFile.toAbsolutePath().toString(), Collections.emptyMap(), KNNEngine.FAISS) + ); } public void testLoadIndex_faiss_valid() throws IOException { @@ -720,24 +722,21 @@ public void testLoadIndex_faiss_valid() throws IOException { testData.indexData.vectors, tmpFile.toAbsolutePath().toString(), ImmutableMap.of(INDEX_DESCRIPTION_PARAMETER, faissMethod, KNNConstants.SPACE_TYPE, SpaceType.L2.getValue()), - FAISS_NAME + KNNEngine.FAISS ); assertTrue(tmpFile.toFile().length() > 0); - long pointer = JNIService.loadIndex(tmpFile.toAbsolutePath().toString(), Collections.emptyMap(), FAISS_NAME); + long pointer = JNIService.loadIndex(tmpFile.toAbsolutePath().toString(), Collections.emptyMap(), KNNEngine.FAISS); assertNotEquals(0, pointer); } public void testQueryIndex_invalidEngine() { - expectThrows( - IllegalArgumentException.class, - () -> JNIService.queryIndex(0L, new float[] {}, 0, "invalid" + "-engine", null, 0, null) - ); + expectThrows(IllegalArgumentException.class, () -> JNIService.queryIndex(0L, new float[] {}, 0, KNNEngine.LUCENE, null, 0, null)); } public void testQueryIndex_nmslib_invalid_badPointer() { - expectThrows(Exception.class, () -> JNIService.queryIndex(0L, new float[] {}, 0, KNNEngine.NMSLIB.getName(), null, 0, null)); + expectThrows(Exception.class, () -> JNIService.queryIndex(0L, new float[] {}, 0, KNNEngine.NMSLIB, null, 0, null)); } public void testQueryIndex_nmslib_invalid_nullQueryVector() throws IOException { @@ -749,18 +748,18 @@ public void testQueryIndex_nmslib_invalid_nullQueryVector() throws IOException { testData.indexData.vectors, tmpFile.toAbsolutePath().toString(), ImmutableMap.of(KNNConstants.SPACE_TYPE, SpaceType.L2.getValue()), - KNNEngine.NMSLIB.getName() + KNNEngine.NMSLIB ); assertTrue(tmpFile.toFile().length() > 0); long pointer = JNIService.loadIndex( tmpFile.toAbsolutePath().toString(), ImmutableMap.of(KNNConstants.SPACE_TYPE, SpaceType.L2.getValue()), - KNNEngine.NMSLIB.getName() + KNNEngine.NMSLIB ); assertNotEquals(0, pointer); - expectThrows(Exception.class, () -> JNIService.queryIndex(pointer, null, 10, KNNEngine.NMSLIB.getName(), null, 0, null)); + expectThrows(Exception.class, () -> JNIService.queryIndex(pointer, null, 10, KNNEngine.NMSLIB, null, 0, null)); } public void testQueryIndex_nmslib_valid() throws IOException { @@ -774,19 +773,19 @@ public void testQueryIndex_nmslib_valid() throws IOException { testData.indexData.vectors, tmpFile.toAbsolutePath().toString(), ImmutableMap.of(KNNConstants.SPACE_TYPE, spaceType.getValue()), - KNNEngine.NMSLIB.getName() + KNNEngine.NMSLIB ); assertTrue(tmpFile.toFile().length() > 0); long pointer = JNIService.loadIndex( tmpFile.toAbsolutePath().toString(), ImmutableMap.of(KNNConstants.SPACE_TYPE, spaceType.getValue()), - KNNEngine.NMSLIB.getName() + KNNEngine.NMSLIB ); assertNotEquals(0, pointer); for (float[] query : testData.queries) { - KNNQueryResult[] results = JNIService.queryIndex(pointer, query, k, KNNEngine.NMSLIB.getName(), null, 0, null); + KNNQueryResult[] results = JNIService.queryIndex(pointer, query, k, KNNEngine.NMSLIB, null, 0, null); assertEquals(k, results.length); } } @@ -794,7 +793,7 @@ public void testQueryIndex_nmslib_valid() throws IOException { public void testQueryIndex_faiss_invalid_badPointer() { - expectThrows(Exception.class, () -> JNIService.queryIndex(0L, new float[] {}, 0, FAISS_NAME, null, 0, null)); + expectThrows(Exception.class, () -> JNIService.queryIndex(0L, new float[] {}, 0, KNNEngine.FAISS, null, 0, null)); } public void testQueryIndex_faiss_invalid_nullQueryVector() throws IOException { @@ -806,14 +805,14 @@ public void testQueryIndex_faiss_invalid_nullQueryVector() throws IOException { testData.indexData.vectors, tmpFile.toAbsolutePath().toString(), ImmutableMap.of(INDEX_DESCRIPTION_PARAMETER, faissMethod, KNNConstants.SPACE_TYPE, SpaceType.L2.getValue()), - FAISS_NAME + KNNEngine.FAISS ); assertTrue(tmpFile.toFile().length() > 0); - long pointer = JNIService.loadIndex(tmpFile.toAbsolutePath().toString(), Collections.emptyMap(), FAISS_NAME); + long pointer = JNIService.loadIndex(tmpFile.toAbsolutePath().toString(), Collections.emptyMap(), KNNEngine.FAISS); assertNotEquals(0, pointer); - expectThrows(Exception.class, () -> JNIService.queryIndex(pointer, null, 10, FAISS_NAME, null, 0, null)); + expectThrows(Exception.class, () -> JNIService.queryIndex(pointer, null, 10, KNNEngine.FAISS, null, 0, null)); } public void testQueryIndex_faiss_valid() throws IOException { @@ -830,25 +829,25 @@ public void testQueryIndex_faiss_valid() throws IOException { testData.indexData.vectors, tmpFile.toAbsolutePath().toString(), ImmutableMap.of(INDEX_DESCRIPTION_PARAMETER, method, KNNConstants.SPACE_TYPE, spaceType.getValue()), - FAISS_NAME + KNNEngine.FAISS ); assertTrue(tmpFile.toFile().length() > 0); long pointer = JNIService.loadIndex( tmpFile.toAbsolutePath().toString(), ImmutableMap.of(KNNConstants.SPACE_TYPE, spaceType.getValue()), - FAISS_NAME + KNNEngine.FAISS ); assertNotEquals(0, pointer); for (float[] query : testData.queries) { - KNNQueryResult[] results = JNIService.queryIndex(pointer, query, k, FAISS_NAME, null, 0, null); + KNNQueryResult[] results = JNIService.queryIndex(pointer, query, k, KNNEngine.FAISS, null, 0, null); assertEquals(k, results.length); } // Filter will result in no ids for (float[] query : testData.queries) { - KNNQueryResult[] results = JNIService.queryIndex(pointer, query, k, FAISS_NAME, new long[] { 0 }, 0, null); + KNNQueryResult[] results = JNIService.queryIndex(pointer, query, k, KNNEngine.FAISS, new long[] { 0 }, 0, null); assertEquals(0, results.length); } } @@ -871,19 +870,19 @@ public void testQueryIndex_faiss_parentIds() throws IOException { testDataNested.indexData.vectors, tmpFile.toAbsolutePath().toString(), ImmutableMap.of(INDEX_DESCRIPTION_PARAMETER, method, KNNConstants.SPACE_TYPE, spaceType.getValue()), - FAISS_NAME + KNNEngine.FAISS ); assertTrue(tmpFile.toFile().length() > 0); long pointer = JNIService.loadIndex( tmpFile.toAbsolutePath().toString(), ImmutableMap.of(KNNConstants.SPACE_TYPE, spaceType.getValue()), - FAISS_NAME + KNNEngine.FAISS ); assertNotEquals(0, pointer); for (float[] query : testDataNested.queries) { - KNNQueryResult[] results = JNIService.queryIndex(pointer, query, k, FAISS_NAME, null, 0, parentIds); + KNNQueryResult[] results = JNIService.queryIndex(pointer, query, k, KNNEngine.FAISS, null, 0, parentIds); // Verify there is no more than one result from same parent Set parentIdSet = toParentIdSet(results, idToParentIdMap); assertEquals(results.length, parentIdSet.size()); @@ -933,7 +932,7 @@ private Map toIdToParentIdMap(int[] ids) { } public void testFree_invalidEngine() { - expectThrows(IllegalArgumentException.class, () -> JNIService.free(0L, "invalid-engine")); + expectThrows(IllegalArgumentException.class, () -> JNIService.free(0L, KNNEngine.LUCENE)); } public void testFree_nmslib_valid() throws IOException { @@ -945,18 +944,18 @@ public void testFree_nmslib_valid() throws IOException { testData.indexData.vectors, tmpFile.toAbsolutePath().toString(), ImmutableMap.of(KNNConstants.SPACE_TYPE, SpaceType.L2.getValue()), - KNNEngine.NMSLIB.getName() + KNNEngine.NMSLIB ); assertTrue(tmpFile.toFile().length() > 0); long pointer = JNIService.loadIndex( tmpFile.toAbsolutePath().toString(), ImmutableMap.of(KNNConstants.SPACE_TYPE, SpaceType.L2.getValue()), - KNNEngine.NMSLIB.getName() + KNNEngine.NMSLIB ); assertNotEquals(0, pointer); - JNIService.free(pointer, KNNEngine.NMSLIB.getName()); + JNIService.free(pointer, KNNEngine.NMSLIB); } public void testFree_faiss_valid() throws IOException { @@ -968,14 +967,14 @@ public void testFree_faiss_valid() throws IOException { testData.indexData.vectors, tmpFile.toAbsolutePath().toString(), ImmutableMap.of(INDEX_DESCRIPTION_PARAMETER, faissMethod, KNNConstants.SPACE_TYPE, SpaceType.L2.getValue()), - FAISS_NAME + KNNEngine.FAISS ); assertTrue(tmpFile.toFile().length() > 0); - long pointer = JNIService.loadIndex(tmpFile.toAbsolutePath().toString(), Collections.emptyMap(), FAISS_NAME); + long pointer = JNIService.loadIndex(tmpFile.toAbsolutePath().toString(), Collections.emptyMap(), KNNEngine.FAISS); assertNotEquals(0, pointer); - JNIService.free(pointer, FAISS_NAME); + JNIService.free(pointer, KNNEngine.FAISS); } public void testTransferVectors() { @@ -1006,7 +1005,7 @@ public void testTrain_whenConfigurationIsIVFFlat_thenSucceed() throws IOExceptio KNNMethodContext knnMethodContext = KNNMethodContext.parse(in); Map parameters = KNNEngine.FAISS.getMethodAsMap(knnMethodContext); - byte[] faissIndex = JNIService.trainIndex(parameters, 128, trainPointer, FAISS_NAME); + byte[] faissIndex = JNIService.trainIndex(parameters, 128, trainPointer, KNNEngine.FAISS); assertNotEquals(0, faissIndex.length); JNIService.freeVectors(trainPointer); @@ -1036,7 +1035,7 @@ public void testTrain_whenConfigurationIsIVFPQ_thenSucceed() throws IOException KNNMethodContext knnMethodContext = KNNMethodContext.parse(in); Map parameters = KNNEngine.FAISS.getMethodAsMap(knnMethodContext); - byte[] faissIndex = JNIService.trainIndex(parameters, 128, trainPointer, FAISS_NAME); + byte[] faissIndex = JNIService.trainIndex(parameters, 128, trainPointer, KNNEngine.FAISS); assertNotEquals(0, faissIndex.length); JNIService.freeVectors(trainPointer); @@ -1063,7 +1062,7 @@ public void testTrain_whenConfigurationIsHNSWPQ_thenSucceed() throws IOException knnMethodContext.getMethodComponentContext().setIndexVersion(Version.CURRENT); Map parameters = KNNEngine.FAISS.getMethodAsMap(knnMethodContext); - byte[] faissIndex = JNIService.trainIndex(parameters, 128, trainPointer, FAISS_NAME); + byte[] faissIndex = JNIService.trainIndex(parameters, 128, trainPointer, KNNEngine.FAISS); assertNotEquals(0, faissIndex.length); JNIService.freeVectors(trainPointer); @@ -1118,7 +1117,7 @@ public void testCreateIndexFromTemplate() throws IOException { spaceType.getValue() ); - byte[] faissIndex = JNIService.trainIndex(parameters, 128, trainPointer1, FAISS_NAME); + byte[] faissIndex = JNIService.trainIndex(parameters, 128, trainPointer1, KNNEngine.FAISS); assertNotEquals(0, faissIndex.length); JNIService.freeVectors(trainPointer1); @@ -1130,11 +1129,157 @@ public void testCreateIndexFromTemplate() throws IOException { tmpFile1.toAbsolutePath().toString(), faissIndex, ImmutableMap.of(INDEX_THREAD_QTY, 1), - FAISS_NAME + KNNEngine.FAISS ); assertTrue(tmpFile1.toFile().length() > 0); - long pointer = JNIService.loadIndex(tmpFile1.toAbsolutePath().toString(), Collections.emptyMap(), FAISS_NAME); + long pointer = JNIService.loadIndex(tmpFile1.toAbsolutePath().toString(), Collections.emptyMap(), KNNEngine.FAISS); assertNotEquals(0, pointer); } + + @SneakyThrows + public void testIndexLoad_whenStateIsShared_thenSucceed() { + // Creates a single IVFPQ-l2 index. Then, we will configure a set of indices in memory in different ways to + // ensure that everything is loaded properly and the results are consistent. + int k = 10; + int ivfNlist = 16; + int pqM = 16; + int pqCodeSize = 4; + + String indexIVFPQPath = createFaissIVFPQIndex(ivfNlist, pqM, pqCodeSize, SpaceType.L2); + + long indexIVFPQIndexTest1 = JNIService.loadIndex(indexIVFPQPath, Collections.emptyMap(), KNNEngine.FAISS); + assertNotEquals(0, indexIVFPQIndexTest1); + long indexIVFPQIndexTest2 = JNIService.loadIndex(indexIVFPQPath, Collections.emptyMap(), KNNEngine.FAISS); + assertNotEquals(0, indexIVFPQIndexTest2); + + long sharedStateAddress = JNIService.initSharedIndexState(indexIVFPQIndexTest1, KNNEngine.FAISS); + JNIService.setSharedIndexState(indexIVFPQIndexTest1, sharedStateAddress, KNNEngine.FAISS); + JNIService.setSharedIndexState(indexIVFPQIndexTest2, sharedStateAddress, KNNEngine.FAISS); + + assertQueryResultsMatch(testData.queries, k, List.of(indexIVFPQIndexTest1, indexIVFPQIndexTest2)); + + // Free the first test index 1. This will ensure that the shared state persists after index that initialized + // shared state is gone. + JNIService.free(indexIVFPQIndexTest1, KNNEngine.FAISS); + + long indexIVFPQIndexTest3 = JNIService.loadIndex(indexIVFPQPath, Collections.emptyMap(), KNNEngine.FAISS); + assertNotEquals(0, indexIVFPQIndexTest3); + + JNIService.setSharedIndexState(indexIVFPQIndexTest3, sharedStateAddress, KNNEngine.FAISS); + + assertQueryResultsMatch(testData.queries, k, List.of(indexIVFPQIndexTest2, indexIVFPQIndexTest3)); + + // Ensure everything gets freed + JNIService.free(indexIVFPQIndexTest2, KNNEngine.FAISS); + JNIService.free(indexIVFPQIndexTest3, KNNEngine.FAISS); + JNIService.freeSharedIndexState(sharedStateAddress, KNNEngine.FAISS); + } + + @SneakyThrows + public void testIsIndexIVFPQL2() { + long dummyAddress = 0; + assertFalse(JNIService.isSharedIndexStateRequired(dummyAddress, KNNEngine.NMSLIB)); + + String faissIVFPQL2Index = createFaissIVFPQIndex(16, 16, 4, SpaceType.L2); + long faissIVFPQL2Address = JNIService.loadIndex(faissIVFPQL2Index, Collections.emptyMap(), KNNEngine.FAISS); + assertTrue(JNIService.isSharedIndexStateRequired(faissIVFPQL2Address, KNNEngine.FAISS)); + JNIService.free(faissIVFPQL2Address, KNNEngine.FAISS); + + String faissIVFPQIPIndex = createFaissIVFPQIndex(16, 16, 4, SpaceType.INNER_PRODUCT); + long faissIVFPQIPAddress = JNIService.loadIndex(faissIVFPQIPIndex, Collections.emptyMap(), KNNEngine.FAISS); + assertFalse(JNIService.isSharedIndexStateRequired(faissIVFPQIPAddress, KNNEngine.FAISS)); + JNIService.free(faissIVFPQIPAddress, KNNEngine.FAISS); + + String faissHNSWIndex = createFaissHNSWIndex(SpaceType.L2); + long faissHNSWAddress = JNIService.loadIndex(faissHNSWIndex, Collections.emptyMap(), KNNEngine.FAISS); + assertFalse(JNIService.isSharedIndexStateRequired(faissHNSWAddress, KNNEngine.FAISS)); + JNIService.free(faissHNSWAddress, KNNEngine.FAISS); + } + + @SneakyThrows + public void testFunctionsUnsupportedForEngine_whenEngineUnsupported_thenThrowIllegalArgumentException() { + int dummyAddress = 0; + expectThrows(IllegalArgumentException.class, () -> JNIService.initSharedIndexState(dummyAddress, KNNEngine.NMSLIB)); + expectThrows(IllegalArgumentException.class, () -> JNIService.setSharedIndexState(dummyAddress, dummyAddress, KNNEngine.NMSLIB)); + expectThrows(IllegalArgumentException.class, () -> JNIService.freeSharedIndexState(dummyAddress, KNNEngine.NMSLIB)); + } + + private void assertQueryResultsMatch(float[][] testQueries, int k, List indexAddresses) { + // Checks that the set of queries is consistent amongst all indices in the list + for (float[] query : testQueries) { + KNNQueryResult[][] allResults = new KNNQueryResult[indexAddresses.size()][]; + for (int i = 0; i < indexAddresses.size(); i++) { + allResults[i] = JNIService.queryIndex(indexAddresses.get(i), query, k, KNNEngine.FAISS, null, 0, null); + assertEquals(k, allResults[i].length); + } + + for (int i = 1; i < indexAddresses.size(); i++) { + for (int j = 0; j < k; j++) { + assertEquals(allResults[0][j].getId(), allResults[i][j].getId()); + assertEquals(allResults[0][j].getScore(), allResults[i][j].getScore(), 0.00001); + } + } + } + } + + private String createFaissIVFPQIndex(int ivfNlist, int pqM, int pqCodeSize, SpaceType spaceType) throws IOException { + long trainPointer = JNIService.transferVectors(0, testData.indexData.vectors); + assertNotEquals(0, trainPointer); + + KNNMethodContext knnMethodContext = new KNNMethodContext( + KNNEngine.FAISS, + spaceType, + new MethodComponentContext( + METHOD_IVF, + ImmutableMap.of( + METHOD_PARAMETER_NLIST, + ivfNlist, + METHOD_ENCODER_PARAMETER, + new MethodComponentContext( + ENCODER_PQ, + ImmutableMap.of(ENCODER_PARAMETER_PQ_M, pqM, ENCODER_PARAMETER_PQ_CODE_SIZE, pqCodeSize) + ) + ) + ) + ); + + String description = knnMethodContext.getKnnEngine().getMethodAsMap(knnMethodContext).get(INDEX_DESCRIPTION_PARAMETER).toString(); + Map parameters = ImmutableMap.of( + INDEX_DESCRIPTION_PARAMETER, + description, + KNNConstants.SPACE_TYPE, + spaceType.getValue() + ); + + byte[] faissIndex = JNIService.trainIndex(parameters, 128, trainPointer, KNNEngine.FAISS); + + assertNotEquals(0, faissIndex.length); + JNIService.freeVectors(trainPointer); + Path tmpFile = createTempFile(); + JNIService.createIndexFromTemplate( + testData.indexData.docs, + testData.indexData.vectors, + tmpFile.toAbsolutePath().toString(), + faissIndex, + ImmutableMap.of(INDEX_THREAD_QTY, 1), + KNNEngine.FAISS + ); + assertTrue(tmpFile.toFile().length() > 0); + + return tmpFile.toAbsolutePath().toString(); + } + + private String createFaissHNSWIndex(SpaceType spaceType) throws IOException { + Path tmpFile = createTempFile(); + JNIService.createIndex( + testData.indexData.docs, + testData.indexData.vectors, + tmpFile.toAbsolutePath().toString(), + ImmutableMap.of(INDEX_DESCRIPTION_PARAMETER, faissMethod, KNNConstants.SPACE_TYPE, spaceType.getValue()), + KNNEngine.FAISS + ); + assertTrue(tmpFile.toFile().length() > 0); + return tmpFile.toAbsolutePath().toString(); + } } diff --git a/src/test/java/org/opensearch/knn/training/TrainingJobTests.java b/src/test/java/org/opensearch/knn/training/TrainingJobTests.java index 9dc461b97..e4c01a8ec 100644 --- a/src/test/java/org/opensearch/knn/training/TrainingJobTests.java +++ b/src/test/java/org/opensearch/knn/training/TrainingJobTests.java @@ -205,7 +205,7 @@ public void testRun_success() throws IOException, ExecutionException { indexPath.toString(), model.getModelBlob(), ImmutableMap.of(INDEX_THREAD_QTY, 1), - knnEngine.getName() + knnEngine ); assertNotEquals(0, new File(indexPath.toString()).length()); } From 715b79ddfd955926401a27009956c7c38ca91b82 Mon Sep 17 00:00:00 2001 From: John Mazanec Date: Mon, 18 Mar 2024 10:24:31 -0700 Subject: [PATCH 223/416] Integrate index state sharing into mem management (#1545) Adds the ability to share index state amongst indices during index load operations into the plugins memory management system. Introduces a manager of the shared state that will properly manage the lifecycle of the shared state. There was a bug in clear cache that had to be fixed to get this change working as well. Previously, only one index file per clear cache would be freed. This fixes that logic to clear everything. Added unit tests and an integration test to confirm functionality. In addition, modified recall integration tests to get more coverage on the different algo configs. Along with this, had to fix a few things around the computation of recall for non-l2 space types. Signed-off-by: John Mazanec --- CHANGELOG.md | 1 + .../org/opensearch/knn/index/IndexUtil.java | 16 + .../opensearch/knn/index/KNNIndexShard.java | 62 +- .../index/memory/NativeMemoryAllocation.java | 31 + .../memory/NativeMemoryEntryContext.java | 32 ++ .../memory/NativeMemoryLoadStrategy.java | 19 +- .../knn/index/memory/SharedIndexState.java | 21 + .../index/memory/SharedIndexStateManager.java | 150 +++++ .../opensearch/knn/index/query/KNNWeight.java | 3 +- .../org/opensearch/knn/index/FaissIT.java | 112 ++++ .../opensearch/knn/index/IndexUtilTests.java | 35 ++ .../knn/index/KNNIndexShardTests.java | 39 +- .../memory/SharedIndexStateManagerTests.java | 62 ++ .../index/memory/SharedIndexStateTests.java | 29 + .../opensearch/knn/recall/RecallTestsIT.java | 528 +++++++++++++++++- .../org/opensearch/knn/KNNRestTestCase.java | 16 +- .../java/org/opensearch/knn/TestUtils.java | 11 + 17 files changed, 1093 insertions(+), 74 deletions(-) create mode 100644 src/main/java/org/opensearch/knn/index/memory/SharedIndexState.java create mode 100644 src/main/java/org/opensearch/knn/index/memory/SharedIndexStateManager.java create mode 100644 src/test/java/org/opensearch/knn/index/memory/SharedIndexStateManagerTests.java create mode 100644 src/test/java/org/opensearch/knn/index/memory/SharedIndexStateTests.java diff --git a/CHANGELOG.md b/CHANGELOG.md index c5f52a431..9f815f792 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), * Disable sdc table for HNSWPQ read-only indices [#1518](https://github.com/opensearch-project/k-NN/pull/1518) * Switch SpaceType.INNERPRODUCT's vector similarity function to MAXIMUM_INNER_PRODUCT [#1532](https://github.com/opensearch-project/k-NN/pull/1532) * Add patch to fix arm segfault in nmslib during ingestion [#1541](https://github.com/opensearch-project/k-NN/pull/1541) +* Share ivfpq-l2 table allocations across indices on load [#1558](https://github.com/opensearch-project/k-NN/pull/1558) ### Infrastructure * Manually install zlib for win CI [#1513](https://github.com/opensearch-project/k-NN/pull/1513) * Update k-NN build artifact script to enable SIMD on ARM for Faiss [#1543](https://github.com/opensearch-project/k-NN/pull/1543) diff --git a/src/main/java/org/opensearch/knn/index/IndexUtil.java b/src/main/java/org/opensearch/knn/index/IndexUtil.java index d7833be46..bb9bbac90 100644 --- a/src/main/java/org/opensearch/knn/index/IndexUtil.java +++ b/src/main/java/org/opensearch/knn/index/IndexUtil.java @@ -23,6 +23,7 @@ import org.opensearch.knn.index.util.KNNEngine; import org.opensearch.knn.indices.ModelDao; import org.opensearch.knn.indices.ModelMetadata; +import org.opensearch.knn.jni.JNIService; import java.io.File; import java.util.Collections; @@ -268,4 +269,19 @@ public static boolean isVersionOnOrAfterMinRequiredVersion(Version version, Stri } return version.onOrAfter(minimalRequiredVersion); } + + /** + * Checks if index requires shared state + * + * @param knnEngine The knnEngine associated with the index + * @param modelId The modelId associated with the index + * @param indexAddr Address to check if loaded index requires shared state + * @return true if state can be shared; false otherwise + */ + public static boolean isSharedIndexStateRequired(KNNEngine knnEngine, String modelId, long indexAddr) { + if (StringUtils.isEmpty(modelId)) { + return false; + } + return JNIService.isSharedIndexStateRequired(indexAddr, knnEngine); + } } diff --git a/src/main/java/org/opensearch/knn/index/KNNIndexShard.java b/src/main/java/org/opensearch/knn/index/KNNIndexShard.java index a12b0df0f..f7597dce6 100644 --- a/src/main/java/org/opensearch/knn/index/KNNIndexShard.java +++ b/src/main/java/org/opensearch/knn/index/KNNIndexShard.java @@ -5,6 +5,9 @@ package org.opensearch.knn.index; +import com.google.common.annotations.VisibleForTesting; +import lombok.AllArgsConstructor; +import lombok.Getter; import org.apache.lucene.index.FieldInfo; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -24,12 +27,13 @@ import java.io.IOException; import java.nio.file.Path; +import java.util.ArrayList; import java.util.Collection; -import java.util.HashMap; -import java.util.Map; +import java.util.List; import java.util.concurrent.ExecutionException; import java.util.stream.Collectors; +import static org.opensearch.knn.common.KNNConstants.MODEL_ID; import static org.opensearch.knn.common.KNNConstants.SPACE_TYPE; import static org.opensearch.knn.index.IndexUtil.getParametersAtLoading; import static org.opensearch.knn.index.codec.util.KNNCodecUtil.buildEngineFilePrefix; @@ -82,14 +86,19 @@ public String getIndexName() { public void warmup() throws IOException { logger.info("[KNN] Warming up index: " + getIndexName()); try (Engine.Searcher searcher = indexShard.acquireSearcher("knn-warmup")) { - getAllEnginePaths(searcher.getIndexReader()).forEach((key, value) -> { + getAllEngineFileContexts(searcher.getIndexReader()).forEach((engineFileContext) -> { try { nativeMemoryCacheManager.get( new NativeMemoryEntryContext.IndexEntryContext( - key, + engineFileContext.getIndexPath(), NativeMemoryLoadStrategy.IndexLoadStrategy.getInstance(), - getParametersAtLoading(value, KNNEngine.getEngineNameFromPath(key), getIndexName()), - getIndexName() + getParametersAtLoading( + engineFileContext.getSpaceType(), + KNNEngine.getEngineNameFromPath(engineFileContext.getIndexPath()), + getIndexName() + ), + getIndexName(), + engineFileContext.getModelId() ), true ); @@ -103,20 +112,21 @@ public void warmup() throws IOException { /** * For the given shard, get all of its engine paths * - * @param indexReader IndexReader to read the file paths for the shard - * @return List of engine file Paths + * @param indexReader IndexReader to read the information for each segment in the shard + * @return List of engine contexts * @throws IOException Thrown when the SegmentReader is attempting to read the segments files */ - public Map getAllEnginePaths(IndexReader indexReader) throws IOException { - Map engineFiles = new HashMap<>(); + @VisibleForTesting + List getAllEngineFileContexts(IndexReader indexReader) throws IOException { + List engineFiles = new ArrayList<>(); for (KNNEngine knnEngine : KNNEngine.getEnginesThatCreateCustomSegmentFiles()) { - engineFiles.putAll(getEnginePaths(indexReader, knnEngine)); + engineFiles.addAll(getEngineFileContexts(indexReader, knnEngine)); } return engineFiles; } - private Map getEnginePaths(IndexReader indexReader, KNNEngine knnEngine) throws IOException { - Map engineFiles = new HashMap<>(); + List getEngineFileContexts(IndexReader indexReader, KNNEngine knnEngine) throws IOException { + List engineFiles = new ArrayList<>(); for (LeafReaderContext leafReaderContext : indexReader.leaves()) { SegmentReader reader = (SegmentReader) FilterLeafReader.unwrap(leafReaderContext.reader()); @@ -131,15 +141,17 @@ private Map getEnginePaths(IndexReader indexReader, KNNEngine // was L2. So, if Space Type is not present, just fall back to L2 String spaceTypeName = fieldInfo.attributes().getOrDefault(SPACE_TYPE, SpaceType.L2.getValue()); SpaceType spaceType = SpaceType.getSpace(spaceTypeName); + String modelId = fieldInfo.attributes().getOrDefault(MODEL_ID, null); - engineFiles.putAll( - getEnginePaths( + engineFiles.addAll( + getEngineFileContexts( reader.getSegmentInfo().files(), reader.getSegmentInfo().info.name, fieldInfo.name, fileExtension, shardPath, - spaceType + spaceType, + modelId ) ); } @@ -148,13 +160,15 @@ private Map getEnginePaths(IndexReader indexReader, KNNEngine return engineFiles; } - protected Map getEnginePaths( + @VisibleForTesting + List getEngineFileContexts( Collection files, String segmentName, String fieldName, String fileExtension, Path shardPath, - SpaceType spaceType + SpaceType spaceType, + String modelId ) { String prefix = buildEngineFilePrefix(segmentName); String suffix = buildEngineFileSuffix(fieldName, fileExtension); @@ -162,6 +176,16 @@ protected Map getEnginePaths( .filter(fileName -> fileName.startsWith(prefix)) .filter(fileName -> fileName.endsWith(suffix)) .map(fileName -> shardPath.resolve(fileName).toString()) - .collect(Collectors.toMap(fileName -> fileName, fileName -> spaceType)); + .map(fileName -> new EngineFileContext(spaceType, modelId, fileName)) + .collect(Collectors.toList()); + } + + @AllArgsConstructor + @Getter + @VisibleForTesting + static class EngineFileContext { + private final SpaceType spaceType; + private final String modelId; + private final String indexPath; } } diff --git a/src/main/java/org/opensearch/knn/index/memory/NativeMemoryAllocation.java b/src/main/java/org/opensearch/knn/index/memory/NativeMemoryAllocation.java index 416980759..286e6265c 100644 --- a/src/main/java/org/opensearch/knn/index/memory/NativeMemoryAllocation.java +++ b/src/main/java/org/opensearch/knn/index/memory/NativeMemoryAllocation.java @@ -91,6 +91,7 @@ class IndexAllocation implements NativeMemoryAllocation { private final String openSearchIndexName; private final ReadWriteLock readWriteLock; private final WatcherHandle watcherHandle; + private final SharedIndexState sharedIndexState; /** * Constructor @@ -111,6 +112,31 @@ class IndexAllocation implements NativeMemoryAllocation { String indexPath, String openSearchIndexName, WatcherHandle watcherHandle + ) { + this(executorService, memoryAddress, size, knnEngine, indexPath, openSearchIndexName, watcherHandle, null); + } + + /** + * Constructor + * + * @param executorService Executor service used to close the allocation + * @param memoryAddress Pointer in memory to the index + * @param size Size this index consumes in kilobytes + * @param knnEngine KNNEngine associated with the index allocation + * @param indexPath File path to index + * @param openSearchIndexName Name of OpenSearch index this index is associated with + * @param watcherHandle Handle for watching index file + * @param sharedIndexState Shared index state. If not shared state present, pass null. + */ + IndexAllocation( + ExecutorService executorService, + long memoryAddress, + int size, + KNNEngine knnEngine, + String indexPath, + String openSearchIndexName, + WatcherHandle watcherHandle, + SharedIndexState sharedIndexState ) { this.executor = executorService; this.closed = false; @@ -121,6 +147,7 @@ class IndexAllocation implements NativeMemoryAllocation { this.readWriteLock = new ReentrantReadWriteLock(); this.size = size; this.watcherHandle = watcherHandle; + this.sharedIndexState = sharedIndexState; } @Override @@ -145,6 +172,10 @@ private void cleanup() { if (memoryAddress != 0) { JNIService.free(memoryAddress, knnEngine); } + + if (sharedIndexState != null) { + SharedIndexStateManager.getInstance().release(sharedIndexState); + } } @Override diff --git a/src/main/java/org/opensearch/knn/index/memory/NativeMemoryEntryContext.java b/src/main/java/org/opensearch/knn/index/memory/NativeMemoryEntryContext.java index 13f8dae10..7f14a2341 100644 --- a/src/main/java/org/opensearch/knn/index/memory/NativeMemoryEntryContext.java +++ b/src/main/java/org/opensearch/knn/index/memory/NativeMemoryEntryContext.java @@ -12,6 +12,7 @@ package org.opensearch.knn.index.memory; import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.Nullable; import org.opensearch.knn.index.IndexUtil; import java.io.IOException; @@ -63,6 +64,8 @@ public static class IndexEntryContext extends NativeMemoryEntryContext parameters; + @Nullable + private final String modelId; /** * Constructor @@ -77,11 +80,31 @@ public IndexEntryContext( NativeMemoryLoadStrategy.IndexLoadStrategy indexLoadStrategy, Map parameters, String openSearchIndexName + ) { + this(indexPath, indexLoadStrategy, parameters, openSearchIndexName, null); + } + + /** + * Constructor + * + * @param indexPath path to index file. Also used as key in cache. + * @param indexLoadStrategy strategy to load index into memory + * @param parameters load time parameters + * @param openSearchIndexName opensearch index associated with index + * @param modelId model to be loaded. If none available, pass null + */ + public IndexEntryContext( + String indexPath, + NativeMemoryLoadStrategy.IndexLoadStrategy indexLoadStrategy, + Map parameters, + String openSearchIndexName, + String modelId ) { super(indexPath); this.indexLoadStrategy = indexLoadStrategy; this.openSearchIndexName = openSearchIndexName; this.parameters = parameters; + this.modelId = modelId; } @Override @@ -112,6 +135,15 @@ public Map getParameters() { return parameters; } + /** + * Getter + * + * @return return model ID for the index. null if no model is in use + */ + public String getModelId() { + return modelId; + } + private static class IndexSizeCalculator implements Function { static IndexSizeCalculator INSTANCE = new IndexSizeCalculator(); diff --git a/src/main/java/org/opensearch/knn/index/memory/NativeMemoryLoadStrategy.java b/src/main/java/org/opensearch/knn/index/memory/NativeMemoryLoadStrategy.java index 568cc892b..cb7dafdfc 100644 --- a/src/main/java/org/opensearch/knn/index/memory/NativeMemoryLoadStrategy.java +++ b/src/main/java/org/opensearch/knn/index/memory/NativeMemoryLoadStrategy.java @@ -11,7 +11,9 @@ package org.opensearch.knn.index.memory; +import lombok.extern.log4j.Log4j2; import org.opensearch.core.action.ActionListener; +import org.opensearch.knn.index.IndexUtil; import org.opensearch.knn.jni.JNIService; import org.opensearch.knn.index.util.KNNEngine; import org.opensearch.knn.training.TrainingDataConsumer; @@ -41,6 +43,7 @@ public interface NativeMemoryLoadStrategy, @@ -92,17 +95,25 @@ public NativeMemoryAllocation.IndexAllocation load(NativeMemoryEntryContext.Inde fileWatcher.init(); KNNEngine knnEngine = KNNEngine.getEngineNameFromPath(indexPath.toString()); - long memoryAddress = JNIService.loadIndex(indexPath.toString(), indexEntryContext.getParameters(), knnEngine); - final WatcherHandle watcherHandle = resourceWatcherService.add(fileWatcher); + long indexAddress = JNIService.loadIndex(indexPath.toString(), indexEntryContext.getParameters(), knnEngine); + SharedIndexState sharedIndexState = null; + String modelId = indexEntryContext.getModelId(); + if (IndexUtil.isSharedIndexStateRequired(knnEngine, modelId, indexAddress)) { + log.info("Index with model: \"{}\" requires shared state. Retrieving shared state.", modelId); + sharedIndexState = SharedIndexStateManager.getInstance().get(indexAddress, modelId, knnEngine); + JNIService.setSharedIndexState(indexAddress, sharedIndexState.getSharedIndexStateAddress(), knnEngine); + } + final WatcherHandle watcherHandle = resourceWatcherService.add(fileWatcher); return new NativeMemoryAllocation.IndexAllocation( executor, - memoryAddress, + indexAddress, indexEntryContext.calculateSizeInKB(), knnEngine, indexPath.toString(), indexEntryContext.getOpenSearchIndexName(), - watcherHandle + watcherHandle, + sharedIndexState ); } diff --git a/src/main/java/org/opensearch/knn/index/memory/SharedIndexState.java b/src/main/java/org/opensearch/knn/index/memory/SharedIndexState.java new file mode 100644 index 000000000..2ffadb22e --- /dev/null +++ b/src/main/java/org/opensearch/knn/index/memory/SharedIndexState.java @@ -0,0 +1,21 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.memory; + +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import org.opensearch.knn.index.util.KNNEngine; + +/** + * Class stores information about the shared memory allocations between loaded native indices. + */ +@RequiredArgsConstructor +@Getter +public class SharedIndexState { + private final long sharedIndexStateAddress; + private final String modelId; + private final KNNEngine knnEngine; +} diff --git a/src/main/java/org/opensearch/knn/index/memory/SharedIndexStateManager.java b/src/main/java/org/opensearch/knn/index/memory/SharedIndexStateManager.java new file mode 100644 index 000000000..896113834 --- /dev/null +++ b/src/main/java/org/opensearch/knn/index/memory/SharedIndexStateManager.java @@ -0,0 +1,150 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.memory; + +import com.google.common.annotations.VisibleForTesting; +import lombok.Getter; +import lombok.extern.log4j.Log4j2; +import org.opensearch.knn.index.util.KNNEngine; +import org.opensearch.knn.jni.JNIService; + +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.locks.ReadWriteLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; + +/** + * Class manages allocations that can be shared between native indices. No locking is required. + * Once a caller obtain an instance of a {@link org.opensearch.knn.index.memory.SharedIndexState}, it is guaranteed to + * be valid until it is returned. {@link org.opensearch.knn.index.memory.SharedIndexState} are reference counted + * internally. Once the reference count goes to 0, it will be freed. + */ +@Log4j2 +class SharedIndexStateManager { + // Map storing the shared index state with key being the modelId. + private final ConcurrentHashMap sharedIndexStateCache; + private final ReadWriteLock readWriteLock; + + private static SharedIndexStateManager INSTANCE; + + // TODO: Going to refactor away from doing this in the future. For now, keeping for simplicity. + public static synchronized SharedIndexStateManager getInstance() { + if (INSTANCE == null) { + INSTANCE = new SharedIndexStateManager(); + } + return INSTANCE; + } + + /** + * Constructor + */ + @VisibleForTesting + SharedIndexStateManager() { + this.sharedIndexStateCache = new ConcurrentHashMap<>(); + this.readWriteLock = new ReentrantReadWriteLock(); + } + + /** + * Return a {@link SharedIndexState} associated with the key. If no value exists, it will attempt to create it. + * Once returned, the {@link SharedIndexState} will be valid until + * {@link SharedIndexStateManager#release(SharedIndexState)} is called. Caller must ensure that this is + * called after it is done using it. + * + * In order to create the shared state, it will use the indexAddress passed in to create the shared state from + * using {@link org.opensearch.knn.jni.JNIService#initSharedIndexState(long, KNNEngine)}. + * + * @param indexAddress Address of index to initialize the shared state from + * @param knnEngine engine index belongs to + * @return ShareModelContext + */ + public SharedIndexState get(long indexAddress, String modelId, KNNEngine knnEngine) { + this.readWriteLock.readLock().lock(); + try { + // This can be done safely with readLock because the ConcurrentHasMap.computeIfAbsent guarantees: + // + // "If the specified key is not already associated with a value, attempts to compute its value using the given + // mapping function and enters it into this map unless null. The entire method invocation is performed + // atomically, so the function is applied at most once per key. Some attempted update operations on this map + // by other threads may be blocked while computation is in progress, so the computation should be short and + // simple, and must not attempt to update any other mappings of this map." + // + // Ref: + // https://docs.oracle.com/javase/8/docs/api/java/util/concurrent/ConcurrentHashMap.html#computeIfAbsent-K-java.util.function.Function- + SharedIndexStateEntry entry = sharedIndexStateCache.computeIfAbsent(modelId, m -> { + log.info("Loading entry to shared index state cache for model {}", modelId); + long sharedIndexStateAddress = JNIService.initSharedIndexState(indexAddress, knnEngine); + return new SharedIndexStateEntry(new SharedIndexState(sharedIndexStateAddress, modelId, knnEngine)); + }); + entry.incRef(); + return entry.getSharedIndexState(); + } finally { + this.readWriteLock.readLock().unlock(); + } + } + + /** + * Indicate that the {@link SharedIndexState} is no longer being used. If nothing else is using it, it will be + * removed from the cache and evicted. + * + * After calling this method, {@link SharedIndexState} should no longer be used by calling thread. + * + * @param sharedIndexState to return to the system. + */ + public void release(SharedIndexState sharedIndexState) { + this.readWriteLock.writeLock().lock(); + + try { + SharedIndexStateEntry sharedIndexStateEntry; + if ((sharedIndexStateEntry = sharedIndexStateCache.get(sharedIndexState.getModelId())) == null) { + // This should not happen. Will log the error and return to prevent crash + log.error("Attempting to evict model from cache but it is not present: {}", sharedIndexState.getModelId()); + return; + } + + if (sharedIndexStateEntry.decRef() <= 0) { + log.info("Evicting entry from shared index state cache for key {}", sharedIndexState.getModelId()); + sharedIndexStateCache.remove(sharedIndexState.getModelId()); + JNIService.freeSharedIndexState(sharedIndexState.getSharedIndexStateAddress(), sharedIndexState.getKnnEngine()); + } + } finally { + this.readWriteLock.writeLock().unlock(); + } + } + + private static final class SharedIndexStateEntry { + @Getter + private final SharedIndexState sharedIndexState; + private final AtomicLong referenceCount; + + /** + * Constructor + * + * @param sharedIndexState sharedIndexStateContext being wrapped + */ + private SharedIndexStateEntry(SharedIndexState sharedIndexState) { + this.sharedIndexState = sharedIndexState; + this.referenceCount = new AtomicLong(0); + } + + /** + * Increases reference count by 1 + * + * @return ++referenceCount + */ + private long incRef() { + return referenceCount.incrementAndGet(); + } + + /** + * Decrease reference count by 1 + * + * @return --referenceCount + */ + private long decRef() { + return referenceCount.decrementAndGet(); + } + } +} diff --git a/src/main/java/org/opensearch/knn/index/query/KNNWeight.java b/src/main/java/org/opensearch/knn/index/query/KNNWeight.java index 7e2fa19cc..1c4d0a646 100644 --- a/src/main/java/org/opensearch/knn/index/query/KNNWeight.java +++ b/src/main/java/org/opensearch/knn/index/query/KNNWeight.java @@ -256,7 +256,8 @@ private Map doANNSearch(final LeafReaderContext context, final B indexPath.toString(), NativeMemoryLoadStrategy.IndexLoadStrategy.getInstance(), getParametersAtLoading(spaceType, knnEngine, knnQuery.getIndexName()), - knnQuery.getIndexName() + knnQuery.getIndexName(), + modelId ), true ); diff --git a/src/test/java/org/opensearch/knn/index/FaissIT.java b/src/test/java/org/opensearch/knn/index/FaissIT.java index 0e05c1e3c..f00942068 100644 --- a/src/test/java/org/opensearch/knn/index/FaissIT.java +++ b/src/test/java/org/opensearch/knn/index/FaissIT.java @@ -39,6 +39,7 @@ import java.util.TreeMap; import java.util.stream.Collectors; +import static org.opensearch.knn.common.KNNConstants.ENCODER_PARAMETER_PQ_CODE_SIZE; import static org.opensearch.knn.common.KNNConstants.ENCODER_PARAMETER_PQ_M; import static org.opensearch.knn.common.KNNConstants.ENCODER_PQ; import static org.opensearch.knn.common.KNNConstants.ENCODER_SQ; @@ -50,6 +51,7 @@ import static org.opensearch.knn.common.KNNConstants.METHOD_HNSW; import static org.opensearch.knn.common.KNNConstants.METHOD_IVF; import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_NLIST; +import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_NPROBES; import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_SPACE_TYPE; import static org.opensearch.knn.common.KNNConstants.MODEL_ID; import static org.opensearch.knn.common.KNNConstants.NAME; @@ -496,6 +498,116 @@ public void testEndToEnd_whenMethodIsHNSWPQAndHyperParametersNotSet_thenSucceed( fail("Graphs are not getting evicted"); } + /** + * This test confirms that sharing index state for IVFPQ-l2 indices functions properly. The main functionality that + * needs to be confirmed is that once an index gets deleted, it will not cause a failure for the non-deleted index. + * + * The workflow will be: + * 1. Create a model + * 2. Create two indices index from the model + * 3. Load the native index files from the first index + * 4. Assert search works + * 5. Load the native index files (which will reuse the shared state from the initial index) + * 6. Assert search works on the second index + * 7. Delete the first index and wait + * 8. Assert search works on the second index + */ + @SneakyThrows + public void testSharedIndexState_whenOneIndexDeleted_thenSecondIndexIsStillSearchable() { + String firstIndexName = "test-index-1"; + String secondIndexName = "test-index-2"; + String trainingIndexName = "training-index"; + + String modelId = "test-model"; + String modelDescription = "ivfpql2 model for testing shared state"; + + int dimension = testData.indexData.vectors[0].length; + SpaceType spaceType = SpaceType.L2; + int ivfNlist = 4; + int ivfNprobes = 4; + int pqCodeSize = 8; + int pqM = 1; + int docCount = 100; + + // training data needs to be at least equal to the number of centroids for PQ + // which is 2^8 = 256. 8 because thats the only valid code_size for HNSWPQ + int trainingDataCount = 256; + XContentBuilder xContentBuilder = XContentFactory.jsonBuilder() + .startObject() + .field(NAME, METHOD_IVF) + .field(KNN_ENGINE, FAISS_NAME) + .field(METHOD_PARAMETER_SPACE_TYPE, spaceType.getValue()) + .startObject(PARAMETERS) + .field(METHOD_PARAMETER_NPROBES, ivfNprobes) + .field(METHOD_PARAMETER_NLIST, ivfNlist) + .startObject(METHOD_ENCODER_PARAMETER) + .field(NAME, ENCODER_PQ) + .startObject(PARAMETERS) + .field(ENCODER_PARAMETER_PQ_M, pqCodeSize) + .field(ENCODER_PARAMETER_PQ_CODE_SIZE, pqM) + .endObject() + .endObject() + .endObject() + .endObject(); + Map in = xContentBuilderToMap(xContentBuilder); + createBasicKnnIndex(trainingIndexName, FIELD_NAME, dimension); + ingestDataAndTrainModel(modelId, trainingIndexName, FIELD_NAME, dimension, modelDescription, in, trainingDataCount); + assertTrainingSucceeds(modelId, 360, 1000); + + createIndexFromModelAndIngestDocuments(firstIndexName, modelId, docCount); + createIndexFromModelAndIngestDocuments(secondIndexName, modelId, docCount); + + doKnnWarmup(List.of(firstIndexName)); + validateSearchWorkflow(firstIndexName, testData.queries, 10); + doKnnWarmup(List.of(secondIndexName)); + validateSearchWorkflow(secondIndexName, testData.queries, 10); + deleteKNNIndex(firstIndexName); + // wait for all index files to be cleaned up from original index. empirically determined to take 25 seconds. + // will give 15 second buffer from that + Thread.sleep(1000 * 45); + validateSearchWorkflow(secondIndexName, testData.queries, 10); + deleteModel(modelId); + } + + @SneakyThrows + private void createIndexFromModelAndIngestDocuments(String indexName, String modelId, int docCount) { + XContentBuilder builder = XContentFactory.jsonBuilder() + .startObject() + .startObject("properties") + .startObject(FIELD_NAME) + .field("type", "knn_vector") + .field("model_id", modelId) + .endObject() + .endObject() + .endObject(); + + Map mappingMap = xContentBuilderToMap(builder); + String mapping = builder.toString(); + createKnnIndex(indexName, mapping); + assertEquals(new TreeMap<>(mappingMap), new TreeMap<>(getIndexMappingAsMap(indexName))); + + for (int i = 0; i < Math.min(testData.indexData.docs.length, docCount); i++) { + addKnnDoc( + indexName, + Integer.toString(testData.indexData.docs[i]), + FIELD_NAME, + Floats.asList(testData.indexData.vectors[i]).toArray() + ); + } + refreshAllNonSystemIndices(); + assertEquals(Math.min(testData.indexData.docs.length, docCount), getDocCount(indexName)); + } + + @SneakyThrows + private void validateSearchWorkflow(String indexName, float[][] queries, int k) { + for (float[] query : queries) { + Response response = searchKNNIndex(indexName, new KNNQueryBuilder(FIELD_NAME, query, k), k); + String responseBody = EntityUtils.toString(response.getEntity()); + List knnResults = parseSearchResponse(responseBody, FIELD_NAME); + assertEquals(k, knnResults.size()); + } + } + public void testDocUpdate() throws IOException { String indexName = "test-index-1"; String fieldName = "test-field-1"; diff --git a/src/test/java/org/opensearch/knn/index/IndexUtilTests.java b/src/test/java/org/opensearch/knn/index/IndexUtilTests.java index c8b29b6ef..00493b293 100644 --- a/src/test/java/org/opensearch/knn/index/IndexUtilTests.java +++ b/src/test/java/org/opensearch/knn/index/IndexUtilTests.java @@ -12,6 +12,8 @@ package org.opensearch.knn.index; import com.google.common.collect.ImmutableMap; +import org.junit.BeforeClass; +import org.mockito.MockedStatic; import org.opensearch.Version; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.metadata.IndexMetadata; @@ -24,12 +26,16 @@ import org.opensearch.knn.index.util.KNNEngine; import org.opensearch.knn.indices.ModelDao; import org.opensearch.knn.indices.ModelMetadata; +import org.opensearch.knn.jni.JNIService; import java.util.Map; import java.util.Objects; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.mockStatic; import static org.mockito.Mockito.when; import static org.opensearch.knn.common.KNNConstants.HNSW_ALGO_EF_SEARCH; import static org.opensearch.knn.common.KNNConstants.SPACE_TYPE; @@ -37,6 +43,15 @@ import static org.opensearch.knn.index.KNNSettings.KNN_ALGO_PARAM_EF_SEARCH; public class IndexUtilTests extends KNNTestCase { + + private static MockedStatic jniServiceMockedStatic; + private static final long TEST_INDEX_ADDRESS = 0; + + @BeforeClass + public static void setUpClass() { + jniServiceMockedStatic = mockStatic(JNIService.class); + } + public void testGetLoadParameters() { // Test faiss to ensure that space type gets set properly SpaceType spaceType1 = SpaceType.COSINESIMIL; @@ -206,4 +221,24 @@ public void testValidateKnnField_EmptyIndexMetadata() { assert (Objects.requireNonNull(e).getMessage().matches("Validation Failed: 1: Invalid index. Index does not contain a mapping;")); } + + public void testIsShareableStateContainedInIndex_whenIndexNotModelBased_thenReturnFalse() { + String modelId = null; + KNNEngine knnEngine = KNNEngine.FAISS; + assertFalse(IndexUtil.isSharedIndexStateRequired(knnEngine, modelId, TEST_INDEX_ADDRESS)); + } + + public void testIsShareableStateContainedInIndex_whenFaissHNSWIsUsed_thenReturnFalse() { + jniServiceMockedStatic.when(() -> JNIService.isSharedIndexStateRequired(anyLong(), any())).thenReturn(false); + String modelId = "test-model"; + KNNEngine knnEngine = KNNEngine.FAISS; + assertFalse(IndexUtil.isSharedIndexStateRequired(knnEngine, modelId, TEST_INDEX_ADDRESS)); + } + + public void testIsShareableStateContainedInIndex_whenJNIIsSharedIndexStateRequiredIsTrue_thenReturnTrue() { + jniServiceMockedStatic.when(() -> JNIService.isSharedIndexStateRequired(anyLong(), any())).thenReturn(true); + String modelId = "test-model"; + KNNEngine knnEngine = KNNEngine.FAISS; + assertTrue(IndexUtil.isSharedIndexStateRequired(knnEngine, modelId, TEST_INDEX_ADDRESS)); + } } diff --git a/src/test/java/org/opensearch/knn/index/KNNIndexShardTests.java b/src/test/java/org/opensearch/knn/index/KNNIndexShardTests.java index fc88a8ea6..079d18e66 100644 --- a/src/test/java/org/opensearch/knn/index/KNNIndexShardTests.java +++ b/src/test/java/org/opensearch/knn/index/KNNIndexShardTests.java @@ -16,9 +16,7 @@ import java.io.IOException; import java.nio.file.Path; import java.nio.file.Paths; -import java.util.ArrayList; import java.util.List; -import java.util.Map; import java.util.Set; import java.util.concurrent.ExecutionException; import java.util.stream.Collectors; @@ -97,38 +95,35 @@ public void testWarmup_shardNotPresentInCache() throws InterruptedException, Exe assertEquals(2, NativeMemoryCacheManager.getInstance().getIndicesCacheStats().get(testIndexName).get(GRAPH_COUNT)); } - public void testGetHNSWPaths() throws IOException, ExecutionException, InterruptedException { + public void testGetAllEngineFileContexts() throws IOException, ExecutionException, InterruptedException { IndexService indexService = createKNNIndex(testIndexName); createKnnIndexMapping(testIndexName, testFieldName, dimensions); - IndexShard indexShard; - KNNIndexShard knnIndexShard; - Engine.Searcher searcher; - Map hnswPaths; - indexShard = indexService.iterator().next(); - knnIndexShard = new KNNIndexShard(indexShard); + IndexShard indexShard = indexService.iterator().next(); + KNNIndexShard knnIndexShard = new KNNIndexShard(indexShard); - searcher = indexShard.acquireSearcher("test-hnsw-paths-1"); - hnswPaths = knnIndexShard.getAllEnginePaths(searcher.getIndexReader()); - assertEquals(0, hnswPaths.size()); + Engine.Searcher searcher = indexShard.acquireSearcher("test-hnsw-paths-1"); + List engineFileContexts = knnIndexShard.getAllEngineFileContexts(searcher.getIndexReader()); + assertEquals(0, engineFileContexts.size()); searcher.close(); addKnnDoc(testIndexName, "1", testFieldName, new Float[] { 2.5F, 3.5F }); searcher = indexShard.acquireSearcher("test-hnsw-paths-2"); - hnswPaths = knnIndexShard.getAllEnginePaths(searcher.getIndexReader()); - assertEquals(1, hnswPaths.size()); - List paths = new ArrayList<>(hnswPaths.keySet()); + engineFileContexts = knnIndexShard.getAllEngineFileContexts(searcher.getIndexReader()); + assertEquals(1, engineFileContexts.size()); + List paths = engineFileContexts.stream().map(KNNIndexShard.EngineFileContext::getIndexPath).collect(Collectors.toList()); assertTrue(paths.get(0).contains("hnsw") || paths.get(0).contains("hnswc")); searcher.close(); } - public void testGetEnginePaths() { + public void testGetEngineFileContexts() { // Check that the correct engine paths are being returned by the KNNIndexShard String segmentName = "_0"; String fieldName = "test_field"; String fileExt = ".test"; SpaceType spaceType = SpaceType.L2; + String modelId = "test-model"; Set includedFileNames = ImmutableSet.of( String.format("%s_111_%s%s", segmentName, fieldName, fileExt), @@ -147,9 +142,17 @@ public void testGetEnginePaths() { KNNIndexShard knnIndexShard = new KNNIndexShard(null); Path path = Paths.get(""); - Map included = knnIndexShard.getEnginePaths(files, segmentName, fieldName, fileExt, path, spaceType); + List included = knnIndexShard.getEngineFileContexts( + files, + segmentName, + fieldName, + fileExt, + path, + spaceType, + modelId + ); assertEquals(includedFileNames.size(), included.size()); - included.keySet().forEach(o -> assertTrue(includedFileNames.contains(o))); + included.stream().map(KNNIndexShard.EngineFileContext::getIndexPath).forEach(o -> assertTrue(includedFileNames.contains(o))); } } diff --git a/src/test/java/org/opensearch/knn/index/memory/SharedIndexStateManagerTests.java b/src/test/java/org/opensearch/knn/index/memory/SharedIndexStateManagerTests.java new file mode 100644 index 000000000..daf02c611 --- /dev/null +++ b/src/test/java/org/opensearch/knn/index/memory/SharedIndexStateManagerTests.java @@ -0,0 +1,62 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.knn.index.memory; + +import org.junit.BeforeClass; +import org.mockito.MockedStatic; +import org.mockito.Mockito; +import org.opensearch.knn.KNNTestCase; +import org.opensearch.knn.index.util.KNNEngine; +import org.opensearch.knn.jni.JNIService; + +import static org.mockito.Mockito.mockStatic; + +public class SharedIndexStateManagerTests extends KNNTestCase { + private static MockedStatic jniServiceMockedStatic; + private final static long TEST_SHARED_TABLE_ADDRESS = 123; + private final static long TEST_INDEX_ADDRESS = 1234; + private final static String TEST_MODEL_ID = "test-model-id"; + private final static KNNEngine TEST_KNN_ENGINE = KNNEngine.DEFAULT; + + @BeforeClass + public static void setUpClass() { + jniServiceMockedStatic = mockStatic(JNIService.class); + jniServiceMockedStatic.when(() -> JNIService.freeSharedIndexState(TEST_SHARED_TABLE_ADDRESS, TEST_KNN_ENGINE)) + .then(invocation -> null); + jniServiceMockedStatic.when(() -> JNIService.initSharedIndexState(TEST_INDEX_ADDRESS, TEST_KNN_ENGINE)) + .thenReturn(TEST_SHARED_TABLE_ADDRESS); + } + + public void testGet_whenNormalWorkfloatApplied_thenSucceed() { + SharedIndexStateManager sharedIndexStateManager = new SharedIndexStateManager(); + SharedIndexState firstSharedIndexStateRetrieved = sharedIndexStateManager.get(TEST_INDEX_ADDRESS, TEST_MODEL_ID, TEST_KNN_ENGINE); + assertEquals(TEST_SHARED_TABLE_ADDRESS, firstSharedIndexStateRetrieved.getSharedIndexStateAddress()); + assertEquals(TEST_MODEL_ID, firstSharedIndexStateRetrieved.getModelId()); + assertEquals(TEST_KNN_ENGINE, firstSharedIndexStateRetrieved.getKnnEngine()); + + SharedIndexState secondSharedIndexStateRetrieved = sharedIndexStateManager.get(TEST_INDEX_ADDRESS, TEST_MODEL_ID, TEST_KNN_ENGINE); + assertEquals(TEST_SHARED_TABLE_ADDRESS, secondSharedIndexStateRetrieved.getSharedIndexStateAddress()); + assertEquals(TEST_MODEL_ID, secondSharedIndexStateRetrieved.getModelId()); + assertEquals(TEST_KNN_ENGINE, secondSharedIndexStateRetrieved.getKnnEngine()); + } + + public void testRelease_whenNormalWorkflowApplied_thenSucceed() { + SharedIndexStateManager sharedIndexStateManager = new SharedIndexStateManager(); + SharedIndexState firstSharedIndexStateRetrieved = sharedIndexStateManager.get(TEST_INDEX_ADDRESS, TEST_MODEL_ID, TEST_KNN_ENGINE); + SharedIndexState secondSharedIndexStateRetrieved = sharedIndexStateManager.get(TEST_INDEX_ADDRESS, TEST_MODEL_ID, TEST_KNN_ENGINE); + + sharedIndexStateManager.release(firstSharedIndexStateRetrieved); + jniServiceMockedStatic.verify(() -> JNIService.freeSharedIndexState(TEST_SHARED_TABLE_ADDRESS, TEST_KNN_ENGINE), Mockito.times(0)); + sharedIndexStateManager.release(secondSharedIndexStateRetrieved); + jniServiceMockedStatic.verify(() -> JNIService.freeSharedIndexState(TEST_SHARED_TABLE_ADDRESS, TEST_KNN_ENGINE), Mockito.times(1)); + } +} diff --git a/src/test/java/org/opensearch/knn/index/memory/SharedIndexStateTests.java b/src/test/java/org/opensearch/knn/index/memory/SharedIndexStateTests.java new file mode 100644 index 000000000..cddbec5c0 --- /dev/null +++ b/src/test/java/org/opensearch/knn/index/memory/SharedIndexStateTests.java @@ -0,0 +1,29 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.knn.index.memory; + +import org.opensearch.knn.KNNTestCase; +import org.opensearch.knn.index.util.KNNEngine; + +public class SharedIndexStateTests extends KNNTestCase { + + private static final String TEST_MODEL_ID = "test-model"; + private static final long TEST_SHARED_INDEX_STATE_ADDRESS = 22L; + private static final KNNEngine TEST_KNN_ENGINE = KNNEngine.DEFAULT; + + public void testSharedIndexState() { + SharedIndexState sharedIndexState = new SharedIndexState(TEST_SHARED_INDEX_STATE_ADDRESS, TEST_MODEL_ID, TEST_KNN_ENGINE); + assertEquals(TEST_MODEL_ID, sharedIndexState.getModelId()); + assertEquals(TEST_SHARED_INDEX_STATE_ADDRESS, sharedIndexState.getSharedIndexStateAddress()); + assertEquals(TEST_KNN_ENGINE, sharedIndexState.getKnnEngine()); + } +} diff --git a/src/test/java/org/opensearch/knn/recall/RecallTestsIT.java b/src/test/java/org/opensearch/knn/recall/RecallTestsIT.java index 889f3916f..c90afaa62 100644 --- a/src/test/java/org/opensearch/knn/recall/RecallTestsIT.java +++ b/src/test/java/org/opensearch/knn/recall/RecallTestsIT.java @@ -11,54 +11,520 @@ package org.opensearch.knn.recall; +import lombok.SneakyThrows; +import org.junit.Before; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.knn.KNNRestTestCase; import org.opensearch.knn.TestUtils; import org.opensearch.knn.index.SpaceType; +import org.opensearch.knn.index.util.KNNEngine; + import java.util.List; +import java.util.Map; import java.util.Set; + +import static org.opensearch.knn.common.KNNConstants.DIMENSION; +import static org.opensearch.knn.common.KNNConstants.ENCODER_PARAMETER_PQ_CODE_SIZE; +import static org.opensearch.knn.common.KNNConstants.ENCODER_PARAMETER_PQ_M; +import static org.opensearch.knn.common.KNNConstants.ENCODER_PQ; +import static org.opensearch.knn.common.KNNConstants.FAISS_NAME; +import static org.opensearch.knn.common.KNNConstants.KNN_ENGINE; +import static org.opensearch.knn.common.KNNConstants.KNN_METHOD; +import static org.opensearch.knn.common.KNNConstants.METHOD_ENCODER_PARAMETER; +import static org.opensearch.knn.common.KNNConstants.METHOD_HNSW; +import static org.opensearch.knn.common.KNNConstants.METHOD_IVF; +import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_EF_CONSTRUCTION; +import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_EF_SEARCH; +import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_M; +import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_NLIST; +import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_NPROBES; +import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_SPACE_TYPE; +import static org.opensearch.knn.common.KNNConstants.MODEL_ID; +import static org.opensearch.knn.common.KNNConstants.NAME; +import static org.opensearch.knn.common.KNNConstants.PARAMETERS; +import static org.opensearch.knn.common.KNNConstants.TYPE; +import static org.opensearch.knn.common.KNNConstants.TYPE_KNN_VECTOR; +import static org.opensearch.knn.index.KNNSettings.KNN_ALGO_PARAM_EF_SEARCH; import static org.opensearch.knn.index.KNNSettings.KNN_ALGO_PARAM_INDEX_THREAD_QTY; import static org.opensearch.knn.index.KNNSettings.KNN_MEMORY_CIRCUIT_BREAKER_ENABLED; +/** + * Tests confirm that for the different supported configurations, recall is sound. The recall thresholds are + * conservatively and empirically determined to prevent flakiness. + * + * This test suite can take a long time to run. The primary reason is that training can take a long time for PQ. + * The parameters for PQ have been reduced significantly, but it still takes time. + */ public class RecallTestsIT extends KNNRestTestCase { - private final String testFieldName = "test-field"; - private final int dimensions = 50; - private final int docCount = 10000; - private final int queryCount = 100; - private final int k = 5; - private final double expRecallValue = 1.0; - - public void testRecallL2StandardData() throws Exception { - String testIndexStandard = "test-index-standard"; - - addDocs(testIndexStandard, testFieldName, dimensions, docCount, true); - float[][] indexVectors = getIndexVectorsFromIndex(testIndexStandard, testFieldName, docCount, dimensions); - float[][] queryVectors = TestUtils.getQueryVectors(queryCount, dimensions, docCount, true); - List> groundTruthValues = TestUtils.computeGroundTruthValues(indexVectors, queryVectors, SpaceType.L2, k); - List> searchResults = bulkSearch(testIndexStandard, testFieldName, queryVectors, k); - double recallValue = TestUtils.calculateRecallValue(searchResults, groundTruthValues, k); - assertEquals(expRecallValue, recallValue, 0.2); + private static final String PROPERTIES_FIELD = "properties"; + private final static String TEST_INDEX_PREFIX_NAME = "test_index"; + private final static String TEST_FIELD_NAME = "test_field"; + private final static String TRAIN_INDEX_NAME = "train_index"; + private final static String TRAIN_FIELD_NAME = "train_field"; + private final static String TEST_MODEL_ID = "test_model_id"; + private final static int TEST_DIMENSION = 32; + private final static int DOC_COUNT = 500; + private final static int QUERY_COUNT = 100; + private final static int TEST_K = 100; + private final static double PERFECT_RECALL = 1.0; + private final static int SHARD_COUNT = 1; + private final static int REPLICA_COUNT = 0; + private final static int MAX_SEGMENT_COUNT = 10; + + // Standard algorithm parameters + private final static int HNSW_M = 16; + private final static int HNSW_EF_CONSTRUCTION = 100; + private final static int HNSW_EF_SEARCH = TEST_K; // For consistency with lucene + private final static int IVF_NLIST = 4; + private final static int IVF_NPROBES = IVF_NLIST; // This equates to essentially a brute force search + private final static int PQ_CODE_SIZE = 8; // This is low and going to produce bad recall, but reduces build time + private final static int PQ_M = TEST_DIMENSION / 8; // Will give low recall, but required for test time + + // Setup ground truth for all tests once + private final static float[][] INDEX_VECTORS = TestUtils.getIndexVectors(DOC_COUNT, TEST_DIMENSION, true); + private final static float[][] QUERY_VECTORS = TestUtils.getQueryVectors(QUERY_COUNT, TEST_DIMENSION, DOC_COUNT, true); + private final static Map>> GROUND_TRUTH = Map.of( + SpaceType.L2, + TestUtils.computeGroundTruthValues(INDEX_VECTORS, QUERY_VECTORS, SpaceType.L2, TEST_K), + SpaceType.COSINESIMIL, + TestUtils.computeGroundTruthValues(INDEX_VECTORS, QUERY_VECTORS, SpaceType.COSINESIMIL, TEST_K), + SpaceType.INNER_PRODUCT, + TestUtils.computeGroundTruthValues(INDEX_VECTORS, QUERY_VECTORS, SpaceType.INNER_PRODUCT, TEST_K) + ); + + @SneakyThrows + @Before + public void setupClusterSettings() { + updateClusterSettings(KNN_ALGO_PARAM_INDEX_THREAD_QTY, 2); + updateClusterSettings(KNN_MEMORY_CIRCUIT_BREAKER_ENABLED, true); + } + + /** + * { + * "properties": { + * { + * "type": "knn_vector", + * "dimension": {DIMENSION}, + * "method": { + * "name":"hnsw", + * "engine":"nmslib", + * "space_type": "{SPACE_TYPE}", + * "parameters":{ + * "m":{HNSW_M}, + * "ef_construction": {HNSW_EF_CONSTRUCTION}, + * "ef_search": {HNSW_EF_SEARCH} + * } + * } + * } + * } + * } + */ + @SneakyThrows + public void testRecall_whenNmslibHnswFP32_thenRecallAbove75percent() { + List spaceTypes = List.of(SpaceType.L2, SpaceType.COSINESIMIL, SpaceType.INNER_PRODUCT); + for (SpaceType spaceType : spaceTypes) { + String indexName = createIndexName(KNNEngine.NMSLIB, spaceType); + XContentBuilder builder = XContentFactory.jsonBuilder() + .startObject() + .startObject(PROPERTIES_FIELD) + .startObject(TEST_FIELD_NAME) + .field(TYPE, TYPE_KNN_VECTOR) + .field(DIMENSION, TEST_DIMENSION) + .startObject(KNN_METHOD) + .field(METHOD_PARAMETER_SPACE_TYPE, spaceType.getValue()) + .field(KNN_ENGINE, KNNEngine.NMSLIB.getName()) + .field(NAME, METHOD_HNSW) + .startObject(PARAMETERS) + .field(METHOD_PARAMETER_EF_CONSTRUCTION, HNSW_EF_CONSTRUCTION) + .field(METHOD_PARAMETER_M, HNSW_M) + .endObject() + .endObject() + .endObject() + .endObject() + .endObject(); + createIndexAndIngestDocs( + indexName, + TEST_FIELD_NAME, + Settings.builder() + .put("number_of_shards", SHARD_COUNT) + .put("number_of_replicas", REPLICA_COUNT) + .put("index.knn", true) + .put(KNN_ALGO_PARAM_EF_SEARCH, HNSW_EF_SEARCH) + .build(), + builder.toString() + ); + assertRecall(indexName, spaceType, 0.25f); + } } - public void testRecallL2RandomData() throws Exception { - String testIndexRandom = "test-index-random"; + /** + * { + * "properties": { + * { + * "type": "knn_vector", + * "dimension": {DIMENSION}, + * "method": { + * "name":"hnsw", + * "engine":"lucene", + * "space_type": "{SPACE_TYPE}", + * "parameters":{ + * "m":{HNSW_M}, + * "ef_construction": {HNSW_EF_CONSTRUCTION} + * } + * } + * } + * } + * } + */ + @SneakyThrows + public void testRecall_whenLuceneHnswFP32_thenRecallAbove75percent() { + List spaceTypes = List.of(SpaceType.L2, SpaceType.COSINESIMIL); + for (SpaceType spaceType : spaceTypes) { + String indexName = createIndexName(KNNEngine.LUCENE, spaceType); + XContentBuilder builder = XContentFactory.jsonBuilder() + .startObject() + .startObject(PROPERTIES_FIELD) + .startObject(TEST_FIELD_NAME) + .field(TYPE, TYPE_KNN_VECTOR) + .field(DIMENSION, TEST_DIMENSION) + .startObject(KNN_METHOD) + .field(METHOD_PARAMETER_SPACE_TYPE, spaceType.getValue()) + .field(KNN_ENGINE, KNNEngine.LUCENE.getName()) + .field(NAME, METHOD_HNSW) + .startObject(PARAMETERS) + .field(METHOD_PARAMETER_EF_CONSTRUCTION, HNSW_EF_CONSTRUCTION) + .field(METHOD_PARAMETER_M, HNSW_M) + .endObject() + .endObject() + .endObject() + .endObject() + .endObject(); + createIndexAndIngestDocs(indexName, TEST_FIELD_NAME, getSettings(), builder.toString()); + assertRecall(indexName, spaceType, 0.25f); + } + } - addDocs(testIndexRandom, testFieldName, dimensions, docCount, false); - float[][] indexVectors = getIndexVectorsFromIndex(testIndexRandom, testFieldName, docCount, dimensions); - float[][] queryVectors = TestUtils.getQueryVectors(queryCount, dimensions, docCount, false); - List> groundTruthValues = TestUtils.computeGroundTruthValues(indexVectors, queryVectors, SpaceType.L2, k); - List> searchResults = bulkSearch(testIndexRandom, testFieldName, queryVectors, k); - double recallValue = TestUtils.calculateRecallValue(searchResults, groundTruthValues, k); - assertEquals(expRecallValue, recallValue, 0.2); + /** + * { + * "properties": { + * { + * "type": "knn_vector", + * "dimension": {TEST_DIMENSION}, + * "method": { + * "name":"hnsw", + * "engine":"faiss", + * "space_type": "{SPACE_TYPE}", + * "parameters":{ + * "m":{HNSW_M}, + * "ef_construction": {HNSW_EF_CONSTRUCTION}, + * "ef_search": {HNSW_EF_SEARCH}, + * } + * } + * } + * } + * } + */ + @SneakyThrows + public void testRecall_whenFaissHnswFP32_thenRecallAbove75percent() { + List spaceTypes = List.of(SpaceType.L2, SpaceType.INNER_PRODUCT); + for (SpaceType spaceType : spaceTypes) { + String indexName = createIndexName(KNNEngine.FAISS, spaceType); + XContentBuilder builder = XContentFactory.jsonBuilder() + .startObject() + .startObject(PROPERTIES_FIELD) + .startObject(TEST_FIELD_NAME) + .field(TYPE, TYPE_KNN_VECTOR) + .field(DIMENSION, TEST_DIMENSION) + .startObject(KNN_METHOD) + .field(METHOD_PARAMETER_SPACE_TYPE, spaceType.getValue()) + .field(KNN_ENGINE, KNNEngine.FAISS.getName()) + .field(NAME, METHOD_HNSW) + .startObject(PARAMETERS) + .field(METHOD_PARAMETER_EF_CONSTRUCTION, HNSW_EF_CONSTRUCTION) + .field(METHOD_PARAMETER_M, HNSW_M) + .field(METHOD_PARAMETER_EF_SEARCH, HNSW_EF_SEARCH) + .endObject() + .endObject() + .endObject() + .endObject() + .endObject(); + createIndexAndIngestDocs(indexName, TEST_FIELD_NAME, getSettings(), builder.toString()); + assertRecall(indexName, spaceType, 0.25f); + } } - private void addDocs(String testIndex, String testField, int dimensions, int docCount, boolean isStandard) throws Exception { - createKnnIndex(testIndex, getKNNDefaultIndexSettings(), createKnnIndexMapping(testField, dimensions)); + /** + * Train context: + * { + * "method": { + * "name":"ivf", + * "engine":"faiss", + * "space_type": "{SPACE_TYPE}", + * "parameters":{ + * "nlist":{IVF_NLIST}, + * "nprobes": {IVF_NPROBES} + * } + * } + * } + * + * Index Mapping: + * { + * "properties": { + * { + * "type": "knn_vector", + * "model_id": {MODEL_ID} + * } + * } + * } + */ + @SneakyThrows + public void testRecall_whenFaissIVFFP32_thenRecallAbove75percent() { + List spaceTypes = List.of(SpaceType.L2, SpaceType.INNER_PRODUCT); + setupTrainingIndex(); + for (SpaceType spaceType : spaceTypes) { + String indexName = createIndexName(KNNEngine.FAISS, spaceType); - updateClusterSettings(KNN_ALGO_PARAM_INDEX_THREAD_QTY, 2); - updateClusterSettings(KNN_MEMORY_CIRCUIT_BREAKER_ENABLED, true); + // Train the model + XContentBuilder trainingBuilder = XContentFactory.jsonBuilder() + .startObject() + .field(NAME, METHOD_IVF) + .field(KNN_ENGINE, FAISS_NAME) + .field(METHOD_PARAMETER_SPACE_TYPE, spaceType.getValue()) + .startObject(PARAMETERS) + .field(METHOD_PARAMETER_NLIST, IVF_NLIST) + .field(METHOD_PARAMETER_NPROBES, IVF_NPROBES) + .endObject() + .endObject(); + trainModel( + TEST_MODEL_ID, + TRAIN_INDEX_NAME, + TRAIN_FIELD_NAME, + TEST_DIMENSION, + xContentBuilderToMap(trainingBuilder), + String.format("%s-%s", KNNEngine.FAISS.getName(), spaceType.getValue()) + ); + assertTrainingSucceeds(TEST_MODEL_ID, 100, 1000 * 5); + + // Build the index + createIndexAndIngestDocs(indexName, TEST_FIELD_NAME, getSettings(), getModelMapping()); + assertRecall(indexName, spaceType, 0.25f); + + // Delete the model + deleteModel(TEST_MODEL_ID); + } + } + + /** + * Train context: + * { + * "properties": { + * { + * "type": "knn_vector", + * "dimension": {TEST_DIMENSION}, + * "method": { + * "name":"hnsw", + * "engine":"faiss", + * "space_type": "{SPACE_TYPE}", + * "parameters":{ + * "m":{HNSW_M}, + * "ef_construction": {HNSW_EF_CONSTRUCTION}, + * "ef_search": {HNSW_EF_SEARCH}, + * } + * } + * } + * } + * } + * + * Index Mapping: + * { + * "properties": { + * { + * "type": "knn_vector", + * "model_id": {MODEL_ID} + * } + * } + * } + */ + @SneakyThrows + public void testRecall_whenFaissIVFPQFP32_thenRecallAbove50percent() { + List spaceTypes = List.of(SpaceType.L2, SpaceType.INNER_PRODUCT); + setupTrainingIndex(); + for (SpaceType spaceType : spaceTypes) { + String indexName = createIndexName(KNNEngine.FAISS, spaceType); - bulkAddKnnDocs(testIndex, testField, TestUtils.getIndexVectors(docCount, dimensions, isStandard), docCount); + // Train the model + XContentBuilder trainingBuilder = XContentFactory.jsonBuilder() + .startObject() + .field(NAME, METHOD_IVF) + .field(KNN_ENGINE, FAISS_NAME) + .field(METHOD_PARAMETER_SPACE_TYPE, spaceType.getValue()) + .startObject(PARAMETERS) + .field(METHOD_PARAMETER_NLIST, IVF_NLIST) + .field(METHOD_PARAMETER_NPROBES, IVF_NPROBES) + .startObject(METHOD_ENCODER_PARAMETER) + .field(NAME, ENCODER_PQ) + .startObject(PARAMETERS) + .field(ENCODER_PARAMETER_PQ_CODE_SIZE, PQ_CODE_SIZE) + .field(ENCODER_PARAMETER_PQ_M, PQ_M) + .endObject() + .endObject() + .endObject() + .endObject(); + trainModel( + TEST_MODEL_ID, + TRAIN_INDEX_NAME, + TRAIN_FIELD_NAME, + TEST_DIMENSION, + xContentBuilderToMap(trainingBuilder), + String.format("%s-%s", KNNEngine.FAISS.getName(), spaceType.getValue()) + ); + assertTrainingSucceeds(TEST_MODEL_ID, 100, 1000 * 5); + + // Build the index + createIndexAndIngestDocs(indexName, TEST_FIELD_NAME, getSettings(), getModelMapping()); + assertRecall(indexName, spaceType, 0.5f); + + // Delete the model + deleteModel(TEST_MODEL_ID); + } + } + + /** + * Train context: + * { + * "properties": { + * { + * "type": "knn_vector", + * "dimension": {TEST_DIMENSION}, + * "method": { + * "name":"hnsw", + * "engine":"faiss", + * "space_type": "{SPACE_TYPE}", + * "parameters":{ + * "m":{HNSW_M}, + * "ef_construction": {HNSW_EF_CONSTRUCTION}, + * "ef_search": {HNSW_EF_SEARCH}, + * } + * } + * } + * } + * } + * + * Index Mapping: + * { + * "properties": { + * { + * "type": "knn_vector", + * "model_id": {MODEL_ID} + * } + * } + * } + */ + @SneakyThrows + public void testRecall_whenFaissHNSWPQFP32_thenRecallAbove50percent() { + List spaceTypes = List.of(SpaceType.L2, SpaceType.INNER_PRODUCT); + setupTrainingIndex(); + for (SpaceType spaceType : spaceTypes) { + String indexName = createIndexName(KNNEngine.FAISS, spaceType); + + // Train the model + XContentBuilder trainingBuilder = XContentFactory.jsonBuilder() + .startObject() + .field(NAME, METHOD_HNSW) + .field(KNN_ENGINE, FAISS_NAME) + .field(METHOD_PARAMETER_SPACE_TYPE, spaceType.getValue()) + .startObject(PARAMETERS) + .field(METHOD_PARAMETER_M, HNSW_M) + .field(METHOD_PARAMETER_EF_SEARCH, HNSW_EF_SEARCH) + .field(METHOD_PARAMETER_EF_CONSTRUCTION, HNSW_EF_CONSTRUCTION) + .startObject(METHOD_ENCODER_PARAMETER) + .field(NAME, ENCODER_PQ) + .startObject(PARAMETERS) + .field(ENCODER_PARAMETER_PQ_CODE_SIZE, PQ_CODE_SIZE) + .field(ENCODER_PARAMETER_PQ_M, PQ_M) + .endObject() + .endObject() + .endObject() + .endObject(); + trainModel( + TEST_MODEL_ID, + TRAIN_INDEX_NAME, + TRAIN_FIELD_NAME, + TEST_DIMENSION, + xContentBuilderToMap(trainingBuilder), + String.format("%s-%s", KNNEngine.FAISS.getName(), spaceType.getValue()) + ); + assertTrainingSucceeds(TEST_MODEL_ID, 100, 1000 * 5); + + // Build the index + createIndexAndIngestDocs(indexName, TEST_FIELD_NAME, getSettings(), getModelMapping()); + assertRecall(indexName, spaceType, 0.5f); + + // Delete the model + deleteModel(TEST_MODEL_ID); + } + } + + @SneakyThrows + private void assertRecall(String testIndexName, SpaceType spaceType, float acceptableRecallFromPerfect) { + List> searchResults = bulkSearch(testIndexName, TEST_FIELD_NAME, QUERY_VECTORS, TEST_K); + double recallValue = TestUtils.calculateRecallValue(searchResults, GROUND_TRUTH.get(spaceType), TEST_K); + logger.info("Recall value = {}", recallValue); + assertEquals(PERFECT_RECALL, recallValue, acceptableRecallFromPerfect); } + private String createIndexName(KNNEngine knnEngine, SpaceType spaceType) { + return String.format("%s_%s_%s", TEST_INDEX_PREFIX_NAME, knnEngine.getName(), spaceType.getValue()); + } + + @SneakyThrows + private void createIndexAndIngestDocs(String indexName, String fieldName, Settings settings, String mapping) { + createKnnIndex(indexName, settings, mapping); + bulkAddKnnDocs(indexName, fieldName, INDEX_VECTORS, DOC_COUNT); + forceMergeKnnIndex(indexName, MAX_SEGMENT_COUNT); + } + + @SneakyThrows + private void setupTrainingIndex() { + XContentBuilder trainingIndexBuilder = XContentFactory.jsonBuilder() + .startObject() + .startObject(PROPERTIES_FIELD) + .startObject(TRAIN_FIELD_NAME) + .field(TYPE, TYPE_KNN_VECTOR) + .field(DIMENSION, TEST_DIMENSION) + .endObject() + .endObject() + .endObject(); + createIndexAndIngestDocs( + TRAIN_INDEX_NAME, + TRAIN_FIELD_NAME, + Settings.builder().put("number_of_shards", SHARD_COUNT).put("number_of_replicas", REPLICA_COUNT).build(), + trainingIndexBuilder.toString() + ); + } + + @SneakyThrows + private String getModelMapping() { + return XContentFactory.jsonBuilder() + .startObject() + .startObject(PROPERTIES_FIELD) + .startObject(TEST_FIELD_NAME) + .field(TYPE, TYPE_KNN_VECTOR) + .field(MODEL_ID, TEST_MODEL_ID) + .endObject() + .endObject() + .endObject() + .toString(); + } + + private Settings getSettings() { + return Settings.builder() + .put("number_of_shards", SHARD_COUNT) + .put("number_of_replicas", REPLICA_COUNT) + .put("index.knn", true) + .build(); + } } diff --git a/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java b/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java index 2bfef125b..6897091af 100644 --- a/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java +++ b/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java @@ -6,6 +6,7 @@ package org.opensearch.knn; import com.google.common.primitives.Floats; +import lombok.SneakyThrows; import lombok.extern.log4j.Log4j2; import org.apache.commons.lang.StringUtils; import org.apache.http.client.utils.URIBuilder; @@ -449,6 +450,13 @@ public int getDocCount(String indexName) throws IOException { * Force merge KNN index segments */ protected void forceMergeKnnIndex(String index) throws Exception { + forceMergeKnnIndex(index, 1); + } + + /** + * Force merge KNN index segments + */ + protected void forceMergeKnnIndex(String index, int maxSegments) throws Exception { Request request = new Request("POST", "/" + index + "/_refresh"); Response response = client().performRequest(request); @@ -456,7 +464,7 @@ protected void forceMergeKnnIndex(String index) throws Exception { request = new Request("POST", "/" + index + "/_forcemerge"); - request.addParameter("max_num_segments", "1"); + request.addParameter("max_num_segments", String.valueOf(maxSegments)); request.addParameter("flush", "true"); response = client().performRequest(request); assertEquals(request.getEndpoint() + ": failed", RestStatus.OK, RestStatus.fromCode(response.getStatusLine().getStatusCode())); @@ -657,6 +665,12 @@ protected Response executeKnnStatRequest(List nodeIds, List stat return response; } + @SneakyThrows + protected void doKnnWarmup(List indices) { + Response response = knnWarmup(indices); + assertEquals(response.getStatusLine().getStatusCode(), 200); + } + /** * Warmup KNN Index */ diff --git a/src/testFixtures/java/org/opensearch/knn/TestUtils.java b/src/testFixtures/java/org/opensearch/knn/TestUtils.java index 1cd0ac3db..1b5accae9 100644 --- a/src/testFixtures/java/org/opensearch/knn/TestUtils.java +++ b/src/testFixtures/java/org/opensearch/knn/TestUtils.java @@ -141,6 +141,12 @@ public static List> computeGroundTruthValues(float[][] indexVectors, pq = new PriorityQueue<>(k, new DistComparator()); for (int j = 0; j < indexVectors.length; j++) { float dist = computeDistFromSpaceType(spaceType, indexVectors[j], queryVectors[i]); + + // Need to invert distance for IP or COSINE because higher is better in these cases + if (spaceType == SpaceType.INNER_PRODUCT || spaceType == SpaceType.COSINESIMIL) { + dist *= -1; + } + pq = insertWithOverflow(pq, k, dist, j); } @@ -201,6 +207,11 @@ public static PriorityQueue computeGroundTruthValues(int k, SpaceTyp for (int id = 0; id < numDocs; id++) { float[] indexVector = idVectorProducer.getVector(id); float dist = computeDistFromSpaceType(spaceType, indexVector, queryVector); + // Need to invert distance for IP or COSINE because higher is better in these cases + if (spaceType == SpaceType.INNER_PRODUCT || spaceType == SpaceType.COSINESIMIL) { + dist *= -1; + } + pq = insertWithOverflow(pq, k, dist, id); } return pq; From 6329f5fe550a45a6bce2b7529477df6387bcacf2 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Mon, 18 Mar 2024 16:31:45 -0500 Subject: [PATCH 224/416] Remove faiss CI patch (#1560) (#1561) Signed-off-by: Naveen Tatikonda (cherry picked from commit 4b0078d08148d61543cc993deec607700fd8a124) Co-authored-by: Naveen Tatikonda --- .github/workflows/CI.yml | 6 ++-- .github/workflows/test_security.yml | 6 ++-- jni/CMakeLists.txt | 4 +-- ...ustom-patch-to-support-AVX2-Linux-CI.patch | 32 ------------------- ...le-precomp-table-to-be-shared-ivfpq.patch} | 0 5 files changed, 6 insertions(+), 42 deletions(-) delete mode 100644 jni/patches/faiss/0002-Custom-patch-to-support-AVX2-Linux-CI.patch rename jni/patches/faiss/{0003-Enable-precomp-table-to-be-shared-ivfpq.patch => 0002-Enable-precomp-table-to-be-shared-ivfpq.patch} (100%) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 504aabada..73335e9ce 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -45,10 +45,8 @@ jobs: cd jni/external/faiss git apply --ignore-space-change --ignore-whitespace --3way ../../patches/faiss/0001-Custom-patch-to-support-multi-vector.patch rm ../../patches/faiss/0001-Custom-patch-to-support-multi-vector.patch - git apply --ignore-space-change --ignore-whitespace --3way ../../patches/faiss/0002-Custom-patch-to-support-AVX2-Linux-CI.patch - rm ../../patches/faiss/0002-Custom-patch-to-support-AVX2-Linux-CI.patch - git apply --ignore-space-change --ignore-whitespace --3way ../../patches/faiss/0003-Enable-precomp-table-to-be-shared-ivfpq.patch - rm ../../patches/faiss/0003-Enable-precomp-table-to-be-shared-ivfpq.patch + git apply --ignore-space-change --ignore-whitespace --3way ../../patches/faiss/0002-Enable-precomp-table-to-be-shared-ivfpq.patch + rm ../../patches/faiss/0002-Enable-precomp-table-to-be-shared-ivfpq.patch cd ../nmslib git apply --ignore-space-change --ignore-whitespace --3way ../../patches/nmslib/0001-Initialize-maxlevel-during-add-from-enterpoint-level.patch rm ../../patches/nmslib/0001-Initialize-maxlevel-during-add-from-enterpoint-level.patch diff --git a/.github/workflows/test_security.yml b/.github/workflows/test_security.yml index f6eebced2..cbdd7983b 100644 --- a/.github/workflows/test_security.yml +++ b/.github/workflows/test_security.yml @@ -45,10 +45,8 @@ jobs: cd jni/external/faiss git apply --ignore-space-change --ignore-whitespace --3way ../../patches/faiss/0001-Custom-patch-to-support-multi-vector.patch rm ../../patches/faiss/0001-Custom-patch-to-support-multi-vector.patch - git apply --ignore-space-change --ignore-whitespace --3way ../../patches/faiss/0002-Custom-patch-to-support-AVX2-Linux-CI.patch - rm ../../patches/faiss/0002-Custom-patch-to-support-AVX2-Linux-CI.patch - git apply --ignore-space-change --ignore-whitespace --3way ../../patches/faiss/0003-Enable-precomp-table-to-be-shared-ivfpq.patch - rm ../../patches/faiss/0003-Enable-precomp-table-to-be-shared-ivfpq.patch + git apply --ignore-space-change --ignore-whitespace --3way ../../patches/faiss/0002-Enable-precomp-table-to-be-shared-ivfpq.patch + rm ../../patches/faiss/0002-Enable-precomp-table-to-be-shared-ivfpq.patch cd ../nmslib git apply --ignore-space-change --ignore-whitespace --3way ../../patches/nmslib/0001-Initialize-maxlevel-during-add-from-enterpoint-level.patch rm ../../patches/nmslib/0001-Initialize-maxlevel-during-add-from-enterpoint-level.patch diff --git a/jni/CMakeLists.txt b/jni/CMakeLists.txt index 574a04c9d..60321ed1b 100644 --- a/jni/CMakeLists.txt +++ b/jni/CMakeLists.txt @@ -168,13 +168,13 @@ if (${CONFIG_FAISS} STREQUAL ON OR ${CONFIG_ALL} STREQUAL ON OR ${CONFIG_TEST} S endif () # Check if patch exist, this is to skip git apply during CI build. See CI.yml with ubuntu. - find_path(PATCH_FILE NAMES 0001-Custom-patch-to-support-multi-vector.patch 0003-Enable-precomp-table-to-be-shared-ivfpq.patch PATHS ${CMAKE_CURRENT_SOURCE_DIR}/patches/faiss NO_DEFAULT_PATH) + find_path(PATCH_FILE NAMES 0001-Custom-patch-to-support-multi-vector.patch 0002-Enable-precomp-table-to-be-shared-ivfpq.patch PATHS ${CMAKE_CURRENT_SOURCE_DIR}/patches/faiss NO_DEFAULT_PATH) # If it exists, apply patches if (EXISTS ${PATCH_FILE}) message(STATUS "Applying custom patches.") execute_process(COMMAND git apply --ignore-space-change --ignore-whitespace --3way ${CMAKE_CURRENT_SOURCE_DIR}/patches/faiss/0001-Custom-patch-to-support-multi-vector.patch WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/external/faiss ERROR_VARIABLE ERROR_MSG RESULT_VARIABLE RESULT_CODE) - execute_process(COMMAND git apply --ignore-space-change --ignore-whitespace --3way ${CMAKE_CURRENT_SOURCE_DIR}/patches/faiss/0003-Enable-precomp-table-to-be-shared-ivfpq.patch WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/external/faiss ERROR_VARIABLE ERROR_MSG RESULT_VARIABLE RESULT_CODE) + execute_process(COMMAND git apply --ignore-space-change --ignore-whitespace --3way ${CMAKE_CURRENT_SOURCE_DIR}/patches/faiss/0002-Enable-precomp-table-to-be-shared-ivfpq.patch WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/external/faiss ERROR_VARIABLE ERROR_MSG RESULT_VARIABLE RESULT_CODE) if(RESULT_CODE) message(FATAL_ERROR "Failed to apply patch:\n${ERROR_MSG}") endif() diff --git a/jni/patches/faiss/0002-Custom-patch-to-support-AVX2-Linux-CI.patch b/jni/patches/faiss/0002-Custom-patch-to-support-AVX2-Linux-CI.patch deleted file mode 100644 index 22d50e66c..000000000 --- a/jni/patches/faiss/0002-Custom-patch-to-support-AVX2-Linux-CI.patch +++ /dev/null @@ -1,32 +0,0 @@ -Temporarily replace the intrinsic '_mm_loadu_si64' with '_mm_loadl_epi64' until centOS7 is deprecated in our CI on Linux OS. -centOS7 only supports gcc version upto 8.x. But, the intrinsic '_mm_loadu_si64' requires gcc version of minimum 9.x. -So, replacing it with an equivalent intrinsic. - -diff --git a/faiss/impl/code_distance/code_distance-avx2.h b/faiss/impl/code_distance/code_distance-avx2.h -index 0aa1535b..6e4e5b55 100644 ---- a/faiss/impl/code_distance/code_distance-avx2.h -+++ b/faiss/impl/code_distance/code_distance-avx2.h -@@ -91,7 +91,7 @@ float inline distance_single_code_avx2_pqdecoder8_m8( - __m256 partialSum; - - // load 8 uint8 values -- const __m128i mm1 = _mm_loadu_si64((const __m128i_u*)code); -+ const __m128i mm1 = _mm_loadl_epi64((const __m128i_u*)code); - { - // convert uint8 values (low part of __m128i) to int32 - // values -@@ -199,10 +199,10 @@ inline void distance_four_codes_avx2_pqdecoder8_m8( - - // load 8 uint8 values - __m128i mm1[N]; -- mm1[0] = _mm_loadu_si64((const __m128i_u*)code0); -- mm1[1] = _mm_loadu_si64((const __m128i_u*)code1); -- mm1[2] = _mm_loadu_si64((const __m128i_u*)code2); -- mm1[3] = _mm_loadu_si64((const __m128i_u*)code3); -+ mm1[0] = _mm_loadl_epi64((const __m128i_u*)code0); -+ mm1[1] = _mm_loadl_epi64((const __m128i_u*)code1); -+ mm1[2] = _mm_loadl_epi64((const __m128i_u*)code2); -+ mm1[3] = _mm_loadl_epi64((const __m128i_u*)code3); - - for (intptr_t j = 0; j < N; j++) { - // convert uint8 values (low part of __m128i) to int32 diff --git a/jni/patches/faiss/0003-Enable-precomp-table-to-be-shared-ivfpq.patch b/jni/patches/faiss/0002-Enable-precomp-table-to-be-shared-ivfpq.patch similarity index 100% rename from jni/patches/faiss/0003-Enable-precomp-table-to-be-shared-ivfpq.patch rename to jni/patches/faiss/0002-Enable-precomp-table-to-be-shared-ivfpq.patch From 1d712da32a330267e18d54205672d68529b0c072 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Mon, 18 Mar 2024 18:48:31 -0500 Subject: [PATCH 225/416] [Backport 2.x] Faiss SQFP16 Range Validation and Clipping (#1563) * Faiss SQFP16 Range Validation and Clipping (#1562) * Add Range Validation for SQFP16 (#1493) * Add Range Validation for SQFP16 Vector Data Signed-off-by: Naveen Tatikonda * Add index setting to clip vector data to FP16 range Signed-off-by: Naveen Tatikonda * Add CHANGELOG Signed-off-by: Naveen Tatikonda * Add an encoder parameter to clip fp16 range Signed-off-by: Naveen Tatikonda * Address Review Comments Signed-off-by: Naveen Tatikonda * Add BWC Tests Signed-off-by: Naveen Tatikonda --------- Signed-off-by: Naveen Tatikonda * SQFP16 Range Validation for Faiss IVF Models (#1557) * SQFP16 Range Validation for Faiss IVF Models Signed-off-by: Naveen Tatikonda * Address Review Comments Signed-off-by: Naveen Tatikonda --------- Signed-off-by: Naveen Tatikonda * Rebase Changes Signed-off-by: Naveen Tatikonda --------- Signed-off-by: Naveen Tatikonda (cherry picked from commit d63ce2766bfd7551b680c1d2801b2c9e986f9613) * Fix EntityUtils package Signed-off-by: Naveen Tatikonda --------- Signed-off-by: Naveen Tatikonda Co-authored-by: Naveen Tatikonda --- CHANGELOG.md | 2 + .../org/opensearch/knn/bwc/FaissSQIT.java | 388 ++++++++++++++++++ .../opensearch/knn/common/KNNConstants.java | 4 + .../org/opensearch/knn/index/Parameter.java | 25 ++ .../index/mapper/KNNVectorFieldMapper.java | 114 ++++- .../mapper/KNNVectorFieldMapperUtil.java | 43 ++ .../knn/index/mapper/LuceneFieldMapper.java | 6 +- .../knn/index/mapper/ModelFieldMapper.java | 2 +- .../org/opensearch/knn/index/util/Faiss.java | 2 + .../org/opensearch/knn/index/FaissIT.java | 355 ++++++++++++++++ .../mapper/KNNVectorFieldMapperTests.java | 79 +++- 11 files changed, 1005 insertions(+), 15 deletions(-) create mode 100644 qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/FaissSQIT.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 9f815f792..fdfedc126 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), * Validate zero vector when using cosine metric [#1501](https://github.com/opensearch-project/k-NN/pull/1501) * Persist model definition in model metadata [#1527] (https://github.com/opensearch-project/k-NN/pull/1527) * Added Inner Product Space type support for Lucene Engine [#1551](https://github.com/opensearch-project/k-NN/pull/1551) +* Add Range Validation for Faiss SQFP16 [#1493](https://github.com/opensearch-project/k-NN/pull/1493) +* SQFP16 Range Validation for Faiss IVF Models [#1557](https://github.com/opensearch-project/k-NN/pull/1557) ### Bug Fixes * Disable sdc table for HNSWPQ read-only indices [#1518](https://github.com/opensearch-project/k-NN/pull/1518) * Switch SpaceType.INNERPRODUCT's vector similarity function to MAXIMUM_INNER_PRODUCT [#1532](https://github.com/opensearch-project/k-NN/pull/1532) diff --git a/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/FaissSQIT.java b/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/FaissSQIT.java new file mode 100644 index 000000000..83c358cad --- /dev/null +++ b/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/FaissSQIT.java @@ -0,0 +1,388 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.knn.bwc; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import org.apache.http.util.EntityUtils; +import org.opensearch.client.Response; +import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.knn.KNNResult; +import org.opensearch.knn.common.KNNConstants; +import org.opensearch.knn.index.KNNMethod; +import org.opensearch.knn.index.SpaceType; +import org.opensearch.knn.index.query.KNNQueryBuilder; +import org.opensearch.knn.index.util.KNNEngine; + +import java.io.IOException; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.TreeMap; + +import static org.opensearch.knn.common.KNNConstants.ENCODER_SQ; +import static org.opensearch.knn.common.KNNConstants.FAISS_NAME; +import static org.opensearch.knn.common.KNNConstants.FAISS_SQ_CLIP; +import static org.opensearch.knn.common.KNNConstants.FAISS_SQ_ENCODER_FP16; +import static org.opensearch.knn.common.KNNConstants.FAISS_SQ_TYPE; +import static org.opensearch.knn.common.KNNConstants.KNN_ENGINE; +import static org.opensearch.knn.common.KNNConstants.METHOD_ENCODER_PARAMETER; +import static org.opensearch.knn.common.KNNConstants.METHOD_IVF; +import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_NLIST; +import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_SPACE_TYPE; +import static org.opensearch.knn.common.KNNConstants.MODEL_ID; +import static org.opensearch.knn.common.KNNConstants.NAME; +import static org.opensearch.knn.common.KNNConstants.PARAMETERS; + +public class FaissSQIT extends AbstractRestartUpgradeTestCase { + private static final String TEST_FIELD = "test-field"; + private static final String TRAIN_TEST_FIELD = "train-test-field"; + private static final String TRAIN_INDEX = "train-index"; + private static final String TEST_MODEL = "test-model"; + private static final int DIMENSION = 128; + private static final int NUM_DOCS = 100; + + public void testHNSWSQFP16_onUpgradeWhenIndexedAndQueried_thenSucceed() throws Exception { + if (!isRunningAgainstOldCluster()) { + KNNMethod hnswMethod = KNNEngine.FAISS.getMethod(KNNConstants.METHOD_HNSW); + SpaceType[] spaceTypes = { SpaceType.L2, SpaceType.INNER_PRODUCT }; + Random random = new Random(); + SpaceType spaceType = spaceTypes[random.nextInt(spaceTypes.length)]; + + List mValues = ImmutableList.of(16, 32, 64, 128); + List efConstructionValues = ImmutableList.of(16, 32, 64, 128); + List efSearchValues = ImmutableList.of(16, 32, 64, 128); + + // Create an index + /** + * "properties": { + * "test-field": { + * "type": "knn_vector", + * "dimension": 128, + * "method": { + * "name": "hnsw", + * "space_type": "l2", + * "engine": "faiss", + * "parameters": { + * "m": 16, + * "ef_construction": 128, + * "ef_search": 128, + * "encoder": { + * "name": "sq", + * "parameters": { + * "type": "fp16" + * } + * } + * } + * } + * } + * } + */ + XContentBuilder builder = XContentFactory.jsonBuilder() + .startObject() + .startObject("properties") + .startObject(TEST_FIELD) + .field("type", "knn_vector") + .field("dimension", DIMENSION) + .startObject(KNNConstants.KNN_METHOD) + .field(KNNConstants.NAME, hnswMethod.getMethodComponent().getName()) + .field(KNNConstants.METHOD_PARAMETER_SPACE_TYPE, spaceType.getValue()) + .field(KNNConstants.KNN_ENGINE, KNNEngine.FAISS.getName()) + .startObject(KNNConstants.PARAMETERS) + .field(KNNConstants.METHOD_PARAMETER_M, mValues.get(random().nextInt(mValues.size()))) + .field( + KNNConstants.METHOD_PARAMETER_EF_CONSTRUCTION, + efConstructionValues.get(random().nextInt(efConstructionValues.size())) + ) + .field(KNNConstants.METHOD_PARAMETER_EF_SEARCH, efSearchValues.get(random().nextInt(efSearchValues.size()))) + .startObject(METHOD_ENCODER_PARAMETER) + .field(NAME, ENCODER_SQ) + .startObject(PARAMETERS) + .field(FAISS_SQ_TYPE, FAISS_SQ_ENCODER_FP16) + .endObject() + .endObject() + .endObject() + .endObject() + .endObject() + .endObject() + .endObject(); + + Map mappingMap = xContentBuilderToMap(builder); + String mapping = builder.toString(); + + createKnnIndex(testIndex, mapping); + assertEquals(new TreeMap<>(mappingMap), new TreeMap<>(getIndexMappingAsMap(testIndex))); + indexTestData(testIndex, TEST_FIELD, DIMENSION, NUM_DOCS); + queryTestData(testIndex, TEST_FIELD, DIMENSION, NUM_DOCS); + deleteKNNIndex(testIndex); + validateGraphEviction(); + } + } + + public void testHNSWSQFP16_onUpgradeWhenClipToFp16isTrueAndIndexedWithOutOfFP16Range_thenSucceed() throws Exception { + if (!isRunningAgainstOldCluster()) { + KNNMethod hnswMethod = KNNEngine.FAISS.getMethod(KNNConstants.METHOD_HNSW); + new Random(); + + List mValues = ImmutableList.of(16, 32, 64, 128); + List efConstructionValues = ImmutableList.of(16, 32, 64, 128); + List efSearchValues = ImmutableList.of(16, 32, 64, 128); + + int dimension = 2; + + // Create an index + /** + * "properties": { + * "test-field": { + * "type": "knn_vector", + * "dimension": 128, + * "method": { + * "name": "hnsw", + * "space_type": "l2", + * "engine": "faiss", + * "parameters": { + * "m": 16, + * "ef_construction": 128, + * "ef_search": 128, + * "encoder": { + * "name": "sq", + * "parameters": { + * "type": "fp16", + * "clip": true + * } + * } + * } + * } + * } + * } + */ + XContentBuilder builder = XContentFactory.jsonBuilder() + .startObject() + .startObject("properties") + .startObject(TEST_FIELD) + .field("type", "knn_vector") + .field("dimension", dimension) + .startObject(KNNConstants.KNN_METHOD) + .field(KNNConstants.NAME, hnswMethod.getMethodComponent().getName()) + .field(KNNConstants.METHOD_PARAMETER_SPACE_TYPE, SpaceType.L2.getValue()) + .field(KNNConstants.KNN_ENGINE, KNNEngine.FAISS.getName()) + .startObject(PARAMETERS) + .field(KNNConstants.METHOD_PARAMETER_M, mValues.get(random().nextInt(mValues.size()))) + .field( + KNNConstants.METHOD_PARAMETER_EF_CONSTRUCTION, + efConstructionValues.get(random().nextInt(efConstructionValues.size())) + ) + .field(KNNConstants.METHOD_PARAMETER_EF_SEARCH, efSearchValues.get(random().nextInt(efSearchValues.size()))) + .startObject(METHOD_ENCODER_PARAMETER) + .field(NAME, ENCODER_SQ) + .startObject(PARAMETERS) + .field(FAISS_SQ_TYPE, FAISS_SQ_ENCODER_FP16) + .field(FAISS_SQ_CLIP, true) + .endObject() + .endObject() + .endObject() + .endObject() + .endObject() + .endObject() + .endObject(); + + Map mappingMap = xContentBuilderToMap(builder); + String mapping = builder.toString(); + + createKnnIndex(testIndex, mapping); + assertEquals(new TreeMap<>(mappingMap), new TreeMap<>(getIndexMappingAsMap(testIndex))); + Float[] vector1 = { -65523.76f, 65504.2f }; + Float[] vector2 = { -270.85f, 65514.2f }; + Float[] vector3 = { -150.9f, 65504.0f }; + Float[] vector4 = { -20.89f, 100000000.0f }; + addKnnDoc(testIndex, "1", TEST_FIELD, vector1); + addKnnDoc(testIndex, "2", TEST_FIELD, vector2); + addKnnDoc(testIndex, "3", TEST_FIELD, vector3); + addKnnDoc(testIndex, "4", TEST_FIELD, vector4); + + float[] queryVector = { -10.5f, 25.48f }; + int k = 4; + Response searchResponse = searchKNNIndex(testIndex, new KNNQueryBuilder(TEST_FIELD, queryVector, k), k); + List results = parseSearchResponse(EntityUtils.toString(searchResponse.getEntity()), TEST_FIELD); + assertEquals(k, results.size()); + for (int i = 0; i < k; i++) { + assertEquals(k - i, Integer.parseInt(results.get(i).getDocId())); + } + deleteKNNIndex(testIndex); + validateGraphEviction(); + } + } + + public void testIVFSQFP16_onUpgradeWhenIndexedAndQueried_thenSucceed() throws Exception { + if (!isRunningAgainstOldCluster()) { + + // Add training data + createBasicKnnIndex(TRAIN_INDEX, TRAIN_TEST_FIELD, DIMENSION); + int trainingDataCount = 200; + bulkIngestRandomVectors(TRAIN_INDEX, TRAIN_TEST_FIELD, trainingDataCount, DIMENSION); + + XContentBuilder builder = XContentFactory.jsonBuilder() + .startObject() + .field(NAME, METHOD_IVF) + .field(KNN_ENGINE, FAISS_NAME) + .field(METHOD_PARAMETER_SPACE_TYPE, "l2") + .startObject(PARAMETERS) + .startObject(METHOD_ENCODER_PARAMETER) + .field(NAME, ENCODER_SQ) + .startObject(PARAMETERS) + .field(FAISS_SQ_TYPE, FAISS_SQ_ENCODER_FP16) + .endObject() + .endObject() + .endObject() + .endObject(); + Map method = xContentBuilderToMap(builder); + + trainModel(TEST_MODEL, TRAIN_INDEX, TRAIN_TEST_FIELD, DIMENSION, method, "faiss ivf sqfp16 test description"); + + // Make sure training succeeds after 30 seconds + assertTrainingSucceeds(TEST_MODEL, 30, 1000); + + // Create knn index from model + String indexMapping = XContentFactory.jsonBuilder() + .startObject() + .startObject("properties") + .startObject(TEST_FIELD) + .field("type", "knn_vector") + .field(MODEL_ID, TEST_MODEL) + .endObject() + .endObject() + .endObject() + .toString(); + + createKnnIndex(testIndex, getKNNDefaultIndexSettings(), indexMapping); + + indexTestData(testIndex, TEST_FIELD, DIMENSION, NUM_DOCS); + queryTestData(testIndex, TEST_FIELD, DIMENSION, NUM_DOCS); + deleteKNNIndex(TRAIN_INDEX); + deleteKNNIndex(testIndex); + deleteModel(TEST_MODEL); + validateGraphEviction(); + } + } + + public void testIVFSQFP16_onUpgradeWhenClipToFp16isTrueAndIndexedWithOutOfFP16Range_thenSucceed() throws Exception { + if (!isRunningAgainstOldCluster()) { + int dimension = 2; + + // Add training data + createBasicKnnIndex(TRAIN_INDEX, TRAIN_TEST_FIELD, dimension); + int trainingDataCount = 200; + bulkIngestRandomVectors(TRAIN_INDEX, TRAIN_TEST_FIELD, trainingDataCount, dimension); + + XContentBuilder builder = XContentFactory.jsonBuilder() + .startObject() + .field(NAME, METHOD_IVF) + .field(KNN_ENGINE, FAISS_NAME) + .field(METHOD_PARAMETER_SPACE_TYPE, "l2") + .startObject(PARAMETERS) + .field(METHOD_PARAMETER_NLIST, 1) + .startObject(METHOD_ENCODER_PARAMETER) + .field(NAME, ENCODER_SQ) + .startObject(PARAMETERS) + .field(FAISS_SQ_TYPE, FAISS_SQ_ENCODER_FP16) + .field(FAISS_SQ_CLIP, true) + .endObject() + .endObject() + .endObject() + .endObject(); + Map method = xContentBuilderToMap(builder); + + trainModel(TEST_MODEL, TRAIN_INDEX, TRAIN_TEST_FIELD, dimension, method, "faiss ivf sqfp16 test description"); + + // Make sure training succeeds after 30 seconds + assertTrainingSucceeds(TEST_MODEL, 30, 1000); + + // Create knn index from model + String indexMapping = XContentFactory.jsonBuilder() + .startObject() + .startObject("properties") + .startObject(TEST_FIELD) + .field("type", "knn_vector") + .field(MODEL_ID, TEST_MODEL) + .endObject() + .endObject() + .endObject() + .toString(); + + createKnnIndex(testIndex, getKNNDefaultIndexSettings(), indexMapping); + Float[] vector1 = { -65523.76f, 65504.2f }; + Float[] vector2 = { -270.85f, 65514.2f }; + Float[] vector3 = { -150.9f, 65504.0f }; + Float[] vector4 = { -20.89f, 100000000.0f }; + addKnnDoc(testIndex, "1", TEST_FIELD, vector1); + addKnnDoc(testIndex, "2", TEST_FIELD, vector2); + addKnnDoc(testIndex, "3", TEST_FIELD, vector3); + addKnnDoc(testIndex, "4", TEST_FIELD, vector4); + + float[] queryVector = { -10.5f, 25.48f }; + int k = 4; + Response searchResponse = searchKNNIndex(testIndex, new KNNQueryBuilder(TEST_FIELD, queryVector, k), k); + List results = parseSearchResponse(EntityUtils.toString(searchResponse.getEntity()), TEST_FIELD); + assertEquals(k, results.size()); + for (int i = 0; i < k; i++) { + assertEquals(k - i, Integer.parseInt(results.get(i).getDocId())); + } + + deleteKNNIndex(testIndex); + deleteKNNIndex(TRAIN_INDEX); + deleteModel(TEST_MODEL); + validateGraphEviction(); + } + } + + private void validateGraphEviction() throws Exception { + // Search every 5 seconds 14 times to confirm graph gets evicted + int intervals = 14; + for (int i = 0; i < intervals; i++) { + if (getTotalGraphsInCache() == 0) { + return; + } + + Thread.sleep(5 * 1000); + } + + fail("Graphs are not getting evicted"); + } + + private void queryTestData(final String indexName, final String fieldName, final int dimension, final int numDocs) throws IOException { + float[] queryVector = new float[dimension]; + Arrays.fill(queryVector, (float) numDocs); + int k = 10; + + Response searchResponse = searchKNNIndex(indexName, new KNNQueryBuilder(fieldName, queryVector, k), k); + List results = parseSearchResponse(EntityUtils.toString(searchResponse.getEntity()), fieldName); + assertEquals(k, results.size()); + for (int i = 0; i < k; i++) { + assertEquals(numDocs - i - 1, Integer.parseInt(results.get(i).getDocId())); + } + } + + private void indexTestData(final String indexName, final String fieldName, final int dimension, final int numDocs) throws Exception { + for (int i = 0; i < numDocs; i++) { + float[] indexVector = new float[dimension]; + Arrays.fill(indexVector, (float) i); + addKnnDocWithAttributes(indexName, Integer.toString(i), fieldName, indexVector, ImmutableMap.of("rating", String.valueOf(i))); + } + + // Assert that all docs are ingested + refreshAllNonSystemIndices(); + assertEquals(numDocs, getDocCount(indexName)); + } + +} diff --git a/src/main/java/org/opensearch/knn/common/KNNConstants.java b/src/main/java/org/opensearch/knn/common/KNNConstants.java index 314d7d4d1..46d82b2bf 100644 --- a/src/main/java/org/opensearch/knn/common/KNNConstants.java +++ b/src/main/java/org/opensearch/knn/common/KNNConstants.java @@ -97,6 +97,7 @@ public class KNNConstants { public static final String FAISS_SQ_TYPE = "type"; public static final String FAISS_SQ_ENCODER_FP16 = "fp16"; public static final List FAISS_SQ_ENCODER_TYPES = List.of(FAISS_SQ_ENCODER_FP16); + public static final String FAISS_SQ_CLIP = "clip"; // Parameter defaults/limits public static final Integer ENCODER_PARAMETER_PQ_CODE_COUNT_DEFAULT = 1; @@ -111,6 +112,9 @@ public class KNNConstants { public static final Integer MODEL_CACHE_CAPACITY_ATROPHY_THRESHOLD_IN_MINUTES = 30; public static final Integer MODEL_CACHE_EXPIRE_AFTER_ACCESS_TIME_MINUTES = 30; + public static final Float FP16_MAX_VALUE = 65504.0f; + public static final Float FP16_MIN_VALUE = -65504.0f; + // Lib names private static final String JNI_LIBRARY_PREFIX = "opensearchknn_"; public static final String FAISS_JNI_LIBRARY_NAME = JNI_LIBRARY_PREFIX + FAISS_NAME; diff --git a/src/main/java/org/opensearch/knn/index/Parameter.java b/src/main/java/org/opensearch/knn/index/Parameter.java index bef5a33e9..e223909d5 100644 --- a/src/main/java/org/opensearch/knn/index/Parameter.java +++ b/src/main/java/org/opensearch/knn/index/Parameter.java @@ -66,6 +66,31 @@ public T getDefaultValue() { */ public abstract ValidationException validate(Object value); + /** + * Boolean method parameter + */ + public static class BooleanParameter extends Parameter { + public BooleanParameter(String name, Boolean defaultValue, Predicate validator) { + super(name, defaultValue, validator); + } + + @Override + public ValidationException validate(Object value) { + ValidationException validationException = null; + if (!(value instanceof Boolean)) { + validationException = new ValidationException(); + validationException.addValidationError(String.format("value not of type Boolean for Boolean parameter [%s].", getName())); + return validationException; + } + + if (!validator.test((Boolean) value)) { + validationException = new ValidationException(); + validationException.addValidationError(String.format("parameter validation failed for Boolean parameter [%s].", getName())); + } + return validationException; + } + } + /** * Integer method parameter */ diff --git a/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapper.java b/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapper.java index 2369a6937..a36a4222b 100644 --- a/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapper.java +++ b/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapper.java @@ -44,6 +44,7 @@ import org.opensearch.knn.index.KNNSettings; import org.opensearch.knn.index.KNNVectorIndexFieldData; import org.opensearch.knn.index.SpaceType; +import org.opensearch.knn.index.MethodComponentContext; import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.VectorField; import org.opensearch.knn.index.util.KNNEngine; @@ -51,14 +52,31 @@ import org.opensearch.search.aggregations.support.CoreValuesSourceType; import org.opensearch.search.lookup.SearchLookup; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.function.Supplier; + import static org.opensearch.knn.common.KNNConstants.DEFAULT_VECTOR_DATA_TYPE_FIELD; +import static org.opensearch.knn.common.KNNConstants.ENCODER_SQ; +import static org.opensearch.knn.common.KNNConstants.FAISS_SQ_CLIP; +import static org.opensearch.knn.common.KNNConstants.FAISS_SQ_ENCODER_FP16; +import static org.opensearch.knn.common.KNNConstants.FAISS_SQ_TYPE; import static org.opensearch.knn.common.KNNConstants.KNN_METHOD; +import static org.opensearch.knn.common.KNNConstants.METHOD_ENCODER_PARAMETER; import static org.opensearch.knn.common.KNNConstants.VECTOR_DATA_TYPE_FIELD; import static org.opensearch.knn.common.KNNValidationUtil.validateByteVectorValue; import static org.opensearch.knn.common.KNNValidationUtil.validateFloatVectorValue; import static org.opensearch.knn.common.KNNValidationUtil.validateVectorDimension; import static org.opensearch.knn.index.KNNSettings.KNN_INDEX; import static org.opensearch.knn.index.mapper.KNNVectorFieldMapperUtil.addStoredFieldForVectorField; +import static org.opensearch.knn.index.mapper.KNNVectorFieldMapperUtil.clipVectorValueToFP16Range; +import static org.opensearch.knn.index.mapper.KNNVectorFieldMapperUtil.validateFP16VectorValue; import static org.opensearch.knn.index.mapper.KNNVectorFieldMapperUtil.validateVectorDataTypeWithEngine; import static org.opensearch.knn.index.mapper.KNNVectorFieldMapperUtil.validateVectorDataTypeWithKnnIndexSetting; @@ -511,10 +529,23 @@ protected String contentType() { @Override protected void parseCreateField(ParseContext context) throws IOException { - parseCreateField(context, fieldType().getDimension(), fieldType().getSpaceType()); + parseCreateField( + context, + fieldType().getDimension(), + fieldType().getSpaceType(), + getMethodComponentContext(fieldType().getKnnMethodContext()) + ); } - protected void parseCreateField(ParseContext context, int dimension, SpaceType spaceType) throws IOException { + private MethodComponentContext getMethodComponentContext(KNNMethodContext knnMethodContext) { + if (Objects.isNull(knnMethodContext)) { + return null; + } + return knnMethodContext.getMethodComponentContext(); + } + + protected void parseCreateField(ParseContext context, int dimension, SpaceType spaceType, MethodComponentContext methodComponentContext) + throws IOException { validateIfKNNPluginEnabled(); validateIfCircuitBreakerIsNotTriggered(); @@ -532,7 +563,7 @@ protected void parseCreateField(ParseContext context, int dimension, SpaceType s context.doc().add(point); addStoredFieldForVectorField(context, fieldType, name(), point.toString()); } else if (VectorDataType.FLOAT == vectorDataType) { - Optional floatsArrayOptional = getFloatsFromContext(context, dimension); + Optional floatsArrayOptional = getFloatsFromContext(context, dimension, methodComponentContext); if (floatsArrayOptional.isEmpty()) { return; @@ -551,6 +582,47 @@ protected void parseCreateField(ParseContext context, int dimension, SpaceType s context.path().remove(); } + // Verify mapping and return true if it is a "faiss" Index using "sq" encoder of type "fp16" + protected boolean isFaissSQfp16(MethodComponentContext methodComponentContext) { + if (Objects.isNull(methodComponentContext)) { + return false; + } + + if (methodComponentContext.getParameters().size() == 0) { + return false; + } + + Map methodComponentParams = methodComponentContext.getParameters(); + + // The method component parameters should have an encoder + if (!methodComponentParams.containsKey(METHOD_ENCODER_PARAMETER)) { + return false; + } + + // Validate if the object is of type MethodComponentContext before casting it later + if (!(methodComponentParams.get(METHOD_ENCODER_PARAMETER) instanceof MethodComponentContext)) { + return false; + } + + MethodComponentContext encoderMethodComponentContext = (MethodComponentContext) methodComponentParams.get(METHOD_ENCODER_PARAMETER); + + // returns true if encoder name is "sq" and type is "fp16" + return ENCODER_SQ.equals(encoderMethodComponentContext.getName()) + && FAISS_SQ_ENCODER_FP16.equals( + encoderMethodComponentContext.getParameters().getOrDefault(FAISS_SQ_TYPE, FAISS_SQ_ENCODER_FP16) + ); + + } + + // Verify mapping and return the value of "clip" parameter(default false) for a "faiss" Index + // using "sq" encoder of type "fp16". + protected boolean isFaissSQClipToFP16RangeEnabled(MethodComponentContext methodComponentContext) { + if (Objects.nonNull(methodComponentContext)) { + return (boolean) methodComponentContext.getParameters().getOrDefault(FAISS_SQ_CLIP, false); + } + return false; + } + void validateIfCircuitBreakerIsNotTriggered() { if (KNNSettings.isCircuitBreakerTriggered()) { throw new IllegalStateException( @@ -600,9 +672,22 @@ Optional getBytesFromContext(ParseContext context, int dimension) throws return Optional.of(array); } - Optional getFloatsFromContext(ParseContext context, int dimension) throws IOException { + Optional getFloatsFromContext(ParseContext context, int dimension, MethodComponentContext methodComponentContext) + throws IOException { context.path().add(simpleName()); + // Returns an optional array of float values where each value in the vector is parsed as a float and validated + // if it is a finite number and within the fp16 range of [-65504 to 65504] by default if Faiss encoder is SQ and type is 'fp16'. + // If the encoder parameter, "clip" is set to True, if the vector value is outside the FP16 range then it will be + // clipped to FP16 range. + boolean isFaissSQfp16Flag = isFaissSQfp16(methodComponentContext); + boolean clipVectorValueToFP16RangeFlag = false; + if (isFaissSQfp16Flag) { + clipVectorValueToFP16RangeFlag = isFaissSQClipToFP16RangeEnabled( + (MethodComponentContext) methodComponentContext.getParameters().get(METHOD_ENCODER_PARAMETER) + ); + } + ArrayList vector = new ArrayList<>(); XContentParser.Token token = context.parser().currentToken(); float value; @@ -610,13 +695,30 @@ Optional getFloatsFromContext(ParseContext context, int dimension) thro token = context.parser().nextToken(); while (token != XContentParser.Token.END_ARRAY) { value = context.parser().floatValue(); - validateFloatVectorValue(value); + if (isFaissSQfp16Flag) { + if (clipVectorValueToFP16RangeFlag) { + value = clipVectorValueToFP16Range(value); + } else { + validateFP16VectorValue(value); + } + } else { + validateFloatVectorValue(value); + } + vector.add(value); token = context.parser().nextToken(); } } else if (token == XContentParser.Token.VALUE_NUMBER) { value = context.parser().floatValue(); - validateFloatVectorValue(value); + if (isFaissSQfp16Flag) { + if (clipVectorValueToFP16RangeFlag) { + value = clipVectorValueToFP16Range(value); + } else { + validateFP16VectorValue(value); + } + } else { + validateFloatVectorValue(value); + } vector.add(value); context.parser().nextToken(); } else if (token == XContentParser.Token.VALUE_NULL) { diff --git a/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperUtil.java b/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperUtil.java index b525b9dc6..283d35f00 100644 --- a/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperUtil.java +++ b/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperUtil.java @@ -23,12 +23,55 @@ import java.util.Locale; +import static org.opensearch.knn.common.KNNConstants.ENCODER_SQ; +import static org.opensearch.knn.common.KNNConstants.FAISS_SQ_ENCODER_FP16; +import static org.opensearch.knn.common.KNNConstants.FP16_MAX_VALUE; +import static org.opensearch.knn.common.KNNConstants.FP16_MIN_VALUE; import static org.opensearch.knn.common.KNNConstants.KNN_ENGINE; import static org.opensearch.knn.common.KNNConstants.LUCENE_NAME; import static org.opensearch.knn.common.KNNConstants.VECTOR_DATA_TYPE_FIELD; +import static org.opensearch.knn.common.KNNValidationUtil.validateFloatVectorValue; @NoArgsConstructor(access = AccessLevel.PRIVATE) public class KNNVectorFieldMapperUtil { + + /** + * Validate the float vector value and throw exception if it is not a number or not in the finite range + * or is not within the FP16 range of [-65504 to 65504]. + * + * @param value float vector value + */ + public static void validateFP16VectorValue(float value) { + validateFloatVectorValue(value); + + if (value < FP16_MIN_VALUE || value > FP16_MAX_VALUE) { + throw new IllegalArgumentException( + String.format( + Locale.ROOT, + "encoder name is set as [%s] and type is set as [%s] in index mapping. But, KNN vector values are not within in the FP16 range [%f, %f]", + ENCODER_SQ, + FAISS_SQ_ENCODER_FP16, + FP16_MIN_VALUE, + FP16_MAX_VALUE + ) + ); + } + } + + /** + * Validate the float vector value and if it is outside FP16 range, + * then it will be clipped to FP16 range of [-65504 to 65504]. + * + * @param value float vector value + * @return vector value clipped to FP16 range + */ + public static float clipVectorValueToFP16Range(float value) { + validateFloatVectorValue(value); + if (value < FP16_MIN_VALUE) return FP16_MIN_VALUE; + if (value > FP16_MAX_VALUE) return FP16_MAX_VALUE; + return value; + } + /** * Validates and throws exception if data_type field is set in the index mapping * using any VectorDataType (other than float, which is default) because other diff --git a/src/main/java/org/opensearch/knn/index/mapper/LuceneFieldMapper.java b/src/main/java/org/opensearch/knn/index/mapper/LuceneFieldMapper.java index 81c7216bf..185ab3dc4 100644 --- a/src/main/java/org/opensearch/knn/index/mapper/LuceneFieldMapper.java +++ b/src/main/java/org/opensearch/knn/index/mapper/LuceneFieldMapper.java @@ -18,6 +18,7 @@ import org.opensearch.common.Explicit; import org.opensearch.index.mapper.ParseContext; import org.opensearch.knn.index.KNNMethodContext; +import org.opensearch.knn.index.MethodComponentContext; import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.VectorField; @@ -75,7 +76,8 @@ public class LuceneFieldMapper extends KNNVectorFieldMapper { } @Override - protected void parseCreateField(ParseContext context, int dimension, SpaceType spaceType) throws IOException { + protected void parseCreateField(ParseContext context, int dimension, SpaceType spaceType, MethodComponentContext methodComponentContext) + throws IOException { validateIfKNNPluginEnabled(); validateIfCircuitBreakerIsNotTriggered(); @@ -96,7 +98,7 @@ protected void parseCreateField(ParseContext context, int dimension, SpaceType s context.doc().add(new VectorField(name(), array, vectorFieldType)); } } else if (VectorDataType.FLOAT == vectorDataType) { - Optional floatsArrayOptional = getFloatsFromContext(context, dimension); + Optional floatsArrayOptional = getFloatsFromContext(context, dimension, methodComponentContext); if (floatsArrayOptional.isEmpty()) { return; diff --git a/src/main/java/org/opensearch/knn/index/mapper/ModelFieldMapper.java b/src/main/java/org/opensearch/knn/index/mapper/ModelFieldMapper.java index 2367d7422..ce92d2967 100644 --- a/src/main/java/org/opensearch/knn/index/mapper/ModelFieldMapper.java +++ b/src/main/java/org/opensearch/knn/index/mapper/ModelFieldMapper.java @@ -61,6 +61,6 @@ protected void parseCreateField(ParseContext context) throws IOException { ); } - parseCreateField(context, modelMetadata.getDimension(), modelMetadata.getSpaceType()); + parseCreateField(context, modelMetadata.getDimension(), modelMetadata.getSpaceType(), modelMetadata.getMethodComponentContext()); } } diff --git a/src/main/java/org/opensearch/knn/index/util/Faiss.java b/src/main/java/org/opensearch/knn/index/util/Faiss.java index 3b21488b9..563311c49 100644 --- a/src/main/java/org/opensearch/knn/index/util/Faiss.java +++ b/src/main/java/org/opensearch/knn/index/util/Faiss.java @@ -32,6 +32,7 @@ import static org.opensearch.knn.common.KNNConstants.FAISS_HNSW_DESCRIPTION; import static org.opensearch.knn.common.KNNConstants.FAISS_IVF_DESCRIPTION; import static org.opensearch.knn.common.KNNConstants.FAISS_PQ_DESCRIPTION; +import static org.opensearch.knn.common.KNNConstants.FAISS_SQ_CLIP; import static org.opensearch.knn.common.KNNConstants.FAISS_SQ_DESCRIPTION; import static org.opensearch.knn.common.KNNConstants.FAISS_SQ_ENCODER_FP16; import static org.opensearch.knn.common.KNNConstants.FAISS_SQ_ENCODER_TYPES; @@ -90,6 +91,7 @@ class Faiss extends NativeLibrary { FAISS_SQ_TYPE, new Parameter.StringParameter(FAISS_SQ_TYPE, FAISS_SQ_ENCODER_FP16, FAISS_SQ_ENCODER_TYPES::contains) ) + .addParameter(FAISS_SQ_CLIP, new Parameter.BooleanParameter(FAISS_SQ_CLIP, false, Objects::nonNull)) .setMapGenerator( ((methodComponent, methodComponentContext) -> MethodAsMapBuilder.builder( FAISS_SQ_DESCRIPTION, diff --git a/src/test/java/org/opensearch/knn/index/FaissIT.java b/src/test/java/org/opensearch/knn/index/FaissIT.java index f00942068..13d1bc64d 100644 --- a/src/test/java/org/opensearch/knn/index/FaissIT.java +++ b/src/test/java/org/opensearch/knn/index/FaissIT.java @@ -20,6 +20,7 @@ import org.opensearch.client.Response; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.settings.Settings; +import org.opensearch.client.ResponseException; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.index.query.QueryBuilders; import org.opensearch.knn.KNNRestTestCase; @@ -34,6 +35,7 @@ import java.net.URL; import java.util.Arrays; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Random; import java.util.TreeMap; @@ -44,8 +46,11 @@ import static org.opensearch.knn.common.KNNConstants.ENCODER_PQ; import static org.opensearch.knn.common.KNNConstants.ENCODER_SQ; import static org.opensearch.knn.common.KNNConstants.FAISS_NAME; +import static org.opensearch.knn.common.KNNConstants.FAISS_SQ_CLIP; import static org.opensearch.knn.common.KNNConstants.FAISS_SQ_ENCODER_FP16; import static org.opensearch.knn.common.KNNConstants.FAISS_SQ_TYPE; +import static org.opensearch.knn.common.KNNConstants.FP16_MAX_VALUE; +import static org.opensearch.knn.common.KNNConstants.FP16_MIN_VALUE; import static org.opensearch.knn.common.KNNConstants.KNN_ENGINE; import static org.opensearch.knn.common.KNNConstants.METHOD_ENCODER_PARAMETER; import static org.opensearch.knn.common.KNNConstants.METHOD_HNSW; @@ -391,6 +396,356 @@ public void testIVFSQFP16_whenIndexedAndQueried_thenSucceed() { validateGraphEviction(); } + @SneakyThrows + public void testHNSWSQFP16_whenIndexedWithOutOfFP16Range_thenThrowException() { + String indexName = "test-index-sqfp16"; + String fieldName = "test-field-sqfp16"; + + KNNMethod hnswMethod = KNNEngine.FAISS.getMethod(KNNConstants.METHOD_HNSW); + SpaceType[] spaceTypes = { SpaceType.L2, SpaceType.INNER_PRODUCT }; + Random random = new Random(); + SpaceType spaceType = spaceTypes[random.nextInt(spaceTypes.length)]; + + List mValues = ImmutableList.of(16, 32, 64, 128); + List efConstructionValues = ImmutableList.of(16, 32, 64, 128); + List efSearchValues = ImmutableList.of(16, 32, 64, 128); + + int dimension = 2; + + // Create an index + XContentBuilder builder = XContentFactory.jsonBuilder() + .startObject() + .startObject("properties") + .startObject(fieldName) + .field("type", "knn_vector") + .field("dimension", dimension) + .startObject(KNNConstants.KNN_METHOD) + .field(KNNConstants.NAME, hnswMethod.getMethodComponent().getName()) + .field(KNNConstants.METHOD_PARAMETER_SPACE_TYPE, spaceType.getValue()) + .field(KNNConstants.KNN_ENGINE, KNNEngine.FAISS.getName()) + .startObject(KNNConstants.PARAMETERS) + .field(KNNConstants.METHOD_PARAMETER_M, mValues.get(random().nextInt(mValues.size()))) + .field(KNNConstants.METHOD_PARAMETER_EF_CONSTRUCTION, efConstructionValues.get(random().nextInt(efConstructionValues.size()))) + .field(KNNConstants.METHOD_PARAMETER_EF_SEARCH, efSearchValues.get(random().nextInt(efSearchValues.size()))) + .startObject(METHOD_ENCODER_PARAMETER) + .field(NAME, ENCODER_SQ) + .startObject(PARAMETERS) + .field(FAISS_SQ_TYPE, FAISS_SQ_ENCODER_FP16) + .endObject() + .endObject() + .endObject() + .endObject() + .endObject() + .endObject() + .endObject(); + + Map mappingMap = xContentBuilderToMap(builder); + String mapping = builder.toString(); + + createKnnIndex(indexName, mapping); + assertEquals(new TreeMap<>(mappingMap), new TreeMap<>(getIndexMappingAsMap(indexName))); + Float[] vector = { -10.76f, 65504.2f }; + + ResponseException ex = expectThrows(ResponseException.class, () -> addKnnDoc(indexName, "1", fieldName, vector)); + assertTrue( + ex.getMessage() + .contains( + String.format( + Locale.ROOT, + "encoder name is set as [%s] and type is set as [%s] in index mapping. But, KNN vector values are not within in the FP16 range [%f, %f]", + ENCODER_SQ, + FAISS_SQ_ENCODER_FP16, + FP16_MIN_VALUE, + FP16_MAX_VALUE + ) + ) + ); + + Float[] vector1 = { -65506.84f, 12.56f }; + + ResponseException ex1 = expectThrows(ResponseException.class, () -> addKnnDoc(indexName, "2", fieldName, vector1)); + assertTrue( + ex1.getMessage() + .contains( + String.format( + Locale.ROOT, + "encoder name is set as [%s] and type is set as [%s] in index mapping. But, KNN vector values are not within in the FP16 range [%f, %f]", + ENCODER_SQ, + FAISS_SQ_ENCODER_FP16, + FP16_MIN_VALUE, + FP16_MAX_VALUE + ) + ) + ); + + Float[] vector2 = { -65526.4567f, 65526.4567f }; + + ResponseException ex2 = expectThrows(ResponseException.class, () -> addKnnDoc(indexName, "3", fieldName, vector2)); + assertTrue( + ex2.getMessage() + .contains( + String.format( + Locale.ROOT, + "encoder name is set as [%s] and type is set as [%s] in index mapping. But, KNN vector values are not within in the FP16 range [%f, %f]", + ENCODER_SQ, + FAISS_SQ_ENCODER_FP16, + FP16_MIN_VALUE, + FP16_MAX_VALUE + ) + ) + ); + deleteKNNIndex(indexName); + validateGraphEviction(); + } + + @SneakyThrows + public void testHNSWSQFP16_whenClipToFp16isTrueAndIndexedWithOutOfFP16Range_thenSucceed() { + String indexName = "test-index-sqfp16-clip-fp16"; + String fieldName = "test-field-sqfp16"; + + KNNMethod hnswMethod = KNNEngine.FAISS.getMethod(KNNConstants.METHOD_HNSW); + Random random = new Random(); + + List mValues = ImmutableList.of(16, 32, 64, 128); + List efConstructionValues = ImmutableList.of(16, 32, 64, 128); + List efSearchValues = ImmutableList.of(16, 32, 64, 128); + + int dimension = 2; + + // Create an index + XContentBuilder builder = XContentFactory.jsonBuilder() + .startObject() + .startObject("properties") + .startObject(fieldName) + .field("type", "knn_vector") + .field("dimension", dimension) + .startObject(KNNConstants.KNN_METHOD) + .field(KNNConstants.NAME, hnswMethod.getMethodComponent().getName()) + .field(KNNConstants.METHOD_PARAMETER_SPACE_TYPE, SpaceType.L2.getValue()) + .field(KNNConstants.KNN_ENGINE, KNNEngine.FAISS.getName()) + .startObject(KNNConstants.PARAMETERS) + .field(KNNConstants.METHOD_PARAMETER_M, mValues.get(random().nextInt(mValues.size()))) + .field(KNNConstants.METHOD_PARAMETER_EF_CONSTRUCTION, efConstructionValues.get(random().nextInt(efConstructionValues.size()))) + .field(KNNConstants.METHOD_PARAMETER_EF_SEARCH, efSearchValues.get(random().nextInt(efSearchValues.size()))) + .startObject(METHOD_ENCODER_PARAMETER) + .field(NAME, ENCODER_SQ) + .startObject(PARAMETERS) + .field(FAISS_SQ_TYPE, FAISS_SQ_ENCODER_FP16) + .field(FAISS_SQ_CLIP, true) + .endObject() + .endObject() + .endObject() + .endObject() + .endObject() + .endObject() + .endObject(); + + Map mappingMap = xContentBuilderToMap(builder); + String mapping = builder.toString(); + + createKnnIndex(indexName, mapping); + assertEquals(new TreeMap<>(mappingMap), new TreeMap<>(getIndexMappingAsMap(indexName))); + Float[] vector1 = { -65523.76f, 65504.2f }; + Float[] vector2 = { -270.85f, 65514.2f }; + Float[] vector3 = { -150.9f, 65504.0f }; + Float[] vector4 = { -20.89f, 100000000.0f }; + addKnnDoc(indexName, "1", fieldName, vector1); + addKnnDoc(indexName, "2", fieldName, vector2); + addKnnDoc(indexName, "3", fieldName, vector3); + addKnnDoc(indexName, "4", fieldName, vector4); + + float[] queryVector = { -10.5f, 25.48f }; + int k = 4; + Response searchResponse = searchKNNIndex(indexName, new KNNQueryBuilder(fieldName, queryVector, k), k); + List results = parseSearchResponse(EntityUtils.toString(searchResponse.getEntity()), fieldName); + assertEquals(k, results.size()); + for (int i = 0; i < k; i++) { + assertEquals(k - i, Integer.parseInt(results.get(i).getDocId())); + } + + deleteKNNIndex(indexName); + validateGraphEviction(); + } + + @SneakyThrows + public void testIVFSQFP16_whenIndexedWithOutOfFP16Range_thenThrowException() { + String modelId = "test-model-ivf-sqfp16"; + int dimension = 128; + + String trainingIndexName = "train-index-ivf-sqfp16"; + String trainingFieldName = "train-field-ivf-sqfp16"; + + // Add training data + createBasicKnnIndex(trainingIndexName, trainingFieldName, dimension); + int trainingDataCount = 200; + bulkIngestRandomVectors(trainingIndexName, trainingFieldName, trainingDataCount, dimension); + + XContentBuilder builder = XContentFactory.jsonBuilder() + .startObject() + .field(NAME, METHOD_IVF) + .field(KNN_ENGINE, FAISS_NAME) + .field(METHOD_PARAMETER_SPACE_TYPE, "l2") + .startObject(PARAMETERS) + .startObject(METHOD_ENCODER_PARAMETER) + .field(NAME, ENCODER_SQ) + .startObject(PARAMETERS) + .field(FAISS_SQ_TYPE, FAISS_SQ_ENCODER_FP16) + .endObject() + .endObject() + .endObject() + .endObject(); + Map method = xContentBuilderToMap(builder); + + trainModel(modelId, trainingIndexName, trainingFieldName, dimension, method, "faiss ivf sqfp16 test description"); + + // Make sure training succeeds after 30 seconds + assertTrainingSucceeds(modelId, 30, 1000); + + // Create knn index from model + String fieldName = "test-field-name-ivf-sqfp16"; + String indexName = "test-index-name-ivf-sqfp16"; + String indexMapping = XContentFactory.jsonBuilder() + .startObject() + .startObject("properties") + .startObject(fieldName) + .field("type", "knn_vector") + .field(MODEL_ID, modelId) + .endObject() + .endObject() + .endObject() + .toString(); + + createKnnIndex(indexName, getKNNDefaultIndexSettings(), indexMapping); + Float[] vector = { -10.76f, 65504.2f }; + + ResponseException ex = expectThrows(ResponseException.class, () -> addKnnDoc(indexName, "1", fieldName, vector)); + assertTrue( + ex.getMessage() + .contains( + String.format( + Locale.ROOT, + "encoder name is set as [%s] and type is set as [%s] in index mapping. But, KNN vector values are not within in the FP16 range [%f, %f]", + ENCODER_SQ, + FAISS_SQ_ENCODER_FP16, + FP16_MIN_VALUE, + FP16_MAX_VALUE + ) + ) + ); + + Float[] vector1 = { -65506.84f, 12.56f }; + + ResponseException ex1 = expectThrows(ResponseException.class, () -> addKnnDoc(indexName, "2", fieldName, vector1)); + assertTrue( + ex1.getMessage() + .contains( + String.format( + Locale.ROOT, + "encoder name is set as [%s] and type is set as [%s] in index mapping. But, KNN vector values are not within in the FP16 range [%f, %f]", + ENCODER_SQ, + FAISS_SQ_ENCODER_FP16, + FP16_MIN_VALUE, + FP16_MAX_VALUE + ) + ) + ); + + Float[] vector2 = { -65526.4567f, 65526.4567f }; + + ResponseException ex2 = expectThrows(ResponseException.class, () -> addKnnDoc(indexName, "3", fieldName, vector2)); + assertTrue( + ex2.getMessage() + .contains( + String.format( + Locale.ROOT, + "encoder name is set as [%s] and type is set as [%s] in index mapping. But, KNN vector values are not within in the FP16 range [%f, %f]", + ENCODER_SQ, + FAISS_SQ_ENCODER_FP16, + FP16_MIN_VALUE, + FP16_MAX_VALUE + ) + ) + ); + deleteKNNIndex(indexName); + deleteKNNIndex(trainingIndexName); + deleteModel(modelId); + } + + @SneakyThrows + public void testIVFSQFP16_whenClipToFp16isTrueAndIndexedWithOutOfFP16Range_thenSucceed() { + String modelId = "test-model-ivf-sqfp16"; + int dimension = 2; + + String trainingIndexName = "train-index-ivf-sqfp16"; + String trainingFieldName = "train-field-ivf-sqfp16"; + + // Add training data + createBasicKnnIndex(trainingIndexName, trainingFieldName, dimension); + int trainingDataCount = 200; + bulkIngestRandomVectors(trainingIndexName, trainingFieldName, trainingDataCount, dimension); + + XContentBuilder builder = XContentFactory.jsonBuilder() + .startObject() + .field(NAME, METHOD_IVF) + .field(KNN_ENGINE, FAISS_NAME) + .field(METHOD_PARAMETER_SPACE_TYPE, "l2") + .startObject(PARAMETERS) + .field(METHOD_PARAMETER_NLIST, 1) + .startObject(METHOD_ENCODER_PARAMETER) + .field(NAME, ENCODER_SQ) + .startObject(PARAMETERS) + .field(FAISS_SQ_TYPE, FAISS_SQ_ENCODER_FP16) + .field(FAISS_SQ_CLIP, true) + .endObject() + .endObject() + .endObject() + .endObject(); + Map method = xContentBuilderToMap(builder); + + trainModel(modelId, trainingIndexName, trainingFieldName, dimension, method, "faiss ivf sqfp16 test description"); + + // Make sure training succeeds after 30 seconds + assertTrainingSucceeds(modelId, 30, 1000); + + // Create knn index from model + String fieldName = "test-field-name-ivf-sqfp16"; + String indexName = "test-index-name-ivf-sqfp16"; + String indexMapping = XContentFactory.jsonBuilder() + .startObject() + .startObject("properties") + .startObject(fieldName) + .field("type", "knn_vector") + .field(MODEL_ID, modelId) + .endObject() + .endObject() + .endObject() + .toString(); + + createKnnIndex(indexName, getKNNDefaultIndexSettings(), indexMapping); + Float[] vector1 = { -65523.76f, 65504.2f }; + Float[] vector2 = { -270.85f, 65514.2f }; + Float[] vector3 = { -150.9f, 65504.0f }; + Float[] vector4 = { -20.89f, 100000000.0f }; + addKnnDoc(indexName, "1", fieldName, vector1); + addKnnDoc(indexName, "2", fieldName, vector2); + addKnnDoc(indexName, "3", fieldName, vector3); + addKnnDoc(indexName, "4", fieldName, vector4); + + float[] queryVector = { -10.5f, 25.48f }; + int k = 4; + Response searchResponse = searchKNNIndex(indexName, new KNNQueryBuilder(fieldName, queryVector, k), k); + List results = parseSearchResponse(EntityUtils.toString(searchResponse.getEntity()), fieldName); + assertEquals(k, results.size()); + for (int i = 0; i < k; i++) { + assertEquals(k - i, Integer.parseInt(results.get(i).getDocId())); + } + + deleteKNNIndex(indexName); + deleteKNNIndex(trainingIndexName); + deleteModel(modelId); + validateGraphEviction(); + } + @SneakyThrows public void testEndToEnd_whenMethodIsHNSWPQAndHyperParametersNotSet_thenSucceed() { String indexName = "test-index"; diff --git a/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java b/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java index 824b61a9f..a9b65878f 100644 --- a/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java +++ b/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java @@ -53,6 +53,10 @@ import static org.mockito.Mockito.doNothing; import static org.mockito.Mockito.doReturn; import static org.opensearch.knn.common.KNNConstants.DIMENSION; +import static org.opensearch.knn.common.KNNConstants.ENCODER_SQ; +import static org.opensearch.knn.common.KNNConstants.FAISS_SQ_ENCODER_FP16; +import static org.opensearch.knn.common.KNNConstants.FP16_MAX_VALUE; +import static org.opensearch.knn.common.KNNConstants.FP16_MIN_VALUE; import static org.opensearch.knn.common.KNNConstants.KNN_ENGINE; import static org.opensearch.knn.common.KNNConstants.KNN_METHOD; import static org.opensearch.knn.common.KNNConstants.LUCENE_NAME; @@ -68,6 +72,8 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; import static org.opensearch.knn.common.KNNConstants.VECTOR_DATA_TYPE_FIELD; +import static org.opensearch.knn.index.mapper.KNNVectorFieldMapperUtil.clipVectorValueToFP16Range; +import static org.opensearch.knn.index.mapper.KNNVectorFieldMapperUtil.validateFP16VectorValue; public class KNNVectorFieldMapperTests extends KNNTestCase { @@ -733,10 +739,16 @@ public void testLuceneFieldMapper_parseCreateField_docValues_withFloats() { when(parseContext.path()).thenReturn(contentPath); LuceneFieldMapper luceneFieldMapper = Mockito.spy(new LuceneFieldMapper(inputBuilder.build())); - doReturn(Optional.of(TEST_VECTOR)).when(luceneFieldMapper).getFloatsFromContext(parseContext, TEST_DIMENSION); + doReturn(Optional.of(TEST_VECTOR)).when(luceneFieldMapper) + .getFloatsFromContext(parseContext, TEST_DIMENSION, new MethodComponentContext(METHOD_HNSW, Collections.emptyMap())); doNothing().when(luceneFieldMapper).validateIfCircuitBreakerIsNotTriggered(); doNothing().when(luceneFieldMapper).validateIfKNNPluginEnabled(); - luceneFieldMapper.parseCreateField(parseContext, TEST_DIMENSION, luceneFieldMapper.fieldType().spaceType); + luceneFieldMapper.parseCreateField( + parseContext, + TEST_DIMENSION, + luceneFieldMapper.fieldType().spaceType, + luceneFieldMapper.fieldType().knnMethodContext.getMethodComponentContext() + ); // Document should have 2 fields: one for VectorField (binary doc values) and one for KnnVectorField List fields = document.getFields(); @@ -770,11 +782,17 @@ public void testLuceneFieldMapper_parseCreateField_docValues_withFloats() { inputBuilder.hasDocValues(false); luceneFieldMapper = Mockito.spy(new LuceneFieldMapper(inputBuilder.build())); - doReturn(Optional.of(TEST_VECTOR)).when(luceneFieldMapper).getFloatsFromContext(parseContext, TEST_DIMENSION); + doReturn(Optional.of(TEST_VECTOR)).when(luceneFieldMapper) + .getFloatsFromContext(parseContext, TEST_DIMENSION, new MethodComponentContext(METHOD_HNSW, Collections.emptyMap())); doNothing().when(luceneFieldMapper).validateIfCircuitBreakerIsNotTriggered(); doNothing().when(luceneFieldMapper).validateIfKNNPluginEnabled(); - luceneFieldMapper.parseCreateField(parseContext, TEST_DIMENSION, luceneFieldMapper.fieldType().spaceType); + luceneFieldMapper.parseCreateField( + parseContext, + TEST_DIMENSION, + luceneFieldMapper.fieldType().spaceType, + luceneFieldMapper.fieldType().knnMethodContext.getMethodComponentContext() + ); // Document should have 1 field: one for KnnVectorField fields = document.getFields(); @@ -803,7 +821,12 @@ public void testLuceneFieldMapper_parseCreateField_docValues_withBytes() { doNothing().when(luceneFieldMapper).validateIfCircuitBreakerIsNotTriggered(); doNothing().when(luceneFieldMapper).validateIfKNNPluginEnabled(); - luceneFieldMapper.parseCreateField(parseContext, TEST_DIMENSION, luceneFieldMapper.fieldType().spaceType); + luceneFieldMapper.parseCreateField( + parseContext, + TEST_DIMENSION, + luceneFieldMapper.fieldType().spaceType, + luceneFieldMapper.fieldType().knnMethodContext.getMethodComponentContext() + ); // Document should have 2 fields: one for VectorField (binary doc values) and one for KnnByteVectorField List fields = document.getFields(); @@ -840,7 +863,12 @@ public void testLuceneFieldMapper_parseCreateField_docValues_withBytes() { doNothing().when(luceneFieldMapper).validateIfCircuitBreakerIsNotTriggered(); doNothing().when(luceneFieldMapper).validateIfKNNPluginEnabled(); - luceneFieldMapper.parseCreateField(parseContext, TEST_DIMENSION, luceneFieldMapper.fieldType().spaceType); + luceneFieldMapper.parseCreateField( + parseContext, + TEST_DIMENSION, + luceneFieldMapper.fieldType().spaceType, + luceneFieldMapper.fieldType().knnMethodContext.getMethodComponentContext() + ); // Document should have 1 field: one for KnnByteVectorField fields = document.getFields(); @@ -851,6 +879,45 @@ public void testLuceneFieldMapper_parseCreateField_docValues_withBytes() { assertArrayEquals(TEST_BYTE_VECTOR, knnByteVectorField.vectorValue()); } + public void testValidateFp16VectorValue_outOfRange_throwsException() { + IllegalArgumentException ex = expectThrows(IllegalArgumentException.class, () -> validateFP16VectorValue(65505.25f)); + assertTrue( + ex.getMessage() + .contains( + String.format( + Locale.ROOT, + "encoder name is set as [%s] and type is set as [%s] in index mapping. But, KNN vector values are not within in the FP16 range [%f, %f]", + ENCODER_SQ, + FAISS_SQ_ENCODER_FP16, + FP16_MIN_VALUE, + FP16_MAX_VALUE + ) + ) + ); + + IllegalArgumentException ex1 = expectThrows(IllegalArgumentException.class, () -> validateFP16VectorValue(-65525.65f)); + assertTrue( + ex1.getMessage() + .contains( + String.format( + Locale.ROOT, + "encoder name is set as [%s] and type is set as [%s] in index mapping. But, KNN vector values are not within in the FP16 range [%f, %f]", + ENCODER_SQ, + FAISS_SQ_ENCODER_FP16, + FP16_MIN_VALUE, + FP16_MAX_VALUE + ) + ) + ); + } + + public void testClipVectorValuetoFP16Range_succeed() { + assertEquals(65504.0f, clipVectorValueToFP16Range(65504.10f), 0.0f); + assertEquals(65504.0f, clipVectorValueToFP16Range(1000000.89f), 0.0f); + assertEquals(-65504.0f, clipVectorValueToFP16Range(-65504.10f), 0.0f); + assertEquals(-65504.0f, clipVectorValueToFP16Range(-1000000.89f), 0.0f); + } + private LuceneFieldMapper.CreateLuceneFieldMapperInput.CreateLuceneFieldMapperInputBuilder createLuceneFieldMapperInputBuilder( VectorDataType vectorDataType ) { From 5cd2ab88a258f4b632f7963099cdc5afe60bfd29 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Mon, 18 Mar 2024 19:57:57 -0700 Subject: [PATCH 226/416] Add release notes for 2.13.0 release (#1564) (#1565) * Add release notes for 2.13.0 release Signed-off-by: Varun Jain * Add release notes for 2.13.0 release Signed-off-by: Varun Jain --------- Signed-off-by: Varun Jain (cherry picked from commit 9a38cfcda78c68f28f90e8197b24a88677ac0d6a) Co-authored-by: Varun Jain --- CHANGELOG.md | 18 +------------- .../opensearch-knn.release-notes-2.13.0.0.md | 24 +++++++++++++++++++ 2 files changed, 25 insertions(+), 17 deletions(-) create mode 100644 release-notes/opensearch-knn.release-notes-2.13.0.0.md diff --git a/CHANGELOG.md b/CHANGELOG.md index fdfedc126..2d9d35ef0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,27 +12,11 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Maintenance ### Refactoring -## [Unreleased 2.x](https://github.com/opensearch-project/k-NN/compare/2.12...2.x) +## [Unreleased 2.x](https://github.com/opensearch-project/k-NN/compare/2.13...2.x) ### Features ### Enhancements -* Optize Faiss Query With Filters: Reduce iteration and memory for id filter [#1402](https://github.com/opensearch-project/k-NN/pull/1402) -* Detect AVX2 Dynamically on the System [#1502](https://github.com/opensearch-project/k-NN/pull/1502) -* Validate zero vector when using cosine metric [#1501](https://github.com/opensearch-project/k-NN/pull/1501) -* Persist model definition in model metadata [#1527] (https://github.com/opensearch-project/k-NN/pull/1527) -* Added Inner Product Space type support for Lucene Engine [#1551](https://github.com/opensearch-project/k-NN/pull/1551) -* Add Range Validation for Faiss SQFP16 [#1493](https://github.com/opensearch-project/k-NN/pull/1493) -* SQFP16 Range Validation for Faiss IVF Models [#1557](https://github.com/opensearch-project/k-NN/pull/1557) ### Bug Fixes -* Disable sdc table for HNSWPQ read-only indices [#1518](https://github.com/opensearch-project/k-NN/pull/1518) -* Switch SpaceType.INNERPRODUCT's vector similarity function to MAXIMUM_INNER_PRODUCT [#1532](https://github.com/opensearch-project/k-NN/pull/1532) -* Add patch to fix arm segfault in nmslib during ingestion [#1541](https://github.com/opensearch-project/k-NN/pull/1541) -* Share ivfpq-l2 table allocations across indices on load [#1558](https://github.com/opensearch-project/k-NN/pull/1558) ### Infrastructure -* Manually install zlib for win CI [#1513](https://github.com/opensearch-project/k-NN/pull/1513) -* Update k-NN build artifact script to enable SIMD on ARM for Faiss [#1543](https://github.com/opensearch-project/k-NN/pull/1543) ### Documentation ### Maintenance -* Bump faiss lib commit to 32f0e8cf92cd2275b60364517bb1cce67aa29a55 [#1443](https://github.com/opensearch-project/k-NN/pull/1443) -* Fix FieldInfo Parameters Mismatch [#1489](https://github.com/opensearch-project/k-NN/pull/1489) -* Upgrade faiss to 12b92e9 [#1509](https://github.com/opensearch-project/k-NN/pull/1509) ### Refactoring diff --git a/release-notes/opensearch-knn.release-notes-2.13.0.0.md b/release-notes/opensearch-knn.release-notes-2.13.0.0.md new file mode 100644 index 000000000..9daa9bd4a --- /dev/null +++ b/release-notes/opensearch-knn.release-notes-2.13.0.0.md @@ -0,0 +1,24 @@ +## Version 2.13.0.0 Release Notes + +Compatible with OpenSearch 2.13.0 + +### Enhancements +* Optize Faiss Query With Filters: Reduce iteration and memory for id filter [#1402](https://github.com/opensearch-project/k-NN/pull/1402) +* Detect AVX2 Dynamically on the System [#1502](https://github.com/opensearch-project/k-NN/pull/1502) +* Validate zero vector when using cosine metric [#1501](https://github.com/opensearch-project/k-NN/pull/1501) +* Persist model definition in model metadata [#1527] (https://github.com/opensearch-project/k-NN/pull/1527) +* Added Inner Product Space type support for Lucene Engine [#1551](https://github.com/opensearch-project/k-NN/pull/1551) +* Add Range Validation for Faiss SQFP16 [#1493](https://github.com/opensearch-project/k-NN/pull/1493) +* SQFP16 Range Validation for Faiss IVF Models [#1557](https://github.com/opensearch-project/k-NN/pull/1557) +### Bug Fixes +* Disable sdc table for HNSWPQ read-only indices [#1518](https://github.com/opensearch-project/k-NN/pull/1518) +* Switch SpaceType.INNERPRODUCT's vector similarity function to MAXIMUM_INNER_PRODUCT [#1532](https://github.com/opensearch-project/k-NN/pull/1532) +* Add patch to fix arm segfault in nmslib during ingestion [#1541](https://github.com/opensearch-project/k-NN/pull/1541) +* Share ivfpq-l2 table allocations across indices on load [#1558](https://github.com/opensearch-project/k-NN/pull/1558) +### Infrastructure +* Manually install zlib for win CI [#1513](https://github.com/opensearch-project/k-NN/pull/1513) +* Update k-NN build artifact script to enable SIMD on ARM for Faiss [#1543](https://github.com/opensearch-project/k-NN/pull/1543) +### Maintenance +* Bump faiss lib commit to 32f0e8cf92cd2275b60364517bb1cce67aa29a55 [#1443](https://github.com/opensearch-project/k-NN/pull/1443) +* Fix FieldInfo Parameters Mismatch [#1490](https://github.com/opensearch-project/k-NN/pull/1490) +* Upgrade faiss to 12b92e9 [#1509](https://github.com/opensearch-project/k-NN/pull/1509) \ No newline at end of file From 5748b8cce27702af820a2174c6f725bb7ba537fe Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Tue, 19 Mar 2024 12:59:48 -0700 Subject: [PATCH 227/416] Upgrade byte-buddy version to 1.14.9 (#1567) (#1568) Signed-off-by: Varun Jain (cherry picked from commit e8c9cedd922ff4d36d7b133685c8a46919fab878) Co-authored-by: Varun Jain --- build.gradle | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build.gradle b/build.gradle index ca26b4921..8c7497069 100644 --- a/build.gradle +++ b/build.gradle @@ -283,9 +283,9 @@ dependencies { api group: 'com.google.guava', name: 'guava', version:'32.1.3-jre' api group: 'commons-lang', name: 'commons-lang', version: '2.6' testFixturesImplementation "org.opensearch.test:framework:${opensearch_version}" - testImplementation group: 'net.bytebuddy', name: 'byte-buddy', version: '1.14.7' + testImplementation group: 'net.bytebuddy', name: 'byte-buddy', version: '1.14.9' testImplementation group: 'org.objenesis', name: 'objenesis', version: '3.2' - testImplementation group: 'net.bytebuddy', name: 'byte-buddy-agent', version: '1.14.7' + testImplementation group: 'net.bytebuddy', name: 'byte-buddy-agent', version: '1.14.9' testFixturesImplementation "org.opensearch:common-utils:${version}" implementation 'com.github.oshi:oshi-core:6.4.13' api "net.java.dev.jna:jna:5.13.0" From 3ec60afb5ce48e1d8f67eb3f3406aecfc1133b55 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Wed, 27 Mar 2024 00:21:53 -0500 Subject: [PATCH 228/416] Add no new development notice to perf tool (#1578) (#1579) * Add no new development notice to perf tool and OSB Added notice to inform developers to consider adding new features to OSB Add no feature development notice to OSB Signed-off-by: Vijayan Balasubramanian (cherry picked from commit 17bbbd9317e84d94d7f1e0192fe6c6c8a5653fd7) Co-authored-by: Vijayan Balasubramanian --- benchmarks/osb/README.md | 1 + benchmarks/perf-tool/README.md | 2 ++ 2 files changed, 3 insertions(+) diff --git a/benchmarks/osb/README.md b/benchmarks/osb/README.md index 0f806a344..0d0b05f8d 100644 --- a/benchmarks/osb/README.md +++ b/benchmarks/osb/README.md @@ -1,3 +1,4 @@ +# IMPORTANT NOTE: No new features will be added to this tool . This tool is currently in maintanence mode. All new features will be added to [vector search workload]( https://github.com/opensearch-project/opensearch-benchmark-workloads/tree/main/vectorsearch) # OpenSearch Benchmarks for k-NN ## Overview diff --git a/benchmarks/perf-tool/README.md b/benchmarks/perf-tool/README.md index 52590d22b..36f76bcdb 100644 --- a/benchmarks/perf-tool/README.md +++ b/benchmarks/perf-tool/README.md @@ -1,3 +1,5 @@ +# IMPORTANT NOTE: No new features will be added to this tool . This tool is currently in maintanence mode. All new features will be added to [vector search workload]( https://github.com/opensearch-project/opensearch-benchmark-workloads/tree/main/vectorsearch) + # OpenSearch k-NN Benchmarking - [Welcome!](#welcome) - [Install Prerequisites](#install-prerequisites) From 5afbbc0f5c373879f15f060e3be0948d77aa224f Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Wed, 3 Apr 2024 11:19:44 -0700 Subject: [PATCH 229/416] Increment version to 2.14.0-SNAPSHOT (#1574) Signed-off-by: opensearch-ci-bot Co-authored-by: opensearch-ci-bot --- .../workflows/backwards_compatibility_tests_workflow.yml | 8 ++++---- build.gradle | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/backwards_compatibility_tests_workflow.yml b/.github/workflows/backwards_compatibility_tests_workflow.yml index 8cda91f14..4b1a0ac23 100644 --- a/.github/workflows/backwards_compatibility_tests_workflow.yml +++ b/.github/workflows/backwards_compatibility_tests_workflow.yml @@ -14,8 +14,8 @@ jobs: strategy: matrix: java: [ 11, 17 ] - bwc_version : [ "1.1.0", "1.2.4", "1.3.8", "2.0.1", "2.1.0", "2.2.1", "2.3.0", "2.4.1", "2.5.0", "2.6.0", "2.7.0", "2.8.0", "2.9.0", "2.10.0", "2.11.0", "2.12.0" ] - opensearch_version : [ "2.13.0-SNAPSHOT" ] + bwc_version : [ "1.1.0", "1.2.4", "1.3.8", "2.0.1", "2.1.0", "2.2.1", "2.3.0", "2.4.1", "2.5.0", "2.6.0", "2.7.0", "2.8.0", "2.9.0", "2.10.0", "2.11.0", "2.12.0", "2.13.0" ] + opensearch_version : [ "2.14.0-SNAPSHOT" ] name: k-NN Restart-Upgrade BWC Tests runs-on: ubuntu-latest @@ -46,8 +46,8 @@ jobs: strategy: matrix: java: [ 11, 17 ] - bwc_version: [ "1.3.8", "2.0.1", "2.1.0", "2.2.1", "2.3.0", "2.4.1", "2.5.0", "2.6.0", "2.7.0", "2.8.0", "2.9.0", "2.10.0", "2.11.0", "2.12.0"] - opensearch_version: [ "2.13.0-SNAPSHOT" ] + bwc_version: [ "1.3.8", "2.0.1", "2.1.0", "2.2.1", "2.3.0", "2.4.1", "2.5.0", "2.6.0", "2.7.0", "2.8.0", "2.9.0", "2.10.0", "2.11.0", "2.12.0", "2.13.0"] + opensearch_version: [ "2.14.0-SNAPSHOT" ] name: k-NN Rolling-Upgrade BWC Tests runs-on: ubuntu-latest diff --git a/build.gradle b/build.gradle index 8c7497069..6a5fa1d13 100644 --- a/build.gradle +++ b/build.gradle @@ -13,7 +13,7 @@ buildscript { ext { // build.version_qualifier parameter applies to knn plugin artifacts only. OpenSearch version must be set // explicitly as 'opensearch.version' property, for instance opensearch.version=2.0.0-rc1-SNAPSHOT - opensearch_version = System.getProperty("opensearch.version", "2.13.0-SNAPSHOT") + opensearch_version = System.getProperty("opensearch.version", "2.14.0-SNAPSHOT") version_qualifier = System.getProperty("build.version_qualifier", "") opensearch_group = "org.opensearch" isSnapshot = "true" == System.getProperty("build.snapshot", "true") From 4c960a0290aaf7231b8916ee957c6023f5a8bd7c Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Wed, 3 Apr 2024 14:14:33 -0700 Subject: [PATCH 230/416] Add micro-benchmarks module in k-NN plugin for benchmark streaming vectors to JNI layer functionality. (#1583) (#1584) Signed-off-by: Navneet Verma (cherry picked from commit cc91c7cf17b2001550463e8e6be4f00269aeba05) Co-authored-by: Navneet Verma --- CHANGELOG.md | 1 + .../org_opensearch_knn_jni_FaissService.h | 8 ++ .../org_opensearch_knn_jni_FaissService.cpp | 18 ++++ micro-benchmarks/README.md | 97 +++++++++++++++++++ micro-benchmarks/build.gradle | 68 +++++++++++++ .../knn/TransferVectorsBenchmarks.java | 87 +++++++++++++++++ .../src/main/resources/log4j2.properties | 19 ++++ settings.gradle | 1 + .../org/opensearch/knn/index/KNNSettings.java | 23 ++++- .../org/opensearch/knn/jni/FaissService.java | 15 +++ .../org/opensearch/knn/jni/JNIService.java | 22 +++++ 11 files changed, 357 insertions(+), 2 deletions(-) create mode 100644 micro-benchmarks/README.md create mode 100644 micro-benchmarks/build.gradle create mode 100644 micro-benchmarks/src/main/java/org/opensearch/knn/TransferVectorsBenchmarks.java create mode 100644 micro-benchmarks/src/main/resources/log4j2.properties diff --git a/CHANGELOG.md b/CHANGELOG.md index 2d9d35ef0..04c972711 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Enhancements ### Bug Fixes ### Infrastructure +* Add micro-benchmark module in k-NN plugin for benchmark streaming vectors to JNI layer functionality. [#1583](https://github.com/opensearch-project/k-NN/pull/1583) ### Documentation ### Maintenance ### Refactoring diff --git a/jni/include/org_opensearch_knn_jni_FaissService.h b/jni/include/org_opensearch_knn_jni_FaissService.h index 64a858f84..ec1f46bc3 100644 --- a/jni/include/org_opensearch_knn_jni_FaissService.h +++ b/jni/include/org_opensearch_knn_jni_FaissService.h @@ -122,6 +122,14 @@ JNIEXPORT jbyteArray JNICALL Java_org_opensearch_knn_jni_FaissService_trainIndex JNIEXPORT jlong JNICALL Java_org_opensearch_knn_jni_FaissService_transferVectors (JNIEnv *, jclass, jlong, jobjectArray); +/* + * Class: org_opensearch_knn_jni_FaissService + * Method: transferVectorsV2 + * Signature: (J[[F)J + */ +JNIEXPORT jlong JNICALL Java_org_opensearch_knn_jni_FaissService_transferVectorsV2 + (JNIEnv *, jclass, jlong, jobjectArray); + /* * Class: org_opensearch_knn_jni_FaissService * Method: freeVectors diff --git a/jni/src/org_opensearch_knn_jni_FaissService.cpp b/jni/src/org_opensearch_knn_jni_FaissService.cpp index c81f23a62..3d9624c25 100644 --- a/jni/src/org_opensearch_knn_jni_FaissService.cpp +++ b/jni/src/org_opensearch_knn_jni_FaissService.cpp @@ -191,6 +191,24 @@ JNIEXPORT jlong JNICALL Java_org_opensearch_knn_jni_FaissService_transferVectors return (jlong) vect; } +JNIEXPORT jlong JNICALL Java_org_opensearch_knn_jni_FaissService_transferVectorsV2(JNIEnv * env, jclass cls, +jlong vectorsPointerJ, + jobjectArray vectorsJ) +{ + std::vector *vect; + if ((long) vectorsPointerJ == 0) { + vect = new std::vector; + } else { + vect = reinterpret_cast*>(vectorsPointerJ); + } + + int dim = jniUtil.GetInnerDimensionOf2dJavaFloatArray(env, vectorsJ); + auto dataset = jniUtil.Convert2dJavaObjectArrayToCppFloatVector(env, vectorsJ, dim); + vect->insert(vect->end(), dataset.begin(), dataset.end()); + + return (jlong) vect; +} + JNIEXPORT void JNICALL Java_org_opensearch_knn_jni_FaissService_freeVectors(JNIEnv * env, jclass cls, jlong vectorsPointerJ) { diff --git a/micro-benchmarks/README.md b/micro-benchmarks/README.md new file mode 100644 index 000000000..0a676004b --- /dev/null +++ b/micro-benchmarks/README.md @@ -0,0 +1,97 @@ +# OpenSearch K-NN Microbenchmark Suite + +This directory contains the microbenchmark suite of Opensearch K-NN Plugin. It relies on [JMH](http://openjdk.java.net/projects/code-tools/jmh/). + +This module draws a lot of inspiration from [Opensearch benchmarks](https://github.com/opensearch-project/OpenSearch/tree/main/benchmarks). + +## Purpose + +Micro benchmarks are intended to spot performance regressions in performance-critical components. + +The microbenchmark suite is also handy for ad-hoc micro benchmarks but please remove them again before merging your PR. + +## Getting Started + +Just run `gradlew -p micro-benchmarks run` from the project root +directory. It will build all microbenchmarks, execute them and print +the result. + +## Running Microbenchmarks + +Running via an IDE is not supported as the results are meaningless +because we have no control over the JVM running the benchmarks. + +If you want to run a specific benchmark class like, say, +`TransferVectorsBenchmarks`, you can use `--args`: + +``` +gradlew -p micro-benchmarks run --args ' TransferVectorsBenchmarks' +``` + +Setting Heap while running the benchmarks +``` +./gradlew -p micro-benchmarks run --args ' -gc true ' -Djvm.heap.size=4g +``` + +Everything in the `'` gets sent on the command line to JMH. The leading ` ` +inside the `'`s is important. Without it parameters are sometimes sent to +gradle. + +## Adding Microbenchmarks + +Before adding a new microbenchmark, make yourself familiar with the JMH API. You can check our existing microbenchmarks and also the +[JMH samples](http://hg.openjdk.java.net/code-tools/jmh/file/tip/jmh-samples/src/main/java/org/openjdk/jmh/samples/). + +In contrast to tests, the actual name of the benchmark class is not relevant to JMH. However, stick to the naming convention and +end the class name of a benchmark with `Benchmark`. To have JMH execute a benchmark, annotate the respective methods with `@Benchmark`. + +## Tips and Best Practices + +To get realistic results, you should exercise care when running benchmarks. Here are a few tips: + +### Do + +* Ensure that the system executing your microbenchmarks has as little load as possible. Shutdown every process that can cause unnecessary + runtime jitter. Watch the `Error` column in the benchmark results to see the run-to-run variance. +* Ensure to run enough warmup iterations to get the benchmark into a stable state. If you are unsure, don't change the defaults. +* Avoid CPU migrations by pinning your benchmarks to specific CPU cores. On Linux you can use `taskset`. +* Fix the CPU frequency to avoid Turbo Boost from kicking in and skewing your results. On Linux you can use `cpufreq-set` and the + `performance` CPU governor. +* Vary the problem input size with `@Param`. +* Use the integrated profilers in JMH to dig deeper if benchmark results to not match your hypotheses: + * Add `-prof gc` to the options to check whether the garbage collector runs during a microbenchmarks and skews + your results. If so, try to force a GC between runs (`-gc true`) but watch out for the caveats. + * Add `-prof perf` or `-prof perfasm` (both only available on Linux) to see hotspots. +* Have your benchmarks peer-reviewed. + +### Don't + +* Blindly believe the numbers that your microbenchmark produces but verify them by measuring e.g. with `-prof perfasm`. +* Run more threads than your number of CPU cores (in case you run multi-threaded microbenchmark). +* Look only at the `Score` column and ignore `Error`. Instead, take countermeasures to keep `Error` low / variance explainable. + +## Disassembling + +Disassembling is fun! Maybe not always useful, but always fun! Generally, you'll want to install `perf` and FCML's `hsdis`. +`perf` is generally available via `apg-get install perf` or `pacman -S perf`. FCML is a little more involved. This worked +on 2020-08-01: + +``` +wget https://github.com/swojtasiak/fcml-lib/releases/download/v1.2.2/fcml-1.2.2.tar.gz +tar xf fcml* +cd fcml* +./configure +make +cd example/hsdis +make +sudo cp .libs/libhsdis.so.0.0.0 /usr/lib/jvm/java-14-adoptopenjdk/lib/hsdis-amd64.so +``` + +If you want to disassemble a single method do something like this: + +``` +gradlew -p micro-benchmarks run --args ' MemoryStatsBenchmark -jvmArgs "-XX:+UnlockDiagnosticVMOptions -XX:CompileCommand=print,*.yourMethodName -XX:PrintAssemblyOptions=intel" +``` + + +If you want `perf` to find the hot methods for you then do add `-prof:perfasm`. diff --git a/micro-benchmarks/build.gradle b/micro-benchmarks/build.gradle new file mode 100644 index 000000000..b1da431fa --- /dev/null +++ b/micro-benchmarks/build.gradle @@ -0,0 +1,68 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +import org.opensearch.gradle.info.BuildParams + +apply plugin: 'opensearch.build' +apply plugin: 'application' +apply plugin: 'java' +apply plugin: 'io.freefair.lombok' + +assemble.enabled = false + +application { + mainClass = 'org.openjdk.jmh.Main' +} + +test.enabled = false + +repositories { + mavenLocal() + maven { url "https://aws.oss.sonatype.org/content/repositories/snapshots" } + mavenCentral() + maven { url "https://plugins.gradle.org/m2/" } +} + +dependencies { + // This will take root project as the dependency + api(project(':')) + api "org.openjdk.jmh:jmh-core:$versions.jmh" + annotationProcessor "org.openjdk.jmh:jmh-generator-annprocess:$versions.jmh" + // Dependencies of JMH + runtimeOnly 'net.sf.jopt-simple:jopt-simple:5.0.4' + runtimeOnly 'org.apache.commons:commons-math3:3.6.1' +} + +// enable the JMH's BenchmarkProcessor to generate the final benchmark classes +// needs to be added separately otherwise Gradle will quote it and javac will fail +compileJava.options.compilerArgs.addAll(["-processor", "org.openjdk.jmh.generators.BenchmarkProcessor"]) + + +run { + // This is required for C++ code + systemProperty "java.library.path", "$rootDir/jni/release" + executable = "${BuildParams.runtimeJavaHome}/bin/java" + var jvmHeapSize = System.getProperty("jvm.heap.size", "6g") + jvmArgs("-Xms" + jvmHeapSize, "-Xmx" + jvmHeapSize) +} + + +// No licenses for our benchmark deps (we don't ship benchmarks) +tasks.named("dependencyLicenses").configure { it.enabled = false } +dependenciesInfo.enabled = false + +thirdPartyAudit.ignoreViolations( + // these classes intentionally use JDK internal API (and this is ok since the project is maintained by Oracle employees) + 'org.openjdk.jmh.util.Utils' +) + +spotless { + java { + // IDEs can sometimes run annotation processors that leave files in + // here, causing Spotless to complain. Even though this path ought not + // to exist, exclude it anyway in order to avoid spurious failures. + targetExclude 'src/main/generated/**/*.java' + } +} + diff --git a/micro-benchmarks/src/main/java/org/opensearch/knn/TransferVectorsBenchmarks.java b/micro-benchmarks/src/main/java/org/opensearch/knn/TransferVectorsBenchmarks.java new file mode 100644 index 000000000..ad1076484 --- /dev/null +++ b/micro-benchmarks/src/main/java/org/opensearch/knn/TransferVectorsBenchmarks.java @@ -0,0 +1,87 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.knn; + +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Warmup; +import org.opensearch.knn.jni.JNIService; + +import java.util.ArrayList; +import java.util.List; +import java.util.Random; +import java.util.concurrent.TimeUnit; + +/** + * The class provides runs some benchmarks and provide the performance data around how much time it will take to + * transfer vectors from java to jni layer for different configuration. + */ +@Warmup(iterations = 1, timeUnit = TimeUnit.SECONDS, time = 300) +@Measurement(iterations = 1, timeUnit = TimeUnit.SECONDS, time = 300) +@Fork(3) +@BenchmarkMode(Mode.SingleShotTime) +@OutputTimeUnit(TimeUnit.SECONDS) +@State(Scope.Benchmark) +public class TransferVectorsBenchmarks { + private static final Random random = new Random(1212121212); + private static final int TOTAL_NUMBER_OF_VECTOR_TO_BE_TRANSFERRED = 1000000; + + @Param({ "128", "256", "384", "512" }) + private int dimension; + + @Param({ "100000", "500000", "1000000" }) + private int vectorsPerTransfer; + + private List vectorList; + + @Setup(Level.Trial) + public void setup() { + vectorList = new ArrayList<>(); + for (int i = 0; i < TOTAL_NUMBER_OF_VECTOR_TO_BE_TRANSFERRED; i++) { + vectorList.add(generateRandomVector(dimension)); + } + } + + @Benchmark + public void transferVectors() { + long vectorsAddress = 0; + List vectorToTransfer = new ArrayList<>(); + for (float[] floats : vectorList) { + if (vectorToTransfer.size() == vectorsPerTransfer) { + vectorsAddress = JNIService.transferVectorsV2(vectorsAddress, vectorToTransfer.toArray(new float[][] {})); + vectorToTransfer = new ArrayList<>(); + } + vectorToTransfer.add(floats); + } + if (!vectorToTransfer.isEmpty()) { + vectorsAddress = JNIService.transferVectorsV2(vectorsAddress, vectorToTransfer.toArray(new float[][] {})); + } + JNIService.freeVectors(vectorsAddress); + } + + private float[] generateRandomVector(int dimensions) { + float[] vector = new float[dimensions]; + for (int i = 0; i < dimensions; i++) { + vector[i] = -500 + (float) random.nextGaussian() * (1000); + } + return vector; + } +} diff --git a/micro-benchmarks/src/main/resources/log4j2.properties b/micro-benchmarks/src/main/resources/log4j2.properties new file mode 100644 index 000000000..2cd74124e --- /dev/null +++ b/micro-benchmarks/src/main/resources/log4j2.properties @@ -0,0 +1,19 @@ +# +# SPDX-License-Identifier: Apache-2.0 +# +# The OpenSearch Contributors require contributions made to +# this file be licensed under the Apache-2.0 license or a +# compatible open source license. +# +# Modifications Copyright OpenSearch Contributors. See +# GitHub history for details. +# + +appender.console.type = Console +appender.console.name = console +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %marker %m%n + +# Do not log at all if it is not really critical - we're in a benchmark +rootLogger.level = error +rootLogger.appenderRef.console.ref = console diff --git a/settings.gradle b/settings.gradle index 9056e382e..fd4369d4a 100644 --- a/settings.gradle +++ b/settings.gradle @@ -8,4 +8,5 @@ rootProject.name = 'opensearch-knn' include ":qa" include ":qa:rolling-upgrade" include ":qa:restart-upgrade" +include ":micro-benchmarks" diff --git a/src/main/java/org/opensearch/knn/index/KNNSettings.java b/src/main/java/org/opensearch/knn/index/KNNSettings.java index ba172bc30..9ac0bf216 100644 --- a/src/main/java/org/opensearch/knn/index/KNNSettings.java +++ b/src/main/java/org/opensearch/knn/index/KNNSettings.java @@ -5,6 +5,7 @@ package org.opensearch.knn.index; +import lombok.extern.log4j.Log4j2; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.opensearch.OpenSearchParseException; @@ -48,6 +49,7 @@ * 2. KNN settings to enable/disable plugin, circuit breaker settings * 3. KNN settings to manage graphs loaded in native memory */ +@Log4j2 public class KNNSettings { private static final Logger logger = LogManager.getLogger(KNNSettings.class); @@ -81,6 +83,7 @@ public class KNNSettings { /** * Default setting values */ + public static final boolean KNN_DEFAULT_FAISS_AVX2_DISABLED_VALUE = false; public static final String INDEX_KNN_DEFAULT_SPACE_TYPE = "l2"; public static final Integer INDEX_KNN_DEFAULT_ALGO_PARAM_M = 16; public static final Integer INDEX_KNN_DEFAULT_ALGO_PARAM_EF_SEARCH = 100; @@ -232,7 +235,11 @@ public class KNNSettings { Dynamic ); - public static final Setting KNN_FAISS_AVX2_DISABLED_SETTING = Setting.boolSetting(KNN_FAISS_AVX2_DISABLED, false, NodeScope); + public static final Setting KNN_FAISS_AVX2_DISABLED_SETTING = Setting.boolSetting( + KNN_FAISS_AVX2_DISABLED, + KNN_DEFAULT_FAISS_AVX2_DISABLED_VALUE, + NodeScope + ); /** * Dynamic settings @@ -386,7 +393,19 @@ public static double getCircuitBreakerUnsetPercentage() { } public static boolean isFaissAVX2Disabled() { - return KNNSettings.state().getSettingValue(KNNSettings.KNN_FAISS_AVX2_DISABLED); + try { + return KNNSettings.state().getSettingValue(KNNSettings.KNN_FAISS_AVX2_DISABLED); + } catch (Exception e) { + // In some UTs we identified that cluster setting is not set properly an leads to NPE. This check will avoid + // those cases and will still return the default value. + log.warn( + "Unable to get setting value {} from cluster settings. Using default value as {}", + KNN_FAISS_AVX2_DISABLED, + KNN_DEFAULT_FAISS_AVX2_DISABLED_VALUE, + e + ); + return KNN_DEFAULT_FAISS_AVX2_DISABLED_VALUE; + } } public static Integer getFilteredExactSearchThreshold(final String indexName) { diff --git a/src/main/java/org/opensearch/knn/jni/FaissService.java b/src/main/java/org/opensearch/knn/jni/FaissService.java index 0da6f54ef..4b5045359 100644 --- a/src/main/java/org/opensearch/knn/jni/FaissService.java +++ b/src/main/java/org/opensearch/knn/jni/FaissService.java @@ -181,6 +181,21 @@ public static native KNNQueryResult[] queryIndexWithFilter( */ public static native long transferVectors(long vectorsPointer, float[][] trainingData); + /** + * Transfer vectors from Java to native layer. This is the version 2 of transfer vector functionality. The + * difference between this and the version 1 is, this version puts vectors at the end rather than in front. + * Keeping this name as V2 for now, will come up with better name going forward. + *

+ * TODO: Rename the function + *
+ * TODO: Make this function native function and use a common cpp file to host these functions. + *

+ * @param vectorsPointer pointer to vectors in native memory. Should be 0 to create vector as well + * @param data data to be transferred + * @return pointer to native memory location for data + */ + public static native long transferVectorsV2(long vectorsPointer, float[][] data); + /** * Free vectors from memory * diff --git a/src/main/java/org/opensearch/knn/jni/JNIService.java b/src/main/java/org/opensearch/knn/jni/JNIService.java index 555c2d6a6..80b56b173 100644 --- a/src/main/java/org/opensearch/knn/jni/JNIService.java +++ b/src/main/java/org/opensearch/knn/jni/JNIService.java @@ -253,4 +253,26 @@ public static long transferVectors(long vectorsPointer, float[][] trainingData) public static void freeVectors(long vectorsPointer) { FaissService.freeVectors(vectorsPointer); } + + /** + * Experimental: Transfer vectors from Java to native layer. This is the version 2 of transfer vector + * functionality. The difference between this and the version 1 is, this version puts vectors at the end rather + * than in front. Keeping this name as V2 for now, will come up with better name going forward. + *

+ * This is not a production ready function for now. Adding this to ensure that we are able to run atleast 1 + * micro-benchmarks. + *

+ *

+ * TODO: Rename the function + *
+ * TODO: Make this function native function and use a common cpp file to host these functions. + *

+ * @param vectorsPointer pointer to vectors in native memory. Should be 0 to create vector as well + * @param data data to be transferred + * @return pointer to native memory location for data + * + */ + public static long transferVectorsV2(long vectorsPointer, float[][] data) { + return FaissService.transferVectorsV2(vectorsPointer, data); + } } From 551068bad2bfffe4a92b31e22d8e572df53fb0d0 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Wed, 3 Apr 2024 14:19:23 -0700 Subject: [PATCH 231/416] Make the HitQueue size more appropriate for exact search (#1549) (#1580) Signed-off-by: panguixin (cherry picked from commit c861966708219b5a0c27fa60e6eb1c150dfc0efa) Co-authored-by: panguixin --- CHANGELOG.md | 1 + .../java/org/opensearch/knn/index/query/KNNWeight.java | 8 ++++---- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 04c972711..03de217a8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ## [Unreleased 2.x](https://github.com/opensearch-project/k-NN/compare/2.13...2.x) ### Features ### Enhancements +* Make the HitQueue size more appropriate for exact search [#1549](https://github.com/opensearch-project/k-NN/pull/1549) ### Bug Fixes ### Infrastructure * Add micro-benchmark module in k-NN plugin for benchmark streaming vectors to JNI layer functionality. [#1583](https://github.com/opensearch-project/k-NN/pull/1583) diff --git a/src/main/java/org/opensearch/knn/index/query/KNNWeight.java b/src/main/java/org/opensearch/knn/index/query/KNNWeight.java index 1c4d0a646..06bf96d63 100644 --- a/src/main/java/org/opensearch/knn/index/query/KNNWeight.java +++ b/src/main/java/org/opensearch/knn/index/query/KNNWeight.java @@ -117,7 +117,7 @@ public Scorer scorer(LeafReaderContext context) throws IOException { * This improves the recall. */ if (filterWeight != null && canDoExactSearch(cardinality)) { - docIdsToScoreMap.putAll(doExactSearch(context, filterBitSet)); + docIdsToScoreMap.putAll(doExactSearch(context, filterBitSet, cardinality)); } else { Map annResults = doANNSearch(context, filterBitSet, cardinality); if (annResults == null) { @@ -131,7 +131,7 @@ public Scorer scorer(LeafReaderContext context) throws IOException { annResults.size(), cardinality ); - annResults = doExactSearch(context, filterBitSet); + annResults = doExactSearch(context, filterBitSet, cardinality); } docIdsToScoreMap.putAll(annResults); } @@ -309,10 +309,10 @@ private Map doANNSearch(final LeafReaderContext context, final B .collect(Collectors.toMap(KNNQueryResult::getId, result -> knnEngine.score(result.getScore(), spaceType))); } - private Map doExactSearch(final LeafReaderContext leafReaderContext, final BitSet filterIdsBitSet) { + private Map doExactSearch(final LeafReaderContext leafReaderContext, final BitSet filterIdsBitSet, int cardinality) { try { // Creating min heap and init with MAX DocID and Score as -INF. - final HitQueue queue = new HitQueue(this.knnQuery.getK(), true); + final HitQueue queue = new HitQueue(Math.min(this.knnQuery.getK(), cardinality), true); ScoreDoc topDoc = queue.top(); final Map docToScore = new HashMap<>(); FilteredIdsKNNIterator iterator = getFilteredKNNIterator(leafReaderContext, filterIdsBitSet); From bc14182a9b1ae80648e471c67a2eba7acfd56873 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Mon, 8 Apr 2024 09:10:50 -0700 Subject: [PATCH 232/416] Fix script to pick up correct zip (#1596) Signed-off-by: John Mazanec (cherry picked from commit 653e7eb49769a60813a680813ec8df341742883d) --- scripts/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/build.sh b/scripts/build.sh index 700ab5731..a5ab477f3 100644 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -129,7 +129,7 @@ fi ./gradlew publishPluginZipPublicationToMavenLocal -Dbuild.snapshot=$SNAPSHOT -Dbuild.version_qualifier=$QUALIFIER -Dopensearch.version=$VERSION # Add lib to zip -zipPath=$(find "$(pwd)" -path \*build/distributions/*.zip) +zipPath=$(find "$(pwd)/build/distributions" -path \*.zip) distributions="$(dirname "${zipPath}")" mkdir $distributions/lib libPrefix="libopensearchknn" From d4e91e071e9ee71d045635a5fa18b94d7dc8a725 Mon Sep 17 00:00:00 2001 From: panguixin Date: Wed, 10 Apr 2024 01:54:18 +0800 Subject: [PATCH 233/416] Support script score when doc value is disabled (#1573) (#1587) * support script score when doc value is disabled Signed-off-by: panguixin * add test Signed-off-by: panguixin * apply review comments Signed-off-by: panguixin * fix test Signed-off-by: panguixin --------- Signed-off-by: panguixin (cherry picked from commit 771c4b54a74b7c4406c71a8bf758378329cfe4d5) --- CHANGELOG.md | 1 + .../knn/index/KNNVectorDVLeafFieldData.java | 28 +- .../knn/index/KNNVectorScriptDocValues.java | 109 ++++- .../org/opensearch/knn/index/FaissIT.java | 6 +- .../index/KNNVectorScriptDocValuesTests.java | 62 ++- .../opensearch/knn/index/LuceneEngineIT.java | 9 +- .../org/opensearch/knn/index/NmslibIT.java | 4 +- .../opensearch/knn/index/OpenSearchIT.java | 5 +- .../knn/index/VectorDataTypeTests.java | 4 +- .../plugin/script/KNNScoringUtilTests.java | 2 +- .../knn/plugin/script/KNNScriptScoringIT.java | 385 ++++++------------ .../knn/plugin/script/PainlessScriptIT.java | 20 +- .../org/opensearch/knn/KNNRestTestCase.java | 49 ++- .../java/org/opensearch/knn/KNNResult.java | 27 +- 14 files changed, 385 insertions(+), 326 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 03de217a8..e82376a6d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Features ### Enhancements * Make the HitQueue size more appropriate for exact search [#1549](https://github.com/opensearch-project/k-NN/pull/1549) +* Support script score when doc value is disabled [#1573](https://github.com/opensearch-project/k-NN/pull/1573) ### Bug Fixes ### Infrastructure * Add micro-benchmark module in k-NN plugin for benchmark streaming vectors to JNI layer functionality. [#1583](https://github.com/opensearch-project/k-NN/pull/1583) diff --git a/src/main/java/org/opensearch/knn/index/KNNVectorDVLeafFieldData.java b/src/main/java/org/opensearch/knn/index/KNNVectorDVLeafFieldData.java index f4caa4f20..85f037c0f 100644 --- a/src/main/java/org/opensearch/knn/index/KNNVectorDVLeafFieldData.java +++ b/src/main/java/org/opensearch/knn/index/KNNVectorDVLeafFieldData.java @@ -5,9 +5,10 @@ package org.opensearch.knn.index; -import org.apache.lucene.index.BinaryDocValues; import org.apache.lucene.index.DocValues; +import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.LeafReader; +import org.apache.lucene.search.DocIdSetIterator; import org.opensearch.index.fielddata.LeafFieldData; import org.opensearch.index.fielddata.ScriptDocValues; import org.opensearch.index.fielddata.SortedBinaryDocValues; @@ -39,10 +40,29 @@ public long ramBytesUsed() { @Override public ScriptDocValues getScriptValues() { try { - BinaryDocValues values = DocValues.getBinary(reader, fieldName); - return new KNNVectorScriptDocValues(values, fieldName, vectorDataType); + FieldInfo fieldInfo = reader.getFieldInfos().fieldInfo(fieldName); + if (fieldInfo == null) { + return KNNVectorScriptDocValues.emptyValues(fieldName, vectorDataType); + } + + DocIdSetIterator values; + if (fieldInfo.hasVectorValues()) { + switch (fieldInfo.getVectorEncoding()) { + case FLOAT32: + values = reader.getFloatVectorValues(fieldName); + break; + case BYTE: + values = reader.getByteVectorValues(fieldName); + break; + default: + throw new IllegalStateException("Unsupported Lucene vector encoding: " + fieldInfo.getVectorEncoding()); + } + } else { + values = DocValues.getBinary(reader, fieldName); + } + return KNNVectorScriptDocValues.create(values, fieldName, vectorDataType); } catch (IOException e) { - throw new IllegalStateException("Cannot load doc values for knn vector field: " + fieldName, e); + throw new IllegalStateException("Cannot load values for knn vector field: " + fieldName, e); } } diff --git a/src/main/java/org/opensearch/knn/index/KNNVectorScriptDocValues.java b/src/main/java/org/opensearch/knn/index/KNNVectorScriptDocValues.java index 9f7d52205..c733c534e 100644 --- a/src/main/java/org/opensearch/knn/index/KNNVectorScriptDocValues.java +++ b/src/main/java/org/opensearch/knn/index/KNNVectorScriptDocValues.java @@ -5,18 +5,22 @@ package org.opensearch.knn.index; +import java.io.IOException; +import java.util.Objects; +import lombok.AccessLevel; import lombok.Getter; import lombok.RequiredArgsConstructor; import org.apache.lucene.index.BinaryDocValues; +import org.apache.lucene.index.ByteVectorValues; +import org.apache.lucene.index.FloatVectorValues; +import org.apache.lucene.search.DocIdSetIterator; import org.opensearch.ExceptionsHelper; import org.opensearch.index.fielddata.ScriptDocValues; -import java.io.IOException; - -@RequiredArgsConstructor -public final class KNNVectorScriptDocValues extends ScriptDocValues { +@RequiredArgsConstructor(access = AccessLevel.PRIVATE) +public abstract class KNNVectorScriptDocValues extends ScriptDocValues { - private final BinaryDocValues binaryDocValues; + private final DocIdSetIterator vectorValues; private final String fieldName; @Getter private final VectorDataType vectorDataType; @@ -24,11 +28,7 @@ public final class KNNVectorScriptDocValues extends ScriptDocValues { @Override public void setNextDocId(int docId) throws IOException { - if (binaryDocValues.advanceExact(docId)) { - docExists = true; - return; - } - docExists = false; + docExists = vectorValues.docID() == docId || vectorValues.advance(docId) == docId; } public float[] getValue() { @@ -43,12 +43,14 @@ public float[] getValue() { throw new IllegalStateException(errorMessage); } try { - return vectorDataType.getVectorFromDocValues(binaryDocValues.binaryValue()); + return doGetValue(); } catch (IOException e) { throw ExceptionsHelper.convertToOpenSearchException(e); } } + protected abstract float[] doGetValue() throws IOException; + @Override public int size() { return docExists ? 1 : 0; @@ -58,4 +60,89 @@ public int size() { public float[] get(int i) { throw new UnsupportedOperationException("knn vector does not support this operation"); } + + /** + * Creates a KNNVectorScriptDocValues object based on the provided parameters. + * + * @param values The DocIdSetIterator representing the vector values. + * @param fieldName The name of the field. + * @param vectorDataType The data type of the vector. + * @return A KNNVectorScriptDocValues object based on the type of the values. + * @throws IllegalArgumentException If the type of values is unsupported. + */ + public static KNNVectorScriptDocValues create(DocIdSetIterator values, String fieldName, VectorDataType vectorDataType) { + Objects.requireNonNull(values, "values must not be null"); + if (values instanceof ByteVectorValues) { + return new KNNByteVectorScriptDocValues((ByteVectorValues) values, fieldName, vectorDataType); + } else if (values instanceof FloatVectorValues) { + return new KNNFloatVectorScriptDocValues((FloatVectorValues) values, fieldName, vectorDataType); + } else if (values instanceof BinaryDocValues) { + return new KNNNativeVectorScriptDocValues((BinaryDocValues) values, fieldName, vectorDataType); + } else { + throw new IllegalArgumentException("Unsupported values type: " + values.getClass()); + } + } + + private static final class KNNByteVectorScriptDocValues extends KNNVectorScriptDocValues { + private final ByteVectorValues values; + + KNNByteVectorScriptDocValues(ByteVectorValues values, String field, VectorDataType type) { + super(values, field, type); + this.values = values; + } + + @Override + protected float[] doGetValue() throws IOException { + byte[] bytes = values.vectorValue(); + float[] value = new float[bytes.length]; + for (int i = 0; i < bytes.length; i++) { + value[i] = (float) bytes[i]; + } + return value; + } + } + + private static final class KNNFloatVectorScriptDocValues extends KNNVectorScriptDocValues { + private final FloatVectorValues values; + + KNNFloatVectorScriptDocValues(FloatVectorValues values, String field, VectorDataType type) { + super(values, field, type); + this.values = values; + } + + @Override + protected float[] doGetValue() throws IOException { + return values.vectorValue(); + } + } + + private static final class KNNNativeVectorScriptDocValues extends KNNVectorScriptDocValues { + private final BinaryDocValues values; + + KNNNativeVectorScriptDocValues(BinaryDocValues values, String field, VectorDataType type) { + super(values, field, type); + this.values = values; + } + + @Override + protected float[] doGetValue() throws IOException { + return getVectorDataType().getVectorFromDocValues(values.binaryValue()); + } + } + + /** + * Creates an empty KNNVectorScriptDocValues object based on the provided field name and vector data type. + * + * @param fieldName The name of the field. + * @param type The data type of the vector. + * @return An empty KNNVectorScriptDocValues object. + */ + public static KNNVectorScriptDocValues emptyValues(String fieldName, VectorDataType type) { + return new KNNVectorScriptDocValues(DocIdSetIterator.empty(), fieldName, type) { + @Override + protected float[] doGetValue() throws IOException { + throw new UnsupportedOperationException("empty values"); + } + }; + } } diff --git a/src/test/java/org/opensearch/knn/index/FaissIT.java b/src/test/java/org/opensearch/knn/index/FaissIT.java index 13d1bc64d..72ffd2b66 100644 --- a/src/test/java/org/opensearch/knn/index/FaissIT.java +++ b/src/test/java/org/opensearch/knn/index/FaissIT.java @@ -147,7 +147,7 @@ public void testEndToEnd_whenMethodIsHNSWFlat_thenSucceed() { List actualScores = parseSearchResponseScore(responseBody, fieldName); for (int j = 0; j < k; j++) { - float[] primitiveArray = Floats.toArray(Arrays.stream(knnResults.get(j).getVector()).collect(Collectors.toList())); + float[] primitiveArray = knnResults.get(j).getVector(); assertEquals( KNNEngine.FAISS.score(KNNScoringUtil.l2Squared(testData.queries[i], primitiveArray), spaceType), actualScores.get(j), @@ -257,7 +257,7 @@ public void testEndToEnd_whenMethodIsHNSWPQ_thenSucceed() { List actualScores = parseSearchResponseScore(responseBody, fieldName); for (int j = 0; j < k; j++) { - float[] primitiveArray = Floats.toArray(Arrays.stream(knnResults.get(j).getVector()).collect(Collectors.toList())); + float[] primitiveArray = knnResults.get(j).getVector(); assertEquals( KNNEngine.FAISS.score(KNNScoringUtil.l2Squared(testData.queries[i], primitiveArray), spaceType), actualScores.get(j), @@ -827,7 +827,7 @@ public void testEndToEnd_whenMethodIsHNSWPQAndHyperParametersNotSet_thenSucceed( List actualScores = parseSearchResponseScore(responseBody, fieldName); for (int j = 0; j < k; j++) { - float[] primitiveArray = Floats.toArray(Arrays.stream(knnResults.get(j).getVector()).collect(Collectors.toList())); + float[] primitiveArray = knnResults.get(j).getVector(); assertEquals( KNNEngine.FAISS.score(KNNScoringUtil.l2Squared(testData.queries[i], primitiveArray), spaceType), actualScores.get(j), diff --git a/src/test/java/org/opensearch/knn/index/KNNVectorScriptDocValuesTests.java b/src/test/java/org/opensearch/knn/index/KNNVectorScriptDocValuesTests.java index a0df3ce64..66e2893c0 100644 --- a/src/test/java/org/opensearch/knn/index/KNNVectorScriptDocValuesTests.java +++ b/src/test/java/org/opensearch/knn/index/KNNVectorScriptDocValuesTests.java @@ -5,6 +5,15 @@ package org.opensearch.knn.index; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.KnnByteVectorField; +import org.apache.lucene.document.KnnFloatVectorField; +import org.apache.lucene.index.BinaryDocValues; +import org.apache.lucene.index.ByteVectorValues; +import org.apache.lucene.index.DocValues; +import org.apache.lucene.index.FloatVectorValues; +import org.apache.lucene.index.LeafReader; +import org.apache.lucene.search.DocIdSetIterator; import org.opensearch.knn.KNNTestCase; import org.apache.lucene.tests.analysis.MockAnalyzer; import org.apache.lucene.document.BinaryDocValuesField; @@ -13,7 +22,6 @@ import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; -import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.store.Directory; import org.junit.Assert; import org.junit.Before; @@ -24,6 +32,7 @@ public class KNNVectorScriptDocValuesTests extends KNNTestCase { private static final String MOCK_INDEX_FIELD_NAME = "test-index-field-name"; private static final float[] SAMPLE_VECTOR_DATA = new float[] { 1.0f, 2.0f }; + private static final byte[] SAMPLE_BYTE_VECTOR_DATA = new byte[] { 1, 2 }; private KNNVectorScriptDocValues scriptDocValues; private Directory directory; private DirectoryReader reader; @@ -32,26 +41,39 @@ public class KNNVectorScriptDocValuesTests extends KNNTestCase { public void setUp() throws Exception { super.setUp(); directory = newDirectory(); - createKNNVectorDocument(directory); + Class valuesClass = randomFrom(BinaryDocValues.class, ByteVectorValues.class, FloatVectorValues.class); + createKNNVectorDocument(directory, valuesClass); reader = DirectoryReader.open(directory); - LeafReaderContext leafReaderContext = reader.getContext().leaves().get(0); - scriptDocValues = new KNNVectorScriptDocValues( - leafReaderContext.reader().getBinaryDocValues(MOCK_INDEX_FIELD_NAME), - MOCK_INDEX_FIELD_NAME, - VectorDataType.FLOAT - ); + LeafReader leafReader = reader.getContext().leaves().get(0).reader(); + DocIdSetIterator vectorValues; + if (BinaryDocValues.class.equals(valuesClass)) { + vectorValues = DocValues.getBinary(leafReader, MOCK_INDEX_FIELD_NAME); + } else if (ByteVectorValues.class.equals(valuesClass)) { + vectorValues = leafReader.getByteVectorValues(MOCK_INDEX_FIELD_NAME); + } else { + vectorValues = leafReader.getFloatVectorValues(MOCK_INDEX_FIELD_NAME); + } + + scriptDocValues = KNNVectorScriptDocValues.create(vectorValues, MOCK_INDEX_FIELD_NAME, VectorDataType.FLOAT); } - private void createKNNVectorDocument(Directory directory) throws IOException { + private void createKNNVectorDocument(Directory directory, Class valuesClass) throws IOException { IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random())); IndexWriter writer = new IndexWriter(directory, conf); Document knnDocument = new Document(); - knnDocument.add( - new BinaryDocValuesField( + Field field; + if (BinaryDocValues.class.equals(valuesClass)) { + field = new BinaryDocValuesField( MOCK_INDEX_FIELD_NAME, new VectorField(MOCK_INDEX_FIELD_NAME, SAMPLE_VECTOR_DATA, new FieldType()).binaryValue() - ) - ); + ); + } else if (ByteVectorValues.class.equals(valuesClass)) { + field = new KnnByteVectorField(MOCK_INDEX_FIELD_NAME, SAMPLE_BYTE_VECTOR_DATA); + } else { + field = new KnnFloatVectorField(MOCK_INDEX_FIELD_NAME, SAMPLE_VECTOR_DATA); + } + + knnDocument.add(field); writer.addDocument(knnDocument); writer.commit(); writer.close(); @@ -83,4 +105,18 @@ public void testSize() throws IOException { public void testGet() throws IOException { expectThrows(UnsupportedOperationException.class, () -> scriptDocValues.get(0)); } + + public void testUnsupportedValues() throws IOException { + expectThrows( + IllegalArgumentException.class, + () -> KNNVectorScriptDocValues.create(DocValues.emptyNumeric(), MOCK_INDEX_FIELD_NAME, VectorDataType.FLOAT) + ); + } + + public void testEmptyValues() throws IOException { + KNNVectorScriptDocValues values = KNNVectorScriptDocValues.emptyValues(MOCK_INDEX_FIELD_NAME, VectorDataType.FLOAT); + assertEquals(0, values.size()); + scriptDocValues.setNextDocId(0); + assertEquals(0, values.size()); + } } diff --git a/src/test/java/org/opensearch/knn/index/LuceneEngineIT.java b/src/test/java/org/opensearch/knn/index/LuceneEngineIT.java index 083b5b370..c53fa4456 100644 --- a/src/test/java/org/opensearch/knn/index/LuceneEngineIT.java +++ b/src/test/java/org/opensearch/knn/index/LuceneEngineIT.java @@ -7,7 +7,6 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import com.google.common.primitives.Floats; import org.apache.http.util.EntityUtils; import lombok.SneakyThrows; import org.apache.commons.lang.math.RandomUtils; @@ -307,14 +306,14 @@ public void testIndexReopening() throws Exception { final float[] searchVector = TEST_QUERY_VECTORS[0]; final int k = 1 + RandomUtils.nextInt(TEST_INDEX_VECTORS.length); - final List knnResultsBeforeIndexClosure = queryResults(searchVector, k); + final List knnResultsBeforeIndexClosure = queryResults(searchVector, k); closeIndex(INDEX_NAME); openIndex(INDEX_NAME); ensureGreen(INDEX_NAME); - final List knnResultsAfterIndexClosure = queryResults(searchVector, k); + final List knnResultsAfterIndexClosure = queryResults(searchVector, k); assertArrayEquals(knnResultsBeforeIndexClosure.toArray(), knnResultsAfterIndexClosure.toArray()); } @@ -365,7 +364,7 @@ private void validateQueries(SpaceType spaceType, String fieldName) throws IOExc List actualScores = parseSearchResponseScore(responseBody, fieldName); for (int j = 0; j < k; j++) { - float[] primitiveArray = Floats.toArray(Arrays.stream(knnResults.get(j).getVector()).collect(Collectors.toList())); + float[] primitiveArray = knnResults.get(j).getVector(); float distance = TestUtils.computeDistFromSpaceType(spaceType, primitiveArray, queryVector); float rawScore = VECTOR_SIMILARITY_TO_SCORE.get(spaceType.getVectorSimilarityFunction()).apply(distance); assertEquals(KNNEngine.LUCENE.score(rawScore, spaceType), actualScores.get(j), 0.0001); @@ -373,7 +372,7 @@ private void validateQueries(SpaceType spaceType, String fieldName) throws IOExc } } - private List queryResults(final float[] searchVector, final int k) throws Exception { + private List queryResults(final float[] searchVector, final int k) throws Exception { final String responseBody = EntityUtils.toString( searchKNNIndex(INDEX_NAME, new KNNQueryBuilder(FIELD_NAME, searchVector, k), k).getEntity() ); diff --git a/src/test/java/org/opensearch/knn/index/NmslibIT.java b/src/test/java/org/opensearch/knn/index/NmslibIT.java index cb15a0eb9..b76a26e69 100644 --- a/src/test/java/org/opensearch/knn/index/NmslibIT.java +++ b/src/test/java/org/opensearch/knn/index/NmslibIT.java @@ -30,11 +30,9 @@ import java.io.IOException; import java.net.URL; -import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.TreeMap; -import java.util.stream.Collectors; import static org.hamcrest.Matchers.containsString; @@ -115,7 +113,7 @@ public void testEndToEnd() throws IOException, InterruptedException { List actualScores = parseSearchResponseScore(responseBody, fieldName); for (int j = 0; j < k; j++) { - float[] primitiveArray = Floats.toArray(Arrays.stream(knnResults.get(j).getVector()).collect(Collectors.toList())); + float[] primitiveArray = knnResults.get(j).getVector(); assertEquals( KNNEngine.NMSLIB.score(KNNScoringUtil.l1Norm(testData.queries[i], primitiveArray), spaceType), actualScores.get(j), diff --git a/src/test/java/org/opensearch/knn/index/OpenSearchIT.java b/src/test/java/org/opensearch/knn/index/OpenSearchIT.java index 24dcf7b1c..f8948db26 100644 --- a/src/test/java/org/opensearch/knn/index/OpenSearchIT.java +++ b/src/test/java/org/opensearch/knn/index/OpenSearchIT.java @@ -39,7 +39,6 @@ import java.util.List; import java.util.Map; import java.util.TreeMap; -import java.util.stream.Collectors; import static org.hamcrest.Matchers.containsString; @@ -143,7 +142,7 @@ public void testEndToEnd() throws IOException, InterruptedException { List actualScores = parseSearchResponseScore(responseBody, fieldName1); for (int j = 0; j < k; j++) { - float[] primitiveArray = Floats.toArray(Arrays.stream(knnResults.get(j).getVector()).collect(Collectors.toList())); + float[] primitiveArray = knnResults.get(j).getVector(); assertEquals( knnEngine1.score(1 - KNNScoringUtil.cosinesimil(testData.queries[i], primitiveArray), spaceType1), actualScores.get(j), @@ -159,7 +158,7 @@ public void testEndToEnd() throws IOException, InterruptedException { actualScores = parseSearchResponseScore(responseBody, fieldName2); for (int j = 0; j < k; j++) { - float[] primitiveArray = Floats.toArray(Arrays.stream(knnResults.get(j).getVector()).collect(Collectors.toList())); + float[] primitiveArray = knnResults.get(j).getVector(); assertEquals( knnEngine2.score(KNNScoringUtil.l2Squared(testData.queries[i], primitiveArray), spaceType2), actualScores.get(j), diff --git a/src/test/java/org/opensearch/knn/index/VectorDataTypeTests.java b/src/test/java/org/opensearch/knn/index/VectorDataTypeTests.java index 4423c85d8..19270717d 100644 --- a/src/test/java/org/opensearch/knn/index/VectorDataTypeTests.java +++ b/src/test/java/org/opensearch/knn/index/VectorDataTypeTests.java @@ -57,7 +57,7 @@ private KNNVectorScriptDocValues getKNNFloatVectorScriptDocValues() { createKNNFloatVectorDocument(directory); reader = DirectoryReader.open(directory); LeafReaderContext leafReaderContext = reader.getContext().leaves().get(0); - return new KNNVectorScriptDocValues( + return KNNVectorScriptDocValues.create( leafReaderContext.reader().getBinaryDocValues(VectorDataTypeTests.MOCK_FLOAT_INDEX_FIELD_NAME), VectorDataTypeTests.MOCK_FLOAT_INDEX_FIELD_NAME, VectorDataType.FLOAT @@ -70,7 +70,7 @@ private KNNVectorScriptDocValues getKNNByteVectorScriptDocValues() { createKNNByteVectorDocument(directory); reader = DirectoryReader.open(directory); LeafReaderContext leafReaderContext = reader.getContext().leaves().get(0); - return new KNNVectorScriptDocValues( + return KNNVectorScriptDocValues.create( leafReaderContext.reader().getBinaryDocValues(VectorDataTypeTests.MOCK_BYTE_INDEX_FIELD_NAME), VectorDataTypeTests.MOCK_BYTE_INDEX_FIELD_NAME, VectorDataType.BYTE diff --git a/src/test/java/org/opensearch/knn/plugin/script/KNNScoringUtilTests.java b/src/test/java/org/opensearch/knn/plugin/script/KNNScoringUtilTests.java index 8c43a4acf..22110accd 100644 --- a/src/test/java/org/opensearch/knn/plugin/script/KNNScoringUtilTests.java +++ b/src/test/java/org/opensearch/knn/plugin/script/KNNScoringUtilTests.java @@ -280,7 +280,7 @@ public KNNVectorScriptDocValues getScriptDocValues(String fieldName) throws IOEx if (scriptDocValues == null) { reader = DirectoryReader.open(directory); LeafReaderContext leafReaderContext = reader.getContext().leaves().get(0); - scriptDocValues = new KNNVectorScriptDocValues( + scriptDocValues = KNNVectorScriptDocValues.create( leafReaderContext.reader().getBinaryDocValues(fieldName), fieldName, VectorDataType.FLOAT diff --git a/src/test/java/org/opensearch/knn/plugin/script/KNNScriptScoringIT.java b/src/test/java/org/opensearch/knn/plugin/script/KNNScriptScoringIT.java index 214ecd158..58cdb3112 100644 --- a/src/test/java/org/opensearch/knn/plugin/script/KNNScriptScoringIT.java +++ b/src/test/java/org/opensearch/knn/plugin/script/KNNScriptScoringIT.java @@ -5,15 +5,18 @@ package org.opensearch.knn.plugin.script; -import org.opensearch.core.xcontent.MediaTypeRegistry; +import java.util.function.BiFunction; +import java.util.function.Function; import org.opensearch.knn.KNNRestTestCase; import org.opensearch.knn.KNNResult; +import org.opensearch.knn.common.KNNConstants; import org.opensearch.knn.index.SpaceType; import org.apache.http.util.EntityUtils; import org.opensearch.client.Request; import org.opensearch.client.Response; import org.opensearch.client.ResponseException; import org.opensearch.common.settings.Settings; +import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; @@ -22,6 +25,9 @@ import org.opensearch.index.query.QueryBuilder; import org.opensearch.index.query.functionscore.ScriptScoreQueryBuilder; import org.opensearch.core.rest.RestStatus; +import org.opensearch.knn.index.VectorDataType; +import org.opensearch.knn.index.mapper.KNNVectorFieldMapper; +import org.opensearch.knn.index.util.KNNEngine; import org.opensearch.script.Script; import java.util.ArrayList; @@ -38,214 +44,19 @@ public class KNNScriptScoringIT extends KNNRestTestCase { public void testKNNL2ScriptScore() throws Exception { - /* - * Create knn index and populate data - */ - createKnnIndex(INDEX_NAME, createKnnIndexMapping(FIELD_NAME, 2)); - Float[] f1 = { 6.0f, 6.0f }; - addKnnDoc(INDEX_NAME, "1", FIELD_NAME, f1); - - Float[] f2 = { 2.0f, 2.0f }; - addKnnDoc(INDEX_NAME, "2", FIELD_NAME, f2); - - Float[] f3 = { 4.0f, 4.0f }; - addKnnDoc(INDEX_NAME, "3", FIELD_NAME, f3); - - Float[] f4 = { 3.0f, 3.0f }; - addKnnDoc(INDEX_NAME, "4", FIELD_NAME, f4); - - /** - * Construct Search Request - */ - QueryBuilder qb = new MatchAllQueryBuilder(); - Map params = new HashMap<>(); - /* - * params": { - * "field": "my_dense_vector", - * "vector": [2.0, 2.0] - * } - */ - float[] queryVector = { 1.0f, 1.0f }; - params.put("field", FIELD_NAME); - params.put("query_value", queryVector); - params.put("space_type", SpaceType.L2.getValue()); - Request request = constructKNNScriptQueryRequest(INDEX_NAME, qb, params); - Response response = client().performRequest(request); - assertEquals(request.getEndpoint() + ": failed", RestStatus.OK, RestStatus.fromCode(response.getStatusLine().getStatusCode())); - - List results = parseSearchResponse(EntityUtils.toString(response.getEntity()), FIELD_NAME); - List expectedDocids = Arrays.asList("2", "4", "3", "1"); - - List actualDocids = new ArrayList<>(); - for (KNNResult result : results) { - actualDocids.add(result.getDocId()); - } - - assertEquals(4, results.size()); - - // assert document order - assertEquals("2", results.get(0).getDocId()); - assertEquals("4", results.get(1).getDocId()); - assertEquals("3", results.get(2).getDocId()); - assertEquals("1", results.get(3).getDocId()); + testKNNScriptScore(SpaceType.L2); } public void testKNNL1ScriptScore() throws Exception { - /* - * Create knn index and populate data - */ - createKnnIndex(INDEX_NAME, createKnnIndexMapping(FIELD_NAME, 2)); - Float[] f1 = { 6.0f, 6.0f }; - addKnnDoc(INDEX_NAME, "1", FIELD_NAME, f1); - - Float[] f2 = { 4.0f, 1.0f }; - addKnnDoc(INDEX_NAME, "2", FIELD_NAME, f2); - - Float[] f3 = { 3.0f, 3.0f }; - addKnnDoc(INDEX_NAME, "3", FIELD_NAME, f3); - - Float[] f4 = { 5.0f, 5.0f }; - addKnnDoc(INDEX_NAME, "4", FIELD_NAME, f4); - - /** - * Construct Search Request - */ - QueryBuilder qb = new MatchAllQueryBuilder(); - Map params = new HashMap<>(); - /* - * params": { - * "field": "my_dense_vector", - * "vector": [1.0, 1.0] - * } - */ - float[] queryVector = { 1.0f, 1.0f }; - params.put("field", FIELD_NAME); - params.put("query_value", queryVector); - params.put("space_type", SpaceType.L1); - Request request = constructKNNScriptQueryRequest(INDEX_NAME, qb, params); - Response response = client().performRequest(request); - assertEquals(request.getEndpoint() + ": failed", RestStatus.OK, RestStatus.fromCode(response.getStatusLine().getStatusCode())); - - List results = parseSearchResponse(EntityUtils.toString(response.getEntity()), FIELD_NAME); - List expectedDocids = Arrays.asList("2", "4", "3", "1"); - - List actualDocids = new ArrayList<>(); - for (KNNResult result : results) { - actualDocids.add(result.getDocId()); - } - - assertEquals(4, results.size()); - - // assert document order - assertEquals("2", results.get(0).getDocId()); - assertEquals("3", results.get(1).getDocId()); - assertEquals("4", results.get(2).getDocId()); - assertEquals("1", results.get(3).getDocId()); + testKNNScriptScore(SpaceType.L1); } public void testKNNLInfScriptScore() throws Exception { - /* - * Create knn index and populate data - */ - createKnnIndex(INDEX_NAME, createKnnIndexMapping(FIELD_NAME, 2)); - Float[] f1 = { 6.0f, 6.0f }; - addKnnDoc(INDEX_NAME, "1", FIELD_NAME, f1); - - Float[] f2 = { 4.0f, 1.0f }; - addKnnDoc(INDEX_NAME, "2", FIELD_NAME, f2); - - Float[] f3 = { 3.0f, 3.0f }; - addKnnDoc(INDEX_NAME, "3", FIELD_NAME, f3); - - Float[] f4 = { 5.0f, 5.0f }; - addKnnDoc(INDEX_NAME, "4", FIELD_NAME, f4); - - /** - * Construct Search Request - */ - QueryBuilder qb = new MatchAllQueryBuilder(); - Map params = new HashMap<>(); - /* - * params": { - * "field": "my_dense_vector", - * "vector": [1.0, 1.0] - * } - */ - float[] queryVector = { 1.0f, 1.0f }; - params.put("field", FIELD_NAME); - params.put("query_value", queryVector); - params.put("space_type", SpaceType.LINF.getValue()); - Request request = constructKNNScriptQueryRequest(INDEX_NAME, qb, params); - Response response = client().performRequest(request); - assertEquals(request.getEndpoint() + ": failed", RestStatus.OK, RestStatus.fromCode(response.getStatusLine().getStatusCode())); - - List results = parseSearchResponse(EntityUtils.toString(response.getEntity()), FIELD_NAME); - List expectedDocids = Arrays.asList("3", "2", "4", "1"); - - List actualDocids = new ArrayList<>(); - for (KNNResult result : results) { - actualDocids.add(result.getDocId()); - } - - assertEquals(4, results.size()); - - // assert document order - assertEquals("3", results.get(0).getDocId()); - assertEquals("2", results.get(1).getDocId()); - assertEquals("4", results.get(2).getDocId()); - assertEquals("1", results.get(3).getDocId()); + testKNNScriptScore(SpaceType.LINF); } public void testKNNCosineScriptScore() throws Exception { - /* - * Create knn index and populate data - */ - createKnnIndex(INDEX_NAME, createKnnIndexMapping(FIELD_NAME, 2)); - Float[] f1 = { 1.0f, -1.0f }; - addKnnDoc(INDEX_NAME, "0", FIELD_NAME, f1); - - Float[] f2 = { 1.0f, 0.0f }; - addKnnDoc(INDEX_NAME, "1", FIELD_NAME, f2); - - Float[] f3 = { 1.0f, 1.0f }; - addKnnDoc(INDEX_NAME, "2", FIELD_NAME, f3); - - /** - * Construct Search Request - */ - QueryBuilder qb = new MatchAllQueryBuilder(); - Map params = new HashMap<>(); - /* - * params": { - * "field": "my_dense_vector", - * "query_value": [2.0, 2.0], - * "space_type": "L2" - * } - * - * - */ - float[] queryVector = { 2.0f, -2.0f }; - params.put("field", FIELD_NAME); - params.put("query_value", queryVector); - params.put("space_type", SpaceType.COSINESIMIL.getValue()); - Request request = constructKNNScriptQueryRequest(INDEX_NAME, qb, params); - Response response = client().performRequest(request); - assertEquals(request.getEndpoint() + ": failed", RestStatus.OK, RestStatus.fromCode(response.getStatusLine().getStatusCode())); - - List results = parseSearchResponse(EntityUtils.toString(response.getEntity()), FIELD_NAME); - List expectedDocids = Arrays.asList("0", "1", "2"); - - List actualDocids = new ArrayList<>(); - for (KNNResult result : results) { - actualDocids.add(result.getDocId()); - } - - assertEquals(3, results.size()); - - // assert document order - assertEquals("0", results.get(0).getDocId()); - assertEquals("1", results.get(1).getDocId()); - assertEquals("2", results.get(2).getDocId()); + testKNNScriptScore(SpaceType.COSINESIMIL); } public void testKNNInvalidSourceScript() throws Exception { @@ -395,10 +206,7 @@ public void testKNNScoreforNonVectorDocument() throws Exception { List hits = (List) ((Map) createParser(XContentType.JSON.xContent(), responseBody).map() .get("hits")).get("hits"); - List docIds = hits.stream().map(hit -> { - String id = ((String) ((Map) hit).get("_id")); - return id; - }).collect(Collectors.toList()); + List docIds = hits.stream().map(hit -> ((String) ((Map) hit).get("_id"))).collect(Collectors.toList()); // assert document order assertEquals("1", docIds.get(0)); assertEquals("0", docIds.get(1)); @@ -624,57 +432,7 @@ public void testHammingScriptScore_Base64() throws Exception { } public void testKNNInnerProdScriptScore() throws Exception { - /* - * Create knn index and populate data - */ - createKnnIndex(INDEX_NAME, createKnnIndexMapping(FIELD_NAME, 2)); - Float[] f1 = { -2.0f, -2.0f }; - addKnnDoc(INDEX_NAME, "1", FIELD_NAME, f1); - - Float[] f2 = { 1.0f, 1.0f }; - addKnnDoc(INDEX_NAME, "2", FIELD_NAME, f2); - - Float[] f3 = { 2.0f, 2.0f }; - addKnnDoc(INDEX_NAME, "3", FIELD_NAME, f3); - - Float[] f4 = { 2.0f, -2.0f }; - addKnnDoc(INDEX_NAME, "4", FIELD_NAME, f4); - - /** - * Construct Search Request - */ - QueryBuilder qb = new MatchAllQueryBuilder(); - Map params = new HashMap<>(); - /* - * params": { - * "field": "my_dense_vector", - * "query_value": [1.0, 1.0], - * "space_type": "innerproduct", - * } - */ - float[] queryVector = { 1.0f, 1.0f }; - params.put("field", FIELD_NAME); - params.put("query_value", queryVector); - params.put("space_type", SpaceType.INNER_PRODUCT.getValue()); - Request request = constructKNNScriptQueryRequest(INDEX_NAME, qb, params); - Response response = client().performRequest(request); - assertEquals(request.getEndpoint() + ": failed", RestStatus.OK, RestStatus.fromCode(response.getStatusLine().getStatusCode())); - - List results = parseSearchResponse(EntityUtils.toString(response.getEntity()), FIELD_NAME); - List expectedDocids = Arrays.asList("3", "2", "4", "1"); - - List actualDocids = new ArrayList<>(); - for (KNNResult result : results) { - actualDocids.add(result.getDocId()); - } - - assertEquals(4, results.size()); - - // assert document order - assertEquals("3", results.get(0).getDocId()); - assertEquals("2", results.get(1).getDocId()); - assertEquals("4", results.get(2).getDocId()); - assertEquals("1", results.get(3).getDocId()); + testKNNScriptScore(SpaceType.INNER_PRODUCT); } public void testKNNScriptScoreWithRequestCacheEnabled() throws Exception { @@ -782,4 +540,121 @@ public void testKNNScriptScoreWithRequestCacheEnabled() throws Exception { // assert that the request cache was hit at second request assertEquals(1, secondQueryCacheMap.get("hit_count")); } + + private List createMappers(int dimensions) throws Exception { + return List.of( + createKnnIndexMapping(FIELD_NAME, dimensions), + createKnnIndexMapping( + FIELD_NAME, + dimensions, + KNNConstants.METHOD_HNSW, + KNNEngine.LUCENE.getName(), + SpaceType.DEFAULT.getValue(), + true + ), + createKnnIndexMapping( + FIELD_NAME, + dimensions, + KNNConstants.METHOD_HNSW, + KNNEngine.LUCENE.getName(), + SpaceType.DEFAULT.getValue(), + false + ) + ); + } + + private float[] randomVector(int dimensions) { + final float[] vector = new float[dimensions]; + for (int i = 0; i < dimensions; i++) { + vector[i] = randomFloat(); + } + return vector; + } + + private Map createDataset(Function scoreFunction, int dimensions, int numDocs) { + final Map dataset = new HashMap<>(numDocs); + for (int i = 0; i < numDocs; i++) { + final float[] vector = randomVector(dimensions); + final float score = scoreFunction.apply(vector); + dataset.put(Integer.toString(i), new KNNResult(Integer.toString(i), vector, score)); + } + return dataset; + } + + private BiFunction getScoreFunction(SpaceType spaceType, float[] queryVector) { + KNNVectorFieldMapper.KNNVectorFieldType knnVectorFieldType = new KNNVectorFieldMapper.KNNVectorFieldType( + FIELD_NAME, + Collections.emptyMap(), + queryVector.length, + VectorDataType.FLOAT, + null + ); + List target = new ArrayList<>(queryVector.length); + for (float f : queryVector) { + target.add(f); + } + KNNScoringSpace knnScoringSpace = KNNScoringSpaceFactory.create(spaceType.getValue(), target, knnVectorFieldType); + switch (spaceType) { + case L1: + return ((KNNScoringSpace.L1) knnScoringSpace).scoringMethod; + case L2: + return ((KNNScoringSpace.L2) knnScoringSpace).scoringMethod; + case LINF: + return ((KNNScoringSpace.LInf) knnScoringSpace).scoringMethod; + case COSINESIMIL: + return ((KNNScoringSpace.CosineSimilarity) knnScoringSpace).scoringMethod; + case INNER_PRODUCT: + return ((KNNScoringSpace.InnerProd) knnScoringSpace).scoringMethod; + default: + throw new IllegalArgumentException(); + } + } + + private void testKNNScriptScore(SpaceType spaceType) throws Exception { + final int dims = randomIntBetween(2, 10); + final float[] queryVector = randomVector(dims); + final BiFunction scoreFunction = getScoreFunction(spaceType, queryVector); + for (String mapper : createMappers(dims)) { + createIndexAndAssertScriptScore(mapper, spaceType, scoreFunction, dims, queryVector); + } + } + + private void createIndexAndAssertScriptScore( + String mapper, + SpaceType spaceType, + BiFunction scoreFunction, + int dimensions, + float[] queryVector + ) throws Exception { + /* + * Create knn index and populate data + */ + createKnnIndex(INDEX_NAME, mapper); + Map dataset = createDataset(v -> scoreFunction.apply(queryVector, v), dimensions, randomIntBetween(4, 10)); + for (Map.Entry entry : dataset.entrySet()) { + addKnnDoc(INDEX_NAME, entry.getKey(), FIELD_NAME, entry.getValue().getVector()); + } + + /** + * Construct Search Request + */ + QueryBuilder qb = new MatchAllQueryBuilder(); + Map params = new HashMap<>(); + /* + * params": { + * "field": FIELD_NAME, + * "vector": queryVector + * } + */ + params.put("field", FIELD_NAME); + params.put("query_value", queryVector); + params.put("space_type", spaceType.getValue()); + Request request = constructKNNScriptQueryRequest(INDEX_NAME, qb, params); + Response response = client().performRequest(request); + assertEquals(request.getEndpoint() + ": failed", RestStatus.OK, RestStatus.fromCode(response.getStatusLine().getStatusCode())); + + List results = parseSearchResponse(EntityUtils.toString(response.getEntity()), FIELD_NAME); + assertTrue(results.stream().allMatch(r -> dataset.get(r.getDocId()).equals(r))); + deleteKNNIndex(INDEX_NAME); + } } diff --git a/src/test/java/org/opensearch/knn/plugin/script/PainlessScriptIT.java b/src/test/java/org/opensearch/knn/plugin/script/PainlessScriptIT.java index 15e3732b2..47e914d04 100644 --- a/src/test/java/org/opensearch/knn/plugin/script/PainlessScriptIT.java +++ b/src/test/java/org/opensearch/knn/plugin/script/PainlessScriptIT.java @@ -53,6 +53,10 @@ protected String createMapping(List properties) throws IOExcept builder.field("dimension", property.getDimension()); } + if (property.getDocValues() != null) { + builder.field("doc_values", property.getDocValues()); + } + if (property.getKnnMethodContext() != null) { builder.startObject(KNNConstants.KNN_METHOD); property.getKnnMethodContext().toXContent(builder, ToXContent.EMPTY_PARAMS); @@ -554,12 +558,14 @@ public void testScriptedMetricIsSupported() throws Exception { public void testL2ScriptingWithLuceneBackedIndex() throws Exception { List properties = new ArrayList<>(); KNNMethodContext knnMethodContext = new KNNMethodContext( - KNNEngine.NMSLIB, + KNNEngine.LUCENE, SpaceType.DEFAULT, new MethodComponentContext(METHOD_HNSW, Collections.emptyMap()) ); properties.add( - new MappingProperty(FIELD_NAME, KNNVectorFieldMapper.CONTENT_TYPE).dimension("2").knnMethodContext(knnMethodContext) + new MappingProperty(FIELD_NAME, KNNVectorFieldMapper.CONTENT_TYPE).dimension("2") + .knnMethodContext(knnMethodContext) + .docValues(randomBoolean()) ); String source = String.format("1/(1 + l2Squared([1.0f, 1.0f], doc['%s']))", FIELD_NAME); @@ -585,6 +591,7 @@ static class MappingProperty { private String dimension; private KNNMethodContext knnMethodContext; + private Boolean docValues; MappingProperty(String name, String type) { this.name = name; @@ -601,6 +608,11 @@ MappingProperty knnMethodContext(KNNMethodContext knnMethodContext) { return this; } + MappingProperty docValues(boolean docValues) { + this.docValues = docValues; + return this; + } + KNNMethodContext getKnnMethodContext() { return knnMethodContext; } @@ -616,5 +628,9 @@ String getName() { String getType() { return type; } + + Boolean getDocValues() { + return docValues; + } } } diff --git a/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java b/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java index 6897091af..60c401648 100644 --- a/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java +++ b/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java @@ -243,10 +243,16 @@ protected List parseSearchResponse(String responseBody, String fieldN @SuppressWarnings("unchecked") List knnSearchResponses = hits.stream().map(hit -> { @SuppressWarnings("unchecked") - Float[] vector = Arrays.stream( - ((ArrayList) ((Map) ((Map) hit).get("_source")).get(fieldName)).toArray() - ).map(Object::toString).map(Float::valueOf).toArray(Float[]::new); - return new KNNResult((String) ((Map) hit).get("_id"), vector); + final float[] vector = Floats.toArray( + Arrays.stream( + ((ArrayList) ((Map) ((Map) hit).get("_source")).get(fieldName)).toArray() + ).map(Object::toString).map(Float::valueOf).collect(Collectors.toList()) + ); + return new KNNResult( + (String) ((Map) hit).get("_id"), + vector, + ((Double) ((Map) hit).get("_score")).floatValue() + ); }).collect(Collectors.toList()); return knnSearchResponses; @@ -323,20 +329,7 @@ protected String createKnnIndexMapping(String fieldName, Integer dimensions) thr * Utility to create a Knn Index Mapping with specific algorithm and engine */ protected String createKnnIndexMapping(String fieldName, Integer dimensions, String algoName, String knnEngine) throws IOException { - return XContentFactory.jsonBuilder() - .startObject() - .startObject("properties") - .startObject(fieldName) - .field("type", "knn_vector") - .field("dimension", dimensions.toString()) - .startObject("method") - .field("name", algoName) - .field("engine", knnEngine) - .endObject() - .endObject() - .endObject() - .endObject() - .toString(); + return this.createKnnIndexMapping(fieldName, dimensions, algoName, knnEngine, SpaceType.DEFAULT.getValue()); } /** @@ -344,12 +337,27 @@ protected String createKnnIndexMapping(String fieldName, Integer dimensions, Str */ protected String createKnnIndexMapping(String fieldName, Integer dimensions, String algoName, String knnEngine, String spaceType) throws IOException { + return this.createKnnIndexMapping(fieldName, dimensions, algoName, knnEngine, spaceType, true); + } + + /** + * Utility to create a Knn Index Mapping with specific algorithm, engine, spaceType and docValues + */ + protected String createKnnIndexMapping( + String fieldName, + Integer dimensions, + String algoName, + String knnEngine, + String spaceType, + boolean docValues + ) throws IOException { return XContentFactory.jsonBuilder() .startObject() .startObject("properties") .startObject(fieldName) .field(KNNConstants.TYPE, KNNConstants.TYPE_KNN_VECTOR) .field(KNNConstants.DIMENSION, dimensions.toString()) + .field("doc_values", docValues) .startObject(KNNConstants.KNN_METHOD) .field(KNNConstants.NAME, algoName) .field(KNNConstants.METHOD_PARAMETER_SPACE_TYPE, spaceType) @@ -474,7 +482,7 @@ protected void forceMergeKnnIndex(String index, int maxSegments) throws Exceptio /** * Add a single KNN Doc to an index */ - protected void addKnnDoc(String index, String docId, String fieldName, Object[] vector) throws IOException { + protected void addKnnDoc(String index, String docId, String fieldName, T vector) throws IOException { Request request = new Request("POST", "/" + index + "/_doc/" + docId + "?refresh=true"); XContentBuilder builder = XContentFactory.jsonBuilder().startObject().field(fieldName, vector).endObject(); @@ -1014,8 +1022,7 @@ public float[][] getIndexVectorsFromIndex(String testIndex, String testField, in int i = 0; for (KNNResult result : results) { - float[] primitiveArray = Floats.toArray(Arrays.stream(result.getVector()).collect(Collectors.toList())); - vectors[i++] = primitiveArray; + vectors[i++] = result.getVector(); } return vectors; diff --git a/src/testFixtures/java/org/opensearch/knn/KNNResult.java b/src/testFixtures/java/org/opensearch/knn/KNNResult.java index 803c2ae72..ee2ba39f7 100644 --- a/src/testFixtures/java/org/opensearch/knn/KNNResult.java +++ b/src/testFixtures/java/org/opensearch/knn/KNNResult.java @@ -5,20 +5,41 @@ package org.opensearch.knn; +import java.util.Arrays; +import java.util.Objects; + public class KNNResult { + private final static float delta = 1e-3f; + private String docId; - private Float[] vector; + private float[] vector; + private Float score; - public KNNResult(String docId, Float[] vector) { + public KNNResult(String docId, float[] vector, Float score) { this.docId = docId; this.vector = vector; + this.score = score; } public String getDocId() { return docId; } - public Float[] getVector() { + public float[] getVector() { return vector; } + + public Float getScore() { + return score; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + KNNResult knnResult = (KNNResult) o; + return Objects.equals(docId, knnResult.docId) + && Arrays.equals(vector, knnResult.vector) + && (Float.compare(score, knnResult.score) == 0 || Math.abs(score - knnResult.score) <= delta); + } } From 172dc846c96fcaa2c7b0fc55df719526f9f60379 Mon Sep 17 00:00:00 2001 From: Navneet Verma Date: Wed, 10 Apr 2024 15:11:16 -0700 Subject: [PATCH 234/416] Implemented the Streaming Feature to stream vectors from Java to JNI layer to enable creation of larger segments for vector indices (#1604) (#1608) Changes include: 1. Add the interface for streaming the vectors from java to jni layer with initial capacity (#1586) 2. Integrating storeVectors interfaces with createIndex and createIndexTemplate functions. (#1588) 3. Update KNN80BinaryDocValues reader count live docs and use live docs as initial capacity to initialize vector address(#1595) 4. Move free vectorAddress from Java to JNI layer to reduce the memory footprint for Nmslib (#1602) Signed-off-by: Navneet Verma --- CHANGELOG.md | 1 + jni/CMakeLists.txt | 3 +- jni/include/commons.h | 37 +++ jni/include/faiss_wrapper.h | 4 +- jni/include/jni_util.h | 4 + jni/include/nmslib_wrapper.h | 2 +- .../org_opensearch_knn_jni_FaissService.h | 24 +- .../org_opensearch_knn_jni_JNICommons.h | 40 ++++ .../org_opensearch_knn_jni_NmslibService.h | 4 +- jni/src/commons.cpp | 41 ++++ jni/src/faiss_wrapper.cpp | 57 +++-- jni/src/jni_util.cpp | 11 +- jni/src/nmslib_wrapper.cpp | 45 ++-- .../org_opensearch_knn_jni_FaissService.cpp | 39 +--- jni/src/org_opensearch_knn_jni_JNICommons.cpp | 60 +++++ .../org_opensearch_knn_jni_NmslibService.cpp | 4 +- jni/tests/commons_test.cpp | 73 ++++++ jni/tests/faiss_wrapper_test.cpp | 39 ++-- jni/tests/nmslib_wrapper_test.cpp | 15 +- jni/tests/test_util.cpp | 8 + jni/tests/test_util.h | 2 + .../knn/TransferVectorsBenchmarks.java | 24 +- .../opensearch/knn/common/KNNConstants.java | 2 + .../org/opensearch/knn/index/KNNSettings.java | 22 +- .../KNN80Codec/KNN80BinaryDocValues.java | 18 +- .../KNN80Codec/KNN80DocValuesConsumer.java | 48 ++-- .../KNN80Codec/KNN80DocValuesReader.java | 45 +++- .../knn/index/codec/util/KNNCodecUtil.java | 75 ++++-- .../index/memory/NativeMemoryAllocation.java | 3 +- .../org/opensearch/knn/jni/FaissService.java | 42 ++-- .../org/opensearch/knn/jni/JNICommons.java | 62 +++++ .../org/opensearch/knn/jni/JNIService.java | 65 +++--- .../org/opensearch/knn/jni/NmslibService.java | 10 +- .../plugin-metadata/plugin-security.policy | 1 + .../org/opensearch/knn/index/FaissIT.java | 104 +++++++++ .../KNN80DocValuesConsumerTests.java | 3 +- .../memory/NativeMemoryAllocationTests.java | 4 +- .../memory/NativeMemoryLoadStrategyTests.java | 4 +- .../opensearch/knn/jni/JNICommonsTest.java | 40 ++++ .../opensearch/knn/jni/JNIServiceTests.java | 217 ++++++++++-------- .../knn/training/TrainingJobTests.java | 10 +- .../java/org/opensearch/knn/TestUtils.java | 26 ++- 42 files changed, 970 insertions(+), 368 deletions(-) create mode 100644 jni/include/commons.h create mode 100644 jni/include/org_opensearch_knn_jni_JNICommons.h create mode 100644 jni/src/commons.cpp create mode 100644 jni/src/org_opensearch_knn_jni_JNICommons.cpp create mode 100644 jni/tests/commons_test.cpp create mode 100644 src/main/java/org/opensearch/knn/jni/JNICommons.java create mode 100644 src/test/java/org/opensearch/knn/jni/JNICommonsTest.java diff --git a/CHANGELOG.md b/CHANGELOG.md index e82376a6d..65e39b77b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Enhancements * Make the HitQueue size more appropriate for exact search [#1549](https://github.com/opensearch-project/k-NN/pull/1549) * Support script score when doc value is disabled [#1573](https://github.com/opensearch-project/k-NN/pull/1573) +* Implemented the Streaming Feature to stream vectors from Java to JNI layer to enable creation of larger segments for vector indices [#1604](https://github.com/opensearch-project/k-NN/pull/1604) ### Bug Fixes ### Infrastructure * Add micro-benchmark module in k-NN plugin for benchmark streaming vectors to JNI layer functionality. [#1583](https://github.com/opensearch-project/k-NN/pull/1583) diff --git a/jni/CMakeLists.txt b/jni/CMakeLists.txt index 60321ed1b..4f32c87b9 100644 --- a/jni/CMakeLists.txt +++ b/jni/CMakeLists.txt @@ -61,7 +61,7 @@ endif() # ---------------------------------------------------------------------------- # ---------------------------------- COMMON ---------------------------------- -add_library(${TARGET_LIB_COMMON} SHARED ${CMAKE_CURRENT_SOURCE_DIR}/src/jni_util.cpp) +add_library(${TARGET_LIB_COMMON} SHARED ${CMAKE_CURRENT_SOURCE_DIR}/src/jni_util.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/org_opensearch_knn_jni_JNICommons.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/commons.cpp) target_include_directories(${TARGET_LIB_COMMON} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include $ENV{JAVA_HOME}/include $ENV{JAVA_HOME}/include/${JVM_OS_TYPE}) set_target_properties(${TARGET_LIB_COMMON} PROPERTIES SUFFIX ${LIB_EXT}) set_target_properties(${TARGET_LIB_COMMON} PROPERTIES POSITION_INDEPENDENT_CODE ON) @@ -236,6 +236,7 @@ if ("${WIN32}" STREQUAL "") tests/faiss_util_test.cpp tests/nmslib_wrapper_test.cpp tests/test_util.cpp + tests/commons_test.cpp ) target_link_libraries( diff --git a/jni/include/commons.h b/jni/include/commons.h new file mode 100644 index 000000000..05367a693 --- /dev/null +++ b/jni/include/commons.h @@ -0,0 +1,37 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ +#include "jni_util.h" +#include +namespace knn_jni { + namespace commons { + /** + * This is utility function that can be used to store data in native memory. This function will allocate memory for + * the data(rows*columns) with initialCapacity and return the memory address where the data is stored. + * If you are using this function for first time use memoryAddress = 0 to ensure that a new memory location is created. + * For subsequent calls you can pass the same memoryAddress. If the data cannot be stored in the memory location + * will throw Exception. + * + * @param memoryAddress The address of the memory location where data will be stored. + * @param data 2D float array containing data to be stored in native memory. + * @param initialCapacity The initial capacity of the memory location. + * @return memory address where the data is stored. + */ + jlong storeVectorData(knn_jni::JNIUtilInterface *, JNIEnv *, jlong , jobjectArray, jlong); + + /** + * Free up the memory allocated for the data stored in memory address. This function should be used with the memory + * address returned by {@link JNICommons#storeVectorData(long, float[][], long, long)} + * + * @param memoryAddress address to be freed. + */ + void freeVectorData(jlong); + } +} diff --git a/jni/include/faiss_wrapper.h b/jni/include/faiss_wrapper.h index 2629eea43..3e1adeac4 100644 --- a/jni/include/faiss_wrapper.h +++ b/jni/include/faiss_wrapper.h @@ -19,13 +19,13 @@ namespace knn_jni { namespace faiss_wrapper { // Create an index with ids and vectors. The configuration is defined by values in the Java map, parametersJ. // The index is serialized to indexPathJ. - void CreateIndex(knn_jni::JNIUtilInterface * jniUtil, JNIEnv * env, jintArray idsJ, jobjectArray vectorsJ, + void CreateIndex(knn_jni::JNIUtilInterface * jniUtil, JNIEnv * env, jintArray idsJ, jlong vectorsAddressJ, jint dimJ, jstring indexPathJ, jobject parametersJ); // Create an index with ids and vectors. Instead of creating a new index, this function creates the index // based off of the template index passed in. The index is serialized to indexPathJ. void CreateIndexFromTemplate(knn_jni::JNIUtilInterface * jniUtil, JNIEnv * env, jintArray idsJ, - jobjectArray vectorsJ, jstring indexPathJ, jbyteArray templateIndexJ, + jlong vectorsAddressJ, jint dimJ, jstring indexPathJ, jbyteArray templateIndexJ, jobject parametersJ); // Load an index from indexPathJ into memory. diff --git a/jni/include/jni_util.h b/jni/include/jni_util.h index 52b08a202..b3d55f1c1 100644 --- a/jni/include/jni_util.h +++ b/jni/include/jni_util.h @@ -69,6 +69,9 @@ namespace knn_jni { virtual std::vector Convert2dJavaObjectArrayToCppFloatVector(JNIEnv *env, jobjectArray array2dJ, int dim) = 0; + virtual void Convert2dJavaObjectArrayAndStoreToFloatVector(JNIEnv *env, jobjectArray array2dJ, + int dim, std::vector *vect ) = 0; + virtual std::vector ConvertJavaIntArrayToCppIntVector(JNIEnv *env, jintArray arrayJ) = 0; // -------------------------------------------------------------------------- @@ -164,6 +167,7 @@ namespace knn_jni { void ReleaseLongArrayElements(JNIEnv *env, jlongArray array, jlong *elems, jint mode); void SetObjectArrayElement(JNIEnv *env, jobjectArray array, jsize index, jobject val); void SetByteArrayRegion(JNIEnv *env, jbyteArray array, jsize start, jsize len, const jbyte * buf); + void Convert2dJavaObjectArrayAndStoreToFloatVector(JNIEnv *env, jobjectArray array2dJ, int dim, std::vector *vect); private: std::unordered_map cachedClasses; diff --git a/jni/include/nmslib_wrapper.h b/jni/include/nmslib_wrapper.h index 9b555580a..08494644f 100644 --- a/jni/include/nmslib_wrapper.h +++ b/jni/include/nmslib_wrapper.h @@ -25,7 +25,7 @@ namespace knn_jni { namespace nmslib_wrapper { // Create an index with ids and vectors. The configuration is defined by values in the Java map, parametersJ. // The index is serialized to indexPathJ. - void CreateIndex(knn_jni::JNIUtilInterface * jniUtil, JNIEnv * env, jintArray idsJ, jobjectArray vectorsJ, + void CreateIndex(knn_jni::JNIUtilInterface * jniUtil, JNIEnv * env, jintArray idsJ, jlong vectorsAddress, jint dim, jstring indexPathJ, jobject parametersJ); // Load an index from indexPathJ into memory. Use parametersJ to set any query time parameters diff --git a/jni/include/org_opensearch_knn_jni_FaissService.h b/jni/include/org_opensearch_knn_jni_FaissService.h index ec1f46bc3..32b6f22f1 100644 --- a/jni/include/org_opensearch_knn_jni_FaissService.h +++ b/jni/include/org_opensearch_knn_jni_FaissService.h @@ -21,18 +21,18 @@ extern "C" { /* * Class: org_opensearch_knn_jni_FaissService * Method: createIndex - * Signature: ([I[[FLjava/lang/String;Ljava/util/Map;)V + * Signature: ([IJILjava/lang/String;Ljava/util/Map;)V */ JNIEXPORT void JNICALL Java_org_opensearch_knn_jni_FaissService_createIndex - (JNIEnv *, jclass, jintArray, jobjectArray, jstring, jobject); + (JNIEnv *, jclass, jintArray, jlong, jint, jstring, jobject); /* * Class: org_opensearch_knn_jni_FaissService * Method: createIndexFromTemplate - * Signature: ([I[[FLjava/lang/String;[BLjava/util/Map;)V + * Signature: ([IJILjava/lang/String;[BLjava/util/Map;)V */ JNIEXPORT void JNICALL Java_org_opensearch_knn_jni_FaissService_createIndexFromTemplate - (JNIEnv *, jclass, jintArray, jobjectArray, jstring, jbyteArray, jobject); + (JNIEnv *, jclass, jintArray, jlong, jint, jstring, jbyteArray, jobject); /* * Class: org_opensearch_knn_jni_FaissService @@ -122,22 +122,6 @@ JNIEXPORT jbyteArray JNICALL Java_org_opensearch_knn_jni_FaissService_trainIndex JNIEXPORT jlong JNICALL Java_org_opensearch_knn_jni_FaissService_transferVectors (JNIEnv *, jclass, jlong, jobjectArray); -/* - * Class: org_opensearch_knn_jni_FaissService - * Method: transferVectorsV2 - * Signature: (J[[F)J - */ -JNIEXPORT jlong JNICALL Java_org_opensearch_knn_jni_FaissService_transferVectorsV2 - (JNIEnv *, jclass, jlong, jobjectArray); - -/* - * Class: org_opensearch_knn_jni_FaissService - * Method: freeVectors - * Signature: (J)V - */ -JNIEXPORT void JNICALL Java_org_opensearch_knn_jni_FaissService_freeVectors - (JNIEnv *, jclass, jlong); - #ifdef __cplusplus } #endif diff --git a/jni/include/org_opensearch_knn_jni_JNICommons.h b/jni/include/org_opensearch_knn_jni_JNICommons.h new file mode 100644 index 000000000..d0758d7c8 --- /dev/null +++ b/jni/include/org_opensearch_knn_jni_JNICommons.h @@ -0,0 +1,40 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class org_opensearch_knn_jni_JNICommons */ + +#ifndef _Included_org_opensearch_knn_jni_JNICommons +#define _Included_org_opensearch_knn_jni_JNICommons +#ifdef __cplusplus +extern "C" { +#endif +/* + * Class: org_opensearch_knn_jni_JNICommons + * Method: storeVectorData + * Signature: (J[[FJJ) + */ +JNIEXPORT jlong JNICALL Java_org_opensearch_knn_jni_JNICommons_storeVectorData + (JNIEnv *, jclass, jlong, jobjectArray, jlong); + +/* + * Class: org_opensearch_knn_jni_JNICommons + * Method: freeVectorData + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_org_opensearch_knn_jni_JNICommons_freeVectorData + (JNIEnv *, jclass, jlong); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/jni/include/org_opensearch_knn_jni_NmslibService.h b/jni/include/org_opensearch_knn_jni_NmslibService.h index 02f58d20f..31422955f 100644 --- a/jni/include/org_opensearch_knn_jni_NmslibService.h +++ b/jni/include/org_opensearch_knn_jni_NmslibService.h @@ -21,10 +21,10 @@ extern "C" { /* * Class: org_opensearch_knn_jni_NmslibService * Method: createIndex - * Signature: ([I[[FLjava/lang/String;Ljava/util/Map;)V + * Signature: ([IJILjava/lang/String;Ljava/util/Map;)V */ JNIEXPORT void JNICALL Java_org_opensearch_knn_jni_NmslibService_createIndex - (JNIEnv *, jclass, jintArray, jobjectArray, jstring, jobject); + (JNIEnv *, jclass, jintArray, jlong, jint, jstring, jobject); /* * Class: org_opensearch_knn_jni_NmslibService diff --git a/jni/src/commons.cpp b/jni/src/commons.cpp new file mode 100644 index 000000000..3c03ac49d --- /dev/null +++ b/jni/src/commons.cpp @@ -0,0 +1,41 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ +#ifndef OPENSEARCH_KNN_COMMONS_H +#define OPENSEARCH_KNN_COMMONS_H +#include + +#include + +#include "jni_util.h" +#include "commons.h" + +jlong knn_jni::commons::storeVectorData(knn_jni::JNIUtilInterface *jniUtil, JNIEnv *env, jlong memoryAddressJ, + jobjectArray dataJ, jlong initialCapacityJ) { + std::vector *vect; + if ((long) memoryAddressJ == 0) { + vect = new std::vector(); + vect->reserve((long)initialCapacityJ); + } else { + vect = reinterpret_cast*>(memoryAddressJ); + } + int dim = jniUtil->GetInnerDimensionOf2dJavaFloatArray(env, dataJ); + jniUtil->Convert2dJavaObjectArrayAndStoreToFloatVector(env, dataJ, dim, vect); + + return (jlong) vect; +} + +void knn_jni::commons::freeVectorData(jlong memoryAddressJ) { + if (memoryAddressJ != 0) { + auto *vect = reinterpret_cast*>(memoryAddressJ); + delete vect; + } +} +#endif //OPENSEARCH_KNN_COMMONS_H \ No newline at end of file diff --git a/jni/src/faiss_wrapper.cpp b/jni/src/faiss_wrapper.cpp index a7075740e..817bdb816 100644 --- a/jni/src/faiss_wrapper.cpp +++ b/jni/src/faiss_wrapper.cpp @@ -81,15 +81,19 @@ bool isIndexIVFPQL2(faiss::Index * index); // IndexIDMap which has member that will point to underlying index that stores the data faiss::IndexIVFPQ * extractIVFPQIndex(faiss::Index * index); -void knn_jni::faiss_wrapper::CreateIndex(knn_jni::JNIUtilInterface * jniUtil, JNIEnv * env, jintArray idsJ, - jobjectArray vectorsJ, jstring indexPathJ, jobject parametersJ) { +void knn_jni::faiss_wrapper::CreateIndex(knn_jni::JNIUtilInterface * jniUtil, JNIEnv * env, jintArray idsJ, jlong vectorsAddressJ, jint dimJ, + jstring indexPathJ, jobject parametersJ) { if (idsJ == nullptr) { throw std::runtime_error("IDs cannot be null"); } - if (vectorsJ == nullptr) { - throw std::runtime_error("Vectors cannot be null"); + if (vectorsAddressJ <= 0) { + throw std::runtime_error("VectorsAddress cannot be less than 0"); + } + + if(dimJ <= 0) { + throw std::runtime_error("Vectors dimensions cannot be less than or equal to 0"); } if (indexPathJ == nullptr) { @@ -109,16 +113,20 @@ void knn_jni::faiss_wrapper::CreateIndex(knn_jni::JNIUtilInterface * jniUtil, JN std::string spaceTypeCpp(jniUtil->ConvertJavaObjectToCppString(env, spaceTypeJ)); faiss::MetricType metric = TranslateSpaceToMetric(spaceTypeCpp); - // Read data set - int numVectors = jniUtil->GetJavaObjectArrayLength(env, vectorsJ); + // Read vectors from memory address + auto *inputVectors = reinterpret_cast*>(vectorsAddressJ); + int dim = (int)dimJ; + // The number of vectors can be int here because a lucene segment number of total docs never crosses INT_MAX value + int numVectors = (int) (inputVectors->size() / (uint64_t) dim); + if(numVectors == 0) { + throw std::runtime_error("Number of vectors cannot be 0"); + } + int numIds = jniUtil->GetJavaIntArrayLength(env, idsJ); if (numIds != numVectors) { throw std::runtime_error("Number of IDs does not match number of vectors"); } - int dim = jniUtil->GetInnerDimensionOf2dJavaFloatArray(env, vectorsJ); - auto dataset = jniUtil->Convert2dJavaObjectArrayToCppFloatVector(env, vectorsJ, dim); - // Create faiss index jobject indexDescriptionJ = knn_jni::GetJObjectFromMapOrThrow(parametersCpp, knn_jni::INDEX_DESCRIPTION); std::string indexDescriptionCpp(jniUtil->ConvertJavaObjectToCppString(env, indexDescriptionJ)); @@ -148,22 +156,30 @@ void knn_jni::faiss_wrapper::CreateIndex(knn_jni::JNIUtilInterface * jniUtil, JN auto idVector = jniUtil->ConvertJavaIntArrayToCppIntVector(env, idsJ); faiss::IndexIDMap idMap = faiss::IndexIDMap(indexWriter.get()); - idMap.add_with_ids(numVectors, dataset.data(), idVector.data()); + idMap.add_with_ids(numVectors, inputVectors->data(), idVector.data()); // Write the index to disk std::string indexPathCpp(jniUtil->ConvertJavaStringToCppString(env, indexPathJ)); faiss::write_index(&idMap, indexPathCpp.c_str()); + // Releasing the vectorsAddressJ memory as that is not required once we have created the index. + // This is not the ideal approach, please refer this gh issue for long term solution: + // https://github.com/opensearch-project/k-NN/issues/1600 + delete inputVectors; } void knn_jni::faiss_wrapper::CreateIndexFromTemplate(knn_jni::JNIUtilInterface * jniUtil, JNIEnv * env, jintArray idsJ, - jobjectArray vectorsJ, jstring indexPathJ, + jlong vectorsAddressJ, jint dimJ, jstring indexPathJ, jbyteArray templateIndexJ, jobject parametersJ) { if (idsJ == nullptr) { throw std::runtime_error("IDs cannot be null"); } - if (vectorsJ == nullptr) { - throw std::runtime_error("Vectors cannot be null"); + if (vectorsAddressJ <= 0) { + throw std::runtime_error("VectorsAddress cannot be less than 0"); + } + + if(dimJ <= 0) { + throw std::runtime_error("Vectors dimensions cannot be less than or equal to 0"); } if (indexPathJ == nullptr) { @@ -183,15 +199,15 @@ void knn_jni::faiss_wrapper::CreateIndexFromTemplate(knn_jni::JNIUtilInterface * jniUtil->DeleteLocalRef(env, parametersJ); // Read data set - int numVectors = jniUtil->GetJavaObjectArrayLength(env, vectorsJ); + // Read vectors from memory address + auto *inputVectors = reinterpret_cast*>(vectorsAddressJ); + int dim = (int)dimJ; + int numVectors = (int) (inputVectors->size() / (uint64_t) dim); int numIds = jniUtil->GetJavaIntArrayLength(env, idsJ); if (numIds != numVectors) { throw std::runtime_error("Number of IDs does not match number of vectors"); } - int dim = jniUtil->GetInnerDimensionOf2dJavaFloatArray(env, vectorsJ); - auto dataset = jniUtil->Convert2dJavaObjectArrayToCppFloatVector(env, vectorsJ, dim); - // Get vector of bytes from jbytearray int indexBytesCount = jniUtil->GetJavaBytesArrayLength(env, templateIndexJ); jbyte * indexBytesJ = jniUtil->GetByteArrayElements(env, templateIndexJ, nullptr); @@ -208,8 +224,11 @@ void knn_jni::faiss_wrapper::CreateIndexFromTemplate(knn_jni::JNIUtilInterface * auto idVector = jniUtil->ConvertJavaIntArrayToCppIntVector(env, idsJ); faiss::IndexIDMap idMap = faiss::IndexIDMap(indexWriter.get()); - idMap.add_with_ids(numVectors, dataset.data(), idVector.data()); - + idMap.add_with_ids(numVectors, inputVectors->data(), idVector.data()); + // Releasing the vectorsAddressJ memory as that is not required once we have created the index. + // This is not the ideal approach, please refer this gh issue for long term solution: + // https://github.com/opensearch-project/k-NN/issues/1600 + delete inputVectors; // Write the index to disk std::string indexPathCpp(jniUtil->ConvertJavaStringToCppString(env, indexPathJ)); faiss::write_index(&idMap, indexPathCpp.c_str()); diff --git a/jni/src/jni_util.cpp b/jni/src/jni_util.cpp index a0c1d5733..a1faa4894 100644 --- a/jni/src/jni_util.cpp +++ b/jni/src/jni_util.cpp @@ -223,6 +223,13 @@ int knn_jni::JNIUtil::ConvertJavaObjectToCppInteger(JNIEnv *env, jobject objectJ std::vector knn_jni::JNIUtil::Convert2dJavaObjectArrayToCppFloatVector(JNIEnv *env, jobjectArray array2dJ, int dim) { + std::vector vect; + Convert2dJavaObjectArrayAndStoreToFloatVector(env, array2dJ, dim, &vect); + return vect; +} + +void knn_jni::JNIUtil::Convert2dJavaObjectArrayAndStoreToFloatVector(JNIEnv *env, jobjectArray array2dJ, + int dim, std::vector *vect) { if (array2dJ == nullptr) { throw std::runtime_error("Array cannot be null"); @@ -231,7 +238,6 @@ std::vector knn_jni::JNIUtil::Convert2dJavaObjectArrayToCppFloatVector(JN int numVectors = env->GetArrayLength(array2dJ); this->HasExceptionInStack(env); - std::vector floatVectorCpp; for (int i = 0; i < numVectors; ++i) { auto vectorArray = (jfloatArray)env->GetObjectArrayElement(array2dJ, i); this->HasExceptionInStack(env, "Unable to get object array element"); @@ -247,13 +253,12 @@ std::vector knn_jni::JNIUtil::Convert2dJavaObjectArrayToCppFloatVector(JN } for(int j = 0; j < dim; ++j) { - floatVectorCpp.push_back(vector[j]); + vect->push_back(vector[j]); } env->ReleaseFloatArrayElements(vectorArray, vector, JNI_ABORT); } this->HasExceptionInStack(env); env->DeleteLocalRef(array2dJ); - return floatVectorCpp; } std::vector knn_jni::JNIUtil::ConvertJavaIntArrayToCppIntVector(JNIEnv *env, jintArray arrayJ) { diff --git a/jni/src/nmslib_wrapper.cpp b/jni/src/nmslib_wrapper.cpp index f63fd2b01..6ea80d727 100644 --- a/jni/src/nmslib_wrapper.cpp +++ b/jni/src/nmslib_wrapper.cpp @@ -32,14 +32,19 @@ std::string TranslateSpaceType(const std::string& spaceType); const similarity::LabelType DEFAULT_LABEL = -1; void knn_jni::nmslib_wrapper::CreateIndex(knn_jni::JNIUtilInterface * jniUtil, JNIEnv * env, jintArray idsJ, - jobjectArray vectorsJ, jstring indexPathJ, jobject parametersJ) { + jlong vectorsAddressJ, jint dimJ, + jstring indexPathJ, jobject parametersJ) { if (idsJ == nullptr) { throw std::runtime_error("IDs cannot be null"); } - if (vectorsJ == nullptr) { - throw std::runtime_error("Vectors cannot be null"); + if (vectorsAddressJ <= 0) { + throw std::runtime_error("VectorsAddress cannot be less than 0"); + } + + if(dimJ <= 0) { + throw std::runtime_error("Vectors dimensions cannot be less than or equal to 0"); } if (indexPathJ == nullptr) { @@ -91,12 +96,18 @@ void knn_jni::nmslib_wrapper::CreateIndex(knn_jni::JNIUtilInterface * jniUtil, J space.reset(similarity::SpaceFactoryRegistry::Instance().CreateSpace(spaceTypeCpp,similarity::AnyParams())); // Get number of ids and vectors and dimension - int numVectors = jniUtil->GetJavaObjectArrayLength(env, vectorsJ); + auto *inputVectors = reinterpret_cast*>(vectorsAddressJ); + int dim = (int)dimJ; + // The number of vectors can be int here because a lucene segment number of total docs never crosses INT_MAX value + int numVectors = (int) ( inputVectors->size() / (uint64_t) dim); + if(numVectors == 0) { + throw std::runtime_error("Number of vectors cannot be 0"); + } + int numIds = jniUtil->GetJavaIntArrayLength(env, idsJ); if (numIds != numVectors) { throw std::runtime_error("Number of IDs does not match number of vectors"); } - int dim = jniUtil->GetInnerDimensionOf2dJavaFloatArray(env, vectorsJ); // Read dataset similarity::ObjectVector dataset; @@ -105,10 +116,12 @@ void knn_jni::nmslib_wrapper::CreateIndex(knn_jni::JNIUtilInterface * jniUtil, J try { // Read in data set idsCpp = jniUtil->GetIntArrayElements(env, idsJ, nullptr); - - float* floatArrayCpp; - jfloatArray floatArrayJ; size_t vectorSizeInBytes = dim*sizeof(float); + // vectorPointer needs to be unsigned long long, this will ensure that out of range doesn't happen for this pointer + // when the values of numVectors * dim becomes very large. + // Example: for 10M vectors of 1536 dim vectorPointer max value will be ~15.3B which is already > range of ints. + // keeping it unsigned long long we will never go above the range. + unsigned long long vectorPointer = 0; // Allocate a large buffer that will contain all the vectors. Allocating the objects in one large buffer as // opposed to individually will prevent heap fragmentation. We have observed that allocating individual @@ -134,18 +147,18 @@ void knn_jni::nmslib_wrapper::CreateIndex(knn_jni::JNIUtilInterface * jniUtil, J memcpy(ptr, &vectorSizeInBytes, similarity::DATALENGTH_SIZE); ptr += similarity::DATALENGTH_SIZE; - floatArrayJ = (jfloatArray)jniUtil->GetObjectArrayElement(env, vectorsJ, i); - if (dim != jniUtil->GetJavaFloatArrayLength(env, floatArrayJ)) { - throw std::runtime_error("Dimension of vectors is inconsistent"); - } - - floatArrayCpp = jniUtil->GetFloatArrayElements(env, floatArrayJ, nullptr); - memcpy(ptr, floatArrayCpp, vectorSizeInBytes); - jniUtil->ReleaseFloatArrayElements(env, floatArrayJ, floatArrayCpp, JNI_ABORT); + memcpy(ptr, &(inputVectors->at(vectorPointer)), vectorSizeInBytes); ptr += vectorSizeInBytes; + vectorPointer += dim; } jniUtil->ReleaseIntArrayElements(env, idsJ, idsCpp, JNI_ABORT); + // Releasing the vectorsAddressJ memory as that is not required once we have created the index. + // This is not the ideal approach, please refer this gh issue for long term solution: + // https://github.com/opensearch-project/k-NN/issues/1600 + //commons::freeVectorData(vectorsAddressJ); + delete inputVectors; + std::unique_ptr> index; index.reset(similarity::MethodFactoryRegistry::Instance().CreateMethod(false, "hnsw", spaceTypeCpp, *(space), dataset)); index->CreateIndex(similarity::AnyParams(indexParameters)); diff --git a/jni/src/org_opensearch_knn_jni_FaissService.cpp b/jni/src/org_opensearch_knn_jni_FaissService.cpp index 3d9624c25..3249ed872 100644 --- a/jni/src/org_opensearch_knn_jni_FaissService.cpp +++ b/jni/src/org_opensearch_knn_jni_FaissService.cpp @@ -13,7 +13,6 @@ #include -#include #include #include "faiss_wrapper.h" @@ -41,11 +40,11 @@ void JNI_OnUnload(JavaVM *vm, void *reserved) { } JNIEXPORT void JNICALL Java_org_opensearch_knn_jni_FaissService_createIndex(JNIEnv * env, jclass cls, jintArray idsJ, - jobjectArray vectorsJ, jstring indexPathJ, - jobject parametersJ) + jlong vectorsAddressJ, jint dimJ, + jstring indexPathJ, jobject parametersJ) { try { - knn_jni::faiss_wrapper::CreateIndex(&jniUtil, env, idsJ, vectorsJ, indexPathJ, parametersJ); + knn_jni::faiss_wrapper::CreateIndex(&jniUtil, env, idsJ, vectorsAddressJ, dimJ, indexPathJ, parametersJ); } catch (...) { jniUtil.CatchCppExceptionAndThrowJava(env); } @@ -53,13 +52,14 @@ JNIEXPORT void JNICALL Java_org_opensearch_knn_jni_FaissService_createIndex(JNIE JNIEXPORT void JNICALL Java_org_opensearch_knn_jni_FaissService_createIndexFromTemplate(JNIEnv * env, jclass cls, jintArray idsJ, - jobjectArray vectorsJ, + jlong vectorsAddressJ, + jint dimJ, jstring indexPathJ, jbyteArray templateIndexJ, jobject parametersJ) { try { - knn_jni::faiss_wrapper::CreateIndexFromTemplate(&jniUtil, env, idsJ, vectorsJ, indexPathJ, templateIndexJ, parametersJ); + knn_jni::faiss_wrapper::CreateIndexFromTemplate(&jniUtil, env, idsJ, vectorsAddressJ, dimJ, indexPathJ, templateIndexJ, parametersJ); } catch (...) { jniUtil.CatchCppExceptionAndThrowJava(env); } @@ -190,30 +190,3 @@ JNIEXPORT jlong JNICALL Java_org_opensearch_knn_jni_FaissService_transferVectors return (jlong) vect; } - -JNIEXPORT jlong JNICALL Java_org_opensearch_knn_jni_FaissService_transferVectorsV2(JNIEnv * env, jclass cls, -jlong vectorsPointerJ, - jobjectArray vectorsJ) -{ - std::vector *vect; - if ((long) vectorsPointerJ == 0) { - vect = new std::vector; - } else { - vect = reinterpret_cast*>(vectorsPointerJ); - } - - int dim = jniUtil.GetInnerDimensionOf2dJavaFloatArray(env, vectorsJ); - auto dataset = jniUtil.Convert2dJavaObjectArrayToCppFloatVector(env, vectorsJ, dim); - vect->insert(vect->end(), dataset.begin(), dataset.end()); - - return (jlong) vect; -} - -JNIEXPORT void JNICALL Java_org_opensearch_knn_jni_FaissService_freeVectors(JNIEnv * env, jclass cls, - jlong vectorsPointerJ) -{ - if (vectorsPointerJ != 0) { - auto *vect = reinterpret_cast*>(vectorsPointerJ); - delete vect; - } -} diff --git a/jni/src/org_opensearch_knn_jni_JNICommons.cpp b/jni/src/org_opensearch_knn_jni_JNICommons.cpp new file mode 100644 index 000000000..ccdd11882 --- /dev/null +++ b/jni/src/org_opensearch_knn_jni_JNICommons.cpp @@ -0,0 +1,60 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +#include "org_opensearch_knn_jni_JNICommons.h" + +#include +#include "commons.h" +#include "jni_util.h" + +static knn_jni::JNIUtil jniUtil; +static const jint KNN_JNICOMMONS_JNI_VERSION = JNI_VERSION_1_1; + +jint JNI_OnLoad(JavaVM* vm, void* reserved) { + // Obtain the JNIEnv from the VM and confirm JNI_VERSION + JNIEnv* env; + if (vm->GetEnv((void**)&env, KNN_JNICOMMONS_JNI_VERSION) != JNI_OK) { + return JNI_ERR; + } + + jniUtil.Initialize(env); + + return KNN_JNICOMMONS_JNI_VERSION; +} + +void JNI_OnUnload(JavaVM *vm, void *reserved) { + JNIEnv* env; + vm->GetEnv((void**)&env, KNN_JNICOMMONS_JNI_VERSION); + jniUtil.Uninitialize(env); +} + + +JNIEXPORT jlong JNICALL Java_org_opensearch_knn_jni_JNICommons_storeVectorData(JNIEnv * env, jclass cls, +jlong memoryAddressJ, jobjectArray dataJ, jlong initialCapacityJ) + +{ + try { + return knn_jni::commons::storeVectorData(&jniUtil, env, memoryAddressJ, dataJ, initialCapacityJ); + } catch (...) { + jniUtil.CatchCppExceptionAndThrowJava(env); + } + return (long)memoryAddressJ; +} + +JNIEXPORT void JNICALL Java_org_opensearch_knn_jni_JNICommons_freeVectorData(JNIEnv * env, jclass cls, + jlong memoryAddressJ) +{ + try { + return knn_jni::commons::freeVectorData(memoryAddressJ); + } catch (...) { + jniUtil.CatchCppExceptionAndThrowJava(env); + } +} diff --git a/jni/src/org_opensearch_knn_jni_NmslibService.cpp b/jni/src/org_opensearch_knn_jni_NmslibService.cpp index 11dd885b1..d037d3337 100644 --- a/jni/src/org_opensearch_knn_jni_NmslibService.cpp +++ b/jni/src/org_opensearch_knn_jni_NmslibService.cpp @@ -38,11 +38,11 @@ void JNI_OnUnload(JavaVM *vm, void *reserved) { } JNIEXPORT void JNICALL Java_org_opensearch_knn_jni_NmslibService_createIndex(JNIEnv * env, jclass cls, jintArray idsJ, - jobjectArray vectorsJ, jstring indexPathJ, + jlong vectorsAddressJ, jint dimJ, jstring indexPathJ, jobject parametersJ) { try { - knn_jni::nmslib_wrapper::CreateIndex(&jniUtil, env, idsJ, vectorsJ, indexPathJ, parametersJ); + knn_jni::nmslib_wrapper::CreateIndex(&jniUtil, env, idsJ, vectorsAddressJ, dimJ, indexPathJ, parametersJ); } catch (...) { jniUtil.CatchCppExceptionAndThrowJava(env); } diff --git a/jni/tests/commons_test.cpp b/jni/tests/commons_test.cpp new file mode 100644 index 000000000..09323f0fb --- /dev/null +++ b/jni/tests/commons_test.cpp @@ -0,0 +1,73 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + + +#include "test_util.h" +#include +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "jni_util.h" +#include "commons.h" + +TEST(CommonsTests, BasicAssertions) { + long dim = 3; + long totalNumberOfVector = 5; + std::vector> data; + for(int i = 0 ; i < totalNumberOfVector - 1 ; i++) { + std::vector vector; + for(int j = 0 ; j < dim ; j ++) { + vector.push_back((float)j); + } + data.push_back(vector); + } + JNIEnv *jniEnv = nullptr; + + testing::NiceMock mockJNIUtil; + + jlong memoryAddress = knn_jni::commons::storeVectorData(&mockJNIUtil, jniEnv, (jlong)0, + reinterpret_cast(&data), (jlong)(totalNumberOfVector * dim)); + ASSERT_NE(memoryAddress, 0); + auto *vect = reinterpret_cast*>(memoryAddress); + ASSERT_EQ(vect->size(), data.size() * dim); + ASSERT_EQ(vect->capacity(), totalNumberOfVector * dim); + + // Check by inserting more vectors at same memory location + jlong oldMemoryAddress = memoryAddress; + std::vector> data2; + std::vector vector; + for(int j = 0 ; j < dim ; j ++) { + vector.push_back((float)j); + } + data2.push_back(vector); + memoryAddress = knn_jni::commons::storeVectorData(&mockJNIUtil, jniEnv, memoryAddress, + reinterpret_cast(&data2), (jlong)(totalNumberOfVector * dim)); + ASSERT_NE(memoryAddress, 0); + ASSERT_EQ(memoryAddress, oldMemoryAddress); + vect = reinterpret_cast*>(memoryAddress); + int currentIndex = 0; + ASSERT_EQ(vect->size(), totalNumberOfVector*dim); + ASSERT_EQ(vect->capacity(), totalNumberOfVector * dim); + + // Validate if all vectors data are at correct location + for(auto & i : data) { + for(float j : i) { + ASSERT_FLOAT_EQ(vect->at(currentIndex), j); + currentIndex++; + } + } + + for(auto & i : data2) { + for(float j : i) { + ASSERT_FLOAT_EQ(vect->at(currentIndex), j); + currentIndex++; + } + } +} diff --git a/jni/tests/faiss_wrapper_test.cpp b/jni/tests/faiss_wrapper_test.cpp index 2b1684cfb..05854f7ed 100644 --- a/jni/tests/faiss_wrapper_test.cpp +++ b/jni/tests/faiss_wrapper_test.cpp @@ -30,17 +30,14 @@ TEST(FaissCreateIndexTest, BasicAssertions) { // Define the data faiss::idx_t numIds = 200; std::vector ids; - std::vector> vectors; + auto *vectors = new std::vector(); int dim = 2; + vectors->reserve(dim * numIds); for (int64_t i = 0; i < numIds; ++i) { ids.push_back(i); - - std::vector vect; - vect.reserve(dim); for (int j = 0; j < dim; ++j) { - vect.push_back(test_util::RandomFloat(-500.0, 500.0)); + vectors->push_back(test_util::RandomFloat(-500.0, 500.0)); } - vectors.push_back(vect); } std::string indexPath = test_util::RandomString(10, "tmp/", ".faiss"); @@ -58,12 +55,12 @@ TEST(FaissCreateIndexTest, BasicAssertions) { EXPECT_CALL(mockJNIUtil, GetJavaObjectArrayLength( jniEnv, reinterpret_cast(&vectors))) - .WillRepeatedly(Return(vectors.size())); + .WillRepeatedly(Return(vectors->size())); // Create the index knn_jni::faiss_wrapper::CreateIndex( &mockJNIUtil, jniEnv, reinterpret_cast(&ids), - reinterpret_cast(&vectors), (jstring)&indexPath, + (jlong) vectors, dim , (jstring)&indexPath, (jobject)¶metersMap); // Make sure index can be loaded @@ -77,17 +74,14 @@ TEST(FaissCreateIndexFromTemplateTest, BasicAssertions) { // Define the data faiss::idx_t numIds = 100; std::vector ids; - std::vector> vectors; + auto *vectors = new std::vector(); int dim = 2; + vectors->reserve(dim * numIds); for (int64_t i = 0; i < numIds; ++i) { ids.push_back(i); - - std::vector vect; - vect.reserve(dim); for (int j = 0; j < dim; ++j) { - vect.push_back(test_util::RandomFloat(-500.0, 500.0)); + vectors->push_back(test_util::RandomFloat(-500.0, 500.0)); } - vectors.push_back(vect); } std::string indexPath = test_util::RandomString(10, "tmp/", ".faiss"); @@ -105,7 +99,7 @@ TEST(FaissCreateIndexFromTemplateTest, BasicAssertions) { EXPECT_CALL(mockJNIUtil, GetJavaObjectArrayLength( jniEnv, reinterpret_cast(&vectors))) - .WillRepeatedly(Return(vectors.size())); + .WillRepeatedly(Return(vectors->size())); std::string spaceType = knn_jni::L2; std::unordered_map parametersMap; @@ -113,7 +107,7 @@ TEST(FaissCreateIndexFromTemplateTest, BasicAssertions) { knn_jni::faiss_wrapper::CreateIndexFromTemplate( &mockJNIUtil, jniEnv, reinterpret_cast(&ids), - reinterpret_cast(&vectors), (jstring)&indexPath, + (jlong)vectors, dim, (jstring)&indexPath, reinterpret_cast(&(vectorIoWriter.data)), (jobject) ¶metersMap ); @@ -486,17 +480,14 @@ TEST(FaissCreateHnswSQfp16IndexTest, BasicAssertions) { // Define the data faiss::idx_t numIds = 200; std::vector ids; - std::vector> vectors; + auto *vectors = new std::vector(); int dim = 2; + vectors->reserve(dim * numIds); for (int64_t i = 0; i < numIds; ++i) { ids.push_back(i); - - std::vector vect; - vect.reserve(dim); for (int j = 0; j < dim; ++j) { - vect.push_back(test_util::RandomFloat(-500.0, 500.0)); + vectors->push_back(test_util::RandomFloat(-500.0, 500.0)); } - vectors.push_back(vect); } std::string indexPath = test_util::RandomString(10, "tmp/", ".faiss"); @@ -514,12 +505,12 @@ TEST(FaissCreateHnswSQfp16IndexTest, BasicAssertions) { EXPECT_CALL(mockJNIUtil, GetJavaObjectArrayLength( jniEnv, reinterpret_cast(&vectors))) - .WillRepeatedly(Return(vectors.size())); + .WillRepeatedly(Return(vectors->size())); // Create the index knn_jni::faiss_wrapper::CreateIndex( &mockJNIUtil, jniEnv, reinterpret_cast(&ids), - reinterpret_cast(&vectors), (jstring)&indexPath, + (jlong)vectors, dim, (jstring)&indexPath, (jobject)¶metersMap); // Make sure index can be loaded diff --git a/jni/tests/nmslib_wrapper_test.cpp b/jni/tests/nmslib_wrapper_test.cpp index 3a21e7401..1fd9471b0 100644 --- a/jni/tests/nmslib_wrapper_test.cpp +++ b/jni/tests/nmslib_wrapper_test.cpp @@ -39,17 +39,14 @@ TEST(NmslibCreateIndexTest, BasicAssertions) { // Define index data int numIds = 100; std::vector ids; - std::vector> vectors; + auto *vectors = new std::vector(); int dim = 2; - for (int i = 0; i < numIds; ++i) { + vectors->reserve(dim * numIds); + for (int64_t i = 0; i < numIds; ++i) { ids.push_back(i); - - std::vector vect; - vect.reserve(dim); for (int j = 0; j < dim; ++j) { - vect.push_back(test_util::RandomFloat(-500.0, 500.0)); + vectors->push_back(test_util::RandomFloat(-500.0, 500.0)); } - vectors.push_back(vect); } std::string indexPath = test_util::RandomString(10, "tmp/", ".nmslib"); @@ -70,7 +67,7 @@ TEST(NmslibCreateIndexTest, BasicAssertions) { EXPECT_CALL(mockJNIUtil, GetJavaObjectArrayLength( jniEnv, reinterpret_cast(&vectors))) - .WillRepeatedly(Return(vectors.size())); + .WillRepeatedly(Return(vectors->size())); EXPECT_CALL(mockJNIUtil, GetJavaIntArrayLength(jniEnv, reinterpret_cast(&ids))) @@ -79,7 +76,7 @@ TEST(NmslibCreateIndexTest, BasicAssertions) { // Create the index knn_jni::nmslib_wrapper::CreateIndex( &mockJNIUtil, jniEnv, reinterpret_cast(&ids), - reinterpret_cast(&vectors), (jstring)&indexPath, + (jlong) vectors, dim, (jstring)&indexPath, (jobject)¶metersMap); // Make sure index can be loaded diff --git a/jni/tests/test_util.cpp b/jni/tests/test_util.cpp index 89b19f9aa..92532b9e2 100644 --- a/jni/tests/test_util.cpp +++ b/jni/tests/test_util.cpp @@ -45,6 +45,14 @@ test_util::MockJNIUtil::MockJNIUtil() { return data; }); + ON_CALL(*this, Convert2dJavaObjectArrayAndStoreToFloatVector) + .WillByDefault([this](JNIEnv *env, jobjectArray array2dJ, int dim, std::vector* data) { + for (const auto &v : + (*reinterpret_cast> *>(array2dJ))) + for (auto item : v) data->push_back(item); + }); + + // arrayJ is re-interpreted as std::vector * ON_CALL(*this, ConvertJavaIntArrayToCppIntVector) .WillByDefault([this](JNIEnv *env, jintArray arrayJ) { diff --git a/jni/tests/test_util.h b/jni/tests/test_util.h index 1e32ad3c3..8e73a8ab0 100644 --- a/jni/tests/test_util.h +++ b/jni/tests/test_util.h @@ -44,6 +44,8 @@ namespace test_util { // TODO: Figure out why this cant use "new" MOCK_METHOD MOCK_METHOD(std::vector, Convert2dJavaObjectArrayToCppFloatVector, (JNIEnv * env, jobjectArray array2dJ, int dim)); + MOCK_METHOD(void, Convert2dJavaObjectArrayAndStoreToFloatVector, + (JNIEnv * env, jobjectArray array2dJ, int dim, std::vector*vect)); MOCK_METHOD(std::vector, ConvertJavaIntArrayToCppIntVector, (JNIEnv * env, jintArray arrayJ)); MOCK_METHOD2(ConvertJavaMapToCppMap, diff --git a/micro-benchmarks/src/main/java/org/opensearch/knn/TransferVectorsBenchmarks.java b/micro-benchmarks/src/main/java/org/opensearch/knn/TransferVectorsBenchmarks.java index ad1076484..2bce54ee6 100644 --- a/micro-benchmarks/src/main/java/org/opensearch/knn/TransferVectorsBenchmarks.java +++ b/micro-benchmarks/src/main/java/org/opensearch/knn/TransferVectorsBenchmarks.java @@ -23,7 +23,7 @@ import org.openjdk.jmh.annotations.Setup; import org.openjdk.jmh.annotations.State; import org.openjdk.jmh.annotations.Warmup; -import org.opensearch.knn.jni.JNIService; +import org.opensearch.knn.jni.JNICommons; import java.util.ArrayList; import java.util.List; @@ -42,9 +42,9 @@ @State(Scope.Benchmark) public class TransferVectorsBenchmarks { private static final Random random = new Random(1212121212); - private static final int TOTAL_NUMBER_OF_VECTOR_TO_BE_TRANSFERRED = 1000000; + private static final long TOTAL_NUMBER_OF_VECTOR_TO_BE_TRANSFERRED = 1000000; - @Param({ "128", "256", "384", "512" }) + @Param({ "128", "256", "384", "512", "960", "1024", "1536" }) private int dimension; @Param({ "100000", "500000", "1000000" }) @@ -61,20 +61,30 @@ public void setup() { } @Benchmark - public void transferVectors() { + public void transferVectors_withCapacity() { long vectorsAddress = 0; List vectorToTransfer = new ArrayList<>(); + long startingIndex = 0; for (float[] floats : vectorList) { if (vectorToTransfer.size() == vectorsPerTransfer) { - vectorsAddress = JNIService.transferVectorsV2(vectorsAddress, vectorToTransfer.toArray(new float[][] {})); + vectorsAddress = JNICommons.storeVectorData( + vectorsAddress, + vectorToTransfer.toArray(new float[][] {}), + dimension * TOTAL_NUMBER_OF_VECTOR_TO_BE_TRANSFERRED + ); + startingIndex += vectorsPerTransfer; vectorToTransfer = new ArrayList<>(); } vectorToTransfer.add(floats); } if (!vectorToTransfer.isEmpty()) { - vectorsAddress = JNIService.transferVectorsV2(vectorsAddress, vectorToTransfer.toArray(new float[][] {})); + vectorsAddress = JNICommons.storeVectorData( + vectorsAddress, + vectorToTransfer.toArray(new float[][] {}), + dimension * TOTAL_NUMBER_OF_VECTOR_TO_BE_TRANSFERRED + ); } - JNIService.freeVectors(vectorsAddress); + JNICommons.freeVectorData(vectorsAddress); } private float[] generateRandomVector(int dimensions) { diff --git a/src/main/java/org/opensearch/knn/common/KNNConstants.java b/src/main/java/org/opensearch/knn/common/KNNConstants.java index 46d82b2bf..6c92afabc 100644 --- a/src/main/java/org/opensearch/knn/common/KNNConstants.java +++ b/src/main/java/org/opensearch/knn/common/KNNConstants.java @@ -72,6 +72,7 @@ public class KNNConstants { // nmslib specific constants public static final String NMSLIB_NAME = "nmslib"; + public static final String COMMONS_NAME = "common"; public static final String SPACE_TYPE = "spaceType"; // used as field info key public static final String HNSW_ALGO_M = "M"; public static final String HNSW_ALGO_EF_CONSTRUCTION = "efConstruction"; @@ -120,6 +121,7 @@ public class KNNConstants { public static final String FAISS_JNI_LIBRARY_NAME = JNI_LIBRARY_PREFIX + FAISS_NAME; public static final String FAISS_AVX2_JNI_LIBRARY_NAME = JNI_LIBRARY_PREFIX + FAISS_NAME + "_avx2"; public static final String NMSLIB_JNI_LIBRARY_NAME = JNI_LIBRARY_PREFIX + NMSLIB_NAME; + public static final String COMMON_JNI_LIBRARY_NAME = JNI_LIBRARY_PREFIX + COMMONS_NAME; // Filtered Search Constants // Please refer this github issue for more details for choosing this value: diff --git a/src/main/java/org/opensearch/knn/index/KNNSettings.java b/src/main/java/org/opensearch/knn/index/KNNSettings.java index 9ac0bf216..8d0ad91e0 100644 --- a/src/main/java/org/opensearch/knn/index/KNNSettings.java +++ b/src/main/java/org/opensearch/knn/index/KNNSettings.java @@ -68,6 +68,7 @@ public class KNNSettings { public static final String KNN_ALGO_PARAM_INDEX_THREAD_QTY = "knn.algo_param.index_thread_qty"; public static final String KNN_MEMORY_CIRCUIT_BREAKER_ENABLED = "knn.memory.circuit_breaker.enabled"; public static final String KNN_MEMORY_CIRCUIT_BREAKER_LIMIT = "knn.memory.circuit_breaker.limit"; + public static final String KNN_VECTOR_STREAMING_MEMORY_LIMIT_IN_MB = "knn.vector_streaming_memory.limit"; public static final String KNN_CIRCUIT_BREAKER_TRIGGERED = "knn.circuit_breaker.triggered"; public static final String KNN_CACHE_ITEM_EXPIRY_ENABLED = "knn.cache.item.expiry.enabled"; public static final String KNN_CACHE_ITEM_EXPIRY_TIME_MINUTES = "knn.cache.item.expiry.minutes"; @@ -93,6 +94,7 @@ public class KNNSettings { public static final Integer KNN_DEFAULT_MODEL_CACHE_SIZE_LIMIT_PERCENTAGE = 10; // By default, set aside 10% of the JVM for the limit public static final Integer KNN_MAX_MODEL_CACHE_SIZE_LIMIT_PERCENTAGE = 25; // Model cache limit cannot exceed 25% of the JVM heap public static final String KNN_DEFAULT_MEMORY_CIRCUIT_BREAKER_LIMIT = "50%"; + public static final String KNN_DEFAULT_VECTOR_STREAMING_MEMORY_LIMIT_PCT = "1%"; public static final Integer ADVANCED_FILTERED_EXACT_SEARCH_THRESHOLD_DEFAULT_VALUE = -1; @@ -100,6 +102,15 @@ public class KNNSettings { * Settings Definition */ + // This setting controls how much memory should be used to transfer vectors from Java to JNI Layer. The default + // 1% of the JVM heap + public static final Setting KNN_VECTOR_STREAMING_MEMORY_LIMIT_PCT_SETTING = Setting.memorySizeSetting( + KNN_VECTOR_STREAMING_MEMORY_LIMIT_IN_MB, + KNN_DEFAULT_VECTOR_STREAMING_MEMORY_LIMIT_PCT, + Setting.Property.Dynamic, + Setting.Property.NodeScope + ); + public static final Setting INDEX_KNN_SPACE_TYPE = Setting.simpleString( KNN_SPACE_TYPE, INDEX_KNN_DEFAULT_SPACE_TYPE, @@ -354,6 +365,10 @@ private Setting getSetting(String key) { return KNN_FAISS_AVX2_DISABLED_SETTING; } + if (KNN_VECTOR_STREAMING_MEMORY_LIMIT_IN_MB.equals(key)) { + return KNN_VECTOR_STREAMING_MEMORY_LIMIT_PCT_SETTING; + } + throw new IllegalArgumentException("Cannot find setting by key [" + key + "]"); } @@ -371,7 +386,8 @@ public List> getSettings() { MODEL_INDEX_NUMBER_OF_REPLICAS_SETTING, MODEL_CACHE_SIZE_LIMIT_SETTING, ADVANCED_FILTERED_EXACT_SEARCH_THRESHOLD_SETTING, - KNN_FAISS_AVX2_DISABLED_SETTING + KNN_FAISS_AVX2_DISABLED_SETTING, + KNN_VECTOR_STREAMING_MEMORY_LIMIT_PCT_SETTING ); return Stream.concat(settings.stream(), dynamicCacheSettings.values().stream()).collect(Collectors.toList()); } @@ -475,6 +491,10 @@ public void onFailure(Exception e) { }); } + public static ByteSizeValue getVectorStreamingMemoryLimit() { + return KNNSettings.state().getSettingValue(KNN_VECTOR_STREAMING_MEMORY_LIMIT_IN_MB); + } + /** * * @param index Name of the index diff --git a/src/main/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80BinaryDocValues.java b/src/main/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80BinaryDocValues.java index 832737a6d..df26766b3 100644 --- a/src/main/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80BinaryDocValues.java +++ b/src/main/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80BinaryDocValues.java @@ -5,6 +5,7 @@ package org.opensearch.knn.index.codec.KNN80Codec; +import lombok.Getter; import org.opensearch.knn.index.codec.util.BinaryDocValuesSub; import org.apache.lucene.index.BinaryDocValues; import org.apache.lucene.index.DocIDMerger; @@ -15,10 +16,13 @@ /** * A per-document kNN numeric value. */ -class KNN80BinaryDocValues extends BinaryDocValues { +public class KNN80BinaryDocValues extends BinaryDocValues { private DocIDMerger docIDMerger; + @Getter + private long totalLiveDocs; + KNN80BinaryDocValues(DocIDMerger docIdMerger) { this.docIDMerger = docIdMerger; } @@ -61,4 +65,14 @@ public long cost() { public BytesRef binaryValue() throws IOException { return current.getValues().binaryValue(); } -}; + + /** + * Builder pattern like setter for setting totalLiveDocs. We can use setter also. But this way the code is clean. + * @param totalLiveDocs int + * @return {@link KNN80BinaryDocValues} + */ + public KNN80BinaryDocValues setTotalLiveDocs(long totalLiveDocs) { + this.totalLiveDocs = totalLiveDocs; + return this; + } +} diff --git a/src/main/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumer.java b/src/main/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumer.java index 804c4f7bb..096df817a 100644 --- a/src/main/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumer.java +++ b/src/main/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumer.java @@ -109,11 +109,11 @@ public void addKNNBinaryField(FieldInfo field, DocValuesProducer valuesProducer, // Get values to be indexed BinaryDocValues values = valuesProducer.getBinary(field); KNNCodecUtil.Pair pair = KNNCodecUtil.getFloats(values); - if (pair.vectors.length == 0 || pair.docs.length == 0) { - logger.info("Skipping engine index creation as there are no vectors or docs in the documents"); + if (pair.getVectorAddress() == 0 || pair.docs.length == 0) { + logger.info("Skipping engine index creation as there are no vectors or docs in the segment"); return; } - long arraySize = calculateArraySize(pair.vectors, pair.serializationMode); + long arraySize = calculateArraySize(pair.docs.length, pair.getDimension(), pair.serializationMode); if (isMerge) { KNNGraphValue.MERGE_CURRENT_OPERATIONS.increment(); KNNGraphValue.MERGE_CURRENT_DOCS.incrementBy(pair.docs.length); @@ -121,31 +121,27 @@ public void addKNNBinaryField(FieldInfo field, DocValuesProducer valuesProducer, } // Increment counter for number of graph index requests KNNCounter.GRAPH_INDEX_REQUESTS.increment(); - // Create library index either from model or from scratch - String engineFileName; - String indexPath; - NativeIndexCreator indexCreator; final KNNEngine knnEngine = getKNNEngine(field); + final String engineFileName = buildEngineFileName( + state.segmentInfo.name, + knnEngine.getVersion(), + field.name, + knnEngine.getExtension() + ); + final String indexPath = Paths.get( + ((FSDirectory) (FilterDirectory.unwrap(state.directory))).getDirectory().toString(), + engineFileName + ).toString(); + NativeIndexCreator indexCreator; + // Create library index either from model or from scratch if (field.attributes().containsKey(MODEL_ID)) { - String modelId = field.attributes().get(MODEL_ID); Model model = ModelCache.getInstance().get(modelId); - - engineFileName = buildEngineFileName(state.segmentInfo.name, knnEngine.getVersion(), field.name, knnEngine.getExtension()); - indexPath = Paths.get(((FSDirectory) (FilterDirectory.unwrap(state.directory))).getDirectory().toString(), engineFileName) - .toString(); - if (model.getModelBlob() == null) { - throw new RuntimeException("There is no trained model with id \"" + modelId + "\""); + throw new RuntimeException(String.format("There is no trained model with id \"%s\"", modelId)); } - indexCreator = () -> createKNNIndexFromTemplate(model.getModelBlob(), pair, knnEngine, indexPath); } else { - - engineFileName = buildEngineFileName(state.segmentInfo.name, knnEngine.getVersion(), field.name, knnEngine.getExtension()); - indexPath = Paths.get(((FSDirectory) (FilterDirectory.unwrap(state.directory))).getDirectory().toString(), engineFileName) - .toString(); - indexCreator = () -> createKNNIndexFromScratch(field, pair, knnEngine, indexPath); } @@ -184,7 +180,15 @@ private void createKNNIndexFromTemplate(byte[] model, KNNCodecUtil.Pair pair, KN KNNSettings.state().getSettingValue(KNNSettings.KNN_ALGO_PARAM_INDEX_THREAD_QTY) ); AccessController.doPrivileged((PrivilegedAction) () -> { - JNIService.createIndexFromTemplate(pair.docs, pair.vectors, indexPath, model, parameters, knnEngine); + JNIService.createIndexFromTemplate( + pair.docs, + pair.getVectorAddress(), + pair.getDimension(), + indexPath, + model, + parameters, + knnEngine + ); return null; }); } @@ -223,7 +227,7 @@ private void createKNNIndexFromScratch(FieldInfo fieldInfo, KNNCodecUtil.Pair pa // Pass the path for the nms library to save the file AccessController.doPrivileged((PrivilegedAction) () -> { - JNIService.createIndex(pair.docs, pair.vectors, indexPath, parameters, knnEngine); + JNIService.createIndex(pair.docs, pair.getVectorAddress(), pair.getDimension(), indexPath, parameters, knnEngine); return null; }); } diff --git a/src/main/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesReader.java b/src/main/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesReader.java index ccfaa68fc..16380c5d9 100644 --- a/src/main/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesReader.java +++ b/src/main/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesReader.java @@ -5,6 +5,10 @@ package org.opensearch.knn.index.codec.KNN80Codec; +import lombok.extern.log4j.Log4j2; +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.util.Bits; +import org.opensearch.common.StopWatch; import org.opensearch.knn.index.codec.util.BinaryDocValuesSub; import org.apache.lucene.codecs.DocValuesProducer; import org.apache.lucene.index.BinaryDocValues; @@ -14,12 +18,14 @@ import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.MergeState; +import java.io.IOException; import java.util.ArrayList; import java.util.List; /** * Reader for KNNDocValues from the segments */ +@Log4j2 class KNN80DocValuesReader extends EmptyDocValuesProducer { private final MergeState mergeState; @@ -30,6 +36,7 @@ class KNN80DocValuesReader extends EmptyDocValuesProducer { @Override public BinaryDocValues getBinary(FieldInfo field) { + long totalLiveDocs = 0; try { List subs = new ArrayList<>(this.mergeState.docValuesProducers.length); for (int i = 0; i < this.mergeState.docValuesProducers.length; i++) { @@ -41,13 +48,49 @@ public BinaryDocValues getBinary(FieldInfo field) { values = docValuesProducer.getBinary(readerFieldInfo); } if (values != null) { + totalLiveDocs = totalLiveDocs + getLiveDocsCount(values, this.mergeState.liveDocs[i]); + // docValues will be consumed when liveDocs are not null, hence resetting the docsValues + // pointer. + values = this.mergeState.liveDocs[i] != null ? docValuesProducer.getBinary(readerFieldInfo) : values; + subs.add(new BinaryDocValuesSub(mergeState.docMaps[i], values)); } } } - return new KNN80BinaryDocValues(DocIDMerger.of(subs, mergeState.needsIndexSort)); + return new KNN80BinaryDocValues(DocIDMerger.of(subs, mergeState.needsIndexSort)).setTotalLiveDocs(totalLiveDocs); } catch (Exception e) { throw new RuntimeException(e); } } + + /** + * This function return the liveDocs count present in the BinaryDocValues. If the liveDocsBits is null, then we + * can use {@link BinaryDocValues#cost()} function to get max docIds. But if LiveDocsBits is not null, then we + * iterate over the BinaryDocValues and validate if the docId is present in the live docs bits or not. + * + * @param binaryDocValues {@link BinaryDocValues} + * @param liveDocsBits {@link Bits} + * @return total number of liveDocs. + * @throws IOException + */ + private long getLiveDocsCount(final BinaryDocValues binaryDocValues, final Bits liveDocsBits) throws IOException { + long liveDocs = 0; + if (liveDocsBits != null) { + int docId; + // This is not the right way to log the time. I create a github issue for adding an annotation to track + // the time. https://github.com/opensearch-project/k-NN/issues/1594 + StopWatch stopWatch = new StopWatch(); + stopWatch.start(); + for (docId = binaryDocValues.nextDoc(); docId != DocIdSetIterator.NO_MORE_DOCS; docId = binaryDocValues.nextDoc()) { + if (liveDocsBits.get(docId)) { + liveDocs++; + } + } + stopWatch.stop(); + log.debug("Time taken to iterate over binary doc values: {} ms", stopWatch.totalTime().millis()); + } else { + liveDocs = binaryDocValues.cost(); + } + return liveDocs; + } } diff --git a/src/main/java/org/opensearch/knn/index/codec/util/KNNCodecUtil.java b/src/main/java/org/opensearch/knn/index/codec/util/KNNCodecUtil.java index 02ab2d833..c5ae469e0 100644 --- a/src/main/java/org/opensearch/knn/index/codec/util/KNNCodecUtil.java +++ b/src/main/java/org/opensearch/knn/index/codec/util/KNNCodecUtil.java @@ -5,18 +5,22 @@ package org.opensearch.knn.index.codec.util; +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.Setter; import org.apache.lucene.index.BinaryDocValues; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.util.BytesRef; +import org.opensearch.knn.index.KNNSettings; +import org.opensearch.knn.index.codec.KNN80Codec.KNN80BinaryDocValues; +import org.opensearch.knn.jni.JNICommons; import java.io.ByteArrayInputStream; import java.io.IOException; import java.util.ArrayList; +import java.util.List; public class KNNCodecUtil { - - public static final String HNSW_EXTENSION = ".hnsw"; - public static final String HNSW_COMPOUND_EXTENSION = ".hnswc"; // Floats are 4 bytes in size public static final int FLOAT_BYTE_SIZE = 4; // References to objects are 4 bytes in size @@ -26,42 +30,63 @@ public class KNNCodecUtil { // Java rounds each array size up to multiples of 8 bytes public static final int JAVA_ROUNDING_NUMBER = 8; + @AllArgsConstructor public static final class Pair { - public Pair(int[] docs, float[][] vectors, SerializationMode serializationMode) { - this.docs = docs; - this.vectors = vectors; - this.serializationMode = serializationMode; - } - public int[] docs; - public float[][] vectors; + @Getter + @Setter + private long vectorAddress; + @Getter + @Setter + private int dimension; public SerializationMode serializationMode; + } public static KNNCodecUtil.Pair getFloats(BinaryDocValues values) throws IOException { - ArrayList vectorList = new ArrayList<>(); - ArrayList docIdList = new ArrayList<>(); + List vectorList = new ArrayList<>(); + List docIdList = new ArrayList<>(); + long vectorAddress = 0; + int dimension = 0; SerializationMode serializationMode = SerializationMode.COLLECTION_OF_FLOATS; + + long totalLiveDocs = getTotalLiveDocsCount(values); + long vectorsStreamingMemoryLimit = KNNSettings.getVectorStreamingMemoryLimit().getBytes(); + long vectorsPerTransfer = Integer.MIN_VALUE; + for (int doc = values.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = values.nextDoc()) { BytesRef bytesref = values.binaryValue(); try (ByteArrayInputStream byteStream = new ByteArrayInputStream(bytesref.bytes, bytesref.offset, bytesref.length)) { serializationMode = KNNVectorSerializerFactory.serializerModeFromStream(byteStream); final KNNVectorSerializer vectorSerializer = KNNVectorSerializerFactory.getSerializerByStreamContent(byteStream); final float[] vector = vectorSerializer.byteToFloatArray(byteStream); + dimension = vector.length; + + if (vectorsPerTransfer == Integer.MIN_VALUE) { + vectorsPerTransfer = (dimension * Float.BYTES * totalLiveDocs) / vectorsStreamingMemoryLimit; + } + if (vectorList.size() == vectorsPerTransfer) { + vectorAddress = JNICommons.storeVectorData( + vectorAddress, + vectorList.toArray(new float[][] {}), + totalLiveDocs * dimension + ); + // We should probably come up with a better way to reuse the vectorList memory which we have + // created. Problem here is doing like this can lead to a lot of list memory which is of no use and + // will be garbage collected later on, but it creates pressure on JVM. We should revisit this. + vectorList = new ArrayList<>(); + } vectorList.add(vector); } docIdList.add(doc); } - return new KNNCodecUtil.Pair( - docIdList.stream().mapToInt(Integer::intValue).toArray(), - vectorList.toArray(new float[][] {}), - serializationMode - ); + if (vectorList.isEmpty() == false) { + vectorAddress = JNICommons.storeVectorData(vectorAddress, vectorList.toArray(new float[][] {}), totalLiveDocs * dimension); + } + return new KNNCodecUtil.Pair(docIdList.stream().mapToInt(Integer::intValue).toArray(), vectorAddress, dimension, serializationMode); } - public static long calculateArraySize(float[][] vectors, SerializationMode serializationMode) { - int vectorLength = vectors[0].length; - int numVectors = vectors.length; + public static long calculateArraySize(int numVectors, int vectorLength, SerializationMode serializationMode) { if (serializationMode == SerializationMode.ARRAY) { int vectorSize = vectorLength * FLOAT_BYTE_SIZE + JAVA_ARRAY_HEADER_SIZE; if (vectorSize % JAVA_ROUNDING_NUMBER != 0) { @@ -96,4 +121,14 @@ public static String buildEngineFilePrefix(String segmentName) { public static String buildEngineFileSuffix(String fieldName, String extension) { return String.format("_%s%s", fieldName, extension); } + + private static long getTotalLiveDocsCount(final BinaryDocValues binaryDocValues) { + long totalLiveDocs; + if (binaryDocValues instanceof KNN80BinaryDocValues) { + totalLiveDocs = ((KNN80BinaryDocValues) binaryDocValues).getTotalLiveDocs(); + } else { + totalLiveDocs = binaryDocValues.cost(); + } + return totalLiveDocs; + } } diff --git a/src/main/java/org/opensearch/knn/index/memory/NativeMemoryAllocation.java b/src/main/java/org/opensearch/knn/index/memory/NativeMemoryAllocation.java index 286e6265c..c8b56436b 100644 --- a/src/main/java/org/opensearch/knn/index/memory/NativeMemoryAllocation.java +++ b/src/main/java/org/opensearch/knn/index/memory/NativeMemoryAllocation.java @@ -13,6 +13,7 @@ import org.apache.lucene.index.LeafReaderContext; import org.opensearch.knn.index.query.KNNWeight; +import org.opensearch.knn.jni.JNICommons; import org.opensearch.knn.jni.JNIService; import org.opensearch.knn.index.util.KNNEngine; import org.opensearch.watcher.FileWatcher; @@ -313,7 +314,7 @@ private void cleanup() { closed = true; if (this.memoryAddress != 0) { - JNIService.freeVectors(this.memoryAddress); + JNICommons.freeVectorData(this.memoryAddress); } } diff --git a/src/main/java/org/opensearch/knn/jni/FaissService.java b/src/main/java/org/opensearch/knn/jni/FaissService.java index 4b5045359..32516ef9d 100644 --- a/src/main/java/org/opensearch/knn/jni/FaissService.java +++ b/src/main/java/org/opensearch/knn/jni/FaissService.java @@ -50,27 +50,33 @@ class FaissService { } /** - * Create an index for the native library + * Create an index for the native library The memory occupied by the vectorsAddress will be freed up during the + * function call. So Java layer doesn't need to free up the memory. This is not an ideal behavior because Java layer + * created the memory address and that should only free up the memory. We are tracking the proper fix for this on this + * issue * * @param ids array of ids mapping to the data passed in - * @param data array of float arrays to be indexed + * @param vectorsAddress address of native memory where vectors are stored + * @param dim dimension of the vector to be indexed * @param indexPath path to save index file to * @param parameters parameters to build index */ - public static native void createIndex(int[] ids, float[][] data, String indexPath, Map parameters); + public static native void createIndex(int[] ids, long vectorsAddress, int dim, String indexPath, Map parameters); /** * Create an index for the native library with a provided template index * * @param ids array of ids mapping to the data passed in - * @param data array of float arrays to be indexed + * @param vectorsAddress address of native memory where vectors are stored + * @param dim dimension of the vector to be indexed * @param indexPath path to save index file to * @param templateIndex empty template index * @param parameters additional build time parameters */ public static native void createIndexFromTemplate( int[] ids, - float[][] data, + long vectorsAddress, + int dim, String indexPath, byte[] templateIndex, Map parameters @@ -173,33 +179,15 @@ public static native KNNQueryResult[] queryIndexWithFilter( public static native byte[] trainIndex(Map indexParameters, int dimension, long trainVectorsPointer); /** + *

+ * The function is deprecated. Use {@link JNICommons#storeVectorData(long, float[][], long)} + *

* Transfer vectors from Java to native * * @param vectorsPointer pointer to vectors in native memory. Should be 0 to create vector as well * @param trainingData data to be transferred * @return pointer to native memory location of training data */ + @Deprecated(since = "2.14.0", forRemoval = true) public static native long transferVectors(long vectorsPointer, float[][] trainingData); - - /** - * Transfer vectors from Java to native layer. This is the version 2 of transfer vector functionality. The - * difference between this and the version 1 is, this version puts vectors at the end rather than in front. - * Keeping this name as V2 for now, will come up with better name going forward. - *

- * TODO: Rename the function - *
- * TODO: Make this function native function and use a common cpp file to host these functions. - *

- * @param vectorsPointer pointer to vectors in native memory. Should be 0 to create vector as well - * @param data data to be transferred - * @return pointer to native memory location for data - */ - public static native long transferVectorsV2(long vectorsPointer, float[][] data); - - /** - * Free vectors from memory - * - * @param vectorsPointer to be freed - */ - public static native void freeVectors(long vectorsPointer); } diff --git a/src/main/java/org/opensearch/knn/jni/JNICommons.java b/src/main/java/org/opensearch/knn/jni/JNICommons.java new file mode 100644 index 000000000..90ad70c3d --- /dev/null +++ b/src/main/java/org/opensearch/knn/jni/JNICommons.java @@ -0,0 +1,62 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.knn.jni; + +import org.opensearch.knn.common.KNNConstants; + +import java.security.AccessController; +import java.security.PrivilegedAction; + +/** + * Common class for providing the JNI related functionality to various JNIServices. + */ +public class JNICommons { + + static { + AccessController.doPrivileged((PrivilegedAction) () -> { + System.loadLibrary(KNNConstants.COMMON_JNI_LIBRARY_NAME); + return null; + }); + } + + /** + * This is utility function that can be used to store data in native memory. This function will allocate memory for + * the data(rows*columns) with initialCapacity and return the memory address where the data is stored. + * If you are using this function for first time use memoryAddress = 0 to ensure that a new memory location is created. + * For subsequent calls you can pass the same memoryAddress. If the data cannot be stored in the memory location + * will throw Exception. + * + *

+ * The function is not threadsafe. If multiple threads are trying to insert on same memory location, then it can + * lead to data corruption. + *

+ * + * @param memoryAddress The address of the memory location where data will be stored. + * @param data 2D float array containing data to be stored in native memory. + * @param initialCapacity The initial capacity of the memory location. + * @return memory address where the data is stored. + */ + public static native long storeVectorData(long memoryAddress, float[][] data, long initialCapacity); + + /** + * Free up the memory allocated for the data stored in memory address. This function should be used with the memory + * address returned by {@link JNICommons#storeVectorData(long, float[][], long)} + * + *

+ * The function is not threadsafe. If multiple threads are trying to free up same memory location, then it can + * lead to errors. + *

+ * + * @param memoryAddress address to be freed. + */ + public static native void freeVectorData(long memoryAddress); +} diff --git a/src/main/java/org/opensearch/knn/jni/JNIService.java b/src/main/java/org/opensearch/knn/jni/JNIService.java index 80b56b173..5a5b6794a 100644 --- a/src/main/java/org/opensearch/knn/jni/JNIService.java +++ b/src/main/java/org/opensearch/knn/jni/JNIService.java @@ -23,22 +23,34 @@ public class JNIService { /** - * Create an index for the native library + * Create an index for the native library. The memory occupied by the vectorsAddress will be freed up during the + * function call. So Java layer doesn't need to free up the memory. This is not an ideal behavior because Java layer + * created the memory address and that should only free up the memory. We are tracking the proper fix for this on this + * issue * * @param ids array of ids mapping to the data passed in - * @param data array of float arrays to be indexed + * @param vectorsAddress address of native memory where vectors are stored + * @param dim dimension of the vector to be indexed * @param indexPath path to save index file to * @param parameters parameters to build index * @param knnEngine engine to build index for */ - public static void createIndex(int[] ids, float[][] data, String indexPath, Map parameters, KNNEngine knnEngine) { + public static void createIndex( + int[] ids, + long vectorsAddress, + int dim, + String indexPath, + Map parameters, + KNNEngine knnEngine + ) { + if (KNNEngine.NMSLIB == knnEngine) { - NmslibService.createIndex(ids, data, indexPath, parameters); + NmslibService.createIndex(ids, vectorsAddress, dim, indexPath, parameters); return; } if (KNNEngine.FAISS == knnEngine) { - FaissService.createIndex(ids, data, indexPath, parameters); + FaissService.createIndex(ids, vectorsAddress, dim, indexPath, parameters); return; } @@ -49,7 +61,8 @@ public static void createIndex(int[] ids, float[][] data, String indexPath, Map< * Create an index for the native library with a provided template index * * @param ids array of ids mapping to the data passed in - * @param data array of float arrays to be indexed + * @param vectorsAddress address of native memory where vectors are stored + * @param dim dimension of vectors to be indexed * @param indexPath path to save index file to * @param templateIndex empty template index * @param parameters parameters to build index @@ -57,14 +70,15 @@ public static void createIndex(int[] ids, float[][] data, String indexPath, Map< */ public static void createIndexFromTemplate( int[] ids, - float[][] data, + long vectorsAddress, + int dim, String indexPath, byte[] templateIndex, Map parameters, KNNEngine knnEngine ) { if (KNNEngine.FAISS == knnEngine) { - FaissService.createIndexFromTemplate(ids, data, indexPath, templateIndex, parameters); + FaissService.createIndexFromTemplate(ids, vectorsAddress, dim, indexPath, templateIndex, parameters); return; } @@ -235,44 +249,17 @@ public static byte[] trainIndex(Map indexParameters, int dimensi } /** + *

+ * The function is deprecated. Use {@link JNICommons#storeVectorData(long, float[][], long)} + *

* Transfer vectors from Java to native * * @param vectorsPointer pointer to vectors in native memory. Should be 0 to create vector as well * @param trainingData data to be transferred * @return pointer to native memory location of training data */ + @Deprecated(since = "2.14.0", forRemoval = true) public static long transferVectors(long vectorsPointer, float[][] trainingData) { return FaissService.transferVectors(vectorsPointer, trainingData); } - - /** - * Free vectors from memory - * - * @param vectorsPointer to be freed - */ - public static void freeVectors(long vectorsPointer) { - FaissService.freeVectors(vectorsPointer); - } - - /** - * Experimental: Transfer vectors from Java to native layer. This is the version 2 of transfer vector - * functionality. The difference between this and the version 1 is, this version puts vectors at the end rather - * than in front. Keeping this name as V2 for now, will come up with better name going forward. - *

- * This is not a production ready function for now. Adding this to ensure that we are able to run atleast 1 - * micro-benchmarks. - *

- *

- * TODO: Rename the function - *
- * TODO: Make this function native function and use a common cpp file to host these functions. - *

- * @param vectorsPointer pointer to vectors in native memory. Should be 0 to create vector as well - * @param data data to be transferred - * @return pointer to native memory location for data - * - */ - public static long transferVectorsV2(long vectorsPointer, float[][] data) { - return FaissService.transferVectorsV2(vectorsPointer, data); - } } diff --git a/src/main/java/org/opensearch/knn/jni/NmslibService.java b/src/main/java/org/opensearch/knn/jni/NmslibService.java index 77896822a..7fdc278d2 100644 --- a/src/main/java/org/opensearch/knn/jni/NmslibService.java +++ b/src/main/java/org/opensearch/knn/jni/NmslibService.java @@ -39,14 +39,18 @@ class NmslibService { } /** - * Create an index for the native library + * Create an index for the native library. The memory occupied by the vectorsAddress will be freed up during the + * function call. So Java layer doesn't need to free up the memory. This is not an ideal behavior because Java layer + * created the memory address and that should only free up the memory. We are tracking the proper fix for this on this + * issue * * @param ids array of ids mapping to the data passed in - * @param data array of float arrays to be indexed + * @param vectorsAddress address of native memory where vectors are stored + * @param dim dimension of the vector to be indexed * @param indexPath path to save index file to * @param parameters parameters to build index */ - public static native void createIndex(int[] ids, float[][] data, String indexPath, Map parameters); + public static native void createIndex(int[] ids, long vectorsAddress, int dim, String indexPath, Map parameters); /** * Load an index into memory diff --git a/src/main/plugin-metadata/plugin-security.policy b/src/main/plugin-metadata/plugin-security.policy index 91624613c..d5ab0be21 100644 --- a/src/main/plugin-metadata/plugin-security.policy +++ b/src/main/plugin-metadata/plugin-security.policy @@ -1,6 +1,7 @@ grant { permission java.lang.RuntimePermission "loadLibrary.opensearchknn_nmslib"; permission java.lang.RuntimePermission "loadLibrary.opensearchknn_faiss"; + permission java.lang.RuntimePermission "loadLibrary.opensearchknn_common"; permission java.lang.RuntimePermission "loadLibrary.opensearchknn_faiss_avx2"; permission java.net.SocketPermission "*", "connect,resolve"; permission java.lang.RuntimePermission "accessDeclaredMembers"; diff --git a/src/test/java/org/opensearch/knn/index/FaissIT.java b/src/test/java/org/opensearch/knn/index/FaissIT.java index 72ffd2b66..1f9b423bc 100644 --- a/src/test/java/org/opensearch/knn/index/FaissIT.java +++ b/src/test/java/org/opensearch/knn/index/FaissIT.java @@ -34,10 +34,12 @@ import java.io.IOException; import java.net.URL; import java.util.Arrays; +import java.util.HashSet; import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Random; +import java.util.Set; import java.util.TreeMap; import java.util.stream.Collectors; @@ -172,6 +174,108 @@ public void testEndToEnd_whenMethodIsHNSWFlat_thenSucceed() { fail("Graphs are not getting evicted"); } + @SneakyThrows + public void testEndToEnd_whenMethodIsHNSWFlatAndHasDeletedDocs_thenSucceed() { + String indexName = "test-index-1"; + String fieldName = "test-field-1"; + + KNNMethod hnswMethod = KNNEngine.FAISS.getMethod(KNNConstants.METHOD_HNSW); + SpaceType spaceType = SpaceType.L2; + + List mValues = ImmutableList.of(16, 32, 64, 128); + List efConstructionValues = ImmutableList.of(16, 32, 64, 128); + List efSearchValues = ImmutableList.of(16, 32, 64, 128); + + Integer dimension = testData.indexData.vectors[0].length; + + // Create an index + XContentBuilder builder = XContentFactory.jsonBuilder() + .startObject() + .startObject("properties") + .startObject(fieldName) + .field("type", "knn_vector") + .field("dimension", dimension) + .startObject(KNNConstants.KNN_METHOD) + .field(KNNConstants.NAME, hnswMethod.getMethodComponent().getName()) + .field(KNNConstants.METHOD_PARAMETER_SPACE_TYPE, spaceType.getValue()) + .field(KNNConstants.KNN_ENGINE, KNNEngine.FAISS.getName()) + .startObject(KNNConstants.PARAMETERS) + .field(KNNConstants.METHOD_PARAMETER_M, mValues.get(random().nextInt(mValues.size()))) + .field(KNNConstants.METHOD_PARAMETER_EF_CONSTRUCTION, efConstructionValues.get(random().nextInt(efConstructionValues.size()))) + .field(KNNConstants.METHOD_PARAMETER_EF_SEARCH, efSearchValues.get(random().nextInt(efSearchValues.size()))) + .endObject() + .endObject() + .endObject() + .endObject() + .endObject(); + + Map mappingMap = xContentBuilderToMap(builder); + String mapping = builder.toString(); + + createKnnIndex(indexName, mapping); + assertEquals(new TreeMap<>(mappingMap), new TreeMap<>(getIndexMappingAsMap(indexName))); + + // Index the test data + for (int i = 0; i < testData.indexData.docs.length; i++) { + addKnnDoc( + indexName, + Integer.toString(testData.indexData.docs[i]), + fieldName, + Floats.asList(testData.indexData.vectors[i]).toArray() + ); + } + + // Assert we have the right number of documents in the index + refreshAllNonSystemIndices(); + assertEquals(testData.indexData.docs.length, getDocCount(indexName)); + + final Set docIdsToBeDeleted = new HashSet<>(); + while (docIdsToBeDeleted.size() < 10) { + docIdsToBeDeleted.add(randomInt(testData.indexData.docs.length)); + } + + for (Integer id : docIdsToBeDeleted) { + deleteKnnDoc(indexName, Integer.toString(testData.indexData.docs[id])); + } + refreshAllNonSystemIndices(); + forceMergeKnnIndex(indexName, 3); + + assertEquals(testData.indexData.docs.length - 10, getDocCount(indexName)); + + int k = 10; + for (int i = 0; i < testData.queries.length; i++) { + Response response = searchKNNIndex(indexName, new KNNQueryBuilder(fieldName, testData.queries[i], k), k); + String responseBody = EntityUtils.toString(response.getEntity()); + List knnResults = parseSearchResponse(responseBody, fieldName); + assertEquals(k, knnResults.size()); + + List actualScores = parseSearchResponseScore(responseBody, fieldName); + for (int j = 0; j < k; j++) { + float[] primitiveArray = knnResults.get(j).getVector(); + assertEquals( + KNNEngine.FAISS.score(KNNScoringUtil.l2Squared(testData.queries[i], primitiveArray), spaceType), + actualScores.get(j), + 0.0001 + ); + } + } + + // Delete index + deleteKNNIndex(indexName); + + // Search every 5 seconds 14 times to confirm graph gets evicted + int intervals = 14; + for (int i = 0; i < intervals; i++) { + if (getTotalGraphsInCache() == 0) { + return; + } + + Thread.sleep(5 * 1000); + } + + fail("Graphs are not getting evicted"); + } + @SneakyThrows public void testEndToEnd_whenMethodIsHNSWPQ_thenSucceed() { String indexName = "test-index"; diff --git a/src/test/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumerTests.java b/src/test/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumerTests.java index cdf3109a4..2ce3a7c83 100644 --- a/src/test/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumerTests.java +++ b/src/test/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumerTests.java @@ -38,6 +38,7 @@ import org.opensearch.knn.indices.ModelDao; import org.opensearch.knn.indices.ModelMetadata; import org.opensearch.knn.indices.ModelState; +import org.opensearch.knn.jni.JNICommons; import org.opensearch.knn.jni.JNIService; import org.opensearch.knn.plugin.stats.KNNCounter; import org.opensearch.knn.plugin.stats.KNNGraphValue; @@ -357,7 +358,7 @@ public void testAddKNNBinaryField_fromModel_faiss() throws IOException, Executio modelBytes, modelId ); - JNIService.freeVectors(trainingPtr); + JNICommons.freeVectorData(trainingPtr); // Setup the model cache to return the correct model ModelDao modelDao = mock(ModelDao.class); diff --git a/src/test/java/org/opensearch/knn/index/memory/NativeMemoryAllocationTests.java b/src/test/java/org/opensearch/knn/index/memory/NativeMemoryAllocationTests.java index fabffab46..7573a4394 100644 --- a/src/test/java/org/opensearch/knn/index/memory/NativeMemoryAllocationTests.java +++ b/src/test/java/org/opensearch/knn/index/memory/NativeMemoryAllocationTests.java @@ -15,6 +15,7 @@ import org.opensearch.knn.KNNTestCase; import org.opensearch.knn.common.KNNConstants; import org.opensearch.knn.index.IndexUtil; +import org.opensearch.knn.jni.JNICommons; import org.opensearch.knn.jni.JNIService; import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.util.KNNEngine; @@ -52,7 +53,8 @@ public void testIndexAllocation_close() throws InterruptedException { Arrays.fill(vectors[i], 1f); } Map parameters = ImmutableMap.of(KNNConstants.SPACE_TYPE, SpaceType.DEFAULT.getValue()); - JNIService.createIndex(ids, vectors, path, parameters, knnEngine); + long vectorMemoryAddress = JNICommons.storeVectorData(0, vectors, numVectors * dimension); + JNIService.createIndex(ids, vectorMemoryAddress, dimension, path, parameters, knnEngine); // Load index into memory long memoryAddress = JNIService.loadIndex(path, parameters, knnEngine); diff --git a/src/test/java/org/opensearch/knn/index/memory/NativeMemoryLoadStrategyTests.java b/src/test/java/org/opensearch/knn/index/memory/NativeMemoryLoadStrategyTests.java index 207b6373b..a84974202 100644 --- a/src/test/java/org/opensearch/knn/index/memory/NativeMemoryLoadStrategyTests.java +++ b/src/test/java/org/opensearch/knn/index/memory/NativeMemoryLoadStrategyTests.java @@ -16,6 +16,7 @@ import org.opensearch.action.search.SearchResponse; import org.opensearch.knn.KNNTestCase; import org.opensearch.knn.common.KNNConstants; +import org.opensearch.knn.jni.JNICommons; import org.opensearch.knn.jni.JNIService; import org.opensearch.knn.index.query.KNNQueryResult; import org.opensearch.knn.index.SpaceType; @@ -53,7 +54,8 @@ public void testIndexLoadStrategy_load() throws IOException { Arrays.fill(vectors[i], 1f); } Map parameters = ImmutableMap.of(KNNConstants.SPACE_TYPE, SpaceType.DEFAULT.getValue()); - JNIService.createIndex(ids, vectors, path, parameters, knnEngine); + long memoryAddress = JNICommons.storeVectorData(0, vectors, numVectors * dimension); + JNIService.createIndex(ids, memoryAddress, dimension, path, parameters, knnEngine); // Setup mock resource manager ResourceWatcherService resourceWatcherService = mock(ResourceWatcherService.class); diff --git a/src/test/java/org/opensearch/knn/jni/JNICommonsTest.java b/src/test/java/org/opensearch/knn/jni/JNICommonsTest.java new file mode 100644 index 000000000..bf27458b0 --- /dev/null +++ b/src/test/java/org/opensearch/knn/jni/JNICommonsTest.java @@ -0,0 +1,40 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.knn.jni; + +import org.opensearch.knn.KNNTestCase; + +public class JNICommonsTest extends KNNTestCase { + + public void testStoreVectorData_whenVaildInputThenSuccess() { + float[][] data = new float[2][2]; + for (int i = 0; i < 2; i++) { + for (int j = 0; j < 2; j++) { + data[i][j] = i + j; + } + } + long memoryAddress = JNICommons.storeVectorData(0, data, 8); + assertTrue(memoryAddress > 0); + assertEquals(memoryAddress, JNICommons.storeVectorData(memoryAddress, data, 8)); + } + + public void testFreeVectorData_whenValidInput_ThenSuccess() { + float[][] data = new float[2][2]; + for (int i = 0; i < 2; i++) { + for (int j = 0; j < 2; j++) { + data[i][j] = i + j; + } + } + long memoryAddress = JNICommons.storeVectorData(0, data, 8); + JNICommons.freeVectorData(memoryAddress); + } +} diff --git a/src/test/java/org/opensearch/knn/jni/JNIServiceTests.java b/src/test/java/org/opensearch/knn/jni/JNIServiceTests.java index 36a3d93be..d6ae13e92 100644 --- a/src/test/java/org/opensearch/knn/jni/JNIServiceTests.java +++ b/src/test/java/org/opensearch/knn/jni/JNIServiceTests.java @@ -83,7 +83,8 @@ public void testCreateIndex_invalid_engineNotSupported() { IllegalArgumentException.class, () -> JNIService.createIndex( new int[] {}, - new float[][] {}, + 0, + 0, "test", ImmutableMap.of(KNNConstants.SPACE_TYPE, SpaceType.L2.getValue()), KNNEngine.LUCENE @@ -96,7 +97,8 @@ public void testCreateIndex_invalid_engineNull() { Exception.class, () -> JNIService.createIndex( new int[] {}, - new float[][] {}, + 0, + 0, "test", ImmutableMap.of(KNNConstants.SPACE_TYPE, SpaceType.L2.getValue()), null @@ -105,12 +107,12 @@ public void testCreateIndex_invalid_engineNull() { } public void testCreateIndex_nmslib_invalid_noSpaceType() { - expectThrows( Exception.class, () -> JNIService.createIndex( testData.indexData.docs, - testData.indexData.vectors, + testData.loadDataToMemoryAddress(), + testData.indexData.getDimension(), "something", Collections.emptyMap(), KNNEngine.NMSLIB @@ -122,13 +124,14 @@ public void testCreateIndex_nmslib_invalid_vectorDocIDMismatch() throws IOExcept int[] docIds = new int[] { 1, 2, 3 }; float[][] vectors1 = new float[][] { { 1, 2 }, { 3, 4 } }; - + long memoryAddress = JNICommons.storeVectorData(0, vectors1, vectors1.length * vectors1[0].length); Path tmpFile1 = createTempFile(); expectThrows( Exception.class, () -> JNIService.createIndex( docIds, - vectors1, + memoryAddress, + vectors1[0].length, tmpFile1.toAbsolutePath().toString(), ImmutableMap.of(KNNConstants.SPACE_TYPE, SpaceType.L2.getValue()), KNNEngine.NMSLIB @@ -136,13 +139,15 @@ public void testCreateIndex_nmslib_invalid_vectorDocIDMismatch() throws IOExcept ); float[][] vectors2 = new float[][] { { 1, 2 }, { 3, 4 }, { 4, 5 }, { 6, 7 }, { 8, 9 } }; + long memoryAddress2 = JNICommons.storeVectorData(0, vectors2, vectors2.length * vectors2[0].length); Path tmpFile2 = createTempFile(); expectThrows( Exception.class, () -> JNIService.createIndex( docIds, - vectors2, + memoryAddress2, + vectors2[0].length, tmpFile2.toAbsolutePath().toString(), ImmutableMap.of(KNNConstants.SPACE_TYPE, SpaceType.L2.getValue()), KNNEngine.NMSLIB @@ -154,13 +159,14 @@ public void testCreateIndex_nmslib_invalid_nullArgument() throws IOException { int[] docIds = new int[] {}; float[][] vectors = new float[][] {}; - + long memoryAddress = JNICommons.storeVectorData(0, vectors, vectors.length); Path tmpFile = createTempFile(); expectThrows( Exception.class, () -> JNIService.createIndex( null, - vectors, + memoryAddress, + 0, tmpFile.toAbsolutePath().toString(), ImmutableMap.of(KNNConstants.SPACE_TYPE, SpaceType.L2.getValue()), KNNEngine.NMSLIB @@ -171,7 +177,8 @@ public void testCreateIndex_nmslib_invalid_nullArgument() throws IOException { Exception.class, () -> JNIService.createIndex( docIds, - null, + 0, + 0, tmpFile.toAbsolutePath().toString(), ImmutableMap.of(KNNConstants.SPACE_TYPE, SpaceType.L2.getValue()), KNNEngine.NMSLIB @@ -182,7 +189,8 @@ public void testCreateIndex_nmslib_invalid_nullArgument() throws IOException { Exception.class, () -> JNIService.createIndex( docIds, - vectors, + memoryAddress, + 0, null, ImmutableMap.of(KNNConstants.SPACE_TYPE, SpaceType.L2.getValue()), KNNEngine.NMSLIB @@ -191,14 +199,15 @@ public void testCreateIndex_nmslib_invalid_nullArgument() throws IOException { expectThrows( Exception.class, - () -> JNIService.createIndex(docIds, vectors, tmpFile.toAbsolutePath().toString(), null, KNNEngine.NMSLIB) + () -> JNIService.createIndex(docIds, memoryAddress, 0, tmpFile.toAbsolutePath().toString(), null, KNNEngine.NMSLIB) ); expectThrows( Exception.class, () -> JNIService.createIndex( docIds, - vectors, + memoryAddress, + 0, tmpFile.toAbsolutePath().toString(), ImmutableMap.of(KNNConstants.SPACE_TYPE, SpaceType.L2.getValue()), null @@ -210,13 +219,14 @@ public void testCreateIndex_nmslib_invalid_badSpace() throws IOException { int[] docIds = new int[] { 1 }; float[][] vectors = new float[][] { { 2, 3 } }; - + long memoryAddress = JNICommons.storeVectorData(0, vectors, vectors.length * vectors[0].length); Path tmpFile = createTempFile(); expectThrows( Exception.class, () -> JNIService.createIndex( docIds, - vectors, + memoryAddress, + vectors[0].length, tmpFile.toAbsolutePath().toString(), ImmutableMap.of(KNNConstants.SPACE_TYPE, "invalid"), KNNEngine.NMSLIB @@ -224,28 +234,11 @@ public void testCreateIndex_nmslib_invalid_badSpace() throws IOException { ); } - public void testCreateIndex_nmslib_invalid_inconsistentDimensions() throws IOException { - - int[] docIds = new int[] { 1, 2 }; - float[][] vectors = new float[][] { { 2, 3 }, { 2, 3, 4 } }; - - Path tmpFile = createTempFile(); - expectThrows( - Exception.class, - () -> JNIService.createIndex( - docIds, - vectors, - tmpFile.toAbsolutePath().toString(), - ImmutableMap.of(KNNConstants.SPACE_TYPE, SpaceType.L2.getValue()), - KNNEngine.NMSLIB - ) - ); - } - public void testCreateIndex_nmslib_invalid_badParameterType() throws IOException { - int[] docIds = new int[] {}; - float[][] vectors = new float[][] {}; + int[] docIds = new int[] { 1 }; + float[][] vectors = new float[][] { { 2, 3 } }; + long memoryAddress = JNICommons.storeVectorData(0, vectors, vectors.length * vectors[0].length); Map parametersMap = ImmutableMap.of( KNNConstants.HNSW_ALGO_EF_CONSTRUCTION, @@ -258,7 +251,8 @@ public void testCreateIndex_nmslib_invalid_badParameterType() throws IOException Exception.class, () -> JNIService.createIndex( docIds, - vectors, + memoryAddress, + vectors[0].length, tmpFile.toAbsolutePath().toString(), ImmutableMap.of(KNNConstants.SPACE_TYPE, SpaceType.L2.getValue(), KNNConstants.PARAMETERS, parametersMap), KNNEngine.NMSLIB @@ -273,7 +267,8 @@ public void testCreateIndex_nmslib_valid() throws IOException { JNIService.createIndex( testData.indexData.docs, - testData.indexData.vectors, + testData.loadDataToMemoryAddress(), + testData.indexData.getDimension(), tmpFile.toAbsolutePath().toString(), ImmutableMap.of(KNNConstants.SPACE_TYPE, spaceType.getValue()), KNNEngine.NMSLIB @@ -284,7 +279,8 @@ public void testCreateIndex_nmslib_valid() throws IOException { JNIService.createIndex( testData.indexData.docs, - testData.indexData.vectors, + testData.loadDataToMemoryAddress(), + testData.indexData.getDimension(), tmpFile.toAbsolutePath().toString(), ImmutableMap.of( KNNConstants.SPACE_TYPE, @@ -301,15 +297,14 @@ public void testCreateIndex_nmslib_valid() throws IOException { } public void testCreateIndex_faiss_invalid_noSpaceType() { - int[] docIds = new int[] {}; - float[][] vectors = new float[][] {}; expectThrows( Exception.class, () -> JNIService.createIndex( docIds, - vectors, + testData.loadDataToMemoryAddress(), + testData.indexData.getDimension(), "something", ImmutableMap.of(INDEX_DESCRIPTION_PARAMETER, faissMethod), KNNEngine.FAISS @@ -321,13 +316,14 @@ public void testCreateIndex_faiss_invalid_vectorDocIDMismatch() throws IOExcepti int[] docIds = new int[] { 1, 2, 3 }; float[][] vectors1 = new float[][] { { 1, 2 }, { 3, 4 } }; - + long memoryAddress = JNICommons.storeVectorData(0, vectors1, vectors1.length * vectors1[0].length); Path tmpFile1 = createTempFile(); expectThrows( Exception.class, () -> JNIService.createIndex( docIds, - vectors1, + memoryAddress, + vectors1[0].length, tmpFile1.toAbsolutePath().toString(), ImmutableMap.of(INDEX_DESCRIPTION_PARAMETER, faissMethod, KNNConstants.SPACE_TYPE, SpaceType.L2.getValue()), KNNEngine.FAISS @@ -335,13 +331,14 @@ public void testCreateIndex_faiss_invalid_vectorDocIDMismatch() throws IOExcepti ); float[][] vectors2 = new float[][] { { 1, 2 }, { 3, 4 }, { 4, 5 }, { 6, 7 }, { 8, 9 } }; - + long memoryAddress2 = JNICommons.storeVectorData(0, vectors2, vectors2.length * vectors2[0].length); Path tmpFile2 = createTempFile(); expectThrows( Exception.class, () -> JNIService.createIndex( docIds, - vectors2, + memoryAddress, + vectors2[0].length, tmpFile2.toAbsolutePath().toString(), ImmutableMap.of(INDEX_DESCRIPTION_PARAMETER, faissMethod, KNNConstants.SPACE_TYPE, SpaceType.L2.getValue()), KNNEngine.FAISS @@ -353,13 +350,15 @@ public void testCreateIndex_faiss_invalid_null() throws IOException { int[] docIds = new int[] {}; float[][] vectors = new float[][] {}; + long memoryAddress = JNICommons.storeVectorData(0, vectors, 0); Path tmpFile = createTempFile(); expectThrows( Exception.class, () -> JNIService.createIndex( null, - vectors, + memoryAddress, + 0, tmpFile.toAbsolutePath().toString(), ImmutableMap.of(INDEX_DESCRIPTION_PARAMETER, faissMethod, KNNConstants.SPACE_TYPE, SpaceType.L2.getValue()), KNNEngine.FAISS @@ -370,7 +369,8 @@ public void testCreateIndex_faiss_invalid_null() throws IOException { Exception.class, () -> JNIService.createIndex( docIds, - null, + 0, + 0, tmpFile.toAbsolutePath().toString(), ImmutableMap.of(INDEX_DESCRIPTION_PARAMETER, faissMethod, KNNConstants.SPACE_TYPE, SpaceType.L2.getValue()), KNNEngine.FAISS @@ -381,7 +381,8 @@ public void testCreateIndex_faiss_invalid_null() throws IOException { Exception.class, () -> JNIService.createIndex( docIds, - vectors, + testData.loadDataToMemoryAddress(), + testData.indexData.getDimension(), null, ImmutableMap.of(INDEX_DESCRIPTION_PARAMETER, faissMethod, KNNConstants.SPACE_TYPE, SpaceType.L2.getValue()), KNNEngine.FAISS @@ -390,14 +391,22 @@ public void testCreateIndex_faiss_invalid_null() throws IOException { expectThrows( Exception.class, - () -> JNIService.createIndex(docIds, vectors, tmpFile.toAbsolutePath().toString(), null, KNNEngine.FAISS) + () -> JNIService.createIndex( + docIds, + testData.loadDataToMemoryAddress(), + testData.indexData.getDimension(), + tmpFile.toAbsolutePath().toString(), + null, + KNNEngine.FAISS + ) ); expectThrows( Exception.class, () -> JNIService.createIndex( docIds, - vectors, + testData.loadDataToMemoryAddress(), + testData.indexData.getDimension(), tmpFile.toAbsolutePath().toString(), ImmutableMap.of(INDEX_DESCRIPTION_PARAMETER, faissMethod, KNNConstants.SPACE_TYPE, SpaceType.L2.getValue()), null @@ -409,13 +418,15 @@ public void testCreateIndex_faiss_invalid_invalidSpace() throws IOException { int[] docIds = new int[] { 1 }; float[][] vectors = new float[][] { { 2, 3 } }; + long memoryAddress = JNICommons.storeVectorData(0, vectors, (long) vectors.length * vectors[0].length); Path tmpFile = createTempFile(); expectThrows( Exception.class, () -> JNIService.createIndex( docIds, - vectors, + memoryAddress, + vectors[0].length, tmpFile.toAbsolutePath().toString(), ImmutableMap.of(INDEX_DESCRIPTION_PARAMETER, faissMethod, KNNConstants.SPACE_TYPE, "invalid"), KNNEngine.FAISS @@ -423,35 +434,19 @@ public void testCreateIndex_faiss_invalid_invalidSpace() throws IOException { ); } - public void testCreateIndex_faiss_invalid_inconsistentDimensions() throws IOException { - - int[] docIds = new int[] { 1, 2 }; - float[][] vectors = new float[][] { { 2, 3 }, { 2, 3, 4 } }; - - Path tmpFile = createTempFile(); - expectThrows( - Exception.class, - () -> JNIService.createIndex( - docIds, - vectors, - tmpFile.toAbsolutePath().toString(), - ImmutableMap.of(INDEX_DESCRIPTION_PARAMETER, faissMethod, KNNConstants.SPACE_TYPE, SpaceType.L2.getValue()), - KNNEngine.FAISS - ) - ); - } - public void testCreateIndex_faiss_invalid_noIndexDescription() throws IOException { int[] docIds = new int[] { 1, 2 }; - float[][] vectors = new float[][] { { 2, 3 }, { 2, 3, 4 } }; + float[][] vectors = new float[][] { { 2, 3 }, { 2, 3 } }; + long memoryAddress = JNICommons.storeVectorData(0, vectors, (long) vectors.length * vectors[0].length); Path tmpFile = createTempFile(); expectThrows( Exception.class, () -> JNIService.createIndex( docIds, - vectors, + memoryAddress, + vectors[0].length, tmpFile.toAbsolutePath().toString(), ImmutableMap.of(KNNConstants.SPACE_TYPE, SpaceType.L2.getValue()), KNNEngine.FAISS @@ -460,16 +455,16 @@ public void testCreateIndex_faiss_invalid_noIndexDescription() throws IOExceptio } public void testCreateIndex_faiss_invalid_invalidIndexDescription() throws IOException { - int[] docIds = new int[] { 1, 2 }; - float[][] vectors = new float[][] { { 2, 3 }, { 2, 3, 4 } }; - + float[][] vectors = new float[][] { { 2, 3 }, { 2, 3 } }; + long memoryAddress = JNICommons.storeVectorData(0, vectors, (long) vectors.length * vectors[0].length); Path tmpFile = createTempFile(); expectThrows( Exception.class, () -> JNIService.createIndex( docIds, - vectors, + memoryAddress, + vectors[0].length, tmpFile.toAbsolutePath().toString(), ImmutableMap.of(INDEX_DESCRIPTION_PARAMETER, "invalid", KNNConstants.SPACE_TYPE, SpaceType.L2.getValue()), KNNEngine.FAISS @@ -482,6 +477,8 @@ public void testCreateIndex_faiss_sqfp16_invalidIndexDescription() { int[] docIds = new int[] { 1, 2 }; float[][] vectors = new float[][] { { 2, 3 }, { 3, 4 } }; + long memoryAddress = JNICommons.storeVectorData(0, vectors, (long) vectors.length * vectors[0].length); + String sqfp16InvalidIndexDescription = "HNSW16,SQfp1655"; Path tmpFile = createTempFile(); @@ -489,7 +486,8 @@ public void testCreateIndex_faiss_sqfp16_invalidIndexDescription() { Exception.class, () -> JNIService.createIndex( docIds, - vectors, + memoryAddress, + vectors[0].length, tmpFile.toAbsolutePath().toString(), ImmutableMap.of( INDEX_DESCRIPTION_PARAMETER, @@ -508,11 +506,12 @@ public void testLoadIndex_faiss_sqfp16_valid() { int[] docIds = new int[] { 1, 2 }; float[][] vectors = new float[][] { { 2, 3 }, { 3, 4 } }; String sqfp16IndexDescription = "HNSW16,SQfp16"; - + long memoryAddress = JNICommons.storeVectorData(0, vectors, (long) vectors.length * vectors[0].length); Path tmpFile = createTempFile(); JNIService.createIndex( docIds, - vectors, + memoryAddress, + vectors[0].length, tmpFile.toAbsolutePath().toString(), ImmutableMap.of(INDEX_DESCRIPTION_PARAMETER, sqfp16IndexDescription, KNNConstants.SPACE_TYPE, SpaceType.L2.getValue()), KNNEngine.FAISS @@ -528,11 +527,13 @@ public void testQueryIndex_faiss_sqfp16_valid() { String sqfp16IndexDescription = "HNSW16,SQfp16"; int k = 10; - + float[][] truncatedVectors = truncateToFp16Range(testData.indexData.vectors); + long memoryAddress = JNICommons.storeVectorData(0, truncatedVectors, (long) truncatedVectors.length * truncatedVectors[0].length); Path tmpFile = createTempFile(); JNIService.createIndex( testData.indexData.docs, - truncateToFp16Range(testData.indexData.vectors), + memoryAddress, + testData.indexData.getDimension(), tmpFile.toAbsolutePath().toString(), ImmutableMap.of(INDEX_DESCRIPTION_PARAMETER, sqfp16IndexDescription, KNNConstants.SPACE_TYPE, SpaceType.L2.getValue()), KNNEngine.FAISS @@ -596,7 +597,7 @@ public void testTrain_whenConfigurationIsIVFSQFP16_thenSucceed() { byte[] faissIndex = JNIService.trainIndex(parameters, 128, trainPointer, KNNEngine.FAISS); assertNotEquals(0, faissIndex.length); - JNIService.freeVectors(trainPointer); + JNICommons.freeVectorData(trainPointer); } public void testCreateIndex_faiss_invalid_invalidParameterType() throws IOException { @@ -609,7 +610,8 @@ public void testCreateIndex_faiss_invalid_invalidParameterType() throws IOExcept Exception.class, () -> JNIService.createIndex( docIds, - vectors, + testData.loadDataToMemoryAddress(), + testData.indexData.getDimension(), tmpFile.toAbsolutePath().toString(), ImmutableMap.of( INDEX_DESCRIPTION_PARAMETER, @@ -634,7 +636,8 @@ public void testCreateIndex_faiss_valid() throws IOException { Path tmpFile1 = createTempFile(); JNIService.createIndex( testData.indexData.docs, - testData.indexData.vectors, + testData.loadDataToMemoryAddress(), + testData.indexData.getDimension(), tmpFile1.toAbsolutePath().toString(), ImmutableMap.of(INDEX_DESCRIPTION_PARAMETER, method, KNNConstants.SPACE_TYPE, spaceType.getValue()), KNNEngine.FAISS @@ -684,7 +687,8 @@ public void testLoadIndex_nmslib_valid() throws IOException { JNIService.createIndex( testData.indexData.docs, - testData.indexData.vectors, + testData.loadDataToMemoryAddress(), + testData.indexData.getDimension(), tmpFile.toAbsolutePath().toString(), ImmutableMap.of(KNNConstants.SPACE_TYPE, SpaceType.L2.getValue()), KNNEngine.NMSLIB @@ -719,7 +723,8 @@ public void testLoadIndex_faiss_valid() throws IOException { JNIService.createIndex( testData.indexData.docs, - testData.indexData.vectors, + testData.loadDataToMemoryAddress(), + testData.indexData.getDimension(), tmpFile.toAbsolutePath().toString(), ImmutableMap.of(INDEX_DESCRIPTION_PARAMETER, faissMethod, KNNConstants.SPACE_TYPE, SpaceType.L2.getValue()), KNNEngine.FAISS @@ -745,7 +750,8 @@ public void testQueryIndex_nmslib_invalid_nullQueryVector() throws IOException { JNIService.createIndex( testData.indexData.docs, - testData.indexData.vectors, + testData.loadDataToMemoryAddress(), + testData.indexData.getDimension(), tmpFile.toAbsolutePath().toString(), ImmutableMap.of(KNNConstants.SPACE_TYPE, SpaceType.L2.getValue()), KNNEngine.NMSLIB @@ -770,7 +776,8 @@ public void testQueryIndex_nmslib_valid() throws IOException { JNIService.createIndex( testData.indexData.docs, - testData.indexData.vectors, + testData.loadDataToMemoryAddress(), + testData.indexData.getDimension(), tmpFile.toAbsolutePath().toString(), ImmutableMap.of(KNNConstants.SPACE_TYPE, spaceType.getValue()), KNNEngine.NMSLIB @@ -802,7 +809,8 @@ public void testQueryIndex_faiss_invalid_nullQueryVector() throws IOException { JNIService.createIndex( testData.indexData.docs, - testData.indexData.vectors, + testData.loadDataToMemoryAddress(), + testData.indexData.getDimension(), tmpFile.toAbsolutePath().toString(), ImmutableMap.of(INDEX_DESCRIPTION_PARAMETER, faissMethod, KNNConstants.SPACE_TYPE, SpaceType.L2.getValue()), KNNEngine.FAISS @@ -826,7 +834,8 @@ public void testQueryIndex_faiss_valid() throws IOException { Path tmpFile = createTempFile(); JNIService.createIndex( testData.indexData.docs, - testData.indexData.vectors, + testData.loadDataToMemoryAddress(), + testData.indexData.getDimension(), tmpFile.toAbsolutePath().toString(), ImmutableMap.of(INDEX_DESCRIPTION_PARAMETER, method, KNNConstants.SPACE_TYPE, spaceType.getValue()), KNNEngine.FAISS @@ -867,7 +876,8 @@ public void testQueryIndex_faiss_parentIds() throws IOException { Path tmpFile = createTempFile(); JNIService.createIndex( testDataNested.indexData.docs, - testDataNested.indexData.vectors, + testData.loadDataToMemoryAddress(), + testDataNested.indexData.getDimension(), tmpFile.toAbsolutePath().toString(), ImmutableMap.of(INDEX_DESCRIPTION_PARAMETER, method, KNNConstants.SPACE_TYPE, spaceType.getValue()), KNNEngine.FAISS @@ -941,7 +951,8 @@ public void testFree_nmslib_valid() throws IOException { JNIService.createIndex( testData.indexData.docs, - testData.indexData.vectors, + testData.loadDataToMemoryAddress(), + testData.indexData.getDimension(), tmpFile.toAbsolutePath().toString(), ImmutableMap.of(KNNConstants.SPACE_TYPE, SpaceType.L2.getValue()), KNNEngine.NMSLIB @@ -964,7 +975,8 @@ public void testFree_faiss_valid() throws IOException { JNIService.createIndex( testData.indexData.docs, - testData.indexData.vectors, + testData.loadDataToMemoryAddress(), + testData.indexData.getDimension(), tmpFile.toAbsolutePath().toString(), ImmutableMap.of(INDEX_DESCRIPTION_PARAMETER, faissMethod, KNNConstants.SPACE_TYPE, SpaceType.L2.getValue()), KNNEngine.FAISS @@ -987,7 +999,7 @@ public void testTransferVectors() { assertEquals(trainPointer1, trainPointer2); } - JNIService.freeVectors(trainPointer1); + JNICommons.freeVectorData(trainPointer1); } public void testTrain_whenConfigurationIsIVFFlat_thenSucceed() throws IOException { @@ -1008,7 +1020,7 @@ public void testTrain_whenConfigurationIsIVFFlat_thenSucceed() throws IOExceptio byte[] faissIndex = JNIService.trainIndex(parameters, 128, trainPointer, KNNEngine.FAISS); assertNotEquals(0, faissIndex.length); - JNIService.freeVectors(trainPointer); + JNICommons.freeVectorData(trainPointer); } public void testTrain_whenConfigurationIsIVFPQ_thenSucceed() throws IOException { @@ -1038,7 +1050,7 @@ public void testTrain_whenConfigurationIsIVFPQ_thenSucceed() throws IOException byte[] faissIndex = JNIService.trainIndex(parameters, 128, trainPointer, KNNEngine.FAISS); assertNotEquals(0, faissIndex.length); - JNIService.freeVectors(trainPointer); + JNICommons.freeVectorData(trainPointer); } public void testTrain_whenConfigurationIsHNSWPQ_thenSucceed() throws IOException { @@ -1065,7 +1077,7 @@ public void testTrain_whenConfigurationIsHNSWPQ_thenSucceed() throws IOException byte[] faissIndex = JNIService.trainIndex(parameters, 128, trainPointer, KNNEngine.FAISS); assertNotEquals(0, faissIndex.length); - JNIService.freeVectors(trainPointer); + JNICommons.freeVectorData(trainPointer); } private long transferVectors(int numDuplicates) { @@ -1120,12 +1132,13 @@ public void testCreateIndexFromTemplate() throws IOException { byte[] faissIndex = JNIService.trainIndex(parameters, 128, trainPointer1, KNNEngine.FAISS); assertNotEquals(0, faissIndex.length); - JNIService.freeVectors(trainPointer1); + JNICommons.freeVectorData(trainPointer1); Path tmpFile1 = createTempFile(); JNIService.createIndexFromTemplate( testData.indexData.docs, - testData.indexData.vectors, + testData.loadDataToMemoryAddress(), + testData.indexData.getDimension(), tmpFile1.toAbsolutePath().toString(), faissIndex, ImmutableMap.of(INDEX_THREAD_QTY, 1), @@ -1255,11 +1268,12 @@ private String createFaissIVFPQIndex(int ivfNlist, int pqM, int pqCodeSize, Spac byte[] faissIndex = JNIService.trainIndex(parameters, 128, trainPointer, KNNEngine.FAISS); assertNotEquals(0, faissIndex.length); - JNIService.freeVectors(trainPointer); + JNICommons.freeVectorData(trainPointer); Path tmpFile = createTempFile(); JNIService.createIndexFromTemplate( testData.indexData.docs, - testData.indexData.vectors, + testData.loadDataToMemoryAddress(), + testData.indexData.getDimension(), tmpFile.toAbsolutePath().toString(), faissIndex, ImmutableMap.of(INDEX_THREAD_QTY, 1), @@ -1274,7 +1288,8 @@ private String createFaissHNSWIndex(SpaceType spaceType) throws IOException { Path tmpFile = createTempFile(); JNIService.createIndex( testData.indexData.docs, - testData.indexData.vectors, + testData.loadDataToMemoryAddress(), + testData.indexData.getDimension(), tmpFile.toAbsolutePath().toString(), ImmutableMap.of(INDEX_DESCRIPTION_PARAMETER, faissMethod, KNNConstants.SPACE_TYPE, spaceType.getValue()), KNNEngine.FAISS diff --git a/src/test/java/org/opensearch/knn/training/TrainingJobTests.java b/src/test/java/org/opensearch/knn/training/TrainingJobTests.java index e4c01a8ec..06b96c57c 100644 --- a/src/test/java/org/opensearch/knn/training/TrainingJobTests.java +++ b/src/test/java/org/opensearch/knn/training/TrainingJobTests.java @@ -25,6 +25,7 @@ import org.opensearch.knn.indices.Model; import org.opensearch.knn.indices.ModelMetadata; import org.opensearch.knn.indices.ModelState; +import org.opensearch.knn.jni.JNICommons; import org.opensearch.knn.jni.JNIService; import java.io.File; @@ -170,7 +171,7 @@ public void testRun_success() throws IOException, ExecutionException { when(nativeMemoryCacheManager.get(trainingDataEntryContext, false)).thenReturn(nativeMemoryAllocation); doAnswer(invocationOnMock -> { - JNIService.freeVectors(memoryAddress); + JNICommons.freeVectorData(memoryAddress); return null; }).when(nativeMemoryCacheManager).invalidate(tdataKey); @@ -197,11 +198,12 @@ public void testRun_success() throws IOException, ExecutionException { int[] ids = { 1, 2, 3, 4 }; float[][] vectors = new float[ids.length][dimension]; fillFloatArrayRandomly(vectors); - + long vectorsMemoryAddress = JNICommons.storeVectorData(0, vectors, (long) vectors.length * vectors[0].length); Path indexPath = createTempFile(); JNIService.createIndexFromTemplate( ids, - vectors, + vectorsMemoryAddress, + vectors[0].length, indexPath.toString(), model.getModelBlob(), ImmutableMap.of(INDEX_THREAD_QTY, 1), @@ -456,7 +458,7 @@ public void testRun_failure_notEnoughTrainingData() throws ExecutionException { when(nativeMemoryCacheManager.get(trainingDataEntryContext, false)).thenReturn(nativeMemoryAllocation); doAnswer(invocationOnMock -> { - JNIService.freeVectors(memoryAddress); + JNICommons.freeVectorData(memoryAddress); return null; }).when(nativeMemoryCacheManager).invalidate(tdataKey); diff --git a/src/testFixtures/java/org/opensearch/knn/TestUtils.java b/src/testFixtures/java/org/opensearch/knn/TestUtils.java index 1b5accae9..37e35f062 100644 --- a/src/testFixtures/java/org/opensearch/knn/TestUtils.java +++ b/src/testFixtures/java/org/opensearch/knn/TestUtils.java @@ -9,12 +9,15 @@ import org.opensearch.core.xcontent.DeprecationHandler; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.common.xcontent.XContentType; -import org.opensearch.knn.index.codec.util.KNNCodecUtil; +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.Setter; import java.io.BufferedReader; import java.io.FileReader; import java.io.IOException; import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.codec.util.SerializationMode; +import org.opensearch.knn.jni.JNICommons; import org.opensearch.knn.plugin.script.KNNScoringUtil; import java.util.Comparator; import java.util.Random; @@ -246,7 +249,7 @@ public static PriorityQueue insertWithOverflow(PriorityQueue idsList = new ArrayList<>(); List vectorsList = new ArrayList<>(); @@ -287,8 +290,7 @@ private KNNCodecUtil.Pair readIndexData(String path) throws IOException { vectorsArray[i][j] = vectorsList.get(i)[j]; } } - - return new KNNCodecUtil.Pair(idsArray, vectorsArray, SerializationMode.COLLECTION_OF_FLOATS); + return new Pair(idsArray, vectorsArray[0].length, SerializationMode.COLLECTION_OF_FLOATS, vectorsArray); } private float[][] readQueries(String path) throws IOException { @@ -319,5 +321,19 @@ private float[][] readQueries(String path) throws IOException { } return queryArray; } + + public long loadDataToMemoryAddress() { + return JNICommons.storeVectorData(0, indexData.vectors, (long) indexData.vectors.length * indexData.vectors[0].length); + } + + @AllArgsConstructor + public static class Pair { + public int[] docs; + @Getter + @Setter + private int dimension; + public SerializationMode serializationMode; + public float[][] vectors; + } } } From cee100f73952551fb24e730237b7310f69a4f884 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Fri, 12 Apr 2024 11:27:56 -0700 Subject: [PATCH 235/416] Remove unnecessary toString conversion of vector field and added some minor optimization in KNNCodec (#1613) (#1615) Signed-off-by: Navneet Verma (cherry picked from commit b48ab3020d75dcf3473fe151f0db813d00bba68b) Co-authored-by: Navneet Verma --- CHANGELOG.md | 1 + .../opensearch/knn/index/codec/util/KNNCodecUtil.java | 5 +++++ .../knn/index/mapper/KNNVectorFieldMapper.java | 4 ++-- .../knn/index/mapper/KNNVectorFieldMapperUtil.java | 10 +++------- .../opensearch/knn/index/mapper/LuceneFieldMapper.java | 4 ++-- 5 files changed, 13 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 65e39b77b..d46a9c41b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), * Make the HitQueue size more appropriate for exact search [#1549](https://github.com/opensearch-project/k-NN/pull/1549) * Support script score when doc value is disabled [#1573](https://github.com/opensearch-project/k-NN/pull/1573) * Implemented the Streaming Feature to stream vectors from Java to JNI layer to enable creation of larger segments for vector indices [#1604](https://github.com/opensearch-project/k-NN/pull/1604) +* Remove unnecessary toString conversion of vector field and added some minor optimization in KNNCodec [1613](https://github.com/opensearch-project/k-NN/pull/1613) ### Bug Fixes ### Infrastructure * Add micro-benchmark module in k-NN plugin for benchmark streaming vectors to JNI layer functionality. [#1583](https://github.com/opensearch-project/k-NN/pull/1583) diff --git a/src/main/java/org/opensearch/knn/index/codec/util/KNNCodecUtil.java b/src/main/java/org/opensearch/knn/index/codec/util/KNNCodecUtil.java index c5ae469e0..e05962608 100644 --- a/src/main/java/org/opensearch/knn/index/codec/util/KNNCodecUtil.java +++ b/src/main/java/org/opensearch/knn/index/codec/util/KNNCodecUtil.java @@ -64,6 +64,11 @@ public static KNNCodecUtil.Pair getFloats(BinaryDocValues values) throws IOExcep if (vectorsPerTransfer == Integer.MIN_VALUE) { vectorsPerTransfer = (dimension * Float.BYTES * totalLiveDocs) / vectorsStreamingMemoryLimit; + // This condition comes if vectorsStreamingMemoryLimit is higher than total number floats to transfer + // Doing this will reduce 1 extra trip to JNI layer. + if (vectorsPerTransfer == 0) { + vectorsPerTransfer = totalLiveDocs; + } } if (vectorList.size() == vectorsPerTransfer) { vectorAddress = JNICommons.storeVectorData( diff --git a/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapper.java b/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapper.java index a36a4222b..0fa026f34 100644 --- a/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapper.java +++ b/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapper.java @@ -561,7 +561,7 @@ protected void parseCreateField(ParseContext context, int dimension, SpaceType s VectorField point = new VectorField(name(), array, fieldType); context.doc().add(point); - addStoredFieldForVectorField(context, fieldType, name(), point.toString()); + addStoredFieldForVectorField(context, fieldType, name(), point); } else if (VectorDataType.FLOAT == vectorDataType) { Optional floatsArrayOptional = getFloatsFromContext(context, dimension, methodComponentContext); @@ -572,7 +572,7 @@ protected void parseCreateField(ParseContext context, int dimension, SpaceType s spaceType.validateVector(array); VectorField point = new VectorField(name(), array, fieldType); context.doc().add(point); - addStoredFieldForVectorField(context, fieldType, name(), point.toString()); + addStoredFieldForVectorField(context, fieldType, name(), point); } else { throw new IllegalArgumentException( String.format(Locale.ROOT, "Cannot parse context for unsupported values provided for field [%s]", VECTOR_DATA_TYPE_FIELD) diff --git a/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperUtil.java b/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperUtil.java index 283d35f00..074be0375 100644 --- a/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperUtil.java +++ b/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperUtil.java @@ -13,6 +13,7 @@ import lombok.AccessLevel; import lombok.NoArgsConstructor; +import org.apache.lucene.document.Field; import org.apache.lucene.document.FieldType; import org.apache.lucene.document.StoredField; import org.apache.lucene.index.DocValuesType; @@ -135,14 +136,9 @@ public static FieldType buildDocValuesFieldType(KNNEngine knnEngine) { return field; } - public static void addStoredFieldForVectorField( - ParseContext context, - FieldType fieldType, - String mapperName, - String vectorFieldAsString - ) { + public static void addStoredFieldForVectorField(ParseContext context, FieldType fieldType, String mapperName, Field vectorField) { if (fieldType.stored()) { - context.doc().add(new StoredField(mapperName, vectorFieldAsString)); + context.doc().add(new StoredField(mapperName, vectorField.toString())); } } } diff --git a/src/main/java/org/opensearch/knn/index/mapper/LuceneFieldMapper.java b/src/main/java/org/opensearch/knn/index/mapper/LuceneFieldMapper.java index 185ab3dc4..d61fa1150 100644 --- a/src/main/java/org/opensearch/knn/index/mapper/LuceneFieldMapper.java +++ b/src/main/java/org/opensearch/knn/index/mapper/LuceneFieldMapper.java @@ -92,7 +92,7 @@ protected void parseCreateField(ParseContext context, int dimension, SpaceType s KnnByteVectorField point = new KnnByteVectorField(name(), array, fieldType); context.doc().add(point); - addStoredFieldForVectorField(context, fieldType, name(), point.toString()); + addStoredFieldForVectorField(context, fieldType, name(), point); if (hasDocValues && vectorFieldType != null) { context.doc().add(new VectorField(name(), array, vectorFieldType)); @@ -108,7 +108,7 @@ protected void parseCreateField(ParseContext context, int dimension, SpaceType s KnnVectorField point = new KnnVectorField(name(), array, fieldType); context.doc().add(point); - addStoredFieldForVectorField(context, fieldType, name(), point.toString()); + addStoredFieldForVectorField(context, fieldType, name(), point); if (hasDocValues && vectorFieldType != null) { context.doc().add(new VectorField(name(), array, vectorFieldType)); From 78a489a6014a01435f8341c8d5986a1594a55b8c Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Fri, 12 Apr 2024 20:22:44 -0500 Subject: [PATCH 236/416] [Backport 2.x] [k-NN] Add Clear Cache API (#1055) * [k-NN] Add Clear Cache API (#740) * Add Clear Cache API Signed-off-by: Naveen Tatikonda * Add Unit and Integration tests Signed-off-by: Naveen Tatikonda * Add BWC Tests Signed-off-by: Naveen Tatikonda * Add CHANGELOG Signed-off-by: Naveen Tatikonda * Address Review Comments Signed-off-by: Naveen Tatikonda --------- Signed-off-by: Naveen Tatikonda (cherry picked from commit 12f4a51289cc2e64c62cbe3ab888e9c16ae8bb1d) Signed-off-by: Naveen Tatikonda * Fix Failing Clear Cache API Test (#1336) Signed-off-by: Naveen Tatikonda --------- Signed-off-by: Naveen Tatikonda Co-authored-by: Naveen Tatikonda --- CHANGELOG.md | 1 + .../org/opensearch/knn/bwc/ClearCacheIT.java | 49 ++++++ .../org/opensearch/knn/bwc/ClearCacheIT.java | 55 ++++++ .../opensearch/knn/common/KNNConstants.java | 3 + .../opensearch/knn/index/KNNIndexShard.java | 39 ++++- .../memory/NativeMemoryCacheManager.java | 18 ++ .../org/opensearch/knn/plugin/KNNPlugin.java | 10 +- .../plugin/rest/RestClearCacheHandler.java | 98 +++++++++++ .../plugin/transport/ClearCacheAction.java | 27 +++ .../plugin/transport/ClearCacheRequest.java | 36 ++++ .../plugin/transport/ClearCacheResponse.java | 49 ++++++ .../transport/ClearCacheTransportAction.java | 164 ++++++++++++++++++ .../opensearch/knn/KNNSingleNodeTestCase.java | 30 ++++ .../knn/index/KNNIndexShardTests.java | 27 +++ .../action/RestClearCacheHandlerIT.java | 111 ++++++++++++ .../ClearCacheTransportActionTests.java | 117 +++++++++++++ .../org/opensearch/knn/KNNRestTestCase.java | 15 ++ 17 files changed, 842 insertions(+), 7 deletions(-) create mode 100644 qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/ClearCacheIT.java create mode 100644 qa/rolling-upgrade/src/test/java/org/opensearch/knn/bwc/ClearCacheIT.java create mode 100644 src/main/java/org/opensearch/knn/plugin/rest/RestClearCacheHandler.java create mode 100644 src/main/java/org/opensearch/knn/plugin/transport/ClearCacheAction.java create mode 100644 src/main/java/org/opensearch/knn/plugin/transport/ClearCacheRequest.java create mode 100644 src/main/java/org/opensearch/knn/plugin/transport/ClearCacheResponse.java create mode 100644 src/main/java/org/opensearch/knn/plugin/transport/ClearCacheTransportAction.java create mode 100644 src/test/java/org/opensearch/knn/plugin/action/RestClearCacheHandlerIT.java create mode 100644 src/test/java/org/opensearch/knn/plugin/transport/ClearCacheTransportActionTests.java diff --git a/CHANGELOG.md b/CHANGELOG.md index d46a9c41b..e3c07fc1d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ## [Unreleased 2.x](https://github.com/opensearch-project/k-NN/compare/2.13...2.x) ### Features +* Add Clear Cache API [#740](https://github.com/opensearch-project/k-NN/pull/740) ### Enhancements * Make the HitQueue size more appropriate for exact search [#1549](https://github.com/opensearch-project/k-NN/pull/1549) * Support script score when doc value is disabled [#1573](https://github.com/opensearch-project/k-NN/pull/1573) diff --git a/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/ClearCacheIT.java b/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/ClearCacheIT.java new file mode 100644 index 000000000..045821e09 --- /dev/null +++ b/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/ClearCacheIT.java @@ -0,0 +1,49 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.bwc; + +import java.util.Collections; +import static org.opensearch.knn.TestUtils.NODES_BWC_CLUSTER; + +public class ClearCacheIT extends AbstractRestartUpgradeTestCase { + private static final String TEST_FIELD = "test-field"; + private static final int DIMENSIONS = 5; + private static int docId = 0; + private static final int NUM_DOCS = 10; + private static int queryCnt = 0; + private static final int K = 5; + + // Restart Upgrade BWC Tests to validate Clear Cache API + public void testClearCache() throws Exception { + waitForClusterHealthGreen(NODES_BWC_CLUSTER); + if (isRunningAgainstOldCluster()) { + createKnnIndex(testIndex, getKNNDefaultIndexSettings(), createKnnIndexMapping(TEST_FIELD, DIMENSIONS)); + addKNNDocs(testIndex, TEST_FIELD, DIMENSIONS, docId, NUM_DOCS); + } else { + queryCnt = NUM_DOCS; + validateClearCacheOnUpgrade(queryCnt); + + docId = NUM_DOCS; + addKNNDocs(testIndex, TEST_FIELD, DIMENSIONS, docId, NUM_DOCS); + + queryCnt = queryCnt + NUM_DOCS; + validateClearCacheOnUpgrade(queryCnt); + deleteKNNIndex(testIndex); + } + } + + // validation steps for Clear Cache API after upgrading node to new version + private void validateClearCacheOnUpgrade(int queryCount) throws Exception { + int graphCount = getTotalGraphsInCache(); + knnWarmup(Collections.singletonList(testIndex)); + assertTrue(getTotalGraphsInCache() > graphCount); + validateKNNSearch(testIndex, TEST_FIELD, DIMENSIONS, queryCount, K); + + clearCache(Collections.singletonList(testIndex)); + assertEquals(0, getTotalGraphsInCache()); + validateKNNSearch(testIndex, TEST_FIELD, DIMENSIONS, queryCount, K); + } +} diff --git a/qa/rolling-upgrade/src/test/java/org/opensearch/knn/bwc/ClearCacheIT.java b/qa/rolling-upgrade/src/test/java/org/opensearch/knn/bwc/ClearCacheIT.java new file mode 100644 index 000000000..24c674d0d --- /dev/null +++ b/qa/rolling-upgrade/src/test/java/org/opensearch/knn/bwc/ClearCacheIT.java @@ -0,0 +1,55 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.bwc; + +import java.util.Collections; + +import static org.opensearch.knn.TestUtils.NODES_BWC_CLUSTER; + +public class ClearCacheIT extends AbstractRollingUpgradeTestCase { + private static final String TEST_FIELD = "test-field"; + private static final int DIMENSIONS = 5; + private static int docId = 0; + private static final int K = 5; + private static final int NUM_DOCS = 10; + private static int queryCnt = 0; + + // Rolling Upgrade BWC Tests to validate Clear Cache API + public void testClearCache() throws Exception { + waitForClusterHealthGreen(NODES_BWC_CLUSTER); + switch (getClusterType()) { + case OLD: + createKnnIndex(testIndex, getKNNDefaultIndexSettings(), createKnnIndexMapping(TEST_FIELD, DIMENSIONS)); + int docIdOld = 0; + addKNNDocs(testIndex, TEST_FIELD, DIMENSIONS, docIdOld, NUM_DOCS); + break; + case UPGRADED: + queryCnt = NUM_DOCS; + validateClearCacheOnUpgrade(queryCnt); + + docId = NUM_DOCS; + addKNNDocs(testIndex, TEST_FIELD, DIMENSIONS, docId, NUM_DOCS); + + queryCnt = queryCnt + NUM_DOCS; + validateClearCacheOnUpgrade(queryCnt); + deleteKNNIndex(testIndex); + } + + } + + // validation steps for Clear Cache API after upgrading all nodes from old version to new version + public void validateClearCacheOnUpgrade(int queryCount) throws Exception { + int graphCount = getTotalGraphsInCache(); + knnWarmup(Collections.singletonList(testIndex)); + assertTrue(getTotalGraphsInCache() > graphCount); + validateKNNSearch(testIndex, TEST_FIELD, DIMENSIONS, queryCount, K); + + clearCache(Collections.singletonList(testIndex)); + assertEquals(0, getTotalGraphsInCache()); + validateKNNSearch(testIndex, TEST_FIELD, DIMENSIONS, queryCount, K); + } + +} diff --git a/src/main/java/org/opensearch/knn/common/KNNConstants.java b/src/main/java/org/opensearch/knn/common/KNNConstants.java index 6c92afabc..a2e539375 100644 --- a/src/main/java/org/opensearch/knn/common/KNNConstants.java +++ b/src/main/java/org/opensearch/knn/common/KNNConstants.java @@ -127,4 +127,7 @@ public class KNNConstants { // Please refer this github issue for more details for choosing this value: // https://github.com/opensearch-project/k-NN/issues/1049#issuecomment-1694741092 public static int MAX_DISTANCE_COMPUTATIONS = 2048000; + + // API Constants + public static final String CLEAR_CACHE = "clear_cache"; } diff --git a/src/main/java/org/opensearch/knn/index/KNNIndexShard.java b/src/main/java/org/opensearch/knn/index/KNNIndexShard.java index f7597dce6..efa09662c 100644 --- a/src/main/java/org/opensearch/knn/index/KNNIndexShard.java +++ b/src/main/java/org/opensearch/knn/index/KNNIndexShard.java @@ -8,9 +8,8 @@ import com.google.common.annotations.VisibleForTesting; import lombok.AllArgsConstructor; import lombok.Getter; +import lombok.extern.log4j.Log4j2; import org.apache.lucene.index.FieldInfo; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; import org.apache.lucene.index.FilterLeafReader; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.LeafReaderContext; @@ -20,6 +19,7 @@ import org.opensearch.index.engine.Engine; import org.opensearch.index.shard.IndexShard; import org.opensearch.knn.index.mapper.KNNVectorFieldMapper; +import org.opensearch.knn.index.memory.NativeMemoryAllocation; import org.opensearch.knn.index.memory.NativeMemoryCacheManager; import org.opensearch.knn.index.memory.NativeMemoryEntryContext; import org.opensearch.knn.index.memory.NativeMemoryLoadStrategy; @@ -30,6 +30,7 @@ import java.util.ArrayList; import java.util.Collection; import java.util.List; +import java.util.Optional; import java.util.concurrent.ExecutionException; import java.util.stream.Collectors; @@ -42,11 +43,11 @@ /** * KNNIndexShard wraps IndexShard and adds methods to perform k-NN related operations against the shard */ +@Log4j2 public class KNNIndexShard { private IndexShard indexShard; private NativeMemoryCacheManager nativeMemoryCacheManager; - - private static Logger logger = LogManager.getLogger(KNNIndexShard.class); + private static final String INDEX_SHARD_CLEAR_CACHE_SEARCHER = "knn-clear-cache"; /** * Constructor to generate KNNIndexShard. We do not perform validation that the index the shard is from @@ -84,7 +85,7 @@ public String getIndexName() { * @throws IOException Thrown when getting the HNSW Paths to be loaded in */ public void warmup() throws IOException { - logger.info("[KNN] Warming up index: " + getIndexName()); + log.info("[KNN] Warming up index: [{}]", getIndexName()); try (Engine.Searcher searcher = indexShard.acquireSearcher("knn-warmup")) { getAllEngineFileContexts(searcher.getIndexReader()).forEach((engineFileContext) -> { try { @@ -109,6 +110,34 @@ public void warmup() throws IOException { } } + /** + * Removes all the k-NN segments for this shard from the cache. + * Adding write lock onto the NativeMemoryAllocation of the index that needs to be evicted from cache. + * Write lock will be unlocked after the index is evicted. This locking mechanism is used to avoid + * conflicts with queries fired on this index when the index is being evicted from cache. + */ + public void clearCache() { + String indexName = getIndexName(); + Optional indexAllocationOptional; + NativeMemoryAllocation indexAllocation; + indexAllocationOptional = nativeMemoryCacheManager.getIndexMemoryAllocation(indexName); + if (indexAllocationOptional.isPresent()) { + indexAllocation = indexAllocationOptional.get(); + indexAllocation.writeLock(); + log.info("[KNN] Evicting index from cache: [{}]", indexName); + try (Engine.Searcher searcher = indexShard.acquireSearcher(INDEX_SHARD_CLEAR_CACHE_SEARCHER)) { + getAllEngineFileContexts(searcher.getIndexReader()).forEach( + (engineFileContext) -> nativeMemoryCacheManager.invalidate(engineFileContext.getIndexPath()) + ); + } catch (IOException ex) { + log.error("[KNN] Failed to evict index from cache: [{}]", indexName, ex); + throw new RuntimeException(ex); + } finally { + indexAllocation.writeUnlock(); + } + } + } + /** * For the given shard, get all of its engine paths * diff --git a/src/main/java/org/opensearch/knn/index/memory/NativeMemoryCacheManager.java b/src/main/java/org/opensearch/knn/index/memory/NativeMemoryCacheManager.java index 8b3a3bce1..9478e1e00 100644 --- a/src/main/java/org/opensearch/knn/index/memory/NativeMemoryCacheManager.java +++ b/src/main/java/org/opensearch/knn/index/memory/NativeMemoryCacheManager.java @@ -27,6 +27,7 @@ import java.io.Closeable; import java.util.HashMap; import java.util.Map; +import java.util.Optional; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -303,6 +304,23 @@ public NativeMemoryAllocation get(NativeMemoryEntryContext nativeMemoryEntryC return cache.get(nativeMemoryEntryContext.getKey(), nativeMemoryEntryContext::load); } + /** + * Returns the NativeMemoryAllocation associated with given index + * @param indexName name of OpenSearch index + * @return NativeMemoryAllocation associated with given index + */ + public Optional getIndexMemoryAllocation(String indexName) { + Validate.notNull(indexName, "Index name cannot be null"); + return cache.asMap() + .values() + .stream() + .filter(nativeMemoryAllocation -> nativeMemoryAllocation instanceof NativeMemoryAllocation.IndexAllocation) + .filter( + indexAllocation -> indexName.equals(((NativeMemoryAllocation.IndexAllocation) indexAllocation).getOpenSearchIndexName()) + ) + .findFirst(); + } + /** * Invalidate entry from the cache. * diff --git a/src/main/java/org/opensearch/knn/plugin/KNNPlugin.java b/src/main/java/org/opensearch/knn/plugin/KNNPlugin.java index 3edff1879..2e5a55092 100644 --- a/src/main/java/org/opensearch/knn/plugin/KNNPlugin.java +++ b/src/main/java/org/opensearch/knn/plugin/KNNPlugin.java @@ -31,6 +31,7 @@ import org.opensearch.knn.plugin.rest.RestKNNWarmupHandler; import org.opensearch.knn.plugin.rest.RestSearchModelHandler; import org.opensearch.knn.plugin.rest.RestTrainModelHandler; +import org.opensearch.knn.plugin.rest.RestClearCacheHandler; import org.opensearch.knn.plugin.script.KNNScoringScriptEngine; import org.opensearch.knn.plugin.stats.KNNStats; import org.opensearch.knn.plugin.transport.DeleteModelAction; @@ -41,6 +42,8 @@ import org.opensearch.knn.plugin.transport.KNNStatsTransportAction; import org.opensearch.knn.plugin.transport.KNNWarmupAction; import org.opensearch.knn.plugin.transport.KNNWarmupTransportAction; +import org.opensearch.knn.plugin.transport.ClearCacheAction; +import org.opensearch.knn.plugin.transport.ClearCacheTransportAction; import com.google.common.collect.ImmutableList; import org.opensearch.action.ActionRequest; @@ -236,6 +239,7 @@ public List getRestHandlers( RestDeleteModelHandler restDeleteModelHandler = new RestDeleteModelHandler(); RestTrainModelHandler restTrainModelHandler = new RestTrainModelHandler(); RestSearchModelHandler restSearchModelHandler = new RestSearchModelHandler(); + RestClearCacheHandler restClearCacheHandler = new RestClearCacheHandler(clusterService, indexNameExpressionResolver); return ImmutableList.of( restKNNStatsHandler, @@ -243,7 +247,8 @@ public List getRestHandlers( restGetModelHandler, restDeleteModelHandler, restTrainModelHandler, - restSearchModelHandler + restSearchModelHandler, + restClearCacheHandler ); } @@ -263,7 +268,8 @@ public List getRestHandlers( new ActionHandler<>(TrainingModelAction.INSTANCE, TrainingModelTransportAction.class), new ActionHandler<>(RemoveModelFromCacheAction.INSTANCE, RemoveModelFromCacheTransportAction.class), new ActionHandler<>(SearchModelAction.INSTANCE, SearchModelTransportAction.class), - new ActionHandler<>(UpdateModelGraveyardAction.INSTANCE, UpdateModelGraveyardTransportAction.class) + new ActionHandler<>(UpdateModelGraveyardAction.INSTANCE, UpdateModelGraveyardTransportAction.class), + new ActionHandler<>(ClearCacheAction.INSTANCE, ClearCacheTransportAction.class) ); } diff --git a/src/main/java/org/opensearch/knn/plugin/rest/RestClearCacheHandler.java b/src/main/java/org/opensearch/knn/plugin/rest/RestClearCacheHandler.java new file mode 100644 index 000000000..2cbc9cd76 --- /dev/null +++ b/src/main/java/org/opensearch/knn/plugin/rest/RestClearCacheHandler.java @@ -0,0 +1,98 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.plugin.rest; + +import com.google.common.collect.ImmutableList; +import lombok.AllArgsConstructor; +import lombok.extern.log4j.Log4j2; +import org.opensearch.client.node.NodeClient; +import org.opensearch.cluster.metadata.IndexNameExpressionResolver; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.core.common.Strings; +import org.opensearch.core.index.Index; +import org.opensearch.knn.common.exception.KNNInvalidIndicesException; +import org.opensearch.knn.plugin.KNNPlugin; +import org.opensearch.knn.plugin.transport.ClearCacheAction; +import org.opensearch.knn.plugin.transport.ClearCacheRequest; +import org.opensearch.rest.BaseRestHandler; +import org.opensearch.rest.RestRequest; +import org.opensearch.rest.action.RestToXContentListener; + +import java.util.Arrays; +import java.util.List; +import java.util.Locale; +import java.util.stream.Collectors; + +import static org.opensearch.action.support.IndicesOptions.strictExpandOpen; +import static org.opensearch.knn.common.KNNConstants.CLEAR_CACHE; +import static org.opensearch.knn.index.KNNSettings.KNN_INDEX; + +/** + * RestHandler for k-NN Clear Cache API. API provides the ability for a user to evict those indices from Cache. + */ +@AllArgsConstructor +@Log4j2 +public class RestClearCacheHandler extends BaseRestHandler { + private static final String INDEX = "index"; + public static String NAME = "knn_clear_cache_action"; + private final ClusterService clusterService; + private final IndexNameExpressionResolver indexNameExpressionResolver; + + /** + * @return name of Clear Cache API action + */ + @Override + public String getName() { + return NAME; + } + + /** + * @return Immutable List of Clear Cache API endpoint + */ + @Override + public List routes() { + return ImmutableList.of( + new Route(RestRequest.Method.POST, String.format(Locale.ROOT, "%s/%s/{%s}", KNNPlugin.KNN_BASE_URI, CLEAR_CACHE, INDEX)) + ); + } + + /** + * @param request RestRequest + * @param client NodeClient + * @return RestChannelConsumer + */ + @Override + protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) { + ClearCacheRequest clearCacheRequest = createClearCacheRequest(request); + log.info("[KNN] ClearCache started for the following indices: [{}]", String.join(",", clearCacheRequest.indices())); + return channel -> client.execute(ClearCacheAction.INSTANCE, clearCacheRequest, new RestToXContentListener<>(channel)); + } + + // Create a clear cache request by processing the rest request and validating the indices + private ClearCacheRequest createClearCacheRequest(RestRequest request) { + String[] indexNames = Strings.splitStringByCommaToArray(request.param("index")); + Index[] indices = indexNameExpressionResolver.concreteIndices(clusterService.state(), strictExpandOpen(), indexNames); + validateIndices(indices); + + return new ClearCacheRequest(indexNames); + } + + // Validate if the given indices are k-NN indices or not. If there are any invalid indices, + // the request is rejected and an exception is thrown. + private void validateIndices(Index[] indices) { + List invalidIndexNames = Arrays.stream(indices) + .filter(index -> !"true".equals(clusterService.state().metadata().getIndexSafe(index).getSettings().get(KNN_INDEX))) + .map(Index::getName) + .collect(Collectors.toList()); + + if (!invalidIndexNames.isEmpty()) { + throw new KNNInvalidIndicesException( + invalidIndexNames, + "ClearCache request rejected. One or more indices have 'index.knn' set to false." + ); + } + } +} diff --git a/src/main/java/org/opensearch/knn/plugin/transport/ClearCacheAction.java b/src/main/java/org/opensearch/knn/plugin/transport/ClearCacheAction.java new file mode 100644 index 000000000..358027a8b --- /dev/null +++ b/src/main/java/org/opensearch/knn/plugin/transport/ClearCacheAction.java @@ -0,0 +1,27 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.plugin.transport; + +import org.opensearch.action.ActionType; +import org.opensearch.core.common.io.stream.Writeable; + +/** + * Action associated with ClearCache + */ +public class ClearCacheAction extends ActionType { + + public static final ClearCacheAction INSTANCE = new ClearCacheAction(); + public static final String NAME = "cluster:admin/clear_cache_action"; + + private ClearCacheAction() { + super(NAME, ClearCacheResponse::new); + } + + @Override + public Writeable.Reader getResponseReader() { + return ClearCacheResponse::new; + } +} diff --git a/src/main/java/org/opensearch/knn/plugin/transport/ClearCacheRequest.java b/src/main/java/org/opensearch/knn/plugin/transport/ClearCacheRequest.java new file mode 100644 index 000000000..029b9babe --- /dev/null +++ b/src/main/java/org/opensearch/knn/plugin/transport/ClearCacheRequest.java @@ -0,0 +1,36 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.plugin.transport; + +import org.opensearch.action.support.broadcast.BroadcastRequest; +import org.opensearch.core.common.io.stream.StreamInput; + +import java.io.IOException; + +/** + * Clear Cache Request. This request contains a list of indices which needs to be evicted from Cache. + */ +public class ClearCacheRequest extends BroadcastRequest { + + /** + * Constructor + * + * @param in input stream + * @throws IOException if read from stream fails + */ + public ClearCacheRequest(StreamInput in) throws IOException { + super(in); + } + + /** + * Constructor + * + * @param indices list of indices which needs to be evicted from cache + */ + public ClearCacheRequest(String... indices) { + super(indices); + } +} diff --git a/src/main/java/org/opensearch/knn/plugin/transport/ClearCacheResponse.java b/src/main/java/org/opensearch/knn/plugin/transport/ClearCacheResponse.java new file mode 100644 index 000000000..3da7b0738 --- /dev/null +++ b/src/main/java/org/opensearch/knn/plugin/transport/ClearCacheResponse.java @@ -0,0 +1,49 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.plugin.transport; + +import org.opensearch.core.action.support.DefaultShardOperationFailedException; +import org.opensearch.action.support.broadcast.BroadcastResponse; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.xcontent.ToXContentObject; + +import java.io.IOException; +import java.util.List; + +/** + * {@link ClearCacheResponse} represents Response returned by {@link ClearCacheRequest}. + * Returns total number of shards on which ClearCache was performed on, as well as + * the number of shards that succeeded and the number of shards that failed. + */ +public class ClearCacheResponse extends BroadcastResponse implements ToXContentObject { + + /** + * Constructor + * + * @param in input stream + * @throws IOException if read from stream fails + */ + public ClearCacheResponse(StreamInput in) throws IOException { + super(in); + } + + /** + * Constructor + * + * @param totalShards total number of shards on which ClearCache was performed + * @param successfulShards number of shards that succeeded + * @param failedShards number of shards that failed + * @param shardFailures list of shard failure exceptions + */ + public ClearCacheResponse( + int totalShards, + int successfulShards, + int failedShards, + List shardFailures + ) { + super(totalShards, successfulShards, failedShards, shardFailures); + } +} diff --git a/src/main/java/org/opensearch/knn/plugin/transport/ClearCacheTransportAction.java b/src/main/java/org/opensearch/knn/plugin/transport/ClearCacheTransportAction.java new file mode 100644 index 000000000..4a294e73b --- /dev/null +++ b/src/main/java/org/opensearch/knn/plugin/transport/ClearCacheTransportAction.java @@ -0,0 +1,164 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.plugin.transport; + +import org.opensearch.action.support.ActionFilters; +import org.opensearch.action.support.broadcast.node.TransportBroadcastByNodeAction; +import org.opensearch.cluster.ClusterState; +import org.opensearch.cluster.block.ClusterBlockException; +import org.opensearch.cluster.block.ClusterBlockLevel; +import org.opensearch.cluster.metadata.IndexNameExpressionResolver; +import org.opensearch.cluster.routing.ShardRouting; +import org.opensearch.cluster.routing.ShardsIterator; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.inject.Inject; +import org.opensearch.core.action.support.DefaultShardOperationFailedException; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.index.Index; +import org.opensearch.index.IndexService; +import org.opensearch.index.shard.IndexShard; +import org.opensearch.indices.IndicesService; +import org.opensearch.knn.index.KNNIndexShard; +import org.opensearch.threadpool.ThreadPool; +import org.opensearch.transport.TransportService; + +import java.io.IOException; +import java.util.List; + +/** + * Transport Action to evict k-NN indices from Cache. TransportBroadcastByNodeAction will distribute the request to + * all shards across the cluster for the given indices. For each shard, shardOperation will be called and the + * indices will be cleared from cache. + */ +public class ClearCacheTransportAction extends TransportBroadcastByNodeAction< + ClearCacheRequest, + ClearCacheResponse, + TransportBroadcastByNodeAction.EmptyResult> { + + private IndicesService indicesService; + + /** + * Constructor + * + * @param clusterService ClusterService + * @param transportService TransportService + * @param actionFilters ActionFilters + * @param indexNameExpressionResolver IndexNameExpressionResolver + * @param indicesService IndicesService + */ + @Inject + public ClearCacheTransportAction( + ClusterService clusterService, + TransportService transportService, + ActionFilters actionFilters, + IndexNameExpressionResolver indexNameExpressionResolver, + IndicesService indicesService + ) { + super( + ClearCacheAction.NAME, + clusterService, + transportService, + actionFilters, + indexNameExpressionResolver, + ClearCacheRequest::new, + ThreadPool.Names.SEARCH + ); + this.indicesService = indicesService; + } + + /** + * @param streamInput StreamInput + * @return EmptyResult + * @throws IOException + */ + @Override + protected EmptyResult readShardResult(StreamInput streamInput) throws IOException { + return EmptyResult.readEmptyResultFrom(streamInput); + } + + /** + * @param request ClearCacheRequest + * @param totalShards total number of shards on which ClearCache was performed + * @param successfulShards number of shards that succeeded + * @param failedShards number of shards that failed + * @param emptyResults List of EmptyResult + * @param shardFailures list of shard failure exceptions + * @param clusterState ClusterState + * @return {@link ClearCacheResponse} + */ + @Override + protected ClearCacheResponse newResponse( + ClearCacheRequest request, + int totalShards, + int successfulShards, + int failedShards, + List emptyResults, + List shardFailures, + ClusterState clusterState + ) { + return new ClearCacheResponse(totalShards, successfulShards, failedShards, shardFailures); + } + + /** + * @param streamInput StreamInput + * @return {@link ClearCacheRequest} + * @throws IOException + */ + @Override + protected ClearCacheRequest readRequestFrom(StreamInput streamInput) throws IOException { + return new ClearCacheRequest(streamInput); + } + + /** + * Operation performed at a shard level on all the shards of given index where the index is removed from the cache. + * + * @param request ClearCacheRequest + * @param shardRouting ShardRouting of given shard + * @return EmptyResult + * @throws IOException + */ + @Override + protected EmptyResult shardOperation(ClearCacheRequest request, ShardRouting shardRouting) throws IOException { + Index index = shardRouting.shardId().getIndex(); + IndexService indexService = indicesService.indexServiceSafe(index); + IndexShard indexShard = indexService.getShard(shardRouting.shardId().id()); + KNNIndexShard knnIndexShard = new KNNIndexShard(indexShard); + knnIndexShard.clearCache(); + return EmptyResult.INSTANCE; + } + + /** + * @param clusterState ClusterState + * @param request ClearCacheRequest + * @param concreteIndices Indices in the request + * @return ShardsIterator with all the shards for given concrete indices + */ + @Override + protected ShardsIterator shards(ClusterState clusterState, ClearCacheRequest request, String[] concreteIndices) { + return clusterState.routingTable().allShards(concreteIndices); + } + + /** + * @param clusterState ClusterState + * @param request ClearCacheRequest + * @return ClusterBlockException if there is any global cluster block at a cluster block level of "METADATA_WRITE" + */ + @Override + protected ClusterBlockException checkGlobalBlock(ClusterState clusterState, ClearCacheRequest request) { + return clusterState.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE); + } + + /** + * @param clusterState ClusterState + * @param request ClearCacheRequest + * @param concreteIndices Indices in the request + * @return ClusterBlockException if there is any cluster block on any of the given indices at a cluster block level of "METADATA_WRITE" + */ + @Override + protected ClusterBlockException checkRequestBlock(ClusterState clusterState, ClearCacheRequest request, String[] concreteIndices) { + return clusterState.blocks().indicesBlockedException(ClusterBlockLevel.METADATA_WRITE, concreteIndices); + } +} diff --git a/src/test/java/org/opensearch/knn/KNNSingleNodeTestCase.java b/src/test/java/org/opensearch/knn/KNNSingleNodeTestCase.java index 323442fff..1c9e75c3a 100644 --- a/src/test/java/org/opensearch/knn/KNNSingleNodeTestCase.java +++ b/src/test/java/org/opensearch/knn/KNNSingleNodeTestCase.java @@ -6,6 +6,12 @@ package org.opensearch.knn; import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.cluster.ClusterName; +import org.opensearch.cluster.ClusterState; +import org.opensearch.cluster.block.ClusterBlock; +import org.opensearch.cluster.block.ClusterBlockLevel; +import org.opensearch.cluster.block.ClusterBlocks; +import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.xcontent.XContentHelper; import org.opensearch.knn.index.query.KNNQueryBuilder; import org.opensearch.knn.index.memory.NativeMemoryCacheManager; @@ -32,9 +38,12 @@ import java.io.IOException; import java.util.Collection; import java.util.Collections; +import java.util.EnumSet; import java.util.Map; import java.util.concurrent.ExecutionException; +import static org.mockito.Mockito.when; + public class KNNSingleNodeTestCase extends OpenSearchSingleNodeTestCase { @Override public void setUp() throws Exception { @@ -208,4 +217,25 @@ public void assertTrainingSucceeds(ModelDao modelDao, String modelId, int attemp fail("Training did not succeed after " + attempts + " attempts with a delay of " + delayInMillis + " ms."); } + + // Add Global Cluster Block with the given ClusterBlockLevel + protected void addGlobalClusterBlock(ClusterService clusterService, String description, EnumSet clusterBlockLevels) { + ClusterBlock block = new ClusterBlock(randomInt(), description, false, false, false, RestStatus.FORBIDDEN, clusterBlockLevels); + ClusterBlocks clusterBlocks = ClusterBlocks.builder().addGlobalBlock(block).build(); + ClusterState state = ClusterState.builder(ClusterName.DEFAULT).blocks(clusterBlocks).build(); + when(clusterService.state()).thenReturn(state); + } + + // Add Cluster Block for an Index with given ClusterBlockLevel + protected void addIndexClusterBlock( + ClusterService clusterService, + String description, + EnumSet clusterBlockLevels, + String testIndex + ) { + ClusterBlock block = new ClusterBlock(randomInt(), description, false, false, false, RestStatus.FORBIDDEN, clusterBlockLevels); + ClusterBlocks clusterBlocks = ClusterBlocks.builder().addIndexBlock(testIndex, block).build(); + ClusterState state = ClusterState.builder(ClusterName.DEFAULT).blocks(clusterBlocks).build(); + when(clusterService.state()).thenReturn(state); + } } diff --git a/src/test/java/org/opensearch/knn/index/KNNIndexShardTests.java b/src/test/java/org/opensearch/knn/index/KNNIndexShardTests.java index 079d18e66..7b6f96d5a 100644 --- a/src/test/java/org/opensearch/knn/index/KNNIndexShardTests.java +++ b/src/test/java/org/opensearch/knn/index/KNNIndexShardTests.java @@ -7,6 +7,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; +import lombok.SneakyThrows; import org.opensearch.knn.KNNSingleNodeTestCase; import org.opensearch.index.IndexService; import org.opensearch.index.engine.Engine; @@ -155,4 +156,30 @@ public void testGetEngineFileContexts() { assertEquals(includedFileNames.size(), included.size()); included.stream().map(KNNIndexShard.EngineFileContext::getIndexPath).forEach(o -> assertTrue(includedFileNames.contains(o))); } + + @SneakyThrows + public void testClearCache_emptyIndex() { + IndexService indexService = createKNNIndex(testIndexName); + createKnnIndexMapping(testIndexName, testFieldName, dimensions); + + IndexShard indexShard = indexService.iterator().next(); + KNNIndexShard knnIndexShard = new KNNIndexShard(indexShard); + knnIndexShard.clearCache(); + assertNull(NativeMemoryCacheManager.getInstance().getIndicesCacheStats().get(testIndexName)); + } + + @SneakyThrows + public void testClearCache_shardPresentInCache() { + IndexService indexService = createKNNIndex(testIndexName); + createKnnIndexMapping(testIndexName, testFieldName, dimensions); + addKnnDoc(testIndexName, String.valueOf(randomInt()), testFieldName, new Float[] { randomFloat(), randomFloat() }); + + IndexShard indexShard = indexService.iterator().next(); + KNNIndexShard knnIndexShard = new KNNIndexShard(indexShard); + knnIndexShard.warmup(); + assertEquals(1, NativeMemoryCacheManager.getInstance().getIndicesCacheStats().get(testIndexName).get(GRAPH_COUNT)); + + knnIndexShard.clearCache(); + assertNull(NativeMemoryCacheManager.getInstance().getIndicesCacheStats().get(testIndexName)); + } } diff --git a/src/test/java/org/opensearch/knn/plugin/action/RestClearCacheHandlerIT.java b/src/test/java/org/opensearch/knn/plugin/action/RestClearCacheHandlerIT.java new file mode 100644 index 000000000..2618519f2 --- /dev/null +++ b/src/test/java/org/opensearch/knn/plugin/action/RestClearCacheHandlerIT.java @@ -0,0 +1,111 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.plugin.action; + +import lombok.SneakyThrows; +import org.opensearch.client.Request; +import org.opensearch.client.ResponseException; +import org.opensearch.common.settings.Settings; +import org.opensearch.knn.KNNRestTestCase; +import org.opensearch.knn.plugin.KNNPlugin; +import org.opensearch.rest.RestRequest; + +import java.util.Arrays; +import java.util.Collections; + +import static org.opensearch.knn.common.KNNConstants.CLEAR_CACHE; + +/** + * Integration tests to validate ClearCache API + */ + +public class RestClearCacheHandlerIT extends KNNRestTestCase { + private static final String TEST_FIELD = "test-field"; + private static final int DIMENSIONS = 2; + + @SneakyThrows + public void testNonExistentIndex() { + String nonExistentIndex = "non-existent-index"; + + String restURI = String.join("/", KNNPlugin.KNN_BASE_URI, CLEAR_CACHE, nonExistentIndex); + Request request = new Request(RestRequest.Method.POST.name(), restURI); + + ResponseException ex = expectThrows(ResponseException.class, () -> client().performRequest(request)); + assertTrue(ex.getMessage().contains(nonExistentIndex)); + } + + @SneakyThrows + public void testNotKnnIndex() { + String notKNNIndex = "not-knn-index"; + createIndex(notKNNIndex, Settings.EMPTY); + + String restURI = String.join("/", KNNPlugin.KNN_BASE_URI, CLEAR_CACHE, notKNNIndex); + Request request = new Request(RestRequest.Method.POST.name(), restURI); + + ResponseException ex = expectThrows(ResponseException.class, () -> client().performRequest(request)); + assertTrue(ex.getMessage().contains(notKNNIndex)); + } + + @SneakyThrows + public void testClearCacheSingleIndex() { + String testIndex = getTestName().toLowerCase(); + int graphCountBefore = getTotalGraphsInCache(); + createKnnIndex(testIndex, getKNNDefaultIndexSettings(), createKnnIndexMapping(TEST_FIELD, DIMENSIONS)); + addKnnDoc(testIndex, String.valueOf(randomInt()), TEST_FIELD, new Float[] { randomFloat(), randomFloat() }); + + knnWarmup(Collections.singletonList(testIndex)); + + assertEquals(graphCountBefore + 1, getTotalGraphsInCache()); + + clearCache(Collections.singletonList(testIndex)); + assertEquals(graphCountBefore, getTotalGraphsInCache()); + } + + @SneakyThrows + public void testClearCacheMultipleIndices() { + String testIndex1 = getTestName().toLowerCase(); + String testIndex2 = getTestName().toLowerCase() + 1; + int graphCountBefore = getTotalGraphsInCache(); + + createKnnIndex(testIndex1, getKNNDefaultIndexSettings(), createKnnIndexMapping(TEST_FIELD, DIMENSIONS)); + addKnnDoc(testIndex1, String.valueOf(randomInt()), TEST_FIELD, new Float[] { randomFloat(), randomFloat() }); + + createKnnIndex(testIndex2, getKNNDefaultIndexSettings(), createKnnIndexMapping(TEST_FIELD, DIMENSIONS)); + addKnnDoc(testIndex2, String.valueOf(randomInt()), TEST_FIELD, new Float[] { randomFloat(), randomFloat() }); + + knnWarmup(Arrays.asList(testIndex1, testIndex2)); + + assertEquals(graphCountBefore + 2, getTotalGraphsInCache()); + + clearCache(Arrays.asList(testIndex1, testIndex2)); + assertEquals(graphCountBefore, getTotalGraphsInCache()); + } + + @SneakyThrows + public void testClearCacheMultipleIndicesWithPatterns() { + String testIndex1 = getTestName().toLowerCase(); + String testIndex2 = getTestName().toLowerCase() + 1; + String testIndex3 = "abc" + getTestName().toLowerCase(); + int graphCountBefore = getTotalGraphsInCache(); + + createKnnIndex(testIndex1, getKNNDefaultIndexSettings(), createKnnIndexMapping(TEST_FIELD, DIMENSIONS)); + addKnnDoc(testIndex1, String.valueOf(randomInt()), TEST_FIELD, new Float[] { randomFloat(), randomFloat() }); + + createKnnIndex(testIndex2, getKNNDefaultIndexSettings(), createKnnIndexMapping(TEST_FIELD, DIMENSIONS)); + addKnnDoc(testIndex2, String.valueOf(randomInt()), TEST_FIELD, new Float[] { randomFloat(), randomFloat() }); + + createKnnIndex(testIndex3, getKNNDefaultIndexSettings(), createKnnIndexMapping(TEST_FIELD, DIMENSIONS)); + addKnnDoc(testIndex3, String.valueOf(randomInt()), TEST_FIELD, new Float[] { randomFloat(), randomFloat() }); + + knnWarmup(Arrays.asList(testIndex1, testIndex2, testIndex3)); + + assertEquals(graphCountBefore + 3, getTotalGraphsInCache()); + String indexPattern = getTestName().toLowerCase() + "*"; + + clearCache(Arrays.asList(indexPattern)); + assertEquals(graphCountBefore + 1, getTotalGraphsInCache()); + } +} diff --git a/src/test/java/org/opensearch/knn/plugin/transport/ClearCacheTransportActionTests.java b/src/test/java/org/opensearch/knn/plugin/transport/ClearCacheTransportActionTests.java new file mode 100644 index 000000000..3222a3eb7 --- /dev/null +++ b/src/test/java/org/opensearch/knn/plugin/transport/ClearCacheTransportActionTests.java @@ -0,0 +1,117 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.plugin.transport; + +import lombok.SneakyThrows; +import org.opensearch.cluster.ClusterName; +import org.opensearch.cluster.ClusterState; +import org.opensearch.cluster.block.ClusterBlockException; +import org.opensearch.cluster.block.ClusterBlockLevel; +import org.opensearch.cluster.routing.ShardRouting; +import org.opensearch.cluster.routing.ShardsIterator; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.index.IndexService; +import org.opensearch.knn.KNNSingleNodeTestCase; +import org.opensearch.knn.index.memory.NativeMemoryCacheManager; + +import java.util.EnumSet; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class ClearCacheTransportActionTests extends KNNSingleNodeTestCase { + private static final String TEST_FIELD = "test-field"; + private static final int DIMENSIONS = 2; + + @SneakyThrows + public void testShardOperation() { + String testIndex = getTestName().toLowerCase(); + KNNWarmupRequest knnWarmupRequest = new KNNWarmupRequest(testIndex); + KNNWarmupTransportAction knnWarmupTransportAction = node().injector().getInstance(KNNWarmupTransportAction.class); + assertEquals(0, NativeMemoryCacheManager.getInstance().getIndicesCacheStats().size()); + + IndexService indexService = createKNNIndex(testIndex); + createKnnIndexMapping(testIndex, TEST_FIELD, DIMENSIONS); + addKnnDoc(testIndex, String.valueOf(randomInt()), TEST_FIELD, new Float[] { randomFloat(), randomFloat() }); + ShardRouting shardRouting = indexService.iterator().next().routingEntry(); + + knnWarmupTransportAction.shardOperation(knnWarmupRequest, shardRouting); + assertEquals(1, NativeMemoryCacheManager.getInstance().getIndicesCacheStats().size()); + + ClearCacheRequest clearCacheRequest = new ClearCacheRequest(testIndex); + ClearCacheTransportAction clearCacheTransportAction = node().injector().getInstance(ClearCacheTransportAction.class); + clearCacheTransportAction.shardOperation(clearCacheRequest, shardRouting); + assertEquals(0, NativeMemoryCacheManager.getInstance().getIndicesCacheStats().size()); + } + + @SneakyThrows + public void testShards() { + String testIndex = getTestName().toLowerCase(); + ClusterService clusterService = node().injector().getInstance(ClusterService.class); + ClearCacheTransportAction clearCacheTransportAction = node().injector().getInstance(ClearCacheTransportAction.class); + ClearCacheRequest clearCacheRequest = new ClearCacheRequest(testIndex); + + createKNNIndex(testIndex); + createKnnIndexMapping(testIndex, TEST_FIELD, DIMENSIONS); + addKnnDoc(testIndex, String.valueOf(randomInt()), TEST_FIELD, new Float[] { randomFloat(), randomFloat() }); + + ShardsIterator shardsIterator = clearCacheTransportAction.shards( + clusterService.state(), + clearCacheRequest, + new String[] { testIndex } + ); + assertEquals(1, shardsIterator.size()); + } + + public void testCheckGlobalBlock_throwsClusterBlockException() { + String testIndex = getTestName().toLowerCase(); + String description = "testing metadata block"; + ClusterService clusterService = mock(ClusterService.class); + addGlobalClusterBlock(clusterService, description, EnumSet.of(ClusterBlockLevel.METADATA_WRITE)); + ClearCacheTransportAction clearCacheTransportAction = node().injector().getInstance(ClearCacheTransportAction.class); + ClearCacheRequest clearCacheRequest = new ClearCacheRequest(testIndex); + ClusterBlockException ex = clearCacheTransportAction.checkGlobalBlock(clusterService.state(), clearCacheRequest); + assertTrue(ex.getMessage().contains(description)); + } + + public void testCheckGlobalBlock_notThrowsClusterBlockException() { + String testIndex = getTestName().toLowerCase(); + ClusterService clusterService = mock(ClusterService.class); + ClearCacheTransportAction clearCacheTransportAction = node().injector().getInstance(ClearCacheTransportAction.class); + ClearCacheRequest clearCacheRequest = new ClearCacheRequest(testIndex); + ClusterState state = ClusterState.builder(ClusterName.DEFAULT).build(); + when(clusterService.state()).thenReturn(state); + assertNull(clearCacheTransportAction.checkGlobalBlock(clusterService.state(), clearCacheRequest)); + } + + public void testCheckRequestBlock_throwsClusterBlockException() { + String testIndex = getTestName().toLowerCase(); + String description = "testing index metadata block"; + ClusterService clusterService = mock(ClusterService.class); + addIndexClusterBlock(clusterService, description, EnumSet.of(ClusterBlockLevel.METADATA_WRITE), testIndex); + + ClearCacheTransportAction clearCacheTransportAction = node().injector().getInstance(ClearCacheTransportAction.class); + ClearCacheRequest clearCacheRequest = new ClearCacheRequest(testIndex); + ClusterBlockException ex = clearCacheTransportAction.checkRequestBlock( + clusterService.state(), + clearCacheRequest, + new String[] { testIndex } + ); + assertTrue(ex.getMessage().contains(testIndex)); + assertTrue(ex.getMessage().contains(description)); + + } + + public void testCheckRequestBlock_notThrowsClusterBlockException() { + String testIndex = getTestName().toLowerCase(); + ClusterService clusterService = mock(ClusterService.class); + ClearCacheTransportAction clearCacheTransportAction = node().injector().getInstance(ClearCacheTransportAction.class); + ClearCacheRequest clearCacheRequest = new ClearCacheRequest(testIndex); + ClusterState state = ClusterState.builder(ClusterName.DEFAULT).build(); + when(clusterService.state()).thenReturn(state); + assertNull(clearCacheTransportAction.checkRequestBlock(clusterService.state(), clearCacheRequest, new String[] { testIndex })); + } +} diff --git a/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java b/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java index 60c401648..0331d49e5 100644 --- a/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java +++ b/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java @@ -84,6 +84,7 @@ import static org.opensearch.knn.common.KNNConstants.PARAMETERS; import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_EF_CONSTRUCTION; import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_M; +import static org.opensearch.knn.common.KNNConstants.CLEAR_CACHE; import static org.opensearch.knn.TestUtils.NUMBER_OF_REPLICAS; import static org.opensearch.knn.TestUtils.NUMBER_OF_SHARDS; @@ -692,6 +693,20 @@ protected Response executeWarmupRequest(List indices, final String baseU return client().performRequest(request); } + /** + * Evicts valid k-NN indices from the cache. + * + * @param indices list of k-NN indices that needs to be removed from cache + * @return Response of clear Cache API request + * @throws IOException + */ + protected Response clearCache(List indices) throws IOException { + String indicesSuffix = String.join(",", indices); + String restURI = String.join("/", KNNPlugin.KNN_BASE_URI, CLEAR_CACHE, indicesSuffix); + Request request = new Request("POST", restURI); + return client().performRequest(request); + } + /** * Parse KNN Cluster stats from response */ From 028f289f4af821f939440cab79402a72de7f2eb7 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Wed, 17 Apr 2024 19:14:51 -0700 Subject: [PATCH 237/416] Separate common lib jni and non-jni into 2 libs (#1624) Creates a new shared non-jni library that the jni libraries can depend on. This is to resolve a build issue on windows that is coming from nmslib not being able to find symbols in the jni common library. The net result will be that we add one more shared library. In addition, I removed all the CPack code from CMakeLists because we dont use it. Signed-off-by: John Mazanec (cherry picked from commit dc0953a3aa4fae48d75fa22b146a7087a2b49ee5) --- build.gradle | 2 +- jni/CMakeLists.txt | 79 ++++++++++++++-------------------------------- 2 files changed, 25 insertions(+), 56 deletions(-) diff --git a/build.gradle b/build.gradle index 6a5fa1d13..b608dbb1a 100644 --- a/build.gradle +++ b/build.gradle @@ -313,7 +313,7 @@ task cmakeJniLib(type:Exec) { task buildJniLib(type:Exec) { dependsOn cmakeJniLib workingDir 'jni' - commandLine 'make', 'opensearchknn_nmslib', 'opensearchknn_faiss' + commandLine 'make', 'opensearchknn_nmslib', 'opensearchknn_faiss', 'opensearchknn_common' } test { diff --git a/jni/CMakeLists.txt b/jni/CMakeLists.txt index 4f32c87b9..b81e0fe20 100644 --- a/jni/CMakeLists.txt +++ b/jni/CMakeLists.txt @@ -9,7 +9,9 @@ project(KNNPlugin_JNI) # ---------------------------------- SETUP ---------------------------------- # Target libraries to be compiled -set(TARGET_LIB_COMMON opensearchknn_common) # Shared library with common utilities +# Shared library with common utilities. Not a JNI library. Other JNI libs should depend on this one. +set(TARGET_LIB_UTIL opensearchknn_util) +set(TARGET_LIB_COMMON opensearchknn_common) # common lib for JNI set(TARGET_LIB_NMSLIB opensearchknn_nmslib) # nmslib JNI set(TARGET_LIB_FAISS opensearchknn_faiss) # faiss JNI set(TARGET_LIBS "") # Libs to be installed @@ -60,8 +62,25 @@ elseif(${CMAKE_SYSTEM_PROCESSOR} STREQUAL x86_64) endif() # ---------------------------------------------------------------------------- +# ---------------------------------- UTIL ---------------------------------- +add_library(${TARGET_LIB_UTIL} SHARED ${CMAKE_CURRENT_SOURCE_DIR}/src/jni_util.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/commons.cpp) +target_include_directories(${TARGET_LIB_UTIL} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include $ENV{JAVA_HOME}/include $ENV{JAVA_HOME}/include/${JVM_OS_TYPE}) +set_target_properties(${TARGET_LIB_UTIL} PROPERTIES SUFFIX ${LIB_EXT}) +set_target_properties(${TARGET_LIB_UTIL} PROPERTIES POSITION_INDEPENDENT_CODE ON) + +if (NOT "${WIN32}" STREQUAL "") + # Use RUNTIME_OUTPUT_DIRECTORY, to build the target library (opensearchknn_util) in the specified directory at runtime. + set_target_properties(${TARGET_LIB_UTIL} PROPERTIES RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/release) +else() + set_target_properties(${TARGET_LIB_UTIL} PROPERTIES LIBRARY_OUTPUT_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/release) +endif() + +list(APPEND TARGET_LIBS ${TARGET_LIB_UTIL}) +# ---------------------------------------------------------------------------- + # ---------------------------------- COMMON ---------------------------------- -add_library(${TARGET_LIB_COMMON} SHARED ${CMAKE_CURRENT_SOURCE_DIR}/src/jni_util.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/org_opensearch_knn_jni_JNICommons.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/commons.cpp) +add_library(${TARGET_LIB_COMMON} SHARED ${CMAKE_CURRENT_SOURCE_DIR}/src/org_opensearch_knn_jni_JNICommons.cpp) +target_link_libraries(${TARGET_LIB_COMMON} ${TARGET_LIB_UTIL}) target_include_directories(${TARGET_LIB_COMMON} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include $ENV{JAVA_HOME}/include $ENV{JAVA_HOME}/include/${JVM_OS_TYPE}) set_target_properties(${TARGET_LIB_COMMON} PROPERTIES SUFFIX ${LIB_EXT}) set_target_properties(${TARGET_LIB_COMMON} PROPERTIES POSITION_INDEPENDENT_CODE ON) @@ -103,7 +122,7 @@ if (${CONFIG_NMSLIB} STREQUAL ON OR ${CONFIG_ALL} STREQUAL ON OR ${CONFIG_TEST} add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/external/nmslib/similarity_search) add_library(${TARGET_LIB_NMSLIB} SHARED ${CMAKE_CURRENT_SOURCE_DIR}/src/org_opensearch_knn_jni_NmslibService.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/nmslib_wrapper.cpp) - target_link_libraries(${TARGET_LIB_NMSLIB} NonMetricSpaceLib ${TARGET_LIB_COMMON}) + target_link_libraries(${TARGET_LIB_NMSLIB} NonMetricSpaceLib ${TARGET_LIB_UTIL}) target_include_directories(${TARGET_LIB_NMSLIB} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include $ENV{JAVA_HOME}/include $ENV{JAVA_HOME}/include/${JVM_OS_TYPE} ${CMAKE_CURRENT_SOURCE_DIR}/external/nmslib/similarity_search/include) set_target_properties(${TARGET_LIB_NMSLIB} PROPERTIES SUFFIX ${LIB_EXT}) set_target_properties(${TARGET_LIB_NMSLIB} PROPERTIES POSITION_INDEPENDENT_CODE ON) @@ -190,7 +209,7 @@ if (${CONFIG_FAISS} STREQUAL ON OR ${CONFIG_ALL} STREQUAL ON OR ${CONFIG_TEST} S ${CMAKE_CURRENT_SOURCE_DIR}/src/faiss_wrapper.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/faiss_util.cpp ) - target_link_libraries(${TARGET_LIB_FAISS} ${TARGET_LINK_FAISS_LIB} ${TARGET_LIB_COMMON} OpenMP::OpenMP_CXX) + target_link_libraries(${TARGET_LIB_FAISS} ${TARGET_LINK_FAISS_LIB} ${TARGET_LIB_UTIL} OpenMP::OpenMP_CXX) target_include_directories(${TARGET_LIB_FAISS} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include $ENV{JAVA_HOME}/include @@ -249,6 +268,7 @@ if ("${WIN32}" STREQUAL "") ${TARGET_LIB_FAISS} ${TARGET_LIB_NMSLIB} ${TARGET_LIB_COMMON} + ${TARGET_LIB_UTIL} ) target_include_directories(jni_test PRIVATE @@ -267,54 +287,3 @@ if ("${WIN32}" STREQUAL "") endif() # --------------------------------------------------------------------------- - -# -------------------------------- INSTALL ---------------------------------- -# Installation rules for shared library -install(TARGETS ${TARGET_LIBS} - LIBRARY DESTINATION lib - COMPONENT library) - -set(KNN_MAINTAINER "OpenSearch Team ") -set(OPENSEARCH_DOWNLOAD_URL "https://opensearch.org/downloads.html") -set(CPACK_PACKAGE_NAME ${KNN_PACKAGE_NAME}) -set(CPACK_PACKAGE_VERSION ${KNN_PLUGIN_VERSION}) -set(CMAKE_INSTALL_PREFIX /usr) -set(CPACK_GENERATOR "RPM;DEB") -set(CPACK_OUTPUT_FILE_PREFIX packages) -set(CPACK_PACKAGE_RELEASE 1) -set(CPACK_PACKAGE_VENDOR "Amazon") -set(CPACK_PACKAGE_CONTACT "Maintainer: ${KNN_MAINTAINER}") -set(CPACK_PACKAGING_INSTALL_PREFIX ${CMAKE_INSTALL_PREFIX}) -set(CPACK_COMPONENTS_GROUPING IGNORE) -get_cmake_property(CPACK_COMPONENTS_ALL COMPONENTS) -list(REMOVE_ITEM CPACK_COMPONENTS_ALL "Unspecified") - -# Component variable -set(KNN_PACKAGE_NAME opensearch-knnlib) -set(KNN_PACKAGE_DESCRIPTION "KNN JNI libraries built off of nmslib and faiss for OpenSearch") - -# RPM -set(CPACK_RPM_PACKAGE_LICENSE "ASL-2.0") -set(CPACK_RPM_COMPONENT_INSTALL ON) -set(CPACK_RPM_PACKAGE_URL ${OPENSEARCH_DOWNLOAD_URL}) -set(CPACK_RPM_PACKAGE_RELEASE ${CPACK_PACKAGE_RELEASE}) - -set(CPACK_RPM_PACKAGE_NAME ${KNN_PACKAGE_NAME}) -set(CPACK_RPM_FILE_NAME "${CPACK_RPM_PACKAGE_NAME}-${CPACK_PACKAGE_VERSION}-${JVM_OS_TYPE}-${MACH_ARCH}.rpm") -set(CPACK_RPM_PACKAGE_DESCRIPTION ${KNN_PACKAGE_DESCRIPTION}) -set(CPACK_RPM_PACKAGE_SUMMARY "OpenSearch k-NN JNI Library with nmslib and faiss") - -# DEB -set(CPACK_DEBIAN_PACKAGE_HOMEPAGE ${OPENSEARCH_DOWNLOAD_URL}) -set(CPACK_DEBIAN_PACKAGE_MAINTAINER ${KNN_MAINTAINER}) -set(CPACK_DEBIAN_PACKAGE_VERSION ${CPACK_PACKAGE_VERSION}) -set(CPACK_DEBIAN_PACKAGE_SECTION "libs") -set(CPACK_DEB_COMPONENT_INSTALL ON) - -set(CPACK_DEBIAN_PACKAGE_NAME ${KNN_PACKAGE_NAME}) -set(CPACK_DEBIAN_FILE_NAME "${CPACK_DEBIAN_PACKAGE_NAME}-${CPACK_PACKAGE_VERSION}-${JVM_OS_TYPE}-${MACH_ARCH}.deb") -set(CPACK_DEBIAN_DESCRIPTION ${KNN_PACKAGE_DESCRIPTION}) -set(CPACK_DEBIAN_PACKAGE_SOURCE ${CPACK_DEBIAN_PACKAGE_NAME}) - -include(CPack) -# --------------------------------------------------------------------------- From 4abe91f1c506ea6ffcdc3326562ae6d4b402de54 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Thu, 18 Apr 2024 13:36:51 -0700 Subject: [PATCH 238/416] Add arm64 check when SIMD is disabled (#1618) (#1626) * Add arm64 check when SIMD is disabled Signed-off-by: Ryan Bogan * Add Changelog entry Signed-off-by: Ryan Bogan * Change entry in changelog to infrastructure Signed-off-by: Ryan Bogan --------- Signed-off-by: Ryan Bogan (cherry picked from commit a2bb6cc9d77d49b4678fc6d96b7b885c06b75bff) Co-authored-by: Ryan Bogan --- CHANGELOG.md | 1 + jni/CMakeLists.txt | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e3c07fc1d..396a86d83 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Bug Fixes ### Infrastructure * Add micro-benchmark module in k-NN plugin for benchmark streaming vectors to JNI layer functionality. [#1583](https://github.com/opensearch-project/k-NN/pull/1583) +* Add arm64 check when SIMD is disabled [#1618](https://github.com/opensearch-project/k-NN/pull/1618) ### Documentation ### Maintenance ### Refactoring diff --git a/jni/CMakeLists.txt b/jni/CMakeLists.txt index b81e0fe20..b894baebf 100644 --- a/jni/CMakeLists.txt +++ b/jni/CMakeLists.txt @@ -148,7 +148,7 @@ if (${CONFIG_FAISS} STREQUAL ON OR ${CONFIG_ALL} STREQUAL ON OR ${CONFIG_TEST} S set(SIMD_ENABLED true) # set default value as true if the argument is not set endif() - if(${CMAKE_SYSTEM_NAME} STREQUAL Windows OR ${CMAKE_SYSTEM_PROCESSOR} MATCHES "aarch64" OR NOT ${SIMD_ENABLED}) + if(${CMAKE_SYSTEM_NAME} STREQUAL Windows OR ${CMAKE_SYSTEM_PROCESSOR} MATCHES "aarch64" OR ${CMAKE_SYSTEM_PROCESSOR} MATCHES "arm64" OR NOT ${SIMD_ENABLED}) set(FAISS_OPT_LEVEL generic) # Keep optimization level as generic on Windows OS as it is not supported due to MINGW64 compiler issue. Also, on aarch64 avx2 is not supported. set(TARGET_LINK_FAISS_LIB faiss) else() From 529c5dedee9547f5ff6511a365ea071934f865d7 Mon Sep 17 00:00:00 2001 From: Junqiu Lei Date: Thu, 18 Apr 2024 15:39:13 -0700 Subject: [PATCH 239/416] Support radial search in k-NN (#1627) Signed-off-by: Junqiu Lei --- CHANGELOG.md | 1 + jni/include/faiss_wrapper.h | 12 + .../org_opensearch_knn_jni_FaissService.h | 8 + jni/src/faiss_wrapper.cpp | 44 ++ .../org_opensearch_knn_jni_FaissService.cpp | 14 + jni/tests/faiss_wrapper_test.cpp | 113 ++++ .../opensearch/knn/common/KNNConstants.java | 2 + .../org/opensearch/knn/index/IndexUtil.java | 2 + .../org/opensearch/knn/index/SpaceType.java | 18 + .../knn/index/query/BaseQueryFactory.java | 95 ++++ .../opensearch/knn/index/query/KNNQuery.java | 66 ++- .../knn/index/query/KNNQueryBuilder.java | 228 +++++++- .../knn/index/query/KNNQueryFactory.java | 120 +---- .../opensearch/knn/index/query/KNNWeight.java | 32 +- .../knn/index/query/RNNQueryFactory.java | 136 +++++ .../org/opensearch/knn/index/util/Faiss.java | 34 +- .../opensearch/knn/index/util/KNNEngine.java | 11 + .../opensearch/knn/index/util/KNNLibrary.java | 20 + .../org/opensearch/knn/index/util/Lucene.java | 33 +- .../org/opensearch/knn/index/util/Nmslib.java | 9 + .../org/opensearch/knn/jni/FaissService.java | 11 + .../org/opensearch/knn/jni/JNIService.java | 23 + .../org/opensearch/knn/index/FaissIT.java | 319 +++++++++++ .../opensearch/knn/index/LuceneEngineIT.java | 205 ++++++- .../knn/index/query/KNNQueryBuilderTests.java | 505 +++++++++++++++++- .../knn/index/query/KNNWeightTests.java | 63 +++ .../knn/index/query/RNNQueryFactoryTests.java | 134 +++++ .../index/util/AbstractKNNLibraryTests.java | 9 + .../knn/index/util/LuceneTests.java | 10 +- .../knn/index/util/NativeLibraryTests.java | 10 + .../LibraryInitializedSupplierTests.java | 10 + .../org/opensearch/knn/KNNRestTestCase.java | 17 + 32 files changed, 2163 insertions(+), 151 deletions(-) create mode 100644 src/main/java/org/opensearch/knn/index/query/BaseQueryFactory.java create mode 100644 src/main/java/org/opensearch/knn/index/query/RNNQueryFactory.java create mode 100644 src/test/java/org/opensearch/knn/index/query/RNNQueryFactoryTests.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 396a86d83..9e7b1c2dc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ## [Unreleased 2.x](https://github.com/opensearch-project/k-NN/compare/2.13...2.x) ### Features * Add Clear Cache API [#740](https://github.com/opensearch-project/k-NN/pull/740) +* Support radial search in k-NN plugin [#1617](https://github.com/opensearch-project/k-NN/pull/1617) ### Enhancements * Make the HitQueue size more appropriate for exact search [#1549](https://github.com/opensearch-project/k-NN/pull/1549) * Support script score when doc value is disabled [#1573](https://github.com/opensearch-project/k-NN/pull/1573) diff --git a/jni/include/faiss_wrapper.h b/jni/include/faiss_wrapper.h index 3e1adeac4..da67c0f59 100644 --- a/jni/include/faiss_wrapper.h +++ b/jni/include/faiss_wrapper.h @@ -73,6 +73,18 @@ namespace knn_jni { // Return the serialized representation jbyteArray TrainIndex(knn_jni::JNIUtilInterface * jniUtil, JNIEnv * env, jobject parametersJ, jint dimension, jlong trainVectorsPointerJ); + + /* + * Perform a range search against the index located in memory at indexPointerJ. + * + * @param indexPointerJ - pointer to the index + * @param queryVectorJ - the query vector + * @param radiusJ - the radius for the range search + * @param maxResultsWindowJ - the maximum number of results to return + * @return an array of RangeQueryResults + */ + jobjectArray RangeSearch(knn_jni::JNIUtilInterface *jniUtil, JNIEnv *env, jlong indexPointerJ, jfloatArray queryVectorJ, + jfloat radiusJ, jint maxResultsWindowJ); } } diff --git a/jni/include/org_opensearch_knn_jni_FaissService.h b/jni/include/org_opensearch_knn_jni_FaissService.h index 32b6f22f1..3715730ab 100644 --- a/jni/include/org_opensearch_knn_jni_FaissService.h +++ b/jni/include/org_opensearch_knn_jni_FaissService.h @@ -122,6 +122,14 @@ JNIEXPORT jbyteArray JNICALL Java_org_opensearch_knn_jni_FaissService_trainIndex JNIEXPORT jlong JNICALL Java_org_opensearch_knn_jni_FaissService_transferVectors (JNIEnv *, jclass, jlong, jobjectArray); +/* +* Class: org_opensearch_knn_jni_FaissService +* Method: rangeSearchIndex +* Signature: (J[F[F)J +*/ +JNIEXPORT jobjectArray JNICALL Java_org_opensearch_knn_jni_FaissService_rangeSearchIndex + (JNIEnv *, jclass, jlong, jfloatArray, jfloat, jint); + #ifdef __cplusplus } #endif diff --git a/jni/src/faiss_wrapper.cpp b/jni/src/faiss_wrapper.cpp index 817bdb816..983cfa8a9 100644 --- a/jni/src/faiss_wrapper.cpp +++ b/jni/src/faiss_wrapper.cpp @@ -587,3 +587,47 @@ faiss::IndexIVFPQ * extractIVFPQIndex(faiss::Index * index) { throw std::runtime_error("Unable to extract IVFPQ index. IVFPQ index not present."); } + +jobjectArray knn_jni::faiss_wrapper::RangeSearch(knn_jni::JNIUtilInterface *jniUtil, JNIEnv *env, jlong indexPointerJ, + jfloatArray queryVectorJ, jfloat radiusJ, jint maxResultWindowJ) { + if (queryVectorJ == nullptr) { + throw std::runtime_error("Query Vector cannot be null"); + } + + auto *indexReader = reinterpret_cast(indexPointerJ); + + if (indexReader == nullptr) { + throw std::runtime_error("Invalid pointer to indexReader"); + } + + float *rawQueryVector = jniUtil->GetFloatArrayElements(env, queryVectorJ, nullptr); + + // The res will be freed by ~RangeSearchResult() in FAISS + // The second parameter is always true, as lims is allocated by FAISS + faiss::RangeSearchResult res(1, true); + indexReader->range_search(1, rawQueryVector, radiusJ, &res); + + // lims is structured to support batched queries, it has a length of nq + 1 (where nq is the number of queries), + // lims[i] - lims[i-1] gives the number of results for the i-th query. With a single query we used in k-NN, + // res.lims[0] is always 0, and res.lims[1] gives the total number of matching entries found. + int resultSize = res.lims[1]; + + // Limit the result size to maxResultWindowJ so that we don't return more than the max result window + // TODO: In the future, we should prevent this via FAISS's ResultHandler. + if (resultSize > maxResultWindowJ) { + resultSize = maxResultWindowJ; + } + + jclass resultClass = jniUtil->FindClass(env,"org/opensearch/knn/index/query/KNNQueryResult"); + jmethodID allArgs = jniUtil->FindMethod(env, "org/opensearch/knn/index/query/KNNQueryResult", ""); + + jobjectArray results = jniUtil->NewObjectArray(env, resultSize, resultClass, nullptr); + + jobject result; + for(int i = 0; i < resultSize; ++i) { + result = jniUtil->NewObject(env, resultClass, allArgs, res.labels[i], res.distances[i]); + jniUtil->SetObjectArrayElement(env, results, i, result); + } + + return results; +} diff --git a/jni/src/org_opensearch_knn_jni_FaissService.cpp b/jni/src/org_opensearch_knn_jni_FaissService.cpp index 3249ed872..ab2a37e84 100644 --- a/jni/src/org_opensearch_knn_jni_FaissService.cpp +++ b/jni/src/org_opensearch_knn_jni_FaissService.cpp @@ -190,3 +190,17 @@ JNIEXPORT jlong JNICALL Java_org_opensearch_knn_jni_FaissService_transferVectors return (jlong) vect; } + +JNIEXPORT jobjectArray JNICALL Java_org_opensearch_knn_jni_FaissService_rangeSearchIndex(JNIEnv * env, jclass cls, + jlong indexPointerJ, + jfloatArray queryVectorJ, + jfloat radiusJ, jint maxResultWindowJ) +{ + try { + return knn_jni::faiss_wrapper::RangeSearch(&jniUtil, env, indexPointerJ, queryVectorJ, radiusJ, maxResultWindowJ); + + } catch (...) { + jniUtil.CatchCppExceptionAndThrowJava(env); + } + return nullptr; +} diff --git a/jni/tests/faiss_wrapper_test.cpp b/jni/tests/faiss_wrapper_test.cpp index 05854f7ed..07b34976f 100644 --- a/jni/tests/faiss_wrapper_test.cpp +++ b/jni/tests/faiss_wrapper_test.cpp @@ -615,3 +615,116 @@ TEST(FaissInitAndSetSharedIndexState, BasicAssertions) { ASSERT_EQ(1, ivfpqIndex->use_precomputed_table); knn_jni::faiss_wrapper::FreeSharedIndexState(sharedModelAddress); } + +TEST(FaissRangeSearchQueryIndexTest, BasicAssertions) { + // Define the index data + faiss::idx_t numIds = 200; + int dim = 2; + std::vector ids = test_util::Range(numIds); + std::vector vectors = test_util::RandomVectors(dim, numIds, randomDataMin, randomDataMax); + + faiss::MetricType metricType = faiss::METRIC_L2; + std::string method = "HNSW32,Flat"; + + // Define query data + float radius = 100000.0; + int numQueries = 2; + std::vector> queries; + + for (int i = 0; i < numQueries; i++) { + std::vector query; + query.reserve(dim); + for (int j = 0; j < dim; j++) { + query.push_back(test_util::RandomFloat(randomDataMin, randomDataMax)); + } + queries.push_back(query); + } + + // Create the index + std::unique_ptr createdIndex( + test_util::FaissCreateIndex(dim, method, metricType)); + auto createdIndexWithData = + test_util::FaissAddData(createdIndex.get(), ids, vectors); + + // Setup jni + JNIEnv *jniEnv = nullptr; + NiceMock mockJNIUtil; + + int maxResultWindow = 20000; + + for (auto query : queries) { + std::unique_ptr *>> results( + reinterpret_cast *> *>( + + knn_jni::faiss_wrapper::RangeSearch( + &mockJNIUtil, jniEnv, + reinterpret_cast(&createdIndexWithData), + reinterpret_cast(&query), radius, maxResultWindow))); + + // assert result size is not 0 + ASSERT_NE(0, results->size()); + + + // Need to free up each result + for (auto it : *results) { + delete it; + } + } +} + +TEST(FaissRangeSearchQueryIndexTest_WhenHitMaxWindowResult, BasicAssertions){ + // Define the index data + faiss::idx_t numIds = 200; + int dim = 2; + std::vector ids = test_util::Range(numIds); + std::vector vectors = test_util::RandomVectors(dim, numIds, randomDataMin, randomDataMax); + + faiss::MetricType metricType = faiss::METRIC_L2; + std::string method = "HNSW32,Flat"; + + // Define query data + float radius = 100000.0; + int numQueries = 2; + std::vector> queries; + + for (int i = 0; i < numQueries; i++) { + std::vector query; + query.reserve(dim); + for (int j = 0; j < dim; j++) { + query.push_back(test_util::RandomFloat(randomDataMin, randomDataMax)); + } + queries.push_back(query); + } + + // Create the index + std::unique_ptr createdIndex( + test_util::FaissCreateIndex(dim, method, metricType)); + auto createdIndexWithData = + test_util::FaissAddData(createdIndex.get(), ids, vectors); + + // Setup jni + JNIEnv *jniEnv = nullptr; + NiceMock mockJNIUtil; + + int maxResultWindow = 10; + + for (auto query : queries) { + std::unique_ptr *>> results( + reinterpret_cast *> *>( + + knn_jni::faiss_wrapper::RangeSearch( + &mockJNIUtil, jniEnv, + reinterpret_cast(&createdIndexWithData), + reinterpret_cast(&query), radius, maxResultWindow))); + + // assert result size is not 0 + ASSERT_NE(0, results->size()); + // assert result size is equal to maxResultWindow + ASSERT_EQ(maxResultWindow, results->size()); + + // Need to free up each result + for (auto it : *results) { + delete it; + } + } +} diff --git a/src/main/java/org/opensearch/knn/common/KNNConstants.java b/src/main/java/org/opensearch/knn/common/KNNConstants.java index a2e539375..2c3b03e47 100644 --- a/src/main/java/org/opensearch/knn/common/KNNConstants.java +++ b/src/main/java/org/opensearch/knn/common/KNNConstants.java @@ -67,6 +67,8 @@ public class KNNConstants { public static final String VECTOR_DATA_TYPE_FIELD = "data_type"; public static final VectorDataType DEFAULT_VECTOR_DATA_TYPE_FIELD = VectorDataType.FLOAT; + public static final String RADIAL_SEARCH_KEY = "radial_search"; + // Lucene specific constants public static final String LUCENE_NAME = "lucene"; diff --git a/src/main/java/org/opensearch/knn/index/IndexUtil.java b/src/main/java/org/opensearch/knn/index/IndexUtil.java index bb9bbac90..92b94c2e2 100644 --- a/src/main/java/org/opensearch/knn/index/IndexUtil.java +++ b/src/main/java/org/opensearch/knn/index/IndexUtil.java @@ -44,12 +44,14 @@ public class IndexUtil { private static final Version MINIMAL_SUPPORTED_VERSION_FOR_IGNORE_UNMAPPED = Version.V_2_11_0; private static final Version MINIMAL_SUPPORTED_VERSION_FOR_MODEL_NODE_ASSIGNMENT = Version.V_2_12_0; private static final Version MINIMAL_SUPPORTED_VERSION_FOR_MODEL_METHOD_COMPONENT_CONTEXT = Version.V_2_13_0; + private static final Version MINIMAL_SUPPORTED_VERSION_FOR_RADIAL_SEARCH = Version.V_2_14_0; public static final Map minimalRequiredVersionMap = new HashMap() { { put("filter", MINIMAL_SUPPORTED_VERSION_FOR_LUCENE_HNSW_FILTER); put("ignore_unmapped", MINIMAL_SUPPORTED_VERSION_FOR_IGNORE_UNMAPPED); put(MODEL_NODE_ASSIGNMENT_KEY, MINIMAL_SUPPORTED_VERSION_FOR_MODEL_NODE_ASSIGNMENT); put(MODEL_METHOD_COMPONENT_CONTEXT_KEY, MINIMAL_SUPPORTED_VERSION_FOR_MODEL_METHOD_COMPONENT_CONTEXT); + put(KNNConstants.RADIAL_SEARCH_KEY, MINIMAL_SUPPORTED_VERSION_FOR_RADIAL_SEARCH); } }; diff --git a/src/main/java/org/opensearch/knn/index/SpaceType.java b/src/main/java/org/opensearch/knn/index/SpaceType.java index 50d8d352c..240bfbe91 100644 --- a/src/main/java/org/opensearch/knn/index/SpaceType.java +++ b/src/main/java/org/opensearch/knn/index/SpaceType.java @@ -36,6 +36,14 @@ public float scoreTranslation(float rawScore) { public VectorSimilarityFunction getVectorSimilarityFunction() { return VectorSimilarityFunction.EUCLIDEAN; } + + @Override + public float scoreToDistanceTranslation(float score) { + if (score == 0) { + throw new IllegalArgumentException(String.format(Locale.ROOT, "score cannot be 0 when space type is [%s]", getValue())); + } + return 1 / score - 1; + } }, COSINESIMIL("cosinesimil") { @Override @@ -170,4 +178,14 @@ public static SpaceType getSpace(String spaceTypeName) { } throw new IllegalArgumentException("Unable to find space: " + spaceTypeName); } + + /** + * Translate a score to a distance for this space type + * + * @param score score to translate + * @return translated distance + */ + public float scoreToDistanceTranslation(float score) { + throw new UnsupportedOperationException(String.format("Space [%s] does not have a score to distance translation", getValue())); + } } diff --git a/src/main/java/org/opensearch/knn/index/query/BaseQueryFactory.java b/src/main/java/org/opensearch/knn/index/query/BaseQueryFactory.java new file mode 100644 index 000000000..3146cd33e --- /dev/null +++ b/src/main/java/org/opensearch/knn/index/query/BaseQueryFactory.java @@ -0,0 +1,95 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.query; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Getter; +import lombok.NonNull; +import lombok.extern.log4j.Log4j2; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.join.BitSetProducer; +import org.apache.lucene.search.join.ToChildBlockJoinQuery; +import org.opensearch.index.query.QueryBuilder; +import org.opensearch.index.query.QueryShardContext; +import org.opensearch.index.search.NestedHelper; +import org.opensearch.knn.index.VectorDataType; +import org.opensearch.knn.index.util.KNNEngine; + +import java.io.IOException; +import java.util.Optional; + +/** +* Base class for creating vector search queries. +*/ +@Log4j2 +public abstract class BaseQueryFactory { + /** + * DTO object to hold data required to create a Query instance. + */ + @AllArgsConstructor + @Builder + @Getter + public static class CreateQueryRequest { + @NonNull + private KNNEngine knnEngine; + @NonNull + private String indexName; + private String fieldName; + private float[] vector; + private byte[] byteVector; + private VectorDataType vectorDataType; + private Integer k; + private Float radius; + private QueryBuilder filter; + private QueryShardContext context; + + public Optional getFilter() { + return Optional.ofNullable(filter); + } + + public Optional getContext() { + return Optional.ofNullable(context); + } + } + + /** + * Creates a query filter. + * + * @param createQueryRequest request object that has all required fields to construct the query + * @return Lucene Query + */ + protected static Query getFilterQuery(BaseQueryFactory.CreateQueryRequest createQueryRequest) { + if (!createQueryRequest.getFilter().isPresent()) { + return null; + } + + final QueryShardContext queryShardContext = createQueryRequest.getContext() + .orElseThrow(() -> new RuntimeException("Shard context cannot be null")); + log.debug( + String.format( + "Creating query with filter for index [%s], field [%s]", + createQueryRequest.getIndexName(), + createQueryRequest.getFieldName() + ) + ); + final Query filterQuery; + try { + filterQuery = createQueryRequest.getFilter().get().toQuery(queryShardContext); + } catch (IOException e) { + throw new RuntimeException("Cannot create query with filter", e); + } + BitSetProducer parentFilter = queryShardContext.getParentFilter(); + if (parentFilter != null) { + boolean mightMatch = new NestedHelper(queryShardContext.getMapperService()).mightMatchNestedDocs(filterQuery); + if (mightMatch) { + return filterQuery; + } + return new ToChildBlockJoinQuery(filterQuery, parentFilter); + } + return filterQuery; + } +} diff --git a/src/main/java/org/opensearch/knn/index/query/KNNQuery.java b/src/main/java/org/opensearch/knn/index/query/KNNQuery.java index 9c78b18a1..0862b2d93 100644 --- a/src/main/java/org/opensearch/knn/index/query/KNNQuery.java +++ b/src/main/java/org/opensearch/knn/index/query/KNNQuery.java @@ -7,6 +7,8 @@ import java.util.Arrays; import java.util.Objects; + +import lombok.AllArgsConstructor; import lombok.Getter; import lombok.Setter; import org.apache.lucene.search.BooleanClause; @@ -30,7 +32,7 @@ public class KNNQuery extends Query { private final String field; private final float[] queryVector; - private final int k; + private int k; private final String indexName; @Getter @@ -38,6 +40,10 @@ public class KNNQuery extends Query { private Query filterQuery; @Getter private BitSetProducer parentsFilter; + @Getter + private Float radius = null; + @Getter + private Context context; public KNNQuery( final String field, @@ -69,6 +75,54 @@ public KNNQuery( this.parentsFilter = parentsFilter; } + /** + * Constructor for KNNQuery with query vector, index name and parent filter + * + * @param field field name + * @param queryVector query vector + * @param indexName index name + * @param parentsFilter parent filter + */ + public KNNQuery(String field, float[] queryVector, String indexName, BitSetProducer parentsFilter) { + this.field = field; + this.queryVector = queryVector; + this.indexName = indexName; + this.parentsFilter = parentsFilter; + } + + /** + * Constructor for KNNQuery with radius + * + * @param radius engine radius + * @return KNNQuery + */ + public KNNQuery radius(Float radius) { + this.radius = radius; + return this; + } + + /** + * Constructor for KNNQuery with Context + * + * @param context Context for KNNQuery + * @return KNNQuery + */ + public KNNQuery kNNQueryContext(Context context) { + this.context = context; + return this; + } + + /** + * Constructor for KNNQuery with filter query + * + * @param filterQuery filter query + * @return KNNQuery + */ + public KNNQuery filterQuery(Query filterQuery) { + this.filterQuery = filterQuery; + return this; + } + public String getField() { return this.field; } @@ -144,4 +198,14 @@ private boolean equalsTo(KNNQuery other) { && Objects.equals(indexName, other.indexName) && Objects.equals(filterQuery, other.filterQuery); } + + /** + * Context for KNNQuery + */ + @Setter + @Getter + @AllArgsConstructor + public static class Context { + int maxResultWindow; + } } diff --git a/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java b/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java index 7287ce3c6..760a91815 100644 --- a/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java +++ b/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java @@ -11,6 +11,7 @@ import java.util.Objects; import lombok.extern.log4j.Log4j2; import org.apache.lucene.search.MatchNoDocsQuery; +import org.opensearch.core.common.Strings; import org.apache.commons.lang.StringUtils; import org.apache.lucene.search.Query; import org.opensearch.core.ParseField; @@ -21,9 +22,8 @@ import org.opensearch.core.xcontent.XContentParser; import org.opensearch.index.mapper.MappedFieldType; import org.opensearch.index.mapper.NumberFieldMapper; -import org.opensearch.index.query.AbstractQueryBuilder; import org.opensearch.index.query.QueryBuilder; -import org.opensearch.index.query.QueryShardContext; +import org.opensearch.knn.common.KNNConstants; import org.opensearch.knn.index.KNNMethodContext; import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.VectorDataType; @@ -32,10 +32,13 @@ import org.opensearch.knn.indices.ModelDao; import org.opensearch.knn.indices.ModelMetadata; import org.opensearch.knn.plugin.stats.KNNCounter; +import org.opensearch.index.query.AbstractQueryBuilder; +import org.opensearch.index.query.QueryShardContext; import static org.opensearch.knn.index.IndexUtil.*; import static org.opensearch.knn.common.KNNValidationUtil.validateByteVectorValue; import static org.opensearch.knn.index.IndexUtil.isClusterOnOrAfterMinRequiredVersion; +import static org.opensearch.knn.index.util.KNNEngine.ENGINES_SUPPORTING_RADIAL_SEARCH; /** * Helper class to build the KNN query @@ -48,6 +51,8 @@ public class KNNQueryBuilder extends AbstractQueryBuilder { public static final ParseField K_FIELD = new ParseField("k"); public static final ParseField FILTER_FIELD = new ParseField("filter"); public static final ParseField IGNORE_UNMAPPED_FIELD = new ParseField("ignore_unmapped"); + public static final ParseField MAX_DISTANCE_FIELD = new ParseField("max_distance"); + public static final ParseField MIN_SCORE_FIELD = new ParseField("min_score"); public static final int K_MAX = 10000; /** * The name for the knn query @@ -59,11 +64,91 @@ public class KNNQueryBuilder extends AbstractQueryBuilder { private final String fieldName; private final float[] vector; private int k = 0; + private Float max_distance = null; + private Float min_score = null; private QueryBuilder filter; private boolean ignoreUnmapped = false; /** - * Constructs a new knn query + * Constructs a new query with the given field name and vector + * + * @param fieldName Name of the field + * @param vector Array of floating points + */ + public KNNQueryBuilder(String fieldName, float[] vector) { + if (Strings.isNullOrEmpty(fieldName)) { + throw new IllegalArgumentException("[" + NAME + "] requires fieldName"); + } + if (vector == null) { + throw new IllegalArgumentException("[" + NAME + "] requires query vector"); + } + if (vector.length == 0) { + throw new IllegalArgumentException("[" + NAME + "] query vector is empty"); + } + this.fieldName = fieldName; + this.vector = vector; + } + + /** + * Builder method for k + * + * @param k K nearest neighbours for the given vector + */ + public KNNQueryBuilder k(Integer k) { + if (k == null) { + throw new IllegalArgumentException("[" + NAME + "] requires k to be set"); + } + validateSingleQueryType(k, max_distance, min_score); + if (k <= 0 || k > K_MAX) { + throw new IllegalArgumentException("[" + NAME + "] requires 0 < k <= " + K_MAX); + } + this.k = k; + return this; + } + + /** + * Builder method for max_distance + * + * @param max_distance the max_distance threshold for the nearest neighbours + */ + public KNNQueryBuilder maxDistance(Float max_distance) { + if (max_distance == null) { + throw new IllegalArgumentException("[" + NAME + "] requires max_distance to be set"); + } + validateSingleQueryType(k, max_distance, min_score); + this.max_distance = max_distance; + return this; + } + + /** + * Builder method for min_score + * + * @param min_score the min_score threshold for the nearest neighbours + */ + public KNNQueryBuilder minScore(Float min_score) { + if (min_score == null) { + throw new IllegalArgumentException("[" + NAME + "] requires min_score to be set"); + } + validateSingleQueryType(k, max_distance, min_score); + if (min_score <= 0) { + throw new IllegalArgumentException("[" + NAME + "] requires min_score greater than 0"); + } + this.min_score = min_score; + return this; + } + + /** + * Builder method for filter + * + * @param filter QueryBuilder + */ + public KNNQueryBuilder filter(QueryBuilder filter) { + this.filter = filter; + return this; + } + + /** + * Constructs a new query for top k search * * @param fieldName Name of the filed * @param vector Array of floating points @@ -95,6 +180,8 @@ public KNNQueryBuilder(String fieldName, float[] vector, int k, QueryBuilder fil this.k = k; this.filter = filter; this.ignoreUnmapped = false; + this.max_distance = null; + this.min_score = null; } public static void initialize(ModelDao modelDao) { @@ -133,6 +220,12 @@ public KNNQueryBuilder(StreamInput in) throws IOException { if (isClusterOnOrAfterMinRequiredVersion("ignore_unmapped")) { ignoreUnmapped = in.readOptionalBoolean(); } + if (isClusterOnOrAfterMinRequiredVersion(KNNConstants.RADIAL_SEARCH_KEY)) { + max_distance = in.readOptionalFloat(); + } + if (isClusterOnOrAfterMinRequiredVersion(KNNConstants.RADIAL_SEARCH_KEY)) { + min_score = in.readOptionalFloat(); + } } catch (IOException ex) { throw new RuntimeException("[KNN] Unable to create KNNQueryBuilder", ex); } @@ -142,7 +235,9 @@ public static KNNQueryBuilder fromXContent(XContentParser parser) throws IOExcep String fieldName = null; List vector = null; float boost = AbstractQueryBuilder.DEFAULT_BOOST; - int k = 0; + Integer k = null; + Float max_distance = null; + Float min_score = null; QueryBuilder filter = null; boolean ignoreUnmapped = false; String queryName = null; @@ -167,6 +262,10 @@ public static KNNQueryBuilder fromXContent(XContentParser parser) throws IOExcep k = (Integer) NumberFieldMapper.NumberType.INTEGER.parse(parser.objectBytes(), false); } else if (AbstractQueryBuilder.NAME_FIELD.match(currentFieldName, parser.getDeprecationHandler())) { queryName = parser.text(); + } else if (MAX_DISTANCE_FIELD.match(currentFieldName, parser.getDeprecationHandler())) { + max_distance = (Float) NumberFieldMapper.NumberType.FLOAT.parse(parser.objectBytes(), false); + } else if (MIN_SCORE_FIELD.match(currentFieldName, parser.getDeprecationHandler())) { + min_score = (Float) NumberFieldMapper.NumberType.FLOAT.parse(parser.objectBytes(), false); } else if (IGNORE_UNMAPPED_FIELD.getPreferredName().equals("ignore_unmapped")) { if (isClusterOnOrAfterMinRequiredVersion("ignore_unmapped")) { ignoreUnmapped = parser.booleanValue(); @@ -220,12 +319,24 @@ public static KNNQueryBuilder fromXContent(XContentParser parser) throws IOExcep } } - KNNQueryBuilder knnQueryBuilder = new KNNQueryBuilder(fieldName, ObjectsToFloats(vector), k, filter); - knnQueryBuilder.queryName(queryName); + validateSingleQueryType(k, max_distance, min_score); + + KNNQueryBuilder knnQueryBuilder = new KNNQueryBuilder(fieldName, ObjectsToFloats(vector)).filter(filter) + .boost(boost) + .queryName(queryName); + if (isClusterOnOrAfterMinRequiredVersion("ignoreUnmapped")) { knnQueryBuilder.ignoreUnmapped(ignoreUnmapped); } - knnQueryBuilder.boost(boost); + + if (k != null) { + knnQueryBuilder.k(k); + } else if (max_distance != null) { + knnQueryBuilder.maxDistance(max_distance); + } else if (min_score != null) { + knnQueryBuilder.minScore(min_score); + } + return knnQueryBuilder; } @@ -242,6 +353,12 @@ protected void doWriteTo(StreamOutput out) throws IOException { if (isClusterOnOrAfterMinRequiredVersion("ignore_unmapped")) { out.writeOptionalBoolean(ignoreUnmapped); } + if (isClusterOnOrAfterMinRequiredVersion(KNNConstants.RADIAL_SEARCH_KEY)) { + out.writeOptionalFloat(max_distance); + } + if (isClusterOnOrAfterMinRequiredVersion(KNNConstants.RADIAL_SEARCH_KEY)) { + out.writeOptionalFloat(min_score); + } } /** @@ -262,6 +379,14 @@ public int getK() { return this.k; } + public float getMaxDistance() { + return this.max_distance; + } + + public float getMinScore() { + return this.min_score; + } + public QueryBuilder getFilter() { return this.filter; } @@ -290,9 +415,15 @@ public void doXContent(XContentBuilder builder, Params params) throws IOExceptio if (filter != null) { builder.field(FILTER_FIELD.getPreferredName(), filter); } + if (max_distance != null) { + builder.field(MAX_DISTANCE_FIELD.getPreferredName(), max_distance); + } if (ignoreUnmapped) { builder.field(IGNORE_UNMAPPED_FIELD.getPreferredName(), ignoreUnmapped); } + if (min_score != null) { + builder.field(MIN_SCORE_FIELD.getPreferredName(), min_score); + } printBoostAndQueryName(builder); builder.endObject(); builder.endObject(); @@ -329,6 +460,24 @@ protected Query doToQuery(QueryShardContext context) { } else if (knnMethodContext != null) { // If the dimension is set but the knnMethodContext is not then the field is using the legacy mapping knnEngine = knnMethodContext.getKnnEngine(); + spaceType = knnMethodContext.getSpaceType(); + } + + // Currently, k-NN supports distance and score types radial search + // We need transform distance/score to right type of engine required radius. + Float radius = null; + if (this.max_distance != null) { + if (this.max_distance < 0 && SpaceType.INNER_PRODUCT.equals(spaceType) == false) { + throw new IllegalArgumentException("[" + NAME + "] requires distance to be non-negative for space type: " + spaceType); + } + radius = knnEngine.distanceToRadialThreshold(this.max_distance, spaceType); + } + + if (this.min_score != null) { + if (this.min_score > 1 && SpaceType.INNER_PRODUCT.equals(spaceType) == false) { + throw new IllegalArgumentException("[" + NAME + "] requires score to be in the range (0, 1] for space type: " + spaceType); + } + radius = knnEngine.scoreToRadialThreshold(this.min_score, spaceType); } if (fieldDimension != vector.length) { @@ -356,18 +505,39 @@ protected Query doToQuery(QueryShardContext context) { } String indexName = context.index().getName(); - KNNQueryFactory.CreateQueryRequest createQueryRequest = KNNQueryFactory.CreateQueryRequest.builder() - .knnEngine(knnEngine) - .indexName(indexName) - .fieldName(this.fieldName) - .vector(VectorDataType.FLOAT == vectorDataType ? this.vector : null) - .byteVector(VectorDataType.BYTE == vectorDataType ? byteVector : null) - .vectorDataType(vectorDataType) - .k(this.k) - .filter(this.filter) - .context(context) - .build(); - return KNNQueryFactory.create(createQueryRequest); + + if (k != 0) { + KNNQueryFactory.CreateQueryRequest createQueryRequest = KNNQueryFactory.CreateQueryRequest.builder() + .knnEngine(knnEngine) + .indexName(indexName) + .fieldName(this.fieldName) + .vector(VectorDataType.FLOAT == vectorDataType ? this.vector : null) + .byteVector(VectorDataType.BYTE == vectorDataType ? byteVector : null) + .vectorDataType(vectorDataType) + .k(this.k) + .filter(this.filter) + .context(context) + .build(); + return KNNQueryFactory.create(createQueryRequest); + } + if (radius != null) { + if (!ENGINES_SUPPORTING_RADIAL_SEARCH.contains(knnEngine)) { + throw new UnsupportedOperationException(String.format("Engine [%s] does not support radial search", knnEngine)); + } + RNNQueryFactory.CreateQueryRequest createQueryRequest = RNNQueryFactory.CreateQueryRequest.builder() + .knnEngine(knnEngine) + .indexName(indexName) + .fieldName(this.fieldName) + .vector(VectorDataType.FLOAT == vectorDataType ? this.vector : null) + .byteVector(VectorDataType.BYTE == vectorDataType ? byteVector : null) + .vectorDataType(vectorDataType) + .radius(radius) + .filter(this.filter) + .context(context) + .build(); + return RNNQueryFactory.create(createQueryRequest); + } + throw new IllegalArgumentException("[" + NAME + "] requires either k or distance or score to be set"); } private ModelMetadata getModelMetadataForField(KNNVectorFieldMapper.KNNVectorFieldType knnVectorField) { @@ -402,4 +572,24 @@ protected int doHashCode() { public String getWriteableName() { return NAME; } + + private static void validateSingleQueryType(Integer k, Float distance, Float score) { + int countSetFields = 0; + + if (k != null && k != 0) { + countSetFields++; + } + if (distance != null) { + countSetFields++; + } + if (score != null) { + countSetFields++; + } + + if (countSetFields != 1) { + throw new IllegalArgumentException( + "[" + NAME + "] requires only one query type to be set, it can be either k, distance, or score" + ); + } + } } diff --git a/src/main/java/org/opensearch/knn/index/query/KNNQueryFactory.java b/src/main/java/org/opensearch/knn/index/query/KNNQueryFactory.java index 2ab0e62af..ec1f53d13 100644 --- a/src/main/java/org/opensearch/knn/index/query/KNNQueryFactory.java +++ b/src/main/java/org/opensearch/knn/index/query/KNNQueryFactory.java @@ -5,11 +5,6 @@ package org.opensearch.knn.index.query; -import lombok.AllArgsConstructor; -import lombok.Builder; -import lombok.Getter; -import lombok.NonNull; -import lombok.Setter; import lombok.extern.log4j.Log4j2; import org.apache.lucene.search.KnnByteVectorQuery; import org.apache.lucene.search.KnnFloatVectorQuery; @@ -17,16 +12,11 @@ import org.apache.lucene.search.join.BitSetProducer; import org.apache.lucene.search.join.DiversifyingChildrenByteKnnVectorQuery; import org.apache.lucene.search.join.DiversifyingChildrenFloatKnnVectorQuery; -import org.apache.lucene.search.join.ToChildBlockJoinQuery; -import org.opensearch.index.query.QueryBuilder; import org.opensearch.index.query.QueryShardContext; -import org.opensearch.index.search.NestedHelper; import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.util.KNNEngine; -import java.io.IOException; import java.util.Locale; -import java.util.Optional; import static org.opensearch.knn.common.KNNConstants.VECTOR_DATA_TYPE_FIELD; import static org.opensearch.knn.index.VectorDataType.SUPPORTED_VECTOR_DATA_TYPES; @@ -35,7 +25,7 @@ * Creates the Lucene k-NN queries */ @Log4j2 -public class KNNQueryFactory { +public class KNNQueryFactory extends BaseQueryFactory { /** * Creates a Lucene query for a particular engine. @@ -82,7 +72,12 @@ public static Query create(CreateQueryRequest createQueryRequest) { final VectorDataType vectorDataType = createQueryRequest.getVectorDataType(); final Query filterQuery = getFilterQuery(createQueryRequest); - BitSetProducer parentFilter = createQueryRequest.context == null ? null : createQueryRequest.context.getParentFilter(); + BitSetProducer parentFilter = null; + if (createQueryRequest.getContext().isPresent()) { + QueryShardContext context = createQueryRequest.getContext().get(); + parentFilter = context.getParentFilter(); + } + if (KNNEngine.getEnginesThatCreateCustomSegmentFiles().contains(createQueryRequest.getKnnEngine())) { if (filterQuery != null && KNNEngine.getEnginesThatSupportsFilters().contains(createQueryRequest.getKnnEngine())) { log.debug("Creating custom k-NN query with filters for index: {}, field: {} , k: {}", indexName, fieldName, k); @@ -93,19 +88,21 @@ public static Query create(CreateQueryRequest createQueryRequest) { } log.debug(String.format("Creating Lucene k-NN query for index: %s \"\", field: %s \"\", k: %d", indexName, fieldName, k)); - if (VectorDataType.BYTE == vectorDataType) { - return getKnnByteVectorQuery(fieldName, byteVector, k, filterQuery, parentFilter); - } else if (VectorDataType.FLOAT == vectorDataType) { - return getKnnFloatVectorQuery(fieldName, vector, k, filterQuery, parentFilter); - } else { - throw new IllegalArgumentException( - String.format( - Locale.ROOT, - "Invalid value provided for [%s] field. Supported values are [%s]", - VECTOR_DATA_TYPE_FIELD, - SUPPORTED_VECTOR_DATA_TYPES - ) - ); + switch (vectorDataType) { + case BYTE: + return getKnnByteVectorQuery(fieldName, byteVector, k, filterQuery, parentFilter); + case FLOAT: + return getKnnFloatVectorQuery(fieldName, vector, k, filterQuery, parentFilter); + default: + throw new IllegalArgumentException( + String.format( + Locale.ROOT, + "Invalid value provided for [%s] field. Supported values are [%s], but got: %s", + VECTOR_DATA_TYPE_FIELD, + SUPPORTED_VECTOR_DATA_TYPES, + vectorDataType + ) + ); } } @@ -144,77 +141,4 @@ private static Query getKnnFloatVectorQuery( return new DiversifyingChildrenFloatKnnVectorQuery(fieldName, floatVector, filterQuery, k, parentFilter); } } - - private static Query getFilterQuery(CreateQueryRequest createQueryRequest) { - if (createQueryRequest.getFilter().isPresent()) { - final QueryShardContext queryShardContext = createQueryRequest.getContext() - .orElseThrow(() -> new RuntimeException("Shard context cannot be null")); - log.debug( - String.format( - "Creating k-NN query with filter for index [%s], field [%s] and k [%d]", - createQueryRequest.getIndexName(), - createQueryRequest.fieldName, - createQueryRequest.k - ) - ); - final Query filterQuery; - try { - filterQuery = createQueryRequest.getFilter().get().toQuery(queryShardContext); - } catch (IOException e) { - throw new RuntimeException("Cannot create knn query with filter", e); - } - // If k-NN Field is nested field then parentFilter will not be null. This parentFilter is set by the - // Opensearch core. Ref PR: https://github.com/opensearch-project/OpenSearch/pull/10246 - if (queryShardContext.getParentFilter() != null) { - // if the filter is also a nested query clause then we should just return the same query without - // considering it to join with the parent documents. - if (new NestedHelper(queryShardContext.getMapperService()).mightMatchNestedDocs(filterQuery)) { - return filterQuery; - } - // This condition will be hit when filters are getting applied on the top level fields and k-nn - // query field is a nested field. In this case we need to wrap the filter query with - // ToChildBlockJoinQuery to ensure parent documents which will be retrieved from filters can be - // joined with the child documents containing vector field. - return new ToChildBlockJoinQuery(filterQuery, queryShardContext.getParentFilter()); - } - return filterQuery; - } - return null; - } - - /** - * DTO object to hold data required to create a Query instance. - */ - @AllArgsConstructor - @Builder - @Setter - static class CreateQueryRequest { - @Getter - @NonNull - private KNNEngine knnEngine; - @Getter - @NonNull - private String indexName; - @Getter - private String fieldName; - @Getter - private float[] vector; - @Getter - private byte[] byteVector; - @Getter - private VectorDataType vectorDataType; - @Getter - private int k; - private QueryBuilder filter; - // can be null in cases filter not passed with the knn query - private QueryShardContext context; - - public Optional getFilter() { - return Optional.ofNullable(filter); - } - - public Optional getContext() { - return Optional.ofNullable(context); - } - } } diff --git a/src/main/java/org/opensearch/knn/index/query/KNNWeight.java b/src/main/java/org/opensearch/knn/index/query/KNNWeight.java index 06bf96d63..6b323e124 100644 --- a/src/main/java/org/opensearch/knn/index/query/KNNWeight.java +++ b/src/main/java/org/opensearch/knn/index/query/KNNWeight.java @@ -277,16 +277,25 @@ private Map doANNSearch(final LeafReaderContext context, final B throw new RuntimeException("Index has already been closed"); } int[] parentIds = getParentIdsArray(context); - results = JNIService.queryIndex( - indexAllocation.getMemoryAddress(), - knnQuery.getQueryVector(), - knnQuery.getK(), - knnEngine, - filterIds, - filterType.getValue(), - parentIds - ); - + if (knnQuery.getK() > 0) { + results = JNIService.queryIndex( + indexAllocation.getMemoryAddress(), + knnQuery.getQueryVector(), + knnQuery.getK(), + knnEngine, + filterIds, + filterType.getValue(), + parentIds + ); + } else { + results = JNIService.radiusQueryIndex( + indexAllocation.getMemoryAddress(), + knnQuery.getQueryVector(), + knnQuery.getRadius(), + knnEngine, + knnQuery.getContext().getMaxResultWindow() + ); + } } catch (Exception e) { GRAPH_QUERY_ERRORS.increment(); throw new RuntimeException(e); @@ -406,6 +415,9 @@ private boolean canDoExactSearch(final int filterIdsCount) { filterIdsCount, KNNSettings.getFilteredExactSearchThreshold(knnQuery.getIndexName()) ); + if (knnQuery.getRadius() != null) { + return false; + } int filterThresholdValue = KNNSettings.getFilteredExactSearchThreshold(knnQuery.getIndexName()); // Refer this GitHub around more details https://github.com/opensearch-project/k-NN/issues/1049 on the logic if (filterIdsCount <= knnQuery.getK()) { diff --git a/src/main/java/org/opensearch/knn/index/query/RNNQueryFactory.java b/src/main/java/org/opensearch/knn/index/query/RNNQueryFactory.java new file mode 100644 index 000000000..cd32ac4f3 --- /dev/null +++ b/src/main/java/org/opensearch/knn/index/query/RNNQueryFactory.java @@ -0,0 +1,136 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.query; + +import static org.opensearch.knn.common.KNNConstants.VECTOR_DATA_TYPE_FIELD; +import static org.opensearch.knn.index.VectorDataType.SUPPORTED_VECTOR_DATA_TYPES; + +import java.util.Locale; + +import lombok.extern.log4j.Log4j2; +import org.apache.lucene.search.ByteVectorSimilarityQuery; +import org.apache.lucene.search.FloatVectorSimilarityQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.join.BitSetProducer; +import org.opensearch.index.IndexSettings; +import org.opensearch.index.query.QueryShardContext; +import org.opensearch.knn.index.VectorDataType; +import org.opensearch.knn.index.util.KNNEngine; + +/** + * Class to create radius nearest neighbor queries + */ +@Log4j2 +public class RNNQueryFactory extends BaseQueryFactory { + + /** + * Creates a Lucene query for a particular engine. + * + * @param knnEngine Engine to create the query for + * @param indexName Name of the OpenSearch index that is being queried + * @param fieldName Name of the field in the OpenSearch index that will be queried + * @param vector The query vector to get the nearest neighbors for + * @param radius the radius threshold for the nearest neighbors + * @return Lucene Query + */ + public static Query create( + KNNEngine knnEngine, + String indexName, + String fieldName, + float[] vector, + Float radius, + VectorDataType vectorDataType + ) { + final CreateQueryRequest createQueryRequest = CreateQueryRequest.builder() + .knnEngine(knnEngine) + .indexName(indexName) + .fieldName(fieldName) + .vector(vector) + .vectorDataType(vectorDataType) + .radius(radius) + .build(); + return create(createQueryRequest); + } + + /** + * Creates a Lucene query for a particular engine. + * @param createQueryRequest request object that has all required fields to construct the query + * @return Lucene Query + */ + public static Query create(RNNQueryFactory.CreateQueryRequest createQueryRequest) { + final String indexName = createQueryRequest.getIndexName(); + final String fieldName = createQueryRequest.getFieldName(); + final Float radius = createQueryRequest.getRadius(); + final float[] vector = createQueryRequest.getVector(); + final byte[] byteVector = createQueryRequest.getByteVector(); + final VectorDataType vectorDataType = createQueryRequest.getVectorDataType(); + final Query filterQuery = getFilterQuery(createQueryRequest); + + if (KNNEngine.getEnginesThatCreateCustomSegmentFiles().contains(createQueryRequest.getKnnEngine())) { + BitSetProducer parentFilter = null; + QueryShardContext context = createQueryRequest.getContext().get(); + + if (createQueryRequest.getContext().isPresent()) { + parentFilter = context.getParentFilter(); + } + IndexSettings indexSettings = context.getIndexSettings(); + KNNQuery.Context knnQueryContext = new KNNQuery.Context(indexSettings.getMaxResultWindow()); + KNNQuery rnnQuery = new KNNQuery(fieldName, vector, indexName, parentFilter).radius(radius).kNNQueryContext(knnQueryContext); + if (filterQuery != null && KNNEngine.getEnginesThatSupportsFilters().contains(createQueryRequest.getKnnEngine())) { + log.debug("Creating custom radius search with filters for index: {}, field: {} , r: {}", indexName, fieldName, radius); + rnnQuery.filterQuery(filterQuery); + } + log.debug( + String.format("Creating custom radius search for index: %s \"\", field: %s \"\", r: %f", indexName, fieldName, radius) + ); + return rnnQuery; + } + + log.debug(String.format("Creating Lucene r-NN query for index: %s \"\", field: %s \"\", k: %f", indexName, fieldName, radius)); + switch (vectorDataType) { + case BYTE: + return getByteVectorSimilarityQuery(fieldName, byteVector, radius, filterQuery); + case FLOAT: + return getFloatVectorSimilarityQuery(fieldName, vector, radius, filterQuery); + default: + throw new IllegalArgumentException( + String.format( + Locale.ROOT, + "Invalid value provided for [%s] field. Supported values are [%s], but got: %s", + VECTOR_DATA_TYPE_FIELD, + SUPPORTED_VECTOR_DATA_TYPES, + vectorDataType + ) + ); + } + } + + /** + * If radius is greater than 0, we return {@link FloatVectorSimilarityQuery} which will return all documents with similarity + * greater than or equal to the resultSimilarity. If filterQuery is not null, it will be used to filter the documents. + */ + private static Query getFloatVectorSimilarityQuery( + final String fieldName, + final float[] floatVector, + final float resultSimilarity, + final Query filterQuery + ) { + return new FloatVectorSimilarityQuery(fieldName, floatVector, resultSimilarity, filterQuery); + } + + /** + * If radius is greater than 0, we return {@link ByteVectorSimilarityQuery} which will return all documents with similarity + * greater than or equal to the resultSimilarity. If filterQuery is not null, it will be used to filter the documents. + */ + private static Query getByteVectorSimilarityQuery( + final String fieldName, + final byte[] byteVector, + final float resultSimilarity, + final Query filterQuery + ) { + return new ByteVectorSimilarityQuery(fieldName, byteVector, resultSimilarity, filterQuery); + } +} diff --git a/src/main/java/org/opensearch/knn/index/util/Faiss.java b/src/main/java/org/opensearch/knn/index/util/Faiss.java index 563311c49..efd8a637c 100644 --- a/src/main/java/org/opensearch/knn/index/util/Faiss.java +++ b/src/main/java/org/opensearch/knn/index/util/Faiss.java @@ -56,6 +56,7 @@ * Implements NativeLibrary for the faiss native library */ class Faiss extends NativeLibrary { + Map> scoreTransform; // TODO: Current version is not really current version. Instead, it encodes information in the file name // about the compatibility version the file is created with. In the future, we should refactor this so that it @@ -68,6 +69,12 @@ class Faiss extends NativeLibrary { rawScore -> SpaceType.INNER_PRODUCT.scoreTranslation(-1 * rawScore) ); + // Map that overrides radial search score threshold to faiss required distance, check more details in knn documentation: + // https://opensearch.org/docs/latest/search-plugins/knn/approximate-knn/#spaces + private final static Map> SCORE_TO_DISTANCE_TRANSFORMATIONS = ImmutableMap.< + SpaceType, + Function>builder().put(SpaceType.INNER_PRODUCT, score -> score > 1 ? 1 - score : 1 / score - 1).build(); + // Define encoders supported by faiss private final static MethodComponentContext ENCODER_DEFAULT = new MethodComponentContext( KNNConstants.ENCODER_FLAT, @@ -301,7 +308,13 @@ class Faiss extends NativeLibrary { ).addSpaces(SpaceType.L2, SpaceType.INNER_PRODUCT).build() ); - final static Faiss INSTANCE = new Faiss(METHODS, SCORE_TRANSLATIONS, CURRENT_VERSION, KNNConstants.FAISS_EXTENSION); + final static Faiss INSTANCE = new Faiss( + METHODS, + SCORE_TRANSLATIONS, + CURRENT_VERSION, + KNNConstants.FAISS_EXTENSION, + SCORE_TO_DISTANCE_TRANSFORMATIONS + ); /** * Constructor for Faiss @@ -315,9 +328,26 @@ private Faiss( Map methods, Map> scoreTranslation, String currentVersion, - String extension + String extension, + Map> scoreTransform ) { super(methods, scoreTranslation, currentVersion, extension); + this.scoreTransform = scoreTransform; + } + + @Override + public Float distanceToRadialThreshold(Float distance, SpaceType spaceType) { + // Faiss engine uses distance as is and does not need transformation + return distance; + } + + @Override + public Float scoreToRadialThreshold(Float score, SpaceType spaceType) { + // Faiss engine uses distance as is and need transformation + if (this.scoreTransform.containsKey(spaceType)) { + return this.scoreTransform.get(spaceType).apply(score); + } + return spaceType.scoreToDistanceTranslation(score); } /** diff --git a/src/main/java/org/opensearch/knn/index/util/KNNEngine.java b/src/main/java/org/opensearch/knn/index/util/KNNEngine.java index 8d03d9a9e..e282c69db 100644 --- a/src/main/java/org/opensearch/knn/index/util/KNNEngine.java +++ b/src/main/java/org/opensearch/knn/index/util/KNNEngine.java @@ -32,6 +32,7 @@ public enum KNNEngine implements KNNLibrary { private static final Set CUSTOM_SEGMENT_FILE_ENGINES = ImmutableSet.of(KNNEngine.NMSLIB, KNNEngine.FAISS); private static final Set ENGINES_SUPPORTING_FILTERS = ImmutableSet.of(KNNEngine.LUCENE, KNNEngine.FAISS); + public static final Set ENGINES_SUPPORTING_RADIAL_SEARCH = ImmutableSet.of(KNNEngine.LUCENE, KNNEngine.FAISS); private static Map MAX_DIMENSIONS_BY_ENGINE = Map.of( KNNEngine.NMSLIB, @@ -152,6 +153,16 @@ public float score(float rawScore, SpaceType spaceType) { return knnLibrary.score(rawScore, spaceType); } + @Override + public Float distanceToRadialThreshold(Float distance, SpaceType spaceType) { + return knnLibrary.distanceToRadialThreshold(distance, spaceType); + } + + @Override + public Float scoreToRadialThreshold(Float score, SpaceType spaceType) { + return knnLibrary.scoreToRadialThreshold(score, spaceType); + } + @Override public ValidationException validateMethod(KNNMethodContext knnMethodContext) { return knnLibrary.validateMethod(knnMethodContext); diff --git a/src/main/java/org/opensearch/knn/index/util/KNNLibrary.java b/src/main/java/org/opensearch/knn/index/util/KNNLibrary.java index ba1d3ac84..f837566b8 100644 --- a/src/main/java/org/opensearch/knn/index/util/KNNLibrary.java +++ b/src/main/java/org/opensearch/knn/index/util/KNNLibrary.java @@ -68,6 +68,26 @@ public interface KNNLibrary { */ float score(float rawScore, SpaceType spaceType); + /** + * Translate the distance radius input from end user to the engine's threshold. + * + * @param distance distance radius input from end user + * @param spaceType spaceType used to compute the radius + * + * @return transformed distance for the library + */ + Float distanceToRadialThreshold(Float distance, SpaceType spaceType); + + /** + * Translate the score threshold input from end user to the engine's threshold. + * + * @param score score threshold input from end user + * @param spaceType spaceType used to compute the threshold + * + * @return transformed score for the library + */ + Float scoreToRadialThreshold(Float score, SpaceType spaceType); + /** * Validate the knnMethodContext for the given library. A ValidationException should be thrown if the method is * deemed invalid. diff --git a/src/main/java/org/opensearch/knn/index/util/Lucene.java b/src/main/java/org/opensearch/knn/index/util/Lucene.java index 63642ae2c..630d7a2c2 100644 --- a/src/main/java/org/opensearch/knn/index/util/Lucene.java +++ b/src/main/java/org/opensearch/knn/index/util/Lucene.java @@ -15,6 +15,7 @@ import java.util.List; import java.util.Map; +import java.util.function.Function; import static org.opensearch.knn.common.KNNConstants.METHOD_HNSW; import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_EF_CONSTRUCTION; @@ -25,6 +26,8 @@ */ public class Lucene extends JVMLibrary { + Map> distanceTransform; + final static Map METHODS = ImmutableMap.of( METHOD_HNSW, KNNMethod.Builder.builder( @@ -45,16 +48,27 @@ public class Lucene extends JVMLibrary { ).addSpaces(SpaceType.L2, SpaceType.COSINESIMIL, SpaceType.INNER_PRODUCT).build() ); - final static Lucene INSTANCE = new Lucene(METHODS, Version.LATEST.toString()); + // Map that overrides the default distance translations for Lucene, check more details in knn documentation: + // https://opensearch.org/docs/latest/search-plugins/knn/approximate-knn/#spaces + private final static Map> DISTANCE_TRANSLATIONS = ImmutableMap.< + SpaceType, + Function>builder() + .put(SpaceType.COSINESIMIL, distance -> (2 - distance) / 2) + .put(SpaceType.INNER_PRODUCT, distance -> distance <= 0 ? 1 / (1 - distance) : distance + 1) + .build(); + + final static Lucene INSTANCE = new Lucene(METHODS, Version.LATEST.toString(), DISTANCE_TRANSLATIONS); /** * Constructor * * @param methods Map of k-NN methods that the library supports * @param version String representing version of library + * @param distanceTransform Map of space type to distance transformation function */ - Lucene(Map methods, String version) { + Lucene(Map methods, String version, Map> distanceTransform) { super(methods, version); + this.distanceTransform = distanceTransform; } @Override @@ -75,6 +89,21 @@ public float score(float rawScore, SpaceType spaceType) { return rawScore; } + @Override + public Float distanceToRadialThreshold(Float distance, SpaceType spaceType) { + // Lucene requires score threshold to be parameterized when calling the radius search. + if (this.distanceTransform.containsKey(spaceType)) { + return this.distanceTransform.get(spaceType).apply(distance); + } + return spaceType.scoreTranslation(distance); + } + + @Override + public Float scoreToRadialThreshold(Float score, SpaceType spaceType) { + // Lucene engine uses distance as is and does not need transformation + return score; + } + @Override public List mmapFileExtensions() { return List.of("vec", "vex"); diff --git a/src/main/java/org/opensearch/knn/index/util/Nmslib.java b/src/main/java/org/opensearch/knn/index/util/Nmslib.java index 617b311f4..64af43520 100644 --- a/src/main/java/org/opensearch/knn/index/util/Nmslib.java +++ b/src/main/java/org/opensearch/knn/index/util/Nmslib.java @@ -68,4 +68,13 @@ private Nmslib( ) { super(methods, scoreTranslation, currentVersion, extension); } + + @Override + public Float distanceToRadialThreshold(Float distance, SpaceType spaceType) { + return distance; + } + + public Float scoreToRadialThreshold(Float score, SpaceType spaceType) { + return score; + } } diff --git a/src/main/java/org/opensearch/knn/jni/FaissService.java b/src/main/java/org/opensearch/knn/jni/FaissService.java index 32516ef9d..b59ac4bcf 100644 --- a/src/main/java/org/opensearch/knn/jni/FaissService.java +++ b/src/main/java/org/opensearch/knn/jni/FaissService.java @@ -190,4 +190,15 @@ public static native KNNQueryResult[] queryIndexWithFilter( */ @Deprecated(since = "2.14.0", forRemoval = true) public static native long transferVectors(long vectorsPointer, float[][] trainingData); + + /** + * Range search index + * + * @param indexPointer pointer to index in memory + * @param queryVector vector to be used for query + * @param radius search within radius threshold + * @param indexMaxResultWindow maximum number of results to return + * @return KNNQueryResult array of neighbors within radius + */ + public static native KNNQueryResult[] rangeSearchIndex(long indexPointer, float[] queryVector, float radius, int indexMaxResultWindow); } diff --git a/src/main/java/org/opensearch/knn/jni/JNIService.java b/src/main/java/org/opensearch/knn/jni/JNIService.java index 5a5b6794a..e846f02d1 100644 --- a/src/main/java/org/opensearch/knn/jni/JNIService.java +++ b/src/main/java/org/opensearch/knn/jni/JNIService.java @@ -262,4 +262,27 @@ public static byte[] trainIndex(Map indexParameters, int dimensi public static long transferVectors(long vectorsPointer, float[][] trainingData) { return FaissService.transferVectors(vectorsPointer, trainingData); } + + /** + * Range search index for a given query vector + * + * @param indexPointer pointer to index in memory + * @param queryVector vector to be used for query + * @param radius search within radius threshold + * @param knnEngine engine to query index + * @param indexMaxResultWindow maximum number of results to return + * @return KNNQueryResult array of neighbors within radius + */ + public static KNNQueryResult[] radiusQueryIndex( + long indexPointer, + float[] queryVector, + float radius, + KNNEngine knnEngine, + int indexMaxResultWindow + ) { + if (KNNEngine.FAISS == knnEngine) { + return FaissService.rangeSearchIndex(indexPointer, queryVector, radius, indexMaxResultWindow); + } + throw new IllegalArgumentException("RadiusQueryIndex not supported for provided engine"); + } } diff --git a/src/test/java/org/opensearch/knn/index/FaissIT.java b/src/test/java/org/opensearch/knn/index/FaissIT.java index 1f9b423bc..82ac12a73 100644 --- a/src/test/java/org/opensearch/knn/index/FaissIT.java +++ b/src/test/java/org/opensearch/knn/index/FaissIT.java @@ -15,6 +15,7 @@ import com.google.common.collect.ImmutableMap; import com.google.common.primitives.Floats; import lombok.SneakyThrows; +import org.apache.http.ParseException; import org.apache.http.util.EntityUtils; import org.junit.BeforeClass; import org.opensearch.client.Response; @@ -276,6 +277,283 @@ public void testEndToEnd_whenMethodIsHNSWFlatAndHasDeletedDocs_thenSucceed() { fail("Graphs are not getting evicted"); } + @SneakyThrows + public void testEndToEnd_whenDoRadiusSearch_whenDistanceThreshold_whenMethodIsHNSWFlat_thenSucceed() { + KNNMethod hnswMethod = KNNEngine.FAISS.getMethod(KNNConstants.METHOD_HNSW); + SpaceType spaceType = SpaceType.L2; + + List mValues = ImmutableList.of(16, 32, 64, 128); + List efConstructionValues = ImmutableList.of(16, 32, 64, 128); + List efSearchValues = ImmutableList.of(16, 32, 64, 128); + + Integer dimension = testData.indexData.vectors[0].length; + + // Create an index + XContentBuilder builder = XContentFactory.jsonBuilder() + .startObject() + .startObject("properties") + .startObject(FIELD_NAME) + .field("type", "knn_vector") + .field("dimension", dimension) + .startObject(KNNConstants.KNN_METHOD) + .field(KNNConstants.NAME, hnswMethod.getMethodComponent().getName()) + .field(KNNConstants.METHOD_PARAMETER_SPACE_TYPE, spaceType.getValue()) + .field(KNNConstants.KNN_ENGINE, KNNEngine.FAISS.getName()) + .startObject(KNNConstants.PARAMETERS) + .field(KNNConstants.METHOD_PARAMETER_M, mValues.get(random().nextInt(mValues.size()))) + .field(KNNConstants.METHOD_PARAMETER_EF_CONSTRUCTION, efConstructionValues.get(random().nextInt(efConstructionValues.size()))) + .field(KNNConstants.METHOD_PARAMETER_EF_SEARCH, efSearchValues.get(random().nextInt(efSearchValues.size()))) + .endObject() + .endObject() + .endObject() + .endObject() + .endObject(); + + Map mappingMap = xContentBuilderToMap(builder); + String mapping = builder.toString(); + + createKnnIndex(INDEX_NAME, mapping); + assertEquals(new TreeMap<>(mappingMap), new TreeMap<>(getIndexMappingAsMap(INDEX_NAME))); + + // Index the test data + for (int i = 0; i < testData.indexData.docs.length; i++) { + addKnnDoc( + INDEX_NAME, + Integer.toString(testData.indexData.docs[i]), + FIELD_NAME, + Floats.asList(testData.indexData.vectors[i]).toArray() + ); + } + + // Assert we have the right number of documents + refreshAllNonSystemIndices(); + assertEquals(testData.indexData.docs.length, getDocCount(INDEX_NAME)); + + float distance = 300000000000f; + validateRadiusSearchResults(INDEX_NAME, FIELD_NAME, testData.queries, distance, null, spaceType); + + // Delete index + deleteKNNIndex(INDEX_NAME); + } + + @SneakyThrows + public void testEndToEnd_whenDoRadiusSearch_whenScoreThreshold_whenMethodIsHNSWFlat_thenSucceed() { + KNNMethod hnswMethod = KNNEngine.FAISS.getMethod(KNNConstants.METHOD_HNSW); + SpaceType spaceType = SpaceType.L2; + + List mValues = ImmutableList.of(16, 32, 64, 128); + List efConstructionValues = ImmutableList.of(16, 32, 64, 128); + List efSearchValues = ImmutableList.of(16, 32, 64, 128); + + Integer dimension = testData.indexData.vectors[0].length; + + // Create an index + XContentBuilder builder = XContentFactory.jsonBuilder() + .startObject() + .startObject("properties") + .startObject(FIELD_NAME) + .field("type", "knn_vector") + .field("dimension", dimension) + .startObject(KNNConstants.KNN_METHOD) + .field(KNNConstants.NAME, hnswMethod.getMethodComponent().getName()) + .field(KNNConstants.METHOD_PARAMETER_SPACE_TYPE, spaceType.getValue()) + .field(KNNConstants.KNN_ENGINE, KNNEngine.FAISS.getName()) + .startObject(KNNConstants.PARAMETERS) + .field(KNNConstants.METHOD_PARAMETER_M, mValues.get(random().nextInt(mValues.size()))) + .field(KNNConstants.METHOD_PARAMETER_EF_CONSTRUCTION, efConstructionValues.get(random().nextInt(efConstructionValues.size()))) + .field(KNNConstants.METHOD_PARAMETER_EF_SEARCH, efSearchValues.get(random().nextInt(efSearchValues.size()))) + .endObject() + .endObject() + .endObject() + .endObject() + .endObject(); + + Map mappingMap = xContentBuilderToMap(builder); + String mapping = builder.toString(); + + createKnnIndex(INDEX_NAME, mapping); + assertEquals(new TreeMap<>(mappingMap), new TreeMap<>(getIndexMappingAsMap(INDEX_NAME))); + + // Index the test data + for (int i = 0; i < testData.indexData.docs.length; i++) { + addKnnDoc( + INDEX_NAME, + Integer.toString(testData.indexData.docs[i]), + FIELD_NAME, + Floats.asList(testData.indexData.vectors[i]).toArray() + ); + } + + // Assert we have the right number of documents + refreshAllNonSystemIndices(); + assertEquals(testData.indexData.docs.length, getDocCount(INDEX_NAME)); + + float score = 0.00001f; + + validateRadiusSearchResults(INDEX_NAME, FIELD_NAME, testData.queries, null, score, spaceType); + + // Delete index + deleteKNNIndex(INDEX_NAME); + } + + @SneakyThrows + public void testEndToEnd_whenDoRadiusSearch_whenMoreThanOneScoreThreshold_whenMethodIsHNSWFlat_thenSucceed() { + KNNMethod hnswMethod = KNNEngine.FAISS.getMethod(KNNConstants.METHOD_HNSW); + SpaceType spaceType = SpaceType.INNER_PRODUCT; + + List mValues = ImmutableList.of(16, 32, 64, 128); + List efConstructionValues = ImmutableList.of(16, 32, 64, 128); + List efSearchValues = ImmutableList.of(16, 32, 64, 128); + + Integer dimension = testData.indexData.vectors[0].length; + + // Create an index + XContentBuilder builder = XContentFactory.jsonBuilder() + .startObject() + .startObject("properties") + .startObject(FIELD_NAME) + .field("type", "knn_vector") + .field("dimension", dimension) + .startObject(KNNConstants.KNN_METHOD) + .field(KNNConstants.NAME, hnswMethod.getMethodComponent().getName()) + .field(KNNConstants.METHOD_PARAMETER_SPACE_TYPE, spaceType.getValue()) + .field(KNNConstants.KNN_ENGINE, KNNEngine.FAISS.getName()) + .startObject(KNNConstants.PARAMETERS) + .field(KNNConstants.METHOD_PARAMETER_M, mValues.get(random().nextInt(mValues.size()))) + .field(KNNConstants.METHOD_PARAMETER_EF_CONSTRUCTION, efConstructionValues.get(random().nextInt(efConstructionValues.size()))) + .field(KNNConstants.METHOD_PARAMETER_EF_SEARCH, efSearchValues.get(random().nextInt(efSearchValues.size()))) + .endObject() + .endObject() + .endObject() + .endObject() + .endObject(); + + Map mappingMap = xContentBuilderToMap(builder); + String mapping = builder.toString(); + + createKnnIndex(INDEX_NAME, mapping); + assertEquals(new TreeMap<>(mappingMap), new TreeMap<>(getIndexMappingAsMap(INDEX_NAME))); + + // Index the test data + for (int i = 0; i < testData.indexData.docs.length; i++) { + addKnnDoc( + INDEX_NAME, + Integer.toString(testData.indexData.docs[i]), + FIELD_NAME, + Floats.asList(testData.indexData.vectors[i]).toArray() + ); + } + + // Assert we have the right number of documents + refreshAllNonSystemIndices(); + assertEquals(testData.indexData.docs.length, getDocCount(INDEX_NAME)); + + float score = 5f; + + validateRadiusSearchResults(INDEX_NAME, FIELD_NAME, testData.queries, null, score, spaceType); + + // Delete index + deleteKNNIndex(INDEX_NAME); + } + + @SneakyThrows + public void testEndToEnd_whenDoRadiusSearch__whenDistanceThreshold_whenMethodIsHNSWPQ_thenSucceed() { + String indexName = "test-index"; + String fieldName = "test-field"; + String trainingIndexName = "training-index"; + String trainingFieldName = "training-field"; + + String modelId = "test-model"; + String modelDescription = "test model"; + + List mValues = ImmutableList.of(16, 32, 64, 128); + List efConstructionValues = ImmutableList.of(16, 32, 64, 128); + List efSearchValues = ImmutableList.of(16, 32, 64, 128); + List pqMValues = ImmutableList.of(2, 4, 8); + + // training data needs to be at least equal to the number of centroids for PQ + // which is 2^8 = 256. 8 because that's the only valid code_size for HNSWPQ + int trainingDataCount = 256; + + SpaceType spaceType = SpaceType.L2; + + int dimension = testData.indexData.vectors[0].length; + + XContentBuilder xContentBuilder = XContentFactory.jsonBuilder() + .startObject() + .field(NAME, METHOD_HNSW) + .field(KNN_ENGINE, FAISS_NAME) + .startObject(PARAMETERS) + .field(KNNConstants.METHOD_PARAMETER_M, mValues.get(random().nextInt(mValues.size()))) + .field(KNNConstants.METHOD_PARAMETER_EF_CONSTRUCTION, efConstructionValues.get(random().nextInt(efConstructionValues.size()))) + .field(KNNConstants.METHOD_PARAMETER_EF_SEARCH, efSearchValues.get(random().nextInt(efSearchValues.size()))) + .startObject(METHOD_ENCODER_PARAMETER) + .field(NAME, ENCODER_PQ) + .startObject(PARAMETERS) + .field(ENCODER_PARAMETER_PQ_M, pqMValues.get(random().nextInt(pqMValues.size()))) + .endObject() + .endObject() + .endObject() + .endObject(); + Map in = xContentBuilderToMap(xContentBuilder); + + createBasicKnnIndex(trainingIndexName, trainingFieldName, dimension); + ingestDataAndTrainModel(modelId, trainingIndexName, trainingFieldName, dimension, modelDescription, in, trainingDataCount); + assertTrainingSucceeds(modelId, 360, 1000); + + // Create an index + XContentBuilder builder = XContentFactory.jsonBuilder() + .startObject() + .startObject("properties") + .startObject(fieldName) + .field("type", "knn_vector") + .field("model_id", modelId) + .endObject() + .endObject() + .endObject(); + + Map mappingMap = xContentBuilderToMap(builder); + String mapping = builder.toString(); + + createKnnIndex(indexName, mapping); + + assertEquals(new TreeMap<>(mappingMap), new TreeMap<>(getIndexMappingAsMap(indexName))); + + // Index the test data + for (int i = 0; i < testData.indexData.docs.length; i++) { + addKnnDoc( + indexName, + Integer.toString(testData.indexData.docs[i]), + fieldName, + Floats.asList(testData.indexData.vectors[i]).toArray() + ); + } + + // Assert we have the right number of documents in the index + refreshAllNonSystemIndices(); + assertEquals(testData.indexData.docs.length, getDocCount(indexName)); + + float distance = 300000000000f; + + validateRadiusSearchResults(indexName, fieldName, testData.queries, distance, null, spaceType); + + // Delete index + deleteKNNIndex(indexName); + deleteModel(modelId); + + // Search every 5 seconds 14 times to confirm graph gets evicted + int intervals = 14; + for (int i = 0; i < intervals; i++) { + if (getTotalGraphsInCache() == 0) { + return; + } + + Thread.sleep(5 * 1000); + } + + fail("Graphs are not getting evicted"); + } + @SneakyThrows public void testEndToEnd_whenMethodIsHNSWPQ_thenSucceed() { String indexName = "test-index"; @@ -1406,4 +1684,45 @@ private void validateGraphEviction() throws Exception { fail("Graphs are not getting evicted"); } + + private void validateRadiusSearchResults( + String indexName, + String fieldName, + float[][] queryVectors, + Float distanceThreshold, + Float scoreThreshold, + final SpaceType spaceType + ) throws IOException, ParseException { + for (float[] queryVector : queryVectors) { + XContentBuilder queryBuilder = XContentFactory.jsonBuilder().startObject().startObject("query"); + queryBuilder.startObject("knn"); + queryBuilder.startObject(fieldName); + queryBuilder.field("vector", queryVector); + if (distanceThreshold != null) { + queryBuilder.field("max_distance", distanceThreshold); + } else if (scoreThreshold != null) { + queryBuilder.field("min_score", scoreThreshold); + } else { + throw new IllegalArgumentException("Invalid threshold"); + } + queryBuilder.endObject(); + queryBuilder.endObject(); + queryBuilder.endObject().endObject(); + final String responseBody = EntityUtils.toString(searchKNNIndex(indexName, queryBuilder, 10).getEntity()); + + List knnResults = parseSearchResponse(responseBody, fieldName); + + for (KNNResult knnResult : knnResults) { + float[] vector = knnResult.getVector(); + float distance = TestUtils.computeDistFromSpaceType(spaceType, vector, queryVector); + if (spaceType == SpaceType.L2) { + assertTrue(KNNScoringUtil.l2Squared(queryVector, vector) <= distance); + } else if (spaceType == SpaceType.INNER_PRODUCT) { + assertTrue(KNNScoringUtil.innerProduct(queryVector, vector) >= distance); + } else { + throw new IllegalArgumentException("Invalid space type"); + } + } + } + } } diff --git a/src/test/java/org/opensearch/knn/index/LuceneEngineIT.java b/src/test/java/org/opensearch/knn/index/LuceneEngineIT.java index c53fa4456..4cc856613 100644 --- a/src/test/java/org/opensearch/knn/index/LuceneEngineIT.java +++ b/src/test/java/org/opensearch/knn/index/LuceneEngineIT.java @@ -15,6 +15,7 @@ import org.junit.After; import org.opensearch.client.Response; import org.opensearch.client.ResponseException; +import org.opensearch.common.Nullable; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.index.query.QueryBuilders; @@ -33,7 +34,6 @@ import java.util.function.Function; import java.util.stream.Collectors; -import static org.opensearch.knn.common.KNNConstants.DIMENSION; import static org.opensearch.knn.common.KNNConstants.METHOD_HNSW; import static org.opensearch.knn.common.KNNConstants.NMSLIB_NAME; import static org.opensearch.knn.common.KNNConstants.VECTOR_DATA_TYPE_FIELD; @@ -50,6 +50,14 @@ public class LuceneEngineIT extends KNNRestTestCase { private static final int M = 16; private static final Float[][] TEST_INDEX_VECTORS = { { 1.0f, 1.0f, 1.0f }, { 2.0f, 2.0f, 2.0f }, { 3.0f, 3.0f, 3.0f } }; + private static final Float[][] TEST_COSINESIMIL_INDEX_VECTORS = { { 6.0f, 7.0f, 3.0f }, { 3.0f, 2.0f, 5.0f }, { 4.0f, 5.0f, 7.0f } }; + private static final Float[][] TEST_INNER_PRODUCT_INDEX_VECTORS = { + { 1.0f, 1.0f, 1.0f }, + { 2.0f, 2.0f, 2.0f }, + { 3.0f, 3.0f, 3.0f }, + { -1.0f, -1.0f, -1.0f }, + { -2.0f, -2.0f, -2.0f }, + { -3.0f, -3.0f, -3.0f } }; private static final float[][] TEST_QUERY_VECTORS = { { 1.0f, 1.0f, 1.0f }, { 2.0f, 2.0f, 2.0f }, { 3.0f, 3.0f, 3.0f } }; @@ -59,7 +67,9 @@ public class LuceneEngineIT extends KNNRestTestCase { VectorSimilarityFunction.DOT_PRODUCT, (similarity) -> (1 + similarity) / 2, VectorSimilarityFunction.COSINE, - (similarity) -> (1 + similarity) / 2 + (similarity) -> (1 + similarity) / 2, + VectorSimilarityFunction.MAXIMUM_INNER_PRODUCT, + (similarity) -> similarity <= 0 ? 1 / (1 - similarity) : similarity + 1 ); private static final String DIMENSION_FIELD_NAME = "dimension"; private static final String KNN_VECTOR_TYPE = "knn_vector"; @@ -318,6 +328,142 @@ public void testIndexReopening() throws Exception { assertArrayEquals(knnResultsBeforeIndexClosure.toArray(), knnResultsAfterIndexClosure.toArray()); } + public void testRadiusSearch_usingDistanceThreshold_usingL2Metrics_usingFloatType() throws Exception { + createKnnIndexMappingWithLuceneEngine(DIMENSION, SpaceType.L2, VectorDataType.FLOAT); + for (int j = 0; j < TEST_INDEX_VECTORS.length; j++) { + addKnnDoc(INDEX_NAME, Integer.toString(j + 1), FIELD_NAME, TEST_INDEX_VECTORS[j]); + } + + final float distance = 3.5f; + final int[] expectedResults = { 2, 3, 2 }; + + validateRadiusSearchResults(TEST_QUERY_VECTORS, distance, null, SpaceType.L2, expectedResults, null, null); + } + + public void testRadiusSearch_usingScoreThreshold_usingL2Metrics_usingFloatType() throws Exception { + createKnnIndexMappingWithLuceneEngine(DIMENSION, SpaceType.L2, VectorDataType.FLOAT); + for (int j = 0; j < TEST_INDEX_VECTORS.length; j++) { + addKnnDoc(INDEX_NAME, Integer.toString(j + 1), FIELD_NAME, TEST_INDEX_VECTORS[j]); + } + + final float score = 0.23f; + final int[] expectedResults = { 2, 3, 2 }; + + validateRadiusSearchResults(TEST_QUERY_VECTORS, null, score, SpaceType.L2, expectedResults, null, null); + } + + public void testRadiusSearch_usingDistanceThreshold_usingCosineMetrics_usingFloatType() throws Exception { + createKnnIndexMappingWithLuceneEngine(DIMENSION, SpaceType.COSINESIMIL, VectorDataType.FLOAT); + for (int j = 0; j < TEST_COSINESIMIL_INDEX_VECTORS.length; j++) { + addKnnDoc(INDEX_NAME, Integer.toString(j + 1), FIELD_NAME, TEST_COSINESIMIL_INDEX_VECTORS[j]); + } + + final float distance = 0.03f; + final int[] expectedResults = { 1, 1, 1 }; + + validateRadiusSearchResults(TEST_QUERY_VECTORS, distance, null, SpaceType.COSINESIMIL, expectedResults, null, null); + } + + public void testRadiusSearch_usingScoreThreshold_usingCosineMetrics_usingFloatType() throws Exception { + createKnnIndexMappingWithLuceneEngine(DIMENSION, SpaceType.COSINESIMIL, VectorDataType.FLOAT); + for (int j = 0; j < TEST_COSINESIMIL_INDEX_VECTORS.length; j++) { + addKnnDoc(INDEX_NAME, Integer.toString(j + 1), FIELD_NAME, TEST_COSINESIMIL_INDEX_VECTORS[j]); + } + + final float score = 0.97f; + final int[] expectedResults = { 1, 1, 1 }; + + validateRadiusSearchResults(TEST_QUERY_VECTORS, null, score, SpaceType.COSINESIMIL, expectedResults, null, null); + } + + public void testRadiusSearch_usingScoreThreshold_usingInnerProductMetrics_usingFloatType() throws Exception { + createKnnIndexMappingWithLuceneEngine(DIMENSION, SpaceType.INNER_PRODUCT, VectorDataType.FLOAT); + for (int j = 0; j < TEST_INNER_PRODUCT_INDEX_VECTORS.length; j++) { + addKnnDoc(INDEX_NAME, Integer.toString(j + 1), FIELD_NAME, TEST_INNER_PRODUCT_INDEX_VECTORS[j]); + } + + final float score = 2f; + final int[] expectedResults = { 1, 1, 1 }; + + validateRadiusSearchResults(TEST_QUERY_VECTORS, null, score, SpaceType.INNER_PRODUCT, expectedResults, null, null); + } + + public void testRadiusSearch_usingDistanceThreshold_usingL2Metrics_usingByteType() throws Exception { + createKnnIndexMappingWithLuceneEngine(DIMENSION, SpaceType.L2, VectorDataType.BYTE); + for (int j = 0; j < TEST_INDEX_VECTORS.length; j++) { + addKnnDoc(INDEX_NAME, Integer.toString(j + 1), FIELD_NAME, TEST_INDEX_VECTORS[j]); + } + + final float distance = 3.5f; + final int[] expectedResults = { 2, 2, 2 }; + + validateRadiusSearchResults(TEST_QUERY_VECTORS, distance, null, SpaceType.L2, expectedResults, null, null); + } + + public void testRadiusSearch_usingScoreThreshold_usingL2Metrics_usingByteType() throws Exception { + createKnnIndexMappingWithLuceneEngine(DIMENSION, SpaceType.L2, VectorDataType.BYTE); + for (int j = 0; j < TEST_INDEX_VECTORS.length; j++) { + addKnnDoc(INDEX_NAME, Integer.toString(j + 1), FIELD_NAME, TEST_INDEX_VECTORS[j]); + } + + final float score = 0.23f; + final int[] expectedResults = { 2, 2, 2 }; + + validateRadiusSearchResults(TEST_QUERY_VECTORS, null, score, SpaceType.L2, expectedResults, null, null); + } + + public void testRadiusSearch_usingDistanceThreshold_usingCosineMetrics_usingByteType() throws Exception { + createKnnIndexMappingWithLuceneEngine(DIMENSION, SpaceType.COSINESIMIL, VectorDataType.BYTE); + for (int j = 0; j < TEST_COSINESIMIL_INDEX_VECTORS.length; j++) { + addKnnDoc(INDEX_NAME, Integer.toString(j + 1), FIELD_NAME, TEST_COSINESIMIL_INDEX_VECTORS[j]); + } + + final float distance = 0.05f; + final int[] expectedResults = { 2, 2, 2 }; + + validateRadiusSearchResults(TEST_QUERY_VECTORS, distance, null, SpaceType.COSINESIMIL, expectedResults, null, null); + } + + public void testRadiusSearch_usingScoreThreshold_usingCosineMetrics_usingByteType() throws Exception { + createKnnIndexMappingWithLuceneEngine(DIMENSION, SpaceType.COSINESIMIL, VectorDataType.BYTE); + for (int j = 0; j < TEST_COSINESIMIL_INDEX_VECTORS.length; j++) { + addKnnDoc(INDEX_NAME, Integer.toString(j + 1), FIELD_NAME, TEST_COSINESIMIL_INDEX_VECTORS[j]); + } + + final float score = 0.97f; + final int[] expectedResults = { 2, 2, 2 }; + + validateRadiusSearchResults(TEST_QUERY_VECTORS, null, score, SpaceType.COSINESIMIL, expectedResults, null, null); + } + + public void testRadiusSearch_usingDistanceThreshold_withFilter_usingL2Metrics_usingFloatType() throws Exception { + createKnnIndexMappingWithLuceneEngine(DIMENSION, SpaceType.L2, VectorDataType.FLOAT); + addKnnDocWithAttributes(DOC_ID, new float[] { 6.0f, 7.9f, 3.1f }, ImmutableMap.of(COLOR_FIELD_NAME, "red")); + addKnnDocWithAttributes(DOC_ID_2, new float[] { 3.2f, 2.1f, 4.8f }, ImmutableMap.of(COLOR_FIELD_NAME, "red")); + addKnnDocWithAttributes(DOC_ID_3, new float[] { 4.1f, 5.0f, 7.1f }, ImmutableMap.of(COLOR_FIELD_NAME, "green")); + + refreshIndex(INDEX_NAME); + + final float distance = 45.0f; + final int[] expectedResults = { 1, 1, 1 }; + + validateRadiusSearchResults(TEST_QUERY_VECTORS, distance, null, SpaceType.L2, expectedResults, COLOR_FIELD_NAME, "red"); + } + + public void testRadiusSearch_usingScoreThreshold_withFilter_usingCosineMetrics_usingFloatType() throws Exception { + createKnnIndexMappingWithLuceneEngine(DIMENSION, SpaceType.COSINESIMIL, VectorDataType.FLOAT); + addKnnDocWithAttributes(DOC_ID, new float[] { 6.0f, 7.9f, 3.1f }, ImmutableMap.of(COLOR_FIELD_NAME, "red")); + addKnnDocWithAttributes(DOC_ID_2, new float[] { 3.2f, 2.1f, 4.8f }, ImmutableMap.of(COLOR_FIELD_NAME, "red")); + addKnnDocWithAttributes(DOC_ID_3, new float[] { 4.1f, 5.0f, 7.1f }, ImmutableMap.of(COLOR_FIELD_NAME, "green")); + + refreshIndex(INDEX_NAME); + + final float score = 0.02f; + final int[] expectedResults = { 1, 1, 1 }; + + validateRadiusSearchResults(TEST_QUERY_VECTORS, null, score, SpaceType.COSINESIMIL, expectedResults, COLOR_FIELD_NAME, "red"); + } + private void createKnnIndexMappingWithLuceneEngine(int dimension, SpaceType spaceType, VectorDataType vectorDataType) throws Exception { XContentBuilder builder = XContentFactory.jsonBuilder() .startObject() @@ -467,4 +613,59 @@ public void test_whenUsingIP_thenSuccess() { assertEquals(expectedScores.get(i), knnResults.get(i), 0.0000001); } } + + private void validateRadiusSearchResults( + final float[][] searchVectors, + final Float distanceThreshold, + final Float scoreThreshold, + final SpaceType spaceType, + final int[] expectedResults, + @Nullable final String filterField, + @Nullable final String filterValue + ) throws Exception { + for (int i = 0; i < searchVectors.length; i++) { + XContentBuilder builder = XContentFactory.jsonBuilder().startObject().startObject("query"); + builder.startObject("knn"); + builder.startObject(FIELD_NAME); + builder.field("vector", searchVectors[i]); + if (distanceThreshold != null) { + builder.field("max_distance", distanceThreshold); + } else if (scoreThreshold != null) { + builder.field("min_score", scoreThreshold); + } else { + throw new IllegalArgumentException("Either distance or score must be provided"); + } + if (filterField != null && filterValue != null) { + builder.startObject("filter"); + builder.startObject("term"); + builder.field(filterField, filterValue); + builder.endObject(); + builder.endObject(); + } + builder.endObject(); + builder.endObject(); + builder.endObject().endObject(); + + final String responseBody = EntityUtils.toString(searchKNNIndex(INDEX_NAME, builder, expectedResults[i]).getEntity()); + final List radiusResults = parseSearchResponse(responseBody, FIELD_NAME); + + assertEquals(expectedResults[i], radiusResults.size()); + + List actualScores = parseSearchResponseScore(responseBody, FIELD_NAME); + for (KNNResult result : radiusResults) { + float[] vector = result.getVector(); + float distance = TestUtils.computeDistFromSpaceType(spaceType, vector, searchVectors[i]); + float rawScore = VECTOR_SIMILARITY_TO_SCORE.get(spaceType.getVectorSimilarityFunction()).apply(distance); + if (spaceType == SpaceType.COSINESIMIL) { + distance = 1 - distance; + } + if (distanceThreshold != null) { + assertTrue(distance <= distanceThreshold); + } else { + assertTrue(rawScore >= scoreThreshold); + } + assertEquals(KNNEngine.LUCENE.score(rawScore, spaceType), actualScores.get(radiusResults.indexOf(result)), 0.0001); + } + } + } } diff --git a/src/test/java/org/opensearch/knn/index/query/KNNQueryBuilderTests.java b/src/test/java/org/opensearch/knn/index/query/KNNQueryBuilderTests.java index 987d44422..e47f583e7 100644 --- a/src/test/java/org/opensearch/knn/index/query/KNNQueryBuilderTests.java +++ b/src/test/java/org/opensearch/knn/index/query/KNNQueryBuilderTests.java @@ -6,6 +6,7 @@ package org.opensearch.knn.index.query; import com.google.common.collect.ImmutableMap; +import org.apache.lucene.search.FloatVectorSimilarityQuery; import java.util.Locale; import org.apache.lucene.search.KnnFloatVectorQuery; import org.apache.lucene.search.MatchNoDocsQuery; @@ -18,6 +19,7 @@ import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.index.IndexSettings; import org.opensearch.index.query.QueryBuilder; import org.opensearch.index.query.QueryBuilders; import org.opensearch.index.query.TermQueryBuilder; @@ -41,8 +43,10 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import java.util.Optional; +import java.util.stream.Collectors; import static org.hamcrest.Matchers.instanceOf; import static org.mockito.Mockito.anyString; @@ -50,11 +54,14 @@ import static org.mockito.Mockito.when; import static org.opensearch.knn.common.KNNConstants.METHOD_HNSW; import static org.opensearch.knn.index.KNNClusterTestUtils.mockClusterService; +import static org.opensearch.knn.index.util.KNNEngine.ENGINES_SUPPORTING_RADIAL_SEARCH; public class KNNQueryBuilderTests extends KNNTestCase { private static final String FIELD_NAME = "myvector"; private static final int K = 1; + private static final Float MAX_DISTANCE = 1.0f; + private static final Float MIN_SCORE = 0.5f; private static final TermQueryBuilder TERM_QUERY = QueryBuilders.termQuery("field", "value"); private static final float[] QUERY_VECTOR = new float[] { 1.0f, 2.0f, 3.0f, 4.0f }; @@ -77,6 +84,32 @@ public void testInvalidK() { expectThrows(IllegalArgumentException.class, () -> new KNNQueryBuilder(FIELD_NAME, queryVector, KNNQueryBuilder.K_MAX + K)); } + public void testInvalidDistance() { + float[] queryVector = { 1.0f, 1.0f }; + /** + * null distance + */ + expectThrows(IllegalArgumentException.class, () -> new KNNQueryBuilder(FIELD_NAME, queryVector).maxDistance(null)); + } + + public void testInvalidScore() { + float[] queryVector = { 1.0f, 1.0f }; + /** + * null min_score + */ + expectThrows(IllegalArgumentException.class, () -> new KNNQueryBuilder(FIELD_NAME, queryVector).minScore(null)); + + /** + * negative min_score + */ + expectThrows(IllegalArgumentException.class, () -> new KNNQueryBuilder(FIELD_NAME, queryVector).minScore(-1.0f)); + + /** + * min_score = 0 + */ + expectThrows(IllegalArgumentException.class, () -> new KNNQueryBuilder(FIELD_NAME, queryVector).minScore(0.0f)); + } + public void testEmptyVector() { /** * null query vector @@ -89,6 +122,18 @@ public void testEmptyVector() { */ float[] queryVector1 = {}; expectThrows(IllegalArgumentException.class, () -> new KNNQueryBuilder(FIELD_NAME, queryVector1, K)); + + /** + * null query vector with distance + */ + float[] queryVector2 = null; + expectThrows(IllegalArgumentException.class, () -> new KNNQueryBuilder(FIELD_NAME, queryVector2).maxDistance(MAX_DISTANCE)); + + /** + * empty query vector with distance + */ + float[] queryVector3 = {}; + expectThrows(IllegalArgumentException.class, () -> new KNNQueryBuilder(FIELD_NAME, queryVector3).maxDistance(MAX_DISTANCE)); } public void testFromXContent() throws Exception { @@ -107,7 +152,39 @@ public void testFromXContent() throws Exception { assertEquals(knnQueryBuilder, actualBuilder); } - public void testFromXContent_WithFilter() throws Exception { + public void testFromXContent_whenDoRadiusSearch_whenDistanceThreshold_thenSucceed() throws Exception { + float[] queryVector = { 1.0f, 2.0f, 3.0f, 4.0f }; + KNNQueryBuilder knnQueryBuilder = new KNNQueryBuilder(FIELD_NAME, queryVector).maxDistance(MAX_DISTANCE); + XContentBuilder builder = XContentFactory.jsonBuilder(); + builder.startObject(); + builder.startObject(knnQueryBuilder.fieldName()); + builder.field(KNNQueryBuilder.VECTOR_FIELD.getPreferredName(), knnQueryBuilder.vector()); + builder.field(KNNQueryBuilder.MAX_DISTANCE_FIELD.getPreferredName(), knnQueryBuilder.getMaxDistance()); + builder.endObject(); + builder.endObject(); + XContentParser contentParser = createParser(builder); + contentParser.nextToken(); + KNNQueryBuilder actualBuilder = KNNQueryBuilder.fromXContent(contentParser); + assertEquals(knnQueryBuilder, actualBuilder); + } + + public void testFromXContent_whenDoRadiusSearch_whenScoreThreshold_thenSucceed() throws Exception { + float[] queryVector = { 1.0f, 2.0f, 3.0f, 4.0f }; + KNNQueryBuilder knnQueryBuilder = new KNNQueryBuilder(FIELD_NAME, queryVector).minScore(MAX_DISTANCE); + XContentBuilder builder = XContentFactory.jsonBuilder(); + builder.startObject(); + builder.startObject(knnQueryBuilder.fieldName()); + builder.field(KNNQueryBuilder.VECTOR_FIELD.getPreferredName(), knnQueryBuilder.vector()); + builder.field(KNNQueryBuilder.MAX_DISTANCE_FIELD.getPreferredName(), knnQueryBuilder.getMinScore()); + builder.endObject(); + builder.endObject(); + XContentParser contentParser = createParser(builder); + contentParser.nextToken(); + KNNQueryBuilder actualBuilder = KNNQueryBuilder.fromXContent(contentParser); + assertEquals(knnQueryBuilder, actualBuilder); + } + + public void testFromXContent_withFilter() throws Exception { final ClusterService clusterService = mockClusterService(Version.CURRENT); final KNNClusterUtil knnClusterUtil = KNNClusterUtil.instance(); @@ -151,7 +228,51 @@ public void testFromXcontent_WithFilter_UnsupportedClusterVersion() throws Excep expectThrows(IllegalArgumentException.class, () -> KNNQueryBuilder.fromXContent(contentParser)); } - public void testFromXContent_invalidQueryVectorType() throws Exception { + public void testFromXContent_wenDoRadiusSearch_whenDistanceThreshold_whenFilter_thenSucceed() throws Exception { + final ClusterService clusterService = mockClusterService(Version.CURRENT); + + final KNNClusterUtil knnClusterUtil = KNNClusterUtil.instance(); + knnClusterUtil.initialize(clusterService); + + float[] queryVector = { 1.0f, 2.0f, 3.0f, 4.0f }; + KNNQueryBuilder knnQueryBuilder = new KNNQueryBuilder(FIELD_NAME, queryVector).maxDistance(MAX_DISTANCE).filter(TERM_QUERY); + XContentBuilder builder = XContentFactory.jsonBuilder(); + builder.startObject(); + builder.startObject(knnQueryBuilder.fieldName()); + builder.field(KNNQueryBuilder.VECTOR_FIELD.getPreferredName(), knnQueryBuilder.vector()); + builder.field(KNNQueryBuilder.MAX_DISTANCE_FIELD.getPreferredName(), knnQueryBuilder.getMaxDistance()); + builder.field(KNNQueryBuilder.FILTER_FIELD.getPreferredName(), knnQueryBuilder.getFilter()); + builder.endObject(); + builder.endObject(); + XContentParser contentParser = createParser(builder); + contentParser.nextToken(); + KNNQueryBuilder actualBuilder = KNNQueryBuilder.fromXContent(contentParser); + assertEquals(knnQueryBuilder, actualBuilder); + } + + public void testFromXContent_wenDoRadiusSearch_whenScoreThreshold_whenFilter_thenSucceed() throws Exception { + final ClusterService clusterService = mockClusterService(Version.CURRENT); + + final KNNClusterUtil knnClusterUtil = KNNClusterUtil.instance(); + knnClusterUtil.initialize(clusterService); + + float[] queryVector = { 1.0f, 2.0f, 3.0f, 4.0f }; + KNNQueryBuilder knnQueryBuilder = new KNNQueryBuilder(FIELD_NAME, queryVector).minScore(MIN_SCORE).filter(TERM_QUERY); + XContentBuilder builder = XContentFactory.jsonBuilder(); + builder.startObject(); + builder.startObject(knnQueryBuilder.fieldName()); + builder.field(KNNQueryBuilder.VECTOR_FIELD.getPreferredName(), knnQueryBuilder.vector()); + builder.field(KNNQueryBuilder.MAX_DISTANCE_FIELD.getPreferredName(), knnQueryBuilder.getMinScore()); + builder.field(KNNQueryBuilder.FILTER_FIELD.getPreferredName(), knnQueryBuilder.getFilter()); + builder.endObject(); + builder.endObject(); + XContentParser contentParser = createParser(builder); + contentParser.nextToken(); + KNNQueryBuilder actualBuilder = KNNQueryBuilder.fromXContent(contentParser); + assertEquals(knnQueryBuilder, actualBuilder); + } + + public void testFromXContent_InvalidQueryVectorType() throws Exception { final ClusterService clusterService = mockClusterService(Version.CURRENT); final KNNClusterUtil knnClusterUtil = KNNClusterUtil.instance(); @@ -179,6 +300,34 @@ public void testFromXContent_invalidQueryVectorType() throws Exception { assertTrue(exception.getMessage().contains("[knn] field 'vector' requires to be an array of numbers")); } + public void testFromXContent_whenDoRadiusSearch_whenInputInvalidQueryVectorType_thenException() throws Exception { + final ClusterService clusterService = mockClusterService(Version.CURRENT); + + final KNNClusterUtil knnClusterUtil = KNNClusterUtil.instance(); + knnClusterUtil.initialize(clusterService); + + List invalidTypeQueryVector = new ArrayList<>(); + invalidTypeQueryVector.add(1.5); + invalidTypeQueryVector.add(2.5); + invalidTypeQueryVector.add("a"); + invalidTypeQueryVector.add(null); + + XContentBuilder builder = XContentFactory.jsonBuilder(); + builder.startObject(); + builder.startObject(FIELD_NAME); + builder.field(KNNQueryBuilder.VECTOR_FIELD.getPreferredName(), invalidTypeQueryVector); + builder.field(KNNQueryBuilder.MAX_DISTANCE_FIELD.getPreferredName(), MAX_DISTANCE); + builder.endObject(); + builder.endObject(); + XContentParser contentParser = createParser(builder); + contentParser.nextToken(); + IllegalArgumentException exception = expectThrows( + IllegalArgumentException.class, + () -> KNNQueryBuilder.fromXContent(contentParser) + ); + assertTrue(exception.getMessage().contains("[knn] field 'vector' requires to be an array of numbers")); + } + public void testFromXContent_missingQueryVector() throws Exception { final ClusterService clusterService = mockClusterService(Version.CURRENT); @@ -253,6 +402,178 @@ public void testDoToQuery_Normal() throws Exception { assertEquals(knnQueryBuilder.vector(), query.getQueryVector()); } + public void testDoToQuery_whenNormal_whenDoRadiusSearch_whenDistanceThreshold_thenSucceed() { + float[] queryVector = { 1.0f, 2.0f, 3.0f, 4.0f }; + KNNQueryBuilder knnQueryBuilder = new KNNQueryBuilder(FIELD_NAME, queryVector).maxDistance(MAX_DISTANCE); + Index dummyIndex = new Index("dummy", "dummy"); + QueryShardContext mockQueryShardContext = mock(QueryShardContext.class); + KNNVectorFieldMapper.KNNVectorFieldType mockKNNVectorField = mock(KNNVectorFieldMapper.KNNVectorFieldType.class); + when(mockQueryShardContext.index()).thenReturn(dummyIndex); + when(mockKNNVectorField.getDimension()).thenReturn(4); + when(mockKNNVectorField.getVectorDataType()).thenReturn(VectorDataType.FLOAT); + when(mockQueryShardContext.fieldMapper(anyString())).thenReturn(mockKNNVectorField); + MethodComponentContext methodComponentContext = new MethodComponentContext(METHOD_HNSW, ImmutableMap.of()); + KNNMethodContext knnMethodContext = new KNNMethodContext(KNNEngine.LUCENE, SpaceType.L2, methodComponentContext); + when(mockKNNVectorField.getKnnMethodContext()).thenReturn(knnMethodContext); + FloatVectorSimilarityQuery query = (FloatVectorSimilarityQuery) knnQueryBuilder.doToQuery(mockQueryShardContext); + assertTrue(query.toString().contains("resultSimilarity=" + KNNEngine.LUCENE.distanceToRadialThreshold(MAX_DISTANCE, SpaceType.L2))); + } + + public void testDoToQuery_whenNormal_whenDoRadiusSearch_whenScoreThreshold_thenSucceed() { + float[] queryVector = { 1.0f, 2.0f, 3.0f, 4.0f }; + KNNQueryBuilder knnQueryBuilder = new KNNQueryBuilder(FIELD_NAME, queryVector).minScore(MIN_SCORE); + Index dummyIndex = new Index("dummy", "dummy"); + QueryShardContext mockQueryShardContext = mock(QueryShardContext.class); + KNNVectorFieldMapper.KNNVectorFieldType mockKNNVectorField = mock(KNNVectorFieldMapper.KNNVectorFieldType.class); + when(mockQueryShardContext.index()).thenReturn(dummyIndex); + when(mockKNNVectorField.getDimension()).thenReturn(4); + when(mockKNNVectorField.getVectorDataType()).thenReturn(VectorDataType.FLOAT); + when(mockQueryShardContext.fieldMapper(anyString())).thenReturn(mockKNNVectorField); + MethodComponentContext methodComponentContext = new MethodComponentContext(METHOD_HNSW, ImmutableMap.of()); + KNNMethodContext knnMethodContext = new KNNMethodContext(KNNEngine.LUCENE, SpaceType.L2, methodComponentContext); + when(mockKNNVectorField.getKnnMethodContext()).thenReturn(knnMethodContext); + FloatVectorSimilarityQuery query = (FloatVectorSimilarityQuery) knnQueryBuilder.doToQuery(mockQueryShardContext); + assertTrue(query.toString().contains("resultSimilarity=" + 0.5f)); + } + + public void testDoToQuery_whenDoRadiusSearch_whenPassNegativeDistance_whenSupportedSpaceType_thenSucceed() { + float[] queryVector = { 1.0f, 2.0f, 3.0f, 4.0f }; + float negativeDistance = -1.0f; + KNNQueryBuilder knnQueryBuilder = new KNNQueryBuilder(FIELD_NAME, queryVector).maxDistance(negativeDistance); + Index dummyIndex = new Index("dummy", "dummy"); + QueryShardContext mockQueryShardContext = mock(QueryShardContext.class); + KNNVectorFieldMapper.KNNVectorFieldType mockKNNVectorField = mock(KNNVectorFieldMapper.KNNVectorFieldType.class); + when(mockQueryShardContext.index()).thenReturn(dummyIndex); + when(mockKNNVectorField.getDimension()).thenReturn(4); + when(mockKNNVectorField.getVectorDataType()).thenReturn(VectorDataType.FLOAT); + when(mockQueryShardContext.fieldMapper(anyString())).thenReturn(mockKNNVectorField); + MethodComponentContext methodComponentContext = new MethodComponentContext(METHOD_HNSW, ImmutableMap.of()); + when(mockKNNVectorField.getKnnMethodContext()).thenReturn( + new KNNMethodContext(KNNEngine.FAISS, SpaceType.INNER_PRODUCT, methodComponentContext) + ); + IndexSettings indexSettings = mock(IndexSettings.class); + when(mockQueryShardContext.getIndexSettings()).thenReturn(indexSettings); + when(indexSettings.getMaxResultWindow()).thenReturn(1000); + + KNNQuery query = (KNNQuery) knnQueryBuilder.doToQuery(mockQueryShardContext); + + assertEquals(negativeDistance, query.getRadius(), 0); + } + + public void testDoToQuery_whenDoRadiusSearch_whenPassNegativeDistance_whenUnSupportedSpaceType_thenException() { + float[] queryVector = { 1.0f, 2.0f, 3.0f, 4.0f }; + float negativeDistance = -1.0f; + KNNQueryBuilder knnQueryBuilder = new KNNQueryBuilder(FIELD_NAME, queryVector).maxDistance(negativeDistance); + Index dummyIndex = new Index("dummy", "dummy"); + QueryShardContext mockQueryShardContext = mock(QueryShardContext.class); + KNNVectorFieldMapper.KNNVectorFieldType mockKNNVectorField = mock(KNNVectorFieldMapper.KNNVectorFieldType.class); + when(mockQueryShardContext.index()).thenReturn(dummyIndex); + when(mockKNNVectorField.getDimension()).thenReturn(4); + when(mockKNNVectorField.getVectorDataType()).thenReturn(VectorDataType.FLOAT); + when(mockQueryShardContext.fieldMapper(anyString())).thenReturn(mockKNNVectorField); + MethodComponentContext methodComponentContext = new MethodComponentContext(METHOD_HNSW, ImmutableMap.of()); + when(mockKNNVectorField.getKnnMethodContext()).thenReturn( + new KNNMethodContext(KNNEngine.FAISS, SpaceType.L2, methodComponentContext) + ); + IndexSettings indexSettings = mock(IndexSettings.class); + when(mockQueryShardContext.getIndexSettings()).thenReturn(indexSettings); + when(indexSettings.getMaxResultWindow()).thenReturn(1000); + + expectThrows(IllegalArgumentException.class, () -> knnQueryBuilder.doToQuery(mockQueryShardContext)); + } + + public void testDoToQuery_whenDoRadiusSearch_whenPassScoreMoreThanOne_whenSupportedSpaceType_thenSucceed() { + float[] queryVector = { 1.0f, 2.0f, 3.0f, 4.0f }; + float score = 5f; + KNNQueryBuilder knnQueryBuilder = new KNNQueryBuilder(FIELD_NAME, queryVector).minScore(score); + Index dummyIndex = new Index("dummy", "dummy"); + QueryShardContext mockQueryShardContext = mock(QueryShardContext.class); + KNNVectorFieldMapper.KNNVectorFieldType mockKNNVectorField = mock(KNNVectorFieldMapper.KNNVectorFieldType.class); + when(mockQueryShardContext.index()).thenReturn(dummyIndex); + when(mockKNNVectorField.getDimension()).thenReturn(4); + when(mockKNNVectorField.getVectorDataType()).thenReturn(VectorDataType.FLOAT); + when(mockQueryShardContext.fieldMapper(anyString())).thenReturn(mockKNNVectorField); + MethodComponentContext methodComponentContext = new MethodComponentContext(METHOD_HNSW, ImmutableMap.of()); + when(mockKNNVectorField.getKnnMethodContext()).thenReturn( + new KNNMethodContext(KNNEngine.FAISS, SpaceType.INNER_PRODUCT, methodComponentContext) + ); + IndexSettings indexSettings = mock(IndexSettings.class); + when(mockQueryShardContext.getIndexSettings()).thenReturn(indexSettings); + when(indexSettings.getMaxResultWindow()).thenReturn(1000); + + KNNQuery query = (KNNQuery) knnQueryBuilder.doToQuery(mockQueryShardContext); + + assertEquals(1 - score, query.getRadius(), 0); + } + + public void testDoToQuery_whenDoRadiusSearch_whenPassScoreMoreThanOne_whenUnsupportedSpaceType_thenException() { + float[] queryVector = { 1.0f, 2.0f, 3.0f, 4.0f }; + float score = 5f; + KNNQueryBuilder knnQueryBuilder = new KNNQueryBuilder(FIELD_NAME, queryVector).minScore(score); + Index dummyIndex = new Index("dummy", "dummy"); + QueryShardContext mockQueryShardContext = mock(QueryShardContext.class); + KNNVectorFieldMapper.KNNVectorFieldType mockKNNVectorField = mock(KNNVectorFieldMapper.KNNVectorFieldType.class); + when(mockQueryShardContext.index()).thenReturn(dummyIndex); + when(mockKNNVectorField.getDimension()).thenReturn(4); + when(mockKNNVectorField.getVectorDataType()).thenReturn(VectorDataType.FLOAT); + when(mockQueryShardContext.fieldMapper(anyString())).thenReturn(mockKNNVectorField); + MethodComponentContext methodComponentContext = new MethodComponentContext(METHOD_HNSW, ImmutableMap.of()); + when(mockKNNVectorField.getKnnMethodContext()).thenReturn( + new KNNMethodContext(KNNEngine.FAISS, SpaceType.L2, methodComponentContext) + ); + IndexSettings indexSettings = mock(IndexSettings.class); + when(mockQueryShardContext.getIndexSettings()).thenReturn(indexSettings); + when(indexSettings.getMaxResultWindow()).thenReturn(1000); + + expectThrows(IllegalArgumentException.class, () -> knnQueryBuilder.doToQuery(mockQueryShardContext)); + } + + public void testDoToQuery_whenPassNegativeDistance_whenSupportedSpaceType_thenSucceed() { + float[] queryVector = { 1.0f, 2.0f, 3.0f, 4.0f }; + float negativeDistance = -1.0f; + KNNQueryBuilder knnQueryBuilder = new KNNQueryBuilder(FIELD_NAME, queryVector).maxDistance(negativeDistance); + Index dummyIndex = new Index("dummy", "dummy"); + QueryShardContext mockQueryShardContext = mock(QueryShardContext.class); + KNNVectorFieldMapper.KNNVectorFieldType mockKNNVectorField = mock(KNNVectorFieldMapper.KNNVectorFieldType.class); + when(mockQueryShardContext.index()).thenReturn(dummyIndex); + when(mockKNNVectorField.getDimension()).thenReturn(4); + when(mockKNNVectorField.getVectorDataType()).thenReturn(VectorDataType.FLOAT); + when(mockQueryShardContext.fieldMapper(anyString())).thenReturn(mockKNNVectorField); + MethodComponentContext methodComponentContext = new MethodComponentContext(METHOD_HNSW, ImmutableMap.of()); + when(mockKNNVectorField.getKnnMethodContext()).thenReturn( + new KNNMethodContext(KNNEngine.FAISS, SpaceType.INNER_PRODUCT, methodComponentContext) + ); + IndexSettings indexSettings = mock(IndexSettings.class); + when(mockQueryShardContext.getIndexSettings()).thenReturn(indexSettings); + when(indexSettings.getMaxResultWindow()).thenReturn(1000); + + KNNQuery query = (KNNQuery) knnQueryBuilder.doToQuery(mockQueryShardContext); + + assertEquals(negativeDistance, query.getRadius(), 0); + } + + public void testDoToQuery_whenPassNegativeDistance_whenUnSupportedSpaceType_thenException() { + float[] queryVector = { 1.0f, 2.0f, 3.0f, 4.0f }; + float negativeDistance = -1.0f; + KNNQueryBuilder knnQueryBuilder = new KNNQueryBuilder(FIELD_NAME, queryVector).maxDistance(negativeDistance); + Index dummyIndex = new Index("dummy", "dummy"); + QueryShardContext mockQueryShardContext = mock(QueryShardContext.class); + KNNVectorFieldMapper.KNNVectorFieldType mockKNNVectorField = mock(KNNVectorFieldMapper.KNNVectorFieldType.class); + when(mockQueryShardContext.index()).thenReturn(dummyIndex); + when(mockKNNVectorField.getDimension()).thenReturn(4); + when(mockKNNVectorField.getVectorDataType()).thenReturn(VectorDataType.FLOAT); + when(mockQueryShardContext.fieldMapper(anyString())).thenReturn(mockKNNVectorField); + MethodComponentContext methodComponentContext = new MethodComponentContext(METHOD_HNSW, ImmutableMap.of()); + when(mockKNNVectorField.getKnnMethodContext()).thenReturn( + new KNNMethodContext(KNNEngine.FAISS, SpaceType.L2, methodComponentContext) + ); + IndexSettings indexSettings = mock(IndexSettings.class); + when(mockQueryShardContext.getIndexSettings()).thenReturn(indexSettings); + when(indexSettings.getMaxResultWindow()).thenReturn(1000); + + expectThrows(IllegalArgumentException.class, () -> knnQueryBuilder.doToQuery(mockQueryShardContext)); + } + public void testDoToQuery_KnnQueryWithFilter() throws Exception { float[] queryVector = { 1.0f, 2.0f, 3.0f, 4.0f }; KNNQueryBuilder knnQueryBuilder = new KNNQueryBuilder(FIELD_NAME, queryVector, K, TERM_QUERY); @@ -272,6 +593,42 @@ public void testDoToQuery_KnnQueryWithFilter() throws Exception { assertTrue(query.getClass().isAssignableFrom(KnnFloatVectorQuery.class)); } + public void testDoToQuery_whenDoRadiusSearch_whenDistanceThreshold_whenFilter_thenSucceed() { + float[] queryVector = { 1.0f, 2.0f, 3.0f, 4.0f }; + KNNQueryBuilder knnQueryBuilder = new KNNQueryBuilder(FIELD_NAME, queryVector).maxDistance(MAX_DISTANCE).filter(TERM_QUERY); + Index dummyIndex = new Index("dummy", "dummy"); + QueryShardContext mockQueryShardContext = mock(QueryShardContext.class); + KNNVectorFieldMapper.KNNVectorFieldType mockKNNVectorField = mock(KNNVectorFieldMapper.KNNVectorFieldType.class); + when(mockQueryShardContext.index()).thenReturn(dummyIndex); + when(mockKNNVectorField.getDimension()).thenReturn(4); + when(mockKNNVectorField.getVectorDataType()).thenReturn(VectorDataType.FLOAT); + MethodComponentContext methodComponentContext = new MethodComponentContext(METHOD_HNSW, ImmutableMap.of()); + KNNMethodContext knnMethodContext = new KNNMethodContext(KNNEngine.LUCENE, SpaceType.L2, methodComponentContext); + when(mockKNNVectorField.getKnnMethodContext()).thenReturn(knnMethodContext); + when(mockQueryShardContext.fieldMapper(anyString())).thenReturn(mockKNNVectorField); + Query query = knnQueryBuilder.doToQuery(mockQueryShardContext); + assertNotNull(query); + assertTrue(query.getClass().isAssignableFrom(FloatVectorSimilarityQuery.class)); + } + + public void testDoToQuery_whenDoRadiusSearch_whenScoreThreshold_whenFilter_thenSucceed() { + float[] queryVector = { 1.0f, 2.0f, 3.0f, 4.0f }; + KNNQueryBuilder knnQueryBuilder = new KNNQueryBuilder(FIELD_NAME, queryVector).minScore(MIN_SCORE).filter(TERM_QUERY); + Index dummyIndex = new Index("dummy", "dummy"); + QueryShardContext mockQueryShardContext = mock(QueryShardContext.class); + KNNVectorFieldMapper.KNNVectorFieldType mockKNNVectorField = mock(KNNVectorFieldMapper.KNNVectorFieldType.class); + when(mockQueryShardContext.index()).thenReturn(dummyIndex); + when(mockKNNVectorField.getDimension()).thenReturn(4); + when(mockKNNVectorField.getVectorDataType()).thenReturn(VectorDataType.FLOAT); + MethodComponentContext methodComponentContext = new MethodComponentContext(METHOD_HNSW, ImmutableMap.of()); + KNNMethodContext knnMethodContext = new KNNMethodContext(KNNEngine.LUCENE, SpaceType.L2, methodComponentContext); + when(mockKNNVectorField.getKnnMethodContext()).thenReturn(knnMethodContext); + when(mockQueryShardContext.fieldMapper(anyString())).thenReturn(mockKNNVectorField); + Query query = knnQueryBuilder.doToQuery(mockQueryShardContext); + assertNotNull(query); + assertTrue(query.getClass().isAssignableFrom(FloatVectorSimilarityQuery.class)); + } + public void testDoToQuery_WhenknnQueryWithFilterAndFaissEngine_thenSuccess() { float[] queryVector = { 1.0f, 2.0f, 3.0f, 4.0f }; KNNQueryBuilder knnQueryBuilder = new KNNQueryBuilder(FIELD_NAME, queryVector, K, TERM_QUERY); @@ -337,6 +694,70 @@ public void testDoToQuery_FromModel() { assertEquals(knnQueryBuilder.vector(), query.getQueryVector()); } + public void testDoToQuery_whenFromModel_whenDoRadiusSearch_whenDistanceThreshold_thenSucceed() { + float[] queryVector = { 1.0f, 2.0f, 3.0f, 4.0f }; + KNNQueryBuilder knnQueryBuilder = new KNNQueryBuilder(FIELD_NAME, queryVector).maxDistance(MAX_DISTANCE); + Index dummyIndex = new Index("dummy", "dummy"); + QueryShardContext mockQueryShardContext = mock(QueryShardContext.class); + KNNVectorFieldMapper.KNNVectorFieldType mockKNNVectorField = mock(KNNVectorFieldMapper.KNNVectorFieldType.class); + when(mockQueryShardContext.index()).thenReturn(dummyIndex); + + when(mockKNNVectorField.getDimension()).thenReturn(-K); + when(mockKNNVectorField.getVectorDataType()).thenReturn(VectorDataType.FLOAT); + when(mockKNNVectorField.getKnnMethodContext()).thenReturn(null); + String modelId = "test-model-id"; + when(mockKNNVectorField.getModelId()).thenReturn(modelId); + + ModelMetadata modelMetadata = mock(ModelMetadata.class); + when(modelMetadata.getDimension()).thenReturn(4); + when(modelMetadata.getKnnEngine()).thenReturn(KNNEngine.FAISS); + when(modelMetadata.getSpaceType()).thenReturn(SpaceType.L2); + ModelDao modelDao = mock(ModelDao.class); + when(modelDao.getMetadata(modelId)).thenReturn(modelMetadata); + KNNQueryBuilder.initialize(modelDao); + IndexSettings indexSettings = mock(IndexSettings.class); + when(mockQueryShardContext.getIndexSettings()).thenReturn(indexSettings); + when(indexSettings.getMaxResultWindow()).thenReturn(1000); + when(mockQueryShardContext.fieldMapper(anyString())).thenReturn(mockKNNVectorField); + + KNNQuery query = (KNNQuery) knnQueryBuilder.doToQuery(mockQueryShardContext); + assertEquals(knnQueryBuilder.getMaxDistance(), query.getRadius(), 0); + assertEquals(knnQueryBuilder.fieldName(), query.getField()); + assertEquals(knnQueryBuilder.vector(), query.getQueryVector()); + } + + public void testDoToQuery_whenFromModel_whenDoRadiusSearch_whenScoreThreshold_thenSucceed() { + float[] queryVector = { 1.0f, 2.0f, 3.0f, 4.0f }; + KNNQueryBuilder knnQueryBuilder = new KNNQueryBuilder(FIELD_NAME, queryVector).minScore(MIN_SCORE); + Index dummyIndex = new Index("dummy", "dummy"); + QueryShardContext mockQueryShardContext = mock(QueryShardContext.class); + KNNVectorFieldMapper.KNNVectorFieldType mockKNNVectorField = mock(KNNVectorFieldMapper.KNNVectorFieldType.class); + when(mockQueryShardContext.index()).thenReturn(dummyIndex); + + when(mockKNNVectorField.getDimension()).thenReturn(-K); + when(mockKNNVectorField.getVectorDataType()).thenReturn(VectorDataType.FLOAT); + when(mockKNNVectorField.getKnnMethodContext()).thenReturn(null); + String modelId = "test-model-id"; + when(mockKNNVectorField.getModelId()).thenReturn(modelId); + + ModelMetadata modelMetadata = mock(ModelMetadata.class); + when(modelMetadata.getDimension()).thenReturn(4); + when(modelMetadata.getKnnEngine()).thenReturn(KNNEngine.FAISS); + when(modelMetadata.getSpaceType()).thenReturn(SpaceType.L2); + ModelDao modelDao = mock(ModelDao.class); + when(modelDao.getMetadata(modelId)).thenReturn(modelMetadata); + KNNQueryBuilder.initialize(modelDao); + IndexSettings indexSettings = mock(IndexSettings.class); + when(mockQueryShardContext.getIndexSettings()).thenReturn(indexSettings); + when(indexSettings.getMaxResultWindow()).thenReturn(1000); + when(mockQueryShardContext.fieldMapper(anyString())).thenReturn(mockKNNVectorField); + KNNQuery query = (KNNQuery) knnQueryBuilder.doToQuery(mockQueryShardContext); + + assertEquals(1 / knnQueryBuilder.getMinScore() - 1, query.getRadius(), 0); + assertEquals(knnQueryBuilder.fieldName(), query.getField()); + assertEquals(knnQueryBuilder.vector(), query.getQueryVector()); + } + public void testDoToQuery_InvalidDimensions() { float[] queryVector = { 1.0f, 2.0f, 3.0f, 4.0f }; KNNQueryBuilder knnQueryBuilder = new KNNQueryBuilder(FIELD_NAME, queryVector, K); @@ -405,11 +826,18 @@ public void testDoToQuery_InvalidZeroByteVector() { } public void testSerialization() throws Exception { - assertSerialization(Version.CURRENT, Optional.empty()); - - assertSerialization(Version.CURRENT, Optional.of(TERM_QUERY)); - - assertSerialization(Version.V_2_3_0, Optional.empty()); + // For k-NN search + assertSerialization(Version.CURRENT, Optional.empty(), K, null, null); + assertSerialization(Version.CURRENT, Optional.of(TERM_QUERY), K, null, null); + assertSerialization(Version.V_2_3_0, Optional.empty(), K, null, null); + + // For distance threshold search + assertSerialization(Version.CURRENT, Optional.empty(), null, MAX_DISTANCE, null); + assertSerialization(Version.CURRENT, Optional.of(TERM_QUERY), null, MAX_DISTANCE, null); + + // For score threshold search + assertSerialization(Version.CURRENT, Optional.empty(), null, null, MIN_SCORE); + assertSerialization(Version.CURRENT, Optional.of(TERM_QUERY), null, null, MIN_SCORE); } public void testIgnoreUnmapped() throws IOException { @@ -424,10 +852,14 @@ public void testIgnoreUnmapped() throws IOException { expectThrows(IllegalArgumentException.class, () -> knnQueryBuilder.doToQuery(mock(QueryShardContext.class))); } - private void assertSerialization(final Version version, final Optional queryBuilderOptional) throws Exception { - final KNNQueryBuilder knnQueryBuilder = queryBuilderOptional.isPresent() - ? new KNNQueryBuilder(FIELD_NAME, QUERY_VECTOR, K, queryBuilderOptional.get()) - : new KNNQueryBuilder(FIELD_NAME, QUERY_VECTOR, K); + private void assertSerialization( + final Version version, + final Optional queryBuilderOptional, + Integer k, + Float distance, + Float score + ) throws Exception { + final KNNQueryBuilder knnQueryBuilder = getKnnQueryBuilder(queryBuilderOptional, k, distance, score); final ClusterService clusterService = mockClusterService(version); @@ -446,7 +878,13 @@ private void assertSerialization(final Version version, final Optional queryBuilderOptional, Integer k, Float distance, Float score) { + final KNNQueryBuilder knnQueryBuilder; + if (k != null) { + knnQueryBuilder = queryBuilderOptional.isPresent() + ? new KNNQueryBuilder(FIELD_NAME, QUERY_VECTOR, k, queryBuilderOptional.get()) + : new KNNQueryBuilder(FIELD_NAME, QUERY_VECTOR, k); + } else if (distance != null) { + knnQueryBuilder = queryBuilderOptional.isPresent() + ? new KNNQueryBuilder(FIELD_NAME, QUERY_VECTOR).maxDistance(distance).filter(queryBuilderOptional.get()) + : new KNNQueryBuilder(FIELD_NAME, QUERY_VECTOR).maxDistance(distance); + } else if (score != null) { + knnQueryBuilder = queryBuilderOptional.isPresent() + ? new KNNQueryBuilder(FIELD_NAME, QUERY_VECTOR).minScore(score).filter(queryBuilderOptional.get()) + : new KNNQueryBuilder(FIELD_NAME, QUERY_VECTOR).minScore(score); + } else { + throw new IllegalArgumentException("Either k or distance must be provided"); + } + return knnQueryBuilder; + } + + public void testRadialSearch_whenUnsupportedEngine_thenThrowException() { + List unsupportedEngines = Arrays.stream(KNNEngine.values()) + .filter(knnEngine -> !ENGINES_SUPPORTING_RADIAL_SEARCH.contains(knnEngine)) + .collect(Collectors.toList()); + for (KNNEngine knnEngine : unsupportedEngines) { + KNNMethodContext knnMethodContext = new KNNMethodContext( + knnEngine, + SpaceType.L2, + new MethodComponentContext(METHOD_HNSW, ImmutableMap.of()) + ); + KNNQueryBuilder knnQueryBuilder = new KNNQueryBuilder(FIELD_NAME, QUERY_VECTOR).maxDistance(MAX_DISTANCE); + KNNVectorFieldMapper.KNNVectorFieldType mockKNNVectorField = mock(KNNVectorFieldMapper.KNNVectorFieldType.class); + QueryShardContext mockQueryShardContext = mock(QueryShardContext.class); + Index dummyIndex = new Index("dummy", "dummy"); + when(mockKNNVectorField.getKnnMethodContext()).thenReturn(knnMethodContext); + when(mockQueryShardContext.index()).thenReturn(dummyIndex); + when(mockKNNVectorField.getDimension()).thenReturn(4); + when(mockQueryShardContext.fieldMapper(anyString())).thenReturn(mockKNNVectorField); + + expectThrows(UnsupportedOperationException.class, () -> knnQueryBuilder.doToQuery(mockQueryShardContext)); + } + } } diff --git a/src/test/java/org/opensearch/knn/index/query/KNNWeightTests.java b/src/test/java/org/opensearch/knn/index/query/KNNWeightTests.java index 2e6895d0b..795635b68 100644 --- a/src/test/java/org/opensearch/knn/index/query/KNNWeightTests.java +++ b/src/test/java/org/opensearch/knn/index/query/KNNWeightTests.java @@ -61,6 +61,7 @@ import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.ArgumentMatchers.anyFloat; import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.anyString; @@ -691,6 +692,68 @@ public void testANNWithParentsFilter_whenDoingANN_thenBitSetIsPassedToJNI() { assertEquals(DOC_ID_TO_SCORES.size(), docIdSetIterator.cost()); } + @SneakyThrows + public void testDoANNSearch_whenRadialIsDefined_thenCallJniRadiusQueryIndex() { + final float[] queryVector = new float[] { 0.1f, 0.3f }; + final float radius = 0.5f; + final int maxResults = 1000; + jniServiceMockedStatic.when(() -> JNIService.radiusQueryIndex(anyLong(), any(), anyFloat(), any(), anyInt())) + .thenReturn(getKNNQueryResults()); + KNNQuery.Context context = mock(KNNQuery.Context.class); + when(context.getMaxResultWindow()).thenReturn(maxResults); + + final KNNQuery query = new KNNQuery(FIELD_NAME, queryVector, INDEX_NAME, null).radius(radius).kNNQueryContext(context); + final float boost = (float) randomDoubleBetween(0, 10, true); + final KNNWeight knnWeight = new KNNWeight(query, boost); + + final LeafReaderContext leafReaderContext = mock(LeafReaderContext.class); + final SegmentReader reader = mock(SegmentReader.class); + when(leafReaderContext.reader()).thenReturn(reader); + + final FSDirectory directory = mock(FSDirectory.class); + when(reader.directory()).thenReturn(directory); + final SegmentInfo segmentInfo = new SegmentInfo( + directory, + Version.LATEST, + Version.LATEST, + SEGMENT_NAME, + 100, + true, + false, + KNNCodecVersion.current().getDefaultCodecDelegate(), + Map.of(), + new byte[StringHelper.ID_LENGTH], + Map.of(), + Sort.RELEVANCE + ); + segmentInfo.setFiles(SEGMENT_FILES_FAISS); + final SegmentCommitInfo segmentCommitInfo = new SegmentCommitInfo(segmentInfo, 0, 0, 0, 0, 0, new byte[StringHelper.ID_LENGTH]); + when(reader.getSegmentInfo()).thenReturn(segmentCommitInfo); + + final Path path = mock(Path.class); + when(directory.getDirectory()).thenReturn(path); + final FieldInfos fieldInfos = mock(FieldInfos.class); + final FieldInfo fieldInfo = mock(FieldInfo.class); + when(reader.getFieldInfos()).thenReturn(fieldInfos); + when(fieldInfos.fieldInfo(any())).thenReturn(fieldInfo); + when(fieldInfo.attributes()).thenReturn(Map.of(SPACE_TYPE, SpaceType.L2.getValue(), KNN_ENGINE, KNNEngine.FAISS.getName())); + + final KNNScorer knnScorer = (KNNScorer) knnWeight.scorer(leafReaderContext); + assertNotNull(knnScorer); + jniServiceMockedStatic.verify(() -> JNIService.radiusQueryIndex(anyLong(), any(), anyFloat(), any(), anyInt())); + + final DocIdSetIterator docIdSetIterator = knnScorer.iterator(); + + final List actualDocIds = new ArrayList<>(); + final Map translatedScores = getTranslatedScores(SpaceType.L2::scoreTranslation); + for (int docId = docIdSetIterator.nextDoc(); docId != NO_MORE_DOCS; docId = docIdSetIterator.nextDoc()) { + actualDocIds.add(docId); + assertEquals(translatedScores.get(docId) * boost, knnScorer.score(), 0.01f); + } + assertEquals(docIdSetIterator.cost(), actualDocIds.size()); + assertTrue(Comparators.isInOrder(actualDocIds, Comparator.naturalOrder())); + } + private SegmentReader getMockedSegmentReader() { final SegmentReader reader = mock(SegmentReader.class); when(reader.maxDoc()).thenReturn(1); diff --git a/src/test/java/org/opensearch/knn/index/query/RNNQueryFactoryTests.java b/src/test/java/org/opensearch/knn/index/query/RNNQueryFactoryTests.java new file mode 100644 index 000000000..5492b8506 --- /dev/null +++ b/src/test/java/org/opensearch/knn/index/query/RNNQueryFactoryTests.java @@ -0,0 +1,134 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.query; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import static org.opensearch.knn.common.KNNConstants.DEFAULT_VECTOR_DATA_TYPE_FIELD; + +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; + +import org.apache.lucene.search.ByteVectorSimilarityQuery; +import org.apache.lucene.search.FloatVectorSimilarityQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.join.BitSetProducer; +import org.opensearch.index.IndexSettings; +import org.opensearch.index.mapper.MappedFieldType; +import org.opensearch.index.query.QueryBuilder; +import org.opensearch.index.query.QueryShardContext; +import org.opensearch.index.query.TermQueryBuilder; +import org.opensearch.knn.KNNTestCase; +import org.opensearch.knn.index.VectorDataType; +import org.opensearch.knn.index.util.KNNEngine; + +public class RNNQueryFactoryTests extends KNNTestCase { + private static final String FILTER_FILED_NAME = "foo"; + private static final String FILTER_FILED_VALUE = "fooval"; + private static final QueryBuilder FILTER_QUERY_BUILDER = new TermQueryBuilder(FILTER_FILED_NAME, FILTER_FILED_VALUE); + private final int testQueryDimension = 17; + private final float[] testQueryVector = new float[testQueryDimension]; + private final byte[] testByteQueryVector = new byte[testQueryDimension]; + private final String testIndexName = "test-index"; + private final String testFieldName = "test-field"; + private final Float testRadius = 0.5f; + private final int maxResultWindow = 20000; + + public void testCreate_whenLucene_withRadiusQuery_withFloatVector() { + List luceneDefaultQueryEngineList = Arrays.stream(KNNEngine.values()) + .filter(knnEngine -> !KNNEngine.getEnginesThatCreateCustomSegmentFiles().contains(knnEngine)) + .collect(Collectors.toList()); + for (KNNEngine knnEngine : luceneDefaultQueryEngineList) { + Query query = RNNQueryFactory.create( + knnEngine, + testIndexName, + testFieldName, + testQueryVector, + testRadius, + DEFAULT_VECTOR_DATA_TYPE_FIELD + ); + assertEquals(FloatVectorSimilarityQuery.class, query.getClass()); + } + } + + public void testCreate_whenLucene_withRadiusQuery_withByteVector() { + List luceneDefaultQueryEngineList = Arrays.stream(KNNEngine.values()) + .filter(knnEngine -> !KNNEngine.getEnginesThatCreateCustomSegmentFiles().contains(knnEngine)) + .collect(Collectors.toList()); + for (KNNEngine knnEngine : luceneDefaultQueryEngineList) { + QueryShardContext mockQueryShardContext = mock(QueryShardContext.class); + MappedFieldType testMapper = mock(MappedFieldType.class); + when(mockQueryShardContext.fieldMapper(any())).thenReturn(testMapper); + BitSetProducer parentFilter = mock(BitSetProducer.class); + when(mockQueryShardContext.getParentFilter()).thenReturn(parentFilter); + final RNNQueryFactory.CreateQueryRequest createQueryRequest = RNNQueryFactory.CreateQueryRequest.builder() + .knnEngine(knnEngine) + .indexName(testIndexName) + .fieldName(testFieldName) + .vector(testQueryVector) + .radius(testRadius) + .byteVector(testByteQueryVector) + .vectorDataType(VectorDataType.BYTE) + .context(mockQueryShardContext) + .filter(FILTER_QUERY_BUILDER) + .build(); + Query query = RNNQueryFactory.create(createQueryRequest); + assertEquals(ByteVectorSimilarityQuery.class, query.getClass()); + } + } + + public void testCreate_whenLucene_withFilter_thenSucceed() { + List luceneDefaultQueryEngineList = Arrays.stream(KNNEngine.values()) + .filter(knnEngine -> !KNNEngine.getEnginesThatCreateCustomSegmentFiles().contains(knnEngine)) + .collect(Collectors.toList()); + for (KNNEngine knnEngine : luceneDefaultQueryEngineList) { + QueryShardContext mockQueryShardContext = mock(QueryShardContext.class); + MappedFieldType testMapper = mock(MappedFieldType.class); + when(mockQueryShardContext.fieldMapper(any())).thenReturn(testMapper); + final RNNQueryFactory.CreateQueryRequest createQueryRequest = RNNQueryFactory.CreateQueryRequest.builder() + .knnEngine(knnEngine) + .indexName(testIndexName) + .fieldName(testFieldName) + .vector(testQueryVector) + .vectorDataType(DEFAULT_VECTOR_DATA_TYPE_FIELD) + .context(mockQueryShardContext) + .filter(FILTER_QUERY_BUILDER) + .radius(testRadius) + .build(); + Query query = RNNQueryFactory.create(createQueryRequest); + assertEquals(FloatVectorSimilarityQuery.class, query.getClass()); + } + } + + public void testCreate_whenFaiss_thenSucceed() { + QueryShardContext mockQueryShardContext = mock(QueryShardContext.class); + MappedFieldType testMapper = mock(MappedFieldType.class); + IndexSettings indexSettings = mock(IndexSettings.class); + when(mockQueryShardContext.getIndexSettings()).thenReturn(indexSettings); + when(mockQueryShardContext.fieldMapper(any())).thenReturn(testMapper); + when(mockQueryShardContext.getIndexSettings().getMaxResultWindow()).thenReturn(maxResultWindow); + final RNNQueryFactory.CreateQueryRequest createQueryRequest = RNNQueryFactory.CreateQueryRequest.builder() + .knnEngine(KNNEngine.FAISS) + .indexName(testIndexName) + .fieldName(testFieldName) + .vector(testQueryVector) + .radius(testRadius) + .vectorDataType(DEFAULT_VECTOR_DATA_TYPE_FIELD) + .context(mockQueryShardContext) + .build(); + + Query query = RNNQueryFactory.create(createQueryRequest); + + assertTrue(query instanceof KNNQuery); + assertEquals(testIndexName, ((KNNQuery) query).getIndexName()); + assertEquals(testFieldName, ((KNNQuery) query).getField()); + assertEquals(testQueryVector, ((KNNQuery) query).getQueryVector()); + assertEquals(testRadius, ((KNNQuery) query).getRadius(), 0); + assertEquals(maxResultWindow, ((KNNQuery) query).getContext().getMaxResultWindow()); + } +} diff --git a/src/test/java/org/opensearch/knn/index/util/AbstractKNNLibraryTests.java b/src/test/java/org/opensearch/knn/index/util/AbstractKNNLibraryTests.java index 916e87414..9e6bd67ea 100644 --- a/src/test/java/org/opensearch/knn/index/util/AbstractKNNLibraryTests.java +++ b/src/test/java/org/opensearch/knn/index/util/AbstractKNNLibraryTests.java @@ -127,6 +127,15 @@ public float score(float rawScore, SpaceType spaceType) { return 0; } + @Override + public Float distanceToRadialThreshold(Float distance, SpaceType spaceType) { + return 0f; + } + + public Float scoreToRadialThreshold(Float score, SpaceType spaceType) { + return 0f; + } + @Override public int estimateOverheadInKB(KNNMethodContext knnMethodContext, int dimension) { return 0; diff --git a/src/test/java/org/opensearch/knn/index/util/LuceneTests.java b/src/test/java/org/opensearch/knn/index/util/LuceneTests.java index 6de46b52d..c9ffd13b2 100644 --- a/src/test/java/org/opensearch/knn/index/util/LuceneTests.java +++ b/src/test/java/org/opensearch/knn/index/util/LuceneTests.java @@ -99,28 +99,28 @@ public void testLucenHNSWMethod() throws IOException { } public void testGetExtension() { - Lucene luceneLibrary = new Lucene(Collections.emptyMap(), ""); + Lucene luceneLibrary = new Lucene(Collections.emptyMap(), "", Collections.emptyMap()); expectThrows(UnsupportedOperationException.class, luceneLibrary::getExtension); } public void testGetCompundExtension() { - Lucene luceneLibrary = new Lucene(Collections.emptyMap(), ""); + Lucene luceneLibrary = new Lucene(Collections.emptyMap(), "", Collections.emptyMap()); expectThrows(UnsupportedOperationException.class, luceneLibrary::getCompoundExtension); } public void testScore() { - Lucene luceneLibrary = new Lucene(Collections.emptyMap(), ""); + Lucene luceneLibrary = new Lucene(Collections.emptyMap(), "", Collections.emptyMap()); float rawScore = 10.0f; assertEquals(rawScore, luceneLibrary.score(rawScore, SpaceType.DEFAULT), 0.001); } public void testIsInitialized() { - Lucene luceneLibrary = new Lucene(Collections.emptyMap(), ""); + Lucene luceneLibrary = new Lucene(Collections.emptyMap(), "", Collections.emptyMap()); assertFalse(luceneLibrary.isInitialized()); } public void testSetInitialized() { - Lucene luceneLibrary = new Lucene(Collections.emptyMap(), ""); + Lucene luceneLibrary = new Lucene(Collections.emptyMap(), "", Collections.emptyMap()); luceneLibrary.setInitialized(true); assertTrue(luceneLibrary.isInitialized()); } diff --git a/src/test/java/org/opensearch/knn/index/util/NativeLibraryTests.java b/src/test/java/org/opensearch/knn/index/util/NativeLibraryTests.java index 00a628f1e..3c3afbee6 100644 --- a/src/test/java/org/opensearch/knn/index/util/NativeLibraryTests.java +++ b/src/test/java/org/opensearch/knn/index/util/NativeLibraryTests.java @@ -64,5 +64,15 @@ public TestNativeLibrary( ) { super(methods, scoreTranslation, currentVersion, extension); } + + @Override + public Float distanceToRadialThreshold(Float distance, SpaceType spaceType) { + return 0.0f; + } + + @Override + public Float scoreToRadialThreshold(Float score, SpaceType spaceType) { + return 0.0f; + } } } diff --git a/src/test/java/org/opensearch/knn/plugin/stats/suppliers/LibraryInitializedSupplierTests.java b/src/test/java/org/opensearch/knn/plugin/stats/suppliers/LibraryInitializedSupplierTests.java index 64cf86381..cff4d5805 100644 --- a/src/test/java/org/opensearch/knn/plugin/stats/suppliers/LibraryInitializedSupplierTests.java +++ b/src/test/java/org/opensearch/knn/plugin/stats/suppliers/LibraryInitializedSupplierTests.java @@ -63,6 +63,16 @@ public float score(float rawScore, SpaceType spaceType) { return 0; } + @Override + public Float distanceToRadialThreshold(Float distance, SpaceType spaceType) { + return 0.0f; + } + + @Override + public Float scoreToRadialThreshold(Float score, SpaceType spaceType) { + return 0.0f; + } + @Override public ValidationException validateMethod(KNNMethodContext knnMethodContext) { return null; diff --git a/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java b/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java index 0331d49e5..ae4d3a6c3 100644 --- a/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java +++ b/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java @@ -203,6 +203,23 @@ protected Response searchKNNIndex(String index, KNNQueryBuilder knnQueryBuilder, return response; } + /** + * Run KNN Search on Index with XContentBuilder query + */ + protected Response searchKNNIndex(String index, XContentBuilder xContentBuilder, int resultSize) throws IOException { + Request request = new Request("POST", "/" + index + "/_search"); + request.setJsonEntity(xContentBuilder.toString()); + + request.addParameter("size", Integer.toString(resultSize)); + request.addParameter("explain", Boolean.toString(true)); + request.addParameter("search_type", "query_then_fetch"); + + Response response = client().performRequest(request); + assertEquals(request.getEndpoint() + ": failed", RestStatus.OK, RestStatus.fromCode(response.getStatusLine().getStatusCode())); + + return response; + } + /** * Run exists search */ From 579bbb1cce20e8efffdc74524b82b731aa7f1bf3 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Mon, 22 Apr 2024 14:33:33 -0500 Subject: [PATCH 240/416] Fix flaky faiss integration tests (#1635) (#1637) (cherry picked from commit 730b7e430f9b25bfddaddcb7b178e15ac2a3ef06) Signed-off-by: Naveen Tatikonda Co-authored-by: Naveen Tatikonda --- src/test/java/org/opensearch/knn/index/FaissIT.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/test/java/org/opensearch/knn/index/FaissIT.java b/src/test/java/org/opensearch/knn/index/FaissIT.java index 82ac12a73..61b0461ef 100644 --- a/src/test/java/org/opensearch/knn/index/FaissIT.java +++ b/src/test/java/org/opensearch/knn/index/FaissIT.java @@ -232,7 +232,7 @@ public void testEndToEnd_whenMethodIsHNSWFlatAndHasDeletedDocs_thenSucceed() { final Set docIdsToBeDeleted = new HashSet<>(); while (docIdsToBeDeleted.size() < 10) { - docIdsToBeDeleted.add(randomInt(testData.indexData.docs.length)); + docIdsToBeDeleted.add(randomInt(testData.indexData.docs.length - 1)); } for (Integer id : docIdsToBeDeleted) { @@ -741,6 +741,7 @@ public void testIVFSQFP16_whenIndexedAndQueried_thenSucceed() { .field(KNN_ENGINE, FAISS_NAME) .field(METHOD_PARAMETER_SPACE_TYPE, "l2") .startObject(PARAMETERS) + .field(METHOD_PARAMETER_NPROBES, 4) .startObject(METHOD_ENCODER_PARAMETER) .field(NAME, ENCODER_SQ) .startObject(PARAMETERS) From ec9ddb4ffe8fa65413735c96c89e9ccb13ab96ce Mon Sep 17 00:00:00 2001 From: John Mazanec Date: Mon, 22 Apr 2024 14:06:47 -0700 Subject: [PATCH 241/416] Add stored fields for knn_vector type (#1640) Fixes bug where we were not creating stored field type for knn_vector even when the mapping parameter is passed. Along with this, clean up the field mapper implementations. Add relevant uTs and iTs to ensure functionality is working as expected. (cherry picked from commit 699510d61dd8c583653118be172fb60a00463760) Signed-off-by: John Mazanec --- CHANGELOG.md | 1 + .../knn/index/KNNVectorScriptDocValues.java | 2 +- .../opensearch/knn/index/VectorDataType.java | 10 +- .../index/mapper/KNNVectorFieldMapper.java | 46 ++--- .../mapper/KNNVectorFieldMapperUtil.java | 42 +++- .../knn/index/mapper/LuceneFieldMapper.java | 11 +- .../index/AdvancedFilteringUseCasesIT.java | 2 - .../knn/index/KNNMapperSearcherIT.java | 182 +++++++++++++++++- .../mapper/KNNVectorFieldMapperUtilTests.java | 54 ++++++ .../org/opensearch/knn/KNNRestTestCase.java | 4 + 10 files changed, 308 insertions(+), 46 deletions(-) create mode 100644 src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperUtilTests.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 9e7b1c2dc..0b37975ba 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), * Implemented the Streaming Feature to stream vectors from Java to JNI layer to enable creation of larger segments for vector indices [#1604](https://github.com/opensearch-project/k-NN/pull/1604) * Remove unnecessary toString conversion of vector field and added some minor optimization in KNNCodec [1613](https://github.com/opensearch-project/k-NN/pull/1613) ### Bug Fixes +* Add stored fields for knn_vector type [#1630](https://github.com/opensearch-project/k-NN/pull/1630) ### Infrastructure * Add micro-benchmark module in k-NN plugin for benchmark streaming vectors to JNI layer functionality. [#1583](https://github.com/opensearch-project/k-NN/pull/1583) * Add arm64 check when SIMD is disabled [#1618](https://github.com/opensearch-project/k-NN/pull/1618) diff --git a/src/main/java/org/opensearch/knn/index/KNNVectorScriptDocValues.java b/src/main/java/org/opensearch/knn/index/KNNVectorScriptDocValues.java index c733c534e..f69ad850e 100644 --- a/src/main/java/org/opensearch/knn/index/KNNVectorScriptDocValues.java +++ b/src/main/java/org/opensearch/knn/index/KNNVectorScriptDocValues.java @@ -126,7 +126,7 @@ private static final class KNNNativeVectorScriptDocValues extends KNNVectorScrip @Override protected float[] doGetValue() throws IOException { - return getVectorDataType().getVectorFromDocValues(values.binaryValue()); + return getVectorDataType().getVectorFromBytesRef(values.binaryValue()); } } diff --git a/src/main/java/org/opensearch/knn/index/VectorDataType.java b/src/main/java/org/opensearch/knn/index/VectorDataType.java index 23b374e9d..98b767f8d 100644 --- a/src/main/java/org/opensearch/knn/index/VectorDataType.java +++ b/src/main/java/org/opensearch/knn/index/VectorDataType.java @@ -37,7 +37,7 @@ public FieldType createKnnVectorFieldType(int dimension, VectorSimilarityFunctio } @Override - public float[] getVectorFromDocValues(BytesRef binaryValue) { + public float[] getVectorFromBytesRef(BytesRef binaryValue) { float[] vector = new float[binaryValue.length]; int i = 0; int j = binaryValue.offset; @@ -56,7 +56,7 @@ public FieldType createKnnVectorFieldType(int dimension, VectorSimilarityFunctio } @Override - public float[] getVectorFromDocValues(BytesRef binaryValue) { + public float[] getVectorFromBytesRef(BytesRef binaryValue) { ByteArrayInputStream byteStream = new ByteArrayInputStream(binaryValue.bytes, binaryValue.offset, binaryValue.length); final KNNVectorSerializer vectorSerializer = KNNVectorSerializerFactory.getSerializerByStreamContent(byteStream); return vectorSerializer.byteToFloatArray(byteStream); @@ -81,12 +81,12 @@ public float[] getVectorFromDocValues(BytesRef binaryValue) { public abstract FieldType createKnnVectorFieldType(int dimension, VectorSimilarityFunction vectorSimilarityFunction); /** - * Deserializes float vector from doc values binary value. + * Deserializes float vector from BytesRef. * - * @param binaryValue Binary Value of DocValues + * @param binaryValue Binary Value * @return float vector deserialized from binary value */ - public abstract float[] getVectorFromDocValues(BytesRef binaryValue); + public abstract float[] getVectorFromBytesRef(BytesRef binaryValue); /** * Validates if given VectorDataType is in the list of supported data types. diff --git a/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapper.java b/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapper.java index 0fa026f34..b314a5dae 100644 --- a/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapper.java +++ b/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapper.java @@ -11,6 +11,7 @@ import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.Objects; import java.util.Optional; import java.util.function.Supplier; import lombok.Getter; @@ -20,6 +21,7 @@ import org.apache.lucene.index.IndexOptions; import org.apache.lucene.search.DocValuesFieldExistsQuery; import org.apache.lucene.search.Query; +import org.apache.lucene.util.BytesRef; import org.opensearch.Version; import org.opensearch.common.Explicit; import org.opensearch.common.Nullable; @@ -52,16 +54,6 @@ import org.opensearch.search.aggregations.support.CoreValuesSourceType; import org.opensearch.search.lookup.SearchLookup; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.Objects; -import java.util.Optional; -import java.util.function.Supplier; - import static org.opensearch.knn.common.KNNConstants.DEFAULT_VECTOR_DATA_TYPE_FIELD; import static org.opensearch.knn.common.KNNConstants.ENCODER_SQ; import static org.opensearch.knn.common.KNNConstants.FAISS_SQ_CLIP; @@ -74,19 +66,17 @@ import static org.opensearch.knn.common.KNNValidationUtil.validateFloatVectorValue; import static org.opensearch.knn.common.KNNValidationUtil.validateVectorDimension; import static org.opensearch.knn.index.KNNSettings.KNN_INDEX; -import static org.opensearch.knn.index.mapper.KNNVectorFieldMapperUtil.addStoredFieldForVectorField; +import static org.opensearch.knn.index.mapper.KNNVectorFieldMapperUtil.createStoredFieldForByteVector; +import static org.opensearch.knn.index.mapper.KNNVectorFieldMapperUtil.createStoredFieldForFloatVector; import static org.opensearch.knn.index.mapper.KNNVectorFieldMapperUtil.clipVectorValueToFP16Range; +import static org.opensearch.knn.index.mapper.KNNVectorFieldMapperUtil.deserializeStoredVector; import static org.opensearch.knn.index.mapper.KNNVectorFieldMapperUtil.validateFP16VectorValue; import static org.opensearch.knn.index.mapper.KNNVectorFieldMapperUtil.validateVectorDataTypeWithEngine; import static org.opensearch.knn.index.mapper.KNNVectorFieldMapperUtil.validateVectorDataTypeWithKnnIndexSetting; /** - * Field Mapper for KNN vector type. - * - * Extends ParametrizedFieldMapper in order to easily configure mapping parameters. - * - * Implementations of this class define what needs to be stored in Lucene's fieldType. This allows us to have - * alternative mappings for the same field type. + * Field Mapper for KNN vector type. Implementations of this class define what needs to be stored in Lucene's fieldType. + * This allows us to have alternative mappings for the same field type. */ @Log4j2 public abstract class KNNVectorFieldMapper extends ParametrizedFieldMapper { @@ -109,8 +99,8 @@ private static KNNVectorFieldMapper toType(FieldMapper in) { public static class Builder extends ParametrizedFieldMapper.Builder { protected Boolean ignoreMalformed; - protected final Parameter stored = Parameter.boolParam("store", false, m -> toType(m).stored, false); - protected final Parameter hasDocValues = Parameter.boolParam("doc_values", false, m -> toType(m).hasDocValues, true); + protected final Parameter stored = Parameter.storeParam(m -> toType(m).stored, false); + protected final Parameter hasDocValues = Parameter.docValuesParam(m -> toType(m).hasDocValues, true); protected final Parameter dimension = new Parameter<>(KNNConstants.DIMENSION, false, () -> -1, (n, c, o) -> { if (o == null) { throw new IllegalArgumentException("Dimension cannot be null"); @@ -483,6 +473,11 @@ public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, S failIfNoDocValues(); return new KNNVectorIndexFieldData.Builder(name(), CoreValuesSourceType.BYTES, this.vectorDataType); } + + @Override + public Object valueForDisplay(Object value) { + return deserializeStoredVector((BytesRef) value, vectorDataType); + } } protected Explicit ignoreMalformed; @@ -561,7 +556,9 @@ protected void parseCreateField(ParseContext context, int dimension, SpaceType s VectorField point = new VectorField(name(), array, fieldType); context.doc().add(point); - addStoredFieldForVectorField(context, fieldType, name(), point); + if (this.stored) { + context.doc().add(createStoredFieldForByteVector(name(), array)); + } } else if (VectorDataType.FLOAT == vectorDataType) { Optional floatsArrayOptional = getFloatsFromContext(context, dimension, methodComponentContext); @@ -572,7 +569,9 @@ protected void parseCreateField(ParseContext context, int dimension, SpaceType s spaceType.validateVector(array); VectorField point = new VectorField(name(), array, fieldType); context.doc().add(point); - addStoredFieldForVectorField(context, fieldType, name(), point); + if (this.stored) { + context.doc().add(createStoredFieldForFloatVector(name(), array)); + } } else { throw new IllegalArgumentException( String.format(Locale.ROOT, "Cannot parse context for unsupported values provided for field [%s]", VECTOR_DATA_TYPE_FIELD) @@ -735,11 +734,6 @@ Optional getFloatsFromContext(ParseContext context, int dimension, Meth return Optional.of(array); } - @Override - protected boolean docValuesByDefault() { - return true; - } - @Override public ParametrizedFieldMapper.Builder getMergeBuilder() { return new KNNVectorFieldMapper.Builder(simpleName(), modelDao, indexCreatedVersion).init(this); diff --git a/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperUtil.java b/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperUtil.java index 074be0375..9b1578a45 100644 --- a/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperUtil.java +++ b/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperUtil.java @@ -13,15 +13,16 @@ import lombok.AccessLevel; import lombok.NoArgsConstructor; -import org.apache.lucene.document.Field; import org.apache.lucene.document.FieldType; import org.apache.lucene.document.StoredField; import org.apache.lucene.index.DocValuesType; +import org.apache.lucene.util.BytesRef; import org.opensearch.index.mapper.ParametrizedFieldMapper; -import org.opensearch.index.mapper.ParseContext; import org.opensearch.knn.index.VectorDataType; +import org.opensearch.knn.index.codec.util.KNNVectorSerializerFactory; import org.opensearch.knn.index.util.KNNEngine; +import java.util.Arrays; import java.util.Locale; import static org.opensearch.knn.common.KNNConstants.ENCODER_SQ; @@ -44,7 +45,6 @@ public class KNNVectorFieldMapperUtil { */ public static void validateFP16VectorValue(float value) { validateFloatVectorValue(value); - if (value < FP16_MIN_VALUE || value > FP16_MAX_VALUE) { throw new IllegalArgumentException( String.format( @@ -136,9 +136,39 @@ public static FieldType buildDocValuesFieldType(KNNEngine knnEngine) { return field; } - public static void addStoredFieldForVectorField(ParseContext context, FieldType fieldType, String mapperName, Field vectorField) { - if (fieldType.stored()) { - context.doc().add(new StoredField(mapperName, vectorField.toString())); + /** + * Creates a stored field for a byte vector + * + * @param name field name + * @param vector vector to be added to stored field + */ + public static StoredField createStoredFieldForByteVector(String name, byte[] vector) { + return new StoredField(name, vector); + } + + /** + * Creates a stored field for a float vector + * + * @param name field name + * @param vector vector to be added to stored field + */ + public static StoredField createStoredFieldForFloatVector(String name, float[] vector) { + return new StoredField(name, KNNVectorSerializerFactory.getDefaultSerializer().floatToByteArray(vector)); + } + + /** + * @param storedVector Vector representation in bytes + * @param vectorDataType type of vector + * @return either int[] or float[] of corresponding vector + */ + public static Object deserializeStoredVector(BytesRef storedVector, VectorDataType vectorDataType) { + if (VectorDataType.BYTE == vectorDataType) { + byte[] bytes = storedVector.bytes; + int[] byteAsIntArray = new int[bytes.length]; + Arrays.setAll(byteAsIntArray, i -> bytes[i]); + return byteAsIntArray; } + + return vectorDataType.getVectorFromBytesRef(storedVector); } } diff --git a/src/main/java/org/opensearch/knn/index/mapper/LuceneFieldMapper.java b/src/main/java/org/opensearch/knn/index/mapper/LuceneFieldMapper.java index d61fa1150..618d77a32 100644 --- a/src/main/java/org/opensearch/knn/index/mapper/LuceneFieldMapper.java +++ b/src/main/java/org/opensearch/knn/index/mapper/LuceneFieldMapper.java @@ -25,7 +25,8 @@ import org.opensearch.knn.index.util.KNNEngine; import static org.opensearch.knn.common.KNNConstants.VECTOR_DATA_TYPE_FIELD; -import static org.opensearch.knn.index.mapper.KNNVectorFieldMapperUtil.addStoredFieldForVectorField; +import static org.opensearch.knn.index.mapper.KNNVectorFieldMapperUtil.createStoredFieldForByteVector; +import static org.opensearch.knn.index.mapper.KNNVectorFieldMapperUtil.createStoredFieldForFloatVector; import static org.opensearch.knn.index.mapper.KNNVectorFieldMapperUtil.buildDocValuesFieldType; /** @@ -92,7 +93,9 @@ protected void parseCreateField(ParseContext context, int dimension, SpaceType s KnnByteVectorField point = new KnnByteVectorField(name(), array, fieldType); context.doc().add(point); - addStoredFieldForVectorField(context, fieldType, name(), point); + if (this.stored) { + context.doc().add(createStoredFieldForByteVector(name(), array)); + } if (hasDocValues && vectorFieldType != null) { context.doc().add(new VectorField(name(), array, vectorFieldType)); @@ -108,7 +111,9 @@ protected void parseCreateField(ParseContext context, int dimension, SpaceType s KnnVectorField point = new KnnVectorField(name(), array, fieldType); context.doc().add(point); - addStoredFieldForVectorField(context, fieldType, name(), point); + if (this.stored) { + context.doc().add(createStoredFieldForFloatVector(name(), array)); + } if (hasDocValues && vectorFieldType != null) { context.doc().add(new VectorField(name(), array, vectorFieldType)); diff --git a/src/test/java/org/opensearch/knn/index/AdvancedFilteringUseCasesIT.java b/src/test/java/org/opensearch/knn/index/AdvancedFilteringUseCasesIT.java index b559b5760..5380dae90 100644 --- a/src/test/java/org/opensearch/knn/index/AdvancedFilteringUseCasesIT.java +++ b/src/test/java/org/opensearch/knn/index/AdvancedFilteringUseCasesIT.java @@ -53,8 +53,6 @@ public class AdvancedFilteringUseCasesIT extends KNNRestTestCase { private static final String FIELD_NAME_VECTOR = "test_vector"; - private static final String PROPERTIES_FIELD = "properties"; - private static final String FILTER_FIELD = "filter"; private static final String TERM_FIELD = "term"; diff --git a/src/test/java/org/opensearch/knn/index/KNNMapperSearcherIT.java b/src/test/java/org/opensearch/knn/index/KNNMapperSearcherIT.java index b2477fa5d..a50691e4b 100644 --- a/src/test/java/org/opensearch/knn/index/KNNMapperSearcherIT.java +++ b/src/test/java/org/opensearch/knn/index/KNNMapperSearcherIT.java @@ -5,20 +5,34 @@ package org.opensearch.knn.index; +import lombok.SneakyThrows; +import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.knn.KNNRestTestCase; import org.opensearch.knn.KNNResult; import org.apache.http.util.EntityUtils; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; import org.opensearch.client.Response; import org.opensearch.knn.index.query.KNNQueryBuilder; +import org.opensearch.knn.index.util.KNNEngine; import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import static org.opensearch.knn.common.KNNConstants.DIMENSION; +import static org.opensearch.knn.common.KNNConstants.KNN_ENGINE; +import static org.opensearch.knn.common.KNNConstants.KNN_METHOD; +import static org.opensearch.knn.common.KNNConstants.METHOD_HNSW; +import static org.opensearch.knn.common.KNNConstants.NAME; +import static org.opensearch.knn.common.KNNConstants.QUERY; +import static org.opensearch.knn.common.KNNConstants.TYPE; +import static org.opensearch.knn.common.KNNConstants.TYPE_KNN_VECTOR; +import static org.opensearch.knn.common.KNNConstants.VECTOR_DATA_TYPE_FIELD; + public class KNNMapperSearcherIT extends KNNRestTestCase { - private static final Logger logger = LogManager.getLogger(KNNMapperSearcherIT.class); + + private static final String INDEX_NAME = "test_index"; + private static final String FIELD_NAME = "test_vector"; /** * Test Data set @@ -239,4 +253,166 @@ public void testLargeK() throws Exception { List results = parseSearchResponse(EntityUtils.toString(response.getEntity()), FIELD_NAME); assertEquals(results.size(), 4); } + + /** + * Request: + * { + * "stored_fields": ["test_vector"], + * "query": { + * "match_all": {} + * } + * } + * + * Example Response: + * { + * "took":248, + * "timed_out":false, + * "_shards":{ + * "total":1, + * "successful":1, + * "skipped":0, + * "failed":0 + * }, + * "hits":{ + * "total":{ + * "value":1, + * "relation":"eq" + * }, + * "max_score":1.0, + * "hits":[ + * { + * "_index":"test_index", + * "_id":"1", + * "_score":1.0, + * "fields":{"test_vector":[[-128,0,1,127]]} + * } + * ] + * } + * } + */ + @SneakyThrows + public void testStoredFields_whenByteDataType_thenSucceed() { + // Create index with stored field and confirm that we can properly retrieve it + int[] testVector = new int[] { -128, 0, 1, 127 }; + String expectedResponse = String.format("\"fields\":{\"%s\":[[-128,0,1,127]]}}", FIELD_NAME); + createKnnIndex( + INDEX_NAME, + createVectorMapping(testVector.length, KNNEngine.LUCENE.getName(), VectorDataType.BYTE.getValue(), true) + ); + addKnnDoc(INDEX_NAME, "1", FIELD_NAME, testVector); + + final XContentBuilder builder = XContentFactory.jsonBuilder(); + builder.startObject(); + builder.field(STORED_QUERY_FIELD, List.of(FIELD_NAME)); + builder.startObject(QUERY); + builder.startObject(MATCH_ALL_QUERY_FIELD); + builder.endObject(); + builder.endObject(); + builder.endObject(); + + String response = EntityUtils.toString(performSearch(INDEX_NAME, builder.toString()).getEntity()); + assertTrue(response.contains(expectedResponse)); + + deleteKNNIndex(INDEX_NAME); + } + + /** + * Request: + * { + * "stored_fields": ["test_vector"], + * "query": { + * "match_all": {} + * } + * } + * + * Example Response: + * { + * "took":248, + * "timed_out":false, + * "_shards":{ + * "total":1, + * "successful":1, + * "skipped":0, + * "failed":0 + * }, + * "hits":{ + * "total":{ + * "value":1, + * "relation":"eq" + * }, + * "max_score":1.0, + * "hits":[ + * { + * "_index":"test_index", + * "_id":"1", + * "_score":1.0, + * "fields":{"test_vector":[[-100.0,100.0,0.0,1.0]]} + * } + * ] + * } + * } + */ + @SneakyThrows + public void testStoredFields_whenFloatDataType_thenSucceed() { + List enginesToTest = List.of(KNNEngine.NMSLIB, KNNEngine.FAISS, KNNEngine.LUCENE); + float[] testVector = new float[] { -100.0f, 100.0f, 0f, 1f }; + String expectedResponse = String.format("\"fields\":{\"%s\":[[-100.0,100.0,0.0,1.0]]}}", FIELD_NAME); + for (KNNEngine knnEngine : enginesToTest) { + createKnnIndex(INDEX_NAME, createVectorMapping(testVector.length, knnEngine.getName(), VectorDataType.FLOAT.getValue(), true)); + addKnnDoc(INDEX_NAME, "1", FIELD_NAME, testVector); + + final XContentBuilder builder = XContentFactory.jsonBuilder(); + builder.startObject(); + builder.field(STORED_QUERY_FIELD, List.of(FIELD_NAME)); + builder.startObject(QUERY); + builder.startObject(MATCH_ALL_QUERY_FIELD); + builder.endObject(); + builder.endObject(); + builder.endObject(); + + String response = EntityUtils.toString(performSearch(INDEX_NAME, builder.toString()).getEntity()); + assertTrue(response.contains(expectedResponse)); + + deleteKNNIndex(INDEX_NAME); + } + } + + /** + * Mapping + * { + * "properties": { + * "test_vector": { + * "type": "knn_vector", + * "dimension": {dimension}, + * "data_type": "{type}", + * "stored": true + * "method": { + * "name": "hnsw", + * "engine": "{engine}" + * } + * } + * } + * } + */ + @SneakyThrows + private String createVectorMapping(final int dimension, final String engine, final String dataType, final boolean isStored) { + XContentBuilder builder = XContentFactory.jsonBuilder() + .startObject() + .startObject(PROPERTIES_FIELD) + .startObject(FIELD_NAME) + .field(TYPE, TYPE_KNN_VECTOR) + .field(DIMENSION, dimension) + .field(VECTOR_DATA_TYPE_FIELD, dataType) + .field(STORE_FIELD, isStored) + .startObject(KNN_METHOD) + .field(NAME, METHOD_HNSW) + .field(KNN_ENGINE, engine) + .endObject() + .endObject() + .endObject() + .endObject(); + + return builder.toString(); + } + } diff --git a/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperUtilTests.java b/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperUtilTests.java new file mode 100644 index 000000000..3fa9f2363 --- /dev/null +++ b/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperUtilTests.java @@ -0,0 +1,54 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.knn.index.mapper; + +import org.apache.lucene.document.StoredField; +import org.opensearch.knn.KNNTestCase; +import org.opensearch.knn.index.VectorDataType; +import org.opensearch.knn.index.codec.util.KNNVectorSerializerFactory; + +import java.io.ByteArrayInputStream; +import java.util.Arrays; + +public class KNNVectorFieldMapperUtilTests extends KNNTestCase { + + private static final String TEST_FIELD_NAME = "test_field_name"; + private static final byte[] TEST_BYTE_VECTOR = new byte[] { -128, 0, 1, 127 }; + private static final float[] TEST_FLOAT_VECTOR = new float[] { -100.0f, 100.0f, 0f, 1f }; + + public void testStoredFields_whenVectorIsByteType_thenSucceed() { + StoredField storedField = KNNVectorFieldMapperUtil.createStoredFieldForByteVector(TEST_FIELD_NAME, TEST_BYTE_VECTOR); + assertEquals(TEST_FIELD_NAME, storedField.name()); + assertEquals(TEST_BYTE_VECTOR, storedField.binaryValue().bytes); + Object vector = KNNVectorFieldMapperUtil.deserializeStoredVector(storedField.binaryValue(), VectorDataType.BYTE); + assertTrue(vector instanceof int[]); + int[] byteAsIntArray = new int[TEST_BYTE_VECTOR.length]; + Arrays.setAll(byteAsIntArray, i -> TEST_BYTE_VECTOR[i]); + assertArrayEquals(byteAsIntArray, (int[]) vector); + } + + public void testStoredFields_whenVectorIsFloatType_thenSucceed() { + StoredField storedField = KNNVectorFieldMapperUtil.createStoredFieldForFloatVector(TEST_FIELD_NAME, TEST_FLOAT_VECTOR); + assertEquals(TEST_FIELD_NAME, storedField.name()); + byte[] bytes = storedField.binaryValue().bytes; + ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(bytes, 0, bytes.length); + assertArrayEquals( + TEST_FLOAT_VECTOR, + KNNVectorSerializerFactory.getDefaultSerializer().byteToFloatArray(byteArrayInputStream), + 0.001f + ); + + Object vector = KNNVectorFieldMapperUtil.deserializeStoredVector(storedField.binaryValue(), VectorDataType.FLOAT); + assertTrue(vector instanceof float[]); + assertArrayEquals(TEST_FLOAT_VECTOR, (float[]) vector, 0.001f); + } +} diff --git a/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java b/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java index ae4d3a6c3..c2ebde0f5 100644 --- a/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java +++ b/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java @@ -108,6 +108,10 @@ public class KNNRestTestCase extends ODFERestTestCase { public static final String INDEX_NAME = "test_index"; public static final String FIELD_NAME = "test_field"; + public static final String PROPERTIES_FIELD = "properties"; + public static final String STORE_FIELD = "store"; + public static final String STORED_QUERY_FIELD = "stored_fields"; + public static final String MATCH_ALL_QUERY_FIELD = "match_all"; private static final String DOCUMENT_FIELD_SOURCE = "_source"; private static final String DOCUMENT_FIELD_FOUND = "found"; protected static final int DELAY_MILLI_SEC = 1000; From d9d2d10f722569ba5e7c1e8c7378e8bd30c3ae31 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Tue, 23 Apr 2024 11:59:50 -0700 Subject: [PATCH 242/416] Serialize all models into cluster metadata (#1499) (#1644) * Remove transport calls in TrainingJobRunner and TrainingJobClusterStateListener Signed-off-by: Ryan Bogan * Fix tests Signed-off-by: Ryan Bogan * Add changelog Signed-off-by: Ryan Bogan * Fix CMake Faiss bug Signed-off-by: Ryan Bogan * Add state checks for existing cluster metadata calls Signed-off-by: Ryan Bogan * Remove CMake bug fix Signed-off-by: Ryan Bogan * Fix changelog Signed-off-by: Ryan Bogan * Fix failing tests Signed-off-by: Ryan Bogan * Refactor and add two more created state checks Signed-off-by: Ryan Bogan * Rebase and fix new tests Signed-off-by: Ryan Bogan * Refactor created checks and modify error messages Signed-off-by: Ryan Bogan * Refactor cluster state listener transport calls Signed-off-by: Ryan Bogan --------- Signed-off-by: Ryan Bogan (cherry picked from commit dc8eb6b919ff6025b7d1e2aeccba28d32793bb96) Co-authored-by: Ryan Bogan --- CHANGELOG.md | 1 + .../org/opensearch/knn/index/IndexUtil.java | 6 ++-- .../knn/index/mapper/ModelFieldMapper.java | 5 +-- .../knn/index/query/KNNQueryBuilder.java | 5 +-- .../opensearch/knn/index/query/KNNWeight.java | 5 +-- .../org/opensearch/knn/indices/ModelDao.java | 8 +---- .../org/opensearch/knn/indices/ModelUtil.java | 30 ++++++++++++++++++ .../TrainingJobClusterStateListener.java | 31 +++++++++++++------ .../knn/index/query/KNNQueryBuilderTests.java | 4 +++ .../knn/index/query/KNNWeightTests.java | 4 ++- .../transport/TrainingModelRequestTests.java | 1 + .../TrainingJobClusterStateListenerTests.java | 2 ++ 12 files changed, 77 insertions(+), 25 deletions(-) create mode 100644 src/main/java/org/opensearch/knn/indices/ModelUtil.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 0b37975ba..7999e7bff 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), * Support script score when doc value is disabled [#1573](https://github.com/opensearch-project/k-NN/pull/1573) * Implemented the Streaming Feature to stream vectors from Java to JNI layer to enable creation of larger segments for vector indices [#1604](https://github.com/opensearch-project/k-NN/pull/1604) * Remove unnecessary toString conversion of vector field and added some minor optimization in KNNCodec [1613](https://github.com/opensearch-project/k-NN/pull/1613) +* Serialize all models into cluster metadata [#1499](https://github.com/opensearch-project/k-NN/pull/1499) ### Bug Fixes * Add stored fields for knn_vector type [#1630](https://github.com/opensearch-project/k-NN/pull/1630) ### Infrastructure diff --git a/src/main/java/org/opensearch/knn/index/IndexUtil.java b/src/main/java/org/opensearch/knn/index/IndexUtil.java index 92b94c2e2..e4523bb5e 100644 --- a/src/main/java/org/opensearch/knn/index/IndexUtil.java +++ b/src/main/java/org/opensearch/knn/index/IndexUtil.java @@ -23,6 +23,7 @@ import org.opensearch.knn.index.util.KNNEngine; import org.opensearch.knn.indices.ModelDao; import org.opensearch.knn.indices.ModelMetadata; +import org.opensearch.knn.indices.ModelUtil; import org.opensearch.knn.jni.JNIService; import java.io.File; @@ -199,8 +200,8 @@ public static ValidationException validateKnnField( } ModelMetadata modelMetadata = modelDao.getMetadata(modelId); - if (modelMetadata == null) { - exception.addValidationError(String.format("Model \"%s\" for field \"%s\" does not exist.", modelId, field)); + if (!ModelUtil.isModelCreated(modelMetadata)) { + exception.addValidationError(String.format("Model \"%s\" for field \"%s\" is not created.", modelId, field)); return exception; } @@ -286,4 +287,5 @@ public static boolean isSharedIndexStateRequired(KNNEngine knnEngine, String mod } return JNIService.isSharedIndexStateRequired(indexAddr, knnEngine); } + } diff --git a/src/main/java/org/opensearch/knn/index/mapper/ModelFieldMapper.java b/src/main/java/org/opensearch/knn/index/mapper/ModelFieldMapper.java index ce92d2967..554871279 100644 --- a/src/main/java/org/opensearch/knn/index/mapper/ModelFieldMapper.java +++ b/src/main/java/org/opensearch/knn/index/mapper/ModelFieldMapper.java @@ -11,6 +11,7 @@ import org.opensearch.index.mapper.ParseContext; import org.opensearch.knn.indices.ModelDao; import org.opensearch.knn.indices.ModelMetadata; +import org.opensearch.knn.indices.ModelUtil; import java.io.IOException; @@ -50,10 +51,10 @@ protected void parseCreateField(ParseContext context) throws IOException { // model when ingestion starts. ModelMetadata modelMetadata = this.modelDao.getMetadata(modelId); - if (modelMetadata == null) { + if (!ModelUtil.isModelCreated(modelMetadata)) { throw new IllegalStateException( String.format( - "Model \"%s\" from %s's mapping does not exist. Because the \"%s\" parameter is not updatable, this index will need to be recreated with a valid model.", + "Model \"%s\" from %s's mapping is not created. Because the \"%s\" parameter is not updatable, this index will need to be recreated with a valid model.", modelId, context.mapperService().index().getName(), MODEL_ID diff --git a/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java b/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java index 760a91815..37114f3cb 100644 --- a/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java +++ b/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java @@ -31,6 +31,7 @@ import org.opensearch.knn.index.util.KNNEngine; import org.opensearch.knn.indices.ModelDao; import org.opensearch.knn.indices.ModelMetadata; +import org.opensearch.knn.indices.ModelUtil; import org.opensearch.knn.plugin.stats.KNNCounter; import org.opensearch.index.query.AbstractQueryBuilder; import org.opensearch.index.query.QueryShardContext; @@ -548,8 +549,8 @@ private ModelMetadata getModelMetadataForField(KNNVectorFieldMapper.KNNVectorFie } ModelMetadata modelMetadata = modelDao.getMetadata(modelId); - if (modelMetadata == null) { - throw new IllegalArgumentException(String.format("Model ID '%s' does not exist.", modelId)); + if (!ModelUtil.isModelCreated(modelMetadata)) { + throw new IllegalArgumentException(String.format("Model ID '%s' is not created.", modelId)); } return modelMetadata; } diff --git a/src/main/java/org/opensearch/knn/index/query/KNNWeight.java b/src/main/java/org/opensearch/knn/index/query/KNNWeight.java index 6b323e124..8939a569e 100644 --- a/src/main/java/org/opensearch/knn/index/query/KNNWeight.java +++ b/src/main/java/org/opensearch/knn/index/query/KNNWeight.java @@ -40,6 +40,7 @@ import org.opensearch.knn.index.util.KNNEngine; import org.opensearch.knn.indices.ModelDao; import org.opensearch.knn.indices.ModelMetadata; +import org.opensearch.knn.indices.ModelUtil; import org.opensearch.knn.jni.JNIService; import org.opensearch.knn.plugin.stats.KNNCounter; @@ -213,8 +214,8 @@ private Map doANNSearch(final LeafReaderContext context, final B String modelId = fieldInfo.getAttribute(MODEL_ID); if (modelId != null) { ModelMetadata modelMetadata = modelDao.getMetadata(modelId); - if (modelMetadata == null) { - throw new RuntimeException("Model \"" + modelId + "\" does not exist."); + if (!ModelUtil.isModelCreated(modelMetadata)) { + throw new RuntimeException("Model \"" + modelId + "\" is not created."); } knnEngine = modelMetadata.getKnnEngine(); diff --git a/src/main/java/org/opensearch/knn/indices/ModelDao.java b/src/main/java/org/opensearch/knn/indices/ModelDao.java index 0c0f08545..6940fcd39 100644 --- a/src/main/java/org/opensearch/knn/indices/ModelDao.java +++ b/src/main/java/org/opensearch/knn/indices/ModelDao.java @@ -341,13 +341,7 @@ private void putInternal(Model model, ActionListener listener, Do ); }, listener::onFailure); - // After the model is indexed, update metadata only if the model is in CREATED state - ActionListener onIndexListener; - if (ModelState.CREATED.equals(model.getModelMetadata().getState())) { - onIndexListener = getUpdateModelMetadataListener(model.getModelMetadata(), onMetaListener); - } else { - onIndexListener = onMetaListener; - } + ActionListener onIndexListener = getUpdateModelMetadataListener(model.getModelMetadata(), onMetaListener); // Create the model index if it does not already exist Runnable indexModelRunnable = () -> indexRequestBuilder.execute(onIndexListener); diff --git a/src/main/java/org/opensearch/knn/indices/ModelUtil.java b/src/main/java/org/opensearch/knn/indices/ModelUtil.java new file mode 100644 index 000000000..3daaed138 --- /dev/null +++ b/src/main/java/org/opensearch/knn/indices/ModelUtil.java @@ -0,0 +1,30 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.knn.indices; + +/** + * A utility class for models. + */ +public class ModelUtil { + + public static boolean isModelPresent(ModelMetadata modelMetadata) { + return modelMetadata != null; + } + + public static boolean isModelCreated(ModelMetadata modelMetadata) { + if (!isModelPresent(modelMetadata)) { + return false; + } + return modelMetadata.getState().equals(ModelState.CREATED); + } + +} diff --git a/src/main/java/org/opensearch/knn/training/TrainingJobClusterStateListener.java b/src/main/java/org/opensearch/knn/training/TrainingJobClusterStateListener.java index 45d2197e8..7e39ff7b3 100644 --- a/src/main/java/org/opensearch/knn/training/TrainingJobClusterStateListener.java +++ b/src/main/java/org/opensearch/knn/training/TrainingJobClusterStateListener.java @@ -109,10 +109,9 @@ protected void updateModelsNewCluster() throws IOException, InterruptedException if (modelDao.isCreated()) { List modelIds = searchModelIds(); for (String modelId : modelIds) { - Model model = modelDao.get(modelId); - ModelMetadata modelMetadata = model.getModelMetadata(); + ModelMetadata modelMetadata = getModelMetadata(modelId); if (modelMetadata.getState().equals(ModelState.TRAINING)) { - updateModelStateAsFailed(model, "Training failed to complete as cluster crashed"); + updateModelStateAsFailed(modelId, modelMetadata, "Training failed to complete as cluster crashed"); } } } @@ -123,11 +122,10 @@ protected void updateModelsNodesRemoved(List removedNodes) throws List modelIds = searchModelIds(); for (DiscoveryNode removedNode : removedNodes) { for (String modelId : modelIds) { - Model model = modelDao.get(modelId); - ModelMetadata modelMetadata = model.getModelMetadata(); + ModelMetadata modelMetadata = getModelMetadata(modelId); if (modelMetadata.getNodeAssignment().equals(removedNode.getEphemeralId()) && modelMetadata.getState().equals(ModelState.TRAINING)) { - updateModelStateAsFailed(model, "Training failed to complete as node dropped"); + updateModelStateAsFailed(modelId, modelMetadata, "Training failed to complete as node dropped"); } } } @@ -158,9 +156,11 @@ public void onFailure(Exception e) { return modelIds; } - private void updateModelStateAsFailed(Model model, String msg) throws IOException { - model.getModelMetadata().setState(ModelState.FAILED); - model.getModelMetadata().setError(msg); + private void updateModelStateAsFailed(String modelId, ModelMetadata modelMetadata, String msg) throws IOException, ExecutionException, + InterruptedException { + modelMetadata.setState(ModelState.FAILED); + modelMetadata.setError(msg); + Model model = new Model(modelMetadata, null, modelId); modelDao.update(model, new ActionListener() { @Override public void onResponse(IndexResponse indexResponse) { @@ -173,4 +173,17 @@ public void onFailure(Exception e) { } }); } + + private ModelMetadata getModelMetadata(String modelId) throws ExecutionException, InterruptedException { + ModelMetadata modelMetadata = modelDao.getMetadata(modelId); + // On versions prior to 2.14, only models in created state are present in model metadata. + if (modelMetadata == null) { + log.info( + "Model metadata is null in cluster metadata. This can happen for models training on nodes prior to OpenSearch version 2.14.0. Fetching model information from system index." + ); + Model model = modelDao.get(modelId); + return model.getModelMetadata(); + } + return modelMetadata; + } } diff --git a/src/test/java/org/opensearch/knn/index/query/KNNQueryBuilderTests.java b/src/test/java/org/opensearch/knn/index/query/KNNQueryBuilderTests.java index e47f583e7..ddc961093 100644 --- a/src/test/java/org/opensearch/knn/index/query/KNNQueryBuilderTests.java +++ b/src/test/java/org/opensearch/knn/index/query/KNNQueryBuilderTests.java @@ -39,6 +39,7 @@ import org.opensearch.knn.index.util.KNNEngine; import org.opensearch.knn.indices.ModelDao; import org.opensearch.knn.indices.ModelMetadata; +import org.opensearch.knn.indices.ModelState; import org.opensearch.plugins.SearchPlugin; import java.io.IOException; @@ -683,6 +684,7 @@ public void testDoToQuery_FromModel() { when(modelMetadata.getDimension()).thenReturn(4); when(modelMetadata.getKnnEngine()).thenReturn(KNNEngine.FAISS); when(modelMetadata.getSpaceType()).thenReturn(SpaceType.COSINESIMIL); + when(modelMetadata.getState()).thenReturn(ModelState.CREATED); ModelDao modelDao = mock(ModelDao.class); when(modelDao.getMetadata(modelId)).thenReturn(modelMetadata); KNNQueryBuilder.initialize(modelDao); @@ -712,6 +714,7 @@ public void testDoToQuery_whenFromModel_whenDoRadiusSearch_whenDistanceThreshold when(modelMetadata.getDimension()).thenReturn(4); when(modelMetadata.getKnnEngine()).thenReturn(KNNEngine.FAISS); when(modelMetadata.getSpaceType()).thenReturn(SpaceType.L2); + when(modelMetadata.getState()).thenReturn(ModelState.CREATED); ModelDao modelDao = mock(ModelDao.class); when(modelDao.getMetadata(modelId)).thenReturn(modelMetadata); KNNQueryBuilder.initialize(modelDao); @@ -744,6 +747,7 @@ public void testDoToQuery_whenFromModel_whenDoRadiusSearch_whenScoreThreshold_th when(modelMetadata.getDimension()).thenReturn(4); when(modelMetadata.getKnnEngine()).thenReturn(KNNEngine.FAISS); when(modelMetadata.getSpaceType()).thenReturn(SpaceType.L2); + when(modelMetadata.getState()).thenReturn(ModelState.CREATED); ModelDao modelDao = mock(ModelDao.class); when(modelDao.getMetadata(modelId)).thenReturn(modelMetadata); KNNQueryBuilder.initialize(modelDao); diff --git a/src/test/java/org/opensearch/knn/index/query/KNNWeightTests.java b/src/test/java/org/opensearch/knn/index/query/KNNWeightTests.java index 795635b68..e80190ff8 100644 --- a/src/test/java/org/opensearch/knn/index/query/KNNWeightTests.java +++ b/src/test/java/org/opensearch/knn/index/query/KNNWeightTests.java @@ -45,6 +45,7 @@ import org.opensearch.knn.index.util.KNNEngine; import org.opensearch.knn.indices.ModelDao; import org.opensearch.knn.indices.ModelMetadata; +import org.opensearch.knn.indices.ModelState; import org.opensearch.knn.jni.JNIService; import java.io.IOException; @@ -167,6 +168,7 @@ public void testQueryScoreForFaissWithModel() { ModelMetadata modelMetadata = mock(ModelMetadata.class); when(modelMetadata.getKnnEngine()).thenReturn(KNNEngine.FAISS); when(modelMetadata.getSpaceType()).thenReturn(spaceType); + when(modelMetadata.getState()).thenReturn(ModelState.CREATED); when(modelDao.getMetadata(eq("modelId"))).thenReturn(modelMetadata); KNNWeight.initialize(modelDao); @@ -254,7 +256,7 @@ public void testQueryScoreForFaissWithNonExistingModel() throws IOException { when(fieldInfo.getAttribute(eq(MODEL_ID))).thenReturn(modelId); RuntimeException ex = expectThrows(RuntimeException.class, () -> knnWeight.scorer(leafReaderContext)); - assertEquals(String.format("Model \"%s\" does not exist.", modelId), ex.getMessage()); + assertEquals(String.format("Model \"%s\" is not created.", modelId), ex.getMessage()); } @SneakyThrows diff --git a/src/test/java/org/opensearch/knn/plugin/transport/TrainingModelRequestTests.java b/src/test/java/org/opensearch/knn/plugin/transport/TrainingModelRequestTests.java index bdae54cad..b39c48635 100644 --- a/src/test/java/org/opensearch/knn/plugin/transport/TrainingModelRequestTests.java +++ b/src/test/java/org/opensearch/knn/plugin/transport/TrainingModelRequestTests.java @@ -661,6 +661,7 @@ public void testValidation_valid_trainingIndexBuiltFromModel() { // Mock the model dao to return metadata for modelId to recognize it is a duplicate ModelMetadata trainingFieldModelMetadata = mock(ModelMetadata.class); when(trainingFieldModelMetadata.getDimension()).thenReturn(dimension); + when(trainingFieldModelMetadata.getState()).thenReturn(ModelState.CREATED); ModelDao modelDao = mock(ModelDao.class); when(modelDao.getMetadata(modelId)).thenReturn(null); diff --git a/src/test/java/org/opensearch/knn/training/TrainingJobClusterStateListenerTests.java b/src/test/java/org/opensearch/knn/training/TrainingJobClusterStateListenerTests.java index 7994e73d2..672a54110 100644 --- a/src/test/java/org/opensearch/knn/training/TrainingJobClusterStateListenerTests.java +++ b/src/test/java/org/opensearch/knn/training/TrainingJobClusterStateListenerTests.java @@ -101,6 +101,7 @@ public void testUpdateModelsNewCluster() throws IOException, InterruptedExceptio ModelDao modelDao = mock(ModelDao.class); when(modelDao.isCreated()).thenReturn(true); when(modelDao.get(modelId)).thenReturn(model); + when(modelDao.getMetadata(modelId)).thenReturn(modelMetadata); doAnswer(invocationOnMock -> { SearchResponse searchResponse = mock(SearchResponse.class); SearchHits searchHits = mock(SearchHits.class); @@ -144,6 +145,7 @@ public void testUpdateModelsNodesRemoved() throws IOException, InterruptedExcept ModelDao modelDao = mock(ModelDao.class); when(modelDao.isCreated()).thenReturn(true); when(modelDao.get(modelId)).thenReturn(model); + when(modelDao.getMetadata(modelId)).thenReturn(modelMetadata); DiscoveryNode node1 = mock(DiscoveryNode.class); when(node1.getEphemeralId()).thenReturn("test-node-model-match"); DiscoveryNode node2 = mock(DiscoveryNode.class); From e65a310e805abfd5a88c3084ed23c07c8845a063 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Tue, 23 Apr 2024 13:24:44 -0700 Subject: [PATCH 243/416] Update gcc restrictions to version 12.0.0 (#1643) Signed-off-by: Peter Zhu (cherry picked from commit 11342beb11f3fc78c4be681a799695c2e3cfc5a1) Co-authored-by: Peter Zhu --- scripts/build.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/build.sh b/scripts/build.sh index a5ab477f3..43a36443d 100644 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -103,12 +103,13 @@ if [ "$JAVA_HOME" = "" ]; then echo "SET JAVA_HOME=$JAVA_HOME" fi -# Ensure gcc version is above 4.9.0 and at least 9.0.0 for faiss 1.7.4+ / SIMD Neon support on ARM64 compilation +# Ensure gcc version is at least 12.0.0 for faiss 1.7.4+ / SIMD Neon support on ARM64 compilation # https://github.com/opensearch-project/k-NN/issues/975 # https://github.com/opensearch-project/k-NN/issues/1138 # https://github.com/opensearch-project/opensearch-build/issues/4386 +# https://github.com/opensearch-project/opensearch-build/issues/4379#issuecomment-2067191682 GCC_VERSION=`gcc --version | head -n 1 | cut -d ' ' -f3` -GCC_REQUIRED_VERSION=9.0.0 +GCC_REQUIRED_VERSION=12.0.0 COMPARE_VERSION=`echo $GCC_REQUIRED_VERSION $GCC_VERSION | tr ' ' '\n' | sort -V | uniq | head -n 1` if [ "$COMPARE_VERSION" != "$GCC_REQUIRED_VERSION" ]; then echo "gcc version on this env is older than $GCC_REQUIRED_VERSION, exit 1" From e4cc592eec8a80a42579cd0f781c208399c20d71 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Tue, 23 Apr 2024 14:31:36 -0700 Subject: [PATCH 244/416] Skip rebuild from scratch after cmake is ran (#1648) By commiting the patch changes, this commit prevents the libraries to be built from scratch every time cmake is ran. This will save additional build time. In addition to this, this commit separates out the different functionality to make the build system more readable. We can continue to iterate on this in the future. Signed-off-by: John Mazanec (cherry picked from commit c35f6add3add241c59ee2966b7958239b1591259) Co-authored-by: John Mazanec --- .github/workflows/CI.yml | 28 ++-- ...backwards_compatibility_tests_workflow.yml | 12 ++ .github/workflows/test_security.yml | 17 +-- .gitignore | 4 +- CHANGELOG.md | 1 + jni/CMakeLists.txt | 133 ++---------------- jni/cmake/init-faiss.cmake | 66 +++++++++ jni/cmake/init-nmslib.cmake | 28 ++++ jni/cmake/macros.cmake | 16 +++ 9 files changed, 152 insertions(+), 153 deletions(-) create mode 100644 jni/cmake/init-faiss.cmake create mode 100644 jni/cmake/init-nmslib.cmake create mode 100644 jni/cmake/macros.cmake diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 73335e9ce..8795ac319 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -38,19 +38,11 @@ jobs: with: submodules: true - # Git functionality in CMAKE file does not work with given ubuntu image. Therefore, handling it here. - - name: Apply Git Patch - # Deleting file at the end to skip `git apply` inside CMAKE file + # Setup git user so that patches for native libraries can be applied and committed + - name: Setup git user run: | - cd jni/external/faiss - git apply --ignore-space-change --ignore-whitespace --3way ../../patches/faiss/0001-Custom-patch-to-support-multi-vector.patch - rm ../../patches/faiss/0001-Custom-patch-to-support-multi-vector.patch - git apply --ignore-space-change --ignore-whitespace --3way ../../patches/faiss/0002-Enable-precomp-table-to-be-shared-ivfpq.patch - rm ../../patches/faiss/0002-Enable-precomp-table-to-be-shared-ivfpq.patch - cd ../nmslib - git apply --ignore-space-change --ignore-whitespace --3way ../../patches/nmslib/0001-Initialize-maxlevel-during-add-from-enterpoint-level.patch - rm ../../patches/nmslib/0001-Initialize-maxlevel-during-add-from-enterpoint-level.patch - working-directory: ${{ github.workspace }} + su `id -un 1000` -c 'git config --global user.name "github-actions[bot]"' + su `id -un 1000` -c 'git config --global user.email "github-actions[bot]@users.noreply.github.com"' - name: Setup Java ${{ matrix.java }} uses: actions/setup-java@v1 @@ -89,6 +81,12 @@ jobs: - name: Checkout k-NN uses: actions/checkout@v1 + # Setup git user so that patches for native libraries can be applied and committed + - name: Setup git user + run: | + git config --global user.name "github-actions[bot]" + git config --global user.email "github-actions[bot]@users.noreply.github.com" + - name: Setup Java ${{ matrix.java }} uses: actions/setup-java@v1 with: @@ -123,6 +121,12 @@ jobs: - name: Checkout k-NN uses: actions/checkout@v1 + # Setup git user so that patches for native libraries can be applied and committed + - name: Setup git user + run: | + git config --global user.name "github-actions[bot]" + git config --global user.email "github-actions[bot]@users.noreply.github.com" + - name: Setup Java ${{ matrix.java }} uses: actions/setup-java@v1 with: diff --git a/.github/workflows/backwards_compatibility_tests_workflow.yml b/.github/workflows/backwards_compatibility_tests_workflow.yml index 4b1a0ac23..43670fe83 100644 --- a/.github/workflows/backwards_compatibility_tests_workflow.yml +++ b/.github/workflows/backwards_compatibility_tests_workflow.yml @@ -26,6 +26,12 @@ jobs: - name: Checkout k-NN uses: actions/checkout@v1 + # Setup git user so that patches for native libraries can be applied and committed + - name: Setup git user + run: | + git config --global user.name "github-actions[bot]" + git config --global user.email "github-actions[bot]@users.noreply.github.com" + - name: Setup Java ${{ matrix.java }} uses: actions/setup-java@v1 with: @@ -58,6 +64,12 @@ jobs: - name: Checkout k-NN uses: actions/checkout@v1 + # Setup git user so that patches for native libraries can be applied and committed + - name: Setup git user + run: | + git config --global user.name "github-actions[bot]" + git config --global user.email "github-actions[bot]@users.noreply.github.com" + - name: Setup Java ${{ matrix.java }} uses: actions/setup-java@v1 with: diff --git a/.github/workflows/test_security.yml b/.github/workflows/test_security.yml index cbdd7983b..78206bada 100644 --- a/.github/workflows/test_security.yml +++ b/.github/workflows/test_security.yml @@ -37,20 +37,11 @@ jobs: uses: actions/checkout@v1 with: submodules: true - - # Git functionality in CMAKE file does not work with given ubuntu image. Therefore, handling it here. - - name: Apply Git Patch - # Deleting file at the end to skip `git apply` inside CMAKE file + # Setup git user so that patches for native libraries can be applied and committed + - name: Setup git user run: | - cd jni/external/faiss - git apply --ignore-space-change --ignore-whitespace --3way ../../patches/faiss/0001-Custom-patch-to-support-multi-vector.patch - rm ../../patches/faiss/0001-Custom-patch-to-support-multi-vector.patch - git apply --ignore-space-change --ignore-whitespace --3way ../../patches/faiss/0002-Enable-precomp-table-to-be-shared-ivfpq.patch - rm ../../patches/faiss/0002-Enable-precomp-table-to-be-shared-ivfpq.patch - cd ../nmslib - git apply --ignore-space-change --ignore-whitespace --3way ../../patches/nmslib/0001-Initialize-maxlevel-during-add-from-enterpoint-level.patch - rm ../../patches/nmslib/0001-Initialize-maxlevel-during-add-from-enterpoint-level.patch - working-directory: ${{ github.workspace }} + su `id -un 1000` -c 'git config --global user.name "github-actions[bot]"' + su `id -un 1000` -c 'git config --global user.email "github-actions[bot]@users.noreply.github.com"' - name: Setup Java ${{ matrix.java }} uses: actions/setup-java@v1 diff --git a/.gitignore b/.gitignore index 160757c41..4cb4ee61c 100644 --- a/.gitignore +++ b/.gitignore @@ -17,9 +17,7 @@ oss/* jni/CMakeCache.txt jni/CMakeFiles jni/Makefile -jni/cmake* -jni/CPack* -jni/_CPack* +jni/cmake_install.cmake jni/release jni/packages jni/CTestTestfile.cmake diff --git a/CHANGELOG.md b/CHANGELOG.md index 7999e7bff..fc0cf4238 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Infrastructure * Add micro-benchmark module in k-NN plugin for benchmark streaming vectors to JNI layer functionality. [#1583](https://github.com/opensearch-project/k-NN/pull/1583) * Add arm64 check when SIMD is disabled [#1618](https://github.com/opensearch-project/k-NN/pull/1618) +* Skip rebuild from scratch after cmake is ran [#1636](https://github.com/opensearch-project/k-NN/pull/1636) ### Documentation ### Maintenance ### Refactoring diff --git a/jni/CMakeLists.txt b/jni/CMakeLists.txt index b894baebf..3429671fe 100644 --- a/jni/CMakeLists.txt +++ b/jni/CMakeLists.txt @@ -7,6 +7,8 @@ cmake_minimum_required(VERSION 3.23.1) project(KNNPlugin_JNI) +include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/macros.cmake) + # ---------------------------------- SETUP ---------------------------------- # Target libraries to be compiled # Shared library with common utilities. Not a JNI library. Other JNI libs should depend on this one. @@ -65,16 +67,7 @@ endif() # ---------------------------------- UTIL ---------------------------------- add_library(${TARGET_LIB_UTIL} SHARED ${CMAKE_CURRENT_SOURCE_DIR}/src/jni_util.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/commons.cpp) target_include_directories(${TARGET_LIB_UTIL} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include $ENV{JAVA_HOME}/include $ENV{JAVA_HOME}/include/${JVM_OS_TYPE}) -set_target_properties(${TARGET_LIB_UTIL} PROPERTIES SUFFIX ${LIB_EXT}) -set_target_properties(${TARGET_LIB_UTIL} PROPERTIES POSITION_INDEPENDENT_CODE ON) - -if (NOT "${WIN32}" STREQUAL "") - # Use RUNTIME_OUTPUT_DIRECTORY, to build the target library (opensearchknn_util) in the specified directory at runtime. - set_target_properties(${TARGET_LIB_UTIL} PROPERTIES RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/release) -else() - set_target_properties(${TARGET_LIB_UTIL} PROPERTIES LIBRARY_OUTPUT_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/release) -endif() - +opensearch_set_common_properties(${TARGET_LIB_UTIL}) list(APPEND TARGET_LIBS ${TARGET_LIB_UTIL}) # ---------------------------------------------------------------------------- @@ -82,58 +75,17 @@ list(APPEND TARGET_LIBS ${TARGET_LIB_UTIL}) add_library(${TARGET_LIB_COMMON} SHARED ${CMAKE_CURRENT_SOURCE_DIR}/src/org_opensearch_knn_jni_JNICommons.cpp) target_link_libraries(${TARGET_LIB_COMMON} ${TARGET_LIB_UTIL}) target_include_directories(${TARGET_LIB_COMMON} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include $ENV{JAVA_HOME}/include $ENV{JAVA_HOME}/include/${JVM_OS_TYPE}) -set_target_properties(${TARGET_LIB_COMMON} PROPERTIES SUFFIX ${LIB_EXT}) -set_target_properties(${TARGET_LIB_COMMON} PROPERTIES POSITION_INDEPENDENT_CODE ON) - -if (NOT "${WIN32}" STREQUAL "") -# Use RUNTIME_OUTPUT_DIRECTORY, to build the target library (opensearchknn_common) in the specified directory at runtime. - set_target_properties(${TARGET_LIB_COMMON} PROPERTIES RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/release) -else() - set_target_properties(${TARGET_LIB_COMMON} PROPERTIES LIBRARY_OUTPUT_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/release) -endif() - +opensearch_set_common_properties(${TARGET_LIB_COMMON}) list(APPEND TARGET_LIBS ${TARGET_LIB_COMMON}) # ---------------------------------------------------------------------------- # ---------------------------------- NMSLIB ---------------------------------- if (${CONFIG_NMSLIB} STREQUAL ON OR ${CONFIG_ALL} STREQUAL ON OR ${CONFIG_TEST} STREQUAL ON) - # Check if nmslib exists - find_path(NMS_REPO_DIR NAMES similarity_search PATHS ${CMAKE_CURRENT_SOURCE_DIR}/external/nmslib NO_DEFAULT_PATH) - - # If not, pull the updated submodule - if (NOT EXISTS ${NMS_REPO_DIR}) - message(STATUS "Could not find nmslib. Pulling updated submodule.") - execute_process(COMMAND git submodule update --init -- external/nmslib WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}) - endif () - - # Check if patch exist, this is to skip git apply during CI build. See CI.yml with ubuntu. - find_path(PATCH_FILE NAMES 0001-Initialize-maxlevel-during-add-from-enterpoint-level.patch PATHS ${CMAKE_CURRENT_SOURCE_DIR}/patches/nmslib NO_DEFAULT_PATH) - - # If it exists, apply patches - if (EXISTS ${PATCH_FILE}) - message(STATUS "Applying custom patches.") - execute_process(COMMAND git apply --ignore-space-change --ignore-whitespace --3way ${CMAKE_CURRENT_SOURCE_DIR}/patches/nmslib/0001-Initialize-maxlevel-during-add-from-enterpoint-level.patch WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/external/nmslib ERROR_VARIABLE ERROR_MSG RESULT_VARIABLE RESULT_CODE) - - if(RESULT_CODE) - message(FATAL_ERROR "Failed to apply patch:\n${ERROR_MSG}") - endif() - endif() - - add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/external/nmslib/similarity_search) - + include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/init-nmslib.cmake) add_library(${TARGET_LIB_NMSLIB} SHARED ${CMAKE_CURRENT_SOURCE_DIR}/src/org_opensearch_knn_jni_NmslibService.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/nmslib_wrapper.cpp) target_link_libraries(${TARGET_LIB_NMSLIB} NonMetricSpaceLib ${TARGET_LIB_UTIL}) target_include_directories(${TARGET_LIB_NMSLIB} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include $ENV{JAVA_HOME}/include $ENV{JAVA_HOME}/include/${JVM_OS_TYPE} ${CMAKE_CURRENT_SOURCE_DIR}/external/nmslib/similarity_search/include) - set_target_properties(${TARGET_LIB_NMSLIB} PROPERTIES SUFFIX ${LIB_EXT}) - set_target_properties(${TARGET_LIB_NMSLIB} PROPERTIES POSITION_INDEPENDENT_CODE ON) - - if (NOT "${WIN32}" STREQUAL "") - # Use RUNTIME_OUTPUT_DIRECTORY, to build the target library (opensearchknn_nmslib) in the specified directory at runtime. - set_target_properties(${TARGET_LIB_NMSLIB} PROPERTIES RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/release) - else() - set_target_properties(${TARGET_LIB_NMSLIB} PROPERTIES LIBRARY_OUTPUT_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/release) - endif() - + opensearch_set_common_properties(${TARGET_LIB_NMSLIB}) list(APPEND TARGET_LIBS ${TARGET_LIB_NMSLIB}) endif () @@ -141,68 +93,8 @@ endif () # ---------------------------------- FAISS ---------------------------------- if (${CONFIG_FAISS} STREQUAL ON OR ${CONFIG_ALL} STREQUAL ON OR ${CONFIG_TEST} STREQUAL ON) - set(BUILD_TESTING OFF) # Avoid building faiss tests - set(BLA_STATIC ON) # Statically link BLAS - - if(NOT DEFINED SIMD_ENABLED) - set(SIMD_ENABLED true) # set default value as true if the argument is not set - endif() - - if(${CMAKE_SYSTEM_NAME} STREQUAL Windows OR ${CMAKE_SYSTEM_PROCESSOR} MATCHES "aarch64" OR ${CMAKE_SYSTEM_PROCESSOR} MATCHES "arm64" OR NOT ${SIMD_ENABLED}) - set(FAISS_OPT_LEVEL generic) # Keep optimization level as generic on Windows OS as it is not supported due to MINGW64 compiler issue. Also, on aarch64 avx2 is not supported. - set(TARGET_LINK_FAISS_LIB faiss) - else() - set(FAISS_OPT_LEVEL avx2) # Keep optimization level as avx2 to improve performance on Linux and Mac. - set(TARGET_LINK_FAISS_LIB faiss_avx2) - string(PREPEND LIB_EXT "_avx2") # Prepend "_avx2" to lib extension to create the library as "libopensearchknn_faiss_avx2.so" on linux and "libopensearchknn_faiss_avx2.jnilib" on mac - endif() - - if (${CMAKE_SYSTEM_NAME} STREQUAL Darwin) - if(CMAKE_C_COMPILER_ID MATCHES "Clang\$") - set(OpenMP_C_FLAGS "-Xpreprocessor -fopenmp") - set(OpenMP_C_LIB_NAMES "omp") - set(OpenMP_omp_LIBRARY /usr/local/opt/libomp/lib/libomp.dylib) - endif() - - if(CMAKE_CXX_COMPILER_ID MATCHES "Clang\$") - set(OpenMP_CXX_FLAGS "-Xpreprocessor -fopenmp -I/usr/local/opt/libomp/include") - set(OpenMP_CXX_LIB_NAMES "omp") - set(OpenMP_omp_LIBRARY /usr/local/opt/libomp/lib/libomp.dylib) - endif() - endif() - + include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/init-faiss.cmake) find_package(OpenMP REQUIRED) - find_package(ZLIB REQUIRED) - find_package(BLAS REQUIRED) - enable_language(Fortran) - find_package(LAPACK REQUIRED) - - # Check if faiss exists - find_path(FAISS_REPO_DIR NAMES faiss PATHS ${CMAKE_CURRENT_SOURCE_DIR}/external/faiss NO_DEFAULT_PATH) - - # If not, pull the updated submodule - if (NOT EXISTS ${FAISS_REPO_DIR}) - message(STATUS "Could not find faiss. Pulling updated submodule.") - execute_process(COMMAND git submodule update --init -- external/faiss WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}) - endif () - - # Check if patch exist, this is to skip git apply during CI build. See CI.yml with ubuntu. - find_path(PATCH_FILE NAMES 0001-Custom-patch-to-support-multi-vector.patch 0002-Enable-precomp-table-to-be-shared-ivfpq.patch PATHS ${CMAKE_CURRENT_SOURCE_DIR}/patches/faiss NO_DEFAULT_PATH) - - # If it exists, apply patches - if (EXISTS ${PATCH_FILE}) - message(STATUS "Applying custom patches.") - execute_process(COMMAND git apply --ignore-space-change --ignore-whitespace --3way ${CMAKE_CURRENT_SOURCE_DIR}/patches/faiss/0001-Custom-patch-to-support-multi-vector.patch WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/external/faiss ERROR_VARIABLE ERROR_MSG RESULT_VARIABLE RESULT_CODE) - execute_process(COMMAND git apply --ignore-space-change --ignore-whitespace --3way ${CMAKE_CURRENT_SOURCE_DIR}/patches/faiss/0002-Enable-precomp-table-to-be-shared-ivfpq.patch WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/external/faiss ERROR_VARIABLE ERROR_MSG RESULT_VARIABLE RESULT_CODE) - if(RESULT_CODE) - message(FATAL_ERROR "Failed to apply patch:\n${ERROR_MSG}") - endif() - endif() - - set(FAISS_ENABLE_GPU OFF) - set(FAISS_ENABLE_PYTHON OFF) - add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/external/faiss EXCLUDE_FROM_ALL) - add_library( ${TARGET_LIB_FAISS} SHARED ${CMAKE_CURRENT_SOURCE_DIR}/src/org_opensearch_knn_jni_FaissService.cpp @@ -216,16 +108,7 @@ if (${CONFIG_FAISS} STREQUAL ON OR ${CONFIG_ALL} STREQUAL ON OR ${CONFIG_TEST} S $ENV{JAVA_HOME}/include/${JVM_OS_TYPE} ${CMAKE_CURRENT_SOURCE_DIR}/external/faiss ) - set_target_properties(${TARGET_LIB_FAISS} PROPERTIES SUFFIX ${LIB_EXT}) - set_target_properties(${TARGET_LIB_FAISS} PROPERTIES POSITION_INDEPENDENT_CODE ON) - - if (NOT "${WIN32}" STREQUAL "") - # Use RUNTIME_OUTPUT_DIRECTORY, to build the target library (opensearchknn_faiss) in the specified directory at runtime. - set_target_properties(${TARGET_LIB_FAISS} PROPERTIES RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/release) - else() - set_target_properties(${TARGET_LIB_FAISS} PROPERTIES LIBRARY_OUTPUT_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/release) - endif() - + opensearch_set_common_properties(${TARGET_LIB_FAISS}) list(APPEND TARGET_LIBS ${TARGET_LIB_FAISS}) endif () diff --git a/jni/cmake/init-faiss.cmake b/jni/cmake/init-faiss.cmake new file mode 100644 index 000000000..44dd4442a --- /dev/null +++ b/jni/cmake/init-faiss.cmake @@ -0,0 +1,66 @@ +# +# Copyright OpenSearch Contributors +# SPDX-License-Identifier: Apache-2.0 +# + +# Check if faiss exists +find_path(FAISS_REPO_DIR NAMES faiss PATHS ${CMAKE_CURRENT_SOURCE_DIR}/external/faiss NO_DEFAULT_PATH) + +# If not, pull the updated submodule +if (NOT EXISTS ${FAISS_REPO_DIR}) + message(STATUS "Could not find faiss. Pulling updated submodule.") + execute_process(COMMAND git submodule update --init -- external/faiss WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}) +endif () + +# Check if patch exist, this is to skip git apply during CI build. See CI.yml with ubuntu. +find_path(PATCH_FILE NAMES 0001-Custom-patch-to-support-multi-vector.patch 0002-Enable-precomp-table-to-be-shared-ivfpq.patch PATHS ${CMAKE_CURRENT_SOURCE_DIR}/patches/faiss NO_DEFAULT_PATH) + +# If it exists, apply patches +if (EXISTS ${PATCH_FILE}) + message(STATUS "Applying custom patches.") + execute_process(COMMAND git am --3way ${CMAKE_CURRENT_SOURCE_DIR}/patches/faiss/0001-Custom-patch-to-support-multi-vector.patch WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/external/faiss ERROR_VARIABLE ERROR_MSG RESULT_VARIABLE RESULT_CODE) + execute_process(COMMAND git am --3way ${CMAKE_CURRENT_SOURCE_DIR}/patches/faiss/0002-Enable-precomp-table-to-be-shared-ivfpq.patch WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/external/faiss ERROR_VARIABLE ERROR_MSG RESULT_VARIABLE RESULT_CODE) + if(RESULT_CODE) + message(FATAL_ERROR "Failed to apply patch:\n${ERROR_MSG}") + endif() +endif() + +if (${CMAKE_SYSTEM_NAME} STREQUAL Darwin) + if(CMAKE_C_COMPILER_ID MATCHES "Clang\$") + set(OpenMP_C_FLAGS "-Xpreprocessor -fopenmp") + set(OpenMP_C_LIB_NAMES "omp") + set(OpenMP_omp_LIBRARY /usr/local/opt/libomp/lib/libomp.dylib) + endif() + + if(CMAKE_CXX_COMPILER_ID MATCHES "Clang\$") + set(OpenMP_CXX_FLAGS "-Xpreprocessor -fopenmp -I/usr/local/opt/libomp/include") + set(OpenMP_CXX_LIB_NAMES "omp") + set(OpenMP_omp_LIBRARY /usr/local/opt/libomp/lib/libomp.dylib) + endif() +endif() + +find_package(ZLIB REQUIRED) +find_package(BLAS REQUIRED) +enable_language(Fortran) +find_package(LAPACK REQUIRED) + +# Set relevant properties +set(BUILD_TESTING OFF) # Avoid building faiss tests +set(BLA_STATIC ON) # Statically link BLAS +set(FAISS_ENABLE_GPU OFF) +set(FAISS_ENABLE_PYTHON OFF) + +if(NOT DEFINED SIMD_ENABLED) + set(SIMD_ENABLED true) # set default value as true if the argument is not set +endif() + +if(${CMAKE_SYSTEM_NAME} STREQUAL Windows OR ${CMAKE_SYSTEM_PROCESSOR} MATCHES "aarch64" OR ${CMAKE_SYSTEM_PROCESSOR} MATCHES "arm64" OR NOT ${SIMD_ENABLED}) + set(FAISS_OPT_LEVEL generic) # Keep optimization level as generic on Windows OS as it is not supported due to MINGW64 compiler issue. Also, on aarch64 avx2 is not supported. + set(TARGET_LINK_FAISS_LIB faiss) +else() + set(FAISS_OPT_LEVEL avx2) # Keep optimization level as avx2 to improve performance on Linux and Mac. + set(TARGET_LINK_FAISS_LIB faiss_avx2) + string(PREPEND LIB_EXT "_avx2") # Prepend "_avx2" to lib extension to create the library as "libopensearchknn_faiss_avx2.so" on linux and "libopensearchknn_faiss_avx2.jnilib" on mac +endif() + +add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/external/faiss EXCLUDE_FROM_ALL) diff --git a/jni/cmake/init-nmslib.cmake b/jni/cmake/init-nmslib.cmake new file mode 100644 index 000000000..edf2296d9 --- /dev/null +++ b/jni/cmake/init-nmslib.cmake @@ -0,0 +1,28 @@ +# +# Copyright OpenSearch Contributors +# SPDX-License-Identifier: Apache-2.0 +# + +# Check if nmslib exists +find_path(NMS_REPO_DIR NAMES similarity_search PATHS ${CMAKE_CURRENT_SOURCE_DIR}/external/nmslib NO_DEFAULT_PATH) + +# If not, pull the updated submodule +if (NOT EXISTS ${NMS_REPO_DIR}) + message(STATUS "Could not find nmslib. Pulling updated submodule.") + execute_process(COMMAND git submodule update --init -- external/nmslib WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}) +endif () + +# Check if patch exist, this is to skip git apply during CI build. See CI.yml with ubuntu. +find_path(PATCH_FILE NAMES 0001-Initialize-maxlevel-during-add-from-enterpoint-level.patch PATHS ${CMAKE_CURRENT_SOURCE_DIR}/patches/nmslib NO_DEFAULT_PATH) + +# If it exists, apply patches +if (EXISTS ${PATCH_FILE}) + message(STATUS "Applying custom patches.") + execute_process(COMMAND git am --3way ${CMAKE_CURRENT_SOURCE_DIR}/patches/nmslib/0001-Initialize-maxlevel-during-add-from-enterpoint-level.patch WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/external/nmslib ERROR_VARIABLE ERROR_MSG RESULT_VARIABLE RESULT_CODE) + + if(RESULT_CODE) + message(FATAL_ERROR "Failed to apply patch:\n${ERROR_MSG}") + endif() +endif() + +add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/external/nmslib/similarity_search) diff --git a/jni/cmake/macros.cmake b/jni/cmake/macros.cmake new file mode 100644 index 000000000..773033b7e --- /dev/null +++ b/jni/cmake/macros.cmake @@ -0,0 +1,16 @@ +# +# Copyright OpenSearch Contributors +# SPDX-License-Identifier: Apache-2.0 +# + +macro(opensearch_set_common_properties TARGET) + set_target_properties(${TARGET} PROPERTIES SUFFIX ${LIB_EXT}) + set_target_properties(${TARGET} PROPERTIES POSITION_INDEPENDENT_CODE ON) + + if (NOT "${WIN32}" STREQUAL "") + # Use RUNTIME_OUTPUT_DIRECTORY, to build the target library in the specified directory at runtime. + set_target_properties(${TARGET} PROPERTIES RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/release) + else() + set_target_properties(${TARGET} PROPERTIES LIBRARY_OUTPUT_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/release) + endif() +endmacro() From 4fc0cfff2dee45877de4e4fa0f693ce0b09cbf38 Mon Sep 17 00:00:00 2001 From: Ryan Bogan Date: Wed, 24 Apr 2024 14:28:55 -0700 Subject: [PATCH 245/416] Ensure ignore_unmapped is set to correct value (#1655) * Ensure ignore_unmapped is set to correct value Signed-off-by: Ryan Bogan * Add changelog entry Signed-off-by: Ryan Bogan --------- Signed-off-by: Ryan Bogan --- CHANGELOG.md | 1 + .../java/org/opensearch/knn/index/query/KNNQueryBuilder.java | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fc0cf4238..05ba3be4c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), * Serialize all models into cluster metadata [#1499](https://github.com/opensearch-project/k-NN/pull/1499) ### Bug Fixes * Add stored fields for knn_vector type [#1630](https://github.com/opensearch-project/k-NN/pull/1630) +* Ensure ignore_unmapped is set to correct value [#1655](https://github.com/opensearch-project/k-NN/pull/1655) ### Infrastructure * Add micro-benchmark module in k-NN plugin for benchmark streaming vectors to JNI layer functionality. [#1583](https://github.com/opensearch-project/k-NN/pull/1583) * Add arm64 check when SIMD is disabled [#1618](https://github.com/opensearch-project/k-NN/pull/1618) diff --git a/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java b/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java index 37114f3cb..3a24c1012 100644 --- a/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java +++ b/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java @@ -267,7 +267,7 @@ public static KNNQueryBuilder fromXContent(XContentParser parser) throws IOExcep max_distance = (Float) NumberFieldMapper.NumberType.FLOAT.parse(parser.objectBytes(), false); } else if (MIN_SCORE_FIELD.match(currentFieldName, parser.getDeprecationHandler())) { min_score = (Float) NumberFieldMapper.NumberType.FLOAT.parse(parser.objectBytes(), false); - } else if (IGNORE_UNMAPPED_FIELD.getPreferredName().equals("ignore_unmapped")) { + } else if (IGNORE_UNMAPPED_FIELD.getPreferredName().equals(currentFieldName)) { if (isClusterOnOrAfterMinRequiredVersion("ignore_unmapped")) { ignoreUnmapped = parser.booleanValue(); } From d15e250d389a09ff7ea346a7bd0ff003ae74a749 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Wed, 24 Apr 2024 15:32:58 -0700 Subject: [PATCH 246/416] Add flag for whether patches should be committed (#1656) Signed-off-by: John Mazanec (cherry picked from commit b3829c3468296c38474562dff43f6d3f5cf9ca23) --- .github/workflows/dco.yml | 18 ------------------ DEVELOPER_GUIDE.md | 7 +++++++ build.gradle | 7 +++++-- jni/CMakeLists.txt | 12 ++++++++++++ jni/cmake/init-faiss.cmake | 4 ++-- jni/cmake/init-nmslib.cmake | 2 +- scripts/build.sh | 6 +++--- 7 files changed, 30 insertions(+), 26 deletions(-) delete mode 100644 .github/workflows/dco.yml diff --git a/.github/workflows/dco.yml b/.github/workflows/dco.yml deleted file mode 100644 index cf30ea89d..000000000 --- a/.github/workflows/dco.yml +++ /dev/null @@ -1,18 +0,0 @@ -name: Developer Certificate of Origin Check - -on: [pull_request] - -jobs: - check: - runs-on: ubuntu-latest - - steps: - - name: Get PR Commits - id: 'get-pr-commits' - uses: tim-actions/get-pr-commits@v1.1.0 - with: - token: ${{ secrets.GITHUB_TOKEN }} - - name: DCO Check - uses: tim-actions/dco@v1.1.0 - with: - commits: ${{ steps.get-pr-commits.outputs.commits }} diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index ca5e7cc52..9c17f4f68 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -237,6 +237,13 @@ If you want to make a custom patch on JNI library 3. Place the patch file under `jni/patches` 4. Make a change in `jni/CmakeLists.txt`, `.github/workflows/CI.yml` to apply the patch during build +By default, in the cmake build system, these patches will be applied and committed to the native libraries. In order to +successfully make the commits the `user.name` and `user.email` git configurations need to be setup. If you cannot set +these in your environment, you can disable committing the changes to the library by passing gradle this flag: +`build.lib.commit_patches=false`. For example, `gradlew build -Dbuild.lib.commit_patches=false`. If the patches are +not committed, then the full library build process will run each time `cmake` is invoked. In a development environment, +it is recommended to setup the user git configuration to avoid this cost. + ### Enable SIMD Optimization SIMD(Single Instruction/Multiple Data) Optimization is enabled by default on Linux and Mac which boosts the performance by enabling `AVX2` on `x86 architecture` and `NEON` on `ARM64 architecture` while building the Faiss library. But to enable SIMD, the underlying processor diff --git a/build.gradle b/build.gradle index b608dbb1a..dd2c162c3 100644 --- a/build.gradle +++ b/build.gradle @@ -18,6 +18,9 @@ buildscript { opensearch_group = "org.opensearch" isSnapshot = "true" == System.getProperty("build.snapshot", "true") simd_enabled = System.getProperty("simd.enabled", "true") + // Flag to determine whether cmake build system should apply patches and commit. In automated build environments + // set this to false. In dev environments, set to true + commit_lib_patches = System.getProperty("build.lib.commit_patches", "true") version_tokens = opensearch_version.tokenize('-') opensearch_build = version_tokens[0] + '.0' @@ -303,10 +306,10 @@ task cmakeJniLib(type:Exec) { workingDir 'jni' if (Os.isFamily(Os.FAMILY_WINDOWS)) { dependsOn windowsPatches - commandLine 'cmake', '.', "-G", "Unix Makefiles", "-DKNN_PLUGIN_VERSION=${opensearch_version}", "-DBLAS_LIBRARIES=$rootDir\\src\\main\\resources\\windowsDependencies\\libopenblas.dll", "-DLAPACK_LIBRARIES=$rootDir\\src\\main\\resources\\windowsDependencies\\libopenblas.dll", "-DSIMD_ENABLED=${simd_enabled}" + commandLine 'cmake', '.', "-G", "Unix Makefiles", "-DKNN_PLUGIN_VERSION=${opensearch_version}", "-DBLAS_LIBRARIES=$rootDir\\src\\main\\resources\\windowsDependencies\\libopenblas.dll", "-DLAPACK_LIBRARIES=$rootDir\\src\\main\\resources\\windowsDependencies\\libopenblas.dll", "-DSIMD_ENABLED=${simd_enabled}", "-DCOMMIT_LIB_PATCHES=${commit_lib_patches}" } else { - commandLine 'cmake', '.', "-DKNN_PLUGIN_VERSION=${opensearch_version}", "-DSIMD_ENABLED=${simd_enabled}" + commandLine 'cmake', '.', "-DKNN_PLUGIN_VERSION=${opensearch_version}", "-DSIMD_ENABLED=${simd_enabled}", "-DCOMMIT_LIB_PATCHES=${commit_lib_patches}" } } diff --git a/jni/CMakeLists.txt b/jni/CMakeLists.txt index 3429671fe..6f0894607 100644 --- a/jni/CMakeLists.txt +++ b/jni/CMakeLists.txt @@ -31,6 +31,18 @@ else() set(CONFIG_ALL OFF) endif () +# `git am` will create commits from the patches in the native libraries. This is ideal for development envs +# because it prevents full lib rebuild everytime cmake is run. However, for build systems that will run the +# build workflow once, it can cause issues because git commits require that the user and the user's email be set. +# See https://github.com/opensearch-project/k-NN/issues/1651. So, we provide a flag that allows users to select between +# the two +if(NOT DEFINED COMMIT_LIB_PATCHES OR ${COMMIT_LIB_PATCHES} STREQUAL true) + set(GIT_PATCH_COMMAND am) +else() + set(GIT_PATCH_COMMAND apply) +endif() +message(STATUS "Using the following git patch command: \"${GIT_PATCH_COMMAND}\"") + # Set OS specific variables if (${CMAKE_SYSTEM_NAME} STREQUAL Darwin) set(CMAKE_MACOSX_RPATH 1) diff --git a/jni/cmake/init-faiss.cmake b/jni/cmake/init-faiss.cmake index 44dd4442a..fefed61e2 100644 --- a/jni/cmake/init-faiss.cmake +++ b/jni/cmake/init-faiss.cmake @@ -18,8 +18,8 @@ find_path(PATCH_FILE NAMES 0001-Custom-patch-to-support-multi-vector.patch 0002- # If it exists, apply patches if (EXISTS ${PATCH_FILE}) message(STATUS "Applying custom patches.") - execute_process(COMMAND git am --3way ${CMAKE_CURRENT_SOURCE_DIR}/patches/faiss/0001-Custom-patch-to-support-multi-vector.patch WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/external/faiss ERROR_VARIABLE ERROR_MSG RESULT_VARIABLE RESULT_CODE) - execute_process(COMMAND git am --3way ${CMAKE_CURRENT_SOURCE_DIR}/patches/faiss/0002-Enable-precomp-table-to-be-shared-ivfpq.patch WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/external/faiss ERROR_VARIABLE ERROR_MSG RESULT_VARIABLE RESULT_CODE) + execute_process(COMMAND git ${GIT_PATCH_COMMAND} --3way --ignore-space-change --ignore-whitespace ${CMAKE_CURRENT_SOURCE_DIR}/patches/faiss/0001-Custom-patch-to-support-multi-vector.patch WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/external/faiss ERROR_VARIABLE ERROR_MSG RESULT_VARIABLE RESULT_CODE) + execute_process(COMMAND git ${GIT_PATCH_COMMAND} --3way --ignore-space-change --ignore-whitespace ${CMAKE_CURRENT_SOURCE_DIR}/patches/faiss/0002-Enable-precomp-table-to-be-shared-ivfpq.patch WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/external/faiss ERROR_VARIABLE ERROR_MSG RESULT_VARIABLE RESULT_CODE) if(RESULT_CODE) message(FATAL_ERROR "Failed to apply patch:\n${ERROR_MSG}") endif() diff --git a/jni/cmake/init-nmslib.cmake b/jni/cmake/init-nmslib.cmake index edf2296d9..a735bcbd8 100644 --- a/jni/cmake/init-nmslib.cmake +++ b/jni/cmake/init-nmslib.cmake @@ -18,7 +18,7 @@ find_path(PATCH_FILE NAMES 0001-Initialize-maxlevel-during-add-from-enterpoint-l # If it exists, apply patches if (EXISTS ${PATCH_FILE}) message(STATUS "Applying custom patches.") - execute_process(COMMAND git am --3way ${CMAKE_CURRENT_SOURCE_DIR}/patches/nmslib/0001-Initialize-maxlevel-during-add-from-enterpoint-level.patch WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/external/nmslib ERROR_VARIABLE ERROR_MSG RESULT_VARIABLE RESULT_CODE) + execute_process(COMMAND git ${GIT_PATCH_COMMAND} --3way --ignore-space-change --ignore-whitespace ${CMAKE_CURRENT_SOURCE_DIR}/patches/nmslib/0001-Initialize-maxlevel-during-add-from-enterpoint-level.patch WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/external/nmslib ERROR_VARIABLE ERROR_MSG RESULT_VARIABLE RESULT_CODE) if(RESULT_CODE) message(FATAL_ERROR "Failed to apply patch:\n${ERROR_MSG}") diff --git a/scripts/build.sh b/scripts/build.sh index 43a36443d..d2571efb6 100644 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -118,12 +118,12 @@ fi # Build k-NN lib and plugin through gradle tasks cd $work_dir -./gradlew build --no-daemon --refresh-dependencies -x integTest -x test -Dopensearch.version=$VERSION -Dbuild.snapshot=$SNAPSHOT -Dbuild.version_qualifier=$QUALIFIER -./gradlew :buildJniLib -Dsimd.enabled=false +./gradlew build --no-daemon --refresh-dependencies -x integTest -x test -Dopensearch.version=$VERSION -Dbuild.snapshot=$SNAPSHOT -Dbuild.version_qualifier=$QUALIFIER -Dbuild.lib.commit_patches=false +./gradlew :buildJniLib -Dsimd.enabled=false -Dbuild.lib.commit_patches=false if [ "$PLATFORM" != "windows" ] && [ "$ARCHITECTURE" = "x64" ]; then echo "Building k-NN library after enabling AVX2" - ./gradlew :buildJniLib -Dsimd.enabled=true + ./gradlew :buildJniLib -Dsimd.enabled=true -Dbuild.lib.commit_patches=false fi ./gradlew publishPluginZipPublicationToZipStagingRepository -Dopensearch.version=$VERSION -Dbuild.snapshot=$SNAPSHOT -Dbuild.version_qualifier=$QUALIFIER From 6a77cfc3a482ab5ec4081bb92cb47a775560198d Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Thu, 25 Apr 2024 12:55:52 -0500 Subject: [PATCH 247/416] Updates Developer guide (#1642) (#1657) - Adds clarifications for developing on Mac M series - Minor misc updates Signed-off-by: Tejas Shah (cherry picked from commit f51dd64c8e266ba74cbaf64105572c8d553c91db) Co-authored-by: Tejas Shah --- DEVELOPER_GUIDE.md | 30 +++++++++++++++++++++++------- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index 9c17f4f68..a19586156 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -64,7 +64,7 @@ One easy way to install on mac or linux is to use pip: pip install cmake==3.23.3 ``` -On Mac M1 machines, install cmake using: +On Mac M series machines, install cmake using: ```bash brew install cmake ``` @@ -84,9 +84,9 @@ Additionally, the `gcc` toolchain needs to be installed on Mac. To install, run: brew install gcc ``` -#### Extra setup for Mac M1 Machines +#### Additional setup for Mac M series Machines -The following commands enable running/building k-NN on M1 machines: +The following commands enable running/building k-NN on M series machines: ```bash // Go to k-NN folder @@ -102,6 +102,7 @@ cd jni sed -i -e 's/\/usr\/local\/opt\/libomp\//\/opt\/homebrew\/opt\/llvm\//g' CMakeLists.txt sed -i -e 's/-march=native/-mcpu=apple-m1/g' external/nmslib/similarity_search/CMakeLists.txt sed -i -e 's/pragma message WARN/pragma message /g' external/nmslib/similarity_search/src/distcomp_scalar.cc +// Change to apple-m1, apple-m2 and apple-m3 according to your mac chipset sed -i -e 's/-mcpu=apple-a14/-mcpu=apple-m1/g' external/nmslib/python_bindings/setup.py sed -i -e 's/__aarch64__/__undefine_aarch64__/g' external/faiss/faiss/utils/distances_simd.cpp @@ -128,7 +129,7 @@ Next, obtain a minimum distribution tarball of the k-NN version you want to buil 4. You should see a opensearch-min--SNAPSHOT-darwin-x64.tar.gz file present in distribution/archives/darwin-tar/build/distributions/ 5. Build k-NN by passing the OpenSearch distribution path in `./gradlew -PcustomDistributionUrl=""` -If you want to start OpenSearch directly on Mac M1, make sure to use JDK for ARM. Otherwise, you will see the following error: `mach-o file, but is an incompatible architecture (have 'arm64', need 'x86_64')`. It is better to start OpenSearch by running `bash opensearch-tar-install.sh` instead of `./bin/opensearch`. To run `./bin/opensearch`, the environment variable `JAVA_LIBRARY_PATH` needs to be set correctly so that OpenSearch can find the JNI library: +If you want to start OpenSearch directly on Mac M series, make sure to use JDK for ARM. Otherwise, you will see the following error: `mach-o file, but is an incompatible architecture (have 'arm64', need 'x86_64')`. It is better to start OpenSearch by running `bash opensearch-tar-install.sh` instead of `./bin/opensearch`. To run `./bin/opensearch`, the environment variable `JAVA_LIBRARY_PATH` needs to be set correctly so that OpenSearch can find the JNI library: ``` export OPENSEARCH_HOME=the directory of opensearch... @@ -178,12 +179,25 @@ Please follow these formatting guidelines: OpenSearch k-NN uses a [Gradle](https://docs.gradle.org/6.6.1/userguide/userguide.html) wrapper for its build. Run `gradlew` on Unix systems. +Tests use `JAVA11_HOME` environment variable, make sure to add it in the export path else the tests might fail. +e.g +``` +echo "export JAVA11_HOME=" >> ~/.zshrc +source ~/.zshrc +``` + Build OpenSearch k-NN using `gradlew build` ``` ./gradlew build ``` +For Mac M series machines use +``` +./gradlew build -PcustomDistributionUrl="" +``` + + ### JNI Library The plugin relies on 2 JNI libraries to perform approximate k-NN search. `./gradlew build` will first build the @@ -213,10 +227,10 @@ run: ./bin/jni_test # To run nmslib tests -./bin/jni_test --gtest_filter=Nmslib* +./bin/jni_test --gtest_filter='Nmslib*' # To run faiss tests -./bin/jni_test --gtest_filter=Faiss* +./bin/jni_test --gtest_filter='Faiss*' ``` ### JNI Library Artifacts @@ -264,11 +278,13 @@ cmake . -DSIMD_ENABLED=true ## Run OpenSearch k-NN ### Run Single-node Cluster Locally -Run OpenSearch k-NN using `gradlew run`. +Run OpenSearch k-NN using `gradlew run`. For Mac M series add ```-PcustomDistributionUrl=``` argument. ```shell script ./gradlew run ``` + + That will build OpenSearch and start it, writing its log above Gradle's status message. We log a lot of stuff on startup, specifically these lines tell you that plugin is ready. ``` [2020-05-29T14:50:35,167][INFO ][o.e.h.AbstractHttpServerTransport] [runTask-0] publish_address {127.0.0.1:9200}, bound_addresses {[::1]:9200}, {127.0.0.1:9200} From 036662039bce8b1e58aaaba2c46b9a1398fceef6 Mon Sep 17 00:00:00 2001 From: Junqiu Lei Date: Mon, 29 Apr 2024 10:01:48 -0700 Subject: [PATCH 248/416] * Support filter and nested field in faiss engine radial search (#1652) (#1658) * Use 0.95 as default ratio for lucene radial search traversal similarity (#1619) Signed-off-by: Junqiu Lei --- CHANGELOG.md | 1 + jni/CMakeLists.txt | 2 +- jni/cmake/init-faiss.cmake | 3 +- jni/include/faiss_wrapper.h | 20 ++- .../org_opensearch_knn_jni_FaissService.h | 14 +- ...patch-to-support-range-search-params.patch | 53 ++++++ jni/src/faiss_wrapper.cpp | 71 +++++++- .../org_opensearch_knn_jni_FaissService.cpp | 19 +- jni/tests/faiss_wrapper_test.cpp | 156 ++++++++++++++++- .../opensearch/knn/common/KNNConstants.java | 5 + .../knn/index/query/KNNQueryBuilder.java | 132 +++++++------- .../opensearch/knn/index/query/KNNWeight.java | 5 +- .../knn/index/query/RNNQueryFactory.java | 17 +- .../org/opensearch/knn/jni/FaissService.java | 31 +++- .../org/opensearch/knn/jni/JNIService.java | 21 ++- .../org/opensearch/knn/index/FaissIT.java | 163 +++++++++++------- .../opensearch/knn/index/LuceneEngineIT.java | 6 +- .../opensearch/knn/index/NestedSearchIT.java | 75 +++++++- .../knn/index/query/KNNQueryBuilderTests.java | 8 +- .../knn/index/query/KNNWeightTests.java | 9 +- 20 files changed, 653 insertions(+), 158 deletions(-) create mode 100644 jni/patches/faiss/0003-Custom-patch-to-support-range-search-params.patch diff --git a/CHANGELOG.md b/CHANGELOG.md index 05ba3be4c..c0dc0b523 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Features * Add Clear Cache API [#740](https://github.com/opensearch-project/k-NN/pull/740) * Support radial search in k-NN plugin [#1617](https://github.com/opensearch-project/k-NN/pull/1617) +* Support filter and nested field in faiss engine radial search [#1652](https://github.com/opensearch-project/k-NN/pull/1652) ### Enhancements * Make the HitQueue size more appropriate for exact search [#1549](https://github.com/opensearch-project/k-NN/pull/1549) * Support script score when doc value is disabled [#1573](https://github.com/opensearch-project/k-NN/pull/1573) diff --git a/jni/CMakeLists.txt b/jni/CMakeLists.txt index 6f0894607..595fa6fea 100644 --- a/jni/CMakeLists.txt +++ b/jni/CMakeLists.txt @@ -36,7 +36,7 @@ endif () # build workflow once, it can cause issues because git commits require that the user and the user's email be set. # See https://github.com/opensearch-project/k-NN/issues/1651. So, we provide a flag that allows users to select between # the two -if(NOT DEFINED COMMIT_LIB_PATCHES OR ${COMMIT_LIB_PATCHES} STREQUAL true) +if(NOT DEFINED COMMIT_LIB_PATCHES OR "${COMMIT_LIB_PATCHES}" STREQUAL true) set(GIT_PATCH_COMMAND am) else() set(GIT_PATCH_COMMAND apply) diff --git a/jni/cmake/init-faiss.cmake b/jni/cmake/init-faiss.cmake index fefed61e2..bc3836b06 100644 --- a/jni/cmake/init-faiss.cmake +++ b/jni/cmake/init-faiss.cmake @@ -13,13 +13,14 @@ if (NOT EXISTS ${FAISS_REPO_DIR}) endif () # Check if patch exist, this is to skip git apply during CI build. See CI.yml with ubuntu. -find_path(PATCH_FILE NAMES 0001-Custom-patch-to-support-multi-vector.patch 0002-Enable-precomp-table-to-be-shared-ivfpq.patch PATHS ${CMAKE_CURRENT_SOURCE_DIR}/patches/faiss NO_DEFAULT_PATH) +find_path(PATCH_FILE NAMES 0001-Custom-patch-to-support-multi-vector.patch 0002-Enable-precomp-table-to-be-shared-ivfpq.patch 0003-Custom-patch-to-support-range-search-params.patch PATHS ${CMAKE_CURRENT_SOURCE_DIR}/patches/faiss NO_DEFAULT_PATH) # If it exists, apply patches if (EXISTS ${PATCH_FILE}) message(STATUS "Applying custom patches.") execute_process(COMMAND git ${GIT_PATCH_COMMAND} --3way --ignore-space-change --ignore-whitespace ${CMAKE_CURRENT_SOURCE_DIR}/patches/faiss/0001-Custom-patch-to-support-multi-vector.patch WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/external/faiss ERROR_VARIABLE ERROR_MSG RESULT_VARIABLE RESULT_CODE) execute_process(COMMAND git ${GIT_PATCH_COMMAND} --3way --ignore-space-change --ignore-whitespace ${CMAKE_CURRENT_SOURCE_DIR}/patches/faiss/0002-Enable-precomp-table-to-be-shared-ivfpq.patch WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/external/faiss ERROR_VARIABLE ERROR_MSG RESULT_VARIABLE RESULT_CODE) + execute_process(COMMAND git ${GIT_PATCH_COMMAND} --3way --ignore-space-change --ignore-whitespace ${CMAKE_CURRENT_SOURCE_DIR}/patches/faiss/0003-Custom-patch-to-support-range-search-params.patch WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/external/faiss ERROR_VARIABLE ERROR_MSG RESULT_VARIABLE RESULT_CODE) if(RESULT_CODE) message(FATAL_ERROR "Failed to apply patch:\n${ERROR_MSG}") endif() diff --git a/jni/include/faiss_wrapper.h b/jni/include/faiss_wrapper.h index da67c0f59..958eca8ac 100644 --- a/jni/include/faiss_wrapper.h +++ b/jni/include/faiss_wrapper.h @@ -74,6 +74,22 @@ namespace knn_jni { jbyteArray TrainIndex(knn_jni::JNIUtilInterface * jniUtil, JNIEnv * env, jobject parametersJ, jint dimension, jlong trainVectorsPointerJ); + /* + * Perform a range search with filter against the index located in memory at indexPointerJ. + * + * @param indexPointerJ - pointer to the index + * @param queryVectorJ - the query vector + * @param radiusJ - the radius for the range search + * @param maxResultsWindowJ - the maximum number of results to return + * @param filterIdsJ - the filter ids + * @param filterIdsTypeJ - the filter ids type + * @param parentIdsJ - the parent ids + * + * @return an array of RangeQueryResults + */ + jobjectArray RangeSearchWithFilter(knn_jni::JNIUtilInterface *jniUtil, JNIEnv *env, jlong indexPointerJ, jfloatArray queryVectorJ, + jfloat radiusJ, jint maxResultWindowJ, jlongArray filterIdsJ, jint filterIdsTypeJ, jintArray parentIdsJ); + /* * Perform a range search against the index located in memory at indexPointerJ. * @@ -81,10 +97,12 @@ namespace knn_jni { * @param queryVectorJ - the query vector * @param radiusJ - the radius for the range search * @param maxResultsWindowJ - the maximum number of results to return + * @param parentIdsJ - the parent ids + * * @return an array of RangeQueryResults */ jobjectArray RangeSearch(knn_jni::JNIUtilInterface *jniUtil, JNIEnv *env, jlong indexPointerJ, jfloatArray queryVectorJ, - jfloat radiusJ, jint maxResultsWindowJ); + jfloat radiusJ, jint maxResultWindowJ, jintArray parentIdsJ); } } diff --git a/jni/include/org_opensearch_knn_jni_FaissService.h b/jni/include/org_opensearch_knn_jni_FaissService.h index 3715730ab..e16677db7 100644 --- a/jni/include/org_opensearch_knn_jni_FaissService.h +++ b/jni/include/org_opensearch_knn_jni_FaissService.h @@ -124,11 +124,19 @@ JNIEXPORT jlong JNICALL Java_org_opensearch_knn_jni_FaissService_transferVectors /* * Class: org_opensearch_knn_jni_FaissService -* Method: rangeSearchIndex -* Signature: (J[F[F)J +* Method: rangeSearchIndexWithFilter +* Signature: (J[FJ[I)[Lorg/opensearch/knn/index/query/RangeQueryResult; */ +JNIEXPORT jobjectArray JNICALL Java_org_opensearch_knn_jni_FaissService_rangeSearchIndexWithFilter + (JNIEnv *, jclass, jlong, jfloatArray, jfloat, jint, jlongArray, jint, jintArray); + +/* + * Class: org_opensearch_knn_jni_FaissService + * Method: rangeSearchIndex + * Signature: (J[FJ[I)[Lorg/opensearch/knn/index/query/RangeQueryResult; + */ JNIEXPORT jobjectArray JNICALL Java_org_opensearch_knn_jni_FaissService_rangeSearchIndex - (JNIEnv *, jclass, jlong, jfloatArray, jfloat, jint); + (JNIEnv *, jclass, jlong, jfloatArray, jfloat, jint, jintArray); #ifdef __cplusplus } diff --git a/jni/patches/faiss/0003-Custom-patch-to-support-range-search-params.patch b/jni/patches/faiss/0003-Custom-patch-to-support-range-search-params.patch new file mode 100644 index 000000000..bdc202bf6 --- /dev/null +++ b/jni/patches/faiss/0003-Custom-patch-to-support-range-search-params.patch @@ -0,0 +1,53 @@ +From af6770b505a32b2c4eab2036d2509dec4b137f28 Mon Sep 17 00:00:00 2001 +From: Junqiu Lei +Date: Tue, 23 Apr 2024 17:18:56 -0700 +Subject: [PATCH] Custom patch to support range search params + +Signed-off-by: Junqiu Lei +--- + faiss/IndexIDMap.cpp | 28 ++++++++++++++++++++++++---- + 1 file changed, 24 insertions(+), 4 deletions(-) + +diff --git a/faiss/IndexIDMap.cpp b/faiss/IndexIDMap.cpp +index 3f375e7b..11f3a847 100644 +--- a/faiss/IndexIDMap.cpp ++++ b/faiss/IndexIDMap.cpp +@@ -176,11 +176,31 @@ void IndexIDMapTemplate::range_search( + RangeSearchResult* result, + const SearchParameters* params) const { + if (params) { +- SearchParameters internal_search_parameters; +- IDSelectorTranslated id_selector_translated(id_map, params->sel); +- internal_search_parameters.sel = &id_selector_translated; ++ IDSelectorTranslated this_idtrans(this->id_map, nullptr); ++ ScopedSelChange sel_change; ++ IDGrouperTranslated this_idgrptrans(this->id_map, nullptr); ++ ScopedGrpChange grp_change; ++ ++ if (params->sel) { ++ auto idtrans = dynamic_cast(params->sel); ++ ++ if (!idtrans) { ++ auto params_non_const = const_cast(params); ++ this_idtrans.sel = params->sel; ++ sel_change.set(params_non_const, &this_idtrans); ++ } ++ } ++ ++ if (params->grp) { ++ auto idtrans = dynamic_cast(params->grp); + +- index->range_search(n, x, radius, result, &internal_search_parameters); ++ if (!idtrans) { ++ auto params_non_const = const_cast(params); ++ this_idgrptrans.grp = params->grp; ++ grp_change.set(params_non_const, &this_idgrptrans); ++ } ++ } ++ index->range_search(n, x, radius, result, params); + } else { + index->range_search(n, x, radius, result); + } +-- +2.39.0 + diff --git a/jni/src/faiss_wrapper.cpp b/jni/src/faiss_wrapper.cpp index 983cfa8a9..5a0910d9a 100644 --- a/jni/src/faiss_wrapper.cpp +++ b/jni/src/faiss_wrapper.cpp @@ -589,7 +589,12 @@ faiss::IndexIVFPQ * extractIVFPQIndex(faiss::Index * index) { } jobjectArray knn_jni::faiss_wrapper::RangeSearch(knn_jni::JNIUtilInterface *jniUtil, JNIEnv *env, jlong indexPointerJ, - jfloatArray queryVectorJ, jfloat radiusJ, jint maxResultWindowJ) { + jfloatArray queryVectorJ, jfloat radiusJ, jint maxResultWindowJ, jintArray parentIdsJ) { + return knn_jni::faiss_wrapper::RangeSearchWithFilter(jniUtil, env, indexPointerJ, queryVectorJ, radiusJ, maxResultWindowJ, nullptr, 0, parentIdsJ); +} + +jobjectArray knn_jni::faiss_wrapper::RangeSearchWithFilter(knn_jni::JNIUtilInterface *jniUtil, JNIEnv *env, jlong indexPointerJ, + jfloatArray queryVectorJ, jfloat radiusJ, jint maxResultWindowJ, jlongArray filterIdsJ, jint filterIdsTypeJ, jintArray parentIdsJ) { if (queryVectorJ == nullptr) { throw std::runtime_error("Query Vector cannot be null"); } @@ -605,7 +610,69 @@ jobjectArray knn_jni::faiss_wrapper::RangeSearch(knn_jni::JNIUtilInterface *jniU // The res will be freed by ~RangeSearchResult() in FAISS // The second parameter is always true, as lims is allocated by FAISS faiss::RangeSearchResult res(1, true); - indexReader->range_search(1, rawQueryVector, radiusJ, &res); + + if(filterIdsJ != nullptr) { + jlong *filteredIdsArray = jniUtil->GetLongArrayElements(env, filterIdsJ, nullptr); + int filterIdsLength = jniUtil->GetJavaLongArrayLength(env, filterIdsJ); + std::unique_ptr idSelector; + if(filterIdsTypeJ == BITMAP) { + idSelector.reset(new faiss::IDSelectorJlongBitmap(filterIdsLength, filteredIdsArray)); + } else { + faiss::idx_t* batchIndices = reinterpret_cast(filteredIdsArray); + idSelector.reset(new faiss::IDSelectorBatch(filterIdsLength, batchIndices)); + } + faiss::SearchParameters *searchParameters; + faiss::SearchParametersHNSW hnswParams; + faiss::SearchParametersIVF ivfParams; + std::unique_ptr idGrouper; + std::vector idGrouperBitmap; + auto hnswReader = dynamic_cast(indexReader->index); + if(hnswReader) { + // Setting the ef_search value equal to what was provided during index creation. SearchParametersHNSW has a default + // value of ef_search = 16 which will then be used. + hnswParams.efSearch = hnswReader->hnsw.efSearch; + hnswParams.sel = idSelector.get(); + if (parentIdsJ != nullptr) { + idGrouper = buildIDGrouperBitmap(jniUtil, env, parentIdsJ, &idGrouperBitmap); + hnswParams.grp = idGrouper.get(); + } + searchParameters = &hnswParams; + } else { + auto ivfReader = dynamic_cast(indexReader->index); + auto ivfFlatReader = dynamic_cast(indexReader->index); + if(ivfReader || ivfFlatReader) { + ivfParams.sel = idSelector.get(); + searchParameters = &ivfParams; + } + } + try { + indexReader->range_search(1, rawQueryVector, radiusJ, &res, searchParameters); + } catch (...) { + jniUtil->ReleaseFloatArrayElements(env, queryVectorJ, rawQueryVector, JNI_ABORT); + jniUtil->ReleaseLongArrayElements(env, filterIdsJ, filteredIdsArray, JNI_ABORT); + throw; + } + } else { + faiss::SearchParameters *searchParameters = nullptr; + faiss::SearchParametersHNSW hnswParams; + std::unique_ptr idGrouper; + std::vector idGrouperBitmap; + auto hnswReader = dynamic_cast(indexReader->index); + if(hnswReader!= nullptr && parentIdsJ != nullptr) { + // Setting the ef_search value equal to what was provided during index creation. SearchParametersHNSW has a default + // value of ef_search = 16 which will then be used. + hnswParams.efSearch = hnswReader->hnsw.efSearch; + idGrouper = buildIDGrouperBitmap(jniUtil, env, parentIdsJ, &idGrouperBitmap); + hnswParams.grp = idGrouper.get(); + searchParameters = &hnswParams; + } + try { + indexReader->range_search(1, rawQueryVector, radiusJ, &res, searchParameters); + } catch (...) { + jniUtil->ReleaseFloatArrayElements(env, queryVectorJ, rawQueryVector, JNI_ABORT); + throw; + } + } // lims is structured to support batched queries, it has a length of nq + 1 (where nq is the number of queries), // lims[i] - lims[i-1] gives the number of results for the i-th query. With a single query we used in k-NN, diff --git a/jni/src/org_opensearch_knn_jni_FaissService.cpp b/jni/src/org_opensearch_knn_jni_FaissService.cpp index ab2a37e84..0aa51987d 100644 --- a/jni/src/org_opensearch_knn_jni_FaissService.cpp +++ b/jni/src/org_opensearch_knn_jni_FaissService.cpp @@ -194,11 +194,26 @@ JNIEXPORT jlong JNICALL Java_org_opensearch_knn_jni_FaissService_transferVectors JNIEXPORT jobjectArray JNICALL Java_org_opensearch_knn_jni_FaissService_rangeSearchIndex(JNIEnv * env, jclass cls, jlong indexPointerJ, jfloatArray queryVectorJ, - jfloat radiusJ, jint maxResultWindowJ) + jfloat radiusJ, jint maxResultWindowJ, + jintArray parentIdsJ) { try { - return knn_jni::faiss_wrapper::RangeSearch(&jniUtil, env, indexPointerJ, queryVectorJ, radiusJ, maxResultWindowJ); + return knn_jni::faiss_wrapper::RangeSearch(&jniUtil, env, indexPointerJ, queryVectorJ, radiusJ, maxResultWindowJ, parentIdsJ); + } catch (...) { + jniUtil.CatchCppExceptionAndThrowJava(env); + } + return nullptr; +} +JNIEXPORT jobjectArray JNICALL Java_org_opensearch_knn_jni_FaissService_rangeSearchIndexWithFilter(JNIEnv * env, jclass cls, + jlong indexPointerJ, + jfloatArray queryVectorJ, + jfloat radiusJ, jint maxResultWindowJ, + jlongArray filterIdsJ, jint filterIdsTypeJ, jintArray parentIdsJ) +{ + try { + return knn_jni::faiss_wrapper::RangeSearchWithFilter(&jniUtil, env, indexPointerJ, queryVectorJ, radiusJ, + maxResultWindowJ, filterIdsJ, filterIdsTypeJ, parentIdsJ); } catch (...) { jniUtil.CatchCppExceptionAndThrowJava(env); } diff --git a/jni/tests/faiss_wrapper_test.cpp b/jni/tests/faiss_wrapper_test.cpp index 07b34976f..e9316dcc2 100644 --- a/jni/tests/faiss_wrapper_test.cpp +++ b/jni/tests/faiss_wrapper_test.cpp @@ -628,7 +628,7 @@ TEST(FaissRangeSearchQueryIndexTest, BasicAssertions) { // Define query data float radius = 100000.0; - int numQueries = 2; + int numQueries = 100; std::vector> queries; for (int i = 0; i < numQueries; i++) { @@ -659,7 +659,7 @@ TEST(FaissRangeSearchQueryIndexTest, BasicAssertions) { knn_jni::faiss_wrapper::RangeSearch( &mockJNIUtil, jniEnv, reinterpret_cast(&createdIndexWithData), - reinterpret_cast(&query), radius, maxResultWindow))); + reinterpret_cast(&query), radius, maxResultWindow, nullptr))); // assert result size is not 0 ASSERT_NE(0, results->size()); @@ -684,7 +684,7 @@ TEST(FaissRangeSearchQueryIndexTest_WhenHitMaxWindowResult, BasicAssertions){ // Define query data float radius = 100000.0; - int numQueries = 2; + int numQueries = 100; std::vector> queries; for (int i = 0; i < numQueries; i++) { @@ -715,7 +715,7 @@ TEST(FaissRangeSearchQueryIndexTest_WhenHitMaxWindowResult, BasicAssertions){ knn_jni::faiss_wrapper::RangeSearch( &mockJNIUtil, jniEnv, reinterpret_cast(&createdIndexWithData), - reinterpret_cast(&query), radius, maxResultWindow))); + reinterpret_cast(&query), radius, maxResultWindow, nullptr))); // assert result size is not 0 ASSERT_NE(0, results->size()); @@ -728,3 +728,151 @@ TEST(FaissRangeSearchQueryIndexTest_WhenHitMaxWindowResult, BasicAssertions){ } } } + +TEST(FaissRangeSearchQueryIndexTestWithFilterTest, BasicAssertions) { + // Define the index data + faiss::idx_t numIds = 200; + int dim = 2; + std::vector ids = test_util::Range(numIds); + std::vector vectors = test_util::RandomVectors(dim, numIds, randomDataMin, randomDataMax); + + faiss::MetricType metricType = faiss::METRIC_L2; + std::string method = "HNSW32,Flat"; + + // Define query data + float radius = 100000.0; + int numQueries = 100; + std::vector> queries; + + for (int i = 0; i < numQueries; i++) { + std::vector query; + query.reserve(dim); + for (int j = 0; j < dim; j++) { + query.push_back(test_util::RandomFloat(randomDataMin, randomDataMax)); + } + queries.push_back(query); + } + + // Create the index + std::unique_ptr createdIndex( + test_util::FaissCreateIndex(dim, method, metricType)); + auto createdIndexWithData = + test_util::FaissAddData(createdIndex.get(), ids, vectors); + + // Setup jni + JNIEnv *jniEnv = nullptr; + NiceMock mockJNIUtil; + + int num_bits = test_util::bits2words(164); + std::vector bitmap(num_bits,0); + std::vector filterIds; + + for (int64_t i = 154; i < 163; i++) { + filterIds.push_back(i); + test_util::setBitSet(i, bitmap.data(), bitmap.size()); + } + std::unordered_set filterIdSet(filterIds.begin(), filterIds.end()); + + int maxResultWindow = 20000; + + for (auto query : queries) { + std::unique_ptr *>> results( + reinterpret_cast *> *>( + + knn_jni::faiss_wrapper::RangeSearchWithFilter( + &mockJNIUtil, jniEnv, + reinterpret_cast(&createdIndexWithData), + reinterpret_cast(&query), radius, maxResultWindow, + reinterpret_cast(&bitmap), 0, nullptr))); + + // assert result size is not 0 + ASSERT_NE(0, results->size()); + ASSERT_TRUE(results->size() <= filterIds.size()); + for (const auto& pairPtr : *results) { + auto it = filterIdSet.find(pairPtr->first); + ASSERT_NE(it, filterIdSet.end()); + } + + // Need to free up each result + for (auto it : *results) { + delete it; + } + } +} + +TEST(FaissRangeSearchQueryIndexTestWithParentFilterTest, BasicAssertions) { + // Define the index data + faiss::idx_t numIds = 100; + std::vector ids; + std::vector vectors; + std::vector parentIds; + int dim = 2; + for (int64_t i = 1; i < numIds + 1; i++) { + if (i % 10 == 0) { + parentIds.push_back(i); + continue; + } + ids.push_back(i); + for (int j = 0; j < dim; j++) { + vectors.push_back(test_util::RandomFloat(-500.0, 500.0)); + } + } + + faiss::MetricType metricType = faiss::METRIC_L2; + std::string method = "HNSW32,Flat"; + + // Define query data + float radius = 100000.0; + int numQueries = 1; + std::vector> queries; + + for (int i = 0; i < numQueries; i++) { + std::vector query; + query.reserve(dim); + for (int j = 0; j < dim; j++) { + query.push_back(test_util::RandomFloat(-500.0, 500.0)); + } + queries.push_back(query); + } + + // Create the index + std::unique_ptr createdIndex( + test_util::FaissCreateIndex(dim, method, metricType)); + auto createdIndexWithData = + test_util::FaissAddData(createdIndex.get(), ids, vectors); + + // Setup jni + JNIEnv *jniEnv = nullptr; + NiceMock mockJNIUtil; + EXPECT_CALL(mockJNIUtil, + GetJavaIntArrayLength( + jniEnv, reinterpret_cast(&parentIds))) + .WillRepeatedly(Return(parentIds.size())); + + int maxResultWindow = 10000; + + for (auto query : queries) { + std::unique_ptr *>> results( + reinterpret_cast *> *>( + + knn_jni::faiss_wrapper::RangeSearchWithFilter( + &mockJNIUtil, jniEnv, + reinterpret_cast(&createdIndexWithData), + reinterpret_cast(&query), radius, maxResultWindow, nullptr, 0, + reinterpret_cast(&parentIds)))); + + // assert result size is not 0 + ASSERT_NE(0, results->size()); + // Result should be one for each group + std::set idSet; + for (const auto& pairPtr : *results) { + idSet.insert(pairPtr->first / 10); + } + ASSERT_NE(0, idSet.size()); + + // Need to free up each result + for (auto it : *results) { + delete it; + } + } +} \ No newline at end of file diff --git a/src/main/java/org/opensearch/knn/common/KNNConstants.java b/src/main/java/org/opensearch/knn/common/KNNConstants.java index 2c3b03e47..7c5bb61ad 100644 --- a/src/main/java/org/opensearch/knn/common/KNNConstants.java +++ b/src/main/java/org/opensearch/knn/common/KNNConstants.java @@ -132,4 +132,9 @@ public class KNNConstants { // API Constants public static final String CLEAR_CACHE = "clear_cache"; + + // Radial search constants + public static final Float DEFAULT_LUCENE_RADIAL_SEARCH_TRAVERSAL_SIMILARITY_RATIO = 0.95f; + public static final String MIN_SCORE = "min_score"; + public static final String MAX_DISTANCE = "max_distance"; } diff --git a/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java b/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java index 3a24c1012..3d3b0969f 100644 --- a/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java +++ b/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java @@ -37,6 +37,8 @@ import org.opensearch.index.query.QueryShardContext; import static org.opensearch.knn.index.IndexUtil.*; +import static org.opensearch.knn.common.KNNConstants.MAX_DISTANCE; +import static org.opensearch.knn.common.KNNConstants.MIN_SCORE; import static org.opensearch.knn.common.KNNValidationUtil.validateByteVectorValue; import static org.opensearch.knn.index.IndexUtil.isClusterOnOrAfterMinRequiredVersion; import static org.opensearch.knn.index.util.KNNEngine.ENGINES_SUPPORTING_RADIAL_SEARCH; @@ -52,8 +54,8 @@ public class KNNQueryBuilder extends AbstractQueryBuilder { public static final ParseField K_FIELD = new ParseField("k"); public static final ParseField FILTER_FIELD = new ParseField("filter"); public static final ParseField IGNORE_UNMAPPED_FIELD = new ParseField("ignore_unmapped"); - public static final ParseField MAX_DISTANCE_FIELD = new ParseField("max_distance"); - public static final ParseField MIN_SCORE_FIELD = new ParseField("min_score"); + public static final ParseField MAX_DISTANCE_FIELD = new ParseField(MAX_DISTANCE); + public static final ParseField MIN_SCORE_FIELD = new ParseField(MIN_SCORE); public static final int K_MAX = 10000; /** * The name for the knn query @@ -65,8 +67,8 @@ public class KNNQueryBuilder extends AbstractQueryBuilder { private final String fieldName; private final float[] vector; private int k = 0; - private Float max_distance = null; - private Float min_score = null; + private Float maxDistance = null; + private Float minScore = null; private QueryBuilder filter; private boolean ignoreUnmapped = false; @@ -78,13 +80,13 @@ public class KNNQueryBuilder extends AbstractQueryBuilder { */ public KNNQueryBuilder(String fieldName, float[] vector) { if (Strings.isNullOrEmpty(fieldName)) { - throw new IllegalArgumentException("[" + NAME + "] requires fieldName"); + throw new IllegalArgumentException(String.format("[%s] requires fieldName", NAME)); } if (vector == null) { - throw new IllegalArgumentException("[" + NAME + "] requires query vector"); + throw new IllegalArgumentException(String.format("[%s] requires query vector", NAME)); } if (vector.length == 0) { - throw new IllegalArgumentException("[" + NAME + "] query vector is empty"); + throw new IllegalArgumentException(String.format("[%s] query vector is empty", NAME)); } this.fieldName = fieldName; this.vector = vector; @@ -97,44 +99,44 @@ public KNNQueryBuilder(String fieldName, float[] vector) { */ public KNNQueryBuilder k(Integer k) { if (k == null) { - throw new IllegalArgumentException("[" + NAME + "] requires k to be set"); + throw new IllegalArgumentException(String.format("[%s] requires k to be set", NAME)); } - validateSingleQueryType(k, max_distance, min_score); + validateSingleQueryType(k, maxDistance, minScore); if (k <= 0 || k > K_MAX) { - throw new IllegalArgumentException("[" + NAME + "] requires 0 < k <= " + K_MAX); + throw new IllegalArgumentException(String.format("[%s] requires k to be in the range (0, %d]", NAME, K_MAX)); } this.k = k; return this; } /** - * Builder method for max_distance + * Builder method for maxDistance * - * @param max_distance the max_distance threshold for the nearest neighbours + * @param maxDistance the maxDistance threshold for the nearest neighbours */ - public KNNQueryBuilder maxDistance(Float max_distance) { - if (max_distance == null) { - throw new IllegalArgumentException("[" + NAME + "] requires max_distance to be set"); + public KNNQueryBuilder maxDistance(Float maxDistance) { + if (maxDistance == null) { + throw new IllegalArgumentException(String.format("[%s] requires maxDistance to be set", NAME)); } - validateSingleQueryType(k, max_distance, min_score); - this.max_distance = max_distance; + validateSingleQueryType(k, maxDistance, minScore); + this.maxDistance = maxDistance; return this; } /** - * Builder method for min_score + * Builder method for minScore * - * @param min_score the min_score threshold for the nearest neighbours + * @param minScore the minScore threshold for the nearest neighbours */ - public KNNQueryBuilder minScore(Float min_score) { - if (min_score == null) { - throw new IllegalArgumentException("[" + NAME + "] requires min_score to be set"); + public KNNQueryBuilder minScore(Float minScore) { + if (minScore == null) { + throw new IllegalArgumentException(String.format("[%s] requires minScore to be set", NAME)); } - validateSingleQueryType(k, max_distance, min_score); - if (min_score <= 0) { - throw new IllegalArgumentException("[" + NAME + "] requires min_score greater than 0"); + validateSingleQueryType(k, maxDistance, minScore); + if (minScore <= 0) { + throw new IllegalArgumentException(String.format("[%s] requires minScore to be greater than 0", NAME)); } - this.min_score = min_score; + this.minScore = minScore; return this; } @@ -161,19 +163,19 @@ public KNNQueryBuilder(String fieldName, float[] vector, int k) { public KNNQueryBuilder(String fieldName, float[] vector, int k, QueryBuilder filter) { if (StringUtils.isBlank(fieldName)) { - throw new IllegalArgumentException("[" + NAME + "] requires fieldName"); + throw new IllegalArgumentException(String.format("[%s] requires fieldName", NAME)); } if (vector == null) { - throw new IllegalArgumentException("[" + NAME + "] requires query vector"); + throw new IllegalArgumentException(String.format("[%s] requires query vector", NAME)); } if (vector.length == 0) { - throw new IllegalArgumentException("[" + NAME + "] query vector is empty"); + throw new IllegalArgumentException(String.format("[%s] query vector is empty", NAME)); } if (k <= 0) { - throw new IllegalArgumentException("[" + NAME + "] requires k > 0"); + throw new IllegalArgumentException(String.format("[%s] requires k > 0", NAME)); } if (k > K_MAX) { - throw new IllegalArgumentException("[" + NAME + "] requires k <= " + K_MAX); + throw new IllegalArgumentException(String.format("[%s] requires k <= %d", NAME, K_MAX)); } this.fieldName = fieldName; @@ -181,8 +183,8 @@ public KNNQueryBuilder(String fieldName, float[] vector, int k, QueryBuilder fil this.k = k; this.filter = filter; this.ignoreUnmapped = false; - this.max_distance = null; - this.min_score = null; + this.maxDistance = null; + this.minScore = null; } public static void initialize(ModelDao modelDao) { @@ -222,10 +224,10 @@ public KNNQueryBuilder(StreamInput in) throws IOException { ignoreUnmapped = in.readOptionalBoolean(); } if (isClusterOnOrAfterMinRequiredVersion(KNNConstants.RADIAL_SEARCH_KEY)) { - max_distance = in.readOptionalFloat(); + maxDistance = in.readOptionalFloat(); } if (isClusterOnOrAfterMinRequiredVersion(KNNConstants.RADIAL_SEARCH_KEY)) { - min_score = in.readOptionalFloat(); + minScore = in.readOptionalFloat(); } } catch (IOException ex) { throw new RuntimeException("[KNN] Unable to create KNNQueryBuilder", ex); @@ -237,8 +239,8 @@ public static KNNQueryBuilder fromXContent(XContentParser parser) throws IOExcep List vector = null; float boost = AbstractQueryBuilder.DEFAULT_BOOST; Integer k = null; - Float max_distance = null; - Float min_score = null; + Float maxDistance = null; + Float minScore = null; QueryBuilder filter = null; boolean ignoreUnmapped = false; String queryName = null; @@ -264,9 +266,9 @@ public static KNNQueryBuilder fromXContent(XContentParser parser) throws IOExcep } else if (AbstractQueryBuilder.NAME_FIELD.match(currentFieldName, parser.getDeprecationHandler())) { queryName = parser.text(); } else if (MAX_DISTANCE_FIELD.match(currentFieldName, parser.getDeprecationHandler())) { - max_distance = (Float) NumberFieldMapper.NumberType.FLOAT.parse(parser.objectBytes(), false); + maxDistance = (Float) NumberFieldMapper.NumberType.FLOAT.parse(parser.objectBytes(), false); } else if (MIN_SCORE_FIELD.match(currentFieldName, parser.getDeprecationHandler())) { - min_score = (Float) NumberFieldMapper.NumberType.FLOAT.parse(parser.objectBytes(), false); + minScore = (Float) NumberFieldMapper.NumberType.FLOAT.parse(parser.objectBytes(), false); } else if (IGNORE_UNMAPPED_FIELD.getPreferredName().equals(currentFieldName)) { if (isClusterOnOrAfterMinRequiredVersion("ignore_unmapped")) { ignoreUnmapped = parser.booleanValue(); @@ -320,7 +322,7 @@ public static KNNQueryBuilder fromXContent(XContentParser parser) throws IOExcep } } - validateSingleQueryType(k, max_distance, min_score); + validateSingleQueryType(k, maxDistance, minScore); KNNQueryBuilder knnQueryBuilder = new KNNQueryBuilder(fieldName, ObjectsToFloats(vector)).filter(filter) .boost(boost) @@ -332,10 +334,10 @@ public static KNNQueryBuilder fromXContent(XContentParser parser) throws IOExcep if (k != null) { knnQueryBuilder.k(k); - } else if (max_distance != null) { - knnQueryBuilder.maxDistance(max_distance); - } else if (min_score != null) { - knnQueryBuilder.minScore(min_score); + } else if (maxDistance != null) { + knnQueryBuilder.maxDistance(maxDistance); + } else if (minScore != null) { + knnQueryBuilder.minScore(minScore); } return knnQueryBuilder; @@ -355,10 +357,10 @@ protected void doWriteTo(StreamOutput out) throws IOException { out.writeOptionalBoolean(ignoreUnmapped); } if (isClusterOnOrAfterMinRequiredVersion(KNNConstants.RADIAL_SEARCH_KEY)) { - out.writeOptionalFloat(max_distance); + out.writeOptionalFloat(maxDistance); } if (isClusterOnOrAfterMinRequiredVersion(KNNConstants.RADIAL_SEARCH_KEY)) { - out.writeOptionalFloat(min_score); + out.writeOptionalFloat(minScore); } } @@ -381,11 +383,11 @@ public int getK() { } public float getMaxDistance() { - return this.max_distance; + return this.maxDistance; } public float getMinScore() { - return this.min_score; + return this.minScore; } public QueryBuilder getFilter() { @@ -416,14 +418,14 @@ public void doXContent(XContentBuilder builder, Params params) throws IOExceptio if (filter != null) { builder.field(FILTER_FIELD.getPreferredName(), filter); } - if (max_distance != null) { - builder.field(MAX_DISTANCE_FIELD.getPreferredName(), max_distance); + if (maxDistance != null) { + builder.field(MAX_DISTANCE_FIELD.getPreferredName(), maxDistance); } if (ignoreUnmapped) { builder.field(IGNORE_UNMAPPED_FIELD.getPreferredName(), ignoreUnmapped); } - if (min_score != null) { - builder.field(MIN_SCORE_FIELD.getPreferredName(), min_score); + if (minScore != null) { + builder.field(MIN_SCORE_FIELD.getPreferredName(), minScore); } printBoostAndQueryName(builder); builder.endObject(); @@ -467,18 +469,22 @@ protected Query doToQuery(QueryShardContext context) { // Currently, k-NN supports distance and score types radial search // We need transform distance/score to right type of engine required radius. Float radius = null; - if (this.max_distance != null) { - if (this.max_distance < 0 && SpaceType.INNER_PRODUCT.equals(spaceType) == false) { - throw new IllegalArgumentException("[" + NAME + "] requires distance to be non-negative for space type: " + spaceType); + if (this.maxDistance != null) { + if (this.maxDistance < 0 && SpaceType.INNER_PRODUCT.equals(spaceType) == false) { + throw new IllegalArgumentException( + String.format("[" + NAME + "] requires distance to be non-negative for space type: %s", spaceType) + ); } - radius = knnEngine.distanceToRadialThreshold(this.max_distance, spaceType); + radius = knnEngine.distanceToRadialThreshold(this.maxDistance, spaceType); } - if (this.min_score != null) { - if (this.min_score > 1 && SpaceType.INNER_PRODUCT.equals(spaceType) == false) { - throw new IllegalArgumentException("[" + NAME + "] requires score to be in the range (0, 1] for space type: " + spaceType); + if (this.minScore != null) { + if (this.minScore > 1 && SpaceType.INNER_PRODUCT.equals(spaceType) == false) { + throw new IllegalArgumentException( + String.format("[" + NAME + "] requires score to be in the range [0, 1] for space type: %s", spaceType) + ); } - radius = knnEngine.scoreToRadialThreshold(this.min_score, spaceType); + radius = knnEngine.scoreToRadialThreshold(this.minScore, spaceType); } if (fieldDimension != vector.length) { @@ -538,7 +544,7 @@ protected Query doToQuery(QueryShardContext context) { .build(); return RNNQueryFactory.create(createQueryRequest); } - throw new IllegalArgumentException("[" + NAME + "] requires either k or distance or score to be set"); + throw new IllegalArgumentException(String.format("[%s] requires k or distance or score to be set", NAME)); } private ModelMetadata getModelMetadataForField(KNNVectorFieldMapper.KNNVectorFieldType knnVectorField) { @@ -588,9 +594,7 @@ private static void validateSingleQueryType(Integer k, Float distance, Float sco } if (countSetFields != 1) { - throw new IllegalArgumentException( - "[" + NAME + "] requires only one query type to be set, it can be either k, distance, or score" - ); + throw new IllegalArgumentException(String.format("[%s] requires exactly one of k, distance or score to be set", NAME)); } } } diff --git a/src/main/java/org/opensearch/knn/index/query/KNNWeight.java b/src/main/java/org/opensearch/knn/index/query/KNNWeight.java index 8939a569e..bac8c03d4 100644 --- a/src/main/java/org/opensearch/knn/index/query/KNNWeight.java +++ b/src/main/java/org/opensearch/knn/index/query/KNNWeight.java @@ -294,7 +294,10 @@ private Map doANNSearch(final LeafReaderContext context, final B knnQuery.getQueryVector(), knnQuery.getRadius(), knnEngine, - knnQuery.getContext().getMaxResultWindow() + knnQuery.getContext().getMaxResultWindow(), + filterIds, + filterType.getValue(), + parentIds ); } } catch (Exception e) { diff --git a/src/main/java/org/opensearch/knn/index/query/RNNQueryFactory.java b/src/main/java/org/opensearch/knn/index/query/RNNQueryFactory.java index cd32ac4f3..db8084864 100644 --- a/src/main/java/org/opensearch/knn/index/query/RNNQueryFactory.java +++ b/src/main/java/org/opensearch/knn/index/query/RNNQueryFactory.java @@ -5,6 +5,7 @@ package org.opensearch.knn.index.query; +import static org.opensearch.knn.common.KNNConstants.DEFAULT_LUCENE_RADIAL_SEARCH_TRAVERSAL_SIMILARITY_RATIO; import static org.opensearch.knn.common.KNNConstants.VECTOR_DATA_TYPE_FIELD; import static org.opensearch.knn.index.VectorDataType.SUPPORTED_VECTOR_DATA_TYPES; @@ -118,7 +119,13 @@ private static Query getFloatVectorSimilarityQuery( final float resultSimilarity, final Query filterQuery ) { - return new FloatVectorSimilarityQuery(fieldName, floatVector, resultSimilarity, filterQuery); + return new FloatVectorSimilarityQuery( + fieldName, + floatVector, + DEFAULT_LUCENE_RADIAL_SEARCH_TRAVERSAL_SIMILARITY_RATIO * resultSimilarity, + resultSimilarity, + filterQuery + ); } /** @@ -131,6 +138,12 @@ private static Query getByteVectorSimilarityQuery( final float resultSimilarity, final Query filterQuery ) { - return new ByteVectorSimilarityQuery(fieldName, byteVector, resultSimilarity, filterQuery); + return new ByteVectorSimilarityQuery( + fieldName, + byteVector, + DEFAULT_LUCENE_RADIAL_SEARCH_TRAVERSAL_SIMILARITY_RATIO * resultSimilarity, + resultSimilarity, + filterQuery + ); } } diff --git a/src/main/java/org/opensearch/knn/jni/FaissService.java b/src/main/java/org/opensearch/knn/jni/FaissService.java index b59ac4bcf..53980bbb7 100644 --- a/src/main/java/org/opensearch/knn/jni/FaissService.java +++ b/src/main/java/org/opensearch/knn/jni/FaissService.java @@ -191,6 +191,28 @@ public static native KNNQueryResult[] queryIndexWithFilter( @Deprecated(since = "2.14.0", forRemoval = true) public static native long transferVectors(long vectorsPointer, float[][] trainingData); + /** + * Range search index with filter + * + * @param indexPointer pointer to index in memory + * @param queryVector vector to be used for query + * @param radius search within radius threshold + * @param indexMaxResultWindow maximum number of results to return + * @param filteredIds list of doc ids to include in the query result + * @param filterIdsType type of filter ids + * @param parentIds list of parent doc ids when the knn field is a nested field + * @return KNNQueryResult array of neighbors within radius + */ + public static native KNNQueryResult[] rangeSearchIndexWithFilter( + long indexPointer, + float[] queryVector, + float radius, + int indexMaxResultWindow, + long[] filteredIds, + int filterIdsType, + int[] parentIds + ); + /** * Range search index * @@ -198,7 +220,14 @@ public static native KNNQueryResult[] queryIndexWithFilter( * @param queryVector vector to be used for query * @param radius search within radius threshold * @param indexMaxResultWindow maximum number of results to return + * @param parentIds list of parent doc ids when the knn field is a nested field * @return KNNQueryResult array of neighbors within radius */ - public static native KNNQueryResult[] rangeSearchIndex(long indexPointer, float[] queryVector, float radius, int indexMaxResultWindow); + public static native KNNQueryResult[] rangeSearchIndex( + long indexPointer, + float[] queryVector, + float radius, + int indexMaxResultWindow, + int[] parentIds + ); } diff --git a/src/main/java/org/opensearch/knn/jni/JNIService.java b/src/main/java/org/opensearch/knn/jni/JNIService.java index e846f02d1..20c418819 100644 --- a/src/main/java/org/opensearch/knn/jni/JNIService.java +++ b/src/main/java/org/opensearch/knn/jni/JNIService.java @@ -271,6 +271,9 @@ public static long transferVectors(long vectorsPointer, float[][] trainingData) * @param radius search within radius threshold * @param knnEngine engine to query index * @param indexMaxResultWindow maximum number of results to return + * @param filteredIds list of doc ids to include in the query result + * @param filterIdsType how to filter ids: Batch or BitMap + * @param parentIds parent ids of the vectors * @return KNNQueryResult array of neighbors within radius */ public static KNNQueryResult[] radiusQueryIndex( @@ -278,10 +281,24 @@ public static KNNQueryResult[] radiusQueryIndex( float[] queryVector, float radius, KNNEngine knnEngine, - int indexMaxResultWindow + int indexMaxResultWindow, + long[] filteredIds, + int filterIdsType, + int[] parentIds ) { if (KNNEngine.FAISS == knnEngine) { - return FaissService.rangeSearchIndex(indexPointer, queryVector, radius, indexMaxResultWindow); + if (ArrayUtils.isNotEmpty(filteredIds)) { + return FaissService.rangeSearchIndexWithFilter( + indexPointer, + queryVector, + radius, + indexMaxResultWindow, + filteredIds, + filterIdsType, + parentIds + ); + } + return FaissService.rangeSearchIndex(indexPointer, queryVector, radius, indexMaxResultWindow, parentIds); } throw new IllegalArgumentException("RadiusQueryIndex not supported for provided engine"); } diff --git a/src/test/java/org/opensearch/knn/index/FaissIT.java b/src/test/java/org/opensearch/knn/index/FaissIT.java index 61b0461ef..2349bb8d0 100644 --- a/src/test/java/org/opensearch/knn/index/FaissIT.java +++ b/src/test/java/org/opensearch/knn/index/FaissIT.java @@ -24,6 +24,7 @@ import org.opensearch.client.ResponseException; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.index.query.QueryBuilders; +import org.opensearch.index.query.TermQueryBuilder; import org.opensearch.knn.KNNRestTestCase; import org.opensearch.knn.KNNResult; import org.opensearch.knn.TestUtils; @@ -34,6 +35,7 @@ import java.io.IOException; import java.net.URL; +import java.util.ArrayList; import java.util.Arrays; import java.util.HashSet; import java.util.List; @@ -55,12 +57,14 @@ import static org.opensearch.knn.common.KNNConstants.FP16_MAX_VALUE; import static org.opensearch.knn.common.KNNConstants.FP16_MIN_VALUE; import static org.opensearch.knn.common.KNNConstants.KNN_ENGINE; +import static org.opensearch.knn.common.KNNConstants.MAX_DISTANCE; import static org.opensearch.knn.common.KNNConstants.METHOD_ENCODER_PARAMETER; import static org.opensearch.knn.common.KNNConstants.METHOD_HNSW; import static org.opensearch.knn.common.KNNConstants.METHOD_IVF; import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_NLIST; import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_NPROBES; import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_SPACE_TYPE; +import static org.opensearch.knn.common.KNNConstants.MIN_SCORE; import static org.opensearch.knn.common.KNNConstants.MODEL_ID; import static org.opensearch.knn.common.KNNConstants.NAME; import static org.opensearch.knn.common.KNNConstants.PARAMETERS; @@ -91,7 +95,7 @@ public void testEndToEnd_whenMethodIsHNSWFlat_thenSucceed() { String indexName = "test-index-1"; String fieldName = "test-field-1"; - KNNMethod hnswMethod = KNNEngine.FAISS.getMethod(KNNConstants.METHOD_HNSW); + KNNMethod hnswMethod = KNNEngine.FAISS.getMethod(METHOD_HNSW); SpaceType spaceType = SpaceType.L2; List mValues = ImmutableList.of(16, 32, 64, 128); @@ -108,10 +112,10 @@ public void testEndToEnd_whenMethodIsHNSWFlat_thenSucceed() { .field("type", "knn_vector") .field("dimension", dimension) .startObject(KNNConstants.KNN_METHOD) - .field(KNNConstants.NAME, hnswMethod.getMethodComponent().getName()) - .field(KNNConstants.METHOD_PARAMETER_SPACE_TYPE, spaceType.getValue()) - .field(KNNConstants.KNN_ENGINE, KNNEngine.FAISS.getName()) - .startObject(KNNConstants.PARAMETERS) + .field(NAME, hnswMethod.getMethodComponent().getName()) + .field(METHOD_PARAMETER_SPACE_TYPE, spaceType.getValue()) + .field(KNN_ENGINE, KNNEngine.FAISS.getName()) + .startObject(PARAMETERS) .field(KNNConstants.METHOD_PARAMETER_M, mValues.get(random().nextInt(mValues.size()))) .field(KNNConstants.METHOD_PARAMETER_EF_CONSTRUCTION, efConstructionValues.get(random().nextInt(efConstructionValues.size()))) .field(KNNConstants.METHOD_PARAMETER_EF_SEARCH, efSearchValues.get(random().nextInt(efSearchValues.size()))) @@ -180,7 +184,7 @@ public void testEndToEnd_whenMethodIsHNSWFlatAndHasDeletedDocs_thenSucceed() { String indexName = "test-index-1"; String fieldName = "test-field-1"; - KNNMethod hnswMethod = KNNEngine.FAISS.getMethod(KNNConstants.METHOD_HNSW); + KNNMethod hnswMethod = KNNEngine.FAISS.getMethod(METHOD_HNSW); SpaceType spaceType = SpaceType.L2; List mValues = ImmutableList.of(16, 32, 64, 128); @@ -197,10 +201,10 @@ public void testEndToEnd_whenMethodIsHNSWFlatAndHasDeletedDocs_thenSucceed() { .field("type", "knn_vector") .field("dimension", dimension) .startObject(KNNConstants.KNN_METHOD) - .field(KNNConstants.NAME, hnswMethod.getMethodComponent().getName()) - .field(KNNConstants.METHOD_PARAMETER_SPACE_TYPE, spaceType.getValue()) - .field(KNNConstants.KNN_ENGINE, KNNEngine.FAISS.getName()) - .startObject(KNNConstants.PARAMETERS) + .field(NAME, hnswMethod.getMethodComponent().getName()) + .field(METHOD_PARAMETER_SPACE_TYPE, spaceType.getValue()) + .field(KNN_ENGINE, KNNEngine.FAISS.getName()) + .startObject(PARAMETERS) .field(KNNConstants.METHOD_PARAMETER_M, mValues.get(random().nextInt(mValues.size()))) .field(KNNConstants.METHOD_PARAMETER_EF_CONSTRUCTION, efConstructionValues.get(random().nextInt(efConstructionValues.size()))) .field(KNNConstants.METHOD_PARAMETER_EF_SEARCH, efSearchValues.get(random().nextInt(efSearchValues.size()))) @@ -279,7 +283,7 @@ public void testEndToEnd_whenMethodIsHNSWFlatAndHasDeletedDocs_thenSucceed() { @SneakyThrows public void testEndToEnd_whenDoRadiusSearch_whenDistanceThreshold_whenMethodIsHNSWFlat_thenSucceed() { - KNNMethod hnswMethod = KNNEngine.FAISS.getMethod(KNNConstants.METHOD_HNSW); + KNNMethod hnswMethod = KNNEngine.FAISS.getMethod(METHOD_HNSW); SpaceType spaceType = SpaceType.L2; List mValues = ImmutableList.of(16, 32, 64, 128); @@ -296,10 +300,10 @@ public void testEndToEnd_whenDoRadiusSearch_whenDistanceThreshold_whenMethodIsHN .field("type", "knn_vector") .field("dimension", dimension) .startObject(KNNConstants.KNN_METHOD) - .field(KNNConstants.NAME, hnswMethod.getMethodComponent().getName()) - .field(KNNConstants.METHOD_PARAMETER_SPACE_TYPE, spaceType.getValue()) - .field(KNNConstants.KNN_ENGINE, KNNEngine.FAISS.getName()) - .startObject(KNNConstants.PARAMETERS) + .field(NAME, hnswMethod.getMethodComponent().getName()) + .field(METHOD_PARAMETER_SPACE_TYPE, spaceType.getValue()) + .field(KNN_ENGINE, KNNEngine.FAISS.getName()) + .startObject(PARAMETERS) .field(KNNConstants.METHOD_PARAMETER_M, mValues.get(random().nextInt(mValues.size()))) .field(KNNConstants.METHOD_PARAMETER_EF_CONSTRUCTION, efConstructionValues.get(random().nextInt(efConstructionValues.size()))) .field(KNNConstants.METHOD_PARAMETER_EF_SEARCH, efSearchValues.get(random().nextInt(efSearchValues.size()))) @@ -330,7 +334,7 @@ public void testEndToEnd_whenDoRadiusSearch_whenDistanceThreshold_whenMethodIsHN assertEquals(testData.indexData.docs.length, getDocCount(INDEX_NAME)); float distance = 300000000000f; - validateRadiusSearchResults(INDEX_NAME, FIELD_NAME, testData.queries, distance, null, spaceType); + validateRadiusSearchResults(INDEX_NAME, FIELD_NAME, testData.queries, distance, null, spaceType, null); // Delete index deleteKNNIndex(INDEX_NAME); @@ -338,7 +342,7 @@ public void testEndToEnd_whenDoRadiusSearch_whenDistanceThreshold_whenMethodIsHN @SneakyThrows public void testEndToEnd_whenDoRadiusSearch_whenScoreThreshold_whenMethodIsHNSWFlat_thenSucceed() { - KNNMethod hnswMethod = KNNEngine.FAISS.getMethod(KNNConstants.METHOD_HNSW); + KNNMethod hnswMethod = KNNEngine.FAISS.getMethod(METHOD_HNSW); SpaceType spaceType = SpaceType.L2; List mValues = ImmutableList.of(16, 32, 64, 128); @@ -355,10 +359,10 @@ public void testEndToEnd_whenDoRadiusSearch_whenScoreThreshold_whenMethodIsHNSWF .field("type", "knn_vector") .field("dimension", dimension) .startObject(KNNConstants.KNN_METHOD) - .field(KNNConstants.NAME, hnswMethod.getMethodComponent().getName()) - .field(KNNConstants.METHOD_PARAMETER_SPACE_TYPE, spaceType.getValue()) - .field(KNNConstants.KNN_ENGINE, KNNEngine.FAISS.getName()) - .startObject(KNNConstants.PARAMETERS) + .field(NAME, hnswMethod.getMethodComponent().getName()) + .field(METHOD_PARAMETER_SPACE_TYPE, spaceType.getValue()) + .field(KNN_ENGINE, KNNEngine.FAISS.getName()) + .startObject(PARAMETERS) .field(KNNConstants.METHOD_PARAMETER_M, mValues.get(random().nextInt(mValues.size()))) .field(KNNConstants.METHOD_PARAMETER_EF_CONSTRUCTION, efConstructionValues.get(random().nextInt(efConstructionValues.size()))) .field(KNNConstants.METHOD_PARAMETER_EF_SEARCH, efSearchValues.get(random().nextInt(efSearchValues.size()))) @@ -390,7 +394,7 @@ public void testEndToEnd_whenDoRadiusSearch_whenScoreThreshold_whenMethodIsHNSWF float score = 0.00001f; - validateRadiusSearchResults(INDEX_NAME, FIELD_NAME, testData.queries, null, score, spaceType); + validateRadiusSearchResults(INDEX_NAME, FIELD_NAME, testData.queries, null, score, spaceType, null); // Delete index deleteKNNIndex(INDEX_NAME); @@ -398,7 +402,7 @@ public void testEndToEnd_whenDoRadiusSearch_whenScoreThreshold_whenMethodIsHNSWF @SneakyThrows public void testEndToEnd_whenDoRadiusSearch_whenMoreThanOneScoreThreshold_whenMethodIsHNSWFlat_thenSucceed() { - KNNMethod hnswMethod = KNNEngine.FAISS.getMethod(KNNConstants.METHOD_HNSW); + KNNMethod hnswMethod = KNNEngine.FAISS.getMethod(METHOD_HNSW); SpaceType spaceType = SpaceType.INNER_PRODUCT; List mValues = ImmutableList.of(16, 32, 64, 128); @@ -415,10 +419,10 @@ public void testEndToEnd_whenDoRadiusSearch_whenMoreThanOneScoreThreshold_whenMe .field("type", "knn_vector") .field("dimension", dimension) .startObject(KNNConstants.KNN_METHOD) - .field(KNNConstants.NAME, hnswMethod.getMethodComponent().getName()) - .field(KNNConstants.METHOD_PARAMETER_SPACE_TYPE, spaceType.getValue()) - .field(KNNConstants.KNN_ENGINE, KNNEngine.FAISS.getName()) - .startObject(KNNConstants.PARAMETERS) + .field(NAME, hnswMethod.getMethodComponent().getName()) + .field(METHOD_PARAMETER_SPACE_TYPE, spaceType.getValue()) + .field(KNN_ENGINE, KNNEngine.FAISS.getName()) + .startObject(PARAMETERS) .field(KNNConstants.METHOD_PARAMETER_M, mValues.get(random().nextInt(mValues.size()))) .field(KNNConstants.METHOD_PARAMETER_EF_CONSTRUCTION, efConstructionValues.get(random().nextInt(efConstructionValues.size()))) .field(KNNConstants.METHOD_PARAMETER_EF_SEARCH, efSearchValues.get(random().nextInt(efSearchValues.size()))) @@ -450,14 +454,14 @@ public void testEndToEnd_whenDoRadiusSearch_whenMoreThanOneScoreThreshold_whenMe float score = 5f; - validateRadiusSearchResults(INDEX_NAME, FIELD_NAME, testData.queries, null, score, spaceType); + validateRadiusSearchResults(INDEX_NAME, FIELD_NAME, testData.queries, null, score, spaceType, null); // Delete index deleteKNNIndex(INDEX_NAME); } @SneakyThrows - public void testEndToEnd_whenDoRadiusSearch__whenDistanceThreshold_whenMethodIsHNSWPQ_thenSucceed() { + public void testEndToEnd_whenDoRadiusSearch_whenDistanceThreshold_whenMethodIsHNSWPQ_thenSucceed() { String indexName = "test-index"; String fieldName = "test-field"; String trainingIndexName = "training-index"; @@ -535,7 +539,7 @@ public void testEndToEnd_whenDoRadiusSearch__whenDistanceThreshold_whenMethodIsH float distance = 300000000000f; - validateRadiusSearchResults(indexName, fieldName, testData.queries, distance, null, spaceType); + validateRadiusSearchResults(indexName, fieldName, testData.queries, distance, null, spaceType, null); // Delete index deleteKNNIndex(indexName); @@ -554,6 +558,32 @@ public void testEndToEnd_whenDoRadiusSearch__whenDistanceThreshold_whenMethodIsH fail("Graphs are not getting evicted"); } + @SneakyThrows + public void testRadialQuery_withFilter_thenSuccess() { + setupKNNIndexForFilterQuery(); + + final float[][] searchVector = new float[][] { { 3.3f, 3.0f, 5.0f } }; + TermQueryBuilder termQueryBuilder = QueryBuilders.termQuery("color", "red"); + List expectedDocIds = Arrays.asList(DOC_ID_3); + + float distance = 15f; + List> queryResult = validateRadiusSearchResults( + INDEX_NAME, + FIELD_NAME, + searchVector, + distance, + null, + SpaceType.L2, + termQueryBuilder + ); + + assertEquals(1, queryResult.get(0).size()); + assertEquals(expectedDocIds.get(0), queryResult.get(0).get(0).getDocId()); + + // Delete index + deleteKNNIndex(INDEX_NAME); + } + @SneakyThrows public void testEndToEnd_whenMethodIsHNSWPQ_thenSucceed() { String indexName = "test-index"; @@ -670,7 +700,7 @@ public void testHNSWSQFP16_whenIndexedAndQueried_thenSucceed() { String indexName = "test-index-hnsw-sqfp16"; String fieldName = "test-field-hnsw-sqfp16"; - KNNMethod hnswMethod = KNNEngine.FAISS.getMethod(KNNConstants.METHOD_HNSW); + KNNMethod hnswMethod = KNNEngine.FAISS.getMethod(METHOD_HNSW); SpaceType[] spaceTypes = { SpaceType.L2, SpaceType.INNER_PRODUCT }; Random random = new Random(); SpaceType spaceType = spaceTypes[random.nextInt(spaceTypes.length)]; @@ -690,10 +720,10 @@ public void testHNSWSQFP16_whenIndexedAndQueried_thenSucceed() { .field("type", "knn_vector") .field("dimension", dimension) .startObject(KNNConstants.KNN_METHOD) - .field(KNNConstants.NAME, hnswMethod.getMethodComponent().getName()) - .field(KNNConstants.METHOD_PARAMETER_SPACE_TYPE, spaceType.getValue()) - .field(KNNConstants.KNN_ENGINE, KNNEngine.FAISS.getName()) - .startObject(KNNConstants.PARAMETERS) + .field(NAME, hnswMethod.getMethodComponent().getName()) + .field(METHOD_PARAMETER_SPACE_TYPE, spaceType.getValue()) + .field(KNN_ENGINE, KNNEngine.FAISS.getName()) + .startObject(PARAMETERS) .field(KNNConstants.METHOD_PARAMETER_M, mValues.get(random().nextInt(mValues.size()))) .field(KNNConstants.METHOD_PARAMETER_EF_CONSTRUCTION, efConstructionValues.get(random().nextInt(efConstructionValues.size()))) .field(KNNConstants.METHOD_PARAMETER_EF_SEARCH, efSearchValues.get(random().nextInt(efSearchValues.size()))) @@ -784,7 +814,7 @@ public void testHNSWSQFP16_whenIndexedWithOutOfFP16Range_thenThrowException() { String indexName = "test-index-sqfp16"; String fieldName = "test-field-sqfp16"; - KNNMethod hnswMethod = KNNEngine.FAISS.getMethod(KNNConstants.METHOD_HNSW); + KNNMethod hnswMethod = KNNEngine.FAISS.getMethod(METHOD_HNSW); SpaceType[] spaceTypes = { SpaceType.L2, SpaceType.INNER_PRODUCT }; Random random = new Random(); SpaceType spaceType = spaceTypes[random.nextInt(spaceTypes.length)]; @@ -803,10 +833,10 @@ public void testHNSWSQFP16_whenIndexedWithOutOfFP16Range_thenThrowException() { .field("type", "knn_vector") .field("dimension", dimension) .startObject(KNNConstants.KNN_METHOD) - .field(KNNConstants.NAME, hnswMethod.getMethodComponent().getName()) - .field(KNNConstants.METHOD_PARAMETER_SPACE_TYPE, spaceType.getValue()) - .field(KNNConstants.KNN_ENGINE, KNNEngine.FAISS.getName()) - .startObject(KNNConstants.PARAMETERS) + .field(NAME, hnswMethod.getMethodComponent().getName()) + .field(METHOD_PARAMETER_SPACE_TYPE, spaceType.getValue()) + .field(KNN_ENGINE, KNNEngine.FAISS.getName()) + .startObject(PARAMETERS) .field(KNNConstants.METHOD_PARAMETER_M, mValues.get(random().nextInt(mValues.size()))) .field(KNNConstants.METHOD_PARAMETER_EF_CONSTRUCTION, efConstructionValues.get(random().nextInt(efConstructionValues.size()))) .field(KNNConstants.METHOD_PARAMETER_EF_SEARCH, efSearchValues.get(random().nextInt(efSearchValues.size()))) @@ -886,7 +916,7 @@ public void testHNSWSQFP16_whenClipToFp16isTrueAndIndexedWithOutOfFP16Range_then String indexName = "test-index-sqfp16-clip-fp16"; String fieldName = "test-field-sqfp16"; - KNNMethod hnswMethod = KNNEngine.FAISS.getMethod(KNNConstants.METHOD_HNSW); + KNNMethod hnswMethod = KNNEngine.FAISS.getMethod(METHOD_HNSW); Random random = new Random(); List mValues = ImmutableList.of(16, 32, 64, 128); @@ -903,10 +933,10 @@ public void testHNSWSQFP16_whenClipToFp16isTrueAndIndexedWithOutOfFP16Range_then .field("type", "knn_vector") .field("dimension", dimension) .startObject(KNNConstants.KNN_METHOD) - .field(KNNConstants.NAME, hnswMethod.getMethodComponent().getName()) - .field(KNNConstants.METHOD_PARAMETER_SPACE_TYPE, SpaceType.L2.getValue()) - .field(KNNConstants.KNN_ENGINE, KNNEngine.FAISS.getName()) - .startObject(KNNConstants.PARAMETERS) + .field(NAME, hnswMethod.getMethodComponent().getName()) + .field(METHOD_PARAMETER_SPACE_TYPE, SpaceType.L2.getValue()) + .field(KNN_ENGINE, KNNEngine.FAISS.getName()) + .startObject(PARAMETERS) .field(KNNConstants.METHOD_PARAMETER_M, mValues.get(random().nextInt(mValues.size()))) .field(KNNConstants.METHOD_PARAMETER_EF_CONSTRUCTION, efConstructionValues.get(random().nextInt(efConstructionValues.size()))) .field(KNNConstants.METHOD_PARAMETER_EF_SEARCH, efSearchValues.get(random().nextInt(efSearchValues.size()))) @@ -1351,7 +1381,7 @@ public void testDocUpdate() throws IOException { String fieldName = "test-field-1"; Integer dimension = 2; - KNNMethod hnswMethod = KNNEngine.FAISS.getMethod(KNNConstants.METHOD_HNSW); + KNNMethod hnswMethod = KNNEngine.FAISS.getMethod(METHOD_HNSW); SpaceType spaceType = SpaceType.L2; // Create an index @@ -1362,9 +1392,9 @@ public void testDocUpdate() throws IOException { .field("type", "knn_vector") .field("dimension", dimension) .startObject(KNNConstants.KNN_METHOD) - .field(KNNConstants.NAME, hnswMethod.getMethodComponent().getName()) - .field(KNNConstants.METHOD_PARAMETER_SPACE_TYPE, spaceType.getValue()) - .field(KNNConstants.KNN_ENGINE, KNNEngine.FAISS.getName()) + .field(NAME, hnswMethod.getMethodComponent().getName()) + .field(METHOD_PARAMETER_SPACE_TYPE, spaceType.getValue()) + .field(KNN_ENGINE, KNNEngine.FAISS.getName()) .endObject() .endObject() .endObject() @@ -1387,7 +1417,7 @@ public void testDocDeletion() throws IOException { String fieldName = "test-field-1"; Integer dimension = 2; - KNNMethod hnswMethod = KNNEngine.FAISS.getMethod(KNNConstants.METHOD_HNSW); + KNNMethod hnswMethod = KNNEngine.FAISS.getMethod(METHOD_HNSW); SpaceType spaceType = SpaceType.L2; // Create an index @@ -1398,9 +1428,9 @@ public void testDocDeletion() throws IOException { .field("type", "knn_vector") .field("dimension", dimension) .startObject(KNNConstants.KNN_METHOD) - .field(KNNConstants.NAME, hnswMethod.getMethodComponent().getName()) - .field(KNNConstants.METHOD_PARAMETER_SPACE_TYPE, spaceType.getValue()) - .field(KNNConstants.KNN_ENGINE, KNNEngine.FAISS.getName()) + .field(NAME, hnswMethod.getMethodComponent().getName()) + .field(METHOD_PARAMETER_SPACE_TYPE, spaceType.getValue()) + .field(KNN_ENGINE, KNNEngine.FAISS.getName()) .endObject() .endObject() .endObject() @@ -1574,9 +1604,9 @@ public void testFiltering_whenUsingFaissExactSearchWithIP_thenMatchExpectedScore .field("type", "knn_vector") .field("dimension", 2) .startObject(KNNConstants.KNN_METHOD) - .field(KNNConstants.NAME, KNNEngine.FAISS.getMethod(KNNConstants.METHOD_HNSW).getMethodComponent().getName()) - .field(KNNConstants.METHOD_PARAMETER_SPACE_TYPE, SpaceType.INNER_PRODUCT.getValue()) - .field(KNNConstants.KNN_ENGINE, KNNEngine.FAISS.getName()) + .field(NAME, KNNEngine.FAISS.getMethod(METHOD_HNSW).getMethodComponent().getName()) + .field(METHOD_PARAMETER_SPACE_TYPE, SpaceType.INNER_PRODUCT.getValue()) + .field(KNN_ENGINE, KNNEngine.FAISS.getName()) .endObject() .endObject() .endObject() @@ -1625,9 +1655,9 @@ protected void setupKNNIndexForFilterQuery() throws Exception { .field("type", "knn_vector") .field("dimension", 3) .startObject(KNNConstants.KNN_METHOD) - .field(KNNConstants.NAME, KNNEngine.FAISS.getMethod(KNNConstants.METHOD_HNSW).getMethodComponent().getName()) - .field(KNNConstants.METHOD_PARAMETER_SPACE_TYPE, SpaceType.L2) - .field(KNNConstants.KNN_ENGINE, KNNEngine.FAISS.getName()) + .field(NAME, KNNEngine.FAISS.getMethod(METHOD_HNSW).getMethodComponent().getName()) + .field(METHOD_PARAMETER_SPACE_TYPE, SpaceType.L2) + .field(KNN_ENGINE, KNNEngine.FAISS.getName()) .endObject() .endObject() .endObject() @@ -1686,26 +1716,31 @@ private void validateGraphEviction() throws Exception { fail("Graphs are not getting evicted"); } - private void validateRadiusSearchResults( + private List> validateRadiusSearchResults( String indexName, String fieldName, float[][] queryVectors, Float distanceThreshold, Float scoreThreshold, - final SpaceType spaceType + final SpaceType spaceType, + TermQueryBuilder filterQuery ) throws IOException, ParseException { + List> queryResults = new ArrayList<>(); for (float[] queryVector : queryVectors) { XContentBuilder queryBuilder = XContentFactory.jsonBuilder().startObject().startObject("query"); queryBuilder.startObject("knn"); queryBuilder.startObject(fieldName); queryBuilder.field("vector", queryVector); if (distanceThreshold != null) { - queryBuilder.field("max_distance", distanceThreshold); + queryBuilder.field(MAX_DISTANCE, distanceThreshold); } else if (scoreThreshold != null) { - queryBuilder.field("min_score", scoreThreshold); + queryBuilder.field(MIN_SCORE, scoreThreshold); } else { throw new IllegalArgumentException("Invalid threshold"); } + if (filterQuery != null) { + queryBuilder.field("filter", filterQuery); + } queryBuilder.endObject(); queryBuilder.endObject(); queryBuilder.endObject().endObject(); @@ -1724,6 +1759,8 @@ private void validateRadiusSearchResults( throw new IllegalArgumentException("Invalid space type"); } } + queryResults.add(knnResults); } + return queryResults; } } diff --git a/src/test/java/org/opensearch/knn/index/LuceneEngineIT.java b/src/test/java/org/opensearch/knn/index/LuceneEngineIT.java index 4cc856613..bf9a6b776 100644 --- a/src/test/java/org/opensearch/knn/index/LuceneEngineIT.java +++ b/src/test/java/org/opensearch/knn/index/LuceneEngineIT.java @@ -34,7 +34,9 @@ import java.util.function.Function; import java.util.stream.Collectors; +import static org.opensearch.knn.common.KNNConstants.MAX_DISTANCE; import static org.opensearch.knn.common.KNNConstants.METHOD_HNSW; +import static org.opensearch.knn.common.KNNConstants.MIN_SCORE; import static org.opensearch.knn.common.KNNConstants.NMSLIB_NAME; import static org.opensearch.knn.common.KNNConstants.VECTOR_DATA_TYPE_FIELD; @@ -629,9 +631,9 @@ private void validateRadiusSearchResults( builder.startObject(FIELD_NAME); builder.field("vector", searchVectors[i]); if (distanceThreshold != null) { - builder.field("max_distance", distanceThreshold); + builder.field(MAX_DISTANCE, distanceThreshold); } else if (scoreThreshold != null) { - builder.field("min_score", scoreThreshold); + builder.field(MIN_SCORE, scoreThreshold); } else { throw new IllegalArgumentException("Either distance or score must be provided"); } diff --git a/src/test/java/org/opensearch/knn/index/NestedSearchIT.java b/src/test/java/org/opensearch/knn/index/NestedSearchIT.java index e4d828a01..d97aa4d40 100644 --- a/src/test/java/org/opensearch/knn/index/NestedSearchIT.java +++ b/src/test/java/org/opensearch/knn/index/NestedSearchIT.java @@ -32,6 +32,7 @@ import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_EF_CONSTRUCTION; import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_M; import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_SPACE_TYPE; +import static org.opensearch.knn.common.KNNConstants.MIN_SCORE; import static org.opensearch.knn.common.KNNConstants.NAME; import static org.opensearch.knn.common.KNNConstants.PARAMETERS; import static org.opensearch.knn.common.KNNConstants.PATH; @@ -152,7 +153,64 @@ public void testNestedSearchWithFaiss_whenDoingExactSearch_thenReturnCorrectResu updateIndexSettings(INDEX_NAME, Settings.builder().put(KNNSettings.ADVANCED_FILTERED_EXACT_SEARCH_THRESHOLD, 100)); Float[] queryVector = { 3f, 3f, 3f }; - Response response = queryNestedField(INDEX_NAME, 3, queryVector, FIELD_NAME_PARKING, FIELD_VALUE_TRUE); + Response response = queryNestedField(INDEX_NAME, 3, queryVector, FIELD_NAME_PARKING, FIELD_VALUE_TRUE, null); + String entity = EntityUtils.toString(response.getEntity()); + List docIds = parseIds(entity); + assertEquals(2, docIds.size()); + assertEquals("3", docIds.get(0)); + assertEquals("1", docIds.get(1)); + assertEquals(2, parseTotalSearchHits(entity)); + } + + /** + * { + * "query": { + * "nested": { + * "path": "test_nested", + * "query": { + * "knn": { + * "test_nested.test_vector": { + * "vector": [ + * 1, 1, 1 + * ], + * "min_score": 0.00001, + * "filter": { + * "term": { + * "parking": "true" + * } + * } + * } + * } + * } + * } + * } + * } + * + */ + @SneakyThrows + public void testNestedWithFaiss_whenFilter_whenDoRadialSearch_thenReturnCorrectResults() { + createKnnIndex(3, KNNEngine.FAISS.getName()); + + for (int i = 1; i < 4; i++) { + float value = (float) i; + String doc = NestedKnnDocBuilder.create(FIELD_NAME_NESTED) + .addVectors( + FIELD_NAME_VECTOR, + new Float[] { value, value, value }, + new Float[] { value, value, value }, + new Float[] { value, value, value } + ) + .addTopLevelField(FIELD_NAME_PARKING, i % 2 == 1 ? FIELD_VALUE_TRUE : FIELD_VALUE_FALSE) + .build(); + addKnnDoc(INDEX_NAME, String.valueOf(i), doc); + } + refreshIndex(INDEX_NAME); + forceMergeKnnIndex(INDEX_NAME); + + Float[] queryVector = { 3f, 3f, 3f }; + Float minScore = 0.00001f; + Response response = queryNestedField(INDEX_NAME, null, queryVector, FIELD_NAME_PARKING, FIELD_VALUE_TRUE, minScore); + String entity = EntityUtils.toString(response.getEntity()); List docIds = parseIds(entity); assertEquals(2, docIds.size()); @@ -215,22 +273,29 @@ private void createKnnIndex(final int dimension, final String engine) throws Exc } private Response queryNestedField(final String index, final int k, final Object[] vector) throws IOException { - return queryNestedField(index, k, vector, null, null); + return queryNestedField(index, k, vector, null, null, null); } private Response queryNestedField( final String index, - final int k, + final Integer k, final Object[] vector, final String filterName, - final String filterValue + final String filterValue, + final Float minScore ) throws IOException { XContentBuilder builder = XContentFactory.jsonBuilder().startObject().startObject(QUERY); builder.startObject(TYPE_NESTED); builder.field(PATH, FIELD_NAME_NESTED); builder.startObject(QUERY).startObject(KNN).startObject(FIELD_NAME_NESTED + "." + FIELD_NAME_VECTOR); builder.field(VECTOR, vector); - builder.field(K, k); + if (minScore != null) { + builder.field(MIN_SCORE, minScore); + } else if (k != null) { + builder.field(K, k); + } else { + throw new IllegalArgumentException("k or minScore must be provided in the query"); + } if (filterName != null && filterValue != null) { builder.startObject(FIELD_FILTER); builder.startObject(FIELD_TERM); diff --git a/src/test/java/org/opensearch/knn/index/query/KNNQueryBuilderTests.java b/src/test/java/org/opensearch/knn/index/query/KNNQueryBuilderTests.java index ddc961093..4b9872131 100644 --- a/src/test/java/org/opensearch/knn/index/query/KNNQueryBuilderTests.java +++ b/src/test/java/org/opensearch/knn/index/query/KNNQueryBuilderTests.java @@ -53,6 +53,7 @@ import static org.mockito.Mockito.anyString; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; +import static org.opensearch.knn.common.KNNConstants.DEFAULT_LUCENE_RADIAL_SEARCH_TRAVERSAL_SIMILARITY_RATIO; import static org.opensearch.knn.common.KNNConstants.METHOD_HNSW; import static org.opensearch.knn.index.KNNClusterTestUtils.mockClusterService; import static org.opensearch.knn.index.util.KNNEngine.ENGINES_SUPPORTING_RADIAL_SEARCH; @@ -417,7 +418,12 @@ public void testDoToQuery_whenNormal_whenDoRadiusSearch_whenDistanceThreshold_th KNNMethodContext knnMethodContext = new KNNMethodContext(KNNEngine.LUCENE, SpaceType.L2, methodComponentContext); when(mockKNNVectorField.getKnnMethodContext()).thenReturn(knnMethodContext); FloatVectorSimilarityQuery query = (FloatVectorSimilarityQuery) knnQueryBuilder.doToQuery(mockQueryShardContext); - assertTrue(query.toString().contains("resultSimilarity=" + KNNEngine.LUCENE.distanceToRadialThreshold(MAX_DISTANCE, SpaceType.L2))); + float resultSimilarity = KNNEngine.LUCENE.distanceToRadialThreshold(MAX_DISTANCE, SpaceType.L2); + + assertTrue(query.toString().contains("resultSimilarity=" + resultSimilarity)); + assertTrue( + query.toString().contains("traversalSimilarity=" + DEFAULT_LUCENE_RADIAL_SEARCH_TRAVERSAL_SIMILARITY_RATIO * resultSimilarity) + ); } public void testDoToQuery_whenNormal_whenDoRadiusSearch_whenScoreThreshold_thenSucceed() { diff --git a/src/test/java/org/opensearch/knn/index/query/KNNWeightTests.java b/src/test/java/org/opensearch/knn/index/query/KNNWeightTests.java index e80190ff8..0d15b5f5f 100644 --- a/src/test/java/org/opensearch/knn/index/query/KNNWeightTests.java +++ b/src/test/java/org/opensearch/knn/index/query/KNNWeightTests.java @@ -699,8 +699,9 @@ public void testDoANNSearch_whenRadialIsDefined_thenCallJniRadiusQueryIndex() { final float[] queryVector = new float[] { 0.1f, 0.3f }; final float radius = 0.5f; final int maxResults = 1000; - jniServiceMockedStatic.when(() -> JNIService.radiusQueryIndex(anyLong(), any(), anyFloat(), any(), anyInt())) - .thenReturn(getKNNQueryResults()); + jniServiceMockedStatic.when( + () -> JNIService.radiusQueryIndex(anyLong(), any(), anyFloat(), any(), anyInt(), any(), anyInt(), any()) + ).thenReturn(getKNNQueryResults()); KNNQuery.Context context = mock(KNNQuery.Context.class); when(context.getMaxResultWindow()).thenReturn(maxResults); @@ -742,7 +743,9 @@ public void testDoANNSearch_whenRadialIsDefined_thenCallJniRadiusQueryIndex() { final KNNScorer knnScorer = (KNNScorer) knnWeight.scorer(leafReaderContext); assertNotNull(knnScorer); - jniServiceMockedStatic.verify(() -> JNIService.radiusQueryIndex(anyLong(), any(), anyFloat(), any(), anyInt())); + jniServiceMockedStatic.verify( + () -> JNIService.radiusQueryIndex(anyLong(), any(), anyFloat(), any(), anyInt(), any(), anyInt(), any()) + ); final DocIdSetIterator docIdSetIterator = knnScorer.iterator(); From 2ac49032f7cdf88695e8b6e9cd802ec2ffc28f58 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Mon, 29 Apr 2024 15:34:45 -0700 Subject: [PATCH 249/416] Fixing KNNCodecServiceTests as reported in the gh issue:[#1593] (#1601) (#1663) Signed-off-by: Navneet Verma (cherry picked from commit 5de10fc382c124fafd249c20ab35ecc79eae0792) Co-authored-by: Navneet Verma --- CHANGELOG.md | 2 +- .../org/opensearch/knn/index/codec/KNNCodecServiceTests.java | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c0dc0b523..2be38fa6b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ## [Unreleased 3.0](https://github.com/opensearch-project/k-NN/compare/2.x...HEAD) ### Features ### Enhancements -### Bug Fixes +### Bug Fixes ### Infrastructure ### Documentation ### Maintenance diff --git a/src/test/java/org/opensearch/knn/index/codec/KNNCodecServiceTests.java b/src/test/java/org/opensearch/knn/index/codec/KNNCodecServiceTests.java index 233b9adf7..dfe4e7f22 100644 --- a/src/test/java/org/opensearch/knn/index/codec/KNNCodecServiceTests.java +++ b/src/test/java/org/opensearch/knn/index/codec/KNNCodecServiceTests.java @@ -36,6 +36,7 @@ public void setUp() throws Exception { super.setUp(); IndexMetadata indexMetadata = mock(IndexMetadata.class); when(indexMetadata.getIndex()).thenReturn(new Index(TEST_INDEX, INDEX_UUID.toString())); + when(indexMetadata.getCustomData(IndexMetadata.REMOTE_STORE_CUSTOM_KEY)).thenReturn(null); when(indexMetadata.getSettings()).thenReturn(Settings.EMPTY); Settings settings = Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, Integer.toString(NUM_OF_SHARDS)).build(); indexSettings = new IndexSettings(indexMetadata, settings); From bfe46000efe3eab258169c02122b0e917a4eb572 Mon Sep 17 00:00:00 2001 From: Ryan Bogan Date: Mon, 29 Apr 2024 18:06:04 -0700 Subject: [PATCH 250/416] Enable script score to work with model based indices (#1661) Signed-off-by: Ryan Bogan --- CHANGELOG.md | 1 + .../mapper/KNNVectorFieldMapperUtil.java | 58 +++++++++++++++++ .../org/opensearch/knn/plugin/KNNPlugin.java | 2 + .../knn/plugin/script/KNNScoringSpace.java | 12 ++-- .../plugin/script/KNNScoringSpaceUtil.java | 3 + .../mapper/KNNVectorFieldMapperUtilTests.java | 63 +++++++++++++++++++ .../knn/plugin/script/KNNScriptScoringIT.java | 54 ++++++++++++++++ 7 files changed, 188 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2be38fa6b..8f2721810 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Bug Fixes * Add stored fields for knn_vector type [#1630](https://github.com/opensearch-project/k-NN/pull/1630) * Ensure ignore_unmapped is set to correct value [#1655](https://github.com/opensearch-project/k-NN/pull/1655) +* Enable script score to work with model based indices [#1649](https://github.com/opensearch-project/k-NN/pull/1649) ### Infrastructure * Add micro-benchmark module in k-NN plugin for benchmark streaming vectors to JNI layer functionality. [#1583](https://github.com/opensearch-project/k-NN/pull/1583) * Add arm64 check when SIMD is disabled [#1618](https://github.com/opensearch-project/k-NN/pull/1618) diff --git a/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperUtil.java b/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperUtil.java index 9b1578a45..8bd7eb6f2 100644 --- a/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperUtil.java +++ b/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperUtil.java @@ -21,6 +21,9 @@ import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.codec.util.KNNVectorSerializerFactory; import org.opensearch.knn.index.util.KNNEngine; +import org.opensearch.knn.indices.ModelDao; +import org.opensearch.knn.indices.ModelMetadata; +import org.opensearch.knn.indices.ModelUtil; import java.util.Arrays; import java.util.Locale; @@ -34,9 +37,22 @@ import static org.opensearch.knn.common.KNNConstants.VECTOR_DATA_TYPE_FIELD; import static org.opensearch.knn.common.KNNValidationUtil.validateFloatVectorValue; +/** + * Utility class for KNNVectorFieldMapper + */ @NoArgsConstructor(access = AccessLevel.PRIVATE) public class KNNVectorFieldMapperUtil { + private static ModelDao modelDao; + + /** + * Initializes static instance variables + * @param modelDao ModelDao object + */ + public static void initialize(final ModelDao modelDao) { + KNNVectorFieldMapperUtil.modelDao = modelDao; + } + /** * Validate the float vector value and throw exception if it is not a number or not in the finite range * or is not within the FP16 range of [-65504 to 65504]. @@ -171,4 +187,46 @@ public static Object deserializeStoredVector(BytesRef storedVector, VectorDataTy return vectorDataType.getVectorFromBytesRef(storedVector); } + + /** + * Get the expected dimensions from a specified knn vector field type. + * + * If the field is model-based, get dimensions from model metadata. + * @param knnVectorFieldType knn vector field type + * @return expected dimensions + */ + public static int getExpectedDimensions(final KNNVectorFieldMapper.KNNVectorFieldType knnVectorFieldType) { + int expectedDimensions = knnVectorFieldType.getDimension(); + if (isModelBasedIndex(expectedDimensions)) { + ModelMetadata modelMetadata = getModelMetadataForField(knnVectorFieldType); + expectedDimensions = modelMetadata.getDimension(); + } + return expectedDimensions; + } + + private static boolean isModelBasedIndex(int expectedDimensions) { + return expectedDimensions == -1; + } + + /** + * Returns the model metadata for a specified knn vector field + * + * @param knnVectorField knn vector field + * @return the model metadata from knnVectorField + */ + private static ModelMetadata getModelMetadataForField(final KNNVectorFieldMapper.KNNVectorFieldType knnVectorField) { + String modelId = knnVectorField.getModelId(); + + if (modelId == null) { + throw new IllegalArgumentException( + String.format("Field '%s' does not have model.", knnVectorField.getKnnMethodContext().getMethodComponentContext().getName()) + ); + } + + ModelMetadata modelMetadata = modelDao.getMetadata(modelId); + if (!ModelUtil.isModelCreated(modelMetadata)) { + throw new IllegalArgumentException(String.format("Model ID '%s' is not created.", modelId)); + } + return modelMetadata; + } } diff --git a/src/main/java/org/opensearch/knn/plugin/KNNPlugin.java b/src/main/java/org/opensearch/knn/plugin/KNNPlugin.java index 2e5a55092..f898b622e 100644 --- a/src/main/java/org/opensearch/knn/plugin/KNNPlugin.java +++ b/src/main/java/org/opensearch/knn/plugin/KNNPlugin.java @@ -14,6 +14,7 @@ import org.opensearch.indices.SystemIndexDescriptor; import org.opensearch.knn.index.KNNCircuitBreaker; import org.opensearch.knn.index.KNNClusterUtil; +import org.opensearch.knn.index.mapper.KNNVectorFieldMapperUtil; import org.opensearch.knn.index.query.KNNQueryBuilder; import org.opensearch.knn.index.KNNSettings; import org.opensearch.knn.index.mapper.KNNVectorFieldMapper; @@ -204,6 +205,7 @@ public Collection createComponents( TrainingJobClusterStateListener.initialize(threadPool, ModelDao.OpenSearchKNNModelDao.getInstance(), clusterService); KNNCircuitBreaker.getInstance().initialize(threadPool, clusterService, client); KNNQueryBuilder.initialize(ModelDao.OpenSearchKNNModelDao.getInstance()); + KNNVectorFieldMapperUtil.initialize(ModelDao.OpenSearchKNNModelDao.getInstance()); KNNWeight.initialize(ModelDao.OpenSearchKNNModelDao.getInstance()); TrainingModelRequest.initialize(ModelDao.OpenSearchKNNModelDao.getInstance(), clusterService); diff --git a/src/main/java/org/opensearch/knn/plugin/script/KNNScoringSpace.java b/src/main/java/org/opensearch/knn/plugin/script/KNNScoringSpace.java index 5a8cdb036..8105539ba 100644 --- a/src/main/java/org/opensearch/knn/plugin/script/KNNScoringSpace.java +++ b/src/main/java/org/opensearch/knn/plugin/script/KNNScoringSpace.java @@ -8,6 +8,7 @@ import org.apache.lucene.search.IndexSearcher; import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.mapper.KNNVectorFieldMapper; +import org.opensearch.knn.index.mapper.KNNVectorFieldMapperUtil; import org.opensearch.knn.index.query.KNNWeight; import org.apache.lucene.index.LeafReaderContext; import org.opensearch.index.mapper.MappedFieldType; @@ -28,6 +29,7 @@ import static org.opensearch.knn.plugin.script.KNNScoringSpaceUtil.parseToLong; public interface KNNScoringSpace { + /** * Return the correct scoring script for a given query. The scoring script * @@ -60,7 +62,7 @@ public L2(Object query, MappedFieldType fieldType) { this.processedQuery = parseToFloatArray( query, - ((KNNVectorFieldMapper.KNNVectorFieldType) fieldType).getDimension(), + KNNVectorFieldMapperUtil.getExpectedDimensions((KNNVectorFieldMapper.KNNVectorFieldType) fieldType), ((KNNVectorFieldMapper.KNNVectorFieldType) fieldType).getVectorDataType() ); this.scoringMethod = (float[] q, float[] v) -> 1 / (1 + KNNScoringUtil.l2Squared(q, v)); @@ -96,7 +98,7 @@ public CosineSimilarity(Object query, MappedFieldType fieldType) { this.processedQuery = parseToFloatArray( query, - ((KNNVectorFieldMapper.KNNVectorFieldType) fieldType).getDimension(), + KNNVectorFieldMapperUtil.getExpectedDimensions((KNNVectorFieldMapper.KNNVectorFieldType) fieldType), ((KNNVectorFieldMapper.KNNVectorFieldType) fieldType).getVectorDataType() ); SpaceType.COSINESIMIL.validateVector(processedQuery); @@ -191,7 +193,7 @@ public L1(Object query, MappedFieldType fieldType) { this.processedQuery = parseToFloatArray( query, - ((KNNVectorFieldMapper.KNNVectorFieldType) fieldType).getDimension(), + KNNVectorFieldMapperUtil.getExpectedDimensions((KNNVectorFieldMapper.KNNVectorFieldType) fieldType), ((KNNVectorFieldMapper.KNNVectorFieldType) fieldType).getVectorDataType() ); this.scoringMethod = (float[] q, float[] v) -> 1 / (1 + KNNScoringUtil.l1Norm(q, v)); @@ -226,7 +228,7 @@ public LInf(Object query, MappedFieldType fieldType) { this.processedQuery = parseToFloatArray( query, - ((KNNVectorFieldMapper.KNNVectorFieldType) fieldType).getDimension(), + KNNVectorFieldMapperUtil.getExpectedDimensions((KNNVectorFieldMapper.KNNVectorFieldType) fieldType), ((KNNVectorFieldMapper.KNNVectorFieldType) fieldType).getVectorDataType() ); this.scoringMethod = (float[] q, float[] v) -> 1 / (1 + KNNScoringUtil.lInfNorm(q, v)); @@ -263,7 +265,7 @@ public InnerProd(Object query, MappedFieldType fieldType) { this.processedQuery = parseToFloatArray( query, - ((KNNVectorFieldMapper.KNNVectorFieldType) fieldType).getDimension(), + KNNVectorFieldMapperUtil.getExpectedDimensions((KNNVectorFieldMapper.KNNVectorFieldType) fieldType), ((KNNVectorFieldMapper.KNNVectorFieldType) fieldType).getVectorDataType() ); this.scoringMethod = (float[] q, float[] v) -> KNNWeight.normalizeScore(-KNNScoringUtil.innerProduct(q, v)); diff --git a/src/main/java/org/opensearch/knn/plugin/script/KNNScoringSpaceUtil.java b/src/main/java/org/opensearch/knn/plugin/script/KNNScoringSpaceUtil.java index c482413fb..889780d7a 100644 --- a/src/main/java/org/opensearch/knn/plugin/script/KNNScoringSpaceUtil.java +++ b/src/main/java/org/opensearch/knn/plugin/script/KNNScoringSpaceUtil.java @@ -19,6 +19,9 @@ import static org.opensearch.index.mapper.NumberFieldMapper.NumberType.LONG; import static org.opensearch.knn.common.KNNValidationUtil.validateByteVectorValue; +/** + * Utility class for KNNScoringSpace + */ public class KNNScoringSpaceUtil { /** diff --git a/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperUtilTests.java b/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperUtilTests.java index 3fa9f2363..ff47dcd69 100644 --- a/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperUtilTests.java +++ b/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperUtilTests.java @@ -13,12 +13,20 @@ import org.apache.lucene.document.StoredField; import org.opensearch.knn.KNNTestCase; +import org.opensearch.knn.index.KNNMethodContext; +import org.opensearch.knn.index.MethodComponentContext; import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.codec.util.KNNVectorSerializerFactory; +import org.opensearch.knn.indices.ModelDao; +import org.opensearch.knn.indices.ModelMetadata; +import org.opensearch.knn.indices.ModelState; import java.io.ByteArrayInputStream; import java.util.Arrays; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + public class KNNVectorFieldMapperUtilTests extends KNNTestCase { private static final String TEST_FIELD_NAME = "test_field_name"; @@ -51,4 +59,59 @@ public void testStoredFields_whenVectorIsFloatType_thenSucceed() { assertTrue(vector instanceof float[]); assertArrayEquals(TEST_FLOAT_VECTOR, (float[]) vector, 0.001f); } + + public void testGetExpectedDimensionsSuccess() { + KNNVectorFieldMapper.KNNVectorFieldType knnVectorFieldType = mock(KNNVectorFieldMapper.KNNVectorFieldType.class); + when(knnVectorFieldType.getDimension()).thenReturn(3); + + KNNVectorFieldMapper.KNNVectorFieldType knnVectorFieldTypeModelBased = mock(KNNVectorFieldMapper.KNNVectorFieldType.class); + when(knnVectorFieldTypeModelBased.getDimension()).thenReturn(-1); + String modelId = "test-model"; + when(knnVectorFieldTypeModelBased.getModelId()).thenReturn(modelId); + + ModelDao modelDao = mock(ModelDao.class); + ModelMetadata modelMetadata = mock(ModelMetadata.class); + when(modelMetadata.getState()).thenReturn(ModelState.CREATED); + when(modelMetadata.getDimension()).thenReturn(4); + when(modelDao.getMetadata(modelId)).thenReturn(modelMetadata); + + KNNVectorFieldMapperUtil.initialize(modelDao); + + assertEquals(3, KNNVectorFieldMapperUtil.getExpectedDimensions(knnVectorFieldType)); + assertEquals(4, KNNVectorFieldMapperUtil.getExpectedDimensions(knnVectorFieldTypeModelBased)); + } + + public void testGetExpectedDimensionsFailure() { + KNNVectorFieldMapper.KNNVectorFieldType knnVectorFieldTypeModelBased = mock(KNNVectorFieldMapper.KNNVectorFieldType.class); + when(knnVectorFieldTypeModelBased.getDimension()).thenReturn(-1); + String modelId = "test-model"; + when(knnVectorFieldTypeModelBased.getModelId()).thenReturn(modelId); + + ModelDao modelDao = mock(ModelDao.class); + ModelMetadata modelMetadata = mock(ModelMetadata.class); + when(modelMetadata.getState()).thenReturn(ModelState.TRAINING); + when(modelDao.getMetadata(modelId)).thenReturn(modelMetadata); + + KNNVectorFieldMapperUtil.initialize(modelDao); + + IllegalArgumentException e = expectThrows( + IllegalArgumentException.class, + () -> KNNVectorFieldMapperUtil.getExpectedDimensions(knnVectorFieldTypeModelBased) + ); + assertEquals(String.format("Model ID '%s' is not created.", modelId), e.getMessage()); + + when(knnVectorFieldTypeModelBased.getModelId()).thenReturn(null); + KNNMethodContext knnMethodContext = mock(KNNMethodContext.class); + MethodComponentContext methodComponentContext = mock(MethodComponentContext.class); + String fieldName = "test-field"; + when(methodComponentContext.getName()).thenReturn(fieldName); + when(knnMethodContext.getMethodComponentContext()).thenReturn(methodComponentContext); + when(knnVectorFieldTypeModelBased.getKnnMethodContext()).thenReturn(knnMethodContext); + + e = expectThrows( + IllegalArgumentException.class, + () -> KNNVectorFieldMapperUtil.getExpectedDimensions(knnVectorFieldTypeModelBased) + ); + assertEquals(String.format("Field '%s' does not have model.", fieldName), e.getMessage()); + } } diff --git a/src/test/java/org/opensearch/knn/plugin/script/KNNScriptScoringIT.java b/src/test/java/org/opensearch/knn/plugin/script/KNNScriptScoringIT.java index 58cdb3112..06aaa413b 100644 --- a/src/test/java/org/opensearch/knn/plugin/script/KNNScriptScoringIT.java +++ b/src/test/java/org/opensearch/knn/plugin/script/KNNScriptScoringIT.java @@ -40,9 +40,23 @@ import java.util.stream.Collectors; import static org.hamcrest.Matchers.containsString; +import static org.opensearch.knn.common.KNNConstants.FAISS_NAME; +import static org.opensearch.knn.common.KNNConstants.KNN_ENGINE; +import static org.opensearch.knn.common.KNNConstants.METHOD_IVF; +import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_NLIST; +import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_NPROBES; +import static org.opensearch.knn.common.KNNConstants.MODEL_ID; +import static org.opensearch.knn.common.KNNConstants.NAME; +import static org.opensearch.knn.common.KNNConstants.PARAMETERS; +import static org.opensearch.knn.common.KNNConstants.TYPE; +import static org.opensearch.knn.common.KNNConstants.TYPE_KNN_VECTOR; +import static org.opensearch.knn.common.KNNConstants.TRAIN_FIELD_PARAMETER; +import static org.opensearch.knn.common.KNNConstants.TRAIN_INDEX_PARAMETER; public class KNNScriptScoringIT extends KNNRestTestCase { + private static final String TEST_MODEL = "test-model"; + public void testKNNL2ScriptScore() throws Exception { testKNNScriptScore(SpaceType.L2); } @@ -541,6 +555,46 @@ public void testKNNScriptScoreWithRequestCacheEnabled() throws Exception { assertEquals(1, secondQueryCacheMap.get("hit_count")); } + public void testKNNScriptScoreOnModelBasedIndex() throws Exception { + int dimensions = randomIntBetween(2, 10); + String trainMapping = createKnnIndexMapping(TRAIN_FIELD_PARAMETER, dimensions); + createKnnIndex(TRAIN_INDEX_PARAMETER, trainMapping); + bulkIngestRandomVectors(TRAIN_INDEX_PARAMETER, TRAIN_FIELD_PARAMETER, dimensions * 3, dimensions); + + XContentBuilder methodBuilder = XContentFactory.jsonBuilder() + .startObject() + .field(NAME, METHOD_IVF) + .field(KNN_ENGINE, FAISS_NAME) + .startObject(PARAMETERS) + .field(METHOD_PARAMETER_NLIST, 4) + .field(METHOD_PARAMETER_NPROBES, 2) + .endObject() + .endObject(); + Map method = xContentBuilderToMap(methodBuilder); + + trainModel(TEST_MODEL, TRAIN_INDEX_PARAMETER, TRAIN_FIELD_PARAMETER, dimensions, method, "test model for script score"); + assertTrainingSucceeds(TEST_MODEL, 30, 1000); + + String testMapping = XContentFactory.jsonBuilder() + .startObject() + .startObject(PROPERTIES_FIELD) + .startObject(FIELD_NAME) + .field(TYPE, TYPE_KNN_VECTOR) + .field(MODEL_ID, TEST_MODEL) + .endObject() + .endObject() + .endObject() + .toString(); + + for (SpaceType spaceType : SpaceType.values()) { + if (spaceType != SpaceType.HAMMING_BIT) { + final float[] queryVector = randomVector(dimensions); + final BiFunction scoreFunction = getScoreFunction(spaceType, queryVector); + createIndexAndAssertScriptScore(testMapping, spaceType, scoreFunction, dimensions, queryVector); + } + } + } + private List createMappers(int dimensions) throws Exception { return List.of( createKnnIndexMapping(FIELD_NAME, dimensions), From 34e55f2f47882080d1f342907c7a1f1e3c8a96f1 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Mon, 29 Apr 2024 19:02:25 -0700 Subject: [PATCH 251/416] Revert 'Support script score when doc value is disabled' (#1662) (#1666) Signed-off-by: Ryan Bogan (cherry picked from commit bd2f403cb1ff439e6f1d88efce71464682472544) Co-authored-by: Ryan Bogan --- CHANGELOG.md | 1 - .../knn/index/KNNVectorDVLeafFieldData.java | 28 +---- .../knn/index/KNNVectorScriptDocValues.java | 108 ++---------------- .../index/KNNVectorScriptDocValuesTests.java | 61 +++------- .../knn/index/VectorDataTypeTests.java | 4 +- .../plugin/script/KNNScoringUtilTests.java | 2 +- .../knn/plugin/script/KNNScriptScoringIT.java | 11 +- .../knn/plugin/script/PainlessScriptIT.java | 4 +- 8 files changed, 33 insertions(+), 186 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8f2721810..b567d64bc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,7 +19,6 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), * Support filter and nested field in faiss engine radial search [#1652](https://github.com/opensearch-project/k-NN/pull/1652) ### Enhancements * Make the HitQueue size more appropriate for exact search [#1549](https://github.com/opensearch-project/k-NN/pull/1549) -* Support script score when doc value is disabled [#1573](https://github.com/opensearch-project/k-NN/pull/1573) * Implemented the Streaming Feature to stream vectors from Java to JNI layer to enable creation of larger segments for vector indices [#1604](https://github.com/opensearch-project/k-NN/pull/1604) * Remove unnecessary toString conversion of vector field and added some minor optimization in KNNCodec [1613](https://github.com/opensearch-project/k-NN/pull/1613) * Serialize all models into cluster metadata [#1499](https://github.com/opensearch-project/k-NN/pull/1499) diff --git a/src/main/java/org/opensearch/knn/index/KNNVectorDVLeafFieldData.java b/src/main/java/org/opensearch/knn/index/KNNVectorDVLeafFieldData.java index 85f037c0f..f4caa4f20 100644 --- a/src/main/java/org/opensearch/knn/index/KNNVectorDVLeafFieldData.java +++ b/src/main/java/org/opensearch/knn/index/KNNVectorDVLeafFieldData.java @@ -5,10 +5,9 @@ package org.opensearch.knn.index; +import org.apache.lucene.index.BinaryDocValues; import org.apache.lucene.index.DocValues; -import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.LeafReader; -import org.apache.lucene.search.DocIdSetIterator; import org.opensearch.index.fielddata.LeafFieldData; import org.opensearch.index.fielddata.ScriptDocValues; import org.opensearch.index.fielddata.SortedBinaryDocValues; @@ -40,29 +39,10 @@ public long ramBytesUsed() { @Override public ScriptDocValues getScriptValues() { try { - FieldInfo fieldInfo = reader.getFieldInfos().fieldInfo(fieldName); - if (fieldInfo == null) { - return KNNVectorScriptDocValues.emptyValues(fieldName, vectorDataType); - } - - DocIdSetIterator values; - if (fieldInfo.hasVectorValues()) { - switch (fieldInfo.getVectorEncoding()) { - case FLOAT32: - values = reader.getFloatVectorValues(fieldName); - break; - case BYTE: - values = reader.getByteVectorValues(fieldName); - break; - default: - throw new IllegalStateException("Unsupported Lucene vector encoding: " + fieldInfo.getVectorEncoding()); - } - } else { - values = DocValues.getBinary(reader, fieldName); - } - return KNNVectorScriptDocValues.create(values, fieldName, vectorDataType); + BinaryDocValues values = DocValues.getBinary(reader, fieldName); + return new KNNVectorScriptDocValues(values, fieldName, vectorDataType); } catch (IOException e) { - throw new IllegalStateException("Cannot load values for knn vector field: " + fieldName, e); + throw new IllegalStateException("Cannot load doc values for knn vector field: " + fieldName, e); } } diff --git a/src/main/java/org/opensearch/knn/index/KNNVectorScriptDocValues.java b/src/main/java/org/opensearch/knn/index/KNNVectorScriptDocValues.java index f69ad850e..349988c93 100644 --- a/src/main/java/org/opensearch/knn/index/KNNVectorScriptDocValues.java +++ b/src/main/java/org/opensearch/knn/index/KNNVectorScriptDocValues.java @@ -6,21 +6,18 @@ package org.opensearch.knn.index; import java.io.IOException; -import java.util.Objects; -import lombok.AccessLevel; import lombok.Getter; import lombok.RequiredArgsConstructor; import org.apache.lucene.index.BinaryDocValues; -import org.apache.lucene.index.ByteVectorValues; -import org.apache.lucene.index.FloatVectorValues; -import org.apache.lucene.search.DocIdSetIterator; import org.opensearch.ExceptionsHelper; import org.opensearch.index.fielddata.ScriptDocValues; -@RequiredArgsConstructor(access = AccessLevel.PRIVATE) -public abstract class KNNVectorScriptDocValues extends ScriptDocValues { +import java.io.IOException; + +@RequiredArgsConstructor +public final class KNNVectorScriptDocValues extends ScriptDocValues { - private final DocIdSetIterator vectorValues; + private final BinaryDocValues binaryDocValues; private final String fieldName; @Getter private final VectorDataType vectorDataType; @@ -28,7 +25,11 @@ public abstract class KNNVectorScriptDocValues extends ScriptDocValues @Override public void setNextDocId(int docId) throws IOException { - docExists = vectorValues.docID() == docId || vectorValues.advance(docId) == docId; + if (binaryDocValues.advanceExact(docId)) { + docExists = true; + return; + } + docExists = false; } public float[] getValue() { @@ -43,14 +44,12 @@ public float[] getValue() { throw new IllegalStateException(errorMessage); } try { - return doGetValue(); + return vectorDataType.getVectorFromBytesRef(binaryDocValues.binaryValue()); } catch (IOException e) { throw ExceptionsHelper.convertToOpenSearchException(e); } } - protected abstract float[] doGetValue() throws IOException; - @Override public int size() { return docExists ? 1 : 0; @@ -60,89 +59,4 @@ public int size() { public float[] get(int i) { throw new UnsupportedOperationException("knn vector does not support this operation"); } - - /** - * Creates a KNNVectorScriptDocValues object based on the provided parameters. - * - * @param values The DocIdSetIterator representing the vector values. - * @param fieldName The name of the field. - * @param vectorDataType The data type of the vector. - * @return A KNNVectorScriptDocValues object based on the type of the values. - * @throws IllegalArgumentException If the type of values is unsupported. - */ - public static KNNVectorScriptDocValues create(DocIdSetIterator values, String fieldName, VectorDataType vectorDataType) { - Objects.requireNonNull(values, "values must not be null"); - if (values instanceof ByteVectorValues) { - return new KNNByteVectorScriptDocValues((ByteVectorValues) values, fieldName, vectorDataType); - } else if (values instanceof FloatVectorValues) { - return new KNNFloatVectorScriptDocValues((FloatVectorValues) values, fieldName, vectorDataType); - } else if (values instanceof BinaryDocValues) { - return new KNNNativeVectorScriptDocValues((BinaryDocValues) values, fieldName, vectorDataType); - } else { - throw new IllegalArgumentException("Unsupported values type: " + values.getClass()); - } - } - - private static final class KNNByteVectorScriptDocValues extends KNNVectorScriptDocValues { - private final ByteVectorValues values; - - KNNByteVectorScriptDocValues(ByteVectorValues values, String field, VectorDataType type) { - super(values, field, type); - this.values = values; - } - - @Override - protected float[] doGetValue() throws IOException { - byte[] bytes = values.vectorValue(); - float[] value = new float[bytes.length]; - for (int i = 0; i < bytes.length; i++) { - value[i] = (float) bytes[i]; - } - return value; - } - } - - private static final class KNNFloatVectorScriptDocValues extends KNNVectorScriptDocValues { - private final FloatVectorValues values; - - KNNFloatVectorScriptDocValues(FloatVectorValues values, String field, VectorDataType type) { - super(values, field, type); - this.values = values; - } - - @Override - protected float[] doGetValue() throws IOException { - return values.vectorValue(); - } - } - - private static final class KNNNativeVectorScriptDocValues extends KNNVectorScriptDocValues { - private final BinaryDocValues values; - - KNNNativeVectorScriptDocValues(BinaryDocValues values, String field, VectorDataType type) { - super(values, field, type); - this.values = values; - } - - @Override - protected float[] doGetValue() throws IOException { - return getVectorDataType().getVectorFromBytesRef(values.binaryValue()); - } - } - - /** - * Creates an empty KNNVectorScriptDocValues object based on the provided field name and vector data type. - * - * @param fieldName The name of the field. - * @param type The data type of the vector. - * @return An empty KNNVectorScriptDocValues object. - */ - public static KNNVectorScriptDocValues emptyValues(String fieldName, VectorDataType type) { - return new KNNVectorScriptDocValues(DocIdSetIterator.empty(), fieldName, type) { - @Override - protected float[] doGetValue() throws IOException { - throw new UnsupportedOperationException("empty values"); - } - }; - } } diff --git a/src/test/java/org/opensearch/knn/index/KNNVectorScriptDocValuesTests.java b/src/test/java/org/opensearch/knn/index/KNNVectorScriptDocValuesTests.java index 66e2893c0..3f98a9136 100644 --- a/src/test/java/org/opensearch/knn/index/KNNVectorScriptDocValuesTests.java +++ b/src/test/java/org/opensearch/knn/index/KNNVectorScriptDocValuesTests.java @@ -5,15 +5,7 @@ package org.opensearch.knn.index; -import org.apache.lucene.document.Field; -import org.apache.lucene.document.KnnByteVectorField; -import org.apache.lucene.document.KnnFloatVectorField; -import org.apache.lucene.index.BinaryDocValues; -import org.apache.lucene.index.ByteVectorValues; -import org.apache.lucene.index.DocValues; -import org.apache.lucene.index.FloatVectorValues; -import org.apache.lucene.index.LeafReader; -import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.index.LeafReaderContext; import org.opensearch.knn.KNNTestCase; import org.apache.lucene.tests.analysis.MockAnalyzer; import org.apache.lucene.document.BinaryDocValuesField; @@ -41,39 +33,26 @@ public class KNNVectorScriptDocValuesTests extends KNNTestCase { public void setUp() throws Exception { super.setUp(); directory = newDirectory(); - Class valuesClass = randomFrom(BinaryDocValues.class, ByteVectorValues.class, FloatVectorValues.class); - createKNNVectorDocument(directory, valuesClass); + createKNNVectorDocument(directory); reader = DirectoryReader.open(directory); - LeafReader leafReader = reader.getContext().leaves().get(0).reader(); - DocIdSetIterator vectorValues; - if (BinaryDocValues.class.equals(valuesClass)) { - vectorValues = DocValues.getBinary(leafReader, MOCK_INDEX_FIELD_NAME); - } else if (ByteVectorValues.class.equals(valuesClass)) { - vectorValues = leafReader.getByteVectorValues(MOCK_INDEX_FIELD_NAME); - } else { - vectorValues = leafReader.getFloatVectorValues(MOCK_INDEX_FIELD_NAME); - } - - scriptDocValues = KNNVectorScriptDocValues.create(vectorValues, MOCK_INDEX_FIELD_NAME, VectorDataType.FLOAT); + LeafReaderContext leafReaderContext = reader.getContext().leaves().get(0); + scriptDocValues = new KNNVectorScriptDocValues( + leafReaderContext.reader().getBinaryDocValues(MOCK_INDEX_FIELD_NAME), + MOCK_INDEX_FIELD_NAME, + VectorDataType.FLOAT + ); } - private void createKNNVectorDocument(Directory directory, Class valuesClass) throws IOException { + private void createKNNVectorDocument(Directory directory) throws IOException { IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random())); IndexWriter writer = new IndexWriter(directory, conf); Document knnDocument = new Document(); - Field field; - if (BinaryDocValues.class.equals(valuesClass)) { - field = new BinaryDocValuesField( + knnDocument.add( + new BinaryDocValuesField( MOCK_INDEX_FIELD_NAME, new VectorField(MOCK_INDEX_FIELD_NAME, SAMPLE_VECTOR_DATA, new FieldType()).binaryValue() - ); - } else if (ByteVectorValues.class.equals(valuesClass)) { - field = new KnnByteVectorField(MOCK_INDEX_FIELD_NAME, SAMPLE_BYTE_VECTOR_DATA); - } else { - field = new KnnFloatVectorField(MOCK_INDEX_FIELD_NAME, SAMPLE_VECTOR_DATA); - } - - knnDocument.add(field); + ) + ); writer.addDocument(knnDocument); writer.commit(); writer.close(); @@ -105,18 +84,4 @@ public void testSize() throws IOException { public void testGet() throws IOException { expectThrows(UnsupportedOperationException.class, () -> scriptDocValues.get(0)); } - - public void testUnsupportedValues() throws IOException { - expectThrows( - IllegalArgumentException.class, - () -> KNNVectorScriptDocValues.create(DocValues.emptyNumeric(), MOCK_INDEX_FIELD_NAME, VectorDataType.FLOAT) - ); - } - - public void testEmptyValues() throws IOException { - KNNVectorScriptDocValues values = KNNVectorScriptDocValues.emptyValues(MOCK_INDEX_FIELD_NAME, VectorDataType.FLOAT); - assertEquals(0, values.size()); - scriptDocValues.setNextDocId(0); - assertEquals(0, values.size()); - } } diff --git a/src/test/java/org/opensearch/knn/index/VectorDataTypeTests.java b/src/test/java/org/opensearch/knn/index/VectorDataTypeTests.java index 19270717d..4423c85d8 100644 --- a/src/test/java/org/opensearch/knn/index/VectorDataTypeTests.java +++ b/src/test/java/org/opensearch/knn/index/VectorDataTypeTests.java @@ -57,7 +57,7 @@ private KNNVectorScriptDocValues getKNNFloatVectorScriptDocValues() { createKNNFloatVectorDocument(directory); reader = DirectoryReader.open(directory); LeafReaderContext leafReaderContext = reader.getContext().leaves().get(0); - return KNNVectorScriptDocValues.create( + return new KNNVectorScriptDocValues( leafReaderContext.reader().getBinaryDocValues(VectorDataTypeTests.MOCK_FLOAT_INDEX_FIELD_NAME), VectorDataTypeTests.MOCK_FLOAT_INDEX_FIELD_NAME, VectorDataType.FLOAT @@ -70,7 +70,7 @@ private KNNVectorScriptDocValues getKNNByteVectorScriptDocValues() { createKNNByteVectorDocument(directory); reader = DirectoryReader.open(directory); LeafReaderContext leafReaderContext = reader.getContext().leaves().get(0); - return KNNVectorScriptDocValues.create( + return new KNNVectorScriptDocValues( leafReaderContext.reader().getBinaryDocValues(VectorDataTypeTests.MOCK_BYTE_INDEX_FIELD_NAME), VectorDataTypeTests.MOCK_BYTE_INDEX_FIELD_NAME, VectorDataType.BYTE diff --git a/src/test/java/org/opensearch/knn/plugin/script/KNNScoringUtilTests.java b/src/test/java/org/opensearch/knn/plugin/script/KNNScoringUtilTests.java index 22110accd..8c43a4acf 100644 --- a/src/test/java/org/opensearch/knn/plugin/script/KNNScoringUtilTests.java +++ b/src/test/java/org/opensearch/knn/plugin/script/KNNScoringUtilTests.java @@ -280,7 +280,7 @@ public KNNVectorScriptDocValues getScriptDocValues(String fieldName) throws IOEx if (scriptDocValues == null) { reader = DirectoryReader.open(directory); LeafReaderContext leafReaderContext = reader.getContext().leaves().get(0); - scriptDocValues = KNNVectorScriptDocValues.create( + scriptDocValues = new KNNVectorScriptDocValues( leafReaderContext.reader().getBinaryDocValues(fieldName), fieldName, VectorDataType.FLOAT diff --git a/src/test/java/org/opensearch/knn/plugin/script/KNNScriptScoringIT.java b/src/test/java/org/opensearch/knn/plugin/script/KNNScriptScoringIT.java index 06aaa413b..2c18bfa90 100644 --- a/src/test/java/org/opensearch/knn/plugin/script/KNNScriptScoringIT.java +++ b/src/test/java/org/opensearch/knn/plugin/script/KNNScriptScoringIT.java @@ -603,16 +603,7 @@ private List createMappers(int dimensions) throws Exception { dimensions, KNNConstants.METHOD_HNSW, KNNEngine.LUCENE.getName(), - SpaceType.DEFAULT.getValue(), - true - ), - createKnnIndexMapping( - FIELD_NAME, - dimensions, - KNNConstants.METHOD_HNSW, - KNNEngine.LUCENE.getName(), - SpaceType.DEFAULT.getValue(), - false + SpaceType.DEFAULT.getValue() ) ); } diff --git a/src/test/java/org/opensearch/knn/plugin/script/PainlessScriptIT.java b/src/test/java/org/opensearch/knn/plugin/script/PainlessScriptIT.java index 47e914d04..e87b9771e 100644 --- a/src/test/java/org/opensearch/knn/plugin/script/PainlessScriptIT.java +++ b/src/test/java/org/opensearch/knn/plugin/script/PainlessScriptIT.java @@ -563,9 +563,7 @@ public void testL2ScriptingWithLuceneBackedIndex() throws Exception { new MethodComponentContext(METHOD_HNSW, Collections.emptyMap()) ); properties.add( - new MappingProperty(FIELD_NAME, KNNVectorFieldMapper.CONTENT_TYPE).dimension("2") - .knnMethodContext(knnMethodContext) - .docValues(randomBoolean()) + new MappingProperty(FIELD_NAME, KNNVectorFieldMapper.CONTENT_TYPE).dimension("2").knnMethodContext(knnMethodContext) ); String source = String.format("1/(1 + l2Squared([1.0f, 1.0f], doc['%s']))", FIELD_NAME); From 143b27b1eb3d54a05cf3119d280392a9b2575ba3 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Mon, 29 Apr 2024 19:38:48 -0700 Subject: [PATCH 252/416] Ensure blas is statically linked (#1667) Signed-off-by: John Mazanec (cherry picked from commit 1961e587feffe213724d4648564bb76827679a47) --- jni/cmake/init-faiss.cmake | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/jni/cmake/init-faiss.cmake b/jni/cmake/init-faiss.cmake index bc3836b06..bef93eda0 100644 --- a/jni/cmake/init-faiss.cmake +++ b/jni/cmake/init-faiss.cmake @@ -41,13 +41,15 @@ if (${CMAKE_SYSTEM_NAME} STREQUAL Darwin) endif() find_package(ZLIB REQUIRED) + +# Statically link BLAS - ensure this is before we find the blas package so we dont dynamically link +set(BLA_STATIC ON) find_package(BLAS REQUIRED) enable_language(Fortran) find_package(LAPACK REQUIRED) # Set relevant properties set(BUILD_TESTING OFF) # Avoid building faiss tests -set(BLA_STATIC ON) # Statically link BLAS set(FAISS_ENABLE_GPU OFF) set(FAISS_ENABLE_PYTHON OFF) From 5487264b4cedccd3758ba3d3b465a86b5a3021cb Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Tue, 30 Apr 2024 11:10:21 -0700 Subject: [PATCH 253/416] Ensure gcc is newer >= 11.2.1 < 12.0.0 when building knn lib (#1669) Signed-off-by: Peter Zhu (cherry picked from commit da4cd1c6bdaa0c0c6ca3a622f15fe821d6ace281) --- scripts/build.sh | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/scripts/build.sh b/scripts/build.sh index d2571efb6..0b1419a93 100644 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -103,18 +103,26 @@ if [ "$JAVA_HOME" = "" ]; then echo "SET JAVA_HOME=$JAVA_HOME" fi -# Ensure gcc version is at least 12.0.0 for faiss 1.7.4+ / SIMD Neon support on ARM64 compilation +# Ensure gcc version is > 11.2.1 and < 12.0.0 for faiss 1.7.4+ / SIMD Neon support on ARM64 compilation # https://github.com/opensearch-project/k-NN/issues/975 # https://github.com/opensearch-project/k-NN/issues/1138 # https://github.com/opensearch-project/opensearch-build/issues/4386 # https://github.com/opensearch-project/opensearch-build/issues/4379#issuecomment-2067191682 +# https://github.com/opensearch-project/opensearch-build/issues/4379#issuecomment-2083623882 +# https://github.com/opensearch-project/opensearch-build/issues/4379#issuecomment-2084133839 GCC_VERSION=`gcc --version | head -n 1 | cut -d ' ' -f3` -GCC_REQUIRED_VERSION=12.0.0 +GCC_REQUIRED_VERSION=11.2.1 COMPARE_VERSION=`echo $GCC_REQUIRED_VERSION $GCC_VERSION | tr ' ' '\n' | sort -V | uniq | head -n 1` if [ "$COMPARE_VERSION" != "$GCC_REQUIRED_VERSION" ]; then echo "gcc version on this env is older than $GCC_REQUIRED_VERSION, exit 1" exit 1 fi +GCC_REQUIRED_VERSION_CEILING=12.0.0 +COMPARE_VERSION_CEILING=`echo $GCC_REQUIRED_VERSION_CEILING $GCC_VERSION | tr ' ' '\n' | sort -V | uniq | tail -n 1` +if [ "$COMPARE_VERSION_CEILING" != "$GCC_REQUIRED_VERSION_CEILING" ] && (echo "$OSTYPE" | grep -i linux); then + echo "gcc version on this env is newer than $GCC_REQUIRED_VERSION_CEILING, exit 1" + exit 1 +fi # Build k-NN lib and plugin through gradle tasks cd $work_dir From 3498870bd8e8f6358e8cfc0efdc7abfe9f41c7da Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Tue, 30 Apr 2024 13:10:35 -0700 Subject: [PATCH 254/416] Remove odfe release notes (#1673) Signed-off-by: John Mazanec (cherry picked from commit a9ebd14079b2a27e6010bbc9bd7d7ccac72e40c0) --- ...earch-knn.release-notes-1.2.0.0-alpha.1.md | 8 ---- ...o-elasticsearch-knn.release-notes-1.3.0.md | 14 ------ ...o-elasticsearch-knn.release-notes-1.4.0.md | 19 -------- ...o-elasticsearch-knn.release-notes-1.6.0.md | 16 ------- ...o-elasticsearch-knn.release-notes-1.7.0.md | 12 ----- ...o-elasticsearch-knn.release-notes-1.8.0.md | 9 ---- ...elasticsearch-knn.release-notes-1.9.0.0.md | 19 -------- ...lasticsearch-knn.release-notes-1.10.1.0.md | 48 ------------------- ...lasticsearch-knn.release-notes-1.11.0.0.md | 21 -------- ...lasticsearch-knn.release-notes-1.12.0.0.md | 13 ----- ...lasticsearch-knn.release-notes-1.13.0.0.md | 34 ------------- 11 files changed, 213 deletions(-) delete mode 100644 release-notes/opendistro-elasticsearch-knn.release-notes-1.2.0.0-alpha.1.md delete mode 100644 release-notes/opendistro-elasticsearch-knn.release-notes-1.3.0.md delete mode 100644 release-notes/opendistro-elasticsearch-knn.release-notes-1.4.0.md delete mode 100644 release-notes/opendistro-elasticsearch-knn.release-notes-1.6.0.md delete mode 100644 release-notes/opendistro-elasticsearch-knn.release-notes-1.7.0.md delete mode 100644 release-notes/opendistro-elasticsearch-knn.release-notes-1.8.0.md delete mode 100644 release-notes/opendistro-elasticsearch-knn.release-notes-1.9.0.0.md delete mode 100644 release-notes/opendistro-for-elasticsearch-knn.release-notes-1.10.1.0.md delete mode 100644 release-notes/opendistro-for-elasticsearch-knn.release-notes-1.11.0.0.md delete mode 100644 release-notes/opendistro-for-elasticsearch-knn.release-notes-1.12.0.0.md delete mode 100644 release-notes/opendistro-for-elasticsearch-knn.release-notes-1.13.0.0.md diff --git a/release-notes/opendistro-elasticsearch-knn.release-notes-1.2.0.0-alpha.1.md b/release-notes/opendistro-elasticsearch-knn.release-notes-1.2.0.0-alpha.1.md deleted file mode 100644 index 5311a7e62..000000000 --- a/release-notes/opendistro-elasticsearch-knn.release-notes-1.2.0.0-alpha.1.md +++ /dev/null @@ -1,8 +0,0 @@ -## 2019-09-17 Version 1.2.0.0-alpha.1 -### New Features - * Adds support for Elasticsearch 7.2.0 - [Commit #](https://github.com/opendistro-for-elasticsearch/k-NN/commit/15ae8c7b3a4ab88e2be974af107161b10d0204bb) - -### Bug fixes - * Performance improvement and bug fixes - [PR 2](https://github.com/opendistro-for-elasticsearch/k-NN/pull/2) - -Note:- For configuring native library for plugin installations from Archive, please refer to [ReadME](https://github.com/opendistro-for-elasticsearch/k-NN/blob/development/README.md#java-native-library-usage) diff --git a/release-notes/opendistro-elasticsearch-knn.release-notes-1.3.0.md b/release-notes/opendistro-elasticsearch-knn.release-notes-1.3.0.md deleted file mode 100644 index d377a5d64..000000000 --- a/release-notes/opendistro-elasticsearch-knn.release-notes-1.3.0.md +++ /dev/null @@ -1,14 +0,0 @@ -## 2019-12-03 Version 1.3.0.0 -### Notable changes - -* opendistro1.3 for KNN for ES 7.3.2 -* Performance improvement and bug fixes -** JNI Memory leak fixes -** Cache time out fix -** Doc values memory fix -** Other minor bugs -** Updated ReadMe -* fixed memory leak in saveIndex -* fixed issue jvm heap leak in JNI - -Note:- For configuring native library for plugin installations from Archive, please refer to [ReadME](https://github.com/opendistro-for-elasticsearch/k-NN/blob/development/README.md#java-native-library-usage) \ No newline at end of file diff --git a/release-notes/opendistro-elasticsearch-knn.release-notes-1.4.0.md b/release-notes/opendistro-elasticsearch-knn.release-notes-1.4.0.md deleted file mode 100644 index 38736b343..000000000 --- a/release-notes/opendistro-elasticsearch-knn.release-notes-1.4.0.md +++ /dev/null @@ -1,19 +0,0 @@ -## 2020-01-24 Version 1.4.0.0 -### Features -#### Elasticsearch Compatibility -* Feature [#11 ](https://github.com/opendistro-for-elasticsearch/k-NN/issues/11): Elasticsearch 7.4.2 compatibility - -#### Documentation -* Feature ( [#40 ](https://github.com/opendistro-for-elasticsearch/k-NN/issues/40 ), [#37 ](https://github.com/opendistro-for-elasticsearch/k-NN/issues/37)). Documentation on knn index creation, settings and stats - -### Enhancements -* KNN Codec Backward Compatibility support [#20 ](https://github.com/opendistro-for-elasticsearch/k-NN/issues/20) - -### Bug Fixes -* Avoid recreating space for each query [#29 ] -* Fix a leak where FileWatchers are added but never removed [#36 ] -* JNI clean up and race conditions [#25 ] -* native memory leak in saveIndex and JVM leak [#14 ] [#15 ] - -### Note -For configuring native library for plugin installations from Archive, please refer to [ReadME](https://github.com/opendistro-for-elasticsearch/k-NN/blob/development/README.md#java-native-library-usage) \ No newline at end of file diff --git a/release-notes/opendistro-elasticsearch-knn.release-notes-1.6.0.md b/release-notes/opendistro-elasticsearch-knn.release-notes-1.6.0.md deleted file mode 100644 index 8dc51aa90..000000000 --- a/release-notes/opendistro-elasticsearch-knn.release-notes-1.6.0.md +++ /dev/null @@ -1,16 +0,0 @@ -## 2020-03-24 Version 1.6.0.0 (Current) -### Features -* Feature [#76](https://github.com/opendistro-for-elasticsearch/k-NN/pull/72): Elasticsearch 7.6.1 compatibility (issue [#71](https://github.com/opendistro-for-elasticsearch/k-NN/issues/71)) -* Feature [#73](https://github.com/opendistro-for-elasticsearch/k-NN/pull/73): Add Github Actions so that changes are automatically tested and artifacts are uploaded to S3 (issue [#74](https://github.com/opendistro-for-elasticsearch/k-NN/issues/74)) - -### Enhancements -* Enhancement [#61](https://github.com/opendistro-for-elasticsearch/k-NN/pull/61): Convert integration tests from ESIntegTestCase to ESRestTestCase, so that they can be run on a remote cluster (issue [#60](https://github.com/opendistro-for-elasticsearch/k-NN/issues/60)) -* Enhancement [#54](https://github.com/opendistro-for-elasticsearch/k-NN/pull/54): Add check in gradle build for license headers (issue [#7](https://github.com/opendistro-for-elasticsearch/k-NN/issues/7)) -* Enhancement [#52](https://github.com/opendistro-for-elasticsearch/k-NN/pull/52): Lazily load efSearch parameter (issue [#51](https://github.com/opendistro-for-elasticsearch/k-NN/issues/51)) - -### Bug Fixes -* Bugfix [#66](https://github.com/opendistro-for-elasticsearch/k-NN/pull/66): Flaky failure in KNN80HnswIndexTests testFooter (issue [#65](https://github.com/opendistro-for-elasticsearch/k-NN/issues/65)) -* Bugfix [#63](https://github.com/opendistro-for-elasticsearch/k-NN/pull/63): Circuit Breaker fails to turn off (issue [#62](https://github.com/opendistro-for-elasticsearch/k-NN/issues/62)) -* Bugfix [#59](https://github.com/opendistro-for-elasticsearch/k-NN/pull/59): Gradle build failure on Mac due to library error (issue [#58](https://github.com/opendistro-for-elasticsearch/k-NN/issues/58)) -* Bugfix [#53](https://github.com/opendistro-for-elasticsearch/k-NN/pull/53): AccessControlException when HNSW library is loaded (issue [#49](https://github.com/opendistro-for-elasticsearch/k-NN/issues/49)) -* Bugfix [#47](https://github.com/opendistro-for-elasticsearch/k-NN/pull/47): Stats API failure in Transport Layer (issue [#45](https://github.com/opendistro-for-elasticsearch/k-NN/issues/45)) diff --git a/release-notes/opendistro-elasticsearch-knn.release-notes-1.7.0.md b/release-notes/opendistro-elasticsearch-knn.release-notes-1.7.0.md deleted file mode 100644 index e370baa4b..000000000 --- a/release-notes/opendistro-elasticsearch-knn.release-notes-1.7.0.md +++ /dev/null @@ -1,12 +0,0 @@ -## Version 1.7.0 (Version compatible with elasticsearch 7.6.1) -### Features -* Feature [#90](https://github.com/opendistro-for-elasticsearch/k-NN/pull/90): Support cosine similarity (issue [#28](https://github.com/opendistro-for-elasticsearch/k-NN/issues/28)). ```Note``` this feature is experimental - -### Enhancements -* Enhancement [#89](https://github.com/opendistro-for-elasticsearch/k-NN/pull/89): Add stats to track the number of requests and errors for KNN query and index operations. (issue [#88](https://github.com/opendistro-for-elasticsearch/k-NN/issues/88)) -* Enhancement [#92](https://github.com/opendistro-for-elasticsearch/k-NN/pull/92): Switched the default value of the circuit breaker from 60% to 50%. (issue [#82](https://github.com/opendistro-for-elasticsearch/k-NN/issues/82)) -* Enhancement [#73](https://github.com/opendistro-for-elasticsearch/k-NN/pull/73): Create Github action that automatically runs integration tests against docker image whenever code is checked into main or opendistro branch. (issue [#74](https://github.com/opendistro-for-elasticsearch/k-NN/issues/74)) - -### Bug Fixes -* Bugfix [#100](https://github.com/opendistro-for-elasticsearch/k-NN/pull/100): Added validation in VectorFieldMapper to check for vector values of NaN and throwing an Exception if so. (issue [#99](https://github.com/opendistro-for-elasticsearch/k-NN/issues/99)) -* Bugfix [#78](https://github.com/opendistro-for-elasticsearch/k-NN/pull/78): Fix debugging integration tests (issue [#77](https://github.com/opendistro-for-elasticsearch/k-NN/issues/77)) diff --git a/release-notes/opendistro-elasticsearch-knn.release-notes-1.8.0.md b/release-notes/opendistro-elasticsearch-knn.release-notes-1.8.0.md deleted file mode 100644 index 091d01e0e..000000000 --- a/release-notes/opendistro-elasticsearch-knn.release-notes-1.8.0.md +++ /dev/null @@ -1,9 +0,0 @@ -## Version 1.8.0 (Version compatible with elasticsearch 7.7.0) -### Features -* Feature [#115](https://github.com/opendistro-for-elasticsearch/k-NN/pull/115): ODFE 1.8 support for Elasticsearch version 7.7.0 (issue [#113](https://github.com/opendistro-for-elasticsearch/k-NN/issues/113)). - -### Enhancements -* Enhancement [#108](https://github.com/opendistro-for-elasticsearch/k-NN/pull/108): Block knn index writes if the Circuit breaker triggers. (issue [#96](https://github.com/opendistro-for-elasticsearch/k-NN/issues/96)) - -### Bug Fixes -* None \ No newline at end of file diff --git a/release-notes/opendistro-elasticsearch-knn.release-notes-1.9.0.0.md b/release-notes/opendistro-elasticsearch-knn.release-notes-1.9.0.0.md deleted file mode 100644 index bbf3f0f3c..000000000 --- a/release-notes/opendistro-elasticsearch-knn.release-notes-1.9.0.0.md +++ /dev/null @@ -1,19 +0,0 @@ -## Version 1.9.0.0 (Version compatible with elasticsearch 7.8.0) -### Features -* Feature [#147](https://github.com/opendistro-for-elasticsearch/k-NN/pull/147): ODFE 1.9 support for Elasticsearch version 7.8.0 (issue [#146](https://github.com/opendistro-for-elasticsearch/k-NN/issues/146)). - -### Enhancements -* Enhancement [#149](https://github.com/opendistro-for-elasticsearch/k-NN/pull/149): Remove/depricate shared library in buildSrc. (issue [#148](https://github.com/opendistro-for-elasticsearch/k-NN/issues/96)) -* Enhancement [#141](https://github.com/opendistro-for-elasticsearch/k-NN/pull/141): Modify artifact release Github action. (issue [#140](https://github.com/opendistro-for-elasticsearch/k-NN/issues/140)) -* Enhancement [#132](https://github.com/opendistro-for-elasticsearch/k-NN/pull/132): Add github action to build library artifacts. (issue [#122](https://github.com/opendistro-for-elasticsearch/k-NN/issues/122)) -* Enhancement [#126](https://github.com/opendistro-for-elasticsearch/k-NN/pull/126): Ability to dynamically update efSearch setting. (issue [#116](https://github.com/opendistro-for-elasticsearch/k-NN/issues/116)) -* Enhancement [#125](https://github.com/opendistro-for-elasticsearch/k-NN/pull/125): Fix test structure. (issue [#124](https://github.com/opendistro-for-elasticsearch/k-NN/issues/124)) -* Enhancement [#123](https://github.com/opendistro-for-elasticsearch/k-NN/pull/123): Build separate artifacts for library using CPack. (issue [#122](https://github.com/opendistro-for-elasticsearch/k-NN/issues/122)) - -### Bug Fixes -* Bugfix [#155](https://github.com/opendistro-for-elasticsearch/k-NN/pull/155): Bad recall from Lucene upgrade 8.5.1. (issue [#154](https://github.com/opendistro-for-elasticsearch/k-NN/issues/154)) -* Bugfix [#143](https://github.com/opendistro-for-elasticsearch/k-NN/pull/143): Add recursive option to zip. (issue [#140](https://github.com/opendistro-for-elasticsearch/k-NN/issues/140)) -* Bugfix [#138](https://github.com/opendistro-for-elasticsearch/k-NN/pull/138): CMake fails to use c++11 CMake 2.8. (issue [#137](https://github.com/opendistro-for-elasticsearch/k-NN/issues/137)) -* Bugfix [#134](https://github.com/opendistro-for-elasticsearch/k-NN/pull/134): Fix Jacoco coverage issue introduced in odfe 1.8. (issue [#127](https://github.com/opendistro-for-elasticsearch/k-NN/issues/127)) -* Bugfix [#130](https://github.com/opendistro-for-elasticsearch/k-NN/pull/130): Fixes parent directory in makeJniLib gradle task. (issue [#137](https://github.com/opendistro-for-elasticsearch/k-NN/issues/137)) -* Bugfix [#125](https://github.com/opendistro-for-elasticsearch/k-NN/pull/125): Flaky test cases caused by Counter Enum. (issue [#124](https://github.com/opendistro-for-elasticsearch/k-NN/issues/124)) diff --git a/release-notes/opendistro-for-elasticsearch-knn.release-notes-1.10.1.0.md b/release-notes/opendistro-for-elasticsearch-knn.release-notes-1.10.1.0.md deleted file mode 100644 index 192bd14b2..000000000 --- a/release-notes/opendistro-for-elasticsearch-knn.release-notes-1.10.1.0.md +++ /dev/null @@ -1,48 +0,0 @@ -## Version 1.10.1.0 Release Notes - -Compatible with Elasticsearch 7.9.1 -### Features - -* Add Warmup API to load indices graphs into memory ([#162](https://github.com/opendistro-for-elasticsearch/k-NN/pull/162)) - -### Enhancements - -* Upgrade nmslib to v2.0.6 ([#160](https://github.com/opendistro-for-elasticsearch/k-NN/pull/160)) - -### Bug Fixes - -* Update guava version to 29.0 ([#182](https://github.com/opendistro-for-elasticsearch/k-NN/pull/182)) -* Add default index settings when parsing index ([#205](https://github.com/opendistro-for-elasticsearch/k-NN/pull/205)) -* NPE in force merge when non knn doc gets updated to knn doc across segments ([#212](https://github.com/opendistro-for-elasticsearch/k-NN/pull/212)) -* Fix casting issue with cache expiration ([#215](https://github.com/opendistro-for-elasticsearch/k-NN/pull/215)) - -### Infrastructure - -* Reset state for uTs so tests run independently ([#159](https://github.com/opendistro-for-elasticsearch/k-NN/pull/159)) -* Pass -march=x86-64 to build JNI library ([#164](https://github.com/opendistro-for-elasticsearch/k-NN/pull/164)) -* Fix versioning for lib artifacts ([#166](https://github.com/opendistro-for-elasticsearch/k-NN/pull/166)) -* Add release notes automation ([#168](https://github.com/opendistro-for-elasticsearch/k-NN/pull/168)) -* Add Github action to build library artifacts ([#170](https://github.com/opendistro-for-elasticsearch/k-NN/pull/170)) -* Flaky rest test case fix ([#183](https://github.com/opendistro-for-elasticsearch/k-NN/pull/183)) -* Add code coverage widget and badges ([#191](https://github.com/opendistro-for-elasticsearch/k-NN/pull/191)) -* Add Codecov configuration to set a coverage threshold to pass the check on a commit ([#192](https://github.com/opendistro-for-elasticsearch/k-NN/pull/192)) -* Add AWS CLI in order to ship library artifacts from container ([#194](https://github.com/opendistro-for-elasticsearch/k-NN/pull/194)) -* Remove sudo from "./aws install" in library build action ([#202](https://github.com/opendistro-for-elasticsearch/k-NN/pull/202)) -* Fix download link in package description ([#214](https://github.com/opendistro-for-elasticsearch/k-NN/pull/214)) - -### Documentation - -* Performance tuning/Recommendations ([#177](https://github.com/opendistro-for-elasticsearch/k-NN/pull/177)) -* Fix cluster setting example in README.md ([#186](https://github.com/opendistro-for-elasticsearch/k-NN/pull/186)) -* Add scoring documentation ([#193](https://github.com/opendistro-for-elasticsearch/k-NN/pull/193)) -* Add 1.10.0.0 release notes ([#201](https://github.com/opendistro-for-elasticsearch/k-NN/pull/201)) - -### Maintenance - -* ODFE 1.10 support for k-NN plugin ([#199](https://github.com/opendistro-for-elasticsearch/k-NN/pull/199)) -* Upgrade Elasticsearch to 7.9.1 and ODFE to 1.10.1 ([#217](https://github.com/opendistro-for-elasticsearch/k-NN/pull/217)) - -### Refactoring - -* Update default variable settings name ([#209](https://github.com/opendistro-for-elasticsearch/k-NN/pull/209)) - diff --git a/release-notes/opendistro-for-elasticsearch-knn.release-notes-1.11.0.0.md b/release-notes/opendistro-for-elasticsearch-knn.release-notes-1.11.0.0.md deleted file mode 100644 index ba9bf4b4f..000000000 --- a/release-notes/opendistro-for-elasticsearch-knn.release-notes-1.11.0.0.md +++ /dev/null @@ -1,21 +0,0 @@ -## Version 1.11.0.0 Release Notes - -Compatible with Elasticsearch 7.9.1 -### Features - -* Pre filter support through custom scoring ([#196](https://github.com/opendistro-for-elasticsearch/k-NN/pull/196)) - -### Enhancements - -* Add existsQuery method implementation to KNNVectorFieldType ([#228](https://github.com/opendistro-for-elasticsearch/k-NN/pull/228)) -* Change "space" parameter to "space_type" for custom scoring ([#232](https://github.com/opendistro-for-elasticsearch/k-NN/pull/232)) -* change space -> space_type ([#234](https://github.com/opendistro-for-elasticsearch/k-NN/pull/234)) -* Add stats for custom scoring feature ([#233](https://github.com/opendistro-for-elasticsearch/k-NN/pull/233)) - -### Bug Fixes - -* KNN score fix for non knn documents ([#231](https://github.com/opendistro-for-elasticsearch/k-NN/pull/231)) -* Fix script statistics flaky test case ([#235](https://github.com/opendistro-for-elasticsearch/k-NN/pull/235)) -* Refactor KNNVectorFieldMapper ([#240](https://github.com/opendistro-for-elasticsearch/k-NN/pull/240)) -* Fix PostingsFormat in KNN Codec ([#236](https://github.com/opendistro-for-elasticsearch/k-NN/pull/236)) - diff --git a/release-notes/opendistro-for-elasticsearch-knn.release-notes-1.12.0.0.md b/release-notes/opendistro-for-elasticsearch-knn.release-notes-1.12.0.0.md deleted file mode 100644 index f52c0f3c9..000000000 --- a/release-notes/opendistro-for-elasticsearch-knn.release-notes-1.12.0.0.md +++ /dev/null @@ -1,13 +0,0 @@ -## Version 1.12.0.0 Release Notes - -Compatible with Elasticsearch 7.10.0 - -### Features - -* Support for hamming bit distance in custom scoring ([#267](https://github.com/opendistro-for-elasticsearch/k-NN/pull/267)) - -### Maintenance - -* k-NN plugin support for Elasticsearch version 7.10.0 ([#271](https://github.com/opendistro-for-elasticsearch/k-NN/pull/271)) -* Bump odfe version to 1.12 ([#273](https://github.com/opendistro-for-elasticsearch/k-NN/pull/273)) - diff --git a/release-notes/opendistro-for-elasticsearch-knn.release-notes-1.13.0.0.md b/release-notes/opendistro-for-elasticsearch-knn.release-notes-1.13.0.0.md deleted file mode 100644 index 0b0199c97..000000000 --- a/release-notes/opendistro-for-elasticsearch-knn.release-notes-1.13.0.0.md +++ /dev/null @@ -1,34 +0,0 @@ -## Version 1.13.0.0 Release Notes - -Compatible with Elasticsearch 7.10.2 - -### Features - -* Support k-NN similarity functions in painless scripting ([#281](https://github.com/opendistro-for-elasticsearch/k-NN/pull/281)) -* Add support for L1 distance in AKNN, custom scoring and painless scripting ([#310](https://github.com/opendistro-for-elasticsearch/k-NN/pull/310)) - -### Enhancements - -* Upgrade nmslib to 2.0.11 ([#302](https://github.com/opendistro-for-elasticsearch/k-NN/pull/302)) -* Upgrade commons-beanutils ([#297](https://github.com/opendistro-for-elasticsearch/k-NN/pull/297)) - -### Bug Fixes - -* Fix find_path bug in CMakeLists ([#280](https://github.com/opendistro-for-elasticsearch/k-NN/pull/280)) -* Add builder constructor that takes algo params ([#289](https://github.com/opendistro-for-elasticsearch/k-NN/pull/289)) - -### Infrastructure - -* Add arm64 support and correct the naming convention to the new standards ([#299](https://github.com/opendistro-for-elasticsearch/k-NN/pull/299)) -* Run KNN integ tests with security plugin enabled ([#304](https://github.com/opendistro-for-elasticsearch/k-NN/pull/304)) -* Update artifact naming ([#309](https://github.com/opendistro-for-elasticsearch/k-NN/pull/309)) -* Change CD workflow to use new staging bucket for artifacts ([#301](https://github.com/opendistro-for-elasticsearch/k-NN/pull/301)) - -### Documentation - -* Add copyright header ([#307](https://github.com/opendistro-for-elasticsearch/k-NN/pull/307)) - -### Maintenance - -* Upgrade odfe version to 1.13.0 ([#312](https://github.com/opendistro-for-elasticsearch/k-NN/pull/312)) - From c58ba53b0a4dab99c7eca699740c7512eb496686 Mon Sep 17 00:00:00 2001 From: John Mazanec Date: Tue, 30 Apr 2024 17:57:29 -0400 Subject: [PATCH 255/416] Add 2.14 release notes (#1675) Signed-off-by: John Mazanec (cherry picked from commit bd8256fd1d783ff7a13d0a597a3971bb9facadae) --- CHANGELOG.md | 15 +-------------- .../opensearch-knn.release-notes-2.14.0.0.md | 19 +++++++++++++++++++ 2 files changed, 20 insertions(+), 14 deletions(-) create mode 100644 release-notes/opensearch-knn.release-notes-2.14.0.0.md diff --git a/CHANGELOG.md b/CHANGELOG.md index b567d64bc..708ce86ff 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,24 +12,11 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Maintenance ### Refactoring -## [Unreleased 2.x](https://github.com/opensearch-project/k-NN/compare/2.13...2.x) +## [Unreleased 2.x](https://github.com/opensearch-project/k-NN/compare/2.14...2.x) ### Features -* Add Clear Cache API [#740](https://github.com/opensearch-project/k-NN/pull/740) -* Support radial search in k-NN plugin [#1617](https://github.com/opensearch-project/k-NN/pull/1617) -* Support filter and nested field in faiss engine radial search [#1652](https://github.com/opensearch-project/k-NN/pull/1652) ### Enhancements -* Make the HitQueue size more appropriate for exact search [#1549](https://github.com/opensearch-project/k-NN/pull/1549) -* Implemented the Streaming Feature to stream vectors from Java to JNI layer to enable creation of larger segments for vector indices [#1604](https://github.com/opensearch-project/k-NN/pull/1604) -* Remove unnecessary toString conversion of vector field and added some minor optimization in KNNCodec [1613](https://github.com/opensearch-project/k-NN/pull/1613) -* Serialize all models into cluster metadata [#1499](https://github.com/opensearch-project/k-NN/pull/1499) ### Bug Fixes -* Add stored fields for knn_vector type [#1630](https://github.com/opensearch-project/k-NN/pull/1630) -* Ensure ignore_unmapped is set to correct value [#1655](https://github.com/opensearch-project/k-NN/pull/1655) -* Enable script score to work with model based indices [#1649](https://github.com/opensearch-project/k-NN/pull/1649) ### Infrastructure -* Add micro-benchmark module in k-NN plugin for benchmark streaming vectors to JNI layer functionality. [#1583](https://github.com/opensearch-project/k-NN/pull/1583) -* Add arm64 check when SIMD is disabled [#1618](https://github.com/opensearch-project/k-NN/pull/1618) -* Skip rebuild from scratch after cmake is ran [#1636](https://github.com/opensearch-project/k-NN/pull/1636) ### Documentation ### Maintenance ### Refactoring diff --git a/release-notes/opensearch-knn.release-notes-2.14.0.0.md b/release-notes/opensearch-knn.release-notes-2.14.0.0.md new file mode 100644 index 000000000..e48d3784a --- /dev/null +++ b/release-notes/opensearch-knn.release-notes-2.14.0.0.md @@ -0,0 +1,19 @@ +## Version 2.14.0.0 Release Notes + +Compatible with OpenSearch 2.14.0 + +### Features +* Support radial search in k-NN plugin [#1617](https://github.com/opensearch-project/k-NN/pull/1617) +* Support filter and nested field in faiss engine radial search [#1652](https://github.com/opensearch-project/k-NN/pull/1652) +### Enhancements +* Make the HitQueue size more appropriate for exact search [#1549](https://github.com/opensearch-project/k-NN/pull/1549) +* Implement the Streaming Feature to stream vectors from Java to JNI layer to enable creation of larger segments for vector indices [#1604](https://github.com/opensearch-project/k-NN/pull/1604) +* Remove unnecessary toString conversion of vector field and added some minor optimization in KNNCodec [1613](https://github.com/opensearch-project/k-NN/pull/1613) +* Serialize all models into cluster metadata [#1499](https://github.com/opensearch-project/k-NN/pull/1499) +### Bug Fixes +* Add stored fields for knn_vector type [#1630](https://github.com/opensearch-project/k-NN/pull/1630) +* Enable script score to work with model based indices [#1649](https://github.com/opensearch-project/k-NN/pull/1649) +### Infrastructure +* Add micro-benchmark module in k-NN plugin for benchmark streaming vectors to JNI layer functionality. [#1583](https://github.com/opensearch-project/k-NN/pull/1583) +* Add arm64 check when SIMD is disabled [#1618](https://github.com/opensearch-project/k-NN/pull/1618) +* Skip rebuild from scratch after cmake is run [#1636](https://github.com/opensearch-project/k-NN/pull/1636) From 9eb09859a58b9a4ac14638013862e6f7733b8fa1 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Tue, 30 Apr 2024 17:58:28 -0400 Subject: [PATCH 256/416] Revert #1634 #1664 to switch back to CentOS7 (#1674) (#1676) * Revert "Ensure gcc is newer >= 11.2.1 < 12.0.0 when building knn lib (#1664)" This reverts commit da4cd1c6bdaa0c0c6ca3a622f15fe821d6ace281. Signed-off-by: Peter Zhu * Revert "Update gcc restrictions to version 12.0.0 (#1634)" This reverts commit 11342beb11f3fc78c4be681a799695c2e3cfc5a1. Signed-off-by: Peter Zhu --------- Signed-off-by: Peter Zhu (cherry picked from commit f432b0e6681ed978a266f9f5e836cd35a46c481e) Co-authored-by: Peter Zhu --- scripts/build.sh | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/scripts/build.sh b/scripts/build.sh index 0b1419a93..38998d66a 100644 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -103,26 +103,17 @@ if [ "$JAVA_HOME" = "" ]; then echo "SET JAVA_HOME=$JAVA_HOME" fi -# Ensure gcc version is > 11.2.1 and < 12.0.0 for faiss 1.7.4+ / SIMD Neon support on ARM64 compilation +# Ensure gcc version is above 4.9.0 and at least 9.0.0 for faiss 1.7.4+ / SIMD Neon support on ARM64 compilation # https://github.com/opensearch-project/k-NN/issues/975 # https://github.com/opensearch-project/k-NN/issues/1138 # https://github.com/opensearch-project/opensearch-build/issues/4386 -# https://github.com/opensearch-project/opensearch-build/issues/4379#issuecomment-2067191682 -# https://github.com/opensearch-project/opensearch-build/issues/4379#issuecomment-2083623882 -# https://github.com/opensearch-project/opensearch-build/issues/4379#issuecomment-2084133839 GCC_VERSION=`gcc --version | head -n 1 | cut -d ' ' -f3` -GCC_REQUIRED_VERSION=11.2.1 +GCC_REQUIRED_VERSION=9.0.0 COMPARE_VERSION=`echo $GCC_REQUIRED_VERSION $GCC_VERSION | tr ' ' '\n' | sort -V | uniq | head -n 1` if [ "$COMPARE_VERSION" != "$GCC_REQUIRED_VERSION" ]; then echo "gcc version on this env is older than $GCC_REQUIRED_VERSION, exit 1" exit 1 fi -GCC_REQUIRED_VERSION_CEILING=12.0.0 -COMPARE_VERSION_CEILING=`echo $GCC_REQUIRED_VERSION_CEILING $GCC_VERSION | tr ' ' '\n' | sort -V | uniq | tail -n 1` -if [ "$COMPARE_VERSION_CEILING" != "$GCC_REQUIRED_VERSION_CEILING" ] && (echo "$OSTYPE" | grep -i linux); then - echo "gcc version on this env is newer than $GCC_REQUIRED_VERSION_CEILING, exit 1" - exit 1 -fi # Build k-NN lib and plugin through gradle tasks cd $work_dir From 04f23ae5d3622e9697fc8234606e06c0f2b0ea26 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Wed, 1 May 2024 11:56:24 -0500 Subject: [PATCH 257/416] Update 2.14 release notes (#1679) (#1680) Signed-off-by: Naveen Tatikonda (cherry picked from commit dea138ef4dc845861104efe0d2dca46b8ec6a583) Co-authored-by: Naveen Tatikonda --- release-notes/opensearch-knn.release-notes-2.14.0.0.md | 1 + 1 file changed, 1 insertion(+) diff --git a/release-notes/opensearch-knn.release-notes-2.14.0.0.md b/release-notes/opensearch-knn.release-notes-2.14.0.0.md index e48d3784a..113a30600 100644 --- a/release-notes/opensearch-knn.release-notes-2.14.0.0.md +++ b/release-notes/opensearch-knn.release-notes-2.14.0.0.md @@ -3,6 +3,7 @@ Compatible with OpenSearch 2.14.0 ### Features +* Add k-NN clear cache api [#740](https://github.com/opensearch-project/k-NN/pull/740) * Support radial search in k-NN plugin [#1617](https://github.com/opensearch-project/k-NN/pull/1617) * Support filter and nested field in faiss engine radial search [#1652](https://github.com/opensearch-project/k-NN/pull/1652) ### Enhancements From b36ba4594ec0617430439eefb6ef60d377b24416 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Thu, 2 May 2024 11:50:19 -0700 Subject: [PATCH 258/416] Update DEVELOPER_GUIDE.md for M Chip Mac (#1682) (#1683) Signed-off-by: Junqiu Lei (cherry picked from commit a405267aca2605047ed3bb1951a21cce688c2878) Co-authored-by: Junqiu Lei --- DEVELOPER_GUIDE.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index a19586156..ff37b456b 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -99,12 +99,12 @@ cd k-NN cd jni // File changes required -sed -i -e 's/\/usr\/local\/opt\/libomp\//\/opt\/homebrew\/opt\/llvm\//g' CMakeLists.txt -sed -i -e 's/-march=native/-mcpu=apple-m1/g' external/nmslib/similarity_search/CMakeLists.txt +sed -i -e 's/\/usr\/local\/opt\/libomp\//\/opt\/homebrew\/opt\/llvm\//g' cmake/init-faiss.cmake +sed -i -e 's/__aarch64__/__undefine_aarch64__/g' external/faiss/faiss/utils/distances_simd.cpp sed -i -e 's/pragma message WARN/pragma message /g' external/nmslib/similarity_search/src/distcomp_scalar.cc -// Change to apple-m1, apple-m2 and apple-m3 according to your mac chipset +// Change -mcpu value to use chip version according to your M series, for example, -mcpu=apple-m1 +sed -i -e 's/-march=native/-mcpu=apple-m1/g' external/nmslib/similarity_search/CMakeLists.txt sed -i -e 's/-mcpu=apple-a14/-mcpu=apple-m1/g' external/nmslib/python_bindings/setup.py -sed -i -e 's/__aarch64__/__undefine_aarch64__/g' external/faiss/faiss/utils/distances_simd.cpp // Install llvm brew install llvm @@ -214,7 +214,7 @@ cmake . make # To just build the libraries -make opensearchknn_nmslib opensearchknn_nmslib +make opensearchknn_faiss opensearchknn_nmslib ``` The libraries will be placed in the `jni/release` directory. From 23ecce54eff7a95c1e258d0665982b2a2c444952 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Mon, 6 May 2024 12:39:45 -0700 Subject: [PATCH 259/416] Fix build CI failure on MacOS (#1685) (#1689) --- .github/workflows/CI.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 8795ac319..e1d85eb78 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -61,7 +61,7 @@ jobs: echo "avx2 not available on system" su `id -un 1000` -c "whoami && java -version && ./gradlew build -Dsimd.enabled=false" fi - + - name: Upload Coverage Report uses: codecov/codecov-action@v1 @@ -75,7 +75,7 @@ jobs: name: Build and Test k-NN Plugin on MacOS needs: Get-CI-Image-Tag - runs-on: macos-latest + runs-on: macos-12 steps: - name: Checkout k-NN @@ -166,4 +166,3 @@ jobs: - name: Run build run: | ./gradlew.bat build -D'simd.enabled=false' - From 144d2bf6cf44636fc09cc1c722880421884286b8 Mon Sep 17 00:00:00 2001 From: Ryan Bogan Date: Thu, 9 May 2024 10:14:34 -0700 Subject: [PATCH 260/416] Block commas in model description (#1695) Signed-off-by: Ryan Bogan --- CHANGELOG.md | 1 + .../opensearch/knn/indices/ModelMetadata.java | 2 + .../org/opensearch/knn/indices/ModelUtil.java | 6 ++ .../plugin/rest/RestTrainModelHandler.java | 2 + .../knn/indices/ModelMetadataTests.java | 60 +++++++++++++----- .../action/RestTrainModelHandlerIT.java | 63 ++++++++++++++++++- 6 files changed, 119 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 708ce86ff..de4319adf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Features ### Enhancements ### Bug Fixes +* Block commas in model description [#1692](https://github.com/opensearch-project/k-NN/pull/1692) ### Infrastructure ### Documentation ### Maintenance diff --git a/src/main/java/org/opensearch/knn/indices/ModelMetadata.java b/src/main/java/org/opensearch/knn/indices/ModelMetadata.java index fa88c8416..f3a5506cd 100644 --- a/src/main/java/org/opensearch/knn/indices/ModelMetadata.java +++ b/src/main/java/org/opensearch/knn/indices/ModelMetadata.java @@ -67,6 +67,7 @@ public ModelMetadata(StreamInput in) throws IOException { // Description and error may be empty. However, reading the string will work as long as they are not null // which is checked in constructor and setters this.description = in.readString(); + ModelUtil.blockCommasInModelDescription(this.description); this.error = in.readString(); if (IndexUtil.isVersionOnOrAfterMinRequiredVersion(in.getVersion(), IndexUtil.MODEL_NODE_ASSIGNMENT_KEY)) { @@ -123,6 +124,7 @@ public ModelMetadata( this.state = new AtomicReference<>(Objects.requireNonNull(modelState, "modelState must not be null")); this.timestamp = Objects.requireNonNull(timestamp, "timestamp must not be null"); this.description = Objects.requireNonNull(description, "description must not be null"); + ModelUtil.blockCommasInModelDescription(this.description); this.error = Objects.requireNonNull(error, "error must not be null"); this.trainingNodeAssignment = Objects.requireNonNull(trainingNodeAssignment, "node assignment must not be null"); this.methodComponentContext = Objects.requireNonNull(methodComponentContext, "method context must not be null"); diff --git a/src/main/java/org/opensearch/knn/indices/ModelUtil.java b/src/main/java/org/opensearch/knn/indices/ModelUtil.java index 3daaed138..4c6230a46 100644 --- a/src/main/java/org/opensearch/knn/indices/ModelUtil.java +++ b/src/main/java/org/opensearch/knn/indices/ModelUtil.java @@ -16,6 +16,12 @@ */ public class ModelUtil { + public static void blockCommasInModelDescription(String description) { + if (description.contains(",")) { + throw new IllegalArgumentException("Model description cannot contain any commas: ','"); + } + } + public static boolean isModelPresent(ModelMetadata modelMetadata) { return modelMetadata != null; } diff --git a/src/main/java/org/opensearch/knn/plugin/rest/RestTrainModelHandler.java b/src/main/java/org/opensearch/knn/plugin/rest/RestTrainModelHandler.java index a4a0de5de..ebbd7fa9b 100644 --- a/src/main/java/org/opensearch/knn/plugin/rest/RestTrainModelHandler.java +++ b/src/main/java/org/opensearch/knn/plugin/rest/RestTrainModelHandler.java @@ -16,6 +16,7 @@ import org.opensearch.core.xcontent.XContentParser; import org.opensearch.index.mapper.NumberFieldMapper; import org.opensearch.knn.index.KNNMethodContext; +import org.opensearch.knn.indices.ModelUtil; import org.opensearch.knn.plugin.KNNPlugin; import org.opensearch.knn.plugin.transport.TrainingJobRouterAction; import org.opensearch.knn.plugin.transport.TrainingModelRequest; @@ -104,6 +105,7 @@ private TrainingModelRequest createTransportRequest(RestRequest restRequest) thr searchSize = (Integer) NumberFieldMapper.NumberType.INTEGER.parse(parser.objectBytes(), false); } else if (MODEL_DESCRIPTION.equals(fieldName) && ensureNotSet(fieldName, description)) { description = parser.textOrNull(); + ModelUtil.blockCommasInModelDescription(description); } else { throw new IllegalArgumentException("Unable to parse token. \"" + fieldName + "\" is not a valid " + "parameter."); } diff --git a/src/test/java/org/opensearch/knn/indices/ModelMetadataTests.java b/src/test/java/org/opensearch/knn/indices/ModelMetadataTests.java index da56a8421..74715671f 100644 --- a/src/test/java/org/opensearch/knn/indices/ModelMetadataTests.java +++ b/src/test/java/org/opensearch/knn/indices/ModelMetadataTests.java @@ -608,20 +608,7 @@ public void testFromResponseMap() throws IOException { String description = "test-description"; String error = "test-error"; String nodeAssignment = "test-node"; - Map nestedParameters = new HashMap() { - { - put("testNestedKey1", "testNestedString"); - put("testNestedKey2", 1); - } - }; - Map parameters = new HashMap<>() { - { - put("testKey1", "testString"); - put("testKey2", 0); - put("testKey3", new MethodComponentContext("ivf", nestedParameters)); - } - }; - MethodComponentContext methodComponentContext = new MethodComponentContext("hnsw", parameters); + MethodComponentContext methodComponentContext = getMethodComponentContext(); MethodComponentContext emptyMethodComponentContext = MethodComponentContext.EMPTY; ModelMetadata expected = new ModelMetadata( @@ -667,6 +654,51 @@ public void testFromResponseMap() throws IOException { metadataAsMap.put(KNNConstants.MODEL_NODE_ASSIGNMENT, null); metadataAsMap.put(KNNConstants.MODEL_METHOD_COMPONENT_CONTEXT, null); assertEquals(expected2, fromMap); + } + + public void testBlockCommasInDescription() { + KNNEngine knnEngine = KNNEngine.DEFAULT; + SpaceType spaceType = SpaceType.L2; + int dimension = 128; + ModelState modelState = ModelState.TRAINING; + String timestamp = ZonedDateTime.now(ZoneOffset.UTC).toString(); + String description = "Test, comma, description"; + String error = "test-error"; + String nodeAssignment = "test-node"; + MethodComponentContext methodComponentContext = getMethodComponentContext(); + + Exception e = expectThrows( + IllegalArgumentException.class, + () -> new ModelMetadata( + knnEngine, + spaceType, + dimension, + modelState, + timestamp, + description, + error, + nodeAssignment, + methodComponentContext + ) + ); + assertEquals("Model description cannot contain any commas: ','", e.getMessage()); + } + private static MethodComponentContext getMethodComponentContext() { + Map nestedParameters = new HashMap() { + { + put("testNestedKey1", "testNestedString"); + put("testNestedKey2", 1); + } + }; + Map parameters = new HashMap<>() { + { + put("testKey1", "testString"); + put("testKey2", 0); + put("testKey3", new MethodComponentContext("ivf", nestedParameters)); + } + }; + MethodComponentContext methodComponentContext = new MethodComponentContext("hnsw", parameters); + return methodComponentContext; } } diff --git a/src/test/java/org/opensearch/knn/plugin/action/RestTrainModelHandlerIT.java b/src/test/java/org/opensearch/knn/plugin/action/RestTrainModelHandlerIT.java index b2f429e2a..a22e1acb8 100644 --- a/src/test/java/org/opensearch/knn/plugin/action/RestTrainModelHandlerIT.java +++ b/src/test/java/org/opensearch/knn/plugin/action/RestTrainModelHandlerIT.java @@ -13,6 +13,7 @@ import org.apache.http.util.EntityUtils; import org.opensearch.client.Response; +import org.opensearch.client.ResponseException; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentType; @@ -193,7 +194,67 @@ public void testTrainModel_fail_tooMuchData() throws Exception { assertTrainingFails(modelId, 30, 1000); } - public void testTrainModel_success_withId() throws IOException, InterruptedException { + public void testTrainModel_fail_commaInDescription() throws Exception { + // Test checks that training when passing in an id succeeds + + String modelId = "test-model-id"; + String trainingIndexName = "train-index"; + String trainingFieldName = "train-field"; + int dimension = 8; + + // Create a training index and randomly ingest data into it + createBasicKnnIndex(trainingIndexName, trainingFieldName, dimension); + + // Call the train API with this definition: + /* + { + "training_index": "train_index", + "training_field": "train_field", + "dimension": 8, + "description": "this should be allowed to be null", + "method": { + "name":"ivf", + "engine":"faiss", + "space_type": "l2", + "parameters":{ + "nlist":1, + "encoder":{ + "name":"pq", + "parameters":{ + "code_size":2, + "m": 2 + } + } + } + } + } + */ + XContentBuilder builder = XContentFactory.jsonBuilder() + .startObject() + .field(NAME, "ivf") + .field(KNN_ENGINE, "faiss") + .field(METHOD_PARAMETER_SPACE_TYPE, "l2") + .startObject(PARAMETERS) + .field(METHOD_PARAMETER_NLIST, 1) + .startObject(METHOD_ENCODER_PARAMETER) + .field(NAME, "pq") + .startObject(PARAMETERS) + .field(ENCODER_PARAMETER_PQ_CODE_SIZE, 2) + .field(ENCODER_PARAMETER_PQ_M, 2) + .endObject() + .endObject() + .endObject() + .endObject(); + Map method = xContentBuilderToMap(builder); + + Exception e = expectThrows( + ResponseException.class, + () -> trainModel(modelId, trainingIndexName, trainingFieldName, dimension, method, "dummy description, with comma") + ); + assertTrue(e.getMessage().contains("Model description cannot contain any commas: ','")); + } + + public void testTrainModel_success_withId() throws Exception { // Test checks that training when passing in an id succeeds String modelId = "test-model-id"; From 46e405132bb005a1cf15e123cf3211829bedb57b Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Mon, 13 May 2024 11:47:03 -0700 Subject: [PATCH 261/416] Add KnnCircuitBreakerException and modify exception message (#1688) (#1697) * Add KnnCircuitBreakerException and modify exception message Signed-off-by: Ryan Bogan * Add changelog entry and remove star import Signed-off-by: Ryan Bogan * Remove default exception constructor Signed-off-by: Ryan Bogan * Add class description and change parameter Signed-off-by: Ryan Bogan * Fix javadocs Signed-off-by: Ryan Bogan --------- Signed-off-by: Ryan Bogan (cherry picked from commit c315862417747a2a8616facff199327eec8ab083) Co-authored-by: Ryan Bogan --- CHANGELOG.md | 1 + .../knn/index/KnnCircuitBreakerException.java | 65 +++++++++++++++++++ .../index/mapper/KNNVectorFieldMapper.java | 5 +- .../opensearch/knn/index/OpenSearchIT.java | 8 +-- 4 files changed, 73 insertions(+), 6 deletions(-) create mode 100644 src/main/java/org/opensearch/knn/index/KnnCircuitBreakerException.java diff --git a/CHANGELOG.md b/CHANGELOG.md index de4319adf..f3d670f17 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ## [Unreleased 2.x](https://github.com/opensearch-project/k-NN/compare/2.14...2.x) ### Features ### Enhancements +* Add KnnCircuitBreakerException and modify exception message [#1688](https://github.com/opensearch-project/k-NN/pull/1688) ### Bug Fixes * Block commas in model description [#1692](https://github.com/opensearch-project/k-NN/pull/1692) ### Infrastructure diff --git a/src/main/java/org/opensearch/knn/index/KnnCircuitBreakerException.java b/src/main/java/org/opensearch/knn/index/KnnCircuitBreakerException.java new file mode 100644 index 000000000..0bcae8dff --- /dev/null +++ b/src/main/java/org/opensearch/knn/index/KnnCircuitBreakerException.java @@ -0,0 +1,65 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.knn.index; + +/** + * An exception to be thrown when the k-NN circuit breaker is triggered. + */ +public class KnnCircuitBreakerException extends RuntimeException { + + /** + * Constructs a KnnCircuitBreakerException with the specified detail + * message. A detail message is a String that describes this particular + * exception. + * + * @param message the String that contains a detailed message + */ + public KnnCircuitBreakerException(final String message) { + super(message); + } + + /** + * Constructs a new exception with the specified detail message and + * cause. + * + *

Note that the detail message associated with {@code cause} is + * not automatically incorporated in this exception's detail + * message. + * + * @param message the detail message (which is saved for later retrieval + * by the {@link Throwable#getMessage()} method). + * @param cause the cause (which is saved for later retrieval by the + * {@link Throwable#getCause()} method). (A {@code null} value + * is permitted, and indicates that the cause is nonexistent or + * unknown.) + */ + public KnnCircuitBreakerException(String message, Throwable cause) { + super(message, cause); + } + + /** + * Constructs a new exception with the specified cause and a detail + * message of {@code (cause==null ? null : cause.toString())} (which + * typically contains the class and detail message of {@code cause}). + * This constructor is useful for exceptions that are little more than + * wrappers for other throwables (for example, {@link + * java.security.PrivilegedActionException}). + * + * @param cause the cause (which is saved for later retrieval by the + * {@link Throwable#getCause()} method). (A {@code null} value is + * permitted, and indicates that the cause is nonexistent or + * unknown.) + */ + public KnnCircuitBreakerException(Throwable cause) { + super(cause); + } +} diff --git a/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapper.java b/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapper.java index b314a5dae..dd8c145db 100644 --- a/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapper.java +++ b/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapper.java @@ -42,6 +42,7 @@ import org.opensearch.index.query.QueryShardContext; import org.opensearch.index.query.QueryShardException; import org.opensearch.knn.common.KNNConstants; +import org.opensearch.knn.index.KnnCircuitBreakerException; import org.opensearch.knn.index.KNNMethodContext; import org.opensearch.knn.index.KNNSettings; import org.opensearch.knn.index.KNNVectorIndexFieldData; @@ -624,8 +625,8 @@ protected boolean isFaissSQClipToFP16RangeEnabled(MethodComponentContext methodC void validateIfCircuitBreakerIsNotTriggered() { if (KNNSettings.isCircuitBreakerTriggered()) { - throw new IllegalStateException( - "Indexing knn vector fields is rejected as circuit breaker triggered. Check _opendistro/_knn/stats for detailed state" + throw new KnnCircuitBreakerException( + "Parsing the created knn vector fields prior to indexing has failed as the circuit breaker triggered. This indicates that the cluster is low on memory resources and cannot index more documents at the moment. Check _plugins/_knn/stats for the circuit breaker status." ); } } diff --git a/src/test/java/org/opensearch/knn/index/OpenSearchIT.java b/src/test/java/org/opensearch/knn/index/OpenSearchIT.java index f8948db26..d6ecdb0c6 100644 --- a/src/test/java/org/opensearch/knn/index/OpenSearchIT.java +++ b/src/test/java/org/opensearch/knn/index/OpenSearchIT.java @@ -189,8 +189,8 @@ public void testAddDoc_blockedWhenCbTrips() throws Exception { Float[] vector = { 6.0f, 6.0f }; ResponseException ex = expectThrows(ResponseException.class, () -> addKnnDoc(INDEX_NAME, "1", FIELD_NAME, vector)); - String expMessage = "Indexing knn vector fields is rejected as circuit breaker triggered." - + " Check _opendistro/_knn/stats for detailed state"; + String expMessage = + "Parsing the created knn vector fields prior to indexing has failed as the circuit breaker triggered. This indicates that the cluster is low on memory resources and cannot index more documents at the moment. Check _plugins/_knn/stats for the circuit breaker status."; assertThat(EntityUtils.toString(ex.getResponse().getEntity()), containsString(expMessage)); // reset @@ -207,8 +207,8 @@ public void testUpdateDoc_blockedWhenCbTrips() throws Exception { updateClusterSettings("knn.circuit_breaker.triggered", "true"); Float[] updatedVector = { 8.0f, 8.0f }; ResponseException ex = expectThrows(ResponseException.class, () -> updateKnnDoc(INDEX_NAME, "1", FIELD_NAME, vector)); - String expMessage = "Indexing knn vector fields is rejected as circuit breaker triggered." - + " Check _opendistro/_knn/stats for detailed state"; + String expMessage = + "Parsing the created knn vector fields prior to indexing has failed as the circuit breaker triggered. This indicates that the cluster is low on memory resources and cannot index more documents at the moment. Check _plugins/_knn/stats for the circuit breaker status."; assertThat(EntityUtils.toString(ex.getResponse().getEntity()), containsString(expMessage)); // reset From bf8d2a909c9a8db0b0dc6dfd1283c84d2b119c43 Mon Sep 17 00:00:00 2001 From: Junqiu Lei Date: Tue, 14 May 2024 16:18:13 -0700 Subject: [PATCH 262/416] Add stats for radial search (#1684) (#1701) (cherry picked from commit 9a52b2bcd4d7e0a05368d8d689b50971f44c6489) Signed-off-by: Junqiu Lei --- CHANGELOG.md | 1 + .../opensearch/knn/index/VectorQueryType.java | 57 ++++++++ .../knn/index/query/KNNQueryBuilder.java | 37 +++-- .../knn/plugin/stats/KNNCounter.java | 6 +- .../opensearch/knn/plugin/stats/KNNStats.java | 20 +++ .../knn/plugin/stats/StatNames.java | 6 +- .../knn/index/VectorQueryTypeTests.java | 30 ++++ .../plugin/action/RestKNNStatsHandlerIT.java | 128 ++++++++++++++---- 8 files changed, 245 insertions(+), 40 deletions(-) create mode 100644 src/main/java/org/opensearch/knn/index/VectorQueryType.java create mode 100644 src/test/java/org/opensearch/knn/index/VectorQueryTypeTests.java diff --git a/CHANGELOG.md b/CHANGELOG.md index f3d670f17..ee01f2ac7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Features ### Enhancements * Add KnnCircuitBreakerException and modify exception message [#1688](https://github.com/opensearch-project/k-NN/pull/1688) +* Add stats for radial search [#1684](https://github.com/opensearch-project/k-NN/pull/1684) ### Bug Fixes * Block commas in model description [#1692](https://github.com/opensearch-project/k-NN/pull/1692) ### Infrastructure diff --git a/src/main/java/org/opensearch/knn/index/VectorQueryType.java b/src/main/java/org/opensearch/knn/index/VectorQueryType.java new file mode 100644 index 000000000..4697a917e --- /dev/null +++ b/src/main/java/org/opensearch/knn/index/VectorQueryType.java @@ -0,0 +1,57 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index; + +import lombok.Getter; +import org.opensearch.knn.common.KNNConstants; +import org.opensearch.knn.plugin.stats.KNNCounter; + +@Getter +public enum VectorQueryType { + K(KNNConstants.K) { + @Override + public KNNCounter getQueryStatCounter() { + return KNNCounter.KNN_QUERY_REQUESTS; + } + + @Override + public KNNCounter getQueryWithFilterStatCounter() { + return KNNCounter.KNN_QUERY_WITH_FILTER_REQUESTS; + } + }, + MIN_SCORE(KNNConstants.MIN_SCORE) { + @Override + public KNNCounter getQueryStatCounter() { + return KNNCounter.MIN_SCORE_QUERY_REQUESTS; + } + + @Override + public KNNCounter getQueryWithFilterStatCounter() { + return KNNCounter.MIN_SCORE_QUERY_WITH_FILTER_REQUESTS; + } + }, + MAX_DISTANCE(KNNConstants.MAX_DISTANCE) { + @Override + public KNNCounter getQueryStatCounter() { + return KNNCounter.MAX_DISTANCE_QUERY_REQUESTS; + } + + @Override + public KNNCounter getQueryWithFilterStatCounter() { + return KNNCounter.MAX_DISTANCE_QUERY_WITH_FILTER_REQUESTS; + } + }; + + private final String queryTypeName; + + VectorQueryType(String queryTypeName) { + this.queryTypeName = queryTypeName; + } + + public abstract KNNCounter getQueryStatCounter(); + + public abstract KNNCounter getQueryWithFilterStatCounter(); +} diff --git a/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java b/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java index 3d3b0969f..88bcc84bc 100644 --- a/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java +++ b/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java @@ -5,42 +5,43 @@ package org.opensearch.knn.index.query; -import java.io.IOException; -import java.util.Arrays; -import java.util.List; -import java.util.Objects; import lombok.extern.log4j.Log4j2; -import org.apache.lucene.search.MatchNoDocsQuery; -import org.opensearch.core.common.Strings; import org.apache.commons.lang.StringUtils; +import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.Query; import org.opensearch.core.ParseField; import org.opensearch.core.common.ParsingException; +import org.opensearch.core.common.Strings; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.index.mapper.MappedFieldType; import org.opensearch.index.mapper.NumberFieldMapper; +import org.opensearch.index.query.AbstractQueryBuilder; import org.opensearch.index.query.QueryBuilder; +import org.opensearch.index.query.QueryShardContext; import org.opensearch.knn.common.KNNConstants; import org.opensearch.knn.index.KNNMethodContext; import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.VectorDataType; +import org.opensearch.knn.index.VectorQueryType; import org.opensearch.knn.index.mapper.KNNVectorFieldMapper; import org.opensearch.knn.index.util.KNNEngine; import org.opensearch.knn.indices.ModelDao; import org.opensearch.knn.indices.ModelMetadata; import org.opensearch.knn.indices.ModelUtil; -import org.opensearch.knn.plugin.stats.KNNCounter; -import org.opensearch.index.query.AbstractQueryBuilder; -import org.opensearch.index.query.QueryShardContext; -import static org.opensearch.knn.index.IndexUtil.*; +import java.io.IOException; +import java.util.Arrays; +import java.util.List; +import java.util.Objects; + import static org.opensearch.knn.common.KNNConstants.MAX_DISTANCE; import static org.opensearch.knn.common.KNNConstants.MIN_SCORE; import static org.opensearch.knn.common.KNNValidationUtil.validateByteVectorValue; import static org.opensearch.knn.index.IndexUtil.isClusterOnOrAfterMinRequiredVersion; +import static org.opensearch.knn.index.IndexUtil.minimalRequiredVersionMap; import static org.opensearch.knn.index.util.KNNEngine.ENGINES_SUPPORTING_RADIAL_SEARCH; /** @@ -246,7 +247,6 @@ public static KNNQueryBuilder fromXContent(XContentParser parser) throws IOExcep String queryName = null; String currentFieldName = null; XContentParser.Token token; - KNNCounter.KNN_QUERY_REQUESTS.increment(); while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { if (token == XContentParser.Token.FIELD_NAME) { currentFieldName = parser.currentName(); @@ -283,7 +283,6 @@ public static KNNQueryBuilder fromXContent(XContentParser parser) throws IOExcep String tokenName = parser.currentName(); if (FILTER_FIELD.getPreferredName().equals(tokenName)) { log.debug(String.format("Start parsing filter for field [%s]", fieldName)); - KNNCounter.KNN_QUERY_WITH_FILTER_REQUESTS.increment(); // Query filters are supported starting from a certain k-NN version only, exact version is defined by // MINIMAL_SUPPORTED_VERSION_FOR_LUCENE_HNSW_FILTER variable. // Here we're checking if all cluster nodes has at least that version or higher. This check is required @@ -322,7 +321,11 @@ public static KNNQueryBuilder fromXContent(XContentParser parser) throws IOExcep } } - validateSingleQueryType(k, maxDistance, minScore); + VectorQueryType vectorQueryType = validateSingleQueryType(k, maxDistance, minScore); + vectorQueryType.getQueryStatCounter().increment(); + if (filter != null) { + vectorQueryType.getQueryWithFilterStatCounter().increment(); + } KNNQueryBuilder knnQueryBuilder = new KNNQueryBuilder(fieldName, ObjectsToFloats(vector)).filter(filter) .boost(boost) @@ -580,21 +583,27 @@ public String getWriteableName() { return NAME; } - private static void validateSingleQueryType(Integer k, Float distance, Float score) { + private static VectorQueryType validateSingleQueryType(Integer k, Float distance, Float score) { int countSetFields = 0; + VectorQueryType vectorQueryType = null; if (k != null && k != 0) { countSetFields++; + vectorQueryType = VectorQueryType.K; } if (distance != null) { countSetFields++; + vectorQueryType = VectorQueryType.MAX_DISTANCE; } if (score != null) { countSetFields++; + vectorQueryType = VectorQueryType.MIN_SCORE; } if (countSetFields != 1) { throw new IllegalArgumentException(String.format("[%s] requires exactly one of k, distance or score to be set", NAME)); } + + return vectorQueryType; } } diff --git a/src/main/java/org/opensearch/knn/plugin/stats/KNNCounter.java b/src/main/java/org/opensearch/knn/plugin/stats/KNNCounter.java index ce04c9078..3bcc3399c 100644 --- a/src/main/java/org/opensearch/knn/plugin/stats/KNNCounter.java +++ b/src/main/java/org/opensearch/knn/plugin/stats/KNNCounter.java @@ -22,7 +22,11 @@ public enum KNNCounter { SCRIPT_QUERY_ERRORS("script_query_errors"), TRAINING_REQUESTS("training_requests"), TRAINING_ERRORS("training_errors"), - KNN_QUERY_WITH_FILTER_REQUESTS("knn_query_with_filter_requests"); + KNN_QUERY_WITH_FILTER_REQUESTS("knn_query_with_filter_requests"), + MIN_SCORE_QUERY_REQUESTS("min_score_query_requests"), + MIN_SCORE_QUERY_WITH_FILTER_REQUESTS("min_score_query_with_filter_requests"), + MAX_DISTANCE_QUERY_REQUESTS("max_distance_query_requests"), + MAX_DISTANCE_QUERY_WITH_FILTER_REQUESTS("max_distance_query_with_filter_requests"); private String name; private AtomicLong count; diff --git a/src/main/java/org/opensearch/knn/plugin/stats/KNNStats.java b/src/main/java/org/opensearch/knn/plugin/stats/KNNStats.java index 07d129652..3ddc8d4b4 100644 --- a/src/main/java/org/opensearch/knn/plugin/stats/KNNStats.java +++ b/src/main/java/org/opensearch/knn/plugin/stats/KNNStats.java @@ -90,12 +90,32 @@ private Map> buildStatsMap() { } private void addQueryStats(ImmutableMap.Builder> builder) { + // KNN Query Stats builder.put(StatNames.KNN_QUERY_REQUESTS.getName(), new KNNStat<>(false, new KNNCounterSupplier(KNNCounter.KNN_QUERY_REQUESTS))) .put( StatNames.KNN_QUERY_WITH_FILTER_REQUESTS.getName(), new KNNStat<>(false, new KNNCounterSupplier(KNNCounter.KNN_QUERY_WITH_FILTER_REQUESTS)) ); + // Min Score Query Stats + builder.put( + StatNames.MIN_SCORE_QUERY_REQUESTS.getName(), + new KNNStat<>(false, new KNNCounterSupplier(KNNCounter.MIN_SCORE_QUERY_REQUESTS)) + ) + .put( + StatNames.MIN_SCORE_QUERY_WITH_FILTER_REQUESTS.getName(), + new KNNStat<>(false, new KNNCounterSupplier(KNNCounter.MIN_SCORE_QUERY_WITH_FILTER_REQUESTS)) + ); + + // Max Distance Query Stats + builder.put( + StatNames.MAX_DISTANCE_QUERY_REQUESTS.getName(), + new KNNStat<>(false, new KNNCounterSupplier(KNNCounter.MAX_DISTANCE_QUERY_REQUESTS)) + ) + .put( + StatNames.MAX_DISTANCE_QUERY_WITH_FILTER_REQUESTS.getName(), + new KNNStat<>(false, new KNNCounterSupplier(KNNCounter.MAX_DISTANCE_QUERY_WITH_FILTER_REQUESTS)) + ); } private void addNativeMemoryStats(ImmutableMap.Builder> builder) { diff --git a/src/main/java/org/opensearch/knn/plugin/stats/StatNames.java b/src/main/java/org/opensearch/knn/plugin/stats/StatNames.java index e9ed2b126..e7f4fd4a2 100644 --- a/src/main/java/org/opensearch/knn/plugin/stats/StatNames.java +++ b/src/main/java/org/opensearch/knn/plugin/stats/StatNames.java @@ -44,7 +44,11 @@ public enum StatNames { KNN_QUERY_WITH_FILTER_REQUESTS(KNNCounter.KNN_QUERY_WITH_FILTER_REQUESTS.getName()), GRAPH_STATS("graph_stats"), REFRESH("refresh"), - MERGE("merge"); + MERGE("merge"), + MIN_SCORE_QUERY_REQUESTS(KNNCounter.MIN_SCORE_QUERY_REQUESTS.getName()), + MIN_SCORE_QUERY_WITH_FILTER_REQUESTS(KNNCounter.MIN_SCORE_QUERY_WITH_FILTER_REQUESTS.getName()), + MAX_DISTANCE_QUERY_REQUESTS(KNNCounter.MAX_DISTANCE_QUERY_REQUESTS.getName()), + MAX_DISTANCE_QUERY_WITH_FILTER_REQUESTS(KNNCounter.MAX_DISTANCE_QUERY_WITH_FILTER_REQUESTS.getName()); private String name; diff --git a/src/test/java/org/opensearch/knn/index/VectorQueryTypeTests.java b/src/test/java/org/opensearch/knn/index/VectorQueryTypeTests.java new file mode 100644 index 000000000..d0fac3f59 --- /dev/null +++ b/src/test/java/org/opensearch/knn/index/VectorQueryTypeTests.java @@ -0,0 +1,30 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.knn.index; + +import org.opensearch.knn.KNNTestCase; +import org.opensearch.knn.plugin.stats.KNNCounter; + +public class VectorQueryTypeTests extends KNNTestCase { + + public void testGetQueryStatCounter() { + assertEquals(KNNCounter.KNN_QUERY_REQUESTS, VectorQueryType.K.getQueryStatCounter()); + assertEquals(KNNCounter.MIN_SCORE_QUERY_REQUESTS, VectorQueryType.MIN_SCORE.getQueryStatCounter()); + assertEquals(KNNCounter.MAX_DISTANCE_QUERY_REQUESTS, VectorQueryType.MAX_DISTANCE.getQueryStatCounter()); + } + + public void testGetQueryWithFilterStatCounter() { + assertEquals(KNNCounter.KNN_QUERY_WITH_FILTER_REQUESTS, VectorQueryType.K.getQueryWithFilterStatCounter()); + assertEquals(KNNCounter.MIN_SCORE_QUERY_WITH_FILTER_REQUESTS, VectorQueryType.MIN_SCORE.getQueryWithFilterStatCounter()); + assertEquals(KNNCounter.MAX_DISTANCE_QUERY_WITH_FILTER_REQUESTS, VectorQueryType.MAX_DISTANCE.getQueryWithFilterStatCounter()); + } +} diff --git a/src/test/java/org/opensearch/knn/plugin/action/RestKNNStatsHandlerIT.java b/src/test/java/org/opensearch/knn/plugin/action/RestKNNStatsHandlerIT.java index 7d11f2e4a..9f28d5a71 100644 --- a/src/test/java/org/opensearch/knn/plugin/action/RestKNNStatsHandlerIT.java +++ b/src/test/java/org/opensearch/knn/plugin/action/RestKNNStatsHandlerIT.java @@ -5,6 +5,7 @@ package org.opensearch.knn.plugin.action; +import lombok.SneakyThrows; import org.apache.http.util.EntityUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -16,41 +17,24 @@ import org.opensearch.cluster.health.ClusterHealthStatus; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; -import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentType; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.index.query.MatchAllQueryBuilder; import org.opensearch.index.query.QueryBuilder; import org.opensearch.index.query.QueryBuilders; import org.opensearch.knn.KNNRestTestCase; -import org.opensearch.knn.index.query.KNNQueryBuilder; import org.opensearch.knn.index.SpaceType; +import org.opensearch.knn.index.query.KNNQueryBuilder; import org.opensearch.knn.plugin.stats.KNNStats; import org.opensearch.knn.plugin.stats.StatNames; -import org.opensearch.core.rest.RestStatus; import java.io.IOException; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import static org.opensearch.knn.TestUtils.KNN_VECTOR; -import static org.opensearch.knn.TestUtils.PROPERTIES; -import static org.opensearch.knn.TestUtils.VECTOR_TYPE; -import static org.opensearch.knn.common.KNNConstants.FAISS_NAME; -import static org.opensearch.knn.common.KNNConstants.KNN_ENGINE; -import static org.opensearch.knn.common.KNNConstants.LUCENE_NAME; -import static org.opensearch.knn.common.KNNConstants.METHOD_HNSW; -import static org.opensearch.knn.common.KNNConstants.METHOD_IVF; -import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_NLIST; -import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_SPACE_TYPE; -import static org.opensearch.knn.common.KNNConstants.MODEL_ID; -import static org.opensearch.knn.common.KNNConstants.MODEL_INDEX_NAME; -import static org.opensearch.knn.common.KNNConstants.NAME; -import static org.opensearch.knn.common.KNNConstants.NMSLIB_NAME; -import static org.opensearch.knn.common.KNNConstants.PARAMETERS; +import java.util.*; + +import static org.opensearch.knn.TestUtils.*; +import static org.opensearch.knn.common.KNNConstants.*; /** * Integration tests to check the correctness of RestKNNStatsHandler @@ -432,6 +416,95 @@ public void testFieldsByEngineModelTraining() throws Exception { assertTrue(faissField); } + public void testRadialSearchStats_thenSucceed() throws Exception { + createKnnIndex(INDEX_NAME, createKnnIndexMapping(FIELD_NAME, 2, METHOD_HNSW, LUCENE_NAME)); + Float[] vector = { 6.0f, 6.0f }; + addKnnDoc(INDEX_NAME, "1", FIELD_NAME, vector); + + // First search: radial search by min score + XContentBuilder queryBuilderMinScore = XContentFactory.jsonBuilder().startObject().startObject("query"); + queryBuilderMinScore.startObject("knn"); + queryBuilderMinScore.startObject(FIELD_NAME); + queryBuilderMinScore.field("vector", vector); + queryBuilderMinScore.field(MIN_SCORE, 0.95f); + queryBuilderMinScore.endObject(); + queryBuilderMinScore.endObject(); + queryBuilderMinScore.endObject().endObject(); + + Integer minScoreStatBeforeMinScoreSearch = getStatCount(StatNames.MIN_SCORE_QUERY_REQUESTS.getName()); + searchKNNIndex(INDEX_NAME, queryBuilderMinScore, 1); + Integer minScoreStatAfterMinScoreSearch = getStatCount(StatNames.MIN_SCORE_QUERY_REQUESTS.getName()); + + assertEquals(1, minScoreStatAfterMinScoreSearch - minScoreStatBeforeMinScoreSearch); + + // Second search: radial search by min score with filter + XContentBuilder queryBuilderMinScoreWithFilter = XContentFactory.jsonBuilder().startObject().startObject("query"); + queryBuilderMinScoreWithFilter.startObject("knn"); + queryBuilderMinScoreWithFilter.startObject(FIELD_NAME); + queryBuilderMinScoreWithFilter.field("vector", vector); + queryBuilderMinScoreWithFilter.field(MIN_SCORE, 0.95f); + queryBuilderMinScoreWithFilter.field("filter", QueryBuilders.termQuery("_id", "1")); + queryBuilderMinScoreWithFilter.endObject(); + queryBuilderMinScoreWithFilter.endObject(); + queryBuilderMinScoreWithFilter.endObject().endObject(); + + Integer minScoreWithFilterStatBeforeMinScoreWithFilterSearch = getStatCount( + StatNames.MIN_SCORE_QUERY_WITH_FILTER_REQUESTS.getName() + ); + Integer minScoreStatBeforeMinScoreWithFilterSearch = getStatCount(StatNames.MIN_SCORE_QUERY_REQUESTS.getName()); + searchKNNIndex(INDEX_NAME, queryBuilderMinScoreWithFilter, 1); + Integer minScoreWithFilterStatAfterMinScoreWithFilterSearch = getStatCount( + StatNames.MIN_SCORE_QUERY_WITH_FILTER_REQUESTS.getName() + ); + Integer minScoreStatAfterMinScoreWithFilterSearch = getStatCount(StatNames.MIN_SCORE_QUERY_REQUESTS.getName()); + + assertEquals(1, minScoreWithFilterStatAfterMinScoreWithFilterSearch - minScoreWithFilterStatBeforeMinScoreWithFilterSearch); + assertEquals(1, minScoreStatAfterMinScoreWithFilterSearch - minScoreStatBeforeMinScoreWithFilterSearch); + + // Third search: radial search by max distance + XContentBuilder queryBuilderMaxDistance = XContentFactory.jsonBuilder().startObject().startObject("query"); + queryBuilderMaxDistance.startObject("knn"); + queryBuilderMaxDistance.startObject(FIELD_NAME); + queryBuilderMaxDistance.field("vector", vector); + queryBuilderMaxDistance.field(MAX_DISTANCE, 100f); + queryBuilderMaxDistance.endObject(); + queryBuilderMaxDistance.endObject(); + queryBuilderMaxDistance.endObject().endObject(); + + Integer maxDistanceStatBeforeMaxDistanceSearch = getStatCount(StatNames.MAX_DISTANCE_QUERY_REQUESTS.getName()); + searchKNNIndex(INDEX_NAME, queryBuilderMaxDistance, 0); + Integer maxDistanceStatAfterMaxDistanceSearch = getStatCount(StatNames.MAX_DISTANCE_QUERY_REQUESTS.getName()); + + assertEquals(1, maxDistanceStatAfterMaxDistanceSearch - maxDistanceStatBeforeMaxDistanceSearch); + + // Fourth search: radial search by max distance with filter + XContentBuilder queryBuilderMaxDistanceWithFilter = XContentFactory.jsonBuilder().startObject().startObject("query"); + queryBuilderMaxDistanceWithFilter.startObject("knn"); + queryBuilderMaxDistanceWithFilter.startObject(FIELD_NAME); + queryBuilderMaxDistanceWithFilter.field("vector", vector); + queryBuilderMaxDistanceWithFilter.field(MAX_DISTANCE, 100f); + queryBuilderMaxDistanceWithFilter.field("filter", QueryBuilders.termQuery("_id", "1")); + queryBuilderMaxDistanceWithFilter.endObject(); + queryBuilderMaxDistanceWithFilter.endObject(); + queryBuilderMaxDistanceWithFilter.endObject().endObject(); + + Integer maxDistanceWithFilterStatBeforeMaxDistanceWithFilterSearch = getStatCount( + StatNames.MAX_DISTANCE_QUERY_WITH_FILTER_REQUESTS.getName() + ); + Integer maxDistanceStatBeforeMaxDistanceWithFilterSearch = getStatCount(StatNames.MAX_DISTANCE_QUERY_REQUESTS.getName()); + searchKNNIndex(INDEX_NAME, queryBuilderMaxDistanceWithFilter, 0); + Integer maxDistanceWithFilterStatAfterMaxDistanceWithFilterSearch = getStatCount( + StatNames.MAX_DISTANCE_QUERY_WITH_FILTER_REQUESTS.getName() + ); + Integer maxDistanceStatAfterMaxDistanceWithFilterSearch = getStatCount(StatNames.MAX_DISTANCE_QUERY_REQUESTS.getName()); + + assertEquals( + 1, + maxDistanceWithFilterStatAfterMaxDistanceWithFilterSearch - maxDistanceWithFilterStatBeforeMaxDistanceWithFilterSearch + ); + assertEquals(1, maxDistanceStatAfterMaxDistanceWithFilterSearch - maxDistanceStatBeforeMaxDistanceWithFilterSearch); + } + public void trainKnnModel(String modelId, String trainingIndexName, String trainingFieldName, int dimension, String description) throws IOException { XContentBuilder builder = XContentFactory.jsonBuilder() @@ -487,4 +560,11 @@ protected Settings restClientSettings() { return super.restClientSettings(); } } + + @SneakyThrows + private Integer getStatCount(String statName) { + Response response = getKnnStats(Collections.emptyList(), Collections.emptyList()); + String responseBody = EntityUtils.toString(response.getEntity()); + return (Integer) parseNodeStatsResponse(responseBody).get(0).get(statName); + } } From 81b514b499066efe40969e64f9dd3ae9c8dad585 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Wed, 15 May 2024 09:21:22 -0700 Subject: [PATCH 263/416] Increment version to 2.15.0-SNAPSHOT (#1686) Signed-off-by: opensearch-ci-bot Co-authored-by: opensearch-ci-bot --- .../workflows/backwards_compatibility_tests_workflow.yml | 8 ++++---- build.gradle | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/backwards_compatibility_tests_workflow.yml b/.github/workflows/backwards_compatibility_tests_workflow.yml index 43670fe83..48925dbc2 100644 --- a/.github/workflows/backwards_compatibility_tests_workflow.yml +++ b/.github/workflows/backwards_compatibility_tests_workflow.yml @@ -14,8 +14,8 @@ jobs: strategy: matrix: java: [ 11, 17 ] - bwc_version : [ "1.1.0", "1.2.4", "1.3.8", "2.0.1", "2.1.0", "2.2.1", "2.3.0", "2.4.1", "2.5.0", "2.6.0", "2.7.0", "2.8.0", "2.9.0", "2.10.0", "2.11.0", "2.12.0", "2.13.0" ] - opensearch_version : [ "2.14.0-SNAPSHOT" ] + bwc_version : [ "1.1.0", "1.2.4", "1.3.8", "2.0.1", "2.1.0", "2.2.1", "2.3.0", "2.4.1", "2.5.0", "2.6.0", "2.7.0", "2.8.0", "2.9.0", "2.10.0", "2.11.0", "2.12.0", "2.13.0", "2.14.0" ] + opensearch_version : [ "2.15.0-SNAPSHOT" ] name: k-NN Restart-Upgrade BWC Tests runs-on: ubuntu-latest @@ -52,8 +52,8 @@ jobs: strategy: matrix: java: [ 11, 17 ] - bwc_version: [ "1.3.8", "2.0.1", "2.1.0", "2.2.1", "2.3.0", "2.4.1", "2.5.0", "2.6.0", "2.7.0", "2.8.0", "2.9.0", "2.10.0", "2.11.0", "2.12.0", "2.13.0"] - opensearch_version: [ "2.14.0-SNAPSHOT" ] + bwc_version: [ "1.3.8", "2.0.1", "2.1.0", "2.2.1", "2.3.0", "2.4.1", "2.5.0", "2.6.0", "2.7.0", "2.8.0", "2.9.0", "2.10.0", "2.11.0", "2.12.0", "2.13.0", "2.14.0"] + opensearch_version: [ "2.15.0-SNAPSHOT" ] name: k-NN Rolling-Upgrade BWC Tests runs-on: ubuntu-latest diff --git a/build.gradle b/build.gradle index dd2c162c3..a0e194811 100644 --- a/build.gradle +++ b/build.gradle @@ -13,7 +13,7 @@ buildscript { ext { // build.version_qualifier parameter applies to knn plugin artifacts only. OpenSearch version must be set // explicitly as 'opensearch.version' property, for instance opensearch.version=2.0.0-rc1-SNAPSHOT - opensearch_version = System.getProperty("opensearch.version", "2.14.0-SNAPSHOT") + opensearch_version = System.getProperty("opensearch.version", "2.15.0-SNAPSHOT") version_qualifier = System.getProperty("build.version_qualifier", "") opensearch_group = "org.opensearch" isSnapshot = "true" == System.getProperty("build.snapshot", "true") From e6c504761d0c34e481c8a4c914e64e271bd2f3e3 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Wed, 15 May 2024 11:32:24 -0700 Subject: [PATCH 264/416] Updating security reachout email (#1702) Signed-off-by: Varun Lodaya (cherry picked from commit 6ea376bdb233e0be1cbd681297b5e04071657f04) --- SECURITY.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SECURITY.md b/SECURITY.md index 0b85ca04e..be4ac7463 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -1,3 +1,3 @@ ## Reporting a Vulnerability -If you discover a potential security issue in this project we ask that you notify AWS/Amazon Security via our [vulnerability reporting page](http://aws.amazon.com/security/vulnerability-reporting/) or directly via email to aws-security@amazon.com. Please do **not** create a public GitHub issue. \ No newline at end of file +If you discover a potential security issue in this project we ask that you notify OpenSearch Security directly via email to security@opensearch.org. Please do **not** create a public GitHub issue. From 2890521597bea67e764b947ed553ffaf11ee4a7b Mon Sep 17 00:00:00 2001 From: Vijayan Balasubramanian Date: Thu, 16 May 2024 14:19:28 -0700 Subject: [PATCH 265/416] Add luyuncheng as maintainer (#1628) (#1704) Add Yuncheng Lu as maintainer to k-NN repository Signed-off-by: Vijayan Balasubramanian (cherry picked from commit ef962ecace122af12cad6b0554a9b09c2d8a58dd) --- .github/CODEOWNERS | 2 +- MAINTAINERS.md | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 2ca3ba012..b72c1da95 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,2 +1,2 @@ # This should match the owning team set up in https://github.com/orgs/opensearch-project/teams -* @heemin32 @navneet1v @VijayanB @vamshin @jmazanec15 @naveentatikonda @junqiu-lei @martin-gaievski \ No newline at end of file +* @heemin32 @navneet1v @VijayanB @vamshin @jmazanec15 @naveentatikonda @junqiu-lei @martin-gaievski @ryanbogan @luyuncheng diff --git a/MAINTAINERS.md b/MAINTAINERS.md index c285c166c..73fdc028a 100644 --- a/MAINTAINERS.md +++ b/MAINTAINERS.md @@ -13,3 +13,4 @@ This document contains a list of maintainers in this repo. See [opensearch-proje | Navneet Verma | [navneet1v](https://github.com/navneet1v) | Amazon | | Vamshi Vijay Nakkirtha | [vamshin](https://github.com/vamshin) | Amazon | | Vijayan Balasubramanian | [VijayanB](https://github.com/VijayanB) | Amazon | +| Yuncheng Lu | [luyuncheng](https://github.com/luyuncheng) | Bytedance | From 2e3da545dff71c7a1eb127f41547f1543daff78e Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Fri, 17 May 2024 09:19:49 -0700 Subject: [PATCH 266/416] Fix flaky test in Faiss JNI range search (#1705) (#1706) Signed-off-by: Junqiu Lei (cherry picked from commit e584822bac172b95ac208c5a2a7fb75c50e63bce) Co-authored-by: Junqiu Lei --- jni/tests/faiss_wrapper_test.cpp | 33 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 17 deletions(-) diff --git a/jni/tests/faiss_wrapper_test.cpp b/jni/tests/faiss_wrapper_test.cpp index e9316dcc2..4cd3b319e 100644 --- a/jni/tests/faiss_wrapper_test.cpp +++ b/jni/tests/faiss_wrapper_test.cpp @@ -25,6 +25,9 @@ using ::testing::Return; float randomDataMin = -500.0; float randomDataMax = 500.0; +float rangeSearchRandomDataMin = -50; +float rangeSearchRandomDataMax = 50; +float rangeSearchRadius = 20000; TEST(FaissCreateIndexTest, BasicAssertions) { // Define the data @@ -621,13 +624,12 @@ TEST(FaissRangeSearchQueryIndexTest, BasicAssertions) { faiss::idx_t numIds = 200; int dim = 2; std::vector ids = test_util::Range(numIds); - std::vector vectors = test_util::RandomVectors(dim, numIds, randomDataMin, randomDataMax); + std::vector vectors = test_util::RandomVectors(dim, numIds, rangeSearchRandomDataMin, rangeSearchRandomDataMax); faiss::MetricType metricType = faiss::METRIC_L2; std::string method = "HNSW32,Flat"; // Define query data - float radius = 100000.0; int numQueries = 100; std::vector> queries; @@ -635,7 +637,7 @@ TEST(FaissRangeSearchQueryIndexTest, BasicAssertions) { std::vector query; query.reserve(dim); for (int j = 0; j < dim; j++) { - query.push_back(test_util::RandomFloat(randomDataMin, randomDataMax)); + query.push_back(test_util::RandomFloat(rangeSearchRandomDataMin, rangeSearchRandomDataMax)); } queries.push_back(query); } @@ -659,7 +661,7 @@ TEST(FaissRangeSearchQueryIndexTest, BasicAssertions) { knn_jni::faiss_wrapper::RangeSearch( &mockJNIUtil, jniEnv, reinterpret_cast(&createdIndexWithData), - reinterpret_cast(&query), radius, maxResultWindow, nullptr))); + reinterpret_cast(&query), rangeSearchRadius, maxResultWindow, nullptr))); // assert result size is not 0 ASSERT_NE(0, results->size()); @@ -677,13 +679,12 @@ TEST(FaissRangeSearchQueryIndexTest_WhenHitMaxWindowResult, BasicAssertions){ faiss::idx_t numIds = 200; int dim = 2; std::vector ids = test_util::Range(numIds); - std::vector vectors = test_util::RandomVectors(dim, numIds, randomDataMin, randomDataMax); + std::vector vectors = test_util::RandomVectors(dim, numIds, rangeSearchRandomDataMin, rangeSearchRandomDataMax); faiss::MetricType metricType = faiss::METRIC_L2; std::string method = "HNSW32,Flat"; // Define query data - float radius = 100000.0; int numQueries = 100; std::vector> queries; @@ -691,7 +692,7 @@ TEST(FaissRangeSearchQueryIndexTest_WhenHitMaxWindowResult, BasicAssertions){ std::vector query; query.reserve(dim); for (int j = 0; j < dim; j++) { - query.push_back(test_util::RandomFloat(randomDataMin, randomDataMax)); + query.push_back(test_util::RandomFloat(rangeSearchRandomDataMin, rangeSearchRandomDataMax)); } queries.push_back(query); } @@ -715,7 +716,7 @@ TEST(FaissRangeSearchQueryIndexTest_WhenHitMaxWindowResult, BasicAssertions){ knn_jni::faiss_wrapper::RangeSearch( &mockJNIUtil, jniEnv, reinterpret_cast(&createdIndexWithData), - reinterpret_cast(&query), radius, maxResultWindow, nullptr))); + reinterpret_cast(&query), rangeSearchRadius, maxResultWindow, nullptr))); // assert result size is not 0 ASSERT_NE(0, results->size()); @@ -734,13 +735,12 @@ TEST(FaissRangeSearchQueryIndexTestWithFilterTest, BasicAssertions) { faiss::idx_t numIds = 200; int dim = 2; std::vector ids = test_util::Range(numIds); - std::vector vectors = test_util::RandomVectors(dim, numIds, randomDataMin, randomDataMax); + std::vector vectors = test_util::RandomVectors(dim, numIds, rangeSearchRandomDataMin, rangeSearchRandomDataMax); faiss::MetricType metricType = faiss::METRIC_L2; std::string method = "HNSW32,Flat"; // Define query data - float radius = 100000.0; int numQueries = 100; std::vector> queries; @@ -748,7 +748,7 @@ TEST(FaissRangeSearchQueryIndexTestWithFilterTest, BasicAssertions) { std::vector query; query.reserve(dim); for (int j = 0; j < dim; j++) { - query.push_back(test_util::RandomFloat(randomDataMin, randomDataMax)); + query.push_back(test_util::RandomFloat(rangeSearchRandomDataMin, rangeSearchRandomDataMax)); } queries.push_back(query); } @@ -767,7 +767,7 @@ TEST(FaissRangeSearchQueryIndexTestWithFilterTest, BasicAssertions) { std::vector bitmap(num_bits,0); std::vector filterIds; - for (int64_t i = 154; i < 163; i++) { + for (int64_t i = 1; i < 50; i++) { filterIds.push_back(i); test_util::setBitSet(i, bitmap.data(), bitmap.size()); } @@ -782,7 +782,7 @@ TEST(FaissRangeSearchQueryIndexTestWithFilterTest, BasicAssertions) { knn_jni::faiss_wrapper::RangeSearchWithFilter( &mockJNIUtil, jniEnv, reinterpret_cast(&createdIndexWithData), - reinterpret_cast(&query), radius, maxResultWindow, + reinterpret_cast(&query), rangeSearchRadius, maxResultWindow, reinterpret_cast(&bitmap), 0, nullptr))); // assert result size is not 0 @@ -814,7 +814,7 @@ TEST(FaissRangeSearchQueryIndexTestWithParentFilterTest, BasicAssertions) { } ids.push_back(i); for (int j = 0; j < dim; j++) { - vectors.push_back(test_util::RandomFloat(-500.0, 500.0)); + vectors.push_back(test_util::RandomFloat(rangeSearchRandomDataMin, rangeSearchRandomDataMax)); } } @@ -822,7 +822,6 @@ TEST(FaissRangeSearchQueryIndexTestWithParentFilterTest, BasicAssertions) { std::string method = "HNSW32,Flat"; // Define query data - float radius = 100000.0; int numQueries = 1; std::vector> queries; @@ -830,7 +829,7 @@ TEST(FaissRangeSearchQueryIndexTestWithParentFilterTest, BasicAssertions) { std::vector query; query.reserve(dim); for (int j = 0; j < dim; j++) { - query.push_back(test_util::RandomFloat(-500.0, 500.0)); + query.push_back(test_util::RandomFloat(rangeSearchRandomDataMin, rangeSearchRandomDataMax)); } queries.push_back(query); } @@ -858,7 +857,7 @@ TEST(FaissRangeSearchQueryIndexTestWithParentFilterTest, BasicAssertions) { knn_jni::faiss_wrapper::RangeSearchWithFilter( &mockJNIUtil, jniEnv, reinterpret_cast(&createdIndexWithData), - reinterpret_cast(&query), radius, maxResultWindow, nullptr, 0, + reinterpret_cast(&query), rangeSearchRadius, maxResultWindow, nullptr, 0, reinterpret_cast(&parentIds)))); // assert result size is not 0 From c2c166043ae5af5d44a9a6b8eaa173a232ef4260 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Tue, 21 May 2024 08:06:49 -0700 Subject: [PATCH 267/416] Support script score when doc value is disabled and fix misusing DISI (#1711) * Revert "Revert 'Support script score when doc value is disabled' (#1662)" This reverts commit bd2f403cb1ff439e6f1d88efce71464682472544. Signed-off-by: panguixin * fix misusing doc value Signed-off-by: panguixin * add changelog Signed-off-by: panguixin --------- Signed-off-by: panguixin (cherry picked from commit 4d59d4c898621e19944f5de42c87a3f9228b0574) Co-authored-by: panguixin --- CHANGELOG.md | 1 + .../knn/index/KNNVectorDVLeafFieldData.java | 28 ++++- .../knn/index/KNNVectorScriptDocValues.java | 117 ++++++++++++++++-- .../index/KNNVectorScriptDocValuesTests.java | 61 +++++++-- .../knn/index/VectorDataTypeTests.java | 4 +- .../plugin/script/KNNScoringUtilTests.java | 2 +- .../knn/plugin/script/KNNScriptScoringIT.java | 53 +++++--- .../knn/plugin/script/PainlessScriptIT.java | 4 +- .../org/opensearch/knn/KNNRestTestCase.java | 10 +- 9 files changed, 234 insertions(+), 46 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ee01f2ac7..3a7e1d6ac 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Enhancements * Add KnnCircuitBreakerException and modify exception message [#1688](https://github.com/opensearch-project/k-NN/pull/1688) * Add stats for radial search [#1684](https://github.com/opensearch-project/k-NN/pull/1684) +* Support script score when doc value is disabled and fix misusing DISI [#1696](https://github.com/opensearch-project/k-NN/pull/1696) ### Bug Fixes * Block commas in model description [#1692](https://github.com/opensearch-project/k-NN/pull/1692) ### Infrastructure diff --git a/src/main/java/org/opensearch/knn/index/KNNVectorDVLeafFieldData.java b/src/main/java/org/opensearch/knn/index/KNNVectorDVLeafFieldData.java index f4caa4f20..85f037c0f 100644 --- a/src/main/java/org/opensearch/knn/index/KNNVectorDVLeafFieldData.java +++ b/src/main/java/org/opensearch/knn/index/KNNVectorDVLeafFieldData.java @@ -5,9 +5,10 @@ package org.opensearch.knn.index; -import org.apache.lucene.index.BinaryDocValues; import org.apache.lucene.index.DocValues; +import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.LeafReader; +import org.apache.lucene.search.DocIdSetIterator; import org.opensearch.index.fielddata.LeafFieldData; import org.opensearch.index.fielddata.ScriptDocValues; import org.opensearch.index.fielddata.SortedBinaryDocValues; @@ -39,10 +40,29 @@ public long ramBytesUsed() { @Override public ScriptDocValues getScriptValues() { try { - BinaryDocValues values = DocValues.getBinary(reader, fieldName); - return new KNNVectorScriptDocValues(values, fieldName, vectorDataType); + FieldInfo fieldInfo = reader.getFieldInfos().fieldInfo(fieldName); + if (fieldInfo == null) { + return KNNVectorScriptDocValues.emptyValues(fieldName, vectorDataType); + } + + DocIdSetIterator values; + if (fieldInfo.hasVectorValues()) { + switch (fieldInfo.getVectorEncoding()) { + case FLOAT32: + values = reader.getFloatVectorValues(fieldName); + break; + case BYTE: + values = reader.getByteVectorValues(fieldName); + break; + default: + throw new IllegalStateException("Unsupported Lucene vector encoding: " + fieldInfo.getVectorEncoding()); + } + } else { + values = DocValues.getBinary(reader, fieldName); + } + return KNNVectorScriptDocValues.create(values, fieldName, vectorDataType); } catch (IOException e) { - throw new IllegalStateException("Cannot load doc values for knn vector field: " + fieldName, e); + throw new IllegalStateException("Cannot load values for knn vector field: " + fieldName, e); } } diff --git a/src/main/java/org/opensearch/knn/index/KNNVectorScriptDocValues.java b/src/main/java/org/opensearch/knn/index/KNNVectorScriptDocValues.java index 349988c93..55ff65516 100644 --- a/src/main/java/org/opensearch/knn/index/KNNVectorScriptDocValues.java +++ b/src/main/java/org/opensearch/knn/index/KNNVectorScriptDocValues.java @@ -6,30 +6,40 @@ package org.opensearch.knn.index; import java.io.IOException; +import java.util.Objects; +import lombok.AccessLevel; import lombok.Getter; import lombok.RequiredArgsConstructor; import org.apache.lucene.index.BinaryDocValues; +import org.apache.lucene.index.ByteVectorValues; +import org.apache.lucene.index.FloatVectorValues; +import org.apache.lucene.search.DocIdSetIterator; import org.opensearch.ExceptionsHelper; import org.opensearch.index.fielddata.ScriptDocValues; -import java.io.IOException; - -@RequiredArgsConstructor -public final class KNNVectorScriptDocValues extends ScriptDocValues { +@RequiredArgsConstructor(access = AccessLevel.PRIVATE) +public abstract class KNNVectorScriptDocValues extends ScriptDocValues { - private final BinaryDocValues binaryDocValues; + private final DocIdSetIterator vectorValues; private final String fieldName; @Getter private final VectorDataType vectorDataType; private boolean docExists = false; + private int lastDocID = -1; @Override public void setNextDocId(int docId) throws IOException { - if (binaryDocValues.advanceExact(docId)) { - docExists = true; - return; + if (docId < lastDocID) { + throw new IllegalArgumentException("docs were sent out-of-order: lastDocID=" + lastDocID + " vs docID=" + docId); + } + + lastDocID = docId; + + int curDocID = vectorValues.docID(); + if (lastDocID > curDocID) { + curDocID = vectorValues.advance(docId); } - docExists = false; + docExists = lastDocID == curDocID; } public float[] getValue() { @@ -44,12 +54,14 @@ public float[] getValue() { throw new IllegalStateException(errorMessage); } try { - return vectorDataType.getVectorFromBytesRef(binaryDocValues.binaryValue()); + return doGetValue(); } catch (IOException e) { throw ExceptionsHelper.convertToOpenSearchException(e); } } + protected abstract float[] doGetValue() throws IOException; + @Override public int size() { return docExists ? 1 : 0; @@ -59,4 +71,89 @@ public int size() { public float[] get(int i) { throw new UnsupportedOperationException("knn vector does not support this operation"); } + + /** + * Creates a KNNVectorScriptDocValues object based on the provided parameters. + * + * @param values The DocIdSetIterator representing the vector values. + * @param fieldName The name of the field. + * @param vectorDataType The data type of the vector. + * @return A KNNVectorScriptDocValues object based on the type of the values. + * @throws IllegalArgumentException If the type of values is unsupported. + */ + public static KNNVectorScriptDocValues create(DocIdSetIterator values, String fieldName, VectorDataType vectorDataType) { + Objects.requireNonNull(values, "values must not be null"); + if (values instanceof ByteVectorValues) { + return new KNNByteVectorScriptDocValues((ByteVectorValues) values, fieldName, vectorDataType); + } else if (values instanceof FloatVectorValues) { + return new KNNFloatVectorScriptDocValues((FloatVectorValues) values, fieldName, vectorDataType); + } else if (values instanceof BinaryDocValues) { + return new KNNNativeVectorScriptDocValues((BinaryDocValues) values, fieldName, vectorDataType); + } else { + throw new IllegalArgumentException("Unsupported values type: " + values.getClass()); + } + } + + private static final class KNNByteVectorScriptDocValues extends KNNVectorScriptDocValues { + private final ByteVectorValues values; + + KNNByteVectorScriptDocValues(ByteVectorValues values, String field, VectorDataType type) { + super(values, field, type); + this.values = values; + } + + @Override + protected float[] doGetValue() throws IOException { + byte[] bytes = values.vectorValue(); + float[] value = new float[bytes.length]; + for (int i = 0; i < bytes.length; i++) { + value[i] = (float) bytes[i]; + } + return value; + } + } + + private static final class KNNFloatVectorScriptDocValues extends KNNVectorScriptDocValues { + private final FloatVectorValues values; + + KNNFloatVectorScriptDocValues(FloatVectorValues values, String field, VectorDataType type) { + super(values, field, type); + this.values = values; + } + + @Override + protected float[] doGetValue() throws IOException { + return values.vectorValue(); + } + } + + private static final class KNNNativeVectorScriptDocValues extends KNNVectorScriptDocValues { + private final BinaryDocValues values; + + KNNNativeVectorScriptDocValues(BinaryDocValues values, String field, VectorDataType type) { + super(values, field, type); + this.values = values; + } + + @Override + protected float[] doGetValue() throws IOException { + return getVectorDataType().getVectorFromBytesRef(values.binaryValue()); + } + } + + /** + * Creates an empty KNNVectorScriptDocValues object based on the provided field name and vector data type. + * + * @param fieldName The name of the field. + * @param type The data type of the vector. + * @return An empty KNNVectorScriptDocValues object. + */ + public static KNNVectorScriptDocValues emptyValues(String fieldName, VectorDataType type) { + return new KNNVectorScriptDocValues(DocIdSetIterator.empty(), fieldName, type) { + @Override + protected float[] doGetValue() throws IOException { + throw new UnsupportedOperationException("empty values"); + } + }; + } } diff --git a/src/test/java/org/opensearch/knn/index/KNNVectorScriptDocValuesTests.java b/src/test/java/org/opensearch/knn/index/KNNVectorScriptDocValuesTests.java index 3f98a9136..66e2893c0 100644 --- a/src/test/java/org/opensearch/knn/index/KNNVectorScriptDocValuesTests.java +++ b/src/test/java/org/opensearch/knn/index/KNNVectorScriptDocValuesTests.java @@ -5,7 +5,15 @@ package org.opensearch.knn.index; -import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.KnnByteVectorField; +import org.apache.lucene.document.KnnFloatVectorField; +import org.apache.lucene.index.BinaryDocValues; +import org.apache.lucene.index.ByteVectorValues; +import org.apache.lucene.index.DocValues; +import org.apache.lucene.index.FloatVectorValues; +import org.apache.lucene.index.LeafReader; +import org.apache.lucene.search.DocIdSetIterator; import org.opensearch.knn.KNNTestCase; import org.apache.lucene.tests.analysis.MockAnalyzer; import org.apache.lucene.document.BinaryDocValuesField; @@ -33,26 +41,39 @@ public class KNNVectorScriptDocValuesTests extends KNNTestCase { public void setUp() throws Exception { super.setUp(); directory = newDirectory(); - createKNNVectorDocument(directory); + Class valuesClass = randomFrom(BinaryDocValues.class, ByteVectorValues.class, FloatVectorValues.class); + createKNNVectorDocument(directory, valuesClass); reader = DirectoryReader.open(directory); - LeafReaderContext leafReaderContext = reader.getContext().leaves().get(0); - scriptDocValues = new KNNVectorScriptDocValues( - leafReaderContext.reader().getBinaryDocValues(MOCK_INDEX_FIELD_NAME), - MOCK_INDEX_FIELD_NAME, - VectorDataType.FLOAT - ); + LeafReader leafReader = reader.getContext().leaves().get(0).reader(); + DocIdSetIterator vectorValues; + if (BinaryDocValues.class.equals(valuesClass)) { + vectorValues = DocValues.getBinary(leafReader, MOCK_INDEX_FIELD_NAME); + } else if (ByteVectorValues.class.equals(valuesClass)) { + vectorValues = leafReader.getByteVectorValues(MOCK_INDEX_FIELD_NAME); + } else { + vectorValues = leafReader.getFloatVectorValues(MOCK_INDEX_FIELD_NAME); + } + + scriptDocValues = KNNVectorScriptDocValues.create(vectorValues, MOCK_INDEX_FIELD_NAME, VectorDataType.FLOAT); } - private void createKNNVectorDocument(Directory directory) throws IOException { + private void createKNNVectorDocument(Directory directory, Class valuesClass) throws IOException { IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random())); IndexWriter writer = new IndexWriter(directory, conf); Document knnDocument = new Document(); - knnDocument.add( - new BinaryDocValuesField( + Field field; + if (BinaryDocValues.class.equals(valuesClass)) { + field = new BinaryDocValuesField( MOCK_INDEX_FIELD_NAME, new VectorField(MOCK_INDEX_FIELD_NAME, SAMPLE_VECTOR_DATA, new FieldType()).binaryValue() - ) - ); + ); + } else if (ByteVectorValues.class.equals(valuesClass)) { + field = new KnnByteVectorField(MOCK_INDEX_FIELD_NAME, SAMPLE_BYTE_VECTOR_DATA); + } else { + field = new KnnFloatVectorField(MOCK_INDEX_FIELD_NAME, SAMPLE_VECTOR_DATA); + } + + knnDocument.add(field); writer.addDocument(knnDocument); writer.commit(); writer.close(); @@ -84,4 +105,18 @@ public void testSize() throws IOException { public void testGet() throws IOException { expectThrows(UnsupportedOperationException.class, () -> scriptDocValues.get(0)); } + + public void testUnsupportedValues() throws IOException { + expectThrows( + IllegalArgumentException.class, + () -> KNNVectorScriptDocValues.create(DocValues.emptyNumeric(), MOCK_INDEX_FIELD_NAME, VectorDataType.FLOAT) + ); + } + + public void testEmptyValues() throws IOException { + KNNVectorScriptDocValues values = KNNVectorScriptDocValues.emptyValues(MOCK_INDEX_FIELD_NAME, VectorDataType.FLOAT); + assertEquals(0, values.size()); + scriptDocValues.setNextDocId(0); + assertEquals(0, values.size()); + } } diff --git a/src/test/java/org/opensearch/knn/index/VectorDataTypeTests.java b/src/test/java/org/opensearch/knn/index/VectorDataTypeTests.java index 4423c85d8..19270717d 100644 --- a/src/test/java/org/opensearch/knn/index/VectorDataTypeTests.java +++ b/src/test/java/org/opensearch/knn/index/VectorDataTypeTests.java @@ -57,7 +57,7 @@ private KNNVectorScriptDocValues getKNNFloatVectorScriptDocValues() { createKNNFloatVectorDocument(directory); reader = DirectoryReader.open(directory); LeafReaderContext leafReaderContext = reader.getContext().leaves().get(0); - return new KNNVectorScriptDocValues( + return KNNVectorScriptDocValues.create( leafReaderContext.reader().getBinaryDocValues(VectorDataTypeTests.MOCK_FLOAT_INDEX_FIELD_NAME), VectorDataTypeTests.MOCK_FLOAT_INDEX_FIELD_NAME, VectorDataType.FLOAT @@ -70,7 +70,7 @@ private KNNVectorScriptDocValues getKNNByteVectorScriptDocValues() { createKNNByteVectorDocument(directory); reader = DirectoryReader.open(directory); LeafReaderContext leafReaderContext = reader.getContext().leaves().get(0); - return new KNNVectorScriptDocValues( + return KNNVectorScriptDocValues.create( leafReaderContext.reader().getBinaryDocValues(VectorDataTypeTests.MOCK_BYTE_INDEX_FIELD_NAME), VectorDataTypeTests.MOCK_BYTE_INDEX_FIELD_NAME, VectorDataType.BYTE diff --git a/src/test/java/org/opensearch/knn/plugin/script/KNNScoringUtilTests.java b/src/test/java/org/opensearch/knn/plugin/script/KNNScoringUtilTests.java index 8c43a4acf..22110accd 100644 --- a/src/test/java/org/opensearch/knn/plugin/script/KNNScoringUtilTests.java +++ b/src/test/java/org/opensearch/knn/plugin/script/KNNScoringUtilTests.java @@ -280,7 +280,7 @@ public KNNVectorScriptDocValues getScriptDocValues(String fieldName) throws IOEx if (scriptDocValues == null) { reader = DirectoryReader.open(directory); LeafReaderContext leafReaderContext = reader.getContext().leaves().get(0); - scriptDocValues = new KNNVectorScriptDocValues( + scriptDocValues = KNNVectorScriptDocValues.create( leafReaderContext.reader().getBinaryDocValues(fieldName), fieldName, VectorDataType.FLOAT diff --git a/src/test/java/org/opensearch/knn/plugin/script/KNNScriptScoringIT.java b/src/test/java/org/opensearch/knn/plugin/script/KNNScriptScoringIT.java index 2c18bfa90..9684d1705 100644 --- a/src/test/java/org/opensearch/knn/plugin/script/KNNScriptScoringIT.java +++ b/src/test/java/org/opensearch/knn/plugin/script/KNNScriptScoringIT.java @@ -7,6 +7,7 @@ import java.util.function.BiFunction; import java.util.function.Function; +import org.opensearch.ExceptionsHelper; import org.opensearch.knn.KNNRestTestCase; import org.opensearch.knn.KNNResult; import org.opensearch.knn.common.KNNConstants; @@ -194,7 +195,7 @@ public void testUnequalDimensions() throws Exception { } @SuppressWarnings("unchecked") - public void testKNNScoreforNonVectorDocument() throws Exception { + public void testKNNScoreForNonVectorDocument() throws Exception { /* * Create knn index and populate data */ @@ -590,7 +591,7 @@ public void testKNNScriptScoreOnModelBasedIndex() throws Exception { if (spaceType != SpaceType.HAMMING_BIT) { final float[] queryVector = randomVector(dimensions); final BiFunction scoreFunction = getScoreFunction(spaceType, queryVector); - createIndexAndAssertScriptScore(testMapping, spaceType, scoreFunction, dimensions, queryVector); + createIndexAndAssertScriptScore(testMapping, spaceType, scoreFunction, dimensions, queryVector, true); } } } @@ -603,7 +604,16 @@ private List createMappers(int dimensions) throws Exception { dimensions, KNNConstants.METHOD_HNSW, KNNEngine.LUCENE.getName(), - SpaceType.DEFAULT.getValue() + SpaceType.DEFAULT.getValue(), + true + ), + createKnnIndexMapping( + FIELD_NAME, + dimensions, + KNNConstants.METHOD_HNSW, + KNNEngine.LUCENE.getName(), + SpaceType.DEFAULT.getValue(), + false ) ); } @@ -616,12 +626,22 @@ private float[] randomVector(int dimensions) { return vector; } - private Map createDataset(Function scoreFunction, int dimensions, int numDocs) { - final Map dataset = new HashMap<>(numDocs); - for (int i = 0; i < numDocs; i++) { + private Map createDataset( + Function scoreFunction, + int dimensions, + int numDocsWithField, + boolean dense + ) { + final Map dataset = new HashMap<>(dense ? numDocsWithField : numDocsWithField * 3); + int id = 0; + for (int i = 0; i < numDocsWithField; i++) { + final int dummyDocs = dense ? 0 : randomIntBetween(2, 5); + for (int j = 0; j < dummyDocs; j++) { + dataset.put(Integer.toString(id++), null); + } final float[] vector = randomVector(dimensions); final float score = scoreFunction.apply(vector); - dataset.put(Integer.toString(i), new KNNResult(Integer.toString(i), vector, score)); + dataset.put(Integer.toString(id), new KNNResult(Integer.toString(id++), vector, score)); } return dataset; } @@ -660,7 +680,8 @@ private void testKNNScriptScore(SpaceType spaceType) throws Exception { final float[] queryVector = randomVector(dims); final BiFunction scoreFunction = getScoreFunction(spaceType, queryVector); for (String mapper : createMappers(dims)) { - createIndexAndAssertScriptScore(mapper, spaceType, scoreFunction, dims, queryVector); + createIndexAndAssertScriptScore(mapper, spaceType, scoreFunction, dims, queryVector, true); + createIndexAndAssertScriptScore(mapper, spaceType, scoreFunction, dims, queryVector, false); } } @@ -669,16 +690,20 @@ private void createIndexAndAssertScriptScore( SpaceType spaceType, BiFunction scoreFunction, int dimensions, - float[] queryVector + float[] queryVector, + boolean dense ) throws Exception { /* * Create knn index and populate data */ createKnnIndex(INDEX_NAME, mapper); - Map dataset = createDataset(v -> scoreFunction.apply(queryVector, v), dimensions, randomIntBetween(4, 10)); - for (Map.Entry entry : dataset.entrySet()) { - addKnnDoc(INDEX_NAME, entry.getKey(), FIELD_NAME, entry.getValue().getVector()); - } + final int numDocsWithField = randomIntBetween(4, 10); + Map dataset = createDataset(v -> scoreFunction.apply(queryVector, v), dimensions, numDocsWithField, dense); + final float[] dummyVector = new float[1]; + dataset.forEach((k, v) -> { + final float[] vector = (v != null) ? v.getVector() : dummyVector; + ExceptionsHelper.catchAsRuntimeException(() -> addKnnDoc(INDEX_NAME, k, (v != null) ? FIELD_NAME : "dummy", vector)); + }); /** * Construct Search Request @@ -694,7 +719,7 @@ private void createIndexAndAssertScriptScore( params.put("field", FIELD_NAME); params.put("query_value", queryVector); params.put("space_type", spaceType.getValue()); - Request request = constructKNNScriptQueryRequest(INDEX_NAME, qb, params); + Request request = constructKNNScriptQueryRequest(INDEX_NAME, qb, params, numDocsWithField); Response response = client().performRequest(request); assertEquals(request.getEndpoint() + ": failed", RestStatus.OK, RestStatus.fromCode(response.getStatusLine().getStatusCode())); diff --git a/src/test/java/org/opensearch/knn/plugin/script/PainlessScriptIT.java b/src/test/java/org/opensearch/knn/plugin/script/PainlessScriptIT.java index e87b9771e..47e914d04 100644 --- a/src/test/java/org/opensearch/knn/plugin/script/PainlessScriptIT.java +++ b/src/test/java/org/opensearch/knn/plugin/script/PainlessScriptIT.java @@ -563,7 +563,9 @@ public void testL2ScriptingWithLuceneBackedIndex() throws Exception { new MethodComponentContext(METHOD_HNSW, Collections.emptyMap()) ); properties.add( - new MappingProperty(FIELD_NAME, KNNVectorFieldMapper.CONTENT_TYPE).dimension("2").knnMethodContext(knnMethodContext) + new MappingProperty(FIELD_NAME, KNNVectorFieldMapper.CONTENT_TYPE).dimension("2") + .knnMethodContext(knnMethodContext) + .docValues(randomBoolean()) ); String source = String.format("1/(1 + l2Squared([1.0f, 1.0f], doc['%s']))", FIELD_NAME); diff --git a/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java b/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java index c2ebde0f5..05e2a5320 100644 --- a/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java +++ b/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java @@ -40,6 +40,7 @@ import org.opensearch.index.query.functionscore.ScriptScoreQueryBuilder; import org.opensearch.core.rest.RestStatus; import org.opensearch.script.Script; +import org.opensearch.search.SearchService; import org.opensearch.search.aggregations.metrics.ScriptedMetricAggregationBuilder; import javax.management.MBeanServerInvocationHandler; @@ -942,9 +943,16 @@ protected Request constructScriptScoreContextSearchRequest( } protected Request constructKNNScriptQueryRequest(String indexName, QueryBuilder qb, Map params) throws Exception { + return constructKNNScriptQueryRequest(indexName, qb, params, SearchService.DEFAULT_SIZE); + } + + protected Request constructKNNScriptQueryRequest(String indexName, QueryBuilder qb, Map params, int size) + throws Exception { Script script = new Script(Script.DEFAULT_SCRIPT_TYPE, KNNScoringScriptEngine.NAME, KNNScoringScriptEngine.SCRIPT_SOURCE, params); ScriptScoreQueryBuilder sc = new ScriptScoreQueryBuilder(qb, script); - XContentBuilder builder = XContentFactory.jsonBuilder().startObject().startObject("query"); + XContentBuilder builder = XContentFactory.jsonBuilder().startObject(); + builder.field("size", size); + builder.startObject("query"); builder.startObject("script_score"); builder.field("query"); sc.query().toXContent(builder, ToXContent.EMPTY_PARAMS); From 40bf38852a268c14b5157d8156a7a391726f9ed9 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Thu, 23 May 2024 09:37:12 -0700 Subject: [PATCH 268/416] Update threshold value after new result is added (#1715) (#1716) Signed-off-by: Heemin Kim (cherry picked from commit 9023604d8abbfa60b99e6b82cd3e03c683c4d7a3) Co-authored-by: Heemin Kim --- CHANGELOG.md | 1 + ...Custom-patch-to-support-multi-vector.patch | 110 ++++++++++++------ .../opensearch/knn/index/NestedSearchIT.java | 8 +- 3 files changed, 84 insertions(+), 35 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3a7e1d6ac..5c837b493 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), * Support script score when doc value is disabled and fix misusing DISI [#1696](https://github.com/opensearch-project/k-NN/pull/1696) ### Bug Fixes * Block commas in model description [#1692](https://github.com/opensearch-project/k-NN/pull/1692) +* Update threshold value after new result is added [#1715](https://github.com/opensearch-project/k-NN/pull/1715) ### Infrastructure ### Documentation ### Maintenance diff --git a/jni/patches/faiss/0001-Custom-patch-to-support-multi-vector.patch b/jni/patches/faiss/0001-Custom-patch-to-support-multi-vector.patch index a22e28130..227630c63 100644 --- a/jni/patches/faiss/0001-Custom-patch-to-support-multi-vector.patch +++ b/jni/patches/faiss/0001-Custom-patch-to-support-multi-vector.patch @@ -1,4 +1,4 @@ -From 0d1385959ddecabb2825957e48ff28ff0e8abf53 Mon Sep 17 00:00:00 2001 +From 35ef01f59b8903dfbd4d08ff874b085e851e4228 Mon Sep 17 00:00:00 2001 From: Heemin Kim Date: Tue, 30 Jan 2024 14:43:56 -0800 Subject: [PATCH] Add IDGrouper for HNSW @@ -7,18 +7,18 @@ Signed-off-by: Heemin Kim --- faiss/CMakeLists.txt | 3 + faiss/Index.h | 8 +- - faiss/IndexHNSW.cpp | 13 ++- - faiss/IndexIDMap.cpp | 29 ++++++ - faiss/IndexIDMap.h | 22 +++++ - faiss/impl/HNSW.cpp | 10 +- - faiss/impl/IDGrouper.cpp | 51 ++++++++++ - faiss/impl/IDGrouper.h | 51 ++++++++++ - faiss/impl/ResultHandler.h | 187 ++++++++++++++++++++++++++++++++++++ - faiss/utils/GroupHeap.h | 182 +++++++++++++++++++++++++++++++++++ + faiss/IndexHNSW.cpp | 13 +- + faiss/IndexIDMap.cpp | 29 +++++ + faiss/IndexIDMap.h | 22 ++++ + faiss/impl/HNSW.cpp | 6 + + faiss/impl/IDGrouper.cpp | 51 ++++++++ + faiss/impl/IDGrouper.h | 51 ++++++++ + faiss/impl/ResultHandler.h | 190 +++++++++++++++++++++++++++++ + faiss/utils/GroupHeap.h | 182 ++++++++++++++++++++++++++++ tests/CMakeLists.txt | 2 + - tests/test_group_heap.cpp | 98 +++++++++++++++++++ - tests/test_id_grouper.cpp | 189 +++++++++++++++++++++++++++++++++++++ - 13 files changed, 838 insertions(+), 7 deletions(-) + tests/test_group_heap.cpp | 98 +++++++++++++++ + tests/test_id_grouper.cpp | 241 +++++++++++++++++++++++++++++++++++++ + 13 files changed, 891 insertions(+), 5 deletions(-) create mode 100644 faiss/impl/IDGrouper.cpp create mode 100644 faiss/impl/IDGrouper.h create mode 100644 faiss/utils/GroupHeap.h @@ -54,7 +54,7 @@ index a890a46f..137e68d4 100644 utils/WorkerThread.h utils/distances.h diff --git a/faiss/Index.h b/faiss/Index.h -index 4b4b302b..3b673d1e 100644 +index 3d1bdb99..a8622858 100644 --- a/faiss/Index.h +++ b/faiss/Index.h @@ -38,9 +38,10 @@ @@ -106,7 +106,7 @@ index 9a67332d..a5e0fea0 100644 if (is_similarity_metric(this->metric_type)) { // we need to revert the negated distances diff --git a/faiss/IndexIDMap.cpp b/faiss/IndexIDMap.cpp -index e093bbda..e24365d5 100644 +index dc84052b..3f375e7b 100644 --- a/faiss/IndexIDMap.cpp +++ b/faiss/IndexIDMap.cpp @@ -102,6 +102,23 @@ struct ScopedSelChange { @@ -198,20 +198,9 @@ index 2d164123..a68887bd 100644 + } // namespace faiss diff --git a/faiss/impl/HNSW.cpp b/faiss/impl/HNSW.cpp -index fb4de678..b6f602a0 100644 +index a9fb9daf..33b56638 100644 --- a/faiss/impl/HNSW.cpp +++ b/faiss/impl/HNSW.cpp -@@ -110,8 +110,8 @@ void HNSW::print_neighbor_stats(int level) const { - level, - nb_neighbors(level)); - size_t tot_neigh = 0, tot_common = 0, tot_reciprocal = 0, n_node = 0; --#pragma omp parallel for reduction(+: tot_neigh) reduction(+: tot_common) \ -- reduction(+: tot_reciprocal) reduction(+: n_node) -+#pragma omp parallel for reduction(+ : tot_neigh) reduction(+ : tot_common) \ -+ reduction(+ : tot_reciprocal) reduction(+ : n_node) - for (int i = 0; i < levels.size(); i++) { - if (levels[i] > level) { - n_node++; @@ -804,6 +804,12 @@ int extract_k_from_ResultHandler(ResultHandler& res) { if (auto hres = dynamic_cast(&res)) { return hres->k; @@ -340,19 +329,20 @@ index 00000000..d56113d9 + +} // namespace faiss diff --git a/faiss/impl/ResultHandler.h b/faiss/impl/ResultHandler.h -index 270de8dc..2f7f3e7f 100644 +index 270de8dc..3199634f 100644 --- a/faiss/impl/ResultHandler.h +++ b/faiss/impl/ResultHandler.h -@@ -12,6 +12,8 @@ +@@ -12,6 +12,9 @@ #pragma once #include ++#include +#include +#include #include #include -@@ -265,6 +267,191 @@ struct HeapBlockResultHandler : BlockResultHandler { +@@ -265,6 +268,193 @@ struct HeapBlockResultHandler : BlockResultHandler { } }; @@ -436,6 +426,7 @@ index 270de8dc..2f7f3e7f 100644 + idx, + group_id, + &group_id_to_index_in_heap); ++ threshold = heap_dis[0]; + return true; + } else { + size_t pos = it_pos->second; @@ -452,6 +443,7 @@ index 270de8dc..2f7f3e7f 100644 + idx, + group_id, + &group_id_to_index_in_heap); ++ threshold = heap_dis[0]; + return true; + } + } @@ -734,10 +726,10 @@ index 00000000..3b7078da +} // namespace faiss \ No newline at end of file diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt -index cc0a4f4c..96e19328 100644 +index 9017edc5..a8e9d30c 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt -@@ -26,6 +26,8 @@ set(FAISS_TEST_SRC +@@ -27,6 +27,8 @@ set(FAISS_TEST_SRC test_approx_topk.cpp test_RCQ_cropping.cpp test_distances_simd.cpp @@ -852,10 +844,10 @@ index 00000000..0e8fe7a7 +} diff --git a/tests/test_id_grouper.cpp b/tests/test_id_grouper.cpp new file mode 100644 -index 00000000..2aed5500 +index 00000000..6601795b --- /dev/null +++ b/tests/test_id_grouper.cpp -@@ -0,0 +1,189 @@ +@@ -0,0 +1,241 @@ +/** + * Copyright (c) Facebook, Inc. and its affiliates. + * @@ -920,6 +912,58 @@ index 00000000..2aed5500 + ASSERT_EQ(bitmap.NO_MORE_DOCS, bitmap.get_group(group_ids[3] + 1)); +} + ++TEST(IdGrouper, sanity_test) { ++ int d = 1; // dimension ++ int nb = 10; // database size ++ ++ std::mt19937 rng; ++ std::uniform_real_distribution<> distrib; ++ ++ float* xb = new float[d * nb]; ++ ++ for (int i = 0; i < nb; i++) { ++ for (int j = 0; j < d; j++) ++ xb[d * i + j] = distrib(rng); ++ xb[d * i] += i / 1000.; ++ } ++ ++ uint64_t bitmap[1] = {}; ++ faiss::IDGrouperBitmap id_grouper(1, bitmap); ++ for (int i = 0; i < nb; i++) { ++ id_grouper.set_group(i); ++ } ++ ++ int k = 5; ++ int m = 8; ++ faiss::Index* index = ++ new faiss::IndexHNSWFlat(d, m, faiss::MetricType::METRIC_L2); ++ index->add(nb, xb); // add vectors to the index ++ ++ // search ++ auto pSearchParameters = new faiss::SearchParametersHNSW(); ++ ++ idx_t* expectedI = new idx_t[k]; ++ float* expectedD = new float[k]; ++ index->search(1, xb, k, expectedD, expectedI, pSearchParameters); ++ ++ idx_t* I = new idx_t[k]; ++ float* D = new float[k]; ++ pSearchParameters->grp = &id_grouper; ++ index->search(1, xb, k, D, I, pSearchParameters); ++ ++ // compare ++ for (int j = 0; j < k; j++) { ++ ASSERT_EQ(expectedI[j], I[j]); ++ ASSERT_EQ(expectedD[j], D[j]); ++ } ++ ++ delete[] expectedI; ++ delete[] expectedD; ++ delete[] I; ++ delete[] D; ++ delete[] xb; ++} ++ +TEST(IdGrouper, bitmap_with_hnsw) { + int d = 1; // dimension + int nb = 10; // database size diff --git a/src/test/java/org/opensearch/knn/index/NestedSearchIT.java b/src/test/java/org/opensearch/knn/index/NestedSearchIT.java index d97aa4d40..22d8ff68f 100644 --- a/src/test/java/org/opensearch/knn/index/NestedSearchIT.java +++ b/src/test/java/org/opensearch/knn/index/NestedSearchIT.java @@ -75,11 +75,13 @@ public void testNestedSearchWithLucene_whenKIsTwo_thenReturnTwoResults() { refreshIndex(INDEX_NAME); forceMergeKnnIndex(INDEX_NAME); - Float[] queryVector = { 1f, 1f }; + Float[] queryVector = { 14f, 14f }; Response response = queryNestedField(INDEX_NAME, 2, queryVector); String entity = EntityUtils.toString(response.getEntity()); assertEquals(2, parseHits(entity)); assertEquals(2, parseTotalSearchHits(entity)); + assertEquals("14", parseIds(entity).get(0)); + assertEquals("13", parseIds(entity).get(1)); } @SneakyThrows @@ -97,11 +99,13 @@ public void testNestedSearchWithFaiss_whenKIsTwo_thenReturnTwoResults() { refreshIndex(INDEX_NAME); forceMergeKnnIndex(INDEX_NAME); - Float[] queryVector = { 1f, 1f }; + Float[] queryVector = { 14f, 14f }; Response response = queryNestedField(INDEX_NAME, 2, queryVector); String entity = EntityUtils.toString(response.getEntity()); assertEquals(2, parseHits(entity)); assertEquals(2, parseTotalSearchHits(entity)); + assertEquals("14", parseIds(entity).get(0)); + assertEquals("13", parseIds(entity).get(1)); } /** From 1ef097407df77a9fe5a41dd88d029b716d1bec78 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Fri, 24 May 2024 09:57:57 -0700 Subject: [PATCH 269/416] Use the Lucene Distance Calculation Function in Script Scoring for doing exact search (#1699) (#1717) * Use the Lucene Distance Calculation Function in Script Scoring for doing exact search Signed-off-by: Ryan Bogan * Add Changelog entry Signed-off-by: Ryan Bogan * Fix failing test Signed-off-by: Ryan Bogan * fix test Signed-off-by: Ryan Bogan * Fix test bug and remove unnecessary validation Signed-off-by: Ryan Bogan * Remove cosineSimilOptimized Signed-off-by: Ryan Bogan * Revert "Remove cosineSimilOptimized" This reverts commit f872d8389683186c9ff64f6a65fd77f170f4a47d. Signed-off-by: Ryan Bogan --------- Signed-off-by: Ryan Bogan (cherry picked from commit 7a88f40dd084cd4c5d9cc5c1ef3d8b26fd25d422) Co-authored-by: Ryan Bogan --- CHANGELOG.md | 1 + .../knn/plugin/script/KNNScoringUtil.java | 32 ++++--------------- .../plugin/script/KNNScoringSpaceTests.java | 4 +-- 3 files changed, 9 insertions(+), 28 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5c837b493..d6f9739f9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ## [Unreleased 2.x](https://github.com/opensearch-project/k-NN/compare/2.14...2.x) ### Features +* Use the Lucene Distance Calculation Function in Script Scoring for doing exact search [#1699](https://github.com/opensearch-project/k-NN/pull/1699) ### Enhancements * Add KnnCircuitBreakerException and modify exception message [#1688](https://github.com/opensearch-project/k-NN/pull/1688) * Add stats for radial search [#1684](https://github.com/opensearch-project/k-NN/pull/1684) diff --git a/src/main/java/org/opensearch/knn/plugin/script/KNNScoringUtil.java b/src/main/java/org/opensearch/knn/plugin/script/KNNScoringUtil.java index 114499100..84e986faa 100644 --- a/src/main/java/org/opensearch/knn/plugin/script/KNNScoringUtil.java +++ b/src/main/java/org/opensearch/knn/plugin/script/KNNScoringUtil.java @@ -10,6 +10,7 @@ import java.util.Objects; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.apache.lucene.util.VectorUtil; import org.opensearch.knn.index.KNNVectorScriptDocValues; import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.VectorDataType; @@ -48,13 +49,7 @@ private static void requireEqualDimension(final float[] queryVector, final float * @return L2 score */ public static float l2Squared(float[] queryVector, float[] inputVector) { - requireEqualDimension(queryVector, inputVector); - float squaredDistance = 0; - for (int i = 0; i < inputVector.length; i++) { - float diff = queryVector[i] - inputVector[i]; - squaredDistance += diff * diff; - } - return squaredDistance; + return VectorUtil.squareDistance(queryVector, inputVector); } private static float[] toFloat(List inputVector, VectorDataType vectorDataType) { @@ -148,20 +143,12 @@ public static float cosineSimilarity(List queryVector, KNNVectorScriptDo */ public static float cosinesimil(float[] queryVector, float[] inputVector) { requireEqualDimension(queryVector, inputVector); - float dotProduct = 0.0f; - float normQueryVector = 0.0f; - float normInputVector = 0.0f; - for (int i = 0; i < queryVector.length; i++) { - dotProduct += queryVector[i] * inputVector[i]; - normQueryVector += queryVector[i] * queryVector[i]; - normInputVector += inputVector[i] * inputVector[i]; - } - float normalizedProduct = normQueryVector * normInputVector; - if (normalizedProduct == 0) { + try { + return VectorUtil.cosine(queryVector, inputVector); + } catch (IllegalArgumentException | AssertionError e) { logger.debug("Invalid vectors for cosine. Returning minimum score to put this result to end"); return 0.0f; } - return (float) (dotProduct / (Math.sqrt(normalizedProduct))); } /** @@ -217,7 +204,6 @@ public static float calculateHammingBit(Long queryLong, Long inputLong) { * @return L1 score */ public static float l1Norm(float[] queryVector, float[] inputVector) { - requireEqualDimension(queryVector, inputVector); float distance = 0; for (int i = 0; i < inputVector.length; i++) { float diff = queryVector[i] - inputVector[i]; @@ -255,7 +241,6 @@ public static float l1Norm(List queryVector, KNNVectorScriptDocValues do * @return L-inf score */ public static float lInfNorm(float[] queryVector, float[] inputVector) { - requireEqualDimension(queryVector, inputVector); float distance = 0; for (int i = 0; i < inputVector.length; i++) { float diff = queryVector[i] - inputVector[i]; @@ -293,12 +278,7 @@ public static float lInfNorm(List queryVector, KNNVectorScriptDocValues * @return dot product score */ public static float innerProduct(float[] queryVector, float[] inputVector) { - requireEqualDimension(queryVector, inputVector); - float distance = 0; - for (int i = 0; i < inputVector.length; i++) { - distance += queryVector[i] * inputVector[i]; - } - return distance; + return VectorUtil.dotProduct(queryVector, inputVector); } /** diff --git a/src/test/java/org/opensearch/knn/plugin/script/KNNScoringSpaceTests.java b/src/test/java/org/opensearch/knn/plugin/script/KNNScoringSpaceTests.java index 6b40f375c..3cfbe56f1 100644 --- a/src/test/java/org/opensearch/knn/plugin/script/KNNScoringSpaceTests.java +++ b/src/test/java/org/opensearch/knn/plugin/script/KNNScoringSpaceTests.java @@ -47,7 +47,7 @@ public void testL2() { public void testCosineSimilarity() { float[] arrayFloat = new float[] { 1.0f, 2.0f, 3.0f }; - List arrayListQueryObject = new ArrayList<>(Arrays.asList(1.0, 2.0, 3.0)); + List arrayListQueryObject = new ArrayList<>(Arrays.asList(2.0, 4.0, 6.0)); float[] arrayFloat2 = new float[] { 2.0f, 4.0f, 6.0f }; KNNMethodContext knnMethodContext = KNNMethodContext.getDefault(); @@ -59,7 +59,7 @@ public void testCosineSimilarity() { ); KNNScoringSpace.CosineSimilarity cosineSimilarity = new KNNScoringSpace.CosineSimilarity(arrayListQueryObject, fieldType); - assertEquals(3F, cosineSimilarity.scoringMethod.apply(arrayFloat2, arrayFloat), 0.1F); + assertEquals(2F, cosineSimilarity.scoringMethod.apply(arrayFloat2, arrayFloat), 0.1F); // invalid zero vector final List queryZeroVector = List.of(0.0f, 0.0f, 0.0f); From 8150a07a05971ee7ae5190917a8335bb47f2abb0 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Thu, 30 May 2024 15:36:20 -0700 Subject: [PATCH 270/416] Add validation for pq m parameter before training starts (#1713) (#1721) * Add validation for pq code count before training starts Signed-off-by: Ryan Bogan * Add integration test Signed-off-by: Ryan Bogan * Add unit tests Signed-off-by: Ryan Bogan * Clean up code Signed-off-by: Ryan Bogan * Remove unnecessary lines Signed-off-by: Ryan Bogan * Add changelog entry Signed-off-by: Ryan Bogan * Change framework to add validation with data Signed-off-by: Ryan Bogan * Remove unused error message Signed-off-by: Ryan Bogan * Add unit tests Signed-off-by: Ryan Bogan * Change space type check name for readability Signed-off-by: Ryan Bogan * Add javadocs Signed-off-by: Ryan Bogan * Modify validation error wording and add json structure to tests Signed-off-by: Ryan Bogan * Change TrainingDataSpec to VectorSpaceInfo Signed-off-by: Ryan Bogan * Add unit tests Signed-off-by: Ryan Bogan --------- Signed-off-by: Ryan Bogan (cherry picked from commit 3701d19d859dc53a4f0c148c730555b8e1c7eea7) Co-authored-by: Ryan Bogan --- CHANGELOG.md | 1 + .../org/opensearch/knn/index/KNNMethod.java | 41 ++++- .../knn/index/KNNMethodContext.java | 11 ++ .../opensearch/knn/index/MethodComponent.java | 38 +++++ .../org/opensearch/knn/index/Parameter.java | 149 +++++++++++++++++- .../knn/index/util/AbstractKNNLibrary.java | 7 + .../org/opensearch/knn/index/util/Faiss.java | 9 +- .../opensearch/knn/index/util/KNNEngine.java | 6 + .../opensearch/knn/index/util/KNNLibrary.java | 11 ++ .../transport/TrainingModelRequest.java | 7 + .../knn/training/VectorSpaceInfo.java | 26 +++ .../org/opensearch/knn/index/FaissIT.java | 136 +++++++++++++++- .../opensearch/knn/index/KNNMethodTests.java | 55 ++++++- .../opensearch/knn/index/ParameterTests.java | 109 +++++++++++++ .../LibraryInitializedSupplierTests.java | 6 + .../org/opensearch/knn/KNNRestTestCase.java | 2 +- 16 files changed, 599 insertions(+), 15 deletions(-) create mode 100644 src/main/java/org/opensearch/knn/training/VectorSpaceInfo.java diff --git a/CHANGELOG.md b/CHANGELOG.md index d6f9739f9..0a0f62ebd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), * Add KnnCircuitBreakerException and modify exception message [#1688](https://github.com/opensearch-project/k-NN/pull/1688) * Add stats for radial search [#1684](https://github.com/opensearch-project/k-NN/pull/1684) * Support script score when doc value is disabled and fix misusing DISI [#1696](https://github.com/opensearch-project/k-NN/pull/1696) +* Add validation for pq m parameter before training starts [#1713](https://github.com/opensearch-project/k-NN/pull/1713) ### Bug Fixes * Block commas in model description [#1692](https://github.com/opensearch-project/k-NN/pull/1692) * Update threshold value after new result is added [#1715](https://github.com/opensearch-project/k-NN/pull/1715) diff --git a/src/main/java/org/opensearch/knn/index/KNNMethod.java b/src/main/java/org/opensearch/knn/index/KNNMethod.java index 2d3672d87..7abd2ce39 100644 --- a/src/main/java/org/opensearch/knn/index/KNNMethod.java +++ b/src/main/java/org/opensearch/knn/index/KNNMethod.java @@ -15,6 +15,7 @@ import lombok.Getter; import org.opensearch.common.ValidationException; import org.opensearch.knn.common.KNNConstants; +import org.opensearch.knn.training.VectorSpaceInfo; import java.util.ArrayList; import java.util.Arrays; @@ -41,7 +42,7 @@ public class KNNMethod { * @param space to be checked * @return true if the space is supported; false otherwise */ - public boolean containsSpace(SpaceType space) { + public boolean isSpaceTypeSupported(SpaceType space) { return spaces.contains(space); } @@ -53,7 +54,7 @@ public boolean containsSpace(SpaceType space) { */ public ValidationException validate(KNNMethodContext knnMethodContext) { List errorMessages = new ArrayList<>(); - if (!containsSpace(knnMethodContext.getSpaceType())) { + if (!isSpaceTypeSupported(knnMethodContext.getSpaceType())) { errorMessages.add( String.format( "\"%s\" configuration does not support space type: " + "\"%s\".", @@ -77,6 +78,42 @@ public ValidationException validate(KNNMethodContext knnMethodContext) { return validationException; } + /** + * Validate that the configured KNNMethodContext is valid for this method, using additional data not present in the method context + * + * @param knnMethodContext to be validated + * @param vectorSpaceInfo additional data not present in the method context + * @return ValidationException produced by validation errors; null if no validations errors. + */ + public ValidationException validateWithData(KNNMethodContext knnMethodContext, VectorSpaceInfo vectorSpaceInfo) { + List errorMessages = new ArrayList<>(); + if (!isSpaceTypeSupported(knnMethodContext.getSpaceType())) { + errorMessages.add( + String.format( + "\"%s\" configuration does not support space type: " + "\"%s\".", + this.methodComponent.getName(), + knnMethodContext.getSpaceType().getValue() + ) + ); + } + + ValidationException methodValidation = methodComponent.validateWithData( + knnMethodContext.getMethodComponentContext(), + vectorSpaceInfo + ); + if (methodValidation != null) { + errorMessages.addAll(methodValidation.validationErrors()); + } + + if (errorMessages.isEmpty()) { + return null; + } + + ValidationException validationException = new ValidationException(); + validationException.addValidationErrors(errorMessages); + return validationException; + } + /** * returns whether training is required or not * diff --git a/src/main/java/org/opensearch/knn/index/KNNMethodContext.java b/src/main/java/org/opensearch/knn/index/KNNMethodContext.java index d4df713c2..ce48b06be 100644 --- a/src/main/java/org/opensearch/knn/index/KNNMethodContext.java +++ b/src/main/java/org/opensearch/knn/index/KNNMethodContext.java @@ -30,6 +30,7 @@ import java.util.stream.Collectors; import org.apache.commons.lang.builder.EqualsBuilder; import org.apache.commons.lang.builder.HashCodeBuilder; +import org.opensearch.knn.training.VectorSpaceInfo; import static org.opensearch.knn.common.KNNConstants.KNN_ENGINE; import static org.opensearch.knn.common.KNNConstants.METHOD_HNSW; @@ -86,6 +87,16 @@ public ValidationException validate() { return knnEngine.validateMethod(this); } + /** + * This method uses the knnEngine to validate that the method is compatible with the engine, using additional data not present in the method context + * + * @param vectorSpaceInfo additional data not present in the method context + * @return ValidationException produced by validation errors; null if no validations errors. + */ + public ValidationException validateWithData(VectorSpaceInfo vectorSpaceInfo) { + return knnEngine.validateMethodWithData(this, vectorSpaceInfo); + } + /** * This method returns whether training is requires or not from knnEngine * diff --git a/src/main/java/org/opensearch/knn/index/MethodComponent.java b/src/main/java/org/opensearch/knn/index/MethodComponent.java index f2e2d878e..256d55ee5 100644 --- a/src/main/java/org/opensearch/knn/index/MethodComponent.java +++ b/src/main/java/org/opensearch/knn/index/MethodComponent.java @@ -17,6 +17,7 @@ import org.opensearch.common.ValidationException; import org.opensearch.knn.common.KNNConstants; import org.opensearch.knn.index.util.IndexHyperParametersUtil; +import org.opensearch.knn.training.VectorSpaceInfo; import java.util.ArrayList; import java.util.HashMap; @@ -102,6 +103,43 @@ public ValidationException validate(MethodComponentContext methodComponentContex return validationException; } + /** + * Validate that the methodComponentContext is a valid configuration for this methodComponent, using additional data not present in the method component context + * + * @param methodComponentContext to be validated + * @param vectorSpaceInfo additional data not present in the method component context + * @return ValidationException produced by validation errors; null if no validations errors. + */ + public ValidationException validateWithData(MethodComponentContext methodComponentContext, VectorSpaceInfo vectorSpaceInfo) { + Map providedParameters = methodComponentContext.getParameters(); + List errorMessages = new ArrayList<>(); + + if (providedParameters == null) { + return null; + } + + ValidationException parameterValidation; + for (Map.Entry parameter : providedParameters.entrySet()) { + if (!parameters.containsKey(parameter.getKey())) { + errorMessages.add(String.format("Invalid parameter for method \"%s\".", getName())); + continue; + } + + parameterValidation = parameters.get(parameter.getKey()).validateWithData(parameter.getValue(), vectorSpaceInfo); + if (parameterValidation != null) { + errorMessages.addAll(parameterValidation.validationErrors()); + } + } + + if (errorMessages.isEmpty()) { + return null; + } + + ValidationException validationException = new ValidationException(); + validationException.addValidationErrors(errorMessages); + return validationException; + } + /** * gets requiresTraining value * diff --git a/src/main/java/org/opensearch/knn/index/Parameter.java b/src/main/java/org/opensearch/knn/index/Parameter.java index e223909d5..a4520636e 100644 --- a/src/main/java/org/opensearch/knn/index/Parameter.java +++ b/src/main/java/org/opensearch/knn/index/Parameter.java @@ -12,8 +12,10 @@ package org.opensearch.knn.index; import org.opensearch.common.ValidationException; +import org.opensearch.knn.training.VectorSpaceInfo; import java.util.Map; +import java.util.function.BiFunction; import java.util.function.Predicate; /** @@ -26,6 +28,7 @@ public abstract class Parameter { private String name; private T defaultValue; protected Predicate validator; + protected BiFunction validatorWithData; /** * Constructor @@ -38,6 +41,14 @@ public Parameter(String name, T defaultValue, Predicate validator) { this.name = name; this.defaultValue = defaultValue; this.validator = validator; + this.validatorWithData = null; + } + + public Parameter(String name, T defaultValue, Predicate validator, BiFunction validatorWithData) { + this.name = name; + this.defaultValue = defaultValue; + this.validator = validator; + this.validatorWithData = validatorWithData; } /** @@ -66,6 +77,15 @@ public T getDefaultValue() { */ public abstract ValidationException validate(Object value); + /** + * Check if the value passed in is valid, using additional data not present in the value + * + * @param value to be checked + * @param vectorSpaceInfo additional data not present in the value + * @return ValidationException produced by validation errors; null if no validations errors. + */ + public abstract ValidationException validateWithData(Object value, VectorSpaceInfo vectorSpaceInfo); + /** * Boolean method parameter */ @@ -74,12 +94,23 @@ public BooleanParameter(String name, Boolean defaultValue, Predicate va super(name, defaultValue, validator); } + public BooleanParameter( + String name, + Boolean defaultValue, + Predicate validator, + BiFunction validatorWithData + ) { + super(name, defaultValue, validator, validatorWithData); + } + @Override public ValidationException validate(Object value) { ValidationException validationException = null; if (!(value instanceof Boolean)) { validationException = new ValidationException(); - validationException.addValidationError(String.format("value not of type Boolean for Boolean parameter [%s].", getName())); + validationException.addValidationError( + String.format("value is not an instance of Boolean for Boolean parameter [%s].", getName()) + ); return validationException; } @@ -89,6 +120,27 @@ public ValidationException validate(Object value) { } return validationException; } + + @Override + public ValidationException validateWithData(Object value, VectorSpaceInfo vectorSpaceInfo) { + ValidationException validationException = null; + if (!(value instanceof Boolean)) { + validationException = new ValidationException(); + validationException.addValidationError(String.format("value not of type Boolean for Boolean parameter [%s].", getName())); + return validationException; + } + + if (validatorWithData == null) { + return null; + } + + if (!validatorWithData.apply((Boolean) value, vectorSpaceInfo)) { + validationException = new ValidationException(); + validationException.addValidationError(String.format("parameter validation failed for Boolean parameter [%s].", getName())); + } + + return validationException; + } } /** @@ -99,6 +151,15 @@ public IntegerParameter(String name, Integer defaultValue, Predicate va super(name, defaultValue, validator); } + public IntegerParameter( + String name, + Integer defaultValue, + Predicate validator, + BiFunction validatorWithData + ) { + super(name, defaultValue, validator, validatorWithData); + } + @Override public ValidationException validate(Object value) { ValidationException validationException = null; @@ -118,6 +179,29 @@ public ValidationException validate(Object value) { } return validationException; } + + @Override + public ValidationException validateWithData(Object value, VectorSpaceInfo vectorSpaceInfo) { + ValidationException validationException = null; + if (!(value instanceof Integer)) { + validationException = new ValidationException(); + validationException.addValidationError( + String.format("value is not an instance of Integer for Integer parameter [%s].", getName()) + ); + return validationException; + } + + if (validatorWithData == null) { + return null; + } + + if (!validatorWithData.apply((Integer) value, vectorSpaceInfo)) { + validationException = new ValidationException(); + validationException.addValidationError(String.format("parameter validation failed for Integer parameter [%s].", getName())); + } + + return validationException; + } } /** @@ -136,6 +220,15 @@ public StringParameter(String name, String defaultValue, Predicate valid super(name, defaultValue, validator); } + public StringParameter( + String name, + String defaultValue, + Predicate validator, + BiFunction validatorWithData + ) { + super(name, defaultValue, validator, validatorWithData); + } + /** * Check if the value passed in is valid * @@ -161,6 +254,29 @@ public ValidationException validate(Object value) { } return validationException; } + + @Override + public ValidationException validateWithData(Object value, VectorSpaceInfo vectorSpaceInfo) { + ValidationException validationException = null; + if (!(value instanceof String)) { + validationException = new ValidationException(); + validationException.addValidationError( + String.format("value is not an instance of String for String parameter [%s].", getName()) + ); + return validationException; + } + + if (validatorWithData == null) { + return null; + } + + if (!validatorWithData.apply((String) value, vectorSpaceInfo)) { + validationException = new ValidationException(); + validationException.addValidationError(String.format("parameter validation failed for String parameter [%s].", getName())); + } + + return validationException; + } } /** @@ -190,6 +306,12 @@ public MethodComponentContextParameter( } return methodComponents.get(methodComponentContext.getName()).validate(methodComponentContext) == null; + }, (methodComponentContext, vectorSpaceInfo) -> { + if (!methodComponents.containsKey(methodComponentContext.getName())) { + return false; + } + return methodComponents.get(methodComponentContext.getName()) + .validateWithData(methodComponentContext, vectorSpaceInfo) == null; }); this.methodComponents = methodComponents; } @@ -216,6 +338,31 @@ public ValidationException validate(Object value) { return validationException; } + @Override + public ValidationException validateWithData(Object value, VectorSpaceInfo vectorSpaceInfo) { + ValidationException validationException = null; + if (!(value instanceof MethodComponentContext)) { + validationException = new ValidationException(); + validationException.addValidationError( + String.format("value is not an instance of for MethodComponentContext parameter [%s].", getName()) + ); + return validationException; + } + + if (validatorWithData == null) { + return null; + } + + if (!validatorWithData.apply((MethodComponentContext) value, vectorSpaceInfo)) { + validationException = new ValidationException(); + validationException.addValidationError( + String.format("parameter validation failed for MethodComponentContext parameter [%s].", getName()) + ); + } + + return validationException; + } + /** * Get method component by name * diff --git a/src/main/java/org/opensearch/knn/index/util/AbstractKNNLibrary.java b/src/main/java/org/opensearch/knn/index/util/AbstractKNNLibrary.java index f97d18810..0fe311094 100644 --- a/src/main/java/org/opensearch/knn/index/util/AbstractKNNLibrary.java +++ b/src/main/java/org/opensearch/knn/index/util/AbstractKNNLibrary.java @@ -11,6 +11,7 @@ import org.opensearch.common.ValidationException; import org.opensearch.knn.index.KNNMethod; import org.opensearch.knn.index.KNNMethodContext; +import org.opensearch.knn.training.VectorSpaceInfo; import java.util.Map; @@ -39,6 +40,12 @@ public ValidationException validateMethod(KNNMethodContext knnMethodContext) { return getMethod(methodName).validate(knnMethodContext); } + @Override + public ValidationException validateMethodWithData(KNNMethodContext knnMethodContext, VectorSpaceInfo vectorSpaceInfo) { + String methodName = knnMethodContext.getMethodComponentContext().getName(); + return getMethod(methodName).validateWithData(knnMethodContext, vectorSpaceInfo); + } + @Override public boolean isTrainingRequired(KNNMethodContext knnMethodContext) { String methodName = knnMethodContext.getMethodComponentContext().getName(); diff --git a/src/main/java/org/opensearch/knn/index/util/Faiss.java b/src/main/java/org/opensearch/knn/index/util/Faiss.java index efd8a637c..bbb58bf1e 100644 --- a/src/main/java/org/opensearch/knn/index/util/Faiss.java +++ b/src/main/java/org/opensearch/knn/index/util/Faiss.java @@ -109,9 +109,6 @@ class Faiss extends NativeLibrary { .build() ); - // TODO: To think about in future: for PQ, if dimension is not divisible by code count, PQ will fail. Right now, - // we do not have a way to base validation off of dimension. Failure will happen during training in JNI. - // Define methods supported by faiss. See issue here: https://github.com/opensearch-project/k-NN/issues/1075 private final static Map HNSW_ENCODERS = ImmutableMap.builder() .putAll( ImmutableMap.of( @@ -122,7 +119,8 @@ class Faiss extends NativeLibrary { new Parameter.IntegerParameter( ENCODER_PARAMETER_PQ_M, ENCODER_PARAMETER_PQ_CODE_COUNT_DEFAULT, - v -> v > 0 && v < ENCODER_PARAMETER_PQ_CODE_COUNT_LIMIT + v -> v > 0 && v < ENCODER_PARAMETER_PQ_CODE_COUNT_LIMIT, + (v, vectorSpaceInfo) -> vectorSpaceInfo.getDimension() % v == 0 ) ) .addParameter( @@ -161,7 +159,8 @@ class Faiss extends NativeLibrary { new Parameter.IntegerParameter( ENCODER_PARAMETER_PQ_M, ENCODER_PARAMETER_PQ_CODE_COUNT_DEFAULT, - v -> v > 0 && v < ENCODER_PARAMETER_PQ_CODE_COUNT_LIMIT + v -> v > 0 && v < ENCODER_PARAMETER_PQ_CODE_COUNT_LIMIT, + (v, vectorSpaceInfo) -> vectorSpaceInfo.getDimension() % v == 0 ) ) .addParameter( diff --git a/src/main/java/org/opensearch/knn/index/util/KNNEngine.java b/src/main/java/org/opensearch/knn/index/util/KNNEngine.java index e282c69db..556785783 100644 --- a/src/main/java/org/opensearch/knn/index/util/KNNEngine.java +++ b/src/main/java/org/opensearch/knn/index/util/KNNEngine.java @@ -10,6 +10,7 @@ import org.opensearch.knn.index.KNNMethod; import org.opensearch.knn.index.KNNMethodContext; import org.opensearch.knn.index.SpaceType; +import org.opensearch.knn.training.VectorSpaceInfo; import java.util.List; import java.util.Map; @@ -168,6 +169,11 @@ public ValidationException validateMethod(KNNMethodContext knnMethodContext) { return knnLibrary.validateMethod(knnMethodContext); } + @Override + public ValidationException validateMethodWithData(KNNMethodContext knnMethodContext, VectorSpaceInfo vectorSpaceInfo) { + return knnLibrary.validateMethodWithData(knnMethodContext, vectorSpaceInfo); + } + @Override public boolean isTrainingRequired(KNNMethodContext knnMethodContext) { return knnLibrary.isTrainingRequired(knnMethodContext); diff --git a/src/main/java/org/opensearch/knn/index/util/KNNLibrary.java b/src/main/java/org/opensearch/knn/index/util/KNNLibrary.java index f837566b8..cac5af2bb 100644 --- a/src/main/java/org/opensearch/knn/index/util/KNNLibrary.java +++ b/src/main/java/org/opensearch/knn/index/util/KNNLibrary.java @@ -15,6 +15,7 @@ import org.opensearch.knn.index.KNNMethod; import org.opensearch.knn.index.KNNMethodContext; import org.opensearch.knn.index.SpaceType; +import org.opensearch.knn.training.VectorSpaceInfo; import java.util.Collections; import java.util.List; @@ -97,6 +98,16 @@ public interface KNNLibrary { */ ValidationException validateMethod(KNNMethodContext knnMethodContext); + /** + * Validate the knnMethodContext for the given library, using additional data not present in the method context. A ValidationException should be thrown if the method is + * deemed invalid. + * + * @param knnMethodContext to be validated + * @param vectorSpaceInfo additional data not present in the method context + * @return ValidationException produced by validation errors; null if no validations errors. + */ + ValidationException validateMethodWithData(KNNMethodContext knnMethodContext, VectorSpaceInfo vectorSpaceInfo); + /** * Returns whether training is required or not from knnMethodContext for the given library. * diff --git a/src/main/java/org/opensearch/knn/plugin/transport/TrainingModelRequest.java b/src/main/java/org/opensearch/knn/plugin/transport/TrainingModelRequest.java index 9035a8e84..5f3913ac5 100644 --- a/src/main/java/org/opensearch/knn/plugin/transport/TrainingModelRequest.java +++ b/src/main/java/org/opensearch/knn/plugin/transport/TrainingModelRequest.java @@ -22,6 +22,7 @@ import org.opensearch.knn.index.IndexUtil; import org.opensearch.knn.index.KNNMethodContext; import org.opensearch.knn.indices.ModelDao; +import org.opensearch.knn.training.VectorSpaceInfo; import java.io.IOException; @@ -281,6 +282,12 @@ public ActionRequestValidationException validate() { exception.addValidationErrors(validationException.validationErrors()); } + validationException = this.knnMethodContext.validateWithData(new VectorSpaceInfo(dimension)); + if (validationException != null) { + exception = new ActionRequestValidationException(); + exception.addValidationErrors(validationException.validationErrors()); + } + if (!this.knnMethodContext.isTrainingRequired()) { exception = exception == null ? new ActionRequestValidationException() : exception; exception.addValidationError("Method does not require training."); diff --git a/src/main/java/org/opensearch/knn/training/VectorSpaceInfo.java b/src/main/java/org/opensearch/knn/training/VectorSpaceInfo.java new file mode 100644 index 000000000..13843486d --- /dev/null +++ b/src/main/java/org/opensearch/knn/training/VectorSpaceInfo.java @@ -0,0 +1,26 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.knn.training; + +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.Setter; + +/** + * A data spec containing relevant information for validation. + */ +@Getter +@Setter +@AllArgsConstructor +public class VectorSpaceInfo { + private int dimension; +} diff --git a/src/test/java/org/opensearch/knn/index/FaissIT.java b/src/test/java/org/opensearch/knn/index/FaissIT.java index 2349bb8d0..699c920ff 100644 --- a/src/test/java/org/opensearch/knn/index/FaissIT.java +++ b/src/test/java/org/opensearch/knn/index/FaissIT.java @@ -1311,8 +1311,8 @@ public void testSharedIndexState_whenOneIndexDeleted_thenSecondIndexIsStillSearc .startObject(METHOD_ENCODER_PARAMETER) .field(NAME, ENCODER_PQ) .startObject(PARAMETERS) - .field(ENCODER_PARAMETER_PQ_M, pqCodeSize) - .field(ENCODER_PARAMETER_PQ_CODE_SIZE, pqM) + .field(ENCODER_PARAMETER_PQ_M, pqM) + .field(ENCODER_PARAMETER_PQ_CODE_SIZE, pqCodeSize) .endObject() .endObject() .endObject() @@ -1646,6 +1646,138 @@ public void testFiltering_whenUsingFaissExactSearchWithIP_thenMatchExpectedScore } } + @SneakyThrows + public void testHNSW_InvalidPQM_thenFail() { + String trainingIndexName = "training-index"; + String trainingFieldName = "training-field"; + + String modelId = "test-model"; + String modelDescription = "test model"; + + List mValues = ImmutableList.of(16, 32, 64, 128); + int invalidPQM = 3; + + // training data needs to be at least equal to the number of centroids for PQ + // which is 2^8 = 256. 8 because thats the only valid code_size for HNSWPQ + int trainingDataCount = 256; + + SpaceType spaceType = SpaceType.L2; + + Integer dimension = testData.indexData.vectors[0].length; + + /* + * Builds the below json: + * { + * "name": "hnsw", + * "engine": "faiss", + * "space_type": "l2", + * "parameters": { + * "encoder": { + * "name": "pq", + * "parameters": { + * "m": 3 + * } + * } + * } + * } + */ + + XContentBuilder xContentBuilder = XContentFactory.jsonBuilder() + .startObject() + .field(NAME, METHOD_HNSW) + .field(KNN_ENGINE, FAISS_NAME) + .field(METHOD_PARAMETER_SPACE_TYPE, spaceType.getValue()) + .startObject(PARAMETERS) + .startObject(METHOD_ENCODER_PARAMETER) + .field(NAME, ENCODER_PQ) + .startObject(PARAMETERS) + .field(ENCODER_PARAMETER_PQ_M, invalidPQM) + .endObject() + .endObject() + .endObject() + .endObject(); + Map in = xContentBuilderToMap(xContentBuilder); + + createBasicKnnIndex(trainingIndexName, trainingFieldName, dimension); + ResponseException re = expectThrows( + ResponseException.class, + () -> ingestDataAndTrainModel(modelId, trainingIndexName, trainingFieldName, dimension, modelDescription, in, trainingDataCount) + ); + assertTrue( + re.getMessage().contains("Validation Failed: 1: parameter validation failed for MethodComponentContext parameter [encoder].;") + ); + } + + @SneakyThrows + public void testIVF_InvalidPQM_thenFail() { + String trainingIndexName = "training-index"; + String trainingFieldName = "training-field"; + + String modelId = "test-model"; + String modelDescription = "test model"; + + List mValues = ImmutableList.of(16, 32, 64, 128); + int invalidPQM = 3; + + // training data needs to be at least equal to the number of centroids for PQ + // which is 2^8 = 256. + int trainingDataCount = 256; + + int dimension = testData.indexData.vectors[0].length; + SpaceType spaceType = SpaceType.L2; + int ivfNlist = 4; + int ivfNprobes = 4; + int pqCodeSize = 8; + + /* + * Builds the below json: + * { + * "name": "ivf", + * "engine": "faiss", + * "space_type": "l2", + * "parameters": { + * "nprobes": 8, + * "nlist": 4, + * "encoder": { + * "name": "pq", + * "parameters": { + * "m": 3, + * "code_size": 8 + * } + * } + * } + * } + */ + + XContentBuilder xContentBuilder = XContentFactory.jsonBuilder() + .startObject() + .field(NAME, METHOD_IVF) + .field(KNN_ENGINE, FAISS_NAME) + .field(METHOD_PARAMETER_SPACE_TYPE, spaceType.getValue()) + .startObject(PARAMETERS) + .field(METHOD_PARAMETER_NPROBES, ivfNprobes) + .field(METHOD_PARAMETER_NLIST, ivfNlist) + .startObject(METHOD_ENCODER_PARAMETER) + .field(NAME, ENCODER_PQ) + .startObject(PARAMETERS) + .field(ENCODER_PARAMETER_PQ_M, invalidPQM) + .field(ENCODER_PARAMETER_PQ_CODE_SIZE, pqCodeSize) + .endObject() + .endObject() + .endObject() + .endObject(); + Map in = xContentBuilderToMap(xContentBuilder); + + createBasicKnnIndex(trainingIndexName, trainingFieldName, dimension); + ResponseException re = expectThrows( + ResponseException.class, + () -> ingestDataAndTrainModel(modelId, trainingIndexName, trainingFieldName, dimension, modelDescription, in, trainingDataCount) + ); + assertTrue( + re.getMessage().contains("Validation Failed: 1: parameter validation failed for MethodComponentContext parameter [encoder].;") + ); + } + protected void setupKNNIndexForFilterQuery() throws Exception { // Create Mappings XContentBuilder builder = XContentFactory.jsonBuilder() diff --git a/src/test/java/org/opensearch/knn/index/KNNMethodTests.java b/src/test/java/org/opensearch/knn/index/KNNMethodTests.java index d4dd989f7..607ca849e 100644 --- a/src/test/java/org/opensearch/knn/index/KNNMethodTests.java +++ b/src/test/java/org/opensearch/knn/index/KNNMethodTests.java @@ -17,6 +17,7 @@ import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.knn.common.KNNConstants; import org.opensearch.knn.index.util.KNNEngine; +import org.opensearch.knn.training.VectorSpaceInfo; import java.io.IOException; import java.util.HashMap; @@ -44,9 +45,9 @@ public void testHasSpace() { KNNMethod knnMethod = KNNMethod.Builder.builder(MethodComponent.Builder.builder(name).build()) .addSpaces(SpaceType.L2, SpaceType.COSINESIMIL) .build(); - assertTrue(knnMethod.containsSpace(SpaceType.L2)); - assertTrue(knnMethod.containsSpace(SpaceType.COSINESIMIL)); - assertFalse(knnMethod.containsSpace(SpaceType.INNER_PRODUCT)); + assertTrue(knnMethod.isSpaceTypeSupported(SpaceType.L2)); + assertTrue(knnMethod.isSpaceTypeSupported(SpaceType.COSINESIMIL)); + assertFalse(knnMethod.isSpaceTypeSupported(SpaceType.INNER_PRODUCT)); } /** @@ -93,6 +94,52 @@ public void testValidate() throws IOException { assertNull(knnMethod.validate(knnMethodContext3)); } + /** + * Test KNNMethod validateWithData + */ + public void testValidateWithData() throws IOException { + String methodName = "test-method"; + KNNMethod knnMethod = KNNMethod.Builder.builder(MethodComponent.Builder.builder(methodName).build()) + .addSpaces(SpaceType.L2) + .build(); + + VectorSpaceInfo testVectorSpaceInfo = new VectorSpaceInfo(4); + + // Invalid space + XContentBuilder xContentBuilder = XContentFactory.jsonBuilder() + .startObject() + .field(NAME, methodName) + .field(METHOD_PARAMETER_SPACE_TYPE, SpaceType.INNER_PRODUCT.getValue()) + .endObject(); + Map in = xContentBuilderToMap(xContentBuilder); + KNNMethodContext knnMethodContext1 = KNNMethodContext.parse(in); + assertNotNull(knnMethod.validateWithData(knnMethodContext1, testVectorSpaceInfo)); + + // Invalid methodComponent + xContentBuilder = XContentFactory.jsonBuilder() + .startObject() + .field(NAME, methodName) + .field(METHOD_PARAMETER_SPACE_TYPE, SpaceType.L2.getValue()) + .startObject(PARAMETERS) + .field("invalid", "invalid") + .endObject() + .endObject(); + in = xContentBuilderToMap(xContentBuilder); + KNNMethodContext knnMethodContext2 = KNNMethodContext.parse(in); + + assertNotNull(knnMethod.validateWithData(knnMethodContext2, testVectorSpaceInfo)); + + // Valid everything + xContentBuilder = XContentFactory.jsonBuilder() + .startObject() + .field(NAME, methodName) + .field(METHOD_PARAMETER_SPACE_TYPE, SpaceType.L2.getValue()) + .endObject(); + in = xContentBuilderToMap(xContentBuilder); + KNNMethodContext knnMethodContext3 = KNNMethodContext.parse(in); + assertNull(knnMethod.validateWithData(knnMethodContext3, testVectorSpaceInfo)); + } + public void testGetAsMap() { SpaceType spaceType = SpaceType.DEFAULT; String methodName = "test-method"; @@ -122,6 +169,6 @@ public void testBuilder() { builder.addSpaces(SpaceType.L2); knnMethod = builder.build(); - assertTrue(knnMethod.containsSpace(SpaceType.L2)); + assertTrue(knnMethod.isSpaceTypeSupported(SpaceType.L2)); } } diff --git a/src/test/java/org/opensearch/knn/index/ParameterTests.java b/src/test/java/org/opensearch/knn/index/ParameterTests.java index 08decd592..2f3f19727 100644 --- a/src/test/java/org/opensearch/knn/index/ParameterTests.java +++ b/src/test/java/org/opensearch/knn/index/ParameterTests.java @@ -17,6 +17,7 @@ import org.opensearch.knn.index.Parameter.IntegerParameter; import org.opensearch.knn.index.Parameter.StringParameter; import org.opensearch.knn.index.Parameter.MethodComponentContextParameter; +import org.opensearch.knn.training.VectorSpaceInfo; import java.util.Map; @@ -31,6 +32,12 @@ public void testGetDefaultValue() { public ValidationException validate(Object value) { return null; } + + @Override + public ValidationException validateWithData(Object value, VectorSpaceInfo vectorSpaceInfo) { + return null; + } + }; assertEquals(defaultValue, parameter.getDefaultValue()); @@ -52,6 +59,29 @@ public void testIntegerParameter_validate() { assertNull(parameter.validate(12)); } + /** + * Test integer parameter validate + */ + public void testIntegerParameter_validateWithData() { + final IntegerParameter parameter = new IntegerParameter( + "test", + 1, + v -> v > 0, + (v, vectorSpaceInfo) -> v > vectorSpaceInfo.getDimension() + ); + + VectorSpaceInfo testVectorSpaceInfo = new VectorSpaceInfo(0); + + // Invalid type + assertNotNull(parameter.validateWithData("String", testVectorSpaceInfo)); + + // Invalid value + assertNotNull(parameter.validateWithData(-1, testVectorSpaceInfo)); + + // valid value + assertNull(parameter.validateWithData(12, testVectorSpaceInfo)); + } + public void testStringParameter_validate() { final StringParameter parameter = new StringParameter("test_parameter", "default_value", v -> "test".equals(v)); @@ -65,6 +95,36 @@ public void testStringParameter_validate() { assertNull(parameter.validate("test")); } + public void testStringParameter_validateWithData() { + final StringParameter parameter = new StringParameter( + "test_parameter", + "default_value", + v -> "test".equals(v), + (v, vectorSpaceInfo) -> { + if (vectorSpaceInfo.getDimension() > 0) { + return "test".equals(v); + } + return false; + } + ); + + VectorSpaceInfo testVectorSpaceInfo = new VectorSpaceInfo(1); + + // Invalid type + assertNotNull(parameter.validateWithData(5, testVectorSpaceInfo)); + + // null + assertNotNull(parameter.validateWithData(null, testVectorSpaceInfo)); + + // valid value + assertNull(parameter.validateWithData("test", testVectorSpaceInfo)); + + testVectorSpaceInfo.setDimension(0); + + // invalid value + assertNotNull(parameter.validateWithData("test", testVectorSpaceInfo)); + } + public void testMethodComponentContextParameter_validate() { String methodComponentName1 = "method-1"; String parameterKey1 = "parameter_key_1"; @@ -109,6 +169,55 @@ public void testMethodComponentContextParameter_validate() { assertNull(parameter.validate(methodComponentContext)); } + public void testMethodComponentContextParameter_validateWithData() { + String methodComponentName1 = "method-1"; + String parameterKey1 = "parameter_key_1"; + Integer parameterValue1 = 12; + + Map defaultParameterMap = ImmutableMap.of(parameterKey1, parameterValue1); + MethodComponentContext methodComponentContext = new MethodComponentContext(methodComponentName1, defaultParameterMap); + + Map methodComponentMap = ImmutableMap.of( + methodComponentName1, + MethodComponent.Builder.builder(parameterKey1) + .addParameter( + parameterKey1, + new IntegerParameter(parameterKey1, 1, v -> v > 0, (v, vectorSpaceInfo) -> v > vectorSpaceInfo.getDimension()) + ) + .build() + ); + + final MethodComponentContextParameter parameter = new MethodComponentContextParameter( + "test", + methodComponentContext, + methodComponentMap + ); + + VectorSpaceInfo testVectorSpaceInfo = new VectorSpaceInfo(0); + + // Invalid type + assertNotNull(parameter.validateWithData(17, testVectorSpaceInfo)); + assertNotNull(parameter.validateWithData("invalid-value", testVectorSpaceInfo)); + + // Invalid value + String invalidMethodComponentName = "invalid-method"; + MethodComponentContext invalidMethodComponentContext1 = new MethodComponentContext(invalidMethodComponentName, defaultParameterMap); + assertNotNull(parameter.validateWithData(invalidMethodComponentContext1, testVectorSpaceInfo)); + + String invalidParameterKey = "invalid-parameter"; + Map invalidParameterMap1 = ImmutableMap.of(invalidParameterKey, parameterValue1); + MethodComponentContext invalidMethodComponentContext2 = new MethodComponentContext(methodComponentName1, invalidParameterMap1); + assertNotNull(parameter.validateWithData(invalidMethodComponentContext2, testVectorSpaceInfo)); + + String invalidParameterValue = "invalid-value"; + Map invalidParameterMap2 = ImmutableMap.of(parameterKey1, invalidParameterValue); + MethodComponentContext invalidMethodComponentContext3 = new MethodComponentContext(methodComponentName1, invalidParameterMap2); + assertNotNull(parameter.validateWithData(invalidMethodComponentContext3, testVectorSpaceInfo)); + + // valid value + assertNull(parameter.validateWithData(methodComponentContext, testVectorSpaceInfo)); + } + public void testMethodComponentContextParameter_getMethodComponent() { String methodComponentName1 = "method-1"; String parameterKey1 = "parameter_key_1"; diff --git a/src/test/java/org/opensearch/knn/plugin/stats/suppliers/LibraryInitializedSupplierTests.java b/src/test/java/org/opensearch/knn/plugin/stats/suppliers/LibraryInitializedSupplierTests.java index cff4d5805..46240e830 100644 --- a/src/test/java/org/opensearch/knn/plugin/stats/suppliers/LibraryInitializedSupplierTests.java +++ b/src/test/java/org/opensearch/knn/plugin/stats/suppliers/LibraryInitializedSupplierTests.java @@ -16,6 +16,7 @@ import org.opensearch.knn.index.KNNMethodContext; import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.util.KNNLibrary; +import org.opensearch.knn.training.VectorSpaceInfo; import org.opensearch.test.OpenSearchTestCase; import java.util.Map; @@ -78,6 +79,11 @@ public ValidationException validateMethod(KNNMethodContext knnMethodContext) { return null; } + @Override + public ValidationException validateMethodWithData(KNNMethodContext knnMethodContext, VectorSpaceInfo vectorSpaceInfo) { + return null; + } + @Override public boolean isTrainingRequired(KNNMethodContext knnMethodContext) { return false; diff --git a/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java b/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java index 05e2a5320..adb1726d4 100644 --- a/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java +++ b/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java @@ -1416,7 +1416,7 @@ public void assertTrainingFails(String modelId, int attempts, int delayInMillis) assertNotEquals(ModelState.CREATED, modelState); } - fail("Training did not succeed after " + attempts + " attempts with a delay of " + delayInMillis + " ms."); + fail("Training did not fail after " + attempts + " attempts with a delay of " + delayInMillis + " ms."); } protected boolean systemIndexExists(final String indexName) throws IOException { From 5ffdb42106772bffc8ac706df523e8869fc7f107 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Fri, 7 Jun 2024 00:07:24 -0700 Subject: [PATCH 271/416] Refractor createParseField function in mappers for code reusability (#1726) (#1727) Signed-off-by: Navneet Verma (cherry picked from commit 623b61013bd53a741213bd225370c34d36b194ea) Co-authored-by: Navneet Verma --- .../index/mapper/KNNVectorFieldMapper.java | 46 ++++++++--- .../knn/index/mapper/LuceneFieldMapper.java | 80 +++++++------------ 2 files changed, 63 insertions(+), 63 deletions(-) diff --git a/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapper.java b/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapper.java index dd8c145db..7e697fed7 100644 --- a/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapper.java +++ b/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapper.java @@ -16,6 +16,7 @@ import java.util.function.Supplier; import lombok.Getter; import lombok.extern.log4j.Log4j2; +import org.apache.lucene.document.Field; import org.apache.lucene.document.FieldType; import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.IndexOptions; @@ -540,6 +541,38 @@ private MethodComponentContext getMethodComponentContext(KNNMethodContext knnMet return knnMethodContext.getMethodComponentContext(); } + /** + * Function returns a list of fields to be indexed when the vector is float type. + * + * @param array array of floats + * @param fieldType {@link FieldType} + * @return {@link List} of {@link Field} + */ + protected List getFieldsForFloatVector(final float[] array, final FieldType fieldType) { + final List fields = new ArrayList<>(); + fields.add(new VectorField(name(), array, fieldType)); + if (this.stored) { + fields.add(createStoredFieldForFloatVector(name(), array)); + } + return fields; + } + + /** + * Function returns a list of fields to be indexed when the vector is byte type. + * + * @param array array of bytes + * @param fieldType {@link FieldType} + * @return {@link List} of {@link Field} + */ + protected List getFieldsForByteVector(final byte[] array, final FieldType fieldType) { + final List fields = new ArrayList<>(); + fields.add(new VectorField(name(), array, fieldType)); + if (this.stored) { + fields.add(createStoredFieldForByteVector(name(), array)); + } + return fields; + } + protected void parseCreateField(ParseContext context, int dimension, SpaceType spaceType, MethodComponentContext methodComponentContext) throws IOException { @@ -554,12 +587,7 @@ protected void parseCreateField(ParseContext context, int dimension, SpaceType s } final byte[] array = bytesArrayOptional.get(); spaceType.validateVector(array); - VectorField point = new VectorField(name(), array, fieldType); - - context.doc().add(point); - if (this.stored) { - context.doc().add(createStoredFieldForByteVector(name(), array)); - } + context.doc().addAll(getFieldsForByteVector(array, fieldType)); } else if (VectorDataType.FLOAT == vectorDataType) { Optional floatsArrayOptional = getFloatsFromContext(context, dimension, methodComponentContext); @@ -568,11 +596,7 @@ protected void parseCreateField(ParseContext context, int dimension, SpaceType s } final float[] array = floatsArrayOptional.get(); spaceType.validateVector(array); - VectorField point = new VectorField(name(), array, fieldType); - context.doc().add(point); - if (this.stored) { - context.doc().add(createStoredFieldForFloatVector(name(), array)); - } + context.doc().addAll(getFieldsForFloatVector(array, fieldType)); } else { throw new IllegalArgumentException( String.format(Locale.ROOT, "Cannot parse context for unsupported values provided for field [%s]", VECTOR_DATA_TYPE_FIELD) diff --git a/src/main/java/org/opensearch/knn/index/mapper/LuceneFieldMapper.java b/src/main/java/org/opensearch/knn/index/mapper/LuceneFieldMapper.java index 618d77a32..59f4867dd 100644 --- a/src/main/java/org/opensearch/knn/index/mapper/LuceneFieldMapper.java +++ b/src/main/java/org/opensearch/knn/index/mapper/LuceneFieldMapper.java @@ -5,26 +5,23 @@ package org.opensearch.knn.index.mapper; -import java.io.IOException; +import java.util.ArrayList; +import java.util.List; import java.util.Locale; -import java.util.Optional; import lombok.AllArgsConstructor; import lombok.Getter; import lombok.NonNull; +import org.apache.lucene.document.Field; import org.apache.lucene.document.FieldType; import org.apache.lucene.document.KnnByteVectorField; import org.apache.lucene.document.KnnVectorField; import org.apache.lucene.index.VectorSimilarityFunction; import org.opensearch.common.Explicit; -import org.opensearch.index.mapper.ParseContext; import org.opensearch.knn.index.KNNMethodContext; -import org.opensearch.knn.index.MethodComponentContext; -import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.VectorField; import org.opensearch.knn.index.util.KNNEngine; -import static org.opensearch.knn.common.KNNConstants.VECTOR_DATA_TYPE_FIELD; import static org.opensearch.knn.index.mapper.KNNVectorFieldMapperUtil.createStoredFieldForByteVector; import static org.opensearch.knn.index.mapper.KNNVectorFieldMapperUtil.createStoredFieldForFloatVector; import static org.opensearch.knn.index.mapper.KNNVectorFieldMapperUtil.buildDocValuesFieldType; @@ -77,54 +74,33 @@ public class LuceneFieldMapper extends KNNVectorFieldMapper { } @Override - protected void parseCreateField(ParseContext context, int dimension, SpaceType spaceType, MethodComponentContext methodComponentContext) - throws IOException { - - validateIfKNNPluginEnabled(); - validateIfCircuitBreakerIsNotTriggered(); - - if (VectorDataType.BYTE == vectorDataType) { - Optional bytesArrayOptional = getBytesFromContext(context, dimension); - if (bytesArrayOptional.isEmpty()) { - return; - } - final byte[] array = bytesArrayOptional.get(); - spaceType.validateVector(array); - KnnByteVectorField point = new KnnByteVectorField(name(), array, fieldType); - - context.doc().add(point); - if (this.stored) { - context.doc().add(createStoredFieldForByteVector(name(), array)); - } - - if (hasDocValues && vectorFieldType != null) { - context.doc().add(new VectorField(name(), array, vectorFieldType)); - } - } else if (VectorDataType.FLOAT == vectorDataType) { - Optional floatsArrayOptional = getFloatsFromContext(context, dimension, methodComponentContext); - - if (floatsArrayOptional.isEmpty()) { - return; - } - final float[] array = floatsArrayOptional.get(); - spaceType.validateVector(array); - KnnVectorField point = new KnnVectorField(name(), array, fieldType); - - context.doc().add(point); - if (this.stored) { - context.doc().add(createStoredFieldForFloatVector(name(), array)); - } - - if (hasDocValues && vectorFieldType != null) { - context.doc().add(new VectorField(name(), array, vectorFieldType)); - } - } else { - throw new IllegalArgumentException( - String.format(Locale.ROOT, "Cannot parse context for unsupported values provided for field [%s]", VECTOR_DATA_TYPE_FIELD) - ); + protected List getFieldsForFloatVector(final float[] array, final FieldType fieldType) { + final List fieldsToBeAdded = new ArrayList<>(); + fieldsToBeAdded.add(new KnnVectorField(name(), array, fieldType)); + + if (hasDocValues && vectorFieldType != null) { + fieldsToBeAdded.add(new VectorField(name(), array, vectorFieldType)); + } + + if (this.stored) { + fieldsToBeAdded.add(createStoredFieldForFloatVector(name(), array)); } + return fieldsToBeAdded; + } + + @Override + protected List getFieldsForByteVector(final byte[] array, final FieldType fieldType) { + final List fieldsToBeAdded = new ArrayList<>(); + fieldsToBeAdded.add(new KnnByteVectorField(name(), array, fieldType)); - context.path().remove(); + if (hasDocValues && vectorFieldType != null) { + fieldsToBeAdded.add(new VectorField(name(), array, vectorFieldType)); + } + + if (this.stored) { + fieldsToBeAdded.add(createStoredFieldForByteVector(name(), array)); + } + return fieldsToBeAdded; } @Override From 71deaf47fdfa8c6d7a545c0029f9f091d585425c Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Tue, 11 Jun 2024 10:44:12 -0700 Subject: [PATCH 272/416] Bump idna from 3.3 to 3.7 in /benchmarks/osb (#1612) (#1739) Bumps [idna](https://github.com/kjd/idna) from 3.3 to 3.7. - [Release notes](https://github.com/kjd/idna/releases) - [Changelog](https://github.com/kjd/idna/blob/master/HISTORY.rst) - [Commits](https://github.com/kjd/idna/compare/v3.3...v3.7) --- updated-dependencies: - dependency-name: idna dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> (cherry picked from commit 3ceb501cb52eb5763da84317b468c0b77bc9c254) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- benchmarks/osb/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmarks/osb/requirements.txt b/benchmarks/osb/requirements.txt index 3d41012f2..261693047 100644 --- a/benchmarks/osb/requirements.txt +++ b/benchmarks/osb/requirements.txt @@ -32,7 +32,7 @@ google-resumable-media==1.1.0 # via opensearch-benchmark h5py==3.6.0 # via -r requirements.in -idna==3.3 +idna==3.7 # via yarl ijson==2.6.1 # via opensearch-benchmark From e932570ad23accc587db850016067a9403b1c64f Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Tue, 11 Jun 2024 10:44:40 -0700 Subject: [PATCH 273/416] Bump idna from 3.2 to 3.7 in /benchmarks/perf-tool (#1611) (#1740) Bumps [idna](https://github.com/kjd/idna) from 3.2 to 3.7. - [Release notes](https://github.com/kjd/idna/releases) - [Changelog](https://github.com/kjd/idna/blob/master/HISTORY.rst) - [Commits](https://github.com/kjd/idna/compare/v3.2...v3.7) --- updated-dependencies: - dependency-name: idna dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> (cherry picked from commit 77b2c1442181343876926a4dbd024deab86f3179) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- benchmarks/perf-tool/requirements.txt | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/benchmarks/perf-tool/requirements.txt b/benchmarks/perf-tool/requirements.txt index fdcff1fa8..46cec00ed 100644 --- a/benchmarks/perf-tool/requirements.txt +++ b/benchmarks/perf-tool/requirements.txt @@ -4,8 +4,6 @@ # # pip-compile # -cached-property==1.5.2 - # via h5py cerberus==1.3.4 # via -r requirements.in certifi==2023.7.22 @@ -16,7 +14,7 @@ charset-normalizer==2.0.4 # via requests h5py==3.3.0 # via -r requirements.in -idna==3.2 +idna==3.7 # via requests numpy==1.24.2 # via From 5b3c4f81387febd3466972caf39b964e2949ff44 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Tue, 11 Jun 2024 10:55:15 -0700 Subject: [PATCH 274/416] Bump aiohttp from 3.9.2 to 3.9.4 in /benchmarks/osb (#1625) (#1741) Bumps [aiohttp](https://github.com/aio-libs/aiohttp) from 3.9.2 to 3.9.4. - [Release notes](https://github.com/aio-libs/aiohttp/releases) - [Changelog](https://github.com/aio-libs/aiohttp/blob/master/CHANGES.rst) - [Commits](https://github.com/aio-libs/aiohttp/compare/v3.9.2...v3.9.4) --- updated-dependencies: - dependency-name: aiohttp dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> (cherry picked from commit 3d0bd4725231dcd82c26adc61ed70785c5f76b7b) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- benchmarks/osb/requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/benchmarks/osb/requirements.txt b/benchmarks/osb/requirements.txt index 261693047..0829cdd54 100644 --- a/benchmarks/osb/requirements.txt +++ b/benchmarks/osb/requirements.txt @@ -4,7 +4,7 @@ # # pip-compile # -aiohttp==3.9.2 +aiohttp==3.9.4 # via opensearch-py aiosignal==1.2.0 # via aiohttp @@ -38,7 +38,7 @@ ijson==2.6.1 # via opensearch-benchmark importlib-metadata==4.11.3 # via jsonschema -jinja2==3.1.3 +jinja2==2.11.3 # via opensearch-benchmark jsonschema==3.1.1 # via opensearch-benchmark From cd311af4cb8716eafddd8f6a33c87a6d308dfdee Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Tue, 11 Jun 2024 13:21:09 -0700 Subject: [PATCH 275/416] Add 2.15.0 release notes (#1738) (#1744) Add release note and prepare changelog for next release Signed-off-by: Vijayan Balasubramanian (cherry picked from commit 34a8b4f083c1bde407f0876c0863536fee7103b3) Co-authored-by: Vijayan Balasubramanian --- CHANGELOG.md | 9 +-------- .../opensearch-knn.release-notes-2.15.0.0.md | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 8 deletions(-) create mode 100644 release-notes/opensearch-knn.release-notes-2.15.0.0.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 0a0f62ebd..62006a6f3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,17 +12,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Maintenance ### Refactoring -## [Unreleased 2.x](https://github.com/opensearch-project/k-NN/compare/2.14...2.x) +## [Unreleased 2.x](https://github.com/opensearch-project/k-NN/compare/2.15...2.x) ### Features -* Use the Lucene Distance Calculation Function in Script Scoring for doing exact search [#1699](https://github.com/opensearch-project/k-NN/pull/1699) ### Enhancements -* Add KnnCircuitBreakerException and modify exception message [#1688](https://github.com/opensearch-project/k-NN/pull/1688) -* Add stats for radial search [#1684](https://github.com/opensearch-project/k-NN/pull/1684) -* Support script score when doc value is disabled and fix misusing DISI [#1696](https://github.com/opensearch-project/k-NN/pull/1696) -* Add validation for pq m parameter before training starts [#1713](https://github.com/opensearch-project/k-NN/pull/1713) ### Bug Fixes -* Block commas in model description [#1692](https://github.com/opensearch-project/k-NN/pull/1692) -* Update threshold value after new result is added [#1715](https://github.com/opensearch-project/k-NN/pull/1715) ### Infrastructure ### Documentation ### Maintenance diff --git a/release-notes/opensearch-knn.release-notes-2.15.0.0.md b/release-notes/opensearch-knn.release-notes-2.15.0.0.md new file mode 100644 index 000000000..3def01638 --- /dev/null +++ b/release-notes/opensearch-knn.release-notes-2.15.0.0.md @@ -0,0 +1,14 @@ +## Version 2.15.0.0 Release Notes + +Compatible with OpenSearch 2.15.0 + +### Features +* Use the Lucene Distance Calculation Function in Script Scoring for doing exact search [#1699](https://github.com/opensearch-project/k-NN/pull/1699) +### Enhancements +* Add KnnCircuitBreakerException and modify exception message [#1688](https://github.com/opensearch-project/k-NN/pull/1688) +* Add stats for radial search [#1684](https://github.com/opensearch-project/k-NN/pull/1684) +* Support script score when doc value is disabled and fix misusing DISI [#1696](https://github.com/opensearch-project/k-NN/pull/1696) +* Add validation for pq m parameter before training starts [#1713](https://github.com/opensearch-project/k-NN/pull/1713) +### Bug Fixes +* Block commas in model description [#1692](https://github.com/opensearch-project/k-NN/pull/1692) +* Update threshold value after new result is added [#1715](https://github.com/opensearch-project/k-NN/pull/1715) From 0738439e15d41c28eaaeef9a0d8b35727c1b30e5 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Tue, 11 Jun 2024 16:43:16 -0700 Subject: [PATCH 276/416] [Backport 2.x] Block delete model requests if an index uses the model (#1745) * Block delete model requests if an index uses the model (#1722) * Block delete model requests if an index uses the model Signed-off-by: Ryan Bogan * Add test Signed-off-by: Ryan Bogan * Add changelog entry Signed-off-by: Ryan Bogan * Change tests that were deleting a model in use Signed-off-by: Ryan Bogan * Debug Signed-off-by: Ryan Bogan * Add extra check to fix incorrect exception Signed-off-by: Ryan Bogan * Remove println Signed-off-by: Ryan Bogan * Change model_id search from contains to parsing response map Signed-off-by: Ryan Bogan * Remove * import Signed-off-by: Ryan Bogan * Combine delete model exceptions Signed-off-by: Ryan Bogan * Move check to UpdateModelGraveyardTransportAction Signed-off-by: Ryan Bogan * Optimize code logic Signed-off-by: Ryan Bogan * Modify method names and throw exception in outer method Signed-off-by: Ryan Bogan * Add test with cases for UpdateModelGraveyardTransportAction Signed-off-by: Ryan Bogan * Fix spotless Signed-off-by: Ryan Bogan * Refactor code and other minor optimizations Signed-off-by: Ryan Bogan --------- Signed-off-by: Ryan Bogan (cherry picked from commit 1c4a131688463f4e3612700b39b424c2c8f40c17) * Fix compile Signed-off-by: Ryan Bogan --------- Signed-off-by: Ryan Bogan Co-authored-by: Ryan Bogan --- .../opensearch-knn.release-notes-2.15.0.0.md | 1 + .../opensearch/knn/common/KNNConstants.java | 1 + ...ception.java => DeleteModelException.java} | 8 +- .../org/opensearch/knn/indices/ModelDao.java | 6 +- .../UpdateModelGraveyardTransportAction.java | 74 +++++- .../opensearch/knn/KNNSingleNodeTestCase.java | 40 +++- .../org/opensearch/knn/index/FaissIT.java | 1 + .../opensearch/knn/indices/ModelDaoTests.java | 136 +++++++---- .../plugin/action/RestKNNStatsHandlerIT.java | 1 + ...ateModelGraveyardTransportActionTests.java | 220 ++++++++++++++++++ .../opensearch/knn/recall/RecallTestsIT.java | 6 + 11 files changed, 436 insertions(+), 58 deletions(-) rename src/main/java/org/opensearch/knn/common/exception/{DeleteModelWhenInTrainStateException.java => DeleteModelException.java} (75%) diff --git a/release-notes/opensearch-knn.release-notes-2.15.0.0.md b/release-notes/opensearch-knn.release-notes-2.15.0.0.md index 3def01638..198c32ce9 100644 --- a/release-notes/opensearch-knn.release-notes-2.15.0.0.md +++ b/release-notes/opensearch-knn.release-notes-2.15.0.0.md @@ -9,6 +9,7 @@ Compatible with OpenSearch 2.15.0 * Add stats for radial search [#1684](https://github.com/opensearch-project/k-NN/pull/1684) * Support script score when doc value is disabled and fix misusing DISI [#1696](https://github.com/opensearch-project/k-NN/pull/1696) * Add validation for pq m parameter before training starts [#1713](https://github.com/opensearch-project/k-NN/pull/1713) +* Block delete model requests if an index uses the model [#1722](https://github.com/opensearch-project/k-NN/pull/1722) ### Bug Fixes * Block commas in model description [#1692](https://github.com/opensearch-project/k-NN/pull/1692) * Update threshold value after new result is added [#1715](https://github.com/opensearch-project/k-NN/pull/1715) diff --git a/src/main/java/org/opensearch/knn/common/KNNConstants.java b/src/main/java/org/opensearch/knn/common/KNNConstants.java index 7c5bb61ad..0b4538ec8 100644 --- a/src/main/java/org/opensearch/knn/common/KNNConstants.java +++ b/src/main/java/org/opensearch/knn/common/KNNConstants.java @@ -25,6 +25,7 @@ public class KNNConstants { public static final String VECTOR = "vector"; public static final String K = "k"; public static final String TYPE_KNN_VECTOR = "knn_vector"; + public static final String PROPERTIES = "properties"; public static final String METHOD_PARAMETER_EF_SEARCH = "ef_search"; public static final String METHOD_PARAMETER_EF_CONSTRUCTION = "ef_construction"; public static final String METHOD_PARAMETER_M = "m"; diff --git a/src/main/java/org/opensearch/knn/common/exception/DeleteModelWhenInTrainStateException.java b/src/main/java/org/opensearch/knn/common/exception/DeleteModelException.java similarity index 75% rename from src/main/java/org/opensearch/knn/common/exception/DeleteModelWhenInTrainStateException.java rename to src/main/java/org/opensearch/knn/common/exception/DeleteModelException.java index 00f6e6e80..d9590c3f8 100644 --- a/src/main/java/org/opensearch/knn/common/exception/DeleteModelWhenInTrainStateException.java +++ b/src/main/java/org/opensearch/knn/common/exception/DeleteModelException.java @@ -10,18 +10,18 @@ import org.opensearch.core.rest.RestStatus; /** - * Exception thrown when a model is deleted while it is in the training state. The RestStatus associated with this + * Exception thrown when a model is deleted while it is in the training state or in use by an index. The RestStatus associated with this * exception should be a {@link RestStatus#CONFLICT} because the request cannot be deleted due to the model being in - * the training state. + * the training state or in use by an index. */ -public class DeleteModelWhenInTrainStateException extends OpenSearchException { +public class DeleteModelException extends OpenSearchException { /** * Constructor * * @param msg detailed exception message * @param args arguments of the message */ - public DeleteModelWhenInTrainStateException(String msg, Object... args) { + public DeleteModelException(String msg, Object... args) { super(LoggerMessageFormat.format(msg, args)); } diff --git a/src/main/java/org/opensearch/knn/indices/ModelDao.java b/src/main/java/org/opensearch/knn/indices/ModelDao.java index 6940fcd39..0bc6c5edb 100644 --- a/src/main/java/org/opensearch/knn/indices/ModelDao.java +++ b/src/main/java/org/opensearch/knn/indices/ModelDao.java @@ -49,7 +49,7 @@ import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.index.IndexNotFoundException; import org.opensearch.knn.common.KNNConstants; -import org.opensearch.knn.common.exception.DeleteModelWhenInTrainStateException; +import org.opensearch.knn.common.exception.DeleteModelException; import org.opensearch.knn.index.MethodComponentContext; import org.opensearch.knn.plugin.transport.DeleteModelResponse; import org.opensearch.knn.plugin.transport.GetModelResponse; @@ -84,7 +84,7 @@ public interface ModelDao { /** - * Creates model index. It is possible that the 2 threads call this function simulateously. In this case, one + * Creates model index. It is possible that the 2 threads call this function simultaneously. In this case, one * thread will throw a ResourceAlreadyExistsException. This should be caught and handled. * * @param actionListener CreateIndexResponse listener @@ -527,7 +527,7 @@ public void delete(String modelId, ActionListener listener) // If model is in Training state, fail delete model request if (ModelState.TRAINING == getModelResponse.getModel().getModelMetadata().getState()) { String errorMessage = String.format("Cannot delete model [%s]. Model is still in training", modelId); - listener.onFailure(new DeleteModelWhenInTrainStateException(errorMessage)); + listener.onFailure(new DeleteModelException(errorMessage)); return; } diff --git a/src/main/java/org/opensearch/knn/plugin/transport/UpdateModelGraveyardTransportAction.java b/src/main/java/org/opensearch/knn/plugin/transport/UpdateModelGraveyardTransportAction.java index df0c26624..d7536a5a7 100644 --- a/src/main/java/org/opensearch/knn/plugin/transport/UpdateModelGraveyardTransportAction.java +++ b/src/main/java/org/opensearch/knn/plugin/transport/UpdateModelGraveyardTransportAction.java @@ -7,6 +7,7 @@ import lombok.Value; import lombok.extern.log4j.Log4j2; +import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.core.action.ActionListener; import org.opensearch.action.support.ActionFilters; import org.opensearch.action.support.master.AcknowledgedResponse; @@ -22,16 +23,22 @@ import org.opensearch.common.Priority; import org.opensearch.common.inject.Inject; import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.indices.IndicesService; +import org.opensearch.knn.common.exception.DeleteModelException; import org.opensearch.knn.indices.ModelGraveyard; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportService; import java.io.IOException; +import java.util.Collections; import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Objects; import java.util.Set; +import static java.util.stream.Collectors.toList; +import static org.opensearch.knn.common.KNNConstants.MODEL_ID; import static org.opensearch.knn.common.KNNConstants.PLUGIN_NAME; /** @@ -42,6 +49,7 @@ public class UpdateModelGraveyardTransportAction extends TransportClusterManager UpdateModelGraveyardRequest, AcknowledgedResponse> { private UpdateModelGraveyardExecutor updateModelGraveyardExecutor; + private final IndicesService indicesService; @Inject public UpdateModelGraveyardTransportAction( @@ -49,7 +57,8 @@ public UpdateModelGraveyardTransportAction( ClusterService clusterService, ThreadPool threadPool, ActionFilters actionFilters, - IndexNameExpressionResolver indexNameExpressionResolver + IndexNameExpressionResolver indexNameExpressionResolver, + IndicesService indicesService ) { super( UpdateModelGraveyardAction.NAME, @@ -61,6 +70,7 @@ public UpdateModelGraveyardTransportAction( indexNameExpressionResolver ); this.updateModelGraveyardExecutor = new UpdateModelGraveyardExecutor(); + this.indicesService = indicesService; } @Override @@ -82,7 +92,7 @@ protected void clusterManagerOperation( // ClusterManager updates model graveyard based on request parameters clusterService.submitStateUpdateTask( PLUGIN_NAME, - new UpdateModelGraveyardTask(request.getModelId(), request.isRemoveRequest()), + new UpdateModelGraveyardTask(request.getModelId(), request.isRemoveRequest(), indicesService), ClusterStateTaskConfig.build(Priority.NORMAL), updateModelGraveyardExecutor, new ClusterStateTaskListener() { @@ -111,6 +121,7 @@ protected ClusterBlockException checkBlock(UpdateModelGraveyardRequest request, private static class UpdateModelGraveyardTask { String modelId; boolean isRemoveRequest; + IndicesService indicesService; } /** @@ -123,7 +134,8 @@ private static class UpdateModelGraveyardExecutor implements ClusterStateTaskExe * @return Represents the result of a batched execution of cluster state update tasks (UpdateModelGraveyardTasks) */ @Override - public ClusterTasksResult execute(ClusterState clusterState, List taskList) { + public ClusterTasksResult execute(ClusterState clusterState, List taskList) + throws IOException { // Check if the objects are not null and throw a customized NullPointerException Objects.requireNonNull(clusterState, "Cluster state must not be null"); @@ -146,6 +158,17 @@ public ClusterTasksResult execute(ClusterState cluster modelGraveyard.remove(task.getModelId()); continue; } + List indicesUsingModel = getIndicesUsingModel(clusterState, task); + // Throw exception if any indices are using the model + if (!indicesUsingModel.isEmpty()) { + throw new DeleteModelException( + String.format( + "Cannot delete model [%s]. Model is in use by the following indices %s, which must be deleted first.", + task.getModelId(), + indicesUsingModel + ) + ); + } modelGraveyard.add(task.getModelId()); } @@ -155,5 +178,50 @@ public ClusterTasksResult execute(ClusterState cluster ClusterState updatedClusterState = ClusterState.builder(clusterState).metadata(metaDataBuilder).build(); return new ClusterTasksResult.Builder().successes(taskList).build(updatedClusterState); } + + private List getIndicesUsingModel(ClusterState clusterState, UpdateModelGraveyardTask task) throws IOException { + Map indices = clusterState.metadata().indices(); + String[] knnIndicesList = indices.values() + .stream() + .filter(metadata -> "true".equals(metadata.getSettings().get("index.knn", "false"))) + .map(metadata -> metadata.getIndex().getName()) + .toArray(String[]::new); + if (knnIndicesList.length == 0) { + return Collections.emptyList(); + } + + return clusterState.metadata() + .findMappings(knnIndicesList, task.getIndicesService().getFieldFilter()) + .entrySet() + .stream() + .filter(entry -> entry.getValue() != null) + .filter(entry -> { + Object properties = entry.getValue().getSourceAsMap().get("properties"); + if (properties == null || properties instanceof Map == false) { + return false; + } + Map propertiesMap = (Map) properties; + return propertiesMapContainsModel(propertiesMap, task.getModelId()); + }) + .map(Map.Entry::getKey) + .collect(toList()); + } + + private boolean propertiesMapContainsModel(Map propertiesMap, String modelId) { + for (Map.Entry fieldsEntry : propertiesMap.entrySet()) { + if (fieldsEntry.getKey() != null && fieldsEntry.getValue() instanceof Map) { + Map innerMap = (Map) fieldsEntry.getValue(); + for (Map.Entry innerEntry : innerMap.entrySet()) { + // If model is in use, fail delete model request + if (innerEntry.getKey().equals(MODEL_ID) + && innerEntry.getValue() instanceof String + && innerEntry.getValue().equals(modelId)) { + return true; + } + } + } + } + return false; + } } } diff --git a/src/test/java/org/opensearch/knn/KNNSingleNodeTestCase.java b/src/test/java/org/opensearch/knn/KNNSingleNodeTestCase.java index 1c9e75c3a..d87a63ad5 100644 --- a/src/test/java/org/opensearch/knn/KNNSingleNodeTestCase.java +++ b/src/test/java/org/opensearch/knn/KNNSingleNodeTestCase.java @@ -16,6 +16,7 @@ import org.opensearch.knn.index.query.KNNQueryBuilder; import org.opensearch.knn.index.memory.NativeMemoryCacheManager; import org.opensearch.knn.index.memory.NativeMemoryLoadStrategy; +import org.opensearch.knn.indices.Model; import org.opensearch.knn.indices.ModelDao; import org.opensearch.knn.indices.ModelMetadata; import org.opensearch.knn.indices.ModelState; @@ -36,13 +37,12 @@ import org.opensearch.test.hamcrest.OpenSearchAssertions; import java.io.IOException; -import java.util.Collection; -import java.util.Collections; -import java.util.EnumSet; -import java.util.Map; +import java.util.*; import java.util.concurrent.ExecutionException; import static org.mockito.Mockito.when; +import static org.opensearch.knn.common.KNNConstants.*; +import static org.opensearch.knn.common.KNNConstants.MODEL_INDEX_NAME; public class KNNSingleNodeTestCase extends OpenSearchSingleNodeTestCase { @Override @@ -181,6 +181,38 @@ protected void addDoc(String index, String docId, String fieldName, String dummy assertEquals(response.status(), RestStatus.CREATED); } + /** + * Index a new model + */ + protected void addDoc(Model model) throws IOException, ExecutionException, InterruptedException { + ModelMetadata modelMetadata = model.getModelMetadata(); + + XContentBuilder builder = XContentFactory.jsonBuilder() + .startObject() + .field(MODEL_ID, model.getModelID()) + .field(KNN_ENGINE, modelMetadata.getKnnEngine().getName()) + .field(METHOD_PARAMETER_SPACE_TYPE, modelMetadata.getSpaceType().getValue()) + .field(DIMENSION, modelMetadata.getDimension()) + .field(MODEL_STATE, modelMetadata.getState().getName()) + .field(MODEL_TIMESTAMP, modelMetadata.getTimestamp().toString()) + .field(MODEL_DESCRIPTION, modelMetadata.getDescription()) + .field(MODEL_ERROR, modelMetadata.getError()); + + if (model.getModelBlob() != null) { + builder.field(MODEL_BLOB_PARAMETER, Base64.getEncoder().encodeToString(model.getModelBlob())); + } + + builder.endObject(); + + IndexRequest indexRequest = new IndexRequest().index(MODEL_INDEX_NAME) + .id(model.getModelID()) + .source(builder) + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + + IndexResponse response = client().index(indexRequest).get(); + assertTrue(response.status() == RestStatus.CREATED || response.status() == RestStatus.OK); + } + /** * Run a search against a k-NN index */ diff --git a/src/test/java/org/opensearch/knn/index/FaissIT.java b/src/test/java/org/opensearch/knn/index/FaissIT.java index 699c920ff..3a9b7d596 100644 --- a/src/test/java/org/opensearch/knn/index/FaissIT.java +++ b/src/test/java/org/opensearch/knn/index/FaissIT.java @@ -1334,6 +1334,7 @@ public void testSharedIndexState_whenOneIndexDeleted_thenSecondIndexIsStillSearc // will give 15 second buffer from that Thread.sleep(1000 * 45); validateSearchWorkflow(secondIndexName, testData.queries, 10); + deleteKNNIndex(secondIndexName); deleteModel(modelId); } diff --git a/src/test/java/org/opensearch/knn/indices/ModelDaoTests.java b/src/test/java/org/opensearch/knn/indices/ModelDaoTests.java index ee2c77d1a..3a25c3064 100644 --- a/src/test/java/org/opensearch/knn/indices/ModelDaoTests.java +++ b/src/test/java/org/opensearch/knn/indices/ModelDaoTests.java @@ -18,6 +18,7 @@ import org.opensearch.ExceptionsHelper; import org.opensearch.ResourceAlreadyExistsException; import org.opensearch.ResourceNotFoundException; +import org.opensearch.action.admin.indices.create.CreateIndexRequestBuilder; import org.opensearch.cluster.ClusterChangedEvent; import org.opensearch.core.action.ActionListener; import org.opensearch.action.DocWriteResponse; @@ -26,7 +27,6 @@ import org.opensearch.action.delete.DeleteAction; import org.opensearch.action.delete.DeleteRequestBuilder; import org.opensearch.action.delete.DeleteResponse; -import org.opensearch.action.index.IndexRequest; import org.opensearch.action.index.IndexResponse; import org.opensearch.action.support.WriteRequest; import org.opensearch.action.support.master.AcknowledgedResponse; @@ -35,7 +35,8 @@ import org.opensearch.index.IndexNotFoundException; import org.opensearch.index.engine.VersionConflictEngineException; import org.opensearch.knn.KNNSingleNodeTestCase; -import org.opensearch.knn.common.exception.DeleteModelWhenInTrainStateException; +import org.opensearch.knn.TestUtils; +import org.opensearch.knn.common.exception.DeleteModelException; import org.opensearch.knn.index.MethodComponentContext; import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.util.KNNEngine; @@ -48,13 +49,11 @@ import org.opensearch.knn.plugin.transport.UpdateModelMetadataRequest; import org.opensearch.knn.plugin.transport.UpdateModelGraveyardAction; import org.opensearch.knn.plugin.transport.UpdateModelGraveyardRequest; -import org.opensearch.core.rest.RestStatus; import org.opensearch.knn.training.TrainingJobClusterStateListener; import java.io.IOException; import java.time.ZoneOffset; import java.time.ZonedDateTime; -import java.util.Base64; import java.util.Collections; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutionException; @@ -66,17 +65,11 @@ import static org.mockito.Mockito.doNothing; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.mockStatic; -import static org.opensearch.cluster.metadata.Metadata.builder; -import static org.opensearch.knn.common.KNNConstants.DIMENSION; -import static org.opensearch.knn.common.KNNConstants.KNN_ENGINE; -import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_SPACE_TYPE; -import static org.opensearch.knn.common.KNNConstants.MODEL_BLOB_PARAMETER; -import static org.opensearch.knn.common.KNNConstants.MODEL_DESCRIPTION; -import static org.opensearch.knn.common.KNNConstants.MODEL_ERROR; import static org.opensearch.knn.common.KNNConstants.MODEL_ID; import static org.opensearch.knn.common.KNNConstants.MODEL_INDEX_NAME; -import static org.opensearch.knn.common.KNNConstants.MODEL_STATE; -import static org.opensearch.knn.common.KNNConstants.MODEL_TIMESTAMP; +import static org.opensearch.knn.common.KNNConstants.PROPERTIES; +import static org.opensearch.knn.common.KNNConstants.TYPE; +import static org.opensearch.knn.common.KNNConstants.TYPE_KNN_VECTOR; public class ModelDaoTests extends KNNSingleNodeTestCase { @@ -568,7 +561,7 @@ public void testDelete() throws IOException, InterruptedException { ActionListener deleteModelTrainingListener = ActionListener.wrap( response -> fail("Deleting model when model does not exist should throw ResourceNotFoundException"), exception -> { - assertTrue(exception instanceof DeleteModelWhenInTrainStateException); + assertTrue(exception instanceof DeleteModelException); assertFalse(modelDao.isModelInGraveyard(modelId)); inProgressLatch2.countDown(); } @@ -636,6 +629,91 @@ public void testDelete() throws IOException, InterruptedException { assertTrue(inProgressLatch3.await(100, TimeUnit.SECONDS)); } + // Test Delete Model when the model is in use by an index + public void testDeleteModelInUse() throws IOException, ExecutionException, InterruptedException { + String modelId = "test-model-id-training"; + ModelDao modelDao = ModelDao.OpenSearchKNNModelDao.getInstance(); + byte[] modelBlob = "deleteModel".getBytes(); + int dimension = 2; + createIndex(MODEL_INDEX_NAME); + + Model model = new Model( + new ModelMetadata( + KNNEngine.DEFAULT, + SpaceType.DEFAULT, + dimension, + ModelState.CREATED, + ZonedDateTime.now(ZoneOffset.UTC).toString(), + "", + "", + "", + MethodComponentContext.EMPTY + ), + modelBlob, + modelId + ); + + // created model and added it to index + addDoc(model); + + String testIndex = "test-index"; + String testField = "test-field"; + + /* + Constructs the following json: + { + "properties": { + "test-field": { + "type": "knn_vector", + "model_id": "test-model-id-training" + } + } + } + */ + XContentBuilder mappings = XContentFactory.jsonBuilder() + .startObject() + .startObject(PROPERTIES) + .startObject(testField) + .field(TYPE, TYPE_KNN_VECTOR) + .field(MODEL_ID, modelId) + .endObject() + .endObject() + .endObject(); + + XContentBuilder settings = XContentFactory.jsonBuilder().startObject().field(TestUtils.INDEX_KNN, "true").endObject(); + + // Create index using model + CreateIndexRequestBuilder createIndexRequestBuilder = client().admin() + .indices() + .prepareCreate(testIndex) + .setMapping(mappings) + .setSettings(settings); + createIndex(testIndex, createIndexRequestBuilder); + + CountDownLatch latch = new CountDownLatch(1); + modelDao.delete(modelId, new ActionListener() { + @Override + public void onResponse(DeleteModelResponse deleteModelResponse) { + fail("Received delete model response when the request should have failed."); + } + + @Override + public void onFailure(Exception e) { + assertTrue(e instanceof DeleteModelException); + assertEquals( + String.format( + "Cannot delete model [%s]. Model is in use by the following indices [%s], which must be deleted first.", + modelId, + testIndex + ), + e.getMessage() + ); + latch.countDown(); + } + }); + assertTrue(latch.await(60, TimeUnit.SECONDS)); + } + // Test Delete Model when modelId is in Model Graveyard (previous delete model request which failed to // remove modelId from model graveyard). But, the model does not exist public void testDeleteModelWithModelInGraveyardModelDoesNotExist() throws InterruptedException { @@ -911,34 +989,4 @@ public void testDeleteWithStepListenersOnFailureModelBlocked() throws Interrupte assertTrue(inProgressLatch1.await(100, TimeUnit.SECONDS)); } - - public void addDoc(Model model) throws IOException, ExecutionException, InterruptedException { - ModelMetadata modelMetadata = model.getModelMetadata(); - - XContentBuilder builder = XContentFactory.jsonBuilder() - .startObject() - .field(MODEL_ID, model.getModelID()) - .field(KNN_ENGINE, modelMetadata.getKnnEngine().getName()) - .field(METHOD_PARAMETER_SPACE_TYPE, modelMetadata.getSpaceType().getValue()) - .field(DIMENSION, modelMetadata.getDimension()) - .field(MODEL_STATE, modelMetadata.getState().getName()) - .field(MODEL_TIMESTAMP, modelMetadata.getTimestamp().toString()) - .field(MODEL_DESCRIPTION, modelMetadata.getDescription()) - .field(MODEL_ERROR, modelMetadata.getError()); - - if (model.getModelBlob() != null) { - builder.field(MODEL_BLOB_PARAMETER, Base64.getEncoder().encodeToString(model.getModelBlob())); - } - - builder.endObject(); - - IndexRequest indexRequest = new IndexRequest().index(MODEL_INDEX_NAME) - .id(model.getModelID()) - .source(builder) - .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); - - IndexResponse response = client().index(indexRequest).get(); - assertTrue(response.status() == RestStatus.CREATED || response.status() == RestStatus.OK); - } - } diff --git a/src/test/java/org/opensearch/knn/plugin/action/RestKNNStatsHandlerIT.java b/src/test/java/org/opensearch/knn/plugin/action/RestKNNStatsHandlerIT.java index 9f28d5a71..9af1f49cc 100644 --- a/src/test/java/org/opensearch/knn/plugin/action/RestKNNStatsHandlerIT.java +++ b/src/test/java/org/opensearch/knn/plugin/action/RestKNNStatsHandlerIT.java @@ -34,6 +34,7 @@ import java.util.*; import static org.opensearch.knn.TestUtils.*; +import static org.opensearch.knn.TestUtils.PROPERTIES; import static org.opensearch.knn.common.KNNConstants.*; /** diff --git a/src/test/java/org/opensearch/knn/plugin/transport/UpdateModelGraveyardTransportActionTests.java b/src/test/java/org/opensearch/knn/plugin/transport/UpdateModelGraveyardTransportActionTests.java index b1983b964..cd60d566c 100644 --- a/src/test/java/org/opensearch/knn/plugin/transport/UpdateModelGraveyardTransportActionTests.java +++ b/src/test/java/org/opensearch/knn/plugin/transport/UpdateModelGraveyardTransportActionTests.java @@ -5,18 +5,41 @@ package org.opensearch.knn.plugin.transport; +import org.opensearch.action.admin.indices.create.CreateIndexRequestBuilder; +import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.action.ActionListener; import org.opensearch.action.support.master.AcknowledgedResponse; import org.opensearch.cluster.ClusterState; import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.knn.KNNSingleNodeTestCase; +import org.opensearch.knn.TestUtils; +import org.opensearch.knn.common.exception.DeleteModelException; +import org.opensearch.knn.index.MethodComponentContext; +import org.opensearch.knn.index.SpaceType; +import org.opensearch.knn.index.util.KNNEngine; +import org.opensearch.knn.indices.Model; import org.opensearch.knn.indices.ModelGraveyard; +import org.opensearch.knn.indices.ModelMetadata; +import org.opensearch.knn.indices.ModelState; import org.opensearch.threadpool.ThreadPool; import java.io.IOException; +import java.time.ZoneOffset; +import java.time.ZonedDateTime; +import java.util.ArrayList; +import java.util.List; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; +import static org.opensearch.knn.common.KNNConstants.DIMENSION; +import static org.opensearch.knn.common.KNNConstants.MODEL_ID; +import static org.opensearch.knn.common.KNNConstants.MODEL_INDEX_NAME; +import static org.opensearch.knn.common.KNNConstants.PROPERTIES; +import static org.opensearch.knn.common.KNNConstants.TYPE; +import static org.opensearch.knn.common.KNNConstants.TYPE_KNN_VECTOR; + public class UpdateModelGraveyardTransportActionTests extends KNNSingleNodeTestCase { public void testExecutor() { @@ -165,4 +188,201 @@ public void testCheckBlock() { .getInstance(UpdateModelGraveyardTransportAction.class); assertNull(updateModelGraveyardTransportAction.checkBlock(null, null)); } + + public void testGetIndicesUsingModel() throws IOException, ExecutionException, InterruptedException { + // Get update transport action + UpdateModelGraveyardTransportAction updateModelGraveyardTransportAction = node().injector() + .getInstance(UpdateModelGraveyardTransportAction.class); + + String modelId = "test-model-id"; + byte[] modelBlob = "testModel".getBytes(); + int dimension = 2; + + createIndex(MODEL_INDEX_NAME); + + Model model = new Model( + new ModelMetadata( + KNNEngine.DEFAULT, + SpaceType.DEFAULT, + dimension, + ModelState.CREATED, + ZonedDateTime.now(ZoneOffset.UTC).toString(), + "", + "", + "", + MethodComponentContext.EMPTY + ), + modelBlob, + modelId + ); + + // created model and added it to index + addDoc(model); + + // Create basic index (not using k-NN) + String testIndex1 = "test-index1"; + createIndex(testIndex1); + + // Attempt to add model id to graveyard with one non-knn index present, should succeed + UpdateModelGraveyardRequest addModelGraveyardRequest = new UpdateModelGraveyardRequest(modelId, false); + updateModelGraveyardAndAssertNoError(updateModelGraveyardTransportAction, addModelGraveyardRequest); + + // Remove model from graveyard to prepare for next check + UpdateModelGraveyardRequest removeModelGraveyardRequest = new UpdateModelGraveyardRequest(modelId, true); + updateModelGraveyardAndAssertNoError(updateModelGraveyardTransportAction, removeModelGraveyardRequest); + + // Create k-NN index not using the model + String testIndex2 = "test-index2"; + createKNNIndex(testIndex2); + + // Attempt to add model id to graveyard with one non-knn index and one k-nn index not using model present, should succeed + updateModelGraveyardAndAssertNoError(updateModelGraveyardTransportAction, addModelGraveyardRequest); + + // Remove model from graveyard to prepare for next check + updateModelGraveyardAndAssertNoError(updateModelGraveyardTransportAction, removeModelGraveyardRequest); + + // Create k-NN index using model + String testIndex3 = "test-index3"; + String testField3 = "test-field3"; + + /* + Constructs the following json: + { + "properties": { + "test-field3": { + "type": "knn_vector", + "model_id": "test-model-id" + } + } + } + */ + XContentBuilder mappings3 = XContentFactory.jsonBuilder() + .startObject() + .startObject(PROPERTIES) + .startObject(testField3) + .field(TYPE, TYPE_KNN_VECTOR) + .field(MODEL_ID, modelId) + .endObject() + .endObject() + .endObject(); + + XContentBuilder settings = XContentFactory.jsonBuilder().startObject().field(TestUtils.INDEX_KNN, "true").endObject(); + + CreateIndexRequestBuilder createIndexRequestBuilder3 = client().admin() + .indices() + .prepareCreate(testIndex3) + .setMapping(mappings3) + .setSettings(settings); + createIndex(testIndex3, createIndexRequestBuilder3); + + // Attempt to add model id to graveyard when one index is using model, should fail + List indicesUsingModel = new ArrayList<>(); + indicesUsingModel.add(testIndex3); + updateModelGraveyardAndAssertDeleteModelException( + updateModelGraveyardTransportAction, + addModelGraveyardRequest, + indicesUsingModel.toString() + ); + + // Create second k-NN index using model + String testIndex4 = "test-index4"; + String testField4 = "test-field4"; + String standardField = "standard-field"; + + /* + Constructs the following json: + { + "properties": { + "standard-field": { + "type": "knn_vector", + "dimension": "2" + } + "test-field4": { + "type": "knn_vector", + "model_id": "test-model-id" + } + } + } + */ + XContentBuilder mappings4 = XContentFactory.jsonBuilder() + .startObject() + .startObject(PROPERTIES) + .startObject(standardField) + .field(TYPE, TYPE_KNN_VECTOR) + .field(DIMENSION, dimension) + .endObject() + .startObject(testField4) + .field(TYPE, TYPE_KNN_VECTOR) + .field(MODEL_ID, modelId) + .endObject() + .endObject() + .endObject(); + + CreateIndexRequestBuilder createIndexRequestBuilder4 = client().admin() + .indices() + .prepareCreate(testIndex4) + .setMapping(mappings4) + .setSettings(settings); + createIndex(testIndex4, createIndexRequestBuilder4); + + // Add index at beginning to match order of list returned by getIndicesUsingModel() + indicesUsingModel.add(0, testIndex4); + + // Attempt to add model id to graveyard when one index is using model, should fail + updateModelGraveyardAndAssertDeleteModelException( + updateModelGraveyardTransportAction, + addModelGraveyardRequest, + indicesUsingModel.toString() + ); + } + + public void updateModelGraveyardAndAssertNoError( + UpdateModelGraveyardTransportAction updateModelGraveyardTransportAction, + UpdateModelGraveyardRequest updateModelGraveyardRequest + ) throws InterruptedException { + final CountDownLatch countDownLatch = new CountDownLatch(1); + client().admin().cluster().prepareState().execute(ActionListener.wrap(stateResponse1 -> { + ClusterState clusterState1 = stateResponse1.getState(); + updateModelGraveyardTransportAction.clusterManagerOperation( + updateModelGraveyardRequest, + clusterState1, + ActionListener.wrap(acknowledgedResponse -> { + assertTrue(acknowledgedResponse.isAcknowledged()); + countDownLatch.countDown(); + }, e -> { fail("Update failed: " + e); }) + ); + }, e -> fail("Update failed: " + e))); + assertTrue(countDownLatch.await(60, TimeUnit.SECONDS)); + } + + public void updateModelGraveyardAndAssertDeleteModelException( + UpdateModelGraveyardTransportAction updateModelGraveyardTransportAction, + UpdateModelGraveyardRequest updateModelGraveyardRequest, + String indicesPresentInException + ) throws InterruptedException { + final CountDownLatch countDownLatch = new CountDownLatch(1); + client().admin().cluster().prepareState().execute(ActionListener.wrap(stateResponse1 -> { + ClusterState clusterState1 = stateResponse1.getState(); + updateModelGraveyardTransportAction.clusterManagerOperation( + updateModelGraveyardRequest, + clusterState1, + ActionListener.wrap(acknowledgedResponse -> { + fail(); + }, e -> { + assertTrue(e instanceof DeleteModelException); + assertEquals( + String.format( + "Cannot delete model [%s]. Model is in use by the following indices %s, which must be deleted first.", + updateModelGraveyardRequest.getModelId(), + indicesPresentInException + ), + e.getMessage() + ); + countDownLatch.countDown(); + }) + ); + }, e -> fail("Update failed: " + e))); + + assertTrue(countDownLatch.await(60, TimeUnit.SECONDS)); + } } diff --git a/src/test/java/org/opensearch/knn/recall/RecallTestsIT.java b/src/test/java/org/opensearch/knn/recall/RecallTestsIT.java index c90afaa62..656b95c2e 100644 --- a/src/test/java/org/opensearch/knn/recall/RecallTestsIT.java +++ b/src/test/java/org/opensearch/knn/recall/RecallTestsIT.java @@ -312,6 +312,8 @@ public void testRecall_whenFaissIVFFP32_thenRecallAbove75percent() { createIndexAndIngestDocs(indexName, TEST_FIELD_NAME, getSettings(), getModelMapping()); assertRecall(indexName, spaceType, 0.25f); + deleteIndex(indexName); + // Delete the model deleteModel(TEST_MODEL_ID); } @@ -387,6 +389,8 @@ public void testRecall_whenFaissIVFPQFP32_thenRecallAbove50percent() { createIndexAndIngestDocs(indexName, TEST_FIELD_NAME, getSettings(), getModelMapping()); assertRecall(indexName, spaceType, 0.5f); + deleteIndex(indexName); + // Delete the model deleteModel(TEST_MODEL_ID); } @@ -463,6 +467,8 @@ public void testRecall_whenFaissHNSWPQFP32_thenRecallAbove50percent() { createIndexAndIngestDocs(indexName, TEST_FIELD_NAME, getSettings(), getModelMapping()); assertRecall(indexName, spaceType, 0.5f); + deleteIndex(indexName); + // Delete the model deleteModel(TEST_MODEL_ID); } From b532fa0b1abade76253054661946b4199348d445 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Wed, 12 Jun 2024 17:11:06 -0500 Subject: [PATCH 277/416] Bump jinja2 from 2.11.3 to 3.1.3 (#1750) (#1751) Signed-off-by: Naveen Tatikonda (cherry picked from commit d0b0214d6b75ca4534ace2c027fcb377a128bfeb) Co-authored-by: Naveen Tatikonda --- benchmarks/osb/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmarks/osb/requirements.txt b/benchmarks/osb/requirements.txt index 0829cdd54..a220ee44f 100644 --- a/benchmarks/osb/requirements.txt +++ b/benchmarks/osb/requirements.txt @@ -38,7 +38,7 @@ ijson==2.6.1 # via opensearch-benchmark importlib-metadata==4.11.3 # via jsonschema -jinja2==2.11.3 +jinja2==3.1.3 # via opensearch-benchmark jsonschema==3.1.1 # via opensearch-benchmark From cf5a754ceb1ea3315a6a7f95a6126d29039924f6 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Mon, 17 Jun 2024 10:16:00 -0700 Subject: [PATCH 278/416] Optimize logic and tests for blocking model deletion when in use (#1757) (#1759) * Optimize logic and tests for blocking model deletion when in use Signed-off-by: Ryan Bogan * Change if statement Co-authored-by: Heemin Kim Signed-off-by: Ryan Bogan --------- Signed-off-by: Ryan Bogan Co-authored-by: Heemin Kim (cherry picked from commit 6e30a3bf86c2b1abe9f4be4f3727eef35d04f838) Co-authored-by: Ryan Bogan --- .../UpdateModelGraveyardTransportAction.java | 26 +---- .../opensearch/knn/KNNSingleNodeTestCase.java | 25 ++++- .../opensearch/knn/indices/ModelDaoTests.java | 105 +----------------- ...ateModelGraveyardTransportActionTests.java | 8 +- 4 files changed, 39 insertions(+), 125 deletions(-) diff --git a/src/main/java/org/opensearch/knn/plugin/transport/UpdateModelGraveyardTransportAction.java b/src/main/java/org/opensearch/knn/plugin/transport/UpdateModelGraveyardTransportAction.java index d7536a5a7..7d5750c2b 100644 --- a/src/main/java/org/opensearch/knn/plugin/transport/UpdateModelGraveyardTransportAction.java +++ b/src/main/java/org/opensearch/knn/plugin/transport/UpdateModelGraveyardTransportAction.java @@ -197,31 +197,17 @@ private List getIndicesUsingModel(ClusterState clusterState, UpdateModel .filter(entry -> entry.getValue() != null) .filter(entry -> { Object properties = entry.getValue().getSourceAsMap().get("properties"); - if (properties == null || properties instanceof Map == false) { + if ((properties instanceof Map) == false) { return false; } - Map propertiesMap = (Map) properties; - return propertiesMapContainsModel(propertiesMap, task.getModelId()); + Map propertiesMap = (Map) properties; + return propertiesMap.values() + .stream() + .filter(obj -> obj instanceof Map) + .anyMatch(obj -> task.getModelId().equals(((Map) obj).get(MODEL_ID))); }) .map(Map.Entry::getKey) .collect(toList()); } - - private boolean propertiesMapContainsModel(Map propertiesMap, String modelId) { - for (Map.Entry fieldsEntry : propertiesMap.entrySet()) { - if (fieldsEntry.getKey() != null && fieldsEntry.getValue() instanceof Map) { - Map innerMap = (Map) fieldsEntry.getValue(); - for (Map.Entry innerEntry : innerMap.entrySet()) { - // If model is in use, fail delete model request - if (innerEntry.getKey().equals(MODEL_ID) - && innerEntry.getValue() instanceof String - && innerEntry.getValue().equals(modelId)) { - return true; - } - } - } - } - return false; - } } } diff --git a/src/test/java/org/opensearch/knn/KNNSingleNodeTestCase.java b/src/test/java/org/opensearch/knn/KNNSingleNodeTestCase.java index d87a63ad5..f9c0161d6 100644 --- a/src/test/java/org/opensearch/knn/KNNSingleNodeTestCase.java +++ b/src/test/java/org/opensearch/knn/KNNSingleNodeTestCase.java @@ -5,6 +5,7 @@ package org.opensearch.knn; +import org.opensearch.core.action.ActionListener; import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.cluster.ClusterName; import org.opensearch.cluster.ClusterState; @@ -37,7 +38,11 @@ import org.opensearch.test.hamcrest.OpenSearchAssertions; import java.io.IOException; -import java.util.*; +import java.util.Base64; +import java.util.Collection; +import java.util.Collections; +import java.util.EnumSet; +import java.util.Map; import java.util.concurrent.ExecutionException; import static org.mockito.Mockito.when; @@ -184,7 +189,7 @@ protected void addDoc(String index, String docId, String fieldName, String dummy /** * Index a new model */ - protected void addDoc(Model model) throws IOException, ExecutionException, InterruptedException { + protected void writeModelToModelSystemIndex(Model model) throws IOException, ExecutionException, InterruptedException { ModelMetadata modelMetadata = model.getModelMetadata(); XContentBuilder builder = XContentFactory.jsonBuilder() @@ -213,6 +218,22 @@ protected void addDoc(Model model) throws IOException, ExecutionException, Inter assertTrue(response.status() == RestStatus.CREATED || response.status() == RestStatus.OK); } + // Add a new model to ModelDao + protected void addModel(Model model) throws IOException { + ModelDao modelDao = ModelDao.OpenSearchKNNModelDao.getInstance(); + modelDao.put(model, new ActionListener() { + @Override + public void onResponse(IndexResponse indexResponse) { + assertTrue(indexResponse.status() == RestStatus.CREATED || indexResponse.status() == RestStatus.OK); + } + + @Override + public void onFailure(Exception e) { + fail("Failed to add model: " + e); + } + }); + } + /** * Run a search against a k-NN index */ diff --git a/src/test/java/org/opensearch/knn/indices/ModelDaoTests.java b/src/test/java/org/opensearch/knn/indices/ModelDaoTests.java index 3a25c3064..75c523332 100644 --- a/src/test/java/org/opensearch/knn/indices/ModelDaoTests.java +++ b/src/test/java/org/opensearch/knn/indices/ModelDaoTests.java @@ -18,7 +18,6 @@ import org.opensearch.ExceptionsHelper; import org.opensearch.ResourceAlreadyExistsException; import org.opensearch.ResourceNotFoundException; -import org.opensearch.action.admin.indices.create.CreateIndexRequestBuilder; import org.opensearch.cluster.ClusterChangedEvent; import org.opensearch.core.action.ActionListener; import org.opensearch.action.DocWriteResponse; @@ -30,12 +29,9 @@ import org.opensearch.action.index.IndexResponse; import org.opensearch.action.support.WriteRequest; import org.opensearch.action.support.master.AcknowledgedResponse; -import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.index.IndexNotFoundException; import org.opensearch.index.engine.VersionConflictEngineException; import org.opensearch.knn.KNNSingleNodeTestCase; -import org.opensearch.knn.TestUtils; import org.opensearch.knn.common.exception.DeleteModelException; import org.opensearch.knn.index.MethodComponentContext; import org.opensearch.knn.index.SpaceType; @@ -65,11 +61,7 @@ import static org.mockito.Mockito.doNothing; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.mockStatic; -import static org.opensearch.knn.common.KNNConstants.MODEL_ID; import static org.opensearch.knn.common.KNNConstants.MODEL_INDEX_NAME; -import static org.opensearch.knn.common.KNNConstants.PROPERTIES; -import static org.opensearch.knn.common.KNNConstants.TYPE; -import static org.opensearch.knn.common.KNNConstants.TYPE_KNN_VECTOR; public class ModelDaoTests extends KNNSingleNodeTestCase { @@ -152,7 +144,7 @@ public void testModelIndexHealth() throws InterruptedException, ExecutionExcepti modelBlob, modelId ); - addDoc(model); + writeModelToModelSystemIndex(model); assertEquals(model, modelDao.get(modelId)); assertNotNull(modelDao.getHealthStatus()); @@ -172,7 +164,7 @@ public void testModelIndexHealth() throws InterruptedException, ExecutionExcepti modelBlob, modelId ); - addDoc(model); + writeModelToModelSystemIndex(model); assertEquals(model, modelDao.get(modelId)); assertNotNull(modelDao.getHealthStatus()); } @@ -450,7 +442,7 @@ public void testGet() throws IOException, InterruptedException, ExecutionExcepti modelBlob, modelId ); - addDoc(model); + writeModelToModelSystemIndex(model); assertEquals(model, modelDao.get(modelId)); // Get model during training @@ -469,7 +461,7 @@ public void testGet() throws IOException, InterruptedException, ExecutionExcepti null, modelId ); - addDoc(model); + writeModelToModelSystemIndex(model); assertEquals(model, modelDao.get(modelId)); } @@ -629,91 +621,6 @@ public void testDelete() throws IOException, InterruptedException { assertTrue(inProgressLatch3.await(100, TimeUnit.SECONDS)); } - // Test Delete Model when the model is in use by an index - public void testDeleteModelInUse() throws IOException, ExecutionException, InterruptedException { - String modelId = "test-model-id-training"; - ModelDao modelDao = ModelDao.OpenSearchKNNModelDao.getInstance(); - byte[] modelBlob = "deleteModel".getBytes(); - int dimension = 2; - createIndex(MODEL_INDEX_NAME); - - Model model = new Model( - new ModelMetadata( - KNNEngine.DEFAULT, - SpaceType.DEFAULT, - dimension, - ModelState.CREATED, - ZonedDateTime.now(ZoneOffset.UTC).toString(), - "", - "", - "", - MethodComponentContext.EMPTY - ), - modelBlob, - modelId - ); - - // created model and added it to index - addDoc(model); - - String testIndex = "test-index"; - String testField = "test-field"; - - /* - Constructs the following json: - { - "properties": { - "test-field": { - "type": "knn_vector", - "model_id": "test-model-id-training" - } - } - } - */ - XContentBuilder mappings = XContentFactory.jsonBuilder() - .startObject() - .startObject(PROPERTIES) - .startObject(testField) - .field(TYPE, TYPE_KNN_VECTOR) - .field(MODEL_ID, modelId) - .endObject() - .endObject() - .endObject(); - - XContentBuilder settings = XContentFactory.jsonBuilder().startObject().field(TestUtils.INDEX_KNN, "true").endObject(); - - // Create index using model - CreateIndexRequestBuilder createIndexRequestBuilder = client().admin() - .indices() - .prepareCreate(testIndex) - .setMapping(mappings) - .setSettings(settings); - createIndex(testIndex, createIndexRequestBuilder); - - CountDownLatch latch = new CountDownLatch(1); - modelDao.delete(modelId, new ActionListener() { - @Override - public void onResponse(DeleteModelResponse deleteModelResponse) { - fail("Received delete model response when the request should have failed."); - } - - @Override - public void onFailure(Exception e) { - assertTrue(e instanceof DeleteModelException); - assertEquals( - String.format( - "Cannot delete model [%s]. Model is in use by the following indices [%s], which must be deleted first.", - modelId, - testIndex - ), - e.getMessage() - ); - latch.countDown(); - } - }); - assertTrue(latch.await(60, TimeUnit.SECONDS)); - } - // Test Delete Model when modelId is in Model Graveyard (previous delete model request which failed to // remove modelId from model graveyard). But, the model does not exist public void testDeleteModelWithModelInGraveyardModelDoesNotExist() throws InterruptedException { @@ -772,7 +679,7 @@ public void testDeleteModelInTrainingWithStepListeners() throws IOException, Exe ); // created model and added it to index - addDoc(model); + writeModelToModelSystemIndex(model); final CountDownLatch inProgressLatch = new CountDownLatch(1); @@ -814,7 +721,7 @@ public void testDeleteWithStepListeners() throws IOException, InterruptedExcepti ); // created model and added it to index - addDoc(model); + writeModelToModelSystemIndex(model); final CountDownLatch inProgressLatch = new CountDownLatch(1); diff --git a/src/test/java/org/opensearch/knn/plugin/transport/UpdateModelGraveyardTransportActionTests.java b/src/test/java/org/opensearch/knn/plugin/transport/UpdateModelGraveyardTransportActionTests.java index cd60d566c..5be907ebd 100644 --- a/src/test/java/org/opensearch/knn/plugin/transport/UpdateModelGraveyardTransportActionTests.java +++ b/src/test/java/org/opensearch/knn/plugin/transport/UpdateModelGraveyardTransportActionTests.java @@ -189,7 +189,7 @@ public void testCheckBlock() { assertNull(updateModelGraveyardTransportAction.checkBlock(null, null)); } - public void testGetIndicesUsingModel() throws IOException, ExecutionException, InterruptedException { + public void testClusterManagerOperation_GetIndicesUsingModel() throws IOException, ExecutionException, InterruptedException { // Get update transport action UpdateModelGraveyardTransportAction updateModelGraveyardTransportAction = node().injector() .getInstance(UpdateModelGraveyardTransportAction.class); @@ -217,7 +217,7 @@ public void testGetIndicesUsingModel() throws IOException, ExecutionException, I ); // created model and added it to index - addDoc(model); + addModel(model); // Create basic index (not using k-NN) String testIndex1 = "test-index1"; @@ -336,7 +336,7 @@ public void testGetIndicesUsingModel() throws IOException, ExecutionException, I ); } - public void updateModelGraveyardAndAssertNoError( + private void updateModelGraveyardAndAssertNoError( UpdateModelGraveyardTransportAction updateModelGraveyardTransportAction, UpdateModelGraveyardRequest updateModelGraveyardRequest ) throws InterruptedException { @@ -355,7 +355,7 @@ public void updateModelGraveyardAndAssertNoError( assertTrue(countDownLatch.await(60, TimeUnit.SECONDS)); } - public void updateModelGraveyardAndAssertDeleteModelException( + private void updateModelGraveyardAndAssertDeleteModelException( UpdateModelGraveyardTransportAction updateModelGraveyardTransportAction, UpdateModelGraveyardRequest updateModelGraveyardRequest, String indicesPresentInException From 4ab53395d45e5aeea15325c55babfd68cbf04de9 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Fri, 28 Jun 2024 14:31:28 -0500 Subject: [PATCH 279/416] Increment version to 2.16.0-SNAPSHOT (#1749) Signed-off-by: opensearch-ci-bot Co-authored-by: opensearch-ci-bot --- .../workflows/backwards_compatibility_tests_workflow.yml | 8 ++++---- build.gradle | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/backwards_compatibility_tests_workflow.yml b/.github/workflows/backwards_compatibility_tests_workflow.yml index 48925dbc2..8946ed2c1 100644 --- a/.github/workflows/backwards_compatibility_tests_workflow.yml +++ b/.github/workflows/backwards_compatibility_tests_workflow.yml @@ -14,8 +14,8 @@ jobs: strategy: matrix: java: [ 11, 17 ] - bwc_version : [ "1.1.0", "1.2.4", "1.3.8", "2.0.1", "2.1.0", "2.2.1", "2.3.0", "2.4.1", "2.5.0", "2.6.0", "2.7.0", "2.8.0", "2.9.0", "2.10.0", "2.11.0", "2.12.0", "2.13.0", "2.14.0" ] - opensearch_version : [ "2.15.0-SNAPSHOT" ] + bwc_version : [ "1.1.0", "1.2.4", "1.3.8", "2.0.1", "2.1.0", "2.2.1", "2.3.0", "2.4.1", "2.5.0", "2.6.0", "2.7.0", "2.8.0", "2.9.0", "2.10.0", "2.11.0", "2.12.0", "2.13.0", "2.14.0", "2.15.0" ] + opensearch_version : [ "2.16.0-SNAPSHOT" ] name: k-NN Restart-Upgrade BWC Tests runs-on: ubuntu-latest @@ -52,8 +52,8 @@ jobs: strategy: matrix: java: [ 11, 17 ] - bwc_version: [ "1.3.8", "2.0.1", "2.1.0", "2.2.1", "2.3.0", "2.4.1", "2.5.0", "2.6.0", "2.7.0", "2.8.0", "2.9.0", "2.10.0", "2.11.0", "2.12.0", "2.13.0", "2.14.0"] - opensearch_version: [ "2.15.0-SNAPSHOT" ] + bwc_version: [ "1.3.8", "2.0.1", "2.1.0", "2.2.1", "2.3.0", "2.4.1", "2.5.0", "2.6.0", "2.7.0", "2.8.0", "2.9.0", "2.10.0", "2.11.0", "2.12.0", "2.13.0", "2.14.0", "2.15.0"] + opensearch_version: [ "2.16.0-SNAPSHOT" ] name: k-NN Rolling-Upgrade BWC Tests runs-on: ubuntu-latest diff --git a/build.gradle b/build.gradle index a0e194811..09851f65f 100644 --- a/build.gradle +++ b/build.gradle @@ -13,7 +13,7 @@ buildscript { ext { // build.version_qualifier parameter applies to knn plugin artifacts only. OpenSearch version must be set // explicitly as 'opensearch.version' property, for instance opensearch.version=2.0.0-rc1-SNAPSHOT - opensearch_version = System.getProperty("opensearch.version", "2.15.0-SNAPSHOT") + opensearch_version = System.getProperty("opensearch.version", "2.16.0-SNAPSHOT") version_qualifier = System.getProperty("build.version_qualifier", "") opensearch_group = "org.opensearch" isSnapshot = "true" == System.getProperty("build.snapshot", "true") From 946b99f078c82d20fda9ec0a6e70c89a22076b34 Mon Sep 17 00:00:00 2001 From: Tejas Shah Date: Wed, 3 Jul 2024 17:55:55 -0700 Subject: [PATCH 280/416] Adds ef_search as a query parameter for Lucene, FAISS and NMSLIB (#1783) (#1791) Currently ef_search is set at index level, this change gives the ability to have query time ef-search parameter without manipulating the index settings. query time value supersedes the value from index in FAISS and NMSLIB. For Lucene, max of k and ef_search is used as ef_search value Signed-off-by: Tejas Shah --- CHANGELOG.md | 1 + jni/CMakeLists.txt | 2 + jni/cmake/init-nmslib.cmake | 3 +- jni/include/commons.h | 5 + jni/include/faiss_wrapper.h | 28 +- jni/include/nmslib_wrapper.h | 2 +- .../org_opensearch_knn_jni_FaissService.h | 8 +- .../org_opensearch_knn_jni_NmslibService.h | 2 +- ...pass-ef-parameter-in-the-query-for-h.patch | 303 +++++++++++++ jni/src/commons.cpp | 14 +- jni/src/faiss_wrapper.cpp | 30 +- jni/src/nmslib_wrapper.cpp | 35 +- .../org_opensearch_knn_jni_FaissService.cpp | 8 +- .../org_opensearch_knn_jni_NmslibService.cpp | 4 +- jni/tests/commons_test.cpp | 19 + jni/tests/faiss_wrapper_test.cpp | 19 +- jni/tests/faiss_wrapper_unit_test.cpp | 251 +++++++++++ jni/tests/nmslib_wrapper_test.cpp | 5 +- jni/tests/nmslib_wrapper_unit_test.cpp | 135 ++++++ .../java/org/opensearch/knn/bwc/ModelIT.java | 2 +- .../org/opensearch/knn/bwc/QueryANNIT.java | 37 ++ .../org/opensearch/knn/bwc/IndexingIT.java | 4 +- .../org/opensearch/knn/bwc/QueryANNIT.java | 49 ++ .../java/org/opensearch/knn/bwc/WarmupIT.java | 3 +- .../opensearch/knn/common/KNNConstants.java | 1 + .../org/opensearch/knn/index/IndexUtil.java | 31 +- .../opensearch/knn/index/MethodComponent.java | 33 +- .../knn/index/query/BaseQueryFactory.java | 2 + .../opensearch/knn/index/query/KNNQuery.java | 50 +-- .../knn/index/query/KNNQueryBuilder.java | 342 ++++++++------ .../knn/index/query/KNNQueryFactory.java | 44 +- .../opensearch/knn/index/query/KNNWeight.java | 1 + .../query/parser/MethodParametersParser.java | 137 ++++++ .../index/query/request/MethodParameter.java | 77 ++++ .../knn/index/util/AbstractKNNLibrary.java | 10 + .../knn/index/util/DefaultHnswContext.java | 33 ++ .../util/EngineSpecificMethodContext.java | 31 ++ .../org/opensearch/knn/index/util/Faiss.java | 8 +- .../opensearch/knn/index/util/JVMLibrary.java | 4 +- .../opensearch/knn/index/util/KNNEngine.java | 5 + .../opensearch/knn/index/util/KNNLibrary.java | 7 + .../org/opensearch/knn/index/util/Lucene.java | 2 +- .../knn/index/util/NativeLibrary.java | 3 +- .../org/opensearch/knn/index/util/Nmslib.java | 2 +- .../org/opensearch/knn/jni/FaissService.java | 9 +- .../org/opensearch/knn/jni/JNIService.java | 16 +- .../org/opensearch/knn/jni/NmslibService.java | 2 +- .../knn/validation/ParameterValidator.java | 64 +++ .../knn/index/FaissHNSWFlatE2EIT.java | 194 ++++++++ .../org/opensearch/knn/index/FaissIT.java | 195 +------- .../opensearch/knn/index/LuceneEngineIT.java | 34 +- .../org/opensearch/knn/index/NmslibIT.java | 123 ++++- .../opensearch/knn/index/OpenSearchIT.java | 2 +- .../KNN80DocValuesConsumerTests.java | 12 +- .../knn/index/codec/KNNCodecTestUtil.java | 3 +- .../memory/NativeMemoryLoadStrategyTests.java | 2 +- .../KNNQueryBuilderInvalidParamsTests.java | 98 ++++ .../knn/index/query/KNNQueryBuilderTests.java | 423 ++++++++++++++---- .../KNNQueryBuilderValidParamsTests.java | 90 ++++ .../knn/index/query/KNNQueryFactoryTests.java | 153 ++++++- .../knn/index/query/KNNWeightTests.java | 96 +++- .../parser/MethodParametersParserTests.java | 84 ++++ .../index/util/AbstractKNNLibraryTests.java | 25 +- .../knn/index/util/NativeLibraryTests.java | 2 +- .../opensearch/knn/jni/JNIServiceTests.java | 66 ++- .../plugin/action/RestKNNStatsHandlerIT.java | 2 +- .../plugin/action/RestKNNWarmupHandlerIT.java | 6 +- .../action/RestLegacyKNNWarmupHandlerIT.java | 6 +- .../action/RestTrainModelHandlerIT.java | 5 +- .../LibraryInitializedSupplierTests.java | 6 + .../org/opensearch/knn/KNNRestTestCase.java | 99 ++-- 71 files changed, 2945 insertions(+), 664 deletions(-) create mode 100644 jni/patches/nmslib/0002-Adds-ability-to-pass-ef-parameter-in-the-query-for-h.patch create mode 100644 jni/tests/faiss_wrapper_unit_test.cpp create mode 100644 jni/tests/nmslib_wrapper_unit_test.cpp create mode 100644 qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/QueryANNIT.java create mode 100644 qa/rolling-upgrade/src/test/java/org/opensearch/knn/bwc/QueryANNIT.java create mode 100644 src/main/java/org/opensearch/knn/index/query/parser/MethodParametersParser.java create mode 100644 src/main/java/org/opensearch/knn/index/query/request/MethodParameter.java create mode 100644 src/main/java/org/opensearch/knn/index/util/DefaultHnswContext.java create mode 100644 src/main/java/org/opensearch/knn/index/util/EngineSpecificMethodContext.java create mode 100644 src/main/java/org/opensearch/knn/validation/ParameterValidator.java create mode 100644 src/test/java/org/opensearch/knn/index/FaissHNSWFlatE2EIT.java create mode 100644 src/test/java/org/opensearch/knn/index/query/KNNQueryBuilderInvalidParamsTests.java create mode 100644 src/test/java/org/opensearch/knn/index/query/KNNQueryBuilderValidParamsTests.java create mode 100644 src/test/java/org/opensearch/knn/index/query/parser/MethodParametersParserTests.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 62006a6f3..d38818a68 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ## [Unreleased 2.x](https://github.com/opensearch-project/k-NN/compare/2.15...2.x) ### Features +* Adds dynamic query parameter ef_search [#1783](https://github.com/opensearch-project/k-NN/pull/1783) ### Enhancements ### Bug Fixes ### Infrastructure diff --git a/jni/CMakeLists.txt b/jni/CMakeLists.txt index 595fa6fea..e2003e0f7 100644 --- a/jni/CMakeLists.txt +++ b/jni/CMakeLists.txt @@ -147,8 +147,10 @@ if ("${WIN32}" STREQUAL "") add_executable( jni_test tests/faiss_wrapper_test.cpp + tests/faiss_wrapper_unit_test.cpp tests/faiss_util_test.cpp tests/nmslib_wrapper_test.cpp + tests/nmslib_wrapper_unit_test.cpp tests/test_util.cpp tests/commons_test.cpp ) diff --git a/jni/cmake/init-nmslib.cmake b/jni/cmake/init-nmslib.cmake index a735bcbd8..387dce6bc 100644 --- a/jni/cmake/init-nmslib.cmake +++ b/jni/cmake/init-nmslib.cmake @@ -13,12 +13,13 @@ if (NOT EXISTS ${NMS_REPO_DIR}) endif () # Check if patch exist, this is to skip git apply during CI build. See CI.yml with ubuntu. -find_path(PATCH_FILE NAMES 0001-Initialize-maxlevel-during-add-from-enterpoint-level.patch PATHS ${CMAKE_CURRENT_SOURCE_DIR}/patches/nmslib NO_DEFAULT_PATH) +find_path(PATCH_FILE NAMES 0001-Initialize-maxlevel-during-add-from-enterpoint-level.patch 0002-Adds-ability-to-pass-ef-parameter-in-the-query-for-h.patch PATHS ${CMAKE_CURRENT_SOURCE_DIR}/patches/nmslib NO_DEFAULT_PATH) # If it exists, apply patches if (EXISTS ${PATCH_FILE}) message(STATUS "Applying custom patches.") execute_process(COMMAND git ${GIT_PATCH_COMMAND} --3way --ignore-space-change --ignore-whitespace ${CMAKE_CURRENT_SOURCE_DIR}/patches/nmslib/0001-Initialize-maxlevel-during-add-from-enterpoint-level.patch WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/external/nmslib ERROR_VARIABLE ERROR_MSG RESULT_VARIABLE RESULT_CODE) + execute_process(COMMAND git ${GIT_PATCH_COMMAND} --3way --ignore-space-change --ignore-whitespace ${CMAKE_CURRENT_SOURCE_DIR}/patches/nmslib/0002-Adds-ability-to-pass-ef-parameter-in-the-query-for-h.patch WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/external/nmslib ERROR_VARIABLE ERROR_MSG RESULT_VARIABLE RESULT_CODE) if(RESULT_CODE) message(FATAL_ERROR "Failed to apply patch:\n${ERROR_MSG}") diff --git a/jni/include/commons.h b/jni/include/commons.h index 05367a693..67a141c8b 100644 --- a/jni/include/commons.h +++ b/jni/include/commons.h @@ -33,5 +33,10 @@ namespace knn_jni { * @param memoryAddress address to be freed. */ void freeVectorData(jlong); + + /** + * Extracts query time efSearch from method parameters + **/ + int getIntegerMethodParameter(JNIEnv *, knn_jni::JNIUtilInterface *, std::unordered_map, std::string, int); } } diff --git a/jni/include/faiss_wrapper.h b/jni/include/faiss_wrapper.h index 958eca8ac..aa747862a 100644 --- a/jni/include/faiss_wrapper.h +++ b/jni/include/faiss_wrapper.h @@ -45,17 +45,27 @@ namespace knn_jni { // Sets the sharedIndexState for an index void SetSharedIndexState(jlong indexPointerJ, jlong shareIndexStatePointerJ); - // Execute a query against the index located in memory at indexPointerJ. - // - // Return an array of KNNQueryResults + /** + * Execute a query against the index located in memory at indexPointerJ + * + * Parameters: + * methodParamsJ: introduces a map to have additional method parameters + * + * Return an array of KNNQueryResults + */ jobjectArray QueryIndex(knn_jni::JNIUtilInterface * jniUtil, JNIEnv * env, jlong indexPointerJ, - jfloatArray queryVectorJ, jint kJ, jintArray parentIdsJ); - - // Execute a query against the index located in memory at indexPointerJ along with Filters - // - // Return an array of KNNQueryResults + jfloatArray queryVectorJ, jint kJ, jobject methodParamsJ, jintArray parentIdsJ); + + /** + * Execute a query against the index located in memory at indexPointerJ along with Filters + * + * Parameters: + * methodParamsJ: introduces a map to have additional method parameters + * + * Return an array of KNNQueryResults + */ jobjectArray QueryIndex_WithFilter(knn_jni::JNIUtilInterface * jniUtil, JNIEnv * env, jlong indexPointerJ, - jfloatArray queryVectorJ, jint kJ, jlongArray filterIdsJ, + jfloatArray queryVectorJ, jint kJ, jobject methodParamsJ, jlongArray filterIdsJ, jint filterIdsTypeJ, jintArray parentIdsJ); // Free the index located in memory at indexPointerJ diff --git a/jni/include/nmslib_wrapper.h b/jni/include/nmslib_wrapper.h index 08494644f..27a013c10 100644 --- a/jni/include/nmslib_wrapper.h +++ b/jni/include/nmslib_wrapper.h @@ -37,7 +37,7 @@ namespace knn_jni { // // Return an array of KNNQueryResults jobjectArray QueryIndex(knn_jni::JNIUtilInterface * jniUtil, JNIEnv * env, jlong indexPointerJ, - jfloatArray queryVectorJ, jint kJ); + jfloatArray queryVectorJ, jint kJ, jobject methodParamsJ); // Free the index located in memory at indexPointerJ void Free(jlong indexPointer); diff --git a/jni/include/org_opensearch_knn_jni_FaissService.h b/jni/include/org_opensearch_knn_jni_FaissService.h index e16677db7..0453864e4 100644 --- a/jni/include/org_opensearch_knn_jni_FaissService.h +++ b/jni/include/org_opensearch_knn_jni_FaissService.h @@ -69,18 +69,18 @@ JNIEXPORT void JNICALL Java_org_opensearch_knn_jni_FaissService_setSharedIndexSt /* * Class: org_opensearch_knn_jni_FaissService * Method: queryIndex - * Signature: (J[FI[I)[Lorg/opensearch/knn/index/query/KNNQueryResult; + * Signature: (J[FI[Ljava/util/MapI)[Lorg/opensearch/knn/index/query/KNNQueryResult; */ JNIEXPORT jobjectArray JNICALL Java_org_opensearch_knn_jni_FaissService_queryIndex - (JNIEnv *, jclass, jlong, jfloatArray, jint, jintArray); + (JNIEnv *, jclass, jlong, jfloatArray, jint, jobject, jintArray); /* * Class: org_opensearch_knn_jni_FaissService * Method: queryIndexWithFilter - * Signature: (J[FI[JI[I)[Lorg/opensearch/knn/index/query/KNNQueryResult; + * Signature: (J[FI[JLjava/util/MapI[I)[Lorg/opensearch/knn/index/query/KNNQueryResult; */ JNIEXPORT jobjectArray JNICALL Java_org_opensearch_knn_jni_FaissService_queryIndexWithFilter - (JNIEnv *, jclass, jlong, jfloatArray, jint, jlongArray, jint, jintArray); + (JNIEnv *, jclass, jlong, jfloatArray, jint, jobject, jlongArray, jint, jintArray); /* * Class: org_opensearch_knn_jni_FaissService diff --git a/jni/include/org_opensearch_knn_jni_NmslibService.h b/jni/include/org_opensearch_knn_jni_NmslibService.h index 31422955f..a9d5238b7 100644 --- a/jni/include/org_opensearch_knn_jni_NmslibService.h +++ b/jni/include/org_opensearch_knn_jni_NmslibService.h @@ -40,7 +40,7 @@ JNIEXPORT jlong JNICALL Java_org_opensearch_knn_jni_NmslibService_loadIndex * Signature: (J[FI)[Lorg/opensearch/knn/index/query/KNNQueryResult; */ JNIEXPORT jobjectArray JNICALL Java_org_opensearch_knn_jni_NmslibService_queryIndex - (JNIEnv *, jclass, jlong, jfloatArray, jint); + (JNIEnv *, jclass, jlong, jfloatArray, jint, jobject); /* * Class: org_opensearch_knn_jni_NmslibService diff --git a/jni/patches/nmslib/0002-Adds-ability-to-pass-ef-parameter-in-the-query-for-h.patch b/jni/patches/nmslib/0002-Adds-ability-to-pass-ef-parameter-in-the-query-for-h.patch new file mode 100644 index 000000000..e55e52cb3 --- /dev/null +++ b/jni/patches/nmslib/0002-Adds-ability-to-pass-ef-parameter-in-the-query-for-h.patch @@ -0,0 +1,303 @@ +From d700d93d5efda7349b90f6f0b2373580ced8097d Mon Sep 17 00:00:00 2001 +From: Tejas Shah +Date: Mon, 27 May 2024 22:02:12 -0700 +Subject: [PATCH] Adds ability to pass ef parameter in the query for hnsw + +It defaults to index ef_ value if its not type HNSWQuery +--- + similarity_search/include/hnswquery.h | 37 +++++++++++++++++++ + similarity_search/include/method/hnsw.h | 1 + + similarity_search/include/space.h | 4 ++ + similarity_search/src/hnswquery.cc | 32 ++++++++++++++++ + similarity_search/src/method/hnsw.cc | 28 +++++++++----- + .../src/method/hnsw_distfunc_opt.cc | 12 +++--- + 6 files changed, 100 insertions(+), 14 deletions(-) + create mode 100644 similarity_search/include/hnswquery.h + create mode 100644 similarity_search/src/hnswquery.cc + +diff --git a/similarity_search/include/hnswquery.h b/similarity_search/include/hnswquery.h +new file mode 100644 +index 0000000..a4f65ac +--- /dev/null ++++ b/similarity_search/include/hnswquery.h +@@ -0,0 +1,37 @@ ++/** ++ * Non-metric Space Library ++ * ++ * Main developers: Bilegsaikhan Naidan, Leonid Boytsov, Yury Malkov, Ben Frederickson, David Novak ++ * ++ * For the complete list of contributors and further details see: ++ * https://github.com/nmslib/nmslib ++ * ++ * Copyright (c) 2013-2018 ++ * ++ * This code is released under the ++ * Apache License Version 2.0 http://www.apache.org/licenses/. ++ * ++ */ ++ ++#ifndef HNSWQUERY_H ++#define HNSWQUERY_H ++#include "global.h" ++#include "knnquery.h" ++ ++namespace similarity { ++ ++template ++class HNSWQuery : public KNNQuery { ++public: ++ ~HNSWQuery(); ++ HNSWQuery(const Space& space, const Object *query_object, unsigned K, unsigned ef = 100, float eps = 0); ++ ++ unsigned getEf() { return ef_; } ++ ++protected: ++ unsigned ef_; ++}; ++ ++} ++ ++#endif //HNSWQUERY_H +diff --git a/similarity_search/include/method/hnsw.h b/similarity_search/include/method/hnsw.h +index 57d99d0..e6dcea7 100644 +--- a/similarity_search/include/method/hnsw.h ++++ b/similarity_search/include/method/hnsw.h +@@ -474,6 +474,7 @@ namespace similarity { + void baseSearchAlgorithmV1Merge(KNNQuery *query); + void SearchOld(KNNQuery *query, bool normalize); + void SearchV1Merge(KNNQuery *query, bool normalize); ++ size_t extractEf(KNNQuery *query, size_t defaultEf) const; + + int getRandomLevel(double revSize) + { +diff --git a/similarity_search/include/space.h b/similarity_search/include/space.h +index fedad46..a0e9ea9 100644 +--- a/similarity_search/include/space.h ++++ b/similarity_search/include/space.h +@@ -63,6 +63,9 @@ class Query; + template + class KNNQuery; + ++ template ++class HNSWQuery; ++ + template + class RangeQuery; + +@@ -263,6 +266,7 @@ class Space { + friend class Query; + friend class RangeQuery; + friend class KNNQuery; ++ friend class HNSWQuery; + friend class Experiments; + /* + * This function is private, but it will be accessible by the friend class Query +diff --git a/similarity_search/src/hnswquery.cc b/similarity_search/src/hnswquery.cc +new file mode 100644 +index 0000000..4ee7b38 +--- /dev/null ++++ b/similarity_search/src/hnswquery.cc +@@ -0,0 +1,32 @@ ++/** ++* Non-metric Space Library ++ * ++ * Main developers: Bilegsaikhan Naidan, Leonid Boytsov, Yury Malkov, Ben Frederickson, David Novak ++ * ++ * For the complete list of contributors and further details see: ++ * https://github.com/nmslib/nmslib ++ * ++ * Copyright (c) 2013-2018 ++ * ++ * This code is released under the ++ * Apache License Version 2.0 http://www.apache.org/licenses/. ++ * ++ */ ++ ++#include "hnswquery.h" ++ ++namespace similarity { ++ ++template ++HNSWQuery::HNSWQuery(const Space &space, const Object* query_object, const unsigned K, unsigned ef, float eps) ++ : KNNQuery(space, query_object, K, eps), ++ ef_(ef) { ++} ++ ++template ++HNSWQuery::~HNSWQuery() = default; ++ ++template class HNSWQuery; ++template class HNSWQuery; ++template class HNSWQuery; ++} +diff --git a/similarity_search/src/method/hnsw.cc b/similarity_search/src/method/hnsw.cc +index 35b372c..69ee9e4 100644 +--- a/similarity_search/src/method/hnsw.cc ++++ b/similarity_search/src/method/hnsw.cc +@@ -46,6 +46,7 @@ + #include + #include + ++#include "hnswquery.h" + #include "sort_arr_bi.h" + #define MERGE_BUFFER_ALGO_SWITCH_THRESHOLD 100 + +@@ -101,9 +102,16 @@ namespace similarity { + return nullptr; + } + ++ template ++ size_t Hnsw::extractEf(KNNQuery* searchQuery, size_t defaultEf) const { ++ auto* hnswQueryPtr = dynamic_cast*>(searchQuery); ++ if (hnswQueryPtr) { ++ return hnswQueryPtr->getEf(); ++ } ++ return defaultEf; ++ } + +- +-// This is the counter to keep the size of neighborhood information (for one node) ++ // This is the counter to keep the size of neighborhood information (for one node) + // TODO Can this one overflow? I really doubt + typedef uint32_t SIZEMASS_TYPE; + +@@ -718,10 +726,11 @@ namespace similarity { + void + Hnsw::Search(KNNQuery *query, IdType) const + { ++ size_t ef = this->extractEf(query, ef_); + if (this->data_.empty() && this->data_rearranged_.empty()) { + return; + } +- bool useOld = searchAlgoType_ == kOld || (searchAlgoType_ == kHybrid && ef_ >= 1000); ++ bool useOld = searchAlgoType_ == kOld || (searchAlgoType_ == kHybrid && ef >= 1000); + // cout << "Ef = " << ef_ << " use old = " << useOld << endl; + switch (searchMethod_) { + case 0: +@@ -1148,6 +1157,7 @@ namespace similarity { + PREFETCH((char *)(massVisited + (*iter)->getId()), _MM_HINT_T0); + } + // calculate distance to each neighbor ++ size_t ef = this->extractEf(query, ef_); + for (auto iter = neighbor.begin(); iter != neighbor.end(); ++iter) { + curId = (*iter)->getId(); + +@@ -1155,12 +1165,12 @@ namespace similarity { + massVisited[curId] = currentV; + currObj = (*iter)->getData(); + d = query->DistanceObjLeft(currObj); +- if (closestDistQueue1.top().getDistance() > d || closestDistQueue1.size() < ef_) { ++ if (closestDistQueue1.top().getDistance() > d || closestDistQueue1.size() < ef) { + { + query->CheckAndAddToResult(d, currObj); + candidateQueue.emplace(d, *iter); + closestDistQueue1.emplace(d, *iter); +- if (closestDistQueue1.size() > ef_) { ++ if (closestDistQueue1.size() > ef) { + closestDistQueue1.pop(); + } + } +@@ -1185,6 +1195,7 @@ namespace similarity { + + const Object *currObj = provider->getData(); + ++ size_t ef = this->extractEf(query, ef_); + dist_t d = query->DistanceObjLeft(currObj); + dist_t curdist = d; + HnswNode *curNode = provider; +@@ -1209,7 +1220,7 @@ namespace similarity { + } + } + +- SortArrBI sortedArr(max(ef_, query->GetK())); ++ SortArrBI sortedArr(max(ef, query->GetK())); + sortedArr.push_unsorted_grow(curdist, curNode); + + int_fast32_t currElem = 0; +@@ -1225,8 +1236,7 @@ namespace similarity { + // PHASE TWO OF THE SEARCH + // Extraction of the neighborhood to find k nearest neighbors. + //////////////////////////////////////////////////////////////////////////////// +- +- while (currElem < min(sortedArr.size(), ef_)) { ++ while (currElem < min(sortedArr.size(), ef)) { + auto &e = queueData[currElem]; + CHECK(!e.used); + e.used = true; +@@ -1255,7 +1265,7 @@ namespace similarity { + currObj = (*iter)->getData(); + d = query->DistanceObjLeft(currObj); + +- if (d < topKey || sortedArr.size() < ef_) { ++ if (d < topKey || sortedArr.size() < ef) { + CHECK_MSG(itemBuff.size() > itemQty, + "Perhaps a bug: buffer size is not enough " + + ConvertToString(itemQty) + " >= " + ConvertToString(itemBuff.size())); +diff --git a/similarity_search/src/method/hnsw_distfunc_opt.cc b/similarity_search/src/method/hnsw_distfunc_opt.cc +index 5c219cd..1913936 100644 +--- a/similarity_search/src/method/hnsw_distfunc_opt.cc ++++ b/similarity_search/src/method/hnsw_distfunc_opt.cc +@@ -120,6 +120,7 @@ namespace similarity { + PREFETCH(data_level0_memory_ + (*(data + 1)) * memoryPerObject_ + offsetData_, _MM_HINT_T0); + PREFETCH((char *)(data + 2), _MM_HINT_T0); + ++ size_t ef = this->extractEf(query, ef_); + for (int j = 1; j <= size; j++) { + int tnum = *(data + j); + PREFETCH((char *)(massVisited + *(data + j + 1)), _MM_HINT_T0); +@@ -131,7 +132,7 @@ namespace similarity { + massVisited[tnum] = currentV; + char *currObj1 = (data_level0_memory_ + tnum * memoryPerObject_ + offsetData_); + dist_t d = (fstdistfunc_(pVectq, (float *)(currObj1 + 16), qty, TmpRes)); +- if (closestDistQueuei.top().getDistance() > d || closestDistQueuei.size() < ef_) { ++ if (closestDistQueuei.top().getDistance() > d || closestDistQueuei.size() < ef) { + candidateQueuei.emplace(-d, tnum); + PREFETCH(data_level0_memory_ + candidateQueuei.top().element * memoryPerObject_ + offsetLevel0_, + _MM_HINT_T0); +@@ -139,7 +140,7 @@ namespace similarity { + query->CheckAndAddToResult(d, data_rearranged_[tnum]); + closestDistQueuei.emplace(d, tnum); + +- if (closestDistQueuei.size() > ef_) { ++ if (closestDistQueuei.size() > ef) { + closestDistQueuei.pop(); + } + } +@@ -153,6 +154,7 @@ namespace similarity { + void + Hnsw::SearchV1Merge(KNNQuery *query, bool normalize) + { ++ size_t ef = this->extractEf(query, ef_); + float *pVectq = (float *)((char *)query->QueryObject()->data()); + TMP_RES_ARRAY(TmpRes); + size_t qty = query->QueryObject()->datalength() >> 2; +@@ -197,7 +199,7 @@ namespace similarity { + } + } + +- SortArrBI sortedArr(max(ef_, query->GetK())); ++ SortArrBI sortedArr(max(ef, query->GetK())); + sortedArr.push_unsorted_grow(curdist, curNodeNum); + + int_fast32_t currElem = 0; +@@ -208,7 +210,7 @@ namespace similarity { + + massVisited[curNodeNum] = currentV; + +- while (currElem < min(sortedArr.size(), ef_)) { ++ while (currElem < min(sortedArr.size(), ef)) { + auto &e = queueData[currElem]; + CHECK(!e.used); + e.used = true; +@@ -237,7 +239,7 @@ namespace similarity { + char *currObj1 = (data_level0_memory_ + tnum * memoryPerObject_ + offsetData_); + dist_t d = (fstdistfunc_(pVectq, (float *)(currObj1 + 16), qty, TmpRes)); + +- if (d < topKey || sortedArr.size() < ef_) { ++ if (d < topKey || sortedArr.size() < ef) { + CHECK_MSG(itemBuff.size() > itemQty, + "Perhaps a bug: buffer size is not enough " + + ConvertToString(itemQty) + " >= " + ConvertToString(itemBuff.size())); +-- +2.44.0 + diff --git a/jni/src/commons.cpp b/jni/src/commons.cpp index 3c03ac49d..c2b2354cc 100644 --- a/jni/src/commons.cpp +++ b/jni/src/commons.cpp @@ -38,4 +38,16 @@ void knn_jni::commons::freeVectorData(jlong memoryAddressJ) { delete vect; } } -#endif //OPENSEARCH_KNN_COMMONS_H \ No newline at end of file + +int knn_jni::commons::getIntegerMethodParameter(JNIEnv * env, knn_jni::JNIUtilInterface * jniUtil, std::unordered_map methodParams, std::string methodParam, int defaultValue) { + if (methodParams.empty()) { + return defaultValue; + } + auto efSearchIt = methodParams.find(methodParam); + if (efSearchIt != methodParams.end()) { + return jniUtil->ConvertJavaObjectToCppInteger(env, methodParams[methodParam]); + } + + return defaultValue; +} +#endif //OPENSEARCH_KNN_COMMONS_H diff --git a/jni/src/faiss_wrapper.cpp b/jni/src/faiss_wrapper.cpp index 5a0910d9a..198f733be 100644 --- a/jni/src/faiss_wrapper.cpp +++ b/jni/src/faiss_wrapper.cpp @@ -22,6 +22,7 @@ #include "faiss/Index.h" #include "faiss/impl/IDSelector.h" #include "faiss/IndexIVFPQ.h" +#include "commons.h" #include #include @@ -296,12 +297,12 @@ void knn_jni::faiss_wrapper::SetSharedIndexState(jlong indexPointerJ, jlong shar } jobjectArray knn_jni::faiss_wrapper::QueryIndex(knn_jni::JNIUtilInterface * jniUtil, JNIEnv * env, jlong indexPointerJ, - jfloatArray queryVectorJ, jint kJ, jintArray parentIdsJ) { - return knn_jni::faiss_wrapper::QueryIndex_WithFilter(jniUtil, env, indexPointerJ, queryVectorJ, kJ, nullptr, 0, parentIdsJ); + jfloatArray queryVectorJ, jint kJ, jobject methodParamsJ, jintArray parentIdsJ) { + return knn_jni::faiss_wrapper::QueryIndex_WithFilter(jniUtil, env, indexPointerJ, queryVectorJ, kJ, methodParamsJ, nullptr, 0, parentIdsJ); } jobjectArray knn_jni::faiss_wrapper::QueryIndex_WithFilter(knn_jni::JNIUtilInterface * jniUtil, JNIEnv * env, jlong indexPointerJ, - jfloatArray queryVectorJ, jint kJ, jlongArray filterIdsJ, jint filterIdsTypeJ, jintArray parentIdsJ) { + jfloatArray queryVectorJ, jint kJ, jobject methodParamsJ, jlongArray filterIdsJ, jint filterIdsTypeJ, jintArray parentIdsJ) { if (queryVectorJ == nullptr) { throw std::runtime_error("Query Vector cannot be null"); @@ -313,6 +314,11 @@ jobjectArray knn_jni::faiss_wrapper::QueryIndex_WithFilter(knn_jni::JNIUtilInter throw std::runtime_error("Invalid pointer to index"); } + std::unordered_map methodParams; + if (methodParamsJ != nullptr) { + methodParams = jniUtil->ConvertJavaMapToCppMap(env, methodParamsJ); + } + // The ids vector will hold the top k ids from the search and the dis vector will hold the top k distances from // the query point std::vector dis(kJ); @@ -340,9 +346,8 @@ jobjectArray knn_jni::faiss_wrapper::QueryIndex_WithFilter(knn_jni::JNIUtilInter std::vector idGrouperBitmap; auto hnswReader = dynamic_cast(indexReader->index); if(hnswReader) { - // Setting the ef_search value equal to what was provided during index creation. SearchParametersHNSW has a default - // value of ef_search = 16 which will then be used. - hnswParams.efSearch = hnswReader->hnsw.efSearch; + // Query param efsearch supersedes ef_search provided during index setting. + hnswParams.efSearch = knn_jni::commons::getIntegerMethodParameter(env, jniUtil, methodParams, EF_SEARCH, hnswReader->hnsw.efSearch); hnswParams.sel = idSelector.get(); if (parentIdsJ != nullptr) { idGrouper = buildIDGrouperBitmap(jniUtil, env, parentIdsJ, &idGrouperBitmap); @@ -371,12 +376,13 @@ jobjectArray knn_jni::faiss_wrapper::QueryIndex_WithFilter(knn_jni::JNIUtilInter std::unique_ptr idGrouper; std::vector idGrouperBitmap; auto hnswReader = dynamic_cast(indexReader->index); - if(hnswReader!= nullptr && parentIdsJ != nullptr) { - // Setting the ef_search value equal to what was provided during index creation. SearchParametersHNSW has a default - // value of ef_search = 16 which will then be used. - hnswParams.efSearch = hnswReader->hnsw.efSearch; - idGrouper = buildIDGrouperBitmap(jniUtil, env, parentIdsJ, &idGrouperBitmap); - hnswParams.grp = idGrouper.get(); + if(hnswReader!= nullptr) { + // Query param efsearch supersedes ef_search provided during index setting. + hnswParams.efSearch = knn_jni::commons::getIntegerMethodParameter(env, jniUtil, methodParams, EF_SEARCH, hnswReader->hnsw.efSearch); + if (parentIdsJ != nullptr) { + idGrouper = buildIDGrouperBitmap(jniUtil, env, parentIdsJ, &idGrouperBitmap); + hnswParams.grp = idGrouper.get(); + } searchParameters = &hnswParams; } try { diff --git a/jni/src/nmslib_wrapper.cpp b/jni/src/nmslib_wrapper.cpp index 6ea80d727..21b34eb83 100644 --- a/jni/src/nmslib_wrapper.cpp +++ b/jni/src/nmslib_wrapper.cpp @@ -12,6 +12,8 @@ #include "jni_util.h" #include "nmslib_wrapper.h" +#include "commons.h" + #include "init.h" #include "index.h" #include "params.h" @@ -24,6 +26,8 @@ #include #include +#include "hnswquery.h" + std::string TranslateSpaceType(const std::string& spaceType); @@ -220,7 +224,7 @@ jlong knn_jni::nmslib_wrapper::LoadIndex(knn_jni::JNIUtilInterface * jniUtil, JN } jobjectArray knn_jni::nmslib_wrapper::QueryIndex(knn_jni::JNIUtilInterface * jniUtil, JNIEnv * env, jlong indexPointerJ, - jfloatArray queryVectorJ, jint kJ) { + jfloatArray queryVectorJ, jint kJ, jobject methodParamsJ) { if (queryVectorJ == nullptr) { throw std::runtime_error("Query Vector cannot be null"); @@ -243,14 +247,34 @@ jobjectArray knn_jni::nmslib_wrapper::QueryIndex(knn_jni::JNIUtilInterface * jni jniUtil->ReleaseFloatArrayElements(env, queryVectorJ, rawQueryvector, JNI_ABORT); throw; } + jniUtil->ReleaseFloatArrayElements(env, queryVectorJ, rawQueryvector, JNI_ABORT); + std::unordered_map methodParams; + if (methodParamsJ != nullptr) { + methodParams = jniUtil->ConvertJavaMapToCppMap(env, methodParamsJ); + } - similarity::KNNQuery knnQuery(*(indexWrapper->space), queryObject.get(), kJ); - indexWrapper->index->Search(&knnQuery); + int queryEfSearch = knn_jni::commons::getIntegerMethodParameter(env, jniUtil, methodParams, EF_SEARCH, -1); + similarity::KNNQuery* query; // TODO: Replace with smart pointers https://github.com/opensearch-project/k-NN/issues/1785 + std::unique_ptr> neighbors; + try { + if (queryEfSearch == -1) { + query = new similarity::KNNQuery(*(indexWrapper->space), queryObject.get(), kJ); + } else { + query = new similarity::HNSWQuery(*(indexWrapper->space), queryObject.get(), kJ, queryEfSearch); + } - std::unique_ptr> neighbors(knnQuery.Result()->Clone()); + indexWrapper->index->Search(query); + neighbors.reset(query->Result()->Clone()); + } catch (...) { + if (query != nullptr) { + delete query; + } + throw; + } + delete query; + int resultSize = neighbors->Size(); - jclass resultClass = jniUtil->FindClass(env,"org/opensearch/knn/index/query/KNNQueryResult"); jmethodID allArgs = jniUtil->FindMethod(env, "org/opensearch/knn/index/query/KNNQueryResult", ""); @@ -265,6 +289,7 @@ jobjectArray knn_jni::nmslib_wrapper::QueryIndex(knn_jni::JNIUtilInterface * jni result = jniUtil->NewObject(env, resultClass, allArgs, id, distance); jniUtil->SetObjectArrayElement(env, results, i, result); } + return results; } diff --git a/jni/src/org_opensearch_knn_jni_FaissService.cpp b/jni/src/org_opensearch_knn_jni_FaissService.cpp index 0aa51987d..57353f9e1 100644 --- a/jni/src/org_opensearch_knn_jni_FaissService.cpp +++ b/jni/src/org_opensearch_knn_jni_FaissService.cpp @@ -109,10 +109,10 @@ JNIEXPORT void JNICALL Java_org_opensearch_knn_jni_FaissService_setSharedIndexSt JNIEXPORT jobjectArray JNICALL Java_org_opensearch_knn_jni_FaissService_queryIndex(JNIEnv * env, jclass cls, jlong indexPointerJ, - jfloatArray queryVectorJ, jint kJ, jintArray parentIdsJ) + jfloatArray queryVectorJ, jint kJ, jobject methodParamsJ, jintArray parentIdsJ) { try { - return knn_jni::faiss_wrapper::QueryIndex(&jniUtil, env, indexPointerJ, queryVectorJ, kJ, parentIdsJ); + return knn_jni::faiss_wrapper::QueryIndex(&jniUtil, env, indexPointerJ, queryVectorJ, kJ, methodParamsJ, parentIdsJ); } catch (...) { jniUtil.CatchCppExceptionAndThrowJava(env); @@ -121,10 +121,10 @@ JNIEXPORT jobjectArray JNICALL Java_org_opensearch_knn_jni_FaissService_queryInd } JNIEXPORT jobjectArray JNICALL Java_org_opensearch_knn_jni_FaissService_queryIndexWithFilter - (JNIEnv * env, jclass cls, jlong indexPointerJ, jfloatArray queryVectorJ, jint kJ, jlongArray filteredIdsJ, jint filterIdsTypeJ, jintArray parentIdsJ) { + (JNIEnv * env, jclass cls, jlong indexPointerJ, jfloatArray queryVectorJ, jint kJ, jobject methodParamsJ, jlongArray filteredIdsJ, jint filterIdsTypeJ, jintArray parentIdsJ) { try { - return knn_jni::faiss_wrapper::QueryIndex_WithFilter(&jniUtil, env, indexPointerJ, queryVectorJ, kJ, filteredIdsJ, filterIdsTypeJ, parentIdsJ); + return knn_jni::faiss_wrapper::QueryIndex_WithFilter(&jniUtil, env, indexPointerJ, queryVectorJ, kJ, methodParamsJ, filteredIdsJ, filterIdsTypeJ, parentIdsJ); } catch (...) { jniUtil.CatchCppExceptionAndThrowJava(env); } diff --git a/jni/src/org_opensearch_knn_jni_NmslibService.cpp b/jni/src/org_opensearch_knn_jni_NmslibService.cpp index d037d3337..e265827cd 100644 --- a/jni/src/org_opensearch_knn_jni_NmslibService.cpp +++ b/jni/src/org_opensearch_knn_jni_NmslibService.cpp @@ -61,10 +61,10 @@ JNIEXPORT jlong JNICALL Java_org_opensearch_knn_jni_NmslibService_loadIndex(JNIE JNIEXPORT jobjectArray JNICALL Java_org_opensearch_knn_jni_NmslibService_queryIndex(JNIEnv * env, jclass cls, jlong indexPointerJ, - jfloatArray queryVectorJ, jint kJ) + jfloatArray queryVectorJ, jint kJ, jobject methodParamsJ) { try { - return knn_jni::nmslib_wrapper::QueryIndex(&jniUtil, env, indexPointerJ, queryVectorJ, kJ); + return knn_jni::nmslib_wrapper::QueryIndex(&jniUtil, env, indexPointerJ, queryVectorJ, kJ, methodParamsJ); } catch (...) { jniUtil.CatchCppExceptionAndThrowJava(env); } diff --git a/jni/tests/commons_test.cpp b/jni/tests/commons_test.cpp index 09323f0fb..630358919 100644 --- a/jni/tests/commons_test.cpp +++ b/jni/tests/commons_test.cpp @@ -71,3 +71,22 @@ TEST(CommonsTests, BasicAssertions) { } } } + +TEST(CommonTests, GetIntegerMethodParam) { + JNIEnv *jniEnv = nullptr; + testing::NiceMock mockJNIUtil; + + std::unordered_map methodParams1; + int efSearch = 10; + methodParams1[knn_jni::EF_SEARCH] = reinterpret_cast(&efSearch); + + int actualValue1 = knn_jni::commons::getIntegerMethodParameter(jniEnv, &mockJNIUtil, methodParams1, knn_jni::EF_SEARCH, 1); + EXPECT_EQ(efSearch, actualValue1); + + int actualValue2 = knn_jni::commons::getIntegerMethodParameter(jniEnv, &mockJNIUtil, methodParams1, "param", 1); + EXPECT_EQ(1, actualValue2); + + std::unordered_map methodParams2; + int actualValue3 = knn_jni::commons::getIntegerMethodParameter(jniEnv, &mockJNIUtil, methodParams2, knn_jni::EF_SEARCH, 1); + EXPECT_EQ(1, actualValue3); +} diff --git a/jni/tests/faiss_wrapper_test.cpp b/jni/tests/faiss_wrapper_test.cpp index 4cd3b319e..1db3df42c 100644 --- a/jni/tests/faiss_wrapper_test.cpp +++ b/jni/tests/faiss_wrapper_test.cpp @@ -245,6 +245,10 @@ TEST(FaissQueryIndexTest, BasicAssertions) { // Define query data int k = 10; + int efSearch = 20; + std::unordered_map methodParams; + methodParams[knn_jni::EF_SEARCH] = reinterpret_cast(&efSearch); + int numQueries = 100; std::vector> queries; @@ -266,6 +270,7 @@ TEST(FaissQueryIndexTest, BasicAssertions) { // Setup jni JNIEnv *jniEnv = nullptr; NiceMock mockJNIUtil; + auto methodParamsJ = reinterpret_cast(&methodParams); for (auto query : queries) { std::unique_ptr *>> results( @@ -273,7 +278,7 @@ TEST(FaissQueryIndexTest, BasicAssertions) { knn_jni::faiss_wrapper::QueryIndex( &mockJNIUtil, jniEnv, reinterpret_cast(&createdIndexWithData), - reinterpret_cast(&query), k, nullptr))); + reinterpret_cast(&query), k, methodParamsJ, nullptr))); ASSERT_EQ(k, results->size()); @@ -339,7 +344,7 @@ TEST(FaissQueryIndexWithFilterTest1435, BasicAssertions) { knn_jni::faiss_wrapper::QueryIndex_WithFilter( &mockJNIUtil, jniEnv, reinterpret_cast(&createdIndexWithData), - reinterpret_cast(&query), k, + reinterpret_cast(&query), k, nullptr, reinterpret_cast(&bitmap), 0, nullptr))); ASSERT_TRUE(results->size() <= filterIds.size()); @@ -397,20 +402,20 @@ TEST(FaissQueryIndexWithParentFilterTest, BasicAssertions) { auto createdIndexWithData = test_util::FaissAddData(createdIndex.get(), ids, vectors); + int efSearch = 100; + std::unordered_map methodParams; + methodParams[knn_jni::EF_SEARCH] = reinterpret_cast(&efSearch); + // Setup jni JNIEnv *jniEnv = nullptr; NiceMock mockJNIUtil; - EXPECT_CALL(mockJNIUtil, - GetJavaIntArrayLength( - jniEnv, reinterpret_cast(&parentIds))) - .WillRepeatedly(Return(parentIds.size())); for (auto query : queries) { std::unique_ptr *>> results( reinterpret_cast *> *>( knn_jni::faiss_wrapper::QueryIndex( &mockJNIUtil, jniEnv, reinterpret_cast(&createdIndexWithData), - reinterpret_cast(&query), k, + reinterpret_cast(&query), k, reinterpret_cast(&methodParams), reinterpret_cast(&parentIds)))); // Even with k 20, result should have only 10 which is total number of groups diff --git a/jni/tests/faiss_wrapper_unit_test.cpp b/jni/tests/faiss_wrapper_unit_test.cpp new file mode 100644 index 000000000..ea9131dd7 --- /dev/null +++ b/jni/tests/faiss_wrapper_unit_test.cpp @@ -0,0 +1,251 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +#include "faiss_wrapper.h" + +#include + +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "jni_util.h" +#include "jni.h" +#include "test_util.h" +#include "faiss/IndexHNSW.h" +#include "faiss/IndexIDMap.h" + +using ::testing::NiceMock; + +using idx_t = faiss::idx_t; + +struct MockIndex : faiss::IndexHNSW { + explicit MockIndex(idx_t d) : faiss::IndexHNSW(d, 32) { + } +}; + + +struct MockIdMap : faiss::IndexIDMap { + mutable idx_t nCalled; + mutable const float *xCalled; + mutable idx_t kCalled; + mutable float *distancesCalled; + mutable idx_t *labelsCalled; + mutable const faiss::SearchParametersHNSW *paramsCalled; + + explicit MockIdMap(MockIndex *index) : faiss::IndexIDMapTemplate(index) { + } + + void search( + idx_t n, + const float *x, + idx_t k, + float *distances, + idx_t *labels, + const faiss::SearchParameters *params) const override { + nCalled = n; + xCalled = x; + kCalled = k; + distancesCalled = distances; + labelsCalled = labels; + paramsCalled = dynamic_cast(params); + } + + void resetMock() const { + nCalled = 0; + xCalled = nullptr; + kCalled = 0; + distancesCalled = nullptr; + labelsCalled = nullptr; + paramsCalled = nullptr; + } +}; + +struct QueryIndexHNSWTestInput { + string description; + int k; + int efSearch; + int filterIdType; + bool filterIdsPresent; + bool parentIdsPresent; +}; + + + +class FaissWrappeterParametrizedTestFixture : public testing::TestWithParam { +public: + FaissWrappeterParametrizedTestFixture() : index_(3), id_map_(&index_) { + index_.hnsw.efSearch = 100; // assigning 100 to make sure default of 16 is not used anywhere + }; + +protected: + MockIndex index_; + MockIdMap id_map_; +}; + +namespace query_index_test { + + std::unordered_map methodParams; + + + TEST_P(FaissWrappeterParametrizedTestFixture, QueryIndexHNSWTests) { + //Given + JNIEnv *jniEnv = nullptr; + NiceMock mockJNIUtil; + + + QueryIndexHNSWTestInput const &input = GetParam(); + float query[] = {1.2, 2.3, 3.4}; + + int efSearch = input.efSearch; + int expectedEfSearch = 100; //default set in mock + std::unordered_map methodParams; + if (efSearch != -1) { + expectedEfSearch = input.efSearch; + methodParams[knn_jni::EF_SEARCH] = reinterpret_cast(&efSearch); + } + + std::vector *parentIdPtr = nullptr; + if (input.parentIdsPresent) { + std::vector parentId; + parentId.reserve(2); + parentId.push_back(1); + parentId.push_back(2); + parentIdPtr = &parentId; + + EXPECT_CALL(mockJNIUtil, + GetJavaIntArrayLength( + jniEnv, reinterpret_cast(parentIdPtr))) + .WillOnce(testing::Return(parentId.size())); + + EXPECT_CALL(mockJNIUtil, + GetIntArrayElements( + jniEnv, reinterpret_cast(parentIdPtr), nullptr)) + .WillOnce(testing::Return(new int[2]{1, 2})); + } + + // When + knn_jni::faiss_wrapper::QueryIndex( + &mockJNIUtil, jniEnv, + reinterpret_cast(&id_map_), + reinterpret_cast(&query), input.k, reinterpret_cast(&methodParams), + reinterpret_cast(parentIdPtr)); + + //Then + int actualEfSearch = id_map_.paramsCalled->efSearch; + // Asserting the captured argument + EXPECT_EQ(input.k, id_map_.kCalled); + EXPECT_EQ(expectedEfSearch, actualEfSearch); + if (input.parentIdsPresent) { + faiss::IDGrouper *grouper = id_map_.paramsCalled->grp; + EXPECT_TRUE(grouper != nullptr); + } + + id_map_.resetMock(); + } + + INSTANTIATE_TEST_CASE_P( + QueryIndexHNSWTests, + FaissWrappeterParametrizedTestFixture, + ::testing::Values( + QueryIndexHNSWTestInput{"algoParams present, parent absent", 10, 200, 0, false, false}, + QueryIndexHNSWTestInput{"algoParams absent, parent absent", 10, -1, 0, false, false}, + QueryIndexHNSWTestInput{"algoParams present, parent present", 10, 200, 0, false, true}, + QueryIndexHNSWTestInput{"algoParams absent, parent present", 10, -1, 0, false, true} + ) + ); +} + +namespace query_index_with_filter_test { + + TEST_P(FaissWrappeterParametrizedTestFixture, QueryIndexWithFilterHNSWTests) { + //Given + JNIEnv *jniEnv = nullptr; + NiceMock mockJNIUtil; + + + QueryIndexHNSWTestInput const &input = GetParam(); + float query[] = {1.2, 2.3, 3.4}; + + std::vector *parentIdPtr = nullptr; + if (input.parentIdsPresent) { + std::vector parentId; + parentId.reserve(2); + parentId.push_back(1); + parentId.push_back(2); + parentIdPtr = &parentId; + + EXPECT_CALL(mockJNIUtil, + GetJavaIntArrayLength( + jniEnv, reinterpret_cast(parentIdPtr))) + .WillOnce(testing::Return(parentId.size())); + + EXPECT_CALL(mockJNIUtil, + GetIntArrayElements( + jniEnv, reinterpret_cast(parentIdPtr), nullptr)) + .WillOnce(testing::Return(new int[2]{1, 2})); + } + + std::vector *filterptr = nullptr; + if (input.filterIdsPresent) { + std::vector filter; + filter.reserve(2); + filter.push_back(1); + filter.push_back(2); + filterptr = &filter; + } + + int efSearch = input.efSearch; + int expectedEfSearch = 100; //default set in mock + std::unordered_map methodParams; + if (efSearch != -1) { + expectedEfSearch = input.efSearch; + methodParams[knn_jni::EF_SEARCH] = reinterpret_cast(&efSearch); + } + + // When + knn_jni::faiss_wrapper::QueryIndex_WithFilter( + &mockJNIUtil, jniEnv, + reinterpret_cast(&id_map_), + reinterpret_cast(&query), input.k, reinterpret_cast(&methodParams), + reinterpret_cast(filterptr), + input.filterIdType, + reinterpret_cast(parentIdPtr)); + + //Then + int actualEfSearch = id_map_.paramsCalled->efSearch; + // Asserting the captured argument + EXPECT_EQ(input.k, id_map_.kCalled); + EXPECT_EQ(expectedEfSearch, actualEfSearch); + if (input.parentIdsPresent) { + faiss::IDGrouper *grouper = id_map_.paramsCalled->grp; + EXPECT_TRUE(grouper != nullptr); + } + if (input.filterIdsPresent) { + faiss::IDSelector *sel = id_map_.paramsCalled->sel; + EXPECT_TRUE(sel != nullptr); + } + id_map_.resetMock(); + } + + INSTANTIATE_TEST_CASE_P( + QueryIndexWithFilterHNSWTests, + FaissWrappeterParametrizedTestFixture, + ::testing::Values( + QueryIndexHNSWTestInput{"algoParams present, parent absent, filter absent", 10, 200, 0, false, false}, + QueryIndexHNSWTestInput{"algoParams present, parent absent, filter absent, filter type 1", 10, 200, 1, false, false}, + QueryIndexHNSWTestInput{"algoParams absent, parent absent, filter present", 10, -1, 0, true, false}, + QueryIndexHNSWTestInput{"algoParams absent, parent absent, filter present, filter type 1", 10, -1, 1, true, false}, + QueryIndexHNSWTestInput{"algoParams present, parent present, filter absent", 10, 200, 0, false, true}, + QueryIndexHNSWTestInput{"algoParams present, parent present, filter absent, filter type 1", 10, 150, 1, false, true}, + QueryIndexHNSWTestInput{"algoParams absent, parent present, filter present", 10, -1, 0, true, true}, + QueryIndexHNSWTestInput{"algoParams absent, parent present, filter present, filter type 1",10, -1, 1, true, true} + ) + ); +} diff --git a/jni/tests/nmslib_wrapper_test.cpp b/jni/tests/nmslib_wrapper_test.cpp index 1fd9471b0..4e0c57044 100644 --- a/jni/tests/nmslib_wrapper_test.cpp +++ b/jni/tests/nmslib_wrapper_test.cpp @@ -182,8 +182,11 @@ TEST(NmslibQueryIndexTest, BasicAssertions) { // Define query data int k = 10; + int efSearch = 20; int numQueries = 100; std::vector> queries; + std::unordered_map methodParams; + methodParams[knn_jni::EF_SEARCH] = reinterpret_cast(&efSearch); for (int i = 0; i < numQueries; i++) { std::vector query; @@ -205,7 +208,7 @@ TEST(NmslibQueryIndexTest, BasicAssertions) { knn_jni::nmslib_wrapper::QueryIndex( &mockJNIUtil, jniEnv, reinterpret_cast(indexWrapper.get()), - reinterpret_cast(&query), k))); + reinterpret_cast(&query), k, nullptr))); ASSERT_EQ(k, results->size()); diff --git a/jni/tests/nmslib_wrapper_unit_test.cpp b/jni/tests/nmslib_wrapper_unit_test.cpp new file mode 100644 index 000000000..ff6b94fc7 --- /dev/null +++ b/jni/tests/nmslib_wrapper_unit_test.cpp @@ -0,0 +1,135 @@ +/* +* SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ +#include "hnswquery.h" +#include "knnquery.h" +#include "nmslib_wrapper.h" + +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "jni_util.h" +#include "jni.h" +#include "test_util.h" +#include "method/hnsw.h" +#include "space/space_dummy.h" + +namespace nmslib_query_index_test { + + using ::testing::NiceMock; + + struct QueryIndexHNSWTestInput { + string description; + int k; + int efSearch; + bool expectedHNSWQuery; + }; + + struct MockNMSIndex : similarity::Hnsw { + mutable int kCalled; + mutable int efCalled = -1; + + explicit MockNMSIndex(const similarity::Space &space, const similarity::ObjectVector &data): Hnsw(false, + space, data) { + std::vector input; + input.emplace_back("ef=10"); + similarity::AnyParams ef(input); + this->Hnsw::SetQueryTimeParams(ef); + } + + void Search(similarity::KNNQuery *query, similarity::IdType id) const override { + auto hnsw = dynamic_cast *>(query); + if (hnsw != nullptr) { + kCalled = hnsw->GetK(); + efCalled = hnsw->getEf(); + } else { + kCalled = query->GetK(); + } + similarity::Object object(5, 0, 3*sizeof(float), new float[] { 2.2f, 2.5f, 2.6f }); + similarity::Object* objectPtr = &object; + bool added = query->CheckAndAddToResult(0.0f, objectPtr); + }; + + void resetMocks() const { + kCalled = -1; + efCalled = -1; + } + }; + + class NmslibWrapperParametrizedTestFixture : public testing::TestWithParam { + public: + NmslibWrapperParametrizedTestFixture() : space_(nullptr), index_(nullptr) { + similarity::initLibrary(); + std::string spaceType = knn_jni::L2; + space_ = similarity::SpaceFactoryRegistry::Instance().CreateSpace( + spaceType, similarity::AnyParams()); + index_ = new MockNMSIndex(*space_, similarity::ObjectVector()); + }; + + protected: + MockNMSIndex* index_; + similarity::Space* space_; // Moved from local to member variable + }; + + + TEST_P(NmslibWrapperParametrizedTestFixture, QueryIndexHNSWTests) { + //Given + JNIEnv *jniEnv = nullptr; + NiceMock mockJNIUtil; + + + QueryIndexHNSWTestInput const &input = GetParam(); + float query[] = { 1.2f, 2.3f, 3.4f }; + + std::string spaceType = knn_jni::L2; + std::unique_ptr indexWrapper( + new knn_jni::nmslib_wrapper::IndexWrapper(spaceType)); + indexWrapper->index.reset(index_); + + int efSearch = input.efSearch; + std::unordered_map methodParams; + if (efSearch != -1) { + methodParams[knn_jni::EF_SEARCH] = reinterpret_cast(&efSearch); + } + EXPECT_CALL(mockJNIUtil, + GetJavaFloatArrayLength( + jniEnv, reinterpret_cast(query))) + .WillOnce(testing::Return(3)); + + EXPECT_CALL(mockJNIUtil, + ReleaseFloatArrayElements( + jniEnv, reinterpret_cast(query), query, JNI_ABORT)); + EXPECT_CALL(mockJNIUtil, + GetFloatArrayElements( + jniEnv, reinterpret_cast(query), nullptr)) + .WillOnce(testing::Return(query)); + + knn_jni::nmslib_wrapper::QueryIndex( + &mockJNIUtil, jniEnv, + reinterpret_cast(indexWrapper.get()), + reinterpret_cast(&query), input.k, reinterpret_cast(&methodParams)); + + if (input.expectedHNSWQuery) { + EXPECT_EQ(input.efSearch, index_->efCalled); + EXPECT_EQ(input.k, index_->kCalled); + } else { + EXPECT_EQ(input.k, index_->kCalled); + } + index_->resetMocks(); + } + + INSTANTIATE_TEST_CASE_P( + QueryIndexHNSWTests, + NmslibWrapperParametrizedTestFixture, + ::testing::Values( + QueryIndexHNSWTestInput{"methodParams present", 10, 200, true}, + QueryIndexHNSWTestInput{"methodParams absent", 5, -1, false } + ) + ); +} diff --git a/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/ModelIT.java b/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/ModelIT.java index fed2d39da..7df651a3c 100644 --- a/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/ModelIT.java +++ b/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/ModelIT.java @@ -177,7 +177,7 @@ public void searchKNNModel(String testModelID) throws IOException { } // Confirm that the model gets created using Get Model API - public void validateModelCreated(String modelId) throws IOException, InterruptedException { + public void validateModelCreated(String modelId) throws Exception { Response getResponse = getModel(modelId, null); String responseBody = EntityUtils.toString(getResponse.getEntity()); assertNotNull(responseBody); diff --git a/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/QueryANNIT.java b/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/QueryANNIT.java new file mode 100644 index 000000000..566f40383 --- /dev/null +++ b/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/QueryANNIT.java @@ -0,0 +1,37 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.knn.bwc; + +import java.util.Map; + +import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_EF_SEARCH; + +public class QueryANNIT extends AbstractRestartUpgradeTestCase { + + private static final String TEST_FIELD = "test-field"; + private static final int DIMENSIONS = 5; + private static final int K = 5; + private static final Integer EF_SEARCH = 10; + private static final int NUM_DOCS = 10; + + public void testQueryANN() throws Exception { + if (isRunningAgainstOldCluster()) { + createKnnIndex(testIndex, getKNNDefaultIndexSettings(), createKnnIndexMapping(TEST_FIELD, DIMENSIONS)); + addKNNDocs(testIndex, TEST_FIELD, DIMENSIONS, 0, NUM_DOCS); + validateKNNSearch(testIndex, TEST_FIELD, DIMENSIONS, NUM_DOCS, K); + } else { + validateKNNSearch(testIndex, TEST_FIELD, DIMENSIONS, NUM_DOCS, K); + validateKNNSearch(testIndex, TEST_FIELD, DIMENSIONS, NUM_DOCS, K, Map.of(METHOD_PARAMETER_EF_SEARCH, EF_SEARCH)); + deleteKNNIndex(testIndex); + } + } +} diff --git a/qa/rolling-upgrade/src/test/java/org/opensearch/knn/bwc/IndexingIT.java b/qa/rolling-upgrade/src/test/java/org/opensearch/knn/bwc/IndexingIT.java index 73adb6db8..10df1a79b 100644 --- a/qa/rolling-upgrade/src/test/java/org/opensearch/knn/bwc/IndexingIT.java +++ b/qa/rolling-upgrade/src/test/java/org/opensearch/knn/bwc/IndexingIT.java @@ -5,8 +5,6 @@ package org.opensearch.knn.bwc; -import java.io.IOException; - import static org.opensearch.knn.TestUtils.NODES_BWC_CLUSTER; public class IndexingIT extends AbstractRollingUpgradeTestCase { @@ -129,7 +127,7 @@ public void testKNNIndexCreation_withMethodMapper() throws Exception { } // validation steps for indexing after upgrading each node from old version to new version - public void validateKNNIndexingOnUpgrade(int totalDocsCount, int docId) throws IOException { + public void validateKNNIndexingOnUpgrade(int totalDocsCount, int docId) throws Exception { validateKNNSearch(testIndex, TEST_FIELD, DIMENSIONS, totalDocsCount, K); addKNNDocs(testIndex, TEST_FIELD, DIMENSIONS, docId, NUM_DOCS); diff --git a/qa/rolling-upgrade/src/test/java/org/opensearch/knn/bwc/QueryANNIT.java b/qa/rolling-upgrade/src/test/java/org/opensearch/knn/bwc/QueryANNIT.java new file mode 100644 index 000000000..080e63241 --- /dev/null +++ b/qa/rolling-upgrade/src/test/java/org/opensearch/knn/bwc/QueryANNIT.java @@ -0,0 +1,49 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.knn.bwc; + +import java.util.Map; + +import static org.opensearch.knn.TestUtils.NODES_BWC_CLUSTER; +import static org.opensearch.knn.common.KNNConstants.FAISS_NAME; +import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_EF_SEARCH; + +public class QueryANNIT extends AbstractRollingUpgradeTestCase { + private static final String TEST_FIELD = "test-field"; + private static final int DIMENSIONS = 5; + private static final int K = 5; + private static final Integer EF_SEARCH = 10; + private static final int NUM_DOCS = 10; + private static final String ALGORITHM = "hnsw"; + + public void testQueryANNIT() throws Exception { + waitForClusterHealthGreen(NODES_BWC_CLUSTER); + switch (getClusterType()) { + case OLD: + createKnnIndex( + testIndex, + getKNNDefaultIndexSettings(), + createKnnIndexMapping(TEST_FIELD, DIMENSIONS, ALGORITHM, FAISS_NAME) + ); + addKNNDocs(testIndex, TEST_FIELD, DIMENSIONS, 0, NUM_DOCS); + validateKNNSearch(testIndex, TEST_FIELD, DIMENSIONS, NUM_DOCS, K); + break; + case MIXED: + validateKNNSearch(testIndex, TEST_FIELD, DIMENSIONS, NUM_DOCS, K); + break; + case UPGRADED: + validateKNNSearch(testIndex, TEST_FIELD, DIMENSIONS, NUM_DOCS, K); + validateKNNSearch(testIndex, TEST_FIELD, DIMENSIONS, NUM_DOCS, K, Map.of(METHOD_PARAMETER_EF_SEARCH, EF_SEARCH)); + deleteKNNIndex(testIndex); + } + } +} diff --git a/qa/rolling-upgrade/src/test/java/org/opensearch/knn/bwc/WarmupIT.java b/qa/rolling-upgrade/src/test/java/org/opensearch/knn/bwc/WarmupIT.java index a34f4b3cf..9cbb99d87 100644 --- a/qa/rolling-upgrade/src/test/java/org/opensearch/knn/bwc/WarmupIT.java +++ b/qa/rolling-upgrade/src/test/java/org/opensearch/knn/bwc/WarmupIT.java @@ -5,7 +5,6 @@ package org.opensearch.knn.bwc; -import java.io.IOException; import java.util.Collections; import static org.opensearch.knn.TestUtils.NODES_BWC_CLUSTER; @@ -47,7 +46,7 @@ public void testKNNWarmup() throws Exception { } // validation steps for KNN Warmup after upgrading each node from old version to new version - public void validateKNNWarmupOnUpgrade(int totalDocsCount, int docId) throws IOException { + public void validateKNNWarmupOnUpgrade(int totalDocsCount, int docId) throws Exception { int graphCount = getTotalGraphsInCache(); knnWarmup(Collections.singletonList(testIndex)); assertTrue(getTotalGraphsInCache() > graphCount); diff --git a/src/main/java/org/opensearch/knn/common/KNNConstants.java b/src/main/java/org/opensearch/knn/common/KNNConstants.java index 0b4538ec8..e73212afe 100644 --- a/src/main/java/org/opensearch/knn/common/KNNConstants.java +++ b/src/main/java/org/opensearch/knn/common/KNNConstants.java @@ -26,6 +26,7 @@ public class KNNConstants { public static final String K = "k"; public static final String TYPE_KNN_VECTOR = "knn_vector"; public static final String PROPERTIES = "properties"; + public static final String METHOD_PARAMETER = "method_parameters"; public static final String METHOD_PARAMETER_EF_SEARCH = "ef_search"; public static final String METHOD_PARAMETER_EF_CONSTRUCTION = "ef_construction"; public static final String METHOD_PARAMETER_M = "m"; diff --git a/src/main/java/org/opensearch/knn/index/IndexUtil.java b/src/main/java/org/opensearch/knn/index/IndexUtil.java index e4523bb5e..5cd4aaf81 100644 --- a/src/main/java/org/opensearch/knn/index/IndexUtil.java +++ b/src/main/java/org/opensearch/knn/index/IndexUtil.java @@ -20,6 +20,7 @@ import org.opensearch.common.ValidationException; import org.opensearch.knn.common.KNNConstants; import org.opensearch.knn.index.mapper.KNNVectorFieldMapper; +import org.opensearch.knn.index.query.request.MethodParameter; import org.opensearch.knn.index.util.KNNEngine; import org.opensearch.knn.indices.ModelDao; import org.opensearch.knn.indices.ModelMetadata; @@ -46,15 +47,29 @@ public class IndexUtil { private static final Version MINIMAL_SUPPORTED_VERSION_FOR_MODEL_NODE_ASSIGNMENT = Version.V_2_12_0; private static final Version MINIMAL_SUPPORTED_VERSION_FOR_MODEL_METHOD_COMPONENT_CONTEXT = Version.V_2_13_0; private static final Version MINIMAL_SUPPORTED_VERSION_FOR_RADIAL_SEARCH = Version.V_2_14_0; - public static final Map minimalRequiredVersionMap = new HashMap() { - { - put("filter", MINIMAL_SUPPORTED_VERSION_FOR_LUCENE_HNSW_FILTER); - put("ignore_unmapped", MINIMAL_SUPPORTED_VERSION_FOR_IGNORE_UNMAPPED); - put(MODEL_NODE_ASSIGNMENT_KEY, MINIMAL_SUPPORTED_VERSION_FOR_MODEL_NODE_ASSIGNMENT); - put(MODEL_METHOD_COMPONENT_CONTEXT_KEY, MINIMAL_SUPPORTED_VERSION_FOR_MODEL_METHOD_COMPONENT_CONTEXT); - put(KNNConstants.RADIAL_SEARCH_KEY, MINIMAL_SUPPORTED_VERSION_FOR_RADIAL_SEARCH); + private static final Version MINIMAL_SUPPORTED_VERSION_FOR_METHOD_PARAMETERS = Version.V_2_16_0; + // public so neural search can access it + public static final Map minimalRequiredVersionMap = initializeMinimalRequiredVersionMap(); + + private static Map initializeMinimalRequiredVersionMap() { + final Map versionMap = new HashMap<>() { + { + put("filter", MINIMAL_SUPPORTED_VERSION_FOR_LUCENE_HNSW_FILTER); + put("ignore_unmapped", MINIMAL_SUPPORTED_VERSION_FOR_IGNORE_UNMAPPED); + put(MODEL_NODE_ASSIGNMENT_KEY, MINIMAL_SUPPORTED_VERSION_FOR_MODEL_NODE_ASSIGNMENT); + put(MODEL_METHOD_COMPONENT_CONTEXT_KEY, MINIMAL_SUPPORTED_VERSION_FOR_MODEL_METHOD_COMPONENT_CONTEXT); + put(KNNConstants.RADIAL_SEARCH_KEY, MINIMAL_SUPPORTED_VERSION_FOR_RADIAL_SEARCH); + put(KNNConstants.METHOD_PARAMETER, MINIMAL_SUPPORTED_VERSION_FOR_METHOD_PARAMETERS); + } + }; + + for (final MethodParameter methodParameter : MethodParameter.values()) { + if (methodParameter.getVersion() != null) { + versionMap.put(methodParameter.getName(), methodParameter.getVersion()); + } } - }; + return Collections.unmodifiableMap(versionMap); + } /** * Determines the size of a file on disk in kilobytes diff --git a/src/main/java/org/opensearch/knn/index/MethodComponent.java b/src/main/java/org/opensearch/knn/index/MethodComponent.java index 256d55ee5..b344772f7 100644 --- a/src/main/java/org/opensearch/knn/index/MethodComponent.java +++ b/src/main/java/org/opensearch/knn/index/MethodComponent.java @@ -19,11 +19,13 @@ import org.opensearch.knn.index.util.IndexHyperParametersUtil; import org.opensearch.knn.training.VectorSpaceInfo; -import java.util.ArrayList; import java.util.HashMap; -import java.util.List; import java.util.Map; import java.util.function.BiFunction; +import java.util.List; +import java.util.ArrayList; + +import static org.opensearch.knn.validation.ParameterValidator.validateParameters; /** * MethodComponent defines the structure of an individual component that can make up an index @@ -75,32 +77,7 @@ public Map getAsMap(MethodComponentContext methodComponentContex */ public ValidationException validate(MethodComponentContext methodComponentContext) { Map providedParameters = methodComponentContext.getParameters(); - List errorMessages = new ArrayList<>(); - - if (providedParameters == null) { - return null; - } - - ValidationException parameterValidation; - for (Map.Entry parameter : providedParameters.entrySet()) { - if (!parameters.containsKey(parameter.getKey())) { - errorMessages.add(String.format("Invalid parameter for method \"%s\".", getName())); - continue; - } - - parameterValidation = parameters.get(parameter.getKey()).validate(parameter.getValue()); - if (parameterValidation != null) { - errorMessages.addAll(parameterValidation.validationErrors()); - } - } - - if (errorMessages.isEmpty()) { - return null; - } - - ValidationException validationException = new ValidationException(); - validationException.addValidationErrors(errorMessages); - return validationException; + return validateParameters(parameters, providedParameters); } /** diff --git a/src/main/java/org/opensearch/knn/index/query/BaseQueryFactory.java b/src/main/java/org/opensearch/knn/index/query/BaseQueryFactory.java index 3146cd33e..a02c090b1 100644 --- a/src/main/java/org/opensearch/knn/index/query/BaseQueryFactory.java +++ b/src/main/java/org/opensearch/knn/index/query/BaseQueryFactory.java @@ -20,6 +20,7 @@ import org.opensearch.knn.index.util.KNNEngine; import java.io.IOException; +import java.util.Map; import java.util.Optional; /** @@ -42,6 +43,7 @@ public static class CreateQueryRequest { private float[] vector; private byte[] byteVector; private VectorDataType vectorDataType; + private Map methodParameters; private Integer k; private Float radius; private QueryBuilder filter; diff --git a/src/main/java/org/opensearch/knn/index/query/KNNQuery.java b/src/main/java/org/opensearch/knn/index/query/KNNQuery.java index 0862b2d93..d123cc149 100644 --- a/src/main/java/org/opensearch/knn/index/query/KNNQuery.java +++ b/src/main/java/org/opensearch/knn/index/query/KNNQuery.java @@ -5,10 +5,8 @@ package org.opensearch.knn.index.query; -import java.util.Arrays; -import java.util.Objects; - import lombok.AllArgsConstructor; +import lombok.Builder; import lombok.Getter; import lombok.Setter; import org.apache.lucene.search.BooleanClause; @@ -23,26 +21,29 @@ import org.opensearch.knn.index.KNNSettings; import java.io.IOException; +import java.util.Arrays; +import java.util.Map; +import java.util.Objects; /** * Custom KNN query. Query is used for KNNEngine's that create their own custom segment files. These files need to be * loaded and queried in a custom manner throughout the query path. */ +@Getter +@Builder +@AllArgsConstructor public class KNNQuery extends Query { private final String field; private final float[] queryVector; private int k; + private Map methodParameters; private final String indexName; - @Getter @Setter private Query filterQuery; - @Getter private BitSetProducer parentsFilter; - @Getter - private Float radius = null; - @Getter + private Float radius; private Context context; public KNNQuery( @@ -123,22 +124,6 @@ public KNNQuery filterQuery(Query filterQuery) { return this; } - public String getField() { - return this.field; - } - - public float[] getQueryVector() { - return this.queryVector; - } - - public int getK() { - return this.k; - } - - public String getIndexName() { - return this.indexName; - } - /** * Constructs Weight implementation for this query * @@ -183,7 +168,17 @@ public String toString(String field) { @Override public int hashCode() { - return Objects.hash(field, Arrays.hashCode(queryVector), k, indexName, filterQuery); + return Objects.hash( + field, + Arrays.hashCode(queryVector), + k, + indexName, + filterQuery, + context, + parentsFilter, + radius, + methodParameters + ); } @Override @@ -192,10 +187,15 @@ public boolean equals(Object other) { } private boolean equalsTo(KNNQuery other) { + if (other == this) return true; return Objects.equals(field, other.field) && Arrays.equals(queryVector, other.queryVector) && Objects.equals(k, other.k) + && Objects.equals(methodParameters, other.methodParameters) + && Objects.equals(radius, other.radius) + && Objects.equals(context, other.context) && Objects.equals(indexName, other.indexName) + && Objects.equals(parentsFilter, other.parentsFilter) && Objects.equals(filterQuery, other.filterQuery); } diff --git a/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java b/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java index 88bcc84bc..1dec98c90 100644 --- a/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java +++ b/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java @@ -5,10 +5,14 @@ package org.opensearch.knn.index.query; +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.Getter; import lombok.extern.log4j.Log4j2; import org.apache.commons.lang.StringUtils; import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.Query; +import org.opensearch.common.ValidationException; import org.opensearch.core.ParseField; import org.opensearch.core.common.ParsingException; import org.opensearch.core.common.Strings; @@ -22,11 +26,15 @@ import org.opensearch.index.query.QueryBuilder; import org.opensearch.index.query.QueryShardContext; import org.opensearch.knn.common.KNNConstants; +import org.opensearch.knn.index.IndexUtil; +import org.opensearch.knn.index.util.EngineSpecificMethodContext; import org.opensearch.knn.index.KNNMethodContext; +import org.opensearch.knn.index.MethodComponentContext; import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.VectorQueryType; import org.opensearch.knn.index.mapper.KNNVectorFieldMapper; +import org.opensearch.knn.index.query.parser.MethodParametersParser; import org.opensearch.knn.index.util.KNNEngine; import org.opensearch.knn.indices.ModelDao; import org.opensearch.knn.indices.ModelMetadata; @@ -35,18 +43,25 @@ import java.io.IOException; import java.util.Arrays; import java.util.List; +import java.util.Map; import java.util.Objects; import static org.opensearch.knn.common.KNNConstants.MAX_DISTANCE; +import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER; +import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_EF_SEARCH; import static org.opensearch.knn.common.KNNConstants.MIN_SCORE; import static org.opensearch.knn.common.KNNValidationUtil.validateByteVectorValue; import static org.opensearch.knn.index.IndexUtil.isClusterOnOrAfterMinRequiredVersion; import static org.opensearch.knn.index.IndexUtil.minimalRequiredVersionMap; +import static org.opensearch.knn.index.query.parser.MethodParametersParser.validateMethodParameters; import static org.opensearch.knn.index.util.KNNEngine.ENGINES_SUPPORTING_RADIAL_SEARCH; +import static org.opensearch.knn.validation.ParameterValidator.validateParameters; /** * Helper class to build the KNN query */ +// The builder validates the member variables so access to the constructor is prohibited to not accidentally bypass validations +@AllArgsConstructor(access = AccessLevel.PRIVATE) @Log4j2 public class KNNQueryBuilder extends AbstractQueryBuilder { private static ModelDao modelDao; @@ -57,6 +72,8 @@ public class KNNQueryBuilder extends AbstractQueryBuilder { public static final ParseField IGNORE_UNMAPPED_FIELD = new ParseField("ignore_unmapped"); public static final ParseField MAX_DISTANCE_FIELD = new ParseField(MAX_DISTANCE); public static final ParseField MIN_SCORE_FIELD = new ParseField(MIN_SCORE); + public static final ParseField EF_SEARCH_FIELD = new ParseField(METHOD_PARAMETER_EF_SEARCH); + public static final ParseField METHOD_PARAMS_FIELD = new ParseField(METHOD_PARAMETER); public static final int K_MAX = 10000; /** * The name for the knn query @@ -67,18 +84,27 @@ public class KNNQueryBuilder extends AbstractQueryBuilder { */ private final String fieldName; private final float[] vector; - private int k = 0; - private Float maxDistance = null; - private Float minScore = null; + @Getter + private int k; + @Getter + private Float maxDistance; + @Getter + private Float minScore; + @Getter + private Map methodParameters; + @Getter private QueryBuilder filter; - private boolean ignoreUnmapped = false; + @Getter + private boolean ignoreUnmapped; /** * Constructs a new query with the given field name and vector * * @param fieldName Name of the field * @param vector Array of floating points + * @deprecated Use {@code {@link KNNQueryBuilder.Builder}} instead */ + @Deprecated public KNNQueryBuilder(String fieldName, float[] vector) { if (Strings.isNullOrEmpty(fieldName)) { throw new IllegalArgumentException(String.format("[%s] requires fieldName", NAME)); @@ -94,61 +120,136 @@ public KNNQueryBuilder(String fieldName, float[] vector) { } /** - * Builder method for k - * - * @param k K nearest neighbours for the given vector + * lombok SuperBuilder annotation requires a builder annotation on parent class to work well + * {@link AbstractQueryBuilder#boost()} and {@link AbstractQueryBuilder#queryName()} both need to be called + * A custom builder helps with the calls to the parent class, simultaneously addressing the problem of telescoping + * constructors in this class. */ - public KNNQueryBuilder k(Integer k) { - if (k == null) { - throw new IllegalArgumentException(String.format("[%s] requires k to be set", NAME)); + public static class Builder { + private String fieldName; + private float[] vector; + private Integer k; + private Map methodParameters; + private Float maxDistance; + private Float minScore; + private QueryBuilder filter; + private boolean ignoreUnmapped; + private String queryName; + private float boost = DEFAULT_BOOST; + + private Builder() {} + + public Builder fieldName(String fieldName) { + this.fieldName = fieldName; + return this; } - validateSingleQueryType(k, maxDistance, minScore); - if (k <= 0 || k > K_MAX) { - throw new IllegalArgumentException(String.format("[%s] requires k to be in the range (0, %d]", NAME, K_MAX)); + + public Builder vector(float[] vector) { + this.vector = vector; + return this; } - this.k = k; - return this; - } - /** - * Builder method for maxDistance - * - * @param maxDistance the maxDistance threshold for the nearest neighbours - */ - public KNNQueryBuilder maxDistance(Float maxDistance) { - if (maxDistance == null) { - throw new IllegalArgumentException(String.format("[%s] requires maxDistance to be set", NAME)); + public Builder k(Integer k) { + this.k = k; + return this; } - validateSingleQueryType(k, maxDistance, minScore); - this.maxDistance = maxDistance; - return this; - } - /** - * Builder method for minScore - * - * @param minScore the minScore threshold for the nearest neighbours - */ - public KNNQueryBuilder minScore(Float minScore) { - if (minScore == null) { - throw new IllegalArgumentException(String.format("[%s] requires minScore to be set", NAME)); + public Builder methodParameters(Map methodParameters) { + this.methodParameters = methodParameters; + return this; + } + + public Builder maxDistance(Float maxDistance) { + this.maxDistance = maxDistance; + return this; + } + + public Builder minScore(Float minScore) { + this.minScore = minScore; + return this; + } + + public Builder ignoreUnmapped(boolean ignoreUnmapped) { + this.ignoreUnmapped = ignoreUnmapped; + return this; + } + + public Builder filter(QueryBuilder filter) { + this.filter = filter; + return this; + } + + public Builder queryName(String queryName) { + this.queryName = queryName; + return this; + } + + public Builder boost(float boost) { + this.boost = boost; + return this; + } + + public KNNQueryBuilder build() { + validate(); + int k = this.k == null ? 0 : this.k; + return new KNNQueryBuilder(fieldName, vector, k, maxDistance, minScore, methodParameters, filter, ignoreUnmapped).boost(boost) + .queryName(queryName); } - validateSingleQueryType(k, maxDistance, minScore); - if (minScore <= 0) { - throw new IllegalArgumentException(String.format("[%s] requires minScore to be greater than 0", NAME)); + + private void validate() { + if (Strings.isNullOrEmpty(fieldName)) { + throw new IllegalArgumentException(String.format("[%s] requires fieldName", NAME)); + } + + if (vector == null) { + throw new IllegalArgumentException(String.format("[%s] requires query vector", NAME)); + } else if (vector.length == 0) { + throw new IllegalArgumentException(String.format("[%s] query vector is empty", NAME)); + } + + if (k == null && minScore == null && maxDistance == null) { + throw new IllegalArgumentException(String.format("[%s] requires exactly one of k, distance or score to be set", NAME)); + } + + if ((k != null && maxDistance != null) || (maxDistance != null && minScore != null) || (k != null && minScore != null)) { + throw new IllegalArgumentException(String.format("[%s] requires exactly one of k, distance or score to be set", NAME)); + } + + VectorQueryType vectorQueryType = VectorQueryType.MAX_DISTANCE; + if (k != null) { + vectorQueryType = VectorQueryType.K; + if (k <= 0 || k > K_MAX) { + final String errorMessage = "[" + NAME + "] requires k to be in the range (0, " + K_MAX + "]"; + throw new IllegalArgumentException(errorMessage); + } + } + + if (minScore != null) { + vectorQueryType = VectorQueryType.MIN_SCORE; + if (minScore <= 0) { + throw new IllegalArgumentException(String.format("[%s] requires minScore to be greater than 0", NAME)); + } + } + + if (methodParameters != null) { + ValidationException validationException = validateMethodParameters(methodParameters); + if (validationException != null) { + throw new IllegalArgumentException( + String.format("[%s] errors in method parameter [%s]", NAME, validationException.getMessage()) + ); + } + } + + // Update stats + vectorQueryType.getQueryStatCounter().increment(); + if (filter != null) { + vectorQueryType.getQueryWithFilterStatCounter().increment(); + } } - this.minScore = minScore; - return this; } - /** - * Builder method for filter - * - * @param filter QueryBuilder - */ - public KNNQueryBuilder filter(QueryBuilder filter) { - this.filter = filter; - return this; + public static KNNQueryBuilder.Builder builder() { + return new KNNQueryBuilder.Builder(); } /** @@ -158,12 +259,14 @@ public KNNQueryBuilder filter(QueryBuilder filter) { * @param vector Array of floating points * @param k K nearest neighbours for the given vector */ + @Deprecated public KNNQueryBuilder(String fieldName, float[] vector, int k) { this(fieldName, vector, k, null); } + @Deprecated public KNNQueryBuilder(String fieldName, float[] vector, int k, QueryBuilder filter) { - if (StringUtils.isBlank(fieldName)) { + if (Strings.isNullOrEmpty(fieldName)) { throw new IllegalArgumentException(String.format("[%s] requires fieldName", NAME)); } if (vector == null) { @@ -230,6 +333,10 @@ public KNNQueryBuilder(StreamInput in) throws IOException { if (isClusterOnOrAfterMinRequiredVersion(KNNConstants.RADIAL_SEARCH_KEY)) { minScore = in.readOptionalFloat(); } + if (isClusterOnOrAfterMinRequiredVersion(METHOD_PARAMETER)) { + methodParameters = MethodParametersParser.streamInput(in, IndexUtil::isClusterOnOrAfterMinRequiredVersion); + } + } catch (IOException ex) { throw new RuntimeException("[KNN] Unable to create KNNQueryBuilder", ex); } @@ -243,9 +350,10 @@ public static KNNQueryBuilder fromXContent(XContentParser parser) throws IOExcep Float maxDistance = null; Float minScore = null; QueryBuilder filter = null; - boolean ignoreUnmapped = false; String queryName = null; String currentFieldName = null; + boolean ignoreUnmapped = false; + Map methodParameters = null; XContentParser.Token token; while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { if (token == XContentParser.Token.FIELD_NAME) { @@ -263,16 +371,16 @@ public static KNNQueryBuilder fromXContent(XContentParser parser) throws IOExcep boost = parser.floatValue(); } else if (K_FIELD.match(currentFieldName, parser.getDeprecationHandler())) { k = (Integer) NumberFieldMapper.NumberType.INTEGER.parse(parser.objectBytes(), false); + } else if (IGNORE_UNMAPPED_FIELD.getPreferredName().equals(currentFieldName)) { + if (isClusterOnOrAfterMinRequiredVersion("ignore_unmapped")) { + ignoreUnmapped = parser.booleanValue(); + } } else if (AbstractQueryBuilder.NAME_FIELD.match(currentFieldName, parser.getDeprecationHandler())) { queryName = parser.text(); } else if (MAX_DISTANCE_FIELD.match(currentFieldName, parser.getDeprecationHandler())) { maxDistance = (Float) NumberFieldMapper.NumberType.FLOAT.parse(parser.objectBytes(), false); } else if (MIN_SCORE_FIELD.match(currentFieldName, parser.getDeprecationHandler())) { minScore = (Float) NumberFieldMapper.NumberType.FLOAT.parse(parser.objectBytes(), false); - } else if (IGNORE_UNMAPPED_FIELD.getPreferredName().equals(currentFieldName)) { - if (isClusterOnOrAfterMinRequiredVersion("ignore_unmapped")) { - ignoreUnmapped = parser.booleanValue(); - } } else { throw new ParsingException( parser.getTokenLocation(), @@ -304,6 +412,8 @@ public static KNNQueryBuilder fromXContent(XContentParser parser) throws IOExcep ) ); } + } else if (METHOD_PARAMS_FIELD.match(currentFieldName, parser.getDeprecationHandler())) { + methodParameters = MethodParametersParser.fromXContent(parser); } else { throw new ParsingException(parser.getTokenLocation(), "[" + NAME + "] unknown token [" + token + "]"); } @@ -321,29 +431,18 @@ public static KNNQueryBuilder fromXContent(XContentParser parser) throws IOExcep } } - VectorQueryType vectorQueryType = validateSingleQueryType(k, maxDistance, minScore); - vectorQueryType.getQueryStatCounter().increment(); - if (filter != null) { - vectorQueryType.getQueryWithFilterStatCounter().increment(); - } - - KNNQueryBuilder knnQueryBuilder = new KNNQueryBuilder(fieldName, ObjectsToFloats(vector)).filter(filter) + return KNNQueryBuilder.builder() + .queryName(queryName) .boost(boost) - .queryName(queryName); - - if (isClusterOnOrAfterMinRequiredVersion("ignoreUnmapped")) { - knnQueryBuilder.ignoreUnmapped(ignoreUnmapped); - } - - if (k != null) { - knnQueryBuilder.k(k); - } else if (maxDistance != null) { - knnQueryBuilder.maxDistance(maxDistance); - } else if (minScore != null) { - knnQueryBuilder.minScore(minScore); - } - - return knnQueryBuilder; + .fieldName(fieldName) + .vector(ObjectsToFloats(vector)) + .k(k) + .maxDistance(maxDistance) + .minScore(minScore) + .methodParameters(methodParameters) + .ignoreUnmapped(ignoreUnmapped) + .filter(filter) + .build(); } @Override @@ -365,6 +464,9 @@ protected void doWriteTo(StreamOutput out) throws IOException { if (isClusterOnOrAfterMinRequiredVersion(KNNConstants.RADIAL_SEARCH_KEY)) { out.writeOptionalFloat(minScore); } + if (isClusterOnOrAfterMinRequiredVersion(METHOD_PARAMETER)) { + MethodParametersParser.streamOutput(out, methodParameters, IndexUtil::isClusterOnOrAfterMinRequiredVersion); + } } /** @@ -381,36 +483,6 @@ public Object vector() { return this.vector; } - public int getK() { - return this.k; - } - - public float getMaxDistance() { - return this.maxDistance; - } - - public float getMinScore() { - return this.minScore; - } - - public QueryBuilder getFilter() { - return this.filter; - } - - /** - * Sets whether the query builder should ignore unmapped paths (and run a - * {@link MatchNoDocsQuery} in place of this query) or throw an exception if - * the path is unmapped. - */ - public KNNQueryBuilder ignoreUnmapped(boolean ignoreUnmapped) { - this.ignoreUnmapped = ignoreUnmapped; - return this; - } - - public boolean getIgnoreUnmapped() { - return this.ignoreUnmapped; - } - @Override public void doXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(NAME); @@ -430,6 +502,9 @@ public void doXContent(XContentBuilder builder, Params params) throws IOExceptio if (minScore != null) { builder.field(MIN_SCORE_FIELD.getPreferredName(), minScore); } + if (methodParameters != null) { + MethodParametersParser.doXContent(builder, methodParameters); + } printBoostAndQueryName(builder); builder.endObject(); builder.endObject(); @@ -450,6 +525,7 @@ protected Query doToQuery(QueryShardContext context) { KNNVectorFieldMapper.KNNVectorFieldType knnVectorFieldType = (KNNVectorFieldMapper.KNNVectorFieldType) mappedFieldType; int fieldDimension = knnVectorFieldType.getDimension(); KNNMethodContext knnMethodContext = knnVectorFieldType.getKnnMethodContext(); + MethodComponentContext methodComponentContext = null; KNNEngine knnEngine = KNNEngine.DEFAULT; VectorDataType vectorDataType = knnVectorFieldType.getVectorDataType(); SpaceType spaceType = knnVectorFieldType.getSpaceType(); @@ -463,10 +539,32 @@ protected Query doToQuery(QueryShardContext context) { fieldDimension = modelMetadata.getDimension(); knnEngine = modelMetadata.getKnnEngine(); spaceType = modelMetadata.getSpaceType(); + methodComponentContext = modelMetadata.getMethodComponentContext(); + } else if (knnMethodContext != null) { // If the dimension is set but the knnMethodContext is not then the field is using the legacy mapping knnEngine = knnMethodContext.getKnnEngine(); spaceType = knnMethodContext.getSpaceType(); + methodComponentContext = knnMethodContext.getMethodComponentContext(); + } + + final String method = methodComponentContext != null ? methodComponentContext.getName() : null; + if (StringUtils.isNotBlank(method)) { + final EngineSpecificMethodContext engineSpecificMethodContext = knnEngine.getMethodContext(method); + ValidationException validationException = validateParameters( + engineSpecificMethodContext.supportedMethodParameters(), + (Map) methodParameters + ); + if (validationException != null) { + throw new IllegalArgumentException( + String.format( + "Parameters not valid for [%s]:[%s] combination: [%s]", + knnEngine, + method, + validationException.getMessage() + ) + ); + } } // Currently, k-NN supports distance and score types radial search @@ -525,6 +623,7 @@ protected Query doToQuery(QueryShardContext context) { .byteVector(VectorDataType.BYTE == vectorDataType ? byteVector : null) .vectorDataType(vectorDataType) .k(this.k) + .methodParameters(this.methodParameters) .filter(this.filter) .context(context) .build(); @@ -569,41 +668,20 @@ protected boolean doEquals(KNNQueryBuilder other) { return Objects.equals(fieldName, other.fieldName) && Arrays.equals(vector, other.vector) && Objects.equals(k, other.k) + && Objects.equals(minScore, other.minScore) + && Objects.equals(maxDistance, other.maxDistance) + && Objects.equals(methodParameters, other.methodParameters) && Objects.equals(filter, other.filter) && Objects.equals(ignoreUnmapped, other.ignoreUnmapped); } @Override protected int doHashCode() { - return Objects.hash(fieldName, Arrays.hashCode(vector), k, filter, ignoreUnmapped); + return Objects.hash(fieldName, Arrays.hashCode(vector), k, methodParameters, filter, ignoreUnmapped, maxDistance, minScore); } @Override public String getWriteableName() { return NAME; } - - private static VectorQueryType validateSingleQueryType(Integer k, Float distance, Float score) { - int countSetFields = 0; - VectorQueryType vectorQueryType = null; - - if (k != null && k != 0) { - countSetFields++; - vectorQueryType = VectorQueryType.K; - } - if (distance != null) { - countSetFields++; - vectorQueryType = VectorQueryType.MAX_DISTANCE; - } - if (score != null) { - countSetFields++; - vectorQueryType = VectorQueryType.MIN_SCORE; - } - - if (countSetFields != 1) { - throw new IllegalArgumentException(String.format("[%s] requires exactly one of k, distance or score to be set", NAME)); - } - - return vectorQueryType; - } } diff --git a/src/main/java/org/opensearch/knn/index/query/KNNQueryFactory.java b/src/main/java/org/opensearch/knn/index/query/KNNQueryFactory.java index ec1f53d13..36987c750 100644 --- a/src/main/java/org/opensearch/knn/index/query/KNNQueryFactory.java +++ b/src/main/java/org/opensearch/knn/index/query/KNNQueryFactory.java @@ -17,7 +17,9 @@ import org.opensearch.knn.index.util.KNNEngine; import java.util.Locale; +import java.util.Map; +import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_EF_SEARCH; import static org.opensearch.knn.common.KNNConstants.VECTOR_DATA_TYPE_FIELD; import static org.opensearch.knn.index.VectorDataType.SUPPORTED_VECTOR_DATA_TYPES; @@ -71,6 +73,7 @@ public static Query create(CreateQueryRequest createQueryRequest) { final byte[] byteVector = createQueryRequest.getByteVector(); final VectorDataType vectorDataType = createQueryRequest.getVectorDataType(); final Query filterQuery = getFilterQuery(createQueryRequest); + final Map methodParameters = createQueryRequest.getMethodParameters(); BitSetProducer parentFilter = null; if (createQueryRequest.getContext().isPresent()) { @@ -79,20 +82,37 @@ public static Query create(CreateQueryRequest createQueryRequest) { } if (KNNEngine.getEnginesThatCreateCustomSegmentFiles().contains(createQueryRequest.getKnnEngine())) { - if (filterQuery != null && KNNEngine.getEnginesThatSupportsFilters().contains(createQueryRequest.getKnnEngine())) { - log.debug("Creating custom k-NN query with filters for index: {}, field: {} , k: {}", indexName, fieldName, k); - return new KNNQuery(fieldName, vector, k, indexName, filterQuery, parentFilter); - } - log.debug(String.format("Creating custom k-NN query for index: %s \"\", field: %s \"\", k: %d", indexName, fieldName, k)); - return new KNNQuery(fieldName, vector, k, indexName, parentFilter); + final Query validatedFilterQuery = validateFilterQuerySupport(filterQuery, createQueryRequest.getKnnEngine()); + log.debug( + "Creating custom k-NN query for index:{}, field:{}, k:{}, filterQuery:{}, efSearch:{}", + indexName, + fieldName, + k, + validatedFilterQuery, + methodParameters + ); + return KNNQuery.builder() + .field(fieldName) + .queryVector(vector) + .indexName(indexName) + .parentsFilter(parentFilter) + .k(k) + .methodParameters(methodParameters) + .filterQuery(validatedFilterQuery) + .build(); } + Integer requestEfSearch = null; + if (methodParameters != null && methodParameters.containsKey(METHOD_PARAMETER_EF_SEARCH)) { + requestEfSearch = (Integer) methodParameters.get(METHOD_PARAMETER_EF_SEARCH); + } + int luceneK = requestEfSearch == null ? k : Math.max(k, requestEfSearch); log.debug(String.format("Creating Lucene k-NN query for index: %s \"\", field: %s \"\", k: %d", indexName, fieldName, k)); switch (vectorDataType) { case BYTE: - return getKnnByteVectorQuery(fieldName, byteVector, k, filterQuery, parentFilter); + return getKnnByteVectorQuery(fieldName, byteVector, luceneK, filterQuery, parentFilter); case FLOAT: - return getKnnFloatVectorQuery(fieldName, vector, k, filterQuery, parentFilter); + return getKnnFloatVectorQuery(fieldName, vector, luceneK, filterQuery, parentFilter); default: throw new IllegalArgumentException( String.format( @@ -106,6 +126,14 @@ public static Query create(CreateQueryRequest createQueryRequest) { } } + private static Query validateFilterQuerySupport(final Query filterQuery, final KNNEngine knnEngine) { + log.debug("filter query {}, knnEngine {}", filterQuery, knnEngine); + if (filterQuery != null && KNNEngine.getEnginesThatSupportsFilters().contains(knnEngine)) { + return filterQuery; + } + return null; + } + /** * If parentFilter is not null, it is a nested query. Therefore, we return {@link DiversifyingChildrenByteKnnVectorQuery} * which will dedupe search result per parent so that we can get k parent results at the end. diff --git a/src/main/java/org/opensearch/knn/index/query/KNNWeight.java b/src/main/java/org/opensearch/knn/index/query/KNNWeight.java index bac8c03d4..794c9af1c 100644 --- a/src/main/java/org/opensearch/knn/index/query/KNNWeight.java +++ b/src/main/java/org/opensearch/knn/index/query/KNNWeight.java @@ -283,6 +283,7 @@ private Map doANNSearch(final LeafReaderContext context, final B indexAllocation.getMemoryAddress(), knnQuery.getQueryVector(), knnQuery.getK(), + knnQuery.getMethodParameters(), knnEngine, filterIds, filterType.getValue(), diff --git a/src/main/java/org/opensearch/knn/index/query/parser/MethodParametersParser.java b/src/main/java/org/opensearch/knn/index/query/parser/MethodParametersParser.java new file mode 100644 index 000000000..e2ba8f26e --- /dev/null +++ b/src/main/java/org/opensearch/knn/index/query/parser/MethodParametersParser.java @@ -0,0 +1,137 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.knn.index.query.parser; + +import lombok.AllArgsConstructor; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import org.opensearch.common.ValidationException; +import org.opensearch.core.common.ParsingException; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.knn.index.query.request.MethodParameter; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.function.Function; + +import static org.opensearch.knn.index.query.KNNQueryBuilder.METHOD_PARAMS_FIELD; +import static org.opensearch.knn.index.query.KNNQueryBuilder.NAME; + +@EqualsAndHashCode +@Getter +@AllArgsConstructor +public class MethodParametersParser { + + // Validation on rest layer + public static ValidationException validateMethodParameters(final Map methodParameters) { + final List errors = new ArrayList<>(); + for (final Map.Entry methodParameter : methodParameters.entrySet()) { + final MethodParameter parameter = MethodParameter.enumOf(methodParameter.getKey()); + if (parameter != null) { + final ValidationException validationException = parameter.validate(methodParameter.getValue()); + if (validationException != null) { + errors.add(validationException.getMessage()); + } + } else { // Should never happen if used in the right sequence + errors.add(methodParameter.getKey() + " is not a valid method parameter"); + } + } + + if (!errors.isEmpty()) { + ValidationException validationException = new ValidationException(); + validationException.addValidationErrors(errors); + return validationException; + } + return null; + } + + // deserialize for node to node communication + public static Map streamInput(StreamInput in, Function minClusterVersionCheck) throws IOException { + if (!in.readBoolean()) { + return null; + } + + final Map methodParameters = new HashMap<>(); + for (final MethodParameter methodParameter : MethodParameter.values()) { + if (minClusterVersionCheck.apply(methodParameter.getName())) { + String name = in.readString(); + Object value = in.readGenericValue(); + if (value != null) { + methodParameters.put(name, methodParameter.parse(value)); + } + } + } + + return !methodParameters.isEmpty() ? methodParameters : null; + } + + // serialize for node to node communication + public static void streamOutput(StreamOutput out, Map methodParameters, Function minClusterVersionCheck) + throws IOException { + if (methodParameters == null || methodParameters.isEmpty()) { + out.writeBoolean(false); + } else { + out.writeBoolean(true); + // All values are written to deserialize without ambiguity + for (final MethodParameter methodParameter : MethodParameter.values()) { + if (minClusterVersionCheck.apply(methodParameter.getName())) { + out.writeString(methodParameter.getName()); + out.writeGenericValue(methodParameters.get(methodParameter.getName())); + } + } + } + } + + public static void doXContent(final XContentBuilder builder, final Map methodParameters) throws IOException { + if (methodParameters == null || methodParameters.isEmpty()) { + return; + } + builder.startObject(METHOD_PARAMS_FIELD.getPreferredName()); + for (final Map.Entry entry : methodParameters.entrySet()) { + if (entry.getKey() != null && entry.getValue() != null) { + builder.field(entry.getKey(), entry.getValue()); + } + } + builder.endObject(); + } + + public static Map fromXContent(final XContentParser parser) throws IOException { + final Map methodParametersJson = parser.map(); + if (methodParametersJson.isEmpty()) { + throw new ParsingException(parser.getTokenLocation(), METHOD_PARAMS_FIELD.getPreferredName() + " cannot be empty"); + } + + final Map methodParameters = new HashMap<>(); + for (Map.Entry requestParameter : methodParametersJson.entrySet()) { + final String name = requestParameter.getKey(); + final Object value = requestParameter.getValue(); + final MethodParameter parameter = MethodParameter.enumOf(name); + if (parameter == null) { + throw new ParsingException(parser.getTokenLocation(), "[" + NAME + "] unknown method parameter found [" + name + "]"); + } + + try { + // This makes sure that we throw parsing exception on rest layer. + methodParameters.put(name, parameter.parse(value)); + } catch (final Exception exception) { + throw new ParsingException(parser.getTokenLocation(), exception.getMessage()); + } + } + return methodParameters.isEmpty() ? null : methodParameters; + } +} diff --git a/src/main/java/org/opensearch/knn/index/query/request/MethodParameter.java b/src/main/java/org/opensearch/knn/index/query/request/MethodParameter.java new file mode 100644 index 000000000..0b76cad4a --- /dev/null +++ b/src/main/java/org/opensearch/knn/index/query/request/MethodParameter.java @@ -0,0 +1,77 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.knn.index.query.request; + +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import org.opensearch.Version; +import org.opensearch.common.ValidationException; +import org.opensearch.core.ParseField; + +import java.util.HashMap; +import java.util.Map; + +import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_EF_SEARCH; +import static org.opensearch.knn.index.query.KNNQueryBuilder.EF_SEARCH_FIELD; + +/** + * MethodParameters are engine and algorithm related parameters that clients can pass in knn query + * This enum holds metadata which helps parse and have basic validation related to MethodParameter + */ +@Getter +@RequiredArgsConstructor +public enum MethodParameter { + + EF_SEARCH(METHOD_PARAMETER_EF_SEARCH, Version.V_2_16_0, EF_SEARCH_FIELD) { + @Override + public Integer parse(Object value) { + try { + return Integer.parseInt(String.valueOf(value)); + } catch (final NumberFormatException e) { + throw new IllegalArgumentException(METHOD_PARAMETER_EF_SEARCH + " value must be an integer"); + } + } + + @Override + public ValidationException validate(Object value) { + final Integer ef = parse(value); + if (ef != null && ef > 0) { + return null; + } + ; + ValidationException validationException = new ValidationException(); + validationException.addValidationError(METHOD_PARAMETER_EF_SEARCH + " should be greater than 0"); + return validationException; + } + }; + + private final String name; + private final Version version; + private final ParseField parseField; + + private static Map PARAMETERS_DIR; + + public abstract T parse(Object value); + + // These are preliminary validations on rest layer + public abstract ValidationException validate(Object value); + + public static MethodParameter enumOf(final String name) { + if (PARAMETERS_DIR == null) { + PARAMETERS_DIR = new HashMap<>(); + for (final MethodParameter methodParameter : MethodParameter.values()) { + PARAMETERS_DIR.put(methodParameter.name, methodParameter); + } + } + return PARAMETERS_DIR.get(name); + } +} diff --git a/src/main/java/org/opensearch/knn/index/util/AbstractKNNLibrary.java b/src/main/java/org/opensearch/knn/index/util/AbstractKNNLibrary.java index 0fe311094..0e0c56128 100644 --- a/src/main/java/org/opensearch/knn/index/util/AbstractKNNLibrary.java +++ b/src/main/java/org/opensearch/knn/index/util/AbstractKNNLibrary.java @@ -22,6 +22,7 @@ public abstract class AbstractKNNLibrary implements KNNLibrary { protected final Map methods; + protected final Map engineMethods; @Getter protected final String version; @@ -34,6 +35,15 @@ public KNNMethod getMethod(String methodName) { return method; } + @Override + public EngineSpecificMethodContext getMethodContext(String methodName) { + EngineSpecificMethodContext method = engineMethods.get(methodName); + if (method == null) { + throw new IllegalArgumentException(String.format("Invalid method name: %s", methodName)); + } + return method; + } + @Override public ValidationException validateMethod(KNNMethodContext knnMethodContext) { String methodName = knnMethodContext.getMethodComponentContext().getName(); diff --git a/src/main/java/org/opensearch/knn/index/util/DefaultHnswContext.java b/src/main/java/org/opensearch/knn/index/util/DefaultHnswContext.java new file mode 100644 index 000000000..c16f1b05e --- /dev/null +++ b/src/main/java/org/opensearch/knn/index/util/DefaultHnswContext.java @@ -0,0 +1,33 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.knn.index.util; + +import com.google.common.collect.ImmutableMap; +import org.opensearch.knn.index.Parameter; +import org.opensearch.knn.index.query.request.MethodParameter; + +import java.util.Map; + +/** + * Default HNSW context for all engines. Have a different implementation if engine context differs. + */ +public final class DefaultHnswContext implements EngineSpecificMethodContext { + + private final Map> supportedMethodParameters = ImmutableMap.>builder() + .put(MethodParameter.EF_SEARCH.getName(), new Parameter.IntegerParameter(MethodParameter.EF_SEARCH.getName(), null, value -> true)) + .build(); + + @Override + public Map> supportedMethodParameters() { + return supportedMethodParameters; + } +} diff --git a/src/main/java/org/opensearch/knn/index/util/EngineSpecificMethodContext.java b/src/main/java/org/opensearch/knn/index/util/EngineSpecificMethodContext.java new file mode 100644 index 000000000..f669704ad --- /dev/null +++ b/src/main/java/org/opensearch/knn/index/util/EngineSpecificMethodContext.java @@ -0,0 +1,31 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.knn.index.util; + +import org.opensearch.knn.index.Parameter; + +import java.util.Collections; +import java.util.Map; + +/** + * Holds context related to a method for a particular engine + * Each engine can have a specific set of parameters that it supports during index and build time. This context holds + * the information for each engine method combination. + * + * TODO: Move KnnMethod in here + */ +public interface EngineSpecificMethodContext { + + Map> supportedMethodParameters(); + + EngineSpecificMethodContext EMPTY = Collections::emptyMap; +} diff --git a/src/main/java/org/opensearch/knn/index/util/Faiss.java b/src/main/java/org/opensearch/knn/index/util/Faiss.java index bbb58bf1e..7cf31ba3c 100644 --- a/src/main/java/org/opensearch/knn/index/util/Faiss.java +++ b/src/main/java/org/opensearch/knn/index/util/Faiss.java @@ -330,7 +330,13 @@ private Faiss( String extension, Map> scoreTransform ) { - super(methods, scoreTranslation, currentVersion, extension); + super( + methods, + Map.of(METHOD_HNSW, new DefaultHnswContext(), METHOD_IVF, EngineSpecificMethodContext.EMPTY), + scoreTranslation, + currentVersion, + extension + ); this.scoreTransform = scoreTransform; } diff --git a/src/main/java/org/opensearch/knn/index/util/JVMLibrary.java b/src/main/java/org/opensearch/knn/index/util/JVMLibrary.java index e1d48cb0a..850679dc2 100644 --- a/src/main/java/org/opensearch/knn/index/util/JVMLibrary.java +++ b/src/main/java/org/opensearch/knn/index/util/JVMLibrary.java @@ -23,8 +23,8 @@ public abstract class JVMLibrary extends AbstractKNNLibrary { * @param methods Map of k-NN methods that the library supports * @param version String representing version of library */ - JVMLibrary(Map methods, String version) { - super(methods, version); + JVMLibrary(Map methods, Map engineMethodMetadataMap, String version) { + super(methods, engineMethodMetadataMap, version); } @Override diff --git a/src/main/java/org/opensearch/knn/index/util/KNNEngine.java b/src/main/java/org/opensearch/knn/index/util/KNNEngine.java index 556785783..ee8be9c5c 100644 --- a/src/main/java/org/opensearch/knn/index/util/KNNEngine.java +++ b/src/main/java/org/opensearch/knn/index/util/KNNEngine.java @@ -149,6 +149,11 @@ public KNNMethod getMethod(String methodName) { return knnLibrary.getMethod(methodName); } + @Override + public EngineSpecificMethodContext getMethodContext(String methodName) { + return knnLibrary.getMethodContext(methodName); + } + @Override public float score(float rawScore, SpaceType spaceType) { return knnLibrary.score(rawScore, spaceType); diff --git a/src/main/java/org/opensearch/knn/index/util/KNNLibrary.java b/src/main/java/org/opensearch/knn/index/util/KNNLibrary.java index cac5af2bb..f9d8429d3 100644 --- a/src/main/java/org/opensearch/knn/index/util/KNNLibrary.java +++ b/src/main/java/org/opensearch/knn/index/util/KNNLibrary.java @@ -58,6 +58,13 @@ public interface KNNLibrary { */ KNNMethod getMethod(String methodName); + /** + * Gets metadata related to methods supported by the library + * @param methodName + * @return + */ + EngineSpecificMethodContext getMethodContext(String methodName); + /** * Generate the Lucene score from the rawScore returned by the library. With k-NN, often times the library * will return a score where the lower the score, the better the result. This is the opposite of how Lucene scores diff --git a/src/main/java/org/opensearch/knn/index/util/Lucene.java b/src/main/java/org/opensearch/knn/index/util/Lucene.java index 630d7a2c2..ae6ea3a70 100644 --- a/src/main/java/org/opensearch/knn/index/util/Lucene.java +++ b/src/main/java/org/opensearch/knn/index/util/Lucene.java @@ -67,7 +67,7 @@ public class Lucene extends JVMLibrary { * @param distanceTransform Map of space type to distance transformation function */ Lucene(Map methods, String version, Map> distanceTransform) { - super(methods, version); + super(methods, Map.of(METHOD_HNSW, new DefaultHnswContext()), version); this.distanceTransform = distanceTransform; } diff --git a/src/main/java/org/opensearch/knn/index/util/NativeLibrary.java b/src/main/java/org/opensearch/knn/index/util/NativeLibrary.java index 5e264ed12..99d4aeeb9 100644 --- a/src/main/java/org/opensearch/knn/index/util/NativeLibrary.java +++ b/src/main/java/org/opensearch/knn/index/util/NativeLibrary.java @@ -36,11 +36,12 @@ abstract class NativeLibrary extends AbstractKNNLibrary { */ NativeLibrary( Map methods, + Map engineMethods, Map> scoreTranslation, String version, String extension ) { - super(methods, version); + super(methods, engineMethods, version); this.scoreTranslation = scoreTranslation; this.extension = extension; this.initialized = new AtomicBoolean(false); diff --git a/src/main/java/org/opensearch/knn/index/util/Nmslib.java b/src/main/java/org/opensearch/knn/index/util/Nmslib.java index 64af43520..7b18ed11d 100644 --- a/src/main/java/org/opensearch/knn/index/util/Nmslib.java +++ b/src/main/java/org/opensearch/knn/index/util/Nmslib.java @@ -66,7 +66,7 @@ private Nmslib( String currentVersion, String extension ) { - super(methods, scoreTranslation, currentVersion, extension); + super(methods, Map.of(METHOD_HNSW, new DefaultHnswContext()), scoreTranslation, currentVersion, extension); } @Override diff --git a/src/main/java/org/opensearch/knn/jni/FaissService.java b/src/main/java/org/opensearch/knn/jni/FaissService.java index 53980bbb7..77b786421 100644 --- a/src/main/java/org/opensearch/knn/jni/FaissService.java +++ b/src/main/java/org/opensearch/knn/jni/FaissService.java @@ -129,7 +129,13 @@ public static native void createIndexFromTemplate( * @param parentIds list of parent doc ids when the knn field is a nested field * @return KNNQueryResult array of k neighbors */ - public static native KNNQueryResult[] queryIndex(long indexPointer, float[] queryVector, int k, int[] parentIds); + public static native KNNQueryResult[] queryIndex( + long indexPointer, + float[] queryVector, + int k, + Map methodParameters, + int[] parentIds + ); /** * Query an index with filter @@ -145,6 +151,7 @@ public static native KNNQueryResult[] queryIndexWithFilter( long indexPointer, float[] queryVector, int k, + Map methodParameters, long[] filterIds, int filterIdsType, int[] parentIds diff --git a/src/main/java/org/opensearch/knn/jni/JNIService.java b/src/main/java/org/opensearch/knn/jni/JNIService.java index 20c418819..6563da296 100644 --- a/src/main/java/org/opensearch/knn/jni/JNIService.java +++ b/src/main/java/org/opensearch/knn/jni/JNIService.java @@ -12,6 +12,7 @@ package org.opensearch.knn.jni; import org.apache.commons.lang.ArrayUtils; +import org.opensearch.common.Nullable; import org.opensearch.knn.index.query.KNNQueryResult; import org.opensearch.knn.index.util.KNNEngine; @@ -173,13 +174,14 @@ public static KNNQueryResult[] queryIndex( long indexPointer, float[] queryVector, int k, + @Nullable Map methodParameters, KNNEngine knnEngine, long[] filteredIds, int filterIdsType, int[] parentIds ) { if (KNNEngine.NMSLIB == knnEngine) { - return NmslibService.queryIndex(indexPointer, queryVector, k); + return NmslibService.queryIndex(indexPointer, queryVector, k, methodParameters); } if (KNNEngine.FAISS == knnEngine) { @@ -188,9 +190,17 @@ public static KNNQueryResult[] queryIndex( // filterIds. FilterIds is coming as empty then its the case where we need to do search with Faiss engine // normally. if (ArrayUtils.isNotEmpty(filteredIds)) { - return FaissService.queryIndexWithFilter(indexPointer, queryVector, k, filteredIds, filterIdsType, parentIds); + return FaissService.queryIndexWithFilter( + indexPointer, + queryVector, + k, + methodParameters, + filteredIds, + filterIdsType, + parentIds + ); } - return FaissService.queryIndex(indexPointer, queryVector, k, parentIds); + return FaissService.queryIndex(indexPointer, queryVector, k, methodParameters, parentIds); } throw new IllegalArgumentException(String.format("QueryIndex not supported for provided engine : %s", knnEngine.getName())); } diff --git a/src/main/java/org/opensearch/knn/jni/NmslibService.java b/src/main/java/org/opensearch/knn/jni/NmslibService.java index 7fdc278d2..294c5a208 100644 --- a/src/main/java/org/opensearch/knn/jni/NmslibService.java +++ b/src/main/java/org/opensearch/knn/jni/NmslibService.java @@ -69,7 +69,7 @@ class NmslibService { * @param k neighbors to be returned * @return KNNQueryResult array of k neighbors */ - public static native KNNQueryResult[] queryIndex(long indexPointer, float[] queryVector, int k); + public static native KNNQueryResult[] queryIndex(long indexPointer, float[] queryVector, int k, Map methodParameters); /** * Free native memory pointer diff --git a/src/main/java/org/opensearch/knn/validation/ParameterValidator.java b/src/main/java/org/opensearch/knn/validation/ParameterValidator.java new file mode 100644 index 000000000..15925fffa --- /dev/null +++ b/src/main/java/org/opensearch/knn/validation/ParameterValidator.java @@ -0,0 +1,64 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.knn.validation; + +import org.opensearch.common.Nullable; +import org.opensearch.common.ValidationException; +import org.opensearch.knn.index.Parameter; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +public final class ParameterValidator { + + /** + * A function which validates request parameters. + * @param validParameters A set of valid parameters that can be requestParameters can be validated against + * @param requestParameters parameters from the request + * @return + */ + @Nullable + public static ValidationException validateParameters( + final Map> validParameters, + final Map requestParameters + ) { + + if (validParameters == null) { + throw new IllegalArgumentException("validParameters cannot be null"); + } + + if (requestParameters == null || requestParameters.isEmpty()) { + return null; + } + + final List errorMessages = new ArrayList<>(); + for (Map.Entry parameter : requestParameters.entrySet()) { + if (validParameters.containsKey(parameter.getKey())) { + final ValidationException parameterValidation = validParameters.get(parameter.getKey()).validate(parameter.getValue()); + if (parameterValidation != null) { + errorMessages.addAll(parameterValidation.validationErrors()); + } + } else { + errorMessages.add("Unknown parameter '" + parameter.getKey() + "' found"); + } + } + + if (errorMessages.isEmpty()) { + return null; + } + + final ValidationException validationException = new ValidationException(); + validationException.addValidationErrors(errorMessages); + return validationException; + } +} diff --git a/src/test/java/org/opensearch/knn/index/FaissHNSWFlatE2EIT.java b/src/test/java/org/opensearch/knn/index/FaissHNSWFlatE2EIT.java new file mode 100644 index 000000000..5b7a99ce9 --- /dev/null +++ b/src/test/java/org/opensearch/knn/index/FaissHNSWFlatE2EIT.java @@ -0,0 +1,194 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.knn.index; + +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import com.google.common.collect.ImmutableList; +import com.google.common.primitives.Floats; +import lombok.AllArgsConstructor; +import lombok.SneakyThrows; +import org.apache.http.util.EntityUtils; +import org.junit.BeforeClass; +import org.opensearch.client.Response; +import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.knn.KNNRestTestCase; +import org.opensearch.knn.KNNResult; +import org.opensearch.knn.TestUtils; +import org.opensearch.knn.common.KNNConstants; +import org.opensearch.knn.index.query.KNNQueryBuilder; +import org.opensearch.knn.index.util.KNNEngine; +import org.opensearch.knn.plugin.script.KNNScoringUtil; + +import java.io.IOException; +import java.net.URL; +import java.util.Arrays; +import java.util.Collection; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; + +import static com.carrotsearch.randomizedtesting.RandomizedTest.$; +import static com.carrotsearch.randomizedtesting.RandomizedTest.$$; +import static org.opensearch.knn.common.KNNConstants.KNN_ENGINE; +import static org.opensearch.knn.common.KNNConstants.METHOD_HNSW; +import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_EF_SEARCH; +import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_SPACE_TYPE; +import static org.opensearch.knn.common.KNNConstants.NAME; +import static org.opensearch.knn.common.KNNConstants.PARAMETERS; + +@AllArgsConstructor +public class FaissHNSWFlatE2EIT extends KNNRestTestCase { + + private String description; + private int k; + private Map methodParameters; + private boolean deleteRandomDocs; + + static TestUtils.TestData testData; + + @BeforeClass + public static void setUpClass() throws IOException { + if (FaissHNSWFlatE2EIT.class.getClassLoader() == null) { + throw new IllegalStateException("ClassLoader of FaissIT Class is null"); + } + URL testIndexVectors = FaissHNSWFlatE2EIT.class.getClassLoader().getResource("data/test_vectors_1000x128.json"); + URL testQueries = FaissHNSWFlatE2EIT.class.getClassLoader().getResource("data/test_queries_100x128.csv"); + assert testIndexVectors != null; + assert testQueries != null; + testData = new TestUtils.TestData(testIndexVectors.getPath(), testQueries.getPath()); + } + + @ParametersFactory(argumentFormatting = "description:%1$s; k:%2$s; efSearch:%3$s, deleteDocs:%4$s") + public static Collection parameters() { + return Arrays.asList( + $$( + $("Valid k, valid efSearch efSearch value", 10, Map.of(METHOD_PARAMETER_EF_SEARCH, 300), false), + $("Valid k, efsearch absent", 10, null, false), + $("Has delete docs, ef_search", 10, Map.of(METHOD_PARAMETER_EF_SEARCH, 300), true), + $("Has delete docs", 10, null, true) + ) + ); + } + + @SneakyThrows + public void testEndToEnd_whenMethodIsHNSWFlat_thenSucceed() { + String indexName = "test-index-1"; + String fieldName = "test-field-1"; + + KNNMethod hnswMethod = KNNEngine.FAISS.getMethod(METHOD_HNSW); + SpaceType spaceType = SpaceType.L2; + + List mValues = ImmutableList.of(16, 32, 64, 128); + List efConstructionValues = ImmutableList.of(16, 32, 64, 128); + List efSearchValues = ImmutableList.of(16, 32, 64, 128); + + Integer dimension = testData.indexData.vectors[0].length; + + // Create an index + XContentBuilder builder = XContentFactory.jsonBuilder() + .startObject() + .startObject("properties") + .startObject(fieldName) + .field("type", "knn_vector") + .field("dimension", dimension) + .startObject(KNNConstants.KNN_METHOD) + .field(NAME, hnswMethod.getMethodComponent().getName()) + .field(METHOD_PARAMETER_SPACE_TYPE, spaceType.getValue()) + .field(KNN_ENGINE, KNNEngine.FAISS.getName()) + .startObject(PARAMETERS) + .field(KNNConstants.METHOD_PARAMETER_M, mValues.get(random().nextInt(mValues.size()))) + .field(KNNConstants.METHOD_PARAMETER_EF_CONSTRUCTION, efConstructionValues.get(random().nextInt(efConstructionValues.size()))) + .field(KNNConstants.METHOD_PARAMETER_EF_SEARCH, efSearchValues.get(random().nextInt(efSearchValues.size()))) + .endObject() + .endObject() + .endObject() + .endObject() + .endObject(); + + Map mappingMap = xContentBuilderToMap(builder); + String mapping = builder.toString(); + + createKnnIndex(indexName, mapping); + assertEquals(new TreeMap<>(mappingMap), new TreeMap<>(getIndexMappingAsMap(indexName))); + + // Index the test data + for (int i = 0; i < testData.indexData.docs.length; i++) { + addKnnDoc( + indexName, + Integer.toString(testData.indexData.docs[i]), + fieldName, + Floats.asList(testData.indexData.vectors[i]).toArray() + ); + } + + // Assert we have the right number of documents in the index + refreshAllNonSystemIndices(); + assertEquals(testData.indexData.docs.length, getDocCount(indexName)); + + // Delete few Docs + if (deleteRandomDocs) { + final Set docIdsToBeDeleted = new HashSet<>(); + while (docIdsToBeDeleted.size() < 10) { + docIdsToBeDeleted.add(randomInt(testData.indexData.docs.length - 1)); + } + + for (Integer id : docIdsToBeDeleted) { + deleteKnnDoc(indexName, Integer.toString(testData.indexData.docs[id])); + } + refreshAllNonSystemIndices(); + forceMergeKnnIndex(indexName, 3); + + assertEquals(testData.indexData.docs.length - 10, getDocCount(indexName)); + } + + // Test search queries + for (int i = 0; i < testData.queries.length; i++) { + final KNNQueryBuilder queryBuilder = KNNQueryBuilder.builder() + .fieldName(fieldName) + .vector(testData.queries[i]) + .k(k) + .methodParameters(methodParameters) + .build(); + Response response = searchKNNIndex(indexName, queryBuilder, k); + String responseBody = EntityUtils.toString(response.getEntity()); + List knnResults = parseSearchResponse(responseBody, fieldName); + assertEquals(k, knnResults.size()); + + List actualScores = parseSearchResponseScore(responseBody, fieldName); + for (int j = 0; j < k; j++) { + float[] primitiveArray = knnResults.get(j).getVector(); + assertEquals( + KNNEngine.FAISS.score(KNNScoringUtil.l2Squared(testData.queries[i], primitiveArray), spaceType), + actualScores.get(j), + 0.0001 + ); + } + } + + // Delete index + deleteKNNIndex(indexName); + + // Search every 5 seconds 14 times to confirm graph gets evicted + int intervals = 14; + for (int i = 0; i < intervals; i++) { + if (getTotalGraphsInCache() == 0) { + return; + } + Thread.sleep(5 * 1000); + } + + fail("Graphs are not getting evicted"); + } +} diff --git a/src/test/java/org/opensearch/knn/index/FaissIT.java b/src/test/java/org/opensearch/knn/index/FaissIT.java index 3a9b7d596..2e7f772b0 100644 --- a/src/test/java/org/opensearch/knn/index/FaissIT.java +++ b/src/test/java/org/opensearch/knn/index/FaissIT.java @@ -37,12 +37,10 @@ import java.net.URL; import java.util.ArrayList; import java.util.Arrays; -import java.util.HashSet; import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Random; -import java.util.Set; import java.util.TreeMap; import java.util.stream.Collectors; @@ -90,197 +88,6 @@ public static void setUpClass() throws IOException { testData = new TestUtils.TestData(testIndexVectors.getPath(), testQueries.getPath()); } - @SneakyThrows - public void testEndToEnd_whenMethodIsHNSWFlat_thenSucceed() { - String indexName = "test-index-1"; - String fieldName = "test-field-1"; - - KNNMethod hnswMethod = KNNEngine.FAISS.getMethod(METHOD_HNSW); - SpaceType spaceType = SpaceType.L2; - - List mValues = ImmutableList.of(16, 32, 64, 128); - List efConstructionValues = ImmutableList.of(16, 32, 64, 128); - List efSearchValues = ImmutableList.of(16, 32, 64, 128); - - Integer dimension = testData.indexData.vectors[0].length; - - // Create an index - XContentBuilder builder = XContentFactory.jsonBuilder() - .startObject() - .startObject("properties") - .startObject(fieldName) - .field("type", "knn_vector") - .field("dimension", dimension) - .startObject(KNNConstants.KNN_METHOD) - .field(NAME, hnswMethod.getMethodComponent().getName()) - .field(METHOD_PARAMETER_SPACE_TYPE, spaceType.getValue()) - .field(KNN_ENGINE, KNNEngine.FAISS.getName()) - .startObject(PARAMETERS) - .field(KNNConstants.METHOD_PARAMETER_M, mValues.get(random().nextInt(mValues.size()))) - .field(KNNConstants.METHOD_PARAMETER_EF_CONSTRUCTION, efConstructionValues.get(random().nextInt(efConstructionValues.size()))) - .field(KNNConstants.METHOD_PARAMETER_EF_SEARCH, efSearchValues.get(random().nextInt(efSearchValues.size()))) - .endObject() - .endObject() - .endObject() - .endObject() - .endObject(); - - Map mappingMap = xContentBuilderToMap(builder); - String mapping = builder.toString(); - - createKnnIndex(indexName, mapping); - assertEquals(new TreeMap<>(mappingMap), new TreeMap<>(getIndexMappingAsMap(indexName))); - - // Index the test data - for (int i = 0; i < testData.indexData.docs.length; i++) { - addKnnDoc( - indexName, - Integer.toString(testData.indexData.docs[i]), - fieldName, - Floats.asList(testData.indexData.vectors[i]).toArray() - ); - } - - // Assert we have the right number of documents in the index - refreshAllNonSystemIndices(); - assertEquals(testData.indexData.docs.length, getDocCount(indexName)); - - int k = 10; - for (int i = 0; i < testData.queries.length; i++) { - Response response = searchKNNIndex(indexName, new KNNQueryBuilder(fieldName, testData.queries[i], k), k); - String responseBody = EntityUtils.toString(response.getEntity()); - List knnResults = parseSearchResponse(responseBody, fieldName); - assertEquals(k, knnResults.size()); - - List actualScores = parseSearchResponseScore(responseBody, fieldName); - for (int j = 0; j < k; j++) { - float[] primitiveArray = knnResults.get(j).getVector(); - assertEquals( - KNNEngine.FAISS.score(KNNScoringUtil.l2Squared(testData.queries[i], primitiveArray), spaceType), - actualScores.get(j), - 0.0001 - ); - } - } - - // Delete index - deleteKNNIndex(indexName); - - // Search every 5 seconds 14 times to confirm graph gets evicted - int intervals = 14; - for (int i = 0; i < intervals; i++) { - if (getTotalGraphsInCache() == 0) { - return; - } - - Thread.sleep(5 * 1000); - } - - fail("Graphs are not getting evicted"); - } - - @SneakyThrows - public void testEndToEnd_whenMethodIsHNSWFlatAndHasDeletedDocs_thenSucceed() { - String indexName = "test-index-1"; - String fieldName = "test-field-1"; - - KNNMethod hnswMethod = KNNEngine.FAISS.getMethod(METHOD_HNSW); - SpaceType spaceType = SpaceType.L2; - - List mValues = ImmutableList.of(16, 32, 64, 128); - List efConstructionValues = ImmutableList.of(16, 32, 64, 128); - List efSearchValues = ImmutableList.of(16, 32, 64, 128); - - Integer dimension = testData.indexData.vectors[0].length; - - // Create an index - XContentBuilder builder = XContentFactory.jsonBuilder() - .startObject() - .startObject("properties") - .startObject(fieldName) - .field("type", "knn_vector") - .field("dimension", dimension) - .startObject(KNNConstants.KNN_METHOD) - .field(NAME, hnswMethod.getMethodComponent().getName()) - .field(METHOD_PARAMETER_SPACE_TYPE, spaceType.getValue()) - .field(KNN_ENGINE, KNNEngine.FAISS.getName()) - .startObject(PARAMETERS) - .field(KNNConstants.METHOD_PARAMETER_M, mValues.get(random().nextInt(mValues.size()))) - .field(KNNConstants.METHOD_PARAMETER_EF_CONSTRUCTION, efConstructionValues.get(random().nextInt(efConstructionValues.size()))) - .field(KNNConstants.METHOD_PARAMETER_EF_SEARCH, efSearchValues.get(random().nextInt(efSearchValues.size()))) - .endObject() - .endObject() - .endObject() - .endObject() - .endObject(); - - Map mappingMap = xContentBuilderToMap(builder); - String mapping = builder.toString(); - - createKnnIndex(indexName, mapping); - assertEquals(new TreeMap<>(mappingMap), new TreeMap<>(getIndexMappingAsMap(indexName))); - - // Index the test data - for (int i = 0; i < testData.indexData.docs.length; i++) { - addKnnDoc( - indexName, - Integer.toString(testData.indexData.docs[i]), - fieldName, - Floats.asList(testData.indexData.vectors[i]).toArray() - ); - } - - // Assert we have the right number of documents in the index - refreshAllNonSystemIndices(); - assertEquals(testData.indexData.docs.length, getDocCount(indexName)); - - final Set docIdsToBeDeleted = new HashSet<>(); - while (docIdsToBeDeleted.size() < 10) { - docIdsToBeDeleted.add(randomInt(testData.indexData.docs.length - 1)); - } - - for (Integer id : docIdsToBeDeleted) { - deleteKnnDoc(indexName, Integer.toString(testData.indexData.docs[id])); - } - refreshAllNonSystemIndices(); - forceMergeKnnIndex(indexName, 3); - - assertEquals(testData.indexData.docs.length - 10, getDocCount(indexName)); - - int k = 10; - for (int i = 0; i < testData.queries.length; i++) { - Response response = searchKNNIndex(indexName, new KNNQueryBuilder(fieldName, testData.queries[i], k), k); - String responseBody = EntityUtils.toString(response.getEntity()); - List knnResults = parseSearchResponse(responseBody, fieldName); - assertEquals(k, knnResults.size()); - - List actualScores = parseSearchResponseScore(responseBody, fieldName); - for (int j = 0; j < k; j++) { - float[] primitiveArray = knnResults.get(j).getVector(); - assertEquals( - KNNEngine.FAISS.score(KNNScoringUtil.l2Squared(testData.queries[i], primitiveArray), spaceType), - actualScores.get(j), - 0.0001 - ); - } - } - - // Delete index - deleteKNNIndex(indexName); - - // Search every 5 seconds 14 times to confirm graph gets evicted - int intervals = 14; - for (int i = 0; i < intervals; i++) { - if (getTotalGraphsInCache() == 0) { - return; - } - - Thread.sleep(5 * 1000); - } - - fail("Graphs are not getting evicted"); - } - @SneakyThrows public void testEndToEnd_whenDoRadiusSearch_whenDistanceThreshold_whenMethodIsHNSWFlat_thenSucceed() { KNNMethod hnswMethod = KNNEngine.FAISS.getMethod(METHOD_HNSW); @@ -1447,7 +1254,7 @@ public void testDocDeletion() throws IOException { deleteKnnDoc(INDEX_NAME, "1"); } - public void testKNNQuery_withModelDifferentCombination_thenSuccess() throws IOException, InterruptedException { + public void testKNNQuery_withModelDifferentCombination_thenSuccess() throws Exception { String modelId = "test-model"; int dimension = 128; diff --git a/src/test/java/org/opensearch/knn/index/LuceneEngineIT.java b/src/test/java/org/opensearch/knn/index/LuceneEngineIT.java index bf9a6b776..38895246a 100644 --- a/src/test/java/org/opensearch/knn/index/LuceneEngineIT.java +++ b/src/test/java/org/opensearch/knn/index/LuceneEngineIT.java @@ -122,7 +122,7 @@ public void testQuery_documentsMissingField() throws Exception { validateQueries(spaceType, FIELD_NAME); } - public void testQuery_multipleEngines() throws IOException { + public void testQuery_multipleEngines() throws Exception { String luceneField = "lucene-field"; SpaceType luceneSpaceType = SpaceType.COSINESIMIL; String nmslibField = "nmslib-field"; @@ -175,7 +175,7 @@ public void testQuery_multipleEngines() throws IOException { validateQueries(nmslibSpaceType, nmslibField); } - public void testAddDoc() throws IOException { + public void testAddDoc() throws Exception { List mValues = ImmutableList.of(16, 32, 64, 128); List efConstructionValues = ImmutableList.of(16, 32, 64, 128); @@ -499,13 +499,18 @@ private void baseQueryTest(SpaceType spaceType) throws Exception { } validateQueries(spaceType, FIELD_NAME); + validateQueries(spaceType, FIELD_NAME, Map.of("ef_search", 100)); } - private void validateQueries(SpaceType spaceType, String fieldName) throws IOException { + private void validateQueries(SpaceType spaceType, String fieldName) throws Exception { + validateQueries(spaceType, fieldName, null); + } + + private void validateQueries(SpaceType spaceType, String fieldName, Map methodParameters) throws Exception { int k = LuceneEngineIT.TEST_INDEX_VECTORS.length; for (float[] queryVector : TEST_QUERY_VECTORS) { - Response response = searchKNNIndex(INDEX_NAME, new KNNQueryBuilder(fieldName, queryVector, k), k); + Response response = searchKNNIndex(INDEX_NAME, buildLuceneKSearchQuery(fieldName, k, queryVector, methodParameters), k); String responseBody = EntityUtils.toString(response.getEntity()); List knnResults = parseSearchResponse(responseBody, fieldName); assertEquals(k, knnResults.size()); @@ -520,6 +525,27 @@ private void validateQueries(SpaceType spaceType, String fieldName) throws IOExc } } + @SneakyThrows + private XContentBuilder buildLuceneKSearchQuery(String fieldName, int k, float[] vector, Map methodParams) { + XContentBuilder builder = XContentFactory.jsonBuilder() + .startObject() + .startObject("query") + .startObject("knn") + .startObject(fieldName) + .field("vector", vector) + .field("k", k); + if (methodParams != null) { + builder.startObject("method_parameters"); + for (Map.Entry entry : methodParams.entrySet()) { + builder.field(entry.getKey(), entry.getValue()); + } + builder.endObject(); + } + + builder.endObject().endObject().endObject().endObject(); + return builder; + } + private List queryResults(final float[] searchVector, final int k) throws Exception { final String responseBody = EntityUtils.toString( searchKNNIndex(INDEX_NAME, new KNNQueryBuilder(FIELD_NAME, searchVector, k), k).getEntity() diff --git a/src/test/java/org/opensearch/knn/index/NmslibIT.java b/src/test/java/org/opensearch/knn/index/NmslibIT.java index b76a26e69..22168b3e4 100644 --- a/src/test/java/org/opensearch/knn/index/NmslibIT.java +++ b/src/test/java/org/opensearch/knn/index/NmslibIT.java @@ -13,6 +13,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.primitives.Floats; +import lombok.SneakyThrows; import org.apache.http.util.EntityUtils; import org.junit.BeforeClass; import org.opensearch.client.Response; @@ -52,7 +53,77 @@ public static void setUpClass() throws IOException { testData = new TestUtils.TestData(testIndexVectors.getPath(), testQueries.getPath()); } - public void testEndToEnd() throws IOException, InterruptedException { + public void testInvalidMethodParameters() throws Exception { + String indexName = "test-index-1"; + String fieldName = "test-field-1"; + Integer dimension = testData.indexData.vectors[0].length; + KNNMethod hnswMethod = KNNEngine.NMSLIB.getMethod(KNNConstants.METHOD_HNSW); + SpaceType spaceType = SpaceType.L1; + + // Create an index + XContentBuilder builder = XContentFactory.jsonBuilder() + .startObject() + .startObject("properties") + .startObject(fieldName) + .field("type", "knn_vector") + .field("dimension", dimension) + .startObject(KNNConstants.KNN_METHOD) + .field(KNNConstants.NAME, hnswMethod.getMethodComponent().getName()) + .field(KNNConstants.METHOD_PARAMETER_SPACE_TYPE, spaceType.getValue()) + .field(KNNConstants.KNN_ENGINE, KNNEngine.NMSLIB.getName()) + .startObject(KNNConstants.PARAMETERS) + .field(KNNConstants.METHOD_PARAMETER_M, 32) + .field(KNNConstants.METHOD_PARAMETER_EF_CONSTRUCTION, 100) + .endObject() + .endObject() + .endObject() + .endObject() + .endObject(); + + final Map mappingMap = xContentBuilderToMap(builder); + String mapping = builder.toString(); + + createKnnIndex(indexName, mapping); + assertEquals(new TreeMap<>(mappingMap), new TreeMap<>(getIndexMappingAsMap(indexName))); + + // Index the test data + // Adding only doc to cut on integ test time + addKnnDoc( + indexName, + Integer.toString(testData.indexData.docs[0]), + fieldName, + Floats.asList(testData.indexData.vectors[0]).toArray() + ); + + expectThrows( + IllegalArgumentException.class, + () -> searchKNNIndex( + indexName, + KNNQueryBuilder.builder() + .k(10) + .methodParameters(Map.of("foo", "bar")) + .vector(testData.queries[0]) + .fieldName(fieldName) + .build(), + 10 + ) + ); + expectThrows( + IllegalArgumentException.class, + () -> searchKNNIndex( + indexName, + KNNQueryBuilder.builder() + .k(10) + .methodParameters(Map.of("ef_search", "bar")) + .vector(testData.queries[0]) + .fieldName(fieldName) + .build(), + 10 + ) + ); + } + + public void testEndToEnd() throws Exception { String indexName = "test-index-1"; String fieldName = "test-field-1"; @@ -104,9 +175,42 @@ public void testEndToEnd() throws IOException, InterruptedException { refreshAllIndices(); assertEquals(testData.indexData.docs.length, getDocCount(indexName)); + // search index + // without method parameters + validateSearch(indexName, fieldName, spaceType, null); + // With valid method params + validateSearch(indexName, fieldName, spaceType, Map.of("ef_search", 50)); + + // Delete index + deleteKNNIndex(indexName); + + // Search every 5 seconds 14 times to confirm graph gets evicted + int intervals = 14; + for (int i = 0; i < intervals; i++) { + if (getTotalGraphsInCache() == 0) { + return; + } + + Thread.sleep(5 * 1000); + } + + fail("Graphs are not getting evicted"); + } + + @SneakyThrows + private void validateSearch( + final String indexName, + final String fieldName, + SpaceType spaceType, + final Map methodParams + ) { int k = 10; for (int i = 0; i < testData.queries.length; i++) { - Response response = searchKNNIndex(indexName, new KNNQueryBuilder(fieldName, testData.queries[i], k), k); + Response response = searchKNNIndex( + indexName, + KNNQueryBuilder.builder().fieldName(fieldName).vector(testData.queries[i]).k(k).methodParameters(methodParams).build(), + k + ); String responseBody = EntityUtils.toString(response.getEntity()); List knnResults = parseSearchResponse(responseBody, fieldName); assertEquals(k, knnResults.size()); @@ -121,21 +225,6 @@ public void testEndToEnd() throws IOException, InterruptedException { ); } } - - // Delete index - deleteKNNIndex(indexName); - - // Search every 5 seconds 14 times to confirm graph gets evicted - int intervals = 14; - for (int i = 0; i < intervals; i++) { - if (getTotalGraphsInCache() == 0) { - return; - } - - Thread.sleep(5 * 1000); - } - - fail("Graphs are not getting evicted"); } public void testAddDoc() throws Exception { diff --git a/src/test/java/org/opensearch/knn/index/OpenSearchIT.java b/src/test/java/org/opensearch/knn/index/OpenSearchIT.java index d6ecdb0c6..a751754fd 100644 --- a/src/test/java/org/opensearch/knn/index/OpenSearchIT.java +++ b/src/test/java/org/opensearch/knn/index/OpenSearchIT.java @@ -58,7 +58,7 @@ public static void setUpClass() throws IOException { testData = new TestUtils.TestData(testIndexVectors.getPath(), testQueries.getPath()); } - public void testEndToEnd() throws IOException, InterruptedException { + public void testEndToEnd() throws Exception { String indexName = "test-index-1"; KNNEngine knnEngine1 = KNNEngine.NMSLIB; KNNEngine knnEngine2 = KNNEngine.FAISS; diff --git a/src/test/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumerTests.java b/src/test/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumerTests.java index 2ce3a7c83..f7c9f3eb8 100644 --- a/src/test/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumerTests.java +++ b/src/test/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumerTests.java @@ -59,6 +59,7 @@ import static org.opensearch.knn.common.KNNConstants.INDEX_DESCRIPTION_PARAMETER; import static org.opensearch.knn.common.KNNConstants.METHOD_HNSW; import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_EF_CONSTRUCTION; +import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_EF_SEARCH; import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_M; import static org.opensearch.knn.common.KNNConstants.MODEL_ID; import static org.opensearch.knn.index.KNNSettings.MODEL_CACHE_SIZE_LIMIT_SETTING; @@ -70,6 +71,9 @@ public class KNN80DocValuesConsumerTests extends KNNTestCase { + private static final int EF_SEARCH = 10; + private static final Map HNSW_METHODPARAMETERS = Map.of(METHOD_PARAMETER_EF_SEARCH, EF_SEARCH); + private static Directory directory; private static Codec codec; @@ -202,7 +206,7 @@ public void testAddKNNBinaryField_fromScratch_nmslibCurrent() throws IOException assertValidFooter(state.directory, expectedFile); // The document should be readable by nmslib - assertLoadableByEngine(state, expectedFile, knnEngine, spaceType, dimension); + assertLoadableByEngine(null, state, expectedFile, knnEngine, spaceType, dimension); // The graph creation statistics should be updated assertEquals(1 + initialRefreshOperations, (long) KNNGraphValue.REFRESH_TOTAL_OPERATIONS.getValue()); @@ -255,7 +259,7 @@ public void testAddKNNBinaryField_fromScratch_nmslibLegacy() throws IOException assertValidFooter(state.directory, expectedFile); // The document should be readable by nmslib - assertLoadableByEngine(state, expectedFile, knnEngine, spaceType, dimension); + assertLoadableByEngine(null, state, expectedFile, knnEngine, spaceType, dimension); // The graph creation statistics should be updated assertEquals(1 + initialRefreshOperations, (long) KNNGraphValue.REFRESH_TOTAL_OPERATIONS.getValue()); @@ -316,7 +320,7 @@ public void testAddKNNBinaryField_fromScratch_faissCurrent() throws IOException assertValidFooter(state.directory, expectedFile); // The document should be readable by faiss - assertLoadableByEngine(state, expectedFile, knnEngine, spaceType, dimension); + assertLoadableByEngine(HNSW_METHODPARAMETERS, state, expectedFile, knnEngine, spaceType, dimension); // The graph creation statistics should be updated assertEquals(1 + initialRefreshOperations, (long) KNNGraphValue.REFRESH_TOTAL_OPERATIONS.getValue()); @@ -411,7 +415,7 @@ public void testAddKNNBinaryField_fromModel_faiss() throws IOException, Executio assertValidFooter(state.directory, expectedFile); // The document should be readable by faiss - assertLoadableByEngine(state, expectedFile, knnEngine, spaceType, dimension); + assertLoadableByEngine(HNSW_METHODPARAMETERS, state, expectedFile, knnEngine, spaceType, dimension); // The graph creation statistics should be updated assertEquals(1 + initialRefreshOperations, (long) KNNGraphValue.REFRESH_TOTAL_OPERATIONS.getValue()); diff --git a/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestUtil.java b/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestUtil.java index c4d50ec27..80e94caf8 100644 --- a/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestUtil.java +++ b/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestUtil.java @@ -326,6 +326,7 @@ public static void assertValidFooter(Directory dir, String filename) throws IOEx } public static void assertLoadableByEngine( + Map methodParameters, SegmentWriteState state, String fileName, KNNEngine knnEngine, @@ -337,7 +338,7 @@ public static void assertLoadableByEngine( long indexPtr = JNIService.loadIndex(filePath, Maps.newHashMap(ImmutableMap.of(SPACE_TYPE, spaceType.getValue())), knnEngine); int k = 2; float[] queryVector = new float[dimension]; - KNNQueryResult[] results = JNIService.queryIndex(indexPtr, queryVector, k, knnEngine, null, 0, null); + KNNQueryResult[] results = JNIService.queryIndex(indexPtr, queryVector, k, methodParameters, knnEngine, null, 0, null); assertTrue(results.length > 0); JNIService.free(indexPtr, knnEngine); } diff --git a/src/test/java/org/opensearch/knn/index/memory/NativeMemoryLoadStrategyTests.java b/src/test/java/org/opensearch/knn/index/memory/NativeMemoryLoadStrategyTests.java index a84974202..876303523 100644 --- a/src/test/java/org/opensearch/knn/index/memory/NativeMemoryLoadStrategyTests.java +++ b/src/test/java/org/opensearch/knn/index/memory/NativeMemoryLoadStrategyTests.java @@ -76,7 +76,7 @@ public void testIndexLoadStrategy_load() throws IOException { // Confirm that the file was loaded by querying float[] query = new float[dimension]; Arrays.fill(query, numVectors + 1); - KNNQueryResult[] results = JNIService.queryIndex(indexAllocation.getMemoryAddress(), query, 2, knnEngine, null, 0, null); + KNNQueryResult[] results = JNIService.queryIndex(indexAllocation.getMemoryAddress(), query, 2, null, knnEngine, null, 0, null); assertTrue(results.length > 0); } diff --git a/src/test/java/org/opensearch/knn/index/query/KNNQueryBuilderInvalidParamsTests.java b/src/test/java/org/opensearch/knn/index/query/KNNQueryBuilderInvalidParamsTests.java new file mode 100644 index 000000000..74c1cca58 --- /dev/null +++ b/src/test/java/org/opensearch/knn/index/query/KNNQueryBuilderInvalidParamsTests.java @@ -0,0 +1,98 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.query; + +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import lombok.AllArgsConstructor; +import org.opensearch.knn.KNNTestCase; + +import java.util.Arrays; +import java.util.Collection; +import java.util.Map; + +import static com.carrotsearch.randomizedtesting.RandomizedTest.$; +import static com.carrotsearch.randomizedtesting.RandomizedTest.$$; + +@AllArgsConstructor +public class KNNQueryBuilderInvalidParamsTests extends KNNTestCase { + + private static final float[] QUERY_VECTOR = new float[] { 1.2f, 2.3f, 4.5f }; + private static final String FIELD_NAME = "test_vector"; + + private String description; + private String expectedMessage; + private KNNQueryBuilder.Builder knnQueryBuilderBuilder; + + @ParametersFactory(argumentFormatting = "description:%1$s; expectedMessage:%2$s; querybuilder:%3$s") + public static Collection invalidParameters() { + return Arrays.asList( + $$( + $("fieldName absent", "[knn] requires fieldName", KNNQueryBuilder.builder().k(1).vector(QUERY_VECTOR)), + $("vector absent", "[knn] requires query vector", KNNQueryBuilder.builder().k(1).fieldName(FIELD_NAME)), + $( + "vector empty", + "[knn] query vector is empty", + KNNQueryBuilder.builder().k(1).fieldName(FIELD_NAME).vector(new float[] {}) + ), + $( + "Neither knn nor radial search", + "[knn] requires exactly one of k, distance or score to be set", + KNNQueryBuilder.builder().fieldName(FIELD_NAME).vector(QUERY_VECTOR) + ), + $( + "max distance and k present", + "[knn] requires exactly one of k, distance or score to be set", + KNNQueryBuilder.builder().fieldName(FIELD_NAME).vector(QUERY_VECTOR).k(1).maxDistance(10f) + ), + $( + "min_score and k present", + "[knn] requires exactly one of k, distance or score to be set", + KNNQueryBuilder.builder().fieldName(FIELD_NAME).vector(QUERY_VECTOR).k(1).minScore(1.0f) + ), + $( + "max_dist and min_score present", + "[knn] requires exactly one of k, distance or score to be set", + KNNQueryBuilder.builder().fieldName(FIELD_NAME).vector(QUERY_VECTOR).maxDistance(1.0f).minScore(1.0f) + ), + $( + "max_dist, k and min_score present", + "[knn] requires exactly one of k, distance or score to be set", + KNNQueryBuilder.builder().fieldName(FIELD_NAME).vector(QUERY_VECTOR).k(1).maxDistance(1.0f).minScore(1.0f) + ), + $( + "-ve k value", + "[knn] requires k to be in the range (0, 10000]", + KNNQueryBuilder.builder().fieldName(FIELD_NAME).vector(QUERY_VECTOR).k(-1) + ), + $( + "k value greater than max", + "[knn] requires k to be in the range (0, 10000]", + KNNQueryBuilder.builder().fieldName(FIELD_NAME).vector(QUERY_VECTOR).k(10001) + ), + $( + "efSearch 0", + "[knn] errors in method parameter [Validation Failed: 1: Validation Failed: 1: ef_search should be greater than 0;;]", + KNNQueryBuilder.builder().fieldName(FIELD_NAME).vector(QUERY_VECTOR).methodParameters(Map.of("ef_search", 0)).k(10) + ), + $( + "efSearch -ve", + "[knn] errors in method parameter [Validation Failed: 1: Validation Failed: 1: ef_search should be greater than 0;;]", + KNNQueryBuilder.builder().fieldName(FIELD_NAME).vector(QUERY_VECTOR).methodParameters(Map.of("ef_search", -10)).k(10) + ), + $( + "min score less than 0", + "[knn] requires minScore to be greater than 0", + KNNQueryBuilder.builder().fieldName(FIELD_NAME).vector(QUERY_VECTOR).minScore(-1f) + ) + ) + ); + } + + public void testInvalidBuilder() { + Throwable exception = expectThrows(IllegalArgumentException.class, () -> knnQueryBuilderBuilder.build()); + assertEquals(expectedMessage, expectedMessage, exception.getMessage()); + } +} diff --git a/src/test/java/org/opensearch/knn/index/query/KNNQueryBuilderTests.java b/src/test/java/org/opensearch/knn/index/query/KNNQueryBuilderTests.java index 4b9872131..10c0155ae 100644 --- a/src/test/java/org/opensearch/knn/index/query/KNNQueryBuilderTests.java +++ b/src/test/java/org/opensearch/knn/index/query/KNNQueryBuilderTests.java @@ -7,7 +7,6 @@ import com.google.common.collect.ImmutableMap; import org.apache.lucene.search.FloatVectorSimilarityQuery; -import java.util.Locale; import org.apache.lucene.search.KnnFloatVectorQuery; import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.Query; @@ -15,21 +14,21 @@ import org.opensearch.cluster.ClusterModule; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.common.io.stream.NamedWriteableAwareStreamInput; import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.index.Index; import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; import org.opensearch.index.IndexSettings; +import org.opensearch.index.mapper.NumberFieldMapper; import org.opensearch.index.query.QueryBuilder; import org.opensearch.index.query.QueryBuilders; +import org.opensearch.index.query.QueryShardContext; import org.opensearch.index.query.TermQueryBuilder; import org.opensearch.knn.KNNTestCase; -import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentFactory; -import org.opensearch.core.xcontent.XContentParser; -import org.opensearch.core.index.Index; -import org.opensearch.index.mapper.NumberFieldMapper; -import org.opensearch.index.query.QueryShardContext; import org.opensearch.knn.index.KNNClusterUtil; import org.opensearch.knn.index.KNNMethodContext; import org.opensearch.knn.index.MethodComponentContext; @@ -46,22 +45,26 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.Locale; +import java.util.Map; import java.util.Optional; import java.util.stream.Collectors; +import static java.util.Collections.emptyMap; import static org.hamcrest.Matchers.instanceOf; import static org.mockito.Mockito.anyString; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -import static org.opensearch.knn.common.KNNConstants.DEFAULT_LUCENE_RADIAL_SEARCH_TRAVERSAL_SIMILARITY_RATIO; -import static org.opensearch.knn.common.KNNConstants.METHOD_HNSW; import static org.opensearch.knn.index.KNNClusterTestUtils.mockClusterService; +import static org.opensearch.knn.index.query.KNNQueryBuilder.EF_SEARCH_FIELD; import static org.opensearch.knn.index.util.KNNEngine.ENGINES_SUPPORTING_RADIAL_SEARCH; public class KNNQueryBuilderTests extends KNNTestCase { private static final String FIELD_NAME = "myvector"; private static final int K = 1; + private static final int EF_SEARCH = 10; + private static final Map HNSW_METHOD_PARAMS = Map.of("ef_search", EF_SEARCH); private static final Float MAX_DISTANCE = 1.0f; private static final Float MIN_SCORE = 0.5f; private static final TermQueryBuilder TERM_QUERY = QueryBuilders.termQuery("field", "value"); @@ -91,7 +94,10 @@ public void testInvalidDistance() { /** * null distance */ - expectThrows(IllegalArgumentException.class, () -> new KNNQueryBuilder(FIELD_NAME, queryVector).maxDistance(null)); + expectThrows( + IllegalArgumentException.class, + () -> KNNQueryBuilder.builder().fieldName(FIELD_NAME).vector(queryVector).maxDistance(null).build() + ); } public void testInvalidScore() { @@ -99,17 +105,26 @@ public void testInvalidScore() { /** * null min_score */ - expectThrows(IllegalArgumentException.class, () -> new KNNQueryBuilder(FIELD_NAME, queryVector).minScore(null)); + expectThrows( + IllegalArgumentException.class, + () -> KNNQueryBuilder.builder().fieldName(FIELD_NAME).vector(queryVector).minScore(null).build() + ); /** * negative min_score */ - expectThrows(IllegalArgumentException.class, () -> new KNNQueryBuilder(FIELD_NAME, queryVector).minScore(-1.0f)); + expectThrows( + IllegalArgumentException.class, + () -> KNNQueryBuilder.builder().fieldName(FIELD_NAME).vector(queryVector).minScore(-1.0f).build() + ); /** * min_score = 0 */ - expectThrows(IllegalArgumentException.class, () -> new KNNQueryBuilder(FIELD_NAME, queryVector).minScore(0.0f)); + expectThrows( + IllegalArgumentException.class, + () -> KNNQueryBuilder.builder().fieldName(FIELD_NAME).vector(queryVector).minScore(0.0f).build() + ); } public void testEmptyVector() { @@ -129,13 +144,19 @@ public void testEmptyVector() { * null query vector with distance */ float[] queryVector2 = null; - expectThrows(IllegalArgumentException.class, () -> new KNNQueryBuilder(FIELD_NAME, queryVector2).maxDistance(MAX_DISTANCE)); + expectThrows( + IllegalArgumentException.class, + () -> KNNQueryBuilder.builder().fieldName(FIELD_NAME).vector(queryVector2).maxDistance(MAX_DISTANCE).build() + ); /** * empty query vector with distance */ float[] queryVector3 = {}; - expectThrows(IllegalArgumentException.class, () -> new KNNQueryBuilder(FIELD_NAME, queryVector3).maxDistance(MAX_DISTANCE)); + expectThrows( + IllegalArgumentException.class, + () -> KNNQueryBuilder.builder().fieldName(FIELD_NAME).vector(queryVector3).maxDistance(MAX_DISTANCE).build() + ); } public void testFromXContent() throws Exception { @@ -154,9 +175,37 @@ public void testFromXContent() throws Exception { assertEquals(knnQueryBuilder, actualBuilder); } + public void testFromXContent_KnnWithMethodParameters() throws Exception { + float[] queryVector = { 1.0f, 2.0f, 3.0f, 4.0f }; + KNNQueryBuilder knnQueryBuilder = KNNQueryBuilder.builder() + .fieldName(FIELD_NAME) + .vector(queryVector) + .k(K) + .methodParameters(HNSW_METHOD_PARAMS) + .build(); + XContentBuilder builder = XContentFactory.jsonBuilder(); + builder.startObject(); + builder.startObject(knnQueryBuilder.fieldName()); + builder.field(KNNQueryBuilder.VECTOR_FIELD.getPreferredName(), knnQueryBuilder.vector()); + builder.field(KNNQueryBuilder.K_FIELD.getPreferredName(), knnQueryBuilder.getK()); + builder.startObject(org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER); + builder.field(EF_SEARCH_FIELD.getPreferredName(), EF_SEARCH); + builder.endObject(); + builder.endObject(); + builder.endObject(); + XContentParser contentParser = createParser(builder); + contentParser.nextToken(); + KNNQueryBuilder actualBuilder = KNNQueryBuilder.fromXContent(contentParser); + assertEquals(knnQueryBuilder, actualBuilder); + } + public void testFromXContent_whenDoRadiusSearch_whenDistanceThreshold_thenSucceed() throws Exception { float[] queryVector = { 1.0f, 2.0f, 3.0f, 4.0f }; - KNNQueryBuilder knnQueryBuilder = new KNNQueryBuilder(FIELD_NAME, queryVector).maxDistance(MAX_DISTANCE); + KNNQueryBuilder knnQueryBuilder = KNNQueryBuilder.builder() + .fieldName(FIELD_NAME) + .vector(queryVector) + .maxDistance(MAX_DISTANCE) + .build(); XContentBuilder builder = XContentFactory.jsonBuilder(); builder.startObject(); builder.startObject(knnQueryBuilder.fieldName()); @@ -172,12 +221,16 @@ public void testFromXContent_whenDoRadiusSearch_whenDistanceThreshold_thenSuccee public void testFromXContent_whenDoRadiusSearch_whenScoreThreshold_thenSucceed() throws Exception { float[] queryVector = { 1.0f, 2.0f, 3.0f, 4.0f }; - KNNQueryBuilder knnQueryBuilder = new KNNQueryBuilder(FIELD_NAME, queryVector).minScore(MAX_DISTANCE); + KNNQueryBuilder knnQueryBuilder = KNNQueryBuilder.builder() + .fieldName(FIELD_NAME) + .vector(queryVector) + .minScore(MAX_DISTANCE) + .build(); XContentBuilder builder = XContentFactory.jsonBuilder(); builder.startObject(); builder.startObject(knnQueryBuilder.fieldName()); builder.field(KNNQueryBuilder.VECTOR_FIELD.getPreferredName(), knnQueryBuilder.vector()); - builder.field(KNNQueryBuilder.MAX_DISTANCE_FIELD.getPreferredName(), knnQueryBuilder.getMinScore()); + builder.field(KNNQueryBuilder.MIN_SCORE_FIELD.getPreferredName(), knnQueryBuilder.getMinScore()); builder.endObject(); builder.endObject(); XContentParser contentParser = createParser(builder); @@ -208,6 +261,37 @@ public void testFromXContent_withFilter() throws Exception { assertEquals(knnQueryBuilder, actualBuilder); } + public void testFromXContent_KnnWithEfSearch_withFilter() throws Exception { + final ClusterService clusterService = mockClusterService(Version.CURRENT); + + final KNNClusterUtil knnClusterUtil = KNNClusterUtil.instance(); + knnClusterUtil.initialize(clusterService); + + float[] queryVector = { 1.0f, 2.0f, 3.0f, 4.0f }; + KNNQueryBuilder knnQueryBuilder = KNNQueryBuilder.builder() + .fieldName(FIELD_NAME) + .vector(queryVector) + .k(K) + .filter(TERM_QUERY) + .methodParameters(HNSW_METHOD_PARAMS) + .build(); + XContentBuilder builder = XContentFactory.jsonBuilder(); + builder.startObject(); + builder.startObject(knnQueryBuilder.fieldName()); + builder.field(KNNQueryBuilder.VECTOR_FIELD.getPreferredName(), knnQueryBuilder.vector()); + builder.field(KNNQueryBuilder.K_FIELD.getPreferredName(), knnQueryBuilder.getK()); + builder.startObject(org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER); + builder.field(EF_SEARCH_FIELD.getPreferredName(), EF_SEARCH); + builder.endObject(); + builder.field(KNNQueryBuilder.FILTER_FIELD.getPreferredName(), knnQueryBuilder.getFilter()); + builder.endObject(); + builder.endObject(); + XContentParser contentParser = createParser(builder); + contentParser.nextToken(); + KNNQueryBuilder actualBuilder = KNNQueryBuilder.fromXContent(contentParser); + assertEquals(knnQueryBuilder, actualBuilder); + } + public void testFromXcontent_WithFilter_UnsupportedClusterVersion() throws Exception { final ClusterService clusterService = mockClusterService(Version.V_2_3_0); @@ -237,7 +321,13 @@ public void testFromXContent_wenDoRadiusSearch_whenDistanceThreshold_whenFilter_ knnClusterUtil.initialize(clusterService); float[] queryVector = { 1.0f, 2.0f, 3.0f, 4.0f }; - KNNQueryBuilder knnQueryBuilder = new KNNQueryBuilder(FIELD_NAME, queryVector).maxDistance(MAX_DISTANCE).filter(TERM_QUERY); + KNNQueryBuilder knnQueryBuilder = KNNQueryBuilder.builder() + .fieldName(FIELD_NAME) + .vector(queryVector) + .maxDistance(MAX_DISTANCE) + .filter(TERM_QUERY) + .build(); + XContentBuilder builder = XContentFactory.jsonBuilder(); builder.startObject(); builder.startObject(knnQueryBuilder.fieldName()); @@ -259,12 +349,17 @@ public void testFromXContent_wenDoRadiusSearch_whenScoreThreshold_whenFilter_the knnClusterUtil.initialize(clusterService); float[] queryVector = { 1.0f, 2.0f, 3.0f, 4.0f }; - KNNQueryBuilder knnQueryBuilder = new KNNQueryBuilder(FIELD_NAME, queryVector).minScore(MIN_SCORE).filter(TERM_QUERY); + KNNQueryBuilder knnQueryBuilder = KNNQueryBuilder.builder() + .fieldName(FIELD_NAME) + .vector(queryVector) + .minScore(MIN_SCORE) + .filter(TERM_QUERY) + .build(); XContentBuilder builder = XContentFactory.jsonBuilder(); builder.startObject(); builder.startObject(knnQueryBuilder.fieldName()); builder.field(KNNQueryBuilder.VECTOR_FIELD.getPreferredName(), knnQueryBuilder.vector()); - builder.field(KNNQueryBuilder.MAX_DISTANCE_FIELD.getPreferredName(), knnQueryBuilder.getMinScore()); + builder.field(KNNQueryBuilder.MIN_SCORE_FIELD.getPreferredName(), knnQueryBuilder.getMinScore()); builder.field(KNNQueryBuilder.FILTER_FIELD.getPreferredName(), knnQueryBuilder.getFilter()); builder.endObject(); builder.endObject(); @@ -406,7 +501,11 @@ public void testDoToQuery_Normal() throws Exception { public void testDoToQuery_whenNormal_whenDoRadiusSearch_whenDistanceThreshold_thenSucceed() { float[] queryVector = { 1.0f, 2.0f, 3.0f, 4.0f }; - KNNQueryBuilder knnQueryBuilder = new KNNQueryBuilder(FIELD_NAME, queryVector).maxDistance(MAX_DISTANCE); + KNNQueryBuilder knnQueryBuilder = KNNQueryBuilder.builder() + .fieldName(FIELD_NAME) + .vector(queryVector) + .maxDistance(MAX_DISTANCE) + .build(); Index dummyIndex = new Index("dummy", "dummy"); QueryShardContext mockQueryShardContext = mock(QueryShardContext.class); KNNVectorFieldMapper.KNNVectorFieldType mockKNNVectorField = mock(KNNVectorFieldMapper.KNNVectorFieldType.class); @@ -414,7 +513,10 @@ public void testDoToQuery_whenNormal_whenDoRadiusSearch_whenDistanceThreshold_th when(mockKNNVectorField.getDimension()).thenReturn(4); when(mockKNNVectorField.getVectorDataType()).thenReturn(VectorDataType.FLOAT); when(mockQueryShardContext.fieldMapper(anyString())).thenReturn(mockKNNVectorField); - MethodComponentContext methodComponentContext = new MethodComponentContext(METHOD_HNSW, ImmutableMap.of()); + MethodComponentContext methodComponentContext = new MethodComponentContext( + org.opensearch.knn.common.KNNConstants.METHOD_HNSW, + ImmutableMap.of() + ); KNNMethodContext knnMethodContext = new KNNMethodContext(KNNEngine.LUCENE, SpaceType.L2, methodComponentContext); when(mockKNNVectorField.getKnnMethodContext()).thenReturn(knnMethodContext); FloatVectorSimilarityQuery query = (FloatVectorSimilarityQuery) knnQueryBuilder.doToQuery(mockQueryShardContext); @@ -422,13 +524,19 @@ public void testDoToQuery_whenNormal_whenDoRadiusSearch_whenDistanceThreshold_th assertTrue(query.toString().contains("resultSimilarity=" + resultSimilarity)); assertTrue( - query.toString().contains("traversalSimilarity=" + DEFAULT_LUCENE_RADIAL_SEARCH_TRAVERSAL_SIMILARITY_RATIO * resultSimilarity) + query.toString() + .contains( + "traversalSimilarity=" + + org.opensearch.knn.common.KNNConstants.DEFAULT_LUCENE_RADIAL_SEARCH_TRAVERSAL_SIMILARITY_RATIO * resultSimilarity + ) ); } public void testDoToQuery_whenNormal_whenDoRadiusSearch_whenScoreThreshold_thenSucceed() { float[] queryVector = { 1.0f, 2.0f, 3.0f, 4.0f }; - KNNQueryBuilder knnQueryBuilder = new KNNQueryBuilder(FIELD_NAME, queryVector).minScore(MIN_SCORE); + + KNNQueryBuilder knnQueryBuilder = KNNQueryBuilder.builder().fieldName(FIELD_NAME).vector(queryVector).minScore(MIN_SCORE).build(); + Index dummyIndex = new Index("dummy", "dummy"); QueryShardContext mockQueryShardContext = mock(QueryShardContext.class); KNNVectorFieldMapper.KNNVectorFieldType mockKNNVectorField = mock(KNNVectorFieldMapper.KNNVectorFieldType.class); @@ -436,7 +544,10 @@ public void testDoToQuery_whenNormal_whenDoRadiusSearch_whenScoreThreshold_thenS when(mockKNNVectorField.getDimension()).thenReturn(4); when(mockKNNVectorField.getVectorDataType()).thenReturn(VectorDataType.FLOAT); when(mockQueryShardContext.fieldMapper(anyString())).thenReturn(mockKNNVectorField); - MethodComponentContext methodComponentContext = new MethodComponentContext(METHOD_HNSW, ImmutableMap.of()); + MethodComponentContext methodComponentContext = new MethodComponentContext( + org.opensearch.knn.common.KNNConstants.METHOD_HNSW, + ImmutableMap.of() + ); KNNMethodContext knnMethodContext = new KNNMethodContext(KNNEngine.LUCENE, SpaceType.L2, methodComponentContext); when(mockKNNVectorField.getKnnMethodContext()).thenReturn(knnMethodContext); FloatVectorSimilarityQuery query = (FloatVectorSimilarityQuery) knnQueryBuilder.doToQuery(mockQueryShardContext); @@ -446,7 +557,12 @@ public void testDoToQuery_whenNormal_whenDoRadiusSearch_whenScoreThreshold_thenS public void testDoToQuery_whenDoRadiusSearch_whenPassNegativeDistance_whenSupportedSpaceType_thenSucceed() { float[] queryVector = { 1.0f, 2.0f, 3.0f, 4.0f }; float negativeDistance = -1.0f; - KNNQueryBuilder knnQueryBuilder = new KNNQueryBuilder(FIELD_NAME, queryVector).maxDistance(negativeDistance); + + KNNQueryBuilder knnQueryBuilder = KNNQueryBuilder.builder() + .fieldName(FIELD_NAME) + .vector(queryVector) + .maxDistance(negativeDistance) + .build(); Index dummyIndex = new Index("dummy", "dummy"); QueryShardContext mockQueryShardContext = mock(QueryShardContext.class); KNNVectorFieldMapper.KNNVectorFieldType mockKNNVectorField = mock(KNNVectorFieldMapper.KNNVectorFieldType.class); @@ -454,7 +570,10 @@ public void testDoToQuery_whenDoRadiusSearch_whenPassNegativeDistance_whenSuppor when(mockKNNVectorField.getDimension()).thenReturn(4); when(mockKNNVectorField.getVectorDataType()).thenReturn(VectorDataType.FLOAT); when(mockQueryShardContext.fieldMapper(anyString())).thenReturn(mockKNNVectorField); - MethodComponentContext methodComponentContext = new MethodComponentContext(METHOD_HNSW, ImmutableMap.of()); + MethodComponentContext methodComponentContext = new MethodComponentContext( + org.opensearch.knn.common.KNNConstants.METHOD_HNSW, + ImmutableMap.of() + ); when(mockKNNVectorField.getKnnMethodContext()).thenReturn( new KNNMethodContext(KNNEngine.FAISS, SpaceType.INNER_PRODUCT, methodComponentContext) ); @@ -470,7 +589,12 @@ public void testDoToQuery_whenDoRadiusSearch_whenPassNegativeDistance_whenSuppor public void testDoToQuery_whenDoRadiusSearch_whenPassNegativeDistance_whenUnSupportedSpaceType_thenException() { float[] queryVector = { 1.0f, 2.0f, 3.0f, 4.0f }; float negativeDistance = -1.0f; - KNNQueryBuilder knnQueryBuilder = new KNNQueryBuilder(FIELD_NAME, queryVector).maxDistance(negativeDistance); + + KNNQueryBuilder knnQueryBuilder = KNNQueryBuilder.builder() + .fieldName(FIELD_NAME) + .vector(queryVector) + .maxDistance(negativeDistance) + .build(); Index dummyIndex = new Index("dummy", "dummy"); QueryShardContext mockQueryShardContext = mock(QueryShardContext.class); KNNVectorFieldMapper.KNNVectorFieldType mockKNNVectorField = mock(KNNVectorFieldMapper.KNNVectorFieldType.class); @@ -478,7 +602,10 @@ public void testDoToQuery_whenDoRadiusSearch_whenPassNegativeDistance_whenUnSupp when(mockKNNVectorField.getDimension()).thenReturn(4); when(mockKNNVectorField.getVectorDataType()).thenReturn(VectorDataType.FLOAT); when(mockQueryShardContext.fieldMapper(anyString())).thenReturn(mockKNNVectorField); - MethodComponentContext methodComponentContext = new MethodComponentContext(METHOD_HNSW, ImmutableMap.of()); + MethodComponentContext methodComponentContext = new MethodComponentContext( + org.opensearch.knn.common.KNNConstants.METHOD_HNSW, + ImmutableMap.of() + ); when(mockKNNVectorField.getKnnMethodContext()).thenReturn( new KNNMethodContext(KNNEngine.FAISS, SpaceType.L2, methodComponentContext) ); @@ -492,7 +619,8 @@ public void testDoToQuery_whenDoRadiusSearch_whenPassNegativeDistance_whenUnSupp public void testDoToQuery_whenDoRadiusSearch_whenPassScoreMoreThanOne_whenSupportedSpaceType_thenSucceed() { float[] queryVector = { 1.0f, 2.0f, 3.0f, 4.0f }; float score = 5f; - KNNQueryBuilder knnQueryBuilder = new KNNQueryBuilder(FIELD_NAME, queryVector).minScore(score); + + KNNQueryBuilder knnQueryBuilder = KNNQueryBuilder.builder().fieldName(FIELD_NAME).vector(queryVector).minScore(score).build(); Index dummyIndex = new Index("dummy", "dummy"); QueryShardContext mockQueryShardContext = mock(QueryShardContext.class); KNNVectorFieldMapper.KNNVectorFieldType mockKNNVectorField = mock(KNNVectorFieldMapper.KNNVectorFieldType.class); @@ -500,7 +628,10 @@ public void testDoToQuery_whenDoRadiusSearch_whenPassScoreMoreThanOne_whenSuppor when(mockKNNVectorField.getDimension()).thenReturn(4); when(mockKNNVectorField.getVectorDataType()).thenReturn(VectorDataType.FLOAT); when(mockQueryShardContext.fieldMapper(anyString())).thenReturn(mockKNNVectorField); - MethodComponentContext methodComponentContext = new MethodComponentContext(METHOD_HNSW, ImmutableMap.of()); + MethodComponentContext methodComponentContext = new MethodComponentContext( + org.opensearch.knn.common.KNNConstants.METHOD_HNSW, + ImmutableMap.of() + ); when(mockKNNVectorField.getKnnMethodContext()).thenReturn( new KNNMethodContext(KNNEngine.FAISS, SpaceType.INNER_PRODUCT, methodComponentContext) ); @@ -516,7 +647,7 @@ public void testDoToQuery_whenDoRadiusSearch_whenPassScoreMoreThanOne_whenSuppor public void testDoToQuery_whenDoRadiusSearch_whenPassScoreMoreThanOne_whenUnsupportedSpaceType_thenException() { float[] queryVector = { 1.0f, 2.0f, 3.0f, 4.0f }; float score = 5f; - KNNQueryBuilder knnQueryBuilder = new KNNQueryBuilder(FIELD_NAME, queryVector).minScore(score); + KNNQueryBuilder knnQueryBuilder = KNNQueryBuilder.builder().fieldName(FIELD_NAME).vector(queryVector).minScore(score).build(); Index dummyIndex = new Index("dummy", "dummy"); QueryShardContext mockQueryShardContext = mock(QueryShardContext.class); KNNVectorFieldMapper.KNNVectorFieldType mockKNNVectorField = mock(KNNVectorFieldMapper.KNNVectorFieldType.class); @@ -524,7 +655,10 @@ public void testDoToQuery_whenDoRadiusSearch_whenPassScoreMoreThanOne_whenUnsupp when(mockKNNVectorField.getDimension()).thenReturn(4); when(mockKNNVectorField.getVectorDataType()).thenReturn(VectorDataType.FLOAT); when(mockQueryShardContext.fieldMapper(anyString())).thenReturn(mockKNNVectorField); - MethodComponentContext methodComponentContext = new MethodComponentContext(METHOD_HNSW, ImmutableMap.of()); + MethodComponentContext methodComponentContext = new MethodComponentContext( + org.opensearch.knn.common.KNNConstants.METHOD_HNSW, + ImmutableMap.of() + ); when(mockKNNVectorField.getKnnMethodContext()).thenReturn( new KNNMethodContext(KNNEngine.FAISS, SpaceType.L2, methodComponentContext) ); @@ -538,7 +672,12 @@ public void testDoToQuery_whenDoRadiusSearch_whenPassScoreMoreThanOne_whenUnsupp public void testDoToQuery_whenPassNegativeDistance_whenSupportedSpaceType_thenSucceed() { float[] queryVector = { 1.0f, 2.0f, 3.0f, 4.0f }; float negativeDistance = -1.0f; - KNNQueryBuilder knnQueryBuilder = new KNNQueryBuilder(FIELD_NAME, queryVector).maxDistance(negativeDistance); + KNNQueryBuilder knnQueryBuilder = KNNQueryBuilder.builder() + .fieldName(FIELD_NAME) + .vector(queryVector) + .maxDistance(negativeDistance) + .build(); + Index dummyIndex = new Index("dummy", "dummy"); QueryShardContext mockQueryShardContext = mock(QueryShardContext.class); KNNVectorFieldMapper.KNNVectorFieldType mockKNNVectorField = mock(KNNVectorFieldMapper.KNNVectorFieldType.class); @@ -546,7 +685,10 @@ public void testDoToQuery_whenPassNegativeDistance_whenSupportedSpaceType_thenSu when(mockKNNVectorField.getDimension()).thenReturn(4); when(mockKNNVectorField.getVectorDataType()).thenReturn(VectorDataType.FLOAT); when(mockQueryShardContext.fieldMapper(anyString())).thenReturn(mockKNNVectorField); - MethodComponentContext methodComponentContext = new MethodComponentContext(METHOD_HNSW, ImmutableMap.of()); + MethodComponentContext methodComponentContext = new MethodComponentContext( + org.opensearch.knn.common.KNNConstants.METHOD_HNSW, + ImmutableMap.of() + ); when(mockKNNVectorField.getKnnMethodContext()).thenReturn( new KNNMethodContext(KNNEngine.FAISS, SpaceType.INNER_PRODUCT, methodComponentContext) ); @@ -562,7 +704,13 @@ public void testDoToQuery_whenPassNegativeDistance_whenSupportedSpaceType_thenSu public void testDoToQuery_whenPassNegativeDistance_whenUnSupportedSpaceType_thenException() { float[] queryVector = { 1.0f, 2.0f, 3.0f, 4.0f }; float negativeDistance = -1.0f; - KNNQueryBuilder knnQueryBuilder = new KNNQueryBuilder(FIELD_NAME, queryVector).maxDistance(negativeDistance); + + KNNQueryBuilder knnQueryBuilder = KNNQueryBuilder.builder() + .fieldName(FIELD_NAME) + .vector(queryVector) + .maxDistance(negativeDistance) + .build(); + Index dummyIndex = new Index("dummy", "dummy"); QueryShardContext mockQueryShardContext = mock(QueryShardContext.class); KNNVectorFieldMapper.KNNVectorFieldType mockKNNVectorField = mock(KNNVectorFieldMapper.KNNVectorFieldType.class); @@ -570,7 +718,10 @@ public void testDoToQuery_whenPassNegativeDistance_whenUnSupportedSpaceType_then when(mockKNNVectorField.getDimension()).thenReturn(4); when(mockKNNVectorField.getVectorDataType()).thenReturn(VectorDataType.FLOAT); when(mockQueryShardContext.fieldMapper(anyString())).thenReturn(mockKNNVectorField); - MethodComponentContext methodComponentContext = new MethodComponentContext(METHOD_HNSW, ImmutableMap.of()); + MethodComponentContext methodComponentContext = new MethodComponentContext( + org.opensearch.knn.common.KNNConstants.METHOD_HNSW, + ImmutableMap.of() + ); when(mockKNNVectorField.getKnnMethodContext()).thenReturn( new KNNMethodContext(KNNEngine.FAISS, SpaceType.L2, methodComponentContext) ); @@ -581,9 +732,15 @@ public void testDoToQuery_whenPassNegativeDistance_whenUnSupportedSpaceType_then expectThrows(IllegalArgumentException.class, () -> knnQueryBuilder.doToQuery(mockQueryShardContext)); } - public void testDoToQuery_KnnQueryWithFilter() throws Exception { + public void testDoToQuery_KnnQueryWithFilter_Lucene() throws Exception { + // Given float[] queryVector = { 1.0f, 2.0f, 3.0f, 4.0f }; - KNNQueryBuilder knnQueryBuilder = new KNNQueryBuilder(FIELD_NAME, queryVector, K, TERM_QUERY); + KNNQueryBuilder knnQueryBuilder = KNNQueryBuilder.builder() + .fieldName(FIELD_NAME) + .vector(queryVector) + .k(K) + .filter(TERM_QUERY) + .build(); Index dummyIndex = new Index("dummy", "dummy"); QueryShardContext mockQueryShardContext = mock(QueryShardContext.class); KNNVectorFieldMapper.KNNVectorFieldType mockKNNVectorField = mock(KNNVectorFieldMapper.KNNVectorFieldType.class); @@ -591,25 +748,42 @@ public void testDoToQuery_KnnQueryWithFilter() throws Exception { when(mockKNNVectorField.getDimension()).thenReturn(4); when(mockKNNVectorField.getVectorDataType()).thenReturn(VectorDataType.FLOAT); when(mockKNNVectorField.getSpaceType()).thenReturn(SpaceType.L2); - MethodComponentContext methodComponentContext = new MethodComponentContext(METHOD_HNSW, ImmutableMap.of()); + MethodComponentContext methodComponentContext = new MethodComponentContext( + org.opensearch.knn.common.KNNConstants.METHOD_HNSW, + ImmutableMap.of() + ); KNNMethodContext knnMethodContext = new KNNMethodContext(KNNEngine.LUCENE, SpaceType.L2, methodComponentContext); when(mockKNNVectorField.getKnnMethodContext()).thenReturn(knnMethodContext); when(mockQueryShardContext.fieldMapper(anyString())).thenReturn(mockKNNVectorField); + + // When Query query = knnQueryBuilder.doToQuery(mockQueryShardContext); + + // Then assertNotNull(query); assertTrue(query.getClass().isAssignableFrom(KnnFloatVectorQuery.class)); } public void testDoToQuery_whenDoRadiusSearch_whenDistanceThreshold_whenFilter_thenSucceed() { float[] queryVector = { 1.0f, 2.0f, 3.0f, 4.0f }; - KNNQueryBuilder knnQueryBuilder = new KNNQueryBuilder(FIELD_NAME, queryVector).maxDistance(MAX_DISTANCE).filter(TERM_QUERY); + + KNNQueryBuilder knnQueryBuilder = KNNQueryBuilder.builder() + .fieldName(FIELD_NAME) + .vector(queryVector) + .maxDistance(MAX_DISTANCE) + .filter(TERM_QUERY) + .build(); + Index dummyIndex = new Index("dummy", "dummy"); QueryShardContext mockQueryShardContext = mock(QueryShardContext.class); KNNVectorFieldMapper.KNNVectorFieldType mockKNNVectorField = mock(KNNVectorFieldMapper.KNNVectorFieldType.class); when(mockQueryShardContext.index()).thenReturn(dummyIndex); when(mockKNNVectorField.getDimension()).thenReturn(4); when(mockKNNVectorField.getVectorDataType()).thenReturn(VectorDataType.FLOAT); - MethodComponentContext methodComponentContext = new MethodComponentContext(METHOD_HNSW, ImmutableMap.of()); + MethodComponentContext methodComponentContext = new MethodComponentContext( + org.opensearch.knn.common.KNNConstants.METHOD_HNSW, + ImmutableMap.of() + ); KNNMethodContext knnMethodContext = new KNNMethodContext(KNNEngine.LUCENE, SpaceType.L2, methodComponentContext); when(mockKNNVectorField.getKnnMethodContext()).thenReturn(knnMethodContext); when(mockQueryShardContext.fieldMapper(anyString())).thenReturn(mockKNNVectorField); @@ -620,14 +794,22 @@ public void testDoToQuery_whenDoRadiusSearch_whenDistanceThreshold_whenFilter_th public void testDoToQuery_whenDoRadiusSearch_whenScoreThreshold_whenFilter_thenSucceed() { float[] queryVector = { 1.0f, 2.0f, 3.0f, 4.0f }; - KNNQueryBuilder knnQueryBuilder = new KNNQueryBuilder(FIELD_NAME, queryVector).minScore(MIN_SCORE).filter(TERM_QUERY); + KNNQueryBuilder knnQueryBuilder = KNNQueryBuilder.builder() + .fieldName(FIELD_NAME) + .vector(queryVector) + .maxDistance(MAX_DISTANCE) + .filter(TERM_QUERY) + .build(); Index dummyIndex = new Index("dummy", "dummy"); QueryShardContext mockQueryShardContext = mock(QueryShardContext.class); KNNVectorFieldMapper.KNNVectorFieldType mockKNNVectorField = mock(KNNVectorFieldMapper.KNNVectorFieldType.class); when(mockQueryShardContext.index()).thenReturn(dummyIndex); when(mockKNNVectorField.getDimension()).thenReturn(4); when(mockKNNVectorField.getVectorDataType()).thenReturn(VectorDataType.FLOAT); - MethodComponentContext methodComponentContext = new MethodComponentContext(METHOD_HNSW, ImmutableMap.of()); + MethodComponentContext methodComponentContext = new MethodComponentContext( + org.opensearch.knn.common.KNNConstants.METHOD_HNSW, + ImmutableMap.of() + ); KNNMethodContext knnMethodContext = new KNNMethodContext(KNNEngine.LUCENE, SpaceType.L2, methodComponentContext); when(mockKNNVectorField.getKnnMethodContext()).thenReturn(knnMethodContext); when(mockQueryShardContext.fieldMapper(anyString())).thenReturn(mockKNNVectorField); @@ -637,23 +819,61 @@ public void testDoToQuery_whenDoRadiusSearch_whenScoreThreshold_whenFilter_thenS } public void testDoToQuery_WhenknnQueryWithFilterAndFaissEngine_thenSuccess() { + // Given float[] queryVector = { 1.0f, 2.0f, 3.0f, 4.0f }; - KNNQueryBuilder knnQueryBuilder = new KNNQueryBuilder(FIELD_NAME, queryVector, K, TERM_QUERY); Index dummyIndex = new Index("dummy", "dummy"); QueryShardContext mockQueryShardContext = mock(QueryShardContext.class); KNNVectorFieldMapper.KNNVectorFieldType mockKNNVectorField = mock(KNNVectorFieldMapper.KNNVectorFieldType.class); when(mockQueryShardContext.index()).thenReturn(dummyIndex); when(mockKNNVectorField.getDimension()).thenReturn(4); when(mockKNNVectorField.getSpaceType()).thenReturn(SpaceType.L2); - MethodComponentContext methodComponentContext = new MethodComponentContext(METHOD_HNSW, ImmutableMap.of()); + MethodComponentContext methodComponentContext = new MethodComponentContext( + org.opensearch.knn.common.KNNConstants.METHOD_HNSW, + ImmutableMap.of() + ); KNNMethodContext knnMethodContext = new KNNMethodContext(KNNEngine.FAISS, SpaceType.L2, methodComponentContext); when(mockKNNVectorField.getKnnMethodContext()).thenReturn(knnMethodContext); when(mockQueryShardContext.fieldMapper(anyString())).thenReturn(mockKNNVectorField); + + // When + KNNQueryBuilder knnQueryBuilder = KNNQueryBuilder.builder() + .fieldName(FIELD_NAME) + .vector(queryVector) + .k(K) + .filter(TERM_QUERY) + .methodParameters(HNSW_METHOD_PARAMS) + .build(); + Query query = knnQueryBuilder.doToQuery(mockQueryShardContext); + + // Then assertNotNull(query); assertTrue(query.getClass().isAssignableFrom(KNNQuery.class)); + assertEquals(HNSW_METHOD_PARAMS, ((KNNQuery) query).getMethodParameters()); } + /** This test should be uncommented once we have nprobs. Considering engine instance is static its not possible to test this right now + public void testDoToQuery_ThrowsIllegalArgumentExceptionForUnknownMethodParameter() { + + QueryShardContext mockQueryShardContext = mock(QueryShardContext.class); + KNNVectorFieldMapper.KNNVectorFieldType mockKNNVectorField = mock(KNNVectorFieldMapper.KNNVectorFieldType.class); + when(mockQueryShardContext.fieldMapper(anyString())).thenReturn(mockKNNVectorField); + when(mockKNNVectorField.getDimension()).thenReturn(4); + when(mockKNNVectorField.getKnnMethodContext()).thenReturn( + new KNNMethodContext(KNNEngine.LUCENE, SpaceType.COSINESIMIL, new MethodComponentContext("hnsw", Map.of())) + ); + + float[] queryVector = { 1.0f, 2.0f, 3.0f, 4.0f }; + KNNQueryBuilder knnQueryBuilder = KNNQueryBuilder.builder() + .fieldName(FIELD_NAME) + .vector(queryVector) + .k(K) + .methodParameters(Map.of("ef_search", 10)) + .build(); + + expectThrows(IllegalArgumentException.class, () -> knnQueryBuilder.doToQuery(mockQueryShardContext)); + }**/ + public void testDoToQuery_whenknnQueryWithFilterAndNmsLibEngine_thenException() { float[] queryVector = { 1.0f, 2.0f, 3.0f, 4.0f }; KNNQueryBuilder knnQueryBuilder = new KNNQueryBuilder(FIELD_NAME, queryVector, K, TERM_QUERY); @@ -663,7 +883,10 @@ public void testDoToQuery_whenknnQueryWithFilterAndNmsLibEngine_thenException() when(mockQueryShardContext.index()).thenReturn(dummyIndex); when(mockKNNVectorField.getDimension()).thenReturn(4); when(mockKNNVectorField.getSpaceType()).thenReturn(SpaceType.L2); - MethodComponentContext methodComponentContext = new MethodComponentContext(METHOD_HNSW, ImmutableMap.of()); + MethodComponentContext methodComponentContext = new MethodComponentContext( + org.opensearch.knn.common.KNNConstants.METHOD_HNSW, + ImmutableMap.of() + ); KNNMethodContext knnMethodContext = new KNNMethodContext(KNNEngine.NMSLIB, SpaceType.L2, methodComponentContext); when(mockKNNVectorField.getKnnMethodContext()).thenReturn(knnMethodContext); when(mockQueryShardContext.fieldMapper(anyString())).thenReturn(mockKNNVectorField); @@ -691,6 +914,7 @@ public void testDoToQuery_FromModel() { when(modelMetadata.getKnnEngine()).thenReturn(KNNEngine.FAISS); when(modelMetadata.getSpaceType()).thenReturn(SpaceType.COSINESIMIL); when(modelMetadata.getState()).thenReturn(ModelState.CREATED); + when(modelMetadata.getMethodComponentContext()).thenReturn(new MethodComponentContext("ivf", emptyMap())); ModelDao modelDao = mock(ModelDao.class); when(modelDao.getMetadata(modelId)).thenReturn(modelMetadata); KNNQueryBuilder.initialize(modelDao); @@ -704,7 +928,13 @@ public void testDoToQuery_FromModel() { public void testDoToQuery_whenFromModel_whenDoRadiusSearch_whenDistanceThreshold_thenSucceed() { float[] queryVector = { 1.0f, 2.0f, 3.0f, 4.0f }; - KNNQueryBuilder knnQueryBuilder = new KNNQueryBuilder(FIELD_NAME, queryVector).maxDistance(MAX_DISTANCE); + + KNNQueryBuilder knnQueryBuilder = KNNQueryBuilder.builder() + .fieldName(FIELD_NAME) + .vector(queryVector) + .maxDistance(MAX_DISTANCE) + .build(); + Index dummyIndex = new Index("dummy", "dummy"); QueryShardContext mockQueryShardContext = mock(QueryShardContext.class); KNNVectorFieldMapper.KNNVectorFieldType mockKNNVectorField = mock(KNNVectorFieldMapper.KNNVectorFieldType.class); @@ -721,6 +951,7 @@ public void testDoToQuery_whenFromModel_whenDoRadiusSearch_whenDistanceThreshold when(modelMetadata.getKnnEngine()).thenReturn(KNNEngine.FAISS); when(modelMetadata.getSpaceType()).thenReturn(SpaceType.L2); when(modelMetadata.getState()).thenReturn(ModelState.CREATED); + when(modelMetadata.getMethodComponentContext()).thenReturn(new MethodComponentContext("ivf", emptyMap())); ModelDao modelDao = mock(ModelDao.class); when(modelDao.getMetadata(modelId)).thenReturn(modelMetadata); KNNQueryBuilder.initialize(modelDao); @@ -737,7 +968,9 @@ public void testDoToQuery_whenFromModel_whenDoRadiusSearch_whenDistanceThreshold public void testDoToQuery_whenFromModel_whenDoRadiusSearch_whenScoreThreshold_thenSucceed() { float[] queryVector = { 1.0f, 2.0f, 3.0f, 4.0f }; - KNNQueryBuilder knnQueryBuilder = new KNNQueryBuilder(FIELD_NAME, queryVector).minScore(MIN_SCORE); + + KNNQueryBuilder knnQueryBuilder = KNNQueryBuilder.builder().fieldName(FIELD_NAME).vector(queryVector).minScore(MIN_SCORE).build(); + Index dummyIndex = new Index("dummy", "dummy"); QueryShardContext mockQueryShardContext = mock(QueryShardContext.class); KNNVectorFieldMapper.KNNVectorFieldType mockKNNVectorField = mock(KNNVectorFieldMapper.KNNVectorFieldType.class); @@ -754,6 +987,7 @@ public void testDoToQuery_whenFromModel_whenDoRadiusSearch_whenScoreThreshold_th when(modelMetadata.getKnnEngine()).thenReturn(KNNEngine.FAISS); when(modelMetadata.getSpaceType()).thenReturn(SpaceType.L2); when(modelMetadata.getState()).thenReturn(ModelState.CREATED); + when(modelMetadata.getMethodComponentContext()).thenReturn(new MethodComponentContext("ivf", emptyMap())); ModelDao modelDao = mock(ModelDao.class); when(modelDao.getMetadata(modelId)).thenReturn(modelMetadata); KNNQueryBuilder.initialize(modelDao); @@ -837,39 +1071,38 @@ public void testDoToQuery_InvalidZeroByteVector() { public void testSerialization() throws Exception { // For k-NN search - assertSerialization(Version.CURRENT, Optional.empty(), K, null, null); - assertSerialization(Version.CURRENT, Optional.of(TERM_QUERY), K, null, null); - assertSerialization(Version.V_2_3_0, Optional.empty(), K, null, null); + assertSerialization(Version.CURRENT, Optional.empty(), K, null, null, null); + assertSerialization(Version.CURRENT, Optional.empty(), K, Map.of("ef_search", EF_SEARCH), null, null); + assertSerialization(Version.CURRENT, Optional.of(TERM_QUERY), K, Map.of("ef_search", EF_SEARCH), null, null); + assertSerialization(Version.V_2_3_0, Optional.empty(), K, Map.of("ef_search", EF_SEARCH), null, null); + assertSerialization(Version.V_2_3_0, Optional.empty(), K, null, null, null); // For distance threshold search - assertSerialization(Version.CURRENT, Optional.empty(), null, MAX_DISTANCE, null); - assertSerialization(Version.CURRENT, Optional.of(TERM_QUERY), null, MAX_DISTANCE, null); + assertSerialization(Version.CURRENT, Optional.empty(), null, null, null, MAX_DISTANCE); + assertSerialization(Version.CURRENT, Optional.of(TERM_QUERY), null, null, null, MAX_DISTANCE); // For score threshold search - assertSerialization(Version.CURRENT, Optional.empty(), null, null, MIN_SCORE); - assertSerialization(Version.CURRENT, Optional.of(TERM_QUERY), null, null, MIN_SCORE); - } - - public void testIgnoreUnmapped() throws IOException { - float[] queryVector = { 1.0f, 2.0f, 3.0f, 4.0f }; - KNNQueryBuilder knnQueryBuilder = new KNNQueryBuilder(FIELD_NAME, queryVector, K); - knnQueryBuilder.ignoreUnmapped(true); - assertTrue(knnQueryBuilder.getIgnoreUnmapped()); - Query query = knnQueryBuilder.doToQuery(mock(QueryShardContext.class)); - assertNotNull(query); - assertThat(query, instanceOf(MatchNoDocsQuery.class)); - knnQueryBuilder.ignoreUnmapped(false); - expectThrows(IllegalArgumentException.class, () -> knnQueryBuilder.doToQuery(mock(QueryShardContext.class))); + assertSerialization(Version.CURRENT, Optional.empty(), null, null, null, MIN_SCORE); + assertSerialization(Version.CURRENT, Optional.of(TERM_QUERY), null, null, null, MIN_SCORE); } private void assertSerialization( final Version version, final Optional queryBuilderOptional, Integer k, + Map methodParameters, Float distance, Float score ) throws Exception { - final KNNQueryBuilder knnQueryBuilder = getKnnQueryBuilder(queryBuilderOptional, k, distance, score); + final KNNQueryBuilder knnQueryBuilder = KNNQueryBuilder.builder() + .fieldName(FIELD_NAME) + .vector(QUERY_VECTOR) + .maxDistance(distance) + .minScore(score) + .k(k) + .methodParameters(methodParameters) + .filter(queryBuilderOptional.orElse(null)) + .build(); final ClusterService clusterService = mockClusterService(version); @@ -901,28 +1134,34 @@ private void assertSerialization( } else { assertNull(deserializedKnnQueryBuilder.getFilter()); } + assertMethodParameters(version, methodParameters, deserializedKnnQueryBuilder.getMethodParameters()); } } } - private static KNNQueryBuilder getKnnQueryBuilder(Optional queryBuilderOptional, Integer k, Float distance, Float score) { - final KNNQueryBuilder knnQueryBuilder; - if (k != null) { - knnQueryBuilder = queryBuilderOptional.isPresent() - ? new KNNQueryBuilder(FIELD_NAME, QUERY_VECTOR, k, queryBuilderOptional.get()) - : new KNNQueryBuilder(FIELD_NAME, QUERY_VECTOR, k); - } else if (distance != null) { - knnQueryBuilder = queryBuilderOptional.isPresent() - ? new KNNQueryBuilder(FIELD_NAME, QUERY_VECTOR).maxDistance(distance).filter(queryBuilderOptional.get()) - : new KNNQueryBuilder(FIELD_NAME, QUERY_VECTOR).maxDistance(distance); - } else if (score != null) { - knnQueryBuilder = queryBuilderOptional.isPresent() - ? new KNNQueryBuilder(FIELD_NAME, QUERY_VECTOR).minScore(score).filter(queryBuilderOptional.get()) - : new KNNQueryBuilder(FIELD_NAME, QUERY_VECTOR).minScore(score); - } else { - throw new IllegalArgumentException("Either k or distance must be provided"); + private void assertMethodParameters(Version version, Map expectedMethodParameters, Map actualMethodParameters) { + if (!version.onOrAfter(Version.V_2_16_0)) { + assertNull(actualMethodParameters); + } else if (expectedMethodParameters != null) { + if (version.onOrAfter(Version.V_2_16_0)) { + assertEquals(expectedMethodParameters.get("ef_search"), actualMethodParameters.get("ef_search")); + } } - return knnQueryBuilder; + } + + public void testIgnoreUnmapped() throws IOException { + float[] queryVector = { 1.0f, 2.0f, 3.0f, 4.0f }; + KNNQueryBuilder.Builder knnQueryBuilder = KNNQueryBuilder.builder() + .fieldName(FIELD_NAME) + .vector(queryVector) + .k(K) + .ignoreUnmapped(true); + assertTrue(knnQueryBuilder.build().isIgnoreUnmapped()); + Query query = knnQueryBuilder.build().doToQuery(mock(QueryShardContext.class)); + assertNotNull(query); + assertThat(query, instanceOf(MatchNoDocsQuery.class)); + knnQueryBuilder.ignoreUnmapped(false); + expectThrows(IllegalArgumentException.class, () -> knnQueryBuilder.build().doToQuery(mock(QueryShardContext.class))); } public void testRadialSearch_whenUnsupportedEngine_thenThrowException() { @@ -933,9 +1172,15 @@ public void testRadialSearch_whenUnsupportedEngine_thenThrowException() { KNNMethodContext knnMethodContext = new KNNMethodContext( knnEngine, SpaceType.L2, - new MethodComponentContext(METHOD_HNSW, ImmutableMap.of()) + new MethodComponentContext(org.opensearch.knn.common.KNNConstants.METHOD_HNSW, ImmutableMap.of()) ); - KNNQueryBuilder knnQueryBuilder = new KNNQueryBuilder(FIELD_NAME, QUERY_VECTOR).maxDistance(MAX_DISTANCE); + + KNNQueryBuilder knnQueryBuilder = KNNQueryBuilder.builder() + .fieldName(FIELD_NAME) + .vector(QUERY_VECTOR) + .maxDistance(MAX_DISTANCE) + .build(); + KNNVectorFieldMapper.KNNVectorFieldType mockKNNVectorField = mock(KNNVectorFieldMapper.KNNVectorFieldType.class); QueryShardContext mockQueryShardContext = mock(QueryShardContext.class); Index dummyIndex = new Index("dummy", "dummy"); diff --git a/src/test/java/org/opensearch/knn/index/query/KNNQueryBuilderValidParamsTests.java b/src/test/java/org/opensearch/knn/index/query/KNNQueryBuilderValidParamsTests.java new file mode 100644 index 000000000..4b97df4b4 --- /dev/null +++ b/src/test/java/org/opensearch/knn/index/query/KNNQueryBuilderValidParamsTests.java @@ -0,0 +1,90 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.query; + +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import lombok.AllArgsConstructor; +import org.opensearch.knn.KNNTestCase; + +import java.util.Arrays; +import java.util.Collection; +import java.util.Map; + +import static com.carrotsearch.randomizedtesting.RandomizedTest.$; +import static com.carrotsearch.randomizedtesting.RandomizedTest.$$; + +@AllArgsConstructor +public class KNNQueryBuilderValidParamsTests extends KNNTestCase { + + private static final float[] QUERY_VECTOR = new float[] { 1.2f, 2.3f, 4.5f }; + private static final String FIELD_NAME = "test_vector"; + + private String description; + private KNNQueryBuilder expected; + private Integer k; + private Map methodParameters; + private Float maxDistance; + private Float minScore; + + @ParametersFactory(argumentFormatting = "description:%1$s; k:%3$s, efSearch:%4$s, maxDist:%5$s, minScore:%6$s") + public static Collection validParameters() { + return Arrays.asList( + $$( + $( + "valid knn with k", + KNNQueryBuilder.builder().fieldName(FIELD_NAME).vector(QUERY_VECTOR).k(10).build(), + 10, + null, + null, + null + ), + $( + "valid knn with k and efSearch", + KNNQueryBuilder.builder() + .fieldName(FIELD_NAME) + .vector(QUERY_VECTOR) + .k(10) + .methodParameters(Map.of("ef_search", 12)) + .build(), + 10, + Map.of("ef_search", 12), + null, + null + ), + $( + "valid knn with maxDis", + KNNQueryBuilder.builder().fieldName(FIELD_NAME).vector(QUERY_VECTOR).maxDistance(10.0f).build(), + null, + null, + 10.0f, + null + ), + $( + "valid knn with minScore", + KNNQueryBuilder.builder().fieldName(FIELD_NAME).vector(QUERY_VECTOR).minScore(10.0f).build(), + null, + null, + null, + 10.0f + ) + ) + ); + } + + public void testValidBuilder() { + assertEquals( + expected, + KNNQueryBuilder.builder() + .fieldName(FIELD_NAME) + .vector(QUERY_VECTOR) + .k(k) + .methodParameters(methodParameters) + .maxDistance(maxDistance) + .minScore(minScore) + .build() + ); + } +} diff --git a/src/test/java/org/opensearch/knn/index/query/KNNQueryFactoryTests.java b/src/test/java/org/opensearch/knn/index/query/KNNQueryFactoryTests.java index 1bb17cfae..56e81d237 100644 --- a/src/test/java/org/opensearch/knn/index/query/KNNQueryFactoryTests.java +++ b/src/test/java/org/opensearch/knn/index/query/KNNQueryFactoryTests.java @@ -6,6 +6,7 @@ package org.opensearch.knn.index.query; import org.apache.lucene.index.Term; +import org.apache.lucene.search.KnnByteVectorQuery; import org.apache.lucene.search.KnnFloatVectorQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.TermQuery; @@ -27,12 +28,14 @@ import java.util.Arrays; import java.util.List; +import java.util.Map; import java.util.stream.Collectors; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; import static org.opensearch.knn.common.KNNConstants.DEFAULT_VECTOR_DATA_TYPE_FIELD; +import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_EF_SEARCH; public class KNNQueryFactoryTests extends KNNTestCase { private static final String FILTER_FILED_NAME = "foo"; @@ -45,6 +48,7 @@ public class KNNQueryFactoryTests extends KNNTestCase { private final String testIndexName = "test-index"; private final String testFieldName = "test-field"; private final int testK = 10; + private final Map methodParameters = Map.of(METHOD_PARAMETER_EF_SEARCH, 100); public void testCreateCustomKNNQuery() { for (KNNEngine knnEngine : KNNEngine.getEnginesThatCreateCustomSegmentFiles()) { @@ -82,6 +86,98 @@ public void testCreateLuceneDefaultQuery() { } } + public void testLuceneFloatVectorQuery() { + Query actualQuery1 = KNNQueryFactory.create( + BaseQueryFactory.CreateQueryRequest.builder() + .knnEngine(KNNEngine.LUCENE) + .vector(testQueryVector) + .k(testK) + .indexName(testIndexName) + .fieldName(testFieldName) + .methodParameters(methodParameters) + .vectorDataType(VectorDataType.FLOAT) + .build() + ); + + // efsearch > k + Query expectedQuery1 = new KnnFloatVectorQuery(testFieldName, testQueryVector, 100, null); + assertEquals(expectedQuery1, actualQuery1); + + // efsearch < k + actualQuery1 = KNNQueryFactory.create( + BaseQueryFactory.CreateQueryRequest.builder() + .knnEngine(KNNEngine.LUCENE) + .vector(testQueryVector) + .k(testK) + .indexName(testIndexName) + .fieldName(testFieldName) + .methodParameters(Map.of("ef_search", 1)) + .vectorDataType(VectorDataType.FLOAT) + .build() + ); + expectedQuery1 = new KnnFloatVectorQuery(testFieldName, testQueryVector, testK, null); + assertEquals(expectedQuery1, actualQuery1); + + actualQuery1 = KNNQueryFactory.create( + BaseQueryFactory.CreateQueryRequest.builder() + .knnEngine(KNNEngine.LUCENE) + .vector(testQueryVector) + .k(testK) + .indexName(testIndexName) + .fieldName(testFieldName) + .vectorDataType(VectorDataType.FLOAT) + .build() + ); + expectedQuery1 = new KnnFloatVectorQuery(testFieldName, testQueryVector, testK, null); + assertEquals(expectedQuery1, actualQuery1); + } + + public void testLuceneByteVectorQuery() { + Query actualQuery1 = KNNQueryFactory.create( + BaseQueryFactory.CreateQueryRequest.builder() + .knnEngine(KNNEngine.LUCENE) + .byteVector(testByteQueryVector) + .k(testK) + .indexName(testIndexName) + .fieldName(testFieldName) + .methodParameters(methodParameters) + .vectorDataType(VectorDataType.BYTE) + .build() + ); + + // efsearch > k + Query expectedQuery1 = new KnnByteVectorQuery(testFieldName, testByteQueryVector, 100, null); + assertEquals(expectedQuery1, actualQuery1); + + // efsearch < k + actualQuery1 = KNNQueryFactory.create( + BaseQueryFactory.CreateQueryRequest.builder() + .knnEngine(KNNEngine.LUCENE) + .byteVector(testByteQueryVector) + .k(testK) + .indexName(testIndexName) + .fieldName(testFieldName) + .methodParameters(Map.of("ef_search", 1)) + .vectorDataType(VectorDataType.BYTE) + .build() + ); + expectedQuery1 = new KnnByteVectorQuery(testFieldName, testByteQueryVector, testK, null); + assertEquals(expectedQuery1, actualQuery1); + + actualQuery1 = KNNQueryFactory.create( + BaseQueryFactory.CreateQueryRequest.builder() + .knnEngine(KNNEngine.LUCENE) + .byteVector(testByteQueryVector) + .k(testK) + .indexName(testIndexName) + .fieldName(testFieldName) + .vectorDataType(VectorDataType.BYTE) + .build() + ); + expectedQuery1 = new KnnByteVectorQuery(testFieldName, testByteQueryVector, testK, null); + assertEquals(expectedQuery1, actualQuery1); + } + public void testCreateLuceneQueryWithFilter() { List luceneDefaultQueryEngineList = Arrays.stream(KNNEngine.values()) .filter(knnEngine -> !KNNEngine.getEnginesThatCreateCustomSegmentFiles().contains(knnEngine)) @@ -106,28 +202,71 @@ public void testCreateLuceneQueryWithFilter() { } public void testCreateFaissQueryWithFilter_withValidValues_thenSuccess() { + // Given final KNNEngine knnEngine = KNNEngine.FAISS; final QueryShardContext mockQueryShardContext = mock(QueryShardContext.class); MappedFieldType testMapper = mock(MappedFieldType.class); when(mockQueryShardContext.fieldMapper(any())).thenReturn(testMapper); when(testMapper.termQuery(Mockito.any(), Mockito.eq(mockQueryShardContext))).thenReturn(FILTER_QUERY); + + final KNNQuery expectedQuery = KNNQuery.builder() + .indexName(testIndexName) + .filterQuery(FILTER_QUERY) + .field(testFieldName) + .queryVector(testQueryVector) + .k(testK) + .methodParameters(methodParameters) + .build(); + + // When final KNNQueryFactory.CreateQueryRequest createQueryRequest = KNNQueryFactory.CreateQueryRequest.builder() .knnEngine(knnEngine) .indexName(testIndexName) .fieldName(testFieldName) .vector(testQueryVector) .k(testK) + .methodParameters(methodParameters) .context(mockQueryShardContext) .filter(FILTER_QUERY_BUILDER) .build(); - final Query query = KNNQueryFactory.create(createQueryRequest); - assertTrue(query instanceof KNNQuery); - assertEquals(testIndexName, ((KNNQuery) query).getIndexName()); - assertEquals(testFieldName, ((KNNQuery) query).getField()); - assertEquals(testQueryVector, ((KNNQuery) query).getQueryVector()); - assertEquals(testK, ((KNNQuery) query).getK()); - assertEquals(FILTER_QUERY, ((KNNQuery) query).getFilterQuery()); + final Query actual = KNNQueryFactory.create(createQueryRequest); + + // Then + assertEquals(expectedQuery, actual); + } + + public void testCreateFaissQueryWithFilter_withValidValues_nullEfSearch_thenSuccess() { + // Given + final KNNEngine knnEngine = KNNEngine.FAISS; + final QueryShardContext mockQueryShardContext = mock(QueryShardContext.class); + MappedFieldType testMapper = mock(MappedFieldType.class); + when(mockQueryShardContext.fieldMapper(any())).thenReturn(testMapper); + when(testMapper.termQuery(Mockito.any(), Mockito.eq(mockQueryShardContext))).thenReturn(FILTER_QUERY); + + final KNNQuery expectedQuery = KNNQuery.builder() + .indexName(testIndexName) + .filterQuery(FILTER_QUERY) + .field(testFieldName) + .queryVector(testQueryVector) + .k(testK) + .build(); + + // When + final KNNQueryFactory.CreateQueryRequest createQueryRequest = KNNQueryFactory.CreateQueryRequest.builder() + .knnEngine(knnEngine) + .indexName(testIndexName) + .fieldName(testFieldName) + .vector(testQueryVector) + .k(testK) + .context(mockQueryShardContext) + .filter(FILTER_QUERY_BUILDER) + .build(); + + final Query actual = KNNQueryFactory.create(createQueryRequest); + + // Then + assertEquals(expectedQuery, actual); } public void testCreate_whenLuceneWithParentFilter_thenReturnDiversifyingQuery() { diff --git a/src/test/java/org/opensearch/knn/index/query/KNNWeightTests.java b/src/test/java/org/opensearch/knn/index/query/KNNWeightTests.java index 0d15b5f5f..021e3a825 100644 --- a/src/test/java/org/opensearch/knn/index/query/KNNWeightTests.java +++ b/src/test/java/org/opensearch/knn/index/query/KNNWeightTests.java @@ -67,11 +67,13 @@ import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.ArgumentMatchers.isNull; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.mockStatic; import static org.mockito.Mockito.when; import static org.opensearch.knn.KNNRestTestCase.INDEX_NAME; import static org.opensearch.knn.common.KNNConstants.KNN_ENGINE; +import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_EF_SEARCH; import static org.opensearch.knn.common.KNNConstants.MODEL_ID; import static org.opensearch.knn.common.KNNConstants.SPACE_TYPE; @@ -83,6 +85,8 @@ public class KNNWeightTests extends KNNTestCase { private static final Set SEGMENT_FILES_NMSLIB = Set.of("_0.cfe", "_0_2011_target_field.hnswc"); private static final Set SEGMENT_FILES_FAISS = Set.of("_0.cfe", "_0_2011_target_field.faissc"); private static final String CIRCUIT_BREAKER_LIMIT_100KB = "100Kb"; + private static final Integer EF_SEARCH = 10; + private static final Map HNSW_METHOD_PARAMETERS = Map.of(METHOD_PARAMETER_EF_SEARCH, EF_SEARCH); private static final Map DOC_ID_TO_SCORES = Map.of(10, 0.4f, 101, 0.05f, 100, 0.8f, 50, 0.52f); private static final Map FILTERED_DOC_ID_TO_SCORES = Map.of(101, 0.05f, 100, 0.8f, 50, 0.52f); @@ -159,7 +163,7 @@ public void testQueryScoreForFaissWithModel() { SpaceType spaceType = SpaceType.L2; final Function scoreTranslator = spaceType::scoreTranslation; final String modelId = "modelId"; - jniServiceMockedStatic.when(() -> JNIService.queryIndex(anyLong(), any(), anyInt(), any(), any(), anyInt(), any())) + jniServiceMockedStatic.when(() -> JNIService.queryIndex(anyLong(), any(), eq(K), isNull(), any(), any(), anyInt(), any())) .thenReturn(getKNNQueryResults()); final KNNQuery query = new KNNQuery(FIELD_NAME, QUERY_VECTOR, K, INDEX_NAME, (BitSetProducer) null); @@ -303,7 +307,7 @@ public void testShardWithoutFiles() { @SneakyThrows public void testEmptyQueryResults() { final KNNQueryResult[] knnQueryResults = new KNNQueryResult[] {}; - jniServiceMockedStatic.when(() -> JNIService.queryIndex(anyLong(), any(), anyInt(), any(), any(), anyInt(), any())) + jniServiceMockedStatic.when(() -> JNIService.queryIndex(anyLong(), any(), eq(K), isNull(), any(), any(), anyInt(), any())) .thenReturn(knnQueryResults); final KNNQuery query = new KNNQuery(FIELD_NAME, QUERY_VECTOR, K, INDEX_NAME, (BitSetProducer) null); @@ -346,6 +350,7 @@ public void testEmptyQueryResults() { @SneakyThrows public void testANNWithFilterQuery_whenDoingANN_thenSuccess() { + // Given int k = 3; final int[] filterDocIds = new int[] { 0, 1, 2, 3, 4, 5 }; FixedBitSet filterBitSet = new FixedBitSet(filterDocIds.length); @@ -353,7 +358,16 @@ public void testANNWithFilterQuery_whenDoingANN_thenSuccess() { filterBitSet.set(docId); } jniServiceMockedStatic.when( - () -> JNIService.queryIndex(anyLong(), any(), anyInt(), any(), eq(filterBitSet.getBits()), anyInt(), any()) + () -> JNIService.queryIndex( + anyLong(), + eq(QUERY_VECTOR), + eq(k), + eq(HNSW_METHOD_PARAMETERS), + any(), + eq(filterBitSet.getBits()), + anyInt(), + any() + ) ).thenReturn(getFilteredKNNQueryResults()); final LeafReaderContext leafReaderContext = mock(LeafReaderContext.class); final SegmentReader reader = mock(SegmentReader.class); @@ -366,7 +380,15 @@ public void testANNWithFilterQuery_whenDoingANN_thenSuccess() { when(liveDocsBits.length()).thenReturn(1000); when(leafReaderContext.reader()).thenReturn(reader); - final KNNQuery query = new KNNQuery(FIELD_NAME, QUERY_VECTOR, k, INDEX_NAME, FILTER_QUERY, null); + final KNNQuery query = KNNQuery.builder() + .field(FIELD_NAME) + .queryVector(QUERY_VECTOR) + .k(k) + .indexName(INDEX_NAME) + .filterQuery(FILTER_QUERY) + .methodParameters(HNSW_METHOD_PARAMETERS) + .build(); + final Weight filterQueryWeight = mock(Weight.class); final Scorer filterScorer = mock(Scorer.class); when(filterQueryWeight.scorer(leafReaderContext)).thenReturn(filterScorer); @@ -406,15 +428,26 @@ public void testANNWithFilterQuery_whenDoingANN_thenSuccess() { when(fieldInfos.fieldInfo(any())).thenReturn(fieldInfo); when(fieldInfo.attributes()).thenReturn(attributesMap); + // When final KNNScorer knnScorer = (KNNScorer) knnWeight.scorer(leafReaderContext); - assertNotNull(knnScorer); + // Then + assertNotNull(knnScorer); final DocIdSetIterator docIdSetIterator = knnScorer.iterator(); assertNotNull(docIdSetIterator); assertEquals(FILTERED_DOC_ID_TO_SCORES.size(), docIdSetIterator.cost()); jniServiceMockedStatic.verify( - () -> JNIService.queryIndex(anyLong(), any(), anyInt(), any(), eq(filterBitSet.getBits()), anyInt(), any()) + () -> JNIService.queryIndex( + anyLong(), + eq(QUERY_VECTOR), + eq(k), + eq(HNSW_METHOD_PARAMETERS), + any(), + eq(filterBitSet.getBits()), + anyInt(), + any() + ) ); final List actualDocIds = new ArrayList<>(); @@ -677,17 +710,47 @@ public void testANNWithParentsFilter_whenDoingANN_thenBitSetIsPassedToJNI() { // Prepare query and weight when(bitSetProducer.getBitSet(leafReaderContext)).thenReturn(bitset); - final KNNQuery query = new KNNQuery(FIELD_NAME, QUERY_VECTOR, 1, INDEX_NAME, null, bitSetProducer); + + final KNNQuery query = KNNQuery.builder() + .field(FIELD_NAME) + .queryVector(QUERY_VECTOR) + .k(1) + .indexName(INDEX_NAME) + .methodParameters(HNSW_METHOD_PARAMETERS) + .parentsFilter(bitSetProducer) + .build(); + final KNNWeight knnWeight = new KNNWeight(query, 0.0f, null); - jniServiceMockedStatic.when(() -> JNIService.queryIndex(anyLong(), any(), anyInt(), any(), any(), anyInt(), eq(parentsFilter))) - .thenReturn(getKNNQueryResults()); + jniServiceMockedStatic.when( + () -> JNIService.queryIndex( + anyLong(), + eq(QUERY_VECTOR), + eq(1), + eq(HNSW_METHOD_PARAMETERS), + any(), + any(), + anyInt(), + eq(parentsFilter) + ) + ).thenReturn(getKNNQueryResults()); // Execute Scorer knnScorer = knnWeight.scorer(leafReaderContext); // Verify - jniServiceMockedStatic.verify(() -> JNIService.queryIndex(anyLong(), any(), anyInt(), any(), any(), anyInt(), eq(parentsFilter))); + jniServiceMockedStatic.verify( + () -> JNIService.queryIndex( + anyLong(), + eq(QUERY_VECTOR), + eq(1), + eq(HNSW_METHOD_PARAMETERS), + any(), + any(), + anyInt(), + eq(parentsFilter) + ) + ); assertNotNull(knnScorer); final DocIdSetIterator docIdSetIterator = knnScorer.iterator(); assertNotNull(docIdSetIterator); @@ -811,10 +874,17 @@ private void testQueryScore( final Set segmentFiles, final Map fileAttributes ) throws IOException { - jniServiceMockedStatic.when(() -> JNIService.queryIndex(anyLong(), any(), anyInt(), any(), any(), anyInt(), any())) - .thenReturn(getKNNQueryResults()); + jniServiceMockedStatic.when( + () -> JNIService.queryIndex(anyLong(), eq(QUERY_VECTOR), eq(K), eq(HNSW_METHOD_PARAMETERS), any(), any(), anyInt(), any()) + ).thenReturn(getKNNQueryResults()); - final KNNQuery query = new KNNQuery(FIELD_NAME, QUERY_VECTOR, K, INDEX_NAME, (BitSetProducer) null); + final KNNQuery query = KNNQuery.builder() + .field(FIELD_NAME) + .queryVector(QUERY_VECTOR) + .k(K) + .indexName(INDEX_NAME) + .methodParameters(HNSW_METHOD_PARAMETERS) + .build(); final float boost = (float) randomDoubleBetween(0, 10, true); final KNNWeight knnWeight = new KNNWeight(query, boost); diff --git a/src/test/java/org/opensearch/knn/index/query/parser/MethodParametersParserTests.java b/src/test/java/org/opensearch/knn/index/query/parser/MethodParametersParserTests.java new file mode 100644 index 000000000..f2924fb4f --- /dev/null +++ b/src/test/java/org/opensearch/knn/index/query/parser/MethodParametersParserTests.java @@ -0,0 +1,84 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.knn.index.query.parser; + +import lombok.SneakyThrows; +import org.opensearch.common.ValidationException; +import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.core.common.ParsingException; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.knn.KNNTestCase; + +import java.util.Map; + +import static org.opensearch.knn.index.query.parser.MethodParametersParser.doXContent; +import static org.opensearch.knn.index.query.parser.MethodParametersParser.validateMethodParameters; + +public class MethodParametersParserTests extends KNNTestCase { + + public void testValidateMethodParameters() { + ValidationException validationException = validateMethodParameters(Map.of("dummy", 0)); + assertEquals("Validation Failed: 1: dummy is not a valid method parameter;", validationException.getMessage()); + + ValidationException validationException2 = validateMethodParameters(Map.of("ef_search", 0)); + assertTrue(validationException2.getMessage().contains("Validation Failed: 1: ef_search should be greater than 0")); + + ValidationException validationException3 = validateMethodParameters(Map.of("ef_search", 10)); + assertNull(validationException3); + } + + @SneakyThrows + public void testDoXContent() { + Map params = Map.of("ef_search", 10); + XContentBuilder builder = XContentFactory.jsonBuilder() + .startObject() + .startObject("method_parameters") + .field("ef_search", 10) + .endObject() + .endObject(); + + XContentBuilder builder2 = XContentFactory.jsonBuilder().startObject(); + doXContent(builder2, params); + builder2.endObject(); + assertEquals(builder.toString(), builder2.toString()); + + XContentBuilder b3 = XContentFactory.jsonBuilder(); + XContentBuilder b4 = XContentFactory.jsonBuilder(); + + doXContent(b4, null); + assertEquals(b3.toString(), b4.toString()); + } + + @SneakyThrows + public void testFromXContent() { + // efsearch string + XContentBuilder builder = XContentFactory.jsonBuilder().startObject().field("ef_search", "string").endObject(); + XContentParser parser1 = createParser(builder); + expectThrows(ParsingException.class, () -> MethodParametersParser.fromXContent(parser1)); + + // unknown method parameter + builder = XContentFactory.jsonBuilder().startObject().field("unknown", "10").endObject(); + XContentParser parser2 = createParser(builder); + expectThrows(ParsingException.class, () -> MethodParametersParser.fromXContent(parser2)); + + // Valid + builder = XContentFactory.jsonBuilder().startObject().field("ef_search", 10).endObject(); + XContentParser parser3 = createParser(builder); + assertEquals(Map.of("ef_search", 10), MethodParametersParser.fromXContent(parser3)); + + // empty map + builder = XContentFactory.jsonBuilder().startObject().endObject(); + XContentParser parser4 = createParser(builder); + expectThrows(ParsingException.class, () -> MethodParametersParser.fromXContent(parser4)); + } +} diff --git a/src/test/java/org/opensearch/knn/index/util/AbstractKNNLibraryTests.java b/src/test/java/org/opensearch/knn/index/util/AbstractKNNLibraryTests.java index 9e6bd67ea..0aab78042 100644 --- a/src/test/java/org/opensearch/knn/index/util/AbstractKNNLibraryTests.java +++ b/src/test/java/org/opensearch/knn/index/util/AbstractKNNLibraryTests.java @@ -15,6 +15,7 @@ import org.opensearch.knn.index.KNNMethodContext; import org.opensearch.knn.index.MethodComponent; import org.opensearch.knn.index.MethodComponentContext; +import org.opensearch.knn.index.Parameter; import org.opensearch.knn.index.SpaceType; import java.io.IOException; @@ -77,6 +78,20 @@ public ValidationException validate(KNNMethodContext knnMethodContext) { assertNotNull(testAbstractKNNLibrary2.validateMethod(knnMethodContext2)); } + public void testEngineSpecificMethods() throws IOException { + String methodName1 = "test-method-1"; + EngineSpecificMethodContext context = () -> Map.of("myparameter", new Parameter.BooleanParameter("myparameter", false, o -> o)); + + TestAbstractKNNLibrary testAbstractKNNLibrary1 = new TestAbstractKNNLibrary( + Collections.emptyMap(), + Map.of(methodName1, context), + "" + ); + + assertNotNull(testAbstractKNNLibrary1.getMethodContext(methodName1)); + assertTrue(testAbstractKNNLibrary1.getMethodContext(methodName1).supportedMethodParameters().containsKey("myparameter")); + } + public void testGetMethodAsMap() { String methodName = "test-method-1"; SpaceType spaceType = SpaceType.DEFAULT; @@ -109,7 +124,15 @@ public void testGetMethodAsMap() { private static class TestAbstractKNNLibrary extends AbstractKNNLibrary { public TestAbstractKNNLibrary(Map methods, String currentVersion) { - super(methods, currentVersion); + super(methods, Collections.emptyMap(), currentVersion); + } + + public TestAbstractKNNLibrary( + Map methods, + Map engineSpecificMethodContextMap, + String currentVersion + ) { + super(methods, engineSpecificMethodContextMap, currentVersion); } @Override diff --git a/src/test/java/org/opensearch/knn/index/util/NativeLibraryTests.java b/src/test/java/org/opensearch/knn/index/util/NativeLibraryTests.java index 3c3afbee6..814712560 100644 --- a/src/test/java/org/opensearch/knn/index/util/NativeLibraryTests.java +++ b/src/test/java/org/opensearch/knn/index/util/NativeLibraryTests.java @@ -62,7 +62,7 @@ public TestNativeLibrary( String currentVersion, String extension ) { - super(methods, scoreTranslation, currentVersion, extension); + super(methods, Collections.emptyMap(), scoreTranslation, currentVersion, extension); } @Override diff --git a/src/test/java/org/opensearch/knn/jni/JNIServiceTests.java b/src/test/java/org/opensearch/knn/jni/JNIServiceTests.java index d6ae13e92..e71930d48 100644 --- a/src/test/java/org/opensearch/knn/jni/JNIServiceTests.java +++ b/src/test/java/org/opensearch/knn/jni/JNIServiceTests.java @@ -527,6 +527,7 @@ public void testQueryIndex_faiss_sqfp16_valid() { String sqfp16IndexDescription = "HNSW16,SQfp16"; int k = 10; + Map methodParameters = Map.of("ef_search", 12); float[][] truncatedVectors = truncateToFp16Range(testData.indexData.vectors); long memoryAddress = JNICommons.storeVectorData(0, truncatedVectors, (long) truncatedVectors.length * truncatedVectors[0].length); Path tmpFile = createTempFile(); @@ -544,13 +545,22 @@ public void testQueryIndex_faiss_sqfp16_valid() { assertNotEquals(0, pointer); for (float[] query : testData.queries) { - KNNQueryResult[] results = JNIService.queryIndex(pointer, query, k, KNNEngine.FAISS, null, 0, null); + KNNQueryResult[] results = JNIService.queryIndex(pointer, query, k, methodParameters, KNNEngine.FAISS, null, 0, null); assertEquals(k, results.length); } // Filter will result in no ids for (float[] query : testData.queries) { - KNNQueryResult[] results = JNIService.queryIndex(pointer, query, k, KNNEngine.FAISS, new long[] { 0 }, 0, null); + KNNQueryResult[] results = JNIService.queryIndex( + pointer, + query, + k, + methodParameters, + KNNEngine.FAISS, + new long[] { 0 }, + 0, + null + ); assertEquals(0, results.length); } } @@ -736,12 +746,15 @@ public void testLoadIndex_faiss_valid() throws IOException { } public void testQueryIndex_invalidEngine() { - expectThrows(IllegalArgumentException.class, () -> JNIService.queryIndex(0L, new float[] {}, 0, KNNEngine.LUCENE, null, 0, null)); + expectThrows( + IllegalArgumentException.class, + () -> JNIService.queryIndex(0L, new float[] {}, 0, null, KNNEngine.LUCENE, null, 0, null) + ); } public void testQueryIndex_nmslib_invalid_badPointer() { - expectThrows(Exception.class, () -> JNIService.queryIndex(0L, new float[] {}, 0, KNNEngine.NMSLIB, null, 0, null)); + expectThrows(Exception.class, () -> JNIService.queryIndex(0L, new float[] {}, 0, null, KNNEngine.NMSLIB, null, 0, null)); } public void testQueryIndex_nmslib_invalid_nullQueryVector() throws IOException { @@ -765,7 +778,7 @@ public void testQueryIndex_nmslib_invalid_nullQueryVector() throws IOException { ); assertNotEquals(0, pointer); - expectThrows(Exception.class, () -> JNIService.queryIndex(pointer, null, 10, KNNEngine.NMSLIB, null, 0, null)); + expectThrows(Exception.class, () -> JNIService.queryIndex(pointer, null, 10, null, KNNEngine.NMSLIB, null, 0, null)); } public void testQueryIndex_nmslib_valid() throws IOException { @@ -792,7 +805,7 @@ public void testQueryIndex_nmslib_valid() throws IOException { assertNotEquals(0, pointer); for (float[] query : testData.queries) { - KNNQueryResult[] results = JNIService.queryIndex(pointer, query, k, KNNEngine.NMSLIB, null, 0, null); + KNNQueryResult[] results = JNIService.queryIndex(pointer, query, k, null, KNNEngine.NMSLIB, null, 0, null); assertEquals(k, results.length); } } @@ -800,7 +813,7 @@ public void testQueryIndex_nmslib_valid() throws IOException { public void testQueryIndex_faiss_invalid_badPointer() { - expectThrows(Exception.class, () -> JNIService.queryIndex(0L, new float[] {}, 0, KNNEngine.FAISS, null, 0, null)); + expectThrows(Exception.class, () -> JNIService.queryIndex(0L, new float[] {}, 0, null, KNNEngine.FAISS, null, 0, null)); } public void testQueryIndex_faiss_invalid_nullQueryVector() throws IOException { @@ -820,12 +833,13 @@ public void testQueryIndex_faiss_invalid_nullQueryVector() throws IOException { long pointer = JNIService.loadIndex(tmpFile.toAbsolutePath().toString(), Collections.emptyMap(), KNNEngine.FAISS); assertNotEquals(0, pointer); - expectThrows(Exception.class, () -> JNIService.queryIndex(pointer, null, 10, KNNEngine.FAISS, null, 0, null)); + expectThrows(Exception.class, () -> JNIService.queryIndex(pointer, null, 10, null, KNNEngine.FAISS, null, 0, null)); } public void testQueryIndex_faiss_valid() throws IOException { int k = 10; + int efSearch = 100; List methods = ImmutableList.of(faissMethod); List spaces = ImmutableList.of(SpaceType.L2, SpaceType.INNER_PRODUCT); @@ -850,13 +864,31 @@ public void testQueryIndex_faiss_valid() throws IOException { assertNotEquals(0, pointer); for (float[] query : testData.queries) { - KNNQueryResult[] results = JNIService.queryIndex(pointer, query, k, KNNEngine.FAISS, null, 0, null); + KNNQueryResult[] results = JNIService.queryIndex( + pointer, + query, + k, + Map.of("ef_search", efSearch), + KNNEngine.FAISS, + null, + 0, + null + ); assertEquals(k, results.length); } // Filter will result in no ids for (float[] query : testData.queries) { - KNNQueryResult[] results = JNIService.queryIndex(pointer, query, k, KNNEngine.FAISS, new long[] { 0 }, 0, null); + KNNQueryResult[] results = JNIService.queryIndex( + pointer, + query, + k, + Map.of("ef_search", efSearch), + KNNEngine.FAISS, + new long[] { 0 }, + 0, + null + ); assertEquals(0, results.length); } } @@ -866,6 +898,7 @@ public void testQueryIndex_faiss_valid() throws IOException { public void testQueryIndex_faiss_parentIds() throws IOException { int k = 100; + int efSearch = 100; List methods = ImmutableList.of(faissMethod); List spaces = ImmutableList.of(SpaceType.L2, SpaceType.INNER_PRODUCT); @@ -892,7 +925,16 @@ public void testQueryIndex_faiss_parentIds() throws IOException { assertNotEquals(0, pointer); for (float[] query : testDataNested.queries) { - KNNQueryResult[] results = JNIService.queryIndex(pointer, query, k, KNNEngine.FAISS, null, 0, parentIds); + KNNQueryResult[] results = JNIService.queryIndex( + pointer, + query, + k, + Map.of("ef_search", efSearch), + KNNEngine.FAISS, + null, + 0, + parentIds + ); // Verify there is no more than one result from same parent Set parentIdSet = toParentIdSet(results, idToParentIdMap); assertEquals(results.length, parentIdSet.size()); @@ -1223,7 +1265,7 @@ private void assertQueryResultsMatch(float[][] testQueries, int k, List in for (float[] query : testQueries) { KNNQueryResult[][] allResults = new KNNQueryResult[indexAddresses.size()][]; for (int i = 0; i < indexAddresses.size(); i++) { - allResults[i] = JNIService.queryIndex(indexAddresses.get(i), query, k, KNNEngine.FAISS, null, 0, null); + allResults[i] = JNIService.queryIndex(indexAddresses.get(i), query, k, null, KNNEngine.FAISS, null, 0, null); assertEquals(k, allResults[i].length); } diff --git a/src/test/java/org/opensearch/knn/plugin/action/RestKNNStatsHandlerIT.java b/src/test/java/org/opensearch/knn/plugin/action/RestKNNStatsHandlerIT.java index 9af1f49cc..cf7b869f8 100644 --- a/src/test/java/org/opensearch/knn/plugin/action/RestKNNStatsHandlerIT.java +++ b/src/test/java/org/opensearch/knn/plugin/action/RestKNNStatsHandlerIT.java @@ -523,7 +523,7 @@ public void trainKnnModel(String modelId, String trainingIndexName, String train assertEquals(RestStatus.OK, RestStatus.fromCode(trainResponse.getStatusLine().getStatusCode())); } - public void validateModelCreated(String modelId) throws IOException, InterruptedException { + public void validateModelCreated(String modelId) throws Exception { Response getResponse = getModel(modelId, null); String responseBody = EntityUtils.toString(getResponse.getEntity()); assertNotNull(responseBody); diff --git a/src/test/java/org/opensearch/knn/plugin/action/RestKNNWarmupHandlerIT.java b/src/test/java/org/opensearch/knn/plugin/action/RestKNNWarmupHandlerIT.java index a2078c291..d433cd285 100644 --- a/src/test/java/org/opensearch/knn/plugin/action/RestKNNWarmupHandlerIT.java +++ b/src/test/java/org/opensearch/knn/plugin/action/RestKNNWarmupHandlerIT.java @@ -36,7 +36,7 @@ public void testNonKnnIndex() throws IOException { knnWarmup(Collections.singletonList("not-knn-index")); } - public void testEmptyIndex() throws IOException { + public void testEmptyIndex() throws Exception { int graphCountBefore = getTotalGraphsInCache(); createKnnIndex(testIndexName, getKNNDefaultIndexSettings(), createKnnIndexMapping(testFieldName, dimensions)); @@ -45,7 +45,7 @@ public void testEmptyIndex() throws IOException { assertEquals(graphCountBefore, getTotalGraphsInCache()); } - public void testSingleIndex() throws IOException { + public void testSingleIndex() throws Exception { int graphCountBefore = getTotalGraphsInCache(); createKnnIndex(testIndexName, getKNNDefaultIndexSettings(), createKnnIndexMapping(testFieldName, dimensions)); addKnnDoc(testIndexName, "1", testFieldName, new Float[] { 6.0f, 6.0f }); @@ -55,7 +55,7 @@ public void testSingleIndex() throws IOException { assertEquals(graphCountBefore + 1, getTotalGraphsInCache()); } - public void testMultipleIndices() throws IOException { + public void testMultipleIndices() throws Exception { int graphCountBefore = getTotalGraphsInCache(); createKnnIndex(testIndexName + "1", getKNNDefaultIndexSettings(), createKnnIndexMapping(testFieldName, dimensions)); diff --git a/src/test/java/org/opensearch/knn/plugin/action/RestLegacyKNNWarmupHandlerIT.java b/src/test/java/org/opensearch/knn/plugin/action/RestLegacyKNNWarmupHandlerIT.java index f74135c0c..e6345faba 100644 --- a/src/test/java/org/opensearch/knn/plugin/action/RestLegacyKNNWarmupHandlerIT.java +++ b/src/test/java/org/opensearch/knn/plugin/action/RestLegacyKNNWarmupHandlerIT.java @@ -43,7 +43,7 @@ public void testNonKnnIndex() throws IOException { executeWarmupRequest(Collections.singletonList("not-knn-index"), KNNPlugin.LEGACY_KNN_BASE_URI); } - public void testEmptyIndex() throws IOException { + public void testEmptyIndex() throws Exception { int graphCountBefore = getTotalGraphsInCache(); createKnnIndex(testIndexName, getKNNDefaultIndexSettings(), createKnnIndexMapping(testFieldName, dimensions)); @@ -52,7 +52,7 @@ public void testEmptyIndex() throws IOException { assertEquals(graphCountBefore, getTotalGraphsInCache()); } - public void testSingleIndex() throws IOException { + public void testSingleIndex() throws Exception { int graphCountBefore = getTotalGraphsInCache(); createKnnIndex(testIndexName, getKNNDefaultIndexSettings(), createKnnIndexMapping(testFieldName, dimensions)); addKnnDoc(testIndexName, "1", testFieldName, new Float[] { 6.0f, 6.0f }); @@ -62,7 +62,7 @@ public void testSingleIndex() throws IOException { assertEquals(graphCountBefore + 1, getTotalGraphsInCache()); } - public void testMultipleIndices() throws IOException { + public void testMultipleIndices() throws Exception { int graphCountBefore = getTotalGraphsInCache(); createKnnIndex(testIndexName + "1", getKNNDefaultIndexSettings(), createKnnIndexMapping(testFieldName, dimensions)); diff --git a/src/test/java/org/opensearch/knn/plugin/action/RestTrainModelHandlerIT.java b/src/test/java/org/opensearch/knn/plugin/action/RestTrainModelHandlerIT.java index a22e1acb8..1ba6eae9b 100644 --- a/src/test/java/org/opensearch/knn/plugin/action/RestTrainModelHandlerIT.java +++ b/src/test/java/org/opensearch/knn/plugin/action/RestTrainModelHandlerIT.java @@ -20,7 +20,6 @@ import org.opensearch.knn.KNNRestTestCase; import org.opensearch.core.rest.RestStatus; -import java.io.IOException; import java.util.Map; import static org.opensearch.knn.common.KNNConstants.ENCODER_PARAMETER_PQ_CODE_SIZE; @@ -35,7 +34,7 @@ public class RestTrainModelHandlerIT extends KNNRestTestCase { - public void testTrainModel_fail_notEnoughData() throws IOException, InterruptedException { + public void testTrainModel_fail_notEnoughData() throws Exception { // Check that training fails properly when there is not enough data @@ -326,7 +325,7 @@ public void testTrainModel_success_withId() throws Exception { assertTrainingSucceeds(modelId, 30, 1000); } - public void testTrainModel_success_noId() throws IOException, InterruptedException { + public void testTrainModel_success_noId() throws Exception { // Test to check if training succeeds when no id is passed in String trainingIndexName = "train-index"; diff --git a/src/test/java/org/opensearch/knn/plugin/stats/suppliers/LibraryInitializedSupplierTests.java b/src/test/java/org/opensearch/knn/plugin/stats/suppliers/LibraryInitializedSupplierTests.java index 46240e830..d66241376 100644 --- a/src/test/java/org/opensearch/knn/plugin/stats/suppliers/LibraryInitializedSupplierTests.java +++ b/src/test/java/org/opensearch/knn/plugin/stats/suppliers/LibraryInitializedSupplierTests.java @@ -12,6 +12,7 @@ package org.opensearch.knn.plugin.stats.suppliers; import org.opensearch.common.ValidationException; +import org.opensearch.knn.index.util.EngineSpecificMethodContext; import org.opensearch.knn.index.KNNMethod; import org.opensearch.knn.index.KNNMethodContext; import org.opensearch.knn.index.SpaceType; @@ -59,6 +60,11 @@ public KNNMethod getMethod(String methodName) { return null; } + @Override + public EngineSpecificMethodContext getMethodContext(String methodName) { + return null; + } + @Override public float score(float rawScore, SpaceType spaceType) { return 0; diff --git a/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java b/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java index adb1726d4..e71399c1e 100644 --- a/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java +++ b/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java @@ -14,7 +14,6 @@ import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.xcontent.XContentHelper; import org.opensearch.core.xcontent.DeprecationHandler; -import org.opensearch.core.xcontent.MediaType; import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.XContentParser; @@ -34,7 +33,7 @@ import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; -import org.opensearch.common.xcontent.XContentType; +import org.opensearch.core.xcontent.MediaType; import org.opensearch.index.query.ExistsQueryBuilder; import org.opensearch.index.query.QueryBuilder; import org.opensearch.index.query.functionscore.ScriptScoreQueryBuilder; @@ -124,7 +123,7 @@ public static void dumpCoverage() throws IOException, MalformedObjectNameExcepti // jacoco.dir is set in esplugin-coverage.gradle, if it doesn't exist we don't // want to collect coverage so we can return early String jacocoBuildPath = System.getProperty("jacoco.dir"); - if (StringUtils.isBlank(jacocoBuildPath)) { + if (org.opensearch.core.common.Strings.isNullOrEmpty(jacocoBuildPath)) { return; } @@ -194,18 +193,7 @@ protected Response searchKNNIndex(String index, KNNQueryBuilder knnQueryBuilder, XContentBuilder builder = XContentFactory.jsonBuilder().startObject().startObject("query"); knnQueryBuilder.doXContent(builder, ToXContent.EMPTY_PARAMS); builder.endObject().endObject(); - - Request request = new Request("POST", "/" + index + "/_search"); - - request.addParameter("size", Integer.toString(resultSize)); - request.addParameter("explain", Boolean.toString(true)); - request.addParameter("search_type", "query_then_fetch"); - request.setJsonEntity(builder.toString()); - - Response response = client().performRequest(request); - assertEquals(request.getEndpoint() + ": failed", RestStatus.OK, RestStatus.fromCode(response.getStatusLine().getStatusCode())); - - return response; + return searchKNNIndex(index, builder, resultSize); } /** @@ -260,8 +248,10 @@ protected Response performSearch(final String indexName, final String query) thr */ protected List parseSearchResponse(String responseBody, String fieldName) throws IOException { @SuppressWarnings("unchecked") - List hits = (List) ((Map) createParser(XContentType.JSON.xContent(), responseBody).map() - .get("hits")).get("hits"); + List hits = (List) ((Map) createParser( + MediaTypeRegistry.getDefaultMediaType().xContent(), + responseBody + ).map().get("hits")).get("hits"); @SuppressWarnings("unchecked") List knnSearchResponses = hits.stream().map(hit -> { @@ -283,8 +273,10 @@ protected List parseSearchResponse(String responseBody, String fieldN protected List parseSearchResponseScore(String responseBody, String fieldName) throws IOException { @SuppressWarnings("unchecked") - List hits = (List) ((Map) createParser(XContentType.JSON.xContent(), responseBody).map() - .get("hits")).get("hits"); + List hits = (List) ((Map) createParser( + MediaTypeRegistry.getDefaultMediaType().xContent(), + responseBody + ).map().get("hits")).get("hits"); @SuppressWarnings("unchecked") List knnSearchResponses = hits.stream() @@ -299,8 +291,10 @@ protected List parseSearchResponseScore(String responseBody, String field */ protected Double parseAggregationResponse(String responseBody, String aggregationName) throws IOException { @SuppressWarnings("unchecked") - Map aggregations = ((Map) createParser(XContentType.JSON.xContent(), responseBody).map() - .get("aggregations")); + Map aggregations = ((Map) createParser( + MediaTypeRegistry.getDefaultMediaType().xContent(), + responseBody + ).map().get("aggregations")); final Map values = (Map) aggregations.get(aggregationName); return Double.valueOf(String.valueOf(values.get("value"))); @@ -450,7 +444,7 @@ protected String createKnnIndexNestedMapping(Integer dimensions, String fieldPat * @return index mapping a map */ @SuppressWarnings("unchecked") - public Map getIndexMappingAsMap(String index) throws IOException { + public Map getIndexMappingAsMap(String index) throws Exception { Request request = new Request("GET", "/" + index + "/_mapping"); Response response = client().performRequest(request); @@ -459,12 +453,12 @@ public Map getIndexMappingAsMap(String index) throws IOException String responseBody = EntityUtils.toString(response.getEntity()); - Map responseMap = createParser(XContentType.JSON.xContent(), responseBody).map(); + Map responseMap = createParser(MediaTypeRegistry.getDefaultMediaType().xContent(), responseBody).map(); return (Map) ((Map) responseMap.get(index)).get("mappings"); } - public int getDocCount(String indexName) throws IOException { + public int getDocCount(String indexName) throws Exception { Request request = new Request("GET", "/" + indexName + "/_count"); Response response = client().performRequest(request); @@ -473,7 +467,7 @@ public int getDocCount(String indexName) throws IOException { String responseBody = EntityUtils.toString(response.getEntity()); - Map responseMap = createParser(XContentType.JSON.xContent(), responseBody).map(); + Map responseMap = createParser(MediaTypeRegistry.getDefaultMediaType().xContent(), responseBody).map(); return (Integer) responseMap.get("count"); } @@ -632,12 +626,14 @@ protected void deleteKnnDoc(String index, String docId) throws IOException { /** * Retrieve document by index and document id */ - protected Map getKnnDoc(final String index, final String docId) throws IOException { + protected Map getKnnDoc(final String index, final String docId) throws Exception { final Request request = new Request("GET", "/" + index + "/_doc/" + docId); final Response response = client().performRequest(request); - final Map responseMap = createParser(XContentType.JSON.xContent(), EntityUtils.toString(response.getEntity())) - .map(); + final Map responseMap = createParser( + MediaTypeRegistry.getDefaultMediaType().xContent(), + EntityUtils.toString(response.getEntity()) + ).map(); assertNotNull(responseMap); assertTrue((Boolean) responseMap.get(DOCUMENT_FIELD_FOUND)); @@ -733,7 +729,7 @@ protected Response clearCache(List indices) throws IOException { * Parse KNN Cluster stats from response */ protected Map parseClusterStatsResponse(String responseBody) throws IOException { - Map responseMap = createParser(XContentType.JSON.xContent(), responseBody).map(); + Map responseMap = createParser(MediaTypeRegistry.getDefaultMediaType().xContent(), responseBody).map(); responseMap.remove("cluster_name"); responseMap.remove("_nodes"); responseMap.remove("nodes"); @@ -745,7 +741,10 @@ protected Map parseClusterStatsResponse(String responseBody) thr */ protected List> parseNodeStatsResponse(String responseBody) throws IOException { @SuppressWarnings("unchecked") - Map responseMap = (Map) createParser(XContentType.JSON.xContent(), responseBody).map().get("nodes"); + Map responseMap = (Map) createParser( + MediaTypeRegistry.getDefaultMediaType().xContent(), + responseBody + ).map().get("nodes"); @SuppressWarnings("unchecked") List> nodeResponses = responseMap.keySet() @@ -761,8 +760,10 @@ protected List> parseNodeStatsResponse(String responseBody) */ @SuppressWarnings("unchecked") protected int parseTotalSearchHits(String searchResponseBody) throws IOException { - Map responseMap = (Map) createParser(XContentType.JSON.xContent(), searchResponseBody).map() - .get("hits"); + Map responseMap = (Map) createParser( + MediaTypeRegistry.getDefaultMediaType().xContent(), + searchResponseBody + ).map().get("hits"); return (int) ((Map) responseMap.get("total")).get("value"); } @@ -789,7 +790,7 @@ protected List parseIds(String searchResponseBody) throws IOException { * Get the total number of graphs in the cache across all nodes */ @SuppressWarnings("unchecked") - protected int getTotalGraphsInCache() throws IOException { + protected int getTotalGraphsInCache() throws Exception { Response response = getKnnStats(Collections.emptyList(), Collections.emptyList()); String responseBody = EntityUtils.toString(response.getEntity()); @@ -1046,7 +1047,7 @@ public void bulkAddKnnDocs(String index, String fieldName, float[][] indexVector } // Method that returns index vectors of the documents that were added before into the index - public float[][] getIndexVectorsFromIndex(String testIndex, String testField, int docCount, int dimensions) throws IOException { + public float[][] getIndexVectorsFromIndex(String testIndex, String testField, int docCount, int dimensions) throws Exception { float[][] vectors = new float[docCount][dimensions]; QueryBuilder qb = new MatchAllQueryBuilder(); @@ -1073,7 +1074,7 @@ public float[][] getIndexVectorsFromIndex(String testIndex, String testField, in } // Method that performs bulk search for multiple queries and stores the resulting documents ids into list - public List> bulkSearch(String testIndex, String testField, float[][] queryVectors, int k) throws IOException { + public List> bulkSearch(String testIndex, String testField, float[][] queryVectors, int k) throws Exception { List> searchResults = new ArrayList<>(); List kVectors; @@ -1110,12 +1111,22 @@ public void addKNNDocs(String testIndex, String testField, int dimension, int fi } } + public void validateKNNSearch(String testIndex, String testField, int dimension, int numDocs, int k) throws Exception { + validateKNNSearch(testIndex, testField, dimension, numDocs, k, null); + } + // Validate KNN search on a KNN index by generating the query vector from the number of documents in the index - public void validateKNNSearch(String testIndex, String testField, int dimension, int numDocs, int k) throws IOException { + public void validateKNNSearch(String testIndex, String testField, int dimension, int numDocs, int k, Map methodParameters) + throws Exception { float[] queryVector = new float[dimension]; Arrays.fill(queryVector, (float) numDocs); - Response searchResponse = searchKNNIndex(testIndex, new KNNQueryBuilder(testField, queryVector, k), k); + Response searchResponse = searchKNNIndex( + testIndex, + KNNQueryBuilder.builder().k(k).methodParameters(methodParameters).fieldName(testField).vector(queryVector).build(), + k + ); + List results = parseSearchResponse(EntityUtils.toString(searchResponse.getEntity()), testField); assertEquals(k, results.size()); @@ -1371,7 +1382,7 @@ public void deleteModel(String modelId) throws IOException { assertEquals(request.getEndpoint() + ": failed", RestStatus.OK, RestStatus.fromCode(response.getStatusLine().getStatusCode())); } - public void assertTrainingSucceeds(String modelId, int attempts, int delayInMillis) throws InterruptedException, IOException { + public void assertTrainingSucceeds(String modelId, int attempts, int delayInMillis) throws InterruptedException, Exception { int attemptNum = 0; Response response; Map responseMap; @@ -1382,7 +1393,8 @@ public void assertTrainingSucceeds(String modelId, int attempts, int delayInMill response = getModel(modelId, null); - responseMap = createParser(XContentType.JSON.xContent(), EntityUtils.toString(response.getEntity())).map(); + responseMap = createParser(MediaTypeRegistry.getDefaultMediaType().xContent(), EntityUtils.toString(response.getEntity())) + .map(); modelState = ModelState.getModelState((String) responseMap.get(MODEL_STATE)); if (modelState == ModelState.CREATED) { @@ -1395,7 +1407,7 @@ public void assertTrainingSucceeds(String modelId, int attempts, int delayInMill fail("Training did not succeed after " + attempts + " attempts with a delay of " + delayInMillis + " ms."); } - public void assertTrainingFails(String modelId, int attempts, int delayInMillis) throws InterruptedException, IOException { + public void assertTrainingFails(String modelId, int attempts, int delayInMillis) throws Exception { int attemptNum = 0; Response response; Map responseMap; @@ -1406,7 +1418,8 @@ public void assertTrainingFails(String modelId, int attempts, int delayInMillis) response = getModel(modelId, null); - responseMap = createParser(XContentType.JSON.xContent(), EntityUtils.toString(response.getEntity())).map(); + responseMap = createParser(MediaTypeRegistry.getDefaultMediaType().xContent(), EntityUtils.toString(response.getEntity())) + .map(); modelState = ModelState.getModelState((String) responseMap.get(MODEL_STATE)); if (modelState == ModelState.FAILED) { @@ -1526,9 +1539,9 @@ public interface IProxy { protected void refreshAllNonSystemIndices() throws Exception { Response response = adminClient().performRequest(new Request("GET", "/_cat/indices?format=json&expand_wildcards=all")); - MediaType xContentType = MediaType.fromMediaType(response.getEntity().getContentType().getValue()); + MediaType mediaType = MediaType.fromMediaType(response.getEntity().getContentType().getValue()); try ( - XContentParser parser = xContentType.xContent() + XContentParser parser = mediaType.xContent() .createParser( NamedXContentRegistry.EMPTY, DeprecationHandler.THROW_UNSUPPORTED_OPERATION, From ef5489e394b88123e2d9aba4f5e93c6de4c16408 Mon Sep 17 00:00:00 2001 From: Junqiu Lei Date: Fri, 5 Jul 2024 09:24:36 -0700 Subject: [PATCH 281/416] Support ef_search parameter in radial search faiss engine (#1790) (#1794) (cherry picked from commit e7d7ec81ec223ed4f54d5f9e8eeff1e776e8662e) Signed-off-by: Junqiu Lei --- CHANGELOG.md | 1 + jni/include/faiss_wrapper.h | 6 +- .../org_opensearch_knn_jni_FaissService.h | 8 +- jni/src/faiss_wrapper.cpp | 29 ++-- .../org_opensearch_knn_jni_FaissService.cpp | 11 +- jni/tests/faiss_wrapper_test.cpp | 15 +- jni/tests/faiss_wrapper_unit_test.cpp | 155 ++++++++++++++++-- .../opensearch/knn/index/VectorQueryType.java | 4 + .../opensearch/knn/index/query/KNNQuery.java | 2 + .../knn/index/query/KNNQueryBuilder.java | 51 ++++-- .../opensearch/knn/index/query/KNNWeight.java | 1 + .../knn/index/query/RNNQueryFactory.java | 22 ++- .../knn/index/util/DefaultHnswContext.java | 2 +- .../util/EngineSpecificMethodContext.java | 4 +- .../org/opensearch/knn/index/util/Lucene.java | 2 +- .../knn/index/util/LuceneHNSWContext.java | 36 ++++ .../knn/index/util/QueryContext.java | 23 +++ .../org/opensearch/knn/jni/FaissService.java | 4 + .../org/opensearch/knn/jni/JNIService.java | 5 +- .../org/opensearch/knn/index/FaissIT.java | 25 ++- .../opensearch/knn/index/LuceneEngineIT.java | 52 ++++-- .../knn/index/query/KNNQueryBuilderTests.java | 75 ++++++++- .../knn/index/query/KNNWeightTests.java | 34 +++- .../knn/index/query/RNNQueryFactoryTests.java | 24 ++- .../index/util/AbstractKNNLibraryTests.java | 21 ++- 25 files changed, 500 insertions(+), 112 deletions(-) create mode 100644 src/main/java/org/opensearch/knn/index/util/LuceneHNSWContext.java create mode 100644 src/main/java/org/opensearch/knn/index/util/QueryContext.java diff --git a/CHANGELOG.md b/CHANGELOG.md index d38818a68..2d75d5f2f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ## [Unreleased 2.x](https://github.com/opensearch-project/k-NN/compare/2.15...2.x) ### Features * Adds dynamic query parameter ef_search [#1783](https://github.com/opensearch-project/k-NN/pull/1783) +* Adds dynamic query parameter ef_search in radial search faiss engine [#1790](https://github.com/opensearch-project/k-NN/pull/1790) ### Enhancements ### Bug Fixes ### Infrastructure diff --git a/jni/include/faiss_wrapper.h b/jni/include/faiss_wrapper.h index aa747862a..3bfc66325 100644 --- a/jni/include/faiss_wrapper.h +++ b/jni/include/faiss_wrapper.h @@ -90,6 +90,7 @@ namespace knn_jni { * @param indexPointerJ - pointer to the index * @param queryVectorJ - the query vector * @param radiusJ - the radius for the range search + * @param methodParamsJ - the method parameters * @param maxResultsWindowJ - the maximum number of results to return * @param filterIdsJ - the filter ids * @param filterIdsTypeJ - the filter ids type @@ -98,7 +99,7 @@ namespace knn_jni { * @return an array of RangeQueryResults */ jobjectArray RangeSearchWithFilter(knn_jni::JNIUtilInterface *jniUtil, JNIEnv *env, jlong indexPointerJ, jfloatArray queryVectorJ, - jfloat radiusJ, jint maxResultWindowJ, jlongArray filterIdsJ, jint filterIdsTypeJ, jintArray parentIdsJ); + jfloat radiusJ, jobject methodParamsJ, jint maxResultWindowJ, jlongArray filterIdsJ, jint filterIdsTypeJ, jintArray parentIdsJ); /* * Perform a range search against the index located in memory at indexPointerJ. @@ -106,13 +107,14 @@ namespace knn_jni { * @param indexPointerJ - pointer to the index * @param queryVectorJ - the query vector * @param radiusJ - the radius for the range search + * @param methodParamsJ - the method parameters * @param maxResultsWindowJ - the maximum number of results to return * @param parentIdsJ - the parent ids * * @return an array of RangeQueryResults */ jobjectArray RangeSearch(knn_jni::JNIUtilInterface *jniUtil, JNIEnv *env, jlong indexPointerJ, jfloatArray queryVectorJ, - jfloat radiusJ, jint maxResultWindowJ, jintArray parentIdsJ); + jfloat radiusJ, jobject methodParamsJ, jint maxResultWindowJ, jintArray parentIdsJ); } } diff --git a/jni/include/org_opensearch_knn_jni_FaissService.h b/jni/include/org_opensearch_knn_jni_FaissService.h index 0453864e4..ef382507a 100644 --- a/jni/include/org_opensearch_knn_jni_FaissService.h +++ b/jni/include/org_opensearch_knn_jni_FaissService.h @@ -125,18 +125,18 @@ JNIEXPORT jlong JNICALL Java_org_opensearch_knn_jni_FaissService_transferVectors /* * Class: org_opensearch_knn_jni_FaissService * Method: rangeSearchIndexWithFilter -* Signature: (J[FJ[I)[Lorg/opensearch/knn/index/query/RangeQueryResult; +* Signature: (J[FJLjava/util/MapI[JII)[Lorg/opensearch/knn/index/query/RangeQueryResult; */ JNIEXPORT jobjectArray JNICALL Java_org_opensearch_knn_jni_FaissService_rangeSearchIndexWithFilter - (JNIEnv *, jclass, jlong, jfloatArray, jfloat, jint, jlongArray, jint, jintArray); + (JNIEnv *, jclass, jlong, jfloatArray, jfloat, jobject, jint, jlongArray, jint, jintArray); /* * Class: org_opensearch_knn_jni_FaissService * Method: rangeSearchIndex - * Signature: (J[FJ[I)[Lorg/opensearch/knn/index/query/RangeQueryResult; + * Signature: (J[FJLjava/util/MapII)[Lorg/opensearch/knn/index/query/RangeQueryResult; */ JNIEXPORT jobjectArray JNICALL Java_org_opensearch_knn_jni_FaissService_rangeSearchIndex - (JNIEnv *, jclass, jlong, jfloatArray, jfloat, jint, jintArray); + (JNIEnv *, jclass, jlong, jfloatArray, jfloat, jobject, jint, jintArray); #ifdef __cplusplus } diff --git a/jni/src/faiss_wrapper.cpp b/jni/src/faiss_wrapper.cpp index 198f733be..692a33aee 100644 --- a/jni/src/faiss_wrapper.cpp +++ b/jni/src/faiss_wrapper.cpp @@ -595,12 +595,12 @@ faiss::IndexIVFPQ * extractIVFPQIndex(faiss::Index * index) { } jobjectArray knn_jni::faiss_wrapper::RangeSearch(knn_jni::JNIUtilInterface *jniUtil, JNIEnv *env, jlong indexPointerJ, - jfloatArray queryVectorJ, jfloat radiusJ, jint maxResultWindowJ, jintArray parentIdsJ) { - return knn_jni::faiss_wrapper::RangeSearchWithFilter(jniUtil, env, indexPointerJ, queryVectorJ, radiusJ, maxResultWindowJ, nullptr, 0, parentIdsJ); + jfloatArray queryVectorJ, jfloat radiusJ, jobject methodParamsJ, jint maxResultWindowJ, jintArray parentIdsJ) { + return knn_jni::faiss_wrapper::RangeSearchWithFilter(jniUtil, env, indexPointerJ, queryVectorJ, radiusJ, methodParamsJ, maxResultWindowJ, nullptr, 0, parentIdsJ); } jobjectArray knn_jni::faiss_wrapper::RangeSearchWithFilter(knn_jni::JNIUtilInterface *jniUtil, JNIEnv *env, jlong indexPointerJ, - jfloatArray queryVectorJ, jfloat radiusJ, jint maxResultWindowJ, jlongArray filterIdsJ, jint filterIdsTypeJ, jintArray parentIdsJ) { + jfloatArray queryVectorJ, jfloat radiusJ, jobject methodParamsJ, jint maxResultWindowJ, jlongArray filterIdsJ, jint filterIdsTypeJ, jintArray parentIdsJ) { if (queryVectorJ == nullptr) { throw std::runtime_error("Query Vector cannot be null"); } @@ -613,6 +613,11 @@ jobjectArray knn_jni::faiss_wrapper::RangeSearchWithFilter(knn_jni::JNIUtilInter float *rawQueryVector = jniUtil->GetFloatArrayElements(env, queryVectorJ, nullptr); + std::unordered_map methodParams; + if (methodParamsJ != nullptr) { + methodParams = jniUtil->ConvertJavaMapToCppMap(env, methodParamsJ); + } + // The res will be freed by ~RangeSearchResult() in FAISS // The second parameter is always true, as lims is allocated by FAISS faiss::RangeSearchResult res(1, true); @@ -634,9 +639,8 @@ jobjectArray knn_jni::faiss_wrapper::RangeSearchWithFilter(knn_jni::JNIUtilInter std::vector idGrouperBitmap; auto hnswReader = dynamic_cast(indexReader->index); if(hnswReader) { - // Setting the ef_search value equal to what was provided during index creation. SearchParametersHNSW has a default - // value of ef_search = 16 which will then be used. - hnswParams.efSearch = hnswReader->hnsw.efSearch; + // Query param ef_search supersedes ef_search provided during index setting. + hnswParams.efSearch = knn_jni::commons::getIntegerMethodParameter(env, jniUtil, methodParams, EF_SEARCH, hnswReader->hnsw.efSearch); hnswParams.sel = idSelector.get(); if (parentIdsJ != nullptr) { idGrouper = buildIDGrouperBitmap(jniUtil, env, parentIdsJ, &idGrouperBitmap); @@ -664,12 +668,13 @@ jobjectArray knn_jni::faiss_wrapper::RangeSearchWithFilter(knn_jni::JNIUtilInter std::unique_ptr idGrouper; std::vector idGrouperBitmap; auto hnswReader = dynamic_cast(indexReader->index); - if(hnswReader!= nullptr && parentIdsJ != nullptr) { - // Setting the ef_search value equal to what was provided during index creation. SearchParametersHNSW has a default - // value of ef_search = 16 which will then be used. - hnswParams.efSearch = hnswReader->hnsw.efSearch; - idGrouper = buildIDGrouperBitmap(jniUtil, env, parentIdsJ, &idGrouperBitmap); - hnswParams.grp = idGrouper.get(); + if(hnswReader!= nullptr) { + // Query param ef_search supersedes ef_search provided during index setting. + hnswParams.efSearch = knn_jni::commons::getIntegerMethodParameter(env, jniUtil, methodParams, EF_SEARCH, hnswReader->hnsw.efSearch); + if (parentIdsJ != nullptr) { + idGrouper = buildIDGrouperBitmap(jniUtil, env, parentIdsJ, &idGrouperBitmap); + hnswParams.grp = idGrouper.get(); + } searchParameters = &hnswParams; } try { diff --git a/jni/src/org_opensearch_knn_jni_FaissService.cpp b/jni/src/org_opensearch_knn_jni_FaissService.cpp index 57353f9e1..d8cf2f9cf 100644 --- a/jni/src/org_opensearch_knn_jni_FaissService.cpp +++ b/jni/src/org_opensearch_knn_jni_FaissService.cpp @@ -194,11 +194,11 @@ JNIEXPORT jlong JNICALL Java_org_opensearch_knn_jni_FaissService_transferVectors JNIEXPORT jobjectArray JNICALL Java_org_opensearch_knn_jni_FaissService_rangeSearchIndex(JNIEnv * env, jclass cls, jlong indexPointerJ, jfloatArray queryVectorJ, - jfloat radiusJ, jint maxResultWindowJ, - jintArray parentIdsJ) + jfloat radiusJ, jobject methodParamsJ, + jint maxResultWindowJ, jintArray parentIdsJ) { try { - return knn_jni::faiss_wrapper::RangeSearch(&jniUtil, env, indexPointerJ, queryVectorJ, radiusJ, maxResultWindowJ, parentIdsJ); + return knn_jni::faiss_wrapper::RangeSearch(&jniUtil, env, indexPointerJ, queryVectorJ, radiusJ, methodParamsJ, maxResultWindowJ, parentIdsJ); } catch (...) { jniUtil.CatchCppExceptionAndThrowJava(env); } @@ -208,12 +208,11 @@ JNIEXPORT jobjectArray JNICALL Java_org_opensearch_knn_jni_FaissService_rangeSea JNIEXPORT jobjectArray JNICALL Java_org_opensearch_knn_jni_FaissService_rangeSearchIndexWithFilter(JNIEnv * env, jclass cls, jlong indexPointerJ, jfloatArray queryVectorJ, - jfloat radiusJ, jint maxResultWindowJ, + jfloat radiusJ, jobject methodParamsJ, jint maxResultWindowJ, jlongArray filterIdsJ, jint filterIdsTypeJ, jintArray parentIdsJ) { try { - return knn_jni::faiss_wrapper::RangeSearchWithFilter(&jniUtil, env, indexPointerJ, queryVectorJ, radiusJ, - maxResultWindowJ, filterIdsJ, filterIdsTypeJ, parentIdsJ); + return knn_jni::faiss_wrapper::RangeSearchWithFilter(&jniUtil, env, indexPointerJ, queryVectorJ, radiusJ, methodParamsJ, maxResultWindowJ, filterIdsJ, filterIdsTypeJ, parentIdsJ); } catch (...) { jniUtil.CatchCppExceptionAndThrowJava(env); } diff --git a/jni/tests/faiss_wrapper_test.cpp b/jni/tests/faiss_wrapper_test.cpp index 1db3df42c..f1d2ee7f4 100644 --- a/jni/tests/faiss_wrapper_test.cpp +++ b/jni/tests/faiss_wrapper_test.cpp @@ -634,6 +634,11 @@ TEST(FaissRangeSearchQueryIndexTest, BasicAssertions) { faiss::MetricType metricType = faiss::METRIC_L2; std::string method = "HNSW32,Flat"; + int efSearch = 20; + std::unordered_map methodParams; + methodParams[knn_jni::EF_SEARCH] = reinterpret_cast(&efSearch); + auto methodParamsJ = reinterpret_cast(&methodParams); + // Define query data int numQueries = 100; std::vector> queries; @@ -666,7 +671,7 @@ TEST(FaissRangeSearchQueryIndexTest, BasicAssertions) { knn_jni::faiss_wrapper::RangeSearch( &mockJNIUtil, jniEnv, reinterpret_cast(&createdIndexWithData), - reinterpret_cast(&query), rangeSearchRadius, maxResultWindow, nullptr))); + reinterpret_cast(&query), rangeSearchRadius, methodParamsJ, maxResultWindow, nullptr))); // assert result size is not 0 ASSERT_NE(0, results->size()); @@ -721,7 +726,7 @@ TEST(FaissRangeSearchQueryIndexTest_WhenHitMaxWindowResult, BasicAssertions){ knn_jni::faiss_wrapper::RangeSearch( &mockJNIUtil, jniEnv, reinterpret_cast(&createdIndexWithData), - reinterpret_cast(&query), rangeSearchRadius, maxResultWindow, nullptr))); + reinterpret_cast(&query), rangeSearchRadius, nullptr, maxResultWindow, nullptr))); // assert result size is not 0 ASSERT_NE(0, results->size()); @@ -787,7 +792,7 @@ TEST(FaissRangeSearchQueryIndexTestWithFilterTest, BasicAssertions) { knn_jni::faiss_wrapper::RangeSearchWithFilter( &mockJNIUtil, jniEnv, reinterpret_cast(&createdIndexWithData), - reinterpret_cast(&query), rangeSearchRadius, maxResultWindow, + reinterpret_cast(&query), rangeSearchRadius, nullptr, maxResultWindow, reinterpret_cast(&bitmap), 0, nullptr))); // assert result size is not 0 @@ -862,7 +867,7 @@ TEST(FaissRangeSearchQueryIndexTestWithParentFilterTest, BasicAssertions) { knn_jni::faiss_wrapper::RangeSearchWithFilter( &mockJNIUtil, jniEnv, reinterpret_cast(&createdIndexWithData), - reinterpret_cast(&query), rangeSearchRadius, maxResultWindow, nullptr, 0, + reinterpret_cast(&query), rangeSearchRadius, nullptr, maxResultWindow, nullptr, 0, reinterpret_cast(&parentIds)))); // assert result size is not 0 @@ -879,4 +884,4 @@ TEST(FaissRangeSearchQueryIndexTestWithParentFilterTest, BasicAssertions) { delete it; } } -} \ No newline at end of file +} diff --git a/jni/tests/faiss_wrapper_unit_test.cpp b/jni/tests/faiss_wrapper_unit_test.cpp index ea9131dd7..d9fdac23f 100644 --- a/jni/tests/faiss_wrapper_unit_test.cpp +++ b/jni/tests/faiss_wrapper_unit_test.cpp @@ -30,14 +30,15 @@ struct MockIndex : faiss::IndexHNSW { } }; - struct MockIdMap : faiss::IndexIDMap { - mutable idx_t nCalled; - mutable const float *xCalled; - mutable idx_t kCalled; - mutable float *distancesCalled; - mutable idx_t *labelsCalled; - mutable const faiss::SearchParametersHNSW *paramsCalled; + mutable idx_t nCalled{}; + mutable const float *xCalled{}; + mutable int kCalled{}; + mutable float radiusCalled{}; + mutable float *distancesCalled{}; + mutable idx_t *labelsCalled{}; + mutable const faiss::SearchParametersHNSW *paramsCalled{}; + mutable faiss::RangeSearchResult *resCalled{}; explicit MockIdMap(MockIndex *index) : faiss::IndexIDMapTemplate(index) { } @@ -57,18 +58,33 @@ struct MockIdMap : faiss::IndexIDMap { paramsCalled = dynamic_cast(params); } + void range_search( + idx_t n, + const float *x, + float radius, + faiss::RangeSearchResult *res, + const faiss::SearchParameters *params) const override { + nCalled = n; + xCalled = x; + radiusCalled = radius; + resCalled = res; + paramsCalled = dynamic_cast(params); + } + void resetMock() const { nCalled = 0; xCalled = nullptr; kCalled = 0; + radiusCalled = 0.0; distancesCalled = nullptr; labelsCalled = nullptr; + resCalled = nullptr; paramsCalled = nullptr; } }; struct QueryIndexHNSWTestInput { - string description; + std::string description; int k; int efSearch; int filterIdType; @@ -76,13 +92,31 @@ struct QueryIndexHNSWTestInput { bool parentIdsPresent; }; - +struct RangeSearchTestInput { + std::string description; + float radius; + int efSearch; + int filterIdType; + bool filterIdsPresent; + bool parentIdsPresent; +}; class FaissWrappeterParametrizedTestFixture : public testing::TestWithParam { public: FaissWrappeterParametrizedTestFixture() : index_(3), id_map_(&index_) { index_.hnsw.efSearch = 100; // assigning 100 to make sure default of 16 is not used anywhere - }; + } + +protected: + MockIndex index_; + MockIdMap id_map_; +}; + +class FaissWrapperParametrizedRangeSearchTestFixture : public testing::TestWithParam { +public: + FaissWrapperParametrizedRangeSearchTestFixture() : index_(3), id_map_(&index_) { + index_.hnsw.efSearch = 100; // assigning 100 to make sure default of 16 is not used anywhere + } protected: MockIndex index_; @@ -93,13 +127,11 @@ namespace query_index_test { std::unordered_map methodParams; - TEST_P(FaissWrappeterParametrizedTestFixture, QueryIndexHNSWTests) { - //Given + // Given JNIEnv *jniEnv = nullptr; NiceMock mockJNIUtil; - QueryIndexHNSWTestInput const &input = GetParam(); float query[] = {1.2, 2.3, 3.4}; @@ -137,7 +169,7 @@ namespace query_index_test { reinterpret_cast(&query), input.k, reinterpret_cast(&methodParams), reinterpret_cast(parentIdPtr)); - //Then + // Then int actualEfSearch = id_map_.paramsCalled->efSearch; // Asserting the captured argument EXPECT_EQ(input.k, id_map_.kCalled); @@ -165,11 +197,10 @@ namespace query_index_test { namespace query_index_with_filter_test { TEST_P(FaissWrappeterParametrizedTestFixture, QueryIndexWithFilterHNSWTests) { - //Given + // Given JNIEnv *jniEnv = nullptr; NiceMock mockJNIUtil; - QueryIndexHNSWTestInput const &input = GetParam(); float query[] = {1.2, 2.3, 3.4}; @@ -218,7 +249,7 @@ namespace query_index_with_filter_test { input.filterIdType, reinterpret_cast(parentIdPtr)); - //Then + // Then int actualEfSearch = id_map_.paramsCalled->efSearch; // Asserting the captured argument EXPECT_EQ(input.k, id_map_.kCalled); @@ -249,3 +280,93 @@ namespace query_index_with_filter_test { ) ); } + +namespace range_search_test { + + TEST_P(FaissWrapperParametrizedRangeSearchTestFixture, RangeSearchHNSWTests) { + // Given + JNIEnv *jniEnv = nullptr; + NiceMock mockJNIUtil; + + RangeSearchTestInput const &input = GetParam(); + float query[] = {1.2, 2.3, 3.4}; + float radius = input.radius; + int maxResultWindow = 100; // Set your max result window + + std::unordered_map methodParams; + int efSearch = input.efSearch; + int expectedEfSearch = 100; // default set in mock + if (efSearch != -1) { + expectedEfSearch = input.efSearch; + methodParams[knn_jni::EF_SEARCH] = reinterpret_cast(&efSearch); + } + + std::vector *parentIdPtr = nullptr; + if (input.parentIdsPresent) { + std::vector parentId; + parentId.reserve(2); + parentId.push_back(1); + parentId.push_back(2); + parentIdPtr = &parentId; + + EXPECT_CALL(mockJNIUtil, + GetJavaIntArrayLength( + jniEnv, reinterpret_cast(parentIdPtr))) + .WillOnce(testing::Return(parentId.size())); + + EXPECT_CALL(mockJNIUtil, + GetIntArrayElements( + jniEnv, reinterpret_cast(parentIdPtr), nullptr)) + .WillOnce(testing::Return(new int[2]{1, 2})); + } + + std::vector filter; + std::vector *filterptr = nullptr; + if (input.filterIdsPresent) { + filter.reserve(2); + filter.push_back(1); + filter.push_back(2); + filterptr = &filter; + } + + // When + knn_jni::faiss_wrapper::RangeSearchWithFilter( + &mockJNIUtil, jniEnv, + reinterpret_cast(&id_map_), + reinterpret_cast(&query), radius, reinterpret_cast(&methodParams), + maxResultWindow, + reinterpret_cast(filterptr), + input.filterIdType, + reinterpret_cast(parentIdPtr)); + + // Then + int actualEfSearch = id_map_.paramsCalled->efSearch; + // Asserting the captured argument + EXPECT_EQ(expectedEfSearch, actualEfSearch); + if (input.parentIdsPresent) { + faiss::IDGrouper *grouper = id_map_.paramsCalled->grp; + EXPECT_TRUE(grouper != nullptr); + } + if (input.filterIdsPresent) { + faiss::IDSelector *sel = id_map_.paramsCalled->sel; + EXPECT_TRUE(sel != nullptr); + } + id_map_.resetMock(); + } + + INSTANTIATE_TEST_CASE_P( + RangeSearchHNSWTests, + FaissWrapperParametrizedRangeSearchTestFixture, + ::testing::Values( + RangeSearchTestInput{"algoParams present, parent absent, filter absent", 10.0f, 200, 0, false, false}, + RangeSearchTestInput{"algoParams present, parent absent, filter absent, filter type 1", 10.0f, 200, 1, false, false}, + RangeSearchTestInput{"algoParams absent, parent absent, filter present", 10.0f, -1, 0, true, false}, + RangeSearchTestInput{"algoParams absent, parent absent, filter present, filter type 1", 10.0f, -1, 1, true, false}, + RangeSearchTestInput{"algoParams present, parent present, filter absent", 10.0f, 200, 0, false, true}, + RangeSearchTestInput{"algoParams present, parent present, filter absent, filter type 1", 10.0f, 150, 1, false, true}, + RangeSearchTestInput{"algoParams absent, parent present, filter present", 10.0f, -1, 0, true, true}, + RangeSearchTestInput{"algoParams absent, parent present, filter present, filter type 1", 10.0f, -1, 1, true, true} + ) + ); +} + diff --git a/src/main/java/org/opensearch/knn/index/VectorQueryType.java b/src/main/java/org/opensearch/knn/index/VectorQueryType.java index 4697a917e..fb7bfaafd 100644 --- a/src/main/java/org/opensearch/knn/index/VectorQueryType.java +++ b/src/main/java/org/opensearch/knn/index/VectorQueryType.java @@ -54,4 +54,8 @@ public KNNCounter getQueryWithFilterStatCounter() { public abstract KNNCounter getQueryStatCounter(); public abstract KNNCounter getQueryWithFilterStatCounter(); + + public boolean isRadialSearch() { + return this == MAX_DISTANCE || this == MIN_SCORE; + } } diff --git a/src/main/java/org/opensearch/knn/index/query/KNNQuery.java b/src/main/java/org/opensearch/knn/index/query/KNNQuery.java index d123cc149..4b875d9a8 100644 --- a/src/main/java/org/opensearch/knn/index/query/KNNQuery.java +++ b/src/main/java/org/opensearch/knn/index/query/KNNQuery.java @@ -7,6 +7,7 @@ import lombok.AllArgsConstructor; import lombok.Builder; +import lombok.EqualsAndHashCode; import lombok.Getter; import lombok.Setter; import org.apache.lucene.search.BooleanClause; @@ -205,6 +206,7 @@ private boolean equalsTo(KNNQuery other) { @Setter @Getter @AllArgsConstructor + @EqualsAndHashCode public static class Context { int maxResultWindow; } diff --git a/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java b/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java index 1dec98c90..61ffc567e 100644 --- a/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java +++ b/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java @@ -36,6 +36,7 @@ import org.opensearch.knn.index.mapper.KNNVectorFieldMapper; import org.opensearch.knn.index.query.parser.MethodParametersParser; import org.opensearch.knn.index.util.KNNEngine; +import org.opensearch.knn.index.util.QueryContext; import org.opensearch.knn.indices.ModelDao; import org.opensearch.knn.indices.ModelMetadata; import org.opensearch.knn.indices.ModelUtil; @@ -215,9 +216,7 @@ private void validate() { throw new IllegalArgumentException(String.format("[%s] requires exactly one of k, distance or score to be set", NAME)); } - VectorQueryType vectorQueryType = VectorQueryType.MAX_DISTANCE; if (k != null) { - vectorQueryType = VectorQueryType.K; if (k <= 0 || k > K_MAX) { final String errorMessage = "[" + NAME + "] requires k to be in the range (0, " + K_MAX + "]"; throw new IllegalArgumentException(errorMessage); @@ -225,7 +224,6 @@ private void validate() { } if (minScore != null) { - vectorQueryType = VectorQueryType.MIN_SCORE; if (minScore <= 0) { throw new IllegalArgumentException(String.format("[%s] requires minScore to be greater than 0", NAME)); } @@ -239,12 +237,6 @@ private void validate() { ); } } - - // Update stats - vectorQueryType.getQueryStatCounter().increment(); - if (filter != null) { - vectorQueryType.getQueryWithFilterStatCounter().increment(); - } } } @@ -529,6 +521,8 @@ protected Query doToQuery(QueryShardContext context) { KNNEngine knnEngine = KNNEngine.DEFAULT; VectorDataType vectorDataType = knnVectorFieldType.getVectorDataType(); SpaceType spaceType = knnVectorFieldType.getSpaceType(); + VectorQueryType vectorQueryType = getVectorQueryType(k, maxDistance, minScore); + updateQueryStats(vectorQueryType); if (fieldDimension == -1) { if (spaceType != null) { @@ -551,16 +545,18 @@ protected Query doToQuery(QueryShardContext context) { final String method = methodComponentContext != null ? methodComponentContext.getName() : null; if (StringUtils.isNotBlank(method)) { final EngineSpecificMethodContext engineSpecificMethodContext = knnEngine.getMethodContext(method); + QueryContext queryContext = new QueryContext(vectorQueryType); ValidationException validationException = validateParameters( - engineSpecificMethodContext.supportedMethodParameters(), + engineSpecificMethodContext.supportedMethodParameters(queryContext), (Map) methodParameters ); if (validationException != null) { throw new IllegalArgumentException( String.format( - "Parameters not valid for [%s]:[%s] combination: [%s]", + "Parameters not valid for [%s]:[%s]:[%s] combination: [%s]", knnEngine, method, + vectorQueryType.getQueryTypeName(), validationException.getMessage() ) ); @@ -641,6 +637,7 @@ protected Query doToQuery(QueryShardContext context) { .byteVector(VectorDataType.BYTE == vectorDataType ? byteVector : null) .vectorDataType(vectorDataType) .radius(radius) + .methodParameters(this.methodParameters) .filter(this.filter) .context(context) .build(); @@ -663,6 +660,38 @@ private ModelMetadata getModelMetadataForField(KNNVectorFieldMapper.KNNVectorFie return modelMetadata; } + /** + * Function to get the vector query type based on the valid query parameter. + * + * @param k K nearest neighbours for the given vector, if k is set, then the query type is K + * @param maxDistance Maximum distance for the given vector, if maxDistance is set, then the query type is MAX_DISTANCE + * @param minScore Minimum score for the given vector, if minScore is set, then the query type is MIN_SCORE + */ + private VectorQueryType getVectorQueryType(int k, Float maxDistance, Float minScore) { + if (maxDistance != null) { + return VectorQueryType.MAX_DISTANCE; + } + if (minScore != null) { + return VectorQueryType.MIN_SCORE; + } + if (k != 0) { + return VectorQueryType.K; + } + throw new IllegalArgumentException(String.format("[%s] requires exactly one of k, distance or score to be set", NAME)); + } + + /** + * Function to update query stats. + * + * @param vectorQueryType The type of query to be executed + */ + private void updateQueryStats(VectorQueryType vectorQueryType) { + vectorQueryType.getQueryStatCounter().increment(); + if (filter != null) { + vectorQueryType.getQueryWithFilterStatCounter().increment(); + } + } + @Override protected boolean doEquals(KNNQueryBuilder other) { return Objects.equals(fieldName, other.fieldName) diff --git a/src/main/java/org/opensearch/knn/index/query/KNNWeight.java b/src/main/java/org/opensearch/knn/index/query/KNNWeight.java index 794c9af1c..539a08a02 100644 --- a/src/main/java/org/opensearch/knn/index/query/KNNWeight.java +++ b/src/main/java/org/opensearch/knn/index/query/KNNWeight.java @@ -294,6 +294,7 @@ private Map doANNSearch(final LeafReaderContext context, final B indexAllocation.getMemoryAddress(), knnQuery.getQueryVector(), knnQuery.getRadius(), + knnQuery.getMethodParameters(), knnEngine, knnQuery.getContext().getMaxResultWindow(), filterIds, diff --git a/src/main/java/org/opensearch/knn/index/query/RNNQueryFactory.java b/src/main/java/org/opensearch/knn/index/query/RNNQueryFactory.java index db8084864..dd5efc93f 100644 --- a/src/main/java/org/opensearch/knn/index/query/RNNQueryFactory.java +++ b/src/main/java/org/opensearch/knn/index/query/RNNQueryFactory.java @@ -10,6 +10,7 @@ import static org.opensearch.knn.index.VectorDataType.SUPPORTED_VECTOR_DATA_TYPES; import java.util.Locale; +import java.util.Map; import lombok.extern.log4j.Log4j2; import org.apache.lucene.search.ByteVectorSimilarityQuery; @@ -69,6 +70,7 @@ public static Query create(RNNQueryFactory.CreateQueryRequest createQueryRequest final byte[] byteVector = createQueryRequest.getByteVector(); final VectorDataType vectorDataType = createQueryRequest.getVectorDataType(); final Query filterQuery = getFilterQuery(createQueryRequest); + final Map methodParameters = createQueryRequest.getMethodParameters(); if (KNNEngine.getEnginesThatCreateCustomSegmentFiles().contains(createQueryRequest.getKnnEngine())) { BitSetProducer parentFilter = null; @@ -79,15 +81,17 @@ public static Query create(RNNQueryFactory.CreateQueryRequest createQueryRequest } IndexSettings indexSettings = context.getIndexSettings(); KNNQuery.Context knnQueryContext = new KNNQuery.Context(indexSettings.getMaxResultWindow()); - KNNQuery rnnQuery = new KNNQuery(fieldName, vector, indexName, parentFilter).radius(radius).kNNQueryContext(knnQueryContext); - if (filterQuery != null && KNNEngine.getEnginesThatSupportsFilters().contains(createQueryRequest.getKnnEngine())) { - log.debug("Creating custom radius search with filters for index: {}, field: {} , r: {}", indexName, fieldName, radius); - rnnQuery.filterQuery(filterQuery); - } - log.debug( - String.format("Creating custom radius search for index: %s \"\", field: %s \"\", r: %f", indexName, fieldName, radius) - ); - return rnnQuery; + + return KNNQuery.builder() + .field(fieldName) + .queryVector(vector) + .indexName(indexName) + .parentsFilter(parentFilter) + .radius(radius) + .methodParameters(methodParameters) + .context(knnQueryContext) + .filterQuery(filterQuery) + .build(); } log.debug(String.format("Creating Lucene r-NN query for index: %s \"\", field: %s \"\", k: %f", indexName, fieldName, radius)); diff --git a/src/main/java/org/opensearch/knn/index/util/DefaultHnswContext.java b/src/main/java/org/opensearch/knn/index/util/DefaultHnswContext.java index c16f1b05e..c2bbb9e6f 100644 --- a/src/main/java/org/opensearch/knn/index/util/DefaultHnswContext.java +++ b/src/main/java/org/opensearch/knn/index/util/DefaultHnswContext.java @@ -27,7 +27,7 @@ public final class DefaultHnswContext implements EngineSpecificMethodContext { .build(); @Override - public Map> supportedMethodParameters() { + public Map> supportedMethodParameters(QueryContext ctx) { return supportedMethodParameters; } } diff --git a/src/main/java/org/opensearch/knn/index/util/EngineSpecificMethodContext.java b/src/main/java/org/opensearch/knn/index/util/EngineSpecificMethodContext.java index f669704ad..edb8e830a 100644 --- a/src/main/java/org/opensearch/knn/index/util/EngineSpecificMethodContext.java +++ b/src/main/java/org/opensearch/knn/index/util/EngineSpecificMethodContext.java @@ -25,7 +25,7 @@ */ public interface EngineSpecificMethodContext { - Map> supportedMethodParameters(); + Map> supportedMethodParameters(QueryContext ctx); - EngineSpecificMethodContext EMPTY = Collections::emptyMap; + EngineSpecificMethodContext EMPTY = ctx -> Collections.emptyMap(); } diff --git a/src/main/java/org/opensearch/knn/index/util/Lucene.java b/src/main/java/org/opensearch/knn/index/util/Lucene.java index ae6ea3a70..d98775f94 100644 --- a/src/main/java/org/opensearch/knn/index/util/Lucene.java +++ b/src/main/java/org/opensearch/knn/index/util/Lucene.java @@ -67,7 +67,7 @@ public class Lucene extends JVMLibrary { * @param distanceTransform Map of space type to distance transformation function */ Lucene(Map methods, String version, Map> distanceTransform) { - super(methods, Map.of(METHOD_HNSW, new DefaultHnswContext()), version); + super(methods, Map.of(METHOD_HNSW, new LuceneHNSWContext()), version); this.distanceTransform = distanceTransform; } diff --git a/src/main/java/org/opensearch/knn/index/util/LuceneHNSWContext.java b/src/main/java/org/opensearch/knn/index/util/LuceneHNSWContext.java new file mode 100644 index 000000000..d9b6ba1c3 --- /dev/null +++ b/src/main/java/org/opensearch/knn/index/util/LuceneHNSWContext.java @@ -0,0 +1,36 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.knn.index.util; + +import com.google.common.collect.ImmutableMap; +import org.opensearch.knn.index.Parameter; +import org.opensearch.knn.index.query.request.MethodParameter; + +import java.util.Collections; +import java.util.Map; + +public class LuceneHNSWContext implements EngineSpecificMethodContext { + + private final Map> supportedMethodParameters = ImmutableMap.>builder() + .put(MethodParameter.EF_SEARCH.getName(), new Parameter.IntegerParameter(MethodParameter.EF_SEARCH.getName(), null, value -> true)) + .build(); + + @Override + public Map> supportedMethodParameters(QueryContext ctx) { + if (ctx.queryType.isRadialSearch()) { + // return empty map if radial search is true + return Collections.emptyMap(); + } + // Return the supported method parameters for non-radial cases + return supportedMethodParameters; + } +} diff --git a/src/main/java/org/opensearch/knn/index/util/QueryContext.java b/src/main/java/org/opensearch/knn/index/util/QueryContext.java new file mode 100644 index 000000000..6bb495814 --- /dev/null +++ b/src/main/java/org/opensearch/knn/index/util/QueryContext.java @@ -0,0 +1,23 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.knn.index.util; + +import lombok.AllArgsConstructor; +import org.opensearch.knn.index.VectorQueryType; + +/** + * Context class for query-specific information. + */ +@AllArgsConstructor +public class QueryContext { + VectorQueryType queryType; +} diff --git a/src/main/java/org/opensearch/knn/jni/FaissService.java b/src/main/java/org/opensearch/knn/jni/FaissService.java index 77b786421..2c537cf00 100644 --- a/src/main/java/org/opensearch/knn/jni/FaissService.java +++ b/src/main/java/org/opensearch/knn/jni/FaissService.java @@ -204,6 +204,7 @@ public static native KNNQueryResult[] queryIndexWithFilter( * @param indexPointer pointer to index in memory * @param queryVector vector to be used for query * @param radius search within radius threshold + * @param methodParameters parameters to be used for the query * @param indexMaxResultWindow maximum number of results to return * @param filteredIds list of doc ids to include in the query result * @param filterIdsType type of filter ids @@ -214,6 +215,7 @@ public static native KNNQueryResult[] rangeSearchIndexWithFilter( long indexPointer, float[] queryVector, float radius, + Map methodParameters, int indexMaxResultWindow, long[] filteredIds, int filterIdsType, @@ -226,6 +228,7 @@ public static native KNNQueryResult[] rangeSearchIndexWithFilter( * @param indexPointer pointer to index in memory * @param queryVector vector to be used for query * @param radius search within radius threshold + * @param methodParameters parameters to be used for the query * @param indexMaxResultWindow maximum number of results to return * @param parentIds list of parent doc ids when the knn field is a nested field * @return KNNQueryResult array of neighbors within radius @@ -234,6 +237,7 @@ public static native KNNQueryResult[] rangeSearchIndex( long indexPointer, float[] queryVector, float radius, + Map methodParameters, int indexMaxResultWindow, int[] parentIds ); diff --git a/src/main/java/org/opensearch/knn/jni/JNIService.java b/src/main/java/org/opensearch/knn/jni/JNIService.java index 6563da296..83afc592f 100644 --- a/src/main/java/org/opensearch/knn/jni/JNIService.java +++ b/src/main/java/org/opensearch/knn/jni/JNIService.java @@ -279,6 +279,7 @@ public static long transferVectors(long vectorsPointer, float[][] trainingData) * @param indexPointer pointer to index in memory * @param queryVector vector to be used for query * @param radius search within radius threshold + * @param methodParameters parameters to be used when loading index * @param knnEngine engine to query index * @param indexMaxResultWindow maximum number of results to return * @param filteredIds list of doc ids to include in the query result @@ -290,6 +291,7 @@ public static KNNQueryResult[] radiusQueryIndex( long indexPointer, float[] queryVector, float radius, + @Nullable Map methodParameters, KNNEngine knnEngine, int indexMaxResultWindow, long[] filteredIds, @@ -302,13 +304,14 @@ public static KNNQueryResult[] radiusQueryIndex( indexPointer, queryVector, radius, + methodParameters, indexMaxResultWindow, filteredIds, filterIdsType, parentIds ); } - return FaissService.rangeSearchIndex(indexPointer, queryVector, radius, indexMaxResultWindow, parentIds); + return FaissService.rangeSearchIndex(indexPointer, queryVector, radius, methodParameters, indexMaxResultWindow, parentIds); } throw new IllegalArgumentException("RadiusQueryIndex not supported for provided engine"); } diff --git a/src/test/java/org/opensearch/knn/index/FaissIT.java b/src/test/java/org/opensearch/knn/index/FaissIT.java index 2e7f772b0..0af06b7e5 100644 --- a/src/test/java/org/opensearch/knn/index/FaissIT.java +++ b/src/test/java/org/opensearch/knn/index/FaissIT.java @@ -59,6 +59,7 @@ import static org.opensearch.knn.common.KNNConstants.METHOD_ENCODER_PARAMETER; import static org.opensearch.knn.common.KNNConstants.METHOD_HNSW; import static org.opensearch.knn.common.KNNConstants.METHOD_IVF; +import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER; import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_NLIST; import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_NPROBES; import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_SPACE_TYPE; @@ -141,7 +142,7 @@ public void testEndToEnd_whenDoRadiusSearch_whenDistanceThreshold_whenMethodIsHN assertEquals(testData.indexData.docs.length, getDocCount(INDEX_NAME)); float distance = 300000000000f; - validateRadiusSearchResults(INDEX_NAME, FIELD_NAME, testData.queries, distance, null, spaceType, null); + validateRadiusSearchResults(INDEX_NAME, FIELD_NAME, testData.queries, distance, null, spaceType, null, null); // Delete index deleteKNNIndex(INDEX_NAME); @@ -201,7 +202,7 @@ public void testEndToEnd_whenDoRadiusSearch_whenScoreThreshold_whenMethodIsHNSWF float score = 0.00001f; - validateRadiusSearchResults(INDEX_NAME, FIELD_NAME, testData.queries, null, score, spaceType, null); + validateRadiusSearchResults(INDEX_NAME, FIELD_NAME, testData.queries, null, score, spaceType, null, null); // Delete index deleteKNNIndex(INDEX_NAME); @@ -261,7 +262,7 @@ public void testEndToEnd_whenDoRadiusSearch_whenMoreThanOneScoreThreshold_whenMe float score = 5f; - validateRadiusSearchResults(INDEX_NAME, FIELD_NAME, testData.queries, null, score, spaceType, null); + validateRadiusSearchResults(INDEX_NAME, FIELD_NAME, testData.queries, null, score, spaceType, null, null); // Delete index deleteKNNIndex(INDEX_NAME); @@ -345,8 +346,11 @@ public void testEndToEnd_whenDoRadiusSearch_whenDistanceThreshold_whenMethodIsHN assertEquals(testData.indexData.docs.length, getDocCount(indexName)); float distance = 300000000000f; + // create method parameter wih ef_search + Map methodParameters = new ImmutableMap.Builder().put(KNNConstants.METHOD_PARAMETER_EF_SEARCH, 150) + .build(); - validateRadiusSearchResults(indexName, fieldName, testData.queries, distance, null, spaceType, null); + validateRadiusSearchResults(indexName, fieldName, testData.queries, distance, null, spaceType, null, methodParameters); // Delete index deleteKNNIndex(indexName); @@ -381,7 +385,8 @@ public void testRadialQuery_withFilter_thenSuccess() { distance, null, SpaceType.L2, - termQueryBuilder + termQueryBuilder, + null ); assertEquals(1, queryResult.get(0).size()); @@ -1663,7 +1668,8 @@ private List> validateRadiusSearchResults( Float distanceThreshold, Float scoreThreshold, final SpaceType spaceType, - TermQueryBuilder filterQuery + TermQueryBuilder filterQuery, + Map methodParameters ) throws IOException, ParseException { List> queryResults = new ArrayList<>(); for (float[] queryVector : queryVectors) { @@ -1681,6 +1687,13 @@ private List> validateRadiusSearchResults( if (filterQuery != null) { queryBuilder.field("filter", filterQuery); } + if (methodParameters != null) { + queryBuilder.startObject(METHOD_PARAMETER); + for (Map.Entry entry : methodParameters.entrySet()) { + queryBuilder.field(entry.getKey(), entry.getValue()); + } + queryBuilder.endObject(); + } queryBuilder.endObject(); queryBuilder.endObject(); queryBuilder.endObject().endObject(); diff --git a/src/test/java/org/opensearch/knn/index/LuceneEngineIT.java b/src/test/java/org/opensearch/knn/index/LuceneEngineIT.java index 38895246a..f5821bf07 100644 --- a/src/test/java/org/opensearch/knn/index/LuceneEngineIT.java +++ b/src/test/java/org/opensearch/knn/index/LuceneEngineIT.java @@ -36,6 +36,7 @@ import static org.opensearch.knn.common.KNNConstants.MAX_DISTANCE; import static org.opensearch.knn.common.KNNConstants.METHOD_HNSW; +import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER; import static org.opensearch.knn.common.KNNConstants.MIN_SCORE; import static org.opensearch.knn.common.KNNConstants.NMSLIB_NAME; import static org.opensearch.knn.common.KNNConstants.VECTOR_DATA_TYPE_FIELD; @@ -339,7 +340,7 @@ public void testRadiusSearch_usingDistanceThreshold_usingL2Metrics_usingFloatTyp final float distance = 3.5f; final int[] expectedResults = { 2, 3, 2 }; - validateRadiusSearchResults(TEST_QUERY_VECTORS, distance, null, SpaceType.L2, expectedResults, null, null); + validateRadiusSearchResults(TEST_QUERY_VECTORS, distance, null, SpaceType.L2, expectedResults, null, null, null); } public void testRadiusSearch_usingScoreThreshold_usingL2Metrics_usingFloatType() throws Exception { @@ -351,7 +352,7 @@ public void testRadiusSearch_usingScoreThreshold_usingL2Metrics_usingFloatType() final float score = 0.23f; final int[] expectedResults = { 2, 3, 2 }; - validateRadiusSearchResults(TEST_QUERY_VECTORS, null, score, SpaceType.L2, expectedResults, null, null); + validateRadiusSearchResults(TEST_QUERY_VECTORS, null, score, SpaceType.L2, expectedResults, null, null, null); } public void testRadiusSearch_usingDistanceThreshold_usingCosineMetrics_usingFloatType() throws Exception { @@ -363,7 +364,7 @@ public void testRadiusSearch_usingDistanceThreshold_usingCosineMetrics_usingFloa final float distance = 0.03f; final int[] expectedResults = { 1, 1, 1 }; - validateRadiusSearchResults(TEST_QUERY_VECTORS, distance, null, SpaceType.COSINESIMIL, expectedResults, null, null); + validateRadiusSearchResults(TEST_QUERY_VECTORS, distance, null, SpaceType.COSINESIMIL, expectedResults, null, null, null); } public void testRadiusSearch_usingScoreThreshold_usingCosineMetrics_usingFloatType() throws Exception { @@ -375,7 +376,7 @@ public void testRadiusSearch_usingScoreThreshold_usingCosineMetrics_usingFloatTy final float score = 0.97f; final int[] expectedResults = { 1, 1, 1 }; - validateRadiusSearchResults(TEST_QUERY_VECTORS, null, score, SpaceType.COSINESIMIL, expectedResults, null, null); + validateRadiusSearchResults(TEST_QUERY_VECTORS, null, score, SpaceType.COSINESIMIL, expectedResults, null, null, null); } public void testRadiusSearch_usingScoreThreshold_usingInnerProductMetrics_usingFloatType() throws Exception { @@ -387,7 +388,7 @@ public void testRadiusSearch_usingScoreThreshold_usingInnerProductMetrics_usingF final float score = 2f; final int[] expectedResults = { 1, 1, 1 }; - validateRadiusSearchResults(TEST_QUERY_VECTORS, null, score, SpaceType.INNER_PRODUCT, expectedResults, null, null); + validateRadiusSearchResults(TEST_QUERY_VECTORS, null, score, SpaceType.INNER_PRODUCT, expectedResults, null, null, null); } public void testRadiusSearch_usingDistanceThreshold_usingL2Metrics_usingByteType() throws Exception { @@ -399,7 +400,7 @@ public void testRadiusSearch_usingDistanceThreshold_usingL2Metrics_usingByteType final float distance = 3.5f; final int[] expectedResults = { 2, 2, 2 }; - validateRadiusSearchResults(TEST_QUERY_VECTORS, distance, null, SpaceType.L2, expectedResults, null, null); + validateRadiusSearchResults(TEST_QUERY_VECTORS, distance, null, SpaceType.L2, expectedResults, null, null, null); } public void testRadiusSearch_usingScoreThreshold_usingL2Metrics_usingByteType() throws Exception { @@ -411,7 +412,7 @@ public void testRadiusSearch_usingScoreThreshold_usingL2Metrics_usingByteType() final float score = 0.23f; final int[] expectedResults = { 2, 2, 2 }; - validateRadiusSearchResults(TEST_QUERY_VECTORS, null, score, SpaceType.L2, expectedResults, null, null); + validateRadiusSearchResults(TEST_QUERY_VECTORS, null, score, SpaceType.L2, expectedResults, null, null, null); } public void testRadiusSearch_usingDistanceThreshold_usingCosineMetrics_usingByteType() throws Exception { @@ -423,7 +424,7 @@ public void testRadiusSearch_usingDistanceThreshold_usingCosineMetrics_usingByte final float distance = 0.05f; final int[] expectedResults = { 2, 2, 2 }; - validateRadiusSearchResults(TEST_QUERY_VECTORS, distance, null, SpaceType.COSINESIMIL, expectedResults, null, null); + validateRadiusSearchResults(TEST_QUERY_VECTORS, distance, null, SpaceType.COSINESIMIL, expectedResults, null, null, null); } public void testRadiusSearch_usingScoreThreshold_usingCosineMetrics_usingByteType() throws Exception { @@ -435,7 +436,7 @@ public void testRadiusSearch_usingScoreThreshold_usingCosineMetrics_usingByteTyp final float score = 0.97f; final int[] expectedResults = { 2, 2, 2 }; - validateRadiusSearchResults(TEST_QUERY_VECTORS, null, score, SpaceType.COSINESIMIL, expectedResults, null, null); + validateRadiusSearchResults(TEST_QUERY_VECTORS, null, score, SpaceType.COSINESIMIL, expectedResults, null, null, null); } public void testRadiusSearch_usingDistanceThreshold_withFilter_usingL2Metrics_usingFloatType() throws Exception { @@ -449,7 +450,7 @@ public void testRadiusSearch_usingDistanceThreshold_withFilter_usingL2Metrics_us final float distance = 45.0f; final int[] expectedResults = { 1, 1, 1 }; - validateRadiusSearchResults(TEST_QUERY_VECTORS, distance, null, SpaceType.L2, expectedResults, COLOR_FIELD_NAME, "red"); + validateRadiusSearchResults(TEST_QUERY_VECTORS, distance, null, SpaceType.L2, expectedResults, COLOR_FIELD_NAME, "red", null); } public void testRadiusSearch_usingScoreThreshold_withFilter_usingCosineMetrics_usingFloatType() throws Exception { @@ -463,7 +464,7 @@ public void testRadiusSearch_usingScoreThreshold_withFilter_usingCosineMetrics_u final float score = 0.02f; final int[] expectedResults = { 1, 1, 1 }; - validateRadiusSearchResults(TEST_QUERY_VECTORS, null, score, SpaceType.COSINESIMIL, expectedResults, COLOR_FIELD_NAME, "red"); + validateRadiusSearchResults(TEST_QUERY_VECTORS, null, score, SpaceType.COSINESIMIL, expectedResults, COLOR_FIELD_NAME, "red", null); } private void createKnnIndexMappingWithLuceneEngine(int dimension, SpaceType spaceType, VectorDataType vectorDataType) throws Exception { @@ -642,6 +643,25 @@ public void test_whenUsingIP_thenSuccess() { } } + @SneakyThrows + public void testRadialSearch_whenEfSearchIsSet_thenThrowException() { + createKnnIndexMappingWithLuceneEngine(DIMENSION, SpaceType.L2, VectorDataType.FLOAT); + for (int j = 0; j < TEST_INDEX_VECTORS.length; j++) { + addKnnDoc(INDEX_NAME, Integer.toString(j + 1), FIELD_NAME, TEST_INDEX_VECTORS[j]); + } + + final float score = 0.23f; + final int[] expectedResults = { 2, 3, 2 }; + + Map methodParameters = new ImmutableMap.Builder().put(KNNConstants.METHOD_PARAMETER_EF_SEARCH, 150) + .build(); + + expectThrows( + ResponseException.class, + () -> validateRadiusSearchResults(TEST_QUERY_VECTORS, null, score, SpaceType.L2, expectedResults, null, null, methodParameters) + ); + } + private void validateRadiusSearchResults( final float[][] searchVectors, final Float distanceThreshold, @@ -649,7 +669,8 @@ private void validateRadiusSearchResults( final SpaceType spaceType, final int[] expectedResults, @Nullable final String filterField, - @Nullable final String filterValue + @Nullable final String filterValue, + @Nullable final Map methodParameters ) throws Exception { for (int i = 0; i < searchVectors.length; i++) { XContentBuilder builder = XContentFactory.jsonBuilder().startObject().startObject("query"); @@ -670,6 +691,13 @@ private void validateRadiusSearchResults( builder.endObject(); builder.endObject(); } + if (methodParameters != null) { + builder.startObject(METHOD_PARAMETER); + for (Map.Entry entry : methodParameters.entrySet()) { + builder.field(entry.getKey(), entry.getValue()); + } + builder.endObject(); + } builder.endObject(); builder.endObject(); builder.endObject().endObject(); diff --git a/src/test/java/org/opensearch/knn/index/query/KNNQueryBuilderTests.java b/src/test/java/org/opensearch/knn/index/query/KNNQueryBuilderTests.java index 10c0155ae..ca191b19a 100644 --- a/src/test/java/org/opensearch/knn/index/query/KNNQueryBuilderTests.java +++ b/src/test/java/org/opensearch/knn/index/query/KNNQueryBuilderTests.java @@ -161,7 +161,7 @@ public void testEmptyVector() { public void testFromXContent() throws Exception { float[] queryVector = { 1.0f, 2.0f, 3.0f, 4.0f }; - KNNQueryBuilder knnQueryBuilder = new KNNQueryBuilder(FIELD_NAME, queryVector, K); + KNNQueryBuilder knnQueryBuilder = KNNQueryBuilder.builder().fieldName(FIELD_NAME).vector(queryVector).k(K).build(); XContentBuilder builder = XContentFactory.jsonBuilder(); builder.startObject(); builder.startObject(knnQueryBuilder.fieldName()); @@ -199,18 +199,22 @@ public void testFromXContent_KnnWithMethodParameters() throws Exception { assertEquals(knnQueryBuilder, actualBuilder); } - public void testFromXContent_whenDoRadiusSearch_whenDistanceThreshold_thenSucceed() throws Exception { + public void testFromXContent_whenDoRadiusSearch_whenDistanceThreshold_whenMethodParameter_thenSucceed() throws Exception { float[] queryVector = { 1.0f, 2.0f, 3.0f, 4.0f }; KNNQueryBuilder knnQueryBuilder = KNNQueryBuilder.builder() .fieldName(FIELD_NAME) .vector(queryVector) .maxDistance(MAX_DISTANCE) + .methodParameters(HNSW_METHOD_PARAMS) .build(); XContentBuilder builder = XContentFactory.jsonBuilder(); builder.startObject(); builder.startObject(knnQueryBuilder.fieldName()); builder.field(KNNQueryBuilder.VECTOR_FIELD.getPreferredName(), knnQueryBuilder.vector()); builder.field(KNNQueryBuilder.MAX_DISTANCE_FIELD.getPreferredName(), knnQueryBuilder.getMaxDistance()); + builder.startObject(org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER); + builder.field(EF_SEARCH_FIELD.getPreferredName(), EF_SEARCH); + builder.endObject(); builder.endObject(); builder.endObject(); XContentParser contentParser = createParser(builder); @@ -219,18 +223,22 @@ public void testFromXContent_whenDoRadiusSearch_whenDistanceThreshold_thenSuccee assertEquals(knnQueryBuilder, actualBuilder); } - public void testFromXContent_whenDoRadiusSearch_whenScoreThreshold_thenSucceed() throws Exception { + public void testFromXContent_whenDoRadiusSearch_whenScoreThreshold_whenMethodParameter_thenSucceed() throws Exception { float[] queryVector = { 1.0f, 2.0f, 3.0f, 4.0f }; KNNQueryBuilder knnQueryBuilder = KNNQueryBuilder.builder() .fieldName(FIELD_NAME) .vector(queryVector) .minScore(MAX_DISTANCE) + .methodParameters(HNSW_METHOD_PARAMS) .build(); XContentBuilder builder = XContentFactory.jsonBuilder(); builder.startObject(); builder.startObject(knnQueryBuilder.fieldName()); builder.field(KNNQueryBuilder.VECTOR_FIELD.getPreferredName(), knnQueryBuilder.vector()); builder.field(KNNQueryBuilder.MIN_SCORE_FIELD.getPreferredName(), knnQueryBuilder.getMinScore()); + builder.startObject(org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER); + builder.field(EF_SEARCH_FIELD.getPreferredName(), EF_SEARCH); + builder.endObject(); builder.endObject(); builder.endObject(); XContentParser contentParser = createParser(builder); @@ -246,7 +254,12 @@ public void testFromXContent_withFilter() throws Exception { knnClusterUtil.initialize(clusterService); float[] queryVector = { 1.0f, 2.0f, 3.0f, 4.0f }; - KNNQueryBuilder knnQueryBuilder = new KNNQueryBuilder(FIELD_NAME, queryVector, K, TERM_QUERY); + KNNQueryBuilder knnQueryBuilder = KNNQueryBuilder.builder() + .fieldName(FIELD_NAME) + .vector(queryVector) + .k(K) + .filter(TERM_QUERY) + .build(); XContentBuilder builder = XContentFactory.jsonBuilder(); builder.startObject(); builder.startObject(knnQueryBuilder.fieldName()); @@ -1192,4 +1205,58 @@ public void testRadialSearch_whenUnsupportedEngine_thenThrowException() { expectThrows(UnsupportedOperationException.class, () -> knnQueryBuilder.doToQuery(mockQueryShardContext)); } } + + public void testRadialSearch_whenEfSearchIsSet_whenLuceneEngine_thenThrowException() { + KNNMethodContext knnMethodContext = new KNNMethodContext( + KNNEngine.LUCENE, + SpaceType.L2, + new MethodComponentContext(org.opensearch.knn.common.KNNConstants.METHOD_HNSW, ImmutableMap.of()) + ); + + KNNQueryBuilder knnQueryBuilder = KNNQueryBuilder.builder() + .fieldName(FIELD_NAME) + .vector(QUERY_VECTOR) + .maxDistance(MAX_DISTANCE) + .methodParameters(Map.of("ef_search", EF_SEARCH)) + .build(); + + KNNVectorFieldMapper.KNNVectorFieldType mockKNNVectorField = mock(KNNVectorFieldMapper.KNNVectorFieldType.class); + QueryShardContext mockQueryShardContext = mock(QueryShardContext.class); + Index dummyIndex = new Index("dummy", "dummy"); + when(mockKNNVectorField.getKnnMethodContext()).thenReturn(knnMethodContext); + when(mockQueryShardContext.index()).thenReturn(dummyIndex); + when(mockKNNVectorField.getDimension()).thenReturn(4); + when(mockQueryShardContext.fieldMapper(anyString())).thenReturn(mockKNNVectorField); + + expectThrows(IllegalArgumentException.class, () -> knnQueryBuilder.doToQuery(mockQueryShardContext)); + } + + public void testRadialSearch_whenEfSearchIsSet_whenFaissEngine_thenSuccess() { + KNNMethodContext knnMethodContext = new KNNMethodContext( + KNNEngine.FAISS, + SpaceType.L2, + new MethodComponentContext(org.opensearch.knn.common.KNNConstants.METHOD_HNSW, ImmutableMap.of()) + ); + + KNNQueryBuilder knnQueryBuilder = KNNQueryBuilder.builder() + .fieldName(FIELD_NAME) + .vector(QUERY_VECTOR) + .minScore(MIN_SCORE) + .methodParameters(Map.of("ef_search", EF_SEARCH)) + .build(); + + KNNVectorFieldMapper.KNNVectorFieldType mockKNNVectorField = mock(KNNVectorFieldMapper.KNNVectorFieldType.class); + QueryShardContext mockQueryShardContext = mock(QueryShardContext.class); + Index dummyIndex = new Index("dummy", "dummy"); + when(mockKNNVectorField.getKnnMethodContext()).thenReturn(knnMethodContext); + when(mockQueryShardContext.index()).thenReturn(dummyIndex); + when(mockKNNVectorField.getDimension()).thenReturn(4); + when(mockQueryShardContext.fieldMapper(anyString())).thenReturn(mockKNNVectorField); + IndexSettings indexSettings = mock(IndexSettings.class); + when(mockQueryShardContext.getIndexSettings()).thenReturn(indexSettings); + when(indexSettings.getMaxResultWindow()).thenReturn(1000); + + KNNQuery query = (KNNQuery) knnQueryBuilder.doToQuery(mockQueryShardContext); + assertEquals(1 / MIN_SCORE - 1, query.getRadius(), 0); + } } diff --git a/src/test/java/org/opensearch/knn/index/query/KNNWeightTests.java b/src/test/java/org/opensearch/knn/index/query/KNNWeightTests.java index 021e3a825..75cd6e7a9 100644 --- a/src/test/java/org/opensearch/knn/index/query/KNNWeightTests.java +++ b/src/test/java/org/opensearch/knn/index/query/KNNWeightTests.java @@ -62,7 +62,6 @@ import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyBoolean; -import static org.mockito.ArgumentMatchers.anyFloat; import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.anyString; @@ -763,12 +762,29 @@ public void testDoANNSearch_whenRadialIsDefined_thenCallJniRadiusQueryIndex() { final float radius = 0.5f; final int maxResults = 1000; jniServiceMockedStatic.when( - () -> JNIService.radiusQueryIndex(anyLong(), any(), anyFloat(), any(), anyInt(), any(), anyInt(), any()) + () -> JNIService.radiusQueryIndex( + anyLong(), + eq(queryVector), + eq(radius), + eq(HNSW_METHOD_PARAMETERS), + any(), + eq(maxResults), + any(), + anyInt(), + any() + ) ).thenReturn(getKNNQueryResults()); KNNQuery.Context context = mock(KNNQuery.Context.class); when(context.getMaxResultWindow()).thenReturn(maxResults); - final KNNQuery query = new KNNQuery(FIELD_NAME, queryVector, INDEX_NAME, null).radius(radius).kNNQueryContext(context); + final KNNQuery query = KNNQuery.builder() + .field(FIELD_NAME) + .queryVector(queryVector) + .radius(radius) + .indexName(INDEX_NAME) + .context(context) + .methodParameters(HNSW_METHOD_PARAMETERS) + .build(); final float boost = (float) randomDoubleBetween(0, 10, true); final KNNWeight knnWeight = new KNNWeight(query, boost); @@ -807,7 +823,17 @@ public void testDoANNSearch_whenRadialIsDefined_thenCallJniRadiusQueryIndex() { final KNNScorer knnScorer = (KNNScorer) knnWeight.scorer(leafReaderContext); assertNotNull(knnScorer); jniServiceMockedStatic.verify( - () -> JNIService.radiusQueryIndex(anyLong(), any(), anyFloat(), any(), anyInt(), any(), anyInt(), any()) + () -> JNIService.radiusQueryIndex( + anyLong(), + eq(queryVector), + eq(radius), + eq(HNSW_METHOD_PARAMETERS), + any(), + eq(maxResults), + any(), + anyInt(), + any() + ) ); final DocIdSetIterator docIdSetIterator = knnScorer.iterator(); diff --git a/src/test/java/org/opensearch/knn/index/query/RNNQueryFactoryTests.java b/src/test/java/org/opensearch/knn/index/query/RNNQueryFactoryTests.java index 5492b8506..af415f9c5 100644 --- a/src/test/java/org/opensearch/knn/index/query/RNNQueryFactoryTests.java +++ b/src/test/java/org/opensearch/knn/index/query/RNNQueryFactoryTests.java @@ -9,9 +9,11 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; import static org.opensearch.knn.common.KNNConstants.DEFAULT_VECTOR_DATA_TYPE_FIELD; +import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_EF_SEARCH; import java.util.Arrays; import java.util.List; +import java.util.Map; import java.util.stream.Collectors; import org.apache.lucene.search.ByteVectorSimilarityQuery; @@ -38,6 +40,7 @@ public class RNNQueryFactoryTests extends KNNTestCase { private final String testFieldName = "test-field"; private final Float testRadius = 0.5f; private final int maxResultWindow = 20000; + private final Map methodParameters = Map.of(METHOD_PARAMETER_EF_SEARCH, 100); public void testCreate_whenLucene_withRadiusQuery_withFloatVector() { List luceneDefaultQueryEngineList = Arrays.stream(KNNEngine.values()) @@ -106,12 +109,24 @@ public void testCreate_whenLucene_withFilter_thenSucceed() { } public void testCreate_whenFaiss_thenSucceed() { + // Given QueryShardContext mockQueryShardContext = mock(QueryShardContext.class); MappedFieldType testMapper = mock(MappedFieldType.class); IndexSettings indexSettings = mock(IndexSettings.class); when(mockQueryShardContext.getIndexSettings()).thenReturn(indexSettings); when(mockQueryShardContext.fieldMapper(any())).thenReturn(testMapper); when(mockQueryShardContext.getIndexSettings().getMaxResultWindow()).thenReturn(maxResultWindow); + + final KNNQuery expectedQuery = KNNQuery.builder() + .field(testFieldName) + .queryVector(testQueryVector) + .indexName(testIndexName) + .radius(testRadius) + .methodParameters(methodParameters) + .context(new KNNQuery.Context(maxResultWindow)) + .build(); + + // When final RNNQueryFactory.CreateQueryRequest createQueryRequest = RNNQueryFactory.CreateQueryRequest.builder() .knnEngine(KNNEngine.FAISS) .indexName(testIndexName) @@ -120,15 +135,12 @@ public void testCreate_whenFaiss_thenSucceed() { .radius(testRadius) .vectorDataType(DEFAULT_VECTOR_DATA_TYPE_FIELD) .context(mockQueryShardContext) + .methodParameters(methodParameters) .build(); Query query = RNNQueryFactory.create(createQueryRequest); - assertTrue(query instanceof KNNQuery); - assertEquals(testIndexName, ((KNNQuery) query).getIndexName()); - assertEquals(testFieldName, ((KNNQuery) query).getField()); - assertEquals(testQueryVector, ((KNNQuery) query).getQueryVector()); - assertEquals(testRadius, ((KNNQuery) query).getRadius(), 0); - assertEquals(maxResultWindow, ((KNNQuery) query).getContext().getMaxResultWindow()); + // Then + assertEquals(expectedQuery, query); } } diff --git a/src/test/java/org/opensearch/knn/index/util/AbstractKNNLibraryTests.java b/src/test/java/org/opensearch/knn/index/util/AbstractKNNLibraryTests.java index 0aab78042..8e5ae24f9 100644 --- a/src/test/java/org/opensearch/knn/index/util/AbstractKNNLibraryTests.java +++ b/src/test/java/org/opensearch/knn/index/util/AbstractKNNLibraryTests.java @@ -11,12 +11,7 @@ import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.knn.KNNTestCase; import org.opensearch.knn.common.KNNConstants; -import org.opensearch.knn.index.KNNMethod; -import org.opensearch.knn.index.KNNMethodContext; -import org.opensearch.knn.index.MethodComponent; -import org.opensearch.knn.index.MethodComponentContext; -import org.opensearch.knn.index.Parameter; -import org.opensearch.knn.index.SpaceType; +import org.opensearch.knn.index.*; import java.io.IOException; import java.util.Collections; @@ -78,9 +73,13 @@ public ValidationException validate(KNNMethodContext knnMethodContext) { assertNotNull(testAbstractKNNLibrary2.validateMethod(knnMethodContext2)); } - public void testEngineSpecificMethods() throws IOException { + public void testEngineSpecificMethods() { String methodName1 = "test-method-1"; - EngineSpecificMethodContext context = () -> Map.of("myparameter", new Parameter.BooleanParameter("myparameter", false, o -> o)); + QueryContext engineSpecificMethodContext = new QueryContext(VectorQueryType.K); + EngineSpecificMethodContext context = ctx -> ImmutableMap.of( + "myparameter", + new Parameter.BooleanParameter("myparameter", null, value -> true) + ); TestAbstractKNNLibrary testAbstractKNNLibrary1 = new TestAbstractKNNLibrary( Collections.emptyMap(), @@ -89,7 +88,11 @@ public void testEngineSpecificMethods() throws IOException { ); assertNotNull(testAbstractKNNLibrary1.getMethodContext(methodName1)); - assertTrue(testAbstractKNNLibrary1.getMethodContext(methodName1).supportedMethodParameters().containsKey("myparameter")); + assertTrue( + testAbstractKNNLibrary1.getMethodContext(methodName1) + .supportedMethodParameters(engineSpecificMethodContext) + .containsKey("myparameter") + ); } public void testGetMethodAsMap() { From bd2dea395ff97fbacc5c15c1edb6b0836061d9e5 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Mon, 8 Jul 2024 09:25:33 -0700 Subject: [PATCH 282/416] Fix linux build CI error due to action runner env upgrade node 20 (#1795) (#1798) Signed-off-by: Junqiu Lei (cherry picked from commit 5139b163bc4a36aef7c99ce53d5e8b14c43eb768) Co-authored-by: Junqiu Lei --- .github/workflows/CI.yml | 2 ++ .github/workflows/test_security.yml | 4 +++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index e1d85eb78..0b9b24d98 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -10,6 +10,8 @@ on: branches: - "*" - "feature/**" +env: + ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true jobs: Get-CI-Image-Tag: diff --git a/.github/workflows/test_security.yml b/.github/workflows/test_security.yml index 78206bada..e0f2dbf45 100644 --- a/.github/workflows/test_security.yml +++ b/.github/workflows/test_security.yml @@ -10,6 +10,8 @@ on: branches: - "*" - "feature/**" +env: + ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true jobs: Get-CI-Image-Tag: @@ -20,7 +22,7 @@ jobs: integ-test-with-security-linux: strategy: matrix: - java: [11, 17, 21] + java: [21] name: Run Integration Tests on Linux runs-on: ubuntu-latest From c47fba1e580814835c83b0819174ad92f86177e6 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Tue, 9 Jul 2024 12:23:42 -0700 Subject: [PATCH 283/416] Fixing the arithmetic to find the number of vectors to stream from java to jni layer (#1804) (#1805) Signed-off-by: Navneet Verma (cherry picked from commit 5a62f46de20efbcfb55fe02ace76fde30f7ac1c1) Co-authored-by: Navneet Verma --- CHANGELOG.md | 1 + .../opensearch/knn/index/codec/util/KNNCodecUtil.java | 10 ++++++---- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2d75d5f2f..61df8c6c0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), * Adds dynamic query parameter ef_search in radial search faiss engine [#1790](https://github.com/opensearch-project/k-NN/pull/1790) ### Enhancements ### Bug Fixes +* Fixing the arithmetic to find the number of vectors to stream from java to jni layer.[#1804](https://github.com/opensearch-project/k-NN/pull/1804) ### Infrastructure ### Documentation ### Maintenance diff --git a/src/main/java/org/opensearch/knn/index/codec/util/KNNCodecUtil.java b/src/main/java/org/opensearch/knn/index/codec/util/KNNCodecUtil.java index e05962608..0a000cadb 100644 --- a/src/main/java/org/opensearch/knn/index/codec/util/KNNCodecUtil.java +++ b/src/main/java/org/opensearch/knn/index/codec/util/KNNCodecUtil.java @@ -63,11 +63,13 @@ public static KNNCodecUtil.Pair getFloats(BinaryDocValues values) throws IOExcep dimension = vector.length; if (vectorsPerTransfer == Integer.MIN_VALUE) { - vectorsPerTransfer = (dimension * Float.BYTES * totalLiveDocs) / vectorsStreamingMemoryLimit; - // This condition comes if vectorsStreamingMemoryLimit is higher than total number floats to transfer - // Doing this will reduce 1 extra trip to JNI layer. + // if vectorsStreamingMemoryLimit is 100 bytes and we have 50 vectors with 5 dimension, then per + // transfer we have to send 100/(5 * 4) => 20 vectors. + vectorsPerTransfer = vectorsStreamingMemoryLimit / ((long) dimension * Float.BYTES); + // If vectorsPerTransfer comes out to be 0, then we set number of vectors per transfer to 1, to ensure that + // we are sending minimum number of vectors. if (vectorsPerTransfer == 0) { - vectorsPerTransfer = totalLiveDocs; + vectorsPerTransfer = 1; } } if (vectorList.size() == vectorsPerTransfer) { From 67b4fa98884e02fd73eb8b9a6c8bf4a5170e504b Mon Sep 17 00:00:00 2001 From: Navneet Verma Date: Wed, 10 Jul 2024 12:10:31 -0700 Subject: [PATCH 284/416] Setting locale to Locale.ROOT while using String.format to ensure that UTs doesn't fail when validating the exception messages (#1812) (#1813) Signed-off-by: Navneet Verma --- .../knn/index/query/KNNQueryBuilder.java | 59 +++++++++++-------- 1 file changed, 36 insertions(+), 23 deletions(-) diff --git a/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java b/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java index 61ffc567e..bab0b2519 100644 --- a/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java +++ b/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java @@ -44,6 +44,7 @@ import java.io.IOException; import java.util.Arrays; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Objects; @@ -199,33 +200,38 @@ public KNNQueryBuilder build() { private void validate() { if (Strings.isNullOrEmpty(fieldName)) { - throw new IllegalArgumentException(String.format("[%s] requires fieldName", NAME)); + throw new IllegalArgumentException(String.format(Locale.ROOT, "[%s] requires fieldName", NAME)); } if (vector == null) { - throw new IllegalArgumentException(String.format("[%s] requires query vector", NAME)); + throw new IllegalArgumentException(String.format(Locale.ROOT, "[%s] requires query vector", NAME)); } else if (vector.length == 0) { - throw new IllegalArgumentException(String.format("[%s] query vector is empty", NAME)); + throw new IllegalArgumentException(String.format(Locale.ROOT, "[%s] query vector is empty", NAME)); } if (k == null && minScore == null && maxDistance == null) { - throw new IllegalArgumentException(String.format("[%s] requires exactly one of k, distance or score to be set", NAME)); + throw new IllegalArgumentException( + String.format(Locale.ROOT, "[%s] requires exactly one of k, distance or score to be set", NAME) + ); } if ((k != null && maxDistance != null) || (maxDistance != null && minScore != null) || (k != null && minScore != null)) { - throw new IllegalArgumentException(String.format("[%s] requires exactly one of k, distance or score to be set", NAME)); + throw new IllegalArgumentException( + String.format(Locale.ROOT, "[%s] requires exactly one of k, distance or score to be set", NAME) + ); } if (k != null) { if (k <= 0 || k > K_MAX) { - final String errorMessage = "[" + NAME + "] requires k to be in the range (0, " + K_MAX + "]"; - throw new IllegalArgumentException(errorMessage); + throw new IllegalArgumentException( + String.format(Locale.ROOT, "[%s] requires k to be in the range (0, %d]", NAME, K_MAX) + ); } } if (minScore != null) { if (minScore <= 0) { - throw new IllegalArgumentException(String.format("[%s] requires minScore to be greater than 0", NAME)); + throw new IllegalArgumentException(String.format(Locale.ROOT, "[%s] requires minScore to be greater than 0", NAME)); } } @@ -233,7 +239,7 @@ private void validate() { ValidationException validationException = validateMethodParameters(methodParameters); if (validationException != null) { throw new IllegalArgumentException( - String.format("[%s] errors in method parameter [%s]", NAME, validationException.getMessage()) + String.format(Locale.ROOT, "[%s] errors in method parameter [%s]", NAME, validationException.getMessage()) ); } } @@ -259,19 +265,19 @@ public KNNQueryBuilder(String fieldName, float[] vector, int k) { @Deprecated public KNNQueryBuilder(String fieldName, float[] vector, int k, QueryBuilder filter) { if (Strings.isNullOrEmpty(fieldName)) { - throw new IllegalArgumentException(String.format("[%s] requires fieldName", NAME)); + throw new IllegalArgumentException(String.format(Locale.ROOT, "[%s] requires fieldName", NAME)); } if (vector == null) { - throw new IllegalArgumentException(String.format("[%s] requires query vector", NAME)); + throw new IllegalArgumentException(String.format(Locale.ROOT, "[%s] requires query vector", NAME)); } if (vector.length == 0) { - throw new IllegalArgumentException(String.format("[%s] query vector is empty", NAME)); + throw new IllegalArgumentException(String.format(Locale.ROOT, "[%s] query vector is empty", NAME)); } if (k <= 0) { - throw new IllegalArgumentException(String.format("[%s] requires k > 0", NAME)); + throw new IllegalArgumentException(String.format(Locale.ROOT, "[%s] requires k > 0", NAME)); } if (k > K_MAX) { - throw new IllegalArgumentException(String.format("[%s] requires k <= %d", NAME, K_MAX)); + throw new IllegalArgumentException(String.format(Locale.ROOT, "[%s] requires k <= %d", NAME, K_MAX)); } this.fieldName = fieldName; @@ -289,12 +295,16 @@ public static void initialize(ModelDao modelDao) { private static float[] ObjectsToFloats(List objs) { if (Objects.isNull(objs) || objs.isEmpty()) { - throw new IllegalArgumentException(String.format("[%s] field 'vector' requires to be non-null and non-empty", NAME)); + throw new IllegalArgumentException( + String.format(Locale.ROOT, "[%s] field 'vector' requires to be non-null and non-empty", NAME) + ); } float[] vec = new float[objs.size()]; for (int i = 0; i < objs.size(); i++) { if ((objs.get(i) instanceof Number) == false) { - throw new IllegalArgumentException(String.format("[%s] field 'vector' requires to be an array of numbers", NAME)); + throw new IllegalArgumentException( + String.format(Locale.ROOT, "[%s] field 'vector' requires to be an array of numbers", NAME) + ); } vec[i] = ((Number) objs.get(i)).floatValue(); } @@ -511,7 +521,7 @@ protected Query doToQuery(QueryShardContext context) { } if (!(mappedFieldType instanceof KNNVectorFieldMapper.KNNVectorFieldType)) { - throw new IllegalArgumentException(String.format("Field '%s' is not knn_vector type.", this.fieldName)); + throw new IllegalArgumentException(String.format(Locale.ROOT, "Field '%s' is not knn_vector type.", this.fieldName)); } KNNVectorFieldMapper.KNNVectorFieldType knnVectorFieldType = (KNNVectorFieldMapper.KNNVectorFieldType) mappedFieldType; @@ -553,6 +563,7 @@ protected Query doToQuery(QueryShardContext context) { if (validationException != null) { throw new IllegalArgumentException( String.format( + Locale.ROOT, "Parameters not valid for [%s]:[%s]:[%s] combination: [%s]", knnEngine, method, @@ -605,7 +616,7 @@ protected Query doToQuery(QueryShardContext context) { if (KNNEngine.getEnginesThatCreateCustomSegmentFiles().contains(knnEngine) && filter != null && !KNNEngine.getEnginesThatSupportsFilters().contains(knnEngine)) { - throw new IllegalArgumentException(String.format("Engine [%s] does not support filters", knnEngine)); + throw new IllegalArgumentException(String.format(Locale.ROOT, "Engine [%s] does not support filters", knnEngine)); } String indexName = context.index().getName(); @@ -627,7 +638,9 @@ protected Query doToQuery(QueryShardContext context) { } if (radius != null) { if (!ENGINES_SUPPORTING_RADIAL_SEARCH.contains(knnEngine)) { - throw new UnsupportedOperationException(String.format("Engine [%s] does not support radial search", knnEngine)); + throw new UnsupportedOperationException( + String.format(Locale.ROOT, "Engine [%s] does not support radial search", knnEngine) + ); } RNNQueryFactory.CreateQueryRequest createQueryRequest = RNNQueryFactory.CreateQueryRequest.builder() .knnEngine(knnEngine) @@ -643,19 +656,19 @@ protected Query doToQuery(QueryShardContext context) { .build(); return RNNQueryFactory.create(createQueryRequest); } - throw new IllegalArgumentException(String.format("[%s] requires k or distance or score to be set", NAME)); + throw new IllegalArgumentException(String.format(Locale.ROOT, "[%s] requires k or distance or score to be set", NAME)); } private ModelMetadata getModelMetadataForField(KNNVectorFieldMapper.KNNVectorFieldType knnVectorField) { String modelId = knnVectorField.getModelId(); if (modelId == null) { - throw new IllegalArgumentException(String.format("Field '%s' does not have model.", this.fieldName)); + throw new IllegalArgumentException(String.format(Locale.ROOT, "Field '%s' does not have model.", this.fieldName)); } ModelMetadata modelMetadata = modelDao.getMetadata(modelId); if (!ModelUtil.isModelCreated(modelMetadata)) { - throw new IllegalArgumentException(String.format("Model ID '%s' is not created.", modelId)); + throw new IllegalArgumentException(String.format(Locale.ROOT, "Model ID '%s' is not created.", modelId)); } return modelMetadata; } @@ -677,7 +690,7 @@ private VectorQueryType getVectorQueryType(int k, Float maxDistance, Float minSc if (k != 0) { return VectorQueryType.K; } - throw new IllegalArgumentException(String.format("[%s] requires exactly one of k, distance or score to be set", NAME)); + throw new IllegalArgumentException(String.format(Locale.ROOT, "[%s] requires exactly one of k, distance or score to be set", NAME)); } /** From 8b63d1db821d4b0d772733ed7dc80f1deca98182 Mon Sep 17 00:00:00 2001 From: Heemin Kim Date: Thu, 11 Jul 2024 11:12:12 -0700 Subject: [PATCH 285/416] Add jni interface to use a binary hnsw index with faiss (#1778) (#1817) * Add jni interface to use a binary hnsw index with faiss (#1747) * Fix memory leak on test code (#1776) --------- Signed-off-by: Heemin Kim --- jni/CMakeLists.txt | 4 +- jni/include/commons.h | 24 +- jni/include/faiss_index_service.h | 113 ++++++++++ jni/include/faiss_methods.h | 42 ++++ jni/include/faiss_wrapper.h | 14 +- jni/include/jni_util.h | 7 + .../org_opensearch_knn_jni_FaissService.h | 29 ++- .../org_opensearch_knn_jni_JNICommons.h | 16 ++ jni/src/commons.cpp | 22 ++ jni/src/faiss_index_service.cpp | 164 ++++++++++++++ jni/src/faiss_methods.cpp | 40 ++++ jni/src/faiss_wrapper.cpp | 205 ++++++++++++++---- jni/src/jni_util.cpp | 51 +++++ .../org_opensearch_knn_jni_FaissService.cpp | 49 ++++- jni/src/org_opensearch_knn_jni_JNICommons.cpp | 23 ++ jni/tests/faiss_index_service_test.cpp | 134 ++++++++++++ jni/tests/faiss_wrapper_test.cpp | 197 +++++++++++++++-- jni/tests/faiss_wrapper_unit_test.cpp | 17 +- jni/tests/mocks/faiss_index_mock.h | 35 +++ jni/tests/mocks/faiss_index_service_mock.h | 44 ++++ jni/tests/mocks/faiss_methods_mock.h | 28 +++ jni/tests/test_util.cpp | 56 +++++ jni/tests/test_util.h | 14 +- .../org/opensearch/knn/index/IndexUtil.java | 13 +- .../opensearch/knn/index/KNNIndexShard.java | 14 +- .../opensearch/knn/index/query/KNNWeight.java | 8 +- .../knn/index/util/FieldInfoExtractor.java | 37 ++++ .../org/opensearch/knn/jni/FaissService.java | 45 ++++ .../org/opensearch/knn/jni/JNICommons.java | 19 ++ .../org/opensearch/knn/jni/JNIService.java | 65 +++++- .../opensearch/knn/index/IndexUtilTests.java | 7 +- .../knn/index/KNNIndexShardTests.java | 4 +- .../knn/index/query/KNNWeightTests.java | 75 ++++++- .../index/util/FieldInfoExtractorTests.java | 42 ++++ .../opensearch/knn/jni/JNIServiceTests.java | 47 ++++ .../java/org/opensearch/knn/TestUtils.java | 43 ++++ 36 files changed, 1639 insertions(+), 108 deletions(-) create mode 100644 jni/include/faiss_index_service.h create mode 100644 jni/include/faiss_methods.h create mode 100644 jni/src/faiss_index_service.cpp create mode 100644 jni/src/faiss_methods.cpp create mode 100644 jni/tests/faiss_index_service_test.cpp create mode 100644 jni/tests/mocks/faiss_index_mock.h create mode 100644 jni/tests/mocks/faiss_index_service_mock.h create mode 100644 jni/tests/mocks/faiss_methods_mock.h create mode 100644 src/main/java/org/opensearch/knn/index/util/FieldInfoExtractor.java create mode 100644 src/test/java/org/opensearch/knn/index/util/FieldInfoExtractorTests.java diff --git a/jni/CMakeLists.txt b/jni/CMakeLists.txt index e2003e0f7..2fe26875d 100644 --- a/jni/CMakeLists.txt +++ b/jni/CMakeLists.txt @@ -20,7 +20,6 @@ set(TARGET_LIBS "") # Libs to be installed set(CMAKE_CXX_STANDARD 11) set(CMAKE_CXX_STANDARD_REQUIRED True) - option(CONFIG_FAISS "Configure faiss library build when this is on") option(CONFIG_NMSLIB "Configure nmslib library build when this is on") option(CONFIG_TEST "Configure tests when this is on") @@ -112,6 +111,8 @@ if (${CONFIG_FAISS} STREQUAL ON OR ${CONFIG_ALL} STREQUAL ON OR ${CONFIG_TEST} S ${CMAKE_CURRENT_SOURCE_DIR}/src/org_opensearch_knn_jni_FaissService.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/faiss_wrapper.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/faiss_util.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/src/faiss_index_service.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/src/faiss_methods.cpp ) target_link_libraries(${TARGET_LIB_FAISS} ${TARGET_LINK_FAISS_LIB} ${TARGET_LIB_UTIL} OpenMP::OpenMP_CXX) target_include_directories(${TARGET_LIB_FAISS} PRIVATE @@ -153,6 +154,7 @@ if ("${WIN32}" STREQUAL "") tests/nmslib_wrapper_unit_test.cpp tests/test_util.cpp tests/commons_test.cpp + tests/faiss_index_service_test.cpp ) target_link_libraries( diff --git a/jni/include/commons.h b/jni/include/commons.h index 67a141c8b..d02439377 100644 --- a/jni/include/commons.h +++ b/jni/include/commons.h @@ -22,10 +22,24 @@ namespace knn_jni { * @param memoryAddress The address of the memory location where data will be stored. * @param data 2D float array containing data to be stored in native memory. * @param initialCapacity The initial capacity of the memory location. - * @return memory address where the data is stored. + * @return memory address of std::vector where the data is stored. */ jlong storeVectorData(knn_jni::JNIUtilInterface *, JNIEnv *, jlong , jobjectArray, jlong); + /** + * This is utility function that can be used to store data in native memory. This function will allocate memory for + * the data(rows*columns) with initialCapacity and return the memory address where the data is stored. + * If you are using this function for first time use memoryAddress = 0 to ensure that a new memory location is created. + * For subsequent calls you can pass the same memoryAddress. If the data cannot be stored in the memory location + * will throw Exception. + * + * @param memoryAddress The address of the memory location where data will be stored. + * @param data 2D byte array containing data to be stored in native memory. + * @param initialCapacity The initial capacity of the memory location. + * @return memory address of std::vector where the data is stored. + */ + jlong storeByteVectorData(knn_jni::JNIUtilInterface *, JNIEnv *, jlong , jobjectArray, jlong); + /** * Free up the memory allocated for the data stored in memory address. This function should be used with the memory * address returned by {@link JNICommons#storeVectorData(long, float[][], long, long)} @@ -34,6 +48,14 @@ namespace knn_jni { */ void freeVectorData(jlong); + /** + * Free up the memory allocated for the data stored in memory address. This function should be used with the memory + * address returned by {@link JNICommons#storeByteVectorData(long, byte[][], long, long)} + * + * @param memoryAddress address to be freed. + */ + void freeByteVectorData(jlong); + /** * Extracts query time efSearch from method parameters **/ diff --git a/jni/include/faiss_index_service.h b/jni/include/faiss_index_service.h new file mode 100644 index 000000000..59f15fda9 --- /dev/null +++ b/jni/include/faiss_index_service.h @@ -0,0 +1,113 @@ +// SPDX-License-Identifier: Apache-2.0 +// +// The OpenSearch Contributors require contributions made to +// this file be licensed under the Apache-2.0 license or a +// compatible open source license. +// +// Modifications Copyright OpenSearch Contributors. See +// GitHub history for details. + +/** + * This file contains classes for index operations which are free of JNI + */ + +#ifndef OPENSEARCH_KNN_FAISS_INDEX_SERVICE_H +#define OPENSEARCH_KNN_FAISS_INDEX_SERVICE_H + +#include +#include "faiss/MetricType.h" +#include "jni_util.h" +#include "faiss_methods.h" +#include + +namespace knn_jni { +namespace faiss_wrapper { + + +/** + * A class to provide operations on index + * This class should evolve to have only cpp object but not jni object + */ +class IndexService { +public: + IndexService(std::unique_ptr faissMethods); + //TODO Remove dependency on JNIUtilInterface and JNIEnv + //TODO Reduce the number of parameters + + /** + * Create index + * + * @param jniUtil jni util + * @param env jni environment + * @param metric space type for distance calculation + * @param indexDescription index description to be used by faiss index factory + * @param dim dimension of vectors + * @param numIds number of vectors + * @param threadCount number of thread count to be used while adding data + * @param vectorsAddress memory address which is holding vector data + * @param ids a list of document ids for corresponding vectors + * @param indexPath path to write index + * @param parameters parameters to be applied to faiss index + */ + virtual void createIndex( + knn_jni::JNIUtilInterface * jniUtil, + JNIEnv * env, + faiss::MetricType metric, + std::string indexDescription, + int dim, + int numIds, + int threadCount, + int64_t vectorsAddress, + std::vector ids, + std::string indexPath, + std::unordered_map parameters); + virtual ~IndexService() = default; +protected: + std::unique_ptr faissMethods; +}; + +/** + * A class to provide operations on index + * This class should evolve to have only cpp object but not jni object + */ +class BinaryIndexService : public IndexService { +public: + //TODO Remove dependency on JNIUtilInterface and JNIEnv + //TODO Reduce the number of parameters + BinaryIndexService(std::unique_ptr faissMethods); + /** + * Create binary index + * + * @param jniUtil jni util + * @param env jni environment + * @param metric space type for distance calculation + * @param indexDescription index description to be used by faiss index factory + * @param dim dimension of vectors + * @param numIds number of vectors + * @param threadCount number of thread count to be used while adding data + * @param vectorsAddress memory address which is holding vector data + * @param ids a list of document ids for corresponding vectors + * @param indexPath path to write index + * @param parameters parameters to be applied to faiss index + */ + virtual void createIndex( + knn_jni::JNIUtilInterface * jniUtil, + JNIEnv * env, + faiss::MetricType metric, + std::string indexDescription, + int dim, + int numIds, + int threadCount, + int64_t vectorsAddress, + std::vector ids, + std::string indexPath, + std::unordered_map parameters + ) override; + virtual ~BinaryIndexService() = default; +}; + +} +} + + +#endif //OPENSEARCH_KNN_FAISS_INDEX_SERVICE_H diff --git a/jni/include/faiss_methods.h b/jni/include/faiss_methods.h new file mode 100644 index 000000000..38d8d756a --- /dev/null +++ b/jni/include/faiss_methods.h @@ -0,0 +1,42 @@ +// SPDX-License-Identifier: Apache-2.0 +// +// The OpenSearch Contributors require contributions made to +// this file be licensed under the Apache-2.0 license or a +// compatible open source license. +// +// Modifications Copyright OpenSearch Contributors. See +// GitHub history for details. + +#ifndef OPENSEARCH_KNN_FAISS_METHODS_H +#define OPENSEARCH_KNN_FAISS_METHODS_H + +#include "faiss/Index.h" +#include "faiss/IndexBinary.h" +#include "faiss/IndexIDMap.h" +#include "faiss/index_io.h" + +namespace knn_jni { +namespace faiss_wrapper { + +/** + * A class having wrapped faiss methods + * + * This class helps to mock faiss methods during unit test + */ +class FaissMethods { +public: + FaissMethods() = default; + virtual faiss::Index* indexFactory(int d, const char* description, faiss::MetricType metric); + virtual faiss::IndexBinary* indexBinaryFactory(int d, const char* description); + virtual faiss::IndexIDMapTemplate* indexIdMap(faiss::Index* index); + virtual faiss::IndexIDMapTemplate* indexBinaryIdMap(faiss::IndexBinary* index); + virtual void writeIndex(const faiss::Index* idx, const char* fname); + virtual void writeIndexBinary(const faiss::IndexBinary* idx, const char* fname); + virtual ~FaissMethods() = default; +}; + +} //namespace faiss_wrapper +} //namespace knn_jni + + +#endif //OPENSEARCH_KNN_FAISS_METHODS_H \ No newline at end of file diff --git a/jni/include/faiss_wrapper.h b/jni/include/faiss_wrapper.h index 3bfc66325..5ac17cfd1 100644 --- a/jni/include/faiss_wrapper.h +++ b/jni/include/faiss_wrapper.h @@ -13,6 +13,7 @@ #define OPENSEARCH_KNN_FAISS_WRAPPER_H #include "jni_util.h" +#include "faiss_index_service.h" #include namespace knn_jni { @@ -20,7 +21,7 @@ namespace knn_jni { // Create an index with ids and vectors. The configuration is defined by values in the Java map, parametersJ. // The index is serialized to indexPathJ. void CreateIndex(knn_jni::JNIUtilInterface * jniUtil, JNIEnv * env, jintArray idsJ, jlong vectorsAddressJ, jint dimJ, - jstring indexPathJ, jobject parametersJ); + jstring indexPathJ, jobject parametersJ, IndexService* indexService); // Create an index with ids and vectors. Instead of creating a new index, this function creates the index // based off of the template index passed in. The index is serialized to indexPathJ. @@ -33,6 +34,11 @@ namespace knn_jni { // Return a pointer to the loaded index jlong LoadIndex(knn_jni::JNIUtilInterface * jniUtil, JNIEnv * env, jstring indexPathJ); + // Load a binary index from indexPathJ into memory. + // + // Return a pointer to the loaded index + jlong LoadBinaryIndex(knn_jni::JNIUtilInterface * jniUtil, JNIEnv * env, jstring indexPathJ); + // Check if a loaded index requires shared state bool IsSharedIndexStateRequired(jlong indexPointerJ); @@ -68,6 +74,12 @@ namespace knn_jni { jfloatArray queryVectorJ, jint kJ, jobject methodParamsJ, jlongArray filterIdsJ, jint filterIdsTypeJ, jintArray parentIdsJ); + // Execute a query against the binary index located in memory at indexPointerJ along with Filters + // + // Return an array of KNNQueryResults + jobjectArray QueryBinaryIndex_WithFilter(knn_jni::JNIUtilInterface * jniUtil, JNIEnv * env, jlong indexPointerJ, + jbyteArray queryVectorJ, jint kJ, jobject methodParamsJ, jlongArray filterIdsJ, jint filterIdsTypeJ, jintArray parentIdsJ); + // Free the index located in memory at indexPointerJ void Free(jlong indexPointer); diff --git a/jni/include/jni_util.h b/jni/include/jni_util.h index b3d55f1c1..97a8f063c 100644 --- a/jni/include/jni_util.h +++ b/jni/include/jni_util.h @@ -71,6 +71,8 @@ namespace knn_jni { virtual void Convert2dJavaObjectArrayAndStoreToFloatVector(JNIEnv *env, jobjectArray array2dJ, int dim, std::vector *vect ) = 0; + virtual void Convert2dJavaObjectArrayAndStoreToByteVector(JNIEnv *env, jobjectArray array2dJ, + int dim, std::vector *vect ) = 0; virtual std::vector ConvertJavaIntArrayToCppIntVector(JNIEnv *env, jintArray arrayJ) = 0; @@ -79,6 +81,8 @@ namespace knn_jni { // ------------------------------ MISC HELPERS ------------------------------ virtual int GetInnerDimensionOf2dJavaFloatArray(JNIEnv *env, jobjectArray array2dJ) = 0; + virtual int GetInnerDimensionOf2dJavaByteArray(JNIEnv *env, jobjectArray array2dJ) = 0; + virtual int GetJavaObjectArrayLength(JNIEnv *env, jobjectArray arrayJ) = 0; virtual int GetJavaIntArrayLength(JNIEnv *env, jintArray arrayJ) = 0; @@ -146,6 +150,7 @@ namespace knn_jni { std::vector Convert2dJavaObjectArrayToCppFloatVector(JNIEnv *env, jobjectArray array2dJ, int dim); std::vector ConvertJavaIntArrayToCppIntVector(JNIEnv *env, jintArray arrayJ); int GetInnerDimensionOf2dJavaFloatArray(JNIEnv *env, jobjectArray array2dJ); + int GetInnerDimensionOf2dJavaByteArray(JNIEnv *env, jobjectArray array2dJ); int GetJavaObjectArrayLength(JNIEnv *env, jobjectArray arrayJ); int GetJavaIntArrayLength(JNIEnv *env, jintArray arrayJ); int GetJavaLongArrayLength(JNIEnv *env, jlongArray arrayJ); @@ -168,6 +173,7 @@ namespace knn_jni { void SetObjectArrayElement(JNIEnv *env, jobjectArray array, jsize index, jobject val); void SetByteArrayRegion(JNIEnv *env, jbyteArray array, jsize start, jsize len, const jbyte * buf); void Convert2dJavaObjectArrayAndStoreToFloatVector(JNIEnv *env, jobjectArray array2dJ, int dim, std::vector *vect); + void Convert2dJavaObjectArrayAndStoreToByteVector(JNIEnv *env, jobjectArray array2dJ, int dim, std::vector *vect); private: std::unordered_map cachedClasses; @@ -193,6 +199,7 @@ namespace knn_jni { extern const std::string COSINESIMIL; extern const std::string INNER_PRODUCT; extern const std::string NEG_DOT_PRODUCT; + extern const std::string HAMMING_BIT; extern const std::string NPROBES; extern const std::string COARSE_QUANTIZER; diff --git a/jni/include/org_opensearch_knn_jni_FaissService.h b/jni/include/org_opensearch_knn_jni_FaissService.h index ef382507a..3d6aef45c 100644 --- a/jni/include/org_opensearch_knn_jni_FaissService.h +++ b/jni/include/org_opensearch_knn_jni_FaissService.h @@ -18,6 +18,7 @@ #ifdef __cplusplus extern "C" { #endif + /* * Class: org_opensearch_knn_jni_FaissService * Method: createIndex @@ -26,6 +27,14 @@ extern "C" { JNIEXPORT void JNICALL Java_org_opensearch_knn_jni_FaissService_createIndex (JNIEnv *, jclass, jintArray, jlong, jint, jstring, jobject); +/* + * Class: org_opensearch_knn_jni_FaissService + * Method: createBinaryIndex + * Signature: ([IJILjava/lang/String;Ljava/util/Map;)V + */ +JNIEXPORT void JNICALL Java_org_opensearch_knn_jni_FaissService_createBinaryIndex + (JNIEnv *, jclass, jintArray, jlong, jint, jstring, jobject); + /* * Class: org_opensearch_knn_jni_FaissService * Method: createIndexFromTemplate @@ -42,6 +51,14 @@ JNIEXPORT void JNICALL Java_org_opensearch_knn_jni_FaissService_createIndexFromT JNIEXPORT jlong JNICALL Java_org_opensearch_knn_jni_FaissService_loadIndex (JNIEnv *, jclass, jstring); +/* + * Class: org_opensearch_knn_jni_FaissService + * Method: loadBinaryIndex + * Signature: (Ljava/lang/String;)J + */ +JNIEXPORT jlong JNICALL Java_org_opensearch_knn_jni_FaissService_loadBinaryIndex + (JNIEnv *, jclass, jstring); + /* * Class: org_opensearch_knn_jni_FaissService * Method: isSharedIndexStateRequired @@ -69,7 +86,7 @@ JNIEXPORT void JNICALL Java_org_opensearch_knn_jni_FaissService_setSharedIndexSt /* * Class: org_opensearch_knn_jni_FaissService * Method: queryIndex - * Signature: (J[FI[Ljava/util/MapI)[Lorg/opensearch/knn/index/query/KNNQueryResult; + * Signature: (J[FILjava/util/Map[I)[Lorg/opensearch/knn/index/query/KNNQueryResult; */ JNIEXPORT jobjectArray JNICALL Java_org_opensearch_knn_jni_FaissService_queryIndex (JNIEnv *, jclass, jlong, jfloatArray, jint, jobject, jintArray); @@ -77,11 +94,19 @@ JNIEXPORT jobjectArray JNICALL Java_org_opensearch_knn_jni_FaissService_queryInd /* * Class: org_opensearch_knn_jni_FaissService * Method: queryIndexWithFilter - * Signature: (J[FI[JLjava/util/MapI[I)[Lorg/opensearch/knn/index/query/KNNQueryResult; + * Signature: (J[FILjava/util/Map[JI[I)[Lorg/opensearch/knn/index/query/KNNQueryResult; */ JNIEXPORT jobjectArray JNICALL Java_org_opensearch_knn_jni_FaissService_queryIndexWithFilter (JNIEnv *, jclass, jlong, jfloatArray, jint, jobject, jlongArray, jint, jintArray); +/* + * Class: org_opensearch_knn_jni_FaissService + * Method: queryBIndexWithFilter + * Signature: (J[BILjava/util/Map[JI[I)[Lorg/opensearch/knn/index/query/KNNQueryResult; + */ +JNIEXPORT jobjectArray JNICALL Java_org_opensearch_knn_jni_FaissService_queryBinaryIndexWithFilter + (JNIEnv *, jclass, jlong, jbyteArray, jint, jobject, jlongArray, jint, jintArray); + /* * Class: org_opensearch_knn_jni_FaissService * Method: free diff --git a/jni/include/org_opensearch_knn_jni_JNICommons.h b/jni/include/org_opensearch_knn_jni_JNICommons.h index d0758d7c8..89de76520 100644 --- a/jni/include/org_opensearch_knn_jni_JNICommons.h +++ b/jni/include/org_opensearch_knn_jni_JNICommons.h @@ -26,6 +26,14 @@ extern "C" { JNIEXPORT jlong JNICALL Java_org_opensearch_knn_jni_JNICommons_storeVectorData (JNIEnv *, jclass, jlong, jobjectArray, jlong); +/* + * Class: org_opensearch_knn_jni_JNICommons + * Method: storeVectorData + * Signature: (J[[FJJ) + */ +JNIEXPORT jlong JNICALL Java_org_opensearch_knn_jni_JNICommons_storeByteVectorData + (JNIEnv *, jclass, jlong, jobjectArray, jlong); + /* * Class: org_opensearch_knn_jni_JNICommons * Method: freeVectorData @@ -34,6 +42,14 @@ JNIEXPORT jlong JNICALL Java_org_opensearch_knn_jni_JNICommons_storeVectorData JNIEXPORT void JNICALL Java_org_opensearch_knn_jni_JNICommons_freeVectorData (JNIEnv *, jclass, jlong); +/* +* Class: org_opensearch_knn_jni_JNICommons +* Method: freeVectorData +* Signature: (J)V +*/ +JNIEXPORT void JNICALL Java_org_opensearch_knn_jni_JNICommons_freeByteVectorData +(JNIEnv *, jclass, jlong); + #ifdef __cplusplus } #endif diff --git a/jni/src/commons.cpp b/jni/src/commons.cpp index c2b2354cc..13f59194e 100644 --- a/jni/src/commons.cpp +++ b/jni/src/commons.cpp @@ -32,6 +32,21 @@ jlong knn_jni::commons::storeVectorData(knn_jni::JNIUtilInterface *jniUtil, JNIE return (jlong) vect; } +jlong knn_jni::commons::storeByteVectorData(knn_jni::JNIUtilInterface *jniUtil, JNIEnv *env, jlong memoryAddressJ, + jobjectArray dataJ, jlong initialCapacityJ) { + std::vector *vect; + if ((long) memoryAddressJ == 0) { + vect = new std::vector(); + vect->reserve((long)initialCapacityJ); + } else { + vect = reinterpret_cast*>(memoryAddressJ); + } + int dim = jniUtil->GetInnerDimensionOf2dJavaByteArray(env, dataJ); + jniUtil->Convert2dJavaObjectArrayAndStoreToByteVector(env, dataJ, dim, vect); + + return (jlong) vect; +} + void knn_jni::commons::freeVectorData(jlong memoryAddressJ) { if (memoryAddressJ != 0) { auto *vect = reinterpret_cast*>(memoryAddressJ); @@ -39,6 +54,13 @@ void knn_jni::commons::freeVectorData(jlong memoryAddressJ) { } } +void knn_jni::commons::freeByteVectorData(jlong memoryAddressJ) { + if (memoryAddressJ != 0) { + auto *vect = reinterpret_cast*>(memoryAddressJ); + delete vect; + } +} + int knn_jni::commons::getIntegerMethodParameter(JNIEnv * env, knn_jni::JNIUtilInterface * jniUtil, std::unordered_map methodParams, std::string methodParam, int defaultValue) { if (methodParams.empty()) { return defaultValue; diff --git a/jni/src/faiss_index_service.cpp b/jni/src/faiss_index_service.cpp new file mode 100644 index 000000000..8c5ba36af --- /dev/null +++ b/jni/src/faiss_index_service.cpp @@ -0,0 +1,164 @@ +// SPDX-License-Identifier: Apache-2.0 +// +// The OpenSearch Contributors require contributions made to +// this file be licensed under the Apache-2.0 license or a +// compatible open source license. +// +// Modifications Copyright OpenSearch Contributors. See +// GitHub history for details. + +#include "faiss_index_service.h" +#include "faiss_methods.h" +#include "faiss/index_factory.h" +#include "faiss/Index.h" +#include "faiss/IndexBinary.h" +#include "faiss/IndexHNSW.h" +#include "faiss/IndexBinaryHNSW.h" +#include "faiss/IndexIVFFlat.h" +#include "faiss/IndexBinaryIVF.h" +#include "faiss/IndexIDMap.h" +#include "faiss/index_io.h" +#include +#include +#include +#include +#include + +namespace knn_jni { +namespace faiss_wrapper { + +template +void SetExtraParameters(knn_jni::JNIUtilInterface * jniUtil, JNIEnv *env, + const std::unordered_map& parametersCpp, INDEX * index) { + std::unordered_map::const_iterator value; + if (auto * indexIvf = dynamic_cast(index)) { + if ((value = parametersCpp.find(knn_jni::NPROBES)) != parametersCpp.end()) { + indexIvf->nprobe = jniUtil->ConvertJavaObjectToCppInteger(env, value->second); + } + + if ((value = parametersCpp.find(knn_jni::COARSE_QUANTIZER)) != parametersCpp.end() + && indexIvf->quantizer != nullptr) { + auto subParametersCpp = jniUtil->ConvertJavaMapToCppMap(env, value->second); + SetExtraParameters(jniUtil, env, subParametersCpp, indexIvf->quantizer); + } + } + + if (auto * indexHnsw = dynamic_cast(index)) { + + if ((value = parametersCpp.find(knn_jni::EF_CONSTRUCTION)) != parametersCpp.end()) { + indexHnsw->hnsw.efConstruction = jniUtil->ConvertJavaObjectToCppInteger(env, value->second); + } + + if ((value = parametersCpp.find(knn_jni::EF_SEARCH)) != parametersCpp.end()) { + indexHnsw->hnsw.efSearch = jniUtil->ConvertJavaObjectToCppInteger(env, value->second); + } + } +} + +IndexService::IndexService(std::unique_ptr faissMethods) : faissMethods(std::move(faissMethods)) {} + +void IndexService::createIndex( + knn_jni::JNIUtilInterface * jniUtil, + JNIEnv * env, + faiss::MetricType metric, + std::string indexDescription, + int dim, + int numIds, + int threadCount, + int64_t vectorsAddress, + std::vector ids, + std::string indexPath, + std::unordered_map parameters + ) { + // Read vectors from memory address + auto *inputVectors = reinterpret_cast*>(vectorsAddress); + + // The number of vectors can be int here because a lucene segment number of total docs never crosses INT_MAX value + int numVectors = (int) (inputVectors->size() / (uint64_t) dim); + if(numVectors == 0) { + throw std::runtime_error("Number of vectors cannot be 0"); + } + + if (numIds != numVectors) { + throw std::runtime_error("Number of IDs does not match number of vectors"); + } + + std::unique_ptr indexWriter(faissMethods->indexFactory(dim, indexDescription.c_str(), metric)); + + // Set thread count if it is passed in as a parameter. Setting this variable will only impact the current thread + if(threadCount != 0) { + omp_set_num_threads(threadCount); + } + + // Add extra parameters that cant be configured with the index factory + SetExtraParameters(jniUtil, env, parameters, indexWriter.get()); + + // Check that the index does not need to be trained + if(!indexWriter->is_trained) { + throw std::runtime_error("Index is not trained"); + } + + // Add vectors + std::unique_ptr idMap(faissMethods->indexIdMap(indexWriter.get())); + idMap->add_with_ids(numVectors, inputVectors->data(), ids.data()); + + // Write the index to disk + faissMethods->writeIndex(idMap.get(), indexPath.c_str()); +} + +BinaryIndexService::BinaryIndexService(std::unique_ptr faissMethods) : IndexService(std::move(faissMethods)) {} + +void BinaryIndexService::createIndex( + knn_jni::JNIUtilInterface * jniUtil, + JNIEnv * env, + faiss::MetricType metric, + std::string indexDescription, + int dim, + int numIds, + int threadCount, + int64_t vectorsAddress, + std::vector ids, + std::string indexPath, + std::unordered_map parameters + ) { + // Read vectors from memory address + auto *inputVectors = reinterpret_cast*>(vectorsAddress); + + if (dim % 8 != 0) { + throw std::runtime_error("Dimensions should be multiply of 8"); + } + // The number of vectors can be int here because a lucene segment number of total docs never crosses INT_MAX value + int numVectors = (int) (inputVectors->size() / (uint64_t) (dim / 8)); + if(numVectors == 0) { + throw std::runtime_error("Number of vectors cannot be 0"); + } + + if (numIds != numVectors) { + throw std::runtime_error("Number of IDs does not match number of vectors"); + } + + std::unique_ptr indexWriter(faissMethods->indexBinaryFactory(dim, indexDescription.c_str())); + + // Set thread count if it is passed in as a parameter. Setting this variable will only impact the current thread + if(threadCount != 0) { + omp_set_num_threads(threadCount); + } + + // Add extra parameters that cant be configured with the index factory + SetExtraParameters(jniUtil, env, parameters, indexWriter.get()); + + // Check that the index does not need to be trained + if(!indexWriter->is_trained) { + throw std::runtime_error("Index is not trained"); + } + + // Add vectors + std::unique_ptr idMap(faissMethods->indexBinaryIdMap(indexWriter.get())); + idMap->add_with_ids(numVectors, inputVectors->data(), ids.data()); + + // Write the index to disk + faissMethods->writeIndexBinary(idMap.get(), indexPath.c_str()); +} + +} // namespace faiss_wrapper +} // namesapce knn_jni diff --git a/jni/src/faiss_methods.cpp b/jni/src/faiss_methods.cpp new file mode 100644 index 000000000..05c8f459a --- /dev/null +++ b/jni/src/faiss_methods.cpp @@ -0,0 +1,40 @@ +// SPDX-License-Identifier: Apache-2.0 +// +// The OpenSearch Contributors require contributions made to +// this file be licensed under the Apache-2.0 license or a +// compatible open source license. +// +// Modifications Copyright OpenSearch Contributors. See +// GitHub history for details. + +#include "faiss_methods.h" +#include "faiss/index_factory.h" + +namespace knn_jni { +namespace faiss_wrapper { + +faiss::Index* FaissMethods::indexFactory(int d, const char* description, faiss::MetricType metric) { + return faiss::index_factory(d, description, metric); +} + +faiss::IndexBinary* FaissMethods::indexBinaryFactory(int d, const char* description) { + return faiss::index_binary_factory(d, description); +} + +faiss::IndexIDMapTemplate* FaissMethods::indexIdMap(faiss::Index* index) { + return new faiss::IndexIDMap(index); +} + +faiss::IndexIDMapTemplate* FaissMethods::indexBinaryIdMap(faiss::IndexBinary* index) { + return new faiss::IndexBinaryIDMap(index); +} + +void FaissMethods::writeIndex(const faiss::Index* idx, const char* fname) { + faiss::write_index(idx, fname); +} +void FaissMethods::writeIndexBinary(const faiss::IndexBinary* idx, const char* fname) { + faiss::write_index_binary(idx, fname); +} + +} // namespace faiss_wrapper +} // namesapce knn_jni diff --git a/jni/src/faiss_wrapper.cpp b/jni/src/faiss_wrapper.cpp index 692a33aee..c4c6e18eb 100644 --- a/jni/src/faiss_wrapper.cpp +++ b/jni/src/faiss_wrapper.cpp @@ -12,6 +12,7 @@ #include "jni_util.h" #include "faiss_wrapper.h" #include "faiss_util.h" +#include "faiss_index_service.h" #include "faiss/impl/io.h" #include "faiss/index_factory.h" @@ -23,6 +24,8 @@ #include "faiss/impl/IDSelector.h" #include "faiss/IndexIVFPQ.h" #include "commons.h" +#include "faiss/IndexBinaryIVF.h" +#include "faiss/IndexBinaryHNSW.h" #include #include @@ -83,8 +86,7 @@ bool isIndexIVFPQL2(faiss::Index * index); faiss::IndexIVFPQ * extractIVFPQIndex(faiss::Index * index); void knn_jni::faiss_wrapper::CreateIndex(knn_jni::JNIUtilInterface * jniUtil, JNIEnv * env, jintArray idsJ, jlong vectorsAddressJ, jint dimJ, - jstring indexPathJ, jobject parametersJ) { - + jstring indexPathJ, jobject parametersJ, IndexService* indexService) { if (idsJ == nullptr) { throw std::runtime_error("IDs cannot be null"); } @@ -109,63 +111,49 @@ void knn_jni::faiss_wrapper::CreateIndex(knn_jni::JNIUtilInterface * jniUtil, JN // so that it is easier to access. auto parametersCpp = jniUtil->ConvertJavaMapToCppMap(env, parametersJ); - // Get space type for this index + // Parameters to pass + // Metric type jobject spaceTypeJ = knn_jni::GetJObjectFromMapOrThrow(parametersCpp, knn_jni::SPACE_TYPE); std::string spaceTypeCpp(jniUtil->ConvertJavaObjectToCppString(env, spaceTypeJ)); faiss::MetricType metric = TranslateSpaceToMetric(spaceTypeCpp); + jniUtil->DeleteLocalRef(env, spaceTypeJ); - // Read vectors from memory address - auto *inputVectors = reinterpret_cast*>(vectorsAddressJ); + // Dimension int dim = (int)dimJ; - // The number of vectors can be int here because a lucene segment number of total docs never crosses INT_MAX value - int numVectors = (int) (inputVectors->size() / (uint64_t) dim); - if(numVectors == 0) { - throw std::runtime_error("Number of vectors cannot be 0"); - } + // Number of vectors int numIds = jniUtil->GetJavaIntArrayLength(env, idsJ); - if (numIds != numVectors) { - throw std::runtime_error("Number of IDs does not match number of vectors"); - } - // Create faiss index + // Index description jobject indexDescriptionJ = knn_jni::GetJObjectFromMapOrThrow(parametersCpp, knn_jni::INDEX_DESCRIPTION); std::string indexDescriptionCpp(jniUtil->ConvertJavaObjectToCppString(env, indexDescriptionJ)); + jniUtil->DeleteLocalRef(env, indexDescriptionJ); - std::unique_ptr indexWriter; - indexWriter.reset(faiss::index_factory(dim, indexDescriptionCpp.c_str(), metric)); - - // Set thread count if it is passed in as a parameter. Setting this variable will only impact the current thread + // Thread count + int threadCount = 0; if(parametersCpp.find(knn_jni::INDEX_THREAD_QUANTITY) != parametersCpp.end()) { - auto threadCount = jniUtil->ConvertJavaObjectToCppInteger(env, parametersCpp[knn_jni::INDEX_THREAD_QUANTITY]); - omp_set_num_threads(threadCount); - } - - // Add extra parameters that cant be configured with the index factory - if(parametersCpp.find(knn_jni::PARAMETERS) != parametersCpp.end()) { - jobject subParametersJ = parametersCpp[knn_jni::PARAMETERS]; - auto subParametersCpp = jniUtil->ConvertJavaMapToCppMap(env, subParametersJ); - SetExtraParameters(jniUtil, env, subParametersCpp, indexWriter.get()); - jniUtil->DeleteLocalRef(env, subParametersJ); + threadCount = jniUtil->ConvertJavaObjectToCppInteger(env, parametersCpp[knn_jni::INDEX_THREAD_QUANTITY]); } - jniUtil->DeleteLocalRef(env, parametersJ); - // Check that the index does not need to be trained - if(!indexWriter->is_trained) { - throw std::runtime_error("Index is not trained"); - } + // Vectors address + int64_t vectorsAddress = (int64_t)vectorsAddressJ; - auto idVector = jniUtil->ConvertJavaIntArrayToCppIntVector(env, idsJ); - faiss::IndexIDMap idMap = faiss::IndexIDMap(indexWriter.get()); - idMap.add_with_ids(numVectors, inputVectors->data(), idVector.data()); + // Ids + auto ids = jniUtil->ConvertJavaIntArrayToCppIntVector(env, idsJ); - // Write the index to disk + // Index path std::string indexPathCpp(jniUtil->ConvertJavaStringToCppString(env, indexPathJ)); - faiss::write_index(&idMap, indexPathCpp.c_str()); - // Releasing the vectorsAddressJ memory as that is not required once we have created the index. - // This is not the ideal approach, please refer this gh issue for long term solution: - // https://github.com/opensearch-project/k-NN/issues/1600 - delete inputVectors; + + // Extra parameters + // TODO: parse the entire map and remove jni object + std::unordered_map subParametersCpp; + if(parametersCpp.find(knn_jni::PARAMETERS) != parametersCpp.end()) { + subParametersCpp = jniUtil->ConvertJavaMapToCppMap(env, parametersCpp[knn_jni::PARAMETERS]); + } + // end parameters to pass + + // Create index + indexService->createIndex(jniUtil, env, metric, indexDescriptionCpp, dim, numIds, threadCount, vectorsAddress, ids, indexPathCpp, subParametersCpp); } void knn_jni::faiss_wrapper::CreateIndexFromTemplate(knn_jni::JNIUtilInterface * jniUtil, JNIEnv * env, jintArray idsJ, @@ -248,6 +236,19 @@ jlong knn_jni::faiss_wrapper::LoadIndex(knn_jni::JNIUtilInterface * jniUtil, JNI return (jlong) indexReader; } +jlong knn_jni::faiss_wrapper::LoadBinaryIndex(knn_jni::JNIUtilInterface * jniUtil, JNIEnv * env, jstring indexPathJ) { + if (indexPathJ == nullptr) { + throw std::runtime_error("Index path cannot be null"); + } + + std::string indexPathCpp(jniUtil->ConvertJavaStringToCppString(env, indexPathJ)); + // Skipping IO_FLAG_PQ_SKIP_SDC_TABLE because the index is read only and the sdc table is only used during ingestion + // Skipping IO_PRECOMPUTE_TABLE because it is only needed for IVFPQ-l2 and it leads to high memory consumption if + // done for each segment. Instead, we will set it later on with `setSharedIndexState` + faiss::IndexBinary* indexReader = faiss::read_index_binary(indexPathCpp.c_str(), faiss::IO_FLAG_READ_ONLY | faiss::IO_FLAG_PQ_SKIP_SDC_TABLE | faiss::IO_FLAG_SKIP_PRECOMPUTE_TABLE); + return (jlong) indexReader; +} + bool knn_jni::faiss_wrapper::IsSharedIndexStateRequired(jlong indexPointerJ) { auto * index = reinterpret_cast(indexPointerJ); return isIndexIVFPQL2(index); @@ -415,6 +416,121 @@ jobjectArray knn_jni::faiss_wrapper::QueryIndex_WithFilter(knn_jni::JNIUtilInter return results; } +jobjectArray knn_jni::faiss_wrapper::QueryBinaryIndex_WithFilter(knn_jni::JNIUtilInterface * jniUtil, JNIEnv * env, jlong indexPointerJ, + jbyteArray queryVectorJ, jint kJ, jobject methodParamsJ, jlongArray filterIdsJ, jint filterIdsTypeJ, jintArray parentIdsJ) { + + if (queryVectorJ == nullptr) { + throw std::runtime_error("Query Vector cannot be null"); + } + + auto *indexReader = reinterpret_cast(indexPointerJ); + + if (indexReader == nullptr) { + throw std::runtime_error("Invalid pointer to index"); + } + + std::unordered_map methodParams; + if (methodParamsJ != nullptr) { + methodParams = jniUtil->ConvertJavaMapToCppMap(env, methodParamsJ); + } + + // The ids vector will hold the top k ids from the search and the dis vector will hold the top k distances from + // the query point + std::vector dis(kJ); + std::vector ids(kJ); + int8_t* rawQueryvector = jniUtil->GetByteArrayElements(env, queryVectorJ, nullptr); + /* + Setting the omp_set_num_threads to 1 to make sure that no new OMP threads are getting created. + */ + omp_set_num_threads(1); + // create the filterSearch params if the filterIdsJ is not a null pointer + if(filterIdsJ != nullptr) { + jlong *filteredIdsArray = jniUtil->GetLongArrayElements(env, filterIdsJ, nullptr); + int filterIdsLength = jniUtil->GetJavaLongArrayLength(env, filterIdsJ); + std::unique_ptr idSelector; + if(filterIdsTypeJ == BITMAP) { + idSelector.reset(new faiss::IDSelectorJlongBitmap(filterIdsLength, filteredIdsArray)); + } else { + faiss::idx_t* batchIndices = reinterpret_cast(filteredIdsArray); + idSelector.reset(new faiss::IDSelectorBatch(filterIdsLength, batchIndices)); + } + faiss::SearchParameters *searchParameters; + faiss::SearchParametersHNSW hnswParams; + faiss::SearchParametersIVF ivfParams; + std::unique_ptr idGrouper; + std::vector idGrouperBitmap; + auto hnswReader = dynamic_cast(indexReader->index); + if(hnswReader) { + // Query param efsearch supersedes ef_search provided during index setting. + hnswParams.efSearch = knn_jni::commons::getIntegerMethodParameter(env, jniUtil, methodParams, EF_SEARCH, hnswReader->hnsw.efSearch); + hnswParams.sel = idSelector.get(); + if (parentIdsJ != nullptr) { + idGrouper = buildIDGrouperBitmap(jniUtil, env, parentIdsJ, &idGrouperBitmap); + hnswParams.grp = idGrouper.get(); + } + searchParameters = &hnswParams; + } else { + auto ivfReader = dynamic_cast(indexReader->index); + if(ivfReader) { + ivfParams.sel = idSelector.get(); + searchParameters = &ivfParams; + } + } + try { + indexReader->search(1, reinterpret_cast(rawQueryvector), kJ, dis.data(), ids.data(), searchParameters); + } catch (...) { + jniUtil->ReleaseByteArrayElements(env, queryVectorJ, rawQueryvector, JNI_ABORT); + jniUtil->ReleaseLongArrayElements(env, filterIdsJ, filteredIdsArray, JNI_ABORT); + throw; + } + jniUtil->ReleaseLongArrayElements(env, filterIdsJ, filteredIdsArray, JNI_ABORT); + } else { + faiss::SearchParameters *searchParameters = nullptr; + faiss::SearchParametersHNSW hnswParams; + std::unique_ptr idGrouper; + std::vector idGrouperBitmap; + auto hnswReader = dynamic_cast(indexReader->index); + // TODO currently, search parameter is not supported in binary index + // To avoid test failure, we skip setting ef search when methodPramsJ is null temporary + if(hnswReader!= nullptr && (methodParamsJ != nullptr || parentIdsJ != nullptr)) { + // Query param efsearch supersedes ef_search provided during index setting. + hnswParams.efSearch = knn_jni::commons::getIntegerMethodParameter(env, jniUtil, methodParams, EF_SEARCH, hnswReader->hnsw.efSearch); + if (parentIdsJ != nullptr) { + idGrouper = buildIDGrouperBitmap(jniUtil, env, parentIdsJ, &idGrouperBitmap); + hnswParams.grp = idGrouper.get(); + } + searchParameters = &hnswParams; + } + try { + indexReader->search(1, reinterpret_cast(rawQueryvector), kJ, dis.data(), ids.data(), searchParameters); + } catch (...) { + jniUtil->ReleaseByteArrayElements(env, queryVectorJ, rawQueryvector, JNI_ABORT); + throw; + } + } + jniUtil->ReleaseByteArrayElements(env, queryVectorJ, rawQueryvector, JNI_ABORT); + + // If there are not k results, the results will be padded with -1. Find the first -1, and set result size to that + // index + int resultSize = kJ; + auto it = std::find(ids.begin(), ids.end(), -1); + if (it != ids.end()) { + resultSize = it - ids.begin(); + } + + jclass resultClass = jniUtil->FindClass(env,"org/opensearch/knn/index/query/KNNQueryResult"); + jmethodID allArgs = jniUtil->FindMethod(env, "org/opensearch/knn/index/query/KNNQueryResult", ""); + + jobjectArray results = jniUtil->NewObjectArray(env, resultSize, resultClass, nullptr); + + jobject result; + for(int i = 0; i < resultSize; ++i) { + result = jniUtil->NewObject(env, resultClass, allArgs, ids[i], dis[i]); + jniUtil->SetObjectArrayElement(env, results, i, result); + } + return results; +} + void knn_jni::faiss_wrapper::Free(jlong indexPointer) { auto *indexWrapper = reinterpret_cast(indexPointer); delete indexWrapper; @@ -510,6 +626,11 @@ faiss::MetricType TranslateSpaceToMetric(const std::string& spaceType) { return faiss::METRIC_INNER_PRODUCT; } + // Space type is not used for binary index. Use L2 just to avoid an error. + if (spaceType == knn_jni::HAMMING_BIT) { + return faiss::METRIC_L2; + } + throw std::runtime_error("Invalid spaceType"); } diff --git a/jni/src/jni_util.cpp b/jni/src/jni_util.cpp index a1faa4894..919191596 100644 --- a/jni/src/jni_util.cpp +++ b/jni/src/jni_util.cpp @@ -261,6 +261,39 @@ void knn_jni::JNIUtil::Convert2dJavaObjectArrayAndStoreToFloatVector(JNIEnv *env env->DeleteLocalRef(array2dJ); } +void knn_jni::JNIUtil::Convert2dJavaObjectArrayAndStoreToByteVector(JNIEnv *env, jobjectArray array2dJ, + int dim, std::vector *vect) { + + if (array2dJ == nullptr) { + throw std::runtime_error("Array cannot be null"); + } + + int numVectors = env->GetArrayLength(array2dJ); + this->HasExceptionInStack(env); + + for (int i = 0; i < numVectors; ++i) { + auto vectorArray = (jbyteArray)env->GetObjectArrayElement(array2dJ, i); + this->HasExceptionInStack(env, "Unable to get object array element"); + + if (dim != env->GetArrayLength(vectorArray)) { + throw std::runtime_error("Dimension of vectors is inconsistent"); + } + + uint8_t* vector = reinterpret_cast(env->GetByteArrayElements(vectorArray, nullptr)); + if (vector == nullptr) { + this->HasExceptionInStack(env); + throw std::runtime_error("Unable to get byte array elements"); + } + + for(int j = 0; j < dim; ++j) { + vect->push_back(vector[j]); + } + env->ReleaseByteArrayElements(vectorArray, reinterpret_cast(vector), JNI_ABORT); + } + this->HasExceptionInStack(env); + env->DeleteLocalRef(array2dJ); +} + std::vector knn_jni::JNIUtil::ConvertJavaIntArrayToCppIntVector(JNIEnv *env, jintArray arrayJ) { if (arrayJ == nullptr) { @@ -302,6 +335,23 @@ int knn_jni::JNIUtil::GetInnerDimensionOf2dJavaFloatArray(JNIEnv *env, jobjectAr return dim; } +int knn_jni::JNIUtil::GetInnerDimensionOf2dJavaByteArray(JNIEnv *env, jobjectArray array2dJ) { + + if (array2dJ == nullptr) { + throw std::runtime_error("Array cannot be null"); + } + + if (env->GetArrayLength(array2dJ) <= 0) { + return 0; + } + + auto vectorArray = (jbyteArray)env->GetObjectArrayElement(array2dJ, 0); + this->HasExceptionInStack(env); + int dim = env->GetArrayLength(vectorArray); + this->HasExceptionInStack(env); + return dim; +} + int knn_jni::JNIUtil::GetJavaObjectArrayLength(JNIEnv *env, jobjectArray arrayJ) { if (arrayJ == nullptr) { @@ -490,6 +540,7 @@ const std::string knn_jni::LINF = "linf"; const std::string knn_jni::COSINESIMIL = "cosinesimil"; const std::string knn_jni::INNER_PRODUCT = "innerproduct"; const std::string knn_jni::NEG_DOT_PRODUCT = "negdotprod"; +const std::string knn_jni::HAMMING_BIT = "hammingbit"; const std::string knn_jni::NPROBES = "nprobes"; const std::string knn_jni::COARSE_QUANTIZER = "coarse_quantizer"; diff --git a/jni/src/org_opensearch_knn_jni_FaissService.cpp b/jni/src/org_opensearch_knn_jni_FaissService.cpp index d8cf2f9cf..5f9c83ea8 100644 --- a/jni/src/org_opensearch_knn_jni_FaissService.cpp +++ b/jni/src/org_opensearch_knn_jni_FaissService.cpp @@ -44,7 +44,32 @@ JNIEXPORT void JNICALL Java_org_opensearch_knn_jni_FaissService_createIndex(JNIE jstring indexPathJ, jobject parametersJ) { try { - knn_jni::faiss_wrapper::CreateIndex(&jniUtil, env, idsJ, vectorsAddressJ, dimJ, indexPathJ, parametersJ); + std::unique_ptr faissMethods(new knn_jni::faiss_wrapper::FaissMethods()); + knn_jni::faiss_wrapper::IndexService indexService(std::move(faissMethods)); + knn_jni::faiss_wrapper::CreateIndex(&jniUtil, env, idsJ, vectorsAddressJ, dimJ, indexPathJ, parametersJ, &indexService); + + // Releasing the vectorsAddressJ memory as that is not required once we have created the index. + // This is not the ideal approach, please refer this gh issue for long term solution: + // https://github.com/opensearch-project/k-NN/issues/1600 + delete reinterpret_cast*>(vectorsAddressJ); + } catch (...) { + jniUtil.CatchCppExceptionAndThrowJava(env); + } +} + +JNIEXPORT void JNICALL Java_org_opensearch_knn_jni_FaissService_createBinaryIndex(JNIEnv * env, jclass cls, jintArray idsJ, + jlong vectorsAddressJ, jint dimJ, + jstring indexPathJ, jobject parametersJ) +{ + try { + std::unique_ptr faissMethods(new knn_jni::faiss_wrapper::FaissMethods()); + knn_jni::faiss_wrapper::BinaryIndexService binaryIndexService(std::move(faissMethods)); + knn_jni::faiss_wrapper::CreateIndex(&jniUtil, env, idsJ, vectorsAddressJ, dimJ, indexPathJ, parametersJ, &binaryIndexService); + + // Releasing the vectorsAddressJ memory as that is not required once we have created the index. + // This is not the ideal approach, please refer this gh issue for long term solution: + // https://github.com/opensearch-project/k-NN/issues/1600 + delete reinterpret_cast*>(vectorsAddressJ); } catch (...) { jniUtil.CatchCppExceptionAndThrowJava(env); } @@ -75,6 +100,16 @@ JNIEXPORT jlong JNICALL Java_org_opensearch_knn_jni_FaissService_loadIndex(JNIEn return NULL; } +JNIEXPORT jlong JNICALL Java_org_opensearch_knn_jni_FaissService_loadBinaryIndex(JNIEnv * env, jclass cls, jstring indexPathJ) +{ + try { + return knn_jni::faiss_wrapper::LoadBinaryIndex(&jniUtil, env, indexPathJ); + } catch (...) { + jniUtil.CatchCppExceptionAndThrowJava(env); + } + return NULL; +} + JNIEXPORT jboolean JNICALL Java_org_opensearch_knn_jni_FaissService_isSharedIndexStateRequired (JNIEnv * env, jclass cls, jlong indexPointerJ) { @@ -132,6 +167,18 @@ JNIEXPORT jobjectArray JNICALL Java_org_opensearch_knn_jni_FaissService_queryInd } +JNIEXPORT jobjectArray JNICALL Java_org_opensearch_knn_jni_FaissService_queryBinaryIndexWithFilter + (JNIEnv * env, jclass cls, jlong indexPointerJ, jbyteArray queryVectorJ, jint kJ, jobject methodParamsJ, jlongArray filteredIdsJ, jint filterIdsTypeJ, jintArray parentIdsJ) { + + try { + return knn_jni::faiss_wrapper::QueryBinaryIndex_WithFilter(&jniUtil, env, indexPointerJ, queryVectorJ, kJ, methodParamsJ, filteredIdsJ, filterIdsTypeJ, parentIdsJ); + } catch (...) { + jniUtil.CatchCppExceptionAndThrowJava(env); + } + return nullptr; + +} + JNIEXPORT void JNICALL Java_org_opensearch_knn_jni_FaissService_free(JNIEnv * env, jclass cls, jlong indexPointerJ) { try { diff --git a/jni/src/org_opensearch_knn_jni_JNICommons.cpp b/jni/src/org_opensearch_knn_jni_JNICommons.cpp index ccdd11882..0bc2e4633 100644 --- a/jni/src/org_opensearch_knn_jni_JNICommons.cpp +++ b/jni/src/org_opensearch_knn_jni_JNICommons.cpp @@ -49,6 +49,18 @@ jlong memoryAddressJ, jobjectArray dataJ, jlong initialCapacityJ) return (long)memoryAddressJ; } +JNIEXPORT jlong JNICALL Java_org_opensearch_knn_jni_JNICommons_storeByteVectorData(JNIEnv * env, jclass cls, +jlong memoryAddressJ, jobjectArray dataJ, jlong initialCapacityJ) + +{ + try { + return knn_jni::commons::storeByteVectorData(&jniUtil, env, memoryAddressJ, dataJ, initialCapacityJ); + } catch (...) { + jniUtil.CatchCppExceptionAndThrowJava(env); + } + return (long)memoryAddressJ; +} + JNIEXPORT void JNICALL Java_org_opensearch_knn_jni_JNICommons_freeVectorData(JNIEnv * env, jclass cls, jlong memoryAddressJ) { @@ -58,3 +70,14 @@ JNIEXPORT void JNICALL Java_org_opensearch_knn_jni_JNICommons_freeVectorData(JNI jniUtil.CatchCppExceptionAndThrowJava(env); } } + + +JNIEXPORT void JNICALL Java_org_opensearch_knn_jni_JNICommons_freeByteVectorData(JNIEnv * env, jclass cls, + jlong memoryAddressJ) +{ + try { + return knn_jni::commons::freeByteVectorData(memoryAddressJ); + } catch (...) { + jniUtil.CatchCppExceptionAndThrowJava(env); + } +} diff --git a/jni/tests/faiss_index_service_test.cpp b/jni/tests/faiss_index_service_test.cpp new file mode 100644 index 000000000..f876edced --- /dev/null +++ b/jni/tests/faiss_index_service_test.cpp @@ -0,0 +1,134 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + + +#include "faiss_index_service.h" +#include "mocks/faiss_methods_mock.h" +#include "mocks/faiss_index_mock.h" +#include "test_util.h" +#include +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "commons.h" + +using ::testing::_; +using ::testing::NiceMock; +using ::testing::Return; + +TEST(CreateIndexTest, BasicAssertions) { + // Define the data + faiss::idx_t numIds = 200; + std::vector ids; + std::vector vectors; + int dim = 2; + vectors.reserve(dim * numIds); + for (int64_t i = 0; i < numIds; ++i) { + ids.push_back(i); + for (int j = 0; j < dim; ++j) { + vectors.push_back(test_util::RandomFloat(-500.0, 500.0)); + } + } + + std::string indexPath = test_util::RandomString(10, "tmp/", ".faiss"); + faiss::MetricType metricType = faiss::METRIC_L2; + std::string indexDescription = "HNSW32,Flat"; + int threadCount = 1; + std::unordered_map parametersMap; + + // Set up jni + JNIEnv *jniEnv = nullptr; + NiceMock mockJNIUtil; + + // Setup faiss method mock + // This object is handled by unique_ptr inside indexService.createIndex() + MockIndex* index = new MockIndex(); + EXPECT_CALL(*index, add(numIds, vectors.data())) + .Times(1); + // This object is handled by unique_ptr inside indexService.createIndex() + faiss::IndexIDMap* indexIdMap = new faiss::IndexIDMap(index); + std::unique_ptr mockFaissMethods(new MockFaissMethods()); + EXPECT_CALL(*mockFaissMethods, indexFactory(dim, ::testing::StrEq(indexDescription.c_str()), metricType)) + .WillOnce(Return(index)); + EXPECT_CALL(*mockFaissMethods, indexIdMap(index)) + .WillOnce(Return(indexIdMap)); + EXPECT_CALL(*mockFaissMethods, writeIndex(indexIdMap, ::testing::StrEq(indexPath.c_str()))) + .Times(1); + + // Create the index + knn_jni::faiss_wrapper::IndexService indexService(std::move(mockFaissMethods)); + indexService.createIndex( + &mockJNIUtil, + jniEnv, + metricType, + indexDescription, + dim, + numIds, + threadCount, + (int64_t) &vectors, + ids, + indexPath, + parametersMap); +} + +TEST(CreateBinaryIndexTest, BasicAssertions) { + // Define the data + faiss::idx_t numIds = 200; + std::vector ids; + std::vector vectors; + int dim = 128; + vectors.reserve(numIds); + for (int64_t i = 0; i < numIds; ++i) { + ids.push_back(i); + for (int j = 0; j < dim / 8; ++j) { + vectors.push_back(test_util::RandomInt(0, 255)); + } + } + + std::string indexPath = test_util::RandomString(10, "tmp/", ".faiss"); + faiss::MetricType metricType = faiss::METRIC_L2; + std::string indexDescription = "BHNSW32"; + int threadCount = 1; + std::unordered_map parametersMap; + + // Set up jni + JNIEnv *jniEnv = nullptr; + NiceMock mockJNIUtil; + + // Setup faiss method mock + // This object is handled by unique_ptr inside indexService.createIndex() + MockIndexBinary* index = new MockIndexBinary(); + EXPECT_CALL(*index, add(numIds, vectors.data())) + .Times(1); + // This object is handled by unique_ptr inside indexService.createIndex() + faiss::IndexBinaryIDMap* indexIdMap = new faiss::IndexBinaryIDMap(index); + std::unique_ptr mockFaissMethods(new MockFaissMethods()); + EXPECT_CALL(*mockFaissMethods, indexBinaryFactory(dim, ::testing::StrEq(indexDescription.c_str()))) + .WillOnce(Return(index)); + EXPECT_CALL(*mockFaissMethods, indexBinaryIdMap(index)) + .WillOnce(Return(indexIdMap)); + EXPECT_CALL(*mockFaissMethods, writeIndexBinary(indexIdMap, ::testing::StrEq(indexPath.c_str()))) + .Times(1); + + // Create the index + knn_jni::faiss_wrapper::BinaryIndexService indexService(std::move(mockFaissMethods)); + indexService.createIndex( + &mockJNIUtil, + jniEnv, + metricType, + indexDescription, + dim, + numIds, + threadCount, + (int64_t) &vectors, + ids, + indexPath, + parametersMap); +} \ No newline at end of file diff --git a/jni/tests/faiss_wrapper_test.cpp b/jni/tests/faiss_wrapper_test.cpp index f1d2ee7f4..c6663a19a 100644 --- a/jni/tests/faiss_wrapper_test.cpp +++ b/jni/tests/faiss_wrapper_test.cpp @@ -19,9 +19,12 @@ #include "test_util.h" #include "faiss/IndexHNSW.h" #include "faiss/IndexIVFPQ.h" +#include "mocks/faiss_index_service_mock.h" +using ::testing::_; using ::testing::NiceMock; using ::testing::Return; +using ::testing::Mock; float randomDataMin = -500.0; float randomDataMax = 500.0; @@ -33,44 +36,81 @@ TEST(FaissCreateIndexTest, BasicAssertions) { // Define the data faiss::idx_t numIds = 200; std::vector ids; - auto *vectors = new std::vector(); + std::vector vectors; int dim = 2; - vectors->reserve(dim * numIds); + vectors.reserve(dim * numIds); for (int64_t i = 0; i < numIds; ++i) { ids.push_back(i); for (int j = 0; j < dim; ++j) { - vectors->push_back(test_util::RandomFloat(-500.0, 500.0)); + vectors.push_back(test_util::RandomFloat(-500.0, 500.0)); } } std::string indexPath = test_util::RandomString(10, "tmp/", ".faiss"); std::string spaceType = knn_jni::L2; - std::string index_description = "HNSW32,Flat"; + std::string indexDescription = "HNSW32,Flat"; std::unordered_map parametersMap; parametersMap[knn_jni::SPACE_TYPE] = (jobject)&spaceType; - parametersMap[knn_jni::INDEX_DESCRIPTION] = (jobject)&index_description; + parametersMap[knn_jni::INDEX_DESCRIPTION] = (jobject)&indexDescription; + std::unordered_map subParametersMap; + parametersMap[knn_jni::PARAMETERS] = (jobject)&subParametersMap; // Set up jni JNIEnv *jniEnv = nullptr; NiceMock mockJNIUtil; - EXPECT_CALL(mockJNIUtil, - GetJavaObjectArrayLength( - jniEnv, reinterpret_cast(&vectors))) - .WillRepeatedly(Return(vectors->size())); - // Create the index + std::unique_ptr faissMethods(new FaissMethods()); + NiceMock mockIndexService(std::move(faissMethods)); + EXPECT_CALL(mockIndexService, createIndex(_, _, faiss::METRIC_L2, indexDescription, dim, (int)numIds, 0, (int64_t)&vectors, ids, indexPath, subParametersMap)) + .Times(1); + knn_jni::faiss_wrapper::CreateIndex( &mockJNIUtil, jniEnv, reinterpret_cast(&ids), - (jlong) vectors, dim , (jstring)&indexPath, - (jobject)¶metersMap); + (jlong) &vectors, dim , (jstring)&indexPath, + (jobject)¶metersMap, &mockIndexService); +} - // Make sure index can be loaded - std::unique_ptr index(test_util::FaissLoadIndex(indexPath)); +TEST(FaissCreateBinaryIndexTest, BasicAssertions) { + // Define the data + faiss::idx_t numIds = 200; + std::vector ids; + std::vector vectors; + int dim = 128; + vectors.reserve(numIds); + for (int64_t i = 0; i < numIds; ++i) { + ids.push_back(i); + for (int j = 0; j < dim / 8; ++j) { + vectors.push_back(test_util::RandomInt(0, 255)); + } + } - // Clean up - std::remove(indexPath.c_str()); + std::string indexPath = test_util::RandomString(10, "tmp/", ".faiss"); + std::string spaceType = knn_jni::HAMMING_BIT; + std::string indexDescription = "BHNSW32"; + + std::unordered_map parametersMap; + parametersMap[knn_jni::SPACE_TYPE] = (jobject)&spaceType; + parametersMap[knn_jni::INDEX_DESCRIPTION] = (jobject)&indexDescription; + std::unordered_map subParametersMap; + parametersMap[knn_jni::PARAMETERS] = (jobject)&subParametersMap; + + // Set up jni + JNIEnv *jniEnv = nullptr; + NiceMock mockJNIUtil; + + // Create the index + std::unique_ptr faissMethods(new FaissMethods()); + NiceMock mockIndexService(std::move(faissMethods)); + EXPECT_CALL(mockIndexService, createIndex(_, _, faiss::METRIC_L2, indexDescription, dim, (int)numIds, 0, (int64_t)&vectors, ids, indexPath, subParametersMap)) + .Times(1); + + // This method calls delete vectors at the end + knn_jni::faiss_wrapper::CreateIndex( + &mockJNIUtil, jniEnv, reinterpret_cast(&ids), + (jlong) &vectors, dim , (jstring)&indexPath, + (jobject)¶metersMap, &mockIndexService); } TEST(FaissCreateIndexFromTemplateTest, BasicAssertions) { @@ -168,6 +208,58 @@ TEST(FaissLoadIndexTest, BasicAssertions) { std::remove(indexPath.c_str()); } +TEST(FaissLoadBinaryIndexTest, BasicAssertions) { + // Define the data + faiss::idx_t numIds = 200; + std::vector ids; + auto vectors = std::vector(numIds); + int dim = 128; + for (int64_t i = 0; i < numIds; ++i) { + ids.push_back(i); + for (int j = 0; j < dim / 8; ++j) { + vectors.push_back(test_util::RandomInt(0, 255)); + } + } + + std::string indexPath = test_util::RandomString(10, "tmp/", ".faiss"); + std::string spaceType = knn_jni::HAMMING_BIT; + std::string method = "BHNSW32"; + + // Create the index + std::unique_ptr createdIndex( + test_util::FaissCreateBinaryIndex(dim, method)); + auto createdIndexWithData = + test_util::FaissAddBinaryData(createdIndex.get(), ids, vectors); + + test_util::FaissWriteBinaryIndex(&createdIndexWithData, indexPath); + + // Setup jni + JNIEnv *jniEnv = nullptr; + NiceMock mockJNIUtil; + + std::unique_ptr loadedIndexPointer( + reinterpret_cast(knn_jni::faiss_wrapper::LoadBinaryIndex( + &mockJNIUtil, jniEnv, (jstring)&indexPath))); + + // Compare serialized versions + auto createIndexSerialization = + test_util::FaissGetSerializedBinaryIndex(&createdIndexWithData); + auto loadedIndexSerialization = test_util::FaissGetSerializedBinaryIndex( + reinterpret_cast(loadedIndexPointer.get())); + + ASSERT_NE(0, loadedIndexSerialization.data.size()); + ASSERT_EQ(createIndexSerialization.data.size(), + loadedIndexSerialization.data.size()); + + for (int i = 0; i < loadedIndexSerialization.data.size(); ++i) { + ASSERT_EQ(createIndexSerialization.data[i], + loadedIndexSerialization.data[i]); + } + + // Clean up + std::remove(indexPath.c_str()); +} + TEST(FaissLoadIndexTest, HNSWPQDisableSdcTable) { // Check that when we load an HNSWPQ index, the sdc table is not present. faiss::idx_t numIds = 256; @@ -289,6 +381,61 @@ TEST(FaissQueryIndexTest, BasicAssertions) { } } +TEST(FaissQueryBinaryIndexTest, BasicAssertions) { + // Define the data + faiss::idx_t numIds = 200; + std::vector ids; + auto vectors = std::vector(numIds); + int dim = 128; + for (int64_t i = 0; i < numIds; ++i) { + ids.push_back(i); + for (int j = 0; j < dim / 8; ++j) { + vectors.push_back(test_util::RandomInt(0, 255)); + } + } + + // Define query data + int k = 10; + int numQueries = 100; + std::vector> queries; + + for (int i = 0; i < numQueries; i++) { + std::vector query; + query.reserve(dim); + for (int j = 0; j < dim; j++) { + query.push_back(test_util::RandomInt(0, 255)); + } + queries.push_back(query); + } + + // Create the index + std::string method = "BHNSW32"; + std::unique_ptr createdIndex( + test_util::FaissCreateBinaryIndex(dim, method)); + auto createdIndexWithData = + test_util::FaissAddBinaryData(createdIndex.get(), ids, vectors); + + // Setup jni + JNIEnv *jniEnv = nullptr; + NiceMock mockJNIUtil; + + for (auto query : queries) { + std::unique_ptr *>> results( + reinterpret_cast *> *>( + knn_jni::faiss_wrapper::QueryBinaryIndex_WithFilter( + &mockJNIUtil, jniEnv, + reinterpret_cast(&createdIndexWithData), + reinterpret_cast(&query), k, nullptr, nullptr, 0, nullptr))); + + ASSERT_EQ(k, results->size()); + + // Need to free up each result + for (auto it : *results.get()) { + delete it; + } + } +} + //Test for a bug reported in https://github.com/opensearch-project/k-NN/issues/1435 TEST(FaissQueryIndexWithFilterTest1435, BasicAssertions) { // Define the index data @@ -409,6 +556,10 @@ TEST(FaissQueryIndexWithParentFilterTest, BasicAssertions) { // Setup jni JNIEnv *jniEnv = nullptr; NiceMock mockJNIUtil; + EXPECT_CALL(mockJNIUtil, + GetJavaIntArrayLength( + jniEnv, reinterpret_cast(&parentIds))) + .WillRepeatedly(Return(parentIds.size())); for (auto query : queries) { std::unique_ptr *>> results( reinterpret_cast *> *>( @@ -488,13 +639,13 @@ TEST(FaissCreateHnswSQfp16IndexTest, BasicAssertions) { // Define the data faiss::idx_t numIds = 200; std::vector ids; - auto *vectors = new std::vector(); + std::vector vectors; int dim = 2; - vectors->reserve(dim * numIds); + vectors.reserve(dim * numIds); for (int64_t i = 0; i < numIds; ++i) { ids.push_back(i); for (int j = 0; j < dim; ++j) { - vectors->push_back(test_util::RandomFloat(-500.0, 500.0)); + vectors.push_back(test_util::RandomFloat(-500.0, 500.0)); } } @@ -513,13 +664,15 @@ TEST(FaissCreateHnswSQfp16IndexTest, BasicAssertions) { EXPECT_CALL(mockJNIUtil, GetJavaObjectArrayLength( jniEnv, reinterpret_cast(&vectors))) - .WillRepeatedly(Return(vectors->size())); + .WillRepeatedly(Return(vectors.size())); // Create the index + std::unique_ptr faissMethods(new FaissMethods()); + knn_jni::faiss_wrapper::IndexService IndexService(std::move(faissMethods)); knn_jni::faiss_wrapper::CreateIndex( &mockJNIUtil, jniEnv, reinterpret_cast(&ids), - (jlong)vectors, dim, (jstring)&indexPath, - (jobject)¶metersMap); + (jlong)&vectors, dim, (jstring)&indexPath, + (jobject)¶metersMap, &IndexService); // Make sure index can be loaded std::unique_ptr index(test_util::FaissLoadIndex(indexPath)); diff --git a/jni/tests/faiss_wrapper_unit_test.cpp b/jni/tests/faiss_wrapper_unit_test.cpp index d9fdac23f..d68ec69c6 100644 --- a/jni/tests/faiss_wrapper_unit_test.cpp +++ b/jni/tests/faiss_wrapper_unit_test.cpp @@ -25,12 +25,12 @@ using ::testing::NiceMock; using idx_t = faiss::idx_t; -struct MockIndex : faiss::IndexHNSW { - explicit MockIndex(idx_t d) : faiss::IndexHNSW(d, 32) { +struct FaissMockIndex : faiss::IndexHNSW { + explicit FaissMockIndex(idx_t d) : faiss::IndexHNSW(d, 32) { } }; -struct MockIdMap : faiss::IndexIDMap { +struct FaissMockIdMap : faiss::IndexIDMap { mutable idx_t nCalled{}; mutable const float *xCalled{}; mutable int kCalled{}; @@ -40,7 +40,7 @@ struct MockIdMap : faiss::IndexIDMap { mutable const faiss::SearchParametersHNSW *paramsCalled{}; mutable faiss::RangeSearchResult *resCalled{}; - explicit MockIdMap(MockIndex *index) : faiss::IndexIDMapTemplate(index) { + explicit FaissMockIdMap(FaissMockIndex *index) : faiss::IndexIDMapTemplate(index) { } void search( @@ -108,8 +108,8 @@ class FaissWrappeterParametrizedTestFixture : public testing::TestWithParam { @@ -119,8 +119,8 @@ class FaissWrapperParametrizedRangeSearchTestFixture : public testing::TestWithP } protected: - MockIndex index_; - MockIdMap id_map_; + FaissMockIndex index_; + FaissMockIdMap id_map_; }; namespace query_index_test { @@ -369,4 +369,3 @@ namespace range_search_test { ) ); } - diff --git a/jni/tests/mocks/faiss_index_mock.h b/jni/tests/mocks/faiss_index_mock.h new file mode 100644 index 000000000..521cbb2d3 --- /dev/null +++ b/jni/tests/mocks/faiss_index_mock.h @@ -0,0 +1,35 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + + #ifndef OPENSEARCH_KNN_FAISS_INDEX_MOCK_H + #define OPENSEARCH_KNN_FAISS_INDEX_MOCK_H + +#include "faiss/Index.h" +#include "faiss/IndexBinary.h" +#include + +using idx_t = int64_t; + +class MockIndex : public faiss::Index { +public: + MOCK_METHOD(void, add, (idx_t n, const float* x), (override)); + MOCK_METHOD(void, search, (idx_t n, const float* x, idx_t k, float* distances, idx_t* labels, const faiss::SearchParameters* params), (const, override)); + MOCK_METHOD(void, reset, (), (override)); +}; + +class MockIndexBinary : public faiss::IndexBinary { +public: + MOCK_METHOD(void, add, (idx_t n, const uint8_t* x), (override)); + MOCK_METHOD(void, search, (idx_t n, const uint8_t* x, idx_t k, int32_t* distances, idx_t* labels, const faiss::SearchParameters* params), (const, override)); + MOCK_METHOD(void, reset, (), (override)); +}; + +#endif // OPENSEARCH_KNN_FAISS_INDEX_MOCK_H \ No newline at end of file diff --git a/jni/tests/mocks/faiss_index_service_mock.h b/jni/tests/mocks/faiss_index_service_mock.h new file mode 100644 index 000000000..7af08c82e --- /dev/null +++ b/jni/tests/mocks/faiss_index_service_mock.h @@ -0,0 +1,44 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +#ifndef OPENSEARCH_KNN_FAISS_INDEX_SERVICE_MOCK_H +#define OPENSEARCH_KNN_FAISS_INDEX_SERVICE_MOCK_H + +#include "faiss_index_service.h" +#include + +using ::knn_jni::faiss_wrapper::FaissMethods; +using ::knn_jni::faiss_wrapper::IndexService; +typedef std::unordered_map StringToJObjectMap; + +class MockIndexService : public IndexService { +public: + MockIndexService(std::unique_ptr faissMethods) : IndexService(std::move(faissMethods)) {}; + MOCK_METHOD( + void, + createIndex, + ( + knn_jni::JNIUtilInterface * jniUtil, + JNIEnv * env, + faiss::MetricType metric, + std::string indexDescription, + int dim, + int numIds, + int threadCount, + int64_t vectorsAddress, + std::vector ids, + std::string indexPath, + StringToJObjectMap parameters + ), + (override)); +}; + +#endif // OPENSEARCH_KNN_FAISS_INDEX_SERVICE_MOCK_H \ No newline at end of file diff --git a/jni/tests/mocks/faiss_methods_mock.h b/jni/tests/mocks/faiss_methods_mock.h new file mode 100644 index 000000000..64a23b895 --- /dev/null +++ b/jni/tests/mocks/faiss_methods_mock.h @@ -0,0 +1,28 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + + #ifndef OPENSEARCH_KNN_FAISS_METHODS_MOCK_H + #define OPENSEARCH_KNN_FAISS_METHODS_MOCK_H + +#include "faiss_methods.h" +#include + +class MockFaissMethods : public knn_jni::faiss_wrapper::FaissMethods { +public: + MOCK_METHOD(faiss::Index*, indexFactory, (int d, const char* description, faiss::MetricType metric), (override)); + MOCK_METHOD(faiss::IndexBinary*, indexBinaryFactory, (int d, const char* description), (override)); + MOCK_METHOD(faiss::IndexIDMapTemplate*, indexIdMap, (faiss::Index* index), (override)); + MOCK_METHOD(faiss::IndexIDMapTemplate*, indexBinaryIdMap, (faiss::IndexBinary* index), (override)); + MOCK_METHOD(void, writeIndex, (const faiss::Index* idx, const char* fname), (override)); + MOCK_METHOD(void, writeIndexBinary, (const faiss::IndexBinary* idx, const char* fname), (override)); +}; + +#endif // OPENSEARCH_KNN_FAISS_METHODS_MOCK_H \ No newline at end of file diff --git a/jni/tests/test_util.cpp b/jni/tests/test_util.cpp index 92532b9e2..2149f8a1a 100644 --- a/jni/tests/test_util.cpp +++ b/jni/tests/test_util.cpp @@ -51,6 +51,12 @@ test_util::MockJNIUtil::MockJNIUtil() { (*reinterpret_cast> *>(array2dJ))) for (auto item : v) data->push_back(item); }); + ON_CALL(*this, Convert2dJavaObjectArrayAndStoreToByteVector) + .WillByDefault([this](JNIEnv *env, jobjectArray array2dJ, int dim, std::vector* data) { + for (const auto &v : + (*reinterpret_cast> *>(array2dJ))) + for (auto item : v) data->push_back(item); + }); // arrayJ is re-interpreted as std::vector * @@ -150,6 +156,15 @@ test_util::MockJNIUtil::MockJNIUtil() { .size(); }); + // array2dJ is re-interpreted as a std::vector> * and then + // the size of the first element is returned + ON_CALL(*this, GetInnerDimensionOf2dJavaByteArray) + .WillByDefault([this](JNIEnv *env, jobjectArray array2dJ) { + return (*reinterpret_cast> *>( + array2dJ))[0] + .size(); + }); + // arrayJ is re-interpreted as a std::vector * and the size is returned ON_CALL(*this, GetJavaFloatArrayLength) .WillByDefault([this](JNIEnv *env, jfloatArray arrayJ) { @@ -249,12 +264,22 @@ faiss::Index *test_util::FaissCreateIndex(int dim, const std::string &method, return faiss::index_factory(dim, method.c_str(), metric); } +faiss::IndexBinary *test_util::FaissCreateBinaryIndex(int dim, const std::string &method) { + return faiss::index_binary_factory(dim, method.c_str()); +} + faiss::VectorIOWriter test_util::FaissGetSerializedIndex(faiss::Index *index) { faiss::VectorIOWriter vectorIoWriter; faiss::write_index(index, &vectorIoWriter); return vectorIoWriter; } +faiss::VectorIOWriter test_util::FaissGetSerializedBinaryIndex(faiss::IndexBinary *index) { + faiss::VectorIOWriter vectorIoWriter; + faiss::write_index_binary(index, &vectorIoWriter); + return vectorIoWriter; +} + faiss::Index *test_util::FaissLoadFromSerializedIndex( std::vector *indexSerial) { faiss::VectorIOReader vectorIoReader; @@ -262,6 +287,13 @@ faiss::Index *test_util::FaissLoadFromSerializedIndex( return faiss::read_index(&vectorIoReader, 0); } +faiss::IndexBinary *test_util::FaissLoadFromSerializedBinaryIndex( + std::vector *indexSerial) { + faiss::VectorIOReader vectorIoReader; + vectorIoReader.data = *indexSerial; + return faiss::read_index_binary(&vectorIoReader, 0); +} + faiss::IndexIDMap test_util::FaissAddData(faiss::Index *index, std::vector ids, std::vector dataset) { @@ -270,15 +302,32 @@ faiss::IndexIDMap test_util::FaissAddData(faiss::Index *index, return idMap; } +faiss::IndexBinaryIDMap test_util::FaissAddBinaryData(faiss::IndexBinary *index, + std::vector ids, + std::vector dataset) { + faiss::IndexBinaryIDMap idMap = faiss::IndexBinaryIDMap(index); + idMap.add_with_ids(ids.size(), dataset.data(), ids.data()); + return idMap; +} + void test_util::FaissWriteIndex(faiss::Index *index, const std::string &indexPath) { faiss::write_index(index, indexPath.c_str()); } +void test_util::FaissWriteBinaryIndex(faiss::IndexBinary *index, + const std::string &indexPath) { + faiss::write_index_binary(index, indexPath.c_str()); +} + faiss::Index *test_util::FaissLoadIndex(const std::string &indexPath) { return faiss::read_index(indexPath.c_str(), faiss::IO_FLAG_READ_ONLY); } +faiss::IndexBinary *test_util::FaissLoadBinaryIndex(const std::string &indexPath) { + return faiss::read_index_binary(indexPath.c_str(), faiss::IO_FLAG_READ_ONLY); +} + void test_util::FaissQueryIndex(faiss::Index *index, float *query, int k, float *distances, faiss::idx_t *ids) { index->search(1, query, k, distances, ids); @@ -377,6 +426,13 @@ float test_util::RandomFloat(float min, float max) { return distribution(e1); } +int test_util::RandomInt(int min, int max) { + std::random_device r; + std::default_random_engine e1(r()); + std::uniform_int_distribution distribution(min, max); + return distribution(e1); +} + std::vector test_util::RandomVectors(int dim, int64_t numVectors, float min, float max) { std::vector vectors(dim*numVectors); for (int64_t i = 0; i < dim*numVectors; i++) { diff --git a/jni/tests/test_util.h b/jni/tests/test_util.h index 8e73a8ab0..ba773fad3 100644 --- a/jni/tests/test_util.h +++ b/jni/tests/test_util.h @@ -46,6 +46,8 @@ namespace test_util { (JNIEnv * env, jobjectArray array2dJ, int dim)); MOCK_METHOD(void, Convert2dJavaObjectArrayAndStoreToFloatVector, (JNIEnv * env, jobjectArray array2dJ, int dim, std::vector*vect)); + MOCK_METHOD(void, Convert2dJavaObjectArrayAndStoreToByteVector, + (JNIEnv * env, jobjectArray array2dJ, int dim, std::vector*vect)); MOCK_METHOD(std::vector, ConvertJavaIntArrayToCppIntVector, (JNIEnv * env, jintArray arrayJ)); MOCK_METHOD2(ConvertJavaMapToCppMap, @@ -64,6 +66,8 @@ namespace test_util { (JNIEnv * env, jfloatArray array, jboolean* isCopy)); MOCK_METHOD(int, GetInnerDimensionOf2dJavaFloatArray, (JNIEnv * env, jobjectArray array2dJ)); + MOCK_METHOD(int, GetInnerDimensionOf2dJavaByteArray, + (JNIEnv * env, jobjectArray array2dJ)); MOCK_METHOD(jint*, GetIntArrayElements, (JNIEnv * env, jintArray array, jboolean* isCopy)); MOCK_METHOD(jlong*, GetLongArrayElements, @@ -109,18 +113,25 @@ namespace test_util { faiss::Index* FaissCreateIndex(int dim, const std::string& method, faiss::MetricType metric); + faiss::IndexBinary* FaissCreateBinaryIndex(int dim, const std::string& method); faiss::VectorIOWriter FaissGetSerializedIndex(faiss::Index* index); + faiss::VectorIOWriter FaissGetSerializedBinaryIndex(faiss::IndexBinary* index); faiss::Index* FaissLoadFromSerializedIndex(std::vector* indexSerial); + faiss::IndexBinary* FaissLoadFromSerializedBinaryIndex(std::vector* indexSerial); faiss::IndexIDMap FaissAddData(faiss::Index* index, std::vector ids, std::vector dataset); - + faiss::IndexBinaryIDMap FaissAddBinaryData(faiss::IndexBinary* index, + std::vector ids, + std::vector dataset); void FaissWriteIndex(faiss::Index* index, const std::string& indexPath); + void FaissWriteBinaryIndex(faiss::IndexBinary* index, const std::string& indexPath); faiss::Index* FaissLoadIndex(const std::string& indexPath); + faiss::IndexBinary* FaissLoadBinaryIndex(const std::string &indexPath); void FaissQueryIndex(faiss::Index* index, float* query, int k, float* distances, faiss::idx_t* ids); @@ -156,6 +167,7 @@ namespace test_util { std::string RandomString(size_t length, const std::string& prefix, const std::string& suffix); float RandomFloat(float min, float max); + int RandomInt(int min, int max); std::vector RandomVectors(int dim, int64_t numVectors, float min, float max); diff --git a/src/main/java/org/opensearch/knn/index/IndexUtil.java b/src/main/java/org/opensearch/knn/index/IndexUtil.java index 5cd4aaf81..7a5e93e14 100644 --- a/src/main/java/org/opensearch/knn/index/IndexUtil.java +++ b/src/main/java/org/opensearch/knn/index/IndexUtil.java @@ -35,6 +35,7 @@ import static org.opensearch.knn.common.KNNConstants.BYTES_PER_KILOBYTES; import static org.opensearch.knn.common.KNNConstants.HNSW_ALGO_EF_SEARCH; +import static org.opensearch.knn.common.KNNConstants.INDEX_DESCRIPTION_PARAMETER; import static org.opensearch.knn.common.KNNConstants.SPACE_TYPE; public class IndexUtil { @@ -258,9 +259,15 @@ public static ValidationException validateKnnField( * @param spaceType Space for this particular segment * @param knnEngine Engine used for the native library indices being loaded in * @param indexName Name of OpenSearch index that the segment files belong to + * @param indexDescription Index description of OpenSearch index with faiss that the segment files belong to * @return load parameters that will be passed to the JNI. */ - public static Map getParametersAtLoading(SpaceType spaceType, KNNEngine knnEngine, String indexName) { + public static Map getParametersAtLoading( + SpaceType spaceType, + KNNEngine knnEngine, + String indexName, + String indexDescription + ) { Map loadParameters = Maps.newHashMap(ImmutableMap.of(SPACE_TYPE, spaceType.getValue())); // For nmslib, we need to add the dynamic ef_search parameter that needs to be passed in when the @@ -268,6 +275,9 @@ public static Map getParametersAtLoading(SpaceType spaceType, KN if (KNNEngine.NMSLIB.equals(knnEngine)) { loadParameters.put(HNSW_ALGO_EF_SEARCH, KNNSettings.getEfSearchParam(indexName)); } + if (KNNEngine.FAISS.equals(knnEngine)) { + loadParameters.put(INDEX_DESCRIPTION_PARAMETER, indexDescription); + } return Collections.unmodifiableMap(loadParameters); } @@ -302,5 +312,4 @@ public static boolean isSharedIndexStateRequired(KNNEngine knnEngine, String mod } return JNIService.isSharedIndexStateRequired(indexAddr, knnEngine); } - } diff --git a/src/main/java/org/opensearch/knn/index/KNNIndexShard.java b/src/main/java/org/opensearch/knn/index/KNNIndexShard.java index efa09662c..e00d36d2e 100644 --- a/src/main/java/org/opensearch/knn/index/KNNIndexShard.java +++ b/src/main/java/org/opensearch/knn/index/KNNIndexShard.java @@ -23,6 +23,7 @@ import org.opensearch.knn.index.memory.NativeMemoryCacheManager; import org.opensearch.knn.index.memory.NativeMemoryEntryContext; import org.opensearch.knn.index.memory.NativeMemoryLoadStrategy; +import org.opensearch.knn.index.util.FieldInfoExtractor; import org.opensearch.knn.index.util.KNNEngine; import java.io.IOException; @@ -96,7 +97,8 @@ public void warmup() throws IOException { getParametersAtLoading( engineFileContext.getSpaceType(), KNNEngine.getEngineNameFromPath(engineFileContext.getIndexPath()), - getIndexName() + getIndexName(), + engineFileContext.indexDescription ), getIndexName(), engineFileContext.getModelId() @@ -171,7 +173,6 @@ List getEngineFileContexts(IndexReader indexReader, KNNEngine String spaceTypeName = fieldInfo.attributes().getOrDefault(SPACE_TYPE, SpaceType.L2.getValue()); SpaceType spaceType = SpaceType.getSpace(spaceTypeName); String modelId = fieldInfo.attributes().getOrDefault(MODEL_ID, null); - engineFiles.addAll( getEngineFileContexts( reader.getSegmentInfo().files(), @@ -180,7 +181,8 @@ List getEngineFileContexts(IndexReader indexReader, KNNEngine fileExtension, shardPath, spaceType, - modelId + modelId, + FieldInfoExtractor.getIndexDescription(fieldInfo) ) ); } @@ -197,7 +199,8 @@ List getEngineFileContexts( String fileExtension, Path shardPath, SpaceType spaceType, - String modelId + String modelId, + String indexDescription ) { String prefix = buildEngineFilePrefix(segmentName); String suffix = buildEngineFileSuffix(fieldName, fileExtension); @@ -205,7 +208,7 @@ List getEngineFileContexts( .filter(fileName -> fileName.startsWith(prefix)) .filter(fileName -> fileName.endsWith(suffix)) .map(fileName -> shardPath.resolve(fileName).toString()) - .map(fileName -> new EngineFileContext(spaceType, modelId, fileName)) + .map(fileName -> new EngineFileContext(spaceType, modelId, fileName, indexDescription)) .collect(Collectors.toList()); } @@ -216,5 +219,6 @@ static class EngineFileContext { private final SpaceType spaceType; private final String modelId; private final String indexPath; + private final String indexDescription; } } diff --git a/src/main/java/org/opensearch/knn/index/query/KNNWeight.java b/src/main/java/org/opensearch/knn/index/query/KNNWeight.java index 539a08a02..fce8e8e04 100644 --- a/src/main/java/org/opensearch/knn/index/query/KNNWeight.java +++ b/src/main/java/org/opensearch/knn/index/query/KNNWeight.java @@ -37,6 +37,7 @@ import org.opensearch.knn.index.memory.NativeMemoryLoadStrategy; import org.opensearch.knn.index.query.filtered.FilteredIdsKNNIterator; import org.opensearch.knn.index.query.filtered.NestedFilteredIdsKNNIterator; +import org.opensearch.knn.index.util.FieldInfoExtractor; import org.opensearch.knn.index.util.KNNEngine; import org.opensearch.knn.indices.ModelDao; import org.opensearch.knn.indices.ModelMetadata; @@ -256,7 +257,12 @@ private Map doANNSearch(final LeafReaderContext context, final B new NativeMemoryEntryContext.IndexEntryContext( indexPath.toString(), NativeMemoryLoadStrategy.IndexLoadStrategy.getInstance(), - getParametersAtLoading(spaceType, knnEngine, knnQuery.getIndexName()), + getParametersAtLoading( + spaceType, + knnEngine, + knnQuery.getIndexName(), + FieldInfoExtractor.getIndexDescription(fieldInfo) + ), knnQuery.getIndexName(), modelId ), diff --git a/src/main/java/org/opensearch/knn/index/util/FieldInfoExtractor.java b/src/main/java/org/opensearch/knn/index/util/FieldInfoExtractor.java new file mode 100644 index 000000000..5ad271969 --- /dev/null +++ b/src/main/java/org/opensearch/knn/index/util/FieldInfoExtractor.java @@ -0,0 +1,37 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.util; + +import org.apache.lucene.index.FieldInfo; +import org.opensearch.common.xcontent.XContentHelper; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.xcontent.DeprecationHandler; +import org.opensearch.core.xcontent.MediaTypeRegistry; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.knn.common.KNNConstants; + +import java.io.IOException; + +import static org.opensearch.knn.common.KNNConstants.INDEX_DESCRIPTION_PARAMETER; + +/** + * Class having methods to extract a value from field info + */ +public class FieldInfoExtractor { + public static String getIndexDescription(FieldInfo fieldInfo) throws IOException { + String parameters = fieldInfo.attributes().get(KNNConstants.PARAMETERS); + if (parameters == null) { + return null; + } + + return (String) XContentHelper.createParser( + NamedXContentRegistry.EMPTY, + DeprecationHandler.THROW_UNSUPPORTED_OPERATION, + new BytesArray(parameters), + MediaTypeRegistry.getDefaultMediaType() + ).map().getOrDefault(INDEX_DESCRIPTION_PARAMETER, null); + } +} diff --git a/src/main/java/org/opensearch/knn/jni/FaissService.java b/src/main/java/org/opensearch/knn/jni/FaissService.java index 2c537cf00..f718ce6d5 100644 --- a/src/main/java/org/opensearch/knn/jni/FaissService.java +++ b/src/main/java/org/opensearch/knn/jni/FaissService.java @@ -63,6 +63,20 @@ class FaissService { */ public static native void createIndex(int[] ids, long vectorsAddress, int dim, String indexPath, Map parameters); + /** + * Create a binary index for the native library The memory occupied by the vectorsAddress will be freed up during the + * function call. So Java layer doesn't need to free up the memory. This is not an ideal behavior because Java layer + * created the memory address and that should only free up the memory. We are tracking the proper fix for this on this + * issue + * + * @param ids array of ids mapping to the data passed in + * @param vectorsAddress address of native memory where vectors are stored + * @param dim dimension of the vector to be indexed + * @param indexPath path to save index file to + * @param parameters parameters to build index + */ + public static native void createBinaryIndex(int[] ids, long vectorsAddress, int dim, String indexPath, Map parameters); + /** * Create an index for the native library with a provided template index * @@ -90,6 +104,14 @@ public static native void createIndexFromTemplate( */ public static native long loadIndex(String indexPath); + /** + * Load a binary index into memory + * + * @param indexPath path to index file + * @return pointer to location in memory the index resides in + */ + public static native long loadBinaryIndex(String indexPath); + /** * Determine if index contains shared state. * @@ -126,6 +148,7 @@ public static native void createIndexFromTemplate( * @param indexPointer pointer to index in memory * @param queryVector vector to be used for query * @param k neighbors to be returned + * @param methodParameters method parameter * @param parentIds list of parent doc ids when the knn field is a nested field * @return KNNQueryResult array of k neighbors */ @@ -143,6 +166,7 @@ public static native KNNQueryResult[] queryIndex( * @param indexPointer pointer to index in memory * @param queryVector vector to be used for query * @param k neighbors to be returned + * @param methodParameters method parameter * @param filterIds list of doc ids to include in the query result * @param parentIds list of parent doc ids when the knn field is a nested field * @return KNNQueryResult array of k neighbors @@ -157,6 +181,27 @@ public static native KNNQueryResult[] queryIndexWithFilter( int[] parentIds ); + /** + * Query a binary index with filter + * + * @param indexPointer pointer to index in memory + * @param queryVector vector to be used for query + * @param k neighbors to be returned + * @param methodParameters method parameter + * @param filterIds list of doc ids to include in the query result + * @param parentIds list of parent doc ids when the knn field is a nested field + * @return KNNQueryResult array of k neighbors + */ + public static native KNNQueryResult[] queryBinaryIndexWithFilter( + long indexPointer, + byte[] queryVector, + int k, + Map methodParameters, + long[] filterIds, + int filterIdsType, + int[] parentIds + ); + /** * Free native memory pointer */ diff --git a/src/main/java/org/opensearch/knn/jni/JNICommons.java b/src/main/java/org/opensearch/knn/jni/JNICommons.java index 90ad70c3d..d0111b115 100644 --- a/src/main/java/org/opensearch/knn/jni/JNICommons.java +++ b/src/main/java/org/opensearch/knn/jni/JNICommons.java @@ -47,6 +47,25 @@ public class JNICommons { */ public static native long storeVectorData(long memoryAddress, float[][] data, long initialCapacity); + /** + * This is utility function that can be used to store data in native memory. This function will allocate memory for + * the data(rows*columns) with initialCapacity and return the memory address where the data is stored. + * If you are using this function for first time use memoryAddress = 0 to ensure that a new memory location is created. + * For subsequent calls you can pass the same memoryAddress. If the data cannot be stored in the memory location + * will throw Exception. + * + *

+ * The function is not threadsafe. If multiple threads are trying to insert on same memory location, then it can + * lead to data corruption. + *

+ * + * @param memoryAddress The address of the memory location where data will be stored. + * @param data 2D byte array containing data to be stored in native memory. + * @param initialCapacity The initial capacity of the memory location. + * @return memory address where the data is stored. + */ + public static native long storeByteVectorData(long memoryAddress, byte[][] data, long initialCapacity); + /** * Free up the memory allocated for the data stored in memory address. This function should be used with the memory * address returned by {@link JNICommons#storeVectorData(long, float[][], long)} diff --git a/src/main/java/org/opensearch/knn/jni/JNIService.java b/src/main/java/org/opensearch/knn/jni/JNIService.java index 83afc592f..ed6a169c1 100644 --- a/src/main/java/org/opensearch/knn/jni/JNIService.java +++ b/src/main/java/org/opensearch/knn/jni/JNIService.java @@ -13,6 +13,7 @@ import org.apache.commons.lang.ArrayUtils; import org.opensearch.common.Nullable; +import org.opensearch.knn.common.KNNConstants; import org.opensearch.knn.index.query.KNNQueryResult; import org.opensearch.knn.index.util.KNNEngine; @@ -22,6 +23,7 @@ * Service to distribute requests to the proper engine jni service */ public class JNIService { + private static final String FAISS_BINARY_INDEX_PREFIX = "B"; /** * Create an index for the native library. The memory occupied by the vectorsAddress will be freed up during the @@ -51,7 +53,12 @@ public static void createIndex( } if (KNNEngine.FAISS == knnEngine) { - FaissService.createIndex(ids, vectorsAddress, dim, indexPath, parameters); + if (parameters.get(KNNConstants.INDEX_DESCRIPTION_PARAMETER) != null + && parameters.get(KNNConstants.INDEX_DESCRIPTION_PARAMETER).toString().startsWith(FAISS_BINARY_INDEX_PREFIX)) { + FaissService.createBinaryIndex(ids, vectorsAddress, dim, indexPath, parameters); + } else { + FaissService.createIndex(ids, vectorsAddress, dim, indexPath, parameters); + } return; } @@ -102,7 +109,12 @@ public static long loadIndex(String indexPath, Map parameters, K } if (KNNEngine.FAISS == knnEngine) { - return FaissService.loadIndex(indexPath); + if (parameters.get(KNNConstants.INDEX_DESCRIPTION_PARAMETER) != null + && parameters.get(KNNConstants.INDEX_DESCRIPTION_PARAMETER).toString().startsWith(FAISS_BINARY_INDEX_PREFIX)) { + return FaissService.loadBinaryIndex(indexPath); + } else { + return FaissService.loadIndex(indexPath); + } } throw new IllegalArgumentException(String.format("LoadIndex not supported for provided engine : %s", knnEngine.getName())); @@ -162,12 +174,13 @@ public static void setSharedIndexState(long indexAddr, long shareIndexStateAddr, /** * Query an index * - * @param indexPointer pointer to index in memory - * @param queryVector vector to be used for query - * @param k neighbors to be returned - * @param knnEngine engine to query index - * @param filteredIds array of ints on which should be used for search. - * @param filterIdsType how to filter ids: Batch or BitMap + * @param indexPointer pointer to index in memory + * @param queryVector vector to be used for query + * @param k neighbors to be returned + * @param methodParameters method parameter + * @param knnEngine engine to query index + * @param filteredIds array of ints on which should be used for search. + * @param filterIdsType how to filter ids: Batch or BitMap * @return KNNQueryResult array of k neighbors */ public static KNNQueryResult[] queryIndex( @@ -205,6 +218,42 @@ public static KNNQueryResult[] queryIndex( throw new IllegalArgumentException(String.format("QueryIndex not supported for provided engine : %s", knnEngine.getName())); } + /** + * Query a binary index + * + * @param indexPointer pointer to index in memory + * @param queryVector vector to be used for query + * @param k neighbors to be returned + * @param methodParameters method parameter + * @param knnEngine engine to query index + * @param filteredIds array of ints on which should be used for search. + * @param filterIdsType how to filter ids: Batch or BitMap + * @return KNNQueryResult array of k neighbors + */ + public static KNNQueryResult[] queryBinaryIndex( + long indexPointer, + byte[] queryVector, + int k, + @Nullable Map methodParameters, + KNNEngine knnEngine, + long[] filteredIds, + int filterIdsType, + int[] parentIds + ) { + if (KNNEngine.FAISS == knnEngine) { + return FaissService.queryBinaryIndexWithFilter( + indexPointer, + queryVector, + k, + methodParameters, + ArrayUtils.isEmpty(filteredIds) ? null : filteredIds, + filterIdsType, + parentIds + ); + } + throw new IllegalArgumentException(String.format("QueryBinaryIndex not supported for provided engine : %s", knnEngine.getName())); + } + /** * Free native memory pointer * diff --git a/src/test/java/org/opensearch/knn/index/IndexUtilTests.java b/src/test/java/org/opensearch/knn/index/IndexUtilTests.java index 00493b293..e6c3e96ee 100644 --- a/src/test/java/org/opensearch/knn/index/IndexUtilTests.java +++ b/src/test/java/org/opensearch/knn/index/IndexUtilTests.java @@ -57,9 +57,10 @@ public void testGetLoadParameters() { SpaceType spaceType1 = SpaceType.COSINESIMIL; KNNEngine knnEngine1 = KNNEngine.FAISS; String indexName = "my-test-index"; + String indexDescription = "HNSW32Flat"; - Map loadParameters = getParametersAtLoading(spaceType1, knnEngine1, indexName); - assertEquals(1, loadParameters.size()); + Map loadParameters = getParametersAtLoading(spaceType1, knnEngine1, indexName, indexDescription); + assertEquals(2, loadParameters.size()); assertEquals(spaceType1.getValue(), loadParameters.get(SPACE_TYPE)); // Test nmslib to ensure both space type and ef search are properly set @@ -84,7 +85,7 @@ public void testGetLoadParameters() { when(clusterService.state()).thenReturn(clusterState); KNNSettings.state().setClusterService(clusterService); - loadParameters = getParametersAtLoading(spaceType2, knnEngine2, indexName); + loadParameters = getParametersAtLoading(spaceType2, knnEngine2, indexName, null); assertEquals(2, loadParameters.size()); assertEquals(spaceType2.getValue(), loadParameters.get(SPACE_TYPE)); assertEquals(efSearchValue, loadParameters.get(HNSW_ALGO_EF_SEARCH)); diff --git a/src/test/java/org/opensearch/knn/index/KNNIndexShardTests.java b/src/test/java/org/opensearch/knn/index/KNNIndexShardTests.java index 7b6f96d5a..42a59d26f 100644 --- a/src/test/java/org/opensearch/knn/index/KNNIndexShardTests.java +++ b/src/test/java/org/opensearch/knn/index/KNNIndexShardTests.java @@ -125,6 +125,7 @@ public void testGetEngineFileContexts() { String fileExt = ".test"; SpaceType spaceType = SpaceType.L2; String modelId = "test-model"; + String indexDescription = "test-description"; Set includedFileNames = ImmutableSet.of( String.format("%s_111_%s%s", segmentName, fieldName, fileExt), @@ -150,7 +151,8 @@ public void testGetEngineFileContexts() { fileExt, path, spaceType, - modelId + modelId, + indexDescription ); assertEquals(includedFileNames.size(), included.size()); diff --git a/src/test/java/org/opensearch/knn/index/query/KNNWeightTests.java b/src/test/java/org/opensearch/knn/index/query/KNNWeightTests.java index 75cd6e7a9..e73e86e90 100644 --- a/src/test/java/org/opensearch/knn/index/query/KNNWeightTests.java +++ b/src/test/java/org/opensearch/knn/index/query/KNNWeightTests.java @@ -54,6 +54,7 @@ import java.util.Arrays; import java.util.Comparator; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Set; import java.util.function.Function; @@ -71,9 +72,11 @@ import static org.mockito.Mockito.mockStatic; import static org.mockito.Mockito.when; import static org.opensearch.knn.KNNRestTestCase.INDEX_NAME; +import static org.opensearch.knn.common.KNNConstants.INDEX_DESCRIPTION_PARAMETER; import static org.opensearch.knn.common.KNNConstants.KNN_ENGINE; import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_EF_SEARCH; import static org.opensearch.knn.common.KNNConstants.MODEL_ID; +import static org.opensearch.knn.common.KNNConstants.PARAMETERS; import static org.opensearch.knn.common.KNNConstants.SPACE_TYPE; public class KNNWeightTests extends KNNTestCase { @@ -147,13 +150,27 @@ public void testQueryResultScoreFaiss() { testQueryScore( SpaceType.L2::scoreTranslation, SEGMENT_FILES_FAISS, - Map.of(SPACE_TYPE, SpaceType.L2.getValue(), KNN_ENGINE, KNNEngine.FAISS.getName()) + Map.of( + SPACE_TYPE, + SpaceType.L2.getValue(), + KNN_ENGINE, + KNNEngine.FAISS.getName(), + PARAMETERS, + String.format(Locale.ROOT, "{\"%s\":\"%s\"}", INDEX_DESCRIPTION_PARAMETER, "HNSW32") + ) ); // score translation for Faiss and inner product is different from default defined in Space enum testQueryScore( rawScore -> SpaceType.INNER_PRODUCT.scoreTranslation(-1 * rawScore), SEGMENT_FILES_FAISS, - Map.of(SPACE_TYPE, SpaceType.INNER_PRODUCT.getValue(), KNN_ENGINE, KNNEngine.FAISS.getName()) + Map.of( + SPACE_TYPE, + SpaceType.INNER_PRODUCT.getValue(), + KNN_ENGINE, + KNNEngine.FAISS.getName(), + PARAMETERS, + String.format(Locale.ROOT, "{\"%s\":\"%s\"}", INDEX_DESCRIPTION_PARAMETER, "HNSW32") + ) ); } @@ -421,7 +438,12 @@ public void testANNWithFilterQuery_whenDoingANN_thenSuccess() { when(directory.getDirectory()).thenReturn(path); final FieldInfos fieldInfos = mock(FieldInfos.class); final FieldInfo fieldInfo = mock(FieldInfo.class); - final Map attributesMap = ImmutableMap.of(KNN_ENGINE, KNNEngine.FAISS.getName()); + final Map attributesMap = ImmutableMap.of( + KNN_ENGINE, + KNNEngine.FAISS.getName(), + PARAMETERS, + String.format(Locale.ROOT, "{\"%s\":\"%s\"}", INDEX_DESCRIPTION_PARAMETER, "HNSW32") + ); when(reader.getFieldInfos()).thenReturn(fieldInfos); when(fieldInfos.fieldInfo(any())).thenReturn(fieldInfo); @@ -480,7 +502,14 @@ public void testANNWithFilterQuery_whenExactSearch_thenSuccess() { final float boost = (float) randomDoubleBetween(0, 10, true); final KNNWeight knnWeight = new KNNWeight(query, boost, filterQueryWeight); - final Map attributesMap = ImmutableMap.of(KNN_ENGINE, KNNEngine.FAISS.getName(), SPACE_TYPE, SpaceType.L2.name()); + final Map attributesMap = ImmutableMap.of( + KNN_ENGINE, + KNNEngine.FAISS.getName(), + SPACE_TYPE, + SpaceType.L2.name(), + PARAMETERS, + String.format(Locale.ROOT, "{\"%s\":\"%s\"}", INDEX_DESCRIPTION_PARAMETER, "HNSW32") + ); final FieldInfos fieldInfos = mock(FieldInfos.class); final FieldInfo fieldInfo = mock(FieldInfo.class); final BinaryDocValues binaryDocValues = mock(BinaryDocValues.class); @@ -531,7 +560,14 @@ public void testANNWithFilterQuery_whenExactSearchAndThresholdComputations_thenS final float boost = (float) randomDoubleBetween(0, 10, true); final KNNWeight knnWeight = new KNNWeight(query, boost, filterQueryWeight); - final Map attributesMap = ImmutableMap.of(KNN_ENGINE, KNNEngine.FAISS.getName(), SPACE_TYPE, SpaceType.L2.name()); + final Map attributesMap = ImmutableMap.of( + KNN_ENGINE, + KNNEngine.FAISS.getName(), + SPACE_TYPE, + SpaceType.L2.name(), + PARAMETERS, + String.format(Locale.ROOT, "{\"%s\":\"%s\"}", INDEX_DESCRIPTION_PARAMETER, "HNSW32") + ); final FieldInfos fieldInfos = mock(FieldInfos.class); final FieldInfo fieldInfo = mock(FieldInfo.class); final BinaryDocValues binaryDocValues = mock(BinaryDocValues.class); @@ -592,7 +628,14 @@ public void testANNWithFilterQuery_whenExactSearchViaThresholdSetting_thenSucces final float boost = (float) randomDoubleBetween(0, 10, true); final KNNWeight knnWeight = new KNNWeight(query, boost, filterQueryWeight); - final Map attributesMap = ImmutableMap.of(KNN_ENGINE, KNNEngine.FAISS.getName(), SPACE_TYPE, SpaceType.L2.name()); + final Map attributesMap = ImmutableMap.of( + KNN_ENGINE, + KNNEngine.FAISS.getName(), + SPACE_TYPE, + SpaceType.L2.name(), + PARAMETERS, + String.format(Locale.ROOT, "{\"%s\":\"%s\"}", INDEX_DESCRIPTION_PARAMETER, "HNSW32") + ); final FieldInfos fieldInfos = mock(FieldInfos.class); final FieldInfo fieldInfo = mock(FieldInfo.class); final BinaryDocValues binaryDocValues = mock(BinaryDocValues.class); @@ -818,7 +861,16 @@ public void testDoANNSearch_whenRadialIsDefined_thenCallJniRadiusQueryIndex() { final FieldInfo fieldInfo = mock(FieldInfo.class); when(reader.getFieldInfos()).thenReturn(fieldInfos); when(fieldInfos.fieldInfo(any())).thenReturn(fieldInfo); - when(fieldInfo.attributes()).thenReturn(Map.of(SPACE_TYPE, SpaceType.L2.getValue(), KNN_ENGINE, KNNEngine.FAISS.getName())); + when(fieldInfo.attributes()).thenReturn( + Map.of( + SPACE_TYPE, + SpaceType.L2.getValue(), + KNN_ENGINE, + KNNEngine.FAISS.getName(), + PARAMETERS, + String.format(Locale.ROOT, "{\"%s\":\"%s\"}", INDEX_DESCRIPTION_PARAMETER, "HNSW32") + ) + ); final KNNScorer knnScorer = (KNNScorer) knnWeight.scorer(leafReaderContext); assertNotNull(knnScorer); @@ -881,7 +933,14 @@ private SegmentReader getMockedSegmentReader() { when(reader.getSegmentInfo()).thenReturn(segmentCommitInfo); // Prepare fieldInfo - final Map attributesMap = ImmutableMap.of(KNN_ENGINE, KNNEngine.FAISS.getName(), SPACE_TYPE, SpaceType.L2.name()); + final Map attributesMap = ImmutableMap.of( + KNN_ENGINE, + KNNEngine.FAISS.getName(), + SPACE_TYPE, + SpaceType.L2.name(), + PARAMETERS, + String.format(Locale.ROOT, "{\"%s\":\"%s\"}", INDEX_DESCRIPTION_PARAMETER, "HNSW32") + ); final FieldInfo fieldInfo = mock(FieldInfo.class); when(fieldInfo.attributes()).thenReturn(attributesMap); when(fieldInfo.getAttribute(SPACE_TYPE)).thenReturn(SpaceType.L2.name()); diff --git a/src/test/java/org/opensearch/knn/index/util/FieldInfoExtractorTests.java b/src/test/java/org/opensearch/knn/index/util/FieldInfoExtractorTests.java new file mode 100644 index 000000000..a0facefbd --- /dev/null +++ b/src/test/java/org/opensearch/knn/index/util/FieldInfoExtractorTests.java @@ -0,0 +1,42 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.util; + +import junit.framework.TestCase; +import lombok.SneakyThrows; +import org.apache.lucene.index.FieldInfo; +import org.mockito.Mockito; +import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.knn.common.KNNConstants; + +import java.util.Collections; +import java.util.Map; + +import static org.mockito.Mockito.mock; +import static org.opensearch.knn.common.KNNConstants.INDEX_DESCRIPTION_PARAMETER; + +public class FieldInfoExtractorTests extends TestCase { + @SneakyThrows + public void testGetIndexDescription_whenNoDescription_thenReturnNull() { + FieldInfo fieldInfo = mock(FieldInfo.class); + Mockito.when(fieldInfo.attributes()).thenReturn(Collections.emptyMap(), Map.of(KNNConstants.PARAMETERS, "{}")); + assertNull(FieldInfoExtractor.getIndexDescription(fieldInfo)); + assertNull(FieldInfoExtractor.getIndexDescription(fieldInfo)); + } + + @SneakyThrows + public void testGetIndexDescription_whenDescriptionExist_thenReturnIndexDescription() { + String indexDescription = "HNSW"; + XContentBuilder parameters = XContentFactory.jsonBuilder() + .startObject() + .field(INDEX_DESCRIPTION_PARAMETER, indexDescription) + .endObject(); + FieldInfo fieldInfo = mock(FieldInfo.class); + Mockito.when(fieldInfo.attributes()).thenReturn(Map.of(KNNConstants.PARAMETERS, parameters.toString())); + assertEquals(indexDescription, FieldInfoExtractor.getIndexDescription(fieldInfo)); + } +} diff --git a/src/test/java/org/opensearch/knn/jni/JNIServiceTests.java b/src/test/java/org/opensearch/knn/jni/JNIServiceTests.java index e71930d48..e17ee5077 100644 --- a/src/test/java/org/opensearch/knn/jni/JNIServiceTests.java +++ b/src/test/java/org/opensearch/knn/jni/JNIServiceTests.java @@ -62,6 +62,7 @@ public class JNIServiceTests extends KNNTestCase { static TestUtils.TestData testData; static TestUtils.TestData testDataNested; private String faissMethod = "HNSW32,Flat"; + private String faissBinaryMethod = "BHNSW32"; @BeforeClass public static void setUpClass() throws IOException { @@ -657,6 +658,21 @@ public void testCreateIndex_faiss_valid() throws IOException { } } + @SneakyThrows + public void testCreateIndex_binary_faiss_valid() { + Path tmpFile1 = createTempFile(); + long memoryAddr = testData.loadBinaryDataToMemoryAddress(); + JNIService.createIndex( + testData.indexData.docs, + memoryAddr, + testData.indexData.getDimension(), + tmpFile1.toAbsolutePath().toString(), + ImmutableMap.of(INDEX_DESCRIPTION_PARAMETER, faissBinaryMethod, KNNConstants.SPACE_TYPE, SpaceType.HAMMING_BIT.getValue()), + KNNEngine.FAISS + ); + assertTrue(tmpFile1.toFile().length() > 0); + } + public void testLoadIndex_invalidEngine() { expectThrows(IllegalArgumentException.class, () -> JNIService.loadIndex("test", Collections.emptyMap(), KNNEngine.LUCENE)); } @@ -943,6 +959,37 @@ public void testQueryIndex_faiss_parentIds() throws IOException { } } + @SneakyThrows + public void testQueryBinaryIndex_faiss_valid() { + int k = 10; + List methods = ImmutableList.of(faissBinaryMethod); + for (String method : methods) { + Path tmpFile = createTempFile(); + long memoryAddr = testData.loadBinaryDataToMemoryAddress(); + JNIService.createIndex( + testData.indexData.docs, + memoryAddr, + testData.indexData.getDimension(), + tmpFile.toAbsolutePath().toString(), + ImmutableMap.of(INDEX_DESCRIPTION_PARAMETER, method, KNNConstants.SPACE_TYPE, SpaceType.HAMMING_BIT.getValue()), + KNNEngine.FAISS + ); + assertTrue(tmpFile.toFile().length() > 0); + + long pointer = JNIService.loadIndex( + tmpFile.toAbsolutePath().toString(), + ImmutableMap.of(INDEX_DESCRIPTION_PARAMETER, method), + KNNEngine.FAISS + ); + assertNotEquals(0, pointer); + + for (byte[] query : testData.binaryQueries) { + KNNQueryResult[] results = JNIService.queryBinaryIndex(pointer, query, k, null, KNNEngine.FAISS, null, 0, null); + assertEquals(k, results.length); + } + } + } + private Set toParentIdSet(KNNQueryResult[] results, Map idToParentIdMap) { return Arrays.stream(results).map(result -> idToParentIdMap.get(result.getId())).collect(Collectors.toSet()); } diff --git a/src/testFixtures/java/org/opensearch/knn/TestUtils.java b/src/testFixtures/java/org/opensearch/knn/TestUtils.java index 37e35f062..d41bbc0fd 100644 --- a/src/testFixtures/java/org/opensearch/knn/TestUtils.java +++ b/src/testFixtures/java/org/opensearch/knn/TestUtils.java @@ -19,6 +19,8 @@ import org.opensearch.knn.index.codec.util.SerializationMode; import org.opensearch.knn.jni.JNICommons; import org.opensearch.knn.plugin.script.KNNScoringUtil; + +import java.util.Collections; import java.util.Comparator; import java.util.Random; import java.util.Set; @@ -250,11 +252,14 @@ public static PriorityQueue insertWithOverflow(PriorityQueue flattenedVectors = new ArrayList<>(indexData.vectors.length * indexData.vectors[0].length); + for (int i = 0; i < indexData.vectors.length; i++) { + for (int j = 0; j < indexData.vectors[i].length; j++) { + flattenedVectors.add(indexData.vectors[i][j]); + } + } + Collections.sort(flattenedVectors); + Float median = flattenedVectors.get(flattenedVectors.size() / 2); + + // Quantize(indexData.vectors[i][j] >= median ? 1 : 0) and + // packing(8 bits to 1 byte) for index data + indexBinaryData = new byte[indexData.vectors.length][(indexData.vectors[0].length + 7) / 8]; + for (int i = 0; i < indexData.vectors.length; i++) { + for (int j = 0; j < indexData.vectors[i].length; j++) { + int byteIndex = j / 8; + int bitIndex = 7 - (j % 8); + indexBinaryData[i][byteIndex] |= (indexData.vectors[i][j] >= median ? 1 : 0) << bitIndex; + } + } + + // Quantize(queries[i][j] >= median ? 1 : 0) and + // packing(8 bits to 1 byte) for query data + binaryQueries = new byte[queries.length][(queries[0].length + 7) / 8]; + for (int i = 0; i < queries.length; i++) { + for (int j = 0; j < queries[i].length; j++) { + int byteIndex = j / 8; + int bitIndex = 7 - (j % 8); + binaryQueries[i][byteIndex] |= (queries[i][j] >= median ? 1 : 0) << bitIndex; + } + } + } + public long loadDataToMemoryAddress() { return JNICommons.storeVectorData(0, indexData.vectors, (long) indexData.vectors.length * indexData.vectors[0].length); } + public long loadBinaryDataToMemoryAddress() { + return JNICommons.storeByteVectorData(0, indexBinaryData, (long) indexBinaryData.length * indexBinaryData[0].length); + } + @AllArgsConstructor public static class Pair { public int[] docs; From af8f04d7baa1412ce48b8666de331d09c7b1f604 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Thu, 11 Jul 2024 19:47:03 -0700 Subject: [PATCH 286/416] Release memory properly for an array type (#1820) (#1821) Signed-off-by: Heemin Kim (cherry picked from commit 31a4d3eaf14877071e127048f6a8610063f263e8) Co-authored-by: Heemin Kim --- CHANGELOG.md | 1 + .../faiss/0001-Custom-patch-to-support-multi-vector.patch | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 61df8c6c0..d4af42608 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Enhancements ### Bug Fixes * Fixing the arithmetic to find the number of vectors to stream from java to jni layer.[#1804](https://github.com/opensearch-project/k-NN/pull/1804) +* Release memory properly for an array type [#1820](https://github.com/opensearch-project/k-NN/pull/1820) ### Infrastructure ### Documentation ### Maintenance diff --git a/jni/patches/faiss/0001-Custom-patch-to-support-multi-vector.patch b/jni/patches/faiss/0001-Custom-patch-to-support-multi-vector.patch index 227630c63..aa34795fb 100644 --- a/jni/patches/faiss/0001-Custom-patch-to-support-multi-vector.patch +++ b/jni/patches/faiss/0001-Custom-patch-to-support-multi-vector.patch @@ -451,7 +451,7 @@ index 270de8dc..3199634f 100644 + /// series of results for query i is done + void end() { + heap_reorder(k, heap_dis, heap_ids); -+ delete heap_group_ids; ++ delete[] heap_group_ids; + } + }; + @@ -528,8 +528,8 @@ index 270de8dc..3199634f 100644 + for (size_t i = i0; i < i1; i++) { + heap_reorder(k, heap_dis_tab + i * k, heap_ids_tab + i * k); + } -+ delete group_id_to_index_in_heap_tab; -+ delete heap_group_ids_tab; ++ delete[] group_id_to_index_in_heap_tab; ++ delete[] heap_group_ids_tab; + } +}; + From c5da4ece95df7107dd8e16475359dfc69a49c474 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Tue, 16 Jul 2024 09:10:30 -0700 Subject: [PATCH 287/416] [BUG] FIX Same Suffix Cause Recall Drop to zero (#1802) (#1832) FIX Same Suffix BUG Signed-off-by: luyuncheng (cherry picked from commit 48478b4888df80e28a710bd96599a07a4d33006d) Co-authored-by: luyuncheng --- CHANGELOG.md | 1 + .../opensearch/knn/index/query/KNNWeight.java | 35 ++++++++++++------- .../knn/index/query/KNNWeightTests.java | 25 +++++++++++++ 3 files changed, 48 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d4af42608..b675d032a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Bug Fixes * Fixing the arithmetic to find the number of vectors to stream from java to jni layer.[#1804](https://github.com/opensearch-project/k-NN/pull/1804) * Release memory properly for an array type [#1820](https://github.com/opensearch-project/k-NN/pull/1820) +* FIX Same Suffix Cause Recall Drop to zero [#1802](https://github.com/opensearch-project/k-NN/pull/1802) ### Infrastructure ### Documentation ### Maintenance diff --git a/src/main/java/org/opensearch/knn/index/query/KNNWeight.java b/src/main/java/org/opensearch/knn/index/query/KNNWeight.java index fce8e8e04..09fca0a52 100644 --- a/src/main/java/org/opensearch/knn/index/query/KNNWeight.java +++ b/src/main/java/org/opensearch/knn/index/query/KNNWeight.java @@ -5,6 +5,7 @@ package org.opensearch.knn.index.query; +import com.google.common.annotations.VisibleForTesting; import lombok.extern.log4j.Log4j2; import org.apache.commons.lang.StringUtils; import org.apache.lucene.index.BinaryDocValues; @@ -49,6 +50,7 @@ import java.nio.file.Path; import java.util.Arrays; import java.util.Collections; +import java.util.Comparator; import java.util.HashMap; import java.util.List; import java.util.Locale; @@ -228,19 +230,7 @@ private Map doANNSearch(final LeafReaderContext context, final B spaceType = SpaceType.getSpace(spaceTypeName); } - /* - * In case of compound file, extension would be + c otherwise - */ - String engineExtension = reader.getSegmentInfo().info.getUseCompoundFile() - ? knnEngine.getExtension() + KNNConstants.COMPOUND_EXTENSION - : knnEngine.getExtension(); - String engineSuffix = knnQuery.getField() + engineExtension; - List engineFiles = reader.getSegmentInfo() - .files() - .stream() - .filter(fileName -> fileName.endsWith(engineSuffix)) - .collect(Collectors.toList()); - + List engineFiles = getEngineFiles(reader, knnEngine.getExtension()); if (engineFiles.isEmpty()) { log.debug("[KNN] No engine index found for field {} for segment {}", knnQuery.getField(), reader.getSegmentName()); return null; @@ -330,6 +320,25 @@ private Map doANNSearch(final LeafReaderContext context, final B .collect(Collectors.toMap(KNNQueryResult::getId, result -> knnEngine.score(result.getScore(), spaceType))); } + @VisibleForTesting + List getEngineFiles(SegmentReader reader, String extension) throws IOException { + /* + * In case of compound file, extension would be + c otherwise + */ + String engineExtension = reader.getSegmentInfo().info.getUseCompoundFile() + ? extension + KNNConstants.COMPOUND_EXTENSION + : extension; + String engineSuffix = knnQuery.getField() + engineExtension; + String underLineEngineSuffix = "_" + engineSuffix; + List engineFiles = reader.getSegmentInfo() + .files() + .stream() + .filter(fileName -> fileName.endsWith(underLineEngineSuffix)) + .sorted(Comparator.comparingInt(String::length)) + .collect(Collectors.toList()); + return engineFiles; + } + private Map doExactSearch(final LeafReaderContext leafReaderContext, final BitSet filterIdsBitSet, int cardinality) { try { // Creating min heap and init with MAX DocID and Score as -INF. diff --git a/src/test/java/org/opensearch/knn/index/query/KNNWeightTests.java b/src/test/java/org/opensearch/knn/index/query/KNNWeightTests.java index e73e86e90..ade564d94 100644 --- a/src/test/java/org/opensearch/knn/index/query/KNNWeightTests.java +++ b/src/test/java/org/opensearch/knn/index/query/KNNWeightTests.java @@ -86,6 +86,11 @@ public class KNNWeightTests extends KNNTestCase { private static final int K = 5; private static final Set SEGMENT_FILES_NMSLIB = Set.of("_0.cfe", "_0_2011_target_field.hnswc"); private static final Set SEGMENT_FILES_FAISS = Set.of("_0.cfe", "_0_2011_target_field.faissc"); + private static final Set SEGMENT_MULTI_FIELD_FILES_FAISS = Set.of( + "_0.cfe", + "_0_2011_target_field.faissc", + "_0_2011_long_target_field.faissc" + ); private static final String CIRCUIT_BREAKER_LIMIT_100KB = "100Kb"; private static final Integer EF_SEARCH = 10; private static final Map HNSW_METHOD_PARAMETERS = Map.of(METHOD_PARAMETER_EF_SEARCH, EF_SEARCH); @@ -172,6 +177,20 @@ public void testQueryResultScoreFaiss() { String.format(Locale.ROOT, "{\"%s\":\"%s\"}", INDEX_DESCRIPTION_PARAMETER, "HNSW32") ) ); + + // multi field + testQueryScore( + rawScore -> SpaceType.INNER_PRODUCT.scoreTranslation(-1 * rawScore), + SEGMENT_MULTI_FIELD_FILES_FAISS, + Map.of( + SPACE_TYPE, + SpaceType.INNER_PRODUCT.getValue(), + KNN_ENGINE, + KNNEngine.FAISS.getName(), + PARAMETERS, + String.format(Locale.ROOT, "{\"%s\":\"%s\"}", INDEX_DESCRIPTION_PARAMETER, "HNSW32") + ) + ); } @SneakyThrows @@ -1005,6 +1024,12 @@ private void testQueryScore( when(fieldInfos.fieldInfo(any())).thenReturn(fieldInfo); when(fieldInfo.attributes()).thenReturn(fileAttributes); + String engineName = fieldInfo.attributes().getOrDefault(KNN_ENGINE, KNNEngine.NMSLIB.getName()); + KNNEngine knnEngine = KNNEngine.getEngine(engineName); + List engineFiles = knnWeight.getEngineFiles(reader, knnEngine.getExtension()); + String expectIndexPath = String.format("%s_%s_%s%s%s", SEGMENT_NAME, 2011, FIELD_NAME, knnEngine.getExtension(), "c"); + assertEquals(engineFiles.get(0), expectIndexPath); + final KNNScorer knnScorer = (KNNScorer) knnWeight.scorer(leafReaderContext); assertNotNull(knnScorer); final DocIdSetIterator docIdSetIterator = knnScorer.iterator(); From 73ede544ddc25c51eff05e87220ffb5259714cf9 Mon Sep 17 00:00:00 2001 From: Heemin Kim Date: Tue, 16 Jul 2024 12:19:48 -0700 Subject: [PATCH 288/416] [feature to main branch] Add binary format support with HNSW method in Faiss Engine (#1829) (#1834) * Add faiss custom patch to support search parameter in binary index (#1815) Signed-off-by: Heemin Kim * Add binary format support with HNSW method in Faiss Engine (#1781) Signed-off-by: Heemin Kim --------- Signed-off-by: Heemin Kim (cherry picked from commit fe1d86fcbdb57d42c4e76c1e75312f914c2f3ba6) --- CHANGELOG.md | 1 + jni/cmake/init-faiss.cmake | 3 +- jni/include/faiss_wrapper.h | 2 +- .../org_opensearch_knn_jni_FaissService.h | 4 +- ...ustom-patch-to-support-binary-vector.patch | 294 +++++++++++++++++ jni/src/faiss_wrapper.cpp | 13 +- .../org_opensearch_knn_jni_FaissService.cpp | 4 +- jni/tests/faiss_wrapper_test.cpp | 16 +- .../knn/common/KNNValidationUtil.java | 33 +- .../org/opensearch/knn/index/IndexUtil.java | 23 +- .../opensearch/knn/index/KNNIndexShard.java | 12 +- .../knn/index/KNNMethodContext.java | 19 +- .../index/KNNVectorSimilarityFunction.java | 53 +++ .../org/opensearch/knn/index/SpaceType.java | 67 +++- .../opensearch/knn/index/VectorDataType.java | 18 +- .../KNN80Codec/KNN80DocValuesConsumer.java | 26 +- .../index/codec/transfer/VectorTransfer.java | 55 +++ .../codec/transfer/VectorTransferByte.java | 66 ++++ .../codec/transfer/VectorTransferFloat.java | 69 ++++ .../knn/index/codec/util/KNNCodecUtil.java | 69 ++-- .../util/KNNVectorSerializerFactory.java | 4 +- .../index/codec/util/SerializationMode.java | 3 +- .../index/mapper/KNNVectorFieldMapper.java | 134 ++++++-- .../mapper/KNNVectorFieldMapperUtil.java | 73 +++- .../knn/index/mapper/LuceneFieldMapper.java | 4 +- .../knn/index/mapper/MethodFieldMapper.java | 2 + .../index/memory/NativeMemoryAllocation.java | 41 +-- .../memory/NativeMemoryLoadStrategy.java | 3 +- .../opensearch/knn/index/query/KNNQuery.java | 44 ++- .../knn/index/query/KNNQueryBuilder.java | 20 +- .../knn/index/query/KNNQueryFactory.java | 40 ++- .../opensearch/knn/index/query/KNNWeight.java | 76 +++-- .../filtered/FilteredIdsKNNByteIterator.java | 79 +++++ .../filtered/FilteredIdsKNNIterator.java | 6 +- .../knn/index/query/filtered/KNNIterator.java | 14 + .../NestedFilteredIdsKNNByteIterator.java | 61 ++++ .../org/opensearch/knn/index/util/Faiss.java | 7 +- .../knn/index/util/FieldInfoExtractor.java | 37 --- .../org/opensearch/knn/index/util/Lucene.java | 2 +- .../org/opensearch/knn/index/util/Nmslib.java | 2 +- .../org/opensearch/knn/jni/FaissService.java | 21 +- .../org/opensearch/knn/jni/JNIService.java | 25 +- .../plugin/rest/RestTrainModelHandler.java | 4 + .../plugin/script/KNNScoringSpaceUtil.java | 2 +- .../knn/plugin/script/KNNScoringUtil.java | 13 +- .../knn/common/KNNValidationUtilTests.java | 41 +++ .../opensearch/knn/index/IndexUtilTests.java | 25 +- .../knn/index/KNNIndexShardTests.java | 4 +- .../knn/index/KNNMethodContextTests.java | 2 +- .../KNNVectorSimilarityFunctionTests.java | 57 ++++ .../opensearch/knn/index/LuceneEngineIT.java | 19 +- .../opensearch/knn/index/SpaceTypeTests.java | 45 ++- .../knn/index/VectorDataTypeIT.java | 5 +- .../knn/index/VectorDataTypeTests.java | 15 + .../KNN80DocValuesConsumerTests.java | 69 +++- .../knn/index/codec/KNNCodecTestUtil.java | 35 ++ .../transfer/VectorTransferByteTests.java | 56 ++++ .../transfer/VectorTransferFloatTests.java | 66 ++++ .../index/codec/util/KNNCodecUtilTests.java | 56 ++++ .../mapper/KNNVectorFieldMapperTests.java | 141 +++++++- .../mapper/KNNVectorFieldMapperUtilTests.java | 52 +++ .../index/mapper/MethodFieldMapperTests.java | 37 +++ .../memory/NativeMemoryAllocationTests.java | 65 ++++ .../memory/NativeMemoryLoadStrategyTests.java | 60 ++++ .../knn/index/query/KNNQueryBuilderTests.java | 35 ++ .../knn/index/query/KNNQueryFactoryTests.java | 29 ++ .../knn/index/query/KNNWeightTests.java | 312 ++++++++++++++---- .../FilteredIdsKNNByteIteratorTests.java | 52 +++ .../filtered/FilteredIdsKNNIteratorTests.java | 2 +- ...NestedFilteredIdsKNNByteIteratorTests.java | 63 ++++ .../NestedFilteredIdsKNNIteratorTests.java | 2 +- .../index/util/FieldInfoExtractorTests.java | 42 --- .../opensearch/knn/jni/JNIServiceTests.java | 34 +- .../plugin/script/KNNScoringUtilTests.java | 6 + .../knn/plugin/script/KNNScriptScoringIT.java | 9 +- 75 files changed, 2569 insertions(+), 431 deletions(-) create mode 100644 jni/patches/faiss/0004-Custom-patch-to-support-binary-vector.patch create mode 100644 src/main/java/org/opensearch/knn/index/KNNVectorSimilarityFunction.java create mode 100644 src/main/java/org/opensearch/knn/index/codec/transfer/VectorTransfer.java create mode 100644 src/main/java/org/opensearch/knn/index/codec/transfer/VectorTransferByte.java create mode 100644 src/main/java/org/opensearch/knn/index/codec/transfer/VectorTransferFloat.java create mode 100644 src/main/java/org/opensearch/knn/index/query/filtered/FilteredIdsKNNByteIterator.java create mode 100644 src/main/java/org/opensearch/knn/index/query/filtered/KNNIterator.java create mode 100644 src/main/java/org/opensearch/knn/index/query/filtered/NestedFilteredIdsKNNByteIterator.java delete mode 100644 src/main/java/org/opensearch/knn/index/util/FieldInfoExtractor.java create mode 100644 src/test/java/org/opensearch/knn/common/KNNValidationUtilTests.java create mode 100644 src/test/java/org/opensearch/knn/index/KNNVectorSimilarityFunctionTests.java create mode 100644 src/test/java/org/opensearch/knn/index/codec/transfer/VectorTransferByteTests.java create mode 100644 src/test/java/org/opensearch/knn/index/codec/transfer/VectorTransferFloatTests.java create mode 100644 src/test/java/org/opensearch/knn/index/codec/util/KNNCodecUtilTests.java create mode 100644 src/test/java/org/opensearch/knn/index/mapper/MethodFieldMapperTests.java create mode 100644 src/test/java/org/opensearch/knn/index/query/filtered/FilteredIdsKNNByteIteratorTests.java create mode 100644 src/test/java/org/opensearch/knn/index/query/filtered/NestedFilteredIdsKNNByteIteratorTests.java delete mode 100644 src/test/java/org/opensearch/knn/index/util/FieldInfoExtractorTests.java diff --git a/CHANGELOG.md b/CHANGELOG.md index b675d032a..96b833811 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Features * Adds dynamic query parameter ef_search [#1783](https://github.com/opensearch-project/k-NN/pull/1783) * Adds dynamic query parameter ef_search in radial search faiss engine [#1790](https://github.com/opensearch-project/k-NN/pull/1790) +* Add binary format support with HNSW method in Faiss Engine [#1781](https://github.com/opensearch-project/k-NN/pull/1781) ### Enhancements ### Bug Fixes * Fixing the arithmetic to find the number of vectors to stream from java to jni layer.[#1804](https://github.com/opensearch-project/k-NN/pull/1804) diff --git a/jni/cmake/init-faiss.cmake b/jni/cmake/init-faiss.cmake index bef93eda0..befed4703 100644 --- a/jni/cmake/init-faiss.cmake +++ b/jni/cmake/init-faiss.cmake @@ -13,7 +13,7 @@ if (NOT EXISTS ${FAISS_REPO_DIR}) endif () # Check if patch exist, this is to skip git apply during CI build. See CI.yml with ubuntu. -find_path(PATCH_FILE NAMES 0001-Custom-patch-to-support-multi-vector.patch 0002-Enable-precomp-table-to-be-shared-ivfpq.patch 0003-Custom-patch-to-support-range-search-params.patch PATHS ${CMAKE_CURRENT_SOURCE_DIR}/patches/faiss NO_DEFAULT_PATH) +find_path(PATCH_FILE NAMES 0001-Custom-patch-to-support-multi-vector.patch 0002-Enable-precomp-table-to-be-shared-ivfpq.patch 0003-Custom-patch-to-support-range-search-params.patch 0004-Custom-patch-to-support-binary-vector.patch PATHS ${CMAKE_CURRENT_SOURCE_DIR}/patches/faiss NO_DEFAULT_PATH) # If it exists, apply patches if (EXISTS ${PATCH_FILE}) @@ -21,6 +21,7 @@ if (EXISTS ${PATCH_FILE}) execute_process(COMMAND git ${GIT_PATCH_COMMAND} --3way --ignore-space-change --ignore-whitespace ${CMAKE_CURRENT_SOURCE_DIR}/patches/faiss/0001-Custom-patch-to-support-multi-vector.patch WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/external/faiss ERROR_VARIABLE ERROR_MSG RESULT_VARIABLE RESULT_CODE) execute_process(COMMAND git ${GIT_PATCH_COMMAND} --3way --ignore-space-change --ignore-whitespace ${CMAKE_CURRENT_SOURCE_DIR}/patches/faiss/0002-Enable-precomp-table-to-be-shared-ivfpq.patch WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/external/faiss ERROR_VARIABLE ERROR_MSG RESULT_VARIABLE RESULT_CODE) execute_process(COMMAND git ${GIT_PATCH_COMMAND} --3way --ignore-space-change --ignore-whitespace ${CMAKE_CURRENT_SOURCE_DIR}/patches/faiss/0003-Custom-patch-to-support-range-search-params.patch WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/external/faiss ERROR_VARIABLE ERROR_MSG RESULT_VARIABLE RESULT_CODE) + execute_process(COMMAND git ${GIT_PATCH_COMMAND} --3way --ignore-space-change --ignore-whitespace ${CMAKE_CURRENT_SOURCE_DIR}/patches/faiss/0004-Custom-patch-to-support-binary-vector.patch WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/external/faiss ERROR_VARIABLE ERROR_MSG RESULT_VARIABLE RESULT_CODE) if(RESULT_CODE) message(FATAL_ERROR "Failed to apply patch:\n${ERROR_MSG}") endif() diff --git a/jni/include/faiss_wrapper.h b/jni/include/faiss_wrapper.h index 5ac17cfd1..2b9bc2c76 100644 --- a/jni/include/faiss_wrapper.h +++ b/jni/include/faiss_wrapper.h @@ -81,7 +81,7 @@ namespace knn_jni { jbyteArray queryVectorJ, jint kJ, jobject methodParamsJ, jlongArray filterIdsJ, jint filterIdsTypeJ, jintArray parentIdsJ); // Free the index located in memory at indexPointerJ - void Free(jlong indexPointer); + void Free(jlong indexPointer, jboolean isBinaryIndexJ); // Free shared index state in memory at shareIndexStatePointerJ void FreeSharedIndexState(jlong shareIndexStatePointerJ); diff --git a/jni/include/org_opensearch_knn_jni_FaissService.h b/jni/include/org_opensearch_knn_jni_FaissService.h index 3d6aef45c..7cc071ff3 100644 --- a/jni/include/org_opensearch_knn_jni_FaissService.h +++ b/jni/include/org_opensearch_knn_jni_FaissService.h @@ -110,10 +110,10 @@ JNIEXPORT jobjectArray JNICALL Java_org_opensearch_knn_jni_FaissService_queryBin /* * Class: org_opensearch_knn_jni_FaissService * Method: free - * Signature: (J)V + * Signature: (JZ)V */ JNIEXPORT void JNICALL Java_org_opensearch_knn_jni_FaissService_free - (JNIEnv *, jclass, jlong); + (JNIEnv *, jclass, jlong, jboolean); /* * Class: org_opensearch_knn_jni_FaissService diff --git a/jni/patches/faiss/0004-Custom-patch-to-support-binary-vector.patch b/jni/patches/faiss/0004-Custom-patch-to-support-binary-vector.patch new file mode 100644 index 000000000..e2bac8f1a --- /dev/null +++ b/jni/patches/faiss/0004-Custom-patch-to-support-binary-vector.patch @@ -0,0 +1,294 @@ +From 4d374aa47d4415cbda04b299788988f4ff6e5da0 Mon Sep 17 00:00:00 2001 +From: Heemin Kim +Date: Wed, 10 Jul 2024 16:06:36 -0700 +Subject: [PATCH] 0004-Custom-patch-to-support-binary-vector + +Signed-off-by: Heemin Kim +--- + faiss/IndexBinaryHNSW.cpp | 59 +++++++++++++------ + faiss/IndexBinaryIVF.cpp | 28 +++++++-- + tests/test_id_grouper.cpp | 117 +++++++++++++++++++++++++++++++++++++- + 3 files changed, 179 insertions(+), 25 deletions(-) + +diff --git a/faiss/IndexBinaryHNSW.cpp b/faiss/IndexBinaryHNSW.cpp +index f1bda08f..32627cb0 100644 +--- a/faiss/IndexBinaryHNSW.cpp ++++ b/faiss/IndexBinaryHNSW.cpp +@@ -189,37 +189,62 @@ void IndexBinaryHNSW::train(idx_t n, const uint8_t* x) { + is_trained = true; + } + ++namespace { ++template ++void hnsw_search( ++ const IndexBinaryHNSW* index, ++ idx_t n, ++ const uint8_t* x, ++ BlockResultHandler& bres, ++ const SearchParameters* params_in) { ++ const SearchParametersHNSW* params = nullptr; ++ const HNSW& hnsw = index->hnsw; ++ ++ if (params_in) { ++ params = dynamic_cast(params_in); ++ FAISS_THROW_IF_NOT_MSG(params, "params type invalid"); ++ } ++#pragma omp parallel ++ { ++ VisitedTable vt(index->ntotal); ++ std::unique_ptr dis(index->get_distance_computer()); ++ typename BlockResultHandler::SingleResultHandler res(bres); ++ ++#pragma omp for ++ for (idx_t i = 0; i < n; i++) { ++ res.begin(i); ++ dis->set_query((float*)(x + i * index->code_size)); ++ hnsw.search(*dis, res, vt, params); ++ res.end(); ++ } ++ } ++} ++ ++} // anonymous namespace ++ + void IndexBinaryHNSW::search( + idx_t n, + const uint8_t* x, + idx_t k, + int32_t* distances, + idx_t* labels, +- const SearchParameters* params) const { +- FAISS_THROW_IF_NOT_MSG( +- !params, "search params not supported for this index"); ++ const SearchParameters* params_in) const { + FAISS_THROW_IF_NOT(k > 0); + + // we use the buffer for distances as float but convert them back + // to int in the end + float* distances_f = (float*)distances; + +- using RH = HeapBlockResultHandler; +- RH bres(n, distances_f, labels, k); ++ if (params_in && params_in->grp) { ++ using RH = GroupedHeapBlockResultHandler; ++ RH bres(n, distances_f, labels, k, params_in->grp); + +-#pragma omp parallel +- { +- VisitedTable vt(ntotal); +- std::unique_ptr dis(get_distance_computer()); +- RH::SingleResultHandler res(bres); ++ hnsw_search(this, n, x, bres, params_in); ++ } else { ++ using RH = HeapBlockResultHandler; ++ RH bres(n, distances_f, labels, k); + +-#pragma omp for +- for (idx_t i = 0; i < n; i++) { +- res.begin(i); +- dis->set_query((float*)(x + i * code_size)); +- hnsw.search(*dis, res, vt); +- res.end(); +- } ++ hnsw_search(this, n, x, bres, params_in); + } + + #pragma omp parallel for +diff --git a/faiss/IndexBinaryIVF.cpp b/faiss/IndexBinaryIVF.cpp +index ab1b9fd8..de996df3 100644 +--- a/faiss/IndexBinaryIVF.cpp ++++ b/faiss/IndexBinaryIVF.cpp +@@ -113,25 +113,41 @@ void IndexBinaryIVF::search( + idx_t k, + int32_t* distances, + idx_t* labels, +- const SearchParameters* params) const { +- FAISS_THROW_IF_NOT_MSG( +- !params, "search params not supported for this index"); ++ const SearchParameters* params_in) const { + FAISS_THROW_IF_NOT(k > 0); ++ const IVFSearchParameters* params = nullptr; ++ if (params_in) { ++ params = dynamic_cast(params_in); ++ FAISS_THROW_IF_NOT_MSG(params, "IndexIVF params have incorrect type"); ++ } ++ const size_t nprobe_2 = std::min(nlist, params ? params->nprobe : this->nprobe); + FAISS_THROW_IF_NOT(nprobe > 0); + +- const size_t nprobe_2 = std::min(nlist, this->nprobe); + std::unique_ptr idx(new idx_t[n * nprobe_2]); + std::unique_ptr coarse_dis(new int32_t[n * nprobe_2]); + + double t0 = getmillisecs(); +- quantizer->search(n, x, nprobe_2, coarse_dis.get(), idx.get()); ++ quantizer->search( ++ n, ++ x, ++ nprobe_2, ++ coarse_dis.get(), ++ idx.get(), ++ params ? params->quantizer_params : nullptr); + indexIVF_stats.quantization_time += getmillisecs() - t0; + + t0 = getmillisecs(); + invlists->prefetch_lists(idx.get(), n * nprobe_2); + + search_preassigned( +- n, x, k, idx.get(), coarse_dis.get(), distances, labels, false); ++ n, ++ x, ++ k, ++ idx.get(), ++ coarse_dis.get(), ++ distances, labels, ++ false, ++ params); + indexIVF_stats.search_time += getmillisecs() - t0; + } + +diff --git a/tests/test_id_grouper.cpp b/tests/test_id_grouper.cpp +index 6601795b..bd8ab5f9 100644 +--- a/tests/test_id_grouper.cpp ++++ b/tests/test_id_grouper.cpp +@@ -14,10 +14,10 @@ + #include + #include + #include ++#include "faiss/IndexBinaryHNSW.h" + + // 64-bit int + using idx_t = faiss::idx_t; +- + using namespace faiss; + + TEST(IdGrouper, get_group) { +@@ -172,7 +172,58 @@ TEST(IdGrouper, bitmap_with_hnsw) { + delete[] xb; + } + +-TEST(IdGrouper, bitmap_with_hnswn_idmap) { ++TEST(IdGrouper, bitmap_with_binary_hnsw) { ++ int d = 16; // dimension ++ int nb = 10; // database size ++ ++ std::vector database(nb * (d / 8)); ++ for (size_t i = 0; i < nb * (d / 8); i++) { ++ database[i] = rand() % 0x100; ++ } ++ ++ uint64_t bitmap[1] = {}; ++ faiss::IDGrouperBitmap id_grouper(1, bitmap); ++ for (int i = 0; i < nb; i++) { ++ if (i % 2 == 1) { ++ id_grouper.set_group(i); ++ } ++ } ++ ++ int k = 10; ++ int m = 8; ++ faiss::IndexBinary* index = ++ new faiss::IndexBinaryHNSW(d, m); ++ index->add(nb, database.data()); // add vectors to the index ++ ++ // search ++ idx_t* I = new idx_t[k]; ++ int32_t* D = new int32_t[k]; ++ ++ auto pSearchParameters = new faiss::SearchParametersHNSW(); ++ pSearchParameters->grp = &id_grouper; ++ ++ index->search(1, database.data(), k, D, I, pSearchParameters); ++ ++ std::unordered_set group_ids; ++ ASSERT_EQ(0, I[0]); ++ ASSERT_EQ(0, D[0]); ++ group_ids.insert(id_grouper.get_group(I[0])); ++ for (int j = 1; j < 5; j++) { ++ ASSERT_NE(-1, I[j]); ++ ASSERT_NE(std::numeric_limits::max(), D[j]); ++ group_ids.insert(id_grouper.get_group(I[j])); ++ } ++ for (int j = 5; j < k; j++) { ++ ASSERT_EQ(-1, I[j]); ++ ASSERT_EQ(std::numeric_limits::max(), D[j]); ++ } ++ ASSERT_EQ(5, group_ids.size()); ++ ++ delete[] I; ++ delete[] D; ++} ++ ++TEST(IdGrouper, bitmap_with_hnsw_idmap) { + int d = 1; // dimension + int nb = 10; // database size + +@@ -239,3 +290,65 @@ TEST(IdGrouper, bitmap_with_hnswn_idmap) { + delete[] D; + delete[] xb; + } ++ ++TEST(IdGrouper, bitmap_with_binary_hnsw_idmap) { ++ int d = 16; // dimension ++ int nb = 10; // database size ++ ++ std::vector database(nb * (d / 8)); ++ for (size_t i = 0; i < nb * (d / 8); i++) { ++ database[i] = rand() % 0x100; ++ } ++ ++ idx_t* xids = new idx_t[nb]; ++ uint64_t bitmap[1] = {}; ++ faiss::IDGrouperBitmap id_grouper(1, bitmap); ++ int num_grp = 0; ++ int grp_size = 2; ++ int id_in_grp = 0; ++ for (int i = 0; i < nb; i++) { ++ xids[i] = i + num_grp; ++ id_in_grp++; ++ if (id_in_grp == grp_size) { ++ id_grouper.set_group(i + num_grp + 1); ++ num_grp++; ++ id_in_grp = 0; ++ } ++ } ++ ++ int k = 10; ++ int m = 8; ++ ++ faiss::IndexBinary* index = ++ new faiss::IndexBinaryHNSW(d, m); ++ faiss::IndexBinaryIDMap id_map = ++ faiss::IndexBinaryIDMap(index); // add vectors to the index ++ id_map.add_with_ids(nb, database.data(), xids); ++ ++ // search ++ idx_t* I = new idx_t[k]; ++ int32_t* D = new int32_t[k]; ++ ++ auto pSearchParameters = new faiss::SearchParametersHNSW(); ++ pSearchParameters->grp = &id_grouper; ++ ++ id_map.search(1, database.data(), k, D, I, pSearchParameters); ++ ++ std::unordered_set group_ids; ++ ASSERT_EQ(0, I[0]); ++ ASSERT_EQ(0, D[0]); ++ group_ids.insert(id_grouper.get_group(I[0])); ++ for (int j = 1; j < 5; j++) { ++ ASSERT_NE(-1, I[j]); ++ ASSERT_NE(std::numeric_limits::max(), D[j]); ++ group_ids.insert(id_grouper.get_group(I[j])); ++ } ++ for (int j = 5; j < k; j++) { ++ ASSERT_EQ(-1, I[j]); ++ ASSERT_EQ(std::numeric_limits::max(), D[j]); ++ } ++ ASSERT_EQ(5, group_ids.size()); ++ ++ delete[] I; ++ delete[] D; ++} +\ No newline at end of file +-- +2.39.3 (Apple Git-146) + diff --git a/jni/src/faiss_wrapper.cpp b/jni/src/faiss_wrapper.cpp index c4c6e18eb..9abb2357f 100644 --- a/jni/src/faiss_wrapper.cpp +++ b/jni/src/faiss_wrapper.cpp @@ -531,9 +531,16 @@ jobjectArray knn_jni::faiss_wrapper::QueryBinaryIndex_WithFilter(knn_jni::JNIUti return results; } -void knn_jni::faiss_wrapper::Free(jlong indexPointer) { - auto *indexWrapper = reinterpret_cast(indexPointer); - delete indexWrapper; +void knn_jni::faiss_wrapper::Free(jlong indexPointer, jboolean isBinaryIndexJ) { + bool isBinaryIndex = static_cast(isBinaryIndexJ); + if (isBinaryIndex) { + auto *indexWrapper = reinterpret_cast(indexPointer); + delete indexWrapper; + } + else { + auto *indexWrapper = reinterpret_cast(indexPointer); + delete indexWrapper; + } } void knn_jni::faiss_wrapper::FreeSharedIndexState(jlong shareIndexStatePointerJ) { diff --git a/jni/src/org_opensearch_knn_jni_FaissService.cpp b/jni/src/org_opensearch_knn_jni_FaissService.cpp index 5f9c83ea8..6e447b034 100644 --- a/jni/src/org_opensearch_knn_jni_FaissService.cpp +++ b/jni/src/org_opensearch_knn_jni_FaissService.cpp @@ -179,10 +179,10 @@ JNIEXPORT jobjectArray JNICALL Java_org_opensearch_knn_jni_FaissService_queryBin } -JNIEXPORT void JNICALL Java_org_opensearch_knn_jni_FaissService_free(JNIEnv * env, jclass cls, jlong indexPointerJ) +JNIEXPORT void JNICALL Java_org_opensearch_knn_jni_FaissService_free(JNIEnv * env, jclass cls, jlong indexPointerJ, jboolean isBinaryIndexJ) { try { - return knn_jni::faiss_wrapper::Free(indexPointerJ); + return knn_jni::faiss_wrapper::Free(indexPointerJ, isBinaryIndexJ); } catch (...) { jniUtil.CatchCppExceptionAndThrowJava(env); } diff --git a/jni/tests/faiss_wrapper_test.cpp b/jni/tests/faiss_wrapper_test.cpp index c6663a19a..36bcea491 100644 --- a/jni/tests/faiss_wrapper_test.cpp +++ b/jni/tests/faiss_wrapper_test.cpp @@ -596,7 +596,21 @@ TEST(FaissFreeTest, BasicAssertions) { test_util::FaissCreateIndex(dim, method, metricType)); // Free created index --> memory check should catch failure - knn_jni::faiss_wrapper::Free(reinterpret_cast(createdIndex)); + knn_jni::faiss_wrapper::Free(reinterpret_cast(createdIndex), JNI_FALSE); +} + + +TEST(FaissBinaryFreeTest, BasicAssertions) { + // Define the data + int dim = 8; + std::string method = "BHNSW32"; + + // Create the index + faiss::IndexBinary *createdIndex( + test_util::FaissCreateBinaryIndex(dim, method)); + + // Free created index --> memory check should catch failure + knn_jni::faiss_wrapper::Free(reinterpret_cast(createdIndex), JNI_TRUE); } TEST(FaissInitLibraryTest, BasicAssertions) { diff --git a/src/main/java/org/opensearch/knn/common/KNNValidationUtil.java b/src/main/java/org/opensearch/knn/common/KNNValidationUtil.java index ca8e1459a..59ca8993a 100644 --- a/src/main/java/org/opensearch/knn/common/KNNValidationUtil.java +++ b/src/main/java/org/opensearch/knn/common/KNNValidationUtil.java @@ -41,7 +41,7 @@ public static void validateFloatVectorValue(float value) { * * @param value float value in byte range */ - public static void validateByteVectorValue(float value) { + public static void validateByteVectorValue(float value, final VectorDataType dataType) { validateFloatVectorValue(value); if (value % 1 != 0) { throw new IllegalArgumentException( @@ -49,7 +49,7 @@ public static void validateByteVectorValue(float value) { Locale.ROOT, "[%s] field was set as [%s] in index mapping. But, KNN vector values are floats instead of byte integers", VECTOR_DATA_TYPE_FIELD, - VectorDataType.BYTE.getValue() + dataType.getValue() ) ); @@ -60,7 +60,7 @@ public static void validateByteVectorValue(float value) { Locale.ROOT, "[%s] field was set as [%s] in index mapping. But, KNN vector values are not within in the byte range [%d, %d]", VECTOR_DATA_TYPE_FIELD, - VectorDataType.BYTE.getValue(), + dataType.getValue(), Byte.MIN_VALUE, Byte.MAX_VALUE ) @@ -71,13 +71,32 @@ public static void validateByteVectorValue(float value) { /** * Validate if the given vector size matches with the dimension provided in mapping. * + * For binary index, the dimension is 8 times larger than vector size because 8 bits is packed into single byte + * * @param dimension dimension of vector * @param vectorSize size of the vector + * @param dataType vector data type */ - public static void validateVectorDimension(int dimension, int vectorSize) { - if (dimension != vectorSize) { - String errorMessage = String.format(Locale.ROOT, "Vector dimension mismatch. Expected: %d, Given: %d", dimension, vectorSize); - throw new IllegalArgumentException(errorMessage); + public static void validateVectorDimension(final int dimension, final int vectorSize, final VectorDataType dataType) { + int actualDimension = VectorDataType.BINARY == dataType ? vectorSize * Byte.SIZE : vectorSize; + if (dimension != actualDimension) { + if (VectorDataType.BINARY == dataType) { + String errorMessage = String.format( + Locale.ROOT, + "The dimension of the binary vector must be 8 times the length of the provided vector. Expected: %d, Given: %d", + dimension, + actualDimension + ); + throw new IllegalArgumentException(errorMessage); + } else { + String errorMessage = String.format( + Locale.ROOT, + "Vector dimension mismatch. Expected: %d, Given: %d", + dimension, + actualDimension + ); + throw new IllegalArgumentException(errorMessage); + } } } } diff --git a/src/main/java/org/opensearch/knn/index/IndexUtil.java b/src/main/java/org/opensearch/knn/index/IndexUtil.java index 7a5e93e14..7a445eda1 100644 --- a/src/main/java/org/opensearch/knn/index/IndexUtil.java +++ b/src/main/java/org/opensearch/knn/index/IndexUtil.java @@ -35,8 +35,8 @@ import static org.opensearch.knn.common.KNNConstants.BYTES_PER_KILOBYTES; import static org.opensearch.knn.common.KNNConstants.HNSW_ALGO_EF_SEARCH; -import static org.opensearch.knn.common.KNNConstants.INDEX_DESCRIPTION_PARAMETER; import static org.opensearch.knn.common.KNNConstants.SPACE_TYPE; +import static org.opensearch.knn.common.KNNConstants.VECTOR_DATA_TYPE_FIELD; public class IndexUtil { @@ -259,14 +259,14 @@ public static ValidationException validateKnnField( * @param spaceType Space for this particular segment * @param knnEngine Engine used for the native library indices being loaded in * @param indexName Name of OpenSearch index that the segment files belong to - * @param indexDescription Index description of OpenSearch index with faiss that the segment files belong to + * @param vectorDataType Vector data type for this particular segment * @return load parameters that will be passed to the JNI. */ public static Map getParametersAtLoading( SpaceType spaceType, KNNEngine knnEngine, String indexName, - String indexDescription + VectorDataType vectorDataType ) { Map loadParameters = Maps.newHashMap(ImmutableMap.of(SPACE_TYPE, spaceType.getValue())); @@ -275,9 +275,7 @@ public static Map getParametersAtLoading( if (KNNEngine.NMSLIB.equals(knnEngine)) { loadParameters.put(HNSW_ALGO_EF_SEARCH, KNNSettings.getEfSearchParam(indexName)); } - if (KNNEngine.FAISS.equals(knnEngine)) { - loadParameters.put(INDEX_DESCRIPTION_PARAMETER, indexDescription); - } + loadParameters.put(VECTOR_DATA_TYPE_FIELD, vectorDataType.getValue()); return Collections.unmodifiableMap(loadParameters); } @@ -312,4 +310,17 @@ public static boolean isSharedIndexStateRequired(KNNEngine knnEngine, String mod } return JNIService.isSharedIndexStateRequired(indexAddr, knnEngine); } + + /** + * Tell if it is binary index or not + * + * @param knnEngine knn engine associated with an index + * @param parameters parameters associated with an index + * @return true if it is binary index + */ + public static boolean isBinaryIndex(KNNEngine knnEngine, Map parameters) { + return KNNEngine.FAISS == knnEngine + && parameters.get(VECTOR_DATA_TYPE_FIELD) != null + && parameters.get(VECTOR_DATA_TYPE_FIELD).toString().equals(VectorDataType.BINARY.getValue()); + } } diff --git a/src/main/java/org/opensearch/knn/index/KNNIndexShard.java b/src/main/java/org/opensearch/knn/index/KNNIndexShard.java index e00d36d2e..2a44eec91 100644 --- a/src/main/java/org/opensearch/knn/index/KNNIndexShard.java +++ b/src/main/java/org/opensearch/knn/index/KNNIndexShard.java @@ -23,7 +23,6 @@ import org.opensearch.knn.index.memory.NativeMemoryCacheManager; import org.opensearch.knn.index.memory.NativeMemoryEntryContext; import org.opensearch.knn.index.memory.NativeMemoryLoadStrategy; -import org.opensearch.knn.index.util.FieldInfoExtractor; import org.opensearch.knn.index.util.KNNEngine; import java.io.IOException; @@ -37,6 +36,7 @@ import static org.opensearch.knn.common.KNNConstants.MODEL_ID; import static org.opensearch.knn.common.KNNConstants.SPACE_TYPE; +import static org.opensearch.knn.common.KNNConstants.VECTOR_DATA_TYPE_FIELD; import static org.opensearch.knn.index.IndexUtil.getParametersAtLoading; import static org.opensearch.knn.index.codec.util.KNNCodecUtil.buildEngineFilePrefix; import static org.opensearch.knn.index.codec.util.KNNCodecUtil.buildEngineFileSuffix; @@ -98,7 +98,7 @@ public void warmup() throws IOException { engineFileContext.getSpaceType(), KNNEngine.getEngineNameFromPath(engineFileContext.getIndexPath()), getIndexName(), - engineFileContext.indexDescription + engineFileContext.getVectorDataType() ), getIndexName(), engineFileContext.getModelId() @@ -182,7 +182,7 @@ List getEngineFileContexts(IndexReader indexReader, KNNEngine shardPath, spaceType, modelId, - FieldInfoExtractor.getIndexDescription(fieldInfo) + VectorDataType.get(fieldInfo.attributes().getOrDefault(VECTOR_DATA_TYPE_FIELD, VectorDataType.FLOAT.getValue())) ) ); } @@ -200,7 +200,7 @@ List getEngineFileContexts( Path shardPath, SpaceType spaceType, String modelId, - String indexDescription + VectorDataType vectorDataType ) { String prefix = buildEngineFilePrefix(segmentName); String suffix = buildEngineFileSuffix(fieldName, fileExtension); @@ -208,7 +208,7 @@ List getEngineFileContexts( .filter(fileName -> fileName.startsWith(prefix)) .filter(fileName -> fileName.endsWith(suffix)) .map(fileName -> shardPath.resolve(fileName).toString()) - .map(fileName -> new EngineFileContext(spaceType, modelId, fileName, indexDescription)) + .map(fileName -> new EngineFileContext(spaceType, modelId, fileName, vectorDataType)) .collect(Collectors.toList()); } @@ -219,6 +219,6 @@ static class EngineFileContext { private final SpaceType spaceType; private final String modelId; private final String indexPath; - private final String indexDescription; + private final VectorDataType vectorDataType; } } diff --git a/src/main/java/org/opensearch/knn/index/KNNMethodContext.java b/src/main/java/org/opensearch/knn/index/KNNMethodContext.java index ce48b06be..ba7a7509c 100644 --- a/src/main/java/org/opensearch/knn/index/KNNMethodContext.java +++ b/src/main/java/org/opensearch/knn/index/KNNMethodContext.java @@ -14,6 +14,8 @@ import lombok.AllArgsConstructor; import lombok.Getter; import lombok.NonNull; +import lombok.Setter; +import org.opensearch.Version; import org.opensearch.common.ValidationException; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; @@ -48,13 +50,15 @@ public class KNNMethodContext implements ToXContentFragment, Writeable { private static KNNMethodContext defaultInstance = null; + /** + * This is used only for testing + * @return default KNNMethodContext for testing + */ public static synchronized KNNMethodContext getDefault() { if (defaultInstance == null) { - defaultInstance = new KNNMethodContext( - KNNEngine.DEFAULT, - SpaceType.DEFAULT, - new MethodComponentContext(METHOD_HNSW, Collections.emptyMap()) - ); + MethodComponentContext methodComponentContext = new MethodComponentContext(METHOD_HNSW, Collections.emptyMap()); + methodComponentContext.setIndexVersion(Version.CURRENT); + defaultInstance = new KNNMethodContext(KNNEngine.DEFAULT, SpaceType.DEFAULT, methodComponentContext); } return defaultInstance; } @@ -62,7 +66,8 @@ public static synchronized KNNMethodContext getDefault() { @NonNull private final KNNEngine knnEngine; @NonNull - private final SpaceType spaceType; + @Setter + private SpaceType spaceType; @NonNull private final MethodComponentContext methodComponentContext; @@ -131,7 +136,7 @@ public static KNNMethodContext parse(Object in) { Map methodMap = (Map) in; KNNEngine engine = KNNEngine.DEFAULT; // Get or default - SpaceType spaceType = SpaceType.DEFAULT; // Get or default + SpaceType spaceType = SpaceType.UNDEFINED; // Get or default String name = ""; Map parameters = new HashMap<>(); diff --git a/src/main/java/org/opensearch/knn/index/KNNVectorSimilarityFunction.java b/src/main/java/org/opensearch/knn/index/KNNVectorSimilarityFunction.java new file mode 100644 index 000000000..7eca6287c --- /dev/null +++ b/src/main/java/org/opensearch/knn/index/KNNVectorSimilarityFunction.java @@ -0,0 +1,53 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index; + +import org.apache.lucene.index.VectorSimilarityFunction; +import org.opensearch.knn.plugin.script.KNNScoringUtil; + +/** + * Wrapper class of VectorSimilarityFunction to support more function than what Lucene provides + */ +public enum KNNVectorSimilarityFunction { + EUCLIDEAN(VectorSimilarityFunction.EUCLIDEAN), + DOT_PRODUCT(VectorSimilarityFunction.DOT_PRODUCT), + COSINE(VectorSimilarityFunction.COSINE), + MAXIMUM_INNER_PRODUCT(VectorSimilarityFunction.MAXIMUM_INNER_PRODUCT), + HAMMING(null) { + @Override + public float compare(float[] v1, float[] v2) { + throw new IllegalStateException("Hamming space is not supported with float vectors"); + } + + @Override + public float compare(byte[] v1, byte[] v2) { + return 1.0f / (1 + KNNScoringUtil.calculateHammingBit(v1, v2)); + } + + @Override + public VectorSimilarityFunction getVectorSimilarityFunction() { + throw new IllegalStateException("VectorSimilarityFunction is not available for Hamming space"); + } + }; + + private final VectorSimilarityFunction vectorSimilarityFunction; + + KNNVectorSimilarityFunction(final VectorSimilarityFunction vectorSimilarityFunction) { + this.vectorSimilarityFunction = vectorSimilarityFunction; + } + + public VectorSimilarityFunction getVectorSimilarityFunction() { + return vectorSimilarityFunction; + } + + public float compare(float[] var1, float[] var2) { + return vectorSimilarityFunction.compare(var1, var2); + } + + public float compare(byte[] var1, byte[] var2) { + return vectorSimilarityFunction.compare(var1, var2); + } +} diff --git a/src/main/java/org/opensearch/knn/index/SpaceType.java b/src/main/java/org/opensearch/knn/index/SpaceType.java index 240bfbe91..a65c4bb4c 100644 --- a/src/main/java/org/opensearch/knn/index/SpaceType.java +++ b/src/main/java/org/opensearch/knn/index/SpaceType.java @@ -12,7 +12,6 @@ package org.opensearch.knn.index; import java.util.Locale; -import org.apache.lucene.index.VectorSimilarityFunction; import java.util.HashSet; import java.util.Set; @@ -26,6 +25,19 @@ * nmslib calls the inner_product space "negdotprod". This translation should take place in the nmslib's jni layer. */ public enum SpaceType { + // This undefined space type is used to indicate that space type is not provided by user + // Later, we need to assign a default value based on data type + UNDEFINED("undefined") { + @Override + public float scoreTranslation(final float rawScore) { + throw new IllegalStateException("Unsupported method"); + } + + @Override + public void validateVectorDataType(VectorDataType vectorDataType) { + throw new IllegalStateException("Unsupported method"); + } + }, L2("l2") { @Override public float scoreTranslation(float rawScore) { @@ -33,8 +45,8 @@ public float scoreTranslation(float rawScore) { } @Override - public VectorSimilarityFunction getVectorSimilarityFunction() { - return VectorSimilarityFunction.EUCLIDEAN; + public KNNVectorSimilarityFunction getKnnVectorSimilarityFunction() { + return KNNVectorSimilarityFunction.EUCLIDEAN; } @Override @@ -52,8 +64,8 @@ public float scoreTranslation(float rawScore) { } @Override - public VectorSimilarityFunction getVectorSimilarityFunction() { - return VectorSimilarityFunction.COSINE; + public KNNVectorSimilarityFunction getKnnVectorSimilarityFunction() { + return KNNVectorSimilarityFunction.COSINE; } @Override @@ -104,8 +116,8 @@ public float scoreTranslation(float rawScore) { } @Override - public VectorSimilarityFunction getVectorSimilarityFunction() { - return VectorSimilarityFunction.MAXIMUM_INNER_PRODUCT; + public KNNVectorSimilarityFunction getKnnVectorSimilarityFunction() { + return KNNVectorSimilarityFunction.MAXIMUM_INNER_PRODUCT; } }, HAMMING_BIT("hammingbit") { @@ -113,9 +125,29 @@ public VectorSimilarityFunction getVectorSimilarityFunction() { public float scoreTranslation(float rawScore) { return 1 / (1 + rawScore); } + + @Override + public void validateVectorDataType(VectorDataType vectorDataType) { + if (VectorDataType.BINARY != vectorDataType) { + throw new IllegalArgumentException( + String.format( + Locale.ROOT, + "Space type [%s] is not supported with [%s] data type", + getValue(), + vectorDataType.getValue() + ) + ); + } + } + + @Override + public KNNVectorSimilarityFunction getKnnVectorSimilarityFunction() { + return KNNVectorSimilarityFunction.HAMMING; + } }; public static SpaceType DEFAULT = L2; + public static SpaceType DEFAULT_BINARY = HAMMING_BIT; private final String value; @@ -126,12 +158,12 @@ public float scoreTranslation(float rawScore) { public abstract float scoreTranslation(float rawScore); /** - * Get VectorSimilarityFunction that maps to this SpaceType + * Get KNNVectorSimilarityFunction that maps to this SpaceType * - * @return VectorSimilarityFunction + * @return KNNVectorSimilarityFunction */ - public VectorSimilarityFunction getVectorSimilarityFunction() { - throw new UnsupportedOperationException(String.format("Space [%s] does not have a vector similarity function", getValue())); + public KNNVectorSimilarityFunction getKnnVectorSimilarityFunction() { + throw new UnsupportedOperationException(String.format("Space [%s] does not have a knn vector similarity function", getValue())); } /** @@ -152,6 +184,19 @@ public void validateVector(float[] vector) { // do nothing } + /** + * Validate if given vector data type is supported by this space type + * + * @param vectorDataType the given vector data type + */ + public void validateVectorDataType(VectorDataType vectorDataType) { + if (VectorDataType.FLOAT != vectorDataType && VectorDataType.BYTE != vectorDataType) { + throw new IllegalArgumentException( + String.format(Locale.ROOT, "Space type [%s] is not supported with [%s] data type", getValue(), vectorDataType.getValue()) + ); + } + } + /** * Get space type name in engine * diff --git a/src/main/java/org/opensearch/knn/index/VectorDataType.java b/src/main/java/org/opensearch/knn/index/VectorDataType.java index 98b767f8d..8add84609 100644 --- a/src/main/java/org/opensearch/knn/index/VectorDataType.java +++ b/src/main/java/org/opensearch/knn/index/VectorDataType.java @@ -24,11 +24,25 @@ import static org.opensearch.knn.common.KNNConstants.VECTOR_DATA_TYPE_FIELD; /** - * Enum contains data_type of vectors and right now only supported for lucene engine in k-NN plugin. - * We have two vector data_types, one is float (default) and the other one is byte. + * Enum contains data_type of vectors + * Lucene supports byte and float data type + * NMSLib supports only float data type + * Faiss supports binary and float data type */ @AllArgsConstructor public enum VectorDataType { + BINARY("binary") { + + @Override + public FieldType createKnnVectorFieldType(int dimension, VectorSimilarityFunction vectorSimilarityFunction) { + throw new IllegalStateException("Unsupported method"); + } + + @Override + public float[] getVectorFromBytesRef(BytesRef binaryValue) { + throw new IllegalStateException("Unsupported method"); + } + }, BYTE("byte") { @Override diff --git a/src/main/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumer.java b/src/main/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumer.java index 096df817a..2eba19f7e 100644 --- a/src/main/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumer.java +++ b/src/main/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumer.java @@ -14,6 +14,10 @@ import org.opensearch.core.xcontent.DeprecationHandler; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.knn.index.KNNSettings; +import org.opensearch.knn.index.VectorDataType; +import org.opensearch.knn.index.codec.transfer.VectorTransfer; +import org.opensearch.knn.index.codec.transfer.VectorTransferByte; +import org.opensearch.knn.index.codec.transfer.VectorTransferFloat; import org.opensearch.knn.jni.JNIService; import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.codec.util.KNNCodecUtil; @@ -54,6 +58,7 @@ import static org.opensearch.knn.common.KNNConstants.PARAMETERS; import static org.opensearch.knn.index.codec.util.KNNCodecUtil.buildEngineFileName; import static org.opensearch.knn.index.codec.util.KNNCodecUtil.calculateArraySize; +import static org.opensearch.knn.index.util.Faiss.FAISS_BINARY_INDEX_DESCRIPTION_PREFIX; /** * This class writes the KNN docvalues to the segments @@ -104,11 +109,18 @@ private KNNEngine getKNNEngine(@NonNull FieldInfo field) { return KNNEngine.getEngine(engineName); } + private VectorTransfer getVectorTransfer(FieldInfo field) { + if (VectorDataType.BINARY.getValue().equalsIgnoreCase(field.attributes().get(KNNConstants.VECTOR_DATA_TYPE_FIELD))) { + return new VectorTransferByte(KNNSettings.getVectorStreamingMemoryLimit().getBytes()); + } + return new VectorTransferFloat(KNNSettings.getVectorStreamingMemoryLimit().getBytes()); + } + public void addKNNBinaryField(FieldInfo field, DocValuesProducer valuesProducer, boolean isMerge, boolean isRefresh) throws IOException { // Get values to be indexed BinaryDocValues values = valuesProducer.getBinary(field); - KNNCodecUtil.Pair pair = KNNCodecUtil.getFloats(values); + KNNCodecUtil.Pair pair = KNNCodecUtil.getPair(values, getVectorTransfer(field)); if (pair.getVectorAddress() == 0 || pair.docs.length == 0) { logger.info("Skipping engine index creation as there are no vectors or docs in the segment"); return; @@ -222,6 +234,18 @@ private void createKNNIndexFromScratch(FieldInfo fieldInfo, KNNCodecUtil.Pair pa ); } + // Update index description of Faiss for binary data type + if (KNNEngine.FAISS == knnEngine + && VectorDataType.BINARY.getValue() + .equals(fieldAttributes.getOrDefault(KNNConstants.VECTOR_DATA_TYPE_FIELD, VectorDataType.FLOAT.getValue())) + && parameters.get(KNNConstants.INDEX_DESCRIPTION_PARAMETER) != null) { + parameters.put( + KNNConstants.INDEX_DESCRIPTION_PARAMETER, + FAISS_BINARY_INDEX_DESCRIPTION_PREFIX + parameters.get(KNNConstants.INDEX_DESCRIPTION_PARAMETER).toString() + ); + parameters.put(KNNConstants.VECTOR_DATA_TYPE_FIELD, VectorDataType.BINARY.getValue()); + } + // Used to determine how many threads to use when indexing parameters.put(KNNConstants.INDEX_THREAD_QTY, KNNSettings.state().getSettingValue(KNNSettings.KNN_ALGO_PARAM_INDEX_THREAD_QTY)); diff --git a/src/main/java/org/opensearch/knn/index/codec/transfer/VectorTransfer.java b/src/main/java/org/opensearch/knn/index/codec/transfer/VectorTransfer.java new file mode 100644 index 000000000..2ab80f776 --- /dev/null +++ b/src/main/java/org/opensearch/knn/index/codec/transfer/VectorTransfer.java @@ -0,0 +1,55 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.codec.transfer; + +import lombok.Data; +import org.opensearch.knn.index.codec.util.SerializationMode; + +import java.io.ByteArrayInputStream; + +/** + * Abstract class to transfer vector value from Java to native memory + */ +@Data +public abstract class VectorTransfer { + protected final long vectorsStreamingMemoryLimit; + protected long totalLiveDocs; + protected long vectorsPerTransfer; + protected long vectorAddress; + protected int dimension; + + public VectorTransfer(final long vectorsStreamingMemoryLimit) { + this.vectorsStreamingMemoryLimit = vectorsStreamingMemoryLimit; + this.vectorsPerTransfer = Integer.MIN_VALUE; + } + + /** + * Initialize the transfer + * + * @param totalLiveDocs total number of vectors to be transferred + */ + abstract public void init(final long totalLiveDocs); + + /** + * Transfer a single vector + * + * @param byteStream a vector in byte stream format + */ + abstract public void transfer(final ByteArrayInputStream byteStream); + + /** + * Close the transfer + */ + abstract public void close(); + + /** + * Get serialization mode of given byte stream + * + * @param byteStream byte stream of a vector + * @return serialization mode + */ + abstract public SerializationMode getSerializationMode(final ByteArrayInputStream byteStream); +} diff --git a/src/main/java/org/opensearch/knn/index/codec/transfer/VectorTransferByte.java b/src/main/java/org/opensearch/knn/index/codec/transfer/VectorTransferByte.java new file mode 100644 index 000000000..fb0f9d470 --- /dev/null +++ b/src/main/java/org/opensearch/knn/index/codec/transfer/VectorTransferByte.java @@ -0,0 +1,66 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.codec.transfer; + +import org.opensearch.knn.index.codec.util.SerializationMode; +import org.opensearch.knn.jni.JNICommons; + +import java.io.ByteArrayInputStream; +import java.util.ArrayList; +import java.util.List; + +/** + * Vector transfer for byte + */ +public class VectorTransferByte extends VectorTransfer { + private List vectorList; + + public VectorTransferByte(final long vectorsStreamingMemoryLimit) { + super(vectorsStreamingMemoryLimit); + vectorList = new ArrayList<>(); + } + + @Override + public void init(final long totalLiveDocs) { + this.totalLiveDocs = totalLiveDocs; + vectorList.clear(); + } + + @Override + public void transfer(final ByteArrayInputStream byteStream) { + final byte[] vector = byteStream.readAllBytes(); + dimension = vector.length * 8; + if (vectorsPerTransfer == Integer.MIN_VALUE) { + vectorsPerTransfer = (vector.length * totalLiveDocs) / vectorsStreamingMemoryLimit; + // This condition comes if vectorsStreamingMemoryLimit is higher than total number floats to transfer + // Doing this will reduce 1 extra trip to JNI layer. + if (vectorsPerTransfer == 0) { + vectorsPerTransfer = totalLiveDocs; + } + } + + vectorList.add(vector); + if (vectorList.size() == vectorsPerTransfer) { + transfer(); + } + } + + @Override + public void close() { + transfer(); + } + + @Override + public SerializationMode getSerializationMode(final ByteArrayInputStream byteStream) { + return SerializationMode.COLLECTIONS_OF_BYTES; + } + + private void transfer() { + int lengthOfVector = dimension / 8; + vectorAddress = JNICommons.storeByteVectorData(vectorAddress, vectorList.toArray(new byte[][] {}), totalLiveDocs * lengthOfVector); + vectorList.clear(); + } +} diff --git a/src/main/java/org/opensearch/knn/index/codec/transfer/VectorTransferFloat.java b/src/main/java/org/opensearch/knn/index/codec/transfer/VectorTransferFloat.java new file mode 100644 index 000000000..d5958b375 --- /dev/null +++ b/src/main/java/org/opensearch/knn/index/codec/transfer/VectorTransferFloat.java @@ -0,0 +1,69 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.codec.transfer; + +import org.opensearch.knn.index.codec.util.KNNVectorSerializer; +import org.opensearch.knn.index.codec.util.KNNVectorSerializerFactory; +import org.opensearch.knn.index.codec.util.SerializationMode; +import org.opensearch.knn.jni.JNICommons; + +import java.io.ByteArrayInputStream; +import java.util.ArrayList; +import java.util.List; + +/** + * Vector transfer for float + */ +public class VectorTransferFloat extends VectorTransfer { + private List vectorList; + + public VectorTransferFloat(final long vectorsStreamingMemoryLimit) { + super(vectorsStreamingMemoryLimit); + vectorList = new ArrayList<>(); + } + + @Override + public void init(final long totalLiveDocs) { + this.totalLiveDocs = totalLiveDocs; + vectorList.clear(); + } + + @Override + public void transfer(final ByteArrayInputStream byteStream) { + final KNNVectorSerializer vectorSerializer = KNNVectorSerializerFactory.getSerializerByStreamContent(byteStream); + final float[] vector = vectorSerializer.byteToFloatArray(byteStream); + dimension = vector.length; + + if (vectorsPerTransfer == Integer.MIN_VALUE) { + vectorsPerTransfer = (dimension * Float.BYTES * totalLiveDocs) / vectorsStreamingMemoryLimit; + // This condition comes if vectorsStreamingMemoryLimit is higher than total number floats to transfer + // Doing this will reduce 1 extra trip to JNI layer. + if (vectorsPerTransfer == 0) { + vectorsPerTransfer = totalLiveDocs; + } + } + + vectorList.add(vector); + if (vectorList.size() == vectorsPerTransfer) { + transfer(); + } + } + + @Override + public void close() { + transfer(); + } + + @Override + public SerializationMode getSerializationMode(final ByteArrayInputStream byteStream) { + return KNNVectorSerializerFactory.getSerializerModeFromStream(byteStream); + } + + private void transfer() { + vectorAddress = JNICommons.storeVectorData(vectorAddress, vectorList.toArray(new float[][] {}), totalLiveDocs * dimension); + vectorList.clear(); + } +} diff --git a/src/main/java/org/opensearch/knn/index/codec/util/KNNCodecUtil.java b/src/main/java/org/opensearch/knn/index/codec/util/KNNCodecUtil.java index 0a000cadb..68b61a070 100644 --- a/src/main/java/org/opensearch/knn/index/codec/util/KNNCodecUtil.java +++ b/src/main/java/org/opensearch/knn/index/codec/util/KNNCodecUtil.java @@ -11,9 +11,8 @@ import org.apache.lucene.index.BinaryDocValues; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.util.BytesRef; -import org.opensearch.knn.index.KNNSettings; import org.opensearch.knn.index.codec.KNN80Codec.KNN80BinaryDocValues; -import org.opensearch.knn.jni.JNICommons; +import org.opensearch.knn.index.codec.transfer.VectorTransfer; import java.io.ByteArrayInputStream; import java.io.IOException; @@ -40,57 +39,27 @@ public static final class Pair { @Setter private int dimension; public SerializationMode serializationMode; - } - public static KNNCodecUtil.Pair getFloats(BinaryDocValues values) throws IOException { - List vectorList = new ArrayList<>(); + public static KNNCodecUtil.Pair getPair(final BinaryDocValues values, final VectorTransfer vectorTransfer) throws IOException { List docIdList = new ArrayList<>(); - long vectorAddress = 0; - int dimension = 0; SerializationMode serializationMode = SerializationMode.COLLECTION_OF_FLOATS; - - long totalLiveDocs = getTotalLiveDocsCount(values); - long vectorsStreamingMemoryLimit = KNNSettings.getVectorStreamingMemoryLimit().getBytes(); - long vectorsPerTransfer = Integer.MIN_VALUE; - + vectorTransfer.init(getTotalLiveDocsCount(values)); for (int doc = values.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = values.nextDoc()) { BytesRef bytesref = values.binaryValue(); try (ByteArrayInputStream byteStream = new ByteArrayInputStream(bytesref.bytes, bytesref.offset, bytesref.length)) { - serializationMode = KNNVectorSerializerFactory.serializerModeFromStream(byteStream); - final KNNVectorSerializer vectorSerializer = KNNVectorSerializerFactory.getSerializerByStreamContent(byteStream); - final float[] vector = vectorSerializer.byteToFloatArray(byteStream); - dimension = vector.length; - - if (vectorsPerTransfer == Integer.MIN_VALUE) { - // if vectorsStreamingMemoryLimit is 100 bytes and we have 50 vectors with 5 dimension, then per - // transfer we have to send 100/(5 * 4) => 20 vectors. - vectorsPerTransfer = vectorsStreamingMemoryLimit / ((long) dimension * Float.BYTES); - // If vectorsPerTransfer comes out to be 0, then we set number of vectors per transfer to 1, to ensure that - // we are sending minimum number of vectors. - if (vectorsPerTransfer == 0) { - vectorsPerTransfer = 1; - } - } - if (vectorList.size() == vectorsPerTransfer) { - vectorAddress = JNICommons.storeVectorData( - vectorAddress, - vectorList.toArray(new float[][] {}), - totalLiveDocs * dimension - ); - // We should probably come up with a better way to reuse the vectorList memory which we have - // created. Problem here is doing like this can lead to a lot of list memory which is of no use and - // will be garbage collected later on, but it creates pressure on JVM. We should revisit this. - vectorList = new ArrayList<>(); - } - vectorList.add(vector); + serializationMode = vectorTransfer.getSerializationMode(byteStream); + vectorTransfer.transfer(byteStream); } docIdList.add(doc); } - if (vectorList.isEmpty() == false) { - vectorAddress = JNICommons.storeVectorData(vectorAddress, vectorList.toArray(new float[][] {}), totalLiveDocs * dimension); - } - return new KNNCodecUtil.Pair(docIdList.stream().mapToInt(Integer::intValue).toArray(), vectorAddress, dimension, serializationMode); + vectorTransfer.close(); + return new KNNCodecUtil.Pair( + docIdList.stream().mapToInt(Integer::intValue).toArray(), + vectorTransfer.getVectorAddress(), + vectorTransfer.getDimension(), + serializationMode + ); } public static long calculateArraySize(int numVectors, int vectorLength, SerializationMode serializationMode) { @@ -104,7 +73,7 @@ public static long calculateArraySize(int numVectors, int vectorLength, Serializ vectorsSize += vectorsSize % JAVA_ROUNDING_NUMBER; } return vectorsSize; - } else { + } else if (serializationMode == SerializationMode.COLLECTION_OF_FLOATS) { int vectorSize = vectorLength * FLOAT_BYTE_SIZE; if (vectorSize % JAVA_ROUNDING_NUMBER != 0) { vectorSize += vectorSize % JAVA_ROUNDING_NUMBER; @@ -114,6 +83,18 @@ public static long calculateArraySize(int numVectors, int vectorLength, Serializ vectorsSize += vectorsSize % JAVA_ROUNDING_NUMBER; } return vectorsSize; + } else if (serializationMode == SerializationMode.COLLECTIONS_OF_BYTES) { + int vectorSize = vectorLength; + if (vectorSize % JAVA_ROUNDING_NUMBER != 0) { + vectorSize += vectorSize % JAVA_ROUNDING_NUMBER; + } + int vectorsSize = numVectors * (vectorSize + JAVA_REFERENCE_SIZE); + if (vectorsSize % JAVA_ROUNDING_NUMBER != 0) { + vectorsSize += vectorsSize % JAVA_ROUNDING_NUMBER; + } + return vectorsSize; + } else { + throw new IllegalStateException("Unreachable code"); } } diff --git a/src/main/java/org/opensearch/knn/index/codec/util/KNNVectorSerializerFactory.java b/src/main/java/org/opensearch/knn/index/codec/util/KNNVectorSerializerFactory.java index 5c1e4ca9b..23a829dfd 100644 --- a/src/main/java/org/opensearch/knn/index/codec/util/KNNVectorSerializerFactory.java +++ b/src/main/java/org/opensearch/knn/index/codec/util/KNNVectorSerializerFactory.java @@ -52,11 +52,11 @@ public static KNNVectorSerializer getDefaultSerializer() { } public static KNNVectorSerializer getSerializerByStreamContent(final ByteArrayInputStream byteStream) { - final SerializationMode serializationMode = serializerModeFromStream(byteStream); + final SerializationMode serializationMode = getSerializerModeFromStream(byteStream); return getSerializerBySerializationMode(serializationMode); } - static SerializationMode serializerModeFromStream(ByteArrayInputStream byteStream) { + public static SerializationMode getSerializerModeFromStream(ByteArrayInputStream byteStream) { int numberOfAvailableBytesInStream = byteStream.available(); if (numberOfAvailableBytesInStream < ARRAY_HEADER_OFFSET) { return getSerializerOrThrowError(numberOfAvailableBytesInStream, COLLECTION_OF_FLOATS); diff --git a/src/main/java/org/opensearch/knn/index/codec/util/SerializationMode.java b/src/main/java/org/opensearch/knn/index/codec/util/SerializationMode.java index 1fb82cbfe..f3a32f53e 100644 --- a/src/main/java/org/opensearch/knn/index/codec/util/SerializationMode.java +++ b/src/main/java/org/opensearch/knn/index/codec/util/SerializationMode.java @@ -7,5 +7,6 @@ public enum SerializationMode { ARRAY, - COLLECTION_OF_FLOATS + COLLECTION_OF_FLOATS, + COLLECTIONS_OF_BYTES } diff --git a/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapper.java b/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapper.java index 7e697fed7..7f7c83f3e 100644 --- a/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapper.java +++ b/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapper.java @@ -57,6 +57,7 @@ import org.opensearch.search.lookup.SearchLookup; import static org.opensearch.knn.common.KNNConstants.DEFAULT_VECTOR_DATA_TYPE_FIELD; +import static org.opensearch.knn.common.KNNConstants.ENCODER_FLAT; import static org.opensearch.knn.common.KNNConstants.ENCODER_SQ; import static org.opensearch.knn.common.KNNConstants.FAISS_SQ_CLIP; import static org.opensearch.knn.common.KNNConstants.FAISS_SQ_ENCODER_FP16; @@ -73,7 +74,7 @@ import static org.opensearch.knn.index.mapper.KNNVectorFieldMapperUtil.clipVectorValueToFP16Range; import static org.opensearch.knn.index.mapper.KNNVectorFieldMapperUtil.deserializeStoredVector; import static org.opensearch.knn.index.mapper.KNNVectorFieldMapperUtil.validateFP16VectorValue; -import static org.opensearch.knn.index.mapper.KNNVectorFieldMapperUtil.validateVectorDataTypeWithEngine; +import static org.opensearch.knn.index.mapper.KNNVectorFieldMapperUtil.validateVectorDataType; import static org.opensearch.knn.index.mapper.KNNVectorFieldMapperUtil.validateVectorDataTypeWithKnnIndexSetting; /** @@ -112,11 +113,13 @@ public static class Builder extends ParametrizedFieldMapper.Builder { value = XContentMapValues.nodeIntegerValue(o); } catch (Exception exception) { throw new IllegalArgumentException( - String.format("Unable to parse [dimension] from provided value [%s] for vector [%s]", o, name) + String.format(Locale.ROOT, "Unable to parse [dimension] from provided value [%s] for vector [%s]", o, name) ); } if (value <= 0) { - throw new IllegalArgumentException(String.format("Dimension value must be greater than 0 for vector: %s", name)); + throw new IllegalArgumentException( + String.format(Locale.ROOT, "Dimension value must be greater than 0 for vector: %s", name) + ); } return value; }, m -> toType(m).dimension); @@ -125,7 +128,7 @@ public static class Builder extends ParametrizedFieldMapper.Builder { * data_type which defines the datatype of the vector values. This is an optional parameter and * this is right now only relevant for lucene engine. The default value is float. */ - private final Parameter vectorDataType = new Parameter<>( + protected final Parameter vectorDataType = new Parameter<>( VECTOR_DATA_TYPE_FIELD, false, () -> DEFAULT_VECTOR_DATA_TYPE_FIELD, @@ -160,7 +163,7 @@ public static class Builder extends ParametrizedFieldMapper.Builder { ValidationException validationException = null; if (v.isTrainingRequired()) { validationException = new ValidationException(); - validationException.addValidationError(String.format("\"%s\" requires training.", KNN_METHOD)); + validationException.addValidationError(String.format(Locale.ROOT, "\"%s\" requires training.", KNN_METHOD)); } ValidationException methodValidation = v.validate(); @@ -233,13 +236,17 @@ public KNNVectorFieldMapper build(BuilderContext context) { // the mappings, setting the index settings will have no impact. final KNNMethodContext knnMethodContext = this.knnMethodContext.getValue(); - validateMaxDimensions(knnMethodContext); + setDefaultSpaceType(knnMethodContext, vectorDataType.getValue()); + validateSpaceType(knnMethodContext, vectorDataType.getValue()); + validateDimensions(knnMethodContext, vectorDataType.getValue()); + validateEncoder(knnMethodContext, vectorDataType.getValue()); final MultiFields multiFieldsBuilder = this.multiFieldsBuilder.build(this, context); final CopyTo copyToBuilder = copyTo.build(); final Explicit ignoreMalformed = ignoreMalformed(context); final Map metaValue = meta.getValue(); if (knnMethodContext != null) { + validateVectorDataType(knnMethodContext, vectorDataType.getValue()); knnMethodContext.getMethodComponentContext().setIndexVersion(indexCreatedVersion); final KNNVectorFieldType mappedFieldType = new KNNVectorFieldType( buildFullName(context), @@ -249,7 +256,7 @@ public KNNVectorFieldMapper build(BuilderContext context) { vectorDataType.getValue() ); if (knnMethodContext.getKnnEngine() == KNNEngine.LUCENE) { - log.debug(String.format("Use [LuceneFieldMapper] mapper for field [%s]", name)); + log.debug(String.format(Locale.ROOT, "Use [LuceneFieldMapper] mapper for field [%s]", name)); LuceneFieldMapper.CreateLuceneFieldMapperInput createLuceneFieldMapperInput = LuceneFieldMapper.CreateLuceneFieldMapperInput.builder() .name(name) @@ -265,11 +272,6 @@ public KNNVectorFieldMapper build(BuilderContext context) { return new LuceneFieldMapper(createLuceneFieldMapperInput); } - // Validates and throws exception if data_type field is set in the index mapping - // using any VectorDataType (other than float, which is default) because other - // VectorDataTypes are only supported for lucene engine. - validateVectorDataTypeWithEngine(vectorDataType); - return new MethodFieldMapper( name, mappedFieldType, @@ -342,7 +344,72 @@ public KNNVectorFieldMapper build(BuilderContext context) { ); } - private KNNEngine validateMaxDimensions(final KNNMethodContext knnMethodContext) { + private void validateEncoder(final KNNMethodContext knnMethodContext, final VectorDataType vectorDataType) { + if (knnMethodContext == null) { + return; + } + + if (VectorDataType.BINARY != vectorDataType) { + return; + } + + if (knnMethodContext.getMethodComponentContext() == null) { + return; + } + + if (knnMethodContext.getMethodComponentContext().getParameters() == null) { + return; + } + + if (knnMethodContext.getMethodComponentContext().getParameters().get(METHOD_ENCODER_PARAMETER) == null) { + return; + } + + if (knnMethodContext.getMethodComponentContext() + .getParameters() + .get(METHOD_ENCODER_PARAMETER) instanceof MethodComponentContext == false) { + return; + } + + MethodComponentContext encoderMethodComponentContext = (MethodComponentContext) knnMethodContext.getMethodComponentContext() + .getParameters() + .get(METHOD_ENCODER_PARAMETER); + + if (ENCODER_FLAT.equals(encoderMethodComponentContext.getName()) == false) { + throw new IllegalArgumentException( + String.format( + Locale.ROOT, + "%s data type does not support %s encoder", + VectorDataType.BINARY.getValue(), + encoderMethodComponentContext.getName() + ) + ); + } + } + + private void setDefaultSpaceType(final KNNMethodContext knnMethodContext, final VectorDataType vectorDataType) { + if (knnMethodContext == null) { + return; + } + + if (SpaceType.UNDEFINED == knnMethodContext.getSpaceType()) { + if (VectorDataType.BINARY == vectorDataType) { + knnMethodContext.setSpaceType(SpaceType.DEFAULT_BINARY); + } else { + knnMethodContext.setSpaceType(SpaceType.DEFAULT); + } + } + } + + private void validateSpaceType(final KNNMethodContext knnMethodContext, final VectorDataType vectorDataType) { + if (knnMethodContext == null) { + return; + } + + knnMethodContext.getSpaceType().validateVectorDataType(vectorDataType); + } + + private KNNEngine validateDimensions(final KNNMethodContext knnMethodContext, final VectorDataType dataType) { final KNNEngine knnEngine; if (knnMethodContext != null) { knnEngine = knnMethodContext.getKnnEngine(); @@ -352,12 +419,16 @@ private KNNEngine validateMaxDimensions(final KNNMethodContext knnMethodContext) if (dimension.getValue() > KNNEngine.getMaxDimensionByEngine(knnEngine)) { throw new IllegalArgumentException( String.format( + Locale.ROOT, "Dimension value cannot be greater than %s for vector: %s", KNNEngine.getMaxDimensionByEngine(knnEngine), name ) ); } + if (VectorDataType.BINARY == dataType && dimension.getValue() % 8 != 0) { + throw new IllegalArgumentException("Dimension should be multiply of 8 for binary vector data type"); + } return knnEngine; } } @@ -382,12 +453,14 @@ public Mapper.Builder parse(String name, Map node, ParserCont // is done before any mappers are built. Therefore, validation should be done during parsing // so that it can fail early. if (builder.knnMethodContext.get() != null && builder.modelId.get() != null) { - throw new IllegalArgumentException(String.format("Method and model can not be both specified in the mapping: %s", name)); + throw new IllegalArgumentException( + String.format(Locale.ROOT, "Method and model can not be both specified in the mapping: %s", name) + ); } // Dimension should not be null unless modelId is used if (builder.dimension.getValue() == -1 && builder.modelId.get() == null) { - throw new IllegalArgumentException(String.format("Dimension value missing for vector: %s", name)); + throw new IllegalArgumentException(String.format(Locale.ROOT, "Dimension value missing for vector: %s", name)); } return builder; @@ -466,7 +539,7 @@ public Query existsQuery(QueryShardContext context) { public Query termQuery(Object value, QueryShardContext context) { throw new QueryShardException( context, - String.format("KNN vector do not support exact searching, use KNN queries instead: [%s]", name()) + String.format(Locale.ROOT, "KNN vector do not support exact searching, use KNN queries instead: [%s]", name()) ); } @@ -578,9 +651,19 @@ protected void parseCreateField(ParseContext context, int dimension, SpaceType s validateIfKNNPluginEnabled(); validateIfCircuitBreakerIsNotTriggered(); + spaceType.validateVectorDataType(vectorDataType); - if (VectorDataType.BYTE == vectorDataType) { - Optional bytesArrayOptional = getBytesFromContext(context, dimension); + if (VectorDataType.BINARY == vectorDataType) { + Optional bytesArrayOptional = getBytesFromContext(context, dimension, vectorDataType); + + if (bytesArrayOptional.isEmpty()) { + return; + } + final byte[] array = bytesArrayOptional.get(); + spaceType.validateVector(array); + context.doc().addAll(getFieldsForByteVector(array, fieldType)); + } else if (VectorDataType.BYTE == vectorDataType) { + Optional bytesArrayOptional = getBytesFromContext(context, dimension, vectorDataType); if (bytesArrayOptional.isEmpty()) { return; @@ -663,31 +746,30 @@ void validateIfKNNPluginEnabled() { // Returns an optional array of byte values where each value in the vector is parsed as a float and validated // if it is a finite number without any decimals and within the byte range of [-128 to 127]. - Optional getBytesFromContext(ParseContext context, int dimension) throws IOException { + Optional getBytesFromContext(ParseContext context, int dimension, VectorDataType dataType) throws IOException { context.path().add(simpleName()); ArrayList vector = new ArrayList<>(); XContentParser.Token token = context.parser().currentToken(); - float value; if (token == XContentParser.Token.START_ARRAY) { token = context.parser().nextToken(); while (token != XContentParser.Token.END_ARRAY) { - value = context.parser().floatValue(); - validateByteVectorValue(value); + float value = context.parser().floatValue(); + validateByteVectorValue(value, dataType); vector.add((byte) value); token = context.parser().nextToken(); } } else if (token == XContentParser.Token.VALUE_NUMBER) { - value = context.parser().floatValue(); - validateByteVectorValue(value); + float value = context.parser().floatValue(); + validateByteVectorValue(value, dataType); vector.add((byte) value); context.parser().nextToken(); } else if (token == XContentParser.Token.VALUE_NULL) { context.path().remove(); return Optional.empty(); } - validateVectorDimension(dimension, vector.size()); + validateVectorDimension(dimension, vector.size(), dataType); byte[] array = new byte[vector.size()]; int i = 0; for (Byte f : vector) { @@ -749,7 +831,7 @@ Optional getFloatsFromContext(ParseContext context, int dimension, Meth context.path().remove(); return Optional.empty(); } - validateVectorDimension(dimension, vector.size()); + validateVectorDimension(dimension, vector.size(), vectorDataType); float[] array = new float[vector.size()]; int i = 0; diff --git a/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperUtil.java b/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperUtil.java index 8bd7eb6f2..03f369d0d 100644 --- a/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperUtil.java +++ b/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperUtil.java @@ -18,6 +18,7 @@ import org.apache.lucene.index.DocValuesType; import org.apache.lucene.util.BytesRef; import org.opensearch.index.mapper.ParametrizedFieldMapper; +import org.opensearch.knn.index.KNNMethodContext; import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.codec.util.KNNVectorSerializerFactory; import org.opensearch.knn.index.util.KNNEngine; @@ -29,11 +30,14 @@ import java.util.Locale; import static org.opensearch.knn.common.KNNConstants.ENCODER_SQ; +import static org.opensearch.knn.common.KNNConstants.FAISS_NAME; import static org.opensearch.knn.common.KNNConstants.FAISS_SQ_ENCODER_FP16; import static org.opensearch.knn.common.KNNConstants.FP16_MAX_VALUE; import static org.opensearch.knn.common.KNNConstants.FP16_MIN_VALUE; import static org.opensearch.knn.common.KNNConstants.KNN_ENGINE; import static org.opensearch.knn.common.KNNConstants.LUCENE_NAME; +import static org.opensearch.knn.common.KNNConstants.METHOD_HNSW; +import static org.opensearch.knn.common.KNNConstants.NMSLIB_NAME; import static org.opensearch.knn.common.KNNConstants.VECTOR_DATA_TYPE_FIELD; import static org.opensearch.knn.common.KNNValidationUtil.validateFloatVectorValue; @@ -90,25 +94,60 @@ public static float clipVectorValueToFP16Range(float value) { } /** - * Validates and throws exception if data_type field is set in the index mapping - * using any VectorDataType (other than float, which is default) because other - * VectorDataTypes are only supported for lucene engine. + * Validates if the vector data type is supported with given method context * - * @param vectorDataType VectorDataType Parameter + * @param methodContext methodContext + * @param vectorDataType vector data type */ - public static void validateVectorDataTypeWithEngine(ParametrizedFieldMapper.Parameter vectorDataType) { - if (VectorDataType.FLOAT == vectorDataType.getValue()) { + public static void validateVectorDataType(KNNMethodContext methodContext, VectorDataType vectorDataType) { + if (VectorDataType.FLOAT == vectorDataType) { return; } - throw new IllegalArgumentException( - String.format( - Locale.ROOT, - "[%s] field with value [%s] is only supported for [%s] engine", - VECTOR_DATA_TYPE_FIELD, - vectorDataType.getValue().getValue(), - LUCENE_NAME - ) - ); + + if (VectorDataType.BYTE == vectorDataType) { + if (KNNEngine.LUCENE == methodContext.getKnnEngine()) { + return; + } else { + throw new IllegalArgumentException( + String.format( + Locale.ROOT, + "[%s] field with value [%s] is only supported for [%s] engine", + VECTOR_DATA_TYPE_FIELD, + vectorDataType.getValue(), + LUCENE_NAME + ) + ); + } + } + + if (VectorDataType.BINARY == vectorDataType) { + if (KNNEngine.FAISS == methodContext.getKnnEngine()) { + if (METHOD_HNSW.equals(methodContext.getMethodComponentContext().getName())) { + return; + } else { + throw new IllegalArgumentException( + String.format( + Locale.ROOT, + "[%s] field with value [%s] is only supported for [%s] method", + VECTOR_DATA_TYPE_FIELD, + vectorDataType.getValue(), + METHOD_HNSW + ) + ); + } + } else { + throw new IllegalArgumentException( + String.format( + Locale.ROOT, + "[%s] field with value [%s] is only supported for [%s] engine", + VECTOR_DATA_TYPE_FIELD, + vectorDataType.getValue(), + FAISS_NAME + ) + ); + } + } + throw new IllegalArgumentException("This line should not be reached"); } /** @@ -131,10 +170,10 @@ public static void validateVectorDataTypeWithKnnIndexSetting( throw new IllegalArgumentException( String.format( Locale.ROOT, - "[%s] field with value [%s] is only supported for [%s] engine", + "[%s] field with value [%s] is not supported for [%s] engine", VECTOR_DATA_TYPE_FIELD, vectorDataType.getValue().getValue(), - LUCENE_NAME + NMSLIB_NAME ) ); } diff --git a/src/main/java/org/opensearch/knn/index/mapper/LuceneFieldMapper.java b/src/main/java/org/opensearch/knn/index/mapper/LuceneFieldMapper.java index 59f4867dd..b8ba688ff 100644 --- a/src/main/java/org/opensearch/knn/index/mapper/LuceneFieldMapper.java +++ b/src/main/java/org/opensearch/knn/index/mapper/LuceneFieldMapper.java @@ -49,7 +49,9 @@ public class LuceneFieldMapper extends KNNVectorFieldMapper { vectorDataType = input.getVectorDataType(); this.knnMethod = input.getKnnMethodContext(); - final VectorSimilarityFunction vectorSimilarityFunction = this.knnMethod.getSpaceType().getVectorSimilarityFunction(); + final VectorSimilarityFunction vectorSimilarityFunction = this.knnMethod.getSpaceType() + .getKnnVectorSimilarityFunction() + .getVectorSimilarityFunction(); final int dimension = input.getMappedFieldType().getDimension(); if (dimension > KNNEngine.getMaxDimensionByEngine(KNNEngine.LUCENE)) { diff --git a/src/main/java/org/opensearch/knn/index/mapper/MethodFieldMapper.java b/src/main/java/org/opensearch/knn/index/mapper/MethodFieldMapper.java index d2db7fb5a..f09ac1b4c 100644 --- a/src/main/java/org/opensearch/knn/index/mapper/MethodFieldMapper.java +++ b/src/main/java/org/opensearch/knn/index/mapper/MethodFieldMapper.java @@ -17,6 +17,7 @@ import static org.opensearch.knn.common.KNNConstants.KNN_ENGINE; import static org.opensearch.knn.common.KNNConstants.PARAMETERS; import static org.opensearch.knn.common.KNNConstants.SPACE_TYPE; +import static org.opensearch.knn.common.KNNConstants.VECTOR_DATA_TYPE_FIELD; /** * Field mapper for method definition in mapping @@ -51,6 +52,7 @@ public class MethodFieldMapper extends KNNVectorFieldMapper { this.fieldType.putAttribute(DIMENSION, String.valueOf(dimension)); this.fieldType.putAttribute(SPACE_TYPE, knnMethodContext.getSpaceType().getValue()); + this.fieldType.putAttribute(VECTOR_DATA_TYPE_FIELD, vectorDataType.getValue()); KNNEngine knnEngine = knnMethodContext.getKnnEngine(); this.fieldType.putAttribute(KNN_ENGINE, knnEngine.getName()); diff --git a/src/main/java/org/opensearch/knn/index/memory/NativeMemoryAllocation.java b/src/main/java/org/opensearch/knn/index/memory/NativeMemoryAllocation.java index c8b56436b..b108fb6f0 100644 --- a/src/main/java/org/opensearch/knn/index/memory/NativeMemoryAllocation.java +++ b/src/main/java/org/opensearch/knn/index/memory/NativeMemoryAllocation.java @@ -11,6 +11,7 @@ package org.opensearch.knn.index.memory; +import lombok.Getter; import org.apache.lucene.index.LeafReaderContext; import org.opensearch.knn.index.query.KNNWeight; import org.opensearch.knn.jni.JNICommons; @@ -87,12 +88,17 @@ class IndexAllocation implements NativeMemoryAllocation { private final long memoryAddress; private final int size; private volatile boolean closed; + @Getter private final KNNEngine knnEngine; + @Getter private final String indexPath; + @Getter private final String openSearchIndexName; private final ReadWriteLock readWriteLock; private final WatcherHandle watcherHandle; private final SharedIndexState sharedIndexState; + @Getter + private final boolean isBinaryIndex; /** * Constructor @@ -114,7 +120,7 @@ class IndexAllocation implements NativeMemoryAllocation { String openSearchIndexName, WatcherHandle watcherHandle ) { - this(executorService, memoryAddress, size, knnEngine, indexPath, openSearchIndexName, watcherHandle, null); + this(executorService, memoryAddress, size, knnEngine, indexPath, openSearchIndexName, watcherHandle, null, false); } /** @@ -137,7 +143,8 @@ class IndexAllocation implements NativeMemoryAllocation { String indexPath, String openSearchIndexName, WatcherHandle watcherHandle, - SharedIndexState sharedIndexState + SharedIndexState sharedIndexState, + boolean isBinaryIndex ) { this.executor = executorService; this.closed = false; @@ -149,6 +156,7 @@ class IndexAllocation implements NativeMemoryAllocation { this.size = size; this.watcherHandle = watcherHandle; this.sharedIndexState = sharedIndexState; + this.isBinaryIndex = isBinaryIndex; } @Override @@ -171,7 +179,7 @@ private void cleanup() { // memoryAddress is sometimes initialized to 0. If this is ever the case, freeing will surely fail. if (memoryAddress != 0) { - JNIService.free(memoryAddress, knnEngine); + JNIService.free(memoryAddress, knnEngine, isBinaryIndex); } if (sharedIndexState != null) { @@ -223,33 +231,6 @@ public void writeUnlock() { public int getSizeInKB() { return size; } - - /** - * Getter for k-NN Engine associated with this index allocation. - * - * @return KNNEngine associated with index allocation - */ - public KNNEngine getKnnEngine() { - return knnEngine; - } - - /** - * Getter for the path to the file from which the index was loaded. - * - * @return indexPath to index - */ - public String getIndexPath() { - return indexPath; - } - - /** - * Getter for the OpenSearch index associated with the native index. - * - * @return OpenSearch index name - */ - public String getOpenSearchIndexName() { - return openSearchIndexName; - } } /** diff --git a/src/main/java/org/opensearch/knn/index/memory/NativeMemoryLoadStrategy.java b/src/main/java/org/opensearch/knn/index/memory/NativeMemoryLoadStrategy.java index cb7dafdfc..3602dd3c0 100644 --- a/src/main/java/org/opensearch/knn/index/memory/NativeMemoryLoadStrategy.java +++ b/src/main/java/org/opensearch/knn/index/memory/NativeMemoryLoadStrategy.java @@ -113,7 +113,8 @@ public NativeMemoryAllocation.IndexAllocation load(NativeMemoryEntryContext.Inde indexPath.toString(), indexEntryContext.getOpenSearchIndexName(), watcherHandle, - sharedIndexState + sharedIndexState, + IndexUtil.isBinaryIndex(knnEngine, indexEntryContext.getParameters()) ); } diff --git a/src/main/java/org/opensearch/knn/index/query/KNNQuery.java b/src/main/java/org/opensearch/knn/index/query/KNNQuery.java index 4b875d9a8..1c4ef25e5 100644 --- a/src/main/java/org/opensearch/knn/index/query/KNNQuery.java +++ b/src/main/java/org/opensearch/knn/index/query/KNNQuery.java @@ -20,6 +20,7 @@ import org.apache.lucene.search.Weight; import org.apache.lucene.search.join.BitSetProducer; import org.opensearch.knn.index.KNNSettings; +import org.opensearch.knn.index.VectorDataType; import java.io.IOException; import java.util.Arrays; @@ -37,9 +38,13 @@ public class KNNQuery extends Query { private final String field; private final float[] queryVector; + @Getter + private final byte[] byteQueryVector; private int k; private Map methodParameters; private final String indexName; + @Getter + private final VectorDataType vectorDataType; @Setter private Query filterQuery; @@ -54,11 +59,7 @@ public KNNQuery( final String indexName, final BitSetProducer parentsFilter ) { - this.field = field; - this.queryVector = queryVector; - this.k = k; - this.indexName = indexName; - this.parentsFilter = parentsFilter; + this(field, queryVector, null, k, indexName, null, parentsFilter, VectorDataType.FLOAT); } public KNNQuery( @@ -68,13 +69,40 @@ public KNNQuery( final String indexName, final Query filterQuery, final BitSetProducer parentsFilter + ) { + this(field, queryVector, null, k, indexName, filterQuery, parentsFilter, VectorDataType.FLOAT); + } + + public KNNQuery( + final String field, + final byte[] byteQueryVector, + final int k, + final String indexName, + final Query filterQuery, + final BitSetProducer parentsFilter, + final VectorDataType vectorDataType + ) { + this(field, null, byteQueryVector, k, indexName, filterQuery, parentsFilter, vectorDataType); + } + + private KNNQuery( + final String field, + final float[] queryVector, + final byte[] byteQueryVector, + final int k, + final String indexName, + final Query filterQuery, + final BitSetProducer parentsFilter, + final VectorDataType vectorDataType ) { this.field = field; this.queryVector = queryVector; + this.byteQueryVector = byteQueryVector; this.k = k; this.indexName = indexName; this.filterQuery = filterQuery; this.parentsFilter = parentsFilter; + this.vectorDataType = vectorDataType; } /** @@ -86,10 +114,7 @@ public KNNQuery( * @param parentsFilter parent filter */ public KNNQuery(String field, float[] queryVector, String indexName, BitSetProducer parentsFilter) { - this.field = field; - this.queryVector = queryVector; - this.indexName = indexName; - this.parentsFilter = parentsFilter; + this(field, queryVector, null, 0, indexName, null, parentsFilter, VectorDataType.FLOAT); } /** @@ -191,6 +216,7 @@ private boolean equalsTo(KNNQuery other) { if (other == this) return true; return Objects.equals(field, other.field) && Arrays.equals(queryVector, other.queryVector) + && Arrays.equals(byteQueryVector, other.byteQueryVector) && Objects.equals(k, other.k) && Objects.equals(methodParameters, other.methodParameters) && Objects.equals(radius, other.radius) diff --git a/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java b/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java index bab0b2519..855bb9ac3 100644 --- a/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java +++ b/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java @@ -27,7 +27,6 @@ import org.opensearch.index.query.QueryShardContext; import org.opensearch.knn.common.KNNConstants; import org.opensearch.knn.index.IndexUtil; -import org.opensearch.knn.index.util.EngineSpecificMethodContext; import org.opensearch.knn.index.KNNMethodContext; import org.opensearch.knn.index.MethodComponentContext; import org.opensearch.knn.index.SpaceType; @@ -35,6 +34,7 @@ import org.opensearch.knn.index.VectorQueryType; import org.opensearch.knn.index.mapper.KNNVectorFieldMapper; import org.opensearch.knn.index.query.parser.MethodParametersParser; +import org.opensearch.knn.index.util.EngineSpecificMethodContext; import org.opensearch.knn.index.util.KNNEngine; import org.opensearch.knn.index.util.QueryContext; import org.opensearch.knn.indices.ModelDao; @@ -595,17 +595,25 @@ protected Query doToQuery(QueryShardContext context) { radius = knnEngine.scoreToRadialThreshold(this.minScore, spaceType); } - if (fieldDimension != vector.length) { + int vectorLength = VectorDataType.BINARY == vectorDataType ? vector.length * Byte.SIZE : vector.length; + if (fieldDimension != vectorLength) { throw new IllegalArgumentException( - String.format("Query vector has invalid dimension: %d. Dimension should be: %d", vector.length, fieldDimension) + String.format("Query vector has invalid dimension: %d. Dimension should be: %d", vectorLength, fieldDimension) ); } byte[] byteVector = new byte[0]; - if (VectorDataType.BYTE == vectorDataType) { + if (VectorDataType.BINARY == vectorDataType) { + byteVector = new byte[vector.length]; + for (int i = 0; i < vector.length; i++) { + validateByteVectorValue(vector[i], knnVectorFieldType.getVectorDataType()); + byteVector[i] = (byte) vector[i]; + } + spaceType.validateVector(byteVector); + } else if (VectorDataType.BYTE == vectorDataType) { byteVector = new byte[vector.length]; for (int i = 0; i < vector.length; i++) { - validateByteVectorValue(vector[i]); + validateByteVectorValue(vector[i], knnVectorFieldType.getVectorDataType()); byteVector[i] = (byte) vector[i]; } spaceType.validateVector(byteVector); @@ -627,7 +635,7 @@ protected Query doToQuery(QueryShardContext context) { .indexName(indexName) .fieldName(this.fieldName) .vector(VectorDataType.FLOAT == vectorDataType ? this.vector : null) - .byteVector(VectorDataType.BYTE == vectorDataType ? byteVector : null) + .byteVector(VectorDataType.BYTE == vectorDataType || VectorDataType.BINARY == vectorDataType ? byteVector : null) .vectorDataType(vectorDataType) .k(this.k) .methodParameters(this.methodParameters) diff --git a/src/main/java/org/opensearch/knn/index/query/KNNQueryFactory.java b/src/main/java/org/opensearch/knn/index/query/KNNQueryFactory.java index 36987c750..af7dad026 100644 --- a/src/main/java/org/opensearch/knn/index/query/KNNQueryFactory.java +++ b/src/main/java/org/opensearch/knn/index/query/KNNQueryFactory.java @@ -5,6 +5,7 @@ package org.opensearch.knn.index.query; +import com.google.common.annotations.VisibleForTesting; import lombok.extern.log4j.Log4j2; import org.apache.lucene.search.KnnByteVectorQuery; import org.apache.lucene.search.KnnFloatVectorQuery; @@ -30,6 +31,9 @@ public class KNNQueryFactory extends BaseQueryFactory { /** + * Note. This method should be used only for test. + * Should use {@link #create(CreateQueryRequest)} instead. + * * Creates a Lucene query for a particular engine. * * @param knnEngine Engine to create the query for @@ -39,6 +43,7 @@ public class KNNQueryFactory extends BaseQueryFactory { * @param k the number of nearest neighbors to return * @return Lucene Query */ + @VisibleForTesting public static Query create( KNNEngine knnEngine, String indexName, @@ -83,6 +88,7 @@ public static Query create(CreateQueryRequest createQueryRequest) { if (KNNEngine.getEnginesThatCreateCustomSegmentFiles().contains(createQueryRequest.getKnnEngine())) { final Query validatedFilterQuery = validateFilterQuerySupport(filterQuery, createQueryRequest.getKnnEngine()); + log.debug( "Creating custom k-NN query for index:{}, field:{}, k:{}, filterQuery:{}, efSearch:{}", indexName, @@ -91,15 +97,31 @@ public static Query create(CreateQueryRequest createQueryRequest) { validatedFilterQuery, methodParameters ); - return KNNQuery.builder() - .field(fieldName) - .queryVector(vector) - .indexName(indexName) - .parentsFilter(parentFilter) - .k(k) - .methodParameters(methodParameters) - .filterQuery(validatedFilterQuery) - .build(); + + switch (vectorDataType) { + case BINARY: + return KNNQuery.builder() + .field(fieldName) + .byteQueryVector(byteVector) + .indexName(indexName) + .parentsFilter(parentFilter) + .k(k) + .methodParameters(methodParameters) + .filterQuery(validatedFilterQuery) + .vectorDataType(vectorDataType) + .build(); + default: + return KNNQuery.builder() + .field(fieldName) + .queryVector(vector) + .indexName(indexName) + .parentsFilter(parentFilter) + .k(k) + .methodParameters(methodParameters) + .filterQuery(validatedFilterQuery) + .vectorDataType(vectorDataType) + .build(); + } } Integer requestEfSearch = null; diff --git a/src/main/java/org/opensearch/knn/index/query/KNNWeight.java b/src/main/java/org/opensearch/knn/index/query/KNNWeight.java index 09fca0a52..503bea42d 100644 --- a/src/main/java/org/opensearch/knn/index/query/KNNWeight.java +++ b/src/main/java/org/opensearch/knn/index/query/KNNWeight.java @@ -32,13 +32,16 @@ import org.opensearch.knn.common.KNNConstants; import org.opensearch.knn.index.KNNSettings; import org.opensearch.knn.index.SpaceType; +import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.memory.NativeMemoryAllocation; import org.opensearch.knn.index.memory.NativeMemoryCacheManager; import org.opensearch.knn.index.memory.NativeMemoryEntryContext; import org.opensearch.knn.index.memory.NativeMemoryLoadStrategy; +import org.opensearch.knn.index.query.filtered.FilteredIdsKNNByteIterator; import org.opensearch.knn.index.query.filtered.FilteredIdsKNNIterator; +import org.opensearch.knn.index.query.filtered.KNNIterator; +import org.opensearch.knn.index.query.filtered.NestedFilteredIdsKNNByteIterator; import org.opensearch.knn.index.query.filtered.NestedFilteredIdsKNNIterator; -import org.opensearch.knn.index.util.FieldInfoExtractor; import org.opensearch.knn.index.util.KNNEngine; import org.opensearch.knn.indices.ModelDao; import org.opensearch.knn.indices.ModelMetadata; @@ -61,6 +64,7 @@ import static org.opensearch.knn.common.KNNConstants.KNN_ENGINE; import static org.opensearch.knn.common.KNNConstants.MODEL_ID; import static org.opensearch.knn.common.KNNConstants.SPACE_TYPE; +import static org.opensearch.knn.common.KNNConstants.VECTOR_DATA_TYPE_FIELD; import static org.opensearch.knn.index.IndexUtil.getParametersAtLoading; import static org.opensearch.knn.plugin.stats.KNNCounter.GRAPH_QUERY_ERRORS; @@ -251,7 +255,7 @@ private Map doANNSearch(final LeafReaderContext context, final B spaceType, knnEngine, knnQuery.getIndexName(), - FieldInfoExtractor.getIndexDescription(fieldInfo) + VectorDataType.get(fieldInfo.attributes().getOrDefault(VECTOR_DATA_TYPE_FIELD, VectorDataType.FLOAT.getValue())) ), knnQuery.getIndexName(), modelId @@ -275,16 +279,29 @@ private Map doANNSearch(final LeafReaderContext context, final B } int[] parentIds = getParentIdsArray(context); if (knnQuery.getK() > 0) { - results = JNIService.queryIndex( - indexAllocation.getMemoryAddress(), - knnQuery.getQueryVector(), - knnQuery.getK(), - knnQuery.getMethodParameters(), - knnEngine, - filterIds, - filterType.getValue(), - parentIds - ); + if (knnQuery.getVectorDataType() == VectorDataType.BINARY) { + results = JNIService.queryBinaryIndex( + indexAllocation.getMemoryAddress(), + knnQuery.getByteQueryVector(), + knnQuery.getK(), + knnQuery.getMethodParameters(), + knnEngine, + filterIds, + filterType.getValue(), + parentIds + ); + } else { + results = JNIService.queryIndex( + indexAllocation.getMemoryAddress(), + knnQuery.getQueryVector(), + knnQuery.getK(), + knnQuery.getMethodParameters(), + knnEngine, + filterIds, + filterType.getValue(), + parentIds + ); + } } else { results = JNIService.radiusQueryIndex( indexAllocation.getMemoryAddress(), @@ -345,7 +362,7 @@ private Map doExactSearch(final LeafReaderContext leafReaderCont final HitQueue queue = new HitQueue(Math.min(this.knnQuery.getK(), cardinality), true); ScoreDoc topDoc = queue.top(); final Map docToScore = new HashMap<>(); - FilteredIdsKNNIterator iterator = getFilteredKNNIterator(leafReaderContext, filterIdsBitSet); + KNNIterator iterator = getFilteredKNNIterator(leafReaderContext, filterIdsBitSet); int docId; while ((docId = iterator.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) { if (iterator.score() > topDoc.score) { @@ -376,21 +393,32 @@ private Map doExactSearch(final LeafReaderContext leafReaderCont return Collections.emptyMap(); } - private FilteredIdsKNNIterator getFilteredKNNIterator(final LeafReaderContext leafReaderContext, final BitSet filterIdsBitSet) - throws IOException { + private KNNIterator getFilteredKNNIterator(final LeafReaderContext leafReaderContext, final BitSet filterIdsBitSet) throws IOException { final SegmentReader reader = (SegmentReader) FilterLeafReader.unwrap(leafReaderContext.reader()); final FieldInfo fieldInfo = reader.getFieldInfos().fieldInfo(knnQuery.getField()); final BinaryDocValues values = DocValues.getBinary(leafReaderContext.reader(), fieldInfo.getName()); final SpaceType spaceType = getSpaceType(fieldInfo); - return knnQuery.getParentsFilter() == null - ? new FilteredIdsKNNIterator(filterIdsBitSet, knnQuery.getQueryVector(), values, spaceType) - : new NestedFilteredIdsKNNIterator( - filterIdsBitSet, - knnQuery.getQueryVector(), - values, - spaceType, - knnQuery.getParentsFilter().getBitSet(leafReaderContext) - ); + if (VectorDataType.BINARY == knnQuery.getVectorDataType()) { + return knnQuery.getParentsFilter() == null + ? new FilteredIdsKNNByteIterator(filterIdsBitSet, knnQuery.getByteQueryVector(), values, spaceType) + : new NestedFilteredIdsKNNByteIterator( + filterIdsBitSet, + knnQuery.getByteQueryVector(), + values, + spaceType, + knnQuery.getParentsFilter().getBitSet(leafReaderContext) + ); + } else { + return knnQuery.getParentsFilter() == null + ? new FilteredIdsKNNIterator(filterIdsBitSet, knnQuery.getQueryVector(), values, spaceType) + : new NestedFilteredIdsKNNIterator( + filterIdsBitSet, + knnQuery.getQueryVector(), + values, + spaceType, + knnQuery.getParentsFilter().getBitSet(leafReaderContext) + ); + } } private Scorer convertSearchResponseToScorer(final Map docsToScore) throws IOException { diff --git a/src/main/java/org/opensearch/knn/index/query/filtered/FilteredIdsKNNByteIterator.java b/src/main/java/org/opensearch/knn/index/query/filtered/FilteredIdsKNNByteIterator.java new file mode 100644 index 000000000..815e621f6 --- /dev/null +++ b/src/main/java/org/opensearch/knn/index/query/filtered/FilteredIdsKNNByteIterator.java @@ -0,0 +1,79 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.query.filtered; + +import org.apache.lucene.index.BinaryDocValues; +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.util.BitSet; +import org.apache.lucene.util.BitSetIterator; +import org.apache.lucene.util.BytesRef; +import org.opensearch.knn.index.SpaceType; + +import java.io.ByteArrayInputStream; +import java.io.IOException; + +/** + * Inspired by DiversifyingChildrenFloatKnnVectorQuery in lucene + * https://github.com/apache/lucene/blob/7b8aece125aabff2823626d5b939abf4747f63a7/lucene/join/src/java/org/apache/lucene/search/join/DiversifyingChildrenFloatKnnVectorQuery.java#L162 + * + * The class is used in KNNWeight to score filtered KNN field by iterating filterIdsArray. + */ +public class FilteredIdsKNNByteIterator implements KNNIterator { + // Array of doc ids to iterate + protected final BitSet filterIdsBitSet; + protected final BitSetIterator bitSetIterator; + protected final byte[] queryVector; + protected final BinaryDocValues binaryDocValues; + protected final SpaceType spaceType; + protected float currentScore = Float.NEGATIVE_INFINITY; + protected int docId; + + public FilteredIdsKNNByteIterator( + final BitSet filterIdsBitSet, + final byte[] queryVector, + final BinaryDocValues binaryDocValues, + final SpaceType spaceType + ) { + this.filterIdsBitSet = filterIdsBitSet; + this.bitSetIterator = new BitSetIterator(filterIdsBitSet, filterIdsBitSet.length()); + this.queryVector = queryVector; + this.binaryDocValues = binaryDocValues; + this.spaceType = spaceType; + this.docId = bitSetIterator.nextDoc(); + } + + /** + * Advance to the next doc and update score value with score of the next doc. + * DocIdSetIterator.NO_MORE_DOCS is returned when there is no more docs + * + * @return next doc id + */ + @Override + public int nextDoc() throws IOException { + + if (docId == DocIdSetIterator.NO_MORE_DOCS) { + return DocIdSetIterator.NO_MORE_DOCS; + } + int doc = binaryDocValues.advance(docId); + currentScore = computeScore(); + docId = bitSetIterator.nextDoc(); + return doc; + } + + @Override + public float score() { + return currentScore; + } + + protected float computeScore() throws IOException { + final BytesRef value = binaryDocValues.binaryValue(); + final ByteArrayInputStream byteStream = new ByteArrayInputStream(value.bytes, value.offset, value.length); + final byte[] vector = byteStream.readAllBytes(); + // Calculates a similarity score between the two vectors with a specified function. Higher similarity + // scores correspond to closer vectors. + return spaceType.getKnnVectorSimilarityFunction().compare(queryVector, vector); + } +} diff --git a/src/main/java/org/opensearch/knn/index/query/filtered/FilteredIdsKNNIterator.java b/src/main/java/org/opensearch/knn/index/query/filtered/FilteredIdsKNNIterator.java index a53cb8d60..fb153989a 100644 --- a/src/main/java/org/opensearch/knn/index/query/filtered/FilteredIdsKNNIterator.java +++ b/src/main/java/org/opensearch/knn/index/query/filtered/FilteredIdsKNNIterator.java @@ -23,7 +23,7 @@ * * The class is used in KNNWeight to score filtered KNN field by iterating filterIdsArray. */ -public class FilteredIdsKNNIterator { +public class FilteredIdsKNNIterator implements KNNIterator { // Array of doc ids to iterate protected final BitSet filterIdsBitSet; protected final BitSetIterator bitSetIterator; @@ -53,6 +53,7 @@ public FilteredIdsKNNIterator( * * @return next doc id */ + @Override public int nextDoc() throws IOException { if (docId == DocIdSetIterator.NO_MORE_DOCS) { @@ -64,6 +65,7 @@ public int nextDoc() throws IOException { return doc; } + @Override public float score() { return currentScore; } @@ -75,6 +77,6 @@ protected float computeScore() throws IOException { final float[] vector = vectorSerializer.byteToFloatArray(byteStream); // Calculates a similarity score between the two vectors with a specified function. Higher similarity // scores correspond to closer vectors. - return spaceType.getVectorSimilarityFunction().compare(queryVector, vector); + return spaceType.getKnnVectorSimilarityFunction().compare(queryVector, vector); } } diff --git a/src/main/java/org/opensearch/knn/index/query/filtered/KNNIterator.java b/src/main/java/org/opensearch/knn/index/query/filtered/KNNIterator.java new file mode 100644 index 000000000..4a105975a --- /dev/null +++ b/src/main/java/org/opensearch/knn/index/query/filtered/KNNIterator.java @@ -0,0 +1,14 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.query.filtered; + +import java.io.IOException; + +public interface KNNIterator { + int nextDoc() throws IOException; + + float score(); +} diff --git a/src/main/java/org/opensearch/knn/index/query/filtered/NestedFilteredIdsKNNByteIterator.java b/src/main/java/org/opensearch/knn/index/query/filtered/NestedFilteredIdsKNNByteIterator.java new file mode 100644 index 000000000..80fba1e41 --- /dev/null +++ b/src/main/java/org/opensearch/knn/index/query/filtered/NestedFilteredIdsKNNByteIterator.java @@ -0,0 +1,61 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.query.filtered; + +import org.apache.lucene.index.BinaryDocValues; +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.util.BitSet; +import org.opensearch.knn.index.SpaceType; + +import java.io.IOException; + +/** + * This iterator iterates filterIdsArray to score. However, it dedupe docs per each parent doc + * of which ID is set in parentBitSet and only return best child doc with the highest score. + */ +public class NestedFilteredIdsKNNByteIterator extends FilteredIdsKNNByteIterator { + private final BitSet parentBitSet; + + public NestedFilteredIdsKNNByteIterator( + final BitSet filterIdsArray, + final byte[] queryVector, + final BinaryDocValues values, + final SpaceType spaceType, + final BitSet parentBitSet + ) { + super(filterIdsArray, queryVector, values, spaceType); + this.parentBitSet = parentBitSet; + } + + /** + * Advance to the next best child doc per parent and update score with the best score among child docs from the parent. + * DocIdSetIterator.NO_MORE_DOCS is returned when there is no more docs + * + * @return next best child doc id + */ + @Override + public int nextDoc() throws IOException { + if (docId == DocIdSetIterator.NO_MORE_DOCS) { + return DocIdSetIterator.NO_MORE_DOCS; + } + + currentScore = Float.NEGATIVE_INFINITY; + int currentParent = parentBitSet.nextSetBit(docId); + int bestChild = -1; + + while (docId != DocIdSetIterator.NO_MORE_DOCS && docId < currentParent) { + binaryDocValues.advance(docId); + float score = computeScore(); + if (score > currentScore) { + bestChild = docId; + currentScore = score; + } + docId = bitSetIterator.nextDoc(); + } + + return bestChild; + } +} diff --git a/src/main/java/org/opensearch/knn/index/util/Faiss.java b/src/main/java/org/opensearch/knn/index/util/Faiss.java index 7cf31ba3c..711c206f5 100644 --- a/src/main/java/org/opensearch/knn/index/util/Faiss.java +++ b/src/main/java/org/opensearch/knn/index/util/Faiss.java @@ -55,7 +55,8 @@ /** * Implements NativeLibrary for the faiss native library */ -class Faiss extends NativeLibrary { +public class Faiss extends NativeLibrary { + public static final String FAISS_BINARY_INDEX_DESCRIPTION_PREFIX = "B"; Map> scoreTransform; // TODO: Current version is not really current version. Instead, it encodes information in the file name @@ -246,7 +247,7 @@ class Faiss extends NativeLibrary { ).addParameter(METHOD_PARAMETER_M, "", "").addParameter(METHOD_ENCODER_PARAMETER, ",", "").build()) ) .build() - ).addSpaces(SpaceType.L2, SpaceType.INNER_PRODUCT).build(), + ).addSpaces(SpaceType.UNDEFINED, SpaceType.HAMMING_BIT, SpaceType.L2, SpaceType.INNER_PRODUCT).build(), METHOD_IVF, KNNMethod.Builder.builder( MethodComponent.Builder.builder(METHOD_IVF) @@ -304,7 +305,7 @@ class Faiss extends NativeLibrary { return ((4L * centroids * dimension) / BYTES_PER_KILOBYTES) + 1; }) .build() - ).addSpaces(SpaceType.L2, SpaceType.INNER_PRODUCT).build() + ).addSpaces(SpaceType.UNDEFINED, SpaceType.L2, SpaceType.INNER_PRODUCT).build() ); final static Faiss INSTANCE = new Faiss( diff --git a/src/main/java/org/opensearch/knn/index/util/FieldInfoExtractor.java b/src/main/java/org/opensearch/knn/index/util/FieldInfoExtractor.java deleted file mode 100644 index 5ad271969..000000000 --- a/src/main/java/org/opensearch/knn/index/util/FieldInfoExtractor.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.knn.index.util; - -import org.apache.lucene.index.FieldInfo; -import org.opensearch.common.xcontent.XContentHelper; -import org.opensearch.core.common.bytes.BytesArray; -import org.opensearch.core.xcontent.DeprecationHandler; -import org.opensearch.core.xcontent.MediaTypeRegistry; -import org.opensearch.core.xcontent.NamedXContentRegistry; -import org.opensearch.knn.common.KNNConstants; - -import java.io.IOException; - -import static org.opensearch.knn.common.KNNConstants.INDEX_DESCRIPTION_PARAMETER; - -/** - * Class having methods to extract a value from field info - */ -public class FieldInfoExtractor { - public static String getIndexDescription(FieldInfo fieldInfo) throws IOException { - String parameters = fieldInfo.attributes().get(KNNConstants.PARAMETERS); - if (parameters == null) { - return null; - } - - return (String) XContentHelper.createParser( - NamedXContentRegistry.EMPTY, - DeprecationHandler.THROW_UNSUPPORTED_OPERATION, - new BytesArray(parameters), - MediaTypeRegistry.getDefaultMediaType() - ).map().getOrDefault(INDEX_DESCRIPTION_PARAMETER, null); - } -} diff --git a/src/main/java/org/opensearch/knn/index/util/Lucene.java b/src/main/java/org/opensearch/knn/index/util/Lucene.java index d98775f94..b5bbfca75 100644 --- a/src/main/java/org/opensearch/knn/index/util/Lucene.java +++ b/src/main/java/org/opensearch/knn/index/util/Lucene.java @@ -45,7 +45,7 @@ public class Lucene extends JVMLibrary { ) ) .build() - ).addSpaces(SpaceType.L2, SpaceType.COSINESIMIL, SpaceType.INNER_PRODUCT).build() + ).addSpaces(SpaceType.UNDEFINED, SpaceType.L2, SpaceType.COSINESIMIL, SpaceType.INNER_PRODUCT).build() ); // Map that overrides the default distance translations for Lucene, check more details in knn documentation: diff --git a/src/main/java/org/opensearch/knn/index/util/Nmslib.java b/src/main/java/org/opensearch/knn/index/util/Nmslib.java index 7b18ed11d..a068901d3 100644 --- a/src/main/java/org/opensearch/knn/index/util/Nmslib.java +++ b/src/main/java/org/opensearch/knn/index/util/Nmslib.java @@ -47,7 +47,7 @@ class Nmslib extends NativeLibrary { ) ) .build() - ).addSpaces(SpaceType.L2, SpaceType.L1, SpaceType.LINF, SpaceType.COSINESIMIL, SpaceType.INNER_PRODUCT).build() + ).addSpaces(SpaceType.UNDEFINED, SpaceType.L2, SpaceType.L1, SpaceType.LINF, SpaceType.COSINESIMIL, SpaceType.INNER_PRODUCT).build() ); final static Nmslib INSTANCE = new Nmslib(METHODS, Collections.emptyMap(), CURRENT_VERSION, EXTENSION); diff --git a/src/main/java/org/opensearch/knn/jni/FaissService.java b/src/main/java/org/opensearch/knn/jni/FaissService.java index f718ce6d5..21de90765 100644 --- a/src/main/java/org/opensearch/knn/jni/FaissService.java +++ b/src/main/java/org/opensearch/knn/jni/FaissService.java @@ -202,10 +202,29 @@ public static native KNNQueryResult[] queryBinaryIndexWithFilter( int[] parentIds ); + /** + * Query a binary index with filter + * + * @param indexPointer pointer to index in memory + * @param queryVector vector to be used for query + * @param k neighbors to be returned + * @param filterIds list of doc ids to include in the query result + * @param parentIds list of parent doc ids when the knn field is a nested field + * @return KNNQueryResult array of k neighbors + */ + public static native KNNQueryResult[] queryBinaryIndexWithFilter( + long indexPointer, + byte[] queryVector, + int k, + long[] filterIds, + int filterIdsType, + int[] parentIds + ); + /** * Free native memory pointer */ - public static native void free(long indexPointer); + public static native void free(long indexPointer, boolean isBinary); /** * Deallocate memory of the shared index state diff --git a/src/main/java/org/opensearch/knn/jni/JNIService.java b/src/main/java/org/opensearch/knn/jni/JNIService.java index ed6a169c1..cefd0af53 100644 --- a/src/main/java/org/opensearch/knn/jni/JNIService.java +++ b/src/main/java/org/opensearch/knn/jni/JNIService.java @@ -13,7 +13,7 @@ import org.apache.commons.lang.ArrayUtils; import org.opensearch.common.Nullable; -import org.opensearch.knn.common.KNNConstants; +import org.opensearch.knn.index.IndexUtil; import org.opensearch.knn.index.query.KNNQueryResult; import org.opensearch.knn.index.util.KNNEngine; @@ -23,8 +23,6 @@ * Service to distribute requests to the proper engine jni service */ public class JNIService { - private static final String FAISS_BINARY_INDEX_PREFIX = "B"; - /** * Create an index for the native library. The memory occupied by the vectorsAddress will be freed up during the * function call. So Java layer doesn't need to free up the memory. This is not an ideal behavior because Java layer @@ -53,8 +51,7 @@ public static void createIndex( } if (KNNEngine.FAISS == knnEngine) { - if (parameters.get(KNNConstants.INDEX_DESCRIPTION_PARAMETER) != null - && parameters.get(KNNConstants.INDEX_DESCRIPTION_PARAMETER).toString().startsWith(FAISS_BINARY_INDEX_PREFIX)) { + if (IndexUtil.isBinaryIndex(knnEngine, parameters)) { FaissService.createBinaryIndex(ids, vectorsAddress, dim, indexPath, parameters); } else { FaissService.createIndex(ids, vectorsAddress, dim, indexPath, parameters); @@ -109,8 +106,7 @@ public static long loadIndex(String indexPath, Map parameters, K } if (KNNEngine.FAISS == knnEngine) { - if (parameters.get(KNNConstants.INDEX_DESCRIPTION_PARAMETER) != null - && parameters.get(KNNConstants.INDEX_DESCRIPTION_PARAMETER).toString().startsWith(FAISS_BINARY_INDEX_PREFIX)) { + if (IndexUtil.isBinaryIndex(knnEngine, parameters)) { return FaissService.loadBinaryIndex(indexPath); } else { return FaissService.loadIndex(indexPath); @@ -260,14 +256,25 @@ public static KNNQueryResult[] queryBinaryIndex( * @param indexPointer location to be freed * @param knnEngine engine to perform free */ - public static void free(long indexPointer, KNNEngine knnEngine) { + public static void free(final long indexPointer, final KNNEngine knnEngine) { + free(indexPointer, knnEngine, false); + } + + /** + * Free native memory pointer + * + * @param indexPointer location to be freed + * @param knnEngine engine to perform free + * @param isBinaryIndex indicate if it is binary index or not + */ + public static void free(final long indexPointer, final KNNEngine knnEngine, final boolean isBinaryIndex) { if (KNNEngine.NMSLIB == knnEngine) { NmslibService.free(indexPointer); return; } if (KNNEngine.FAISS == knnEngine) { - FaissService.free(indexPointer); + FaissService.free(indexPointer, isBinaryIndex); return; } diff --git a/src/main/java/org/opensearch/knn/plugin/rest/RestTrainModelHandler.java b/src/main/java/org/opensearch/knn/plugin/rest/RestTrainModelHandler.java index ebbd7fa9b..fb8ccc4ce 100644 --- a/src/main/java/org/opensearch/knn/plugin/rest/RestTrainModelHandler.java +++ b/src/main/java/org/opensearch/knn/plugin/rest/RestTrainModelHandler.java @@ -16,6 +16,7 @@ import org.opensearch.core.xcontent.XContentParser; import org.opensearch.index.mapper.NumberFieldMapper; import org.opensearch.knn.index.KNNMethodContext; +import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.indices.ModelUtil; import org.opensearch.knn.plugin.KNNPlugin; import org.opensearch.knn.plugin.transport.TrainingJobRouterAction; @@ -97,6 +98,9 @@ private TrainingModelRequest createTransportRequest(RestRequest restRequest) thr trainingField = parser.textOrNull(); } else if (KNN_METHOD.equals(fieldName) && ensureNotSet(fieldName, knnMethodContext)) { knnMethodContext = KNNMethodContext.parse(parser.map()); + if (SpaceType.UNDEFINED == knnMethodContext.getSpaceType()) { + knnMethodContext.setSpaceType(SpaceType.L2); + } } else if (DIMENSION.equals(fieldName) && ensureNotSet(fieldName, dimension)) { dimension = (Integer) NumberFieldMapper.NumberType.INTEGER.parse(parser.objectBytes(), false); } else if (MAX_VECTOR_COUNT_PARAMETER.equals(fieldName) && ensureNotSet(fieldName, maximumVectorCount)) { diff --git a/src/main/java/org/opensearch/knn/plugin/script/KNNScoringSpaceUtil.java b/src/main/java/org/opensearch/knn/plugin/script/KNNScoringSpaceUtil.java index 889780d7a..5f9efd3cb 100644 --- a/src/main/java/org/opensearch/knn/plugin/script/KNNScoringSpaceUtil.java +++ b/src/main/java/org/opensearch/knn/plugin/script/KNNScoringSpaceUtil.java @@ -116,7 +116,7 @@ public static float[] convertVectorToPrimitive(Object vector, VectorDataType vec for (int i = 0; i < primitiveVector.length; i++) { float value = tmp.get(i).floatValue(); if (VectorDataType.BYTE == vectorDataType) { - validateByteVectorValue(value); + validateByteVectorValue(value, vectorDataType); } primitiveVector[i] = value; } diff --git a/src/main/java/org/opensearch/knn/plugin/script/KNNScoringUtil.java b/src/main/java/org/opensearch/knn/plugin/script/KNNScoringUtil.java index 84e986faa..0bc21e3d9 100644 --- a/src/main/java/org/opensearch/knn/plugin/script/KNNScoringUtil.java +++ b/src/main/java/org/opensearch/knn/plugin/script/KNNScoringUtil.java @@ -59,7 +59,7 @@ private static float[] toFloat(List inputVector, VectorDataType vectorDa for (final Number val : inputVector) { float floatValue = val.floatValue(); if (VectorDataType.BYTE == vectorDataType) { - validateByteVectorValue(floatValue); + validateByteVectorValue(floatValue, vectorDataType); } value[index++] = floatValue; } @@ -195,6 +195,17 @@ public static float calculateHammingBit(Long queryLong, Long inputLong) { return Long.bitCount(queryLong ^ inputLong); } + /** + * This method calculates hamming distance between query vector + * + * @param queryVector query vector + * @param inputVector input vector + * @return hamming distance + */ + public static float calculateHammingBit(byte[] queryVector, byte[] inputVector) { + return VectorUtil.xorBitCount(queryVector, inputVector); + } + /** * This method calculates L1 distance between query vector * and input vector diff --git a/src/test/java/org/opensearch/knn/common/KNNValidationUtilTests.java b/src/test/java/org/opensearch/knn/common/KNNValidationUtilTests.java new file mode 100644 index 000000000..56e462fc1 --- /dev/null +++ b/src/test/java/org/opensearch/knn/common/KNNValidationUtilTests.java @@ -0,0 +1,41 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.common; + +import org.opensearch.knn.KNNTestCase; +import org.opensearch.knn.index.VectorDataType; + +import static org.hamcrest.Matchers.containsString; + +public class KNNValidationUtilTests extends KNNTestCase { + public void testValidateVectorDimension_whenBinary_thenVectorSizeShouldBeEightTimesLarger() { + int vectorLength = randomInt(100); + Exception ex = expectThrows( + IllegalArgumentException.class, + () -> KNNValidationUtil.validateVectorDimension(vectorLength, vectorLength, VectorDataType.BINARY) + ); + assertThat( + ex.getMessage(), + containsString("The dimension of the binary vector must be 8 times the length of the provided vector.") + ); + + // Expect no exception + KNNValidationUtil.validateVectorDimension(vectorLength * Byte.SIZE, vectorLength, VectorDataType.BINARY); + } + + public void testValidateVectorDimension_whenNonBinary_thenVectorSizeShouldBeSameAsDimension() { + int dimension = randomInt(100); + VectorDataType vectorDataType = randomInt(1) == 0 ? VectorDataType.FLOAT : VectorDataType.BYTE; + Exception ex = expectThrows( + IllegalArgumentException.class, + () -> KNNValidationUtil.validateVectorDimension(dimension, dimension + 1, vectorDataType) + ); + assertThat(ex.getMessage(), containsString("Vector dimension mismatch")); + + // Expect no exception + KNNValidationUtil.validateVectorDimension(dimension, dimension, vectorDataType); + } +} diff --git a/src/test/java/org/opensearch/knn/index/IndexUtilTests.java b/src/test/java/org/opensearch/knn/index/IndexUtilTests.java index e6c3e96ee..d500fc342 100644 --- a/src/test/java/org/opensearch/knn/index/IndexUtilTests.java +++ b/src/test/java/org/opensearch/knn/index/IndexUtilTests.java @@ -28,6 +28,7 @@ import org.opensearch.knn.indices.ModelMetadata; import org.opensearch.knn.jni.JNIService; +import java.util.HashMap; import java.util.Map; import java.util.Objects; @@ -39,6 +40,7 @@ import static org.mockito.Mockito.when; import static org.opensearch.knn.common.KNNConstants.HNSW_ALGO_EF_SEARCH; import static org.opensearch.knn.common.KNNConstants.SPACE_TYPE; +import static org.opensearch.knn.common.KNNConstants.VECTOR_DATA_TYPE_FIELD; import static org.opensearch.knn.index.IndexUtil.getParametersAtLoading; import static org.opensearch.knn.index.KNNSettings.KNN_ALGO_PARAM_EF_SEARCH; @@ -57,15 +59,17 @@ public void testGetLoadParameters() { SpaceType spaceType1 = SpaceType.COSINESIMIL; KNNEngine knnEngine1 = KNNEngine.FAISS; String indexName = "my-test-index"; - String indexDescription = "HNSW32Flat"; + VectorDataType vectorDataType1 = VectorDataType.FLOAT; - Map loadParameters = getParametersAtLoading(spaceType1, knnEngine1, indexName, indexDescription); + Map loadParameters = getParametersAtLoading(spaceType1, knnEngine1, indexName, vectorDataType1); assertEquals(2, loadParameters.size()); assertEquals(spaceType1.getValue(), loadParameters.get(SPACE_TYPE)); + assertEquals(vectorDataType1.getValue(), loadParameters.get(VECTOR_DATA_TYPE_FIELD)); // Test nmslib to ensure both space type and ef search are properly set SpaceType spaceType2 = SpaceType.L1; KNNEngine knnEngine2 = KNNEngine.NMSLIB; + VectorDataType vectorDataType2 = VectorDataType.BINARY; int efSearchValue = 413; // We use the constant for the setting here as opposed to the identifier of efSearch in nmslib jni @@ -85,10 +89,11 @@ public void testGetLoadParameters() { when(clusterService.state()).thenReturn(clusterState); KNNSettings.state().setClusterService(clusterService); - loadParameters = getParametersAtLoading(spaceType2, knnEngine2, indexName, null); - assertEquals(2, loadParameters.size()); + loadParameters = getParametersAtLoading(spaceType2, knnEngine2, indexName, vectorDataType2); + assertEquals(3, loadParameters.size()); assertEquals(spaceType2.getValue(), loadParameters.get(SPACE_TYPE)); assertEquals(efSearchValue, loadParameters.get(HNSW_ALGO_EF_SEARCH)); + assertEquals(vectorDataType2.getValue(), loadParameters.get(VECTOR_DATA_TYPE_FIELD)); } public void testValidateKnnField_NestedField() { @@ -242,4 +247,16 @@ public void testIsShareableStateContainedInIndex_whenJNIIsSharedIndexStateRequir KNNEngine knnEngine = KNNEngine.FAISS; assertTrue(IndexUtil.isSharedIndexStateRequired(knnEngine, modelId, TEST_INDEX_ADDRESS)); } + + public void testIsBinaryIndex_whenBinary_thenTrue() { + Map binaryIndexParams = new HashMap<>(); + binaryIndexParams.put(VECTOR_DATA_TYPE_FIELD, "binary"); + assertTrue(IndexUtil.isBinaryIndex(KNNEngine.FAISS, binaryIndexParams)); + } + + public void testIsBinaryIndex_whenNonBinary_thenFalse() { + Map nonBinaryIndexParams = new HashMap<>(); + nonBinaryIndexParams.put(VECTOR_DATA_TYPE_FIELD, "byte"); + assertFalse(IndexUtil.isBinaryIndex(KNNEngine.FAISS, nonBinaryIndexParams)); + } } diff --git a/src/test/java/org/opensearch/knn/index/KNNIndexShardTests.java b/src/test/java/org/opensearch/knn/index/KNNIndexShardTests.java index 42a59d26f..aaeee31e1 100644 --- a/src/test/java/org/opensearch/knn/index/KNNIndexShardTests.java +++ b/src/test/java/org/opensearch/knn/index/KNNIndexShardTests.java @@ -125,7 +125,7 @@ public void testGetEngineFileContexts() { String fileExt = ".test"; SpaceType spaceType = SpaceType.L2; String modelId = "test-model"; - String indexDescription = "test-description"; + VectorDataType vectorDataType = VectorDataType.FLOAT; Set includedFileNames = ImmutableSet.of( String.format("%s_111_%s%s", segmentName, fieldName, fileExt), @@ -152,7 +152,7 @@ public void testGetEngineFileContexts() { path, spaceType, modelId, - indexDescription + vectorDataType ); assertEquals(includedFileNames.size(), included.size()); diff --git a/src/test/java/org/opensearch/knn/index/KNNMethodContextTests.java b/src/test/java/org/opensearch/knn/index/KNNMethodContextTests.java index 1330e8da0..cb294bc3d 100644 --- a/src/test/java/org/opensearch/knn/index/KNNMethodContextTests.java +++ b/src/test/java/org/opensearch/knn/index/KNNMethodContextTests.java @@ -290,7 +290,7 @@ public void testParse_valid() throws IOException { KNNMethodContext knnMethodContext = KNNMethodContext.parse(in); assertEquals(KNNEngine.DEFAULT, knnMethodContext.getKnnEngine()); - assertEquals(SpaceType.DEFAULT, knnMethodContext.getSpaceType()); + assertEquals(SpaceType.UNDEFINED, knnMethodContext.getSpaceType()); assertEquals(methodName, knnMethodContext.getMethodComponentContext().getName()); assertTrue(knnMethodContext.getMethodComponentContext().getParameters().isEmpty()); diff --git a/src/test/java/org/opensearch/knn/index/KNNVectorSimilarityFunctionTests.java b/src/test/java/org/opensearch/knn/index/KNNVectorSimilarityFunctionTests.java new file mode 100644 index 000000000..691941dc3 --- /dev/null +++ b/src/test/java/org/opensearch/knn/index/KNNVectorSimilarityFunctionTests.java @@ -0,0 +1,57 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index; + +import junit.framework.TestCase; +import org.apache.lucene.index.VectorSimilarityFunction; +import org.opensearch.knn.plugin.script.KNNScoringUtil; + +import java.util.Set; + +import static org.apache.lucene.tests.util.LuceneTestCase.expectThrows; +import static org.opensearch.knn.index.KNNVectorSimilarityFunction.COSINE; +import static org.opensearch.knn.index.KNNVectorSimilarityFunction.DOT_PRODUCT; +import static org.opensearch.knn.index.KNNVectorSimilarityFunction.EUCLIDEAN; +import static org.opensearch.knn.index.KNNVectorSimilarityFunction.HAMMING; +import static org.opensearch.knn.index.KNNVectorSimilarityFunction.MAXIMUM_INNER_PRODUCT; + +public class KNNVectorSimilarityFunctionTests extends TestCase { + private static final Set FUNCTION_SET_BACKED_BY_LUCENE = Set.of( + EUCLIDEAN, + DOT_PRODUCT, + COSINE, + MAXIMUM_INNER_PRODUCT + ); + + public void testFunctions_whenBackedByLucene_thenSameAsLucene() { + float[] f1 = new float[] { 1.5f, 2.5f, 3.5f, 4.5f, 5.5f }; + float[] f2 = new float[] { 6.5f, 7.5f, 8.5f, 09.5f, 10.5f }; + byte[] b1 = new byte[] { 1, 2, 3 }; + byte[] b2 = new byte[] { 4, 5, 6 }; + for (KNNVectorSimilarityFunction function : KNNVectorSimilarityFunction.values()) { + if (FUNCTION_SET_BACKED_BY_LUCENE.contains(function) == false) { + continue; + } + assertEquals(VectorSimilarityFunction.valueOf(function.name()), function.getVectorSimilarityFunction()); + assertEquals(function.getVectorSimilarityFunction().compare(f1, f2), function.compare(f1, f2)); + assertEquals(function.getVectorSimilarityFunction().compare(b1, b2), function.compare(b1, b2)); + } + } + + public void testFunctions_whenHamming_thenFloatVectorIsNotSupported() { + float[] f1 = new float[] { 1.5f, 2.5f, 3.5f, 4.5f, 5.5f }; + float[] f2 = new float[] { 6.5f, 7.5f, 8.5f, 09.5f, 10.5f }; + + Exception ex = expectThrows(IllegalStateException.class, () -> HAMMING.compare(f1, f2)); + assertTrue(ex.getMessage().contains("not supported")); + } + + public void testFunctions_whenHamming_thenReturnCorrectScore() { + byte[] b1 = new byte[] { 1, 2, 3 }; + byte[] b2 = new byte[] { 4, 5, 6 }; + assertEquals(1.0f / (1 + KNNScoringUtil.calculateHammingBit(b1, b2)), HAMMING.compare(b1, b2)); + } +} diff --git a/src/test/java/org/opensearch/knn/index/LuceneEngineIT.java b/src/test/java/org/opensearch/knn/index/LuceneEngineIT.java index f5821bf07..0d2ae9e24 100644 --- a/src/test/java/org/opensearch/knn/index/LuceneEngineIT.java +++ b/src/test/java/org/opensearch/knn/index/LuceneEngineIT.java @@ -7,17 +7,16 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import org.apache.http.util.EntityUtils; import lombok.SneakyThrows; import org.apache.commons.lang.math.RandomUtils; -import org.apache.lucene.index.VectorSimilarityFunction; +import org.apache.http.util.EntityUtils; import org.apache.lucene.util.VectorUtil; import org.junit.After; import org.opensearch.client.Response; import org.opensearch.client.ResponseException; import org.opensearch.common.Nullable; -import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.index.query.QueryBuilders; import org.opensearch.knn.KNNRestTestCase; import org.opensearch.knn.KNNResult; @@ -64,14 +63,14 @@ public class LuceneEngineIT extends KNNRestTestCase { private static final float[][] TEST_QUERY_VECTORS = { { 1.0f, 1.0f, 1.0f }, { 2.0f, 2.0f, 2.0f }, { 3.0f, 3.0f, 3.0f } }; - private static final Map> VECTOR_SIMILARITY_TO_SCORE = ImmutableMap.of( - VectorSimilarityFunction.EUCLIDEAN, + private static final Map> VECTOR_SIMILARITY_TO_SCORE = ImmutableMap.of( + KNNVectorSimilarityFunction.EUCLIDEAN, (similarity) -> 1 / (1 + similarity), - VectorSimilarityFunction.DOT_PRODUCT, + KNNVectorSimilarityFunction.DOT_PRODUCT, (similarity) -> (1 + similarity) / 2, - VectorSimilarityFunction.COSINE, + KNNVectorSimilarityFunction.COSINE, (similarity) -> (1 + similarity) / 2, - VectorSimilarityFunction.MAXIMUM_INNER_PRODUCT, + KNNVectorSimilarityFunction.MAXIMUM_INNER_PRODUCT, (similarity) -> similarity <= 0 ? 1 / (1 - similarity) : similarity + 1 ); private static final String DIMENSION_FIELD_NAME = "dimension"; @@ -520,7 +519,7 @@ private void validateQueries(SpaceType spaceType, String fieldName, Map> expected = Map.of( + SpaceType.UNDEFINED, + Collections.emptySet(), + SpaceType.L2, + Set.of(VectorDataType.FLOAT, VectorDataType.BYTE), + SpaceType.COSINESIMIL, + Set.of(VectorDataType.FLOAT, VectorDataType.BYTE), + SpaceType.L1, + Set.of(VectorDataType.FLOAT, VectorDataType.BYTE), + SpaceType.LINF, + Set.of(VectorDataType.FLOAT, VectorDataType.BYTE), + SpaceType.INNER_PRODUCT, + Set.of(VectorDataType.FLOAT, VectorDataType.BYTE), + SpaceType.HAMMING_BIT, + Set.of(VectorDataType.BINARY) + ); + + for (SpaceType spaceType : SpaceType.values()) { + for (VectorDataType vectorDataType : VectorDataType.values()) { + if (expected.get(spaceType).isEmpty()) { + Exception ex = expectThrows(IllegalStateException.class, () -> spaceType.validateVectorDataType(vectorDataType)); + assertTrue(ex.getMessage().contains("Unsupported method")); + continue; + } + + if (expected.get(spaceType).contains(vectorDataType)) { + spaceType.validateVectorDataType(vectorDataType); + } else { + Exception ex = expectThrows(IllegalArgumentException.class, () -> spaceType.validateVectorDataType(vectorDataType)); + assertTrue(ex.getMessage().contains("is not supported")); + } + } + } + } } diff --git a/src/test/java/org/opensearch/knn/index/VectorDataTypeIT.java b/src/test/java/org/opensearch/knn/index/VectorDataTypeIT.java index 96486e707..2d6ffb1e4 100644 --- a/src/test/java/org/opensearch/knn/index/VectorDataTypeIT.java +++ b/src/test/java/org/opensearch/knn/index/VectorDataTypeIT.java @@ -34,6 +34,7 @@ import static org.opensearch.knn.common.KNNConstants.DIMENSION; import static org.opensearch.knn.common.KNNConstants.LUCENE_NAME; import static org.opensearch.knn.common.KNNConstants.METHOD_HNSW; +import static org.opensearch.knn.common.KNNConstants.NMSLIB_NAME; import static org.opensearch.knn.common.KNNConstants.VECTOR_DATA_TYPE_FIELD; import static org.opensearch.knn.index.VectorDataType.SUPPORTED_VECTOR_DATA_TYPES; @@ -282,10 +283,10 @@ public void testByteVectorDataTypeWithLegacyFieldMapperKnnIndexSetting() { .contains( String.format( Locale.ROOT, - "[%s] field with value [%s] is only supported for [%s] engine", + "[%s] field with value [%s] is not supported for [%s] engine", VECTOR_DATA_TYPE_FIELD, VectorDataType.BYTE.getValue(), - LUCENE_NAME + NMSLIB_NAME ) ) ); diff --git a/src/test/java/org/opensearch/knn/index/VectorDataTypeTests.java b/src/test/java/org/opensearch/knn/index/VectorDataTypeTests.java index 19270717d..8e6b6d7f7 100644 --- a/src/test/java/org/opensearch/knn/index/VectorDataTypeTests.java +++ b/src/test/java/org/opensearch/knn/index/VectorDataTypeTests.java @@ -13,8 +13,10 @@ import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.VectorSimilarityFunction; import org.apache.lucene.store.Directory; import org.apache.lucene.tests.analysis.MockAnalyzer; +import org.apache.lucene.util.BytesRef; import org.junit.Assert; import org.opensearch.knn.KNNTestCase; @@ -106,4 +108,17 @@ private void createKNNByteVectorDocument(Directory directory) throws IOException writer.commit(); writer.close(); } + + public void testCreateKnnVectorFieldType_whenBinary_thenException() { + Exception ex = expectThrows( + IllegalStateException.class, + () -> VectorDataType.BINARY.createKnnVectorFieldType(1, VectorSimilarityFunction.EUCLIDEAN) + ); + assertTrue(ex.getMessage().contains("Unsupported method")); + } + + public void testGetVectorFromBytesRef_whenBinary_thenException() { + Exception ex = expectThrows(IllegalStateException.class, () -> VectorDataType.BINARY.getVectorFromBytesRef(new BytesRef())); + assertTrue(ex.getMessage().contains("Unsupported method")); + } } diff --git a/src/test/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumerTests.java b/src/test/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumerTests.java index f7c9f3eb8..847cad04e 100644 --- a/src/test/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumerTests.java +++ b/src/test/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumerTests.java @@ -26,6 +26,7 @@ import org.opensearch.knn.KNNTestCase; import org.opensearch.knn.common.KNNConstants; import org.opensearch.knn.index.KNNMethodContext; +import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.mapper.KNNVectorFieldMapper; import org.opensearch.knn.index.MethodComponentContext; import org.opensearch.knn.index.SpaceType; @@ -62,7 +63,9 @@ import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_EF_SEARCH; import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_M; import static org.opensearch.knn.common.KNNConstants.MODEL_ID; +import static org.opensearch.knn.common.KNNConstants.VECTOR_DATA_TYPE_FIELD; import static org.opensearch.knn.index.KNNSettings.MODEL_CACHE_SIZE_LIMIT_SETTING; +import static org.opensearch.knn.index.codec.KNNCodecTestUtil.assertBinaryIndexLoadableByEngine; import static org.opensearch.knn.index.codec.KNNCodecTestUtil.assertFileInCorrectLocation; import static org.opensearch.knn.index.codec.KNNCodecTestUtil.assertLoadableByEngine; import static org.opensearch.knn.index.codec.KNNCodecTestUtil.assertValidFooter; @@ -146,7 +149,8 @@ public void testAddKNNBinaryField_noVectors() throws IOException { Long initialMergeSize = KNNGraphValue.MERGE_TOTAL_SIZE_IN_BYTES.getValue(); Long initialMergeDocs = KNNGraphValue.MERGE_TOTAL_DOCS.getValue(); KNN80DocValuesConsumer knn80DocValuesConsumer = new KNN80DocValuesConsumer(null, null); - knn80DocValuesConsumer.addKNNBinaryField(null, randomVectorDocValuesProducer, true, true); + FieldInfo fieldInfo = KNNCodecTestUtil.FieldInfoBuilder.builder("test-field").build(); + knn80DocValuesConsumer.addKNNBinaryField(fieldInfo, randomVectorDocValuesProducer, true, true); assertEquals(initialGraphIndexRequests, KNNCounter.GRAPH_INDEX_REQUESTS.getCount()); assertEquals(initialRefreshOperations, KNNGraphValue.REFRESH_TOTAL_OPERATIONS.getValue()); assertEquals(initialMergeOperations, KNNGraphValue.MERGE_TOTAL_OPERATIONS.getValue()); @@ -329,6 +333,69 @@ public void testAddKNNBinaryField_fromScratch_faissCurrent() throws IOException assertNotEquals(0, (long) KNNGraphValue.MERGE_TOTAL_SIZE_IN_BYTES.getValue()); } + public void testAddKNNBinaryField_whenFaissBinary_thenAdded() throws IOException { + String segmentName = String.format("test_segment%s", randomAlphaOfLength(4)); + int docsInSegment = 100; + String fieldName = String.format("test_field%s", randomAlphaOfLength(4)); + + KNNEngine knnEngine = KNNEngine.FAISS; + SpaceType spaceType = SpaceType.HAMMING_BIT; + VectorDataType dataType = VectorDataType.BINARY; + int dimension = 16; + + SegmentInfo segmentInfo = KNNCodecTestUtil.segmentInfoBuilder() + .directory(directory) + .segmentName(segmentName) + .docsInSegment(docsInSegment) + .codec(codec) + .build(); + + KNNMethodContext knnMethodContext = new KNNMethodContext( + knnEngine, + spaceType, + new MethodComponentContext(METHOD_HNSW, ImmutableMap.of(METHOD_PARAMETER_M, 16, METHOD_PARAMETER_EF_CONSTRUCTION, 512)) + ); + knnMethodContext.getMethodComponentContext().setIndexVersion(Version.CURRENT); + + String parameterString = XContentFactory.jsonBuilder().map(knnEngine.getMethodAsMap(knnMethodContext)).toString(); + + FieldInfo[] fieldInfoArray = new FieldInfo[] { + KNNCodecTestUtil.FieldInfoBuilder.builder(fieldName) + .addAttribute(KNNVectorFieldMapper.KNN_FIELD, "true") + .addAttribute(KNNConstants.KNN_ENGINE, knnEngine.getName()) + .addAttribute(KNNConstants.SPACE_TYPE, spaceType.getValue()) + .addAttribute(VECTOR_DATA_TYPE_FIELD, dataType.getValue()) + .addAttribute(KNNConstants.PARAMETERS, parameterString) + .build() }; + + FieldInfos fieldInfos = new FieldInfos(fieldInfoArray); + SegmentWriteState state = new SegmentWriteState(null, directory, segmentInfo, fieldInfos, null, IOContext.DEFAULT); + + long initialRefreshOperations = KNNGraphValue.REFRESH_TOTAL_OPERATIONS.getValue(); + long initialMergeOperations = KNNGraphValue.MERGE_TOTAL_OPERATIONS.getValue(); + + // Add documents to the field + KNN80DocValuesConsumer knn80DocValuesConsumer = new KNN80DocValuesConsumer(null, state); + RandomVectorDocValuesProducer randomVectorDocValuesProducer = new RandomVectorDocValuesProducer(docsInSegment, dimension); + knn80DocValuesConsumer.addKNNBinaryField(fieldInfoArray[0], randomVectorDocValuesProducer, true, true); + + // The document should be created in the correct location + String expectedFile = KNNCodecUtil.buildEngineFileName(segmentName, knnEngine.getVersion(), fieldName, knnEngine.getExtension()); + assertFileInCorrectLocation(state, expectedFile); + + // The footer should be valid + assertValidFooter(state.directory, expectedFile); + + // The document should be readable by faiss + assertBinaryIndexLoadableByEngine(state, expectedFile, knnEngine, spaceType, dimension, dataType); + + // The graph creation statistics should be updated + assertEquals(1 + initialRefreshOperations, (long) KNNGraphValue.REFRESH_TOTAL_OPERATIONS.getValue()); + assertEquals(1 + initialMergeOperations, (long) KNNGraphValue.MERGE_TOTAL_OPERATIONS.getValue()); + assertNotEquals(0, (long) KNNGraphValue.MERGE_TOTAL_DOCS.getValue()); + assertNotEquals(0, (long) KNNGraphValue.MERGE_TOTAL_SIZE_IN_BYTES.getValue()); + } + public void testAddKNNBinaryField_fromModel_faiss() throws IOException, ExecutionException, InterruptedException { // Generate a trained faiss model KNNEngine knnEngine = KNNEngine.FAISS; diff --git a/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestUtil.java b/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestUtil.java index 80e94caf8..6acdfec5d 100644 --- a/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestUtil.java +++ b/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestUtil.java @@ -33,6 +33,8 @@ import org.apache.lucene.util.StringHelper; import org.apache.lucene.util.Version; import java.util.Set; + +import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.query.KNNQueryResult; import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.codec.util.KNNVectorSerializer; @@ -49,7 +51,9 @@ import static com.carrotsearch.randomizedtesting.RandomizedTest.randomFloat; import static org.junit.Assert.assertTrue; +import static org.opensearch.knn.common.KNNConstants.INDEX_DESCRIPTION_PARAMETER; import static org.opensearch.knn.common.KNNConstants.SPACE_TYPE; +import static org.opensearch.knn.common.KNNConstants.VECTOR_DATA_TYPE_FIELD; import static org.opensearch.test.OpenSearchTestCase.randomByteArrayOfLength; public class KNNCodecTestUtil { @@ -343,6 +347,37 @@ public static void assertLoadableByEngine( JNIService.free(indexPtr, knnEngine); } + public static void assertBinaryIndexLoadableByEngine( + SegmentWriteState state, + String fileName, + KNNEngine knnEngine, + SpaceType spaceType, + int dimension, + VectorDataType vectorDataType + ) { + String filePath = Paths.get(((FSDirectory) (FilterDirectory.unwrap(state.directory))).getDirectory().toString(), fileName) + .toString(); + long indexPtr = JNIService.loadIndex( + filePath, + Maps.newHashMap( + ImmutableMap.of( + SPACE_TYPE, + spaceType.getValue(), + INDEX_DESCRIPTION_PARAMETER, + "BHNSW32", + VECTOR_DATA_TYPE_FIELD, + vectorDataType.getValue() + ) + ), + knnEngine + ); + int k = 2; + byte[] queryVector = new byte[dimension]; + KNNQueryResult[] results = JNIService.queryBinaryIndex(indexPtr, queryVector, k, null, knnEngine, null, 0, null); + assertTrue(results.length > 0); + JNIService.free(indexPtr, knnEngine); + } + public static float[][] getRandomVectors(int count, int dimension) { float[][] data = new float[count][dimension]; for (int i = 0; i < count; i++) { diff --git a/src/test/java/org/opensearch/knn/index/codec/transfer/VectorTransferByteTests.java b/src/test/java/org/opensearch/knn/index/codec/transfer/VectorTransferByteTests.java new file mode 100644 index 000000000..abcd89a0e --- /dev/null +++ b/src/test/java/org/opensearch/knn/index/codec/transfer/VectorTransferByteTests.java @@ -0,0 +1,56 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.codec.transfer; + +import junit.framework.TestCase; +import lombok.SneakyThrows; +import org.opensearch.knn.index.codec.util.SerializationMode; +import org.opensearch.knn.jni.JNICommons; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.util.Random; + +import static org.junit.Assert.assertNotEquals; + +public class VectorTransferByteTests extends TestCase { + @SneakyThrows + public void testTransfer_whenCalled_thenAdded() { + final ByteArrayInputStream bais1 = getByteArrayOfVectors(20); + final ByteArrayInputStream bais2 = getByteArrayOfVectors(20); + VectorTransferByte vectorTransfer = new VectorTransferByte(1000); + try { + vectorTransfer.init(2); + + vectorTransfer.transfer(bais1); + // flush is not called + assertEquals(0, vectorTransfer.getVectorAddress()); + + vectorTransfer.transfer(bais2); + // flush should be called + assertNotEquals(0, vectorTransfer.getVectorAddress()); + } finally { + if (vectorTransfer.getVectorAddress() != 0) { + JNICommons.freeVectorData(vectorTransfer.getVectorAddress()); + } + } + } + + @SneakyThrows + public void testSerializationMode_whenCalled_thenReturn() { + final ByteArrayInputStream bais = getByteArrayOfVectors(20); + VectorTransferByte vectorTransfer = new VectorTransferByte(1000); + + // Verify + assertEquals(SerializationMode.COLLECTIONS_OF_BYTES, vectorTransfer.getSerializationMode(bais)); + } + + private ByteArrayInputStream getByteArrayOfVectors(int vectorLength) throws IOException { + byte[] vector = new byte[vectorLength]; + new Random().nextBytes(vector); + return new ByteArrayInputStream(vector); + } +} diff --git a/src/test/java/org/opensearch/knn/index/codec/transfer/VectorTransferFloatTests.java b/src/test/java/org/opensearch/knn/index/codec/transfer/VectorTransferFloatTests.java new file mode 100644 index 000000000..1de513a0b --- /dev/null +++ b/src/test/java/org/opensearch/knn/index/codec/transfer/VectorTransferFloatTests.java @@ -0,0 +1,66 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.codec.transfer; + +import junit.framework.TestCase; +import lombok.SneakyThrows; +import org.opensearch.knn.index.codec.util.KNNVectorSerializerFactory; +import org.opensearch.knn.jni.JNICommons; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.util.Random; +import java.util.stream.IntStream; + +import static org.junit.Assert.assertNotEquals; + +public class VectorTransferFloatTests extends TestCase { + @SneakyThrows + public void testTransfer_whenCalled_thenAdded() { + final ByteArrayInputStream bais1 = getByteArrayOfVectors(20); + final ByteArrayInputStream bais2 = getByteArrayOfVectors(20); + VectorTransferFloat vectorTransfer = new VectorTransferFloat(1000); + try { + vectorTransfer.init(2); + + vectorTransfer.transfer(bais1); + // flush is not called + assertEquals(0, vectorTransfer.getVectorAddress()); + + vectorTransfer.transfer(bais2); + // flush should be called + assertNotEquals(0, vectorTransfer.getVectorAddress()); + } finally { + if (vectorTransfer.getVectorAddress() != 0) { + JNICommons.freeVectorData(vectorTransfer.getVectorAddress()); + } + } + } + + @SneakyThrows + public void testSerializationMode_whenCalled_thenReturn() { + final ByteArrayInputStream bais = getByteArrayOfVectors(20); + VectorTransferFloat vectorTransfer = new VectorTransferFloat(1000); + + // Verify + assertEquals(KNNVectorSerializerFactory.getSerializerModeFromStream(bais), vectorTransfer.getSerializationMode(bais)); + } + + private ByteArrayInputStream getByteArrayOfVectors(int vectorLength) throws IOException { + float[] vector = new float[vectorLength]; + IntStream.range(0, vectorLength).forEach(index -> vector[index] = new Random().nextFloat()); + + final ByteArrayOutputStream bas = new ByteArrayOutputStream(); + final DataOutputStream ds = new DataOutputStream(bas); + for (float f : vector) { + ds.writeFloat(f); + } + final byte[] vectorAsCollectionOfFloats = bas.toByteArray(); + return new ByteArrayInputStream(vectorAsCollectionOfFloats); + } +} diff --git a/src/test/java/org/opensearch/knn/index/codec/util/KNNCodecUtilTests.java b/src/test/java/org/opensearch/knn/index/codec/util/KNNCodecUtilTests.java new file mode 100644 index 000000000..04c1c038f --- /dev/null +++ b/src/test/java/org/opensearch/knn/index/codec/util/KNNCodecUtilTests.java @@ -0,0 +1,56 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.codec.util; + +import junit.framework.TestCase; +import lombok.SneakyThrows; +import org.apache.lucene.index.BinaryDocValues; +import org.apache.lucene.util.BytesRef; +import org.opensearch.knn.index.codec.transfer.VectorTransfer; + +import java.io.ByteArrayInputStream; +import java.util.Arrays; + +import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +public class KNNCodecUtilTests extends TestCase { + @SneakyThrows + public void testGetPair_whenCalled_thenReturn() { + long liveDocCount = 1l; + int[] docId = { 2 }; + long vectorAddress = 3l; + int dimension = 4; + BytesRef bytesRef = new BytesRef(); + + BinaryDocValues binaryDocValues = mock(BinaryDocValues.class); + when(binaryDocValues.cost()).thenReturn(liveDocCount); + when(binaryDocValues.nextDoc()).thenReturn(docId[0], NO_MORE_DOCS); + when(binaryDocValues.binaryValue()).thenReturn(bytesRef); + + VectorTransfer vectorTransfer = mock(VectorTransfer.class); + when(vectorTransfer.getSerializationMode(any(ByteArrayInputStream.class))).thenReturn(SerializationMode.COLLECTIONS_OF_BYTES); + when(vectorTransfer.getVectorAddress()).thenReturn(vectorAddress); + when(vectorTransfer.getDimension()).thenReturn(dimension); + + // Run + KNNCodecUtil.Pair pair = KNNCodecUtil.getPair(binaryDocValues, vectorTransfer); + + // Verify + verify(vectorTransfer).init(liveDocCount); + verify(vectorTransfer).getSerializationMode(any(ByteArrayInputStream.class)); + verify(vectorTransfer).transfer(any(ByteArrayInputStream.class)); + verify(vectorTransfer).close(); + + assertTrue(Arrays.equals(docId, pair.docs)); + assertEquals(vectorAddress, pair.getVectorAddress()); + assertEquals(dimension, pair.getDimension()); + assertEquals(SerializationMode.COLLECTIONS_OF_BYTES, pair.serializationMode); + } +} diff --git a/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java b/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java index a9b65878f..c3ddcf185 100644 --- a/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java +++ b/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java @@ -13,20 +13,20 @@ import org.apache.lucene.index.VectorEncoding; import org.apache.lucene.util.BytesRef; import org.mockito.Mockito; -import org.opensearch.common.Explicit; -import org.opensearch.index.mapper.FieldMapper; -import org.opensearch.index.mapper.ParseContext; -import org.opensearch.knn.KNNTestCase; import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.common.Explicit; import org.opensearch.common.ValidationException; import org.opensearch.common.settings.IndexScopedSettings; import org.opensearch.common.settings.Settings; -import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.index.IndexSettings; import org.opensearch.index.mapper.ContentPath; +import org.opensearch.index.mapper.FieldMapper; import org.opensearch.index.mapper.Mapper; import org.opensearch.index.mapper.MapperService; +import org.opensearch.index.mapper.ParseContext; +import org.opensearch.knn.KNNTestCase; import org.opensearch.knn.index.KNNMethodContext; import org.opensearch.knn.index.KNNSettings; import org.opensearch.knn.index.MethodComponentContext; @@ -47,11 +47,15 @@ import java.util.HashSet; import java.util.List; import java.util.Locale; +import java.util.Map; import java.util.Optional; import java.util.stream.Collectors; import static org.mockito.Mockito.doNothing; import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import static org.opensearch.Version.CURRENT; import static org.opensearch.knn.common.KNNConstants.DIMENSION; import static org.opensearch.knn.common.KNNConstants.ENCODER_SQ; import static org.opensearch.knn.common.KNNConstants.FAISS_SQ_ENCODER_FP16; @@ -60,6 +64,7 @@ import static org.opensearch.knn.common.KNNConstants.KNN_ENGINE; import static org.opensearch.knn.common.KNNConstants.KNN_METHOD; import static org.opensearch.knn.common.KNNConstants.LUCENE_NAME; +import static org.opensearch.knn.common.KNNConstants.METHOD_ENCODER_PARAMETER; import static org.opensearch.knn.common.KNNConstants.METHOD_HNSW; import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_EF_CONSTRUCTION; import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_M; @@ -68,10 +73,9 @@ import static org.opensearch.knn.common.KNNConstants.NAME; import static org.opensearch.knn.common.KNNConstants.NMSLIB_NAME; import static org.opensearch.knn.common.KNNConstants.PARAMETERS; -import static org.opensearch.Version.CURRENT; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; import static org.opensearch.knn.common.KNNConstants.VECTOR_DATA_TYPE_FIELD; +import static org.opensearch.knn.index.KNNSettings.KNN_INDEX; +import static org.opensearch.knn.index.VectorDataType.SUPPORTED_VECTOR_DATA_TYPES; import static org.opensearch.knn.index.mapper.KNNVectorFieldMapperUtil.clipVectorValueToFP16Range; import static org.opensearch.knn.index.mapper.KNNVectorFieldMapperUtil.validateFP16VectorValue; @@ -364,10 +368,6 @@ public void testTypeParser_parse_invalidVectorDataType() { String fieldName = "test-field-name-vec"; String indexName = "test-index-name-vec"; String vectorDataType = "invalid"; - String supportedTypes = String.join( - ",", - Arrays.stream((VectorDataType.values())).map(VectorDataType::getValue).collect(Collectors.toCollection(HashSet::new)) - ); Settings settings = Settings.builder().put(settings(CURRENT).build()).build(); @@ -402,7 +402,7 @@ public void testTypeParser_parse_invalidVectorDataType() { Locale.ROOT, "Invalid value provided for [%s] field. Supported values are [%s]", VECTOR_DATA_TYPE_FIELD, - supportedTypes + SUPPORTED_VECTOR_DATA_TYPES ), ex.getMessage() ); @@ -817,7 +817,8 @@ public void testLuceneFieldMapper_parseCreateField_docValues_withBytes() { when(parseContext.path()).thenReturn(contentPath); LuceneFieldMapper luceneFieldMapper = Mockito.spy(new LuceneFieldMapper(inputBuilder.build())); - doReturn(Optional.of(TEST_BYTE_VECTOR)).when(luceneFieldMapper).getBytesFromContext(parseContext, TEST_DIMENSION); + doReturn(Optional.of(TEST_BYTE_VECTOR)).when(luceneFieldMapper) + .getBytesFromContext(parseContext, TEST_DIMENSION, VectorDataType.BYTE); doNothing().when(luceneFieldMapper).validateIfCircuitBreakerIsNotTriggered(); doNothing().when(luceneFieldMapper).validateIfKNNPluginEnabled(); @@ -859,7 +860,8 @@ public void testLuceneFieldMapper_parseCreateField_docValues_withBytes() { inputBuilder.hasDocValues(false); luceneFieldMapper = Mockito.spy(new LuceneFieldMapper(inputBuilder.build())); - doReturn(Optional.of(TEST_BYTE_VECTOR)).when(luceneFieldMapper).getBytesFromContext(parseContext, TEST_DIMENSION); + doReturn(Optional.of(TEST_BYTE_VECTOR)).when(luceneFieldMapper) + .getBytesFromContext(parseContext, TEST_DIMENSION, VectorDataType.BYTE); doNothing().when(luceneFieldMapper).validateIfCircuitBreakerIsNotTriggered(); doNothing().when(luceneFieldMapper).validateIfKNNPluginEnabled(); @@ -918,6 +920,115 @@ public void testClipVectorValuetoFP16Range_succeed() { assertEquals(-65504.0f, clipVectorValueToFP16Range(-1000000.89f), 0.0f); } + public void testBuilder_whenBinaryFaissHNSW_thenValid() { + testBuilderWithBinaryDataType(KNNEngine.FAISS, SpaceType.UNDEFINED, METHOD_HNSW, 8, null); + } + + public void testBuilder_whenBinaryWithInvalidDimension_thenException() { + testBuilderWithBinaryDataType(KNNEngine.FAISS, SpaceType.UNDEFINED, METHOD_HNSW, 4, "should be multiply of 8"); + } + + public void testBuilder_whenBinaryFaissHNSWWithInvalidSpaceType_thenException() { + for (SpaceType spaceType : SpaceType.values()) { + if (SpaceType.UNDEFINED == spaceType || SpaceType.HAMMING_BIT == spaceType) { + continue; + } + testBuilderWithBinaryDataType(KNNEngine.FAISS, spaceType, METHOD_HNSW, 8, "is not supported"); + } + } + + public void testBuilder_whenBinaryNonFaiss_thenException() { + testBuilderWithBinaryDataType(KNNEngine.LUCENE, SpaceType.UNDEFINED, METHOD_HNSW, 8, "is only supported for"); + testBuilderWithBinaryDataType(KNNEngine.NMSLIB, SpaceType.UNDEFINED, METHOD_HNSW, 8, "is only supported for"); + } + + private void testBuilderWithBinaryDataType( + KNNEngine knnEngine, + SpaceType spaceType, + String method, + int dimension, + String expectedErrMsg + ) { + ModelDao modelDao = mock(ModelDao.class); + KNNVectorFieldMapper.Builder builder = new KNNVectorFieldMapper.Builder("test-field-name-1", modelDao, CURRENT); + + // Setup settings + Settings settings = Settings.builder().put(settings(CURRENT).build()).build(); + + builder.knnMethodContext.setValue( + new KNNMethodContext(knnEngine, spaceType, new MethodComponentContext(method, Collections.emptyMap())) + ); + builder.vectorDataType.setValue(VectorDataType.BINARY); + builder.dimension.setValue(dimension); + + Mapper.BuilderContext builderContext = new Mapper.BuilderContext(settings, new ContentPath()); + if (expectedErrMsg == null) { + KNNVectorFieldMapper knnVectorFieldMapper = builder.build(builderContext); + assertTrue(knnVectorFieldMapper instanceof MethodFieldMapper); + if (SpaceType.UNDEFINED == spaceType) { + assertEquals(SpaceType.HAMMING_BIT, knnVectorFieldMapper.fieldType().spaceType); + } + } else { + Exception ex = expectThrows(Exception.class, () -> builder.build(builderContext)); + assertTrue(ex.getMessage(), ex.getMessage().contains(expectedErrMsg)); + } + } + + public void testBuilder_whenBinaryFaissHNSWWithSQ_thenException() { + ModelDao modelDao = mock(ModelDao.class); + KNNVectorFieldMapper.Builder builder = new KNNVectorFieldMapper.Builder("test-field-name-1", modelDao, CURRENT); + + // Setup settings + Settings settings = Settings.builder().put(settings(CURRENT).build()).build(); + + builder.knnMethodContext.setValue( + new KNNMethodContext( + KNNEngine.FAISS, + SpaceType.HAMMING_BIT, + new MethodComponentContext( + METHOD_HNSW, + Map.of(METHOD_ENCODER_PARAMETER, new MethodComponentContext(ENCODER_SQ, Collections.emptyMap())) + ) + ) + ); + builder.vectorDataType.setValue(VectorDataType.BINARY); + builder.dimension.setValue(8); + + Mapper.BuilderContext builderContext = new Mapper.BuilderContext(settings, new ContentPath()); + Exception ex = expectThrows(Exception.class, () -> builder.build(builderContext)); + assertTrue(ex.getMessage(), ex.getMessage().contains("data type does not support")); + } + + public void testBuilder_whenBinaryWithLegacyKNNDisabled_thenValid() { + // Check legacy is picked up if model context and method context are not set + ModelDao modelDao = mock(ModelDao.class); + KNNVectorFieldMapper.Builder builder = new KNNVectorFieldMapper.Builder("test-field-name-1", modelDao, CURRENT); + builder.vectorDataType.setValue(VectorDataType.BINARY); + builder.dimension.setValue(8); + + // Setup settings + Settings settings = Settings.builder().put(settings(CURRENT).build()).put(KNN_INDEX, false).build(); + + Mapper.BuilderContext builderContext = new Mapper.BuilderContext(settings, new ContentPath()); + KNNVectorFieldMapper knnVectorFieldMapper = builder.build(builderContext); + assertTrue(knnVectorFieldMapper instanceof LegacyFieldMapper); + } + + public void testBuilder_whenBinaryWithLegacyKNNEnabled_thenException() { + // Check legacy is picked up if model context and method context are not set + ModelDao modelDao = mock(ModelDao.class); + KNNVectorFieldMapper.Builder builder = new KNNVectorFieldMapper.Builder("test-field-name-1", modelDao, CURRENT); + builder.vectorDataType.setValue(VectorDataType.BINARY); + builder.dimension.setValue(8); + + // Setup settings + Settings settings = Settings.builder().put(settings(CURRENT).build()).put(KNN_INDEX, true).build(); + + Mapper.BuilderContext builderContext = new Mapper.BuilderContext(settings, new ContentPath()); + Exception ex = expectThrows(Exception.class, () -> builder.build(builderContext)); + assertTrue(ex.getMessage(), ex.getMessage().contains("is not supported for")); + } + private LuceneFieldMapper.CreateLuceneFieldMapperInput.CreateLuceneFieldMapperInputBuilder createLuceneFieldMapperInputBuilder( VectorDataType vectorDataType ) { diff --git a/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperUtilTests.java b/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperUtilTests.java index ff47dcd69..c7c945bfa 100644 --- a/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperUtilTests.java +++ b/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperUtilTests.java @@ -13,16 +13,20 @@ import org.apache.lucene.document.StoredField; import org.opensearch.knn.KNNTestCase; +import org.opensearch.knn.common.KNNConstants; import org.opensearch.knn.index.KNNMethodContext; import org.opensearch.knn.index.MethodComponentContext; +import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.codec.util.KNNVectorSerializerFactory; +import org.opensearch.knn.index.util.KNNEngine; import org.opensearch.knn.indices.ModelDao; import org.opensearch.knn.indices.ModelMetadata; import org.opensearch.knn.indices.ModelState; import java.io.ByteArrayInputStream; import java.util.Arrays; +import java.util.Collections; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -114,4 +118,52 @@ public void testGetExpectedDimensionsFailure() { ); assertEquals(String.format("Field '%s' does not have model.", fieldName), e.getMessage()); } + + public void testValidateVectorDataType_whenBinaryFaissHNSW_thenValid() { + validateValidateVectorDataType(KNNEngine.FAISS, KNNConstants.METHOD_HNSW, VectorDataType.BINARY, null); + } + + public void testValidateVectorDataType_whenBinaryNonFaiss_thenException() { + validateValidateVectorDataType(KNNEngine.LUCENE, KNNConstants.METHOD_HNSW, VectorDataType.BINARY, "only supported"); + validateValidateVectorDataType(KNNEngine.NMSLIB, KNNConstants.METHOD_HNSW, VectorDataType.BINARY, "only supported"); + } + + public void testValidateVectorDataType_whenBinaryFaissIVF_thenException() { + validateValidateVectorDataType(KNNEngine.FAISS, KNNConstants.METHOD_IVF, VectorDataType.BINARY, "only supported"); + } + + public void testValidateVectorDataType_whenByteLucene_thenValid() { + validateValidateVectorDataType(KNNEngine.LUCENE, KNNConstants.METHOD_HNSW, VectorDataType.BYTE, null); + validateValidateVectorDataType(KNNEngine.LUCENE, KNNConstants.METHOD_IVF, VectorDataType.BYTE, null); + } + + public void testValidateVectorDataType_whenByteNonLucene_thenException() { + validateValidateVectorDataType(KNNEngine.FAISS, KNNConstants.METHOD_HNSW, VectorDataType.BYTE, "only supported"); + validateValidateVectorDataType(KNNEngine.NMSLIB, KNNConstants.METHOD_IVF, VectorDataType.BYTE, "only supported"); + } + + public void testValidateVectorDataType_whenFloat_thenValid() { + validateValidateVectorDataType(KNNEngine.FAISS, KNNConstants.METHOD_HNSW, VectorDataType.FLOAT, null); + validateValidateVectorDataType(KNNEngine.LUCENE, KNNConstants.METHOD_HNSW, VectorDataType.FLOAT, null); + validateValidateVectorDataType(KNNEngine.NMSLIB, KNNConstants.METHOD_HNSW, VectorDataType.FLOAT, null); + } + + private void validateValidateVectorDataType( + final KNNEngine knnEngine, + final String methodName, + final VectorDataType vectorDataType, + final String expectedErrMsg + ) { + MethodComponentContext methodComponentContext = new MethodComponentContext(methodName, Collections.emptyMap()); + KNNMethodContext methodContext = new KNNMethodContext(knnEngine, SpaceType.UNDEFINED, methodComponentContext); + if (expectedErrMsg == null) { + KNNVectorFieldMapperUtil.validateVectorDataType(methodContext, vectorDataType); + } else { + Exception ex = expectThrows( + IllegalArgumentException.class, + () -> KNNVectorFieldMapperUtil.validateVectorDataType(methodContext, vectorDataType) + ); + assertTrue(ex.getMessage().contains(expectedErrMsg)); + } + } } diff --git a/src/test/java/org/opensearch/knn/index/mapper/MethodFieldMapperTests.java b/src/test/java/org/opensearch/knn/index/mapper/MethodFieldMapperTests.java new file mode 100644 index 000000000..ef2a2768e --- /dev/null +++ b/src/test/java/org/opensearch/knn/index/mapper/MethodFieldMapperTests.java @@ -0,0 +1,37 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.mapper; + +import junit.framework.TestCase; +import org.opensearch.index.mapper.FieldMapper; +import org.opensearch.knn.index.KNNMethodContext; +import org.opensearch.knn.index.SpaceType; +import org.opensearch.knn.index.VectorDataType; + +import java.util.Collections; + +public class MethodFieldMapperTests extends TestCase { + public void testMethodFieldMapper_whenVectorDataTypeIsGiven_thenSetItInFieldType() { + KNNVectorFieldMapper.KNNVectorFieldType mappedFieldType = new KNNVectorFieldMapper.KNNVectorFieldType( + "testField", + Collections.emptyMap(), + 1, + VectorDataType.BINARY, + SpaceType.HAMMING_BIT + ); + MethodFieldMapper mappers = new MethodFieldMapper( + "simpleName", + mappedFieldType, + null, + new FieldMapper.CopyTo.Builder().build(), + KNNVectorFieldMapper.Defaults.IGNORE_MALFORMED, + true, + true, + KNNMethodContext.getDefault() + ); + assertEquals(VectorDataType.BINARY, mappers.fieldType().vectorDataType); + } +} diff --git a/src/test/java/org/opensearch/knn/index/memory/NativeMemoryAllocationTests.java b/src/test/java/org/opensearch/knn/index/memory/NativeMemoryAllocationTests.java index 7573a4394..ab73c3946 100644 --- a/src/test/java/org/opensearch/knn/index/memory/NativeMemoryAllocationTests.java +++ b/src/test/java/org/opensearch/knn/index/memory/NativeMemoryAllocationTests.java @@ -12,9 +12,11 @@ package org.opensearch.knn.index.memory; import com.google.common.collect.ImmutableMap; +import lombok.SneakyThrows; import org.opensearch.knn.KNNTestCase; import org.opensearch.knn.common.KNNConstants; import org.opensearch.knn.index.IndexUtil; +import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.jni.JNICommons; import org.opensearch.knn.jni.JNIService; import org.opensearch.knn.index.SpaceType; @@ -91,6 +93,69 @@ public void testIndexAllocation_close() throws InterruptedException { executorService.shutdown(); } + @SneakyThrows + public void testClose_whenBinaryFiass_thenSuccess() { + Path dir = createTempDir(); + KNNEngine knnEngine = KNNEngine.FAISS; + String indexName = "test1" + knnEngine.getExtension(); + String path = dir.resolve(indexName).toAbsolutePath().toString(); + int numVectors = 10; + int dimension = 8; + int dataLength = dimension / 8; + int[] ids = new int[numVectors]; + byte[][] vectors = new byte[numVectors][dataLength]; + for (int i = 0; i < numVectors; i++) { + ids[i] = i; + vectors[i][0] = 1; + } + Map parameters = ImmutableMap.of( + KNNConstants.SPACE_TYPE, + SpaceType.HAMMING_BIT.getValue(), + KNNConstants.INDEX_DESCRIPTION_PARAMETER, + "BHNSW32", + KNNConstants.VECTOR_DATA_TYPE_FIELD, + VectorDataType.BINARY.getValue() + ); + long vectorMemoryAddress = JNICommons.storeByteVectorData(0, vectors, numVectors * dataLength); + JNIService.createIndex(ids, vectorMemoryAddress, dimension, path, parameters, knnEngine); + + // Load index into memory + long memoryAddress = JNIService.loadIndex(path, parameters, knnEngine); + + @SuppressWarnings("unchecked") + WatcherHandle watcherHandle = (WatcherHandle) mock(WatcherHandle.class); + doNothing().when(watcherHandle).stop(); + + ExecutorService executorService = Executors.newSingleThreadExecutor(); + NativeMemoryAllocation.IndexAllocation indexAllocation = new NativeMemoryAllocation.IndexAllocation( + executorService, + memoryAddress, + IndexUtil.getFileSizeInKB(path), + knnEngine, + path, + "test", + watcherHandle, + null, + true + ); + + indexAllocation.close(); + + Thread.sleep(1000 * 2); + indexAllocation.writeLock(); + assertTrue(indexAllocation.isClosed()); + indexAllocation.writeUnlock(); + + indexAllocation.close(); + + Thread.sleep(1000 * 2); + indexAllocation.writeLock(); + assertTrue(indexAllocation.isClosed()); + indexAllocation.writeUnlock(); + + executorService.shutdown(); + } + public void testIndexAllocation_getMemoryAddress() { long memoryAddress = 12; NativeMemoryAllocation.IndexAllocation indexAllocation = new NativeMemoryAllocation.IndexAllocation( diff --git a/src/test/java/org/opensearch/knn/index/memory/NativeMemoryLoadStrategyTests.java b/src/test/java/org/opensearch/knn/index/memory/NativeMemoryLoadStrategyTests.java index 876303523..43ad7e968 100644 --- a/src/test/java/org/opensearch/knn/index/memory/NativeMemoryLoadStrategyTests.java +++ b/src/test/java/org/opensearch/knn/index/memory/NativeMemoryLoadStrategyTests.java @@ -16,6 +16,7 @@ import org.opensearch.action.search.SearchResponse; import org.opensearch.knn.KNNTestCase; import org.opensearch.knn.common.KNNConstants; +import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.jni.JNICommons; import org.opensearch.knn.jni.JNIService; import org.opensearch.knn.index.query.KNNQueryResult; @@ -80,6 +81,65 @@ public void testIndexLoadStrategy_load() throws IOException { assertTrue(results.length > 0); } + public void testLoad_whenFaissBinary_thenSuccess() throws IOException { + Path dir = createTempDir(); + KNNEngine knnEngine = KNNEngine.FAISS; + String indexName = "test1" + knnEngine.getExtension(); + String path = dir.resolve(indexName).toAbsolutePath().toString(); + int numVectors = 10; + int dimension = 8; + int dataLength = dimension / 8; + int[] ids = new int[numVectors]; + byte[][] vectors = new byte[numVectors][dataLength]; + for (int i = 0; i < numVectors; i++) { + ids[i] = i; + vectors[i][0] = 1; + } + Map parameters = ImmutableMap.of( + KNNConstants.SPACE_TYPE, + SpaceType.HAMMING_BIT.getValue(), + KNNConstants.INDEX_DESCRIPTION_PARAMETER, + "BHNSW32", + KNNConstants.VECTOR_DATA_TYPE_FIELD, + VectorDataType.BINARY.getValue() + ); + long memoryAddress = JNICommons.storeByteVectorData(0, vectors, numVectors); + JNIService.createIndex(ids, memoryAddress, dimension, path, parameters, knnEngine); + + // Setup mock resource manager + ResourceWatcherService resourceWatcherService = mock(ResourceWatcherService.class); + doReturn(null).when(resourceWatcherService).add(any()); + NativeMemoryLoadStrategy.IndexLoadStrategy.initialize(resourceWatcherService); + + NativeMemoryEntryContext.IndexEntryContext indexEntryContext = new NativeMemoryEntryContext.IndexEntryContext( + path, + NativeMemoryLoadStrategy.IndexLoadStrategy.getInstance(), + parameters, + "test" + ); + + // Load + NativeMemoryAllocation.IndexAllocation indexAllocation = NativeMemoryLoadStrategy.IndexLoadStrategy.getInstance() + .load(indexEntryContext); + + // Verify + assertTrue(indexAllocation.isBinaryIndex()); + + // Confirm that the file was loaded by querying + byte[] query = { 1 }; + KNNQueryResult[] results = JNIService.queryBinaryIndex( + indexAllocation.getMemoryAddress(), + query, + 2, + null, + knnEngine, + null, + 0, + null + ); + assertTrue(results.length > 0); + } + @SuppressWarnings("unchecked") public void testTrainingLoadStrategy_load() { // Mock the vector reader so that on read, it waits 2 seconds, transfers vectors to the consumer, and then calls diff --git a/src/test/java/org/opensearch/knn/index/query/KNNQueryBuilderTests.java b/src/test/java/org/opensearch/knn/index/query/KNNQueryBuilderTests.java index ca191b19a..ee498e5b7 100644 --- a/src/test/java/org/opensearch/knn/index/query/KNNQueryBuilderTests.java +++ b/src/test/java/org/opensearch/knn/index/query/KNNQueryBuilderTests.java @@ -844,6 +844,7 @@ public void testDoToQuery_WhenknnQueryWithFilterAndFaissEngine_thenSuccess() { org.opensearch.knn.common.KNNConstants.METHOD_HNSW, ImmutableMap.of() ); + when(mockKNNVectorField.getVectorDataType()).thenReturn(VectorDataType.FLOAT); KNNMethodContext knnMethodContext = new KNNMethodContext(KNNEngine.FAISS, SpaceType.L2, methodComponentContext); when(mockKNNVectorField.getKnnMethodContext()).thenReturn(knnMethodContext); when(mockQueryShardContext.fieldMapper(anyString())).thenReturn(mockKNNVectorField); @@ -900,6 +901,7 @@ public void testDoToQuery_whenknnQueryWithFilterAndNmsLibEngine_thenException() org.opensearch.knn.common.KNNConstants.METHOD_HNSW, ImmutableMap.of() ); + when(mockKNNVectorField.getVectorDataType()).thenReturn(VectorDataType.FLOAT); KNNMethodContext knnMethodContext = new KNNMethodContext(KNNEngine.NMSLIB, SpaceType.L2, methodComponentContext); when(mockKNNVectorField.getKnnMethodContext()).thenReturn(knnMethodContext); when(mockQueryShardContext.fieldMapper(anyString())).thenReturn(mockKNNVectorField); @@ -1200,6 +1202,7 @@ public void testRadialSearch_whenUnsupportedEngine_thenThrowException() { when(mockKNNVectorField.getKnnMethodContext()).thenReturn(knnMethodContext); when(mockQueryShardContext.index()).thenReturn(dummyIndex); when(mockKNNVectorField.getDimension()).thenReturn(4); + when(mockKNNVectorField.getVectorDataType()).thenReturn(VectorDataType.FLOAT); when(mockQueryShardContext.fieldMapper(anyString())).thenReturn(mockKNNVectorField); expectThrows(UnsupportedOperationException.class, () -> knnQueryBuilder.doToQuery(mockQueryShardContext)); @@ -1259,4 +1262,36 @@ public void testRadialSearch_whenEfSearchIsSet_whenFaissEngine_thenSuccess() { KNNQuery query = (KNNQuery) knnQueryBuilder.doToQuery(mockQueryShardContext); assertEquals(1 / MIN_SCORE - 1, query.getRadius(), 0); } + + public void testDoToQuery_whenBinary_thenValid() throws Exception { + float[] queryVector = { 1.0f, 2.0f, 3.0f, 4.0f }; + byte[] expectedQueryVector = { 1, 2, 3, 4 }; + KNNQueryBuilder knnQueryBuilder = new KNNQueryBuilder(FIELD_NAME, queryVector, K); + Index dummyIndex = new Index("dummy", "dummy"); + QueryShardContext mockQueryShardContext = mock(QueryShardContext.class); + KNNVectorFieldMapper.KNNVectorFieldType mockKNNVectorField = mock(KNNVectorFieldMapper.KNNVectorFieldType.class); + when(mockQueryShardContext.index()).thenReturn(dummyIndex); + when(mockKNNVectorField.getDimension()).thenReturn(32); + when(mockKNNVectorField.getVectorDataType()).thenReturn(VectorDataType.BINARY); + when(mockKNNVectorField.getSpaceType()).thenReturn(SpaceType.HAMMING_BIT); + when(mockQueryShardContext.fieldMapper(anyString())).thenReturn(mockKNNVectorField); + KNNQuery query = (KNNQuery) knnQueryBuilder.doToQuery(mockQueryShardContext); + assertArrayEquals(expectedQueryVector, query.getByteQueryVector()); + assertNull(query.getQueryVector()); + } + + public void testDoToQuery_whenBinaryWithInvalidDimension_thenException() throws Exception { + float[] queryVector = { 1.0f, 2.0f, 3.0f, 4.0f }; + KNNQueryBuilder knnQueryBuilder = new KNNQueryBuilder(FIELD_NAME, queryVector, K); + Index dummyIndex = new Index("dummy", "dummy"); + QueryShardContext mockQueryShardContext = mock(QueryShardContext.class); + KNNVectorFieldMapper.KNNVectorFieldType mockKNNVectorField = mock(KNNVectorFieldMapper.KNNVectorFieldType.class); + when(mockQueryShardContext.index()).thenReturn(dummyIndex); + when(mockKNNVectorField.getDimension()).thenReturn(8); + when(mockKNNVectorField.getVectorDataType()).thenReturn(VectorDataType.BINARY); + when(mockKNNVectorField.getSpaceType()).thenReturn(SpaceType.HAMMING_BIT); + when(mockQueryShardContext.fieldMapper(anyString())).thenReturn(mockKNNVectorField); + Exception ex = expectThrows(IllegalArgumentException.class, () -> knnQueryBuilder.doToQuery(mockQueryShardContext)); + assertTrue(ex.getMessage(), ex.getMessage().contains("invalid dimension")); + } } diff --git a/src/test/java/org/opensearch/knn/index/query/KNNQueryFactoryTests.java b/src/test/java/org/opensearch/knn/index/query/KNNQueryFactoryTests.java index 56e81d237..02b64cba5 100644 --- a/src/test/java/org/opensearch/knn/index/query/KNNQueryFactoryTests.java +++ b/src/test/java/org/opensearch/knn/index/query/KNNQueryFactoryTests.java @@ -216,6 +216,7 @@ public void testCreateFaissQueryWithFilter_withValidValues_thenSuccess() { .queryVector(testQueryVector) .k(testK) .methodParameters(methodParameters) + .vectorDataType(VectorDataType.FLOAT) .build(); // When @@ -226,6 +227,7 @@ public void testCreateFaissQueryWithFilter_withValidValues_thenSuccess() { .vector(testQueryVector) .k(testK) .methodParameters(methodParameters) + .vectorDataType(VectorDataType.FLOAT) .context(mockQueryShardContext) .filter(FILTER_QUERY_BUILDER) .build(); @@ -259,6 +261,7 @@ public void testCreateFaissQueryWithFilter_withValidValues_nullEfSearch_thenSucc .fieldName(testFieldName) .vector(testQueryVector) .k(testK) + .vectorDataType(VectorDataType.FLOAT) .context(mockQueryShardContext) .filter(FILTER_QUERY_BUILDER) .build(); @@ -294,6 +297,7 @@ public void testCreate_whenNestedVectorFiledAndNonNestedFilterField_thenReturnTo .fieldName(testFieldName) .vector(testQueryVector) .k(testK) + .vectorDataType(VectorDataType.FLOAT) .context(mockQueryShardContext) .filter(FILTER_QUERY_BUILDER) .build(); @@ -322,6 +326,7 @@ public void testCreate_whenNestedVectorAndFilterField_thenReturnSameFilterQuery( .fieldName(testFieldName) .vector(testQueryVector) .k(testK) + .vectorDataType(VectorDataType.FLOAT) .context(mockQueryShardContext) .filter(FILTER_QUERY_BUILDER) .build(); @@ -343,6 +348,7 @@ public void testCreate_whenFaissWithParentFilter_thenSuccess() { .fieldName(testFieldName) .vector(testQueryVector) .k(testK) + .vectorDataType(VectorDataType.FLOAT) .context(mockQueryShardContext) .build(); final Query query = KNNQueryFactory.create(createQueryRequest); @@ -379,4 +385,27 @@ private void validateDiversifyingQueryWithParentFilter(final VectorDataType type assertEquals(expectedQueryClass, query.getClass()); } } + + public void testCreate_whenBinary_thenSuccess() { + QueryShardContext mockQueryShardContext = mock(QueryShardContext.class); + MappedFieldType testMapper = mock(MappedFieldType.class); + when(mockQueryShardContext.fieldMapper(any())).thenReturn(testMapper); + BitSetProducer parentFilter = mock(BitSetProducer.class); + when(mockQueryShardContext.getParentFilter()).thenReturn(parentFilter); + final KNNQueryFactory.CreateQueryRequest createQueryRequest = KNNQueryFactory.CreateQueryRequest.builder() + .knnEngine(KNNEngine.FAISS) + .indexName(testIndexName) + .fieldName(testFieldName) + .vector(testQueryVector) + .byteVector(testByteQueryVector) + .vectorDataType(VectorDataType.BINARY) + .k(testK) + .context(mockQueryShardContext) + .filter(FILTER_QUERY_BUILDER) + .build(); + Query query = KNNQueryFactory.create(createQueryRequest); + assertTrue(query instanceof KNNQuery); + assertNotNull(((KNNQuery) query).getByteQueryVector()); + assertNull(((KNNQuery) query).getQueryVector()); + } } diff --git a/src/test/java/org/opensearch/knn/index/query/KNNWeightTests.java b/src/test/java/org/opensearch/knn/index/query/KNNWeightTests.java index ade564d94..bb5d7c4f6 100644 --- a/src/test/java/org/opensearch/knn/index/query/KNNWeightTests.java +++ b/src/test/java/org/opensearch/knn/index/query/KNNWeightTests.java @@ -29,6 +29,7 @@ import org.apache.lucene.util.FixedBitSet; import org.apache.lucene.util.StringHelper; import org.apache.lucene.util.Version; +import org.junit.After; import org.junit.Before; import org.junit.BeforeClass; import org.mockito.MockedStatic; @@ -38,6 +39,7 @@ import org.opensearch.knn.KNNTestCase; import org.opensearch.knn.index.KNNSettings; import org.opensearch.knn.index.SpaceType; +import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.codec.KNNCodecVersion; import org.opensearch.knn.index.codec.util.KNNVectorAsArraySerializer; import org.opensearch.knn.index.memory.NativeMemoryAllocation; @@ -70,6 +72,7 @@ import static org.mockito.ArgumentMatchers.isNull; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.mockStatic; +import static org.mockito.Mockito.times; import static org.mockito.Mockito.when; import static org.opensearch.knn.KNNRestTestCase.INDEX_NAME; import static org.opensearch.knn.common.KNNConstants.INDEX_DESCRIPTION_PARAMETER; @@ -82,6 +85,7 @@ public class KNNWeightTests extends KNNTestCase { private static final String FIELD_NAME = "target_field"; private static final float[] QUERY_VECTOR = new float[] { 1.8f, 2.4f }; + private static final byte[] BYTE_QUERY_VECTOR = new byte[] { 1, 2 }; private static final String SEGMENT_NAME = "0"; private static final int K = 5; private static final Set SEGMENT_FILES_NMSLIB = Set.of("_0.cfe", "_0_2011_target_field.hnswc"); @@ -98,6 +102,7 @@ public class KNNWeightTests extends KNNTestCase { private static final Map DOC_ID_TO_SCORES = Map.of(10, 0.4f, 101, 0.05f, 100, 0.8f, 50, 0.52f); private static final Map FILTERED_DOC_ID_TO_SCORES = Map.of(101, 0.05f, 100, 0.8f, 50, 0.52f); private static final Map EXACT_SEARCH_DOC_ID_TO_SCORES = Map.of(0, 0.12048191f); + private static final Map BINARY_EXACT_SEARCH_DOC_ID_TO_SCORES = Map.of(0, 0.5f); private static final Query FILTER_QUERY = new TermQuery(new Term("foo", "fooValue")); @@ -123,7 +128,6 @@ public static void setUpClass() throws Exception { knnSettingsMockedStatic.when(KNNSettings::state).thenReturn(knnSettings); knnSettingsMockedStatic.when(KNNSettings::isKNNPluginEnabled).thenReturn(true); - jniServiceMockedStatic = mockStatic(JNIService.class); nativeMemoryCacheManagerMockedStatic = mockStatic(NativeMemoryCacheManager.class); final NativeMemoryCacheManager nativeMemoryCacheManager = mock(NativeMemoryCacheManager.class); @@ -141,6 +145,12 @@ public static void setUpClass() throws Exception { @Before public void setupBeforeTest() { knnSettingsMockedStatic.when(() -> KNNSettings.getFilteredExactSearchThreshold(INDEX_NAME)).thenReturn(0); + jniServiceMockedStatic = mockStatic(JNIService.class); + } + + @After + public void tearDownAfterTest() { + jniServiceMockedStatic.close(); } @SneakyThrows @@ -384,45 +394,177 @@ public void testEmptyQueryResults() { } @SneakyThrows - public void testANNWithFilterQuery_whenDoingANN_thenSuccess() { + public void testScorer_whenNoFilterBinary_thenSuccess() { + validateScorer_whenNoFilter_thenSuccess(true); + } + + @SneakyThrows + public void testScorer_whenNoFilter_thenSuccess() { + validateScorer_whenNoFilter_thenSuccess(false); + } + + private void validateScorer_whenNoFilter_thenSuccess(final boolean isBinary) throws IOException { // Given int k = 3; - final int[] filterDocIds = new int[] { 0, 1, 2, 3, 4, 5 }; - FixedBitSet filterBitSet = new FixedBitSet(filterDocIds.length); - for (int docId : filterDocIds) { - filterBitSet.set(docId); - } jniServiceMockedStatic.when( - () -> JNIService.queryIndex( + () -> JNIService.queryIndex(anyLong(), eq(QUERY_VECTOR), eq(k), eq(HNSW_METHOD_PARAMETERS), any(), any(), anyInt(), any()) + ).thenReturn(getFilteredKNNQueryResults()); + + jniServiceMockedStatic.when( + () -> JNIService.queryBinaryIndex( anyLong(), - eq(QUERY_VECTOR), + eq(BYTE_QUERY_VECTOR), eq(k), eq(HNSW_METHOD_PARAMETERS), any(), - eq(filterBitSet.getBits()), + any(), anyInt(), any() ) ).thenReturn(getFilteredKNNQueryResults()); + final SegmentReader reader = mockSegmentReader(); final LeafReaderContext leafReaderContext = mock(LeafReaderContext.class); - final SegmentReader reader = mock(SegmentReader.class); + when(leafReaderContext.reader()).thenReturn(reader); + + final KNNQuery query = isBinary + ? KNNQuery.builder() + .field(FIELD_NAME) + .byteQueryVector(BYTE_QUERY_VECTOR) + .k(k) + .indexName(INDEX_NAME) + .filterQuery(FILTER_QUERY) + .methodParameters(HNSW_METHOD_PARAMETERS) + .vectorDataType(VectorDataType.BINARY) + .build() + : KNNQuery.builder() + .field(FIELD_NAME) + .queryVector(QUERY_VECTOR) + .k(k) + .indexName(INDEX_NAME) + .filterQuery(FILTER_QUERY) + .methodParameters(HNSW_METHOD_PARAMETERS) + .vectorDataType(VectorDataType.FLOAT) + .build(); + + final float boost = (float) randomDoubleBetween(0, 10, true); + final KNNWeight knnWeight = new KNNWeight(query, boost); + final FieldInfos fieldInfos = mock(FieldInfos.class); + final FieldInfo fieldInfo = mock(FieldInfo.class); + final Map attributesMap = ImmutableMap.of( + KNN_ENGINE, + KNNEngine.FAISS.getName(), + PARAMETERS, + String.format(Locale.ROOT, "{\"%s\":\"%s\"}", INDEX_DESCRIPTION_PARAMETER, "HNSW32") + ); + + when(reader.getFieldInfos()).thenReturn(fieldInfos); + when(fieldInfos.fieldInfo(any())).thenReturn(fieldInfo); + when(fieldInfo.attributes()).thenReturn(attributesMap); + + // When + final KNNScorer knnScorer = (KNNScorer) knnWeight.scorer(leafReaderContext); + + // Then + assertNotNull(knnScorer); + if (isBinary) { + jniServiceMockedStatic.verify( + () -> JNIService.queryBinaryIndex( + anyLong(), + eq(BYTE_QUERY_VECTOR), + eq(k), + eq(HNSW_METHOD_PARAMETERS), + any(), + any(), + anyInt(), + any() + ), + times(1) + ); + } else { + jniServiceMockedStatic.verify( + () -> JNIService.queryIndex(anyLong(), eq(QUERY_VECTOR), eq(k), eq(HNSW_METHOD_PARAMETERS), any(), any(), anyInt(), any()), + times(1) + ); + } + } + + @SneakyThrows + public void testANNWithFilterQuery_whenDoingANN_thenSuccess() { + validateANNWithFilterQuery_whenDoingANN_thenSuccess(false); + } + + @SneakyThrows + public void testANNWithFilterQuery_whenDoingANNBinary_thenSuccess() { + validateANNWithFilterQuery_whenDoingANN_thenSuccess(true); + } + + public void validateANNWithFilterQuery_whenDoingANN_thenSuccess(final boolean isBinary) throws IOException { + // Given + int k = 3; + final int[] filterDocIds = new int[] { 0, 1, 2, 3, 4, 5 }; + FixedBitSet filterBitSet = new FixedBitSet(filterDocIds.length); + for (int docId : filterDocIds) { + filterBitSet.set(docId); + } + if (isBinary) { + jniServiceMockedStatic.when( + () -> JNIService.queryBinaryIndex( + anyLong(), + eq(BYTE_QUERY_VECTOR), + eq(k), + eq(HNSW_METHOD_PARAMETERS), + any(), + eq(filterBitSet.getBits()), + anyInt(), + any() + ) + ).thenReturn(getFilteredKNNQueryResults()); + } else { + jniServiceMockedStatic.when( + () -> JNIService.queryIndex( + anyLong(), + eq(QUERY_VECTOR), + eq(k), + eq(HNSW_METHOD_PARAMETERS), + any(), + eq(filterBitSet.getBits()), + anyInt(), + any() + ) + ).thenReturn(getFilteredKNNQueryResults()); + } + final Bits liveDocsBits = mock(Bits.class); - when(reader.maxDoc()).thenReturn(filterDocIds.length); - when(reader.getLiveDocs()).thenReturn(liveDocsBits); for (int filterDocId : filterDocIds) { when(liveDocsBits.get(filterDocId)).thenReturn(true); } when(liveDocsBits.length()).thenReturn(1000); + + final SegmentReader reader = mockSegmentReader(); + when(reader.maxDoc()).thenReturn(filterDocIds.length); + when(reader.getLiveDocs()).thenReturn(liveDocsBits); + + final LeafReaderContext leafReaderContext = mock(LeafReaderContext.class); when(leafReaderContext.reader()).thenReturn(reader); - final KNNQuery query = KNNQuery.builder() - .field(FIELD_NAME) - .queryVector(QUERY_VECTOR) - .k(k) - .indexName(INDEX_NAME) - .filterQuery(FILTER_QUERY) - .methodParameters(HNSW_METHOD_PARAMETERS) - .build(); + final KNNQuery query = isBinary + ? KNNQuery.builder() + .field(FIELD_NAME) + .byteQueryVector(BYTE_QUERY_VECTOR) + .vectorDataType(VectorDataType.BINARY) + .k(k) + .indexName(INDEX_NAME) + .filterQuery(FILTER_QUERY) + .methodParameters(HNSW_METHOD_PARAMETERS) + .build() + : KNNQuery.builder() + .field(FIELD_NAME) + .queryVector(QUERY_VECTOR) + .k(k) + .indexName(INDEX_NAME) + .filterQuery(FILTER_QUERY) + .methodParameters(HNSW_METHOD_PARAMETERS) + .build(); final Weight filterQueryWeight = mock(Weight.class); final Scorer filterScorer = mock(Scorer.class); @@ -433,35 +575,13 @@ public void testANNWithFilterQuery_whenDoingANN_thenSuccess() { final float boost = (float) randomDoubleBetween(0, 10, true); final KNNWeight knnWeight = new KNNWeight(query, boost, filterQueryWeight); - final FSDirectory directory = mock(FSDirectory.class); - when(reader.directory()).thenReturn(directory); - final SegmentInfo segmentInfo = new SegmentInfo( - directory, - Version.LATEST, - Version.LATEST, - SEGMENT_NAME, - 100, - true, - false, - KNNCodecVersion.current().getDefaultCodecDelegate(), - Map.of(), - new byte[StringHelper.ID_LENGTH], - Map.of(), - Sort.RELEVANCE - ); - segmentInfo.setFiles(SEGMENT_FILES_FAISS); - final SegmentCommitInfo segmentCommitInfo = new SegmentCommitInfo(segmentInfo, 0, 0, 0, 0, 0, new byte[StringHelper.ID_LENGTH]); - when(reader.getSegmentInfo()).thenReturn(segmentCommitInfo); - - final Path path = mock(Path.class); - when(directory.getDirectory()).thenReturn(path); final FieldInfos fieldInfos = mock(FieldInfos.class); final FieldInfo fieldInfo = mock(FieldInfo.class); final Map attributesMap = ImmutableMap.of( KNN_ENGINE, KNNEngine.FAISS.getName(), - PARAMETERS, - String.format(Locale.ROOT, "{\"%s\":\"%s\"}", INDEX_DESCRIPTION_PARAMETER, "HNSW32") + SPACE_TYPE, + isBinary ? SpaceType.HAMMING_BIT.getValue() : SpaceType.L2.getValue() ); when(reader.getFieldInfos()).thenReturn(fieldInfos); @@ -477,18 +597,26 @@ public void testANNWithFilterQuery_whenDoingANN_thenSuccess() { assertNotNull(docIdSetIterator); assertEquals(FILTERED_DOC_ID_TO_SCORES.size(), docIdSetIterator.cost()); - jniServiceMockedStatic.verify( - () -> JNIService.queryIndex( - anyLong(), - eq(QUERY_VECTOR), - eq(k), - eq(HNSW_METHOD_PARAMETERS), - any(), - eq(filterBitSet.getBits()), - anyInt(), - any() - ) - ); + if (isBinary) { + jniServiceMockedStatic.verify( + () -> JNIService.queryBinaryIndex( + anyLong(), + eq(BYTE_QUERY_VECTOR), + eq(k), + eq(HNSW_METHOD_PARAMETERS), + any(), + any(), + anyInt(), + any() + ), + times(1) + ); + } else { + jniServiceMockedStatic.verify( + () -> JNIService.queryIndex(anyLong(), eq(QUERY_VECTOR), eq(k), eq(HNSW_METHOD_PARAMETERS), any(), any(), anyInt(), any()), + times(1) + ); + } final List actualDocIds = new ArrayList<>(); final Map translatedScores = getTranslatedScores(SpaceType.L2::scoreTranslation); @@ -500,15 +628,56 @@ public void testANNWithFilterQuery_whenDoingANN_thenSuccess() { assertTrue(Comparators.isInOrder(actualDocIds, Comparator.naturalOrder())); } + private SegmentReader mockSegmentReader() { + Path path = mock(Path.class); + + FSDirectory directory = mock(FSDirectory.class); + when(directory.getDirectory()).thenReturn(path); + + SegmentInfo segmentInfo = new SegmentInfo( + directory, + Version.LATEST, + Version.LATEST, + SEGMENT_NAME, + 100, + true, + false, + KNNCodecVersion.current().getDefaultCodecDelegate(), + Map.of(), + new byte[StringHelper.ID_LENGTH], + Map.of(), + Sort.RELEVANCE + ); + segmentInfo.setFiles(SEGMENT_FILES_FAISS); + SegmentCommitInfo segmentCommitInfo = new SegmentCommitInfo(segmentInfo, 0, 0, 0, 0, 0, new byte[StringHelper.ID_LENGTH]); + + SegmentReader reader = mock(SegmentReader.class); + when(reader.directory()).thenReturn(directory); + when(reader.getSegmentInfo()).thenReturn(segmentCommitInfo); + return reader; + } + @SneakyThrows public void testANNWithFilterQuery_whenExactSearch_thenSuccess() { + validateANNWithFilterQuery_whenExactSearch_thenSuccess(false); + } + + @SneakyThrows + public void testANNWithFilterQuery_whenExactSearchBinary_thenSuccess() { + validateANNWithFilterQuery_whenExactSearch_thenSuccess(true); + } + + public void validateANNWithFilterQuery_whenExactSearch_thenSuccess(final boolean isBinary) throws IOException { float[] vector = new float[] { 0.1f, 0.3f }; + byte[] byteVector = new byte[] { 1, 3 }; int filterDocId = 0; final LeafReaderContext leafReaderContext = mock(LeafReaderContext.class); final SegmentReader reader = mock(SegmentReader.class); when(leafReaderContext.reader()).thenReturn(reader); - final KNNQuery query = new KNNQuery(FIELD_NAME, QUERY_VECTOR, K, INDEX_NAME, FILTER_QUERY, null); + final KNNQuery query = isBinary + ? new KNNQuery(FIELD_NAME, BYTE_QUERY_VECTOR, K, INDEX_NAME, FILTER_QUERY, null, VectorDataType.BINARY) + : new KNNQuery(FIELD_NAME, QUERY_VECTOR, K, INDEX_NAME, FILTER_QUERY, null); final Weight filterQueryWeight = mock(Weight.class); final Scorer filterScorer = mock(Scorer.class); when(filterQueryWeight.scorer(leafReaderContext)).thenReturn(filterScorer); @@ -525,9 +694,7 @@ public void testANNWithFilterQuery_whenExactSearch_thenSuccess() { KNN_ENGINE, KNNEngine.FAISS.getName(), SPACE_TYPE, - SpaceType.L2.name(), - PARAMETERS, - String.format(Locale.ROOT, "{\"%s\":\"%s\"}", INDEX_DESCRIPTION_PARAMETER, "HNSW32") + isBinary ? SpaceType.HAMMING_BIT.getValue() : SpaceType.L2.getValue() ); final FieldInfos fieldInfos = mock(FieldInfos.class); final FieldInfo fieldInfo = mock(FieldInfo.class); @@ -535,23 +702,36 @@ public void testANNWithFilterQuery_whenExactSearch_thenSuccess() { when(reader.getFieldInfos()).thenReturn(fieldInfos); when(fieldInfos.fieldInfo(any())).thenReturn(fieldInfo); when(fieldInfo.attributes()).thenReturn(attributesMap); - when(fieldInfo.getAttribute(SPACE_TYPE)).thenReturn(SpaceType.L2.name()); + if (isBinary) { + when(fieldInfo.getAttribute(SPACE_TYPE)).thenReturn(SpaceType.HAMMING_BIT.getValue()); + } else { + when(fieldInfo.getAttribute(SPACE_TYPE)).thenReturn(SpaceType.L2.getValue()); + } when(fieldInfo.getName()).thenReturn(FIELD_NAME); when(reader.getBinaryDocValues(FIELD_NAME)).thenReturn(binaryDocValues); when(binaryDocValues.advance(filterDocId)).thenReturn(filterDocId); BytesRef vectorByteRef = new BytesRef(new KNNVectorAsArraySerializer().floatToByteArray(vector)); - when(binaryDocValues.binaryValue()).thenReturn(vectorByteRef); + + if (isBinary) { + when(binaryDocValues.binaryValue()).thenReturn(new BytesRef(byteVector)); + } else { + when(binaryDocValues.binaryValue()).thenReturn(vectorByteRef); + } final KNNScorer knnScorer = (KNNScorer) knnWeight.scorer(leafReaderContext); assertNotNull(knnScorer); final DocIdSetIterator docIdSetIterator = knnScorer.iterator(); assertNotNull(docIdSetIterator); - assertEquals(EXACT_SEARCH_DOC_ID_TO_SCORES.size(), docIdSetIterator.cost()); + assertEquals(1, docIdSetIterator.cost()); final List actualDocIds = new ArrayList<>(); for (int docId = docIdSetIterator.nextDoc(); docId != NO_MORE_DOCS; docId = docIdSetIterator.nextDoc()) { actualDocIds.add(docId); - assertEquals(EXACT_SEARCH_DOC_ID_TO_SCORES.get(docId) * boost, knnScorer.score(), 0.01f); + if (isBinary) { + assertEquals(BINARY_EXACT_SEARCH_DOC_ID_TO_SCORES.get(docId) * boost, knnScorer.score(), 0.01f); + } else { + assertEquals(EXACT_SEARCH_DOC_ID_TO_SCORES.get(docId) * boost, knnScorer.score(), 0.01f); + } } assertEquals(docIdSetIterator.cost(), actualDocIds.size()); assertTrue(Comparators.isInOrder(actualDocIds, Comparator.naturalOrder())); @@ -749,7 +929,7 @@ public void testANNWithParentsFilter_whenExactSearch_thenSuccess() { // Verify final List expectedScores = vectors.stream() - .map(vector -> SpaceType.L2.getVectorSimilarityFunction().compare(QUERY_VECTOR, vector)) + .map(vector -> SpaceType.L2.getKnnVectorSimilarityFunction().compare(QUERY_VECTOR, vector)) .collect(Collectors.toList()); final DocIdSetIterator docIdSetIterator = knnScorer.iterator(); assertEquals(1, docIdSetIterator.nextDoc()); diff --git a/src/test/java/org/opensearch/knn/index/query/filtered/FilteredIdsKNNByteIteratorTests.java b/src/test/java/org/opensearch/knn/index/query/filtered/FilteredIdsKNNByteIteratorTests.java new file mode 100644 index 000000000..aabbb1f9c --- /dev/null +++ b/src/test/java/org/opensearch/knn/index/query/filtered/FilteredIdsKNNByteIteratorTests.java @@ -0,0 +1,52 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.query.filtered; + +import junit.framework.TestCase; +import lombok.SneakyThrows; +import org.apache.lucene.index.BinaryDocValues; +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.FixedBitSet; +import org.opensearch.knn.index.SpaceType; + +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class FilteredIdsKNNByteIteratorTests extends TestCase { + @SneakyThrows + public void testNextDoc_whenCalled_IterateAllDocs() { + final SpaceType spaceType = SpaceType.HAMMING_BIT; + final byte[] queryVector = { 1, 2, 3 }; + final int[] filterIds = { 1, 2, 3 }; + final List dataVectors = Arrays.asList(new byte[] { 11, 12, 13 }, new byte[] { 14, 15, 16 }, new byte[] { 17, 18, 19 }); + final List expectedScores = dataVectors.stream() + .map(vector -> spaceType.getKnnVectorSimilarityFunction().compare(queryVector, vector)) + .collect(Collectors.toList()); + + BinaryDocValues values = mock(BinaryDocValues.class); + final List byteRefs = dataVectors.stream().map(vector -> new BytesRef(vector)).collect(Collectors.toList()); + when(values.binaryValue()).thenReturn(byteRefs.get(0), byteRefs.get(1), byteRefs.get(2)); + + FixedBitSet filterBitSet = new FixedBitSet(4); + for (int id : filterIds) { + when(values.advance(id)).thenReturn(id); + filterBitSet.set(id); + } + + // Execute and verify + FilteredIdsKNNByteIterator iterator = new FilteredIdsKNNByteIterator(filterBitSet, queryVector, values, spaceType); + for (int i = 0; i < filterIds.length; i++) { + assertEquals(filterIds[i], iterator.nextDoc()); + assertEquals(expectedScores.get(i), (Float) iterator.score()); + } + assertEquals(DocIdSetIterator.NO_MORE_DOCS, iterator.nextDoc()); + } +} diff --git a/src/test/java/org/opensearch/knn/index/query/filtered/FilteredIdsKNNIteratorTests.java b/src/test/java/org/opensearch/knn/index/query/filtered/FilteredIdsKNNIteratorTests.java index dce703050..cf8582a05 100644 --- a/src/test/java/org/opensearch/knn/index/query/filtered/FilteredIdsKNNIteratorTests.java +++ b/src/test/java/org/opensearch/knn/index/query/filtered/FilteredIdsKNNIteratorTests.java @@ -33,7 +33,7 @@ public void testNextDoc_whenCalled_IterateAllDocs() { new float[] { 17.0f, 18.0f, 19.0f } ); final List expectedScores = dataVectors.stream() - .map(vector -> spaceType.getVectorSimilarityFunction().compare(queryVector, vector)) + .map(vector -> spaceType.getKnnVectorSimilarityFunction().compare(queryVector, vector)) .collect(Collectors.toList()); BinaryDocValues values = mock(BinaryDocValues.class); diff --git a/src/test/java/org/opensearch/knn/index/query/filtered/NestedFilteredIdsKNNByteIteratorTests.java b/src/test/java/org/opensearch/knn/index/query/filtered/NestedFilteredIdsKNNByteIteratorTests.java new file mode 100644 index 000000000..01a4eb2b1 --- /dev/null +++ b/src/test/java/org/opensearch/knn/index/query/filtered/NestedFilteredIdsKNNByteIteratorTests.java @@ -0,0 +1,63 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.query.filtered; + +import junit.framework.TestCase; +import lombok.SneakyThrows; +import org.apache.lucene.index.BinaryDocValues; +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.util.BitSet; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.FixedBitSet; +import org.opensearch.knn.index.SpaceType; + +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class NestedFilteredIdsKNNByteIteratorTests extends TestCase { + @SneakyThrows + public void testNextDoc_whenIterate_ReturnBestChildDocsPerParent() { + final SpaceType spaceType = SpaceType.HAMMING_BIT; + final byte[] queryVector = { 1, 2, 3 }; + final int[] filterIds = { 0, 2, 3 }; + // Parent id for 0 -> 1 + // Parent id for 2, 3 -> 4 + // In bit representation, it is 10010. In long, it is 18. + final BitSet parentBitSet = new FixedBitSet(new long[] { 18 }, 5); + final List dataVectors = Arrays.asList(new byte[] { 11, 12, 13 }, new byte[] { 14, 15, 16 }, new byte[] { 17, 18, 19 }); + final List expectedScores = dataVectors.stream() + .map(vector -> spaceType.getKnnVectorSimilarityFunction().compare(queryVector, vector)) + .collect(Collectors.toList()); + + BinaryDocValues values = mock(BinaryDocValues.class); + final List byteRefs = dataVectors.stream().map(vector -> new BytesRef(vector)).collect(Collectors.toList()); + when(values.binaryValue()).thenReturn(byteRefs.get(0), byteRefs.get(1), byteRefs.get(2)); + + FixedBitSet filterBitSet = new FixedBitSet(4); + for (int id : filterIds) { + when(values.advance(id)).thenReturn(id); + filterBitSet.set(id); + } + + // Execute and verify + NestedFilteredIdsKNNByteIterator iterator = new NestedFilteredIdsKNNByteIterator( + filterBitSet, + queryVector, + values, + spaceType, + parentBitSet + ); + assertEquals(filterIds[0], iterator.nextDoc()); + assertEquals(expectedScores.get(0), iterator.score()); + assertEquals(filterIds[2], iterator.nextDoc()); + assertEquals(expectedScores.get(2), iterator.score()); + assertEquals(DocIdSetIterator.NO_MORE_DOCS, iterator.nextDoc()); + } +} diff --git a/src/test/java/org/opensearch/knn/index/query/filtered/NestedFilteredIdsKNNIteratorTests.java b/src/test/java/org/opensearch/knn/index/query/filtered/NestedFilteredIdsKNNIteratorTests.java index d732376ef..508b0d3d6 100644 --- a/src/test/java/org/opensearch/knn/index/query/filtered/NestedFilteredIdsKNNIteratorTests.java +++ b/src/test/java/org/opensearch/knn/index/query/filtered/NestedFilteredIdsKNNIteratorTests.java @@ -38,7 +38,7 @@ public void testNextDoc_whenIterate_ReturnBestChildDocsPerParent() { new float[] { 14.0f, 15.0f, 16.0f } ); final List expectedScores = dataVectors.stream() - .map(vector -> spaceType.getVectorSimilarityFunction().compare(queryVector, vector)) + .map(vector -> spaceType.getKnnVectorSimilarityFunction().compare(queryVector, vector)) .collect(Collectors.toList()); BinaryDocValues values = mock(BinaryDocValues.class); diff --git a/src/test/java/org/opensearch/knn/index/util/FieldInfoExtractorTests.java b/src/test/java/org/opensearch/knn/index/util/FieldInfoExtractorTests.java deleted file mode 100644 index a0facefbd..000000000 --- a/src/test/java/org/opensearch/knn/index/util/FieldInfoExtractorTests.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.knn.index.util; - -import junit.framework.TestCase; -import lombok.SneakyThrows; -import org.apache.lucene.index.FieldInfo; -import org.mockito.Mockito; -import org.opensearch.common.xcontent.XContentFactory; -import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.knn.common.KNNConstants; - -import java.util.Collections; -import java.util.Map; - -import static org.mockito.Mockito.mock; -import static org.opensearch.knn.common.KNNConstants.INDEX_DESCRIPTION_PARAMETER; - -public class FieldInfoExtractorTests extends TestCase { - @SneakyThrows - public void testGetIndexDescription_whenNoDescription_thenReturnNull() { - FieldInfo fieldInfo = mock(FieldInfo.class); - Mockito.when(fieldInfo.attributes()).thenReturn(Collections.emptyMap(), Map.of(KNNConstants.PARAMETERS, "{}")); - assertNull(FieldInfoExtractor.getIndexDescription(fieldInfo)); - assertNull(FieldInfoExtractor.getIndexDescription(fieldInfo)); - } - - @SneakyThrows - public void testGetIndexDescription_whenDescriptionExist_thenReturnIndexDescription() { - String indexDescription = "HNSW"; - XContentBuilder parameters = XContentFactory.jsonBuilder() - .startObject() - .field(INDEX_DESCRIPTION_PARAMETER, indexDescription) - .endObject(); - FieldInfo fieldInfo = mock(FieldInfo.class); - Mockito.when(fieldInfo.attributes()).thenReturn(Map.of(KNNConstants.PARAMETERS, parameters.toString())); - assertEquals(indexDescription, FieldInfoExtractor.getIndexDescription(fieldInfo)); - } -} diff --git a/src/test/java/org/opensearch/knn/jni/JNIServiceTests.java b/src/test/java/org/opensearch/knn/jni/JNIServiceTests.java index e17ee5077..6fab6c795 100644 --- a/src/test/java/org/opensearch/knn/jni/JNIServiceTests.java +++ b/src/test/java/org/opensearch/knn/jni/JNIServiceTests.java @@ -22,6 +22,7 @@ import org.opensearch.knn.TestUtils; import org.opensearch.knn.common.KNNConstants; import org.opensearch.knn.index.KNNMethodContext; +import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.query.KNNQueryResult; import org.opensearch.knn.index.MethodComponentContext; import org.opensearch.knn.index.SpaceType; @@ -53,6 +54,7 @@ import static org.opensearch.knn.common.KNNConstants.METHOD_HNSW; import static org.opensearch.knn.common.KNNConstants.METHOD_IVF; import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_NLIST; +import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_SPACE_TYPE; import static org.opensearch.knn.common.KNNConstants.NAME; import static org.opensearch.knn.common.KNNConstants.PARAMETERS; @@ -264,6 +266,10 @@ public void testCreateIndex_nmslib_invalid_badParameterType() throws IOException public void testCreateIndex_nmslib_valid() throws IOException { for (SpaceType spaceType : KNNEngine.NMSLIB.getMethod(KNNConstants.METHOD_HNSW).getSpaces()) { + if (SpaceType.UNDEFINED == spaceType) { + continue; + } + Path tmpFile = createTempFile(); JNIService.createIndex( @@ -591,6 +597,7 @@ public void testTrain_whenConfigurationIsIVFSQFP16_thenSucceed() { .startObject() .field(NAME, METHOD_IVF) .field(KNN_ENGINE, FAISS_NAME) + .field(METHOD_PARAMETER_SPACE_TYPE, SpaceType.DEFAULT) .startObject(PARAMETERS) .field(METHOD_PARAMETER_NLIST, ivfNlistParam) .startObject(METHOD_ENCODER_PARAMETER) @@ -667,7 +674,14 @@ public void testCreateIndex_binary_faiss_valid() { memoryAddr, testData.indexData.getDimension(), tmpFile1.toAbsolutePath().toString(), - ImmutableMap.of(INDEX_DESCRIPTION_PARAMETER, faissBinaryMethod, KNNConstants.SPACE_TYPE, SpaceType.HAMMING_BIT.getValue()), + ImmutableMap.of( + INDEX_DESCRIPTION_PARAMETER, + faissBinaryMethod, + KNNConstants.SPACE_TYPE, + SpaceType.HAMMING_BIT.getValue(), + KNNConstants.VECTOR_DATA_TYPE_FIELD, + VectorDataType.BINARY.getValue() + ), KNNEngine.FAISS ); assertTrue(tmpFile1.toFile().length() > 0); @@ -801,6 +815,10 @@ public void testQueryIndex_nmslib_valid() throws IOException { int k = 50; for (SpaceType spaceType : KNNEngine.NMSLIB.getMethod(KNNConstants.METHOD_HNSW).getSpaces()) { + if (SpaceType.UNDEFINED == spaceType) { + continue; + } + Path tmpFile = createTempFile(); JNIService.createIndex( @@ -971,14 +989,21 @@ public void testQueryBinaryIndex_faiss_valid() { memoryAddr, testData.indexData.getDimension(), tmpFile.toAbsolutePath().toString(), - ImmutableMap.of(INDEX_DESCRIPTION_PARAMETER, method, KNNConstants.SPACE_TYPE, SpaceType.HAMMING_BIT.getValue()), + ImmutableMap.of( + INDEX_DESCRIPTION_PARAMETER, + method, + KNNConstants.SPACE_TYPE, + SpaceType.HAMMING_BIT.getValue(), + KNNConstants.VECTOR_DATA_TYPE_FIELD, + VectorDataType.BINARY.getValue() + ), KNNEngine.FAISS ); assertTrue(tmpFile.toFile().length() > 0); long pointer = JNIService.loadIndex( tmpFile.toAbsolutePath().toString(), - ImmutableMap.of(INDEX_DESCRIPTION_PARAMETER, method), + ImmutableMap.of(INDEX_DESCRIPTION_PARAMETER, method, KNNConstants.VECTOR_DATA_TYPE_FIELD, VectorDataType.BINARY.getValue()), KNNEngine.FAISS ); assertNotEquals(0, pointer); @@ -1098,6 +1123,7 @@ public void testTrain_whenConfigurationIsIVFFlat_thenSucceed() throws IOExceptio .startObject() .field(NAME, METHOD_IVF) .field(KNN_ENGINE, FAISS_NAME) + .field(METHOD_PARAMETER_SPACE_TYPE, SpaceType.DEFAULT) .startObject(PARAMETERS) .field(METHOD_PARAMETER_NLIST, ivfNlistParam) .endObject() @@ -1121,6 +1147,7 @@ public void testTrain_whenConfigurationIsIVFPQ_thenSucceed() throws IOException .startObject() .field(NAME, METHOD_IVF) .field(KNN_ENGINE, FAISS_NAME) + .field(METHOD_PARAMETER_SPACE_TYPE, SpaceType.DEFAULT.getValue()) .startObject(PARAMETERS) .field(METHOD_PARAMETER_NLIST, ivfNlistParam) .startObject(METHOD_ENCODER_PARAMETER) @@ -1149,6 +1176,7 @@ public void testTrain_whenConfigurationIsHNSWPQ_thenSucceed() throws IOException .startObject() .field(NAME, METHOD_HNSW) .field(KNN_ENGINE, FAISS_NAME) + .field(METHOD_PARAMETER_SPACE_TYPE, SpaceType.DEFAULT.getValue()) .startObject(PARAMETERS) .startObject(METHOD_ENCODER_PARAMETER) .field(NAME, ENCODER_PQ) diff --git a/src/test/java/org/opensearch/knn/plugin/script/KNNScoringUtilTests.java b/src/test/java/org/opensearch/knn/plugin/script/KNNScoringUtilTests.java index 22110accd..a8d37b6c5 100644 --- a/src/test/java/org/opensearch/knn/plugin/script/KNNScoringUtilTests.java +++ b/src/test/java/org/opensearch/knn/plugin/script/KNNScoringUtilTests.java @@ -267,6 +267,12 @@ public void testZeroVectorFailsCosineSimilarityOptimized() throws IOException { dataset.close(); } + public void testCalculateHammingBit_whenByte_thenSuccess() { + byte[] v1 = { 1, 16, -128 }; // 0000 0001, 0001 0000, 1000 0000 + byte[] v2 = { 2, 17, -1 }; // 0000 0010, 0001 0001, 1111 1111 + assertEquals(10, KNNScoringUtil.calculateHammingBit(v1, v2), 0.001f); + } + class TestKNNScriptDocValues { private KNNVectorScriptDocValues scriptDocValues; private Directory directory; diff --git a/src/test/java/org/opensearch/knn/plugin/script/KNNScriptScoringIT.java b/src/test/java/org/opensearch/knn/plugin/script/KNNScriptScoringIT.java index 9684d1705..d7bad8332 100644 --- a/src/test/java/org/opensearch/knn/plugin/script/KNNScriptScoringIT.java +++ b/src/test/java/org/opensearch/knn/plugin/script/KNNScriptScoringIT.java @@ -588,11 +588,12 @@ public void testKNNScriptScoreOnModelBasedIndex() throws Exception { .toString(); for (SpaceType spaceType : SpaceType.values()) { - if (spaceType != SpaceType.HAMMING_BIT) { - final float[] queryVector = randomVector(dimensions); - final BiFunction scoreFunction = getScoreFunction(spaceType, queryVector); - createIndexAndAssertScriptScore(testMapping, spaceType, scoreFunction, dimensions, queryVector, true); + if (SpaceType.UNDEFINED == spaceType || SpaceType.HAMMING_BIT == spaceType) { + continue; } + final float[] queryVector = randomVector(dimensions); + final BiFunction scoreFunction = getScoreFunction(spaceType, queryVector); + createIndexAndAssertScriptScore(testMapping, spaceType, scoreFunction, dimensions, queryVector, true); } } From 27a3b379db48b9f650433a895389ddc478db1009 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Tue, 16 Jul 2024 12:49:41 -0700 Subject: [PATCH 289/416] Bump faiss commit to 33c0ba5 (#1796) (#1814) * Bump faiss commit to 33c0ba5 and update patches Signed-off-by: Naveen Tatikonda * Add Changelog Signed-off-by: Naveen Tatikonda * Set cmake minimum requirement to 3.24.0 Signed-off-by: Naveen Tatikonda --------- Signed-off-by: Naveen Tatikonda (cherry picked from commit 8fb779df1c0983c3dee12001422f6175df990e4c) Co-authored-by: Naveen Tatikonda --- CHANGELOG.md | 1 + DEVELOPER_GUIDE.md | 4 +-- jni/CMakeLists.txt | 2 +- jni/external/faiss | 2 +- ...Custom-patch-to-support-multi-vector.patch | 35 +++++++++---------- ...ble-precomp-table-to-be-shared-ivfpq.patch | 14 ++++---- 6 files changed, 29 insertions(+), 29 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 96b833811..bf943731c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,4 +25,5 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Infrastructure ### Documentation ### Maintenance +* Bump faiss commit to 33c0ba5 [#1796](https://github.com/opensearch-project/k-NN/pull/1796) ### Refactoring diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index ff37b456b..ecc97eb78 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -57,11 +57,11 @@ In addition to this, the plugin has been tested with JDK 17, and this JDK versio #### CMake -The plugin requires that cmake >= 3.23.3 is installed in order to build the JNI libraries. +The plugin requires that cmake >= 3.24.0 is installed in order to build the JNI libraries. One easy way to install on mac or linux is to use pip: ```bash -pip install cmake==3.23.3 +pip install cmake==3.24.0 ``` On Mac M series machines, install cmake using: diff --git a/jni/CMakeLists.txt b/jni/CMakeLists.txt index 2fe26875d..3c071fc1f 100644 --- a/jni/CMakeLists.txt +++ b/jni/CMakeLists.txt @@ -3,7 +3,7 @@ # SPDX-License-Identifier: Apache-2.0 # -cmake_minimum_required(VERSION 3.23.1) +cmake_minimum_required(VERSION 3.24.0) project(KNNPlugin_JNI) diff --git a/jni/external/faiss b/jni/external/faiss index 12b92e9fa..33c0ba5d0 160000 --- a/jni/external/faiss +++ b/jni/external/faiss @@ -1 +1 @@ -Subproject commit 12b92e9fa5d8e8fb3da53c57af9ff007c826b1ee +Subproject commit 33c0ba5d002a7cd9761513f06ecc9822079d4a2f diff --git a/jni/patches/faiss/0001-Custom-patch-to-support-multi-vector.patch b/jni/patches/faiss/0001-Custom-patch-to-support-multi-vector.patch index aa34795fb..7afdabc1f 100644 --- a/jni/patches/faiss/0001-Custom-patch-to-support-multi-vector.patch +++ b/jni/patches/faiss/0001-Custom-patch-to-support-multi-vector.patch @@ -1,4 +1,4 @@ -From 35ef01f59b8903dfbd4d08ff874b085e851e4228 Mon Sep 17 00:00:00 2001 +From 9e5affabe2caacf38f5585a0b906620dd35deef5 Mon Sep 17 00:00:00 2001 From: Heemin Kim Date: Tue, 30 Jan 2024 14:43:56 -0800 Subject: [PATCH] Add IDGrouper for HNSW @@ -13,12 +13,12 @@ Signed-off-by: Heemin Kim faiss/impl/HNSW.cpp | 6 + faiss/impl/IDGrouper.cpp | 51 ++++++++ faiss/impl/IDGrouper.h | 51 ++++++++ - faiss/impl/ResultHandler.h | 190 +++++++++++++++++++++++++++++ + faiss/impl/ResultHandler.h | 189 +++++++++++++++++++++++++++++ faiss/utils/GroupHeap.h | 182 ++++++++++++++++++++++++++++ tests/CMakeLists.txt | 2 + tests/test_group_heap.cpp | 98 +++++++++++++++ tests/test_id_grouper.cpp | 241 +++++++++++++++++++++++++++++++++++++ - 13 files changed, 891 insertions(+), 5 deletions(-) + 13 files changed, 890 insertions(+), 5 deletions(-) create mode 100644 faiss/impl/IDGrouper.cpp create mode 100644 faiss/impl/IDGrouper.h create mode 100644 faiss/utils/GroupHeap.h @@ -26,7 +26,7 @@ Signed-off-by: Heemin Kim create mode 100644 tests/test_id_grouper.cpp diff --git a/faiss/CMakeLists.txt b/faiss/CMakeLists.txt -index a890a46f..137e68d4 100644 +index 1b0860f3..f3d72df3 100644 --- a/faiss/CMakeLists.txt +++ b/faiss/CMakeLists.txt @@ -54,6 +54,7 @@ set(FAISS_SRC @@ -45,10 +45,10 @@ index a890a46f..137e68d4 100644 impl/DistanceComputer.h impl/FaissAssert.h impl/FaissException.h -@@ -183,6 +185,7 @@ set(FAISS_HEADERS - invlists/InvertedLists.h +@@ -184,6 +186,7 @@ set(FAISS_HEADERS invlists/InvertedListsIOHook.h utils/AlignedTable.h + utils/bf16.h + utils/GroupHeap.h utils/Heap.h utils/WorkerThread.h @@ -81,10 +81,10 @@ index 3d1bdb99..a8622858 100644 virtual ~SearchParameters() {} }; diff --git a/faiss/IndexHNSW.cpp b/faiss/IndexHNSW.cpp -index 9a67332d..a5e0fea0 100644 +index 8e5c654f..d473b6ad 100644 --- a/faiss/IndexHNSW.cpp +++ b/faiss/IndexHNSW.cpp -@@ -354,10 +354,17 @@ void IndexHNSW::search( +@@ -320,10 +320,17 @@ void IndexHNSW::search( const SearchParameters* params_in) const { FAISS_THROW_IF_NOT(k > 0); @@ -198,10 +198,10 @@ index 2d164123..a68887bd 100644 + } // namespace faiss diff --git a/faiss/impl/HNSW.cpp b/faiss/impl/HNSW.cpp -index a9fb9daf..33b56638 100644 +index 3ba5f72f..c574ce39 100644 --- a/faiss/impl/HNSW.cpp +++ b/faiss/impl/HNSW.cpp -@@ -804,6 +804,12 @@ int extract_k_from_ResultHandler(ResultHandler& res) { +@@ -831,6 +831,12 @@ int extract_k_from_ResultHandler(ResultHandler& res) { if (auto hres = dynamic_cast(&res)) { return hres->k; } @@ -329,20 +329,19 @@ index 00000000..d56113d9 + +} // namespace faiss diff --git a/faiss/impl/ResultHandler.h b/faiss/impl/ResultHandler.h -index 270de8dc..3199634f 100644 +index 713fe8e4..d307fd70 100644 --- a/faiss/impl/ResultHandler.h +++ b/faiss/impl/ResultHandler.h -@@ -12,6 +12,9 @@ - #pragma once +@@ -13,6 +13,8 @@ #include -+#include + #include +#include +#include #include #include - -@@ -265,6 +268,193 @@ struct HeapBlockResultHandler : BlockResultHandler { + #include +@@ -267,6 +269,193 @@ struct HeapBlockResultHandler : BlockResultHandler { } }; @@ -726,7 +725,7 @@ index 00000000..3b7078da +} // namespace faiss \ No newline at end of file diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt -index 9017edc5..a8e9d30c 100644 +index 3980d7dd..c888a5a6 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -27,6 +27,8 @@ set(FAISS_TEST_SRC @@ -1090,5 +1089,5 @@ index 00000000..6601795b + delete[] xb; +} -- -2.39.3 (Apple Git-145) +2.37.0 diff --git a/jni/patches/faiss/0002-Enable-precomp-table-to-be-shared-ivfpq.patch b/jni/patches/faiss/0002-Enable-precomp-table-to-be-shared-ivfpq.patch index dfc5099aa..619832f62 100644 --- a/jni/patches/faiss/0002-Enable-precomp-table-to-be-shared-ivfpq.patch +++ b/jni/patches/faiss/0002-Enable-precomp-table-to-be-shared-ivfpq.patch @@ -1,4 +1,4 @@ -From c5ca07299b427dedafc738b98bd20f8f286f6783 Mon Sep 17 00:00:00 2001 +From a33e6ef35385009f24200586294f96235cb95d61 Mon Sep 17 00:00:00 2001 From: John Mazanec Date: Wed, 21 Feb 2024 15:34:15 -0800 Subject: [PATCH] Enable precomp table to be shared ivfpq @@ -167,7 +167,7 @@ index d5d21da4..850bbe44 100644 }; diff --git a/faiss/IndexIVFPQFastScan.cpp b/faiss/IndexIVFPQFastScan.cpp -index d069db13..09a335ff 100644 +index 2844ae49..895df342 100644 --- a/faiss/IndexIVFPQFastScan.cpp +++ b/faiss/IndexIVFPQFastScan.cpp @@ -46,6 +46,8 @@ IndexIVFPQFastScan::IndexIVFPQFastScan( @@ -302,13 +302,13 @@ index 00dd2f11..91f35a6e 100644 /// same as the regular IVFPQ encoder. The codes are not reorganized by /// blocks a that point diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt -index 9017edc5..0889bf72 100644 +index c888a5a6..83ecedfd 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt -@@ -33,6 +33,7 @@ set(FAISS_TEST_SRC - test_partitioning.cpp - test_fastscan_perf.cpp +@@ -37,6 +37,7 @@ set(FAISS_TEST_SRC test_disable_pq_sdc_tables.cpp + test_common_ivf_empty_index.cpp + test_callback.cpp + test_ivfpq_share_table.cpp ) @@ -508,5 +508,5 @@ index 00000000..f827315d + "IVF16,PQ8x4fsr", "/tmp/ivfpqfsip", faiss::METRIC_INNER_PRODUCT); +} -- -2.39.3 (Apple Git-145) +2.37.0 From 3e9e5cfd5527d67998048d24689525c72519cac5 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Tue, 16 Jul 2024 13:35:20 -0700 Subject: [PATCH 290/416] Switch from byte stream to byte ref (#1835) Avoids copy during serialization and deserialization by switching from requiring byte streams to only requiring byte refs. This can speed up operations by 15-20% for exact search. This will not have impact on KnnVectorsFormat structures. Those will use lucenes vector serde which is already optimized. This change is meant to add a boost in performance until that is used completely. Signed-off-by: John Mazanec (cherry picked from commit e5b90ce36fb69f417a644fd50ad49061f5c95753) --- CHANGELOG.md | 1 + .../opensearch/knn/index/VectorDataType.java | 6 +- .../index/codec/transfer/VectorTransfer.java | 11 ++-- .../codec/transfer/VectorTransferByte.java | 14 ++--- .../codec/transfer/VectorTransferFloat.java | 12 ++-- .../knn/index/codec/util/KNNCodecUtil.java | 15 +++-- .../util/KNNVectorAsArraySerializer.java | 6 +- ...NVectorAsCollectionOfFloatsSerializer.java | 13 ++--- .../index/codec/util/KNNVectorSerializer.java | 7 ++- .../util/KNNVectorSerializerFactory.java | 30 +++++----- .../filtered/FilteredIdsKNNIterator.java | 6 +- .../transfer/VectorTransferByteTests.java | 18 +++--- .../transfer/VectorTransferFloatTests.java | 18 +++--- .../index/codec/util/KNNCodecUtilTests.java | 7 +-- .../codec/util/KNNVectorSerializerTests.java | 56 ++++++++++--------- .../mapper/KNNVectorFieldMapperUtilTests.java | 11 +--- 16 files changed, 114 insertions(+), 117 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bf943731c..bfa785204 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), * Adds dynamic query parameter ef_search in radial search faiss engine [#1790](https://github.com/opensearch-project/k-NN/pull/1790) * Add binary format support with HNSW method in Faiss Engine [#1781](https://github.com/opensearch-project/k-NN/pull/1781) ### Enhancements +* Switch from byte stream to byte ref for serde [#1825](https://github.com/opensearch-project/k-NN/pull/1825) ### Bug Fixes * Fixing the arithmetic to find the number of vectors to stream from java to jni layer.[#1804](https://github.com/opensearch-project/k-NN/pull/1804) * Release memory properly for an array type [#1820](https://github.com/opensearch-project/k-NN/pull/1820) diff --git a/src/main/java/org/opensearch/knn/index/VectorDataType.java b/src/main/java/org/opensearch/knn/index/VectorDataType.java index 8add84609..c9fd131a7 100644 --- a/src/main/java/org/opensearch/knn/index/VectorDataType.java +++ b/src/main/java/org/opensearch/knn/index/VectorDataType.java @@ -15,7 +15,6 @@ import org.opensearch.knn.index.codec.util.KNNVectorSerializer; import org.opensearch.knn.index.codec.util.KNNVectorSerializerFactory; -import java.io.ByteArrayInputStream; import java.util.Arrays; import java.util.Locale; import java.util.Objects; @@ -71,9 +70,8 @@ public FieldType createKnnVectorFieldType(int dimension, VectorSimilarityFunctio @Override public float[] getVectorFromBytesRef(BytesRef binaryValue) { - ByteArrayInputStream byteStream = new ByteArrayInputStream(binaryValue.bytes, binaryValue.offset, binaryValue.length); - final KNNVectorSerializer vectorSerializer = KNNVectorSerializerFactory.getSerializerByStreamContent(byteStream); - return vectorSerializer.byteToFloatArray(byteStream); + final KNNVectorSerializer vectorSerializer = KNNVectorSerializerFactory.getSerializerByBytesRef(binaryValue); + return vectorSerializer.byteToFloatArray(binaryValue); } }; diff --git a/src/main/java/org/opensearch/knn/index/codec/transfer/VectorTransfer.java b/src/main/java/org/opensearch/knn/index/codec/transfer/VectorTransfer.java index 2ab80f776..c23bd4317 100644 --- a/src/main/java/org/opensearch/knn/index/codec/transfer/VectorTransfer.java +++ b/src/main/java/org/opensearch/knn/index/codec/transfer/VectorTransfer.java @@ -6,10 +6,9 @@ package org.opensearch.knn.index.codec.transfer; import lombok.Data; +import org.apache.lucene.util.BytesRef; import org.opensearch.knn.index.codec.util.SerializationMode; -import java.io.ByteArrayInputStream; - /** * Abstract class to transfer vector value from Java to native memory */ @@ -36,9 +35,9 @@ public VectorTransfer(final long vectorsStreamingMemoryLimit) { /** * Transfer a single vector * - * @param byteStream a vector in byte stream format + * @param bytesRef a vector in bytes format */ - abstract public void transfer(final ByteArrayInputStream byteStream); + abstract public void transfer(final BytesRef bytesRef); /** * Close the transfer @@ -48,8 +47,8 @@ public VectorTransfer(final long vectorsStreamingMemoryLimit) { /** * Get serialization mode of given byte stream * - * @param byteStream byte stream of a vector + * @param bytesRef bytes of a vector * @return serialization mode */ - abstract public SerializationMode getSerializationMode(final ByteArrayInputStream byteStream); + abstract public SerializationMode getSerializationMode(final BytesRef bytesRef); } diff --git a/src/main/java/org/opensearch/knn/index/codec/transfer/VectorTransferByte.java b/src/main/java/org/opensearch/knn/index/codec/transfer/VectorTransferByte.java index fb0f9d470..5e9831708 100644 --- a/src/main/java/org/opensearch/knn/index/codec/transfer/VectorTransferByte.java +++ b/src/main/java/org/opensearch/knn/index/codec/transfer/VectorTransferByte.java @@ -5,10 +5,11 @@ package org.opensearch.knn.index.codec.transfer; +import org.apache.lucene.util.ArrayUtil; +import org.apache.lucene.util.BytesRef; import org.opensearch.knn.index.codec.util.SerializationMode; import org.opensearch.knn.jni.JNICommons; -import java.io.ByteArrayInputStream; import java.util.ArrayList; import java.util.List; @@ -30,11 +31,10 @@ public void init(final long totalLiveDocs) { } @Override - public void transfer(final ByteArrayInputStream byteStream) { - final byte[] vector = byteStream.readAllBytes(); - dimension = vector.length * 8; + public void transfer(final BytesRef bytesRef) { + dimension = bytesRef.length * 8; if (vectorsPerTransfer == Integer.MIN_VALUE) { - vectorsPerTransfer = (vector.length * totalLiveDocs) / vectorsStreamingMemoryLimit; + vectorsPerTransfer = (bytesRef.length * totalLiveDocs) / vectorsStreamingMemoryLimit; // This condition comes if vectorsStreamingMemoryLimit is higher than total number floats to transfer // Doing this will reduce 1 extra trip to JNI layer. if (vectorsPerTransfer == 0) { @@ -42,7 +42,7 @@ public void transfer(final ByteArrayInputStream byteStream) { } } - vectorList.add(vector); + vectorList.add(ArrayUtil.copyOfSubArray(bytesRef.bytes, bytesRef.offset, bytesRef.offset + bytesRef.length)); if (vectorList.size() == vectorsPerTransfer) { transfer(); } @@ -54,7 +54,7 @@ public void close() { } @Override - public SerializationMode getSerializationMode(final ByteArrayInputStream byteStream) { + public SerializationMode getSerializationMode(final BytesRef bytesRef) { return SerializationMode.COLLECTIONS_OF_BYTES; } diff --git a/src/main/java/org/opensearch/knn/index/codec/transfer/VectorTransferFloat.java b/src/main/java/org/opensearch/knn/index/codec/transfer/VectorTransferFloat.java index d5958b375..af6d9490e 100644 --- a/src/main/java/org/opensearch/knn/index/codec/transfer/VectorTransferFloat.java +++ b/src/main/java/org/opensearch/knn/index/codec/transfer/VectorTransferFloat.java @@ -5,12 +5,12 @@ package org.opensearch.knn.index.codec.transfer; +import org.apache.lucene.util.BytesRef; import org.opensearch.knn.index.codec.util.KNNVectorSerializer; import org.opensearch.knn.index.codec.util.KNNVectorSerializerFactory; import org.opensearch.knn.index.codec.util.SerializationMode; import org.opensearch.knn.jni.JNICommons; -import java.io.ByteArrayInputStream; import java.util.ArrayList; import java.util.List; @@ -32,9 +32,9 @@ public void init(final long totalLiveDocs) { } @Override - public void transfer(final ByteArrayInputStream byteStream) { - final KNNVectorSerializer vectorSerializer = KNNVectorSerializerFactory.getSerializerByStreamContent(byteStream); - final float[] vector = vectorSerializer.byteToFloatArray(byteStream); + public void transfer(final BytesRef bytesRef) { + final KNNVectorSerializer vectorSerializer = KNNVectorSerializerFactory.getSerializerByBytesRef(bytesRef); + final float[] vector = vectorSerializer.byteToFloatArray(bytesRef); dimension = vector.length; if (vectorsPerTransfer == Integer.MIN_VALUE) { @@ -58,8 +58,8 @@ public void close() { } @Override - public SerializationMode getSerializationMode(final ByteArrayInputStream byteStream) { - return KNNVectorSerializerFactory.getSerializerModeFromStream(byteStream); + public SerializationMode getSerializationMode(final BytesRef bytesRef) { + return KNNVectorSerializerFactory.getSerializerModeFromBytesRef(bytesRef); } private void transfer() { diff --git a/src/main/java/org/opensearch/knn/index/codec/util/KNNCodecUtil.java b/src/main/java/org/opensearch/knn/index/codec/util/KNNCodecUtil.java index 68b61a070..04aeb337f 100644 --- a/src/main/java/org/opensearch/knn/index/codec/util/KNNCodecUtil.java +++ b/src/main/java/org/opensearch/knn/index/codec/util/KNNCodecUtil.java @@ -14,7 +14,6 @@ import org.opensearch.knn.index.codec.KNN80Codec.KNN80BinaryDocValues; import org.opensearch.knn.index.codec.transfer.VectorTransfer; -import java.io.ByteArrayInputStream; import java.io.IOException; import java.util.ArrayList; import java.util.List; @@ -41,16 +40,22 @@ public static final class Pair { public SerializationMode serializationMode; } + /** + * Extract docIds and vectors from binary doc values. + * + * @param values Binary doc values + * @param vectorTransfer Utility to make transfer + * @return KNNCodecUtil.Pair representing doc ids and corresponding vectors + * @throws IOException thrown when unable to get binary of vectors + */ public static KNNCodecUtil.Pair getPair(final BinaryDocValues values, final VectorTransfer vectorTransfer) throws IOException { List docIdList = new ArrayList<>(); SerializationMode serializationMode = SerializationMode.COLLECTION_OF_FLOATS; vectorTransfer.init(getTotalLiveDocsCount(values)); for (int doc = values.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = values.nextDoc()) { BytesRef bytesref = values.binaryValue(); - try (ByteArrayInputStream byteStream = new ByteArrayInputStream(bytesref.bytes, bytesref.offset, bytesref.length)) { - serializationMode = vectorTransfer.getSerializationMode(byteStream); - vectorTransfer.transfer(byteStream); - } + serializationMode = vectorTransfer.getSerializationMode(bytesref); + vectorTransfer.transfer(bytesref); docIdList.add(doc); } vectorTransfer.close(); diff --git a/src/main/java/org/opensearch/knn/index/codec/util/KNNVectorAsArraySerializer.java b/src/main/java/org/opensearch/knn/index/codec/util/KNNVectorAsArraySerializer.java index 751a229db..929a9aa3e 100644 --- a/src/main/java/org/opensearch/knn/index/codec/util/KNNVectorAsArraySerializer.java +++ b/src/main/java/org/opensearch/knn/index/codec/util/KNNVectorAsArraySerializer.java @@ -5,6 +5,8 @@ package org.opensearch.knn.index.codec.util; +import org.apache.lucene.util.BytesRef; + import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; @@ -31,8 +33,8 @@ public byte[] floatToByteArray(float[] input) { } @Override - public float[] byteToFloatArray(ByteArrayInputStream byteStream) { - try { + public float[] byteToFloatArray(BytesRef bytesRef) { + try (ByteArrayInputStream byteStream = new ByteArrayInputStream(bytesRef.bytes, bytesRef.offset, bytesRef.length)) { final ObjectInputStream objectStream = new ObjectInputStream(byteStream); final float[] vector = (float[]) objectStream.readObject(); return vector; diff --git a/src/main/java/org/opensearch/knn/index/codec/util/KNNVectorAsCollectionOfFloatsSerializer.java b/src/main/java/org/opensearch/knn/index/codec/util/KNNVectorAsCollectionOfFloatsSerializer.java index 13530afc5..31b9b8fea 100644 --- a/src/main/java/org/opensearch/knn/index/codec/util/KNNVectorAsCollectionOfFloatsSerializer.java +++ b/src/main/java/org/opensearch/knn/index/codec/util/KNNVectorAsCollectionOfFloatsSerializer.java @@ -5,7 +5,8 @@ package org.opensearch.knn.index.codec.util; -import java.io.ByteArrayInputStream; +import org.apache.lucene.util.BytesRef; + import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.util.stream.IntStream; @@ -26,15 +27,13 @@ public byte[] floatToByteArray(float[] input) { } @Override - public float[] byteToFloatArray(ByteArrayInputStream byteStream) { - if (byteStream == null || byteStream.available() % BYTES_IN_FLOAT != 0) { + public float[] byteToFloatArray(BytesRef bytesRef) { + if (bytesRef == null || bytesRef.length % BYTES_IN_FLOAT != 0) { throw new IllegalArgumentException("Byte stream cannot be deserialized to array of floats"); } - final byte[] vectorAsByteArray = new byte[byteStream.available()]; - byteStream.read(vectorAsByteArray, 0, byteStream.available()); - final int sizeOfFloatArray = vectorAsByteArray.length / BYTES_IN_FLOAT; + final int sizeOfFloatArray = bytesRef.length / BYTES_IN_FLOAT; final float[] vector = new float[sizeOfFloatArray]; - ByteBuffer.wrap(vectorAsByteArray).asFloatBuffer().get(vector); + ByteBuffer.wrap(bytesRef.bytes, bytesRef.offset, bytesRef.length).asFloatBuffer().get(vector); return vector; } } diff --git a/src/main/java/org/opensearch/knn/index/codec/util/KNNVectorSerializer.java b/src/main/java/org/opensearch/knn/index/codec/util/KNNVectorSerializer.java index 35f1ff5be..f7e7a6743 100644 --- a/src/main/java/org/opensearch/knn/index/codec/util/KNNVectorSerializer.java +++ b/src/main/java/org/opensearch/knn/index/codec/util/KNNVectorSerializer.java @@ -5,7 +5,7 @@ package org.opensearch.knn.index.codec.util; -import java.io.ByteArrayInputStream; +import org.apache.lucene.util.BytesRef; /** * Interface abstracts the vector serializer object that is responsible for serialization and de-serialization of k-NN vector @@ -20,8 +20,9 @@ public interface KNNVectorSerializer { /** * Deserializes all bytes from the stream to array of floats - * @param byteStream stream of bytes that will be used for deserialization to array of floats + * + * @param bytesRef bytes that will be used for deserialization to array of floats * @return array of floats deserialized from the stream */ - float[] byteToFloatArray(ByteArrayInputStream byteStream); + float[] byteToFloatArray(BytesRef bytesRef); } diff --git a/src/main/java/org/opensearch/knn/index/codec/util/KNNVectorSerializerFactory.java b/src/main/java/org/opensearch/knn/index/codec/util/KNNVectorSerializerFactory.java index 23a829dfd..d2a991153 100644 --- a/src/main/java/org/opensearch/knn/index/codec/util/KNNVectorSerializerFactory.java +++ b/src/main/java/org/opensearch/knn/index/codec/util/KNNVectorSerializerFactory.java @@ -6,10 +6,9 @@ package org.opensearch.knn.index.codec.util; import com.google.common.collect.ImmutableMap; +import org.apache.lucene.util.BytesRef; -import java.io.ByteArrayInputStream; import java.io.ObjectStreamConstants; -import java.util.Arrays; import java.util.Map; import static org.opensearch.knn.index.codec.util.SerializationMode.ARRAY; @@ -51,25 +50,24 @@ public static KNNVectorSerializer getDefaultSerializer() { return getSerializerBySerializationMode(COLLECTION_OF_FLOATS); } - public static KNNVectorSerializer getSerializerByStreamContent(final ByteArrayInputStream byteStream) { - final SerializationMode serializationMode = getSerializerModeFromStream(byteStream); + public static KNNVectorSerializer getSerializerByBytesRef(final BytesRef bytesRef) { + final SerializationMode serializationMode = getSerializerModeFromBytesRef(bytesRef); return getSerializerBySerializationMode(serializationMode); } - public static SerializationMode getSerializerModeFromStream(ByteArrayInputStream byteStream) { - int numberOfAvailableBytesInStream = byteStream.available(); - if (numberOfAvailableBytesInStream < ARRAY_HEADER_OFFSET) { - return getSerializerOrThrowError(numberOfAvailableBytesInStream, COLLECTION_OF_FLOATS); + public static SerializationMode getSerializerModeFromBytesRef(BytesRef bytesRef) { + int numberOfAvailableBytes = bytesRef.length; + if (numberOfAvailableBytes < ARRAY_HEADER_OFFSET) { + return getSerializerOrThrowError(numberOfAvailableBytes, COLLECTION_OF_FLOATS); } - final byte[] byteArray = new byte[SERIALIZATION_PROTOCOL_HEADER_PREFIX.length]; - byteStream.read(byteArray, 0, SERIALIZATION_PROTOCOL_HEADER_PREFIX.length); - byteStream.reset(); - // checking if stream protocol grammar in header is valid for serialized array - if (Arrays.equals(SERIALIZATION_PROTOCOL_HEADER_PREFIX, byteArray)) { - int numberOfAvailableBytesAfterHeader = numberOfAvailableBytesInStream - ARRAY_HEADER_OFFSET; - return getSerializerOrThrowError(numberOfAvailableBytesAfterHeader, ARRAY); + + for (int i = 0; i < SERIALIZATION_PROTOCOL_HEADER_PREFIX.length; i++) { + if (bytesRef.bytes[i + bytesRef.offset] != SERIALIZATION_PROTOCOL_HEADER_PREFIX[i]) { + return getSerializerOrThrowError(numberOfAvailableBytes, COLLECTION_OF_FLOATS); + } } - return getSerializerOrThrowError(numberOfAvailableBytesInStream, COLLECTION_OF_FLOATS); + int numberOfAvailableBytesAfterHeader = numberOfAvailableBytes - ARRAY_HEADER_OFFSET; + return getSerializerOrThrowError(numberOfAvailableBytesAfterHeader, ARRAY); } private static SerializationMode getSerializerOrThrowError(int numberOfRemainingBytes, final SerializationMode serializationMode) { diff --git a/src/main/java/org/opensearch/knn/index/query/filtered/FilteredIdsKNNIterator.java b/src/main/java/org/opensearch/knn/index/query/filtered/FilteredIdsKNNIterator.java index fb153989a..7e554fb7d 100644 --- a/src/main/java/org/opensearch/knn/index/query/filtered/FilteredIdsKNNIterator.java +++ b/src/main/java/org/opensearch/knn/index/query/filtered/FilteredIdsKNNIterator.java @@ -14,7 +14,6 @@ import org.opensearch.knn.index.codec.util.KNNVectorSerializer; import org.opensearch.knn.index.codec.util.KNNVectorSerializerFactory; -import java.io.ByteArrayInputStream; import java.io.IOException; /** @@ -72,9 +71,8 @@ public float score() { protected float computeScore() throws IOException { final BytesRef value = binaryDocValues.binaryValue(); - final ByteArrayInputStream byteStream = new ByteArrayInputStream(value.bytes, value.offset, value.length); - final KNNVectorSerializer vectorSerializer = KNNVectorSerializerFactory.getSerializerByStreamContent(byteStream); - final float[] vector = vectorSerializer.byteToFloatArray(byteStream); + final KNNVectorSerializer vectorSerializer = KNNVectorSerializerFactory.getSerializerByBytesRef(value); + final float[] vector = vectorSerializer.byteToFloatArray(value); // Calculates a similarity score between the two vectors with a specified function. Higher similarity // scores correspond to closer vectors. return spaceType.getKnnVectorSimilarityFunction().compare(queryVector, vector); diff --git a/src/test/java/org/opensearch/knn/index/codec/transfer/VectorTransferByteTests.java b/src/test/java/org/opensearch/knn/index/codec/transfer/VectorTransferByteTests.java index abcd89a0e..7e837fbf2 100644 --- a/src/test/java/org/opensearch/knn/index/codec/transfer/VectorTransferByteTests.java +++ b/src/test/java/org/opensearch/knn/index/codec/transfer/VectorTransferByteTests.java @@ -7,10 +7,10 @@ import junit.framework.TestCase; import lombok.SneakyThrows; +import org.apache.lucene.util.BytesRef; import org.opensearch.knn.index.codec.util.SerializationMode; import org.opensearch.knn.jni.JNICommons; -import java.io.ByteArrayInputStream; import java.io.IOException; import java.util.Random; @@ -19,17 +19,17 @@ public class VectorTransferByteTests extends TestCase { @SneakyThrows public void testTransfer_whenCalled_thenAdded() { - final ByteArrayInputStream bais1 = getByteArrayOfVectors(20); - final ByteArrayInputStream bais2 = getByteArrayOfVectors(20); + final BytesRef bytesRef1 = getByteArrayOfVectors(20); + final BytesRef bytesRef2 = getByteArrayOfVectors(20); VectorTransferByte vectorTransfer = new VectorTransferByte(1000); try { vectorTransfer.init(2); - vectorTransfer.transfer(bais1); + vectorTransfer.transfer(bytesRef1); // flush is not called assertEquals(0, vectorTransfer.getVectorAddress()); - vectorTransfer.transfer(bais2); + vectorTransfer.transfer(bytesRef2); // flush should be called assertNotEquals(0, vectorTransfer.getVectorAddress()); } finally { @@ -41,16 +41,16 @@ public void testTransfer_whenCalled_thenAdded() { @SneakyThrows public void testSerializationMode_whenCalled_thenReturn() { - final ByteArrayInputStream bais = getByteArrayOfVectors(20); + final BytesRef bytesRef = getByteArrayOfVectors(20); VectorTransferByte vectorTransfer = new VectorTransferByte(1000); // Verify - assertEquals(SerializationMode.COLLECTIONS_OF_BYTES, vectorTransfer.getSerializationMode(bais)); + assertEquals(SerializationMode.COLLECTIONS_OF_BYTES, vectorTransfer.getSerializationMode(bytesRef)); } - private ByteArrayInputStream getByteArrayOfVectors(int vectorLength) throws IOException { + private BytesRef getByteArrayOfVectors(int vectorLength) throws IOException { byte[] vector = new byte[vectorLength]; new Random().nextBytes(vector); - return new ByteArrayInputStream(vector); + return new BytesRef(vector); } } diff --git a/src/test/java/org/opensearch/knn/index/codec/transfer/VectorTransferFloatTests.java b/src/test/java/org/opensearch/knn/index/codec/transfer/VectorTransferFloatTests.java index 1de513a0b..1f36f320d 100644 --- a/src/test/java/org/opensearch/knn/index/codec/transfer/VectorTransferFloatTests.java +++ b/src/test/java/org/opensearch/knn/index/codec/transfer/VectorTransferFloatTests.java @@ -7,10 +7,10 @@ import junit.framework.TestCase; import lombok.SneakyThrows; +import org.apache.lucene.util.BytesRef; import org.opensearch.knn.index.codec.util.KNNVectorSerializerFactory; import org.opensearch.knn.jni.JNICommons; -import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.DataOutputStream; import java.io.IOException; @@ -22,17 +22,17 @@ public class VectorTransferFloatTests extends TestCase { @SneakyThrows public void testTransfer_whenCalled_thenAdded() { - final ByteArrayInputStream bais1 = getByteArrayOfVectors(20); - final ByteArrayInputStream bais2 = getByteArrayOfVectors(20); + final BytesRef bytesRef1 = getByteArrayOfVectors(20); + final BytesRef bytesRef2 = getByteArrayOfVectors(20); VectorTransferFloat vectorTransfer = new VectorTransferFloat(1000); try { vectorTransfer.init(2); - vectorTransfer.transfer(bais1); + vectorTransfer.transfer(bytesRef1); // flush is not called assertEquals(0, vectorTransfer.getVectorAddress()); - vectorTransfer.transfer(bais2); + vectorTransfer.transfer(bytesRef2); // flush should be called assertNotEquals(0, vectorTransfer.getVectorAddress()); } finally { @@ -44,14 +44,14 @@ public void testTransfer_whenCalled_thenAdded() { @SneakyThrows public void testSerializationMode_whenCalled_thenReturn() { - final ByteArrayInputStream bais = getByteArrayOfVectors(20); + final BytesRef bytesRef = getByteArrayOfVectors(20); VectorTransferFloat vectorTransfer = new VectorTransferFloat(1000); // Verify - assertEquals(KNNVectorSerializerFactory.getSerializerModeFromStream(bais), vectorTransfer.getSerializationMode(bais)); + assertEquals(KNNVectorSerializerFactory.getSerializerModeFromBytesRef(bytesRef), vectorTransfer.getSerializationMode(bytesRef)); } - private ByteArrayInputStream getByteArrayOfVectors(int vectorLength) throws IOException { + private BytesRef getByteArrayOfVectors(int vectorLength) throws IOException { float[] vector = new float[vectorLength]; IntStream.range(0, vectorLength).forEach(index -> vector[index] = new Random().nextFloat()); @@ -61,6 +61,6 @@ private ByteArrayInputStream getByteArrayOfVectors(int vectorLength) throws IOEx ds.writeFloat(f); } final byte[] vectorAsCollectionOfFloats = bas.toByteArray(); - return new ByteArrayInputStream(vectorAsCollectionOfFloats); + return new BytesRef(vectorAsCollectionOfFloats); } } diff --git a/src/test/java/org/opensearch/knn/index/codec/util/KNNCodecUtilTests.java b/src/test/java/org/opensearch/knn/index/codec/util/KNNCodecUtilTests.java index 04c1c038f..2ff0f08e5 100644 --- a/src/test/java/org/opensearch/knn/index/codec/util/KNNCodecUtilTests.java +++ b/src/test/java/org/opensearch/knn/index/codec/util/KNNCodecUtilTests.java @@ -11,7 +11,6 @@ import org.apache.lucene.util.BytesRef; import org.opensearch.knn.index.codec.transfer.VectorTransfer; -import java.io.ByteArrayInputStream; import java.util.Arrays; import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS; @@ -35,7 +34,7 @@ public void testGetPair_whenCalled_thenReturn() { when(binaryDocValues.binaryValue()).thenReturn(bytesRef); VectorTransfer vectorTransfer = mock(VectorTransfer.class); - when(vectorTransfer.getSerializationMode(any(ByteArrayInputStream.class))).thenReturn(SerializationMode.COLLECTIONS_OF_BYTES); + when(vectorTransfer.getSerializationMode(any(BytesRef.class))).thenReturn(SerializationMode.COLLECTIONS_OF_BYTES); when(vectorTransfer.getVectorAddress()).thenReturn(vectorAddress); when(vectorTransfer.getDimension()).thenReturn(dimension); @@ -44,8 +43,8 @@ public void testGetPair_whenCalled_thenReturn() { // Verify verify(vectorTransfer).init(liveDocCount); - verify(vectorTransfer).getSerializationMode(any(ByteArrayInputStream.class)); - verify(vectorTransfer).transfer(any(ByteArrayInputStream.class)); + verify(vectorTransfer).getSerializationMode(any(BytesRef.class)); + verify(vectorTransfer).transfer(any(BytesRef.class)); verify(vectorTransfer).close(); assertTrue(Arrays.equals(docId, pair.docs)); diff --git a/src/test/java/org/opensearch/knn/index/codec/util/KNNVectorSerializerTests.java b/src/test/java/org/opensearch/knn/index/codec/util/KNNVectorSerializerTests.java index 1d08df6a0..9ec6d937a 100644 --- a/src/test/java/org/opensearch/knn/index/codec/util/KNNVectorSerializerTests.java +++ b/src/test/java/org/opensearch/knn/index/codec/util/KNNVectorSerializerTests.java @@ -5,9 +5,9 @@ package org.opensearch.knn.index.codec.util; +import org.apache.lucene.util.BytesRef; import org.opensearch.knn.KNNTestCase; -import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.DataOutputStream; import java.io.ObjectOutputStream; @@ -26,14 +26,11 @@ public void testVectorSerializerFactory() throws Exception { final DataOutputStream ds = new DataOutputStream(bas); for (float f : vector) ds.writeFloat(f); - final byte[] vectorAsCollectionOfFloats = bas.toByteArray(); - final ByteArrayInputStream bais = new ByteArrayInputStream(vectorAsCollectionOfFloats); - bais.reset(); - + final BytesRef vectorAsCollectionOfFloats = new BytesRef(bas.toByteArray()); final KNNVectorSerializer defaultSerializer = KNNVectorSerializerFactory.getDefaultSerializer(); assertNotNull(defaultSerializer); - final float[] actualDeserializedVector = defaultSerializer.byteToFloatArray(bais); + final float[] actualDeserializedVector = defaultSerializer.byteToFloatArray(vectorAsCollectionOfFloats); assertNotNull(actualDeserializedVector); assertArrayEquals(vector, actualDeserializedVector, 0.1f); @@ -46,18 +43,16 @@ public void testVectorSerializerFactory() throws Exception { assertNotNull(collectionOfFloatsSerializer); } - public void testVectorSerializerFactory_throwExceptionForStreamWithUnsupportedDataType() throws Exception { + public void testVectorSerializerFactory_throwExceptionForBytesWithUnsupportedDataType() throws Exception { // prepare array of chars that is not supported by serializer factory. expected behavior is to fail final char[] arrayOfChars = new char[] { 'a', 'b', 'c' }; final ByteArrayOutputStream bas = new ByteArrayOutputStream(); final DataOutputStream ds = new DataOutputStream(bas); for (char ch : arrayOfChars) ds.writeChar(ch); - final byte[] vectorAsCollectionOfChars = bas.toByteArray(); - final ByteArrayInputStream bais = new ByteArrayInputStream(vectorAsCollectionOfChars); - bais.reset(); + final BytesRef vectorAsCollectionOfChars = new BytesRef(bas.toByteArray()); - expectThrows(RuntimeException.class, () -> KNNVectorSerializerFactory.getSerializerByStreamContent(bais)); + expectThrows(RuntimeException.class, () -> KNNVectorSerializerFactory.getSerializerByBytesRef(vectorAsCollectionOfChars)); } public void testVectorAsArraySerializer() throws Exception { @@ -66,21 +61,17 @@ public void testVectorAsArraySerializer() throws Exception { final ByteArrayOutputStream byteStream = new ByteArrayOutputStream(); final ObjectOutputStream objectStream = new ObjectOutputStream(byteStream); objectStream.writeObject(vector); - final byte[] serializedVector = byteStream.toByteArray(); - final ByteArrayInputStream bais = new ByteArrayInputStream(serializedVector); - - final KNNVectorSerializer vectorSerializer = KNNVectorSerializerFactory.getSerializerByStreamContent(bais); + final BytesRef serializedVector = new BytesRef(byteStream.toByteArray()); + final KNNVectorSerializer vectorSerializer = KNNVectorSerializerFactory.getSerializerByBytesRef(serializedVector); // testing serialization - bais.reset(); final byte[] actualSerializedVector = vectorSerializer.floatToByteArray(vector); assertNotNull(actualSerializedVector); - assertArrayEquals(serializedVector, actualSerializedVector); + assertArrayEquals(serializedVector.bytes, actualSerializedVector); // testing deserialization - bais.reset(); - final float[] actualDeserializedVector = vectorSerializer.byteToFloatArray(bais); + final float[] actualDeserializedVector = vectorSerializer.byteToFloatArray(serializedVector); assertNotNull(actualDeserializedVector); assertArrayEquals(vector, actualDeserializedVector, 0.1f); @@ -94,26 +85,37 @@ public void testVectorAsCollectionOfFloatsSerializer() throws Exception { final DataOutputStream ds = new DataOutputStream(bas); for (float f : vector) ds.writeFloat(f); - final byte[] vectorAsCollectionOfFloats = bas.toByteArray(); - final ByteArrayInputStream bais = new ByteArrayInputStream(vectorAsCollectionOfFloats); - - final KNNVectorSerializer vectorSerializer = KNNVectorSerializerFactory.getSerializerByStreamContent(bais); + final BytesRef vectorAsCollectionOfFloats = new BytesRef(bas.toByteArray()); + final KNNVectorSerializer vectorSerializer = KNNVectorSerializerFactory.getSerializerByBytesRef(vectorAsCollectionOfFloats); // testing serialization - bais.reset(); final byte[] actualSerializedVector = vectorSerializer.floatToByteArray(vector); assertNotNull(actualSerializedVector); - assertArrayEquals(vectorAsCollectionOfFloats, actualSerializedVector); + assertArrayEquals(vectorAsCollectionOfFloats.bytes, actualSerializedVector); // testing deserialization - bais.reset(); - final float[] actualDeserializedVector = vectorSerializer.byteToFloatArray(bais); + final float[] actualDeserializedVector = vectorSerializer.byteToFloatArray(vectorAsCollectionOfFloats); assertNotNull(actualDeserializedVector); assertArrayEquals(vector, actualDeserializedVector, 0.1f); } + public void testVectorSerializer_whenVectorBytesOffset_thenSuccess() { + final float[] vector = getArrayOfRandomFloats(20); + int offset = randomInt(4); + for (SerializationMode serializationMode : SerializationMode.values()) { + final KNNVectorSerializer vectorSerializer = KNNVectorSerializerFactory.getSerializerBySerializationMode(serializationMode); + assertNotNull(vectorSerializer); + byte[] bytes = vectorSerializer.floatToByteArray(vector); + byte[] bytesWithOffset = new byte[bytes.length + 2 * offset]; + System.arraycopy(bytes, 0, bytesWithOffset, offset, bytes.length); + BytesRef serializedVector = new BytesRef(bytesWithOffset, offset, bytes.length); + float[] deserializedVector = vectorSerializer.byteToFloatArray(serializedVector); + assertArrayEquals(vector, deserializedVector, 0.1f); + } + } + private float[] getArrayOfRandomFloats(int arrayLength) { float[] vector = new float[arrayLength]; IntStream.range(0, arrayLength).forEach(index -> vector[index] = random.nextFloat()); diff --git a/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperUtilTests.java b/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperUtilTests.java index c7c945bfa..ad7f49cb6 100644 --- a/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperUtilTests.java +++ b/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperUtilTests.java @@ -12,6 +12,7 @@ package org.opensearch.knn.index.mapper; import org.apache.lucene.document.StoredField; +import org.apache.lucene.util.BytesRef; import org.opensearch.knn.KNNTestCase; import org.opensearch.knn.common.KNNConstants; import org.opensearch.knn.index.KNNMethodContext; @@ -24,7 +25,6 @@ import org.opensearch.knn.indices.ModelMetadata; import org.opensearch.knn.indices.ModelState; -import java.io.ByteArrayInputStream; import java.util.Arrays; import java.util.Collections; @@ -51,13 +51,8 @@ public void testStoredFields_whenVectorIsByteType_thenSucceed() { public void testStoredFields_whenVectorIsFloatType_thenSucceed() { StoredField storedField = KNNVectorFieldMapperUtil.createStoredFieldForFloatVector(TEST_FIELD_NAME, TEST_FLOAT_VECTOR); assertEquals(TEST_FIELD_NAME, storedField.name()); - byte[] bytes = storedField.binaryValue().bytes; - ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(bytes, 0, bytes.length); - assertArrayEquals( - TEST_FLOAT_VECTOR, - KNNVectorSerializerFactory.getDefaultSerializer().byteToFloatArray(byteArrayInputStream), - 0.001f - ); + BytesRef bytes = new BytesRef(storedField.binaryValue().bytes); + assertArrayEquals(TEST_FLOAT_VECTOR, KNNVectorSerializerFactory.getDefaultSerializer().byteToFloatArray(bytes), 0.001f); Object vector = KNNVectorFieldMapperUtil.deserializeStoredVector(storedField.binaryValue(), VectorDataType.FLOAT); assertTrue(vector instanceof float[]); From a7603e8fcc4293fa859583faf86ac5fad53d0060 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Tue, 16 Jul 2024 13:36:08 -0700 Subject: [PATCH 291/416] Fixed LeafReaders casting errors to SegmentReaders when segment replication is enabled during search (#1808) (#1836) Signed-off-by: Navneet Verma Co-authored-by: Navneet Verma --- CHANGELOG.md | 1 + .../opensearch/knn/index/KNNIndexShard.java | 4 +- .../opensearch/knn/index/query/KNNWeight.java | 6 +- .../knn/index/SegmentReplicationIT.java | 94 +++++++++++++++++++ .../org/opensearch/knn/KNNRestTestCase.java | 35 ++++++- 5 files changed, 134 insertions(+), 6 deletions(-) create mode 100644 src/test/java/org/opensearch/knn/index/SegmentReplicationIT.java diff --git a/CHANGELOG.md b/CHANGELOG.md index bfa785204..0118b5d95 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), * Switch from byte stream to byte ref for serde [#1825](https://github.com/opensearch-project/k-NN/pull/1825) ### Bug Fixes * Fixing the arithmetic to find the number of vectors to stream from java to jni layer.[#1804](https://github.com/opensearch-project/k-NN/pull/1804) +* Fixed LeafReaders casting errors to SegmentReaders when segment replication is enabled during search.[#1808](https://github.com/opensearch-project/k-NN/pull/1808) * Release memory properly for an array type [#1820](https://github.com/opensearch-project/k-NN/pull/1820) * FIX Same Suffix Cause Recall Drop to zero [#1802](https://github.com/opensearch-project/k-NN/pull/1802) ### Infrastructure diff --git a/src/main/java/org/opensearch/knn/index/KNNIndexShard.java b/src/main/java/org/opensearch/knn/index/KNNIndexShard.java index 2a44eec91..431c0849c 100644 --- a/src/main/java/org/opensearch/knn/index/KNNIndexShard.java +++ b/src/main/java/org/opensearch/knn/index/KNNIndexShard.java @@ -10,12 +10,12 @@ import lombok.Getter; import lombok.extern.log4j.Log4j2; import org.apache.lucene.index.FieldInfo; -import org.apache.lucene.index.FilterLeafReader; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.SegmentReader; import org.apache.lucene.store.FSDirectory; import org.apache.lucene.store.FilterDirectory; +import org.opensearch.common.lucene.Lucene; import org.opensearch.index.engine.Engine; import org.opensearch.index.shard.IndexShard; import org.opensearch.knn.index.mapper.KNNVectorFieldMapper; @@ -160,7 +160,7 @@ List getEngineFileContexts(IndexReader indexReader, KNNEngine List engineFiles = new ArrayList<>(); for (LeafReaderContext leafReaderContext : indexReader.leaves()) { - SegmentReader reader = (SegmentReader) FilterLeafReader.unwrap(leafReaderContext.reader()); + SegmentReader reader = Lucene.segmentReader(leafReaderContext.reader()); Path shardPath = ((FSDirectory) FilterDirectory.unwrap(reader.directory())).getDirectory(); String fileExtension = reader.getSegmentInfo().info.getUseCompoundFile() ? knnEngine.getCompoundExtension() diff --git a/src/main/java/org/opensearch/knn/index/query/KNNWeight.java b/src/main/java/org/opensearch/knn/index/query/KNNWeight.java index 503bea42d..a061e740e 100644 --- a/src/main/java/org/opensearch/knn/index/query/KNNWeight.java +++ b/src/main/java/org/opensearch/knn/index/query/KNNWeight.java @@ -11,7 +11,6 @@ import org.apache.lucene.index.BinaryDocValues; import org.apache.lucene.index.DocValues; import org.apache.lucene.index.FieldInfo; -import org.apache.lucene.index.FilterLeafReader; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.SegmentReader; import org.apache.lucene.search.DocIdSetIterator; @@ -29,6 +28,7 @@ import org.apache.lucene.util.DocIdSetBuilder; import org.apache.lucene.util.FixedBitSet; import org.opensearch.common.io.PathUtils; +import org.opensearch.common.lucene.Lucene; import org.opensearch.knn.common.KNNConstants; import org.opensearch.knn.index.KNNSettings; import org.opensearch.knn.index.SpaceType; @@ -203,7 +203,7 @@ private int[] bitSetToIntArray(final BitSet bitSet) { private Map doANNSearch(final LeafReaderContext context, final BitSet filterIdsBitSet, final int cardinality) throws IOException { - SegmentReader reader = (SegmentReader) FilterLeafReader.unwrap(context.reader()); + final SegmentReader reader = Lucene.segmentReader(context.reader()); String directory = ((FSDirectory) FilterDirectory.unwrap(reader.directory())).getDirectory().toString(); FieldInfo fieldInfo = reader.getFieldInfos().fieldInfo(knnQuery.getField()); @@ -394,7 +394,7 @@ private Map doExactSearch(final LeafReaderContext leafReaderCont } private KNNIterator getFilteredKNNIterator(final LeafReaderContext leafReaderContext, final BitSet filterIdsBitSet) throws IOException { - final SegmentReader reader = (SegmentReader) FilterLeafReader.unwrap(leafReaderContext.reader()); + final SegmentReader reader = Lucene.segmentReader(leafReaderContext.reader()); final FieldInfo fieldInfo = reader.getFieldInfos().fieldInfo(knnQuery.getField()); final BinaryDocValues values = DocValues.getBinary(leafReaderContext.reader(), fieldInfo.getName()); final SpaceType spaceType = getSpaceType(fieldInfo); diff --git a/src/test/java/org/opensearch/knn/index/SegmentReplicationIT.java b/src/test/java/org/opensearch/knn/index/SegmentReplicationIT.java new file mode 100644 index 000000000..02b0fcf71 --- /dev/null +++ b/src/test/java/org/opensearch/knn/index/SegmentReplicationIT.java @@ -0,0 +1,94 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.knn.index; + +import lombok.SneakyThrows; +import lombok.extern.log4j.Log4j2; +import org.apache.http.util.EntityUtils; +import org.junit.Assert; +import org.opensearch.client.Response; +import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.knn.KNNRestTestCase; +import org.opensearch.knn.KNNResult; + +import java.util.List; + +/** + * This IT class contains will contain special cases of IT for segment replication behavior. + * All the index created in this test will have replication type SEGMENT, number of replicas: 1 and should be run on + * at-least 2 node configuration. + */ +@Log4j2 +public class SegmentReplicationIT extends KNNRestTestCase { + private static final String INDEX_NAME = "segment-replicated-knn-index"; + + @SneakyThrows + public void testSearchOnReplicas_whenIndexHasDeletedDocs_thenSuccess() { + createKnnIndex(INDEX_NAME, getKNNSegmentReplicatedIndexSettings(), createKNNIndexMethodFieldMapping(FIELD_NAME, 2)); + + Float[] vector = { 1.3f, 2.2f }; + int docsInIndex = 10; + + for (int i = 0; i < docsInIndex; i++) { + addKnnDoc(INDEX_NAME, Integer.toString(i), FIELD_NAME, vector); + } + refreshIndex(INDEX_NAME); + int deleteDocs = 5; + for (int i = 0; i < deleteDocs; i++) { + deleteKnnDoc(INDEX_NAME, Integer.toString(i)); + } + refreshIndex(INDEX_NAME); + // sleep for 5sec to ensure data is replicated. I don't have a better way here to know if segments has been + // replicated. + Thread.sleep(5000); + // validate warmup is successful or not. + doKnnWarmup(List.of(INDEX_NAME)); + + XContentBuilder queryBuilder = XContentFactory.jsonBuilder().startObject().startObject("query"); + queryBuilder.startObject("knn"); + queryBuilder.startObject(FIELD_NAME); + queryBuilder.field("vector", vector); + queryBuilder.field("k", docsInIndex); + queryBuilder.endObject().endObject().endObject().endObject(); + + // validate primaries are working + Response searchResponse = performSearch(INDEX_NAME, queryBuilder.toString(), "preference=_primary"); + String responseBody = EntityUtils.toString(searchResponse.getEntity()); + List knnResults = parseSearchResponse(responseBody, FIELD_NAME); + assertEquals(docsInIndex - deleteDocs, knnResults.size()); + + if (ensureMinDataNodesCountForTestingQueriesOnReplica()) { + // validate replicas are working + searchResponse = performSearch(INDEX_NAME, queryBuilder.toString(), "preference=_replica"); + responseBody = EntityUtils.toString(searchResponse.getEntity()); + knnResults = parseSearchResponse(responseBody, FIELD_NAME); + assertEquals(docsInIndex - deleteDocs, knnResults.size()); + } + } + + private boolean ensureMinDataNodesCountForTestingQueriesOnReplica() { + int dataNodeCount = getDataNodeCount(); + if (dataNodeCount <= 1) { + log.warn( + "Not running segment replication tests named: " + + "testSearchOnReplicas_whenIndexHasDeletedDocs_thenSuccess, as data nodes count is not atleast 2. " + + "Actual datanode count : {}", + dataNodeCount + ); + Assert.assertTrue(true); + // making the test successful because we don't want to break already running tests. + return false; + } + return true; + } +} diff --git a/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java b/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java index e71399c1e..179caf951 100644 --- a/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java +++ b/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java @@ -235,7 +235,11 @@ protected Response searchExists(String index, ExistsQueryBuilder existsQueryBuil } protected Response performSearch(final String indexName, final String query) throws IOException { - Request request = new Request("POST", "/" + indexName + "/_search"); + return performSearch(indexName, query, ""); + } + + protected Response performSearch(final String indexName, final String query, final String urlParameters) throws IOException { + Request request = new Request("POST", "/" + indexName + "/_search?" + urlParameters); request.setJsonEntity(query); Response response = client().performRequest(request); @@ -667,6 +671,35 @@ protected Settings getKNNDefaultIndexSettings() { return Settings.builder().put("number_of_shards", 1).put("number_of_replicas", 0).put("index.knn", true).build(); } + protected Settings getKNNSegmentReplicatedIndexSettings() { + return Settings.builder() + .put("number_of_shards", 1) + .put("number_of_replicas", 1) + .put("index.knn", true) + .put("index.replication.type", "SEGMENT") + .build(); + } + + @SneakyThrows + protected int getDataNodeCount() { + Request request = new Request("GET", "_nodes/stats?filter_path=nodes.*.roles"); + + Response response = client().performRequest(request); + assertEquals(RestStatus.OK, RestStatus.fromCode(response.getStatusLine().getStatusCode())); + String responseBody = EntityUtils.toString(response.getEntity()); + + Map responseMap = createParser(MediaTypeRegistry.getDefaultMediaType().xContent(), responseBody).map(); + Map nodesInfo = (Map) responseMap.get("nodes"); + int dataNodeCount = 0; + for (String key : nodesInfo.keySet()) { + Map> nodeRoles = (Map>) nodesInfo.get(key); + if (nodeRoles.get("roles").contains("data")) { + dataNodeCount++; + } + } + return dataNodeCount; + } + /** * Get Stats from KNN Plugin */ From 765e9ebe8acf7cad20bb4ad5f193d96e7c96d6c6 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Tue, 16 Jul 2024 16:30:08 -0700 Subject: [PATCH 292/416] Implement script scoring for knn field with binary data type (#1826) (#1838) Signed-off-by: Heemin Kim (cherry picked from commit 9705144b433c8dcdfd4960289c239dae652ba277) Co-authored-by: Heemin Kim --- CHANGELOG.md | 1 + .../opensearch/knn/index/VectorDataType.java | 9 +- .../mapper/KNNVectorFieldMapperUtil.java | 10 +- .../knn/plugin/script/KNNScoringSpace.java | 308 ++++++++---------- .../plugin/script/KNNScoringSpaceUtil.java | 20 +- .../knn/plugin/script/KNNScoringUtil.java | 1 + .../knn/index/VectorDataTypeTests.java | 6 +- .../mapper/KNNVectorFieldMapperUtilTests.java | 17 +- .../plugin/script/KNNScoringSpaceTests.java | 106 ++++-- .../script/KNNScoringSpaceUtilTests.java | 35 ++ 10 files changed, 304 insertions(+), 209 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0118b5d95..8501e4842 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), * Adds dynamic query parameter ef_search [#1783](https://github.com/opensearch-project/k-NN/pull/1783) * Adds dynamic query parameter ef_search in radial search faiss engine [#1790](https://github.com/opensearch-project/k-NN/pull/1790) * Add binary format support with HNSW method in Faiss Engine [#1781](https://github.com/opensearch-project/k-NN/pull/1781) +* Add script scoring support for knn field with binary data type [#1826](https://github.com/opensearch-project/k-NN/pull/1826) ### Enhancements * Switch from byte stream to byte ref for serde [#1825](https://github.com/opensearch-project/k-NN/pull/1825) ### Bug Fixes diff --git a/src/main/java/org/opensearch/knn/index/VectorDataType.java b/src/main/java/org/opensearch/knn/index/VectorDataType.java index c9fd131a7..b36d508f1 100644 --- a/src/main/java/org/opensearch/knn/index/VectorDataType.java +++ b/src/main/java/org/opensearch/knn/index/VectorDataType.java @@ -39,7 +39,14 @@ public FieldType createKnnVectorFieldType(int dimension, VectorSimilarityFunctio @Override public float[] getVectorFromBytesRef(BytesRef binaryValue) { - throw new IllegalStateException("Unsupported method"); + float[] vector = new float[binaryValue.length]; + int i = 0; + int j = binaryValue.offset; + + while (i < binaryValue.length) { + vector[i++] = binaryValue.bytes[j++]; + } + return vector; } }, BYTE("byte") { diff --git a/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperUtil.java b/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperUtil.java index 03f369d0d..af482813f 100644 --- a/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperUtil.java +++ b/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperUtil.java @@ -228,19 +228,21 @@ public static Object deserializeStoredVector(BytesRef storedVector, VectorDataTy } /** - * Get the expected dimensions from a specified knn vector field type. + * Get the expected vector length from a specified knn vector field type. * * If the field is model-based, get dimensions from model metadata. + * For binary vector, the expected vector length is dimension divided by 8 + * * @param knnVectorFieldType knn vector field type - * @return expected dimensions + * @return expected vector length */ - public static int getExpectedDimensions(final KNNVectorFieldMapper.KNNVectorFieldType knnVectorFieldType) { + public static int getExpectedVectorLength(final KNNVectorFieldMapper.KNNVectorFieldType knnVectorFieldType) { int expectedDimensions = knnVectorFieldType.getDimension(); if (isModelBasedIndex(expectedDimensions)) { ModelMetadata modelMetadata = getModelMetadataForField(knnVectorFieldType); expectedDimensions = modelMetadata.getDimension(); } - return expectedDimensions; + return VectorDataType.BINARY == knnVectorFieldType.getVectorDataType() ? expectedDimensions / 8 : expectedDimensions; } private static boolean isModelBasedIndex(int expectedDimensions) { diff --git a/src/main/java/org/opensearch/knn/plugin/script/KNNScoringSpace.java b/src/main/java/org/opensearch/knn/plugin/script/KNNScoringSpace.java index 8105539ba..04d08fc9c 100644 --- a/src/main/java/org/opensearch/knn/plugin/script/KNNScoringSpace.java +++ b/src/main/java/org/opensearch/knn/plugin/script/KNNScoringSpace.java @@ -17,11 +17,13 @@ import java.io.IOException; import java.math.BigInteger; +import java.util.Locale; import java.util.Map; import java.util.function.BiFunction; import static org.opensearch.knn.plugin.script.KNNScoringSpaceUtil.getVectorMagnitudeSquared; import static org.opensearch.knn.plugin.script.KNNScoringSpaceUtil.isBinaryFieldType; +import static org.opensearch.knn.plugin.script.KNNScoringSpaceUtil.isBinaryVectorDataType; import static org.opensearch.knn.plugin.script.KNNScoringSpaceUtil.isKNNVectorFieldType; import static org.opensearch.knn.plugin.script.KNNScoringSpaceUtil.isLongFieldType; import static org.opensearch.knn.plugin.script.KNNScoringSpaceUtil.parseToBigInteger; @@ -34,7 +36,7 @@ public interface KNNScoringSpace { * Return the correct scoring script for a given query. The scoring script * * @param params Map of parameters - * @param field Fieldname + * @param field field name * @param lookup SearchLookup * @param ctx ctx LeafReaderContext to be used for scoring documents * @param searcher IndexSearcher @@ -44,28 +46,20 @@ public interface KNNScoringSpace { ScoreScript getScoreScript(Map params, String field, SearchLookup lookup, LeafReaderContext ctx, IndexSearcher searcher) throws IOException; - class L2 implements KNNScoringSpace { - + /** + * Base class to represent linear space + * + * As of now, all supporting spaces are linear space except hamming. + * LinearSpace does not support binary vector. + */ + abstract class LinearSpace implements KNNScoringSpace { float[] processedQuery; BiFunction scoringMethod; - /** - * Constructor for L2 scoring space. L2 scoring space expects values to be of type float[]. - * - * @param query Query object that, along with the doc values, will be used to compute L2 score - * @param fieldType FieldType for the doc values that will be used - */ - public L2(Object query, MappedFieldType fieldType) { - if (!isKNNVectorFieldType(fieldType)) { - throw new IllegalArgumentException("Incompatible field_type for l2 space. The field type must " + "be knn_vector."); - } - - this.processedQuery = parseToFloatArray( - query, - KNNVectorFieldMapperUtil.getExpectedDimensions((KNNVectorFieldMapper.KNNVectorFieldType) fieldType), - ((KNNVectorFieldMapper.KNNVectorFieldType) fieldType).getVectorDataType() - ); - this.scoringMethod = (float[] q, float[] v) -> 1 / (1 + KNNScoringUtil.l2Squared(q, v)); + public LinearSpace(final Object query, final MappedFieldType fieldType, final String spaceName) { + KNNVectorFieldMapper.KNNVectorFieldType knnVectorFieldType = toKNNVectorFieldType(fieldType, spaceName); + this.processedQuery = getProcessedQuery(query, knnVectorFieldType); + this.scoringMethod = getScoringMethod(this.processedQuery); } public ScoreScript getScoreScript( @@ -77,43 +71,95 @@ public ScoreScript getScoreScript( ) throws IOException { return new KNNScoreScript.KNNVectorType(params, this.processedQuery, field, this.scoringMethod, lookup, ctx, searcher); } - } - - class CosineSimilarity implements KNNScoringSpace { - float[] processedQuery; - BiFunction scoringMethod; + private KNNVectorFieldMapper.KNNVectorFieldType toKNNVectorFieldType(final MappedFieldType fieldType, final String spaceName) { + if (isKNNVectorFieldType(fieldType) == false) { + throw new IllegalArgumentException( + String.format(Locale.ROOT, "Incompatible field_type for %s space. The field type must be knn_vector.", spaceName) + ); + } - /** - * Constructor for CosineSimilarity scoring space. CosineSimilarity scoring space expects values to be of type - * float[]. - * - * @param query Query object that, along with the doc values, will be used to compute CosineSimilarity score - * @param fieldType FieldType for the doc values that will be used - */ - public CosineSimilarity(Object query, MappedFieldType fieldType) { - if (!isKNNVectorFieldType(fieldType)) { - throw new IllegalArgumentException("Incompatible field_type for cosine space. The field type must be knn_vector."); + KNNVectorFieldMapper.KNNVectorFieldType knnVectorFieldType = (KNNVectorFieldMapper.KNNVectorFieldType) fieldType; + if (isBinaryVectorDataType(knnVectorFieldType)) { + throw new IllegalArgumentException( + String.format( + Locale.ROOT, + "Incompatible field_type for %s space. The data type should be either float or byte but got %s", + spaceName, + knnVectorFieldType.getVectorDataType().getValue() + ) + ); } - this.processedQuery = parseToFloatArray( + return knnVectorFieldType; + } + + protected float[] getProcessedQuery(final Object query, final KNNVectorFieldMapper.KNNVectorFieldType knnVectorFieldType) { + return parseToFloatArray( query, - KNNVectorFieldMapperUtil.getExpectedDimensions((KNNVectorFieldMapper.KNNVectorFieldType) fieldType), - ((KNNVectorFieldMapper.KNNVectorFieldType) fieldType).getVectorDataType() + KNNVectorFieldMapperUtil.getExpectedVectorLength(knnVectorFieldType), + knnVectorFieldType.getVectorDataType() ); + } + + protected abstract BiFunction getScoringMethod(final float[] processedQuery); + + } + + class L2 extends LinearSpace { + public L2(final Object query, final MappedFieldType fieldType) { + super(query, fieldType, "l2"); + } + + @Override + public BiFunction getScoringMethod(final float[] processedQuery) { + return (float[] q, float[] v) -> 1 / (1 + KNNScoringUtil.l2Squared(q, v)); + } + } + + class CosineSimilarity extends LinearSpace { + public CosineSimilarity(Object query, MappedFieldType fieldType) { + super(query, fieldType, "cosine"); + } + + @Override + protected BiFunction getScoringMethod(final float[] processedQuery) { SpaceType.COSINESIMIL.validateVector(processedQuery); - float qVectorSquaredMagnitude = getVectorMagnitudeSquared(this.processedQuery); - this.scoringMethod = (float[] q, float[] v) -> 1 + KNNScoringUtil.cosinesimilOptimized(q, v, qVectorSquaredMagnitude); + float qVectorSquaredMagnitude = getVectorMagnitudeSquared(processedQuery); + return (float[] q, float[] v) -> 1 + KNNScoringUtil.cosinesimilOptimized(q, v, qVectorSquaredMagnitude); } + } - public ScoreScript getScoreScript( - Map params, - String field, - SearchLookup lookup, - LeafReaderContext ctx, - IndexSearcher searcher - ) throws IOException { - return new KNNScoreScript.KNNVectorType(params, this.processedQuery, field, this.scoringMethod, lookup, ctx, searcher); + class L1 extends LinearSpace { + public L1(Object query, MappedFieldType fieldType) { + super(query, fieldType, "l1"); + } + + @Override + protected BiFunction getScoringMethod(final float[] processedQuery) { + return (float[] q, float[] v) -> 1 / (1 + KNNScoringUtil.l1Norm(q, v)); + } + } + + class LInf extends LinearSpace { + public LInf(Object query, MappedFieldType fieldType) { + super(query, fieldType, "l-inf"); + } + + @Override + protected BiFunction getScoringMethod(final float[] processedQuery) { + return (float[] q, float[] v) -> 1 / (1 + KNNScoringUtil.lInfNorm(q, v)); + } + } + + class InnerProd extends LinearSpace { + public InnerProd(Object query, MappedFieldType fieldType) { + super(query, fieldType, "innerproduct"); + } + + @Override + protected BiFunction getScoringMethod(final float[] processedQuery) { + return (float[] q, float[] v) -> KNNWeight.normalizeScore(-KNNScoringUtil.innerProduct(q, v)); } } @@ -124,7 +170,7 @@ class HammingBit implements KNNScoringSpace { /** * Constructor for HammingBit scoring space. HammingBit scoring space expects values to either be of type - * Long or Base64 encoded strings. + * Long or Base64 encoded strings, or KNN type with binary data type. * * @param query Query object that, along with the doc values, will be used to compute HammingBit score * @param fieldType FieldType for the doc values that will be used @@ -136,13 +182,39 @@ public HammingBit(Object query, MappedFieldType fieldType) { } else if (isBinaryFieldType(fieldType)) { this.processedQuery = parseToBigInteger(query); this.scoringMethod = (BigInteger q, BigInteger v) -> 1.0f / (1 + KNNScoringUtil.calculateHammingBit(q, v)); + } else if (isKNNVectorFieldType(fieldType)) { + KNNVectorFieldMapper.KNNVectorFieldType knnVectorFieldType = (KNNVectorFieldMapper.KNNVectorFieldType) fieldType; + if (isBinaryVectorDataType(knnVectorFieldType) == false) { + throw new IllegalArgumentException( + String.format( + Locale.ROOT, + "Incompatible field_type for hamming space. The data type should be binary but got %s", + knnVectorFieldType.getVectorDataType().getValue() + ) + ); + } + this.processedQuery = parseToFloatArray( + query, + KNNVectorFieldMapperUtil.getExpectedVectorLength(knnVectorFieldType), + ((KNNVectorFieldMapper.KNNVectorFieldType) fieldType).getVectorDataType() + ); + // TODO we want to avoid converting back and forth between byte and float + this.scoringMethod = (float[] q, float[] v) -> 1 / (1 + KNNScoringUtil.calculateHammingBit(toByte(q), toByte(v))); } else { throw new IllegalArgumentException( - "Incompatible field_type for hamming space. The field type must " + "of type long or binary." + "Incompatible field_type for hamming space. The field type must of type long, binary, or knn_vector." ); } } + private byte[] toByte(final float[] vector) { + byte[] bytes = new byte[vector.length]; + for (int i = 0; i < vector.length; i++) { + bytes[i] = (byte) vector[i]; + } + return bytes; + } + @SuppressWarnings("unchecked") public ScoreScript getScoreScript( Map params, @@ -161,125 +233,27 @@ public ScoreScript getScoreScript( ctx, searcher ); - } - - return new KNNScoreScript.BigIntegerType( - params, - (BigInteger) this.processedQuery, - field, - (BiFunction) this.scoringMethod, - lookup, - ctx, - searcher - ); - } - } - - class L1 implements KNNScoringSpace { - - float[] processedQuery; - BiFunction scoringMethod; - - /** - * Constructor for L1 scoring space. L1 scoring space expects values to be of type float[]. - * - * @param query Query object that, along with the doc values, will be used to compute L1 score - * @param fieldType FieldType for the doc values that will be used - */ - public L1(Object query, MappedFieldType fieldType) { - if (!isKNNVectorFieldType(fieldType)) { - throw new IllegalArgumentException("Incompatible field_type for l1 space. The field type must " + "be knn_vector."); - } - - this.processedQuery = parseToFloatArray( - query, - KNNVectorFieldMapperUtil.getExpectedDimensions((KNNVectorFieldMapper.KNNVectorFieldType) fieldType), - ((KNNVectorFieldMapper.KNNVectorFieldType) fieldType).getVectorDataType() - ); - this.scoringMethod = (float[] q, float[] v) -> 1 / (1 + KNNScoringUtil.l1Norm(q, v)); - } - - public ScoreScript getScoreScript( - Map params, - String field, - SearchLookup lookup, - LeafReaderContext ctx, - IndexSearcher searcher - ) throws IOException { - return new KNNScoreScript.KNNVectorType(params, this.processedQuery, field, this.scoringMethod, lookup, ctx, searcher); - } - } - - class LInf implements KNNScoringSpace { - - float[] processedQuery; - BiFunction scoringMethod; - - /** - * Constructor for L-inf scoring space. L-inf scoring space expects values to be of type float[]. - * - * @param query Query object that, along with the doc values, will be used to compute L-inf score - * @param fieldType FieldType for the doc values that will be used - */ - public LInf(Object query, MappedFieldType fieldType) { - if (!isKNNVectorFieldType(fieldType)) { - throw new IllegalArgumentException("Incompatible field_type for l-inf space. The field type must " + "be knn_vector."); - } - - this.processedQuery = parseToFloatArray( - query, - KNNVectorFieldMapperUtil.getExpectedDimensions((KNNVectorFieldMapper.KNNVectorFieldType) fieldType), - ((KNNVectorFieldMapper.KNNVectorFieldType) fieldType).getVectorDataType() - ); - this.scoringMethod = (float[] q, float[] v) -> 1 / (1 + KNNScoringUtil.lInfNorm(q, v)); - } - - public ScoreScript getScoreScript( - Map params, - String field, - SearchLookup lookup, - LeafReaderContext ctx, - IndexSearcher searcher - ) throws IOException { - return new KNNScoreScript.KNNVectorType(params, this.processedQuery, field, this.scoringMethod, lookup, ctx, searcher); - } - } - - class InnerProd implements KNNScoringSpace { - - float[] processedQuery; - BiFunction scoringMethod; - - /** - * Constructor for innerproduct scoring space. innerproduct scoring space expects values to be of type float[]. - * - * @param query Query object that, along with the doc values, will be used to compute L-inf score - * @param fieldType FieldType for the doc values that will be used - */ - public InnerProd(Object query, MappedFieldType fieldType) { - if (!isKNNVectorFieldType(fieldType)) { - throw new IllegalArgumentException( - "Incompatible field_type for innerproduct space. The field type must " + "be knn_vector." + } else if (this.processedQuery instanceof BigInteger) { + return new KNNScoreScript.BigIntegerType( + params, + (BigInteger) this.processedQuery, + field, + (BiFunction) this.scoringMethod, + lookup, + ctx, + searcher + ); + } else { + return new KNNScoreScript.KNNVectorType( + params, + (float[]) this.processedQuery, + field, + (BiFunction) this.scoringMethod, + lookup, + ctx, + searcher ); } - - this.processedQuery = parseToFloatArray( - query, - KNNVectorFieldMapperUtil.getExpectedDimensions((KNNVectorFieldMapper.KNNVectorFieldType) fieldType), - ((KNNVectorFieldMapper.KNNVectorFieldType) fieldType).getVectorDataType() - ); - this.scoringMethod = (float[] q, float[] v) -> KNNWeight.normalizeScore(-KNNScoringUtil.innerProduct(q, v)); - } - - @Override - public ScoreScript getScoreScript( - Map params, - String field, - SearchLookup lookup, - LeafReaderContext ctx, - IndexSearcher searcher - ) throws IOException { - return new KNNScoreScript.KNNVectorType(params, this.processedQuery, field, this.scoringMethod, lookup, ctx, searcher); } } } diff --git a/src/main/java/org/opensearch/knn/plugin/script/KNNScoringSpaceUtil.java b/src/main/java/org/opensearch/knn/plugin/script/KNNScoringSpaceUtil.java index 5f9efd3cb..e2bade320 100644 --- a/src/main/java/org/opensearch/knn/plugin/script/KNNScoringSpaceUtil.java +++ b/src/main/java/org/opensearch/knn/plugin/script/KNNScoringSpaceUtil.java @@ -55,6 +55,16 @@ public static boolean isKNNVectorFieldType(MappedFieldType fieldType) { return fieldType instanceof KNNVectorFieldMapper.KNNVectorFieldType; } + /** + * Check if the KNN field type is a binary vector data type + * + * @param fieldType KNN vector field type + * @return true if the KNN field type is a binary vector data type + */ + public static boolean isBinaryVectorDataType(final KNNVectorFieldMapper.KNNVectorFieldType fieldType) { + return VectorDataType.BINARY == fieldType.getVectorDataType(); + } + /** * Convert an Object to a Long. * @@ -87,15 +97,15 @@ public static BigInteger parseToBigInteger(Object object) { * Convert an Object to a float array. * * @param object Object to be converted to a float array - * @param expectedDimensions int representing the expected dimension of this array. + * @param expectedVectorLength int representing the expected vector length of this array. * @return float[] of the object */ - public static float[] parseToFloatArray(Object object, int expectedDimensions, VectorDataType vectorDataType) { + public static float[] parseToFloatArray(Object object, int expectedVectorLength, VectorDataType vectorDataType) { float[] floatArray = convertVectorToPrimitive(object, vectorDataType); - if (expectedDimensions != floatArray.length) { + if (expectedVectorLength != floatArray.length) { KNNCounter.SCRIPT_QUERY_ERRORS.increment(); throw new IllegalStateException( - "Object's dimension=" + floatArray.length + " does not match the " + "expected dimension=" + expectedDimensions + "." + "Object's length=" + floatArray.length + " does not match the " + "expected length=" + expectedVectorLength + "." ); } return floatArray; @@ -115,7 +125,7 @@ public static float[] convertVectorToPrimitive(Object vector, VectorDataType vec primitiveVector = new float[tmp.size()]; for (int i = 0; i < primitiveVector.length; i++) { float value = tmp.get(i).floatValue(); - if (VectorDataType.BYTE == vectorDataType) { + if (VectorDataType.BYTE == vectorDataType || VectorDataType.BINARY == vectorDataType) { validateByteVectorValue(value, vectorDataType); } primitiveVector[i] = value; diff --git a/src/main/java/org/opensearch/knn/plugin/script/KNNScoringUtil.java b/src/main/java/org/opensearch/knn/plugin/script/KNNScoringUtil.java index 0bc21e3d9..8493ea5bd 100644 --- a/src/main/java/org/opensearch/knn/plugin/script/KNNScoringUtil.java +++ b/src/main/java/org/opensearch/knn/plugin/script/KNNScoringUtil.java @@ -197,6 +197,7 @@ public static float calculateHammingBit(Long queryLong, Long inputLong) { /** * This method calculates hamming distance between query vector + * and input vector * * @param queryVector query vector * @param inputVector input vector diff --git a/src/test/java/org/opensearch/knn/index/VectorDataTypeTests.java b/src/test/java/org/opensearch/knn/index/VectorDataTypeTests.java index 8e6b6d7f7..f760a6e88 100644 --- a/src/test/java/org/opensearch/knn/index/VectorDataTypeTests.java +++ b/src/test/java/org/opensearch/knn/index/VectorDataTypeTests.java @@ -118,7 +118,9 @@ public void testCreateKnnVectorFieldType_whenBinary_thenException() { } public void testGetVectorFromBytesRef_whenBinary_thenException() { - Exception ex = expectThrows(IllegalStateException.class, () -> VectorDataType.BINARY.getVectorFromBytesRef(new BytesRef())); - assertTrue(ex.getMessage().contains("Unsupported method")); + byte[] vector = { 1, 2, 3 }; + float[] expected = { 1, 2, 3 }; + BytesRef bytesRef = new BytesRef(vector); + assertArrayEquals(expected, VectorDataType.BINARY.getVectorFromBytesRef(bytesRef), 0.01f); } } diff --git a/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperUtilTests.java b/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperUtilTests.java index ad7f49cb6..7eebc140e 100644 --- a/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperUtilTests.java +++ b/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperUtilTests.java @@ -59,10 +59,14 @@ public void testStoredFields_whenVectorIsFloatType_thenSucceed() { assertArrayEquals(TEST_FLOAT_VECTOR, (float[]) vector, 0.001f); } - public void testGetExpectedDimensionsSuccess() { + public void testGetExpectedVectorLengthSuccess() { KNNVectorFieldMapper.KNNVectorFieldType knnVectorFieldType = mock(KNNVectorFieldMapper.KNNVectorFieldType.class); when(knnVectorFieldType.getDimension()).thenReturn(3); + KNNVectorFieldMapper.KNNVectorFieldType knnVectorFieldTypeBinary = mock(KNNVectorFieldMapper.KNNVectorFieldType.class); + when(knnVectorFieldTypeBinary.getDimension()).thenReturn(8); + when(knnVectorFieldTypeBinary.getVectorDataType()).thenReturn(VectorDataType.BINARY); + KNNVectorFieldMapper.KNNVectorFieldType knnVectorFieldTypeModelBased = mock(KNNVectorFieldMapper.KNNVectorFieldType.class); when(knnVectorFieldTypeModelBased.getDimension()).thenReturn(-1); String modelId = "test-model"; @@ -76,11 +80,12 @@ public void testGetExpectedDimensionsSuccess() { KNNVectorFieldMapperUtil.initialize(modelDao); - assertEquals(3, KNNVectorFieldMapperUtil.getExpectedDimensions(knnVectorFieldType)); - assertEquals(4, KNNVectorFieldMapperUtil.getExpectedDimensions(knnVectorFieldTypeModelBased)); + assertEquals(3, KNNVectorFieldMapperUtil.getExpectedVectorLength(knnVectorFieldType)); + assertEquals(1, KNNVectorFieldMapperUtil.getExpectedVectorLength(knnVectorFieldTypeBinary)); + assertEquals(4, KNNVectorFieldMapperUtil.getExpectedVectorLength(knnVectorFieldTypeModelBased)); } - public void testGetExpectedDimensionsFailure() { + public void testGetExpectedVectorLengthFailure() { KNNVectorFieldMapper.KNNVectorFieldType knnVectorFieldTypeModelBased = mock(KNNVectorFieldMapper.KNNVectorFieldType.class); when(knnVectorFieldTypeModelBased.getDimension()).thenReturn(-1); String modelId = "test-model"; @@ -95,7 +100,7 @@ public void testGetExpectedDimensionsFailure() { IllegalArgumentException e = expectThrows( IllegalArgumentException.class, - () -> KNNVectorFieldMapperUtil.getExpectedDimensions(knnVectorFieldTypeModelBased) + () -> KNNVectorFieldMapperUtil.getExpectedVectorLength(knnVectorFieldTypeModelBased) ); assertEquals(String.format("Model ID '%s' is not created.", modelId), e.getMessage()); @@ -109,7 +114,7 @@ public void testGetExpectedDimensionsFailure() { e = expectThrows( IllegalArgumentException.class, - () -> KNNVectorFieldMapperUtil.getExpectedDimensions(knnVectorFieldTypeModelBased) + () -> KNNVectorFieldMapperUtil.getExpectedVectorLength(knnVectorFieldTypeModelBased) ); assertEquals(String.format("Field '%s' does not have model.", fieldName), e.getMessage()); } diff --git a/src/test/java/org/opensearch/knn/plugin/script/KNNScoringSpaceTests.java b/src/test/java/org/opensearch/knn/plugin/script/KNNScoringSpaceTests.java index 3cfbe56f1..690c9b53a 100644 --- a/src/test/java/org/opensearch/knn/plugin/script/KNNScoringSpaceTests.java +++ b/src/test/java/org/opensearch/knn/plugin/script/KNNScoringSpaceTests.java @@ -5,10 +5,16 @@ package org.opensearch.knn.plugin.script; +import java.lang.reflect.Constructor; +import java.lang.reflect.InvocationTargetException; import java.util.Locale; + +import lombok.SneakyThrows; +import org.opensearch.index.mapper.MappedFieldType; import org.opensearch.knn.KNNTestCase; import org.opensearch.knn.index.KNNMethodContext; import org.opensearch.knn.index.SpaceType; +import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.mapper.KNNVectorFieldMapper; import org.opensearch.index.mapper.BinaryFieldMapper; import org.opensearch.index.mapper.NumberFieldMapper; @@ -22,10 +28,29 @@ import java.util.function.BiFunction; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; public class KNNScoringSpaceTests extends KNNTestCase { - public void testL2() { + private void expectThrowsExceptionWithNonKNNField(Class clazz) throws NoSuchMethodException { + Constructor constructor = clazz.getConstructor(Object.class, MappedFieldType.class); + NumberFieldMapper.NumberFieldType invalidFieldType = mock(NumberFieldMapper.NumberFieldType.class); + Exception e = expectThrows(InvocationTargetException.class, () -> constructor.newInstance(null, invalidFieldType)); + assertTrue(e.getCause() instanceof IllegalArgumentException); + assertTrue(e.getCause().getMessage().contains("The field type must be knn_vector")); + } + + private void expectThrowsExceptionWithKNNFieldWithBinaryDataType(Class clazz) throws NoSuchMethodException { + Constructor constructor = clazz.getConstructor(Object.class, MappedFieldType.class); + KNNVectorFieldMapper.KNNVectorFieldType invalidFieldType = mock(KNNVectorFieldMapper.KNNVectorFieldType.class); + when(invalidFieldType.getVectorDataType()).thenReturn(VectorDataType.BINARY); + Exception e = expectThrows(InvocationTargetException.class, () -> constructor.newInstance(null, invalidFieldType)); + assertTrue(e.getCause() instanceof IllegalArgumentException); + assertTrue(e.getCause().getMessage().contains("The data type should be either float or byte")); + } + + @SneakyThrows + public void testL2_whenValid_thenSucceed() { float[] arrayFloat = new float[] { 1.0f, 2.0f, 3.0f }; List arrayListQueryObject = new ArrayList<>(Arrays.asList(1.0, 2.0, 3.0)); KNNMethodContext knnMethodContext = KNNMethodContext.getDefault(); @@ -37,15 +62,15 @@ public void testL2() { ); KNNScoringSpace.L2 l2 = new KNNScoringSpace.L2(arrayListQueryObject, fieldType); assertEquals(1F, l2.scoringMethod.apply(arrayFloat, arrayFloat), 0.1F); + } - NumberFieldMapper.NumberFieldType invalidFieldType = new NumberFieldMapper.NumberFieldType( - "field", - NumberFieldMapper.NumberType.INTEGER - ); - expectThrows(IllegalArgumentException.class, () -> new KNNScoringSpace.L2(arrayListQueryObject, invalidFieldType)); + @SneakyThrows + public void testL2_whenInvalidType_thenException() { + expectThrowsExceptionWithNonKNNField(KNNScoringSpace.L2.class); + expectThrowsExceptionWithKNNFieldWithBinaryDataType(KNNScoringSpace.L2.class); } - public void testCosineSimilarity() { + public void testCosineSimilarity_whenValid_thenSucceed() { float[] arrayFloat = new float[] { 1.0f, 2.0f, 3.0f }; List arrayListQueryObject = new ArrayList<>(Arrays.asList(2.0, 4.0, 6.0)); float[] arrayFloat2 = new float[] { 2.0f, 4.0f, 6.0f }; @@ -58,7 +83,6 @@ public void testCosineSimilarity() { knnMethodContext ); KNNScoringSpace.CosineSimilarity cosineSimilarity = new KNNScoringSpace.CosineSimilarity(arrayListQueryObject, fieldType); - assertEquals(2F, cosineSimilarity.scoringMethod.apply(arrayFloat2, arrayFloat), 0.1F); // invalid zero vector @@ -71,19 +95,35 @@ public void testCosineSimilarity() { String.format(Locale.ROOT, "zero vector is not supported when space type is [%s]", SpaceType.COSINESIMIL.getValue()), exception1.getMessage() ); + } - NumberFieldMapper.NumberFieldType invalidFieldType = new NumberFieldMapper.NumberFieldType( - "field", - NumberFieldMapper.NumberType.INTEGER + public void testCosineSimilarity_whenZeroVector_thenException() { + KNNMethodContext knnMethodContext = KNNMethodContext.getDefault(); + KNNVectorFieldMapper.KNNVectorFieldType fieldType = new KNNVectorFieldMapper.KNNVectorFieldType( + "test", + Collections.emptyMap(), + 3, + knnMethodContext ); - IllegalArgumentException exception2 = expectThrows( + + final List queryZeroVector = List.of(0.0f, 0.0f, 0.0f); + IllegalArgumentException exception1 = expectThrows( IllegalArgumentException.class, - () -> new KNNScoringSpace.CosineSimilarity(arrayListQueryObject, invalidFieldType) + () -> new KNNScoringSpace.CosineSimilarity(queryZeroVector, fieldType) ); - assertEquals("Incompatible field_type for cosine space. The field type must be knn_vector.", exception2.getMessage()); + assertEquals( + String.format(Locale.ROOT, "zero vector is not supported when space type is [%s]", SpaceType.COSINESIMIL.getValue()), + exception1.getMessage() + ); + } + + @SneakyThrows + public void testCosineSimilarity_whenInvalidType_thenException() { + expectThrowsExceptionWithNonKNNField(KNNScoringSpace.CosineSimilarity.class); + expectThrowsExceptionWithKNNFieldWithBinaryDataType(KNNScoringSpace.CosineSimilarity.class); } - public void testInnerProdSimilarity() { + public void testInnerProd_whenValid_thenSucceed() { float[] arrayFloat_case1 = new float[] { 1.0f, 2.0f, 3.0f }; List arrayListQueryObject_case1 = new ArrayList<>(Arrays.asList(1.0, 2.0, 3.0)); float[] arrayFloat2_case1 = new float[] { 1.0f, 1.0f, 1.0f }; @@ -114,12 +154,12 @@ public void testInnerProdSimilarity() { innerProd = new KNNScoringSpace.InnerProd(arrayListQueryObject_case3, fieldType); assertEquals(140_000_000_001F, innerProd.scoringMethod.apply(arrayFloat_case3, arrayFloat2_case3), 0.01F); + } - NumberFieldMapper.NumberFieldType invalidFieldType = new NumberFieldMapper.NumberFieldType( - "field", - NumberFieldMapper.NumberType.INTEGER - ); - expectThrows(IllegalArgumentException.class, () -> new KNNScoringSpace.InnerProd(arrayListQueryObject_case2, invalidFieldType)); + @SneakyThrows + public void testInnerProd_whenInvalidType_thenException() { + expectThrowsExceptionWithNonKNNField(KNNScoringSpace.InnerProd.class); + expectThrowsExceptionWithKNNFieldWithBinaryDataType(KNNScoringSpace.InnerProd.class); } @SuppressWarnings("unchecked") @@ -130,9 +170,6 @@ public void testHammingBit_Long() { KNNScoringSpace.HammingBit hammingBit = new KNNScoringSpace.HammingBit(longObject1, fieldType); assertEquals(0.1111F, ((BiFunction) hammingBit.scoringMethod).apply(longObject1, longObject2), 0.1F); - - KNNVectorFieldMapper.KNNVectorFieldType invalidFieldType = mock(KNNVectorFieldMapper.KNNVectorFieldType.class); - expectThrows(IllegalArgumentException.class, () -> new KNNScoringSpace.HammingBit(longObject1, invalidFieldType)); } @SuppressWarnings("unchecked") @@ -158,8 +195,29 @@ public void testHammingBit_Base64() { ), 0.1F ); + } + + public void testHammingBit_whenKNNFieldType_thenSucceed() { + List arrayListQueryObject = new ArrayList<>(Arrays.asList(1.0, 2.0, 3.0)); + KNNMethodContext knnMethodContext = KNNMethodContext.getDefault(); + KNNVectorFieldMapper.KNNVectorFieldType fieldType = new KNNVectorFieldMapper.KNNVectorFieldType( + "test", + Collections.emptyMap(), + 8 * arrayListQueryObject.size(), + knnMethodContext, + VectorDataType.BINARY + ); + KNNScoringSpace.HammingBit hammingBit = new KNNScoringSpace.HammingBit(arrayListQueryObject, fieldType); + + float[] arrayFloat = new float[] { 1.0f, 2.0f, 3.0f }; + BiFunction scoringMethod = (BiFunction) hammingBit.scoringMethod; + assertEquals(1F, scoringMethod.apply(arrayFloat, arrayFloat), 0.1F); + } + public void testHammingBit_whenNonBinaryVectorDataType_thenException() { KNNVectorFieldMapper.KNNVectorFieldType invalidFieldType = mock(KNNVectorFieldMapper.KNNVectorFieldType.class); - expectThrows(IllegalArgumentException.class, () -> new KNNScoringSpace.HammingBit(base64Object1, invalidFieldType)); + when(invalidFieldType.getVectorDataType()).thenReturn(randomInt() % 2 == 0 ? VectorDataType.FLOAT : VectorDataType.BYTE); + Exception e = expectThrows(IllegalArgumentException.class, () -> new KNNScoringSpace.HammingBit(null, invalidFieldType)); + assertTrue(e.getMessage().contains("The data type should be binary")); } } diff --git a/src/test/java/org/opensearch/knn/plugin/script/KNNScoringSpaceUtilTests.java b/src/test/java/org/opensearch/knn/plugin/script/KNNScoringSpaceUtilTests.java index b5bc4b95f..ace3dabc8 100644 --- a/src/test/java/org/opensearch/knn/plugin/script/KNNScoringSpaceUtilTests.java +++ b/src/test/java/org/opensearch/knn/plugin/script/KNNScoringSpaceUtilTests.java @@ -75,4 +75,39 @@ public void testParseKNNVectorQuery() { String invalidObject = "invalidObject"; expectThrows(ClassCastException.class, () -> KNNScoringSpaceUtil.parseToFloatArray(invalidObject, 3, VectorDataType.FLOAT)); } + + public void testIsBinaryVectorDataType_whenBinary_thenReturnTrue() { + KNNVectorFieldMapper.KNNVectorFieldType fieldType = mock(KNNVectorFieldMapper.KNNVectorFieldType.class); + when(fieldType.getVectorDataType()).thenReturn(VectorDataType.BINARY); + assertTrue(KNNScoringSpaceUtil.isBinaryVectorDataType(fieldType)); + } + + public void testIsBinaryVectorDataType_whenNonBinary_thenReturnFalse() { + KNNVectorFieldMapper.KNNVectorFieldType fieldType = mock(KNNVectorFieldMapper.KNNVectorFieldType.class); + when(fieldType.getVectorDataType()).thenReturn(randomInt() % 2 == 0 ? VectorDataType.FLOAT : VectorDataType.BYTE); + assertFalse(KNNScoringSpaceUtil.isBinaryVectorDataType(fieldType)); + } + + public void testConvertVectorToPrimitive_whenBinaryWithValidInput_thenReturnPrimitive() { + Number number1 = mock(Number.class); + when(number1.floatValue()).thenReturn(1f); + Number number2 = mock(Number.class); + when(number2.floatValue()).thenReturn(2f); + List vector = List.of(number1, number2); + float[] expected = new float[] { 1, 2 }; + assertArrayEquals(expected, KNNScoringSpaceUtil.convertVectorToPrimitive(vector, VectorDataType.BINARY), 0.01f); + } + + public void testConvertVectorToPrimitive_whenBinaryWithOutOfRange_thenException() { + Number number1 = mock(Number.class); + when(number1.floatValue()).thenReturn(128f); + Number number2 = mock(Number.class); + when(number2.floatValue()).thenReturn(-129f); + List vector = List.of(number1, number2); + Exception e = expectThrows( + IllegalArgumentException.class, + () -> KNNScoringSpaceUtil.convertVectorToPrimitive(vector, VectorDataType.BINARY) + ); + assertTrue(e.getMessage().contains("KNN vector values are not within in the byte range")); + } } From 87733a31608e18982bee9aade8ade5127a0ff966 Mon Sep 17 00:00:00 2001 From: Tejas Shah Date: Tue, 16 Jul 2024 18:46:32 -0700 Subject: [PATCH 293/416] Adds Nprobe as a method parameter in knn query (#1841) Adds integration test for invalid method parameters (#1782) Signed-off-by: Tejas Shah (cherry picked from commit b4224661f3ad1a01a4fdd77e835af08ce2f0aa48) Signed-off-by: Tejas Shah --- CHANGELOG.md | 1 + jni/src/faiss_wrapper.cpp | 14 +- jni/tests/faiss_wrapper_unit_test.cpp | 191 ++++++++++++++---- .../knn/index/query/KNNQueryBuilder.java | 7 +- .../query/parser/MethodParametersParser.java | 3 + .../index/query/request/MethodParameter.java | 37 +++- .../knn/index/util/DefaultIVFContext.java | 30 +++ .../org/opensearch/knn/index/util/Faiss.java | 2 +- .../org/opensearch/knn/index/FaissIT.java | 16 +- .../opensearch/knn/index/LuceneEngineIT.java | 23 +-- .../knn/index/query/InvalidSearchQueryIT.java | 156 ++++++++++++++ .../knn/index/query/KNNQueryBuilderTests.java | 5 +- .../parser/MethodParametersParserTests.java | 13 ++ .../org/opensearch/knn/KNNRestTestCase.java | 21 ++ 14 files changed, 443 insertions(+), 76 deletions(-) create mode 100644 src/main/java/org/opensearch/knn/index/util/DefaultIVFContext.java create mode 100644 src/test/java/org/opensearch/knn/index/query/InvalidSearchQueryIT.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 8501e4842..933be1f3c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Features * Adds dynamic query parameter ef_search [#1783](https://github.com/opensearch-project/k-NN/pull/1783) * Adds dynamic query parameter ef_search in radial search faiss engine [#1790](https://github.com/opensearch-project/k-NN/pull/1790) +* Adds dynamic query parameter nprobes [#1792](https://github.com/opensearch-project/k-NN/pull/1792) * Add binary format support with HNSW method in Faiss Engine [#1781](https://github.com/opensearch-project/k-NN/pull/1781) * Add script scoring support for knn field with binary data type [#1826](https://github.com/opensearch-project/k-NN/pull/1826) ### Enhancements diff --git a/jni/src/faiss_wrapper.cpp b/jni/src/faiss_wrapper.cpp index 9abb2357f..45830aff6 100644 --- a/jni/src/faiss_wrapper.cpp +++ b/jni/src/faiss_wrapper.cpp @@ -319,7 +319,6 @@ jobjectArray knn_jni::faiss_wrapper::QueryIndex_WithFilter(knn_jni::JNIUtilInter if (methodParamsJ != nullptr) { methodParams = jniUtil->ConvertJavaMapToCppMap(env, methodParamsJ); } - // The ids vector will hold the top k ids from the search and the dis vector will hold the top k distances from // the query point std::vector dis(kJ); @@ -358,7 +357,10 @@ jobjectArray knn_jni::faiss_wrapper::QueryIndex_WithFilter(knn_jni::JNIUtilInter } else { auto ivfReader = dynamic_cast(indexReader->index); auto ivfFlatReader = dynamic_cast(indexReader->index); + if(ivfReader || ivfFlatReader) { + int indexNprobe = ivfReader == nullptr ? ivfFlatReader->nprobe : ivfReader->nprobe; + ivfParams.nprobe = commons::getIntegerMethodParameter(env, jniUtil, methodParams, NPROBES, indexNprobe); ivfParams.sel = idSelector.get(); searchParameters = &ivfParams; } @@ -374,10 +376,11 @@ jobjectArray knn_jni::faiss_wrapper::QueryIndex_WithFilter(knn_jni::JNIUtilInter } else { faiss::SearchParameters *searchParameters = nullptr; faiss::SearchParametersHNSW hnswParams; + faiss::SearchParametersIVF ivfParams; std::unique_ptr idGrouper; std::vector idGrouperBitmap; auto hnswReader = dynamic_cast(indexReader->index); - if(hnswReader!= nullptr) { + if(hnswReader != nullptr) { // Query param efsearch supersedes ef_search provided during index setting. hnswParams.efSearch = knn_jni::commons::getIntegerMethodParameter(env, jniUtil, methodParams, EF_SEARCH, hnswReader->hnsw.efSearch); if (parentIdsJ != nullptr) { @@ -385,6 +388,13 @@ jobjectArray knn_jni::faiss_wrapper::QueryIndex_WithFilter(knn_jni::JNIUtilInter hnswParams.grp = idGrouper.get(); } searchParameters = &hnswParams; + } else { + auto ivfReader = dynamic_cast(indexReader->index); + if (ivfReader) { + int indexNprobe = ivfReader->nprobe; + ivfParams.nprobe = commons::getIntegerMethodParameter(env, jniUtil, methodParams, NPROBES, indexNprobe); + searchParameters = &ivfParams; + } } try { indexReader->search(1, rawQueryvector, kJ, dis.data(), ids.data(), searchParameters); diff --git a/jni/tests/faiss_wrapper_unit_test.cpp b/jni/tests/faiss_wrapper_unit_test.cpp index d68ec69c6..e23ca8528 100644 --- a/jni/tests/faiss_wrapper_unit_test.cpp +++ b/jni/tests/faiss_wrapper_unit_test.cpp @@ -20,6 +20,8 @@ #include "test_util.h" #include "faiss/IndexHNSW.h" #include "faiss/IndexIDMap.h" +#include "faiss/IndexIVFFlat.h" +#include "faiss/IndexIVFPQ.h" using ::testing::NiceMock; @@ -30,6 +32,46 @@ struct FaissMockIndex : faiss::IndexHNSW { } }; +struct MockIVFIndex : faiss::IndexIVFFlat { + explicit MockIVFIndex() = default; +}; + +struct MockIVFIdMap : faiss::IndexIDMap { + mutable idx_t nCalled{}; + mutable const float *xCalled{}; + mutable idx_t kCalled{}; + mutable float *distancesCalled{}; + mutable idx_t *labelsCalled{}; + mutable const faiss::SearchParametersIVF *paramsCalled{}; + + explicit MockIVFIdMap(MockIVFIndex *index) : faiss::IndexIDMapTemplate(index) { + } + + void search( + idx_t n, + const float *x, + idx_t k, + float *distances, + idx_t *labels, + const faiss::SearchParameters *params) const override { + nCalled = n; + xCalled = x; + kCalled = k; + distancesCalled = distances; + labelsCalled = labels; + paramsCalled = dynamic_cast(params); + } + + void resetMock() const { + nCalled = 0; + xCalled = nullptr; + kCalled = 0; + distancesCalled = nullptr; + labelsCalled = nullptr; + paramsCalled = nullptr; + } +}; + struct FaissMockIdMap : faiss::IndexIDMap { mutable idx_t nCalled{}; mutable const float *xCalled{}; @@ -83,13 +125,14 @@ struct FaissMockIdMap : faiss::IndexIDMap { } }; -struct QueryIndexHNSWTestInput { - std::string description; +struct QueryIndexInput { + string description; int k; - int efSearch; int filterIdType; bool filterIdsPresent; bool parentIdsPresent; + int efSearch; + int nprobe; }; struct RangeSearchTestInput { @@ -101,9 +144,9 @@ struct RangeSearchTestInput { bool parentIdsPresent; }; -class FaissWrappeterParametrizedTestFixture : public testing::TestWithParam { +class FaissWrapperParameterizedTestFixture : public testing::TestWithParam { public: - FaissWrappeterParametrizedTestFixture() : index_(3), id_map_(&index_) { + FaissWrapperParameterizedTestFixture() : index_(3), id_map_(&index_) { index_.hnsw.efSearch = 100; // assigning 100 to make sure default of 16 is not used anywhere } @@ -112,9 +155,9 @@ class FaissWrappeterParametrizedTestFixture : public testing::TestWithParam { +class FaissWrapperParameterizedRangeSearchTestFixture : public testing::TestWithParam { public: - FaissWrapperParametrizedRangeSearchTestFixture() : index_(3), id_map_(&index_) { + FaissWrapperParameterizedRangeSearchTestFixture() : index_(3), id_map_(&index_) { index_.hnsw.efSearch = 100; // assigning 100 to make sure default of 16 is not used anywhere } @@ -123,16 +166,24 @@ class FaissWrapperParametrizedRangeSearchTestFixture : public testing::TestWithP FaissMockIdMap id_map_; }; -namespace query_index_test { +class FaissWrapperIVFQueryTestFixture : public testing::TestWithParam { +public: + FaissWrapperIVFQueryTestFixture() : ivf_id_map_(&ivf_index_) { + ivf_index_.nprobe = 100; + }; - std::unordered_map methodParams; +protected: + MockIVFIndex ivf_index_; + MockIVFIdMap ivf_id_map_; +}; - TEST_P(FaissWrappeterParametrizedTestFixture, QueryIndexHNSWTests) { - // Given +namespace query_index_test { + TEST_P(FaissWrapperParameterizedTestFixture, QueryIndexHNSWTests) { + //Given JNIEnv *jniEnv = nullptr; NiceMock mockJNIUtil; - QueryIndexHNSWTestInput const &input = GetParam(); + QueryIndexInput const &input = GetParam(); float query[] = {1.2, 2.3, 3.4}; int efSearch = input.efSearch; @@ -184,24 +235,23 @@ namespace query_index_test { INSTANTIATE_TEST_CASE_P( QueryIndexHNSWTests, - FaissWrappeterParametrizedTestFixture, + FaissWrapperParameterizedTestFixture, ::testing::Values( - QueryIndexHNSWTestInput{"algoParams present, parent absent", 10, 200, 0, false, false}, - QueryIndexHNSWTestInput{"algoParams absent, parent absent", 10, -1, 0, false, false}, - QueryIndexHNSWTestInput{"algoParams present, parent present", 10, 200, 0, false, true}, - QueryIndexHNSWTestInput{"algoParams absent, parent present", 10, -1, 0, false, true} + QueryIndexInput {"algoParams present, parent absent", 10, 0, false, false, 200, -1 }, + QueryIndexInput {"algoParams present, parent absent", 10, 0, false, false, -1, -1 }, + QueryIndexInput {"algoParams present, parent present", 10, 0, false, true, 200, -1 }, + QueryIndexInput {"algoParams absent, parent present", 10, 0, false, true, -1, -1 } ) ); } namespace query_index_with_filter_test { - - TEST_P(FaissWrappeterParametrizedTestFixture, QueryIndexWithFilterHNSWTests) { - // Given + TEST_P(FaissWrapperParameterizedTestFixture, QueryIndexWithFilterHNSWTests) { + //Given JNIEnv *jniEnv = nullptr; NiceMock mockJNIUtil; - QueryIndexHNSWTestInput const &input = GetParam(); + QueryIndexInput const &input = GetParam(); float query[] = {1.2, 2.3, 3.4}; std::vector *parentIdPtr = nullptr; @@ -267,23 +317,26 @@ namespace query_index_with_filter_test { INSTANTIATE_TEST_CASE_P( QueryIndexWithFilterHNSWTests, - FaissWrappeterParametrizedTestFixture, + FaissWrapperParameterizedTestFixture, ::testing::Values( - QueryIndexHNSWTestInput{"algoParams present, parent absent, filter absent", 10, 200, 0, false, false}, - QueryIndexHNSWTestInput{"algoParams present, parent absent, filter absent, filter type 1", 10, 200, 1, false, false}, - QueryIndexHNSWTestInput{"algoParams absent, parent absent, filter present", 10, -1, 0, true, false}, - QueryIndexHNSWTestInput{"algoParams absent, parent absent, filter present, filter type 1", 10, -1, 1, true, false}, - QueryIndexHNSWTestInput{"algoParams present, parent present, filter absent", 10, 200, 0, false, true}, - QueryIndexHNSWTestInput{"algoParams present, parent present, filter absent, filter type 1", 10, 150, 1, false, true}, - QueryIndexHNSWTestInput{"algoParams absent, parent present, filter present", 10, -1, 0, true, true}, - QueryIndexHNSWTestInput{"algoParams absent, parent present, filter present, filter type 1",10, -1, 1, true, true} + QueryIndexInput { "algoParams present, parent absent, filter absent", 10, 0, false, false, 200, -1 }, + QueryIndexInput { "algoParams present, parent absent, filter absent, filter type 1", 10, 1, false, false, + 200, -1}, + QueryIndexInput { "algoParams absent, parent absent, filter present", 10, 0, true, false, -1, -1}, + QueryIndexInput { "algoParams absent, parent absent, filter present, filter type 1", 10, 1, true, false, -1, + -1}, + QueryIndexInput { "algoParams present, parent present, filter absent", 10, 0, false, true, 200, -1 }, + QueryIndexInput { "algoParams present, parent present, filter absent, filter type 1", 10, 1, false, true, + 150, -1}, + QueryIndexInput { "algoParams absent, parent present, filter present", 10, 0, true, true, -1, -1}, + QueryIndexInput { "algoParams absent, parent present, filter present, filter type 1",10, 1, true, true, -1, + -1 } ) ); } namespace range_search_test { - - TEST_P(FaissWrapperParametrizedRangeSearchTestFixture, RangeSearchHNSWTests) { + TEST_P(FaissWrapperParameterizedRangeSearchTestFixture, RangeSearchHNSWTests) { // Given JNIEnv *jniEnv = nullptr; NiceMock mockJNIUtil; @@ -323,6 +376,7 @@ namespace range_search_test { std::vector filter; std::vector *filterptr = nullptr; if (input.filterIdsPresent) { + std::vector filter; filter.reserve(2); filter.push_back(1); filter.push_back(2); @@ -356,16 +410,79 @@ namespace range_search_test { INSTANTIATE_TEST_CASE_P( RangeSearchHNSWTests, - FaissWrapperParametrizedRangeSearchTestFixture, + FaissWrapperParameterizedRangeSearchTestFixture, ::testing::Values( RangeSearchTestInput{"algoParams present, parent absent, filter absent", 10.0f, 200, 0, false, false}, - RangeSearchTestInput{"algoParams present, parent absent, filter absent, filter type 1", 10.0f, 200, 1, false, false}, + RangeSearchTestInput{"algoParams present, parent absent, filter absent, filter type 1", 10.0f, 200, 1, false + , false}, RangeSearchTestInput{"algoParams absent, parent absent, filter present", 10.0f, -1, 0, true, false}, - RangeSearchTestInput{"algoParams absent, parent absent, filter present, filter type 1", 10.0f, -1, 1, true, false}, + RangeSearchTestInput{"algoParams absent, parent absent, filter present, filter type 1", 10.0f, -1, 1, true, + false}, RangeSearchTestInput{"algoParams present, parent present, filter absent", 10.0f, 200, 0, false, true}, - RangeSearchTestInput{"algoParams present, parent present, filter absent, filter type 1", 10.0f, 150, 1, false, true}, + RangeSearchTestInput{"algoParams present, parent present, filter absent, filter type 1", 10.0f, 150, 1, + false, true}, RangeSearchTestInput{"algoParams absent, parent present, filter present", 10.0f, -1, 0, true, true}, - RangeSearchTestInput{"algoParams absent, parent present, filter present, filter type 1", 10.0f, -1, 1, true, true} + RangeSearchTestInput{"algoParams absent, parent present, filter present, filter type 1", 10.0f, -1, 1, true, + true} + ) + ); +} + +namespace query_index_with_filter_test_ivf { + TEST_P(FaissWrapperIVFQueryTestFixture, QueryIndexIVFTest) { + //Given + JNIEnv *jniEnv = nullptr; + NiceMock mockJNIUtil; + + QueryIndexInput const &input = GetParam(); + float query[] = {1.2, 2.3, 3.4}; + + int nprobe = input.nprobe; + int expectedNprobe = 100; //default set in mock + std::unordered_map methodParams; + if (nprobe != -1) { + expectedNprobe = input.nprobe; + methodParams[knn_jni::NPROBES] = reinterpret_cast(&nprobe); + } + + std::vector *filterptr = nullptr; + if (input.filterIdsPresent) { + std::vector filter; + filter.reserve(2); + filter.push_back(1); + filter.push_back(2); + filterptr = &filter; + } + + // When + knn_jni::faiss_wrapper::QueryIndex_WithFilter( + &mockJNIUtil, jniEnv, + reinterpret_cast(&ivf_id_map_), + reinterpret_cast(&query), input.k, reinterpret_cast(&methodParams), + reinterpret_cast(filterptr), + input.filterIdType, + nullptr); + + //Then + int actualEfSearch = ivf_id_map_.paramsCalled->nprobe; + // Asserting the captured argument + EXPECT_EQ(input.k, ivf_id_map_.kCalled); + EXPECT_EQ(expectedNprobe, actualEfSearch); + if (input.parentIdsPresent) { + faiss::IDGrouper *grouper = ivf_id_map_.paramsCalled->grp; + EXPECT_TRUE(grouper != nullptr); + } + ivf_id_map_.resetMock(); + } + + INSTANTIATE_TEST_CASE_P( + QueryIndexIVFTest, + FaissWrapperIVFQueryTestFixture, + ::testing::Values( + QueryIndexInput{"algoParams present, parent absent", 10, 0, false, false, -1, 200 }, + QueryIndexInput{"algoParams present, parent absent", 10,0, false, false, -1, -1 }, + QueryIndexInput{"algoParams present, parent present", 10, 0, true, true, -1, 200 }, + QueryIndexInput{"algoParams absent, parent present", 10, 0, true, true, -1, -1 } ) ); } diff --git a/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java b/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java index 855bb9ac3..013a4d9f7 100644 --- a/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java +++ b/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java @@ -51,6 +51,7 @@ import static org.opensearch.knn.common.KNNConstants.MAX_DISTANCE; import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER; import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_EF_SEARCH; +import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_NPROBES; import static org.opensearch.knn.common.KNNConstants.MIN_SCORE; import static org.opensearch.knn.common.KNNValidationUtil.validateByteVectorValue; import static org.opensearch.knn.index.IndexUtil.isClusterOnOrAfterMinRequiredVersion; @@ -75,6 +76,7 @@ public class KNNQueryBuilder extends AbstractQueryBuilder { public static final ParseField MAX_DISTANCE_FIELD = new ParseField(MAX_DISTANCE); public static final ParseField MIN_SCORE_FIELD = new ParseField(MIN_SCORE); public static final ParseField EF_SEARCH_FIELD = new ParseField(METHOD_PARAMETER_EF_SEARCH); + public static final ParseField NPROBE_FIELD = new ParseField(METHOD_PARAMETER_NPROBES); public static final ParseField METHOD_PARAMS_FIELD = new ParseField(METHOD_PARAMETER); public static final int K_MAX = 10000; /** @@ -223,9 +225,8 @@ private void validate() { if (k != null) { if (k <= 0 || k > K_MAX) { - throw new IllegalArgumentException( - String.format(Locale.ROOT, "[%s] requires k to be in the range (0, %d]", NAME, K_MAX) - ); + final String errorMessage = "[" + NAME + "] requires k to be in the range (0, " + K_MAX + "]"; + throw new IllegalArgumentException(errorMessage); } } diff --git a/src/main/java/org/opensearch/knn/index/query/parser/MethodParametersParser.java b/src/main/java/org/opensearch/knn/index/query/parser/MethodParametersParser.java index e2ba8f26e..41b69f441 100644 --- a/src/main/java/org/opensearch/knn/index/query/parser/MethodParametersParser.java +++ b/src/main/java/org/opensearch/knn/index/query/parser/MethodParametersParser.java @@ -32,6 +32,9 @@ import static org.opensearch.knn.index.query.KNNQueryBuilder.METHOD_PARAMS_FIELD; import static org.opensearch.knn.index.query.KNNQueryBuilder.NAME; +/** + * Note: This parser is used by neural plugin as well, breaking changes will require changes in neural as well + */ @EqualsAndHashCode @Getter @AllArgsConstructor diff --git a/src/main/java/org/opensearch/knn/index/query/request/MethodParameter.java b/src/main/java/org/opensearch/knn/index/query/request/MethodParameter.java index 0b76cad4a..17f04d7e2 100644 --- a/src/main/java/org/opensearch/knn/index/query/request/MethodParameter.java +++ b/src/main/java/org/opensearch/knn/index/query/request/MethodParameter.java @@ -21,7 +21,9 @@ import java.util.Map; import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_EF_SEARCH; +import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_NPROBES; import static org.opensearch.knn.index.query.KNNQueryBuilder.EF_SEARCH_FIELD; +import static org.opensearch.knn.index.query.KNNQueryBuilder.NPROBE_FIELD; /** * MethodParameters are engine and algorithm related parameters that clients can pass in knn query @@ -34,11 +36,7 @@ public enum MethodParameter { EF_SEARCH(METHOD_PARAMETER_EF_SEARCH, Version.V_2_16_0, EF_SEARCH_FIELD) { @Override public Integer parse(Object value) { - try { - return Integer.parseInt(String.valueOf(value)); - } catch (final NumberFormatException e) { - throw new IllegalArgumentException(METHOD_PARAMETER_EF_SEARCH + " value must be an integer"); - } + return parseInteger(value, METHOD_PARAMETER_EF_SEARCH); } @Override @@ -47,11 +45,30 @@ public ValidationException validate(Object value) { if (ef != null && ef > 0) { return null; } - ; + ValidationException validationException = new ValidationException(); validationException.addValidationError(METHOD_PARAMETER_EF_SEARCH + " should be greater than 0"); return validationException; } + }, + + NPROBE(METHOD_PARAMETER_NPROBES, Version.V_2_16_0, NPROBE_FIELD) { + @Override + public Integer parse(Object value) { + return parseInteger(value, METHOD_PARAMETER_EF_SEARCH); + } + + @Override + public ValidationException validate(Object value) { + final Integer nprobe = parse(value); + if (nprobe != null && nprobe > 0) { + return null; + } + + ValidationException validationException = new ValidationException(); + validationException.addValidationError(METHOD_PARAMETER_NPROBES + " should be greater than 0"); + return validationException; + } }; private final String name; @@ -74,4 +91,12 @@ public static MethodParameter enumOf(final String name) { } return PARAMETERS_DIR.get(name); } + + private static Integer parseInteger(Object value, String name) { + try { + return Integer.parseInt(String.valueOf(value)); + } catch (final NumberFormatException e) { + throw new IllegalArgumentException(name + " value must be an integer"); + } + } } diff --git a/src/main/java/org/opensearch/knn/index/util/DefaultIVFContext.java b/src/main/java/org/opensearch/knn/index/util/DefaultIVFContext.java new file mode 100644 index 000000000..dbccbe9cb --- /dev/null +++ b/src/main/java/org/opensearch/knn/index/util/DefaultIVFContext.java @@ -0,0 +1,30 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.knn.index.util; + +import com.google.common.collect.ImmutableMap; +import org.opensearch.knn.index.Parameter; +import org.opensearch.knn.index.query.request.MethodParameter; + +import java.util.Map; + +public final class DefaultIVFContext implements EngineSpecificMethodContext { + + private final Map> supportedMethodParameters = ImmutableMap.>builder() + .put(MethodParameter.NPROBE.getName(), new Parameter.IntegerParameter(MethodParameter.NPROBE.getName(), null, value -> true)) + .build(); + + @Override + public Map> supportedMethodParameters(QueryContext context) { + return supportedMethodParameters; + } +} diff --git a/src/main/java/org/opensearch/knn/index/util/Faiss.java b/src/main/java/org/opensearch/knn/index/util/Faiss.java index 711c206f5..5f3add832 100644 --- a/src/main/java/org/opensearch/knn/index/util/Faiss.java +++ b/src/main/java/org/opensearch/knn/index/util/Faiss.java @@ -333,7 +333,7 @@ private Faiss( ) { super( methods, - Map.of(METHOD_HNSW, new DefaultHnswContext(), METHOD_IVF, EngineSpecificMethodContext.EMPTY), + Map.of(METHOD_HNSW, new DefaultHnswContext(), METHOD_IVF, new DefaultIVFContext()), scoreTranslation, currentVersion, extension diff --git a/src/test/java/org/opensearch/knn/index/FaissIT.java b/src/test/java/org/opensearch/knn/index/FaissIT.java index 0af06b7e5..204d9f468 100644 --- a/src/test/java/org/opensearch/knn/index/FaissIT.java +++ b/src/test/java/org/opensearch/knn/index/FaissIT.java @@ -617,6 +617,7 @@ public void testIVFSQFP16_whenIndexedAndQueried_thenSucceed() { indexTestData(indexName, fieldName, dimension, numDocs); queryTestData(indexName, fieldName, dimension, numDocs); + queryTestData(indexName, fieldName, dimension, numDocs, Map.of("nprobes", 100)); deleteKNNIndex(indexName); validateGraphEviction(); } @@ -1622,12 +1623,23 @@ protected void setupKNNIndexForFilterQuery() throws Exception { refreshIndex(INDEX_NAME); } - private void queryTestData(final String indexName, final String fieldName, final int dimension, final int numDocs) throws IOException { + @SneakyThrows + private void queryTestData(final String indexName, final String fieldName, final int dimension, final int numDocs) { + queryTestData(indexName, fieldName, dimension, numDocs, null); + } + + private void queryTestData( + final String indexName, + final String fieldName, + final int dimension, + final int numDocs, + Map methodParams + ) throws IOException, ParseException { float[] queryVector = new float[dimension]; Arrays.fill(queryVector, (float) numDocs); int k = 10; - Response searchResponse = searchKNNIndex(indexName, new KNNQueryBuilder(fieldName, queryVector, k), k); + Response searchResponse = searchKNNIndex(indexName, buildSearchQuery(fieldName, k, queryVector, methodParams), k); List results = parseSearchResponse(EntityUtils.toString(searchResponse.getEntity()), fieldName); assertEquals(k, results.size()); for (int i = 0; i < k; i++) { diff --git a/src/test/java/org/opensearch/knn/index/LuceneEngineIT.java b/src/test/java/org/opensearch/knn/index/LuceneEngineIT.java index 0d2ae9e24..454f5c724 100644 --- a/src/test/java/org/opensearch/knn/index/LuceneEngineIT.java +++ b/src/test/java/org/opensearch/knn/index/LuceneEngineIT.java @@ -510,7 +510,7 @@ private void validateQueries(SpaceType spaceType, String fieldName, Map knnResults = parseSearchResponse(responseBody, fieldName); assertEquals(k, knnResults.size()); @@ -525,27 +525,6 @@ private void validateQueries(SpaceType spaceType, String fieldName, Map methodParams) { - XContentBuilder builder = XContentFactory.jsonBuilder() - .startObject() - .startObject("query") - .startObject("knn") - .startObject(fieldName) - .field("vector", vector) - .field("k", k); - if (methodParams != null) { - builder.startObject("method_parameters"); - for (Map.Entry entry : methodParams.entrySet()) { - builder.field(entry.getKey(), entry.getValue()); - } - builder.endObject(); - } - - builder.endObject().endObject().endObject().endObject(); - return builder; - } - private List queryResults(final float[] searchVector, final int k) throws Exception { final String responseBody = EntityUtils.toString( searchKNNIndex(INDEX_NAME, new KNNQueryBuilder(FIELD_NAME, searchVector, k), k).getEntity() diff --git a/src/test/java/org/opensearch/knn/index/query/InvalidSearchQueryIT.java b/src/test/java/org/opensearch/knn/index/query/InvalidSearchQueryIT.java new file mode 100644 index 000000000..071162789 --- /dev/null +++ b/src/test/java/org/opensearch/knn/index/query/InvalidSearchQueryIT.java @@ -0,0 +1,156 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.knn.index.query; + +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import lombok.AllArgsConstructor; +import lombok.SneakyThrows; +import org.opensearch.client.Request; +import org.opensearch.client.ResponseException; +import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.knn.KNNRestTestCase; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Collection; + +import static com.carrotsearch.randomizedtesting.RandomizedTest.$; +import static com.carrotsearch.randomizedtesting.RandomizedTest.$$; + +@AllArgsConstructor +public class InvalidSearchQueryIT extends KNNRestTestCase { + + private String description; + private XContentBuilder xContentBuilder; + + @ParametersFactory(argumentFormatting = "description:%1$s; request:%2$s, expectedexception:%3$s") + public static Collection parameters() throws IOException { + /** + * Valid query: + * { + * query: { + * knn: { + * test_field: { + * vector: [1.0, 2.0], + * k: 1, + * method_parameter: { + * ef_search: 10 + * } + * } + * } + * } + * } + */ + + return Arrays.asList( + $$( + $( + "Empty method_parameter", + XContentFactory.jsonBuilder() + .startObject() + .startObject("query") + .startObject("knn") + .startObject(FIELD_NAME) + .field("vector", new float[] { 1.0f, 2.0f }) + .field("k", 1) + .startObject("method_parameter") + .endObject() + .endObject() + .endObject() + .endObject() + .endObject() + ), + $( + "ef_search string", + XContentFactory.jsonBuilder() + .startObject() + .startObject("query") + .startObject("knn") + .startObject(FIELD_NAME) + .field("vector", new float[] { 1.0f, 2.0f }) + .field("k", 1) + .startObject("method_parameter") + .field("ef_search", "string value") + .endObject() + .endObject() + .endObject() + .endObject() + .endObject() + ), + $( + "ef_search less than 0", + XContentFactory.jsonBuilder() + .startObject() + .startObject("query") + .startObject("knn") + .startObject(FIELD_NAME) + .field("vector", new float[] { 1.0f, 2.0f }) + .field("k", 1) + .startObject("method_parameter") + .field("ef_search", -1) + .endObject() + .endObject() + .endObject() + .endObject() + .endObject() + ), + $( + "nprobes string", + XContentFactory.jsonBuilder() + .startObject() + .startObject("query") + .startObject("knn") + .startObject(FIELD_NAME) + .field("vector", new float[] { 1.0f, 2.0f }) + .field("k", 1) + .startObject("method_parameter") + .field("nprobes", "string value") + .endObject() + .endObject() + .endObject() + .endObject() + .endObject() + ), + $( + "nprobes less than 0", + XContentFactory.jsonBuilder() + .startObject() + .startObject("query") + .startObject("knn") + .startObject(FIELD_NAME) + .field("vector", new float[] { 1.0f, 2.0f }) + .field("k", 1) + .startObject("method_parameter") + .field("nprobes", -10) + .endObject() + .endObject() + .endObject() + .endObject() + .endObject() + ) + ) + ); + } + + @SneakyThrows + public void testEndToEnd_whenMethodIsHNSWFlat_thenSucceed() { + Request request = new Request("POST", "/dummy_index/_search"); + request.setJsonEntity(xContentBuilder.toString()); + + request.addParameter("size", Integer.toString(10)); + request.addParameter("explain", Boolean.toString(true)); + request.addParameter("search_type", "query_then_fetch"); + + expectThrows(ResponseException.class, () -> client().performRequest(request)); + } +} diff --git a/src/test/java/org/opensearch/knn/index/query/KNNQueryBuilderTests.java b/src/test/java/org/opensearch/knn/index/query/KNNQueryBuilderTests.java index ee498e5b7..db37259d6 100644 --- a/src/test/java/org/opensearch/knn/index/query/KNNQueryBuilderTests.java +++ b/src/test/java/org/opensearch/knn/index/query/KNNQueryBuilderTests.java @@ -866,7 +866,6 @@ public void testDoToQuery_WhenknnQueryWithFilterAndFaissEngine_thenSuccess() { assertEquals(HNSW_METHOD_PARAMS, ((KNNQuery) query).getMethodParameters()); } - /** This test should be uncommented once we have nprobs. Considering engine instance is static its not possible to test this right now public void testDoToQuery_ThrowsIllegalArgumentExceptionForUnknownMethodParameter() { QueryShardContext mockQueryShardContext = mock(QueryShardContext.class); @@ -882,11 +881,11 @@ public void testDoToQuery_ThrowsIllegalArgumentExceptionForUnknownMethodParamete .fieldName(FIELD_NAME) .vector(queryVector) .k(K) - .methodParameters(Map.of("ef_search", 10)) + .methodParameters(Map.of("nprobes", 10)) .build(); expectThrows(IllegalArgumentException.class, () -> knnQueryBuilder.doToQuery(mockQueryShardContext)); - }**/ + } public void testDoToQuery_whenknnQueryWithFilterAndNmsLibEngine_thenException() { float[] queryVector = { 1.0f, 2.0f, 3.0f, 4.0f }; diff --git a/src/test/java/org/opensearch/knn/index/query/parser/MethodParametersParserTests.java b/src/test/java/org/opensearch/knn/index/query/parser/MethodParametersParserTests.java index f2924fb4f..984c72b89 100644 --- a/src/test/java/org/opensearch/knn/index/query/parser/MethodParametersParserTests.java +++ b/src/test/java/org/opensearch/knn/index/query/parser/MethodParametersParserTests.java @@ -35,6 +35,9 @@ public void testValidateMethodParameters() { ValidationException validationException3 = validateMethodParameters(Map.of("ef_search", 10)); assertNull(validationException3); + + ValidationException validationException4 = validateMethodParameters(Map.of("nprobes", 0)); + assertTrue(validationException4.getMessage().contains("Validation Failed: 1: nprobes should be greater than 0")); } @SneakyThrows @@ -80,5 +83,15 @@ public void testFromXContent() { builder = XContentFactory.jsonBuilder().startObject().endObject(); XContentParser parser4 = createParser(builder); expectThrows(ParsingException.class, () -> MethodParametersParser.fromXContent(parser4)); + + // nprobes string + builder = XContentFactory.jsonBuilder().startObject().field("nprobes", "string").endObject(); + XContentParser parser5 = createParser(builder); + expectThrows(ParsingException.class, () -> MethodParametersParser.fromXContent(parser5)); + + // nprobes Valid + builder = XContentFactory.jsonBuilder().startObject().field("nprobes", 10).endObject(); + XContentParser parser6 = createParser(builder); + assertEquals(Map.of("nprobes", 10), MethodParametersParser.fromXContent(parser6)); } } diff --git a/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java b/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java index 179caf951..73e0bd724 100644 --- a/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java +++ b/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java @@ -1642,4 +1642,25 @@ protected void addKnnDocWithAttributes( Response response = client().performRequest(request); assertEquals(request.getEndpoint() + ": failed", RestStatus.OK, RestStatus.fromCode(response.getStatusLine().getStatusCode())); } + + @SneakyThrows + protected XContentBuilder buildSearchQuery(String fieldName, int k, float[] vector, Map methodParams) { + XContentBuilder builder = XContentFactory.jsonBuilder() + .startObject() + .startObject("query") + .startObject("knn") + .startObject(fieldName) + .field("vector", vector) + .field("k", k); + if (methodParams != null) { + builder.startObject("method_parameters"); + for (Map.Entry entry : methodParams.entrySet()) { + builder.field(entry.getKey(), entry.getValue()); + } + builder.endObject(); + } + + builder.endObject().endObject().endObject().endObject(); + return builder; + } } From ee4b37bb35ecabb32686b0c04c1eb1827c32fcaa Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Wed, 17 Jul 2024 07:11:45 -0700 Subject: [PATCH 294/416] Apply custom patch only once by comparing the last patch id (#1833) (#1843) Signed-off-by: Heemin Kim (cherry picked from commit 1b425df7c90fe78fc28d305be3be4dfbdbe03ba9) Co-authored-by: Heemin Kim --- .github/workflows/CI.yml | 2 -- .github/workflows/test_security.yml | 2 -- CHANGELOG.md | 1 + jni/cmake/init-faiss.cmake | 44 ++++++++++++++++++++++------- jni/cmake/init-nmslib.cmake | 39 +++++++++++++++++++------ 5 files changed, 66 insertions(+), 22 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 0b9b24d98..e6dec4daf 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -37,8 +37,6 @@ jobs: steps: - name: Checkout k-NN uses: actions/checkout@v1 - with: - submodules: true # Setup git user so that patches for native libraries can be applied and committed - name: Setup git user diff --git a/.github/workflows/test_security.yml b/.github/workflows/test_security.yml index e0f2dbf45..77b726a69 100644 --- a/.github/workflows/test_security.yml +++ b/.github/workflows/test_security.yml @@ -37,8 +37,6 @@ jobs: steps: - name: Checkout k-NN uses: actions/checkout@v1 - with: - submodules: true # Setup git user so that patches for native libraries can be applied and committed - name: Setup git user run: | diff --git a/CHANGELOG.md b/CHANGELOG.md index 933be1f3c..7636ce9d5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), * Release memory properly for an array type [#1820](https://github.com/opensearch-project/k-NN/pull/1820) * FIX Same Suffix Cause Recall Drop to zero [#1802](https://github.com/opensearch-project/k-NN/pull/1802) ### Infrastructure +* Apply custom patch only once by comparing the last patch id [#1833](https://github.com/opensearch-project/k-NN/pull/1833) ### Documentation ### Maintenance * Bump faiss commit to 33c0ba5 [#1796](https://github.com/opensearch-project/k-NN/pull/1796) diff --git a/jni/cmake/init-faiss.cmake b/jni/cmake/init-faiss.cmake index befed4703..c2ec24a3b 100644 --- a/jni/cmake/init-faiss.cmake +++ b/jni/cmake/init-faiss.cmake @@ -12,20 +12,44 @@ if (NOT EXISTS ${FAISS_REPO_DIR}) execute_process(COMMAND git submodule update --init -- external/faiss WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}) endif () -# Check if patch exist, this is to skip git apply during CI build. See CI.yml with ubuntu. -find_path(PATCH_FILE NAMES 0001-Custom-patch-to-support-multi-vector.patch 0002-Enable-precomp-table-to-be-shared-ivfpq.patch 0003-Custom-patch-to-support-range-search-params.patch 0004-Custom-patch-to-support-binary-vector.patch PATHS ${CMAKE_CURRENT_SOURCE_DIR}/patches/faiss NO_DEFAULT_PATH) +# Define list of patch files +set(PATCH_FILE_LIST) +list(APPEND PATCH_FILE_LIST "${CMAKE_CURRENT_SOURCE_DIR}/patches/faiss/0001-Custom-patch-to-support-multi-vector.patch") +list(APPEND PATCH_FILE_LIST "${CMAKE_CURRENT_SOURCE_DIR}/patches/faiss/0002-Enable-precomp-table-to-be-shared-ivfpq.patch") +list(APPEND PATCH_FILE_LIST "${CMAKE_CURRENT_SOURCE_DIR}/patches/faiss/0003-Custom-patch-to-support-range-search-params.patch") +list(APPEND PATCH_FILE_LIST "${CMAKE_CURRENT_SOURCE_DIR}/patches/faiss/0004-Custom-patch-to-support-binary-vector.patch") -# If it exists, apply patches -if (EXISTS ${PATCH_FILE}) - message(STATUS "Applying custom patches.") - execute_process(COMMAND git ${GIT_PATCH_COMMAND} --3way --ignore-space-change --ignore-whitespace ${CMAKE_CURRENT_SOURCE_DIR}/patches/faiss/0001-Custom-patch-to-support-multi-vector.patch WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/external/faiss ERROR_VARIABLE ERROR_MSG RESULT_VARIABLE RESULT_CODE) - execute_process(COMMAND git ${GIT_PATCH_COMMAND} --3way --ignore-space-change --ignore-whitespace ${CMAKE_CURRENT_SOURCE_DIR}/patches/faiss/0002-Enable-precomp-table-to-be-shared-ivfpq.patch WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/external/faiss ERROR_VARIABLE ERROR_MSG RESULT_VARIABLE RESULT_CODE) - execute_process(COMMAND git ${GIT_PATCH_COMMAND} --3way --ignore-space-change --ignore-whitespace ${CMAKE_CURRENT_SOURCE_DIR}/patches/faiss/0003-Custom-patch-to-support-range-search-params.patch WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/external/faiss ERROR_VARIABLE ERROR_MSG RESULT_VARIABLE RESULT_CODE) - execute_process(COMMAND git ${GIT_PATCH_COMMAND} --3way --ignore-space-change --ignore-whitespace ${CMAKE_CURRENT_SOURCE_DIR}/patches/faiss/0004-Custom-patch-to-support-binary-vector.patch WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/external/faiss ERROR_VARIABLE ERROR_MSG RESULT_VARIABLE RESULT_CODE) +# Get patch id of the last commit +execute_process(COMMAND sh -c "git --no-pager show HEAD | git patch-id --stable" OUTPUT_VARIABLE PATCH_ID_OUTPUT_FROM_COMMIT WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/external/faiss) +string(REPLACE " " ";" PATCH_ID_LIST_FROM_COMMIT ${PATCH_ID_OUTPUT_FROM_COMMIT}) +list(GET PATCH_ID_LIST_FROM_COMMIT 0 PATCH_ID_FROM_COMMIT) + +# Find all patch files need to apply +list(SORT PATCH_FILE_LIST ORDER DESCENDING) +set(PATCH_FILES_TO_APPLY) +foreach(PATCH_FILE IN LISTS PATCH_FILE_LIST) + # Get patch id of a patch file + execute_process(COMMAND sh -c "cat ${PATCH_FILE} | git patch-id --stable" OUTPUT_VARIABLE PATCH_ID_OUTPUT) + string(REPLACE " " ";" PATCH_ID_LIST ${PATCH_ID_OUTPUT}) + list(GET PATCH_ID_LIST 0 PATCH_ID) + + # Add the file to patch list if patch id does not match + if (${PATCH_ID} STREQUAL ${PATCH_ID_FROM_COMMIT}) + break() + else() + list(APPEND PATCH_FILES_TO_APPLY ${PATCH_FILE}) + endif() +endforeach() + +# Apply patch files +list(SORT PATCH_FILES_TO_APPLY) +foreach(PATCH_FILE IN LISTS PATCH_FILES_TO_APPLY) + message(STATUS "Applying patch of ${PATCH_FILE}") + execute_process(COMMAND git ${GIT_PATCH_COMMAND} --3way --ignore-space-change --ignore-whitespace ${PATCH_FILE} WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/external/faiss ERROR_VARIABLE ERROR_MSG RESULT_VARIABLE RESULT_CODE) if(RESULT_CODE) message(FATAL_ERROR "Failed to apply patch:\n${ERROR_MSG}") endif() -endif() +endforeach() if (${CMAKE_SYSTEM_NAME} STREQUAL Darwin) if(CMAKE_C_COMPILER_ID MATCHES "Clang\$") diff --git a/jni/cmake/init-nmslib.cmake b/jni/cmake/init-nmslib.cmake index 387dce6bc..56c52bb69 100644 --- a/jni/cmake/init-nmslib.cmake +++ b/jni/cmake/init-nmslib.cmake @@ -12,18 +12,41 @@ if (NOT EXISTS ${NMS_REPO_DIR}) execute_process(COMMAND git submodule update --init -- external/nmslib WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}) endif () -# Check if patch exist, this is to skip git apply during CI build. See CI.yml with ubuntu. -find_path(PATCH_FILE NAMES 0001-Initialize-maxlevel-during-add-from-enterpoint-level.patch 0002-Adds-ability-to-pass-ef-parameter-in-the-query-for-h.patch PATHS ${CMAKE_CURRENT_SOURCE_DIR}/patches/nmslib NO_DEFAULT_PATH) +# Define list of patch files +set(PATCH_FILE_LIST) +list(APPEND PATCH_FILE_LIST "${CMAKE_CURRENT_SOURCE_DIR}/patches/nmslib/0001-Initialize-maxlevel-during-add-from-enterpoint-level.patch") +list(APPEND PATCH_FILE_LIST "${CMAKE_CURRENT_SOURCE_DIR}/patches/nmslib/0002-Adds-ability-to-pass-ef-parameter-in-the-query-for-h.patch") -# If it exists, apply patches -if (EXISTS ${PATCH_FILE}) - message(STATUS "Applying custom patches.") - execute_process(COMMAND git ${GIT_PATCH_COMMAND} --3way --ignore-space-change --ignore-whitespace ${CMAKE_CURRENT_SOURCE_DIR}/patches/nmslib/0001-Initialize-maxlevel-during-add-from-enterpoint-level.patch WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/external/nmslib ERROR_VARIABLE ERROR_MSG RESULT_VARIABLE RESULT_CODE) - execute_process(COMMAND git ${GIT_PATCH_COMMAND} --3way --ignore-space-change --ignore-whitespace ${CMAKE_CURRENT_SOURCE_DIR}/patches/nmslib/0002-Adds-ability-to-pass-ef-parameter-in-the-query-for-h.patch WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/external/nmslib ERROR_VARIABLE ERROR_MSG RESULT_VARIABLE RESULT_CODE) +# Get patch id of the last commit +execute_process(COMMAND sh -c "git --no-pager show HEAD | git patch-id --stable" OUTPUT_VARIABLE PATCH_ID_OUTPUT_FROM_COMMIT WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/external/nmslib) +string(REPLACE " " ";" PATCH_ID_LIST_FROM_COMMIT ${PATCH_ID_OUTPUT_FROM_COMMIT}) +list(GET PATCH_ID_LIST_FROM_COMMIT 0 PATCH_ID_FROM_COMMIT) +# Find all patch files need to apply +list(SORT PATCH_FILE_LIST ORDER DESCENDING) +set(PATCH_FILES_TO_APPLY) +foreach(PATCH_FILE IN LISTS PATCH_FILE_LIST) + # Get patch id of a patch file + execute_process(COMMAND sh -c "cat ${PATCH_FILE} | git patch-id --stable" OUTPUT_VARIABLE PATCH_ID_OUTPUT) + string(REPLACE " " ";" PATCH_ID_LIST ${PATCH_ID_OUTPUT}) + list(GET PATCH_ID_LIST 0 PATCH_ID) + + # Add the file to patch list if patch id does not match + if (${PATCH_ID} STREQUAL ${PATCH_ID_FROM_COMMIT}) + break() + else() + list(APPEND PATCH_FILES_TO_APPLY ${PATCH_FILE}) + endif() +endforeach() + +# Apply patch files +list(SORT PATCH_FILES_TO_APPLY) +foreach(PATCH_FILE IN LISTS PATCH_FILES_TO_APPLY) + message(STATUS "Applying patch of ${PATCH_FILE}") + execute_process(COMMAND git ${GIT_PATCH_COMMAND} --3way --ignore-space-change --ignore-whitespace ${PATCH_FILE} WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/external/nmslib ERROR_VARIABLE ERROR_MSG RESULT_VARIABLE RESULT_CODE) if(RESULT_CODE) message(FATAL_ERROR "Failed to apply patch:\n${ERROR_MSG}") endif() -endif() +endforeach() add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/external/nmslib/similarity_search) From 5c4fad370080feb947852c9ee3175e5e17e8071f Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Wed, 17 Jul 2024 18:02:12 -0700 Subject: [PATCH 295/416] Add painless script support for hamming with binary vector data type (#1839) (#1847) Signed-off-by: Heemin Kim (cherry picked from commit de6084ff2040ecc475405d203c35cb7903968f36) Co-authored-by: Heemin Kim --- CHANGELOG.md | 1 + .../knn/plugin/script/KNNScoringUtil.java | 315 ++++++++++++------ .../knn/plugin/script/knn_allowlist.txt | 1 + .../knn/common/KNNValidationUtilTests.java | 2 +- .../plugin/script/KNNScoringUtilTests.java | 49 +++ 5 files changed, 263 insertions(+), 105 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7636ce9d5..dd712c66e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), * Adds dynamic query parameter nprobes [#1792](https://github.com/opensearch-project/k-NN/pull/1792) * Add binary format support with HNSW method in Faiss Engine [#1781](https://github.com/opensearch-project/k-NN/pull/1781) * Add script scoring support for knn field with binary data type [#1826](https://github.com/opensearch-project/k-NN/pull/1826) +* Add painless script support for hamming with binary vector data type [#1839](https://github.com/opensearch-project/k-NN/pull/1839) ### Enhancements * Switch from byte stream to byte ref for serde [#1825](https://github.com/opensearch-project/k-NN/pull/1825) ### Bug Fixes diff --git a/src/main/java/org/opensearch/knn/plugin/script/KNNScoringUtil.java b/src/main/java/org/opensearch/knn/plugin/script/KNNScoringUtil.java index 8493ea5bd..f61ae4349 100644 --- a/src/main/java/org/opensearch/knn/plugin/script/KNNScoringUtil.java +++ b/src/main/java/org/opensearch/knn/plugin/script/KNNScoringUtil.java @@ -7,6 +7,7 @@ import java.math.BigInteger; import java.util.List; +import java.util.Locale; import java.util.Objects; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -40,6 +41,52 @@ private static void requireEqualDimension(final float[] queryVector, final float } } + /** + * checks both query vector and input vector has equal dimension + * + * @param queryVector query vector + * @param inputVector input vector + * @throws IllegalArgumentException if query vector and input vector has different dimensions + */ + private static void requireEqualDimension(final byte[] queryVector, final byte[] inputVector) { + Objects.requireNonNull(queryVector); + Objects.requireNonNull(inputVector); + if (queryVector.length != inputVector.length) { + String errorMessage = String.format( + "query vector dimension mismatch. Expected: %d, Given: %d", + inputVector.length, + queryVector.length + ); + throw new IllegalArgumentException(errorMessage); + } + } + + private static void requireNonBinaryType(final String spaceName, final VectorDataType vectorDataType) { + if (VectorDataType.BINARY == vectorDataType) { + throw new IllegalArgumentException( + String.format( + Locale.ROOT, + "Incompatible field_type for %s space. The data type should be either float or byte but got %s", + spaceName, + vectorDataType.getValue() + ) + ); + } + } + + private static void requireBinaryType(final String spaceName, final VectorDataType vectorDataType) { + if (VectorDataType.BINARY != vectorDataType) { + throw new IllegalArgumentException( + String.format( + Locale.ROOT, + "Incompatible field_type for %s space. The data type should be binary but got %s", + spaceName, + vectorDataType.getValue() + ) + ); + } + } + /** * This method calculates L2 squared distance between query vector * and input vector @@ -52,13 +99,13 @@ public static float l2Squared(float[] queryVector, float[] inputVector) { return VectorUtil.squareDistance(queryVector, inputVector); } - private static float[] toFloat(List inputVector, VectorDataType vectorDataType) { + private static float[] toFloat(final List inputVector, final VectorDataType vectorDataType) { Objects.requireNonNull(inputVector); float[] value = new float[inputVector.size()]; int index = 0; for (final Number val : inputVector) { float floatValue = val.floatValue(); - if (VectorDataType.BYTE == vectorDataType) { + if (VectorDataType.BYTE == vectorDataType || VectorDataType.BINARY == vectorDataType) { validateByteVectorValue(floatValue, vectorDataType); } value[index++] = floatValue; @@ -66,24 +113,35 @@ private static float[] toFloat(List inputVector, VectorDataType vectorDa return value; } + private static byte[] toByte(final List inputVector, final VectorDataType vectorDataType) { + Objects.requireNonNull(inputVector); + byte[] value = new byte[inputVector.size()]; + int index = 0; + for (final Number val : inputVector) { + float floatValue = val.floatValue(); + if (VectorDataType.BYTE == vectorDataType || VectorDataType.BINARY == vectorDataType) { + validateByteVectorValue(floatValue, vectorDataType); + } + value[index++] = val.byteValue(); + } + return value; + } + /** - * Allowlisted l2Squared method for users to calculate L2 squared distance between query vector - * and document vectors - * Example - * "script": { - * "source": "1/(1 + l2Squared(params.query_vector, doc[params.field]))", - * "params": { - * "query_vector": [1, 2, 3.4], - * "field": "my_dense_vector" - * } - * } + * This method calculates cosine similarity * * @param queryVector query vector - * @param docValues script doc values - * @return L2 score + * @param inputVector input vector + * @return cosine score */ - public static float l2Squared(List queryVector, KNNVectorScriptDocValues docValues) { - return l2Squared(toFloat(queryVector, docValues.getVectorDataType()), docValues.getValue()); + public static float cosinesimil(float[] queryVector, float[] inputVector) { + requireEqualDimension(queryVector, inputVector); + try { + return VectorUtil.cosine(queryVector, inputVector); + } catch (IllegalArgumentException | AssertionError e) { + logger.debug("Invalid vectors for cosine. Returning minimum score to put this result to end"); + return 0.0f; + } } /** @@ -111,68 +169,6 @@ public static float cosinesimilOptimized(float[] queryVector, float[] inputVecto return (float) (dotProduct / (Math.sqrt(normalizedProduct))); } - /** - * Allowlisted cosineSimilarity method that can be used in a script to avoid repeated - * calculation of normalization for the query vector. - * Example: - * "script": { - * "source": "cosineSimilarity(params.query_vector, docs[field], 1.0) ", - * "params": { - * "query_vector": [1, 2, 3.4], - * "field": "my_dense_vector" - * } - * } - * - * @param queryVector query vector - * @param docValues script doc values - * @param queryVectorMagnitude the magnitude of the query vector. - * @return cosine score - */ - public static float cosineSimilarity(List queryVector, KNNVectorScriptDocValues docValues, Number queryVectorMagnitude) { - float[] inputVector = toFloat(queryVector, docValues.getVectorDataType()); - SpaceType.COSINESIMIL.validateVector(inputVector); - return cosinesimilOptimized(inputVector, docValues.getValue(), queryVectorMagnitude.floatValue()); - } - - /** - * This method calculates cosine similarity - * - * @param queryVector query vector - * @param inputVector input vector - * @return cosine score - */ - public static float cosinesimil(float[] queryVector, float[] inputVector) { - requireEqualDimension(queryVector, inputVector); - try { - return VectorUtil.cosine(queryVector, inputVector); - } catch (IllegalArgumentException | AssertionError e) { - logger.debug("Invalid vectors for cosine. Returning minimum score to put this result to end"); - return 0.0f; - } - } - - /** - * Allowlisted cosineSimilarity method for users to calculate cosine similarity between query vectors and - * document vectors - * Example: - * "script": { - * "source": "cosineSimilarity(params.query_vector, docs[field]) ", - * "params": { - * "query_vector": [1, 2, 3.4], - * "field": "my_dense_vector" - * } - * } - * - * @param queryVector query vector - * @param docValues script doc values - * @return cosine score - */ - public static float cosineSimilarity(List queryVector, KNNVectorScriptDocValues docValues) { - float[] inputVector = toFloat(queryVector, docValues.getVectorDataType()); - SpaceType.COSINESIMIL.validateVector(inputVector); - return cosinesimil(inputVector, docValues.getValue()); - } - /** * This method calculates hamming distance on 2 BigIntegers * @@ -204,6 +200,7 @@ public static float calculateHammingBit(Long queryLong, Long inputLong) { * @return hamming distance */ public static float calculateHammingBit(byte[] queryVector, byte[] inputVector) { + requireEqualDimension(queryVector, inputVector); return VectorUtil.xorBitCount(queryVector, inputVector); } @@ -216,6 +213,7 @@ public static float calculateHammingBit(byte[] queryVector, byte[] inputVector) * @return L1 score */ public static float l1Norm(float[] queryVector, float[] inputVector) { + requireEqualDimension(queryVector, inputVector); float distance = 0; for (int i = 0; i < inputVector.length; i++) { float diff = queryVector[i] - inputVector[i]; @@ -224,26 +222,6 @@ public static float l1Norm(float[] queryVector, float[] inputVector) { return distance; } - /** - * Allowlisted l1distance method for users to calculate L1 distance between query vector - * and document vectors - * Example - * "script": { - * "source": "1/(1 + l1Norm(params.query_vector, doc[params.field]))", - * "params": { - * "query_vector": [1, 2, 3.4], - * "field": "my_dense_vector" - * } - * } - * - * @param queryVector query vector - * @param docValues script doc values - * @return L1 score - */ - public static float l1Norm(List queryVector, KNNVectorScriptDocValues docValues) { - return l1Norm(toFloat(queryVector, docValues.getVectorDataType()), docValues.getValue()); - } - /** * This method calculates L-inf distance between query vector * and input vector @@ -253,6 +231,7 @@ public static float l1Norm(List queryVector, KNNVectorScriptDocValues do * @return L-inf score */ public static float lInfNorm(float[] queryVector, float[] inputVector) { + requireEqualDimension(queryVector, inputVector); float distance = 0; for (int i = 0; i < inputVector.length; i++) { float diff = queryVector[i] - inputVector[i]; @@ -261,6 +240,46 @@ public static float lInfNorm(float[] queryVector, float[] inputVector) { return distance; } + /** + * This method calculates dot product distance between query vector + * and input vector + * + * @param queryVector query vector + * @param inputVector input vector + * @return dot product score + */ + public static float innerProduct(float[] queryVector, float[] inputVector) { + requireEqualDimension(queryVector, inputVector); + return VectorUtil.dotProduct(queryVector, inputVector); + } + + /** + ********************************************************************************************* + * Functions to be used in painless script which is defined in knn_allowlist.txt + ********************************************************************************************* + */ + + /** + * Allowlisted l2Squared method for users to calculate L2 squared distance between query vector + * and document vectors + * Example + * "script": { + * "source": "1/(1 + l2Squared(params.query_vector, doc[params.field]))", + * "params": { + * "query_vector": [1, 2, 3.4], + * "field": "my_dense_vector" + * } + * } + * + * @param queryVector query vector + * @param docValues script doc values + * @return L2 score + */ + public static float l2Squared(List queryVector, KNNVectorScriptDocValues docValues) { + requireNonBinaryType("l2Squared", docValues.getVectorDataType()); + return l2Squared(toFloat(queryVector, docValues.getVectorDataType()), docValues.getValue()); + } + /** * Allowlisted lInfNorm method for users to calculate L-inf distance between query vector * and document vectors @@ -278,19 +297,29 @@ public static float lInfNorm(float[] queryVector, float[] inputVector) { * @return L-inf score */ public static float lInfNorm(List queryVector, KNNVectorScriptDocValues docValues) { + requireNonBinaryType("lInfNorm", docValues.getVectorDataType()); return lInfNorm(toFloat(queryVector, docValues.getVectorDataType()), docValues.getValue()); } /** - * This method calculates dot product distance between query vector - * and input vector + * Allowlisted l1distance method for users to calculate L1 distance between query vector + * and document vectors + * Example + * "script": { + * "source": "1/(1 + l1Norm(params.query_vector, doc[params.field]))", + * "params": { + * "query_vector": [1, 2, 3.4], + * "field": "my_dense_vector" + * } + * } * * @param queryVector query vector - * @param inputVector input vector - * @return dot product score + * @param docValues script doc values + * @return L1 score */ - public static float innerProduct(float[] queryVector, float[] inputVector) { - return VectorUtil.dotProduct(queryVector, inputVector); + public static float l1Norm(List queryVector, KNNVectorScriptDocValues docValues) { + requireNonBinaryType("l1Norm", docValues.getVectorDataType()); + return l1Norm(toFloat(queryVector, docValues.getVectorDataType()), docValues.getValue()); } /** @@ -310,6 +339,84 @@ public static float innerProduct(float[] queryVector, float[] inputVector) { * @return inner product score */ public static float innerProduct(List queryVector, KNNVectorScriptDocValues docValues) { + requireNonBinaryType("innerProduct", docValues.getVectorDataType()); return innerProduct(toFloat(queryVector, docValues.getVectorDataType()), docValues.getValue()); } + + /** + * Allowlisted cosineSimilarity method for users to calculate cosine similarity between query vectors and + * document vectors + * Example: + * "script": { + * "source": "cosineSimilarity(params.query_vector, docs[field]) ", + * "params": { + * "query_vector": [1, 2, 3.4], + * "field": "my_dense_vector" + * } + * } + * + * @param queryVector query vector + * @param docValues script doc values + * @return cosine score + */ + public static float cosineSimilarity(List queryVector, KNNVectorScriptDocValues docValues) { + requireNonBinaryType("cosineSimilarity", docValues.getVectorDataType()); + float[] inputVector = toFloat(queryVector, docValues.getVectorDataType()); + SpaceType.COSINESIMIL.validateVector(inputVector); + return cosinesimil(inputVector, docValues.getValue()); + } + + /** + * Allowlisted cosineSimilarity method that can be used in a script to avoid repeated + * calculation of normalization for the query vector. + * Example: + * "script": { + * "source": "cosineSimilarity(params.query_vector, docs[field], 1.0) ", + * "params": { + * "query_vector": [1, 2, 3.4], + * "field": "my_dense_vector" + * } + * } + * + * @param queryVector query vector + * @param docValues script doc values + * @param queryVectorMagnitude the magnitude of the query vector. + * @return cosine score + */ + public static float cosineSimilarity(List queryVector, KNNVectorScriptDocValues docValues, Number queryVectorMagnitude) { + requireNonBinaryType("cosineSimilarity", docValues.getVectorDataType()); + float[] inputVector = toFloat(queryVector, docValues.getVectorDataType()); + SpaceType.COSINESIMIL.validateVector(inputVector); + return cosinesimilOptimized(inputVector, docValues.getValue(), queryVectorMagnitude.floatValue()); + } + + /** + * Allowlisted hamming method that can be used in a script to avoid repeated + * calculation of normalization for the query vector. + * Example: + * "script": { + * "source": "hamming(params.query_vector, docs[field]) ", + * "params": { + * "query_vector": [1, 2], + * "field": "my_dense_vector" + * } + * } + * + * @param queryVector query vector + * @param docValues script doc values + * @return hamming score + */ + public static float hamming(List queryVector, KNNVectorScriptDocValues docValues) { + requireBinaryType("hamming", docValues.getVectorDataType()); + byte[] queryVectorInByte = toByte(queryVector, docValues.getVectorDataType()); + + // TODO Optimization need be done for doc value to return byte[] instead of float[] + float[] docVectorInFloat = docValues.getValue(); + byte[] docVectorInByte = new byte[docVectorInFloat.length]; + for (int i = 0; i < docVectorInByte.length; i++) { + docVectorInByte[i] = (byte) docVectorInFloat[i]; + } + + return calculateHammingBit(queryVectorInByte, docVectorInByte); + } } diff --git a/src/main/resources/org/opensearch/knn/plugin/script/knn_allowlist.txt b/src/main/resources/org/opensearch/knn/plugin/script/knn_allowlist.txt index 6b6e6434e..388cdda8a 100644 --- a/src/main/resources/org/opensearch/knn/plugin/script/knn_allowlist.txt +++ b/src/main/resources/org/opensearch/knn/plugin/script/knn_allowlist.txt @@ -13,4 +13,5 @@ static_import { float innerProduct(List, org.opensearch.knn.index.KNNVectorScriptDocValues) from_class org.opensearch.knn.plugin.script.KNNScoringUtil float cosineSimilarity(List, org.opensearch.knn.index.KNNVectorScriptDocValues) from_class org.opensearch.knn.plugin.script.KNNScoringUtil float cosineSimilarity(List, org.opensearch.knn.index.KNNVectorScriptDocValues, Number) from_class org.opensearch.knn.plugin.script.KNNScoringUtil + float hamming(List, org.opensearch.knn.index.KNNVectorScriptDocValues) from_class org.opensearch.knn.plugin.script.KNNScoringUtil } diff --git a/src/test/java/org/opensearch/knn/common/KNNValidationUtilTests.java b/src/test/java/org/opensearch/knn/common/KNNValidationUtilTests.java index 56e462fc1..4b4337880 100644 --- a/src/test/java/org/opensearch/knn/common/KNNValidationUtilTests.java +++ b/src/test/java/org/opensearch/knn/common/KNNValidationUtilTests.java @@ -12,7 +12,7 @@ public class KNNValidationUtilTests extends KNNTestCase { public void testValidateVectorDimension_whenBinary_thenVectorSizeShouldBeEightTimesLarger() { - int vectorLength = randomInt(100); + int vectorLength = randomInt(100) + 1; Exception ex = expectThrows( IllegalArgumentException.class, () -> KNNValidationUtil.validateVectorDimension(vectorLength, vectorLength, VectorDataType.BINARY) diff --git a/src/test/java/org/opensearch/knn/plugin/script/KNNScoringUtilTests.java b/src/test/java/org/opensearch/knn/plugin/script/KNNScoringUtilTests.java index a8d37b6c5..2cc20c8f9 100644 --- a/src/test/java/org/opensearch/knn/plugin/script/KNNScoringUtilTests.java +++ b/src/test/java/org/opensearch/knn/plugin/script/KNNScoringUtilTests.java @@ -5,6 +5,7 @@ package org.opensearch.knn.plugin.script; +import java.util.Arrays; import java.util.Locale; import org.opensearch.knn.KNNTestCase; import org.opensearch.knn.index.KNNVectorScriptDocValues; @@ -25,6 +26,10 @@ import java.io.IOException; import java.math.BigInteger; import java.util.List; +import java.util.function.BiFunction; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; public class KNNScoringUtilTests extends KNNTestCase { @@ -273,6 +278,50 @@ public void testCalculateHammingBit_whenByte_thenSuccess() { assertEquals(10, KNNScoringUtil.calculateHammingBit(v1, v2), 0.001f); } + private void validateThrowExceptionOnGivenDataType( + final BiFunction, KNNVectorScriptDocValues, Float> func, + final VectorDataType dataType, + final String errorMsg + ) { + List queryVector = Arrays.asList(1, 2); + KNNVectorScriptDocValues docValues = mock(KNNVectorScriptDocValues.class); + when(docValues.getVectorDataType()).thenReturn(dataType); + Exception e = expectThrows(IllegalArgumentException.class, () -> func.apply(queryVector, docValues)); + assertTrue(e.getMessage().contains(errorMsg)); + } + + public void testLInfNorm_whenKNNVectorScriptDocValuesOfBinary_thenThrowException() { + validateThrowExceptionOnGivenDataType(KNNScoringUtil::lInfNorm, VectorDataType.BINARY, "should be either float or byte"); + } + + public void testL1Norm_whenKNNVectorScriptDocValuesOfBinary_thenThrowException() { + validateThrowExceptionOnGivenDataType(KNNScoringUtil::l1Norm, VectorDataType.BINARY, "should be either float or byte"); + } + + public void testInnerProduct_whenKNNVectorScriptDocValuesOfBinary_thenThrowException() { + validateThrowExceptionOnGivenDataType(KNNScoringUtil::innerProduct, VectorDataType.BINARY, "should be either float or byte"); + } + + public void testCosineSimilarity_whenKNNVectorScriptDocValuesOfBinary_thenThrowException() { + validateThrowExceptionOnGivenDataType(KNNScoringUtil::cosineSimilarity, VectorDataType.BINARY, "should be either float or byte"); + } + + public void testHamming_whenKNNVectorScriptDocValuesOfNonBinary_thenThrowException() { + validateThrowExceptionOnGivenDataType(KNNScoringUtil::hamming, VectorDataType.FLOAT, "should be binary"); + } + + public void testHamming_whenKNNVectorScriptDocValuesOfBinary_thenSuccess() { + byte[] b1 = { 1, 16, -128 }; // 0000 0001, 0001 0000, 1000 0000 + byte[] b2 = { 2, 17, -1 }; // 0000 0010, 0001 0001, 1111 1111 + float[] f1 = { 1, 16, -128 }; // 0000 0001, 0001 0000, 1000 0000 + float[] f2 = { 2, 17, -1 }; // 0000 0010, 0001 0001, 1111 1111 + List queryVector = Arrays.asList(f1[0], f1[1], f1[2]); + KNNVectorScriptDocValues docValues = mock(KNNVectorScriptDocValues.class); + when(docValues.getVectorDataType()).thenReturn(VectorDataType.BINARY); + when(docValues.getValue()).thenReturn(f2); + assertEquals(KNNScoringUtil.calculateHammingBit(b1, b2), KNNScoringUtil.hamming(queryVector, docValues), 0.01f); + } + class TestKNNScriptDocValues { private KNNVectorScriptDocValues scriptDocValues; private Directory directory; From 4cc44fdf6cb8f4f21bdc01aaf0d1825847b06373 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Thu, 18 Jul 2024 11:07:09 -0700 Subject: [PATCH 296/416] Fix build failure due to patch conflict (#1851) (#1852) Signed-off-by: Heemin Kim (cherry picked from commit 881364f73b5f06761a1c4beffea6fb182e9512d5) Co-authored-by: Heemin Kim --- .gitignore | 1 + build.gradle | 27 ++++++++++---- jni/cmake/init-faiss.cmake | 71 +++++++++++++++++++------------------ jni/cmake/init-nmslib.cmake | 56 +++++++++++++++-------------- scripts/build.sh | 4 ++- 5 files changed, 92 insertions(+), 67 deletions(-) diff --git a/.gitignore b/.gitignore index 4cb4ee61c..7ff764056 100644 --- a/.gitignore +++ b/.gitignore @@ -28,6 +28,7 @@ jni/bin/ jni/lib/ jni/jni_test* jni/googletest* +jni/cmake/*.cmake-e benchmarks/perf-tool/okpt/output benchmarks/perf-tool/okpt/dev diff --git a/build.gradle b/build.gradle index 09851f65f..73f11933b 100644 --- a/build.gradle +++ b/build.gradle @@ -18,8 +18,15 @@ buildscript { opensearch_group = "org.opensearch" isSnapshot = "true" == System.getProperty("build.snapshot", "true") simd_enabled = System.getProperty("simd.enabled", "true") - // Flag to determine whether cmake build system should apply patches and commit. In automated build environments - // set this to false. In dev environments, set to true + // This flag determines whether the CMake build system should apply a custom patch. It prevents build failures + // when the cmakeJniLib task is run multiple times. If the build.lib.commit_patches is true, the CMake build + // system skips applying the patch if the patches have been applied already. If build.lib.commit_patches is + // false, the patches are always applied. To avoid patch conflicts, disable this flag manually after the first + // run of buildJniLib + apply_lib_patches = System.getProperty("build.lib.apply_patches", "true") + // Flag to determine whether cmake build system should commit the patch or not. In automated build environments + // set this to false. In dev environments, set to true. If false, repetitive execution of cmakeJniLib may fail. + // To prevent this, set build.lib.apply_patches to false after the first cmakeJniLib run. commit_lib_patches = System.getProperty("build.lib.commit_patches", "true") version_tokens = opensearch_version.tokenize('-') @@ -304,13 +311,21 @@ task windowsPatches(type:Exec) { task cmakeJniLib(type:Exec) { workingDir 'jni' + def args = [] + args.add("cmake") + args.add(".") + args.add("-DKNN_PLUGIN_VERSION=${opensearch_version}") + args.add("-DSIMD_ENABLED=${simd_enabled}") + args.add("-DCOMMIT_LIB_PATCHES=${commit_lib_patches}") + args.add("-DAPPLY_LIB_PATCHES=${apply_lib_patches}") if (Os.isFamily(Os.FAMILY_WINDOWS)) { dependsOn windowsPatches - commandLine 'cmake', '.', "-G", "Unix Makefiles", "-DKNN_PLUGIN_VERSION=${opensearch_version}", "-DBLAS_LIBRARIES=$rootDir\\src\\main\\resources\\windowsDependencies\\libopenblas.dll", "-DLAPACK_LIBRARIES=$rootDir\\src\\main\\resources\\windowsDependencies\\libopenblas.dll", "-DSIMD_ENABLED=${simd_enabled}", "-DCOMMIT_LIB_PATCHES=${commit_lib_patches}" - } - else { - commandLine 'cmake', '.', "-DKNN_PLUGIN_VERSION=${opensearch_version}", "-DSIMD_ENABLED=${simd_enabled}", "-DCOMMIT_LIB_PATCHES=${commit_lib_patches}" + args.add("-G") + args.add("Unix Makefiles") + args.add("-DBLAS_LIBRARIES=$rootDir\\src\\main\\resources\\windowsDependencies\\libopenblas.dll") + args.add("-DLAPACK_LIBRARIES=$rootDir\\src\\main\\resources\\windowsDependencies\\libopenblas.dll") } + commandLine args } task buildJniLib(type:Exec) { diff --git a/jni/cmake/init-faiss.cmake b/jni/cmake/init-faiss.cmake index c2ec24a3b..08f3ccc40 100644 --- a/jni/cmake/init-faiss.cmake +++ b/jni/cmake/init-faiss.cmake @@ -12,44 +12,47 @@ if (NOT EXISTS ${FAISS_REPO_DIR}) execute_process(COMMAND git submodule update --init -- external/faiss WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}) endif () -# Define list of patch files -set(PATCH_FILE_LIST) -list(APPEND PATCH_FILE_LIST "${CMAKE_CURRENT_SOURCE_DIR}/patches/faiss/0001-Custom-patch-to-support-multi-vector.patch") -list(APPEND PATCH_FILE_LIST "${CMAKE_CURRENT_SOURCE_DIR}/patches/faiss/0002-Enable-precomp-table-to-be-shared-ivfpq.patch") -list(APPEND PATCH_FILE_LIST "${CMAKE_CURRENT_SOURCE_DIR}/patches/faiss/0003-Custom-patch-to-support-range-search-params.patch") -list(APPEND PATCH_FILE_LIST "${CMAKE_CURRENT_SOURCE_DIR}/patches/faiss/0004-Custom-patch-to-support-binary-vector.patch") +# Apply patches +if(NOT DEFINED APPLY_LIB_PATCHES OR "${APPLY_LIB_PATCHES}" STREQUAL true) + # Define list of patch files + set(PATCH_FILE_LIST) + list(APPEND PATCH_FILE_LIST "${CMAKE_CURRENT_SOURCE_DIR}/patches/faiss/0001-Custom-patch-to-support-multi-vector.patch") + list(APPEND PATCH_FILE_LIST "${CMAKE_CURRENT_SOURCE_DIR}/patches/faiss/0002-Enable-precomp-table-to-be-shared-ivfpq.patch") + list(APPEND PATCH_FILE_LIST "${CMAKE_CURRENT_SOURCE_DIR}/patches/faiss/0003-Custom-patch-to-support-range-search-params.patch") + list(APPEND PATCH_FILE_LIST "${CMAKE_CURRENT_SOURCE_DIR}/patches/faiss/0004-Custom-patch-to-support-binary-vector.patch") -# Get patch id of the last commit -execute_process(COMMAND sh -c "git --no-pager show HEAD | git patch-id --stable" OUTPUT_VARIABLE PATCH_ID_OUTPUT_FROM_COMMIT WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/external/faiss) -string(REPLACE " " ";" PATCH_ID_LIST_FROM_COMMIT ${PATCH_ID_OUTPUT_FROM_COMMIT}) -list(GET PATCH_ID_LIST_FROM_COMMIT 0 PATCH_ID_FROM_COMMIT) + # Get patch id of the last commit + execute_process(COMMAND sh -c "git --no-pager show HEAD | git patch-id --stable" OUTPUT_VARIABLE PATCH_ID_OUTPUT_FROM_COMMIT WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/external/faiss) + string(REPLACE " " ";" PATCH_ID_LIST_FROM_COMMIT ${PATCH_ID_OUTPUT_FROM_COMMIT}) + list(GET PATCH_ID_LIST_FROM_COMMIT 0 PATCH_ID_FROM_COMMIT) -# Find all patch files need to apply -list(SORT PATCH_FILE_LIST ORDER DESCENDING) -set(PATCH_FILES_TO_APPLY) -foreach(PATCH_FILE IN LISTS PATCH_FILE_LIST) - # Get patch id of a patch file - execute_process(COMMAND sh -c "cat ${PATCH_FILE} | git patch-id --stable" OUTPUT_VARIABLE PATCH_ID_OUTPUT) - string(REPLACE " " ";" PATCH_ID_LIST ${PATCH_ID_OUTPUT}) - list(GET PATCH_ID_LIST 0 PATCH_ID) + # Find all patch files need to apply + list(SORT PATCH_FILE_LIST ORDER DESCENDING) + set(PATCH_FILES_TO_APPLY) + foreach(PATCH_FILE IN LISTS PATCH_FILE_LIST) + # Get patch id of a patch file + execute_process(COMMAND sh -c "cat ${PATCH_FILE} | git patch-id --stable" OUTPUT_VARIABLE PATCH_ID_OUTPUT) + string(REPLACE " " ";" PATCH_ID_LIST ${PATCH_ID_OUTPUT}) + list(GET PATCH_ID_LIST 0 PATCH_ID) - # Add the file to patch list if patch id does not match - if (${PATCH_ID} STREQUAL ${PATCH_ID_FROM_COMMIT}) - break() - else() - list(APPEND PATCH_FILES_TO_APPLY ${PATCH_FILE}) - endif() -endforeach() + # Add the file to patch list if patch id does not match + if (${PATCH_ID} STREQUAL ${PATCH_ID_FROM_COMMIT}) + break() + else() + list(APPEND PATCH_FILES_TO_APPLY ${PATCH_FILE}) + endif() + endforeach() -# Apply patch files -list(SORT PATCH_FILES_TO_APPLY) -foreach(PATCH_FILE IN LISTS PATCH_FILES_TO_APPLY) - message(STATUS "Applying patch of ${PATCH_FILE}") - execute_process(COMMAND git ${GIT_PATCH_COMMAND} --3way --ignore-space-change --ignore-whitespace ${PATCH_FILE} WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/external/faiss ERROR_VARIABLE ERROR_MSG RESULT_VARIABLE RESULT_CODE) - if(RESULT_CODE) - message(FATAL_ERROR "Failed to apply patch:\n${ERROR_MSG}") - endif() -endforeach() + # Apply patch files + list(SORT PATCH_FILES_TO_APPLY) + foreach(PATCH_FILE IN LISTS PATCH_FILES_TO_APPLY) + message(STATUS "Applying patch of ${PATCH_FILE}") + execute_process(COMMAND git ${GIT_PATCH_COMMAND} --3way --ignore-space-change --ignore-whitespace ${PATCH_FILE} WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/external/faiss ERROR_VARIABLE ERROR_MSG RESULT_VARIABLE RESULT_CODE) + if(RESULT_CODE) + message(FATAL_ERROR "Failed to apply patch:\n${ERROR_MSG}") + endif() + endforeach() +endif() if (${CMAKE_SYSTEM_NAME} STREQUAL Darwin) if(CMAKE_C_COMPILER_ID MATCHES "Clang\$") diff --git a/jni/cmake/init-nmslib.cmake b/jni/cmake/init-nmslib.cmake index 56c52bb69..b2c16f1fe 100644 --- a/jni/cmake/init-nmslib.cmake +++ b/jni/cmake/init-nmslib.cmake @@ -12,32 +12,36 @@ if (NOT EXISTS ${NMS_REPO_DIR}) execute_process(COMMAND git submodule update --init -- external/nmslib WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}) endif () -# Define list of patch files -set(PATCH_FILE_LIST) -list(APPEND PATCH_FILE_LIST "${CMAKE_CURRENT_SOURCE_DIR}/patches/nmslib/0001-Initialize-maxlevel-during-add-from-enterpoint-level.patch") -list(APPEND PATCH_FILE_LIST "${CMAKE_CURRENT_SOURCE_DIR}/patches/nmslib/0002-Adds-ability-to-pass-ef-parameter-in-the-query-for-h.patch") - -# Get patch id of the last commit -execute_process(COMMAND sh -c "git --no-pager show HEAD | git patch-id --stable" OUTPUT_VARIABLE PATCH_ID_OUTPUT_FROM_COMMIT WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/external/nmslib) -string(REPLACE " " ";" PATCH_ID_LIST_FROM_COMMIT ${PATCH_ID_OUTPUT_FROM_COMMIT}) -list(GET PATCH_ID_LIST_FROM_COMMIT 0 PATCH_ID_FROM_COMMIT) - -# Find all patch files need to apply -list(SORT PATCH_FILE_LIST ORDER DESCENDING) -set(PATCH_FILES_TO_APPLY) -foreach(PATCH_FILE IN LISTS PATCH_FILE_LIST) - # Get patch id of a patch file - execute_process(COMMAND sh -c "cat ${PATCH_FILE} | git patch-id --stable" OUTPUT_VARIABLE PATCH_ID_OUTPUT) - string(REPLACE " " ";" PATCH_ID_LIST ${PATCH_ID_OUTPUT}) - list(GET PATCH_ID_LIST 0 PATCH_ID) - - # Add the file to patch list if patch id does not match - if (${PATCH_ID} STREQUAL ${PATCH_ID_FROM_COMMIT}) - break() - else() - list(APPEND PATCH_FILES_TO_APPLY ${PATCH_FILE}) - endif() -endforeach() + +# Apply patches +if(NOT DEFINED APPLY_LIB_PATCHES OR "${APPLY_LIB_PATCHES}" STREQUAL true) + # Define list of patch files + set(PATCH_FILE_LIST) + list(APPEND PATCH_FILE_LIST "${CMAKE_CURRENT_SOURCE_DIR}/patches/nmslib/0001-Initialize-maxlevel-during-add-from-enterpoint-level.patch") + list(APPEND PATCH_FILE_LIST "${CMAKE_CURRENT_SOURCE_DIR}/patches/nmslib/0002-Adds-ability-to-pass-ef-parameter-in-the-query-for-h.patch") + + # Get patch id of the last commit + execute_process(COMMAND sh -c "git --no-pager show HEAD | git patch-id --stable" OUTPUT_VARIABLE PATCH_ID_OUTPUT_FROM_COMMIT WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/external/nmslib) + string(REPLACE " " ";" PATCH_ID_LIST_FROM_COMMIT ${PATCH_ID_OUTPUT_FROM_COMMIT}) + list(GET PATCH_ID_LIST_FROM_COMMIT 0 PATCH_ID_FROM_COMMIT) + + # Find all patch files need to apply + list(SORT PATCH_FILE_LIST ORDER DESCENDING) + set(PATCH_FILES_TO_APPLY) + foreach(PATCH_FILE IN LISTS PATCH_FILE_LIST) + # Get patch id of a patch file + execute_process(COMMAND sh -c "cat ${PATCH_FILE} | git patch-id --stable" OUTPUT_VARIABLE PATCH_ID_OUTPUT) + string(REPLACE " " ";" PATCH_ID_LIST ${PATCH_ID_OUTPUT}) + list(GET PATCH_ID_LIST 0 PATCH_ID) + + # Add the file to patch list if patch id does not match + if (${PATCH_ID} STREQUAL ${PATCH_ID_FROM_COMMIT}) + break() + else() + list(APPEND PATCH_FILES_TO_APPLY ${PATCH_FILE}) + endif() + endforeach() +endif() # Apply patch files list(SORT PATCH_FILES_TO_APPLY) diff --git a/scripts/build.sh b/scripts/build.sh index 38998d66a..12798633b 100644 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -122,7 +122,9 @@ cd $work_dir if [ "$PLATFORM" != "windows" ] && [ "$ARCHITECTURE" = "x64" ]; then echo "Building k-NN library after enabling AVX2" - ./gradlew :buildJniLib -Dsimd.enabled=true -Dbuild.lib.commit_patches=false + # Skip applying patches as patches were applied already from previous :buildJniLib task + # If we apply patches again, it fails with conflict + ./gradlew :buildJniLib -Dsimd.enabled=true -Dbuild.lib.commit_patches=false -Dbuild.lib.apply_patches=false fi ./gradlew publishPluginZipPublicationToZipStagingRepository -Dopensearch.version=$VERSION -Dbuild.snapshot=$SNAPSHOT -Dbuild.version_qualifier=$QUALIFIER From 2c6e0bf180d8bf88c6a55cf60778794120a71d41 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Thu, 18 Jul 2024 18:14:39 -0700 Subject: [PATCH 297/416] Fix few bugs on binary index with Faiss HNSW (#1850) (#1856) Signed-off-by: Heemin Kim (cherry picked from commit 3a5fe64a14b51ceea15027962cb963ceedbc5ad1) Co-authored-by: Heemin Kim --- .../org/opensearch/knn/index/IndexUtil.java | 13 ++++ .../org/opensearch/knn/index/KNNMethod.java | 9 ++- .../org/opensearch/knn/index/KNNSettings.java | 1 + .../index/mapper/KNNVectorFieldMapper.java | 2 +- .../knn/index/mapper/LegacyFieldMapper.java | 9 ++- .../knn/index/query/KNNQueryBuilder.java | 3 + .../opensearch/knn/index/query/KNNWeight.java | 9 ++- .../opensearch/knn/index/IndexUtilTests.java | 21 ++++++ .../mapper/KNNVectorFieldMapperTests.java | 20 +++++- .../knn/index/query/KNNQueryBuilderTests.java | 24 +++++++ .../knn/index/query/KNNWeightTests.java | 68 +++++++++++++++++++ 11 files changed, 169 insertions(+), 10 deletions(-) diff --git a/src/main/java/org/opensearch/knn/index/IndexUtil.java b/src/main/java/org/opensearch/knn/index/IndexUtil.java index 7a445eda1..c88da4d3c 100644 --- a/src/main/java/org/opensearch/knn/index/IndexUtil.java +++ b/src/main/java/org/opensearch/knn/index/IndexUtil.java @@ -192,6 +192,19 @@ public static ValidationException validateKnnField( return exception; } + String vectorDataType = (String) fieldMap.get(VECTOR_DATA_TYPE_FIELD); + if (VectorDataType.BINARY.toString().equalsIgnoreCase(vectorDataType)) { + exception.addValidationError( + String.format( + Locale.ROOT, + "Field \"%s\" is of data type %s. Only FLOAT or BYTE is supported.", + field, + VectorDataType.BINARY + ) + ); + return exception; + } + // Return if dimension does not need to be checked if (expectedDimension < 0) { return null; diff --git a/src/main/java/org/opensearch/knn/index/KNNMethod.java b/src/main/java/org/opensearch/knn/index/KNNMethod.java index 7abd2ce39..d456256f5 100644 --- a/src/main/java/org/opensearch/knn/index/KNNMethod.java +++ b/src/main/java/org/opensearch/knn/index/KNNMethod.java @@ -22,6 +22,7 @@ import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Set; @@ -57,8 +58,10 @@ public ValidationException validate(KNNMethodContext knnMethodContext) { if (!isSpaceTypeSupported(knnMethodContext.getSpaceType())) { errorMessages.add( String.format( - "\"%s\" configuration does not support space type: " + "\"%s\".", + Locale.ROOT, + "\"%s\" with \"%s\" configuration does not support space type: " + "\"%s\".", this.methodComponent.getName(), + knnMethodContext.getKnnEngine().getName().toLowerCase(Locale.ROOT), knnMethodContext.getSpaceType().getValue() ) ); @@ -90,8 +93,10 @@ public ValidationException validateWithData(KNNMethodContext knnMethodContext, V if (!isSpaceTypeSupported(knnMethodContext.getSpaceType())) { errorMessages.add( String.format( - "\"%s\" configuration does not support space type: " + "\"%s\".", + Locale.ROOT, + "\"%s\" with \"%s\" configuration does not support space type: " + "\"%s\".", this.methodComponent.getName(), + knnMethodContext.getKnnEngine().getName().toLowerCase(Locale.ROOT), knnMethodContext.getSpaceType().getValue() ) ); diff --git a/src/main/java/org/opensearch/knn/index/KNNSettings.java b/src/main/java/org/opensearch/knn/index/KNNSettings.java index 8d0ad91e0..18213f060 100644 --- a/src/main/java/org/opensearch/knn/index/KNNSettings.java +++ b/src/main/java/org/opensearch/knn/index/KNNSettings.java @@ -86,6 +86,7 @@ public class KNNSettings { */ public static final boolean KNN_DEFAULT_FAISS_AVX2_DISABLED_VALUE = false; public static final String INDEX_KNN_DEFAULT_SPACE_TYPE = "l2"; + public static final String INDEX_KNN_DEFAULT_SPACE_TYPE_FOR_BINARY = "hammingbit"; public static final Integer INDEX_KNN_DEFAULT_ALGO_PARAM_M = 16; public static final Integer INDEX_KNN_DEFAULT_ALGO_PARAM_EF_SEARCH = 100; public static final Integer INDEX_KNN_DEFAULT_ALGO_PARAM_EF_CONSTRUCTION = 100; diff --git a/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapper.java b/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapper.java index 7f7c83f3e..dc9827189 100644 --- a/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapper.java +++ b/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapper.java @@ -307,7 +307,7 @@ public KNNVectorFieldMapper build(BuilderContext context) { // Build legacy if (this.spaceType == null) { - this.spaceType = LegacyFieldMapper.getSpaceType(context.indexSettings()); + this.spaceType = LegacyFieldMapper.getSpaceType(context.indexSettings(), vectorDataType.getValue()); } if (this.m == null) { diff --git a/src/main/java/org/opensearch/knn/index/mapper/LegacyFieldMapper.java b/src/main/java/org/opensearch/knn/index/mapper/LegacyFieldMapper.java index d67ffc73c..4959e7bc0 100644 --- a/src/main/java/org/opensearch/knn/index/mapper/LegacyFieldMapper.java +++ b/src/main/java/org/opensearch/knn/index/mapper/LegacyFieldMapper.java @@ -12,6 +12,7 @@ import org.opensearch.common.settings.Settings; import org.opensearch.index.mapper.ParametrizedFieldMapper; import org.opensearch.knn.index.KNNSettings; +import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.util.IndexHyperParametersUtil; import org.opensearch.knn.index.util.KNNEngine; @@ -78,17 +79,19 @@ public ParametrizedFieldMapper.Builder getMergeBuilder() { ); } - static String getSpaceType(Settings indexSettings) { + static String getSpaceType(final Settings indexSettings, final VectorDataType vectorDataType) { String spaceType = indexSettings.get(KNNSettings.INDEX_KNN_SPACE_TYPE.getKey()); if (spaceType == null) { + spaceType = VectorDataType.BINARY == vectorDataType + ? KNNSettings.INDEX_KNN_DEFAULT_SPACE_TYPE_FOR_BINARY + : KNNSettings.INDEX_KNN_DEFAULT_SPACE_TYPE; log.info( String.format( "[KNN] The setting \"%s\" was not set for the index. Likely caused by recent version upgrade. Setting the setting to the default value=%s", METHOD_PARAMETER_SPACE_TYPE, - KNNSettings.INDEX_KNN_DEFAULT_SPACE_TYPE + spaceType ) ); - return KNNSettings.INDEX_KNN_DEFAULT_SPACE_TYPE; } return spaceType; } diff --git a/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java b/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java index 013a4d9f7..6f1617193 100644 --- a/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java +++ b/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java @@ -651,6 +651,9 @@ protected Query doToQuery(QueryShardContext context) { String.format(Locale.ROOT, "Engine [%s] does not support radial search", knnEngine) ); } + if (vectorDataType == VectorDataType.BINARY) { + throw new UnsupportedOperationException(String.format(Locale.ROOT, "Binary data type does not support radial search")); + } RNNQueryFactory.CreateQueryRequest createQueryRequest = RNNQueryFactory.CreateQueryRequest.builder() .knnEngine(knnEngine) .indexName(indexName) diff --git a/src/main/java/org/opensearch/knn/index/query/KNNWeight.java b/src/main/java/org/opensearch/knn/index/query/KNNWeight.java index a061e740e..5301f8fa6 100644 --- a/src/main/java/org/opensearch/knn/index/query/KNNWeight.java +++ b/src/main/java/org/opensearch/knn/index/query/KNNWeight.java @@ -476,8 +476,15 @@ private boolean canDoExactSearch(final int filterIdsCount) { if (isExactSearchThresholdSettingSet(filterThresholdValue)) { return filterThresholdValue >= filterIdsCount; } + // if no setting is set, then use the default max distance computation value to see if we can do exact search. - return KNNConstants.MAX_DISTANCE_COMPUTATIONS >= filterIdsCount * knnQuery.getQueryVector().length; + /** + * TODO we can have a different MAX_DISTANCE_COMPUTATIONS for binary index as computation cost for binary index + * is cheaper than computation cost for non binary vector + */ + return KNNConstants.MAX_DISTANCE_COMPUTATIONS >= filterIdsCount * (knnQuery.getVectorDataType() == VectorDataType.FLOAT + ? knnQuery.getQueryVector().length + : knnQuery.getByteQueryVector().length); } /** diff --git a/src/test/java/org/opensearch/knn/index/IndexUtilTests.java b/src/test/java/org/opensearch/knn/index/IndexUtilTests.java index d500fc342..6c1384e9d 100644 --- a/src/test/java/org/opensearch/knn/index/IndexUtilTests.java +++ b/src/test/java/org/opensearch/knn/index/IndexUtilTests.java @@ -228,6 +228,27 @@ public void testValidateKnnField_EmptyIndexMetadata() { assert (Objects.requireNonNull(e).getMessage().matches("Validation Failed: 1: Invalid index. Index does not contain a mapping;")); } + public void testValidateKnnField_whenBinaryDataType_thenThrowException() { + Map fieldValues = Map.of("type", "knn_vector", "dimension", 8, "data_type", "BINARY"); + Map top_level_field = Map.of("top_level_field", fieldValues); + Map properties = Map.of("properties", top_level_field); + String field = "top_level_field"; + int dimension = 8; + + MappingMetadata mappingMetadata = mock(MappingMetadata.class); + when(mappingMetadata.getSourceAsMap()).thenReturn(properties); + IndexMetadata indexMetadata = mock(IndexMetadata.class); + when(indexMetadata.mapping()).thenReturn(mappingMetadata); + ModelDao modelDao = mock(ModelDao.class); + ModelMetadata trainingFieldModelMetadata = mock(ModelMetadata.class); + when(trainingFieldModelMetadata.getDimension()).thenReturn(dimension); + when(modelDao.getMetadata(anyString())).thenReturn(trainingFieldModelMetadata); + + ValidationException e = IndexUtil.validateKnnField(indexMetadata, field, dimension, modelDao); + + assert (Objects.requireNonNull(e).getMessage().contains("is of data type BINARY. Only FLOAT or BYTE is supported")); + } + public void testIsShareableStateContainedInIndex_whenIndexNotModelBased_thenReturnFalse() { String modelId = null; KNNEngine knnEngine = KNNEngine.FAISS; diff --git a/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java b/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java index c3ddcf185..ad3a3d6c5 100644 --- a/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java +++ b/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java @@ -192,14 +192,12 @@ public void testBuilder_build_fromLegacy() { ModelDao modelDao = mock(ModelDao.class); KNNVectorFieldMapper.Builder builder = new KNNVectorFieldMapper.Builder("test-field-name-1", modelDao, CURRENT); - SpaceType spaceType = SpaceType.COSINESIMIL; int m = 17; int efConstruction = 17; // Setup settings Settings settings = Settings.builder() .put(settings(CURRENT).build()) - .put(KNNSettings.KNN_SPACE_TYPE, spaceType.getValue()) .put(KNNSettings.KNN_ALGO_PARAM_M, m) .put(KNNSettings.KNN_ALGO_PARAM_EF_CONSTRUCTION, efConstruction) .build(); @@ -207,9 +205,25 @@ public void testBuilder_build_fromLegacy() { Mapper.BuilderContext builderContext = new Mapper.BuilderContext(settings, new ContentPath()); KNNVectorFieldMapper knnVectorFieldMapper = builder.build(builderContext); assertTrue(knnVectorFieldMapper instanceof LegacyFieldMapper); - assertNull(knnVectorFieldMapper.modelId); assertNull(knnVectorFieldMapper.knnMethod); + assertEquals(SpaceType.L2.getValue(), ((LegacyFieldMapper) knnVectorFieldMapper).spaceType); + } + + public void testBuilder_whenKnnFalseWithBinary_thenSetHammingAsDefault() { + // Check legacy is picked up if model context and method context are not set + ModelDao modelDao = mock(ModelDao.class); + KNNVectorFieldMapper.Builder builder = new KNNVectorFieldMapper.Builder("test-field-name-1", modelDao, CURRENT); + builder.vectorDataType.setValue(VectorDataType.BINARY); + builder.dimension.setValue(8); + + // Setup settings + Settings settings = Settings.builder().put(settings(CURRENT).build()).build(); + + Mapper.BuilderContext builderContext = new Mapper.BuilderContext(settings, new ContentPath()); + KNNVectorFieldMapper knnVectorFieldMapper = builder.build(builderContext); + assertTrue(knnVectorFieldMapper instanceof LegacyFieldMapper); + assertEquals(SpaceType.HAMMING_BIT.getValue(), ((LegacyFieldMapper) knnVectorFieldMapper).spaceType); } public void testBuilder_parse_fromKnnMethodContext_luceneEngine() throws IOException { diff --git a/src/test/java/org/opensearch/knn/index/query/KNNQueryBuilderTests.java b/src/test/java/org/opensearch/knn/index/query/KNNQueryBuilderTests.java index db37259d6..aa4cd88ab 100644 --- a/src/test/java/org/opensearch/knn/index/query/KNNQueryBuilderTests.java +++ b/src/test/java/org/opensearch/knn/index/query/KNNQueryBuilderTests.java @@ -745,6 +745,30 @@ public void testDoToQuery_whenPassNegativeDistance_whenUnSupportedSpaceType_then expectThrows(IllegalArgumentException.class, () -> knnQueryBuilder.doToQuery(mockQueryShardContext)); } + public void testDoToQuery_whenRadialSearchOnBinaryIndex_thenException() { + float[] queryVector = { 1.0f }; + KNNQueryBuilder knnQueryBuilder = KNNQueryBuilder.builder() + .fieldName(FIELD_NAME) + .vector(queryVector) + .maxDistance(MAX_DISTANCE) + .build(); + Index dummyIndex = new Index("dummy", "dummy"); + QueryShardContext mockQueryShardContext = mock(QueryShardContext.class); + KNNVectorFieldMapper.KNNVectorFieldType mockKNNVectorField = mock(KNNVectorFieldMapper.KNNVectorFieldType.class); + when(mockQueryShardContext.index()).thenReturn(dummyIndex); + when(mockKNNVectorField.getDimension()).thenReturn(8); + when(mockKNNVectorField.getVectorDataType()).thenReturn(VectorDataType.BINARY); + when(mockQueryShardContext.fieldMapper(anyString())).thenReturn(mockKNNVectorField); + MethodComponentContext methodComponentContext = new MethodComponentContext( + org.opensearch.knn.common.KNNConstants.METHOD_HNSW, + ImmutableMap.of() + ); + KNNMethodContext knnMethodContext = new KNNMethodContext(KNNEngine.FAISS, SpaceType.HAMMING_BIT, methodComponentContext); + when(mockKNNVectorField.getKnnMethodContext()).thenReturn(knnMethodContext); + Exception e = expectThrows(UnsupportedOperationException.class, () -> knnQueryBuilder.doToQuery(mockQueryShardContext)); + assertTrue(e.getMessage().contains("Binary data type does not support radial search")); + } + public void testDoToQuery_KnnQueryWithFilter_Lucene() throws Exception { // Given float[] queryVector = { 1.0f, 2.0f, 3.0f, 4.0f }; diff --git a/src/test/java/org/opensearch/knn/index/query/KNNWeightTests.java b/src/test/java/org/opensearch/knn/index/query/KNNWeightTests.java index bb5d7c4f6..3cf687bcb 100644 --- a/src/test/java/org/opensearch/knn/index/query/KNNWeightTests.java +++ b/src/test/java/org/opensearch/knn/index/query/KNNWeightTests.java @@ -863,6 +863,74 @@ public void testANNWithFilterQuery_whenExactSearchViaThresholdSetting_thenSucces assertTrue(Comparators.isInOrder(actualDocIds, Comparator.naturalOrder())); } + /** + * This test ensure that we do the exact search when threshold settings are correct and not using filteredIds<=K + * condition to do exact search on binary index + * FilteredIdThreshold: 10 + * FilteredIdThresholdPct: 10% + * FilteredIdsCount: 6 + * liveDocs : null, as there is no deleted documents + * MaxDoc: 100 + * K : 1 + */ + @SneakyThrows + public void testANNWithFilterQuery_whenExactSearchViaThresholdSettingOnBinaryIndex_thenSuccess() { + knnSettingsMockedStatic.when(() -> KNNSettings.getFilteredExactSearchThreshold(INDEX_NAME)).thenReturn(10); + byte[] vector = new byte[] { 1, 3 }; + int k = 1; + final int[] filterDocIds = new int[] { 0, 1, 2, 3, 4, 5 }; + + final LeafReaderContext leafReaderContext = mock(LeafReaderContext.class); + final SegmentReader reader = mock(SegmentReader.class); + when(leafReaderContext.reader()).thenReturn(reader); + when(reader.maxDoc()).thenReturn(100); + when(reader.getLiveDocs()).thenReturn(null); + final Weight filterQueryWeight = mock(Weight.class); + final Scorer filterScorer = mock(Scorer.class); + when(filterQueryWeight.scorer(leafReaderContext)).thenReturn(filterScorer); + + when(filterScorer.iterator()).thenReturn(DocIdSetIterator.all(filterDocIds.length)); + + final KNNQuery query = new KNNQuery(FIELD_NAME, BYTE_QUERY_VECTOR, k, INDEX_NAME, FILTER_QUERY, null, VectorDataType.BINARY); + + final float boost = (float) randomDoubleBetween(0, 10, true); + final KNNWeight knnWeight = new KNNWeight(query, boost, filterQueryWeight); + final Map attributesMap = ImmutableMap.of( + KNN_ENGINE, + KNNEngine.FAISS.getName(), + SPACE_TYPE, + SpaceType.HAMMING_BIT.name(), + PARAMETERS, + String.format(Locale.ROOT, "{\"%s\":\"%s\"}", INDEX_DESCRIPTION_PARAMETER, "BHNSW32") + ); + final FieldInfos fieldInfos = mock(FieldInfos.class); + final FieldInfo fieldInfo = mock(FieldInfo.class); + final BinaryDocValues binaryDocValues = mock(BinaryDocValues.class); + when(reader.getFieldInfos()).thenReturn(fieldInfos); + when(fieldInfos.fieldInfo(any())).thenReturn(fieldInfo); + when(fieldInfo.attributes()).thenReturn(attributesMap); + when(fieldInfo.getAttribute(SPACE_TYPE)).thenReturn(SpaceType.HAMMING_BIT.getValue()); + when(fieldInfo.getName()).thenReturn(FIELD_NAME); + when(reader.getBinaryDocValues(FIELD_NAME)).thenReturn(binaryDocValues); + when(binaryDocValues.advance(0)).thenReturn(0); + BytesRef vectorByteRef = new BytesRef(vector); + when(binaryDocValues.binaryValue()).thenReturn(vectorByteRef); + + final KNNScorer knnScorer = (KNNScorer) knnWeight.scorer(leafReaderContext); + assertNotNull(knnScorer); + final DocIdSetIterator docIdSetIterator = knnScorer.iterator(); + assertNotNull(docIdSetIterator); + assertEquals(EXACT_SEARCH_DOC_ID_TO_SCORES.size(), docIdSetIterator.cost()); + + final List actualDocIds = new ArrayList<>(); + for (int docId = docIdSetIterator.nextDoc(); docId != NO_MORE_DOCS; docId = docIdSetIterator.nextDoc()) { + actualDocIds.add(docId); + assertEquals(BINARY_EXACT_SEARCH_DOC_ID_TO_SCORES.get(docId) * boost, knnScorer.score(), 0.01f); + } + assertEquals(docIdSetIterator.cost(), actualDocIds.size()); + assertTrue(Comparators.isInOrder(actualDocIds, Comparator.naturalOrder())); + } + @SneakyThrows public void testANNWithFilterQuery_whenEmptyFilterIds_thenReturnEarly() { final LeafReaderContext leafReaderContext = mock(LeafReaderContext.class); From 39b9cd864e50d71fa78badba2e905c710f579d4b Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Fri, 19 Jul 2024 11:55:48 -0700 Subject: [PATCH 298/416] Switch space type name from hammingbit to hamming for binary index (#1854) (#1858) Signed-off-by: Heemin Kim (cherry picked from commit 76434c17431e08031467adf1e0b6869cbff54727) Co-authored-by: Heemin Kim --- jni/include/jni_util.h | 2 +- jni/src/faiss_wrapper.cpp | 2 +- jni/src/jni_util.cpp | 2 +- jni/tests/faiss_wrapper_test.cpp | 4 +- .../org/opensearch/knn/index/KNNSettings.java | 2 +- .../org/opensearch/knn/index/SpaceType.java | 4 +- .../org/opensearch/knn/index/util/Faiss.java | 2 +- .../knn/plugin/script/KNNScoringSpace.java | 142 +++++++++--------- .../plugin/script/KNNScoringSpaceFactory.java | 10 +- .../opensearch/knn/index/SpaceTypeTests.java | 2 +- .../KNN80DocValuesConsumerTests.java | 2 +- .../mapper/KNNVectorFieldMapperTests.java | 8 +- .../index/mapper/MethodFieldMapperTests.java | 2 +- .../memory/NativeMemoryAllocationTests.java | 2 +- .../memory/NativeMemoryLoadStrategyTests.java | 2 +- .../knn/index/query/KNNQueryBuilderTests.java | 6 +- .../knn/index/query/KNNWeightTests.java | 10 +- .../FilteredIdsKNNByteIteratorTests.java | 2 +- ...NestedFilteredIdsKNNByteIteratorTests.java | 2 +- .../opensearch/knn/jni/JNIServiceTests.java | 4 +- .../script/KNNScoringSpaceFactoryTests.java | 30 +++- .../plugin/script/KNNScoringSpaceTests.java | 21 +-- .../knn/plugin/script/KNNScriptScoringIT.java | 10 +- 23 files changed, 157 insertions(+), 116 deletions(-) diff --git a/jni/include/jni_util.h b/jni/include/jni_util.h index 97a8f063c..1579522d0 100644 --- a/jni/include/jni_util.h +++ b/jni/include/jni_util.h @@ -199,7 +199,7 @@ namespace knn_jni { extern const std::string COSINESIMIL; extern const std::string INNER_PRODUCT; extern const std::string NEG_DOT_PRODUCT; - extern const std::string HAMMING_BIT; + extern const std::string HAMMING; extern const std::string NPROBES; extern const std::string COARSE_QUANTIZER; diff --git a/jni/src/faiss_wrapper.cpp b/jni/src/faiss_wrapper.cpp index 45830aff6..2d5054635 100644 --- a/jni/src/faiss_wrapper.cpp +++ b/jni/src/faiss_wrapper.cpp @@ -644,7 +644,7 @@ faiss::MetricType TranslateSpaceToMetric(const std::string& spaceType) { } // Space type is not used for binary index. Use L2 just to avoid an error. - if (spaceType == knn_jni::HAMMING_BIT) { + if (spaceType == knn_jni::HAMMING) { return faiss::METRIC_L2; } diff --git a/jni/src/jni_util.cpp b/jni/src/jni_util.cpp index 919191596..ee4c382b5 100644 --- a/jni/src/jni_util.cpp +++ b/jni/src/jni_util.cpp @@ -540,7 +540,7 @@ const std::string knn_jni::LINF = "linf"; const std::string knn_jni::COSINESIMIL = "cosinesimil"; const std::string knn_jni::INNER_PRODUCT = "innerproduct"; const std::string knn_jni::NEG_DOT_PRODUCT = "negdotprod"; -const std::string knn_jni::HAMMING_BIT = "hammingbit"; +const std::string knn_jni::HAMMING = "hamming"; const std::string knn_jni::NPROBES = "nprobes"; const std::string knn_jni::COARSE_QUANTIZER = "coarse_quantizer"; diff --git a/jni/tests/faiss_wrapper_test.cpp b/jni/tests/faiss_wrapper_test.cpp index 36bcea491..5ae443837 100644 --- a/jni/tests/faiss_wrapper_test.cpp +++ b/jni/tests/faiss_wrapper_test.cpp @@ -87,7 +87,7 @@ TEST(FaissCreateBinaryIndexTest, BasicAssertions) { } std::string indexPath = test_util::RandomString(10, "tmp/", ".faiss"); - std::string spaceType = knn_jni::HAMMING_BIT; + std::string spaceType = knn_jni::HAMMING; std::string indexDescription = "BHNSW32"; std::unordered_map parametersMap; @@ -222,7 +222,7 @@ TEST(FaissLoadBinaryIndexTest, BasicAssertions) { } std::string indexPath = test_util::RandomString(10, "tmp/", ".faiss"); - std::string spaceType = knn_jni::HAMMING_BIT; + std::string spaceType = knn_jni::HAMMING; std::string method = "BHNSW32"; // Create the index diff --git a/src/main/java/org/opensearch/knn/index/KNNSettings.java b/src/main/java/org/opensearch/knn/index/KNNSettings.java index 18213f060..3279e74bc 100644 --- a/src/main/java/org/opensearch/knn/index/KNNSettings.java +++ b/src/main/java/org/opensearch/knn/index/KNNSettings.java @@ -86,7 +86,7 @@ public class KNNSettings { */ public static final boolean KNN_DEFAULT_FAISS_AVX2_DISABLED_VALUE = false; public static final String INDEX_KNN_DEFAULT_SPACE_TYPE = "l2"; - public static final String INDEX_KNN_DEFAULT_SPACE_TYPE_FOR_BINARY = "hammingbit"; + public static final String INDEX_KNN_DEFAULT_SPACE_TYPE_FOR_BINARY = "hamming"; public static final Integer INDEX_KNN_DEFAULT_ALGO_PARAM_M = 16; public static final Integer INDEX_KNN_DEFAULT_ALGO_PARAM_EF_SEARCH = 100; public static final Integer INDEX_KNN_DEFAULT_ALGO_PARAM_EF_CONSTRUCTION = 100; diff --git a/src/main/java/org/opensearch/knn/index/SpaceType.java b/src/main/java/org/opensearch/knn/index/SpaceType.java index a65c4bb4c..43ff45e1d 100644 --- a/src/main/java/org/opensearch/knn/index/SpaceType.java +++ b/src/main/java/org/opensearch/knn/index/SpaceType.java @@ -120,7 +120,7 @@ public KNNVectorSimilarityFunction getKnnVectorSimilarityFunction() { return KNNVectorSimilarityFunction.MAXIMUM_INNER_PRODUCT; } }, - HAMMING_BIT("hammingbit") { + HAMMING("hamming") { @Override public float scoreTranslation(float rawScore) { return 1 / (1 + rawScore); @@ -147,7 +147,7 @@ public KNNVectorSimilarityFunction getKnnVectorSimilarityFunction() { }; public static SpaceType DEFAULT = L2; - public static SpaceType DEFAULT_BINARY = HAMMING_BIT; + public static SpaceType DEFAULT_BINARY = HAMMING; private final String value; diff --git a/src/main/java/org/opensearch/knn/index/util/Faiss.java b/src/main/java/org/opensearch/knn/index/util/Faiss.java index 5f3add832..e4d0490bf 100644 --- a/src/main/java/org/opensearch/knn/index/util/Faiss.java +++ b/src/main/java/org/opensearch/knn/index/util/Faiss.java @@ -247,7 +247,7 @@ public class Faiss extends NativeLibrary { ).addParameter(METHOD_PARAMETER_M, "", "").addParameter(METHOD_ENCODER_PARAMETER, ",", "").build()) ) .build() - ).addSpaces(SpaceType.UNDEFINED, SpaceType.HAMMING_BIT, SpaceType.L2, SpaceType.INNER_PRODUCT).build(), + ).addSpaces(SpaceType.UNDEFINED, SpaceType.HAMMING, SpaceType.L2, SpaceType.INNER_PRODUCT).build(), METHOD_IVF, KNNMethod.Builder.builder( MethodComponent.Builder.builder(METHOD_IVF) diff --git a/src/main/java/org/opensearch/knn/plugin/script/KNNScoringSpace.java b/src/main/java/org/opensearch/knn/plugin/script/KNNScoringSpace.java index 04d08fc9c..2aa203b12 100644 --- a/src/main/java/org/opensearch/knn/plugin/script/KNNScoringSpace.java +++ b/src/main/java/org/opensearch/knn/plugin/script/KNNScoringSpace.java @@ -5,13 +5,14 @@ package org.opensearch.knn.plugin.script; +import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.search.IndexSearcher; +import org.opensearch.index.mapper.MappedFieldType; import org.opensearch.knn.index.SpaceType; +import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.mapper.KNNVectorFieldMapper; import org.opensearch.knn.index.mapper.KNNVectorFieldMapperUtil; import org.opensearch.knn.index.query.KNNWeight; -import org.apache.lucene.index.LeafReaderContext; -import org.opensearch.index.mapper.MappedFieldType; import org.opensearch.script.ScoreScript; import org.opensearch.search.lookup.SearchLookup; @@ -19,11 +20,11 @@ import java.math.BigInteger; import java.util.Locale; import java.util.Map; +import java.util.Set; import java.util.function.BiFunction; import static org.opensearch.knn.plugin.script.KNNScoringSpaceUtil.getVectorMagnitudeSquared; import static org.opensearch.knn.plugin.script.KNNScoringSpaceUtil.isBinaryFieldType; -import static org.opensearch.knn.plugin.script.KNNScoringSpaceUtil.isBinaryVectorDataType; import static org.opensearch.knn.plugin.script.KNNScoringSpaceUtil.isKNNVectorFieldType; import static org.opensearch.knn.plugin.script.KNNScoringSpaceUtil.isLongFieldType; import static org.opensearch.knn.plugin.script.KNNScoringSpaceUtil.parseToBigInteger; @@ -31,7 +32,6 @@ import static org.opensearch.knn.plugin.script.KNNScoringSpaceUtil.parseToLong; public interface KNNScoringSpace { - /** * Return the correct scoring script for a given query. The scoring script * @@ -47,17 +47,29 @@ ScoreScript getScoreScript(Map params, String field, SearchLooku throws IOException; /** - * Base class to represent linear space - * - * As of now, all supporting spaces are linear space except hamming. - * LinearSpace does not support binary vector. + * Base class to represent vector space for knn field */ - abstract class LinearSpace implements KNNScoringSpace { + abstract class KNNFieldSpace implements KNNScoringSpace { + public static final Set DATA_TYPES_DEFAULT = Set.of(VectorDataType.FLOAT, VectorDataType.BYTE); + float[] processedQuery; BiFunction scoringMethod; - public LinearSpace(final Object query, final MappedFieldType fieldType, final String spaceName) { - KNNVectorFieldMapper.KNNVectorFieldType knnVectorFieldType = toKNNVectorFieldType(fieldType, spaceName); + public KNNFieldSpace(final Object query, final MappedFieldType fieldType, final String spaceName) { + this(query, fieldType, spaceName, DATA_TYPES_DEFAULT); + } + + public KNNFieldSpace( + final Object query, + final MappedFieldType fieldType, + final String spaceName, + final Set supportingVectorDataTypes + ) { + KNNVectorFieldMapper.KNNVectorFieldType knnVectorFieldType = toKNNVectorFieldType( + fieldType, + spaceName, + supportingVectorDataTypes + ); this.processedQuery = getProcessedQuery(query, knnVectorFieldType); this.scoringMethod = getScoringMethod(this.processedQuery); } @@ -72,7 +84,11 @@ public ScoreScript getScoreScript( return new KNNScoreScript.KNNVectorType(params, this.processedQuery, field, this.scoringMethod, lookup, ctx, searcher); } - private KNNVectorFieldMapper.KNNVectorFieldType toKNNVectorFieldType(final MappedFieldType fieldType, final String spaceName) { + private KNNVectorFieldMapper.KNNVectorFieldType toKNNVectorFieldType( + final MappedFieldType fieldType, + final String spaceName, + final Set supportingVectorDataTypes + ) { if (isKNNVectorFieldType(fieldType) == false) { throw new IllegalArgumentException( String.format(Locale.ROOT, "Incompatible field_type for %s space. The field type must be knn_vector.", spaceName) @@ -80,13 +96,17 @@ private KNNVectorFieldMapper.KNNVectorFieldType toKNNVectorFieldType(final Mappe } KNNVectorFieldMapper.KNNVectorFieldType knnVectorFieldType = (KNNVectorFieldMapper.KNNVectorFieldType) fieldType; - if (isBinaryVectorDataType(knnVectorFieldType)) { + VectorDataType vectorDataType = knnVectorFieldType.getVectorDataType() == null + ? VectorDataType.FLOAT + : knnVectorFieldType.getVectorDataType(); + if (supportingVectorDataTypes.contains(vectorDataType) == false) { throw new IllegalArgumentException( String.format( Locale.ROOT, - "Incompatible field_type for %s space. The data type should be either float or byte but got %s", + "Incompatible field_type for %s space. The data type should be %s but got %s", spaceName, - knnVectorFieldType.getVectorDataType().getValue() + supportingVectorDataTypes, + vectorDataType ) ); } @@ -106,7 +126,7 @@ protected float[] getProcessedQuery(final Object query, final KNNVectorFieldMapp } - class L2 extends LinearSpace { + class L2 extends KNNFieldSpace { public L2(final Object query, final MappedFieldType fieldType) { super(query, fieldType, "l2"); } @@ -117,7 +137,7 @@ public BiFunction getScoringMethod(final float[] proces } } - class CosineSimilarity extends LinearSpace { + class CosineSimilarity extends KNNFieldSpace { public CosineSimilarity(Object query, MappedFieldType fieldType) { super(query, fieldType, "cosine"); } @@ -130,7 +150,7 @@ protected BiFunction getScoringMethod(final float[] pro } } - class L1 extends LinearSpace { + class L1 extends KNNFieldSpace { public L1(Object query, MappedFieldType fieldType) { super(query, fieldType, "l1"); } @@ -141,7 +161,7 @@ protected BiFunction getScoringMethod(final float[] pro } } - class LInf extends LinearSpace { + class LInf extends KNNFieldSpace { public LInf(Object query, MappedFieldType fieldType) { super(query, fieldType, "l-inf"); } @@ -152,7 +172,7 @@ protected BiFunction getScoringMethod(final float[] pro } } - class InnerProd extends LinearSpace { + class InnerProd extends KNNFieldSpace { public InnerProd(Object query, MappedFieldType fieldType) { super(query, fieldType, "innerproduct"); } @@ -163,6 +183,28 @@ protected BiFunction getScoringMethod(final float[] pro } } + class Hamming extends KNNFieldSpace { + private static final Set DATA_TYPES_HAMMING = Set.of(VectorDataType.BINARY); + + public Hamming(Object query, MappedFieldType fieldType) { + super(query, fieldType, "hamming", DATA_TYPES_HAMMING); + } + + @Override + protected BiFunction getScoringMethod(final float[] processedQuery) { + // TODO we want to avoid converting back and forth between byte and float + return (float[] q, float[] v) -> 1 / (1 + KNNScoringUtil.calculateHammingBit(toByte(q), toByte(v))); + } + + private byte[] toByte(final float[] vector) { + byte[] bytes = new byte[vector.length]; + for (int i = 0; i < vector.length; i++) { + bytes[i] = (byte) vector[i]; + } + return bytes; + } + } + class HammingBit implements KNNScoringSpace { Object processedQuery; @@ -170,7 +212,7 @@ class HammingBit implements KNNScoringSpace { /** * Constructor for HammingBit scoring space. HammingBit scoring space expects values to either be of type - * Long or Base64 encoded strings, or KNN type with binary data type. + * Long or Base64 encoded strings. * * @param query Query object that, along with the doc values, will be used to compute HammingBit score * @param fieldType FieldType for the doc values that will be used @@ -182,39 +224,13 @@ public HammingBit(Object query, MappedFieldType fieldType) { } else if (isBinaryFieldType(fieldType)) { this.processedQuery = parseToBigInteger(query); this.scoringMethod = (BigInteger q, BigInteger v) -> 1.0f / (1 + KNNScoringUtil.calculateHammingBit(q, v)); - } else if (isKNNVectorFieldType(fieldType)) { - KNNVectorFieldMapper.KNNVectorFieldType knnVectorFieldType = (KNNVectorFieldMapper.KNNVectorFieldType) fieldType; - if (isBinaryVectorDataType(knnVectorFieldType) == false) { - throw new IllegalArgumentException( - String.format( - Locale.ROOT, - "Incompatible field_type for hamming space. The data type should be binary but got %s", - knnVectorFieldType.getVectorDataType().getValue() - ) - ); - } - this.processedQuery = parseToFloatArray( - query, - KNNVectorFieldMapperUtil.getExpectedVectorLength(knnVectorFieldType), - ((KNNVectorFieldMapper.KNNVectorFieldType) fieldType).getVectorDataType() - ); - // TODO we want to avoid converting back and forth between byte and float - this.scoringMethod = (float[] q, float[] v) -> 1 / (1 + KNNScoringUtil.calculateHammingBit(toByte(q), toByte(v))); } else { throw new IllegalArgumentException( - "Incompatible field_type for hamming space. The field type must of type long, binary, or knn_vector." + "Incompatible field_type for hammingbit space. The field type must of type long or binary." ); } } - private byte[] toByte(final float[] vector) { - byte[] bytes = new byte[vector.length]; - for (int i = 0; i < vector.length; i++) { - bytes[i] = (byte) vector[i]; - } - return bytes; - } - @SuppressWarnings("unchecked") public ScoreScript getScoreScript( Map params, @@ -233,27 +249,17 @@ public ScoreScript getScoreScript( ctx, searcher ); - } else if (this.processedQuery instanceof BigInteger) { - return new KNNScoreScript.BigIntegerType( - params, - (BigInteger) this.processedQuery, - field, - (BiFunction) this.scoringMethod, - lookup, - ctx, - searcher - ); - } else { - return new KNNScoreScript.KNNVectorType( - params, - (float[]) this.processedQuery, - field, - (BiFunction) this.scoringMethod, - lookup, - ctx, - searcher - ); } + + return new KNNScoreScript.BigIntegerType( + params, + (BigInteger) this.processedQuery, + field, + (BiFunction) this.scoringMethod, + lookup, + ctx, + searcher + ); } } } diff --git a/src/main/java/org/opensearch/knn/plugin/script/KNNScoringSpaceFactory.java b/src/main/java/org/opensearch/knn/plugin/script/KNNScoringSpaceFactory.java index 8d0506f58..2508fc015 100644 --- a/src/main/java/org/opensearch/knn/plugin/script/KNNScoringSpaceFactory.java +++ b/src/main/java/org/opensearch/knn/plugin/script/KNNScoringSpaceFactory.java @@ -13,9 +13,11 @@ * Factory to create correct KNNScoringSpace based on the spaceType passed in. */ public class KNNScoringSpaceFactory { + public static final String HAMMING_BIT = "hammingbit"; + public static KNNScoringSpace create(String spaceType, Object query, MappedFieldType mappedFieldType) { - if (SpaceType.HAMMING_BIT.getValue().equalsIgnoreCase(spaceType)) { - return new KNNScoringSpace.HammingBit(query, mappedFieldType); + if (SpaceType.HAMMING.getValue().equalsIgnoreCase(spaceType)) { + return new KNNScoringSpace.Hamming(query, mappedFieldType); } if (SpaceType.L2.getValue().equalsIgnoreCase(spaceType)) { @@ -36,6 +38,10 @@ public static KNNScoringSpace create(String spaceType, Object query, MappedField return new KNNScoringSpace.CosineSimilarity(query, mappedFieldType); } + if (HAMMING_BIT.equalsIgnoreCase(spaceType)) { + return new KNNScoringSpace.HammingBit(query, mappedFieldType); + } + KNNCounter.SCRIPT_QUERY_ERRORS.increment(); throw new IllegalArgumentException("Invalid space type. Please refer to the available space types."); } diff --git a/src/test/java/org/opensearch/knn/index/SpaceTypeTests.java b/src/test/java/org/opensearch/knn/index/SpaceTypeTests.java index a9a28cda0..d10cf5819 100644 --- a/src/test/java/org/opensearch/knn/index/SpaceTypeTests.java +++ b/src/test/java/org/opensearch/knn/index/SpaceTypeTests.java @@ -79,7 +79,7 @@ public void testValidateVectorDataType_whenCalled_thenReturn() { Set.of(VectorDataType.FLOAT, VectorDataType.BYTE), SpaceType.INNER_PRODUCT, Set.of(VectorDataType.FLOAT, VectorDataType.BYTE), - SpaceType.HAMMING_BIT, + SpaceType.HAMMING, Set.of(VectorDataType.BINARY) ); diff --git a/src/test/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumerTests.java b/src/test/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumerTests.java index 847cad04e..53066c1cc 100644 --- a/src/test/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumerTests.java +++ b/src/test/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumerTests.java @@ -339,7 +339,7 @@ public void testAddKNNBinaryField_whenFaissBinary_thenAdded() throws IOException String fieldName = String.format("test_field%s", randomAlphaOfLength(4)); KNNEngine knnEngine = KNNEngine.FAISS; - SpaceType spaceType = SpaceType.HAMMING_BIT; + SpaceType spaceType = SpaceType.HAMMING; VectorDataType dataType = VectorDataType.BINARY; int dimension = 16; diff --git a/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java b/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java index ad3a3d6c5..cb069ad34 100644 --- a/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java +++ b/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java @@ -223,7 +223,7 @@ public void testBuilder_whenKnnFalseWithBinary_thenSetHammingAsDefault() { Mapper.BuilderContext builderContext = new Mapper.BuilderContext(settings, new ContentPath()); KNNVectorFieldMapper knnVectorFieldMapper = builder.build(builderContext); assertTrue(knnVectorFieldMapper instanceof LegacyFieldMapper); - assertEquals(SpaceType.HAMMING_BIT.getValue(), ((LegacyFieldMapper) knnVectorFieldMapper).spaceType); + assertEquals(SpaceType.HAMMING.getValue(), ((LegacyFieldMapper) knnVectorFieldMapper).spaceType); } public void testBuilder_parse_fromKnnMethodContext_luceneEngine() throws IOException { @@ -944,7 +944,7 @@ public void testBuilder_whenBinaryWithInvalidDimension_thenException() { public void testBuilder_whenBinaryFaissHNSWWithInvalidSpaceType_thenException() { for (SpaceType spaceType : SpaceType.values()) { - if (SpaceType.UNDEFINED == spaceType || SpaceType.HAMMING_BIT == spaceType) { + if (SpaceType.UNDEFINED == spaceType || SpaceType.HAMMING == spaceType) { continue; } testBuilderWithBinaryDataType(KNNEngine.FAISS, spaceType, METHOD_HNSW, 8, "is not supported"); @@ -980,7 +980,7 @@ private void testBuilderWithBinaryDataType( KNNVectorFieldMapper knnVectorFieldMapper = builder.build(builderContext); assertTrue(knnVectorFieldMapper instanceof MethodFieldMapper); if (SpaceType.UNDEFINED == spaceType) { - assertEquals(SpaceType.HAMMING_BIT, knnVectorFieldMapper.fieldType().spaceType); + assertEquals(SpaceType.HAMMING, knnVectorFieldMapper.fieldType().spaceType); } } else { Exception ex = expectThrows(Exception.class, () -> builder.build(builderContext)); @@ -998,7 +998,7 @@ public void testBuilder_whenBinaryFaissHNSWWithSQ_thenException() { builder.knnMethodContext.setValue( new KNNMethodContext( KNNEngine.FAISS, - SpaceType.HAMMING_BIT, + SpaceType.HAMMING, new MethodComponentContext( METHOD_HNSW, Map.of(METHOD_ENCODER_PARAMETER, new MethodComponentContext(ENCODER_SQ, Collections.emptyMap())) diff --git a/src/test/java/org/opensearch/knn/index/mapper/MethodFieldMapperTests.java b/src/test/java/org/opensearch/knn/index/mapper/MethodFieldMapperTests.java index ef2a2768e..04fc7cf06 100644 --- a/src/test/java/org/opensearch/knn/index/mapper/MethodFieldMapperTests.java +++ b/src/test/java/org/opensearch/knn/index/mapper/MethodFieldMapperTests.java @@ -20,7 +20,7 @@ public void testMethodFieldMapper_whenVectorDataTypeIsGiven_thenSetItInFieldType Collections.emptyMap(), 1, VectorDataType.BINARY, - SpaceType.HAMMING_BIT + SpaceType.HAMMING ); MethodFieldMapper mappers = new MethodFieldMapper( "simpleName", diff --git a/src/test/java/org/opensearch/knn/index/memory/NativeMemoryAllocationTests.java b/src/test/java/org/opensearch/knn/index/memory/NativeMemoryAllocationTests.java index ab73c3946..b530c9d8d 100644 --- a/src/test/java/org/opensearch/knn/index/memory/NativeMemoryAllocationTests.java +++ b/src/test/java/org/opensearch/knn/index/memory/NativeMemoryAllocationTests.java @@ -110,7 +110,7 @@ public void testClose_whenBinaryFiass_thenSuccess() { } Map parameters = ImmutableMap.of( KNNConstants.SPACE_TYPE, - SpaceType.HAMMING_BIT.getValue(), + SpaceType.HAMMING.getValue(), KNNConstants.INDEX_DESCRIPTION_PARAMETER, "BHNSW32", KNNConstants.VECTOR_DATA_TYPE_FIELD, diff --git a/src/test/java/org/opensearch/knn/index/memory/NativeMemoryLoadStrategyTests.java b/src/test/java/org/opensearch/knn/index/memory/NativeMemoryLoadStrategyTests.java index 43ad7e968..134f0fe42 100644 --- a/src/test/java/org/opensearch/knn/index/memory/NativeMemoryLoadStrategyTests.java +++ b/src/test/java/org/opensearch/knn/index/memory/NativeMemoryLoadStrategyTests.java @@ -97,7 +97,7 @@ public void testLoad_whenFaissBinary_thenSuccess() throws IOException { } Map parameters = ImmutableMap.of( KNNConstants.SPACE_TYPE, - SpaceType.HAMMING_BIT.getValue(), + SpaceType.HAMMING.getValue(), KNNConstants.INDEX_DESCRIPTION_PARAMETER, "BHNSW32", KNNConstants.VECTOR_DATA_TYPE_FIELD, diff --git a/src/test/java/org/opensearch/knn/index/query/KNNQueryBuilderTests.java b/src/test/java/org/opensearch/knn/index/query/KNNQueryBuilderTests.java index aa4cd88ab..651fe75c0 100644 --- a/src/test/java/org/opensearch/knn/index/query/KNNQueryBuilderTests.java +++ b/src/test/java/org/opensearch/knn/index/query/KNNQueryBuilderTests.java @@ -763,7 +763,7 @@ public void testDoToQuery_whenRadialSearchOnBinaryIndex_thenException() { org.opensearch.knn.common.KNNConstants.METHOD_HNSW, ImmutableMap.of() ); - KNNMethodContext knnMethodContext = new KNNMethodContext(KNNEngine.FAISS, SpaceType.HAMMING_BIT, methodComponentContext); + KNNMethodContext knnMethodContext = new KNNMethodContext(KNNEngine.FAISS, SpaceType.HAMMING, methodComponentContext); when(mockKNNVectorField.getKnnMethodContext()).thenReturn(knnMethodContext); Exception e = expectThrows(UnsupportedOperationException.class, () -> knnQueryBuilder.doToQuery(mockQueryShardContext)); assertTrue(e.getMessage().contains("Binary data type does not support radial search")); @@ -1296,7 +1296,7 @@ public void testDoToQuery_whenBinary_thenValid() throws Exception { when(mockQueryShardContext.index()).thenReturn(dummyIndex); when(mockKNNVectorField.getDimension()).thenReturn(32); when(mockKNNVectorField.getVectorDataType()).thenReturn(VectorDataType.BINARY); - when(mockKNNVectorField.getSpaceType()).thenReturn(SpaceType.HAMMING_BIT); + when(mockKNNVectorField.getSpaceType()).thenReturn(SpaceType.HAMMING); when(mockQueryShardContext.fieldMapper(anyString())).thenReturn(mockKNNVectorField); KNNQuery query = (KNNQuery) knnQueryBuilder.doToQuery(mockQueryShardContext); assertArrayEquals(expectedQueryVector, query.getByteQueryVector()); @@ -1312,7 +1312,7 @@ public void testDoToQuery_whenBinaryWithInvalidDimension_thenException() throws when(mockQueryShardContext.index()).thenReturn(dummyIndex); when(mockKNNVectorField.getDimension()).thenReturn(8); when(mockKNNVectorField.getVectorDataType()).thenReturn(VectorDataType.BINARY); - when(mockKNNVectorField.getSpaceType()).thenReturn(SpaceType.HAMMING_BIT); + when(mockKNNVectorField.getSpaceType()).thenReturn(SpaceType.HAMMING); when(mockQueryShardContext.fieldMapper(anyString())).thenReturn(mockKNNVectorField); Exception ex = expectThrows(IllegalArgumentException.class, () -> knnQueryBuilder.doToQuery(mockQueryShardContext)); assertTrue(ex.getMessage(), ex.getMessage().contains("invalid dimension")); diff --git a/src/test/java/org/opensearch/knn/index/query/KNNWeightTests.java b/src/test/java/org/opensearch/knn/index/query/KNNWeightTests.java index 3cf687bcb..2905d047e 100644 --- a/src/test/java/org/opensearch/knn/index/query/KNNWeightTests.java +++ b/src/test/java/org/opensearch/knn/index/query/KNNWeightTests.java @@ -581,7 +581,7 @@ public void validateANNWithFilterQuery_whenDoingANN_thenSuccess(final boolean is KNN_ENGINE, KNNEngine.FAISS.getName(), SPACE_TYPE, - isBinary ? SpaceType.HAMMING_BIT.getValue() : SpaceType.L2.getValue() + isBinary ? SpaceType.HAMMING.getValue() : SpaceType.L2.getValue() ); when(reader.getFieldInfos()).thenReturn(fieldInfos); @@ -694,7 +694,7 @@ public void validateANNWithFilterQuery_whenExactSearch_thenSuccess(final boolean KNN_ENGINE, KNNEngine.FAISS.getName(), SPACE_TYPE, - isBinary ? SpaceType.HAMMING_BIT.getValue() : SpaceType.L2.getValue() + isBinary ? SpaceType.HAMMING.getValue() : SpaceType.L2.getValue() ); final FieldInfos fieldInfos = mock(FieldInfos.class); final FieldInfo fieldInfo = mock(FieldInfo.class); @@ -703,7 +703,7 @@ public void validateANNWithFilterQuery_whenExactSearch_thenSuccess(final boolean when(fieldInfos.fieldInfo(any())).thenReturn(fieldInfo); when(fieldInfo.attributes()).thenReturn(attributesMap); if (isBinary) { - when(fieldInfo.getAttribute(SPACE_TYPE)).thenReturn(SpaceType.HAMMING_BIT.getValue()); + when(fieldInfo.getAttribute(SPACE_TYPE)).thenReturn(SpaceType.HAMMING.getValue()); } else { when(fieldInfo.getAttribute(SPACE_TYPE)).thenReturn(SpaceType.L2.getValue()); } @@ -899,7 +899,7 @@ public void testANNWithFilterQuery_whenExactSearchViaThresholdSettingOnBinaryInd KNN_ENGINE, KNNEngine.FAISS.getName(), SPACE_TYPE, - SpaceType.HAMMING_BIT.name(), + SpaceType.HAMMING.name(), PARAMETERS, String.format(Locale.ROOT, "{\"%s\":\"%s\"}", INDEX_DESCRIPTION_PARAMETER, "BHNSW32") ); @@ -909,7 +909,7 @@ public void testANNWithFilterQuery_whenExactSearchViaThresholdSettingOnBinaryInd when(reader.getFieldInfos()).thenReturn(fieldInfos); when(fieldInfos.fieldInfo(any())).thenReturn(fieldInfo); when(fieldInfo.attributes()).thenReturn(attributesMap); - when(fieldInfo.getAttribute(SPACE_TYPE)).thenReturn(SpaceType.HAMMING_BIT.getValue()); + when(fieldInfo.getAttribute(SPACE_TYPE)).thenReturn(SpaceType.HAMMING.getValue()); when(fieldInfo.getName()).thenReturn(FIELD_NAME); when(reader.getBinaryDocValues(FIELD_NAME)).thenReturn(binaryDocValues); when(binaryDocValues.advance(0)).thenReturn(0); diff --git a/src/test/java/org/opensearch/knn/index/query/filtered/FilteredIdsKNNByteIteratorTests.java b/src/test/java/org/opensearch/knn/index/query/filtered/FilteredIdsKNNByteIteratorTests.java index aabbb1f9c..7583f50bc 100644 --- a/src/test/java/org/opensearch/knn/index/query/filtered/FilteredIdsKNNByteIteratorTests.java +++ b/src/test/java/org/opensearch/knn/index/query/filtered/FilteredIdsKNNByteIteratorTests.java @@ -23,7 +23,7 @@ public class FilteredIdsKNNByteIteratorTests extends TestCase { @SneakyThrows public void testNextDoc_whenCalled_IterateAllDocs() { - final SpaceType spaceType = SpaceType.HAMMING_BIT; + final SpaceType spaceType = SpaceType.HAMMING; final byte[] queryVector = { 1, 2, 3 }; final int[] filterIds = { 1, 2, 3 }; final List dataVectors = Arrays.asList(new byte[] { 11, 12, 13 }, new byte[] { 14, 15, 16 }, new byte[] { 17, 18, 19 }); diff --git a/src/test/java/org/opensearch/knn/index/query/filtered/NestedFilteredIdsKNNByteIteratorTests.java b/src/test/java/org/opensearch/knn/index/query/filtered/NestedFilteredIdsKNNByteIteratorTests.java index 01a4eb2b1..c4b7859d0 100644 --- a/src/test/java/org/opensearch/knn/index/query/filtered/NestedFilteredIdsKNNByteIteratorTests.java +++ b/src/test/java/org/opensearch/knn/index/query/filtered/NestedFilteredIdsKNNByteIteratorTests.java @@ -24,7 +24,7 @@ public class NestedFilteredIdsKNNByteIteratorTests extends TestCase { @SneakyThrows public void testNextDoc_whenIterate_ReturnBestChildDocsPerParent() { - final SpaceType spaceType = SpaceType.HAMMING_BIT; + final SpaceType spaceType = SpaceType.HAMMING; final byte[] queryVector = { 1, 2, 3 }; final int[] filterIds = { 0, 2, 3 }; // Parent id for 0 -> 1 diff --git a/src/test/java/org/opensearch/knn/jni/JNIServiceTests.java b/src/test/java/org/opensearch/knn/jni/JNIServiceTests.java index 6fab6c795..78b878b90 100644 --- a/src/test/java/org/opensearch/knn/jni/JNIServiceTests.java +++ b/src/test/java/org/opensearch/knn/jni/JNIServiceTests.java @@ -678,7 +678,7 @@ public void testCreateIndex_binary_faiss_valid() { INDEX_DESCRIPTION_PARAMETER, faissBinaryMethod, KNNConstants.SPACE_TYPE, - SpaceType.HAMMING_BIT.getValue(), + SpaceType.HAMMING.getValue(), KNNConstants.VECTOR_DATA_TYPE_FIELD, VectorDataType.BINARY.getValue() ), @@ -993,7 +993,7 @@ public void testQueryBinaryIndex_faiss_valid() { INDEX_DESCRIPTION_PARAMETER, method, KNNConstants.SPACE_TYPE, - SpaceType.HAMMING_BIT.getValue(), + SpaceType.HAMMING.getValue(), KNNConstants.VECTOR_DATA_TYPE_FIELD, VectorDataType.BINARY.getValue() ), diff --git a/src/test/java/org/opensearch/knn/plugin/script/KNNScoringSpaceFactoryTests.java b/src/test/java/org/opensearch/knn/plugin/script/KNNScoringSpaceFactoryTests.java index 52bc22eff..e24acc483 100644 --- a/src/test/java/org/opensearch/knn/plugin/script/KNNScoringSpaceFactoryTests.java +++ b/src/test/java/org/opensearch/knn/plugin/script/KNNScoringSpaceFactoryTests.java @@ -6,6 +6,7 @@ package org.opensearch.knn.plugin.script; import org.opensearch.knn.KNNTestCase; +import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.mapper.KNNVectorFieldMapper; import org.opensearch.knn.index.SpaceType; import org.opensearch.index.mapper.NumberFieldMapper; @@ -17,9 +18,11 @@ public class KNNScoringSpaceFactoryTests extends KNNTestCase { public void testValidSpaces() { - KNNVectorFieldMapper.KNNVectorFieldType knnVectorFieldType = mock(KNNVectorFieldMapper.KNNVectorFieldType.class); when(knnVectorFieldType.getDimension()).thenReturn(3); + KNNVectorFieldMapper.KNNVectorFieldType knnVectorFieldTypeBinary = mock(KNNVectorFieldMapper.KNNVectorFieldType.class); + when(knnVectorFieldTypeBinary.getDimension()).thenReturn(24); + when(knnVectorFieldTypeBinary.getVectorDataType()).thenReturn(VectorDataType.BINARY); NumberFieldMapper.NumberFieldType numberFieldType = new NumberFieldMapper.NumberFieldType( "field", NumberFieldMapper.NumberType.LONG @@ -46,7 +49,14 @@ public void testValidSpaces() { ); assertTrue( KNNScoringSpaceFactory.create( - SpaceType.HAMMING_BIT.getValue(), + SpaceType.HAMMING.getValue(), + floatQueryObject, + knnVectorFieldTypeBinary + ) instanceof KNNScoringSpace.Hamming + ); + assertTrue( + KNNScoringSpaceFactory.create( + KNNScoringSpaceFactory.HAMMING_BIT, longQueryObject, numberFieldType ) instanceof KNNScoringSpace.HammingBit @@ -54,6 +64,22 @@ public void testValidSpaces() { } public void testInvalidSpace() { + List floatQueryObject = List.of(1.0f, 1.0f, 1.0f); + KNNVectorFieldMapper.KNNVectorFieldType knnVectorFieldType = mock(KNNVectorFieldMapper.KNNVectorFieldType.class); + when(knnVectorFieldType.getDimension()).thenReturn(3); + KNNVectorFieldMapper.KNNVectorFieldType knnVectorFieldTypeBinary = mock(KNNVectorFieldMapper.KNNVectorFieldType.class); + when(knnVectorFieldTypeBinary.getDimension()).thenReturn(24); + when(knnVectorFieldTypeBinary.getVectorDataType()).thenReturn(VectorDataType.BINARY); + + // Verify expectThrows(IllegalArgumentException.class, () -> KNNScoringSpaceFactory.create(SpaceType.L2.getValue(), null, null)); + expectThrows( + IllegalArgumentException.class, + () -> KNNScoringSpaceFactory.create(SpaceType.L2.getValue(), floatQueryObject, knnVectorFieldTypeBinary) + ); + expectThrows( + IllegalArgumentException.class, + () -> KNNScoringSpaceFactory.create(SpaceType.HAMMING.getValue(), floatQueryObject, knnVectorFieldType) + ); } } diff --git a/src/test/java/org/opensearch/knn/plugin/script/KNNScoringSpaceTests.java b/src/test/java/org/opensearch/knn/plugin/script/KNNScoringSpaceTests.java index 690c9b53a..5a86deb7e 100644 --- a/src/test/java/org/opensearch/knn/plugin/script/KNNScoringSpaceTests.java +++ b/src/test/java/org/opensearch/knn/plugin/script/KNNScoringSpaceTests.java @@ -29,6 +29,7 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; +import static org.opensearch.knn.plugin.script.KNNScoringSpace.KNNFieldSpace.DATA_TYPES_DEFAULT; public class KNNScoringSpaceTests extends KNNTestCase { @@ -37,7 +38,7 @@ private void expectThrowsExceptionWithNonKNNField(Class clazz) throws NoSuchMeth NumberFieldMapper.NumberFieldType invalidFieldType = mock(NumberFieldMapper.NumberFieldType.class); Exception e = expectThrows(InvocationTargetException.class, () -> constructor.newInstance(null, invalidFieldType)); assertTrue(e.getCause() instanceof IllegalArgumentException); - assertTrue(e.getCause().getMessage().contains("The field type must be knn_vector")); + assertTrue(e.getCause().getMessage(), e.getCause().getMessage().contains("The field type must be knn_vector")); } private void expectThrowsExceptionWithKNNFieldWithBinaryDataType(Class clazz) throws NoSuchMethodException { @@ -46,7 +47,10 @@ private void expectThrowsExceptionWithKNNFieldWithBinaryDataType(Class clazz) th when(invalidFieldType.getVectorDataType()).thenReturn(VectorDataType.BINARY); Exception e = expectThrows(InvocationTargetException.class, () -> constructor.newInstance(null, invalidFieldType)); assertTrue(e.getCause() instanceof IllegalArgumentException); - assertTrue(e.getCause().getMessage().contains("The data type should be either float or byte")); + assertTrue( + e.getCause().getMessage(), + e.getCause().getMessage().contains(String.format("The data type should be %s", DATA_TYPES_DEFAULT)) + ); } @SneakyThrows @@ -197,7 +201,7 @@ public void testHammingBit_Base64() { ); } - public void testHammingBit_whenKNNFieldType_thenSucceed() { + public void testHamming_whenKNNFieldType_thenSucceed() { List arrayListQueryObject = new ArrayList<>(Arrays.asList(1.0, 2.0, 3.0)); KNNMethodContext knnMethodContext = KNNMethodContext.getDefault(); KNNVectorFieldMapper.KNNVectorFieldType fieldType = new KNNVectorFieldMapper.KNNVectorFieldType( @@ -207,17 +211,16 @@ public void testHammingBit_whenKNNFieldType_thenSucceed() { knnMethodContext, VectorDataType.BINARY ); - KNNScoringSpace.HammingBit hammingBit = new KNNScoringSpace.HammingBit(arrayListQueryObject, fieldType); + KNNScoringSpace.Hamming hamming = new KNNScoringSpace.Hamming(arrayListQueryObject, fieldType); float[] arrayFloat = new float[] { 1.0f, 2.0f, 3.0f }; - BiFunction scoringMethod = (BiFunction) hammingBit.scoringMethod; - assertEquals(1F, scoringMethod.apply(arrayFloat, arrayFloat), 0.1F); + assertEquals(1F, hamming.scoringMethod.apply(arrayFloat, arrayFloat), 0.1F); } - public void testHammingBit_whenNonBinaryVectorDataType_thenException() { + public void testHamming_whenNonBinaryVectorDataType_thenException() { KNNVectorFieldMapper.KNNVectorFieldType invalidFieldType = mock(KNNVectorFieldMapper.KNNVectorFieldType.class); when(invalidFieldType.getVectorDataType()).thenReturn(randomInt() % 2 == 0 ? VectorDataType.FLOAT : VectorDataType.BYTE); - Exception e = expectThrows(IllegalArgumentException.class, () -> new KNNScoringSpace.HammingBit(null, invalidFieldType)); - assertTrue(e.getMessage().contains("The data type should be binary")); + Exception e = expectThrows(IllegalArgumentException.class, () -> new KNNScoringSpace.Hamming(null, invalidFieldType)); + assertTrue(e.getMessage(), e.getMessage().contains("The data type should be [BINARY]")); } } diff --git a/src/test/java/org/opensearch/knn/plugin/script/KNNScriptScoringIT.java b/src/test/java/org/opensearch/knn/plugin/script/KNNScriptScoringIT.java index d7bad8332..0d6248830 100644 --- a/src/test/java/org/opensearch/knn/plugin/script/KNNScriptScoringIT.java +++ b/src/test/java/org/opensearch/knn/plugin/script/KNNScriptScoringIT.java @@ -274,7 +274,7 @@ public void testHammingScriptScore_Long() throws Exception { Long queryValue1 = -9223372036818526181L; params1.put("field", FIELD_NAME); params1.put("query_value", queryValue1); - params1.put("space_type", SpaceType.HAMMING_BIT.getValue()); + params1.put("space_type", KNNScoringSpaceFactory.HAMMING_BIT); Request request1 = constructKNNScriptQueryRequest(INDEX_NAME, qb1, params1, 4, Collections.emptyMap()); Response response1 = client().performRequest(request1); assertEquals(request1.getEndpoint() + ": failed", RestStatus.OK, RestStatus.fromCode(response1.getStatusLine().getStatusCode())); @@ -312,7 +312,7 @@ public void testHammingScriptScore_Long() throws Exception { Long queryValue2 = 10L; params2.put("field", FIELD_NAME); params2.put("query_value", queryValue2); - params2.put("space_type", SpaceType.HAMMING_BIT.getValue()); + params2.put("space_type", KNNScoringSpaceFactory.HAMMING_BIT); Request request2 = constructKNNScriptQueryRequest(INDEX_NAME, qb2, params2, 4, Collections.emptyMap()); Response response2 = client().performRequest(request2); assertEquals(request2.getEndpoint() + ": failed", RestStatus.OK, RestStatus.fromCode(response2.getStatusLine().getStatusCode())); @@ -380,7 +380,7 @@ public void testHammingScriptScore_Base64() throws Exception { String queryValue1 = "gAAAAAIpIBs="; params1.put("field", FIELD_NAME); params1.put("query_value", queryValue1); - params1.put("space_type", SpaceType.HAMMING_BIT.getValue()); + params1.put("space_type", KNNScoringSpaceFactory.HAMMING_BIT); Request request1 = constructKNNScriptQueryRequest(INDEX_NAME, qb1, params1, 4, Collections.emptyMap()); Response response1 = client().performRequest(request1); assertEquals(request1.getEndpoint() + ": failed", RestStatus.OK, RestStatus.fromCode(response1.getStatusLine().getStatusCode())); @@ -418,7 +418,7 @@ public void testHammingScriptScore_Base64() throws Exception { String queryValue2 = "AAAAAAIpIBs="; params2.put("field", FIELD_NAME); params2.put("query_value", queryValue2); - params2.put("space_type", SpaceType.HAMMING_BIT.getValue()); + params2.put("space_type", KNNScoringSpaceFactory.HAMMING_BIT); Request request2 = constructKNNScriptQueryRequest(INDEX_NAME, qb2, params2, 4, Collections.emptyMap()); Response response2 = client().performRequest(request2); assertEquals(request2.getEndpoint() + ": failed", RestStatus.OK, RestStatus.fromCode(response2.getStatusLine().getStatusCode())); @@ -588,7 +588,7 @@ public void testKNNScriptScoreOnModelBasedIndex() throws Exception { .toString(); for (SpaceType spaceType : SpaceType.values()) { - if (SpaceType.UNDEFINED == spaceType || SpaceType.HAMMING_BIT == spaceType) { + if (SpaceType.UNDEFINED == spaceType || SpaceType.HAMMING == spaceType) { continue; } final float[] queryVector = randomVector(dimensions); From 2091af3ff4ed2c32c5fc98c709faa93fc880a350 Mon Sep 17 00:00:00 2001 From: Junqiu Lei Date: Mon, 22 Jul 2024 12:48:34 -0700 Subject: [PATCH 299/416] Add binary format support with IVF method in Faiss Engine (#1784) (#1863) Signed-off-by: Junqiu Lei --- CHANGELOG.md | 1 + jni/include/faiss_wrapper.h | 13 + .../org_opensearch_knn_jni_FaissService.h | 16 + jni/src/faiss_wrapper.cpp | 133 ++++++++ .../org_opensearch_knn_jni_FaissService.cpp | 28 ++ .../opensearch/knn/common/KNNConstants.java | 1 + .../org/opensearch/knn/index/IndexUtil.java | 178 +++++++---- .../opensearch/knn/index/VectorDataType.java | 2 + .../KNN80Codec/KNN80DocValuesConsumer.java | 83 ++--- .../index/mapper/KNNVectorFieldMapper.java | 12 +- .../knn/index/mapper/ModelFieldMapper.java | 8 +- .../index/memory/NativeMemoryAllocation.java | 12 +- .../memory/NativeMemoryEntryContext.java | 15 +- .../memory/NativeMemoryLoadStrategy.java | 13 +- .../knn/index/query/KNNQueryBuilder.java | 1 + .../opensearch/knn/index/query/KNNWeight.java | 12 +- .../org/opensearch/knn/index/util/Faiss.java | 2 +- .../org/opensearch/knn/indices/ModelDao.java | 1 + .../opensearch/knn/indices/ModelMetadata.java | 155 +++++----- .../org/opensearch/knn/jni/FaissService.java | 29 ++ .../org/opensearch/knn/jni/JNICommons.java | 13 + .../org/opensearch/knn/jni/JNIService.java | 12 +- .../plugin/rest/RestTrainModelHandler.java | 12 +- .../TrainingJobRouterTransportAction.java | 17 +- .../transport/TrainingModelRequest.java | 28 +- .../TrainingModelTransportAction.java | 6 +- .../training/ByteTrainingDataConsumer.java | 81 +++++ .../training/FloatTrainingDataConsumer.java | 67 ++++ .../knn/training/TrainingDataConsumer.java | 46 ++- .../opensearch/knn/training/TrainingJob.java | 19 +- .../opensearch/knn/training/VectorReader.java | 20 +- .../opensearch/knn/KNNSingleNodeTestCase.java | 14 +- .../org/opensearch/knn/index/FaissIT.java | 286 ++++++++++++++++-- .../opensearch/knn/index/IndexUtilTests.java | 85 ++++-- .../index/KNNCreateIndexFromModelTests.java | 3 +- .../KNN80DocValuesConsumerTests.java | 32 +- .../knn/index/codec/KNNCodecTestCase.java | 10 +- .../mapper/KNNVectorFieldMapperTests.java | 18 +- .../memory/NativeMemoryAllocationTests.java | 18 +- .../memory/NativeMemoryCacheManagerTests.java | 4 +- .../memory/NativeMemoryEntryContextTests.java | 22 +- .../memory/NativeMemoryLoadStrategyTests.java | 9 +- .../knn/index/query/KNNQueryBuilderTests.java | 3 + .../knn/index/query/KNNWeightTests.java | 4 + .../knn/indices/ModelCacheTests.java | 38 ++- .../opensearch/knn/indices/ModelDaoTests.java | 43 ++- .../knn/indices/ModelMetadataTests.java | 141 ++++++--- .../opensearch/knn/indices/ModelTests.java | 110 ++++++- .../plugin/action/RestKNNStatsHandlerIT.java | 16 +- .../transport/GetModelResponseTests.java | 8 +- ...oveModelFromCacheTransportActionTests.java | 4 +- ...TrainingJobRouterTransportActionTests.java | 100 +++++- .../transport/TrainingModelRequestTests.java | 48 ++- .../TrainingModelTransportActionTests.java | 6 +- ...ateModelGraveyardTransportActionTests.java | 4 +- .../UpdateModelMetadataRequestTests.java | 10 +- ...dateModelMetadataTransportActionTests.java | 4 +- ...va => FloatTrainingDataConsumerTests.java} | 8 +- .../knn/training/TrainingJobTests.java | 28 +- .../knn/training/VectorReaderTests.java | 101 ++++--- .../org/opensearch/knn/KNNRestTestCase.java | 35 +++ 61 files changed, 1762 insertions(+), 486 deletions(-) create mode 100644 src/main/java/org/opensearch/knn/training/ByteTrainingDataConsumer.java create mode 100644 src/main/java/org/opensearch/knn/training/FloatTrainingDataConsumer.java rename src/test/java/org/opensearch/knn/training/{TrainingDataConsumerTests.java => FloatTrainingDataConsumerTests.java} (88%) diff --git a/CHANGELOG.md b/CHANGELOG.md index dd712c66e..85a91dab2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), * Add binary format support with HNSW method in Faiss Engine [#1781](https://github.com/opensearch-project/k-NN/pull/1781) * Add script scoring support for knn field with binary data type [#1826](https://github.com/opensearch-project/k-NN/pull/1826) * Add painless script support for hamming with binary vector data type [#1839](https://github.com/opensearch-project/k-NN/pull/1839) +* Add binary format support with IVF method in Faiss Engine [#1784](https://github.com/opensearch-project/k-NN/pull/1784) ### Enhancements * Switch from byte stream to byte ref for serde [#1825](https://github.com/opensearch-project/k-NN/pull/1825) ### Bug Fixes diff --git a/jni/include/faiss_wrapper.h b/jni/include/faiss_wrapper.h index 2b9bc2c76..5ad0dedc4 100644 --- a/jni/include/faiss_wrapper.h +++ b/jni/include/faiss_wrapper.h @@ -29,6 +29,12 @@ namespace knn_jni { jlong vectorsAddressJ, jint dimJ, jstring indexPathJ, jbyteArray templateIndexJ, jobject parametersJ); + // Create an index with ids and vectors. Instead of creating a new index, this function creates the index + // based off of the template index passed in. The index is serialized to indexPathJ. + void CreateBinaryIndexFromTemplate(knn_jni::JNIUtilInterface * jniUtil, JNIEnv * env, jintArray idsJ, + jlong vectorsAddressJ, jint dimJ, jstring indexPathJ, jbyteArray templateIndexJ, + jobject parametersJ); + // Load an index from indexPathJ into memory. // // Return a pointer to the loaded index @@ -96,6 +102,13 @@ namespace knn_jni { jbyteArray TrainIndex(knn_jni::JNIUtilInterface * jniUtil, JNIEnv * env, jobject parametersJ, jint dimension, jlong trainVectorsPointerJ); + // Create an empty binary index defined by the values in the Java map, parametersJ. Train the index with + // the vector of floats located at trainVectorsPointerJ. + // + // Return the serialized representation + jbyteArray TrainBinaryIndex(knn_jni::JNIUtilInterface * jniUtil, JNIEnv * env, jobject parametersJ, jint dimension, + jlong trainVectorsPointerJ); + /* * Perform a range search with filter against the index located in memory at indexPointerJ. * diff --git a/jni/include/org_opensearch_knn_jni_FaissService.h b/jni/include/org_opensearch_knn_jni_FaissService.h index 7cc071ff3..025fb12e8 100644 --- a/jni/include/org_opensearch_knn_jni_FaissService.h +++ b/jni/include/org_opensearch_knn_jni_FaissService.h @@ -43,6 +43,14 @@ JNIEXPORT void JNICALL Java_org_opensearch_knn_jni_FaissService_createBinaryInde JNIEXPORT void JNICALL Java_org_opensearch_knn_jni_FaissService_createIndexFromTemplate (JNIEnv *, jclass, jintArray, jlong, jint, jstring, jbyteArray, jobject); +/* + * Class: org_opensearch_knn_jni_FaissService + * Method: createBinaryIndexFromTemplate + * Signature: ([IJILjava/lang/String;[BLjava/util/Map;)V + */ + JNIEXPORT void JNICALL Java_org_opensearch_knn_jni_FaissService_createBinaryIndexFromTemplate + (JNIEnv *, jclass, jintArray, jlong, jint, jstring, jbyteArray, jobject); + /* * Class: org_opensearch_knn_jni_FaissService * Method: loadIndex @@ -139,6 +147,14 @@ JNIEXPORT void JNICALL Java_org_opensearch_knn_jni_FaissService_initLibrary JNIEXPORT jbyteArray JNICALL Java_org_opensearch_knn_jni_FaissService_trainIndex (JNIEnv *, jclass, jobject, jint, jlong); +/* + * Class: org_opensearch_knn_jni_FaissService + * Method: trainBinaryIndex + * Signature: (Ljava/util/Map;IJ)[B + */ +JNIEXPORT jbyteArray JNICALL Java_org_opensearch_knn_jni_FaissService_trainBinaryIndex + (JNIEnv *, jclass, jobject, jint, jlong); + /* * Class: org_opensearch_knn_jni_FaissService * Method: transferVectors diff --git a/jni/src/faiss_wrapper.cpp b/jni/src/faiss_wrapper.cpp index 2d5054635..1d4437414 100644 --- a/jni/src/faiss_wrapper.cpp +++ b/jni/src/faiss_wrapper.cpp @@ -70,6 +70,9 @@ void SetExtraParameters(knn_jni::JNIUtilInterface * jniUtil, JNIEnv *env, // Train an index with data provided void InternalTrainIndex(faiss::Index * index, faiss::idx_t n, const float* x); +// Train a binary index with data provided +void InternalTrainBinaryIndex(faiss::IndexBinary * index, faiss::idx_t n, const float* x); + // Converts the int FilterIds to Faiss ids type array. void convertFilterIdsToFaissIdType(const int* filterIds, int filterIdsLength, faiss::idx_t* convertedFilterIds); @@ -223,6 +226,76 @@ void knn_jni::faiss_wrapper::CreateIndexFromTemplate(knn_jni::JNIUtilInterface * faiss::write_index(&idMap, indexPathCpp.c_str()); } +void knn_jni::faiss_wrapper::CreateBinaryIndexFromTemplate(knn_jni::JNIUtilInterface * jniUtil, JNIEnv * env, jintArray idsJ, + jlong vectorsAddressJ, jint dimJ, jstring indexPathJ, + jbyteArray templateIndexJ, jobject parametersJ) { + if (idsJ == nullptr) { + throw std::runtime_error("IDs cannot be null"); + } + + if (vectorsAddressJ <= 0) { + throw std::runtime_error("VectorsAddress cannot be less than 0"); + } + + if(dimJ <= 0) { + throw std::runtime_error("Vectors dimensions cannot be less than or equal to 0"); + } + + if (indexPathJ == nullptr) { + throw std::runtime_error("Index path cannot be null"); + } + + if (templateIndexJ == nullptr) { + throw std::runtime_error("Template index cannot be null"); + } + + // Set thread count if it is passed in as a parameter. Setting this variable will only impact the current thread + auto parametersCpp = jniUtil->ConvertJavaMapToCppMap(env, parametersJ); + if(parametersCpp.find(knn_jni::INDEX_THREAD_QUANTITY) != parametersCpp.end()) { + auto threadCount = jniUtil->ConvertJavaObjectToCppInteger(env, parametersCpp[knn_jni::INDEX_THREAD_QUANTITY]); + omp_set_num_threads(threadCount); + } + jniUtil->DeleteLocalRef(env, parametersJ); + + // Read data set + // Read vectors from memory address + auto *inputVectors = reinterpret_cast*>(vectorsAddressJ); + int dim = (int)dimJ; + if (dim % 8 != 0) { + throw std::runtime_error("Dimensions should be multiply of 8"); + } + int numVectors = (int) (inputVectors->size() / (uint64_t) (dim / 8)); + int numIds = jniUtil->GetJavaIntArrayLength(env, idsJ); + if (numIds != numVectors) { + throw std::runtime_error("Number of IDs does not match number of vectors"); + } + + // Get vector of bytes from jbytearray + int indexBytesCount = jniUtil->GetJavaBytesArrayLength(env, templateIndexJ); + jbyte * indexBytesJ = jniUtil->GetByteArrayElements(env, templateIndexJ, nullptr); + + faiss::VectorIOReader vectorIoReader; + for (int i = 0; i < indexBytesCount; i++) { + vectorIoReader.data.push_back((uint8_t) indexBytesJ[i]); + } + jniUtil->ReleaseByteArrayElements(env, templateIndexJ, indexBytesJ, JNI_ABORT); + + // Create faiss index + std::unique_ptr indexWriter; + indexWriter.reset(faiss::read_index_binary(&vectorIoReader, 0)); + + auto idVector = jniUtil->ConvertJavaIntArrayToCppIntVector(env, idsJ); + faiss::IndexBinaryIDMap idMap = faiss::IndexBinaryIDMap(indexWriter.get()); + idMap.add_with_ids(numVectors, reinterpret_cast(inputVectors->data()), idVector.data()); + // Releasing the vectorsAddressJ memory as that is not required once we have created the index. + // This is not the ideal approach, please refer this gh issue for long term solution: + // https://github.com/opensearch-project/k-NN/issues/1600 + delete inputVectors; + // Write the index to disk + std::string indexPathCpp(jniUtil->ConvertJavaStringToCppString(env, indexPathJ)); + faiss::write_index_binary(&idMap, indexPathCpp.c_str()); +} + jlong knn_jni::faiss_wrapper::LoadIndex(knn_jni::JNIUtilInterface * jniUtil, JNIEnv * env, jstring indexPathJ) { if (indexPathJ == nullptr) { throw std::runtime_error("Index path cannot be null"); @@ -634,6 +707,57 @@ jbyteArray knn_jni::faiss_wrapper::TrainIndex(knn_jni::JNIUtilInterface * jniUti return ret; } +jbyteArray knn_jni::faiss_wrapper::TrainBinaryIndex(knn_jni::JNIUtilInterface * jniUtil, JNIEnv * env, jobject parametersJ, + jint dimensionJ, jlong trainVectorsPointerJ) { + // First, we need to build the index + if (parametersJ == nullptr) { + throw std::runtime_error("Parameters cannot be null"); + } + + auto parametersCpp = jniUtil->ConvertJavaMapToCppMap(env, parametersJ); + + jobject spaceTypeJ = knn_jni::GetJObjectFromMapOrThrow(parametersCpp, knn_jni::SPACE_TYPE); + std::string spaceTypeCpp(jniUtil->ConvertJavaObjectToCppString(env, spaceTypeJ)); + faiss::MetricType metric = TranslateSpaceToMetric(spaceTypeCpp); + + // Create faiss index + jobject indexDescriptionJ = knn_jni::GetJObjectFromMapOrThrow(parametersCpp, knn_jni::INDEX_DESCRIPTION); + std::string indexDescriptionCpp(jniUtil->ConvertJavaObjectToCppString(env, indexDescriptionJ)); + + std::unique_ptr indexWriter; + indexWriter.reset(faiss::index_binary_factory((int) dimensionJ, indexDescriptionCpp.c_str())); + + // Set thread count if it is passed in as a parameter. Setting this variable will only impact the current thread + if(parametersCpp.find(knn_jni::INDEX_THREAD_QUANTITY) != parametersCpp.end()) { + auto threadCount = jniUtil->ConvertJavaObjectToCppInteger(env, parametersCpp[knn_jni::INDEX_THREAD_QUANTITY]); + omp_set_num_threads(threadCount); + } + + // Train index if needed + auto *trainingVectorsPointerCpp = reinterpret_cast*>(trainVectorsPointerJ); + int numVectors = trainingVectorsPointerCpp->size()/(int) dimensionJ; + if(!indexWriter->is_trained) { + InternalTrainBinaryIndex(indexWriter.get(), numVectors, trainingVectorsPointerCpp->data()); + } + jniUtil->DeleteLocalRef(env, parametersJ); + + // Now that indexWriter is trained, we just load the bytes into an array and return + faiss::VectorIOWriter vectorIoWriter; + faiss::write_index_binary(indexWriter.get(), &vectorIoWriter); + + // Wrap in smart pointer + std::unique_ptr jbytesBuffer; + jbytesBuffer.reset(new jbyte[vectorIoWriter.data.size()]); + int c = 0; + for (auto b : vectorIoWriter.data) { + jbytesBuffer[c++] = (jbyte) b; + } + + jbyteArray ret = jniUtil->NewByteArray(env, vectorIoWriter.data.size()); + jniUtil->SetByteArrayRegion(env, ret, 0, vectorIoWriter.data.size(), jbytesBuffer.get()); + return ret; +} + faiss::MetricType TranslateSpaceToMetric(const std::string& spaceType) { if (spaceType == knn_jni::L2) { return faiss::METRIC_L2; @@ -692,6 +816,15 @@ void InternalTrainIndex(faiss::Index * index, faiss::idx_t n, const float* x) { } } +void InternalTrainBinaryIndex(faiss::IndexBinary * index, faiss::idx_t n, const float* x) { + if (auto * indexIvf = dynamic_cast(index)) { + indexIvf->make_direct_map(); + } + if (!index->is_trained) { + index->train(n, reinterpret_cast(x)); + } +} + std::unique_ptr buildIDGrouperBitmap(knn_jni::JNIUtilInterface * jniUtil, JNIEnv *env, jintArray parentIdsJ, std::vector* bitmap) { int *parentIdsArray = jniUtil->GetIntArrayElements(env, parentIdsJ, nullptr); int parentIdsLength = jniUtil->GetJavaIntArrayLength(env, parentIdsJ); diff --git a/jni/src/org_opensearch_knn_jni_FaissService.cpp b/jni/src/org_opensearch_knn_jni_FaissService.cpp index 6e447b034..2394e2951 100644 --- a/jni/src/org_opensearch_knn_jni_FaissService.cpp +++ b/jni/src/org_opensearch_knn_jni_FaissService.cpp @@ -90,6 +90,21 @@ JNIEXPORT void JNICALL Java_org_opensearch_knn_jni_FaissService_createIndexFromT } } +JNIEXPORT void JNICALL Java_org_opensearch_knn_jni_FaissService_createBinaryIndexFromTemplate(JNIEnv * env, jclass cls, + jintArray idsJ, + jlong vectorsAddressJ, + jint dimJ, + jstring indexPathJ, + jbyteArray templateIndexJ, + jobject parametersJ) +{ + try { + knn_jni::faiss_wrapper::CreateBinaryIndexFromTemplate(&jniUtil, env, idsJ, vectorsAddressJ, dimJ, indexPathJ, templateIndexJ, parametersJ); + } catch (...) { + jniUtil.CatchCppExceptionAndThrowJava(env); + } +} + JNIEXPORT jlong JNICALL Java_org_opensearch_knn_jni_FaissService_loadIndex(JNIEnv * env, jclass cls, jstring indexPathJ) { try { @@ -220,6 +235,19 @@ JNIEXPORT jbyteArray JNICALL Java_org_opensearch_knn_jni_FaissService_trainIndex return nullptr; } +JNIEXPORT jbyteArray JNICALL Java_org_opensearch_knn_jni_FaissService_trainBinaryIndex(JNIEnv * env, jclass cls, + jobject parametersJ, + jint dimensionJ, + jlong trainVectorsPointerJ) +{ + try { + return knn_jni::faiss_wrapper::TrainBinaryIndex(&jniUtil, env, parametersJ, dimensionJ, trainVectorsPointerJ); + } catch (...) { + jniUtil.CatchCppExceptionAndThrowJava(env); + } + return nullptr; +} + JNIEXPORT jlong JNICALL Java_org_opensearch_knn_jni_FaissService_transferVectors(JNIEnv * env, jclass cls, jlong vectorsPointerJ, jobjectArray vectorsJ) diff --git a/src/main/java/org/opensearch/knn/common/KNNConstants.java b/src/main/java/org/opensearch/knn/common/KNNConstants.java index e73212afe..77a884a5c 100644 --- a/src/main/java/org/opensearch/knn/common/KNNConstants.java +++ b/src/main/java/org/opensearch/knn/common/KNNConstants.java @@ -67,6 +67,7 @@ public class KNNConstants { public static final String SEARCH_SIZE_PARAMETER = "search_size"; public static final String VECTOR_DATA_TYPE_FIELD = "data_type"; + public static final String MODEL_VECTOR_DATA_TYPE_KEY = VECTOR_DATA_TYPE_FIELD; public static final VectorDataType DEFAULT_VECTOR_DATA_TYPE_FIELD = VectorDataType.FLOAT; public static final String RADIAL_SEARCH_KEY = "radial_search"; diff --git a/src/main/java/org/opensearch/knn/index/IndexUtil.java b/src/main/java/org/opensearch/knn/index/IndexUtil.java index c88da4d3c..109ec7101 100644 --- a/src/main/java/org/opensearch/knn/index/IndexUtil.java +++ b/src/main/java/org/opensearch/knn/index/IndexUtil.java @@ -49,29 +49,10 @@ public class IndexUtil { private static final Version MINIMAL_SUPPORTED_VERSION_FOR_MODEL_METHOD_COMPONENT_CONTEXT = Version.V_2_13_0; private static final Version MINIMAL_SUPPORTED_VERSION_FOR_RADIAL_SEARCH = Version.V_2_14_0; private static final Version MINIMAL_SUPPORTED_VERSION_FOR_METHOD_PARAMETERS = Version.V_2_16_0; + private static final Version MINIMAL_SUPPORTED_VERSION_FOR_MODEL_VECTOR_DATA_TYPE = Version.V_2_16_0; // public so neural search can access it public static final Map minimalRequiredVersionMap = initializeMinimalRequiredVersionMap(); - private static Map initializeMinimalRequiredVersionMap() { - final Map versionMap = new HashMap<>() { - { - put("filter", MINIMAL_SUPPORTED_VERSION_FOR_LUCENE_HNSW_FILTER); - put("ignore_unmapped", MINIMAL_SUPPORTED_VERSION_FOR_IGNORE_UNMAPPED); - put(MODEL_NODE_ASSIGNMENT_KEY, MINIMAL_SUPPORTED_VERSION_FOR_MODEL_NODE_ASSIGNMENT); - put(MODEL_METHOD_COMPONENT_CONTEXT_KEY, MINIMAL_SUPPORTED_VERSION_FOR_MODEL_METHOD_COMPONENT_CONTEXT); - put(KNNConstants.RADIAL_SEARCH_KEY, MINIMAL_SUPPORTED_VERSION_FOR_RADIAL_SEARCH); - put(KNNConstants.METHOD_PARAMETER, MINIMAL_SUPPORTED_VERSION_FOR_METHOD_PARAMETERS); - } - }; - - for (final MethodParameter methodParameter : MethodParameter.values()) { - if (methodParameter.getVersion() != null) { - versionMap.put(methodParameter.getName(), methodParameter.getVersion()); - } - } - return Collections.unmodifiableMap(versionMap); - } - /** * Determines the size of a file on disk in kilobytes * @@ -90,37 +71,6 @@ public static int getFileSizeInKB(String filePath) { return Math.toIntExact((file.length() / BYTES_PER_KILOBYTES) + 1L); // Add one so that integer division rounds up } - /** - * This method retrieves the field mapping by a given field path from the index metadata. - * - * @param properties Index metadata mapping properties. - * @param fieldPath The field path string that make up the path to the field mapping. e.g. "a.b.field" or "field". - * The field path is applied and checked in OpenSearch, so it is guaranteed to be valid. - * - * @return The field mapping object if found, or null if the field is not found in the index metadata. - */ - private static Object getFieldMapping(final Map properties, final String fieldPath) { - String[] fieldPaths = fieldPath.split("\\."); - Object currentFieldMapping = properties; - - // Iterate through the field path list to retrieve the field mapping. - for (String path : fieldPaths) { - currentFieldMapping = ((Map) currentFieldMapping).get(path); - if (currentFieldMapping == null) { - return null; - } - - if (currentFieldMapping instanceof Map) { - Object possibleProperties = ((Map) currentFieldMapping).get("properties"); - if (possibleProperties instanceof Map) { - currentFieldMapping = possibleProperties; - } - } - } - - return currentFieldMapping; - } - /** * Validate that a field is a k-NN vector field and has the expected dimension * @@ -137,7 +87,8 @@ public static ValidationException validateKnnField( IndexMetadata indexMetadata, String field, int expectedDimension, - ModelDao modelDao + ModelDao modelDao, + VectorDataType expectedVectorDataType ) { // Index metadata should not be null if (indexMetadata == null) { @@ -192,17 +143,27 @@ public static ValidationException validateKnnField( return exception; } - String vectorDataType = (String) fieldMap.get(VECTOR_DATA_TYPE_FIELD); - if (VectorDataType.BINARY.toString().equalsIgnoreCase(vectorDataType)) { - exception.addValidationError( - String.format( - Locale.ROOT, - "Field \"%s\" is of data type %s. Only FLOAT or BYTE is supported.", - field, - VectorDataType.BINARY - ) - ); - return exception; + if (expectedVectorDataType != null) { + if (VectorDataType.BYTE == expectedVectorDataType) { + exception.addValidationError( + String.format(Locale.ROOT, "vector data type \"%s\" is not supported for training.", expectedVectorDataType.getValue()) + ); + return exception; + } + VectorDataType trainIndexDataType = getVectorDataTypeFromFieldMapping(fieldMap); + + if (trainIndexDataType != expectedVectorDataType) { + exception.addValidationError( + String.format( + Locale.ROOT, + "Field \"%s\" has data type %s, which is different from data type used in the training request: %s", + field, + trainIndexDataType.getValue(), + expectedVectorDataType.getValue() + ) + ); + return exception; + } } // Return if dimension does not need to be checked @@ -336,4 +297,95 @@ public static boolean isBinaryIndex(KNNEngine knnEngine, Map par && parameters.get(VECTOR_DATA_TYPE_FIELD) != null && parameters.get(VECTOR_DATA_TYPE_FIELD).toString().equals(VectorDataType.BINARY.getValue()); } + + /** + * Tell if it is binary index or not + * + * @param vectorDataType vector data type + * @return true if it is binary index + */ + public static boolean isBinaryIndex(VectorDataType vectorDataType) { + return VectorDataType.BINARY == vectorDataType; + } + + /** + * Update vector data type into parameters + * + * @param parameters parameters associated with an index + * @param vectorDataType vector data type + */ + public static void updateVectorDataTypeToParameters(Map parameters, VectorDataType vectorDataType) { + if (VectorDataType.BINARY == vectorDataType) { + parameters.put(VECTOR_DATA_TYPE_FIELD, VectorDataType.BINARY.getValue()); + } + } + + /** + * This method retrieves the field mapping by a given field path from the index metadata. + * + * @param properties Index metadata mapping properties. + * @param fieldPath The field path string that make up the path to the field mapping. e.g. "a.b.field" or "field". + * The field path is applied and checked in OpenSearch, so it is guaranteed to be valid. + * + * @return The field mapping object if found, or null if the field is not found in the index metadata. + */ + private static Object getFieldMapping(final Map properties, final String fieldPath) { + String[] fieldPaths = fieldPath.split("\\."); + Object currentFieldMapping = properties; + + // Iterate through the field path list to retrieve the field mapping. + for (String path : fieldPaths) { + currentFieldMapping = ((Map) currentFieldMapping).get(path); + if (currentFieldMapping == null) { + return null; + } + + if (currentFieldMapping instanceof Map) { + Object possibleProperties = ((Map) currentFieldMapping).get("properties"); + if (possibleProperties instanceof Map) { + currentFieldMapping = possibleProperties; + } + } + } + + return currentFieldMapping; + } + + /** + * This method is used to get the vector data type from field mapping + * @param fieldMap field mapping + * @return vector data type + */ + private static VectorDataType getVectorDataTypeFromFieldMapping(Map fieldMap) { + if (fieldMap.containsKey(VECTOR_DATA_TYPE_FIELD)) { + return VectorDataType.get((String) fieldMap.get(VECTOR_DATA_TYPE_FIELD)); + } + return VectorDataType.DEFAULT; + } + + /** + * Initialize the minimal required version map + * + * @return minimal required version map + */ + private static Map initializeMinimalRequiredVersionMap() { + final Map versionMap = new HashMap<>() { + { + put("filter", MINIMAL_SUPPORTED_VERSION_FOR_LUCENE_HNSW_FILTER); + put("ignore_unmapped", MINIMAL_SUPPORTED_VERSION_FOR_IGNORE_UNMAPPED); + put(MODEL_NODE_ASSIGNMENT_KEY, MINIMAL_SUPPORTED_VERSION_FOR_MODEL_NODE_ASSIGNMENT); + put(MODEL_METHOD_COMPONENT_CONTEXT_KEY, MINIMAL_SUPPORTED_VERSION_FOR_MODEL_METHOD_COMPONENT_CONTEXT); + put(KNNConstants.RADIAL_SEARCH_KEY, MINIMAL_SUPPORTED_VERSION_FOR_RADIAL_SEARCH); + put(KNNConstants.METHOD_PARAMETER, MINIMAL_SUPPORTED_VERSION_FOR_METHOD_PARAMETERS); + put(KNNConstants.MODEL_VECTOR_DATA_TYPE_KEY, MINIMAL_SUPPORTED_VERSION_FOR_MODEL_VECTOR_DATA_TYPE); + } + }; + + for (final MethodParameter methodParameter : MethodParameter.values()) { + if (methodParameter.getVersion() != null) { + versionMap.put(methodParameter.getName(), methodParameter.getVersion()); + } + } + return Collections.unmodifiableMap(versionMap); + } } diff --git a/src/main/java/org/opensearch/knn/index/VectorDataType.java b/src/main/java/org/opensearch/knn/index/VectorDataType.java index b36d508f1..9283e5ee6 100644 --- a/src/main/java/org/opensearch/knn/index/VectorDataType.java +++ b/src/main/java/org/opensearch/knn/index/VectorDataType.java @@ -136,4 +136,6 @@ public static VectorDataType get(String vectorDataType) { ); } } + + public static VectorDataType DEFAULT = FLOAT; } diff --git a/src/main/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumer.java b/src/main/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumer.java index 2eba19f7e..d0008b7a0 100644 --- a/src/main/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumer.java +++ b/src/main/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumer.java @@ -5,7 +5,6 @@ package org.opensearch.knn.index.codec.KNN80Codec; -import com.google.common.collect.ImmutableMap; import lombok.NonNull; import lombok.extern.log4j.Log4j2; import org.apache.lucene.store.ChecksumIndexInput; @@ -13,6 +12,7 @@ import org.opensearch.common.StopWatch; import org.opensearch.core.xcontent.DeprecationHandler; import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.knn.index.IndexUtil; import org.opensearch.knn.index.KNNSettings; import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.codec.transfer.VectorTransfer; @@ -109,30 +109,10 @@ private KNNEngine getKNNEngine(@NonNull FieldInfo field) { return KNNEngine.getEngine(engineName); } - private VectorTransfer getVectorTransfer(FieldInfo field) { - if (VectorDataType.BINARY.getValue().equalsIgnoreCase(field.attributes().get(KNNConstants.VECTOR_DATA_TYPE_FIELD))) { - return new VectorTransferByte(KNNSettings.getVectorStreamingMemoryLimit().getBytes()); - } - return new VectorTransferFloat(KNNSettings.getVectorStreamingMemoryLimit().getBytes()); - } - public void addKNNBinaryField(FieldInfo field, DocValuesProducer valuesProducer, boolean isMerge, boolean isRefresh) throws IOException { // Get values to be indexed BinaryDocValues values = valuesProducer.getBinary(field); - KNNCodecUtil.Pair pair = KNNCodecUtil.getPair(values, getVectorTransfer(field)); - if (pair.getVectorAddress() == 0 || pair.docs.length == 0) { - logger.info("Skipping engine index creation as there are no vectors or docs in the segment"); - return; - } - long arraySize = calculateArraySize(pair.docs.length, pair.getDimension(), pair.serializationMode); - if (isMerge) { - KNNGraphValue.MERGE_CURRENT_OPERATIONS.increment(); - KNNGraphValue.MERGE_CURRENT_DOCS.incrementBy(pair.docs.length); - KNNGraphValue.MERGE_CURRENT_SIZE_IN_BYTES.incrementBy(arraySize); - } - // Increment counter for number of graph index requests - KNNCounter.GRAPH_INDEX_REQUESTS.increment(); final KNNEngine knnEngine = getKNNEngine(field); final String engineFileName = buildEngineFileName( state.segmentInfo.name, @@ -144,30 +124,53 @@ public void addKNNBinaryField(FieldInfo field, DocValuesProducer valuesProducer, ((FSDirectory) (FilterDirectory.unwrap(state.directory))).getDirectory().toString(), engineFileName ).toString(); + + // Determine if we are creating an index from a model or from scratch NativeIndexCreator indexCreator; - // Create library index either from model or from scratch - if (field.attributes().containsKey(MODEL_ID)) { - String modelId = field.attributes().get(MODEL_ID); + KNNCodecUtil.Pair pair; + Map fieldAttributes = field.attributes(); + + if (fieldAttributes.containsKey(MODEL_ID)) { + String modelId = fieldAttributes.get(MODEL_ID); Model model = ModelCache.getInstance().get(modelId); if (model.getModelBlob() == null) { throw new RuntimeException(String.format("There is no trained model with id \"%s\"", modelId)); } - indexCreator = () -> createKNNIndexFromTemplate(model.getModelBlob(), pair, knnEngine, indexPath); + VectorDataType vectorDataType = model.getModelMetadata().getVectorDataType(); + pair = KNNCodecUtil.getPair(values, getVectorTransfer(vectorDataType)); + indexCreator = () -> createKNNIndexFromTemplate(model, pair, knnEngine, indexPath); } else { + // get vector data type from field attributes or provide default value + VectorDataType vectorDataType = VectorDataType.get( + fieldAttributes.getOrDefault(KNNConstants.VECTOR_DATA_TYPE_FIELD, VectorDataType.DEFAULT.getValue()) + ); + pair = KNNCodecUtil.getPair(values, getVectorTransfer(vectorDataType)); indexCreator = () -> createKNNIndexFromScratch(field, pair, knnEngine, indexPath); } + // Skip index creation if no vectors or docs in segment + if (pair.getVectorAddress() == 0 || pair.docs.length == 0) { + logger.info("Skipping engine index creation as there are no vectors or docs in the segment"); + return; + } + + long arraySize = calculateArraySize(pair.docs.length, pair.getDimension(), pair.serializationMode); + if (isMerge) { + KNNGraphValue.MERGE_CURRENT_OPERATIONS.increment(); + KNNGraphValue.MERGE_CURRENT_DOCS.incrementBy(pair.docs.length); + KNNGraphValue.MERGE_CURRENT_SIZE_IN_BYTES.incrementBy(arraySize); recordMergeStats(pair.docs.length, arraySize); } + // Increment counter for number of graph index requests + KNNCounter.GRAPH_INDEX_REQUESTS.increment(); + if (isRefresh) { recordRefreshStats(); } - // This is a bit of a hack. We have to create an output here and then immediately close it to ensure that - // engineFileName is added to the tracked files by Lucene's TrackingDirectoryWrapper. Otherwise, the file will - // not be marked as added to the directory. + // Ensure engineFileName is added to the tracked files by Lucene's TrackingDirectoryWrapper state.directory.createOutput(engineFileName, state.context).close(); indexCreator.createIndex(); writeFooter(indexPath, engineFileName); @@ -186,18 +189,19 @@ private void recordRefreshStats() { KNNGraphValue.REFRESH_TOTAL_OPERATIONS.increment(); } - private void createKNNIndexFromTemplate(byte[] model, KNNCodecUtil.Pair pair, KNNEngine knnEngine, String indexPath) { - Map parameters = ImmutableMap.of( - KNNConstants.INDEX_THREAD_QTY, - KNNSettings.state().getSettingValue(KNNSettings.KNN_ALGO_PARAM_INDEX_THREAD_QTY) - ); + private void createKNNIndexFromTemplate(Model model, KNNCodecUtil.Pair pair, KNNEngine knnEngine, String indexPath) { + Map parameters = new HashMap<>(); + parameters.put(KNNConstants.INDEX_THREAD_QTY, KNNSettings.state().getSettingValue(KNNSettings.KNN_ALGO_PARAM_INDEX_THREAD_QTY)); + + IndexUtil.updateVectorDataTypeToParameters(parameters, model.getModelMetadata().getVectorDataType()); + AccessController.doPrivileged((PrivilegedAction) () -> { JNIService.createIndexFromTemplate( pair.docs, pair.getVectorAddress(), pair.getDimension(), indexPath, - model, + model.getModelBlob(), parameters, knnEngine ); @@ -237,13 +241,13 @@ private void createKNNIndexFromScratch(FieldInfo fieldInfo, KNNCodecUtil.Pair pa // Update index description of Faiss for binary data type if (KNNEngine.FAISS == knnEngine && VectorDataType.BINARY.getValue() - .equals(fieldAttributes.getOrDefault(KNNConstants.VECTOR_DATA_TYPE_FIELD, VectorDataType.FLOAT.getValue())) + .equals(fieldAttributes.getOrDefault(KNNConstants.VECTOR_DATA_TYPE_FIELD, VectorDataType.DEFAULT.getValue())) && parameters.get(KNNConstants.INDEX_DESCRIPTION_PARAMETER) != null) { parameters.put( KNNConstants.INDEX_DESCRIPTION_PARAMETER, FAISS_BINARY_INDEX_DESCRIPTION_PREFIX + parameters.get(KNNConstants.INDEX_DESCRIPTION_PARAMETER).toString() ); - parameters.put(KNNConstants.VECTOR_DATA_TYPE_FIELD, VectorDataType.BINARY.getValue()); + IndexUtil.updateVectorDataTypeToParameters(parameters, VectorDataType.BINARY); } // Used to determine how many threads to use when indexing @@ -349,4 +353,11 @@ private boolean isChecksumValid(long value) { // https://github.com/apache/lucene/blob/branch_9_0/lucene/core/src/java/org/apache/lucene/codecs/CodecUtil.java#L644-L647 return (value & CRC32_CHECKSUM_SANITY) != 0; } + + private VectorTransfer getVectorTransfer(VectorDataType vectorDataType) { + if (VectorDataType.BINARY == vectorDataType) { + return new VectorTransferByte(KNNSettings.getVectorStreamingMemoryLimit().getBytes()); + } + return new VectorTransferFloat(KNNSettings.getVectorStreamingMemoryLimit().getBytes()); + } } diff --git a/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapper.java b/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapper.java index dc9827189..5c19a4989 100644 --- a/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapper.java +++ b/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapper.java @@ -603,7 +603,8 @@ protected void parseCreateField(ParseContext context) throws IOException { context, fieldType().getDimension(), fieldType().getSpaceType(), - getMethodComponentContext(fieldType().getKnnMethodContext()) + getMethodComponentContext(fieldType().getKnnMethodContext()), + fieldType().getVectorDataType() ); } @@ -646,8 +647,13 @@ protected List getFieldsForByteVector(final byte[] array, final FieldType return fields; } - protected void parseCreateField(ParseContext context, int dimension, SpaceType spaceType, MethodComponentContext methodComponentContext) - throws IOException { + protected void parseCreateField( + ParseContext context, + int dimension, + SpaceType spaceType, + MethodComponentContext methodComponentContext, + VectorDataType vectorDataType + ) throws IOException { validateIfKNNPluginEnabled(); validateIfCircuitBreakerIsNotTriggered(); diff --git a/src/main/java/org/opensearch/knn/index/mapper/ModelFieldMapper.java b/src/main/java/org/opensearch/knn/index/mapper/ModelFieldMapper.java index 554871279..adaaef28e 100644 --- a/src/main/java/org/opensearch/knn/index/mapper/ModelFieldMapper.java +++ b/src/main/java/org/opensearch/knn/index/mapper/ModelFieldMapper.java @@ -62,6 +62,12 @@ protected void parseCreateField(ParseContext context) throws IOException { ); } - parseCreateField(context, modelMetadata.getDimension(), modelMetadata.getSpaceType(), modelMetadata.getMethodComponentContext()); + parseCreateField( + context, + modelMetadata.getDimension(), + modelMetadata.getSpaceType(), + modelMetadata.getMethodComponentContext(), + modelMetadata.getVectorDataType() + ); } } diff --git a/src/main/java/org/opensearch/knn/index/memory/NativeMemoryAllocation.java b/src/main/java/org/opensearch/knn/index/memory/NativeMemoryAllocation.java index b108fb6f0..0b0f1e615 100644 --- a/src/main/java/org/opensearch/knn/index/memory/NativeMemoryAllocation.java +++ b/src/main/java/org/opensearch/knn/index/memory/NativeMemoryAllocation.java @@ -13,6 +13,8 @@ import lombok.Getter; import org.apache.lucene.index.LeafReaderContext; +import org.opensearch.knn.index.IndexUtil; +import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.query.KNNWeight; import org.opensearch.knn.jni.JNICommons; import org.opensearch.knn.jni.JNIService; @@ -248,6 +250,7 @@ class TrainingDataAllocation implements NativeMemoryAllocation { private int readCount; private Semaphore readSemaphore; private Semaphore writeSemaphore; + private VectorDataType vectorDataType; /** * Constructor @@ -256,7 +259,7 @@ class TrainingDataAllocation implements NativeMemoryAllocation { * @param memoryAddress pointer in memory to the training data allocation * @param size amount memory needed for allocation in kilobytes */ - TrainingDataAllocation(ExecutorService executor, long memoryAddress, int size) { + public TrainingDataAllocation(ExecutorService executor, long memoryAddress, int size, VectorDataType vectorDataType) { this.executor = executor; this.closed = false; this.memoryAddress = memoryAddress; @@ -265,6 +268,7 @@ class TrainingDataAllocation implements NativeMemoryAllocation { this.readCount = 0; this.readSemaphore = new Semaphore(1); this.writeSemaphore = new Semaphore(1); + this.vectorDataType = vectorDataType; } @Override @@ -295,7 +299,11 @@ private void cleanup() { closed = true; if (this.memoryAddress != 0) { - JNICommons.freeVectorData(this.memoryAddress); + if (IndexUtil.isBinaryIndex(vectorDataType)) { + JNICommons.freeByteVectorData(this.memoryAddress); + } else { + JNICommons.freeVectorData(this.memoryAddress); + } } } diff --git a/src/main/java/org/opensearch/knn/index/memory/NativeMemoryEntryContext.java b/src/main/java/org/opensearch/knn/index/memory/NativeMemoryEntryContext.java index 7f14a2341..b5ddff1e2 100644 --- a/src/main/java/org/opensearch/knn/index/memory/NativeMemoryEntryContext.java +++ b/src/main/java/org/opensearch/knn/index/memory/NativeMemoryEntryContext.java @@ -14,6 +14,7 @@ import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.Nullable; import org.opensearch.knn.index.IndexUtil; +import org.opensearch.knn.index.VectorDataType; import java.io.IOException; import java.util.Map; @@ -169,6 +170,7 @@ public static class TrainingDataEntryContext extends NativeMemoryEntryContext trainingDataAllocation.writeUnlock(), ex -> { // Close unsafe will assume that the caller passes control of the writelock to it. It // will then handle releasing the write lock once the close operations finish. diff --git a/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java b/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java index 6f1617193..e319fa388 100644 --- a/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java +++ b/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java @@ -545,6 +545,7 @@ protected Query doToQuery(QueryShardContext context) { knnEngine = modelMetadata.getKnnEngine(); spaceType = modelMetadata.getSpaceType(); methodComponentContext = modelMetadata.getMethodComponentContext(); + vectorDataType = modelMetadata.getVectorDataType(); } else if (knnMethodContext != null) { // If the dimension is set but the knnMethodContext is not then the field is using the legacy mapping diff --git a/src/main/java/org/opensearch/knn/index/query/KNNWeight.java b/src/main/java/org/opensearch/knn/index/query/KNNWeight.java index 5301f8fa6..4e5dec89f 100644 --- a/src/main/java/org/opensearch/knn/index/query/KNNWeight.java +++ b/src/main/java/org/opensearch/knn/index/query/KNNWeight.java @@ -215,6 +215,7 @@ private Map doANNSearch(final LeafReaderContext context, final B KNNEngine knnEngine; SpaceType spaceType; + VectorDataType vectorDataType; // Check if a modelId exists. If so, the space type and engine will need to be picked up from the model's // metadata. @@ -227,11 +228,15 @@ private Map doANNSearch(final LeafReaderContext context, final B knnEngine = modelMetadata.getKnnEngine(); spaceType = modelMetadata.getSpaceType(); + vectorDataType = modelMetadata.getVectorDataType(); } else { String engineName = fieldInfo.attributes().getOrDefault(KNN_ENGINE, KNNEngine.NMSLIB.getName()); knnEngine = KNNEngine.getEngine(engineName); String spaceTypeName = fieldInfo.attributes().getOrDefault(SPACE_TYPE, SpaceType.L2.getValue()); spaceType = SpaceType.getSpace(spaceTypeName); + vectorDataType = VectorDataType.get( + fieldInfo.attributes().getOrDefault(VECTOR_DATA_TYPE_FIELD, VectorDataType.FLOAT.getValue()) + ); } List engineFiles = getEngineFiles(reader, knnEngine.getExtension()); @@ -251,12 +256,7 @@ private Map doANNSearch(final LeafReaderContext context, final B new NativeMemoryEntryContext.IndexEntryContext( indexPath.toString(), NativeMemoryLoadStrategy.IndexLoadStrategy.getInstance(), - getParametersAtLoading( - spaceType, - knnEngine, - knnQuery.getIndexName(), - VectorDataType.get(fieldInfo.attributes().getOrDefault(VECTOR_DATA_TYPE_FIELD, VectorDataType.FLOAT.getValue())) - ), + getParametersAtLoading(spaceType, knnEngine, knnQuery.getIndexName(), vectorDataType), knnQuery.getIndexName(), modelId ), diff --git a/src/main/java/org/opensearch/knn/index/util/Faiss.java b/src/main/java/org/opensearch/knn/index/util/Faiss.java index e4d0490bf..7e33db30c 100644 --- a/src/main/java/org/opensearch/knn/index/util/Faiss.java +++ b/src/main/java/org/opensearch/knn/index/util/Faiss.java @@ -305,7 +305,7 @@ public class Faiss extends NativeLibrary { return ((4L * centroids * dimension) / BYTES_PER_KILOBYTES) + 1; }) .build() - ).addSpaces(SpaceType.UNDEFINED, SpaceType.L2, SpaceType.INNER_PRODUCT).build() + ).addSpaces(SpaceType.UNDEFINED, SpaceType.L2, SpaceType.INNER_PRODUCT, SpaceType.HAMMING).build() ); final static Faiss INSTANCE = new Faiss( diff --git a/src/main/java/org/opensearch/knn/indices/ModelDao.java b/src/main/java/org/opensearch/knn/indices/ModelDao.java index 0bc6c5edb..37edcd3ae 100644 --- a/src/main/java/org/opensearch/knn/indices/ModelDao.java +++ b/src/main/java/org/opensearch/knn/indices/ModelDao.java @@ -292,6 +292,7 @@ private void putInternal(Model model, ActionListener listener, Do put(KNNConstants.MODEL_DESCRIPTION, modelMetadata.getDescription()); put(KNNConstants.MODEL_ERROR, modelMetadata.getError()); put(KNNConstants.MODEL_NODE_ASSIGNMENT, modelMetadata.getNodeAssignment()); + put(KNNConstants.VECTOR_DATA_TYPE_FIELD, modelMetadata.getVectorDataType()); MethodComponentContext methodComponentContext = modelMetadata.getMethodComponentContext(); if (!methodComponentContext.getName().isEmpty()) { diff --git a/src/main/java/org/opensearch/knn/indices/ModelMetadata.java b/src/main/java/org/opensearch/knn/indices/ModelMetadata.java index f3a5506cd..6bfb3aaf2 100644 --- a/src/main/java/org/opensearch/knn/indices/ModelMetadata.java +++ b/src/main/java/org/opensearch/knn/indices/ModelMetadata.java @@ -26,6 +26,7 @@ import org.opensearch.knn.index.IndexUtil; import org.opensearch.knn.index.MethodComponentContext; import org.opensearch.knn.index.SpaceType; +import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.util.KNNEngine; import java.io.IOException; @@ -48,6 +49,7 @@ public class ModelMetadata implements Writeable, ToXContentObject { final private String timestamp; final private String description; final private String trainingNodeAssignment; + final private VectorDataType vectorDataType; private MethodComponentContext methodComponentContext; private String error; @@ -81,6 +83,12 @@ public ModelMetadata(StreamInput in) throws IOException { } else { this.methodComponentContext = MethodComponentContext.EMPTY; } + + if (IndexUtil.isVersionOnOrAfterMinRequiredVersion(in.getVersion(), KNNConstants.MODEL_VECTOR_DATA_TYPE_KEY)) { + this.vectorDataType = VectorDataType.get(in.readString()); + } else { + this.vectorDataType = VectorDataType.DEFAULT; + } } /** @@ -95,6 +103,7 @@ public ModelMetadata(StreamInput in) throws IOException { * @param error error message associated with model * @param trainingNodeAssignment node assignment for the model * @param methodComponentContext method component context associated with model + * @param vectorDataType vector data type of the model */ public ModelMetadata( KNNEngine knnEngine, @@ -105,7 +114,8 @@ public ModelMetadata( String description, String error, String trainingNodeAssignment, - MethodComponentContext methodComponentContext + MethodComponentContext methodComponentContext, + VectorDataType vectorDataType ) { this.knnEngine = Objects.requireNonNull(knnEngine, "knnEngine must not be null"); this.spaceType = Objects.requireNonNull(spaceType, "spaceType must not be null"); @@ -128,6 +138,7 @@ public ModelMetadata( this.error = Objects.requireNonNull(error, "error must not be null"); this.trainingNodeAssignment = Objects.requireNonNull(trainingNodeAssignment, "node assignment must not be null"); this.methodComponentContext = Objects.requireNonNull(methodComponentContext, "method context must not be null"); + this.vectorDataType = Objects.requireNonNull(vectorDataType, "vector data type must not be null"); } /** @@ -211,6 +222,10 @@ public MethodComponentContext getMethodComponentContext() { return methodComponentContext; } + public VectorDataType getVectorDataType() { + return vectorDataType; + } + /** * setter for model's state * @@ -241,7 +256,8 @@ public String toString() { description, error, trainingNodeAssignment, - methodComponentContext.toClusterStateString() + methodComponentContext.toClusterStateString(), + vectorDataType.getValue() ); } @@ -259,6 +275,7 @@ public boolean equals(Object obj) { equalsBuilder.append(getTimestamp(), other.getTimestamp()); equalsBuilder.append(getDescription(), other.getDescription()); equalsBuilder.append(getError(), other.getError()); + equalsBuilder.append(getVectorDataType(), other.getVectorDataType()); return equalsBuilder.isEquals(); } @@ -273,6 +290,7 @@ public int hashCode() { .append(getDescription()) .append(getError()) .append(getMethodComponentContext()) + .append(getVectorDataType()) .toHashCode(); } @@ -284,81 +302,60 @@ public int hashCode() { */ public static ModelMetadata fromString(String modelMetadataString) { String[] modelMetadataArray = modelMetadataString.split(DELIMITER, -1); + int length = modelMetadataArray.length; - // Training node assignment was added as a field in Version 2.12.0 - // Because models can be created on older versions and the cluster can be upgraded after, - // we need to accept model metadata arrays both with and without the training node assignment. - if (modelMetadataArray.length == 7) { - log.debug( - "Model metadata array does not contain training node assignment or method component context. Assuming empty string node assignment and empty method component context." - ); - KNNEngine knnEngine = KNNEngine.getEngine(modelMetadataArray[0]); - SpaceType spaceType = SpaceType.getSpace(modelMetadataArray[1]); - int dimension = Integer.parseInt(modelMetadataArray[2]); - ModelState modelState = ModelState.getModelState(modelMetadataArray[3]); - String timestamp = modelMetadataArray[4]; - String description = modelMetadataArray[5]; - String error = modelMetadataArray[6]; - return new ModelMetadata( - knnEngine, - spaceType, - dimension, - modelState, - timestamp, - description, - error, - "", - MethodComponentContext.EMPTY - ); - } else if (modelMetadataArray.length == 8) { - log.debug("Model metadata contains training node assignment. Assuming empty method component context."); - KNNEngine knnEngine = KNNEngine.getEngine(modelMetadataArray[0]); - SpaceType spaceType = SpaceType.getSpace(modelMetadataArray[1]); - int dimension = Integer.parseInt(modelMetadataArray[2]); - ModelState modelState = ModelState.getModelState(modelMetadataArray[3]); - String timestamp = modelMetadataArray[4]; - String description = modelMetadataArray[5]; - String error = modelMetadataArray[6]; - String trainingNodeAssignment = modelMetadataArray[7]; - return new ModelMetadata( - knnEngine, - spaceType, - dimension, - modelState, - timestamp, - description, - error, - trainingNodeAssignment, - MethodComponentContext.EMPTY - ); - } else if (modelMetadataArray.length == 9) { - log.debug("Model metadata contains training node assignment and method context"); - KNNEngine knnEngine = KNNEngine.getEngine(modelMetadataArray[0]); - SpaceType spaceType = SpaceType.getSpace(modelMetadataArray[1]); - int dimension = Integer.parseInt(modelMetadataArray[2]); - ModelState modelState = ModelState.getModelState(modelMetadataArray[3]); - String timestamp = modelMetadataArray[4]; - String description = modelMetadataArray[5]; - String error = modelMetadataArray[6]; - String trainingNodeAssignment = modelMetadataArray[7]; - MethodComponentContext methodComponentContext = MethodComponentContext.fromClusterStateString(modelMetadataArray[8]); - return new ModelMetadata( - knnEngine, - spaceType, - dimension, - modelState, - timestamp, - description, - error, - trainingNodeAssignment, - methodComponentContext - ); - } else { + if (length < 7 || length > 10) { throw new IllegalArgumentException( "Illegal format for model metadata. Must be of the form " - + "\",,,,,,\" or \",,,,,,,\" or \",,,,,,,,\"." + + "\",,,,,,\" or " + + "\",,,,,,,\" or " + + "\",,,,,,,,\" or " + + "\",,,,,,,,,\"." ); } + + KNNEngine knnEngine = KNNEngine.getEngine(modelMetadataArray[0]); + SpaceType spaceType = SpaceType.getSpace(modelMetadataArray[1]); + int dimension = Integer.parseInt(modelMetadataArray[2]); + ModelState modelState = ModelState.getModelState(modelMetadataArray[3]); + String timestamp = modelMetadataArray[4]; + String description = modelMetadataArray[5]; + String error = modelMetadataArray[6]; + String trainingNodeAssignment = length > 7 ? modelMetadataArray[7] : ""; + MethodComponentContext methodComponentContext = length > 8 + ? MethodComponentContext.fromClusterStateString(modelMetadataArray[8]) + : MethodComponentContext.EMPTY; + VectorDataType vectorDataType = length > 9 ? VectorDataType.get(modelMetadataArray[9]) : VectorDataType.DEFAULT; + + log.debug(getLogMessage(length)); + + return new ModelMetadata( + knnEngine, + spaceType, + dimension, + modelState, + timestamp, + description, + error, + trainingNodeAssignment, + methodComponentContext, + vectorDataType + ); + } + + private static String getLogMessage(int length) { + switch (length) { + case 7: + return "Model metadata array does not contain training node assignment or method component context. Assuming empty string node assignment and empty method component context."; + case 8: + return "Model metadata contains training node assignment. Assuming empty method component context."; + case 9: + return "Model metadata contains training node assignment and method context."; + case 10: + return "Model metadata contains training node assignment, method context and vector data type."; + default: + throw new IllegalArgumentException("Unexpected metadata array length: " + length); + } } private static String objectToString(Object value) { @@ -387,6 +384,7 @@ public static ModelMetadata getMetadataFromSourceMap(final Map m Object error = modelSourceMap.get(KNNConstants.MODEL_ERROR); Object trainingNodeAssignment = modelSourceMap.get(KNNConstants.MODEL_NODE_ASSIGNMENT); Object methodComponentContext = modelSourceMap.get(KNNConstants.MODEL_METHOD_COMPONENT_CONTEXT); + Object vectorDataType = modelSourceMap.get(KNNConstants.VECTOR_DATA_TYPE_FIELD); if (trainingNodeAssignment == null) { trainingNodeAssignment = ""; @@ -407,6 +405,10 @@ public static ModelMetadata getMetadataFromSourceMap(final Map m methodComponentContext = MethodComponentContext.EMPTY; } + if (vectorDataType == null) { + vectorDataType = VectorDataType.DEFAULT.getValue(); + } + ModelMetadata modelMetadata = new ModelMetadata( KNNEngine.getEngine(objectToString(engine)), SpaceType.getSpace(objectToString(space)), @@ -416,7 +418,8 @@ public static ModelMetadata getMetadataFromSourceMap(final Map m objectToString(description), objectToString(error), objectToString(trainingNodeAssignment), - (MethodComponentContext) methodComponentContext + (MethodComponentContext) methodComponentContext, + VectorDataType.get(objectToString(vectorDataType)) ); return modelMetadata; } @@ -436,6 +439,9 @@ public void writeTo(StreamOutput out) throws IOException { if (IndexUtil.isVersionOnOrAfterMinRequiredVersion(out.getVersion(), IndexUtil.MODEL_METHOD_COMPONENT_CONTEXT_KEY)) { getMethodComponentContext().writeTo(out); } + if (IndexUtil.isVersionOnOrAfterMinRequiredVersion(out.getVersion(), KNNConstants.MODEL_VECTOR_DATA_TYPE_KEY)) { + out.writeString(vectorDataType.getValue()); + } } @Override @@ -456,6 +462,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws getMethodComponentContext().toXContent(builder, params); builder.endObject(); } + if (IndexUtil.isClusterOnOrAfterMinRequiredVersion(KNNConstants.MODEL_VECTOR_DATA_TYPE_KEY)) { + builder.field(KNNConstants.VECTOR_DATA_TYPE_FIELD, vectorDataType.getValue()); + } return builder; } } diff --git a/src/main/java/org/opensearch/knn/jni/FaissService.java b/src/main/java/org/opensearch/knn/jni/FaissService.java index 21de90765..1f23f6fcd 100644 --- a/src/main/java/org/opensearch/knn/jni/FaissService.java +++ b/src/main/java/org/opensearch/knn/jni/FaissService.java @@ -96,6 +96,25 @@ public static native void createIndexFromTemplate( Map parameters ); + /** + * Create a binary index for the native library with a provided template index + * + * @param ids array of ids mapping to the data passed in + * @param vectorsAddress address of native memory where vectors are stored + * @param dim dimension of the vector to be indexed + * @param indexPath path to save index file to + * @param templateIndex empty template index + * @param parameters additional build time parameters + */ + public static native void createBinaryIndexFromTemplate( + int[] ids, + long vectorsAddress, + int dim, + String indexPath, + byte[] templateIndex, + Map parameters + ); + /** * Load an index into memory * @@ -249,6 +268,16 @@ public static native KNNQueryResult[] queryBinaryIndexWithFilter( */ public static native byte[] trainIndex(Map indexParameters, int dimension, long trainVectorsPointer); + /** + * Train an empty binary index + * + * @param indexParameters parameters used to build index + * @param dimension dimension for the index + * @param trainVectorsPointer pointer to where training vectors are stored in native memory + * @return bytes array of trained template index + */ + public static native byte[] trainBinaryIndex(Map indexParameters, int dimension, long trainVectorsPointer); + /** *

* The function is deprecated. Use {@link JNICommons#storeVectorData(long, float[][], long)} diff --git a/src/main/java/org/opensearch/knn/jni/JNICommons.java b/src/main/java/org/opensearch/knn/jni/JNICommons.java index d0111b115..31a8f43cc 100644 --- a/src/main/java/org/opensearch/knn/jni/JNICommons.java +++ b/src/main/java/org/opensearch/knn/jni/JNICommons.java @@ -78,4 +78,17 @@ public class JNICommons { * @param memoryAddress address to be freed. */ public static native void freeVectorData(long memoryAddress); + + /** + * Free up the memory allocated for the byte data stored in memory address. This function should be used with the memory + * address returned by {@link JNICommons#storeVectorData(long, float[][], long)} + * + *

+ * The function is not threadsafe. If multiple threads are trying to free up same memory location, then it can + * lead to errors. + *

+ * + * @param memoryAddress address to be freed. + */ + public static native void freeByteVectorData(long memoryAddress); } diff --git a/src/main/java/org/opensearch/knn/jni/JNIService.java b/src/main/java/org/opensearch/knn/jni/JNIService.java index cefd0af53..2a8d3ea8f 100644 --- a/src/main/java/org/opensearch/knn/jni/JNIService.java +++ b/src/main/java/org/opensearch/knn/jni/JNIService.java @@ -83,8 +83,13 @@ public static void createIndexFromTemplate( KNNEngine knnEngine ) { if (KNNEngine.FAISS == knnEngine) { - FaissService.createIndexFromTemplate(ids, vectorsAddress, dim, indexPath, templateIndex, parameters); - return; + if (IndexUtil.isBinaryIndex(knnEngine, parameters)) { + FaissService.createBinaryIndexFromTemplate(ids, vectorsAddress, dim, indexPath, templateIndex, parameters); + return; + } else { + FaissService.createIndexFromTemplate(ids, vectorsAddress, dim, indexPath, templateIndex, parameters); + return; + } } throw new IllegalArgumentException( @@ -308,6 +313,9 @@ public static void freeSharedIndexState(long shareIndexStateAddr, KNNEngine knnE */ public static byte[] trainIndex(Map indexParameters, int dimension, long trainVectorsPointer, KNNEngine knnEngine) { if (KNNEngine.FAISS == knnEngine) { + if (IndexUtil.isBinaryIndex(knnEngine, indexParameters)) { + return FaissService.trainBinaryIndex(indexParameters, dimension, trainVectorsPointer); + } return FaissService.trainIndex(indexParameters, dimension, trainVectorsPointer); } diff --git a/src/main/java/org/opensearch/knn/plugin/rest/RestTrainModelHandler.java b/src/main/java/org/opensearch/knn/plugin/rest/RestTrainModelHandler.java index fb8ccc4ce..e0b94ec76 100644 --- a/src/main/java/org/opensearch/knn/plugin/rest/RestTrainModelHandler.java +++ b/src/main/java/org/opensearch/knn/plugin/rest/RestTrainModelHandler.java @@ -17,6 +17,7 @@ import org.opensearch.index.mapper.NumberFieldMapper; import org.opensearch.knn.index.KNNMethodContext; import org.opensearch.knn.index.SpaceType; +import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.indices.ModelUtil; import org.opensearch.knn.plugin.KNNPlugin; import org.opensearch.knn.plugin.transport.TrainingJobRouterAction; @@ -40,6 +41,7 @@ import static org.opensearch.knn.common.KNNConstants.SEARCH_SIZE_PARAMETER; import static org.opensearch.knn.common.KNNConstants.TRAIN_FIELD_PARAMETER; import static org.opensearch.knn.common.KNNConstants.TRAIN_INDEX_PARAMETER; +import static org.opensearch.knn.common.KNNConstants.VECTOR_DATA_TYPE_FIELD; /** * Rest Handler for model training api endpoint. @@ -83,6 +85,7 @@ private TrainingModelRequest createTransportRequest(RestRequest restRequest) thr String trainingIndex = (String) DEFAULT_NOT_SET_OBJECT_VALUE; String trainingField = (String) DEFAULT_NOT_SET_OBJECT_VALUE; String description = (String) DEFAULT_NOT_SET_OBJECT_VALUE; + VectorDataType vectorDataType = (VectorDataType) DEFAULT_NOT_SET_OBJECT_VALUE; int dimension = DEFAULT_NOT_SET_INT_VALUE; int maximumVectorCount = DEFAULT_NOT_SET_INT_VALUE; @@ -110,6 +113,8 @@ private TrainingModelRequest createTransportRequest(RestRequest restRequest) thr } else if (MODEL_DESCRIPTION.equals(fieldName) && ensureNotSet(fieldName, description)) { description = parser.textOrNull(); ModelUtil.blockCommasInModelDescription(description); + } else if (VECTOR_DATA_TYPE_FIELD.equals(fieldName) && ensureNotSet(fieldName, vectorDataType)) { + vectorDataType = VectorDataType.get(parser.text()); } else { throw new IllegalArgumentException("Unable to parse token. \"" + fieldName + "\" is not a valid " + "parameter."); } @@ -126,6 +131,10 @@ private TrainingModelRequest createTransportRequest(RestRequest restRequest) thr description = ""; } + if (vectorDataType == DEFAULT_NOT_SET_OBJECT_VALUE) { + vectorDataType = VectorDataType.DEFAULT; + } + TrainingModelRequest trainingModelRequest = new TrainingModelRequest( modelId, knnMethodContext, @@ -133,7 +142,8 @@ private TrainingModelRequest createTransportRequest(RestRequest restRequest) thr trainingIndex, trainingField, preferredNodeId, - description + description, + vectorDataType ); if (maximumVectorCount != DEFAULT_NOT_SET_INT_VALUE) { diff --git a/src/main/java/org/opensearch/knn/plugin/transport/TrainingJobRouterTransportAction.java b/src/main/java/org/opensearch/knn/plugin/transport/TrainingJobRouterTransportAction.java index 9b2d3a9de..78f3769c5 100644 --- a/src/main/java/org/opensearch/knn/plugin/transport/TrainingJobRouterTransportAction.java +++ b/src/main/java/org/opensearch/knn/plugin/transport/TrainingJobRouterTransportAction.java @@ -22,6 +22,7 @@ import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.ValidationException; import org.opensearch.common.inject.Inject; +import org.opensearch.knn.index.VectorDataType; import org.opensearch.search.builder.SearchSourceBuilder; import org.opensearch.tasks.Task; import org.opensearch.transport.TransportRequestOptions; @@ -133,7 +134,9 @@ protected void getTrainingIndexSizeInKB(TrainingModelRequest trainingModelReques trainingVectors = trainingModelRequest.getMaximumVectorCount(); } - listener.onResponse(estimateVectorSetSizeInKB(trainingVectors, trainingModelRequest.getDimension())); + listener.onResponse( + estimateVectorSetSizeInKB(trainingVectors, trainingModelRequest.getDimension(), trainingModelRequest.getVectorDataType()) + ); }, listener::onFailure)); } @@ -144,8 +147,14 @@ protected void getTrainingIndexSizeInKB(TrainingModelRequest trainingModelReques * @param dimension dimension of vectors * @return size estimate */ - public static int estimateVectorSetSizeInKB(long vectorCount, int dimension) { - // Ensure we do not overflow the int on estimate - return Math.toIntExact(((Float.BYTES * dimension * vectorCount) / BYTES_PER_KILOBYTES) + 1L); + public static int estimateVectorSetSizeInKB(long vectorCount, int dimension, VectorDataType vectorDataType) { + switch (vectorDataType) { + case BINARY: + return Math.toIntExact(((Byte.BYTES * (dimension / 8) * vectorCount) / BYTES_PER_KILOBYTES) + 1L); + case BYTE: + return Math.toIntExact(((Byte.BYTES * dimension * vectorCount) / BYTES_PER_KILOBYTES) + 1L); + default: + return Math.toIntExact(((Float.BYTES * dimension * vectorCount) / BYTES_PER_KILOBYTES) + 1L); + } } } diff --git a/src/main/java/org/opensearch/knn/plugin/transport/TrainingModelRequest.java b/src/main/java/org/opensearch/knn/plugin/transport/TrainingModelRequest.java index 5f3913ac5..16a1a103a 100644 --- a/src/main/java/org/opensearch/knn/plugin/transport/TrainingModelRequest.java +++ b/src/main/java/org/opensearch/knn/plugin/transport/TrainingModelRequest.java @@ -21,6 +21,7 @@ import org.opensearch.knn.common.KNNConstants; import org.opensearch.knn.index.IndexUtil; import org.opensearch.knn.index.KNNMethodContext; +import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.indices.ModelDao; import org.opensearch.knn.training.VectorSpaceInfo; @@ -41,6 +42,7 @@ public class TrainingModelRequest extends ActionRequest { private final String trainingField; private final String preferredNodeId; private final String description; + private final VectorDataType vectorDataType; private int maximumVectorCount; private int searchSize; @@ -65,7 +67,8 @@ public TrainingModelRequest( String trainingIndex, String trainingField, String preferredNodeId, - String description + String description, + VectorDataType vectorDataType ) { super(); this.modelId = modelId; @@ -75,6 +78,7 @@ public TrainingModelRequest( this.trainingField = trainingField; this.preferredNodeId = preferredNodeId; this.description = description; + this.vectorDataType = vectorDataType; // Set these as defaults initially. If call wants to override them, they can use the setters. this.maximumVectorCount = Integer.MAX_VALUE; // By default, get all vectors in the index @@ -103,6 +107,11 @@ public TrainingModelRequest(StreamInput in) throws IOException { this.maximumVectorCount = in.readInt(); this.searchSize = in.readInt(); this.trainingDataSizeInKB = in.readInt(); + if (IndexUtil.isVersionOnOrAfterMinRequiredVersion(in.getVersion(), KNNConstants.MODEL_VECTOR_DATA_TYPE_KEY)) { + this.vectorDataType = VectorDataType.get(in.readString()); + } else { + this.vectorDataType = VectorDataType.DEFAULT; + } } /** @@ -213,6 +222,10 @@ public int getSearchSize() { return searchSize; } + public VectorDataType getVectorDataType() { + return vectorDataType; + } + /** * Setter for search size. * @@ -314,7 +327,13 @@ public ActionRequestValidationException validate() { } // Validate the training field - ValidationException fieldValidation = IndexUtil.validateKnnField(indexMetadata, this.trainingField, this.dimension, modelDao); + ValidationException fieldValidation = IndexUtil.validateKnnField( + indexMetadata, + this.trainingField, + this.dimension, + modelDao, + this.vectorDataType + ); if (fieldValidation != null) { exception = exception == null ? new ActionRequestValidationException() : exception; exception.addValidationErrors(fieldValidation.validationErrors()); @@ -336,5 +355,10 @@ public void writeTo(StreamOutput out) throws IOException { out.writeInt(this.maximumVectorCount); out.writeInt(this.searchSize); out.writeInt(this.trainingDataSizeInKB); + if (IndexUtil.isVersionOnOrAfterMinRequiredVersion(out.getVersion(), KNNConstants.MODEL_VECTOR_DATA_TYPE_KEY)) { + out.writeString(this.vectorDataType.getValue()); + } else { + out.writeString(VectorDataType.DEFAULT.getValue()); + } } } diff --git a/src/main/java/org/opensearch/knn/plugin/transport/TrainingModelTransportAction.java b/src/main/java/org/opensearch/knn/plugin/transport/TrainingModelTransportAction.java index 33b420e2c..a9eca609d 100644 --- a/src/main/java/org/opensearch/knn/plugin/transport/TrainingModelTransportAction.java +++ b/src/main/java/org/opensearch/knn/plugin/transport/TrainingModelTransportAction.java @@ -51,7 +51,8 @@ protected void doExecute(Task task, TrainingModelRequest request, ActionListener NativeMemoryLoadStrategy.TrainingLoadStrategy.getInstance(), clusterService, request.getMaximumVectorCount(), - request.getSearchSize() + request.getSearchSize(), + request.getVectorDataType() ); // Allocation representing size model will occupy in memory during training @@ -68,7 +69,8 @@ protected void doExecute(Task task, TrainingModelRequest request, ActionListener modelAnonymousEntryContext, request.getDimension(), request.getDescription(), - clusterService.localNode().getEphemeralId() + clusterService.localNode().getEphemeralId(), + request.getVectorDataType() ); KNNCounter.TRAINING_REQUESTS.increment(); diff --git a/src/main/java/org/opensearch/knn/training/ByteTrainingDataConsumer.java b/src/main/java/org/opensearch/knn/training/ByteTrainingDataConsumer.java new file mode 100644 index 000000000..70cfb4f4c --- /dev/null +++ b/src/main/java/org/opensearch/knn/training/ByteTrainingDataConsumer.java @@ -0,0 +1,81 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.knn.training; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.action.search.SearchResponse; +import org.opensearch.knn.jni.JNICommons; +import org.opensearch.knn.index.memory.NativeMemoryAllocation; +import org.opensearch.search.SearchHit; + +import java.util.ArrayList; +import java.util.List; + +/** + * Transfers byte vectors from JVM to native memory. + */ +public class ByteTrainingDataConsumer extends TrainingDataConsumer { + private static final Logger logger = LogManager.getLogger(TrainingDataConsumer.class); + + /** + * Constructor + * + * @param trainingDataAllocation NativeMemoryAllocation that contains information about native memory allocation. + */ + public ByteTrainingDataConsumer(NativeMemoryAllocation.TrainingDataAllocation trainingDataAllocation) { + super(trainingDataAllocation); + } + + @Override + public void accept(List byteVectors) { + long memoryAddress = trainingDataAllocation.getMemoryAddress(); + memoryAddress = JNICommons.storeByteVectorData(memoryAddress, byteVectors.toArray(new byte[0][0]), byteVectors.size()); + trainingDataAllocation.setMemoryAddress(memoryAddress); + } + + @Override + public void processTrainingVectors(SearchResponse searchResponse, int vectorsToAdd, String fieldName) { + SearchHit[] hits = searchResponse.getHits().getHits(); + List vectors = new ArrayList<>(); + String[] fieldPath = fieldName.split("\\."); + int nullVectorCount = 0; + + for (int vector = 0; vector < vectorsToAdd; vector++) { + Object fieldValue = extractFieldValue(hits[vector], fieldPath); + if (fieldValue == null) { + nullVectorCount++; + continue; + } + + byte[] byteArray; + if (!(fieldValue instanceof List)) { + continue; + } + List fieldList = (List) fieldValue; + byteArray = new byte[fieldList.size()]; + for (int i = 0; i < fieldList.size(); i++) { + byteArray[i] = fieldList.get(i).byteValue(); + } + + vectors.add(byteArray); + } + + if (nullVectorCount > 0) { + logger.warn("Found {} documents with null byte vectors in field {}", nullVectorCount, fieldName); + } + + setTotalVectorsCountAdded(getTotalVectorsCountAdded() + vectors.size()); + + accept(vectors); + } +} diff --git a/src/main/java/org/opensearch/knn/training/FloatTrainingDataConsumer.java b/src/main/java/org/opensearch/knn/training/FloatTrainingDataConsumer.java new file mode 100644 index 000000000..d742a9184 --- /dev/null +++ b/src/main/java/org/opensearch/knn/training/FloatTrainingDataConsumer.java @@ -0,0 +1,67 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.knn.training; + +import org.apache.commons.lang.ArrayUtils; +import org.opensearch.action.search.SearchResponse; +import org.opensearch.knn.jni.JNIService; +import org.opensearch.knn.index.memory.NativeMemoryAllocation; +import org.opensearch.search.SearchHit; + +import java.util.ArrayList; +import java.util.List; + +/** + * Transfers float vectors from JVM to native memory. + */ +public class FloatTrainingDataConsumer extends TrainingDataConsumer { + + /** + * Constructor + * + * @param trainingDataAllocation NativeMemoryAllocation that contains information about native memory allocation. + */ + public FloatTrainingDataConsumer(NativeMemoryAllocation.TrainingDataAllocation trainingDataAllocation) { + super(trainingDataAllocation); + } + + @Override + public void accept(List floats) { + trainingDataAllocation.setMemoryAddress( + JNIService.transferVectors( + trainingDataAllocation.getMemoryAddress(), + floats.stream().map(v -> ArrayUtils.toPrimitive((Float[]) v)).toArray(float[][]::new) + ) + ); + } + + @Override + public void processTrainingVectors(SearchResponse searchResponse, int vectorsToAdd, String fieldName) { + SearchHit[] hits = searchResponse.getHits().getHits(); + List vectors = new ArrayList<>(); + String[] fieldPath = fieldName.split("\\."); + + for (int vector = 0; vector < vectorsToAdd; vector++) { + Object fieldValue = extractFieldValue(hits[vector], fieldPath); + if (!(fieldValue instanceof List)) { + continue; + } + + List fieldList = (List) fieldValue; + vectors.add(fieldList.stream().map(Number::floatValue).toArray(Float[]::new)); + } + + setTotalVectorsCountAdded(getTotalVectorsCountAdded() + vectors.size()); + + accept(vectors); + } +} diff --git a/src/main/java/org/opensearch/knn/training/TrainingDataConsumer.java b/src/main/java/org/opensearch/knn/training/TrainingDataConsumer.java index 6732bd3f4..9d0683fdc 100644 --- a/src/main/java/org/opensearch/knn/training/TrainingDataConsumer.java +++ b/src/main/java/org/opensearch/knn/training/TrainingDataConsumer.java @@ -11,19 +11,25 @@ package org.opensearch.knn.training; -import org.apache.commons.lang.ArrayUtils; -import org.opensearch.knn.jni.JNIService; +import lombok.Getter; +import lombok.Setter; +import org.opensearch.action.search.SearchResponse; import org.opensearch.knn.index.memory.NativeMemoryAllocation; +import org.opensearch.search.SearchHit; import java.util.List; -import java.util.function.Consumer; +import java.util.Map; /** - * Transfers vectors from JVM to native memory. + * TrainingDataConsumer is an abstract class that defines the interface for consuming training data. + * It is used to process training data and add it to the training data allocation. */ -public class TrainingDataConsumer implements Consumer> { +public abstract class TrainingDataConsumer { - private final NativeMemoryAllocation.TrainingDataAllocation trainingDataAllocation; + @Setter + @Getter + private int totalVectorsCountAdded = 0; + protected final NativeMemoryAllocation.TrainingDataAllocation trainingDataAllocation; /** * Constructor @@ -34,13 +40,25 @@ public TrainingDataConsumer(NativeMemoryAllocation.TrainingDataAllocation traini this.trainingDataAllocation = trainingDataAllocation; } - @Override - public void accept(List floats) { - trainingDataAllocation.setMemoryAddress( - JNIService.transferVectors( - trainingDataAllocation.getMemoryAddress(), - floats.stream().map(ArrayUtils::toPrimitive).toArray(float[][]::new) - ) - ); + protected abstract void accept(List vectors); + + public abstract void processTrainingVectors(SearchResponse searchResponse, int vectorsToAdd, String fieldName); + + /** + * Traverses the hit to the desired field and extracts its value. + * + * @param hit The search hit to extract the field value from + * @param fieldPath The path to the desired field + * @return The extracted field value, or null if the field does not exist + */ + protected Object extractFieldValue(SearchHit hit, String[] fieldPath) { + Map currentMap = hit.getSourceAsMap(); + for (int pathPart = 0; pathPart < fieldPath.length - 1; pathPart++) { + currentMap = (Map) currentMap.get(fieldPath[pathPart]); + if (currentMap == null) { + return null; + } + } + return currentMap.get(fieldPath[fieldPath.length - 1]); } } diff --git a/src/main/java/org/opensearch/knn/training/TrainingJob.java b/src/main/java/org/opensearch/knn/training/TrainingJob.java index aa2786c0a..928396289 100644 --- a/src/main/java/org/opensearch/knn/training/TrainingJob.java +++ b/src/main/java/org/opensearch/knn/training/TrainingJob.java @@ -16,7 +16,9 @@ import org.apache.logging.log4j.Logger; import org.opensearch.common.UUIDs; import org.opensearch.knn.common.KNNConstants; +import org.opensearch.knn.index.IndexUtil; import org.opensearch.knn.index.KNNSettings; +import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.jni.JNIService; import org.opensearch.knn.index.KNNMethodContext; import org.opensearch.knn.index.memory.NativeMemoryAllocation; @@ -32,6 +34,8 @@ import java.util.Map; import java.util.Objects; +import static org.opensearch.knn.index.util.Faiss.FAISS_BINARY_INDEX_DESCRIPTION_PREFIX; + /** * Encapsulates all information required to generate and train a model. */ @@ -66,7 +70,8 @@ public TrainingJob( NativeMemoryEntryContext.AnonymousEntryContext modelAnonymousEntryContext, int dimension, String description, - String nodeAssignment + String nodeAssignment, + VectorDataType vectorDataType ) { // Generate random base64 string if one is not provided this.modelId = StringUtils.isNotBlank(modelId) ? modelId : UUIDs.randomBase64UUID(); @@ -84,7 +89,8 @@ public TrainingJob( description, "", nodeAssignment, - knnMethodContext.getMethodComponentContext() + knnMethodContext.getMethodComponentContext(), + vectorDataType ), null, this.modelId @@ -182,6 +188,15 @@ public void run() { KNNSettings.state().getSettingValue(KNNSettings.KNN_ALGO_PARAM_INDEX_THREAD_QTY) ); + if (VectorDataType.BINARY == model.getModelMetadata().getVectorDataType()) { + trainParameters.put( + KNNConstants.INDEX_DESCRIPTION_PARAMETER, + FAISS_BINARY_INDEX_DESCRIPTION_PREFIX + trainParameters.get(KNNConstants.INDEX_DESCRIPTION_PARAMETER).toString() + ); + } + + IndexUtil.updateVectorDataTypeToParameters(trainParameters, model.getModelMetadata().getVectorDataType()); + byte[] modelBlob = JNIService.trainIndex( trainParameters, model.getModelMetadata().getDimension(), diff --git a/src/main/java/org/opensearch/knn/training/VectorReader.java b/src/main/java/org/opensearch/knn/training/VectorReader.java index aeebae129..f1fd744fd 100644 --- a/src/main/java/org/opensearch/knn/training/VectorReader.java +++ b/src/main/java/org/opensearch/knn/training/VectorReader.java @@ -30,7 +30,6 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; -import java.util.function.Consumer; public class VectorReader { @@ -59,13 +58,13 @@ public VectorReader(Client client) { * @param vectorConsumer consumer used to do something with the collected vectors after each search * @param listener ActionListener that should be called once all search operations complete */ - public void read( + public void read( ClusterService clusterService, String indexName, String fieldName, int maxVectorCount, int searchSize, - Consumer> vectorConsumer, + TrainingDataConsumer vectorConsumer, ActionListener listener ) { @@ -89,7 +88,7 @@ public void read( throw validationException; } - ValidationException fieldValidationException = IndexUtil.validateKnnField(indexMetadata, fieldName, -1, null); + ValidationException fieldValidationException = IndexUtil.validateKnnField(indexMetadata, fieldName, -1, null, null); if (fieldValidationException != null) { validationException = validationException == null ? new ValidationException() : validationException; validationException.addValidationErrors(validationException.validationErrors()); @@ -136,14 +135,14 @@ private SearchScrollRequestBuilder createSearchScrollRequestBuilder() { return searchScrollRequestBuilder; } - private static class VectorReaderListener implements ActionListener { + private static class VectorReaderListener implements ActionListener { final Client client; final String fieldName; final int maxVectorCount; int collectedVectorCount; final ActionListener listener; - final Consumer> vectorConsumer; + final TrainingDataConsumer vectorConsumer; SearchScrollRequestBuilder searchScrollRequestBuilder; /** @@ -162,7 +161,7 @@ public VectorReaderListener( int maxVectorCount, int collectedVectorCount, ActionListener listener, - Consumer> vectorConsumer, + TrainingDataConsumer vectorConsumer, SearchScrollRequestBuilder searchScrollRequestBuilder ) { this.client = client; @@ -181,12 +180,9 @@ public void onResponse(SearchResponse searchResponse) { // Either add the entire set of returned hits, or maxVectorCount - collectedVectorCount hits SearchHit[] hits = searchResponse.getHits().getHits(); int vectorsToAdd = Integer.min(maxVectorCount - collectedVectorCount, hits.length); - List trainingData = extractVectorsFromHits(searchResponse, vectorsToAdd); - this.collectedVectorCount += trainingData.size(); - - // Do something with the vectors - vectorConsumer.accept(trainingData); + vectorConsumer.processTrainingVectors(searchResponse, vectorsToAdd, fieldName); + this.collectedVectorCount = vectorConsumer.getTotalVectorsCountAdded(); if (vectorsToAdd <= 0 || this.collectedVectorCount >= maxVectorCount) { // Clear scroll context diff --git a/src/test/java/org/opensearch/knn/KNNSingleNodeTestCase.java b/src/test/java/org/opensearch/knn/KNNSingleNodeTestCase.java index f9c0161d6..06431bf07 100644 --- a/src/test/java/org/opensearch/knn/KNNSingleNodeTestCase.java +++ b/src/test/java/org/opensearch/knn/KNNSingleNodeTestCase.java @@ -46,8 +46,17 @@ import java.util.concurrent.ExecutionException; import static org.mockito.Mockito.when; -import static org.opensearch.knn.common.KNNConstants.*; +import static org.opensearch.knn.common.KNNConstants.DIMENSION; +import static org.opensearch.knn.common.KNNConstants.KNN_ENGINE; +import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_SPACE_TYPE; +import static org.opensearch.knn.common.KNNConstants.MODEL_BLOB_PARAMETER; +import static org.opensearch.knn.common.KNNConstants.MODEL_DESCRIPTION; +import static org.opensearch.knn.common.KNNConstants.MODEL_ERROR; +import static org.opensearch.knn.common.KNNConstants.MODEL_ID; import static org.opensearch.knn.common.KNNConstants.MODEL_INDEX_NAME; +import static org.opensearch.knn.common.KNNConstants.MODEL_STATE; +import static org.opensearch.knn.common.KNNConstants.MODEL_TIMESTAMP; +import static org.opensearch.knn.common.KNNConstants.VECTOR_DATA_TYPE_FIELD; public class KNNSingleNodeTestCase extends OpenSearchSingleNodeTestCase { @Override @@ -201,7 +210,8 @@ protected void writeModelToModelSystemIndex(Model model) throws IOException, Exe .field(MODEL_STATE, modelMetadata.getState().getName()) .field(MODEL_TIMESTAMP, modelMetadata.getTimestamp().toString()) .field(MODEL_DESCRIPTION, modelMetadata.getDescription()) - .field(MODEL_ERROR, modelMetadata.getError()); + .field(MODEL_ERROR, modelMetadata.getError()) + .field(VECTOR_DATA_TYPE_FIELD, modelMetadata.getVectorDataType().getValue()); if (model.getModelBlob() != null) { builder.field(MODEL_BLOB_PARAMETER, Base64.getEncoder().encodeToString(model.getModelBlob())); diff --git a/src/test/java/org/opensearch/knn/index/FaissIT.java b/src/test/java/org/opensearch/knn/index/FaissIT.java index 204d9f468..4258f309e 100644 --- a/src/test/java/org/opensearch/knn/index/FaissIT.java +++ b/src/test/java/org/opensearch/knn/index/FaissIT.java @@ -44,6 +44,7 @@ import java.util.TreeMap; import java.util.stream.Collectors; +import static org.opensearch.knn.common.KNNConstants.DIMENSION; import static org.opensearch.knn.common.KNNConstants.ENCODER_PARAMETER_PQ_CODE_SIZE; import static org.opensearch.knn.common.KNNConstants.ENCODER_PARAMETER_PQ_M; import static org.opensearch.knn.common.KNNConstants.ENCODER_PQ; @@ -55,18 +56,25 @@ import static org.opensearch.knn.common.KNNConstants.FP16_MAX_VALUE; import static org.opensearch.knn.common.KNNConstants.FP16_MIN_VALUE; import static org.opensearch.knn.common.KNNConstants.KNN_ENGINE; +import static org.opensearch.knn.common.KNNConstants.KNN_METHOD; import static org.opensearch.knn.common.KNNConstants.MAX_DISTANCE; import static org.opensearch.knn.common.KNNConstants.METHOD_ENCODER_PARAMETER; import static org.opensearch.knn.common.KNNConstants.METHOD_HNSW; import static org.opensearch.knn.common.KNNConstants.METHOD_IVF; import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER; +import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_EF_CONSTRUCTION; +import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_M; import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_NLIST; import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_NPROBES; import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_SPACE_TYPE; import static org.opensearch.knn.common.KNNConstants.MIN_SCORE; +import static org.opensearch.knn.common.KNNConstants.MODEL_DESCRIPTION; import static org.opensearch.knn.common.KNNConstants.MODEL_ID; import static org.opensearch.knn.common.KNNConstants.NAME; import static org.opensearch.knn.common.KNNConstants.PARAMETERS; +import static org.opensearch.knn.common.KNNConstants.TRAIN_FIELD_PARAMETER; +import static org.opensearch.knn.common.KNNConstants.TRAIN_INDEX_PARAMETER; +import static org.opensearch.knn.common.KNNConstants.VECTOR_DATA_TYPE_FIELD; public class FaissIT extends KNNRestTestCase { private static final String DOC_ID_1 = "doc1"; @@ -107,13 +115,13 @@ public void testEndToEnd_whenDoRadiusSearch_whenDistanceThreshold_whenMethodIsHN .startObject(FIELD_NAME) .field("type", "knn_vector") .field("dimension", dimension) - .startObject(KNNConstants.KNN_METHOD) + .startObject(KNN_METHOD) .field(NAME, hnswMethod.getMethodComponent().getName()) .field(METHOD_PARAMETER_SPACE_TYPE, spaceType.getValue()) .field(KNN_ENGINE, KNNEngine.FAISS.getName()) .startObject(PARAMETERS) - .field(KNNConstants.METHOD_PARAMETER_M, mValues.get(random().nextInt(mValues.size()))) - .field(KNNConstants.METHOD_PARAMETER_EF_CONSTRUCTION, efConstructionValues.get(random().nextInt(efConstructionValues.size()))) + .field(METHOD_PARAMETER_M, mValues.get(random().nextInt(mValues.size()))) + .field(METHOD_PARAMETER_EF_CONSTRUCTION, efConstructionValues.get(random().nextInt(efConstructionValues.size()))) .field(KNNConstants.METHOD_PARAMETER_EF_SEARCH, efSearchValues.get(random().nextInt(efSearchValues.size()))) .endObject() .endObject() @@ -166,13 +174,13 @@ public void testEndToEnd_whenDoRadiusSearch_whenScoreThreshold_whenMethodIsHNSWF .startObject(FIELD_NAME) .field("type", "knn_vector") .field("dimension", dimension) - .startObject(KNNConstants.KNN_METHOD) + .startObject(KNN_METHOD) .field(NAME, hnswMethod.getMethodComponent().getName()) .field(METHOD_PARAMETER_SPACE_TYPE, spaceType.getValue()) .field(KNN_ENGINE, KNNEngine.FAISS.getName()) .startObject(PARAMETERS) - .field(KNNConstants.METHOD_PARAMETER_M, mValues.get(random().nextInt(mValues.size()))) - .field(KNNConstants.METHOD_PARAMETER_EF_CONSTRUCTION, efConstructionValues.get(random().nextInt(efConstructionValues.size()))) + .field(METHOD_PARAMETER_M, mValues.get(random().nextInt(mValues.size()))) + .field(METHOD_PARAMETER_EF_CONSTRUCTION, efConstructionValues.get(random().nextInt(efConstructionValues.size()))) .field(KNNConstants.METHOD_PARAMETER_EF_SEARCH, efSearchValues.get(random().nextInt(efSearchValues.size()))) .endObject() .endObject() @@ -226,13 +234,13 @@ public void testEndToEnd_whenDoRadiusSearch_whenMoreThanOneScoreThreshold_whenMe .startObject(FIELD_NAME) .field("type", "knn_vector") .field("dimension", dimension) - .startObject(KNNConstants.KNN_METHOD) + .startObject(KNN_METHOD) .field(NAME, hnswMethod.getMethodComponent().getName()) .field(METHOD_PARAMETER_SPACE_TYPE, spaceType.getValue()) .field(KNN_ENGINE, KNNEngine.FAISS.getName()) .startObject(PARAMETERS) - .field(KNNConstants.METHOD_PARAMETER_M, mValues.get(random().nextInt(mValues.size()))) - .field(KNNConstants.METHOD_PARAMETER_EF_CONSTRUCTION, efConstructionValues.get(random().nextInt(efConstructionValues.size()))) + .field(METHOD_PARAMETER_M, mValues.get(random().nextInt(mValues.size()))) + .field(METHOD_PARAMETER_EF_CONSTRUCTION, efConstructionValues.get(random().nextInt(efConstructionValues.size()))) .field(KNNConstants.METHOD_PARAMETER_EF_SEARCH, efSearchValues.get(random().nextInt(efSearchValues.size()))) .endObject() .endObject() @@ -296,8 +304,8 @@ public void testEndToEnd_whenDoRadiusSearch_whenDistanceThreshold_whenMethodIsHN .field(NAME, METHOD_HNSW) .field(KNN_ENGINE, FAISS_NAME) .startObject(PARAMETERS) - .field(KNNConstants.METHOD_PARAMETER_M, mValues.get(random().nextInt(mValues.size()))) - .field(KNNConstants.METHOD_PARAMETER_EF_CONSTRUCTION, efConstructionValues.get(random().nextInt(efConstructionValues.size()))) + .field(METHOD_PARAMETER_M, mValues.get(random().nextInt(mValues.size()))) + .field(METHOD_PARAMETER_EF_CONSTRUCTION, efConstructionValues.get(random().nextInt(efConstructionValues.size()))) .field(KNNConstants.METHOD_PARAMETER_EF_SEARCH, efSearchValues.get(random().nextInt(efSearchValues.size()))) .startObject(METHOD_ENCODER_PARAMETER) .field(NAME, ENCODER_PQ) @@ -424,8 +432,8 @@ public void testEndToEnd_whenMethodIsHNSWPQ_thenSucceed() { .field(NAME, METHOD_HNSW) .field(KNN_ENGINE, FAISS_NAME) .startObject(PARAMETERS) - .field(KNNConstants.METHOD_PARAMETER_M, mValues.get(random().nextInt(mValues.size()))) - .field(KNNConstants.METHOD_PARAMETER_EF_CONSTRUCTION, efConstructionValues.get(random().nextInt(efConstructionValues.size()))) + .field(METHOD_PARAMETER_M, mValues.get(random().nextInt(mValues.size()))) + .field(METHOD_PARAMETER_EF_CONSTRUCTION, efConstructionValues.get(random().nextInt(efConstructionValues.size()))) .field(KNNConstants.METHOD_PARAMETER_EF_SEARCH, efSearchValues.get(random().nextInt(efSearchValues.size()))) .startObject(METHOD_ENCODER_PARAMETER) .field(NAME, ENCODER_PQ) @@ -531,13 +539,13 @@ public void testHNSWSQFP16_whenIndexedAndQueried_thenSucceed() { .startObject(fieldName) .field("type", "knn_vector") .field("dimension", dimension) - .startObject(KNNConstants.KNN_METHOD) + .startObject(KNN_METHOD) .field(NAME, hnswMethod.getMethodComponent().getName()) .field(METHOD_PARAMETER_SPACE_TYPE, spaceType.getValue()) .field(KNN_ENGINE, KNNEngine.FAISS.getName()) .startObject(PARAMETERS) - .field(KNNConstants.METHOD_PARAMETER_M, mValues.get(random().nextInt(mValues.size()))) - .field(KNNConstants.METHOD_PARAMETER_EF_CONSTRUCTION, efConstructionValues.get(random().nextInt(efConstructionValues.size()))) + .field(METHOD_PARAMETER_M, mValues.get(random().nextInt(mValues.size()))) + .field(METHOD_PARAMETER_EF_CONSTRUCTION, efConstructionValues.get(random().nextInt(efConstructionValues.size()))) .field(KNNConstants.METHOD_PARAMETER_EF_SEARCH, efSearchValues.get(random().nextInt(efSearchValues.size()))) .startObject(METHOD_ENCODER_PARAMETER) .field(NAME, ENCODER_SQ) @@ -645,13 +653,13 @@ public void testHNSWSQFP16_whenIndexedWithOutOfFP16Range_thenThrowException() { .startObject(fieldName) .field("type", "knn_vector") .field("dimension", dimension) - .startObject(KNNConstants.KNN_METHOD) + .startObject(KNN_METHOD) .field(NAME, hnswMethod.getMethodComponent().getName()) .field(METHOD_PARAMETER_SPACE_TYPE, spaceType.getValue()) .field(KNN_ENGINE, KNNEngine.FAISS.getName()) .startObject(PARAMETERS) - .field(KNNConstants.METHOD_PARAMETER_M, mValues.get(random().nextInt(mValues.size()))) - .field(KNNConstants.METHOD_PARAMETER_EF_CONSTRUCTION, efConstructionValues.get(random().nextInt(efConstructionValues.size()))) + .field(METHOD_PARAMETER_M, mValues.get(random().nextInt(mValues.size()))) + .field(METHOD_PARAMETER_EF_CONSTRUCTION, efConstructionValues.get(random().nextInt(efConstructionValues.size()))) .field(KNNConstants.METHOD_PARAMETER_EF_SEARCH, efSearchValues.get(random().nextInt(efSearchValues.size()))) .startObject(METHOD_ENCODER_PARAMETER) .field(NAME, ENCODER_SQ) @@ -745,13 +753,13 @@ public void testHNSWSQFP16_whenClipToFp16isTrueAndIndexedWithOutOfFP16Range_then .startObject(fieldName) .field("type", "knn_vector") .field("dimension", dimension) - .startObject(KNNConstants.KNN_METHOD) + .startObject(KNN_METHOD) .field(NAME, hnswMethod.getMethodComponent().getName()) .field(METHOD_PARAMETER_SPACE_TYPE, SpaceType.L2.getValue()) .field(KNN_ENGINE, KNNEngine.FAISS.getName()) .startObject(PARAMETERS) - .field(KNNConstants.METHOD_PARAMETER_M, mValues.get(random().nextInt(mValues.size()))) - .field(KNNConstants.METHOD_PARAMETER_EF_CONSTRUCTION, efConstructionValues.get(random().nextInt(efConstructionValues.size()))) + .field(METHOD_PARAMETER_M, mValues.get(random().nextInt(mValues.size()))) + .field(METHOD_PARAMETER_EF_CONSTRUCTION, efConstructionValues.get(random().nextInt(efConstructionValues.size()))) .field(KNNConstants.METHOD_PARAMETER_EF_SEARCH, efSearchValues.get(random().nextInt(efSearchValues.size()))) .startObject(METHOD_ENCODER_PARAMETER) .field(NAME, ENCODER_SQ) @@ -998,7 +1006,7 @@ public void testEndToEnd_whenMethodIsHNSWPQAndHyperParametersNotSet_thenSucceed( .field(NAME, METHOD_HNSW) .field(KNN_ENGINE, FAISS_NAME) .startObject(PARAMETERS) - .field(KNNConstants.METHOD_PARAMETER_M, mValues.get(random().nextInt(mValues.size()))) + .field(METHOD_PARAMETER_M, mValues.get(random().nextInt(mValues.size()))) .startObject(METHOD_ENCODER_PARAMETER) .field(NAME, ENCODER_PQ) .startObject(PARAMETERS) @@ -1205,7 +1213,7 @@ public void testDocUpdate() throws IOException { .startObject(fieldName) .field("type", "knn_vector") .field("dimension", dimension) - .startObject(KNNConstants.KNN_METHOD) + .startObject(KNN_METHOD) .field(NAME, hnswMethod.getMethodComponent().getName()) .field(METHOD_PARAMETER_SPACE_TYPE, spaceType.getValue()) .field(KNN_ENGINE, KNNEngine.FAISS.getName()) @@ -1241,7 +1249,7 @@ public void testDocDeletion() throws IOException { .startObject(fieldName) .field("type", "knn_vector") .field("dimension", dimension) - .startObject(KNNConstants.KNN_METHOD) + .startObject(KNN_METHOD) .field(NAME, hnswMethod.getMethodComponent().getName()) .field(METHOD_PARAMETER_SPACE_TYPE, spaceType.getValue()) .field(KNN_ENGINE, KNNEngine.FAISS.getName()) @@ -1417,7 +1425,7 @@ public void testFiltering_whenUsingFaissExactSearchWithIP_thenMatchExpectedScore .startObject(FIELD_NAME) .field("type", "knn_vector") .field("dimension", 2) - .startObject(KNNConstants.KNN_METHOD) + .startObject(KNN_METHOD) .field(NAME, KNNEngine.FAISS.getMethod(METHOD_HNSW).getMethodComponent().getName()) .field(METHOD_PARAMETER_SPACE_TYPE, SpaceType.INNER_PRODUCT.getValue()) .field(KNN_ENGINE, KNNEngine.FAISS.getName()) @@ -1592,6 +1600,230 @@ public void testIVF_InvalidPQM_thenFail() { ); } + @SneakyThrows + public void testIVF_whenBinaryFormat_whenIVF_thenSuccess() { + String modelId = "test-model-ivf-binary"; + int dimension = 8; + + String trainingIndexName = "train-index-ivf-binary"; + String trainingFieldName = "train-field-ivf-binary"; + + String trainIndexMapping = XContentFactory.jsonBuilder() + .startObject() + .startObject("properties") + .startObject(trainingFieldName) + .field("type", "knn_vector") + .field("dimension", dimension) + .field("data_type", VectorDataType.BINARY.getValue()) + .startObject(KNN_METHOD) + .field(NAME, KNNEngine.FAISS.getMethod(METHOD_HNSW).getMethodComponent().getName()) + .field(METHOD_PARAMETER_SPACE_TYPE, SpaceType.HAMMING.getValue()) + .field(KNN_ENGINE, KNNEngine.FAISS.getName()) + .startObject(PARAMETERS) + .field(METHOD_PARAMETER_M, 24) + .field(METHOD_PARAMETER_EF_CONSTRUCTION, 128) + .endObject() + .endObject() + .endObject() + .endObject() + .endObject() + .toString(); + + createKnnIndex(trainingIndexName, trainIndexMapping); + + int trainingDataCount = 40; + bulkIngestRandomBinaryVectors(trainingIndexName, trainingFieldName, trainingDataCount, dimension); + + XContentBuilder trainModelXContentBuilder = XContentFactory.jsonBuilder() + .startObject() + .field(TRAIN_INDEX_PARAMETER, trainingIndexName) + .field(TRAIN_FIELD_PARAMETER, trainingFieldName) + .field(DIMENSION, dimension) + .field(MODEL_DESCRIPTION, "My model description") + .field(VECTOR_DATA_TYPE_FIELD, VectorDataType.BINARY.getValue()) + .field( + KNN_METHOD, + Map.of( + NAME, + METHOD_IVF, + KNN_ENGINE, + FAISS_NAME, + METHOD_PARAMETER_SPACE_TYPE, + SpaceType.HAMMING.getValue(), + PARAMETERS, + Map.of(METHOD_PARAMETER_NLIST, 1, METHOD_PARAMETER_NPROBES, 1) + ) + ) + .endObject(); + + trainModel(modelId, trainModelXContentBuilder); + + // Make sure training succeeds after 30 seconds + assertTrainingSucceeds(modelId, 30, 1000); + + // Create knn index from model + String fieldName = "test-field-name-ivf-binary"; + String indexName = "test-index-name-ivf-binary"; + String indexMapping = XContentFactory.jsonBuilder() + .startObject() + .startObject("properties") + .startObject(fieldName) + .field("type", "knn_vector") + .field(MODEL_ID, modelId) + .endObject() + .endObject() + .endObject() + .toString(); + + createKnnIndex(indexName, getKNNDefaultIndexSettings(), indexMapping); + Integer[] vector1 = { 11 }; + Integer[] vector2 = { 22 }; + Integer[] vector3 = { 33 }; + Integer[] vector4 = { 44 }; + addKnnDoc(indexName, "1", fieldName, vector1); + addKnnDoc(indexName, "2", fieldName, vector2); + addKnnDoc(indexName, "3", fieldName, vector3); + addKnnDoc(indexName, "4", fieldName, vector4); + + Integer[] queryVector = { 15 }; + int k = 2; + + XContentBuilder queryBuilder = XContentFactory.jsonBuilder() + .startObject() + .startObject("query") + .startObject("knn") + .startObject(fieldName) + .field("vector", queryVector) + .field("k", k) + .endObject() + .endObject() + .endObject() + .endObject(); + Response searchResponse = searchKNNIndex(indexName, queryBuilder, k); + List results = parseSearchResponse(EntityUtils.toString(searchResponse.getEntity()), fieldName); + assertEquals(k, results.size()); + + deleteKNNIndex(indexName); + Thread.sleep(45 * 1000); + deleteModel(modelId); + deleteKNNIndex(trainingIndexName); + validateGraphEviction(); + } + + @SneakyThrows + public void testIVF_whenBinaryFormat_whenIVFPQ_thenSuccess() { + String modelId = "test-model-ivfpq-binary"; + int dimension = 8; + + String trainingIndexName = "train-index-ivfpq-binary"; + String trainingFieldName = "train-field-ivfpq-binary"; + + String trainIndexMapping = XContentFactory.jsonBuilder() + .startObject() + .startObject("properties") + .startObject(trainingFieldName) + .field("type", "knn_vector") + .field("dimension", dimension) + .field("data_type", VectorDataType.BINARY.getValue()) + .startObject(KNN_METHOD) + .field(NAME, KNNEngine.FAISS.getMethod(METHOD_HNSW).getMethodComponent().getName()) + .field(METHOD_PARAMETER_SPACE_TYPE, SpaceType.HAMMING.getValue()) + .field(KNN_ENGINE, KNNEngine.FAISS.getName()) + .startObject(PARAMETERS) + .field(METHOD_PARAMETER_M, 24) + .field(METHOD_PARAMETER_EF_CONSTRUCTION, 128) + .endObject() + .endObject() + .endObject() + .endObject() + .endObject() + .toString(); + + createKnnIndex(trainingIndexName, trainIndexMapping); + + int trainingDataCount = 50; + bulkIngestRandomBinaryVectors(trainingIndexName, trainingFieldName, trainingDataCount, dimension); + + XContentBuilder trainModelXContentBuilder = XContentFactory.jsonBuilder() + .startObject() + .field(TRAIN_INDEX_PARAMETER, trainingIndexName) + .field(TRAIN_FIELD_PARAMETER, trainingFieldName) + .field(DIMENSION, dimension) + .field(MODEL_DESCRIPTION, "My model description") + .field(VECTOR_DATA_TYPE_FIELD, VectorDataType.BINARY.getValue()) + .startObject(KNN_METHOD) + .field(NAME, METHOD_IVF) + .field(KNN_ENGINE, FAISS_NAME) + .field(METHOD_PARAMETER_SPACE_TYPE, SpaceType.HAMMING.getValue()) + .startObject(PARAMETERS) + .field(METHOD_PARAMETER_NPROBES, 1) + .field(METHOD_PARAMETER_NLIST, 1) + .startObject(METHOD_ENCODER_PARAMETER) + .field(NAME, ENCODER_PQ) + .startObject(PARAMETERS) + .field(ENCODER_PARAMETER_PQ_CODE_SIZE, 8) + .field(ENCODER_PARAMETER_PQ_M, 8) + .endObject() + .endObject() + .endObject() + .endObject() + .endObject(); + + trainModel(modelId, trainModelXContentBuilder); + + // Make sure training succeeds after 30 seconds + assertTrainingSucceeds(modelId, 30, 1000); + + // Create knn index from model + String fieldName = "test-field-name-ivfpq-binary"; + String indexName = "test-index-name-ivfpq-binary"; + + String indexMapping = XContentFactory.jsonBuilder() + .startObject() + .startObject("properties") + .startObject(fieldName) + .field("type", "knn_vector") + .field(MODEL_ID, modelId) + .endObject() + .endObject() + .endObject() + .toString(); + + createKnnIndex(indexName, getKNNDefaultIndexSettings(), indexMapping); + Integer[] vector1 = { 11 }; + Integer[] vector2 = { 22 }; + Integer[] vector3 = { 33 }; + Integer[] vector4 = { 44 }; + addKnnDoc(indexName, "1", fieldName, vector1); + addKnnDoc(indexName, "2", fieldName, vector2); + addKnnDoc(indexName, "3", fieldName, vector3); + addKnnDoc(indexName, "4", fieldName, vector4); + + Integer[] queryVector = { 15 }; + int k = 2; + + XContentBuilder queryBuilder = XContentFactory.jsonBuilder() + .startObject() + .startObject("query") + .startObject("knn") + .startObject(fieldName) + .field("vector", queryVector) + .field("k", k) + .endObject() + .endObject() + .endObject() + .endObject(); + Response searchResponse = searchKNNIndex(indexName, queryBuilder, k); + List results = parseSearchResponse(EntityUtils.toString(searchResponse.getEntity()), fieldName); + assertEquals(k, results.size()); + + deleteKNNIndex(indexName); + Thread.sleep(45 * 1000); + deleteModel(modelId); + deleteKNNIndex(trainingIndexName); + validateGraphEviction(); + } + protected void setupKNNIndexForFilterQuery() throws Exception { // Create Mappings XContentBuilder builder = XContentFactory.jsonBuilder() @@ -1600,7 +1832,7 @@ protected void setupKNNIndexForFilterQuery() throws Exception { .startObject(FIELD_NAME) .field("type", "knn_vector") .field("dimension", 3) - .startObject(KNNConstants.KNN_METHOD) + .startObject(KNN_METHOD) .field(NAME, KNNEngine.FAISS.getMethod(METHOD_HNSW).getMethodComponent().getName()) .field(METHOD_PARAMETER_SPACE_TYPE, SpaceType.L2) .field(KNN_ENGINE, KNNEngine.FAISS.getName()) diff --git a/src/test/java/org/opensearch/knn/index/IndexUtilTests.java b/src/test/java/org/opensearch/knn/index/IndexUtilTests.java index 6c1384e9d..1b00ecfaa 100644 --- a/src/test/java/org/opensearch/knn/index/IndexUtilTests.java +++ b/src/test/java/org/opensearch/knn/index/IndexUtilTests.java @@ -117,7 +117,7 @@ public void testValidateKnnField_NestedField() { when(trainingFieldModelMetadata.getDimension()).thenReturn(dimension); when(modelDao.getMetadata(anyString())).thenReturn(trainingFieldModelMetadata); - ValidationException e = IndexUtil.validateKnnField(indexMetadata, field, dimension, modelDao); + ValidationException e = IndexUtil.validateKnnField(indexMetadata, field, dimension, modelDao, null); assertNull(e); } @@ -138,7 +138,7 @@ public void testValidateKnnField_NonNestedField() { when(trainingFieldModelMetadata.getDimension()).thenReturn(dimension); when(modelDao.getMetadata(anyString())).thenReturn(trainingFieldModelMetadata); - ValidationException e = IndexUtil.validateKnnField(indexMetadata, field, dimension, modelDao); + ValidationException e = IndexUtil.validateKnnField(indexMetadata, field, dimension, modelDao, null); assertNull(e); } @@ -158,7 +158,7 @@ public void testValidateKnnField_NonKnnField() { when(trainingFieldModelMetadata.getDimension()).thenReturn(dimension); when(modelDao.getMetadata(anyString())).thenReturn(trainingFieldModelMetadata); - ValidationException e = IndexUtil.validateKnnField(indexMetadata, field, dimension, modelDao); + ValidationException e = IndexUtil.validateKnnField(indexMetadata, field, dimension, modelDao, null); assert Objects.requireNonNull(e).getMessage().matches("Validation Failed: 1: Field \"" + field + "\" is not of type knn_vector.;"); } @@ -182,7 +182,7 @@ public void testValidateKnnField_WrongFieldPath() { when(trainingFieldModelMetadata.getDimension()).thenReturn(dimension); when(modelDao.getMetadata(anyString())).thenReturn(trainingFieldModelMetadata); - ValidationException e = IndexUtil.validateKnnField(indexMetadata, field, dimension, modelDao); + ValidationException e = IndexUtil.validateKnnField(indexMetadata, field, dimension, modelDao, null); assert (Objects.requireNonNull(e).getMessage().matches("Validation Failed: 1: Field \"" + field + "\" does not exist.;")); } @@ -206,7 +206,7 @@ public void testValidateKnnField_EmptyField() { when(trainingFieldModelMetadata.getDimension()).thenReturn(dimension); when(modelDao.getMetadata(anyString())).thenReturn(trainingFieldModelMetadata); - ValidationException e = IndexUtil.validateKnnField(indexMetadata, field, dimension, modelDao); + ValidationException e = IndexUtil.validateKnnField(indexMetadata, field, dimension, modelDao, null); System.out.println(Objects.requireNonNull(e).getMessage()); @@ -223,32 +223,11 @@ public void testValidateKnnField_EmptyIndexMetadata() { when(trainingFieldModelMetadata.getDimension()).thenReturn(dimension); when(modelDao.getMetadata(anyString())).thenReturn(trainingFieldModelMetadata); - ValidationException e = IndexUtil.validateKnnField(indexMetadata, field, dimension, modelDao); + ValidationException e = IndexUtil.validateKnnField(indexMetadata, field, dimension, modelDao, null); assert (Objects.requireNonNull(e).getMessage().matches("Validation Failed: 1: Invalid index. Index does not contain a mapping;")); } - public void testValidateKnnField_whenBinaryDataType_thenThrowException() { - Map fieldValues = Map.of("type", "knn_vector", "dimension", 8, "data_type", "BINARY"); - Map top_level_field = Map.of("top_level_field", fieldValues); - Map properties = Map.of("properties", top_level_field); - String field = "top_level_field"; - int dimension = 8; - - MappingMetadata mappingMetadata = mock(MappingMetadata.class); - when(mappingMetadata.getSourceAsMap()).thenReturn(properties); - IndexMetadata indexMetadata = mock(IndexMetadata.class); - when(indexMetadata.mapping()).thenReturn(mappingMetadata); - ModelDao modelDao = mock(ModelDao.class); - ModelMetadata trainingFieldModelMetadata = mock(ModelMetadata.class); - when(trainingFieldModelMetadata.getDimension()).thenReturn(dimension); - when(modelDao.getMetadata(anyString())).thenReturn(trainingFieldModelMetadata); - - ValidationException e = IndexUtil.validateKnnField(indexMetadata, field, dimension, modelDao); - - assert (Objects.requireNonNull(e).getMessage().contains("is of data type BINARY. Only FLOAT or BYTE is supported")); - } - public void testIsShareableStateContainedInIndex_whenIndexNotModelBased_thenReturnFalse() { String modelId = null; KNNEngine knnEngine = KNNEngine.FAISS; @@ -280,4 +259,56 @@ public void testIsBinaryIndex_whenNonBinary_thenFalse() { nonBinaryIndexParams.put(VECTOR_DATA_TYPE_FIELD, "byte"); assertFalse(IndexUtil.isBinaryIndex(KNNEngine.FAISS, nonBinaryIndexParams)); } + + public void testValidateKnnField_whenTrainModelUseDifferentVectorDataTypeFromTrainIndex_thenThrowException() { + Map fieldValues = Map.of("type", "knn_vector", "dimension", 8, "data_type", "float"); + Map top_level_field = Map.of("top_level_field", fieldValues); + Map properties = Map.of("properties", top_level_field); + String field = "top_level_field"; + int dimension = 8; + + MappingMetadata mappingMetadata = mock(MappingMetadata.class); + when(mappingMetadata.getSourceAsMap()).thenReturn(properties); + IndexMetadata indexMetadata = mock(IndexMetadata.class); + when(indexMetadata.mapping()).thenReturn(mappingMetadata); + ModelDao modelDao = mock(ModelDao.class); + + ValidationException e = IndexUtil.validateKnnField(indexMetadata, field, dimension, modelDao, VectorDataType.BINARY); + System.out.println(Objects.requireNonNull(e).getMessage()); + + assert Objects.requireNonNull(e) + .getMessage() + .matches( + "Validation Failed: 1: Field \"" + + field + + "\" has data type float, which is different from data type used in the training request: binary;" + ); + } + + public void testValidateKnnField_whenPassByteVectorDataType_thenThrowException() { + Map fieldValues = Map.of("type", "knn_vector", "dimension", 8, "data_type", "byte"); + Map top_level_field = Map.of("top_level_field", fieldValues); + Map properties = Map.of("properties", top_level_field); + String field = "top_level_field"; + int dimension = 8; + + MappingMetadata mappingMetadata = mock(MappingMetadata.class); + when(mappingMetadata.getSourceAsMap()).thenReturn(properties); + IndexMetadata indexMetadata = mock(IndexMetadata.class); + when(indexMetadata.mapping()).thenReturn(mappingMetadata); + ModelDao modelDao = mock(ModelDao.class); + + ValidationException e = IndexUtil.validateKnnField(indexMetadata, field, dimension, modelDao, VectorDataType.BYTE); + System.out.println(Objects.requireNonNull(e).getMessage()); + + assert Objects.requireNonNull(e) + .getMessage() + .matches("Validation Failed: 1: vector data type \"" + VectorDataType.BYTE.getValue() + "\" is not supported for training.;"); + } + + public void testUpdateVectorDataTypeToParameters_whenVectorDataTypeIsBinary() { + Map indexParams = new HashMap<>(); + IndexUtil.updateVectorDataTypeToParameters(indexParams, VectorDataType.BINARY); + assertEquals(VectorDataType.BINARY.getValue(), indexParams.get(VECTOR_DATA_TYPE_FIELD)); + } } diff --git a/src/test/java/org/opensearch/knn/index/KNNCreateIndexFromModelTests.java b/src/test/java/org/opensearch/knn/index/KNNCreateIndexFromModelTests.java index 11a8bdb15..e9b78e7ec 100644 --- a/src/test/java/org/opensearch/knn/index/KNNCreateIndexFromModelTests.java +++ b/src/test/java/org/opensearch/knn/index/KNNCreateIndexFromModelTests.java @@ -63,7 +63,8 @@ public void testCreateIndexFromModel() throws IOException, InterruptedException "", "", "test-node", - MethodComponentContext.EMPTY + MethodComponentContext.EMPTY, + VectorDataType.FLOAT ); Model model = new Model(modelMetadata, modelBlob, modelId); diff --git a/src/test/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumerTests.java b/src/test/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumerTests.java index 53066c1cc..6837a5ce5 100644 --- a/src/test/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumerTests.java +++ b/src/test/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumerTests.java @@ -125,8 +125,21 @@ public void testAddBinaryField_withoutKNN() throws IOException { DocValuesConsumer delegate = mock(DocValuesConsumer.class); doNothing().when(delegate).addBinaryField(fieldInfo, docValuesProducer); + String segmentName = String.format("test_segment%s", randomAlphaOfLength(4)); + int docsInSegment = 100; + + SegmentInfo segmentInfo = KNNCodecTestUtil.segmentInfoBuilder() + .directory(directory) + .segmentName(segmentName) + .docsInSegment(docsInSegment) + .codec(codec) + .build(); + + FieldInfos fieldInfos = mock(FieldInfos.class); + SegmentWriteState state = new SegmentWriteState(null, directory, segmentInfo, fieldInfos, null, IOContext.DEFAULT); + final boolean[] called = { false }; - KNN80DocValuesConsumer knn80DocValuesConsumer = new KNN80DocValuesConsumer(delegate, null) { + KNN80DocValuesConsumer knn80DocValuesConsumer = new KNN80DocValuesConsumer(delegate, state) { @Override public void addKNNBinaryField(FieldInfo field, DocValuesProducer valuesProducer, boolean isMerge, boolean isRefresh) { @@ -148,7 +161,19 @@ public void testAddKNNBinaryField_noVectors() throws IOException { Long initialMergeOperations = KNNGraphValue.MERGE_TOTAL_OPERATIONS.getValue(); Long initialMergeSize = KNNGraphValue.MERGE_TOTAL_SIZE_IN_BYTES.getValue(); Long initialMergeDocs = KNNGraphValue.MERGE_TOTAL_DOCS.getValue(); - KNN80DocValuesConsumer knn80DocValuesConsumer = new KNN80DocValuesConsumer(null, null); + String segmentName = String.format("test_segment%s", randomAlphaOfLength(4)); + int docsInSegment = 100; + + SegmentInfo segmentInfo = KNNCodecTestUtil.segmentInfoBuilder() + .directory(directory) + .segmentName(segmentName) + .docsInSegment(docsInSegment) + .codec(codec) + .build(); + + FieldInfos fieldInfos = mock(FieldInfos.class); + SegmentWriteState state = new SegmentWriteState(null, directory, segmentInfo, fieldInfos, null, IOContext.DEFAULT); + KNN80DocValuesConsumer knn80DocValuesConsumer = new KNN80DocValuesConsumer(null, state); FieldInfo fieldInfo = KNNCodecTestUtil.FieldInfoBuilder.builder("test-field").build(); knn80DocValuesConsumer.addKNNBinaryField(fieldInfo, randomVectorDocValuesProducer, true, true); assertEquals(initialGraphIndexRequests, KNNCounter.GRAPH_INDEX_REQUESTS.getCount()); @@ -424,7 +449,8 @@ public void testAddKNNBinaryField_fromModel_faiss() throws IOException, Executio "Empty description", "", "", - MethodComponentContext.EMPTY + MethodComponentContext.EMPTY, + VectorDataType.FLOAT ), modelBytes, modelId diff --git a/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestCase.java b/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestCase.java index b82bc85e0..66fe9770d 100644 --- a/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestCase.java +++ b/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestCase.java @@ -20,15 +20,16 @@ import org.opensearch.knn.KNNTestCase; import org.opensearch.knn.common.KNNConstants; import org.opensearch.knn.index.KNNMethodContext; +import org.opensearch.knn.index.KNNSettings; import org.opensearch.knn.index.MethodComponentContext; +import org.opensearch.knn.index.SpaceType; +import org.opensearch.knn.index.VectorDataType; +import org.opensearch.knn.index.VectorField; import org.opensearch.knn.index.query.KNNQueryFactory; import org.opensearch.knn.jni.JNIService; import org.opensearch.knn.index.query.KNNQuery; -import org.opensearch.knn.index.KNNSettings; import org.opensearch.knn.index.mapper.KNNVectorFieldMapper; import org.opensearch.knn.index.query.KNNWeight; -import org.opensearch.knn.index.SpaceType; -import org.opensearch.knn.index.VectorField; import org.apache.lucene.codecs.Codec; import org.apache.lucene.document.Document; import org.apache.lucene.document.FieldType; @@ -213,7 +214,8 @@ public void testBuildFromModelTemplate(Codec codec) throws IOException, Executio "", "", "", - MethodComponentContext.EMPTY + MethodComponentContext.EMPTY, + VectorDataType.FLOAT ); Model mockModel = new Model(modelMetadata1, modelBlob, modelId); diff --git a/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java b/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java index cb069ad34..0d3c6ac8a 100644 --- a/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java +++ b/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java @@ -175,7 +175,8 @@ public void testBuilder_build_fromModel() { "", "", "", - MethodComponentContext.EMPTY + MethodComponentContext.EMPTY, + VectorDataType.FLOAT ); builder.modelId.setValue(modelId); Mapper.BuilderContext builderContext = new Mapper.BuilderContext(settings, new ContentPath()); @@ -690,7 +691,8 @@ public void testKNNVectorFieldMapper_merge_fromModel() throws IOException { "", "", "", - MethodComponentContext.EMPTY + MethodComponentContext.EMPTY, + VectorDataType.FLOAT ); when(mockModelDao.getMetadata(modelId)).thenReturn(mockModelMetadata); @@ -761,7 +763,8 @@ public void testLuceneFieldMapper_parseCreateField_docValues_withFloats() { parseContext, TEST_DIMENSION, luceneFieldMapper.fieldType().spaceType, - luceneFieldMapper.fieldType().knnMethodContext.getMethodComponentContext() + luceneFieldMapper.fieldType().knnMethodContext.getMethodComponentContext(), + VectorDataType.FLOAT ); // Document should have 2 fields: one for VectorField (binary doc values) and one for KnnVectorField @@ -805,7 +808,8 @@ public void testLuceneFieldMapper_parseCreateField_docValues_withFloats() { parseContext, TEST_DIMENSION, luceneFieldMapper.fieldType().spaceType, - luceneFieldMapper.fieldType().knnMethodContext.getMethodComponentContext() + luceneFieldMapper.fieldType().knnMethodContext.getMethodComponentContext(), + VectorDataType.FLOAT ); // Document should have 1 field: one for KnnVectorField @@ -840,7 +844,8 @@ public void testLuceneFieldMapper_parseCreateField_docValues_withBytes() { parseContext, TEST_DIMENSION, luceneFieldMapper.fieldType().spaceType, - luceneFieldMapper.fieldType().knnMethodContext.getMethodComponentContext() + luceneFieldMapper.fieldType().knnMethodContext.getMethodComponentContext(), + VectorDataType.BYTE ); // Document should have 2 fields: one for VectorField (binary doc values) and one for KnnByteVectorField @@ -883,7 +888,8 @@ public void testLuceneFieldMapper_parseCreateField_docValues_withBytes() { parseContext, TEST_DIMENSION, luceneFieldMapper.fieldType().spaceType, - luceneFieldMapper.fieldType().knnMethodContext.getMethodComponentContext() + luceneFieldMapper.fieldType().knnMethodContext.getMethodComponentContext(), + VectorDataType.BYTE ); // Document should have 1 field: one for KnnByteVectorField diff --git a/src/test/java/org/opensearch/knn/index/memory/NativeMemoryAllocationTests.java b/src/test/java/org/opensearch/knn/index/memory/NativeMemoryAllocationTests.java index b530c9d8d..a98e07182 100644 --- a/src/test/java/org/opensearch/knn/index/memory/NativeMemoryAllocationTests.java +++ b/src/test/java/org/opensearch/knn/index/memory/NativeMemoryAllocationTests.java @@ -315,7 +315,8 @@ public void testTrainingDataAllocation_close() throws InterruptedException { NativeMemoryAllocation.TrainingDataAllocation trainingDataAllocation = new NativeMemoryAllocation.TrainingDataAllocation( executorService, memoryAddress, - 0 + 0, + VectorDataType.FLOAT ); trainingDataAllocation.close(); @@ -341,7 +342,8 @@ public void testTrainingDataAllocation_getMemoryAddress() { NativeMemoryAllocation.TrainingDataAllocation trainingDataAllocation = new NativeMemoryAllocation.TrainingDataAllocation( null, memoryAddress, - 0 + 0, + VectorDataType.FLOAT ); assertEquals(memoryAddress, trainingDataAllocation.getMemoryAddress()); @@ -354,7 +356,8 @@ public void testTrainingDataAllocation_readLock() throws InterruptedException { NativeMemoryAllocation.TrainingDataAllocation trainingDataAllocation = new NativeMemoryAllocation.TrainingDataAllocation( null, 0, - 0 + 0, + VectorDataType.FLOAT ); int initialValue = 10; @@ -387,7 +390,8 @@ public void testTrainingDataAllocation_writeLock() throws InterruptedException { NativeMemoryAllocation.TrainingDataAllocation trainingDataAllocation = new NativeMemoryAllocation.TrainingDataAllocation( null, 0, - 0 + 0, + VectorDataType.FLOAT ); int initialValue = 10; @@ -422,7 +426,8 @@ public void testTrainingDataAllocation_getSize() { NativeMemoryAllocation.TrainingDataAllocation trainingDataAllocation = new NativeMemoryAllocation.TrainingDataAllocation( null, 0, - size + size, + VectorDataType.FLOAT ); assertEquals(size, trainingDataAllocation.getSizeInKB()); @@ -434,7 +439,8 @@ public void testTrainingDataAllocation_setMemoryAddress() { NativeMemoryAllocation.TrainingDataAllocation trainingDataAllocation = new NativeMemoryAllocation.TrainingDataAllocation( null, pointer, - 0 + 0, + VectorDataType.FLOAT ); assertEquals(pointer, trainingDataAllocation.getMemoryAddress()); diff --git a/src/test/java/org/opensearch/knn/index/memory/NativeMemoryCacheManagerTests.java b/src/test/java/org/opensearch/knn/index/memory/NativeMemoryCacheManagerTests.java index 718df0b1f..85eaf3322 100644 --- a/src/test/java/org/opensearch/knn/index/memory/NativeMemoryCacheManagerTests.java +++ b/src/test/java/org/opensearch/knn/index/memory/NativeMemoryCacheManagerTests.java @@ -16,6 +16,7 @@ import org.opensearch.common.settings.Settings; import org.opensearch.knn.common.exception.OutOfNativeMemoryException; import org.opensearch.knn.index.KNNSettings; +import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.plugin.KNNPlugin; import org.opensearch.plugins.Plugin; import org.opensearch.test.OpenSearchSingleNodeTestCase; @@ -186,7 +187,8 @@ public void testGetTrainingSize() throws ExecutionException { NativeMemoryAllocation.TrainingDataAllocation trainingDataAllocation = new NativeMemoryAllocation.TrainingDataAllocation( null, 0, - allocationEntryWeight + allocationEntryWeight, + VectorDataType.FLOAT ); NativeMemoryEntryContext.TrainingDataEntryContext trainingDataEntryContext = mock( diff --git a/src/test/java/org/opensearch/knn/index/memory/NativeMemoryEntryContextTests.java b/src/test/java/org/opensearch/knn/index/memory/NativeMemoryEntryContextTests.java index 495f20347..f87a069a2 100644 --- a/src/test/java/org/opensearch/knn/index/memory/NativeMemoryEntryContextTests.java +++ b/src/test/java/org/opensearch/knn/index/memory/NativeMemoryEntryContextTests.java @@ -15,6 +15,7 @@ import org.opensearch.cluster.service.ClusterService; import org.opensearch.knn.KNNTestCase; import org.opensearch.knn.index.IndexUtil; +import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.util.KNNEngine; import java.io.BufferedOutputStream; @@ -122,13 +123,15 @@ public void testTrainingDataEntryContext_load() { trainingLoadStrategy, null, 0, - 0 + 0, + VectorDataType.DEFAULT ); NativeMemoryAllocation.TrainingDataAllocation trainingDataAllocation = new NativeMemoryAllocation.TrainingDataAllocation( null, 0, - 0 + 0, + VectorDataType.DEFAULT ); when(trainingLoadStrategy.load(trainingDataEntryContext)).thenReturn(trainingDataAllocation); @@ -145,7 +148,8 @@ public void testTrainingDataEntryContext_getTrainIndexName() { null, null, 0, - 0 + 0, + VectorDataType.DEFAULT ); assertEquals(trainIndexName, trainingDataEntryContext.getTrainIndexName()); @@ -160,7 +164,8 @@ public void testTrainingDataEntryContext_getTrainFieldName() { null, null, 0, - 0 + 0, + VectorDataType.DEFAULT ); assertEquals(trainFieldName, trainingDataEntryContext.getTrainFieldName()); @@ -175,7 +180,8 @@ public void testTrainingDataEntryContext_getMaxVectorCount() { null, null, maxVectorCount, - 0 + 0, + VectorDataType.DEFAULT ); assertEquals(maxVectorCount, trainingDataEntryContext.getMaxVectorCount()); @@ -190,7 +196,8 @@ public void testTrainingDataEntryContext_getSearchSize() { null, null, 0, - searchSize + searchSize, + VectorDataType.DEFAULT ); assertEquals(searchSize, trainingDataEntryContext.getSearchSize()); @@ -205,7 +212,8 @@ public void testTrainingDataEntryContext_getIndicesService() { null, clusterService, 0, - 0 + 0, + VectorDataType.DEFAULT ); assertEquals(clusterService, trainingDataEntryContext.getClusterService()); diff --git a/src/test/java/org/opensearch/knn/index/memory/NativeMemoryLoadStrategyTests.java b/src/test/java/org/opensearch/knn/index/memory/NativeMemoryLoadStrategyTests.java index 134f0fe42..7fac05271 100644 --- a/src/test/java/org/opensearch/knn/index/memory/NativeMemoryLoadStrategyTests.java +++ b/src/test/java/org/opensearch/knn/index/memory/NativeMemoryLoadStrategyTests.java @@ -22,7 +22,7 @@ import org.opensearch.knn.index.query.KNNQueryResult; import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.util.KNNEngine; -import org.opensearch.knn.training.TrainingDataConsumer; +import org.opensearch.knn.training.FloatTrainingDataConsumer; import org.opensearch.knn.training.VectorReader; import org.opensearch.watcher.ResourceWatcherService; @@ -150,12 +150,12 @@ public void testTrainingLoadStrategy_load() { logger.info("J0"); doAnswer(invocationOnMock -> { logger.info("J1"); - TrainingDataConsumer trainingDataConsumer = (TrainingDataConsumer) invocationOnMock.getArguments()[5]; + FloatTrainingDataConsumer floatTrainingDataConsumer = (FloatTrainingDataConsumer) invocationOnMock.getArguments()[5]; ActionListener listener = (ActionListener) invocationOnMock.getArguments()[6]; Thread thread = new Thread(() -> { try { Thread.sleep(2000); - trainingDataConsumer.accept(vectors); // Transfer some floats + floatTrainingDataConsumer.accept(vectors); // Transfer some floats listener.onResponse(null); } catch (InterruptedException e) { listener.onFailure(null); @@ -176,7 +176,8 @@ public void testTrainingLoadStrategy_load() { NativeMemoryLoadStrategy.TrainingLoadStrategy.getInstance(), null, 0, - 0 + 0, + VectorDataType.FLOAT ); // Load the allocation. Initially, the memory address should be 0. However, after the readlock is obtained, diff --git a/src/test/java/org/opensearch/knn/index/query/KNNQueryBuilderTests.java b/src/test/java/org/opensearch/knn/index/query/KNNQueryBuilderTests.java index 651fe75c0..18e2b914a 100644 --- a/src/test/java/org/opensearch/knn/index/query/KNNQueryBuilderTests.java +++ b/src/test/java/org/opensearch/knn/index/query/KNNQueryBuilderTests.java @@ -953,6 +953,7 @@ public void testDoToQuery_FromModel() { when(modelMetadata.getSpaceType()).thenReturn(SpaceType.COSINESIMIL); when(modelMetadata.getState()).thenReturn(ModelState.CREATED); when(modelMetadata.getMethodComponentContext()).thenReturn(new MethodComponentContext("ivf", emptyMap())); + when(modelMetadata.getVectorDataType()).thenReturn(VectorDataType.DEFAULT); ModelDao modelDao = mock(ModelDao.class); when(modelDao.getMetadata(modelId)).thenReturn(modelMetadata); KNNQueryBuilder.initialize(modelDao); @@ -990,6 +991,7 @@ public void testDoToQuery_whenFromModel_whenDoRadiusSearch_whenDistanceThreshold when(modelMetadata.getSpaceType()).thenReturn(SpaceType.L2); when(modelMetadata.getState()).thenReturn(ModelState.CREATED); when(modelMetadata.getMethodComponentContext()).thenReturn(new MethodComponentContext("ivf", emptyMap())); + when(modelMetadata.getVectorDataType()).thenReturn(VectorDataType.DEFAULT); ModelDao modelDao = mock(ModelDao.class); when(modelDao.getMetadata(modelId)).thenReturn(modelMetadata); KNNQueryBuilder.initialize(modelDao); @@ -1025,6 +1027,7 @@ public void testDoToQuery_whenFromModel_whenDoRadiusSearch_whenScoreThreshold_th when(modelMetadata.getKnnEngine()).thenReturn(KNNEngine.FAISS); when(modelMetadata.getSpaceType()).thenReturn(SpaceType.L2); when(modelMetadata.getState()).thenReturn(ModelState.CREATED); + when(modelMetadata.getVectorDataType()).thenReturn(VectorDataType.DEFAULT); when(modelMetadata.getMethodComponentContext()).thenReturn(new MethodComponentContext("ivf", emptyMap())); ModelDao modelDao = mock(ModelDao.class); when(modelDao.getMetadata(modelId)).thenReturn(modelMetadata); diff --git a/src/test/java/org/opensearch/knn/index/query/KNNWeightTests.java b/src/test/java/org/opensearch/knn/index/query/KNNWeightTests.java index 2905d047e..d08f7e0ce 100644 --- a/src/test/java/org/opensearch/knn/index/query/KNNWeightTests.java +++ b/src/test/java/org/opensearch/knn/index/query/KNNWeightTests.java @@ -38,6 +38,7 @@ import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.knn.KNNTestCase; import org.opensearch.knn.index.KNNSettings; +import org.opensearch.knn.index.MethodComponentContext; import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.codec.KNNCodecVersion; @@ -62,6 +63,7 @@ import java.util.function.Function; import java.util.stream.Collectors; +import static java.util.Collections.emptyMap; import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyBoolean; @@ -218,6 +220,8 @@ public void testQueryScoreForFaissWithModel() { when(modelMetadata.getKnnEngine()).thenReturn(KNNEngine.FAISS); when(modelMetadata.getSpaceType()).thenReturn(spaceType); when(modelMetadata.getState()).thenReturn(ModelState.CREATED); + when(modelMetadata.getVectorDataType()).thenReturn(VectorDataType.DEFAULT); + when(modelMetadata.getMethodComponentContext()).thenReturn(new MethodComponentContext("ivf", emptyMap())); when(modelDao.getMetadata(eq("modelId"))).thenReturn(modelMetadata); KNNWeight.initialize(modelDao); diff --git a/src/test/java/org/opensearch/knn/indices/ModelCacheTests.java b/src/test/java/org/opensearch/knn/indices/ModelCacheTests.java index 3a5255cd3..e0111204d 100644 --- a/src/test/java/org/opensearch/knn/indices/ModelCacheTests.java +++ b/src/test/java/org/opensearch/knn/indices/ModelCacheTests.java @@ -19,6 +19,7 @@ import org.opensearch.knn.KNNTestCase; import org.opensearch.knn.index.MethodComponentContext; import org.opensearch.knn.index.SpaceType; +import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.util.KNNEngine; import java.time.ZoneOffset; @@ -45,7 +46,8 @@ public void testGet_normal() throws ExecutionException, InterruptedException { "", "", "", - MethodComponentContext.EMPTY + MethodComponentContext.EMPTY, + VectorDataType.DEFAULT ), "hello".getBytes(), modelId @@ -82,7 +84,8 @@ public void testGet_modelDoesNotFitInCache() throws ExecutionException, Interrup "", "", "", - MethodComponentContext.EMPTY + MethodComponentContext.EMPTY, + VectorDataType.DEFAULT ), new byte[BYTES_PER_KILOBYTES + 1], modelId @@ -140,7 +143,8 @@ public void testGetTotalWeight() throws ExecutionException, InterruptedException "", "", "", - MethodComponentContext.EMPTY + MethodComponentContext.EMPTY, + VectorDataType.DEFAULT ), new byte[size1], modelId1 @@ -156,7 +160,8 @@ public void testGetTotalWeight() throws ExecutionException, InterruptedException "", "", "", - MethodComponentContext.EMPTY + MethodComponentContext.EMPTY, + VectorDataType.DEFAULT ), new byte[size2], modelId2 @@ -200,7 +205,8 @@ public void testRemove_normal() throws ExecutionException, InterruptedException "", "", "", - MethodComponentContext.EMPTY + MethodComponentContext.EMPTY, + VectorDataType.DEFAULT ), new byte[size1], modelId1 @@ -216,8 +222,8 @@ public void testRemove_normal() throws ExecutionException, InterruptedException "", "", "", - MethodComponentContext.EMPTY - + MethodComponentContext.EMPTY, + VectorDataType.DEFAULT ), new byte[size2], modelId2 @@ -266,7 +272,8 @@ public void testRebuild_normal() throws ExecutionException, InterruptedException "", "", "", - MethodComponentContext.EMPTY + MethodComponentContext.EMPTY, + VectorDataType.DEFAULT ), "hello".getBytes(), modelId @@ -312,7 +319,8 @@ public void testRebuild_afterSettingUpdate() throws ExecutionException, Interrup "", "", "", - MethodComponentContext.EMPTY + MethodComponentContext.EMPTY, + VectorDataType.DEFAULT ), new byte[modelSize], modelId @@ -381,7 +389,8 @@ public void testContains() throws ExecutionException, InterruptedException { "", "", "", - MethodComponentContext.EMPTY + MethodComponentContext.EMPTY, + VectorDataType.DEFAULT ), new byte[modelSize1], modelId1 @@ -423,7 +432,8 @@ public void testRemoveAll() throws ExecutionException, InterruptedException { "", "", "", - MethodComponentContext.EMPTY + MethodComponentContext.EMPTY, + VectorDataType.DEFAULT ), new byte[modelSize1], modelId1 @@ -441,7 +451,8 @@ public void testRemoveAll() throws ExecutionException, InterruptedException { "", "", "", - MethodComponentContext.EMPTY + MethodComponentContext.EMPTY, + VectorDataType.DEFAULT ), new byte[modelSize2], modelId2 @@ -487,7 +498,8 @@ public void testModelCacheEvictionDueToSize() throws ExecutionException, Interru "", "", "", - MethodComponentContext.EMPTY + MethodComponentContext.EMPTY, + VectorDataType.DEFAULT ), new byte[BYTES_PER_KILOBYTES * 2], modelId diff --git a/src/test/java/org/opensearch/knn/indices/ModelDaoTests.java b/src/test/java/org/opensearch/knn/indices/ModelDaoTests.java index 75c523332..b18a7259e 100644 --- a/src/test/java/org/opensearch/knn/indices/ModelDaoTests.java +++ b/src/test/java/org/opensearch/knn/indices/ModelDaoTests.java @@ -35,6 +35,7 @@ import org.opensearch.knn.common.exception.DeleteModelException; import org.opensearch.knn.index.MethodComponentContext; import org.opensearch.knn.index.SpaceType; +import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.util.KNNEngine; import org.opensearch.knn.plugin.transport.DeleteModelResponse; import org.opensearch.knn.plugin.transport.GetModelResponse; @@ -139,7 +140,8 @@ public void testModelIndexHealth() throws InterruptedException, ExecutionExcepti "", "", "", - MethodComponentContext.EMPTY + MethodComponentContext.EMPTY, + VectorDataType.DEFAULT ), modelBlob, modelId @@ -159,7 +161,8 @@ public void testModelIndexHealth() throws InterruptedException, ExecutionExcepti "", "", "", - MethodComponentContext.EMPTY + MethodComponentContext.EMPTY, + VectorDataType.DEFAULT ), modelBlob, modelId @@ -187,7 +190,8 @@ public void testPut_withId() throws InterruptedException, IOException { "", "", "", - new MethodComponentContext("test", Collections.emptyMap()) + new MethodComponentContext("test", Collections.emptyMap()), + VectorDataType.DEFAULT ), modelBlob, modelId @@ -248,7 +252,8 @@ public void testPut_withoutModel() throws InterruptedException, IOException { "", "", "", - MethodComponentContext.EMPTY + MethodComponentContext.EMPTY, + VectorDataType.DEFAULT ), modelBlob, modelId @@ -310,7 +315,8 @@ public void testPut_invalid_badState() { "", "", "", - MethodComponentContext.EMPTY + MethodComponentContext.EMPTY, + VectorDataType.DEFAULT ), modelBlob, "any-id" @@ -347,7 +353,8 @@ public void testUpdate() throws IOException, InterruptedException { "", "", "", - MethodComponentContext.EMPTY + MethodComponentContext.EMPTY, + VectorDataType.DEFAULT ), null, modelId @@ -386,7 +393,8 @@ public void testUpdate() throws IOException, InterruptedException { "", "", "", - MethodComponentContext.EMPTY + MethodComponentContext.EMPTY, + VectorDataType.DEFAULT ), modelBlob, modelId @@ -437,7 +445,8 @@ public void testGet() throws IOException, InterruptedException, ExecutionExcepti "", "", "", - MethodComponentContext.EMPTY + MethodComponentContext.EMPTY, + VectorDataType.DEFAULT ), modelBlob, modelId @@ -456,7 +465,8 @@ public void testGet() throws IOException, InterruptedException, ExecutionExcepti "", "", "", - MethodComponentContext.EMPTY + MethodComponentContext.EMPTY, + VectorDataType.DEFAULT ), null, modelId @@ -493,7 +503,8 @@ public void testGetMetadata() throws IOException, InterruptedException { "", "", "", - MethodComponentContext.EMPTY + MethodComponentContext.EMPTY, + VectorDataType.DEFAULT ); Model model = new Model(modelMetadata, modelBlob, modelId); @@ -570,7 +581,8 @@ public void testDelete() throws IOException, InterruptedException { "", "", "", - MethodComponentContext.EMPTY + MethodComponentContext.EMPTY, + VectorDataType.DEFAULT ), modelBlob, modelId @@ -604,7 +616,8 @@ public void testDelete() throws IOException, InterruptedException { "", "", "", - MethodComponentContext.EMPTY + MethodComponentContext.EMPTY, + VectorDataType.DEFAULT ), modelBlob, modelId1 @@ -672,7 +685,8 @@ public void testDeleteModelInTrainingWithStepListeners() throws IOException, Exe "", "", "", - MethodComponentContext.EMPTY + MethodComponentContext.EMPTY, + VectorDataType.DEFAULT ), modelBlob, modelId @@ -714,7 +728,8 @@ public void testDeleteWithStepListeners() throws IOException, InterruptedExcepti "", "", "", - MethodComponentContext.EMPTY + MethodComponentContext.EMPTY, + VectorDataType.DEFAULT ), modelBlob, modelId diff --git a/src/test/java/org/opensearch/knn/indices/ModelMetadataTests.java b/src/test/java/org/opensearch/knn/indices/ModelMetadataTests.java index 74715671f..c23b7e2fd 100644 --- a/src/test/java/org/opensearch/knn/indices/ModelMetadataTests.java +++ b/src/test/java/org/opensearch/knn/indices/ModelMetadataTests.java @@ -19,6 +19,7 @@ import org.opensearch.knn.common.KNNConstants; import org.opensearch.knn.index.MethodComponentContext; import org.opensearch.knn.index.SpaceType; +import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.util.KNNEngine; import java.io.IOException; @@ -45,7 +46,8 @@ public void testStreams() throws IOException { "", "", "", - MethodComponentContext.EMPTY + MethodComponentContext.EMPTY, + VectorDataType.DEFAULT ); BytesStreamOutput streamOutput = new BytesStreamOutput(); @@ -67,7 +69,8 @@ public void testGetKnnEngine() { "", "", "", - MethodComponentContext.EMPTY + MethodComponentContext.EMPTY, + VectorDataType.DEFAULT ); assertEquals(knnEngine, modelMetadata.getKnnEngine()); @@ -84,7 +87,8 @@ public void testGetSpaceType() { "", "", "", - MethodComponentContext.EMPTY + MethodComponentContext.EMPTY, + VectorDataType.DEFAULT ); assertEquals(spaceType, modelMetadata.getSpaceType()); @@ -101,7 +105,8 @@ public void testGetDimension() { "", "", "", - MethodComponentContext.EMPTY + MethodComponentContext.EMPTY, + VectorDataType.DEFAULT ); assertEquals(dimension, modelMetadata.getDimension()); @@ -118,7 +123,8 @@ public void testGetState() { "", "", "", - MethodComponentContext.EMPTY + MethodComponentContext.EMPTY, + VectorDataType.DEFAULT ); assertEquals(modelState, modelMetadata.getState()); @@ -135,7 +141,8 @@ public void testGetTimestamp() { "", "", "", - MethodComponentContext.EMPTY + MethodComponentContext.EMPTY, + VectorDataType.DEFAULT ); assertEquals(timeValue, modelMetadata.getTimestamp()); @@ -152,7 +159,8 @@ public void testDescription() { description, "", "", - MethodComponentContext.EMPTY + MethodComponentContext.EMPTY, + VectorDataType.DEFAULT ); assertEquals(description, modelMetadata.getDescription()); @@ -169,12 +177,31 @@ public void testGetError() { "", error, "", - MethodComponentContext.EMPTY + MethodComponentContext.EMPTY, + VectorDataType.DEFAULT ); assertEquals(error, modelMetadata.getError()); } + public void testGetVectorDataType() { + VectorDataType vectorDataType = VectorDataType.BINARY; + ModelMetadata modelMetadata = new ModelMetadata( + KNNEngine.DEFAULT, + SpaceType.L2, + 12, + ModelState.CREATED, + ZonedDateTime.now(ZoneOffset.UTC).toString(), + "", + "", + "", + MethodComponentContext.EMPTY, + vectorDataType + ); + + assertEquals(vectorDataType, modelMetadata.getVectorDataType()); + } + public void testSetState() { ModelState modelState = ModelState.FAILED; ModelMetadata modelMetadata = new ModelMetadata( @@ -186,7 +213,8 @@ public void testSetState() { "", "", "", - MethodComponentContext.EMPTY + MethodComponentContext.EMPTY, + VectorDataType.DEFAULT ); assertEquals(modelState, modelMetadata.getState()); @@ -207,7 +235,8 @@ public void testSetError() { "", error, "", - MethodComponentContext.EMPTY + MethodComponentContext.EMPTY, + VectorDataType.DEFAULT ); assertEquals(error, modelMetadata.getError()); @@ -244,7 +273,9 @@ public void testToString() { + "," + nodeAssignment + "," - + methodComponentContext.toClusterStateString(); + + methodComponentContext.toClusterStateString() + + "," + + VectorDataType.DEFAULT.getValue(); ModelMetadata modelMetadata = new ModelMetadata( knnEngine, @@ -255,7 +286,8 @@ public void testToString() { description, error, nodeAssignment, - MethodComponentContext.EMPTY + MethodComponentContext.EMPTY, + VectorDataType.DEFAULT ); assertEquals(expected, modelMetadata.toString()); @@ -275,7 +307,8 @@ public void testEquals() { "", "", "", - MethodComponentContext.EMPTY + MethodComponentContext.EMPTY, + VectorDataType.DEFAULT ); ModelMetadata modelMetadata2 = new ModelMetadata( KNNEngine.FAISS, @@ -286,7 +319,8 @@ public void testEquals() { "", "", "", - MethodComponentContext.EMPTY + MethodComponentContext.EMPTY, + VectorDataType.DEFAULT ); ModelMetadata modelMetadata3 = new ModelMetadata( @@ -298,7 +332,8 @@ public void testEquals() { "", "", "", - MethodComponentContext.EMPTY + MethodComponentContext.EMPTY, + VectorDataType.DEFAULT ); ModelMetadata modelMetadata4 = new ModelMetadata( KNNEngine.FAISS, @@ -309,7 +344,8 @@ public void testEquals() { "", "", "", - MethodComponentContext.EMPTY + MethodComponentContext.EMPTY, + VectorDataType.DEFAULT ); ModelMetadata modelMetadata5 = new ModelMetadata( KNNEngine.FAISS, @@ -320,7 +356,8 @@ public void testEquals() { "", "", "", - MethodComponentContext.EMPTY + MethodComponentContext.EMPTY, + VectorDataType.DEFAULT ); ModelMetadata modelMetadata6 = new ModelMetadata( KNNEngine.FAISS, @@ -331,7 +368,8 @@ public void testEquals() { "", "", "", - MethodComponentContext.EMPTY + MethodComponentContext.EMPTY, + VectorDataType.DEFAULT ); ModelMetadata modelMetadata7 = new ModelMetadata( KNNEngine.FAISS, @@ -342,7 +380,8 @@ public void testEquals() { "", "", "", - MethodComponentContext.EMPTY + MethodComponentContext.EMPTY, + VectorDataType.DEFAULT ); ModelMetadata modelMetadata8 = new ModelMetadata( KNNEngine.FAISS, @@ -353,7 +392,8 @@ public void testEquals() { "diff descript", "", "", - MethodComponentContext.EMPTY + MethodComponentContext.EMPTY, + VectorDataType.DEFAULT ); ModelMetadata modelMetadata9 = new ModelMetadata( KNNEngine.FAISS, @@ -364,7 +404,8 @@ public void testEquals() { "", "diff error", "", - MethodComponentContext.EMPTY + MethodComponentContext.EMPTY, + VectorDataType.DEFAULT ); ModelMetadata modelMetadata10 = new ModelMetadata( @@ -376,7 +417,8 @@ public void testEquals() { "", "", "", - new MethodComponentContext("test", Collections.emptyMap()) + new MethodComponentContext("test", Collections.emptyMap()), + VectorDataType.DEFAULT ); assertEquals(modelMetadata1, modelMetadata1); @@ -406,7 +448,8 @@ public void testHashCode() { "", "", "", - MethodComponentContext.EMPTY + MethodComponentContext.EMPTY, + VectorDataType.DEFAULT ); ModelMetadata modelMetadata2 = new ModelMetadata( KNNEngine.FAISS, @@ -417,7 +460,8 @@ public void testHashCode() { "", "", "", - MethodComponentContext.EMPTY + MethodComponentContext.EMPTY, + VectorDataType.DEFAULT ); ModelMetadata modelMetadata3 = new ModelMetadata( @@ -429,7 +473,8 @@ public void testHashCode() { "", "", "", - MethodComponentContext.EMPTY + MethodComponentContext.EMPTY, + VectorDataType.DEFAULT ); ModelMetadata modelMetadata4 = new ModelMetadata( KNNEngine.FAISS, @@ -440,7 +485,8 @@ public void testHashCode() { "", "", "", - MethodComponentContext.EMPTY + MethodComponentContext.EMPTY, + VectorDataType.DEFAULT ); ModelMetadata modelMetadata5 = new ModelMetadata( KNNEngine.FAISS, @@ -451,7 +497,8 @@ public void testHashCode() { "", "", "", - MethodComponentContext.EMPTY + MethodComponentContext.EMPTY, + VectorDataType.DEFAULT ); ModelMetadata modelMetadata6 = new ModelMetadata( KNNEngine.FAISS, @@ -462,7 +509,8 @@ public void testHashCode() { "", "", "", - MethodComponentContext.EMPTY + MethodComponentContext.EMPTY, + VectorDataType.DEFAULT ); ModelMetadata modelMetadata7 = new ModelMetadata( KNNEngine.FAISS, @@ -473,7 +521,8 @@ public void testHashCode() { "", "", "", - MethodComponentContext.EMPTY + MethodComponentContext.EMPTY, + VectorDataType.DEFAULT ); ModelMetadata modelMetadata8 = new ModelMetadata( KNNEngine.FAISS, @@ -484,7 +533,8 @@ public void testHashCode() { "diff descript", "", "", - MethodComponentContext.EMPTY + MethodComponentContext.EMPTY, + VectorDataType.DEFAULT ); ModelMetadata modelMetadata9 = new ModelMetadata( KNNEngine.FAISS, @@ -495,7 +545,8 @@ public void testHashCode() { "", "diff error", "", - MethodComponentContext.EMPTY + MethodComponentContext.EMPTY, + VectorDataType.DEFAULT ); ModelMetadata modelMetadata10 = new ModelMetadata( @@ -507,7 +558,8 @@ public void testHashCode() { "", "", "", - new MethodComponentContext("test", Collections.emptyMap()) + new MethodComponentContext("test", Collections.emptyMap()), + VectorDataType.DEFAULT ); assertEquals(modelMetadata1.hashCode(), modelMetadata1.hashCode()); @@ -550,7 +602,9 @@ public void testFromString() { + "," + nodeAssignment + "," - + methodComponentContext.toClusterStateString(); + + methodComponentContext.toClusterStateString() + + "," + + VectorDataType.DEFAULT.getValue(); String stringRep2 = knnEngine.getName() + "," @@ -564,7 +618,9 @@ public void testFromString() { + "," + description + "," - + error; + + error + + "," + + VectorDataType.DEFAULT.getValue(); ModelMetadata expected1 = new ModelMetadata( knnEngine, @@ -575,7 +631,8 @@ public void testFromString() { description, error, nodeAssignment, - MethodComponentContext.EMPTY + MethodComponentContext.EMPTY, + VectorDataType.DEFAULT ); ModelMetadata expected2 = new ModelMetadata( @@ -587,7 +644,8 @@ public void testFromString() { description, error, "", - MethodComponentContext.EMPTY + MethodComponentContext.EMPTY, + VectorDataType.DEFAULT ); ModelMetadata fromString1 = ModelMetadata.fromString(stringRep1); @@ -620,8 +678,8 @@ public void testFromResponseMap() throws IOException { description, error, nodeAssignment, - methodComponentContext - + methodComponentContext, + VectorDataType.DEFAULT ); ModelMetadata expected2 = new ModelMetadata( knnEngine, @@ -632,7 +690,8 @@ public void testFromResponseMap() throws IOException { description, error, "", - emptyMethodComponentContext + emptyMethodComponentContext, + VectorDataType.DEFAULT ); Map metadataAsMap = new HashMap<>(); metadataAsMap.put(KNNConstants.KNN_ENGINE, knnEngine.getName()); @@ -643,6 +702,7 @@ public void testFromResponseMap() throws IOException { metadataAsMap.put(KNNConstants.MODEL_DESCRIPTION, description); metadataAsMap.put(KNNConstants.MODEL_ERROR, error); metadataAsMap.put(KNNConstants.MODEL_NODE_ASSIGNMENT, nodeAssignment); + metadataAsMap.put(KNNConstants.VECTOR_DATA_TYPE_FIELD, VectorDataType.DEFAULT.getValue()); XContentBuilder builder = XContentFactory.jsonBuilder().startObject(); builder = methodComponentContext.toXContent(builder, ToXContent.EMPTY_PARAMS).endObject(); @@ -678,7 +738,8 @@ public void testBlockCommasInDescription() { description, error, nodeAssignment, - methodComponentContext + methodComponentContext, + VectorDataType.DEFAULT ) ); assertEquals("Model description cannot contain any commas: ','", e.getMessage()); diff --git a/src/test/java/org/opensearch/knn/indices/ModelTests.java b/src/test/java/org/opensearch/knn/indices/ModelTests.java index 13579acad..59bfe035f 100644 --- a/src/test/java/org/opensearch/knn/indices/ModelTests.java +++ b/src/test/java/org/opensearch/knn/indices/ModelTests.java @@ -15,6 +15,7 @@ import org.opensearch.knn.common.KNNConstants; import org.opensearch.knn.index.MethodComponentContext; import org.opensearch.knn.index.SpaceType; +import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.util.KNNEngine; import java.time.ZoneOffset; @@ -41,7 +42,8 @@ public void testInvalidConstructor() { "", "", "", - MethodComponentContext.EMPTY + MethodComponentContext.EMPTY, + VectorDataType.DEFAULT ), null, "test-model" @@ -62,7 +64,8 @@ public void testInvalidDimension() { "", "", "", - MethodComponentContext.EMPTY + MethodComponentContext.EMPTY, + VectorDataType.DEFAULT ), new byte[16], "test-model" @@ -80,7 +83,8 @@ public void testInvalidDimension() { "", "", "", - MethodComponentContext.EMPTY + MethodComponentContext.EMPTY, + VectorDataType.DEFAULT ), new byte[16], "test-model" @@ -98,7 +102,8 @@ public void testInvalidDimension() { "", "", "", - MethodComponentContext.EMPTY + MethodComponentContext.EMPTY, + VectorDataType.DEFAULT ), new byte[16], "test-model" @@ -117,7 +122,8 @@ public void testGetModelMetadata() { "", "", "", - MethodComponentContext.EMPTY + MethodComponentContext.EMPTY, + VectorDataType.DEFAULT ); Model model = new Model(modelMetadata, new byte[16], "test-model"); assertEquals(modelMetadata, model.getModelMetadata()); @@ -135,7 +141,8 @@ public void testGetModelBlob() { "", "", "", - MethodComponentContext.EMPTY + MethodComponentContext.EMPTY, + VectorDataType.DEFAULT ), modelBlob, "test-model" @@ -155,7 +162,8 @@ public void testGetLength() { "", "", "", - MethodComponentContext.EMPTY + MethodComponentContext.EMPTY, + VectorDataType.DEFAULT ), new byte[size], "test-model" @@ -172,7 +180,8 @@ public void testGetLength() { "", "", "", - MethodComponentContext.EMPTY + MethodComponentContext.EMPTY, + VectorDataType.DEFAULT ), null, "test-model" @@ -192,7 +201,8 @@ public void testSetModelBlob() { "", "", "", - MethodComponentContext.EMPTY + MethodComponentContext.EMPTY, + VectorDataType.DEFAULT ), blob1, "test-model" @@ -209,17 +219,50 @@ public void testEquals() { String time = ZonedDateTime.now(ZoneOffset.UTC).toString(); Model model1 = new Model( - new ModelMetadata(KNNEngine.DEFAULT, SpaceType.L1, 2, ModelState.CREATED, time, "", "", "", MethodComponentContext.EMPTY), + new ModelMetadata( + KNNEngine.DEFAULT, + SpaceType.L1, + 2, + ModelState.CREATED, + time, + "", + "", + "", + MethodComponentContext.EMPTY, + VectorDataType.DEFAULT + ), new byte[16], "test-model-1" ); Model model2 = new Model( - new ModelMetadata(KNNEngine.DEFAULT, SpaceType.L1, 2, ModelState.CREATED, time, "", "", "", MethodComponentContext.EMPTY), + new ModelMetadata( + KNNEngine.DEFAULT, + SpaceType.L1, + 2, + ModelState.CREATED, + time, + "", + "", + "", + MethodComponentContext.EMPTY, + VectorDataType.DEFAULT + ), new byte[16], "test-model-1" ); Model model3 = new Model( - new ModelMetadata(KNNEngine.DEFAULT, SpaceType.L2, 2, ModelState.CREATED, time, "", "", "", MethodComponentContext.EMPTY), + new ModelMetadata( + KNNEngine.DEFAULT, + SpaceType.L2, + 2, + ModelState.CREATED, + time, + "", + "", + "", + MethodComponentContext.EMPTY, + VectorDataType.DEFAULT + ), new byte[16], "test-model-2" ); @@ -234,17 +277,50 @@ public void testHashCode() { String time = ZonedDateTime.now(ZoneOffset.UTC).toString(); Model model1 = new Model( - new ModelMetadata(KNNEngine.DEFAULT, SpaceType.L1, 2, ModelState.CREATED, time, "", "", "", MethodComponentContext.EMPTY), + new ModelMetadata( + KNNEngine.DEFAULT, + SpaceType.L1, + 2, + ModelState.CREATED, + time, + "", + "", + "", + MethodComponentContext.EMPTY, + VectorDataType.DEFAULT + ), new byte[16], "test-model-1" ); Model model2 = new Model( - new ModelMetadata(KNNEngine.DEFAULT, SpaceType.L1, 2, ModelState.CREATED, time, "", "", "", MethodComponentContext.EMPTY), + new ModelMetadata( + KNNEngine.DEFAULT, + SpaceType.L1, + 2, + ModelState.CREATED, + time, + "", + "", + "", + MethodComponentContext.EMPTY, + VectorDataType.DEFAULT + ), new byte[16], "test-model-1" ); Model model3 = new Model( - new ModelMetadata(KNNEngine.DEFAULT, SpaceType.L1, 2, ModelState.CREATED, time, "", "", "", MethodComponentContext.EMPTY), + new ModelMetadata( + KNNEngine.DEFAULT, + SpaceType.L1, + 2, + ModelState.CREATED, + time, + "", + "", + "", + MethodComponentContext.EMPTY, + VectorDataType.DEFAULT + ), new byte[16], "test-model-2" ); @@ -274,7 +350,8 @@ public void testModelFromSourceMap() { description, error, nodeAssignment, - MethodComponentContext.EMPTY + MethodComponentContext.EMPTY, + VectorDataType.DEFAULT ); Map modelAsMap = new HashMap<>(); modelAsMap.put(KNNConstants.MODEL_ID, modelID); @@ -287,6 +364,7 @@ public void testModelFromSourceMap() { modelAsMap.put(KNNConstants.MODEL_ERROR, error); modelAsMap.put(KNNConstants.MODEL_NODE_ASSIGNMENT, nodeAssignment); modelAsMap.put(KNNConstants.MODEL_BLOB_PARAMETER, "aGVsbG8="); + modelAsMap.put(KNNConstants.VECTOR_DATA_TYPE_FIELD, VectorDataType.DEFAULT.getValue()); byte[] blob1 = "hello".getBytes(); Model expected = new Model(metadata, blob1, modelID); diff --git a/src/test/java/org/opensearch/knn/plugin/action/RestKNNStatsHandlerIT.java b/src/test/java/org/opensearch/knn/plugin/action/RestKNNStatsHandlerIT.java index cf7b869f8..850f24bb7 100644 --- a/src/test/java/org/opensearch/knn/plugin/action/RestKNNStatsHandlerIT.java +++ b/src/test/java/org/opensearch/knn/plugin/action/RestKNNStatsHandlerIT.java @@ -35,7 +35,21 @@ import static org.opensearch.knn.TestUtils.*; import static org.opensearch.knn.TestUtils.PROPERTIES; -import static org.opensearch.knn.common.KNNConstants.*; +import static org.opensearch.knn.TestUtils.VECTOR_TYPE; +import static org.opensearch.knn.common.KNNConstants.FAISS_NAME; +import static org.opensearch.knn.common.KNNConstants.KNN_ENGINE; +import static org.opensearch.knn.common.KNNConstants.LUCENE_NAME; +import static org.opensearch.knn.common.KNNConstants.MAX_DISTANCE; +import static org.opensearch.knn.common.KNNConstants.METHOD_HNSW; +import static org.opensearch.knn.common.KNNConstants.METHOD_IVF; +import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_NLIST; +import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_SPACE_TYPE; +import static org.opensearch.knn.common.KNNConstants.MIN_SCORE; +import static org.opensearch.knn.common.KNNConstants.MODEL_ID; +import static org.opensearch.knn.common.KNNConstants.MODEL_INDEX_NAME; +import static org.opensearch.knn.common.KNNConstants.NMSLIB_NAME; +import static org.opensearch.knn.common.KNNConstants.PARAMETERS; +import static org.opensearch.knn.common.KNNConstants.NAME; /** * Integration tests to check the correctness of RestKNNStatsHandler diff --git a/src/test/java/org/opensearch/knn/plugin/transport/GetModelResponseTests.java b/src/test/java/org/opensearch/knn/plugin/transport/GetModelResponseTests.java index 2a59874d5..06cebff7b 100644 --- a/src/test/java/org/opensearch/knn/plugin/transport/GetModelResponseTests.java +++ b/src/test/java/org/opensearch/knn/plugin/transport/GetModelResponseTests.java @@ -21,6 +21,7 @@ import org.opensearch.knn.index.KNNClusterUtil; import org.opensearch.knn.index.MethodComponentContext; import org.opensearch.knn.index.SpaceType; +import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.util.KNNEngine; import org.opensearch.knn.indices.Model; import org.opensearch.knn.indices.ModelMetadata; @@ -44,7 +45,8 @@ private ModelMetadata getModelMetadata(ModelState state) { "test model", "", "", - MethodComponentContext.EMPTY + MethodComponentContext.EMPTY, + VectorDataType.DEFAULT ); } @@ -69,7 +71,7 @@ public void testXContent() throws IOException { Model model = new Model(getModelMetadata(ModelState.CREATED), testModelBlob, modelId); GetModelResponse getModelResponse = new GetModelResponse(model); String expectedResponseString = - "{\"model_id\":\"test-model\",\"model_blob\":\"aGVsbG8=\",\"state\":\"created\",\"timestamp\":\"2021-03-27 10:15:30 AM +05:30\",\"description\":\"test model\",\"error\":\"\",\"space_type\":\"l2\",\"dimension\":4,\"engine\":\"nmslib\",\"training_node_assignment\":\"\",\"model_definition\":{\"name\":\"\",\"parameters\":{}}}"; + "{\"model_id\":\"test-model\",\"model_blob\":\"aGVsbG8=\",\"state\":\"created\",\"timestamp\":\"2021-03-27 10:15:30 AM +05:30\",\"description\":\"test model\",\"error\":\"\",\"space_type\":\"l2\",\"dimension\":4,\"engine\":\"nmslib\",\"training_node_assignment\":\"\",\"model_definition\":{\"name\":\"\",\"parameters\":{}},\"data_type\":\"float\"}"; XContentBuilder xContentBuilder = MediaTypeRegistry.contentBuilder(XContentType.JSON); getModelResponse.toXContent(xContentBuilder, null); assertEquals(expectedResponseString, xContentBuilder.toString()); @@ -85,7 +87,7 @@ public void testXContentWithNoModelBlob() throws IOException { Model model = new Model(getModelMetadata(ModelState.FAILED), null, modelId); GetModelResponse getModelResponse = new GetModelResponse(model); String expectedResponseString = - "{\"model_id\":\"test-model\",\"model_blob\":\"\",\"state\":\"failed\",\"timestamp\":\"2021-03-27 10:15:30 AM +05:30\",\"description\":\"test model\",\"error\":\"\",\"space_type\":\"l2\",\"dimension\":4,\"engine\":\"nmslib\",\"training_node_assignment\":\"\",\"model_definition\":{\"name\":\"\",\"parameters\":{}}}"; + "{\"model_id\":\"test-model\",\"model_blob\":\"\",\"state\":\"failed\",\"timestamp\":\"2021-03-27 10:15:30 AM +05:30\",\"description\":\"test model\",\"error\":\"\",\"space_type\":\"l2\",\"dimension\":4,\"engine\":\"nmslib\",\"training_node_assignment\":\"\",\"model_definition\":{\"name\":\"\",\"parameters\":{}},\"data_type\":\"float\"}"; XContentBuilder xContentBuilder = MediaTypeRegistry.contentBuilder(XContentType.JSON); getModelResponse.toXContent(xContentBuilder, null); assertEquals(expectedResponseString, xContentBuilder.toString()); diff --git a/src/test/java/org/opensearch/knn/plugin/transport/RemoveModelFromCacheTransportActionTests.java b/src/test/java/org/opensearch/knn/plugin/transport/RemoveModelFromCacheTransportActionTests.java index a2da83dad..381831fc7 100644 --- a/src/test/java/org/opensearch/knn/plugin/transport/RemoveModelFromCacheTransportActionTests.java +++ b/src/test/java/org/opensearch/knn/plugin/transport/RemoveModelFromCacheTransportActionTests.java @@ -19,6 +19,7 @@ import org.opensearch.knn.KNNSingleNodeTestCase; import org.opensearch.knn.index.MethodComponentContext; import org.opensearch.knn.index.SpaceType; +import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.util.KNNEngine; import org.opensearch.knn.indices.Model; import org.opensearch.knn.indices.ModelCache; @@ -78,7 +79,8 @@ public void testNodeOperation_modelInCache() throws ExecutionException, Interrup "description", "", "", - MethodComponentContext.EMPTY + MethodComponentContext.EMPTY, + VectorDataType.DEFAULT ), new byte[128], modelId diff --git a/src/test/java/org/opensearch/knn/plugin/transport/TrainingJobRouterTransportActionTests.java b/src/test/java/org/opensearch/knn/plugin/transport/TrainingJobRouterTransportActionTests.java index 56c50aca1..63c770a26 100644 --- a/src/test/java/org/opensearch/knn/plugin/transport/TrainingJobRouterTransportActionTests.java +++ b/src/test/java/org/opensearch/knn/plugin/transport/TrainingJobRouterTransportActionTests.java @@ -24,6 +24,7 @@ import org.opensearch.cluster.service.ClusterService; import org.opensearch.knn.KNNTestCase; import org.opensearch.knn.index.KNNMethodContext; +import org.opensearch.knn.index.VectorDataType; import org.opensearch.search.SearchHit; import org.opensearch.search.SearchHits; import org.opensearch.transport.TransportService; @@ -307,7 +308,8 @@ public void testTrainingIndexSize() { trainingIndexName, "training-field", null, - "description" + "description", + VectorDataType.DEFAULT ); // Mock client to return the right number of docs @@ -339,6 +341,102 @@ public void testTrainingIndexSize() { transportAction.getTrainingIndexSizeInKB(trainingModelRequest, listener); } + public void testTrainIndexSize_whenDataTypeIsBinary() { + String trainingIndexName = "training-index"; + int dimension = 8; + int vectorCount = 1000000; + int expectedSize = Byte.BYTES * (dimension / 8) * vectorCount / BYTES_PER_KILOBYTES + 1; // 977 KB + + // Setup the request + TrainingModelRequest trainingModelRequest = new TrainingModelRequest( + null, + KNNMethodContext.getDefault(), + dimension, + trainingIndexName, + "training-field", + null, + "description", + VectorDataType.BINARY + ); + + // Mock client to return the right number of docs + TotalHits totalHits = new TotalHits(vectorCount, TotalHits.Relation.EQUAL_TO); + SearchHits searchHits = new SearchHits(new SearchHit[2], totalHits, 1.0f); + SearchResponse searchResponse = mock(SearchResponse.class); + when(searchResponse.getHits()).thenReturn(searchHits); + Client client = mock(Client.class); + + doAnswer(invocationOnMock -> { + ((ActionListener) invocationOnMock.getArguments()[1]).onResponse(searchResponse); + return null; + }).when(client).search(any(), any()); + + // Setup the action + ClusterService clusterService = mock(ClusterService.class); + TransportService transportService = mock(TransportService.class); + TrainingJobRouterTransportAction transportAction = new TrainingJobRouterTransportAction( + transportService, + new ActionFilters(Collections.emptySet()), + clusterService, + client + ); + + ActionListener listener = ActionListener.wrap( + size -> assertEquals(expectedSize, size.intValue()), + e -> fail(e.getMessage()) + ); + + transportAction.getTrainingIndexSizeInKB(trainingModelRequest, listener); + } + + public void testTrainIndexSize_whenDataTypeIsByte() { + String trainingIndexName = "training-index"; + int dimension = 8; + int vectorCount = 1000000; + int expectedSize = Byte.BYTES * dimension * vectorCount / BYTES_PER_KILOBYTES + 1; // 7813 KB + + // Setup the request + TrainingModelRequest trainingModelRequest = new TrainingModelRequest( + null, + KNNMethodContext.getDefault(), + dimension, + trainingIndexName, + "training-field", + null, + "description", + VectorDataType.BYTE + ); + + // Mock client to return the right number of docs + TotalHits totalHits = new TotalHits(vectorCount, TotalHits.Relation.EQUAL_TO); + SearchHits searchHits = new SearchHits(new SearchHit[2], totalHits, 1.0f); + SearchResponse searchResponse = mock(SearchResponse.class); + when(searchResponse.getHits()).thenReturn(searchHits); + Client client = mock(Client.class); + + doAnswer(invocationOnMock -> { + ((ActionListener) invocationOnMock.getArguments()[1]).onResponse(searchResponse); + return null; + }).when(client).search(any(), any()); + + // Setup the action + ClusterService clusterService = mock(ClusterService.class); + TransportService transportService = mock(TransportService.class); + TrainingJobRouterTransportAction transportAction = new TrainingJobRouterTransportAction( + transportService, + new ActionFilters(Collections.emptySet()), + clusterService, + client + ); + + ActionListener listener = ActionListener.wrap( + size -> assertEquals(expectedSize, size.intValue()), + e -> fail(e.getMessage()) + ); + + transportAction.getTrainingIndexSizeInKB(trainingModelRequest, listener); + } + private Map generateDiscoveryNodes(List dataNodeIds) { Map nodes = new HashMap<>(); diff --git a/src/test/java/org/opensearch/knn/plugin/transport/TrainingModelRequestTests.java b/src/test/java/org/opensearch/knn/plugin/transport/TrainingModelRequestTests.java index b39c48635..9434a6e41 100644 --- a/src/test/java/org/opensearch/knn/plugin/transport/TrainingModelRequestTests.java +++ b/src/test/java/org/opensearch/knn/plugin/transport/TrainingModelRequestTests.java @@ -25,6 +25,7 @@ import org.opensearch.knn.common.KNNConstants; import org.opensearch.knn.index.KNNMethodContext; import org.opensearch.knn.index.MethodComponentContext; +import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.mapper.KNNVectorFieldMapper; import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.util.KNNEngine; @@ -61,7 +62,8 @@ public void testStreams() throws IOException { trainingIndex, trainingField, preferredNode, - description + description, + VectorDataType.DEFAULT ); BytesStreamOutput streamOutput = new BytesStreamOutput(); @@ -74,6 +76,7 @@ public void testStreams() throws IOException { assertEquals(original1.getTrainingIndex(), copy1.getTrainingIndex()); assertEquals(original1.getTrainingField(), copy1.getTrainingField()); assertEquals(original1.getPreferredNodeId(), copy1.getPreferredNodeId()); + assertEquals(original1.getVectorDataType(), copy1.getVectorDataType()); // Also, check when preferred node and model id and description are null TrainingModelRequest original2 = new TrainingModelRequest( @@ -83,7 +86,8 @@ public void testStreams() throws IOException { trainingIndex, trainingField, null, - null + null, + VectorDataType.DEFAULT ); streamOutput = new BytesStreamOutput(); @@ -96,6 +100,7 @@ public void testStreams() throws IOException { assertEquals(original2.getTrainingIndex(), copy2.getTrainingIndex()); assertEquals(original2.getTrainingField(), copy2.getTrainingField()); assertEquals(original2.getPreferredNodeId(), copy2.getPreferredNodeId()); + assertEquals(original2.getVectorDataType(), copy2.getVectorDataType()); } public void testGetters() { @@ -117,7 +122,8 @@ public void testGetters() { trainingIndex, trainingField, preferredNode, - description + description, + VectorDataType.DEFAULT ); trainingModelRequest.setMaximumVectorCount(maxVectorCount); @@ -156,7 +162,8 @@ public void testValidation_invalid_modelIdAlreadyExists() { trainingIndex, trainingField, null, - null + null, + VectorDataType.DEFAULT ); // Mock the model dao to return metadata for modelId to recognize it is a duplicate @@ -170,7 +177,8 @@ public void testValidation_invalid_modelIdAlreadyExists() { "", "", "", - MethodComponentContext.EMPTY + MethodComponentContext.EMPTY, + VectorDataType.DEFAULT ); when(modelDao.getMetadata(modelId)).thenReturn(modelMetadata); @@ -211,7 +219,8 @@ public void testValidation_blocked_modelId() { trainingIndex, trainingField, null, - null + null, + VectorDataType.DEFAULT ); // Mock the model dao to return true to recognize that the modelId is in graveyard @@ -257,7 +266,8 @@ public void testValidation_invalid_invalidMethodContext() { trainingIndex, trainingField, null, - null + null, + VectorDataType.DEFAULT ); // Mock the model dao to return null so that no exception is produced @@ -300,7 +310,8 @@ public void testValidation_invalid_trainingIndexDoesNotExist() { trainingIndex, trainingField, null, - null + null, + VectorDataType.DEFAULT ); // Mock the model dao to return null so that no exception is produced @@ -346,7 +357,8 @@ public void testValidation_invalid_trainingFieldDoesNotExist() { trainingIndex, trainingField, null, - null + null, + VectorDataType.DEFAULT ); // Mock the model dao to return null so that no exception is produced @@ -397,7 +409,8 @@ public void testValidation_invalid_trainingFieldNotKnnVector() { trainingIndex, trainingField, null, - null + null, + VectorDataType.DEFAULT ); // Mock the model dao to return null so that no exception is produced @@ -452,7 +465,8 @@ public void testValidation_invalid_dimensionDoesNotMatch() { trainingIndex, trainingField, null, - null + null, + VectorDataType.DEFAULT ); // Mock the model dao to return null so that no exception is produced @@ -509,7 +523,8 @@ public void testValidation_invalid_preferredNodeDoesNotExist() { trainingIndex, trainingField, preferredNode, - null + null, + VectorDataType.DEFAULT ); // Mock the model dao to return metadata for modelId to recognize it is a duplicate @@ -574,7 +589,8 @@ public void testValidation_invalid_descriptionToLong() { trainingIndex, trainingField, null, - description + description, + VectorDataType.DEFAULT ); // Mock the model dao to return metadata for modelId to recognize it is a duplicate @@ -618,7 +634,8 @@ public void testValidation_valid_trainingIndexBuiltFromMethod() { trainingIndex, trainingField, null, - null + null, + VectorDataType.DEFAULT ); // Mock the model dao to return metadata for modelId to recognize it is a duplicate @@ -655,7 +672,8 @@ public void testValidation_valid_trainingIndexBuiltFromModel() { trainingIndex, trainingField, null, - null + null, + VectorDataType.DEFAULT ); // Mock the model dao to return metadata for modelId to recognize it is a duplicate diff --git a/src/test/java/org/opensearch/knn/plugin/transport/TrainingModelTransportActionTests.java b/src/test/java/org/opensearch/knn/plugin/transport/TrainingModelTransportActionTests.java index 950ce1fd0..9ca790350 100644 --- a/src/test/java/org/opensearch/knn/plugin/transport/TrainingModelTransportActionTests.java +++ b/src/test/java/org/opensearch/knn/plugin/transport/TrainingModelTransportActionTests.java @@ -17,6 +17,7 @@ import org.opensearch.knn.KNNSingleNodeTestCase; import org.opensearch.knn.index.KNNMethodContext; import org.opensearch.knn.index.SpaceType; +import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.util.KNNEngine; import org.opensearch.knn.indices.ModelDao; @@ -72,9 +73,10 @@ public void testDoExecute() throws InterruptedException, ExecutionException, IOE trainingIndexName, trainingFieldName, null, - "test-detector" + "test-detector", + VectorDataType.DEFAULT ); - trainingModelRequest.setTrainingDataSizeInKB(estimateVectorSetSizeInKB(trainingDataCount, dimension)); + trainingModelRequest.setTrainingDataSizeInKB(estimateVectorSetSizeInKB(trainingDataCount, dimension, VectorDataType.DEFAULT)); // Create listener that ensures that the initial model put succeeds ActionListener listener = ActionListener.wrap( diff --git a/src/test/java/org/opensearch/knn/plugin/transport/UpdateModelGraveyardTransportActionTests.java b/src/test/java/org/opensearch/knn/plugin/transport/UpdateModelGraveyardTransportActionTests.java index 5be907ebd..bad8d368b 100644 --- a/src/test/java/org/opensearch/knn/plugin/transport/UpdateModelGraveyardTransportActionTests.java +++ b/src/test/java/org/opensearch/knn/plugin/transport/UpdateModelGraveyardTransportActionTests.java @@ -17,6 +17,7 @@ import org.opensearch.knn.common.exception.DeleteModelException; import org.opensearch.knn.index.MethodComponentContext; import org.opensearch.knn.index.SpaceType; +import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.util.KNNEngine; import org.opensearch.knn.indices.Model; import org.opensearch.knn.indices.ModelGraveyard; @@ -210,7 +211,8 @@ public void testClusterManagerOperation_GetIndicesUsingModel() throws IOExceptio "", "", "", - MethodComponentContext.EMPTY + MethodComponentContext.EMPTY, + VectorDataType.DEFAULT ), modelBlob, modelId diff --git a/src/test/java/org/opensearch/knn/plugin/transport/UpdateModelMetadataRequestTests.java b/src/test/java/org/opensearch/knn/plugin/transport/UpdateModelMetadataRequestTests.java index 3719d124a..d2291c4ea 100644 --- a/src/test/java/org/opensearch/knn/plugin/transport/UpdateModelMetadataRequestTests.java +++ b/src/test/java/org/opensearch/knn/plugin/transport/UpdateModelMetadataRequestTests.java @@ -15,6 +15,7 @@ import org.opensearch.knn.KNNTestCase; import org.opensearch.knn.index.MethodComponentContext; import org.opensearch.knn.index.SpaceType; +import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.util.KNNEngine; import org.opensearch.knn.indices.ModelMetadata; import org.opensearch.knn.indices.ModelState; @@ -42,7 +43,8 @@ public void testStreams() throws IOException { "", "", "", - MethodComponentContext.EMPTY + MethodComponentContext.EMPTY, + VectorDataType.DEFAULT ); UpdateModelMetadataRequest updateModelMetadataRequest = new UpdateModelMetadataRequest(modelId, isRemoveRequest, modelMetadata); @@ -67,7 +69,8 @@ public void testValidate() { "", "", "", - MethodComponentContext.EMPTY + MethodComponentContext.EMPTY, + VectorDataType.DEFAULT ); UpdateModelMetadataRequest updateModelMetadataRequest1 = new UpdateModelMetadataRequest("test", true, null); @@ -107,7 +110,8 @@ public void testGetModelMetadata() { "", "", "", - MethodComponentContext.EMPTY + MethodComponentContext.EMPTY, + VectorDataType.DEFAULT ); UpdateModelMetadataRequest updateModelMetadataRequest = new UpdateModelMetadataRequest("test", true, modelMetadata); diff --git a/src/test/java/org/opensearch/knn/plugin/transport/UpdateModelMetadataTransportActionTests.java b/src/test/java/org/opensearch/knn/plugin/transport/UpdateModelMetadataTransportActionTests.java index ab0e4f506..c35c7effb 100644 --- a/src/test/java/org/opensearch/knn/plugin/transport/UpdateModelMetadataTransportActionTests.java +++ b/src/test/java/org/opensearch/knn/plugin/transport/UpdateModelMetadataTransportActionTests.java @@ -19,6 +19,7 @@ import org.opensearch.knn.KNNSingleNodeTestCase; import org.opensearch.knn.index.MethodComponentContext; import org.opensearch.knn.index.SpaceType; +import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.util.KNNEngine; import org.opensearch.knn.indices.ModelMetadata; import org.opensearch.knn.indices.ModelState; @@ -68,7 +69,8 @@ public void testClusterManagerOperation() throws InterruptedException { "", "", "", - MethodComponentContext.EMPTY + MethodComponentContext.EMPTY, + VectorDataType.DEFAULT ); // Get update transport action diff --git a/src/test/java/org/opensearch/knn/training/TrainingDataConsumerTests.java b/src/test/java/org/opensearch/knn/training/FloatTrainingDataConsumerTests.java similarity index 88% rename from src/test/java/org/opensearch/knn/training/TrainingDataConsumerTests.java rename to src/test/java/org/opensearch/knn/training/FloatTrainingDataConsumerTests.java index d5a66c5b6..27e02b46b 100644 --- a/src/test/java/org/opensearch/knn/training/TrainingDataConsumerTests.java +++ b/src/test/java/org/opensearch/knn/training/FloatTrainingDataConsumerTests.java @@ -23,7 +23,7 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; -public class TrainingDataConsumerTests extends KNNTestCase { +public class FloatTrainingDataConsumerTests extends KNNTestCase { public void testAccept() { @@ -38,7 +38,7 @@ public void testAccept() { // Capture argument passed to set pointer ArgumentCaptor valueCapture = ArgumentCaptor.forClass(Long.class); - TrainingDataConsumer trainingDataConsumer = new TrainingDataConsumer(trainingDataAllocation); + FloatTrainingDataConsumer floatTrainingDataConsumer = new FloatTrainingDataConsumer(trainingDataAllocation); List vectorSet1 = new ArrayList<>(3); for (int i = 0; i < 3; i++) { @@ -47,10 +47,8 @@ public void testAccept() { vectorSet1.add(vector); } - when(trainingDataAllocation.getMemoryAddress()).thenReturn(0L); - // Transfer vectors - trainingDataConsumer.accept(vectorSet1); + floatTrainingDataConsumer.accept(vectorSet1); // Ensure that the pointer captured has been updated verify(trainingDataAllocation).setMemoryAddress(valueCapture.capture()); diff --git a/src/test/java/org/opensearch/knn/training/TrainingJobTests.java b/src/test/java/org/opensearch/knn/training/TrainingJobTests.java index 06b96c57c..0852c39de 100644 --- a/src/test/java/org/opensearch/knn/training/TrainingJobTests.java +++ b/src/test/java/org/opensearch/knn/training/TrainingJobTests.java @@ -18,6 +18,7 @@ import org.opensearch.knn.index.KNNMethodContext; import org.opensearch.knn.index.MethodComponentContext; import org.opensearch.knn.index.SpaceType; +import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.memory.NativeMemoryAllocation; import org.opensearch.knn.index.memory.NativeMemoryCacheManager; import org.opensearch.knn.index.memory.NativeMemoryEntryContext; @@ -67,7 +68,8 @@ public void testGetModelId() { mock(NativeMemoryEntryContext.AnonymousEntryContext.class), 10, "", - "test-node" + "test-node", + VectorDataType.DEFAULT ); assertEquals(modelId, trainingJob.getModelId()); @@ -96,7 +98,8 @@ public void testGetModel() { mock(NativeMemoryEntryContext.AnonymousEntryContext.class), dimension, description, - nodeAssignment + nodeAssignment, + VectorDataType.DEFAULT ); Model model = new Model( @@ -109,7 +112,8 @@ public void testGetModel() { description, error, nodeAssignment, - MethodComponentContext.EMPTY + MethodComponentContext.EMPTY, + VectorDataType.DEFAULT ), null, modelID @@ -183,8 +187,8 @@ public void testRun_success() throws IOException, ExecutionException { modelContext, dimension, "", - "test-node" - + "test-node", + VectorDataType.DEFAULT ); trainingJob.run(); @@ -262,8 +266,8 @@ public void testRun_failure_onGetTrainingDataAllocation() throws ExecutionExcept modelContext, dimension, "", - - "test-node" + "test-node", + VectorDataType.DEFAULT ); trainingJob.run(); @@ -330,8 +334,8 @@ public void testRun_failure_onGetModelAnonymousAllocation() throws ExecutionExce modelContext, dimension, "", - - "test-node" + "test-node", + VectorDataType.DEFAULT ); trainingJob.run(); @@ -397,7 +401,8 @@ public void testRun_failure_closedTrainingDataAllocation() throws ExecutionExcep mock(NativeMemoryEntryContext.AnonymousEntryContext.class), dimension, "", - "test-node" + "test-node", + VectorDataType.DEFAULT ); trainingJob.run(); @@ -470,7 +475,8 @@ public void testRun_failure_notEnoughTrainingData() throws ExecutionException { modelContext, dimension, "", - "test-node" + "test-node", + VectorDataType.DEFAULT ); trainingJob.run(); diff --git a/src/test/java/org/opensearch/knn/training/VectorReaderTests.java b/src/test/java/org/opensearch/knn/training/VectorReaderTests.java index 209c9cc73..b69b43a39 100644 --- a/src/test/java/org/opensearch/knn/training/VectorReaderTests.java +++ b/src/test/java/org/opensearch/knn/training/VectorReaderTests.java @@ -8,25 +8,23 @@ * Modifications Copyright OpenSearch Contributors. See * GitHub history for details. */ - package org.opensearch.knn.training; +import lombok.Getter; import org.opensearch.core.action.ActionListener; import org.opensearch.action.search.SearchResponse; import org.opensearch.cluster.service.ClusterService; -import org.opensearch.common.ValidationException; import org.opensearch.knn.KNNSingleNodeTestCase; +import org.opensearch.knn.index.VectorDataType; +import org.opensearch.knn.index.memory.NativeMemoryAllocation; +import org.opensearch.common.ValidationException; +import org.opensearch.search.SearchHit; import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashSet; -import java.util.List; -import java.util.Random; +import java.util.*; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; -import java.util.function.Consumer; import java.util.stream.Collectors; public class VectorReaderTests extends KNNSingleNodeTestCase { @@ -36,9 +34,9 @@ public class VectorReaderTests extends KNNSingleNodeTestCase { private final static String DEFAULT_FIELD_NAME = "test-field"; private final static String DEFAULT_NESTED_FIELD_PATH = "a.b.test-field"; private final static int DEFAULT_DIMENSION = 16; - private final static int DEFAULT_NUM_VECTORS = 100; + private final static int DEFAULT_NUM_VECTORS = 50; private final static int DEFAULT_MAX_VECTOR_COUNT = 10000; - private final static int DEFAULT_SEARCH_SIZE = 10; + private final static int DEFAULT_SEARCH_SIZE = 120; public void testRead_valid_completeIndex() throws InterruptedException, ExecutionException, IOException { createIndex(DEFAULT_INDEX_NAME); @@ -56,9 +54,9 @@ public void testRead_valid_completeIndex() throws InterruptedException, Executio // Configure VectorReader ClusterService clusterService = node().injector().getInstance(ClusterService.class); VectorReader vectorReader = new VectorReader(client()); + TestFloatTrainingDataConsumer trainingDataConsumer = new TestFloatTrainingDataConsumer(createMockTrainingDataAllocation()); // Read all vectors and confirm they match vectors - TestVectorConsumer testVectorConsumer = new TestVectorConsumer(); final CountDownLatch inProgressLatch = new CountDownLatch(1); vectorReader.read( clusterService, @@ -66,14 +64,14 @@ public void testRead_valid_completeIndex() throws InterruptedException, Executio DEFAULT_FIELD_NAME, DEFAULT_MAX_VECTOR_COUNT, DEFAULT_SEARCH_SIZE, - testVectorConsumer, + trainingDataConsumer, createOnSearchResponseCountDownListener(inProgressLatch) ); assertLatchDecremented(inProgressLatch); - List consumedVectors = testVectorConsumer.getVectorsConsumed(); - assertEquals(DEFAULT_NUM_VECTORS, consumedVectors.size()); + List consumedVectors = trainingDataConsumer.getTotalAddedVectors(); + assertEquals(DEFAULT_NUM_VECTORS, trainingDataConsumer.getTotalVectorsCountAdded()); List flatVectors = vectors.stream().flatMap(Arrays::stream).collect(Collectors.toList()); List flatConsumedVectors = consumedVectors.stream().flatMap(Arrays::stream).collect(Collectors.toList()); @@ -98,21 +96,21 @@ public void testRead_valid_trainVectorsIngestedAsIntegers() throws IOException, VectorReader vectorReader = new VectorReader(client()); // Read all vectors and confirm they match vectors - TestVectorConsumer testVectorConsumer = new TestVectorConsumer(); final CountDownLatch inProgressLatch = new CountDownLatch(1); + TestFloatTrainingDataConsumer trainingDataConsumer = new TestFloatTrainingDataConsumer(createMockTrainingDataAllocation()); vectorReader.read( clusterService, DEFAULT_INDEX_NAME, DEFAULT_FIELD_NAME, DEFAULT_MAX_VECTOR_COUNT, DEFAULT_SEARCH_SIZE, - testVectorConsumer, + trainingDataConsumer, createOnSearchResponseCountDownListener(inProgressLatch) ); assertLatchDecremented(inProgressLatch); - List consumedVectors = testVectorConsumer.getVectorsConsumed(); + List consumedVectors = trainingDataConsumer.getTotalAddedVectors(); assertEquals(DEFAULT_NUM_VECTORS, consumedVectors.size()); List flatVectors = vectors.stream().flatMap(Arrays::stream).map(Integer::floatValue).collect(Collectors.toList()); @@ -149,21 +147,22 @@ public void testRead_valid_incompleteIndex() throws InterruptedException, Execut VectorReader vectorReader = new VectorReader(client()); // Read all vectors and confirm they match vectors - TestVectorConsumer testVectorConsumer = new TestVectorConsumer(); final CountDownLatch inProgressLatch = new CountDownLatch(1); + TestFloatTrainingDataConsumer trainingDataConsumer = new TestFloatTrainingDataConsumer(createMockTrainingDataAllocation()); + vectorReader.read( clusterService, DEFAULT_INDEX_NAME, DEFAULT_FIELD_NAME, DEFAULT_MAX_VECTOR_COUNT, DEFAULT_SEARCH_SIZE, - testVectorConsumer, + trainingDataConsumer, createOnSearchResponseCountDownListener(inProgressLatch) ); assertLatchDecremented(inProgressLatch); - List consumedVectors = testVectorConsumer.getVectorsConsumed(); + List consumedVectors = trainingDataConsumer.getTotalAddedVectors(); assertEquals(DEFAULT_NUM_VECTORS, consumedVectors.size()); List flatVectors = vectors.stream().flatMap(Arrays::stream).collect(Collectors.toList()); @@ -192,21 +191,21 @@ public void testRead_valid_OnlyGetMaxVectors() throws InterruptedException, Exec VectorReader vectorReader = new VectorReader(client()); // Read maxNumVectorsRead vectors - TestVectorConsumer testVectorConsumer = new TestVectorConsumer(); final CountDownLatch inProgressLatch = new CountDownLatch(1); + TestFloatTrainingDataConsumer trainingDataConsumer = new TestFloatTrainingDataConsumer(createMockTrainingDataAllocation()); vectorReader.read( clusterService, DEFAULT_INDEX_NAME, DEFAULT_FIELD_NAME, maxNumVectorsRead, DEFAULT_SEARCH_SIZE, - testVectorConsumer, + trainingDataConsumer, createOnSearchResponseCountDownListener(inProgressLatch) ); assertLatchDecremented(inProgressLatch); - List consumedVectors = testVectorConsumer.getVectorsConsumed(); + List consumedVectors = trainingDataConsumer.getTotalAddedVectors(); assertEquals(maxNumVectorsRead, consumedVectors.size()); } @@ -364,21 +363,21 @@ public void testRead_valid_NestedField() throws InterruptedException, ExecutionE VectorReader vectorReader = new VectorReader(client()); // Read all vectors and confirm they match vectors - TestVectorConsumer testVectorConsumer = new TestVectorConsumer(); final CountDownLatch inProgressLatch = new CountDownLatch(1); + TestFloatTrainingDataConsumer trainingDataConsumer = new TestFloatTrainingDataConsumer(createMockTrainingDataAllocation()); vectorReader.read( clusterService, DEFAULT_INDEX_NAME, DEFAULT_NESTED_FIELD_PATH, DEFAULT_MAX_VECTOR_COUNT, DEFAULT_SEARCH_SIZE, - testVectorConsumer, + trainingDataConsumer, createOnSearchResponseCountDownListener(inProgressLatch) ); assertLatchDecremented(inProgressLatch); - List consumedVectors = testVectorConsumer.getVectorsConsumed(); + List consumedVectors = trainingDataConsumer.getTotalAddedVectors(); assertEquals(DEFAULT_NUM_VECTORS, consumedVectors.size()); List flatVectors = vectors.stream().flatMap(Arrays::stream).collect(Collectors.toList()); @@ -386,29 +385,47 @@ public void testRead_valid_NestedField() throws InterruptedException, ExecutionE assertEquals(new HashSet<>(flatVectors), new HashSet<>(flatConsumedVectors)); } - private static class TestVectorConsumer implements Consumer> { + private void assertLatchDecremented(CountDownLatch countDownLatch) throws InterruptedException { + assertTrue(countDownLatch.await(DEFAULT_LATCH_TIMEOUT, TimeUnit.SECONDS)); + } + + private ActionListener createOnSearchResponseCountDownListener(CountDownLatch countDownLatch) { + return ActionListener.wrap(response -> countDownLatch.countDown(), Throwable::printStackTrace); + } + + private NativeMemoryAllocation.TrainingDataAllocation createMockTrainingDataAllocation() { + return new NativeMemoryAllocation.TrainingDataAllocation(null, 0, 0, VectorDataType.FLOAT); + } - List vectorsConsumed; + // create test float training data consumer class extending FloatTrainingDataConsumer + private static class TestFloatTrainingDataConsumer extends FloatTrainingDataConsumer { + @Getter + private List totalAddedVectors = new ArrayList<>(); - TestVectorConsumer() { - vectorsConsumed = new ArrayList<>(); + public TestFloatTrainingDataConsumer(NativeMemoryAllocation.TrainingDataAllocation trainingDataAllocation) { + super(trainingDataAllocation); } @Override - public void accept(List vectors) { - vectorsConsumed.addAll(vectors); - } + public void processTrainingVectors(SearchResponse searchResponse, int vectorsToAdd, String fieldName) { + SearchHit[] hits = searchResponse.getHits().getHits(); + List vectors = new ArrayList<>(); - public List getVectorsConsumed() { - return vectorsConsumed; - } - } + String[] fieldPath = fieldName.split("\\."); - private void assertLatchDecremented(CountDownLatch countDownLatch) throws InterruptedException { - assertTrue(countDownLatch.await(DEFAULT_LATCH_TIMEOUT, TimeUnit.SECONDS)); - } + for (int vector = 0; vector < vectorsToAdd; vector++) { + Object fieldValue = extractFieldValue(hits[vector], fieldPath); + if (!(fieldValue instanceof List)) { + continue; + } - private ActionListener createOnSearchResponseCountDownListener(CountDownLatch countDownLatch) { - return ActionListener.wrap(response -> countDownLatch.countDown(), Throwable::printStackTrace); + List fieldList = (List) fieldValue; + vectors.add(fieldList.stream().map(Number::floatValue).toArray(Float[]::new)); + } + + totalAddedVectors.addAll(vectors); + setTotalVectorsCountAdded(getTotalVectorsCountAdded() + vectors.size()); + accept(vectors); + } } } diff --git a/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java b/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java index 73e0bd724..df45fe2cc 100644 --- a/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java +++ b/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java @@ -6,6 +6,7 @@ package org.opensearch.knn; import com.google.common.primitives.Floats; +import com.google.common.primitives.Ints; import lombok.SneakyThrows; import lombok.extern.log4j.Log4j2; import org.apache.commons.lang.StringUtils; @@ -1033,6 +1034,28 @@ public void bulkIngestRandomVectors(String indexName, String fieldName, int numV } + /** + * Bulk ingest random binary vectors + * @param indexName index name + * @param fieldName field name + * @param numVectors number of vectors + * @param dimension vector dimension + */ + public void bulkIngestRandomBinaryVectors(String indexName, String fieldName, int numVectors, int dimension) throws IOException { + if (dimension % 8 != 0) { + throw new IllegalArgumentException("Dimension must be a multiple of 8"); + } + for (int i = 0; i < numVectors; i++) { + int binaryDimension = dimension / 8; + int[] vector = new int[binaryDimension]; + for (int j = 0; j < binaryDimension; j++) { + vector[j] = randomIntBetween(-128, 127); + } + + addKnnDoc(indexName, String.valueOf(i + 1), fieldName, Ints.asList(vector).toArray()); + } + } + /** * Bulk ingest random vectors with nested field * @@ -1370,6 +1393,18 @@ public Response trainModel( return client().performRequest(request); } + public Response trainModel(String modelId, XContentBuilder builder) throws IOException { + if (modelId == null) { + modelId = ""; + } else { + modelId = "/" + modelId; + } + + Request request = new Request("POST", "/_plugins/_knn/models" + modelId + "/_train"); + request.setJsonEntity(builder.toString()); + return client().performRequest(request); + } + /** * Retrieve the model * From c7cbba4abeef87e07d76f9bb4cdbbf0c83a9e181 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Mon, 22 Jul 2024 17:26:09 -0700 Subject: [PATCH 300/416] Adds release notes for 2.16 (#1866) Signed-off-by: Tejas Shah (cherry picked from commit aa5312ed89acdd1b86ca25f7e448ae24fcc1aee2) --- CHANGELOG.md | 18 ++----------- .../opensearch-knn.release-notes-2.16.0.0.md | 25 +++++++++++++++++++ 2 files changed, 27 insertions(+), 16 deletions(-) create mode 100644 release-notes/opensearch-knn.release-notes-2.16.0.0.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 85a91dab2..d10752c01 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,25 +12,11 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Maintenance ### Refactoring -## [Unreleased 2.x](https://github.com/opensearch-project/k-NN/compare/2.15...2.x) +## [Unreleased 2.x](https://github.com/opensearch-project/k-NN/compare/2.16...2.x) ### Features -* Adds dynamic query parameter ef_search [#1783](https://github.com/opensearch-project/k-NN/pull/1783) -* Adds dynamic query parameter ef_search in radial search faiss engine [#1790](https://github.com/opensearch-project/k-NN/pull/1790) -* Adds dynamic query parameter nprobes [#1792](https://github.com/opensearch-project/k-NN/pull/1792) -* Add binary format support with HNSW method in Faiss Engine [#1781](https://github.com/opensearch-project/k-NN/pull/1781) -* Add script scoring support for knn field with binary data type [#1826](https://github.com/opensearch-project/k-NN/pull/1826) -* Add painless script support for hamming with binary vector data type [#1839](https://github.com/opensearch-project/k-NN/pull/1839) -* Add binary format support with IVF method in Faiss Engine [#1784](https://github.com/opensearch-project/k-NN/pull/1784) ### Enhancements -* Switch from byte stream to byte ref for serde [#1825](https://github.com/opensearch-project/k-NN/pull/1825) ### Bug Fixes -* Fixing the arithmetic to find the number of vectors to stream from java to jni layer.[#1804](https://github.com/opensearch-project/k-NN/pull/1804) -* Fixed LeafReaders casting errors to SegmentReaders when segment replication is enabled during search.[#1808](https://github.com/opensearch-project/k-NN/pull/1808) -* Release memory properly for an array type [#1820](https://github.com/opensearch-project/k-NN/pull/1820) -* FIX Same Suffix Cause Recall Drop to zero [#1802](https://github.com/opensearch-project/k-NN/pull/1802) ### Infrastructure -* Apply custom patch only once by comparing the last patch id [#1833](https://github.com/opensearch-project/k-NN/pull/1833) ### Documentation ### Maintenance -* Bump faiss commit to 33c0ba5 [#1796](https://github.com/opensearch-project/k-NN/pull/1796) -### Refactoring +### Refactoring \ No newline at end of file diff --git a/release-notes/opensearch-knn.release-notes-2.16.0.0.md b/release-notes/opensearch-knn.release-notes-2.16.0.0.md new file mode 100644 index 000000000..c24d3552b --- /dev/null +++ b/release-notes/opensearch-knn.release-notes-2.16.0.0.md @@ -0,0 +1,25 @@ +## Version 2.16.0.0 Release Notes + +Compatible with OpenSearch 2.16.0 + +### Features +* Adds dynamic query parameter ef_search [#1783](https://github.com/opensearch-project/k-NN/pull/1783) +* Adds dynamic query parameter ef_search in radial search faiss engine [#1790](https://github.com/opensearch-project/k-NN/pull/1790) +* Adds dynamic query parameter nprobes [#1792](https://github.com/opensearch-project/k-NN/pull/1792) +* Add binary format support with HNSW method in Faiss Engine [#1781](https://github.com/opensearch-project/k-NN/pull/1781) +* Add script scoring support for knn field with binary data type [#1826](https://github.com/opensearch-project/k-NN/pull/1826) +* Add painless script support for hamming with binary vector data type [#1839](https://github.com/opensearch-project/k-NN/pull/1839) +* Add binary format support with IVF method in Faiss Engine [#1784](https://github.com/opensearch-project/k-NN/pull/1784) +### Enhancements +* Switch from byte stream to byte ref for serde [#1825](https://github.com/opensearch-project/k-NN/pull/1825) +### Bug Fixes +* Fixing the arithmetic to find the number of vectors to stream from java to jni layer.[#1804](https://github.com/opensearch-project/k-NN/pull/1804) +* Fixed LeafReaders casting errors to SegmentReaders when segment replication is enabled during search.[#1808](https://github.com/opensearch-project/k-NN/pull/1808) +* Release memory properly for an array type [#1820](https://github.com/opensearch-project/k-NN/pull/1820) +* FIX Same Suffix Cause Recall Drop to zero [#1802](https://github.com/opensearch-project/k-NN/pull/1802) +### Infrastructure +* Apply custom patch only once by comparing the last patch id [#1833](https://github.com/opensearch-project/k-NN/pull/1833) +### Documentation +* Update dev guide to fix clang linking issue on arm [#1746](https://github.com/opensearch-project/k-NN/pull/1746) +### Maintenance +* Bump faiss commit to 33c0ba5 [#1796](https://github.com/opensearch-project/k-NN/pull/1796) \ No newline at end of file From bfed576fe9074ba880dcd8499cbb4f302f984c16 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Mon, 22 Jul 2024 18:09:25 -0700 Subject: [PATCH 301/416] Block PQ support when data type is binary format (#1868) (#1869) (cherry picked from commit 87db66a8233b3834fd80e86d027a1543b79f8713) Co-authored-by: Junqiu Lei --- .../org/opensearch/knn/index/IndexUtil.java | 39 +++++- .../transport/TrainingModelRequest.java | 3 +- .../opensearch/knn/training/VectorReader.java | 2 +- .../org/opensearch/knn/index/FaissIT.java | 114 ------------------ .../opensearch/knn/index/IndexUtilTests.java | 54 +++++++-- .../transport/TrainingModelRequestTests.java | 6 + 6 files changed, 87 insertions(+), 131 deletions(-) diff --git a/src/main/java/org/opensearch/knn/index/IndexUtil.java b/src/main/java/org/opensearch/knn/index/IndexUtil.java index 109ec7101..524c9267e 100644 --- a/src/main/java/org/opensearch/knn/index/IndexUtil.java +++ b/src/main/java/org/opensearch/knn/index/IndexUtil.java @@ -88,7 +88,8 @@ public static ValidationException validateKnnField( String field, int expectedDimension, ModelDao modelDao, - VectorDataType expectedVectorDataType + VectorDataType trainRequestVectorDataType, + KNNMethodContext trainRequestKnnMethodContext ) { // Index metadata should not be null if (indexMetadata == null) { @@ -143,27 +144,53 @@ public static ValidationException validateKnnField( return exception; } - if (expectedVectorDataType != null) { - if (VectorDataType.BYTE == expectedVectorDataType) { + if (trainRequestVectorDataType != null) { + if (VectorDataType.BYTE == trainRequestVectorDataType) { exception.addValidationError( - String.format(Locale.ROOT, "vector data type \"%s\" is not supported for training.", expectedVectorDataType.getValue()) + String.format( + Locale.ROOT, + "vector data type \"%s\" is not supported for training.", + trainRequestVectorDataType.getValue() + ) ); return exception; } VectorDataType trainIndexDataType = getVectorDataTypeFromFieldMapping(fieldMap); - if (trainIndexDataType != expectedVectorDataType) { + if (trainIndexDataType != trainRequestVectorDataType) { exception.addValidationError( String.format( Locale.ROOT, "Field \"%s\" has data type %s, which is different from data type used in the training request: %s", field, trainIndexDataType.getValue(), - expectedVectorDataType.getValue() + trainRequestVectorDataType.getValue() ) ); return exception; } + + // Block binary vector data type for pq encoder + if (trainRequestKnnMethodContext != null) { + MethodComponentContext methodComponentContext = trainRequestKnnMethodContext.getMethodComponentContext(); + Map parameters = methodComponentContext.getParameters(); + + if (parameters != null && parameters.containsKey(KNNConstants.METHOD_ENCODER_PARAMETER)) { + MethodComponentContext encoder = (MethodComponentContext) parameters.get(KNNConstants.METHOD_ENCODER_PARAMETER); + if (encoder != null + && KNNConstants.ENCODER_PQ.equals(encoder.getName()) + && VectorDataType.BINARY == trainRequestVectorDataType) { + exception.addValidationError( + String.format( + Locale.ROOT, + "vector data type \"%s\" is not supported for pq encoder.", + trainRequestVectorDataType.getValue() + ) + ); + return exception; + } + } + } } // Return if dimension does not need to be checked diff --git a/src/main/java/org/opensearch/knn/plugin/transport/TrainingModelRequest.java b/src/main/java/org/opensearch/knn/plugin/transport/TrainingModelRequest.java index 16a1a103a..f7ad997b2 100644 --- a/src/main/java/org/opensearch/knn/plugin/transport/TrainingModelRequest.java +++ b/src/main/java/org/opensearch/knn/plugin/transport/TrainingModelRequest.java @@ -332,7 +332,8 @@ public ActionRequestValidationException validate() { this.trainingField, this.dimension, modelDao, - this.vectorDataType + vectorDataType, + knnMethodContext ); if (fieldValidation != null) { exception = exception == null ? new ActionRequestValidationException() : exception; diff --git a/src/main/java/org/opensearch/knn/training/VectorReader.java b/src/main/java/org/opensearch/knn/training/VectorReader.java index f1fd744fd..e94d037bd 100644 --- a/src/main/java/org/opensearch/knn/training/VectorReader.java +++ b/src/main/java/org/opensearch/knn/training/VectorReader.java @@ -88,7 +88,7 @@ public void read( throw validationException; } - ValidationException fieldValidationException = IndexUtil.validateKnnField(indexMetadata, fieldName, -1, null, null); + ValidationException fieldValidationException = IndexUtil.validateKnnField(indexMetadata, fieldName, -1, null, null, null); if (fieldValidationException != null) { validationException = validationException == null ? new ValidationException() : validationException; validationException.addValidationErrors(validationException.validationErrors()); diff --git a/src/test/java/org/opensearch/knn/index/FaissIT.java b/src/test/java/org/opensearch/knn/index/FaissIT.java index 4258f309e..33a37a4a4 100644 --- a/src/test/java/org/opensearch/knn/index/FaissIT.java +++ b/src/test/java/org/opensearch/knn/index/FaissIT.java @@ -1710,120 +1710,6 @@ public void testIVF_whenBinaryFormat_whenIVF_thenSuccess() { validateGraphEviction(); } - @SneakyThrows - public void testIVF_whenBinaryFormat_whenIVFPQ_thenSuccess() { - String modelId = "test-model-ivfpq-binary"; - int dimension = 8; - - String trainingIndexName = "train-index-ivfpq-binary"; - String trainingFieldName = "train-field-ivfpq-binary"; - - String trainIndexMapping = XContentFactory.jsonBuilder() - .startObject() - .startObject("properties") - .startObject(trainingFieldName) - .field("type", "knn_vector") - .field("dimension", dimension) - .field("data_type", VectorDataType.BINARY.getValue()) - .startObject(KNN_METHOD) - .field(NAME, KNNEngine.FAISS.getMethod(METHOD_HNSW).getMethodComponent().getName()) - .field(METHOD_PARAMETER_SPACE_TYPE, SpaceType.HAMMING.getValue()) - .field(KNN_ENGINE, KNNEngine.FAISS.getName()) - .startObject(PARAMETERS) - .field(METHOD_PARAMETER_M, 24) - .field(METHOD_PARAMETER_EF_CONSTRUCTION, 128) - .endObject() - .endObject() - .endObject() - .endObject() - .endObject() - .toString(); - - createKnnIndex(trainingIndexName, trainIndexMapping); - - int trainingDataCount = 50; - bulkIngestRandomBinaryVectors(trainingIndexName, trainingFieldName, trainingDataCount, dimension); - - XContentBuilder trainModelXContentBuilder = XContentFactory.jsonBuilder() - .startObject() - .field(TRAIN_INDEX_PARAMETER, trainingIndexName) - .field(TRAIN_FIELD_PARAMETER, trainingFieldName) - .field(DIMENSION, dimension) - .field(MODEL_DESCRIPTION, "My model description") - .field(VECTOR_DATA_TYPE_FIELD, VectorDataType.BINARY.getValue()) - .startObject(KNN_METHOD) - .field(NAME, METHOD_IVF) - .field(KNN_ENGINE, FAISS_NAME) - .field(METHOD_PARAMETER_SPACE_TYPE, SpaceType.HAMMING.getValue()) - .startObject(PARAMETERS) - .field(METHOD_PARAMETER_NPROBES, 1) - .field(METHOD_PARAMETER_NLIST, 1) - .startObject(METHOD_ENCODER_PARAMETER) - .field(NAME, ENCODER_PQ) - .startObject(PARAMETERS) - .field(ENCODER_PARAMETER_PQ_CODE_SIZE, 8) - .field(ENCODER_PARAMETER_PQ_M, 8) - .endObject() - .endObject() - .endObject() - .endObject() - .endObject(); - - trainModel(modelId, trainModelXContentBuilder); - - // Make sure training succeeds after 30 seconds - assertTrainingSucceeds(modelId, 30, 1000); - - // Create knn index from model - String fieldName = "test-field-name-ivfpq-binary"; - String indexName = "test-index-name-ivfpq-binary"; - - String indexMapping = XContentFactory.jsonBuilder() - .startObject() - .startObject("properties") - .startObject(fieldName) - .field("type", "knn_vector") - .field(MODEL_ID, modelId) - .endObject() - .endObject() - .endObject() - .toString(); - - createKnnIndex(indexName, getKNNDefaultIndexSettings(), indexMapping); - Integer[] vector1 = { 11 }; - Integer[] vector2 = { 22 }; - Integer[] vector3 = { 33 }; - Integer[] vector4 = { 44 }; - addKnnDoc(indexName, "1", fieldName, vector1); - addKnnDoc(indexName, "2", fieldName, vector2); - addKnnDoc(indexName, "3", fieldName, vector3); - addKnnDoc(indexName, "4", fieldName, vector4); - - Integer[] queryVector = { 15 }; - int k = 2; - - XContentBuilder queryBuilder = XContentFactory.jsonBuilder() - .startObject() - .startObject("query") - .startObject("knn") - .startObject(fieldName) - .field("vector", queryVector) - .field("k", k) - .endObject() - .endObject() - .endObject() - .endObject(); - Response searchResponse = searchKNNIndex(indexName, queryBuilder, k); - List results = parseSearchResponse(EntityUtils.toString(searchResponse.getEntity()), fieldName); - assertEquals(k, results.size()); - - deleteKNNIndex(indexName); - Thread.sleep(45 * 1000); - deleteModel(modelId); - deleteKNNIndex(trainingIndexName); - validateGraphEviction(); - } - protected void setupKNNIndexForFilterQuery() throws Exception { // Create Mappings XContentBuilder builder = XContentFactory.jsonBuilder() diff --git a/src/test/java/org/opensearch/knn/index/IndexUtilTests.java b/src/test/java/org/opensearch/knn/index/IndexUtilTests.java index 1b00ecfaa..809c7d930 100644 --- a/src/test/java/org/opensearch/knn/index/IndexUtilTests.java +++ b/src/test/java/org/opensearch/knn/index/IndexUtilTests.java @@ -28,6 +28,7 @@ import org.opensearch.knn.indices.ModelMetadata; import org.opensearch.knn.jni.JNIService; +import java.util.Collections; import java.util.HashMap; import java.util.Map; import java.util.Objects; @@ -38,7 +39,10 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.mockStatic; import static org.mockito.Mockito.when; +import static org.opensearch.knn.common.KNNConstants.ENCODER_PQ; import static org.opensearch.knn.common.KNNConstants.HNSW_ALGO_EF_SEARCH; +import static org.opensearch.knn.common.KNNConstants.METHOD_ENCODER_PARAMETER; +import static org.opensearch.knn.common.KNNConstants.METHOD_IVF; import static org.opensearch.knn.common.KNNConstants.SPACE_TYPE; import static org.opensearch.knn.common.KNNConstants.VECTOR_DATA_TYPE_FIELD; import static org.opensearch.knn.index.IndexUtil.getParametersAtLoading; @@ -117,7 +121,7 @@ public void testValidateKnnField_NestedField() { when(trainingFieldModelMetadata.getDimension()).thenReturn(dimension); when(modelDao.getMetadata(anyString())).thenReturn(trainingFieldModelMetadata); - ValidationException e = IndexUtil.validateKnnField(indexMetadata, field, dimension, modelDao, null); + ValidationException e = IndexUtil.validateKnnField(indexMetadata, field, dimension, modelDao, null, null); assertNull(e); } @@ -138,7 +142,7 @@ public void testValidateKnnField_NonNestedField() { when(trainingFieldModelMetadata.getDimension()).thenReturn(dimension); when(modelDao.getMetadata(anyString())).thenReturn(trainingFieldModelMetadata); - ValidationException e = IndexUtil.validateKnnField(indexMetadata, field, dimension, modelDao, null); + ValidationException e = IndexUtil.validateKnnField(indexMetadata, field, dimension, modelDao, null, null); assertNull(e); } @@ -158,7 +162,7 @@ public void testValidateKnnField_NonKnnField() { when(trainingFieldModelMetadata.getDimension()).thenReturn(dimension); when(modelDao.getMetadata(anyString())).thenReturn(trainingFieldModelMetadata); - ValidationException e = IndexUtil.validateKnnField(indexMetadata, field, dimension, modelDao, null); + ValidationException e = IndexUtil.validateKnnField(indexMetadata, field, dimension, modelDao, null, null); assert Objects.requireNonNull(e).getMessage().matches("Validation Failed: 1: Field \"" + field + "\" is not of type knn_vector.;"); } @@ -182,7 +186,7 @@ public void testValidateKnnField_WrongFieldPath() { when(trainingFieldModelMetadata.getDimension()).thenReturn(dimension); when(modelDao.getMetadata(anyString())).thenReturn(trainingFieldModelMetadata); - ValidationException e = IndexUtil.validateKnnField(indexMetadata, field, dimension, modelDao, null); + ValidationException e = IndexUtil.validateKnnField(indexMetadata, field, dimension, modelDao, null, null); assert (Objects.requireNonNull(e).getMessage().matches("Validation Failed: 1: Field \"" + field + "\" does not exist.;")); } @@ -206,7 +210,7 @@ public void testValidateKnnField_EmptyField() { when(trainingFieldModelMetadata.getDimension()).thenReturn(dimension); when(modelDao.getMetadata(anyString())).thenReturn(trainingFieldModelMetadata); - ValidationException e = IndexUtil.validateKnnField(indexMetadata, field, dimension, modelDao, null); + ValidationException e = IndexUtil.validateKnnField(indexMetadata, field, dimension, modelDao, null, null); System.out.println(Objects.requireNonNull(e).getMessage()); @@ -223,7 +227,7 @@ public void testValidateKnnField_EmptyIndexMetadata() { when(trainingFieldModelMetadata.getDimension()).thenReturn(dimension); when(modelDao.getMetadata(anyString())).thenReturn(trainingFieldModelMetadata); - ValidationException e = IndexUtil.validateKnnField(indexMetadata, field, dimension, modelDao, null); + ValidationException e = IndexUtil.validateKnnField(indexMetadata, field, dimension, modelDao, null, null); assert (Objects.requireNonNull(e).getMessage().matches("Validation Failed: 1: Invalid index. Index does not contain a mapping;")); } @@ -273,7 +277,7 @@ public void testValidateKnnField_whenTrainModelUseDifferentVectorDataTypeFromTra when(indexMetadata.mapping()).thenReturn(mappingMetadata); ModelDao modelDao = mock(ModelDao.class); - ValidationException e = IndexUtil.validateKnnField(indexMetadata, field, dimension, modelDao, VectorDataType.BINARY); + ValidationException e = IndexUtil.validateKnnField(indexMetadata, field, dimension, modelDao, VectorDataType.BINARY, null); System.out.println(Objects.requireNonNull(e).getMessage()); assert Objects.requireNonNull(e) @@ -298,8 +302,7 @@ public void testValidateKnnField_whenPassByteVectorDataType_thenThrowException() when(indexMetadata.mapping()).thenReturn(mappingMetadata); ModelDao modelDao = mock(ModelDao.class); - ValidationException e = IndexUtil.validateKnnField(indexMetadata, field, dimension, modelDao, VectorDataType.BYTE); - System.out.println(Objects.requireNonNull(e).getMessage()); + ValidationException e = IndexUtil.validateKnnField(indexMetadata, field, dimension, modelDao, VectorDataType.BYTE, null); assert Objects.requireNonNull(e) .getMessage() @@ -311,4 +314,37 @@ public void testUpdateVectorDataTypeToParameters_whenVectorDataTypeIsBinary() { IndexUtil.updateVectorDataTypeToParameters(indexParams, VectorDataType.BINARY); assertEquals(VectorDataType.BINARY.getValue(), indexParams.get(VECTOR_DATA_TYPE_FIELD)); } + + public void testValidateKnnField_whenPassBinaryVectorDataTypeAndPQEncoder_thenThrowException() { + Map fieldValues = Map.of("type", "knn_vector", "dimension", 8, "data_type", "binary", "encoder", "pq"); + Map top_level_field = Map.of("top_level_field", fieldValues); + Map properties = Map.of("properties", top_level_field); + String field = "top_level_field"; + int dimension = 8; + + MappingMetadata mappingMetadata = mock(MappingMetadata.class); + when(mappingMetadata.getSourceAsMap()).thenReturn(properties); + IndexMetadata indexMetadata = mock(IndexMetadata.class); + when(indexMetadata.mapping()).thenReturn(mappingMetadata); + ModelDao modelDao = mock(ModelDao.class); + MethodComponentContext pq = new MethodComponentContext(ENCODER_PQ, Collections.emptyMap()); + KNNMethodContext knnMethodContext = new KNNMethodContext( + KNNEngine.FAISS, + SpaceType.INNER_PRODUCT, + new MethodComponentContext(METHOD_IVF, ImmutableMap.of(METHOD_ENCODER_PARAMETER, pq)) + ); + + ValidationException e = IndexUtil.validateKnnField( + indexMetadata, + field, + dimension, + modelDao, + VectorDataType.BINARY, + knnMethodContext + ); + + assert Objects.requireNonNull(e) + .getMessage() + .matches("Validation Failed: 1: vector data type \"binary\" is not supported for pq encoder.;"); + } } diff --git a/src/test/java/org/opensearch/knn/plugin/transport/TrainingModelRequestTests.java b/src/test/java/org/opensearch/knn/plugin/transport/TrainingModelRequestTests.java index 9434a6e41..0fb478c83 100644 --- a/src/test/java/org/opensearch/knn/plugin/transport/TrainingModelRequestTests.java +++ b/src/test/java/org/opensearch/knn/plugin/transport/TrainingModelRequestTests.java @@ -255,6 +255,7 @@ public void testValidation_invalid_invalidMethodContext() { when(knnMethodContext.validate()).thenReturn(validationException); when(knnMethodContext.isTrainingRequired()).thenReturn(false); + when(knnMethodContext.getMethodComponentContext()).thenReturn(MethodComponentContext.EMPTY); int dimension = 10; String trainingIndex = "test-training-index"; String trainingField = "test-training-field"; @@ -454,6 +455,7 @@ public void testValidation_invalid_dimensionDoesNotMatch() { when(knnMethodContext.validate()).thenReturn(null); when(knnMethodContext.isTrainingRequired()).thenReturn(true); + when(knnMethodContext.getMethodComponentContext()).thenReturn(MethodComponentContext.EMPTY); int dimension = 10; String trainingIndex = "test-training-index"; String trainingField = "test-training-field"; @@ -511,6 +513,7 @@ public void testValidation_invalid_preferredNodeDoesNotExist() { KNNMethodContext knnMethodContext = mock(KNNMethodContext.class); when(knnMethodContext.validate()).thenReturn(null); when(knnMethodContext.isTrainingRequired()).thenReturn(true); + when(knnMethodContext.getMethodComponentContext()).thenReturn(MethodComponentContext.EMPTY); int dimension = 10; String trainingIndex = "test-training-index"; String trainingField = "test-training-field"; @@ -573,6 +576,7 @@ public void testValidation_invalid_descriptionToLong() { KNNMethodContext knnMethodContext = mock(KNNMethodContext.class); when(knnMethodContext.validate()).thenReturn(null); when(knnMethodContext.isTrainingRequired()).thenReturn(true); + when(knnMethodContext.getMethodComponentContext()).thenReturn(MethodComponentContext.EMPTY); int dimension = 10; String trainingIndex = "test-training-index"; String trainingField = "test-training-field"; @@ -623,6 +627,7 @@ public void testValidation_valid_trainingIndexBuiltFromMethod() { KNNMethodContext knnMethodContext = mock(KNNMethodContext.class); when(knnMethodContext.validate()).thenReturn(null); when(knnMethodContext.isTrainingRequired()).thenReturn(true); + when(knnMethodContext.getMethodComponentContext()).thenReturn(MethodComponentContext.EMPTY); int dimension = 10; String trainingIndex = "test-training-index"; String trainingField = "test-training-field"; @@ -660,6 +665,7 @@ public void testValidation_valid_trainingIndexBuiltFromModel() { KNNMethodContext knnMethodContext = mock(KNNMethodContext.class); when(knnMethodContext.validate()).thenReturn(null); when(knnMethodContext.isTrainingRequired()).thenReturn(true); + when(knnMethodContext.getMethodComponentContext()).thenReturn(MethodComponentContext.EMPTY); int dimension = 10; String trainingIndex = "test-training-index"; String trainingField = "test-training-field"; From f84caf8c8fa4bf67cd5b146f98d5b00c563f6e75 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Mon, 22 Jul 2024 21:37:52 -0500 Subject: [PATCH 302/416] Add support for Lucene inbuilt Scalar Quantizer (#1848) (#1871) --- .../opensearch-knn.release-notes-2.16.0.0.md | 1 + .../opensearch/knn/common/KNNConstants.java | 6 + .../org/opensearch/knn/index/Parameter.java | 81 +++++++ .../codec/BasePerFieldKnnVectorsFormat.java | 88 +++++--- .../KNN920PerFieldKnnVectorsFormat.java | 5 +- .../KNN940PerFieldKnnVectorsFormat.java | 5 +- .../KNN950PerFieldKnnVectorsFormat.java | 5 +- .../KNN990PerFieldKnnVectorsFormat.java | 16 +- ...KNNScalarQuantizedVectorsFormatParams.java | 82 +++++++ .../codec/params/KNNVectorsFormatParams.java | 45 ++++ .../index/mapper/KNNVectorFieldMapper.java | 4 +- .../org/opensearch/knn/index/util/Lucene.java | 39 ++++ .../opensearch/knn/index/LuceneEngineIT.java | 200 ++++++++++++++++++ .../opensearch/knn/index/ParameterTests.java | 34 +++ ...alarQuantizedVectorsFormatParamsTests.java | 110 ++++++++++ .../params/KNNVectorsFormatParamsTests.java | 56 +++++ 16 files changed, 746 insertions(+), 31 deletions(-) create mode 100644 src/main/java/org/opensearch/knn/index/codec/params/KNNScalarQuantizedVectorsFormatParams.java create mode 100644 src/main/java/org/opensearch/knn/index/codec/params/KNNVectorsFormatParams.java create mode 100644 src/test/java/org/opensearch/knn/index/codec/params/KNNScalarQuantizedVectorsFormatParamsTests.java create mode 100644 src/test/java/org/opensearch/knn/index/codec/params/KNNVectorsFormatParamsTests.java diff --git a/release-notes/opensearch-knn.release-notes-2.16.0.0.md b/release-notes/opensearch-knn.release-notes-2.16.0.0.md index c24d3552b..da60c2cbf 100644 --- a/release-notes/opensearch-knn.release-notes-2.16.0.0.md +++ b/release-notes/opensearch-knn.release-notes-2.16.0.0.md @@ -10,6 +10,7 @@ Compatible with OpenSearch 2.16.0 * Add script scoring support for knn field with binary data type [#1826](https://github.com/opensearch-project/k-NN/pull/1826) * Add painless script support for hamming with binary vector data type [#1839](https://github.com/opensearch-project/k-NN/pull/1839) * Add binary format support with IVF method in Faiss Engine [#1784](https://github.com/opensearch-project/k-NN/pull/1784) +* Add support for Lucene inbuilt Scalar Quantizer [#1848](https://github.com/opensearch-project/k-NN/pull/1848) ### Enhancements * Switch from byte stream to byte ref for serde [#1825](https://github.com/opensearch-project/k-NN/pull/1825) ### Bug Fixes diff --git a/src/main/java/org/opensearch/knn/common/KNNConstants.java b/src/main/java/org/opensearch/knn/common/KNNConstants.java index 77a884a5c..de46cdaa8 100644 --- a/src/main/java/org/opensearch/knn/common/KNNConstants.java +++ b/src/main/java/org/opensearch/knn/common/KNNConstants.java @@ -74,6 +74,12 @@ public class KNNConstants { // Lucene specific constants public static final String LUCENE_NAME = "lucene"; + public static final String LUCENE_SQ_CONFIDENCE_INTERVAL = "confidence_interval"; + public static final int DYNAMIC_CONFIDENCE_INTERVAL = 0; + public static final double MINIMUM_CONFIDENCE_INTERVAL = 0.9; + public static final double MAXIMUM_CONFIDENCE_INTERVAL = 1.0; + public static final String LUCENE_SQ_BITS = "bits"; + public static final int LUCENE_SQ_DEFAULT_BITS = 7; // nmslib specific constants public static final String NMSLIB_NAME = "nmslib"; diff --git a/src/main/java/org/opensearch/knn/index/Parameter.java b/src/main/java/org/opensearch/knn/index/Parameter.java index a4520636e..50d792ebd 100644 --- a/src/main/java/org/opensearch/knn/index/Parameter.java +++ b/src/main/java/org/opensearch/knn/index/Parameter.java @@ -14,7 +14,9 @@ import org.opensearch.common.ValidationException; import org.opensearch.knn.training.VectorSpaceInfo; +import java.util.Locale; import java.util.Map; +import java.util.Objects; import java.util.function.BiFunction; import java.util.function.Predicate; @@ -204,6 +206,85 @@ public ValidationException validateWithData(Object value, VectorSpaceInfo vector } } + /** + * Double method parameter + */ + public static class DoubleParameter extends Parameter { + public DoubleParameter(String name, Double defaultValue, Predicate validator) { + super(name, defaultValue, validator); + } + + public DoubleParameter( + String name, + Double defaultValue, + Predicate validator, + BiFunction validatorWithData + ) { + super(name, defaultValue, validator, validatorWithData); + } + + @Override + public ValidationException validate(Object value) { + if (Objects.isNull(value)) { + String validationErrorMsg = String.format(Locale.ROOT, "Null value provided for Double " + "parameter \"%s\".", getName()); + return getValidationException(validationErrorMsg); + } + if (value.equals(0)) value = 0.0; + + if (!(value instanceof Double)) { + String validationErrorMsg = String.format( + Locale.ROOT, + "Value not of type Double for Double " + "parameter \"%s\".", + getName() + ); + return getValidationException(validationErrorMsg); + } + + if (!validator.test((Double) value)) { + String validationErrorMsg = String.format( + Locale.ROOT, + "Parameter validation failed for Double " + "parameter \"%s\".", + getName() + ); + return getValidationException(validationErrorMsg); + } + return null; + } + + @Override + public ValidationException validateWithData(Object value, VectorSpaceInfo vectorSpaceInfo) { + if (Objects.isNull(value)) { + String validationErrorMsg = String.format(Locale.ROOT, "Null value provided for Double " + "parameter \"%s\".", getName()); + return getValidationException(validationErrorMsg); + } + + if (!(value instanceof Double)) { + String validationErrorMsg = String.format( + Locale.ROOT, + "value is not an instance of Double for Double parameter [%s].", + getName() + ); + return getValidationException(validationErrorMsg); + } + + if (validatorWithData == null) { + return null; + } + + if (!validatorWithData.apply((Double) value, vectorSpaceInfo)) { + String validationErrorMsg = String.format(Locale.ROOT, "parameter validation failed for Double parameter [%s].", getName()); + return getValidationException(validationErrorMsg); + } + return null; + } + + private ValidationException getValidationException(String validationErrorMsg) { + ValidationException validationException = new ValidationException(); + validationException.addValidationError(validationErrorMsg); + return validationException; + } + } + /** * String method parameter */ diff --git a/src/main/java/org/opensearch/knn/index/codec/BasePerFieldKnnVectorsFormat.java b/src/main/java/org/opensearch/knn/index/codec/BasePerFieldKnnVectorsFormat.java index b9e280e2e..f3738452a 100644 --- a/src/main/java/org/opensearch/knn/index/codec/BasePerFieldKnnVectorsFormat.java +++ b/src/main/java/org/opensearch/knn/index/codec/BasePerFieldKnnVectorsFormat.java @@ -10,14 +10,19 @@ import org.apache.lucene.codecs.KnnVectorsFormat; import org.apache.lucene.codecs.perfield.PerFieldKnnVectorsFormat; import org.opensearch.index.mapper.MapperService; -import org.opensearch.knn.common.KNNConstants; +import org.opensearch.knn.index.codec.params.KNNScalarQuantizedVectorsFormatParams; +import org.opensearch.knn.index.codec.params.KNNVectorsFormatParams; import org.opensearch.knn.index.mapper.KNNVectorFieldMapper; +import org.opensearch.knn.index.util.KNNEngine; -import java.util.Map; import java.util.Optional; -import java.util.function.BiFunction; +import java.util.function.Function; import java.util.function.Supplier; +import static org.opensearch.knn.common.KNNConstants.LUCENE_SQ_BITS; +import static org.opensearch.knn.common.KNNConstants.LUCENE_SQ_CONFIDENCE_INTERVAL; +import static org.opensearch.knn.common.KNNConstants.METHOD_ENCODER_PARAMETER; + /** * Base class for PerFieldKnnVectorsFormat, builds KnnVectorsFormat based on specific Lucene version */ @@ -29,15 +34,34 @@ public abstract class BasePerFieldKnnVectorsFormat extends PerFieldKnnVectorsFor private final int defaultMaxConnections; private final int defaultBeamWidth; private final Supplier defaultFormatSupplier; - private final BiFunction formatSupplier; + private final Function vectorsFormatSupplier; + private Function scalarQuantizedVectorsFormatSupplier; + private static final String MAX_CONNECTIONS = "max_connections"; + private static final String BEAM_WIDTH = "beam_width"; + + public BasePerFieldKnnVectorsFormat( + Optional mapperService, + int defaultMaxConnections, + int defaultBeamWidth, + Supplier defaultFormatSupplier, + Function vectorsFormatSupplier + ) { + this.mapperService = mapperService; + this.defaultMaxConnections = defaultMaxConnections; + this.defaultBeamWidth = defaultBeamWidth; + this.defaultFormatSupplier = defaultFormatSupplier; + this.vectorsFormatSupplier = vectorsFormatSupplier; + } @Override public KnnVectorsFormat getKnnVectorsFormatForField(final String field) { if (isKnnVectorFieldType(field) == false) { log.debug( - "Initialize KNN vector format for field [{}] with default params [max_connections] = \"{}\" and [beam_width] = \"{}\"", + "Initialize KNN vector format for field [{}] with default params [{}] = \"{}\" and [{}] = \"{}\"", field, + MAX_CONNECTIONS, defaultMaxConnections, + BEAM_WIDTH, defaultBeamWidth ); return defaultFormatSupplier.get(); @@ -48,15 +72,43 @@ public KnnVectorsFormat getKnnVectorsFormatForField(final String field) { ) ).fieldType(field); var params = type.getKnnMethodContext().getMethodComponentContext().getParameters(); - int maxConnections = getMaxConnections(params); - int beamWidth = getBeamWidth(params); + + if (type.getKnnMethodContext().getKnnEngine() == KNNEngine.LUCENE + && params != null + && params.containsKey(METHOD_ENCODER_PARAMETER)) { + KNNScalarQuantizedVectorsFormatParams knnScalarQuantizedVectorsFormatParams = new KNNScalarQuantizedVectorsFormatParams( + params, + defaultMaxConnections, + defaultBeamWidth + ); + if (knnScalarQuantizedVectorsFormatParams.validate(params)) { + log.debug( + "Initialize KNN vector format for field [{}] with params [{}] = \"{}\", [{}] = \"{}\", [{}] = \"{}\", [{}] = \"{}\"", + field, + MAX_CONNECTIONS, + knnScalarQuantizedVectorsFormatParams.getMaxConnections(), + BEAM_WIDTH, + knnScalarQuantizedVectorsFormatParams.getBeamWidth(), + LUCENE_SQ_CONFIDENCE_INTERVAL, + knnScalarQuantizedVectorsFormatParams.getConfidenceInterval(), + LUCENE_SQ_BITS, + knnScalarQuantizedVectorsFormatParams.getBits() + ); + return scalarQuantizedVectorsFormatSupplier.apply(knnScalarQuantizedVectorsFormatParams); + } + + } + + KNNVectorsFormatParams knnVectorsFormatParams = new KNNVectorsFormatParams(params, defaultMaxConnections, defaultBeamWidth); log.debug( - "Initialize KNN vector format for field [{}] with params [max_connections] = \"{}\" and [beam_width] = \"{}\"", + "Initialize KNN vector format for field [{}] with params [{}] = \"{}\" and [{}] = \"{}\"", field, - maxConnections, - beamWidth + MAX_CONNECTIONS, + knnVectorsFormatParams.getMaxConnections(), + BEAM_WIDTH, + knnVectorsFormatParams.getBeamWidth() ); - return formatSupplier.apply(maxConnections, beamWidth); + return vectorsFormatSupplier.apply(knnVectorsFormatParams); } @Override @@ -67,18 +119,4 @@ public int getMaxDimensions(String fieldName) { private boolean isKnnVectorFieldType(final String field) { return mapperService.isPresent() && mapperService.get().fieldType(field) instanceof KNNVectorFieldMapper.KNNVectorFieldType; } - - private int getMaxConnections(final Map params) { - if (params != null && params.containsKey(KNNConstants.METHOD_PARAMETER_M)) { - return (int) params.get(KNNConstants.METHOD_PARAMETER_M); - } - return defaultMaxConnections; - } - - private int getBeamWidth(final Map params) { - if (params != null && params.containsKey(KNNConstants.METHOD_PARAMETER_EF_CONSTRUCTION)) { - return (int) params.get(KNNConstants.METHOD_PARAMETER_EF_CONSTRUCTION); - } - return defaultBeamWidth; - } } diff --git a/src/main/java/org/opensearch/knn/index/codec/KNN920Codec/KNN920PerFieldKnnVectorsFormat.java b/src/main/java/org/opensearch/knn/index/codec/KNN920Codec/KNN920PerFieldKnnVectorsFormat.java index ae1ef206c..7cca04319 100644 --- a/src/main/java/org/opensearch/knn/index/codec/KNN920Codec/KNN920PerFieldKnnVectorsFormat.java +++ b/src/main/java/org/opensearch/knn/index/codec/KNN920Codec/KNN920PerFieldKnnVectorsFormat.java @@ -22,7 +22,10 @@ public KNN920PerFieldKnnVectorsFormat(final Optional mapperServic Lucene92HnswVectorsFormat.DEFAULT_MAX_CONN, Lucene92HnswVectorsFormat.DEFAULT_BEAM_WIDTH, () -> new Lucene92HnswVectorsFormat(), - (maxConnm, beamWidth) -> new Lucene92HnswVectorsFormat(maxConnm, beamWidth) + knnVectorsFormatParams -> new Lucene92HnswVectorsFormat( + knnVectorsFormatParams.getMaxConnections(), + knnVectorsFormatParams.getBeamWidth() + ) ); } } diff --git a/src/main/java/org/opensearch/knn/index/codec/KNN940Codec/KNN940PerFieldKnnVectorsFormat.java b/src/main/java/org/opensearch/knn/index/codec/KNN940Codec/KNN940PerFieldKnnVectorsFormat.java index d9a1a9251..1ed9c929c 100644 --- a/src/main/java/org/opensearch/knn/index/codec/KNN940Codec/KNN940PerFieldKnnVectorsFormat.java +++ b/src/main/java/org/opensearch/knn/index/codec/KNN940Codec/KNN940PerFieldKnnVectorsFormat.java @@ -22,7 +22,10 @@ public KNN940PerFieldKnnVectorsFormat(final Optional mapperServic Lucene94HnswVectorsFormat.DEFAULT_MAX_CONN, Lucene94HnswVectorsFormat.DEFAULT_BEAM_WIDTH, () -> new Lucene94HnswVectorsFormat(), - (maxConnm, beamWidth) -> new Lucene94HnswVectorsFormat(maxConnm, beamWidth) + knnVectorsFormatParams -> new Lucene94HnswVectorsFormat( + knnVectorsFormatParams.getMaxConnections(), + knnVectorsFormatParams.getBeamWidth() + ) ); } } diff --git a/src/main/java/org/opensearch/knn/index/codec/KNN950Codec/KNN950PerFieldKnnVectorsFormat.java b/src/main/java/org/opensearch/knn/index/codec/KNN950Codec/KNN950PerFieldKnnVectorsFormat.java index 05ce7271f..978b22003 100644 --- a/src/main/java/org/opensearch/knn/index/codec/KNN950Codec/KNN950PerFieldKnnVectorsFormat.java +++ b/src/main/java/org/opensearch/knn/index/codec/KNN950Codec/KNN950PerFieldKnnVectorsFormat.java @@ -23,7 +23,10 @@ public KNN950PerFieldKnnVectorsFormat(final Optional mapperServic Lucene95HnswVectorsFormat.DEFAULT_MAX_CONN, Lucene95HnswVectorsFormat.DEFAULT_BEAM_WIDTH, () -> new Lucene95HnswVectorsFormat(), - (maxConnm, beamWidth) -> new Lucene95HnswVectorsFormat(maxConnm, beamWidth) + knnVectorsFormatParams -> new Lucene95HnswVectorsFormat( + knnVectorsFormatParams.getMaxConnections(), + knnVectorsFormatParams.getBeamWidth() + ) ); } diff --git a/src/main/java/org/opensearch/knn/index/codec/KNN990Codec/KNN990PerFieldKnnVectorsFormat.java b/src/main/java/org/opensearch/knn/index/codec/KNN990Codec/KNN990PerFieldKnnVectorsFormat.java index abf40f2ef..e8ecfad18 100644 --- a/src/main/java/org/opensearch/knn/index/codec/KNN990Codec/KNN990PerFieldKnnVectorsFormat.java +++ b/src/main/java/org/opensearch/knn/index/codec/KNN990Codec/KNN990PerFieldKnnVectorsFormat.java @@ -5,6 +5,7 @@ package org.opensearch.knn.index.codec.KNN990Codec; +import org.apache.lucene.codecs.lucene99.Lucene99HnswScalarQuantizedVectorsFormat; import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat; import org.opensearch.index.mapper.MapperService; import org.opensearch.knn.index.codec.BasePerFieldKnnVectorsFormat; @@ -16,6 +17,7 @@ * Class provides per field format implementation for Lucene Knn vector type */ public class KNN990PerFieldKnnVectorsFormat extends BasePerFieldKnnVectorsFormat { + private static final int NUM_MERGE_WORKERS = 1; public KNN990PerFieldKnnVectorsFormat(final Optional mapperService) { super( @@ -23,7 +25,19 @@ public KNN990PerFieldKnnVectorsFormat(final Optional mapperServic Lucene99HnswVectorsFormat.DEFAULT_MAX_CONN, Lucene99HnswVectorsFormat.DEFAULT_BEAM_WIDTH, () -> new Lucene99HnswVectorsFormat(), - (maxConnm, beamWidth) -> new Lucene99HnswVectorsFormat(maxConnm, beamWidth) + knnVectorsFormatParams -> new Lucene99HnswVectorsFormat( + knnVectorsFormatParams.getMaxConnections(), + knnVectorsFormatParams.getBeamWidth() + ), + knnScalarQuantizedVectorsFormatParams -> new Lucene99HnswScalarQuantizedVectorsFormat( + knnScalarQuantizedVectorsFormatParams.getMaxConnections(), + knnScalarQuantizedVectorsFormatParams.getBeamWidth(), + NUM_MERGE_WORKERS, + knnScalarQuantizedVectorsFormatParams.getBits(), + knnScalarQuantizedVectorsFormatParams.isCompressFlag(), + knnScalarQuantizedVectorsFormatParams.getConfidenceInterval(), + null + ) ); } diff --git a/src/main/java/org/opensearch/knn/index/codec/params/KNNScalarQuantizedVectorsFormatParams.java b/src/main/java/org/opensearch/knn/index/codec/params/KNNScalarQuantizedVectorsFormatParams.java new file mode 100644 index 000000000..79bf1cbdb --- /dev/null +++ b/src/main/java/org/opensearch/knn/index/codec/params/KNNScalarQuantizedVectorsFormatParams.java @@ -0,0 +1,82 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.codec.params; + +import lombok.Getter; +import org.opensearch.knn.index.MethodComponentContext; + +import java.util.Map; + +import static org.opensearch.knn.common.KNNConstants.ENCODER_SQ; +import static org.opensearch.knn.common.KNNConstants.LUCENE_SQ_BITS; +import static org.opensearch.knn.common.KNNConstants.LUCENE_SQ_CONFIDENCE_INTERVAL; +import static org.opensearch.knn.common.KNNConstants.LUCENE_SQ_DEFAULT_BITS; +import static org.opensearch.knn.common.KNNConstants.METHOD_ENCODER_PARAMETER; + +/** + * Class provides params for LuceneHnswScalarQuantizedVectorsFormat + */ +@Getter +public class KNNScalarQuantizedVectorsFormatParams extends KNNVectorsFormatParams { + private Float confidenceInterval; + private int bits; + private boolean compressFlag; + + public KNNScalarQuantizedVectorsFormatParams(Map params, int defaultMaxConnections, int defaultBeamWidth) { + super(params, defaultMaxConnections, defaultBeamWidth); + MethodComponentContext encoderMethodComponentContext = (MethodComponentContext) params.get(METHOD_ENCODER_PARAMETER); + Map sqEncoderParams = encoderMethodComponentContext.getParameters(); + this.initConfidenceInterval(sqEncoderParams); + this.initBits(sqEncoderParams); + this.initCompressFlag(); + } + + @Override + public boolean validate(Map params) { + if (params.get(METHOD_ENCODER_PARAMETER) == null) { + return false; + } + + // Validate if the object is of type MethodComponentContext before casting it later + if (!(params.get(METHOD_ENCODER_PARAMETER) instanceof MethodComponentContext)) { + return false; + } + MethodComponentContext encoderMethodComponentContext = (MethodComponentContext) params.get(METHOD_ENCODER_PARAMETER); + if (!ENCODER_SQ.equals(encoderMethodComponentContext.getName())) { + return false; + } + + return true; + } + + private void initConfidenceInterval(final Map params) { + + if (params != null && params.containsKey(LUCENE_SQ_CONFIDENCE_INTERVAL)) { + if (params.get(LUCENE_SQ_CONFIDENCE_INTERVAL).equals(0)) { + this.confidenceInterval = (float) 0; + return; + } + this.confidenceInterval = ((Double) params.get(LUCENE_SQ_CONFIDENCE_INTERVAL)).floatValue(); + return; + } + + // If confidence_interval is not provided by user, then it will be set with a default value as null so that + // it will be computed later in Lucene based on the dimension of the vector as 1 - 1/(1 + d) + this.confidenceInterval = null; + } + + private void initBits(final Map params) { + if (params != null && params.containsKey(LUCENE_SQ_BITS)) { + this.bits = (int) params.get(LUCENE_SQ_BITS); + return; + } + this.bits = LUCENE_SQ_DEFAULT_BITS; + } + + private void initCompressFlag() { + this.compressFlag = true; + } +} diff --git a/src/main/java/org/opensearch/knn/index/codec/params/KNNVectorsFormatParams.java b/src/main/java/org/opensearch/knn/index/codec/params/KNNVectorsFormatParams.java new file mode 100644 index 000000000..52134bc7e --- /dev/null +++ b/src/main/java/org/opensearch/knn/index/codec/params/KNNVectorsFormatParams.java @@ -0,0 +1,45 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.codec.params; + +import lombok.Getter; +import org.opensearch.knn.common.KNNConstants; + +import java.util.Map; + +/** + * Class provides params for LuceneHNSWVectorsFormat + */ +@Getter +public class KNNVectorsFormatParams { + private int maxConnections; + private int beamWidth; + + public KNNVectorsFormatParams(final Map params, int defaultMaxConnections, int defaultBeamWidth) { + initMaxConnections(params, defaultMaxConnections); + initBeamWidth(params, defaultBeamWidth); + } + + public boolean validate(final Map params) { + return true; + } + + private void initMaxConnections(final Map params, int defaultMaxConnections) { + if (params != null && params.containsKey(KNNConstants.METHOD_PARAMETER_M)) { + this.maxConnections = (int) params.get(KNNConstants.METHOD_PARAMETER_M); + return; + } + this.maxConnections = defaultMaxConnections; + } + + private void initBeamWidth(final Map params, int defaultBeamWidth) { + if (params != null && params.containsKey(KNNConstants.METHOD_PARAMETER_EF_CONSTRUCTION)) { + this.beamWidth = (int) params.get(KNNConstants.METHOD_PARAMETER_EF_CONSTRUCTION); + return; + } + this.beamWidth = defaultBeamWidth; + } +} diff --git a/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapper.java b/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapper.java index 5c19a4989..026e9f469 100644 --- a/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapper.java +++ b/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapper.java @@ -349,7 +349,7 @@ private void validateEncoder(final KNNMethodContext knnMethodContext, final Vect return; } - if (VectorDataType.BINARY != vectorDataType) { + if (VectorDataType.FLOAT == vectorDataType) { return; } @@ -380,7 +380,7 @@ private void validateEncoder(final KNNMethodContext knnMethodContext, final Vect String.format( Locale.ROOT, "%s data type does not support %s encoder", - VectorDataType.BINARY.getValue(), + vectorDataType.getValue(), encoderMethodComponentContext.getName() ) ); diff --git a/src/main/java/org/opensearch/knn/index/util/Lucene.java b/src/main/java/org/opensearch/knn/index/util/Lucene.java index b5bbfca75..caf4200cb 100644 --- a/src/main/java/org/opensearch/knn/index/util/Lucene.java +++ b/src/main/java/org/opensearch/knn/index/util/Lucene.java @@ -7,19 +7,30 @@ import com.google.common.collect.ImmutableMap; import org.apache.lucene.util.Version; +import org.opensearch.knn.common.KNNConstants; import org.opensearch.knn.index.KNNMethod; import org.opensearch.knn.index.KNNSettings; import org.opensearch.knn.index.MethodComponent; +import org.opensearch.knn.index.MethodComponentContext; import org.opensearch.knn.index.Parameter; import org.opensearch.knn.index.SpaceType; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.function.Function; +import static org.opensearch.knn.common.KNNConstants.DYNAMIC_CONFIDENCE_INTERVAL; +import static org.opensearch.knn.common.KNNConstants.ENCODER_SQ; +import static org.opensearch.knn.common.KNNConstants.LUCENE_SQ_BITS; +import static org.opensearch.knn.common.KNNConstants.LUCENE_SQ_CONFIDENCE_INTERVAL; +import static org.opensearch.knn.common.KNNConstants.LUCENE_SQ_DEFAULT_BITS; +import static org.opensearch.knn.common.KNNConstants.MAXIMUM_CONFIDENCE_INTERVAL; +import static org.opensearch.knn.common.KNNConstants.METHOD_ENCODER_PARAMETER; import static org.opensearch.knn.common.KNNConstants.METHOD_HNSW; import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_EF_CONSTRUCTION; import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_M; +import static org.opensearch.knn.common.KNNConstants.MINIMUM_CONFIDENCE_INTERVAL; /** * KNN Library for Lucene @@ -27,6 +38,30 @@ public class Lucene extends JVMLibrary { Map> distanceTransform; + private static final List LUCENE_SQ_BITS_SUPPORTED = List.of(7); + + private final static MethodComponentContext ENCODER_DEFAULT = new MethodComponentContext( + KNNConstants.ENCODER_FLAT, + Collections.emptyMap() + ); + + private final static Map HNSW_ENCODERS = ImmutableMap.of( + ENCODER_SQ, + MethodComponent.Builder.builder(ENCODER_SQ) + .addParameter( + LUCENE_SQ_CONFIDENCE_INTERVAL, + new Parameter.DoubleParameter( + LUCENE_SQ_CONFIDENCE_INTERVAL, + null, + v -> v == DYNAMIC_CONFIDENCE_INTERVAL || (v >= MINIMUM_CONFIDENCE_INTERVAL && v <= MAXIMUM_CONFIDENCE_INTERVAL) + ) + ) + .addParameter( + LUCENE_SQ_BITS, + new Parameter.IntegerParameter(LUCENE_SQ_BITS, LUCENE_SQ_DEFAULT_BITS, LUCENE_SQ_BITS_SUPPORTED::contains) + ) + .build() + ); final static Map METHODS = ImmutableMap.of( METHOD_HNSW, @@ -44,6 +79,10 @@ public class Lucene extends JVMLibrary { v -> v > 0 ) ) + .addParameter( + METHOD_ENCODER_PARAMETER, + new Parameter.MethodComponentContextParameter(METHOD_ENCODER_PARAMETER, ENCODER_DEFAULT, HNSW_ENCODERS) + ) .build() ).addSpaces(SpaceType.UNDEFINED, SpaceType.L2, SpaceType.COSINESIMIL, SpaceType.INNER_PRODUCT).build() ); diff --git a/src/test/java/org/opensearch/knn/index/LuceneEngineIT.java b/src/test/java/org/opensearch/knn/index/LuceneEngineIT.java index 454f5c724..1a047ac95 100644 --- a/src/test/java/org/opensearch/knn/index/LuceneEngineIT.java +++ b/src/test/java/org/opensearch/knn/index/LuceneEngineIT.java @@ -33,11 +33,20 @@ import java.util.function.Function; import java.util.stream.Collectors; +import static org.opensearch.knn.common.KNNConstants.ENCODER_SQ; +import static org.opensearch.knn.common.KNNConstants.LUCENE_SQ_BITS; +import static org.opensearch.knn.common.KNNConstants.LUCENE_SQ_CONFIDENCE_INTERVAL; +import static org.opensearch.knn.common.KNNConstants.LUCENE_SQ_DEFAULT_BITS; +import static org.opensearch.knn.common.KNNConstants.MAXIMUM_CONFIDENCE_INTERVAL; import static org.opensearch.knn.common.KNNConstants.MAX_DISTANCE; +import static org.opensearch.knn.common.KNNConstants.METHOD_ENCODER_PARAMETER; import static org.opensearch.knn.common.KNNConstants.METHOD_HNSW; import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER; +import static org.opensearch.knn.common.KNNConstants.MINIMUM_CONFIDENCE_INTERVAL; import static org.opensearch.knn.common.KNNConstants.MIN_SCORE; +import static org.opensearch.knn.common.KNNConstants.NAME; import static org.opensearch.knn.common.KNNConstants.NMSLIB_NAME; +import static org.opensearch.knn.common.KNNConstants.PARAMETERS; import static org.opensearch.knn.common.KNNConstants.VECTOR_DATA_TYPE_FIELD; public class LuceneEngineIT extends KNNRestTestCase { @@ -466,6 +475,197 @@ public void testRadiusSearch_usingScoreThreshold_withFilter_usingCosineMetrics_u validateRadiusSearchResults(TEST_QUERY_VECTORS, null, score, SpaceType.COSINESIMIL, expectedResults, COLOR_FIELD_NAME, "red", null); } + @SneakyThrows + public void testSQ_withInvalidParams_thenThrowException() { + + // Use invalid number of bits for the bits param which throws an exception + int bits = -1; + expectThrows( + ResponseException.class, + () -> createKnnIndexMappingWithLuceneEngineAndSQEncoder( + DIMENSION, + SpaceType.L2, + VectorDataType.FLOAT, + bits, + MINIMUM_CONFIDENCE_INTERVAL + ) + ); + + // Use invalid value for confidence_interval param which throws an exception + double confidenceInterval = -2.5; + expectThrows( + ResponseException.class, + () -> createKnnIndexMappingWithLuceneEngineAndSQEncoder( + DIMENSION, + SpaceType.L2, + VectorDataType.FLOAT, + LUCENE_SQ_DEFAULT_BITS, + confidenceInterval + ) + ); + + // Use "byte" data_type with sq encoder which throws an exception + expectThrows( + ResponseException.class, + () -> createKnnIndexMappingWithLuceneEngineAndSQEncoder( + DIMENSION, + SpaceType.L2, + VectorDataType.BYTE, + LUCENE_SQ_DEFAULT_BITS, + MINIMUM_CONFIDENCE_INTERVAL + ) + ); + } + + @SneakyThrows + public void testAddDocWithSQEncoder() { + createKnnIndexMappingWithLuceneEngineAndSQEncoder( + DIMENSION, + SpaceType.L2, + VectorDataType.FLOAT, + LUCENE_SQ_DEFAULT_BITS, + MAXIMUM_CONFIDENCE_INTERVAL + ); + Float[] vector = new Float[] { 2.0f, 4.5f, 6.5f }; + addKnnDoc(INDEX_NAME, DOC_ID, FIELD_NAME, vector); + + refreshIndex(INDEX_NAME); + assertEquals(1, getDocCount(INDEX_NAME)); + } + + @SneakyThrows + public void testUpdateDocWithSQEncoder() { + createKnnIndexMappingWithLuceneEngineAndSQEncoder( + DIMENSION, + SpaceType.INNER_PRODUCT, + VectorDataType.FLOAT, + LUCENE_SQ_DEFAULT_BITS, + MAXIMUM_CONFIDENCE_INTERVAL + ); + Float[] vector = { 6.0f, 6.0f, 7.0f }; + addKnnDoc(INDEX_NAME, DOC_ID, FIELD_NAME, vector); + + Float[] updatedVector = { 8.0f, 8.0f, 8.0f }; + updateKnnDoc(INDEX_NAME, DOC_ID, FIELD_NAME, updatedVector); + + refreshIndex(INDEX_NAME); + assertEquals(1, getDocCount(INDEX_NAME)); + } + + @SneakyThrows + public void testDeleteDocWithSQEncoder() { + createKnnIndexMappingWithLuceneEngineAndSQEncoder( + DIMENSION, + SpaceType.INNER_PRODUCT, + VectorDataType.FLOAT, + LUCENE_SQ_DEFAULT_BITS, + MAXIMUM_CONFIDENCE_INTERVAL + ); + Float[] vector = { 6.0f, 6.0f, 7.0f }; + addKnnDoc(INDEX_NAME, DOC_ID, FIELD_NAME, vector); + + deleteKnnDoc(INDEX_NAME, DOC_ID); + + refreshIndex(INDEX_NAME); + assertEquals(0, getDocCount(INDEX_NAME)); + } + + @SneakyThrows + public void testIndexingAndQueryingWithSQEncoder() { + createKnnIndexMappingWithLuceneEngineAndSQEncoder( + DIMENSION, + SpaceType.INNER_PRODUCT, + VectorDataType.FLOAT, + LUCENE_SQ_DEFAULT_BITS, + MAXIMUM_CONFIDENCE_INTERVAL + ); + + int numDocs = 10; + for (int i = 0; i < numDocs; i++) { + float[] indexVector = new float[DIMENSION]; + Arrays.fill(indexVector, (float) i); + addKnnDocWithAttributes(INDEX_NAME, Integer.toString(i), FIELD_NAME, indexVector, ImmutableMap.of("rating", String.valueOf(i))); + } + + // Assert that all docs are ingested + refreshAllNonSystemIndices(); + assertEquals(numDocs, getDocCount(INDEX_NAME)); + + float[] queryVector = new float[DIMENSION]; + Arrays.fill(queryVector, (float) numDocs); + int k = 10; + + Response searchResponse = searchKNNIndex(INDEX_NAME, new KNNQueryBuilder(FIELD_NAME, queryVector, k), k); + List results = parseSearchResponse(EntityUtils.toString(searchResponse.getEntity()), FIELD_NAME); + assertEquals(k, results.size()); + for (int i = 0; i < k; i++) { + assertEquals(numDocs - i - 1, Integer.parseInt(results.get(i).getDocId())); + } + } + + public void testQueryWithFilterUsingSQEncoder() throws Exception { + createKnnIndexMappingWithLuceneEngineAndSQEncoder( + DIMENSION, + SpaceType.INNER_PRODUCT, + VectorDataType.FLOAT, + LUCENE_SQ_DEFAULT_BITS, + MAXIMUM_CONFIDENCE_INTERVAL + ); + + addKnnDocWithAttributes( + DOC_ID, + new float[] { 6.0f, 7.9f, 3.1f }, + ImmutableMap.of(COLOR_FIELD_NAME, "red", TASTE_FIELD_NAME, "sweet") + ); + addKnnDocWithAttributes(DOC_ID_2, new float[] { 3.2f, 2.1f, 4.8f }, ImmutableMap.of(COLOR_FIELD_NAME, "green")); + addKnnDocWithAttributes(DOC_ID_3, new float[] { 4.1f, 5.0f, 7.1f }, ImmutableMap.of(COLOR_FIELD_NAME, "red")); + + refreshIndex(INDEX_NAME); + + final float[] searchVector = { 6.0f, 6.0f, 4.1f }; + List expectedDocIdsKGreaterThanFilterResult = Arrays.asList(DOC_ID, DOC_ID_3); + List expectedDocIdsKLimitsFilterResult = Arrays.asList(DOC_ID); + validateQueryResultsWithFilters(searchVector, 5, 1, expectedDocIdsKGreaterThanFilterResult, expectedDocIdsKLimitsFilterResult); + } + + private void createKnnIndexMappingWithLuceneEngineAndSQEncoder( + int dimension, + SpaceType spaceType, + VectorDataType vectorDataType, + int bits, + double confidenceInterval + ) throws Exception { + XContentBuilder builder = XContentFactory.jsonBuilder() + .startObject() + .startObject(PROPERTIES_FIELD_NAME) + .startObject(FIELD_NAME) + .field(TYPE_FIELD_NAME, KNN_VECTOR_TYPE) + .field(DIMENSION_FIELD_NAME, dimension) + .field(VECTOR_DATA_TYPE_FIELD, vectorDataType) + .startObject(KNNConstants.KNN_METHOD) + .field(KNNConstants.NAME, KNNEngine.LUCENE.getMethod(METHOD_HNSW).getMethodComponent().getName()) + .field(KNNConstants.METHOD_PARAMETER_SPACE_TYPE, spaceType.getValue()) + .field(KNNConstants.KNN_ENGINE, KNNEngine.LUCENE.getName()) + .startObject(KNNConstants.PARAMETERS) + .field(KNNConstants.METHOD_PARAMETER_M, M) + .field(KNNConstants.METHOD_PARAMETER_EF_CONSTRUCTION, EF_CONSTRUCTION) + .startObject(METHOD_ENCODER_PARAMETER) + .field(NAME, ENCODER_SQ) + .startObject(PARAMETERS) + .field(LUCENE_SQ_BITS, bits) + .field(LUCENE_SQ_CONFIDENCE_INTERVAL, confidenceInterval) + .endObject() + .endObject() + .endObject() + .endObject() + .endObject() + .endObject() + .endObject(); + + String mapping = builder.toString(); + createKnnIndex(INDEX_NAME, mapping); + } + private void createKnnIndexMappingWithLuceneEngine(int dimension, SpaceType spaceType, VectorDataType vectorDataType) throws Exception { XContentBuilder builder = XContentFactory.jsonBuilder() .startObject() diff --git a/src/test/java/org/opensearch/knn/index/ParameterTests.java b/src/test/java/org/opensearch/knn/index/ParameterTests.java index 2f3f19727..18b892499 100644 --- a/src/test/java/org/opensearch/knn/index/ParameterTests.java +++ b/src/test/java/org/opensearch/knn/index/ParameterTests.java @@ -125,6 +125,40 @@ public void testStringParameter_validateWithData() { assertNotNull(parameter.validateWithData("test", testVectorSpaceInfo)); } + public void testDoubleParameter_validate() { + final Parameter.DoubleParameter parameter = new Parameter.DoubleParameter("test_parameter", 1.0, v -> v >= 0); + + // valid value + assertNull(parameter.validate(0.9)); + + // Invalid type + assertNotNull(parameter.validate(true)); + + // Invalid type + assertNotNull(parameter.validate(-1)); + + } + + public void testDoubleParameter_validateWithData() { + final Parameter.DoubleParameter parameter = new Parameter.DoubleParameter( + "test", + 1.0, + v -> v > 0, + (v, vectorSpaceInfo) -> v > vectorSpaceInfo.getDimension() + ); + + VectorSpaceInfo testVectorSpaceInfo = new VectorSpaceInfo(0); + + // Invalid type + assertNotNull(parameter.validateWithData("String", testVectorSpaceInfo)); + + // Invalid value + assertNotNull(parameter.validateWithData(-1, testVectorSpaceInfo)); + + // valid value + assertNull(parameter.validateWithData(1.2, testVectorSpaceInfo)); + } + public void testMethodComponentContextParameter_validate() { String methodComponentName1 = "method-1"; String parameterKey1 = "parameter_key_1"; diff --git a/src/test/java/org/opensearch/knn/index/codec/params/KNNScalarQuantizedVectorsFormatParamsTests.java b/src/test/java/org/opensearch/knn/index/codec/params/KNNScalarQuantizedVectorsFormatParamsTests.java new file mode 100644 index 000000000..bcba8ebbd --- /dev/null +++ b/src/test/java/org/opensearch/knn/index/codec/params/KNNScalarQuantizedVectorsFormatParamsTests.java @@ -0,0 +1,110 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.knn.index.codec.params; + +import junit.framework.TestCase; +import org.opensearch.knn.index.MethodComponentContext; + +import java.util.HashMap; +import java.util.Map; + +import static org.opensearch.knn.common.KNNConstants.ENCODER_SQ; +import static org.opensearch.knn.common.KNNConstants.LUCENE_SQ_CONFIDENCE_INTERVAL; +import static org.opensearch.knn.common.KNNConstants.LUCENE_SQ_DEFAULT_BITS; +import static org.opensearch.knn.common.KNNConstants.METHOD_ENCODER_PARAMETER; +import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_EF_CONSTRUCTION; +import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_M; +import static org.opensearch.knn.common.KNNConstants.MINIMUM_CONFIDENCE_INTERVAL; + +public class KNNScalarQuantizedVectorsFormatParamsTests extends TestCase { + private static final int DEFAULT_MAX_CONNECTIONS = 16; + private static final int DEFAULT_BEAM_WIDTH = 100; + + public void testInitParams_whenCalled_thenReturnDefaultParams() { + KNNScalarQuantizedVectorsFormatParams knnScalarQuantizedVectorsFormatParams = new KNNScalarQuantizedVectorsFormatParams( + getDefaultParamsForConstructor(), + DEFAULT_MAX_CONNECTIONS, + DEFAULT_BEAM_WIDTH + ); + + assertEquals(DEFAULT_MAX_CONNECTIONS, knnScalarQuantizedVectorsFormatParams.getMaxConnections()); + assertEquals(DEFAULT_BEAM_WIDTH, knnScalarQuantizedVectorsFormatParams.getBeamWidth()); + assertNull(knnScalarQuantizedVectorsFormatParams.getConfidenceInterval()); + assertTrue(knnScalarQuantizedVectorsFormatParams.isCompressFlag()); + assertEquals(LUCENE_SQ_DEFAULT_BITS, knnScalarQuantizedVectorsFormatParams.getBits()); + } + + public void testInitParams_whenCalled_thenReturnParams() { + int m = 64; + int efConstruction = 128; + + Map encoderParams = new HashMap<>(); + encoderParams.put(LUCENE_SQ_CONFIDENCE_INTERVAL, MINIMUM_CONFIDENCE_INTERVAL); + MethodComponentContext encoderComponentContext = new MethodComponentContext(ENCODER_SQ, encoderParams); + + Map params = new HashMap<>(); + params.put(METHOD_ENCODER_PARAMETER, encoderComponentContext); + params.put(METHOD_PARAMETER_M, m); + params.put(METHOD_PARAMETER_EF_CONSTRUCTION, efConstruction); + + KNNScalarQuantizedVectorsFormatParams knnScalarQuantizedVectorsFormatParams = new KNNScalarQuantizedVectorsFormatParams( + params, + DEFAULT_MAX_CONNECTIONS, + DEFAULT_BEAM_WIDTH + ); + + assertEquals(m, knnScalarQuantizedVectorsFormatParams.getMaxConnections()); + assertEquals(efConstruction, knnScalarQuantizedVectorsFormatParams.getBeamWidth()); + assertEquals((float) MINIMUM_CONFIDENCE_INTERVAL, knnScalarQuantizedVectorsFormatParams.getConfidenceInterval()); + assertTrue(knnScalarQuantizedVectorsFormatParams.isCompressFlag()); + assertEquals(LUCENE_SQ_DEFAULT_BITS, knnScalarQuantizedVectorsFormatParams.getBits()); + } + + public void testValidate_whenCalled_thenReturnTrue() { + Map params = getDefaultParamsForConstructor(); + KNNScalarQuantizedVectorsFormatParams knnScalarQuantizedVectorsFormatParams = new KNNScalarQuantizedVectorsFormatParams( + params, + DEFAULT_MAX_CONNECTIONS, + DEFAULT_BEAM_WIDTH + ); + assertTrue(knnScalarQuantizedVectorsFormatParams.validate(params)); + } + + public void testValidate_whenCalled_thenReturnFalse() { + KNNScalarQuantizedVectorsFormatParams knnScalarQuantizedVectorsFormatParams = new KNNScalarQuantizedVectorsFormatParams( + getDefaultParamsForConstructor(), + DEFAULT_MAX_CONNECTIONS, + DEFAULT_BEAM_WIDTH + ); + Map params = new HashMap<>(); + + // Return false if encoder value is null + params.put(METHOD_ENCODER_PARAMETER, null); + assertFalse(knnScalarQuantizedVectorsFormatParams.validate(params)); + + // Return false if encoder value is not an instance of MethodComponentContext + params.replace(METHOD_ENCODER_PARAMETER, "dummy string"); + assertFalse(knnScalarQuantizedVectorsFormatParams.validate(params)); + + // Return false if encoder name is not "sq" + MethodComponentContext encoderComponentContext = new MethodComponentContext("invalid encoder name", new HashMap<>()); + params.replace(METHOD_ENCODER_PARAMETER, encoderComponentContext); + assertFalse(knnScalarQuantizedVectorsFormatParams.validate(params)); + } + + private Map getDefaultParamsForConstructor() { + MethodComponentContext encoderComponentContext = new MethodComponentContext(ENCODER_SQ, new HashMap<>()); + Map params = new HashMap<>(); + params.put(METHOD_ENCODER_PARAMETER, encoderComponentContext); + return params; + } +} diff --git a/src/test/java/org/opensearch/knn/index/codec/params/KNNVectorsFormatParamsTests.java b/src/test/java/org/opensearch/knn/index/codec/params/KNNVectorsFormatParamsTests.java new file mode 100644 index 000000000..dca054046 --- /dev/null +++ b/src/test/java/org/opensearch/knn/index/codec/params/KNNVectorsFormatParamsTests.java @@ -0,0 +1,56 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.knn.index.codec.params; + +import junit.framework.TestCase; + +import java.util.HashMap; +import java.util.Map; + +import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_EF_CONSTRUCTION; +import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_M; + +public class KNNVectorsFormatParamsTests extends TestCase { + private static final int DEFAULT_MAX_CONNECTIONS = 16; + private static final int DEFAULT_BEAM_WIDTH = 100; + + public void testInitParams_whenCalled_thenReturnDefaultParams() { + KNNVectorsFormatParams knnVectorsFormatParams = new KNNVectorsFormatParams( + new HashMap<>(), + DEFAULT_MAX_CONNECTIONS, + DEFAULT_BEAM_WIDTH + ); + assertEquals(DEFAULT_MAX_CONNECTIONS, knnVectorsFormatParams.getMaxConnections()); + assertEquals(DEFAULT_BEAM_WIDTH, knnVectorsFormatParams.getBeamWidth()); + } + + public void testInitParams_whenCalled_thenReturnParams() { + int m = 64; + int efConstruction = 128; + Map params = new HashMap<>(); + params.put(METHOD_PARAMETER_M, m); + params.put(METHOD_PARAMETER_EF_CONSTRUCTION, efConstruction); + + KNNVectorsFormatParams knnVectorsFormatParams = new KNNVectorsFormatParams(params, DEFAULT_MAX_CONNECTIONS, DEFAULT_BEAM_WIDTH); + assertEquals(m, knnVectorsFormatParams.getMaxConnections()); + assertEquals(efConstruction, knnVectorsFormatParams.getBeamWidth()); + } + + public void testValidate_whenCalled_thenReturnTrue() { + KNNVectorsFormatParams knnVectorsFormatParams = new KNNVectorsFormatParams( + new HashMap<>(), + DEFAULT_MAX_CONNECTIONS, + DEFAULT_BEAM_WIDTH + ); + assertTrue(knnVectorsFormatParams.validate(new HashMap<>())); + } +} From deea892ace50a513a2c42a603263af5d4288332f Mon Sep 17 00:00:00 2001 From: Tejas Shah Date: Wed, 24 Jul 2024 13:10:18 -0700 Subject: [PATCH 303/416] Removes benchmark folder from 2.x branch to resolve CVEs from the folder (#1875) Signed-off-by: Tejas Shah --- benchmarks/osb/README.md | 478 --------- benchmarks/osb/__init__.py | 0 benchmarks/osb/extensions/__init__.py | 0 benchmarks/osb/extensions/data_set.py | 202 ---- benchmarks/osb/extensions/param_sources.py | 217 ---- benchmarks/osb/extensions/registry.py | 13 - benchmarks/osb/extensions/runners.py | 121 --- benchmarks/osb/extensions/util.py | 71 -- benchmarks/osb/indices/faiss-index.json | 27 - benchmarks/osb/indices/lucene-index.json | 26 - benchmarks/osb/indices/model-index.json | 17 - benchmarks/osb/indices/nmslib-index.json | 27 - benchmarks/osb/indices/train-index.json | 16 - benchmarks/osb/operations/default.json | 53 - benchmarks/osb/params/no-train-params.json | 40 - benchmarks/osb/params/train-params.json | 38 - benchmarks/osb/procedures/no-train-test.json | 73 -- benchmarks/osb/procedures/train-test.json | 127 --- benchmarks/osb/requirements.in | 4 - benchmarks/osb/requirements.txt | 96 -- benchmarks/osb/tests/__init__.py | 0 benchmarks/osb/tests/data_set_helper.py | 197 ---- benchmarks/osb/tests/test_param_sources.py | 353 ------- benchmarks/osb/workload.json | 17 - benchmarks/osb/workload.py | 18 - benchmarks/perf-tool/.pylintrc | 443 -------- benchmarks/perf-tool/.style.yapf | 10 - benchmarks/perf-tool/README.md | 449 -------- .../perf-tool/add-filters-to-dataset.py | 200 ---- .../perf-tool/add-parent-doc-id-to-dataset.py | 291 ------ benchmarks/perf-tool/dataset/data-nested.hdf5 | Bin 74496 -> 0 bytes .../dataset/data-with-attr-with-filters.hdf5 | Bin 2404096 -> 0 bytes .../perf-tool/dataset/data-with-attr.hdf5 | Bin 306896 -> 0 bytes benchmarks/perf-tool/dataset/data.hdf5 | Bin 527648 -> 0 bytes benchmarks/perf-tool/knn-perf-tool.py | 10 - benchmarks/perf-tool/okpt/__init__.py | 6 - benchmarks/perf-tool/okpt/diff/diff.py | 142 --- benchmarks/perf-tool/okpt/io/args.py | 178 ---- .../perf-tool/okpt/io/config/parsers/base.py | 67 -- .../perf-tool/okpt/io/config/parsers/test.py | 81 -- .../perf-tool/okpt/io/config/parsers/util.py | 116 -- .../perf-tool/okpt/io/config/schemas/test.yml | 35 - benchmarks/perf-tool/okpt/io/dataset.py | 222 ---- benchmarks/perf-tool/okpt/io/utils/reader.py | 84 -- benchmarks/perf-tool/okpt/io/utils/writer.py | 40 - benchmarks/perf-tool/okpt/main.py | 55 - benchmarks/perf-tool/okpt/test/__init__.py | 5 - benchmarks/perf-tool/okpt/test/profile.py | 86 -- benchmarks/perf-tool/okpt/test/runner.py | 107 -- benchmarks/perf-tool/okpt/test/steps/base.py | 60 -- .../perf-tool/okpt/test/steps/factory.py | 50 - benchmarks/perf-tool/okpt/test/steps/steps.py | 987 ------------------ benchmarks/perf-tool/okpt/test/test.py | 188 ---- .../filtering/relaxed-filter/index.json | 27 - .../relaxed-filter/relaxed-filter-spec.json | 42 - .../relaxed-filter/relaxed-filter-test.yml | 40 - .../filtering/restrictive-filter/index.json | 27 - .../restrictive-filter-spec.json | 44 - .../restrictive-filter-test.yml | 40 - .../release-configs/faiss-hnsw/index.json | 27 - .../faiss-hnsw/nested/simple/index.json | 35 - .../nested/simple/simple-nested-test.yml | 37 - .../release-configs/faiss-hnsw/test.yml | 35 - .../release-configs/faiss-hnswpq/index.json | 17 - .../faiss-hnswpq/method-spec.json | 15 - .../release-configs/faiss-hnswpq/test.yml | 59 -- .../faiss-hnswpq/train-index-spec.json | 16 - .../filtering/relaxed-filter/index.json | 17 - .../filtering/relaxed-filter/method-spec.json | 9 - .../relaxed-filter/relaxed-filter-spec.json | 42 - .../relaxed-filter/relaxed-filter-test.yml | 64 -- .../relaxed-filter/train-index-spec.json | 16 - .../filtering/restrictive-filter/index.json | 17 - .../restrictive-filter/method-spec.json | 9 - .../restrictive-filter-spec.json | 44 - .../restrictive-filter-test.yml | 64 -- .../restrictive-filter/train-index-spec.json | 16 - .../release-configs/faiss-ivf/index.json | 17 - .../faiss-ivf/method-spec.json | 9 - .../release-configs/faiss-ivf/test.yml | 59 -- .../faiss-ivf/train-index-spec.json | 16 - .../release-configs/faiss-ivfpq/index.json | 17 - .../faiss-ivfpq/method-spec.json | 16 - .../release-configs/faiss-ivfpq/test.yml | 59 -- .../faiss-ivfpq/train-index-spec.json | 16 - .../filtering/relaxed-filter/index.json | 26 - .../relaxed-filter/relaxed-filter-spec.json | 42 - .../relaxed-filter/relaxed-filter-test.yml | 38 - .../filtering/restrictive-filter/index.json | 26 - .../restrictive-filter-spec.json | 44 - .../restrictive-filter-test.yml | 38 - .../release-configs/lucene-hnsw/index.json | 26 - .../lucene-hnsw/nested/simple/index.json | 34 - .../nested/simple/simple-nested-test.yml | 37 - .../release-configs/lucene-hnsw/test.yml | 33 - .../release-configs/nmslib-hnsw/index.json | 27 - .../release-configs/nmslib-hnsw/test.yml | 35 - .../release-configs/run_all_tests.sh | 102 -- benchmarks/perf-tool/requirements.in | 7 - benchmarks/perf-tool/requirements.txt | 37 - .../faiss-sift-ivf/index-spec.json | 17 - .../faiss-sift-ivf/method-spec.json | 8 - .../sample-configs/faiss-sift-ivf/test.yml | 62 -- .../faiss-sift-ivf/train-index-spec.json | 16 - .../filter-spec/filter-1-spec.json | 24 - .../filter-spec/filter-2-spec.json | 40 - .../filter-spec/filter-3-spec.json | 30 - .../filter-spec/filter-4-spec.json | 44 - .../filter-spec/filter-5-spec.json | 42 - .../lucene-sift-hnsw-filter/index-spec.json | 27 - .../lucene-sift-hnsw-filter/test.yml | 41 - .../nmslib-sift-hnsw/index-spec.json | 28 - .../sample-configs/nmslib-sift-hnsw/test.yml | 38 - 113 files changed, 8080 deletions(-) delete mode 100644 benchmarks/osb/README.md delete mode 100644 benchmarks/osb/__init__.py delete mode 100644 benchmarks/osb/extensions/__init__.py delete mode 100644 benchmarks/osb/extensions/data_set.py delete mode 100644 benchmarks/osb/extensions/param_sources.py delete mode 100644 benchmarks/osb/extensions/registry.py delete mode 100644 benchmarks/osb/extensions/runners.py delete mode 100644 benchmarks/osb/extensions/util.py delete mode 100644 benchmarks/osb/indices/faiss-index.json delete mode 100644 benchmarks/osb/indices/lucene-index.json delete mode 100644 benchmarks/osb/indices/model-index.json delete mode 100644 benchmarks/osb/indices/nmslib-index.json delete mode 100644 benchmarks/osb/indices/train-index.json delete mode 100644 benchmarks/osb/operations/default.json delete mode 100644 benchmarks/osb/params/no-train-params.json delete mode 100644 benchmarks/osb/params/train-params.json delete mode 100644 benchmarks/osb/procedures/no-train-test.json delete mode 100644 benchmarks/osb/procedures/train-test.json delete mode 100644 benchmarks/osb/requirements.in delete mode 100644 benchmarks/osb/requirements.txt delete mode 100644 benchmarks/osb/tests/__init__.py delete mode 100644 benchmarks/osb/tests/data_set_helper.py delete mode 100644 benchmarks/osb/tests/test_param_sources.py delete mode 100644 benchmarks/osb/workload.json delete mode 100644 benchmarks/osb/workload.py delete mode 100644 benchmarks/perf-tool/.pylintrc delete mode 100644 benchmarks/perf-tool/.style.yapf delete mode 100644 benchmarks/perf-tool/README.md delete mode 100644 benchmarks/perf-tool/add-filters-to-dataset.py delete mode 100644 benchmarks/perf-tool/add-parent-doc-id-to-dataset.py delete mode 100644 benchmarks/perf-tool/dataset/data-nested.hdf5 delete mode 100644 benchmarks/perf-tool/dataset/data-with-attr-with-filters.hdf5 delete mode 100644 benchmarks/perf-tool/dataset/data-with-attr.hdf5 delete mode 100644 benchmarks/perf-tool/dataset/data.hdf5 delete mode 100644 benchmarks/perf-tool/knn-perf-tool.py delete mode 100644 benchmarks/perf-tool/okpt/__init__.py delete mode 100644 benchmarks/perf-tool/okpt/diff/diff.py delete mode 100644 benchmarks/perf-tool/okpt/io/args.py delete mode 100644 benchmarks/perf-tool/okpt/io/config/parsers/base.py delete mode 100644 benchmarks/perf-tool/okpt/io/config/parsers/test.py delete mode 100644 benchmarks/perf-tool/okpt/io/config/parsers/util.py delete mode 100644 benchmarks/perf-tool/okpt/io/config/schemas/test.yml delete mode 100644 benchmarks/perf-tool/okpt/io/dataset.py delete mode 100644 benchmarks/perf-tool/okpt/io/utils/reader.py delete mode 100644 benchmarks/perf-tool/okpt/io/utils/writer.py delete mode 100644 benchmarks/perf-tool/okpt/main.py delete mode 100644 benchmarks/perf-tool/okpt/test/__init__.py delete mode 100644 benchmarks/perf-tool/okpt/test/profile.py delete mode 100644 benchmarks/perf-tool/okpt/test/runner.py delete mode 100644 benchmarks/perf-tool/okpt/test/steps/base.py delete mode 100644 benchmarks/perf-tool/okpt/test/steps/factory.py delete mode 100644 benchmarks/perf-tool/okpt/test/steps/steps.py delete mode 100644 benchmarks/perf-tool/okpt/test/test.py delete mode 100644 benchmarks/perf-tool/release-configs/faiss-hnsw/filtering/relaxed-filter/index.json delete mode 100644 benchmarks/perf-tool/release-configs/faiss-hnsw/filtering/relaxed-filter/relaxed-filter-spec.json delete mode 100644 benchmarks/perf-tool/release-configs/faiss-hnsw/filtering/relaxed-filter/relaxed-filter-test.yml delete mode 100644 benchmarks/perf-tool/release-configs/faiss-hnsw/filtering/restrictive-filter/index.json delete mode 100644 benchmarks/perf-tool/release-configs/faiss-hnsw/filtering/restrictive-filter/restrictive-filter-spec.json delete mode 100644 benchmarks/perf-tool/release-configs/faiss-hnsw/filtering/restrictive-filter/restrictive-filter-test.yml delete mode 100644 benchmarks/perf-tool/release-configs/faiss-hnsw/index.json delete mode 100644 benchmarks/perf-tool/release-configs/faiss-hnsw/nested/simple/index.json delete mode 100644 benchmarks/perf-tool/release-configs/faiss-hnsw/nested/simple/simple-nested-test.yml delete mode 100644 benchmarks/perf-tool/release-configs/faiss-hnsw/test.yml delete mode 100644 benchmarks/perf-tool/release-configs/faiss-hnswpq/index.json delete mode 100644 benchmarks/perf-tool/release-configs/faiss-hnswpq/method-spec.json delete mode 100644 benchmarks/perf-tool/release-configs/faiss-hnswpq/test.yml delete mode 100644 benchmarks/perf-tool/release-configs/faiss-hnswpq/train-index-spec.json delete mode 100644 benchmarks/perf-tool/release-configs/faiss-ivf/filtering/relaxed-filter/index.json delete mode 100644 benchmarks/perf-tool/release-configs/faiss-ivf/filtering/relaxed-filter/method-spec.json delete mode 100644 benchmarks/perf-tool/release-configs/faiss-ivf/filtering/relaxed-filter/relaxed-filter-spec.json delete mode 100644 benchmarks/perf-tool/release-configs/faiss-ivf/filtering/relaxed-filter/relaxed-filter-test.yml delete mode 100644 benchmarks/perf-tool/release-configs/faiss-ivf/filtering/relaxed-filter/train-index-spec.json delete mode 100644 benchmarks/perf-tool/release-configs/faiss-ivf/filtering/restrictive-filter/index.json delete mode 100644 benchmarks/perf-tool/release-configs/faiss-ivf/filtering/restrictive-filter/method-spec.json delete mode 100644 benchmarks/perf-tool/release-configs/faiss-ivf/filtering/restrictive-filter/restrictive-filter-spec.json delete mode 100644 benchmarks/perf-tool/release-configs/faiss-ivf/filtering/restrictive-filter/restrictive-filter-test.yml delete mode 100644 benchmarks/perf-tool/release-configs/faiss-ivf/filtering/restrictive-filter/train-index-spec.json delete mode 100644 benchmarks/perf-tool/release-configs/faiss-ivf/index.json delete mode 100644 benchmarks/perf-tool/release-configs/faiss-ivf/method-spec.json delete mode 100644 benchmarks/perf-tool/release-configs/faiss-ivf/test.yml delete mode 100644 benchmarks/perf-tool/release-configs/faiss-ivf/train-index-spec.json delete mode 100644 benchmarks/perf-tool/release-configs/faiss-ivfpq/index.json delete mode 100644 benchmarks/perf-tool/release-configs/faiss-ivfpq/method-spec.json delete mode 100644 benchmarks/perf-tool/release-configs/faiss-ivfpq/test.yml delete mode 100644 benchmarks/perf-tool/release-configs/faiss-ivfpq/train-index-spec.json delete mode 100644 benchmarks/perf-tool/release-configs/lucene-hnsw/filtering/relaxed-filter/index.json delete mode 100644 benchmarks/perf-tool/release-configs/lucene-hnsw/filtering/relaxed-filter/relaxed-filter-spec.json delete mode 100644 benchmarks/perf-tool/release-configs/lucene-hnsw/filtering/relaxed-filter/relaxed-filter-test.yml delete mode 100644 benchmarks/perf-tool/release-configs/lucene-hnsw/filtering/restrictive-filter/index.json delete mode 100644 benchmarks/perf-tool/release-configs/lucene-hnsw/filtering/restrictive-filter/restrictive-filter-spec.json delete mode 100644 benchmarks/perf-tool/release-configs/lucene-hnsw/filtering/restrictive-filter/restrictive-filter-test.yml delete mode 100644 benchmarks/perf-tool/release-configs/lucene-hnsw/index.json delete mode 100644 benchmarks/perf-tool/release-configs/lucene-hnsw/nested/simple/index.json delete mode 100644 benchmarks/perf-tool/release-configs/lucene-hnsw/nested/simple/simple-nested-test.yml delete mode 100644 benchmarks/perf-tool/release-configs/lucene-hnsw/test.yml delete mode 100644 benchmarks/perf-tool/release-configs/nmslib-hnsw/index.json delete mode 100644 benchmarks/perf-tool/release-configs/nmslib-hnsw/test.yml delete mode 100755 benchmarks/perf-tool/release-configs/run_all_tests.sh delete mode 100644 benchmarks/perf-tool/requirements.in delete mode 100644 benchmarks/perf-tool/requirements.txt delete mode 100644 benchmarks/perf-tool/sample-configs/faiss-sift-ivf/index-spec.json delete mode 100644 benchmarks/perf-tool/sample-configs/faiss-sift-ivf/method-spec.json delete mode 100644 benchmarks/perf-tool/sample-configs/faiss-sift-ivf/test.yml delete mode 100644 benchmarks/perf-tool/sample-configs/faiss-sift-ivf/train-index-spec.json delete mode 100644 benchmarks/perf-tool/sample-configs/filter-spec/filter-1-spec.json delete mode 100644 benchmarks/perf-tool/sample-configs/filter-spec/filter-2-spec.json delete mode 100644 benchmarks/perf-tool/sample-configs/filter-spec/filter-3-spec.json delete mode 100644 benchmarks/perf-tool/sample-configs/filter-spec/filter-4-spec.json delete mode 100644 benchmarks/perf-tool/sample-configs/filter-spec/filter-5-spec.json delete mode 100644 benchmarks/perf-tool/sample-configs/lucene-sift-hnsw-filter/index-spec.json delete mode 100644 benchmarks/perf-tool/sample-configs/lucene-sift-hnsw-filter/test.yml delete mode 100644 benchmarks/perf-tool/sample-configs/nmslib-sift-hnsw/index-spec.json delete mode 100644 benchmarks/perf-tool/sample-configs/nmslib-sift-hnsw/test.yml diff --git a/benchmarks/osb/README.md b/benchmarks/osb/README.md deleted file mode 100644 index 0d0b05f8d..000000000 --- a/benchmarks/osb/README.md +++ /dev/null @@ -1,478 +0,0 @@ -# IMPORTANT NOTE: No new features will be added to this tool . This tool is currently in maintanence mode. All new features will be added to [vector search workload]( https://github.com/opensearch-project/opensearch-benchmark-workloads/tree/main/vectorsearch) -# OpenSearch Benchmarks for k-NN - -## Overview - -This directory contains code and configurations to run k-NN benchmarking -workloads using OpenSearch Benchmarks. - -The [extensions](extensions) directory contains common code shared between -procedures. The [procedures](procedures) directory contains the individual -test procedures for this workload. - -## Getting Started - -### OpenSearch Benchmarks Background - -OpenSearch Benchmark is a framework for performance benchmarking an OpenSearch -cluster. For more details, checkout their -[repo](https://github.com/opensearch-project/opensearch-benchmark/). - -Before getting into the benchmarks, it is helpful to know a few terms: -1. Workload - Top level description of a benchmark suite. A workload will have a `workload.json` file that defines different components of the tests -2. Test Procedures - A workload can have a schedule of operations that run the test. However, a workload can also have several test procedures that define their own schedule of operations. This is helpful for sharing code between tests -3. Operation - An action against the OpenSearch cluster -4. Parameter source - Producers of parameters for OpenSearch operations -5. Runners - Code that actually will execute the OpenSearch operations - -### Setup - -OpenSearch Benchmarks requires Python 3.8 or greater to be installed. One of -the easier ways to do this is through Conda, a package and environment -management system for Python. - -First, follow the -[installation instructions](https://docs.conda.io/projects/conda/en/latest/user-guide/install/index.html) -to install Conda on your system. - -Next, create a Python 3.8 environment: -``` -conda create -n knn-osb python=3.8 -``` - -After the environment is created, activate it: -``` -source activate knn-osb -``` - -Lastly, clone the k-NN repo and install all required python packages: -``` -git clone https://github.com/opensearch-project/k-NN.git -cd k-NN/benchmarks/osb -pip install -r requirements.txt -``` - -After all of this completes, you should be ready to run your first benchmark! - -### Running a benchmark - -Before running a benchmark, make sure you have the endpoint of your cluster and - the machine you are running the benchmarks from can access it. - Additionally, ensure that all data has been pulled to the client. - -Currently, we support 2 test procedures for the k-NN workload: train-test and -no-train-test. The train test has steps to train a model included in the -schedule, while no train does not. Both test procedures will index a data set -of vectors into an OpenSearch index and then run a set of queries against them. - -Once you have decided which test procedure you want to use, open up -[params/train-params.json](params/train-params.json) or -[params/no-train-params.json](params/no-train-params.json) and -fill out the parameters. Notice, at the bottom of `no-train-params.json` there -are several parameters that relate to training. Ignore these. They need to be -defined for the workload but not used. - -Once the parameters are set, set the URL and PORT of your cluster and run the -command to run the test procedure. - -``` -export URL= -export PORT= -export PARAMS_FILE= -export PROCEDURE={no-train-test | train-test} - -opensearch-benchmark execute_test \ - --target-hosts $URL:$PORT \ - --workload-path ./workload.json \ - --workload-params ${PARAMS_FILE} \ - --test-procedure=${PROCEDURE} \ - --pipeline benchmark-only -``` - -## Current Procedures - -### No Train Test - -The No Train Test procedure is used to test `knn_vector` indices that do not -use an algorithm that requires training. - -#### Workflow - -1. Delete old resources in the cluster if they are present -2. Create an OpenSearch index with `knn_vector` configured to use the HNSW algorithm -3. Wait for cluster to be green -4. Ingest data set into the cluster -5. Refresh the index -6. Run queries from data set against the cluster - -#### Parameters - -| Name | Description | -|-----------------------------------------|--------------------------------------------------------------------------| -| target_index_name | Name of index to add vectors to | -| target_field_name | Name of field to add vectors to | -| target_index_body | Path to target index definition | -| target_index_primary_shards | Target index primary shards | -| target_index_replica_shards | Target index replica shards | -| target_index_dimension | Dimension of target index | -| target_index_space_type | Target index space type | -| target_index_bulk_size | Target index bulk size | -| target_index_bulk_index_data_set_format | Format of vector data set | -| target_index_bulk_index_data_set_path | Path to vector data set | -| target_index_bulk_index_clients | Clients to be used for bulk ingestion (must be divisor of data set size) | -| target_index_max_num_segments | Number of segments to merge target index down to before beginning search | -| target_index_force_merge_timeout | Timeout for of force merge requests in seconds | -| hnsw_ef_search | HNSW ef search parameter | -| hnsw_ef_construction | HNSW ef construction parameter | -| hnsw_m | HNSW m parameter | -| query_k | The number of neighbors to return for the search | -| query_clients | Number of clients to use for running queries | -| query_data_set_format | Format of vector data set for queries | -| query_data_set_path | Path to vector data set for queries | - -#### Metrics - -The result metrics of this procedure will look like: -``` ------------------------------------------------------- - _______ __ _____ - / ____(_)___ ____ _/ / / ___/_________ ________ - / /_ / / __ \/ __ `/ / \__ \/ ___/ __ \/ ___/ _ \ - / __/ / / / / / /_/ / / ___/ / /__/ /_/ / / / __/ -/_/ /_/_/ /_/\__,_/_/ /____/\___/\____/_/ \___/ ------------------------------------------------------- - -| Metric | Task | Value | Unit | -|---------------------------------------------------------------:|------------------------:|------------:|-------:| -| Cumulative indexing time of primary shards | | 1.82885 | min | -| Min cumulative indexing time across primary shards | | 0.4121 | min | -| Median cumulative indexing time across primary shards | | 0.559617 | min | -| Max cumulative indexing time across primary shards | | 0.857133 | min | -| Cumulative indexing throttle time of primary shards | | 0 | min | -| Min cumulative indexing throttle time across primary shards | | 0 | min | -| Median cumulative indexing throttle time across primary shards | | 0 | min | -| Max cumulative indexing throttle time across primary shards | | 0 | min | -| Cumulative merge time of primary shards | | 5.89065 | min | -| Cumulative merge count of primary shards | | 3 | | -| Min cumulative merge time across primary shards | | 1.95945 | min | -| Median cumulative merge time across primary shards | | 1.96345 | min | -| Max cumulative merge time across primary shards | | 1.96775 | min | -| Cumulative merge throttle time of primary shards | | 0 | min | -| Min cumulative merge throttle time across primary shards | | 0 | min | -| Median cumulative merge throttle time across primary shards | | 0 | min | -| Max cumulative merge throttle time across primary shards | | 0 | min | -| Cumulative refresh time of primary shards | | 8.52517 | min | -| Cumulative refresh count of primary shards | | 29 | | -| Min cumulative refresh time across primary shards | | 2.64265 | min | -| Median cumulative refresh time across primary shards | | 2.93913 | min | -| Max cumulative refresh time across primary shards | | 2.94338 | min | -| Cumulative flush time of primary shards | | 0.00221667 | min | -| Cumulative flush count of primary shards | | 3 | | -| Min cumulative flush time across primary shards | | 0.000733333 | min | -| Median cumulative flush time across primary shards | | 0.000733333 | min | -| Max cumulative flush time across primary shards | | 0.00075 | min | -| Total Young Gen GC time | | 0.318 | s | -| Total Young Gen GC count | | 2 | | -| Total Old Gen GC time | | 0 | s | -| Total Old Gen GC count | | 0 | | -| Store size | | 1.43566 | GB | -| Translog size | | 1.53668e-07 | GB | -| Heap used for segments | | 0.00410843 | MB | -| Heap used for doc values | | 0.000286102 | MB | -| Heap used for terms | | 0.00121307 | MB | -| Heap used for norms | | 0 | MB | -| Heap used for points | | 0 | MB | -| Heap used for stored fields | | 0.00260925 | MB | -| Segment count | | 3 | | -| Min Throughput | custom-vector-bulk | 38005.8 | docs/s | -| Mean Throughput | custom-vector-bulk | 44827.9 | docs/s | -| Median Throughput | custom-vector-bulk | 40507.2 | docs/s | -| Max Throughput | custom-vector-bulk | 88967.8 | docs/s | -| 50th percentile latency | custom-vector-bulk | 29.5857 | ms | -| 90th percentile latency | custom-vector-bulk | 49.0719 | ms | -| 99th percentile latency | custom-vector-bulk | 72.6138 | ms | -| 99.9th percentile latency | custom-vector-bulk | 279.826 | ms | -| 100th percentile latency | custom-vector-bulk | 15688 | ms | -| 50th percentile service time | custom-vector-bulk | 29.5857 | ms | -| 90th percentile service time | custom-vector-bulk | 49.0719 | ms | -| 99th percentile service time | custom-vector-bulk | 72.6138 | ms | -| 99.9th percentile service time | custom-vector-bulk | 279.826 | ms | -| 100th percentile service time | custom-vector-bulk | 15688 | ms | -| error rate | custom-vector-bulk | 0 | % | -| Min Throughput | refresh-target-index | 0.01 | ops/s | -| Mean Throughput | refresh-target-index | 0.01 | ops/s | -| Median Throughput | refresh-target-index | 0.01 | ops/s | -| Max Throughput | refresh-target-index | 0.01 | ops/s | -| 100th percentile latency | refresh-target-index | 176610 | ms | -| 100th percentile service time | refresh-target-index | 176610 | ms | -| error rate | refresh-target-index | 0 | % | -| Min Throughput | knn-query-from-data-set | 444.17 | ops/s | -| Mean Throughput | knn-query-from-data-set | 601.68 | ops/s | -| Median Throughput | knn-query-from-data-set | 621.19 | ops/s | -| Max Throughput | knn-query-from-data-set | 631.23 | ops/s | -| 50th percentile latency | knn-query-from-data-set | 14.7612 | ms | -| 90th percentile latency | knn-query-from-data-set | 20.6954 | ms | -| 99th percentile latency | knn-query-from-data-set | 27.7499 | ms | -| 99.9th percentile latency | knn-query-from-data-set | 41.3506 | ms | -| 99.99th percentile latency | knn-query-from-data-set | 162.391 | ms | -| 100th percentile latency | knn-query-from-data-set | 162.756 | ms | -| 50th percentile service time | knn-query-from-data-set | 14.7612 | ms | -| 90th percentile service time | knn-query-from-data-set | 20.6954 | ms | -| 99th percentile service time | knn-query-from-data-set | 27.7499 | ms | -| 99.9th percentile service time | knn-query-from-data-set | 41.3506 | ms | -| 99.99th percentile service time | knn-query-from-data-set | 162.391 | ms | -| 100th percentile service time | knn-query-from-data-set | 162.756 | ms | -| error rate | knn-query-from-data-set | 0 | % | - - ---------------------------------- -[INFO] SUCCESS (took 618 seconds) ---------------------------------- -``` - -### Train Test - -The Train Test procedure is used to test `knn_vector` indices that do use an -algorithm that requires training. - -#### Workflow - -1. Delete old resources in the cluster if they are present -2. Create an OpenSearch index with `knn_vector` configured to load with training data -3. Wait for cluster to be green -4. Ingest data set into the training index -5. Refresh the index -6. Train a model based on user provided input parameters -7. Create an OpenSearch index with `knn_vector` configured to use the model -8. Ingest vectors into the target index -9. Refresh the target index -10. Run queries from data set against the cluster - -#### Parameters - -| Name | Description | -|-----------------------------------------|--------------------------------------------------------------------------| -| target_index_name | Name of index to add vectors to | -| target_field_name | Name of field to add vectors to | -| target_index_body | Path to target index definition | -| target_index_primary_shards | Target index primary shards | -| target_index_replica_shards | Target index replica shards | -| target_index_dimension | Dimension of target index | -| target_index_space_type | Target index space type | -| target_index_bulk_size | Target index bulk size | -| target_index_bulk_index_data_set_format | Format of vector data set for ingestion | -| target_index_bulk_index_data_set_path | Path to vector data set for ingestion | -| target_index_bulk_index_clients | Clients to be used for bulk ingestion (must be divisor of data set size) | -| target_index_max_num_segments | Number of segments to merge target index down to before beginning search | -| target_index_force_merge_timeout | Timeout for of force merge requests in seconds | -| ivf_nlists | IVF nlist parameter | -| ivf_nprobes | IVF nprobe parameter | -| pq_code_size | PQ code_size parameter | -| pq_m | PQ m parameter | -| train_model_method | Method to be used for model (ivf or ivfpq) | -| train_model_id | Model ID | -| train_index_name | Name of index to put training data into | -| train_field_name | Name of field to put training data into | -| train_index_body | Path to train index definition | -| train_search_size | Search size to use when pulling training data | -| train_timeout | Timeout to wait for training to finish | -| train_index_primary_shards | Train index primary shards | -| train_index_replica_shards | Train index replica shards | -| train_index_bulk_size | Train index bulk size | -| train_index_data_set_format | Format of vector data set for training | -| train_index_data_set_path | Path to vector data set for training | -| train_index_num_vectors | Number of vectors to use from vector data set for training | -| train_index_bulk_index_clients | Clients to be used for bulk ingestion (must be divisor of data set size) | -| query_k | The number of neighbors to return for the search | -| query_clients | Number of clients to use for running queries | -| query_data_set_format | Format of vector data set for queries | -| query_data_set_path | Path to vector data set for queries | - -#### Metrics - -The result metrics of this procedure will look like: -``` ------------------------------------------------------- - _______ __ _____ - / ____(_)___ ____ _/ / / ___/_________ ________ - / /_ / / __ \/ __ `/ / \__ \/ ___/ __ \/ ___/ _ \ - / __/ / / / / / /_/ / / ___/ / /__/ /_/ / / / __/ -/_/ /_/_/ /_/\__,_/_/ /____/\___/\____/_/ \___/ ------------------------------------------------------- - -| Metric | Task | Value | Unit | -|---------------------------------------------------------------:|------------------------:|-----------:|-----------------:| -| Cumulative indexing time of primary shards | | 2.92382 | min | -| Min cumulative indexing time across primary shards | | 0.42245 | min | -| Median cumulative indexing time across primary shards | | 0.43395 | min | -| Max cumulative indexing time across primary shards | | 1.63347 | min | -| Cumulative indexing throttle time of primary shards | | 0 | min | -| Min cumulative indexing throttle time across primary shards | | 0 | min | -| Median cumulative indexing throttle time across primary shards | | 0 | min | -| Max cumulative indexing throttle time across primary shards | | 0 | min | -| Cumulative merge time of primary shards | | 1.36293 | min | -| Cumulative merge count of primary shards | | 20 | | -| Min cumulative merge time across primary shards | | 0.263283 | min | -| Median cumulative merge time across primary shards | | 0.291733 | min | -| Max cumulative merge time across primary shards | | 0.516183 | min | -| Cumulative merge throttle time of primary shards | | 0.701683 | min | -| Min cumulative merge throttle time across primary shards | | 0.163883 | min | -| Median cumulative merge throttle time across primary shards | | 0.175717 | min | -| Max cumulative merge throttle time across primary shards | | 0.186367 | min | -| Cumulative refresh time of primary shards | | 0.222217 | min | -| Cumulative refresh count of primary shards | | 67 | | -| Min cumulative refresh time across primary shards | | 0.03915 | min | -| Median cumulative refresh time across primary shards | | 0.039825 | min | -| Max cumulative refresh time across primary shards | | 0.103417 | min | -| Cumulative flush time of primary shards | | 0.0276833 | min | -| Cumulative flush count of primary shards | | 1 | | -| Min cumulative flush time across primary shards | | 0 | min | -| Median cumulative flush time across primary shards | | 0 | min | -| Max cumulative flush time across primary shards | | 0.0276833 | min | -| Total Young Gen GC time | | 0.074 | s | -| Total Young Gen GC count | | 8 | | -| Total Old Gen GC time | | 0 | s | -| Total Old Gen GC count | | 0 | | -| Store size | | 1.67839 | GB | -| Translog size | | 0.115145 | GB | -| Heap used for segments | | 0.0350914 | MB | -| Heap used for doc values | | 0.00771713 | MB | -| Heap used for terms | | 0.0101089 | MB | -| Heap used for norms | | 0 | MB | -| Heap used for points | | 0 | MB | -| Heap used for stored fields | | 0.0172653 | MB | -| Segment count | | 25 | | -| Min Throughput | delete-model | 25.45 | ops/s | -| Mean Throughput | delete-model | 25.45 | ops/s | -| Median Throughput | delete-model | 25.45 | ops/s | -| Max Throughput | delete-model | 25.45 | ops/s | -| 100th percentile latency | delete-model | 39.0409 | ms | -| 100th percentile service time | delete-model | 39.0409 | ms | -| error rate | delete-model | 0 | % | -| Min Throughput | train-vector-bulk | 49518.9 | docs/s | -| Mean Throughput | train-vector-bulk | 54418.8 | docs/s | -| Median Throughput | train-vector-bulk | 52984.2 | docs/s | -| Max Throughput | train-vector-bulk | 62118.3 | docs/s | -| 50th percentile latency | train-vector-bulk | 26.5293 | ms | -| 90th percentile latency | train-vector-bulk | 41.8212 | ms | -| 99th percentile latency | train-vector-bulk | 239.351 | ms | -| 99.9th percentile latency | train-vector-bulk | 348.507 | ms | -| 100th percentile latency | train-vector-bulk | 436.292 | ms | -| 50th percentile service time | train-vector-bulk | 26.5293 | ms | -| 90th percentile service time | train-vector-bulk | 41.8212 | ms | -| 99th percentile service time | train-vector-bulk | 239.351 | ms | -| 99.9th percentile service time | train-vector-bulk | 348.507 | ms | -| 100th percentile service time | train-vector-bulk | 436.292 | ms | -| error rate | train-vector-bulk | 0 | % | -| Min Throughput | refresh-train-index | 0.47 | ops/s | -| Mean Throughput | refresh-train-index | 0.47 | ops/s | -| Median Throughput | refresh-train-index | 0.47 | ops/s | -| Max Throughput | refresh-train-index | 0.47 | ops/s | -| 100th percentile latency | refresh-train-index | 2142.96 | ms | -| 100th percentile service time | refresh-train-index | 2142.96 | ms | -| error rate | refresh-train-index | 0 | % | -| Min Throughput | ivfpq-train-model | 0.01 | models_trained/s | -| Mean Throughput | ivfpq-train-model | 0.01 | models_trained/s | -| Median Throughput | ivfpq-train-model | 0.01 | models_trained/s | -| Max Throughput | ivfpq-train-model | 0.01 | models_trained/s | -| 100th percentile latency | ivfpq-train-model | 136563 | ms | -| 100th percentile service time | ivfpq-train-model | 136563 | ms | -| error rate | ivfpq-train-model | 0 | % | -| Min Throughput | custom-vector-bulk | 62384.8 | docs/s | -| Mean Throughput | custom-vector-bulk | 69035.2 | docs/s | -| Median Throughput | custom-vector-bulk | 68675.4 | docs/s | -| Max Throughput | custom-vector-bulk | 80713.4 | docs/s | -| 50th percentile latency | custom-vector-bulk | 18.7726 | ms | -| 90th percentile latency | custom-vector-bulk | 34.8881 | ms | -| 99th percentile latency | custom-vector-bulk | 150.435 | ms | -| 99.9th percentile latency | custom-vector-bulk | 296.862 | ms | -| 100th percentile latency | custom-vector-bulk | 344.394 | ms | -| 50th percentile service time | custom-vector-bulk | 18.7726 | ms | -| 90th percentile service time | custom-vector-bulk | 34.8881 | ms | -| 99th percentile service time | custom-vector-bulk | 150.435 | ms | -| 99.9th percentile service time | custom-vector-bulk | 296.862 | ms | -| 100th percentile service time | custom-vector-bulk | 344.394 | ms | -| error rate | custom-vector-bulk | 0 | % | -| Min Throughput | refresh-target-index | 28.32 | ops/s | -| Mean Throughput | refresh-target-index | 28.32 | ops/s | -| Median Throughput | refresh-target-index | 28.32 | ops/s | -| Max Throughput | refresh-target-index | 28.32 | ops/s | -| 100th percentile latency | refresh-target-index | 34.9811 | ms | -| 100th percentile service time | refresh-target-index | 34.9811 | ms | -| error rate | refresh-target-index | 0 | % | -| Min Throughput | knn-query-from-data-set | 0.9 | ops/s | -| Mean Throughput | knn-query-from-data-set | 453.84 | ops/s | -| Median Throughput | knn-query-from-data-set | 554.15 | ops/s | -| Max Throughput | knn-query-from-data-set | 681 | ops/s | -| 50th percentile latency | knn-query-from-data-set | 11.7174 | ms | -| 90th percentile latency | knn-query-from-data-set | 15.4445 | ms | -| 99th percentile latency | knn-query-from-data-set | 21.0682 | ms | -| 99.9th percentile latency | knn-query-from-data-set | 39.5414 | ms | -| 99.99th percentile latency | knn-query-from-data-set | 1116.33 | ms | -| 100th percentile latency | knn-query-from-data-set | 1116.66 | ms | -| 50th percentile service time | knn-query-from-data-set | 11.7174 | ms | -| 90th percentile service time | knn-query-from-data-set | 15.4445 | ms | -| 99th percentile service time | knn-query-from-data-set | 21.0682 | ms | -| 99.9th percentile service time | knn-query-from-data-set | 39.5414 | ms | -| 99.99th percentile service time | knn-query-from-data-set | 1116.33 | ms | -| 100th percentile service time | knn-query-from-data-set | 1116.66 | ms | -| error rate | knn-query-from-data-set | 0 | % | - - ---------------------------------- -[INFO] SUCCESS (took 281 seconds) ---------------------------------- -``` - -## Adding a procedure - -Adding additional benchmarks is very simple. First, place any custom parameter -sources or runners in the [extensions](extensions) directory so that other tests -can use them and also update the [documentation](#custom-extensions) -accordingly. - -Next, create a new test procedure file and add the operations you want your test -to run. Lastly, be sure to update documentation. - -## Custom Extensions - -OpenSearch Benchmarks is very extendable. To fit the plugins needs, we add -customer parameter sources and custom runners. Parameter sources allow users to -supply custom parameters to an operation. Runners are what actually performs -the operations against OpenSearch. - -### Custom Parameter Sources - -Custom parameter sources are defined in [extensions/param_sources.py](extensions/param_sources.py). - -| Name | Description | Parameters | -|-------------------------|------------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| bulk-from-data-set | Provides bulk payloads containing vectors from a data set for indexing | 1. data_set_format - (hdf5, bigann)
2. data_set_path - path to data set
3. index - name of index for bulk ingestion
4. field - field to place vector in
5. bulk_size - vectors per bulk request
6. num_vectors - number of vectors to use from the data set. Defaults to the whole data set. | -| knn-query-from-data-set | Provides a query generated from a data set | 1. data_set_format - (hdf5, bigann)
2. data_set_path - path to data set
3. index - name of index to query against
4. field - field to to query against
5. k - number of results to return
6. dimension - size of vectors to produce
7. num_vectors - number of vectors to use from the data set. Defaults to the whole data set. | - - -### Custom Runners - -Custom runners are defined in [extensions/runners.py](extensions/runners.py). - -| Syntax | Description | Parameters | -|--------------------|-----------------------------------------------------|:-------------------------------------------------------------------------------------------------------------| -| custom-vector-bulk | Bulk index a set of vectors in an OpenSearch index. | 1. bulk-from-data-set | -| custom-refresh | Run refresh with retry capabilities. | 1. index - name of index to refresh
2. retries - number of times to retry the operation | -| train-model | Trains a model. | 1. body - model definition
2. timeout - time to wait for model to finish
3. model_id - ID of model | -| delete-model | Deletes a model if it exists. | 1. model_id - ID of model | - -### Testing - -We have a set of unit tests for our extensions in -[tests](tests). To run all the tests, run the following -command: - -```commandline -python -m unittest discover ./tests -``` - -To run an individual test: -```commandline -python -m unittest tests.test_param_sources.VectorsFromDataSetParamSourceTestCase.test_partition_hdf5 -``` diff --git a/benchmarks/osb/__init__.py b/benchmarks/osb/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/benchmarks/osb/extensions/__init__.py b/benchmarks/osb/extensions/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/benchmarks/osb/extensions/data_set.py b/benchmarks/osb/extensions/data_set.py deleted file mode 100644 index 7e8058844..000000000 --- a/benchmarks/osb/extensions/data_set.py +++ /dev/null @@ -1,202 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# -# The OpenSearch Contributors require contributions made to -# this file be licensed under the Apache-2.0 license or a -# compatible open source license. - -import os -import numpy as np -from abc import ABC, ABCMeta, abstractmethod -from enum import Enum -from typing import cast -import h5py -import struct - - -class Context(Enum): - """DataSet context enum. Can be used to add additional context for how a - data-set should be interpreted. - """ - INDEX = 1 - QUERY = 2 - NEIGHBORS = 3 - - -class DataSet(ABC): - """DataSet interface. Used for reading data-sets from files. - - Methods: - read: Read a chunk of data from the data-set - seek: Get to position in the data-set - size: Gets the number of items in the data-set - reset: Resets internal state of data-set to beginning - """ - __metaclass__ = ABCMeta - - BEGINNING = 0 - - @abstractmethod - def read(self, chunk_size: int): - pass - - @abstractmethod - def seek(self, offset: int): - pass - - @abstractmethod - def size(self): - pass - - @abstractmethod - def reset(self): - pass - - -class HDF5DataSet(DataSet): - """ Data-set format corresponding to `ANN Benchmarks - `_ - """ - - FORMAT_NAME = "hdf5" - - def __init__(self, dataset_path: str, context: Context): - file = h5py.File(dataset_path) - self.data = cast(h5py.Dataset, file[self.parse_context(context)]) - self.current = self.BEGINNING - - def read(self, chunk_size: int): - if self.current >= self.size(): - return None - - end_offset = self.current + chunk_size - if end_offset > self.size(): - end_offset = self.size() - - v = cast(np.ndarray, self.data[self.current:end_offset]) - self.current = end_offset - return v - - def seek(self, offset: int): - - if offset < self.BEGINNING: - raise Exception("Offset must be greater than or equal to 0") - - if offset >= self.size(): - raise Exception("Offset must be less than the data set size") - - self.current = offset - - def size(self): - return self.data.len() - - def reset(self): - self.current = self.BEGINNING - - @staticmethod - def parse_context(context: Context) -> str: - if context == Context.NEIGHBORS: - return "neighbors" - - if context == Context.INDEX: - return "train" - - if context == Context.QUERY: - return "test" - - raise Exception("Unsupported context") - - -class BigANNVectorDataSet(DataSet): - """ Data-set format for vector data-sets for `Big ANN Benchmarks - `_ - """ - - DATA_SET_HEADER_LENGTH = 8 - U8BIN_EXTENSION = "u8bin" - FBIN_EXTENSION = "fbin" - FORMAT_NAME = "bigann" - - BYTES_PER_U8INT = 1 - BYTES_PER_FLOAT = 4 - - def __init__(self, dataset_path: str): - self.file = open(dataset_path, 'rb') - self.file.seek(BigANNVectorDataSet.BEGINNING, os.SEEK_END) - num_bytes = self.file.tell() - self.file.seek(BigANNVectorDataSet.BEGINNING) - - if num_bytes < BigANNVectorDataSet.DATA_SET_HEADER_LENGTH: - raise Exception("File is invalid") - - self.num_points = int.from_bytes(self.file.read(4), "little") - self.dimension = int.from_bytes(self.file.read(4), "little") - self.bytes_per_num = self._get_data_size(dataset_path) - - if (num_bytes - BigANNVectorDataSet.DATA_SET_HEADER_LENGTH) != self.num_points * \ - self.dimension * self.bytes_per_num: - raise Exception("File is invalid") - - self.reader = self._value_reader(dataset_path) - self.current = BigANNVectorDataSet.BEGINNING - - def read(self, chunk_size: int): - if self.current >= self.size(): - return None - - end_offset = self.current + chunk_size - if end_offset > self.size(): - end_offset = self.size() - - v = np.asarray([self._read_vector() for _ in - range(end_offset - self.current)]) - self.current = end_offset - return v - - def seek(self, offset: int): - - if offset < self.BEGINNING: - raise Exception("Offset must be greater than or equal to 0") - - if offset >= self.size(): - raise Exception("Offset must be less than the data set size") - - bytes_offset = BigANNVectorDataSet.DATA_SET_HEADER_LENGTH + \ - self.dimension * self.bytes_per_num * offset - self.file.seek(bytes_offset) - self.current = offset - - def _read_vector(self): - return np.asarray([self.reader(self.file) for _ in - range(self.dimension)]) - - def size(self): - return self.num_points - - def reset(self): - self.file.seek(BigANNVectorDataSet.DATA_SET_HEADER_LENGTH) - self.current = BigANNVectorDataSet.BEGINNING - - def __del__(self): - self.file.close() - - @staticmethod - def _get_data_size(file_name): - ext = file_name.split('.')[-1] - if ext == BigANNVectorDataSet.U8BIN_EXTENSION: - return BigANNVectorDataSet.BYTES_PER_U8INT - - if ext == BigANNVectorDataSet.FBIN_EXTENSION: - return BigANNVectorDataSet.BYTES_PER_FLOAT - - raise Exception("Unknown extension") - - @staticmethod - def _value_reader(file_name): - ext = file_name.split('.')[-1] - if ext == BigANNVectorDataSet.U8BIN_EXTENSION: - return lambda file: float(int.from_bytes(file.read(BigANNVectorDataSet.BYTES_PER_U8INT), "little")) - - if ext == BigANNVectorDataSet.FBIN_EXTENSION: - return lambda file: struct.unpack('= self.num_vectors + self.offset: - raise StopIteration - - if self.vector_batch is None or len(self.vector_batch) == 0: - self.vector_batch = self._batch_read(self.data_set) - if self.vector_batch is None: - raise StopIteration - vector = self.vector_batch.pop(0) - self.current += 1 - self.percent_completed = self.current / self.total - - return self._build_query_body(self.index_name, self.field_name, self.k, - vector) - - def _batch_read(self, data_set: DataSet): - return list(data_set.read(self.VECTOR_READ_BATCH_SIZE)) - - def _build_query_body(self, index_name: str, field_name: str, k: int, - vector) -> dict: - """Builds a k-NN query that can be used to execute an approximate nearest - neighbor search against a k-NN plugin index - Args: - index_name: name of index to search - field_name: name of field to search - k: number of results to return - vector: vector used for query - Returns: - A dictionary containing the body used for search, a set of request - parameters to attach to the search and the name of the index. - """ - return { - "index": index_name, - "request-params": { - "_source": { - "exclude": [field_name] - } - }, - "body": { - "size": k, - "query": { - "knn": { - field_name: { - "vector": vector, - "k": k - } - } - } - } - } - - -class BulkVectorsFromDataSetParamSource(VectorsFromDataSetParamSource): - """ Create bulk index requests from a data set of vectors. - - Attributes: - bulk_size: number of vectors per request - retries: number of times to retry the request when it fails - """ - - DEFAULT_RETRIES = 10 - - def __init__(self, workload, params, **kwargs): - super().__init__(params, Context.INDEX) - self.bulk_size: int = parse_int_parameter("bulk_size", params) - self.retries: int = parse_int_parameter("retries", params, - self.DEFAULT_RETRIES) - - def params(self): - """ - Returns: A bulk index parameter with vectors from a data set. - """ - if self.current >= self.num_vectors + self.offset: - raise StopIteration - - def action(doc_id): - return {'index': {'_index': self.index_name, '_id': doc_id}} - - partition = self.data_set.read(self.bulk_size) - body = bulk_transform(partition, self.field_name, action, self.current) - size = len(body) // 2 - self.current += size - self.percent_completed = self.current / self.total - - return { - "body": body, - "retries": self.retries, - "size": size - } diff --git a/benchmarks/osb/extensions/registry.py b/benchmarks/osb/extensions/registry.py deleted file mode 100644 index 5ce17ab6f..000000000 --- a/benchmarks/osb/extensions/registry.py +++ /dev/null @@ -1,13 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# -# The OpenSearch Contributors require contributions made to -# this file be licensed under the Apache-2.0 license or a -# compatible open source license. - -from .param_sources import register as param_sources_register -from .runners import register as runners_register - - -def register(registry): - param_sources_register(registry) - runners_register(registry) diff --git a/benchmarks/osb/extensions/runners.py b/benchmarks/osb/extensions/runners.py deleted file mode 100644 index d048f80b0..000000000 --- a/benchmarks/osb/extensions/runners.py +++ /dev/null @@ -1,121 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# -# The OpenSearch Contributors require contributions made to -# this file be licensed under the Apache-2.0 license or a -# compatible open source license. -from opensearchpy.exceptions import ConnectionTimeout -from .util import parse_int_parameter, parse_string_parameter -import logging -import time - - -def register(registry): - registry.register_runner( - "custom-vector-bulk", BulkVectorsFromDataSetRunner(), async_runner=True - ) - registry.register_runner( - "custom-refresh", CustomRefreshRunner(), async_runner=True - ) - registry.register_runner( - "train-model", TrainModelRunner(), async_runner=True - ) - registry.register_runner( - "delete-model", DeleteModelRunner(), async_runner=True - ) - - -class BulkVectorsFromDataSetRunner: - - async def __call__(self, opensearch, params): - size = parse_int_parameter("size", params) - retries = parse_int_parameter("retries", params, 0) + 1 - - for _ in range(retries): - try: - await opensearch.bulk( - body=params["body"], - timeout='5m' - ) - - return size, "docs" - except ConnectionTimeout: - logging.getLogger(__name__)\ - .warning("Bulk vector ingestion timed out. Retrying") - - raise TimeoutError("Failed to submit bulk request in specified number " - "of retries: {}".format(retries)) - - def __repr__(self, *args, **kwargs): - return "custom-vector-bulk" - - -class CustomRefreshRunner: - - async def __call__(self, opensearch, params): - retries = parse_int_parameter("retries", params, 0) + 1 - - for _ in range(retries): - try: - await opensearch.indices.refresh( - index=parse_string_parameter("index", params) - ) - - return - except ConnectionTimeout: - logging.getLogger(__name__)\ - .warning("Custom refresh timed out. Retrying") - - raise TimeoutError("Failed to refresh the index in specified number " - "of retries: {}".format(retries)) - - def __repr__(self, *args, **kwargs): - return "custom-refresh" - - -class TrainModelRunner: - - async def __call__(self, opensearch, params): - # Train a model and wait for it training to complete - body = params["body"] - timeout = parse_int_parameter("timeout", params) - model_id = parse_string_parameter("model_id", params) - - method = "POST" - model_uri = "/_plugins/_knn/models/{}".format(model_id) - await opensearch.transport.perform_request(method, "{}/_train".format(model_uri), body=body) - - start_time = time.time() - while time.time() < start_time + timeout: - time.sleep(1) - model_response = await opensearch.transport.perform_request("GET", model_uri) - - if 'state' not in model_response.keys(): - continue - - if model_response['state'] == 'created': - #TODO: Return model size as well - return 1, "models_trained" - - if model_response['state'] == 'failed': - raise Exception("Failed to create model: {}".format(model_response)) - - raise Exception('Failed to create model: {} within timeout {} seconds' - .format(model_id, timeout)) - - def __repr__(self, *args, **kwargs): - return "train-model" - - -class DeleteModelRunner: - - async def __call__(self, opensearch, params): - # Delete model provided by model id - method = "DELETE" - model_id = parse_string_parameter("model_id", params) - uri = "/_plugins/_knn/models/{}".format(model_id) - - # Ignore if model doesnt exist - await opensearch.transport.perform_request(method, uri, params={"ignore": [400, 404]}) - - def __repr__(self, *args, **kwargs): - return "delete-model" diff --git a/benchmarks/osb/extensions/util.py b/benchmarks/osb/extensions/util.py deleted file mode 100644 index f7f6aab62..000000000 --- a/benchmarks/osb/extensions/util.py +++ /dev/null @@ -1,71 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# -# The OpenSearch Contributors require contributions made to -# this file be licensed under the Apache-2.0 license or a -# compatible open source license. - -import numpy as np -from typing import List -from typing import Dict -from typing import Any - - -def bulk_transform(partition: np.ndarray, field_name: str, action, - offset: int) -> List[Dict[str, Any]]: - """Partitions and transforms a list of vectors into OpenSearch's bulk - injection format. - Args: - offset: to start counting from - partition: An array of vectors to transform. - field_name: field name for action - action: Bulk API action. - Returns: - An array of transformed vectors in bulk format. - """ - actions = [] - _ = [ - actions.extend([action(i + offset), None]) - for i in range(len(partition)) - ] - actions[1::2] = [{field_name: vec} for vec in partition.tolist()] - return actions - - -def parse_string_parameter(key: str, params: dict, default: str = None) -> str: - if key not in params: - if default is not None: - return default - raise ConfigurationError( - "Value cannot be None for param {}".format(key) - ) - - if type(params[key]) is str: - return params[key] - - raise ConfigurationError("Value must be a string for param {}".format(key)) - - -def parse_int_parameter(key: str, params: dict, default: int = None) -> int: - if key not in params: - if default: - return default - raise ConfigurationError( - "Value cannot be None for param {}".format(key) - ) - - if type(params[key]) is int: - return params[key] - - raise ConfigurationError("Value must be a int for param {}".format(key)) - - -class ConfigurationError(Exception): - """Exception raised for errors configuration. - - Attributes: - message -- explanation of the error - """ - - def __init__(self, message: str): - self.message = f'{message}' - super().__init__(self.message) diff --git a/benchmarks/osb/indices/faiss-index.json b/benchmarks/osb/indices/faiss-index.json deleted file mode 100644 index 2db4d34d4..000000000 --- a/benchmarks/osb/indices/faiss-index.json +++ /dev/null @@ -1,27 +0,0 @@ -{ - "settings": { - "index": { - "knn": true, - "number_of_shards": {{ target_index_primary_shards }}, - "number_of_replicas": {{ target_index_replica_shards }} - } - }, - "mappings": { - "properties": { - "target_field": { - "type": "knn_vector", - "dimension": {{ target_index_dimension }}, - "method": { - "name": "hnsw", - "space_type": "{{ target_index_space_type }}", - "engine": "faiss", - "parameters": { - "ef_search": {{ hnsw_ef_search }}, - "ef_construction": {{ hnsw_ef_construction }}, - "m": {{ hnsw_m }} - } - } - } - } - } -} diff --git a/benchmarks/osb/indices/lucene-index.json b/benchmarks/osb/indices/lucene-index.json deleted file mode 100644 index 0a4ed868a..000000000 --- a/benchmarks/osb/indices/lucene-index.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "settings": { - "index": { - "knn": true, - "number_of_shards": {{ target_index_primary_shards }}, - "number_of_replicas": {{ target_index_replica_shards }} - } - }, - "mappings": { - "properties": { - "target_field": { - "type": "knn_vector", - "dimension": {{ target_index_dimension }}, - "method": { - "name": "hnsw", - "space_type": "{{ target_index_space_type }}", - "engine": "lucene", - "parameters": { - "ef_construction": {{ hnsw_ef_construction }}, - "m": {{ hnsw_m }} - } - } - } - } - } -} diff --git a/benchmarks/osb/indices/model-index.json b/benchmarks/osb/indices/model-index.json deleted file mode 100644 index 0e92c8903..000000000 --- a/benchmarks/osb/indices/model-index.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "settings": { - "index": { - "knn": true, - "number_of_shards": {{ target_index_primary_shards | default(1) }}, - "number_of_replicas": {{ target_index_replica_shards | default(0) }} - } - }, - "mappings": { - "properties": { - "{{ target_field_name }}": { - "type": "knn_vector", - "model_id": "{{ train_model_id }}" - } - } - } -} diff --git a/benchmarks/osb/indices/nmslib-index.json b/benchmarks/osb/indices/nmslib-index.json deleted file mode 100644 index 4ceb57977..000000000 --- a/benchmarks/osb/indices/nmslib-index.json +++ /dev/null @@ -1,27 +0,0 @@ -{ - "settings": { - "index": { - "knn": true, - "knn.algo_param.ef_search": {{ hnsw_ef_search }}, - "number_of_shards": {{ target_index_primary_shards }}, - "number_of_replicas": {{ target_index_replica_shards }} - } - }, - "mappings": { - "properties": { - "target_field": { - "type": "knn_vector", - "dimension": {{ target_index_dimension }}, - "method": { - "name": "hnsw", - "space_type": "{{ target_index_space_type }}", - "engine": "nmslib", - "parameters": { - "ef_construction": {{ hnsw_ef_construction }}, - "m": {{ hnsw_m }} - } - } - } - } - } -} diff --git a/benchmarks/osb/indices/train-index.json b/benchmarks/osb/indices/train-index.json deleted file mode 100644 index 82af8215e..000000000 --- a/benchmarks/osb/indices/train-index.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "settings": { - "index": { - "number_of_shards": {{ train_index_primary_shards }}, - "number_of_replicas": {{ train_index_replica_shards }} - } - }, - "mappings": { - "properties": { - "{{ train_field_name }}": { - "type": "knn_vector", - "dimension": {{ target_index_dimension }} - } - } - } -} diff --git a/benchmarks/osb/operations/default.json b/benchmarks/osb/operations/default.json deleted file mode 100644 index ee33166f0..000000000 --- a/benchmarks/osb/operations/default.json +++ /dev/null @@ -1,53 +0,0 @@ -[ - { - "name": "ivfpq-train-model", - "operation-type": "train-model", - "model_id": "{{ train_model_id }}", - "timeout": {{ train_timeout }}, - "body": { - "training_index": "{{ train_index_name }}", - "training_field": "{{ train_field_name }}", - "dimension": {{ target_index_dimension }}, - "search_size": {{ train_search_size }}, - "max_training_vector_count": {{ train_index_num_vectors }}, - "method": { - "name":"ivf", - "engine":"faiss", - "space_type": "{{ target_index_space_type }}", - "parameters":{ - "nlist": {{ ivf_nlists }}, - "nprobes": {{ ivf_nprobes }}, - "encoder":{ - "name":"pq", - "parameters":{ - "code_size": {{ pq_code_size }}, - "m": {{ pq_m }} - } - } - } - } - } - }, - { - "name": "ivf-train-model", - "operation-type": "train-model", - "model_id": "{{ train_model_id }}", - "timeout": {{ train_timeout | default(1000) }}, - "body": { - "training_index": "{{ train_index_name }}", - "training_field": "{{ train_field_name }}", - "search_size": {{ train_search_size }}, - "dimension": {{ target_index_dimension }}, - "max_training_vector_count": {{ train_index_num_vectors }}, - "method": { - "name":"ivf", - "engine":"faiss", - "space_type": "{{ target_index_space_type }}", - "parameters":{ - "nlist": {{ ivf_nlists }}, - "nprobes": {{ ivf_nprobes }} - } - } - } - } -] diff --git a/benchmarks/osb/params/no-train-params.json b/benchmarks/osb/params/no-train-params.json deleted file mode 100644 index 58e4197fd..000000000 --- a/benchmarks/osb/params/no-train-params.json +++ /dev/null @@ -1,40 +0,0 @@ -{ - "target_index_name": "target_index", - "target_field_name": "target_field", - "target_index_body": "indices/nmslib-index.json", - "target_index_primary_shards": 3, - "target_index_replica_shards": 1, - "target_index_dimension": 128, - "target_index_space_type": "l2", - "target_index_bulk_size": 200, - "target_index_bulk_index_data_set_format": "hdf5", - "target_index_bulk_index_data_set_path": "", - "target_index_bulk_index_clients": 10, - "target_index_max_num_segments": 10, - "target_index_force_merge_timeout": 45.0, - "hnsw_ef_search": 512, - "hnsw_ef_construction": 512, - "hnsw_m": 16, - - "query_k": 10, - "query_clients": 10, - "query_data_set_format": "hdf5", - "query_data_set_path": "", - - "ivf_nlists": 1, - "ivf_nprobes": 1, - "pq_code_size": 1, - "pq_m": 1, - "train_model_method": "", - "train_model_id": "", - "train_index_name": "", - "train_field_name": "", - "train_index_body": "", - "train_search_size": 1, - "train_timeout": 1, - "train_index_bulk_size": 1, - "train_index_data_set_format": "", - "train_index_data_set_path": "", - "train_index_num_vectors": 1, - "train_index_bulk_index_clients": 1 -} diff --git a/benchmarks/osb/params/train-params.json b/benchmarks/osb/params/train-params.json deleted file mode 100644 index f55ed4333..000000000 --- a/benchmarks/osb/params/train-params.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "target_index_name": "target_index", - "target_field_name": "target_field", - "target_index_body": "indices/model-index.json", - "target_index_primary_shards": 3, - "target_index_replica_shards": 1, - "target_index_dimension": 128, - "target_index_space_type": "l2", - "target_index_bulk_size": 200, - "target_index_bulk_index_data_set_format": "hdf5", - "target_index_bulk_index_data_set_path": "", - "target_index_bulk_index_clients": 10, - "target_index_max_num_segments": 10, - "target_index_force_merge_timeout": 45.0, - "ivf_nlists": 10, - "ivf_nprobes": 1, - "pq_code_size": 8, - "pq_m": 8, - "train_model_method": "ivfpq", - "train_model_id": "test-model", - "train_index_name": "train_index", - "train_field_name": "train_field", - "train_index_body": "indices/train-index.json", - "train_search_size": 500, - "train_timeout": 5000, - "train_index_primary_shards": 1, - "train_index_replica_shards": 0, - "train_index_bulk_size": 200, - "train_index_data_set_format": "hdf5", - "train_index_data_set_path": "", - "train_index_num_vectors": 1000000, - "train_index_bulk_index_clients": 10, - - "query_k": 10, - "query_clients": 10, - "query_data_set_format": "hdf5", - "query_data_set_path": "" -} diff --git a/benchmarks/osb/procedures/no-train-test.json b/benchmarks/osb/procedures/no-train-test.json deleted file mode 100644 index 01985b914..000000000 --- a/benchmarks/osb/procedures/no-train-test.json +++ /dev/null @@ -1,73 +0,0 @@ -{% import "benchmark.helpers" as benchmark with context %} -{ - "name": "no-train-test", - "default": true, - "schedule": [ - { - "operation": { - "name": "delete-target-index", - "operation-type": "delete-index", - "only-if-exists": true, - "index": "{{ target_index_name }}" - } - }, - { - "operation": { - "name": "create-target-index", - "operation-type": "create-index", - "index": "{{ target_index_name }}" - } - }, - { - "name": "wait-for-cluster-to-be-green", - "operation": "cluster-health", - "request-params": { - "wait_for_status": "green" - } - }, - { - "operation": { - "name": "custom-vector-bulk", - "operation-type": "custom-vector-bulk", - "param-source": "bulk-from-data-set", - "index": "{{ target_index_name }}", - "field": "{{ target_field_name }}", - "bulk_size": {{ target_index_bulk_size }}, - "data_set_format": "{{ target_index_bulk_index_data_set_format }}", - "data_set_path": "{{ target_index_bulk_index_data_set_path }}" - }, - "clients": {{ target_index_bulk_index_clients }} - }, - { - "operation": { - "name": "refresh-target-index", - "operation-type": "custom-refresh", - "index": "{{ target_index_name }}", - "retries": 100 - } - }, - { - "operation": { - "name": "force-merge", - "operation-type": "force-merge", - "request-timeout": {{ target_index_force_merge_timeout }}, - "index": "{{ target_index_name }}", - "mode": "polling", - "max-num-segments": {{ target_index_max_num_segments }} - } - }, - { - "operation": { - "name": "knn-query-from-data-set", - "operation-type": "search", - "index": "{{ target_index_name }}", - "param-source": "knn-query-from-data-set", - "k": {{ query_k }}, - "field": "{{ target_field_name }}", - "data_set_format": "{{ query_data_set_format }}", - "data_set_path": "{{ query_data_set_path }}" - }, - "clients": {{ query_clients }} - } - ] -} diff --git a/benchmarks/osb/procedures/train-test.json b/benchmarks/osb/procedures/train-test.json deleted file mode 100644 index ca26db0b0..000000000 --- a/benchmarks/osb/procedures/train-test.json +++ /dev/null @@ -1,127 +0,0 @@ -{% import "benchmark.helpers" as benchmark with context %} -{ - "name": "train-test", - "default": false, - "schedule": [ - { - "operation": { - "name": "delete-target-index", - "operation-type": "delete-index", - "only-if-exists": true, - "index": "{{ target_index_name }}" - } - }, - { - "operation": { - "name": "delete-train-index", - "operation-type": "delete-index", - "only-if-exists": true, - "index": "{{ train_index_name }}" - } - }, - { - "operation": { - "operation-type": "delete-model", - "name": "delete-model", - "model_id": "{{ train_model_id }}" - } - }, - { - "operation": { - "name": "create-train-index", - "operation-type": "create-index", - "index": "{{ train_index_name }}" - } - }, - { - "name": "wait-for-train-index-to-be-green", - "operation": "cluster-health", - "request-params": { - "wait_for_status": "green" - } - }, - { - "operation": { - "name": "train-vector-bulk", - "operation-type": "custom-vector-bulk", - "param-source": "bulk-from-data-set", - "index": "{{ train_index_name }}", - "field": "{{ train_field_name }}", - "bulk_size": {{ train_index_bulk_size }}, - "data_set_format": "{{ train_index_data_set_format }}", - "data_set_path": "{{ train_index_data_set_path }}", - "num_vectors": {{ train_index_num_vectors }} - }, - "clients": {{ train_index_bulk_index_clients }} - }, - { - "operation": { - "name": "refresh-train-index", - "operation-type": "custom-refresh", - "index": "{{ train_index_name }}", - "retries": 100 - } - }, - { - "operation": "{{ train_model_method }}-train-model" - }, - { - "operation": { - "name": "create-target-index", - "operation-type": "create-index", - "index": "{{ target_index_name }}" - } - }, - { - "name": "wait-for-target-index-to-be-green", - "operation": "cluster-health", - "request-params": { - "wait_for_status": "green" - } - }, - { - "operation": { - "name": "custom-vector-bulk", - "operation-type": "custom-vector-bulk", - "param-source": "bulk-from-data-set", - "index": "{{ target_index_name }}", - "field": "{{ target_field_name }}", - "bulk_size": {{ target_index_bulk_size }}, - "data_set_format": "{{ target_index_bulk_index_data_set_format }}", - "data_set_path": "{{ target_index_bulk_index_data_set_path }}" - }, - "clients": {{ target_index_bulk_index_clients }} - }, - { - "operation": { - "name": "refresh-target-index", - "operation-type": "custom-refresh", - "index": "{{ target_index_name }}", - "retries": 100 - } - }, - { - "operation": { - "name": "force-merge", - "operation-type": "force-merge", - "request-timeout": {{ target_index_force_merge_timeout }}, - "index": "{{ target_index_name }}", - "mode": "polling", - "max-num-segments": {{ target_index_max_num_segments }} - } - }, - { - "operation": { - "name": "knn-query-from-data-set", - "operation-type": "search", - "index": "{{ target_index_name }}", - "param-source": "knn-query-from-data-set", - "k": {{ query_k }}, - "field": "{{ target_field_name }}", - "data_set_format": "{{ query_data_set_format }}", - "data_set_path": "{{ query_data_set_path }}" - }, - "clients": {{ query_clients }} - } - ] -} diff --git a/benchmarks/osb/requirements.in b/benchmarks/osb/requirements.in deleted file mode 100644 index a9e12b5d3..000000000 --- a/benchmarks/osb/requirements.in +++ /dev/null @@ -1,4 +0,0 @@ -opensearch-py -numpy -h5py -opensearch-benchmark diff --git a/benchmarks/osb/requirements.txt b/benchmarks/osb/requirements.txt deleted file mode 100644 index a220ee44f..000000000 --- a/benchmarks/osb/requirements.txt +++ /dev/null @@ -1,96 +0,0 @@ -# -# This file is autogenerated by pip-compile with python 3.8 -# To update, run: -# -# pip-compile -# -aiohttp==3.9.4 - # via opensearch-py -aiosignal==1.2.0 - # via aiohttp -async-timeout==4.0.2 - # via aiohttp -attrs==21.4.0 - # via - # aiohttp - # jsonschema -cachetools==4.2.4 - # via google-auth -certifi==2023.7.22 - # via - # opensearch-benchmark - # opensearch-py -frozenlist==1.3.0 - # via - # aiohttp - # aiosignal -google-auth==1.22.1 - # via opensearch-benchmark -google-crc32c==1.3.0 - # via google-resumable-media -google-resumable-media==1.1.0 - # via opensearch-benchmark -h5py==3.6.0 - # via -r requirements.in -idna==3.7 - # via yarl -ijson==2.6.1 - # via opensearch-benchmark -importlib-metadata==4.11.3 - # via jsonschema -jinja2==3.1.3 - # via opensearch-benchmark -jsonschema==3.1.1 - # via opensearch-benchmark -markupsafe==2.0.1 - # via - # jinja2 - # opensearch-benchmark -multidict==6.0.2 - # via - # aiohttp - # yarl -numpy==1.24.2 - # via - # -r requirements.in - # h5py -opensearch-benchmark==0.0.2 - # via -r requirements.in -opensearch-py[async]==1.0.0 - # via - # -r requirements.in - # opensearch-benchmark -psutil==5.8.0 - # via opensearch-benchmark -py-cpuinfo==7.0.0 - # via opensearch-benchmark -pyasn1==0.4.8 - # via - # pyasn1-modules - # rsa -pyasn1-modules==0.2.8 - # via google-auth -pyrsistent==0.18.1 - # via jsonschema -rsa==4.8 - # via google-auth -six==1.16.0 - # via - # google-auth - # google-resumable-media - # jsonschema -tabulate==0.8.7 - # via opensearch-benchmark -thespian==3.10.1 - # via opensearch-benchmark -urllib3==1.26.18 - # via opensearch-py -yappi==1.2.3 - # via opensearch-benchmark -yarl==1.7.2 - # via aiohttp -zipp==3.7.0 - # via importlib-metadata - -# The following packages are considered to be unsafe in a requirements file: -# setuptools diff --git a/benchmarks/osb/tests/__init__.py b/benchmarks/osb/tests/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/benchmarks/osb/tests/data_set_helper.py b/benchmarks/osb/tests/data_set_helper.py deleted file mode 100644 index 2b144da49..000000000 --- a/benchmarks/osb/tests/data_set_helper.py +++ /dev/null @@ -1,197 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# -# The OpenSearch Contributors require contributions made to -# this file be licensed under the Apache-2.0 license or a -# compatible open source license. - -from abc import ABC, abstractmethod - -import h5py -import numpy as np - -from osb.extensions.data_set import Context, HDF5DataSet, BigANNVectorDataSet - -""" Module containing utility classes and functions for working with data sets. - -Included are utilities that can be used to build data sets and write them to -paths. -""" - - -class DataSetBuildContext: - """ Data class capturing information needed to build a particular data set - - Attributes: - data_set_context: Indicator of what the data set is used for, - vectors: A 2D array containing vectors that are used to build data set. - path: string representing path where data set should be serialized to. - """ - def __init__(self, data_set_context: Context, vectors: np.ndarray, path: str): - self.data_set_context: Context = data_set_context - self.vectors: np.ndarray = vectors #TODO: Validate shape - self.path: str = path - - def get_num_vectors(self) -> int: - return self.vectors.shape[0] - - def get_dimension(self) -> int: - return self.vectors.shape[1] - - def get_type(self) -> np.dtype: - return self.vectors.dtype - - -class DataSetBuilder(ABC): - """ Abstract builder used to create a build a collection of data sets - - Attributes: - data_set_build_contexts: list of data set build contexts that builder - will build. - """ - def __init__(self): - self.data_set_build_contexts = list() - - def add_data_set_build_context(self, data_set_build_context: DataSetBuildContext): - """ Adds a data set build context to list of contexts to be built. - - Args: - data_set_build_context: DataSetBuildContext to be added to list - - Returns: Updated DataSetBuilder - - """ - self._validate_data_set_context(data_set_build_context) - self.data_set_build_contexts.append(data_set_build_context) - return self - - def build(self): - """ Builds and serializes all data sets build contexts - - Returns: - - """ - [self._build_data_set(data_set_build_context) for data_set_build_context - in self.data_set_build_contexts] - - @abstractmethod - def _build_data_set(self, context: DataSetBuildContext): - """ Builds an individual data set - - Args: - context: DataSetBuildContext of data set to be built - - Returns: - - """ - pass - - @abstractmethod - def _validate_data_set_context(self, context: DataSetBuildContext): - """ Validates that data set context can be added to this builder - - Args: - context: DataSetBuildContext to be validated - - Returns: - - """ - pass - - -class HDF5Builder(DataSetBuilder): - - def __init__(self): - super(HDF5Builder, self).__init__() - self.data_set_meta_data = dict() - - def _validate_data_set_context(self, context: DataSetBuildContext): - if context.path not in self.data_set_meta_data.keys(): - self.data_set_meta_data[context.path] = { - context.data_set_context: context - } - return - - if context.data_set_context in \ - self.data_set_meta_data[context.path].keys(): - raise IllegalDataSetBuildContext("Path and context for data set " - "are already present in builder.") - - self.data_set_meta_data[context.path][context.data_set_context] = \ - context - - @staticmethod - def _validate_extension(context: DataSetBuildContext): - ext = context.path.split('.')[-1] - - if ext != HDF5DataSet.FORMAT_NAME: - raise IllegalDataSetBuildContext("Invalid file extension") - - def _build_data_set(self, context: DataSetBuildContext): - # For HDF5, because multiple data sets can be grouped in the same file, - # we will build data sets in memory and not write to disk until - # _flush_data_sets_to_disk is called - with h5py.File(context.path, 'a') as hf: - hf.create_dataset( - HDF5DataSet.parse_context(context.data_set_context), - data=context.vectors - ) - - -class BigANNBuilder(DataSetBuilder): - - def _validate_data_set_context(self, context: DataSetBuildContext): - self._validate_extension(context) - - # prevent the duplication of paths for data sets - data_set_paths = [c.path for c in self.data_set_build_contexts] - if any(data_set_paths.count(x) > 1 for x in data_set_paths): - raise IllegalDataSetBuildContext("Build context paths have to be " - "unique.") - - @staticmethod - def _validate_extension(context: DataSetBuildContext): - ext = context.path.split('.')[-1] - - if ext != BigANNVectorDataSet.U8BIN_EXTENSION and ext != \ - BigANNVectorDataSet.FBIN_EXTENSION: - raise IllegalDataSetBuildContext("Invalid file extension") - - if ext == BigANNVectorDataSet.U8BIN_EXTENSION and context.get_type() != \ - np.u8int: - raise IllegalDataSetBuildContext("Invalid data type for {} ext." - .format(BigANNVectorDataSet - .U8BIN_EXTENSION)) - - if ext == BigANNVectorDataSet.FBIN_EXTENSION and context.get_type() != \ - np.float32: - print(context.get_type()) - raise IllegalDataSetBuildContext("Invalid data type for {} ext." - .format(BigANNVectorDataSet - .FBIN_EXTENSION)) - - def _build_data_set(self, context: DataSetBuildContext): - num_vectors = context.get_num_vectors() - dimension = context.get_dimension() - - with open(context.path, 'wb') as f: - f.write(int.to_bytes(num_vectors, 4, "little")) - f.write(int.to_bytes(dimension, 4, "little")) - context.vectors.tofile(f) - - -def create_random_2d_array(num_vectors: int, dimension: int) -> np.ndarray: - rng = np.random.default_rng() - return rng.random(size=(num_vectors, dimension), dtype=np.float32) - - -class IllegalDataSetBuildContext(Exception): - """Exception raised when passed in DataSetBuildContext is illegal - - Attributes: - message -- explanation of the error - """ - - def __init__(self, message: str): - self.message = f'{message}' - super().__init__(self.message) - diff --git a/benchmarks/osb/tests/test_param_sources.py b/benchmarks/osb/tests/test_param_sources.py deleted file mode 100644 index cda730cee..000000000 --- a/benchmarks/osb/tests/test_param_sources.py +++ /dev/null @@ -1,353 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# -# The OpenSearch Contributors require contributions made to -# this file be licensed under the Apache-2.0 license or a -# compatible open source license. - -import os -import random -import shutil -import string -import sys -import tempfile -import unittest - -# Add parent directory to path -import numpy as np - -sys.path.append(os.path.abspath(os.path.join(os.getcwd(), os.pardir))) - -from osb.tests.data_set_helper import HDF5Builder, create_random_2d_array, \ - DataSetBuildContext, BigANNBuilder -from osb.extensions.data_set import Context, HDF5DataSet -from osb.extensions.param_sources import VectorsFromDataSetParamSource, \ - QueryVectorsFromDataSetParamSource, BulkVectorsFromDataSetParamSource -from osb.extensions.util import ConfigurationError - -DEFAULT_INDEX_NAME = "test-index" -DEFAULT_FIELD_NAME = "test-field" -DEFAULT_CONTEXT = Context.INDEX -DEFAULT_TYPE = HDF5DataSet.FORMAT_NAME -DEFAULT_NUM_VECTORS = 10 -DEFAULT_DIMENSION = 10 -DEFAULT_RANDOM_STRING_LENGTH = 8 - - -class VectorsFromDataSetParamSourceTestCase(unittest.TestCase): - - def setUp(self) -> None: - self.data_set_dir = tempfile.mkdtemp() - - # Create a data set we know to be valid for convenience - self.valid_data_set_path = _create_data_set( - DEFAULT_NUM_VECTORS, - DEFAULT_DIMENSION, - DEFAULT_TYPE, - DEFAULT_CONTEXT, - self.data_set_dir - ) - - def tearDown(self): - shutil.rmtree(self.data_set_dir) - - def test_missing_params(self): - empty_params = dict() - self.assertRaises( - ConfigurationError, - lambda: VectorsFromDataSetParamSourceTestCase. - TestVectorsFromDataSetParamSource(empty_params, DEFAULT_CONTEXT) - ) - - def test_invalid_data_set_format(self): - invalid_data_set_format = "invalid-data-set-format" - - test_param_source_params = { - "index": DEFAULT_INDEX_NAME, - "field": DEFAULT_FIELD_NAME, - "data_set_format": invalid_data_set_format, - "data_set_path": self.valid_data_set_path, - } - self.assertRaises( - ConfigurationError, - lambda: self.TestVectorsFromDataSetParamSource( - test_param_source_params, - DEFAULT_CONTEXT - ) - ) - - def test_invalid_data_set_path(self): - invalid_data_set_path = "invalid-data-set-path" - test_param_source_params = { - "index": DEFAULT_INDEX_NAME, - "field": DEFAULT_FIELD_NAME, - "data_set_format": HDF5DataSet.FORMAT_NAME, - "data_set_path": invalid_data_set_path, - } - self.assertRaises( - FileNotFoundError, - lambda: self.TestVectorsFromDataSetParamSource( - test_param_source_params, - DEFAULT_CONTEXT - ) - ) - - def test_partition_hdf5(self): - num_vectors = 100 - - hdf5_data_set_path = _create_data_set( - num_vectors, - DEFAULT_DIMENSION, - HDF5DataSet.FORMAT_NAME, - DEFAULT_CONTEXT, - self.data_set_dir - ) - - test_param_source_params = { - "index": DEFAULT_INDEX_NAME, - "field": DEFAULT_FIELD_NAME, - "data_set_format": HDF5DataSet.FORMAT_NAME, - "data_set_path": hdf5_data_set_path, - } - test_param_source = self.TestVectorsFromDataSetParamSource( - test_param_source_params, - DEFAULT_CONTEXT - ) - - num_partitions = 10 - vecs_per_partition = test_param_source.num_vectors // num_partitions - - self._test_partition( - test_param_source, - num_partitions, - vecs_per_partition - ) - - def test_partition_bigann(self): - num_vectors = 100 - float_extension = "fbin" - - bigann_data_set_path = _create_data_set( - num_vectors, - DEFAULT_DIMENSION, - float_extension, - DEFAULT_CONTEXT, - self.data_set_dir - ) - - test_param_source_params = { - "index": DEFAULT_INDEX_NAME, - "field": DEFAULT_FIELD_NAME, - "data_set_format": "bigann", - "data_set_path": bigann_data_set_path, - } - test_param_source = self.TestVectorsFromDataSetParamSource( - test_param_source_params, - DEFAULT_CONTEXT - ) - - num_partitions = 10 - vecs_per_partition = test_param_source.num_vectors // num_partitions - - self._test_partition( - test_param_source, - num_partitions, - vecs_per_partition - ) - - def _test_partition( - self, - test_param_source: VectorsFromDataSetParamSource, - num_partitions: int, - vec_per_partition: int - ): - for i in range(num_partitions): - test_param_source_i = test_param_source.partition(i, num_partitions) - self.assertEqual(test_param_source_i.num_vectors, vec_per_partition) - self.assertEqual(test_param_source_i.offset, i * vec_per_partition) - - class TestVectorsFromDataSetParamSource(VectorsFromDataSetParamSource): - """ - Empty implementation of ABC VectorsFromDataSetParamSource so that we can - test the concrete methods. - """ - - def params(self): - pass - - -class QueryVectorsFromDataSetParamSourceTestCase(unittest.TestCase): - - def setUp(self) -> None: - self.data_set_dir = tempfile.mkdtemp() - - def tearDown(self): - shutil.rmtree(self.data_set_dir) - - def test_params(self): - # Create a data set - k = 12 - data_set_path = _create_data_set( - DEFAULT_NUM_VECTORS, - DEFAULT_DIMENSION, - DEFAULT_TYPE, - Context.QUERY, - self.data_set_dir - ) - - # Create a QueryVectorsFromDataSetParamSource with relevant params - test_param_source_params = { - "index": DEFAULT_INDEX_NAME, - "field": DEFAULT_FIELD_NAME, - "data_set_format": DEFAULT_TYPE, - "data_set_path": data_set_path, - "k": k, - } - query_param_source = QueryVectorsFromDataSetParamSource( - None, test_param_source_params - ) - - # Check each - for i in range(DEFAULT_NUM_VECTORS): - self._check_params( - query_param_source.params(), - DEFAULT_INDEX_NAME, - DEFAULT_FIELD_NAME, - DEFAULT_DIMENSION, - k - ) - - # Assert last call creates stop iteration - self.assertRaises( - StopIteration, - lambda: query_param_source.params() - ) - - def _check_params( - self, - params: dict, - expected_index: str, - expected_field: str, - expected_dimension: int, - expected_k: int - ): - index_name = params.get("index") - self.assertEqual(expected_index, index_name) - body = params.get("body") - self.assertIsInstance(body, dict) - query = body.get("query") - self.assertIsInstance(query, dict) - query_knn = query.get("knn") - self.assertIsInstance(query_knn, dict) - field = query_knn.get(expected_field) - self.assertIsInstance(field, dict) - vector = field.get("vector") - self.assertIsInstance(vector, np.ndarray) - self.assertEqual(len(list(vector)), expected_dimension) - k = field.get("k") - self.assertEqual(k, expected_k) - - -class BulkVectorsFromDataSetParamSourceTestCase(unittest.TestCase): - - def setUp(self) -> None: - self.data_set_dir = tempfile.mkdtemp() - - def tearDown(self): - shutil.rmtree(self.data_set_dir) - - def test_params(self): - num_vectors = 49 - bulk_size = 10 - data_set_path = _create_data_set( - num_vectors, - DEFAULT_DIMENSION, - DEFAULT_TYPE, - Context.INDEX, - self.data_set_dir - ) - - test_param_source_params = { - "index": DEFAULT_INDEX_NAME, - "field": DEFAULT_FIELD_NAME, - "data_set_format": DEFAULT_TYPE, - "data_set_path": data_set_path, - "bulk_size": bulk_size - } - bulk_param_source = BulkVectorsFromDataSetParamSource( - None, test_param_source_params - ) - - # Check each payload returned - vectors_consumed = 0 - while vectors_consumed < num_vectors: - expected_num_vectors = min(num_vectors - vectors_consumed, bulk_size) - self._check_params( - bulk_param_source.params(), - DEFAULT_INDEX_NAME, - DEFAULT_FIELD_NAME, - DEFAULT_DIMENSION, - expected_num_vectors - ) - vectors_consumed += expected_num_vectors - - # Assert last call creates stop iteration - self.assertRaises( - StopIteration, - lambda: bulk_param_source.params() - ) - - def _check_params( - self, - params: dict, - expected_index: str, - expected_field: str, - expected_dimension: int, - expected_num_vectors_in_payload: int - ): - size = params.get("size") - self.assertEqual(size, expected_num_vectors_in_payload) - body = params.get("body") - self.assertIsInstance(body, list) - self.assertEqual(len(body) // 2, expected_num_vectors_in_payload) - - # Bulk payload has 2 parts: first one is the header and the second one - # is the body. The header will have the index name and the body will - # have the vector - for header, req_body in zip(*[iter(body)] * 2): - index = header.get("index") - self.assertIsInstance(index, dict) - index_name = index.get("_index") - self.assertEqual(index_name, expected_index) - - vector = req_body.get(expected_field) - self.assertIsInstance(vector, list) - self.assertEqual(len(vector), expected_dimension) - - -def _create_data_set( - num_vectors: int, - dimension: int, - extension: str, - data_set_context: Context, - data_set_dir -) -> str: - - file_name_base = ''.join(random.choice(string.ascii_letters) for _ in - range(DEFAULT_RANDOM_STRING_LENGTH)) - data_set_file_name = "{}.{}".format(file_name_base, extension) - data_set_path = os.path.join(data_set_dir, data_set_file_name) - context = DataSetBuildContext( - data_set_context, - create_random_2d_array(num_vectors, dimension), - data_set_path) - - if extension == HDF5DataSet.FORMAT_NAME: - HDF5Builder().add_data_set_build_context(context).build() - else: - BigANNBuilder().add_data_set_build_context(context).build() - - return data_set_path - - -if __name__ == '__main__': - unittest.main() diff --git a/benchmarks/osb/workload.json b/benchmarks/osb/workload.json deleted file mode 100644 index bd0d84195..000000000 --- a/benchmarks/osb/workload.json +++ /dev/null @@ -1,17 +0,0 @@ -{% import "benchmark.helpers" as benchmark with context %} -{ - "version": 2, - "description": "k-NN Plugin train workload", - "indices": [ - { - "name": "{{ target_index_name }}", - "body": "{{ target_index_body }}" - }, - { - "name": "{{ train_index_name }}", - "body": "{{ train_index_body }}" - } - ], - "operations": {{ benchmark.collect(parts="operations/*.json") }}, - "test_procedures": [{{ benchmark.collect(parts="procedures/*.json") }}] -} diff --git a/benchmarks/osb/workload.py b/benchmarks/osb/workload.py deleted file mode 100644 index 32e6ad02c..000000000 --- a/benchmarks/osb/workload.py +++ /dev/null @@ -1,18 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# -# The OpenSearch Contributors require contributions made to -# this file be licensed under the Apache-2.0 license or a -# compatible open source license. - -# This code needs to be included at the top of every workload.py file. -# OpenSearch Benchmarks is not able to find other helper files unless the path -# is updated. -import os -import sys -sys.path.append(os.path.abspath(os.getcwd())) - -from extensions.registry import register as custom_register - - -def register(registry): - custom_register(registry) diff --git a/benchmarks/perf-tool/.pylintrc b/benchmarks/perf-tool/.pylintrc deleted file mode 100644 index 15bf4ccc3..000000000 --- a/benchmarks/perf-tool/.pylintrc +++ /dev/null @@ -1,443 +0,0 @@ -# This Pylint rcfile contains a best-effort configuration to uphold the -# best-practices and style described in the Google Python style guide: -# https://google.github.io/styleguide/pyguide.html -# -# Its canonical open-source location is: -# https://google.github.io/styleguide/pylintrc - -[MASTER] - -fail-under=9.0 - -# Files or directories to be skipped. They should be base names, not paths. -ignore=third_party - -# Files or directories matching the regex patterns are skipped. The regex -# matches against base names, not paths. -ignore-patterns= - -# Pickle collected data for later comparisons. -persistent=no - -# List of plugins (as comma separated values of python modules names) to load, -# usually to register additional checkers. -load-plugins= - -# Use multiple processes to speed up Pylint. -jobs=4 - -# Allow loading of arbitrary C extensions. Extensions are imported into the -# active Python interpreter and may run arbitrary code. -unsafe-load-any-extension=no - - -[MESSAGES CONTROL] - -# Only show warnings with the listed confidence levels. Leave empty to show -# all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED -confidence= - -# Enable the message, report, category or checker with the given id(s). You can -# either give multiple identifier separated by comma (,) or put this option -# multiple time (only on the command line, not in the configuration file where -# it should appear only once). See also the "--disable" option for examples. -#enable= - -# Disable the message, report, category or checker with the given id(s). You -# can either give multiple identifiers separated by comma (,) or put this -# option multiple times (only on the command line, not in the configuration -# file where it should appear only once).You can also use "--disable=all" to -# disable everything first and then reenable specific checks. For example, if -# you want to run only the similarities checker, you can use "--disable=all -# --enable=similarities". If you want to run only the classes checker, but have -# no Warning level messages displayed, use"--disable=all --enable=classes -# --disable=W" -disable=abstract-method, - apply-builtin, - arguments-differ, - attribute-defined-outside-init, - backtick, - bad-option-value, - basestring-builtin, - buffer-builtin, - c-extension-no-member, - consider-using-enumerate, - cmp-builtin, - cmp-method, - coerce-builtin, - coerce-method, - delslice-method, - div-method, - duplicate-code, - eq-without-hash, - execfile-builtin, - file-builtin, - filter-builtin-not-iterating, - fixme, - getslice-method, - global-statement, - hex-method, - idiv-method, - implicit-str-concat-in-sequence, - import-error, - import-self, - import-star-module-level, - inconsistent-return-statements, - input-builtin, - intern-builtin, - invalid-str-codec, - locally-disabled, - long-builtin, - long-suffix, - map-builtin-not-iterating, - misplaced-comparison-constant, - missing-function-docstring, - metaclass-assignment, - next-method-called, - next-method-defined, - no-absolute-import, - no-else-break, - no-else-continue, - no-else-raise, - no-else-return, - no-init, # added - no-member, - no-name-in-module, - no-self-use, - nonzero-method, - oct-method, - old-division, - old-ne-operator, - old-octal-literal, - old-raise-syntax, - parameter-unpacking, - print-statement, - raising-string, - range-builtin-not-iterating, - raw_input-builtin, - rdiv-method, - reduce-builtin, - relative-import, - reload-builtin, - round-builtin, - setslice-method, - signature-differs, - standarderror-builtin, - suppressed-message, - sys-max-int, - too-few-public-methods, - too-many-ancestors, - too-many-arguments, - too-many-boolean-expressions, - too-many-branches, - too-many-instance-attributes, - too-many-locals, - too-many-nested-blocks, - too-many-public-methods, - too-many-return-statements, - too-many-statements, - trailing-newlines, - unichr-builtin, - unicode-builtin, - unnecessary-pass, - unpacking-in-except, - useless-else-on-loop, - useless-object-inheritance, - useless-suppression, - using-cmp-argument, - wrong-import-order, - xrange-builtin, - zip-builtin-not-iterating, - - -[REPORTS] - -# Set the output format. Available formats are text, parseable, colorized, msvs -# (visual studio) and html. You can also give a reporter class, eg -# mypackage.mymodule.MyReporterClass. -output-format=text - -# Put messages in a separate file for each module / package specified on the -# command line instead of printing them on stdout. Reports (if any) will be -# written in a file name "pylint_global.[txt|html]". This option is deprecated -# and it will be removed in Pylint 2.0. -files-output=no - -# Tells whether to display a full report or only the messages -reports=no - -# Python expression which should return a note less than 10 (10 is the highest -# note). You have access to the variables errors warning, statement which -# respectively contain the number of errors / warnings messages and the total -# number of statements analyzed. This is used by the global evaluation report -# (RP0004). -evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) - -# Template used to display messages. This is a python new-style format string -# used to format the message information. See doc for all details -#msg-template= - - -[BASIC] - -# Good variable names which should always be accepted, separated by a comma -good-names=main,_ - -# Bad variable names which should always be refused, separated by a comma -bad-names= - -# Colon-delimited sets of names that determine each other's naming style when -# the name regexes allow several styles. -name-group= - -# Include a hint for the correct naming format with invalid-name -include-naming-hint=no - -# List of decorators that produce properties, such as abc.abstractproperty. Add -# to this list to register other decorators that produce valid properties. -property-classes=abc.abstractproperty,cached_property.cached_property,cached_property.threaded_cached_property,cached_property.cached_property_with_ttl,cached_property.threaded_cached_property_with_ttl - -# Regular expression matching correct function names -function-rgx=^(?:(?PsetUp|tearDown|setUpModule|tearDownModule)|(?P_?[A-Z][a-zA-Z0-9]*)|(?P_?[a-z][a-z0-9_]*))$ - -# Regular expression matching correct variable names -variable-rgx=^[a-z][a-z0-9_]*$ - -# Regular expression matching correct constant names -const-rgx=^(_?[A-Z][A-Z0-9_]*|__[a-z0-9_]+__|_?[a-z][a-z0-9_]*)$ - -# Regular expression matching correct attribute names -attr-rgx=^_{0,2}[a-z][a-z0-9_]*$ - -# Regular expression matching correct argument names -argument-rgx=^[a-z][a-z0-9_]*$ - -# Regular expression matching correct class attribute names -class-attribute-rgx=^(_?[A-Z][A-Z0-9_]*|__[a-z0-9_]+__|_?[a-z][a-z0-9_]*)$ - -# Regular expression matching correct inline iteration names -inlinevar-rgx=^[a-z][a-z0-9_]*$ - -# Regular expression matching correct class names -class-rgx=^_?[A-Z][a-zA-Z0-9]*$ - -# Regular expression matching correct module names -module-rgx=^(_?[a-z][a-z0-9_]*|__init__)$ - -# Regular expression matching correct method names -method-rgx=(?x)^(?:(?P_[a-z0-9_]+__|runTest|setUp|tearDown|setUpTestCase|tearDownTestCase|setupSelf|tearDownClass|setUpClass|(test|assert)_*[A-Z0-9][a-zA-Z0-9_]*|next)|(?P_{0,2}[A-Z][a-zA-Z0-9_]*)|(?P_{0,2}[a-z][a-z0-9_]*))$ - -# Regular expression which should only match function or class names that do -# not require a docstring. -no-docstring-rgx=(__.*__|main|test.*|.*test|.*Test)$ - -# Minimum line length for functions/classes that require docstrings, shorter -# ones are exempt. -docstring-min-length=10 - - -[TYPECHECK] - -# List of decorators that produce context managers, such as -# contextlib.contextmanager. Add to this list to register other decorators that -# produce valid context managers. -contextmanager-decorators=contextlib.contextmanager,contextlib2.contextmanager - -# Tells whether missing members accessed in mixin class should be ignored. A -# mixin class is detected if its name ends with "mixin" (case insensitive). -ignore-mixin-members=yes - -# List of module names for which member attributes should not be checked -# (useful for modules/projects where namespaces are manipulated during runtime -# and thus existing member attributes cannot be deduced by static analysis. It -# supports qualified module names, as well as Unix pattern matching. -ignored-modules= - -# List of class names for which member attributes should not be checked (useful -# for classes with dynamically set attributes). This supports the use of -# qualified names. -ignored-classes=optparse.Values,thread._local,_thread._local - -# List of members which are set dynamically and missed by pylint inference -# system, and so shouldn't trigger E1101 when accessed. Python regular -# expressions are accepted. -generated-members= - - -[FORMAT] - -# Maximum number of characters on a single line. -max-line-length=80 - -# TODO(https://github.com/PyCQA/pylint/issues/3352): Direct pylint to exempt -# lines made too long by directives to pytype. - -# Regexp for a line that is allowed to be longer than the limit. -ignore-long-lines=(?x)( - ^\s*(\#\ )??$| - ^\s*(from\s+\S+\s+)?import\s+.+$) - -# Allow the body of an if to be on the same line as the test if there is no -# else. -single-line-if-stmt=yes - -# List of optional constructs for which whitespace checking is disabled. `dict- -# separator` is used to allow tabulation in dicts, etc.: {1 : 1,\n222: 2}. -# `trailing-comma` allows a space between comma and closing bracket: (a, ). -# `empty-line` allows space-only lines. -no-space-check= - -# Maximum number of lines in a module -max-module-lines=99999 - -# String used as indentation unit. The internal Google style guide mandates 2 -# spaces. Google's externaly-published style guide says 4, consistent with -# PEP 8. Here, we use 2 spaces, for conformity with many open-sourced Google -# projects (like TensorFlow). -indent-string=' ' - -# Number of spaces of indent required inside a hanging or continued line. -indent-after-paren=4 - -# Expected format of line ending, e.g. empty (any line ending), LF or CRLF. -expected-line-ending-format= - - -[MISCELLANEOUS] - -# List of note tags to take in consideration, separated by a comma. -notes=TODO - - -[STRING] - -# This flag controls whether inconsistent-quotes generates a warning when the -# character used as a quote delimiter is used inconsistently within a module. -check-quote-consistency=yes - - -[VARIABLES] - -# Tells whether we should check for unused import in __init__ files. -init-import=no - -# A regular expression matching the name of dummy variables (i.e. expectedly -# not used). -dummy-variables-rgx=^\*{0,2}(_$|unused_|dummy_) - -# List of additional names supposed to be defined in builtins. Remember that -# you should avoid to define new builtins when possible. -additional-builtins= - -# List of strings which can identify a callback function by name. A callback -# name must start or end with one of those strings. -callbacks=cb_,_cb - -# List of qualified module names which can have objects that can redefine -# builtins. -redefining-builtins-modules=six,six.moves,past.builtins,future.builtins,functools - - -[LOGGING] - -# Logging modules to check that the string format arguments are in logging -# function parameter format -logging-modules=logging,absl.logging,tensorflow.io.logging - - -[SIMILARITIES] - -# Minimum lines number of a similarity. -min-similarity-lines=4 - -# Ignore comments when computing similarities. -ignore-comments=yes - -# Ignore docstrings when computing similarities. -ignore-docstrings=yes - -# Ignore imports when computing similarities. -ignore-imports=no - - -[SPELLING] - -# Spelling dictionary name. Available dictionaries: none. To make it working -# install python-enchant package. -spelling-dict= - -# List of comma separated words that should not be checked. -spelling-ignore-words= - -# A path to a file that contains private dictionary; one word per line. -spelling-private-dict-file= - -# Tells whether to store unknown words to indicated private dictionary in -# --spelling-private-dict-file option instead of raising a message. -spelling-store-unknown-words=no - - -[IMPORTS] - -# Deprecated modules which should not be used, separated by a comma -deprecated-modules=regsub, - TERMIOS, - Bastion, - rexec, - sets - -# Create a graph of every (i.e. internal and external) dependencies in the -# given file (report RP0402 must not be disabled) -import-graph= - -# Create a graph of external dependencies in the given file (report RP0402 must -# not be disabled) -ext-import-graph= - -# Create a graph of internal dependencies in the given file (report RP0402 must -# not be disabled) -int-import-graph= - -# Force import order to recognize a module as part of the standard -# compatibility libraries. -known-standard-library= - -# Force import order to recognize a module as part of a third party library. -known-third-party=enchant, absl - -# Analyse import fallback blocks. This can be used to support both Python 2 and -# 3 compatible code, which means that the block might have code that exists -# only in one or another interpreter, leading to false positives when analysed. -analyse-fallback-blocks=no - - -[CLASSES] - -# List of method names used to declare (i.e. assign) instance attributes. -defining-attr-methods=__init__, - __new__, - setUp - -# List of member names, which should be excluded from the protected access -# warning. -exclude-protected=_asdict, - _fields, - _replace, - _source, - _make - -# List of valid names for the first argument in a class method. -valid-classmethod-first-arg=cls, - class_ - -# List of valid names for the first argument in a metaclass class method. -valid-metaclass-classmethod-first-arg=mcs - - -[EXCEPTIONS] - -# Exceptions that will emit a warning when being caught. Defaults to -# "Exception" -overgeneral-exceptions=StandardError, - Exception, - BaseException diff --git a/benchmarks/perf-tool/.style.yapf b/benchmarks/perf-tool/.style.yapf deleted file mode 100644 index 39b663a7a..000000000 --- a/benchmarks/perf-tool/.style.yapf +++ /dev/null @@ -1,10 +0,0 @@ -[style] -COLUMN_LIMIT: 80 -DEDENT_CLOSING_BRACKETS: True -INDENT_DICTIONARY_VALUE: True -SPLIT_ALL_COMMA_SEPARATED_VALUES: True -SPLIT_ARGUMENTS_WHEN_COMMA_TERMINATED: True -SPLIT_BEFORE_CLOSING_BRACKET: True -SPLIT_BEFORE_EXPRESSION_AFTER_OPENING_PAREN: True -SPLIT_BEFORE_FIRST_ARGUMENT: True -SPLIT_BEFORE_NAMED_ASSIGNS: True diff --git a/benchmarks/perf-tool/README.md b/benchmarks/perf-tool/README.md deleted file mode 100644 index 36f76bcdb..000000000 --- a/benchmarks/perf-tool/README.md +++ /dev/null @@ -1,449 +0,0 @@ -# IMPORTANT NOTE: No new features will be added to this tool . This tool is currently in maintanence mode. All new features will be added to [vector search workload]( https://github.com/opensearch-project/opensearch-benchmark-workloads/tree/main/vectorsearch) - -# OpenSearch k-NN Benchmarking -- [Welcome!](#welcome) -- [Install Prerequisites](#install-prerequisites) -- [Usage](#usage) -- [Contributing](#contributing) - -## Welcome! - -This directory contains the code related to benchmarking the k-NN plugin. -Benchmarks can be run against any OpenSearch cluster with the k-NN plugin -installed. Benchmarks are highly configurable using the test configuration -file. - -## Install Prerequisites - -### Setup - -K-NN perf requires Python 3.8 or greater to be installed. One of -the easier ways to do this is through Conda, a package and environment -management system for Python. - -First, follow the -[installation instructions](https://docs.conda.io/projects/conda/en/latest/user-guide/install/index.html) -to install Conda on your system. - -Next, create a Python 3.8 environment: -``` -conda create -n knn-perf python=3.8 -``` - -After the environment is created, activate it: -``` -source activate knn-perf -``` - -Lastly, clone the k-NN repo and install all required python packages: -``` -git clone https://github.com/opensearch-project/k-NN.git -cd k-NN/benchmarks/perf-tool -pip install -r requirements.txt -``` - -After all of this completes, you should be ready to run your first performance benchmarks! - - -## Usage - -### Quick Start - -In order to run a benchmark, you must first create a test configuration yml -file. Checkout [this example](https://github.com/opensearch-project/k-NN/blob/main/benchmarks/perf-tool/sample-configs) file -for benchmarking *faiss*'s IVF method. This file contains the definition for -the benchmark that you want to run. At the top are -[test parameters](#test-parameters). These define high level settings of the -test, such as the endpoint of the OpenSearch cluster. - -Next, you define the actions that the test will perform. These actions are -referred to as steps. First, you can define "setup" steps. These are steps that -are run once at the beginning of the execution to configure the cluster how you -want it. These steps do not contribute to the final metrics. - -After that, you define the "steps". These are the steps that the test will be -collecting metrics on. Each step emits certain metrics. These are run -multiple times, depending on the test parameter "num_runs". At the end of the -execution of all of the runs, the metrics from each run are collected and -averaged. - -Lastly, you define the "cleanup" steps. The "cleanup" steps are executed after -each test run. For instance, if you are measuring index performance, you may -want to delete the index after each run. - -To run the test, execute the following command: -``` -python knn-perf-tool.py [--log LOGLEVEL] test config-path.yml output.json - ---log log level of tool, options are: info, debug, warning, error, critical -``` - -The output will be a json document containing the results. - -Additionally, you can get the difference between two test runs using the diff -command: -``` -python knn-perf-tool.py [--log LOGLEVEL] diff result1.json result2.json - ---log log level of tool, options are: info, debug, warning, error, critical -``` - -The output will be the delta between the two metrics. - -### Test Parameters - -| Parameter Name | Description | Default | -|----------------|------------------------------------------------------------------------------------|------------| -| endpoint | Endpoint OpenSearch cluster is running on | localhost | -| port | Port on which OpenSearch Cluster is running on | 9200 | -| test_name | Name of test | No default | -| test_id | String ID of test | No default | -| num_runs | Number of runs to execute steps | 1 | -| show_runs | Whether to output each run in addition to the total summary | false | -| setup | List of steps to run once before metric collection starts | [] | -| steps | List of steps that make up one test run. Metrics will be collected on these steps. | No default | -| cleanup | List of steps to run after each test run | [] | - -### Steps - -Included are the list of steps that are currently supported. Each step contains -a set of parameters that are passed in the test configuration file and a set -of metrics that the test produces. - -#### create_index - -Creates an OpenSearch index. - -##### Parameters -| Parameter Name | Description | Default | -| ----------- | ----------- | ----------- | -| index_name | Name of index to create | No default | -| index_spec | Path to index specification | No default | - -##### Metrics - -| Metric Name | Description | Unit | -| ----------- | ----------- | ----------- | -| took | Time to execute step end to end. | ms | - -#### disable_refresh - -Disables refresh for all indices in the cluster. - -##### Parameters - -| Parameter Name | Description | Default | -| ----------- | ----------- | ----------- | - -##### Metrics - -| Metric Name | Description | Unit | -| ----------- | ----------- | ----------- | -| took | Time to execute step end to end. | ms | - -#### refresh_index - -Refreshes an OpenSearch index. - -##### Parameters - -| Parameter Name | Description | Default | -| ----------- | ----------- | ----------- | -| index_name | Name of index to refresh | No default | - -##### Metrics - -| Metric Name | Description | Unit | -| ----------- | ----------- | ----------- | -| took | Time to execute step end to end. | ms | -| store_kb | Size of index after refresh completes | KB | - -#### force_merge - -Force merges an index to a specified number of segments. - -##### Parameters - -| Parameter Name | Description | Default | -| ----------- | ----------- | ----------- | -| index_name | Name of index to force merge | No default | -| max_num_segments | Number of segments to force merge to | No default | - -##### Metrics - -| Metric Name | Description | Unit | -| ----------- | ----------- | ----------- | -| took | Time to execute step end to end. | ms | - -#### train_model - -Trains a model. - -##### Parameters - -| Parameter Name | Description | Default | -| ----------- | ----------- | ----------- | -| model_id | Model id to set | Test | -| train_index | Index to pull training data from | No default | -| train_field | Field to pull training data from | No default | -| dimension | Dimension of model | No default | -| description | Description of model | No default | -| max_training_vector_count | Number of training vectors to used | No default | -| method_spec | Path to method specification | No default | - -##### Metrics - -| Metric Name | Description | Unit | -| ----------- | ----------- | ----------- | -| took | Time to execute step end to end | ms | - -#### delete_model - -Deletes a model from the cluster. - -##### Parameters - -| Parameter Name | Description | Default | -| ----------- | ----------- | ----------- | -| model_id | Model id to delete | Test | - -##### Metrics - -| Metric Name | Description | Unit | -| ----------- | ----------- | ----------- | -| took | Time to execute step end to end | ms | - -#### delete_index - -Deletes an index from the cluster. - -##### Parameters - -| Parameter Name | Description | Default | -| ----------- | ----------- | ----------- | -| index_name | Name of index to delete | No default | - -##### Metrics - -| Metric Name | Description | Unit | -| ----------- | ----------- | ----------- | -| took | Time to execute step end to end | ms | - -#### ingest - -Ingests a dataset of vectors into the cluster. - -##### Parameters - -| Parameter Name | Description | Default | -| ----------- | ----------- | ----------- | -| index_name | Name of index to ingest into | No default | -| field_name | Name of field to ingest into | No default | -| bulk_size | Documents per bulk request | 300 | -| dataset_format | Format the data-set is in. Currently hdf5 and bigann is supported. The hdf5 file must be organized in the same way that the ann-benchmarks organizes theirs. | 'hdf5' | -| dataset_path | Path to data-set | No default | -| doc_count | Number of documents to create from data-set | Size of the data-set | - -##### Metrics - -| Metric Name | Description | Unit | -| ----------- | ----------- | ----------- | -| took | Total time to ingest the dataset into the index.| ms | - -#### ingest_multi_field - -Ingests a dataset of multiple context types into the cluster. - -##### Parameters - -| Parameter Name | Description | Default | -| ----------- |-----------------------------------------------------------------------------------------------------------------------------------------------------------| ----------- | -| index_name | Name of index to ingest into | No default | -| field_name | Name of field to ingest into | No default | -| bulk_size | Documents per bulk request | 300 | -| dataset_path | Path to data-set | No default | -| doc_count | Number of documents to create from data-set | Size of the data-set | -| attributes_dataset_name | Name of dataset with additional attributes inside the main dataset | No default | -| attribute_spec | Definition of attributes, format is: [{ name: [name_val], type: [type_val]}] Order is important and must match order of attributes column in dataset file | No default | - -##### Metrics - -| Metric Name | Description | Unit | -| ----------- | ----------- | ----------- | -| took | Total time to ingest the dataset into the index.| ms | - -#### ingest_nested_field - -Ingests a dataset with nested field into the cluster. - -##### Parameters - -| Parameter Name | Description | Default | -| ----------- |------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| ----------- | -| index_name | Name of index to ingest into | No default | -| field_name | Name of field to ingest into | No default | -| dataset_path | Path to data-set | No default | -| attributes_dataset_name | Name of dataset with additional attributes inside the main dataset | No default | -| attribute_spec | Definition of attributes, format is: [{ name: [name_val], type: [type_val]}] Order is important and must match order of attributes column in dataset file. It should contains { name: 'parent_id', type: 'int'} | No default | - -##### Metrics - -| Metric Name | Description | Unit | -| ----------- | ----------- | ----------- | -| took | Total time to ingest the dataset into the index.| ms | - -#### query - -Runs a set of queries against an index. - -##### Parameters - -| Parameter Name | Description | Default | -| ----------- | ----------- | ----------- | -| k | Number of neighbors to return on search | 100 | -| r | r value in Recall@R | 1 | -| index_name | Name of index to search | No default | -| field_name | Name field to search | No default | -| calculate_recall | Whether to calculate recall values | False | -| dataset_format | Format the dataset is in. Currently hdf5 and bigann is supported. The hdf5 file must be organized in the same way that the ann-benchmarks organizes theirs. | 'hdf5' | -| dataset_path | Path to dataset | No default | -| neighbors_format | Format the neighbors dataset is in. Currently hdf5 and bigann is supported. The hdf5 file must be organized in the same way that the ann-benchmarks organizes theirs. | 'hdf5' | -| neighbors_path | Path to neighbors dataset | No default | -| query_count | Number of queries to create from data-set | Size of the data-set | - -##### Metrics - -| Metric Name | Description | Unit | -| ----------- |---------------------------------------------------------------------------------------------------------| ----------- | -| took | Took times returned per query aggregated as total, p50, p90, p99, p99.9 and p100 (when applicable) | ms | -| memory_kb | Native memory k-NN is using at the end of the query workload | KB | -| recall@R | ratio of top R results from the ground truth neighbors that are in the K results returned by the plugin | float 0.0-1.0 | -| recall@K | ratio of results returned that were ground truth nearest neighbors | float 0.0-1.0 | - -#### query_with_filter - -Runs a set of queries with filter against an index. - -##### Parameters - -| Parameter Name | Description | Default | -| ----------- |-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------------------| -| k | Number of neighbors to return on search | 100 | -| r | r value in Recall@R | 1 | -| index_name | Name of index to search | No default | -| field_name | Name field to search | No default | -| calculate_recall | Whether to calculate recall values | False | -| dataset_format | Format the dataset is in. Currently hdf5 and bigann is supported. The hdf5 file must be organized in the same way that the ann-benchmarks organizes theirs. | 'hdf5' | -| dataset_path | Path to dataset | No default | -| neighbors_format | Format the neighbors dataset is in. Currently hdf5 and bigann is supported. The hdf5 file must be organized in the same way that the ann-benchmarks organizes theirs. | 'hdf5' | -| neighbors_path | Path to neighbors dataset | No default | -| neighbors_dataset | Name of filter dataset inside the neighbors dataset | No default | -| filter_spec | Path to filter specification | No default | -| filter_type | Type of filter format, we do support following types:
FILTER inner filter format for approximate k-NN search
SCRIPT score scripting with exact k-NN search and pre-filtering
BOOL_POST_FILTER Bool query with post-filtering | SCRIPT | -| score_script_similarity | Similarity function that has been used to index dataset. Used for SCRIPT filter type and ignored for others | l2 | -| query_count | Number of queries to create from data-set | Size of the data-set | - -##### Metrics - -| Metric Name | Description | Unit | -| ----------- | ----------- | ----------- | -| took | Took times returned per query aggregated as total, p50, p90 and p99 (when applicable) | ms | -| memory_kb | Native memory k-NN is using at the end of the query workload | KB | -| recall@R | ratio of top R results from the ground truth neighbors that are in the K results returned by the plugin | float 0.0-1.0 | -| recall@K | ratio of results returned that were ground truth nearest neighbors | float 0.0-1.0 | - - -#### query_nested_field - -Runs a set of queries with nested field against an index. - -##### Parameters - -| Parameter Name | Description | Default | -| ----------- |-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------------------| -| k | Number of neighbors to return on search | 100 | -| r | r value in Recall@R | 1 | -| index_name | Name of index to search | No default | -| field_name | Name field to search | No default | -| calculate_recall | Whether to calculate recall values | False | -| dataset_format | Format the dataset is in. Currently hdf5 and bigann is supported. The hdf5 file must be organized in the same way that the ann-benchmarks organizes theirs. | 'hdf5' | -| dataset_path | Path to dataset | No default | -| neighbors_format | Format the neighbors dataset is in. Currently hdf5 and bigann is supported. The hdf5 file must be organized in the same way that the ann-benchmarks organizes theirs. | 'hdf5' | -| neighbors_path | Path to neighbors dataset | No default | -| neighbors_dataset | Name of filter dataset inside the neighbors dataset | No default | -| query_count | Number of queries to create from data-set | Size of the data-set | - -##### Metrics - -| Metric Name | Description | Unit | -| ----------- | ----------- | ----------- | -| took | Took times returned per query aggregated as total, p50, p90 and p99 (when applicable) | ms | -| memory_kb | Native memory k-NN is using at the end of the query workload | KB | -| recall@R | ratio of top R results from the ground truth neighbors that are in the K results returned by the plugin | float 0.0-1.0 | -| recall@K | ratio of results returned that were ground truth nearest neighbors | float 0.0-1.0 | - -#### get_stats - -Gets the index stats. - -##### Parameters - -| Parameter Name | Description | Default | -| ----------- |-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------------------| -| index_name | Name of index to search | No default | - -##### Metrics - -| Metric Name | Description | Unit | -| ----------- |-------------------------------------------------|------------| -| num_of_committed_segments | Total number of commited segments in the index | integer >= 0 | -| num_of_search_segments | Total number of search segments in the index | integer >= 0 | - -### Data sets - -This benchmark tool uses pre-generated data sets to run indexing and query workload. For some benchmark types existing dataset need to be -extended. Filtering is an example of use case where such dataset extension is needed. - -It's possible to use script provided with this repo to generate dataset and run benchmark for filtering queries. -You need to have existing dataset with vector data. This dataset will be used to generate additional attribute data and set of ground truth neighbours document ids. - -To generate dataset with attributes based on vectors only dataset use following command pattern: - -```commandline -python add-filters-to-dataset.py True False -``` - -To generate neighbours dataset for different filters based on dataset with attributes use following command pattern: - -```commandline -python add-filters-to-dataset.py False True -``` - -After that new dataset(s) can be referred from testcase definition in `ingest_extended` and `query_with_filter` steps. - -To generate dataset with parent doc id based on vectors only dataset, use following command pattern: -```commandline -python add-parent-doc-id-to-dataset.py -``` -This will generate neighbours dataset as well. This new dataset(s) can be referred from testcase definition in `ingest_nested_field` and `query_nested_field` steps. - -## Contributing - -### Linting - -Use pylint to lint the code: -``` -pylint knn-perf-tool.py okpt/**/*.py okpt/**/**/*.py -``` - -### Formatting - -We use yapf and the google style to format our code. After installing yapf, you can format your code by running: - -``` -yapf --style google knn-perf-tool.py okpt/**/*.py okpt/**/**/*.py -``` - -### Updating requirements - -Add new requirements to "requirements.in" and run `pip-compile` diff --git a/benchmarks/perf-tool/add-filters-to-dataset.py b/benchmarks/perf-tool/add-filters-to-dataset.py deleted file mode 100644 index 0624f7323..000000000 --- a/benchmarks/perf-tool/add-filters-to-dataset.py +++ /dev/null @@ -1,200 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# -# The OpenSearch Contributors require contributions made to -# this file be licensed under the Apache-2.0 license or a -# compatible open source license. -""" -Script builds complex dataset with additional attributes from exiting dataset that has only vectors. -Additional attributes are predefined in the script: color, taste, age. Only HDF5 format of vector dataset is supported. - -Output dataset file will have additional dataset 'attributes' with multiple columns, each column corresponds to one attribute -from an attribute set, and value is generated at random, e.g.: - -0: green None 71 -1: green bitter 28 - -there is no explicit index reference in 'attributes' dataset, index of the row corresponds to a document id. -For instance, in example above two rows of fields mapped to documents with ids '0' and '1'. - -If 'generate_filters' flag is set script generates additional dataset of neighbours (ground truth) for each filter type. -Output is a new file with several datasets, each dataset corresponds to one filter. Datasets are named 'neighbour_filter_X' -where X is 1 based index of particular filter. -Each dataset has rows with array of integers, where integer corresponds to -a document id from original dataset with additional fields. Array ca have -1 values that are treated as null, this is because -subset of filtered documents is same of smaller than original set. - -For example, dataset file content may look like : - -neighbour_filter_1: [[ 2, 5, -1], - [ 3, 1, -1], - [ 2 5, 7]] -neighbour_filter_2: [[-1, -1, -1], - [ 5, 6, -1], - [ 4, 2, 1]] - -In this case we do have datasets for two filters, 3 query results for each. [2, 5, -1] indicates that for first query -if filter 1 is used most similar document is with id 2, next similar is 5, and the rest do not pass filter 1 criteria. - -Example of script usage: - - create new hdf5 file with attribute dataset - add-filters-to-dataset.py ~/dev/opensearch/k-NN/benchmarks/perf-tool/dataset/data.hdf5 ~/dev/opensearch/datasets/data-with-attr True False - - create new hdf5 file with filter datasets - add-filters-to-dataset.py ~/dev/opensearch/k-NN/benchmarks/perf-tool/dataset/data-with-attr.hdf5 ~/dev/opensearch/datasets/data-with-filters False True -""" - -import getopt -import os -import random -import sys - -import h5py - -from osb.extensions.data_set import HDF5DataSet - - -class _Dataset: - """Type of dataset container for data with additional attributes""" - DEFAULT_TYPE = HDF5DataSet.FORMAT_NAME - - def create_dataset(self, source_dataset_path, out_file_path, generate_attrs: bool, generate_filters: bool) -> None: - path_elements = os.path.split(os.path.abspath(source_dataset_path)) - data_set_dir = path_elements[0] - - # For HDF5, because multiple data sets can be grouped in the same file, - # we will build data sets in memory and not write to disk until - # _flush_data_sets_to_disk is called - # read existing dataset - data_hdf5 = os.path.join(os.path.dirname(os.path.realpath('/')), source_dataset_path) - - with h5py.File(data_hdf5, "r") as hf: - - if generate_attrs: - data_set_w_attr = self.create_dataset_file(out_file_path, self.DEFAULT_TYPE, data_set_dir) - - possible_colors = ['red', 'green', 'yellow', 'blue', None] - possible_tastes = ['sweet', 'salty', 'sour', 'bitter', None] - max_age = 100 - - for key in hf.keys(): - if key not in ['neighbors', 'test', 'train']: - continue - data_set_w_attr.create_dataset(key, data=hf[key][()]) - - attributes = [] - for i in range(len(hf['train'])): - attr = [random.choice(possible_colors), random.choice(possible_tastes), - random.randint(0, max_age + 1)] - attributes.append(attr) - - data_set_w_attr.create_dataset('attributes', (len(attributes), 3), 'S10', data=attributes) - - data_set_w_attr.flush() - data_set_w_attr.close() - - if generate_filters: - attributes = hf['attributes'][()] - expected_neighbors = hf['neighbors'][()] - - data_set_filters = self.create_dataset_file(out_file_path, self.DEFAULT_TYPE, data_set_dir) - - def filter1(attributes, vector_idx): - if attributes[vector_idx][0].decode() == 'red' and int(attributes[vector_idx][2].decode()) >= 20: - return True - else: - return False - - self.apply_filter(expected_neighbors, attributes, data_set_filters, 'neighbors_filter_1', filter1) - - # filter 2 - color = blue or None and taste = 'salty' - def filter2(attributes, vector_idx): - if (attributes[vector_idx][0].decode() == 'blue' or attributes[vector_idx][ - 0].decode() == 'None') and attributes[vector_idx][1].decode() == 'salty': - return True - else: - return False - - self.apply_filter(expected_neighbors, attributes, data_set_filters, 'neighbors_filter_2', filter2) - - # filter 3 - color and taste are not None and age is between 20 and 80 - def filter3(attributes, vector_idx): - if attributes[vector_idx][0].decode() != 'None' and attributes[vector_idx][ - 1].decode() != 'None' and 20 <= \ - int(attributes[vector_idx][2].decode()) <= 80: - return True - else: - return False - - self.apply_filter(expected_neighbors, attributes, data_set_filters, 'neighbors_filter_3', filter3) - - # filter 4 - color green or blue and taste is bitter and age is between (30, 60) - def filter4(attributes, vector_idx): - if (attributes[vector_idx][0].decode() == 'green' or attributes[vector_idx][0].decode() == 'blue') \ - and (attributes[vector_idx][1].decode() == 'bitter') \ - and 30 <= int(attributes[vector_idx][2].decode()) <= 60: - return True - else: - return False - - self.apply_filter(expected_neighbors, attributes, data_set_filters, 'neighbors_filter_4', filter4) - - # filter 5 color is (green or blue or yellow) or taste = sweet or age is between (30, 70) - def filter5(attributes, vector_idx): - if attributes[vector_idx][0].decode() == 'green' or attributes[vector_idx][0].decode() == 'blue' \ - or attributes[vector_idx][0].decode() == 'yellow' \ - or attributes[vector_idx][1].decode() == 'sweet' \ - or 30 <= int(attributes[vector_idx][2].decode()) <= 70: - return True - else: - return False - - self.apply_filter(expected_neighbors, attributes, data_set_filters, 'neighbors_filter_5', filter5) - - data_set_filters.flush() - data_set_filters.close() - - def apply_filter(self, expected_neighbors, attributes, data_set_w_filtering, filter_name, filter_func): - neighbors_filter = [] - filtered_count = 0 - for expected_neighbors_row in expected_neighbors: - neighbors_filter_row = [-1] * len(expected_neighbors_row) - idx = 0 - for vector_idx in expected_neighbors_row: - if filter_func(attributes, vector_idx): - neighbors_filter_row[idx] = vector_idx - idx += 1 - filtered_count += 1 - neighbors_filter.append(neighbors_filter_row) - overall_count = len(expected_neighbors) * len(expected_neighbors[0]) - perc = float(filtered_count / overall_count) * 100 - print('ground truth size for {} is {}, percentage {}'.format(filter_name, filtered_count, perc)) - data_set_w_filtering.create_dataset(filter_name, data=neighbors_filter) - return expected_neighbors - - def create_dataset_file(self, file_name, extension, data_set_dir) -> h5py.File: - data_set_file_name = "{}.{}".format(file_name, extension) - data_set_path = os.path.join(data_set_dir, data_set_file_name) - - data_set_w_filtering = h5py.File(data_set_path, 'a') - - return data_set_w_filtering - - -def main(argv): - opts, args = getopt.getopt(argv, "") - in_file_path = args[0] - out_file_path = args[1] - generate_attr = str2bool(args[2]) - generate_filters = str2bool(args[3]) - - worker = _Dataset() - worker.create_dataset(in_file_path, out_file_path, generate_attr, generate_filters) - - -def str2bool(v): - return v.lower() in ("yes", "true", "t", "1") - - -if __name__ == "__main__": - main(sys.argv[1:]) diff --git a/benchmarks/perf-tool/add-parent-doc-id-to-dataset.py b/benchmarks/perf-tool/add-parent-doc-id-to-dataset.py deleted file mode 100644 index a4acafd03..000000000 --- a/benchmarks/perf-tool/add-parent-doc-id-to-dataset.py +++ /dev/null @@ -1,291 +0,0 @@ -# Copyright OpenSearch Contributors -# SPDX-License-Identifier: Apache-2.0 - -""" -Script builds complex dataset with additional attributes from exiting dataset that has only vectors. -Additional attributes are predefined in the script: color, taste, age, and parent doc id. Only HDF5 format of vector dataset is supported. - -Output dataset file will have additional dataset 'attributes' with multiple columns, each column corresponds to one attribute -from an attribute set, and value is generated at random, e.g.: - -0: green None 71 1 -1: green bitter 28 1 -2: green bitter 28 1 -3: green bitter 28 2 -... - -there is no explicit index reference in 'attributes' dataset, index of the row corresponds to a document id. -For instance, in example above two rows of fields mapped to documents with ids '0' and '1'. - -The parend doc ids are assigned in non-decreasing order. - -If 'generate_filters' flag is set script generates additional dataset of neighbours (ground truth). -Output is a new file with three dataset each of which corresponds to a certain type of query. -Dataset name neighbour_nested is a ground truth for query without filtering. -Dataset name neighbour_filtered_relaxed is a ground truth for query with filtering of (30 <= age <= 70) or color in ["green", "blue", "yellow"] or taste in ["sweet"] -Dataset name neighbour_filtered_restrictive is a ground truth for query with filtering of (30 <= age <= 60) and color in ["green", "blue"] and taste in ["bitter"] - - -Each dataset has rows with array of integers, where integer corresponds to -a document id from original dataset with additional fields. - -Example of script usage: - - create new hdf5 file with attribute dataset - add-parent-doc-id-to-dataset.py ~/dev/opensearch/k-NN/benchmarks/perf-tool/dataset/data.hdf5 ~/dev/opensearch/datasets/data-nested.hdf5 - -""" -import getopt -import multiprocessing -import random -import sys -from multiprocessing import Process -from typing import cast -import traceback - -import h5py -import numpy as np - - -class MyVector: - def __init__(self, vector, id, color=None, taste=None, age=None, parent_id=None): - self.vector = vector - self.id = id - self.age = age - self.color = color - self.taste = taste - self.parent_id = parent_id - - def apply_restricted_filter(self): - return (30 <= self.age <= 60) and self.color in ["green", "blue"] and self.taste in ["bitter"] - - def apply_relaxed_filter(self): - return (30 <= self.age <= 70) or self.color in ["green", "blue", "yellow"] or self.taste in ["sweet"] - - def __str__(self): - return f'Vector : {self.vector}, id : {self.id}, color: {self.color}, taste: {self.taste}, age: {self.age}, parent_id: {self.parent_id}\n' - - def __repr__(self): - return f'Vector : {self.vector}, id : {self.id}, color: {self.color}, taste: {self.taste}, age: {self.age}, parent_id: {self.parent_id}\n' - -class HDF5DataSet: - def __init__(self, file_path, key): - self.file_name = file_path - self.file = h5py.File(self.file_name) - self.key = key - self.data = cast(h5py.Dataset, self.file[key]) - self.metadata = None - self.metadata = cast(h5py.Dataset, self.file["attributes"]) if key == "train" else None - print(f'Keys in the file are {self.file.keys()}') - - def read(self, start, end=None): - if end is None: - end = self.data.len() - values = cast(np.ndarray, self.data[start:end]) - metadata = cast(list, self.metadata[start:end]) if self.metadata is not None else None - if metadata is not None: - print(metadata) - vectors = [] - i = 0 - for value in values: - if self.metadata is None: - vector = MyVector(value, i) - else: - # color, taste, age, and parent id - vector = MyVector(value, i, str(metadata[i][0].decode()), str(metadata[i][1].decode()), - int(metadata[i][2]), int(metadata[i][3])) - vectors.append(vector) - i = i + 1 - return vectors - - def read_neighbors(self, start, end): - return cast(np.ndarray, self.data[start:end]) - - def size(self): - return self.data.len() - - def close(self): - self.file.close() - -class _Dataset: - def run(self, source_path, target_path) -> None: - # Add attributes - print(f'Adding attributes started.') - with h5py.File(source_path, "r") as in_file: - out_file = h5py.File(target_path, "w") - possible_colors = ['red', 'green', 'yellow', 'blue', None] - possible_tastes = ['sweet', 'salty', 'sour', 'bitter', None] - max_age = 100 - min_field_size = 10 - max_field_size = 10 - - # Copy train and test data - for key in in_file.keys(): - if key not in ['test', 'train']: - continue - out_file.create_dataset(key, data=in_file[key][()]) - - # Generate attributes - attributes = [] - field_size = random.randint(min_field_size, max_field_size) - parent_id = 1 - field_count = 0 - for i in range(len(in_file['train'])): - attr = [random.choice(possible_colors), random.choice(possible_tastes), - random.randint(0, max_age + 1), parent_id] - attributes.append(attr) - field_count += 1 - if field_count >= field_size: - field_size = random.randint(min_field_size, max_field_size) - field_count = 0 - parent_id += 1 - out_file.create_dataset('attributes', (len(attributes), 4), 'S10', data=attributes) - - out_file.flush() - out_file.close() - - print(f'Adding attributes completed.') - - - # Calculate ground truth - print(f'Calculating ground truth started.') - cpus = multiprocessing.cpu_count() - total_clients = min(8, cpus) # 1 # 10 - hdf5Data_train = HDF5DataSet(target_path, "train") - train_vectors = hdf5Data_train.read(0, hdf5Data_train.size()) - hdf5Data_train.close() - print(f'Train vector size: {len(train_vectors)}') - - hdf5Data_test = HDF5DataSet(target_path, "test") - total_queries = hdf5Data_test.size() # 10000 - dis = [] * total_queries - - for i in range(total_queries): - dis.insert(i, []) - - queries_per_client = int(total_queries / total_clients + 0.5) - if queries_per_client == 0: - queries_per_client = total_queries - - processes = [] - test_vectors = hdf5Data_test.read(0, total_queries) - hdf5Data_test.close() - tasks_that_are_done = multiprocessing.Queue() - for client in range(total_clients): - start_index = int(client * queries_per_client) - if start_index + queries_per_client <= total_queries: - end_index = int(start_index + queries_per_client) - else: - end_index = total_queries - - print(f'Start Index: {start_index}, end Index: {end_index}') - print(f'client is : {client}') - p = Process(target=queryTask, args=( - train_vectors, test_vectors, start_index, end_index, client, total_queries, tasks_that_are_done)) - processes.append(p) - p.start() - if end_index >= total_queries: - print(f'Exiting end Index : {end_index} total_queries: {total_queries}') - break - - # wait for tasks to be completed - print('Waiting for all tasks to be completed') - j = 0 - # This is required because threads can hang if the data sent from the sub process increases by a certain limit - # https://stackoverflow.com/questions/21641887/python-multiprocessing-process-hangs-on-join-for-large-queue - while j < total_queries: - while not tasks_that_are_done.empty(): - calculatedDis = tasks_that_are_done.get() - i = 0 - for d in calculatedDis: - if d: - dis[i] = d - j = j + 1 - i = i + 1 - - for p in processes: - if p.is_alive(): - p.join() - else: - print("Process was not alive hence shutting down") - - data_set_file = h5py.File(target_path, "a") - for type in ['nested', 'relaxed', 'restricted']: - results = [] - for d in dis: - r = [] - for i in range(min(10000, len(d[type]))): - r.append(d[type][i]['id']) - results.append(r) - - - data_set_file.create_dataset("neighbour_" + type, (len(results), len(results[0])), data=results) - data_set_file.flush() - data_set_file.close() - -def calculateL2Distance(point1, point2): - return np.linalg.norm(point1 - point2) - - -def queryTask(train_vectors, test_vectors, startIndex, endIndex, process_number, total_queries, tasks_that_are_done): - print(f'Starting Process number : {process_number}') - all_distances = [] * total_queries - for i in range(total_queries): - all_distances.insert(i, {}) - try: - test_vectors = test_vectors[startIndex:endIndex] - i = startIndex - for test in test_vectors: - distances = [] - values = {} - for value in train_vectors: - values[value.id] = value - distances.append({ - "dis": calculateL2Distance(test.vector, value.vector), - "id": value.parent_id - }) - - distances.sort(key=lambda vector: vector['dis']) - seen_set_nested = set() - seen_set_restricted = set() - seen_set_relaxed = set() - nested = [] - restricted = [] - relaxed = [] - for sub_i in range(len(distances)): - id = distances[sub_i]['id'] - # Check if the number has been seen before - if len(nested) < 1000 and id not in seen_set_nested: - # If not seen before, mark it as seen - seen_set_nested.add(id) - nested.append(distances[sub_i]) - if len(restricted) < 1000 and id not in seen_set_restricted and values[id].apply_restricted_filter(): - seen_set_restricted.add(id) - restricted.append(distances[sub_i]) - if len(relaxed) < 1000 and id not in seen_set_relaxed and values[id].apply_relaxed_filter(): - seen_set_relaxed.add(id) - relaxed.append(distances[sub_i]) - - all_distances[i]['nested'] = nested - all_distances[i]['restricted'] = restricted - all_distances[i]['relaxed'] = relaxed - print(f"Process {process_number} queries completed: {i + 1 - startIndex}, queries left: {endIndex - i - 1}") - i = i + 1 - except: - print( - f"Got exception while running the thread: {process_number} with startIndex: {startIndex} endIndex: {endIndex} ") - traceback.print_exc() - tasks_that_are_done.put(all_distances) - print(f'Exiting Process number : {process_number}') - - -def main(argv): - opts, args = getopt.getopt(argv, "") - in_file_path = args[0] - out_file_path = args[1] - - worker = _Dataset() - worker.run(in_file_path, out_file_path) - -if __name__ == "__main__": - main(sys.argv[1:]) \ No newline at end of file diff --git a/benchmarks/perf-tool/dataset/data-nested.hdf5 b/benchmarks/perf-tool/dataset/data-nested.hdf5 deleted file mode 100644 index 4223d72810785f11ba801ac0fe819b0a72d0ada6..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 74496 zcmeFY2T&DF+bv2EK@cR01OX8cQBV+3VfR|(3?c#w0+JO_5haR%iWzgxVnWP{m=lU% zz#K7WF=xf>neThw_f&me^^-Fs(O?Vj%G-Lt2s`|0OdYxNXgPj7WqLshwd z9V#kv&Ez!yefrPWpT7^e_VWKO`k(o~E5!Ys{&iJOZt~yx3WdK;PdO*E4Sa`$32Qoxaom zXXO1=kd-_#>tAPCBjQtsNojmm)`--EY^nIK>;Kg5`d9XU`zplhap`(=Yz_xt!Fx%D=BGDE!r+%HK0Z zQC_<6*J1E?`gfA|pV|MzxxT@@J>}(^|Ihh6x%q$p2mb0UIk{@xe|r9ZUElTZ;tP8J zeZBsF*Sp{PzpwXC=b0(~ukZIy=|Ar+PwAhM|1;_TZd~>M+kXD@|Nd+K|Him#_xBSn z_|GT(e|B8`U+s^-mjStd`s1Ix;@|T~M;aFYTFm~Nd9~-S=dOC!p3RXb6qz>mjBx7i!LwogsAV(;0|VN#wrd~* zIwr5Eh2o<~q2TP!^DSG@N+plkA0ERfx|7R!?-#Ns0sc(vH<*JvZ^Py# z#++J_C<|#G!eFlt80ut?6o|AjMukBvejub#`iQ~ zXk`ZHY;)lrlT5alQ6(0BRTH_N4&z0UJcmqpFLExwf_%U|+0AJIjI9}r4jM+NmABzI zh333;`5x}Q^QX$^K9E1;LM6v25#RAUTGSkNX-Miq_tWJ#?>-y#j*TwE2DIbptGlow zZ73qvhEe->$kAbrT(L+i-kcF~=Qu7qSo9{B}^D z!nOuew^l>p;t{me?aC|lU3h%v3lXFl$8R6Huyf-|^mLeoG|z0RbgdQEo!ZjlaVjgL z$u8qRiPPWmu=%SS6ZHo1;z&bkZZhL*tJ5MgyAo+rM{~iGI=nlZz+oQwTzGplBMiJa zy2PHrecQ2Yss)md#L4`U(zqxnhmC8vO5O(A8d2-4$O`yO1IWR15MSHQSTA5+pj zXu7wUx^;RyS-M8#PnrywK`*{3ZHEF~LoQmi4o7=D$J!?f=rUq9mO35A`{lQwme&=J zE}OI8{5N7szsWe38qPs;y;ztSf_-l_S)b^^k9XcWx9psaLm{oWWQz-r1O#H|iZ{r8 zI@&BC+IY$1s&^xxoAfb+eY-{k_kJ|Hm(C){Q{UA z^I7<(1fpH_6IsDWdwzPD&b!+mVosk6&|6TB@(rOJJ?1+`pGd*lBtI^EKN6<%b8z`h zARi_>aG1F*Efohd?usto_K)JM8N+Ck+>YKyZp%KT_aIudz(uKi{iIfmt=C|mMthDt z+KH=GqNvn4jVXOoxYi+n{r`;M2N$4eQ7v4SoI+yN5J>rICt5Wcu=mv= zB6rXSF<^jxeQ>uKy^`m=&?s{5tlVof1zPGKA90aZtOd!Qh28ID9UP-!)dE{iW-e{wJ2LjB;^g z#4{K;(vHJUq4B#MP=9r4>Jhy4~9qPED6Uz9K6;@uE>*JLv2 zZ52#5XmHo@7{08NUH9gRN~TDZ0NJtj{|WxEtDGvaWZ?tk-I*r* z7Y}5#LoZ%bsz9*UMXdbZkJZNG{4Trx z%9po=EsvMmiwN0poEW1`GmCFxtByI3sw>0YD2*pu4;3SHqj1P-2nWVRa^B?@d=gr~ z<&|UMA*~&Ur4`^{rXYf z*Ht_XTPqCmD`i1tHdyhtE3b_RrG5D=SXw!=MN~Q`Z?R|DZ3C_?_d@CuWo}<@!@h?G z(Bow=f4sHEmVJS=dz^qW<%ignyc%ga-I;7Vn&U$*;Mdv$?rbU(3jX(G;>VEimt77xZGO-#dG4=>w6W%Wp&PWynv)oQLibqypmYCaFq1^yK>}m zcV6q5En4&}=J+qkuzy~Uep?*b-oFUHX3j*DsWsJS?2wf`{fR@{1t0j>h^5lpZ(Jl# zn^%VD+u4S;Uwqi!tPAyV35_Ywa3MZNR4L}s!lDZD=XBuaW5MAwqPg`HIL>ZAtR8h` z#+%WsHSiL#T8g}Ty#b|#ew+}x4GrH6**8v(rS(%0v8FvnH&0oz8H2IGp)hwZkv%%2z*%#PxF&c%GKZL>qH-!+ zmJH#t@wMo=(TD{ihbFdZOtg2QnR}y1s8(Xb=7l1Az+f7rgm8y>H<7x=mTJn;oZRah znkCIe*vuHNC`zQF{W)>Ta0VuRkH=Wg!PI;D4i6fJP<3oO29^Y))lV;OJ~tPMnFr7% zOM#y)RtlpnePMsanZ0}GGVy>bb2V(Kv_A%!+kG*x)nVDHh^t6BFdu717W2KwWSDNw z#vOAbj`hi53xhXchXOXM*#h;@_i%gag&8kPu}n3QuKj9obDo80RW+IwD=%YX;dBiB zQ->+LoM_q zOKF|Va@H&)-R;H)ixgOM`xcH|)S`BE9s2(u@4xvV(^+1?=l(?;)$=O$Y>46)4KMaO zn!=S;Eot;3kw@E)rf?oE)S5={WrsSHd=C&uGj2j*!5esL^oPP*J${yal(-+JypnFn zQGRL2PM(Ppv-R+uD8tVDyBM>jHv=MPi_l&Dxas+19KEZ}2RGJY!>hiWoi;|CaB0tx zhh|}>_CTHrQ4=Z$idpi#6`Rz`;JChnXjW0gU(Tf%k(N!Ld5Kha2;$^HP8?9&gF9R9 zLwYB&^70!Ly0*oH&!_Nboj?40nWNac89Vna#qN?o=4czjXGtb^n5A*Mw>fnO7gD~r z2YU7RjW&}8^6jE*Mh-uP*;D$npH4eo|LVnd1K;7N=N)M0_Tp|a9a)ifIBDyMX&C{0 z+{T{1HJTWwP$s&)3t{Wv2%b!A5XPt4^I_FEJfG;mY&8w+w(KHicX*DgVkrwQIXD)Vp zJtc4Yg`uj=zy!4G_G zaS(IT^0|F(G?)2|n}XO*Ev72 zzwLMER~yp$$!@V?c@kG_ca+9^2>%?tj;N6HFxDEtW?qGyWPSm2&Vds#PwX`4$ml`e zF?VA#YJKm*gBhdPB{3T%13joVNgt1+jhMQ26!zrhQr=?(@6Tz;jbqz##rjLaQzwgl z!85R@dI>D_&*Mpo9A!OL;D&WI=6*ec-IA|1c6c{zbsQ?1cU*!|7VpvYxC8yT6r-<= zNI{nK2YsW~Dl+BoqAkp|R^s)hzKj!c ztRAh$-=6(B=zcL(kLT0l+9)b5j)c{aFwUMC$mogo)IXuiS9{lp^Mg*{i*79zp4or{ z!-&Y~&!9Ijoa6hw#*8he5ZA39p~l0xMOxQqHPzvQ)+pW`@5U$vQ${U*hmyCJ7<}v_ z<}S%&p8?5Y*smD+xensIaBn&cOQfybQ8-29BEjN{i+ zjWDWtDO+6Ci75|t=;atKsNuw2$6jFk*SV4%Y=E-51=C9W7&7bwv_?O`ZaGWd88;2e zLEGSQ&x;Q3yU_RRd93lW=EG$@Xcj-1%8DIXwsHUtJ(G#SYNud$u@qL)97=7kjbUou zE>E+o5ZM(XJkCqDdXO4j!!o#d!68IEa%7cB3obb^9@C}%-g;RcQ%2`gMCuEb*ILrp zGi8$8PrO3btI+dp@aoOIZ=+>+! zHXf1D`S@A!=z2O|gys?#-ij?N{du#xJKxv^v8P;1o^IvLdq$4zxvn*@8FuFxF2>U( z(JT&Vhm3P4U{)Q5*+HH8D8?V=ZY?=>UJ_M)d2&!|4JPaO^OpZ?IPG36I_93lo8|+k zq?yk?_3~_$*^FL|9(0MlE>!!QaZaH(w{5xy<)xibY9z;)ULSDr&|cKb%@cROJEDDj z4ScpQM%EgCo~jlwZ?zFa&ic^x%TyF@+l7#~YKTq`M`C0L7Q2kZM=7R0IWLG6j}AiH zBUtE8{2(jp-kBA?ZxNi{3f*7(@xsnX_8XK)Y<_@4()^2kZi}^l9^-utc-=QpwyeP& zCnprs{FpKq>}bnz26ou5wM68MHRP|$v#?svh>n(hG3@km@z!IlSUl+_UisOua@JUU zcbW&yXWumA~{fIeb z?-5=%15f8Eq2&V?E-r5mwN(YUp3oVer$o`}k{)X$n-W!a8lAQ~&}+RRr?0d_-@_T4 zazw&h>_d4^PKW7te!%9X65sZ?i~YM7pkI;;=I}T6tAh7Zr-*qKFOd_MPOWQ7SfYMG z{Bk@2)!OzLzeS5DYhTM&MfSa-&t*cCqw+G$*ayhL)$} z#m4qmU?atw=6H<5y`?$fs=-qZ+3dCZo3NO2Pu6>V7DqlT!Ru-j9&2e0`wtFmYB|aI z)k1SPHY`T`?%ybKvg4I!zT!*UfiThN&Ff3Enf6eh3p0P9ZM+HmHCBl7TT+aXSq1N7 zgZXotKBqanh3VEoToKj`pUk^qd*f+ryYmuyUuB5C6Z2=p~ZlCxRUA5`*P9LYA0woX$IzqRk#X& zYLE2dx0>cO2utAzdwKqPZ_cG79c5cij28)=4q@?ESB&u(&65jdRMU>;>vnxOr7>Q( z<<5amO&sTKbl`HQ57=SR4w*$S!~}xiF;>2ns%e2_#RudK(D02Qp$YhhRplQAzPvyk$=*&(?8qkMttlwbHfneSpqr+#KyJY=? zoFMyrR_q(ymFE88R6BeXJ8Z$BTW<<&ol;rc>dUB2Z-@6qM*JQ15J`95U{vuej9;-s zw)A%ZTa@QwMS=_K^%sb9ty5(t-(6^-*IKgOO_=aBh2C+od_Eyh+&gp?yDw_equ2^g zir3+MMUN|DAHrsDN3OM5hx(iVE-un!S36JUX6tf*xjREX5kGIrp-*WRzEpjcFm`!5 zYM$yL`m%D=7wnHI;OA~*WfN*kMZ2s0xb>(q)E91s;cFL;3T;V`hqi40 z{3V=|>(IUNozR~CQQVk+2Mboo7<_Okrreq*i%sn=-re(KRC7lvrN;6_Lkdm%?tz(v zg&b3`#H+rJ+`Dj~p!fX$7y}8SL1?lLjR+zU=x`_&H9&lV!Vbysm(4*Gh5HJtraNHDh^~ z5`38)K=sF6c=)0@?KJ|}?1wxL7f!{Nc}BdwXgf|9R$_WYCeyWh^5W&@ur{z`Mp`t7 zkG+E67BSRLv*rE+seJr4n|H4I(k`?CT}oVKvXWI8@mUWQwR&tmra#Imb1-w&J%oMC zM`-U%4(#c~wpL-h)}+T-QT7}auFKi;pF*arLb=6B%zr8&@E<;}lus&vS^cHDZ}T z1~s|_(QBsU$L9EQ*7g({&j{nq^>c7OyEB5dJ9Az7I#JqS2ji15Tm@e~He3vcr-H|y zmBMk{5ISG^j&-dUqI;wR$DDA6l4&OUT^T3dd~QRf$~<28Fr~|yg?QK8PP`624SBg? zTvDRL8y?L#D%+pmAG(SD!)}XS!+&B%mJW}d2w;iERWZ2RBaz(Amby!aa9ijcq};oL z2TDmepsCA4e^fYQ?>RhSD&zGN*zm-j-BMMlwV)3(yiXw}XFD1Ux^b0~DYTlrXf`YX z>y}$^{>vh+b@`1g2DXfJ+=iK}vqY4Z1JWl2aKS}=CT(_R`AbWd1Zv>;($i>e;L08g zw?gGg2kK`uU`#@LM2K9(XY59gWeuphG@O$bti>wHU;gQH6`JLhqA+x?aC~mVh;vfB z`}zsyT<=P~4f1^PY&FgY2}Epe%d%4vj~CsMYGWe!@Y`y!H#VM*!-wCU}DoBVY{R+H}2D>&520f(;3D~ z$!T0OJA^-;e}VNaX+3(<9lER9@wmi4ym$saIcmsr_31Pk=gLUUwIbK-qv$%iuh`k4 z7XozFWBA$>);xFS>8RFh+PnpO?rg!&`u==;Es%3ua(GKunKya_^V2D9EHK(F8yI{- zT*&OgYeRlv{22wjS(?vbPU{gW{{X)mtfknr6N8Ulz|`fYV!8fIY`)QtImMcMJ@gks z%=+`pt9r>C=&gv(GY*LZ8wYn|$LVG{-R^{{I7gbJu6+n0Q=aA3u z%a$@Zerat?s~78_yx*5s%Y*1J-bwcG)-h~#uNOH(wfSLr8kbJZVCwddIPgx37qX3+ zsqzZ*uTI3cqH@4+6mOfBptElczQ=}BOMfK0Z4ReXsV}=}8Z&8hAcNOrik>qji-gQD zI&9WrPisf6*dHW&F-eOCyF(eSk`31yJ>HjiJMl7+XTNu)cLxK0+M7h*xsIIBs|0cF zGw8c=EE;aU!*Bl-E-R1Z>m6RqeLsjfMML=g;!@-&IItncgXR1C^WhCO?CT}Z&WcSk zmrX~-L!G7Qw``qwuA;z>{d2iyha%Ve?GXyuj(nL{Cp<>9=EFtGG^!elwomQE(16kG zI_WO%z8Z@*i=8<^>j|oQ>_+LdbgpnrWWqoNRx}M@_!b*>t_)`T@va=a>;^iY>x7V$ zV9uC%P4qaF#xCvWA=*rV_Q&kl+s2&cZmC@B-jO3SuE}&fjd92I6P9&6iDw6L_$MQt zvqoHm%4!9Ea__^DCWARpeI0ZUHDDk%A;>+I4@=e~sjLRYWtWB7*)-~E-Nd_gmeo-Pm{b=@`7;*m9IzEpak)bOW)M{+jwSbv9Vh-?1+~@coMM^m za%ZDGBMNo-yvmE+MFX5qHo^9;7cV6Jz^7;VoG0^Ok#or{?sXrpOB>8MHS$i?2rK@NHQ)Uh8`V zuTQT)xk@*#`BWhr^uZDRUk|6lML)K*d5(cyx8TLp!OTvJ;j{2>aFh7Z#<>G%+a`}! zrUbFiX&E0l1yIsRaqjecNHvS1^^ZB?cB|fa`plSmofPpsN(mM_h9fR)G|u(ADSr8_ z#qG5-p#IE^#zRe+EBRqfwGGg*t;Y6s&qdm<0=ih7f_D98{Mj7JVRICj{<n-fpO1*3WqmU`9v#kC$A{8zn2)$ReI5=(+{fz2@syMNoU>nxsAuHI zFZ0UqwaL&qa;zn$?}&l$D8|;|^IMfGsQ-CN9FqCksntvSCej=3&cbZ{}^*|A52x0xooywYb)c?OL=_KDwhkHucE6yDp? zih~|ZL_?b*Iz>0hD+h=Y3`@pRKbgY4>(60T(I6;g4xwZJ-YiOx zNxs1ovB9`C^+&abwKOfq_8H1ET6we@-%E&XPCWLm6~kQ_PO7vz_Z~yE4LN#N68HL@$IkKxIA7UUVK_~8gT^^x={g<#_e%4D{c%5e2aa;XWxu;sHV#JXnhspK5UKS3L%O7PRV{i#3m|MExIiUbK;X zo_8_O)=lQ#Tnny=y#v?gdDK0t%0)g;QQ0$?fh*dfYK|sTyu-Qt6`1Ntmd`nlOO{g1 zt!BvLjz`gIeqVNp5sY8cisqL`BJuN86xld&YOWHo^jQX?53|b1t__Y$ts&~R+!CrV)N%(I`Dt`~l z;ItuIVQtrlGY>0Z^!=ewTxZ3}d*=v`hnw(T=>b~wyoR4m2J}jZq5sPdXh^n`t-cfj z`*}9lw=s(WgTLV2MLQl|1!`rQV?w7exP8fCdP5G^tNC(CxC!pMjpoa~nH*sq#_o?5 zxIKIgRLm9esk|I#vc4jC&M>wMms zIa>r8^ZMv?{@Is+fD|X%9ME94We82(y}1)+{Q5?f^VH3dK13N_Go$g#cqkXU4rapB z7*Y9VKE4+e&~3tN%-oR0j?P-LPVaQF^Vn1*Sr+lGZ(Gjx%VVooZ(Me5=nj9?ew<*o z6#DY!nC0|Zn0332v26{x@zGmZQ$-M`K8#^~O$c?{Ji(CFCY0+MM>7W{PLSqmhDr)A z>wbZfP^4;*KlRJCIqQ6_n7(y7YV>-dg=8-Z_myGXT?&oIR3v)-8bd2w$2E-6gDiBz`Kxg-5Slf}Pt< zSUvG&lhZfZu*E7Ivo%N9MZ2?n>qwMK8U>YC?wH;Et;o_@Ay(WSi(j)PJbcJ)c-`A6 z>%00k4&F`UgraO-ZRUrg6{d9Wz8ACi+QVYMF{hoYL$NFo5qn=?_Z1Vy)m%ZlPt9q& zz6VE+lQH+66%BvXV0HEzoO~_$Ul-CuhsQb$th8ab%Mn}`6fJ4GzKi~z3&f@DS8y9B z>3<&2l+E$9M3>F?@pQ`?5hP(*Iq5D?2w#G8Z}fP3!8k;o=tsF9H)Xx_Kj2-1B?Got zF?W0tU3p7nI)N|c454&5oCA6XvHqJ8{Y<$=j8DfE&n;7_x@T)Dpu z*H_IG;}h@TxtX-Cn#Un(qcn~}wqeyZXDlh1iJ8@x@z!XqOUo-}OrCNE*~4NOQ*{oS z!8M|Baeu~3eG%{7hu!Vh+(=+pRPgc&(WNDz=R%kBY8MBl~tMRS_GX_R_pr#i zKx(ruqn*vMR@;C+4z=Q{pAPp79!=AcVT|uE5WU-};#JHz44@Iyx7xAOyjJ|Zb^?m` z^yIc(50IPo1|P3wvt~+XgzhlJh3PS@G#$-uU*l+5-Y%CUN`6L&fkzO`S{}k zOCT4Y&4>{;w6-0>aXWXSF6o7sw4xI8nml=Isya3oP81&xw&d*x6J)E~zk^o$-6)&* z3}Y+0N}Pa%S?4!lmAVELtB-+iB~7Z`K%NZQfYs4)bPtc`_wR=hcCsfPY3Jhd21DAF zTtP|6XM}5d(eHL!TDLQ0P2biG_Wgq=~W*;IZm_d)PtZ?H0>4{|un9gg=L9 zd5V?01P4|3p-Z#}n)8}xA1zs{3#xn9_9I8*NgN=xl% zUZV$(mTG)Dt_=;hEP>Cq42F$#WaISDcsigOMUUI#gMTPmJJn#yoL{m%D*l|_!-oA0 zcZ*3uBUyGi5^p~A;c%-C?Cqz|l*(lhow%=+ zC68uBvia9%FbP-S@Y%bOI&i0OF7e`k4?FSnNE)}b8;8|vf?$)KjH?A_#nT~{T=T75 zl)v7OXU$_NFZtqOlFoQh>n!R`>d!0dlvpg=jPW|n`CxuDXB*Xv;B*zPvD6jjr#o`& zmqhkb`i7w&lJLDWlK$JHnBTY=2ZMj$!z&LISUI4MSE15L7r);)L+?c)WA61uXT5iL zuN%Z3HSbZi_!2@DYti=WN(|mLQ}~{lfJ?I)usv=xlJ@CvTeF1-(e$Oo)FN639fr#b z2NsHShSqAc;gSii9eIswae)j`-zY?{-7s1{n6pX#PfYFF zf=(+78MIZChmS>xn)?fouX<8^e%piYRzuNp@(TU(7@5>{c7NPsaWwNnL<|16GGkl2; z6-xp|M|&k+Dplpi&-VD_cMGqryEAR0GSAYVN5a9rdr9vxETwGvqvK+AmguN5I;9F9|9}UM`@?vm0SDw2i(Lwmc5{X% zLVibxX3vN7_nk{EcH@=m$QLoqGO`tBvZc*lJ^^SZh#ke!a z-o)vh_hokTM(|*094G9tL26aCm^;Or#gC7Q6EB|1B95;Ru1zU;@@gASE;%4^gF0+) zs)G%qThpwA9X93N5u5j&LWq4Ol-{hu2a^Imy}S&usd=2<<}|iQxbGcPZElk=A*bR3 zzKtvt@oOS*ZM6*!+YMpZ{WDNHYJhJuM&PP>rR=A@5~d&QN0<(zU(ROe{+uAXjZ=o| zYdKyp@4=+ogSoJwH}AFy=9v8^tbZ~SDW?l~-rI}Ea`ULIVSw}J-Kml7&H>7iG&Ng@ zp|w3}x-yPMW$W?7>WFwUbPbB5?_p_t5Z|YoXL1r}BvI&^vlWTw9?0Pw%pEV(0KMDrA_l-mx=VKRzRDZ(4GsnE{8q zmhQFK1I`kEuID@%-o0zFx^9G|VUlnN#SS>AJ{s>YxzJ>N3r?Bco`sUnw)H?B6(lUX z&7heQznjCt)O5~tXo=fq&>ZZ+%^Zsl%$A+n&dr)~_9o|ng#axf2 zV);NbTH8#6dqp`;UK!4`S#q=tZNoKs-8pws5$|s-B#MqlB(Zy2Y##foqY)DU>f7+WfD~wrs%L^4fM{tzR z1hK-S1+HFFxRBx@q^+Chv{4QO*p8rA^Uv3n$2g%d!@@z5p>lQp19nB3| z3A}o3GfI24V$P)IoZmYWE^!%RSk4oqZPP;g$GMzZ(uDOBa;a=E3Az{F38T+NJh|~6 z^7Rj4Gmqm`_zS$1cyxzFWzxDMY5)4UG2Uq@Zg;3cr~V35zr9$@oa4@C$H&XEZidtU z`U#OYxIaq^t8rBOtVox%f|nXKX&a}&w_43<|63hX8Y1|>a47PnK3z8SJ63;DWcx$0 zJS{gB+bwrtQHcYM&4SsttQ4;*N8w0cL7ff{@UVCxeB5F=c}lei+b{$3ZXR}7yJ|Il z={aJ?Y!^maRN~L$&p5F3IBuV-#k&bl@bT1Hlq^i*>ZNU1Z|BVz?R4lFgfaSMFx1Cb zvkl&g>Ybm(R5K-(BzNJ|V=Blvo6Ambui)`~IsWXG#!;_U<7c!f#E0JEXv1Z=%|0S* zzct8|rgf3dc@iyS1e5f5>>XPp5+_Lea(Ycb zX;d6LD((?#5sB=)!jNeyUa+Y9< zoyApqQ0wZ#`7y=3+rA3T)CWtLcS{79crf+F6!?uc=8(dHOdT^9!JG1B_L?O)(Ort; zvVe$*g3*(_0(s9wFXmsgLtDfhdF1QIotjl?tUnjIiz?~<(M*L zYj?n`;Q`cb|JO=vXyGen4jIfbIYmr$RKUCP zIDWY)>ES2#gLb3DGis!A{LLtI``Upiy%Mo!T^8)CA~|b!1;P)G;~-Ppv1Y<*lb80`u6iWd3RQ_;iT7l|26tR*y8y1CFND<;1>V`#oADRg^6vcKxM*}ijEu5F z`qeUw4Ul7k{(7v`P=<+XEEOc3fRAlEo{4w>+3dc&p6e*>t?kOKl>>Qpk2@|?TE}}U z!TnewH6nM3!B14Uciwq8B^qFzPX;e9E^#)rlKf27U-+6k4wY(T*Tg9HF$!l>sU3V~ zNc-gcCyEvOtD&?inmgJj<89b3wDpOgh2;p|8ks_Eg`w=KFbfaYyD~61n~zFu*-V<&v$K7&ukTN$X3qb3t@nR>*O>!};xbJ}>W^ zi~&Onx!XUVkJgW%!yE?|t~PpwPTY7})3` z)=r)84b7kMqk0@$D-EUOaFOqYa7? zdip(Xe_0Bnsllvz6pC${cg$Bi5bl1P6uA7_uvyJ*KH* z^~v)noU}}~Qqrr&oztaH!4BA1`tU%2l~9oI4mbZb_!_thg_g<8TU;sIZnYWZ2REXm zn+NYdj>n_yW+-%4!aet&5NALBd(YpT9lxNudAls{{02n7^yak!U7mT=mN|bGpwd*1 zPX>0ONkTs^moWdQKKT}0qbgt~X%Fre6|;1R1uX_jai)?R2c>^S+q&iA zW7Q|ryqtq}GxfRoWh{plER(HWlgo_Tdi-@GfdTRhFlJ*YcYl70!H$8no;Hk~I{5N{ zx`aFR^Q7siUbKk23dhiwcrh$VazFY%z5tuNBBuMlg7JDkG?*udq_00Q zY`zjz3#9!}lGgUrxreasn#s~{HE1Z@fM>}&gzdE`Y5g8b%Pe{Ep)81!2o)O_45QBE zJJ3`fi^Z{nu`;S0Uz{7E-QEV%KKG&4TV?iK5zltpwqQe19JeO@LHCYTSkTd#m%kWs zr>;HApBzX0@tLqQ+^jW@FM$ zX^+T;0dy_1=KrARyaTCz|0rHogv=s)res7R+~=G`8WcrRl+lop6_G@P_FhUUN>h7% zwfEk8Ylt+p_xL@(|EsHeug~*-&ikC#Ss5K5BF4y!&h&YrNp@e~x_$Ya>8!#HHojDw zco~KdtI@^6krTv2JV-i+v=&;TqpmT+vnPvHPv!e-_YDqb)VOkGKI3e4cxG}}<}Ju# ztIdNst5Jzx*Z6S5xhBlgn}}(j!a4845Js>y)#oW=L~eUdbIZXbIm@Mon_{71v3QVj z1}0}!Y43IciKVT@jR2tePakYQtH*Dj%sBt?I0o7j3Lon=Sllv*bN5S~SNi&cWuENj zVyUnEI3Su%4`)aB5Ki0m2ve>NV)wN{n6lCyeFhKb=b$?5v@eJ1-atHUG=Z7vZwwEK zW0k)Jcf2UhWX&Zi?x)Rm<^)Tf3V<@k;<1+IU{<3n% z`85M+opc%pFXUZ(T&vcNa|j7{u+Xq@QHXE`@`9ogd$==&+=K23wji z;`%hn5OCz()45z|+LWEkPT^L;VceZ-1l#))k#e{NR3>?I@zkMQdN+ncHV@~2x*mMp z+?uMt;^As_5m6ao2tM757q6?rN@YCSMXIvz*(9HzH_nL>lQh|~m5F%QtWgpAU z%;&^asm!Q4iiQ~xOg^Lu`}b++yxyK8W5)Z|dPnoebq79M-;Q^mcH@z@&7_xqGA5a} z!{IJ>u+c*aAL2?-;+;*c`)&DKC7lt!g4q4zNX}^+su(M?a2*c0<4&K^yu8;6alv|g zdq{BEH#;70nyK(g=*8|J(A9M5RJGB)vs{65%aUUej4}{sMbPi5!fs%p2 z^y#X~2U4r*q_bOQQD0zMttsd4+k#`G+wy^CI2V4m=k$TkMWCyKGcp62H6@P5Nk?GQ zKaVPU&dfY|3u&4@bUS($TgO?-4CP|+v$`dxZ`9zVCx21c+XMB3U%~pUCi`s*#_Lms zd}GFDwui+F|CzpQKClN*6fO`k zkDGI+d`>Se*)1*w$eFS-lEW@8!#mp%Od1v{bY0hA={7UmU8l^OTiy6*h&r6Q_^}~! zJ8rIa$HmwGVDGR5Hs8Woe^{9X0ZJ@5>??g10jym$kh744fDgee4zQv6m^@BfcMgwF zbYV%E%(6Jn6!R`#7b~L_9DceBKZbNe<(6bj*w&X{3SAVVRxQS+ZV!dSBTL?2G@KWY zjYE+|bH2#w3{&Ny9Co7aPX+oiw%Q-K*fOJBdwp?zYvVG%ZG4CXZ(W8N~$#ub(EoLA8oSIlzwZ)XKM z&dCz*J#C>~=gmB&{U{tY62?tuqvrHkSdG{z3>0}BxvW@doX_A|eRtMujpSLKzC1iX zhdFMx=(6PuA`1Fo?w5QnH8baU*JOD%1!8cQBTyX>PMh*>Jm7L0eSR8p=15ImygC&I zr>2UswJ)K&Gl*VJFVIQyBig1+LdIX2;c6=Lbz%J((yLtIaoQf9vztOEbToGU(4um$ zXdZoS$gulo#hS?XzB2}!%8c0vrrCeS=v#Nh#Fu$2JWcB8Tk$~lI;^>)&Upj>LZiEp za4CC>kRvI0d$1#ZZXP0c&^o?AEhNj;T^!PCX zxmH7h`uYcIPbj~ zchr4F^ypYdYFtCt9+DNY!-eO{K4R|)xvS90=a}K2<$c|SD=Qyk!Luw}Egs7-9T)zP zo`D`?PvN`acwXIe4^1A^d7NMoe&LtM5;7XZ%bIl$!D`;Jkb5X zXj;{$@V@0YJg*o=9ZPk#-WEfhTDi-Oo(JotJ^0~mA(}m1ASUG>L326J&bp?;!eK-3 zY{+uqTDeTVkAwJKu?r)*yHnYF51M5E#X+51Ha^RsTKHRRoNde&QCDHz(TBg26FF+d zYfL;RcMcvhgT6hHK9gpm*X0NVYgdY*sd;F7_9@zF{6mJ!gW6BdVYh3)eNWuV;fZTt z?9yq9tWl3(f3r`?AU@KLrYNh z>L|YNQ{sSTomnV#-gCht#MYZ#eDAxRzzMe_7;ya`^3ITN_LpG%#}t|bnKP=BJs&7! z2IqA(X4jA7uR|)lAhQ%9*(O|3dmam$3};4e4`wDt(DaltgNMZOscKg?&XJkQV`)sX zw5N4eD%TZ&)#sz(R%FBx*KPQFQv%)A7mHt8CgINQKpySq!vjubSUBYicFT;n<36dA z)K{UU!#d1b{|^&>MB(^42VNR-0H#`YT)QBahy1$m^^^Db;^NA>b3;(5u7VLi61m6J zhLPTXFw&zn+s&_n>jMwo%rD2CynNa#9fn`#LRMRosCN z4o2MJyc-APz3JjI4frpq7Qc$(=%RfUdfG9*iT5fHq@%{#ZH>?jIe}*HO;GgKnen&% znYD5aUOo@zqe1x`+rvfXlhSB+L+0e#tBI7lAtK~0c(Yjk-v099=Mb3@k$1EWGdJUP zO*|Lek^h&0v(R?_ExbK4S6JCOqx4Tt_K4~tuJ+ssjlW~XXP;qgJNvRwZpMK}TlTMpYUJ!{J`6+CB$MZ~hdQE9W8Xt1Vmp zY0dlZ(z$l-YsGiH1Xf5ddR=KdRt)dJ4!*u(u&p5u$$dwqK_8hNY|dXNPeDz7ZdH0M zQQoIJt;Sez;?-DoEbYV&ooqN$w+&|u{)?)^iL|-B1O1-(FuUh)8b!6nx=#zmgdh8b z=3iTmT5=gRE#+<}wE(e>n%rSFfG4F7{DZ|mMBGzD^~cFRLmp3u?VCO%>UuTYYC%indiUyV5I# zCo(=EZ=K|oJ&57$*+ckM&xCQopI~q6$GeYW+2N=+7defk%KdbjWIPlhF)>uzuw95F z)1b7d3rfGKOFiQ#mbLa^+L)0TK2GlQjk-g*pEIL8Y(B?I_nNTF;7s1Ac!6JUhjQ}oWGGch590H#*c+J6K{|%;?U5@Pc2di~WXzfXWnN54 zqlkL(rfWJkgl-Y;l3nsP zGL?JUMoDkRNLoeCfur4MF5W#GJBsg$x%GC8wAzIuvu;DbpfzI#ZbFPL8SD`&+S~Tx zh%e#Hkh#y!4GZw~oijImK8aSz?~pvn3>%$QnfALaf4!Lrl0r!t(#LMzay)g{SaL&FI1Sdc=8MlU>=%&12?2h>rCSB;W0s+;S1XPTUxOjblW`;_ zh(}td@Th^zxP`WpGq$`BO>E7@E;8e~(oU2*eZs|)P1t{xC9f{pg7%bYO`&EM_o zJ|&GW-1g#QwDkW>nB|_N z%>p}1G2jIm+dZDzp--`}z5==3CZhkR<{Z_|p7_&_e_+5~gy zwKYG~gNwWL=RB$XT|Z#XumWYd*It0JQFmaVb{}`!$=B*Q97_({@n7|Lc2A7sfJ?hk zbHx*LcX_~JWeShQh4FdqU|MJDGG|i}dQX{+RUMl1WnL@Z9e)RJyUN}A+d(|#z8Lv0 z?urrV(qDh$1eBWnMtRX+(e<_o#}-XPpQajUdu26Zrh9O}ejhAbH;(oi&k!;CqP&McAshuaZ)xYE!c+dIC+vKKv=t>}pU*OV9~Gd~-n3b5?@HI&=wqJzx_ zJn@}~u+ufT7#+{nwsGY9Y^Lmef!m5jqOuG;{(KOpre4BFHFIuUN(SuCW5Cw&TrWSr zK93vGJM6ER5U?20)tb^T@Kfe7FXLD7Yb=pjkzrRHVIUbP+s3D(sa-2h`WS%GcFMeE zSR)Jv74U+lrrgb~RCGUO!kTB#6wwEo(_wf%-6dW@ZV>`gm!w8|Pq80D9^cA|>f-uNBlYuoq*2`{-71d|aC)XVDc^z3<^bs3ZwBX7K8CcZ*AnH1N z5u?{j&!*!HRJJhV;zhHtTWuiM@A!-qt>>7dHyhSR{CVE91s2-G(AHmzUe20aWMRwd zIR~+Jrys|~4dEN%P1mK{(EglcrfKH!y42h}v%9e*!Bw(=H0h}H1fO*W@sV;AJ$L!> zciSI|nnkgaarqb)m68{!bxHWVFUNK3zDV~B5uK~9VM&Q6vre4Dbp0~4c_+Q43zFns zH-%$Zi+|@!Q6>Ga+grBgu$eEg#-u+NA+fasX zl>7Co2l3l>6r1nu%fRM)pg(N~ruV2pf<*#`*dBsoAL-Am_vP@Y=f1C{zp2h4iM_4b z(K|nqOOIT`>ivyCt9#-{yLNce(uy0>jp%i8x;U9#ihq0EG3WSS;gOKb<8nW-M*7!x zWP~ZM%Und`)uV8czOVq_;Y|^ zBii;TVA9W64wlc}%FPxm3DZNC%=3;amEME$>-aV29V*f#)7f(};y<Xuv+Oq=vdx`uG%RyKK4?S8YxQ-fMiUpyp6AOKjNPBcO~9>0G0Y(zU6NW zU}6%>8iy&Oer^XE>m@SQ`R+l?oCPe!raul}^&i3cZiMeMql;^`)1(em+6XusTs zk9o;dTW!tXefMC6oCW7h>%}fz3}~n^A8+H_xxM8q1itCd`1!9E-_3bC#THF`i*2z48>2b`Peg!3Y>)oNDsd?iudRa0Z6*HIIR=OJlA|~wn$4qz)9_oJaBWkBc0)bj zckVij=3hb2%@dI@XgOSW`l2ByO1|zzyl!;o{S_bZc*Zp}i=Bz6(!pHkpUFSJ3^Dvn zGwN>{AvVo%BmYjon$DR~lW-6w! zr@Y(uVz1^tG?3nW?aUIm4j)KGwTbvR<3Ege7luLO2XgO}Xoh$j@cVzYQ0?E5i!&W~ zaJ$UHwAm|j9{Ma)5h7nK5bkG!S@!EM%D?_Vn9kED2!op;@?JJqwjLoA_>9 zBl#hxqBz<|YG>~}SXO?{H@?RZ8jbMh^8dU=f0N( zQ~EwQF7*5)L*ctAbu~tcmCao7Vbg!;^SA*9CL6@B_6|IO4On%06d#XGlV?I6+ivf} z68$ft$#M^tzkZ0~7ZEgj6d}DnR@ApxffaUx7;L0Ux2eW7zIRG|-mS`z4P}Tf31RDh z1^5)>NaOxeM^^Dc=*&@!jFY+f`-W_?s0Di;%j0cRS01pCS+`x}M1v?g`kCO^KwG+P zYQZ(@B-bU_9_`fBSf48QD9@yppnXWxeskfO;E!TLdNll{-gfSkAMW^{^R-rQ$HvO% zxYO>o%#sqr#Zl}Z)tOtXJK*ByoA7D=7>3=hVp3TedripY38f3Z_P_Gz8Q+cDzxRUP za3iKaRp51$mxHEEB^Q=LhEBi3@SqR?@kZce| zC}V2B;n=717<-@uBk%OU&xAJIaYVBERt#e6mzyzp`Y~kJIWiRknAU!=7-f~rH76=D zcAyU)Ta4jpujvT8*(jOShWK=JC4!yFjekzy#k>dTH?S#p;;HBrpU&xe`Skwt9lDK; zShwUin)Q1wj;C!CE6w{tDb|$hVg@#t#tEY6Fgu9b?N^Br;3-hjWo{zL32b8w*a!zJ{9PDCU8vif1y z@+)H5;Nv2wej83^=Ch#AL)>oAf^jFAe~XM3O|%vv-z$q&TV{yyfuAs?+MUO@N3;LE z>rm`=#;WXHmw|DBH&!=o^zAeMiy%A`A$CQz)Ls)oq3);)wceg-I z{`Xpmaid*nn&`_*F@>}bYJ+q83TSgMSZtPjd)=4j)IOWX4b3*fF|N0`Jhm-QM_Kah zWOW`r-H+Gi6(hD^AGSRp|Gk>a==VpRSr)qd6av)q(;_m~+^Ra_VLm3yIfQ+mQa z7htvNFrIem1*1u;vHom4Uq$Coc~>SE{gC&ctOUj$ZO<=v7og#t2G>|lz~)WDZA{BpS!M@bmFAC|PgI&No z>?;{!&rAQpDOup7vzb`YTaz|PD==@40iFdVFaX0CZKlIim25inS%NhdE_}CTBd}mA zx*>%*-dWtTR)fE*euxe7JgF()3-z6O9AzBG=u0bMx^sr;Jl&EbTY~BThBCQ#6()Uf zVd01XUYOgCwVSqM=AU3Ldm#DFGN1M)(2nI(cVfb_X8inftC%>oGoL>&;sePNyZNaF zdbB(sdb(HPPgGO--(QQ=on2_x*c??!Gw?IHrF?x0_W$I=PT@Kn8Pbzo7ev!yqueR` zbVc~c*%-I5B}+UK=~UZ_v$v|CZOTkkNoMca@%bG1xv%W=xQdX49}uJaR5&hnV}3@4 zXt7Z~)@;@y#iS6ucDLgiS4WONn8DV6?U-tn!kW+i^w#`@3!esa)JrGs*c!<%I`>80 zt|pAVqsNC*5Bl<>LHxRLPq;acpz4@&7?Y+W8C)xI@?1Eld-h|j-2FcrVa~PXZtOdE zq|6&dbNgZs+;MHoTXvoK&3U%mZ@R%L=_m#z+wo!NV;G>+5w1@zDhj4+ibvxlOLkUM z-Yb&K`M_6~1Z&YXMJiZK&{*hY1ln7CW=nk5r^ zT-Gav{)#~6ryNDf^JIp(NS17+D%Vuhp|F! z0oL`a7cT$Wvf@k}pTCObn88~xta%(Bd5+`&@AVkHPK~y0d$Y*@jyUA+A5ahlk(ZiQ4QZ zig~RH`SH2n>1QkP)XbZWDI4)LS09hRXo;y~X5yEv^j>LCM?>2NC^n>1`@}7DEE~xF zPg-K6{LUVJY{NBUKZ}2!t#EX}Vw?>q7qM?gaZOPRrf-qYvAP6aR@x=L+#Sk=((itJ zVZNNH(lPK`9(CjyqtWfCP|3fFhF|AJqx>$Zwut5BgM+YZcQdGZpT(i2o=m9q7p-@E zMOyJ$B;MPCb$Kzo+TuS<(VK~uZB`;IpbK|iFBVhf^C|sb9$QV(XHmvF+@K3?U(A@|EwTho_P!i`$#@d172gdUx@{o4+1> zW?%z@l!1KzUl~-M3=-aLx^nC`UuqqUqUM{XTz)c_h5Gjpb1|1kT8x3$>@F;_Ov3yb z)p+@M6pQM+aLeLJqWRH&JZ=%d^q8)k7WhNBZLJq+a^5n%+JQ49ujRqm0{RpN;G$$m zw_lhnf;Xihr0ErO`7ZCF!U?Zj>u{q|kDpazxN1)qamW0DIAdZ@gJD*vmi`+P#R#@B zY0a8-OEGEKP#T;YLys?UJl9O#N0owjf4H)o%{y|+${w7eA$jLNjN#iDkKFVa8sF&x z3rauhwpvWQmQT+dOWM4tQEZ$N$ZL-dVaq!Uz8(D+MSOt8k-rt|qatbE>>tKE>+wSR zX7QyU9ol6fl2J2NJhialw9SQF-_C$tj>WNmvL$o9ZwTXaZ!qkk+{Nu4uUHX29wWLo zqvhL`FfWbamNS9k{I7k8jZn}^^B}x_gyLw{9mR>-UOe1RGNsavDJ)j^k{MVv>Yub= zP!id8fEP2j`s2s6Dv6Z@Fh)#X3*&oB;P2IvvySM(dYm)6Z25%UI~;ts=pDl03JqFWCer%2 z8?0o4VTfA}%hOu$&SXDE$n4aH*dA;gqE4Imc=qWa^^5QMaR2&M(XdjRU31;}ah@&B zR}bZpUZ)gZpEqIPzp)(nNSp81`Eri!7o-)<$A?{=VSZbWePrHXM7H#LNWCsY_7y~4 zEQ6`kyy|YYV6noUn+p3etuct^*$SRdYKg$PN_;l&GR_UsF+_2YZ)`Ll+i_ zm_2?x{2C=)S%jDJ-1(B$-6u)vlH2OFAxSvgqN5HKttE(( zy1DZC#b`FAKaU2DXM4v;(Wdi!I4zfc(}IEgEx&YMGOMt0oCch0oA8y+1T5TPCfsW0 zBIsN-O4^2SVND3bHc4iLd=^Z#&f*)H>B~;|iedkBrB|RAPneZp*0~bVs$>QxZs{r( zU6~?QB?fcn>o+3oKpk!-r*p2hF2e)+aMR{Tu=?pB^{`%0>so=670t0JWdX)jO5gV% z9iPidC(z{DW!#@U7InL2-@xiZzIm`hQM^P4E%zmeyRl!fQu_An`Wf*0@|j{qNn1vz zj^@8%mtp&_jR-G&=sR>u4(|*J;locxqUY9;eEei8ma4_@Tj66|dVdS;Yqz4|_hc-a zq>6x-jfkt%v6bKWWcv{_(_dw15ye;qhExLyCLXPysRL0Y= z%T#E-UxxjiYejL%9C1r>;>@J>rrrG}+|up&+NCXfylh2-gbu7KDT1b-3SUI2QT@hz zgnn@1r5>Ab@oOvI4-!y%lF02UvsX=s`xcYu=)D&Fy_>N8vkZI-j%L8Tq3CqK4(r#gl$~wvjCSZN z(vm-6`>~PKYws)l6>7|t?2LdZs=O$(4~{clA?9y?sI?Tb2gZ_Jj&|h4bMufnRB}O- z6R3X8l{qp;H>PDF9=)H5nXM&1X>|`q?e>F)@|Mq&|2r+L9C6H{rAJi}02V z1<$*kq>;idpenOA&fCMEa>&8IaG2pS)1`ua%GMq~a-J2p4>I7xO~W|4VkthpmYk7*?rgtvsCX!QSEifI!)^Tl zPP*_OtxpcXPx;*Gl#|T1p^j)`t;{BMtI;qqn3raFa@OH6T1(yFWxeFy?Ytl^|I(*n zb{IasxCL9YJ>s6S{92sh7-EgEu2V`8?cg&m&=CE>{F*^Q@&NmM9zZx%L{2tC);-b^%Oj z{TQd8l;fH7Y*jt%%=2YZ=a(F1v!w56a@&thw#D7$w2Rx*Rz zj)Aj@HM_YEXVZi7XHNbH6WP!8$KH@$X0_1kTnnEW1y~qh#g?U?agiEOJ7XHF;7JXm!W~WEJ;U9cQ zBy33)!Q(Y~>)>6)A2ATYDJO9q&McE%LeDK5@LsZy<{0GDW2_%HUWtlr;@vv_eC&5qDOW)9dR^xg04G^l6O!eW%k(QIhN}mkAk19d0->=1`!#T{V z%VEnI%@j)&cciu%jQAzV$Zq}y$__(#v$gD#JgAG*!Vai7(Uu+qWY5mIcv}0-MyzDP zcntW0%}alXj^=W98(W01n&+q(Y{Lz|m!XYqHusE4#e%b*&vzap=a%&JXZT(h!nXbkFmpIXdMATAz>LmX3XCAorgypFYFmcHL+`bcOh{ zX&@tC??BD_ccO<{lw<*_N^Sc&7PL4cwS38%HSWeZr8uURORnfVS01U4l3bYMi23gj zPWB#)gKBlyxZw!+$zL+B?Zvfk8f;#<2`kDcz{*5s@IJO>R?=pq9bFBTwVOq@O_J}I zj2x!AXfoRH1k?stNng7z_cm+8T@gPKaxQ`gCD*rJEb-@kFs( z?U(5IxI(P(EMV*l$-n5Ez==QPY?@-idt=ST2E*2@cp1qBjps$P5w{d`J9)6`dJ&v2 zOQ!dd-y(AS0B&EF%Z^YJKJ$mm`FsvsKg$_#t`?6h8pnN!KCD+ChrfH|U3S|ZJdL)% zd82`RZQF$BrAMfx>l*Q*pEmQ)>N0Oul;~NT!()enxNzrSEEcw0dc}uh`{?qa>?*i_ zLH3z-k=YT+lxtpIgNS3vi1kM8Kg1` zZ~F`orUm=Oh&8P+ecm6K?ir0k|39)aW{v8Y8d5Q_hg5 zp5=3RwK4O8Wp-RA5JN1R@#KV3?5y#}p-0!y>Qa@smwf{L4lPH8x+YCyo1rF%AAE6;9PvG&+$pta`-&LGs69sN6>CnGbGbbX z>D@bp1-nP{r_VfbZbF6d(7c3=a@Kr#awP@?U4_;84=9)27iR{K;kPTh#e=#QC{J|Y zPW5Ht%J4wRTfKs97w#!~wRU6U{$PIhYs#+%vZt@P{6008jJ8EjeUme;z#(if64h@Z ztjvJhD=QS;bqv_)v%oN$DD2CcfTQ!Wc(#58@-mI+uWZ156SraI)Z;i*zZ^SPW>T+9 zGA7>`%_ADNa^JT?Z2x)#Qlf!xaFQq+(}AHPkf(F9d48SjvHAdvaJ~bpzs_8C%L&D5 zlX0Vy?DM<)7;jg~`>)|n)Xg!L{Zq0B?O=cO^^`qb?G0JJy(zB`JTBUVnJM(=jKrT8 zk_olppCVy!EbrcL!}-!-Xw@kjLDzfIElye9K_=kC?tF$n&0?~jG3PBgq$vGu zPqnhP{OKgw5K(r_|M@|7^Ss4NZ6!MIa%cPaQJf|Ha(5OQ@Z|yN`&$+cC$H~tGLp>J zE60)j+L1F|PKu>!^Rcp8GJxMTWn!<{h}znj57j*hf3Y{8zGgckB{ zUf5CUXPPgebZ;7hI-C%rUudJZc!|>2WB-kTdz1`PP&bOIpHxy8*q@PKu`|8n9>cT6igiFynb6j>mh@WI-Zx8vF8T zU4LdI>(IsBQbbPbPp47M_^!)v8lU=y-M_jp%p{1xp7({B^iWT4vH<#ne+c)gCRlAZ zj1P}Iz#WS_B699A@okR+cc%D=o%-Fm!hM|Nq9ya;LzzL)`3C!gT{(T4kx-I+$#YBE zGk=sTx3;yVx@7&Y{nrc=3;M9%uN9bna6T4xtjB=(=_qV*8LOTy6ZR&}Id;@VT(cg} zeMjb)~KLAvpaT!W+{2vSN-mFSi&hUj1jrwF^rXzpt9GGB+OA%g1rQ+gS9u zB7gt#1E~3o9P`GH?)S&B&E4Z@e%OhfE@vo)^qY$fbDOjLV<&!QBCuOeahvY<)ZXH(USbPas*p1-YD}1Sxnq)Cig7aJU#0-93;=fIe)6$H|Mf* zoi#Px+7Y8hi0!Kb(0F_a6p~*SIcy#5R*y$;y#aT>uw~;HT`FA(RU|?Bfwz9gpT$*B zydT0)^%%qy<O7v&R^D8YVT?lOu{`u}3zApaFut3< zdM(F94aM3m7 zf#G?)xn#NGeyTsrZ;g~)va67LZUwXqx^r7EZQeQ@$$Q6xnJ#l7K?iJErreA9mw&>~ zRF!&5&SFPW8Vq;((t2|MxW*D>9wXmn;w={xa#T0g1|3g^G_NZv7%xq5#)$w5$M<+X8iH7v%wmoHFJ zJCZN-QW!l;Re?e=} z!AWv)pB%)@XK9@4HCOSZ)h0N0-zpAI4q%(ZMTlwMlx1#n6gRcA6{`j=!~z=^1~W-! zcE_@MvA5!~vl-7w9)ZF4G_+Zz#4DC#SQeq+B==c`t;}e9~V*LA~_5mefVB0fxCMTW2W5w8V^2!iw<%p z)l`=o;^k-NH%Isu8^JQJldoS>EqcmMgFkUkaHL%i2HvQ|+0jN!oKoi7R(4C3YmA`7 z7b7$YR%2R&5*_UC3xhK;Xt*IW_z#hJ~x_9UF?Yddh$pM2go?#;@d>74a022Cfo{Qnun<{xudencyj-~*aPwxYdc*Omt4 z)2!EY#l)4iV(h$f+?Xl(nOkBdzd4X)L5E z?6mp%02ckQVb{Q=C{W5^;@7P(y4sEPj)Az>pv55zlc_)65eY8$74~Y0aGxN1isz?_ zn_Y9c?7Rid!+*i=>PHxLE5oLTDG2PWi9N?;WHW3Ef;@)^2oy|F;h~+Ws99WRL8-rq|Hph~P+@L7a0v ziN-Fm+>_Xz&2(i3bBivml#ifQ`e<(Y)}KMrbKWt@j-&gX6rU?E;qNCQ`;opVoKz-b zwof1%*2>NtX%%SrY{Fq7l8JA53ZolZ@pQZkZPLAANq<~AM?0WH~ctc9F8y7PG2djv*Wuw&3^ z{Qi@{OX{=mX68~1oH+wa^Dkg}@J0kYaN;ScPc6y1AnrL2g>dnp;c6`g_n3t2Z%Hh5 zlXLcm3;5C3jGHgI<3@5f;UJmdz8c!xBXt0q>UwB)aAo_!tvGME84NBh#RZRBV#w8I zJiSvgR^_agDCUUODmTT<=ewlF4y0ZPlKU2a4m~nK(JgK-Dz)}vw>XQnNpf%es|PmJ zNPRC{ekbmJMM-o5d*!{tn-fNCYW55vdb(`)wi%~5j-b8g2%grSg$C~w4j)@2rW@$- z$D;lCuW1~u4h%+li!WH;B!O$>ndLg&j;o9Y!7-yFPrvHP+3O3rruZQ=GwNV61F}Q- z5ZZf~!uIqfXmxoaLWg}2`E|0xF5(WJ*ent2&0C3UOop4}td9)rD_-A~`NP}Ycr;)c z95+g)hEFwo4F|IK&fe5sIUf~rC!RigJPW0kTqL>8QCeN-Io%EI#;y@v)uZ7u(wHx< z)!-`}*#BFZ^qGCf^^0*F_tBRn#_LdLR*$P1>Rdi$JBANv#qzB!MPIXa$h)W^wd)*q z_#8{CPVqFa>&}w-$Br8 zXC?|IgC(=#v$$%h#Q@Jpt_hQSdC7SC^23!GyX7uF{fEN1UldDA=b=>U##f4cg@)xp z6qLWkK+}F~=8?ryD(z_<<-oh{Z!p0x2(icfc*5h27Hz`)7g{tdOW@FR(u=u$iQN4dG4K9h7ETFa zVjjPJ?E?-lGF z?LnvZrI5bR|kZm++B~EMv-(I6V93wp`1FTv#4LI!pR3i71vD0QfsW#-P`q+^IQv_ zva*4h^mrIU$9mfe;E%b-&&szLhExB^W9oeMU3i#Cx zlHC9wVc}ke5ZTGMT;&il+bQwq^9uBTY`_^2Q^kJ~*3?R02iqI}Ywt^-dR)7BqmUso zrIaEhv*yzO-9Lm3X)uMPl!!#lp(y6c?vE#A+&|9js18Q$mFy8@Q&O@xEEL$_P#!DZ84nh-1MWL)1wjdJeM zoglHVvAQho*L_Pa14Zq;T3e`v(?N3j+z~^TT~pS)5Z^g6?$D%K4H0=)C(aC*(})LC zX!dq-@8xk*I{siQ-Ak#B&>LM5A#(i0Wj&E}X9MLl>V|l~k`(M{ghEcsXh@&}8gf}V z<5Gtnl>SOZW8R2+u3a%g^m&cFC-$Qv`{Ck@mU z*A10kH-}L6*^c!1Xf}1LZ-&KAf0y&Gv|#RF71;FaQ- z=|JJhC#iSy9`wFkecYTqn-E)<*TZk;r*&g_$0Q=+o9MGENGV8IJXR(Z=F9VfG6OOY_CX!F7>Q-Vl$9^}t9kFFOCOB8K>v z!o=X#czAmx3ci}cIM{-Qx9)(LbtSR7Q#c;Z5xrDz8{y;IX%sLrpW2klqJ^Eu)AsY* zmGosDFePUgx`}W4d}QBSI-W|rywsfO%VO+2GoFYemy*X zTNQ;)d*VZNU)mK^f=4pTsQu7}sA+bYKKoZgV)J;|Zs>qsE``y^xV-4E5KL=xt6&0&ewPJ< zDduE6UW*>^gDRL)*qUBQ|FiB=D_@taN zzo68pUG8L!_@QFY@$+-~ zw9%Z7%%4g*mS?EsyPae*tp>iXx=W9}#TooL$0>H8#Pnap9+Ro~Cj3(AbFU43Ts}>lhdWF6OkA*UUoku$G@Uj_ zTp{nRP4Oh*ciQFF0P6$A`Ov@)*zwH`J&e{;LJ?PVPA`j)I3G-j?u+ngnYtM>k`b2I z6!t!gb(jAr1CR6xO7hWEx_B!Y4#gYc#O!s-h~xDrx#ANF=w1aQ%Zan3O%lnbbF8TI zeu)yAR>NnP`83mPKh;mqqXyzEM}nx~^<>aJ3f`7V*S^)lQ0EyGW@z9Z=%1) z`)5i}(nzcq>)_Z11&aNkwm2SDMAQ{LPBrg~^JRsTQEf*Eg`Fw_gV;nG+v<)kYC~x( zFPpAJJ%|xC?K_aor;>P5x-=?<#o@epXWZPRYx zt44XjTAbG~?68i4_NR;1p>32tdm52%(PYt!&zy#qv?hDQZ{%(*=8*g9(6?RpXieF# zv}5x#YUTP|w>HrkCtti!_J<#&Q5E`7%3BZW-=Z((F0M#V<}Rl2)xH=v<~zlXIiL&Z zHI{0$DvjzNTH=GZIBT)Lu;*Kc8B|@I?KztMzMC zXW54iXBR`xr^&QxSuVABZB5@-i(KePm|46vjrFWS1?~nES!X&G%y7r0T}9~8g%-Gf zcde+wvPKDyM53(WedfGT%@Cm zZK#q!P3OzUp~0#VD9A62qdl#ueb2iT^Gw{`{+LNUn&c?f_Fky_ygUth zHbm5Yt3@N4E>!Z5_rl~2w`s$v08)~H>ZglQ^koC2in`v{7xhw1o{AcDZAa4~M{m^M z6iaQZc1MqMxyrafI^fE6IwilF7 zZWEQS%O;3=zrQN>XNsZp&0Dl)@S&G2cE6;b&&h%sZujhi+6z|PDG1-lwy8RssvsPXgU zMmP9v4X1f;#GZpuA2iwXT=6ZPOM#n))4AXZc@whwkT9Njhc)eOAGt8!Bvw@lznF@nL1QP(C(fz z(ZvEC|1gEk>H0K6RH_D!gL(sT+Klr4VL&3G_G}uziY4YbO4d&NW!Y$j-l(D{u znlqj5p0&s5=MmU6=mdG37rpT|i26&#tYMyEhsRCK5wSS{X_n&d+O$qsnfsj_T^pmD zr59SPD@xTbom3WlK2Dn-e5V65g3xqe9p$jDK3u=-BPDt;j@7D*4Pg((K3FSU>{=X! zVmeWlou@c2cS^}VY^1al-^n{a6|ocR0*JlYwKoP*ps1TN+qEO^WccD(vq(BF4m}SH zD2YSeYE#k1X-ZUGW9pFentIsg(Z^a{5gBHSPNAJJX6JIHZPOZfKd&mDhX1bkJt7<` zw3|K~tfk}67m<77Guq#;65LW~|2t75>DxC739?1+tOf|H z-VK}c#?j{(ZydX4fs{(sXtwSml^!PU?)S4t@QP35SR@SlMBRs%75<>$^{w&cwGYOR zx<`$xl*QvI;$B_aGsW9-G}&zRhv7|MT<#<8FTPqvX>C_2cA^eP)z?utw)hLhRvw9W z8y?X0mnW1C=@pQ;#R~fdR>sG@2g$2QA&fgWnlkNeal+^{6*}4rM{5cno!5$@A0AQ4 z)e(IhYGu)@`=ZX_@K2sTJ7sgw?Pm6aNvj}?Z<&CJ39IOCQ)7DQaF_h9_eQBj=fxe;nslm1 zH>@`-M2i?Vq zP_sVn)VI|=?p7aj9H!Iw-Z3=XxjZaxm&dddp=fZGaAc0?t7$L#SMD`LeLHV_8)A{oODnr`pu>d`G~qCTdo!)4$nI6-%((j!Kg5<6Ya|I5qqRBXz7z*XvfKwWF0vOU7i>sWkoEW zJiMt4ORfNu?82Da@s+3@UzyezyVJqbi7 z^JHP%%4mXFVy|-kq!DN=&Y>91uPx3%+lxKG<;t7+e(*lFo7`jjQoF{YFWHy-w6s*F zZq^`U@>|`LuBT_wxF=SW^-0wK9A}L(T}3ZjPf`xGnxP~)8Q`v|=+EfskI<!p?Sp>iVZ7ql1am<04_Wva|NbcUAn8i8qjPEh^S-f-|J3M0|GHl&j; zdXz9j|N7j*$)JlRwisv8 z+%?TmWKcMKBE|k`h1pahB^gii*Xbff&C9Rl#hs7mC&*#uH^r%W7+PF!0$bOn__jEk z?wJQ*O@-!Ya?6i=_a0JS+gal6UL#~30%mz6iZfpUI5(pNPIYh>=iR2zqTszsf=zcA z)$2@#4^Hczjtij?!<{ia3#e_<5%;R4Q0x~&8h6D)chWnMvRg+|v!%5#V8c!FCu0na zd`br0{V?@r0uElS3|gD4Ox<4w57!0Z57z@UZqOmBWl$d8&0Of~phV0Rb0ysD0*gQ| zbotGYEb63D$Qn;9xe$fvo~P)Hy$QVY3*l|554>$+ME(ED-`9}#+;mZH4y%bv%S656%@Zj)XdHb#=81{zN5JSyH@xdU6ca?x50|s`QKF3l zo`$Zc(2&`p=5rfVuF@0{W<4>k-vRo)Y$vz}dcxqh*<{_$9vj>;D56?@Jb2aves;q| zUB}Hd`|@dOBuj3Wi#(8^tBiqUFyQ^)hjv~+!6IWJ}Q3Kn^T@eGfcBLRd(3K zVyUe=#*A=A)%K!?>6HmoWBd)8KEDgn2xfgPQ|IJ$G9_JgwqM)2qj0LOW3}W~(tBUsndF2ZbnmDvH`YnU$30m1mIe)L^V` z;z+g5?4nOfO^iFch~C{3wU{?_!MPkeC8uix1c*CG<%adc_ZiXX_G=xw=mr$u5QN(= ztBV?t(WqGC7sV`e5?yY(m-4rX`-dOIna9}|sB4ejSk)twQksal6C#yaRx_1JPCIG& zj#g-Nc_3ML*P&t87&2S4kzR_r0zS6a>CKLoI4JJXEs7TXV@1!D-AzPa-1oImxkn0J zyeDd0Y}luy56Y&_mU*J)j5~anS67akI^boc-b!lRak{kB0LdTM&>vgwD77t$)4BQm zU_9RiU7F0KB`>@%`k}ac{Jj|rMeSVA7iDnqmcP#Wjt6B#1|wVCiOan-jsm~c(Cz4L zgWZ$b6Oqoi9a^5MA3dP#t!R(t;;h8xf>;c`UmCOWex--^=Th^eBeX330CibkSL~}6 zr5oZq$U)orx;9%x4V?vq!CU$%HSk&K8GVaJn)C!Y6r+mXg;2gl8!~$DN~v+%mBD74 z$;rkIom@&#t(M&|ARrJi#^Jag-WV3sF466+wGgv6%d?W>8pR^`Bh{PJ2p7wiM`v*^ zGirP((&@yScG+QSR@MZYa{i#v(-p)_ct~L{yr@h$56b(zNl~_xN2U2yDfx|*#l;c8sO2(Th!XN85~-OTw2v^PR-i?Sd&Z z!NeQS`{|GxGLddHv!^xWrZgDvk?yt-cM>YK#^{UUe8VlV4>>v(x?{lJh}zwGf^3yN0y%R93Cn9A%qUY z%AMsYK2(Q&4FTUy7d&&Etgvf!C_EqRRL*2aP}z(1@&4W-va9owuC4Bi6d!9mzj{~M z-l;MyL9{$Cjgn&GQT(6=#zZO|9uF41ye=w5n>nCR zSsPJ@uRPq%c2bhjD>|H+P8}l}((VjXnk#B%3~zgr-b}NF=>l_nD;$H+I7eLE8$tEZ zjTRSo!_Y&9NbLQX_Fo&06QWMjg29Hc_&5|E8RzN4n>?k}p-6F0Zy1K!deQU)6X@~S zlXTl+H`Ns9-s|;=$LQ9jXvoeWq(GgZT}hb9^~`-04JC+q+YfxB{him(DPG8jKPK&&lrDdgbJ< zOl4gDdvbmeAZnWz$5X>SN}gXK?E6+6Q_6iM!;F4d8lFS@z2d2pe|H=gXM3|PqR4a8 zc-`g;qQ>dOXSDF?NHn!?iYn(sjo*e15p6aMS85(5bJ6!TyJS}sEC@u0Bu^NxeN4go zUnpDWET)g09Z*%5LHEZe)AdeAXwNtU^0<;i&%_<7-QqmzxZ+-LJUC04Gi4DC5%rTy z{ie{_yACLOts^}gEoyYH4#LY3Ls0MJ3fiQz!#0bVboSmL)O)8x`>`u1&h!#xh1Y?R zt2y$;y!d6F6ZR;L;A=kwb$#cNLuOfwo->CY=iZ=A^&XK=rV}3azDZ@DHuOARcQ3UT zIXu(!rSiPX6?*ohG!ka_r=#NB?LB3Uu7qPC-kX1+XMW-w?wp!1ZGD9b|DH;2UxjZ) ziGF;#Jf+j;cv0)HG*%RyL;FmsC`LI!*pfa&cjsIyls7lQ5Z8RA#L#teQ2|Be>$C1o6?=@!_ay^Z7r9KUM6NJcu1)7HG?wV?Vvre zZBh5qBwD>79!-bYk_mNz(yloo-yNeg;q#f9ixmGI#uRqtmTutIp*UTkKXyBLV!_G1 zRIZ&k=XbUPGQ_#n7taeLDph>r3@D2EX2a-xjdQer;6CbXm8eu*)SOIa&Q{`AWYCG8 zH4#wU4{Kf;;Z@C(;@tmw>TEPgS2v@lf`KtgJ5d{?<%K}}=K4yvtbypq4ZkIqLj67tNcFVCA3cWPaR#uk=0wGBmoJ_e ziu$k<$CLB%D-`A56E2lRt=l&BaMHzuI(4zZmhAS(KNO4TA=OdD?m8X#G>Pi>7!2>! z04lS@367pN_>>cd>FatcA18!hS4uE03@(Xvwk>qWg6g2n&^0vu%TlW0c$Z~XaN7LF+?g-yX7Cl*e zInnfn;=Zv_Pvj@3Dl_cNabMhnH;o^L%+W3gn2?Olb|Z0T=@nZ0)fr>gR=|i{8_?J! zYVxE!S@#m>sl@HN^TV&}T;0UEuz}_#%4C?)qMy1u+K*nO_?!^6ko!BqZ2bzlFt0Y|e9og@21dyB>xbR<*XyP&6@4Fz*TeXS zMQBIgr{s2}F=_>b(%IAV>HDp&BQai_Q~My!gMY7hn|hzm zQM!g2(3~gll~Lo3VANdn@x0y?cUN?Q*WF%pYF|^VK0Qa-e73D}CTJwW{om2Kj2ycC zRn)z_dW*LIS`R0di~Y(nLos{16+Q7!q1t;UD&JauqpS_yNGcSG*70uexEx2iHbkK# z7m;W0D9kmlf?Fr2D)w$i>C8({(f_Rt_FN00T<765e)U#S8&LEs5qa&5Jq(sD~+&Cq1LxV(-A^>2>8kp&b~(@gZK6ul8g z+hOe7H_XgK>HJ;W*Yjg3vJ%0x!#0mR; zzDm>b8zLyQF$Pwu2iHR<=A6*^q7TVFFQ)#qy zp%q4W{7T_VKGRjR(Mrcj4)|seuk76@>NPgV)_rT3O4I(ROra}U;_mJw968VcNuoY8 zni-LWxSud9Z7^1i9EnVEX6VMRCCSsPAIi2UiMnUIqQt7on6bPHUZN^ZIZzlk9=c=Q zPSGnTd#ciKdnpWU(-z|@UZlt8t6<>!Ug&&jBZVgo!?4vZ$Y}15vkQID$S03VU2#Ok zhnCdY-%U5KaTSy_c2oSt8SI=9HL&Js2r87bLE8Kt;ylj+`Xkv0jYThFk0n;fJy!_5 zp5IdLjIWE&r)?2u*bZ&2d|*5*hx)ZN!oUt8C@8s+`VY-g_K3TR_V-3mdbX&UJhl)e z_Pjx%I$&w^Fg(pUP8BTsqRq@)`mlLCxt_kR zWOoijYcap~99x_U`roGxc1_`C*930HHz{v$QJnbl=iamN3R8@9bnp~5!}sGeV3J)Ob`CLo8Cs?tB|6A8Uvw1>MnR z_+9FnwU8D)83N}^(bPlKE-mq(EQ*aW!>J`n*kU2>ab63B*}|vv=KEthHT(iSit0^c z#@|p1n}pNom08rTN_+SR48!z!lgQ7kFdf~VD%Pt{$T-Fjzdmb)C$;)upV%*)Std@@ zDKNqCt7ZtOA-)4H^+Of=&x+Fa2^B1;MPI_JDxWs>#*j;|m8zeIW83wmx|+ReV`x_o z+T+%k))x`|;%sxNr+*JrjI*Q8PRZytz64V1Wza9h&(OxPIn;F6HG=4inYY&oO&^H< zu4%uL`Ek*oM(k}|zQ0TK94>=rjY1Idy)4297ss)ZT@~k>Uz8Ms3$*M)38a-gO_OtG z=zPQ;XR#AyDC^=7YG^o$+U;6I?Y>vRhv)qiml-9g%gE=-r*4A?jVq(pnM9a9D~@DQ z^U1iksJqinoI}e3qQ@avwLA z7Ju!G$fs+mXuYn;h)%|*1CuEG)Dk+eDFSCNZPiuF_)N#`v&g9VMsZ)MBl35aL24mS zWR6)a&eEBnMeXjG^!PEgz26z@BaTwHtb5Asef6l{n|SQm)(ItUh(3Vgn@IH+jc__E zoxa@&Q0{EdA^XZMGJjD6c54TcYsOs)ez;Veso73F4;4jvyE0hXwFS%@wm@j1H0tuL z0gYNT6mDmV;=-LU{Boxw9bVa;?%FoODc=y-?-b{790L$mtT_($v4C!4Em}F}iEdM2 z6Xc2BVQ&vvQJHOG4_xdQ7Pl9@bHseAtjBOTw-(HcPB_`s8^!v0!?$b&-M#moG&L(l_j-66M3f6A_tdde^l2B}ad!g@ zE#*!NX15^wwqNOSehihdjH0uL3SoTB>PY(>j_K`0-@eGUX!hn7y{uK6k}^u-UDcIz zEXEFd%DPeW=g}x0Y=r2{v&!?s{lvE(W2|~A?zM|`@#T`GU?sjmxsRzP&Y<_f;^`A9 z>{=mgJXsTS>a>N6embM>=a=1Ou*E<=SL}!$RY&02kfP!|(teuY7=~724>W$kSjERc)Z9IEjC5TZ!fIuE znC-O0Hn%3YX**GOJpCx0&K7gJ)>fjYjEK(`qITP!Ksfp@rfS8#vGi?CEZ7@Hx{B2? z*}V{w#N5kb(+%O}O;sjr>_e;W zED__s72ZYW()@`I7}qP64y1IW%vZsRx5j6F}lekC(@HtQKzWbJ$mb3 zjTTznQxbZYgU^>*w02@7=9jLG4yDBRmS+KYKfOFv4J=FHh6yP6xQI5)`Q&lqktgcv zgihAHq2xRl%Gxqo^jHXkTgeVsy4)6hHs7Zl(UrD$=e<-v#~MN2^>KWuGaTDnBPn7E zz3Q1PYj`4X&;KKS3{tuS8frDfJ#^dVv-}o3VtZ#gL z#9xqZ?*9!Zlf%*Nzv24CMnpva_0PuX~g9MJ>vcI@X?5lxee^%X7 zeK64Qr@}cq{kNVOj{Q${15JD|U4C-buVvT!u&B^7uCQrd$8ZTsSe0+%oWp>B{mm%fXmkB>H1}Fg}^QjE~`* zRrb{f1E$7s9uhxbeEbU+N&L$AQ`M)5BgVo{E_&_8e&r^i%UD=vH$6ZW4f*qIruHp{n_!u)mdL2XUwwrkm^t3_p5t)94%z&G^~ZLTBa7@=hlGoS|*lx0Mi{V&*t`fgzIHv2U1{}vZ%d0=H zF+L65ALiA>FWZgfaFF-`(`Eh1#u<(N)bYq&WnXzow%Ww{o@k1uBY@Ns6#80yIstlj%RvP_j z?565ZHh=uhEJAvSq`?Zvn0-04#uobvo(5QIL0TN z|FNDm@=D@}sd0QbN%&*EFkRU^k>zE-@=yigcw>AFC$k&N!JNwM!FpkN-6e8p;8cH+ ztwWhE%PX5dGd||;r`G|hVqa{mMmR4-K3eZ$CSKmgONCe=1qIys96_)&Y!<n|><+SCWl zJ`&3-i*x2rV-LwZjPn}nS+@RQ?)gcUhnOzI$^44nYUoP*h56%O$ox-6SLRpj2aL~A zZ$4l++4_zBp5@R#-(`I661vPE+xKVVT>rX^{a!Y2VmX*U**ML5X1~(E-e)*jTrxF= z({Eqavn(!Ua{O#Qm96t+`1FsLtY?}2Wcb`9^9_b$e6A8W)*nCVr_1=%>7-^{;CN$u z$oOMFV0(C|X-a+2tmBzK**u%&)xb&OgW;I&KiHSYX-WQLd>n5-?WYoa%)N#`i5#py zwukJyp9W4H_htM18aP#6HJxhg%lU;lmE|GE$8zY850*ogCuR6#-wkAN&guk^{ffC~ zyUF}d2IusXc5{%#nJ|dv)K;T}=r3 z_3Wkw9K&hos`wlvde-Pqh0{L|)5t5q$NFPGknM{y96!mv3-Mc~D_g&@y!;FO>wSik z*+ZijHGXCJUS>BLKIV`0qTlZ|ekzGK4PBK#)m=3Ch2dC#vVBaJm-XT-S?@C(^XD${ z0~uWzK89nuKRsSb^diewvV9e%%X*Q0A7p$iuai0<;&_w6{lmDR=BuC0yRvbC?ZI-$ z;+(b1c}O;2V|*;HZ2yJz$Czd76ozBJm(A}vK3ERf_{MOIPc|;d=*s5pGJLY{1{ydu z-eh@={ebl%+b?D~*0anW49ECna2k14{$%Sx4IKNQ>c;AW0n^p+*XYOlAAcKYzGJgJ zWb0oIU3Hw1>5rvkKak}~4S%Y9zin6B(R z5=+T&GP*K+vb@W1oOku>kLk+tB+JX3{_MO{Hcqo2Fh1FM#BW)SfB4SC{wx_UWqNUv z%_c^KP`;qP1o{ri86Pd5K!{jr{9^2+>KHjia$oL}_EFU#R3 zS+BCZ?5DE*9F~JQ(9a*k$?`MPWw@VR*Xp0IvAq8p?gYUrx#ZrQ$>22SNqwoca2RpDfGS%0i&*?LgpFDiek z8_4yi!pXh|Fh1r~Hco5g<@lA@LnDU@CtLSu;8gx(^LFNs{aJS2hq>oClC3K=a4KC{ z9LfAjwk~5im_L~xFdX}zY+cLpvLERGPRtU?&hs-qhLg?jnLmb;(dDQ(3R{9fZWPYH5Q{~XV4$$O3z4L6wEb|wZm*HgJQ>6A#Cr<1KQa!7$ z)d$V_A@=8=AAf&Duzq~9ePx!H_4fmt_WnL=gY72UFVVy~`-`e*=8yHldE+1QxH?~x ztzyGP(@M^2*{;BZsO#Ssbz5H275Yst=m;=xn!t znD4S2lJQc*pQ;zxdPb&a*>`N2US#`(EH7i0eb-?AG;r#Kob|%^INoIAC1YXz=|88% zzfjYVCO`9A#;2-T1IO{farqD9jA}R8zOoFTY+PkHjbBOh$8zwKn*KC$@LR?w+oxuG z@GoTZ4dzc{H+3Ln`?4Qs^rtGC;g~MN>0e(mf3i4Yx-743++x3HIb{2t%%29Ins`|b z=8FAFHlAzrr{)*gc{rAX@yYyN1E>1m&*oFvx>h5HiqE0Zj}b{KuL`GsACv8?f4`XZ zEZbMn$gAc*{rf6xUm1Vw2TWHTE;Vt)aI7KOI!eQzYG2uSDZ?j=Hw`|PS2ctBpqcM7 zf3p0d!N>glFt0{lhGTp`JAUY&-?Ker--($T`-^Pfg5!woCW|+ggXzl7N66r0`xXqx zbY<&R=8xr&RdbQw`(EQjnqm~7nA|NV{mlYO6NdHG3p zZ;|!Key^q%jb4~Pj!XUXPL@}8&W-W09QyM;O~r^cIpKVbg! z=L3zus1s{mJYrTmLc~)BV{vm+gzPyo^u( zc+MUn%LmLK>sg&s;*}PMu7xsHKVR8J*^ddX&sDV@MrvJPJ`>Fo@NtQ@9 z&afQp2eLSl@h3YstbtSYA{(a}j_JzqF@G$tY#&bJS1MiEc&Wk1{vw%&Y2Z|Q$i53{ z;8eP@_?7vo%%5d)$l^^#SN6SI1E$ZKjoklia{YAo;1u6y+7 z1D01duao(KY#yPJSJks@{ME>z$}8hf>Iaf}9ovKBQYMFnuF9WG4wjek$?~{HUbe3! zJ~aHPbY<%@8Go|<4b}_uC-Y|wK2^^$|0`rr#Gufh^)UbX&;Q@h!1S-5ni?Azi2sTIn*8-=5%K^2{7vD)eYj~_Ps;U7Q#zoD@EpYoNh3e`WHOBy(T|Bv_}HF(|Lh0QGd@1HZ`fe*&wsNO9nrT>WS9uOkZ1v^Ha{@n ze~*pm-!oBtq5bzbfhznzzxmrYeni#Zxc~k0Uk&`Lfqymde?kL4^2@(}|El1v zs4NO`4vm9WYa8&YZ*&fr6$^V{E)-fYauoMw5Di`kiUtHh4)mRIX8QRYL9{kpmTSI_a+{qBC_YhSzd(VmWPJ5Rgy z+rRkYPv3G!+g%m-`qxhX=*K7gojL2jGTU4K@K^cMFUj_mTi^f5=l8|GKK_LM=1*Sd zl~-SU;{VjV>QBt)eZBpKk3R1o?|J)6A6;Ib?E9mi{QLVpy7lq>ufF=?Kwp|G}P7>+j5AAHDcmyY>CE{e{{7>3RG&&FzP?J)ir3 zXtv*+`~Ti-zdpzRm9xD!+kZRzpUmyQI{Tl_{eNS&zc5?hg|E&2AI$#$XdeH$x&7vB zfB$U1JlnI`@AlmO>fAog?bm1fjoJRtY+s$%`_Hrg~fh+3(NJ z?cX)~zc%;(p}GBwbNl_-@4w9L4`=)DXZx4t@o&%ef6o1XYqq~_j`thp@t>USubKO= zX1{+h_y5dn|MG1A`#k<^Za{wg z{r{V7D^P(7RGu6q@h`5%nfMl;;&uG);jEX7awty6qqrWI;%r=tv++Ux#?QFe_O8rURDlXqpaK=B zKm{sLfeKWh0u`u$0(^td@FPCN3%Cj=;XfRTzi<`)#FKagKjA~%fh+Mc?!bw7%Gvk} z2jgnyvUmoM;xW92vvDI{!k0MMhvO$4iT`jAF2l*V2#@1tT!#;GLEgo)IMV$&KiV#F4lYH{)x3ii2<@{=?Ha7f0en9EH>I zr1S9%-on3l9f#vKyo~2?5{|;F_!?K?NqmPN@*Ez-n>Zq`;yav)8*;P9a~}MUBl00G z#*H`;&*5MkiofwW-p9W<8Smmu+=+{EINrp?cqLclOMH|gp3Ql2JnqS<_#&s`jy#Ze z@j?!^y(_a7RiFYDs6YiOP=N|mpaK=BKm{tG059S}yohUEyUNV(a2?*nkGKy1;S}76 zfAK5s#dr7?2jf7Thj(!c&cWUI4d3EoJnLfIgIDn+zQsTI7T@4G+>VoRJr2fqPNowF z~ebWC4R?;RxjgQyo^(EGG4^T z_!@uXR6L4TaW7uSDft>d<9ytVhw(n%#>KeW>Q@|$bLCgQ3EEbm0u`u01u9U13RIv1 z6{tW3wp)PTa1JiSr?>>S;VqnriV=YGTz2u$Bco28vbNq%|aUt%s zIv8iI0C2O zHN0bWEH1yoFEjAil;qxD&VFE}V-0a4BxXe=g^Ixf!?Par}(m z@G)M->9`S>;!T{8GjTN@!>u?DU*a!ZksIk%pOF1f+>zkkg6{tW3 zDo}w6RG=ccR7rY@EdMp9*eJV6JEl*cn_c9M_i2? z@i0!svA7l&<6S(ATk#Se#I^VvH{m_Jj*syy&c%cH7x&?BoQkLMA^ykDI2f1Wd_0ER z@D^^zpST#W;x=56hjATF$&>gIpW;jA;rJ1!;!50zPjNoJ#pC!F$Kz|>;&*O0Bt~eRD;#=FjLR(n{Do}w6RG?|SaR z4R`?`;R8H~-*6a?!JYUPXW|x|gVEy8{a&c9=wjvzA?w+ znSB#fpaK=BKm{sLfeKWh0u`u01uC%J0$he~@DeV-Ww-**;3ZsvlkgjE!*@6p$Kp7= zjJt3Yo^?6Sz&$t!|KL{~g->uSuECo)8yDkRoQ_{z`^~E_7QmG_3s2*D9E98P5}w0% zcnlBXC>)SW9mZXF4Tmuw#eF!>!`Yw9aYBy6DLEv!;*`9Jk8wLr$oY5_U*mpUj2rS$ zzQ!dv9AA4f?=N@ai=2$7@lI~WF*)4%9EYEp-{OtDkaKcP4z}GZw3St$0u`u01u9U1 z3RIv16{tW3D!`360>9!!9EYE95{|%g-XA~UAiRpla4c@cFSrk1;yRp!U-62oaT+ee zLwFg7;%oecEAh9pSubzn7`%h;@FkwZy|^Cd<5XOSpYS0*$XPhsqd6Zw#)G(?xh;Og zzql30;#NG38*xAW!>{<@abEv0TMl(GeYhHr;(gqc({VO_#KCwQ@8eV4iRcAJcqQlLSiFpn^-WNL3RIv16{tW3Do}w6RGu&FxCx))Rjc=K5YEIo_z@@L zLcERN@E~r-0r?OQdN@ABYxo`a;zAsd%Uq5l@k7pYf9~gvJnMAs=TAI`L-9Xe#fNy^ zan{M9cp{JDihPduaVc)blei;dIfeKWh0u`u0 z1u9U13RIv18!o_E_y|YhH++M)a1EZsnO5)O2Rw>v@gvT_&G-i|;YGZP4{@*6Z@3#L z<58T3({U2+#eFyrkK#w1i;M9XUc$5Z9lzp7oQL1=KCZ^WI2G^We*BDkaU{;fWw;cl z<9D2q192rz#&P%+pW;WHi39OHj>(a@5O3t2yz*+?h*$9_9>=Bl)nQzTPw`Mb_iP^L zavY7HaWX#1J2@w}`e4?z;Y+loRiFYDs6YiOP=N|mpaK=BKm~SRfLrkle!@Mt4Ikk^ z+<;^79L~X)xCGzfQRcdM2cO|doQHRD3~s|6_z-{LP#lMc@iRWd;dl%m;bC6&6<^~!yo-9`>uL)!qe`LckmZ}#x1xOm*8T@aSqr{N16gs<=g9>fiJ56|FR$MFW9!%J50;!^yB zXK*xb!-4n2#xJ=g2jpKIk^6Bj-pI37SLB1djtla-lX-m($I7EJ3BTeI(ZKl zW6%d3-TiV$3?j!XXJn!j6+_|`nev@;!?creD>phT$Z!)wu{+s z!BVk z8Qv$HA;xxRBTX8%-#m%@6pW;27 zjmvNr{=~g_*l~P`Q*po7=YC$qiMSKD;x#;u=Wsk;$Ip1=ak}#^Uc~J<7LR&s*7L?Z zk7sc*F2#*bW`C~6=QtFH<9M8kWARXq$ESD{C**G2k3;e^uD1Ixf&x^a0u`u01u9U1 z3RIv16{tW3He7(0a3P+MR*3s;uZXe%kUmvaW?+Jg*X?N;beS> zi*PP(!B=<(58`edhSzbXt8o>M!^L3W{D~`u8xg77> z{TD$2Do}w6RG1MM z&chssm+>iX#lQF>kND5tub&pE&1TN}PaTUrGwP=N|mpaK=BKm{sL zfeKV$_XW5Kr|^58pZML+kGKdI;3AxffA9@{vw983;Sk(~Yw#Iv!Y#PV_i`mp!jt$3 zXX0Yqg`aRQ-ox9t8W-VOoQtFJ7jDMocotvcMLdx6aX=2kk9Zrm;$pmx)9^;V$B}p> zzvD;Tj$d*h-uGmDiJNgt&d1|8Ca2^=T#d)wo#XMwlW`;-#;N!vH+?YsaW^i;AGsNq zx}4Wp9gG8VM;^xQcK=0CfC^Ng0u`u01u9U13RIv16{x_53vd%o!!i7>=Pc%qxDF@a z51fJLa3X%eU3k{{I0M%)hs3Qo3TNOb{D^Px6kf(R_!zI@7QBY{a5Y}Ti#Qf%;W2#0 zJQg40fZU8f@gvT|tKJ-!;by#vd-0gZ)9Kl4`4A7{UA%^)aV_q~ulOR*;gNigCvr^Q z#)}TKZhpwYxFet9gZz#&@kP$ZBYEh>oF}j3T0GTU7SG~{oRzC_RW8e4Iow-w9{h5{ zmuO3?Km{sLfeKWh0u`u01u9U13hcfB-{2S=g+uWXUc-C%2!G&C+=1tCFRs8(xDR*W zW}J#!a2JlkMK}__;40jOKXEp0#g%v&e_I`flkhKY#{2jjXW~k{gvW3}UdC0p74PAC zJcUE?LjK0Rcp7)&Vw{Z6a2qbgjrbq;;)}fP;rPyj*>WeI#*?@jcjHPNkSB5@F3R~h z94F;UnCJdvPj0!+$sn_u*ok zjxTX7ZpT5m6>s8H9F5~~Fs^kR58-iqjJxqWF2!3o94F#oT#5tnF3!fAxDQw3ah!=m zalg~?8ji!Ocodi9V|e2KQS3RIv16{tW3Do}w6RGbxb}hZ51zv}xD#LC zCftY@aSZOjU$_~s;89$M7p)$~Z8#4%;&L2|Yj78i#NT)o&){0zj?eKW4#!)#9?#-3 z{EKUGA->1Wcos+ERosZja2-y>{kRj~;x*igdi{H*U$t4zo^P$F<&?;dIfeKWh z0u`u01u9U13RIv18!o_Q_!0NvPrQXwaU~wYS@;#V;bOdl3-K(j#BHvmglBOPesVcZ!i{(sPvUu;k5BO!&cg-8l}A;#%B@%W)&F!@+pZ`5cF@aVf6EwfH8d;%r=q>v1kV$>mPx zb+{dO<4-)0t8p`K_~CTnQCy7M@jSl8S2-8Y+VCaX(kf7a3RIv16{tW3Do}w6RGT6{tW3Do}w6RG^HaUc%Hi#Q=i<1svq({Vzs#Sb|S*W)<+iYFcC^|=)<;)#5SPo9nQ@Gkzw zhwjhgoQr#MM()J9I3};-lRS(wUd(Y1vt8Zm+1$?o`6B=1r(BWWaZb**;Y+loRiFYD zs6YiOP=N|mpaK=BKm~SRfDds79>kHj2shybJceKJ6>h|P_yCXLJ)DP|aUL$isdyMi z<7?c4PnqlCKYWa{aUCwlQMd@d;dz{ntMDo=!)bUKALDbpjeqewuJma9huHXg@)I1Ml2IJ}8d@jK4Ny*M1dGB3rWI2cd7n)BjZe2Rl{N{+|RI1{hq zdOVABo{v{?I3C5Tj`KK&O|azQ*jUO!o_$Lzu-LljDv6GvBABW*RJd3As9}dUsINHT{ z4~OG_T#!TYOK!;Bj^jbxk}L604#vs&5l`Zv{E=gEQtrjo_!bxCbDWF+aYnw%%eW)= z;&~gsL|a+~Do}w6RG7O1y#R@Tb?u zEjSoo;xgQY?{E~Z!q<2YSL11Xi`(!#4#shK54YniyovL1IF7Y?9Vg^Q9E{8HG2X=k zc^3aUj?-{P9%l}Vr}4;(*`JefG492kR!`zoJdYFdBM!yKxEUAZg?y9eaXdcBxA-dm z;->tPGjdP9#k+VWSLC$(cEguwORGQyDo}w6RG9gah##{>Qzz3y zQ#c?e<11W{OL0D~!v%Q|f8>UIi0|+(&cpM#9Jk_oyoi(WE1t=x_#)rrb-a-ap3nQn z^*9{2;+A}k_i;3C#=-a#|Kfm$aVoCH!+0CFy_ofE_!4bt6{tW3Do}w6RG%!7ca|U*jB{i_`EG ze#YOp2Cw63oQMZ;EY8NU_!{@(A^eM<@D>in0XYso;$D1(x9~-t!~ZxIpW#$e*|m zcjSQFjAwEzF2?aVCD-Dn=C*her{Z=Ob3R;-cX2^}$t$@S_vDbAk2~_b4PT-ytpXLO zKm{sLfeKWh0u`u01uC%n0z7~-aSOh~Q8*CC;8EO#M{o+B!xi`m594H9h(~ZOPQ;bC z85iSM{E2sPB~HP+I18WQMm&Yba2W2xbvPOq<2YQ619Byf!_D{_m*ZKhlW{MOXYPwr zaYdfRceoFqHSWUyxE){OZ2X5K@h=|6zqlG_;$b}IbUJZ7{>LXd9N*zu zT#)Z@DL%&+c^U`gVVCo|yzt4~=AitEdvZ#y#1}aw&*EiV^zj_`t?A0&xYdKXpW|^; zzR5i~Du?5i8@@zaS_LXlfeKWh0u`u01u9U13RGbC1F@a zx8hBlQ4#uN68OP#me3EZ+EY8Kv_#2nwc)R~1C_n`&P=N|mpaK=B zKm{sLfeKV$!v%N%$KV?rf;0J@&ms5>7vfC(hI4Qsj>03j6CYXqfn#tU?!ce;7N_77 zoQ><4E8-~Jg#U0TKE}Iv32)*5AjBGUc8bQaWmfaY}|=2aTsFD}Vhc^b#! zth@grC_n`&P=N|mpaK=BKm{sLfeKV$!v#2p-}^j=A8{6b#4~sbhvGFnhuiQmuEmEq z1yA5-oPnS42=hREhu3f!&c=^;3J2k1T!rIsD9*)McoLuCWL%Fg@gknb&3F}O;wyZR zL-8k`#^E>*U*R{rkl%4ZZp3?d8>ito{E0{ML(a&%?v5+*#)soyoRTA6&HWdPYjH8& z$-g)ucjBBpi%;=2e#tfYAqV7@obhDV!?(B`zvN`xjMr`W5^ZS}s6YiOP=N|mpaK=B zKm{sLf!!D2Kiq0{CcpPN50~N$yoD>7%i#r_h~un&!a4X0zv4xlgZJyjI^M?PI26z0MEs1u@DP5)-8c|e<31dP)0~a-@F*_C!#Egseu@IOw- zm$)0(;XwR|Q*kmr$gy}IkK>IT?7_S~cj9Eciqml>PQ)d79WQ)sj>jMQ((7|S&*E1+ zkpJ;Cj>#YS9_QkLeC~p~?f#pf02Qb}1u9U13RIv16{tW3Do}y#7T`d9$XpIr;WAu` zS8)n%!K*k6r{X-kg}-nrZpI%t6EEXpT<18RahR=nE z;x+t+-|;o>#k=?k|Kn;risSGw4#dfL9B<-1JcjS_GY-YUxFK)kb-a(`@ixxJ5&0j7 zyY{T<&XsrXqO<7AwQgYmJ|tN0oh+wK+G z$|_KS3RIv16{tW3Do}w6RGEn91Q+8le1&Im zFCM{V_!Q^jZ02{k760K+T!^P|BR;})_!f`jHk^i^@glCp0r?FV<0Kr4H}M@V#RIwB z>RkMeBXS%r!>hOrcj9W-jy3+ok$4-=<83^NJ8>p{#aV@ULf4CNh;XPc9XYnY`$cy+Hk2L?q zzploaI3@Stjhu=n@<#5(nK&W;;#fS1AMq~k#;dsKan{4DcqpghVZ4!Bt**r<`4^w# zX1r~?S7Ng`ALQaYpXNf4CCY;(mOM5AwOQaX%i!6?qXa z0IWqYl%XqjEJK$X|IGm*bNBt8an|RG(v)`@HQ^VnRpau+vKG#ff+zhdiCv=V`p|IPVL$PJx7tYAH_zwr;Ph63^UAyG0>vXm}lIL+h zPR8dr6c6G_9EfLeG#1KF)bQ=fxR09QWdTJdk&C zxxNW1P=N|mpaK=BKm{sLfeKWh0u|V90q(V%m6sO`{ypI#| z9S+3NI3c&f0B64&Ba-0Ez+ ziA(ZP{=}X56>mJu^LQJNzRcHD*!y*B5|m-r6<<0l-7e{nbd!ngPm2jXn}hWGG2&cm@d94F+H z9FXsELhi;7xg$s8T#x2FITw9?4Tr z=KizUkH7IxKFbaHYTpDEs6YiOP=N|mpaK=BKm{sLfeLK505{+Zyou{@iPP}_^H#io zS8x$t!F%`*_u))@hA;63j>eC;3P<4){D$lB6+Xo^j^i0TiGy(;-otZv318tQJdO8o zJubypcpl&3JzRxTaYC-gwKxmM<8YjaBl1HI#_c|w^W{){h0Pfe2U}nKie<2 zujF`Kl#lU4p2xMW=Jh!hms|agt8vDQIWLacH$ep|P=N|mpaK=BKm{sLfeKWh0^2RX zad;44TD^vw@GQQ;clZ(i;VpcLi*X^2a2S8!R{VvZaSZ;&$9NhK;xrtFV{tXk!fUu4 zH{wXs*e5@F4!gb9fA2;Wb>31M)LI$HRCbH{w8ih|lpf4#~CnA_wC& z{E4&it`Fz^;7Qz&Bc9IrayuTzhj`@49B1_;e#oVG8DHd6&*pX3IX>>iC%GNp;&hyh zQ}H+M#npJ*cCXM@R)GpspaK=BKm{sLfeKWh0u`tLpW#y6il1;C4#Y=z2k+rZtK)DY z4#TIo57*%doQOx6cj8Wbig)lZ-o?}S3Mb-d+=+wmA8x~6_zK_SO1zC9@gY9Oi5`yM z@HL*tXLuBc<49cZZ2X1`9_M&`hL7O$KrMzj|cWmP=N|mpaK=BKm{sL zfeKWh0u`vhb_;M7?!ke00dL|7e1!+`4)akwfJg8pPQqch4Ts@YJcB232yVqwI0@(B z9Q=y6a1)Ng+xQk=;v!syqi`DD#XYzP-{VhwhhG9 z4bNM>ilgx*esdTfI+-oUbv&2!G>d9F9A2BksfVI2E7ahg^`8UClbV5Le`P9EPv)MJ~r{cn_cCXdI3saWTHg zw>S?Md^qcRG+Q3U3lDQYzvG1*k#C(Yb7nm3Y;iko#RYjN7vzon?*H(zIUXlFpW9rl zZ-NR`paK=BKm{sLfeKWh0u`u01-4s&&u|D1!U1>*m%4V1S6=`CZ{b7yhLdm)KE)Y0 zlDQ(j!?QRN$KXCZi<59Be#J%j2FKt<{DKeh5A#EOjSq1!zQjE^3fJQu-IvdeKI zp2neg9RK8}JdiK)Gj4h{>*t|7?2UPyx8`xa$nP%ZabCyoxZQTI&{kG~3RIv16{tW3 zDo}w6RGg`5uf6Coa}rYiEDB@p2wYd7B}N!9FAY{ zDt^b)I2oViU%ZUVajm`yDo}w6RGY z^Md7$KgdhjPr3&PRPHw9cSZJeCh5Smxu8xp2eeh7`NkFypMZvE)KWdE3}nW zpaK=BKm{sLfeKWh0u`u01uDQRI2E_yK^%(T@F?EGvv>qw;ZuBx3$6aaceoB;;U)Zv zd+-^K#a;LZx8WuniNo+5{=`o>9OvOMoa}7;gq!dfZo`AP4@coHoQY3y6Q0NGI16Xv zEqsjM@UPVyxz@?}5=Y`pe37$pOMb|M_!+MI3{t z@HURbU3e4^;w)T>zwtAE!`ZIJQ}_$t<1D;}5Ah|A#(8)Wm*IUg?62CH^cYmJ8A2}Hp;(EN3w{bR(#Je~bALEN$lIL+XF2&!tAh+a< zoQwlX>XcjQN0 zkYk;Uqj4(k#Kkx$XXB^mvux@*`JR+p4)vBRG2Dr{bjCl$UZ$UdcJR zByZ(l{F7sGO`gYJKbZH6hjC4=y4@?Zl~teu6{tW3Do}w6RGLZ zfAA?D!AE!y-{C-9g3oX;p2e*=1-IZIyon$2E^fqgcmyxvVBCvCaUPDr>3AH+;$A$D zPjL?J#p5^-@8dSSh1+nc%W)ts#Jl(#*Wx>zinnkge#CD$62Iem{ESoaM)O%5kYDjH zPQ<-V#(((agSpL}xEBB8UA&G%@iE@Sp}5@L>BIXj<~FzFqFnNgdEDF=ui~9tkvsB0 zzQs%XCa6FKDo}w6RGKV7!Hwa32nG z9RJ`ee25qE8xF=bxCcMsO?-==a5KKaGq?(e<1-wMlW;RG#fLZ>$KgjDi=Xfy4#ZxwEzZYhcnm+}a$JT7aXrq)=XfLM;+0&CpYbOC#>2QC2jpv9j(c%YPRJK| z5l`b~JdJm8F0RGL4)eL;lDv>-axFf^4fz!x7@BRMg0^EY%a1Tzwp*RkQ;#T~L_wXYw!(I3e|Kd74hEMS{Uc^~A7U$w~ z9E9_5IxfR=cpX>bD13*na3!9^hd2oz;V0Z~bw8fOp*R}v;XxdVFYz<($GdnD7vg5z zhr@9n&c+kDB_HE`ypTt7L(atuxz_5BJdh)D!w(ma;&)dcPR76Z6Bpx;Jd9`Y zud8`IF3A!5Ca6FKDo}w6RGw&c$E2 z5KrP_JcP$^CCZa4gQnr?>`pS)Gjo@jFh$$M_GI;vSrcXYm?7!s&Ps zKjTx}h@Wx1%ki4SI19hyXIzY1@fTjjo4DP{9EU5JTjDyLhrd0X{rM2b<8V{tT|$yd1*U*vJTi>vZAUdFMwA{X24723)wP=N|mpaK=B zKm{sLfeKWh0u|sZ9EgkX9$v(EcoHAsUwncO@g~m2gSZY~;yql91Mv~Q!qqq%H{o2o zgk$jz4#%B%8Q0(=e2!OfFs{VcI1Yc~cf5qZaX)Uyjkp^(<5YZw7jYb3#J#ut|ZMlQy+coC1|MI4I%aW_uIGr1jC<4_!rE1BctRs4{<@iqR$ z_c$ES;*1>iVtkOJ@vOcHDo}w6RG4A953WEoQ&h~IS$5;co+BKm3)nB@g+{kp*SAr z;zV4IYjQTO$JO{8m*ZnxidS+iF2%!mD(~WnoQr?)Kt9FqxD{{XY1_R*TUiAvP=N|m zpaK=BKm{sLfeKWh0vw4?a19>7F?ayC;Vb-rCvhB3u(}aP;81*v7jYkRKD>r=@i898 zA$Sev;A9--iMgBk`*FO7FL4>3#P#?JPvd;thDUK3^G6(v2XYu5$Y=N)=ix$p>NxA= za-5D+@fwcB`?wXC;&pt8pK(b3#IHCScjJH@j8F1R&c*{d79ZkBJd6u+I}XUTI2HHe zi;t!Uzv6G_^Zs%^PRAXu=6M|U+Fj>)eG^om0u`u01u9U13RIv16{tW3DzM!GJb+_x zAHVbY7DwS!+=6596;8r|cnvS&Kir5DaV-AAeYgXM;TwF2=Wq)y#xpn*uP}ebjV{M8 z_zgeeO?-~SaUX8NN%$7u<5~QNw{aUz$Dz0v_v1*MkneFS^Ge+9IOoBa_!0NxIQ)v2 z@kH*&E%_0TyqM#2Ee>@uF2p}M7q{YC{E$O&(r0sAPQ@Yl9>3&-e2nk$Nsh_M_#J=b zW8906^1tm~p{=X}6{tW3Do}w6RGWq;8Gqwpyo$eZJ6_4X`X;DA1u9U13RIv16{tW3 zDo}w6RA9RWc*;62#i3T;;z`_wOK~qg#JRW=H{l(8jEC?mPR5t`6DQ+UyodjA9Uj6x zcnsg*CVY)A@f{w-b9ma-I1LZtZ`_W%a5qlF#dsT+;bi=U6LFPC^LiYMqwznk#i2M2 z7rPwK;!Zq@Gw~$O#N&7lN8@ljkXvygp2w;99na%-{Esi5%=vLEj>s9g8b{=De2iyt zJFdm`I2G6AS=@_f@xJX|p{=X}6{tW3Do}w6RGG z!K1hV7vekSrZ@$6;1FDjPjHj7aTxPqT!**u9FD_D_!DR1F`SKeag@t>9bUvecov7_ zb9|3$aVkE>r}zlJ;(Z*62k|T3!*RF_r+YH%oOJdbI2EVkpB#((@hbktnYbpu;$u9NOC6^RPvdf&lh<84=B$hR^-WNL z3RIv16{tW3Do}w6RGce#5(X z6CdGM{DV7i6+Xhpcn9y{Nt}uw9mX-Z2S?*yoQ9il8t%rMI1NwZe4L9H@h*#xiC*fcGg?sTM-oa^j6#wE}oP`TbT{sd);cI+~!|*YF#aH+t-{U*HjDK-1p2wZI5oh8;{D@O= zsl(!2{EyRdB0k8YI1>NkMVyf9@g?5IA-UjT-VfgRY;K>;md9}@?!-g68t=NA^WlkH zic|3`zQ_Z)THgc}s6YiOP=N|mpaK=BKm{sLfeLK506*antJiQIKEM|^7SG@Xe1%8x zF^<5Mj^hoSf=}=zzQs+r0~g^)Jce^{7mmVncm>blSzL^>aV9Ruvp5VF;v_tVCviRw z$9eb>XX7v2i05!B-onLBrWa@9J^b(SxDVIkYCMW>@j!0G`}iYg;%mH(!|^%J#($p7 z>zn)Hn3r=u@8plX?l_OXIp@dAxFSF0q1=#L@vOrn}SMfF;wmJ@n;aFUVhjA7@ z#8LPX&$>Ha!lyVCuj6w3hF9?;&c%uN8~@`#Jc+MyFuuoUxEF`wK75S}@-NQEo_Zq;$@tQ zTX8X7#kDS`!*;LGR#t%uRG>&zv6aWhkJ1-Zo`K-9G~Gh zyo(3%C!WLKxDtosL41YRa30>s(YPWPtyb z5Le@A{D;SJC2qxA-W)IEbzF+Ga2tNd#rO$_;zk^X)A2km#a}oZzvEB5kl*kip2qDs zBv<2ToQt#ZLk`HhxFzS}OPrBA@jE`p<#-o2;zJygZ}B(&#ND_VUwb_BQ9O&U@iRWT zdf#E*KTgH3_!uYUVB5VyTUiAvP=N|mpaK=BKm{sLfeKWh0$hjl@Frfvm$(MM;ud^? zhj1bO#B2B!x8Yx$jGyryuEZhu4X@x!9ECISr}J?SUc=vb6K~@`oQFU07!Ji#_!WoY zUR>@tZo=ER3U}i*rFukGt?B{>8iQ&+~W`x8Yixh%a&~-p6xzA`j$# zeC=w^lQ(i9{>k^a9w+0WT#R4wIR3>a`5k}bSe%Q~@iGp`xi}g3<6c~gEA~xLfeKWh z0u`u01u9U13RIv16{x^=3-A+;z>)X@FW>-v@ADd-wYmi+AxU-oR@(3g6)*JcZBjAg;v4c$7IR&cVly;~m_K%kUS@#+~@c>S&ya zhw&%A#IraFr{P1qjOTDN{>FXy5f9@n{Ebg>E>81g{Da4EjT zdCZmZDjvqC_!{TqXuOJd@;L6q4_}-2ix%q9mnH>yo*EeHcrYbITxSeuAGl= zT};=$2`W&53RIv16{tW3Do}w6RG28}Hy89Ej)eFmA%pxC?*cC|rm0a2npnwKxek z<669r8*v*R!|!+<|Km|ykQbhh$8bB&$d`B}XX9|(j7#w^KE>&{4=3bj{OgT*eQwDE zxgU??bNrHjayUNqc#g-}c-&zejC*l2zIi#%<9$4f7xFpYx7{nWl~teu6{tW3Do}w6 zRG!dn;9DGpYw-!*!h1Lf zkK;ldieK?G{>8_53eVzR9E{6wIex{VI0--FTpWsLaTV^w4S68v<4c@{-*G-}#HqL< z$KpmDh=cJvj>dJk5Z~fVJdXGAI*!Gy_!$4sK(5`W`*9E!*B zAAZM;xEsIXavYLJ@g!cy=XerF#U;(9@imUhy*StEeB5ojS7jCb%5p2Nww6#w8`e1#{OL*qOAgro5@9=7@rzv4B#hbwU^{>5{6 z8pq*y*Df=j#rgOnH{@uX&HNVE<4e5i^*J8r;#1s>`<=~P6DQ+}T$3Yl%9A-R2jieT zj0^HsUdmB96=&mz9E~6HJO0QC`zEMB1u9U13RIv16{tW3Do}w6RA9RWcmn_7Dm;ew za4X)#tM~%n;s$(#XK@(Lz`uA22jdj{<#HT^UvUv0WRA)_5BK0&yoM)nGmgY%I0{GO zL41TS@fnW7_jnBd<6)eKGw~TN#Oe4B$Kf;PinyM6DE`LN9*qC+HeSai`4r#bT>R>4 zp3mhtBM0PXylQo{C-Xk>EpEjR`4&IonOun@@+LlcF~{Y3e2=g3Cr-*A@6PkMBS++T z{E9bj_X=%g6{tW3Do}w6RG<-@_@hGmvoy3l6}cs!<3(JLkMT0z$;r;icfMa9=3WD_v4lPkDqco{`uj&&;0Ld zZgWY#dNJ$bf&7(6a#B9K-7B<}RiFYDs6YiOP=N|mpaK=BKm{tmPxuj^;whYi53LTx zwaoKyBCf;1xC4jcM0|)>aWr1TuXqrb;zay~PjC}1!H+l>f8r_JjpOk#ZpV{293SCx zyoM8TqmywJKEvs_3qRviT#3u^70$)IxDq#79m~8G2jW&-kjrsLuEyW^7%$>hJc$=^ zEWX5_xF+A@ejJR;@g=@!eu_i!ME=F+_|y6L6tCoIe2iQ1RKCXdcpbm&o1g*}s6YiO zP=N|mpaK=BKm{sLf$bLXJD&sa91g*o_zGv@8vKG!aUPDuPk0D_;t(8#x9}q##c_BA zFXAISh7<7#uEt9qjVJLg4#Pz*$3q^>e%yoG@S3yPkN5EPCp=i(0hgE#Rap2HQm24~?sT!TyTEB?Y~coEOwDZGZ$ zJRDcyaQug_@hN`CO?V4u<6WGEmvJ)Q#BKNw@8f?QinE=L$8a3J!xwoU-{UkKiL>!9 z9>$w^4(H*2yo$?lO@7Jy_#~I&cHEJF@v!qbA3n!9IUC>PgxrnOaZV2S!MuO`>11wm zEk4RKujX+s$qV^m-vkw?Km{sLfeKWh0u`u01u9U13T(FkN8n9dhvV=aZo>~a5SQRd zJb+hmARfh^coJ9P538SCj(czvUc@1I2`AwoyyGx#!8?v~n`dwre#1X_7^mYPJdPJ} zC|<<*xDtQkO}vV;albcbU7U#ja67KWcQ_yq;(k1cV{sr3$8GoyPvcU2?AjrxKVRc> z+>&2$GQN2_&*OT$j018~p2neg-{X1yThoPOaX{Y5CwUsTG|BxwRj>A^!OgfD58`Cp ziVty=)ou6_x8ri$ip%gHPQ?2-3jg6~yoaxFHr~UDI3FkEihPUi%y zNe;;Wc;Lf%-s*-tiJS2t-pA*7C{N^We2GtTHXg>K_!TGQb3E~C*2iJb#<}<(7v+?^ zj?eP0?Ovg+tO6CNKm{sLfeKWh0u`u01u9U1KQ_PjZ_oDcjtBhAY`-@5|HZle&fNZ~ zx&5uV{m5-{@1zxch2@JbDV!OkN>{8|BJKzA9Mc?&+Xqb`~TG3e(Lyh z`_A0{o3s4`vwd$K|IxYqgL9mB=l(u{6{tW3Do}w6RGfDf(i z!+ZD>KjK@wi*xZE-o>{#702O0yoe`pFP_G!I2JeKUEGPEt^UQKI1xAFaD0hVaWF2% z-#8Zs<8b_n+wm)o_VeRYJd0a#EDrYR?@o_apaK=BKm{sLfeKWh0u`u01u9U13h)-L z#DRDb@8Mh=i%apW=gZs?-{Czxiudp@9>vQz5>Mk`oQi|-A-=@Tc+uPAM|_MM@i<<_ z$M_Rh;%i)vtMM?-#qW3*C*x;)jJI(xp4BI?0u`u01u9U13RIv16{tW3Do}w6RN&JK za3@~Ft9Ta2;zj(58*wL2#f$hA592`Qp7;_!<5C=n197f<<751XQ}HOywYnOo;$ggr zD{(2_#MihMcjIi_ia&8R?#17D8aLx!oQr3D`n%Jk6{tW3Do}w6RG;#vHMQ*jx-!?*YqXW~%&iIZ_9KE|!M7?&B6iI1%=#*H`=pW|bz zXK^fZR=kXV-5CetaJ-9eaV_&!T#Hw6Hm>#A-lg`cKm{sLfeKWh0u`u01u9U13RIv1 z`z^plsSsaT?aUt%+!#LKxIW8CDU*^2H5trjf z+>F2RE*{05I1>lsa=eMNt?tGD_!Q6Las14@8IRlVC$BTFKm{sLfeKWh0u`u01u9U1 z3RIv1pRE9Q;yhf5OPNFBN*swt@h_glxi}R^;z3+$^)imctN0S%;!jVTw*&+?4q;?saGJu%~mIFYzdj#>sdZuj6t&kI!*1zQxPlo&SBr{eJQ~^9od;0u`u0 z1u9U13RIv16{tW3D)89~@EzX5oA?utG4I5gco*N{L41fO@hZN>g*X-O;zoRj_wY21 z#f!KXzv5YZiGT4fzQo<`j8pMB&cx@;qj5I=#rt?3SL0Fq?7QPr9FTu;FYfo*-lg`c zKm{sLfeKWh0u`u01u9U13RIv1`z^q2coh%gMSO?%@T_~|KfHz?@h$Gd!*~@h<2gKu zGw~;`#i_UvU*ck%i7)Z4r?Wne#@qN9&*E?Vj!SVe9>>Et9*5#=Jc?KGtkvy!7vI|N zC$BTFKm{sLfeKWh0u`u01u9U13RIv1pREAD;!b>s`*19-#i6(jFXB=>iidG1zQl*P zlzA-9#ih6pALC^lilgx^-gIZ&iZ5{`KKAy^MR7SEwK^7G;$XasV{tHU$E&y;$Kq}r zjJI*D&-O00R|P6ifeKWh0u`u01u9U13RIv171(b9F2t{R6Sv}3yoU$z9v;QJcnwE08{gwvT*~|w zx8hx#ig)q0J99q!{p5A#6{tW3Do}w6RGE!?+h$<88c%$8jg_#?QFa>SmmYdvPl6#nCt) zFXLF8j-T-?UdF39+0*8)_IuHF<`t+w1u9U13RIv16{tW3Do}w6yi@_s#C!M(&*C?H zhdc2aKE!pn3~%CByom2`Ebhg3co-MrWPFNOaW}5Uy?7UQ<5j$gPjR?=<4Ampr*SY2 z#lQF&Z{t(noxhLx&TM&|c`F{q_xRgOy;<#CfeKWh0u`u01u9U13RIv16{tW3c36P7 z@EqR6iFgZ-;Z9tL&u}BI!*h5L_u)`HhaYhw?!>!z84u%6T#ch~Ebhj$co^^EM!b%{ zaWziGuecol;%V1TH(qsT9E=BYt<~q=p8emQ?G8V3opJ>#P=N|mpaK=BKm{sLfeKWh z0u^|v0(^!KaTR{WU$_?k;X=HK1FfFKb2!xMK=;OdxED9#LmZ1g@iTtKop=#P<7T{w zx4koN#L2iFPvdL+jdO7=j>p^h9iQV^oXs2=2jptz&0gxwYUc`6paK=BKm{sLfeKWh z0u`u01uC$^0(^)gaUEX8o45|I;z>M+*E}DO;z!(w>+m4XWNwOwaWgK&&$tyQ;$o`@ z@ii{RulU-v*NtOwC396AjKgs~zQxNp*3-q`I2T{zU|f!Sz1#fN4zIROxdIiaKm{sL zfeKWh0u`u01u9U1n-t(p?~SMMBW}Z&cnpW)I~<2+@fps=lXw`1;#K^MQ*k8TbZ>l! zTk#@(#ou@kALCM-?Ap_o`755qrMQ>*DDK3;cpT5-dYp>8nY-d^T#WN^uA97PJ*NT{ zs6YiOP=N|mpaK=BKm{sLfeP%Y06*ebJccKk$Kpghi9@aK#j|)4Z{k-xjQ?;W{=>WY z83*G>oQwnUB0j~P_z-vFQoM?D@hHy4o%kB3Gl#~b_!zI_ZnwwBc-5WpHx9<#_}rd8 zYn^KaDo}w6RG8C)6-VQ2yojsuDDK9wI2pg=OShN#DNe?(xYzS>Htxr>xEz1u zSX_^>3M*Wz5OYw;oO#D_Q)pW;Ovi8paK-p1EfALCVg?D=>S*WyUriQ92Ce#YUr7-!>G z9F2?dDt^TQxfk!+(`T)7tw04TP=N|mpaK=BKm{sLfeKWh0yinZowya};YNIGbu1pm zgSZvn;!qrnXYr=n^Y`?45SQXbcgC4G6>s8fT#PsIAs)upxDqepS*v&PG5*A-I1`8B zSbU1RaVef>{)}&}PR7S>@}Bja3RIv16{tW3Do}w6RGh<90lXm+`On#_6~k=i*s=`mA-X6{tW3Do}w6RGR()p)4eke#jkkZ>Rp`cyR$#<<5%2@i{0cs>p2yuKm{sLfeKWh0u`u01u9U13RGZE z1-KFa;ZVGZGw~vx#JTtsN8(`o=g!PE@veJw+nf|X<5fI~mvJ6W#+f(~PvcBniW9Al z#jiLT593hWj+gN`{>82MoH;GN#m`o^;#qu-gYD_F*11-o0u`u01u9U13RIv16{tW3 zDo}x&6yQkQhFftSF2ixS4v*qM{D?DgBwodtcohfZUYv{na4N3E#dsO#dOjY;gUmUt z9>>%85ijFU=CrsMuX}HqqvBosjPLO*?#8+J8%N_{JnJU!SdW6vyE;oQg|vB<{n-I1(@8Qe2AznQP)~oQz-b zt$X8IT#l>pI-X|Uj5~2MKF0BQ6wl&d9Ez9mGrq;&cpv}bSbO@cb*>etKm{sLfeKWh z0u`u01u9U13RK`G1^5gv;z!(vYjGle#g8}<_jx*Q#Iv~9ow=W9aW5Xkv3MD0;!%8v z1DTKFOk9YI@iI=u$9Nsj;%wZGukkD%$DueGr{Y)~jIZ%4?#8Ql7Z1D1d)9L*P=N|m zpaK=BKm{sLfeKWh0u`vho(ga$-o$Nq71!cUJc@7eCCg?JVp z<3{|7e{nIM#jT!>m+>RM#Fe-c*Wy-Oi)V2rPR6_V8Xx0QJdjuMG`_~cIM<#&Yn^Ka zDo}w6RGSq+wiXE%iq)EGQ5l5@Gw5atvD9n z;%6L;Yw@Mkr#KWx;$HlSTX8zx#o@RVU*d3_if?f`ZpOj573bn*9FEuVH?GIWZt|Y> zoC;K+0u`u01u9U13RIv16{tW3DzK*l=BW4&KjJ<7iWBi6K4s2{fAJ-*!)-Vb|KeI) zieK>_j>N|}6er?KoQtP%DDL&n{P(Z(C+@_TI33sGZ@i61aV+k|vAEv7aXc=^&-fg# z;&&WtPoK5UwE`8WKm{sLfeKWh0u`u01u9U13f!as_u)i5h#zqxuEeLf6c^$*e205+ zF>b@HxEUwoO#F!-@iKnKnRpWS;%7XGi}52)#oag^Z{k+mjh8(gm*Q32iI4F#Ud88l z6}RJH+>E2$1>s%{PfeKWh0u`u01u9U13RIv16{x^X3UDA!#ILvu*WoyP zhI{cNb5)#*5AiMz#&tLpui{P|i;rp2yuKm{sLfeKWh0u`u01u9U13RGZE1$Yd{;yJvC zLvb1&#g%v#&*4;@iBsJh*WpCGhlBAiPQ9-q z8mHrU{EKgKG=9a;_}uDeoNP~@wa&Ey6{tW3Do}w6RGW3W1PyI6^G+e+>OU^s(0t_ z1#%{y#?3evC;MJr#kY7DU*lC=j<4P1J?l9Ys6YiOP=N|mpaK=BKm{sLfeKV$PX){i zaVgHiuXq;U;y`?fH*p~D!*}=(C*xpTix=@H9>uG86DQ+YJc*axTm0(U-NvQNm91{Z z*|^l|Pn?OH@h~pM?RXj2<5awkmvKB^wx`cp=URaZRG;Y3`CNAW4n#ml%5N8)L`izo3cF2uih8BgL> z{EV-yzQn>s%{PfeKWh0u`u01u9U1 z3RIv16{x^X3UD7D#br1X$Kp|(io@_C&ckhZ6}RCyJjQVajcuX zXFaC^6{tW3Do}w6RGV9Jk|C?=Ev# zJZw*&wa&Ey6{tW3Do}w6RGFKx=|KfK%i z#=ZCy@8Vh9jfd^&v(~v*paK=BKm{sLfeKWh0u`u01u9U1n-t(b9E5gKXDu$#GSay z>Nq@!L-8XH#&@_5_u^xGibwGyUdFR{5r^Vre2RzhF>c1iR*&Lt=C?Qx7I)%XoQH#Pq<7NDbyYa5ov-lV9<7C{9m-%~?d;H9G&K0OY1u9U1 z3RIv16{tW3Do}w6RN!R_@FD)gg?JT@;!_-m`|vIf#*KIr|KdV?iH~ue)rI)a^JNZ; z7x5&H#=AHecj8byjVEy?j&^5Whb!?d-o)v68yDkY+>O(5K7Pjs&24eFmwB(+w*nQY zKm{sLfeKWh0u`u01u9U13hc3fIU>%qIuyU+S{#cjajw;q_!HmZM&_q@4tL^6e2PQy zEiS~lxEUAYO5BP!@i|_^={OZ<<4oL%M{zG+$GP|y=i*s>Z*@8j#;>^A9zS!Pa|J3; zfeKWh0u`u01u9U13RIv16?mBfe1`L^?!sGm6360Dyyo7_FL4=O!G8zKfIbDvre6xD(GaU&ZZs98cqNeCzr66_?^< zT#kovwL7!_P2RJfQ-KOppaK=BKm{sLfeKWh0u`u01@=^c2XQR!#F6+DXW~BQl(-h3 z;!>Q7gYhPQ#Je~ZhvHORjE8X~PQ{T{cjH!EiA(V)?!>LQ6sO}{{E18Pr*~%VisSJ$ zF2&Ed6%XTDoG!oG(^b~FR-ghEs6YiOP=N|mpaK=BKm{uBc?xhN9>s^a6VKvUT#Eyl z*WyMzi}!FZPQ{D(4iDp1{EByRACAPWcosL~PODdOD2~L}xE;6RU>t9CDjvqYI24cL zR@{!eaVoyW?YP(HdGC5|1u9U13RIv16{tW3Do}w6RGc$F7ve^@XYPpG z@Tq(A_uzOBcj898hkJ1wuEl+L4_D${oQ$7wGVaE$_!NgSZ^gN|(du-(jeBt_uEn3Y z7hmI4{Eo-*tf$lE?dithI2O0t!)L5>tUv`SP=N|mpaK=BKm{sLfeKWh0-vVfZi7eC`yoQ{ieEc0M|i{EiA4#=;#*ynlgdTs?OP=N|mpaK=B zKm{sLfeKWh0u}gQ?EQbp=k=ZM_g|aS*6dN~#bhI8CqrgjjwH3)b*3KD?qulHP)ucJ z&-@@JX5t=fd$g{a(>XH|WCTY@IjmE|#bjn(8E08#afuMP9Gv4JgM`a0^?*tzI0*5d zj6&Uw2xl~)-RQL46Yn=E*I_9fb57gK=aN#$Kf=* ziTCg(F2tAk48P%EoQC7@9xk@pt+)|y;!ymHH}N$-#=H0#=i*)*iq~;BZpY>L7LVgn ztG$dv@h+an?RXmJTFTA%8o&EE{5LkgaXK&^m<~(_rUTP~>A-YgIxroW4onB81Ji-& zz(?r-AL3ISiw|)s-ou-?6{q4_oQjL_FuufsBDe2PPHJx<59xEhz^R6L8HaW9_rQU2Y{+3CP^U^*}zm<~(_ zrUTP~>A-YgIxroW4onB810RPD@FlLoVK@$#;zfLm8*w21!^8L%&*4YhiVN{5Uc{F; z5m)0_T#Kvu`=}@KC~n22xEn`$Hz(syyons6hGsB9FK2( z9R3@d-#8tZ4onB81Ji-&z;s|bFddigha*Q*kKH#k1aRA6Md1e2y>iyf3t$Tk$*o z#^dA-YgIxroW4onB81Ji-&z;s|bFdg_fbb#;h zBu>MRcoDDRU7Uwk`Fks_#l3hKPvT%aiwE%|-oROp&cS%xQhzV@arkd+e&cjtIxroW4onB81Ji-&z;s|b zFddi!B2@fwcBxA+u?;#>TPH*qjN#GSa$1Njd3;#fTF z-8_g>aX7xkp?Dgf<40VHn{h6##lv_Wf8uRCi*s=-e#N(V7a!wt<5wT$zuGxF9heSG z2c`qlf$6|>U^*}zm<~(_rUTP~>A=US1DuLaaTzYec{mMc;z!(y-|!%=#h*A8&*3;+ ziVtxgPQ=f+5+CAcoQg~FIDW>jxEV+KYW~H^_!S4^R=kd@@jc$h;ds;&tN!nBaWfvq z*SOxt>A$u4tf+?%{KMjqm+IzQw!#P5zzDnd!iEU^*}zm<~(_rUTP~>A-YgIxroW4onB8 z10Saja4Js3v3M1i;#|CmYxz5<)egj?I1tC-SUidM@Fq^huXqx7;$<9bwO{c!zQ)P8 z6Sv}OJdJblsMQX~?f4pp;#)k6pYb(r#j`lr_|?bhf6C^!P6ws~(}C&0bYMC#9heSG z2c`qlf$6|>U^?(oI>3>*6vyILe2ZssD9**Tcopa2P@Kr$M{y;t#f^9r7vo_(ia+r$ z4#mB=lE3>}?Oz=0-Tpp`EAcR1wc4Zj7;oZr9E!j3s>S}kif{2O{>HI5*+=U^*}zm=1g#I>1{v6))mCe2Z6cAA-YgIxroW4onB81Ji-&z(?r-|KU>{ia+rz z{=|j&5ijCYoQC`G9Ny*cq_`Bv;Z}T%oAEI2#ksfR{#{JWd8(}C&0bYMC#9heSG2c`qlf$6|>U^*}zm<~(_ zJ`Nq=LfmP!Q*k2h!(I3l|MK@s+=l0HAHKwUco%=-WL#;rr|~d8#hLgRPvT=dihuDd z-p02$8n5C^JdNW$z3TriJ742%oQtn5t@__1 zU^*}zm<~(_rUTP~>A-a0qjZ2@@f%LXk$4PW;ypZy4{AM+=>VBD}PtTqc|F;<4C-YhjA!Q#;N#~zdz$!JdEdkq2ICE^|&8L<69r) z-`$*@4onB81Ji-&z;s|bFddixE0=ixJa ziHmVCUcA-YgIxroW4onB810SUWoQMzc z9j?TQI1VRzH!tEi{EKUGF8;-J_!w8>Slo(xaVxIHz4#JG;#Yi%L-8qo#@U|kd-JFd zovV4y#+MfR-vi`Oyo=}YD6You_!m#(TpWw{eUyK9b9Oo~9heSG2c`qlf$6|>U^*}z zm<~(_rUTP~>A=UK1N@0&@gv^EiFgl3;#u5?Tk$5|!;kzu5|`p++=_p3EuQw(T!|;~ zDh|bi_!M{JQoQWl?%{Eqi%)T}2l6T&#hEx8m*Q7EiYI z4onB81Ji-&z;s|bFddiBQ?H=e}TKJ=;faXgO3#dsH|<8T~`>+vX_ z#lu#+7SG~)ALZZOoShC#2c`qlf$6|>U^*}zm<~(_rUTP~>A-YgI`DDm0GHuDoQUJ_ zAMV6?I2C8&I{yBMcX1$Y!?E}nf8t#HibwG=?!?#peHc&UVejTq{D`yhB7fJ#+jtm< z;%@wlU-|nlKE zU^*}zm<~(_K1v7p3g_WUoQTu#C|<;AxD5Z|QM`#~@g*L{zjzM6;!iw?cX1}}#Fuy* zf8udGji2#2j>g@17MJ2;yo#ssCZ5Le_!V#Ce7uZ*@w6}GYyST0qx`#@v(tg;z;s|b zFddiL^P8NcFN zT!@eHtksUR+KIRof8t$Silgx_-o)QH9S`GN{E4G+Io`&j_#UTvH=p8qe2`mlu<@&p z!~b;6Z=4QH2c`qlf$6|>U^*}zm<~(_rUTP~>A-a0SJ45U#D#bif8td9iSuwLuElHk z4bS36T#Fa+AWp`OI28BcN8D+(Pw_Uc#=ra>6bIu@9El(CIbOw=xEJ5!UH%@6S8+f7 z#_PD+SMw~+#pC$buj1d~+%X-P4onB81Ji-&z;s|bFddi# z)&9hXI1snuH~fj;@G#EAZ@3r_<4e4WgK;X}!_7Dr_u^2TjU(|T4(0E#co-++R@{vz zaXik%qxjX+c@)>;RGf}`aX3!K$2b~qTg(4#&2O6yOb4a|(}C&0bYMC#9heSG2c`ql zf$6|>U^?)t=m4+cSR9H6@h8s2cR0{u|M$vQdlF~jJiLfc@h>jMg?JK&;$D1>NAV=S z#?5#dXX9o3iktB=-o(qe8;{~(e2b^?sMSuzy?7h{TJ2Wc>sRsbaPF86Ob4a|(}C&0 zbYMC#9heSG2c`qlf$6|>U^=jtJHU7N5U1ij{E2UI81BPoI2X6#KRkyIz1!a}@f-fc zt9TZN;#L06id*q2f5*hnxD*HDah#1q@i{)m&$txF;%>ao-&1ib4#(xV6({?He2=H` zFD|y0|J$11HXWD_Ob4a|(}C&0bYMC#9heSG2c`qlf$6|>;8)QBe#M8l5Z~cL9ES(- zCvL^N_!Jl7U;Ktk@f|M2u{arT;%S`hLm$h*co+ZTN1Tgi@iXqmkvJ8<<4`<|!|^cw z#h>^Z$6D=RypLn8_OD;Xzr(p>IxroW4onB81Ji-&z;s|bFddiU^*}zm<~(_ zrUTP~>A-YgIxrphRdj$W@fgm+i}(vq;y1jB8}TAe#J6}Bm*Q6Zh+lCXZp5uP6<6bB zT#PS0(Epy^yEzhn<3yZ_Gw~>1#nt#2&*F02j*s!VFZ6kwj|cKAUdFX}8~6HE{5zaG zrUTP~>A-YgIxroW4onB81Ji-&z;s|bFddiA-YgIxroW4onB81Ji+DWe0c)ms#yVe97N4 z@fd!?z4#8_;yRp(A8{jI#JM;U-{M}}hX-*buEo80m%pFlR-BAm@i}hCt9TuM<8i!< zPw_gg_e3tmxwsy0<7-@tTk*hO<-g0hYdSC;m<~(_rUTP~>A-YgIxroW4onB81Ji-& zz*_78f8t2IhXe5>Uc>x6j>q3P6@TMt+=|a}w5RJ6@8V>9>sR^j za_*WAOb4a|(}C&0bYMC#9heSG2c`qlf$6|>U^=iCJHUDPkH5q6_eWfd|8Oj>#FID< z4_oay{!WWm@gSbVyZ9Ji;$nP@2k{}k#MyWihvHJ4jhk^bp2oE}8Q0=ryo|5$D30~j zRsP4-coz@ka^qKP@qe1;H%$ko1Ji-&z;s|bFddiPcnyc)IUI>c@h(2bb2t(&<2e5Ai#zcx9>t+}6IbFw+>F0*H@?Ql9_a6__!C#- zalDLUaVdVr-*^|#<6&ROvA7lY;&Ggh@BQ2OcRF`Y2c`qlf$6|>U^*}zm<~(_rUTP~ z>A-YgIxrnrOC5-F7QXq9Zoln+_;3I5KRNIZZ@vA_ga7EC{KL&3to&>9TmF{$fazb` zACUcP^MC(?d*YsjYkZOahx%&^|G~m9&vfwD7C!gO6TkL}4<`Qm&VTI_AN<4W@!wfk z{N?$@PcAI{&HwW^|G(|O{D1%Ze>3`5H+-4`mv;QA|?Y~=C_#gjotN-r)@&AQ>d}G50KkxsO{rGp+`T3;jz;s|b@c-`) z9PI1=qW|ydf71VV^P$Fztv^)t-syPj4>!N1@#h=ASG~1*W$T~o{x3FvY4guE{*9{l zQU6EdBh}6A|LyAS)vH>6t9o7Y|5cy2xA9HYlhvi_f8M_DbpFeY54P_8)>oVVjq0~s z|4jA9>fY90@4lO=r(6GA^$)9Gul|1ZyX}9h@eS42sy}P}nm)&~$%)omUN~c1eX05{ zoNs)-@t3NHs*kpQdE+~)%hfAdf4uR@>aREdM)fzFzohT`Th-mw51z#q7M^V1+3M-) zO7+jW@2?wgZM?mERvLe!`eyZ0)t`3m?&|L9_UgL!y;D8V{Qutg+3G*7o~zzcU97&_ zx&Nv0w(51wKh^kD^^NAex4qc>r>lFaKd!!9y{fvg^Pi{>KW}_>>;GHhyBq)A#(!LW zqWNn(zfkR2Z(&d4f3NyqRM&TIXZO5N{gdjZ_We%#{#o-os&`fYe*1pd*n9NnJNMtT z{-=%q&+0#K|EB6+HGjDAx2k{H{J!cBs;^Y{x9{ny_t)NA|9SiFsBWzORr~L6{M-7{ z{Qp|r-TFVQ?rZ*v_FriJNcDTI|8DEM8h^j_%EFggzpwRw*}^-GAE`dj{D0d1_Z#0_ z{bch$YX5&&{Yv$}ZvC0Yn;UO#e5`xku70igzt{Mi)&HgXYW4TqccFSi_08(_)t|Kg zSmXb=&;M-W_p86z{PD(H8sE`)TlL=RV(Y)x_)wqoeB*!6_>}o`h)6i z)&H{lcT_iY{+{Z8*8H`NcUCWUesA^m>Q(Lk_3DGoFL&;@8t-rXyN%`GrN;lLx|S?s ze#>-VIxroW4onB81Ji-&z}oKsXSlQH*|)0lvpmFSV>yw0&OPKFKE_4(4%g!dau8?a5vTin`H(|# zLO#K5uIu})@0?sEe=fJq53X#V+{%mOG){JF=Qtow;3Su{E(h-GUjD_+e$hPlINv!w zC2z{vHYYjjyQ6(d#?U75NhHTxmZKYq;=U=QzX0#vF`e{jhod zu(=C30k4+3-)uj3;$**kx%)S3-0Qja$;bSLH*+I?aYLWGt8xB*Y+ zSKHdhhfg-EO|+B(1IH#~?RaYv5Eg?J2?TIqZ7Og{Cc=C7*qKn}?N`OqWnA-YgIxroW4onBuQU~N_&&zU{oa;H9H^}315s&HddVy2O7xJ^bDxdJ8 z6Fq119eGjSmp{1*Z{kJrwfrle$&tK$ybYgubtJsos-|>VmbBF z)_IbADX;Slz9Zl9M$W{8_IDpolNaS&IbQCPx8*dskauz!{>bO#c<#ed`5%9igE=Vw z<3{{Jetx08eX+`I~&*f^ooEz{M6Y5h!-wIYlLzvl4XyLAO`YdCcQt;ZeM{AMo9BsdHRfSF_iXb# zpR;kLW8L?5_1^Y#EPl$By@%n*hui<7>O0-@`NltJ{O#(K&3g~SE4@$nRQuM_v&{UK z>A-YgIxroW4onB81Ji-E-vJK9UHFtd&VA%`dFDjVzw#QleQn?%Jndxj{N_UA_xj=w zH|89F+&nkCvUUD@sC8b=H6Co9bMjiQ#RV^GA0Oc$FZDUxg9CFQUdk~pwx92Fzir*W zqq?PieDl)A*HroF^Nla>9KX7+F~{I4T#DoI4?cCebNptdG55N)F^BOUgLD0U>sR!> zIP?9D`3jHWlbqfAgX8UA?!Mhs?(;_D!`0`i9Gcf&Xr3eTGET{#xa6n%-0M5Xi@5KT z&F^X7rp8B%^H+}iK=a(|(Z*lw9G~O12U}li{ZwN; zFddiORnZ%{DrG<6F$U?I0_fy zKirEG@uuy4PyWFp7W+P2?dHbvyIjjR0a$T zci?LBJ%9Lm`{X8mBLB;i^4|;fgEPL?`LoTx-1zavCmM4;9w0Bv-JFdpaX1ddL3kNY z;(a`A?#20j+J5<5uH&0ri6?P6KHA4F$l*^l!EN{=C%LwD z?#iR??t9$P`f_8gu&#N|_hJ5So@a1gxtQy{(mvkGAHLh?@HH+c*I(5-Z{jl?kFRhi zj&pD4sm*_cYH}o@t$z9B$01IN9;$xzhf| zJdwZi8a~KjFKa*dx!CvE(LAT;f$Lkp&^rI)QM_nh>r3swzsk3Ex6bDtZGC@rU-d|J zZ~G56=64sH=SF;wr}0_7#Lsvm=i1agd%B-5@-yyttabj!+c?)!_pGI7nfWc#f$6|> zU^*}zm<~(_rUPrg13P<;l{?QiUhOwrhnMgKj&-tgi#<2Xw{jOB;sqQ=UVOH5d`Vv6 zXL2{c*xG*XBS*<=T;@Xik z`|u4Ov(mi0DGwd!dvHWKQf`w+IgA|6hpy|qoF>QeH~CDi{uVc`mTMF~@kP^KvH_JlDKj&Es`5Sm<*%DM&l8s0 z&r8p@{&bZO@J5~~AM+b|`pwSC<9zc>pTp(&4BzJ8JcfhvJbu8bIMRvk2T!&9`4}QD8&*w)x zUz1aPH zlMC{k@3zi)8ZO-3dCq#ch272XYJ6{%t8&Sk+Q+S4Zy(p^qMvR4_p0BkE>$@>f90;6 znosU;Kd1A4(U<6qL4OY$qOwf3KI<`+%}rUTP~>A-Yg zIxroW4y>gP$T2+T`JRKHt#TZWaI|?&!i#u{yvi>=F5{6h}nJo3$j_RGEU9gp~7zmr>W9eJ_k1$j$;;)9%lm&vVH^ttzS!H(+X zRe7FAY;0cs=55@CugUw|?8(l3u_{N)_54Kcezot#B{p@Ahsg8tEFa@*Cpy2abGI~> zH}7dIFY+3m!x?zn9qs3$d~;{_^DI7dd-HsPqsiy|fIFUTKd0k0_xHJ%wayzk0*B#k z^7+Be$<-WFj^5Tf=f0-#!9HheW8S&1F$d>5zi6JL$n}5KJxi_2;hUT1FMOQ)a1<`} zQ2TinzvASaloPGA|Gnx{eLlD3i~NO~Ue*3L+Rqg@^`7Q0tu9wN+Kr9BUghgAbT2>I z(mHqkRO99Lahwf}ISz-~-TY$pudCdTQ}QZ)!}T8M^VYlP6CeBopX4)V+V|rs=jV=G zkXs#TA4lU?-mkpX=Xqc9cKffZ@>UMY%lOiU_VF;@$+vjo`99~J_IoeE+4vc^yx6`^ zS1SvLyN4t3rEA;3x41MnA-YgIxroW z4onB81Ji-E-vPeEr{u~-Jn!L6JU~8@f4IrU&ha9? z^nCZqqsJO^nio5_smj;n&Xdh^1>Ur`d*n>6B;WHRZXsi0A^Bjk-$k!a}T>JSKCzIcJ8+UrJd*x*=!MSeeKEA`_xEJ^2UR;P@@&+F9 zOrOUi`GY*jb@+f>f3okvZ{$GQY?x8XdGb&j8LMGhdB^E&>+)p)4< zd}p7xt$Vjt_chOB^6pUX$DY|L4>4#(suT!hQ-k%L;0N-6d?G)}xm-xjkt^lH1AU(SASZDpIgkU%o$`a+$Q$Hw zIYutKrO$b}Di87)d6>sM-S3v?j&)9cjPgdnh?#4CbAo-5R$mfrDo;&gvxsE$4`;aI#- zzU3;{^}DaBawUGk!#4GOd6FF6^n$$lOcPvGj^~UI_c{FQipK9$d4_!WUiWd3o7%_A z_BQ5hd{JIL+WPz5!%KMs=aScX0SA)P`71BE(7jwqp5`YU^IZGnZ#kJ;{ZM>^7*ZsrhFIIW;p2mFPvz^;t<&6AOb4a|(}C&0bYMEL z_B$ZY$^G)%>bxOO$$g%;<<=8DM|+<4{4eiwCvL@wJkNX1=Kyk(=WjWJL+}9khqHLD z=V)@CJns3QAH3D`yu2-6as;j<-<)qhCy-}(2mg^*txFUrUA7RQo*<#E0u4}YZ( zcfXGt@<%?#3pm&_otNLa4u|3~yn&nWiJhI7*X3k+oC99n_j;|my>maT%ISQM3%$@f zSKtG3Fn8V3y?l=Q@*nwB?)`4}a$^p{1vnW$;ji-Y*E-K7<$UgYxb>5LuVYm?nm7HT zd2S$Y|G0U+!C!eEf0DC#0>|K7ySs zU^=jtIw04`-8@C!mDk?vd6#3zm;6JHk~ifFekAX5C(qGcyCB!|3*IBQ$zk%c9K#p5 zg*?VDI#1=fawt#biktfP{+reP z-E+JuzssL|hktQ`E$!dYJ-pzB#@nkW+9$X2L$1J0IT`1WhxykRyKh7H$?<%M+iq+h z|J&AnUd;$=3FBYX1FP`!BZ65qZsG^E~hV#y3{?R5=f4J>R{Y@b1Q3>O$k? zDu?EjS2e%0$_KgmzCM@RbIoVk&!4#A_uGF%>wNq(&A-$9=RWblKRnc!tMR%go99kE zenay=tA46``RY<*4){p>?y4?TxhRMGLFYc-`q9R_8y{?Zw((kemYLr&9heSG2c`ql zf$6|>U^=k&JHVeV?YWj4a1c3&hsaYrh`UG?$l>cuLD<4jk#|Gn1bWL~kQb&kg)+P=Ww zIM!;9kmuw$xsfxT>;5y<*SnaP9cg~Cc}~bL<-|>`bDR^MlgBy2f%b8d^Nl&C+|M`U z-p{sAUYA2T# zf1vTt8}r%=&2tuB&igqlN8@9hhEs8O{&}*`;YfVMdj^io!=CSZJX+<@&vef#>>X z4LA&!Ho1m}=bfs`*?fd+@fcooWBd6HA9$>JUdt8u6HnkxJo4_& zZSQ-2x-sYEW}N8G_VHYKTn^`w+{51`eXDakVyVyBSLI(^j@QV~{w{@g$oo%rFW2Ll z{6Lyugetxmie%`@n_~Dz~%Sjg7&xiO3Z{k3_hFkM6UUas5H+0{T z>aOO`HRe%#>b}+wS2-9@yuSG_RXHMeKi0img+FmQ??;w8zpee8()$6P!@<_IZ$szq zX#94S%kd#@x~ctqmLG2I+zqWaT=2f*-sXSO0-rwKJV*5YWq12HE;o9={Tz#P@^8+{ ztvDXnyS95T>z-E|AFW<&o;!0n{^|Y8a{IaVT6&h5-!dJT4onB81Ji-&z;s|bu=YE^ zAvg|ikaKto7m^F)Hu;rr@D@2!PM6o@SNZJnp1U_zx!B&Gvw4Xe${n~1&yb%_bdI;l z7fa1^9r<6ry3jhG;9_#jf!2B2vyJ6O`Aa_9+3%6R)8y?Yf1h`Ug3hb8sjQbh3T&{%S{%x4VAf z7wwljKiA(QaFt72mlMx+kQ;K!N87i#%7>QQ&$T$)7n|ovoRy<;E`GYB{l~kXQ=M=A z=H}(rEsf=Fe!@wXI>#+}7`Nek+>|TwB;MffCHWaod9V9d=jPvQ{^h>ko~m5TZ+J0R zIVdOP&)kmJ@dX~X zt@B)vv#sy@-d*KLZ?>Oj@*D2`Q0ttPPcL^5ui+^?i*x+AeOs${b|0_c8@%$C_VHp) zbgp@>!M%7WANf}IbC0ig?wRJf4aefhJY=Q)yqEXzV2<^A_wh%5wxRR9WncTa3@`S+ z;mY>^pvu#qXw20vHr~{IZ&i66e_H2U>s*V6o@$*d?`j|4;%6MzdjJm3wJzx#Fa1$t zZg+2Eo^`CxU^*}zm<~(_)=~%L z6M61P&$)6Ox009n5BHG!<@gVCLC?p0Kd`a=H8{hspT|yN{1- zYb-a)7d%WJ<__G9-^lAvb&veRGvx0b{Z9FZ6Ueh1g^Qf-KDmtd$j7|l{`Ngql?(Zd zyv_%BnS8_}d5??8Z*nB3kQaHMT>oVE^Se*?xjaXn<}lod6Yx1cv)Fz7a8u)3x_4jo*{b}> zEiY}JQ*yPY7q06cPE>i?BYn=5&CB^*hCd%_-%IUVY5ZdKXH_1+<>Y>D$R{}oC;MsV z_jW&*;RhR9=QLd6aP#st2YH}-zT5iADhJ|P-1h0#xjP^Da^A`N_~rB6@9(epD8J+5 zueOgr>}kxu_#rRhE*ILzSvWJVzo&I>`G<|qRk;_h;%9HP&VxDg?&i70+4{i)In-6n za}wUMy?HLf>7Qx-X!n1!@hgqjH@>sVQ+VD|^PF;P=bmf+RO5H54>r%ScpAs$@fX_1 zw%6r_82gwx&`uxNFUXI44T3%S4U*%T*B#+C( zTu`2u_c#x4;6{9CN8f|9$Td&(MdUY*Ajk10Igek+hn$4hagulXJo!=1<1@U0AMheB zb!X=}9e3j~@){TA0X%7E=Qme>((mLQe5>UJ`SGUqan{G%AfL;ZkG1dm*5zDzoGbE* z_u9w#c%~eCuyy&KTm4n@{EVx}`|`3}%gMgdxp%AbJ+HXCc|OF0I0={C-#$*k0r|+G z`oOce&S#qEQT%`pu5|C7_HhdNp8ITU-?sK$tlnScmi&O@@DF~%?|;~RE1h3z%#(RK z7viS;_ju>t>KrfNKHHn;fSiMCaYL?pU;7ukm%DS1_04mqD;o1(zR!($&WZNjUFBL^ z8goSc$u$@IK0j#Rw#F|t=78Qa@PEE?vHkq>g&~wo!edISJyUP`%gIY3#S9qf$6|>U^*}zm<~(_)=~%LRk>T9 z^E~ajSMHIU``k^U+PyRCWe58i2>^B!*fPpaJXvyCsWdVj&o zI3utAQTx2l;J~Mw|8$>wePizTVB?k-PW2DoKP|PsmY!wiw@e471Ji-&z;s|bFddi< zto;snj+YPQJ5C{g%58G4oZ$Iip7mTT*Ld!KvFBya>v9SQkwbZsTr2SDUlMm(8^?k7XBLDAeo>%Y-c~hR1ueSC5_z#!7qkH5bF1Vxpa`2|cH?^NT@RZ}t z%f0-Dw;gF+E|){OPS-E+B!0r_IHkPL2jy6~pUZGgIg$@?#shsHzVXw(!1*d4`B~$~ z+Rvx>6))g^@-J`XKHTK8?&AggOWx)oe2zcxBDtB%$^9FbCCZS1&Zr zCwTdj%|BA*soNW$=$@78HPw5nuU7fSnZ|#&^IV8ma4ufK*>3C{r{l~#lGE{zPqlxs zx~0koFX{6?Uwx~}>3A|P_MYOZ_HnY^-TTSr&sI6g=RWbl1UKTse3290)w!=!&vozJ zjXCr8o9D`$mRl~ipObQazRbJ5H{qhI-HA_nKk$cruJC1CvIr{ z-p+CVx0~m~-rIbqeLrsf?&`X#_abjJ=AgHA?x8AoytaKmZ+`7R;mj|b4onB81Ji-& zz;s|bFdbM+9oXJ;tbET03Nr8y?D&Y_H*C+8!uP+)ANmapFGd~F7`e5wogvwA)Jf*@hQ2McRbTQE8W9I_$8l^ zqgQ(q_u$JUZW8J^6%7yqZKjY!Nf@AXop0%fYZmzEMd8ezK?(NpsRWEOyyB=$vlklxg z?c*67k|%mkz(qLKhR*RE-pRji=<|5|W$piFmHYC&7n}|}mjx=9cc%^^fa^8zv+x&G^UdxZqb}w&! zr+wU!V?NS8{>RZjnFHR@`cmhwZ~Q>xyBc3r<$g&&Be%=XH+4d*qoHdv4|oa!Zfh3-Z3_>O(Eah1^G; z;0SV*{3CD5`|{Sg?voF>gj~Q~xQx8T)#M1dj#s?ay{kP#o|NY~fn3M?cGMTXCD-z* zpSCWiJk{s%Jh@B$mTx+?Am{M|?y;#2>$^{$mGk6cp2O?*wV#v8kd z=H*W=@QdazR{0=b;y8SU6Y)QedA`r#43{+iN%!(Mj?0f8ZJob;y77jp{LdA3b}#Rf z^S{x4&MK$#g8N(lohk?Bhs(`BQRNFSH|CcI8*@{B{KtJS-oOjE>EYH-cK*sLpX7UQ zG{39;w={mQF*oJ*JcalDQTyJi@+v;?d(HDFPQw-by&13L~ zq569DXVu%Q{*LRe#y{vj9?G-${%!5!Y8;i@9_>7TtfkfXVmoWmpJOL2({m$(#f;C%e%MBnFp=j2_vm74JnZSloJ}5->o^<_;JEknz1CMb z#9wuvJk6ChH!pwk$cvrdSLMEZfaCBHuEELP?;LlKQ+Y8b`(mFjS92w9%NOKqUi^6H zcrRDx+q{d<^39!{6pUcgbm)aURh`TkJz{Pa|P ze5%SPxX+vIUupfx#ycDH22RV}u4|nSZEJi@b$6AkoNLUHZ*I(Sxe#yVFI@Lx-;2X> zC0@smIqe(mzpr|v&+{IFoA6EkeMS3Ts_y9g#_G?jw^n(-_a6L|V_n|9XR6%huEuZo zIo?BX%wx@SARhLo_IIA=ywW@myS;V(!|AxsqwVM6i|yaq{GRG9)ye|L;!NC*cYD8Z zS@&_J<;I+nv%lZ`U-SVNs+*eUq5O|iU*Ed-PTZ4EuC%_Eo@M5@Ob4a|(}C&0bYMC# z9heTR{SL^xT;kfEkLB@0jpc5>#h0#borlQtd}mkB$sCJQ$zdGd<$5S{TC*XwKk~^O7{*&Fa(pV1VtbF3HTj!vh;$-tY_hR?)!G;U_+Q$p|6F1-{ zT;UfEc20ih4Lo^&>+(2%*wj3~;&N{`&k6X*Yki(veyDv*)oraGY0P_g)XlALsq%2H z#Xoj*o0aLH(!6E{SWp%j#VG2avA=2uKnJl@MWIL zpE&xb+RxW`F+V)o`h`A^BXLo#yu0UC%%l16;pV;n;9Ivf|M@CkTl-Hq^9!c~(}C&0bYMC#9heSG z2i8&t1c?w5DDh`cFpaS6}C97T@Z({s6;At!pSm(QN-d73B4e_V?9a0UJ)PxCU* z>vEzzBLB&u@+aTnAMynkl5@F^+$=xxG0vF;D@)M7ctL0LD zzzO6;9?0#^*XO7D_j0T%ugTM#YJK|;w2#a15uU&?}il}+K%8eW92UnD{xyEaqm&@;M|F-JJs=Ud? z_|jvY=LdY1GjfAn-Lt96GnX3kKVHV4R$7<0U+wew3THUkJnwnA@n2LqEZ^W+9Q;n- z?-L*VgZ#}2xGf)eq5YS3KYuygJWuAEJeoUpY=PJORp&SW&*n2Xb>KvmTXWCF&hrZH z_NVRV6ugXwbLai-(w7JjYG#ulfH_zj?H|8?@iRWM6`V)OV596ZC&2tmZ z^LppFD3{@Joa(pQccgoG-&4D{Kd_;7o^`14_u9wBzt#Bh>ig~UK7oJor~U1FyM5lfaJajh=X)H@`;X^4 z&%1cta`T*y7jjO{#WQ#JIlP-E-`xD~cc1qIYw1~Le#>-VIxroW4onB81Ji-&z}oMC z=WMx;cX)1><2lK*JqOE0d}DR~kx%7c&++n_e8?mC0YB<7d%<(PoFNBt4t}t)1M-QS z&2cyZUy?uNFgb8Zss>UN1o?({DlkL(dWrO{6lV( z+qeub;U01*ALA=>E=PKy&*3Wa8DEooI0xV3k-UyW$c_9!-h8M2%6a@q{+Gk}lswAS zp6|RoD#!7a%ljS2+b36Z3Ep;D`{c)!#yhKWru?|A`_@-)sB#?MAfI!tL)|C8^O%#( zpXeUBlgIwF^{uV@`w6-Kt=730_mOYqb2*lSe7gI17?0s6T!j~LE>6ZXxDVIn4Oe$R z&%3kl$shR5H=Ey8{b7}h{-`mp;2ivgPw^~ypBKsJd}d4Ei$grw{al?te64x@_)udG zz}NW$xBFE0AFb}~+#^+f#@ViF{)^RbcaA%9l#9*tv)dbgrE{G3So8dIee2wcYy113 z^R4qA9{Wu5f8Kp(8*`c~n?G4ysa6&^^rKCDrn~75W*ZvdE{KDzLbYMC#9heSG z2c`qlfwj~D`AHs@>pVxxr<{q~a1^=O^S2x;C&>YFpS&X{%Qy0~=XR|py~Nx7o;N!upK-BETjwcq=!WL+>)fWQ{B^8#4kzE;(mdypzj?sn?vb11 z?suE#Wc=?;_wh=OC^yTWd{54n&pFKL?s=`c+~;wwla2W)pW{)ypFL!`_?t) z>D=!3T7R+1yEeDqdzgD$=RMxn>}uZo65hw0(=z;s|bFddi< zOb4a|(}A_$0eM%B+}3k1Uy|!N)S1>V>p7S|aTCrZ$8T!i&MJR+tMSF|<7m8vzwk4j z#eL)v9wdiz1%AP|_y@0&U*&Zk!;$3N!+lRVo}2Ky-)Nol^m7(g`-ptZ*SOys9gwf( zLHYdc*7?v%<43Ffi<9s-Ih+^B-TX^FmEU;5sqW=sa(6#-L5|~DPqx56_!Iw-U(aYw}C}#Qk^;*OYTP9N*+Wa-rPE8@bx?K9~Q}{UAa6&$Huyy&EtH{l~nL~02UT}ZsIn;^9oP`f8_x<@D_uwVm zkqdDa?srG`@+;0G$1k_eX?VbfK9{?3(RZ8Y2>gbxaupu;Soa^TURmWeE8WKtxgK9U z+xmM|-pAG6=|29rp)uFFuJP^Ff8ITOlYerayISW;w>G}C^E`uN@g$zbF?h?C&h4x6 zjLqG9vU+#bdxlrr&yy~+&gbrFokw%YFXv+W+t2MEYYDj(APKk8dqCexmw=Dxds$_wb|hjd>LB=16>OWBYj4 z$?mJ*9heSG2c`qlf$6|> zU@dh(o{|6MRr&10=lY&^qgFHk& zlgpm%bB|QdRpmyxQeKhw%H7-gV)CK<#{1+a`TWi9k?XhxAK+c`C4#D-`UvymF|(JZ*0t!_#JOqY+XL)PV%k1D#yvSD}CNW)vtGt+{fAY(fzGIUzM}@ z!uIar2|F9}4c^Mj`2fe3`{i`b&M~;&@h;-oa=e_suk~B2{CT<0 zl+`ddJn+cydU7;{F0|}R__V;oc98s?{|4W!n1h#lFx4+SLI)vk=uHoabNo{ zc79Lw&gM@w=6v3-cwey8KJP6KHRj2jj1zlr!nfAav&{UK>A-YgIxroW4onB81Ji-E z-vK#{PxQFFAYWhB^Dy_259P?aJHT7ywQbFF34ZZL^YWFPEQj6PItP-Uxe6zfqhILz zaW`JY0pv=#S&r-20`HN><&y(_53a}QIG8-i6CQ0p-{BsdimPnygE@$N&nv#!IzQwn zyoXo^_nk*DNizQLdPf&9)-p6Yk*?0d?Ia@;Rk-_bhH;Y4yFKam@` z5tlsHdA=#P$z?o>7i?-jkCRI|p1du;@fgneTHl**aYUXe?{gClz}w{g+xi~zH^<;h zKW&}k@k>s%q3^*1bq5b#sB!r$=0vwzB5&B z$-y3Lo`dib?!asIwU3)|3QqNOpZ8q#^(sf?8T|LX)?e;kuJV5KoQdynn=S3Alw()kOmAE@#!PQ}~rYJGj@HZze07_c!KGyW7tl?`VBn^OsaVTjgdvlwY22ogZ^tuEYU%cR#Qt_ zxGWDl*uGuuJKFeXjkz#4<6gXq&+@mm|AaHYa5^v@m<~(_rUTP~>A-YgEp>oP$!T(t ze8#EdX8Ge>&&`~KH}Dtvi&M0H!E-$K;UMy!T*}Ef0cVqoAMHXppGWWne!$^4k34u! z=eYyFd#ZW)oyW-$9E;CfR&F@eIgY@aQ zFLa*g@ihL+e{SkNPIIjB#_Dy}t32fS#(z=0tNJfZQmz9@xew8^WDyI|*D* z4u{*)KK{1cz6YA$*!VkDUbxsgkAA52ZB_obyD>*R-I&X+_9HIFyN>obS9Om6@u#KM zU#;HNKK^}e^St|P>z}W_Tji*0=~-re%XDBmFddiv!EAIpn!D;My*E|IqinNzkI!`3*<>oz^mjgd6I9f?;Z~HweH~?yhaY<0CM`> zotHCp1Rymbl$m1N47jvYG>w}T#pCw z-|M@FH*jg5!)y8TEqxAG<_7%nbn9T5* z#ednDXYhRf`18(jH4edZ`4+EO?mnLTgT~zSRAcWIc69$e&7Z6CjEjvq(^HK(>4nBe zt9<0X?!CRrjkmRLvHD2s4_5b9`4e9~*nS?vZTK%=(l#)q1Jr+rsc zx!{Jz-gofVCp-6NtuHk`TjidV-;twFHZO1T2f2c`$Q7RRO9OUo60K(Y2T^FoZ$Y(ype1CgXZ}jFW`s#>E`yy z`&{No^L&Ws@Z{sI^Epn-rMVyH<5nDnUvic=`+XeuK=;4jJU2Vv_>wB$<6}ID|L_<7 z)U_Y}-|ONNTw_!B@*KX(nK%PC=PulW3-JX|MTp_o#*-7 z?}pAlU%jh+JoAp``QXLI7u$EX`hN4gh?j99?si%Gw^hCG;QRcT=Wgv@?>CNgjz@hm ze?8Maj&`i^=JxZggXTNO$-H;qWE^^_^K0o@W`4_bU^*}zm<~(_rUTP~>A>3W02kmX zH}xDWr}UV+Aa~1Ayyxon^AheM*UE|V(8>18^E`w@@iF;DUXg2gg?z#vPIRw4B`3>? z@+?o{JRC*7l7m0Y9o;JrKGv9DaX)#A+kKed+Asg{Ek4B`xPx5JdH%Y80i1|u$Z0%_ zugR-&FIQRL_xN#@dvOEKBOl9caw-4en0NNQc-*<}HFfzES1e#DbG;PLix+uOV7)71xCzrQi};W1qLfz~;jCgYDyA-Yg zEp>n!$*KHE4wVDAjC{AQ=VopqZ*eVtB)@VF{=#E80_WP&y_|xF@Ep0Fv+)gHB!Ban ztGn-T&+T%lyeGf&0$%iZ`#B8Hk(;^8rQI)Qa=?x+$l>y%ydj_Q8ZL3ZgS>`&@hv{b z&Cd1tJnukbF2@IX)3>^xTk<>(!uz^rK^~RgI24yT*+zaR*YbiRt#b>xRerv*bvcb2 z%B8$b?&qK{^gH;2oXO!h5O?E7T!wqe<2;GC@j-c7?w9wk?R#^QryAed_u@73H$UP8 zkF}pO$djCi>&VB~wEzApH+ik`p(>x+*ZAe?&DEdw{dQGPG|wqH@%Gkv%E`um*nSRi zvGMMzoXuzXAMfP@`?`-?o^ns~T#zsFC9c3{IobZsy;Wv%mH&ck>3*B$NWy4;o1@)0ig+3w}lk2Ky`eW%Lpj&(nu+25GcthAp4zSR0x zn&*)`k00^qt?lPVyk~v$ypAvNFy6xjPxpBo)BBYdo99Q|=$Ym(uP#-Ws~4+$??T^) z2Xg6y&2wk(B{)5&eYA?vs6Axt#lP7CDi3a3r~NrSHdy`0TaK^ODt0w6}G+lk>=x@-GkJeH??k z@d^I(#eUEE>I+q_%7ysuqkWH+_HlzNn_sT-6y7TLZ)u(1$@zSdJ8?+9z&RHCT=|?o zbHYoz|C%b#yQwih*w&bHa}BQbtv=_L_HnWAHP81rFgN0bCp)*)es0H^dC{)cf6;yp z#8G$+7dzB`&cX9LzQFhHYaa*M(D+OnFKK+LF}LRP&$P~g-frKyDxc$wZ?*sR>YY`- z!|^ya&*yU2w|{H*zftAFdm3}F>l)u!J>Gdf?7as6;&}JAkK6NL{>`8G5VyLkb6oLk zWB&YbV~)(dy$|7UJoWv)Hy3}X^E{Ue^30|7AFHmb-d^RAT+e%ecUph4`@P5D<-42T zRQ*)@f8Ln0UDf#W)gQO-+g0z04!7=o)!Ki;nO`^^m<~(_rUTP~>A-YgIbBd5}x+H2Lw_?vYF6Wv<2}Swemkdd9Ba8xq5YVrOGc>`h2-qKIfHhwl0@*m9KXX zA9%ho|KpULR<7nT^6dkilSk!g4t!1PJeCvkDE=U?^FAKHmwD%M_sQEFm3Lm~_ws5v zp675>Ih>Pm0xol^&*5jBjl*yfp7WJHcTeZ|!c&d80Vn!y`?=jCjbE>xtlrW2-*5b4 z^+@v@s(hR~a|j-_y>s_fm5ZI{M?8~DawaZvfBU^Z;1`>^aAWJd>(R#VwZ5ymzIkr{ za^tzRgvR$!8vH{@p4M;>29*c=vM~ z-o}9sw$9tPH1=MEukjz=xuefLU%jOIy($;-UWPXOb4a|(}C&0bYMEL_B+69SQ;@G-8y@pu%^IMEm2cXBg7x~=dtz0~Z-RH}xD~%tj{z;WfbBfP4e^vFlD$jYQF{ik!F<<2iJcL8Z^IY*%-=7y; z*I4f6eEdwV<_R24Zs!9$=Ec4jf9HV@_c>gNmvNZeJI7&pA2&SPIv;wgG3VwSo0`A0 z&*Ku@eNXdOS9iCcFKua_3vX_GN0k>YHRj^{f`4#&-oaCN`!${CQd=AI6@Gea-;Z}+ zY!eg z6M2F4y8C^5UlY%kB2IpF40DE+Oa1E&N9QJ>PxXx<@|Z zE{`__@(yoJTAu#_>=t2z2)x9yXV#J zd9g7E;RQT`FI?UJ8>%O(+>e9sr4y|`Tjke0^ZGuQSMslW+Rx+oFwfvwe2N=$InKl( z`4o?PrTcl;V&l839E%TgHtz>G$vf@;v2*<%9_9VR+pXW!I`85n+neX1-)+pb4>$hp z?%_f|Y0QsLHRdPWg>Ukb7rJk6b-BvDmb!%h>w0(i&^J5;%_4w76KHqzeN4nrERZh*@{;>T=s}Htsf0b9> z)|e0S($6>Vea}*3ZoHPBW#+d`2c`qlf$6|>U^*}zm=3J{4y?{ce1j)&EYIz7x#w#6 zSB~xRc!9t0gg0A|(>TETp36Cp+{BSM1;5}8a^UgK%O&z9@8V=~fjswc=XUhHc!u2j z_0~BUCyQ9B#UiE{kSeIB3SpL~h0$i;G@oVKm=a=6?luk#8n z%1b#IU%0Y+_@F%fSo6GvmvFOFt#dQ3bFulwe)py-_v58+w2!OuhR?R2=kOM;z<+tJ z+z-@<=T)w;(wHOj zxsAzdzL<>p%(^OtXSFGux0V5NN=kn`W${x4PUZ{O>Uxy|{;ueHw0E^D4& zamX|6^L}A>`*u}1H%B|%{PUfwEZpAwauYoJWb+4`f4A{nRetv;jqhmR>Bf9)sqrU2 z@xed5+P<^R^IGp`j<(L-*ZvdE{KDzLbYMC#9heSG2c`qlfwj~D&+D8=F521iuROtT zIEGxr9poZTBuB`*7rRf6^l^T)_Y2ZZ5(%3G8DtTNkKHR>2-NS9Z(!4x>sqa-z<0@Zl zA5W8u`G{OD&+(}3os+-iaz3%Yb*>=yali*#KiT(^LoYOz$9bfj$pbfbo=0+Fp2GDw z8V}%w^6P3p=2YB@r(W6jI#GSG&*y|3kH>PE^X=PI-P3ul$}JBxe|`J7#iPyh%PoyL z&Prom%8{Pz9&Rqr^HzDCXK@4`PQ>H+eYSayyROgU zs63A2b2}c!SFi2dQuTrAceK(59MK{DaxTGPcmy|kqTj(w zpYEI-exb3v{%HFjt8$?)wXf*~zHwRe9F-4kZ=Pew?>vI<{dpHY*Z%t(f3q>q+TVCz z>z}S}Y@X-vE?)Ff`*yUCFYvh^HqR}%{jttnQ{_wikiY++buPZteS52WTIa$yH|8)L z^Un5jQO?YdUu^xA&hKjceD&4l@2GN;_gm+`7g|4BA-YgIxroW4onB818b=R^18o! zkfY^Zxm-@YwdZ8HhYQF(@*oeAYj_UllArjPTq%e1qcc5s%f)iF{CsumatHU}YjQki zlS4U{JTE_;>OOgwV{ow(t@9E-#uXOpPuu=4_U-W1>vG@o|6<>1pQFOUrf!TQsRv>r zHlyOLu?6D3(INDVODRsR`x+nOH+{# zwT~Cby(^o4qCW1&9pxB#mOst4Z)^K_7a!yYJmPBoaw3o8jXaQRawq=W#%}m1A)nemv8DuETMjZhoq|(D|G}KIRH9wtlS2Yj~b~ z%zrrm*OQaE;U9V~ui;oToiCTm(>#P*@*v*9#V&R(pW|LUi7)W6$@;k*pW-K+m2dLp z%k5jLa-=nl`Py9Lb^TrZfHS_@Ja6YQe2I7Pm7VqT75>c2IriH6PFA0+^2Pfab1ojq z$!@mpcZCldzft8t-#6af-+8LawKx-x;cK(?-K{=YJ<)l*ZBt_&^-*J9#8vmTe{+>j z9<6_|db)KUb*1rQmCte1`9RP9{h4C*SRI==J<~fkOz$duMp87ZlcevJjOt#KV^1FQcdgpR2`9?mH^W_$KNq&f3NqEL*>QU*0~s8`J#Sa zv$OGr`d@6!J@_IQk<&RO*W+69EVtr-T!;hpmSC3{y-=UL&jFvUj~{Xy4#EfJ{zIL^DfrCG%`aBZ zbS}r{7hGaZ>kHNW?fYYudvIs2cd&JyztnzCaiZ~C^(|ES(9?~%Cii)!eJ5N0AC39! zW6g6p{>NcB7{B6;oas*ccX#emV{Un>`RAISX?(1)-#um;bEDJsaeto1k3OiM`|!6j z&HKH=?;`x3zkl8N{Ox>WzU_Ame(QIEE1l10IV=x-sdGNBzFFmLes5XdIgeMLuO6xY zo5pt=zuS1e>i3tmt#ei$_pi-!=N;|)Q}Z{fT=e6{e(&JS-?q>1Ps{%cXZ(fZ2F4AH z8yGh*ZeZNNxPfs4%jpK>2oB`?o$vULma{mKoFhMTqHTR2%Za|n~Ih-9C9ozLOicz=ypjkK%P4>Y>)ZhiafcedH%>dxWmJ(Z|NMlS8kS9 zIY#Xh+)fVVnewk3Czr~}yp8*CR{q9a_=tQd&&vHrdtdG9EIbIHH2f3C^}FLeH`Dlg@q zYntcFTxhm=KF00-Pv`Tdef3Q@f4=d-DhJ}J9OJXrdCKX=JZXDl{<)^}`54#a7@X&) z`urZie~&dkRpl&vnu{;A&L#N>_jsW9TJN03+=Z9C-~42K9B;DuU)-lLpS<1pjrMU& zzaw04exW{Y$9wqoztp!>-=@Z&H0E`jjQ9UzeLU>fzxsJ%sd>Kx9B90?^Z3@)<}X$+ zwa$Ne`C{u6t@F;uo9ByHTK~7I-$@!yaPL!1@WR85S9acS8n0^XccJeZ&sLYyx6Jrk z#tn=c7&kC(VBEmCfpG)l2A2N~$Zy<)XUJ7tWo_TX@}ykKQTT-%EbrgxJK1+NkCIo9 zcdnfIL}Sjx5BLU8;WP4?yv(!adMa4A>ZOSJo1~)lcVKluEZDl+kEdYC-J$pog>%s zPq|UP-ccV<;~(5ozTH{h#h%BVH~}x>RPyla_1~y+gu9LTj9f0)bGmCims`Btn8)mI z{A}mRySth{R+UrbYu?MnII!Hvk9h3C&f!bEjt6il`Ch)h*}gNqA4j~PQi(J@(-QI9~w@4*g5=-FUbG@uf`2kezLkT zH`>ye4_xis@0#CT{iyk~jX4<4;H5l(->s>iPdwe2|8gvz$e;JrcVExtvb=?t^S!0| zIPiOoIT*j@qdb-Kak$sp$JIF@Z~as6^>%&y_2cG`HovhkXXZxt`U@xH+}GQ`qWxTn z4{O$vpagO>At#7Sz;KjxdwU3)|D1OL)IPtpr`O@c|!xN`l z=ZXB7mvguI`X;Nqo-^^SpISdq<;y(t;pX|=-rk4*amqvWZEu}FFaIx`@fVI87&kC( zVBEmCfpG)l2F496ryDrk_bgZ9P;&eFzI(Zp{L5vy2JiA6%^}|Gd_Kl0jH#_TfNWm&f$)Hi5JM>d~vFMyyV{-^DsH@i{|B5-oriQZhrEwy$3J7)R;T*T^=J( z@9#PCygbQO8ZVMpPxc(%DsS^F9gO@s+G9R!{X`RsjkzU%TGqg{zt34?Qmmmx~=~C=3i{gb0!;e(+BF~{oIO6oo$_y?rXfa%8A!??%L|D*1v4b z1vfSSyXL2>+=_4U_fz%ru+@#XRM%DKJBO=1?flkxA-8(J_2u*}Gyaxw1LFq94U8KY zH!yBs+`zbj<$nVlhxhOkUd0vU$zy%@av>hYzc>fq;UheUC&-z+NWPcDIZ*mQ} zL>}MOdwo=u19_O7CNIkk9E=CZ-}0W^&tLd}JjS0kloRBWo4v>D&GRWfD9`X3-nYJe z{D)Va>l_~VjlZ+XM6Wj^Efi2jN{!PjD%YC4bAyavL|3hk2Ad&2M;x+{umP zVLl`O@+oe1xfkOhJV@^4Opo<auYPXiIdV7m;0p3S zKj02LRc@EZIV>0AFk5@>vsEt2fw=F-t>0h0ROO{#^!LfZyp1pO#;@w*HyrMR=6MOX z;~D(0wuzVfkCjyp^q+b!xqD6HJ@u_<%;9(yXFO6LM|!O>Z`s-SO6PDpp2mSV=$ZOC zGRKkUXFG>8@(8}M*#5Ki9causdCi^XAF2N0h3(tX{O-n_m#c71e$UCiZ{K^>pQ`6O zhv)L|spdHiXXAfdmybSEKUY4{_*QkQ=RH~FBpk=@7#xht@Y=W9$N9L`Li5+Fe2T+! zO&;X;1Xp83h&_p$NCPI=j1#tAusV2ej+b(D89f|IG9`}|M4$Q!I?M%|GM3G zx_m8%$+Iu^-kj}V=gaS$ zgR9AHa`#Lx!kIW4uX(-m!x1-JTMoz@JZua+bRe4>$q*>}~z^&XHd?Ha}D4bv%@3U2Pvv`KmET+1{8x%H1nEm*4T^ zL-lhqUM=@uZXaKg>$%LMt#bl%=-JNU+gyzcahd&{%Z>Of?|HoSSF2oy|2@?F$tvgJ%e-)F z@AE-@e{RhG_BDU8`upm=zV<@^x3=s`XXP@2Yab2USG-NWlOs8eT=!wm{h}(5^C6DFyKeV)@UXj$ z<)b}~<@S!9;0c_7PjLx3hZn5w96rRkIE)<4|Mu0-CFIfV&2x(Nz0jE|_mJx!Y5v73 zci@3Xo976VjgM40Bp2gC{BmROBd2m$|NcO(=eWn~=V()nxz|k3!8a;Yz%QgK{dadAjHC zsB$%q#YK4Gsh-O-ITGjQ&HRf?@kd_CU%u`ms{sQa=Bb7FYjtUPvu$s zge!3>KE{QndJfm=xQUzn2hV!AvHZzr`KO#NcYoTs@*OXd9C?xsvL|5UFkX8k|%Km-XS;hS`Nfd)_1L zp2s)eZ_G`34p;hp`#I)RWA6Q6`=4x{8=Y@{zIhI|wlRdE$T z;A4$>llIZ|^qG$9dHU&GV+yJ(rW6X`Ro1*7|>`UT>Wr&Nlw4%GLNV$6RdP z?-hqTk2i8OzaQ{1ZuorrPqlA3eano$W!%8HfpG)l2F4AH8yGh*ZeaP}fE>&R@g6!L`pDSvYy4#p|?0iTky`H)<-+xZ6LJxEk(=c7U!0+R-*x^I zRlf3S>pYK3@e)oVcXB!Td`suasdD7A%^$7uG496mxD~hHIPwsO;w^F^4|%-zmn(T6 zzqvQ>$wNHvzV`D(ZX*x!77oTUc?!3g?eF~O&Xe;wA=lv-a^tG{*K{7g&*-pF2Q>r?Y(%*WaHIUp1G^@-)x>+OgFyTIw#?J9GNqGUjGAC zF7jpjIV=~IyE*4?>*Hf58}kJDUjFAJoR=GNuG_r_XXXIhj7xCCz4h~V9>}M-4WHvb z{OMTlyVN<~8&|m<=jIUH>8JW$Z~wZ+9EH>IUrxNEJ|4z7c`BDV+qspAi|yk(TU+4h zT$A7P#S8WM9pe4Y;ZPij5Ak_kbEtk^wW;$qHGiY=mFiscJawUYF6Z}&ZT0>L*pM#Pj@K!k?~pE-#*M%vn!$ z4%g&Z{Pu9`ypd-u|1X^J7mgbkH!yBs+`zbjaRcK9#tkf|8#vl`?Kgc7FIDAm`9vO) zSLH(aXnp(S7x{rRaSARZulXK-y7!Qm9e1acv9lpj>*7Q8NnYZy3 zzQnmWgPba_$U${bLFbDozJ=COnI9(%Gr;#pC|GeuEl9SsE^0UsVjP(+Th;uL#@v$&$<>Ei=Y+h7x7^n`@-DyPct5nCLvS*={8H=NtGt(= z@ostia(#Dt?y>5l&GSS)wXJoowX-ol_}`5=&z{CSg@1D;4#F$>6i?aRdvLzrbsk6M zjGUS;?y8TwO*ZCaM;b3y@Ae$7&sVwN*48;P$KfvTx6a4-F2~?+e1vcDL$1d^xC$TP ztvrOoFZSHIs^2+y3g=$i-+8J&zl%K3*zX-L)OUaLTOLEh7t#e1-?DvY-TIX6^&hJInTVL1n zS69Dmp6mX;@oejUx7gBnIep8Fzh&IOxPfs4;|9hJj2jp?Fm7P^-vEc;CUTX$cd+kY zzQ6(Y_WjFgI1KNS$K@W5!=>b0UUIK@aSC2?uIC-i;)ZZKazCzP^W9wyiGr#IWr;lAlPe5U0Id3K?74#3xV&{FH$s`52Iln*%! zZ;^N9()I0^m*nr)o9C3=M^5I1JcDm>8TopuKa7Lj?A$ZW^9w#9hsm3Avi!(TH+2rL z;xXKhiyrDZ^7`h+|D}4c%CC5mJh`!T?#6e{^nCfBW8LeX+=jPuHo2F}@iuv!H}Mo+ z!XLRDS9+%R$PQ%aM?e9}Q>wHeZmAMD6yj|ZLFWNA2o;co?_gOdOy0ah}D_ z<3KNT&f4ZrH$GYAgS?14?yP^RzT=Jke!;=`C+}OVpHFcu9>%fw+kEHlY2WF_+?`u- zwAuQW{};~q3&#zN8yGh*ZeZNNxPfs4;|7+~4ahIP&*eKgmmA1Ia_P3df8|1XV_ow+ zi|g3e^!{qt3h#c|{h z`Hd6Fd2+gZ#Es-S`Ag1}qvb<+MoyOB<$SrF({Mk&z%S$#-u8BXk9@=9c+4BEPj(My zt8&^0t#c*0niKL)Iau!JCmcZz<8g8t=ixJ)l8=1S`#f6ZgmN)Qn`wPj@5`6`J3uZf z_i?Tj?Yr4NF2I@Oz)h{os|OqN4>^<<$;EQ|BlRDye%f=n@W#fwtDjf-lstc;c@DAE zc^vOt>#sHcL*vQDa`n57xyzyY_%h$)6kLOga*M6)<5g=K^8vY^(=GJeXY1n}KQ_-H zc$B=)v3Lg$m-Bf7-;nb+_Z%M0y}22mU8s*^Kh*p1J?_i_YMHdRH@+`i4qy4|BaH=Uakop2IgsBm_#Ovb-?<;xzp2XiuC>n7xFe_B)%s8ETU~v+ z%Gdv3Tt7#hZT@uiiR%BV@5SnD)$am)oJaCxzd!gL=ve#qb}m2WfPRnQ;#|=0GW`GB z&f#C2Y&m_)jK5{vz_@{N1LFq94U8KYH!yBs`QHF9kTc|0uE1UR4Ns73f>|V zU|;Ke@r(MVn&;P?m=~UCozL=N?#P)ZJ8wso|L_1V!b7*!_g&}n%IU^Djr-kdo-^_J zfA1W=!)G|%8SB-_&f(WDHReV9^IY@qRJqpv&fz!QV{3iLTojh|?p z&s=ZJdpMil0gkrsQk8rE*YX~< zpRKN`@=2c0lRs#EPu1@?^NqP7SN*K#@TKqTTig8mRSvbYvEKzaBTt-a|CajhR{8t# z|H2u6;kbcu1LFq94U8KYH!yBs+`w|W0pG{m!gu(+c}T94Kd<#&?7RMAV}8VcE!CydOkPcHS&dgyQ;o-tMWJRc&~ZBCI84rv)$8&_3;|c$EWy=JjsnX7Uz=7_@+EB zAIXa!^*p(2p)pV6d`(Yqz1zL-LoIA<%wyzR&h>Qb{6=o(U0i{;@ecW)C-71ZJK6K) zWX{QxxG2Bi9`Z9k;~3nES8%G$y}!KvFFkit^Kvx*<93`wKIH?vM@~Q6IgfQNH|7ps zxBh081IW{SgU8O*$2nFs=D9rXrRL|WeCCDTgS)-jn5Xb2-pj`?bPgxuHtXBZA$X`< z&tdoluRGelt(|+OF{kFCkG8(Eb$-AdIO8|1^Sm4FJK6l<#+-!X?`?iTBU-#V9fbF&xgi3Tg?Y~t0 zOa1(eC;z(n+9voY*W$9gbajm!*Y7$@o#XcizZcxs{I^x!%&WQjvDOzlf2rzs0Y1K* zzGcSWGHzhpz_@{N1LFq94U8KYH?aI~fCq63c}tGx3cQI6`Tpl3@+d#zKl1y{zNa~a z94)WQ-SR6h<1O;6QzuSxR5c&Rc>pNQKjPe+7`>}PNDM!hja;`kY z9r>iZFW<}CTuiQ%$EJFoJj^lVVGhT^I08rHR{VlL$Yq?3pU8(tx(`mvXZYY;>qo0{ zqg=btd%scT0X&fN@FPwk@5;m6hsW_=uDY-1bIzSTkE`)6?#dbXpuEbz_I4iE;7~kz zRp;}E8}-Za9E=}qZ=KWdIR7q$n{pC9!4rAwgPp@8coTm(+&VAj$H#i#E3F@F%r&_l z@8cny^H%$~5BHSspX)i0{tmc#MY>DJd(uU02}PUV;XpW$~+@Lf*FX*kv=^>I5c z$O*Y2x8UvE=qK;-tDpaa=Wv&8t#cfn_j%9dveS*Hs*6?5%2UoZ|8HGYpnl%QA!eHAIr2C!<;`*{KjVq=IhWy!JeNE0QI58w_u@C}8}~XB z7y6GMs#~jkbyNNPi#u~KuErfs)_=3g2{(4`-71H8wlTl-?@eBBo)5m*dAq7>Tffto z19MS6!Q(jDWc?Sbe1P+8=-fS3p1M%~>s7A78F>VM<{*DpKW~}s+-=QsNUrx$^Vh3C zR{fs9zxWVmJX#-L;UT<(tGrwvU**~FH9yz;ab51l?|Bff+}}RV%8&TyZ(HYN+>JYO z=417(==q$D`*7;}Tj!~Kb#?O`mb0zvTrRk`zIU6Su0CAlmwf7c{qwEgXv`5bp_pN*<56a7Onj9iO%dx(zc@0;QFXVjR^S+WHbJtW4)+ZOrdGgnn zt;=?h`#{efu~8XW&B|j`Q*x`F~yeFL$0i|5@Wh)iw3~xiLT5)R=?uluw)I z*Sz`t=AZ05zO}3I>DD(j=5q&|=P`?|`<>vy*7?fz#$1bYa$fGlrFi62`?w_!;#Pd= zllrE5kEJUA;r#pR<1Bod*G{+Zwdx1e!|mfLZ#2*KUuoU%0lfF|`Zz4V`^VVG5<1hOg%g=Jm&gSJ@J|Q2-M@#+v{6~)F4}5{|@tO6VFURo}`AGhdD>)C( z;%M^Pir$0o@H}~&2h7y>A9_A#lcVHWF3A(+Gx<@j;u3Q6h0f;!axDkrBl4|$$E!HW z(azxqyo59HFz(2c`I(>H>>l_W2jsNdT9@bEYb+P@8?H3hIv4z*=YQBd zZ{uj3fHR$_pFeU|{={ea2_NQ;_j&~{y_VYyU{8fFNf;(_lKFCvdIJfh7!bxZY1_0`TfQsticRTB7BA$XQ>|~VpA#=Ozpnae>zs(2EjItV`hIcW#{BeTW4||SYRvI? zDd*zioNQJ7k5s3toRFW*Ht*lNa?e+KPmaeEm(#b*_*=#ej2jp?Fm7Pnz_@{N1LFpk z{|#^`xx#lpSCAL^0x!68&-XNM;9c^STq?i%p7$NkUF2Q4X=msAE|;%(4@ck<@|%3l zC*(VMOD>SZxSX8HdANr>CEv&!au6ruaB}mzy}vv(*;wA;Ta)FAyH(!7$K+ft;otAb zBS$+|{yx$}mg0m6vdbndTRIfLz51-y!*jg9#m*Wn7>fOq(J9;e#JkG6F_-{3y- zEC-dN`5yP>KXT~@oyUJJ^t=t#wbf^<{EClpF!@zJ=Ds&OpGV2@Jck$lSRaQz)br$d z{>k4C*3Tz6;&aXO4!*O$d9M9v=bx^wY<;0Im)X=j7vZkF`$qjMs{EXTaL1FK!)G|u z`sU}_&pYJ)zqEd@$KI}=m+|Op_46EF&!PDsU*PB4JCE1$nOn{ORh6s!y65a`o=e`> zJn!LHpVfD@_2;XcoO6BCJm2Ng+;?^7d|MxPoM?V)mGhpdkFRfOoi840{pBiG^!vjv zzSuf{=fT`(x^sEhZ2kWI+NIXH^TEbH)yG%)HHWNi;=TGf{?QixzImQE-~0>Bzti|w zV{YYl1-~EdtdE~^LcYvPcXiHEb#MFlmfwYVEVo|%UpV7095*m-VBEmCfpG)l2F4AH z8(2;^AWv}>j>Uuci~P2u?_H|7=V+9yvOZOmJ^ z&8p_*cKL_9$>}^{XZ>=p{LL%)h1|vuqci(BA5AY;@^jPb>i5D#OT&}`t{JRbLnQLzAoUf{Kz8udl=375fJ<)Tx z92eutE9>K1+=Q3N-^V+L3ve$Ed!}=)Re9=E`*;Np;F$dClluPH{3ikCgn{BP^$M5mhPBmC?{eZTlvebdczw|{M(FZn%$e{v>X z(EZZ`#*gsa(mKlG`xPfs4;|9hJj2jp?Fm7Pn z!1BKVJ|SnH>^qmIa1S|IevqT(^*wzL%g?Jihrh^`YwFw7IzQl7{6!w;W;^TWL;ObW z$lLsdC&}r&U|sumR29y^Z8)Y}D?iKE{83)!eH@K@?(Y5hj=aoIPPhI_ zmEZ6sd0j4**LjuPC#T7SypPW=^oPp3Jn5(Ac^JQukGb9U-jB!0*$3LsUAXFZ^)FWC zaNcvXee$z>%`G{dynTP?@D_QPFUkKrl}BuRlN7^ZquU`JwS!ox_LsHs#lXH(KYuT=Jjm<1f6B%eHNT z=Y8J9WR<%;**SifIM#S=eMhTY>_}r?!ku`^RQo5ZT#3(cH9p2yxg^iwd_0?jec!np zeoODe)p*=g^Dozbxbe2e{FY1dB0j%VANRW6n5S|oZufELpQ!S#|Jj&#-D=E}_$jyL zqCD|#{cC$ZPvgR;TAy!yU*l=>Rlc_Tzi`H1IBsCvz_@{N1LFq94U8KYH?W*;z<2Jp zzIWwZ9_2fpYh392SFYqp{DzD0AUXI@=W_!2-}k&+$*Fw5^DTKv4)k3v_i+gML@wq< za-19`U&@Q}5$EAhSNeP96)wahGD@^gQ{8Tgaz;MSkZfGwoZd-t2$BcdL9*9^TaaH&uC>d&-AAL2j0}pF)U$cOT;oX)ZM4IliV{qiIKu45+n(0^#+eEUCa%qe*iKY6irzQ?7w_My(<%{+$J@#_E6Ih>!< z@Tse<^B+F_Y3Fl_?Tz^?r~m8r^G5#1r`Na6_uuav-ttiE9GR=XS>L8AUp~>8f2?f( zy5{}fz;C%6uX&)py;a+|Bp>F~kJj*;Du?H--0`W_w^aEsuj5%9j3@1`pKm|h-}6NC zbJbPN@2cKzowX8g_jd+fezN|J^>bbB|DW?$-hZ+F+4_0j>Bb*d{eHtG zS2xe=c`=Wg?c8Hk9`;K6{GP%$m(#b*_*=#ej2jp?Fm7Pnz_@{N1LFpk{|(4D3w`hU z{^m~?`tIc@bM2D@cniniNH^Qh zm*kgg&GQWoCbx67Jv~R><|{w7pIqQ^Bit|sr?*IKIfXehjYn?zc@$xdBsxaa*YG6b5MEx za(&#BL&>*t_}2QkpB&BG_#_YbOV8sse1YFS(|KHhhw=cv!+ALlzvc_?_MCsJ%F*&Z z|Ku#3h9`5Cwoh=qeZAkh7N#1@^E`qV@ocW}VEsIYPs#uMlY{V~#rE-et})en@Gg$E zzj>~7sPS~?9IU?8{A6Q3a;hZkNq`k^Nm+mzpFmde%{Hw*0%mc>zs~P z@pgXf_lYg-z>wCc^UWMB=X+2`i@lPV6GvL$fF#AA8;Wa@OtmT1LS-;@ZLPd z+qjk7&NXH_kE0yyJ>|@X6MT%D$c^$YFO=(_>RkEkW(RRXz9vWWxcBPgmWMir+sUzV zp}hIq_VGua#Tj`lXXF69bZ^h$0XsU6w{S*Y!B2R^NA>^EIdbS%&GQqUGu=F&Ir1nh)?GUd>l{1b>nHx#2?3<$c_g^F7fz z_v93to980(^{37AVE*)W@6YMZ)yH{mH~)6 zJl&YH@-t5KX6xK>W8+V%cdLt49?d5>(ec)K4Yzo!_ve8;jVu1x`quXIKfb~1{Jy|1 z`Q-m>-~P_wDlat8Z8;S8;i~*-vi*nKx2iD@J<gwuTm4o|Tf&==Ucjj@MkRPtN37yi5Mz0P?T=&6juxC*ebKD&Kgh=gIYQsr=1>ZuIx^ z1-V6Dl|SV-ZX&13hw|Tl?76&$PjNf$A!qX&PR1qHcg~jHk2i5WKD4bq{=q#s+pDcF zbdFs3QS-b<9^YCYm*hus9G~E*kJkTF=gHG@A(xVy*VgafVel~CvZ-@;>(0iP+qb5% zoP4hz@Tj@^cmNOKrM#DeuB`v{o+Gy}H2+Uk9(bfa4k?G<>psua$0_9ZspjSTzwUf) zAh&0$D3=+4>>kB=U|8H`bvo~!<#uljv`oRdRwq*L}+XY1eG_H2h|C?-{D{{w)_AgdvtHGGYN zFAvLE@~6Ciw)5Yq%0Vyo{&F9u;c)UEkJ;FMIi9C*32t_?b9v5mV|ncR#&S6?m{w>*REv$1y%_Ki}KgemU@Se}D75g^Mnf1)~5*IeM zmbKrH?WXHnX#dOAmzw{idbE0`^|z}0 zg#U8h_v`0soN~T-uE+a0&hI;y@9|PTy1M;$tKU@@tGn9&MD_nQ&)NUI`4_6(hBx{> zU|s7x$nO&`cwXx~iR<~@U`u_k)_1<~n#TL9v(0lYj?L*g|Ks(qt)HLnZamvM_d3~_ zUvsvD_48mp$>(?~U*nR?>04&}E#n5p4U8KYH!yBs+`zbjaRbZ$2Dl67;Y0G894Rkw z6%NFexB^e$16+kq@hF}ncku!~CRfVYyh8qzWBD3~lIJ;*yvP^i6>h=ZI1xwU5_0g3 z@_`)AFL;Oiezv};s{F~r_ys5Xp+0$EzK~DlOSw}H=O}XJq0ZrAT#^%V5;^W_IY=&( zM|q$e%**6J`IC#ty}X7?$eEw@Uh**ahqoRO!K^k>)z`tSL)-4Je5Ck25xn`{&Q81v9{;VRX=T= z19Qc@&HvvjU*>VV>%XRgeA%5K6@ejFOe&c_Swr^eM@CNx?KD<~TKak@& znjFl{Pu_mBKHe+8 z@{ak|xzW1Do2$HmBQ5qm9FL3atDk?$-|Oq&+q%5XgZX1^6MTwu$@9F2bMvNe+Q^|k z?l~NfGx8`t#B2B}KY6e7-l)p|JnXsVc>y=!o>yDv75toMaVd^*qxa=-ypx0T!87$e zUFA4Di8t}c)AjKqo`0eFHP!vSFZbXn$D03t_3^>KYJ9r&qm7pu->KeLU22^p&Nt6V zHnh%LcC~(~dA{#=k@L;ptdH-{G|y%D_e1r))cW&{e_iG6T<`w+PF1-(C-gf3x0>yo zzo{Oo^5aJu``v^0ZEOD9Dwo>S_=)PjRC&@=i0n_w#X1c(T41t7od5jc*@p zex~}Tp2K_JX}p}iWyaq!ZeZNNxPfs4;|9hJj2jp?u>5a8&X$AZPI*PHmYd`+-^n-o zK9ZBz5|f1<^MCNX#(akp$>%&se&iIKiTBF&H#)atCvG?9GhCFv@~|iB zvzd9ypaR)0KUZUxRabMr?2mQ)>h}Le_y@Q`|tst z$r0sxzRAB%w{L5eFRtp`z0LDJuFWsyd(Od~9&i7zfA#Z!Om)uXDnI37Jev>jFRsfQ zj&&|S;GXBD|x$&0vP4*nF#7(v}&(*jIXIaxaSK=N|HqR^nThHN+ zCtK&{eD4qSU2UD?{HFQyRqpmdeV;YYgT8LOu62IOi9T=r;nqKH{QounsWGSGVm#|s z>-=iAF%R>*iQijz>D~I?=s8@Er@h;}-z6?Ko@^hd=8w~@f7kkJjpwVpZnp8M`mQ(r zzRHbxH;3bY%l`{!{DtEN#tn=c7&kC(VBEmCfpG)N=?3ofoh!%5=klvO?)#mSa3gt- zv&elb`kwY(&o$%^z9N@#gN^OuBXSbglE3A8xsS8)4373p=Uu4EpFBXm+1xq@k(>F` zLifbgb~fe@+)193Gvq~H#aHCrGd-8f$ZOnBPUdJli8FCJd5sUrNxbmso+Dp#mC61N zp20;H>X%E8*0-%HkMWVE`uWDq`qo!@$A&pB%||D+Zyxo}PUxGI->t3J-c4S1a#d#pabGTnJ!HGi+;@i1P@L-_!2mAg3`r#asF zyn#y|?0NDn&yu@&#H;mjY2NT$^PG%Ba-fzcI521WwgrB|ulUO^E?FOE;5nR--*8LL z_h^6bOI2R;Y2*1S4?5SF3$N_Cay?(->@)SBt*)(~+i<$~Tfbf9XB>EK^ZbRY@R0{v z=S94M*4eALrtyoQj+NzP>Fzcecv^_!9@`@SN+#_8)8?AH3H5-&8s3 zY~$thEi?X>aRcK9#tn=c7&kC(VBEmCf#rV#yoV3@UYFasfn3Ie_z=h96Wm3fmE+|m z-XXX1gdKfP|Iqh0*ICh+r|>qpk)O$JTuPpiPvzb*Ef=0>T@IF$AlE+IdvQ$;#*_G{TqzfGuch{LNx4rh z=LGwE{ufoQ#R)cd4tL>QawC_0us-g^-FOTqJzC$&p0}s^VRc)Tb8<{>$9?!E_c`9V zTy?537vKz>Oit$_@-%PZ8*`n*rMTI?<~a?Q<^gg(|2@>-C1;;*|N7=R5-;H>ypvnK z(>`v?1LS>9daQFFu5vwI&td*k=W!FxwyAymVzMzuJKsLO_um_H%=;U2w#CL=Wxn%x z7$4;-+>8%?*8ZQ`$G`aaT=RVMaQ$40H|}npxACFN&2t$J%SpDkUYYQF0uO(!=ku_8 zUFT98pRA9s@*W<>UB0hxU3IF;b2#CDZ2pbv2UUK=hqm_o1FiFS&c?qd>*Hg5m>=^v zPR5I_b}k>g<)W%wkBi>xRgbiv+i}V1<~i7v&f{yR8n3F4t6y(CTV4KNIO8uIH!yBs z+`zbjaRcK9#tn=cSWY(}pRDV9R*shsxeO1G=j9*yOJ0|o^{MKn{>M~ zk6a^X@eMiMzZ2mN@&;GnCi30gzU$>e`B|QkJNTWPC{M}3at>eMG+c^D$fa_myt2^W z$8+Qs{=ymL5}t6ReR8^7DL2WTa-7^K*UEp~LH_2Kyp8w0-1FovUbwe=l>g)=z9Qdp zBl(^?oohdrknhem&qdcZ=7fBYtH^2cHy`3ae0FDlhdjssc&(g#qvxHhpC`zn{9#Ay zJeJpd)I2}odEYcIKXcag&41E!`6I{WDrf5Bz`U$&6a4SNCZ?;rh$F0Mo#iJ>PTq_6v=B+yw98Ra|UW4X5jWrpi_Cw9YO5(wG}^wvFw3 zu62%murY_}#I0aan%8w)f<#{OYOp z9cX>A%4>McrTUInc|TV>(>#yjiA$aLr`Bg1e_OrXJRe@um=p6V?&NonFWdi4{hO;i z)9)#7*Ed~VsPAUurN+GK#peB9u%UIo+wjTz>ht^1uNxn$9&VrCBbL**%=lZz4U8KY zH!yBs+`zbjaRcK9mj4a-zW06ZdtMHe2jnxk{oe0qt|4d1V;o7Ye5vneIbAO1OmZ8) zkiWL|{mpM~HkP;LH~IQ(&ymw_H0EktMDCXR`G8#RJ74aVKX&$f`Ibk>$?}K1@K*hD zFSk0^d(BkkW6mId$TwVqyYMo(YoX`x5)SfH^GA9P2a@l(Cay<`| zKM%D}j^-3iPsn#)_dE_H*M8o-{Kye`ik!yX3h6xB|c94t$OC z@CBaBF(*5ZPjTQi&2u%rC)aY5XZw5QYfi@f_?8^Zv-pl&%K_zGp1~2h=H;Hpr~P}G zeZ4Q2JzgJw8ZWfIr^>Y+Zp^PZ_LI%?4}Qp{ zziyo;-E|Mue{7wX9&Mht?QYBq_BQ4`k2dCDe1~gr6aI9v=S{Vb&u~0$&yV;opZlQw zTiSoH`upZN$(82)PB7K_YgPVtzP{_tbG)t1|GCO>CL5oqKHomR#Gg3N>H2uq>-9fW z-BtBF$oGwJw~w3gHy-+3>)Wd<+voQYzYkn$o&T?@Z+(^HO*i&C#{=!-bgLWBRJ8$`a;f%j< z+`zbjaRcK9#tn=c7&kC(U^(5uj=p1g24DK7`3rpja}RDJ_i`qV!7I4JjrPlVJc?Iw z7j7Uga-=ow=QJFRZ_KoQrpldWdVk)u(3orR6u)0{x3itYU-%MNkxO~q>-FF4e7+=K zu5F!P@di#PPxB%^Bj0l_PA8{7)43O`T#pBFFwVl)dd-QW{m1_5t5x}$FRti(`JE4a zQ9noGPn?oN$ZsFk&qp}o51r3%HZ z-#oYD6+D_Vaq2 zIoo7?ocLJlT#fggs_#Ta zU*t1Bz^gdHh5il>vZMF>CC9hUKe&|@)I#sSqQ0Y5xp{44&h=CMJV}0)GkFj<*w}uaAn(hcM_T6ryh7gRNphily}tM0 zZJQd;cb=Rp_pfRG^Xkd^xeaIJUc2jCs`5xK!HM_`FW?hgl&i?YJh8`5@Hsh~uiogn za{HCW>)Ob9W}4?^T#}1%h%eeFkDh5CpX7}EhO5fwT;c7`;T~Ui4$tCb+(-WBTk=1L zdb<5Q?Pllm23~uvzC*2Z%<1Mi4bS3DoNJ*zzQ%tzv>eYBI2-?N`^1kupXYql_+M+_ zI&;l)K+efo=3D2^ON}|t{>Ge`Z-3HrxDDs|L-XgWx2l|hFHJW8PL+pl>vr3wg|Q^&PCdbf3(UG z|EB#Mm9uZCkIx*dkAvOoP4{~B`}K1=4tA<@CtK%2ymL$Id}~*ITbut=m6uMp&MA2m zPxL#%xApNhzR0_-w$9V|)TN&HV(a{Kb@M!smo5JBZ_ZRnEg3&+>^Q{RhvG z_oh0B8{KWpuQ-$3uDSNW5da1!~7cU|TpYx{fTt)rd8<9LJI$W?aM$0got%pK(3%bhQ0$$^}M zZ^&i*jyG`--XhQO)z>?pJMmb~$sIY%o!(Q9ztDbe$49Qz$9d&(xt1d>)VHQ{%TYu%F%N9yY=xHzQl#OlAOvLUTxpz-iH%%H~E>T$@$z-F6Nn>gOl+lE_%J^ zak&#cm+$h@!_7Zi{k!(@qM62Aj_2I#d^7Fm7Cilx=J$6FSKZqDW6hteeo)=mI)CA^ zJeG@oUq9dBmPgyqU3NF-y*%mu<~bR^<22lGy613~742JWp3BWO{&VY3Rc}|h-P`r; zseW1Iq@UH#k@?=u`p&n`+xYeN=6N1B`KWy>s}I!2A$i@t=C4#=uAifEE1tPf-%|Cy z`VUljD`(|$2konmK`P{4a;f$^3~g$T#wW?|9$i+=B=B{+D0)gWN1X?d(0|VflAk z^Z&cb#rOk<;tBGJ94AjM_Wds}$#L>8FOs|D5_yT+@gjLbe&7XsNsg1lvj&zQzst4*%jVoJEdY(>=y2zTP7e28Cs z*TKAXYhy0Ni}=Kr)_3$A9?VU70bk*(ypZQ{9Degs=W;#%%z-xbyd(83RIfG9AGj#T zIMzCMRCyX#y-^?6=iOX~`+Z&C&CYqFdZPK6 zDnCBgn6pjR|8Bh5- zA8!140EiApUBhl^3I+o?{gY?QNH0#JdD@K-@7_T zUg1g$&C73mjSF1qJ>_3caLl3zI-7rN7Xo$WdD=%bBaZk-cw zKK`|>^#|&g|K*>Rt_>deYU-P18>z}Xk1Rim;_u-`S zubeH1^C`KRk8m>1!u{lZj>_41lKjlixEODo>b>MbUdK-+`g_*bCs%V{`IQUTK5@DK z;F$d3h0ftG{NmI4`O9QuuJu#0L)yku?jypaF!g-xx?)Au*#$Zz$$?aj;k zyz$G{IXM5hRX>O3(>I#uV7!xCUGKcBRc^%}c^|)%-!FFVH|?8hEbkv`%pEp1=09AA z@9_?v#8r3$=jG}TcHUx@8*>Y8#x`<4Rw5K0ljn-+!%6SGgO1=1C`8=bv2n zSo3>3pQCXpuE@2xG{55G{ER<7-?_`_TW0(%;|9hJj2jp?Fm7Pnz_@{N1Izyg zFUa}wobO@Z&s;&?k~g`NTqj@1IdYSHCcnxfoIuWzOYii(%@cT$oFhNT1M(M#<18IB zA-Bm*a+Vw*|88poZ{i_5faCBYzICCz!6W3KU-C}p9;|*e^ zyV;mGY--FMpQ(>iOt#KFIHFw5iQcMDev{ATV!2oT0;UGDj3I)}ULY`nd7d3H@>IhmvJ*r+r{7UQGpU-ih z2U=g%{!@*44!3@x^%Jf0p(mT4Y<_i>mmF`**Jc|Zt@35RNAPv7&0+Z%C-%Dl@8$XY zly`BYtv&D8zxw%w_w?R;e0S>~H2?c5&*qc6nqOb#Qt!9FGQl4Y)aUn?TQzXpR~jFz zK3+fX<*rN3Z>f)S%{TTt&dU0D7(bnDerNs5{|jgQh2sXs4U8KYH!yBs+`zbjaRbZg z2DpM8C5OtHa;3cGJ6S&AAo8y7;<>($|MMhXAP32( za)8{!#rAcM+#s*Z6LLDI^4%|w@&FDXx5_i}F8`8$xDo%6!?>OtE)UD8kCiv%MGhd> zp6l;%=X2OKJ%@wwP7cU_dD5Qxx!&f+9F;HeD1Ob=cJ{v9fnz+~ zIoqq;g)d%eezE;L=cC3vf|KxFt|^c6+CO#P8|~+7T=BK$`O0&R_qLC_b1N=5SKrq9 zxa|4HXR17%zw@KdTj!!2=z8b!4$i=#cqO;sc6^S%Ug&)O$g?;kcl_I&hO^&o-`Bkd z|9HMJ@A;rHkKxd#+RvYO82{PW`jsjN{I2m-=W@@-8}ldb{%!Mqzj(LtPgTDMtZ%%d z%BfB^o~(Y|^G`J9k3Tl{I}9(o*gQvkv;DiPJoCZEr>ng5aRcK9#tn=c7&kC(VBEmCf#rV# za_7;$i~GEt;32o0=Q!WgAQ$r>-oy#yJb9iYaTxC8yPZF91@7gwxEOzskGPmz|7aVy zid-u1U-Jjn$Ei3HpWrwAh~x1PIgB@O6Mn;q=6eor;$U)y9M4bqk$ii+{k%!u;Y0jG z9-Zs&lec(*Jj+wK1Q(JUSM)r&noG&Q$2xy=eLQV_^Ku^#;}RT$ck&>PC^z#RIhlj- z9Im>v=Wz+n#yj|je9Ren10R#~p6T!5KX*Ex8_2m|HP2aZwU4{;0zM_T@*-|4hs(b_ zgv)O0IXq3SzT5M-m0Zg?_!2kaX54i{`+4(Rr(T)>vx^Y!+0Uro^1U>`<|+D zfUB+Zika3q9v}N(^>LV^jn`Ipw~y=anGag$Qy1$ySbec|{=y0QH@D$6+}!T~T#EPc zO?BS&0YKR-QE9}nYzJe7CyyRGf#Q>zv=lk8_+xoXO|NF+v{|jgQh2sXs4U8KYH!yBs+`zbjaRbZg2EOS#mLu^D{=u<0 z4&S-mckt=z`6@@?Eb_8kvDouCg}ljQI0-k9pX51tkiQ)3Jg&tB);G`37SE`G1?wI){fA95}Ck!QFMPvVDMhCA?sOFf^Ta4Zfd z$8tP5{95n9msT~F^W-q@#Mw9y*W()ekXOjFa=9EXcX0ywnV;~a)4h*;$$@q@&x!aP z=UePPV1E1E~GA~9%vs= z<8Pdm*YK9Z_4AFdJC7rAZ2q{j{%!T~7Y@fumRdhrKj-5?{EhGPMn1v^dFp)UoTxtC z`MiV&^45QE{apQDHs(H)&2z5l#_v=)&fUfwi7RpDnbzk!|7P`U^HYrvR(ap8*7r2e z<2X7unyqhBeSRn5f3G(GeU-yK-k2Az?VPpEU#)V;)t>XKpBMNVFS}kpcRJ9RbMeae zo1d+6F>W~B{Fa`#oW5no-!g7s+`zbjaRcK9#tn=c7&ox|Z$M6z>*Zcfz$fH{dvm({ zCLi7CyI78uKjolr`hMm+Jd6|gu9v&y8F}Pp`+xC=#@vn1$iZ@)JSd0A(|nAx$ieas zUy*M(&7Gbrr}8ZxCYQ)dm%11EbzA%8O7#+?K)-^}V@`0U zgSezT$|bl8|M;N&oP*EsK_2#ceR3wxU+P3xRQe&(D{ zw9db{248!#bvcvE^?DN=fzR+yJ|?%H@0^d?D5uKln_7Ri%73`!isrwr-s=f+Cs&b| zGM9!F%;_u9c0shkX84^L%r%@s-Y#tG{iYSFUd?@AE~u zo_qgP9}nW%+~RW2;X{0Ds(HS^V-o9mnK?Jp54S@xK4iIXuh1Z~39VebtBR z`)zfh`a<4%)Kj($q_w)L1SNRxc<-6B<-e>hqHRdAIjd!=cxyq|K z_aB<)>d&{2bMmOyn*VeC>l*Vmj>ca&4tJSr--+tS)$5((_XZxey>(uAvA)GBpPg;Y z16MWXWdGcFPnGL!?0g>27w4Pj^Stg<^AC59-v#&)XXRuZ^>q6Vw2zlAHqSHfHs*l0 z8gnndhj2G8`AU7u{|jgQh2sXs4U8KYH!yBs+`zbjaRbZg2IM!M!G(B%?|M0jW61v; zfxpNNa&>b>X(D~7boBdd;0tK*UwY<5Z5?cpIpk#O0l@t9}>Y%be_k`9q$RcX$&Q;4tz7$MAhFzsTWSL++L*pnRZU%A;l=liMgLi?Ypa#x=CN%L~AJS<l~AF@C!b{U1vI&d+cuibItR)*BbL9uE{}p?__=Q`i;)p(frZI zJmAsBe0NiQGtKi|PVi*w9Oh8tKiB_h^@-}ns+`Y*_&a~%Ag|QVb+|RR;mLfI%l`UT zKfee!;n8pOew^Z37P*58!hA=g0c4R1enwP~%*vuHuie~_dnP?hkK;)`Sz`E%LaZeZNNxPfs4;|9hJj2jp?u$*o{zL&@4LHUJ;$mMd+ zV&BJnVxjNi-OcZ8%o)}-<}RFpC#-4TiYf>5z0M=}k6a<2%H47X_mW%J_k4aMuk*Xf z);Smlkh|p|xl=BeGx!Q8;xKaA*8VPeUvB0cao^O4Thls^;wM~k zf9o8E!*NN@!bdnBC*ufmIzPJH-zT5#=sEmMzUD1*ITw^4xgO8sOdL>tm8a#%FM8fo z@4;s{%Y5tdq}=;h^E{SMa9O@6cXM9u$1%8v{L8gC2+xsMISa4if*fnEzk^4-+#wEwr&rYGcgPIG??E9>K3+?3~W=i~Lu-F%q4oo$^fUu-O2^Josj z7vAdmH|poJ9O+u?cdGI|=a%a^0q5q0-*xU2RgTFaw{`Bnw!W$HOk=KkviY~GJZ4v8 z($QPQ{{jh^9 z^2_$VkNK0FC};bwm*+VaXW~C{zr4+LI0~PV_j~LF=aJ9kZ*C*6ax%Ul7t8$|LOzm5 zb2-yPjpZJmAdhn_ z&c$`)XP$JZbAEBGU;R9>wfDJPl@sN&$@Xz4UdHJ->4x@itnv^p!5`&mIZY1cFmfG- z;v_s#zUL#Hk1MR}eK`fE;!<+z(fW8Lhv69?_jhof3yr_7@;DxIyLow8PUlsejW_U9 zJ|V9^;=MYLYrWZcclGlsN0FD0wSQH0qCRfH8Mzuy<|^CT$F+WOna-K2@@#p2p?wFc zJeG&?W6pNv|7YLr!*{Ol|NnPaqbMSF{77L_q;k=J#4MJJd<-A+ITB4J+V79g?baXXaqOS!-Eq86_x*F<;`KVQE|-@J%CY6G^5R+SlMl!rXqU(} zAdg)_Ef1_mJs6ey%5~(w@*g?0e02oxUw$VSlCO?oUEVRDS{}QY`VcBl{2uQ| z9@dQdL9{C>AC~jVh2$#o6uHn)US|*c<)d<4`Mlh30CV!#-PAtU=s+!Rl^Z=u-{%tx zseL{$koR$rzWhrb>GK8or(Et3^OaG#V{Oi>gUa0=rk0<{spOrs_MNPHQ$*_!pUb+r-QNLy znOAuqa+Rsn<|*@gDd))FdNL;;Gsl`A&6o0nVnSAhK zURN%*mfGAV&oK|1Yvm^LM!B2(OAa8vFmKB%<$`ih^Sil`w_PH?l#|JGSMFeDV&tq+F;T=a`4hxtG$P$o>V?aw9pMd|Eytza7H9N$fMne?%?Ul#9u2 z%-!Z@dD;Z_$wlN%au+#l8+o+sOo**ATz~|n= zTqpDp+Ku(@sN72)Fo3@NsgzpYRf$@T@-xnzh{^%)VNO0Gx0HX*XI+limRi0gH?70D z^4SlVlXv?3KrSN3TFl%!RL(PpT7J}wT3#$amebc}U5?e1x-lw$k_XH8l@qjE}l(W#s(=aMtYm*nDd(tXUI%XxCHm+9Y*%2VZs zbD7`H`cvq1`f~R>>C0j3v)+-uTu*K%kCMk#V_ptdfqn9;QPjnBm&s>|192b@#DO>v z2jV~+D1HazNnCeJGYeH6NOj2XoF_s5#P{DL;`X$b)7u zZ(cW#%WceSU6_-D$j{8r=3#kQA^+;ke)*5wt1W%GqFlqgE3cF9O!xD%??GyLklaUJ zC?_;8%4g*=@&Ngl{K~v9w~`;pA9wKj@=1AuIa?l7i8;B8c~rh=j_$zcn2XBgC5{DQp-6?spVmEt2VrjJZ&#?a*a~f@1ZZZlHD%$$dN`dCm$L{-4h+kez~~3Y76Uf zE_tO~Nlqv~{2B9dCHdL^(3j`Vqwa*t2jya`=r=+i=KaaRJX7k@v_L-N=yXJgzio8K?W8O38%TspqI`Ri|xVgYwZ;m$) zo4@6E<^}nQ+{D}=M=+PR0nqC?Ef!Q`6efBDu7KF1*D&GWUX z<(-AR#XK!1G4INqyRyFvx*au_n|qhgm;cDg}%}nU_b& zk>qXi*2>Ju%jE=B>C0t$^Ewx!atnFP8v1fc`Hwuk8RyA8<+E}Z`4eL$a+D7^S6;P; z0lBtZR$ll3^UY9s$0GL0d*xt5nUizPq279yR}) zx6Q}$6M2^T`OEt=_iFizInsPC50J~$W?sG`$C4k(6XXwtd`o^|elVBGBjhadEV+|8 z)O=|^mYbMcuI7H<0X5f|L%Q((%|+%{^NBfAo*<8Vi}U0al{iOUV_uUVnWxRcay7Y} z`Ac3Q7yF#ok)z2e&6je7Qs(55@+3KzyitB6=aQ$G`%C#8awPMpc}(spr!!xhCkL`$ z{%0QjGI#Sn&2j#oZXSKPgq(wRi9F9-EMJl5nOn`-@}`eCz`Q7Lk{imi}$*U@}*tW@|e2J$$RBQa;o0U%RNR=%cJFI@(VedywRL4?~*Udh4ymZExZr;K^tnh zlsruyBL9&~$XVvHPd+clkTXr^^~Rusn3qS$3FX*w=0nVFLFckx9yXO)?k*pa-!)}j zE+(IoM-5|Lp7S$aM-C+yk%QgCyd1Lv^V87>QF&7*=8+Qlrd;b>7Ua}-vR@82j`cqD zA@?%(nA6N{ z@*Z@9A7d2%xInEXP1Vvdy$$vxy$TX}tRlsv)QK8)9q!8eR>5{^yPC^sm;BIct7$i^YJM9a)I{j zlat5=8ZjpyGj9*3FNcAO(N|K*7CLHYeb=Hz zU%q=5{X?kSPM#;v@_ENY?3cgD3*{+t-~P6^>U_3|dUl=;eB@4el8CI>R# zm}}%*gLobDlKI-)D*rKmnJeUFatw37yu%zNUy&=xdCceLKy$SD*?c1JFo(*Mk z?ZiIwx_nJ;Aa8n`IeDRcrZRoGmHAp8W?rqsoc!Ss^#R_8JfbbNIbOaY_mtx>RwD14 zOhc|^o|M&b`Y9`bK<@;$t++@}V$xmlj}Ij`TG^$OH-tdZ36YWe4M=H=>*spSB2kxKOC zp*^VOX>x~;cpZ6TGwN#0%eCcTL+Q&C?qRk;BQ4v2jV~+hy!t;m>iJfm=nz}=05YGxlS&#nR~E&MouKRDrJ2p_h)&KIpRh7 z=6my;oMsUF&G$W-Z$rP3`^b6ZTJoYT%$xt^CFT+HkNH(zA#jxRC~^Tgg!ykT zbG3P0dD>L^=5_gmyhKi9E;UEV0nFo9bFTbAUSuwk)5sr6d0qLDd09^JWzON;38=YO z&M1#E&zhsmr{;I_lDtH|BFB(t$zSAV=5O=YSl)ME)O=>Hmk*eK<&1}zZ-p-5^T;ja zjBl}CkM*T!XZmtFdD+GE{nt8%V3Cts0E$@{Ca|90lOQOn!MP|LC8gmN}HuUxVk^Q$;d?k8{hfc2+Xmn+LZr?WmD zU5Cnv+fvJc<@0iTqf;HH`)NuY8MR zO6K!_nxk^XPVC!*%J<}Djp$E8eLm2PbL4mJsTZU2xs~kmIfy)0o^(9x^2Pv2a4YTbDjCvd$9Lz@9E}t z@9E|a^PTy{oFGq-*Hq=+>^?sW6m7%)Rm} zbB^4=T)vFgHTRg`W^ry8*3GSQl!^4^LFVDx*|&>z^N>06e^{4`m?PzYb6GbJ*W?^I zj-2Tu=FIK#x~a^YJLP-wExF5E%$YyU`ErxbIN#hS_cTA67e{c;OjHhH4wVOtXI(BJ zhmiNmgU#u3JNbxwuOshEekb=a*PqPmn0MtCa%4GQl(tIN4^b@_|@>NM83prbiYt|h;zO!`DWA~QV*b(+sd=ArZ4Z7%gO)cg!0-#ZaRYf@~A_+-U|A1Z8>Kt{XNXd z2k)Vl&&p%>)0g|oAN~F5Ue-HtuAFQFwOm&I+LQS`tjl5L(qmbd7y3NH=Q;A$am>kg z+jHK}&~sTIjMhPYZqu2%5A(yR<+Bgcmv>!3-31+o`h1}_^%&0axsm+O=Mu$#!^tO% z192b@#DO>v2jV~+C?*HYapr3Cid@4yW{x%o$`j;6-jB`y<}7oR`BE++Px1b5PBB-R zXXHBOC-3X#6uF8V%p5K!G6$K<%;DzwgWU7wF6J+DzTCh(Be&blz7NO;=0&%gw{jl1}{;|70lmqAM>VpTOJ{AkcY}Y z=ow&0u~jYA(H)dITz0Tt_WueT#ZJD*rQ2%XP-FF0YlZns?=K zr{a&iR$e7flK)KPJh`yEq%nQ-u)ItTVE#SCTw7E=Azzlq%2(tQa!Wb)diKjF%>BLS zPv`T%dCu{yk45FIa!Wa( z&p-Mzw;h!?`W%9nEg8@Mna11#RGuXFm5a(BtFh1L21BWvp;OVq=P2^3QM``4R=zfb zzI?1X=PX9$K=Y~PfDf`y{wjx*>&vO+XYwUEe{arPgI47{xs=Z#xnX}|yat(8-Ih12cSmzcZFsdCd%%*m6?$CuKdhstT?GH=tD2g|$U zn-}vw>!Wg5`GQm^J ze>t<SGt{#NXlPnh@RW%Bfe%*%=73|r{SJ3CU#>qbz^f8@XBb@__iOrE@#_w_&Q zm+!Tqp3b^_Ro?gk{e#TOIR{b8Bi1q}UphoBciccNXOqia%{g+wG0g2j>$1KItwdkW z;d6n`^yMlenU~Ya?R-9QEb}L$NQsTD&*gwpv?nT8k*AMiUcNMz zxjLx)R~|Q)c{#RxUB3DV>vC`TuY9?h=QAh2mgn8W`g~Ll--mg*`9RkD(f7H*c>3~v z`L5hb{w3EP%6uo(=O#YqnaKHjP`T+`<_FN12g+;Z&^~99M;8ALC!a75#DO>v2jV~+ zhy!t;m>lrFYyR%Y{a5Z_jx!gTKfEuScfG&MALJ_LAUT%Y!5p}S*OP;p-^>kimcHE2 z&4K1yb68{6krH#HoXead|C0a6W#m;KaDZIFJR(mpugNDq;r*LS zpFG2yEiaQ-ncL)l<{EQ;8~sB^p>i6z-3VUS{3BqCkN=wyd0t~`<9||xkJ?E{HpAeTgY){&|invW1rl2616-@z9XNK zx5<8m#z>_eP0j@OZA{fvGyR6f;(x(V~AQFlP)JbPJ}6UtfT^6OcbpUAhG)0e-< zd*sgY?bXb!W~f95B1ru?Q3^X4!4k32`NC0{ddn{&)r=J9)Yzve4BhCD#7B8M;!nn&e?%Q#1V zBWGAlUrr=HlM}UKU2gDY9;PqvGvCWc%zttL^Q(DR4j?x%N1Chd#7Fb_%hd9yiPY11 zA97RqlHA1n`!nXvp>k0<*;dY3#hkoPULaqp#+-b~d@d)Fm-J-L+$s-|Yqa5Y_2eq%YV&L**5w#-5xJjyPyW)D`L(D#Pkv!umqSe8eH~z4 zZa0K_0PFHh^R;}y{4Mu8mwnw)`HY-HPATt_S8m~TTzSc^>m0FehJ-i8ODaXtdF3UpU^Im6Lp{=Z<4RbAAOFomvf$Cej(~}gQl$eyrd3w z4fImxv z2jV~+hy!t;m>e*Fny0+)o3qSS-kZJeo6F@T@)NGhCFXncytzsqVjd_aFY{UAKpcnz zaUc%FfjAHc;y@gT192b@#DO>v2fld+3b~TJs*r2RrQ}=ksX~q=-;#gHspMPoEP2>B z|A}&99EbyPAP&TVI1mToKpcnzaUc%FfjAHc;y^JuAWxDr$)n^;@+`TOzti$}RPrym zmfTC;CC8Fq$-VsDRxx>*&k_gXKpcnzaUc%FfjAHc;y@gT192b@#DO^Q%{w5kl1Is@ z3OUn%;!<)eIhTA({v{WabIHr(S>OC8%879x4#a^t5C`Hw9EbyPAP&TVI1mToKpcnz z#pHloN^T{Ol55GO{JoUFtCCOo`zpDW+)GX+7c1mg;aA1v2jV~+Cv2jW24IUv81OUaw$R`M+Qlw3Q2a`|9wd7v%F!@$__Ac)| z4#a^t5C`Hw9EbyPAP&TVI1mToKpcnzaiHuS_)q*wP9?{ZL&>WOd6d83l4Hrc{JoVN zOs*AvRrddE^9pew4#a^t5C`Hw9EbyPAP&TVI1mToKpZFr2jo)z{z~p7xAON`axQ;A zCBKq$6>=##m)uIOCFhcB6@!QQ3~?Y1#DO>v2jV~+hy!sT4#a^t5C`Hw9EbyD=YV{w zkUzv2jV~+hy!sT4#a`7cR=n`$eH9)a;QRXC4cgFS8^_YcU8#4S1!n{Hp zhy!sT4#a^t5C`Hw9EbyPAP&TVI1mToKzVjRj^yvImC3%y4Nvv2jV~+hy!sT4#a^t5C`Hwd3Hb^ zC0CM5$)n^`a;`!iCGRTaT!p{Ol8?!)v2jV~+hy!sT4#a^t5C_V$19B*TS0#^vJr2ZyI1mToKpcnzaUc%FfjAHc;y@gT197129gs)KvE)v2DY=!rO8z9@D&$`B zF1eMQN`58Zl6RH;C(J9vfjAHc;y@gT192b@#DO>v2jV~+hy!sT4wPpH{JoW2NnRzF zl55GUkl3&TgCtZ%y8N9EbyPAP&TVI1mToKpcnzaUc%FfjAHc zzC8!zNpdQAl-x-UCBKq)$)n_5{=Q1?C7&wfU2-mY*|+ye^Gb0b4#a^t5C`Hw9EbyP zAP&TVI1mToKpcnz<v2jV~+hy!sT4#a^t@a;Jucam4hujEwnE%}w)N**QOl4HrMv2jV~+hy!sT4#a^t@a;Ju zuaYy#mkK$R{7EiV$h``Amt0HkRmiu(ufDy1#(AYU5C`Hw9EbyPAP&TVI1mToKpcnz zaUc$qX9wh0{%)#}Qx)v2jV~+hy!sT4wPpHv2jV~+D31=v ztK?VmCOMTHOHL(sl3&Tcv2jV~+ zhy!sT4#a^t@a;KJ$dd|rlpISACC`#y$+hHP@-R7;+)D1{@3iD(-`*$9E5(605C`Hw z9EbyPAP&TVI1mToKpcnzaUc$qM+f9jawj>Hd`Vs<$MSbmaw&O~JWKv1zmjYD`z$$_ z+^jr$miHV7;y@gT192b@#DO>v2jV~+hy!sT4#a^t@a;JuuaZy6vE);{ZN?s+G zl2ggOv2jV~+hy!sT4#a^t5C`Hwd3Hcmb^gv2jV~+hy!sT4#a^t5C`Hw9Ebzu*#UW!oJme4kCIOnaxVFk ze9PZq$*KJPmb^-?B_ET2m1pnr-s3V9EbyPAP&TVI1mToKpcnzaUc%F zfjAHc%CiH7oJ#H_Z}N9naw++i+)EB6-;!6!wd7U)E=$hk@4CvfcX{t|AP&TVI1mTo zKpcnzaUc%FfjAHc;y@gT17+`kyh%W-X({UbIGUV zQu3`ro+S^HYn5m3^4{Y>9EbyPAP&TVI1mToKpcnzaUc%FfjAHc%H9F_mE1|*B!7}m z6>=;&l-x_6V9EbyPAP&TVI1mToKpcnzaUc%FfjCeM z4is`Jd6xXDkUPn<>U&Y{IK0_Rc192b@#DO>v2jV~+hy!sT z4#a^t5C`Hw**PG;l2gg8{O<+jRq`phmA|8scgd&ZR`MzNm0U}HCHE@3PnMJ8Kpcnz zaUc%FfjAHc;y@gT192b@#DO>v2a3S~Ig%VpZY8ghQ^~dDT=FXUmb^-yCAX4i74j=N zSTT5*&kzUVKpcnzaUc%FfjAHc;y@gT192b@#DO?ab`Hp=)({7Nn+$C7uI-6zY*aUc%FfjAHc;y@gT z192b@#DO>v2jV~+hy%sofE=ojE6J(kTJkG-mpn>7CFd&SUvezDmYhrORSX{HGsJ;7 z5C`Hw9EbyPAP&TVI1mToKpcnzaUc$qoda?!e{UtPl55GSK* zIhOoN9#(drEGNf-I1mToKpcnzaUc%FfjAHc;y@gT192b@6oUhDD>;+gN{%I$l2gf_ zv2jV~+hy!sT4#a^t5C`Hw**PGu zk~_(%v2jV~+hy!sT4#a^tP<9T;qvTNjo~n>D$+P5C@+ov2jV~+hy!t;7#xrz$)V&}@+gh1^Q+B&U*F6>=*1)qmn+a;#$TFrOg~#DO>v2jV~+hy!sT4#a^t5C`Hw z9EbyPpzIv*_f+yKxl|#~l26H{{5_TbeV`mnUM2UEclrA+c~|&V+5I!k$#EbK#DO>v z2jV~+hy!sT4#a^t5C`Hw94ICSPm*WJndDUdcZKpO`IcO&kZ;Me zPF%HCmI1mToKpcnzaUc%FfjAHc;y@gT197039FQl;t>jAbCOMVd zN?s+`l5@$av2jajt?|{5YjwP>>H_5f+S@J6Rl)Or=CC8F?$*<&A zav2jV~+hy!sT4t#SC$dlw$ z@+tX~e5#N`$-CrQ@+^6n9ITLQ$*tsL-`pq3NpTv2jV~+hy!sT4#a^t5C`Hw z9EbzO=zttbP9?{ZOUa@9U6q_lE+vnWgUPStUve*bm3++KV-=&P`Al&j4#a^t5C`Hw z9EbyPAP&TVI1mToKpcnz-<$*TD7lr~Np2N0z z$w_e_4#a^t5C`Hw9EbyPAP&TVI1mToKpcnz#prv2jV~+hy!t;m>rNu$*JU5a;-xCRLG~~Sn@5omt3omSNZ!Zd08=g zo6i;p;y@gT192b@#DO>v2jV~+hy!sT4#a^t@bw*#Bgv`cO>(S4F6Hm2v2jV~+hy!sT4#a^t5C`Hw94KZ7 zE+yZRYZY=Rd6m3Nt|hv2jV~+hy!sT4#a^t z@bw*#JIS%+O>!)GmE1`VRmiX8QF1FemfWk5hsn#n{wK%@aUc%FfjAHc;y@gT192b@ z#DO>v2jV~+hy%s!fE=liR~7#5O8z95l6%R!m%K|(Chsa{Z}ZvWKpcnz zaUc%FfjAHc;y@gT192b@#DO>v2fn@oaw&P0d`b>g$hG8E@+Uc#zq680$;ISa@~c9w z_4PkNPKX0>AP&TVI1mToKpcnzaUc%FfjAHc;y@fIW(VX-@+$v(K{=NENlqo7l3)4z zDmj;YOP(bklUvEjv2jV~+hy!sT z4#a^tP|OaCOMUSN**Qel3&TEv2jV~+hy!sT4#a`4?|^(s{v@Z8PsyYF{gu2+ZdJ&+t z#DO>v2jV~+hy!sT4#a^t5C`Hw9EbyPAPy9>19ByKlUzwICEt>3$*1H{axS@+98B&d z=aPTPv;19GF?*ZO76;-$9EbyPAP&TVI1mToKpcnzaUc%FfjIE>9gs`Ot>ja3C;63p zOD-jcD&$u3EcuswOkO3&l7D^uPmmMhKpcnzaUc%FfjAHc;y@gT192b@#DO>v2a4GN zc~v2&k~7J*{9ToNO711cl2^&Mk1==Cj3tI1mToKpcnzaUc%FfjAHc z;y@gT192b@e0>MxS8^tKl>ACwC6AJ4$)V&}@~J`&Cig1jTj5t<|DRn>hy!sT4#a^t z5C`Hw9EbyPAP&TVI1mToz~SJ4yh$GAev2jV~+`05VGrwVzL9810>xAJ#b@-4ZRJj&l~ z$;0GTg&a%H_0`{bcE*7?5C`Hw9EbyPAP&TVI1mToKpcnzaUc#H4h|IZDSsy=zmh-s zJ1cpYyh@%WkCKDQvE*FxE4h~3>u~TmpD_-^fjAHc;y@gT192b@#DO>v2jV~+hy!up zt2-b^k~_(v{JoX@N=_x$l5@$wv z2jV~+hy!sT4#a`O!2$V`oJpP~r;=;Qv*cU;u1XFipORn6yX0T;F1ePx>u~TmpD_-^ zfjAHc;y@gT192b@#DO>v2jV~+hy!upt2-cqBk~_(_ zv2jV~+hy!sT4#a^t5C^`x19B*T zS0!JPTgjc|Q*tf2m;6fpC6|(0$-CrVaxFR6SAXZ(83*D(9EbyPAP&TVI1mToKpcnz zaUc%FfjDqDI3S0TU&)>1RdOu(l)Or=C7+Um$+zTKax1x*+)9phICz}T7zg4&9EbyP zAP&TVI1mToKpcnzaUc%FfjIEh9gr`{ujEd0DmjyUN}eUBl1s_A#W1v=?(# z(eI#5&`IcI<}aY0NIjW)I4bwLf%Qi8U!cB*dJXFD!KSc&8~uK0efmeC?dbPK8}hkY zQlCP7A@%8KCH60)J_)UX-p+b;_79l&%BdM35W9WZ@cBMa-x(anO^e*(b=#%JK z=m4}c`=3WoLq9@aWv(rHCH*JR|E2!}KF_W6kEGuP?T$Xl`ghR==ycYfN6$pZq4Sx$ zkk^rap2+&CzCLw3>K~y^(Dym#O>`*p%hB`DYnfk3eG>I0)c2w%qutP}nCpW6Kl=Yd zxBEG$w^4UTKcl}1U4Wj)dPC}3=#}WN&{NR|&>rZg?C(Q8oBB7@kD-Im39Q%PbF8PH z$$CY!4(siyx1vqyS76`q)KghMhMMJ)#q=++PhFFNHSRL(fFJp_S18=G>Xo zr?dZZ>Q$&*e=2o5_LWj^qkaPYjQ%F{X!<@c_$&R^^xN~grPRI9b5|yQo*7@1d8oeh6)Z)<7>~t~2#&^p~j5ef~e*e_iH}q~1r}nE8KDFQuM^4nrHV z?(?P_>CZvW^&bTKXXclqbWW%`dZ zcQXC+(T`cLhwfx<6!kgOw^Q$+?ukB!w&J`+)Em(E(828gC-pPb3(>pLdaU0^{Wf|Q zS`D4e+<#I3h5CN92mLRoC!hn-M!es;)YX~4hW<|Kdssh<{tWt$(yvYZ4!VN=0qQ?c zcS3)Hp22!g^f=DHi1nJ(SD+u$??!zDx`+N)bRX(--Hx2UivIc3&r+X_K7}GBQ<%Gy z*BMEDBK3z1?x(*2-Aw-%=rsCcsLy8qz4R;7e+Rt^J&5*3E3&T&`zp}yN_`vk7W8%W1GEow9bKn>pL!2< zdvrH?2J08|zOSIap8hN78LS^K`)Koc_P*-%bBDbRhjZsPCk%%D(5Qe}vwT zj$pnex)A+4>wlxZ2E73NHM)bj@zj;6pQfHeeGB??^ardjqP`ye4)3==b$#mk)UByI zpik4U!+FEeb67u@`fBuMbRN2q{fE#2sK1XmmUAAYKbL(!rv4td%wJu z^R_W>?rKN>N%|wGyHJ~p{ri0=>)unoLwz3V{cJMz8>sjDnbZ$)zWK%6a1HC`E$?^U zlf0K*z?}D~0i4&FzPa~I`ra?AQ#WC*4f-klchQNcyyqBdbGG-j-?4uis`x$PeW)7? zM{-VI>Ivvd)VyIXs>HnCr{*K?YxlD5{l5qGZ_!4aD_6XZ{v-6|N+aoef0#^tD*7B+ zm3`g^%{k@^zgMR)FURt}D@S;d^^-WSH97=ci<+CKGT#gJ{=S&G4ygClTGSsizXj{E_+y)ccxTPd+8j zxs3DWpK>GbqqnhcE_jLhKF)7}o`A{~{>)r$`rbc!)AxSqJ?DMq%xP7q=Q3~pt52Wn zN{Rf=d#oJA`!XcD*vp= zoV=(n^-A;|=FRusvwP8B!CYgsCVD#Sa){d0@1x$A*Dx<9mlJhou9Wpps9&WyFPkynp1#kY`{{2-PocjQ-HuL2UuSMAdL?QudXoBPbS?XiruO;F zZ2Beq^Bzt;4&7wUKDna#UQT)zbMnB)sO62nqMnD!dwxp22bFKmQVMW>^i(8j3GwLYb;h;~9}ab8>W2DB>r3$z+4moTrkray)A zhEU7phEn%N&EtEie~2z)|Kq6qZZ@^gFXW-0v9Av6ThW{752K!ejzaHdeIP0i@cFDf zXg%w4h+WhVagJPUKlKdOA3)_+=ThH+%0Ir#z7NqItaqjEkKTvghz>+c(R0}MGPS(t zQtH#t3z?HMuA+VxZOWgY0nATBJFz|iJxISEwa>XXQujmWqVmTRIDaAi`keD8^m()q z>++Uk=x?Q8om#HCfxaB3ChM1@x3IpB`k&N$sFzbej9!3_ML%Y4ANn+U06oZDW$HfY z+vs0dA5DE8_1~$VLEGcc4(jhSHw~40-b#Hfa|_Vt(ADU8<~~4IqjK2|%r&L|15{4^ zSJvCmmpk4|U;cIn^)<|mL0_R?UUGfjXB;>j9Ps|`-%rc=bGn^BxBk8C-=F?HZ{F}e z=RMk-?BA#6Oz+9&55LE~ANaj0&){v8m>q*{^ z-;45{*393=y7}M`{TlS!QG0)p`z~SK`>eTY0e!#U&D-8D>a%X%9Lo8-Q1guUE$?UM zd-L=n=G$`KGV}obJ=86zFF>o(Hy?OUG{^sib?;YI+1G`>_Z0JzoX-2DIm7$-RqQh_ zRHAOoxyMk?qV9@bhW?Rtd4ajgd*lk{Q(SIBh+(|;A+je1|6 zL47{w$dA15e@g#R=DxgFQG1`hjk#s0_qpeplY5V){s(h%j|tS>SvRjOre6iEhRSQr z&5yC~CDzR$H`ABvpF}Okl>^<%KJ)7zm>Wp{F6v9rmGtGREveU`*P}l|z3+Oz9>h5Z zQ181Pm_HVsj1ETsjQ$%vmi^{u%90-ZA9K%n%nzhL94$qUK)L>x%wgVq@EG+l=4PU! z(chpxZ+e~j5c(SX<(Eg&uZxzl{uAm?(RMqV*No>9`Zi>|BHTe^b6Fy zH=4Q)Y95@=e!1uSteb0UQNPFBi|7aF6KF@~o}!iy+)V#t`sUy%^jp!F7vE1kjdeNr zMbsCf70|yiFNbW$KVWm+rOcUg%~>1iU&;QFsGRnn)N(0vSZDetqbH(UIqzxe5!4l^ zpP;@SeFLqG?qu#>>XXo6sJU<-bKB^zq?Uthp)Y^f#`+-o{ZRAnOlo=LZ<%jKzd!XZ z^g~n*cm;E(qYYXA6*`yx(bVSt&D0}V?@PTHJq7&%T9G-Qd-+^Pjqwa=@2(QiqAEp-Fxsnqk) z*U|lGE#`($mr#F!p3k3$XIYoeFQIONjzf<@Z)bly^q1(#I5EVf1MFd#PWceuw%_>hGZY(AKQi=byVAz90QF z(9X=aM+eaNc}qv?QLOK$ZbZF^`d;*Y`gf!A&>AZ5$LIg@>FVr19GE~pV;m?i4!pvj zOLYwxHxjyRmw0Yq(`fsA<3-i6-qjI3TIoJI3WA^#I>iyHaypDPKg!dkEqkP2g zsjHaxKJNWR?j%R^{gu>+6{F-Z6x^-l$x} z{PH35@|`x+=B=UBelL!pmfLur_#J)kA6|+3HHVr<{=s=Cu|5-h5;Y&*K;0cRAINPQbB_Ge9J-W#6|^$@Uqj8IgIV`px{!Jq zdO!1i@6V%_J8q=zgjQmoIrv5T=J&0vo4XEB??%mY&rr)LB~Lav;R~2OQ`#x*V3Pk{uwpT z-^N@E`lZyDplea{x!j>8eRJ5W)U(kI=oZes8GRe|-fI4-!hCJi`@i?i(d;`LH3vP$ zyd19y^()Mo!_2>P=pSJ2OzK}!Z=`;d`rp((hcKVYOa8^&iRcCBHq?CZWj>=Xr!^0o zPv7A6s-iv6Yfw4$dDP|(^Vbmi=CucSz1`>z)|DCjAMN(U0lxK|iNInc5uxKK;|tpR(SUx&rk?YB`nstUl{^pl6{YnD;r;ALy?{ zuR$MVt_5{9>RG6~NuF7mek100pmWgmtPe-m(7%%UBXkgbpF7A)USi!Gdx$x6>ptps zsLvChpq4v7Mcsk(no~EY9!OmYZHi82eF}AN>Q&S?qnqfzMSVZDdHZFh7lTdFVUz<*%#Vk6yw0z0@_SzfbLRGN1d*V|^X^ zTjr`$pGjQ{twCSTbTs|WXg${dirz$jEcz?d=l=4bHO$|K`h0H|^L5eIXcOkfP|Le! zP`6~g4fTDf{C+6)IOe)i&qu#Qzaq6fsU!6rtUpXGFTRy}oPG3mv=RFn^6%4y^xr`z zp~oa55yVO6YHV+?5|1tXiO}&uX`_(S`@+G;%Hv00MW$crqm_y_*&6vLheT4Zg z)NQE8pyrhR)VHCN(7V}J6|F`83ADHC)UQxKiM~$X`>^+BbDcT)80LS0$`9+a-+Rke z>H+9c=nd>UNIeDJMBn^qo|?_N_p4W^TQPqY^%rPk`tn6NoLugC*3E5gspTZ*hP&u{ zUo|%^;&o1No!a}|2lVBm4Out;yg>abbB|Nkqc$)1rCx&G&78UP4EjCLnXJnblI)0eLt$@~-aFQe{@ z{)PS;^eR-Iyp!6Tvy**QoaY68M1Kgn6nz?%Bgp~esKYpiv6AEXKR2U2(9_Xt(IeP* z3AzCNg!Mht_oDK%Cs~&ln&)nzZywxC{W$w4Q+uELF72j`Gkjy}iw zY19p<~|dOPYJ)N-PW=+~$J5Gqgl4(oTK@1y2{^Qq;-a)k#t$DHc(h0XNu zN9Bzps9T~<(Noa3(DtZ2?_bn%*NVI!xuAKg33KMvj?9(PH-9&#Z+@36R%gzf`8u`S z><#wqMn7i#UTX7k9s2L1@1b&(LCl|ons4h+%ez~%Pu?VdGM8S@{0iodM#rPASw9bL zL4O+cLhA3M8|Ys|Ew_~inbXf^y&Ln>s4Gza8I^y0#(FjS&G>tI0)6xB7S`v`uR+}i zo$EaNOX!Z=y^!HG&ram35h^}G%pVV@iTd8}pK91VwVn3j+h5nnlJE-qL z<&$$*KNFRs{eiig(I29pqDM0?r|U#5e_7Am40JHs6CI1*gbqdJV>fc%edr-{5_4xz z%e$IWw@2lLvzXsc|3d0@)H_hQl6>$y=H$dia)O<{P{f*eUpAG>L;kZ-yEd&ep!V%bGLb4 zj$yvr$ef&OI_Hd}zn1!J)W83$vTnY4j&<+(-nYHi$lC@mcRBlCL>r>s8<$Z}M*aT0 zl>O$Z9ju#AhqK<7zPaDOuf3OePx>iy@`63o=83wTcO-r9!QJS0q5oTSAZosP#P@^l zV*LZuJk^VJ^TQ>B}|Vr+%7sx&0RB=r5$coBDb5UDW(`8FOdSH*elce;_3zC(kfP>}GyHIs{$J{04Ltx)+t39!vch`<|n|29;ZnrZ%5#qn6Xm zqn?V&ohDL0&-vG*2kHL=HHV%<{VsD8sD1v`fcm?v%heiCoAbsn_kFZ6dL?t8qUWQ% zSeIYkM1M7EUVMl-bM+$X0jRm{pVSM{b)54U^$7F=bQWqZJcId>=n1ULL5EXUWc?U) zB`Od919NiHBdFhCPVVsw`rYYQroIksPk%D{L;Ag`Yg5aAd`?x5{>{wEOU&8-VqLy7 zpZb^R3+$VPuA|?XdNC^3kpsxb<}mN?hF4R!K+Wk_Q4iugxz|`~dDt_|{gwVJ)IQJo z8}-jwUxv!7?x1eQ+UUK7a{TYn??ZnuwLI~U^heY8 zdBP&<-mF)lmizCZmdEy?K8`P*bA3JZhxDIj?nbm4>!+am=x;<@pcPPg=k3(zp*OOB z4fW~NU!Z@Ye-U+aYI&CYtGwj;yw5mrI5^<<>683jK$JMTH( ztIT_Hg||5G%kMdA?@fI6ZE zV*V&TPZ!khBlD2?ZzJo&*teY8+~a-scdW}3-{N6W*|CV!}M7_^7pA~1`nynh)lID5g3f2%@9#_KKaI*)CNekK{nW>y-uKOS=DJH+ z_ddRr{oXIU_Y9$bB&ifBv`z78t`ZMpn$oszc1Md~yXT3k~;aoYz zo1D`g^?qmmn@wMiuz>xq(4R*AD(ZdKd)4jqmoh(t+WRN%67N%kS(ootq&CNPrLKaG z;Q%>fGx{~rk*s$>L>Qb9;KB0b&^W+!ivGuIiMCFm*x6fmr{O@Dt=AmoR zmZ-VIJm$SnJ~M>52hk5W-#jZ9X~KFv)SPdg+QhzXtaqpW6)NvLopqmE+{OBCbPzff zox=QI(Z2Lop>jFz!Nci$58Xg5|Ju(v=F*+44@A#GXQRJB*P>6b{|ok;(8jF)29^KcLv8M>%zQomVyMsh zR{G|@*3^wq^UqM`%^UL8{`4P0PNd%um9uoDHs9aG zzDX+Uz0f7-4%GbgYijvSDf>T1Uu6Aq^bT|&>qnt;(4VqCgIbPz8MVCNO6q>-chS?( zU7Yg{`eXXt&=#oApFU=;Gg=9qYt6izZYlj<^v|Gv1%03XHE1{b70@2&$!I&~R-(=6 z`+TQ5^%<(B%AJE1qyuS)$H+JL^#bF0ypH_0m&F*gRi60Oa= zoa9*gXQGQ(Kb^Wa^PF1}o%%L(3%Z~6cd302<8z!9te?;NKhX2}dvzG=x1#doL)6>Q8fa(sAC1y3xt01D z8gkVO=*xdbQEx=$I2EXU?sf|2oQw8BpJi?ebpvX-tNd&qeYw`%)TQjZ3Vjc)i++k; zgPwrOzp7EM;XFC*0Q&c$FR(rn?MMG2^q1&%)(;0Jkk1$g%8LWb`SZDeKcC)jKBSgI z&0uZ<+7~tN%4_B-{%-1Xs8>_3 zM{jWsy`27&s5#C1h4+T#tedapCMR)DUDg|-=HR$!qkfKB&b5oa zeCIyux0pK;eI0$1^@pkVpwH01oce9*&!}gi_2{oay)T=yr_(^1s-^?A=>DNLxq2?3w-!b%G;5_rzE%cYspF!>Y zY7e!1r8{%x*2%2PdF9oYFn54;x#?Q^H>0n!{xbSFeea)gj#ugbnK|?M_o(Gmaxd?b z@)~oGx%&am^IqAWdNV4wIgaz?X!5xx%xz;`4kQnLiM|}b+FXuc@eFwEU@KO4e(1qx9bO7__i5uv9 zAN?Wq`{-2WpQQGA!aVvHq2A~J=6?F}XZhRjSzm`PK)W(8x7kO(AN^+N8u~R+^T!pe zZ>3LJaxMR7G};o~jQ%(Cz0fnzWvt6}%tM`7mxq~aA7TD1^nT`$lE(Ba)A$KGAFYL& zcbZW@!9Ka?2>SAfiPZA%Y0S$_K4$$7sQK$c=H8>Mzj0qi3QuIHx1^Fw{Ks zH0xE-;i!4f=LIX>&-z2C&xLMcT|RLo>x<~$h|2f6QU8m%`>Ewh=GFf6vm< zKEIKB$S0mnQ2Qh&xid5+IDenfvTI-dFFXk+vV^cmFWKx5b^ zSGj2&qwE?{g_)zU4dGD(24pJ*7u|G=MP!`0KE^DTePH}iprfPa1Q_4l(b;I z9sPHxPp58!cB3y(Xhgp!{gbFqryh*Tn~tWQ#GL%Tl>S8eH&Ks7tI=Ibr^)Jy?^jA>J%WkKC2D*s#an#kQ zH&UNMeKI-}ZO(dK>QB)Z=wC#AF4};89aOG$BkNDoKN>v`l^2$mT%Y$D2Mz}Z{QLhn z{+yo3pJVTp2dFoq%TVtxPg7rlZbe6No`3IoU-$cB8gu3u|9;pX+`| z-~0Kc)P67ig7dwvd4KZ0G=z1(*IsA8_y1R@{XV&YdHKj_*3FGusmkG&t6(--o2yvNL^ewh8< z!>*)Qt*#92d7Hx{kC(Nt=;QaSluSdNZHJAUC^@jA7?ff6_eM_icXZ}M} zE+cO{o4LnOIq37$li4TVGtYg@`U2K#qWjVQXnpil)ccuy<#WzC$a;756;$rig?co4 zCHpp@70|m_ACJn}z3(1m?kLt*qvpr1)YmX)p85%WbMjwVmz$b*%olevKa4qZLuG37 z=N{&sqd$n+++#kLW0*_5$D3E>q-)t<8NHKpt5GkdJ_=n*|0U|vspVBO=*tO?q27&F zVtyL+Otcw&`NwVa>!OoU`STOZ&qwcN-JG|R`UuvKrkYe!#> zax}I1{C?(;l4tlo=FRJ<<*JLQYj92vv>N&%>vBEw_FUHcq4SuVKz%DJzZyi%$1Ra> zO=RD5=uOPsLVZ8=Sn7XM%d^g)p33?ayg(=F6|BoQYSWi5R%iWobOY<3qVkQ#tjiJK zV!a~TjP)((KJ#47#z6yO5 zy%}wU%4IvS?46WSde%=+J{Z>4TP?Q_R9^nEVShxJA16{uW#HuK}???8R-=I=26M1Ldd^Ru(r ze>gCKe8xCXUL5f68Sj5R`E%<%%slNq-FuVYi+)e~eds;KzZbn9`uE!$&Nu&gul0Lx zH|xJc{XUw?-yeQI_J?QsYL-u)( zF;AJZHnZNH^SmFlr0?Iu53$|@^JO#k%*iwbb6X|48lM|K8udcYDwIm^ttL zU8w!O{)pG}9&28bw|H+g$8KhSd(?Zu7S1s@`F$~r{s`2(e=_^MFHdCsc~lAC6}=NA$|Fk_igWW-jls|n_EV+|8(|mLtmudmHHm)-qiKb z+PTiZ2;Peq)9=db{heBl^fZ0%uTQWpcap<;e{9VBR`dpRC3+3}+n}rHFQt|f$R+M# zeFf{Y&~xa^k5~)p%d%ymZPrk$8u zf_7v5d{kZ{e=v{MWd1$0KY9oH2K!r4H>2)>)<@4|{UYj`=qdEiLe1@Rk&l`CKI^|m ztE0a_w=!q`pHBVXtly4aPybzX7yT;eF|JcrrZ%75Pv7Sja)+DgS3;Me@|rt2XD5Am zK}Y&c`Fm*+wH#|U^OvJLP@f~#V*bbIchPCgEk^G{FF-$M?jh8i`W*EOs2t=%YM-z8 zd}aveorm^D-(lY8TJp<*te=Q(Lgms;sjH#p=9{SH0`GDDeEL2&y^X$nSI$0wIiCad zqLyFmrCx)6hjac$eJk}rbS{0L$6Y}GX0#Ob`K+AxH2QL|zfwPjRz;8G{Eg_j=oGXO zb8^e^^lw3ZzA0ajTYSPkxk+E<$I*YDdO13S{*~xV^l|hp=KhIRr(XlDjm||+VD5j> zs_4&Ie;Pd-y@>Ujs8^w_>9<9%LFHeRCHSL(?L47av0P1JaDd;xVYoI?x7qI?I>PG10 z^czrj;`93)^cB{(qb<;0=nvRem%1NydCB#8pK;)DaG(c&KFw)A;=6L@;^8VpHbrI(@M7?Kw-}T-q5Awd=oO$ne z-FbcQ1O7e6SjiIq-lFj#dMoPpl==HH=I=ti-?V1_Wz_qL_ak$Jx$boK`+Z%R+B|8V z{Rw^VN9GlCb!T4B@5xW8%~Re#2hsQM`Om1m4|=bi&g&&gH1w;p1B9{bJDXH$EBk>f39{zlaMwfALn+I6gd z%sFx#IimM`dB$$$%pvlbN9lWCHupYFUq0~)wfE%;y#BHDpG3`z-pl21-rv0kd!I9B z{E_`vq28lZDya8Y^Mm?<0M9o%>iHje6huf|{|CC#lz=n^{;! zEpOOF-}{REr3?M~%&(=MK|Ke3kA8P*^ON@_?^(lHH&1k}&IjfjoPHm3dN8g;(@djFh3Zb zh;BrWL=P~(6+MCeW7OW857L*jt!Lf5*qrqa^v#DWsoS74(bt%tiv9uh-t9ep2lH~$ zN2upAzaG5-{S)hQ%ZBvLndWu#vpHe|b1m85mHPi@?_R%tuG9CAUll_!5z$(tFo_UR z5=&82Le1C|WjkqX3adquK~W76O)06dt05s|n@l#@tfGw>+Zsg52ur?;NhzX!H@=VK zdt-ioK`XbOj^jLEuj_I(*XMnn$J<&{0h{x?;}EQf@94V(lgD09cf)USmvg)5qICA# ztM~)?CGko;58ub^TTSUvxWPTy-#+9o<)4MwW7eqmz~npo=-!x|ZH)6Z`Ge?w^iy>5 zr{VlPm|QM-L|Oi0*xUK+g^TIgI2E(6Xa7CG|I|IT{l1*w|C-M(m`86`zm@(cy#||L zRrTU{h+hsL;J-<~fXDdBL2C22VD{bT=v(m5&NZh?(R1*A{(o{mCPyAmKdJ9JdLUhz zeg%`S9-_a}_ZNB^{Sp@D*Q7tfC{_H4! z0sc{aI&S4p$K=qL(|PYPk)G+?3c4Ac_Y!%(*^$2y*W-oGCAa>J?xDVt?ujG#Z(u$C zudpb;1)ZE~FrD{u7wbFD?}XL-LO-GY62Cj0JTQ6WLiOS5XFFekUmHuS55}jliuzbO zIp{b1$=CqfVPE}Y>FQXBpSI{R6k&l}YP4*ceHlz1b%X_ zYw6@+C-t3+b=6nWnS(Q5SLNTV?;;$@&*y#et>kaXe;(4Ge4~_e9r)SfX7O*pBl?=CXTBQ3&-^lh{u=*=nHTmszkr`yE6?-fjM*oXqy0huc;`FNZ{y3@SN&1? zAf22h`&t9_@wixD_VwfuThx;){P(;)K)nlQeyl)eJ}Bb;%+4K+#YO!8!OGYklgniv$zEMTJ@Z@k#cKRxn7u#ydNY3JqK0%C zY~}k}ipB7BwE5#F`dR1N(>3TH=w*R?pm={4e|KWc3mH^FHE5dKV@? zX`}xY{%v&jlkDZ0lLzU$Ti*qk{bem(2TSPhj*a-qHx}^U}d!g(b+250&jMA4px1M`G<0rqW#ZOMXN&R0~7Mtly-gG$k zVfMAR=nwHG=ij5VXZ7P}FDb452qwo!{+Ycjc}3onBsVVVzU;qwuke`r8sa)^p??yc zocC5b`%pDHdvEsdfABwXPffZj4Fy~M;Tv49uOIHf`sy#>o&2+WpufpKg(2P)#@|p z{n&~BP&8hGyRf>xf78S1HP{QM<7N8#;|V;bp7#Pr`P=!)57zOM7tN>7$8!1?(sQvC ze?5JOK7ybE^FE|4|0DiX%zK!X>br4g&eK)s2kGPo zPw|&xdG+LvW9Uli$$y@pn_v%2?$eXL3a`Yu?yE;> zlUN)#;K%qW-h+qSvkLPbqcENP=1=q~=bomMwE0FeL`=T*bJ^?9e4l520>6ZxK<3HpqnRK5+7^uW`<%TYpEH>kv*%3E*B-OCWG+9+ z&s;pt7kG|;2Uf=9EwkwCIeGq;<4(D3Y=9uT=+v>@=GXEv-cv(HU(KB@B>&#)vIWos&Zl0+BEnheL zcM&>yLY@;f^)JTE)m!MyMagaM=Vy-UKxePa^Ci#2%%|t->*xEZj$JW(RG!btO*6M7 zH~B$-=BwH6J&jXvzP=mi_H_1&A#^SEM)Y#bo|xx(_Py+z7wXHLkUe89KYMwet80Co z?8BKqZrA^kdgh7D&mGnCoIXci^5W0cci=vJQeWoL%tHs&v)5!E8>T<|R!x1Gw;HM! z!`A98G5KjpdRETs|10k2H^zJUnPW=u2Vv&6%u(z4$#1thUm9Cra`B4#vrlL5PCk`= z<9&U}nP$?-H*PhD&ERLxOS!ylRs4CkHGJ-sC)X*nYX&p znbVHbP4%CHoA`z4Mf4ubK9fD`CVuk%H=N5pIe>l<+v)e?Dkw_N!}Iigh@JS;@L5d0 zkoO=1`P1<`=d+I{r+S9}s=fhu0VeNnqOT0U3)aS6>dWY>F*#-*^_y{&dI`*XlI%;_ zhYG1z!|eBa-IM$_xz|Acxpw;KA3$Vd;3Ct{W1GZ4fmDjXJ462UyI58Khu8;CU2-mchO&*-iyugHQbKbqe?q> z8Gkuu4_ct!ng0ph0J9%YqO_hoP2O&?MJ7zgo_msaK{ z&zVgp586amaBd8?;%}pS;7)#b`YX(SK36?^`uXZ7`9pCd|8?xfPu?|>9)^weJw_kH z*)@3Kl*CFKgms!FFl}s8!p5`&JU&M zU@5Gi-kZJx*YUrmdtq|PE0du zsyJSKCcT=@`~24Y>X_W;l)h`-n_TRaz9x78H|uYRJ#i%_*Sto5asC(d^LUz{{QXY4 zE{?*y=lPUw>HH-)oPR&o=dYlDM=!wc{5kj@{t_~QGk?c3KY^diPvHOdJ-?raFnd?_ zy5vsT_r~}=PY&>wbJ>q(sb^o$KApKUpX2qN`$&H+%p5pXJ@a~=gPC)itFOn*)9=#B z4Kgo|=4VgI{FD87rh4YzmA;=m|C8G!H^@Ac{rK0;SH$djnRD{oO}>};eVnh8edi83 zdwJ&658RvmKYL9x{=Lrq_r95Y?4-WTHOa-2KkQSl=$>};PMn4N)w5@8qOt}weOdrK(ozH&u5S?6g3B4ExIG6l5``}gT2h^uv=CjP_o%q>f zv**3+d~&Ka`Wj>A?d;J*_${2fnLb7*cWuePlz$&y%g-E}IWYTk_LVyNvPYMsGcPWu zU0d*`KeXllh%e$`oTBe>%-)xI^%H&PVCLHFAMApY&S#&@99qqMm;Jqr1I6^eLT66g zL|=}{ljrMqUBOhk4tCXFoBlJs0Fzf0rpxNPj!vGIJ+7g8_WDNps$duOCo%K*TzZ1O zhj0>RzfJD{t^N9&5NN3;OKxbc04tgVhx^v0#Gsh29FNzcO^~Yg&55A-? zx!y>6gnDPZn|~Xw!UpO;VD^^eGuc~esDF&V$GngF#C>frdrCQd7xR;I{e@o}U&Z7w z9q8m1HJndg*-yO^Kl^KYI`2dBek8eHaRQ-^9R?}l}1NOkYXQ@i}bZ!Q{f<6~}@sr0Mr+edXFz<~Xrr&XX8~r8j zW1 z9LT>Pw_peLysufwACJkYM(Asb-PPC7op2q$FSg@9NWV@O#{K+X;c%>?-i=PqTY;ZE zqNDns_)p+SesYh^^mO&@xCehyzn`v*%dj-&JPwzJT;DPNkMu;G!%u$QfZq+1^Ch=l&d>XT$DFUq|GDh-XTHxf zKY?GuPax0zJQp+n=ed2@f z&fKz}&U})*BXjn=^yA|=7^h+8v*Zrdd>@(5vR`C>Xz$!7_*cx_K7r2jHGBV`_<6o& z&c2cVxv$$Er(^QgmUQ;F?6JvNrmElVzU){1>5DP*Sw%YeU2>Ft{Bzxx{Gtw>y(76= zbAEEin)C)-<-U?wg`b=_duQgc%zc?Ni#nIRBXd_j_hcT*+&6=t`E3AQ8Yj9hd&fe4 z^1U+Zr!f0*=J4A3-^J|NmFfC83-{rp?)wA2!q2`?f?oua|7Tyy`-trKC7r)Yf9C5- z{8s#~*b%cYt)(+(CZE{ieDeJ2^sV@hu~KJ8u}mD4$oJgj*Bt#cyhYzHE*kD?@jKRe5S2?v*%{7 z8^%xGd!Kvu@b92c(%I)r^DFQdMB`)p6?Agrytm3;lRaRN{+*aT_CtEKdy8tg1q5n;Hr3cZ;ZJ(lNVBXhlqq9ffLT|@|xXZnp>EuEU=)B+iQr{u|D14FsTb$3o z4Ci8N9FCjxZ^Ml^5MR@GIh`D+7M*?kh`#L81L@`Zn&3K2-taelEBSrtW0)NCd3v6{ z1NZ>$R4+u2!tyv5@5DaQ{`Zx9poe?%zIm*^&6qrQBApzwl5^|%i|7vYar!TG6*@Uu zVg6ctK>aiNC^o=x>d6-`rSDcROwYt6{NxkE>7m#gdth?8vCh}yUqiR1%i((d`}8TS z%1?gPivMHo#kzd|ISP{dlu~~hKi1coKFfhA^o4jf_QT!IC7(J*w{!jpeP#H`&#vH4 z;#b5s_&exYbSK=-Z%lWm7vcSweCV*gPx)_Pb^b*qQB-}~-$&S$R5UYY$d&)JUp{*0MV+qyr`h3qAb^d$$&JoAG7z4)%a6 zE`8aL%hB0qH|R@lR7w4I{$zSBX3uOwzlq7YewD9_nG3VuUZ6j7X!7HF{FkvNW*<7` zzU1n^rZa~&rZX329=wpB`Su@l=C_*k5?^mRX8$>^J|2@_We!cgb*}mZ{Kh?XFnj*1 z^rct=YeYNW2a{9&Q$6$B0eUkgpUS>i$NA(JJM?8=&;FQyFO>Ywv4YI0nP=NOKLTfH zoJ((`Z=^HdPNQ#C&mORrP7Zuf-w*tk@JGyiU6_6dt2o!0Hr5vm<5$K(>IZQa|1-K7 zow@ibIyuBXeH-`>U_ZEtCB)4i|{uEciEU4hvXlQXU1x6z+HHG5O?{K5MEM_+Q3V{~Qp zhcWs8b?O!Q^Xcnx3;#6zIL_p+qo?2n{s(ipdiMY1HqG^AkI&wpJnIeT8tLnRtN3f^ zvGhp#Bqnc3{{9)iz5em^7W@xCds_03Cj3uuz5e8?!@O_H-hCIp43>AU5GEHYOOModqyJ9D=!WVoG4E57bKJ|HgxRkL(AC}3 zihdUJUN(9D5q>Sqd)T+>y4cBmqv;{^U+EKcNqm?;6qn&_^{w>p>6hr1^dop0lNTqy z_?&;gbIDnE@{@;5rEk)g{G}UzIUZF{PG6e;DsEIShF$oZa4>%+K7{9DZyc@vX8aw$ z3SF2UgJ0k{^<}u7{~&!4R>0Pn_krK*ugfojoA~e1TQTpK7SqYm56}~xTY=sA73i<9 zFTXo|K7P-?7W1CuPP&V}ymwf@pN7xi>-u)n-ROOoob?L26c*Co92f9c<8AmgZq!$V zE>GX&_crgn>+vtqH-^spxxW1U{Q6jt-vC?V9Q7V_L;4yz@4KF$e=d9dneX$=PvDpE z6Udx9&F^PE|MNN3fj`slbv}Rc`Ig`Bd>-VvoacX@=b1Y)4`lAhd~(v)$@4mMS3bY9 zx8=FLM1SUyJkM_6XWz@`c;>QvF6B9t{dbSQU*?`y=zP9s-}%J-`P|H0kmt_l&RwT3 z&%Nxq*^An%XD-h^^c4RD=6RL*F!RqK_3S74+%Dnk=eb&g&b*wxDDzO}x(V*fe3s|$ zVt!@J{yxjSFX2e_%*lCfWZ%txn)xaFQ0D2EozEPa=h|@hXJ4vHm()Ly&R&qYFLQSj z_3W{moZE$&*DC8bPyEbv$xkznC3nrf zpZUDDb8X#oHJ!b1zUjVw^mMF^nJg)lN)w73{bkAD;Etvc)^Rn@*;2HH}&Q-(g>sP6t!b0j}=!2NNsW{yn8|%;how+jm zRdVU%^riKGirG`Mr}yS3pZLnxd6(ajK7`pHl5^GJPtu<``5pcSOb+{;WU`X?WCq-{N6@@|El_*;^N@e}Kt}&UJnmKXZ5H_J-=& zCl1gTU@Khc{Ax^2pSq{Jhsm9xy`v67}R>f95~Q?}Y!x^6K3%?`3+> z*^3^gPdnELlOtt6&AvWW|8x4P;e;vF@vHGmV{)#%Z@Q8{ zSO0Zb4!7e7eaS!f^Rq8kr}Mrhxj_s5WzKKLmHZdzE3gNS!@S?g`-|T?m%a9WeWUnY z@g(Mb=2`l-;%+RiuMNJ-Uy0xFYthRvx%EDJJNDJTp6-fMv92_EKpLXsW z%zLz5>UrN;D6Zrk{L!5%iTZ-yWuDwV-IoWu+6(;vuK_~Ax z?4G-^A6C+r_goLq$zMwP`@GMeGmd<!9*zxghyM34 z`Fl_G$M`Q}E!?R7Yq}|x!e!W4Uvl9p^g#Sl-$eWue;$s)ydSxgz7dx=SBAcou1+UU z{gKZ5*}v%Pg~|ns{NWA$P#lZDgiPSf-|@^(;OFua$melB|1+Ou56Rq={WWt>o^P3R z^SsD&Kl6E4zxR1QtP4{QM$sW;|e-QJ0okb@n$vlvG(3dU9^KXm(j+lAoFr7UhbJ%GYWFAl6 znmOfW=NjRg`io=cvdjg)*SA%@JZ7FbL4Tm{FF1vt`J^v@HvWVi^(Du+h0b$3^Fa3K z?A6Piua3z{=h4r&C-ZFP_~gIIX_n~gjhP2#(8*7er!I1T=J;3W+c9%t=I2fBNe-I5 zppyRf>Zj<;$4mK@FnPl$edGC=-)GWA@ozW=lgnnGdYzxSEc5S+?hoG4mz=U0-4^HK z92|^pG@vhM}IlY9*{jX`%gLNi{UuTKAfDXIsc&hD$+;khw0VW z3-ey4hQ3|=tLcmJ9{xbAf}`;veQW6K)0gop^RJ-W(R=B9V-Z{Rc2PXl3=zn4IJxeIxiS=m)V4KA@iWKR@u3%k-nK*H;ne@;6{|{_LYK>AMdn z=}RuUke-ZP^wq;A{7Lj0yb6;i45DAv{{`K`gZ*JTd+&Aha!hV>p7X2uPh%BKK9)VT zhyHfz$%m3R&Q$M$$t}v$oiY1xY4?=GLh6O-iF951Iy$*&5&8=JPx}2F<~w&MKkrZ8 z=Kr04lD?cSgTKQ2)mvlUOBGeG9s0UM=}oxXxpj2*@~Zr5&efzFhWgjhf5J|fT(zyf zyRZ``2iT^60RI#{oPLA8)A!Me{$9N*e#Jk6JNfl6d0gHLyvHA{|3++zi`75HiTurU zN!-rgNKdEp9;QA$U;QH7$sbH7M=!*GgFlSU`=)99;rxH%WBl&87H?C33h%|~>MQ8H zZ_Ru43hE26gTCi6dEfc|yR7Foz~qo4o&S-a-1lRC-Vaq!?~9M9ccX9db-t$aKIMFR zxBldqW9X{RJwtENzZnN$DZD^`4?6Fw57GPG(-SY@zeKmBufeVSC+HRQ96W|Um%aYX z_j%?g@Jsj!H1T_xoF;qCQvQpWd7&HK0W$~Yc|P9n|02xi*$n-)`FTEM?#rH#yheLL z=IQL=H!I{hH^hCJt9Gg9b3F6z1bvxL8q%L&a=Av%XHLqV-Ai9`u;d3L^k)w#rtf9` zxpZ=)%)y!Wv+qsOKOT2D*OD$z=Q-JspZwr?^*l$eRnNXNT|N6(=APt#$$yf+)zx1K zU&rhh$upD3WuEBc>vqLW&Yi@UaUb@^y81KcCI`#jTwGt~ugoWzAMWesYd!boSS~=n?ue$0k?$jDJ*L=8^0XxAQZ{ z{+XWR+;f$NL>e(0i@b_b7 z^@EuGC;Q+QecRPDrzSVZe3iW~`A+td?0YlZlf0>i`{wcgLC?Z<{AY0%PQ>j0*{d?w zw^cul$zP_@rSkWnhtWmpu9$r11ND;FPd)SFRyuiLEqx#21?t(ilfM*EPrh_QU-F~F z^rZNhylkNJ*?&r?|Hyxm{*InP_ryQoe9V6HI9=Ph%NUbru^*N*Cu zZF6o9Kl{{reixjk{vMt84Dawe;$HRAblx-ENVib0MOVY@qgB=K!MpJ#9HhSucE&a8 zOE9_K*XpJ4*Xqf~U*I?3SEm=z+3T}U@8l=Pszh(WfzDro*-M+y<8UN)$C~(-dp6S9 z-?KMoPfi|HA>R)s*GpcJ_bZRO?;w`MzhVo_`-!|aOHNKA~@3e0n{u;J2a=(aBXh(aD8g(^r;X6R+ea@2kO2{+wKRuD&0! zj=tnG<@mSZaGa}eDvsep!A^fjPI-*ps(&Qb#{+nczMJp>CYRZ*ZxnwMT@J6{AIC}j z^7u5rwh!LP{CfPxn7sA4dQts(|5Zd^Z}q(I9m~I!Uj|R&C>*VCFx`Xxlx{@##mo7p z=&x`rzcnUD&HK||LMCwL?|9}X@N@YI&9$=h*%@VhcoYg zuRn8B_S0?r%*C0L_VY8RCZAi$Z}00R&uK{yRnK!T`9q#d2lXZ2$eh%{xj(BP!B_d2 zqw3R@Fweb%boS_P={(Q(((N(x>|#3m#R*?G^Ya8cdt@_u1J1zPae(_eV+nrdo;)}A z@H3C)IlP#koGyEMU493=3*Yp0G8Z@EUx_Q!@1dWevqxnvP%g;4JB0Co{yT6TzY=Eu zD6gLRKXctAeX}ro$_P4pdUBM^Mae|@yz`>0RA>~Gnd zm-#xyFndIDzByYJI`dKHxXJ2m)UzkO!_V9~ zlisW^d-@IhZkYToIsI_`nTt=-d4JK{x#ShuKlVT&sU7ojw0``Y=w_SBBnBCts|<&)%5)Xpp|Xm_4?i z{!Z9W{YN_c(L(;?{DwFKtE$(gr(<&3!Rib6voU+;C-hfX0(&_35S{$`B|3Xj-qXCQ z|2g%gn0&7%ot&wT{%iQj*YD(Kf9|FJ7=H@Ah^|Z*#a5Uc`XYVT;dyvSU-Gh6{Jc-f z9@|yl59-6{b=Zxc++ijCBK`-HgJqwq&)@By-_yypYVnhIB=1e`@V@@#nEiPgy$|y~ z?QQoZU(DWq3BMU8SNhKRkNB%F?>mxTt`eo<(WAed|>DsshYhm6$FQEs!ubH36Bm7;M_Y~df_4=>ELYQ2&m%ij6t?351 zPk(ZiDf}jQA-2(%yyHQB0e=DA6sz<9LjRL4Oplla1 z{dC@^b*GPF^5wB~^2ANfAH{m=m*W21kB#~5=;V*TqR&!ahAsFja3@Ywe;V_?v8VcQ zerc?N8*x6C(f>T1oVh%|Q@$SN-(M8g-;iGslP`WoUx82P?}0y;z5dMidFChZOZW*4 z^n04m$;^k{{Jv(6$n$En{%_T@x7VSS|L@;Tb>L@C&-_|XW1jo{=q38IALhB0&-?62 zbM@`TGQ=!!P`Wtw$03+KV-7tHlMiH0%Kn;tFY`$DwugLQ|Gl3-<-RKF{pe@08b3L1Nk8{e z>gT9eqVv3emCpW^JZiT7=&Gf1ooTWiFh>zsr4X@o|1~sLZ9=OS6w9r)lNfTbMa!I-R|CDgB)LKcw%*ke$ocY^z48Nn_woiISTpug@MgPT!-r&b`?Os`0n+vrmkmlUps+H-lfq z@7YW`?@u=A%lo9?s87IIn4Gwz{^S9*)yLxq_3Zb_2ivRnR_{hHN=*zygi@r`hdF?X(OZ>U?YxKkP zDEcN`&%d6&3R`3L^2_z@=5NDi_?_wG{ZskLMXsg$U|CE~u!VloJ)P-VbYZ*zo8oDG zZSV)YTK!9UA0}VvOHac?`n%G}Ut00A_byj|AbslJ=N=q}W7H4ed;IM6+xac=0i28{ z^&h~p{Ny&D(aBY|(k=C`#)16X>56y^FT%mt1KT=x5JzGeT%zySxDscmKaF2w-n%|Q zzksEkOKy^!YdgQFzVq>K*c`vYa{7~Z?BoyS7oji2t`1wZDW<#T&8 z|5JW>EWsa)d9T=nZmMr8{RusjE=g~s%g}A;uJ{d~EI5}Q%NUOR`Ss~Rblx-dcNCLXS5m)%pZ9;q=09Yq{(B_X9<6>O zHpb+%Yn-dc|0QGsXa0_7egZ$2pFrmMetti%!R+JNWAgc*JwE%%C;Id6cEd{#$hB+ZO!HOL>0R;%82%?p*fMh)eFL1!-kRt3Nq+W*2kFcmHR+pi7}jxro}Ux=+1nbbSHMwtj=mC@ z`6oH%E8`a^8X zUq)yCEXGeB(49_xvxKgM3vjx7ZlPz=**i1$W$vD%Z#8BQ&i-*Re<{v&U-G$y{NyRg zfwD&>NA9n$k@K0;Gr#?rzg}PF(#HJkb;%Rf=$jwvtAN=TldBcuCzs6r(#iRO>Y3}u z^S{T;r^z>x-;Po5hS_^-)8D!GbDV_9SF;a{(?3o9UU~|y;Xg{}{l-u_bMf!>WiNhB zeINe>UWeHuUZk_PWdEI#KKGo)x0m_4o+{j$ETxCi&*V13C8 z4)cfb$6|B5UOjo(UHmclta|d9H|crmh3U?C5x*{d9=(pPirFKxw>RS7uD=<5ix2KC z{0aO6^ieuF!VmoYcn|&-vp4UglYcIEPd)xj`a(K;?qq&@esatI(8=vu_`ZAVe@T4= zuESmG*6Q75^!-R5z;bxM`V!20ylQmb6Ktcie-Cti6#oSMBR!k`3;i5DnI1-;qFZ1Y zd`JCzdK-NfCKpN$l=tL=^moFgSk1ZJ^hSCEZo!GT6Q9D;*b;Z(OZYqz(*5afn7pYD{RXziea>Bt?f9E8c}-dM3-EIFygy3b@{sy7SV-R==_>Tgc!*z$ zZc699^5$r~4-4z>i^)?LslS0$)j!6f{Jd9P!B5_}Tm51Fv7Ez&nD<9L=mj|4`MiI5 zjDH>fY`QN^P~-3-RDD|KY3o{`Iz}Fb4Na}@;RLOAahdY zkw)&#{Phq$*4NFRnCIadeph{K>HV1f^l3Vuhwtmp=V$ije2#vYx^vwz&#~;SneQ@x zW=^Q@eCE&W&+FY&fP*o4$a&6vfOFN859Ray2L60Zu93MTbNy8JWX{aIl$cE9uOMN9ngP^Ktfw?0K1+lD}kL{U2ZFMdvd| zcnlU~pUm8QQeRi+vX6aBXK%^=eUP7A@liT+@(%YUKh9kHIsZ-ldoZ~`=GPbanTyIf zS04MTCm+bZn0-0(?qYqzFuCnvdYt=eVdkRO)TiP~{2rHLa+cN3XC6tuk$EBW{t11x zov(pkVdla$!XT8XU_k^xtW;# zAbVXs{ZrJ>!pi)*^e6Z(CKvov-vHdLei>$;Evvo@vp-Fu@73QBPhuBLe!X3PAI$!7 z7rjA$Q9Aq9gLL+o>{T21#hgnXFq;1uCU-cZFFEvb^-=t_bl$gQuO6*_uKE$2#ZN9m z7L-;$jt6lj{)E||ve%Dv!6Eg$XKT-|#-Bvz{Y`Iv_QaBO_RHjOpYZ>duY<|EUZwLs zC3$Ea51Kma$!lu5uQB#hpGE(U&c3&ppPaj#`T?w?{yJR^vv(G!|E_NfZsa$mlh-w& zi>Utzd+}@1&*MTIuf7-mz+a6c`7`KS>Es5p`N_>bqrcPF1_$E;_3Y=1=%SdMW553M z{P*}hoqTTzUCX(7n7#Wq>MQV79HZ}9yajJpKa5rRy>T5s?+<>UOQ{d1gNONV<5n!| z?=y&Qg~xCO&T&!R4_2jnVk3RK@EQKKxQ&0Beu}=6z8RA@p5^=6$bT4<%XV|VJAV^> zAzp)(a0ez&`iFBv`NQZ6bZ5F3-JiaKz8YV~+i(arz`QT|*!h7t14rR0{XgP8{Jj5c z!e58yV{)T@`fucajSt{b^__SLe>2?|cVpf=^wswb4!|GveUIHR@0Sid9~0(;YDH-H*`?u`afDPc8T6JxhIlXPm3Q3ogT7 zLMCwL?|9}X@N@YIWKYU`n9sk=f!VY3d6LiJd;C6U@5<*=KCkk*lsTrW&xOowYw64x z{prjJH_(6fb^c9fp3Z()Sv`A0KDRTUWv=Yt{7n5N>38YO2g!Zd1^?8S=Xjo1nG5ez z?}S%6c$8j&*(WNh=ee2ZfA)avH zIG1@p&*7EonddVPHPJsoJ@ZTU!Q?<^t0&*c^C5G0@{!DAnGdG9|54{(q%$A4=kLMf z8_7SCr)3|@T$TJH`)BgZ?IV}5I=A*s-KFzR} z{yf(w@;hMm{^VM<_}SlnODCTzMklw(9G-nBbAN4LcZ2>KcsD=XJ1XelKC&W z(q#R~Z!;&q;(R0Z?3oSu$-i3Cna_vn&mNe0yPUp{)HBy-p3l6Qc{}skyZV3a{CGP1 zPWI|K{LF#Lt;RW*d9OH~d2|iE+xhJM*^4sI*4LNZBy;E}=Ptl;cs-WKNzU)X7X0Kn zx6#?7vwuz1ze@dloWLJ~TlsV8BlHw{HD-Uw-abj+Me3P*->3Ir_PIg&vrlwY{|{!q zPrlcfpZq_0P<8iQih0kFdA+}LnZFzAOO8;6PCi(Peg+R=_Wu)fHO!uKfd0(aTZ>=t zOVF3m+2^xYPFJ6+p7%r*=@w`IZ$tK{5&Ug9%J~{}_KW0Nd-?BRVeI2v_T_K*6VaC~ z=td_$c)+=~{8scTyqBN-?J#{(y#&3H&Yr%A{~0DnOK!K5Kh(J%xD5AWU45r8dv#Ow zC;1EM=9u?>{piK%!)Nhs^}Nq|ou9n>CH3r=+54W;m-mC+^mXBv#^g97)Su@MqUX~^ zu>n8%-efv?V>|jdHg~QQeK|J553!KG;W(fF1}5h|t^PN>AFJvcj%_jf{|tR!VJY>R z_yYebEYJTHF5qv#Z?Tg9;r66I*Pr(oE9vAOGxcra=e@)ybaIf&`ljF-gn}RaVJcpX zd2dqKeaY>h&3n6%{OXwZ@l}1D zr=5QSXX`7AMX?PgR~}0bcJBY)kI)5J6;JEmL0^e=`N>@e^4noA^}@d2Kk|>^SZszZ zu_Hc?N8NKPF2eEnn!d(Zg?}AgkFJE>`P=bh{`d4TI{z->A^uC4d~PZIss1+fGQ5zV z_W@h!QtH3MdN^3U0&eD)q5IP7@iBfiKi}4L3w#d?;YR0vE_?l%@AJ%0;Fs_d$mdj^ z^O?6Z2jz1ob6uVTnU_ZRz3zY$FmugxI-kGEUoxj;ZqMgjp3j*xwmF}NT;YdY<1i=m9uVU*_jm={&E_)0a6k^I7(}a{4o8<++%7_Xp?i z!aV;nPfm0Gzn|~PRR^eNpH1%gHUF@CGrv4U7gHZXH>5L9R-lKeSHki5g8DT2Ma(?( zsCqkoS32|9EBtQ!%%9068mni{PtKElD0^>msK@kY9%$;`%=63Dvj?oB_hNFQ59#E1 z-_gm@l9MiX|18WLmc2LmYx0^q^d+y){QeIAYRq1rIW)QEO5euf|1G^GiAR0lv(yMjyn?vB?)_ z@w3kqcP?}CRdn`^{_W^B(6X`d#PGp}(Q;r%T{v`8qyVv&U^wFNu5f9ig-L zFQzxCucmLI4`W|^9?RIZ-d`@Gv+vHO`^R@Kx#3QJdt9OZ5nY--g*|btdL8;Gy&n(p&+#(>#=LjkM}Mp@IaI%VJ=~37=+FC?TlvYc zlb;>s-=_Z~ z{++mjpZ6`v7m|-XqVG@oCelyP$w3~VTd8-#ytgh-PtiAq9*LjuFU90bpQ`Yb=L(FL{!l?|gH5F8-VUPwdVA3+8=bC-ph}c(>AH0B{-0d{oNIiMN4*s9{Bk0dB9izhs_D4yL`}zxVoi3gxh?`xesK z5As~wrrr^glV(plM6d+E$s+3yzdv;Sp}uFB8;;oB<6eDxyZGJV5vA^$GSd$ehENzB}KE4|YBZJ7Ca zk9svsZjc-%xpOt=GT$$uw>mci@8g%Ezo)z4JGcsm=qrjX`I*Ps(u>tIKW5&_{GU7~ zIm$QsUvs`RHpJv)$@|9h4?34U;1l{8tc~6DXa8+Qk5bPZd^&b&3;CH}{xY&KYu?$X7{}B7|JK+MXsood6 zV{(D_>Fi%C=t0;HD_}89j^4!A`7QrP{0@_UXWwkD|05i)FS%|dehGf^)Ajsm_&ko* zSDCI#CkI(WpR4{Z&g1`$&i?%sy-EECoR9b8Ont|(7C*W1Zn}bca*n0@v-_l&{1>aWrr z>AaUou9khbj=o-4PyZBp08YdfzMs55twi^YtgEFHEmhuZI`mF5IW@ zDy++YfF6rC^JmjNFnL_^$GQ5JsV}GZ<1xHKeH`|~&(*8rCH&-0x6l=_Gro-FovVn+ zwWg>)iwD$q;&!}7-LFeQDf%<@IoKboU@IJmOK_v}nV-Nf;V1Bc-_ty|v)|?YROX_+`Z9k_rzc|O$$X9< z;Xi}Wup9j@2&zF3DXP&R@Tyf0vH_x-=QhVLgAG2@fd7tNP@__7B`JCMCTu;oL zx`EE~IQwhn<`urLC-pVO5&Z1AZ}PLJ6{KI^RJsP{dH11u=A=A_M)Q+jWzNjLl-wZC ztIYkG+Xgs)udkQD#aVBAxuRcfK!v_Uh*R#{A-R=K8nzmH6xE!TfOOq?t2{D@lVjn(LbWIzf`BU;&Yh&qltU6XJ#*7p|7+0O>}a* z8|ds)=hF}CPfj_N9*kY}WzYLFKYMKU`f2=c^ymG_J9P4mZuJ6~_2)|X|MrTjn!+)2b{Q7MECVuw0*8V%+t=<@yU~-ug&Rxm>BR+)#)obEo z*hqafCf~{XlvngUs(t}xzfV5s+bnohy$ohAxyL=9V$e(9YDRlHd*?`g_Py=&LwGG_ zPfPCD+C9k^^WLnaZ}f8KvJYLN{~;``J|45zj#PhypFON2-2Hy+o&9_P z|6weop4=mQWAd5``pW66OwYtm_<28Aoi2k7u%G^~a1LIEoiMpna^7yvUx_31CHHF0 z--5}Z3+o%hzm8svo%qR#4$vjBEB4j@2K_bt4E;8peRu&sd+vpFa>F8YY3FXGTVYN9 zA29j)YxM8+Rl=40{&e1ZT+Hu-hw-$&MOc>qANnXggRY7{@^8b6I39ncFS+Iv{xx_Z zF304cwdlOhZSI~T{2rLRwjP~)>m|B_bC=S2PcetTouBt~#pnms%hN6C7wEjtIYzg_ zRF$veKM&&7fG8|NnA6Z~gzIOcukD1BF8A@zf}47=c59E-{2XFFdXXRE(Q zFUQ)L+_yVDL;vsaEbNZS(}w9k&2K}uq5II~=x#U>7hz$2EimuLuTal>(y{8hG4Fpz z>D$U5f*bjL>AWA=NY};WaPQMM`dn||d~)u3>aX*wVnhBfArm{k_D+8G;&bTCb5FV;^WgycpuWsKGwJKqkI>0&7V_uv z^ErBe{}w-U_aXiV{LEd+eI8ZMJd%ANb4zFSefiR9&(dw;7xxl`uApZJ;EGUu(}CnxLV{^U@Zk1Ofhf$Q`&rGJC9`Pr9N z(3uBM=v&LrKD3u!t6qX`Nw>phI2(`YOP(-@{=NF;I195U?xR=Z1m`Zs>_I2hckz4D z*~6;vlTYnYPhNVQP99c4|BL+0*-QCb_<0XeiY|#4=*#|=T&S!15%tXd!|CL5%k-_m zF_?*67=Ug>aj`?29+?|H1Er z-(e3thV}G+PbZg{LSLo+8J@rn>doooFAezFf0I8X&;C?>93~GeMo)9!_gI?$0=){e zr?#P!kIbhp!H#&Pd$Najq-)?X9D~a-d-P6v5x(r+Yv~qe|L-5F(o1nYZqUD<9*zIx z&%oryr|F{l7Sr45AMg|YFih^ak3OI;xyC8FmU?oVdJtj{$i=Kk-J9iD8{Np3~Tue^zH@duY_v2}-r2aCUTw(|R zJ^oSb%kNK*r;B0oxL$N}-c57^ERR=X1@~Que*FL6zbi@(^tf}^;T6uc#(w;~_v}p< zQGcC&36rNkqCS&z0*wW6P=C*nqY0+T-$rr*HX*up*C=|XfvO#Z)2{pYgR zpZPw|`~-dpKY_!3Pha!Z~t&{&V`0`(@sq%Fo`C zJ+2|YDNeu-+?zeA9Y1qX_MPm%gVdkI>|rhFbKI9)>`i|5i{ycM9wb-F{ys;4@`B>- zTg1=abO)WiGtc|o`m@((F3R)#R`q-E2)^UKM=`m^ZFJ_cFZHj&vY5STkN(=2`82uR zO#OLYXP&9Ae~5baj?6{L)yC-Cj$QS4#OwJ3a0`DNo&6*8_!9nO`pVF+&_ijmpb`BO zqXu1x&OS7RpM5BMMNNGj)cauO)MNCwnE7H7oq2P)d$I>)uiV4WUYh(aIo}2DNnZB{ zI&`!cmT8Ko}j;R z??Ac8R&*{6dvGb$oKV$N_%*F5W-@;MOWgni*FUL>Lei#2-%${6A zU-FciboP+!8Rt1SLj4rIm(Ct}C7nH>Ib8v7z-7*7j(?iJ4tL{0%>J7_X$8NQd)i?! ze)i3k{K0rLPQ^ty!@0coxs2X`E%a5!EsR%>-!h~1+0kK=SS+Rfj8wI%zLKn=^ycv zgLk9f#j|`M^mO07{NK{aeGbr9sSm@V&L=N;i9b&Ne0&wxU~7Fh;&D8QZT02-)k%6Y zmen_w-im+aH=wicColMcf0n+pu{0+4`<1@6nD?tq>B2Y>XJPW09dr|X!2N46dDd(? z?^*st7r_UeAArBcE7XtEU(?O#ygw_&KZ;e=ucnjZ59g1@ypJtOC!eWL-|u`~?2B8l z0nW#I&b^NdusV*`SCM`O|IPmdljqe|FUhZh$)ECGV}-t#a6dN3NxkEPWw$MRTM{VQz9e}W#1Utk&aiF9|&`=ig)ui@uC(I);C_#KYH zf9l^!C->?^_fY>OWCCaYj%R)XKbN0Cowsg!9Qw@_CtgGjrQH`tn@K9GN+$xO+3dC-=#`kX$9t?N#o}=hnT>XPzlR zXCE%2e=7fDx-y+PEYH=>c0It=v3Ah}ucs?0l?KT9~DdEf^3bmf<)|Av|0 zUZyjT-K{_SboRu|?U@U^sRLK2*>C+>K5?o_%gUKXYmJg2nvot=s9$o4b7<$t4e}Z|5gx zYfiVtHTrtdZ7{h*a_}nrGML<^rt=f{$shXB$yc(^WS&0k+){j8e>=?kf~9mV%s#r; zx$N0H)Td#6%)XZ#a+!0=WnHBQg8;b?$kNUx_||?1JQ; z$=Ca;XK&1YpS}4p_2e1J%NDzEmilQdi4$=;W-mxia)_UN>U;Nn&d(l}J#q$rp}wZr zg+GHXO8D!;nF z=W_Zr_2i@@a!&nUbaKLm{JcMTgD!{Jn-@6u2V929_ZQK5FMGT5gZRIqx6#>~eftHc z`N@giq<_TooF7Q%JwO%yFn)5|+5DsY`gC%EvHYr-_cX)w-H-pB+x?z@y>lgTGXF35 zSN<*>z<-F&d*$os_tcY5Ji&j2pZ6tyrh8*@#@qDw#lq@e)5&d;FCbWJ+%O`oQd3(ciB>OYr03%B3_Ob$Co|3Q9o zqP+j9&c9M$JG_`*$^SqL@oQrpT!5G8zk>b-^WI{%dh)JTbn>oy^tZsFI0WnIFNepm z5thYu`isz|u>wv~Z%l8ahtmt`>GW876L!Z!>OJY?>Z9o7tGnr+%U*xx`#kd#_$B-V zGS_9#`poZh_V87HZ?jk4NjKEDhi-z|cjl}2;^*_TEuB267M*!&fOGlWu0dy>Dn(~r z%$zjNJtfqWJJqHus<)++FKnjo#mqg~cQdc|QP1b~$L`B>V+x&lGy7tmUwNKvaL-!I zb1a|tH@l}LW={K(KIz<2%;$gR-OLTiJ$CBLbLj*3wBcvpZp6?0oac7tw(Mh1I=5E; zE^N(TkC_9qPh6`nb9UyeTl6ne&-^i+&b*s__*{MmT#lL3`nfNAZRX4_`Z8Y~q%Xp^ zohw7n#J^+a|Lh&v6EasF(bvp9o$2H=>+*eJ_OU5+_T%KOnSTenCv$H0w66T*U@Pev z*bo0Zr#S5EWp62~Z!16hZ*rf^b%*p7(^nVA;vn^_=%#cVyqmugN8$wBsjm#3eKB*! zRQ^Nyo}=5-nTs>uOynoO$UacTx$H-E=RJr>q~HJ$k^dGDosZU6WGgI1iMxwkSsTH`15 z%b30IsCxF4?7ai{_4Pl2*;lidci?Bf{?NGv*aJ7?^ZJ+4c`uM0DD&(H^`rXR;TnGS z&PDvj{Jj6jUiG+o_L^RFa?JCbtHqy(7vgI5mh^-4VmkZIP&#|pTz$zckEs{NTh%+$ zUtx0bbJV9{_NDCkbM#kMUx#D(oiOh;M$i-Vji<9uenH=@-U;)b<{3JB%_sCLn1Apehkb*?L2j!sUJyr!o*6kOmB?J@gj_S{PDNiH#*-l0GHadUoho4glm zt}lCG_PQ?o($00nA(;KLslG=sxkpvHD2{aQQ!L4EKp&)&cTeTl;}4{>m$sslyDiYy z18b}2z0|Mh$#^+#z`PIW?tC}?F8T_bjB7AC+-Q0W{y+Ba{p;sCegFSe5u(VhwMc^! zA)*wPN=-@XolQ|TDSd28R*P(kifS05DJ7L{L&BhJN=-J|EYWVVt0uB3Z(+%MQ6z-E zKYVY;{4oDPD?i;G$9X=U*X1(T>v^8XqqQbES#p_o_{k-b8&vmoO5#GSt^aMhDEQ~X1((`d4&QiaKZcXQ&W*5IA)>O|u)Y)`$gXBXy^zT-0 zfHU|v`F^X=xqm4_msig{(38&Bz})-YqrU}yiuda4O;4sDq(8?+cmiLI)<1w=N1vo0 z#pC#k`kgp}KMjxKrRsmd|Mv`%-{X#k^1wV%=<3;OXltDsd>LW z<9zap4xTfCpM5g(LgxJS`tttFJd*h@^K@6w>8yV`4#&*L+1ryBW?snNv`>HXi@e9~ z^Ze|4rRlryGv}t#J?Xr^7SVa1E~7Ii+(2i4y3upy@;}Bgm^rAmzSsGA&zI#F<$sBN z_~q&3MVbFIM`VA^yp|m4J?Aoy9ieAn_TaI;PUeut^b$;-($cx?S&h_Z@{i&*{B1ZK zJ7DITGw9cx%bc~FP9D>oep`R$y5wKU53^ro?my4Dna*X8{D989xt`7(n*AfWS@xUE z0fY5te_cvvF8kHj%^a2eBJ)q?xyI0=(uWKN&$+*#^>#((f%z%6(VUa0S{^fbC7U7T)0AEx(W zS^QkRG@bjfhw0=^-_kYoSEB!d?_%Woeyk+Fl>V*s<+vDY;{r^6@(BHk^H2+`6Is) zzc76OlNWcVE8uc;tl$EFIL6g*{}Y{@WG!78YwAmGIFi4H-`u~?)%?f!d+6`*A^uMK9c+Vj)qla{>KD_c^dN^w5;7jTchWxEqjb9Yc$HwYQG5JRBKfmW+g~^w@I6t0W4_Dw+JfZJ?Y{B1* zCGfENJnY0DPgka=(U;Qa(qGX1=mO0BPx8%6^i}ciIiJ2x|02wNK{vXQzR~nHdImj@ z9)acgpW+h!aZHX=Q@tI(N5~&c7olIohFC~_G#=m&!{kYW=yLj!XFkhMF7<$V^4&7( zhxp0y3h^uQpQamNH-7G?y7QCwuTvk&zZQQld;O{J^VCn^kMI*X=J&EcW{=wC_cwcQ z_T_wzk5SLOQQG5^7@5{cLeXpN-=Jo7J*$XnSE%m(QH0|kYu!-j+_gTo_fSKQS=xfYRE>MepGT)E) zS?0(}>X~mcFJw>1zI4?2>G&LGUOw(QnQyaq7w4D5e>yjrCJP$TcVXu2n{+UY8^qlNlnNPA`-LC!_&c*C=-+In|e)gab=%VV$r?Rh|c_=wda?I>A z+1DrOPcC_}=f8%TYaXYQQ;%})2)`wrdGu;}hWf`?m7lq037y<^FuexX<9#^Lb4KC{ zeo6W>%s!qxJ9BVyxa{4T9~V3KD<+pnzIc#7(bp?aC*SaETTqRkeLeYFC+D&+4A+-j zq$i!dI5}GKjeDHmjSKW=f4PFc7vI6V^>xSV`Pq*$cW1B4J}^Sx8(7x)-|!IsBFsLV zJ?MS@9?V=km7d@^$&a#^WR84YUk!ab=}+i)F}XnI`3?Lb`Y*$$FnPrQeLMJ>ryrwB zs}IBEJhSK@^nHc*<4H_z`YL@lZg;*FPT(iE&3(#tep7v0@I0J^gYX&sr{gAm9jwMr z-Zz^*fytFx)7j&*UzFgl#^j*m=~2E;a+62-Q}~OpENo&9sqe$=M|aW#^<}@lfF6rQ@DfZ8 znmzbE&-sUX?ki@|HGJ^ggva!E!8w@xxwmsq<5f5X>*pN(GM#&~ZFF+5?B~gwYO7~o zecN+B;@84bSP$>O&H5YAy)ilLUb+G%FWTnZdw7%jBDyYS&%R6jJN^jzODxKthkwUg z)#uYaunNB+-Iq?Tn4EKrdMRv!r}_6Q;au(^E>~ZNo7Jb$Tj@3UAV2p357Esqxqa@9 zvd8aJ@8CJPx0y&^r`{B2@+;$7{$V=zyJP8>)TiUK*az$21N!g96`1_-puR%*jrwwW zBev!5qTiyI(aBdQ^FQa8$4%Hn{WQ8FU6I~`H{yrtTj_`BET*!MTpJOBae&@X~gU;t;=9|1H@}6$&b0DAdjujO5hsV^jU*>ZvpG(Q5 zGSB7nCv$Z6hJ5bkJ)QSc-iLd9z09e3pX75pds61n%$<3EXTQjNmAUh7&z6FrC~u^Jq)`JJmCPWFF5RcZ>QNc$@uT4xK#d?{spE%&&EP zoy1z!hZNRPSL-D&U`V8}WAL=`ee-G~DUq^pJpNHAEhN++BfgROn;rZ&xryKIKPp+X$ zVfL3J&i%qKPG5$}FOw_m(D%0b&Db8ZmmbkKo1gvfYI-_gl%!vX`$^_n3mu z{NWq^2Xqrmo;pOmHvdmJ5!8_^5YXVWLK27f7i80+HG z>OWyw`~qiTHT==JUDyjZsSl@OS&zc`_7N}H{o9O+{d(`lcQZvx5IOB ztn;~VSxZ+?-xlqEM-}J^_&7GyUzF~SE%_H=5B`YsWAd5*p_8X3M_uRK*XpfsH;z*; zi;eiXUpb4Pyl+r6*3_Te{WzWbu`B)kU(tV1eGP8FlUP_^Z@M~t2>W7z`gnW}d#YEb zN8?La8Xv$#_@;BO(E0Bga*uYH-$~zm%)Rmwx`Dp#xRE~?TVoX*fo1d`q_3qP!87?S z@M63K*Xvu0Lvexn<^COVf7gcpB$mV9%U*x#`#kj%_#^xTvOjk6`@9D;x8;45`7rx@ zK9BN#&wCDq-;ve^Y-A^ay%jan3r0ge|_cB+mbT0FB-fMZEW*?a1>tsHAnaBXjv~{r{bpWX|dCIg|9=OfSP3{A2WXWEZsXhs^(_)XQTz_05<$We$C( zzEzkxF7sIOfjiW9IhQ#%bILXR>H0F~BoBFypZzuaTk`DeRgLsb^Srz8Y<~8ojr`g8 zGhTt&!?Uks->G9hc~;*{dB2*Z9~!K1+_)grA(ZtZy*;(th?Hr{;fEYUyIIuk$aHK)Xz}Qyj+*xn4kSD_Xfk%vtO2_k7M%C zR{mYH4s0r{&a0T!2bumht3|9y<{Q3wZ1Yqi$5Gk;#XKjUt_Gn zzlW}aZ87&+*$*$^CqGy~SM;3M>2CN>{y{oeq(AgW`q35W-Kqntbjx$F6 zuj-TO`Se737p~z?q93Fm#fI2fy)L%H^*CPN4tfy%FM1TcnXZnN_;2HF{9>5<&!g(^ z^0U8QOD7*{N}q{CF!;CUoWSHUdo?V=3hK#mvUk43zYe$KQ#jT6?BU7ba^KKGy|Dhz zaU;J9R_0H}SMj8Jb$SX8;Qxw)uoYI+m;1l3=&SHKOrAWDei4(?Or_gRs34Oou2hdgV2tKJ^58Lxg`rJCmuZ@4hU+@9_ zh3MR)yiH%O{u#E!v6vjDx&HP1oAD<8YPugjkM*&F~4$vNLwAL}_+;uii*&RxRa zjXTx`o_-6zHIBwte4RJ(27bqo{}?vH+((_zmmIk_T@u&n zUyhsbB=*75`q$FQ>AUiK@{8gjY>Hj6n*K+yKIUGqxxSkGS@ey#2oJ0G!9PMKaO&@P z>L>7f`3WQk$sBUn?`?9H%%hn{GCyqAmp$+P{NDRLpGjxW$vl#IIr&oNnw`#7#$}j% zBzao)Er>Kt1`DW^1OvOT>mOMd;ETWQ-0>DF8sbYMLlzDFZv1fx9I6OmtUT) zNw>g}{Nz+?>1yhK!e=qL-(Y%&{=u02HTzN%eMi+>(%H-IqTj*nDXZuX&K<{I{N!a_ z`1A1*^?vja?1q{9voAizpXl6!bW6-VKyuE^?<3U5;Q-9O`UQOzF7*85m(TIP;O9P~ zD4l#Gdtqb!->GL0n@(q6{951h{LIgF=-fw4)z_GxJv4h%fA#EhyXowME9f7bpGz;t zFYy(ejLr1#!Q=zk&$1U^tp7B9FJN{4*K|vJ)ew4(`WVc;#Zo#sz*PMu_**ge1nt$6 zAFWry~WFLCZP&%cbWg6HW=?lPJ#to{+5 zdx+%j-TBw*YmPJUg!(f2D87wX;2?cX@N@no+<_bLbbpVvbS39!}b)$1{bR~FZ(_r?eCH}wWs92cv9fb;Mb+@fz9eSmI( zxxY!CI9A^*%)Li_{mI8S(g$&v{;pV)-;RC`H}I23CcjILwp?HCF_JHq=0AnW+j770 zt>?a^K9qh0tKpqkQD0a3Lp;LI{mltFd0A0?k7M$@r}cfqFNwL&e3CALt*{V|cfKE1 z#TA(U?qU(0oP4(P4f)ma_p;ZY`aVzn1pWv=fqd>|4$1tI-|xJqvu_OddwksQcjoyO zbmpLebmsZYZ^@xDm*suY!*k!k?2D`Dl9)L)du_m%f}a;)SI zng46(>w=kIGk5IcAHmF#c^~FIlKK2fec8jZ_hnzqoS(gas(ydHg6s)X`1w4~TvCDm zAm;P^Qx9m3$v^VG$();gr?0*nJSX#5a-htc*(=}Em$~83bO#*ldBy0Z^mfeLb)I_W zxy)h7q4M6%`zU!{=A^tgv&Vno`PrK@Z+!3TB&W;!KlAEc&SgH%o<4(r3s&)*lx`N#R0)03a=RNs$-^kq(;PbbgaOD7K+MQ1+F{Fgl?^Ijc&GqEjZ z9^FqTXZ!EHZZdvv-f+SLQ#A=kPD4 zH`3kcg>?3`iOFe_J2d0}!}$rg67Rv5cnEKDZVArE66(qMrqdm;kiLO*1ALiZ z2CHHA+~iTojjF1D?tJ#x()`?0B)^%!Z;WSQa=hdfO+0V5`f}XNA4zw_=2%L-Ki0!X z@iu+iFnQc(>dAF;pP9XTiTYtoPTQPbkGnm$Jl4aHa4){9e+&ME?bVZ~&Y@Rg?k%&A zH|5`U`v3ojZUqJW4y4Xp*1tzzwPq)KH_zMog8$IX0bM@p%$;H>|&wcH?bTHQ+a^F{% zeg%_j9;W|0UwnjL-}h4i&*c9FC*qrUn!eh&2H(e;`U+!jep`AD{WB)tdxq|cFXC3` zlD8*U`y0PHw$k5+PHxOR5`O-Bfd=$w%>Bn7ArmTfOKR&gb(vb9vtXnWNv( zpU>&9oC|JOzYDV`W-i&MF}cz#I-k3lukv}Fy{)3>b3pu|bM_d&m-9>M%=b^w`CQNE{7im(+>EaMzKz+l zv)?37$onpHW#)|H&Sn1?NN3L2#kC-O=nKcyzvFUEoOfB&AH6CHS%@wJ#360IG=ekd(>6x znS+yOZ0AqFdP>5>2q}Eo3ot1oBs+XhfluLng1+i zzRaGITr2Zc_N=S*XAjD}H{93Vk9#qD^MB{_T|6f_{1JLBo{v>AxnOeE%&EyG+xU71 z^-smOu%vqO(BuT$)sNx=EQWQQyBOQy3H4I+-B<Ys?oX>zZWy(u|T_PFfhk2yEY^Rq|%z|THEjjoSP^(SX5L+9S$C4JeK zs?lro9mM1=%jq)uR?y|~J^ng;2rH{s#k+8o`UQ9y>dmmH zzTC6)=8xc?=6}bz_nE7ndz35aqWT-r$pdmvV*KP**@M^fH|g6f9rAPx=NrxkUcE#N-fD)t}aXfIf%LJwS4SEL-}ATwhv0QsNq-4!&aZ-{`3=&DJcI6mt@X9Sm%rTBkqrU-pE2uDqu|pT~J`9dhnJ?;(A8ALny8d-GBCyx+24 z}d-;U=(J4%ATpcV4^?deS8~T(YfTx%jxWq$$K*|WDd%llsqJJRrbT|J$b*^_w{b{ z{Okiy^0O!0qdpneVq1OfFnLt=jO=+I=+9hVoX)(EJ$r%YWUtAbyi|V^_2gYU>D}tt zdoyoke$RYAQh!U#K49!A$i9<#r;z8iz^2Y6Kgzs*lK&yzjLA8&Klb7G#_VUApNjL7 zPd4>+vu{tLGZ!A9XF8X8e;|J}Kl}2d{Kotvn0%@-{eSw_;C{?JpSiOS{|QXqm)tA) z<7&@6jLFBU`oSl6%zef}&r80Od^WkyKJ}`&1SjCXe4Qcmbj)7%Go5`ld1UhH<>Tlv;Sls$=s6nOXlEw z4mEQw`#|=~7X0kxnL{#f=RKUfBlB({=ZgA!WZupkzeD{;^>gXuL?`HcK33Ov6!RX) zdoXiJ-b;C}RP~&3xD_)eW{rIc+nNLUNxxQ}p&dlvI)Qe;GgC2AZ9N_u2Fmvd9^+Wg< zT&XX)Rra*J-^-}4(|;eGJor+6a*52*WA$Y&olS4Y>`~c&uI4AdsqMK%unTt8mt5*k zbSL%9sU_(G_0@E8i_96B_bygHL%)B9g5(0r_~mi3{*L$+zddHIUZ9?Q{~h(@A=w|^ z)>l_O`_}^c2xgwozLkBtp7TBQCFi=9eoQ^{ZgPd0>Y2|oAAiBmoZf`aJfD4LH-8dV z#KXQ`W4bDR68rNz<6ZpZ0FThS)W^{^=nlnzO%6seuBwM_R+~>GH;LZoVT&1 zz7q66`ddt%K8wyAd%gbTBunVb%lFWkt3RXv<~hlaCefL@r_wWVKUQ(R9i2SreSY?p z?ALGbzs51nW&cZFIGq1W>i8}uC*DPG$J{48Nar4Dsjv4XKl@zv#^mhDmEP08AE!Il z+%M>G{$WghG>d*%|9kj2CV$$Z?*Kph(G>nvd<&EN{8|6g{JZJo80>sSdFshE2k{@^|3HtRlOKJ` zZ^3U#UxM540=yB|<5=gCqmAMx|LI0|)wd1%;tFh~uNeIko%_w{^m*!a=stKk-k|0UvyaU(iJBVfYxzE~7U#Fg2?nC~+`B&1(!@AM+ zaS?9D*3PZPrr21$C;fZb>rZ{3r+xx|gr7h@FTe2nnS3hqT=wwq{T}bbY1quU%+=Xz zGA}mNmpMJ};mon!)j!6(&;Lp1b29Jk(|x_{VVVDu*JOU*t1ok0Cpz=PYR}L7lzpKe zKXYdG;g|V~J?CCJ^LOUk%>O6#?Z>>2Yth;Bljr3<*u-;}U%w3r;$~(V6U-E#=$(eh8)we-k7hKNIoVk+D9JN(n=Df@q6ZqHQBUl$_C8tf_}QO((a8my zI-k8T^L+_^75y#nuXq$^Vh>CX`y8FzIrG9No}2kQ^ZC#ElVeTNm;bILdDdC{?2p+? zKjUZq%-(Xl=Vm`jUOby$72Dxt&sjq6q>s_b8#4Fx<-e`31_sGXdoZ%6zTw;xxIsO8 z_dERLYL)0$^kvSxmac;1Z~<;}t~7ptnd86Fmz?KEI=S!t`gdUV$K*6)`Ae~=^T`=c z(j(N9yB+1v=VxBtOixu`gj;c!dS`qO?@~XF&ir^jT>}SV=Kqp(F`Vl;edx^N$)D@0 z-={tX=ktrx*^B%0xA05SnX8jeO;x`XJK_NS!|(`Rh{yG{$E$ILdiKKS`7dDR{6lp1 zm+XuG%OCGKkKp_KC+XiXImSoy0DYO;hwy9TYV|wm-E`*q&)~VZ5i|cMZ|dlI zJ23Z1$z83l1t-*JVr$P^OmD*MmD#ty(wBRW?7`*umpV6@&YqooY#~4U{O9yvoa;(w z@7Yc#FG=p0JTLjxqB4UeD>Cs^uy}8e<;OIemH~9o|JrGgmVSz$-xKHx$kMD zZzVtb>j^qJMDm@A{Oo@Z(35aFR`%TNnaOjj^2cEI&x6h-Un#4;5R>z~MX%AH+@>qP z8b5hL?r}=14^yv1e@6d}zK&joZTTBSyB*{N>mXkEze2Yj}R{UB2TdzuK(-Y5s2dVjRT39*bZ%te`(^0_?H?|DA&^0}0~C-Y}L&&%gb=9$b(Q#>c{jqIJ-5BsTSu4qN) z{hxh1^VL#cFLP_=$IQu@PxJYi`6GEm_HiHU1$iH2Z~NHS&-|J9SLTEr>X|e0-p`y} z#rK!b%k0tlygWla`}jwmm-q1L>e-`?(Zw;J+m)Srg5L=d!v9fX;rF_jl&1%!4)bUFLbs=*&yc^T+cC;3qglJ@d#S{y*_P z^@lKf;kWA9Q}@uBgCEp?4`v?ufzF)1%ekhQJh8OC_^EZvxg1U-ybtqXAe)VHBCKvO>!4@LGq#GlUse=%)ey8Sq^MqBv;J5T}pj|`dgU$ zm%Hi3xCFD$?{$6#|9d+7^F)4f&g^Z;{gMMD@0;y>JLmS`%lvoftMMOr0FyJd(SIR6 zfd9Q073cSHem~uXev>|*{tKPGtS$YvdiLJrpl7OIf_<@!{_L~Y@?YRjq)(&g&_nP! z{!yHb2h^XY$71f2hO1ZR&&L`3>`PPW#(2NJFX-e@7x7=k^VC<+N9bNylm8r@eR(~7 zQ2jyrH!P34)q7$e{si2|??9iRlT&4{9mSunZv`e#X-_BDxG~Sg68JZL!|9^*H#mrY z4mRN*p?A{BJxB6~;bQe>bbWd-4&@i3AEE2g&(L$}7oy-0BFnwcjH}2P0 z#QQAwaaHNB^lj0X`?TB#cjfQJ+{<1|Cm%dSZ^GnJ3+P*LIyT0pn7ng|?`H(w==(fG zcg7Of(D@3uk)Qj6H~1}aAa2IwpHt`-&R4+!{F~^LbPu{V-5Cu2Te^WI9%k@;RCJME1g!`tv@y-ShK4%KLB}Kl5GY zrpz;w)bn|kJtTW{_L}U2c`rQX>t{cE!gJbV-m`gMC!fjumG?&WtCpU(43m=;^SqUK zgL?9v%sH9MlB3PkpSdG*hL8J#$-YqE^Rl1Ur88e87x=U1Wxs8%FY`s-xBuoRPiaDDUfAq8$?>x9WdFQaJ@dd7 z&S&44sh)W;^Y>@^vL9p)&VE_axe2&IKUt96_&)wgh0L-2=;Vdn>C8_TId=y?bM8nw z`E%y}2lUrbzY>$zFICSxIg6g9uM57-Prk5+&Rm%}`Z4|o*cfl}i}e+qJuLH5=BFCI zUgp@$UF)2C9y6bPtA8T@2)@S8eq4)R2g|Bg3VnacN&D(M7q7;-n0t&}&iBD`*baZi z9E%P0W#8GyuYnI?a+lNm zoKELw9iB*~@yOO(HrN1tg!FMrx^WDzR#!GO7zGvwJSQCFyPtKJ5Df>?LndGKpoU4ZW zoxckwVe-#+^nHi-VDjnY(Bt_P@e|Ma7bZ{HqCOg*#`E!J{d@5!|08+|uHo;dFU2AJ zhBylE!kzfF{@?IHertLG-3Pzqm#2?o5B?_lPC9vH_PCeSx2XRwJq)kM+y_+DR}7Pv zB|k~NJk7Zs`jUfWk37yl6CcLhLo{-JDE~=(8waS@rf;FU;X$0HUJF0Q?E&4)w89j<#hb#CS=-PDh%zOFCSvspv z=9kA4{AzSPY>waHMtwtZ1YU(p^^M2eD?UUo$K+pIol8zPSUq`qQ9rk9^_@^ZhPi)C zezaU)a<-QAXZjCeVb5JiH`7-I7h`GlU3Bir4)6>357IZ`NPc1Lf!E=um^?A}+cWr$ zJSX>(pYx$0_g616%KQ8Dqfg=>Ea7?Q(3j%b{AQT{9^o)O8k4*K5i)^Of5%flf#1td zAahvW%T4^g*7f_HdE^?p16IP!m6>lE^V?!_rp&j=F)}yr@%1`l=K0K_+3#z4UMtM! z^N-Hu^XV))`^=5{GZ#NaXI_|1XFte1oA+$q%l$ofIaYQq^KSO&ymyl~ysSU_Yv#HB z&QDU$T=fD!d-!zqt$>W+j*VVbqubJ;N zFJ*5TqkpFU%p=KBM)9*Z?9g9wNAb zX3@zFuB45<1(}bhdtUa)mHLv0j8V^in0Yz*+|}xfo%@a+N+%y!!cR`!PyHYKc61d? zj`NCo_P;miVt9l8;q);qgq;xzmg!%Q$xBwz4`BAM>{r=e_IuD1SQWQ9*A+|ik7IJL zE9mS!pXoo$Z-Z<26X*|c3;!scoGSa^EdEm5tiL*)d#FM5Nt~)Ld(L+L8Mp?MqYT%- zil4lqB7GV@irE8m-;q7Jj_0)3H;7Qzy?_u`8e>?XTKe@&o^xxG7KP3Jzf3O!lr}SDZrf&&8!(T^F!Q%Xln0zI95Poh4bC9wV%&9>TmKFU~-@@>D;$>(qEl_KfM)m4{}I-1pfg33G3j6`nJ){=!JMK zzao7$HsbHY2K;0A4Q{~7*g*db+>FVA%IQn~+nRnD4`On?dGS4`6kV21jWk^|&JD%8aiIDrOg>+M?u1A1D%_2I@oUfPOxrt z$(f#;xivXNafFb5_=B9UORu4q<42e|_epw{{_GKD`I#${6RhBWq5m&*=A{+<%%9o2 zJMib~KaAOTGQTGu&c1z@bMvt%&T@V#JqByyH1+I#ZRmeua|A^LXlcjFwK zpng7`IbEDCdCz7LO9^0;- zd~h+H`SwqqH=LjSICFM#n|syw>tBt>a2lS5{q-jwP2QCIhvXVN_2oXH7oGWcndfD{ z$^7}TzRZ_Z^<}TPna+MzME@p!O?oN46|+ZYZ-16Q8Cy7)oP0EY3jb>QZ*+3|iFEeP zUZvbVI-w*)iKPNK6PmGE_1@@v!C>$B%2FK@2C12eZLUwg%KGWX}6qaA-dc62^D zN%p_w@X3#JPqfXsZ}Dg6n_%vHcBub~xes|u-wggtI=N5wmFz!<)XUpb);pK|>xg>x z{jcc%&QoeTcMOYT4gJmO?1%ryABoxfhw95-d=u^4EBJ+8@7(wF)!44dL6=ey(o@UO!G*fm<;5PAXj#Lv|KL4S@9@V8(w z%)Qz`eaWj{Rv(Lh#`0K6e_Q%J?8JW$lUE<4yXl*X*Yj_qyV1+BtbfQe)F0ztNGI>u zOE<)xxDelWE_wfE{uoRy)L-9RR4!OUuVkF0tJCNDK9d*JSMQDo^{u7L(39xofe+F5 z;Y0eKp$pMh(Yxud@n4u+buGP4Kl~@B9>qAIZxG#yejAJNPtv*P>%;HE|2w^o{t|oO zXuKOs>92*!?|Q3O#eV9^?XTh&;Wx(QqD|G0;5_w5=sI-r(j|2Ak-_wp`Uheu?5v)9 z!gBNt>bZxkNEcS$j1#bfdRKgqpL@u=>6TboUnTsGzl$D;_4v2odHhT0&h*XL8lP6L zM!$*4r>dwwkIB1oulFMVVdu8dTj|pD96EVo8aHBN^>gS&^dBJ;IQ4ft^%MBL`~;G3eB$@>1HZ2|=qKnF*d7lLHsL2vsmaftGeUhOKl|N1{7L-GKNs( zH@)S2RetuUyr;|ZSLyo}$MKUdWPW{{-%;NzdKjHKy$W3cKf*cs|AZBAzj{Ngj#Je$ zFHNC);9Iy){~>&b{}N`uov%I(k7D-X%k)p+KSC#0BMZ*;hvZJly-w#3a6Y+j_L%Gq znXgvq&)ohLeKVfvIZNr+>C8!)i}vyh^koju9-KLMuzK>l!OmsxIHsO?{%`8bF!@yW zxxea9Ubm9Yo}T?FdCe@(*@T(j+tA~2INs%XncI?&WdBY6l)bWvb6+_3Ee=J7Z_ZZFK9(Hj8h-Z3 z?5o51*^@?lZaI7mi|V@;lk+4edR*UMu#3J2Fu6h_^{xD`=-279>9%xoh~z-I|M*(p zOdNshohyTjF?oFUf$jRU50uo`i@yxTV z;Mbo2DelEz)o;Vk`D5v7boQ|gbW0LEOy2g>J8~VI1qnPzns1b-{T+0 zFERV~0)55#KhVADZJ2yM_Yj5n$;<0HcQw{kFGnZe`<|bDdcS(^MP8+ouXmu6W3+UB zH`Z1kh1;;5`ginl`Uoz;omd#x>TgTe$M)D#eHcEJ=i?=~4wGL@r?2vys@Qwg)OcdeisIk%2Zo|XH*k?OnDo8ff^>Lu!%G5K#dda?f8ANA)a7aOJC&huu_ zSKEi&p(fzK_?ep%ioNJ)RUio$luAo4hLZotb`TyUqQFV z+*9R#=Ysh7G$s$*>NyAa$3r~ zOb(HKw}$>F)su^4&fmdLj*$H?b64iGyl)13-ek=Fe2o6jeBnKl_j%@%eZJm&%$$GR z`6ihC@Tk7Le;2Ahhnb(U$8P6mzB}xE=D*c+_R@*;eCM)1-cDzJ+(Gxo>=PB}4w$+6 z1Nsrqe}NuGpM{w-vv>62H`m`D^FA-Fp1mvcPCI=i@Cj^zjumA88o)TJp8a_^ow;`+ zo!o6RUBvm!>&Zhm@=IcU{n=x-(%Ey9vmMg^sQN`XoS!);dqqe7Y<&ypb#(TfKl7{d zYt!e_Gw8q4*=v{3M{pcwzsWw>#`(;**@s@?FT;k;&!jidZ{VH$ukd4Bu6{kv!k;lY z(p36&=d!;(A0M+fXCE8L-{9Ptm^~%)Lgt*h`jaPJtUr6$emeP8_K?5uGw=5Goa`Od z)C>5@`=;@;pO;Zj?wfr(`)cx;cl5pHeD>tB^keEP>EtoV!M^4H6Zh-yLHEPVhpp)* z*dOa*_LH4-cg$RzdG!Q8b7c0X?0e08AJgz2%>6?Z=acWxQP2LC{r4b0c}Y>9*OU0u zJ?9!aIZN`<%-`p$zl!hU-8jy3vd^~W58{76q<956Yi(&SsJLu-P((~q_ zf3AWS>b3aUE3@|{H)y9X`&{^q7R>FGtmcHag1L^zKzrv?5c|`69UgA%7 zZVC+r|KjJqtcpTKJWstUKEh9)vW#A@{wp5lPoa}bz0H3Q7vo)+J^n^I`}(fQ(6F7AUfyzU<>;=zn8bEU$kTcHt-AI>ygE)l~X4eRHuS?!gE2 zHK0q;f1-QQt^7Qa|19Qr!rUtirfP(3-(-{_6{_u&@)H`pjtuS=g! zFQr$}o$*2by?8CZ9Gx6)A^kWemr5R3lRq1i4?f^|7xD+uxo_FVzkq)Q?!x47Gw5e= z5tecO0DXWSkLU1nPxK;vwt6ePj{iMfj_!lW1X&zcO8sPOjRM{}I0$-Hra7&b>z+x|w=mx-mVH-b5$Q9L-PO_eaPCPW>HE{RDn5 zKY?R@KPO}MgzTZO@H01OPtKg4TqgI{**~&>C%?!(c#Y>x!t6825i&<+PtKl@edROf zvafXUd6W6=a`pCDADik+zBH1)0T*EAJw^L5_4 z$rCac)b+gV=gGC!@qhH3%vDF|%&C3#WlmbAo;~#i_3UZe=;oOHGW%)vqs^X^xoan# z+$H(USNtNl!*es2Wo}Jwnfc*4eaYi~qMyLb3E5+B=VuS|7F>(xf*mIOpca4C;L}&xHI$>bAC2158xKuiXUL|_rKDg;sDQE zKqu!c$Nvj9QqP{;oIix0J$MJd9Y6cTGxSmQ0(uIaJfs5uO#VRZ!vB(9Lf58~>yGBP z!JDxZexZL6om~18{ulht^Z=~=Wfr}!`;|kU-tQp zba&jR?;*U3zbkdjew=;mi2fh&Nqui)a^K^05q-%iN72RAv&Ze^Pv-B&Z*dwXr&>X; z!@W2N4|!g4g16}3)VtFs=&HDkKLr=@=i?RpKG+!F!{eC!x`J~Xu#@^`OdkI=y%Dp| zKj7Rj{`puGi}>7Vpl=S2z}%-!pmPs9$N5|N$qyUR$-%Op-^PE@xl6Gy-mQKNm-BPa zlKd;VP)&XF_4UR|xCTGRH}$_sCr8*zkHFkd6w*J2f0%Ac7s9Kswffz3Kdg`gTz=`^%(JL@HeR8Xt^d-;kp>HH!ho}4BcWM2l=V-C*5>{HnX zZg4K|fs*>#@H00(#_x**)H8Qxj+(;XrZ1n*z364?*|)~fn=o^5-m|Nm%UoPRUvj(N zbR9g3c@Jk!&ilNVdUB@B_nGH!_w|0&_W+%JeivN>^S;S@F!N{T!sJ?Y^q2D7gY?bx zbC`UuJ)QS-_P-(g*3KtSol7TA8l~^gn7O+KJqPPKcapx3ElnFm+W#nfNL%r(iMGIwP^%Dg^7fA*mIbT!N#mprHzKl|r&Uwed1 zPfzDM;a9j5Z^KrYJpXIYdyGE=OYpPbCl}hzUxJrmU*}fSSK~6=qP~YF3l{ssw^$x8 z#`7@q?(5Ef!q0w`x&0dT>_5rHv)5!_o~Z9DOb(U3HT&I1zRqy0jZ>W)i^+pCZ)RUf z{&%jv&dzVgcd;@)uP=L0Px@^vu5SUZ!gcEZq%#Nqi>|G{k4}#E8lAl5Q+?xcrh4+< zGW1=TxqY<$#K4%e99#PfF3wdiSh1%C`y;b#vi;D3gtFndvQr|dh) z(OT-u9#+wFKj7a=SE2ub?eIPIDfD-^mY;p;75-mwl6vxiZVe*5| z^>5=>r59sz;68Nr&*UbD_{n4L^qhOKrg{l_BM!qev4Xw>_%lvW&pl>UdN=-$z8~p6 z_%L3l-VAH2dlj zEU#}U-4+kxY+Rsk9{mL-k89!QQ;C0({tK`hChxkOuB$&eRattFdT)FdTVZm)N_2Hx z?fg-?C~m>zDrM-u=}#{6GQAw1#N@>Pou78~oR0d6V(!7`sISH3mh<)H{v!8cgZLA1 zDJG}7)^lFMZt7#P5nh6u^yNOcE`1}e(^nWr<5oOPUspQ!8inZO(g*0=_cn0u5dTB$ z$4?&Ike-H1^tHg`Q_1ZL=}T_gL*I-1KSCyO>hE~!C-8gu31r?L?)UU{{K4;Q_OZ-$ z!}(w7%jet_I-md9W3ta@&&+!y`%vb~%%yoRzw3F)HTKYXZ>{uw^uqn>$sMxKE#qe| zNbZq&IiKg512aEmE@t!CULa)}B_xeIQdrNYa>>ts zm^>tTR`${C%ZK&1!0EpJ2zm_t1U-%(f{)>T^~{ak=~wap@UK_`k6}B!0uNvpEP~0S z9;1_AWuM>d`^o;694FlS=J$C_SA5Nb4vwr1*p^$o*q=6e6XYvPP=H~1fJM|?G-9%^qSV0ePz9){s-MAjtV)ob{ zozK3My|^F0kN%2u9XfmSN`7O0P3+I_fEBQodUBWK3#D>iUuin~bFrMqht#{`8f>LL zgFkr)=dzy#S01-u?iZ4`*5+r=9O=1hF#FySdWZhn^mzJo9L3N6 z_6olfKYM8M)#N63>#Jk`zf}J%{7dm|{=Ik_{{}kw#Cvq|k=z$N%zqM3IG^0?m-v|c zv7x@~o7?Dr=j;2vZ>G=3t32m8w!?AigE09>b@j?P5%=nQ8$00y^?mfM^qKex_QB-T zWA!ILm_WmS{(qmxJGTRG)Uc4AirLE_poiiYm>mCm`XOxYdAY}%!}sGXs7oj3eMf)p zRkO!0(w7|RWjZ;=2D%&;#oWUcr?-0k61p$;;3t==N7qyTg8mdI@sppg;19(b*i2tj z{1ubGeoeQ=j?PuZqnO+z`SIQQlUsgE7t{Y7Zoos>RbMGQi3RG9&`;A1a28&ozJNZD z9*bA;PhuhdK&-+~j@FD`j1S@&m>g@X^Beg+=!4iE8{hyetp6PB&2Nawi`LP-^_8Z# zVDhEG^da1-e;!?w9)gqkFVe}OpQoo`cYVnn`|)RBa;$3lelL6dsqgdDPvDR66UcwB zl-%Vpzqgr7_xZh@&o6=5*E7GJ(3kyWyT0U8`5e#ta=-q}1&8U(f0?^8A7sDCe!tH1 zt2mcAEAQXD$C~O}jUDx8{zz`~mU=!v^Zq)>&s>|mpglkHe)jk?JU{cu3;OcD%D!}^ zzI)UsV?KwoKW1;sJYL(m%q7Ve^WJ#JbMn6EqCYv)?dmu2GZ(%~XaD_A4xoRg`d7FY z_uzVLu74^fZ(2(4*7qA`Zp<9hLSOdLEu|+M_2Q+k7S-J={eaeTIypLWFEeTu?jQyP1m2jtSz0~b3T199>s;8 zlbmlP|3Us^^nQ8;ogC|*boPTW`jTTjO5cZ>=aUOv&2Ndx^O7SZ|7@n7ef=(9_gj2l zJ@fx^{!{!eSf8JL|7^Oy`V2aI^4Ihc_2KkC=%I99dI?^J$xFJ@$z29IcYzQ(^0Yx8qIl|5u7 z|3`ht@JXztekWa@J{OZCHC6u`R>$k~W&cTTGfh4FVHx@kY~|cue3^e1{(`rtufXI! z_2}dh$>TrZx4}l9lRfMqe(p2s(4F<&hE4cu@frLXb1&3Se=E#h)==MKetAsZ-Ag^W zUpw_unB1&Cy&1C?=KkpbKe+FpJVQma}U&wKMgn`Sx)SsMi48H;YLwY8*>g{!5%nNUrk)gzkn`;<@j519VXXmKv&kkgf30*qW?ek?e**D zI(`56RT0HRcCDopN`w+6u}svIq`uiSlx=B5VOb;^lr}>|Q%Wk^iG)G6$s`I(G^G%k z$fk_2wR6W=K8$P^LUxuL*#z&G5#ohx$nJ}?ut+2 zEd9TSOyKn2@$^sNxAGH6F0;e$Y4WVRSF;~x&pyZR@%{R<7iW%Xq@Fpi51sjNgL6yx zM=|^PZu$j%$rZ}v`%}+c-=FTN{tKO)AbVeOv#t6vA8eveIaeODSKp>y4KttCqgP}0 z)(y_L#VP82>5iCv<*@oP%zl*pA-Td`>eo46oX#GUxis@c=HG+*viHw({sjL5dM%yY z=T?69z-{W8i`vqen~KtzACtS?;W>lUn_~&gJeAzB<w3ZaQ;+_P6W{UG-JUIjqFboboJxEWak5c{Dk7_KNJ+ zE%jwT%D$X@B6(JC&&_-@-?@MBufietNA*{+3jb>y!_Pi_51oBE`%dP!cFtx0dQg9I z&YkqTn7J}@)h_41#ar}sptG-JKg!;fc{aJ&Uj1)6UyeSK=inkt&c2DBfnVWZ&zVH8 zr%Ta&>FjHD=vvqZyJ2#JRrHshGl{N@U+`CB=Hsi?lQSmQNnSHX|MU8i3*>$`xyf{W z%Q3lnGv{u?`s$g>b5D@GF?&KW{mC7(H?83J!m-hw`xe~}-{H^2$T^(#0@2arVLNrx)x0o4!Bbqx_$+GcHyyfyMDN z_2gGi@IT?N!^`>Oa1sAI+{4ek+ZOsbUZXF$>V3OS$7bq(rE@PZkxuTIe=nCDw2JzTI1AT!-ba`mq^J6o{NxWa`Q`8m9D%F! zC%;Xuw4MKmzH{l)^sjXCj+6Z4I@Q&a19hXP>RW(4a0t%SR|u=}zrnirp8DO`0{5w3 zix2Z_VDi`{>Z9-wCWm=Qe+7P5oXkIi{yTjMogB6y{ZD*MU-IJr;U^!gqh5sn6n5b^ zz&kNH*@tv;($aM9$&(8uS30DgoHO}sMPKg$_2hBu=<8F*%WyUh#d%l=U&h?u752R3 zS;_TA@qa6O{ps)X^iSaT@Ds>fmOV9}i}p zYkh9Kj@hSf)?dKiO=oXyL+3q|_ed{(_Ol(%=W{>rm-YPYTY2AQ58CNDnalpDzcv3H zOx~9HqJ+M@pECzNz<&a>H|29Q@5g_8ZsxSON@90Z@lK0jCeVN~~-{t-Lwt6?rdp-MK_U!HI<9wY7 z_yNBpj>jqL$t^M$ChuIPo}BqxI`8Am@drI`KW0wJT$_347WEFElRddToq1s&y&5xT z9HpmX_NkjZH~V{E_3V4uZ|m_l>Q5e&y}Jp&M(ViVxy=7(@^@n9;pBKN`R%c@^V#Q; zcVv!A4z*1GYD`W&!1=@c|Gl^6SI5jvKhmXehI7~8$NXaSmslOYR?oh5grB*qwt91% zj@GAwvUKLx?6VhpP9OC~bmqr(^g;E^r)&7xd-|zo?rub9zqo>)5g%9KX3twfXCA$p z-wJP2-%2MBYe(OJCvhZZ-#h928#oiE=zAAmr*++}>4`3B+;(Yd*ZFKh6rZZY zIo%Y4mj003HTlvQ&)J08C$FRz;acb4q?4~F2f9?f9VQp9q`xS?IQGEgDgE?y!QSev z>3x`edosOL-$V2U`bPRzdKmo}-4*BJ5$uom>aR*y!6BHOp|ZZ@rSH?pnLeYFyYF>= z0Y7Yw6=_z{-CWBR|tlK!5xu!X)k>UHVliOGNJtB=MLn4F{(-4+{S?yWX^?(_U-u`s^~ z*20n45ohbakG_Yli$CEgd>NZza-HqY{|g_*+oO8bLnANhTj{<@`uwa=}uUW z|2+K#=6H<0c~|4Qe6^bF6hs(u-s%O63{cCHhh`=y%v za+tjH3Vm03PHjA_ue^G4v5ovw{N!)HhfLt~-|_TM;J5M<80+^lpZ9scW-rKlGIMqI z)-`^=FL6HmT;8L}QL?9IA5X54Tp)8<=7r=!ojpH$WZsAQJk5K%n7-^m)##giU-{ha z#lMSx6lkAbJAH#FyGu1bEEJx{ryzdvMd$=sUv?Q!+Yjg@@A$rrNsBp=C~mAOB8 zVh!ic#N;j6HQ(5zm>gxkdiLwi^nWq?O6K3> z0Qaje^}O68WWRbreHk9b8!)+2_L&*{37EV&ds6nqyfB3Wh+?TYW_h9y)tG!UO&phHeH|r~lyK$}hS9EP0#lHrV^Yx&U zFK53`9@y0R&-C3#H^L5h0q(><>VFO|!sNdr>84oUxuW#bnERl9>fNv;-hsIf+UMM1 z{!V%ze#D=NL-~8@##jh*kC7auF8_#gtLfZFUO~51Pad>}uBASXUPo7sh)f$ z`+4^M)%xzm@AW@{<@m`b*3qr7nZAv9Bma4NJ-*COu9SPz;_AsM-c27?P~S}_zZpa) zpM8LCjI;1Pd=1-sUJ<$heTqJe!?BNgCwecg<(H-(!pHbk={|Tn{;J-UzK%XYx2CsZ za=PktO?`i(lP6cA=c&JhqxctM^6G=?Q}|VV?hT`ttIx_gJdb~p-bOE@m(s21f70J! zcg#I{{=Gqe{ygWF<8Ztco9o+x$&Y?kufSh|3;D^P{)c}tKA?UCTkx;NE?6IPAMyj8 zd%xexUVr-gJpB{+J^Tdn`8mVyb>`4~PGoP)yqeF&yiaEM{r&!*ji!*2b?;hcE zbdvL*`uchQBwxzsbMmf5`c^qNm;MuGj?CwN=9bKN6Pz2NKY32}yUc6ZBRN5FL{4wZpdD6gY%hd8ato;EOXy-e&)Nr&SmbcsGhku^I+aX zng22mOxHgfZ}Gh2n0ddl`u+TTzGuIjrk=SW`)}rrvz*JEnB3tQKl{QN^nTCFoWFyg zIqWhz@AvG{nR_xKd&!2;Ld0zJK?3LLA)~olx%&QZe--5|6O6bd6nmPOr z`sU*vec4a8)15K<(_;N+@^{fEFmplXjD`9};WPMw{;hORe1+c$FTis#^J-`M7aWJ< z@p5eDx&My#d!K!3DnIjZ@|6djyG=cF*;sxHOpe!0U*?m{pKJKZJGVHuf}gxDc`3W# zS$(^38D5RA<4_06)1xtSeCE02T5Z*nUnJM~D?j^0HM)uCzerc5Gv^Pa=cw<)amuH`7r`||CxpwxPH~0%M_XvZX%icAS zE~{@QT?})tQJl_wLH3~JYsnpld(Hs;$)kLH6eRbUr#>H#JD2=oHvb`h^25<|@_^(D z$(zQi-yQnGmw4XWSYEvhy#Wv7$LiVda<5Q{|0CY&`D2``fM2Nhqu0{OCEue5VRe0D z=t6WEx(1zn@F>3?KY4EM#SW-{u09a2<6nw9`J3?X{OjpQupu^APd@)x`u!X_>Z`y{ zE}UHCTJ@Id&(X=H>hTBQQ1wIf`SfBO#?Sp$_UxzBv%lU%pRK<+oqQ|#W$qzH>8p+n za0w1`z6Sk1{XN|Rd*YoqUEfsf&#y>7N*B=E=tgu|yp`XLZbo;ZufVR@2RrI3?7vrX zjNa<)F*!n4{mIqNQEz~)utR*$T|y_9c%E*83$U2yFQ+fWZ}|u4+~>8Se^pP8c@e(^ zKlgKA)9ch9!rT*Arr*PHc+9!s^at3R--&*U&V5EvTDjmne>jMR@gAI_|3^A`P91)7 z`sCK7^i@$Gj59I0=U?>I<-bXPPVd2s`T2JMx6>!pPtm35E?5K)<3?Pge-@p4ltOgw zYrdkBlWnJmIR7O5C0!L);7QDV;?vQ16hHKw-$N#F`tNx9C-7VO2{iMb$ljHFC3|B2 z{Zi(u$NfGp!n}7U(L?>7XaC5)m;E67#|`?EAN8g4-q__i*`r@n&wi0PGr7(L^?lAI zkNBF-zLov)ef^pD8`GIrvoE~O@8>z^V&>oE0Uz@xV)ljXOPM=XssGh;GtXsC%6yym zYTh5M_2)f(8T}bH!_1AV=udsU|#^i(t z>Eu(L^(R+M-tjkmnJ3HZ%iPpnJ@4;n^!57IWA>D$>M!$G(lv1ke?DEC9+Q5|K9~Le z4*he~XW$n8BCL*?)9%vu3;$b0F>`Pgx+9L#pPV8&PA&C6sMo?K*dH6=CjF~1|1Ryn^iBAf{v9|8 zi>qh<$UgWk|2v$m{}s$0y;gl0W)Dp+x={Zp^>UbfehHoZG5crogBi{b!f&xNKH_}# z=I(UvEs|^8%RdXVPe187pYYG7lb=kb+hARNeds4}H5?=QBu5*^ID##G&TOM!z(;WdcEHu1UlpI@Z^Y!9CFtAqt;5_a z&Qs4`+=woO*ZRKhr?ZED;W-!3mwO7Iv&F07M17)&Luy-f`6Dlp5C8+tfc-qeFc3Hojj!w|4jZ5cnY^;7uy3dx9T$?<#AUDcn)3;Ap4x9Gl@ylf2J0oUV8&TXfM)8*-t*a|OJ{}LbISIs%U zC$soV@Su9~&WHKy_#e{$qzmJ8tgqe;lhc)?>tb@s+nsC1KaW0~=ixl{2KZaq>ra25 zr+)&!ho3<9?&5x5_h2*3epuP>?`Hlxbmo-o_1W8xsIS5im|P)wK<2;f=gC`k`8s)T zXV1(YmG@HSo?)K99BX26yutI5tF@!^-uSn^HvE6kHR$AI*(b7BuTo@HwA|(+>Imo* zWX|cUo;)&pTK4Mfe+~3a^_-A-B%bb_FyuH4em^|TM z^exWU#LREq>EzLe^k=X7MEy3Lqn^Amdtc`FW$MXElB2YBKJ(jTI`d`r+Bf(+akJ;W zf|;wbFYVQr{3i2v=H%QnW^SG6Ihn6Mrbm0uS{%)KbL=k&K^>MpLw(zU0z>VypCUlK8o2V+tW)hdqno`H=NIY zpZq&>cEs#X_?7T_ z^}Y1X$S!E&4|DNtoQ9X-Rn8r!-%lUEHYRU4m(E^!4xPMYwe!zn_Q1dETf+6K}^l`jU&R zq?1q8*H?gds%KySoF1j#n65%6pWMoC$Ztol!Q_zZ>Fn3P(4Fuq=byvm>B&iQKk}IV zL74shpmW)K%d02U>g&KSN?%MT?|79y zj6L<8q6_2M{N#I6`N_Zjq`n0&QE!PS_z%*#A9#Tthh;FiP8;W{=DB_l+vqBIj&n0{ z2med@NxB$)1#ZO$)IX=Y(|fQ#&cf3As^fe7@t9nH0G<1&Npw-?O5tw&LOu5i7ts6F zE7I$*6?RlV1LyO*(k#NMBX; zCG=7J4}UoQBQC%V>d)f=Jc4gwH~qQCUP?E>W;hsoJGTxS;SemVFZWV!(NC+_#pIe@ z)sqVzru*XWArm@Wb6@8F%w5@Q@;=Qzkk8l5#rZtC z+2_C*pBoM7?16btWe)A9FY|mpU-NmL&zH;#zj)p=&gFgkH9vDl5jyih=7@a0X79>8 zo;@X>!=s(g9+LM~-p|=vG8g6Z^)BDnddz&8`7!Ukw(1RVEoOfk>g(q{m3!pL`tmuP z{cf}V?ENL^8}!$~_WWZwjDIyw;}4)Sr(}-I+<2Y77Wx{~c@MnJZ->b%E~GP`XAhpn z@8dZO>CEq$4>QM=R?pm;d8i9N@2@?cmwBxVoqam{N#@k-X_*u99=^wO2RT0wlNTi4 zx?W%Amnr(*<>$Sb{VDrQ=9?JzS`+|$?iR0cjHQZ|HSUN1rO@$OJ7K@qHE)a{4cRM|20g0R+!$5Yp}a>t1l@Fnj^+51 z=oxqu|33OrIyu>3ep&uVIyqGCW0$Mf!`m@=)|qr;JcNJ5<~Z5&lk+@ACpYMzZzC?o zar$ngPtwU@AE2wN=RUCqy+r*ox+eV--4H+L=YD4n|0{k|Y>BVpE?kC>I5&rGfaUP8 zdL6nZy@8&9xmV1+#t416&+JO?!&{yE8}{Ol!eQ}o9)CN|$1Bxm(?jU;xQqWO-2`)= z)k*y}erfy}XQ>aT-=oX<{5b3a&`n`OJreI9>g}_!8#+ zxd{C~nES}}&KLFfJVIB}SD((k;5Yo-%jI4+_g%S{yHa0C9OXH`mA(G-_j&p!@O$_P z`1LHve7Su&spbmra6Pv7&8>dQPbhM$~an0htL9`!n%c_ed8 z=F{wdd-dh>>PgRUjtABAKFa)1gP%Dgdqn2B3eIOQKS1}y%<)w{CpklMi@fhX*8iry zr!b$(c|V`zH`0HAE`{0uk|$>0%AQxux$I+4(3LQAOWxmw_}P~_(yw6Nr#*fB%uSi= z&f#Zo&A!);pZzIw{{Q1I@cet|u5_<_U4Aj#k44q97qsV>;b*SNK5)Bw_SUuZ|L#-H zCBM0j&igy_?9Kelb7MWHA7;8fc%bwPQ&itLdZJYD+agM(1JK4`OKW9!lSAPTjrRZgJ_Qm7_b<}TA z&mKOMKa}4TAIEdlYthf(U6{OQm%i+O*^4vRX8z4yo}BUp&l!)?G5bL^&)v!2j@c7F zr}ydm8k?c9t6-_V()cA_jM=x&axV7{jp@vZnY)r_FILYSpLs3$##GPG+&oc#^5b^& zaeW`qNAZ3BSo&Ysfj@*kLXX6jnB2cJo%@eL&Lw}!e%hU%J#;#~4o~1Tyx()PCp|=8 zpk9H_-rt@-nV)=Q2fal7dipIodrW11_VVTGL-^TKvxg4g->t6`{UiM&eG?YN4cHK~ zuVg>TebKL;^Qyl5`HZ+rB0=O<^#KDmwGMql>B?01XQlV7&gm;B}q`fs=y z`#N_v{Q><2o&B&3Kl|Qf^$Yn$@IC%b{yR+MkHIT2IZtc-mTE)3KxL^Hg`Vf61{Q$1Vdg{q>E7IlFtJB}nclz%&o8FC0 z^)*v(gQwKr#nHGKldl|cKDpZ=^-lb{bPc*IU5oxJeL0=H`of&YRd|j5zte>=_qQGB z+L)X&`Q#bSCx5?G-?RKCbaKK)c@AEIEwKS6$4ajDn&*vD--@5{lW&dWkKiX)OkP!5 zy|?;Rm>jmM|6Y6b{T?!b(|^a)KY`!MPayAseBM3d_qUDT*ZlrAr1M_P=XDGHtub@o zD0(Pno=v`zy*i)2**{;@pZ(xfU-x^==UevD2Kq0;ye~5kWe(5$n>jw8-}zk5dvv<5 zlX)$9$o-z5y&#|KnPal==DnBC*~~vBoNtFWVD{+abmo}srF(U zeK|;9_N1P^PILZRx*z5}Je$^5kohWmeDaju`m%Q<*WAd@JefHv`9tQ#yY&4L*ZG3k zbKCP*@c)3xH#3*5;b#vz;#~H>zbn z|5J4Kk_!Bxn7OJmJyricI{W&1I(cz1I=NWpqU1njJty-$OT|GuNKQ zpT^IAw4Ohn{{)>mwK0DcKYQa4I`j89eN!;|&bjna{WoHNybZ6xBe+%I{EAh{zv@Un7kU+_*S&*DK`|4-v*EzSH-i-%vrut3vbWE=D4V_%&9C|V4 zKJJL;w8B4QQS6~Vxpxu%&A1w$*Y_5kT--z^ zE6i! zK<4$#0f+ov@Amte{qi09IrZdMFVfjR-qW`qGv`dx*M@%wKEuBYliMU$$h^{CfA+2m z=^dCkZw0*-*WfSJhw1D&d>axxi53tBK3PPdBO-fxz{+)Ykyd5FQk)Ky~58t{knQD zeowp^E8|YgzTbroGJg*ByzGJ5i!$G4?@PW=On-T7?txS2qBsY$H?5_MW9G+a=_@_w z7u>+FPfx()OP8s)$K>5-(#b)Se?7#{-doG__Vd5PG<~|^~+`H=8TfX<) z^4J7tV)pjrJ;@#WdQL-q$%RVLC)KOdWAO^iK9D@RF@F;#XUP7MeX6kMcGuU3UW(aA z=c)I@2h>Mm_Oj&lP5IyIFG63BO>nvTar$h!IM%~`>J2b^_ZB+)$Nls%Z0Gz_boRJH z^b#D0#q>|XS^R$375|CJk#3{Ua_%^e!KRq~x-9)E{sWW8E}++8bGx`1L z-E{8vl6P<7Z^ZZUdFLw7$)&f`$?vk?j^ZcJDdl`}wlC;9`o?2&*pJkc&*z?Dw!WM2 zRs0NdpOXFhOz*SB>SeKm=O>RWL+4)VX?@%HUt{*-=hffhuf=Noe_Y41`f5t~WCplS7x*}%(E~&pVzZ~t$6%?V{ zVncno-{?=b!2&!7o8h0FZ%My|OZd0bt?1-%Bk7u02s`4%&UK?p(kM9n_O=4CYtG7Wjg`N9d(^H-3dj_4URT_!m5^uM1rRpTPY8T1M&n zhQ9_&`w#jPy+_{_^!4;SdKcY;hJyP3keuk0z7KId?$x)EPJYvzpWN(9`Z0Z1`#w+c zSMux8L+IM{=ky6WxzR5AJe-c5@LGHXU-g_y^ygTIUjZM*nG_Jkq)op=|X#4mmQEjSL7r~H9#t3UJMh4h11TVFMNk$;TtOrMF#CC;Fm z>dPK8oS(d?3Z4A#a{XiZ*_X11ovWVxt}~tdCHHLE)61*peUf=G`$u_S=WRTI=i^7t zC-+%H--jP#^2IXFWxwm8o;_$Xoq6&({n_s`FC}kzQ$4xDWpr}D_MSHuv)^{q_Z~m_ zOLcms`XM^=TXKxd`}Nc_H&1ph^LX~0M*PgF**h}7W#1m}dD#;V&^I{$EM_mxzFkUR za)>edvM*+j&3;l$fA+$PboRn4>Er|T>1CdqdDA#jkh$p&eL?opcNm$MGuLOYoa4C@ zG4sNg^gidi(V6Fyhd$3w9=KnBEB*vJ`Elm?tN5A!7V59ePhK#e|1yri>{SQ#e}&l- z7STiXKTKyXOMdi>`U>o?FMCcex}kdJyUbac1Fu$JjeDJ&L9d`QZ$HE@fhW{6SHD76 zSAT=YQLFKJGf)1SS+slWemesRowOapqF^Vws2(lfA=z5(=n z9E#VdXOGU_G=-m>ezpE>{N$%K>Evk1iI(!S$7fHS#b1E$V(tl&`yb`kz~#RFh1iw9 z3$ve2QBTfti+UISE_{mr1wDc8K;Mc_@N41o{OjrO>23H6R>ka(XX)?5--)}in0j)% zf&APr-ll##R#G2<|KTsAH_?UZPFNPNz|yz__uxS1lhZEdSK{}k$%5QdOk-5VbM$Sb zw_-_59+>>Ob-s@O{V%1T#0t(OryfMVg&Xwsqi55t>EwFJ0g@vXRex3gU^@347t_fB zO6og|H>xKezMo!?$u*MSC1-oc`TF{L;|@&zxlmsb{&n)k^!-oH z(_845a2~%0eF5Dvb!?8mmA(G-_j&p!@O$_PWdF;Ym_0YY*O^E1`Ixz)vETE|*V(7; z@_U}WBcHpO<1&A|tuJ$E-Wy~2gFHX`bMlhR<9*d%!Ku#Wb24*bKJW7WI;KCL%XyDy zuF71H&$G-;C!EV(o%c?1gUsQXuak%Lb}o6!V!9;O@_iS>%)QCG@?M>!zde4UzbLNf zXAU02uf|_U*P}0@^WMo^l>OlmeRynK^g>zmoa! z5S=+Hb6xh`>>Z!$>+k#cH#Xt_H+9Uu?#mRM&(D1Ip7WV++N)>2KS_VBZ$AD5%j0z1 zsJ}LSCS8lpd~|@$oR}OsIYQ>wp_4;pu1;>!Q$2g@0MA*;Z%AivPaav3pZ(@z zx(}Y=eCD^z%`fsRVDjWurTEJ*bKx3#pT3TCa)xpI%;h)G zJ@gHu_hRP#Z>2N0CkM%1nY}yvQ)m4h z@J&pv^R4G@=QqNK`N<8kzu(DUj$8C6KdVgdSI>Svk3W;2Jz+Wj3Vw2ucj(Ob$qOd% z2jfI6?m2CI;N(8)QuQU+R^JfJexE%rIc0J_#|n}kJmuU6SO&A_oaI1y{$DWrby@W? zSW|rg<{sh$_3U%mZ&&N~rLHw0J7WZO5^&xcf%IEmW5k68cd7tY(sOWw{@lAHuUgCR zfUR+}bMIsF#pHg^@V~(t&fSeS@rPmwe)iadbPM&f>2t9Ue+%7!9*)oA9QAwXzhWhR z^2?$0W-Q|8wnTqJoQ=8X$~|T7YeqXiSzm3;{mn=^`*?M_xN{Zh+{^Z+SK+Pt)?rzG zM}&eS>X$Gc!rc4Rh{oCalaGz0zf`YIPsUFCI&>TQTD&cNc!GbJew3bo>#!9jhs{0F z<<8B;X8I=LANVbB7eD#VXgc?cxvy`jzX;aRcMn|{b8mNu{tD;nUrGNRkK;JJTHj;z zPxLJM3Oe_PZ}F@7-(hm3fAN=MW9JrQ?mwESFTyqIqwq6KzWAZOm-!p;4Sp56E7r%t zzQ6AJ_FyUXX6gg^x!23R$wq#2{g+`m{wex9yb(vKKZ(i3zMx0w8%F1TJh}AmArmEW#tqGm?;{nghewz1B z=ATdWCkM%#Glif1U_G6^JnxU>5WRezc{|&gus+dtTqTc@8p7X&|G>;2mFVnsnSY=1oK09l-x2JA7vOSzGq5n$R!Z19KdX(iPM@H0neZq5EzLp^zG z=E$q~nS&S6+weFh|9ONy>FZ9Sm(%alFJSh9^6GE$Z>4L|-_n_XXV87HxxVZxWBE_; z`_ZlOBtQFkdw%xQ`{+jcM$*^Q$*EV;xevHf-&THUI(tO}ei_XEl^m^#{_I8H>3bE= z!{qE&(r4l^=euB8{xz6=JbT%H`Pmne`{mxHoqG1!+#4h(&7N7**ISJJoJ)>0o$iE# zalZZ^={EFx_zKol&puOxpL{?0!BBmFQ}2q&3CMy!`9lx=73n|H*=x4)+u_UVm($7H z#_^x$_r}-wZ(%+DJy?$46O(6Ep_2<=sQ-EXH0;FB-gt=Lg1-=xGgYUPI~>*j93IB{ z`g&r2{KEZeO#YL5lMedtRZpIs+@mIc6z;>Nc((IH=y~)KT#5(PSJT7k=N_j#e++*no!sI-^lp4K)PE^H&F_uX@uNH!3t>g|u9%#4HoXS#)StW~dD1EM zTlt-`jXpi=hwly*Z{lX zD4d1Kt8)Llj{jTP>ra25r+)&!ho3-xe=}d!_Io-Gv!7<3%3hW|DsyG_gUp?Izvq4a zt>@k4^Cj~^=GMITGIwXc&3us0r>UOvw&&z?ekz@ODf4wcU-O>Id#04<OxhWLAA9;!w^h`W5fO?2kbIs8TZ ziF8Hmh5yFvgLzM6@4U$Q#`^kU_Ke-??_=iO%y--P?VZc}D0x&he)gWshuM1yIiGzh z`&jmn>_geVk}D+V*zNh5yRy&h^xPLP@3*mZG3UBs_Lt0E$>}mDyy)CM{h6OK&oolM zPu=UbAoFVGhU7Ne^>@RMoy-1MnNFUWxiRy}R_CtL*9S9Ku2nxDl?#$XC9lXnoOv(v zRp#dz&NszpFuCfp^eOzaFEp3Vyj+xSgPDK6*1r=Qs3&huj`@Ll_JDWj>|5C<7xOcZ zPN5HAU(bIWljCJh&Ky@of9BTAo7tZ-=VY$?lm7jfyrqV(bAq3HfX@7t{N&?1=;_!| zU-qaw_>=gf>9cS({~&!OX5YL=J$v;_^csEh>EoFBbTpkkI`DTd+L;Q9Omo6_cxFPd(ur6kM*qJ3fioUruT)iI1p{z#3RyJ$w2>e)hLoezBMF zt78ew{GYu(dsjKny#aI2lRbMcKl@@C&$}6uBW8|I9{m^R8tGd>51^aVy>Je;R_~9w z_sAa9Utcr4ACKy zH=#%3C;SrF7{_5@eaA8R<~;R=DqW8I`2uZse*p|%v}}eW1fEy-^R>ucj#-47psq7?_&`j)StcX2%Y!PK014M-j4(Mv#~nv#>`pE>FoED zeBDj_cj@Ffi|NeIx6|3fXVBTRG8boG$zJu2=Wo)VJvr~m%)veNWj`LRKl9Nb^&*%# zFLPRQit_5oL-M}Q9GJbdO`h+6V97sf@w3lnKF)kF-19R}XO5cbT=tKn`quCtqYGj3 z%JFpe-sE>@IhXx5dqn2A%x9IHUxn8?x1Y{@x`Y25_EpcEUX0Ejo7|)^e>FD6UY^qo zx8fZ2$~YQVsAoP*ewcl6jlSeI4fSU)szxUt%AR)IxdZCS@sqocP_LnWg8mXm@;hPj zku7uu?1)pH%eSjS^Xj`rQR8f;vDtll*u)&;U_mq&R4~`8`O)_W9jTiJ?ZQ}3-x7h zTS32qxA^_-=Uhkrvvl^A%5?UI1^U|Yvkzxqn#KPK|A-$sHyzK#Pt@yS_MOA($zQ)w zFONkr`|K|L?QnzoFuFC}kk0;>oGg1za<5|gC+p8XI-h<<{T{51`!M-Sa)9m5&BEN{ z+(+mBZ!ulMf1m7^t@u}ae(ueZr@qghh;8++rO(0KTV(Glt?y1u4)=)u-0QSauaDUu zAEvwM{}T6O56nH=1NzV7XV3kf-yO5xW?wv0|1MmqZ!evF^9g!~dIS2en0)tcb#_5r zx-9lpNDlW4e-hrLehUB2Pi~WZtQ`MkeaUUv1^@RxW2-{;_bqgC_#f%(G54t>JU@BT zM!Jmu9cBOTz@LC6@lT%j16>^-;^!Xm5ItADK9L1HV18#hxzQE$XV_m~S4<9AQN1L;34K0Z zk5}RVeO2fJJcOe#`P7s8YvEM&-gF~+9i98RadZQ`OyAR(e6p#Z({}zw{jXyF{X$vw zdHiwoAL!lmxAc2-a`@a=KgzF(P4o|?3t>h6CG>f|&j;zxu`Z6({}vX;FK{V#!`w6G zUiL=M>7c#`yYd&{75pLe+4OH^uRr~Lp8g5^9)1F+{9f+Ds(8%rZRUz2{CwVJp2+86 z=JL$*)%0in%G{9mblzk4`#PD|vJXvgKJ#BSeR;o}pfl(ETYvWbyhpQ7=5sK6eDay> zMTd8 zd{h06aFY7n^iDc+Yxb1n0&Uf^cV$jUUh%kk^0~~-*}t-HRnnLDdghANzK^`$v)|ec^NCo}JFz?IEZU1%dN%fUjo}YQQJpV<0=GSd>W!!{$uO^pk$4|cb z6P@?m5YK;`zlzR0u#n%G-xbT?T1@Vfye9K*=A*^>GA}0&%iNc}Eqg%r%=*qv^?j7Y z>}&n$N^WhVD^v9)1UG)@2&Jaf9-P~IF`c9S0B@}v4eB-aS(qzW{=&h-VlqZA50&=3ucbXzO$9zLw`~FZ}eFF7iJD$ zp|1}=xzYFZ9`&~{`^^pXu$-gY)3?!^=@OWIAba>T`jVq0pLvj*5=}-cn>2iURdr|hobJUYVeW0%we=nW=x+A}Z4}y{OVQi11 zv90s}L$AhtSOagzSM=}4m-tYSJvDp!>k7N|m%}^oOZDVrU&P0+@D1FCbDe8Ux6aqW zr*JsFi6`|Jq5EO(FHh3l^(E)IimvYG+J^3=KYRaTdb#>m?8-lYg|Q`;!4>*1q3huj z`~mbv+`&(NHi&-)j!{qEGL1f^o}Az)og8%y?=^M1hd@K@)O7tP@x!GU-P|Ezx&=02eX-BVvP`c_~6JAM`VAl*b? z?)j28{!@LQ`a`$}?@>=)RF!^M{Zg!mL)Dki9q}Rl7Wxn-hiOMQ*7q$vf_@s4w_i(- z*0+GZkG>H9!%zO&lRt`|ym}qK6ING$4s#E=UHw+9pnfNP9k$}Pz)}3NbOZc5zYA93 zzeit1--z|G4_5ZyIrnqx_}wx0By~J5`SHc-74ZS}2AEvY&)lCmG;{1WUpMY0mM@%!>C<0<}?xQKri zom}7={#<+kGxui>TkG5;^~>qxXI<%sF>~w&{h7bVs%OufOlLpM+_}ZM%;ithkLb_d zkv+V)dUBZLC&{xiZzh+h=s6$bv(9I)KFptk`!Rd=6uLdu#+9CvJvejWFX|`NllOF^ zOXCE6uVCipLiE-8?x1JU$tlZ>5J%TcodtepFyvnlOtr`s--?( zeFJ{ZZ-d!)vX5pzJ)xc)Ec;aU?d;X%^d%qf=Iiw4zl-f~Bj#Qp`~3`l2j{czf5QI@ zCYPN+Cw~~}+(LeG-&^@L_{qf@(z~!HPQeA3eYb+=+>QTLpGapvFT~%Bxo;RvPsi+) zx6wb~UYv@_18ev?3o-Yd$<2~SB|mNMd|Ca;>!#5+s2`zo-}WnATYU$;5wkD9s(vxf z!7H(@{$=a!$xI#Vo>7)9pU@!ISu}SLwo=@@H^6SyLpDIr`RKJW~g~=Iat3Qv)qqfk^ ze7)=GBF@+5kEDBG1FVj-aUL%9oaEf4_{l3y(Pw!6%k)oJ7hl5`o;Mde;-{Ew#y zsXmJ?jo0#T#P4vR`e$@!gBoNO;`K*{mdVsFL_*Ys^s3u zw<_uz6*~W0+3Qb#pQnEUzlWbd_RE|6o|eG8*YZC9BR_k?WWUe%^Rs^p;P>U{^D6mI z_R@ULWPWe%Ts{Z?O@DydBQhuC{g*k&mn+D7Ebpa!p67j+_e0*pnJY4v9`~RxeBJCF z+3$v_U!|V+N8Y=cqj%`b=Vd+p*?04KojEphZ|1*c&SwsJoX(t@IVpQq_J=!tUib3# z&UY^R(Mo>yoXiK=ck|xKoRRswq;tuqG8etU&*%IcI&*dMo9(`jpD=Sl@|nDEC#qNW zoV&3RZdJb;Gq)zkdXRqzS7K-9v#)Qa$71%d%jp@I95;Kze$UIEnmsD}arTyC`jQ)F zuG#6i*{_mUX1+T|y*y@5UhH`_u%i0ybmooZ3}>m&RnLC(0iAh$oxVl95<9n$v-PQxDhlXEwuvp;mBlaFVg>CIn@ zl{_ywOh^7qe)g1q@@IwWOE9@IS+Lw6j_Th{Cx6SHRYrXZCYMO=bildfQQ5~g=-;Yd zA3x_mhuJqDQeVj5L|3QVVfM}!>52MYrIYVBrjs*duV1Wxjrs&S`&jPB`m0}$Rj_ON zu_|W&t43d?{}=iK?2l`)AC~m_n)?c}Ap7+=ed|5_wS#5Aic6B%eIr`Qn&)`>6ipd?VFA;AanBL4T>9x$<$kB4*ERNhg;WN|(j!G5dED zx*cY(xrxr)wTRBVlf7o2?{AQL=G)|snfLyouMxJ@pB(ZRe&&Hac`j!DZ{^(kn0z7c z@ys*H`?4pD)1SR%1ii-dGv8#c>&(yGFY200rjCQvGiPU>&Yqv# zXoUXT^|i#0KG`6JFJU(Vi^ysozTD(5oyW_~Qr&pe;~^AY|??B_XG z;9Z!xak0M4pKs8gVsf=9&Q;;}p`XF&n7MKy{e=FFbn@ohFJwN>UfNXuE4ahC%5+aW z&cA?84pN7o{bQPX=EU7}?kBQO&U0=RX1<(5Xa4N({3bqGkU4oi|13PHVHcgeA-Q() z`V;z^U^D$t(8C`dR!>gBU$Izr(KjI?&12k{3+i{}V64`*9y8 zuT9RGy()Q4^2ERDPp;FLu8)uUex}jq&|7dRzXHC1mDDTXL@bBdL+_zKaqc|KeZfHW zpZM8tE~aPWFn!19+$$wloW!q&YjF+sa6bE0FMjf}8FVFl8CyBm7XN^ou%o`@m@D}O z{QKy+n0!C`RQBTRVY#n3!w1$Z=dQvbo}2r;&*&lQw_;J}*QqCG`6Hcsv6b|G=c>?$ zFneC|u>AX&srn1)%l`Q;o&Em_`bquQ`Mz$Wug5a_lD9oTXFq?B&V5_<*INAYp7RcF z#L{?eo=g25wf+To zn4jFZIh`D}9G!e%opT@Iv+8T<|bFV5iq z4eMfc_2ewI_~Y>h^$GM~Om29Do{x(#`QO=eNzd6q|CgRk@20!ch4E4T8JIk8xO#He z-2W!eU9J8f=UUP==_6Q`|0=$P^KdKH$K->f=$Aa_6t3t0j2$rdBunY5eePUAcf;Sx zUVr-gJpB{+J^TbRx1I0zdy3!F?{OSI^VM-W`9k)O>{ZDTGG}LR$>)Cd@b#W^v#*o< z<{5tGw1?=-VK35|Yl_jCFET$i_uPEG_oqM5pZPv{RX)c{sAn!uo^YM>dEf4#GnZse zN*-{7=VUL;du{^%YCMRUFNV>b@m1W3*)NkLCU41JJXT*j%>LVr&Ro=l9_sJ$Hl2C( z2|9b-({$#Axpd~I?CaSxvd7oacNbQ}p}4}=|2O`~KSmG0?8W=&%wc(tHFfSLtfp@< z?%-$MUPh0>9u_U+f{&H9RBa?Q?k_QJ{fFXw-Y+pq|>)t9}eHJ!QV z5S@H8dB8&FOJW(!K9hVibKVK{6`q%Qb`ZZYPRGna**CMVcTmqfG0^i)V)o~+^;N>; zYT2tR>u-Y(>B~Gbnf?GXXKvE}JZ68$eBGB{%(>(ZpY!kF-$*BK%HEayV}!m|`ZC{K z$Uh5{R~DiVVo&D|(2vs<>C9>G(SOEzc%lBKnB3wvdaS-_Sf8J~pcb8+GJEMc{LI%+ zI-j{SdwffMcdKVEPhNP5dh){L1P%CIu&47!>Ex^<>DKB8=)rgoW4_o6opXd4dlM5W@*W@RcXvc4c4KVjMx%VsUTymAa(Ao2H zU%21-#_HLt7t(9hbMG^UUZb8otRFwQWOsU>z6JORW}h!Yzlq7=O3__B=L2yX;V1tn%uimO`$%CrFQyw}a-_HD<<2D^e~i9X z{R;ewKZ^c{P7d`GKY35`sN6%1R=*0TWAgTvp7$vBQD2PZ_?77K^ejxC`YC-W&c@{D zb?KU^d){_BdESTgDfK$^DEvKS0;m6ur+)&!m7hRAzo)}6pTpUoiut|GUY5ORI{zqc za=rq#!?BorYP0_A_t{%}>T9Te7#r}L)7hW0e`X%Z9-V#Yfc_htzk|;Hlf31)dgiM! z^ac8h(V0gxU#{SPuP<|V=FsGCnX8fm<^9~+`OL$a*E0uXPt87`eeqT27W=xtVDiJE z^ju6nmt5r;=aYXG(f2iG-fp399e*EY-`-7E)pv}3o6eq=c{BUk^XdmN@AE0ncj33i z?4_&eoAtHBJ$Q|J=JU+q*%ub6XI`94XV1LObAI4wzRx^4iJv_r^WI4Qea>e-|B;`3 zDSJTX)@u4Qw`C8^UaVYjz2|Jet1Mnlw@qQS$fi(Kqll(aQbU*_gHV)YvdLzNrcAaq$fn%FlJBCFBJ};` zam)|*f6&TLPseeduh(_Crt9-Q&*SaBG5d4&j1A6D!t8l>(wXZs55CB6;=bgw!}t$j z=78;V=HA)P-OW#qdx%cHkoh?C)!WW3*Y|hq$j_Xyl%G8^dtT=6?BiYaWnN5P+?IbE zW{+<`zwYauM<@5mTzHV5`84zY`OYP8NUodQ{giXrQ=g%$Wqx=5Y5pbHou6E*Fx?uP z=}XR#y|6EThrT;W4A=S?1jd z{4X%`?A4(Uemt#{LfMYN@&e!y6=bysY_$}xg zFnf0PzoPu?|681UjDHUPji0^sbnf@RV=cOi{^Y|~((5oeNA{Jz&K+0Zjo0(D|0d^n zP(6G7<#bj3+0T~oCu4H7x9FR&pL4zGLUdEQKKg4Gl%?~2s4ksct0(=jd$J!+!&8^s@e5*j0TnR>oTD73sXE*-mFa>ZI=^9>d|-R{wmu0X+(5^QY3; zN3%!fJ;0OdBk>Bnzz?>Bd-mfV^}LUqPG79P8OQVIVqMJsUXuP?fAYQ&{CD`fa6OK| z(l{4?z-rEK#3TF*unhKAPj0)HKc7Df*W&N-ZGFvg2=2k}^*v7~x7$XaRR4xPNLQsN z)1~k%{s7#@FG5$L-^14YKhe$cQGRlw^Z2Xy$sgY2zk|t3@;;)W{(0CBZ$>C6?fgDQ z@`oC9ZS^vk+&=lqtNh#X&v*ovxaVBD3B8(bgJ=8ixEcMazWZUe z)|cFJBEKD8hNJW)4_wFZ!GDDAg_Zd==&$Gk?8#qD7slj9AF0>JcB#3)z42d7u4tg}zz(w$piDXAarR&wiA>By(2wpUeYw+}9Nk zV4e^8T+8QuANB0pnRoKM$v)E5_mSsh=B&&SnbTW%&Sc-tUX%PL&$~YE>*ed`IgmLc z&!x;&nfG6DF8f0p=kMp=O`oK{qce|X9?!gy=VP8*Gn~sjmc1p<|9`1xo=R?(JvIB< zL;5n0Wxl+J-_rM${ADSf=S7~Y+1t80mpm@d^X$vnGn1bjc3<|<%&(ajJGd{;)s^(? z&dsB5qz_^8sLbh^=kxqMs4x4^4RoHX1KpcBD*0WWuZz_)Ph^hCT>OpuuG2RYGp8np zZN$&Kdx>+I4-cwmUTaHdzbr|QbZ#&X=hvpQhh$#K+?@P2`}j5bcVK7Ce33c&CVq0r z0lr@5-Ph^N(aDn=IJZRodQ4t*KApMjS^e3kl4nihZ_(cZJMlBOW&h7y|0jLP(UR9! zbS`u3dVOp7*Wy0Re3hIlb5!<}{`wxk_c3$eBD%P*Gmy@_Hjdu~Gyi9v&t9{|xt;g~ z{@JJ3k(iGbMk?dzK06jupI3CtoV3&8B z@gn!;z1c8+QEaT9ed;!T_NnA_dHYKBdLGeKI#!VUVhH~y zg@4e=yOPUa%E!nv=w~pw#<%*X;SOA{FMI8A{$&1sdL8{2tdEWH z9{)j@9(njchP^P{^SkGy`SNa#N<7BPg}_O-gp|f;ap5EoPYoQz5BMRuf;w5 zW|;ST73miGl2f&%kEqwfhWz9k$tOPGCl6bo|0(`-Y=+D6W-RT$mvf!_5XY-0AFaW! ziPhATKaHSksh`5U?|M^xEPpm0#Xag{=n8b+f91W(Ab#?y#p!eIRr&?o$Dd6;aUo4F|SM?QZu&%B~PpQC%}%!`BFlYJv|c%D0Vt7i_%{CEvNduCJj4#Y2U z5oVsrzK}UT`|)e;%jaiutXug*-IKg0d-40~nOCznZ{gR$8?hp0kI23=&;5B0WncQ4 zpZPSO$9e9Y<-R<3^ITfUPcD^xFmrYt_in{Jk1nFG!sG7Ab3b!so)?+7X6SFEKlAn> zdKz}b%;lNmvcKi|l>M=RdoxGpd762%mHV>KXWrPsPkx)6i zJiyOfk~w)hKXc*pbaI{S`{C8+~o*f6`~um(dT>^Y9=)`&IIg@6-=qFZ@FP!#ExD zeqcSlUjI-`4p^MtsILhg#?shYU-rVzbn=np_BHusoqK~$zLs1ixyU|!^>G{SbnX=W zA%4nFUVa1JOg--ll22qW9;}``a|Hb!*2cWASmxf1xJ><8x*tBy|Aa12e}?Dct2jwt z4SFK3z|rdO;H9`h{Ve~TO{J@==e^Eq`Ul*H7hv*>lJpqt>AvJSi}`c-FVp{v4f)&X zcj(*jJe+`K^xZ+{yEpC0MlF}#nz zkWOy@B;8Q`cDffPXC0+}5?f&M@juXck3tsYy-{+%vF`n~?Dc29&oe)P-@;GeF29%A zcL)0g{)vAG-{tS6&%ymTP(69p5`GuV9`n7v>ih$kyeoOqU-e}V?x$}ge+%BizaKZm z_qmz*>l*$J%wG7pb9eKP(wXlv2PO}ypzkmy$IIT^k$)>D{}@mI?Ea3J9BCz;y|peq z)w!kEjh}t@IQ^P>f4Ue>@AD$MgZ_osf}c5WAbn6hxkn@VIeZwC8;+y1kH0{>u3)G?%;5K-U!>dM3T&(1 z0motXh~&Y|^bf=L@d+%3*=ru8vo|Hz%09D8{c7Cp>+ZlJc%ypuv%k>UH(t=U73bnS z{1=vY?qfQ+cXENu&DnD^FJ^wryp}v{xqB{gZ}y4o`#aT-t0!0LM%Td;cpLV?u9&&A zmiyLV74_sN$#33P&t6ttU-se5cbS8~)!!5!)1Nth44t_+xzJ_$lOMcCZ^xpz()syx z_WB3-{qT8AzPeF=a*O2Wuj-qveiD0N_T1U}*71{1{)e9&sT|!{U)~4&#IMQE{__w& zdwePN>|>v+55WrRU*Kq5r+xu$;aA2<_;1YnoUU|F=N_fI<0Sr{=r)*qC^>TW%%l2G z;Vb%YqMycQ{QSG4?Ell%%d3y38)I?)YnXj?vwAfgul_y0g)Mxpo~N%b_Q%yYOaEki z7)Rj|ec5009;F_CJ#Nx}l3q(EcPmMkP)|P4f?lnDK4!1|8(jzU{%A0LFDB0&?!GJd zRWR>M#?YhmRi#hU*}Ic}%;PuEcNTpWU5UP){t&nEr_inGTIheCf_>`Ca4Htow+fRd zRZ>rmyp7(4H|U>E55wfT$LKv+UwFPjLkQ06h^m z^ONJ&a$z?{O3V9ePnT*1#+AclsX3{qg;rle5g> zUykSMuS{QxrLdLyV)|m7iEHr@eLv6xFz>nc(8(i{KNaw=biN(V=a{V0g4eHr{vZrKU@1y>?{v-5W zI=Rs`{E?VFB=dIm(3#F1!sHa$M@#cFKP6AQ$-QmWZ=`2n_P7ajSIl!V^L}!@Chp0c zmANMQ!V}JCekelEb}oBYa)OTh#`v!O?3>wZ?^4e`lYDAB|8D29*Zz^8Id3hU{j8_{ z%s<`LlQ$&S*{yFXUZ-yj{X9N}L)5!q=8rn+2l?5LX3*7fu)cp__Ml$${rY~v>`U3( z&*Nv`S?JtKemlAt4#ng%nb%wB&%RSp-y;4=%p8^I`IrpVX6E z9Hxt5W9N3@)p(A2_P)RJGiR=&2V&;v?8(_XGWRamcfNZwZ*Ao_!puii>FiOrJGY;o zeW5Zxc~}Say8QWc=D2zM>`fKax8tAH*WfK!0gq$m=vmI4z}C1$-zmBqeFYBVPr;vX z5Wb?XKHZhhzZ1lH+gT zkHAlGJisKG{*3_WDZEnbW_d|Ah@O`{73CreS~e8JHg#+o=?eG0yYw_!JZm(lOj zS7Ld7-n&)dZ{R)(29vsF$n{I%$_`~Uc(hIO3{|EdFR#RV$4Kev&a-fC$9hiUT z{gQi1@_S+OmU{FUyjK79^#9Ud(3^24zZ0F@wI#h2-_%#ee_*ZnUHGrko#?!;Jj(x+ z|2Vx0=i)~E2-o2wxXJmdbaJJ&{3ZCPdP6#S-w6H%{K8llOGW$p&C#D+t+IN3{%QKx zve%#aKF|CFehWW=?S3yaugvuOn&;dD>Y20Wt7l%!b3XG+_S}5#CMQWgmCuu(+_M+6 zKV?tMoS7UX`)KC19lo!@n7z9kz2Cjb1@bwZJ?Us?>lo%UAm?3 zBXf41^X2(H_2qfng#HZkylC9yZoS(o?u92Lm9Y1?X-V>~MF8fS(dI@fHE_3n!qrX*OjoCXh?_b4llzZ?n zzU!XkEra+M^UGl7?rZ32`f6e3tp8q%=3%&W&g=s|2F?;ya<1APjaGv&_mS!gs<}#VBWhdRd3JF z-Z7Z}J-->996s}R_Stdj>+~nLyOFMkwK4f)7drdg0lKvNR$%stF^x1q;i9~`MZm9CD-&F)nnz%NX9 zrT<7br;pZesadk>Expm^}WSU?(+owDt5%#`ftHe{3@8eE$mR5foi}G)xC*h~q0_WhPn7m}F^ELUur~6{|=$iB!`tlxNE!|4}pID54 zg#H4X<76z0tM%u-Qj>fgY>RvF1-uj&p*6l>j(hiEa{s({*rq@4g_7%jssA1IuP`}H zRr+S!qyIj-20p`o6i4Cp>TBs9blw{l=a=PIrJK?OkWAfMD>UZELI9&fRoPzt* zC(tYDQ}`jj2sY%8r%T~j{xG^qtcukU5d z`_8B6e|{i*YQ!CH@vNfir){Ge3b}%THjA-_y(k*&`qE`J{|^nCIJWU-w4L=VCsuGACwlSmgZIn9tQb zf4cLt7o6wbJZH22WxwmHo;kFs^LdWdQO~}1n9g(Ir2c&F<~g%NUp{BEf8=?Qd7-xR zHT7q{JVNKWna}0S1DPi>Ul!MY$bES}=lPQ7XXfYRf5|;oJGaz5Yv_FL=W{-}JoloJR37lyyu=0{LJl{Cs(Myub#am`)lU9HR=!RPwtm}D0B2z`m*09 zC&^rzd32|~%zew3G-vri;f%Dj_(@kRG0M^4U{Ir}B` z>_08szk%O|zL?%hXV1Ey&VIj}&K!8FbKhZK^(J^XKe8qO(6`PwUUm+?gEYVSe(PNx2V~`1*(F;kcfE1Y7gV;X3|M+=%b0=RFBoFu@=4 zej+*SHT-*>pGXg&vo|N_U96sbqq4qP*j)W4%pTKP{TzNbOun>7eI$P?CRfOwH(K8& zJf<(XcmsM0W}nSo@gRS%^MAwr{NxkKm!IHguRBU#i}Ua<_w>T){PJ}6qU^^r`Tx=P zJkG%6nAvBu-<@!-y}s;~ZRwHf)|i69{Bis@aUs7sT^9%PlRvMd|E*qz9z|!bOMabP zAba7H`W9pMtGxgFnLpmWwQ&Si!p{1t)4MS5PcEnH;s@9m`#JwC{SMt5%lLs6p_4DQ z(%+x|0{sK7#qH|JcW$Q(sW+pOf1Kd&;BUj}I5+383MSVqsxLXjYW24KJL#GDSDcJJ zaD)ENbUhrwA4liC$3N2N=T$*p^2!U;Z^o&Z9AUox&HPPxJtl7&r0+F;EiA-uNe`s& z!0T}(=Dl(9t2WMc!Q1q0p~uk4&+g#oJ#ljFtN4#%a?uv_LHyEvlkhM6@98mgN4$uC z8kgcU^+)N!SP~E7aed!lCw_ANCj7VgSJOqYF#k$?HClZq{QzBuUQd_DypOA>{sQJb z%1M2n^BdrT-0Q!uQuI&idGC9*`qS!N)bpNt6}=zp;}_TitKn()bJr=qr`P%YeTq&lmAUCl_3TU2=wjFyOE|v|v!7)j%6^%-F8gJk zPnrLBxG(#5KDV>i^-+J<*Xf0sD?e7xp0R*lrZ4+ja=Ba7PhsZEJjZ|GKjVDzlI%S# z)$<&^icU^6#JQFH?9n&!dt&CP1NySJ)u1z{X0Q0fxy_Ib`_UiFwQ$ezAe|F`&>{v~wgo0|OWd&AW; zf8{wmQs39=UGV_FDP5LME|GmJ&*AKY)AcvgpShwqKl6D$GS{E2o?P)8IyqPc=T`7*;97p>yySo{s%I}w zPBC785qt&j)}Q??^YcLU+3Lybk`v#_&s=s~e{p{D@W=TD{K|A~I&<=^^jrA9_yp!X zLf%tkE>DhT3{$or4p>dC{W(x2z8XkMm3EzVv%I6Q5Vl{`o0g7oX9W{UGyw z2mWgO6DCh6?R*n{a>n=hQ}{FRW`0A={+K=OM}4=c@1u{>kI-{4`OH`9-TAe#G`7P_ z^licBn7p`@zUll8^g2wAwVeJZj@Ca97vhz8Sl=Nm!+(v=o_zy9`B(O%{`_wE0A|0u zn$CXJ*1d!HC$J$u`9t#3NArDo@YSQoVBQDJa8LHcVf1lKuH1?4flaY0CTCnrm%}2s z0mor=-$!No2+rm=p!2@xC;mGAD9oOmJgT|A5$acAa=rfQ$^GxAU)I-#?uLKF-{A-P z{)$KVqi{WcKfZ)j)NjK`k?+v^c`59e-qB-@5BfBk73@Yokfq; zw;vl}D{QQ<1+M4!!lU?S+^KIfJrcj?C!f!IkSEnUt6xcPq>~FLH`~rXjfM3u#pn3d z=)Uyj_$EK^+miEr#?QZh?MeR@GJ!LH$1^{HU&~LRhu_NyexLK)$nW{x|L6BV^H_3% z7u56loaa|QUoy|+b0nWfRh-M`Smu_@3z^5K>r1|r`7Luzo;$iDXAf2Q+%&Rms!EYGbxKk{74b0GU<=9uf< zH{ShK>CCa2dy+dS7i2Cf#>iZi`66>lo(E63e*rGUJf|~<%;Gn7?|JlK`W39sZ$M`c z$ef$J8;9U0H?6oEJWgd7?UmYBvK7!6ZntapPSn$63TKzLH z^U~|;+4HjJ4CI&9|18~t&RljO|5bkG)yxb3Pd)kaE&7tfWj@VbnD_nJHy_o%**)1W zvJdP~&)n0BZlgc@@>YK4(#%_#UouBs?OYlC$;syNvp<%m$6@w|PVt>To8E<)<1>$D zjyj~CIeLQgefT?Y0IpHbe$=O^T`GN%D)lkV)D}DiH-Ol zJ71e_MrU74j!KTJ9N-=iv*uOy3>!&2;v$Lj278m#eqK z3Yh)l8U4vAv-c$DzF7ZieH(BeevKt?BDQiadv@Mq>{kC@_2%>qboRRJZOPB~>&w1z zlKvdGI)51dj@i4G(w+7HME9W=;-9b==DpKy{ataIdQHsUHd6gZemPvvznAWSN3f#$ zYFx?hhF9T3>V>f--mjkbQN8*9@!_@O$WeD9rLB727fhlT>syKka1##4hWI2Vw?5^bfAee7D{&A$kICmB)_({8 zPP~qv{H+qdrJw76^E&=I{mJw4p6?U&XRtDUhl{W*{?0vb(Jx{0k-Udm%YR6JX&l4f zhHdz>@gBUu-=n0yQn*q5FlNuMrv5trDSA}yqxWOp#}uc>;#t0K^46pLp_um@Q{7um ze-*lcz9N|SWcSfE^qr*h{^o7^Q~XHZ3VI0k=D$H7z(@GUun~^HqQ1`>`tv^RN_v65 z*7PzuIn(2GRrQwiD7rG;f?kaK5DI$u!yJ4WZ^AB^_gi@{)t&zpCSSjVKH=-P!(XtM z`WN(Qx+(U<+ts_{hgjFoZ3TS_8#*_Vo`d!9TJ_{(^XdG1gW~#yC} z&&@oiGxz3mBlBmT>yw?&eAArH=h`awW`57-Se~buOY-@fediqKvcF`W$=sLc>}$SW zKIhM)M?1e9^SmuaXO7Hsp^tOP3-USsHh(VWx%)Tw9LL`3d0u4h$R3gB%x?X!U_PIR z)7fJ#c38O)^hZ@qL}fnV7jS&-u*z znRhckHrD?a_x}T@VDir~^#9=BF*#D^)@l4j?(Kk?o9Cz}Ke|Od$o^i0k@;*7-2#&f z5*&~|pvj?rFtK$j%H)8Vt zTj>Ls9B2T&1^YwFc-mA=gB?dUD)nZx_@C-4to_L3**GI%{^56#|q z+&$T!lFMg*N{(1WUj@wGHPpSOu(=1;xAbBCcVlwJ1#~@JhJSKyI!?uRun#_94d=??x9JfeOoKD`#R z_hoPH$4?HDJozX7EF6j1C+5+!eVydjZ_=&RlM_y&*QqB*8O3jo$yHa-uVORIKHr1x z?7r;nkJ3Zbr_&eUCftP$aX22u_wx1V7wE~D+`k6>j=u3&9$&@0hg+xrbIkk268Z}B zlV|khcjYJl`ktRXGkMH*ej&{J%3jWQ#0Gd7zKI)fFt%_{ak?e_7$ztFhTg2NHYWe= zPXCA-^smNk{LgR{{}4{X8u%{m)BhsD%b@=$-UQ`ZRsEzh7nkUDycoe(lfnR_9962k9~N&v-vpc7I>` zHcY;goNOU~8b0sd>k zaFF_Fesb1b{JOXlCt_JV;#~6Zw*0RAI`nqDhJOW}_a+tSr`3N)&&Tg^B~H}Wi7rf^ zOE;nGV^zFT{XzOB?13Zjw~z^(`8%HZ3H(}q0-1lauV;=;zEa%pZE~1(boM8|_62Y8 z=inlRJ@g>VJdyk)&)@75$!!ifpS>dUQ*ZZVPpD33uB+}`9sYed2A8Y<-{)ZFpzN`m z^*6^3&XEQC{2}?;IqJ9Izik?%y-G3lf$i5&;DK4`5W;BCa=Af{(pGP`3LBa={9sT?2j`r z^L*y=+xR!*XYNU^mAzmPKRL@!`jZQ_Q7?-9@FBbhGtV!kvyUd1%e>aX{TJe6*d4Q1 z_i|5izs$eo_}}7g=d#~s&&%9?xq9;aujql8yea$I1ozHTzZ8q&8oV3xej)o;_L1Zg z7r8HU{78Dd^XFkx{vR>9%j-AM}e=AImxrn|V*XW;2AEyW4gZ$lea=GM+$tSYEWq!}zUCp_Y?zxJ7hW<0X0Q=xv zoT)GS(@1&?UV~TT037Rl_V8wOPyCC%&*|FswOVu~9IkIWZpB9G$vZCQC&%hazlNRh z9p`?*M=*J2L;4^3PvQIcADpgl5_Z8II9p$G{1W_2osx24~tuf*hw>*$U8 zI^qicG;GhGj>#+kss15u$Gm@gn;weEI~u#EBEL1Aydt^6GWC|~3oyCg3-mZ_tA8QA zh3-c`N+(zNo<1Kd>ARkui0k=(#e;Yfmtq&}hRJ`<_uu6_{{8MvE?S76eg8219Ny#n zDCZ9GtKr}HEirlGH1&o!7|Ubw#pEQNoPQPT;#vAvV%{qxpYNeB?<vrOg<6nl|v7-7gdJ~EBMLBchZ-r z*Q9&nSpI8tUH`lE;2-6Gfo1Vp_2h$(^XK5x>c5t~{>=Az<|pu5_zCRvd)d|RY4*14 zN6+v-!$Wwva}#hXE>eGy&K}X2e;AXaWKO%6zsI?rxEv?p%lHmvFHBBwBR_jU6*|wU zBfd_5es#JXo!lpL?z8;tt;gvZ&Sj5Z&ClMNIVy8W@|2m*WuEImPj^1^a&m_!)w|(7 zeJAP6-IeLWSVv#xxa2)8)VHYT`SJ=sdr;=6>~YycvNt9V%5yHy(d3b{+?P2hd-V)o zceQ%-VXonqaXz_T=Htw7CG=&FO?z>FsH~p3K6}>> z{6e@;f1dMe={f3|1CkFVzr9pFdB$YAnDYzhf6>X6Gyi_Cp1pGkojf7ASn|p?>J4y^ zdo#a3#?PKqPW|7QeLeZoIs87(ZKZ#QQ~1e8GXG_+TBn|Q_a?e74tCGSnEdh__2gE? z={5RZrIW*c&9BL?O^>9rFPzJ7f^VrO-(N{*FB?IZ(4Sl}d&~fS2Ys1ycknas-bQEM zzFYqUelJW;xm!JZ`XV|x>s9)X@V8?2yv(uLN0T33<Ayi=@{H_L?fKd3r|KVx_0+T14WhHxC)eoA-;cNA{qD&gy@#%Zr7(G8a+jx^PmZ^P z&VHQzrz-z*Y>odn*O~3>Y{3;cSO0K)1IuIH4-KS~Umkb9HD(V_{{1R{Jl4V9?x{vE z$J00*@6-1#R>y~N7hbPFd)-g`q5P(F_Tb(8?7i8KHtS2CxL03EyaKfqjG=2{5qwEQ z^3dcZ+tmMz*=v*6_Ti7l?d~ayz3^gOfyoR1lzXua_Q&Qpz`0tOz4-lTJgC1GW*<$i z(?Z{R^*_*e&`t3K4pUEl(1oA9xQ2Rp{wZw6ABxMcG$s#xPybB5kHLZu`PsXZi#Fn) zr++@?{a0!AG5lThYuFwOtCzvv{QYfZHe}R9*?&`Df0Dl_2k3LEdq)XDtv0Bly zaXmhSb8|0#$A6RVj@$8OKgi4IPMEwcd28N_jB?*7eR)qfpB{$Y^nFRcM(6!NKmMcG z7Vp7&_?2@lF}c?rbYmQh$DDhfE=1>j?pXe5T#J9jzF0Tkho4Uu{y6@RcpWC6`is8X z`6uZGbVa%W-7G%dslJX*E?buFr(OzsU~-}4c*)SD=&ECeLlfzgFLGArmt#;=@1Dn>;C!BYf8+mypSdY> zb>_t#>e&l2cOB+uzRrG`=iYet_te)8oA5LD9O7pW&+{d7c%Jjw$Mc*{UiG#6zQxX% z`TSz{FTm{SJ@ggB>G-0)%!kEuUi}r!b2Rfu8GV@x@_byv@9$im^LcJRq25$I^KCQ! zH2x-R&+mYv@P75%=*&skcc$~9Ap30g`r`_jC(m@9;8c4GF-t$3Es#rLT9fy$7Vcuec3~f@@wM=^$W2w zKl^0%w&bq6^u3R3@g3(X(-r9^blz{(rcbJurSl$Y6+KZsdCtT98?XzWt8Y1O=C}3p z$oqyb)PGP<9+f@6kossWq3?4%fDht+?4>{XQ#bl7^^J6Yx(xj&ojo-B-Zu3`>L;)_ zzNP+qT+Yva+=0ImAHuo%l3zT@pNaEv1m5H4cM+ZUF$3IF4xfqE_dMN>PClCV2|Lxd z;~snrn>$wx@4;0#3-cb~-}E-;lVc~JE2}<4eI7Q%t@yaUiF9)8OX&x&mA=+=-V1%m zugA}O#8>IOH@S*#idCF@j?R0vp8Q9!mHMmnYC1X320HI6D(UNjC-545$qADaRN|My z3i>D0$+yPP8*ziaoBa1ulD~_8Gu?(>h@ayd>V+^l$EWnY`d-KD`A6ug^nQ9QuIDF* zok9Ojy*qBkr`3~-UQHLlyf14(&vR}n&gbu=U!t#|oBR3PM&~`^Bz@Q73+iL(G4#vy z5PA;Q=O;fuLa$fob+<~0^jFX^ius-(#gq2@^9jora!~&nD@!w=u5tJ zhkE|+TpH2c^%bT^(>?Ilve%#aKF|CFehWW=u6|E5=VV?w%wK_-|1(!D@cTR-liy?? zJe!~Aedgvo7xFotc|7}k=8WvUQ{9vO;a|R=?3L}+vj^lk^*%p)LY}uPo&Ot7)7Khz z^Ro}v;b#xY9GK@va)RuOC7sV)lf5j@&93e%uP@KFRs787$wQLUWDb5&-?Nzcv9kLz zH$R|$7ynjz9i4n(GM)Xf8J)c&`(JXs>FUW5=ef5y&cZw=cjzC-KMQwYWlZjvJ#HC4 z^JezGj_%9cka_8l{xz8W@ou`0bIA`f4`q(Y+?<^58t0N5w4t+i?R8(~&ARFz^OJLC zj_$|L^E>lE_NL@TE%epG%$u2Kvwuub&)ihS*Pnp9G5b)S!^uUmmuBurE|mQxb4cdE z^L)L`Grj5Ce4Xs=*{9y(AJW$tn_zN-=T3B7gX|xDVTg~toyPDXa9MKpE>XYI=RLx?#bT%oO);eG0dJ+j?Vsbq5cE-SIoSe z`Fc42eCM+#WM4eUFRCwlY)SeT^~|?Z_-pyu>kjkp!*?;c&1t&0bLHv&boRozbWQc_ z_nCvQSI<6?+#qvy_KSngCr8~tpX+=}9ETm$7vrV4T|ImM5`OZM>_^GdYv}K!Zxa0k z{TiKpatOZ|cE(lulB=HLkHq`c^L}eGzW~?a?fQ~8WpB&A@r=H&^=+bC(^ujP{Ny0n zW0tEY=g8iaeD6o~70%bfya$;>Z^SkFx6w=KK6HJYi<|KRd>aQlw*<2{7op$OcY?kJ zkMfsdS$;>l34X!vi`QXt>FodC^UuL8&Nsuh{02Cfe=%nNoTgrbe-lpS*Tt>;?KqVG zXIzwid+iYVdi{-XKED_?;Qs@=;b!%f^gzr$e_Fj7|66(y-4wHT|CD|_gA`hHJmUr$bx_XF?f>!Gg^X5YV3{cV0j z`l+15`l*>n+Thi^-^PcAh{++l<{V3)=-FQ0hjrP#rJ6FTk>BcX_ zUqB~+tgY`W_3!XMSW5pe%zMFB>d66<+wA2Z!|u*Kipgg;(AD+jea1cfllWlfU@F)J0bVYg? zzR7PvSEVnZlZRKL`(izP$*+@>uIJC!x05b}V{r~P*7qU(TgU{?{2kBy1b!_)fy@h; zw=&meKFK_l&;6NxkMnty=WOP`d_H6k%RG@iX{67C?CIHOl80oT%ID(U`ltH3ncp*q z&Qi~Qme0S;bJ-V<=z9Y59Le04=i@5%>{ZDF^7)^=GoP1v4kbs)evo}M&z;OO#oV9g zM4rcg^Y_d1H1lAdn*-G|Pdx9QWq1My=(~~5^D}d0K94)7Czr}Tm*?JY_askDUXyvQ zr1RNBE9lSj^c6bK^E@|8IG24QbNz+?zrA-}QBB>U)FEUXuCebM@qWTj|W#4>&gllhgK~U(>&xevf_& zGnZz5S;eoQzcHOXD*NmV^~{x%={5Sl#%cV_QxEeqH$SJontzJUJbp9%0sf*d`&8zi zY5e4D<>{`@9l|cy4wI9v)t`K3tol|wUwt{;q>Ff!a*V{XHzrIzSI z{T;A3e+%Bl&%QX6zaMvDO>C#XF7C(VIC)Qyee8PY2J35s-T2wlvp@A$zeBw*=6ygN z_2K;0cn$8u4LDu@WAsG)9J8M{r?1BBW6hmk!O#1xkNC-BvImdW_XC!}{CoPp(0Lz` zoOcd?4YqfG8GM#sf-dU6uXpIYr>IXSS3K!l6WoiVaI^mN@OrF;3vd!v!^QY-tmnSu z!bj;czR%>NwfJlCJ}i!Za$j=C?B$*KZ|kc;=l$93zE1L$?7^M&kJR59f5f-c>tcP( z-v0@`1M|Ko?=KI#r;7SN>EGcBtn2%`N?&rh`Rb4IFQPxjSNY}W;>a$j<_~}7cg6wy z*_iy~XZ3abe%O+KJD$dg*i7Fky0E`b-a{osDdVo^MTJ+Yy4arJZQs#usm6`$e1NzbL1;UK(Ky&gRlKf^uhed(R_ZS+z6o}c%bOZmev z?~l*+^ZlOR50lqjNsqvH+*_G0g*Wj(#a{d=n4Gn#`bONLUO+Fwa=x#Z=*qZJe+#-O zZo`G@$M8wad%FhuPUEj-uRrsBp7{y<7JdR<{l1p)`?(ymcb!zfke@j)&%x~Jd7kET zIQ#Aq=dypca9{TCmGq_hvbSY!Y0b}^nRzGAiOgHsH!Ha>pVP@9%JVb#W-oux{cF{; zKNsWQjrqLnN+*BWM`v%$zBAQ*nR_#zX0OO|>P6?5C8JN={(o+yi4wtTrT_DUj4g$-RvcqTaT#EP|y68x%rTKo>!mK+c3|^ zS@bFQ^vC2M$qgFlOHPrTBy&;n(+>K6!h@K7a)JBXVfKaz`quD!V4i2-8lOs@0_{ekBU=uG}@yg|Jfom^uR-ABC# zoqas>@qB*f`OKY9JC}KXn!d)EJScnm0sW(KJ?1?_CpzyplA9(^O)hq>`^#eXuI$gv z`G3RA|0~^pBW6E;n|=e6n`Cdv+}y@}*(2-d-;S--+tZUVdsJt7xV|d%3wS3M!V`E) z`Z2j%_R{17H|bCQ@Sgs%{Ny$r`71E_SoRL>1;zd0b$#71b9{1*()#XK&z@SDU!4CC zJ(12Ha}WKD`WJNG2PL1JrT!F_($@^D@h`?Un0)dzx~BfT7n)5kRL?$gfS>((JUvI> za7^C*i~0<#gIn|!#fLF@Q+0X~PIvBJOg{aMdUDw0D#Q1kF&86HpOxJH_=P61V1@?KYrf#C0D=M|3Jwj=juD{{D0`=h5h(z`R`*h zd{sU9UoHA8^+8yTpZCYf0~@ICQ@;&;QDw zt8Wz!!6$K`zP_0EHS5*);AX6!bC|sIL%M}~TG0!z3s%IccvAn5^iE7}R!RLd<~?yY zx-pi*OEB*#A9r8!<>X-X_{sm1(~jj|>z;}@hyN(%J@IcL6FBpCJo6LywfqFKm*?|t zo8Q-b4rkwZ!tZtFox%Dt@8|R69e(!Q?CZ%HJbns#>09RB%o%yU-Oc+?V_& zbIBBb=DN)DKQOU`U`+J;rt}&fGBl%H(emhJa zmH9DqZ|0hB+_y{rDmwFIa?2O_nWswX&)%IpJGs+7{Uh;f{mJXA@RJj4Qoov?`7iU< z#r*RybLW5aGk$XD%)Qx5_U3*cpy$%r8oC*yPNoavd1SE z+RQ(SnO8FpFXj)!O76W4pW`RDYDW)JUr7(7v!@QFAI9u|6X{o-+la|a>Znh}pYUCM z*?+S46;p45$Mqe=+xf{M_VBCo+u~8ozL$Nhvi^pc{q`;Wv+y%4tnUC0$0_RBpD*NR z?@GS$F#kRMQ|ZgGHD*s-PjAQM1=+J6ET zVcr)ccWBBlg~?Gaa8L63A?hz+N%iFAH_$~f@4LF_Z-^_gzP<`{a<AkN$yt*7?dLbwKbp?| zpMCdr{yUiWWyx)qIzI|4`&?eCKlxh`dN*#xQFhs13ngC|t8Wc{ zAL>6w51{it;0XU2ejRLykEk!FlaD^c{}ax}j(7<3-g-Oz4E}<5VGH-)hWGOG9(WO* z_np1<&B2rEm(a;ilCvcTuBP4;*I@F;Q|>8_-PDuwzCl;OYw$75dzFfG755}(+)w9y z#moA3^5@g{)5)ne@TcR4>eth~un_iCKSB?|`TXzb+5hjRv%hBUe2|~q=?*&2<=VauyI`|F9K-A@CFwSpIXH93bobn# z-i*%tGnf8E{b4%u?ru8IljJDbpEG}#)VB`b#-r{ni^;?8qkCdLxAXa4&iQlHx8rsE z!f8b5PFiD>ogDTC?c3EAIQ^2_5k?C756xPqTKI{Q=Rr{s!x zj%V)M?|gY|iDmG5_a?u}zLGuuPJP+?vX8yyT;}3k`eyPo?@i-p|Ic2Mc`LczpvJ#$PGe)g`1>E-$s(*MBZhne$}M`hlB#koqD`MDNd z!#$brlgl(#UyG~sB}Y!amc98e>Y1yu*KXl=bx-E8%*zMVAHZY!GG}L>&%QKK{iOc> z^lW#1i{;aPyJ%O%7XRj@ruY=k9TIT&ulklgj>*fCLrrpi zhI;mt?0eaFPV1YZ?*xv+yq`)wko~EudTHnXNRP!+{Oqv<=+StOzNVP{CAr06eM8jO z(dS{_rxjCAPM`g;x4zHSufTufPW9xo$;~SAf6+IG{)j$7Uy5JyXV9(aQ`n6ECf$}! zj+=a;J%0oi!n_x`!};uY$-DC2>LLBT^lhM%Q)QoA$v?|}SwnwSelcu;Kj2)Phf{FB z^W$*=?!aOCe7SW}fdC!pidE!T@T=h> zd|%%-+=I#CR_R;8--?T|9=5{z`YY3)(d+3Ebn?EJ`J4I4hu+{nhIzl0Tz#beqj;^p zKj1(4kK$VXy>tmIgw1ijz8<)b-;JJ(d9RlDHx2m{@o9YA`Jz~b-;#a=lPh+k>tXW2 z59w0QCkHITZ-h@`a`0;U=kR|EnZTL9|FNNed?Y0$-k=5!C`;M{`9tU$%&E^)Yf+Zo9oMS^>%(EemOdG zLVLQ4diL?;2Vbk_d2k+`z3q4OTujbU!hM}_u=;h_0%zi#`d-6z{LGEXy^=pA2g!V% z{2|Z7?05IM?>+Y=PkDvTo|Sp#sQ%UJnU4qZ$KV`1s4sch4g6mGT6E_5 z6sI#6X8!M|{}aqyk~#8z=l-FdIkXx7Km6o}&+u>J-$G}9%p8_^HFIh9{KL*AkC{rh zzzOcVo32G??_SN%o{;&o2tV`u!}JR0Ge13>d+jN2sqe+?#q;zfuSu@fn%_l#_Osb^ zW4u}4G}9s0BPwW0T7_L}4>hbh1?kS+RVcs{KrnB!|rT<2L_T=IGWQvk;#$!>QCNTnNALqT(gGr$M7pWseckC$6iY>!|YcL=u^(Wi+OMNf_nD&uIiJp zH?Gn*gIACMSDd{Ui4zuc%8;Q%|mve6=Ni0ltqFolAZ*oZp>49+%)) ztdDc`52Y`{jr^;z6kdq)@dHfm^Q!YN@uy%XesZdL^l|m%3(19&vmeu!ysn)7o_RX+c%ILf_!1CtNjsc$7edv4~D4eDLg>(H4mGe2b>%)FfWJ$ul+ zKl8*t_*M81V>|w}IG*qEU+@K8LOr>0=GN>rKdG<6>_^!HlBZ@r%l@`q|17)#GnZw* znCJUAqMmszxk2)d?7v<0z2)2vI`dlg)5GeSTjuNQgqd@H(U<)(dvb4na?TIwN;n-0 zxi9%~@{G)-1N8OBXZ26P`}vRHLVhn?fRoiTk7ll$sh-^QV|@qsr|7?7=DVKi7vnSP zBQW`I_M92|D&ba~ifx^1Lnl94NB2?BUQ>bYuU?Z*j=7W0-aeZil&^>9^IxPhfA6Mi zV|#3_KksL<&poETKs`Cm6S>DPP8;>)l;5lG<7W>_E|)#!diB!!vu`ZppXL7Ky6gFk zaH4Y?>FmYXOZW4)WA>7+bW7(y$9~uV$6)re8uVW=|L(Ck-Os(Xu`quvy$Ty(6TCp* zd^&l0b^bU!8?)~f*Pk4zh@aO~e)g^GcW*dfQhgr15j$gYv=8XP`X9yQ3FpzT;b#4l zFu88>knF4N_2>OgUAjCjbN)Ccw;!V(WWWDPU-G0Q8j@e+{m^QCi||2w4(np}yX?1T z@%Q3l_ZOvC(0T7ul|KN>sAsPnL?2V%hsniu(#c0#(EoPsG?w$fcS-fhSXF%@uH{dn zm*O_8r#=E(<9f_qT!j7<|AWbo%hHW;mitd&^4-zu+c0@&-lMJ7-xHha%lm*~{I~g~ z>6__x^c=bep2REh3Vq30l5<_g@1n0SU55SyU*+e0RULW{zJ~`f`PL}tBpN zAQY_Aw}jCU-{j|g#5??v{O)ve#A^IzSPskUOYWQZ;?4P&V=w)iG4IjuQ@@@6B;J9^ z!yD2c;aKNtVg5b9x$3{e`goPTQJ6e2Ie*?8J*uAkY@zdc@70R#gjZtn&tvpy_l%`~ z3z@)~zvG#oz^~;eaFyT7%&}elz8=8jHrWHF^EWwnA)V*wjr0V}KG>hW4WGb#z9hd| z$j@Fl!`I0^l$>przWdel`I|i^dtdgDhxKO3g;~VJghnWw1 zIG@}h``1eS$sdR7%if;(ICD;N?`HazIo}Se@CVV2=@;pL)0t0u@>lSeU~hi%tpohb z&Dk3=cmH=U(w8|bbJr{UxAFJ*IsV^0nq4r?_i+tgmUHyibO&t0A5CZO&ip(_J-Nyh zI{VZ#=d#aa&&j-*eCu0%nIp5`{6F_yjMcCXmcU1xpGjXqXCD3k?cMv=&Sm<>@uy-? zCL&skQ7G9)l$6y*O$jyGG?Z;=Y$D4dRJ3nMQBqRb&X7UKHkm{=t7uBu)L=XKrpbNAez>pYL^S*!TQ@f|j)i|=+d9ltd5b65MU#|zYD ze$D)Gn|$WP5_J0TT67!h&%js3H)H(A$?~}$=}%|wjqecuCI0;_>MG+dENcJwvz6%h zg2thO^y^ctOTYa)_3{7WvsF-cE)G$bxi9nDB>A`H-=W{Y7UECSnU~^2)E2MpjmH1a zytc#ors^_h%n&asegjs+`SN?|_$L*`-xU9dZcWDz%KTnjyeF zPkj$8t-cfffCJ?_VtmGb(wV20sLx#a7`;+mKYUjFd+dUl4|}W2oSONwk$8Nqd#sDk zSX(~6LVTe3Cz%J=s5@@`TzWq}mfnIB#E0P_jIZ2FUFOHkx2x3U9_fU-uW+vX9{Ov# z6+MpbLjQyAP8X(M#Y=G`9#Pko-bIhc8sY87UL_mq@kdp`fdW7@p3$ZL)ATkxlf6oI8t5w;!f)RB;J-@i)AssKr#C7 z>MMCZ#46U!#F02$z9!x%z7CZheC}7#xi?-< zFIJ!Xqt)W^!Aj9B)qQ|##eXex{i*kP>J#`ad;;n3^W0Crn7%#F^ZR^%(_iI0$UHVt zKF_OSbmnM}m4ZB%c6u(Hr7q9!oR67LhN(NI?kjo>o%1@+`}8w&I{x;`yo`<)q8-eNX9;NfVOkY*iemS4gALRLaR6cXS zB6L^&v{T+Jbik6i_D`xs!!iKhTeqNSl^9KKlBm(9low^GNw=bmwfuq z|H!9bi=PyqB>i3Hp!5xUtlMC}rF7=q^f~k8(@$ky%sibrGXB}s>NA&CqvHeZ^g7MO z(=X&aUo9Sgwm|&=@rjted${~cY$2aMBlBwLS6)}$PuNeq2)zq4C#Ap1e3X9FvV!=$uUZ#hpbz~Drf+`B!iU8(FQhNd z9I`-NYn-P(zFsBqDdL;xe)J{S1B=L~kBq81qZ8ZM`x~DLB|)$eP!m2((>ovi`c<_-(XXWU)Gr}ss14T zfa%lY!=}$ZB!9yC_)76PGY90JrG)w?)NiJn;ZeLpK67*p`mb0YAHnK)8!oZWUb-*l zp662eYccceGMv{^RHL>crLO9w+w)CB)~c8&79Gj2{`lJ9A!#)MI>^%r(2k z=io2>g7^xT%V*x{GDJ9+}E_DbKg}%qf&xLj@ol&l%gT4eL)ce-4LvLMco}BC-%5XvHLa_QwZ(hl zf5k7MPvCxx|8xue4UVubK6GWeJQh+HfAS%EDz3--v(E45a`E_+-Q{bD=U%xZ9l!ip zdK3=9!Z^}C`|v99S~v*L#&fY8es0}$bR9bWTYQ$*;!E*XeA>FI_!Q=zX}`LQ#N)FM z7tj6YY4TTzm#1&?`JGRXm0yo%%f}ySq3%ohpRp1)#sfGAJ6qQnH;PZ9SI{3|?!mgy z@nfD*Umx$p!MF@_UsBxq)BPMb)2q~di}&IecmS8H|2;0j+&_J)?m5i;+GpzGqi>@d z;>+se%MTUbgZJU{*h+m@I`=3S(pBW!_&M(uKPi4a4#W8CzXea=)X#Y86Zo}!0(m}_ z@IB4>z1mkMd$o# zDxNta&-Xm%az3Vy`tSA1Igo?VIPu+5f%sD%YP9L{W{d9aE+v2V2AI7UO{!RL&oZFd;(hsD6oM*qx z8TtE#%oRBwGoSBMKO9GS-S~SsUo#h`-`lNz4#rPfOQ#=5pOwCKn*HNjW$qa#9-pr^ zojG%_eLoUUpFCSU{aO0Q^d0Hf(m$u)jBilKewpji2Tm8SW&g|xKhT+n(obdX&z#&> zUHYYK=*+)$=tmRxdi$`4c>K86=(q4yJc_^cYtv8GmLF)}jr3uR|2j!Nb6QI}^H_Y# z0pjs>2H5AAcrUs)y@-xa5?|*N`S?)B)y0=RgT54tt8Yke!pwi)(D4gPt8axJ@CvMe z?J@oD3Hq4*GJjl4r$0^KdxQ86jE{MO&YU>IzUeE6(1)=tW`2l2Q$ze+v~9sk|Cohi za0SMPNS_=3Ha<)G-G27{3QJnwEU%|8jz1b-K6CGEb(!-s|2DBt=KOy2JoQI0bJk7r zE5(1t_*wCfwu(Qj{s=C{P4Y$Q@py%wqdGl7-9c<3{{?+OU1#iy?_o#$7~@C8FNz;f zO+NQd!|b2A<4O5y;_(O9iQg@rd$2O%@&9x0k-6kb^)=M}8{>hYm|vc8`D z3s^+F1jZkGobIXaE}Ve5_p3>tp?)x)g_#S_QWyXKM)`K)m(UCG0`dCv8v3tv5ju15 zcj8;{b@}+(@vZ90my%yhPsaFrh2{PF6!fQG#@<*Lufo!3yMp*al?CEU#K$bF{xZy5 zehwWU_Z92+Bu>XyXh$E%%Icr`>ys>!v_V}mQi^un{Nxw;V;-||iQ zH^d9!`FOKmnD{MS#Gkh=zFhqCLgKl97^{9Ko+tkZRuUhK9k7&q6&#HP@(u7byc#!S zBlT;sD}E~fI6f%84!;)Pfbsp~SLR;f1ND!oTR`XU0XEQo#H#9+(05=3@!TI36+a}t zlWu{#@kh)(UVOgXBfez)LwK9|V)RIQ3f+gkAD_UP_^!J77{7G}JyBg3+$CO*UW7gH zKKZ%WP`o?Fr~kFg^{3wFsZZdy@Cl^9-{pInz9hay`mD?^IWN;MWj@RNmA)%|O>_Ik zXV^ifzj~OCf6~8~*JOy=y&6**UOZl<4XZ(aO`6825Mx``g3F7r$J z@62WChrU-|Uj4One1P=j@xT5dKMt2-=Dwfoleum?oq2by`pn(&G2+)uR-e8x{bBma zoZp%IGJig4-}EnA?34K>{$l3QPV$-m>R5jh#?Q!k`;GV@Jb)`Oena}RLH5sFdW*Uq zSQOK5w^9Fwc;>|PmpN~j$hT6T{x0+6LHW}1bFmDL!p``d`qgwrTH9E#olc*Sel&ej zZ}~aaryt0?ocTNb(WC0)KgHMGD&EmPncw55Z54k}UFL`)bo!Fvbo#o^bo!U{`xC{} z*S%u@^#32xrPXB)Z6F@M_JsU3c(?pg+=H1b-&glBK8=;sEyK(Ona5kH8z;Xi6#qYr zPxcOdJ|4s6)^(xVV*2bY^kJNYC9SJQPp4~R{ND=nX?}s~tIr%8|0=#{eCNzFnKR>a z#82sKpZE~hd7bzjGw6n?!|u3QK67M$dN2;bI`}1~FVB1$-?poL&&6u$-^RUokNoAB zxh{Um&FaQr=GpicW34NfJT4Y*OLxKR#N&fE6dx&`IbuBBTfPt-pKK-lH~GvdQ^fxz zejhFrZ-j-#kJ6W7e~kY=OkL)=%rOVWx ze@j=SPh#f8zVrlq1P9_W>uX?qu21AMPo6Gc9pgW3Q&(JkJ!bwaB40*4zQkhjm&I$+ zb^P3W#eWo^kGV&yLTAp4-}hhf+&h-D&&A^Ly=tXSem}jN-i&*26n4Urcp5%veH%J{ zN?AY8a`9n!rux@$3RaPizxJVcDe(pL7xX&%E_xo;5WfT0aSqY(MbA|?TYNTM zm_8fhPsIN?rf!4$`}9BYF`OX(0G&DiCGmUlb3BeWU=!;mVjuDNZ8hm`SXA9#=>Ong zjBl}9-Gky6(*5XJbneyer#s3&i`U~Eya3~`uC%T)28-0q#5*NA(BtXcGZctJ!F&Eu z3$Ic?i=L15@EMGM)>8dd;s@yAn0xk8bai|cb5B_|+CEk3o%oP=b8IRepKrBz2k~a~ zyBPm(jr*aI`;$de||3>-*zKC78l|V_Nh-F#?SCg9E#mB ze)Q?ycYNQS_KVMT2_1hqe>d=ieMaI*b@6w93!cEKpYhZu@N4-5(wC)w&-Xn2W1eq0 zKl6Ob^DOgJ&doe`kNMu$#rP9>-sb#DKbGfl&d>C*eLP2Uu4F#Ryf|GxK1oSB{dD@} zJU?^JrcX?Nv&g#4Gb6n3^dI-o>7UbYR~FB?kbdTL@$>APId7tPo}ZoNbDmVAH(>gi zAL;bR=^t`#u9Q!IlD?~zc>0j}^ce5s9n85gM!p;79LjnBg8J#0^SzP!oIB}Hn~2}1 zK7C?ax*eW_$_vt;jup>*afgD<;xjOFRs5a7>e45qe@q{CmUU04%RH9zJ9ANd#PkPe zs864fJ~I7z=9%W|ZtyypQ`4WPufJA4^LH^ibNVX#ATC?C%()3^mqCBFn(y}n)to1 z$Zx`;){n>H*j7G$;>Y6Ydq0$~h5hBH(dkEz(&^)_rqfTR4}94A`20KQ5;z+tV_*BE z-#tWUu6s${ix@xaFLe6HHuMhbAER?Gl76JJ#TZkR; z4|q1F|NhAOCgQn&(KZ*HCLSMgBOO0)fb}D=BgP*~-`(4~%>@fih~58t3OA7uXOB;L!qd3aJh^Ya|KB`#IhnyyX%osJ)p`6WI?{NzgN-&8-E zZj23Zfc$v+Tsre}=93ow!!(tjtbRFG74MFvF#g@W^jrA2byw2Ma0zDaY)%Wm-ffr!>y|dJ{$BMX5-9g+f{wF&381aS6%3q9`7oVZO!l$htikD$^j1Rh4{b2F( zI8^)~&J#aDXYSlZ_mV$GucxoV*5dJpCyBp?cjD{nX3;m{I?R3Dt?K5Aufpx(HL;NR zee@js28&~9b*r(i_J1Wjgno@n6eWw?)1WPR01@Z>xJ4 z;}e}wS5Le!{?G5AH{%iYTj=ZPt1s|UO`J3t78&>ssj1;e^ZW~=5hl#Jn^RWeXRW}IZhvwesIdvb% z|61nyQ}6TCC-7VN1g`b{OdpZCDScJu&CCy(XQuf+=X}jPoIWA_PiOm|gXz2DL!__E zxpSs{$Ebf4GhgO;lldp}*Iae!QzzLczC`+sJQsi2e`Y>--#+OB;tS+_%{h_&BIjVv z;q)mvx6>D-4~;L9ek49Y`pul9*LWX`{2co+^UPWDnUB&BWbVk^GQhg@otaj@$|R*#J?6#A2LKdbLAX5ePH_QoQEZ>|D(Fy^d8(Ho_VpIc+Q{r z3CG0K2c^GF|MjB$boT%mu9}5 zt}cB_3p(>s`qF0NU9pC~B=gek;_=PmJFgI5=6$9=T~5bW987z0iZ6Jb_!xEZqvGd$ zB;HuvQOrHXX!=KW!!UDqP5B2fefuBi%vBBP^xv8L&a>Z_I7r&xP&@>}s-@ywOY==c!T)a}L0eev^^|L^}d5TCq=#H*Nl$;$RwgQMj) z(B0@JbRoJNZV``vxK4Zz<{qg6T?bF#f2^N@4e&GhUf2M0e>+%Re50}Qld&n*!t>R~ z=lH#N7xCeA{G@5(@%ulJ?}*RJpQO9vZY+k6tBbGu1$_xFP)8QL z#rQ05%CEMzme{Px&P`T|Csn(EQN1k{H$H- z`-)e`HMkh_cMMhN!q%OQEyb^*>(eLc--0J_>SsLl3H(|+Hnf2Nx6{Z8DE=~MDNOTU!9A^pHY`!>V)2I%%N(7)_o%v@r|BQ=5zqXR zez1*r`rdYQ`q<1xnfubGr5{P(__TGI58`jemrehdz9&Az%hv724H*9{{ZQtDpM75G zr{l|HPDx+)mVMHv+-cna%$%}B-74|S1(%6`D1HpnpN^1E9~*yXrub~U2s>M!zNa<) z%f2xE%RKAi-^QPfAD_9avVAfKq~9+pp8jedeU5#b(tpt54rtQhc)bEV?|Fz~b^ZViWPJ@dAuLHcDOQl=brQ;}+0O zajE(_I7Ykqo~w{)%3PEiisj1Nsd6wZrGon)`$o#N%`3 zelvdV0r`Vi(f;f3EOGzY3l5485id)(qMOrG>7LjZAD6E|-%rP1J6k+HM(%}+iN~+_ z0}Tc7$GY1u_f47aYb(53zC5pDkE7h;2Z=o;86<7p&VS8L`-Ff(i_((eb)Nt`Z z;$7+7OH`mglb=GLpj*&$>3wwW>z<=K%D;l~Oa3AMC;U>rDjnZ=B%S-4Qgl1@edr2w zZ@f-Ce(00*BAkzf)#u)?6P^3D_{sOF?E%Tj}rfoJqf&=W6=R z_$fK3GY3BEIdY@-k$FCI(M0i_=Q+1Czvp?JIWK+1nbu9f^l@L&@y-5AU+aBje$Bb^ zmUw4OKQ`F9^vhf6^dmLZ$LApn(if(0O+S2G!3d1sH<8ZynEv~pUN8OYPjvi)^a=5= zzP8UE%z2l&F6VOQjr29g?Az43%vqU3f0R#un)xjLPx_8l>T>R+FaMYLG5hz!_$+tG zPsO3~>HklPr=Ol7pFSt&Y37?I@{O#EkCA!g3GwvptLPrqjiDc=U#D{}$Ct{yo^w9^ z!DRI-u^RTr%#Y*kpMHOeeCCS|@+&cORQmJu(|5^dZj2vyr~Tqrr_Y-$p7}C!*n{Ff z<8f?f|0VQ%yi`1W(0lYw9H8!Wx(b~>FMfIEoXqPb)E&j!txw+;Um|nE2Kmg}JLvch z*U+VLI%Zxf;&rNuXKqZN7hiF#y7ZSfs2?Z3A2(zA=DOEF^X$A7FXpFThHQ|8g4UN3!Y=9BoIqpUxK`!GK7i`Lf^&%6@9JAGqib!+fHxD+!7 z&7=olTO5n=F*AW#X^WOX!|-e3QHBws?`c z_&a|Ue;B*T-%sa0D!$Qu;_=V6s~;ggl71Cyh-dz)$Gkj0K zf$o~aXA*ptMD{zV}tm04d^*o62Ddd44&?N<(_De zy0hiS(+|*HG50O~=|Wgb{b9`gchrvcUaKyC@?7zTSRH?l7h-vAWqoVaO!6~^$Gl1K7pLq=__-txAFZ=pPT1I`q>il z{e7=@V9xE%be>b0gY#V5Z+*_WbLh+$>Ce)yZ?a#`xtw3i#WQzgZppdRTz(2x^*X0x z=IBf0Gk3;cxLW)>{KmTU)idezkNfEK4e6)e6Hh<0g3h`2iv81v<(%jxp67G=%giS) z+NYAb%nR{*>d0rlxt31~l6 zeJ}YU^y75;m7{d#`a9`5>eELr6;HpPzW4(1GV0?qr*HkQ_;FmTKK<@adYya^I`iRb z@lxWMm*N{`F50Iq{dWy|7sj8-d{o6Rz+n0G^YIPZd%Xkl2kB*WP5fLu^U&wwjc|l~ ze2>gWW8~A%ucW8p)7BNGD`QWLFP8rE3GvKBtLRGhi4WG0&K%Q1-Fhr0KaS4)I7PfO zmc-n*9H28drO!S~{3(nN5dSs)-6Z)Na2{^L_^6xd%rR@c&&(;K==jaY)W_HCN9X=w zJDs_+hxK>hUGnAWpK&}+!psGas?WSpm;OJz9NS|2ji&TP*bUUYq}bpIq}c*T5O^&^Y;?*7UEUu6_|UlVe}|<3+UVjloLNBUY&jk z-@>x;@d5UTZxE0FHC=qN_+}h}$FRP-`1hHE>xsXQQ`L9Jvf?-4mEr?2_jHfZpWtEj z+vxbZ^XLb$rQd`2z;9X?pMorS!9QwY?h}gHryrKW_@aI30oGNdi6J{*izjV{7L*Et`|Q{=RSXicuVo)bbOL7;-8Aw#>=p0wBL{TlGVibT3;IP z6>m)6N5_ZnO`j(pA2s)5MdhEDuSlPy57F_-hSLpjx4Mb+T)Gcd!Q1d|T&I2{{QzDq zeg?f07h>*Ta)1A$_-oen!GB{%j9*?I1cM^}kxJ&-mGS{Da MpQk>7-@+&G|0YWOpa1{> diff --git a/benchmarks/perf-tool/dataset/data-with-attr.hdf5 b/benchmarks/perf-tool/dataset/data-with-attr.hdf5 deleted file mode 100644 index 22873b06c220b5cf9feb60c69295374c50b70ff7..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 306896 zcmeF%2mIIb|Nr}Fmr5xqG_|x9p|rPlO4)_Tipmykdt{_Rl2As&9z{b`HX$;iWMxET z=3MW6`CopQ|G1oUxtz;6=W;IRe12}<+x_u)z21$_=kwv0^V=7w_HX0=_y4E916@0{>yy9mpnv|>hyLT$|2${^ zlm*T?b@*wg4;?Xb)IW{Ur;Zw(IC|ue;pgy>(W6HWA9}v_e{T5qxvln}@_+c>-3k=h z_5A+lfAWOZ-F44yyZ`5tr~LDYLI3Pgy|8n*JzSzH4?~c3qoBZdyP@urr zlm1zl|BFAuKYy75)&A>W=6`bUzx%^Qh5siP|4XadKl!R6|E2l=!uWsmx%ziE{`a~1 zw=U>;$+J+lTGjvdTvaSqpyCz(d@ugj&)3++|NJN_{nu~X|M+~J{$DTuU!MQ>z90Xt zuYdpk|9$;`;`_1opYO-i|N58wfBO6JU;ixsSLSSvzk86r;|BJyx8&P#oI2m@EVIVQ z!*V;OyVinlzl}dILwx}cLk-zJSD>pr1aIyWM!70qjbreu`UJk8Z{%4x3%95%_`Tlef;>mB81|RD^EP~juUxN! zCi+FW6wlNz%=vm&Zk^0~)7kuXuH$K}*834nV9 za^B2y@tgcHTFQmc)@R>Qw~@Qc%lSCGtGAV3#cX*mevUuE>GEg%IiJss;G+e)sQYqT zl#_qK@p8tRU*#3J66-JoBXFc^`}**n@>}u_OqD0$4jh6b&|ZHF55r?PO#KLd%`ag+ zI;%_IFVw>~dKGvhI?Da{O^nC8sO0yp&68cLhTYXaaeckd`FXw%=cBGVFC?eP)A#{y z?*9D6*U}5vA7@}F`st5#Pal3vJxE@OqvhRr9T#%#Fnp(O!6nfX`>K1oeiy&Ohx4O6 z3kS&6-P4$#Ro9Zs%Mal+JfYVEXX9Yk%HdS~Qg~jkG}q^xFGtH$`DGr#-P}7{epoIo zx5wY=>(o!-9(g~#L$F<49Z%?eiAor&ZpIh#7kn%h;2j*{Gtck`Sda7E^9?`5PvUY^ zS5M+s(GcZuzutJBfF1H{xC+Opck(Sb6ZQQ18@L+wa;>-eOCG0gDBmqlllSG<@sj*I z-^+b*A`Vm^ixTcXOuaXE!FG8d?}i`bYw;82yZ3Ot=G+Hw=pDijbE8~`|8`B$Gq{6$ zPv$F7R{q|_U*r#PGxkF@{ZI5S!X4^sxRKsOc@Oz@9Em@1Jc_xt(zP-usvf`>aBIE? zom_vKFTqkArC!EOxhcPiLRf-ZeeO#Bk-z7CxDJ}hgHTJZ?{nwyi}(~L==b4{>B%4R zYy1PcV=E5SYw2@cZTj5E(tloh4T^&UBe6=a z9iEXV@JU?6uW=gJ;2PY9d*DHNfA?IR`_N3UIrqfRuARY)@sa!&`eQh1yRSH|moIa@ zqP&QYQrARVxh4*f&x>6@T<#^G!Uv&@-c5Wej?%A!RdUXpr8rC8hU}M_0kTf>bIO^Y zpZ(d$GrAwL=CW7iXOKNJ&-rT3{&KFoS2H`fh=&-SIpH!{vUQb956oQ0IG+{bet1%x%#V zzpCHDZOA-Sn7=|tIeWle?%9ldIyk)V>+thQ~d?JCuiTdORk9WxCeROs_4&xU_ddas z_;9qBOLJy|wb)nAny7^1)!AdRzCKjvTq(ng_&n5*UEVb(W`A0t&U(*U&Dz_8FGOAW zK0GS7#xS`qXK$;9X6o!shoZ511d7UKIs0)1zsa|JvtHJD5BXTMmb>wK%tcA{8oY)^ z$PDT8_ws!ddqnSm*D5%zg_QKWw?>zeXfR~ zcu)NZKg2I^=8Kc%ujRL}yLznpc{%40>SH+XJul1C zkaOvEz7Dq|GhPMv%$AS9>2l73tFQ%C^qxn~zAbzwGEcs!o{5&|hV1Y2_zbLbPu}$k z$$59XU!4cL30*ATgpXr&A^jKS%)bq}7SH0$pObkpGW(p)KOi&gVxMaz&%qn=G(3rd zcp42*9W`CgEZY`K)yve&_*&eFk?N26ModP1tkoOAdH4L4`=c28Vm|i4D)-!n_85vi zQC0sUE{Ct=tI<);3~{R5MxMc=_$g#29L)v!c+Oeefq%s3=!6ySKO6TTbL0J7hcoy7 z=KB8X{qPx1RiDpyV}d-EFXuDS6Zv<8L3;1YMdYvf26g7SU-)kI)yPcNoG-yS`W5jq zPQ>^4LVqd-$fscvPQjU2j%Kbs#h0Ord>J~(*K=(?lryKk!4IQ}JY4?{WUi=&4f=g? zfZi?qCs*hEJIF^o42zJtHSY#b${SI`eXFn#*P)(X=9a?pXYyX0+4%$cD0y#m#yRSD z`5XR$U*KEO0@q@@-cNWCzvEB6J@^E?ir3Uv^1-};PsT&&?)UnF5755@nRCzOqxBxg z-|_-}6F1BE@~8YJGD~NU`athnG|_8^-Spm3x0N%8PLwm>UBbQf#_%V7Ub#3*^)K!vYcn=TkNOKvz}+GH0S4AhOgy3k6BlFCXV9~+z8*vS$j3*9-NtP zho4mvXK~KWXY||i3j8kTx&A>uoLgW| zIeTT+`1|VDIBV`mIs4|G=!yP%SsU5&Guve!$bLGAk8~}2;~;!^KJIbSvX&B4z9$F$a~;W z{TJoGuuRSjy1QHr%hf0HugG3H4i)w8LFR~@c_s92P*;<$=gbiwVuE}Sc48r}L|d%F zBd%vQ?IeGQ?9G*VJ?FfBjPvgKDbK}MnC*HW%$H9^=AiBTFq+FVaD==Mj>T2DLoe^Q zYvugg@L}rO@}XQ5@5z}ryCP>u2fdtUZRL;Ti_lHJ0(;{dWNylN&`;hEW$~H&9^^~8 zD9_~!u^M~eOTCF)9_M3%I%msIyjlGUXP(P!uu+}!d$ZnI@-W;b-+?)D=F7~7Q`9f3 zo8mP2Rvy7$<9p<+&jZkfTB_^tI9w*z#bhi}7vfcDflqLS-VlBkRnQK_^zts*4#VYj zdi(M~9>b5K7J91lE_^QMEWVBp!~0l-*6x|g&9T4SfS=O?sxyyObbTbg zQrA}xkk?>~`X#Q)`8SN?xRrVx*W%84Ps=m$2wuc5sO;J(F2Fl+i2EK^cH*Vn7&oCX zE_A&s#^XWtfj9!q)W`BE+zYSEx8qdYhDmxiV4z%%Z^u)3TU{3Y@d5tSdkg#H3w*3s zhpY3!JOhWKH_q0Zf(PWAuwA|cJJ1)~&{4k_@}5vzT>@_)?+0~I6Q`-)bk8QfmH$8? z`A1&Duk&SG1V5pTx`xl?9b%Y#Ao}Td$8dQps=M|%_erh462Hjr@dmEP&+tTCC0~xk zs1STM|JIT@>_l~D^!B{cy?bH2yo7fk^F?NopZ7N%Gn3Amh#MHR?57Vz2F(w^GxN8xKurt ztMf&8Q+^cnQne(w+9NCXQ*DEj2!92W(><2^n zMjp)BS6`D~GY|2%+}1tK<)%0kJ&`s27cb-&_(|S^>@$a`pO=U8 z4CMUD{FwbGd)$?-Wt~>TmvRR#qJKZ1jO;1j@n@Vhej%zL=V^HqmAfMQ(;PmGZ{cfk zqWleKZ`=oM&_QoMWQORa9*SkiJId|+7Um-}YkHP#{L?@nAVx57X?hx_pXCSarMXW&(2->IaReY`5l$onGu=H==u@huvmfZi-* zE^N$`wU%Hdaz^Cbd_ex)wGrF^Td8gIkja^81x)~#3nj9w^<+Gv5z$iDh8I?0)v z-;9@yD zoTXmCgOC|D`~JRqnVWvo`vo^+vtHh}hR6@0E}H1=z-%ZBoP;sL1$+w^=UZ_$4nzyg zL(cGH@r&Gu_rWUMpgx4Z!JXI*BQZh07`~H_!glTbLZ zXUTaF$}IAP`X~IQHw-^vBf28*(sTI;*E45kp8rZcA3gLA;)D21E{P_XuD%yDLXtw-^%UOck#KnDK##{=cwnNx~?B8zmD5cSnpsy4x{j`dUw78)#S`uqqze< zM?w9}Pv3Ipu$;G9TP4&h@DVy=2Yz$^@4SZJ;*p%QvA8^oH*pm%!cTHDe2R;4Bfi$p z?DP)zKxT!_oH=KO>+6v{xgBSIX)2$Csj0bbuHzcytb9W6VLlBP;C5WDwtb^TACfsf?^Sc9Am z_banbm*5D!hP)H?<(#9LIoGK(|1H$(Ew|^k>g=OqFD1RsIy2Z|kbF zk3Wjqde7k=xg}Re&dnj|>n-KW`B^>?$6z(Gp62M^i7x8Q0dJwMynweLd&G_E4R}R; zCf>(q>dE{DX36j3DEThjhnec^WtoHTluPTK&6!L5w`-s6$m1{suetU%GJiHvA1P-> z|B^3KXAl39%W&qaUh*iMq<)?sMCPTO-d#TJXgdZjMqP$b6&5L&y^p+RQWp$LthNldkW9X{@bsW-UxwA>uu<9?j4HzPH_!Y}ZdToj$<-?0b>sWb0=EEh*PbyLonR$hKn z&ffikJQ&;5HSwTa0k6qda&!KG|HKxQQs08ijQgn{lC$^C#9%DYtIf5zAvVi*W1D;; zy2@X0=9|OhlTkzc6>h*F6w$j3A0m6?A>7IRIlHe`?~T06ROC-^o$FU|p3{x1=8wJf-%wx0k8| z&n8TgGv8jvnL9H7FVo)x)%5;C54jAM$z9O_8!;JA=wFF_<;-S3$VcE!bsZGL+v;yP zGs?MqtokM{qHf2P@wZ+tz634hah&<{0l5J(qur}_20Gza6w)7q%yoUaFy7a{8#l}6 z@haSfV{t7`MSs^ub7rwqIaBgHeuT_jcdD=n(B zIVI1?q4F!7=VvyGAam;-`kUk(ybljWdyK=!$ljARpY>gZ+xkq-+O_h5dY|$AcvZd| zMPqgLfb5Ssw=;j<#dG!c^_SrV zWR|Li7V>iUEaJ{mH^E6OGGcFh<3EmrGg&uJ}Ritg&0FbvnIKjFMH zRN$OPy^;MT=f`Jw8;_y`K165NpX2BGQ=EzH&%f*KFVEri+!oo7ZdBLhuADQip8Sja zH`<|+`b(^npGV%K+Nm>t9m|6-L;oml%V(jGocDmE`3DR@ZT;Wyh+LAhPyU5-uv4!( za!%ZaMS90z4ce%8aAt)M<>q)$eLXtrZB!SQpX46=9AC~0@fLEXbVLLBDxQd+WFOsi zeqO+jt6xKA^RDWdxJUgnXAeCC<>c(KTlp7^!n5iR&=GH-6t2_%k$3ZJ73a(Xzv+#~ z0`X8)z-#h}ob#<2zoou~JE&`M zMRk3QkZWGYA$D^*_hj-@Qd4@_ZGx zqnCO<>dAlexqK5pg#mu8M(U^KF?blSW1HUV$ooi5^$GHMoEh_J`AWIH{0twjF2i{@ z>5MPr%)}?~0k|0@)Xnp2pp*Qd-YF=ju7aQBk8n7OBJ=Bo{JCof^C|o`*WsO5C1-X! zOg>1?td&_T&&xbC*8dGJ$Q`&Q@_go*{$4L@I_o9tubO-^=iI5Jms#Lt-u1KRtg|7m zFT-Kjq|QE&XL|*v>OGE|<*)c&yoBscS;v_j@~jNPqv{WlpI5$LdvTuEEqYJLS%(vF zC$b*0k9;D(h(^dh^d~Ps=8gfJJ!&y>7A!>#%u#0#&)Qw6z8aaq@~k$L3%b@H`B~SrxJg|%|d*NyU4Ip@n- z`D0#>wwQ$M=kM#!lD|e~u2a?LAp6rJdU+08%MT*+$ZhKB@;N+#OK|qWeRw8rm8WqB zWFPxX{S;>{jl%B8+?2I{hWr6%AKFu%%vo11?>ejV+@B_Ajb-o4Go0r<`%3nvA2@Ts zKDPm&+uw~;-2D3`~(a#dczKk;0=g5v5@{03i& z`tps)UYYYW?=a&yXG#y=L%+KG8rM+Y%2n_g=BwLqJ!D@Wtv-P>6K6lpJbs$q4XCf5 zv$!xWSLckbtk)T*IPaRL^U=5pP4q`|OTHKVTtAlc?)jNefST&h*QvpxCCda3*rHsuD+WyM|b4E&<(>-5w-EPdvXRIB@e{m$a~~=y?!W(MX`QS{paQ5 zlI!G7qf{|e^gAoW-@bl+Zlpxz?1lKbLYc?$N&W$2HE`h)Ph z+!v?Fd-89nCFkD?%E_65p5tD4M*nHPhPz>3IrBtuxdEP0*Wu&&S$-3f@VEL;?!wDB zvrPUCpq+dzDk8I3N1lpX+;=5TmJ8rSs(@`6p)Ta!Z}~@Dg+!vuxhea>-RfH~1UU;Exi$o2kbNR+ zK6~{7jFSK4<@iXx1MB5HPY0lsyaY|;t=x>?`B{kCqBT{n1SMGoR5+7 z{eJZxn5ACG_u&9!UdTEgCD%sQXIo@1dyQM_eSs&D88PeYQ)JIAi>$d0*bP1Ovu>;K zBUmgC;#nw&y-*n^>3_^;;ww4lNi`IaGjkQze^kz%e~!F@YvOJ>>+%P*bzf$N!TLGN zrt4*9&g|Y=Z#90`%S`*Xybj-~&)^sNsUY8jX>#_Iv+y0B!&<%UMLBD0%9*DwRCmEW z>Q#I_XKz1&uU6MXcYK1(E+zDz$J+GOjd>$7w{6zXyi-v9gxm%>^L|%nzIua~scRtf z-%@Ud%K>7i(m0Mis@%A8Y)+pzr<2JhKYJTcp;v^CF:`IP!@g(lV zaJ}|C6xl~Q@rU?XedY?L-+Mhs3_mYJCIrbRc@|#2di`Vpf^ZOVn z*We;N0hc2)$K`tapaIU+%WPAIbM9ZFS4pmbCO8p4>SadAyL?A=F+8ld6MgWQ`gYFT z^)&yc&RkO)@5#Mznf`hF6>gRfLVe7|L&!VbjjmmT%#&y99g3X)`=UJ#!6FRPU&kxC zC}yB9cIf?rAF=D-%Jja+r-LfXLCia!)-B89*%l)DW1>A za(S-9H}iR%*{QT#NPdoQN8USnsh8t6^{IRWU%}^`Xk>}%9c_T72X5PyTR91b0 zYniPslP{3}!ZNg0_vE*E8#W>Da;tcM{o?2=FXHj|Mt%b?$(h?Oln=x+>YKP6f5cUI zKa7`0^FdsVf5oHnVcZlF2aav>ay%#Zormq1z93gCBqgcWG6m35Q#^dMRz z&r$ZE&-f3FmGk$?zL=lS9mqN;!TGagKhO6(-|IXFqmg|jYxR41E;=C3&PB+Y&VG_L znYB2I^9;NwXCGa{`FZ8%GZdLauGGtZlr!oxIp=Hkvplz7yC-XK5WbKPzzpQ)caPqW z@=DHGn!WK`InPYi!4za3$j>nQ>k#$3I0%h{YuT&%xt8ZRXYZ4o_45ed%Gs;4c898u z#&hyWWM+Fx-Gz6@Wb9DqneQ!UJ=c-f;466qvd854%ii%GXU6?^eGgE7>z+^09mCZ{ zIOoH>d`7;87jkCjoI7Q(LNCwtB+he_^Y$yAsF!`Yty}|H{B4nEb1|}aRpE|$nQ!it zhhRVCIjhIDF#|b|vUlfs%^GZl`;oo$KCY#ob2RI$4c1>NkNoDBp{_k#oE-Z$(Mh?%+9SDzC*2xIkSE(~+5duHGY@hfPLmgWpyz$RUe6s_z9i#KF4)fu5Qbt_#Bk?xl7b7#490otO}qi;%C$J}+i%NxUpYbj47%W9z58*K`~~tZI}oenkC6A&My}t5 z{JHb{X3ZaqtkYAFJuqu7KmR-j`Pt|B&)+HQKY!*2)E98}ocxTps_#O6PBW43aegNE z@Z%`v+CXlMd_O92Wwep=z3;&v@)XoSp07O5`TOMOS=aT&a`yV10Xb)~)<4nDzBX9S zbJtZ`+Z|u)`zP2Iz zN%s287}-B|ps4y2?v6a08;d}SSEU3SvTp74O&oS#6R zy^}FnuRLFYJoBy9-^-bq%5m1Ixa%A!$=QpqM9!Kz>dkmeJ%GQ$IQe+4juM!P?7KO` zdhxqxsh4y5EY3Q5n={KU=0nwmxF)~A^KrgBB|Y~nY}k$rL~D!Sg4vp;7KdKLrpHgM*m2j%Q3bJh2vpxjo!mRw){3`6k> zdh4~t8}jq~D{|(qQjft)>Ys5RnyGu@D7>O>h-c)>F&~9}Zkk@^^~O9AC+O!rZps+t~KTCNsl1!C_B}y@U?m@-@u)njp|d^F$6Iahz;?3rKU8~m=$Ui%&Aytz{Di?h^k;#y=+%{$r_-i9l1m3wl| zKQFJ~Ke;%%$T^c9;+%U6^vWae0MqsQA#-hCy$Q%UyOam&zlTGR*yp1cP);;WE%o=WPr@)3LncR@+4Krg+^@TGh`kK{hw#OH43%ux61 zb;cLSdvj&|);J0kafkl1n1mMiL$4}sKudMbwEg6c@&bMoXQH$Ip`7=vypN4?{a_rA z*Y)%M_=j9P_j6g4mOtjq@lCiK?n2JYQ#oh#diOjcmy|D%Ge76uVYpu2<&NW=|3C8E zDB_;?_&OegbMYq@=@oEa3pxYKQAnNHxwf2nYCOM#S-25G?Dk@ zk^CSU;V|{7oOjtbxQ+T#ZqL1uxr%oE`^hyzW|n$rr}r^R%ekQg{ldTVO}vTwxpts@ zxx9g|8Pw~F|0gB3NFcl?GO0O_~&SztyJe(iK3~W|cKr{Jtew@d1J#?0P za4q!p=NPBXd-@XIg41z0jzc^5RO1o+Ef3>kxFGWU-N%>mlbo3$>m}dwoICf(d44i0 z^w!&oEAEr z)$8cKujCVvHJ3g6MY*5;&B(L#E9Xq;#or-6gR?k4gWouNLiVCNaXG%ne7x#fQ_gx^ zjXdKg=vC*DJR6zkG6U4tDqs)al zN1w$h$o}>+=ef;Zw#7Yzcssv~%$nIVGe2zQ?9G{7rg8SIN}RK?z0W)-XFgcM-ywTg zQM@A;$2>fyex9=jPUoDrH^|wqvi85k(>M~1^k?(&Tn5>La)xBT&6)P2oZ0zkUW11) z8Kv@h&YC$vp2Rsvvxax_)6m{EFTKm#6rW&)oO$45IcunsT#);thP;mN<_D0qT7}p1 zV9vRab(wX0F~-U{A8I0N^*(hw_w36jr6*^99wB$sD}v0n-8p-4X5Y*Y*;`BE4ZRn5 z1^DDQO@}Zd;=~(dwh;oxW#?h7p~z) z&{-~rHE4#%@hpzP+c+J0=gGTP_PbxzndA3DS-HC2)u@bH)aT(ge1zTcyZ&b=EzjcY zX@AN+@xGjM>LBz;AFrS-j?$aM%lKZN86$h%X8uAi^X^n$q<$Ymu}Yo!v~DtTj^+H# z**V;`5A<5`0Nx+nC2DyZ#qNAM~586Ub`-LP3M$Y*jd*Y`o@x@)KhRvh znm1vh{0TDuWiA*e?~SoI)Agw+ibrvs-UEC$7NR%4zy|%CAB}zfH+3ml+%@M^;y3lW z@N2vrZBa+P2fmgo;~e=`JSN|YpXAIKjpesc7*o&*6I|=Y<@iAim$&79bdWQb-zHC% z59JGaCGUm1kQw-EJ{jZjjqBI*9KIceZ>c@2ff#@2q){G$aDE^K0O)Vpebsju+Q}5 z8MxnlOSmhajl3V_J^pX`SiFvxP~JTa`C}f&OZhZRk~iXE9IP&aVsdxR%ykL(;Ih1* z{x5tS>gtVG-!7lZnI#Khv3jCD?sfLETPw;4qwxfjHN- z4Y)}@5QXH-sfTjj3-;A}Ms9W&EcBM=bIyhK za-P4AcwOFxvoIc+v+m=ZyRA6;YhQdPPe*(GL--!O-%%U~sh9Fte2!D_6|$$*K}Xjs z;z~K+R^)d3(n1yrIV|X@yiOeQF)%lrbt~*S=4Uf8( z^K~b`r7oBr9>85VA19!b>)9Vl;W2rC&NEw7&i;2NvL9qWu86F|%zCfz4V?Y$OZfrL z^Vym&L>qKdAJ5Y;1dph{MGXu_=DJVxb4Hd?XAS54sVo14M_kK!+Ew0xtf9lX0{X~L zVw+r&|3Z;u_x~+V!F=4PH<;`5A3P3j*|pc@tT{seAml9Upw1rq9v4K;lFZes<$PuS ztEbl=N2)L2Vd-iPfNnDj@aQ55@Xn+=a**EUtCAnxHi{uyN$I)Njm!ILh-<06&U(e|MjHl%DakSinZ^x_h%Ule3$0@JQ``vTuBTx$$ z;ZQu`TIP$ZI5Tn+y@SylAM17FMtB%+sNcih=&Ejrt*EX3n2X>EtVMghm1rlgLiYdj z)b-@Oa2Xy%=F*%ClU*w!XRajLkiG5#9A>@=8>dyKwgK5nLB1BInx`uC2nW z>UX&TAD}-(_TR3Vc)ReD+>6iSO?V$I)VHFid=z%`xyRIh;!}KrLRgQVUC$ZVNWOsc zuD83~68+^tdZ%L`4%E9CrR6nzG=Gk}(MkOo@5bL?Pu!^f0B_1`csQPu=i)OgQ{T^9 zaGX4v@8v^MqY6I6fqI*GA}_}osDimTKz|f|!%ym~k@t`v)a&J(+j;MNQGL03A(uvF zc`-kP%qiv6XUi|)Yx#RDlQRc&;e69O;1n@){95@=`AVEC*XB2Hjy#kvM_UX-=9kmB2}+bo7=rBedB!r!W^H7q_(QJ}vi=@$|6jNh3)QW8FlS%A z6phra_*vxK$@5lA&e>F+^NennS8&eJJTrMVR_NvI%YK$;y9eLSd2aJ;ZNjm7d6o*w znWZim@H?X$eQ_5o&7j#?+RqU&wkbbS!;*ul|=T!zUYn2u6yz(%tJe5&A)m%8GV;a9y3C%QHT9CZt^BsU7rG;JeKq~Pg)MjynL%&jV=>9~e*6uO=ggnK$`w#cuB0~vcdLIv zd3kUC7|Z0m+vNPnewXv}X8phA#hmkT6z3ew-dmsdRcCL#1BW2*79Z<(!V&6~$l3iJ z4o1$J6ZH0wKSMX%i}lzCoiW+Nj1EV3uJ#I1N$y@-$Jclj_phMa*jc|Ni~AJ22~ zDr%yZ`%B>wc`QH4CEV8u_o<&!zrf@8QoaC(VVb%${zm4{6ZL+;_xM8Z9-NIckXa&U z=yHA(c}MDuCGLGsJ&sqQpPX6XN8ZX;;&pi{Uxrr5`%o`_33s3muETPCjicTF0iVPb z(H}Y2-ojSo?B3J$hVti}S>-g&8J#(3sD3?O#{+O39(Mf`evfm;?k9gFf6wpWVtEZR z+t$c6Jb(#$nQ@xQZ(x7*wdfzC!?I^8&j(^AH_!h70)x=ti zP|xR6cqJFWc)AEz;tI6UE6Zc?6EdfF;iCE-IP>|J^3U?Ed<{m*m*5HcO|+M*^CdVy zz6mGe2z6^d2ItAm{92hiGB2H@J`#&j$n_?;Tz(Mcu)q3Du7?-pANdSqCVi5t<=4=k zjT&+zwd>h}-%eaSYedPo3 zvifSAi9d0OYnjvjbgeDEz?=Gq<1{QqX4mHWd&r;i&G<{+P5&r92v;HRcN;kGDo44e zH>zNw-f(o3TVRRYlNaGs`9ZA0uCs~1M$VGwIWtouzSg~i`3dAqn5WM9{JxyMC*R8! zcv)_Q>zJNaVVkL+Db_43?)%LS2j zc0F>2meQ|=oI_bhd5$voWxu>d&R(DAsXw>X%bv4Fo&7p{`ZQ#(@2Vauujlrh8SHm? z0WyDO9@va(t_|c5Fb`YQkMnAN12Zul`PtlvkCA7g0pG-5^0}O!clP2<$k~&%ah+>F z@El~mTd1Bbk3ycitFZtpFb1Evo^|pK@~mdP55T>0Ph`F1c|Hqyc5czj`mBK;aHHNG zJeo7dKF*Kge7(%nIggsCYvK#NJF!j9`pf=yoVpW^Rp+dkg-Xa=wOqdqvUjc0`wsi5 zbGBZ}-PIj=AGE|z>dd>n<*M@OI7H4qlJn_PWKI9d*_WGg=B1pmd&*hs&oceD|39zn zJAZ5JcdEzXQe@9~S8o~$BKt|sgzUTd`kn8?k@5u2K2aA9H_aI|~^CgW9P4|@pHT|0?K z^H<1TKa+C~=6&O5*Vd|kL(aIIYdz%6T*vjbd>i(_8EEHPA0EdibI!!%~xLvQdYd6bR%9Hqb43dw;*?1m@=~coX$o`wNG;_=s za^|l3>UYo!c`tldzaO&yR@SS{`(d$sJsM-I`W`-!kKxP-_2rG}<2-eB&KdTjJXp^A zR6As*9Fgm}rvJTsAgZY=t9Qri>H~OdG76!dUS@%|Tn;(M+UV~gKZ&p9Wqd7OK^OHf zF6_QKe46?`Uc$w3hMbu>bL=njnW%{)koUd|_&N6-f_3ue$c*`!`a|rCvU;ugd^E$+ z>dZh}`CxR^tEv8y-&1F?+)hQ*d6&*P`ztQeyMQ0#^7tsI7vVfSjX`=3qd%@t=R94* zZ(=t*pr7}`4>|8n!}MC>EOiwW!U1?q?=Q}I`=b1`JcL(sMSg_a<7%u^&*9lvCXeAw z=q;B;Yx!Z`z=!g7z6$TiUtuI>rN_;2p*$RK;TUxne}<*W$?2To_yABAAF;>QgxL_x^HE9IL*F$Ky&l^TV4wSG^ve;BWPCu7PK;7xKRL z0B5$_&VS)ReCyuo*oM5P70~+wnSaXh;aVfOJ3oa-%V>gCD2jvdy=yCZ2{J3@z3Nf9h5qF{9+%-X^uvjG%e7mv0lT|DvtTj( zyi@ejn~0y(J9#M2;0HOg{$X-wK82st%M6&=s{>Z*J&U~SjCHM!ob{4tInPV>*8H5? zxRy0G0j1QB@wGgOyW$9ZgY3`QACBg0)Kic%Hv4U6lPA=ZUCY|ZzL>pqj@~wIfXp&? z@Rxd7msxY4p}X9Qa|YylRS<8;C6MQKvN}J{eB~V9TQAS=Wyt>l}l>P8t-lEPr+9~hL3vwOfa0%uh>-i7lTrKKzbvf($TAZdo z7zfBbk>~zh^)zHJoz8jwCLrhiMV#j@`(Qh4bkAVSMb5f!^?Gy8pPZ2m)px50acj;w zGD_YLJJ4TmBxerDy00c@rhJtfV>61o=WJx=sHAQx-;T_O-8ge>13t>NHF!mSp7%s8 zxfI@!XLHWMx4D2i`%W=r|K7k^k45G4kTsgUV?VB`e;;RmI3IgqhTe-z73KrZf0KyoPU(*?f_{xa-}dBF|Jz=Q~hCuH&Ab zJPSFyOX`o67vp*PQoJoM=T|WhrPW{YTz(YU4}ViXz)vIRW#-*`^adbj<9z)q^R z{0Xn({`g&9$>n)3-k%r67%MN~T7HeZS6-+788_%e2Ob-cg#%PofUKMhSG)|Bf?LW}a;7-umiO`8T|bq3UVKTz?~hFOy=K_C+=KJj6%gtXQ4%db~W$y_qeGaQ;o=CcW?EH#xKP z49?uzQm;B*QlE@Rf?Dg7w+Wp*d6ELBiAxl{lo+LY96m&mY>5@@dVkWeFl$075OyYjlboQcv7y(1%2i{ z`9V1|T6bQ8RX7|~QOWi1`7~b3Q}}LV-{{VH9*4+T`+5GdM?8+n>O4OSxF)jxx9Vqo z=lNcPJkN`f=ORD%{F!H<3UWr}S)403mUB+#`Iv;9KL;S|Kl56imuLttof{+e1BiSGC9x6Q=Dggi(a0)?3H(M zXFim(9`k+8xs$Ud&qAK9V>v&w37kD;fczx>K=$Dg+{(4gUjG{p_5r@-{QiIZON)vM z*|an?iHs=ilq6{=MWqr#Nk)`5*=43siAoxj(4e9rqf`jdQZll)@PE8Ij^E$mc|2d| zdEF(S`?}8ed9{u696f~XcvGG8ej@*hi}bROo+r#|=A zmfuD8+1q$Laz_nMb3ByaTzn3vlkW0i5$_ zsGR5K5EN18xmx3zfA`JGZbE(ZP=C$SIWwjk7w62T#`sFkGn8F9_jwaE)nBBS^Ct7U z3V*GaIg)$%0N$*ZbM$Dw5AVqNkC#I$aUw5A?(@u`oM{cz*)!*JcCW(vXJIokH!>JX zxi(VFex8}}B&y?7y_^SoIqww*ac0gr$lS=xI85%1cd=8Q=e{~#lBePs{HA^wwUGIn zxsx61K=pY1qCOe#$#d`!o>rGZ2~>;q`r>V5C;f`w)33l^@^}0`UxFi$^Xq)QlaW2* zW4)ZqIm>d+XWz>|S7No^Cho+e@T~kiUyqG)&V+`@%$mZ@^)h2mlb7>{yc>VZkK$is z?iJ!2P*Lv6`A^<1m&YygZcLXy=X3cZyegl>pCWs~7wUK9YtbK>fv0eGy=KX1k7xCM z=bRgNVw`#)*Fbij>^UR#DsX>w&b+C}+PomajrzEI|j?p23O8IaH1R z<7M2IbH2^uJ?gw;4B|cNk2t%>&nPe7h@0gzP*$G9^{`(4hlgRByplWO36%2tRnp73 zoxP=nUL*ByoHIMS)hzYd>UQXi*VW5-0UnU=;J->?a6Bqsz?Jwm9>MeZA7uB=j<8j} zQa`(I-jP00XU~|-<8Y8`tRvcbZh~*nPkmDQ@+3Z(|KQV*_o;S#sa`?(CwVTW;68kV8LnN#pP{Gx z1+pWb$cLbyehbW%m*PD9groG>h(Zk?%IMAkWn;a^~(D{tEdSmQa6( z+*8#!Gb=MZ&sU!9@jjDhw}^Tkm&d#EbUcbX)R|BD9&+F0EZfKzAote-*D}X5zux61 zQ4smgN9tv^S4K(oO`Ll-vop_m=2`>(7wcT#j6AR9)kEdnXRqO9e1!pe&ti~#0EWn! zGda^|s_V%eFcsgZ^ZYHx&FVbI*>P9P*C5YmcHx}4ljK@xFQ39W^J?RHWR~Zd%3L`b zm-WTt8Lb%LVyv&bhc7*=ve(3!LLx=G4m6a%O$@^7G}qLmbMVB4<|4(yP%| z{swQ#m+;r9D4)gIk51+Z>KXhR-a&TN-}UCo1(35jXL4q5=G`EE#Py=cKiMBA@iP2| z%)2{y3Vx6?V}9j!_*Tvy@w=S4TTh*HYaZXLu7tPb>@Ww)x67GRnPolXx|})rC6_`Q zyr5o+yf0**`d%KaKMdnAL%oGtqO9vjsy~)9R~z%KI80uP67p_77XP5Vx+}kq_vFmb z?bs=o!x6FHvl_40e}bDKXWm}*A#!J4fFI@0k$1AYc$D7pyayxX(cA?)aE!Vq_8{+9 zWBF74-dvWOV6=QAn&LhDqjxqs$vLNThEB#bs&Ym3`EpD71b!b& zs?S?T?gak0a%0VrcZG8j00TDzW$=RH6BJAWS_c>7b82sw?20<`lFoQiF(7i53WV_ zi;{T7_0{U{cqXzlcH`TyN`Dc*fI@gkeHHiRu3Qvn%9rtpTmfz6y8Jd5@_X%&zs4ze zO79kKz#s4f{4$#3K=on#1HZ*(`AQ7N!5D^0e!s%p$MwVce|$V$fz##M{4-zZbKB$! z@^2W4Gx4$B3gjK)3iVL=NPd!QVy)bW%b=Bf5I4giScr4}N&miAbzT+yKpZnnS zF%0?d=f2GMkZ1Q@Wg>*kLP?RPa{8v2a#trXHjOzKggVTPJgmI z0y!%)(~IdfRp)!le7zMpd*aQn8SaBMa(3-{@^bklT#cL6FY>d<-kbgRMZL_;r??d|A2R1Mf7amwb>?h2?!dy|@V#^tRylMwkDLXkyZ#PElj zD13>Za5-{LWM?k#GsCzmXMedt9woQr>(Lh1s)wSX>wl}4pg-#AmBsV&JIH&?|J08n zyGtLQz}c5y;2oHVOZDrZC>p3I@G|VgNyzSS3qS4JHg2iDk$=M(ScET7)b-1-RK5ol z@d#>Qr+#mAkbmLqu+up2_?L3_jY_T^BX>n(^`YvLD5x&Pk8^E4gWu)s%|+x>aUy<1 zcF;MlwUb-Pugb5;e{mC3l1rm7u2f%$yjvcit`+1VybhnawvuxWSC*@Ab*>xqSMts1 zg6zxJ=(R^jbU{`9jTk8J<&iv#3!|OSx8XnZD&rgZdwe3-!QXOr!My9NP=AE_xJ$nw znqi)LI6lSW>i4-MHpy#wDqqc~^Bg_`Tjfviwmb->ko__58wK@esc+}JS5A>f%I{&Q z+zZQbo_Y-4Ko3mSTg;{Nd*C?rTy)kOhwPEp@%1=ZUd5AKKa&S@_SHN1U-ka-2l8vk zJL7SB{cxMQCpXpmiSwRzFn^|>efDmy%=hu9`s?vN2I3Hua;-P#?=imST|RdOj+e8m zoh09nM{$APsn{S7=fyk|d+;+F-~;`qk(vFn`Vn+g=f3SJcSfG&%*fn>52K;pi=6+5 zXXRRQ&f3+S?U5__@#_nk~P})3H*%A34LO^W!)YIhQlnGXvY>9}LsWE;3m@ zgy-??d;oXH0KBCxiT-k){|(4KSW|BeN~y2Gck&EmFKNp;OR|3yk@vX%C$a;5%dIgX zHU8AQ37Oj^@UJ|Yf9EIo3S5Rc$T>Zf|KMR#yF zAlEVWF=bnPiVmT%$Gd;{9aJGmnKvVD8$!SX

|mC?y}pziw2{B&>>yn@d(Sg^hs)JbL(V=}P2P{Q zKd;bh!O!!*IN7!AD@V&`;XGWVHM zrSU5M!kxHFe-QWPPx)+q1lP#@xDxN>Z}5P8JzvP#b@OhYpJC=n?zM4pe&(4u-=er) zzK;TO&Zc~qzpMY@%)ZR2{OoU(^9CY`FuB3^z;9ido|zByU6#N|F`@<-;Ugm`95>c<}Aw3Q`~pH#%u1il!#kBaN(UeAnrO8$et<2;Lh${Ue+w;g#7cOi2od)^Yc zlYVxY%$W=1A)NbTDf0d9*1G|jrFr&p4|e9}7@_w)o|f}Gp3Xbb80CE?{|j+2HX}Pg&WD^mYq&2CmXGA|$Qd#f3sDT$ zqJdwN^Z#UI&Ygw<@*VsNaz-uSMrewoaTf~s%&}N1Kgox1p0_;H6VXx*J?J8ZxK>K;4^f5?ltJ>Eq%WM_F< z|1J4pG{7tP1=aL##0WWaD(BZmeo%cs%F8)ND$8H;Gx$sX2jAmwG}e0^Z_3^IaBjkW zt9@rmLm~V1CF-rn`EVQ;!N>9p&b-=zJt&N3I1tNSo6I+12EIi{Ox7>WV|f!s%f}(- zPxhZ%&<_=Y-uc|0b6#}e{_1}@^Zf;R7!O5VJd4VDwRkFu`!yxigRojX7tbN*)rWjP zf5RovTfTv_+ds;g?|Bb+ag$d^@j1Px)so!6Wz)*~1>- z)A5Q=c`b7G_Tc07&PDd4-_>u+ zuOshF*{_!AeWZSv58-WiRQ``2!Yq7_rYMD0xEXVN=1{DVPvJT^3XiD!;6ZsEe}~ro zJLU0?ygydCc095_l~vz`P3VWcdf89jmzU#ib$etdJ5#+=ev#k68Tdt=caz`byf@V5 zC-nv(d*cVZ1AR~nAEPFUx%Ltd^7(V++vL2LZB=iU^Ip`8SE-xHCvia>q<0mvPn3yq zAv^$kCD6JaWqyJ(yuF*MRv^3 z`9uuY?~R+}{QqVi^hECSe9w33|H>u!WX?aC$(@n;mpM}hzpHc4_LuV=za;19^9u4k z=jWR_|GfToJS8tjc{$JAlgNFLJ#2KY$(g4!VuGbzZU1L zxxuw-P2{09>JM|ndSA>7s+|{kLK)MmvNrAd-bMs zM{bW{as$^hM{<_soXFnNh8G~u%2=L?oOdtt{TM5+;fs+yvIXa?_)|a6=Y80WJm=SQ zp0)A(G;iT|Q2^WUz1|(jd&+m5U1|>J+0Bf)1;=2u`YrU3cXQ@ncK74eUm@=nAM?rj zSKv%}3+HSo#{a0l;ewcjOVoLmBTv-K!>l)Q&u zz_W4@{b`){sC9aI_p6}ZhVht=?Aqt^+bHQXIkRiY4`Y-%=W=$&PU@@Wo;-`Q^WDpH zahYDuoL%x&at-c>%=+s53NDo!Bkvs9L$W_jP-pilsQyfT5g*|+^?!Ui=A(>yHjm@? z`EkCLm+<~*Cs*RW$Qgez*U)>B=i>-miJW;u`ApZ2PsX5(AE*LfL6$}b>i(fR5V z{JMGE3oZ55@Xg!>WnJ6MXXsVNLitgCo}b|x`A%+xRq_dVL#~ID@RNEcUx=KwmGGIo z5%?0@vvM#Ci9B zOdc*jghS=*97l2XuuXh`{=t}p7Wf87>;KK!A3otR>M|IDBI?7CeYUfD2R^}-_!#H8 zRuLb_d$6C+y|4aMJ_aS^LCE`a18$}_nLp%d+yo`%vv@5U%B9g(?v7t%+V^hsfG`pR z)TeVFz6E(tY^A;eSF4NTD!DGt#oKakESCFlH@*|q{kpyUrrr;jBxjc|$0y-Ly}T1Y z#XlqO2fg&am78--^)Sx9f0tfEoP+Gj%k=BW2O;yZKTbu?-OTX(+;7De`9sdm@Swa) z&KdbG=UG^aYI0^m&Z(Sh9nI+j7vX3m*%Z$Gbx$oOxx?c96`uvJ|InU$b+!x2k z>YV%U@(y+Gx!>_Bo<_dE3CO*YpXpmT1i9BQ_G>QV*~mSUd+G-M9+}Cj)wzG)QRhs` zynRakQO+*%7>~erdYR=p<7%TlwxK*SzjH1<&1HS|d(4zyM@e}BU&t+ynUi}W&sgri z`lzX&`?so`vn_k(xpHRfUC7+XZnDj1_Mj7Tz6{jQS(bb5FFCXGdgT7D&P|Xr^mabk zXL7HMmvdg!m5;-nD4mYl-@)Cu8t>0VI5Tk#x95eNc~A@$E5rl3WgRF;(4^r*Y1r zdvUd#Sv6B$iW+j}=uz^qcp1;@HPCxO&Mq-deIzbXKf{ZV{d<5qv*B!Y5zccuLSDti z`5feY$oaiYZ%$C(g9Y+&+!ph(LY-Z$n4H}*Gk7QJsk`t^oSFBqob&$=&iu%ZkRADM zbx-7MtHo<@kvs`U%D?kgOhESC%*bQ$plegO2S0&j@?g$;(GboYYOS{urP*YGFK zz=3!K@1cR~*YXoMQ2qyn{QBqAP2}v+H?n{CokazNj<`g95l`pyk@My^w81_23qRv| zba(w=9>@L^Hc`qBy-|21UL7aW*RJlLiR_6@+T+TaLIX+Dg*)*dA7O`cf}EMZ7#|y zxiU^i&epTJmHu@&S8m8n_1E(o7_avw|G)?F-`It2XpEePpCJ3{BA=-vzroY+A7-e( z!f+ghoOKiR+o6-X2LH&fb4%B<3%(*>jYsuIVI?-JbGEJHn^9LUyU#UpYg9m4z3c!D z@tb@m8tJd%hqyOJW4ro13`2GG!#tV4=9Bnwu7xskWxrQrl$T%B`!0yjeh2IEI6lD`oZ$0^@&))9k768t@M|{l2^fxs z*r0a=Ux5K~Puz^`Mq~M8%)<4qU&vqZDd-@tL-v(d`Eb2bs3hk-F#E?1>ibYo?{col zmH0{inh)W@I3CT^&HS3|xBo*AdSCBKo;sa}9g&0`lJ1n@eFXnqsQY=6!6ae1^Qge3@KOE+wDCh1I|Cr^w!Qh5AH{RQE$2 z{GvV|xo3}6XO?AFbl`on!nHgznQ!@d<(~LPy^!>Rd=DSR`Tt#w z{4A<*eimy|b1(dd+ygc6BQjTVFQ1NsaF#mf>MG=W9E9u}MU`*id3AQ5Q|0`O>#OG= zdqnR2%*M&c%ou}>$X<9c=ibVFQ;K>R&{95EFZb*cWDfj*1;~8L zT+JDi=j#}s?TtNnM4kKDrG2~a8oiuRZ=jL<7kB5J5#Pzba^~|xa?aS@a(2qxFM}}} zAG?-)J9B3%=Id=iF*!5kAvw>~SL&m23+4dk8PDU4GUXJzI z3y@jS7kR!X;v#fbXUEGN_yE}f%Jb`7L_c$I8|vwm#=iFoo|&w_h)W`8b9SVoR89zc7&B3#k$mEG(& z^)mI7*d|Xy=IOoaI(S0f59Q_B{2@lmZ}JHK6a(>@`UK?6J6ydP4Y2_e(8IO5_zRb! zF|zA#a;+~iZ#(ik`kAA{aUrfp=Ji|pna_vg9lgxWVsg&xyX2u5h41y=!y@@kK8Y9b za&G0bIXmydf%z<^%M+3FsUudyzx&=He&gMkhU@SjGV?RbEBH{(d*O3jB_yXyw|ss4QQNm2%!|{^mo}Bl%v=KR5EL*nlQD+~;oLHuxEPd}cAvHmiktC@dLd`G&ApbnpZSpg*UW*Ov-xl4zRR4*&oAdgH_n-n zKWqN%9XV$}{=9j%x+Bl`a^x8qz}vVe=Q+szUmqjn{24Oqa<=Z^%)6XZr?^%gAFA{7 z&NH5w(pjDR>^=1~l$77(JOkN-^3QvE`T6Ev&b^iUqk-IqyP$^N54;#(VUxNna<6Sh z=4)pB2XdaP-Fyu)KXdLKD9=Woi_EW`d^Z{*&t_)GMVOD5kmsp_e&%WJ&n0@9o6jK6 zW&k>iWq2n)@ORrgJ>4l1NJvcw+$DHTc(`w2`;!re3_QYbm7nzkg12SuC;T^qdxLD4cjXcpE;XPMqRm*{so+QcoGlMU(cuV(fl@NUQR;h<&o;llFYt5({HK| z;)gkNzN9>pGXt_?J*dtzUz`i;XTB_x_uw;i&g&cbeD!yD760ONWEaV7>4Q6crW7*M zALM#C1{2g@pru>^*>y&%&y&aUE68~=nP1mC7~|zzu|a;Gzvm}-DY~M#dIFA+GoM!T z7MvyLyt#!lb5E7K<43%!&fJ`WbJQD=J*cX>4D!Bnxn5@BQ|bY7=3ZSn=YDCvQm-k> z$(g73@z=Oc&e{H_Tm_luU*dE9<=7^d;1f9KbbNEH| z=g8ic{qIxx6AW?fEcC`g>_S=BYrD1=C!(3&XnuiP;3E|Dnf^R4ecX?~aj{KV9AeiS*Ya^{TH%e!)8?x}YvABx5DD2$VD;W>N&E|S0FGr1jSKb)A)AvT`l8*5NdZzP7xb1)0p_sj4<`mgaMz7mz?k=QA>=bYKk%HN=e`VhP!SLcWLq+FAa zL1D~Qzk^?KJxb+sDChUNB7J!)8X@mYm+=xbaBUja$$1z0kY}lzqdcC#GWd7je$v8c zCM%9b-n$CvXHWcv`|IU>c(YstU*T@ORlL9cU2+@Tr0&fpawDF}*^#T`4!J7wcPW)| zkL$znoZjI$0Bw=|{3w0^gIwSDF2#e@`NLMmPvSrLSiTL{$The&XJ@`i9wv9-tGJL} z0bH*B84uuGT!8E&m+&zB;Iqv!125uXJflCK&*6%E2G`+oXd?H*CfuX`k3Zl)_-C%+ z_r5`X6xsb==11_UYi)TU4wJWX-d{Gzhsfun4hE@@L;n8geRXNMAFsktxr+XU+!8BL z5Nln_pE399k9Yz3@8{Xh&wi{r&)q0=#(3n}nTq^;vUBA5$jr;1{R$LBQ@?H_=l&Xn z+<*CV)z{0>g5@^ zfs662{1NKP!#Q){INUCm!+bo9hxFdX6!{V4nHsFlbJ+piP~n94b*ubSRl8Q z*K*FrmvM}`EGEgBd%NW$u}prPGr#g~kl9&T{S()~9y!nSL-Hz|ib3kl$XPOp55)b* zu5mgy@R`i4=g>qx9xuxE_&KaY_URIO*->(4W>>gaT@UZ@QLqZx{;T7j@p#u9QpQ1obnV-63-^=f-+< zNo00zaXm9S?|k2@Z^1mAg>|ki<)-{C`e7Un*ZY-cpf)mp^UrnYi|nL%Cn>GpNPP=` zhMjUlo{5%n&ZwLCcl8in%F{3=*Z5NY91qCd*vI$n=cn4EUt#i^4WYL zCdxM=zqmSGgaa`WOI-gR*>5l7yi1+WZQiM8?& z6p?o!yZ7mw_nf`@O}U?XtNaiD#dcWc^{HWf;*^ja-yn+6B9iQPnyx}w3Fk5bp8OUB=9zV!= z4_o2-dH5Ok;7t8Xxh79WBY6Pdk3-~@Xy(_oS64-Ayo*=#AH-eQA7y>!IzAbnBYzk2 zzR&f+CiQ-JRUV8t(GJb^79;OuE7aM~viIL6->m-@9+!u6c8hDdjk*i+cNaJFF?d4% zZoUKE*_j&ZeTs|mvR-ZehOfuL^1-|ctL2YT9v9+hA-+dIxd#`|O1k)y>r9_z@hBRyY;!=%2`6@=G`cbubGLAn&XF zUB5~$gFEHMdjE1w9EOjOzY{3LdEe^FXJVnxUyll?i@)_|;znd9W`3NcUtF$-oPA$% zOH@Y(oQ6dx?%J{ZEqCT-$Qh6wB)eD6r*i6?eOI`4zMQivGjX)~Gfb6#L@7Bt-9LP` z`Wl|ZH{fG=1drlc7%d;g^Z9aql!O3A<23uPsK(Yp@KHQB*HyclOS<>YeI$I5RrWan8g4nUc>9SRX=C%b@K6?8F2tEQ0F;r&-1uE z^1Nn_o{sD(zo@^#GwMT87at%yT=vF~TzgnO0=wnR)DLkVYN4lo&doMl0e9++#ldnb z9>fdKO>WM4&aaT)#9!+5oO3F(YdadtcVjE|M?Jm1Xef8XYjS3CHF-Ja<4637@hITh zvpg8*U;r|c-p52#RX>i!xCq$`?&Iv*!M}MFvhja&FAGmc?dFVA60)L z-;P`5oLz^@=VH2g81~=;bvxI-l_y{?Uevpd59IH7GOy&D@eAgv58#($zxGDni@g8k zog;hsMe3iBbMjj5jJ25O*IkX$$XS>(YP|j$^wsNvxu~x0hwsn}*#VB?)yVm=7`Gtj zS_eK3#ZW-q3JuU$y#uS|fBARh-Qhg`P_G5v#>48zkiEQ|`d*Y!XLnvDPr*3crB|LS z^B7*n2lAVkEl=W{yWj9S_1~C-D{(od={MxxxF%leW@-3YA zzDDw4^11TMoPD;xULiD)@8yvwf|FgZ!0%$6d_RBA+jtN@MN9mF|MaszO^_R4KlOW9 zCC|dE^0{d0Gk>Wo;Q?Ha>=>Q6AhKHz@wu<%YM6m$I7KhJ^j(~Hi*|Y|@QC_YJ``V} zHRkB$9i$L1SJ%N2*o-~+LjNSxmapXp@SfZR&&j=!{VqG`^Lh)_i})Vw!p}HS??Lp( zR&`YzFAqibrlwpLUGXGd!7)Bl7K4#}>j}O4@um6`Y{IE(m-g+kpYu!%Kt;5{O?U}w zeeOZNkY^#g`EBab$a_Rx{upI&De`XF2*Z8$DJ<0M#~1QPToZ4}xwmp|WZvYt*{;qr zHx-YgsA~<7pWpZD{OqsAMC94a{hj;zYg9qLr%yRQ$IQCDC@&XB?xlR!*`?O-b__ty zrUrVse{xK%y7klDXnuOaTnU+RsVS@H_j z$z!=Ds^BB!9?bKfXY@+-yL=Qj%Y!)kY3{u#@{vsYc8&J@H;zN*ZstxO9E9u;pXg;i zP=ec@EegWOozwtHxA( zcGuPFtC4f?0R21zIU`%>oue)*7nd{h7O2bdN}j;`W`cZ?JR6&Fi26$&kFxR~XfI#I z%Q&;O17E>K@iI!P^PFd1{>6E&ipiO8PxEqQ&g9t}f?x5S&)kJ&@}u|$oz;2X=ivxk zikv0c53h6W19j&6P`Q+RHFBQRRS!c4^`D$~gSUB=x+=fQ8+a@-XB((<{^aa@28-pc zd_EeZ?m&y4;EnLOtXxIvXG8|B8)r&cQA6n{syh{pEFXH4MXr zdKc@RgO`xK;~M_cwFO)QmtzcW(yNZ4I9t6GtC07sr}PHPrI7RXV0Csu|L)rpui>Zl zOJf$gsXs;jxk|m1J8{maJLFyX4juJ2@eMc)IWNw|h5FeY^KM!amGnNw@%lI5Q2Alp zjGS#Na0hNv=ZrW7m!LPM>gQ~mp1wSRbFR*nZ$v?Lc7j9X>>tPR(R$^18a|YZ=vU>^ zI1~N#zrm5Hgqb)Ywa<0o!l;Ps#yx$$21co~PhBXl#|U*Lbs@e^o%ix1Gd?^2j=5iZUmoG!!1D@g5dgZW0 zK92{WplipfFOvK6xA;dMhp*%p`EdS`f8gxg&&rkYulhWckxxQd`C;Tew6MB~JfE|V zJ|VZlsW=Raa3yLYe`nCb=ZeU^kv+H$XRm68N9E40m*L!yGij5YciUooz4{owfFI&F zIPYYC%Ri$HvM2th|Gm5(*&Y7iK6(f8aK4Ev@OTWD8}fZT5{=}%kKf5BqM+Vuyqg!| zQaq0baFqT-cno=Gc#coUbFTfwE3gQ)&{{8hbT9c?`F!k$N$Or)-nHB1f8@5@mizM2 z+!K%EMZAGR`Yo|juHe_5$a!~vK<_i;?-K6R+aTYHwQ@hs`_WpijqGqw>93Zvvp2+< z>g@45^y*-Sx-#m@d2Xu9v(X1FaTE@B?PB)tzWpjQx0{^%GjsABxj4Q>0p!^pjm(09 zyjw5#dY-Roay|Kb&Q9}!{0N>`=jZSx?!^Li=GxbIOV0Oy0!GVwcrfy8{GL8%cIMgX zitHnw@T>ZH-ipgFVW*s**8q z&eoiP+c90;9+Q#hB6BBa#@DXpOnw0+^=9+Qm?>Y)zjDrl%-MU@nJ3rs>71Rq2sg*C zdYw7*CFk5l^1;YXHBp^szb-l;vobRw^W-zG$wP3OoE@ktH^3FBqJKG0;2Syj^Mm{* za@OWNe@gFlJdJPkuSHQg&(}}#L2~x0TR7)Y&Vb3t9{H8)Zz1RS<9hSuzBpgbY&!!@ zQ68`9zrmU3dG7zhD0L6Mid&(vJQf=;0(apWG;(bVzsc`%&ijgfp98tRdJtDf-Vw%g z_NVXgzH4(hXVorwf8;$Ob0#yXF|t49x&KdIi#+GI`P^d8zSc#~{`5ST)B6}J+h<$dsq+?p$4 zI&waJ!AmjTwZVKWZ^B>lSd{gd&FU-UoEH`46}VWv1ry}oIQ!X0e4P3b&YXK$Ue9?i zI)qE2lwQu0>^|Mq@8Ns(iCB%1$iBCR7rM3#JLG|U4(GheS(tg*Nj*hffnVVt_(slL z&d&aeoIPNwx*8hD^Y!kL|K;rWFY*3d0Ke$pjBk;>@fp_&%O9ZtI;uzU5R8+Dq84sd zui^H{8GN!j=jTqIrroIJ_0Hm6+=$=hzW5!z@iXe^ zx5FRuk9YznsXxb`=&WwRjr_OIlye>~QfKe~RQ({bGZy11`ls?Ad?-J}ucMH>2W{oc zxGc}XJ@WB<32Mlv;}TTE?YJ8UV-b$@nYR21UPKf1lNckPhL-*emHB0?!aJ_*;=Ayk z{4CzUP-GAPM*koA0G`L2_-}kD59ixCJLnhur8>LAUaqOWi2tX~PSQne!?pFxaVhoF z@}U@n|Diu+=i9;C)FpT+&*l60N~}W8_fL60Jb~4&pUT-G-sS^wieAq0N%ArBmG}qc z)K7ADfsf@C@(<`I58#Kn7-q=X3$oK)rOsZxj?cugt~EptT!UluPQh{VVerxa|93#G z_0PvJoPiTv8-@$fU!C{%tGS-K2xiD-_)*S#!)@|3xd~?{-^wHLpI&cNlt18pTpBka zyHI7l*W~-Tpn9gf85gRX;#Kri_vD}W2s|a9i#l?7UdOeO{dBInfqXq4MpZn98}!@o zQ5Yvbjy z@2$9e2cUR82JQj_Sy)g4@GBSsAmeiG-Aw&S9*?)=-#N3rshoYWgL=N4XY6G; zXG+e+TX-~bMzll$*Rz++mosOxGv>RfqHe`Axv*Yw`9-+`GV3ow&aT|2pK)e=&f+{@ zxi6b@H++xR)Ps@zp%R~moacE~ce?&My2*$0RNjJGSf$R6H;r@e<+(YXZ&hzZp0Vs{ zck69Jcl9MWO|FN5Sg(GNGhZ{;UXq7$X?}{&!w@9-pi%*nj`z(Wt`cwTmNf} zK_|R`FEGgU)|?rgb7D63=zYkaqCPTTGAmZfnO}K6HgINkf6nvxHL|xI!C#`CYcuet z{0ra2H**8tg)8O4ycM4yd&N8yL|3%KSf6deNV#o3s>Y_Y~v&Zh0KajU0=Ti3X8ggc9c8Pmj z&kSv$cN|K|^?4zm%lBXuwMYG2pCjYEg zNWMy5EB8m$1-t`XVBI?IbL0yr{ z@&wNLc>|Z>=eaFb$p6FDRE~4bOvK3;8S5`ac9D~Kuig{58rd)E@DxnPzqkSK=>LnF z@{>42?uG0hNAo;8(|Kf#*cc-_(E=o-!NRg z4Xc8dvcz+y+_8e9R4St?SE?ed{Rob8_CpZssDmUT*|Hg-kOwuPon(9rB0xNX}kZ zi%Y62Vy2w;qs9Cv>R}gJAbanHoPqNa<-H^O%ro+#7p{@>zV|tws?Ph}8@x_^BKlwte#hN-&$UlD`&Sq4tC2`EGLmZ{|ER z#pTSt>@&HKvv=g4dmZ_6_CTJM64-+x`jhzs&NEhC&a;{OCHHqDoQ7MG=Q;CfKJqik zJ)c>X`#v+ZV`|Q<&i9>rb1R?2XCe1`XTAj&>mSZJ%f@qlj`^OyQ)dVKLB3SJAjm5@ z|FlO#{DcO2vvGu+`!3&aU1ZmrtoI>uz8r}1Xn})V?~2?znGMVJTB_g2uV|zG4w)6h zxESY*$&Ad7mYwPlIrE|fa9EAq@fsm^nGvATgeXG~^Z?$-_YL(cuU z6b~Zzb635l@@Vvz|HlK7=d~keCY>f{rdPodbwk{Zb~syaIZxwu$b5biYmhUrCx7L7 z_WYa=bCB7PnK)XWht=wwSx3s7FcP2XWzXE5o}At25iWohXpf>;=K59K7Uv?nenl>g zZ(Pf6Qk}=E$8k44jQewTj?ClCq;K_($IBR|*8$t)cRBATFLO=gtlF-hGxB5g6r7Hn zBNKR`Yfp1#Rd$e0>e9!K`0seB(+`OMc`ga5!*`8NKTbG|&wFQ}X0Qu$dv zl@Gvw*rq;(d*C(sZG0tXf1bjF)lKwVR$@HTSxexP4m-ie$?FLGw-4*lP_7hjIv_yvFJt>ByZ65hboP)g3bUlDYc z&qp8q0_ra~JIMk(C09ii`A0ODPvH}gv+G74qc;}c%C$K&{UNy)+WO4(+#QwV!koF@ zhxbQ4IeY3S@=M71{57A;Gx$l@8{iWhhl}*OU=gy{WyiZ;UW>B06H9&W5loi{@oKz^ zCAeB|3--$2^GrM{FXo>y0+rQO{l1I%8Flv4QMs=Eh%2a%;=1bnfgk9@2Du>F5l1DfuG|a z)Ti?}egKt`_pjUacFI?BS@r+zbFJ0&^h;u*T!GKyd-0fj5pU+97%0DnR`Mji4lVp% zH>mS>4`=WSy<;$3zKw52L)S)dW4)8OnfhwZZh4+wKfHnS^_p-Y{)w}%S zetrCi`_!l6cllF30GG(6F#r!^G0OY3b+`h0W17#M$!BqOE{>NU;uDc~@n3kB-g9^bZ>Y0# zwUP7XccYdVf?IKoejOZ*-;np6v$-`6$4Z|m!iVBGRKf(kb9f_<&~AcXD>#&-gq2%z~U3pUac*x_S=Up{{xhH$dk69Q9RlW=MV>?d9x7H95~` z6AZ+=$j`MJuSTBLL-=#fGx;E&iW2f3)IwQxd(L_PKKDSLyJ09G=ibb$?92ICejsn< zd{0lvnJ3@z1a-dmd~fgYa^$e@p2M@&WLQqIT&(HW1Z7jd5F z{&MDN&cDo&OZC3OVB~o`9=mY_j>W4!Q<|sY72Jal^fE(NaOQF5Yo5XHT|1Y%a^`dP zn3d}KxCFVEpWvLGxhFFxo|ZGW265(8&X?|T175<oa%HnXn4P1;jah~I^ zQB1Cl2XKq}ByPqt&=wb~mmts39XuYN$fY@FQ||4_@(nzeJ8@=Go|DJrlF0d#8Cp^A z8Jvuqsh1ORiN9;zm4%-^2p>QLf94 z@vFRt3;T1NDc3_G^+U+>dOuFl%QKpN`Y|q}&Od*mxSSc3`C6Xu()*B~#C6CS^n>1Q zn6CaE1Lf0sIC9p_;LgYlTZ6gijrTE5KYP^6@)+bDB{MsF&qf}FAAF_*mdV=t&b)KD zi6ZlUm}`sGS8;aJqH;sjRL{m&I1EemF5shZy*!UItA}vr{_}d-MSqqHp}D#%e}hK) z1=Le;pn8AxOfIipiM!=4$h*%d9Z3V3Mb7B#D!;0)(D(1Yy|0sSqFm6mkGLG}#Yl|B$FAqh zYk~S`g}hg`(0@VB`LIU56h(2U-gUSHw<2dxGYm%dgUWof{sDXke~OZF_Wd*E-E#K) z$#Na}CN9WrIJ?_ixi@Ob=lNV=9-*GdmGBPU#(Bv5NH5oV%A5Jbd=5vcmvMHvyLdfv z{$?M`zHyf8^VK=q{~t~E0siIu_J90GJCf0`wWkV|hG-9?(h#y*lD)E{p(T>Y9xZ86 z8b&Hj*)x=kCfOsQCHLdiaompI^Z$IE=k=+(zMtzl-{;l;1+J{AK*3o zi^pR9oB1Q&%eP_=rl_m%KYT7m%DeF_cA}Qv-Kc~sk=^q^J{9@*gS@le?{imUrQV@< zLf*v<`FL*6c~3e)K1F^Wb>;u~UVfWT#!zfh7xmdD@_Kxro{xjD9qrIVe<3Q%H)E?j z4kyax{Mr?A861iJ_)q^t{qb`4wCmJ0afA92?#-u>IJCpT3L05;wN-C z{>B5yS@S9vK%U9mTlp{ZJb(8E{5dj1M{}Nwob@>`mdlwZ&&YWuav#*h^?EroGZQm6 zvVY}1ejfSGpGM|?1>_vv#hHgUah`!u_!YVDaz8bdGYdJ<^MF<{ZlB;XT(T z@fDnV<2BCDHTT>S&i&tta~`zT%l$VSZ{s24S?q+#a_;91D21Zx%+Sw}@9uiNGJFwM zBhOwXzh|C_Vd}SdJ67Qp^&H;8Q<1ZA8$Ya$`Q)d=r_bP&%8}I68rk3T$kaKRD&*dC@osUA!r;m{t zlv&-(XENt=58s9Je0DMC+1`yjqi16i^89CRzQO~L8ThCEIk*bn<30U6AD`j~bk=*0 zGou^uN7#Y;_4B-C7C*{4M|1WZAkW6*n5WKjm}j<&x&r^gwedc(`(#e_L=)VJq4>?U zZdfm0$PKv%f5q8PkK;bbY#oH0@0sK8$eGpca25VjUyIH1I?Rw?!d%q#`RvDI^rs+m z@HYJA+812UwKusc&cXf2xs>^}hBMRZ-~?n|tmM^r8Do4V^F8x2=Vi`^<@&w#n?bqn ztRKja=wE|+@&vw#3v+hH=9nnor*|?wL09z)yo4*b-bP-Fjd&W_9Ww7`x?Y@HV!AvQ zU*RBB)ccGx4`0P_JgavBAHc2AME;w9L@9irK9r9|-Ysw74%mQduC3u?kh$FgcjIMb zmpIw=r*M|~5o8zY%5Nci#l5b5$G0PA!G~OqTj+m+j#!J9$WB|+_1mzY`VG#x(VWlZ zycgwtv!r~m-XQ)OWpOv^Vj+&lT;x3|JIO8bqg;>6bI#b>ybw>KmbzGww_q$TKn=hC zbp0RXygywl|A>$T?zyb6EH+5fVez9IjicMKQe-;o{s zKXqSu9FD|N90Bcp@3yVLp@;Cp!^u93I!d~`+~^*^`= zXQ{77-Xq@TR`^A~3J#XDBNgIp=%H5_hhd!h${>G@%5qU0jmGN5*ed6}{U`Zb^up_S zN527QpYJI@A)kn4n1p}z`k@NWP_M(`az)gY55(W{i?~o8&!_Sed>HhoUKVp@rT}{0qL5b2jGO%+K~A^&dD?9+J#CKXaeXQ!mGz z^5c9JzrdL%Iomp;yI%II+?ylhoCA~PA}D}7!zbylmb>u|Px&di3}4G9==tBi zGiCzkZ0@apA^wp^pdJR|3gq0(E>{7U_{>(!)z6Gvfjdx7??=qXMe4q2A!qKFwjW%ENGhJcEDY&(RNikzMyOz6k4GKO0ZV*_ZQPHCVk~ z-Hg9NS>&vIns3KFuKkMr@rAlJD$AMkYcLc4;%@y;sW~(CKzS9i&pyHT>KDVU^4*vy zzlLj32V?YdUcAZ|s6R*c?HXKIFT2L+a!GkE3dx<2Go+OIH+dzm;_Slb%Go`ttMeX` zUF>@KO#Sy!6*uETz1Q(G`l`?6+c@XtvHUQK;AB)scG*u|dj-?gcXDgJ8S(%ep?(n6 z^x7il)GqYI^VsY&*WoXD2sgk)&d6jovHBj^9@Onp_73<@2yn9?74fsQe-( z;4JkV?t#m2H8$&&M}5CnL-mbvKh%=5&pyOc)F0qrxij|DKTw@_qPpr+<-_?StdpPR zulY?}Dd(ML17CwS$oom&2XhwR>N7vQ+ zaq8!AAFfAjT&~{(%a9%PPrWZuFxFejt8tF}9cK@nCBGvtK_$62+RM-I2V4eU;1pb@ zw;vzHy>OKL6|J1=PU6su29!Ub9pI#ms|S1E6V+oQ64v_ zAH$_`AHJO%aXFve$`#ew8DHQ%>fiCMJQa=Qy2!5fpt^$GgEM3DjO3Zm_57LhUF7%A z_nE)XDCD~-fvIwS26@i%J>{Nk%pa)pGtYmycMsq@)H%cQ=gyzC0A~i|&vl?WbK!I3 z=azdhJ4C+g+%MnjWd^+^=X-k)`58}E-;B(f{48>RKB;al=XuS{xrnE6=5)TZ^W>U1 z5f{7GopWC-!avB*r-y#-sd9J>`FZDi$upMcC^NDH@-xnNI*T*M9^#WZJ4>FcTh&La z58~VxH*(JZHr!r)H!>S?4`f#6jLfsP1V7_#oR8)DS7VmE6S*fcFU~~1zuX(m{Jxvz ze5e2aEdHhcxL)qJjmY=kiu2t6g40s_e4eYE!7m5-Ue0~LK+Zfn1`G8v$KS_I@5i75$o=XFM~h8S>0M$0bl&?u5*Z>{!+GzQx7rmyk1cJ|EAye{YgoVzm4a@~q^3 z{{Q*)uDTsEXD4x-~kL@&LY;dm}sj+v@DZ+37N) zpU}?^a3x=lN%9exgUpx)_*Gtp!MGbm{MzUB>&ZEr50%GYv3g*x;{s&1^yV(eJgKM7 z?s<>;1&l(@%IvtOyO#Hjrrbe4XWVXN@7$=)ykEs*&_wRVc}F@;-hivr$MSn9gI}>z zFFR{VzF)lyuc1HYVWR#r&a8X}Psv{(?++E!FJc{r>%EGTklBBdo{#LC>-))c)L*4~{;7?rsnpdJ9X5$vUo}9gHy1Ygn$5VJTYRJ#xH#z6~be^qV zhspRHKcE`|{KgX+h0=CN!aCX+b=ba~CrdJ(j%RljL+!KA|_4rEO#kFuJ zE=55G?DIpBeeMp^FYl*o^!`>~ z$a7FpZxMg1w}R_)U#ynzLf+{+ae0iyVXpnjP5BlSmLK6WxC*M{5`3)pA%@=7&9$!QJI~K8^Wqoe&zyTX-&>x^sp^CIM)a5S z^UZUYd!rok9lWfrCAY`7_+6cuTavFr?#s-a(|8s#t8*Xb49Y#zQLhc>%*#ER8B>IF zujQF9h&^&Y&VHDmQJ&2&^u{1FA#-FoXU^r`$bELX-bCd2euVSv=4{LTQ5)CjFXueJ znWN*8`{NoO;o4|qE@rN*(p!srklpGwo`8Jkd4_t*nUT$rGco7J4RW5vWAQdt`ds#v zJgXJZO1%+}BhO8~%LX`Fo&Bc^=lRS0xE}`~b1=`}PP`)L-pFp5dn@zwCGLew^>ZH$ zMO``1!5zGn|9{8GGu2n`X}!~V38u=sxHzxp+=KqNZ@0)CJy32zO#Fp< z$eEfM{iW+0QCoeVdK&J*67<$@$rG>``{5J4%$Ch^6Xf~qt~VRq|!;s3_1U1^PW{wZ^uTI$ zd2Eo&;Tb%Jy?Rx7B5sy<<93{lxAcxe-h1Zq8iIe43=#O{RWB3w2mrp_&zpgiD zf1R(tKz=yC4qwS_^{dNUu|<71x6(Tk56TPh1J1$*6u~e|NA{C7T!p{lu6}(degKck zFQAotL6Gl2BNX%bfAz*9`}uV|UcVwr%HQ(g{1Gq37wCY!xE_bP_95qeqmO)rTvjg1 z+tCbtP}#Lxa1u^M-ih1k*N|s$YrKX^`jzzl#`PGWSDVk`JR5&=c96`NJR7;+a^J7x zoIU-JXJ8s<&adHF_y^gC@*HhJIV?f;k?a%M*T{WA-yq+?^Ln}0iXh+fuiOQ1 z%egm~%ej{_YjPiF$LhxU`L0HGv3!5|E{m)4Jy*j;*oMrK%!AEx?&Z;33i)~FyI&+9 zfgkZI@=P6!+{a^(`S&||%b9_h&AFeZA1ZXD*?a8!{u@{O2^gK-TC;2Y$5UBS2E zTV%i5U#}!8sxz-D@P&9%Zp}v`JMx+8Hge{2Y56kFyv{S7{V~s6p6NW#lac2)&(AeH z6Q|4T@d&b~+@tq3GM~TDd!I9RZ;O$8GIKBUDfjfxa%TP$Xr`ALll%2u*Pi9u9hg^;;>m7&tkX`XE-sIX{$UJ!(nfsri33jSGa?XyNE$7Me zk$K%-y)v1*W4Am2zsh@XmE4efVh(Om_vZqB?Lf}lxRh_u&um?fJJe6{ARHoJ%~x_q z|3mibkHqXZ8`Ou( zd2jfGf5K&YyZC2b$aU~ArmM^I0ek_Uht6_Myn;pQPRKcVC(7xagz<9DuD|5sa%+Bv z3+bIGUx`BMxwu$vga>dmu1DTm55g4JFHq0mgp9Lh}089j*$=l9{vippi|MNXhK<@J+(Hd7G-@&_n zQ72r3e4qJEGQTt1Z$vAtJhKIG9da(cg51k3xjXLixw>2onQe>J>v6Q4=dmpEJU7KP za_+T!zu67*Y~>!>ssAbG9LQOjd7AtFTNF^|-py<|k290{W2$^6%FEeNtI1uEGpD`Y zGw6(*r+EgmJLmrT3A2!0Av0~EyaA`mS6~p1K8jX6?qRn!{x{hw?+RqY{cz)nSFDRJ^Dw! zU;kcy8<|~@*KZDrcGcVvIsgv`ou`jv4U2I3<9%+xi! z7ainVIrI7-`6JvbKg4D5lDvvra1nlxGh>(VN>q?D&sO0A`B^T{*;n4;g~)mIv^x7( zFO)!bkY{)}E=C{ME9#BsN$SO%b9JO#j&Hz0@>!@N--p%mrTh{oy`6m=bGn|2^ksUSrXhWY_ zp?;Nn<6#ugZ>d*WZiDIalUR;R&=K3w*0n{LgICoDB75O;+#Ln=-{9L&AJf&3;4oCi zpZEjWfhzC^ocE>Pe5g9_Lx*tom~-)@+!)1?-7ot<6UmF*ai^ z`lAv)bNzk31>Yij!2rF(SK8fU(a=Lp?oLKK=#j*xEmT_2!6mnxE-y1{{PJczHg@_tYiqvfhNTmO2lq?f(>GI@aB z2CSB|%XZ?DyjL&puhrH0x10L>0P=2>`#1M(p1C{pa+WUSoV(9)?xT)8!u9;z@_#pf z|KE^5<5ifdpPygmS?*&m-qgo`av|!aNx>Kg*qw`*R-GMoXN9oZp$xInQ$sWIp7+KR_?D z^)}9Z|2-Z-?%$DmnG1PNaxeakJfpX|)&nKwQ+PP%nW}@#1XMO~J!;i>W*v$3n+!J|^*ufv-XjH&tpSu>> zdox$M;AQ{v7AagZO0bi_EXQYpm9L3a_g3u2%)wZQfA##+mX7e232r-v@`GfO;H`mG4Gn`Az;4ucNnmDIdo7@G=~MGI&MrH2#b~!Zm22E{6&@ z4=eRDpN~Zc6xJJ!8!!qV>&@ZpR^{Yd@vgcJU%`2=x=tP`?4pxaz5rfd`m2X`M1TK>nrddCg}~~-}xZk%bR%?e}iXn8%{(G{hZe`_#JgQ`2*gF7J4V(6S=G{e?U9AJPwr`q74?RXYze0D^JB` z=%hXy2Vsx8D+8& z{*mY530#d`7>d_idxo#%yZ9w+!ryoRMg2Y-UHbqZso&!H#dJtctrO#cWmFFXW zulyXcJLK;$9qlm#nHTx~mLO+Z?%ir~W^B%w+@Ir+`#9fQ?#DYY6aVUGKINW%gmX{i zS=%2u7sl!zf&s-_sIx?!BIJ?#T%#hULgi{uS@w4%ah-|B<_J?$24A=jmNg zSw0t!sWT5=#&I|r59%Gk^EvZ2dsFV!wR&gpTRf0=@D%(gKh8Oua_`rabMKc#59GP> zzkPez1^fqc?w-tuivyY6Cdn0E=c81*FcjF}2@=Ryex8@S+Tk)Sfh%ewv@C-i0IeOiY-79lF=ho}U z96o^aOyr!Nt8S>yTwKgecs;gYt2*cFb~&^53VD>=5t+?e`}W+N4|)Cyx;~iC;LL?) z$SyTQy;pt%rz5+^e7)=C>yYy&&*}kk_SHPI%`jS>cax{(%!%JIO)qn{Ic`%wz&S5+ z=H}huR=vi0JCOaW9Ny3?j&gdx@CtsDf5Q+tyJ}`_=FatccOrXbcDT&{?5nHf68f1v zvpBnb1!P~Xh?n%TD`n5_CjX2IScE6>zF&752FlN2GycL@y)|4G_aW~Ehv{X{T+2oD z&gID%j|EtzmvgihH^C=*Pw@smk1yd~T%M<*v)rApz^};8_dQodc7vU+*O2e$8~H{a zj<#}R&KI5ipny10eHETZZ*_5;BoD(fY*Sa^Z}}UXi8=TXndfDYxt%j{gM5qrLe4wx z34F79HWxz9y%*HkZL-Teq*oUw>RpBIawj~C!dR@gg-^rhxLW-p*2$k@q+A$H<)e5% z&Y4kB{*fDSJ3Nm5I1rt%%eBfpf-B((`C%T3B67~t6XlLLP9Db#xhPlX<8dvfsjH&1 zd=GM-HTU;=md`@o?@xFALiEGgdgo&+M&NzD&$$mD!Px=Nl(R<;<*)P}EFXe_-}p$ zry#rY6P$OlJ^U&z^y^;7rO1w1PH#6JQjgAOxDr;%7x6{>J&r{UoP*u^rTyBExhDQW zBmKOGtl*{UyYK91ZoZ<2(6Ld@NVtb*L;KgR=4o_!`+? z59a)P$xU3}^`ZPOj+c9IYuBc5-nqx{#aM%bT>p|wqoiv;sEzx-KZfV!s($SRK2-e)s-vI!b_|g(N8U}oQ)lPOyKvr72B;Sy``MHHfzQq4d-0Q8 z53A+u$94Dt^(fpg=igg?;XkogZ!>?(#q?k2%&7cXa^~dcc(r^qGAFxno~y!mNv?(N zcosRQG8_LwW^tZ@JQw-SGHdeZX~UU&`R;P&oWVIObMI$H#pLtRcMYWGbW>jX{ zCj22E!lgLRWbVa()%kl?MSJYTa^#-KGm-Bz-%kZ(&ODB>v{HzBeNsV$spdv zW07Yf|J^Uw1hb8spSK+zR^JR{DJMPAr*oft> z__H z=2j8zi9T|6q~rKPb!JXUJ_P^MTSYnZ^Gsw0R6zlK<-RjG=kgMD8LW_>!3FqS{Uy#v zX4_o7ypPP6pG0QvJk*hMKFsCp8#_4jF=xYUoS0w7m*I4|Dc`}FOA9buJq>TknOUF8 znS)LEF3$Ykil=cD{?=>E@1uhJGVkS^_+p$VkLDG4LB0jw$&Yh29*L#cj;4C8@tItL zkHI_09`Q40?>K^I;~}4!hmP`rd>dzP+rYCh1DQQ7T+9BO{b7-uxpfIL%iiPBoZY5} z{Is0?IWumOJPmd68_K)hjSFCboH?B{bc#GdZ!JH>H8^|V;qnOlgY57%_+?~={*Kq< zZJ*71%awB8^%ko?lON%nD?{X*PaQbB&ntSr;W>3X^$7VVIXmp%a(12>e1qOA7=S|R zukgEkIZwnS%tA%ImKY*m#MuiT*4&+F7@^Sjt6Ux}Q%r>H06G4&qq%jNZd=ezMB zu0cIi)-Q~gd~Q05=&i($@(w(Ya_XTxoo_=aEL4BNjqtggJ)%6HiqrMJMP;nQi+U}P z{q%PAHF0Jq8`k)7yW^%?mbzlx5ij&E=*TH|Qf2jgS;VQ#?pB0J8d+=stGMfpZv zhO^~6xfr&|l{oL)&&g%wL0lWxqb^$MJ%;x3Q9O#f^TqfVcc}9&@?t)V3231n=vrHJ z#YcL7;xM^_Yg@UwI{V!l@*4SM&JOm5e5u@(PvY!Zukku`(mRgF^M`1L#_B6LfA>$3 zUFssef8_JU%h|_z_;H?!nh`X5K;As9uk~ z@^8pJH~}@~oXgn_e^uw6X~K)p-L+Qg%JO zGqqdq6S*gr%9D9H=lSc+xhKCyp7lHfqwyq8!!4+Tsrft}mm4E9?pywhb2c4_p7Jj^ z7|MP3$M?ce`7Gp~JX1Xo$78(uNKBM-UglX_Cg+UI9`GyYIl79oH#U>6L}uZE$h;}< zT6W7bcsO=q8ZJY2O2{uD&uR9UQ}|)^Vic9PAv@<@9;uhzZl_!cbJUq*>v$5E<=6OYd?#OwS8$$s zKhEqpnycY9WR~{ek|>6;>e;wn&iu<>Ux$0Cv#ZpVACl+r0i1L4CV2(EQkUdAxSam| z@+7$yF2K#|Vfakm!Ka}j_NdRtsThQ1dO4TI$YXJ_yq#Mk=hJ3>LhlwVkOy&F{*SXK ze1Oc=xA8=JtKSPaJ_TY z*$ZdN1JDZBqmq7gbi@tnnm)6g4_0TdoXS@q=lDqd|H(UW7xI4BNNc&AGwyBv7enO7 zI6KWzoU`Y0ej0zd_8gbvoLSkSEAT_;ia%Xz!=>;Nnz=rU-_*~p(nr1omDTMzXMEn* z4&`BbWB5)Sjbreh-e4Y$=JFDpBd^9&^5@)}JMfo08%5+txscy?l$>*XGZ*4U_)c!Z zA9CKsuH%(B2pdq*wL|m^%R}W`u@+0zPa%8Y7wX#hM?D!^8SGaB4@8UgwNn3cs>dtJIT8~Q(C@(e@8p{KK_Ph@L^bi`_$P*o{;kn zTaHiQ`ud~fdodW()p^fv%;VK1_$ThiOSvI-$uDv>pUeB#M!7vYV4Z8(orlO<v_KK`$A{>Og05faa}7`(ckBJcA941uJjfI2Zxl=)r{gGO&#S@L`RrkwJ^WOD zP5mFb$eqwvet@sWrE*oD+nnp1KhurK-z(2pp11rQM32pUr@8{kfEoY9ENB-_PJM;H0fSj>!`CRV7{gK(e3kM)S%ep9s zJJlB=&*wwx<;c&knYyNaEBP7Z-YTiivyii66Q&{G&24&_o4HSF%9&?RbH2l3a?a}E z@;7n?&OO~l-YMst$}|1GoSB;EzBvxo+sosT?>zTF?d408M#;A;kS4oPC@SdefP5dtLUY74F<@W3mZ^c{XJ(MT`6CJ z9>{a>6X#jV%=^w~n&B$>Q2Zrl$IJ7d^YRLQ41RrRu)T#W1F zC43tGK%VQ<^!9K^z6KA;mGKBVs82@bZ7ILkZuwD+R8K zz(>fO&)(UNhq`tXGP}BRExi`}EdG|WYvjM|GubyYQ@_^B49cbA_(V9CvoC(YnP)jy z55|+O%|hO}F5-n)gq*)WVygTtFTksE6Fi0Np?BzI@5{Md5;KwWA#?c>xtrXDf5BPu z_jp9k88crl;B%+)bo`F&Mx9-&h|AGlFLV9?TrdB|J^2Mbn(yYZSSp{x<^1}2@`-Zx z)xumIHzMyhhq;z>u%5apepPSgv3xR~lK121`CJ~z_w!E7#P8}M{4nw!IA49WJc7&d zJv@h}a_0Ud{y{y28>656DHrCWPzkNok8xRE#yfZ$u0jp<{hV|3WByZpCI5#q*s8t) z*UL|$7k*ZE=WFq=JP6sd`p38z&$+e?*-6gNXZ2R_t;lZCTHPH7Vzk}}F3Rulm;658 zj)Txdy@R)K_Sg>cC=A0wy*8X3^h-WdeGPxmJhs?+PEv*;f{G_KZjMcjG$b z-$RD!Ka8#TO|OXF{-}V@^}f=}J8S+up^y9{@^1l6IXmNa{sM)3wm0U=XW}_D#s)M) zZ!B`XBDX@`wMVKapryJBTIcsuUyK9sE*k2O<0E-BH}{z-a!1^Tb;wRTkR2@8<>E#KWkXolRYnG2b< z`TOR($howf^Zj&@^WEnDc%1V*=Q-QLPpk81&rEKCoTY!OUyMe+P5zrD!+uoP*pEy zXYP}`)S1Q8IcG|~pQ-A($UJ=&i(Jchml;_^{TUa=|>}K~Ipr74if_gpYzRu2&ohq|Cv+`~}3Df1AyLFJ~F!x&* zy#i>7oPq81a&PCJS|g9)>>o|!+`sG8xv#PdW?r{O&aEmK?fR?8eOnKiwbPJUklFPf zGAlEGa!!@RllVkG|DA?^&s&)CtZz$08rFq9#te07EpWY0f!96igeiGj!Ga)l;h5i)Wp?)2W(N_O+&dkoU-a36b&tA2B zUVSNl#AA?Wb%XjT-huIQIj)J?D5X9o_&r+Tb@@j9LvSi)Bkw`O^}oaQ*oM;RhTX`1 ze-dBlvv2ZooVj`|Uc?DFTmK7`!#L!;&wMJ#tJMW@o%$@4#KUN)mwEj&#>&fthm7%~#~-k^OlvXHItG*Z3B0$~hPQk~2SV<(|ksbPCSF6)$Exi_!nVVs%yja&-Zxw3a;?l(Xk9EO*1EY9_roP!r(CaPnke$MOr@H=YYAHC|x zj+7alS)3jBAymd6_|5h6@HDcs&ez+BOOTyCJNgsyQFzB^-bZ$auhb7<2wuj~INh}$ zu~UAFC*nlI{(zrA4#8_^DSGRFjs5lRQs+IPD4Od1 ztKNmYYhTU%kez3#Yt3*9I-&=@#Cxu9$KfdCb1!il{cBMk3sDh=>VL;KAn&QatN$lo zNo`S1ZDm%n5 z&fooez5w0i%$l6lHRb%@KAW524>^C{;ux-Ojm)lZc`$O%pQ)b7nX#APALJg){dp6X zqMUj-m*C8;oTa%Rvjf!Ae*{0^RJ@Cvftd+6`AqhJr}gq2k44Vk=hW-*D)Jn*)6e}o z8996I;LN>}a?YXr_bjr5)zDuo=jWUKVhA@xX6Oa3&F0TJbM6P^+`Eb=VL9fg$0E2`+tKxYM((`@ z$Q;kC&K%C3^ssBa)tSK;qXS-4=N>-{$EY*&rgHY~4>`|Op7G3y_s|e^)CG{4eiq-x zPhpjO4nC8!!*AvG>X-O2tdO(6Wd7vbzLGNwv(J=LKQFgLaeR*^$eer}nM&j zg+hAYBIm{+y|2(k{XQyS6z)SM{ReQZJP?l}GvF%j$q(|e{1gAr@3n>wz-GN3d^V29 zAY|4o(|-o#efCoI7BrLd+@FtETx-GC<3#yMzMM11-oR^e_VH?*eL3eP|AZJAzzh`FmZ?4^m3dsKajb0xds@{XcDA)(D1bLnSnp2c9dQJ&)SJae@qV}!Idk$3)lR=Q&exmI z)6f#1qKw`(_*K3iKgpk<3yP@o&XF^&1x{4wUGqNIYRdnjGwxRxT@xT)`cYGj_D!W3t>GFUZq5=l&aV8+@m(s@^Wo#!Piv{ur(0{rL_)fxqCg zyb5{0FVD~D_2N-@7ayXS-Xi=b--D0jiCBOG)T=Q;-iGWW*;jYUPwQu|%)cw=rac5( z@FO~5nRZ`va=jNHr1t{PP1gI2AJHojnA*af5`Wm`TDb*XE*mm&c6Tfo1An19qypcGmxL>A9!3n0_Pz2UVg@zJ0<0L zTo>)-`FsX4TNbPH%;w&k&t3GhH{2tSMPX##)WKxeb|QPvCF-N(JR|kwvrtvNit~Kl zBA>@Obh6uAuFkz!RI7;kYB{sHGG=0m{>%7Q&MY|v&&jVL_jhK<6wY4txcn)uQs2&X z_)YGDrFaXOQG<9TXU>+AbHC?&UM7Dk-_3ajmtlW2Rd?sv{5LYcYjI`HGnzB=Jf4H1 znBm$aeh4-2tUBjOXU-Y*Di_wzF7_2KM`rp!{qN+hcpguxzvQEld;ej+1kcHXc_rt3 zX(MNboXw5YnR$hH8s0^Ap3J|y^$TJKD&j_rbNy5lm*2-4Iqw_&`Fmvcwnm=W_2}(0 z3pumsIE;`F=bVk1bJmT+c!&c-)5bLDM0^P_@#v-}qS52qvNOA+N4at&V0UnA$~tvF5ZK7Iz- zrO!}j#^gQW0lkUxshn9g8hQ6Pj?ctX$T|17>vzeS>1XpItkoONPvbA#i68Y2L`V5} zK9O_YJ;QlNXeV#NmFjcxC^AFy{?tP5h9_`=&(uQBro2o1pjUy)T{Y({{LO%c=awEfNJ{fcnE%%f59kx zp#Fy^^H5}m&Dq&cZVPeWd*#PI^Q}Ap+vGzz?`%0kYRen+cJi&Lgz9)#?=gHS_uxIq z`IPye_k-5z_s|Sq==H_Rauu}4PwM7;Gy25pfjk#i;63$p{s5QDU!fSX*X8~7HvQ~e zkLW#&mMDXb_{_BdXp3WTj@~P{P<{p3;s4~fajAZ9OhOBF2fmb-qLj}SP~Ra}9zvEGuj=DHe?^m3Ovg+!5F`tf3xDQ8Th<;^$0okp}tH&U_Sl)FXly7kD8~&OP z<>xu?3nkG=T@Pz8M!y^%#Mg0GK9=9$laP15nfxN!V5)0p;e5G1w#c2Z8jaNl@oStN z;vGI5uj}P`s3zaZXW)FyRWHEf7^}{%)s1srt>-z|;Nr z=F)f+MeeDbzxN|&(ldOh{@cj8-9V~L_N$QQ9XKJF{TE2{D;b*x8wqw3} z9CEJBQ9p~F$hmid-jj09+*{;QO#628>`d9c_Sd*sFFSNg&YaBiwi20dgK;_vySA9u z@MPo}%Cqw$=Qqv#T7l1znQ|8n#|C8QyUWLJ#ccI1Waf>=5)@bW=U&KpF+E+L;I zzlZEGc~8o&n!RnUYtt|if4KfJ?vxMXg(%`PnUU-Djz;EWN%d9oY7CXP;$!&-+=zL| zUYT8Vv7C9^8x`@V&(`DYr}xO;%lYprxh1ZZvlD;H12Ijn6(5Mc=%;=aMs`*6P2DynFu0yY)Kcn)-Df zhY$2S;9O?zQBGsM%{*UK33)5)feJP z`D(6@(@@3ru6niPuQ=~A#rO|(bu5q{ZnU|_Q7xEoT0zs zH{_k`F)qg4xe(srfpZA-h2vJSR8sxeBN+_rZNQ7uiR%Z$IbS8|p{+5L_!C#1nWKH{y@@1upONqxd^@ z8#GrhQm?}{_04=K7Rq@iyh;8N*~RYF>&8?0WL%E^u4PC1MIMYQ>g=h{au0Paxfy2{ zxtI%bbG`vr`pgx$NPY};aTu}-jp4?wmE#xrGVaUMald>pcjdD=`(Sr@X8Jf&Z#DYJ zlTb!3%dI%?0aH1<_J?}wP*9zBgE!%GJ;mdyzA7dU~$qoER<-LThBtcu#+m+?#V2<@wAr zoA2!|JcG8#dAf`D-RE-d$2$t_Wv0N6z3 z`|t=Hsy7xnBL*PP&u(LF)l=1yk$qv|DSC{N>;`4ddRx$4ic25)1jUJJ~?Unqb> z^hfg~WKK?2=WNNjcP7T@WzOfUoGcHKbFSvh$g_SM3gZ*~!pNN7hCEw!ID1qT`B&uZ z_>KRM2KxXXseb!7{x4e;p`yr^kx(Sc9%UpnwUDHNR4NUj&{9e%B(rFsK}jkzQfbnZ z8SR}kJ+F`Jdb)Z)@6Uaoqvzl6ch3F2Pua%`@&@&-oSAgBoH<-Wo%#MhWQQ!yvz0lI z%gEWi*KuFHoV_{ovg<6=nYnjiP!3W@I?8Vi3 zg|HcCqZo1?oQ{*R!aZI28P06=|7PDV*@^4wZNWU`96CVnOyrz7gC}AM&T##ER769} z*Q@KEQ+XJ&Zw}Dki0o4%IeS6Aahk{v;8FL@V1MuXM#%ST6~*B=RIe(olaI%1cu)N| zcFTkLB+Nkes^|4?mS5zTaW)I>wSKJTFlx_lYFlqcd3xh?06 z%+7UTa@^Kf*;1iXq;{#?%YnmAIu8rg%!@dvJbhwK^y)q~~F`7@jl)Xi~> zyg$FftM~$3F84-Dxg2M|Tg26I6S7B*cCC*56b8wcqMCdI=Ns({xsrSwc4DOZVxEJ$ zu>@uGe&U~aG7gmUU3fp2RG)^{@|V0GUF32Yg6wp)dAa__cvBwFJvjUFGP%F}Ev}MB zaUE2aEA!#p8q?&hoL%=ad97T7FXr`}Z>9R&L45}b$<6pq{)CT3_R;a`?0^H*+wqtB zQDi>moO*~ei{`qPd6pez7C(U=df5wp;G8Ed^rj+nIA?m^`TYKGLFVHmyyg1W$gJ(7 zzJia%IcS7ddaeA~%+EW~4$mQTXB>LT|KUYs?yW^3xi9BDYKtG_TD$=H-Q^wqgfrvv zEayzgIaff=Gm>ZUM!6rdC%&!Dj#Cv!$T|O}%Ny{RdJ;dunNc%wqTC#RV+H2wy~CML zBamk*=XRdcW3X7A-RfQWOZ=?PJD0iIPF)zcVvqiVoSo%r&U-cnui+Hjr?;8g;|uv@ zpcSwBcFUh$7hLzVkRUC41>8&hvebes;WH_-5pp&byv7tfxA& zKYyK#oH0-09y!lp=2Ui$(VR18vTK=P9dNLmv-y2)?OJB-mE00t_2%$HxKPgdkeT}< z*W%)QIcLXuhX2I%dLJRPrHZ-@Iwj*l?5Dn(|G;&*=Fesh{4T$PzmeJ5o@?hG7#JzG+-k(RJ3_e$%#6MyuGLO&I+sm1I!}(V%MrU<0w?rpNL~z>!i*%zELKjCw1!mG$P!)hLb`|@Y_T7HfXK{>f6%FDO%37m89Wce03 z=iD~=P(Bk^$#?Ld{6C&SXYtWofeZ66^pSH8ZIGABIlJm1JMnn+O1TH0f>QWUy@a!i zuaiHNFW_tWNq(8LJ51we)F1IO{sbT6LR3am>~?K4F2F9FqjwiRK!1FX7xV}4cRUEk z%h}m>riYtQLw^sx#R&ED$X@&pchM_?)8)JP0M41Q1CPqPum!X6EwTqK=bv4Fm%qUc z_)VR2cs}pt{`@aj#cT2mERmnZaqY9l?#qp_Mek|89r<>DfTy6Xe!lPTke`-s=IZJ$oL#UIpMboSIa^zZ zr(&IZ@5j+-jzYLdKi_r-@?q-p`7bP!^Ier)D?3gdz3F&X|4RHWSL8K#NnX$Mcpg{A zH@F@7wkfDTLEg=+xFC*4clG&v4eBBPEhD?@V{$oUFE51b(ucTrnBJM(0FC9rd@3K# z=W=$Bv$+~N=yk-?f78j|1 z;;r~hzJzOFjywPZQ3wCx5&hY`9QmHep7n;@Q9pb9CGwqUjo#Q_?_k$T@t53L?*?>0 zzJE&V4MH`PK=!2Hcrx6wZ^z4ClzsFw^(f8kh1rp6$_Jt)R=ai^Z|6#UIXA>`j6-ky zr$3(8qqIN2iC@v1#NTrhK7ccGhau;7H~t}6e-@YJ=6o|3;IW)Jp5JfIp>cBNT;@cc zy(P%AlXo=lV`l6*oV|wjowJ!&c}A{LXK%}M^cUtL=RoFXJJ<5_-O2~4+i(x${k&0q zHLjI2Px6k=;4k#@v$}#`Q|DZmz@H$qX(aLtt#UmxssdWM=Sr@RKQR*}UC+GA^LT}N z5Ar+DjGT_l#>}Ih{5N09IfHldQONHozxRKTUEvSTY*@>gnIHKx9XZc*_O3jy^-)Uy zS3ZRo;vN)GpTl|Hat@W}ThLuUyYB+dPIQ-ETV(zng#kDMnNMH4=V`ei|Hpl~2p`Hd zT+exMB(kf0z?pw{q7Npi??QIBJkJNq?K$VjbonE6LNC4Hyqrs7syqOhU-R%1hN>?_ z=1TU6Cdho)!Fg{7;R*LG;v#qy+tir{D^N`RF!Ii{R%egOJDGDJ^Kc5X!}jM!sDYd{ zBX~RNqN#fy$9MAGyob*~Z)AqA;6;2MUlwB{GIw_1RGfx>sfi zOVI#VyY@Ptgx7GN`g43G=dYKs1&1Je*BGveLy@^t9uLd2cpYcXX9l)FW_rF4GSi#O zm2tZK2yaA9WUu;HFXzx&9*cUI>Dq(Ho}IaodA(YFq@2C6w0wcQ886@$RK~sfkKr;r zj|22J@N>Kh@5*Okv7EW|HtMMl!*_CaxoMm|pItuZoWJ*- z$@l1G<`zbFzGi%)YrD7s*G1<3Y(9=xVgjnD2jPG6TPTIh^Xs+lM-TPETn^pj8vHjO z&Xw@0+=XxDDtH2C;zhm1JOf+tnED-L-an)sgQ@DA@A>w~oW4(QFpib4zko^;Qn_yhCNO0PPv z=P$S_zl?+AAzY2S;%NCdWN$f3-5Yh(CHMo3MqyOY`w72ein_J?tMV@Vpw|^|%0FWW zu0m_Q4yYqP$+bA&WXt4Ra0!Ota%3O6gS)%uZO*smMf|BcXaAS-t#Tvml}|=z`9I#y zXJR+Hx%UXYR`MwPgEi{gFj#(^7w~l$iidC^vWpkMwXQc-KgG@X8ScqTcrw4p@9{v+ zH^@5PggN+Le*>~F-N83uB~C(d9O9na_ze7t^>_-;=|96CBD-BDTqIXSb=NB4Vq^z9 zmXFb|EcfN?R%Q7H{VQ=dPD6IXid+k?yZ$E^#plSrQHDG4IM=e1Oyp&_86%K={8B#E zeXsIp?uY#Mf^*eHk^lDbwB9?&KH8S^jZoUPXHiS8f*a&V@Cvd!wRGPDI2{-3J&5CQ z0&3$E{c`R*hTG#^y_@&}&OV**yF&7KY{J*r&-H%1g{vXoQ3chX$@vEONp8=L)xXKB zagCh8)0Agpsj?HYPi5CDs5cLn=-tnm4ViCa)wA%eI%h?mkJWPC@3GjZ&cELb&hH}g zI%jcnbqzVcv&{F*i+A+$OuoR`BfjGN&R5_H(_=gA(`}f8k6Vho6x5yAQHo72#9z&*AJ- zoB31q8073|h3thb)NOE=>v=~?B4^Y%&a*NQdA8@^FIN7Ae_@QA&E@%QycE=# zu{-7LT5HwW5yq<@mKSio16IigA-mi^dNc8@dMjrRltj+y%(}jMqtO7F135!7A0ES0 zoUY!CQ_%;d^)iDpJF+)^pqH63U%w*S$dB@T{bhI#IR~@%{jQ(Ce!za}U)7maqts8} z0Qqn{gSXXfIOpJHXe57`J|0jvK`(hEGM{orpQ!hVx+kW|nTM~*IpcB$3_xeSE7Wu4 zVaUA7yxpOf-SrcG9m{d8e$I-^Faaa53-93_+<~9n*AQY>X59V9B80un>x-I%)4Km|T)1M(P!O8Ngdu zxA-W$A@AnOoZaD1l*MQGRPS~!$IW;r{*ZfPI=)dKf(__|dDx@>7TU`9qMJO9Cvwh_ zoV&y2`#ERQ8|bLc?wfg^^PmAL=^cX4k#Cvo!1d&uS9iL;2034*^B!z-?H8_x$@2cV zOU`-mkbIt;v+HtPk489OzrO27%h_Mc^9H@nI2zBZdt*Pj9cL$ak+W|cf$jQN@d*4Y zAI9ssGyjnuvST*Y%XePRsW0S@kX@x6+H2pWegUW82pq5X56;9S^$mDRu7%3T`ME{! z5&0W_9_L|-`aABzg|QApkUjoE{ha4h*x&nh@5x+Ev5njFBA$uWxD?IxZsQM;?~N~c z5Kh+5x9-z&Yl!>yne5M7<-)EnM85kA@)Eu5L=(A*dL*BK_fc8D92Z4B`2b$*o=x~# zUW%)6iMk>;9wQJ?N``pMOI?xfK3Geu&r6D87#y^I$xQ+Un=HCN7k}$Due7*-szj<6Zj?1?6F= zB4>x|z%S!+Ohf7X8E%P9@_KwASK@(u3IB@E%P-->hO8OYh3XSje~o|C*c_aV>3 zwVdZUXL}R%h3ejT6sO}&y|Xw!*ZkbyR_FZ5ysRhZoyxP6ee_hl{2mu_eusUK=Vy%i z3GT@qIWyx0&Y5}y?>p!9=E@uK4ZgyAI9Y!tN}`qfU*TQ)d+|FmNAjL^!ZFy1j(E|v zWqdqV$$7?)Mc$)}uuU)TVxH?Kk=^nUb>5%6f7wOL${%8f`fJYl+?F$!@>ib!emtE& z;u3fZkE`?ijpdX0VGNTC;4$P~?}a_Nrq1l0%bD$~aWp z|IDN8_$PB$y-7TYZ|A+-motmT$(bpce69-Q}bikv-fAU9P1 zgS<=G-ye}r#76aN*o6`J2k&7TG7n#I&k&x<2XN+P1!9dZJa%~xO_F1q5-mx^y9X!55vE5RW5>`<@x*^ zDq)_wJ9=R|zSYZ_`Ir1P=bNhnf1tjMKjK>W7z6M!X6esB8~IF3ldGYgT*RNxJYFIH z%!T&aN;{u7%d>%=WJG^O%jCQ)PH1PRD8LZ_rboh~oGI zo%NRE0Xcin&Dlef0(MsW?%72Y29ic7l~Am0a9sIxZ`^=-`)9k94t3*?JK^NPr-h28JvK2n2aN^-nAn+yZAlw zrSe1^BQM~5dpykr)kSy;*G9g7vj^6c@6`X2C!hryt5@@66he25*4v6wsEAMWTJqf} z;Gc82`fiLzGnCWo;@Y>^jWsB!ca>`cajZO;^KEveYsbs^<=jOFab+Hb^IV(9EAYO2 zG1uk~at)>N5H7)O`gQP*T!{;!oO~}f;az0E8o-TF+4ZG-Fz@7i^VGsk@~arHpZ|8S zUH%@&srzFTo>ISoFXZf5d2Xg4=VUSTm2=jW!bfO{yvsv4@A`Z465fNHpF8+o&Usyy z+o?0_uEqK4zj-?6?8sjBshnr?L;jB|Am_(sK824$=0x^{KkzzoM!o0yIhd)wn$Jh( z>ss}%$UJ)mnc10-IhV)DJ<&ui!I{e?kr{Cj@+@b@{J=l*ecS`j;2(8m|NKMcx8!rt zQO|-iEf|>XcXW)PO-_TZ^ zjOW$!(Ow?Ie_<@Tsb}z;{2iu3xo;<2AY3nBj^E^yu?3rOJC^Hz!0Y%c)Waj_p!Y9l zFKNZkqlsSTbN1gm)j4JHHB^!F-2Wiofk|>b z&V0yDo;@PF@BR9jl}~Wa!=KR|FCeo!=U&eF*W{PbQGF$k;JiD}@?vhz^|?IX#z){J z`61Mhw_rLRROgI5n=7gF?&VzEfPu)o&imX8IS&TtWhdRtFClXx@8dStzeIcaJkI?6 zUe3(VtjV6WiyzZFgm>p2EJ97aoV9~F-v}S-&BbRJuJCHha z+<+7H#-gx$-p_Sphsl1j)Ah3Im-%hp#nt#+OvDOxZ$5%6^8r|i?9;8ej(!*946B4& z^-AJ8`FE_u!|Fc#A|HejSg+3hbc_6^{0?f$lW`#yslUZe`D;{j?Xa_ApXD`?sIwHIbKybO?`?wyT~-T7+RnuCZIxa ztvBC~>}->{v_CV7U&Iaa9a+P(p3h%$eRP#?#0Wg1E{A0psP2U8@|)W)4yrJo%*`%epb6Ry!O&c~sQ zyqDj_zjF4L$K+~q9ljgcZ!Y24de7kwIlI;@v{1NoQ1(ykJIp# zYscX#xh}H%?o}_rcx3aD)&hC36KY^R{PUq~|T{*kr z0KJpY2u1WpqMzIhry_gVtGpW7Wy^3!eu}dro$CIdf){I0)^F$g1(Gvo{R<>#N@ z{pa030-e-(E;7GzPGz=c9+#CLMP^T5y*v}yC06qcy;0a9FThdAIr4(ucJ9l0pYr=2 z#(B4AaXUQWTAqzOa|_j%AkXM!_yKM8H*wD1$?_w3Oujs#y?Ek}D`&Hie{U_JKX!UP=Fy_j~Vx4>de~iqYI=n?MXU1+h z?`!5*&YXtmhQj)D*rk2DV`fm!!%F%?xGlHDN;$LQ9r;K3RIHIpBJ+NzI&-fjzrfd^ zwmcWtU==p%jldE)^Q8fL;0>Ij_bGqOz4$(E!T;heWM1Vg`WLhCBx>r<#7nqH{Sb1N zW#0b9rPN*dS^f)?Yc$~A?NC1o`LoH zOL3|E9=GPL+z|sXR(%uq#{%rewYX5f7T3US$XQ=i?@c*7?yGWrtjGO&`Nr7A&#ANb z_mICwS$V47BKc;{9+sVPkNgl;Av;WVxxwhT&yc1RMzH9HHAs$!1 zjn!C-gYc*RG<=Iv{%l7+hEH>CzPykdBfDi?K0)s>p1}X$DfwVN2PdM4`!?#`Bj15S zauFPZyKp|9*6+<{b7TCDLopsV>rdnd&W6tek40Je zN=%YF<92xkUx4G~d-!YK!1=B}1Q)66;WNDvcpp=6o!$-nH^0Q^a~s}^!I+MGKnbx zv~|2w-Ijkx-n|)|=dmzyj^>@bf^%L>%YB@4C3{5Po1D$9(A2d_xEn3?^W5);$?BrW zJkNVqPA|`HW=%n5-OXQg?uh*CJ9Bz}3;H~I}x>%t1GUu;rk#{7sB+t}e z>b(C|VOYtbXQbcJr?CIe1=uFkgf><>B}Nd9U_&-xpjQ zmmu%xdiTxdx#2(2w)36+`xb%=$*ioZQRh^>RjM zHuP5KJeY&e)erJKp3mDj^YT<)fb1Q4crs)E7LRgI&WAhYhmrUGDlUucb>FB*%ZpJJ z3)R^pC&+on3#tbp`|8npId7EvcAU(MPZhJU969?(qLzMURo?w4@R43={sqPHIBrDd zZDwfpliRRGotbej|A*`KYVg%K4A~zZ<7f4ACicPg>Mpz;nZ-}5a~_-}zben;zE~<( z(Lavcst@9Oc_hEWd(Z-7aF^a({sWI7`*(KB%=y;1&-Hu{6yqN;&^^;pSZ>Nk@>-t7 zMX(VAkh5k9zw26dh!5p6)|Qa}KsgHMs(oBj-fU zf!SP$|HWaLf)RSpqC2j{F?wThx}5K)GkGTt)yoWTpqDv)joePZ9CqL>^?`gpK0`y) z)cYFaaV)a$UC!CNvy;9rKaP{#yPkVAsSGl{fPZSSvq?(ef^Ch2|)T!tT#*(ZsdJ>fQW0=R3MKuTytN z_L((&jNW9ftbRwXA#X(mT&~VHTOB#y*k7q1lHWsBb!9#stMwnm`MHi#dM|L!+hNH5 zvw<(v8;*r|554qm;}Sd*H^`^(lNgGex%vLex8@bDwZOr8Ib)l1&ga+pLcHYK&ln-! z!iR8nf_JzFI_Z6Y*7804E*_Ax59J%ag!)^w)5|XKQm&z*-VWZxH*)shi@6n^!pnFV z=ek~moAH1AD^|+4^Jf?$U%U=6M!Six`;$#eg@<4w(I#l*XR69vv;lK{0tW9t&$gTTV&Q{ZshlocWEQ?K0ksx^|IsS z{K>nWS^K7(=Rd#GJd1@mGxR5{k}twkWahMTJ=KYSyv=P!_VwL0>g<~+?im3J}ca%NGUqoq6_mGye6b7p;noK2mPnUFd9 zuHNOy%y|T#;D4^=897Klvn4aHB%Z}iy|yTc%;%i5o45}$$8r|DsF!)Ozr0QU2YH6` z>_4wJ0^>0Tujzftc@D4Ob(kn$fJ0FnnKfVQ_u;=W8BK~AIBJ<=3UW2B{d;STQ%jY0F?s1&I?$F=>&WtM9jTP#4$Tv(W zJ{OrA)%j+=4%zd*S1*$PMiuof`~foO4sopuPL|K)yLc;S#%5mT8)28c7}+D&axYws z0_yJWnI>naY9tp&d-ci4?9W+nlis6T0N=~MqapsmUd+(H7Z2kp^+xW^HP9FrtIy(` zLDTq7e5_ZW%U}cUMLoUKF-1O+FT)_*hu8JK!3?Za_r?PGD9*ggob9Dmy`WrHuFG9H=g}VSiR1O=A>XOr zso$4xKohwdHp?GkoZOVJ!HMXm&Ns+S>GSz|73I5-UE?@)cljmUEnl8KhGDe2x_ee~ zJN%?qnqS0NIp3c-Q%53aTpKQ|zaHJ>>}1(huTqak9W=udjCH*(zsgtPXZc&4hKcGc z_*Bf1J7PAjRafTg_-_19egU70Ex8;z%1_`DxfA!}+USqL>LL6)pN~TRw`F&@Qm+Zl z#Xxme?$6olPC+qsGxWfD$$CpTJHb8N0ngz8e`Xnae z{D?~G@+c&qjhXUq{3<_>M##6xFP!gygI%j3566>Oi0rx-={J@iK-XJ^~Qx4AxoKjvCkj(?GF?NKrIV6}U4X62k{ z#gq6ky~B`aKJVnsdW(=}?ol4inaeqI3dotuE#&M%S95+R+mL6uxB3F^jQiy8kvWpR z<`MZT{qj7YGsCxV&Wv;PUPb2RUbI5yPI1mW&3s)b4^0o*-6rrDy>&blnXNek@-Ez@ zzlyKtoB2e}yO&v%XDYvsGmz)7ItMIm*y>c;UXKbX`P`-k5K4eDaz0bb+ zFz>`&n5=&YW*~p9!5sYM+AJ)PbB4aaH>8iu*Nv{-8PsQU=GWI;AMKI-A?IdO*Ymvg zl1uOyY()0OA9>&VF86W0FpuMHxYf1FTmgB%&Q*`#JZFz%o!k`fA~X18y$9qKTn{tl zXSpFih_$(foFCKlyI{V&mFMy&ya;trRh?PVlmF&F&`d7F)j01#W=dvSp52^v?d8nP zL2BB!_kYc^kh3Q$WBEAF8GNK%U%m$i z$b&e0__zG5I`ix<`A=kqPT}nI!}PP~XMfDO(H0*g-xV$Ra@R}aO}Q(w>n%`MlKWF9V~$aml#jsxbilR98PyUWx}G`zI&!XT=Qg;>wJ-1} zN~j-0&YOY!628ULu1&7@ei3usb0faQsp_2f zedKH8^SL}$%aB<0EbK;ifBtj67uVrE zoQLi1dmXiKxB3ITfHvw&agclpzk_^Bw^cuboP#Y<5Xb5FI^XA4$o=(4;c$5rD(kON&y%l4cCwLN8V}+;G{C#K-+i0-cCNtx zq6aFeKSpz0tge;o{yEu8pHjC|FX11#E3T5?;nn;EPv@0f2y<|P`VzGD=Sy*8WEbu1 zS|_;+ABbObFP_nBhMc)2)FrVKov;j5UE9bn@U{FSXOGDqbgR4@tI))?J2~fn&h)(Z z->R$QWW0@S?#cU@=k#~<#*H}1wfw#hL*BQsybSeF0xk5i%k|>?zTVKgmCwSNn5M4F z+1FaiW6)V%q4$;iE$4k%&m}qU`QgZWzYsOuyO;CV&-(K*6OXA!<1FM%{GIR8-;P!| z26OZ-;_O_f^G$dP&GqvxFOtj1fAD6mgJA zeIVz5U%4wzbnP!>&&l(fbLwHdCZCV&=mR+CbX|VLwV`|`XHWk;_u^l?fg#9ww#_|_ z_z&L8J$Wv2uAi;WOd7+PMTh8bk*`7r`8&KT=N(^)W7V07=c2rNEuNHf{x9bC>Uzj~ zo|$%;TtogzJ{oVTPr-aS3*2#(^Cxri75Qtvi?ffMBj>%If#T?aA^Nj;Gd9W-k#}-E zavrqBE@VDu&$&mx1v39$*2}w}IW!d=)p=Lf$rJc3Opq&cS^fyi<+a?=pF2c;07LO7 z2I^nVIn(dsoWI!}a`qhTTE08J*UKFHMZEz3s56_l$`z3ra}-zLa=1fof+F%B93kfn z8;5hT1ew1%e`mRtGdtf2r|M<)wURT#o|ngB5ONM&f(P`w@z;De&X7;%Rh+Y_n_L9L z> zN&ip24#(p(RMh*KZ$Q3x&f)2LZIJz~lzI{7sB`W&m8;4PxF_F?qtIJD9m_EZgOT0g zG1qdwWtTopUWrHXpWX*(jN$70IA_!qa%VJBe})yveszN0MRI=dr&7-0_j$a!71!V| z`BR)I_u@4uApebb<(!{?;{w#zyBqm#$X=0etV7k?UAqZ4BIoaqdO5Fi{#DRxjn{Dt zcBIzNcUx)q?T>s1`}===@gfT9O~s|SSKY?NkNG@xQLe<<8y@0cafRN0TnL5bS8<&@ z8^>d%x+b^fPjI9>7~OH9`X78L4?wc`ab>}WAyTDZISc*7vVh1 zU+d>N&(Cau`gP9C&ChXyIx{xE^E`LmkasITgPwR7nG1RU^Guw>&F~iXV6Mmgiq6N#rGr6VSdgF z^?D)Cd3zqohjV6I&br4@UT+A_lAp(8$h)6+xs06Il6R^f^1I!sznb&T_LFlSPM0%B zat3^@&hI_vd7kZG@C5SwosApi8;~=)9h&OpotcU3N-wGNtYo%lK0m9UIhg0YZ!$K> zc_(vb=lRTBs)q06IhgEP&YV1hdEOV`we(!eewKHw50>kd#xcnI`xP=vW+F3k9g3r- z`d;L(&Nu>j@0WAl*-6NI))(19TdMz+Kj5-hgY%K|INt*k*|_$a+QFjRh$8)KpTFg`?i z^(-ERU*ycXUUE%TmYecHJR8O2dyw5Bv%d%5hGX=U`_AlqpPZtXd9xXPaHMOO@ZGow zUn1vN=G8E~BG<)mWT(E7^ZiuIwf*E8JeRZAo-XISug`blCcXFZ2;Rm9y)Q9Aev)tD z+I%oFqdr&ng?yO`5Np+ z1^0Z)6_MGRS)TppPyM@4MgMx#s`^!~ zjI*#p{SiOGm!XaP3IE7f^WnUTv#&Jd)#{H?LjDZnaFP09&hE8RE+lV6c8TJ=2AAo- zj(n#bq0XFN!iS+H&QO=aK=}szfz#A2@PIs<%W^~h1o?*hmao+-iRb0cdgFMA`X4@l z>+(-{0Xg?7VTzn@(mLEkeIXyhkMdl!k$dxX{67q*a(owOPdZe-LoO^YlN-p#b7y3~ z+Mk>2wc?*}61riAUR&IRztx-gJpKSD$uII`p2n^5ubiFtVtzqg2kYg08(%Ijk*jk( z{+&-mTfDE%H`;u8p!^w5Mt3}fZFmLOxIUko;Sijzeu;nOpV3IJ$M5kYm?-BvIp08A z)px4%tvyB_EO$gT^v74J^>4)>c`?Q#|NUcZdU!^?3BSwF^3nV>-^LZNP411ie+PJr8{=toas59O(r=3cH^4) zkY`{6y6U&VIy|NxnI7`o9g2U^(zWk6bMs)%88uzbUYT~0P_vC%d?=J6tF)oapkC}aWcXDp^l5>V+F0IzfO!{8^9RGl` z<^G)ari6R|zClrS=5Ef6bJbZZb z-{KZI&wu7y-sjW#2%IA48J;QMjqBt-ocC{yoZa*uz7N@@&)`}33^@mPxUVTnqnzGC zUdTCjF2~z=5jlfC$2$30p3m1JXL;sZ-oN7d&G=w0$(hUd;T8D_+=xc%_GlsJ9LqUA zT0NR`_MObX@XcI|kKr-siR?<-^jgZ(I5VjUGOKciXNSF4Jq7pc&Ed@3qp(@dUb~4q zBD1$RzmKovEf^tpVz=$vRm!M8MkVANDWIS4pH`fik=bxG@?G$vI&(7b_kTQFKl5y= zoSpp%?uEbfUgJG@Ri4C)QNlgvsc(~a^IhnUe7E%BH}zMdmHI{=sCN-^241GlSvQB@ zK^Ogh_;K`=+oGd^u1msE+L5E7dEo z)V0j_f^xnCa^{rR>%dcZt$xn1%=A_AdwLz^)8x10oN-m;a`M%fE5C#5<%WDD=i7F% zyi>l6=X1`iukpG1Os?#n%lTCGyL=NbLQVa1)UD*f=px^WjrdZ1H44kG@diEtW#ybZ zGvvd$2ws-AaL%MQ{+Uzdad;55ksas^*Q&{rF;?D+yX8-Dt-KZ=$bTX`_j2_|a=r~u zmD}?pTv_h{JdSH{B#y(InBe+QT!~HSiAVJZAm5Hxsk?DMZlG6{v*W%lKP+$O-uy9M zz!G)-s)q&g>9|vVlJDU`Tpi!b?fHD1gQwMn_$3~Q-g0Yx2HE*<;4d&4_q+B1&&51> zJeJ`w^$`4lTh$BrEZ)wapcW2M*T6|~BmRa*@lV{Ef5ydFjdyXpes=82TtQujzva$+ zs{YgR4&19g8J+a<4fLPhOPGL-sP0}6= zN^Z=naI8Fz3nBkq;XH1L?5WvL_j5hp9o6*ikUQ`I+>BGO2qjS)-O&u$Q?q|%mwVWs z8L#&#*319XyNm9WvtMSnDx*wI0JdFj>dJ!JDd0E z201_X3G$ho*)oaq442{AoZs2=a=G-7_ao0_et&uA@~mc0x<)@|KxSD5Y*rtFoG&xg z&+-q*vs?-}<1#1njOE?Sdzg9hJ5S6#{0nmSnR+Gc`NJmB>4KE3%7zpw4@r z_vtmvMNhqx@Q$4G>s)!coM-ra*!~3A4dkU+wuhv2dz3gkPkeO6k zomn?So*=iymvYXa4>3x;g@^G;oP8sEY%#epF2dg!>-x=TkE^i}LP>*afBB@aUb48d!zEkP%_8js=r=!rkoP0$}Z)j4Ob;OXjoQ`Y5C>Ye-} zKfuj-6N<~NaI;(jd$3%+mK)+T`Exuh=lkPF`9o~S2poVfk#D-}@N48l`C~o`=OE|O zB#gtG>Kgni#-gFRDtF*2T!*Kk9qOv*^Ak8rp2bJ==g7{N^P&dUse7Xm=Ho!Ug}fE_ z$~jL4^PlRj@>2Q#zh$<`7wi9qN994h3vc6f^)C1$`@Rudb9U!0dO5pVsMpI)coTom zJ9sePf$!uec_O#L9L&R3Y``)ULuvQC$j|X^^uXyTis`uAwe0gnz>@=tH zC+fHO5@e6rWq4Ztl?(7hX2%mV3E9;yJHaAx&K1F zYw*}!Ioz`E+b`e2H!2QAW%+FV_fS^-Jm1bA zp*DJ>pWc4%JA$88|AuGqnfg*ZDObYjI29xGs^K;id@&cy(| z!}$x0L(Zq={3+I8o$Ft5p1JHCGdO2yea@^q4SkSjq6cy&mT`a1<&B)5cV*7+{zhaT zf3BCA_8PJStx)$#&Dlpv%9$qz;W)XKUY^@y)a`jRW?+JP7;^sajj>8U=SWx1Jk6Z! zjg`prcDQ?Lx}NuKK5kGi#I1M$XXx$6?a@;%!>#d{ya0L6vPagGbEcMe?IbzB<25L;eN3ak08Jev}uWF)l%AJccT+6~fJOX6P*WW8~Rfq<1z};Z9^P zo2=gqwd6iRXz+q^u|C8%q%U>Yx?gcXow?gr{u90A zk(_g-Ag|{oyakhSfO;mLl5=hh<^FgAwf!@<>YXAN;J@%7PDakxZ@C=a#x&fI0`7f| z-$5PuB%C6@iLrQ6eJ$_ceE0qjAIcx#Rs4;6v@3FU{Y{+x@NB*Da?a(K>+o`M{&NX zrpWmYdcn2D@*lWP{u%R-|He>{E8}0+N};NJFaLse@*(WDedlWtJ`m^NMAs%F-&)u5 znfM>FgBIgXXsgwo@5d7PBXpJrqlcWm?=G%~>~2QCn`x&+syX6m%U8b8l z``podlU@V)KY4|GJm1C_qM`gg|HIjbw()_Og9~sRuE9g@*^WcyDwr<6&yBbuu0wXQ z!8`)l!Jg;Fc-eh5_+WmD>tceu2${dHa^}Q1tU}(S>?76WyyrP<^Bx_Et?HY(KF&sF z=xdzcRp#q$*o>a&jeqg6Kl2GDV?Ija9Q{0?hw}h5R1V~;aFl!>|H*xLf8^}1q~1?H znKQRkeOQn3-KLFW3B7i zVa}4*$i1-+*(1y9^Yj>zzKLoU61`E_q|tL_*A|8 zm2<2newSZB=2PZj&Zqg9iafu^>3=LA%U|IVc`;{?y+_V|_>H=U{0L|Ft}54;^FBW> zk3&^;&hTgD2XH4|ME3f%{G;oiAv1FkM(bV0FCp*d3+mr^3@^vexK^FHJ%~?KHw@|( z>IQQ5<%#?{cInOKbyzNEeifFt;AZ(!J_vbt|3PNgM6ScLco1jyUBS=sC|-p#<=?ms z#$q-G>Mh~dI5R!-Xt(+>{DdR$fL>eP!sW3Mv(%Y~9q@*nIoXlB^8&qx<$SZ`T$-xR z3@w8-n4|uK$D@GU6yM61apqsXo3axo}3(EI#=G8n5QRh43Q7(o? zXpGfp@A@zv%1yY2`*M!Hi6`+5#=4e0q$S?Qxq8cRqMUi0Gb}SXyTT^@oI5%1vzwPw zmsOX*Lj0}X%{g!S<3e>S_uVUBjSrDsqZ>EJA+BGCll5At|B_3fu(}yPu6HDIwq3z< zae@9y43nS5Xk;HfRxfAe`#clXknfMzU9XPM)!BpVarU<dvU5z7rqGukeRlh%Z1n)J7Rh(|?%1!2Q^X z8}za-cfg_Yi+WY~N~V4L)CM8nNsn_a6x08J$MC0I3J1#BAG61wAy?t0JOUM6&u)Js z&%=9qL-}bwgR=+bjLz=UfQ#xq#KXC&UL`(BT?3`?D_ZMyK^xcmashsqvnN#M{Exqs zPWV3>>;wGE`R&8_pQdO?Mnrp4X-J{Hl(dbE29;Gdp=hb>A|sTBN+DaB4YZ6xh@zq> zGqWOjUhj_MaX63jeO>pr==Z(v>vP@l75EC-Vc*qz0Y%g&aB<$n=V6ar16$Bhy`Jyn zOZYkrlE-5w2B4wdNN$a@(Lvo6r{EU#Fy4w1xCRUHGkUw$0^2b{J(uV3?`SIji&nTt zJ%%sjX*_`Y@EWW}Pj&XFC*^Wzq3+2A(FU*L3w)@52lmRJVYQt1#qRPnIe&LCm)}u; z$M0b?N@J8>-Y@Rw2i1*{{kRH#laJQ#%=P(mz8BBRm-6AM ze4dN^d2`>lP-m{?T7Xz~gfM zyqN=?)%i2!9@vfiymD{md3>33ALq}IXKxQy$vIc^z2~uXl$^y9* z>Z%XKGC8|gp1<5*Q`P4n&)d5wiZ$xj`F_-pU*MdRIfomo%c=AIf7Z`@&Ha`$X9jYA|KeJ{w}a)O=!Y}a`9Aw2 zyIr35-g=q41<_KS`IVV{2ChZs#75ULr!sGHULGZv#ct$2%G|m^&b-*cc}DZx=Ugm^ zkL2roW+mFm-=s#)VE^vhMe}^VCg=G%2bq!o@+Q6KkhA_VJ`!{Av1_eyyu3fx<;>X& zi9_VfvCN#ln5y26%)tg2qL+D)`FIRp zjO-_usdFCQpk6AU%zd~JD#)4ftN2Rnr}rQ_$pbJ^J{ozC_(=UF9>);9@A0;rvu_Ye z%WojFVH(O}IkvfW9%mnTST2EPxKwW(e}~M{I#`0?>J!jR&OFY!+#%QW*5Gfvgj4js zLQ}k^euTf^QJi_0U1PTTMEno^aGQ88&qDT{HoR5uOKy&<G%MP@ww|K zawYzra~5aD&6B%hv0Rh0bM%#8mM0`2{ zl>bBaj>-HT za)w{)+88-|>78=U_Z9L=xr}@aXHRIv^U=q(50Uq?yZJqwi(YsOKl{w5coW$%vmgE{ zkHu!x_L*XQCb!~SQBrQqzi?MB%$K7Wu2g@{O?f=B6W_-3_0Hmn#-oxj*Rvy*l`*^x*$78HK3we*bL){s*aFAXF&hUGYu8{x1U3eSq^;Y9h z`8@b{-+o?RxEROzb+uzWxN-^MYukn#Al%p{!?#7 z-h+3j*W(%WQoaj+<6-pFtA}CopZp}(ZjE;IJ?lD@uxhfAt6SPMmpV`Ehxb`jbzE)b@k#AJzeK_w}AIn|ztKcwsFJFZp z&=M2$Mj$)bC!D?O0sSiS?L3Z$;{g5a@hj!rLwWv|$ocaR!l}qU^dg^te_YEwnEUhp z|DI}wG#wuhsJQ2C?^8JmwFmzsaNp( z=#K)(cbQ%HK<=Ubfs1noKAVeTo7|T3oa7lgPrVtLMSGCvdpR=Se#Rv}b36CtEnE)8 zy7)>c|US zdx;C9g5C!H8`)tRxOR=4=lx$yl`|_B<6gX}z6Y71N2+JZnW63Fy4(|6<;J6JvkIoJAfbNxH`ME-!Yw=b3V%IoQ+X%K zAm>9*{gb#VZ%5|*4|otIki9+c49Cg)W4F&-h(qND7>b-3-|#!gT>T1p2kWiQd6##U z?5um_qOL8#wQ_c$Ieb2DMj!p>@P~W`zlI0UNS#?-M9w=$&hcCIMym(nKIET^&>EMz zR)J69PxuS;#w2w$&Ym_xei(PFufRj{(`YQ8g~Q~_FjvklRaMT;e3bfbc{a|MvwO_s z9qMZ6D!+hw$UanybH1O#yIdd3ck=HjAveKgcuajA9>8t*6Ibik#hthW-Sn=+YYWVS-gtZak}0wSSF9>ydOL$pDTCgoaL{{1MsUl z?_JaQa&>m#Bl&*yJg$Pzum#tnnf~>tj8W<(1_*&nhK?Nw(-yh?8bdf-XDe9;H+Ay|!;>TcTO(OJC^Yvdw)s%u^OBRq@W zkX>y*J{bkkR{cD`h3qsR`?aOz>iicU%%^cpehb4;2uJ5L$lr~$;R})X@s6&wLKAiN zs$+P7dM;1m{9QvOydyu2hWgq2zLe{u5t?EUCZL^bPjYsu{C&b|c@*;hmCs~;9M2CU zXIq}roEtd@GB$9G&r^ z&*kU;INBkzEkCP#|M?C}Bll(Q=lq#7hx48OD(7DQgY(ShKF`l_sycIWBIoCj*_ZPv zb38xuvYdN1-+P|je0O7z=dcD}gs**WEoa`$mtT@U!4x?^!%>|3t^yCiO4M~d-%rl8 z+@Iy-6=)$}i_W_p=h7jS062KgL0-*W-EhMXCnnNbu4^>QX`mUC{5=4*K< za^^pVEy(lM7kQ2|*Eh=dAos;8o{R%s8;H#F+~awUf8xxU-0u_R-0$1uJg0f4nxn5; zd*58jeV;w%OU}$5#H(D(IaP_zQD<)CdHDi$RCo|#^IeWt}&KX%* zKhNoz$jp0!GjlueShVn&+juW}>SxwJgFHvaa#5UtJa3z@5lz)?T+bexvv`qSX6?(! z*^#qs0cYmSz>nDE*IbRv#NqruWR~T5FD#EhRrOQ3mS4|#zTW3N`>pXkvZK`2Yb)=; zPsj|*J4`?QrE=aOzT!FPgUsFYkTc^n^;G#jE`Vk7FWeu;;cA?v_cSv5@5OSx=lE}A z-rUDIZ}XnCU7hncXGhNeZt~eEFJHZ;%1%+Oj~5?8tQJnG8x_%Jj>-t%(q zeutb_Wl#cb&;tvR89CGS3(#D>ge&OXAa{|6U_N@OuS52Oyl-XR|Ct_lM1Sm3zlr1J z?4&vK4p6t}#+);uzWlR%DAvh+uv(sn>=2)-^G@?JchD=r=kta9HqTAZ^(yM&@=P2m z&p;#jHQXo{^qJ4)r?6Z741di7_+(U)PvITth=cGhTKjcBy7rfR0_R-3i5ue;T!weB z8E4}|6hT#d>$A^ex|}nrfqa~N5Dt@nz#H-v$Qk_=*TXvfkNG!jkPGo5jF)%tXZ!^| zkjEis+cv&UZzNYh1010q&Bc-PZnpYTzLg*1D{(!t*PfvF55}v{UssRYsMlbyJQANEJ5ncP z&$v)s#`Ww;U6K8yl-@OZd4DV}zk~ceL3Z^gd*0!-eKxT<;C(#cvWu4cidCwUr(y{zyH-(uAX=*TQ~xeclk?tl3xBH~!~da<-ht{_@;FS8yYdw{ z9NpF5#dup@jYH%rxJ1ql*p0LEROh@q{Krl3ug_eBNpe%3%k}wvRK#p`Z~hby%ID(; zJcjJ*TlqKFUc^Z#i`(@EpfhUXA-x%V1M=U?{h4R&9BfqQdwNcO1Z{DzdKKsXE-AO> zoSC`jW+La->%2-Yf9`yrv()(x@(kv_$TM<1@*TC|b8xe~6Px7Bp`4-lbLRQJ2Q}oJ zLzxd7)Hx%6lsC$`|H|_F$T`~_`L0(W_ihi)eV=)jb0FW{qsVueIlGc`&xreWp?Bmw zn-}0{>{kDR7m;W9G`(IpM!ugH;xQbdeuEbw&(43`lV9b0$Gf?O|48nqd?)WAds=2v zzNgHUJTpV^nfwxl;TLu8iJT>W$(g_T&bQ%d{oF@4%1@ydwjj@9zVpRcCQrp$`4z6r z3wS7Jc6Gv~a(O%k;scR9{;T#?^KL-~7d!P&dM;yZD*Ugq3^$eEVCqJ_K+9WX?% zBfh|VbtT+~CF-1|nFZMy2di`LELUegIa$s;KM6IE-R4$2gugLb??>c)BkvBk>(y58 z;dQt}J|Ec~wyQsvk4E<5^Z0SS%$l6rqtR79p8w#I$U8~qL}$IsuBGzbI7)sE7s&lF z5yO#L`4n>QjOUy;PxxG4-oPIqv*%w-kuOBvPo}HOp|rXmiekPxyJq%*W@w-^LFPtoI;+!qnw-(C%<%!rX z&qdDP>{u)LKumWn=ht&QR-N7QEIuCFP#BAl^Y|^F`4Uee?`C~D@52kYB}T4 zzb5aY*T`?mQ*gHY9v^^Cavcni@8nXMqA_@%=xrGe~qPj-|>SOEZ@TI@s-?^C-G1&!)M}W*}waC%|rQW zMRmX5Kk`(xQt!oGa`uzid>AI_Whc%*+3E8RR9*f8Ifp;zyfgg4CnE3hxBE5Ocgm_i zmJjAK>YUf-svkuuwATNPYvE$~P2~M}2S20N1SjAS^wR%{iy%8{W1p#pzttCGr`(0J z8x`Wa)%WrYev7MNf?S`kz#924-rwi<G@IXUw_#n0~&u%=nj(@3Iw+&|ibG$a$0b_yp(AobUc_ zWJa#l%lG{ddBNQ3jhva2 zxt(+TB>8OQ%*g$o-C#Ip@5>yU=yO$(v*KsY`CU<7kKDuk_3~^iozKZFW;wLz}#Qxk^{Rh`W z&Vh2+jI;DN;dpdIp0U5N6M4R#@|o;)nco|c9iss6L1yq=9*XVAY}@Yh@AGTC6h5+V zR+QzuZ!FgPQ67)dsEJj03g^1^9Bz^yLC&{N)hDBfx;X!mK6mB|c?>ekzu-&tit=dA zjGc^yxB^S`ujDEC72m3#M&@@j9v5SZd=4LsQ{@xzsaz6uF-TpG$MTz;9i+1V{z-h1 zUe1i>JPwuga#k$Dt#WppgXEH!gAb7z*avlyz5Z?f!DrvaV06bkT#ox)JCw7tej>ku z1Jo7ML*DNOan7+#=pw&>>`aIAGpHuF;YOG#zmA2-KSy!q@f4Ijbje=KmW!&9Cc)HF81DEdN!` zxpM$Nfb2_)TpJ+|!FO`@g`B77sk`zW{5#IUeK^}^I&(uF&C9S2wb2O==zouUW&iGb zZ+@0@-rmX6G`Db1K8#PpI(Yy-Nyd82#0EU0{~LEg3#?H0!WN9gDSG2M@0YWF@J%9@{cHlx76LZ43@|ZP+o3~TjU$~ zV|*oNN4rVh5BIBY#%%0Ww|4CVd6%3WcQ!wyew8Qj3S?h+R(%9&sq-#&Bezwbi9&K; z9)?l!A((+y*s7Nuwx#^DoLys>e7gK6S5#lbd7r3*()z!l1pd>n$@B3SZp04s)XzK0 zlbm;*u6&vPH{22J^?p!~l&i|W$nE636K4;dtFDY~dKcqz`CC4Vvm;gKtJJrlGAiI! zy;u1>Tr7XiJMfPDG`Hd2+=93A?K}#t{C?%t4dwgzF&@s(b6-9RpUK&mFOw(8+4EP+ z=cAVTPMoIqrMf+iR-eu%pu1d!yYd7+l$+u%`9f5cU*kG_8d{K?oi{pCE92V)VMtFOb= zXp4L|IiHV_vvYi}&dySpU&bE&p~wuF%ReIXdo9|zzLG!T1>A~n#{bZdj>MPp6`beh zCN9JWaCW-PluI1d! zv;PnNRcGfJ#E0Nfy|?%Qu7`_JFj?<3ev)%WW-nc?z6ZTg3b)`a{DO+8?z1I1XKri0 zO8qJShKVSHjVR=LS=XMGr*IQ&LFQUFJ_5V(gn9&a$akWId>YT;({Y#Ff-gkQz0BQH z+(zYJBaQ z*1jEhJ^!bdvms~Ag?u~r<$*X#F2}R63mw$^@e*#$3-B50VyxbC_z)kd&%kz6Mb5OG zZF}S&Tz?bU<8n6STrI(eA@2)=T)RSk1-&pJdd1xNBhi~_(na2KSR#gXpx{EBEB0e(%2WQhck<`{ZW%C0wlD!N;Pgd>@bIhxtnWl3zw2 z`CHCj^&oFkui?B?t>&8Q?7r8_c}J_tL-bC>N%H&LjuC z7vjaZRlXO8$=Ok}C)LAF_3irEzrx%F7xxu zeUR@U_gC)kv7DKad**Ncgfm-PW1PAU=ljV0m9rqT=_}5ky|LZ{ob#?YPgCE6{5&&X zGIze?H<0IGIA@-`j67$V^#kSH^QX&0_+WKr_)=s(=RVGvm+vq4eCApy`3Gc|%CmL3 zJQMj@&*#jo>@HvOpU8dNp35Wm(iG&Z=y(TaQ0Q zGfYNV+E_LQ)=J|_qOWY{8@Y$Rd zc~8juMh(4RTpPl9Kgj%hLSCYG9Nv}x!g@J#>3hBe?c{ab3%|)9ad*zSlpScS{0L^r zy^%Afhq@Uu>ki^PESW7giASTU+zpq?-(oq=MrQhMzS*^$Q<*0@H*+?2mY3i_bq9Qn zZK$ZXhs&X#d>=;QT9n0exWKhjIqyzu`^Z`FA0Mheo;Py^ewwpi6heJ<&a_|oDBghwFdLWZ?d6j>b8`@P=gi<^ zaE1CkG>|Le0{J}5lQVbc;>n;sfnVj;oLxKrEXFSN7S2wV`Cdjn17GRQMfRN^d6V8@ z$Qk+}f1%d|AIWzjXV7i97%!;TaxLzSN92h-g&*XrxCQq?&esL%&#+v*6Wj5q`c}@n z<#l`_s$!u2JA5lwN6wHMT!(Y6|05rc>rg@c9$$-6a^7S7yYEbVnAi9<3;9!C!LRe- zC@de0+E}Ze!FBm9ydYp=aTkrw8B0I@cKEbuu zuu|@c>_+|Z26~_kM(OXyBwUQVr{rBbXZ)?` zC<@{&{DnhYyBFCfKH==*qxCEDxgudrbrNDe@M+8;au>$gUAbxHM=T%Yg4^|&8J^@ib9T&Uj5z5JTI>s_h7Q2hrgsoU~7dQWrq zxVAW5o`)qk3J1FW8+OTyP*%?UklFg8oO?FULl-&!y|d&^oO^eqoO^CEXAk&6&i{9R zoQwnYbALUA+&?)h^W0<>oyob6^0OGnIVYNO?$6A$S@KW#7N6?nnfoYK--z6I7a;f4 zshI9sZOp)Wv_WUI@O$O?Dj?^%&-}?;d^x`cXCUW7?!}+=|Ksd~wK>meJW?#nO5`k$L;DerEXXyj(BOO`h?7 z$i4n1_vJ@0SH7C_{5D2r!yCxF+m6ZU;bu9rE;B9Xb#|WHF&Lfn%b~ZN9WpcHI&4&L z#j(ijyGAejxPSNUcgL&WmNSbo7Y;&O6jJ9ryGFhP*Q#rx2lDJM(91l{+?Xl9kL+YK z)K%pxc`W82b6^T?(a-b$u{@vi-jbQL5P3$A;0|~IU#Tl`JzR#r)rHYRJ{(8L=W~5t zhS~Bt*n+(0^XNIb0Dh4V=dY1j^B`xIX2&mr zJ#ssA)c5bc{pTvqd&S+Hv+o%fzLQ_&e#kjeMm-C?(ASeoj7~y!TajN zIA_^e{I0qfN+9nlH|qT*jg-57QkSFn}K|Ybo zxR$+c0ROIjjk~FHe$|z?^O1NNAL9@V({G4Ua%CQfujPk$7@vZZva!#M$3i0P3 z|8epk@?tzM58yX&h5R{=m%rx<{42l82csj_tG_@s`9lPXR2V;i(D;}2fz2=$y8*Rm1*n*rF>+~`! zGw(9T^Zoqe+8S=dE%e9AdA9qjC*orDCm4f?$n&;>4@91k%#V8Lf_x{}apqT9Uc{N9 zKVh&s-$h9|bFn#2K<>|PInPg?(KZ+%H$%>>+&7QP_3#VwY>eZ6cn}+LgT59>xo3OxOdiDBk>{`--^wrY2`DFL9u>wd$hk9$^E^C*Jfk_EbB1Il7vjwM`rHFG za4VMRtwg^2Gm-oL58ln4QBA&q563oiQ9sL>Egj@ms3HG=1^7x`6>lK-)&00e|3&2a zKABf>?(Jic`*|sHCQswJdY_^wW~%$~P@axz@>Xm>o|htgw*Fe~#d#)oAUo|jdM}_9 zMxhKc>$AsZR_u|NaAws_atq8sGvs;9*_LzqT;%@0(dYg}&ecuoipWlvGxlJ)l51z+ zbX=^?vo)O;p$9T&*6Q7j%-}!LlMm$V1Ru$#$u0Q~ejEklHn?5x&Nca{^l_8AApe5P zdM(mvzzCvJ3!r9e*`jDe&*KP zP5(@JJ2DfqQdKYI0zJq(If8?8S7KW=coA#op`WBwV<9IQT#8mlf zbd|Gf9nPiIS7Lv88D};%moLS2n2pTtoM-pBmUp(*JP<|X%U!!!F2}|B63#!b@PEku zocDou^>W4y*881b#+~x}ct(DW^Y@^~%h_oPs>jM7;|ci-+%G?XdvK6C=ltpNo5XWx@BvEWLcLG$7nJ+X&CNpI_ZF%vA!qjhy_4_*@*dny z|6jDnVxk!Y zJkHlUpD*Cb{1SJ?5cwhG?*;O%n?1_E`}Wpv^k0?V#$CvcGKMSg4(^17KGO$zUpR(O zMt*~*d9j#xhGE=LT@y#h>oGxI!P%|;lpjO(o6k7!llO5QY{eYp9iyAie}L=JTJK6M zz+vht_y!f#kMNVoj{H7X##6{UMI$~owaDZ9J_310@~rm8WVG;meyTT9&Y5-@pRCTzYbtL?7r9&duIHJ!TfIyDBRAvI_$tZAGv`5G!2cjS z;;sA)va9@rqVm{$*0mz?CS;Cv)%#gqjcv&HG#p#x>b#p@v=f&Op3v%Dh z(7%w+#x3$PZo|FtEppE_(93MQfHON^LZ0{Oe4o#pfWq<$9$17HxCV1^2X^{Q_LZ&j+ww>}B6s0a@QIvx zewTbX=R7Hj!hX$K^-#G09+Ru-?d3O-eehU5-?i+~HTgZX(ksS2k=>yoU!k{~H{wKj zD1MdqoqIlWk^Bk%)vv-;`F+m$@CHvocC`825?8u@DHr4x+z2l3+dcCBHcWjyvSSUx zD%WzxJS=CH&*i$jf%~Gf{3O=lCmgPKH6Mn|@=5AT&`-Vr6>+TkXkLjgP#@>&?ZHU- zFTMq5$fxtOXe^J%JUQ<_)8(8~pCRYZVEvZz#TbECFb>r)3MG7|q|bbZuhj+B**PAU z%jtcL`{k$5T0Wk;;s=z%r+UZn7(6e3h5xV`3o!=UT|0y)@l*UKMx(m=LyW}^bqSQg zrRwa>TjXcuC$JnptN%f3`8n>08}T*%(7Ti8^XI&W```!pd#=h;xHR^YufgYX5w67L z@iP8IcfBL{F&@nwalJf*55Q76=lyzqTAlsxdCrb;gkIizn(#BY5}Wng^Cq-{fA{To zdDs6|aj<#;*Fqh+3{I18=Ik90a(mo`9eB;P!rYwS=3dy}=Q{8QdVTODvL|L&SSkPQ zT4mnJ74Q`n;}jf<&s=+*^FB96ZYr1OBl$~yl=tU1I6Gzbq~FyOa1sXVSHb0SL)Ws4 zc9xgpDE(4=Dz`)S#J6}V_UdOReONvVl`&o~JL7xuK=~?ufWP5OP*JYMSMg?)kn=+w zM6YroWFK0geoJn|7a@CDO+HTVFRq~O#(B^iP3c0w(v6K zXPFtE=QMLab1e6BUGzhqvu((ClmGYptS>{(+B{GBS?7CvTW*aOa_*TttGVYh4>NnG zah|cV$ZXG!a~p3)?v2ZEiEH@|@*L!QeFF3K9^i|3Cwif;x(Dy!-}o-HmNNqnLGGK% zJR5m#FW}7X+`B8~BAh)gXJ5{ii;?Fx^P(rZ%A+wGuOr__zT=$5xt9v)_2Zv7_v1U9 zGj)+(_K;t26EbTu7kVT2&l1eke};3WG?RZs_NdIj$FTz6>*cx6d_P6bebY+57J2qE ziyz|LL$B)P-p=zfpMT&Jcn*djJ8@I4gkxO$ncHxl`R|cu_6%hI%b8V59_RYId;{OY zb@({W^Kd;X`drT22J!}sa_v0MUYX~$F!D_0K7Sf5GE z>Q&*KDKGGWxJmAa_wgA%&|AQn&!^)>^*Ov0v+xh5qhNVpV z&>aVcgb6M7P6b=on(?;IrTtv zl|R7kn56!c3vu38dh@Nw&hV@L3~az^z2mT3p2FYpdwdTX;bipmYmQ9K^?0Dq9Ed;V z3dr7krMd>HsgK8-@_Md_w(@SCgJ*H7`W()gTbl2{dcFVfk^CjM%72UZBD>HU{StCx zl)+GS-j8!GpD)+c%MG4)gImRM$bRz!=kE&&x^^jgVlO_&>6qyHQ{01}MBc;x;B|V- z@wMCmc_%)ayXa+~`bIuO9?UyXA5GM`39}2lDZhpnv_|sd=zv9Ng+qMiPS*A!XM*B>{1uy#XN$q;VV#8&fhy+ARj2_=axTfeioDQnqGd^d4_UU zXC@cs{P%KB=VzGzZqC3toM&(a3L?)}o`d|%^1M7C=jXA7Gb3)0^E1c{eo@YuoA0v+ z@=Rq`G>}_xo`V*8gZWzCgayd6liAvzGdFWqKBCUgC_jVjTC2D%Zqm!1(}iDAx5HX_ zE!Rae9F5G{!}uB3@?B=$|08F9js7bMJ4EvoB}A_(jefo5-0bnWJ^&%&E+|+1!b9zvNk1g4~;Da?aQ@@VfeMJb^vx zKloa-k(Y61{6-XzcVL#BIW_~+)HkA_{48?j=GnhmOG-goHJ<%>Z!AfpTM2er8qlL=HCGMC|oUH z!?&U;a=u@ILi#@;bE~I%IOpu`$766L4#6X?eacVp$Gim_@sm0;F6U1r+>f)7IaP>n zcI_&@9ytrYQm;qez1DFB6m#tuZpvl3D~e+&dSRP>8LUF~lX?7}e&+e5sE(5AMLd$T zvrdthVm|KG+sa>|xtw>toM%(jIkU3==G=W>o%hBm@-6aZ$S#pHa1bAz8rciW@rmex zFYqXaVkD{spIOO8_*B2PmwW~<uNfxcV`NA8O@A9M!6vNN zAA-m60}A`hYF_QyaNfx`A-iB@J_sM`H^(nH1l{#=hMdQHIqwj)<;QTSdM;-N%RW(4 z{Vf{dbA0LAzx)nf#0To^ICsgX$Q}4{j7A>PkLU(u{~W_FqYkp4_pB zl3Y{%PQD!D)PJFm{1RWqdA}GTuaujhy?hC>|F&0;#~!?eF33ATXKv*)dFOgS_V2#; zz$%<3#d*F9D=S=27=0x_MRrwr#L3Y5Ljpv{(vP<00 z|M>hlD2XLFOYeA|&ab1bd?CilyZI5mnup>*%uxTwIeWLu8!-!+GubP%vrSgt>N9n? zEY6cRVGgFMyWvCRIl2Th^q=6P(JA(O%+{NTv(Z&Ad(hK-fjYDBI_`wb+P>;(D2~(B zFYqGdImn)Rs$3MexR%+O9pFdxjdFI-1w0&?^@nqB*Iq&wInVyJ_+C8>*(r8%Q{0Br zF$CEo@8g`ocXR%^3A5$Pcp~@0V&oY)1HEw{{zY5AzJqHw=Q@VtKxC$0#5o7DV`Y|q zrLKpKe%%HXK{IvEkb5|@_-?(zypEs3Ggyfu^;U6q`gL+{jQ@_zl;|7oslyMmt=F_t6jYajefRLnV0&vbVJ6=Qw9a4f!{;S7-M8 z%7eKD59OJhv-S?WjLf2+xU&8y`~m0Of0@5PQ7qAamLI}G^jBYrBjlc3j(_F3d?y#c zhjMm_ycayEo~Pb}fylWyPwzdxfRDgu$ewfc=kx0Fcuc++ zIkWqtH11O84EdCA#8GwiAp3XrpgQWuU27*dl#j&i>XUJh{0OSbAEP?%SO07*`}V+Z`3+1*A^lT$DQB0eAZK6yoGb8Ge2Qy7@MwOVZ^qAZ_V2&s zuhB@o2~FkfUTgdKRTisFq1gnv= zxgA$W8`lb|7t1+6_g7~JKbNo3%lmZ``A_+2d@CQuOVLLz$dmX7WG_CEAJ!Ym`>R*- z!Rqx~iyLwYz61?iZ^akmZ;Wy6G)$5Y<(=G-vlG;p=gL!%9qnDd6g%{fM)rg37KP-l zxYe~Uk$3qb?B9L+;9zdAcnXKg&AAl{;w5!!d?Y{53;3U8)Iv=Z*B^$T@SXaB^!zun zhde1)!AZCZg?*+XUykx9jc$5l@v2-7b8x46IrrhK&;Wm`uR-2LzvfYTry{%5<(xlI zO?ne&;aPPlyoK!0W%Nd11+p*StA9OSSHH%CxCt7evw9$&lpn+dIeXJZa^7<{tG|)Y zO)cj^$nH5nUZb3VcjOj&T{(NhSpE&Kx%MdEfm`Km$nG*xeUsb+IWwknXSBr{*E0Jv zw{s3(re1|l_3{kuz!o_>Q3H7#Rv_~_^R2Ah0&{%kN8Ex%I9)H#OrFQ=ZY}u?bu*rg zoH=LmWb{Msq0Gw4JQL035u9f;&rvTPj@N(ehF7YU;Y0uA2V?PGQacOzr@9`M1B!x$p_*lWPi!*>!M$hkH!Ld5AsZ9 zFUhWv8S{p^r~Y%CXFj`1=Hf2Y(EAU+$iw(RKAAV59-65OUZ5oaZ<@=p^+s z$jm=V?+e_eUWPU}3cHb+^#q2?e{*-NmH*`nxf*Vkr(zeb!vlJaIOjk=Is5Z??u6q| z4oABF2QqU$#|*uLcsqW<+sHmKSbv(Ff9}S;n6ECL&&Zc@=4)%tJI7icgLknS*%6-b zxeB;lo%84%{si|S?=j8!A=gjB?{X#n1KBsSfBmTUo%%r@#c!d5+{U$$@+wE56-~% z$hnd8XR+Li+wsN7F7mVb0t`jYi64-6hU{Lg_3p>ndarP443IbRN;Jj}e5f}ASIgPW zyU97ja;|4jxLSQIy5In`#b9KA%=^qSKC?tUjwd7g#cf;?uj_ZlwjeP_sGxlW^6%AbQRPnhB^()JV^DC%{Bh_tPt0b?H+w%xM8rR5Q<7Rn(Zj58)TX-SAfGg!caSy7i zUqut_@ArI@HzRw`OT5WvF6Rw+K+f)UZ+a-DcL|P@>+>CGD4)m;@Qu8lKgJyST(02P zXGiWTpQQIXKh0(Iy2ijo%$y<=~AZJtNMfQqk)Ol_<;08Q|!|QL)S8MTFO81ulzP<;VE@yUgkqvJ`PRsA%4M&t`Eg^a(2;j zav5Z9HQ*V1EZ#-t&KtN9EqwMHba#rnC??E*@ zqu&Ubad~IROnX)TH)I}Xu4Pw!PW=xK$L09b=d!=d;v3bYF;jj4d8c|^eYU&`nVmP_ z54kKl$a!zbyvq4>AO4ZwL-x65{0Zt~l52H1d*%83kUD3=Hu(@f28&Tlow>P~H*?O< zA=s-vfU^S~#`AH3-e|lpU(VUxipvMc{dg+hjQdc~wZd4V|D!tRV_~_MdV7y~-I0ICAM#iDOnwt5$mie(IeTIi&W_qZ?_tbWXEzzn z58|zSo)==C+y>+2O32wz*=KL(&-E`uQ8~Ns34AbecDL7GFQ0^oI9Yu%-^FjBo}9g4 zqC6ABF%}#3DupAMkgD7Y<@CSC zFu4zE;a2taI0@sC_sA3Vm&*gWGFs^!q&`%x${qM&l$4)G{s%|VrFy3#JJ-8>i{4+D zif`4ExhF5dzj#Mo1cT8;eLt7R1o>vZGN0oy$o{^8t8(^@)vo1z;Xl3ZD6PJj3u2Ib z39_G_&h7D%{tFQI{T<0JVTC*lRZtP-ak%~wI7>bScj7Je#dr-bsUs*oq^R z)%h_Dz$*1O_z3gVEAb!xR#)y6~>!}awW@CfdM&ib!ng4`9E zSx+MOSN@#2?{kLr<*W4a{O12N|DC0J`SWD9&Xn`B%Fif!N9Iveb>?yI>q>HFQ2w7Y zH`>WLKQbH7)Z2-M@&nk0{2B5y%RRLb`Ce9VGvsHMKU?mr%$dzLuQlD03@w=^*}*Z$UHUKFB>fAK5jtoAlRz8?Wmv<9uiN zuCmi^lV{6$Hgd1`z$jdZ%E`|cFZ zo|R`V=h{R$JN!bHY zQ}O?3v=8tv=l1{OH%&!~24%D(4I!bTT^dT1ifAbrk;CKeGtL zeYUH|1NnGYZK4aHzZkW92PqBa40*4ra@=bUd{c`WaS%lqC@r|=V4=2}}m9`!Ixon3!5 zZ%|K0Y55`^hk@!Obum%k-bbyYeQSg2&X?;C{Il3d)^$7@vt#up0S$h%b2_y14#5U&yWa zAufl_n4r$?ccz@bi^%&yZTVVcAKuFy@in&l>;t%4{(yhQ1o?cFlW)dT!{QV#~R*+`uGt2Q3L<^Y#G;g@)NjU z?>%(K#QZs!ipA>e-?g|pw&|7S8hS(JyXEYP)8z4ZTzxyggEDe8+<hw%}(TwcnTB6Hz&b!O_Vcp5A9J8;g6IdTj6Ut~7q9^5OB=Imn|IJ?6P z?xdglrZ#7Xxj|knzsIlRHTiTNfINS>r@unZi<41PK8SmA=IR@A=G^5dgjISgaig4b zKhIlsuzV+ZCOTsw{`Q%{d>^LBeUKeAXJnq0?1ZoCRdoFTOqbh0d*7M3g}=a0xZm~X zInPXH&T*V`+;3xfCueq_j^6l6uL+mo@6cY(J-3PH;zYDj=bk?h z$05%^X4Uz~?wB(nvoFudXMCB@m*Wr6LmtJ;Ip@T1&V86?w;*0qKgb8*2l--jmS-R{ z<23arcpiB!o8U;-GNTXX%%#k>7W%#UNi@~ljqQVKj*zT3pXS0Gu2U4KEn0v8ktdf zU;32Kga7S2?=r(OS88aiK?h{kwZ=d^qR#oA_omC#+3o*O&z9fk^Z0r$iCS{@qEC2` z`X4?F6Xd!$2^ZlE+@XIAABl72-|&Kb6)(pE+@a1n^%m-3v|bC&?Ci@uxH-Sfw_%KY zD9(_xOP1gj>MEEi&qp))ROB4WdGUu{EA^NB3t!ETVFQlBXL=WN9UO{oI7)9b+9Lb( zA$oJ=xA{$Ee|m|ZMRoo0$h+ubUWQ}Q3qQF2ElSJxbI!`f@`uQtm3cly?uS0O%V&<~ z%>1tMHS!*uDmTL>xrl2!`C4`MniF^e@~$yo{|VfJzfeH`0`89{sD|u{>$xn7;B8F7 zF+N*~f5fA51J1kGpYogXIJA^=)@2`RE05BvjqGtl)OnZa#rNoS;fpX2{csiz!i%m= z$B*(jY?OClGAgTE;Tq)qbC}*w7^!{-JLE;YmW$ydbXPa^*`s+H%HnAKr+5i3<}dgJ zJb-i57111d2U^G{^DD0XCl|wS^0la}zeD|xd=fv#4`Pk{4F8TL^4A!QD=|~=K7JCf zprE>?KPP+cGwQtaWRI95Ux50^yV(s~55J?m-}5+fcGu!gxD#9O0gC#}&%6$K537jV zR zH9p3>>aqA$einIW$h*)8y`Jjt_*lFnXLp{;*}?AC+a_O$|4>MM9rFJ70^hDzgIC}x z`D$cGyjfjUeuDFUnjP>xxfN#@A0YR~44=s}lJ7Ua{~YAc%QIXU1#lPg^Ui&ef2TZO z`RC@}w=z%S{CDM<`d-e@JkLYs-CX_bM|sBctmQt)3>&V_z5503mh(((;yja^Irq_D z@{(M~B)#g$ci$DcC+qT0I0mn}HVuEvxo7jt=3K~qknbQ}X=ftjqnMXYF{DmQUfuC?FroQ;=DDK{6lC`=hD+31=Q=Zj_Vr zJblZVUo-SBmp{T#tW#&6pU-c{=!ZJ$lkf>0@nP?_|fXuyu>deHdayQ(nuD~04G_pgk;LPsq=tsErvivr##FyBKgK;0u zcKtJ6&N)x>p7W{tbM;Ti?7Rpc%5AwZ=X}Y{ekAQeX6iruKDN0217;!T!3!9v-xHbt zf8ug^8E(U+$a~+X`n~b6I`jTGz7u2Rhwz6y20x=L?!iX=^SB3o#F6Tkv0Cnj+8C%V zgih$C&O6{`^0V>|{tvD7Zc*nQ=>~pXFYitR_%JNe>x}p1AJIxK&DqVD%jI#fdI2`z zf2cq?d)9LHkgj^w_yK-S?*)8?O&E@y`ls>;43huF2KhLy==ZJWJJdJvi##38ggmUya|f6sMt~{wOZ$&pC%{s`HL|EI+O8%(wG5Sc&XYH*sTB zbL|rDgmH2u?#R#KOZeZuU8}$F7;eKoXo@RbyO0a}^YX6voBCq)x%douw>XOT=y$_j zT&n(%*YbEY!Cg;V#a(g~dFT30~>b!eYRZm0* zbv~c-Dreep>f9T*AouiSWHx3#<)4{9|43w3{DC}+nGty&Xx~3Cvmp0PJ=BtOU*+CB z8~N|gKeG`k;X<%RQd|jy1>(&A{FVz59`$ z_i`M7pIsl1mymlq_hE54GbZ23-hp&H3Iw<*CTe zayW0q5&D@Qc@FZO=6=Zxd_m5=*$+9N7N7u@A@^7hzL0aDSLXk)0GR{1H#0lCq9Pip z@8)@!nc8P2a9_+qc7fK&j+8m`u>1`7z$QGV9?F?HdA3?0zjAID;XJcDxFf#z+4`LO ztUKqPZ6fEao}zwFo!v0cz`g1`OU;9v=cp*oSO4GhI85DGo#*`?c|P(?uF@Ne^|%0) zu^QQ(>3`di9V^eoLX8(uR(>2S}I34hLQa2q~W=WK5$pC#Ae%+539%80^tI9eK}a z$s_f?;-8S+XDDyRbnJEQCa#M-zd5Jw*QOnsUzAS@IlY?<|4}$lmmW-i>%xeF(SJ`-#h` zGfOw{4>$+eL!agou-mnjd>hZjY&?a_(M!J%vbUV1zFp32%ud!oUMF|J8#ohH^|qor z?p8PB+Bi-whH|)A-OBH;!=u%A@fOZ|-L1GgzW5pVj;#Nvve^( z>i1;de}X%!=ks;^0N#|dhm4coLR(}M)+)h?#VfCujLzfE|s_&?s?pHoS0k4tbG@}7GqZ*jdOf6WtkE&fI?RPvcu^skazA^S{rwPo_dxWl#V zd$YMSj@KK&WAHze#TTgIGpB0h-SHmvK)i#`_1@wkd<)je75PxU8by(J>1*^l^LTak z;ZM}{@B;2b8T{zlt(^CUuKXQ(>(%9L_**^^)#aOU1@cZ-fk*0JiG$>|{3QR1)woZ6 z0T<%Jd(fqnF_}`4Jo==iYmYkKsI{b&-4i5AKDF zFaa;={olQl=P*C>oSE;s))qbVXy5tYK`0=%z-!one#kwY=kZtkt3I4BMb4W_d>kG@ z?uEv@+-GwBY~cLi*>9$b{~_n*dwR{~%4mcF>g=BjSd44Y+i|( zxD}b(c@7S7tqHoxkNQ0ekb5nAZtj^pSM8AdyeDTa&c|W+3Ykw2;(ByM&b0=9-%)Zy z%$A==U)+Msx7qq@9_iJ3Jy&qm--oS>iCI16X0 zD*d<6)x+u7gKl~)xhWU$nVjq2sekA8+y}4AUAPT~U_K@zv#}ML;XR+p zo>oiF%)17q@uxcbQwO;VisDYa!8{P>;trJ4%iMep)8zM&XS)ZN!gaV9*WyzB>^Rvi zGqcL7f5HlUs?PcNGG{hsmQ=-VxdrF_|r@yX1KOceKd+<9lp^!hcnaj&hv%( zA7qaxhs?{dd@mn>*U?0M7P7na;W@|*zTCB+u}FP0|Ap;#h(kk^Q2P z>qp8rqqF=!y`1+e#ZKzHH$Tnu)Ft>XOu=4s)mz4!c`#qgA0X%cuUrfj^-tmZuu-0l zpXIN4E&szW;6n66_Ml=|EcbFf`)@({6Zr=|oeT5rIM=mt>Tl(TP$O1%!U#Eg=Tv?e zr{a2)M|RNc9T)iAG}Oa>`m2$5qr7_@t=AvNBYSUqJTI3*K@9TQYCHtZPy~x{wfk*nZh9I3vAH*wxehRHRtR(&@&zyxHMU9VRkPoO69?$Q$F zUB6G=5I4x<`4H5VvlpEqXLqfyu7K=*Z}C9=tMD9NP=CXB;9dC&WIsEb|I-_V>T=!@ zKIZRml-_ZC879gv^8KjiGyT=S$RDB%PEu!g{9L{Xk71bZv;- zl8-?*IXg_|%NXSR`itkg_BZE@eVKE9xVD( z{^7$pGpiB`A@^imz1GNcJBK^+W}eRZuBM`bx(0GiWVYt4$^Me>Cf|GRmF$-b)mP#q znv4Vfi-@V9(3XRpgV&Uw5QZISb0rrwEMn*Yb!ky*4K4o1$g{5lYo z)R}pi^}oplu}q!&aXmk#&i$D^Cj0Qw{5=M`_F($_G{43tb9V2{nqTG7cpf>U$K><) z4b9O-zX7h1Gv9JnUB}r^Gr!BKH((Ryp|)NDF2i!e={nbQtSkv(`PAE#f3$8zRpVVr?Q>b!$2 z=heut+RLMGu(B^_J`a^YlzSlaCNuLR+=I-XR=hu7$LDYdo`FkI#pgfO%lY~YS3wiG zwrfAiTafc4vu252cE)m?vv!GIS4I6uEwL-j8%ACe>fgNc~tl3y`jHT9)|3nf2lho=jk7MvoKA4FY4kl zbq#(A|6w>T(p!pSklkVse}Yq78_$2@M!5+ei&ycfdM)n4t;mdjS3kSs7jj8dht|Gx z>o_ig$r>9_Le9S0LSBqV)LU^Ua>l)?zE;;l&a)rX9kEf}fzRY7m@40dJ@PG_GyFoli<;`Q&>rR0 z|Duh2BUYk14#lzh%dtQ{4E}Nd|Ns9aWOw~ceHspM?MdVv^I1MrZvub9_oF)ssqf&@ zd>QXT3)E5Hh6m&iaTYFCFT_{!T7DQShs}T#HiZigWRb zYX{*3`7`c>>|E!l&y*YCdi<`wg9~#0o?xQT+h{yC#<>xp%RDG_A>~YzvC(1{;UV*b4XKy}RK1naLAopFK z^PC@oTRF2LXKcQ+*Eut9 zFmlf3p39E4Q~n7(kv%E1?+DkP#z%606vtdl(({phJ3#KKsmOgZnRDNlmfLae+uS41 zV~9HUWA3}N)HfsdMiX`BO&j@SJS=ZQOLSL%&pr4&bdbk!FK)%lk!S8t%!U8$JNuRi ze<3rXk6z|!Eo5H*$Jbzg`8qCu%$Ga|4{}Q$%6)kW=Nwqay)hiS^&23&^S!)7uLLsF z8X>zwVg1avH!uh}KS$_okyqjgxi%NkuPf)7se-Q9tDk4(7R->VpeUNFZ{)v`=cb{$ zCqBSfRK}h9IlD85^E_UqzJPB+=3@o!s+T!5jx+OT;YIXSKg)TBEBfpM>db`w(N^ys zb@r&tm9;odFY~t;&&D;#y!t`!QaLjxGqWS-og-&g_Q#o=U*$P-`#1S|e2ko7!}N-A z6Q8||pHgQwtjCdZBm624!|llYt%uHXcCFs>7@V%oUhynHisE`-<2d;-o`U7_H~3QC zgayd^PG-(g^5d>OkJIHMoEiI%{5wX=dB-XwpN7oxSM}=RL}YKtyHOX{)}fQ$srVha zQE$cNn2s9w4llbjkT2qAkUi@_uEs}j-pfjH_Qykgu9-X^Z_2B;D8p@yJN>suL_#@fx{lxVZSdXXD<0JVD&O2ru{zqMgYpF-dIVabsTjM@;_LxC( zV_b-(dWDhq-Iw@Xy{>X$e5n3So&D(vxgf^lV7>S86)wRpT&ll{`|*DK4iDj3{4;iA z3`*k>TQXRnbJ#qxdxw$HCTwe%WU9Q zyw&x_a`ye~dj;^lUKza`cri}J7TlzFA7|%3n)~2vy%*70&U;{XNc7S{}&Q?$3mpQx4@$%c8ccO~&h1i2v^bY6td?JpO|K^yk$>cByo)E|4s29+Kz8c+ ze30J7n2xK}=kU82DHrsaB$@+0(|#P31%R zJ#0g5{crTLZ=I>m?sytstXGJ~bAR-f^K9k$+khYBRrpdq7*FF~>_&E&oHzT+xks}% zyv+G7@-xl6%Q>8TeWg5G&MbXR&RNtQb>)1oE9CqPGDDjp_tomu{189Po$;ysH|omy zz6x=k&CG&)FGG=YrX+7b&V<6Q<-Y07<#|4D#~NhjeU8$|OuF6mN%9zE9#m3iUiFdN z%Psj$-hv15EZX2RWaj2fnZt9rqTiR@aj`o4-GRIixj#O_6nQ^rIfpNIy)EbY$+MBUbRx3Xov5GlxC?SGHAJ4roC%pjZ{Q1L zM%2UW^8MV62P5}g?$^xw-0RsRvio$@uZ7HrhRA%$?45xt)Op@2^9+27%=P9mUPpF? zoM*Y;tGKomx$pC=AAme3m!XwjX7p&}3|OT8mK)&=?!#>u?D_&eoM+)+c^vP+e7QDu z<12L4%bvH42jB$cY|U)Q%>O{n9K8nlRS`3hooS-$Yvr7QnbQL~&-9C&-C!+GRp*?| zEW4e1s=w#oIWxR0XI_`m`wG*MS(G_7TrQUDoH>~@=4&phJ_uz|5_|PBQ)bFluwKqN z@q>I3mf{a&Px+4zbiF(e;hZIp^Krb0GjC4e$=nOq;xKjQRCV;l*LvAu-{!*VPMCro z>gRAYwyDeEc{%$-=5+qvtES#7dKcgU`DVVKE8qsXINrli>YluYJK#m^o1y-UfAzzm&hjPI)J{#3s2jAB(%>>ht*w+$w*Ft8g-!_~+*g{#7o3%|3G-x8cd$6+g)h_z{_4zsIVV$l2Sb$|dAe`Boeu_vhcZe)>LBlt<|;z)kW-uB6|IyQ}kF z)>s}Rx8qIx3;N3Ua(m9Z+z$Cyxu)Dto`r(y5qMh8K3NAhsXxQd_yC8vb_=dVW!KNv z%PyF`Y(9+}VL z5&1|QAV10jaSl4Dci}9V_U(}qgfHdn$OAb0_(^(~A@6kK^s*lnQ9ma)!`b*#J%W$q zTX4Gk1Ukyq_$bahKt;X^C+KCDzk{>Kw$R%skHnMm$6OGlFcFWTk^U>3dp^%?p8L$J z{Il}>fHBvZtqa%=d}`hFjVh!&b^e`v0Tp2st4xa2jpJ()aSBCe!;J* z^PC^VC6VWII#yF5laHXze>^>*(h@ zyq~vn=1@ichI23HIqrn)n=k6;KFO?EjLh!bGu7om`ag5_yWw0`z3<+ahsd)~7?stT zeaHG-?wQQCI-Gkf^JKi7=RUJ}gljnmGh=ceE|rhtM>uo&JIWI!yN}i21@<>#W8*$EpoO|=tnG+9Uo!(A7EazDpBxhD-Uf-_w zG9QkA{-{L}aL-yMB`q@_|s;|Hn6w`Z&oAA{b zEzjfa*k9hmm*F3*R_E-xUGBKr*f;zJ|JMlJoHKyn{#D03c)idSn5-sGf zIWxVfyg+`9^Xp!@CN4&1;fKhXR8`#t&Eyq)1`p)VQAJ+NbFm*L;ZB^Ue+`67T#|d>GI=xF z$?cJI=tk_qCpZq>^y^_4&R6Fhx-B1%oHaQM+UoViD!sz$&*UTUDh|`jj&cl-$8RW&ob7jW zD`cNO%4Z*w*P**y8f&o>WAu9AV7aKzKE-+Wsn2)m|ARB|1;(MF{$75U55Os?g{ScV zYT!9+zzIH=eft+K%^83X$+fT^-)UXM2jVt4d&}osLY?#U0(psClOI9eOIxF_-Ye>! zJeE7-4>ZR2dW%pG^>Mb|8~hPk`n`GQ{z0#@`dRLSgRlzO6>ICCC}+QWlCM>ld`hM@D+yQ^oqL{N88PEx0nS zmNz1M&JNeFMN#~S0XWsQwYXP)jxR+&`5yio=VAaJ)LW6y<4ydfSAw(87UpUwq&F3_ z@gY9Y%lqGGenR~?uf#QIsXl=x;ysj8AI2y14qPRF%&qtr{3_?2`~~?I~uZ5-?4HPc@#gzmG}@emEXdp@=R3p+3dqN%T4rK;~jjY&KL7M%DTRtC!-SHaj7F0 z#a1j+Uxb_F-}y=&!Oi`?269#YFV^dgrZ`yr5^`?*qb|jn4}Ij^M>&h9^R38zm!Iht z`A5##)={p5!?7Ai=Q?uF73Mrso%wn`gY(_BLI-s-uE?2#`PpVaC@*JjDdp7d@58=$9QJ9TMSc%&DGx!E%hUGb~r1vsr$;a?%ST1L# zW$(=WmHXpHF54`+aOh?w9Ye z5dZqj5d17>rzjvdkOyHA^6X`=?ZJPdq3d6B&Zwv5Zpf^ipC#yI&Ip2KVKH}d?<<4g3HqCGa_V!g7+d67AmIhq-mXLT3n`CTAq zM*qN5kmu+g&e?pXoZU7%VfNaS)m>3Vp2{!!OmiHI%(V|WGb6iw=INct`$V4IL$RN_ zfI72e0rD(gq4%lWn2Ym!T!lB`a=AbMhEwF-$ozUzoq3r#GE)7o`e8Ie_MM!?Utv7X z!bUXFzkoBNvnS=A-OKL;X1v}=q^8qwRjUfaX5y%b_6QRcOmCZ(NA$o1l5H-|C;CpmI_R{Jz6_qE;LvayqL-v>p`2p8f z@u?^x*W=?b0Z-sE{EShqwoaa_({(0R+0Zh{tlua zH^Lz}3OAv>{?25-C;NN$vlHaJ$7G+LD4*o|k$6nL7Wq4f>{;K-Q}uHeufbO2=Xa-G zGcJRh<%zgn&J4{z`zd)SzLN{UAGPnCSt;lKpMuYj|JK}Zxkq1<_m>OF%{brDwY(U) zH_pND*n`ad{OmH{ORHa3=U#eIE-$~#Im>cJWzOYU7$RrBWai~NdlhG@oAWZ{`U`3_ri7hEFuM`mIf zJSgXR&9l$|o$-k}Gv*`YyU%>hb8(Bh7Sh0U>V+Y?Erj@ z?a0}h9eIoVDKc*|kEh8Uk^7)KPSDT2RtI@zGf%(8BJ@PgiXuL@7$2ZCj>b6sa>)HY z34P?;gE>>~lIJ0_b%)*`oOyjXXXpA$??1T_XJ%%<86tN>W@C1uqvTtWc~b*dBJ(HD z!8iPgx*@MYp8M>aljOq4b5#-_BKvvmd`dMG8ue=a9+^KQku&N>9)tq857S)#jvv80 za(0Hg@@)KxrRvP>%6{)_TwE{b{~7$3`fF^*2DH}OfF1IyI7aS@gXCYhFGgXox)RpP zr*h7>7v-zvMZ6I2p_sZ5ev`lA%+1~MaGuGB@e(e@IUA0ZkH8>#InU;*=!~328+jYv zN6vuLedZEgf{GZSF3&IXLHM71GChFI%hT1_!-vbyqa6O$`jVqAdxW4NZK~AIh&y<`Ue1m`xE)`K{CXc7 zF+iR7k^8uVdMYZaS8{gJN<0{O_qm-%`do87ik|8PydTbzFGEvYhwO{9IPVKLx}Lox zJK!$4t^RnPj+}LKcrBj7QrGfMT8hubJiLH$xWe^HuD!s|qX<657W|9sEcg0c6%<6? ztqjpYPP~f&=BNcr3n<-{FVR)#qBO z_vaJTHMxiSX1Ru(b9|z_4!`20*!ACeDChm-M|red7oBjm`X&CKYXj*vJc{%2Eo$p; z2fD`pkj&U8jxKdb{Mj@7$yQSbmawarULR<+*r8 zeGX2MFGVZY-cX+>7eonp0FFoYuOIbpz?15q`9oy4DuJ?c6*R?1XzVlB@K&6Qi`A2P z6;6}S;(ol4mvb#{&0k@qya->*hjZ?!1LS;1IXk=Z7g*}rYupR@esYg>$4~mXmybct z_;dI$5$?--XQcgSaH}-?sr>Fa~)(=DOaP`*IuP z8K16xoO7nXfjlRJ)R|e6@G0^gXK%^bn^{;={U(+m=UN$jj*Bn=`T6GOUKR!9Z8%xZ z9?+9>|1?K__L*A~_!9kF@Q}Qd@8{hA2gmBn@9UAhpqO6H^VRC|*oyP@+G3TQ`Ihq_ z&t>k-?3zuurG93>E;+NPJa{QMf-cIiWc{U%!_aigw6ZHZ)&;QAI7&+q` z>h+fM{C>dWa5J(CHE=Cw!8e?H_d7g|8?jO^v$C7KKicD364%6nP%L<%#???!@=%mpSL#uM0`7geUZ|4@g0?n`(gV7(iVue;YSFZ^!#?|UM_yISl-{b>O&7W6X zeLV7xvQF$M{v=!$)JgJfF|zf6+#Mg1hpc+>*Ov z6yC$_dIdS}U}wtnkTWH_SkC6WXCB1^^)BVNxv5^xrz_Q~)%(dcoYW zSGYctck#V=8Xu~&<9;OHB8&UZ#9QS8ic;#cB9zk{dnKa@f<+=c9U)3~WiN8&hK zgw802UtRkQJ26sy6?S72>iNB!^xu&OAiG@;eqJv-_4}M%WIsL;Z{gA4Gv$ySG5_NA z^!~()I6^D$XBWs7kR5Ajdh$^?4%stWa3MT{i%}IXA@8DjCm-nd6hhv!HgaXo?(&pe z8c+D_9PWnJ@{h6WYvp5c3O>|(0awXoT$>=@ff{%QEA&s|t$Z$aAiLihy;bsFY{7Fl z0cWGWYnAvtJ{%k6QG5=vJ08u`agYAHoc-t#xsQAu=Up}L6=T&u;}@KyUsbOIKdZhL z>*e$KW*m%yXp8Ugq-*{0rThr`$qyp$zIn$xRlW}+ag*!W!-{fcbtk?W*+={F3CMo= z3LoNncFH31i#QDHjc#wVk!oO`)DE>!2d z{f>8Xe*On=&X9xnab#~ekq`Enp158<86B|z?ey-)n{wtt_N?qm*&)u=tIg*j_eRc@ zi+GIwPrRJ7b4=x&@7L(%nfV1T>t$9AlXD+Wz+?DYuK+j3rSg?r3x{KaI=kLyoLO@x z`kkE&sMHwKYf);su%K^JR421L7j6o_xMn3le=*Czy|VI zWVg#K&P*<;9)j^cvz#-t^DJ*sUx|<9Px%QqFGv<8n07+kwp6oL8UAmm<$n zUA~QbBWM2_b#r8PU0U3xcThx`o}^0`&~r}{FqM9!}> zk@MyXbio%m8*?zlXR-qpmoJg`aB)5jIR|FoKDjSG*k(64?j?eIq`T=B@FRWfK-;R~Y{BOYj>Mup!aqm)JEpO!B z{02Y3yU_xdVU6B=UXAQ;E7V=(leh`S%XcH^{m-1Uu&e%Rl#@TzTPNSj)o>kdQFq2n zypKtG>rp{&$rJb^>_T_-@jMWDcWb169dDtuUf%PwpWUH;Rh@G%JHg-Tdfbp7Mpb0z z?Zi)G1&(w*yUTESt$Z==#q*em`gq*68@U|bk&nkAD1~S7Bwj@JnQ1;VS)PdtkGkcexopKz6*cu1}J)<75ZjEDyj>$oo@vxufMixEYmw{$PFz{qc}` z2%p5G(O53e7vo6y?fZ7l!rTeRYwV|A6o05E@?0D)&%?W@fg-pTBQOftLE8A-{qiRm zhV!uvL-q5HcouI&cAo4O+4-^;jq{o9I0pq?AIY0IJIz3@fogikp(y&Pi}5tRf@`8K zZomt84Bugd>)ChC;JqlMw;Pr8K2$G}&&FNyPW&q82l5NOjkS6gaew}tPvjT*SllDG zxxV@;j6y;66MQIcl1HZJb7ka5<@<07odM2BXD@RgGb=MN z^ZY2i>39u!rmO4kln>%{yodALPLVh8C+IJ4K`G?S%DvY^K97&bM{=I8JeSAdCESY4 zuguR^eC9^(i5ulU$n%+*cpS#-=RTh&SLV#qLFlGF3HQqWvHRxc!Ri)@%$pD8%t#d9n=)(HP(8<+;tQUCck~_2quN3TMf~(OX``A92o`JjXfHe&9SK zPwHKO%$~i-S(Nj!DgKZz2#?__FcZDhS7VucDW1k| z^@GR^xrrC(Wq*BL{!0FrA3~nx0XPDq)uWJ|IrDnHUPWxdY;<+4Id?%zIs49=^5ybq zuE0b1Ej|u&a5p|i&e})$8rO5)7LzA&=GUjle654-XzNf0`r+!+U{p99+hP$ZG zL3YYt^}dv|8-K!?o!Q?qdry(?z@c*H>q_kB+9a;V8~HTc=lVOGb1-MtE_oZ4pf@_{ zJ;LwvU^Ku4E?P3ngHBCeOOMb7Y9+=~z3`|+;)0Gi9$8Ls1; zTirSDY4x}~R_cx9zqq>o(eec3S0{X?Ka)?ykH{{N_qLbypTQ-_zIi*>!xcWW6gfN2 zQ16E?aI@ZXC@QbwyrbR2->a*kfO;sG)_at%;otd7>_v8|&U#nkWA*>g5gWw+P}cRT zD5>`#s_ET__3{J!0=D7=^(b`4XX@K{BXa%~;kVFOewIH#Uz9;3EY$x1dyt)Zyxvwc zm-FuVn%vH{m+>10pow1IOKy>u$cJ(>?#5O4G1PaxKmUVcQQfud9#w*Tq25(on%}@Q zIq!ybc?Xu`PW`X>DU8E=*rAtw?hyHHe50@d;&M+5H?9$oU8>$aO{{Ew_epOtmei!+Bk&D!=<)3&2pNZ`Kr}C?q znCn;}*W&zL%v!z@h4l`?C8&?udgJgnvRCH4q!6a6uf^v+lf!8l)Mq)xGf*vcqJi&XqU0R*G{TcFrkgqc4%xYu z$$ORak-3rQcsb_`n8ORvOE2>$Khw0QuQxc4a@7!=PYC{p3J$wPC@Rw+>@DKFS=F?xi9nVj>A~}^YA*}Q0ELTE@wZ<^Yt{c7yiq| zkZ1jTpF16!k$HW&UOWDSU&gn{Oniw8@GyKN=h^Jezw!aR3V+M{adyt^n%}FNaPI9Q z`k7NL<(zX3cs2gQTdp0>*_nUfHR}6$4`*j@%sJCCNBZlJ=Fd21Oy<&RWH+sd;d(j0 zGuvL3^N!GlSNcphRF<#dMm&i(@C*!)SMl?>OkTuaAu}$scZZxapd{xR{*DjAR4hSr zUwm=fkj5?twGp@w|vL>mK9l)Y)BT%6;*$I_E$) z&a6K*xSsj;6EZ(C;tbBaN;|#EsDdw0R6o0QC2p=B#maqibG|zBurRV~W+yySuQW#My^Z?vY%G^o z@FVCX=PVi{=bWg9IzewQ=RKy7&wPqA)dwN7`B8OIZiZv3ya_2tN!l{4}jUZV7b zIp^x1u3yYw@B^qR55r>l5Hygl;5jHG=S=>cZ^b~|qJJ_zL=ilQJJHm&Z;<`zZ1%r> z?*RRUtvCtU8UEz4uHA&<>e4(>ubF(I+(GWn8LFx9@SiX3==UyeJ+l}(driX2vhKi-f7r^k;p!gJ^EeO zYVg-Q60J}`-Pvcplw0z(+!b%i&tZi8A6LXna!uTd`e>o|9-q&5pt3*fUiByPFWiJX zp^IG2wHtV`x;2-b86&2Fo9ypnNHwz$3UxuOp@+ zJJqS2zjt_tpL0F$iRKDd*q$ z05tJ?{cqpDSN=VBpt%0`oS$Xxlg!)ka{e9nxVD|kaqfi~ock|7@66KoIM2%nJc&m! zQa@)`?&&6S?z7CMx8*-L^CUm_X2{QE5{By!=ggCZ@=(sanI-4H{UZ4sz6*KQ^K9RY z{7j~C4bJzS?>P5&etv(UB{D0Y;ir-NFXz_fyhfexr5-ZpvIl&}f9Pk%%;3#vEcfR; zTd(s^m?J-oJnyd|v*L4g&ec4#*?qD@@_>|I?4xe?(gNuJ-Z&+mvh$s55s6CGSent4LYgw z4CK5kB{x82b!KDspX{A|)Q9N5%10p2$0p8vIROQb=ivXI>5E)zqrQ`G<-ajRF2Y|R zGd|B#=6dFdxNmQMO_(O<9KMRDA~S0#Zj>`KGn+DFpGFrf(LbEC=arBv$#Z!#58>Jv zh$8BW$S(SjdJy*DUA;HZ9WP>`-Zj{Oe(G`j2p7c)yp1#TY9Y^mX6Z8Si&1h9&P-p8 z>@0buS4Fe z4%QnXZ{@c53+JhS!{w-n+fW}j;aHr77kut(9*n%3_2tPpUjJ@BojdRxEI?-Z{b-2B z*n-UB$GI40;~DkKm@hwzL-8ou;Y$3B=Ux8^b-XTiN7lrV`WR>|-c#LYw&7v?j^}ZkYx8hD zrlXMF(Wr=F>I-o$vdf*Xw^Lq(>GFp>2ph2*=jeTgx$=EjE?%)Me2W$J?|-X zVY#6EGC!uy+1s8c;xxRfpZDt*xq z>xo(T6&v&i<4yT-E{hY<1RvpN9Oqg+)R8a6PWfc6jel{E`UjrJ*^}DIU*RrwVRdD$ zfP#A2H?QOQsDgKJb*}5LlCx_LP+x+Y>Zbe#n#^HE(}r!aMJ&WX>=wo3eP$4B3~a#`EbIWgySod!3-j3B-QC^Y@n7?;^}Eis z?t5lGAm@Gdo|%mY&?D#(G%;TjfffauTSk#D0S#do?1Ro=ccR%7s2uoQ`2J{p z;^ydI;&LzsauVAe!G5k(#P;>JKketcM{Y4{=gHok?eS&AbHRFoeLg3^J_p+y{`k$T z+52k=*6$vZ+k-EN+B;5xuL3XeZ^1rO`~R>#VrPCKxsj-y8QZ^^pdPXPtarga5BvPH zlegYs=hgOUBS=CnIeGwH3f8x+r`x->J#Oc>KI~_0Il2$6gW8$0yY2?bukd!x?A=X9ZBO+X&*3#b936{J zMQ4NU9jlSndyZQkp27ym4Clc1rqx(Ge^yskpf)pA2dnLKQJen?qt>rp!b!Z|F*a7Tw0B?x?r=i_4Oje@rdnQuf!L`CxRyUFR0b>5~%fT>y2IUN5PZW>ajoG zfsVw-Q0wPK(Ee}?{~Fri)1tG`Eie;)zz6vMS=i2gVdCECCG;zN$1j4zPz#QO&6rVW z6^P=QpJ)fvYH=snj^7P7%bXz&$Nz>{e0;PhB*&LUFN5`w=U}s-2h;}Z8^6dk#+L-E zQCXl7SY7#pK7gNi>xmsvtE~c7E83!ga1ZVh|3L?%0qAPfYQS*xD};j8Mw?0U!!BU? zfAjKM%xpLS707Kvt>0T+sS7rDSY7FkzYnQduZr5NVl(O^;)%r8*Vd!fa|@tWcPf*! zzS|w$MEnD+p1&oo00)Wx!49}ed>ieHeutf4HQMe8mr*z3c~Fu#A3C4hC#Xz(9eqzO z25o`1g1yiLPJ#7un{PhiC!=}MU~*&dx$uJ_o8@5%oF{IMmgkvtc)Ls3UB~KZY4StK z<%er<2)>ZBS@S=97qEFh8vRAS82SVzLSo`nVDr{MG?H8(`VDOc^C5t>0%&~l!yqFh z1M3&3%(Lc)FOLty7eco}Hhc;6G&&Sbj9Sn7fWHY#h_e$X!uvuR;+<%J@Ph`#i&2~N z65^5n-Oa2I*?q4c+6gkkBi0t8CBYZ0KbAo~!2Ub|>)q9eeel-TZ5~aJ76 za-cRN&%i&!|A!_8>(OB_h}>_~2VDW>p*4&mHyr){p80~fBC*{aY`!grw>e-0`8@C! zGLWj$fOsR=oNcqS-ECG8FJR5)jGE{jNJwrC`WX#| zA^2!?4%q#?9NHTE$WH*9JJ%6UgH^-{AU?UQXjXE$(MrTNSF|LqL>vKv9}oNSc1K8n z&j)jeAHrU|%{dv+evqBqFzAeLjP^p)z*)!z(;*J|QV zd}fRBHs{qK?@oLI{r}(T4IzF5y~x!-i^DB^5}1TvigrOequtO!um)_7-9;`IY_qRM z7hogcF}@hO9a2IwXa_sUw}*{TmbeWB@wFB0d*L60owGp@19rx(w%b0lb8CIY`p9tz zz}tDO$-2z~_nbH zO9fV!E~8d2qVRoDtBd8p&fZ62dk0>4!B0VLugrppkO(}<*?hJPKNkiQ??LOLzrgy? zU9es`8qTp6A6|gX5i7_ohTmZI!s?0b)ynV+Z+$KWelS!ZP5`IiCRh)sjwS)?e@)SU zP{f{vASevB&z?gc@Pzkp1`dGD2X@~3@O9SjtrrFpw*xn@dRU*eRjBnbt1q^{c0+CQ zRs*)-uj4gNKyo%0#QrP0agN$|j1&ntzu z+Ex(%2X&x1xjpadZ58{mLHRnHF;=5Ppd}QAvbGLwU<>gLSO(=F2RZ9sJ@9p47z~FZa0{#k4(A!0qg$bA zp#VNFYumu;dq;9F(Uzzm#Ne$5UBg$x`@?MbM{NBuK7I~96PgNk;zLmfB0!1lUQiZ4 z310)9hju~}pf=APL>CfUzqeYw6g@#a6m5@gMFSuq{s&qVdVt+4#*iC`--}wE{)YdA zo-9 zCZ5ZU9|{F6NBjsvk+-;e#AZD3;qO}1MQBkMeW|&4b2Em$axbtN2d`Nhtc?QumQ4x&CmzQ`+)uV zKyEv%friA5(RrxN0CpdWL(G5wyMG+Q6ofLw&0sA)7K*_ruo=$gK zC$W7l@!=t{z5gK8J}28#_V-xr`G$5u?WI}0w%)Xo(B7BL7`CTVqlr+fX;#1Nv#@<< zb;90>oxAN|wR-}wotu`Z)i8S>R_A>1_W9dci-*sETHmprX`e+4VtY4v(HH1N)b^&G z&$oCxXWvk(U$*yc4;@79+}N3}jkhym`^G-ID-esHjs6Gro@T=}XbtvWZSPKml`xsu z&f;DOz*hi!ug{<|zCCK^!)l53?P6k#Z20=NnKGKn#?NRFiGx2t2GlT8#_^6!$I}bbHDc)*)O$fwW&)bGt z{j}ODXiHQf3Eq13QM~QhrNl$gGjI^!3%w0iXYB0S-YrEw7M2rlM(aQXSe*|eXSL1h zyv+mC@eZ6P-xI7iSRGgb<%#V)*jYOSFX1-ygu7t-y+3*j_Q3zw89Q@D!0O{+^aW%B zt2vj+S+6Ms8^LO`)r@G=YQS&QYTpA$kGI+7F#3Vm`bP{DA-1|d7px|oB3Be)G+^@YB)eXhpOXMB(S5_tCUqebs7o2n-=#h1_CvKjgu; zfiw_EZ1Z4S{1zBUoEt3-`(QU%&#H;qJ#aAVb72dlg6HH9z(;&G)aJxw_>5>H)B$gJ zP5cQ2?*~s|3REX&cP5+9(-Y^08RV=_je`4N_oWf!M#3N{4*4Jv`JJo}!+XI<{3qCi z|AVfB$IyZJCwc)b1NHEapcQ@*%z}n+h};rbi?0CZ!S27+$Yq0##M98D=v}k`B*3>v z+rkET4#~lT{4wyw=Y)6oK)zQyd@#Nwx)1CQSsiUhZaq2;JqZcO+nk(^+)d&xupFv^ z-6h76-v*hp*i3xe!Jy)rXT(~J_!DS2XQ3&5ffX-z)=nd`_^wZ|B?IS5lZwu0IsQ+uv`s+|I7`hK=N{ zt}Mb^&$s<<|9@7)t$vh7t!^GhZO>YNwmog{)%K8m_O{pU{SPN+XTtVdLNpoJ>|p!f z`n1&s`>bq_IItc64y-mO1FPea1py$Z4Pa;5>XW_qR1ie`6nzG^&+ZXhp9#a;d$RiW0&ji$C)jzh{cg3z z&Ysn<$*>Y%1oc5}26>CNC$|1*Gr|bs1o%~8`}#2P0(2YL+*%ZkPtIyz3p6!qd;JvN z&dvzp1E}?{S@<)k?ax~HH822v5-$X+u~rW*Wg#m_(+ft{fO=qqy8?`@C9fc3ML zsO|L-)au44u)0tcY!+#S`lELKt!CE&o4Kq8*erek-N(AshW)SuhQSB0a}b6WhU{Ry zVhfrAwKIGVwZ5|rTEc5$n+vR_+KkYK*v|M`G%s;3)M~2rq}(u`*!uci{BE$heG9oB zc$;@Sp;k9+X1<8O%vvfm4{H6@1MN)ghX%lLu)6#N-3)Erl_eITkzjWW ztI4lmAMsqY7n&J;h&BYP$5w0n@%bT^wZ3RMSPDtuD%gDX7@Fa8z$usyFUT!JH$e@2 z1-K5Gh|fb5SYMiiwgsE953(MLw>mZwpAEGdJ_P>*6rT;Xx<4DWndu6-J@}JQ5&r{H zkPjrDj{gPU@MEAFG$$?q;rM9O8#2Qgu$o^EzTwM))w?ys8Q>Q5A^rm=@E5@5qCLbV zU_S&xMRMO!H`ZsO?d>`AAsQF`4b>oqxFF9?z<7V(H&s5y(4jB^cKv=zX9v# ziY7thf!!B+qAkJd^c2>9!5!iTFb7&g6-W-jta*d=fG@-W_|H%emckS056z%FM8JLc zk7w7z35X9R$?Zmuqeszh=rpK+x8833sU29qtPSbmEb(nfi?`lX73~Gq>obzi3opU$ zQW?n~!du_5JIEno`(wS%?g(e`Ss^jZ1iO#>@VwnePNUPwT|`@>mC*!fM>G?96Sdj$ zJpLKj{Vxkz1Z;-?gFfP!)ldqq5O;u5P@eT+U{G=dc5 zBG9p@%|$El;V_Z-2sDAw#Mf*ca^wG^lc5bfg(NVLyv^3v(L%799P+<=&pfm=6oai$ z7FI!19;gYykO9&_dcH>w)EB(*bJ4*(y8!GPR)U?d5I(^m(l(pe{p2pO&3yIAb%&SG z8;X(N1rP9EkR2Zj>F~ZV9@Y_GfV9w)cmw($>V|fOGWZ-&5TBI4BO&A`?ngWceFR0x zH6gx@zl7SXbq!5Pd=C8pLo8362U0^N=tj=&8(;CEuofyqJaVI0yMw=vMxvk5iBJ)r z2pU5)e1c=-FQHw)?w&J`~`o5jxgD@Vf#@KsH3M0WjqiDQ^ZSQG2UwOOuX&;SzxoC&5wQYR*y!o_5*LV!}gZd zE33n{=NsT#g4OV$V6~t=Sd9t;t3y`z_v7sx+ZnO?FaovuYBR0P&UW_geAzrO4Sfn1 z!R9(UYgRk|p;pIirm;G0=Xwj+xv(12kTt85QK;4C-QvwE2Xb%QNn=kYbTldl0) z@O63C&Uaqo;l%$?JKsMbJ>CPXF3%^nbDABsGg%sK4;{hk?Mc*nr5_rI7KL_Df%qWU znLkOq57y$(pcUZ?R3mPTT2HV(SP1_Lg7H=ptxncK2f=!XB3=Tq_-C*KzY~J+o8U11 z1{?vKbIy<(X&!8@x=4Hz?A~BCJDS`gcn1D(3#^Cwpf;;?LWjXuSj5*|g);c5@CH5; zPe%)*R<~B6sfo{`{b4bzCH8_;@Rm3^l!wa1*5h1gHmJbYPbJQczYkWMXP_6r=D*Eo z71%?51I)%3f!1I>Z#lUJ@B!iyhr?WW2Q^_o`G#mBc!^KI_pAiRh-(n{L+_ynz`+kg zo1%A7n~B?^R>xaHPZ-3S&5F+MjNQxL5ZnE?7J3`j!Bc1sHal+P*(cBd-v<2%FIWqN z0GLnQ5B(1H;S})$$cq0Bm%;86qtI~Z0CicfgqDQeP>%Qp8h~B~o0WPK+nv55`iER^ zxCtJ_o^TzWfX$<;(dsahb(=ZIpw{P-;V0sY<9niZZ?}2+J^mATlRpm`@yVbZp5_1D zF+QQg$)!hYqczdxXe4xnfy7@RKYkk84NBot!Zmmc<@jbvP@4%dqx|>3d$-MJZSc9# zTxekyT&Nd>g3X@k(9-0Op^qRKAA=5oDd0zZ01`t7XaqaSXMk1sS?DCNIVy;FEc796 zg$9zdnI;SV3c01|A2bBrh`vVcUOXB<4&MZw0XAzaBd!j&i1VO6( z)b48E@#FBF;Szow41sje2YNs)ILCSd^doA&tC^0z2fIsWB_AKHN?Zb84K_kM;Z01Qqhu>+Icaffit&v7OOjs3&Um>NePUC_o$^wa|1@Y z8uc3{KzZT?=oOgEbGC1+&h5ZkjroPzH?X~Id+<3tAbtk6Cpw|F@2u|Hx%mlI$Yn+C zEL$%)4Bv>Y7g;^AelZYld%rf`-lf&K6Y!IKci06Dh?m1iyw%6Q_?BR`YA>uNZ|BB( zkju5Yh5&LSz~%+3$AeI-dq<%i@ocm&x&*a-XY$2IElCZV11w` z{Da>RMt(OM3q|pEzV^aHh$0sc&4pG%?VM)=yBAo^wmzGR*lNrR^oOnS+!xf&^H;pp z(~NM@Jlq32yFsiwcn??!*U2S@JoqPIeaO!IQ~Vs%>O~-aF8&?dfEUD8uY6IPeb=K_ zlkbvyWqEQHi4}$u+j*`4R&00y+8yewDI5%1x-Hc{~CirvE1aCd%0)7+T6ZHnWque3hhku8*h3xQ$ zI3;w(+su#{oeC=;HGBr^IbC?BDYSu`(3AWKC<}|A0$hdctogxlC{AqsZYJLP_-^79 z5RQ+8PtY7fAt!m8XNKbY!Xe_;&<&r1yv;!E@$29&egZlPoe5j;`JtmhY`wZ1-WR$O z*FlS*xzY4c4RR3MO!OaqD83w6-~UMb5x)wSf%W4J=nt@-=)-ziI0J{^2081c<)9co znA}=ehhK&cK@~lN+ANS3{RhWjHTgv3_M?f>3*-{Qe^3~Xkhk947CjBq$^C+Ua2~Fb zvt~96{{|WT1I3@8sG#YIMb|>qBCV+zEzrsW)X5SBr;NP=mb7vm>Hkbr`?0L8ZP}@0MA*sI<^M2`tTaHJ#1%gFL;u(^YIoA5Zn8n zfwyyM@1g_VK0`Z$RxfVBWb)ZjtEDH>TWb!C-sY2dxe^6I?+{g9kKX&GuOW z)Xr8`bUK>XPj%8yTqKZtGvn`eIz+x_SwvDIINT@VH%;XBmknJwVpw?bE_1Gb0a zpb5d|umL|GtVh|bJ{z@qWoK>;bigM@2cmW^exTE! zA^d_OtXVDEgSVPzb*3Y6HZ&bt4s51~Pi*z$3Gr_@hqv1D5Ufs}C)W<&9z6zW@K(QV z=CNA+2>%+aUgbw0!$15K)aDVZ-MP?V#8%&45`RG(q3_X8=svUnT*X_BzKRck;$U;@ z1oA1s<_4?VRs&|ER__kN3h*R10^IOE5D7MiS=|_fw>o_kvh&O%7>qBATHOu7Uw|aU zb5ZLvOYtorH4GuP`_+4V8L+yQ%<@nNZ~eFv`U2d+=DkI%1;Qq{3_;}DvUUS+HTX1G z&7BH1hg%=|2{w;*N9{gy7Y&CPo@oaSpc%0j`VZ#8BB%oW$y-hJMQxURgJuLj2nCx9 zsz6R?4cB?rHgq=R8*wW%Et(Z><86Mox#1S^X{Z5}$yY>_Rgqw$r{5pW!z3+Ce=pm|a2Ep^dzP??nU{DczZ z?CxOwtQh_n zsLk68&g*YAbf*|5c z=l~c44sl}Ce%G)L{|w5*WT-{{Cj7#Gf(!VsXaLWy#@E89gdF(aV6#I6@e}++^ek$# zT4A&}M4A#;1e+!E62IiR+^7fI7R`)Sg!Xv5Cv-wH5Zk<(6mR!TH*`EXZ}czv8MPT} zEZU6t1X>E7;uoMB(Jl}Sxu6%+faY+X^-1UlG#YiG8PWLA9N!rp;s>Ey(O~`_`(4U6 z{48=g(f-f{e+`Q@4S-dkvCi!8!Cu z(P~;Pu4tu$x%JxG@iJ<`5sCjO)QML%P|qb#qPg9AY1GHZ;?=l-|GA1UT~C6mYIuXT z+P2^ZSvx1Qj#(M*3P}{AH7b0RjwOAiRgO2Z_h@D*@w2k-uM?%c$GwsH3EIoF|Gf3u zx=>AZ(py`U&Z@OzJL{AU{<=MDS4}cui!<|kGNk&mX@Z@qi z7dTl?`91UR)F`{2ys}jmM0S+=OY3TyH?8zmz)fj4d#jYq99MgI=F(DS$IHs5ZDi8c zo09K#ejRpnpbj}4=(s+n(I30xs#EBYj9I^5;$7G$$ZzOSkB>#P z;IoBNy4WO%EFYzAMUOan3uMut$L%z>v1*ZFpPiFGddb@Fs%brRSpx`e#)Q4eLKeDn@#0 z)rrMi{i5n?#@Y|%N0CDMxXNWo^tOgJ>D*Hnc)XJRNhipe?w(ry%n;c;aE{|Uc8^oL zU0*FUG^^aWm`U=rDJYdHJ&^)~lBnO&8aiv!0L@arw9Xm6NwPH0BLP){wN;ZZPSw`c zB>Vm+uD+jq_4C8_dUxYPnNsV#q?@r^manR<{RaJ%e#ctN(ugWL^Fw#ZHa%J{y{)DX zTNTqzK7}+_{I=Tsa$0@YFia;+=%o2uW!H-PZo59VFQt+siCiGRVq_|9dMc&XdgiO$ z4;0YFZeg0RV5GLL(^{7n3)Y7Jy6H!Mk#Dn(i2vNvax~dv85)#9eRjN%nB=MhJ>RZg5A_0{FU)imz@ zlA8OOr`EjE!RguNqtmF7OX~!c%Kp zc-Gyycp{UW+%sMN>`Sc2SA23#dk@efKG$TY_X+38_0CShS3@M(&6HYW<{{a8wuk;q zxlnRkye8xSHPuvJJ!N0F=i*tix7LmBs!t=@>#K5ZI&@EOb+6;0KMICucvv%a+wee= zcIv4q?yeP&A4_HG!e6rT(;C;eEL&vNsqxOFcwyQs&nFr4y1cv&S>qH4{VuH&RMfcZ zW;pjxIr{9$GAG%}^4g=?R*C+XRO61#FJ)$=(re3qNRMi7of^*`%ZkboTIuZx349o= zgHkru+#%cLbcRy;EUUZb%Nn7D&Te)(e~T+)vN`OP+|J)`Dmk|P5O>(|5=CVw>WJ1L{PKs~$dfgf4RGr;S8y7317ZR?N z8Wk?c!k_sxQL@!i_f~G%y?B|-^_(Q*Gv1O$!^>!kl{uZei$iom`Jb+9uPf_qr;r|A z?(5Wdb&+F((x`XNAI^FYA3cyFu>^WW>hbg)oo;EvWKZr6+N^n~PP>#upVaQF^J51~ z5WQpXn5;7My0_$=pIG-!ORd3Snp zaGus(>Uj2!bybhaFALrk)vMiVtMBq#k}G#vO%m2#N3HYKnA@JZba@47_aw1yT#;Yv z>}jMyuWIP8cX?&qf7P_`Es%y4( z=OyX%%C6^us#Uj@*Zun@OS{ie`Z@6n$+Ne;Mr1rC`zz0rEOq|Kzlat(Z-0I5d@8l$ zULitbXZGR!7uD|b1NCaYC?`qjfjaDKEAf4CN*b*5(;QU?$e)Q5}IN)GiYm>-rO-LkjPd+>eTC=ePazh-U?-se63Ad+oZ6?O#QQ*WMu4e`L}+&Ex3U zQ)8t5k{r@6QG88MC|a)^z9~nZCD!m~FJ#G?%sRJ?YWx!;C8F~lSC6n3`s&m`c~&!4 z`glEbd;*8M9-WS-lco;PB{ga8tG%?Z6RlYbMQXUOzj_Bg zb6W0CsMpudbfOxyRnOKTy2+=w({4#2O_sQ^jw{JQwx=V`mh^Jq2E_A%s)e+yzWwmC_ zo*KR@KzpVvqzQL6k**snOS4paU5n~pk=DDW%aZN`^+V7o$+k91?)Z4=;7ZY&#Ph9a zp1zvk&^k#``-7BwT0thf8Y}aXhHF5BLvnLkPAAote!6nuC0X5nymb6`R7P(tsoB0Y z(UC9X>!s9x;a{c07@vibgLM~*HPO}o}ivqJ9x1P`C;Fyv+D_&J6aml&Pj-q>< z+)uk{9ap3-tLmxk%7y8!bxoz_SC?ilc1x<>O0N66=GL2!;_AfS)%4w-3G$#}Ux{w) z;dsYha*8Ktr!Us0&?dpDG}GCHT47sC89Te0&T!l4+VXO&{H)hoS9JI!fgT?Ec>HIH z7@J+I&+eylvGKG|f*}%l=&Ng)WY$V${PgYDRC;qy zGF{#3kW@&x%#~}>B#F3NTp!Ger-yFel6@E4HO>B`((sq+{kI=o9`pO^i>d>(Pw6YN zeN~u#O<6(99B8czw!z>r9F?IzgPjANZ%VuwZ>4<7h7#{x zI{m_o()?F8z1%*N_NfvnQLQG*Q12B|d4x+g_r5EG*45PD(8*5itqpX|i&1jmZW?`X zW2vlqT}LNJ4swq9=g{tZCdtG!&GbY_GRJM#Kppxsm3~VWBYrFLI0;q`&_89zO1H=; ztu!rMQxvPNquP|zMgvRf=H&lL`vR)5m)=VMfGjfn%L)0ntg2Kk<0Av}CeVU)#>%## z)igRyCaE;1t8Vg+)QuH=G;Q1d8n4xMkPxr+j13gCg*t}vB}%((7LO|BjpfzI{Bp>dsAJ#6Hk@&?hmEe{YKj0 zYFWJ=n@-mR2FidjzFMO8cqjSWDN?s-77YmBDY-ZojhjA_VZP<{c+(cL*X_2v{k}my zO>U=~UzO9_$Fl0kCGqu6h9a7#R!RLjq_$?-mqcscnJSyUpK!Wtj@0X!!gYPCc)Bl5 zM=j@9NxgCx(ys+8YD>4?daLMAY1FN>o>&~Ll8`!{t7@jLzA1PI9eOOv3gvbY@r00iCsX@r_bp#vs|dvz?aP-obGS)RI>T zF3ZV`v*p_F5*jD#P&t~TnKsy+U*~kmB85Ud)HNqq{-pF%zj^62;jwJm^hyRj*!P!w zPqJI4ME2H=Qyc5NO5OGLnwj!3p_d+78lff9wpO?K6}4APdHs=OfIfe4&A)$6)u!nW z%8$Ai<$+r|{{`7R^`GAvIlrsFy6;J%2_{w6M|BT7X?C2H5%ZdA+YIv@uk3fkGixhd zasGjPpYmIFX89=@_Gi+(PqsM==11s)jehL+5dC-HnuLX%lg#doG(m;_I@0I7OgSq$ zLZ&&JJ@aYfHa}(R>ICZkvxx5Q+((OqN6FA;L7Hr226^1rOWQ5&BinoR)VM+2^!}9O zx_WRHU9jS!Q{JP8R;e*Tw(p-KIWwG-C#~bCtJDIyk$1mL{eD`uF{=&kQe4*ib#xNt znFEB&r^*E897Ir+Dban>fNuD*RT>tHP^Q5(NV_2Al?!gsf1s9j3$ z<=rHe(#4AJ#6o>dZ8&Rafp!_i=omabKFyP2EmEk&$UOO1a-XbkJij@qOA;PaoRn ztjyu38!B~lW{fB#KJE+U&6+ye+=-+6`=!&rBuzfLz10cL;JCeUXYXaxh&aFH^&&@q zhg^_|BP}&P^WWuO8Kh~s92&n!dtJUNSRU`pqz?~umkNzSo#tb>PbSZ$i~knVR_)xh z=yp#Tx+vP^a`n}5M+$3=2sbCu%dYyS?;BaN@TPQ6epiO)57N)+8cUiHZrZ>Y&&-GATBhB@dTx1 z^*)!DJ$%M_bgjL{QpIxsl9bUu4by!Y3Y$<+($qb60wr(AL!JS{@q{*>1?=~HSekE(jB>SQUoZMKuI z=W%(PsF5a2-CJv&imR!*CeRAcg4Dn1H7999Z=KS=qHb7oPZG~9C}X|iXp=G@<-(pF zaw^U==k8BG$e&7%YT zyUQo;)W@e)*OiZUOPZh>PTCP4T?0xK)RmRrNsacYq{N#ldVX`LHfR&Bvi5=O;rumy zQAn2ldn_LgiC(K*%{A|Opd24QP<;+2))||!=n&7sveSKz6FoSS{<$zZ2eUd3LAEe&V3G#GWLP`F>UuQ4RA;}i?m1`{v%9qh$ zn)_lpJw#0ji#aI;HWt$gD>CW$g@vWg-p)FDAHQ?)t*!Uscxd}Oza;;wg!-=3UD>&9 zhBS!qmnr&Jb|w|Q*KV{kZRIP8Zr)zquO^f^Db728{EkV|BROQ)I(I#Oj?XL_dSy(SW zuk3uy(oC|Ztf|-LMrq{33_7#xFUis(t5i+7z*&BaJ4V+nQt@D0{k$cWzzH#->2Opi=-Osh|*f6JMfju(&dw>iI>ZiwNx-?mu#`-3EEgk)=g;TEQ z6sdHmxlUVMOy`&UD4TL-m#za|I>U=)*ZzOv>FBYUb@Yb3@{xXis;rc$Qbr3EarD-X^jiB*Lf=HtbEM1m7?@lI!AxJt*d97RMCf7+iT{4tgb-sY8P}OugJYu~oNvRMW_~@?Wn>bzu zPns+vPR7WKkI{OZyH=8oIqCb!^hApBvbBGVjJc9jyWi?6tLJ3XPrE*ee_S^`zO}9< z$>gOig7-P~%fFIojce<$NMAjBxV8GV>8WE!uage9mdccmQyq_>*D^nDOWAyTwj?V% zQhH=5D_J(bloUBLY0j-(rG5J3nrhfNxzRD&shzpG3^?*l#+>rfA?VMwa%V~+I zukx$(csbwcraX8bM|vG9>+C3YR?3YYBrO-GcCPO$rB6b8Xq!GgG}q(b60b*Q4L;vn zs}I;MtvhDYaVsCl;VJLr>$$o*K3QGu)3$`pqz6>GcTmE1)Yp^q+_lK9Z!*g>zD^$@ zF4w5b@@;w*JrUhR4@@i|5shl;+q`dO%B~uEcaVoZPrcREuzE>xeL3U&*RQDhRIR7U z_Fj=qg+x29zv-m$80%`j_>vrHpItr-@Y26w4<+KxTj?`!k_=n0$u;+HuqIjFQx>%J z*Ham0IA_zhb7lSMuQ}7Dr*?mn;ZIv@#pX@*#qeIvy**cC+lAB`G%&Z6jDJnaUQVYA znm&~LJM!t${LAE2bg<4IkXnltF0VbK(rP20Kn?k<^802SsWrBTeBJV$-?7KlVyj=t zf|+;aTGCD$zkFql?ekT3HtDNhiVt=TKQh+Iex-q~Kaf~b%v>v(-uP>u+Q~KOVIj@& z;+2$bbyP|`d+($f|H-*A{f^97sZ+&ov^<%&RSqBRt67$E zPr6sqai%5E;-ix};{wU01JuOQ(~%#v^shEQ*kAB(>Svu zwD*U8dirGsZSrEJ#HpD=S04ZA9E>_3wIUu!O!acoppTC(FXiZP`eBDHKjdM~;ncLA zTKwA&$&}w+67{Je*W6 zC+Me#9;xH5*9Mhh9_05=N4n|lr2W;a$R9Z#7+=QT?xLRab7_trO*J%02kG;VN6(%yrMpxEJ8(VAU33c`6iYam~ zs-VUZM1`FjWZXtk7bcNxeMQ>0f(?PQTNPJdK3^>7sykADZ5E9=!N21*&Zo%+)z^nV7CSqdwotz=UG(mlVmhhl5tjt_ zR$Vkrs@C}?hw`V?V<$4`=&8e`NW&MBb^l9=-`HKZt|=xXzWs9w&8efS|4XC!kA>Stq|wHPpvftLYT~XuXv-vEC?ELqDHL zBQw0Vxti5D=A7?ZM6Y)EBg0O|lecqwYp0Scq;}i~^7ndP?xqE_#)0!PW_~tjeujy% z_C^DZ9++C+bo?VB-VOEi>)yJ}l~|g$tfc4tYsh7+&x8?|(C?VU{%gZ)1<#_E= zPK$sbz4@n=9_ig$es@|fy;28iuC>W!WzG0faKavE*XsVx!OxL8yILzbT{znHVr(a! z;n_f@BwIz@TrY3#*3g{XF{6hi(H>X4HQD$~8kiWF^Cc{Q0bR3mgrxk^OalhB&{Ju`c~8-L??+q5t6nAb`LC(Inh>nZrzdh! zly9I5((IIUi+bzBuSs>x>tGEGd?9gr*U{vzIQlhxX3hO_nIzs>S+6XwuEmCxbUnOv zP}T>Ya-ut?(U0>Zb?%tX+HT_~+4bIC&qrm}u5Pbo`jrtfWWaKfOnvlrwxLq6@*(-z zw4SD{Jx8nKhzcHLbCvt5bTyD5quDx>{_lyOz%DrwevgcfB0xuAbX! zYdyCp2{@Ea@AL0=&Z}@e^RuW{%;Tw_cSLCAseU@V%us2bqqA0CI9RUVd@p~iw$^#e zL-oz33fl8S8;u^&L4RJDE79?a>GdW-x_oCteRw08{8uKf7L5PR<-g{D^U!0iG?=%{ zdEpjM^EK?LOE$&V6;-x7@uK|nRj;E?P`C8@a8_dV+A>(OJT2^W4DP2zN8XjYuLn!U z*=2RO`xDtxYMYE5*IpO+g=@=Z@pR?4Mp|!Oek~YVLvsuZ(6;k#NWrrOB&2l>oiOpL zQ)*A77RfP98hgi6--Cs(J!*3Zu}=#h`jb<2xWlKGjpylVc;6>#v9)2>JcIoG$aHZHy2x%`KJ zJ5AuJBX1PYdcDeMwVGj4^-oo)+^oBvj4h=5!#ri=(NK+hI;R#4UN2$Idpa3zR@bEb z8%xi(g>}T=MUrfB3LTxRmH(a9zFNP3T79vlf|hWuOWEV!q|n_8dOqTpe16_rr@4Z3 zK>mK}IVXd*&3{J@ObS(xscteNvX>5CP+oEj?WSX16xA*%SIhgHS0r}L5oi5>MP&6; zAFVttzMjw8S$)0F$ak-^@?&0cy;^6Vyg9i*mb(?#C7)Ni+I;krhHtuPu?tl+dHxsD ztmry{yp^B)J9q;V=uity1Ldn>Cy)!gEelPL^^f+J!$71rg?u& zac-xoDNmng)^r8p%g?Zcl5$_ivfXh7Qgp z<2N-CCupFo?=oEq_bDb@GsKtQDGJM*?z?3Dgb=MXwFjT^LZ{n}5Pkctx30fiO-H}) zq5Zn^Z?XHDY8rlzIY;Vi{6HT)%^j#oWBz@(ZanF`*-c0Piqv^CKT53=4Q1caK=sR8 z)0sRcp61+;UZ=-2*Pj0FTFl=|&kWw@RP;_QMP6snL(4mB=Ai$azegWCJ1Vr+d+SnZ zn+GH0dd2}-vT-79*65Sm?ByY`M@#Fvf4Ma5>nJ%E;il_PU69?|AIq<+Ep$hzS(3ih zDT(*x0>94+*TI=%WMAg`+Ab!CY>o2K*i-p+;LFt#R%EF(Jl z!(+7Z9$IdAdr(#@<>u z-WK`wG^uVM9;`$9wUUsvjWwZE(Nukk$kWu*<>I*d5?ykDCSOrW7v^fuFkV#4EpDMH z+hviEGOaaL!@OE^ZEhJ|^8YwG@3@}7KZ++ZB57Hry@V)=>T}PLQqe@EC=FUF6{(C0 z$;yoET_U?-W<+G~z4ArbWXt~D$M662&inIz-+N!@JkR;4(){vABxrW#)Wu_XaL^T$ z7eB$3vR3qNn=4fN`0&ln2wrM2ftfFRuu1k3?0DqJ0-YmpR}zTN?#Ag=zKpe725s9x zeAgJmAprphYw5}dS>&px<7q!c@*<%T{N>P;-}Q17cGH!3>(W95?c0tiqmE#}Y(MD( z45d%322HsDCC@w`LZPW0#e?alEFAR+{hMb{`;sbG zx<5x*KOcInH$p{zJ4U+&aNS!l&Xp|Azk=&#a^`N`nb}=WK)qlXdxi=|6spts#srN1 zdJ9=r_MGsqUeWBRzC7gjg|90tMCuAJX7=5JHqqA1_2~?|86U*S;ivI?>Ui;VMgr!vy^cADS0mCkQ&Aps z0&BIhIVWr+E^Ypfhf`PM(%pd=ePliktXqz^AA2LVM>4dXGP!ooOhg*_v$8RT14~Y! zTieIDP?HG<)gky^QYWIu#)`W)qG?#~#KoJ(QvXi}_S`s@D%n~5J>LyqE~PN|#dkR0 zb{D75#0%}4U776E1E)`W@dKL zJ?gvT!lfvBuK_~`sf)E$<*;3T5Uv$c|ILZx-*HKtpRgO2)-Q1J={D&8d@7o5G3T6v z`NHMt4t!L3f>!-*-R-uTosk~{tlL*>Bj(8#EPN#tVm-)Tb4 zDESN)jb_V(o-mWzp6h?hd#eXnq_$usng{eh!hDZQ~=yqP6L zRBIifwsZ#EW`9Ogw-&4mv17|CWyCdOE7|4tb8eu(K^I&=H82F3Sn z-kkR|lozV~sNJz137hq))GM5Zwkph(?`l%ZXx`BN1{Kkis@{Xyp;(iPubdVOb}vA+ zPCvAg>qX|FjhInu$|{?7yx0E_jB_S&d|EN8Xhr9CgIKHd7S?{wc%|^>_6`f6*wtR@ z#fRX!WC9;;Sq``H9@J0VC(PZYKb7Rf6SuF!+W9`r>pl40{)b}Rs+OF#J55-J3}Er@ z2`H9~Ld&-Uu*9xGq-w1f>uaau&l0JJCp^T!$K{G)n;+sx?O5h!P2{a+gK%P-0SDL} z#FB$HFg>it`Il>utr(5KgRimwrar@~Z^GzHbN1NUmlI|vnEu$Doqtth^ThW!`%a#} zu8tKJbz1Z)vtsWXf!yL9BAKqABFJ^QxIXbMoWmslQ@2Qw?`npgJO9J;U4_D1YFTOV zPEZb5jmz(K_;C3Q1fLmBrC;|I4jn$>!(B6a?lNck%rWfG2O`BDe52GED#rpi($||8 ze&}+Lz8k`Zzd?Sf^bZ@Z<)0)v7s5FJY4To{V;f~x|u}sVy{RpoN<-TehjuG4CbL6)N zn{GQ|bF+$#p2)7}$v)PKL4JioQ zIE*@3TXD|)B1{Tg=n~eBiw=Cl=I^ucsG$zm{3kJT-UneVxvrk4@8Q*ol_;q5WMP~o z!V|tgHF`1!DKEvG3lZG&vj_cD{=(~l1Mh8`gxfuzioFZk((-Ev7gg%hrDg(;jf!JM zyCi0{9mAjoU+7C;{Z&IUj}5h;;;$Qr^nHw#mQ&>K9L5kwV-#t2qPy*Bachtk55!KT z!32LsScJjXNEL5GXJ90C8Nb__-Il5IZ&5C?5A@@neNT`+_B}q|p2+IC-7&PZGp;TO zWtqWL_HGQPS(Ymc9~jWDWGUw#26q|T(?u1v&U8NrddZfx7*3KnOlbN@se54H@KFL*VHu(Szll|B@>m{ad>m@w_saa3@j!kN9(X{F$*dUoy z>oA`6E5YWFa1IEF;Lo4O;D5Fso@u7zMoDK{=iEe2&Q}Dq8^}QqyU^0efYrm=)5qf< z>Wizzw$}4;t)Dg5>0HHOts#7PP>G}Uh#xQG=%ixA&(~rwGR&3MPT^d0sF#>DFPLpx zCUSLQGpAP?y?Nzm63ph^gMY7RzE={Q6W^XAjJt8?pgoxLp$1=dn{w^@LcG{_1`3lO zIJ190iod0!q~4y!TD~k;(oh4~>}r5!jy5!%y8ucz zEji0VlO-Ga(Pz3E8;;pZ-)@zn-M^94llrrthO5}HPcWux2%SP)*nIR0vF>&TBT`56 z*NJp4oB0{j-(>Jxw@AjP1n|W3E%^L!6mte$MM;1mU576~kJC;JJuLO&Ely0`*aL@y z=VIcrK!%tv5KFqP6z?K>2@i`0h-_W}jY+1w+-A9AmQEU{={R%6+V|)sb&iAPsT?*c z6tmjgMy~{Q@!;iLVG|-(tE;N~;B^R1OY7lw<|cA;rlEBCcQJ5=Bllm}CSJAJ0ZY?J zy07>xM6@NnzW1Q(h&`h6>rWWW_&oN zxSP_AS8kohkQXHw&7TdvVGtTikl`OW369uzs;4i#l4e<)leC z?i?pFvh^k3>?Y^9{yb@y$VSQc=yZ$Y+Q4o+q;AF&sljaC_!9a7%1mCeA8}#j!ZBwc zM}8{D^W$T=$7lvN7ka~LViayoxg?$^n6dCju_%6b7%!WTViS25_m`aU%J!+$nH|KN zTU3~>*om21&H1DtgiCZU2%q?tEHu*=#^<|o`nS<^Q2BwxPh;>iFPMY(j^LyhJ8{J4 z7e2jp!4z{_)bJKscGJe+_m0qcoypM0!_Zyl13qedvv2iBRIIv=p-oStOXCK_?pq{0 zF6QF;;=9-zJ{4mQX>m`p74U24LDP9zwDdj(r`NX36!9E-T9bFL>*MzEceowyML)Ie zLOAS)?z&hm*_276Kg8|E_H3v~;kl2waB?!`6`KhN@OPBl>>qJ~f zQ+QQ%j>k@p5Y_)J$0XIW;%h@+ z4lqwdn>oL5y2Vh=jTpxr1y$HS!XEFUGU>7P8J1mpjw_YMoL)9cln(uYU*lY;x@a(` z1g%5l$pwxXpIacnGlf0|zI^feKQwVx=f5pIsXaMQbXho@!RdFLTFLXDD11nDrhTg1B9GQ_V_9;ttb74F9{S6e&UM2JIqwDB+WhhsA+(gD} z$p^mfi3Gy{?o+-DrQPdL|4vt&UvGo2DU*tEVQ|m?l@6o7#y9Z}iS4wYCizWtID4E)xh8EV?k?~0EJai6zHf2zG zzX_l8r||iWwHOtb!37=9W0%x@9~o$JkJNm=V zek>f1TgGLI-!>{(aAY`P5XM1iJE8qMSM;9I0;=znc-6Qs$2^SXio3q7?dZd4hxK`( zei5S2PvI4}fjpU>LCv`$^KMMGT?8sb*&SmRNq;wunQ`%4WZMQ04=fagb9+D` z^Bn!UDCi>J`Hki-O#9FkM`vylS>Y|Yw^bQVwja)n`~A?vP;#HwM=~ZlT{O89%#JIp z8FX|2|Mb>m(v`t9%C}6Vs0JTr8DOc)8nG_Skd{{SF<@IU&fZMs*u_dT8`_bD zI(A$-JB$Bq&m?MYV^+-n9=f>=nxHuO}+*DKu%M@yN0Lha{%GypQSk z&!VT9^u{+C(lDVPt~~eUqHTK2doU2&`inSLNRdUA`ZAR8;M*z@3x6-W9wSdN*E(B{5BCePfAO+PDi(riQRYBa*jn??j%1I@4x1XMt}DoWhgDxU_nV z-J=1Mx^&LV`Hro*>1@$yHngvP5V~Kpcy{|^OzLnHJ9!G{0$!s*`slVRH_ClSvVX&! z8DYN$4=pOtEl8Pa4_Apr`2+a!)J#R{{QwTWb4Fyu1~D(Q3MVu#iFnBhUVqV!J;IgQ zpwXN*f7LMeZXlm@PQ)a6PuC{?#O7~JX>xQF&nwNtUbB5znPW>mLmv*?n1{DzlW}~Q zpq9lGJk4GKcjr-@Gq+0kmn_7x`^TJ$Hf_cqojzE&#EHSCW%yV36_smF;o;@e_>fzV z&*v^7XT=z9Uel2mtlb!@84sOK{tS8J1GQY(%G%y zCh7{5_}gJDC%@f{-ysGNpM1rMyEovxh(p))c4xmvMwLn#|B0G@9XQ3Z zEu%fWc`q-GX%`*2#O4QTKNTx%quto@WD6*YN?~~S3Dm5VxYtIXueR3+6MY}PlIM@W zr=t<9J`+<8+*O3i=SnR+gqmfAIDS=n9B-4vA)_k1z#r+wM#H-?h^}>GIpsqPi=VjA zrpTG^YSXwjv^OU;jeyFXX~HhBJElI7{cloSVn7MutTRx2D8}sh;&uGdKZZ=eT zy$xON2yVHZNv%(k-O3LTQ^)^PI39QAwHe1mz@~>NXon1 zqi9_-iaYZpho&O^(M)x0yRs6#buH*$1orD2!)Y(*f zY)OSpMKBld--duAllkL(9ex}?gfBN9Ve!>`tj)DU@aO*gcXkV21{H`sYEg7w+nKS={_a^4sYh;;Hll_gWg_^=4pM#w4157=sqAUSL&1Tdpc=$1cl8 z@q?usw;goi{^^Foe8WTRC{Co(C&>hfb|S{AJ-3yrv3-p*jcS`9err8eyAFcG@nDoI zNzTzClb>c9^4aL`=-RyudK@oxOX7%Q8az{2fuv)j*y4KuUQSbCt>IQt;|K*_>OP7WX9HSe4`0 z*<7A8RsWzdeFn-}lf6cd;1Jyae$TUp`$CzM8$3&_KU@WsO(85bi9&<_K6G&pq^a3> zK9~?qP31)PQeKRwTl>??XCj~F^9l3U z%<`Hi^2ZLN;bm|3Uz^Dp+R6O!Y7%c8nuC#vncP2k5}$1yPuqN3W)*he(GCSTW#`Bj zM$Ki`)SUj8nsI{6D=DQFBGdT0=wzjXQPyhw*x8tx!JEX4Ge$g=8c2E0|#x$$2TdSeG;pT6dqL`;SO= za0|fFfNJb^|A;GVW7zPv6R%a?#?(&P7<&FA9)4Q`-FZH&cyG@|A&YQLbm#jHBcYel zgX(uCu;F7j$@z9;uSc2mS!j$|XDJmIktO{_a$VT&(V zm8iw1rY$(_=4Wg<+YPqL-I1_wBKyu)#pbhDkU4v;VuR#W!!K*oeM%{;%-mV&X)ctd z-r+pB5RG1&kZBghj8$cdz2-Ynd}KRvdb{wyx(Ga**bJGDDtJ8LH^imS|Ihs8m;Qn3 z&b^9^D{u*(p6^BW9CNCf22fpDdX;XQ#jkrK*x#T%gYSFtVXz8jMm<3P5!+xW z*@N1wZ004H(lkuYnJUhViEl)gnswrH#TQh+$%oOR4&3=>6ceYcRTLGbGwGoY|J;kD zXOrcawtXn~e|?VFK3=q(KaSljJXonFb*JI3G&tu#)9_p9GxQCf#qL3F|Mt9|eHxF8 zy_nd0Ae$Kb@yy~PQL)%Cq8Q&CUqI8u3iH1Xp+-Xs_FEr8 zqdmJ&k`>O~WB$ReYXz2fb>xk2x-8eWVR8K_M65e1Uj5vF9ao(&rPhuMrDoTlcLB$9 zvN%HHBF>bJrj43CN1yD?Ar1{^X422B)2ezM)%d0~~H*O8qPY zzPLV-EA!?2Y-uJGdnR*Kw-<7*Q{v}oMlhC|(BXP59$i*~*C#u2kZX4iPEzA5M?t@H zCDJ#~$5vB0b6wP+!ME0&|8@yxm&=Su$w>C!Xvy^OIY3K|+ z-QS4vjo=O!eU^UcAnK(4shykzn{_8}6#KB>H4lAK>HZkenlIW zNUrzVvS>DRzK0p#-oj#>CeO9$#_boB6qEe2c=1LvwrgU}FSFO<;9(7n`u!M9x0-TQ zmjwPYw8ic3BL8bgWyR4`ZgEsy{YEw^6?3aXWa}&k~{l@%(V>x8naj18ig{@y7 zi5p{!u;rfvCO((5%l>X``Pdkp!>jP_@>d*?vy4yVIG%HE3%{OPyz=iYIvlg2_qYZO zn{I*WIfi88d6+Ki%88y!5qZ^~A1<_~NSTAf9vzrDkMw(G%lL9zoZQ%z^Q_yk!yF~L z7JKrgj^x}P%y!r?Vl+N@`0~Tf4dR*1_!zyj7GoQ&G5wP#_im6pNnyFdLSE-bqaiJp zXHjdH5(BT#ksSh7yjzpOrMgYnZ{2C!$~uO-i#ovM{&XZ9Yl~(x99S@WG*{jYH< z@Ov;1v8SeT`i5~#*-?d;^8y)vv^C5=B%$9{GscExIz4j;;g9PUe7dzO?>-sGgYS+CuKI4ulTF4e`o;{VyWuFw^lgC@YWZW)R65OW&y<5h*{7oq#x8M#UScvs z$F;?ZU@tljP~iiqRrOJ?kXh6hnDb1ROAhVAi3yf`(Au9%8_hU3xK6nBRB+ySH>S>t zV8^k?VH_}-%`|#&eAO)^wRWUU)miMGVjwe=1>)yH9nLLj#TozoMYgvco`=4M(b?7< zy2lG|PG|Fp-bk9IjVJtfh|QjnLch~{+?{O((XS1cv{)s6^_iw?$77Pnlj5(nOVo2Bxm8uXB&b!A7kPA(TlmR#%!53 znRANI;c<0;u2?6tELM49@x|+6{YV9)YWnk&??4poipR7)L-;k@T9LS+0NV#X6c&#R zcz;098 zML1jE>BO8dS?nqK>p#DZ#kz<(G2Z4AMu&S-!~GBNAsBsaoKXA5jvp1_y!|1RF(rK^ zpKQXH6N?%nFYG-N(XMP5$W;!F0`Td_43k+-KC`$bVHR-!C=H zuWy_x*O_9>twU(+Xw8cqWZvm=K1MiP2UmL0TQb=GZ<@36$X-z~$OZ>H>9d(q4T>BV z;>Wk8C~DuGjoOlDRP)DIPgk~UdRaUfw;c+%P7JK*$@im2vY=tRm~r(pu7p|8sf`6P zbbIlUlLK3RFN4|le%veh{?A!@>?gT?$D@Zth4xJBObO*RO**JNjlSQOwfPV83bneVA0p9T&dTEnLXp>-sFbR{>Py*+@Ho92eQ)oHhg|+GcUF^ zFJ7Gut<$r`x-GBJuFR8tyT3pm*&ktw)fl^xYxo{Ae#Wv?7MA zPvznddMvH~hL8#23~F@^13Y9`M5#5;t^0)i)zYg_pUSkTFLGYD=K7+?$bXuOtGSan zOx>D4Bs1VK=`5DHRbZAVK6BggZjyv;P`kS4h;|Bh40;I zHpv1f?DYBQ>@4h#?8P4EmqB^RBiwA-3Vv?;vEW)XO%pBXSsF?EdwOiur8!$)ux4Jq z9o#D?(C~Qz?;Cta-KH^AH)zRDd%~#xOnSK?i($0VgCE~zqs5baF=Og+sLJ!~{AT6*5$~lVE#7F4h62(W?sraCI3{~f*(x$r#?##E~s>o?jQf;EwM4Egr(3S@ptU}w)R9NEW= z4-_(k^X4EHJfFf}N1O42%u@JH?98>#&SR-k6jL%hI6f|rx~H4cYg9O&s0?7kLYb*N zk;JhEW;9A2$KovT;Q0{P%&E=OJ%%=0bH%S+GjQj&8>@yovacg6*A*( zbx7(Y&v!t_q8RhH{=>8%BXP3Wf|o{BLbr`6x8#TOsB?e5`R@b1TK8oAxlzb&*$mM? z;o`tq^5&1#eta@H51s{mc*?p~%noV55%0rr+;vu@MWrF)q8p1;LYRLaI{!=Kh=3?QKd8*U zxxHv%(SfBsDo`osri-gu!DabD@oQ!Tt<|nVLoLiH?%pPNsyFAeJq>8>Ta6YUI%DR$ z9*n&0!qoMJcva`cMwhC1+#c>QR7hD!CyC%UU zN}iLp_{ljv9pOFmpy(j`NZt)*y;ziZIpt_5bklCwotG?we_gPqh+udFDCy3DX87HJ9XsJ`5>cOh2Z{3$LMEyl2K zCe-=UiTB?pbIYPPibjnXZjvl|{pzmV6xEBpoSa0ci8hW(zoSUYM`j0A`RmkaG?$;- z4h^K%jZLke@aOaLY#* zv_2!V4Z3|ee{BRSEwc}>9n{HgCw{5UT-xlcgXC@ z-~?8ue8%Kr*;n=;jJFqz;x~=XjPUvlGZSateH6}KRSsO%eG;49PiE(ohr%~3jLo<0 z72^0DC@<@e)!$o6J);V%I@vKPEf!Hzq@Ulx9ZiS!;7B_Y-ibC8;|!go_Su`mj-_z& z@<`0-<%0?PJF|c2c;49b0>9pkX3p<;DDRLAV%-4jcT467b!|9#WXKLXspVhl$XeH? zyqJ*0-S5m0sdWpHJ!-_46Bi{PVvN&KiQE}3-`6#*aDVt@4v)QvOATR+Z_|;wvYTeN z;uH+)vf%S^sgt>SsW7_V7K_>>v%XS8_E>eqr#79DJa{FxpO56Dz~|!afMjm-+a+ve zcgee;aolG)QnDGbGz?k@E7J)qs91o~+`D4Yb5jNxmgD&R+tAGF#IWG)2s0tQ?7~Gi zlfjJs>dzFJ`|S5JAK%{h;Px-4&_4b>;%Dfgq=yQVep~X_+dN>MCI_Eei1s%_@g~og z9p%hrxZsW8{BYWAdWPd(S-8_*lZ9idVf8T@-wu1jT&D%blp8QcuI)@8w_}LSvhVsJMFaddpCFw z9MAbxj*PqDNIU)ET-L{$cAJd3x|7tpK8}Uj&NX=9(g3Zw12|@JJewBC?4(sI#Fr&- zi*66DjvLRJUn4}zZ)?$G$|$4z=5c4CQfI+77bqB(t|YSr5!rdz1fsLdCuHwoQ_;|cfLwbqu zHIZ4S24eUNGTc3qYJN|!^!X-a44jUD&#Fx9YDWBNz|0K;aIm*6Kla><*dje9y7Z%d zkTH|)JXF+s7h#Oe2274?$7SW?gs+ApU&>tih1J$5uF1gZKN?I9vgL-{A1Dj^BM$DK z%v_$l$8eZRGip9C)fDOG=`D$`|-p#y&cLSui{%!b z(P?A+Yq9A$akY(30^ji-Jp7W}RgN>=}{M)Hj@vS)D3LLcUj-iy8eY4g(6 z>3I3`EJCG+(ROPuM&&t5|2TqaYwqLOKDoE+8u4`YOS~&CLVcT&96v?oxnzH3?7e|> zKIty?T{G^t>cs zTiY->e;;xx#<4{417okqJiqs4bS-e@fbd?>w(9`D4Izs6yM~B%dpt41Xgu9&sj%8s zieC@T3t{NY`v)6^nVJTDV}|oWS)O>*wYyv^yYN)qI9>`}jF?q(@b`-zo}P4rLTWF= zKjn(^MzX`s^tqTi#0z?_eVClvMe2)cY@_u|xCDJe?vOwDWIKx1y@IIUd^FE#WpKm& z7w8!=f@lB6v3LL09Gzp$mIVPE^`tFhYiFYVYcqOWTCPayo{$8@B z`D3N8o4_hv?8lKz{M`r9bO_mmisTHs z=WG=x%tm9hA9=n+_S}?CgxUBhSQXes9QyhIjbmlcgd!9BlPZuKb{G18{6(uy-^GG= ze)KDm{`%D;_-&F%)%`>0rn(QBb4oGSqZTpxF&JfX6jnZx&wTF0sF8I}ZzSJTZ!wnM zhF$3}HHa&ZU&H3Z4M6*Q;z!r6_)o`>+mbu5@5Q;|)Wp^Jx8D{EPyQ8lF&R85{fR=! zua~9_Q(Tw1h=!|Gu$Ej}?Xo<~8vYWeWmZi|*GgsuyqNmpIQ$xAN61EFHv80>eb$Y_ zoQqX>Qku!btpmJg}=2lMNc-_ADMn=j>UwJ{PO~0wH8R=HloG)~sw#>mEBr@cq>|aV(U-{%CXi zRAnxk7%w%l?dauSg(i~=@o3OYyx%I%j%hi_v4~7wU6V|{r?vNrY>X9g&6vWXbTM_y`oB3ZhpWz;gS5Nx)ZG=RZZnall0m$r>PNi`>0*zbFUO8uCH5F$CY7HbR99zI4aSkRgm#RxrfK3%MSVkepS~p1x+Ml+`a@B zy-U!|YA`dBmSO+j2DE(XLzOLo*kVwMz!uM-c4H4#s<+~>Z%>@guJq>k73a{h_aKU| zd5Sq4Bxn1<9IWa{Ey?DqjbDMDQNdIk>?}UbJAml-!!RNn#^t{#8i<>9Tk`o-IcJ|(m>o%Tqiu4%84$&@LbHG6a$o%rM;1RsF<_!)R^kB^2HTX9x zn}&~@(CNWRq2Vk$6dHBeu2rm9-=ZfzZa)B@$1kDPd7Jpv&4Sg~h7C1|d^{;h?g^7= zxz~p)G`|X^HFn(i<{@%l1XAx&pk#dvsj0sfYfVSctAh${W_M)Ad#A;h3KjZpTZfPp zzU=fb3!lTR*fBur$juz#mzT((2$`F|uT7<8ZRvetGH>hlWTn2$x|NgDwMNp)xid}# zo6u%^TNW0}UKcMjbZy?8&&Nq0<*C#X)Q*a0->rGZ>ywz490C`qx1Brfj5{vpoQzs_ zWkXRN?sUB^v!q0nsKVjIe%yVq7cPFe2}jk(&>nadGu9<>@U#q8D_?Lj`!$*NkpsE6 zaWFKZIxzW(0&hmje!TF$cp`P4y+;RO=gEV3+NU`e$ehk2gAC>lkHkXhtNKTk;f$*% zGt?w^_;0A}q8!BQ#+n$fDRX)~vaqImGFNLq#dtvC+DnbfU7BpSxE zu(}A7f*tW#KaDkg=VI8+2H9Dyjn7r<;njmI`BRM-iyvTUa1)l{i5MK2%()s<>F}o! z?HU?Ty!46PF&{ z4n_HzD`HjXN#Xf?4^EAr%B*@jar?VC!&UGqey+h@!=Xv>p( zLl|)HIusRFJlrXQz4j*&*(yA6RrWq?)L`J4b6EA+m(v|$sp9kw+iUK@WZ(!MJJXp) zfgUvL62LpRro-l&>@Zuk14*;26*oVRX3#oE&TKlAKUU2G_KjxMhraOb8OUM1L%CHi zl9A6=>t)F8vs%ON**NyKZ$j7HO>lYh9EI&7s4u%PW=Y-a)}sh6 z*!LWder6!&-2|3Y7K?rMZ(ylzD7uH5@OGI7d_GTP>v!u=wLcJ@?&vaTvoEu+?m{=| zeGhbN%>!?g88M+Jb>p0PDJ+|2p5{1rD2v8Nyu?n~Z{O}!7pk3|%xx`7U=`slE>E&# z%}4{D&1uQ1nxVY5I2YkVeP~%Je_!oo4E@uRsrv2s*-@Wi8>UkEnJ0VwaG^z5q|D-u zrO48yd+}ucd*saW%?4t#JQH~-569iV?RfTgSN_mRW-I4(mdFm%9m?G#6J|@b-kUJ? zpBtaglbJG&4T^IagZbk`FZ^?wfxb_!AflCmkK}hE>>GI(Gq!26EVlq!b}MncOBw_NY$Xg&U=-VAkZ72=A#0- z95cs>tP|tuEYDMEiyavHvL%aTkK@>bdtq=aRAH~-hK)-#rdYa(Dad~%+{1c z1YU;Ot1WOosGwO#Eo@onj=Z&Z@cE@SOAlu7{NP16Xpz8~vWqv>Mwxd{=<)QX>%y#P zKbqf^OxTw!Y}Osan(l+qVa8@`JsZi_A?a*dKAy{d$oVHVhT&D+`1Ni+UcPU|LW618 zx!r~1TK7bUdByVkBZB$!dW6gX>QeE$QcOFu660#qdHVQx*lNO)vb)4d_N^>0ROU;Yw$$1%g*J(w z#nInc4A?t}>4$G%xwj_O?%8qh6?fh-c#262^f>-wbN+H#Cwo^$^V_4%PHiGOaPt)d zQTJ;n))!=Ri0lxnTm2W^;{`tT&=YIDThn;#S}b0ug{Ph|bj28k=&5sDvx&6uS&l+| zYrfxA0_5+;KqN5TA(gwfwBqj_Kg2e~03jggoq!oxNu>r!x4<5cz!`SK+(#Bf{D} z5mp5@oSKp%+Lp*=&3FqEI%i{WMOPN~v|{Fw6n6S+%5jDXto`Cbht{8Q;d3YxUv+2c z?jU|uzc1>`l^A|UgAb)1^!3L}@$1GtVbdd;Drx7CmZUB_xYpy;Ie*TzAIfm){XdQF z!YvzZIAl?*%o~MpZ-E`|^t9wH(|-KkV}bNHZP0ye6-LCH@?pOd7_QtKJ^#C?$eP<) zJenfAWal^Gy_vFe-fa)=7{9{wl3CcCkR>eFWC^wM9WX0MX3maerr$GCq)qdrxyl?d zy+Zai%TDYmsjn59Yuz|Cp$Z9g@f>C?yJU-0xJ7>=_g?FZxb10lc;SYF4`p}a#|iAE zsfp=o!?>w=K8lAv7uNqQx#>&<>t2U5Ep!*gs7BzCeJqDNY{i7)<}|VN=1iA6;?#pW znJHa~z|_;2Z+!$easoK~M0f7Ekjdl2&4qg97SX~k9~IM@aj3myRKq>6wV+yDue*nb z-`|U86aP~zZlBFhb%Hfd*W-zv0~-=b@FYVMkH5ANv(xhM%S5tQYIE_@@+B18lBrgG z3%%C`GvGfR#L8#(*kfZBPWmGL*|$g4@B*B5-6+D}C9-g4TPE+4>sWmZFDsXeuXjgt zspQ>HE}bgRRLKbbKAGxrk7+fqN;I2#6)%6C7Y*`RQfV8`%ST3_yrKnE9M0nCN_)mU za}k|Nzac62EaL9%Lh=77IuE~``!9^QHAyAy(x#F&b$`xPN>&=0q@kh|?Wtw&k)0$X zGZL90du9~bo0L7`vDfeX{s6Dn<8j~L`+U#2uJ?O#3|hJ#q*>kPQxD533ik0t{q>n* zHb0-z|4oMWEDg-a*g|=($*8H=BKmE$!@}eVoVSdjK`A|G-2yG>eIAY9qfN1JK^qUdp@jh3~|5{`{3)nNOuTH>hEuJEkU= z;FH9XxjX_+b<9#rdnQimN8@{1CAnJIQm9K`6r8D{mSeq1Lxvf2my2*?x05imVivBQ zIv%gHLiF9Y^d)*9RmC{tVD&57V1Jx07wVGofD?2oXt}s`N)|(#b+CA%6|OUT$5}xY zcV?JDj9*9&SAuZ=D?5N9-SMk!jbuD$z|0@n)3UAI5NmXcKFVuhd&C90oH!6Z9b=Gc zl!vn^)5WbC4V*4?!T?Vv3X+rtLNSreAL3+$U%<=sp9Cmex zG@iHgTl!%h^IGl~l)$~zhb}Wi+Gbgb@T*Cq0O@A({Lb&8!j@j!w^Qq;?)XnW2J7~F zira?w#d$poXeOA@AyAt5JH?lhbW&Rzz>J#N;;E4-=IkuR*4~=%JUJGjDaObj z&?;>=%~$!p;DB3AFGZ{3ciL3x&OV8MBp-2>R-G%Ozn^+h$aCguY6IT=vN7qL z2N@O|rDyVkG1zZ16yS?)vJudE$63o$(U|7lL3cJBqw&)>lg{0hEv?5I7;^5zp z_ZQ!b$non}$;&D=49s`Lr!q4bZj8tAVP_?7|JBf_e+3xzPz@iqcwmv)7fPG4ggW>3 zC&OFa;m>`8@j0CJ;C)>N`wAj1uO)rnd9~l@ikT7%)Rc}y+K1sV%#olexd-_!mcg^K zt8{U+3Tp3o(c}BFI67jzXuPys#O(LNsjvR%-8`MfKWm`HBU&UOoNo!yTTZX|cjrsm zVE1I+OV)L$5tDFG+f9+=R98|Y@6BbKR#2B&p*S(T7&d((gk}FFWV?p*rX{2Bn?H14 zGI!ASLM5_mlfvt6Q)t;91L4rNn1)|GM60X=u^ojk9`9hOY!!8rDW#HZq%b8 zQQV3BN>!ZOGaspm>uctV%2n1FlbVHp30KMNpQQ+?c;FE~D-XA$1MuK;Pcd{?5*|O9 zP2~zP_*VLuu6(>nHf_7;y!t?6H>u!AcST{|pSch_z3}Fb1Oa}lg_-jm z(LXsy%v<(RY~=fy$_aD!JQ>6KT0Rb$8DN0YQ^~jumr2;XrC#0F(E4fmsEa8PdZuwO zxykQ=DxtW=@B3W`2I5j4=VCU+LCbSCseD{bhx@mQnX49wo6Lzb;N6?r;2Y$SZh<#_ ztuf@4HZ&9aVaKW&q~axq7m*53Y+XWuowm3#q=qhk)yBQyf@Gd#B2luR{ypj>DQ^XQ z)V6@%ehd7y%Mxw-4Kd8SQ*w+MIiE|rL+)QU9NspEq?7puI6V=oKE|O?{l27HX%3B& z3PD4MA?$J*=v=XkWPe!}nqTqRq6erSQC13n4GK`t=NOQ0-MEo^oj0o?&ouz8k2 z-~7hFr!1bF@3qs`EmiDnbHo^{Ad!~xnRcH{LU$Vv&Q~ZPpV=8cv*dA^yAOTly{4GI zp`_45um{E%o+q3z^V!mxzCG~WT(Q$$iCxB$%5i4i-gbb)(qj}Hl#90+OGxRtKHBvQq3rg9 ziuMGMJMRbYjxoj*8!0>sxl99?fxL2vBJ4B`De1lk_RR(IE(If|)d+5Xlu0f>6Kxrv zm@9LN0xpci`r{f{%D<_97q^jB{$6HOJK(U!H?m)=hKee7!c3S+*HXGsLnE`shKG^S z^!)Bl-pv2r~Sie<@99enm2j;_+CaEj3QzGFd8T`!S2zTcuMspwOQ7(Pu9~j&TQ>?&>u}}dC$)rWrO7JBz4ORQgv}? z{hJDzWM|qkVY9H5-zq*R?4rYoQ;2GuVLayrnZ}HuvSm2q;S5M_>XrB&6YFi!QU%1;RAc+a<+>}^@{NAJpezq9U{|14fOB0 z8Pyi^yr%V&-X{`0PP|K#zuKbbP%GR@^`bVDH0Db^6?4yWPJOE$DM&x3;Gc5%>7tCA zrm66DOcS01$Kq_q2rQhD?QtgA2|jN6$WARm@mEu5r#Fz<_5p}X*i2-rPj8|sh3ctZ zLe41!S+lCh^-(MZy&eJQhauIT6PINH-1SU znSHcKvj{E)UfA9o$?t*O56R4j=Bf8&9>87VGfDIyC>ztrOLTBf)Y`F%A42(WU-)m_}X3|c=RooN{J*ves`R9@`xBgJYj#hE^t`4#$wvujB z7;2q@pvm{oS5KeOodsiDl@%T{uU9%;n=-8A5Nqo+?R~w`Fs)C|Hm`nVpSYp zUWkSX?&we~q`&+5UAAsNJso32O+829jhPghI3v`X^JBQVea&!^y_s zSXLiJD}))!o83|1uZ{=oD!6xveP*8A9bu-N^7=;-Sk7%sPf-uZ_M{gT6Ce6=>sE&D_I`?IL=UluB)Wa-08-nDqz zz&*o89DSq&rGW)dN@^ijo*~aZE5f)#I+*Oo-SKX|6m8rEXQot8{UdK`e0Yttuj~+a zb57I9#x=A-Q3d+3%GBc}^Gviz+Bca1qT}4+ms$SJ5w^VxDF-#p)Xd<5#Jw( zdfr7_Y=}XO!edHp?uF?*ms^k)28@`9l6_hD>0Tx-PT3$_RIbo=o;6>csiN@VEo9Qv zN$c5taXu;+-2Yyh*`pHL-is2Fc)VnsEM4P=aYBHDphr6J^=B&{0W~Gkd~Y zRuc_V>!@n>DQfIkL-kde=c4SDz_i?zA^5GZ_cE-wd7 zTi9dONi^Q>Hktgj!|I#1G*e+ZwK}uU@9JZESH_IykN9Wzn z`rXpFKI)XP3^tHxEJ~t3FPI6n?VltuDi(L{Sz-xij3@T;mS{WY(D3U+;V@Q~-$ABO z=e{C@Jk3Ummk!ESHcBdfTR>s0HGbMM8zRyiML#>)&GU|4smZ`@uOn>YCSd{R^Bn?9J|_lzi4SVC2Ym;wA=8WV;sq=>o%@pT91xc=xN z?oI~&^yvdrt*#hK)2J)Io0mHA{!HZ+$=sbo!~2~USufORgm^{gTyp8tiZDuS>yW%U zXGMRf@_XL0F;r+VhAvL%kL1-3So_k0TGjeslwWV|X-XmRIxu@2b538`(auY2=zZ8s zkG(#!BJbu;@*ZUf^*4by8*qf3Fv*;8SWD68i)j8keQa3SgB*8j!Y%ELczXIJ?cceX z+++ff@%#gwigSV9(h11>5QL}gp~y(-hQ5x*B4S!7Y$tWWd(UyuIs1?H{qjVx-f;N2 z-V+9#p`I(Xlr*A#2*({#w9z~P502lb+eWuV#Nw0U+kOe%p5-p;H3p;7v5>iFDR}UJ zI|$vrk;T!0m^-JZkYT>$#g#TFnq-e%)@D#-*8k>zU1(~FKSutlq`60z&@!hE3Xhvh zrCqPmx~Hp!g`P4BCS9g$y^7IL?h8%-Yow*Ph5GfhM$BPZROG%8=eO>WWS`(W3n_{T z1GDI)Bo8gCH`3+R<8kcuNwVznndCP0$EisJVW!qdw*R8h%GsC7MFViPYm|6>&;Xm4 zRY-of=wVZS99>^uh{FyApH9Hqw;=H-NPI`=4g-|cYW6Lo#irTpI{99s~Qj872COQkc|mD4fxKI zgL4ahlNIwk?22ady*VHK+j~L9p*LvKc(Hq<4}Cbbk|fM8i%8f)<{OL2uR|02UYOy- z7j?)q2TGEO^T4~l)1MVPNb)fnfr>E{Gb0cB4SR%rejaif&5(aM83!U$@YSvuV@w8M zX@(A!+7+PjK`o`MGez7W4dz(rLw8D*c)VjS?YNMJ!-EA5yh_I6zJuu4c5|2no~MwK zDO6^XjZ+b|61Sp>NI3J1)VBqo>0UIhT$jh8#pU$S#U347i?EMB*W)CU>BFpE^qB8@ z>aTdGXSj)KxT_I(u8R7qYvIVa$+)p{jpSacHyPecVwdbX%D-4is+xmQH%tvTk450_ zDLV@7T$0KUK8jk){gf}~? z($xlGjBP2+UVehU?)pLRJLF)OSucicabW%tXMpG3qbt89sJs-9n{VR9^GidR|64~r z?aRdbHg&%7ts--cL=@_5pm(~pw6Bk)D7~mpGlvhQ$R}GRJHJw@EgVdQHoQ5x8&$6r<7EU=rt7^Z| zngQ163t$scB#MA(=b71vs=~faI#30nRgzK=XSVS+0{ovvDrg`fA{+e<%Gt zY=NoooDhBJKU#Q*e=~U=G4DMT|9-k*4DU9^Z;8g?^#c$;A{{6FL$US8Wm?sjISelT z_^6tQeIpW($+usfsMB=Wif>ZV>ev>?|7Ko`gvZRDWIWc{!%JEfuIx1UGxiA`?>z** zt()jVR!>ZrwbsL$-BRn7#>47MPm=OeK-xFGWG{~HD6PkHEQ&ZdgYR_gTY#%#lH)a_eO1TIYzYbM8Gl@HHpZu2Eqme|tN8{v37 zue+#c%OG_0@uj;jRS~@` z1scVDD6#K7iG{)ha-71R;w7o##=v~6ZZd*l$S?A0`9wVjt)-d=6Un!~3hh71o&Cf= zqWs(%seAWuNo?WnBpIvd!N5jdcLC8)_hN|=vs^Y!P;Sc7B8)A;Qdz`e>vrzLb6vH`l?v!kf ztdVELf19q*-_L^mNM9tja?@#{yDwgDW@ipZ1z!HAhlBuT;v1i(te4t27pH>X%#yjh zCK5ISbD-X6Pxt#=_h@Q7Kn|}15Y#0e&zlWVu`CNL#f9q^vJuf?VPW8k{E zg=}tI;+$MJ*u6hQdJcPdr|^~e8s&UTMuh{<*`1f@N00$_UUsC`ltIFZncyBuYS_gfHh3ug5h z4-41vIH$INUJjUuaRoEPT+QzIvHUO{lpYI{BT=-z>lfN8m56Ho&9a|sj&(gpQ=bec zoO?YK3%8b{dgcRC$!I6NdBhIkMzV3yC$n=`NY(R+2u$b{MeXdc3%gBEOjnAnhT7s9 zQpkZh>q)*r;>{iIAKn^-6F#e{&vs^NxE~@9tx*_JKLTo1OK1b%#M2iRqm*~aGnm^P zsp<*WxenC3pjr%697BDRbnxQZBl=2K2>rH}b7tS^`sJ}G{N#aEI$NmSpo3bJ6tQO3 zZWOE@4r}>ILE=TeK1xnIY}97RWZ0uk6J^F+2`viyykxs z6C38z#DpcxDI7xwpYv|%nNsC7I)?2%Mh<3A-7Gjrm1W>aR=U8#CP-n)i{(s8(9mwO&k% zh!W*HGBNn8G?|_LOP9Xvpp|;FX=e}aszxwdCnp5?XRKhp!x?=uhG6K78oKvF6^5L5{5r5|xJIbmV>t;>X^h9V3TBBb+&n z6M%)YLi+Cv?S<+EN2ss;SlaleJES+WOGwTaS3@GO-$;U(aomkaaKjO|`yNUe@-(t^ zyNJnm!S}Egq_IX9Wsa#BG9eWq2i@>;Yyx6OpB%8Nx;w^KxstK80_3Nc(XwE75(oJ2 zIg|G}b!TXost-QJ4aMV+5{ww*0$ZC3S}=_rS3Ta+n*E+=-=+lL>QGo6)1iEod9-Zl z3_34k0f%mTB$1sxST^0s43;$f=gVy8tQ*u49!TpQyCbt_1RQch@aS|PW=HoI9h>Da z{b-=%nqC1^3wZC|dj!vOU2)dLlnglIDLZf|<}*)mwuK^EcV}YL!Sy1cPziks1x%KC zliANU`m>R_aym|s8dgbO?W5TZ@QI8Z*HQpG`PRraQf6-%{CvKFLLX~lUf67LFsv6; z)3=aWs}`z{<->tHbVvFBgG>D}nw`m}p9VRd zA*Wv<$gRFFwtwI|N7XZ$Yh{n5HVJ12bf~yvJ}p1Qy_Z*B)YNf+o-gW+*hixh<}*>2j~Z#1$4R4B?(4$@B5lp8KUxpzdfOCRd!`iB(K|KpzPC=|2jHS;;|qmq(w zt9UWJD@;K;yTktOmBXY)9rO(F$Iqj%zNeD+9BxS49cG@q0`$k=xEJr z$-6Em*~xjAUM3HK8fTlTxrg8R^C3k(kQE7!bE)on2)(*cN2Bd5@sasR^8SYOVTr8R z)R2vsGv0JGD~vME=_4SeA6@*Wf+^Z^q&VCLPDc;Wu~6pbh6Ui$bp>R8E})Qg#n^54 zgPdd7!E|&aI*Q`R;bbXlwWHCp;|O_qoDfehrHKjS@}QJ*f;#?;f}fl%WXD9HZ+NkI zeyUk)Pb}a$^qv^g$^3#%GSIlXh3fs-rQh;=9NCWRHos`m=brZycaZS;uFC*fG*K=GxQX}nJ~ebcbRE6863~z|sL{>F0s{G-LZPoOzf6 z@3m5x$un9dcZhy-9)4Z4KV6PLMj8(s!z0TWCC9tYDYwQ#t(jvY3KO9?~$? zT3Q=6iw@m5Br2IezZFKb1b4l3I7}4A?kM3PB#Lq}?QqN;w-6V4; zKD5QFFD8(_>VfZO{&XZ#nJzDRK$gRu5EB}J>fUc??+<63Fn>*(dznLreV7+k`oeaf zH9A9+DR`?j-c@pL{Jyn#)@>XXy?;eMWtuSXFf3TlwJf(($0dV zh9?Z_L(p8OPbuA;;C!qA*vT)+M8F%+!#K z!Vtw=sQl7`=AR`rvcMh|+IQ$zU@zqRWW#iKFcMs3&`(j5{R&aE4X?OB)+_!o{51ZeT z>kU7;^)duGuJ$;iDUVjAZWudf2=XV|(tFo3vdf<-)*m=aXNE>m_M@NTiq1`8ZspI8 zwD07zYa?BF$~>b{eW^!e1vzcE#y@63eE)AS-xd0Z7P~}z^>3&37aFM6elFGCpFuC3 z{Lr;FO8mY%`p76)uJwd}V7&-T-$r*rjgVcyJI7z|>E|9DI=^}W?bUB4 zm9K|MZLvB2Y;B`ggE@o0qKPusKBdpq$Hhu^U%nY&%x?7g6tXv#EHBB^;HJ^Y=X`0M zXN{;f7>9nwx{zDvi0^6^h(9+7&vq=51Zl3PrgM5?$Kf1QXH}BaH!~>bBvE63<{P(n z(xtL#ykk=5oA5`G_x=z>=nWwdYHT^fAA3$OF8(Gj=4*cHzC z(C}b1{B=XL+z!fDxE3#1`>mJUdD{&GDrbw*3*~g{ zNhutZ`r*>@ouatOmP&iPrl2u~DAC|7sb@YJhh;M7{Wj%!^};Wg)wERmB-vHAP+!h+ zymK{^WBcK-{adm5m{7%UHl%L*|$?zXADPEiac`# zo5-S_^JOxn=+zKIaaZIam6=a7hCGv`?Cy$9-77^(M>;d@gUR@(3SM{Zik@*WHT%-#nn)6GSkj>M znH*C*F=j>)!uGzW`Tq>W+J~9AbxjRhr+CAN^BS_jJ1O#HC96Y+iNwZ!8B(?2{XuiNLQGB_r>?YoV7S9u>yFXN)30e!i#pWiZlY-? zmkX`O2T3n=6zx`!r}Al~(9ln$b9G&?_vbv?T78-ZeKw+h+xS`NNLZ>glxBFCQ)hoE zO6pTaolE-T_7Qn{d2)5Ts7EE>YQVk`|v&) zbeN&@R41*L$sU^tK$0#KjV+|EbTN6Xk05cf6s>69$7M&AB1Uv;RyT_1D~p3m|y9Q&%8(4 ze@hvv*NySSE)YF)%Ej}yzRYxtLr`H7db)H_M<{o4SCvw@q=lUFBP2_QY$0uPSF?)HxwiuT+q zjI6z3^InrO-sLj)&5DY>){2Ks37EJ0Deb-zM4}WhyV`|P??|DXx!w=fB?z@Q%%B@S zmCia2hFyIo4L2Es=+@I>W`+cNZ%!vJT8L*UHT0&TFE)xyEDLNBj@B)bb#F$}rM?y9 zYg`cmbbb7Uw$EEjj!k?!bl*ik9^McywC1xL`MxkHX13*uSX6!M zgB7L6Xv7%<(PfV(y`QfLp9vax%>3!;>#x$sqo!my={%*@+JnyC6bb2Rc*dNukDM2* zXRo_@!d`MnSVxPQ7cs=RkK}D%!e;r0G_K|{m1Xg);ckHqin8cFF98E}Pm6!6`jOwr z2}s?bfdD5(thSCreDe~TXJLgHuRj#E=q%}uEEL~f_CP6_(9St8>C9v;jQEe8!OJUX zqvRF!9LOx*HIBHH%05b-4-B8kVCUp`NuLL5qOTg~G}b+)MO*feHZy)+KN^jI1M#%# zKi)aWCBn1uz3}gPn!@V~sWmDN`9(d+`_}@B&?kJEmX8Bz-)O#dDV&41(dI|4n0Wj! z_i8(6XyR!6Irq?GaAJVua@i^|W`Q^ET3e#<+i+Y?VWb6j*0@>U(jSM8{n zpK0fI7NDW_|IFSaG$!E*8TE;WwEO{TQwhhl_Dq_%u1@H8$kTRqxOE1U3$2!NIyd43 zRWx$$amsg^UYCwI_HMkIaEv}4D!^Z6km2EUx>D31v#Qe(bEp>rDy~WD0`Ji2)e?Lj z`k!c=lM8pBWcXETp!0qi<>>QF^Kg~uyV^p;>kp&(GyIXVqKw+ETVv|`aj4I@L<4TH z8*exBm%121r^*tqJasT`Ul1zvxx2PF6kAUJBS%+vjMn$Ypq&b2cKfne^Q(#Wb^N1K zOCsTw)<>L^*unMp2@kn1Mkzr$J(OxKJvc9SMbw>>6TW;W4|~Jk zolF;ice6VlrBOI@Q7)qYn=;bax% zOV}%QnLo2Oo9QdN<@LJ{MIp`!x8GmHlgs^Sj&HL>e(?w9HMvo>ygg3p4xl8{o*3lh zM_t*OXY}D7z4#c5;UyDrJu(t6wm5?ROQ9?6zSzBEFcwy1!?7-$;>zTa`?F14%2C6( zo6%Ugl=}>(p}03)mQI}Lh3C2KQ#Xl1@%*_Yb>$@ux$6o`&Z-~(HBNG9Pa=+n&!JOG zO_1ldh)#ZGM$+HE6ccHR@wI&sXEqx9T4vI(^uf6JTo;Ra_M+vIThw(jclVR65w-az zIm*Z31alugYQ3eXT|@Evvma(nc~0(z-SKKZ_v$L%iNX3)$@oAZWFPzEP9pagKUGu3 z@U6mWlM2qYrx=tZL~~J97>Te|q>GUPAg|M(jh4hI-W}%HZ2u_|joG zQNxY^g)A()rvo+XO4`5esKh{8A7Oej_%-dXDDL)KOnuyspJfBkKR`{W{tU$=TSKfF zf0s7=Q=@wCZ{#|m2d0Eq(U6r%I3u&j!)U51Ml4N++K@=h^3B7XysgycrARLv+9=?` zc&M(s!5vZyx)MDayJV%wG1nN1-%n9}k3MLee_L#@d`oRp^2KQL>2z<)3R*BrpDwO{ zDQ*s#Ayi#BGqg_+x!Nrh&FBJ;Da==z>;d%=Jm)SbqQT71nX=4Gq&o3jtTPHxrTb~Y z=dY3z$;`UFr^cSU$29*e`)lS+qdK|I6ga5_O(%beUNH&uQb&Wfn+KyWzb|Nf><@)- z=Kov)U}1-6cBYb7qwTQ5p^RpYPp3j>P3S(=#Ntb_=zEWFeg%6qt=V6BOcr*QgYh@l z8k?izkp9ja8<$KG5!{Q@KFRy}jRm;h>Vw0kcPVWS&!icubli9t`7xJd|9v_1l`^D# z!3uctD+)`e{Gnl452?;|B^^xgr{CPe-E;AQ=o6xd)1r#Z^tI_=R~6pN&8FW2KTxmE z3$%FE8#1bJ#P&W+T)hO$Fj2ywkv(aLPY_vXpQd3s$7r%hlWeK`Mar}G)3}jckUTB~ z7qo(~h%>KK*M;I>zj>sh97=~{`k_3XZ?|)LLbkqBNKy(>J};PdSh*sh;4X!yE+;i9 z;PenZj2_tF(P!jW3LlmOt(l>8q{@$Xq#tPG>uJ<*c?%gOWgz0UEEa9f#OoK2#pF^g zsMX0}WymLH$LrB{#r|~WYCdG&cflC_Zrt(xLgABM(z$sz=#CFRM`YONd07Tesywlb zca^K>7Q>x$C~~WNa|YU)cYvG3f2#v9_~KFOpE-d>xU-k+cROuVt&uFtP^5ruW9dO< zEzNvwK(#-a|2fkL-A1v?)`P^^AxlJ&lN8$2+0W<^h}Z=;X-kDUW^2fx;Z`s+lr~77 zaaO->^d=FWYDDKxb9b@O1+I6tP!E3g7jbu_{9r7cdor^irHK8}^^%F_Ysl%NGXD5? zgQ{^9!gVU>!{$`OdAr4u{iM(*_Drn!vF=yvAZ@V)F08x+L&mkfxpjVap}thWLa@W z{=FsL?zfhCaVBWDNP^dce0X`Bp~s4fBo(2L{ff1;a{EBYXT-xViT9^k%Sm}rDc(HX zDM@1HJFZIJ%#5MpLT41#0=?Bj@VwU| z%KR-$Gww!s7_&hI=F8xGd@Rsi2tc9@ua*1!W#rsM%T#gC9!c zOSvBg8>jR0k2wbH3mp=`YnVyZ;>!vd;o(>zvM0z=<6{@`c(MgiBXp8exsyZdEj{mmNCZMI*p+GIJgG(egW2sUP!j_C*YWyXg)|Q?)bBr@Lb7hnb&g|9;Y$S-82Ai3_YmKEE}^&FfZnSA~o&ohN~GdqOk|F zdunbkg3whdwe_z|Zsn{ny}&Gu)$Fm&*QFcBUNlWH0VF zE9gZpqFc|Iak2Y^sLZINF#Q&0&h&@>CNt5b?tqUy$BXi;Cc3>*3Z*}`)7$;eL~mUs zYF(WS#nmo|@LWn8K6qp53+^8O8wgou=X!kThFebpB}UH%P*qYC>bMhk`u0o;|7$L3 z7;lWDb4Oxf%Siemm5P9~kICu40PGquOPG%OLi?w7((-C`kz>4owzCVf^DJ{ex9yO) zNk-yWtQBgV?|ZyEs)Uj|&Ej6h2fA^~gjq{VX#XNv_d}mDP-V4J42e>~yu%X_dAl2J zU3Psi;-H|hu3uIPtb-8B)$xy+PVsw9!{ zJ8ktj8rWS8^?Tpa)G~qe*)J&WgEw{47(gw*>V?>^iJq%XsPsRcQ%9sid&C0r8S9R# zcdT*7>pwD$P{h-v(a`SK7cW0Pp`oS&;V_7w)lS@@oT7>wpkw@1!5pHi$b3J%?iIEzUPQ;XUNkOh_&Q$Hw*Mj=&vK)0!h$(jUG30C#_w**Cz|@<{PR*uQGcMCp-J@i|xp&*dY)l=hO1Xz4 zvFLX+dMQ`Yl4*m;eDFbu3h|w0tq+p()7djW77h#3v9E*oh0&Z(DeR5!@m-+v2nfs8 zz>mXDWN>HG)D1TouO32$T@MN5D*PW)BE ze2qUOTa}EB@q6i{cQ%CvjzJS=d+T&l$fJIiWZz9@oX&YiYu}W>%i0Tut<3nfw@0e> zWZbp5KsxMwty39=&Nbl(F7kllj#m_Q@`E_AVm$qe;wsAKS@1cK3#8fK==D0^kOPAy0=B*V{tBQFK?!Li6su| zE~V?wGhq8wf{`;eQr*6?w}i(gn`lJ{Q|1%8udqrt_QL z1#k{`g$2}y-X)o9<>dB`iFCE7H&r^@LDuLb9ndI6 zf|@ouUl0ucR8iH}25QV44x8I^Y1^7?cuh7XHHv^3;e({F7pa0j^EEZ=L|}s=#oc`( zNjp%0t6CFr)X4*DE*~R}5uEe89*io^t$uhfgOqZGSPK3QDm-2W~Llbb8CsTwPgmM%sx8^rf!IF7r1l2rF) zAJ<@J5apkcxSrOLw76yA3wI|zHxA}op%NsDb_n~^3&%@UVVCHDat}+qjn2iZDqyX} z91(EDAFpMZ4?AZTIXB&B+3yVYayJqDG++#@Jss5)aR2A~n|x@|F+i)X%wO z7oCH_i;;aWtYABp{oY9Cj!$XY z;7QQ+%@a#{w}_3W|9M0z&ZZuG+ZfNhzhBP9Sk&o@Chx&$-PbJoY}SOMdT)ql{MjmH zCu@QemDzLOSZ*vHmX?bpmO5zX9=v+?WYkP`LD1|{gjts0*~YuHvC>T1K7^C(`rDtEl4SPP#jiC}C>6h?e5p zr+Xvmtl3FPr(U19W|`|PC!66A#crIwC8r1?8VWVaCp^>JsP zdVh?Zx=#3AVixj5Cur~5OgC5c#)@AplprOC(*emi+P+J&cq4lsm25HVg*-J(ctdV? z-C-3JOV_WirhiXHk!vGR+b0txmOU^gD-s?tC78uIweOq<|JUOwjlZ#1jEa?_6|cXE zDYN7t=ffV)2VQ8~90BjP1iEs<3)`-)5c{qV7tN6+h!6ZqtyO#J=^y4^-g`oa7TV&{ zCf-+eE5PzY2J|{`5%q4IBmVmSrP|$tQ6!y&q1kR2a3_l-!-!%_){)2f6s*)S#FNVl zgtglRYX0cK{ zZ<&m?wFBX%+e{ZYqm%LJ5j7N~BjCN2aQGC9E81JBzON%P#+hNgEZ(Ev2`LC%l87_^K9OGENjRtNBuR@`gmDK!$!UX_wOmyUzCMv%)!}q{%XV@d zvWqHCD`R^>DL&lUNf{G;Fd?avA}zGpQ^{_`sg{_r@;}NOHH9vodP%hfL7b&xALv)A_m?&&x@f0{${ zRvS^)&mDAg=Qr^+#08qEX85|+lFnYKpaW|SP&{BE#c%jU_q3;qkhu={E0ryd?O|SH z-#W=(`*K?RR*zyg`=ae=5ze3Liz4PTW1t-Aaz9~NMGm%>l%R$)Lys1!kcW3Nx(`x; z&Gk`G-l~Tsn+)+0CN%$)3?9AckDZ6vl~cDs*dJ0w!LZ?&+2a-|h34z^d**|8~NZ*DIWC40UJ{%L>?G$Y31vg7ixG6rSmK+6K`u+div?GlSU_PnGKn$j8 z@nF*+Dl(`b%?rF+*pV!DE`2E6SDzP)ZYZLM)KK^*kH*1|i%90N6xjyk^9;C*Bztzz zyPxfJVY(hls+6ETQXwt`jbWsjDT~*g zV=%0+jYie3rFE}!;e0!lqM2Q)+|eCfrfcKMh9d0O4FLdSqErm{Tc6&r(LYZ52&ZYNkCi_LA4+`vmr4wj6VUR|orDD;APY z6Z>g+w{fTa2s?+n;a$HNjQiIe@i|Jks4_}8KmIKiN!_ICo64wAxk~f)E|K{0jZcDUN6E&Irz- zRTW`me`$Jm>nwdPJ5OV6rqlb$JRA2@#Z&VfG2qQi>i&m&=a$vC*9kmiMo*Nv7#+2UQ6m_)X>82urFr~sM|r_f%Cpl$(r3c zJWq8WPzdLte7Ep;NTm}ml790}$;3QgGR}3t{k)x$^q_;(e&L0Ii|LkV?CwTB?^B@^C5P0S>*D>nWWMz% zV(T03wex%N9Tim=@C~Z}^j@4nPsIANITUwa8hb8VU`3zda0%>(3sYW`7WY9u4frPH zLQHYn{Rdq)lEUZ{DksxOpq1!G9|S_6R|=Nip8#DsUd@B+Yh= z!w}wqX0Mqc{G^z*JuCCmx&5ku+6apsRH}(+xFXrwo5T2Qj;? zF&vJ8>#3L0U~K$ifi=hCNYcX$^ZHAph-WX|`bXp)&kR=YBz&G0LWlc~aj!bt8=s!J z)94o^bV@xKmC17`cT)_i4i>|pLJ_48{@32Ug=$(>aRC2jrc9;?(jsBR%7QF0=G*7Z zIPVJ@btB>-$^#J|wvI45p<|O!X+&rm(T$;W5JiZb3`LU;P>@8>MMV&FQ&CxnAfi}y zy6Brdv;V(EExxY0+jBAVe((C$>A%*q)_R`zoxNwjBlf+I3}5@mfqh@Tac%tM?59us z=JDA(8)rW~JiL7Gu=Jkq4tM|R!u`)Z_2O{&JA1}GUzz=Xf}^wFJ^Q7bhWpPuYkcL; zKOR2!nHPqmv!B7j;qi?)&ylC-FP8%-x=fe2HnX{ia_`sXTNB_7u-u|=6{=Yqa-~Kxv zzkc}oA0C|b|I^369(a1V|E}ry*=z3`e)NSahDTod@cundoj-o-^0$oVeRJm5!bi`& z`%`xf&)qlsIYqB|X86lRZyUaP`ZN2FTzATN!N1-)9KGwnc>n9)HD3C<+4Gj?FCJg{ z;;F-1KlFy-nq5c67ytgi@YucoSpV&_`^K}+oni8gmxtSbu{=Eb?N85s$HLX)>T53@ zAA0!A@%ktKG(0i;t+dyD@CU;=PnVR*_Rm?zK23J)JbvS{otqyg zyN>@e`OnY0CMR#Qo;v&b?%8)Z13Z5ViTwDYNe% z`*Td&4TmnfW-|Hs=I8Cdwh`DyU>kvL1hx^_M&SR4z|{wD-8e4&>KhIp-njYL_3Cs> z-}N_dY}~Z@0d;XfSAfZrzkSnJlj~)T<62zHRgN(sDyRvaxY^ z^Y-%67PZqmUEiYKym9S`OE#}WGTvfUcsqaBL&Zxnk5G9&jb4(A-r$V4qw)UA5~FD(7poHe)rLy%T5>_@$Do}V{}0>SgkgaNs&24FC?S4Lu0jzV0cBJ zzw|C9+l$`VywXu4nZL!VU=^b#o#cg}0V^?DOXjdyjHZboUetxyjF;qTqZe;Sg_X5X z^ahQ!=*2v!!c_iDd{St+_R^5^qttGGMMO_V*WFD=j=Ae?IUa=V! zFCsQ9|SDh?YK{9Bdx>`)ec6xQIPOpB1%4S%# zl9N}kV%2iu>t2a|hh!?lTKp|~L*-GYcQICpz6fT4*1C}hE~mCbgZ1*cSQXx-CcgY# z2pS#lN*BTS8;9C%wV14fhP+yBkVKs}{<2vw>(Edc=*7DhqfrZzTjgrf#Yn9ySY<7U ze&KDRPi_im*Qk9<#UM`g2aRXhqB({7c8 z8e)k;=aC3PqpN16ZZy%yt9tVib$ttrVikYk-0?0p*-o-)L~ihRwTob~uIpB;BDq)< zoFjNrkW6l@Ck(8DC&7b@Pws7Q`gOQ9BzLR|>XBtvo`5iB_wtJ>)c zHI;#65$tvZ^r}}KZ#M;zY`?%}o-bu8K&5vn6}XFFm+g4X8=d5}xFn*RHKdjz@L`9U9#pht1~6B3bm)%R0$UPCDvpa#Ggu$b^iS=c+eeHtS`@ z=Ac1x$+~znzal}BE%bJis_(&>UR0+LY(}+?pix#zs%SwY8Ee`syg?&NuyjdLSWdkv zlEb@}=(E(-{9318@z+{t=Sd!=5keAg8eKc(svGpOd7<$L&Pmv||E|O;I73p`U^Bhl zJc+s##U!IzM?9j}#7HlXmXbLgua!b?kvz9MM{va|tIX_O@+wr*CMti`SRS#e>ylPz z!rSHste`=!YKB^nOz$q#tQ1|NL$a$3JaXp779QEP!HQK@Xt63dt97XKcKry|PE!=v zr-$;_3Z)k>o4ZPgx0@$fMQ?}3ax#{`B&*(qs^)~ZcoeECN-wL{!rMu_ z=QMy-B?>W%WYm=`QB-=(Dm;Q!>rwK`l5BMs ztAb>k8;QQjoHm)$)~oQgdz^TbqEg~ZN4FNlSFM92-uYcp*0sD_g|Mv~Y8_r^K(eJK zb7Y+u+0zwXNM;@3rB|243k})so}4tPcGR}R#;PpA+EYSRRb(9~gR@@NT9qNGc$6s2e@^FFIjiJ` z)y+#+&Nt}At4IcU0aDhz1c9ZS$v%%6)Gf~N?3S1YIx-p z$vn~ryC~SF6n`U`&9GvXT^qe{79&`Z;$1boY@Qz@c||h4^Lgnl{w9LD6|DGc8Y#TV z4O0QDc$CyA&&yD)P-y7vx-+XFNir&%N$#w&J1)G5ezzZlvoqBWjc#IOmAyvCYwZ`S zg0me^!3s4MxRb2%qe9ZYn_e-phs7aAtb#MF?D+5&$*~!)@9FHUDv~n=;f1rlPadju zL^7*VCfj$JRE83TtRv^Ka32faF7@hDN~-dGR?IY}~qbsQeyHMbYuM6i1YgXEOwHeBws-n{K>_N1o8IRShw6FC)3#FGRA_0P;kE1^ViH!Y>vOsV2k*g8Bks6?C!bp#H91rgv=vY$(&+yR@wZ$q|RP= z<8S+hMyQbNqR>r!{7r~z^ds5oF8;0VS8Oi4$w~7SkMKJEU=^w}D3axs zxxL^VkMwTx#Md)ER*|fl;dN!or5CSvnQ+KwJ(Ndq*2@Yt5wucZh1axMsNr>fRV2eI zsly|ZGcwdYYQ@t3$%6p8hRcT0tnlw%`SR*WS3faBbm)SVzatYc$1T9zcsHyb;{I9 zcKU%T`qPx8&LewvRu!8wtP~^@Ba-pb+o1vHEV5dkBROMEosohwtVD)h&wyCPqppXd zS}CwXZC7ZlvKH{N&jbZYlEoJXtdh=#O2MY1kgsNwad z1b@{W)w{#WbGnY#dz(Chl@puJs&2mzNtLkZjYsy~MRL$^E{YeXIvpy#U3_^YCp&+2 z$xgCoPc6#Uw1BIsQ;daXh0mt_8mK3-H4GG2P^C3!@zX|wRgBbA>=u3*(! zg^I(KUPdpfeQ1|Crt6L>+sP|f1!!pao}D$T@Os9F!Ww3kea}9$#3*^?4Ef(ZK?SV< diff --git a/benchmarks/perf-tool/dataset/data.hdf5 b/benchmarks/perf-tool/dataset/data.hdf5 deleted file mode 100644 index c9268606d224befefbc07e7222bd4eaaae061703..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 527648 zcmeFY2T)Z_w=IewiYOvUB&bLh1qA^Wc8@_4P@)P5NKTRzP?7-*hzTQ#Ig6MRV#JJM z0CU0|F)J7`d)N2>-&b|(ysC3wy<2tOt-5FJs@1Fa>b-jRTs^zz9COU+o^I|c%DT#O ze;rCna?R!1{p<3d)1SW&0~Muzef3ZMpZigNUo`u>oc4FQQU2c-&E)?2$jNp1`+K#& zpZ_1*1q|xfPflw2&-4E&|Nfa=&*uNW`uD&1=iC1-|8J3iXFq5Ezs4H;_kjESU7q=S zp8Z{w%N!Y$?&0Ny8Fj(ngN@yFpyisLbdQvGM=Z|CzT2f8YO)MKie; z|C+!5KK(UA&HvS;nf$+h8T@zu|7gvBtSd+}|F4#62Aw6{`s*-|(^itx9{;avIXOi+ zrGMR*m;d(zmHyTgg=Pk=|8BDI@A6-Ty43!!RezqF?|&ti|JrruzgYbL-Q9mn;9nBZDUk7X&u5~)?R3;H z--DCYW7)CPnsyKDg;>y>WjzY;WttaNp7!98%Z9XW>&50jn(;{93~XDV$GeMn;#^)i zW(8$1MZFI%Uu^+%9V@0KhcSKPHTbp)r+Ts_4^}4eX+sw8UH7EbpgQy@aFWRi)?nmk zEo?lc#TFBSvAR4PbJjdSz{k-T2_}$H2HpSncDCA!7_#)?2X9dJt19zTvTPA;OY!*sJLq zbS>1-VvH|3zf==rp1&8C{u^cIHiz?k`{mHmSO^{4k^JGW#MY|^@zQQRnenox!u7ib zPYqV%%?Z8f6ktKiQzLm-Igffhe&LLZ0*daYQ)h(;T^0P>hRN-Qla=o$FK9 zi=sL!=${Ga8hG-l?h@EM6Fm8%2)2__>2U2k*0)~-yAT^rIPCz%&KVqXZIXERxdRo; zb9vLXGac70LSqXn@z(bon#qme(gF?Ma&68rSp)d}v9kysaaZ(B|B2a|8a#2@iv?}3 zi^N_}M0_txYA#FVjzPspcyJAm6ys3YPLqfKC~@}w3#exjqqSpLS8vT;Ny=1PIFM=X zXAz#g6LmVhxJI!v)SCLyU_=bouP|oG>wK0v{>C;PONQ9)z?`+2B2>)=DO0_;@Uk}J zwmPuvwFwKn+v4Q1b7-OC#NLaxL+P3swbSY_A;t(nA_viFd(nG&9V)J*bLzrUtl6i7 zpB~rIuB=?-4caekUs^Eef-a}Ltw-_Ap48gZjIUm-#YG>1psgLb`s@&@g>_}?2|;}P zZLQcJ5l!3lbl#s~!?`_A$>24bT(bZJ1~;L~qAgFK)#mj1lhGsiB|05@1%)s*?%864 zDNTQb<0Vc zD>{o6+H^V0?@59OLVYC2Y2J*B%VMRX+LmE0KvDBq2N2T46 zX}Ia*p3^5R?|KF=Dzo_~Et+#jUWU?Id46&k$WfgV8L6@!nuqHUi7oJPN#f&z4TxJ^ zh4HJe3WM{>)Kt5T##P5LVRw7I$ zq$#YPP7+G8wc^y|BbdL~mi2cms58`-H~OZLcPjCs=Rv4{_$ppa&BY9@j*RQnlEoIW zV$ly>K56|52Zv;G@2+TOCB&kCZXbSnuFX>)qFC{=2Kvtp@H*#Q7spyL ztj{rV?Nx=Fm_z0KAP%G2b_ihi^cb#OS-1k3yVcq#4&KD`*t1u|FW zTa2U5Qf($$+{5v?A=H?!gel3noVd~rMg=1|<7H2#x800J9u$@s|Li;{iM`LiA%$sm;GKZ?a&RX;p?p--)D z3iuwX2;*Jphzb~o3qx*;U;RsQw{$jCUKmh+cxUG9@5-iAb1ccw z>NQ*OXX_x2C{|#~+aa8*S|Qul{sxxDB+_C>C6<~R;!bEf2T1+gbr0YU^RtdS=ROy; zca^!nD3gjdp;$N9jxI4?cxdOuU0>cqrJE{m-^t_4kwG-6X-?bY>3nl?ICV#Oi0iW! zpfczo);^7FzeAhIZ+wXgGdSfQXjZ$ORqY+eJ9>JvuqJoPdYdx|U9`*>z z0rD8TTZz+uBy;(q4;Xkh7)PsJXlw2#<}H<{@s9Q^Ssle3M>X0w>hb)r}l^<58$-_=jW{(cF3uuYZ29!)`AhkV+HwP57XkGP$yf%57;-1f(WpLUX1qfHIV;&8QRBUIZ=N#j_L<7X|)mdWErx&#)9KtZANI1QUsq# z;QkF4uqJ;P6f;t38|=sY7@1@n)Qe5}?WsM+2L#wQ9LEo!nQ59rirYsLDX6 zSSFigAa3VqTFF=7$1`Q_EA--|aq;lq8b(F*r|OsCd-chjCzFAR?9AcqgA+ z@^~ce2Wn$_cr<_XRbZ2Tfy{2A9Iv0Bjo>|-F*f25T&DWd*>(_p6V+(^sR|c<)nM3X zLDRuGSog$K)cjH5WedsXX$*(DW<2-j7;|03Jvg<^7Mz$7=atoS0XnAq}hYh8}-dK{`HgV`foFnXOT4X=(u?C0yqx3K4^KlQTahdVLk z$9|bol?H|{bYy6xJrC8j7n*NW7*f}lwqxS?vXvY|Ryc{_i5dJb6Yh>% zit}~-5PN7Q_N`cmH}88P(J~#Hj(J?ZV=`j81hDdJ2K#L~iEeElpr$$xeU%5IY12~? zpPVS_t|ri+-jQ?HCbQEoZT485OvUl}{5jJdpU!76A8r*zFfB;p?-6O7nYtb3R?kuUcq8<_KNbq>O*w6Uv2cC71@9Ffp;ez7_}Qex zelg)3@cIMl;;m$BulU1yfdvk1&ZJl37c^eB;*m9=T81GCy9L1cOEy#LvbmwPCzl3x z!hPp)d_6dWBh3S7_f(!c1J^;xPywIH%21p66~4tI*u`rmGeP=rw9?>T1%+Rp9J@*AEDshl260z zsC1Fskv*0}qvd(*oFQ9z>+|Ng6#h971Fr;oT2!`WmWe+*ySQ^V4EXh(G8d>AASJaW z`ppT$Fa6z_;2 zH}4$xY_h`u!A`2y5O;bwXRn0F@6be-K89?nabfy;JX^uR8;|i>tZ^N|SlTG&DWFwX+al-a&VHM`WvhAZ#rjSdeH!hfG*B~-AR*IGP zC*s$ihRi+&Gw%LcE#i$nL5S(u;2>&^S)_{Pq3vD=S%`>kPoP@gj|RAam>7D4-8 zVehq0jHkXVEH=^s)IkI9m6ZF{n5YM))6F!w6k)7fQ`M{;P@J@?&7fwRR=^>Q+ zaa-0``vV&5Oz5@ElsS{*=)^lB!ybGsrwhd+fgI}R!mK>W4-?DW`?3Y$!5s{IJ1-GlLUR~VM2=J0|}HiL$F({_dh zpSX14)!QyyIB__JUE3*}7Y|uPygi#W+(Mx4UX0O|W5*i~+-97H-T4nii;>E7(R88} z6WC8~EPomd=Bk4oxS?W!m>hc#FAb!9)i4U7o27Z=zXNM-IACeP9Lza(6%Bf&j%}_P zFn)S1vPOh6yy61d`BsVNOM)3K$s*c)AnmL-;Bi9+B3BQlR{jQ@^{9nWi7Q{9GR`Q|jaq1RcE?SI|r(Rr_WR9rRPf$)6!~XK~F|8(=JHA`e zU-37*@AT!ZHKTE($78W`R$H2X4&$84PIRpv#Ul|(tZ0|U{I+onZtz2=>A`&2kj^86 z%qaWq&VjujV6pjF>6wEW=3t0Ybsc)xoD$djYj9uUICdTtz-Y5b_;pdnoA5~(N*is{CDAi1GXSaL1lU$Vq;Ok2kVdHN86q?b5}iS>Y`2JdVA-M$shSjqC1oruWBE zv{cHZhNHaW_9q7+wuI74eh3dVx8ym`R;;qfm)_3|yxOEm3%Nnm{oR#v`X2mnsu6b| z#PH|qRw&+GgX0p;q}r}rBwh^VP(63HwY-SA<8!!ou|2NecV>@wCxvovV2fHyu4r>w zRBqcL3@-YxY(oy+%*{D7Px9#(b`obl`oU(k7t^9$slBg2ye^fa!Nxy07wF78Cr3)_ zbqmhx{tYRTjX%k|0CLe;3>s-cbIVjt+Pxdqaj(SGmE~B_;%|Qj%2Gyp7Gy=wP8_o35@3V??(`D zrVpN|=iurlU0M}fLqWl31h(tP{&zdlyh~?R4Q@|g&p)Uys}dVq&&1_ER$QTV2?sR> z^6q{)#_AK_UL?^`u?s(59)_WjZnScY;<5uh#poF!Y^yY!OV>4be5uxp7Z0bwWX3H7 z^i1G8Il*Zu?HOv=ja&Qgz_i9{eAZLo@^|a-e9vjfjK1OY-n}UMl7mh4_B7P+W68W8 zB5#g6HE&Oc|1-%x`V`2d`GL#{%%OVM&S>snP2=gaAa~82Q_R%4X;mNkPE=vT5gYE< zvP9PI&rs?wRHDC{n^?6+aM-bdbPRK4i`eI4#f@A>XU6i!@f5(h)3BWpnr!%Sz1dEFL^x6wU? zr`a9Ev?zhvXk%VzvrslgE1MIvoVjTEJM@%rj{T;Y92^mjDQ#|`XR4~W^J2QN4wF`^ zOUi8YJ^+PX^>9CZ4TS|0uxnwH=r_rMduuj|mo2xz+&G3Fi+&1`U{3EQOS*;b5S3@H zV#>!<&e1xJq9Q9AR%yYtO=~`z)PcI&mcnC48Usez^7*XKcsBGH@}C;v!+=3(Z(oJ! z#lK|xlm>8CZwm(N?iEvgMsfAk5WM>^km;so^z&3kFV!)O88KG+O%)0^w`O|Qd@(s8 z2L7F`#iQ{S+*)YKE~eUCImeaJ*PHRU%TTU-)eMswkI3pWy7A)ma~SyiD)i=BarXAR zBDq}#k9;ws$!>3sU1EdlkA4X2OfA;Wbzo@+b1IDVKjh56CTeDVT-RX&?!)!>GSp?DRQ@PDCozbA9mx}(PZxEG6`$f`M@G8 z9@od77tc~nxb9n-D0_PlFIq&fnPiIxJWk=__LvEc^yjKH1YeL z1GHY{G5o<`bk}Ocdrcqqu6mD(C08&=;S@T4U4_IwbA)GYA+F4=!_KI2h&!Oc9nBZP zznv$IXXMk|=Lj5M*)UI}aL_4r)?Mj@8%N*bMwB=GRW=LJcQ5o-By!%GJa+j--1yp_ z4HX$Y`@Rs4j>f!bJqm#V4%~J6fWz(G#+Yz5pL-piilh$4@IKaokM*CxdU^rOdu8(d zL<=sRbOdGzeK4bEE84HhqtEtsJaQsbR6SgX(aLAU=Z4;NF&&OJ(|+Jo%R!tTJ%U?G zj$w1CJ>JIW(R0HSEV%p(7b^`pu{=WT8uSf6Mz~UW&H#=LUV)esvmA0iwnU&;27NpG z@%gKVXy&ZSKkIu?b4-!wIC}^~a&9`dlI%b8NgAAVHG|KmE6^&sEt2-iL*+m&VZRu5 zmzT>XE}f4+sZIJ)4=NORi>}s+yi%mh&7ZCDssA0kHMe8(=9WCq0aWzR=6j_izT08P zPdcW2maz+tZ9TZ{S~(mV?b&In9%ns2C3_I}9wV}QvH1ukI+v(A=A=6zdO-@+^qO<- zO`W6KK=nKPY!_o z6dexD9WVCCE3o3$%0O<6i!VE zsDHBqXO>oCL6`=OI%{Clxb`$Kv%;3#dt&Q>v+%brhvK_6_|R!ApIu#!h@@Q3>TnL* zB;5C2XLat7Fd_T#W7!arC!*H{;l^4E9I;Aez{6T79@oLQ*&}h?uw3@jS`o7j4Iw&5 zvVZngX#Ok|y(YDU@>@AxGVIN`yNO&>=g0dUd^zD@C)U)@LBhGQyy)JKCvtMB-c|<} zFS@X8mJ5fr3}I)3MHqgn4?C}lV*ct4_+fff)DK^W@nH|JEZT?flMHeC$P&z6nZlwg z5(aR~iWAP|(DKh{>>ocHAN8%ocyA{jH@Ja@WzG3(T7T{cQsb3Bag=!m^5ozIZXdTp z7}uLJZqFuBn;~PDPELGNY(`0=ICp>!dki@x`u5W2rta3v`#cHFjz#n5*g&=%X$P(2 zN5zemYP9NS$~h0hvFPL~%<7tfl@7i6;P408VEfMe;XD>A8-wY-dXm_k9zglD&aAQR z&h}4hh2?D%jxx|;>RYL=C9ZIYQlpl`G`Rbn!rJPQ95__MAr#DTNM#(}UvXro4XrqR zo)PmTn{9h#F6AXGyThyi(}g>aC6uQIy9z;w*ovaQs?46+f+c<#aEwY5BeLs}yh9B}Pjfho*RZZGunO`?wVDgTd+J;?k*8? zie32P=u-7b%w>;mY%Nw2YExgIWt(|5m~Dx*$H%9gfkG zOqUP;j>=TCRpQk4S`^esKJm{iM)s9whs+o780X2LbAq$3jzx&9 zBaejVu*j_yj~}<;sIPjEUsQqAY&Z5_JQ!;~+{09#ws_t*mG4#lpj~*b6qeCLr<7>y zpI9Yg3s>N6-$E3HMzO2HKG8ZTmfctCGFhn~EUIgf8Llt+Dksox%_!cret;X6W7xUx zU@pp5qvr)LZtJbW@fG`U%E^%>;p2JVr~=JZ5+%&L4SWk+ne=Kp`j69RYF;FhCd`NL zmeDfnb_FST=FS0@XX=9NlR z7}d+y;pinP#_=Xi9O!Zk&+$u&#U{Y#YcSoOCUb1#FqS=XrFE$@-`>yW^6*|9tq={x zn-hdxP>v8nAYyf_=m;QRgQzhVy(vs>}n=U=kc)e+oUq|0B5QhYQ| z6&o)uhTqec3@8Qr^d80u&+SLefwF;YBd5Sry ziJXv~&m>!UG?qp2%k5r#G-U|XpG$GZwn?0PI~2XXnlYhoEcUI>gmpy-=kDEzz(ZsB z?c7s*J9+@0uHM7kOT}1TXorxGPW*FbJzfNth#e~N^jPl1JdFZuP3y$NemBwah&={}aaYru~ z2L8z4v9qN(x2+YAEUpqBV=o~8uDPa;m80pf zvje}J+9DoH@xYzt26(j=SdcrKrj2oE+3Go#q_*Xf@^)ma&F3)+Tp{F0Nz^WD-I zdPI$<*Hs|xNCaCpmEgq$Mcy~qAgcW~qVmspe%?Ajwm7R1VSOsZS>2JCJSdq@3udFw zoA)q?(MQ=iJ=!UH(lgkBy5|Epse?OIJmTmdnS&1JTgf&r*)NK7%S2GwFw|XagA2zc zy*Bfw=ZpZfFRg&d%m^L{^5U8&(psS`#aE);IChf@ZggA-r$MiT>2!JC+u_IPOC5Q? zR<&)3PEHY&mv{mmmR(pI^a!$fgLyN@mZnNQxxGA+ z=l8kbDy4ngPZ2IBV%auik4UUn;{F8}VIQl5^&V-wx}?BC*Hp4Im4D%D&Lot#CVR$) za-d!yn~JR9G5aiB222qv4<3W!nlSD%ibq4h9(42wqOr+H-WioZb@}1!DL)sFH#pJT zH;Ye-EZM41inlHA&2xKSVdLykZ1!U}@>ca^f9EiIw)u&^vy`divR&5dy$)j^W{(Zk*k_044`diPMU9oH5W8{r$(&$~p?a zEWYAI?ov#OYKIHoTC;I~0p{qb(QJb)99u{@!_jG?vPUe8@4Uf<+xtYG<{D&L&qq&R zD;`bhjE+@}2#rpmYN-i6eaq$jAw}r9*PCPXZbHAfnd1s+t(f9HLlh?urojatIxWxR zB+YbwdpVj{4@|?*;d$IUU^Jg>7)hIA8|JUm=3(s;oV0V`^DZrDTW!jK^UXP`?>ES0 zuS1?;lhCoyLWGqH-|HGuJ!FlTbh--m__VKbmWgof1(KJXjM}gD>jsFt%?? zl)T<6DuNw(EqWZ!G#!Ly(M3!$iQ$Y4J*-HZ0gbw6=>9E++U|jvA6SL$9`A8+`7k!T z(c$IF8yKfE9)r%k$K5Z>pf|&p74PghCu|Nbi|%}|@8FfLE9%FaG!o-7>UZz4`!aJM?tixCDHOMoGXYP`6 z*-q1~C_A(n1-)GP@M$!jWHm>ggCZWd{De6F5i1*dvUt}oC~w^<%e}Y>VXxhJW2`1? zpLAsQpM@yzEXVrD9_$n|gexS>|5=$5!Xj+wI@gi!pB7@aayDw@4vM1caLPNB3EyWs zF@BmURg43vDlf$<-PejAw?gUExjjQ}d+~0FA|}V*fm7&47)acM`}yNplxj@lNXgGs zbmp*>ujp94LVT?FgsRuY=rTu}TVF?T_}JyL(semZyQ{@tw_@nkY#}CW9>l$$pCQrK zo8~h|u$!4DD^(=iX^0y;pY2QIsOzvD^cqhRcc9RzJ+F*Eg$HHc9Nw!Rn;H7^^v+vo zxVKB(sXB<^S&gDN>YQltE`!TeI&u0QO?GrOW%ZgdY?Sy{(JT6Mz|%{x$jxWUfH%wY%F(U5kPmCy0r1DsGK6-D+vlkx2yk`cBzEz003%a%LjDyXL*(tv>pI^!1;$q2vHa8Kn z9b*{L?YZRF$?@ZaE-;iZp@a1rJiK5NUY+Q`{%+klAWelY9R&T)Zj$2W#n@mh`CPSX z?EIxQXTF(-sk?JnyJ;w$R-1EWn41X74&(CabA?>iK*#z6<5*!&{Z6kWM>wti)07CP;*GeROP%UYN4d1is<%{Nz=Sa+B3i-0O}<4Z6_R`-ynJXl`-s#9fWr z;;DpxYNn^bdc|=Z#vbf-D?;zVZD{o|h8ySo!i!K(XdWTkFKWY0$=1BQAb}0Kw=n6; z8<>qy=h-&hxVc77HrhX*wO5<7T{BaDnz|DE52_*J=L0BgFy@kusr+GJgLCUb*d@6d zmD2m{xagSZzxod{6}*_$Ck^9GjhGVr6EpiIbKr!dP}P}&4WI9ctI4HU|EDjqo=LvT z-fmQSU9xn$4An)VbK({RvJn|U)x|CcsQ*=>UptGjZBRXb`=lcQUi7hh;eJhwYj`>qO& zMWd%58@H|!PnI;I%Udgv{M8B*KX`HHDv3|BZnw-#>gU^61DY+&r^Yrp23?tktM4pW zSDnlGdd=8p#VK6RKZ3eB+Az935vfPoqWPr0ESVb1#dVR4*c#7$?dk zErwrIvO(s5X9f~aj^o5tBbc$} z7+%Z>V#?vxFlkIfpA9BVjLdU<(l?CXu9)%hhOVr8)Q?BaTXM&X0!-5DiX%=pvB^#z z@1qxCQQs`Kxoys$&C?n5!;2mt5;@!4UpD%u9eWlB)GUuJv0i6HUgwsQw3ilo|&KT*=%qh|AkbD$|fn(TQ&5|RJ zT}N7L2U;IHgY9EG)3d5XeBZCd>6=<{(!<{v?_-B&;jhr;OluC>;f>d)#`BT>P?}_q zBm%aGwO%ozlg>NTO*Mh&(}wd}E)hR$C&=2>PsYx{F~YUTk;);StQub|A|EKTTw13~ z7FCGzZjwz|8qB!bC3t6)z~nf8(XPijEZ(7yx-tb`+iuN!5lZOpEh@-@_aO0CzA)JY1xV`sgp_?Ih?PTSjIVa$2fuzO-Ijt0I|lOecq`fPRVCQm@2)Vr*O|8$#IyGJSWNDu%;!0M zpr;VaxU06*sZyoFp(2^z#oeg1c?z}k{cye~sQ6Hq!wjOhz9NdPZ|blhE}uP;BKYg4 zp;!_9RE)I#fY>M>s(JhZ8bi?A+7b6(+wr|DiZ>d=Ic!sJiJxrGl>5^Ddv6Y&pMQuI zUYS(gmC3sj*J7XKyZVNoMc-%MTy}pbt1kpg?MykrY#f!x+2Pgq93H;Vo~t?!;hs5H zu;PgVhc&l=!^7K{ov6-V9iy3}-i;3ieTK)Rr#STR7_;*a5Fwu=3DOQPJNT`*k|8xm-0$`_9I^Wl;?aoBkh$wHsGJ^;C1~hTx_1lxoZaEqJ9qll&?Xr*_ooz-Uur7 zeK|({AjS_*M2BXxaKHKt3=+zPhHMNImlTRt=Q6ln-G=wK2lI^TKpvTw!yIcPIBh$P zpnPA<`8R$ASF?{`fWCAMbmrK!+3HC3!w{}S!W zz1X|^bJ#}3(>!$&GJfmPwpj+>4i2Q>fYma)Y7^McY6eyRQ7He`h6)41c@M^_ybi%P|~ZO{%IJurjL*>uxG>Zpd%6^3WDmE8fEIXe!V@xHBcxdQM$xdm2mCkKa$@uYOk96JxNnGnOg@s1!3*%BSrfb-Jb;^15IuIevEb|s z4D}+bM-DOo?Y<)d#j{ah3YtF$A6Oibt|qceSqS}nYdIqnuArX_)X#rc#b}WubO$h zwC5J&?#_g+>RcR_)`eZ&a}Z!2&Pz|L#N*$eWS=K{GHl8MEEQ?IoSVx1ig9e(cmuo7 z_=|)IHf;a(5pL!cAw(+l z%X14LKky!|DYSyW`(DhtoIvB@X7t(>L;G9$Y}K&^Th~~zsMro3m7{3zES0xAH{t1; zII4D5qRx&;sy>loxnXnBWw9sUza5X3kBY^laYvyn*|Re*H|O}c2t1BhCVG@Ek>nW4 zud>}p@UWpm-#w7a`i(=Xxomix!4?5;v1wKZwjFv2U3xk2XG#)>FMo}RXQh~foei^g zCedM15e8fcg11Vkm^^h1%+EYR*H(X!F>W+X3Ub)*@=wRA>p85t3`Q-UCd(Xt58AHr zymy7NM)e~&c6J=CyDQ@6OfxQtnE*NE7OXhA2nJ0Xkg;nUB4QU|#j9iZdO)5-TK3_1 z3FkfQoglVfb8@_GU4<&^qZo4K55}A(-yB?oybr0=_3Fr>wkEtI3*x!g`!Va;SpGQN zoHbG|g{%n>Zh2lt-~ zgY{%>CR{P(&&|VVy`fP2*ft3_Z@BZ=AO}`C?0lp;_HWv_-xgK^=Bh6Ua2_}z9n&wo*{$#{z9Ui4!h3V zf*yD5cx~Kj+#EBGCh|w%?EOm$NlDzXOl7Vd>d3$w%~^SQFk3qYAU!;oy{@<6cUc#i z>-TAxxbzX4o1_WL)EX>b@B|hkUgF-JAera1eYjY>8XGEI;rM4d<`u6Lr9F@0_!=L6 zTk6k8V~XHaV#||O_r=t(1|0G^2#0NFM0R{OqHEn*s+>gspW_+OArr$2zv6VrT=e;J z37;1mvcjPlSF1+j*4=(o>K4!Eu4#x2u%+{#Y=#EL^VxoR_Acy6Gc#@OvaCR**z1T~dN$D~_p;HBDvPj)n*wObp5s`efjYV7W`hEY4`Ec*#y$?=Si3If&{}1etai zr@bA(n9Vn3*C%#hzb$z*cdo&Rq*?eV@q51A@}jm}A(kZ?(51%~q)a~~8?mt?T83G1 zkCcm1ljVV`F0FWd>@+B8o|DbK8^|B5l!?9_(Isyw4!I;Se%pH4ufq?pC%!F}57}|1 zat@a^=5g|g59rf8kjT$~s`4xN4zpmV9fin!lLMo9aZFL$AjZ^r@?Nnrmsq>;hLJo` z@I{>T&16T3BjfS*wCu;Z?{I4+&(z78Jgb>4E)?o8y+D%V{nq0A&uVDbnPKuv$+yT) zz(V~!u#+&z8)rn)sgV>F6lJjWlssig%Q1`0`AhGJl*rg%;9#+oGn0)xI7y$nM06m!jBf zsV#fj8nURJDQATL#+D;VG`z73gC06CtA9MThw7m0<9spU+d!Qfy~L zKB6pIbC>=Qo|Jgt?>qfL(5)8O|DnJk;=yznz47Jv6$)IU+aPNuSB#O*b2z-d1|d%6 zxX_^(r!RdGeV!OfwwF5B`|Gj!xdD#8j}&=A!GIsu26Fv-SF}DYTqa^~`AR+??Z zag%F!wQ3N51ZQ*1#FKFDGlolVC(?1QErTWA>O1|VFyA8Ot_(L@&trYLp!;YxznxCqjJv`wGLkJe?iAwaG{`S-!qO%s3C}o& zB|3IY%T7f6SSg;b?E!^BmK%}XD%~!BrN}Y2i|v6U~Os|x4$((jK+1uSXPTq z$7>~ih#^kJ59ih>Nne+>!tEhrI3%$a=NlrK(xwCTzR0*;b`l0p^Wpn`zN4w?F45(7 zTg+*b&iYC_=iGh+rD?8ecVH=v%c!^n`$h%_R-?V?0CqXA6#9Ka0M*6j157+>C5 za`UH?XrJ;9DUqRQz%q%)-cc)$?bdbX z#>@a}tkdE1Pmvttmca>b&cdqS8kj^b!HNOxnHaDR5zA6=G}4Plby9guL&|aU?<(2Y zl0P(2hb30JG+t>emUjP$+LLk&T-BME7VO6L=BFIDHg$tfzmc4I%z;TY4z%kugbQq~ zXt&0YOLZiy>wPj*wl2eS*9K@zcVXPz6e^S~kVROuLP~im*XvnwY0^ke{v0iozAT63 z*a&(^>#!{DwwR^pMH>lMx7W{P>9-7?$u?xVPCR-$8t}ZUGwRk}!MzX#CVzS=>o~6k zC$FaHV6_8-s?69T{Sb~P%JG1;LDtB`j4w{w3(8|=hl1iAxz@`Oq(ed z9FSw^{bB4pXgQ|Lb)x&ZU!rsJcijFM#@NU-+Bi=^rIeGoIL)6ij|X$uXC+wddW5JG z{ne^XEEQnvl#N6jPi(~ivJ_b zf3^m>{U##tqcVqgH6eaAVDc&#?C)j6_dRwZu~eVKUHh<8up!fK-j&t+lp@Z06~-jB zJ^WLM~wMre;z%OqB-Pz1@2$8$DG}EFk6|*KF6WE z0c<_kj9*WFm(BY5RLEKU!7_>8IP7RkE}QMk;@F+o`B0PRFHOXY?`IG$#W31#=*jpZ z2Pu9W&Fp2j@nny*x9fG`sqruHwyYHOZH98>SSg?D@=7G$>PP1j9umH5!o3!KxJcqZ z+z3*`#TS9t+3PKqJojXltQP_=%X6rd^RsDaK9*d$jMc{N(9>`u9y(6M;OhIR4U3_U zQ8d|@#nipeaYMF1l&%0zJPqa45$ExtMMrL0OuAK!q1*O6Zjj!e?}G;T4E`-9xRoGm ze`|>^@LkHsynr8tudz_d6^XlO0gbE8up=)W&5YY~(g!z;GFISq&HF+#G@muCTT8LI zl`@Z0y1f7Ru`H}onP&0hXcKQkj}@wH*EN{c4gOfWKU((alN7_=(1z*7dr(j@f}3RD zkbKdGzCIVwwZx4sQ9Yq)rw#vAVX}AI28wn&yb#)DB;D^*X0dS>e%v`Hgn=_}@Bb=H zRMhAJ{+{Qjhm$0yt&ldzW|9}2~}E{>4QClY7pN#s?&FOi5p)9R?HdQ6uIj5!{Uug_teW8@Uw`;P9zMn+B^%r0p zYbUPe8!`6gAj<8R{=bCj_#ZwG2ZP>@T8fA8d0G%vT?V04%I~tT4#2X0y|Jpb3aljC za6svHG3Y*sH~QQypTs}+O+`twUL4+YC2q`~hegICS@FAC+*q>_X5H^%u#}^*@|!0f z{FvzzUOB#YJt`-whVE@R;$duCRh#dP%*FnuR+ON*1GxNa)5`2>H? zEyWgz|GHC4hjB&EvChzz*LIzTcYldj@7;&3UTBGNlX|dt;Twz|rp6ZSU&+1>EE27{ z+4I40KYlB87Qa*nioK1M`0(Ngd~N*jAl{C(5+}ayP6GRetwBY4E#=~sYgrPCy4z4fR6CMmvt=@5Pz z4X5(nfpk~i1NCXUFx~S$hIJZ-2&2QW@Rj(P&m0*)^r_=(iQiOjmP{XmuIxK5n2V2I z#@d4oK>J(bTi335sAa&7>DugFJ6)X2T8ck=Z7}=9Z(%nqmnWq7#5#$;zAIy}?242Z z(QxS)tR!CA{RKssGUNqLNx5ordKOZyfHyOrABF$dD1@yxWb+TYv|SN_X|>1jXjdK& zj%dbQuZxHY_K|pDG0L6J0^Q?;89(awYtR?N<~Gc_em@iDFXuCd9rk?k>YgjZw6g9>mKn`qAWKuGnr8%+%BsV!MSl0&jYw zUB7EGm*C4t-3HkADq!l*1jfpjg9GT}GQ7pX&rPuLm-g~UgJc8GrlacsI zpRcQHMAmdisliRbx-J>Q<5wD5d&V&5wJw)kwUs`#576FdEY`?xz}b1C_ljAm^oWn6 zW%)2RYzu{Rl_L-845Y`pJ6NamA6lE=g_hcBG}Lrf6dNc@4nQ1VufC11^FQLA^mh%r z^#CgMySz)^>cGe-fsgH`iTe2+Y1n2M6TEJTZM8jks?TIhmHXA7?W*wLq?Sn7_)0vj zFcf+R zu*)Wv-`^;>?D0ahlAee!wQ~?UP?d8(JK|+be-2erFsA2qOo1~ynP>5;d77BjdIu^j z(pfr2frdGn9CY6oO+UUzoozGTc^}56S$+8UKpF#!n{lVxDXjVS6YtFKiyj+g9>Dkn zie^@0+1QP69DfFr3!Bh#)(;%)AIB~c8Pp%P9K)`yfy!+q{%w66b`C|DKOu%KqK8oT zTb*#Ror3mB9`HGT9R>@k(R=GeBoAB(mnv_(42+h4?j^iwaO3@wM%2u@hGq$~5nUY1 zjlS9Z^Gg>)&NicV$3r`07RGk3%z0?DP(_G*u~4|34dSX_e^L7N2cjx<_@~t|v398+ryibz z`}^i#{mmxc+shOte6tTLQ3mPl_L*JU0&@tL9emU6jBsOEsnGviR zlOgYfv9#aOm&>%jh$bsNSo-E6roM=v$)gDA^|7F~SqVz42QtV&m9EnbX?X9n_`F+{ z!JAhhW?3-X{wu(z_zpA-mpZaaSM-}bf>DEHZvMV5n=Ed{KF7!Mwy_Hjn#ru&E^?wy zG&}eh;rIY6x>mGe*~Z^E5oCk*YHF-cmwS|FQcKW0ENZ_w^K8&ZF(ESszEW>Hf7%Ck ze9wEgQ*Y0P_0Msq{cV{gC5DJ&I546Mx9#hQOP_C|Yl|A__PB~kt1=icA&)1OE_&Pi z8cWZ_9^COg0BwdCF!QMbZ=$!uH=#40N}XrN;ojJKVjrG$QsZ2i(|Kf`$ElHtm?!tD zVMD5L)-RBGn$mapF9aIIx@F)hImAjf}FXg?m zAJd-&^KGPLgDCnjzV;iAf4YD%2bUq`PA~jSw&l*FHhfevkZoUW#pIdCF{-Wu(-FxG zhb3Z!MH1I84yRr(>%TaZsQeWG$7&Dfd(o8Rvi_b|uiJ#PV zR;eo%%xw_t*;=vZ)i10u{f*=qnk+xIPCOgbnIod`3aMebKzdQejgb}9T0O^NI z?uFJ74e-ec!LF6nVnyr;5m>(+r?T@|Q0F0Tztn(XCz*eXiV;mT79+oN4lOEYiShoQ zFm0b3Pwa?c_`T~;?C!t=Z3nUA-VCBpmH%Cp+=tROj2L_#D?SBtqE`x4z2BkY%sp83 z7|0`Mjo2=t7i~<#dFR$dxPG-j^A%f=G0jiBiJQ^ zq3*HVWRl3l+G1RlcZ^c_7;!1s3!^()uwq(E^s7y0XV0eeo4OXhZ|YH|HHc=Cg)vR) zUbh|%;@my;c=R(5li!VG`N551kLMfMx3&@1lry)X4JZ1 z7GAA{gWP@h@Ndcg-Y9Xy;@r&yD5gX65KG zsE@cZ#-3-Q&3SIJI**+R;kBZvNC@dm`-AfLdwc~Uf7F>{ro~TP%^1HXpGviX?D)f% zcJYZai3m zMoss%Nd4!}`dKnl)@F_3d|m*59PfyK-jmSzX*C8lSMbrbqu3LD7n3$?vuf%R=y)v0 zf3=P%du#&p-Fj5buf&a=D`3;~l}K>WM)1wiwC*C{^~!>sHUxA11vACV=|wyZwJhe#40@7_@i`DqvDLcwz#0&5z3DO zWwvG)M8p-?yxIVteG1wb>R`jXo|s*72cKT*a_7E0UI>_veRjh+rEdgtT$OnDxCu|M zy)JCl?}ge;=?VK>fOW=$dB!mS29wrd)44>xj>)C+u52#;A>Tha$xJxrz%O?f;^q71 zEHj^gtrfmZZ|MSqSsPKbX8?a*A0#t?##H=1C?=d;j`YX5y#LSw-_GFM z_EzjIGn>kFi*PdkF#4QH>>NHtG>mcP=M!hKY?BpFNOp<0Ylt~{tRR0nlQUjjlaBCN$yG#zdl;$tue@e z>#EJg^Iu!BdPyPs*AM0M;=gcA6Zq(4B1-zSq-AOeistFySzt2#FqknWt(mSeigtaM zqRh;h?<>oJh1<{r!X??a}Qlm2qhx+m2{PwJ^*n!!9g1F*AC>O|l+FO5XmQJt2gcZ&B`R6tS!x?$pky|c~=g~-8(fZ&9(ad8Zc27`Yh^O?ZCiKFlB`3x8 z=lAgN+j~(v>WQLAzmOlF3!Ztl8c$8U*f6{tPxG`<^F>2UA3Ynttfco!b0%Kezl36Q zCN)pqg8Qlg41c1B6#2~_sj+0)n9t&$r#_BFF2Om!Qjzd(1k0whVrHd0$Lf-KMQNA# zayN;Kq~HC-qI@}1Wn#d$v1~2xnC3l>36=b-c=_vsXprBMYO4fZIW!Qvb~l5n*Et+s z?#bj@U(t5wS7b~*hhg_Bv2koXueSOR)7s32p6zOc`gLQ~^{HaIJfAZEjivrHZBEJB zh+NN9Rupd(9XB~~Zt5`Uw~R-RG(D`Dqf4Vt1NnQPCFjnm#eu(OsQcqZ)3O%ac-lqS zy~{z>^)R;Y;*H=DAy{50weq>|#C7@o68mWYAs8=gl4N!qm=@wN6^BS!2hzdvEb2ZZ}rP zcjA9#udvqTC@zg_ixytTaH9WWar=ZadsiEB{%|{9m+YNRY8t#d#foC+T)1BD&j(-A zusFIaf7Py343!=*>&Grww5c@{%x>d@sxHf;&f!vWcly=GFx@Pdr_&~j+ZEb8InJG4 z-8$p#Uk^Uhv4qa>0sQdaDyTdeD7CuRweKPRQXY@C8ja3# zyK#znDi+MzhgUTtIHkTDE0;_XEsllogqa^Rnca(`t14F5jWT5wBh9aASQNepZd=nmygb9n%NmtdR|M23ugC z^xqgMhSJukEgx@Oj!A=)sB?ZaJ-!U$`DXGxsualkLzLxg?#^kedvRKGvc(TWcsC>> zFEgHoce=rh($BiR789@K(=*qcmTw;`%BT7B+M~m$d~e2gBmZIwA7Dw;Z^fqQD4I6= zhw)BrcrkOU_)?Gw%~ipYQ8QgUHM8W5t%cmwUWeU|4`O(lIrF@32*dMlG5Dd}#qA!i zC|S z46GWpPnj_=m9&rS%r%{ z`m?oi6b;`=ujPq2PV%kC-L*$Bbn*rm-dhUa&U&14R159KIk8*iC+yy7=Uv(6FpjKk zP7Cv4Y5+iloa<_(69l3ov~ z*JWwPGwRYR7)#Bo?q(}aRoJkiFoYQmfixYZ;DuB@_|I43v!W|FKd>b$@A{zTfie&E zDHVq=E*9~7e0bt(U$(EFjG@o=V}75jiZJPK2{T%ZSMuKZlF_qks?;U7*K0zOa9G(| z9V!}25iNCd}TX(#ot0H}3ei&JY`pknw! zj9V{#-+x+ny^?woO|D(R{mElcx7(kI>k9eyL5X7O($>)1pDgYse8p<%+p`YQ;q{fX zMaeRI#-xwrzrj~v_0Lv>6+iS&nwHBualw4}$w2hpmcp7R)3IDFp5F>!Qh}3!m86<(Is`1n6>DGSSR;qEst4q zy_-4hujO%{l?lC?KUEA_dkMnfE!wqNiPA~N+#X*bj4TsraZA1fTZZ$teDAmF-j5e^ zr7vcEB6Yh>N6QZ@aG*=An7VA9xFtDpCQ^IT?0FNenKpdmY|mb=^r@5Fkz1BcK}#PM zzKB+%`i%wX*XYR0y()0&t3K}s3Mf6vX0l=r{ylC)6JIrc(6?dG9vl938Yyb`m@>e( zQE@~vay}Kdp~}D3Jg|8Nnx@Jf;N)a3`;f?SS`QRu&1YahlQ8bDH>K04{WvpTNwKGB zB&%Np@#}djj(udqLtY<+dJhkp<_^Y4OI>d8?8CNRP3Z6}3*Ulb=vS14PWS7uY2#|V zE^uRvU4N00_6a+Vr?8EKxAa%2F;B8H{HCe$lFUAInDrX*f5V}sClEH@oZXJObK?0T zWG6{3h;lO3uemT+=IBQ272?r{*_ho{@{`u}V)SkwG*_O8M`}}r>ysu_36V2z=Wm#) z=)^3~Tn=wG2rZ93Lx(%AbX|A^q5VhmZPo%be{Re=<8f@!`3LfM2BWLg2k*t0bAm$? zJ`1}9ue-y!w3j-abWD-*z?-{glDQW{8Gpl!o&U6eN?tZ=vp!0$%n1aa3t{O|Z7!5| z>fiabu*=&8t(&eqp#2Rlt29}>x)J?`O~tje*4Teovc>`qY3b@ID@<2ji#BPmI25 z#;08(v2=1O+ge}n{_(vCYt5qR@+gihvfz$^I(%3$m?PIN$HzC4Gve2i4pm9wVT1I{ zniS!-wjU>5{D8KnBJoq6JDqaVXy2~`nzU19le%?yIWdTrXL)kYkx;giy1}b@$-S$( zD6ag{rtYXve0*^WRwjGIJ!Q$g?C-@&qYPg8<${{g<2iayDbh9HVP(!@*i_4GipFFZ zj*8{j0Ch#d(|Wjm97j)ir)IZFKLrd>76?Tc~NJ`Hu>^LTHrH@9V9!J%}?u$(!P z>GqNtY(E-KM(x}*ntHeG7bbyfiu`B_k}_%klCIn>U! z6Q8BFP;EOE5C2}qu!bN$8&xhwRH@Uyy%&FO-UrKl`|42WHvo>tcoSP9G)}Y=D(+z%Ic+sOA19#y>ptxC zs1JOD?ug{dbP+VZC2t+Ni^QWkB53$2T!#}^&Haq$<}dLX5~Z3ZE6X&Oegcnf8_INodzc`! z{qh_$9M2@XUHDOb3A0xX#-)eeJnz_plk8@o{m_wk+Vnz~(Q!%5aZAkgE)&mGpDSW+|3UGV8{*zQLyjDN18pxvu+qIhb>#l@)zfFFY2SmU zNhRV>#Q;XV*@?#=-iuyp(UJwIDz)wBSlH^U)bb^3*02W$DGg$JspN_lx$tOxwB*8^ zK>UA)ajMT498#-8`R1eGCtu0Dwh`C9HD`oYbwzIJOQd8@7s3 zmZ{!fvT~X3+>$Z6C!rQ;A${#y+}q5SyCQxf_~w^YdL7E;m)^)+zBun;Lx5 ze~bd5%BXjF^pJdHMUEj}-Msj(7=f zc|1|ph6@G*_{ORUFG!D&o=cf{5u(Zbb6OlbCtCEb&E@gKfm~D-izUK}%d5L`OkXWN zoTtP47gMO$O=d?VQ?5nnV?-QJL&WbS4sw=DAtx8Ki5bZUr4_i+I+eqctR;_4hoJ#m z(D`(8p7uD0)k@*i^lpImWgR+2tirm8d3bRxge@z+!Y4O@bEf*T)zgJ?AJ~>i_u#9# zC;_#zu(7(Jz+#7WgA%|CDt$IruC$xax zOUX0!$&kGAPMEm6PH4Ps#n(wn7=HDfV&7O}M2XvYd#pWQ>F4uruQy`WI}4hYufUEQ z<578d7jzq2Q(tmfBh*|mu)QT7cIt#w$>Q3V@&{k+Bu9KtKUPUC+GcG$w9U3&Ey&Vt<|`Lk=0I6q;n@Mw7%<#N`1b!s&t1Fyp3LL*9tzs1?u(fn4uTRf<1 zh0Yc!yV-N_5)gMu)@^oFn~mcNXdJ)j{d|TM-7w&fno^ zAepVzCot+w2hMgrC6=o#!0LUH0sOuxhXu?<)b_#R>lW$b`eVezyIK6xp#v>-Td_AL zp_P1_7rIORtmP{x-J5~Hjwi*)7nQa0VfhYXtZz` za~t~eXiYo1%EO8qrxoBy8_)umLT7QGhq3)bH!$2rW zzU29(4$L3n!fp0eRF|y(4gZ>9VnJVq{3^lBLkqCTy&jQ?Gf~*;3f4SbA#99VaLkBH zxYlkw_b>OSPTyaYFT~kRI~6&{(VbSBhvE1yjyI(DrDUELue6F4um3aQhDF7S-&c*eJ}(j1OULnm z>lpN{mOp>_er)-S9R1dZZuiI0_U;L^IO51oSF#jwA@i|$ehZd<%*LX5X;7Q0LF3dm z{28i4wR1C2pk+nJXI;3Vzw|pd`m!=z=6(#qq<=ucDY@?as@s8C3mj;+az3sbuE2*D zdqkO@HT7m_!%1d+O68vAlJq{&ocy{#cAh7(Jnko>Z!!5d+{ZajkOb-4S56&t>2QK`D0A{EjPyzM*w zEZG9ZhdB0Ak4OBJTpI7+DO~b$IqI+#^A4nPZ*&^JI*n(Hg%=lQ8KTf>3=co7MA{ll zCic*l91CN%ov>QeY?+BI=Q4Ppr{LaKshscJ14qiOY0>X2!U`s!$RdX)qAC@g^M`Zr zsb|pI9Lx*%<9PYHD)-G_jz=CYtlyN+-SWF0q8N*YY3)!W_j+2dq^4)O9u+dH(eKP^ zIBV(h;E=JrxpbxCe!4GAZ>6Ak*&5`XFM)t6M4=DchYV%xY)u~esZQUtaAa!sV2ooSrY}B*uiJj$ zeZ2~uva3X|jjob^Bt5{h?&I=r1xqd_@z$Fp@%&;h$^YFB1D7K4zE(@__?E$1JDKAQ z*Wz8}K`g8MhLv9SY}eHft2`SqKc*G` z3*3N?j*^S}SZL|Y zAg0R9?ilV{;-$FaWWuwON1*dP1Ga0FSZzLzl>cTv^Mvp%|3v*3wO<2f-j zo6gg(z*h1S@3-pAlCxWJ?MWCjH*{phpM2go?8EgxGdbsdJep3{`~Mxq79Vq2U!8;E z>>FqdYAxA@-_ZJ-0sAe?5G%(fa+#l;(>l*qTwdUai8muzGv$@Y3H^xd$VN1a(x;7N z*B1Nb(#H)czIW@UopH~Vu{;9(eKsIse`BZDW+z?Kpt=KNTW9gAPX z__iOGY3uRS@G6Y_@DBeuS727zF`S#A!&J*F$llwQo7}_cdrQvcK@KcwwGXZDNq+8u zOwN|G)7I;eobtnx-TjxNKq-sEzHWoT)gG+x;E#JRH5j)jjoRZoAldo8!bWWv+$JoM z%-wWxvwI#_Tri_)*f028{Ro2|t5ET9IQ+Y`#Gd0av!DD28Yh29&u$z%Ml_+O-zuSE z(2S+VJ=yK~K)O5Z7s|s^snPTaR!iOHK!Y_Cd$}-5HJ^$C>ES(N$)kz#elt#IsO%3J z9qh(oKc+#uMH)W-nub|#<$K>wa#hxk6OiN}kE465$d=&bbBU$k+oPpAF?w)GRks+tV=k=HI_mkl9`Ck-{ zDw8p{t3O|EkexZwD)92N5eElLCcgP;jC`rjGl?zvTe4&>t&FBa_fgb3?1Bd!u6tiN zyce#ogW11X5}#L_vUpJr&lZ=%dPN3jOAUDEz1QO6o)~&=xC)1x7o|_GHJ#q?gOTe_ zsVTh0{-d5K3T#T7-_cwa_flL5lX*?QQW$u=6zb#2-=5NE*LNlAa=P)$wlHkk>?Y@O znIkI8U`G?FX}f-tSwETG-Fr}EU;Qh#MJ4i(+I(zx9gheLNpvj=t8X)vhQB#io& z%3@bJXE$ENkNzgyddUqp(s~Fx$prUquE{-82e8~%kCq)>=n$*Vq9G>Gxx5?~J#LA( ztIc?(N-|dEtTs%{6YErNirLS1NsS#yzZfX@Ext@TIziE6&|a+9*o)ob95$rNz45PJ z*!)=PdtvgMxce2$Vv-p!_BGy~G+4 z>a^jH#Ru?T(?PU27>m+YU$Ci3GRx$hda+mbbBKD4Q>?qb?!8YxPvE_OT{KreQ}LxaFv|(6#xF>&0U#4yxoJx z{8petxnyc|-3M>o0qj%NhnlMwV6EJVXU-kZLa8NBk=*8Jjc)Xu>5BGa%0zed7&xaG z^2N2s_zF9Qe_JJeX5VrB(jboe=*?w@8&PLckE_kqxpLYL42jfd={7yl-=sapUTQA2 z>s)sHoIs0Ci8QV2$z=abUY>JKlABiwQn~NI34plOAs+&(4fmy9eX`e|NnjpWhN5 zmgi6=bNB{sEqMK6I7Y`erF%vgmqw+q^BQ%kN2XBUVL7gD)aB-a!JOD>K3vCpa)bXu zC>>2_zu6fNvya0824Rf$Pav10j_Z1()x6jmqy;^LPr zSZXvKTlHjCHA=E|Mujo&lpU?NbfR-sFZP~Nf%`8ssJkkeN#~^(bH`G-`!V3y`>`yX z7R-i|In)kyzW zVcnZHY`Q^q390z=N>~*4m?;=PL}nufcjm#)54@XasUoDXT*T*j@O#7(Xs>L`BDZw* z8kWwm|2p&KputQSc-(7QSsM;r?FsXyYE+$EghioUA@<7u++*?tP!H2nB6u{Ucz zDA*^)gN_cxm@`RsT&DbR8YW$0z%7dR!M#zd?nQ5A{=AGa5sUF;aJL-$%}K zt$5nP5+>5)socFcXG@;qbQ^Wv*pbck|CNftI6;d=zOedPi$CimSI*F#O#({b zQ#Vj{1AK&;+bRTqTqIVi97c9~CH{QA7U4BIoE0%${1?%V8krkmbwihBNAl<@GjxaK z--Ach5lqjPoQ#K6=(y`So<&PtV{;qUIedXfhGgg2?LiOsQ*i$k#?fUDM0tbUbF6-b znRYHrsZ~hNfFZ`$&&J|?GWYVz2N&x1;`zMxOn5wiL*yAhqihiKpY6b|jsuw!+!C>F zDs1XrhS8CN7=2e%dvrj3t3Oc7{48^>133PM3$vd~Jt`%Yx5v-NyK!mEl-*%}cd2m1 zVM7`O_vN9pTG%fakmtImu#?o4+v|r>uM?3sR)Le7vv8>5wc=f~<0$aFhnJ~d)Rf+) zGMU3~{P_sc50%B>n$g&PJq)kTZO1?-TYivyB-OsAXjq^u*6+_@{3%}?8X1ACGsX-~ z>xlE;T5^KE3e*D~=zeG~j)Y5YZbUFYURPuGr!fdyGoCw~exOsl>|i<+!utF~xE?QL zrG6Z*ZaD}a?_=WW#SAfQNG_YF9Yg(}0Sr=cq;gCYog>GK=O?N~dGZ)Jhu#-4jgnum zUWwXQHliv>cIkBMjduxJtaxS2W+fRs7*o*IF82jS3|}cGUvJL3BRjEC?*xqce-v&N zu1G4pgrR5A2^V^u#?QTbFr_?zryixzcU2Qkl`~q2%n<#Se)u(U zeQ_!23@o%2?DxosZccp}(DnkjxB-d9eYta#16S)P^JTL*7Wn$&=2tzA?x(>SvAy}? zX#pGmXi+`3Eync@W#-nF+#H_77fYOZ^Qj7de3_5_1&R`ThhrIg}teE(-OJ0vO&X=yrGjPW0)|DeL3#=*Dt*ns*duwrs)| zcj?b7?#@}&wK#O*xgz3Fyr|wk3-_;&qjRZvwn>Y?W}3|K+e!h{pq;do{izD2;F4QcO}v{{=i;5 zYdwVX-oJuhkq%A#WuEzAYqsd^!xv%a@%8!xWag72^heTGyBkfa!dSiC7-_BD+3Cm_ z4%IxPxS%mz#QR=>;r<60(Jw*VTH=b>zU6TH)D~vnBvV5%fW6d5v*j;c>ik)NkTEW_ z(Z7pd{n|0lFNc;pLOIw&i5=B-WWPczwwyHOba-%C<0xca%HgM7sm#(fL}K}1mi(W) z)JsKb6rQ}{I3G`%AIG>I9dYQ&Q}HgZ6?UWs@`2qGJgR;R&znKG{W6TBJY9H7N0m34 zx8|T3y_h%L5$`>V;FLF2l1$UZI1y>Xgi{ed|zqZa*qrW}@=J6ufi~Vyl8Naz=S1k~AaOT)uY) z9vnd1Ro&RPUzO;WxfyrE&6qPrYL36&hp%(5;b3QHZi|%u(8y5k|J#{yDqE1N>dAl}s0f=a%F4lR~;S@5qabw~Fx>98swE z8vSETS)eVwq}}ph9+53M?{|>v(~iG9R$!t2aX6J+g|qZ>#XO9{ckxQ{I%nl zP79Ez)fVRQlJ9m@_IrGLC!+HUxJ|x;vz;4-eO4eZq^U}-;04&!Nq?DAA=~YbN8)8w zHp$MzlwQviX**hReVY=IR-Y-E_MtHU*^;kYwW2}dNZvGzV9lOT+?d_^~dZv1T{LxbuPHW>h4P+tp(0`f@}M(5IX7Xu97CqIc3! zXq=fWGMj|ZCc&HC4qU-quNfFvcN}wGccz8P1t@|KDE2r11h=$qjG2G=P~*BepE&iS!Kmfp`CEU< zbWCLbaVa$Ls7HOc%;YXBM5N*>-1DLo3wv#ZzS#{?TAR#>d3%rpju?UliBSb6^QG$OPJaF(&4=hvfhoB+&4Rn_gN(# zT^P(+JDy_4<^B+bq}7#XNWa^J%O%(Q;hMoh^NnQC1y00ix1MyW%0{3?4C8K`6jQSl z+;wX*WY9u9ORK<}{mxt`vbm_=1>t6YRk7wx2rfDo!{0F8C%sn zLHVpzaJwLPhh4Yf$D^C#h3;%9%b+2N6_b#iDHM-hxt-3PX%o@z@ZBL4S%Q_+`WEj)eYBShfoh$4UnN+<1vux}b z@AC(-^G>5}$T;!sr5+1mfvq!M;?!7O_WA4sx5dR+r+5W}?vlm3(v26>Wgn%S4@{pZ zaqHM5MTdu)!dX-LG}hE$-o~BKmyDm+j|Vb%ZxWV$mYRb~GP@mqFZ#ARiO8yPxDlJk zynF-r{+fd*WAMwQJnqf-hS~Oo>=e2g8yof$-#ay4S~p+T78(UxIvwP=3D3FGCNc4q4s?yvkmv-cok20wvWha@&t z-HY0mk-S!yjp1vy3*&lKl*2nOlho$dv!goyGp2VO7>ec-@rUcsnG2shJrKSKIPcH$x8D-JiwA zGP^cEoEuO6gPUhp4m9>;K`zf@u0=bP#70PW1eJ3?h9L&)Ye0 zN8$^qgZ1L=0nOPoGaQw+-qJ63S!_R}BK+l^JmQV~*~#`Gr8c|eaRwqKH)XMB7@w`~ z%k$k+a6y`$Gy1pW>4EK`*0orqIjAFa*C)hTUd0c)0Zd7>WOzb2Cm&cZ0)4FcZJ7n% zBwZ81FTvAI58+pnEx7Q04Z7sL!*LfEtO&XSwd1Rir;xo;m*h9=umNA=y3n{yZ;s;` z(fRim@#In$%5$>o>mbZL}IIv-_#`jNR}PZmNkqCb~Wn{w)niKwu*@xZ}1$Z#|e?E(Q9c6 zPbtmwHk+uyJ_}Q+*(;jU{Bt=YcN1!T)bYZ#7QqjPQe(|cnIW~o<+y>|rrZ>6qs^)Q z{RFD?I`Hu9J7TTvThva-69cU$rgt&_k#9^E>w$@{GZDtZJhYb$x`uZpc9w47Gc`Z zOpNQK!?sU#IR9b-o$rHZm&jgCd)Z%kM43*uJ^6REJvYQ9GV`4;*DaVJqGT>k|G3oW z*Nx$W8-6@sc@G&g2SfurUD2$EcqJr7jq+{6^SL)G}?S1cD!_=A9vkFecyDgGPW z7r$i=cjx(sqC=QEPm0yBGSGtP5>3N;FCY=9uoQ0Mx!f_zJ zBbR5&-R%qmDpxfMMcOznpB0KNcAgwO<{l!`7eliNd9s%g2X^1@-67;FA_GQIcWO8e zt`3qKX#>{1o`n6EHo`0=i&3wYId4NYU%#jkV+(buxm}4%!#+xOyb;RPyWrH7JSx9$ z#u($)GUNLNkt1H>%&eQZ>nG04CPF8sK{I zc||aLox*~8GMtL!EFU59lV6^cb!>ImLE2oFmtG4-_xDt}7;=TtMc9w57Hy&+Eb zS|IY>o3K_(_A`3-W5S$U*jQ}M>Do%%e>+q%l-4SqNw5C)f$K$Nx*5)%l-b2`9`wAs z5qk3M&zISe<^LtHlYwLwq~**0=qknVvlVdv(1L&Zwx))8EF%qz(Xb(%?NypFb%5%t6fz|qf(5TJDfIA!U-6)UtD?{WvCYN!&w8Wy*)mS@tJm)80 zgwyh&boElBitMh95AVyk7Wy3Sq|Lw26L|5Sz*W%suLrSX$nls#!@4N|E}h?qal&I?Sg3lu<|sy$SY5JzWVdVf)>0S+C}=^ zW@An45s_;iLlwseDA!+6yqOx0@#8viTqW6FGmOvM%|rHYWlX)-R&l9kB)0cXLHBib z9I>MY{h-b+0$%{mRcCOYid-2;EJ^0-jW zO00=u+elwV9aTo#4#kKs_vYGLX)N--jB0yL_I%WoUzP{4r+KD4|0Ksi_J#HemNLwU zGVx`JlJIsb7CFO|ak$1q)Qq*^oifQQ-aP~9(Npo~yf}t71aiOde&t`gZQ=cj<{ac5lxx0ksOv#iCHDKSiU76HO zjYli5!EMP1PCoGpBU?W}*}__ZKh5ao;lR!xKjKns7#+QSh~S5QxZ1Wm=i6(E{pQ(R zXW505$9JMdi0m-EHyzf~9;0YQ6dl85?@oyyt8PgC^XBg`YTAXlnT9;<>%a%<(-gA| z0$7tFdG40Md}Hd3BC8xu?<09Jd)0AaYinM~iWi6VB)g}=K=>IgK;JpB+}zC#cGU;* zQ`m6o^)>kVT(X#VMDfNhTd`|^GyBU7lJ?kC{#%gFfpa_Hc4xBrj%YspU?mxl>D05H zB=i$z;;zpTJo-=O55G&#$yXTo`-?lk!tVpH$3J*qgTZpv{zMS|% zW{>}Mr?O<{dN;J@?I-;dX3xB^Iwh9dWhU)Dh#Jp!Qe%@fNlsH=**WN zo}jm7ce?hFXSKV`P)^X`&2SmrZk&zjnmzeGRl())Gw`^(J<8EpIFI;&+8#2KV9=Wr zZ%co}6RAT^%%H{rp}3G}W%AgZ?yL7Ur7 z{PuheY&(3wgU!P@FUXAV@7Icb;YKtju>OA>odsAH+ZKjFQ3(qh3q-|41u^+&Efo<3 z12M4`EbIUUQS3Z+f{NYUiG|(W-QAt{-S<4_z2CRi{G4--%wBuV{4;x>Pm=iF{nhEP zn~d}9s$H)P<{hv%lEhwOlWG-G=Mi13MlYh-%j>F@zJ`z5ySlRuUv{mWYq!KZe3YZB z=Sky^Wo2)2R++_`8G}OZ$>+(Qnssg-{hOw@w(IAsSNC<4(%M26I#trxBWZPDrx$YY z)*wB_I!$w<)2iE#Sgn$LS-yWhZ`D51h4=L0G}bdfrW_h0FGil02X1?%D9^o@?%ZF8 z)yXI^yIbm{-)*#zbFxew9wC~P5j<9WNmXN zeU&!FI^V!v5Bzn~3EBQg+Tn8r2N87euVU5$VZ)EcjF|41IwZR0r_{c|HZ#9-@ z!&swxWlMcOI7UmKUM8DuUb@3=np}L`M@xUTY1qhR(l6^ZSr<`K9V+J0JM_ix&zIK} ztDM#^9;2n|&laE6t~zYyOnH%dS2mY^E_GLz*JqvXiR-Jf{wGWAlRDJ!)mh(LZ=2tc zH!m~mfEnH8IM3ZawpQ7keVggGJip~l1D?a3SyZ#uxglvT%#lieIJdg7AD`{K)#O)y z);i3r%hJ!31DOg~4qIC4ws})+4=>f$?0GV2OvO7^kvHu$!o7v08#X~&t@|qVC+85S zZYA|)wdqnTva@V;>@H{HOUOJwZ%te9plr{Upplt!>4#^c1^y&U^4DFG(lrgr&yLXtS=SFc`Rh3;~c4ma-GdTo!@vpKr!-tzuB_w+u=R*&cW zE;iC+o?CtQHjQ?h!*`qw(`#a`IQdrOk{s-HK$_+sXn8ECDVe6tu=+1cmQ(GEYD1?6 zy6U}yek^*L=l<7AQ-^W3QpxQt?bX|=$J!vZt~AsA6+haRlwm)vAl4unc)(UMHIMCl zrGEN_cPBoj1o2&=liD1;wdtQix<4VKdUy8GIsRVyx=oC}NLHO+bgb22Pknuvmi1xB zjuyX@H>6u=d##Y0wQhq;>*)%aq)Bsk-L@f2?;Po?J!1-MI7Y~AcCvnqX{&oCwbm=qF1pq;&~~C_Nezx&B~yMcmLk3nWOz^yb*nYN znp)z#wK(;ke@n+Pl9TT?Ix>3^boO+qcBi{Mj+-q>2iMAtFpp5 zBhkQ;RoPFyvW5*?VAVavTFBkYYp(Unzr}hVA6y*+ltwKU^xNsIDv7SBH4z)K>jks(;%dI-2LyzVkfzznl-G z)8#Ezcso0p`SP1JbhLvy)MOve+tu~avgR7_C{oTGsIDu|&a}2(46)9&9HJ4SU*%Hr z7J2Z8buVw;mz@(!>#3#OugnswGj`^em!XrSM9Ns}Z>_(wZex(f+jr4A{VQpe>;1$Q zEYfbs0`c$EO=smPp!ZMzXBDq>T+Y4sXaBcgO}W)pQvC+W=#|@98<71<*tgGBMoGrv zQQA7Mk8YhaK&;Y7Y+XO~(&^4QtmfTAbn}!V&fS(EbObcxzNPHJ}OBWvTT=$!L$`e=R)t>ku2j`NI8pO1HCS8Q)>@V1!c^RbLeX5o1MM3L&v4*;ONEW@kvyVPlyicwLx6qx< zS&yoDyo^2fP5PAjZdd4EXO-QVQDcKcbX3l(^5Sv( zRoRQU%A)+5ddXfR-`=+#jxMFY&U$LUwDmM3e_eHq+agi59JE)Xw)(+klXQ<=XQlA2 zV)4g=W!?tXOde@31G`_BX)Ov%mCm`eHv2hT-TKoUB#S8*hqfm1>EHB!cyW@62kZGpBK1&BmVJWB(IB_sL>6?5>>cltZ1@*VCToY`T3{ zVeQ)#3t>jfrD>>ei^U>+`)NgZd+E>CF zus*4O4Goed*6pQ%5}!XwvLEN(!kQ>+?X){q)x=}gq|1())2@!zk7}Vi-cOP=_w1x} zgMsvb)5W%Fy}bGPM2?Tht3#5VG}ow%vf@Mr9XTdmdQJ?JUQ7SUt0d8`sSUKqa}SMi znNW$06mjy3l)bCnPX~Wv3&QD!6!-!mZW>LIubK^bETkSO0 z{8#e%-wQc2=!!h=)=5T;zH6n)6d}WwuakNO!ZfsDoKBfNP8#G&Bgc2n;d=F@IQCAf z6W`R8 zWO@7=q&sdewiS&mp|RmrB&AYSS)Y#m;yhEOeP|oa+0RRwmQT0}x1bWS#n z+#=QEZi%ul=J|c)wfa-`cTJioc}}uF4fi&#KiR{c!&&rAxwhKzpQ}bhJLw6RaLe!B zZ)=j>6=g^YlHLS9|yt#TrK213$?Msc2w{i5wP zi9^jzysy+)@9fT^bL{dYKxv8k++S06 zG*Rcf>;uSmB8A_T)3fX5$=|yTt%nt~C3re>$KKayVO z)ytxb!vi%>*+6Y)pCrxymXVKJ%q*bDB(gp1s4q9LXF9^<382CTL9Zn#699DinA8Q(-thX-kfs35KHn#1<^o4@?GZj$ZO zpkVEot+iC1Gg8vOYHmAO*-m3KR+hOl0;PD!A9-=7w`9rJO)eg>*U?1_YtpX>ol=i| z`?`c^jnDVxeX$Y}pX{PvJ(kOf-d>vGT1jfY?WsW@rc#WCAPDabn5S8lGhRSIy{UG-N;XwJsk#$&L)iAm4%NC#z%ZwIZ- zJ<$GhM_P65SabKt39&UVtNE9QX|CP*bVsFXde3vL?c}`Ua&`kfU7h^wDZ}^-WbL+; zX6hTdPzpH(>EbU%b?&|nV#`@r$5*!3czQ3l&37drf;CtJy6C6jjb(S4R#lUal+ce4 zt4fP!L*!7_MmjHQti&vBtI0bCYyKohO}Nb-u{Bpqa@#Kcb9+?Q!K^-UXA{qCluDJQ z6=sTGnO!peN`33)_7Bp3XFkn0kM{}Iu=nh&o|0IjxPE(jM8=JHCu>-*$$5JZS(Brs zhL+E%y9*DM;sMtseRvD?pU_kTf*t?M3@*Mzfn*7wvo)|gG5WyQlq%>ViIYnN0>9P6W_BIn4VNiAgc$JSQ9?E$(z z++Ay|<-g0fw{858vGQ^b>l9^pEMGzk$^86}tpS~~Y2Dw&WX;$vnwYtSHpiE3?)N}0kGmlw!_UdxQ|)ER@@VaMj5QF3owh1g2+%W)o3Tgx zP-z=dSdWyet99wOf)07Q?-Vk%jm=&odY%c<73I!s+9k^S}$!+ zLU>=fSr_e6dV{P@%CF;BM%Y5rhid)6kJ6)}k8~{jM)n=eFBN{))&q^AHTFNQ!yQ=r zW|xOvTNkcbPI_qj$^|uSb1$8?!C!Y2?4wSTwpz2lyI2E0vsUU5d+k2zgB5e>wb+N; zv)%kJMBAixkz$$i$r1aKIy_B!y?E)K+oHKxibzi0kETNcujSGGxVR1ayow6SJ;(@}3#yD420w@KCf_Us){R4cK*`pPz) zS-j=Qa7IRWEBh{qeD^ zmVU-}qP|lks%Er0mOEk#Qe7<^7os+A|GA zMD@a=d)6Ap>dJaK^^8v?T^ddKbnw=#h5ul}s6uPp{olUh-38a&z8 z79ZVI^MpN=A7#Duah=PO{%w72-D?2PK)#X#&XeR!KR2y;X1HugNV0q;q*!5H2dVdn zf^y?xUMbPJv;+k`VSPv^_9U&Zb2bgp0!=Gx(x^>Rpj|Pk#`=IATYs^FJJpp!2cFml zee%)I54-8zjSpp7!}F4J)^b_CDoo>t{*?G*on&cb9iH*)C+^gROK(}HfHlc`dU|Wo zv|Y5_2t&)n9lUyLbVoVlob1JhoeCDJ54wllzjx97@ z=_u{gs1xg=glO}Bef6Wi$hSF1#DDH-IqLFQMg-*MorM>&Fr#XM+gsbc^8IvlfA-mS zaF=@}SBhssLwzGBZ1V>`fi*U#Lv!IkdT zfX*MSmaLW5s2b0f9^$^Ge<>~6XomECuwS|!Js|_)YwNX^m!%``kuEsvYF#{$S5BtP zlt243=(5S=eP{dY&&&&?@WpE~k@p?5 zEgUg{@*^TlYhS$XBi zkTUu;!v(o;H&kn-_R)Ipwo1WOnRVOY*81wGonD+;S6gTOE+?ZRt{kw{AUFxw$)a?RrEr-l~OhGtWbB0sG4***1O zrVLUeAJ35I>S6WG(Lz$Zx@+5Z;X30I&s2vE()pW*NdR-lzVQWR_H_>_K0kx*n~_yR zT5OU+bvjAGqh+j+LPuo9h3VFp@p<&>e+6`Gvu0{vyPEa1@lwkzcC)SC!jk{r3-@Z@ zF!fn}ONx3`Ri_r+bnH6zG`sDlOP8}&HP16_Tv1XRrL@$5SM~MRyW+BLS6wahxPvUr z@Q}TP7E4r21?}h^uOmax%b%r#bj!DeR+`{@HtY0r*;?(Dl~{!L%Jyf}V(ByMtWavl zqByOVD?9HLrjdkc?X=3ztzuouq*MLQ%iIt3)V=F@ah_S*_PmB_@b;Q|VEOw5_B<)j=#Tu1Y_IbUuy??ztd?tFq;8%a$M3(K_T#?D)e_N`Q{@C5 z`L(0?ykITDb$(hnc!>O&%zO3j#WmBUO}2$ke@n_nOCJQ4wB|DQpUtt?l3e#UDqT{& zzXoYxkFuJZl}Mg-dM@YN$5>m_4%8w=wu=4P>{30bi1wb;O4pwd9bRU)cs(koz23&_ z5w}3AjcZ!Hd+oYR7|i?fVH@Q7kG$Hbot;iNHC~!7DJ=2n)2f4aj9xi>Q;s~#pb^hr z$dWVpb#7JEFd1`5mZh z&rX+!K6|8W|1|og$O0?hx<=x2xvDm3FhC>rRMP>Oy*1tL*3xHVZE2fjpKVdoE7EDt zOj*(|K|chH75BB#a>p~D4hxD=C%3nv#RjQE>N?33_CcyY4U|c*CdfSJ2(8vERc_8G zVr5}XfRzg_$?CxqrN_UcGHzQ%b^q2{$M9^!rL2GCO19H-{UUqr)M2fo3k9U-ob>vl zRHEfL;JF+cUs30z3AQp+oNMhYmtuK6?W>J!QMxSHO}kca!MlQOqyhK;3;EuX;9D8> zKp!u?`N&=;$JW(%DU;+u^Fb2RD!b*e`I1%Mp{u@FpGjMXWYN55(`n%L%rYUduFi7Y zXxsX7g8Xa}t1G&Hk{a2$=g*ookrN7Oy+pphv1jk`9lIov_ZD*v+hEmtFh;IUNTYEM z!zC*9t8JO&*PtqX`t~dPxTm=2>W)1Bnr@k`=#(iEdAGbin3G0RZ{L#r7hE;lfuqv= zm+JkuA8py^57HOGLwI+My&P8Yy?Ew8t#YuFF4)RD>@OqqVBvVRs`j>AzV+2t#aMs+ zXNYyM*G)+?>#fwx++5PI4$~KUly<+|^>Vko8doPuqC0Z`#bbrk9&M8?v3F(Yx&|5& zKGh1_)=byD7%KBd^0n=ieU!PN@Ls4pkv-?Cr{ zuHq>P#T~SCqY1KoL|u)^mREw3`sgN)DBW1gQ*(3~to988q)Mf~l85{A@8+;BPwx{l zbzF08mc5W(`yQx;+P;_HHSb8a0abLnHBtKW4F7R&Kl!g0>$m6eVei(gGCa*ftNi;= z-j{2t$0M#=`FT(E;nv~ug7>(iT{6q|qGheA>>;%|V>cbqc(r8DJY1emeJRJ@)KiZP z(UYIIDa`?K}X>V83*&&ylCmS#OVN1kmeH$qzWt*j>&SF_EHi;&u-E2{GqJ6rB; zOJu`F)>B=`GdOEi4~*ei=^JNda%?aC>Xch&z8)&}o#xBan*(&^_13I8{!#KT_tqOd z^Xb=`S@ie8a4Y@3oi^8TRWxb)IVrl5^&lG$m2y01_&sQ^E#IhQ`4v&rs@|LL*fv+Q zJqpdMqsI=^S>;$aBx_lDu=TC@JUC-L9y(7>U$>XUnEbW|JQo~Wexn4=8Y=sCch%}U zx?5JMhVshcvYcf7gloSms9nJka?T zO|?y~L-M2XMS0+u(?6MaCjR-Ik@I^7t80prI!vjpj~XAgvh6%6qvy5JF1hDf`3l_; zw}KsY#rX&FecEr?UErtWK9E<7KiO_2&yUpPjeeZ(q5ALOHE9ugPV&39RENO9I)-OA zr=1lYEiQ9m#2a=>n?sM{_lbzalmgPqA12X;lY1vM%HmrAfS?|}w zN?#&L;);BbZ;y*<9i1!jSNrLiLVK)|JH}gU9qOr1Tz(y<4u;l)~wOq;_13T-mGb)?JV~0iqENkYc|);yzk+BI93C$#%a35aPjKiSf{df zX{*sbn)_G|?yW7c&UHQ}Uvsd|=IK?kr>9Ez#OIQe`@AEYy^%@lPDs1*rz9+YZ(YZ{ zKIPj{Iqw>$cSly&7HQnIMdEuI@vf+JIrK@UCk@nwEjwC0|FqUR)jI2pCbiU;=dHc% z4obzQ1EfQd%l^3w{Is4Om?t?KW>Bg=d)|zYFUy`uzSLK?#I3AZ{4l!)`n9q&b4A^D z=%tMOKAk#vT{2`U!ZQeUG_>bOagBc<+wHh7GyFfvP>=Qf?**!FjcwBC`#D(>SX>{@ z<9&|yT{J^l)+}DoQc|AVtS&AmB=3a@;>8&1S~#2ZbgAY4Gwrn?DzA8i>@6M|aQ~OC^%W$h5&#BDKSX39Yj!(y~j#_Sqn~Yc# zW3$-?>3>Jq%Pi8-O8=7mc?Z3bB@1q{#?DAal>rMyFBS-sz;RS%!B z9$o9EFT)0?oPTGnTfp<=tegI}Og*h^mr+k|5d79Kp6G93} zud~O*<3MAXT8}kU_Ehw=a93kmhNX*)8OlP8O-`yq3>Lz6>Ez`gcs8U|9gmcy ztP2xR-^wxiqiqQLIjpSxPU?5dA{E}$VO{NTZPqzLW$gn=Vf?jW9iXNE9?OSR(QCEq z+U8xaA;(80sOO;!I%`t_9qv{}cDp87F~joeAKrgloRd8-iZ+s-C+A!50+w2dV{Xdp zIwf`UlwtC->L}T>`kQ=teZ%VM-AG#P8*I6>IV!Vb+v@VZ4K=uTh_(z4)zPUHwO}Sc z=`n7!b)f4fnYQqQG&wp+p3X=o86Ws-BJWtcEE*)&I+T_#<65W}?@6RmQ(7!MDWx|0 zY9QZdO(z9Ul;Tx@0w*Rn-jJs!Ru%f^A zdpJVg9B|Y_8H=;NqpyC;IHv0B*`DHeJyF_k|I53!ytDbdw)M3@TPc{ifnJ*%tx*qi z>+C+iqyX>I1ZPgRmfzwUqt8~Ub*PK}-jG}W^L;1o>pN?5V+Z-{Szb0iJ1HCPypo*X zZPM=hY}>Nqx248N?mH#al}9PdzWrZ4Z~#JCiNleY%6Tpbu!Vs)v?*vq4r-V`BPt)!7XT_{XZ9Mi$Dg8=h~L z$P?@x(Q<}d?h~x{?OLg8A=YvpGfAddi{y#~YqoxY`Xe>Hx;5^keSPfp&j(MP+t1Io z?$}7HL#Y%={9a9l2E^;}**0~_#@Z-_8tS-b?XBtqrb$q0JDsuGSLav!D4U8Dl0HLT zTBCdm>EJ(UblimeI&MR8`N(|h75AC7Mdw-BEBuyAqwYu(o^6@=JW?9BbJb#}s!EkQ zRai#O(px)oY1pT%Iw9h#)q*;CF1M%p77W!PznbfXu#M8zt+hT|T1;oSu(ws$g|eJy zg|_@XDld=jmS?=9@cr}xasQq{zkjFZt(qfip7`lQzUSMpu!#)F@Kt*Lt|&AAaQ;{B zq`&`F&|Q(kWlhz8lI{CjN%*h5+-SqRvhA687xvJ`owDfbVzqQm3HsWgX#JJb(jPp7 zb+&aKeOR!Y=C4-JR>Q+xPk6Y=yw#ri^yoA>;r-kiI5&=U6C3D!w|&-K)*22^$)N-8 zchJqXZp-qR*|M;Ni=;&`}Bp^5^_p;a9oGOKC!lS!VVlw-ZO_?gAPcD=fA7j|VwmZ*Z7Gd6Z(G!^_ z%C^A^xexEG{ciP<)k*I9Y0oF|w|CUz+ZwA=-hA3VWWUw4<|~=eDojU4`RLihoz#!- zrzVVBC*679cN*{7XAgKS^X)sxmfMNqQgw{`dWo9{Q_1YaY(Ph5m-5jXZ zN=}!a#hyyrE-m%in2}chbzQAvi>qse=&$mt@detR!Z`M_e4ANEPsFfJ_vBI%*|MR&E&f)f@l4R& zq1p9$)@`=t^(u<(%Nc7I>$Z9ZH&K^;S7ej7Xpi+bt!&vR*xD_=BuBaxk`L@j^0&o9 ziM;bx;u5CF$mC77xqm~{Y54$2?!dDMxo25tb9J>9{K?wSIa%ZL&^H;ycOkXfwb2)& z23q%0uE_QaSv4TRODd+lCRH!z)Z{j-1-!F_E-krCPQ`?1;*hNDFI`gyMCZ_!JS!Rc zS>^XlJ83wfzkJ>LoqO!|>bv@tB+tGp*PMH5THXQP9QRdrw;rTl$`7-RIx@j3#B)>Y z4`z@|v)4-AH~tzImQe#9dTZequcT_nqf+77dn?<-Pu7i@cVyNgo7UeuSH|6%VQbU1 zg7yAh9c_``Pt$c}Z?fy1)VJFF9-%C0$;Rwk9`^lA{gtX!x%fnfKdE0{`~X5>DJlw6nAO zX>Vuu7A>8|&h8?9Hnbwn3hALNxWQ~lWxo>Zv>C<;pj>iJ30Gyn#HKRkd% z^a2crtKi1J*9#rZF%L*X{0S{Z?g_ddT@KMuoY)y6@pI7Cs2`s<|KTT*!%zUCpaX=F zAIfJ!(XGVI@u|=ZZ;xI^9XM76-V#?rU7#N1Bd*8s1?XwCD!K(th9dYve5NeAhqx#{ z2YxMdhVA6)LJz3Gv24(hd?wgWE;CvRO$*KNv(Qs$FSHJyU5Hs;&%}jbJGtlJ21ALx}t|Gb)En|)Yf7|7OBnaTM5oiLq z;@@%bEB+=-gZ$t@{vP>(FpqdVT87+6d`A2^2!Y?w8XP&6%CRhvfw&18g9^F?0y(}5 zje`_uOndiqmg z@bPF?j;}{|64yt^!eAIgY=a$eiue{>!#99RP>Y}Y4}C^{8k!LeKu^FLSWVm$GDBZD zLasXO#*aYTq3QW&bVfZS@=Fh~`m+WX+v>Mcd@5C2jCYXLG9r^-l;!O=$z-JzV`OEw+&A4oePlL8cr^9L3 z0_NN>bAt<*^Uc&!Ge3loUqw6;*5O;AQ=tU@Eu_Uahgo>jLnYwN*hmC-ctCzVniVy5 zV*x%lJmyf!GN(wI~KPwRbx@3att?@R?E56P$#6crzy4pf$0nF=l+- zCN}d*7IY)p2R!i{{(qln>eC@&Gv3WuHDfO$Isl5}m%|pkKqq{0)YLW)C{Ju^Q)MVi z+zT?`T~SkybMh~FgH9u7#(70e_M=HR?IK0nWfej-{ez ze!7nzgEw`?2VWF?@v&%A^qz^~5~Snvj^wW6+oOKyY4jXw=4e;cpSUmD3Xb7RK`wA3 zE(WGAu+V&P8%zzm1m?OS8Vx6R5dDTaqx*or|9{Pu1ua8x1NDFoaE*8~x)wc*n*JgJ z{}O)@(hv_N-j6r)nGZe~Um8Cg{|ihlehuxQGx2!T^hsConZW!XFnxJ#bSa@T90Lbp zQ#U`tFua+|YQbM()5m1Me}R4YqG&K|hfw0J&;*|eoekIEEpZFfT=$&9&jB+pokRbF z*-hwW8GZ&17r%;gGelT<2Q*=IB8FQ`YfH&7} zD~QdBT@H!GPl3Bei5(M81L!1QE( zXdLt;pBqj=TX+Z0$j^c%_|7m2BB2``gz_BQiN-?^J{|(_6Hza;GHUwN^XNJ#hwn;$ z9+c|2PMdvz9k+0AwD~5 zdgq(?#`s)N3wjb?MPH%s(ZlErs00(>Gr3Q&27bbCav9M!a0bp2k3lP-x6lY!3&H%m zo}op^&jZuv_C}kL+Xnye>(L7^4ZjS1fL;L8OPfCGCb>6Ij$C!HCwGOo8s7AwBk`uc zi$fcb8;p)e8^d0_sfFhM<6(G;_e9Nqb9pd-rq|IqP#w%TH}%ZSk>=0A%vG6DQ@>0N zHs@bEj+=AXoI|EZFn^gEnh#pwGosDFoD=3eJ%O6vp{bY6Q8Px(*w2KTIyVhW4KwFj zCe-vkbJ1DErk^uozZ9{lA!d$mKz=6P^fT*GGf&hZXZ}phxHmP{)H^e#4!{?3E@0~1 zTh!DEQ$yQ8ar_T3^Y&|W5t#aB#)LUv20=wIb=LGVwNNu|)1rxB<`y$1PT|ct^ak=1 zn{(ZqTbWVw`^|z*K+XAR#+5lITA{trGVm5}#$I83UDWh^?eVR_)E6^W%=r`x3%~=+ z`MZStH}J-r+FKCJn0o{(ArG-RS4=;nXbd@1PygU^qP0;oCQL8=08Jrh>dJ9g2-^(g z&0K2E^#Ec|FmqRb;+Nn}Y{s7%mu4*BOO=hdJcZ!z#QPdwapu zr5NHvV8-qSFn#=F;t6;&KA+%wpr&`2jt|F=M*E|C&@7M*Oh0Vq$0GP-j%`PGLKHp& z`5dSh`iop|m=DXyoq=xnS1=fV1q4nLo8ERSYJYNnGeG7 z`N0()^0_tWAT$HI6deG^AtO8|HxkVO(J+G8%q^eL$Hb>m)6bdS;6AaLzaNw9j_(8u z@bh3X-t?EIKb%T@ir5D_<7c3~(3kKI%v^6yfFQ)5xEMMN;_<~{G;AbxK#xErxCaU3 z+M|1*AXEoOa^_mHI&{TfCYKLwiVjA%KvAeiY_5fSqh>Dt53K?>U?T{hnT7g60eoq6 zA6gE~^{MG4O~3Mn*z{w$Io=mu5SJouf?hI zego5An4Y8+zAt%CFz4rK^aq>(Q;Q#wGv}A7#dBc@n0cWw$ILmlo!HbGGk0Ev!gw?8 z?x8)2&F|Wrry+P#2hCVA=bY)4OuuGo!ETP5bIQySgNT=+h0uX;0lx)G;m!GF{ydw5 z84vx)wFYluQ>WjdnbB&fnGgD-<-wnvIX}kZA3!F&=~>MAy#UO)Z^nKjv=5m1D=o1Z z`@6u5<0ZuD&{|L&{~OkWsa?0wcH|#`>E&h-d!WaOUGb(ynVz5tdL4Yo&qJTXZTu2g z2qTHjc<+hkhnkR;d|EJbV-GO%(>CB^|DSX2;+??M<0s^D;1|P6I0~j7bU-Jg;i##r zXYpszQmE-ylhOT<1u~L90A?=BOnegG2Q~F&2c#!HgpNhWq2?Uk1g6itgEj;g@^w)& zPNQKfnA&gV6;eDVo)B`i7|DYSt!{`q5DVRD_k$6A8 zE1Cpm{xSWdsXwO1jp3LXrylSeAAqJOzXFW_Q&Zle4^cD52S7nE^JxyqfUgavJ}pM8 zpwrO_&=&s+HMKDhCZw$Gr_=BjxWia#aUzmuuK~vZVE8r%KfcqSef-_+1jvF~s$K4?dJ|CF6IfQr& zyn!-cM=lvmUsx6$O=>Tsf|(=Cytx|xlViQm(l8u<2c3g{fPZ*%y<_IO>%?E6KDdGx zRD#D~>gqZO#GAhP8h#Jn%y-A}S)m0q0MpA?KusNP#&I)GOoM)4=8=yuj(m0WHJS>y z;0y6HFn=|IMNpJ{4fqS)iPxje!Stx6?&l+C`lcV`zQQzkOwL?ywa2f8;!uvH_5;k$4#-;aj8U(I~i!KZTA(&77MFEk;}q67bp4k6`+u9K^Z7 z%$?@i@hN^AOyYQMC`!(YxD_0Pn&iUJTHua9f|}ml0qsTn8NGpaLW5x#m_9rbeF1GC z7spGJ8-~^(z6rbWPUM>6x8M&$INl$M!c5{lXlK;)x2AXAN<0-d!Um{8&Ri=s#CPR$ zz0sH81|cvMf;fH*^+L<@neSkF2h%UUCqIwaOfcsV(>oVIFOV~Ra!xdecqIA{a^Z`Z z&*06qs=4mlMD8j~C6^s7jTVM3_`GON)bvnoQ3v8MD1^TP9q|)jG?@D_rXLziKA!jz zItn82%fSN_0?Czy@p#jhEkbR?)zDG!3(Pf1e{>4@Cg^5Jk9R~Lp{7>~#mC`ipw)>N zpuJ&=iD3Xdff9VCIL9mF&%ta+N3H_e3i`nt;xuRic;HRH)gKLjCy7U-9rVlgo ztr=U+#E0Mx)PisDgU|m&PoNjkzNncSo$$%%BeWoz9^HYKhX)V~li?+K(>q;3>w@VO z9;2quImGenU}|!8)buYt_?|G!#Ar2h9L9s0E4PzdhjxY-m<>b7Wre)>rO?{M=w8%} z3Bj*KcfwJ;sZXY6O$0OFnDJre)O5rH!StA(#AXcThlkJ=(n1H0|3;JGF5V7KfSDUs z5Snqi7c6q#=wB#>H}k0JIWH5N{%-@h2KX9iHDXgo2jfj``3k#BIJ3%&LAJoigIq( zn+xHvo}3q26!nJ3_(kvm-xg})pP{C|se+FHPvRFa37SKCa`A8*OpUCF2J(3`caI~^ z1?E~NH+m2L$A!5qgA!rQCwC5i6g9osar|reOgs`@1zjK&%-mRj{B5`nrpM?` zJ`+9%Y6gcpQUKE7mAvVV1`(IWw?mJi4loUrxB^tg&qH^hMZgVz4NgH#;+yC$sDd}W z^BL6iJ7(T3gKx^QRCFF>z@LP^_z=|e7C-RW@ILtI_!;=Ja359^=YgE~60i|(`s0Im z)8EWN+mQPRoxxlejUZ>{)>o+MDT=}-Xv*=9=m4|ftr461b!{# z!>>YLpo`G~XfC(|Gl`d>C(()MM=<^1RN`3h#M{W_ffV=xX8s+Dwt)nQOP^HUz+Z7!BLWkAZx6)0@4=TX2E67&yWu;#a8YQF@~-iKn3HiL0Y| z;UBsBXdL+ChoPqbT#YXcrbk;wE(!wSJ2;RZ45qJZh^B)Z`}(EP$3U0Xjk> zj`c@PFBXX(1%JTw@w3U7#7CgMXfT+5aXE2g_)1(EHND?e{4+2;k?9vJkgo^hp%D4P zXc%n8n;!cgemBej)9(h6e}X@P7J+K`P%u5&6ykXNIf%p819J}F0y8#ET{%zA4t0eS zSPNZ>nsM_RHT5D3IWw1UUVv$xw;2x&T~`4%z5QX+z&Nl()75dKACZD zYVme3wW2(jKE<3PmGP%hbN(!Z^kDkddE_7AzoB{14p0Mz!Cf%5$BcP1z6+w&_!%?T zp2U|V_YhqHXYh+4gHd8r15AB1^S0?f7obbYWkrv|N&|5Oiu`}fm^xk#@O_^zEN#q-&)zI$X zfH&6x&CvJI1iZ-qfX#Rp)YQp8&=dZWD+Feqm(9)b#S1@ni62E_n7jE8nm1WG|2 z^55YFz9QNRodu?c?MQq9B8Vq}nOB>k_u(z{;#eNojqeIU_*du-xQb6k>%$da z2*xi!6VX7lHJT4K{g1iMEQF6DXL=Gd|9&Bkfz8kthQk()FNdr6Fmw;B!<%{i41P5} z1b+do3iV73-8p9ZunFiySWE5-YUcmF_(phhE$5G3BkqeH0@G*3p+4lULt6Y=u;I6& z@n|R<1#>;Pgxp6keT*-;-uQ+v5Il&3$iG8PFYJl_A%6^A1vT)uATQqZzNWt@Mm!x> zgXuZUU(p;N2M3@u@la@p4@J#2!W?`&-dt;#{yUubJ3J*f9}PiUKmoivngbmRpP@eS zDky>fhxSINpgW-n|Ew~^yYPcy9h`#?) z%$%#<__Q2r1m<^X=16nSm>S;_?hr?#htZKHk1jw>Q8&Ge8N=VeoTp|!oeF1QC75wr zoV*!}JK-eW%&+(HukmJnIf=iEUWaNh3QRq}K|UG(5=_t4k+>(A`m~vxIS&QD224LP zlQIy1ABF!PkM;rn<=p;%{H7%( z6&XoWQ`1UByOfp|Sq(&#O_b44Mo1JYqY{-!R;bWW$_$y&US>v$DF4T+t;0=O;V%0h}|gJ0FaD zTx*8Oas%YKJB)AV_9*7F1J#$y2O(#`67H>6ku#fj@oe>8&iwzH4?|Nq^Xq;2L@ta& zZvOsXL08H8f;Sc{c@Nt--HhG zDxSuJF<$-%1(BVzzh2%kK3DgaU*gY@ncbHU#TRk~Ud(^+G<<-f>VtSC55ckW7-UxF zJk2}IFwU9Mo{Q^Ol0V|g>Tz5i8!%Jdgli&m{d)CBMA z&YTN)oB9Fnh|KPW+(54bmqHEnRCnfound=|9|`jLcoSQZGoUZ-l_&5b{DW82tuYu^ zs0-j(T&AAF*`r(X9<)V29FA)E!Dn&?c9VPH6y!Z}hh7&H!<<;Zi2mF1`SS0mB`?6Y zawp`h>Y*-zb5Raw=zoBjI8Hqjb$sp+uBA5zjpeiOvpg0@;}&$q3;O3^w|o{ZmP_*A zs4C~b6_l2<1HHu^F;D*uzK7f5C^`E?QMoqesjKq^JfA$tR+8;hkWi%TqOSwy{HU+!&T~nKKnEOj=S)eeslg5c~7{Lwff8|*c|IE(Q7W>z*l38oPB$bya{E{MXw;leLG_z;YYc;&o$%~>g+i~;@8FktD*DNdFdSRd@9?Xf=jT3bm$Nrz9!ya`gg(ewSl6{axD}ZbdFC^#XX85g zUtWao{&m^c?Pq~jO6$A_s24

f(4p z{Q^IOV~~9z&+&C~HRSnhip;W)_*A{Ecn#SR^L(vGW^O6unQMXl&|W{!ZFznPFUsfe z6DWW~P!_%Q*YFkiUe0+^5ryUKT!r*smNWCOloxXqOp)_k{)(nPmz|-fe$KK9dfAz? zyPvMN9J}?h)BY!~!Y}H}c^SVRyW)|lYaJ{0_wSP6XeX>tXQC4xRTsl+cpUxpnsZ-d zjJd8THOV9uI?K{PV zUU&&*k@?+#r@4L<55eK`c;1EV`XBNMdXM2=c?X_G&W4lmmE4m@VT7EW_YFDkau2A_ zm#@N9xtZSWa{hrf&@gP4cOrZ4f9e|YP}ISb>YUxJP*45?18|yaCn0CjseGQ^a*RfH zyi<5Dvh%Ix+9>05o%sf2uFt}gD2bbqT`Xr!-eErQnX&n-{{8r$-X)a%psjkId@-&^ z5B#Zj7k5J;`BH9+Vi=qyCcjB*ctz4N4^9bCA z>>Ri09f8`oS}(iJ!JKn{u-=h!8PrEF{H~WBA@A}n)d%2ty}js+SJaPi_O3VhZ*}&X zYWPy_h+Fip;qURFd>oF&G(3mA(~WWM9%P>!sCP1Q{vU>qVDIdf$`8+<9tMh|= z4QF>MAs3Y2;zyA8j*jX@7_YvBPvhIUG`GVc@^5H~GciE3XUx8r9jKK0Lf5ie-6G#8 z@4-TxuI|8}@o(6Oyvr@+qxFm8EO`zO$B*(#yensK8zk4lJoN)yn!n@nTn@wK>-jjY z$baEwxe=d$s`AmuzICyBmRt}gBKu>0_XnbsYyZP;e22w2K`YNqo~Kzj9l4J(gEsI# z7%J!AD|0d5&tu4Qa3JTOE%Q9T^Zc%JA6$>jl{~B4`0Va4oZ{i)*=$bN0T*d468vahzG5XSc8VY`i63i|lN# ztK0DY7=>Nx-1D8}JkQnT75HAh3Yjsv|1vwi?b)V_qvw-{|VRf%;aod zqnAB7v;KU&ZO9IKxjN5L0Xb(}=1xI%&c6F_JX+!_G)H}8Kgc<^8Q-ca@mk~zE``hV zW^xVWuPw;){w6Qu?Pw&A#UQyZKY@?r7kC1<;-`27@4$!ht2hihaVaXJ0`lx`=N4)o%jlb@ro>zS)3 z>Gj77oPpbLgMMadan3pRD`v|BIQvNEc+Sev@;c7h)`4@*Wv^W&7soB?()>0Tz}xa% zK9Z}Wj{G#HAm?}?{t3lgdyJ=|f&3os$BpWWn1Jl;)AU}#9qODVzwkC>zVD|#h(ADc zoT|Q-H(?lxs^7umXsPb(&%2ces&n=}o6pHBF%%`$V^GnxSGbv8&V}qaWz~mch~Dd{ zs&}w@i~J1g$vgN>ER-|<^G?)F{Z}$)r>Vqwe>yy$!)>lL4=A|2TWuF{mY9hDrKu)a&K5-b6>>gNWl8_LJv6#S$fgEF{6UDRhL@yqHzaF~1_ zPQr&6uJ=ItK6iwAF!J9we&q{L#kJqLH7=B|$IUn@*Eze#5!?cKuP(tSx>laQ=C*t$ z9>wLTiR{qrT{~2s&kv%x-ihkHcnDA9W88y1uK$4J@Eo$!e#BL9p6k82HY&dM!k_&<7&Ao=Y9J#IqxeMs^_5%p4Xd>2js2DyKE0EmA^yYQ|r3^B=XOlzi*!T z^N{ECVq^y9nalT|`yk(a?*IHd<@wJ)^Rwz3IWs5U@lWa}k?&~|@;lCV@)W;{16=FD z^^o7kkz5u{=>CJ zyHG^EmfIos<~n5F=XO|K_hb=gbH>_w*2X9gddU;SqTp zzmF&61<2gYtXicvfwOP!l(Pf=!f)wi_so5qzw#X9xooAEneeQ90?$S6y^AnP?@+!S zx#v$;Z&Rc=RK1+j z131soXPjO3MLtWBnlA6fo5=HcgkENI z_O?mtW07a&4a`6R^zxZy{BDd^a`uH4ocZ{<-ei2Ip2nvjb0af8=iWW~9guUUn$NeC zZ$d46g3QUjINbF!IP*C(Xg<2@t>)}Sv*gT_Y3ip@KyIpERX$eUfWG(uo%EVwrTjMk zg`D|I)wkkZ^`CeK$E!P_8$M9i!94jk%tRqS_o!a>^?G~O%SF*f z&YAQa=iHmEcPR1>FhQ>ive%xaHv&0l-{c5PHj>;s^O) zyd@V(PcF`N`7Y$0=SX!^`82+q+n^X0qodv}*e2h{*K%jB@8=%j>`>G7T4O8n-dtAy zbacbvcuapjMxr79(W`*_ajH6JS~wMXf7||2lxek2d`s_x-F*SDU`zd`aj|jxeSiN9F)-e8TI9od@avHT{Kc( z!g-fn$xYPjxjA=2_A1)<-%sulva{4gGrct^A?E`vXeaOH2Y4fQajlkoo4lGw^RHY1 zH_IRKdVDQ^gCg<@Jd6WzkX|9)%mZo?a6X#(Ohvwmf98ICE*C)Vzh`&|zsA`a^1S4Co^$6ZIrmR?g-&`u;ZD8T zm>{3SyO47@|6Dh5o`KAmd2$V8o@EZ@8OV&u&a;j4oFB(|wtLFWIQMVT1XH9j^{n-Y^(NwRc&wVfV zLY}$I>}7Hn{RfeI=`YTi(2;*ZzJmdr@8ECFOvo&H9JgURX5vHF8gQP+<;XpLpqySo*xb*|lCvKy;9rm#Rs^5RMKJ@f zsNd$yzzLl5_I^3@D$o8lyn!=OSAQ~Jzy~8UC}&9KZO*jcdrCwoVw{6Fn#o2wcKV-HR!%Dq(crpKhgK@h0 z6t0YuumO38S+Bng*P$r#>}N+if{*c;%%Yt0Blv#Yi00Ug#(2o*G8gXQm(W@+jTJZ^ zuVOyV#b>w-dFRQyR_5Jb>g@4{pp;xm?{1XEL+WcV9^YYq?AG6a67mzAnf9;T0bk2G zr;bDW^zi|jqMP1SUdT`Lq!^ibn|P~U_T7hhj{0l##Zq!czR_#U z-T7#=l`lcg#%k&zToG@{(N6<;eNan(x8~ z@=m-b_r+`SerSe8>VC*h(-xcL0(=E`bo~fquN%#^^~T_4`5Jzb|G^3J-MkTZ$ZL`P zFMGi-`7jK{6|O&wB6u11>OIR-@B%tvD^~00{HW*W|5hI)i~IJRBl#1(HvADULQ_;% z7sn5BSzIZP!z=PQ{2^z@s3(7lLKury7~xu5F3q#hU;ZthM+-T7`FMGhd@>K>C42~; zM0VgG_#zC)kFMXxQ~41Tl52BUexD!b>{hkpia1YwEcVJT@I=m@zMfCTTY6t&jGXt6 z?(!0OF29R!ut$9u{)dHB(dSCax5#608{Sh-;tzQkI->`&dvxJ@u*&u4xRTyFxgkzN z6}=mH7p|2rMRuC+)VIsG;8=AT{Hpg6=HMdzUObII#sP9W&dxQM+jA){r@xcWM-9E< z>PO^DIJ;y)yr{lIy}x|4To2#l5WT^;7y0inRrMZ_bEEE{$Iu7^(F0ezwi*w}wNOyb zo_aFpz2GRld2$n+h=S^xcu?NRWsrTlI?j@>*I&t9a65{r2cVNYlXEUKmvjHM#K-b) z7=Yo(p7jjp+-=O6uV>*Gc>hd1#YZtCZk z^MmT#hb{4toL#M#e)fX#>T~5c@C2?_-^!DD8?u{pQ0F_%Ue`z-kC$D``MQ@sRToGP z&*DkkfD6&e^~{HZ@QQpi=bkMhXZ}5o%!AD1!;$AOyWR(UKWDyelb_|>pQrO6G(lVS z1^g)b;3f6XsEnS-Ubj|1XXL@^Ji|GEYRG@!CD(GEo+0l-o}p8?3_8oN;WxP$??K^Y zpZ`xDilI+#X<@}ZXuclsCoT?lGdkz{p7e2@I%nrPz7XdjJHl4iX5mxyNUp#W zIWu|Puow-H-QaaT*!4@$Sk8VhM}9|s z6Zb+1Ru)p^8v^^&Y|kO-@T!auc-gUdaOl3e1$(<&ly-(zLE2;x4(QUy2|J1 zU4|a0rFSz*$Se44-i#?|rQX2%@sB8pG3sydiM)dQ<1Kj_HejK8I&Z=G^7Z^QpOhNq z@hxiUZR9(65iUo0Ov5qy*Wqvcp}re=5BXL7m7H@s@14ukx2a#?5-2Ob$j>2rN@?{# z`Ca@VZ^uG8dq5k`FTDjW7PH5{CqFLViL2#myb@Q+efbv7ds27#eR&GwD9>toV}$t--Lf%zn8NM zA0r=&ZtB5!K;DXRXX#t(7|Wb)JzE<(xym@%704wh33u4ft{hYa04S8m>KjhiT44bGoSMG=+K`-AyZT**#Irb^?eErP*v0lzIcrHfB z`Cf7_WR4WYHOgA(gUtHeW7%c%Y-FeSNAF1F`Fqyq_uz57pgx^@a^~s~)Kxd;`N+AG z`>m>+v*}RIJ-S0)%sEGM&*a`%te3Md^DOsvdwzs--{#)hi1YMvFBOopORq~Uf6bdP zO3s|fGqX*dd7NkOc4XdXp4CR4wNvzpA+zu-bV7F5lDrW!&+;*kuA6zfT+S>mj7hjmo#%TxXE$hrXXKo5 zd$}L7Ul!y2eQs)MJ_Fs68JM%ZB9Fug@?YG655NF9^DfWY#d4m*o!nYo04wD}{1UH4 z=5JUN5KEUT;r0ZSyM;^}EKmU@;psIYNULQ)IhfgdEFY!LY<&zTA@3Gz^jqOH^%CUl{sku@XU&Cr#pMlX zi>L7wjzDXSa{Uk-BhN=>@~Iey57l${0{(&<;T7ZzoWwJc`FsIS!-uGXsy=@ZUXq9M zYkZ*3wZb#%*VXUvFdo7;q7fcdm%x9>{@F|KS8T^ty{9k`mm|AG&d^2tGV+ep8Vh{( zOZ6~biY{_?f#3Njz7rqI5A!W(jJywZ%e1*dui}3AM!tj#_*^gfM|n0shaD)Pev5bGHvEi_^{U`KT&14L7xNPSABNLS zxD&UdiC!rlia(G&y$u)9Z^7Bmx5 z$NRIgcVu6>Qhg>~L_yc<<2HE~O5w&D~0Q*bHXM0VE`^oz^u`9bWF_tWpj$KfvI{cbhqU8S4PbV7ODq1PWR<%U=w zci=g=M4pAE*mpMZ56D^aHfLw5%cFg^C(lLBgc<6b&tJ=#J^5WW#QSnxWHwAuXLjaZ z$jr(a))wy~^Y0yGKIFX5%(w@c=Xu6{k#ELyWOlu&m;3f-E`U5|_aSHKLHZSub12VI z?xXB|nU@dAnf1A!x^h#!%$ybK%q(o;(}(y7ntiMfSTF)RX1wko)d# z%*JBeinXriIr$N}SM$7g$J25LcJOH_O9@5M6SsA}#jNW5>J!g-7m0!jUdfBOS z9@SS@!B)M;@td6IFZ1nubt{~w&RO#WjzspVMfy#U*|k*f7nD=yY#qYw)Ghf4oQglx z*>^k1739lslAJk`^JzWuO#j81%MCdDQqI_ta-Q}1O#lD?=asqhpT=H(7=!V+dKiWv zGvf=rM^ONoCpi-`ck|b7egL6$GDR>%<)m3q}d?7~RLu7_MhY7Cr=Ii-;WY$mOoP&Aa_|vub z)PEyqT+X%j@+PkC`g=Sc#c?^Bxz?G7@kN|7F|*`;^*U@o3AECy!58o?D1m>~*_CU{ zXW%$w4(ALX$PZvK#_Io#UGkxvb8i=afSivr^|RMz?oN~Q-jP{1NPPe%>V1Ul)(5Ju zlW#|FjKw2*)m(c}zEd8_7vLQEObo=^Xry-}{z2w%&eH5LTjlIs$ErU^N94Wm3;iz0 z{4J|jjmzOh`99RcQ1w&Xi_hik3CGIo(#JLGN}Mz7ce$sW_o-&cPI*_O}R93jy2IQF29B!U^pC8D79eak-qG zIeY9*`3h9QX~=uujr^9+orG2LW@N|Qp#By|p_Ja~d;^Zh+3M^-Kk7ki%3A^Kc0AzV<9C4>mhb{(SIm3W%W^~g+jux`!KLVeUij3taafK0eLlP30s48T=%{xG{!s7bzC4j< zad!Pia%;Yr-_pwtnBA)dmgvn#-gSn$R$b2Xl6yJ#OJ-}n=O(V@85@Cv)UWVp9?56m zH2jFn=gfz*`5yIH?x{=Eg6FE#i$wcS+4~E*H=6qUly)uWQD1fTyeqgG-oz`Ifm2=2{n1^{ zbG!n%Pp{|SxjkoA<=*VgITOCn%glXGK8ee6&W1cgEs=TkAfKW?1F${w-6?vZjLC)18ey#@RxgL#6^?TwNxdU?Ff1!R9nWYmr z_umNQyuXQa-(?Oq!#bboiD}4L_p@Fn&iRuwvW|L+`W!x;bB*$>-t_So9o&9xQyKz^G`qN;olK9?tR&cV<4f9lMg z1CaT>n)5ssk*`6X(aer=Tt)vG&V0B5hhU=KGQ23)_PH|hzsOm;MXw&-!Bcvbxglm? z6K>V(g(u}U+!{0GgE{+V5k6DhP~O9NSNcyc=X^=!^T_O%CTtiV6G}D$CV&R$eUen&ox zvoGe|alLw+`h4V!?V>J<`toi377Ac9uEIPF^Vv20J70`n<($D|_)HAe>y5mJE#%|% zF652ehi~TW)_3zW$ouaq&UxC3zrr?r>$9J5cA!R__u|cZKgf6CVYvlfkdMK0I16`R zrT#{&!bkWRgYdL#)p!T$qM>>_a&FG$br^*8uFXeNEW_)Vtp6ro&3_>K>VrI6Z!50E zH|m0X7mh+jpLveEVL+_Tc|Bb2=d;-@3v>RP!~=T2$e(a_>4}`Z^;Eq|cvpQ9UXst{ z@;Dwl)%S5#euAImLcADP%4_&m{E9|+NUu3(-)zPe)Gd*n;2d=!c{*N|vlHCGx8Vvb zK|hQ~3!ixaC(0-ASzH$H%FFp*RF|`Fb(Z($vp7z^3flYjozi;u;&b%{yq^n?^P$)u z*Wf$XvRD1VJ@{@Ou3w7Z!t3%)ocEtHoIS3l-Xysqo|nJox_myqlAG`){1W;jJ5P46 zpY&I&ALi$|7JrQFNb}UUpuPHXz7FN(OL;&3nXkobauqJ%XTFqY$=T7`@d7NxDX4%W zUEj`^@_Rg%ryz5q9p`@RBj?%A{g)Z>Dn_Yu|IFqp$n*b`exC2#-z$*&`9{@oO^_=Hu95&z`bnLu8-5g(o0qL}pcHTkesua%RN_^umF9xuFbFW>;Igf_Qd2ULhh&s>Jd#?H4zFFB@I30b}Kky9BjOoqAIkTw+zL)b1 zWmnF9-V&|!=j-LX$-J(?Kj>wSfd1l5FsErf!avuE4d9OH#Gjp0Cb0ah3P`MvI#AbD# z``UO_o`$EfQ#~H_kolXrlO3z5dIEN+8{i}P1&qT}>Iyg*$HsaC@IJDWe$OB2SK+Pv z3xC4r;c(>q>Y&#E*(1Kz%ekDhEa!aoz5LZ5%k{qEo;(uI$j|Y$SS9C7Xok$JsoX{{ zGxj8TF@M3^@Q?fm_9Any5MPg~@&%l~XL*ysrr}$$o zh70AD7%WdmDfxOXiYsuBx+OZuRk;w4#(F%Yo{4$*N_{OKj-zoF^8Wjm{!$#LE`pV* z`7@l3yx-l+FC*^C?>m21 z?!Vl_xli*<u|E1dm?9AW_Mxq#Pi7Pz8ild-)&|= zo~xVX%-#3+d*pX`u=*F|o;sE@vogc;eC64m;4^u4i>P1b%J@*8fk$wsI`b*tL++cL zWvjRoa(~TrEpt5c>qCA51(EOk4!z9w>L{rm#JP7fJM)}pt~KSoSn2w=$n#oRJygzp z_7+~l_qa~)84Q*Wz-@BoOwROK>eJ*Nn1&zKdHxpTCUu_U?6}M2fygtOT{vg%WVs%? z%g1xhyyNg3GRyN!Wv-OPg+7xrGxzm*a%N4=qxPH`m;JIUXZGaWzD7UK?0$02|7CJx z{DW6~ZWB*KJ!Cc%L!OIroEg4>FGS|vD*l8s%k!Mh$Ln|unQ7m+{v{9Ri@2~qXC-GY zKBkxRprpK*TOqsELu%T$*ZsjYkaM)Y>nF(nazP%=ITyDfdrfg}izcpRPAy9NzNXzOU~RqO`UV=Wxh{c4J+jAFh%9t z<;uoH@CbOQADfR4+u{7qU-nmv7Y{h6gcIy_VadqU%Sfzm_vsTksGZDu0ZF zsx?hlfza=kma`pqyd-_3o1UV0SL*?Wgw{zC+%DitAr1 zcj0&VO)S9<^;M`V&%+wI88*pZ;cEF^F2#K~?}k(OEcFMt33=Z*hkKw5?sa_!@~-)< zdYzp2uTK0A_Qwl2RIj(|=cBgzL`;yc!+XeX`Z#CLIKZ{+>o2K0Vmvw{`&2Jpfb0N2 z`P@0U29@+`>J8`1a3!)|l*H?q9%Xt7_zz3p{+>2{+ z6?BnL;rF?aKWn4>1CGa&dN*@Z{+u7+*U$z<)ra!0yn-uof82@^7=~*8yuy5$>xc3G zxIA5ohVpUz2cPA0>*Xr)PTYY;_*!o%@(yvSdZ>H^Kf!hJvD}<1puKz$w?=8qLleF1 z+Ew`&UZPik2dOv9e_|rORL{m;$a_=;ZiZJ~E5mKL7H7ZR%a!>Uz58%EwD$eCk}mqs z%eUcS%?J1@&OdX0zxjRUJIdcTGa&!Ind5m@kKmktdG2x#WrpQ@%=4RjK6A4V@@)1( z{(18|$aj_RxCZCzrv-=_PyUBOg+x6^jWBE#TSDwogIN!-r$nW7H z9XcZ4)c~BK?~mR$donw6Za%GlFlr#rNWQmUkmoJ4qXlZqdDb4~ zjogH{p#yS{&EV`BxvzJr^BiPG94`CczMUoa$fxpo$g@=rKOr-9Jn!Mm!=^shlw0bZ z&5b#`)hlvw_43SIjc4&6&eO|I(Ta0l<{mDJ zoVW8`uZ8TlvpMI>{jOy;WoB03%kZ9@UHde7vHU8oz)k9x_!(sH&Hnq6UgqYL+#Z<^ znRA&xE73`vIa`T)^1qz9_F8)Q3z;8x>5u2k%;|gpCL`x!o~_aHgPfiDZf=2@dY|Jy z3_=yX^>`Qq)%oi!&aQd}cgJ&>h1-ywYNpTBlmF&%oc(wNH{#5~+*H{+&K0u<7129b z&OCWk-oQI}C+|gOW_ivzy`6vNF*r=lS$iFhl`rC+T#CD4t-KBwqoO|}=hMgf->Wxa zs+_$nv-L5|!M9lBTK2p?a?Zzt`0ezOv!J2tAK?p}u9sbUKhBJOM(=C+Ph?*xfa1t` z_KRL2`Drwe%ObNhXJO`kS-BVQ=PpwXXpoV6KqBH zmQj2dGOJJJysNJAGe=@8e#b?~Igy>Yvd;|T3po4B_3}u$9bbzsxKce7&0POS{U)x# zX?hj$ocsau9`ir-BgpP@8BgNu%P;aqJcbMPPsD*}s-DD)uo?A{-Qi|_%C+^}PJJWq z#L1YCO*qi?URWsKi>i1Sb+B2#A9~0;I6Lfg&O82voPDF3Ye&f!poRJnbx9Od7vhP0 z9G}b|a`xsT@(HMk-;f>j1=qUC?d3P**X7;Z64m4~D2)EgOCcdjkt zoWs@S8eE%C3HrbtQ7eKAdM0hh_2fuq%PaIW5i$R2qWUyBm*yFA(T zMtm!0U%iv}s`r;am)}C(8IRVx8n>t~=2m*YbKcWR@OS#zXGe2&zMog?e~M3V14^Tm zYyCL?9peYy;&YdxyqsOFzB~qxpp)JS_)H$o3wRcO#vf>k&-I@|X7+39hjF$#_wB{< zxyZAe8JT-p)hRAu=%BeFyzvV}f z-*XAh?3sW(2dD5CC@*KnYs`7Ba;9Fcf4rQ1Eb}hUPws)Mc_cEAi{T0RBrb{q$o$E9 zkvX{?k7GQ(cC8a1#kqI#Y~*~;cm5#qy`7|241F<8oiifOR5hR5#o2+f=N%_k!c1ho zW=B2K&lKj|^8@5QSSjZ@+Knf$30L4a*ES>j*$#Da^hTc3JRg|_xhEdvZ~1-Rz>9H> z{1Y;3&g1McIqNbXGUs!K9L#y1&cUJbWS+_e@H@`NNWJXQnJw9o*6^?Tc_wa@+jC~i z2%g1bk@>R;nGa38ZP7me3co^5?b)0$kj{GOj z!7XU5&e^zHen{S*-^YRYSKS?d$$L06r#bqnyCSnF=Vx}K7JRbaO}q?suu=Uh-ozGl zHO`ED6ASUWx~iW&k{4m3Ue4G%I6GAZ?yuhn!{zq8hBIqNqcD!aVfz1ZcA4xf@2P95 z^Zr;-UN1Mn1dPIBynusT%Y4kPc%^)u{5IBMIbPAboU;>6;|?f*G(D*Da1ow_c!K@H>#tFG5g&W_hxJ_nEBX1y-B0E^VGBD>x1 z>e4tI2kG6%`4_EE;!iP1E{`9uMBS7h!kxGTwJ}%!W6s$- zmvdg;ub2JkXnqAf@Dy6Oo?RgCTr1>Tu?;2hDB22QTlUXC z6rZW@L_=)Fe|jgNIksYq-Yk5FyVB>CD2tlvjl7d5@ljYJ-;3{_$s$#Q;o|8dUl>>{~mbI<3ysiB{L$K0#=em+FL*ZjNX-}!dre$4lodp2iLexKsL z^Ckaod6tGC_sR@h;97Q!8uI({3S_2EN51#$Jd^kwPh)8{)O`_{w=RU z=G_M5IoyKGne2IQ%02b7%Vf@Ul5gYO9}AK1cbnez$SlpXmwWJBZi5ke+wqi~=dmGg zMhjFz?vZUgi1Up9Dd%3@fS$-4>W!DN7JvIp4?YPK)3%~^>4%oIddxK*D8KUJqDHKoFmoct^720%X_dL z|Dc85+jvLr!-sK8_DAhIQ<@3cug_DjL(Yezxd^_NXL9D%M*NJzXpN#+?AjC_gqipW zXJd+f86L}DVU&C{a{grhxfxfZO3>@T*Kp2@^Z6R}Ue0`fQ69!aaSEP6b-j8#4aNOT zN%deXSI@z-$a(bzkKrHrVDyu(=j`^6aOQj7!=BP_seY8#W32odvJ+m)ck3-h1C+yo zdMDshbW_j9ufR6d?h z#F2PdeHk8-SMo3D=a z5}b@3>b#rml=I$j96zCV9kMrm&Kq$V>fvkDMKRZ2<-tCGhJ1^h_p){BZ{@rfUBd6G zTg&yiAP&;I9N8x-#5fC$u*vo8AI;pZzej&u zrEY>Aa`w%_yq|g?&*M_u4h!%NChFabNM0AWd3E&oQPfO z+_Trn`Ho+e^ZR)n`JVIp&76Nue*>PB7o)PAXYL8)KFA(6D%a%9)0y%HAsN**$V! zwey)9k$ZSI=YGwuce`9!FZWJuehisyJ2^X6D>?gQ{sD4+w0EmIr$Bq#OwG|Zj8*%%%F?p%&XhDIIhET3_<4XES`#- z-FdchzMZc=2)Xy`an73ST^lI>-&uRNTvhKH9)lWkW^Zvv>94Ja6~u&ET`SJBGL0lvp2vCWyw{ADGefUeKZ?xsoS7f%{fQ>X z+&vKE@h>XkY5nrZe975yp};>O5)%(-)n-Zacab9LS`o^b6U^{rSY zZ$S-t6=${{DUXzY<`?mdTtt65=RInrUf%txsMlixW+1zE2Yw$VeI{piJ$XDvs&g)9 zXY8rILcW-v=j?p<@f`Hh%bBx9zFe-uS0l5&HouMwNWCwT$!HwFD~HP4}3dXVYGe=Y{gRb z!<=*aEAE6V@fdcywiSoT*=5#nEqsPf`U^2gZqF6*2M)(z9FKnJ?)o;|i86Rq?;yO1 z)#_((FP37U-tU}u|3~HF@;DqKXXiMQvxj}f2k4i;WVFSPD69VuXMgyH?^aj9Z78BX z6xnCbRd2*M=#Q^)rfXI4x%@Ns^SMvdtL39`usj%fe{RaH^``I_Je^yjqP-2yeGC-Uy3W##c{cO3eUm&az8APFXP^P7i#;tfB7A~ zUolzEE?_MmV>*~e)G8g9oC?BhH?tjP|)w$<(;SD^6e1DUWdnLcq6)26|>*x5HUOXGQXL3(n z&)boiyj-38_XBm#q|DnV<=^D&B9HP2{Gyjxo-^(^bjNyBM&@_UrKh-}&u+&o`E8Vx z=ki(H4w*T*C-RKt{yQCY^>hE$l5@6Y&pboUY`q7W8`(|P`^?YiiJUJt=;tiUJ-1uV z?7SAaziV?#BWGsjXMedVEBBp=f2y0yWs%wbC+GRkTs}(u4(FcDIg|Y`_k5n+ z%$9k`eSI!}gq-u|Ap6J^{Tt-P{3YMXz4=()pNnv2;(Oek=W*siF;tOr?&ZDU5_R^J z3hGzoN|=Lb>Q+3Ra~9o)E9A_oS@J^Ekuyh+l*{2YJg3)G??pMg#DnT1aGv^Uo{#L` z*Qqlb8mo(Np3@QXyIh=`Am>BQ??rkq1ogd`D<92W@Cufyv#S-8vs-2cZ^mir^LY?w z=8czg{{O|9AK4MIBac>JjGS%t_&s!$C*uft7q7!4WZ%t+4$FP$J?gZI@r!`8@oC#%FK z>1EH(Zaz}ZEdE&kC*<6?98c&CMRw88cr7;INY@5qDSlE9;Rmo0ZE>1jLp~d$F-D!8 zhCcehazX)WBT3DQ(cFDP%At>l|< zie7e~fpSMwK}Egn0L`#dZiMFg@A5eAhf&y|J`=-GTRonq@DE&{CvrVhkgNN%TA;H0 zlHOk4iUUy={q@#yMW1<9z6cMi$8g?%YjECUR&indoUMcSF7@Ah3)<@KRX;1=$Kx1bl+{o|*`q;4T1b?U?Dc=VI*#Ap;dwt_=XrhV&i8X&=li@`%ll!7oc$(ee*P|G zAa}t_sP6hE{*d3pYCMXK7>CL}e;BvJ20Vm`_|dOf&y_I(O|V9_!v# zM9jtYuD9nexh^i0KSK7ESGbJcA*d?nJuv&n4eGm5Q11$^!d3YR{+18r+i(n8s$2Rs z*>C@c?)0(V|Iin`QApVj6`7>nix>9`{MydOuKK825Mef<7)R|?O6&Lcp zS>alqnasERymC)`r(VSQeO{G^^R--p^Z%Q<(}8Q~<$KL{mHQ>ndY*^;dADXD|yCVLpQzNSRsFn>GEA%j`QF97V@*G$@y8llbUcvGrqzbSf>Af=R+I0wO;PorN|uk5icV1DRVVvOrEbJeYO{N<3V-q zXP5TvzVGPejCvhS8m3#$bbM*|#%ywqT*&W*jJI zrra;*ncAs70!LziUe50?;? zIj|1d0V?t`E~1|~xD}1`N@L%91wWmvznDuRXLEL>a&mUvyrbldEvBBYe>5IJW@h%4 z)%ux7b@hIf^KP<@|5gu1D|LN!BRTt3c`QWs!s+_0u^!p6vYR~XS|^@{6Hovb=w%1K ziqFFyRKOg3iR=r1p@f`sZ6NRDX*>jH;a*&zSA?thy|SDAs(wxV1h&dEk$E~^T_2Cj z`=O$I5`Tg*@*6yoKf{gKq^^vdd1cg}qY2hv61uzA5P#w_G(&d%^{(|n=IurNo_^-& z2(-ub$h>}AKl8Z^-qXw6JW$S=eTO^@qw#~@2UslM&L{ATyn@g1*_@qsplCjeS@L7Z z`E(K9f`9kDL;T9$U?%$FKV;@-mRItjoXKPPTXkXiV!66JiOb^}xe7`mXH|B*Pvj~2 zH5iGqdUxtQDd)W_yX}cV{UEZt)N=hy^jFW~bGa8UKx5aR;)ijB{tlcd&*tjLS#=P< zthX8;$}PE)eq%18z6DpywRj@t;{o+HTrPjg%Xuu%;8vWodybsF>zv*|tAUvm6 zh@a;J@DPgQYrSGV+m;VUE4`{{Ccl8%@;v1HtfXFo)9@s^>fecb@Fspk3!LNHE>x4R z!Ad#rHGlD;>QOwN^UqEE3f7=G%J|$sJ`X=*x6dr$r#a_u-jTCgR#RVz(_K55Kj*vo zWUQ5s;(stsKA*qmpU_;+zPTShly5@zA@Wc@hmXdk z^3~iI1LWhlE}p>IxD}^jwrkD(f34$ta4c@dWY@0X8MqyPtN%q;Iq&gPxUqU7ujGR< z7yZ>ucn_|_VBDtn5;kI(`WSpH_roUS9jKq)UfhU(^?pYW(R9)99;uCd#-g(9|Q#z}2pM9X7iIVag zoM#|=Q2zNqFF)Vh%el95e>9eRa~B-1_aiUCPOMj#NA9)Hk@=b#zfR6`^$lN(%+H*A zMdf+ObCLP=72k;_$g`Ol(h&>s0`fdn($752{kc>xbMq@aGtN6OS|R$(GvO2 zG6S=dFTeMS<@3$mT{28mx%-SHoBmatda?ZR0a?Y^KgvEFa*{e(OWS=>m zPeenxs(w4pJUoGi=&$B_d<4JCnU_GyFX9EnNlFVRM>gzP$F z)Mv{R_+{ignaa!b4#p$$tym*J!$0ujyd0O}AoV1al{25-;>|cs&UrJCGjr?7-S89M zS7&Zc!>A zFCK-D?AvR4$SrW4`T{h+lD*YhAc1PRBbas5c5DG$)oUGJ`#oToccZdg6nZeK8Ff^ zpDWXsx1cHV&U861MPt`y;v+flLZ9$lbt_cF<9H4J-M62dZYhkQh1Npm@YPiew5tyS_1_z)6vY(gZ zdojrMeeY6yn>v5kYS<(GgAMXvTq_^XCvkS>o8{s1g?u#^(kp-~)IZ~1v_?B*AGw5w z<42!uh1qx>58x^Ng?uJg;ZwLiS3q;QC)VRG^?$sMf9Ie1c)#}z@EsFZW_E&i9g;`wr*6E6e%WWY%O( zEJU7<$MB`=Gm-o4AkIu$%ZIp@bE!^x_*0$x^-?)A;vk=!F6W-U33-Nd-{+YfiWB5K zL%C0KzUJq=TF$wW`Lu-_>-Ujsp`H3#&i&C{zL{t6jTkFuZavBQev2V9B+qBgk0bO8 z$%Q%hM&{37d=YXlWF}10`@j2fr20nnX3qXtRX!PeuuCt`$PIiT-^!n&p*(^!2ad*2 zxdImA0oO7YhqMMv&HP2qo`pM`aXI}g(PsEux17k4CwF1a||AVt9 z4MXnVmg*T;tj@e#$=i?_m|f=`&VKhBXD7Rx7yI11Jd2B?67rnp9Bl40*;hKM^Nul> z&(_abSQj~eGe?T>Qt>&tJG#j^qhG)_Jg2@GkE5|V?*lK&ZR8I*XX8saQe7TXd%q0WC|C>J;<)nfE)WvX4V`umyf~o zawDFDkC1)3gkE-(oSE4bI;o4{E#wS66mR1Rb#?v*nJw8NXL089L9S&UUBv5f0**&B zz09^dFbR1#`;XsnE$7~F6u@3}-qWs^GykR@ z!pnIk#^oAc#$VuGxhwnlzWw}Uejbzb+oBmZs!!vZs3xDz?eUm=6Y`5|QAZTTC@gjT z2V}p!obxVqE_Z-`_q|^g5gtZW^-;(fo%fqx<=*;_a1VSa4?z+6Yh?FszYR;LMckM-TPdtP3F#w17Oi#QhXJ_2V7h{S1E+58!<4UYhpU6M* z*Z5Y>`{{nj?llrk!1V-JJa>yTa?}k7d||5AeFrY{fje6=oxQeMS5z=RNFA*U!e!xC^K1U&gh0 zDw@gz_#PZ8uS83~?gDiUw8i^)S^qxVf&Ee5XZrGq_yYO6kdJ+?H`c57!z=P_cmwC7 zmEIEMo$O6@_OtB$gXLTF-^Qcz2+nSC9iOM}g8bdZEqo*%*T0j;qO1G~jzxB+CVHQt z6JF9giNE9Paj<+aufkjM22@0Q9D%%N|H;p~eilE2?eYhxEN{g%7>_5>MgM8eo>ZRK zqnBQB&VHZ0u!_2+x&l9lV{i`Y;XVE1cn80Lx~Px2xEFa}?eF^4^0ByGZl?Dy*G4IP ziu|2GAoM$jQnfU+D z=y{4G&`s|%o{ux+b2;DpTKN)W-e=D(&a?3?a-NjMYW#rAiqp6-_TXzAuh#*iEqYaQfWv}`>Jf%Ji z4Y3Z{;j%Y=>e>V9k@!Z=O#K9RqYirL=iEGxE8%v%2{>3jhX?VC=qk73Jm*)+Z{Sb$ zYR);8*|iPL;#D~_xu(1V3-J^Fz#}N&+6=x8XJP;{lRm~{sG)uo zOVAP73-0FZ+9Q|565n- zQ=jkJE_o9E#`AiExhQ|nQ+Xxdg1vZ7eE`1{`?WXm-^lx4-Z`?DcU12|&dKYzGd{!u zzwR28M$W>VQIF`qgFbp)@EmHX`{8@^M0S93{1$S4EWtqJT)U8u#(^lHJ_n7_OuZd% z$^Y_i$h*VY{E6OKcoz?-A4c}_3hMDFq0a8SN}h&^xI?cZSL1Q~8W-g^Fi)PsId{L~ zkJNwRSzLuHFjKz?|H`#Fd;BIoTAf{_i`-Pc3a!*Ri(c02$-VhwK8}~+Sov_?&N-W| zmA{ft#S?NboGzct2XYl`#F}KEznHH^&cWe6a|-T6cJA}_Pr{SxOE~Xw+5fVuy{XPQ zd7^7q;}ATB`udOajd&Doe5L{#$Xi{z3%AJo>HWwLt9$T3&U;@|xs=>meu=Zs_SY+f z#`1U`g(5i7^-BCcK9cX@FL)~t!e?lM-|?S*_NPg5W9+B?0ITG=ctviF7C!T*x+?C) z^~jEKF&9L3>mfe(ty~kc(Gqp_vP<8=dAB%U?@c_YK8g>+PPE0ddU*#a#4FVGQ5K(L zH@?(A0Vm1V@xAy!ZjL!}FJ!;V4*HDVBK2av3t!`B9H)05`eTc_29A-3A$wB`E{{v` z1YX9GK2sjIA^XY8@ye{Cg`ETB& zz7m-c?;-b7&ah^#Ex{OdON^Ftt}T>5=KrNnQ@JMcJ->rIFZr(Skh44erp~jUd7bYl zXLP>DF`RjKBHxHsa(4{H7L3v>ip!DNze2AG?!=$!wVYYPd(-5@nfC1(7w}&=8kxJ9JH1gH*&#OSWj^E?$?i8(ooD1^b!J!Q zdhWk1Jf8Dhy(iB{SM{&F6*tQ7@?-oB|Ba={{+BtD^D586+sM71Io$~JaU$|;AA>sb zvp4{GE;3W6V3%vN@dZ|@KfzaWf7~ipLK|czWS9Mozg9oU+3m8szNNkfIR_8W&ohuS z@+`eG)#c@bdg0H z@*(my$a&IGJscOR|KPkEyvwW9HTV@?!xNA>+gP3RCuiqVSR!A_=c1`xkMk~){q78P z=KH~1S^p+1lc(|^bd$&PNVGuCv7EKj;TzcGD{0_&W{r4q|WZy zOx}Pq)Fm)WK8K5=5poutj&=IKV6B{UaI^e|oZWtZ`6Ia|hNHb+C%rTA0@xZdwvm^)}!b{TpzY`~Yr2&bBu(7K7C}BkJN3 z^ul!goNcqxmnU(~)#v1!P*9zn;7~dH$I*O*UPYdXPvj!{HMlfRMSuP8a5$>sX;e<_ zb6vPFsvx^@51&6Cqt)4`+RLjkQe9PDi2JJZUS3wdQ$CGP!2)zqKg>J0B)XxWdM@wb z)A(+@B43Z3i_fWZ_Lbsa^@{Le{1aNq=b@H-Ir1Lx6t~r@fTi-;JOBk$E!|HMCXcJ3K+HTlSxdO7d52lDmmBe@;l&#!ad$^Mdm#(Bt|_@Dj{ z@@iyv_?>&}73C3pGgsnAFhXv^ck?JTmGeG+J0FLFdav>~ya<=!8QhC<`uF2u{wrgQyqj-Bo{itq=giJLJC`E+$VPreKhN7i@(cJ% z&d=;`9FLs8)AgFjQ#d`Dk>?_F zCuhdDuH{UgkCJ-x_(VJ{U&Fs}&V$U^@#@Tz>$m}Dr!K;+@QYq&&V0!^*HJzg*{L2= z=h<(F3z1ownUHz1iEHx^oGfPts=u)3H_@jCYc7sJ=Xvh`#AtPQ zzM9WLGkF5mU?lFqwP@sQmiI^A z6EbHqlbRv>Q=a?((FWw|X^#Vq7}`jVI95!Y_xqj)|3 zlqaCP&wQ@FQqFl%MSc^V)SEF${*|+zeac6xALPurm*mx)_o73&Bo5Kbd6M0yoB9L% zpgs<7VHC3Oy~B%Kdkx#=8~IGmd6ly;^YUW#G<79@nSbP)ICD8W`(8PFz;ty@G?o|Y z-6j9a+3#QA{kZ`4>feH0$lmyrYlY=cQ2-aINAnO&l!u`XZdJd-7a(WwiRzr6U-1IH z?7BIh&X*71rCc5#BWFZez1OfB1+iA|H15ew`F-w#-_Q#`qrU$6_+9=9kK+XO7x)96 z)n{>2|KF#|IS&`Bv-f|dz7N?M59De3_4s!_jPK`VC?xMj2l;X?&(Gp6`53+g$IA_H z32I^}?!>`ZjKh7V18>ChXs&(&?BU<(|05s3 z3wS;Mg-_%WJe0G8e#tx3*&Y7o+Ukz{KXrDJF5-E7l70m~ME#_E7zW{g=ug@Cw)0kX z30}_6@!fnCRw3v6XS^RC$6Kz~d)R>~vSFvsZt_r{XBrnxH$b#gTe-akN|tKKlRv4ydjExfqU9aGYzy(H{NP zd2hdl8>x$6wtOr<#CdNREYFmib9VAAJPQBm^+FYS9rxqXxEa}ns_DHd-@^sfPs^X9 zy}AWnK@W8g-os@vS#FK`az*}#PeS(7=hThm>+t|;;9=aLe;${^MEOyCh!M!{S%`PK zHkaq1jogz9azixpnMLYd@>ARiO5z!;T<{W$XNM6IrIBc&bd4R&5*q? z^J^+HhjW%Rlv^UtVs~|443;0k+wyOm+1^6VzIdT}p`2&zB{^qG&c%T|200_zpn&Vy z%NEL+v)LK*T~t?}!%uTzy@TZE5WQB&zIQoi_I#uNEym$u%*U4)deGG8()R?3-Qc|O)~W_Ewh^Y|^Yx0U4`IN!C|_(R^yH}fsrn7_tVa$(+r zjmTcH00nU=&c_6wnSp`WpkBx4@%NaCoOh=oGdpL`QoWqr_2j%)%Z|9N7j+eRghWsLKm#1*f!pHas&bz|+&$pK5;P*P~U`4<-c%0$|L9SSpAx4svdybSh1$!Q=4;_ETT!dUts^ zH$(QD9eje`e>e?8ae)5i`u*^b`aW)oXYnL{(tC~Da})fE5$dgY6~| z%Bknbc@MjVi{N^_kvtin;YM|F94G(EFY_L>mapf$#~#W*qMBY^&bve2K}MIqyeH_#rgH z*EkE=d)spc&I^?Hj_fl}$*=gei!fNu{`I9i5r?U(;z$%yZ|Cp0A&=mDu@$A&k8)33 zE9ZUh3$CZm``zpOk@`6F#%}zEJMn>Q8#((|7w)Cb_g9m1-{;vn0r@$7jaE3y_52Lp z=53sx*)rr_|3NR$^G9-K&1huLn9cd`&YZ}zTY&T3h1c~(kbH;U*_<#n8AtfHJ}GxtmG@1{5zw<6DT=F>vtXOeq9vnuy}X6i+$IkP(7 zckazCd?ufU-0PirAUf%n;hbfUaDI;Yp1)UT2mDdKOl}wCm7ISrKojgiW4(DOE9bt; z_uCNJ^``25f}Af!Q4wdMxa*f9_fBTR3cWV!kMRr6Q-6=lis5`9=ZwjW%#N0w>QFiJ zq6BiUo}un0AIVdY`|%g#nSDZ?=dzQ!u{vi=W?t^sHTYf5{kRsNCJv_^LQDqI@hxt86e7C)k%$X&S< z_vh>!na7z)yY!C1OBk+qA-2lzbKXy0;@ZerwM{=~QgX5uE3dx2lEbS?b}oHKG08YhGR8z2KMDoc^FE`*}oR> zEcHLgd@qSdk^N{o-;Gs1^DQ6GzhjF$m^X0Fml-@?-4d6{Gq@fffd8;nU6;G#Rry`) zl(Roi`5>~-1k?vX!4dEAcWKKCGI$%FVUyn>~;MsG9zmVe-<@sPZP zf5u2uQ`hkOF6O7y*-uC3y82VDq%Ow|)%(kH`5pBUToupBmtzs~KJk2N{kJ*q`RB+3 z<-)v{a}Hi1|0Rq2-s!W;EL8uCvM8p1Gv~ddiab=lhqD9E;UCovcp~46YRLQ7P`$6@ ztGK-S|Mt1I>PGq{@t9nR&*t%XSnkN5^Dx{fzlw9@Dcl!l`MqvX=kFd);Wzb;#0YsX z--0HtjpSx}CvZ#kHJsh@Y`uPX9p~yb=R&-Pv#;cw&pz~}x~cvd_z8Eb>)|)~Gd=*9 z$fYp=4`2x@`nC1B5_(~#&z;JraV_M%{w{8>_Y8l?MR`AbAg|^f$UgkE`WyLhWbes) zT@Cqo{R?rFTtTm(TnCfYC-QSRM!uhqL*B*r@?5<+cp0y&vvZv%=gaR(Z7>A4;#&Rs zI0C;S?>nb+Ta>{{pDDtJ;b>IFB)v0vEsx^)Je^1IA-ESGVLN)Fplerh-WhJ^?7W-! zd;QFUoEKlnQ?N|^EY3$m^=592%=u^4SIe0p`FUI*XE&VlviKCIqyH_?#OdD90lauo0*k;I6upE@)pkb^thaP@;y&d=X=lh z_8zZ5zWe;lo1-1_jDN+MFMrA}AwTW`6S$nf$9^uCC^3& zbW$%xo}sb)2sX;4IcHPu?Wyt&Jb^Ff%%nUgkIE&H^C>g5ir!N=5jj(@K%S|OFb(%0 z^S3x|#U9tc#wz6e_zUCpf98X^88S<*QxD>%JdWSMi}FL~Xu)YZ!~ntl3VK31K7{=h+UW>Ds9MZQDt6Mh1Hku&H=y}_8J{tY+E4R{1{*3IV5 z$P9Z2&!HDS#zg(>Q7_5kkav{K?Cd>jc{G0XnG5lntiA8dJCmC$GVh1GwnTk3XE!Y- zH$iRnJnTd%EZ1wt<#4^cfHSLyaOVCqdf7#PmJ6Yk`cnQ5P4x??r=h5NfA!N`QN0p( z%3Y9mpV9n?UiQ%ky`kuVydQ1i?YIu#xxNNt z&`o^=XQ#*+on7S@^_BYm-M9B$EF32nbnR2FfbkfG3E1Fz&b+g52F^j=E6>uOFXw!C zN4^ZjaF||ST!LGXv!^9)L-vDee2e}8JeEI0Njdxesq!~+_WY@GefefC$mel(x98+u zs3V{4bA@@N`Z2DG_wX*xM&3tyy4FMfoR7=rP)_|CXP3K!S0m?d_Oa|6r@6jRowI!| zS5ue33#g@Dzz5)cG*$o0ukv2xJ@+%-k2mRmDIbmo^6@+$FJO+kGI!_pDC+ZHt2@d^ zat$t|SDbSeSK@8@JGeLM<16({R6;ACsla*PD#B&>C(e7@FrTfCX7c%Zm!l2dMOXb+ zJcax5i?~*P0@uhNU=jYo!?FHN{1N}lx8QF)rLM}q@kO{@{tMIb2aeYpiORSJ**y>9 zM#$eE7%e=(}aH(`f-KTed(`?asjWpD&;z<>HD>Q9!lr}b0U#P#aS`3laSdb`|K zUXD4~u6_meT`Q~3P+U!i<5%2^oHZ|V0pyv?y_J75&-35@3V(vk(D9t-B4>ThixqO_ z$rEy(iQES@(O)lTW@cjMM)t4V$Il?&`Qym^uZWzZe{$yGjhtuT4t$T?ce$U=lrsy9 zt8Yc_p>g;@zcVJuIk&#&JQJCzA0zi(o}tW$oyg4Vfy?w}U@S7*E=KOz?2mbV^GwWB z?~Jm@of*0g`R@AbmElXV26^@>`#tkaj8?zJd$AHPsps?WJQFz^_wWOHd3GA&D0S|~ zCGrh?3)&%P$f=km|A8~)FEAFJk>{-dCcBnrF3<7~&fe0R7pOA}vU?Rl&W(5VGgHg* z!^k-|%ja^Ay~;--=hMf?49cu->@%73xrc|Klg}>YJllUE&**vBhCKh7n=kQTWCs4M ze*ya78@#8V=VL9(p@ZHG&WvurAK`c0t)J&Dv-ly-IhwQQK>1lbf(7b4hk0f@sw?tt zu8sGR-6wOZ8&1b<7=<5PyAYe@E_@1i<9O1cnXSW+^F4F?9XYeQHTvK`bzf|k zH)4+b0-i@*pU-}LkN#8092|&WT-(6)TwBG}Z~^W{&ZW$+PdPKK4o*Pk#T&c^FXA4b z$$Zbe%z2sfVTJw``i-I7ch(Q(2lcN*J$VY>$b~sOV-rl3@6tONt8u=1E-&MXuD6ue zV=EpZ&3;#s1N01kax@L zxh=M!x@({E(a79xijjB`*(FYP{V|-Qeh}G(&gZv~z2Z*SzUD#5S@0p3=cf9r(GKg; z4B2UGx;_y5so&(B8%_9Z&U;bbH%rJR^@i~#l*LHY#bVUL^T>Nrc9NUrhqxYB;GD6w zc`+v8cy-Yre~AgW1U3Bnv-Ee$d4K9Fe~S|GTyBiKUk-LXXG{TkfBp_t^v>leyb?dl z+5fVezA68tcQhB`ACVpWKXotpejI`2C-*dgb={X6+9bjPcBN527QpYJ9=DxZkgFb)6c^+r{+QE$ZIawXK255lkVbLb*Z z=0^M|U(V$*T|S;i;1GE&zs@%xUt-?-vhNkHVSw6HWDQU0WxEEa-Q*GycA94i#a>N>8@vfXXklAy&7ZGnVH+TAvbodCJ$3* zr^@U&TAi8t8<$7!@3zR9@CVPp-Kgc--zbWlxd);&UUuyX&Odp6PDLj?h|Ia>exJJvbFr* z&hxSa*@K7hDr9El?E09yW1!F8f!E{`{2tE6fyi#3`JEY+-7e>C&V|f_#jgK_t;nuW zz~^#?{VYEwm*KvAf}Vf(oiS55XY&>MUGSSc7WFU`S0m?UcDahU+-G*+dHu}D*KsTA z>3xfZxK!N>P36q}vV2Ic>8<0O51FCIsBh<|P)hy}d*#Zw9%msl{uTZPnM>1n5@&Df z$VZ_8G9#Wub@@wvhhM~EIkRxJJd{tsaID8odiP_SoY{3NAIhU~i9CnD<4@2V|028Y z!+a?=yM7)f$=R3lUNv03S>2dFM_J^oe4GbioNM1>e{4|KMin`8{!=`Se=t(NeQM53 zJxE@O?6Z&Zo%%&_i#!rjiTV>{->$)h^|EW6C6|z&M=LHMRInJ_T_kKqSU1SjJ-WS3p* z+Dn+NzKvVx&5;MAoO&Xv>$O78sXx&T&tSXH^usUm2yTF>$eDFgJ|jQKMg7_@_z+|- z%Y&bt>CIS~ohIkl0`JLy-?tuOD z4^roysIIz^d^mrEjq-Hf#H(vXM?sv) z8e^8;9+bz2>R-9A&pyESt7qUY^ha%6rQa2=Av@^LdY_?Sthb!k-~#z;&K^2Xen(!0 z%JLOBSAK#&;4;{N({QEUetaZ%$C2{qXn{A?oA^4;FwCCPM*IkE_0L2_`3`=8v-kZW zFOf$fyUc&;{GH1C>dA6X&JMGb4?|0=#~Ub)LOydaACG%64$t8qobTFRz6KxTYIS`y zk(c8~xtZU)l6*rlD&Ttc!{{mZ;6Z#Um-pEnTuGgsaW4O@{t@rWGtp44i|lF>)fMHg zoEej6B+q=V=g*w)BENsW&;0k?fqXZ`F;mXZAkSI8r`(eb`2%%+=J_Z0?ty%(I%in^ z-1)N>;LL#hxeiiiE_{Og+;T5whsbxG`{f(G%%HdAd~XwxpYfCGn~+(PpGEG^N$MtY zp4ZHrOL-P&PUk!8B-g}=xXiWAocm%4enWmfUG;NMmB+)#&pY2sp0PYfnUQUgpK-p^ zd7L?RAD_(GS@K-nqCQG}Fz3D)z&Zb0^114f$ZW_xkXe~CGSAvF?84i)7%TLz#XR{B zo&Mjm_>2A{db!`WBHw>=&U3#TXQcM|JXbk`Ukvh{ocsP2 zIrHpjEY`~$e;+r>Q*k)*{Cvk%^=opT@yw*g$TK&Ci=(vM9+?~2v8wBRh0D}0AZO}A zuEn{3ZtmJ<2^fK6r_I7v|t7z3lF%$X8(%N+PrQch`R71Nj7e zgPiNz^cL`0xEQ_FZ*tDA-{q${f1pM@7F%!^@-DcOSG&B4-#~9Xi<|YjarU;^@~86s z{1lHz4S70#kaNDz=4aI#@gzRMP83CVbVNBk?sNTk2Cw8P*el=5*;(_R*Gaxo?>L+% zf5!v48+yo_@wxmbACKG66X*N2g?Tcn$T`P%$zyPn&yPa(xm!8!lnwNAp02<`DEGZL ze#WmVPDl3Lq3T+4-d~R8$8e!uef%e%z*TTG7OVSWt-L?CMiIH1UwbM)uWp5><$uvq zzq)#<{445Ugu1-`9)206uoiPr1wZ-BEyxb?HTS^={nL0XvY$>+A0jtHU3nc}z%96m z>u+%@oTm3J@*dWKv!gZT3o*g{%$X+sB?_Bv>&hDFi_ak||-g(%8N7SWos{9!W`OGWoyki#Q?1S5RP%>)xOaZ;T zTl_6o;NAKs^F#cz-YTp{S^SGa`gtenCs&jAqPaYPb3;6eyK%L$8sCfiaVW0BBFuH| zSX?hZggSn|yq|um_p7=K&qpP_CH$4%>s+6EVU0Wld8cpB6>vKabM0q7lW#_0`9W^O zRdF0H$H#gfViZQGpT}QvUB6#xbWjgNX*9*_uH`)H!sp-{O_aD$wmZ=So{8|9Jj;6?TE^11j5KdLixOK>0LzRb)ygXbZ$I`?7D zpxiU<^jdPxyxgOiF-17{TAulW_*?GH*$?wG%Cosa?;d0(WR9%h%(>hfxzDcBn~FT& z4|1N}oNc*3YU4Wn6`bccb96Fte_Y37T^o>ftTdm z8`&*$Z)Kjoz}?YPKljln)Rprb+{!!n|8|T#Q@!*a*E@@sVW#{i7vs&Gd(gl8c8ko> zl5!c8*35mk5qVCY!A1ISU<~qn=eeDQ^YOLonMHX%&X8wto~7&zFL61&oOMsi+2eB# zoGMS}PAD#C=C6^T#xJOcoT-`7pSivjwbgg2XJH(c;R^j`JOxX!A6DySwrrPAN1o5l zde7oQc?inDNA{i9k88*Rh(V!3;m(U%&ez(C-QtA#HVAV zYmL>VkU9T@-eGtI*;6uuG6VOxmOsP@?kaBM%$`e;U92WvL(acvc}Z%YD}_DyO`Tc3 zSkC@hiGRT^JmuP89*+CbS>2M~!d|?q&Q3c*UXB})bMXs4(Y4IpANY0kmHazj%bWNP zZp!6Q6S;{d;RF;=W++HvJYqO zR+V#B&fqWAH*zifDChj0Eq@~)%H`EV$s^G}>}xVn6S+<^!1LVkvi#RU0i9)<7l zy81pI!%y&hevH4yD)|e{6kMjRgA4I7-qw2*+vGv~KEK0vVx#;4s^Mt$0-Pqd zMNzC%=Ux2|eo>vX?o3{+Zp?)-6#3^BBK$29$% zt@Tj_d8cWkSID&w&=qUc6|hAvk0 z`piIiBF@D$T&(vd-a%oX$va9@9H4#<{d}fAm&JKHR0)#1-*- zYMiY%jEC`wxLD5n&%yG~sD<%3T>m$|iVJcxpIN|l)$gGl9zaF?*7_gIXQ7pR8%~y2 z;s(5{zK1X8i})mz@$0VO?5_*;Uy&ckufykZEB)i-9r#i`lAG(DjfwIr*okwn1w}9# zvyuJeQ?AON^Z9;#d%hPB$#cJ2=KQBT55FP%P@bbNQ69^XeI)xtcDA|dMn0E$ z_671CJfoL;tqAfxf6pE9ww!x&g`9gSvnKa(cB~6IKi@UTE|%{v-(@j%zUS(=6nl_a zl6kOQ&b>UIOCdkceD_P_a`+Z6BhOSxrSIuaNy}f4vf@q|Usm$Xzf=Zox+&JM!7;mU8BDY57Xdyv{S7 z{V~s6p6NW#Pa@B6o}cUZX`Cf*#)HV7GEQ$3GM_i-z0aAu17qZ#%-qX-%00bH&dh%l zjrB5Pa=*Uo+H}sIv_md~>go%)8QRL(N&b}ctV}`9ir@8{BhU7!cmbJ5WATRkH~)af za-PMUwcR-9#sHqmnMr+cn|dD4z_W50uEA|^hnzj@LoUx%^p3?{$gVhqx4AY1nJ14U zbAKI9#~{`qbFal zpV_(@x2hlGVK_v-mapM<$h*o*d>lrgH*UuxxCuE2*Kz|)zzgcUmraqg*WS%*^^V6c z@&TOvq5(HlZ{*8efP^^;gL*9*+a?bUfuLsGS z&`Qp`%*Fg9KGypShsoKkX3NdxK`4qz>gPFUSyL{I%=_%Sncpv}o2au-WdWENAGT!&OE_@7E=FZrOM{qR$#aP##;tqU1pTVPXmfRi_aUZVI z%lpq6^8b*%`Xe#>%@+0Ha^4$$;_q;!-k-dS7jqpvfZ6H_d>~)KozOw9iI=cM-5xn7 zZ$o*#lQ3D%+4YNDOm4yNa3Q@DzAnK@CqCv=UlEO ze}bIzrTKas=h}Yi*ZBnXXWW-h;Er5@%kpWwfG6T!v{e_vZuu>g_IvE*UN}>~HD=-; z^)mi||KPiL0oTJI`E)#t57dui8&+d~zu$*^v}<`kyF>1UJJs1YuI0Sz*4HbDXYjJ# zPV|)Xo>!LXe+B)|C^_#|U#m-DmAW6U!hh=eI1r1~ui`m8gv<3l$8`BeZiD=$=c*sT zIOQD9d)7&E-V@gF&ls)W12yCe`EIP1vzulwTqs|mw;XwY%KrJ4d?QQ_%svz%p7x9t$J@`%hF4yFLcrFf*2O~S-f9etVOkDzbPw&p#F${Z91m~e39`)H9 zP!X>of7i5#%ej{Kgs1rwbv3*y=f5lWd!FY!5BW8@@7n3-|7WcHH?K#2kJpfAXCN{I zhjX6U>`-|w^IhbgyAlhL^XLtfKvPUXH{==3eVcjFSkC>t8-JU_ylg#hT_5ooJC%fnF$ znK60Bv-|DO8~9e<%$X0RQB&T@)o>}!Q7^zMe5QU1IiF^z&&PPVHvh++coXNj&F=HO z`f~I^-YNdn|3xm!B{{Pt`${i$cIEYato~?ZU!A9ZSN@S3@Zq>v9>QHwRsI|6a22w{ zeX0KgwqlT8X5V~dkN%eL*1wbAMrPJ-b!NdG>Qhh~T`>z6U<7_ZcDu~rRx<6|JF-7* zLuTdu`c-f&hT>BF%+yc$U$m8P;mqsb@y=3-U^C%0+l0XT~n$H&9W|JX?uN z7Wp zJ^2Nmj_flvIPU(4_FHqL%FOCBvRN8VX;CN;qv)I@o`gMH>N z`8ynd#%P1bksUSr=qWz)y830l0uP{oelxw&a!bsXCt(FHM?36AE7z7_K3-NIgzSYg zxHAgszsUnpAG6gD;xJUf&-e-1fhzI`ocE3CYb z92@03_!6{6N%f655x1bN-WNDu9>d$w#P9Q-Iy-a`>yN@=1QbBhVLx z)vseD9>Y(1b9q0mg;jV3HS`LhpL{l7fM?|EaliaM|ANYL{*EMrZJfFXCSsFT-Vdr_ zyj%_E>G$W#dfB_Lln3i=!5TTcY^=xXYN+LoTZC7 z=k5&7ebkP}x}Kj~{(tk|{{!-8?1P#5`T1p@xvTcX*`DWOw~aP zT!Qg>H*inntjmndxv-KmBMNYK**w!_@P@h;XRhR)UyD397a;fbf5_~qj+{LQa&dl% zt8x==h|Kw?_*K2kjKb>d*$w2(r|bxuGub<)$ODkwpbPFpEp_(l%-|<*uDUrd zL*~ofoM)(*e5w2p@-C6R=rZ{(WNu_0KkqY{nRy;FGc*4i@e2Js^JDP?zD3T$#;za7 z-H`W)-}yrvg^GC6=lUXhZ{|uzyr{pHJK$$|Ay4CZC?U7<*__dvkzMbP|3{O3fd5#( z{U868t%RawWD6MyNy;86$w)gh8By8;AyTBYBV`t)R8modM7uO;8X4_fDC&N^9LL@9 zd;XuV^SnM^cfOzNI^XBjpLb#rilLs*)Id+{RhQxmP#FW&-Th2gE{*SyxqqZo)8HDAs-ugbV~h1`LQAhYaK^wi59I7^-- ze}Z%5(cFXwAoDBl8XxLS#}akk^=cry%^Gz-oGLfq8+>LwXMTP!=X{;Z4bf7+KMp|w z^*yL4--PP&Yy3N2MnCmRK7?=ORX75N;sw2v`D0#(VQ8(cgeo{2uj*w!SH$TktT!2# zV2dy#$@H z166S%E_Q7gmg7Ek-n&_IlJ5Ua`uExToy(3 z@4yyZiW@NkV_a*+6}SU8!GZF(=qOjl!E#fy$4lx3d>amv=iy>>RiA-_ut(hk1?A&# z628W0l*e1H9nQ4xy?s3oLLrT(k$1oW{DmYHfK!k&wG&jINw|D#~ZK!d-XG)a!=pKxhL{0=N?&y%!|z4{CmAX z`JLwe_(PrVzlA#YNABIsqx}AIj_y|Ho~g(AF7o@Wi2Ux`ab{`ui#^;Icj}kMf#|Pp zi~Rm__RW$%!eeq~M@ z%Q@%KB6$+p1a;28w#Yxzu?452oZe7;E4TOOWT#)L&bgQUJkLR%%?g}-T&hI@*13i>>i8t zhRc^B=TDy11LW+hd1hN;vO4c356PJmzhJ&z=4>lmr=G$&FLLJQ-QiljW_r7j{i_nz z=oLpLy&riszs5gftejmnvo>>QxZX%)ugngY`Ja7ttz1Gsvu7b^x37Zit5xxwUUsGI z**)bSPzB5IAl~tFC*wx>QGAU*ai`vTJ`A@Z?*)hGWzXEeMfA?%IhcW^SgV(Fv@SQt z2YS3yob#5Ly@_iGjWrAjs8oVciaYiwfZS8gq(X%s9qikxSy{JrM#X~_Hi zDXyQ7fjC3&9PGdZyrcIC_va%xJK(8u_Q-Mkh2H%<8_&wW@c=%D-{xy_jkAkCD?f=+ zD5Uo>8sk5daDA!%3>>Mh$NMAiBcu6Iy%W(#J_ASV-^xY!FTM{aBD?c6&O6y2UV`)e z+{?HK*)c2W{e!#Jlk*v_hF9ea_yYb06;T^!;vfC8e)fH?gPk}*Kkp%{d8K+3-o@*< zO7Bq4PIH+u_L)1^<6%129fG6ades(4w zto{JCFi?Fx#>#_`ca!bv>|A*l&O6Eo^)h5XdywDtxdl84-^-2gp`87=K2K3k#O-qa zd&`gfJO0)CnzwN={g*j2D*rAybMku}Dwjv*WKYg>RT$67b#X4{BIi_Q+<{miJ`XF2~fPYR-_c12`HWu|SxuX1TF!+9oiFYZ<6 z?^zR_u^X=-_e7qFe4qJ#svvV_D!xSS+k0HkvtJyU9eGYh^WS_Y@=WBP+vPfZAM!iP zy_x4GXHR~QnSXhP-q7EIzR1iwm~Tc6*S2wY{)qEk<@x$fo#*ug^+k9{pIXPNAdaU z%$$;38voN${^YDrKbDV?Bwnch*A6YCvh|Jt) zQD4sau!ys7?BdMFoDEOm*!&z{j8o(md;@1LEyd01`FLH<%=$>q9Bj^`IP-f49zr?% zrPqw#K^6IV{+F-f3vsqQnOEaU`5J7Or}EK!3s&L}w9spZkL40v0dFFE#1EXk;|P8V zcl*q<=prA;*Kzi?O}r3~BeSQqYuSIZKP;0ow|XJ7>}{UR*=>5u56Rh|Gvj8<^Kl%0 zMrGG~ase!rGpBQgJ|fT5+rW2oZO$HexI6(nksZD^KacFt+j%42@Y%e#43YD$w_N?P zd>`jr87t>}I-RroyrB0p9#wZ#PmsTtv%~%+XXkmGFV}kkBTz{F1%8nS@hr^7LR8gj zgR$}joV}m~{BPg8%|Jd#v4HpR9e5ZAsvkise2cDViN*K_zvCcO@Y!4W2fmBv@g;ao zev`-GHu*Qyl!tLYT!uff9D^|&c?UYrXSQNGilQ-Ax>lAe@sqrs$72>cs-HnYc>{)` zH_k!M!4CM)^`DSE>NNfAGIP{lPk#jtmh%o?R{j+Cs0(oe9)i2O8+9U50 zd58GVwF6L3FK75W{F(X;ehZuBA;`IVqIwP7#g9veqlW4bC02j z-mCad-i0SnNj;7qG*KOV9Nu7{`?iH%2)6zoFR|oV)#w2#(Cd< zR6a}|&Gj%G$Dys>1L!Q5Z9L7jJzr}J6NL~Hepu5~~Utke4)hsaf2+rh2W z+3(iK>*Xe#9c+z!k=%nDa`vnj`7?CYJBnxUduWMf>cO19`$xzwb%EYa`D|>He?Vb* zF1P0+kePO=`YL%Ln#!4fx!*E(2H*|kzFfljJLf*Ei2Z$j6z6%*4wHNCTRH!(wfQ^b zIX<6rkFRw-&)qw60r^T~b`((G%9+LY@k{F7d?{z<9fZy5jrdpo8My~%qK=$%IlJLc z>fAHUc^S@it*tt<`$6?wbr)WaW97{4FXY_!IWrEG7s)xRGoSKI{iFAR+y^V=Is6Le z`Rm8IC%;6V^*jTU@gPpdHK>ny`8=k|&5#+ljX&m`O$VZn{3A+1x$pk?Mi?ibhTM~< zs&B+Gn4vxrv*etYdDd3RIU}VF5&ErE#)D|EIbgIHZrO=%#%|2V#mJ6) zJ7-7Fz5J%1$z02<`%|4A>LK|=xec-hWRLrii|gk+${F@HXLfAh?pzev!545}y(&Bu zo6r*v>n-E0d@BBybJpiszXH$7r}5F8e~Qc5aW9tJ;1Tsm+$Dd2oTsgMv0nDH3Vgph zv!Vs(?5)h%9W(!@%8SrM{t5@=w-LtEmy-c>ddjvcs3u#FY=ezE?}9N8>tVmiFe7 zD26-LPhq&6`Io)EKKE5;S2<3;TYiQQ;GC0J$*b{&x+LGgmGp0yXUlbQ9;cb-tQ2CvU_( zI0kF=&f*>1jxRzhf9{d$TTvPx;&7~Yts(xB&*Lq~yJ8D9?R)pw%k$Ay?D zKjFMj<%}#V*VTKN_i*;K=j1n0PyH_6#QnJuujh@(o{_!tX1%l2*$WrQBhVJZQBA)V zy5Mqk9iRDwOR2M0&f~$zIev@&|Kwd5g}mRLp!JHJGwu!Ei?Q+poSmi|=j<8858+SO z9_31$Gb=lE6}}ri@Vjg6xeT5|OV=0jYx>z$`pdmgUEP^;#^-(QU>>h`2aiN~RKVMM zV|X%J$t!TC{2`{xpKw1uoj>QNP(;3;3;A;=$~nis=0bb|w#&`=JW&A1*!X{L8 zt+ak&d7OMLHeiK%IN(gdcf(y6f$U$6_yN}%a6xoLcD?KMYT|HA(fbX@ zp^iH5riaRt-XRh zIeS%U?!-s%ViZDllDB-Otb95Df{yZS{1rdWhhQ~sQ)d^MCg&Zt5;x$+`V-|z7=y>u zdCzagGt?#cdmhLuxhejZpXQ@|F7IEP<<2-ApShOZd91ucu7Ok3Yt+}HHHzR$yoVkr z==%9ScRXt0CcW=@9cK^AgFK7=Lc#QL3d$jSUTwb2XAj}*;V1En>i^JP?ur5O6dsC; ze5&662-)%c*Rve4`-_A3W=WG_|yK2LE7N+7#^{01^B@}1{i&Uv5TQ|`|# z=%Al_KlkorIdiNs@^{bKnZI`d^Ea!P?$}g*HVWGT}U&a404LQ$`RAxqe zz?t`TxetmUJLgorBjjS}BtMUQF`55Mk$b#2&*z*?i#X>&?z`Okc@FdZAB_fbd(N|V z2d2qaBlqf?{5s!=6Ons=-@UBA1by{}VT7EyunA?=-*D#95V;q6BhSJ2oM$OBZ@bU5 z#3k~<_*2e~m*+p{h|A;^d@_DTp6iqK_HY*- zhP&nJxDThRn;`SHj6dri`F`A@o{gL#FR176(izh6bqRSU)**8~duK-;=h{`s z?CQaF^;+{>{3U1C$UoUR zoWI{;p1h5hVu{=w(~&*&2EFWkIhRXf0dhWMF87jq%H8=#oF;#R`{bN4i{%17cM?B_ zUy$9Xn`>1u2%YsZ=MTVe`DgCKPjY#_iSNWp`An|l=bx32m9wuF=32M{dA~WtwVZ>E z)GhFn`fI+En_#-UA3woo@r`^t@5TcBq8`ilBJY8V)kEb8T#0YxXLufG?$73L)nmCC z2Ff3CVJ?SiXs3RF593w5i+{rfC7)Ppfxeh_`}gZf;)6no{- z$ewj&j7fOZwN=PYa!x+0x0rLRI{3d_S@9_0F2+h^Icq?a*JzbuN z@pwtEJ!c2~oKICZL=SlskK=;*jO*9P_2panDL#~c3r<2X;f zl1C%|ed0XMUbaU6Hk^%na52`pR-g0V72cDxuPo>68Tae|gUgWr9x`74UhKfndPVg1 zM-_ac_k~{GS@Yi$`pe%T|1F>eXJ`C_KSd#*?T1D3sdyC4un8^E56fJy%59N%?OW6{ z(MDYZ?egcTFT{a(3r+R!;UoD&ZsjwN$X#$9K0|ieS=`C>YUrk}%nfk0eh2=SFXm(T zDts&-;Byt_k@64tO}>kt!PB@|UC(Fo-w}$+74RM^s|#}W$+_GLd0#ut&uo)N;7MGd zuE`%E?-c#j$II`dl=>odZ!}h4&%Y!8P3a8vm+~4^Q)hPNd%Q&dZ{(hT82Me~J8p^m z4(90P-!1ol?#-W&du=QZa4q-S>zv}vDXxz}@#)4qN2VEH*5f|`0cJ9D3mQfC%F#yL~+ z{mfG@LgwibEORa2U1nqv^~YQoQ|0~mTx33Ee&?CX+4~h|4rD$SQ0H9C+{v8F4E#y& z0c4NNvzy&>gno99nd*(4`#L*AcB;(o%*va%As&-+?$$@1!`yG(^$MU3at3zP%e|d@ zYQ201Xa8s}=l=ano%<@gVCHo@fRw z`R8Qpl(Ucb<$u*jaAwSv$Syg6^Nf{8HP?D@=FUXU-o92Z``#d~r`H5IOY@GgTracW zHoeFBaqfd>-Y}jS>2?b z&bu%}uEcdv4`tL9fIm#I%fN!*L3dYRWh;7<8y9)>~k1K7{c?yoM0(&}Tm z1Qy6SH(!vSK=$V`oH^N(U*v1J1?OD&Q_lRnmir+4(1|z`gYl|fGrk{1at>aA1*nBv^mAU{hF?${JN0TIJ5pwJW^s1hyHOp# z;%C>-#zV-?x>#>BdLcV|cJyg-IlSpJ?;yLw7wRb(i|0`ur?~bVcFV8xEW9S~;JUa` zZiA+ngtM^~8(jMjL*%Z!h==22`A|NB_wbvzSpJkR<@GoV*+UCsja$)HIGMY zbXMoRq#PGRcCG9UW8`CauFq_dZ$&F)N66W;2ZeDt@_uuI&;5gO>ZKTsXK}UOF?xv_j54wQ3d4&<_^tk;9P zA!qgbxKHlSFY+lUCC|nV`7$2BPjFEljN(|0DtfQt8MzeBmoGqV%)~Ox(l74+ekcB3 z-I=fEEj*nUa#dczlXx9g$$#<(I7aS($?^#9&Dkyb@+S3#7>bA0hhnc>osUIF^ulI* zjt^ZMhd0nbeG4CtA-a!C|x#{q(=Y{(7U-c~2;c7J7Tter7^_ilNYaH#r9{ukHdAlDAWL2_M`kiSNIwDH-$xQTu>o`U0$ zcZWRJ`A%|g=9y~F_0@TXPR6;YfXtFSZ#mDJ;Y_@P>`IwY*&)Vr{_fxKdFUx;*5s_N zBj^A28Qc=T%K7&#j+@o(klD43#~}CospPkmsiowX4mFI1ioOv}u&hC->IeYuQ9bNASzQsr}1?>9c$$?@v)p8eg}6} zKgSPXwVeGW^C#!_5Y8;jK2t{hgxm(j@eP_Ib8;#&mpZ*LuDA3><^e$gEkV|1c{1>_zIW zXesBpKL;c3{AkX;obxKPJ#%ssKdXN)ipq;QGx03BA+JQ{ z-;c;X*+M_>MUV6AXo9w`Y3-Y3kIFgEUQwSepUrpkDYz2Z2~Xlj^k2eec^qGiW%3ea z*S}KznB0>qBRkLi>dtrs#n4uN0&nKOxgattFJ%AQw~wvh_i>}ft2_w9kh7w_KQpuY zXV*re3bH?crPm(^tM}j#xe;f!e~q{Cm%1BXlgr~l`DJ_|@8;}YIg=ijm*X@2nw)pX z>-k^I)~m}KQ2?)@u--`I9dQD`s<)8K@qV}#Idk$3)lt75&e2=U^U($$;ZVI{_({GU z-^(AMJBq0D&XF^&HI7y1UGp~A>d1T14Y#Wc@)P(MU*ZMiY+3JG4Sb@`{?LPGsz>1; zxgrk3Me4eo^R9^eIj+EQxC8&7x@+rVl*B-F-i!9bck;{l5JzJwI-wT&;1=XPWH5I@ z&e+NRj5+eTcv60hbMCK^+he=Bruq;0DJ)QT;P=r^-k)#a2K*@>#%q!H`^x;VUSFPw zx9}c{=`F*5@~v1W&%#n1p#Bgu<=>EfB>U=a`62!6mHF?=xoJydE51WltkNEUuCDjx zgY=%{Mag;}^L=_{<8O?6*+7B}ZSt2yg(_Eke>^7_0w=NV}vpN5+1wVdbk8u@I_p_AQakUIBb zQLQ5Cp>k$(bu7Rm`p;vVoLO=r9+h82?(fWyM>u=cRQV%ZqQ0K%^K0B4EAcuqqekBc#umT+PH>|$T=E67a0QGdI<15e;V_2*n3x%cnoUU*a@AAm-cdI$QgWsIy0{j z&&ONH&Xf6flYT)wj;go<_qcu%ip%d{y`1-rEBPB__O?Tw*^TJuGcR#w(NUNnAI>=& zGv~6GEY<6cpY=1p*7D!T?3v5^;VSthZiALSbGSP1Jol@&%6X^BIh6T&36Iv#d|JVo z?KvCQaOTPza^^=B_1E(2{6Cz6oG(R`pUSm)Ie&?qr`O_Sz1#R!W6w( z@=2UoH5qyLIEqijbmW}-%k@!mX8IYt3>)-r=7;bnM&diY1JOl3hL7c(cMo&k5jx6S zFhqS8?nh>5-k(~_JuwaE`Al8pY|6XDw|Z6hG<+p*!k?}+N8a(asGG?DaNecPk{4p0 zdOm00$-K^-|AP-zzb}`-F7=1}3IG2ta*XbKDn zpXv?3^KuPz#`o%0d^P&V>Kl0x2IFn@WBe`#$zPxtve)JP^*a6RT=(hSi#9kEoAI%0 zBhUdAaHif1IA4AN+2McZH*k@DKg>pJ_33;OuS6N2E1pA zmXF5{c>~@-0rdm?9@feJx9|V2WRlQI{sq^|=kt$v7Qd?RLoxiT&O7i-9lgDBT!2!p)xndvOkIMj@Rd9RdG{EjF2Kig5xr)}9^6vB3s0-B;d-2Xco|Pp zkK#+wM7|ECQ3@C8!w^irex*Qi$e})HfuDU;l$a$x}fV-fQUIlEHxAR0ihU0Lo z-cL9chpB7vg?tLS;x?4WSpDk!B(hsoR^Nf_VtLoSOTOH-ulP$on4jRhFOPA?P zJM=5@L3|na;EKG48zS#~3;1cY$2`|g!#Q$eY?Zs>Lo`z##4mDoh&TCgysVe!;b?gz zcfvVXq+W`txKo{7t0(8Y+Q`pfi))!1qdEIcNq$bhI3K}F_-y=z%%vG9iriB@tjg*x|$AKf?mGV}W2PyH=8BN>;= zAMk5vEqCw(*^6=(WOvJRb*G%&C-Wflb0;#le$wxSeAoZ#)!^x9F1O&?$gI9a{feA( zqnG>;FGeNT@(fke+m08J=f5K!!9v%naAth&wK8~BZxg1<{g6E>^W<4n)ISfoKOa}G z=JCj^tBfP@I-YRtC>)8O)lE?ov(=k9&(tiroqRDb#1C?7{DH;ldysSO8TDN3M$Wwk zdJoDubFY!hFzwsTvomG)+F#>pz3k9!ICC=3+pEZY8-r6&*tO-np64LXP@bLdIDcs7 z*J^x%%#=|$9Gj4xZK6oBQtL@R-m~0O74rC7dNZ-_cO0>&eH;V-^$rX9+yAG zaQvg!ji2XfJOO9Rd0r3X&v^v5^mA`;F?Dm~eEdlLygZ$+#V(wWWA#??4>(o64%s!U z;zzx`n5CE9B{%eX^*FH}>dM3MpPVx_=g$HRQ)kCGS-xA&`P5s!P2R>sF#(z1Q}oK> z2E3_P6fKZlCHu?>{hM&2-n-m~2caEKP(Ok%dBpPy?j5O!d`U72K~2?chB$mAHCCaP5m<8gLm~$$GviO9*tY& z!klyC1FnjexDjW$HW9xgJ7)IR%>5~@eTw~1LEWBnKGx)4)aTZ}>Xwl#k=%@q&C4@8KEz zF`t4(@&WvPdbmvf3VFvUq0W0yL3Q2@>Z?yeeRWyRKDbTJ8Tu1`M&7v|;9`6(7s6XU z(*-*)5Xb0E!dUq$?#Y{ZCGWyAe2tT^6@~Dg>t}L9oP#dv@fen$LoKXBc7yhKR6gG4 zs-UslAGhHwWFO7G{ithe)c0{|Tq+;LGkF!C!0Y%)uI%#@dAqtjTB(<*Kf`b8tN9|l zBo^9)lX{?5T6Px4N#}lCz6k$OX9-UydO@GZ+`h z58yZ)g6u+fa5L8`@zZ=U58%geyIhKU@EM$a@LYL8`Z!hZL-dzt<52l9Zp(QOn8(?* z-_!dH1=V>sSR-$hTVbYrJr_X*lv6k2uE@J$19jfnDyGI*RKl0(^DoH0_8Z@Yr}ZyF zGZe*A{Gnfyk46jpi=2s%rRQ4CiJRroXou_>Z|iT7`*F^qJfC@H^SzD2!{~sVr+@Rl z`&`cbm}lV!eg->S+k@u#2)p(A^K-Zz*Q)cZ6~IRIPUM-+v-}tGoUi4UocnaT{EW}! zoa)Y*1&j5za4)`$Gi#5;xyUSg8P~XWCm)9Ka&crvWcCh6NBJD&K0E>k>)nZ*5hIZ2 z=O6UMff$KS$Um2J&Wfw$-Ez*+34A}MBRkz^T*URvyX>%;;U6M%wIq7D-Vkf?C2}ri zkNbnKRae6VInUAWd>$6)E#h@N8kym3)K&NxF3pGYJe0@H>RWgj=f1Dx&)OwVML%_B zbz5{pBjn7?+{rAzU%diN<@x+Pe}G4DmiiN{#~WCw*BX!GPZU6D{mDEVnUizWIa_k> zor*j3GUszv&XLE;IahOL{cqb#>p?iLP>UT!mwi8NC$)ka?9eJTqsnx*#8e6Xna$UY?6y$a_#_y_^|& zXUVzp1ahtv+Qg5JcaC4Z*x)CGyl5CHRW1dP5muDsxBn|DW8U%pE-M;m7hRs ze2hN$4)a~he9jC$0Cn*nuFxxljW`>{kn^AiI^q?d>C8`aW~=`*`*z7rTt{ycoiEp*d>yiH4%Ghy*{8;E_JX|QG?DMY!#+2Q{cqnp zLf&61E6U+8y(+jyJ{~LZzWN{hC11)XVHUDiJ*ziTet}=Y*_erT$d0(qwdL|R+#jpt zt!OD<%$H!MJQ&;L`><1P!#N|fb6qLt9c3Knj5#Q%tKc=cDPBeP?OS|iGxn%+wh!a~ zkUgTM{`vS`eK0@B+0*Zp7vW2FV@#79auME(-grp8kaM;lC?762!;$jwI2RpU+Yi6V zU%6I~^S+ud`Wu_$VA9UxMr~XZX2E{1VPb-g_S4QRslX@iI#Jxt#AcaFlutvIkA( z4_*5K*)@i$FO|RG&v8OfpNRVM{=A%D@8`tWsM`lZ9b!&7=#=Y22eFg8tHM!}LpQra9 zXYP&S-?0Rzs%!B(m?76d=Ei||5ZSGA#(blH7Vg6Hcoij)S>D0t%3&>zMg{%*P~Wvq z>bwKI&#&oS#W~ZS;|c09`5eAcH%0b|H`H%oEw0tOjnCj;@CCla%g8&!8on8K=4be7 zUc`r@tlS+($T#r`oO7?Ee50ImZi{>v_r&G$t-PE6$Ft}xuFFSqVZIK1t`UCj~9){!P>}=c9!w6K@--YinTKz1t7w_TI^opQ|Jc$qBoEcm3 zu)G7CFc;q=d(bNW)%A7!9j?b8>YT$1`9B`Od$|f$$g}XG{0th&!_f>cU=MO8_TVF3 z{{V;MeD!C@yTAti5II{1p_1Nsb$MhDD#~~07m@pNV{FoUf^R|I-S6QC(MCV-_qWPV z$RoL$`ZUfiSdmXa9?G1pCyS@!O`p9Bb#WpJp`U)!eF0yIW03!rk=^wXxh%4m z7eaRFLw)u-y`J0vjpa-EbS}r|adwZhxhmT0b;hOgMLdFw<865;n#=vU3IEQsu?*Si z7VF)Ih3W;kT<(H8ay1->e(GO%Gd`Cu=IWRy55!Q^#$HU3M{dFgaAxjx$hqBxe@fP$!)5qH z9?1oG0%wlr`^`BtSj_`)pF)Zp3zzSm0o^Vm+%U8&V{@AQ)D)cLGGbfUC)d< z5-ok^GOmZ+n2i#yXI|xgyhObV`OY&VXCkvP^Qb%j!-F_y@OC~L`Hu3v??HBjot)Y5 zI%j6C_cI+h_jLBI+^_XeO8<90g%{&?6i}bTx!-aQ9li>=8|n`LLDq+z!KIKKC3K!Nb_1&OCSp#ncZV&rBa&mH2l+PM#b=^7GQ(ft=lFac9ODyY?rg>B zI0JbP+2Aub$fxl{Wa9at^)D6L1V>yLK9?UawIf zC1-CeEng^a#Pj$Km2ijtBN&Wlae&^(yog`LI{7Rtku#Ux!7=K?@q?UQZU$#hIf&oV zpN9GJ&o~5U<8w^W&n};H&j0qE$+zoe<`zbFzGmFqwH@4m>mYM~E;r)UxEqz#!|*@( zZInXh`L$Yip_}?(E{iU5b^eFTaYei=pT;+FWjuzSctLLo&%$OrqJ9sV_xG!B#zX3y z?|JvgoW4`0g7iUqVTFI9KJ)s4M@0>@8=hd!x4cK>iTpQ5Z++ z{enO7pt_aMSK%G_Nv|{Bl7Gc;T#i;-D|hvocR25w z{rEF=&i-xkO>#s0CwIiD^1u8w_rzc5;VJ&vOSkd}tj0+wjzfKB9G{8b@h%o% zk^a;C5whELLO;0zs<~DX7a=>?aa>=&lH8ZGTb1GK^)JJ1I0M-YD{xJ$b$vG%#TUrF zaR|5P$*yH5xsO+3Bt|3q_#i&r=U(RV+#mVx1?Q=YBLD5<3BC7_eY6ed9ig;q&!DDU z8Q06xu^ic*TKL>O=z)v$?nNV%wH#5bJp{1se+{9S+6 z`wux&R&eI$?Z}K;%=!JUmNNq$L7vUr*ftVv_{&$g2kauvN*;+UXd2VyR4w9Flfa}dUdr=YjS2=r78NLG95ArPMJjlGw zzPOGv|1b3UUNLeuAHip1X;5dzZkMxby{^uVFh%`<{1oRs;8nRKvdit!n~m?)n>ll! z1aeMi*7emJj|Rvb$QhFP@CY745A{Zzjy@=@ml>4Vk-hOlz08b-`W4VxewY{Puf!tc z9L(PLr+)tV3HzykS7%m@RX>IUiP0@$h^wD-Kv+}^;2GpRk&I|XT@OLjnUYF_wgQX#jifs z8JY7rCv&b|slE-v<@J0Tm*aZ85@+ERbu+GwtK@TWB#Nu+;2Km_m*U!dFyF+*PzS@* zZO|XDAv69A{aNyh=qSI;w{lm0n~%mD@?TtuvpeiY8GMe<^lss@+>Ezlr`#Je@tyim ze2h+b61()@MjQDKbde|XeVlV7=k6%^F3y?s20Ey-`)1zfJZON5dWYf*34}2OI#+w+9?D6;N=RALq{cqpyJ&mg> zws1Rsj%Q;H2H`}#ar_bTzOjvmp`(7@b)S%1LEN{`WPjc)7j}Iy^4?#NU)0M^bRRcS zkKq&W0V?U2<)S!7K7g0_%$N99UWO}hvAO~u%g6FZ{1zUP^Nz4e9*3jRPp>^%;vZx$ zzLYnhE6($oD%g$e6xqQq;C|{&d^i`w?dYrifd4>$xfK3Ieu>x6SiX}R^QCwkwbYBa z271Xq;xHV9?5Fqg@vi-gg7S5!EN6#n&o5yJW}tL_hFjoE`CWV{SLC65G5?Ow<#V_( zu0|pCLAXqAitIw`_%gkN`3roF!_?W8M#;zF4{X(Ynv3#{cv!xR+xXlaJQoA>vV&eJ z&%=|bu9thd1>c~~eE9*-$OmG)oacEj=UK^Y-iYi_`TgcO$UXG4Ye&erkJ};l=wEyn z79+Eu7!Su_*XCdyDyy%>PPr|=&PzG>VeX5TsHR@a=W(8o8*yT?-a_O#&F?4o#_xDT zz6s~Zvyihn_izEd+$VW%?nLg1t2y^`&h{qiUh3X>7(MWo-dUXAYku$VsB`{gULGUo znaaJCee`s_e2Pm{>lB{pJ(!VJ`iu?QFZRW349Vifa~M}cm#RY&&94>Q)l+h=gjss zI2%vsWv<`M3plf86ko!byOWUTKl3O%en;-CH$n4C1bduZ!*%J@t?6XU_J#sdm z?Q@&(lX@vKM^|Eu{1f-#hMY6J1a4CQgR3x4owGCV2itgqdKcHg1o<%@$Juj>%U56- z8X)^fe{SRYb=WIc;Uf4|UdW435l^bS;#_=<@AYzK{w;sYdFMKkKU5Fq^;{DhFc2?c zj{YpPmV07`TouR2Mf`l`@hkGLT$umn(WoPz#SOVMvMaSz|AtNK>E$a~#y>eaYj-5$51mHy>=IdeCwpO*7}ly}E6>J!!3(f;My>I*r0Ms|g~PZv{H z!x-0IxkZYK|rXX1C{tj+t!Q*vSC z-F6hu^V!y1h=0LD*dOnrn9t;W?;?3FUiY~Q_!wWIDL%(tu2n}_}T>FLx z@hR9(J_IMAEvBJ7-gWIL&MtntJV?F|_2s8H?;cNZL3I&+kZU3Dzu5z8$P@Lq@!dEX zjn!*-8VaE+#_MfHDOA9xdM)@i6!7PiQ{RRuXoj+Sr@8h${=#c0sCT(*LvfsZDd*kn zGS`lm^W{vUl3a<$;(XVhj}MLmn(;veu3l>2tVr-WV*~_{3F@_@ZI?waD$ayeO zFFWZ*UW&|#Jdayk--dSb`JDOtqnw$aS(80$2S1{BDF2nu;5pRL%UOFV=N;iAz4`bY zqx2r;b^K$2$Hl1W+N)d~8{~(%3O<(C;uU0HX`uHB+Nc-dUfizEIh=DQ z`@=jOg_UTj*U9xYXq}(K3Hq6DdGE;H_?LVDvO5mq>`klqH=i5EQ!z|_lF!BS@>BRn zuI{s+$cLi{vWMg>x=LQ+dUoHO6+Pwb4|no4>de#!@eJnZ4MzU?l&?l>`49A$M_>iU zt1ssU+!x;?=Rq&MoT<$?J85?7yk}H%{cybpumyY6i*YwTSI^;JaRkm%7v*l8Gq|uk zQvQn1=gj1Od^~68sLx$7NpBun;(9dKn}EVT^Fgj7J52VI?XH(mzr^qG4z9}Q;Xb^g z?#<=75+8uo$UfbQYwMqeoM9DllU@m2BmarjctG8UU*M8B5bvt9Kiw$5CBKIn@-+0q zbL#K0UH%qTefCrS8^`Hqk1fY}M{cb58E!@?y+`pImZ@tYXZ=XN5Vzn_9E6>C!{=Py zcaB#P&QPDC&Mq=TE{2oQ0(axc;976K3)$JGacMs@mS4d2@-D8z*`^KXDX-%;2wDv z_LC>`QJmekIX{LGdObLMc4yA6I8g5-G(-`-G3YNh!|BLg_A;+QcG*L?125q0NT>V! zFY?#OJM2yB?5^33E6Eq>XUBVwo2EuJJc03e8r@wh$oHUz&lXZ=-^vX+g+A8LeU)c8 z_el@r@118dzn{#pa_ZcNnIkQcXExtkc~nAXMLoRjGc%DH^O3qOUYGxet+WC?k?;EZ z7{f3IIYYkkx%~d~J+4E}iTwU@uVmh5K5yha%ejY6;xoAz-joZViu^4~%DFdhrsn(0v+)I2@R@@!6ou4xpd$`P zW^3-T)8ySe5qUoI{N!0#$hoic{pZ;)k51~`7nxrW#=eYo}h%1zhjS=b?{YiF{5k&uMn6<@#%p{r@`GewX*X|H-v6Ui}9j zjQR3$cvC)rHz2d8HgD3)nemsL=QVRIXU?(cg2MXq*rk2DV`fm!!;1RDxed3#YB{sw zJ^5$(bi5{)K<52b>dd_s{5)TYTJn5ciC6K7-e|liXTCH*H@tyU^giPad@kR~C-YuR zLgrP@qP>`d$5BInHkP8F`hMgr%e?)YOQ}2aGyFHE$;~hjH>$7ZhjFWX2{*xb`3l~S zBjw?IGje|AoI4OdpgK-Od!NY+?~O0ym3%j6e<~+$k^kdIk@M>mu7K=_IagNUCp@Fq z2ahA~Aphw7A&nR>+>0|EtD5?V`R6i!zbuH z!guo?ERYZ8bI=?`eC`vy+vQtPNG^i92Kxa2a(??T{--HYR4CEjX;BF6MP*cID3v5q_Q$&o1_ndXJoaZaYuF(R%s$=f3(>&OMWv zwNqY+3Ubbu%#~rty_DS}=h%<(T)mv#pKor=nOnJ@ zpW9Hmt6YyOB7dfQANlhwNACB<$c)cS$$iyM|8>s%%=58WeF(qKdA7&%-)M`E^mD(r zgG>A7bbh{h-gDpNyU6aekSDpG`TPX0##p@6q)CF?#k%p-p#Bj zuFf3Kv#^^p54R)tY+1ejocnOAT%Nb!4AjAMdarPPU5h*;nI*ZW{!!=ouO#QZ&Rou( zkmoQnF#BR1*Y-#DiDUIMN3)xEmV02n`cS?AZ_BsfC*-->-{-#MVz>x-M&I+fr}%g- z!9VaWR7d7Op1t3BIWCsFV-)T~-UkM7p0xvzJs`8b5i%$Da#_8c(U}c>)j1DlVx#(D zewOF*R?fUUou5PYj@&$%vHyxSeJ1C_-SVTzbAKh5M)tZL>O15YQ5g%=*(1lvdBzK> zha&sxF?u;~l>2s^%!|(z)36LV`$wRLer8pk{ipGX-XZ)eieUzBMCNT~X!euav00s& z@c{3^^?KF#Djb3A57T*`e$K>xxL)0fw;{85jymVTx$+YES?-TlS*L-lgTp2Rty-{SM}l54+UxO^)g#@Pwhau=MY z_Yqpk_w)OBNX|Z#cld+U-=mFQc7d024M*vH#b5A^oc;GgZh_f&1&^Ya>qWRJ@8RFD zLcWtX;0n19e}ipuaejwO@x^E*-xTB;xB?yV3-;;{4{co0L>Z}J#E64}>(R!@~{$fJ3wdNxl+BOI*04GrY~uu*=EU*#_RH0NEetNguu z3AQ7DFH#WMeXdnM@A^{QiS{@{ZzO-l*@ga+kCLnM>HIS0%h{Pa%SYov^`AIXew$Ch zaq=fTfKNo;n||dL$h&WTHnUI)`ETUR%Dl;}&7VKt<1D@ZzgP3@XL)7V5nrKgX?+S(mwy?<3FBdgOV29Czzw$I1DVXFId@Z8`UU zzSG=`g*Y>GC*F`Rz$9emv~WH5X72CI(EKc#$$6ftdi?Ir|0oL*`h{g2j57H~Y(5<=x0VoO^%1-f)b;D7>!s zIp;pSl-FRq+y{rF7&2?V)9=Us;xSA_b)2TZI`-cyDxV}jg_d&ml0keb=b4=#pNZ^( z7pZ?jo{7wp^1Ko!AkX<{SSI&CcHHAQzwXlCiq}v<-GH+fP3Ak)U$bA7eC z2WLmf&Tzeat6oiR$78t$XGY$@op>(3MR#>WySO~ zNA*JaUsO`x${!(f?l9Lnp@V!5-@{utGdA-w?+D+>FCu&7D(;D^P(Xc_&pat-r)ne@ zLtAwRWcKGQxJmB`E`T59KhO}nu@_VHAHbuStzOT4xf&YdLiM?vGw4aa8=vac<5F0M z2T)h98z#yp^CcLHv3N^w2c}?!x-XuSYjWmY=4?-OOLcSPJs|V^W_cQ3z+K4u%`HB+ z896T|aT)z*_(0^HsW2aeyw|+SAG>}G-^s%`JJn@O`}T|H<$3r=y%uxii#hw`I6hUK zbH6$!sLu^@1^r{?cJdJUe0c$vmg{h5&Uv(pyW)7gXOZ{RAJiYpH=v1J1)JoL zai4qwUyYM-hC1&cH>Jr;a+)g`WS{g)Kz_E1-HRYy+e30M$38s z%$Yg@IpbP!Vg2`ThMb)&yXuwdF{q8Ecp0Nzuft3DO8g>!k25h|eJP)gnR0tPg=^In z`8vKA|C1MCqu89wpuPMwE|O2<0bC0MahduG{)2m?kpH*r4wvgSK`#tZcjkedz3x;T zpl*sTI4@c6WzJ4;KexjRIMDA|%BQ25d?at@A2{#F=kVd!tCu~tyqtHQ66*bVGq2UV z7$fBBoWF~x%&*`v9HRd>?v!6g1Nn0Fk+<=)co2DaD$0}L(!QPkVBu#RtuBi~a(7IX zf9EAUAB~WAlV3US0f)L)O}+)sU;(o0UZCGtp3Wcg*Em|v-`RBJ|I~Z3T;X_k5no zoAnkV_uLbF2WKwl%qbveE;p023th$ePPQWVa$j{H?ufDSkH{R!Uh}y8t$taa%bDSu zIcG)>y(P%J+=~{-+$qMHrA%6(^UZuR=h@3F z$~~3u<80(Utjh1z+EYG=M`1m(FaFH? zo_D#Q>xKC~-iq5?tH|Y$=c|``IOjf_fi?08cpsU;9rPZOmvdcAmFICoei*BA4LLua z)bE73@)n-OpYcM}MrCzoNmu@t|3Xu_6j$Lq2bn3EX}NcE*0q&0H;1Ze-`@WnPeab0 z%=|+9i|eaVUS7&S@J>|6vFe{V^Ciz#X5Ak+Mb2zI2@_C8?|z7XHIsoYxoxAJ?0ti>od1;X2N3JFIVO^ z_#BhenIWr@8FnN7!T@z+mx>eo2$DaXiNFd6OjvghUtZ$!tU zj5_b<7=FTrKs%MdS1lW^3R+-CVS9r@}GDEO(Q97L`!X=R5d8ejG#OiO4>X^S{5`87I5; z53=Xv{>?e{C|;L)BRl#a&N*F&A9w8@DM^f${_qMiH$-k0-?zlvkknTfqnR=o<($T|OCxFe>brFsr` z;Or5P$(3+|`WtQoaoI1nxb|B|_S)QQ(H(!V2aVCz^ z`-N{n-gkQNWW83%{#H`G5Hr;|_fL>3%MG|I-;A2*tDcNyn1IWW-C??GIp4BNpDC}v z6WF8o5gOwb^;pgsb*bDDjno^k9NDi<)Vn~=5B_w@Is74yQMce~yq!PC$#PF#i30M! zcwf%>`7ipQp5DF4dqeh$yki}%-saj(xDh#jf7Z)+o%64pUQ4`%Tk%zD{k*pw;&b~W z@4^20UwUtWa0y*8CYN z$d}@-BQL-^st@1&qK4 zdiSFZvK#-vf8tpT$3;Gqzrz^GFRHVfUN5)8tH>_eluyGUsDN8=xz84GZ7ny)?I@$4 zcgJUWq`EP(t5(I2dIOQ&@i)E-`D}jHxd-$A{|oZ}lxIBiG0$jb^eW`u%FjKsGC#M> zrTl-l(R)}vnWu5S@63&Bavk}(<>#CKX71~;{4Yl7<=)yX=l(CkxtG7w&wZYs*>mc* zI5Rgt$8qY+*nH=?@6JG;t^5qS;(cT;=iy{Ib0lZLcj|oaInQ%% z|B9!P`>#80ly5-J=r%Y(FVD?3c^ITplzle!GpJyw(^Fsbo?+%{CUvi$Y zO;{tJgC6oEF3d&zo+a|<7=h(_nKjunuj86}ci|fO8E%XP@}u||W!2MoBz~1M>w3!7 zQBgjDOYl=TK)xT@9Wwj7@a?Ftr`&gD=l$eVz08|U=!XifUBvg|0&GXlvCOMsculT@ zTacalM$Y?F4cGRQtMM$(UfWI1d0&t3#!Y%3;&Hr-b$Z(|PJV`OX4f*4hwHCKW_9-SmV7n#qMXls&PO4$HM2bX z&)@p@ppyRe{1u**dvng63cNoL;_3Pi;cYyC-%(Bf5WX14<8<{Bu84E7T>S|@&6l8+ z{2BkuSMibj24`Ps$Zx7Y#X<50+=mO)k8*ae6>=eYJ+ey_02hd^~qV_N)E*B)t~A6Q|$| zOwntNoA9st3qFrO!YT4%evF^wmiSN3&U+y*QrE_Na^8&x%P-4Sxi0^~C!;k!ROcOS zt~^BEfDSkdk66i{$(PCP zQ3V6>ZEF47FjRgKW01f97@Z!TSAT&&E)nPYh-f59%j{0knYod2Wcn)p~fne%Mq85)Vqp-uQk zFaNLo<>5F3KkMat%zd6SrVjE9Y)78&qj41e(mw=0$d@8Bw1&C>vLoakSclH~t*{2u z)g#hF?z_YBADX-N17~g?$~mJZ%h@Y)uH>t2|k@GRLFV9ZSt)6nukj$kw^)i!wRKLJKp}Rbg^V}RHABY_& zs?OZanbAvKTAgR}RzU6uD z#^rIUoO^hxd@rt(`*EJXm2!5|`*|#~OP|ftumL#-zw)^ga0triE#L*5b7wH##bV?P z+K4ssJf6$fB4>H#Tb{pS`c3&zF3y?DWAU2&G;TyAbz3x(bB^U4ze9Zo=j`jizw*s| z0N3YH=!)!0TlJdDPjY5b6J%E949^bxfO;Yx)SJnfw>7az&R+Wkw?}4gG5!$W%9}A< zK8<~B->y`tjTPE6_rHBM;HL067CMQRl3i$!pO`e>cxSf4Mc< z%NO(Ed>oFF&*Y!D7f<1Tagv-J=1ciy`2>EJGsi2-PsrnWDSyYeBWK_rJPhaR_u=39 zcW#BE_y8v(vpaJ-?}62MuR8BgkI326ws1YY>u{&s1y>+vNM?Q&%vYCi{a>_H7vTYV zCHYtWjZfs9quC1{m+N9GR_a}d3vd&rA@h4Vsv-MM-UF&4JNOFqa=hwV=6gXo?*Tb; z%IdY_iM&cbXIN(X8}bKw?d3D&cjTOLmEeNp~ezJ%v;&aCgS zQGE_q^qIkYy83;-i5H@}eh+mE`7(5pZ^L?QS6_v~^6R{gPef@s=gt)QNG^g`6r(j9SPJbhc|%invU=j3j< zTYiS`=b>B`Kgw;nH+o=>x)8s_L(o@l$l9>asKeZ;fytULxwafJE`{Ds@p z&+)mujXy&T9Hy>@Q{+axgGcgCZppvkLcEFhalC$Z?224YU7Nq>j(ocQ9Qi9epzeUv z^zsh0NAD$!!+KP8EqlqI{4RFt_2I_+9@oJ@c`xeA*K>7#lC$?8DPJi!<~MMxd>}+sNXY2?{mB-|4;88 zx?9eEncb?C`f_zWOqa7K*T6S&e)joujB@Q9PXA=iJ(BYwKf5zsI~#e{$KVcR&b-Un zLB8_Y{Cw(TEHWb>(fbAY|1RP)P4&tn_eY+cF31eZoXgKFKbzbuGm+=AH2S-ixtO0_ zo`=qyGcDhB{_GX?N}wm2A!lHIcAMpYkmsTj@+?k4o~vVU9rDcPdAdQ)&wZSH4rjJZ z;M~Kd_$khJHeW829`bzTp3L``XD;_@_N1%za|UFVmBS|WVaWM1RXvY?Lhj{~$QhS8 znR_hHUY^6ulRtQTKEuBvXJ78+?0pOQ|KBfje_Wu>z3?NiS6|GzA2YX#%emikFO20I zv0T0sXUOkx?uQ|K4e|_@Rp(xsBj;D1v7D*dAudOr(c6$+^dohi`#evtV-~vVor1M; z&aYnbGCB8fZ{)dK$(hH0x|TCG=SOyipXBUkuW>!sGm~@1oxnYieJ#&(FBJBfqR1Th z4{h~2@LGJ1rRvOv5y)&k3U?sSS3f+Z|1IbIe4n#VAIQ!1^UO?=%gA}2KIF`ViptaE z%+;N8X4MenJpB?okXg}yvjgTC&+d>}lo^wGKZtWS<$TE=w9d8Vn6B=Fi;#2YL9EvA z&doV(F)qNr814GaXp5_`9`_+cRZtha!--~l=3$z}Yvy0iSM z{QvJVTjdM&f5#K@Q2qw*;w|+z@LTr1Bevx1&Ykpfb~RJKCpY0Q_(%SVFXOxLgZvDS z=T?}BXR!tAuoMU25T9AhFYuq}f^H~^$r$Wf_W2@m8@z@4^`7PIG^g`t>Ua1eWRLBr zF3FduXUo~)evtRzP?U@Hf8!pkj;rIYv)%#nZN0{kMf_g}Bhp82r4Kh97eheB9^ zoAj>2lk%{0)Ph4Lmo3q#Sy=g-%>7Jpz2>gj#T zC%gU%&chYht=AL(;5yXAEqEG(;UoLr{qi1sqvCK>l)LMHfYR#ud?$Z`TIh=bdi(iY zd7h{K9nWKf`eHmIAB}D}9mDmi;C8h0`%cu`jmy=G_*l+6%bT3PFFJxd>({`Y@@j66 zgXLTJCC@?X0@*MTTAiX2`OWcQ?Ps{jo zti&4EzvbL>**T_g&eD3ES$8J-A@@WV{{d; z$!yI%mwC`kZo-dq=5`PMOx=Lnb8}3PALZ7Zb8wd2LeBl3o%k@$u97pcG+&BGk@<3i z&z>#+ia&9ox+Q*=pF?9@ghMbLm0T-?o8`>VY4WGYy}3}YJKn(E$X@oCepA$tpU`_4 zvoIV-pr!s!*T0j$M4rjfxK%GZT1TFVF7htE0>8U{J^IKwGk%oE@eKSX&*Rf@8=9+& z@qC`dZ@975pl4u05}=ifeG8-aW{fk@t|o zatA!*+IAF?PrxAgC$y87@;_Xd55n>IR(%pT<(#pf@d9<`Zh!f2^pr<%&Xt1v9>2_+ z@fZ$NPsMCG=f*WW5Kp6)f9DpxQ{@8u4<1Gb3(kJnU2lw>bNMAXyG>a>6x;OI@dj>$i8w@E z$Y)k@-bJ3~ya&y3trZ@@d&oQ32d?eUi`8Xtp1KKVM=dR1Dd(NCB1Xy^^)@5tth|(m@e%kFcOtvSB>mzzA1~k*WDmO1^`3I}kh|rYoOh~;a^8a$x%Q&` z7p{|k!L!KUG1TRX_|LVHs4PFgzoLzN82i}1^R);cj2<}IweiTi*0p>N{)gcV zt#uYZh?nJ0&{4h&UF7V2_i$ZgUnt0?2j=Jm)!U^Bh&c7WK_s58aU&`a0*k%6z>Y zo6r?~@gF|*dp^Tsn2X}*p`RP{NFIcS${~CuYRY5zZ|=wYBWM57>iy&noVk6coE@kd zcUEU7tj-hEng5v`IWLOo<$URv&mg-(&fIcxzLU&?%-nKVfFE!OR=J)X=3IHD+!t$* zJ+ic3&X&RI%y5nYk@F`|Ukk9}|%o`U5xPyO8rSXXA*6Tlv!g3+*&IjTNR7B36V|X=A#B1uh>|eR>x$4BH>*ZI@vFi9kUWCl2 z%)^{db1@0Ie~;7uR6dr!#YOUqoIUn_Is4%bbr<<@&hA}Vt|#Yto-f~r%Iciq^W=wc zHx?s%{VM+1^$p0(T!=gLuH=`HXLFJIcOJ#d@C&Y0XKoMWlhqA_dbzrRoPBvbzlCq~ zX7L&>AItc^VJp%)U$c1s=(7;2imPZiUf!3Pbc>=GQqh zJ@e>K^%2;K@_0zEHE-s!SdVGy%)@qAEoV-)=d<`Zy+`G|v*cWwq|OX2g_W48{)@+; zfP4bJmoMSWzq~hPCwfL*6kTx}es=8^&fG63kLAp(XK{r(?-@_<0ceEAcoS`1AI8^k z6RzfSIY-~dGuVOAu4NBtj(5>ZZz)ceGmmqIWhQ4=_(DJDPR{%6=4I5S)dyh#{#F0U zId2Bye02+-dqBPlA0xZQ8GI5BbNxDW&}*jtM?MIJ)lGSZUIpZAyOd|4kNye_ljq?M zWFI|NFK6Y4JQY=t_m8(+uZoT8?7?+8``bi*6*+g`O%Jp2qxuDY33-=kiCuCXT!y8t zABlnTR^$x(UOg3uAbZJe`q#^Cc>r(42zebp&gXG^+$z_{{qhO?3va})_!l+sihcoH zAzy$^_zDN=We>bTuEM9PJED^MZhRuY#vgMb?t?O@g;ID@|55%D58`XwpqG8Q9S)Zl z>s8{*nfC2d>x8@~&EOh1K>s5i#h-IY94u#l%pQNXT!~-h;W*0m?DjYEv-m*o8lJ;v zbN0ZT(b;_(a8bQS_!h3LcQl`(u7;BM4K4LLp_S|Xxd6Y<*%K;q{^u{I6Zvv{iR`d% z>&-_IbrUYmySO)Y%Qf*8PE&8-d-)Q+78l8P;xAl?tI}hN=d?BCD?_({xsk2ATmdm4sx*Hcn8@!Cq z@qzw5*du>}wQ}AUyU3H}{N2UV{HFRFeg|7{C`Rh#{o-MMM14H6A6Lb%@=^Moxeo-~oj+6VfxnTTSMJR`kBd0>asCW>_I6{9 zoO3nbYtHt8$V|(w(UvdAU#@3ISuB@Cef5EOOU^Eq=P&oy6Y5^b^Y%81;(hh2{4i?D z^Eu~a&f(+L<<)pG8$jyN58hVp!!gxpidx}N*`Q;bDs`D^+=;A7-|{7FCa zHTPT2oTn z^YTc!EdECBqs*;ga^}TO&NG_lKIdXVd?;V*Gpo@~-j*6UgZ;B_7tQneik#=C7cwLN z7iVx+n6i<}B%;m-&$AaEAINZtUNc`TV;&&*YIj z0;S~4vCN#lctZUhG6#>vP`%89%*UhoVq`zLRGss1n0mQ@@pNS0 zX~Wy~zToD#QqDP_zfagGpN{vi6rZ_%0$1i8oU=GH?islYmddp_J4au6k^B%cFKh58 zcpSH25)RTo$7k~{Sw;Oadg$GbQu0OkQ_h~4U!TYi@Cbe#rI8t4K(Cy90dlUEQI|$t z`32;C=yi3@k(_~V@c@1Vl~GB39na>GC@R-QJ^2XU#asAuEWuXP(90P;kKe>Lz5Uf+ z%ReITliB4O>%V}j^akNLc`veeOyF;jGyEFYZkMx{-Ye&PUnQ@WOUp-d_Jqbf7w5b7 z0rGx!Kficp7iTK2Mg_)~Rl?uses zqi)3K@NpO-pNf}q4ZHPza3em6yQ2{Hs<$KW!8_F(@SJ)%--mzj7`o{-z)kWWJezCrWym}A zv+A0hUFcrUbgGlw&?-jz?r zCbUIay*yu;H#5~ckZ1R7^<+#}ui|&n9|e%_GP~}9+*SQO7v~P#gNx!@xh>~8$urbj zy#<*?yOHO6B{JWB!X-X)H}~bQxIBu9-@Kgz8*`{;N45vC!}UsPL zzmDvsi_~|?UvOtW8Yf_=Iy3)vc`Keoo{37Fb0xEB1U^Oyy+1J+hhwSUEEGaR_1pLr zEs`2X<`3cSp-phyL9lhQ-5JQl;^Ns%Fs3c#_J&+l2IQ+A3 zMtsI4F-#-#DzoKyxel`HW-io|UvzCD7e+FO){khi>{OaW(!9ne*S{5gd%{?RjT7M&2KP`^*58 zk&ngo$eFQ?-$dr>m&iL(9gQ@;Uqp#-p)1 zv$}|!caEIncj(=w9*A+suR&;y!LC*0Q+OSJj-Gf(U7fS14VNFo{p!o{sQfIBm%HOI z`BFSBXP2rbXJdedO#O(|D)4I?j~mqXDuH)#jYSFtXD_->E{00-Eqo(#24}B)+2@Z^PsG>gh!%KB{{{XF z|H`-W7A}vUkn?yFpQ?Wxev$KT^rL(p&R1uDuf=sS2H&EBYdN1^MPd1NzMh}Q%Xk&1 z>-~(kS`782;_(h%fuE~6vIy-PVepvksSH&mz3fG~T{&lE=k?PA) z8>{kp6huY!Ky1eY>Kgu?mvP=rxA79pacw1T!|Cem57~+KsIwzpsW%*5FXs@Sl7Lcj8*S z0ym)$j>>0{zZ+@81CaOe(_CwX6V=(Pj^+#1PxC{ZziX(BH|1w>oPPGcFXTpOjHb8< zW6;jE*_@p!f1j{c9*O*avfp&c^Y^0UhKpYQNcBk%zRw=>1!rbo#A{s3IaQf^sWUh7ynK#& z@+e$^Bh)|R9UOxE%8ZyOXK%QPb4FIt&vSY%GV^9}W^M-_g%&<@7w zrk;~)`Td;d>s`*X-x@oR9i@(5TX{ErL}pmtVfyJWm-7zsCC|k9$lUFXoEe?ePsroA z0N#>+=KeSaSE0M!v&ig!5G(cO@juAC8OJ$q^PcpbI_GcBj-3Bz$vsd(9?UsUaxP6* zw?#QQb9xW5Z)U&yO|IhFZun>4K5z*SM`rndD2ejwt8q8#pp#xX z`ZdlBt<5EHrE4#szWfXyhGxinUe4Wbkn^fE4n`Yv#f!*{oaXukXs%wy74?S6XUIb_ z7iX)lMfQWdZ)M*9kshCh{@A5nf`)Q-(wun*sN3`LoHL=3{F7V;pUUTBt^5qKL#$Wl zon{et&^ws>@Bn^;pH9#9s_L8NX(%I4MPvCD+#nb9na|`o_*?xPZ{-X5WK@<<;hi`Q z2jOkB_WOQt?N7N0=UlvzkH<^66mMb+df)>TK{b5svoBzZoHOcJ`55^i943E{*W}BQ zGx|$zfKT;5;$N{*F2t|nPI)JP!k^rR1u(M9vO)7H8+F!FhMs%O~PL zpBaRQcy{SA5`S0ca z%(K=Do7DN9=E)P$77wb|aPIFCa%;|+nR{*;a&EoKYxMHx&i6T8o$nyeVD5`NBiA9{ zQ5)`sTjam6S@xQ_eY*`LI!)Gx7&{lbrjnEWeAKv(1t3dKGf-cIDjn znP)i%^4&d-e3zNCt2y_KxNjGFQ_iz_0gl4o>YuRyd4@abosFaAhxtW3i6hjn@$1O5 zvzNQ^%bf4{Z*Ji~lKUy&$veoNmYJ0ADRU*y%uswHFT_pwS)F?#XUU&(=5N09Z}F^t z?xP#zIjD`VkY_R9`BJ6);QA{Lg&KOdcRF;*UA!RUMoz z=l)rT?&?dCGjW`{y?i5zAZJXT@jS!Bag4eg{y_GK+{=0Puf;jYvspl|zMS(dvncoY zoqCy7RgmZL8P5FOrN2_Xi3?(zJdqDT_PosO0($4;E;+OHCC>9*p7R`6;x}-dyn|bC z_O37aURp(re&{aA-{zV7^-&~KF3^jWsJizb`6<23nw;CW;Y_(9 z|IQ_lcaqGB&U%?$%jNrVr2Gmlko#jSZboM19OT@&lXKq8^0~gek>5jR&wrRC4?x~e zrl`x~P<1~P#awlE&FllsaIAV7XXmSyp86opem6nROurPF;q&n!mSH^F`ph-19gg1W zyErqsGM}Q(OnrfSa?ZER_u29vxsKypTZNgpPhE`P=Hrps{hKTjU(f`LsW8#d5uG_z?_| zZ{_y*Qf|r*@%3Dq&&5x&fA;N~W%w#Z4gbD>oFb6%gPejFvyTK`wBjf>?a$ouh5eopU1 zG(jnxt^XqzL3Y&ReWp78QD2O|XLFh9?8dIG7eFnfKE6D!}PA?YRDI| z6Jsz!c`AmZKAzIMoSXAn&O7X5+>V=aRsE63j@bd{>wTgAh+EM$7 z-hl0LF@6PkcW9u_-&0(tu8Vf+0_r>Df*7Q(hzfGvV+QeuXpDRGpG6_8!>hOm|MR(R z^smq4-Mp84!+ z=jdg2e}buU0d$a?Bj?r=>T~5yoSBokopbyoxd(D)+^EY2>mKOe9Dof{zMKzVFIcm2(1h_jGq>>vDvJYREsCVO4x_aR~kMEG#_MOka%dhZq_{hFlQI_+*u~hE|`A!^)T3CZQIM20txKW;noNw#Z zC!>hEIRBhJcjf_nJ2J~Z=fQeK`8Ljsoq!i{IhN^P!ISU{zE(er%scq7axLC z_q&TY=h$YPA6W&Bggk9?IF9UgIfvS8pZ%$Pe-&T!lx~*>$FHKV&B7EX!`u zn2$xy>W4V<|23ZM_nnRR<$|19{)?P*=Kvm$>`Sk^He4QxZ{+L?IZu15&*XdfH}t|d z^zfO^d>r4#Z{b_iK}U?&-+>2Z|Ll8jex7sQ-pi9Uzv6Cu7xL?~=3Q&frJY zFY`mZ3fUK)S091e>b#5Hz-`s%qLAE|Z^B5q6sDpTw(DhwZ7KgGXVE|6~Hs@XEOukfq8=r>udf%%@$kpVZ<#ux3iL-}3 zt*(M^^)AL`^4Hv*vm@2uE7f|7v8~Hui&5zcrhzr$Ck^TEk&ihWDlbZ71 z$R3fs<9EGX>aM&A{pCE9hv0QIS6_>(&=&b_ay}m|XXn_V&dySp7h$*l^~el(ntwp% z_s3}G`f6Usi@6owg8!i(mBSbE<(%i{MlQq$aCW-PlSF3O zd>608Z}MCmF7M!+w^Pvz4N(@$UCX(dXa8UPr_RoE5tqW_dMkK5H^9Xxn5@@{XLHWT z?4>K!51=PX;#PFW&!~hNK3jrw=Cqx9zC1AM4H2j8I@a;D{M+bw_Z`VwT1%h`}~^d0)85waev~&=Vga z?=U%|e{^k@`aI4#{kD7op2g$HesGU#^W_)0J?hBW+fL@KxD-?I0&?~p!PSUj&I|~_zM1l7vX&QYtCNv2!E@7pYu+&mTRfA`(7vK9jzJ<)jI(v z$?tMI&aN{;?jaw~9o6OdWOY@31D9Zq`hH|jx{L3{XE;rLnd>LWMYy55qkNA%7)?WkK!`m?2boqBX#zk`dm^y zn@iz1xh1kIea|=Oy~o+luI4=GH&F|@5HH0Y@`E@`&W@TrsR8~{->siLE<03J^&n*T z&z@D$we1*#ZTe&IqudH5kb5Ba?WPGym{9&TMUs(dv4f?<4nD z&VtOQFFAkqw~G~{PJmovAryL`caAop#1u7KQ2laRBbv(Hw> zxoD2h^&24brJH(?yc5gOPn~nN0k20hOh8%W45*GaJ~xYVmSuO&9Ni)3+&YJ!bS-mu z2p_1PjO?G!sjrkb@op}F5ptgMV)6l84sT$wx(@!3DW&ivC=jtxKA6;UcgK_vvE53{PruH~G{JjuD4v$3=%X5NS!n7 z7e11AVmxMG@c+?hA7DS$@BhcYWtP$s4V68zS0c$Kk*(4~NeN9_WJkNCQXyMZv_xr$ zLK;enqC!d9TSNLkUjE0=(erq|&hz@*eBJkTo$vEz^#(qXd*KoJUY^Pi^3~joFGSAQCF+e>t=^4Yn4%uXdAA(OC!-=R z(SL)-ab@HTslqim=lVZ#DGWs!_1ioIhst@6@!!5P@nL??*DU3acn!bArBGNt4Arnf zJ(FwkN<1rPe{AJ9%KOX+c@@UGHW^o-r}{wLFW2XDcm&=>XJjXt#wWS93~$Qak=>{- zUPd>x!f5@!FbS6-?;(VnHNYD6&s+rm%7rjm{telud#j(4b6(ft=fxd(6*)IY z@oU)RT4u~}ltIqPF327*j+^_;k$f_8_O{_($c%4+>@InJ7t49pGUuD}*SJZ}%*xK0 zS@5XdX*fr3F#m(xw_o8VJd3Wby@H(ixfc)D>yFIZhxIeVZ{yW^d2aHI_d)LUSGYGn zg8A|_oaeV8G8vW>K&+#%)V>& zvXA?3-+os|{koi4l(}#aTBDFU=h?OL?HHo2if+iWzeF$dG;?E?{0_2{%~V&Eui{%U z519i~F+xAj{|E9y&U;H{(o*CZJ&N1m0eq$|$F(sKf2#|ln_LP<$ro^4UWK{xx%d)! z&&j#Y0egj4j7=GItbL!cO@`?t`2oCDe1!Lw*BGa5%n2cUt3+v#+j@^9}D!i;#DYC)F*Hv+-gs#QWiK*K1;|-lNDn?z=ozZ#TE)qI&D( z>?Egh-ZNM5H?CFWpRfSg%ku6roM)q;-kZqz^aAI69>F8kC&`oe^dO(iC0xs1*N=Zy zFXJxioL{x%U3?6l#|Jn9!}ROpP`Nx`g6;BT9>%AliQJ6uzz0|a7n&Xe;wA~R`uJQ^S_4to4g#) z$o=?bTq%EyI`TJMhJWT4`7pG{X7wkiB)^ZM^80ukTQCND_4ntyc_a47<@p3O#x`{W zJS<;?YjBUcGj`wz6v2i1dH>rfAA|L%fFt!=;#2uTJSLAp1H6C}^|EJFkbjVC@gwLU z|AHxU0oQNl?2&)))5v@LV$P0Om=E>Y&-e?@KJ|p$0$-{Ja3NeL?}zMw6Vw^n52&kh zcIs7H8`anI_sE`gwYrO3o&P`?y??l}-b&8S_@MkGp2BE+f<{=0?2`BJM4!$6w3pvg z@5UAKd7O8zrSeO1ZMiD<$0+1EKMXVFpYgDqpKG4kztLL!6JH|d#U{PX%FMgW@%()L zaqT^B#m)3@k@IZ#Ro{!t)E{Cj?nR!rHCzmNMlwHYqZ9Ho8OoVoNAs(k8Tvg2sq?cq zM9y414JRS@=U1HPC(mdrjFg)qXIAc;DROQ6h&&tPxep%1=eW|>=h@Frd9r#49zyQf zUObEY^Dg8$Y{TRDIX(%c<;pg_IXr!LX%~4CvUYgxJXWaqnmii-+x$*y2?S2CDz$?7+8k5A}C^GtS0vb!O9E6jhJlNj#pH^C(P{x1+P1U8@uy zuD%NU%d0rE=`{Ir48>ezcIQ01&$Ya>t>a5jM83kco8{7cFc0ATdV&8#_UF70yrq{j zZjj!u{5&V$@3aW3BKfq)1C%9jJ0C(dcbq&l{+wo&K z9qZJUQ4Rm8Z{TD2BVNFFbI!0|@RYokCt;>MowHN?D!;&s`5@kn1@bGXERW$cP)j}p z=gN!9X_`j>v5dk%ltR0%h?C6=3eT9xjpaZ z#+>u9qU&?yqvgBj#3xL z0QqZv25+H-dJKxlbMP(>$HjUd;tweIotxW)yzeblmqX6(etM_iTjV{sjsCxAi$`$+ z_PExM8}f-ffg9jFoTARZ4YWmf^%3eaxvqX+F2udnt&#n+xL!q6l1u1Kk#kP>=daY| zN$y@cW zL(ce#+(>UKpUuZ`cIeCH-l(9i$JKDCd@UZpt?K>wch0-O)%-F3(rb^WQ3n_3UBDM{ zd0xz&F<73A{JTKjb+bqLZ{Obfh5n23>$nryQO0r^{)x}PQlGgHd0#l5n;^fzle}Ea zJHs%puda%tw)#ZCIL>`ZP-!l7)p#bJ$9{$7z*GD4H zNS@VRcnHn>o*(JWl5?gFpV9ll0>PQ&q~T$%sk=lCsdjY}{`o%7-@{t>zFX6j$e=U{}q zid%6ne1+U|_4G2^F5=A2#mMtMgYWa1lTcV*gPgU$@L9+VEW~eJ&bgRhC-eKLjm-S%dUH9m>?Zj&6{@wEIZPsiQz zxp)IFsdFZ+movlOK+e@Z$g`i_x~JR*ANWi|p2Y`n_Lw}|-ykz1bMQdda<*qD&U?YX zdO5?!ab?bPI$Ay+{p9RVh0qm!k#~`Ht{sH0ksTrD(UsWZdK>lWa`w!=`~n8*e~wM^ zWX@T080un`I&z8vuZpIDZ^1j_8?{CA@b&wsaKi0aIGv;AA zvwS|+;w{`89p%Td3E!iX-ZfkjndOtz1JFml0cBBLeH_1uPf!=<>FvQN`41j}p7NRe z6dKACut3iH&kQ-|)JEj|8KmD_z6>Mr0>+~fMx(gT9O5%Ov0YtIot8aaU-_lH@!P}A%Dz!_(FUuf5R1dDj$yhl;OgWMK(;wLO~tuUX)uX7LV?{n?=UA+tOII<^ZS9nwY+qLq%o6F!c zEXS!h5*uB6iSs@;Pi`!qz{l{XJcak?mpMCS_M~6c_uv!^(yxFkrr;>w_N)Ii`CgfC&~xQ{kWz2J^4H|!ad0MD?h9J z-176w_b9ti?vXNjIWO|u=KK9M2FsuG8qW7T=ihSmWAZ1QdouT8o`)}a74p5z49|0# zxt}?f`?(hSAkW#?$j>JK+w;91h@7=~p7OoU&+&D+C7Q{(XY#D(p3gkY?48be#*Rj2 zdv=^#c^h(X48#D}@-xVDke};gSg7{^U&_1D1HIMVcn|-=ccP`78F&P8-<0RM$a8xU zXLje_eN!&N+2eBd(Uo7R`F#YlDL&n#qyWk+ww%hV?$d)9$m zUavXlxu1YrkTapY>oa+SYn^bOoaZ>__gZ-o#;70R@41=&adKwPI`t~9kDTFOAY@&=)Tt&-)V2b9@kn$Q}7$+=mO0S(#awxp*+|Lgx4eJ`kT`IPUbhr%)1^ zK}YCizMp~)xC=+>t>$4kP0o%!L0*dc)KhSzT%Et;GOlN4=3OT9v#efm*K&@v##nVN ze307p-uyQ*FDs~XE>BaRfkJxs<54urXI#s>*F$RMzP-7sP(d!qU65VnY#yaofpexj z%f)b$+#c^>Bi_|p!kN!!;yLxXyaIFZ4`!fw{!G_?Mm5~6w-&8X3G?-u@(t*UgVdS* z*-1ZCXFvH;?=<-g6u=C19nQb^OyPgj=W}De7ux&YeOd{7)HkYka(g_F&yk(-Htvrr zT)zc*|C!E@>b;NCaTrSJ<=ymVd6!(2OLG$pM?dvsZo|LxsmK{HNWBu-4fAfA^Qww^ zF^+dVXZktvS@>JNmAB#mJcF}*{xTkc{L0R7qx`45gXbW-S>8z|>6KPrg3j{0xDAuk zA8{eh`${h!hwKbL>(9g%tkbK3zvQXBli%jM(Ev@*-PatGnrrhVK2r?8%Vm(g`6_i4 zR8rT$EAnP8i`MdAJP%LdboIHMGxu=59h>$3!+QBsZjnEW_aeK{d-}!YhA4sS)prZeuejIrZ`;9m0t;Tk_9r8|m z9Cy;oKJ|s%Qy#=Wp)MM!a}#D4ctu`@XSGK0Y_!9xXn`Yq<_^~i;|<)ecQu9~`|XK( z)lga85o7U-x;p+v-btU-yGLGtU$I?%5^k4|<-BWFM&8GUs-cCiz?CSamwziL#}}&) zM?svdF2Wz+IsBk$oYPJ$(a#1$oU>*20tg~%+1fU8}dwLR@9T5 zah`)_dV_cfZ^aVi*~x6}%bA-wD<4tkdz9}%cCEGC8aL@>&*{W3sM}zJyn$<>DUL&C zZAo6_T7H(9_y5S5ANgKoM&;+WpSmOG{>smDF|tGD+-a`AOP$$P3!9LiV=1nIzvLe{ z_w7#3ocT~MKbJh4x!30EHI_5SzQatMi(4@gd9HJ&i7H|n6ADN1?8uZGdIsp?#Y}L(~x;|pKIsH*~Ky^8pzk^4dl$o%3VhM z;YIR0a^`*JYxdyr+((_+l{uJsQk!qV`Fdw_cF#I;=HWbb8GeH^ONt_A@@hPdoJW7^ zl}0c1TR04vt*dw_a*my+eqC;lmU7Od!KkgyE`Ac9p+1ze6J`GOlaIwU^0hn;6_NA( zN)*!n9+_L+)x$YwZyz3uV{il>aqT01j6dKn@j1R%XU66HDTn)U4l<_-@y)JX&DSAk z;b-d2$h+4jE`tMIJDwZ!(cBpaV+DHPYyA>fi|i*0_-*~n^UF~ghp1oWQJkH1s=NXV zF;Q;^e}>cKyz}Keo2Jg0mHjv8?mOzdH%^sD$ODmGB4=QKJ}xz~7nbIe(G8zq3a-Z} zR0=-xCKut;eQgi9CokoC*k3*g<>lUdDUQM-e52PClkfquBmAQOH3ncSHtP?@Y?e7rdD+*zkC*l8^AETR!_=*K zKEK7s@P{ajQ}H0C=%2_BV>xb9KZ6tSo_asO{|v68-o~eJMZMR#0RPT8Tfahab){-PL#EK)i?@n6CdgCdpr*k^FHoKZ7UZU`n1Jk@UAQHxx^@YU#Q(6J4(F%Q5KpK#;zYS5XW#xuz6Wos z7jkp7$K&{2?`FJ!8tU`78DGlZ@`ZdIzsSGx-+VUe;z<jnGyefb73*)f2D>E71vg2k6Kxd?xQ)56J%8_a0b*vnSre z9guVJ2CTuK>hJKJ{3RE_PE5oQy;&HEW$Fv?hMal+8&~14ksYiVChMJpvv8IEc;0|k za&Nqs>~ndp>tPFW9)09mCAk}Zm2=Ky9%N2r?^&DA;YVZ#%-MJ@S|hu}{rr#5pNm8A z8qU_M!!!6Lw3aW%E%IOd2w%h3qZnqY|Kps!yX4O?2bnY3E3>mbq#oxpHTh_qFK@*> zOjCEk`^a-N05kO;Gkuvx6?-;mE8n#l2ixj81Z%{UP{9Jq+0?c5`FgiZd}7*(2}c zoWXZ-e%*w*@<6_qdtf>8jPyh=+=qYB+Sj*p?dDv^a1=vk`lXz6AUjrO`Dg0d_}tfR zK@l`n=M1@7^q=C%Sc<;tt8kRuolEo2JfH920(f7}E|K?w z2h|JIdvFPIE-uh}n=j&{uo2mlN^%qZyo=n;f2vR6YC$hEy?j2eJ^_!)mmz0%UmT7* z)j2~x;u~?S-e%l}x7Ee5K`x6acn6hz{&?<#9j^Dsb#gyUk;iZkWM5jU&R+hI`at{D>(Jb_^*9=P)T?<14n=15A$mFMZn->){Ee}$b-*O~NZ!ruIXgjJdA>Xq+0owO%kh)`amap<-J+1(8RK006nU2~ z!v5R04-VqCiYIWSd>XevK`d6c#CrJ|Uc&z*<3v=&!TQ7SJ$9-eNY9^+q@0#3oz zDC{$3`3jtX!_h_W7Q85z#ys4iUd2LqT6}2-s#9Lbp_`yRFz)A*?3BQ zC{`jnbP2tYScB}#6ZNmdOX_9ZpBte8I;tl9;jZ6 zkM!~k{e&;&>_qkC@mPb*>&&;KW^Z#QnIies-GA)iZFC-YUKWqj0180^}K}rOvbcjhvZ27gbOnH|Wnn zX88cFgx>o9VIgMW0AzmWxnIl&;x+j>oGll_P00R|+1E+`5Izn|* z>hAhabDsI^Dw&IaqKe*s_)#9l#kdJ?L2Wcu7raX8nVp=uIY+)1nXhFz`&lpjJg;@|l$_@{JLn|!B4p+ttM>`+ zRIfrS9E-n@ne`Zk%YSoMY>@xui@6eRmZ#xQ48;R_4LIjOA36K;1U>^bP#VX${u?rL zKE_PFgLoHy#OugDF-U*9oL_fgA{MF-&S&JyIrFt8=bd8%kHuS9hwKQC`CJ*?rp|eE zE`NyokoTC=c(UuK;8(dE|Ay=v*}uNi+o^t#NApUwlUunqO8y*~(=(E4A$Uo@Bfdgu^hM6>?0avyuH5&I*q(PG zJJD>-SCBnms@@b7#q)Z-xIO=X!tzttEkBEM-~$<25pAH|!H{qYQ(E!WjQflHzU9#$_xzKLJa%h-rd)fIR;enDd_ z(fgFw<6YFkm3Rkl;2zhX;i32gmDL||_Tv*dyI?7P5_?=*imLJ@SPqx>{o6tjZikhw z-@_C55tKmA{R5D*`y=%Z9FG@qq|g4v*YcHoKK9C&Bm3~x>i+U*o{DOit)7NQ&;}KJ z?qdD&aw&cRRdI~EwQJ?%_vE%bl8?i+@^;)T@6QcUT^_+p`B_{g|Bky+S^Xj!VSm5p zE4&TadlvInpSgm!-~l$4+wmQT@p ziJ#;WdR^oTGQ7 zd?DB2!;wAqadmdKvg!u%Ud}V{3Fq0EC+BD07nxsAsWS(1|K*t&f_yLXEM(8f9+eqZ zNu59QPx(vaJjmIUd6B(hkvh-q7TkczD2bi=Imh!oS)p#l5_i_#_=Ap>>dX;N=X7gO+9?Xo)xs-c6&q4OkoQ-*Y zHXzSH9dy>qOv=4>4EN{UBR|NeapqFa!4Ku!cW?z&V zIe!MLv)eY4GsALE6+rH@H@PL}S;;+`S@;d^l&ACgI0N6|d5qVe%QHE9=&SN>`8Yg@ zo78zu@4z)Erq1)7=PqY)cD93fE;2Vey7mcfQD4uQ51Wv)ZHYRwBQq%HUG|ZjU6<&s z(982tglph1z32HFzM8ur&(j~=i|Zr%P@acZ^>!g=>0g|4dNi+a?Q1+FXAhXpXCpJU zta=@1-uB~Vc;B_moaXZP{4>9fIe0>ynV0#{nromj-p7x4&h_gtRL(A1S}uXit$I9@ ztK%(X?!1f}(adK*;+pys`4D_0@4#!wzEy?a)6aa`gk$B*o}5*C)q7A0i}V{HGcNB8 znQ1TT|ANfp%(d*QPpkhyDO`cyeJ=aU9KKOK2D9X6k$0+>)aS@+k=c0zev^+zJ2~$S znO8ZV?!!Ow+sHoGls`mWOmeLzXRo|~C#!QNd@Ucr$74ATRA+83=WU$xb1?R*58&)T zC3zt((i?+!7x?D`!uvz}Zpj={<~v>g*=N`9ZAA=Xog> z$gMCzE{B{A<$d-x{#bt?iptr2PvXOnv%9VSX89D{izezOd?&w*+H&@Sd*xXej$81# zei`JP*@=twc4Lp6cc`&^6Q-go4s@*q2H*(w(_ug)4IQjdiZ&ec?a7t~gwM85cxuDXp2=;o zSpQjw`~HpOPhpKb3>8ooC!m!6Q8-&Z6?b5z`Z6rTbL#Bom5|@88Ft_(Wo3R8{jgU3 z1=eGM`c3?Yzt!b@Z41u6^gEx3r(BzawtAyD`*2agW>?nOTn`_gDU& zx$kp^_2#Sf^8Dt1Xa1ZOdincgw$76Cy~_6}dq?I`V|C_n?(1@LW>Ee&XKu8SbADtt zo~5@N_2mcfHS%}J_bm6+=g7}x4L3!;XZgG3zRH{#gC*F3e2;T)^h4(J0X~;|wT5~O z=ia*>9o2m}GpZSKALo8r$+bAoK+dNhk)Qb=UUh3h|a?yp(Y3srC+@|@>Bdl@HSEz0O;t}WpF%DrD zgPZfs+!Y1ni*ba!9pmLqXd>rax((~mQtuD!@6T<>r(hbEV3h04_)>f+KZ>1lc8Y^J zyZvy^ne;2y$1p5H2fYE9Eg!=hDEU#|*(hUPv~f$QKEG{IG_-@+sDjNBTRqlC{E;p|jhT>BYUsIS3Y^6yxUPt?^= zSZ>P&xg8hsb@%ch{H?czKgMi4ksFawMApTY<7dYp{M)kE=!Tpb1F z&OC%q$H{mN`FDt~cn*5F{sUjYZTL|xg$=k}o!#$rIsX=s_k$Ypb;v&aA9unx*y^)W zaJRgXf5YwaxhN$M$6maInR?gp4LB6v;4i&zah{y_nA7B^<)cwsE{Z~ORXm70)m8DS z+?Q+dI^K$7P~GowoZel?ZkKnoJ^CN1^N!JwpH*Lt?D=1)510Ss{2N4Vj8PZ{_!Kq+Ag<;sbO+1N~n)@2pqK`B{Cd&hz*$PLlII93^LO%Cojro%<{^eI|;b z2@Y}XD|C{lbIza3_||-eGpq9R%v{`}cP!_7nD5*v0y!P#MMlwXrSAv4=JPpo^({GbZXgO_x!%jTNzS>S=Pf%}ekOS)y5ceX<1<(B!?;iGi|n8|BlE0e zCtRaf#`Oa*S#AgIeP`k({t`dq5!aV-o|(*?YMgUq5O!gN-Wtv`nrAZi+j!o=ncYp% z2Vd(o<0JWdbdYn;t>@XOhj!}R^9P|C@(g5For~;_ITJGb@|d&wYc`lpbIM*_x58=$E%(hngefTLf*V~0- za1(Oxw{$(TFZ1?n%)xE?hjBSXpZaLj7R#chT#H-t zLQI#x;Qw$Yh9mDYl~7bJ?Rs{N%&5FCea>gWfBVk6%&^Rr${H)s5t(&uaT(^TbH3-j z>2h^;``zjp@<)6Y-@u2Wx}3e}Gajto!$;wExfbf^sDf3=q~?`m*gw?dCbEd z>YP*Wq7KICwc^aqi?|oJxYS;{1ZTJf_KW(L!#7oI^P;cI&lKf5pG@ z)%-Z#!SUFp*N-$wSQS9vKa>EDLDi!S2DsEXeB(e>|8 zLVkpER-P(vM)s`C^K0ab(HD35Ol{80?Ud zkGn>lcZuFSNv{WAh&i|zXX0SI;@V{VBu~IPc_${KoVpE$BJZC=^nS)@_50W^FXWYc zAhzHfb#tFRfgeOkoS^>RLWhjRMc)qCXn z{5Vg=3V8|tfv4qfa22k=G`)xUDZGJ#>ehar?72(SdFROC#qV1M-m{6DTl-?%n~Pe%o8RUd=qctZUtE|srG-ZA&%d-QhV z2f379W4Q~Os5@buJOKa5zw({vF1N*}_&_}#zsXCHcZR$Rjnq3|{XJL3`*L>Y2RJ*} z-FlzMS70v+sjo-g|6bx-^(ylVxKh3v*%61UE67iB-cPdwekix$?Bf09OYo4-Seb1BMoak6|rSLDo{Ja5f>KED>B81jtfnOK4R42I)bEJp6hJSUH#nd=Ae z4g3JA$PIZU=U&X5%sk4zFo^TB?ttuNxzFbzKiAvvoa^UcnVkD}-~B3Ig;K~fu>rYX zckwQ6hb!eg=Y9AZE{5DEHzB)Op2Z<@87z>u@-);z=1CJ|XX)iy9bAM%_0HnlyO(k; zTq}>|Hh5I_CHu~^!+163BF|cOvROWpGbD3+2#%C*;ZB@q_AvQDIkP7B>G$gF*{$TM z$c)MTnzJS6M$V$^^gr;q-2a)knVDOV=cWty;5?%_|282rCC^{Zy4?SH)@q}Kd@?UW z0r?2N51FOsCG*j|Kbp&*apqy>MkzVZ(|4Tt^^o2`c?+(^Ds|@hx%^&?i&0ZuAD>}{ zx+Tx#*2r9Lpgs>dce00%*2`X!y)HY-{^$nc4)~<3x`#i)Cf9$&bmTmE38VGTN9O;Z7$`5s7z{w(`##t2gPH2g`)YhA#>eum_d@*X^GIb$zMsIcA0WX)A%G-G_+Unh`&O6eLyjCyoPW|~P zEY$0YkK~`wMlQkG&7YT#!6E8-cnAN(v6QoCC1(%msaKJw@QZpc;cKkN2<*^r#3M0S z{ul4a)wqnWdyVf<593#OGFr+*@kXv8|30=+Kj+?HKAq3NN%B#sA(z2_=o@p z+@Eu(d^p}$|A_K(&g9BGOFfDwgJ-Ca9~@4*-P`>C@N|Ej(TZ|OBqH|4w&&EV-M zDVO7hK2roO)fe)Kdd0CvuB3M+KaF>>Mt?cqh|A@Am@j{gQ{=1h2cAP?oT5LLi~4=K z@rmlZqgLf-)m`~k{uVDI`_wQ#6%}2(lsjXBT$VfWi}(uu+qY|7B0P>Un1tpS^&71Z~jqdK3@d6hG*nmYH!&B#4H5t)sdPx&+R`yYqQirvVw zm>H4hf%g4*nFYCT>Y%!u`zrTl7v%4sKeGwSVk+|N>dt8IzyMK=mwSwNG@d~2 zzen}{!sT*)HhC^GoAYz~oF72Gmm_!`O6zBS;Uf z;tM$Uc{$#TdB_~dy_wnB6J^jueK*g+wA4OxJ70w9$S%+p*^x45X39&r7uMr(^|hRt zlV__H@+;?d5zaHaojc(NpKZXo&(7i8v(4n3)%U4CRA)ELGjP8;&r-`E=Q%2hbJhR% zJPuKxs?PI1NuG;5lgstSV>Ql0IlPAK&h)>n$c~j~;xUa^P*Q#ZFUUC;hj2l4MSLXR zfQK*~nE~ISnB0OF@|&EQat0r%&N*|woM&P!zSrA`ov5L%!WVOPx@PR_JAp zoy{B6nN`=wcj6)S*T{U`p<`x1Y= z3)wr1;8Txh{6O_A-y**_eT6Fc7`<`yzWw zef6z!W@C1;hVm-8Bi_R4sG#>T&cWU47F+|>R@>w`n zZiw=D67S&#baL$=?#yL;O+C51d_A(Of6E`})x-(NKHiUa<2~0N!0Y%}{UIjE-{DbA z!HGBt+gy8&zv6rOKkSf8@D|Qq-41!@{YCE^y$ZNd9)O&o>-cEB&$&PUf$T>wbKa}I zaDJI4v?*1l#ZPL^LqZM=oPdK+<=d<^f;f8bR4f4=ri z`9iTRF2^+7i(kD*H?$t-L!RdN|Vg0DtV}_!Imi*FziKxd#$$2&sN1(2LX5)06t}cr_|JmWQ3$B)HxK;;E@i)%aYt7BMfY0Py|6ctE zci_HQEBD}bxCV1E5t)r`&;lR&O!l`xu#9w>@C^{(Q}&>eT6lwRiM zi+E7}2zj=9@nN_g{cs%y=x4{tZkd@?QvEYtz~}0mkFRrPV`fPO?2=n?-cPpi5M&R_ z`SOr!kEu^UQLMtZdfhlPJ?}hUs`ns!L@8umj_3RN0IWqb^_j@-(wAo;GkBnDzhI$y zIRAyM$PV|uUgqI$&ih(R`A>O0vR9qU*(aaZFM@UIa@+-9V3c|ZpNv)VRKAG+!nL?m z{W%}Y*;5P2Ia>;22eJ!P=0kZYYRKcbjL-fc--&1Ov)({{0`ugY4dbx^^>BrLZN3(7 z$@d`Va69ggcl7VZM{-U5>O4qYhDY!x*o4OF>=>2g(#Ve1hu_2_I3LgWO!lPz_-bUg zZNzh2`xh@@BWCMO!3Ef?-ogtw?E*nCS?sLNd-JnAM_run!hQG;J@ppz z2EK}~+E6uIO6!svZ2A zdMi4s+w=Z<*-ff)O=Q1ap?@jv!dEDy|1oDr|3h9Q?o*(cE5M|GW{#@ zB3@E|%Xi=dc@VOnb>Y2wV^K-YJHn^@J&xC_#+TzB`4xTyb$sR$^{?_~9Etks?2cc^ z!|*sp>wSv8av!vj55qY5LmZFqP(d&I%R@W^OZ4vL%+V(DkH~(JXY?$2JNL({n1bed zllU1Fm2*DlzHN%k#d65c@F&-WoGoXE$$Ys5Ilunmxvu@gIb&bvoL_mya_?u4 z$T>MrooBZf@6w-*%%Pm)BlRxES9*K+XwJ+ki$cgfSxc`i^4!kiPP~C9bADD2;8=BK z3pOO&*73$QgZ0K9Aqg5}`4O-qv5FcLg`cS*U{>^wuJ~W&!p0I0&osGH)-$gX+wj zGw>X;2k+o&`ZakxXMPsOX;`SvJIM3=8ZxZ@|>?%e@1V0U#yVd=kvJ(GGo`OC(7sIN@VZJe9XQxTV0V?s4tRNaZ5DDGIf8n zkO%X7TogI)#v*oJ~0= zw{saBj_i<)c$CkSk+b97tKNkzdYw5lyaHF`k1z;b@t@w+Sbz;!jwkd-U_OpPCBNTW z`a9$y$o~14x+`*??$(=*2i5nZ7A{v;=2x*7BXFVKbEtys7K8aSob1|d{3mXbn{idF z!RP9gxC^6@8UKNPcE>N}Ls1D@`_8RuTm%y}-a&CW`)VtB5$3Bu#+}F+_o3b_c_6Nq zZ{tt+NDM$}EJt?4F|JL=0qB8i&>q*ip7*-ydgvdO_M^HJ)~P%4 z>D&wt$iwird^6_^zW^WLMD>~IfMeAEqMdvbUPdJxfvWn?W1f5z{Bi&P|NkUpcipCL zj00SI3VFv|%17wk&Y$rkI0uE)cW?>5oOhxXYO2Rzio6+TqM!OPd@Zl!nRr3|3ct$P ziAHi$^}85?L)F>0y6^(5!)18FwVC)uevZU$jZh{-|hx!gK$oY4Id*r=X zjmfwYbzB>b{JX^k>bvFpup2FK5;h|H)ZMPPmG|(od>l9AcF8zH{WfPWK9a9eAC61# zg*v;>A98lNZ`I$*?{P)EBp1Oj`B5&8FEJW}^ghL#@{9Nx8&L+2>o?*TIXhGXJ`vgD zvRB_DAMg6HoZUEka~HY3US>h=yFBMPKQd?5x|Z3US#`8J=fJ&k&Y=8^^7E_0*{?eA zUFviBP2~HNGh(2edwc^jdomvjpbByi=YIZ9e+^IL>}S8p|03VNqMTgFibuF`R-`OHRBp=T?$8#TSlOM$#InP~YTIRwQ&iu@C`JUGEb``^Lj52#s2d3TmYFbc@Cy>Yrd8* z;-@+1z$)&85!j{Q5ZRsY=k0pMk(t&6*&Pb&XTH6K!N~bJQg4&|GMFA9e0GXDGhu(U*PEoy9+kPW64mrFe-Gpt7>dlRAN2;vnK_x6ojC6t zIlHnyPUHMKhBLQ+mv6wQ$Qd?7??7(mvzPNT>dc1KI8JVY-{c{<6`8+v&{fW^)knSs zP1V^emhu7=)B6V1>l|KjFa<@RY-1( z%wy!?jiDtk(#CAUEnL48&wq#`k#LrOWt2UV`jd2XRF%&3P|7jI%!; z;d3qIxp-S%&aL$i2*=1_W?{VIV%E%Ys zZ@i&*G7s4D-%In&GV;W$=rDreWao!`fMdd<~`%Ad(CxCbxBM_8$zkAleF@GhUF z--_?%SvXn#i7WC>z6W<;ow_5kQ_tmt_4;8lu2y&B4=`FT=rgss2>vH`;zi;NJSG2! zujGkbiQD^38_s+66us`KkGu5l;oEr<-^h=nfjkCt%kz4@&Ut_C+wJ>#zt1`Eo36wf zoFczfF7sA1ac0L_^eVYR^nTg{UHF`>oWEuGgHR(R3_GmyFN4EglqHppJ8jhx5ygw&yruo0;dr{P0> zENw@7Bl|AHnQeRSg$KVEW&RYdK2J4Wz-XMw(ky#=AEc?Bp zwbjVJ&sjeZIVablv0Q5OP^1S;6K|vE;(VlEpDC9)CHs4~T>3$JVFB?;d{^Xt&`X?t z(+;UY&CnkmtnZ=~Xf=8&QsYlR?ih7xKRktd(A)ZCT85583I1NX0h9Psuo*w0tz723 z*Jw|ij`Y^lhSdD8_|(x`kU!<{EHX2Vu>LNe9+*0P4b7Q;n`Sn6myQ;v&!(2$M>~nP z)4ynHcxjq?eTdvocoL~asZ#^_qFJY@lj$*A=mFy5D2z`EME~z`1FZi_$x6R zJCQl%KYEb$qi7$RUh*yTqeCCJL>HOZSs@(f>m*Z)E03Az@#vObyti*BRPIM8y2ybIw4fQ?t$PY(nx!f5? z(r3hV=>O=)Xu>u}=A!h4W@M=RZ2B`c@tbHPtl(SG6Y(&gnIZM~9q~o#AjG+Z8e=oKf(2UO}vcHaWH1$c583pGJKCBctw5}ZHOKG9L&Sv;*Ruu z+|94WO*j*E{rl;`fA9tHg=cP~P3fbwEq>$AqPNpykiIlQT$Qhf@qAHwBu?Y+p{uY4 z{lyRAb!1k%gPtv43p4m)IE#M)-SN5jRBXi!SSZ&I>9zlftMI?k*Jx-|Xr@2p`D zzCeBXzOb}!PPm1C5lwIy{*j+bpTiRD`(2{k5&R=qiwnesupR?(vD`#7;L}?(ljJT~ z8L!IskgtXG-xI|%`OIyj`Gfg$=-oJm??L~hHIw(u0d$DmWZcOwr{(2a()Qxq%j)pM z_~vv4-GOfWP?%&`sSmh*!#ga3vW#32}gmvFB9`!st#=Qig)^(y~X z&QH#3-pgc~pD%kb_243!@BJE|^N>C;oNvLWo@c(;&gb*YdCmW~HO7laAvI=-IQ=C5 zzj-)C?pHbhd1pD}**}Z$u(dHXpLhCK{?B=*nLSct)64P>bB?mFa-NGJ?;(3H@8&sV zUmb(&p?>1*`<&Yc#reGEV;B0$okz2mQak4G`K&tNdHjOxg>|0G9J!G$73Z86rzMf| z`6NABE_-P-vPa)X_Dl(ydiydx1le-wmXt#ryU?--jQE!l)ol?K{zP*)yqa z)oAuu>dF0l&V6e0AZzJ^sWI7yv-y+gRGPZHlIAR>PJW5ZhO21mTUR`Q8sf~ysW)T! ztFeZkL#H8iC1+z1KNv^z=hF0o^u0;q)QM-XSZ)(0@HuO}_|&S@>-*&1qh+v%A4gN` z%Ah{p#$LG(X?j7<&;(k59!0y*ZMXvMkhyk=eCCxA;v29EMdjY1b?MC*%D+I@V}E`Z zy$-vvP@LX%AK#YVNT0-Id~x45eg9i=A&kN2@^|3TJcFzGobjV*YEovO26=|xNmH{v z!Dc>nHFfYYv=?`#Q;=CYvp{vZ2gIqpnTgl%3o%N*HujV2B7T<7EYXnPLQ~W0@ss%v zX#PCTSH)FGEnJKAq~pbHP@jL3o=>l#-=iWwoleAl7>S2aOa255LsL90Hw5dj47cC{ zKg0FbIwHL;b-WSZ%32S)1+Vi(@dRH4_u>O_X7X8doA_{4;!DyFxQ_n<&G`$FKGYvS z;#-`AcJim;M|2eDj&dkH8@J1~L3jRcIuohs>C=5^GkOzp52-F5$iIg(@B==@LDv4X zHW9appTd_|A-;f~iu9WFf(zujVS!v>@pt?&Sc)U%GNYV8@5i4gh4l7EXk%oaKF+f* z@r%)(FNJq88^h!}qXd6|XJ4hc`_!Zl%kRed*ob>kTYfM7oF0g?aWZD%X;i^nEW_!Z z%e=jVmZB+u&+#W?34WHkk{*P6_{=Te(}Tt7r$l{W}e5#_($$wnt8S`t%O2yqcI+9@s(We ze?#dj;+N@s+=52p)9DDT#1Y~n>6vr`ZsNb8jp+{j!RJmskG~7~{lotFgYS(^)-(H^ z$XDct&{t`BdMN7gpWs^lS(Ni^=HUT+UHK;X3}1`$V%|b&>+9*GD38x9wV(%JHC_{6 zi97kf=#6v`ZQ$$9;*Y2QMdiAn9!iMcLHfpS@gX$zp(~$#lwSNKy&Ktg`Ak>wztZ%! z7JN07!9tYHI}Du^QPg+c5=uJktli z^O-3M@MrP8Fbz3-nQJ@Hzfs%z7MdP4i*JY2+A-qAv$HX(c99lBKR2AaCPi~dZ{ zK+bXM({%o3I)FZi4{;%K4v&_fO{?K^{3Gs4Po?kE>*+9>*&#FNTQu|ZaJg&vE%aO3 z7nO0DxE(Epd->Pt^K=pZMb6I)^lJIpxDa39D!I}~zepWR9Ze0)S^bgb{7&Xmqko~J zkaP4HO>b_-XSU5un7Q^$aa&a6N7J`G(*P$Tb!{z8&B$z@dioG@pUBxg4Eu=-h*L`@ zBWL*rxpjOUT8yrw73m6G&-b8z;%t61Qom-1Q!i6T28;KIC*WLU-bpY13HRe-EJt1W z%V}zK=A_(>GW%u@&TLlL*B*}ai}ca0`~`Rp&){t=w00hr@-yjFWadcCPR(i~pFZ^* zUma_3n_Mfj=byp5_!yl~1_P}fg9`k^NPjL$o8xD`Al-$Ac?LD`0H(=R#CLppeLX&N z!W{A8e1CN07oY)BhyC5xbDyW(k-B&=GD{q0u`_=wEsulvYv@xnHTw|SS$rw2Oow7E zpZfm-t(RvoMSM37m1~OHs3JZF+tCJ@OUKfatQFy3q_2rr<0pO@eFBg3FVQ9#h}>~= zzt}533Dxn1`~m2N+i(J2$3$zFA+!BfapvM>;_vy9^k#ZIy78mw5i~vJH>?+rqv

    nr5;9Rh0JoL`3=@C!KeKFS>rA3y9?H#x#Lcd zE6U$a3(}dm7mdW7XlC)d`1O1NY~<(C)9GgV3LfDrdF}xIQNAy(#68Fya|M0c+5&nG zitwk=lkfmu!FBi@L#*YlbTR!~{3BjOE!4#*`4#kg+8XotohZQ{iQE@@iw~!{H`GVw z(3j=@v;GzBhTi-jx&&{cDQ=W|0>kng8u_}+;wSUpS-%CvQAKJCP1XVou20hFA z5AozFcc^YjS5hV^6d5`Qc5JBZ9#+xgM*>BWn%8u|PllB-V- z#{hl=Zs$`&^KZY(_r=eA0r*z?`pkSj`+pR^L;kPXZ`q?C@cZ+H`1&;O=vF!%*&D6# z7j_|aKc8LddnxgU;_Rij`J?#vXnI+CRO(#LLLWZ$B{eVaY$=+F8_?H~_csxhF-v|7 z+T(I@S(@HO_Vv?uXctt&E@c0dKBfT{< z@+y8EQg2d^pWs^{`=C8em(O0ShMd{d(`}fBPDrmP;<@Sg3Z+mM_sSoE?DvuA%4ZLz zr##HRfYjCvayx12bs3tO>mRxQ_;YD$X6Bndd^@BzW+pn0zZxiNvuWY^HcH+hGUal?h6(8=lK;lmCxDD**XRXB4@vuT1Jzn`JecIF-KgK-jB@h=Zm-UU(n1mpVJ*k zok)$Di$;9T`CS-J(i(~+%jz7m*NDzEsFCyXg3T&Z*h6N%b!Ej-`?hL z=BLpqSc#(ILim&aiKcFD<_FSe>5+6MJ%pw=oX8)8Ui=(7o*s`@NH1DW*WgQ}2b|}b ztLaRX!yxfd^gUV}|KrajPb2m6JaOjmf&5!I0{_bWPIHgz!KcsH#i7V-aj@Jl+#!A# zCHXIC9eMzAhiops0_R{MZa`+^n%0lzGpFt455q^uJ>z5gCK@7h^P8S4feXb4&@1sc z{{Xt6iD!n;Pvra2TKJ8>gyvqHemq(H01Ba!_!s03@gSWfm)^0HHm5ftf4;Ek2##rOj?qSpVSeQE9=zw$%*>S&3Z#qZGnS-XbZgBQ^e+i;5f8afnv zaUmW>Wna?*jq#-TG8FXe>-2rOuC#&pI{qc>79U45Ph@7hh_8aWxCe{zm}fquuV4>< z5IXbSk(qfaJsM}opBcq%Xk)D~j^!&@J05rPmF0fqbKiMP{0%>YcBYw^KIJFkeQ|4? z&0mYg);G7@U zkw{-mZMhnmoANG~q8YLmzw@=#Al%nq{$f(ki_?I^fCR(pUyV0h|8Glmz zGEGna2stOc#Hm>$u?~61nOo9(QwvLqKgLX?uN{u>aV2^ppKm_*(kQ^M!I^yKfKD{~ zrvdWWr*1t!ua>_H&+(h+SepG`A}UV(z8#qhipr&*FBBhz)#xa90T%G7Z|Mg)m)V<{ zHS5ww@~Hto@~K5f(UvHMv*k0BokP>Zo6CL0kEg}ySfplsE1t~f{GW*lNRO{A*M-md z{fgd;0mv+LmbLVPtu%Z0XUsx>%$G~8Y{&193vny5ci)rGJk^}0E**pC_)};d^-e2>w|Bem^hRP}u7Wn-L+Pp|~FDV=pN0)6fX#CKkot6jI5qn} zI!|sJ=JKmhlYfLZrWI-GCb^XGD0!Pk`=$RCUw#e>id=b{#d z$-j#Z{J*ps9Yf2}^ufdV%oyoEt>n_HZ=sz~SZ+K%#r-IcKal#GIyuKP4d^X+oIeY% z@&C|=>3y^zeG~OD9lg;5ccYK>x9Af{pZF8M;UAoc%vGuRPt!?sBE1K<@pUj1g~h4! z|Iuf~yD$p5L#CH+m#-kMgv0oY=?LuMhoUd;$E9+0aTRVBKaXEm3xUUrMqZZtmh}uOXwanEE;wZI2;X ziTmUV(%iwC@h>1fC9_z1bMBeP(QD+crJvGza_LVuiWiFa)iI z{w)2GK8{&fE6$AjHGdb+?(2zn^92Nlh%e?JqrK^8coqM{A*hdskvZ=PTF=t4I0;vx z6^h~yYu{lL28(aPW~{(zzIKKDXM9g&w(CIOlFLl}CCx0dA3Xz~;KksXBaj&}Klo{K zf8%W&BbEEv<^0jej5Rws{y3b3%oz=7A-s;Oa6H~Y?xMMqU*l^EA@{81v;xiS@+w~n zuXy%(+7338|f=o z(qZzy(K$3T$7q`Vev4er%nrOKms-`2&pv(tFJX&Z0a^#w@;B0xQ3lJznf1P-sWlIw zD{6}$MNxhnGCz$IXJ-2U-|OXkHnx`ebSW(&*Fz8 zvt4R&YH~?&AKdSmIW#ppXL*(QMtsAsqpx5ze+P1|4;636^{6Yi0jam?SKss3B4?>O zy@z%|djBGE1Eh9jFHc0ptckBiIsPJ~t{+5G8&W%a<0IsJWQJ}h-x)86Q(KzyU-6kO zhw}CKhBSRD=ky!+G4Jjc-P@|EeubT%GD`bq8rck^Fc zdyY28HogNM;cum>kAI^addr=S+^udGAIblY4{;cNlpBBz{8n1Xa|`I-;_J`|>0ix| zescrbU?VQU^BCos%z(xCtNC5D7_Ei$fiZZB??x}eE&PQz2Q%;=o|Hcj>ES=oXYd;C zL3-!obg$>m!)^GVcr|j*`H!a0oRW1ueR3e5xnl+0EzX^!BmD#4$i0E|%k+eD*1q7= zlQM(fB=Bb;WpeQZiQ#@ zB}U3E!O?sp`T+eJKcc<(RC*0^cRN@7AwEGVx!m(JpFJpEDo!8FOz^MxG+LWZ!12h; z+mg=0n>f~bW|x8dyZlvn9B*L+YT{*U{pk_-j6W5J;t;%n8F(9+XP)rPqx`eDoc{!w zpQh7)C11GV^4%$jq0y=w8pPM{5+cKA5hcnQ5+}RZvOpBpiV5;-d5kdIPPB z>bL{*@DhH;AnTcT&Zm1(NNzLA%dHj9;*$-EsIzJFOd#MAdS*d}k z=f}xCi4Tx7U0Hq;Uz{$cyJ*hsD1I6J7Crb?I0Wfg*?S%M%jl{2n$P*lxjX^y;BKUT zrGCEenf|mB`tx0p^O>4>67HAJK7WC)KvPe9p`Ew}9_RhD`|4&1aYI4s&00Qn`)@2l z>U-*DX5XCW^pEtDHJE}r*eaKEn_4@a{wmjvcBcz)F+UJp_-XWOntqdWoSyaz%^8^? zcR5mf_9DF~{jnZ)^4HPKrti~on23V%nZcXU)RDe&WoREdjNX7}(OG;mUgNLDENm7( zgVc~a>14UguOIS1@q6fV$XV`*V=z=a1euvruP4cs!zzqNTWbwy8#LlG?|jT(&kv+;BlrgCwJ*|Jtf$`=>)3bi$*I*{PprzbY z`X%j+v+w}Y7pl@HaVawU9z=V4wi*iYOXwn;h4XQt+`*W^XVyP~ABFvqo^g`$-PK#p?}dIuos!7TFKprZ^ZvY3oK**Luu>B zqomw3s3i9kmhey0d036p#Y4~v--+*`%aQ(9gnoiL{2TNubi?5|7gOZF!Y*WHzF%%N z8t}P$e!w@k_8$I3Pt=vmz2q)_CVv>MPutOo^d;1^-h=MOiKuKXvq#0iUmhHeaSG>j4=bxkJ;7a}#Y~nA% zt0?FD4VSx$&n#JmZ;H}5N-nc>X7<|RBaz>KTp)iut`&cd{9fcraTESGI*2wy=Kgc& zQjEwt-sDfF`CZJr^hOkxD~_vC6Q{`Ci+_>1GWU`~7%jdP-+Ly_={1tO<|XuUoQTY? zf6?RdpJ#8z2tM!s6k1x`1wSD(Ols;xeucF|X!=3+)_HUc(tkG7J~&mrG0ogGfaYh& z%=H?-S2zi&8#%{wXnMf&bP77lrT*kIOh7;dt;>GN*}hIbvqL`r zr?8uUoIZ_w_NiT|PpPvHBmE$I@FM;~q^{h9amdW@|2=4+XHuiDMhk1H5!p)x(HiM_ z#gKE8`g1W-7tf^GUuPrxE_*Wd>uqaAk$su7doPB|Uxp9ykvKiL7@zqh=W7-+7w(}& zk+a^>bLZg;q+VYy*PQO8?_nEK6W^f)Xg_?-=WMp8f6xQz0{qMGM>BI~*4!?xOS89& z$fr&<;?w8Oq6_g4KCxDYW@i3{E)tKWyJ%+iIy5~kb)<*npOc19J zw4R@_U38X%z4$bCIMq9oBhVf4zy)ZTTce)xAF+rSr zSYMjHnt9@Un!8GKxeBO=jW|F)vvqmeKs!$gG)}@L0K07%KNEYVzYT zhkuh!MN2-tXc(V9Q3=(8++Lb{Od-##Lo;!4q&B}OK7iK833ye!g+5A8qvdGku~+yH z`8G7OQ#XDPKNAPxS}c*9g-(3>e}8(m_z@g|YjBv{75Edki#uUH-az`pJo)sryXYd? z2lw&6qdS_3dm+6qy{#^v8Lg|e%%;`EHLy&adEkFEb4mKq$HLv>8T1$W6nz(4aiO>g z9^=>G8UB9!fp5guBRwlUvNc^SpS$87em!Q0f27~huV^d$iLv7D^aa|J7NVKko}i~9 zbK>VTef4kaSJ931X;kI=VLE>(&f;&N&*N}DJ^3$sH?F~5@@HZ#ir^VMgnHJtBJ~^Qmq5Os}JPFFAYp{2Sm| zK0ot;sOxL}-S>0l=iGo|^4n=X%j}cX+xz+ajJvF@r$^B2g)ubyFQ0d6=}MaOG6*y9 zA_mK+cV$o4<+IOHn?B`t($tfD?)8z+WF!X452UFlQ~16#^=2HO|NE7EYx*#9)^oN8 zAfL$-vjvbvg^#f9YG-(?#OEm(!3smpR~Px>G(iW(@rTb@(1M zXX``y8=mJUAm{x9q*i<{PG8NL&FqsIB4>FWGQVc;W?%p4nH6$x(D^tQIX|ggugV{Y zuaF)yS#BeW@u@TC(!S#J=m|IynQJ!4wcv}>?C&|qo?U{>%jvcMLqGB?Qqvy5BD56e z45Z&3!k>i-;?%~>KbbqbiVu}vN{>O##|oPIaykki=ivX(^p)0{iXWnP(|^&2FG4pW zH9qGlbv^Zj-8VOXz&ydH58p&bBQs|ziIlGb+MIQ~zXnwGh?FipMsgYL1`mS)~e9lQ;> zSCx<(#IL3o;2&Hj{u9@uD(*o|+=&x$G3I%03+;{E&AQP?ajN_y^gMbIeIApMnm!h_ zQ3tD#TKqCCit%_|{2nIpZ{RSzh~~HvzvC_Izo9y6qn+HF$n2Ist1(=DG0j~43V$n# z;x)OF*iZg!apuM^>5_Q)+`Z~yHeUiW<)5bMEoJ%4?dcmg@Rw%|nLSdQf8eL%8srXp zmt6Yvo#N4`fz;_M{=!?h$Jz_H z9Z#Z=Tv?PuKk*f~6q)5Z%5CDO;Yofi?SA{@6ngU>Ae@y5vYZw^0{BXP0teFK-bW6a*xu?8$Ibk=xS{`YTzhrllul+vhKOk za-A>^e_)w>Z+y(ZOiSYo)Wz5M9Vb~k4b}K-v57yEo`OAiO#BOdfo4u>&i{mm#f8Nc z=+P)BmwEFxItdl=8E($H`~p6+W>4|es4A{UKSBfkXPPG3iO=sz?xUkH6ooLt@;02q zUxwVrPNrwb-9ZmVDIA3_<<6&{(9X0v?Ll|YHkgkKF-)!sF2-QwPWX%b@%;BRcZ3IM z?hm=E?L-ocH?-RU6D>`$A}#d2G*9#>-(e#HW7#W4@L z6MiE1AaYM#DVO`fH?$TWlh1u(IR7KA!$W8#e-*u%<_=Ynufz|aZN;7Gf8u($AIFPx zN661rLY%M9{>;y}PHqCtdCI%X9)CeR7U>E3d_F)vhwP2o{48sEe_fG&w^43Be=^NG zF3vZ@7JfVOndbkIJ(4s3oZN7lGn4&$6xxXMvwn+w9{K#s%cpnbyk?(v#DQ`L(S>v( z(znyo^RBbUbAH=mfwf8W9om}aOl2?RbG(4hd+EYAMLxriY37^k&xfpKKaQg<#MdKd z;&56Lsb`r{w(>b=d3Pge_GDkVw#YttndZ#ChADUtWzZh8tzC%zI2V~oa+WeDWpAan zXFuk=rUuN!<4B*$na;bf%#Wto7e8Se-;F*%N6|6J-bsy2?^&8We!z4bgXPHHOpVR? zOI?{vGc#R;)b_D7^ItWZezgT3@~KJF_?*iRY0iK4cFtvLRO&)z<@ELH;{C-r*HieD z`NHVL|4CEJYtr=W^v2%&Vw4f59@pmU;0Ey$ntooL4!{zA8?NC?(I@F_Jb{1k8~%S5 z&e^XfzMXzf|H9w=c)W*8uoRgw`_X3T=b2sfdYW3?3$OF@kXdG#xEsG4ukr=xsW^zQ zMZZA$$WO>DcoB6VdC{SLeM^u!xzYU_DOt+<-j$2xo~-b3%E zb?J*VHDDBFx_N$VHT%qp2t zzZX9uPG6fw({oRvsXI;O(s%pNnc{ztdcIxU2#dsf@dmyWe@*+-zcCl7(YYslK@Si= zjWfk3(r4s$ps9Ej{Z(!p?Lx1@+vtRkkY1ko=68N1J%KityPvPb-;J`#<9U24?o1nc z=5RiDiQIKkN6(hON3JTq!n^oWF0Cqi9NNzW+KqsRidf;BUq4fWI=4NqYaqbS8Z|m`y z1BS_0!GAbHF87xQ_(`}#{2(od+-(-qv*pU+BHSgOfy~*NopZODDt^{l=8P8f8x)dz ziT**GV=#XUeG0js*P}g9L;hi8?tDf37^aE~qJZ3S^f6Y-RI1K#E{=Nv)rM0vR(xRJk{-bN3_C#ZzS@jv;qupf%zT&zZ2OUrQ;J`oSZ zt^B9>j$eV?OEP=D!oM$n7oA1x%dMaduu`rws$ib@GStDb;tHNSfd7NvKwIJmTq@3- zx0636F`qf_V);YFAJG5*o!(&a&$vzQVtNMF@&{rBKat);dV5{-|~4+v-vT|*-XznLhg4=5$F81 zq1$l1Tpx7ga~|vRU*Z@3eLRkq_(`rc-A>QNdZfNTA(xt*cbGkt8uKJN;%K=NNL~7Z zre5^p`_t6LddS)PU7UAtG=CodD9v7Z5|5xT%E)Ctdx0N|yT$YArSxB<4}FXD%KNa~ zS^;cE=7`C16R;bp7pW)Nt4*<$PoFE!-;GA%g7^q)kREUmJrL=CSJ1tvo@dbj)sTJm z6YfA6{EB5*h|C8$?|1mR^!xO}=HhE`08$UzTANDK$5LOizviKpd}_cPeigri9!6*2 zaE!ucWX9@3AGE%NmZMVpX6fwdoSD>~oXKANbUyp{Z=@fWMb6;>@ix8^|25x|k)Re-tf&xqN$C2iWlGqQoT5n4; zca+8g{(cmde;#Z3iL??ujV?uI=R?J__*L{%dL^BSs{CchT=N-y5Z}rFLI0pv)3Y%Z zr7%tI6M89~Mhnq{ks07zx=ZdM`aNAi-=lBf5!{92@QnN|^d&sXr^la3uf~_yjqRw9 zrk;5h&*CO=W1NH@DB$Z3knhHSk0JO@oSEPUdZ+kS{K9`k};W8&*V(lF9!Sq)=fS-}yn+%{g;!ynSnG@*~D9Y!4UYb9RuR=Fq7;Y1{Kr=ow zWM;x8;>*Ps(6{gq|1svFl=bp*pP;e$czP}Uh8Cu|_hil~&S$oMlYWk;@EZQ~Tuu5H z*7GarD;SQR$b7#)GJ9{Zo(6Ra9gX9Knc)jz5Z^>@1ig{|hs@qL$IJCazCQIL|A!UG z&zPS*H6nc>`zxQ%E^$7W0@xtV`)@$=Ib~1f-%0J+NpGTgX{p!gO;dztE1yO7UFt;MNzUEtNbP=DoO9EariSHxq|Q~~^ZDnj?av=U(|6J{^I3Ee=iQt@ zH`5g~do$;AJD+p5lcs)U?`ID!qB%D?)2;ZNk?fm%b}R8JKZd@Eyr(hv3O$hbn!Wo7 zreLHvXYp;+@mpxE9T7xUS(6UFz?Wmv@DM!!btOwMlh?pgA$Vv=|kZH?|oo$nx*+Lk(> zdEjxr4nCE?73mGB1J9$OIOic}?Okla*SHnmBKv&+U5okn|2mU1R~@O3@6pXT3aL3? z$fehuf*D9{PR+Q5rUvY$seRv}G@se!J^GtC{bL-ei&OWrevIFaf=-r5^o3(~lm- zNMr{4jdnw7#Z>E=W!j3@(Yt6}`P88@{DE{8Zbt51&FSA*YHc!#@Y~Q5xd(KSdkvj% z3=T$dYw4jM(x&3v%UMo_~dZoOY&-=~?KrJ9GxP-y;$8GJdJfv~-{UHN zJf1{5yes!SUgjI&Q{?`8pSH;q0-;ZBZ7zn?@$Apk+MJ_`v>#1vNKI~s z{#cBM@tSAbAbX|-`tYeI+xYKj_V-Xe{pW2u9XZpfPkHYp&_MhH{Snz`>%{3Z9r(Pb z)VIxi`t~m5ykx(pw&d)khK&oUc$eGyRb|A98zOb55C}2E8gZu(~ig)I*D$T zOTW(^9f$O@DKvY%B~9J97pV&uAhXC-v^LH8Pt9(P%v`AfnZ*~UnRiPP7=<>w)D z`)s+seCFL=G<730^9ue8Yb9tEn*Mqiy-{3)*2Pk!E^nfWqQO?m+BjdS>lIGHc%-zbEt;ycA7==V5Pu9Ns{{&SjH>nnP&csczI zgOe9mK}j^h&2qVK{KdD&%V>)Ihdw$k+^h?}^ zoQ<6QJ~U?~pH=oxW|$M`2K<7Yi~Q_4d--g#ud|0=qp9H+U^CM9bB-6`A*9CSy%oi9 z^Soxm#3_ zOT9dh9)Q`%dEACWuOB6@%&$jk z^Jw}RGXKq@%`jhnI>zwT(F5suljJVLZ`e=V4P)^OT3~^EJ6af9_=3J?6D$#5EWVSj zqYF`oA40F7-_p#)SJTw-uDBJ0tYudG2?vW`M=2a2o=kJ!?L#xOucz;zWnwh)%vrPm zZG%dlDJvd}tvD6^aSAr#F3%i8zonUBaxeQ?ocr%3^lQ9=AJ7?@9p`v<6E5R>(%-S! zT7A^TGvYhx-)M`E#NXjKemA~A?i2UYZnz$oT5m$n#5~j!ucmeB7s#x1yEu3H#&nO| z?O21u#AUDwn~-_*MS2mQu%4Ooewsd4jDLhbgTIyL-ky2-SAGY|$$yF?`C_QYTi$oa z*g=QNm8Pv|3p$Bji{2P0{u5RC$LP&Ci$54&VLR&iW(U&D1jkT+_uacQpY`M`(@M0O zg>&iAXphXEhtsp=m(cIgoF7LA<5AQQFGLYskB*ote*~uTPtp;{9MwSl0PYZ9MeECD zra6{hDL0Ysp)KhQx{c;ud>{WH--(V!W{nrb7h$cq3aubFg61A~DD5DB6Frvp!ufpW z*t_ZLcon&i-)-#>6hiLL&&glG=YE@c{5o+Daqj%J_(yRfipjN=dy9?}UrKXV` zMGIm%KN$`2zBpf#dU_>V;sg0X$hocS-ec}z8I+>p( z^(vpuUGkYVvj5V12B4u_`fqCcvwZqk_T*qb`!M}CpI>?nyKn9)D!&uYi0{ODWKP>e z&y!ED%e$M6F34w`GdhHpp{ZA2Bj@2HaRHjoz~6m+JZC@kB{ga{9!5j)vveg!dM^7W zb?yy5HRfNMZ;-v5J@^y86aR?piC#4OE_FBOW*3^t9Y=GP(+l3iPH}os>Ph;=Kt6lF z6`yyRI`;u~$@jrqxJ*0=!}-+5|M;#*t$G_T%IDmq2W2+Ro=WX*in?;cka;2Xco0q9 zTa0VOV`zW+Jk7q#y!0KNiLdYxvR}s`b$5bz5o+@%(7yN$Pm5C@CnELra`7Uh{-@rh zZe5A&<-4t?FSQl7qnYP(zNhi)@FAc6kUnrL_F^|W$j_s%;#5B8>uqd6Ke_#BCEA4M zoR>xJ1*zHTvn9l-F`MZ>S@YbVH0Sv*KK1lStcb@t|2&(X#kUo?% z{{#ONO}(hkkL7>CYHSv#UY$lW`%a^&$=}LtOLG{Up94&1_$g{{$7pALDyGg!xGQPThKmE=50NR%%b5 zLS}{Bf4cbk1CjnV6@{(Mr=QTe$h=b!C-M_;i?uUpIdtTQ(7E()+=kasN$weR;M>uU zFrGh-9!~q)89$RR zL(3s|lyAiE@W0TjQ67852jfOQGeZ$N3X@S1e;_^QCeK`fo>+rh<%gjb#-kCwM0sm9 zuoP#A)9;?()5qtDi=!KVEq0(Y+T(cn%rp1!{qe522X5vo%4Y_;j(-K;@(dXvA0uZQ&g-^IW4Q}F`QkEhdrke*nQ^UdX6#w+{_bTB=iE~A+RO3}Sof*0ir%PpWq=x1_;@FuF^efjj}tLevh zT<%}oiBIvRT$`n*J+#fUVmBu@AQ{Q@Gh$X3op`y!-5{cQFgcTFc%}ZO9oK z#}CC1$h*xvwbe5xiF2+}pR=Fx{&EiH;%;OwSD+UmGr>xFHV#8aYuRT7Y0lPh^l^F+ z@{Uu7s`5qoMRX1_&;BFM{b+?a_4jsJgL>+^V^VmRh4eUUj9*< zc_g*F5*;E=ecK}blXj%P(jD|2dJ?|mQ=`A+>*5S#Zhct3I5Ib+Zl?x}p{aL^Fd1d! zhT#Cd0R0|8q;3^U z9)LNjkZ{i%D7#=8ts9DkRG~{wvq2ZFA(>m3&gKtKk-2EM|}F@47?@o zg!}j=bQqTMmGBJz9X)}jw_Hk>qp4ha;yk`3twR5ggR_9DGX1(Zh@F4iu|`K7yT^{- zYj<~d$Eag>cee-zpfu9motN%zkd_ve@A$daUgtdTz35sjyk|dq?}LZnn)JW#k$0tk zCcTx^-g53NeOEz+oCivMDzFxGlJoERnk#*T)Q18C>2;*W3vLL$3&zN^-*<=K^K6v# z-h$6^9xHW^)J{@o3C2r*M)3P?I!16`dLMayQEG^wO8R7}zxl9*)Kh|G(*Nekrt;eF z*)8S&?{I6R86pUh^Y6Pztkef`ZYK3VK|8^3PI^@833=X2>Tm9PPI_JGCklT5xh6;# z+!JWYYjXq(rT3LOQ0i~K{Z8sEL6P8_Jbx)@B>gDCFX?NfY6^A;3ZC6h zgiBu~Rgw1^Nxw|`)`C9L{}BA<4m;(XE&U^@DyhG@+Ayi31^a(jInNgS=8^;C{8irf zx77Ah_e$+9b++Ju^uO;3hoyFw^KZV|T>9Vl%%)Nw$g^Hje@U&7`kQ0jle$ID0aC{a za-@GJ)m-Wk!2&@)!D+!_!B#<-y#82fnbea~Crj-jRYR~<`hNu((qEQpBXxoNJHNG* zYUy9gv%XR<2#!b}E7&A>FL01&zxleOR1?8Uc{Wn&d8tbT3DPT4*GZiwb-KKFv-I}T z_m%!Phs=_G$>|N$Y~M`TqNB8oI>)TT7l~?cg$FmTuyD#X?!;ZA`Gy+|uDn4-kJ~JY zx=U}fM?`#o#MbVQ$)5L^6$_pa5%`3heNVaQ@|1UOPl@{eoUG&*ydCqB{6#PMc=Q#% zi(hji@(nRLZ)h`3laa49@ix_@dy*z8ZQqjS`IdQwZ#npf7H)^MSb0Q?PDizvXQD+` ze{F7+X`|lp9R~)yBk0FF9v=KaKZ6gr9Qug&P91`;>frl8hhxDy?D(QfZI~__$LR6v zj2>~-ddxTZL^tD1kfmov@KH!@?4lNqkwW;_ixVt}~0DJF|MG3+?B+@L`?{gI!(t=;6Za2p1N|y701~-An@o;C8mpc*Z z?v&@cb91T(K{_66^YgK#fwFUy%_D}#YrD87Uz1=vWYj}dw3JN)*JVO-c%p==HMG|GT(Xg z@q;(|Ufwi|^TsIKo2lJ>*f-4womW1z$njxDCtt3d6=eBhm+Q-z27WXf>__EvKi19h zhF~wG1bMzy-)DL?#eIAa80AmOKmN2_>d(bp z{yaYJkHs^8KAHOCWbIFf5Py^if4XV}(6(^^Gnxc2b$S5SO9J?{DS%B|11LTpK<`@t z+_)XUsb>LH>j#kU7r?-X00yfE^0HANT5SVqI5d##S%LJM8_45zfjm4O$la%bEL96) zSc4$qn+NgCI*75}LF~&5Vq>+SE{MHq!QAW?O!nGf{9Xk!${?5pUcu-G2J?4DFz1Sb zSyL5E!uMc2wuZ3wc?fFSA?Uh?aJi+5haFX@^;3~QLdEoRDt25~Vfb9d&^IdjX{&J8 zQ?c1fMTnh>JSP=v+*I80RPjb|F=0j19$lMJPcA zq0B1^tq=99m1*T6wcJX;hdfs&c(UmTsas{-j#4%4a2c@ls+V!m1W`d z>J!1OQ4x%s9D&J-2=w+x@bAG0_S}h}w|NAA*hWzA89@)P2m(VRc$O7`x1jrgNaml2 z#OGlo=j$Vx**=QfMo}a9fsBO2>d(L8+@O{o^Vc~3s*2;*k2se0j_2&i zczliGIg}WWxkdsn4ks`#DuHe-6InPqk?pe*X|XC1U9&`jTodW7O2oHw5;~)j$UB(C z1;rVyx}ic5=BdiF`hb5AOB4y01?J{4Q@R5s?P z@^_~+dUZ>qM&Pn1jqx|qFzuO+(f)Mo52my3P&!p_)9LD%&ZMYxE;PvCqmsdLqYPSl zWU!+ugTywObRV0E{o+irv@$8O%)~1olU`|=v>BJhl0{j3Tc5>^omu1`%3`5W7Ri2D ztotXMFPpPDzb~5;+Sx?4%Hi{j9Hwo_Vey$9=J@3BE+&T#opSMt1;6tNAE>9`s z;_*VB%r9cw$|96cMY!r0(cZ9#HD*QVIu~K&Qp8BNB4)=G@o{J|GjA8;l~+uw+G2Kh zD4}Rx3H#QUu;N?^3v^0QsY>WpQNrWVrJO%nO6R~*tnZeQq*q3ST^Y|E%H%V=jEU{O z(XIbCR;>R<2fJ_h)P3XX*m9m8D`(Ela#r6f$JwNuXD=%_WLiOwzzS@ZR${xUlIs^L z`TV?+ua=c`cCD13)hf!qRnga`hL&YDSZ}LE&$X8Es#<=yekalMJ7ZVXp>e&AzE*Yo z=<$Qtr9Wu@`Uh1HfAZ$#Pfj}hB;evN2HyY0k&X4}3Ptj7r>3m#p{7h6rlt(qr>2x$ zQd8PlswpvcYRWl(HD!uQO>wxTt~4`MS5ku2l|lIpl)QZ!%Ady@Dw*nylo`t#DHjGd zR>A_BDt6Y*lm`2oE7pTrD_u9YR_s0hrz~1+0^uvdQ!+M~utv|6 zwxy=jRGHGNry1>^nBf#-MnbYV3pQF{HPMpGvn{DOW69faOM(YkF?6RDnpdpwIb==a zqt=YkwC1b5HPd~piRorTUJo0Z#oDms4_lsWw`J`uTZTB;a{7OE7+kmG*lRoXXW0>T z&7S-L4xF3dz|H9ntl8;+Wt9W&H4ZF2=7?FYBMoXCnbXdRF=L%LApBxWne^SAG3e!t zFwmb9eku#U9o#~V2Ohi8yc1(4lr*Mr)(Jow0cA@*9u3Ve$N|WWT1RA(vp>jo; zD12hK8_O@dp%d>$=R!B~=epDJygOYUyOZbOj`MI27L4#9b*Bfd(mj~1=E=8>p46Td z&hXHa7SBBCB3NMTNqLed82M)zIySY(2KSe!Xa)5N6__VkfAqA-Mrb9;LXDE zK13|`!P(A-@akctC-hA4y(* z_VK3gFSP0LHutV7Yw&K28D1 zhZEMm0c_kA$l7axDAxmNWEn_tpCGIU1~Fqo5IdI!F>!Se{Wb@2>`4%79D_Jj8pLga znV=KQT(@8nBZEmE93tE_1j`8_99yj=!c+g9m3O?5O%7oP;aH8dVq@Bi7M`% zRnhFO3YQlu>SS*3HdK)kreasNiXW{*X(2P&cW5Z@riL<=Q0k}4Onw#0B)d>LxP}s3 z7fR>eVazoT!1MUrkE$z8Wd_NPSRs~#mhH;Pmx3fntT zvo|%A#c_RP9RHgXNAb2eehD@mh~wd-I2P;1anvi0FA*|( zTgMZw8&ADuJnNj}>691G($aYB|487E_X(7mC*Wm~z;%}dPRA$ksB0o?mI`(xlJ{RC z_NNj_y^u&h`$S@!C*jvUiCeRhc(yc&7t4~Eay^L|pOR=+mqc)nWDX5XX28N^ayKV) z`*1QdER$*XE19>0Qdmj~AE&3#Vs{GDZ>CVIox&vD6r4;`NHb4iYHAAS(o-nQkn@aG z*33!8XGtof)}>;4B9*cCQt^A5ik5pSeHEF%8`4m2rZM+U8kLXJNP3w@@T)Z1>!i^w zC5`*-(&f&V&h%^PYzF6A9_nZGa>!kF{I#7_ujiHU@b-G|Xd=Z5{`*^9ZfT!>&Uck^lVKA&pye5UpN z%BbaE>3#PrM!H{V*RudGrGQDZ3wXP!fYn9?%yucDiCX~!(xp}uFztH*sf`O+*S(PX zK85r-R46}th1{tqWXP-{nh9T+rCWrBUJ-fbMfh12Vc=QB1ivDlbuDJ`YQzM6kS!}1%S zLceh^=^K}(ma{}D=Z;S~X1?V#4JfB1r<@iYDsb?sppAD0A9__XaZx2JuT;{_v66jG zg7#Ic?^A`RU^`XREU%*R?kb)Pp`_b>+22 z17$?>21h54V2~G8z|pzHc&JQ8Yutkp`rLr(@@5J)KFIHYAE6U8p?)w4V7Lu z8Y(U98!8Rl8!AV>H&jYmHB!EHY^1zB+(_Bt)JXC0Z=`Uwv9fto6Qy)b6D9CX6GhLs ziP9yciPAHyiBdPZsp7Q0sWS0cQ)Nh8Qzda=GeviHGv)C8W=dd8Gv$8+n=5}bYN_n& z-BNMu(^3gt)l%_`ZmC$WY^Aha)k-N?+*;}TcN--n?SIOSp6!%DN=K#H>`uzz*PWEG z{A2XJbDB~q6^h-ivOM<&$5!2>*6}9yvTst*=`L%g+$G}0Lmc)$Vq484g4#bO zuE!JFhP>qX$X5jNiqsvi3ETIYs!p0rovJA_Nt3!unjF~wmcx2)S-V1u&beCb`>ajc z!FRlgd&khDAJ~=ofu))sd1w2PZ=N5?OZiC4bvm39w7joF%28d?8tYNgLl28RpBZ!H zGm86XivH5)ZLB`?Q}h}1&Hzsb1M0mDCVLuNGX@P&p`zF>Ow3p-4{(9Ze`{vKaA z7xjftt<32=)|{V{%-M0@obD#(oU$_~y{!cvvn<$p+JcKk7W`^uNp(+4F7CFZmy;#S zoGqE@ZON#=ttf40jd4e7HombYx2FxihTHI2u_1Z04M{s~&~>n3otNlO6*j!8w4v~C zTQ=;qB|p*@pIloW*4c8ammQ}!*ipO3j%#=9=-fzjs44aoueax&zdd_%?TOy*fS-mV zx=kDzbHkDAnU0Ka=!9QOCprs1%ut+&KIz1Sn@;q8?u3Q26X7mS1iCw+o$JJ$hR(ci zkcK zxxe0(w0ExbbatipR5$uBb>q%vH@tVb@#d`?raEq<>bfy8)J^WKqE)525!=%pjY;mb zP`P8+-UIDU9+-uBaJ;W4#?w5xy~LB*dp+57$&={jUUJX!;@L1SO7?lN@qia|FMDC2 z=_Q|?UNl#EVJ_U#x1Tp%-;35Hc#z^v&*na;jq%~1Q$F%J>%;r|K3uc)AvoTL>&ZUs z%=6(}D_<_P_vJ=sU;dcs%bPjAs80Ffam$ybcYM*&^d;ZMm%vzGZYKKDx5Ss|Reo5m z@e@r(bf^1%9C_r&kY|2;Hu2+7x*r-PeyErG(Yl{Mz5n&+a*1eB-~G}2=}%mn06qx^ zZM-&saKVlB0sMV2fP_Z@jQkKlRAd0!$pNevZK?dPK#~Rok}x8WQYDbOi-9=Y2;}C? zK=!^2q|KW^ei;YyUqB#>TZ+!qJ&4l2LG+#&MD@lX*8LYmx7R^rIg9R;9mIIio3tAR zqpJindV4VLH-lOACYWxn!B7~?{a?Y{X%a&C+z|de6oTRP5W3zDp`A|%wy7a#k5OST zLnZfj6@3@0=(Sr#k3%Xh2xm1m7Y-XBT2!!#wvj4Q^Hf+?s<7`J%8C-A3@5? z2&R6HAR{UQhk^)_+eLEbpGc-1j>N%4^diehG#w*Z-GsOH zJQ_pz#Ted-j#Ts{hADb6T(OPew08{3w-~h5Vp-WKmY)4$$rDW}@?|VWZ(=ziy3#P6 zSms&BlI{`9eN`+WiLtnTkLAiAar{^i$FfCnX;ejJyC>rOL`V^?4t z=R@Q8w;+xvjd%(s#`E-4Je6nTNxK+No?|?#GUFLMJb_!I6WBE-0oU0HWGolnyEOr` zI|)oQNnpQe0{_cRkh@|6RTT+z{v(lf0}|;mBoWV$L_Sp}QvNHE4u2-GbD`i;5?dc8 zam+o5>HbL^O;6(cfMjBKCiC-9GU2zANxGXXd^{N|^JMbdq@dL&g#r30#FnOD(k+#U zW2p=|m&%gQseE)u<=l)k_RJMNJ3ozwi_!?%okrQIG^RXD3>UB!ids84M1|U|2~8d77DcyJX^*lgVL?EYjL$ zq3p||?r;`O9%k{+(=1ZFviL77i%;LPXfZXLoa@>2yp=6?4$+*nvZ)Nqrgcd+FIwl2 zvm}STcXBvqp2IH39J)s2kdmE4*tA?cX69mYBp06NawH=c<9T^h-OS^1cpk?K^0050 z&$C_mWS`GxmsUQH1t-JvnHQH&$J~6nZ~RK|*{>Koe&u7~S9Vu?Wnc^8pSuc}{=9%o zjs>_o7cjcEfG+b3c`2Op^@TzXx)lmvD&*2Q(V&zf`aLhA<=Y}61B-Z4RfJkQ(W5#S z)4qGL{QoaTb+?$qiN&-lDCTvK5>ls>u;xMuStcdqyO+?Zw1g8aOWAg%losR5xU{JZ z+x=yz^vhT-8dcNqGEQZd5xL^K| zR#k9vO$Eg{75o}gMd89K1_W0#D?xZ*T{ZtSsiE_v8hTx;;lRThZfMjp<6$j2PiuM7 z;yd2|`_3Ns@6=ANKU0|&xVz1O8IUzWo-*}rKF9zGO4q=vPrb5;nq^U)RmAJb!EVj2FeVl z2Ex}GDEq1#D6cwcC==T>QXY?Mqzv(Hq=e3BtW@7_tn~C~tUMdsL`h9-qAcmyRJpRK zsp7Y}sj~l%7Rsw>Efg!iR>}b9Ic#rrg}qa*l5_F~9fVW08GMU>r{AGq+I=>id&qxn zpRlpZ6W;89#v<$I)Ym;{?4%d;@qEe5$FG>%^$lkIG-)$i6ZN@o8Q)rqbAp3$TB7%9 zlhE!1V}wV2GWx(gzYm;xtV5ok4g)&t@-$YLJ;U@^TcpR-6`#1ZPQ1IE&#bEcjP?e7 zzWL}=i2*jt4R96z?t-=ft7eHNcHWTrO^k58WJE>05xcXEIN!jSz&^(89AV6xRmO~O zVnWltChW2^p+S)e4{A)<(!~^$>8AMYGiCS|Gqzke!}q5d7Rnc%&G=7dgH zXM%P)6Z^rL0&i!|_&am4i3|UpcfqLCh29-psp#g)z1gm;-{4C19#=E1qSp z9B=2wsJU(o&2q!A!i|uw?i^O!dAG|Q>nrZ8HFKxJ(H*l8cbbQ}b2-kPai#K}rXHj% z@W5@I2Zwiii09}*R}&9Hwc{22lC*xjt^2aMr8YX(uHPw@K-#yv)!;_HS zUKEV+V(&sP{tk}P7P*woLN`+r(1oEV3AZNA&a^ZX+*R%p@`6&>Sv_QG<1X58G$mNAW+}sw# z4WjScAe_8{xEdJ5>5w37BZBb06HI~Vd(Wza(WnomW0w%l_X(k| za3lxK5Ik)~<8uh%lWPc>Ng?>vg>d{273RZ*FD+H^d6kNzr^J8LQekGGV!xkwZb2&f z*%s|DMm#(5-hK=UrQhOEY%YYN|1^~C#-S8=g%aW&N@_wV4b{WQIvvKAt6_}47sd?3 zFnk=tM6U?L#Vd@`z%bEh!pMmZqhnzh(~84rRvX6UR^eO|J@DzMaE46|=i}mVI1Gyxmg#^pF<<)yD@?X=OQ?AJ_3KU2zFXT;3<4-P|Ha2+D1~>F_M9Q zMbheIBpp9Sa_WDg{|TCQjiO4ha&8p9D}-+yh@#!0DDtmIaj00HHHpT%c{G2GjAq%? zXnG3Yy1h0Uoh{KgABg7Bp=fMQMAObAnxepH&R0frv?`jy>S%)c$DlqdhS~FC$Xg`d zo^Y^{+A;Kqh@p}Ad5cqHI9M7(gnF##;IUNojHO~&ELLM3#A7Edozc-KYo z@^*}fBX36>EguT!GKgbdQXHqUX3kA-(NPE6xwqjWs4h+Zg~qDpkbrA;$9F(iWrOERcB zlEKWo8FFvT;9ZhnwRn8nGP$)UlhyxaqW(G)*UC(olEnkE@L8Qj?0;Fzxt&GMqb%-P zWN}0-8-4NkeD-9s{B<@J&e_b1%w~7T974N^J~%jszMFE$Fv($FP7X;ebFm$nOQS=% z^naGiox)rkgl~O6n1|-Sd9**1N2*;O=COG!%gN(@VIJk*^GLsv&*f|RWPZ=bw&7Q@ z#(m|j@|An$Upeae74sbhcw8#rwP^tcp#?Ih3u(2ikkzq;KpTgXN@TS#t!vzESr}za7;OG z_LLKIpq$s@1-^Jy&JY{nTq)%|>|cTQf(jb#tiVG&L965nG8AWvuki(TEmo*8Z5rmh&NPA z^@my-zT{-YXUD0gZK)E`qfpTt217%^2Xlu(fl=2)6WwdugW&6d(N>M~( zB`v&(a{OphZV zLCTF=WSZRJipD+39NpvE`g@G?yT{6f_xXJIp=hCxIKT2SKa?lv?s`IMjbyP}Jmvb2 zr^LN|M!mr^T-!Y7?fK`JJ$a79)E8`g{eoLLFPPuqCF8cgq~&elY13a}pnA>xL2pPE zsFlBA%|T6`x@z)0Rg>j2-*WM`7NeZCn6pTmH@CF;6{^id$z=I9c*l@F?@&MU4wcJ0 zo>sr7>BtYvdH#V5-9IwJ;3Hw8i9P+F4$sVVD0I|eSyNrCTkA5kgD&oZCWg9$Qj|JcL@EHDyzEeK2W78+`9zNl5>Jvp5Katw+Gdl)-=F`^C=ylY`d96M@ zU+D9PojyYg^%?e;0ZkVe@a~`i2jdNx)XPxrgNAfkZO9IPL(T~@gAMV?HYBFRkcEwm z*fZYqxP2PPg*h^=mlG@hc4F>G zCqmaa@y~WAS|4?yEYXQ&DNg(^(~04gPMB6XG4>B_@^_uo1D4x(3xQ2t!F8p}s`)^ds(xACvX{xMl6f4|_j`diXKS*N-L1ev$+A zqhp01TEc~w4fDrlxIa53v-bA1KMt4t(ZA_W*c*R71o?ApbpSmN1<>!`0R9sHdW}Z_ zOMV29RUg2-{ejGSC45)(Kv&_p?L`x8F1)tQp&&HPf=CuFd%sCAxot%Mlib;o!=nG4 z4#wbWFvo8M^XNk`S4@Jj^9yF#=nyofi{`g7gxdxo%rg&RmuP@LG9`P~LdDL>DppTd zF-Y{lbD{_8Emz4bRB`L23JuZ!x;GBRqD3fo#KYF_6)N}2P}0VQvSvXjwL3$(eI%5p zCqw!7K`1`LcZWHKQYBnBG9Z*)8p2_lhhf<{jL@lJSR4r>#2}2S>M#})PSDnHVs?Zx z^m#Z3Ux%~LIh=QH;k-(g=fflDvN(b*Ya~auGlErm5yUvivuMekB}ZWPN_4@`k?b=U zjnFO<9YG^iBoh)N`A{86ixE*Y8xuv^g($g$MzPd8ibBbn)%A&H%;{*}-;O5TLi}y- zXwpKWxfvVHfUIcN7K(oOBbw%tH*2(3bivIrbl($0oo)=y&Y}-y$MCa2JnxaQlEsbX z*7;a0ZDR3qiRF&)(aEFY*gjdb!1-~U-V;aY^Eiy!#&fQ7JY9~(V*@PxABsGBq(e|vCB=T#S=zA+AceW*wjlPL^ zwMnAq#3cSBi9v6Zc%UnJv)##52>(pBN~VQpGH1U@w(LhTuEINepHJbMV~S`^DKw2v z!B?_mONOR$`$a0Xk{z3&o624PRF;;ck|7+^bx0a!)6)2`GL7`?G~%k#=sP)G@|D6Z z7o~GxTRO$N)0q;J&Yik+eAj0XdOd^5@fplb7jBu6!JGOF4i3#EV_qf!D>7+*Jd-zd zSxgcBm~<&yKAW>~7TwM>BAbCfvbonVhv~u}cYhK6E;xtLk`FVupG)K1TrTg*V~1uQ zNx~6(bQdppU_N)&=cD&EpGPJ6*oh{m9Vxn;AU^Lab?pmi*tLLm-3s_aFm8T5D&?T6l->=$(X{C|mNffD{?%_7MwfHBTLlwVRPZLK0{v^1 zD5I*d`dY=U;wns9SF=W`ra-R-E#*6D>UFH~tiyfq4`#RjN&o3T`7-bqcennc@$-6G z&sS5l{#I88k5*UocQjBQ?$l7syEIl_4{NMs#O&d9i+%hx^dN70{KplibF>M+#5VIQ zWDmbeQNVQ!yl-H2_a?1x-{P$PEevyRF`?5vj_i-z?-H-XCC!S%;!A#JzNAcfMb`zdB!ln@uj*I$ zb$iXp0dJVR=nW$dzoEx}Z&!Ob2gLIhwMTdUH;$2qiu(gLS`Qvms zcU%{@)4CkFsLS|TUCP_)acG(z>kai-^hJ**V?L2F^AqQvf1;VeC%O#zEZ?Kg)Jaxt z$m+7N!ug|=%`i%a=0E@p3s2ys+|CShFxx#=g8w^C} zHNd6NfS*$hIjC>QTz5l;#u&0$$B4;hMtsOO;-!OlqVdMGPBdokS7Wwmn8=;Wgux?C z*f`sS?n_KW|25(2erI%`-xRH_rreZlToYqc9+{bv z8)!T^wbfz>!%S9BIALkv)eTF+J>v`V~hGUvuQ_Ek{hO9qAI`DB6W1QN@lJ z)jG2Emm`IpoH#egiQU5U8qIQI&p{`A&Wo?x%$XH!ooPSanYkC7nXKh3nQ>=6dpNUM z_+0N`XWB(MvnSq}PU+56PITeAaJj?FTo}2=g{X@zJbvIpg6toheeKE(eOK%PU5V=G zM#H{t)Q)mv%Q49ao|N-xH}*buW4d^}%d{msXyb-ffEx{h-0(?tqj!-T3tGBk*T$W@ z+a*JI#GN8PcQQJ7pw+_z-FY4~+~k4zaSuG6c!)mc!73*Y4AyvJFW#@$Nl#{+^2FOh z_+Nl{!O@;P%k(6vy%$M}7YBEHG2&nGfKPdmXXnMZ3gLKdyd}f$O~@K=EG~OX9>ANy zX5Pevdy`)4%|GG^@9FEqqmhy|6kj+g+=o`VqF*-jrEH=v8|M3RP`qD@cfK^%@s&HA zaJf!?-22l{_H+E$Bc8AM4cSW&-}j1eIX1K^ByYG?%b%tC{&x$pIR1z1V z4+3fJ7RcM6Kw6aqvh7i)geH6@0 z!(g_j2Q$4Sn8#B?L`x7Z_bvo|lMwz%58-Z62!qvB^lq)SQWv?u92csM^dpVk_E>i8F4w16}KZ<{4$dEwvm|j zj-pflD6)n`5jQ@H>C>a=vLK58iKdvdCyJ+{DRMN5;ipAw6i@erX%rv7MDf!xir&K0 z^4mt^I4l~~*l5~L6VA3i8l!8`^sGKedC2p=1i#QB*?IExQE<3$oz(~~%zpMn|ypM0FtQgf;X}YbcQvJE9(q~L_WpMThI+vfpYWD@y zoUd?;Ys@sb&9Xo5V&(sU@ug4M+gWz}CTp@MU6W&Jknv~dmYYo(q-Q$J$lX5qxla#9u4_~o%SaZ zZ9e1b|CzfgeO5LyU``(cM!uB%RI%*!4Kc*rQ*g=%(=a1OY&T}}Yhy0NNQUaR3E`sW zWEPvye4;7&GW#74n_+j`FJHmqfSYqH|$^8gRGHS*D!;1edThU20n;UA@v>b1Z&vI+-owVk-oizsn ztVxua9@gE4uET5ydThh0S2lG1XhXES4UPS5sEfD3>t9>Sj@Y8=W5-`p?68<;hjxS= zNs^gLUnRQB8hhHr*wbsM1A8YrFm;v#d+HtNtM16`QI1U6@5twW9ocnUbd}{!JQ7du zzP=N7sZOlU7tC{J@O)=Z>=s?*oHMVioQ3Z=Gg93Jy>+6E>~~@7F&FVFT?l>U!aLDG z_KM%PysIm{{&8hkS2uo1w%$T!=X{x!TO|**S>~j-N^(&pqKo`+qxpXx%s=NreVzwb z3p}V4SiSS4la0*4j$Ztn;>E7D;>(FQmvGmM>-W9*7V5>{WnOF+UoLxw=oYuUslO+h z#y4-8-0~sU$OrwwzO)vulbtL-igNv^&-bII+K->LvWF)c#pcQWk`?mjhIn_mEB#q3GjNQh zKknkyos9J7NB;n}tq5S(jR0QX6@9`ufO7HbLW2X)2@9Z;n(WXG4a96#AV!MAHpxawbIlu@FWDhj3anjEW&DCf-(wuP5`)RK+z16+a^+ zx0I-&N!w6%&k#9!CH3VQhUChGG)N*iGU5 z@iv?_pTgN17>-+7I8DEXQ}au(D*~5e5u7*^!JPLIWR*r>^&@kvSb#jOEyWeOV`>*(`ZaIc{8Hv zurZnzucFcNjpkBXG{f4*P%oLK&DUdSBHF`q*BEZa#!yozIi`WJOxhZY^^RDM+>GV9 zWRyG&WXH`=_Te&PX)M0j%=4l@xW%#2J&qGmaZJpOCu_Ml!qqNT#u5@SiSE zW=~u)oAZ-N9h}1C5h*+#o5GtFDTJI%VdUEssz1sOob12#4^F`{K7|usQ@Ad>Z^!;f z?$u3wL z3v1J8BJ*;}{B&mQOUG1p;@ZhhT-Vp>ypz2BKNaa5uTMu^X6NU28BFb!fy4L=J}i_y zIl+xp8O*vNewt+Nqeo`S=c)K;M>9zgePhMROujqI-ki+O=SAYH&CX)-_AD&ki>LN6 z3#};0*#FETyLmP@x@J>zEStV(vYBp?ja~a3hTqNMdTtJ;WjPdUV0 zSV+XhLK2_J#-(u~y-W+ad$b7ri|F}BG5h-!(^)*V`TE6-cPqxlruhyi5 zZ5buJs3~DieTnP{mr^{nlnygXv6xkgno%kF`K6fEm2&&9GPZ3mW2;_Cjo=aGlua(D=v+Bzm&)lK zT2Arm3dy=vu;*w6lmDyW(#Z;TCRFfmvq~Bauf%hFB`;4_vh8&x!%QkU=2S_pTP1zE zRG1-x?MN*04OX zhPtmc6tAjf&&FDQ?ylwN#airIeW!nq@3NaGaF%_(aN!d-E(!)5#r%xC;ykor$z z8~ozOfM3j5F1e`HzvzDR7ZY@U;k>P$REv7{7uTa!QIBOCHRWb!HO0NFnqoatO?kXj zO|g5frbIhP^-@!Apsu`CsVn7!8Yp8YG*E8cYoN?=Y@j?#Z=fuxZ=gKssi7Di(omY+ z(NLD$*HF3`YAC9D4Q1+~hKiL*L&Zg~T-8u{v#ODj@v4zhc)78%=0jtp&A=wgVfQA= z{4q_H&(E4FG3}cv&(1YdYHl=B(r-3XW_vbM$^)AzM$?)rVQZT!>phw)OZD zfFq%jIa=1l2@NYJ&Z^11y}5WX%bex+IyjSX%bD&=UAS;Taz!^>sMzJoKQ*pAYVF3( z4Q@=h=Ef>3H+r^q=RtROE?Bvv?(fd}5_j6ky?dCZ2Szy_^7nh9-^Y{K%bt?8@svFf zPqxKKE~t?gSC4s7EO+b8j$W+m?~TDsZ!VwmX8RRy-rx0Rf$)ru5#H>*=tDPOAJoMY z8C~GRt}ebbT;@x2ZC|{+ec2rE%NpSq*E;)gZK9trKH(Cp{P@${k3ntx(d!_4_#yr> zul?CSAb@Fug(u7nVB8Y%HctN^NoO6^b+&eKT2N8!ZXLU^V}Dh|ZtPaa9#IgFj>7EZDD>-y!mO?+tUkqVy>>K~88Q2u%();X8h0}3f0RaZ zevjtt%z1#B9fjN&gh<7rc5^H~zKF$jW_8}S#p3&tIH+%qqZSp1?8|X@LEl2+ZX8bY zJuI3Rhms0@T@{B*%r%G7v$%dN9u+?EkROtO0F4CPuu6ah=YhQ`;Mf2fyl-PtKHUyy>d=N_QGI{!2rKQaUuBrsL7ObPVMke_1>OMVHxIzn=ls7ktCM%Yc4j29EOW+AuK_ zz3Z4IKAwr)CVbO6GZ&MdiOSRT6y9giv&}*ayX^P=%f=XidEpIw&)&_(zVF%iYMzaq z*4bE?&G+rV9H_3!!PSj9P^9PZ@;-a+PjgWEo?SYJ92^U07N(AOeNPVd&&b8q`OLWR zj_Z2j*jPL_Q|v z<>Om>KKe~8fb|OY<{S#pKfC}<2?dCd<9*CGtzvB<2B{RG?H#*rAB&)FU4)ChMYwmn znD6sqB++|_kS&4Z?h@>JT!QycN+1(bLQYC4dR0oXU$qpMG)occUW)C3rErZY#kG!7 z{2o|_uX1G=F{})}qsr(_mBD=%zur@Z*H_A*`GEe!t1?8WmEkh)=+DMw_~pyLCanzX zTlx8~47p>=vBsP4-}M!!zg&UGuPU(nQw6$eD&QwsiMsxkXkN#+@Tp4de_V-S|9cm+ z<2EO#5@E5G7+YD16}6SPEXTgv^eUWGsN(yfikX@!#D=l!mR-es2R)6cyrZ{OWAM>x zSbwd?V$EtyNUw%-RW(L*S7V`E4eXaN5A(eSC*5oK-$f0KP>U3uTC9zzh0KIH$S$gb z$+9}wtgOSzRdrZv#J*cZ9S-xDHr8RVEW2&v>M?J6J)D@6$)X<;a-|-P2K+j)9ub-K z=u=#exZZjM4rxH``UWgJ)qvgXxD9n@$1RKbFv?S|M7}3a_?S*luk@T}c~cd)nB`ZAbFpcKGxDFTBu>JiT`8GHpkVLpz2C zx8q!4J67E3z?i!on8*%X!qZOpnRP;;qZ5UFyD)M@7pAT0g8Pv!=wIm~f29i(?sg$a zm*2PS0z$iRJEn_yUiRfU4>%v`MxSHd*m1iXvM;-l_=frEh;E$E>c(hhcywp?;MmR{ z6rJwD39}xoN$Ek%)_?eZjQzR0|Df`eU;F;Unf!nBEqjr?PfpbZdgm6SjQh0b;Qb<3~`GiM7O;X5^k`menNeKsrNC_*(NC|@=C5Ud8 z5{e#535tKDglGSxgx7uh2(M=J5vD)yBglU4Ba8{|BRmn27H*A@7J^1g3&-|K3(1$H zgi+llrlH*iT6P z-e0IG?=Pqgl@+?T%L)sR$O=ec+f5lc2_)Wy~vx`e;}?- zeT-h7ccBr+)31~HXGHJF7$NG$Sfyi(K_n1e#u)vY>NfduG5dpX6F8|B^-h*ktA;g$pcpO)vO?X!WvzdtdTEmgB^Ws&@sUV zf0(V;xWTTXwGB3<+F;sZTYRMMw)=xEmTKA}&BzuZR<;ngvBeZ8TU@3NCp@%c$IcF4 z&FmmwZ3pj9_E^8b0r@K&upqzz+cO=os>uOi%-(+*>4<_+%ukJRguy~bq%CvAnjMZ< zF6o5MYfjkXW@>xlK6p8jJ;(7rkiO_czsm<(KGP4=q@JhagI|t5s1#w}QHl%= zIbVF6>WfYP`C`!wU*s7{+~!%-7Ic$oPinkUGLo|?ZO&h+ue5IJU{Ciuf{yFa}de|8-F5%k<2W8e5= z_jmr-!5^2V(Q4|7UkNSqJh?4@6r)AhePK5naK4z_B2_I2(jiX8$E~f}kK43}c>(e!)05 zAQ;Dn1jC2CuaC!qvE*zpzN-gg3bRwA3xaXKf_bXyVE(;=;UvZ`0NEavN+Gy-Is}hD zhQR7`2!@-5V4O<`PWyylY%=?pNZ7SBi|YaPej1was<+%BXBZ?K3z@(Zpuc&X%;=XC6Q1g+w0EL zNX(%I=^;wJaSgiy>!VP{eAO8BC`>VqLhE39Z_A^xa$PhgDMrKUP&A~DMPpS_G>YfO zz=WBq#s@K2MJ+PEE(Sw7V{lw77Im_*kW^p~U{fr9vM*40J(j&VW~%Tb)=5T=X)eK|jwp*ppV?2uFG zFs4_x{Zl%5PU)D~#*CF@24>C9z}ih2usWQ91=KJro@Lyw9z|K;KE;yldcyXI$Z9=pK#NT6T$^H@FtPSCgeO%6y-K12%gF|?9rP66r;7C=eA z04E~~a3-e!yUGjTN!{^6Vj+$*S2b~Z5$t#@mlPp?M-g5!Q`LC82uai#7ffeIa$PYh z&ljWb^J1)}SNHR4F?xRN}uDjYsq zh1_>lm{U;&rzYm{N7kTAz6Q&8)?lep4bmQxsqvV+4GZ=i`R)j5tijxUwRo#s%ekT! zdWY-a#++5Xc^%G|)uDrLjIlfF@t;*a3{NybsiOfR){SV-X++N?_7^X+x7fcKZ_hU) zf!f~md(9Xc!Tw?yHNJ{w=ymePDJ_^$*#hM;tvGeQ6;mF!qV7X09DcQ8x<5G=nXTwQ zxs7?{Ht5;5!LOkmVu{Sl(;Gae(1|hsbwPD)53(z$zfI_cmXxUAaal~*Iz>XL)|M3R zab6tCS#SRHP5AQiGIMsfaYcBF6D_YHC8CPf{%SCZQA1m?8iGsIV5I#X$GhL-?4A#3 z*Z&ApsNSS$fd8Lhm?%H_%hPf&o9o#L^!Ms<$V6j9O3)OVl8P)~E@who(4~jd<8ojB9&D7H{ z^*4HE|Hj$ZJQMVBM$m^P&*>0m+b5dl(iJv1&w1-K^Gsg(HYn|53zh!1n7qUm z*7Q8@eYVA=U$(fFL=Uvy7TdSl!Qr4C{@Z4cVXEYlYT9FnojsgI9WZny84;@;Fi&v6 z==BZ=x#fV*dJf2{a)45!158IaV&!`BBAz%RLCX;v|2U$`%n@5%9GRnWgiSl=GtOh- z%sXkEcEX!8PI#^DglJtS7+5>8YvzPOvh+n)J0n)v8CucKn3>{?RbnoXI7Z&YZ5Nzn zwtYCW?Q-v3A!10q(U!fz^I`6g81D|%74Dd`*&PYW?y$Mw4$Zsd zpQgCORn`NQ6FGa$^T47V9@uh^^B1#B&pCVP*RwM@nOUS$p6GAjiD{F(uy3sw@-KQ} z-hD5`_hyl|_-3x>13sTO#{j*JO^4bEPiyUHEB5lAmJFVq{FsotC;`}5Z3U~UumXAV_`Xdx+%%T^yhQhWd6eY4@n6V}d9qYnyds7&09S%dYb{JgQ z!@Lk5hPno_RJVsilbQ33kKy=a5Dux>a5PPfK-km>s9cS}ExiaxGIJgl%y}tcePt6HKE z*cpX&y-~29%(-b+G*Wj)1 z{;pFTO#a28rcXSS#>YcyT0Aw$cJi0%~?7 z;K1z!w0=#XznTC8>bqq<2~eiKd!t_>rcX~K;~)_)_9sI3bRyo&PlEr_BphC!gv8DK ze8br)E(v=JlHgvF1dyqgD3uJ`qsh$hCu1Ld-(qUO&DP2IXqyZPmt^eWiO)?Yqca%` z<}iCcKLuEpf)IriG@ngj2aODi<`k4oW)E^oDh95l{`)=^5!8fzWtlM_o(B8rX*jl$ z{mAQSu)0OBm;P^03H{*uG^|{hj>whiu>YA3#~Sk0re#23X9jj0Ay@4#ecm4#i0{h4 zLNVscmu0f6nu(Frf7b?OVs30E+OjgCOJ6toUKUP0$%0vL7M_pF#){k7c%YdLDW7ck z1(PYk9Mhk|Yz%#vgGA#TJhaR~QDzRJsP(E0%|-8qTs%3Qi@}$3;cc3WCG0oOOkl1l zHy78cbFmM3Sn`E^MvpwG#N}5-zW^b%1(11H2r+{~T+S>+crjTLBZ?rgvIv8DJoi!u)+$1`MG@pu zr~?-k!LX6@(tu)!Z!3o8$zs$!E=JMF62yHb@2tE8#hiuC9xBBX^35KpmqI+G6uu+N znB^_Q0>v^isLK%lh0N02GN_#{N7-$%Nv{~*t%puwJ)X8ula*vP>T?5(5*jdVNFxTE zY(#@oBSyqD;_s{`^f}#x$?uvVvA-EzhnkUntQm^GnxW#OzaVtFckpZ!`4Tt}>A&)asr&Ak*d{bxmloj*l{Ff;0jULwNUCFFH&6BGOni3!Vts0${D z3&YGM1j7mm!Fh(HU^%gmkS8f8gzP?qX9Z_*rT8_TcfY}uX`c|g?lYd5euGWhH~L)P z@#VyKNR0S_4VgcXzxXHa)&9ihotn%IXfj))gP7gF5VKzwB8&CVWT!_K!*BTN{YFOM zZ&XG9#==Xw+n7MzT^@yCSK}Kj^VuYGzBOEwFjp>asA`Y9NB=-M%d*8R3Lh+g@ z?s=O+GuRYErOj}1k{SLjHbc~AGh82Tj!J4wLj&2*8)Sh|LoJZF%>tJ`S-{VPdQzbU zRLL_{q(|BBmnBRcErD!HNHVw8=wO8!>PeqJFwbpi4X=29=3C>Zq77P(*+5FyhFlUG z46P@JUCI`IrnZoBu*K#GYD^`zs9R@;1)OOvow8&8(+(Fe((k0Uq7|~MB8PpLCw*Cd z-tz?BdqT#=6IP*~c*nk+9JAHlWu7>`!3**Gy|DE-^`QrxVN|@J_S6dsv0l)h>WwgJ zMHAP1LxJ76aWT}4;=FNlm=C@x`XKv+53Z>*^X2J7zP}G=S!Ti}`Jx~7pLGttcoybM zj+rke*O1*V${9q#4_of~;fj_YGPC`dm-ok<9sVe}>yHg8{^)t{k2nK=q$l{3lM;Xh zBLbkgDge?70chGvedlTblA<_oBn6;6n_u??U@938K??%`cHL4P~vl5d<({zSpDAPN_TMw3Gm zjf$($h^23=xbwmA4`DO%LG`Pljq=>fFaDBoh?j&LtoAxgP6NU zBF?O1-g+~c>xPN&izeSSF%b$4iO`ymgn7#3V!chmcZVbt1tc;5nS|hoBrKoEob}6O z`k%>|ZJ&(gZpmniNXDxo^0KOu@w$O?$jB57dz*sp?AnG^k8XWBxG>JC>(m*uFGuWA4nBTF#1} zE*%?&q~q&Ua<2}hW8t}UnBPgqFy^i8zouiaSvp?0GKcL$MpguWT}C>8@22BupA2@| zGGH(#0}mH6gT|R9+aLppwi$TiOr}F!2Kulox0#GAt2OM!F^i_~Ig`vn&Mx7ZSd@^7 zP}wYqvM(nqpM_-#Sr~IC3xl3!VT*be=38fBQFs=VcogKbA-*{qnOm}PdV4l}US;FV z7jk2DvN6&!8!oQdun6GX6O;|ZP<}5r8)9TT6n>@dDml?W6X?^Y~OAb4T>GwaIZP&%=wQWM&EZkT{!QP?np6O>2s@;FtaUQg6mUDFzr|g{9MRva4W$QANt)nC0IC(+*Wq%mbg*# z8CM2uD8t7KWw3Q9LtJJVy+C^03guY6sT?Ld$;4vsE;PHG{svjB!z=L4y#nq_$-mO5 z#F1~6SmsoTs64XS-&A3oUKOt9RpIO~=EhbrCx&YDzg`WkcVx6%RKvxp8c!OlF}%GR z*<#d#*4L27Q3JClH4q~Q%kge4H15}8oLMb?F+cYHB=w-5oO$x$&{;dc>D^jG}kdWF2hgt1tF=)q$(hhh^bYiqrCwB*QV(eCC zwk~$URjvzB3FKRebz{KiZWI`I!zi>Hk0ZOWdQA^P4)$QZXAj;8|DZah7q0TXXnEF) z*cZKcJAe#!MG;}uTM;2fj+}){qC$7LsPH>dROnwWDy$qLCR}_eCR}D#Yg~tzpfFfm z_%&BtXnQR#46zm$#FE5?!V?mL%oz#c@-qoRC0asQRwp4yK~h+)CM8S?ml6yE`v~%h zeS||}qy=+n8KIx7jBsL-jIiaPj9@3#S1|k0S8zxgAQ+7tNX*(D+{(C%RILa6vq23Z z)TcVmen5orC)^(W1>Gw&Fzv}#EQzo4~559#OivGu1u zG|Kex#M}Urat+Y9fxSF?Lqy&BgY7^6z;v|{X1z3mZi*4EJvGMRz9v{EX@*NN<~V%) zFZAk|Wj|$woH5pTIL;b|&#W=`8@W|I)O7~hV(3#l9O<%y?<9N3`P!p>xC6AQ$t+MO z(=*Tk<4!puu7;hnx6b6XFn2!21><(HBX-vX{?t%DO1R-tKQ|nrkEO839b&)fR}J*Q zoZ}u)AU8@qjyqtoJ@A~~lTWTEq?~wPXXR2 zj_`&z{ifsneb6p zVb|9nochUZ7w>*#1z{@LnHzc6zhpn^@|X}DSsa4rt3r^pIRvA>hQP&}{!C2>EV{X? zMUENr$-K9ZhobvbC=zal;^f0nj5Q0z@v2bN(4%=vjbZNTF#J0khSQ(IaFia+Xg@MB zgZRCEWVFu>M=ZPUa_Zr*peJLzjdwcl@M)VO(Qr8u*VV~tBI9xzbKlQ+UpGo{f6BNh z?ByBB{!_pz=D&B4&rU9T-nl5;Js*Wr*P^iP9T}MPW8P6?sHlpCsS8 ziN-W`qb6&xC&g~m<*I0e@Eqcm0bvZ2BFC2Y<+IFOS3C zZg%0>Yqyrz%p47y{q&FUW<|nYzLakvHneCes z=ocqI{!#)y*d*WgD>G*uJRWt)csVWwX6sXM zTrUOf5h>`YOTm^=saThkifu*oY-*S%?`E%kW*SBuNyBWlG$iV!QLjoP!#^EUc!wW( zL&lP7ItKI3jv>p^HZL9N)D^Pmy$p@afMt3H-10MUj@rU##({6y2u{w1L~AxYCgtGf?i{kGa`4~J98A;8L8lG#Xx`K$e7IAE z8MK3~IatiR*`4p?x$EVk$0Zk?LAiLClZ%7-xoDZ32iqxmxVDCR#I8K3?asrE6xF$MJRDD z!mWTJ%q%IwGoE!r$$3A;?3;QqtiBcFMnEwc9VOUjS%MKgB}jc+idOGZ)K!#1Wl9-_ zttrFa^<|i}sSKl;ZQK8t{3fF^vLnkds;&$Z{*_^dVmU;V%Q5;|IWq2-L;FQJw!S4l z^L;tS$Cab2s~r7%%GsN#z{ztJsJvSNQ@si-_pd;|+zQMss6Zq8U;SrPQe&;eiF=i} zsKdOQcO{AvD)G3k5{*h#czlRH(u*qOX;e|etipn!)##c~jmu}MQN>KUId{Ia*ztRA z)!6GGnxLH>RtA6#kCs)s;QaxVnsfQ-DmkaOf zQJz5dW;!`h8TD9@%g^q5{GQnWTQ71qa~p6=vJu_G8!>)EBgXbL;&4Ca*H$*ce0vlA z>}i7TktXh{;@9LtE%s<4*Ru&1(wY!9s2P8^HABs#86C;Zn32~5vkqE4ofSZyS5^&vK3Fs7HAvXhWV4)uy6|V>Sx=q=0zLK0^9I1stplg z?PPk8&Az)6la)Jh_<1LuyzhjuO(&U!olxA~h0RyH;NjPW;HWOLsk#u?(}g$vyRn>_ zPWkU{jIiy-;?!Ekc*L{w%Ri_aQ2(j_hpiKPp}M^n z@h5umCyW`hoL+2VcKzlw5n-{Ys4$(3&&4@pdp3v)Sv;k!qJpECn4n1B)cJShPML`b zb8Wxu52;nm995;$*E$2-yoI1f%B?!UAmxVSuiLpq(NiL=BJ>7B7|* z%2!AVGqy+y1jNs`mBOD8o5mJ(6gaHf5bl=}skl)i!&`s+nyin^e zsA%^WMkMtY8deMt?wAe~N@Ir#-k*+O$?;3j8+H=`X3tZ67y)@qm{{|=(B-=TfxN9b68#J=K>=+*dyk@=sn==o<1pR0j8A2p!Z z^bP;ce24Nw@<^S2P%HTfarvLz_wf^!fj^OcOp}>#O_U^RVOftB^v7!B@(yjZoY%&} z3~fjs&_(iBUEFrog)#JS;kh2v()I9kp+1JH=;IYnBjWBzQ5dtq8!RWpbGUglO&{AWF-8Dv}r7`|U zn?P!#3Cw#;uyCX)qCS{HA;1*l!c7sHVTvnj&EP}N@U=bpr=Dim>T8BC$!6SHVvdFt z<_NrNj$Rvc$Yhygk>g*?bpH#xjK5IL{0mtr3#?gfL4Jk>v^H5lc83K{wOHWM3`?Xg zx5R;!WXzwi#KKdSP`_gd`B#?gno@VOwuC4(xV;sYm@Yzws+bivOy!Otbt@e6w!)f8 z)(D(ojnFODICRn)mkc)N3r$PObL?cllH9?`q)VRXeF%5V7TXb*GtE+ty*VSU{J z&u=>*lKNcvbM8re?|?Dx@{skCc?Fen^biN0i(0$nn*RDGuzRC&h zBb`z4!5MuuxYH=q8KH&FXd36joQezQ5puGwvy;gjn8Y_1ga*3cd5H^p$X3|6#FhMa zS1hq`MQw~LR=2a8Ii0#)7U$=2?&zKM|9<15HSVZVWajG}J;&?r(9&>+l9f9~+PLF7 zy~n9XJdpd)1Ia2LxbwjS5C3?ei@sxSmIn&jJ)kAxiL?=(Fd#=^)O1e-&+)|0)t-pu z`LUgTfzl`+M2_=8&J;3USCgTz z&IisHeE7eeo~4@)p6B`?s?7%q{e5w6tS?qC^Tog&zU15c;>9Un9O7(W{){@|2Vb;N zFPx?Ci>L@+e25_vR@@KvGJbeD%nv(9`q9t!!)6E0`uX%Km-wTUKIO@E{%E=9kC%`A zv0c+2^KASP(B=>Gf8;HYvk<~OnDm?g*s=4WwKo8plmobTDgf_8x&JXQ07q*B;I$+W zeLMpZ5*vu(v_Sk+AX8y`5a!db{C+D4>+h16MGowtRB{*k2E&BeufjpW&>9vDi$lRs zycG;F@?fn$^ZoEG7!!1YvDGmck>T9+$Q;x=#h#*>$C{8a5vO_>WJ!&+zUm2^M5(8wPg`l+8F_dsgYQ5HWKfYBVqTN zYz7nhp41d$V)%XbQN0I7lf@T}Q=6ihU7~M!G#Z6^JPy&YE{VoWc1~jUkQ;lD`=Fd+ z5W%yQEC%f}v6#uu$ruA>z}(5v=X+vlV=Qtcxc_N?9QRk!x4aXFk^jqn)yBcTjrySs z-wb?5#7~aLuX%hkER4tCW$`fF#|}y}Sy_V<=u0Hv!I}hEDUhdclz<%(31lksT~U&N z1p^cDK}bZyZgx_$$a)>hT~E{ov4UKNm?Z8`NP^02@)VY_k7AGvQN9hLCCO+VPT%oe z3SRM?c%OpVb}0~iQ_vKgf}=SpQ0q)V*rrqzoK1!GlT@ry2Ov{hwkTeR9U4Xo9_p;_H+yo z%Rs@#48)LupQDiho39yY56OUHNd_jdZ)&xdj0M$9?D)v8sYfR6H)q0+tOb>w?5F5v zVZUJ(eJy@1LME$NHg58r@N7~x0{Ko@N^Nf-dn>71v+?{W-wMC8G0Y>I*@|pt=Ch$% znvGvA+4%A(2X23JFw`~&vdnc!r0~ta^OXMOsm-|v-j$2O{&_G58Cm=DpnNe8DG&KJ zc$LRaWgc^lc`$X$qvk`+uOSb&=Hz1;vt7}A16+#A$63z%Z%!1zi#?S@rvk{HD1^Rg)=bt*b!prV1!$SHO{7PXlpwMwBYC zO1ToLKRIU`R^q8!C6=Ut>$*T-=63Bipg@T08lG?bs*Zfde0?sTp+Pq)9i~(A`*J)l2_OL@>T1A~c(b z2&brrxzZz)c`GiQ))E&ckGT zJoyFnyEV}E^(!3wzQgmWCX$tO&}5?n>!@F3spw);v@Uu?$Y^K3(0ivo9_=zjdF>zA ztulhm1|tM08^Pj}5fWUCaF6rLcybvUUz^~+2!8f4h4w{LgejWgVmp1mo#qG%H75tr zoNu+ixZd#>)#ok9tgt`?GfaA;ExBvS60bH{!EzP%D4ScupLwOFDmM5#)D|D)ZK31N z9m*rfTvy;en|F2?{oM{Go_25wv%{$w_PG6s9Cm$s_$xA3#QWahoD)jUJK-De`jj>& z$cQ_`__{O3+;T>yiZdR5CQnJ4cRO{kU5BZE{dGa5r3*TIU9dda1%t-A;!2JyUKhFI z!XRe3AGkq^S?)cdZpe&ugJ=aAn6mC<5s|;Zdp(c$`Jg;^7|VKK;A0QYJsv30C;Re* zCt7ZLayKP;>ZM*}xOrpzT5t65?)70FTPC00TLHbcg}hH6`Opvd!I>exuvkk!Z42+q z?Y_`@;fo2XzBswT4-d&sZ@W&ef~y}E7W$!A#2@2k`eT5aKPtuqUIX%Ife1aY5r5avz}#;@b- zKVA$*wrMb)bq7OY8uhVHA+U9#C&o;3b7Tm)_}nu?FU+P-D60B%zs!~KgIhog?irzV_i zW8Q7MBcP>2pG!XiCO#3kGCqZfoiNjy!mpcVA9+|y4!~1KTV?5@IC7@8AJB61f;QG1*4BDT7 z)Ab41*2~<@_5a@)Jbhde?vW*=${mF#jo70kzj0o762?>ClAlWNYB6&(LNe#oWDMO# zj>5%cocKd-tb6ky+&#-t@NOQm{NN1#@y!nAc0eC3Y=$ zu1LjA-fQJ|Q<42O75&+-yvcs$?m=mIvM~*sr_-SIg!_WuGnX8ghWUkQcvPN-O|5D0 zoXFiR%qNdpoDRv&)WZIxb}IoL#fE8HOmYsz!DPljyA_8iE_=AvRsF1j|cTX}&x7qzbG*0~sxmdg%i zE^|D&D3H(NpKE#Cy~H~$CJ$#?^01m6%d->6cbt}wxzw{(9^k&hhull}nb{leD|~04 zk1c-WjVJKuMGMI6r7y*t@~0&QP`g-woAE@nNxn%TL9Nhh0xnkh=B(S@tL`t zZ{G_cXpuMMNj79@Aw(JqF?txY$|H;LmfWGDx%925V`=Ou!j>0B=>M?@GFC-6W?O_W zWD$J{;of1r{i`7)~*x0H*Kc(EAsbc%7=jJ-}jvK7LMQ65zcvsmshTu!#+ zni43kW6yJc2^8#0aNDT_?j0p)<#{VgK2d+NCC8US`75`0i=LQ$HInnHF{85@ zR#G+C#rszMP7O>nY9Loy12+*e7N*wXQm23tA$N< zEjd`+NhnL!f?OSr{j7s_eI4W)>d@7~-GzPY(N2D);rDv7pX-rQP>;oR^>EqHfEF>xvK@T`&zK{PzxgNaM$ts7A*PK0tb;+yqUy3gm+qz zaK9C^p0r}7ZYw*ltr%?9if>(HWxBNCst0F=q3y_8+>Yn5?TAch$HDq`1kUTgo?{*8 z|D*$hB088Y?8HlIS=FyPF-Wr$_ojEja(@@x?sviDK^KO9>w<_b2pV^6MV_(C;BrmrMqke>lGCAJSI$l3(9T z&5l}@2K$p!M1*56nQ2lL5r*iA2sb#hTo)4+9<3D>)+$rWVmI=Ig{UykT~zoQCMsm| z6u%J@TGYja)nCMfKRlMEV#0WKBzua)gaKV*Lf$_yVXT?bJy?kC*a)L*c>AuDWoI8bnZHBbmO zA0%|Y9x6OH9W7X#J%EX2FEHoGEAEbXjUqKwWR6iobNPFu+*L=CuR0dQe!{vtpRs<1 z20E5~MX}g-n8|*J!T#^~FXTIN%YWdc(@#ie|HP3{O)SmSL`9nxcZuq-_pO70KXouX z_7^>lUl^*Pi)u|>vaED*c84B9na|2zrjKbo`fwa+fa#qE*!Ir=E4LV8u#F*JX#c^> z-`qiHXoNQA)5p{sA*#`coH8TW^)<%b`&70vjIm3Bd2~k;xVD-=_n!$2PMKoe6;qVP zm_i6QL+o;MJQDkhCyIYzcK9#%cK?M;*Aeo0Ie0>k13B_O(P;vnAYG zEZLvHmV}#5jRo zr-38pS~+55wj&x!93frihAzXmSwr(>M3>wZ@HoVwHwZQa)u0cgJYE&mPomy&j5E+&v3`g z4EgAo=D&0iOJ_ZK@%KXuX@7snJ2EMdSZ8( zCsKMm$)xmx|!-3~=;D z0q4<)x!%Yv^u}5C1A}M!;1509)^|P-^JO-D9y@>sd=Yep-9U9;4EW?r_M|UvhWnx* z+829Te6eqoAF3AnL7Lv}_BKBlFZ0KV-TvsK&#s?^KYnr_;-zGNoG#+dpMl(mxH*8l zfB=N<4?whf0CQZ-e2D~N6Fpt+<$;J@#a!3wKqPDlgx~%^j5rpEEy{tAO=SLy=W_}f z>FI&csR)EiYanXGf^cI+5TrmoXMGTIH;|D|HhSyzAUNL$LgXE4JT*bo`GR4yh`W9s z1>-$CedZa#2#^jzq#W6qE1C78H(d0N8FYIdazEtr=o>SO?$i{5jt=J2B|U5fOOYrCWHmp<~`vC)uWuTO>gj^CMRYRu7?z&Y6}BN|6Dqw%khd9jr-Sh9v} z&0R6bQH_DO7I%J|#2`OB22Rqk+&Rp>h-YIl;0YOHunX~f#&N!@GOb!{CMc*FJzH>b0o{Gn!i}85HbLCb%B4nwuW*K(A-2+>`KI-BV>}jzthgWoFuwjwC_ODG7^%nH{T4!X0)5 ze<>s*<5)7BUNAG}$elpG$+(b8znN!mI=?TGg7sUtckv*1{hVfo>|qL?zoc%I${ZQz zY~5k0DCUtI%b9x-IqF+dap+Jg#P!(|%p}*Oj{6olbBq0#hT>Uim^3#Hv7EjAnJIg5 zkJ&QyG>jExZtPk*q^TeMt4pVLmyQM9>FC!t1G|Q1V3d%7zlSr>Yn_49^D<#X&1lBa zOzug@#QTCwTwI<7?Mqqcy_|))*QgzF?ymZfg%u`Qcpu3;SSGVz6Sm+HfTtiE^*3@L|0xG64A=wAphnb0&4~H0Lh3}TZ|1^RH5Z9P^6+bZ z9)5GS{!fpYcEdcZb;-jk=G*V|o8>+mF*5cH}($ z@H&rf0etBPk2T{=?N@-9r~*VK6`;3?-1U?~ESyI+=Yb;hxy%{5o_i9l$rEGVYjRF8 zJaWl4A>YJfP6=kIaxY?234gDYLUJhiC7PUh=a%92<}&g+%dl=(IaVAk$6ETghW6#S zR>ppxQUzXKsX*_O3jAVbeejzKOx3Bt<_DEf@vB5Web=d+Usrl@h7DwvE4UgDHq{`Q z*{vP6HFzIfgZw!5{N!q3yPy_7ACm#5T8n1-tZHNHU=~TP$l!V`npKaYdG$DQq8{(n z>XG@ISuW0@%3B*?sMA2U6IowL++%pM5sTCt(Pi6+@V!mUAvB@Hp$V&Gnjt>38RzFT zV|aEme)TqEt8@#$--7+KTVR#ng59IY^jOh~zxl1$Sx;}YwH2Qxw88!g86Ku>IG5Lk zy+v*C=Xqbt{W+X3|J`NYE215y!#Yqfx&w`M9k}Y(i6Z+h9GcOMADkE0WppEQR1a2q z^dKmx2b*GhaP@gF?gaJX>jV)Y!a_ti%Z}jY8KT0~g`$Gd9Z?~Di@&ya``%Ddig~oPS_^s}`)sY2$&19<-k6 z!y#WE*1h!PCK_PscLV4>F~quNLj<-OqEqG%avuDFWbz-BRR6&x8DreIYK)O_#z+t| z!OD{+D7NI@nM4yjE#Ta9(hPrZbHAXv8RXi`&`37IIC2o2c-E4C@Pk=(>pXgA)T|<> zSfcbc_X~2T;K}(`IFo9HiD_1NmuH2ov#rVXA#YqDYkZS6f=#XQB+CY?*4uK&j4hTu zwZ#d}H+n&n74Q7MyW-k25YYW4@8uuGmk`IO6S$FOkkDtaV1t2p0q|aDmkh7bqILz|-6X zA$Bg@pXP!|oJ(fUCEI(g8{|H?VU)TX^8UC%)0SEEcsDo|xxr3}bIEaT={iZaXe2*H zJupF4BqUj;7P6Qaa;_$a9|1Qt`y9Y)^QUl2@?6 z3vHTSWGZ>#k%%`e&UnMY-y8KsWO~<|PWIj!*pchz+@a`;akqT& zoc>X_jW1$csEvjALMh4@FXMb+*FcSIx*z=?KTL>V?wm8n!EOFHzn62zZ-2!5`(tB@ zKMJJi1uY0bv04BKX+=dw0DejaB7%K3+>pnB%8e{_KaaSNus9i9Q&=XoJ7J_-4B?c}J;ZEHU zXpA2HnlO%56R4ouUN>g z|Ee%VI+4rm69&VsFr-S5KRPuW%`?OCU`IGoAB1B~ZaAtt$Rynxfw9bt4?7JEl<^gPYXD)=!E>6SLt1lwzT6%pEDrh3{*j zzjHAT=k?e>3yvelkQ^~;VD{_ck#mIk@8|Kz(~rkY^LUIXW*%&30&?Fb;BGrJ(Cc3O+ndf&XjnGx^Coni=nmmJ~RTOy&D574pht zj=ds3TRRo2|D+I=Q*oCb&ylcHJP1$4vvhK`L58*}J7&y;jryF1D9to{@*-oD zJNO2Pq~mzB&VzNZ=JbF55%<8!ep%SJ zJ_~nLvLNqB?TlHmzHwR1{$)X$9?&b9Y%Cp;jV+_dD_cRprZ`|=#zf0BdIra2HM<)W9lu*cNPs_&3brk9JUow-n@PImk_`)Swt)biM`%ELH* zj{mKs=hK^qwX*E+%jLsoP(JcU=OcVfKFau+e%3A@ozeMtTapj8Va#~VDuDma0-TE} zfO}g3)^rtM;-EsjWv1&ccT##k6k_eqLhQ5QCZB&HOhOCMQ(uV0PVzjQBmp2r4SZab~>7O=8BIdn%Kud&#Wk{_(>STuv#$YkvOQ zQ16mr#;bmLDKd7J;`#^P0{*4A6J3hLX=V7qo4|Nk8BX6QV-Kke(Y|Hmv6ew%XgO?- zm80fbIhMR(uFJI?VuLD}>94@R#TEF?Jl8#=3fzeyQ&gl9F$0(ZA5w|)lPYoGU?l>$ zjlAG%C4$2%p)6a43nR${SV|piWfh)ia<4D43WLZA$ZxK~X}xOfkF3Ut8#VavRSo{+ z*TA>A28GnTiuTsx@!?whI$DdlFKRK!pcW_ie%sVybOJNt>12%N)^cNuo5@S-u$tMe zyt{P>e8P=nC;C&)b!6n%VQe$Gqtf+QKfWGf6Y6npK|S1#*TdMl9@~7$Fr$BEI-voh zd4J?8vynSjjaXRQgadrB(#<%|eAabuvPCnRkvO-7UUdsz z-)Z3{LkkiW$qu8Q_55=yuKZ;#E3XxEsApyDZsT|SHn=FY!9=SKxy)l7t!RU%2sN!? z?WpEkI=Y=6T07F%tMi@S0V(QSvPV0hVAg@muntJ3bJtI{6FI9pQMZTQSxzTHmUcnx zOcyTR??Sa>7f!^GC6?aB&!jG_oYxK4*WEbi(+%C2fB3bchdkIGWPAPZ=}hXyry0FC z|DYFk%DtG+otX_cLL&^2p zASMhwDJEq96%)24i3vJ|VuE9tnDBU*xbS<1xDZC)?#MTB!P|~}uT2ty!(9oXfLzfJ z0TO~DwX6142_a8TQiz!&DfC^!?PfB)vQ9|~gP%$Y(r+Y%PBlrPxK~OrTqrFZ)Rh*_ zRrL|3bdWVVKvp;dSz(%uoUp{TpOCHDU$~#&U(n1MC`k1iB&cT$7D~m22#$Y-3!C0N zL&o;EXbgIfO{|Pf!`C!0=#~bKCuzV! z{tv8`G%;ttHvYS#jq<5~5x!p!A}jRqw!;9~^9`}*v?1168sft_BWT|;LOJg)OA%vk z${NFRgbDmVnIQbI85F$C*oQKMR=yd;zL;Z2qB)-FSnz+(64t(!h(BtDkYiR@<7x#5 z3v1LiTI1>tZtWS_vR`G(-3vQ7_T%23BH5*{?a1x2!?--^Mh$joma@mW`S#d&+aA3k z_IMfKfX(w9xyi#0-+4zodPAP+XeSI<;so{U+}3;Ogd>B>et+o##b5Ldf4jiM#s%H=E|@%%8+wafF@FykQHHMAl|(&A(haJ8-SB0e z8;;*{gYhRf+_>wG#gE*PAL0&`Ge%iu}{RZAp_q{yuJlq4zMtH)R?19P4+3RD* z$Mvlz&PIBYFX_qNju(`jykIcg8*fK=Lv*D#7H{^(j=kKu)AWY(UvG3;dn3Er8|zxQ zlhW>uFJe9rU*H3YA3m_w^ukFn8zlp#o_VAQ7A~2vd0^d5Q&rFKMqUn*`iy#}| zBR7~WB9RpwiJ9CpnKwELUGt-`_iq%kYojoHU^GtClkBsQ%+?#!T%JWE-jus^@zDrK zkA_}(H1gV_;oTdJG3lp*TX(Boe5^=|tFE9}UnZ1dhnS?)N13dgM32HCc$8$--kIW=Ymq^C-e%zp= zw$eS48GU-2d(V;sa6TCt#>t2YA~!aK*_;G=oy_Nanw5g1$GM?=np^<-nMYrezf{8f zKJT|%<9NRB{Jhx{Yh9u+gk!~A}id_Fe#=HvW`0?eJtEdHAU>>*nq%(?(CZO9pjF2K$- z`kRu4n6bYQsRs)2I-?NH6Pc-BT!husU?!d@!f`%@Q$^^yT!i7*i(sr(gup0f?_+tZ z#ZjN>D#Ety`u#?lMLxcIslM(>L;?pHDV=wA-eEyk)jYic?FV@T>%D zzLLviSAsDPC0OoQg5+-U;l)a^W=Sbj_m*P%(^6{rrI=$`ild>Wuum_A>2$K;H`B9J zF2m8PGJZ~!(zv-?-H1zD zxc~C8iQhSzpg*J;Tfxn})4bE(He+@cbMz7|Q2Kul@_GvfyllmT*R427eW;j#*fSL}q$qfYKsb>d-nC;ssT3SDU3*o7S4W1XwJ;cH0GF{T??^Z#Mzt$#S% z@ee;o^&t6R4>#L-m~HEUUuh3kSN5R2r3b;Ada=Q#7m;MVyIv3xcBzX9_Z>t8WwPB9 zokWG4x9g z`wQiSm7DqrZ43Jg@{Rokm-ha`Q_+FKszLvu{=frn@Vv$d-FNtM_!9zslp*y03hOst z*&F~VC!8?_zl-$_C*Vd!P*$RRfpbyF1KIw zaPf#fHwg@||CIqV^vsY~7-0`z?h|8Fy)cHVhB0!cnc(d~6RaF%hDo>0P?cf^?co;i znq-Mf=PZ#`X$g7expNL!A-Bj1cMn>l>5DZSRk)95ZH@XoYsBZ-Kuyw?-6UIVJ;lvC zFI!w7A8N%Id!#M0$FarsSRY|e{;~s-oUL-jf6QEsVdl!jky|(P$c{Fo(TNv$~Zwej4lxpJBe}UEvF()!dHT>5G8V^uFkcO>^}{eW5S%R{D{#>WA)oen{)uM< ze%!f0eC3^F{~{2Nz6K(hyTFcNf#{|_<;WaY>RI|)?}BjnBRL1usMbUV!89QV(Jeu^ zw=Wo0*MjlSJs90db>U;#|{fNLJ(+J#+i@>zD2+SNF35iXS`28gkTIP{RVwP)^ zByXlw+oxlApA*iJq0fd=~iy9pq15 zr6(0*9fQNvsT2apquk8fXgjmVPh#}9XlhmToYwIs`gA`DP0V>! z8YH34CkYN@7C4SihPXj8w@8w)D2mL2*kttWOormL6z;TAyLymOio#*MZ5FbxxSAU{^pHj>q~XAeG-Umxzto(DY1FCKQKOo=CLPPq zbLUNk`7PCSEFynMuDolR4N-2)3E{8=Z-OWm$N*Eej&MxC_UAUwwm;q$kqRD zn}xkmS=`pkLS=pyUX7*aG%g!UK4#;OK{ob!W@E5#HkR{k_G1<-BpZr_+1OB=4ZYDh z@IRb`oJ%uEd43oMn^64kUugXujl7ObZb70Rr8s($;Yy+e0-yxwUj-^ z5r+y;LN>u+HFg@A`7$P_GR>3xr8@qzzJ+)`wGf|96{6!7H7wOa>N1st9qi z+{PJNgx<-#mxLmmJi}h1LJ`g~-*tm`(zNBwbAg)H#$u$9%XH*eF;*%T+yb_*QDslNG_j2Bo)vrwUfq5nM=t@kSP=&cd6_l*1a4(&A6Zr>cw(wRm zuZF&DHO#ZBF}b7~qleU>X;cknPOQPWWi=SSyarEqkb6K6?fl^yT+FS(cRo=6YUrs! z*oa!(W^ZxG?po;A*JAb%_7#WL;XAdiG~QJU|JLD?WgR@->hQ#?4*tP)D66bvf2R&F znDu%^B*jH(R^+e{m z6k4G|{ma6s6?d9j5jCp~PGsnh({0107-qq<+TbPA4xK3%+s0dJ2RZr0-Kf9v z4}kbKFioB#y8A`pX zlHI}!Qi9G4DdA$Sl<;bfw6JN1v@p(FTG;qcT3FqukDxlckMO~{kC2ttNBErHN7yhx zM$n0t5xy142+!CbtcaHtdK8(%8rNS)^&23hP8%q!&l)5+g$@?ZL=6!fwmw2g$7}Qr zdxO?tpP@D5Gg{rg;_Q-d7;g6sp++j`y7Ln?;i_1jqzc^`YFMyH4a)v%aO(Pv!YS(b zvO%5s0Cg;FR!8A*4Fsoa0Du3Ws!S6mleJK#qlHms+E}tx7t$uW(44Lh=TdzHM;kzF zksTOLF5M$q%@gev8E{`0!$G;f$Xc9W*Fvg zhJlS{_%P8NDtFBhXJC$$6m!f$JtNp%q7~RYK8;s z9Ub82;egIjju0+7LiP!@FmgFf#GJ6EuM_ zx7}XYHOz*I++85~~{IFgiFLR3@l(&*sb;l1Puh~a?&yAsX5$6=4We<{f6l z3=*?PhXcak#``RXx7n@CFpMQf+^JtUTrP&gmrv$eIBxuAUo1ZyQAOeOdzeX*iNI`T zl77r(e~i9jBzt3`A0o(%jG!+U3AKtytmB;)zn{BA%o;7wi6RS+`@NY_I6E&IZ@Ja$ zb|M;2Zbn1*PBb2Th(^<&XqebXqa}+!%c2oj8_liQ7<`)^gHN+#P|Uo&t1Is|?iGEW zME2B{SctGcw)PftMZvN7o*0YfqFCs5#-f{ zz=@0m81wnHkV!Q(5g$$_!sAyW`p1$pRho#zZtfmRB*B6=+~6rm*t9VTshg6RQ%u5` z_eq%jEr~m)-2Bx^!sK1#PFzaH==;g!#V|*tLk`v86u8TC^Jq>A9xP2k_`VcGKVk1o zHwC9WnALAc!7|xYZ10@hAhQpJYJi z1v5h)888jXz{Ys$TG_n)CS+m)^{Xj+GBN6KCi*?eM7U}uV)Qa0uTP)SArr11%o!DN z`>-Sv@im!9@6N=HapYR8%0l`&@+_`pVWdSC6cV%0n3sjmyz%n*_OxapYz*`K`?F!D zo{cs7*|0F>%@@TRFC!c7)V5Cd$-&@hIoNY52k-CZAn$Pw^yyXZcFci}6L*v%a!^r{ zgV((AjJtC%Wi&HLOW0N0!Yw7bF>C=nOZt_|67w)`Wj^dSkw>wGJk1N-P14Lq zNI0|pHDptW7od9u857S6pu_BbZchQa#*v{poea$xg$UFvgf4waao0kO^(aJ5TOn3; zkV7@32;UwT;rYiRT>esof2!O&^ew_`f3nD<$fIB;Y1q+X1Ya%2nA^pW|5S`U{_M8p z6|-km!X4)lyd;lGRi^~idd%tjl%Oi61XjHzNaLHlq7<>@Zl3;9in+`x-MLeSuKQ(p z{+xZcPh}WyR)*r4<*3_Tj-oT==zFmoBd${0;_X(gRgNJc<#<aAYU#PxVtQUJ0=jAuwWbayw$}~bfdY~RW03lvpkFW9#2z}Rpt9lKnYHomYW+RRkl8N5Yh@n%PFji>7 zyx(L<7&PHipJu2E%~(L*r_)3Fl{U>d-O~)=7IzQdwNS5Vfl3K8Q}r#}*K0-Sl2-IP z)`~vYT2XDm-QX?>G16Has+L$D*`$94c(bRHY8A(dfV%#}2GY z>3~mu2b?>&u{@;{yQqcfBzM8Ivt%azU_cK{7WLqzWe>(Q_CRA> zFKSQqV%{hb;nOk^VXhz|m~z8#k+!I?wvU)#Gf+&hIwvMHM2ZPQvzSoFJk@b!2|<~9 zmWm8{4t$pPC57`2yvKYbg(&9Vm!(PzK70EJtq1!EXR7-My>c%XsxeYGGzOAF(QwBXGwd{4eM6bAmqzVUxC zDB&+g^yuK>C2|+8lFR-`9|;-y_^D=q|Hxd>oM?!gDTZhqZiH1Mm;pOtgudsEVE)Vq zg(r<6uf(jos|gm0m||FwDffy^;W)?)XB5p~WnhK~b2I4LneqFTIgak-o#w$CP0|9J z7FgghnWYs`7RVPTH({J5l;tfkPt_8)buDr6uobiwte~!8g>8<^h1GHwr`-zC@2pXh zV~w?gZ7^h~4Zc!iD_&rWBTH?e-^UIjTkVkl-3}jP>@ZQz9;>ObecfgcIp)HYKiYGj z&>k+%_GqlOM@F|jO8ayFW+itTuR8F1nF9(+9bnk$fD-26pU~%2IpK&`yzvsAIbx9v z`J_{wuxX_e#;ZEvjgJ$iq&Q(;rxVupGXKtgq_UwibiABlGS~&rmXUdI!UfA-yI_O5 z3(^hAF->-XN|Or;7rA2cI#-A@18>Y5@YqRLNPcp~$4*yln&yV+CO52Wb7R)k9UkxK zdwz4rBS&{kb9SfhOCEb0Gw$u~614?!C2=OjOCHRaES?q zKl5nO9l=<$kvw&6vd`R^Z>M)^9YNiXY=TMCL$Ty68K&f$=?5@}7RNk$S|~y@xl7Y0 z48CK+aAa{9q_%~@Q#A}hiD7t>P0o6C7%UsZU^O=!e;1KmdY>8g$KkvW!x3j4j_1zd z*y0wBcJ@3M?Da@vQW6nL{Wdw$Ni@-#7JX2jFuum!y*JL75AVlKX(Ma4r z9SM1dNc8oL#8~>LPZXkX`Cb&9m}S?tjY6Jp6t31pA(u>o9Vh9V>XAXjzL#TAdo>2qUt?gV8iSS81~r*eSD!%b?|3ZYUh=+t8;dO7lsc7UtkWwUu$5W$ zb8*nV7KaHhctlibHT*9L{nd(@icOS?8HoXFk1=H|4pP@hBqK zbhkD)wNvBaEz4f#gamY-NkGS&1bk9YV7?^*1?35FYD|DlGd0521nirYh_Oo(QN1IP zw<0}MzU}`dVw*xD7Til@Z#)t0d^3W`J{3tqHTA=Ryfa7f#$0W|o$cZzeC|y`?+|9v z!;+z$MZc6z9Qxpl6y)Tj;9enlXJnpzXi7mu|5RvwPeq4L zD)jAAVaX@Tyju@B1}7Gzp@f>@9%j$i{Yk_3I5JHW(qNgFhTirxOdXt#y=0oEO(R!* z26?8d)A5lS;x2YT)lJiJkQ(CSVcebhF9R(%G9bs?+fd$>)lJO3No7KJY9>l%W+Hca zCg!fEnsOx*^}jNqqs?s^=S;{f&4MTI$6xeCS3b&uihdTq&oiqY!!7OfEUYQZf)-!< zGP2KBXJh+1ve0&Bqtb`m((-J?b!Owr0qTF($thK$9;lVW?ZX`IN9I5?gnV|9Tu8j& zo%lPK8>xACFe?v@tMX7qp1}|MJlqP*!>PhNbdSwPtU2#SmwfKU7GMzyFn$B`Z1)Rr z?NI^d{UWnWrvP)T3Sf1<5GOAdGOtvK{dWtwQC$daYJX$NEA>n-gqKtidZre^i5lOC zy~Vgq4>Xy#VJm&mcjS=;E0^GOG;?a@>}PU^W=fw@XiP4}=aZ$}>MDh?Q7Q7Llp$2$ z=Jnn(6r3Of;UsrBnPETLOZ|>~(ya>RIFUq;le{r?fxPtt+`7J1fdzjn&}vr!y_O32 zP`6u7-A-4%62Fb8+qqX_Ms_7`%2(kaw`&46Rw3#SeNOW##Cues+P4Y{)m6}ttLA=G zHTFKJ!EMzVY^|%omF2bAf2|gNV!Y4p)Zr4fHhI;0oacR(nNW`}3Jv&V)BwfA22?gO z)85knnL~~IJF*c2Z*s@-C-*J08ev`8i1i{(P#15)!XZrN5JxvgDtD91^AEGhaG&HOoBOlZy#69W2+3;X27h1s*j1&P_>!rSxWg2p3pL9efb&^29BxU^16 zI9wzpe6E!ehMTc>XxK-X+AJ#=4U`iGOT8j5NEts3zd-cAABY&P0;Lzfs86cHR9yoB z^R)2AK@V}UdN_MtpF7D0SWbS_;BZ5HOfkX|9bu_gmASWr~fiub!T@a}6f+1I3xl`&2%bjlc;o`>cUT%;R;ph4&cYG~%htR># z`C%T|MP{h_C{J8m;R)*iPoxg?g6R$~`iEZdB~vJTjt^?~`{3kBA0)ow=A5Aq=FMim z>?w2IhQ4qv=jL;hFV-w$=IE>+`NV!0Ez0iLHh+w`L?$60pChQA4bVJUOn z{7gMRIuKPG*$aCe$j_BPM9DH!w3@plu|a5`5e(DY!O$kp=eB(?S^|PGZa96g@zhwR zgdk9ZIqi@T9H|Jw3Nj5I4-18fdMI)dLy@wEd`|`Tx7g=erWOXV-(dhdU3*KyP{dsJ zz|rCS4i*m4RpBsG3CB|L2r`NyP(F{$9e&2G$cw}cNC`Xje;US12@@4(JNq9$T%7*7R(6o z^KM3QG*(nIzs=mx^+o)wTT1TE?HJs$k3nc-3^Ply7*5aY=I>ZUxW?j8SuCcC$FUb2 zhl@AJ-U*Mxc5Xf|m5#@@2l4nAK~6z5`P_VpG4TkKPC%+0ch;UH!1R3rR*} z6RqdyS*c{=o=qm!(x+J*eaPs4>mQV7Gi6<&*4WzIsRj zwhtu>b8P_zBo{zYocqob3bAcUA#d?QOxaWjW9D+?uNPv@<3ji|M}1C>d;#-981l0x zrMnR4`xn7&Z4s{1Z;IYS&BCwfetzT9F^FtL_`?Ug=+7%d;QUQ&Dm2iAki4=!Q z>~^Z;-vL#4dZP-7|Nj{@kqqtG)!072n)y_+0W{gaV(vzUxtlRD)ws;JDzO?{QkkEw z;Ad5BHHJvk;3PGN;Xj$hF{;6c9krNeRLdSv9ga?{!-`3DXeI0NPgWfi%IlE5nEcB1 z^*FM>o?Oy;obBS~THgkooZ5ij^Bb^sWdp9Wn>BVPJ)+AE$kt#UCyW|I8uf=lW^`m5 zF?ekw&KzsR$1{ykc;1Lx?%W(HXheEhBmRtSBAcfP=2!XiZWC0fO(-WcA!ukbJ21@{ zHmezV>zeUyV>5K`H6tsgncci*+?Q`b@un8+ecOT@G6VjZwV>Li1+9V1@U*ues$VNE zk7~smzSu3Tu-n-R9kW*K^=UT<= z4xAC|fZ?bPgs$ws>f0Sy_OS!QsAcrm~%Ix38t#q;mj&~vSP8W3Ec2Q67!u7~5yh-Z9!3=IZOLRkf3Oi&oyHR+z z8)5wX-Dlg4GgaNF{YM@FHIZ4v|G{D6KYWn?hXGsu;V8KSM%n+czUm)(hxVX(T@RXs z9&EYQ17Euy)JXNBdUP)~kL^X;>Rybw(u*MaTWTMAu|kc%*6f9ZUN5FMkXO*zi_g16 zguVO8+B_p7%uo;!M$!M;^H4xdy5JA)M%z9iwVyc zi3`GUapCb3aUu4-xN!TYxNu2}cZ4N(oukAB-&%2DOTD=8aiD~-feg=*#S%ixMhRiY zehJ~;F$rPoH3`A@wuEr(i-ZuZ!tA)Qgpgm(Z5UZeA$%2Co`)rc*GD9Uejbv-vouLT zDPL0X>y{KYGSeh0A|+@KlM?ohl@cz@mlE>UN(mEpa{u{~l(0clO5oU|VB$$mLaLMy zo+c$^G5h3lL0VY(LRv8PlNJ=4qy@`E17{pT$*Lf|JEpQ`ii)7NE%3lC+5@)B7AtFLBd{j^7dcv3*Qf=(>3>l> zLI-Nkbnw7l2Zx!*N|Vt;tD_#4#OYzqNNxoB=;LLvKCEmFv9O)|g+WH>|Jn%WWQ=iS zk})Q(Fvg*Hva*(#;25>135LvR1)5+E@3^SR-2Pi&hNz`xWS*O0&v!Had&@1ra&z1r zY60~z7C5hJLH@P{H_FNKOt4@+)&hEU7SP>ni5uaTPb~B9B{<-JC4|+1$ zwW$Bsqs*Prcf1Q`&ZYLWl>CL2F0i$7!N@unRBa-QbEhlh%UyAx!4(T;yWxT-d7LTC z-j8!f7C8zNgSe?fy=ileJ5DNika^}o9-JpC&XUdb*AwI8Ju$q*6CbNRv8u%r|0ZxN zXSNp(FY>}->PrPtUKp9jedQ){G-r6@{4#GmzU_@QpS_{1!pyymH+E%vV^kMCNM`Mq zN&DdAMIVHH^T8z*9~=zwK}ebpmNRc}agRGW-+kE$^~F>xX35-rVPEKr%ywU9H2m<5 zdebQtKV0RzXYYskVSZ4{=lx6G*V%vE^czL(=?R%%8vg8{`;!;SEggE1|MHk6E2IX+ zd-!r4yN_bzUm^gDl&M4g450SS8#z3HoUZ`9O%K4Xh1|_q&d%e?Km?MXf8z*uk*^0r zp}04cn|{onwLT9+m?ig;>1kGFgh7s) z)H`~be*MD{F)19mlc_(g3&+qy{Iyv)_Joj$DH(wYvSf+%kAUY`axf1vyMHnQs?4T& z7<2oNoUs`Z5xB{G+B@n|HX|ZYv@{Z4hp9z9_jy$SWpGVJ35F_=Z$@FIK@_{bQTWgsg)d{Gk-sDwBHN;2|127HU!rk1hx^HrF-RbjzjbO1RxF8u z)#@0s9bz!`Rt&EFjlmi5SV+jk!emG+^2TuQ?{F;6UyJ45Vl2Y#WAWD^77NqZyUdM+ z^z1m?+YkpmK70C}JE=__s*i(8R~%N$$3uL7Jnl2EmPVaQ`7>{N&v-b}^XyqmZYC1& zWPbvd-y}OhKLMjGxVbYk5yw_1Vl~;BZ!aWb!lgtUx}6BEM~T?^G!aQJ*vHgJ#4q0b zV{#MWo1X~Nszg||C1TP=X4m$sO@@BI6d24Pdy`)1^&=^exx{=w*_&a>Da^z$*VZQ$FXU6PZDuO22&srU z!~LClejkt_muyHHZmvz^-#KZJB%kcZKXx-^(s7>|ewpdiqPC~w?#FaY|C0`bzv<*T zq@yA|9nT8WaeE;1Y|}DOMjzDcL8mrKWD>p3Z*ypwy)_ zlQYr8*IS*5iWymOF=bBQEDKXdXJhV_Y=l#Dnq8F*V{x)GSLWcVkc0ZI>{VXNf%WYi z)IQBYhdTNEY0Rm~=E8*E9csq#@w-Efb1qz5a6-_Sv^;*N z$wS70d~_enNAoLk_SLCLIpo7RkekWD`TV;gAH8MV&goNtqw)oKa;|`SBlBr?1t_a6 zK!s=_l-Cr(ax*)YPuR88Erfy*^ZTY`=i4)zMqe|UeakIrh2-Qjr&h_%rF0Rll1(;q zO%YTt@F%|~tY|Dk)P!P8T~Lf|r^p_or`b2T7&0Zr*tom|(ML+~$fN{LLMd#|mO}Ge zDN3D6x#?SqxU5nv*+*6{DZTD;b%#Sdd<$gFEonnjM7R2}ZEts`f*4vSva zVSFGvlEu`tMC);w9m!|&>yf^w9-lYV3bm{gM;bBe9B+KnMtq&u3`f;w ze(!5WzJ3dwLRw%KNB%->3l{gbAZ9@;4k@<6@OCSicf85ld9%-F)_zGFzMg7>HT^@Q zuk1HkcjDQQF0AX{jd^ps@#zt_bOO3j7uAij%RPv+=)q_8UZnG$9azi78U#^Fc+^bDvXK{6%><2h5yn-g=LMR!rKA7P1D7M15?;V zd?q2(lAks87rjIF4?z#{w}PZ_h#J<`T{fSQ+o41a=m(ypsUS{M9gDtcz+IUu>jf8)H(S1<|6ov?yR zrWLH^$e|*?JmCnp13PSBCSwboS+;oo-4;`akvf@w-b~?oNy?~2@bhV zI9=j|;xW!h&vZsrn=`i2SGy7If=6xSGW4aF=0J{AjVsw7Zsh&CVX?m(g7e+rL+)td zM|T8klPm7$jVZRv9*`zKT>g|NuAK2iydwQFFHgAjWoK}- z7kk-W&{^$;%x_+ZH1)!oIxqa|^+KtVHzIAl@u8S(sQx}!vdss-ZulVI-v|9VeNaBl z7yr%kMaw5&?27e;BE7Ly%l*)AogZ$V^us78ZVMmtXV=yrf$zD|^Vc5^y8ifS<_}YI z^12)R;nC_3r`-W~^o%Tqj{z7J&;6dcfw-`Wx5vdm;C&zz`6m7igt>U zfjDzi5=|kH+{8O#KX-j>Ls91v%8hMq@*QNJ{Z<%q--ltmT^Jrv!<)LDJna|k|0$8b z%`B7{?}gX{5oo*-0nJYl@Y0Sz7Q25(n|V95N8kZ{t93&op-f%RhPvK4M8bV*B${j^ zvBEnNSN!R1t&hT-t=!|=8HLJSQRuivPs=z8O8)%weD3~~ai^~`3LE-H)lT2* z$lo~L_VM^Jo=m1`WS1_DM`lty7U#vozaM$q!xEUUOTgfr3DhPNkgSpbm0#p9MI>O7 zTp~h7Ct}|rb^=e(zxtHO-U>7DQHd}~=MB*(39nWp;m(>QXr4?$&gUd_|KO&NUJ_=K z<4{6=!vf~tH_zfe@NDk(T~EdoYKNB0PW5^wLtqx(B#CT?^kkqr87rr!pm8&`#62mH zxRt^^oD|%$q|fD(g5cm32ql^ zC$GgjV(3k7`rYQHFnfV#3eqsJZ#wo&Bhz6{I=RN_7j~UqdH3R<{lhG8M0rA!hj2oDVo)zo_ zZXuiLcqaMlnRsWI32$bqY%cqK3JOx5&$jIXH3ke>wTBU&(!N=0A&OhN^Ea zM$E}Y^Wt2{@6W|GCGG+J&BYZ{YMaz9jU02ay_@?$|8fyOg?m4X^MI{nKb&+u?+0VXxFcQ~JZ88yu(^wRb{E5xJcg*fn~5OOYs&?5Wk z9`je{SO1UyB!0XIo#&`)x^Vl4d{kR$`f7uUac@{Ln#L4Emj2p$&tfPC(Pv94#>e_% zAi0G9K9?eJW+|>N;FttnSP+O`7CuFO+~@>XfBfbKNz9Q#%xi&?2V3RQUbn%ytM8b?l44BghiS>=JJKD&7vqpTm#tz|ovZibrq3hTPnaNE! zw5tgx_=-)N5bW6mh0G@G&ToQD0eMu-O}N|H#LihW^y$;RZf!fln+aX`Z)z7x4|gFwrVD*jJ-sK%VWqO zWd`1kEQi!9V!{zcG2!kfF+n4YS@>)*VU`4Sy6xgZ2ea^3?~`L?BO%POmlT#dOA66a zQoa>5 z?+vbudt`V9UI%gLYFA^wcoWlFdcr9_`w8M_^n@jFNv z8|HsOt^OA*TKE;U`@W)W+BaCN`i^s{-|?#WJE9K$K;%>v+?xCoqb+`-w1`Zu^Qstk zMHQ*#)X!>E;oGZ<7{y;0Z1f8|kEx+eK@CaAe&hP}-|%|y8~Gd6G2^s4nqH~n(sy;d zo~3~u`~G0TjX!Am@Q2;>KbZJl6Wg*ivBrwJo39q$6=>ma8TB_wZA>4b4Zq3SNZY9m zix=A5j{6Ij+kc_>>@QlD>R|bP9n}BPVb@p(4aGX>Jg5t^7rHne#h;nFm{G5bDSh?O zepwI7|LNhjfgS><@V1<$kLI=1?Jn!%(M#T$zx3f|tdAahZZr-xK*}To$gD8HNp%CL z|1p4rp8?+OHpIR2hM2zIi2vOhu@7d1=6gmkdtii}+D34GXpD`z#`v9P4DniHEK4=P zr79D4%uUdC+7$gSo3ayXiepF3(C3~RuDvmX{0B3<{bh#1gUz{FXinY09DVPZ_ROC7S>Qz^_j_JgV$54h6n(dZrHLhMq^zL3*b386SpjdXAo|G) zr^#SFu4{$h9QwgUR)`m~#%=n+BQ9`5(%u>m#@OJ-20mLG%}QKLt8CHs!4|!ywm9Wui=A1v@NMFwUYO0!!CV77%qX$L!VWv!7qy2Ub8A}1m`!75 zO`LhPML+DZ#FRaRPJ3=T+hgu(2gDq4z(wkV!S)VV<(2qXOxU^hW=RgOy@hJ*2S5*4QI@c zbjFofXV?}yLrKa7!za6t>B8>mHEM}3T=1sc1rF2@9Xnm%w!#%Ht6U)*bcO#B=GZQ| zVzrhlzF4^;G{+Ua^{!Y!p2vs2ZWu7!4KJ6t(bJ*MXz7NVwq&~6x#6=ovu%>@*+GLwAKi~XNvDOdjH>puR@I#WbAI{18Lvn;aVrKbc)?DuM zob$(l8~%9vnw(b^e|Yikk6ajlG0OrlYYlh4FOmy#gDjB80jQ#9Z_8|)t9k%7I|R_@ z<4?~3_=ob(^Z14Z;w^b0@8;9TKNX1KHv=)wAP@?c{f_P45RK5hJMTgh@WGp;&T{t zdSS5E2*Z~k{;Uea@a{0wP=6fCT-@qC;ka;=yF7}_$$e$l!ZjQl0>ZI1EF3KjWR8rC zK+b{)ZevH_2z5xW`w<9x7=f+NBdEZvS6=8VwY|t zo@7R1B)b?MHOvx_H!@{V6nT$PSiC3-)sLd^AG2~^646+-E*hWKMoF@`ry3|?r(U;%rze$`~b z$;F~=QY@?&#=>)bEV>lQg5&oKQRWM7r^MoCO)U1*#bOn8$;&_D@LnyB|DUj*F({s0 zn|M^*;rECa@yIt|RxT1^NnP>7*Pc3oExB~Lu3fN~Tz$4}gR8JMMYf*@2 z+#;BDz6hId7BSaQgo_^B=wVJED4pAyPl|Dh%no^nVpOdxf$y#o?0sAU*WV@Ng)zS- zQ;G@q$m4ic3O{ZN3|A{9UxnKN`V~l`j<-v`5)G>>*|n@h5VryJBzPN7tHR!3;W0Yh4cemJhRg! zV?Ym{UV4ZgsgHLv^znJVKE62^V8uB@%=IwD*TY6Qn_z-zugoC(gZ|zw3q0(xfa_UH z%pmKk%g*xuNIJ*xy7%*w{jnpTkL;?C?9u4syA6P%f~84zuF6w|HU5{|I??=Deh?(C1ta54 zFiyM+hVAqaeny923z-W}w?d$<83MgOA&9jK!8&eF+g3A+HkNu7bJw@|`QjiOhUe46 zpt&dvQx=E8XKxsD!Q@skhxVtIde+QvXwT;7#@TRe{vHl1-EdsuXUUxGaB_emAW*9+ zz8wLHuMyb7ZeK}g1Po&$;M<=}@ClK)JCoaQx{=JlMIx{`5?@Eq!)>2=Lr#2oh01oA$pWo=A=Hgzn;mkGH1 zGXd462^blefb+~{U!IhR>C~`d<|pFgoH zS-7|)=wv3ryo5ie=QVpW^{PY3ko!cPN}FseuVf6%PsSK-RLc}5V^cRjZw8VJJ}d=G z$PB-Bnw`I&{63WZzd~}usaH+emJ08kshDvu6=NQxB7z!KT~I3jcSyy?id0-_VINR} zJ;32<_`Nd?cGRnSC8r^%EDhm9((&v?Iu^c3$LM$DkVdBCQ%*W8>e4ZycLoAwGa%27 zU)k3ToY%~NvS$X2+sGvCBDbnfCVEX|CVO%w4$jO(J74b=-1E7SiG(}cqWhZ(l|fl3 z9g_vqC0Q8Aoc5$ES@=nfY6w{llH`lW>1JVsBXui!WwleYv2uMj?w!m=%(-kN{me!< zb*h<$n z&YiqcG99Xm*#Yd8hx_C6*wyDo-J(3?EXl*P)%4G(OU>}igZ~8XOg|>4^ld)c-shu_ zVm{pd2D8dAZB7}E&1L=@Ww^AF`V@7l zgEz{UVJ%~>s|@O)We}&9VP8!dw0o2@pIVN+e5yh@-hVE~+VACPw&8OwM`{SYw!(5` zRg>{Bt^%Ud6;MB0fjwFk=-{5ujiL(Xa4YG-lY_Og5}!Vjfu&gqXR}IJSX83Rt`a|) z&zs%JpO2|R{K_gkpdVMDM)reW6(pxuvrAMBA*dR?;;XT!yP7(44Mdx2$TDUwj~r7w z+ZrsBt%byvTK*1R3x#L382_pkZxm}`$G0P?76XUZLD*P_Z`O5K>{5rVvh~>1w;n^L zk&AVy9yr*5(Bs^g{={COTLTXMYrw2l?o6L=B-fc6)9)Lx(WVjFWIEI(G@^cK6KWqf zqr;PYtdwRXr#53o4zqR4(A^*p>ux{``ZTn_V*NkNRsDx^w&XYD{X^WgRxEnjimOV@ z)QxS!o2hLm-P{gG-X*>4+aX`qj#(2s(72)lM?Z9+k3IMPygIO9Qzx$e*NL=-fpPGMNo=-6$R`A@o!r^XjyO@LWSec)UVVSa?KIc*wW4FLf*V9)jBB z9>PGfNW0h(ob`-7LFUlZ$-(mJBPCoMEhT*CQg;46GO(CKyI#rtKxt`V_%LaqaF(=i zB3D|FAKp_qw5q4j@}j43@m)_LFR7=nNWPaae{wHjH*;a(4`l_>6ImhicOT)HjGS=L zUrunEH%REAFhsb2=pz(gr;d=^g~T@Sv67l@)wLcf8+e~KPddCg^#awpmkLjEAHyDH=&2!sd@-Hp^pKb zVVy99RIwrc>t%$C3ym@Ak}+D>{>9t;zsT?Yi?72>G2sfglfRkbM}KN9 z%gnHLyag`3w?xk(ON_l~h0)Kfa97C+TX}!EUbaTrD>B^GtfAp$jUXi(oN=+i%oH0O zjp24pzAf|@*zt4J4hIbEu+-TO^?`Oc6mEy$WIH_RL(OA`JuW)hBlD~SPM>!`EP3t~ zVGi66=3n=4gzrXf6rXU!v{m#5AG3?6!EddKo>4sgYZm8gU)yo}b zyva5$bI0}T?!5Wk(afEhS=1wL&LGo$n+N(fk?%g%6Dsr$o&7vkYkXZ=C(-4dWg@5N#k=^tca> zonywI8K^b9!(8lmi+M8-HPIKRminURx-Yap_~N3WFZ9{5JGRIV1)Kd4M&?5KD?g-D zPcU}zLsy6&Ch{J;Q|t%FN+sMq? zO}*f{7_syiCx4-)pe2UBBR7FvxE-^K`oVE-!|9gL^aWR1=a zf$q5w*qrB`#v3lfCj^JsA8=uIsfoFzeg{IiM-vMBuux2H4&^_q+(O(E1`GD(+OKj4 zk=~=KB7H~YFx2vib;%)h%qC5E6-i%!oQ7|aWN1Vpj6S1*1b1Cz_&dL9 z6ty^RBHBk`q)QY;{@hD!jKYgP{2hNEfA?32rq&(}caLZ!FN{I(!5I8G5`$kKV%V#S zL0&X7Nl7s{Jt3C81~M0Z#-i0AmTccx?!v|4$0PDYzr{gOg*`d*ICwh7aX&Z?j;+im z4T^^=y+nbzqYm=m?H$>TV-GGbJsvUh63}lodvCWB5T=v>H!`%E>=SU!m-iQaM1$l6 zSY;&O-IPRl^9BpskqC{miP(1~5z$Wk>+(c&_DzDxf+Tp6zmQargemk1XE1MMHY^!8 zE+yl|vt(SOS2+HAGE~SB{oI;Nu4@X;j9@QrTMC*KQlLw=R*Y&2TC7tr-HBgIQlL_n zg8EwSB=$?ihVk4)oRkXJC1fxhO+|klb`#9Fn;4x6t@KpH&?_wD4VF7H4Vp{R*o{ns zk5oDa^h`(6$aJWUXLn&PxeJZy^c2a`x{?9^2O02vm4WKQ3|uYBK-~CDC{4-4w|kk~ zoymkva3);BGSMBI$!sLGg7H~UlF!1yo!m=2nT3X{^#88&=3*9U$Db@@hGfCHD+~9@ zPYB@KuqT^))Y&jRoQ>{B*>HbBEx|k+9trF+BxjR!wY(ULyzR)=4ED- z-sjKrm9_%N~r4x>wOZC(lLcX2QAYYArCl%OiM1pEG!BA>otej;zPA!U#nT?XGp zWQ@`~{J4fah+Sof7LgtATZY(xGROp$K_aM(nawg>P3Ds+N6m?HtR!b^l3_UpWR)YY zo;TW@3V4yjV5d+4*9#S}Wlut^T7i163N-tYy_H;nk9_k+RO0rGN=##K;_E8jXluB) zsKCC&sY;xDT8SI4_@8-HB8^)%l>wD7>QDC8*eWQ`t|D`x3JZg(5FK8H=7K6@kFSQ* zvuZq7tj1X8r$l+x!0;NBkEubnT@AWDYTzGIgJZpGxzkpQ9C0m-Giu>lSPP#cbvS>n z4ok1sVb8}p%>Pn{AZj^L)NwX=)yKd8WNuaA#Bt^7pl1 zwqpy7NB@KTjDK*t@DHH_T5-_06_E~P$hWnkw{si&uWcB?&cbM&c6Q|2@nKd6y(T_u z_7m22!qAP~gy>Ebt?Gi>mM)yR-GyZjy5Rn(i<^C2)OfovDYgqCgSw%1rW^6C-5ADB zq3LICBG#}|z-*QT@3i&gI2fC88z)Uta9h?xkY{(Whqjd9_(oa?wvrase3cPaN%Rs< z_2Qe-OHdH{3TKu33E$%S2?N(Wf{M}$Ogj7)6Q8`p;)HKd7^8?YgOuTKse&07HSo3H zZ@kd^jng*t*J^&F`<@moUu!}1T^kN$TK$tTK)b#HM(v`8P=G%&KfFDt)V&G2BH!h zXe_g3cheRj_O|F4WJgbs+sz{8%1_(l-6ea}1lpsa%pRht4rGEm&} zGvi)^nGv7mh#iWKFmmL!vyUT`2Rh-w1}ARmIFsG$j7yiDA$QFgKe){|exD2ad%9pw zj0<%0T~OTNg2>*k_&MDbMP05qxyB6vJKeCXn!7h!+#$V>H_%&m%vtC`mMOWR4?Qqh z&x2ZwC)$2^qGu3qo@h_>jiUzx5h8Yp@c5JnQS@JWtBSB$Lxe;-5oY>|@I@>l+fRh1 zbP?L=$IPN8wU;;1rxD($oaIfQ(;Eq2*>U{AO`D(IxKrefOSRsRpWuUpne03+VE%iX z51w50!9Dsf=4L+Fy2_V#3HelHQ(aK=!*_Sr@ zVrDP;E(`o%zQzwe>-?~Ks~>Eh_#wrb%z`LCC{>U{THudQBk83aqE>ZW3=exT^LS!p zO9h~HVgOFP3&1{W>Q{E;h$aM}H}9T7WL2Ge6o>_64D4)U&T4xQ3@!yB?P?JG6oW8L zDG1{-f{@%CglFUl3|bV7hXONJM}l#uM+h7Sh2X@b5Y$fz!F_rs`{|opF%Ln0cnF-Q zkQ2Hr6c28PLYmxwHAURT8ykl0^h9=_rvGuBdX;q;`H$p<(g%6c$?SHYaOy_k__8(} zC%16V=3F?WUW7w}euy)lG5eHha%6^%i9pVd2zV()AXJlgP-X;Fdz~!9rQ;oM&a1wD5&Xk=gx~6tKukZ>lKau)46FwoeGP&b%SUqoFGTw zYcwX9FsnU3hD^>F?kB}yc}NV@Gh<*fDi+4_v6xQ%Y2K??98qGI(m59Gp|N<@Ld{7w z4pTC`g<`a`Kaz+Y*sW<)kl!DpxI5st;&>!V(BXAG;bSnPbPUTiNIRKwh5&I_#Z=#kANK%7J>`i@%8q|t8nOH_P zfSe6`olcp!5SodB9hnFooQ0jtM+NT6LIQ7~bfYXx7c(ank%fuOXV0vsr_#i)6SMJk zI=NE+WkcacHgtIp-FlXdtzOv(<2|%LFB>Iw**L#42fpueaB4C)?Kb5?WoIr%pJmpG zw~uHL?;P?%e~h5Mw4C4H%tJZv9VM+i2xP0rd*vZE|Noxm-F$5PoR8W1`G_?mE7XFz zQ&B!n_AWp%pUTw&@=)l%3@wD^S@P3W3eiu!5Q{_D*UT!!e&(=mbNj}W&thj0?zk1< zX;$UkGs$03w#eQ=Rf!42=u8yJ8zaaveZYdtU$mf-Yn-TpsHDc9gY=9iR90tDzKAU zQU`C9-AkE;+FJ=%-YTC?Rzf&e2}^hGL`zrUSVk3dDAic;iClG~YIM6)!-LGI_sV2_ z8rI;saSbX4)FOOjEe4FK#RK*^txj_1Mx_=2suhpK zt@sn!3frz$Y#Y#qx}9zKtlWmK$Tr-TX~*0-?a*J@j)~jb(O}e$d1E?w?~)O^wF6VRU;A|8-qJ2q?C65?*Dh>L?!wosE<6~~ zjYWaoco)@;ODWyBFhW8Qrbr0G4@(HYeo6?c<`RNpqy$;%5|s8NgywGke5a&vk{PRU z5t2gB7)jwfHLI{uJp>gp+pF4p2!Dr42`5KN2{Fs0ghPj<;c6&*lXB96{a`Y# zwnz(hs?4aHNDF8Eq=kdqWCY!tGQ!Yz+{tm55yW{if@x|`;bTotVXBzS(LKF|^W1yh zHCk3EwUHH8Imrs93z)&O>nC)!_Y(>d`wP#u$O&iF2MSk`1`6S04eqr(U%o*1J!))<-02u(O=f-C1u_~(@=Y=4;{^^`eA*|F>AZH`xY=2-Z` z0;}uU_Zx1Br3#i9NuTgRh$TGatZ+cELfS4Xc>S@$DC$!k2G-a%!v@=8ZOFy4McpY| zr2b-VhdViqyurMExUoFR4wqKgVb&RHRUho|KHCmG=i6hCo;{MS?6Eu49ydGeVffVn z8=13lJLZgE|B(gNNbgUF{X0_^bf>tWD#HcKkGf*<9aku8xgs!#nVE1`xW95kk&PR+ zy1QXyq8n~^xncHm?il}Y$EYB8*hRX-p}-woBRtT2h`c9n@}5dPFo>D;{Lh{kqUuQ= zId=o2JaKmdZ>eb_NIYQoE{!Zuz8Tpf+-zny?>akp65OvD%sc9YH&(y)MgTJ{s&3v` zm*$PF8Q%C)>Wzz)-neD%1ASX&T*7!;P4(qQGRMr}qg8Y+X~+8vD1y+aU6ZE4ib5bgkk zV2^VMwxx#PQ8sfb73{?=45cm@3XvumO}e4X(1fDKHWZf~Lt*0;iYFDJczP@hpP6aj zWfq2uHeslV4TCiE?N9H8W0@;`!jN!8F;6n-2AN9_BT)4z0`6uJcpSppiZ|7koCtK6 zl2Oqdf#!*k>@Y?mVg)&$%&%uWBLDMeB&Pg|geGsSitY6Nc15A<3Aq$sq9Fbrh3E7I z{ZpdId5*%)8sTM5Ie3!Mj%y9?3GR&fc4N zHF=-RtS=$^;y-$TIxb1jDold*cxq2`==-fqhPpyB+D|6K;ZZU-Qj&4cHyKG}Gc8)3 zg25ZOJIEJ&E(QIrr;taNf`cY0xL}onrC}*}T%JO12DfbXq~i0TRQ$b|3Lk@1*xROJ zst9~H9T&DZntf3ZVW|V=MB4$+*$azoYmaqgF?u(dLQOiWspG@2rlj~lai5lir zR5oQ{E$^<&yR%^RISXMXS=bPeh1&&Lkf_Z90dKg&o9pDFY#d+5uj{kvr)6XM?rfAt zWuqb|8{ez4kuWs}nR9bsxjY9cmvXTDX$~q&xN}pUgKo)Oh)3r_b9F8vH!_<}rpCJS z+#041mG&bSU8>x?vB|}r$Xt9)$c1lKF0`AOUGJi9H8T$yi=8zmfU$BOeKT*}C~SZKv0fp%mSooW_U2p|HZI}?dTZaf%bc45SVZkXThhOKHhR#|pqLQ*%3yScNxO+whY zPeL%@J4t>;#Wx9IE%$KjJehA#k`O+oaksFyq%ftAq!3Jo#lV@8LhU?Bp^2RLyrq)D z6|&yvDoG0Keo6|vizJ0tJ$nej2f3j$M@lF#mJ(9uNDF%ZNehbZ(t>NFwBR>VM#x$& zBb@mwBXo@KEj;_sTX69v6GB2(I5taGh}tVF^f@LgjHWj@WPTr^NB_RUJi~s1l~F&T zaIc&YbZvmJ_NjnzrN_wOz6s^tkMVl;GqQo-LTBkals|nBEz|cfIPwW8`eZgpf5zWA zU+_x$JJRkbL4Vax1a16@@&iAi(UY0~y;&`UpVP+MJKAv8B;!3x2R9pa&<$NQ9?-=>7hNpQ<@X(W zP&=iM0v&za9d3YmQw&gj(*S#J8Nlb80p{Bn;Af!$dTuu4!n78v9GNE6hLA^ZKl z32r?w!2$~tXoZmnpKAg=X22&Hn?l*b6uN1qSUBGdmT$~(wv;Ffer>YE#>rM#bkGXFoUJgg(h7>Q)<_)2J8+CO zq;#!OQDhC(IX39K#|B>KZ1M6Iv)rGk%Z1wFG&Q+m`WJk809l8?jpp-)2k~g7o zA=&Xm?UBC39>b5=<1P8|8_exdVrh@Pyc3%}?C~kf9`DCFVBth^AozyOaA2p-0ikOh zU>WTI$2~0VJZFq0`{9(iGh#xV@j}uCt^Jta9^isQGhC2;zy*GnT=3+J3v%^ckS^_t$K%`$|bfFLCF7|=NR_cIneUL+5#0*Oxe2OHaw8#hdOMP&Dk}oRBM(cCY z7YcWLG2pc?#@hSBm)YzBveI@h@k8xC?lj)ZYMHd6`cmr?UU4iV92Vy?E zn>WcU4R;SjN>Ct5B!akQz?+x3xXWZjOfU+ZhceXQw5m-b|wh#T*IU($lk%LeDF?0vHrqm$o z?}WgE8<3}|L&nC0;8JA>w4_3DUoI4?@}c;?A{2=aLNS21b5Ffc$m^4tZyAazQ7D=N zLh&Y+T#1xW45MbblX*KQ>Xs(^!f@w67}gyL!E^NF*K~ibTi@=JS3;Vm)+o-;g&qZ|y>HH2Dkv%WcOXd1wrb=Eh)@ zQ4BhR$Um))L2g3~q6WvJ5C31;`;lCUud&=qh{eio=I^L&#>|g{{cUErpTuDxb9H_W zahTypkC*Sr)Oak~8xOr3WXRoT&W`?WAbAo^9n?C>k@Mu{_qO#3klviY&RGH`?cuIB zZ|<(M^mT0#_}@JN+o^4)QrAouC!!~JB3n|Jv1_5`*_wy}Lz2+4BnkJnC1JN|6845A z@y{ggM^e{Ro12X1)HV$sBtzjV_r4RErK7GnV(0^m7%t?@ivRU3Cgp zF-!OA5i{IssZjJwMQa1O68-7du1-VN(KMVVN8jo#canakLDnD*9-(RINe{QXZ#tC7 zk%(GFFLy^enm(nY_t$jxGSYF_lB|4(bd*GsHxZkTBkA0VEaH|@aXOAu>vU;j4|E`J z>Oo|jF3P}%O&Qp_H3Q;Z8F>3E1CcuPf|qj(Nih@qA~Nx!CiDL@>(qH)8(pT3NljB^ zodv0+EX=RTf_MS>5}UIzkQ(O4@66I^XJd_1Hga#};LOV$G*{-}dQA?__a{?gQ7%67 zX{@1_yE&IWcrKiHH;1U?qOO9vCcB#lTG`>`R^-)3d6+zhS-7?N*nTV@8{G5pBbpqE zynMVW&BuHl;Z3`_ z7;=L9GKJy%U_KM?@0#4izX;$b8}Iz8Lv0B;9zd*&*j+qIKIyj(Bv6l2%;~_(6PQX3DJ>Y(9_F7}w{ zvLmPqxhMJv*3yUEZ3B457$828`!^eE%v%UjP_&A_5(E<5s4tUe#fbXx^4Ks8^1bbkK zqn+_Y)fqBY&Ul^WjFo*{s7JZroIJOfXS(7hwW-!gZg_ai4Tpcb;gJlpPmkTvKZ+Vu zt~<_*@xZ3H9(bSVf%*auJlo_6^$VUbx$KEUQ^+Z$&)0fg1oWJnFyB- zg~RJ*IO5*WuTu(#yof)ersNmH?{m2MM{Q~I5^f{kBLCnQH)Ql8&?AZ5Q`ty}m`%$1 z5{c8gk<@7-5$73+dCZsZGLC{_coZrNqi~A;+#m9irjLn6(fVjS|HFM4YDM$ONKz>z z|A0Ai^}%EtjEzCl@fdQFV=z`N29nEoNgR%a-qBdtTw(_N3HwK1Vo}7;FeRs0=pdVHhKu=Gc9y** zeufPmkby}S8IbqO!1M^_!$)Oei)kiKn~_yukqN`}OpKDs!nbuly zqAWZq$ihuY>OFn4@o9B7(*3hxN4;lzeKuB0=kPYjf%cpnXe=N{^(>j)cXH79BL`|_ z}n^1+7i>lzdm|xSYs6AK1 zbzL>s7)Ru_YG~i8#>4y7u+pkVzEw3|)>K1rIk%11)u3oAcU%!^^S{#h1g<)zf><8DeJ4THsrVcCnkrOUgkKD=i zP@B#?_kwzSILhyB$tGx}_H%;F@N*5&{@8%6O58nmY{1&s1~lX~V0v)_mew~wkm2^r zxJE?GYs8lwjTrHu5w(hqSXkGHk&~KWypH=XVw*uHwwrY# zTcQhtW_D3i?ZTA6F05JKjf+>hQG2%=@{hZj=jnzawWJ5VC4}>nBm|$8)Q-qS^>`^E z$XZJXPT}OBXG;k8b0ma7YDr^eNeVe+C5+xfR>DI`VLNrB*LCEfa+gM(US2ynsHZRX z5R|X=5Qe^B{=B${Ahn*Hgcnl6lpj*UVk0Txfti%B)Rycik(7{5=IN-B+!93_D-+atGB&#k9$sb?=i1MTbMmhR)|hk*v z;^2P5_9Oj;!yDybT*;{mUbT>_?Aq z@7R0t06#!&=0|AeF@Ncym z_w{i3s~$E_)u#@qk5B^xh%F7VImr+vEr$3hV~o1t#^@YrjJFevv2>O(Ca4+nzBFbh z$CR5prsym;#oKOEC=N2igEeM2dBhBd^~_=Zke$R&7FZX>J1&l2J1wBkm(O;>_jk`91WND>YL8Zuww>@>I1J?NSpAF{w+h9PB4YR|xXn0`Dt~bAW+F~4Uyn;D) z_`2K#BtBsr4}kVuATvoo$O zcfr(0E?C2NkZ(+m3q-sXZDzQl|2|j99(KjvE6mV)yW*6uD@va*D;Dd9zPuZAR=ZPQ zcc&NNj+XV@F5FM%)+G;Y-ROyb)P0`+5>Z1VE7VGaS5EBcM~U#0cVkO$FT6PJg&ybF z!@uVRW%luRDtmEH#0%{OUJ!bEW9CS2?Bq+|>WwG&*;&-}#zI?fY;<6c(a)QC8*;d0 z$i(kO-DebiOnDzX*yMxtM|@Dl+cF~Ahc^nd_4z(nG=%-dz3l4qUD)RfH5F#XB7G5^ z&hEaHA38?+;np@kNWJsJ7)3v9`Rj)!GDAID{cvI#d-~h_v70^pig*6_UGI-cW5u{I zUW^TsxmUA7j4$iyZ!$Ztt;FtQf*6-_#5gl90M{0Ck9to4)Tr@PJY#lVHvkFA+@LNF zz~DoHczTLVE}KBS?hItlAqexf2O&W_h}#E2xam#LGcpK6=y9q`1>+EP9@+81P_+w2 znK&2+qUm{N1>^VB5WJon!cJBQV))wlhHeT$9Q%;(?{ky-2U(ydAvl%7otj$a%Nj$F zHk%sF{7~evi|@W96hoGB+mNsMUMP;d3B?Y2qnXrke*WX%H$DvGdB?h)3`5J;Fih79 zLw|Pg3&{qxwrA$vi+hOlM$O0twb>VrJ&(fi=|eaf_+)&gajRx<1hx&QAIjX>s{;|Z zd?W(?|M8YR9RbxdMK*kvRI28MLqD`D;ePT!$Tf zZrODDL?TGcu4Ov2_~nu49zox9N)*zUMe+Y5cJ#k-)5bLl_3lx4;u!_)T=p#`qVZ*L zG$N?sl$b~3ihVR0^6XRQM`NTcb(_!Ba*WCSH;=(hX4MK4V-U$~{sGxo+@Ovlxi*%Y z8O*1>kHy>{v1qV}g+oIuKJf1SHY5%fm*a5bH}xFqI9l{fV}rPHlN5(j%&Mtfj>q}C z%&2L_W4L!b4)FHPYGfZ@hJE|B+_a{a6Gjc^(A@-#dy;?)Qi<3&A`$25kp`S2C-inA zWXSm1`z{ei*1UiDKKn4cA4}F(W+Gx|C*iR|5-Q*G7XFk3m(TQ1HImRHfjf%H?B(}P zChsX3fn$>KhrP)%dZ(bqle9A#nb)|Vc%Pb&VKS_YxR=O#S>Bbqin09rlai4`4|RzP zbN$}DlZ#Vu2&w3EE*0vJ$qRkT{Qmn?{;tWKz6F_I*1U}~Q?b4-74szMm-b7;JNY!6 zoWxCRg*5K5q+#jlH0*i6yc@k#ZL2g~W2WusjdXmtla7cd^ix&H7PaB#jdMD(;<%xm zmJVBH+6)(GVAt6U47#3y_?sEfeZsuINd|JmGSC*CfoCb)$j;&S%(o5ckqPtundn@Q ziF4O7QF%8L%IcXIrjv%09vOt(SiDL;DLG#bqCAZ9 z&qKtte5gOj$0%m|#q>{&#oRN9%*QW!rV8=}IQO^!USto{=oMgPL;=Ke3ZeGC5K&6p z!nWW(wnHHT50+K%v93ZzA~W`#Rd_mpyo0ONn18bx=bO28J-i0}C)VKf+#2ZZ zslnNUHK@5=gRdSnnAAqj*T!1>q(2&~Sqn+0TKq1qg+^H|>e_2D`AQuIysN_pi3S`P zOg`5z-o(_X{&yeuwP?T{YE$ZZjgZrC#E&6Os2kk`-wn*ocQ@h8+-8hBZa21QcjLzf za!JV{-F8P(n8O?Knp7C=_}Ms zyo;vbC+q~h!iv)OcsSw<`dNHKcLw*JFDv1;suKD;Dj|p2tT7jq(O;wtx5+>8_x4Y$ zyQG5Q{wkOvtI9kknN-f|=oz4n+S3}a)X^ZXP!r1vH1X#svs+HT$O`_AF>`+7%9Y

    >O=!8I9O>|db=1-`A1_3>PdokIW=B z(F?q8h?7NznA%{7M;^w^%o?LTfeiSfzp%Y%!W@|?o;)=}oS!*V+RX8>A8)L|7HIfv zf$^0V*uTk=_lYHUpe*r`8d}~=D|jrk!kPP4c=6l{N)A@|TyKTwf!0ta$LZTnYfO4# zja&b0U^|pr8sDj{w$u`AF@Qeb>U3K?nrerp^>)~`(+(F!{Jx9(f~xi~O|pk#sXhG1 zJ7DB=2OK--09!Q&*yuQ5ZiWL6^moLmeU8}q#1YGU9I?2`5e@5|a9xXRh-@bWw>iO+ zS^VjTo#A}J8S+n@(Mnc?-dAVn#5*H`o}Kau-dsOiVB+tBxwS6bmvF_?zOLAM(-kL7 zT_Nd1oi3C6ft{|fpWuez6>jXRldF{AhPFPuyJos$z=Z>Het%@ zorZcKA;|-++1v@->xnb>Jjs9b#3DUUn1^~|M;JM%iJmAKE`r8H5lW|vpgfNq#?>Mi z{}kaMGyD||B77L;g{n2Y(ZpVGZS=wt=JxxJ^2Y16-q`%!8y&yAF;JgUQ+r=AQ?_IG!8~ z$${h}4X4K@Plh|axC02mkE^`n?u9_|8+QerLSW)eFD^L*y|Y4KP#A)~_3RI9B+ue% zC^yzaQAQuGiy5}rWV-L<-49J9RT(Bk@NdFCdD6Cv$6a;$ko;gM5nu?hBU1V9{ic+u6Hx5ag$zHt>kE!qC;r}}xPs7M;kBVmwAOVZUk=?#O0qs}FZuet%ppCtPeu;2l zb}ju`BF=r}E%r4LihNU*6H%s>$X%jDoJdN9e?}s1WhFw1TTV4|lW<}`c^1dGD`=jC z@3KU{WbO!X>**hNu7{@J z{LvJ4r^sS;Cd(oug`bZpm@*)hT)b46?oLH2vuu9XnPUq}g&cd7u|=s^%NuOx+BA4a zrQu&j8uMmp*wBM_)>`r^*eNKymd^ZJI%fYP)8aw~PV&C`X_x`O@(idp^3H0>zzg;$ z=PP8Q|LaUtsbu1ddnWfpGx5DJlYMLMK5e9K>5+wu0PamUWZ~ph`e*NX8_`dbYGYPy z0@>}mbC_Ms!QzKGm}HiNCifiN-^HBSkzAzz&V>oJ$PjWTCR8z>CYOg5X(tPp(^SLdQk4ISrFk4ax-Q9)kSQX-kYay0r7UD()_nGPn;Ve^xzXOZ# zQ@#kN*A`&`HOC%)Mc8agll9U2wukdVOJX9KyW3hWg&*TZX20|swxfU|D{HePAO zZq-J7RByyHn?^WiH{zXS6Xq^&LeME@_Qy42z=vksDQQLmvtI>!TOe_XoCm{y7;W?q z&l{Nil4`}0!Q4t3+ltFaTM>G#l^Nhxv^Taw;$$0{n(g>By92FD$RU;L#Gh52s6`ic z9q2->d^fU|b;IOEH@=5=!?zbby}lBH)EEij$7Bh?`~vx;5fXwr-$+T`PqmW5rVBj; z)hlE=a8pRjxQEanDxCi4u|@E5l6VCqf%kAbUy`oo`i{H&;d5A2qxe zsbSO>b-cKx4z(xhSnjQkLHjgtmf3mD`I>mKSraaoG}$}X#1ef?=#^;V-HzWVKJW)~ z#eWdIpBb$TZTJg1__a%ifA;8LtQEDeHM&@Nl{%QRE~FB4vF@rKy2t5;GQgG-28bPCNPo%*!)6-6U(*Qt{frQc2G`mm>$c?ka23TVFR!gK@vqXlrC2s3j!r#LZmws5`oiclr8dmfn ztdXW*jV4!XbO%{OFVY&rv#jxPnhkcYpsu!qol0991bErNxY`DL>TRGkz?T18+G3)< zEqtwP(Z|k~Thq4i=gl{Fh#ews*x|U29eQN(=9^`Y$BXSDx5^$vPuL^np*;p^*yEv= zJua2oV;;4)>vha|4RJv42@d$RfH&Y)2TXtC0DV$D`oiIk#39A~M5H-XZuNFF^VWl&+9Cb$eRcACm zc82~_XWa2}Mroikj+1RMvdNjB=`LguyWqhB7c4#Gf(v}&qb?|Y?!tbb3%dAXQeCjM zke+V|I|&soI4t9ef{Ct3r=Dl5<%&l(u4s1SOW;OOD>Grk-4M?EGIf(1-rseD(nmMA zTDZZ~(ha3P{F%QSmRGq!Zm&Dyf4XDGU+x99F&j3OJpH#Gm_=Uau~-k>=dD@O?E&#Z zPq^~llzYjJ=m)a)Ej{5C?}@~GPxPww#HmKIGlq$n<7R(hi3objM3}o$#7u(-QfEcb zd?kXTiHKV#B7CG~80{{?(hw2!sT~ff7h$rb7rxH%LjS#9_;Aq+*PnWkn@cTmHan#& zy^-;sH?p33quIb4`Elg+XL^$r$=#tYZ%7ScPHemn_U-n8+XWwF{3g@W$p=f_d@x@` z{ze43p4mPqn(YgNMZPGaA3Z_Q7m;4R_}1==EuFsbGL?v~WKpmGNdB?~hGdc%Lc-jTiCNF@y!cF{Ek3$9OgFm=Rh1)4TPq3Ao}_TqJ>?H ze)PaUcLyS@Ul0x~3qs?jAS^l*1mhz?xP2@LUvC6K??n(&-v{C6#~`#S2O%Xl2v!x` z1g#8$`~UK1yT}8jPcFV0j07d-&vJti!8c+hdl$z;P* zZ=~*+y+0Hu?}yS04@Ci=!aFiSfPCb`6mK z={1QNjd?M!JHk%Ub1|^K9)kjAWAmBUSdbQjSM@Qt)gFT+#aQ$XWna;}SS(x;i}%Z8 zF>-S(D$m72`2lC*x3Sp9J>(>pSme3KqR=xI8@?xWsk1P3t>@Eh}eGY2ijYN{$Gl4U4vYHvNgmH71R65 zEJH&vW;Pb%$|q{!ze7cK)#knd-Pe-@e}?xi`l zT9{*#sRcdm7I>M*j++8YOdDav9kLb1_RLm)rsj%? zYw4w?CU$s=J7%qPN4vK>=Ek{mU+957yF9Qu!UN;#JfPXvlNm5iymR(MUYsXnmw93C zRxb>^=>>=9WGB4yLJpsC=aRj6{~^vpz>cNTAd>Ch`a+Y>P&myL?QPVKkp5q zaNa2j8MV}!nxioGOf&{wiH0UKc}6PC=DmxCn_D!rlcM367L94-J*|=^|pcE;npeLU{SvOk&{(yGbyNv%sj?x6(ie3F2zpXsG)Bm1c<0a_Ol5%Vk& z(khAY*CDsrJP{>{iI_Jn3Cm_D;Vm_!fUWdUQA0}Pjv`V%86GS7nf;dCFx*dMXePtf zEE%tCl93mbjHKjbWLGC6bx;aa>7}ZlNxl;`q|yNTs6zOep2A&4PYUi%qgJ#;gznWM z)Sjc4>XHZ#AMrE(rwIMsxu1xlPdG<}dhR8@{z`?PDY=?nsSqnn#USxCeCeNt-^0?# z7vbmiwKN>oOM|6f8tzJ_qe(g){_KG%r+@17(R5VYX3v}ldte&r8KyUQfLsPOdU|{p zWMKB{46Iv|fdh6K(4!x?tB7Y3`=~c$;=>O5e-E(pML&}>XeM4G3lZkLif7YXbu|a!I^;W5kz>%3 z15N5WPxj^_>`*Q)h3Dc*G(Y>(bFm;J7qwZr)Tg<#n45|ebC?W^D2wyyku*i#x4Zf@22E%2iSSQ7nZSbsy`| zrMS9;&k%Aox2u+7Aw9)sic6tXS&Dvr%h<=xUb=~8$l)HO>;Lv3^jQTxFT)7SGNgtv zAJRdS<-ekA+-E!P`S`HJpauoWNBZhjBNn<&Fc9v7m zufXj66_|9Wg1I-o8dV_NocfV>1ysW-kjK4Aym%!vBq})@R#F41g!tM@e$G~+a(5-P zUsU4hOMd-RC7RSLvBs&6kV={`>$%`|E@-+ z9d|6n)fiY(jor-R4O3)?3^k|i18QJ1xduP>(GPu|`8;OxK11N&vz zEA!5k{^+V&T$QTB%gJ?Uol=Km+I28XU|x?Lg&AAfCv%}5@1E7e=6yZ(Yu9rZT@T-U z_G7bWX2OI9q#SI3`ct2`J z=K=0|X7x|8>+X*d{L6F|%iEPVVPA_D%1Q5ETBE5Zd36 z@2MsstPhnCKBr0u3&bS_BSlF;wv;{7Vp75tc1gFNkP=qiloDzmNC{alq=XU8Qo;xJ zM{igpEo7~c7Owt~7Ve6qg*VjXjA(4t<331G0i1{Z*Sk z%L*MDvcmB{%mAj!3eFX>!pAyUL3^p3u>K!8VR5jW&{rfUj4PEB4oJ%j7%wmUJtZ%E z;Vx?O4Fw@Ysjo0~U|(VHmA=AD)xJWijG}P=RX?GhT0i0R7$xCxsgiKET}gOS++P@= zGEnH$7$_KN4;DJsd_z{Uq@nHfmsilLj7T{-(cO3vNEzi0Y?< zv$J$@GEo=jqV&+2tB2~*21tEofL$L9VR_34e%FohUBU#WBTdL6F@c@C2@c;RAGF>K zGK0+#U~7(5qbxA@tOc%@Sz!GlOKg;(|7pGzX6`48q1Fm^6RojkjWq&{tg)@!8oKjs zV9r@ge6%e)PHpjfvn~4Hr4NdpsMbinPqjrw8sB%>VsMfj>@)3rc+G*K@}A2xk~CcfrWD z%*lOnf%9kXNyu)njOF`zu83XYidROiIPdO?-y_^mPLKSXd2SHf>jvjnJf&`U>f(;j zPIt(6yW_=p`kYif;1}$H7o6)Btn`G`dQZGK=n2sqPo#bH#L+a)eB5~`j3TR@oUA)v zy>KMR3%ZKl*uWjeL(X^AZrV5T}gzULpvYhCy)84TAGw&UTi;Nap>2?~M@bco6~>rx4gP zZzfIl`fw#?%BWw>KSBn3ODJ9_h9Q*P&dMiYi2NRg_4F7WXb6MRkZ^?YOf(3`#?kZ) zEs4PBf5}{b%R9V(1oKak7|;~SKhr4e*vC6L@8T-=*%xddg^tuHL=KCF!lr1Hki!{$ zD;kH3+54wRZ_hNc(gm`p4#gn#MGVT>yOEV1gWe2s)7xT@CeBV^rC3~95ev(IdDp%h zi@|E-r3b{)Kg=$UEPCO0)7x`64zBm&5cww#yUC?`)J-4!)_ANx7*7p~{ByT>%#Nbx zou{ZG9#8irKscX(cPjMy(BJOqOi#N%xm5CrIK;cP-eG30J|tmxKoTZ1e`Q~vgcW^~ zQN1l0r;c$RyO)d+50bH!of(VWsS_0^BYJKM;@6~*GtWGhSqf&c+?v#Ck2#@L0c~?r^sESOg(c~X2OhwqT zR3z?9MbYk5^rK!B@DKA_-`VF!zx#NdR46y6lHZhu)l z_)$x$+Mfo=z%=MnM+(egw_iS4>9u5}k7uT9emaymyWKySjvD%UidECGu|6I8Vi_1d zgnhshGZ3+ubJ~RrY`>gA4r~U7$YkQ(j7+R1t5g0|CjNOyFT80cd>S(`YjqYm3|YwX z&%%vP9?5J(D`eyBA$IiL&4w+{t%uog>BAULZ4~Hn7XzYs+;NeImoV&bL1nuE5<_VUQO=Il2Q8#r0-{>atU6ZAV1x( z1j}vd`w1?=u*wqrbE6b#PfHQ@fqc#%)VuzcV!vf6#@d$h?^-EV)s?cxt`twFl%da= zGHBjl=ifK_>sOGQzO|gp^>U26#yRc_cMhEIioN+hx*R3+0)5l2z+STo>}USWFth?~ z+(U>BtAzQfN{l&6Uit-g|8ZX1M$Xmd6l!7Jm6*1z3dTRH;GX!uzG4e%U?=Gf`pH>M zIjw!Q5>u_dHJ-l{MKf70t43m1K#n$6* zd_5kIX~3uj4NxFYVOC-TN-`SQtxJ}|-A3kr*!kDZS&bf|stZjx|EURfZB2N%v>ESK zG&76b3`y5!yohba_PS=s(?7pu3cHU_wm|Jd3(Po=N!@C}-`g#ydf$RNzZNWPYJu&7 zR(#*witE(c`g@Um6+_)kvJFlB+n{>C4WClmuyTGo{vK^d*xhzA!`d;en2hy?c6Q9R z!(P4vSkQs{9UajB*a`jCPV%9<;C;6Xi{Er%Omr7Js=J}qzXz92_fR+Lf#_BbcU?Ug z{0aah;5??!Jt0^tp_%pL5!1Z5d&3rHl|bwvRCJOdny7Q6Hi8rmW!e zQ&y;3BrjYxlNTOKCph#P9DMrYaF?4jq4w+5&6>^3#QrNn&khw?Z$1iMbcUFvA)^jqpmG< z`q&|7fF0Ub*}>|89rUT68SS)(%T;^0huCAIm;**pAA5hu0r$_6Qx5rkT&;# zk3BneYCSNc)dS}?Qjgm239|>DSR3z&mJ-fvQeJ2rOrQEDFXWkeA?=DczQ6NEran7$ z(!F6RkWu>Dho7}%mb%l2?oa&F_CWkgzmjNV5LVx(?nAAoBa#gE(bR6n2IJ3}U|3uW#yXu~cohfZLUk}|l|s;W zH}x5MkOqDZ!JFS9sA>+u78%ZN3Zd}eELX886pOBgVq$A3LYZqEyN}t%BVmv^K_0ti z7_KCQ0TSe6EeePCCHjm0`(N(=G3UBre&Nsv42NWSI4rn}bN(H{F5?IckB-2|{0K-E zGW$O;5`kMHQFoB@-pNSlGVi#PS;roaNL&k#L_LpO67!EmWS5rk*EUDuQhOu@c0^*Z zToh`CP`5FOLSPs@<5}$9;ofdbPZWYAquC`Fjh-3N7(JW)Z1ju2JR6NP9`(P`808!d zGwL-j2gP9VbY}di*J!Va!H9h^h~f@U{eBF_n#bUAP7Ex$v+MsO79k$7P~)6vnnyHPJGIIiQZnw3OGeN9WH=)kp<9x1i!)`qdNT42$SrM7#%ft+_vKTtcOAQN zcBDZ0Pzu@7DX{pRf-ANuxMs)wUIur2oGZJS@t2_IXx|zU5|u@`bVP(RH${+SZgQ}P z2)AQ6cXF2Ws}VtUMk>y3PDRe~RDAl83MXnaFBXxBwT{`x&1o=GPs4NlG~_cA>E}W{ zri9FP`E)#_?|aFlbo#!y)7wXe>4|jI-b%-*7wO3Tkd70a6<64?LnkpEu4(D$;#ts? zj**HPup3667H3BzYC1pWWx#(^2C6q_VD!!msQiYIgvxGcDH zS2$mZ-la*|FkF`ny=&Px^dK7-Y}mD9pN)`MdYf|T5iiWf;So7#UY>)Hw>j8pz*is6 zpmjMA@6N%Q;kjs?my5OB752ZI3+oT`gKOkMGS zs??6O*};FO|QhG9PnV z3z+vPgv!Q3sFLH}c%l%i$UmKTrx4e)n8(y6+ugMgmplvk`&J$P4+_K)m)rtYM}UEjty?*BT< z(bJzZ;M@uf*ieD_54hL+uL5(q>wD%?0hiJW$n~j&u$B z=qBT|C7!JIf+}3Et->I&YII+##&nHpSoZL}0(W%xYET$f!`%UUbFyk+URQ$^LM>i% zXSd^EEe<@Y#qX9{_;Ln3GN%sOI-CW4*^`q}huk;yIBr(YKB{_5&8UaxxCU5VY(P+W z16KBHgkMY}Dm|Oo)AKZ-Y2>v^&SzQRLPR)%hLxP|^X_ah+sD zlDQs7HWueKcik??ly_mkyl(8<$^F^xZUmm~#@_4Qm~P$;QcXd#16nwvJ=N-E-h|xI z!D7NAcKjub6c>)X5*KcM6&DT`iVOdgbN-quA^2aD5c>X<5NxW*8s*$&vPn|7!wh1& zhoo?p`NL5SoWEKm1@}%#p{Bo-P--kC40hoxCMPXCG#E zo`3ccB%S&QcBy@Ynamu1Cl4!eiL5Y6S5_F-Dl14%krS%V%L!|{<%JM1XE?gAFv3Pr zxc2BQdk&w&QeFiop1q@2_ak-sPl!6ndCTuJ8e+de*ZMm~&G?0l2EVY^O9RXAlHHJ} zg}d*xaXwia&0lpef21yIUh83GhaP|K{s0Pputfb20_la4;>;DwY@m9S0Rp59apt`t zN-i3)cgF~)GK^qlPOd}OUz|Q}j0>8^@Q^pbpv@+jchCe!E}KA|{-^$ZO!4@!DXiw2 zLE#5k4~xuU>TZEre+%4PZwa@TWRf~rVqdZ)Tx%`yLCgvUL#(itY^o2B=u=O%!jaY7 zhpAe_Owk5*7j00kX@ei#HaH{BTp(w#IeTm|>^wb8Z*9@?!xjTGZILg}4#T;2@H=3K z3eI3BL+x;hyRi=k?4gxqkF_s2cfEALg#ZUkAtPdu$N@uJ*dx5s5t|N@F?Gig7fY#a zwmIVSJ|}b?WWV9RPB_4sEajUMu0?Sk>n2NHAVYqSGwz;qhTeT=c2AHWKgSix3th2G zaK*88uE^T&itTq?VQbBv`}BzN4p?T$-t-4XQJ9g$}4 z2)A=bK!`j3jbd(ah6j2Ud5|kX-})^N)Yy68RvcfCdh+h=iA*IgNU!$7*7aVnKF%5K zt`}@xdtt#xFMQV}|8$l&uHA7CzWj=>xf5 zALMHI(jV!E8;||)?uj2V9Q<(EjXZfTKkVaKTjK{^2lnhZvA@^dAM?Ha@v_z*Vg1?F zyEykp1{{3cxMoT zXVh0Gkw+mnE)*h!;;v69v$O2v%?PEBEflA@i#t0o4BZpLU`hYDvvU~UW`?1uH4LJ$ z;W)EA922%uSG^F9jBnuxC8uIsL^#I2kH9Lj8AQdP($&bAKm8Ew!*T291?5*fS&+gEnw?_dJ&R z3Yit3V)4>47T(^mxIQcnDI?f*(5!{mGbLlYoE+ z30N7CfWm|XSiMTb=XdPDr4Oc8ClQnM6LHpn{e?D((5BzpAt@0bIE$SpSLyk`Nof0; zgdb5!*bj3Y_O&>*k0a#IS9>`H;guMUVF}5e}Rb!JoOixer9h_#wgtdSn#n^FGdfo+FP&vIyf- zMHpAe8LXV%@M;lC8u`5?sVLY&zxRbySo}$ap)Z-H;i>F#=bR>&hRXSA=zk>*=U=D6 zKr;<0Bgre}uFhm-IxelD|JyYkL#eTTs7{CN!VH{K&OnAr2DTYy;5g6kNOl>fa0a7y zrb;Rk`zK^VM>&(ZM|SKwX5ye*Ci;bD;!HA`@}-=un#fp^%EDvr?YtjQbA6Zv$yZtQ z2(sJIp2wSv`K&Bh@!0l|YcV$)PVi)_sH&c*<;ivR&wM;La?) z3){&az2(XsS#=LGrRY;1+zZj6UTABRL8{YBc4;r19DAWACnn4vAtp$W9Wi~8n6Q1b zn4rPE*pFs3inkd zh3I)wg4l5>q30MGqw;+Om-~H$IQu@rwGpzyGCx^i(N#HNdWO8PbY)+mO59V5JN<-sso_H2*5z=@I!i76Uo07T7lo@Iz+ZTTkm*lQzxN3?bUa0@;Y-*x zzQUHSS7?(`!S^vLSUCPQ^Y?GC?Ad>~T=O4lcfH5`2k+7L={=Gse#C!MK0#&g7fgEo zg}nPO?0o-%FjG}zxqZW})Nd#mq6YsbYPh8Q9m1{e7+v@s{Vx9?xA6zw@R*GLiI$`4 z7#^h#u{d>1efkTEw!ff|tpT@bzj0^BZ>Y#?;?79!(0*uQx|Svydo(d;k`|sU(#DV# z+Boo08%qMT5m2j*`<>bt)vb-0)HsJN*TJMUI@q>O2Wm%j5TdO^Zi)^)a=J)})WzT~ zUECb2hx=N3*zc@|Cn7y;YSu$=rylcJe_%QB4;0q^LE{*GvbM>V2-Iiavp#Hk^)W@v z0GH+)AnkwwW?Z9B&A|XiW*8!Mi6I$J^sJc~va`VuVl{?1*lCCu7-7DS5jjvsuq2D3 zV-mTjQ~zT4jlbj`{$=m{UzDr;#Rn69&+IR5jjG1b(lEy3 zW@EfKV1hh-vhsUOa8=9{Ibo*QRBMW&MpInrHbrL-J^2b|^tYJ7{iGRZPcwAYn9=XW zjO9#oIP5mZv%}_idcqt@^wF8CnZuXdvM~nc>;N@~WUM)UQrk7YY5`?+3;ausSG>sr zVJ#N8vELHXS1rktvBcY8OMEP`0(X>4pF5!V7dfqtoSD5Ga5vfkS}DwY);b`ad&@(c9kKSABXg~e_^Ic} z=O;b=%zG}*bi^H=v|ZizC8kJHcv!6Pgw|;r3!D{6ihOhI6(2BPVQn>I7>& z@>{c=Fk!GW-Ys>;{uT87Z*s;z7o0Knt~1P@IJ4Kq8P`pnxqo!VdJkt<@kE3><4`8E zqo6{{nUs)Vskznlt()vS44j zgQT)8^tPCLCYd7w8|QyUN2 z=?(iQ-T-I&mQru{)_9XG=>r>{Im3OhXtNJwcKeV&&AsdqA5`nJ*WAYkF@8QcQO!NA ztS@vn`oeRkFTBq)pK9rgaC-fp6tQ=+iM!fSe#o5Q2m2B~TrKrOfxJKGC4bcX-?3$< zrXO>t3I0&b@JB4Or+s$@;5R*y+GN1`vNOu8HUJsi)kZE4gfST)15O0ux@RC7astWl z3q)#1AalJzFxU|UwTnSG?-m4;GjBtGD!#yv(my_2;An!QOPj+s`Mqpf71bYf1 zA-f_Hh3uP(;l1I~iYPc8jKYSiQHUVR&e1js=?>iK(uXM|(T7>j4o;b9EE*Y&@F~&Q z0p1agM`Qo5XuR;I?>>(Gn`CZ|!G}?? zD4P`v=Y_G@kVp0%?+HVv#bGfu|2IeCaLX(X??iE!Ru%_+GFm;S#Upw~JSvt_``;Lk z$*1Fya-H7H1nzk2;$glr0h$}h_Slku@>2=)HzlyUA%R@11bBZ>0NfIo8BV}Y6LLK) z5>XkNi1;k_P_0bD99}t~}&N8y=fi!fC<95- znK0UyiJbSD(6nRrvm_I9Hj}{H95}*y~##A z=lN8n9Jqm9P}_6Rb%eXoQ#sJM#eHdT4rWiuMVekNR=VY~6D=2`x^kg9D-XF}$-^+D z=Dv{r$hY~>vZ0>t$1Z`n1z6!!0Ob;P1ITdp-a^KlcOk+g*#$7D2vXyV(571i19~1W z`%@PuXHI%RF%FL8%&k_8=z8{2N|OmkR@}%?YT5av%xITlrE(cuPL%OysSJgQW$cQm z!1H*{#ub&Qn^pyfxm8%QsTyYYYhYqugYPvpI4fU^hl#c9`6N5y61i%R8W5=5f^i>P zv3N}fZoKLs@3{ldr-%tl<-~gQ& zg+E}k_y_K2|Ag&4b=-Wf4%d(B(CYUKq7BRdGV|vstBKp^v@qnc4wg(#S05A84464F!1M)%@HR0-x~CzI9x+1m10&e{GJr8y?twZJxA z3s`wtVBthdYz?zS7>`*yGkXiIQ27tF#4A=s(S6Cc##YuX+P6%$W+v5iHV{T~HV=r{R8;Z8m%YWP*v9V;L zCc5J=pL3hud0?}a2SmjlST0Gu@PsD>X50GE%m1j`6E!p0$+OH0>G!?h`oRmm6Unt& z;*D9qywUdC8+R+b^(Umq-s;ZCZYj8sEkOlAj9s^Eu}^yn|6Uf4_@ z*JBlb*#7k=w;}+?oXMdgZ~Rz(00v8vHMJlRXG{a(TNVhzCEPV_41(=WX1_kLpU^A_ zV_SmYI-Jjy<-z>34~Fdt=Dt+ODk%iLE%FcvkDJ>C+GF+IWDS%+ZH zA~L6ThR`P(f@$AFApB(xEF}cNg&%WYQDJzV7>1I{FpN^<^F=8fZIj6}SRIZ` z9$A5zFy;Vep9tsgLpYq62h%6t;C4kgS}Vg*)e(+4qqwu09Rc~95xDOWfsA(Y4*Er6 zGx-He7er$HnMm~KGpLIhu$*k_k7O8l$wtvr7lk1gqj36T6kryG<=#VlbWFEXG=YTrm&c;|QZ|3u8a2$N8C;nX%2UWzu@oF3lp2p$gw>b3Z#$h^rXqE$FAR&yd=h$Z3FW$(XS^84g#HaZ@iDZ;P2RC`(2Tb;%V9%o%J>fyc`fXr!g!1hWN3eMFF-B*N75 zBHXeO!Q4lLUVjlrbJw+9oEZY8R4m%Xz1HDWtmB?*eoiXt6w}bVBMmd2vNKwRS|jzx zta#?crlsRHxl+?0t}d9WR*;`@_uG zUq1I-`90Z8Oj5|idmbyrOav=sLUcG285gJ_T4bWun)>4KEL*)XrnM!rN2uF0~uMcmG|zzyeQV;=xWhWn9T#_7XFqf2^&^)&H9kAj$ROu_ zOJx}I221lWkG{1b8}e{-a~`BmR?7GY3ie zxZRu&XE}E7>@Pq~cmeW~3i$6RWY)S6*4Bj>-%LN6QW1=X7h%K*_C6mjB7=vW&$&ei zRV>DYq-eOKQ6(|=OxTy^XtAP z%<7dQc}^+R7qfR~8TnEjrSR@5#p-@#upd>1Gh54W{~Q?$Gs@wpT#h{-$}vHq0#ld= zczvw`+O8F32vxvYrV^LPO*prNulwj3qlZlYDsur2mCRjL;i`2N4vVVrO{y9PS5~9u z9`(Tw?AcMThN^2duBBGPv91~)=pWPRsAj&s26so-AZ$hr;(2nH)No&4L*1?huWM`2 z$o}SgqigY=zOwU+Y9Y6|7Pk)6V$=0nyi%#f#u|FddTQZfP=}i2Iy|ecLkgJ-T9fLr z>tQ{nCD-E*GjxxRH(>R{2JB&uE~JR>$26kiT5Vsb0uOIvZVxs`d% zHte!%Lt0uJ6sRq>?{0_F<965?v|}lEUXI=EP}_R8v-*&>#xfACncVRSl zTu(!~&_A&YRViKMWpv}&QDp{~`=wz&@L=Lkei<6(93D4k*nFY10(-#9Oc zkx}JgjeoPO;Q<@G3AKSlh7FF+vW4?VJ6L4dA*tC8vO68HQq=*kog7g4*AZF`j>wto zgjXk=5P!=F?eCpnKy7X*_XDjZ&M47#g`A!%E|t<7)#ZkBGu>gW?2ZSo++qI49Vzkd zD9d-pjXHO@DKNLc#{)YxJTT4B19wb3AnoV@+jtM0mG-12=ZWFGzw3B-@{Z<-P1NCn z2YSJWI@~ed+s|{xIi}}@zUE%A_w&L&32&Uf;EgHQd1trvMp}+HYRbIfsKmSZKp%WN z;e+=l$$@`Nre~rL^>?zP7BRE9+83XAR-E;P+P}WAx$leRe!l1`vhahn ztsfbpe)t;ZhnXY%=|%Bp=O*v#JN#jB#2@!|GgI#tK;~Tl+{lA6To#B;>VbG_7l;^U z=zq-&;@|rqgh>Qr=OF5AGlH>tZ7?=biyQon^AS6oyA^o9o==a|^$>hZCc~*C1m{PG z;(!oJ23ROYy$dBLCKS5@IqL+4!kF{U7&4rSx>CkZ44hzSA&O4Vm>v$m?BQ3)bTf}~qD-pP-8iDbV%)G}%U=6!a z2Gp^Ca0oN*Gb3?pVH)ef&^wHml#`Z_im~G5%6tXnCJfiU?JQ^z1(Qs~z#;lfT>=Tb6Q#A&! zdEa*57K0(%dB0X;7m7Rexk!5K(_)~O5d)hp-nVDQVijWX#3UA5U1MS0$$Pgfy;AaV z_^lMjd}bW>tc^qOr#RUB=B&e+CYv)&^s;!UZHvc#&N1ER*+VFi0ErU`xS^JS{q6~P z>y?0kwF#(~OXTNRA~L5XV*1rY`b`tjM4e8ackGYF>>aF4_oramwG^DYm4f0w z^iBO`zP&dE3DV4}D~PaJiQ3&Lc0R8c!KIj8gA?hipUljDc-{ z9rNcc+g)AZ6`WVf>}z4n|_ zx_#I+*hOv=ef4#F*z0_p_xL9{So1RnrkXie&s^jHD>5*%a!{Duys&AQt6YAiOPq}=mPYeUVyS)1xVaofcgLMe*dz7 zeAfb;{8a#3>TOOL1yH4*NpF84WDgaxf29zI?iE5YxDXQTMS)Zi+<5<=J+lZ(7mJwp zXYMhq2nu;c@RchD#ulS`d@&>^6=U4uVvIRZjJZdOar$~O&b=*$%4hoDxgR*6P|W-$ zeNJ;qFnv`CcAP4~XzFsaewM(nxCFNJ!soS=Kz~>%Vn&ogc}ywx&Md{Dg{2s}vJ|Q7 zN-^hPDb63GFaCNd^qx|$3!pEa*~oEGWYF{E=9D3_q73uun8TMLA4-=wqCBj#t6-1ieuYtI+>d74~V>5|p({K0(V&uY9Q zn?9F$M#(8Pc=eL`M%@~`8g=V% zTCWai7VH`fsl!OlICtCXFk($TmLINX-#|TFHS6h7Xuy?G4cNM`0q?FiAn`>5>c2L? zFRcOZ$=WnLNKNl!BVPJ8GMmwazZ;qmf4m9nE;K>oI+>d{n;;q8ghdHWs7z{tRYeoT z*%7V1i|2kbHn=onPYU~@Ra=nY&;kd^R^0v2iZqi}JP&OpiboG$pU>%yu}^u)V#p}>{z z$*1ZZ)&q~RWOq*L!7SAtI0p4#3VJd3YcG;psq=OBB4@0apfguYIJa0#Sa*sUMO!gJ zg?)lDo5Y0&+r#f9GM;=<2xabcgdgz#pigdq1;LKr{~mU_8_ zFo}BK_cxM)Kf9dOl%<4@V^YEeb1A`_ea^E(rG&-tQbKW-l+bxaS{NEHEsVCLuiaHf z@EzVqsJTPNJALin?D`0sOZy0FL+QO5E-U;FmlK|5$_aUr@X_@5(FkJ3c^s;b#OL`3$EaUvSw~6$M_ZI3ZF) zsNZ)iQ22pkJ)E;Le&P|&tP*wlK!4%u_g{$Eu7UJx8ra(JH~MVx+}o#?~f!Li(ZJ@|vS zW}Lx_^|5-S0S@0bAfL_v6+;Yh>4G7C-!+7+y&<}E>E{YF!gMm;uP*uvoqd0CQ{^wS z4}W3P{1+qk7~^oFF=}g!F|NfJ&3ie&{V~C&x26c!G)1PKDS`q_QJ-&$ofFOQXObE8 zOw4d4$c&syGw!<0AWd)mD;0AD=}SP;i>eQb!Bib+N#X0t+lE zwm{YnOLQN%L9v4U>36)KpMoOZ&RyCrMRY}R<5WR2Om)`+`j zgZ|XVvR~T3{XZLs3~k^bW(%q|y7uTEM_<@p2i!a4fQ`qgecf=tIWnv&>>QAhm+84Z?jl%rkalzn~kG zC8#^m%inp_9a1*#SjGJ1-E4PwOL!opuLrDFcp&Yh2l{$?U{?v*Q{^5oALfaO8J^g) z$rD?3Jh3d=6Mod3CMbGAm5kHBcgQ+*^rBD7n?0`HxVPCG($tzR?(#xaq{ezo zINS-uNbNwx`UFBgh&=lAKnyJmgnoS>3VQGgoLx?(LV}8 zF3{*Ek-j4x2@QnloN!(6+!sapL;{j&6Pugk+3-!-ur{`;3)a^dchcA6O7#w z)TGAIvo@DLwS^)0LjI}7%MfV2BE#N|y@g&OFz^n6d1MHbD?_mKBso#%LQ(QM6op^7 zOZ*v%INeYHw<6x*ipzi`lc8AynQ%R2ZiDE#BlhY z2}k(3a6~)^hby@uOF2(hS%uTn8;&J0;g}r9-a_V3caP)F@O1=y{zM?up0Cbip>ocC zS{(t?wg~LFO1tg8>i-qh+W)!x?VbqQ|Y*CKGhimL2ydQ^fHNN+XgBW>I3VCs`B|la2 zI5n^L^vikhH7p)=Jkzt-@w6lXqU{M-eD<^ z`9v6BPQ<9?NqDJF21!g3`j(MLLI%m}JlEgABhyNfTS zAoW8EUK%mGzKq4Ws6yL1}jycAZE=*;&rm^uawOZ$z#t4YHiE6}K>p@HHJ}+z<92 zlL0VGP{e)UHSPkZ*|T%7HUpc*GtoFD6PspaqI5?l+PMP^)6K+F{Y;ebe2dA%$rS2j z)W$|!&VqJ97LrP{@SuhBHg&O=gR&7eEgR!zXXAu%He|AxBdDhDO+E*n{c%i*&QFtG4m5W{^{lHmnW;^0%unX^54lkUFA`Mj}s~p zpjL@NF_p+{Wv=sM6?C|tn_pH18O3UTZji%4-R%H592@PcVH8HrdwC5k$kOmuuEWco zI^6cI$IqO4#8|N#M_`0)rQ^)WLSlE;DAvl=haT!*w%$Bd%O7A z-i=~ZSZ~K7C-ya}~`V7c_Wky03Jw-pDZmo%|Dot2y*Tw0g zKR9&G5G#_6FqkvS76}s^wKu_@Y!f{6FvE~Dk?;t4s^z8vd(YybwSKX7u=od0v&o*e4|{*6QWn#$`w&Au5fI2#SjxWXxO-+ z#m5Z=bICH_NzTGGcY0jh;i~42hkWJ>k{-Cf&;#>KJ;+A%fcg+m+*R|$0S!-B=zC(= zJbJ?;y~re_UtEsQc2jRmwD(2>cTg7se2^yN3w`pkUXZ0=dW)SgS-v>lpZxLTe$=G= zuqTqw;9NgARrui}Iga&YWSNKhLvMTlqE`iA+1UVWz7qh&_W@`#<8yd%Ag-JuU;J7i z#M;?EI4TH6$AcilXX$tc`n&^!@H#mNzUe`@P(po9ojrpUM>{+82qguaVr2j>K+yVk5ZAV)8i(Q#828qQjou zuxQv%kH&!6d{$8}YPcVbRO@K`TN(pJ>OWQMVxe{<79UGvVKX=mT1s(9c^`+BYJC1s z>rt|ZLqGR8EC`LmvVu4~XC~X6-d5@>vb)vd@tDt<0C7C}%OxOV1N(8865w(>0lwz+ zts=OEOD;D`67ZvnjD^1m7$?gel|}TT$iRx)lZbD(5@BS{U0kn^XDQi*OxJ%>OU6QN=CWP?=SyvkNG2~e883>H z(KsLlu0vCxxITppfD~9iN?|WDg??NL*6dD&(u-8wjY}nWG7ZgB(r|Wl8osPc!)tD! z9J-nY+h=JQWs!z)zTCo%O~d1J+`xU7j^5hbyVXg@^`Gp%{Z7X*vaYgI()pf|4#kRe z`1Z{}=+O-PIFSLR$iwPQuS!NL8^?xZx)#W0Jtb_@&s+M|C^U8cgZOq5Fvy z=X}hw$cKePJ`%V;I69MlRnG#5dKKX0{sMM<3!r(s07_4o$L1bNvt|Le#|r696ynsv zLTp}Kh$ZAJDkc=-QSTzGm|TQzGm5Z#JGCHYwJ$o7k>$iboJ$e6x2Orl6yb)r2-)(* zXzj@@l<~zlyO7L;#l_g;Sqzz#C1_s5T=q^r3!j&;%UXiRu_bs)K31L7|34Q?@oh*c z_jPz~a3|$-z#uHQh{6f71+le!g=h&3I3I69aV)X53BI(Z51Lc z*ntbKf+V&IQ{~7-7*!412i4sFs)kQcH5B5i;lf<@MCP&|y42uQYz_3XYcRH@hFl)@ z;go9m8CHwvKV%>z)WWry|Ga`bgP-eA&`I_Ix#7_#>rv}h52vtt94f8HTl!ejpL0{t zv;i^R4bTc{z^!SG7_+_+?H3y{`${8o1dY&jZiHPHbKCWec;4Q~eceV3mTQ9l@FqmB zr?#`V30rSBVd}#sGSr&T*3^V)^35>o(~Q(%%~-y@8C#As&;70$8THM$GqZ)A>K1f7 zrtb5s1say(1K&BUoxG!Ve7B)CRMd`)zwH>;zXO*?cc6A^2RcYa%H z*@<8D$L5rD;$>+k8f)mC4eG+3kzIJplku~Qer*?CZ}^Li+yBDo`d=K>_zSz(zbIJz z4>LFZL)YGaxT^9G;otwk)#M+fivQu}ASq$QbSdFJH)7;BO9{JoN(slwQTQAzCA>(H z5-zkz2`Qtb$wiPBZhe;)-fBw=b-$$ry;5mmMY*)lv06qLBghCg8)Sqh^x~E>Q(mns zBUC+-5u%>S2qLnu`cQA`LDs_eG#TM^fsAmfTt?V5PF4tCEGvli$O^4DWQF^QvO;N* ztYB5jPGFg=P}3!J-|z1IicrYIYH*1oRBwGUNBIX7jhfr1vJSE zYh4tC!@deaZIXiEGqjs9>rpo$n0a*}sGE?M)=gOVsJqbknQ4#I?!v`>0>w2oZaF2!TmG1l5&2g}U0Ff{!sff=<1J+FN~us{MV1=y?MLPxpbsQ@26FSGU1} zLdFomdGs*hQT`D;Q9O^(SLd<)?k#LqyNm5RAE15RLr6`1i7Ux3F)HO1Ox0EJ&hHH- z&3lIsm3Npi>J!XvsiIf8D#mE2V|Rf%<_-Of9XCJY_VO?6^nRi5^cA<3enqb*UvcT} zH#i>nj)#Z7L#?+4`Uhx`o2UT?Xu?TR6G|I3;c`h6k^)VLS~RgzS&O?3S{QIzi#>HM zvXHcKSE7wWk8~Mt)rHI=J^o#1hjE=Aw2kyIA%G`I4$6FjWE&Bj_~(m9^Ky&g_VvNGWHKXQj=5N`v(Wy{y^E64BDCWdJm8v zf5i#Yt~;UUT_?B&IbmxEcONC(e3W&@u)fZ?yUUqOR%fg`>WokHgkxViV+1>#i(;LT z*xLom*1O=XvJ1vtbHTx1F1YUDf|JQESd!y{kA0XIpXiFdmj~9f@DuVddpcCY(F=@u)BL zfC}&Mi%)sL@1h4X*#DgW&;v61%+q;!-~(OnnGZct@rGHtx1Qt$d*bONFUZgF!qm-P z`2E-mS#P~?TEh#nznCMB@FFMO3tc(%q~*PFaECXR9$;=EtV zSrv-tHPq<#g~I-HC`2Daar+bBMO;Hsog9k!siD-o!tj1!7_KOi?Rz>5s*l5@0SFFnI?A(Xq6%$n<@G8b4L zj=;ZU$IXhsCnxH7c@aqK;wB||B9nSWqP2G<%BIo-UmgilzSFe5jKswce8(}MKW-F> z-u~qO22s09jD&GcB>Lw@V&0%AtQ;N%!x7A%Z;iqPr6}AxLKg5@<^o@`)A}U}l5bIn z(2hdsVCK-NQiDC`%55lt~DIT?#SDzVVvw$UPM z?i_i=;vT*5LFByux%j_*BR9SudFhe+OMPxla2)nim)p}!w);cwt3M}CL_HqYo#K&@ z8V}Vh^5=@<$#EcGq$3`ShO)c5B>{?u5^z&B0S~OX!)`-woEbmem;}6MzOUEtL}c0%y&5vY=((2XsQTm3q;%y=g#^zz9%V*;Cw@bu-hWIei6~z z;IG-zPp5aTPyWc=<6`W4LH39Sb9_ExD2m0{E)io#D}8n9co#-UP&kHLljPTpc*J)o zYI!I9Bsh>me_ftjk#5Ya4^P6y-APzdnS`*KB)pcPXU>e@^WN;SE=jL z&E=&juu)1u@wF6mJWV0vDuvmA6l}CjffBcjv`Xl!H>MzHKq?gHreZx(G4eLEe&16u z$(h-|(Ph{D7&yzN^g5idoC2l=b`7~JaYN+5dJC;GyCOJKg&n5c0R5q=c7Wu0QcS5 z86|^5t(gAxgAf;qSwJ`(NEa?GC*)5HvnynFlLG|VscH` z|29LPN(=0lSOO_la9UuEN!zT^qmj?#adx;k%MKI%*dva9RsBn9P}}%S+~tVzQ}`TI zaDpj&RwwkGF`z~5xGT4xT=6);6~WAPpL^!UJf|DkrEZwN!W|LE-7#Fl z9Rsu6>D%zRw8#VI8$4h**Av;Do_M{=3(l{-aHW?weFJZ#Z1%>ut==fQ<_)m}cZul- z{l4e}skzK#pYkOe&<``n-)v$wsm7L^^HHaq z%GJyR?TsRffc<^LD73lrbC90JzQ0l22#!XFQZ)XYk4Eo1(a?I#ZvOLVWT-}Shb|f$ zkHjGJLkxy~jzIxW@$VS4S;Zi2Tr9Rsj>V>ru~_|u`RD%BeU`={pPzS=uf*Zi4SwEz zjl-CtI5_vx!!(ZU^F(scdnKTMzXZM`CooT) zfCXeI4W%Y@Xb?Zs_<1I2N`y;aZn5WB z={@+CBw<`#5;k?xli+8TgnT5`xycy0i2TjR)O$W8!`?3$Rm>;LPy?zhB#*O&{hN+t zXvmPM*)s*p#-?EBCVr0Hpbm7GtWD;CEW=W;Jv9aIt5TrS&b&~+RG4nzRxUrwrdXsR z+kqM8l2mro(vUPQ4L5d@g=CzDr__e7N2Rg%nugFiGSSIE%4LQ*uhjUXp#^j zzL}7sUL?AiiC8smxFu)eNMR;E)@9<(KYs0>g+8;$+&q^BTe6T+K4jq`*_+;GS-9+x z1*gO;Ed9&wPq%C=@0E>}%rm!LVQ%PNHZB=tV|P$C^1``|8^`a-;=DOM2j*lW=`CPZ zi2Jw`H|Jo=)*Kvqk%RU^_V=Z75x+PW`X}jsgy$l*M;>Y?v!A~t5Blr#pmR45TWt8f zLmmz$=Akhu57%4rkR?wB60^*Q7Ukm=HKYa9iRM`6V_HZ)^qFnGUPE1p9Cgiw1(>v= z04iGwuqmtn15yi6POc^@3$Uz%-J-wj7Of{6X=@?OUNK|zj(Z&1g^*>x=uluGI?K4F zORY%#K@sY{P%E-5!g{|VC{Hbhm>Hr*rDCjeD2A*@F?MBhtD}**qV6Tg8B2ENgc5iv zm7vGI64=}>LFB^{D6r#iZB;_HTM4&cN)QoL!aoD@)r(8eCR>Un@}=+}Uy9>OrFhRg zv^qckOTU%E%Agb)$)$K#S&F>&QmoiihV?<*oGU6r4?YtX>@3Il{pHBMUXH3eT+aRZ3PjDWK#5`nG>)_Lf2soWm?_fHtHABx3Mgh(pr=A5j7=-~ zY^cPr)Joiyt-|{$Raid13Zs@*Ve*M8YMf+nN~$oeybALFs<3fFHTcWX zylbU4#dH2IcklXf_wGUyjGi=MxHJ^2kn@y(T-bI?I^G5VBd<&&Uc-7uhxm6m`-G7cA~4e6BAlH zF|cPBOox&Mb-fFHYrF7Z)L*oT|3b`XcHwL?JD2^#x6S`B_ToSMdGZe??5iazNePC# zrG%?b$>`*fe?e{Oy_9e$R7%*FC?(92NC}%Wq=d?HDWRaRv@n+ZsM^WWLdH^Q!D557 zV7pmbShQPOkUt?U{3TQBM1Om+fx{;*HiEt z(MyOu-Ah=si0t;0y#=K&y@lbT-omGlK0;$vAE9e2dF{vg35y@}6YO907hcE>5H<}T zBqVnZ7EVkY!aD3wVd0}?P`$GYt+nUj`}Q`nmOVm|>N6A%cn!&+HyHoy9WKPbhv4}E z-M4&%;&3%?2&(fg{~4ycKjZM4FZjv5XZzh>F+N@s-9~D$tDpt*N7^vZ(Z6&IuI)Nl*-IB`({!=jNf*+wdMFtA1Dcn9U|!GvNFU7XCtV&8*!`62$L#|kTdERMz5uY^wtEbpG?r; zVgkob6CAx~iV;=pBBq+5y~7M&uA1Yb#2leg7KkRh)QjBG**X@u-`x_Q=5p7EY%Sxp zmKc26lKZljSSX@K#B+bO6cbmCM#pJz>M1n+@OPZQzmmzg*QeMO&!9wneFh zEy~(#p(A4lrEYf6H?hM%Z|16`$u_+~&E|zY+&^QBfL*Ll5gRNH%*R^CzF1btP>o2Il+1{H=*}9VT_v- zsw?%bcXA27gUa={&Lg>%P+d1kFyK*2eR8wO(wjbE2JG9QMK9<#gno?ZSAZs=#{1~m^idi3n~H}TiUkV`++9j~UjBW4YI{pZ|a^2Z$^neLdG z<&K(x%uvxowXtLNKimWQ@g6W4Mg4_c#wE_4SP<@sInnF^q zJ^2E^zwU*&+g@1plpRMEFU)yI9p=3kY=gWInCpf1axav1_lD6@?(@MLF1%0A-|5Xx zr8ll#^v0+g-pr(O>xjF4%fIkvF4S!La>t1ItC)>G@mLhkWtM*cZC&2E07+ zzuo9@AN|nlmmjp{{V_^`-AHONlQ;OInP%w6>k!S+5O*tRtUW|ulKcLQS4;Kha;Yh*K&3-ueXQe zQeXD{r$#``-sP3L6GTVF!B&Rzst3X;~D0 zu=77zk$cexqp$l}{}trA?v2LJ zBhh%Q9F4voqRBmvhGsq)0n*f82E-tQ`KcGoPmSWvw9kI-^IeOw=-WU1RU4d*@+;_>zt-&UJkuYtv+yky z(MGB4h;l0+hZ;>)DrWVh4%0sknFG@BWIh=Kcha!yDLrrVG<jg@wSjqfNd~LU$UJN~oyYyGJbWi3%#!aN`hD^- zfqjKe(|oKc%tw1sK37a<7tE*_4^4{UvAP7M zyGyX(LkWCUOE5iz?-D%DktO)Crxb?4^qR}dAbq18>z|b)(5oCZ{VR}VSb>I!3iL~> zz_-bj7<{l2>poSYmi$!N!b&_DTgAQiDhx8@ecp@R$G~b>Bvr%dcnyB&vfF4;14G`~ zXY{MZfzh@2^1K#JskOKgU5DDLI+Q=H$1_#t`|WxErXPGXt^u9QVx6pPfVW{IE*Cc< zR)_iggcg(*wP5*V^6B@tp;EaGa=*wqjcp@$unlh8+94Ixj^wW$D7xK=bgNE8C3fOP zekVRm>f*jq7lcLs;9y8^)a)Pk-Tz@oZz3wmV%J?v6~>(*iATkx{t83*IAhLe+cs>kKy+4F*HM8pm~8R`3b71_^5_} zN$T+0`VI5HeTQ-FcXAW8u;{H8+|;#k&PN-APm$&BrGxHEbkTjaE@M=2tW4o3O)i6;85GRT(d=r@zMDCpqZU}}$K3f6OSsW1yWnk!zSFIce#{D~53G?U zvBuscYYdrS1IZ;D+%UDl;t(79>^68d-xiaVY;o?O9eU?bbE~pLTZbJi_Sob4X?xy@ z>~VUH15|V!5YIc!G#>|)l{?_r5=TfKcEp$y^r&t*;vKn4ukZeWc&ZbE1tP?N?8%M&txlPA=M~64x&3tg}EIpymJ~*A?gY;S-DAE^N zanBc_W@IoV`{GF@cY#Ov(M#~d?=(Mn&hp2J`Tkh%;quOlPC&+UbjHlObx^g-Y-UNCa*y~5b}wi%dGWp`(TV_R;+kx2=_cgARR-mrXmEp$!16$6^erS z^kR;M!kQk<#qVS@=!9ZHVknH-xY64;3^q%`uxAB5nLT0He1lx~fG}K23q$|RFxZU@ z$AELpR5M4G!{`2baX608h`=D`$F8r9V4frbcAnhrk*Ak3KN4eCMsg!A5>@nEZhwzN z@$@Lf^Eq!uAI0}v6p}AQVXX?a#r7zS?c(!)Tr?baMk9pUVkL9Yv!6!8P$o)(}-=+jK(_1+s@R`i~nT{4UNS*^$ ziLkRyM6iD%Ugsv_iHryndy0@lUuD)_5ghMwm+`v@cZ@|iWg>!A1U1Ss5en-?_}M7J zZgLa6coNo&p}S9vv}aIWCO_d~YO6K}FZB^l_Rn}L?U8L&8?iBC^6F;G2|_6YN7WDJ=d z%|egw)EFD-$uwo57q!MIa@p`%pN%@6hKJb@Yi2{DmyHgqY&`QNCp3x~>&9$u$L8SY z%N%S}%fTLV?)`Z2Yg`VdX6N9ed@lLwxwy757f&|lV#a@b?*GaK$V@2CC5Nab7qgjJ z%cK4nMPFz2u{^|F%)>ydJV@~lU>cZ*!NKGwWaZ&}HuKo@dq%IzhuqnGcz(@C2#=L< zKEkZ>QRJ7;-ST`4=3T+DdjUG87qBZwHoHK^0$E13Q3a4LD}cDV02{i2?|3|0=_I+cFHyEyK%vb{CtuA>2}ilHKK~)hfqHy>hZR z%W-}r`3qwz@MUTRrru^>(YgX_!z*CeR)M4AD=|r_5>ZzxQT?crUTY=Pd@Ir9SBa#E zN`w?tvR_w)m21gs*h0VRbQQ{;R$;}fDx~plQeaYr-E*q3Wo0#XZK+0}b~P>;Rio6d z8UvTsAoxlx9y`|Jm=`;V!L=~p9p&DfIt<%U2dCS0P!FoZk0bRkRbys6x*mBM^@thK z01m^W-K&8H9~V}BlHy;VXfJSnR$&kwU(X3tIfFls~I*sTTo@!LjGe5PLs3n zdr&JDQQPZKX~koYR_MyN;qk~eY|d)Km;83<^y$Fc>+Bd7bRfQ=1Mk;#qIPX3cA9lU z;>hoNbU|rw7m^LSxKr7MQ#M^tjO)UdiY^Qa{R{oBzZiR%8#@o!E#&rL>0T+}*8}Qw zT2jK01ZKjh%dKJ#OvGNH^C4*=Kwnz$8%Tb;wv2E{Pe!|!Jtr&U3e~J1%Z=NBwU0<1V;EwVa!TD?PC2y$2?J^nlpj1MfpT zxaaEu18O=O&v|0kWl#L~hZ*@edY*uH@jorP>JzIRjD_W zd-%{F^nuxGA9(KZL0XUx88JTC-Rgs0TYYhc85s{lKWK@_WDx(4op`y3`Iz4R@EpPX z%S?Z?Qj5tr><<;*i4I>2K*Q4jzQwf1CH*Ss@hhSi2 z2#&UdU>Gwj_g9kDuptyrZib@nUMS9ckl)aY8VI?FZY%%CZ@9NF3_%ydAbp9O)avwC zzmnx(!j0;{Ff=8F;Wc#=y(V_g28LtEeDV^1heKo~P@WV^eKr;jKCx&Hh{Y^25tYlB->)RcVNx7?*To@;-LWiswTWJF@DGc_ z*be6QN5o?;?@BkAOI0lQ)| z5^-u?B5KH(l8K;VLOo+R^Cz#n5|Px6`I0&0NX-`^L}318p9od#hi(4N&1#-yej=oW zbNfbtccJNGwCokb^&E8$=1=y%5#ykR7%%HvG z@2e67>qy|FE5ThC33iDj_|z%ET;@>vk~Ou1JO_QpBzU;V#=f+5i3Ybk*Oc>~jcX-Kr)H@!*D;szGc$X59!7k3mxBP4@ zY{*8yT;8dur}*%GrE-*=wFfz9`p%sj?Hux?>Feg_;5PFzPeuROHR&%^%J)LVE0w&uZLM;>19%0tuMJk%@aL4}!`nP-`+ z(d54~EuWj6`8aznAD3>kC-9w~u5LcI8|LH2?|k(0&&QrU1z2#f0QdZ;qvlJIPN>P2F4Ev4BP-$F--L+--ucHi_$EmwKD#sZ| zX804hTPoNC0J=$N?Ly5YMqfH` zZZ)_K83(v0{hZr{pWE7{MzmWa=7gFQ?A!ge@oN@byp_9mI5V#vn4tpUv z4K~lYRrpFuXycu#Nm^R?M1M2bQCg7HOAD^#uxF^t2o_p0!tI|j!oXG;;h?gt@cE^z zkZC9@2vM@a^*(Zf;bA#phnk$=q#-A)*OC*`BIE?FTL|XcAE9(of1!9` ze<4n@zi`2Pkf85;6v@Fi(4v1ELnq$BK;Ju1DY%1zU#Lv7e+G(Y={6}ewvzxXS9uKbFXd&zdl z`HCAJ-*D9b8~%iR!?%dZP*Ob!3pYSRq;C5OZG$Wi@NaK#jLxk9xhnw!Nu|iT+jZ5@oj2n zPk*9+96iT}`Z&TfXPN;n&NRUJ{|xcO*pQl`A*K#8LhVW;eA{J&UB`@&aoh-H{>-|I zjc{q>FQ_T~;%E0S?wcFq(*1tq#k|a}Xa-v5WNKFprXu3#EPRksd)65~g!vd2+EYL5>0 zRrV(@b+9FN%(sNzUQ5Itqi1=~65Vtykz>ScySXLqQnyp}v4l?+dz;!;(9N>KOmZQ1 z*IMD?U+!o3vBtW2*2w#0jqTyq@aV9HfwB#zU9=%%%m!Av>~xB3@HW>53tMckpt~)G zjCO5Vp+QQ}s@3GdlSW#;W(P2C6JIBoWO*fei1tsXWCioY45f3GuI-P^jyKr@ZriOL1bh z-U-bWPB=N4n?uUnAiByu$al_oqRI{_bARIuo$<`anb~h=v?n>^h_nmV^>e|8{w|PR z?1GeF7tD-iE-;fGY9n>Wwal<@b4AW>SF+(<@jT2GL#Q?Oly<|_iEh}unqR-Tp-$J0 z85cMHUNa}y&z=7-xnts3cji~zshjYQy~Q1_&)soB#2q90r(3xvd8y4EE6;f#^@9hT zq&zW-cWkN8o(Sg=c-IaH_JrO@FRZxig;SrHV{i3B%V2Mq&G3feGH zW5aiEtk&{|w!Jr2_PUZpQ;ZG9mV?u(tZzWCehi+wG=xXG-$ z=MF!tzU_y!H-1q0;D;xl{GhJtN3Y5cgV}{mt@gvX)&6KW>(8A+e_VX&j~y!h*r4f; z{+{$zi~J#37l0YmJL`T1z{D&72gpbN8XbUbIoz}y5Qr(8c+Yp*eM%?r{jar z=SC2YeV|ur5`^D2q@UW`F9h9dxUaN` zyxLV{*&>wwawv2Ta7#%!lp9yfTmIs597WiH!=H?lfFD0CCB*vwQw@}!pSlVhe~g9+eSnnV0HwwHgFH~a|HFw z2t0O(fU+BZCL4(xOCr&iU0g3~Zl60x!qtmjYJ4OhUX6+j0*FP}ZsTzkD z>Tx*rIgV_DI4rE@`$J|$k<1wQto}8%!ay1jsZ3A~Q>63=sO~9Rp3E1h7z`R=mR`VU=(5ggw=85!;6EXAx zx0+tlC;gJho+Ep}nu++MON>DzBn(Iml zo@|l8MoogNf!t>gW9~hI9NTPeKGjN~FgOWE4<})*3%8v-_Yjp}?J3+yPr-$5sSqZmqSrcZXI`e}`iOh$uTq&0B=?Oy z;qi01fw>@!eAP5KzDdKnpfobL)3K}@`@o~=nJ!IdUN4=w&UAcWwtfA-bna%5Me&-P z3TmpR@$3Wh{XkDK6J7t2P4PSvav!*L>6OV{Lw12{xYg8~yjgmrTIaJ-M(xx}pWR+Z z<~JqT=-Vd;o&9pKlXv_w9?v~F@JP$S^@bcI-p<87*IcYf%f-Px+@w52?u1buy^+ot|Sw?60cTQ;j9m{hyr=-(bZrA0(H}C@#se#a@TQ3XksIGw8$2l*My{$W~|I^ zhQ*y0W_wzpUf2Qou1@qEFDryimlL)ZEWn2Y2as^^20q)o?s!n4a+o=BrfEqOFRAAT>y>QpeFqb^g6m z$K3Cq5yAf8_jO;o;sx`cCez2F6rqU{H$&(mOSf7^{hX*_v=1tc97@ zTBz=!jTMu$QL#fCQ+&0t`I!zg(Bx_<=(0DZi%C0m5hT_{Xo)T!ztSVCitKnRJxmts zp`XSNZp{9~-fKT`?DbFVnWRr9FWFjB23Ro106kR;(>4Xv!R#Dc=7u z#W{0RtnM_WerblC$IQ^6Muvr*8KOp+qtASEco~@^OVa`)4J=UaVF3qU3#0^Dz+A=> zr{pX#l6NYFDV8{Y!4k{qd!Dl3J!+U09#6EwuLD-FIK%u`s}){uVZU#OHTGp%qcz(a zue#YFkv`}`6C0?f+hA&i4MN7+VherHT{~>i&eL(u7Q0n#(ZA6alP22HBecV|Ks(Hc zw!>^ z+39n0!R~k$OiXaWpla^S@O+lyE{r_yMC?y&p6|*{23J%*q%W)Gig+_uOiXsg@G4gf zyzWMCfLk$PZYbmZsH(;d&uiV#$a~Ve0q$g`vR`q;9s94i!}Fy(g6-Uy1*dl#knX^18+XL7^ z48X0E+=aOmfHkkkpVbe*Dc=B0PYHlmVE|5c1|Xgu@RU=5e2)%2}h-p`KTsbFk22*wz*U=*YT zK={LBcp*)0 zUdQ}nARicm#gQ=xlElC{nLMq680?@1YPFBdtc$Tw(r4Z)G#0t>{GML%oWHT?*uX8x zYjKd)j6=Lu975R9+xsUDuE}wjQyGWh?5s@cACLHv+=Q7MkBe91QO_J$^T&AH(~2kG zl-(8TqIwna_&P8Fs|Jy|KY`xy!~}RxPQZC)!dCE}<3J|Ax^V&&OcL-P?>IN zi1})X_`o}iNo*qSh!YW8pNP5rcz2n|tk-%G1|FkMdR~O&-y)o&hkP`FcN?(?IvLD^ z6^PJ>p7Q6RV)Wr1X8S2I!l{|QyCR0Rp%^@T^%p}mUW_Pq_v~g+ zJ6$5dC3g3G)Ft@Oi0qSM_V@mg$Hh)e>+&Szol8RSjU+_8O~SxmN!ah5gobwh+R|iH zEK4SrfjKcdo(ar}v9~w%c?wRMrC?l23KE-B5IrIlp@&l0jY!2X-eu0cNW(_wG&p&r zp*SWD!~3K|)IS})4)eZpBpuSn(qVKh9Ty&?W9+YVe05L9e3=Z`jnBaSvl-mo&cM-# z4DP9C@L8IHE14POHfLa1aRx43&%_^ht) zM^E#7+>gsgN_svr7ISm^Fj=X{{@yJ3Hy)48ywL)BTEyN1;OS&y9 zLfqCOZ00>_;_zbpGbl!H+Y)TfC_zbG36yz9vX?H!-5I5bI8=(}H`F9`OQBp|j#0ZS zaJQlY*E%aupjZjXflBNeS_OsURaknCy}xT!%;|EcaaT1``qtp%tQr(3)!^B^8fdyQ z-xXbheCmYX(rb~y!f)R6N zMp!b{7`Na3#>iPF=>O0Jvk!3Jrl&c~E^yn<+5$aYE!exYf`TS_oLbf>e_@O5nzqoK zZjVDd?6LALd06CJCHgwTaOofR-nsd9gPMiLA8dBu3FNcen48IsPT=YTWZ01(x`a6< z_7CR=y5d8HD{lREg94w68vEQa?Wa3-Rk8m*%>%J~wmmw^O!Y}DVAyX6ZxK9eqQ;B!dP4{DeE;7z8a^J70;dFqFed@onN z#_aPGf3#>Zn`GpV15y4cS&VYPe~y5O%KAsB|&(` zywTljL6El&!f>gnrv|G9KRtBQzoyPb0~Mj0?u3#9*jz4?&&R}r!ELeR2y`k&;>7kyTvBD zF^d!!iT3D71aM=_CN~mpvQg0Q$Bma!e6H}lS@l2^wp@?G_c(fL8QghMApg8~Gz^(J zO6x_w`H&b$jbw)SA@jt3%oml#phh7UW`kle|3NGgf3VL@-lANeILw;Oy_Sn{7=J$w ztKFCn_KAZ+P#n2HaroC12bGpMq)(zZww${xyW=rfIUZ{C(bl|=M=qZiJ*?R8=Ci^= zNI>a6@~@62;4PmAbL_d{Qjvhm5Gl? zPeyiXGOiXU<7i1To(!cwHYo+Zi&BufCIx4i7k*6+;}&wyuifJY@y8Uz8l+&R54|(y zg~u;T#q+hPF#nH!*u7L3s-{B6IF-M@sStUmVoq`@PNk;eRz)hB_^zvUB@Gw8rs1eY z8Y037&_go=13feF&?|$UCp(SoH;!hW zC?F{VdO6G)F+V(QVkQFSWa97qOy0;dF?d%dT&nnQ zmE6e5E5{&o1YlW;w33ms4-5K;5MZTsEvgg>waJ=#$NtWw*V1B}#f% z!gqNkMhTS|%dWfeo=Qx+R*4xmDpB~X63%v&aO_t_UN!fL7gb^I$|_)06=uKU?#tII zEMf;<|5p{V`Ai(py&CCzsbbRAPo_cx?`jS3AHvSzF)0C z?(XjH4!QG;`(q7I&yT(LII`Yv#xwa$J=TUC#WwsYZR5Syj=j&@F-R5yP(k%D7IL74WI=#ErI2 zl=gB9c_Gy~csO}i^F##=_6-ZZi3;tb@>?baG4C8Z9To`a%T#&viE=W8Q7pllr_xT|%Ku28IX)G>GwG$Ukv1hnCU0ks5 zBOw?Kl@L~PV>xq)gdqP+LWot95Ul9uxpA*X+CoBb@?rOIn4~a;oU6FCl0x%3Ng?v3 zq_D|YQW#Y%DJ+nX5=M=Y62|Y95}uey394>V!Vz{27fg^Aj?R!4Zfuhl623_bkG%T` zU3q;3rqiQ$|BV(j2d>6U`D4gjb_#h9&q3qu zHQZBugyV&e$t-(*CtRc_Xn#}^!+!S?0@5}%pWui{R0n`KQImcgW|P+ zq3rn=z8!y&;i`#u*R*i|xfb62<`%QKHllWFW6gbSGT^kK+O3TZLv+wKo>}>uI!FlD zL1noPeJ@?!Bf97=(dC{CZ@x+T^osQH;(|Vo8S6tgLLYl7_2Ib10G;&qn(YnocbF00 z9W%nLEF&b6RaG*|827Ij!~KUb-Z~qjsK6LH%-5SeG2u>!33Q@N;5dmHv6W_+x5*4! zs?FFFGRKZ4{i(@@`A}P|NV3J(p>}XyV+V0HJE${{uf=TH;!t}mh_T1D`P7k)IiUQy z0~FsoV1T;=bg3`>h<1RD1U)uc@}j6MJqqP!B3Z4qe8x67;?;8}+}Cp=Th|#A$B_*s z?+o+vWVy3{xH!!jm-3tu+`>KL0WLT;!UempxnOjM3nsFUI7!|W2QItf`y*F~QY%Vk zAMu(G8SehBFo|)+9_H0TW8AQJtUEfUxMR#*cXF-WVSK_JD^=Vv+u9v(=W%ODo?D8$ z$bM4vzyxCt?6vcN6dCS6@;&gb$OE6H>7h+w?{F!1gYS6ald>nQ96Zr5$qRR9dU4Oy z3$reHVLx@Dd5p~H^2OV&yszntRa*HnN9>D%WPBX2_r*RDKj<&?gC%oq7q0rDlbTTUPd`lQ z^us-AfBYFrF8f4(NG~KKc7;DRU4QI8PTtdX_6;BU!$6Um&|813R3#_Gk^kEy>PB(_ zXjUNS=}rLFy$pbuY5-n`upiK$deC_KVmAX(_$(0K>Vf$8HxMgA0}+0U{PkZ!^aFy> zM(roFCkO*31jB6}`+`@vSyT}WyS>~Az7c{>r4VfW6aozsb_6^^&=MH}``i$WWoK}I zc?hzXgraACD4e#2!unJw{MZqA%S|H>n@~(=Pr$t-6o$;feQ9J)ZfO{v9S*~Gb_S#$ zF%M_X{YE~gGq~H>%^rbNI2K&t7VxKV9IXt;c6wUA5fR*{j=&#oF+O2WaLbTL^7td6 zbutpIHzP6l9(AG*k(jJUttgTmf+%(bb0gs>!+U)~6jm>wCUhbSX?#4W4}H;LZ-Cm6 zIP-8-4N;iVhkTNa>;Z0x#_4o=)cNzknM@$CGgG9(NeEk}$}DS-Hq0$fwcUI!ce~6gvdZld<$`GA=P6Cl<-i z19izb#tp{HLJI63Q`b>V!8CVr*CSKVEk#X-nK;>*WS(qD#d3EtO~ROwi>7bYk%|X0 zY49AAhC|b+`|Re!o8Rhc8e+e*BS`LgsXe*3aoirJ24pLwqlx-Yp=LVN%=wkKe*av0 zTT&VLMQ@8EmE_iD;MG$yn~JzSJS!7vYq?`Y9mv2p6O+c1t3I84+sj!sy(F1Zz-Z2yRBO7DfvQa^=Yh7$MzO`p# z+4vkNPS3&XV>!^ekORM4Ihd=NgFKfUoKMMNRw@UR2j$}ZI_@vt$i=byxj4rx+(5rv zZj9&Q`LaChJ(Gu3hIuG%$itJF`7k|5rtQgmSl`aad+mH&v*o>CnU8#l0@w~Mz`p<3 z8N674)9eiv3@yadS%q-?uMpSwu{X&4+Wv>+lqeQry?ha#f?W1ZMYwp1+eWHI*xpP& zNjJLz)5)&o{a#3IDBqmi+T3DXpgwf%1bcy}*bltP-e4AWBI#1-4=qI$Z}V8CQpgvT z!lttngWbxo!>$^2@_(-l)W({>0 zZY3|T#lUN|aIvh#YRNik>~*Lc#~XQH9W0dVxPw)XLHYHVUtEvNn;NiiJF{n>8^|AO zz(j*aSk7#M_<<(!l$vnGr3uMiO}M_U8AmoYqkK>c5*M`Kx=1T-iMDcovlU7vt?a_K zLtzV_!|hld$vaoE1LBgM*we2Q(=T?ymbo$2zFo*W)P?oz7Jk~m9mAo$@Si9mIIb2E z%)>+kcV_5sj}Q|a-iZtSco)j?=F|8fDa>9eMfR$c5XCz#bgqmrXiGmqh#Me$8+rz> zhuy*Dy)RL@|22H)z2Rnw3icY&{~PcbdQU!MpY9hVu2(~rxf%|x`wr_1-{I2l2Zmby zfc%<2IQ#Msmc9RjVL=*@j{J*{y_$%etp&U5x_Go%57~$GaQvknH;nagIZqFXgY>ak zLm%_~^tmmjPw!bD9p4SmFu)Km?;GN>o*|apH-<-$G1h$}A3n_lgW^nKO@2b`U~_tb z+^L}s)|zaA#B>YvW?5j*8A~jGLpElPB?gIEVf#5N_GPTFjQ7dl6l<*9V?+JThTT~k zWUJdi{-`b9YT81{$QEYw!2XvPQkvy}?nMr;+~$DA`yB9LzXPUFNBjHG0Wq%}p#RZ< zJRS%1bLVz*ha|TnGlMh5$^+|D$T4X5fDRc261)?>%XnehDld#a z=mqWbUP$%z!pLARBnLPF4Ug3>B8@y3;+#7NiyfLDf+b{4zPoNL__3?%HJLY4Y ze38UVOr@9~d=B}+(ZUZ>&dkF^`(a6pAAWWF;agvS1da^ALgrl}-v;2U4f*gL0c5Sv z%VEc_jC$L1`Z~X;wT+q*1pC$Wa!P`*l{xk;V}mhjSupo+g3)^<7*CjA@p{Ky!e7Dg zHwuQmSO`w_3BiC_AsDnN1SVTU@ai>xZOIN`b_hMs5Zq^;UGXCO|IDl0&<=%BR4BCh z4)042V|sTeDyh3Ye;9^k-`Vy1%Pc$nom;74*jdc4?P2ig-$N;&4WhyUVU|$V`btOBOqUMRDZIa+CQV-|J%W z7%d;qpTXqQ>v0n$J|3kD_|9IEfUM92WVEsO$2|L<<=jGEnTWZY647)v5j)-_Vy#{x zelc&d&WHU!`Zax-Gx3;1?}a{1%vSm^hm!E_OA>DVPJ*6Z5?0S-S6|>J%6`6=|EJTP zO-7$9%(mZ6#-=yP%(`$pIf45qsnqQQ?O7Xl?-8S z6Fw(T@ntH)`F#1A3e5=i`*TteSkCQ2(KMWxkOpCP8jdbV!(+`fd^AmiqZ9e{{%Oz% z=B9Ea|2&89^9FLt2B+h~Y-)0=(&2VLoy^d5Tz5|AR&xe!O{SkSB?G3@Gmvm7gP%93 z(M4upTV)2$iDqKJyi6GKiP%XVnK^wP7ixEYnb;YXiKy&MOzz3VhNJBCD`a8TlPm+`Zg1z0As-i-^0B8QA5s7E z(f?2Z`oAo|$yWvN)TWkaSO9y!0-Px%BVc$Tj>mCVIld6$DTTODUx+bn^noT8VaK#0 zeB?Xd`84w}7mLtvrwCgNipT*g!q)mCtZpbm@!?{4Yf+=KDTaeHb-VInq*A9_F|!1g z3%Oyqy95;{OAz&@1koQ#pr%^_iMSF3b(UZmyMYI_n62?DMQV8|HE!y3{mKw~tPEGL z@eWWZL-V&XC^?tmbSU=<*$ou^ubjIJ<>+8Ha2Pv*=`s~aoLzw$GU*%cRgmXdfpxN# zcr?5cXJ=PJfzOCJmAu<3v0bqeeN`$k_ERPMN|lgnByVhD6(%m^HX-kbrM*=Mo>`3@ z|5ancj%sYWTa62ks?n=YrdV7xCY4pgf&0w67SyoYUPD$@4c-;jpnpvb7K+rujd}M` z$7->SkCs9$8XnamM2Q-maxJ3M$rqbiM}7sptMED`l-J?LaBdW?smIU_WE9-1M@2|I zx2PL%XH^5F-5QV(KuwMsoO65wL`OH`AUR>9nj4WbuL;k#HKE122|7hh*xA^G;L&8$ z&u0H`XEWraTJUFB3+|5M{V}EmKM%LS>`V(5lTq)^C+kNGet5UwWIzjCLt3zTK`UB? zR`l7`iii_!7_Y$2U{4#4EpA8ml6DMMX~*Y-9oTic0}A9HttWG6KM&ZSWM0B&P_7EzIS1OaTly2x*%23g~r?6nE0|ANBK+(>BhE_Zfte#foFLS zikf>cZvH=5uK0%;UjLx6lB|Svz04E#;#_Sny2V6<_NgMmpT#1A!BY|8M!JX)(J3Mb z6Geq**F*(ZE$$SivQs#mJwoymEN+MivUkOVU`6KN$>>_UN?fQ(5f>WC;#z-SLQpj# zcgsyeSU?un&xKNg%2g@BLP<)Hdn+vr(v}vsRY(gTDy4;n%07ZK`R9+3eAQ61=-aD1o1P&gw|VQ1QRtSSRGSEip(b{_k7|8jvCGn`+<}_Kk@JIFQmWz zg|my9gAz2LV6TDA9%@!pIWmJuPu~pcQJDSRr7I6>bb*#%Pf>-tFT3 zwA7Y=*KJX@%ntwLnU_9nkMcr${A{&{A8)4q)()^<;RspYM=?8`;1J>jIp&?C-a13e zj@x=}&bZGUPvZm^Y)E#2>MK`7ySt*gz!kdV+z@$+9Hp= zJTQGVck-B1z7*}jo7)qzKRvO}le^9Xn6ugE1`s3pw z-bP!=LD&(1-)90K_nDrgW&rh&04xh4YpFB`m8RY?BoH4i2SPP15LX@s;Z$f4 zE`nB+fuR?-n0f#7i;lsx0kN2TCKesPW6@vWmXMv1`M}gFDQb)H9mnFrl0LX+C|&#WU;3j0)nRb0{8rj>dCyj$1s+@#y|S z7OQ_eeloifQy-7ILCo9EO2B|+36R^FKn*7WzP}T&`7gKNbP}-EF#%s(xZ9J=t>zr= zHV;l@ewlaC*F+5Wq<<-vglU74kUg3=(In~?$C7Z{ju~8cZp87)t4qR@9`cQPlQ3Wb z^S6tWA$f$`anv$qoMCR|K{6D17d27KIGx4Ko?$8IH!cNA=Th+aehLKJ6c`4jK(;sq zk>kieSeXiwedM=3NQJsmDrD$q?jv{lx?~zojv$|v`o*S?WFwfR;j1}&ehz6^;GTvb zb!mv$k&X&>`25K~C>WDLttSIw=a|90n*rr}8DuTfpR~@zqJ&J$s^h)G4DP-wSy=m! z_lzPlDymt?pl)&FH+y^rS(s}`?y6lDrbp0|>?PZ1Z#LiW*|_~P8|$f4y!7JtapW6i zWy82E8`ZpXvUuMNoW-sl^DBe-?0c1i6tY*}{9va~JqL$%bC6_|gXhllCCjN{)aJmM zn{six$UxYii^_w!WVZ4KV!qew3U}soac{ z?~C(rw~IceOg>gGqn06`j{>iJSVYmgWZuO*lUsDf-1e!+N9cwE)I2P}PJ40?=u_Se zFF@A-cJmgIvq}y^TT&s;%`U>*A4T|W#4R3XUBtFf`Z8t-n@K>U3TY@KSzVW@#dD|>$(HHed_Mct5E>|0ceIm>GC z-;P==zEX5q89pUyo0oBvEQT?d#!4*!oHSyoLcHXwPaP);!jyEtX9>r8(qhL zt~xYI)FPR(n=(zDGNf4>=aueh;i-Hc?1X6(%3_TK-V!e)#V<-Q(oqt>4-NJ?%& zNLmXXRJ7n;O$(N_x8Mt(cNarIY z+J^PDZ7||9_&_@|Y%S@$TSWQzsm!b>h~uPVU-v zlBLy&FQ%RFric0V3AcORb>X#l7ZQTIke1Yixu3d`;n0oq`D9J?@4@0_Jy28N9mNh| zQa}$>gL?3aH&w;tf9z%bgYfwuB)|W|iG5@<-0OwBbuZ?O77>!!E7X=35ypKM5hm!0 z2)B$x1RZjzf?Y&}N3J4*can&(c9*D7b&?xBcSVIU>=?S!yZk*=OqjQsj8qdbLCI81 z@bKf;8nRQx#Dy60Q2#}U3$~f!f^4INpgcuVm|-F*Scgdpg~z3YA_FNQo7~b%snWup zt1`kJdl{jIJA;37`wG9sWQEh(vcgx9{(_a#Fd=X9aA8XSk-}2zBd|0%i9Rzf17$Zc zaN$GPY7K7KnNA>m(P>lb8@26Ffx&9-rJAT6FbzjhP?<>yl zRpTbvH(ZMPjta@2*uU~8cHa64D^)Tb)~n;>OLg)F)nOL%8^4l&W5g`-M#KN0Tb_5@ z84X-{s)4!!4agO0pmHEPfM@^0^vYkXv;GU~r<(9vz@3}ZTKG1CKH)$e%wM4cEfFj?>ci`=127Kt%#YhTzz<-HXvi$05OjZFlwC7rg)-eiYx5!)zJ4_=*IjxJA3l&=Ip9i z;N?0CEZt(k?Jo-yJ+i>RZ+yJT;7YJSO{N9HmsrAXrzQ59P+Oy>CO_E<8SAacY~s$d zrxgao^7s0ZMS7pTJzHx?Ct71^n>8BcZQ!qE15G0v42`pa1#iX54Yo+TX^Ye++Xob=X2$&K~a`*~9a>JzR9{VQ*j$DKmSlOtr^<>Gn_)9B}-s1Ey*_ zz>9k_kE$K;y3PShhB-ogydy?laD-l#$G+Sq$zGlpXZLR^aM-)a?82Z9sMVGz;v+(Ms4vxj)DhdwLOrL z=Yb^pf$v*A=mB^lBi9rCpL(I;8~bvO?AB4&8<*&X2Q^+~jC(_Kpf_p`*Ewj_##H%7dCzUAUn_xuSWXe-&jA4p5zCyBYp@y>4!N+^aPX1B2Du{A@5(E zCO^Il{87BoANnW!G4iQDdj|f9WLIu{H8bHY{IiL?d&ve{%Dec*kpNtyei)X?-{U>( zAj?c1^WkZ0xYx{l_&hi6zj#wmEDS_;Bfpmmg8twjOdTDBRhxsz0|>%mdW09gb8FHu z2>1PYH}?rf1p9GjM}v|5H5ivQ`Mo(=qxtN~l>}oNb;YnbA=t~j_(XwU6+%#A76Mm# zg+rf&;`z%^ykt&%5_96l65JkG7lsw=z3rh7X!(*m>K3j9Ub}kNxIF zp#EqCdQ~Ez{~-dAnh{v%5rG-oxH)+s5^C2Yaf<$*o?;}XFsFBjI^oElQLv!Drz9GU zgaOg;+#HQH^!EN7h{oT8`XJPWa!WSk!7Vzh}T4UjcQ*O7`Bkm;8AJ+OUOfHSsvC+V9#x8K8_+EpN{bkzL}3-cgP204=(aPSr(f4 z*zc1Mv50&GbmrrkWB~$x7C`?`0qP72kTRqYbLZ3hJ70(;KMV2Bunz_b;$dj-fLu@OwL-ixu!ws6du0d0%c7n6F$3<5%1f_*ID(?@9>LRS09Y{KV@j1pTN& z^KY`esOvrMUrj!0H7c05d-t*$CcI+pVhJ(TCx6!SjvuFdHFxr4i%5H{!8QBbsU&ad=b{o^EKul#|?mW>!7It_khDP2bLMg*|V~ zi(*K=L6Te-#m?Zx9n^J>zELYH`8(thi7bC@$aJjD$#IvNz44p$-6rn|!RN~%FgFds3*QiKYlq-#NeFyKQcGIN ztox2o?qr2Rtq;Al6=8V6yo`od800d;pxzvY=o{g9q`+;nr{rk=4M&9@+1laZIG7%e zth{in?g_{3g%N056#<8>5wN}zfrK9s+<%S4-v^N>ATMf1JhScjk??Jbgv$IV{E?@J zwvN8qnJBo^W1El?g@+|Bo6k>vU~6yIFLDZXTJA_(kB}$8IS#Z#tw;x z)1-JrPmPDt+IS3c=azFO`PbR;7%?gVCpRSEHM1`1DhVj$bE7JO84_+c(=#(TLmt%a zL`-{^h{GCWWriim5J5c>5uKm#FPD*{JWnCGjcS~WM*PVRVL&cGa)x5 z3%%>P-$RXQyiyhpG5fwfki6`mEb{!b&{dFyAsy72=%HE2G3&mO58tJ?6tfZkKMzei z8&|EFgKx-28TFzOjX791lseJ8T#VhB3kT{%mY;JWo|KCSdSn-8<>4~5q2267ojuQ8 z=gWB*9-N1&=sesi%ERpv_NAC}A2N#X$kX)66v)(6%I7;GA6Nazd@9Umwm%;a2Nz)c z*aD1NT|ky(0eqkF9ZBD;ShWD&e+yveRe+1^P4(?p$W1i98@Cnmb9W&=e=X$B5V@JI zd}qoQA)a~n@zj$#nREBcFJj+`JI}pE)bxr`(}&MVa-FCrrKuOA`Y(A;rp0LGqrvQq zJl~!7c9QLMyaX$6m0;GL5**Sj!9xRXJR6ok&yo64TM3?YaMwqi85+q_JY7}_)h(qc zxy(JEo27X8wiL?-mSOFIGH#ldk%L6GCOtGw=GwPqaRa)t3>ve`v2X|dvuEYJwab}h zD95qja%iQLW0QCV&JLx9bfE$jDiv7rs{)386_{6Cf%u*Z6bz_DIeYGV*>hK3SBd_Q z=&vbNLWR#7jY?GM(q}WP#6kB;L z_HZv+r=Fdldi;@W!1YlLm~yxQ5+@q)_F@A{?=(Qsya6hn4RBiANEUG;x?&pPAlihn zVog|ixrslgny~q86W$v(LCv;_n}kj5;51=ja1%Sb&CuM_%#Kkro~t%v@vmm`2b$6B z+KdUz&F>rDf>36J>{hfwY;P-~ZnR>rDf|BW+YoWK4TG}VV7sXuBX74uT(cd2blag^ z*^Y`)9q7K?0omJRk-zJ}DAf+gId>qHPsQR+_BY9)GVX*@ODBxSGfQOIh4Df3%|;sqBH<vl7CRw-SQaZwbMfTh7 zgvHiU!lGm;L8Dws*exe5yq_s8tiB~JWRa`hN34(Vdr%)?_j+#TJm**4KEnR?KEg~< z8No(gM!4P}Bg}u(S5WcnE36nHD`+do3dIZh33{jc302Dd1W~7cLeirFLUzjlp>vs> zFlx7)5GgfSP@FMXNVqsyNP93?xbtZ=a#6`58~AeSz9LHEsx!yD;|~mVEmLGx_frG3Ez0Jov$l zaCHQ~QpX;1bsV=-#~Y15xa>_v)ksb3DJMH&j24#7&_a`*7XCVEp}SEF2S#f{X}LCP zHf!VB5p7uf)Pa?o4x+PkFnJ<#Tq|{PcDF8fDRpr=R+sv|F2V-up?Q`b^xgHaZGk>= zzUbplxjyE-GC-`F0q(dNAb+M2*6c8X=`SPPPBVhj0Ao0;W|sZDF(x@1BO%Hdn%Tzm z=9z6DWdf~jrucW#6tj~}VOVZToyQbs2Ad(C&%s${C|PZWB|FU!=4poW*=9hg85TA2 z>l|}T-@uIf33H5j!o0hxIfi{RhlxM$NNEc+?6kn4eHJ)##sW~cz|SeXBkx&4+R+k; zahBNn&k|kKwydvM;r(MPoYb;HEcLC2g;r3Pw8oXc);MisjrZ=>cp+y4v31P1Z?l2^ zNgHhXWy9RA4f@L3B5E!<^owlq>5MJ%FW6$QqAeQLY`J-0hf6!{pm4$t%dPET5@m;* z(RMgq!98W(s)wcQQPR&IdzX_#wc4KlovDTWwTG**Js!K-!!6t%!}IJBT4B!(X9pbk z&jA*Dm{+?%4#P7Cl)ZF-!fOZkjb=7p-Vy$AM8przEmgL(fvG+N_iTjaG*uBLG z5?SO^F)!aS)0z9a)Wr6Yz3{~uW9Y{|-N9W-8y8r5xj;J51&>o)FsZLAQs%f~(K6n} zn_Urih?~jBT@m!n74xiJ5lHX$S|xQdar(97P@P}thJ}i5_^3_}RUI|64*puk9V2Aj zAv44s%G2CozuX;`o5@l*>yG~}xWoFBI~4TY(J$5=dc3WlWw>K=3%%cVcP#87e_^Bt zq83q4`^;S*P44{UdO(F)xj#~#Fz(}tGow85VFGov$)3#Tc;d?${vN$yTN_U--0Fqd z^o1?eyvVoo!W%0uEPl;xOy1?S{9{1JY2iN@=)Xk?a0b88?5=1-}S{fxn~-()863FK!2yIg)>9fN@) zu_zIX#aezwup1hSs)?~!IX4yxcHA(Ij)iaEI9wnH>ZTlXb?m@yK26@!dG34CZ|>5I zgSmYihGZ~n*Eb#>Gvbj(U-^AhJo;qNb7nT)LXtl6hy)m}OhEjZ1PI*48GI)JD{>Q% z*pJ=Ufn+&d;Mc2(Sa>}V|2`zbL_HA-p=8SDP#^1`go*T#KTS@8>|^d0t0rNEdlIyx zk}xtR2`?Ly0C{o`&U3f;BKL_eC1VTy;{p1~uuV$FpQ>cI^0PzNkQ59YnSy6iQ{b>R z1(Oe?AoT?qPW(*pc4sQK-bsbhy;O{)cdTT|E#oTY;b$@Hz9=JfcFL4X!aytGhb00^YpA{U_QJa{KwX-siJUasoOEU2LbOt`(%)mI63|Jdx zAjq3M*wL9dIi8G!b!6sWp|_loNq%A`vu~N0$2|Og-7H8l1D{bty{s?$tBaX;zmW}P zrELDYWh3rwHZ(pl?{1Kd^bqn9^0P6bJR7CV%qdB7xAPd;`T9993gk{s0oe#$Wa^K| z#X2RjUCBatrJ4)-pJeMhVtC5Lb;IgA;(+(JZl6e4DE5%ZcwDB@$Jz+Njea*ou<{^b?Hu&xNh2NvVS z+hRQb!Hv$aVi;u>V=VLWSL=#lF`)$YTT7slSAt?Gdd1Ag&+A``;j>El8HLXs_Eevi z;u+afev)PQdA1B;c4a6nF5}P4GGq=dN9Epf?)sKv%IXSCy;K2L`oI%qDq--FxwqU( z?kZK`%AP7nQxBU=JxtcJ3PL@(3*(t-yUvW86WOinYS8LX1I2B%sQFb3>-<_w{a%O1 zfpypw%R6~dJx0E*CsUc4mtQ?%V(M{kPy_DnYry%h4R}}5Kwqu_b(P#@9?}SLYF^H_ zxKpXngiST%u*x=L;(%s2XErl~+=^AJTXBz`>H3jv2zk(k81FW22e+Y6wjC~?+R1yN zPkN5NXG1r4TYE53wwHSxy~xfL5e97+753_qcY2uq;|X!$Za#a576;I+cpsTH_c7q@ z6TE(`4BZQ_u&j>?F8Y1MiHuKhy8i>W2mi#b6+iJ&`gFJc(y!yqbI8REPixI!xZND{ zwRy)}u)x}%7O*k0fb=>`@q4ceJ|uk^A(*a};WKWobk2wS}QY76x=Tl8CMhlnBeI5XEC<{P<>GuZ)ejhKBe zcYwxNN6h)|2oX(3+?nBowa1*0ciIW!mz>aD;DqB>opJ1%GcKw)W0MJ(1hsiI=^e zcr?`u^U0+6Neym@F8g#HUa*z)h8r{TfhWCj{E|0*t9rx0fcrU_-gv@Z-9Q;1erEE) zx5+;6U+shHbv`)H8=&H^54qXQie>tska@A8^w+!(`ofAV3sc?zA549*$ckT+eetl) z7w6~t;l*`wnm`JQRiz%$51mFS~d)90q*< zujvYhZg)8LGHXA1N(3&hkHEiU%k=RriiMpCd@;9RJ-w^KW@HS{(5(OP{kF@sMw#)6Po$dW0)gxsV{2jZ~dIC&V?;;@R^q3Nf1eA0+V zS8+Uy|8aXqhW)qc379>Xy*SRWL06#PK=i<1#!<*nkSpp_W zCgRz!M5Jv^#MWntaMI^Co>?L)Z4%MtoQT`ri5S8BePB!?X2vJtPDdhE?@c0qg*xI* z-UbhnFyk#VXeLRhbxy(}-U_d?$)B%H!cEa+^pj)O{u#S-b;;z-rC>U9_Tm=Q6)jW9 zbl_L^?MlI)D0V$3{K0s{r|;q^DlFl?ePeOrYTSychGZtQ_C3`|{Y3snV-2SCuf_FqwQ#=AOg(kINyfF1$lyMAT`i0z z*CFo+b-X`y{H#}p@4gYZ2ei>5_N%?xDF<<}i96Nma^)TU1cddIp z@>1(j_pJc~iyIK$)qogg>#dm~yKUEqagt3?Vy~}Lu?gpXHsSSu&FGxp3>)zl>^{;0 zpd0vMcti?ZU;|UD!@-u77(MPR{S9cG-;*yKYE0c4P6U9@NEAiy;_?5zL9 zgxSAva?3AhEC0gqwqFRBRY(8J>e$ZvOl=|A?C*cW_6F~>ygx{*CBvch4{A4RpuIx_ zZ4!TxqWl-ND>ZTCnkFQkX+rn4CZ2!S#1J!0DDW|r)q*fti}?|<+ka@`vV#_TUE0v? zr-P}#bTB)eJJGc|7e}LOS=4No9 zHg>Vt4FBF!6H772&a)2(y_#mL`!UvwZhP4 z>`n49*>8ncXRT22(F(@jt+2<%3Mayt|BA4}>>?`&%~p6LYt3FfHyLMJvj;#P*#>J2 zeQiw+y)}kaTO(_R4a}JVJCbL^ePtU2zOluDU$zi;vBmyi?o{V97q*?e(Jyv*nPG>Z z0`l2M*t5@Lk5LQBXWwj(2h_s;ytl`u@Aj~DwntkIx2gBDhk4Ke;@2IpkeM(s5l8IU z;fSRNxmW$b5x-*`F)iH@E^D12NzJQ#lM|k7c7n2&6N1g0V7|Z^2Nsh#y4e{k4m%?| z$r+EboN;Qi3+hk1V0xMhv@>1cQS5^K&AelIzq+k(#ib*zn106DT+s<`(BAKc0rZQDMBULl*d6+&?kG-m$H+$J!uoh1M2?!(LEf>t9=ypt@I=Iu znSHX*eO3`oa0{gH9v?y z;$G5kYF_$&=nnS7UiL}Dn*6X>lKC()CBn#ZTjj@%>U4jMYxjrq&;ZErww06*!18SY z)QAJ%$NM%-ArP-V1!8MB84;<0*gYc%!{-LU`M)5%+!BOk+sIQt5QM8IxIcY92=ON5 zyEz2GFDVF;H9@dx3c^Wx-nX9w<5@U45|QkmHU%SjatIc$K+z5|Q6h?7NDLEANWuf>b6~PTuE>j~5zD7;#K{y;9hhw=KHz=u%Z4V5`u8?rt zZwiOjm`j7{)J<)2emW=-zPypu4ke=faw0ZApuf%= zIoTo+GU4oD6p@WSo_mus$%$K#gc}087y{g-^(g&`m}A)HGD=Ov9*sX}GhWtaBrB z;KF&=My5e7Dh)gG)39AUow=8E3|^X!$Yb0|vP{Q6KBsu=E=;3dmP;;remc|#WUvQH zj_8F9Zf0fRvTg>dthuGcoY|H8Oi0(-)^M7W0|ANY%N>ADV}yqo{+;&x7_FYGd@qkKD~z^u%%5J0$qP#n7`I0XKf0Y?YhX@Qn3uizsu0%QHB#` zWsqQ(Q}-nM3{T7P=}$Q}G?pW+w;UHm={@hOz>CWji2Tky$KVPKNUgxbTV$_g@Se=6 z#JSJhCStzpeoqz7?5oB;`p11Ws@WHw+&EE?7bxpj?DINb>C=tfkfktzDM8S(lp zFpX$|ad!(8c>^lBwj!jb6~ot4!)j=Q(6=4cciYjD(aw%zJC0VgW1M9NzBF~>Lt7_4 z$FLWd z9W z3iBUXLGhUte7;$6|H%p%!CPgfHD;V;zwx3qB;Hs<_lGrnm@oS=&<2Z#+n{>24RcL4 zkl1O%+_(+C^|8gzL$=t*+}WJFw&;3K|If}AeWPsA)NYG_4qFKS+g99Uhty+s$T?*P zv9oq~aLEprAJ`#J#g2NW9ftQI|LQaE8clm#PGMe+dG|r1$yQe&hl00_Znpz^`Z?ml zGe?}zV8)$#oRz5~ZdyB{f4C!*#hh?ZaKa|uIIpicq5e9(y%_3rbxuec=!^k7oMCW@ z8r^MY$h>q$-79KzpUHsLb%wL4GaAC2F{jBH-cQNJQg*?pbQi3Vb;as~uGoIj6`7Cd z@%?tiKR52ph;n0wch04GW;Bd?ig6l?Z;+!{#E*v(x|^ddSP54mfy(C+42_gz9Ba1gd&sJTvtp#Jq5Ok{7fVdco<4 z7e*cRqWBjW?e7&$d$P35G)+*pe&5DKIFhA{$;`83fr3aXw?#sy{|>c6y{i<`&QGcbt*5_&rnZ3IqI6Ug6K5 zp#iuq`h&cs;sNk8~ z6i9wV5FQ>3!k-gCsQF6%`i~%31q5Nyc=B)AL-UIoVuo8Vndre-oXvi2?gzb_*+nB4 zg71SuU^gO!9e3ogFK1WxX8P*4grJCCd(r4n(baJ*th zbrdy4?bi{I`WAtOe>gk2M_`>K0txNxq?C)qxPg)ENFpouR3zr!jKrQ-k+^LZ32n|t z=>wv0O*INSXQEJeg-muee*G{CJDho3qi~#gRqyU7DAJ#QVr(>ACP$;+6z&gKM&rlY zXdK$X4C|3-OlXb9E}m7~C(7=LLCiUF+b_oOXG9FTpTxlIDcM{G;+&vf%6CLu^?c;G|hzLtZh|n`pgdv=bzONBM{elSd z-iR=qGtz5U5&ndWa6V3i;CKEG<5~RvWO!sIKz<=ThRmd@ zA4j*35v%Au>|M*fVI6md?VOulBw<`_64r4acx`ktR5nmI{Fuyq9X<6v$rvO` zM!rl6bf^<1^W64K!H>ukRB{%It)-4Qip;FTsR;j)3U~KZyi=i0h%_uc$8#wSEf3O= z&sq#io&{p|#8ikejogjO z1rp5PLbm!-32v!Np!!_`AI?Pwxf9$KOMia61T{4h8Kk2u zklozW6_=66akM`<>w_~eOC0K_)I_DykM|(;hHqOh#+Gly#q?-rL zusrlH%R{SDKAv94hc&(QZ}akz(6<0~uL>}6Lm{?qE<}!5A$r^kF;hZ)FrQu8)B{JI zDZ=_EMeG!0uJlI{0^^EsXiG5ywTofSU0vI*5bpRxmSd^ofGnDIA&QPiqD85hu z*=rTpX;gtjrWNRvuY^-0GoLf7;D5dfuisYTy=N6F!^qL9sbaSkxfV;RG5#hqn^SAZ z1g{~dx&|$>wahK?9OCZnX)U6IY9Y3)L)e&l6kV>zf4A!Ki+eWbw0fwWXdnlV9=h3$ zm^Psad9_VgIHwtPn$5^$wo-|^G(*D{#Cx}3_3~CsUEF~qY8|{QI&o!M7j7=;!t^^` zIAqadTf;va>?F zrxmo1TjQ&VHS{LgAZf5IY*yOh&~97Q{j|lp1$G#I%?=0Nb3XWBheu!7Vb@@XCoAl+ z?|?l%+Sy}5f<4A(+Qajd1LPi&gW2PVVeD7wen_2-cXQrYH>_RmhBX7+@oT(09&5Ry zDZ(A91|Im+>4Ctj^uCC_@S=}5p3L&b4&H@_^}Nv#Lp1Ya)7O!eaK#^`PyCT> z=#L5`e;A!)Kk^^?*QsfRx(6`hL5-^{0874+n~={u*cjeL--0moR}e0f+y0tw{rCRC zP*Mm%{*VytSs4QVSNvqgE664UpF|;8%Z%3sg;1}6n8N!e zFbs1yfy{ zJH*AFIj?|790?*HDl8Ib$424lUr~^uPi1TgIZ!oG%o#`H->qaE{6k;**SPWwCu*+DBxi8)m^-(cc`ZX5S&*Nb8Ck|cS z>hWdNqEr%~vois1yeC#=Pl!63|>`369lZZv+t|yx(Vt{KRygk`loI_7a z9(%>A6CtAaM4gPMI=-Rj?xTK056WUP*zf#If=d}SrG_M^%uUA4Ey?hu@5JpxGTKa% zF)KM4W?9L2RmnSrev=1h$z8vaf=B&QF?bo->nl=GO9o@gvs4smr(&fYGg{(Q_T{Ex z)3G$(H|!BhPlK$27!Jx}_85sV-%pGKL1NfMh_N?aOvZy46IZZ5Y@Gxbc}M*HOM)AY z68se=foHu0fvpl)ckuU=cn{ETVzV?Irl*+Ix=jQ#K|Z&c-lu4oa?N~*^?&Htqqn|@Q4!P$eAUYpE>0kNZeUvb~0Ko?e z@XdiNM{nLuwFL+oU5K`$g^-~~ec~fB9S;|wmHaOFf9JFwWbcBKrKWFFjpLFN

    @-%P z?zDt`Y&$EVqgIJ!4=VW{uEf+Im6&2x2{W5YSf*7%qplJbU6mLzfgDMC&`WqHF4|s& zg`rhwNUg$$v?@qSs&I=8gy-w4u_2s2$I;aY7qkC(L=Cjp)L`w+8Vu5@!AR#CIL)lZ zYu=Ibuh+uQq86WdN3PANMfT`A+#X+tRb(A(|4|23N3sq=$(tnaU{G-#vkvvhUs?~X zt@Y%nb3bvk9$gRXVeyfk8?$;mwXVlhr+QSC)ZUvYQ)GR?vuFW1676tqIvXnh=uIgz1@0*s4Tc`$A^Vt~KM!lV+%rTdA7R%s)HL zm{-<};4dxcYtq6@YzwTp^Vr_f0ws4i`p^! zpLR?(C!;;0on1Wbc%IdcDcd^Wyocv}2Le8j(eBcL&qW>BP}Yg%ZDh0e>q5-i}MPaU3QP@+XC^RW337d{833v6#Z8uO7`iYf<#RGc@i&yj#*1qW_$a?h> zE-vgXjMMBbNWSzIT;=)*T=vm%hP<84r zxQ7l9MpY;a>y`}^?A-K=%GSWE;Oj@1}P! zJf?~L|7qfH3r+Of_z8{sKEdkSXH4k+0^7}B@igx%EX=>buJarB6k6!lUk4habYS{{ zJ%X9v+0*$QeWvNc>wq5mCh0+6ULU$+sN?Nq|MC%ikGeT<;6(1?9qMo{}^gv-?TY+{UXF4+ifCBIPH_6y0~zsT=1 zhSM2iJhC^TXTt=&3ryI@Y=ZfNsof1R!)$WeGfm9!DaZ`&v1TxTP9{S!nGBPu-OaUt z^-~L^Pxy_T+tlsse`DT+KbRXyjZVoDrZX(@;ENS}y{zyu)e0qPR>&`~!g*?QaU-p< zk^c0DA30x|T0_Uu8b*_CFlsaPw>|Wy@3)~()&{FS7(xKgof8>fA{wIl_FIBW7Q9 z#Fbn}TR=6IbpAd6U@Aw$Vg|W-vR1t@12nu)By!-Tt~4Oh80YIR2= zy<*C89{8?EZQK=WS#&|PJ;Ej$m-tf5MjbGQjajuHHq9s0LyZf*!%LnV#eXzRQ2gU#T z!t9DKCO`KDLVY1U)(;b>`$2w@A3m0H?^EW7&@t4=7W1s|r|#*GyNc{F+7y6Uy8|F) zK)+YNK>B;g1q%tp$+SSctO!Kb(jW}k8H6gcAn5)Mf`1NkoTWk3+o*dD560cPU|gCJ zf-&@L^)m>8hEWK_^lN#hk(JKwqCn<5uPTPZsedR+xu2=JL{6A;7&ffu?qxT1tPf$R z~_)r#T@AA=k$P^keluwh0Wa0%qeFda2wfTecAmxJ{pm$ zqA_AaG(?*@U+;;=tbNf~d6d3z&fZ!%(RkbzjSgk*WroM#!-yCV1P2dw>Rdcs=`m2x zjlsT|v1k!uA+sYEtm&=>Z!=e!{4DByNgF*fw_R{S}YvM)Blg#iNFMp2@E9I26Eq zDd+ag=6D?LErRMa5%w$*VfJGYvbpcEu@fQDk4%Lk5vtcEpx{LUCX%0^W0`;{g=CNQ zOT^o0iTE%h5%bAWcz!$)J`d^t`k4qdn?y7Pao5wr{SJ>sS0XMdBys1F1V_&Bb5BwS zGfYC9WfF5^$cYKIQ?I3E@ z`y{Yemq6NBf^SX|Z1I#ps!)PyebRA#V>*VOPRGd0>6pfuet~&9o~P3*R-2C73+M;i zk%8R@IKv+%mw_Jh4*Lvz@*Om&yLCOccpxVf)}LTpN;w zK5ki9BhJG3oGj$FW}!)-kNiS5<~+*AiI{BU*JqR4oCEV$Ieiq7-c?FQ5qS!^WthRac_w#5@|xv{cP+=d)^fbkU`F$E z1y<#=r*C#8j=FLFO{wJ1;!1qUuY~)*Rrq|r3M(Iyo1o1+rgIg(WmVx%c@^GKZ@W0G z8snJ1e4Sj49o!EkY+~-xwFY~eYH(?CE%Ll-QLa&kAo+T%)2TM=TjbE~+X zY=s8&EN{T7jXbBBuRKf6sY3%)hc#m9ghsSnY(#(yS?f)Wi0^2G(7y>%)0;5n-zHq$ z-;A$@&6xJT9HmDyZZ|h0W?&0O(xdfY5jkDETCn64z30zc*ag*s_lm99@}m_SpSNLb z7`<8%oLfuVU>w_y+$kLxtu!o56y8(-XYd@{%0*cUbE8k?)pINZ$S#5@rC; z&(Og-Hyy0b*TG}5sBW&-M{2KM+$sIS>9@vsPtVcp{q&tL|AVHze_$W<2d>KOfia?< zmP@}+p*5^F*&zOy4R*e?!D$W75f!$u3$?=~PkU6>+QWD|I|uJOVy~kUeXC9wxz`z@ zc`nR8tpxzrP@;;b1-3Q_HbxTS4qG%2K za$oo&^{X!o&-^>=EI<7kapXUX_nB1bPZ`f-a6i7yB zAZCXLA!uqa4CV(Td2ulQ{KMYbm()o@g3(zPjIP>X?!rQ#xi170e0dLxLvUp-y(QE@ zstcAvCC5Wj8ILi?M2I^f!dBi}SKo=S;R|&POA&thl9doELR*OlYUBZ(QBHsn zJ3=f^v)BGf0;IeX5So;LSJ?>|TAP3cLla@JlKzc7+(lI-BC2l^ti~ka+Z6J{7bTIi zn1rz7Nti)jbG|me{*eSPxn!)TZgGGfj<6}rJ%uM@Okc7NHl)CHR|=+`Ou^WzDae0B zE|7Z)+?7%h%d?dCSCnxo7LW-PH7AYlzBGKemqwNseakP>;G#oDkSnuJqs91Vp%?}` zc|YwD<0m->-%Q1@r+1@+XSuQj59rz0`iXq-S?O?C&kWN^cD`F!dY0v zUD5BeSy=cW3s)XzL02ma(NS5rA(IW~0rVkLgDBsejd^FYVSF(gU8&iao1e`+Qx1k8 zhu&)LeaJGHel-WBcXP1UHwVrmbFp@AE*31vh0n8Gc6{fe{aY@k`q7)5#D0(ZT#Ql9 z!_~ogcrZT?2RG!wkZ09@d5CzOhdY{i_$x3EEA#VUT9F6++B`%y=b?T=K7MRr7wmCr z6$Sa2Rhkd+$^wM!VTOsGk7Kq4_}Ec^_ezDRpF!r&>_W1$3L!dONWWhp9Kxt!kO8!+ zp%B_UkJpob$(_**^CDbMp)aJe2#0Hn@RfJi(v`(fTwRR2HpQ?b^YW8n38YO*Fxa64 zKjL_A#h1XRm^{qt68vJ$X&UdaEM}cXs+2US1xAmp!1bvWxVF85Sq-uaPO+;th@bHl5M@*#t9K<1jiG<}Y9$6g z>N`I>1nm(qSj)YYAtN{*FyG2E#J7cP|dBy-$%$3e^dwe);c^J zTaVIN^$=#)W7}ftGZ*V|>2^K)!0M68T~a3R;{&}LaI1dQ!BcI@g6;&$>{_JOB6TZWNs9 z##Zi@rgnDY?BO2RUhRQ)4tbgdJ;)v+CCD$964vpI<~}LPL`oPXqDQ-eyCo@UVe2Ys zVeu1bA(1EijkGZGqqOiUP+BmJ;QlBh6&A1$=KgP4;YgUQa6Na zqF{MaQFwSwQShXf{7aUiFjG-U@Li%L%-ow&L=H-ViNBK2609Wrv!s`B zpT|e7m+-E#m++{$m+;T<-a^xAGC|Q>n0>0Zu<5VfWGeI(9)Id9Y}(mR(AMoI=(+Y2 zDyIz)>=q9Y%>EuAuJj-w{Lo-wYTjUBx8D$9TksGeDndoL z_-d$NSUObraA}wzw{*CmFzy`lnb)z}{2rc`-ov9I_wm5{8O{uUft_>J*#V`FEyG`; zJNhO3OI~8L{Trpv@&;%^;jAWRrD!7H_eYe+f5edDuiOcL zW9Ii8RFl5pcIG$a&(}hDs}}T*Y2#&wHd1pqzYWqwhqo?XNBqDsMLld%(S!F|JwzVU z!{azTNLT5hVun6;E!M|Mdwt9s`4iUTe!d*j5F*L;5ctZrF z8Nz(A5$ewyLF$eXO70qAP&PHr9wW%kVuo>%F}7Sb#^(_x=r_^?4Yy1X{oMpH{w4@j zGsAy(&EWFW47vek=%iQug0eX@#NCe_TP||{sX5me-O(VwC*yw^xT0(kFvyyYnGVi#9dgMCFUqvp;NFz zOoSB#b!+%*Sws1|H7;(mf#kFesxR6g(%Od1Ya8@+w!sE}8~oj5!+rqz&y}dZ4zxwh zNL&154)V?ba!BXfVcS(ZyuNOSbC2xsInNH?YwTblZIAQH^kvPo$9=Nxhg`Ks7rp71 zo$N6&lr!%{GVcF&z_8yAIO631l{5!T>2yHFJV%6`cEr1jj`;A{5f8K+(bk6^^bt;Q z+~I@;ubl9Sd$O^?PB>A`&t@m2taHYh{p8}GcE&|@XN(DQhFzpHyc4)LQ+7e&U>8`b zu)}be3p%H{;Pf3A?6G3TGS>z1fwh) zk<4etvG=f?9;|9Vs5JUvBTqB^>^qhGae2Bw7A*6}wXOaT4v_PImfrOn{`7eGV|j!> z#zy(0A>E&MK>&WN55W6F0f^BIz@Og%(6Cu4S*1Vi@9wh2i-~b}G@|zK5LwozKEBLpzKbZWwck+!J06N5lPaY^tRX zt1%oo(<3mEd%*e?5g1I(+*6yZ2krr97)Ic8Tm(LF_jf~x#F~?l7-|#=3%^Kgl4Caj zcYN9}qY(0eedXTtr{_lT-}`9hu-K6_Cz^T`I}f?XQ{ek$s9H1@1oJZ~8nMOEFq#)b z_Gt`OFta(+hWdGQ4Ce}VA?C&4b!QAr*T-Vhf3cWj9E*U)SUeiRtmgbU_BqBOejk~q zeA~ovk9URox@K}5{>x#H05$cC((#xzHXg^9#pCOacwAPCM}~SlYG20VNqRimO4*%M z5s$ShA{^f=g2rhPMw}I)(uw@HND&&9=uxM(9(Fqc=RWW~6PAFLk<4@UPQ=ZniO^d{ zuEV-STv1JgrZc~0W^-Tw`KM{jc9H|Xz#ZIVzIFPMi5hiV0@-g8{HIH&r4QbZZ8t@2CHYK6c+z8F28)KxPv;sr@q1IXx4H zSJS&r50)0+M%T46kw*`^nEH93LKf23W?{^MEId1th1LsM>|xHr(%39CR%bEyosBEQ zvTUwDk1E3n>d`9C%Yo4qkmhd9-n9bVp%r-2P=N`(EPvI&oP&a%MFk>|p~WaShlzr4inaO^{yKjO>bLG}5!ask<2-Yg>?a zq6IJITJgxW6^Hk><8f67#x!k?p;bQOr;Y8;eJR7nef5u)!>fhI~pyDQm zNU1^BRt;Oq??Ua_1I(E62v?3g!X@4RV5js15euKg>A-V5xcCBT>Mt-N@C7QHUvLki zj(g27p&|1MyB%IZr2HEF_q@i&)334r)EoR){sy13-y)^qE!rH|Z#eB8T8_WN?4R#2 zDC`~luDs{p=X;EL`~gGNKj7(Wo{5@BKc$Jge`(_MbxpjK{)qL;+_O#kNR8qnrrrOD zn?4`$Am$^s&iq8a-6zbl{DjzuPgqj_34Mjn&^`8^shm*Tv*Ny4dfc3+q^2C}r!Svrrca<+|*g{Q>2E zKX88B54dgk0r`_Z;QHSWEHeE8)6gF%DIm9g2{qJ>deA(i$G$B+j5lW&5s#{s9j#uz3=jy`AF=~GQ`caaI+uHjsJ-2}(qn_!KB2@X2&du=9ARW^lWh$-5~ znxfxzQ%q7bMebcwEPr7NTTN4DnoaRngekn@OmVHo6#8}i>qeMCm|%uctIeo|nBm49 zGcxbZ$QdxlyXofGsB4Zao>*^lDEg3tTVYO*o&{b{wZOP(7MOL^0;4WlActr54GWCt z?0jC+0#AG_uqK+nFS5Yw91FPQS|F~{0?DI(L;c8a47mB5e)8W~V9mMO>o-i9|Fq6y zj-rEoPxIITe&i2UT>L}-9(Cqle~@bZ2ebUx1zyZdXcK#&CR$?fK1<}hv81M8iOaT@ z@JO=6?QC{Ke)fTszyD;3#d7eGva!+ls=7lYCzuMxppDnbb=&#J?e@@yC ziT&*mG|Z0N2s^yuY=1zTJT?Q&5$pXO zakR)0+shoW<1Z)N*KoprTFj#!cShq&b`ZQ~HiKORhf|z!axxhyOPDu(e-3@)2(O4hp#*TD1oEk#ye2hCDE_X-oi|#ngoJPN=?ilyN9j?0W*vmJ;!*+KZ z>T*XWb#-yH?*g}D zp3r7iqf6ZreYL2!yLn<%peJ^VJh33f6Y_j3>>uNW(Cc2fk>Q1xeY|mv`n$teZ#=u~ z4aq}qtWsyMsf{;~=#5YbwRvjs{x#kRn&*S)MLy(T`(S324~nYjPvtuxdM5p-+sS7+ z?2Fc8zKFWv3j=*$INDMJNMZ+q*cVy(zEB~rZN+B#R?qm6pX~?ZKYpmU^26K!KM-nw z4_VAyLv8%w{zDV~|P!{6+*InEL(RT>;qmEPy`80C+V7K)qKWhV~D{_Vt0d z_8}1Q=7E@z$j(%D9lY)cgq2DV=FTLW zek=s<&V*pKeh6lo()$_~LRNbStW!d;YzVsyhK14t5sF_ILlL1BN)99E`j}88a<+fZ zjK%T+Vd%9h42O<~;guEpQhma3E;|gfPIHET9*%!shePE(`EBmu?5YjNmw<5CQs*DS z+1_0>0=T7p&im^IL_=v=G&(u!7gL`wD2m3*@@UMdiH4y;460>f zapOfSF0{lVW(e6ULL5FFh{H02I5cjFXWuFrY~k_jd5(uqFJ>j?iXi?j!l`r-vdTo5 zLe^TGY652OO<*r-0`y!G5cnz)zYG$wL@o*6za^o?j(YfU>IH9-(fK|ZBV5=YFgk^d zVP-v}QeZ8giZNSL+2O!!!>&{;KADPF&Z$ro(lB&g8oy5CJ2(wC^rvbE@at0M9@^93 zGgS<0FEO@-i1BcO1cg4-1vnf3kxxhP$aGx3ngN&o?C{>goxf@(S>2gaszq0BmK&8RJ#%<&Dpy&i_D9o)tD+z-q!J2EE`^jX|TlmCjUk5z&A@c zQ#*V<$QsY~Sfk%h>W9qMO*vr0zE&GNTxN^?GwtwqUk3!NaX@sr1AETdmrKKm57bdn*3mcQ`Pv;5-nrv^j63(r?$|oY10^Tv6-pv=a`UFRH_R zF{a!XE9<$V=;Mbm^ah=L~C^4@06%Gf4%e%kOp^e&373nWu3$7Qk%z z^mr8VU2gFv9;-Fzr+F8TSBCV^(Dxpj5s%Ow>XC{f^cf(+!!aWIVMK6UEQ0Pf5ndh^ zq5VG*P6w02*vXD#p4tfsaM_oDVE+WT^NsG5%xpP*@W(n6aQ;*x3hyM+V?-bPQohHH z_%p(oJ$;EuC`wO)jxsY*`DRZ}rQaqE z^3%ykq9&=dGYuE7q+#QOG{n+Jv%`_y#jR=B$#=WrAu%Ei#W)rt#@l)^TBRjW9wdRv zI0^E1lDTLuL4RinI#VP#E|%cj;dHE_E*bJ~I{Rgqhl)zaQhH+^jLSgeiVV!#L|@FF z40Z`;&`-kdKC=uc$7UdgKKGNA8IV@U#H#h=CNUFbaU&CxKeE?PHxu%|GO^Y(6CXk{ zVIE7?I=f1gsb8*d$ONR=Q!+mbL)K-X;z}0o+Gat=KMNU=S-7`48=o}zPPfa3U12tM zS7pO>EHmaCbMW9F@)yrjr+kuwOD}WK|2KO}{^a1rfm|dV&BdY%WG)V5|1cRzHzrbx zyqZU@33W*Rtkg5h!y>-RHKOydIxY_n%kprE`sAs=eC*H8M|n*?Ce-C)I(_R$mlt6C z+5*@Eiwe0(IBv!moqF+$%JV`F$RpX)`)GLe1@PW)E#nLj&YNWTlm3|!Rmg9$&qk3dHhO8;a;?3n)eX1PIzu4Pn zTaIs`)HEZ?krdD0tK;{mT^?Rjfyuio@bG*EPQRx2lVwkqjsrYg&cE!x>b=QRRx*& zDyXGap-asDL@u*n`BmuL#7>m$)fme2gIO?}+tf6lRpT1<&D-YHXwD%^vW{7?74-k? zs6p~>@|v#KAcKB*QG5-aq}8B0uLcXt$Z{HAiu zJh|P5XQuQMrIF3v+=g`>ZJ0c_9m}Xoe!bO>`nT<{w`fO|O*`r;+woDZ0}p0);Ng)D zSYGRZQcVYj&17HlnofLv+zEfHPE457h4_tKn0BTMmv42ULA{HNt}ZwRcfq`;3wiS0 zxYUn5Fv{IH=GToS)!o>w*n{&ErG$Qhl;D1pp7+aA!Yy|FNd-s=C&o$(jy$(#Nehow zNDDID>G<4|7H+6Z3%(lC!gg(GLBmg4*cd7;=yp=4q|f~?D;Z%kz3qESWrWC;vI4ii z!qdaD!e2*Zg<0ydg7QmQK|NSjI7uCH!U{QI(0VzchuWj=b2*_uyO+=OmKPc}$P4YxffN|Li3^9@blUyt20t^`f`Xc&U#t){c%CuX&1~{&3TUAu!HKZFiCX9QaLv~eZhCEnH#eG$qj90&T+asg0{Lt zk2BqtyY85n?v4Y+?oerS$3iD&9i2Te)7=AAO&+M1^TdHAp6I{K6G6+#4cOy}A1^%F zL*t1{rJfjYimdQYUSvRW)~oVD)gW(--|3CDe2c#2JGB0bHx$2nBbwea$0Ot`sqs_C z2ajFYzbBzK)#Zc3qkQ4H*B32LzPRd3e|V`c_Ez~qeTW~K@O~Ko)ek@I{n+6|RubpH zv(%RcT=R$Mu0Q@w^M~o&0Q{a8fT64SxiWPgv2pcgU%-&V6nZ*2tL&_jNo z?4w3{$8(A!I2)4-evR+R+mYN;M52WArswlWykhQhZ$TucDzImoe(}MRqVSzs(x+qm z^&`%n^oSobi^9dIDC{hyKfEjoXXp=CUK9{1IgdV+3HZ1n z5%<^^?fpFwDWQqjU6cq@&YttU$@I>r&#f~FddyTdu1?0!qx5DUOGfxD&Y-W8aYrK= z4%(bUbCS`oJ{blIDHu341=Gp$4z^6eW*g3)4k?f?Oo0Yx&XB#SkkLv-nNBJq?8)tn zO@(T6Ds)xS&^#%Pew{Q-(Mm%p=S#0PdbsI(dpDhZ(5u9Fzg5h)207n<#0Y2Za$2Dn zCCujg*NRb9C&r34db)>7Fl!ij-}@zSzbauj8~xc15`;3N-y>-ZT^aIeYFa%*0Ec`@OO-oc_6Old|yqZWbb*QzO#L z!hU8fdpnZl8I}d__$+)b%|aygA}3}l=iJD~!rSbSzLO2b*Ua&2F~jecjrve_O2=m7 zk%Zrq&B4kIoJ9}kVEA7-FnF4Svp;fRZJ2}6b~%U($w4)>q_iFM)a}lN&AD9k)6Rvw zCpDzNTx^wQ56~gzDJ}DGu00QLC*`AZW!S;NpEiPd19Xo(_ z6yV0u0%*M?x6_?7YE1!u{xfsAvyfSQzW?7ABH6zXsr1ijClumo?;`XcTm-#&Metf% zgyE`1n5$NV**Zl?bS2Nzy$B9GE5&33$QI+rhGN9o7efdwMuemo68gZ8ZRR}sxCG3i zxC+%TtB?`QY-Juj<<(V~(a65(-kd3?R%70_YBCS2aov(!r^C!vey&0Ppc>fJ)xf>0 z2B&0dAwylP`71wX)nWY?_V*;#A&^JCybjj%wq0jN(h?1*+tvVWYGi+NJ`^(fsY*Zj zrABrx^DraX70u7#O>q0(g#VPAp*gu3y~rp~z1563>S#qBoDbKv;MAoSJpR=REu}Ub zI@E@vXWCG7r48M-ocj*8BkkXIdKTKDG`|DWCU#<8XeXwN*~4>yGu-no_RDpl|Ke`W z58W8*(~UEw-B6p@1GBX~DA?ZvgTH#%G13DmYHv9@JxDU^!H0fQLV*fBW+RwY94#eG zGm{dm=smy1{$?mk3)AmN3+c=#CI(0gV}`KL=-DhM7`&Df(nIBhrv-At#U1j( zw|aSD#4rV66=$}k7ZruJ7)2p}uoC~fm4p+_8j3dc7A%Z=3wLdL3xhg(3u(7~^jy71nk3&qpA%$oc_r_&Fd{;G#7CVKeyj~>L- zwDcyzuA_#103{ec>} z?&%4C5O&HEtu~fOen@R4&I&61$k!ZhjpohPFu2aSEYq5-4;wO+ZO}5DTE{c$BGcG& zxWf+q^eFY3!~U4<_OQHdkNoW~^e8BJ7kN7*xXRJgmNX08V|uJyoJ>IZq`IbBll#8>)`biihRS#!cC-3mPm>RNmiS`&YG2eG@P)@MU&uW1 z#WrVOwA1@zea{bx5B;E&?uXON;TcMkL)pt8Q^xpX;(LEM+WTW*h(CNQ{ppbnfM`nq z+D>rhvLlxwpT8!XqMs5u>wN;zXKWx^X9qHiABd{Ufe29#gj8T4e&hw>LNyueb@cDF z(c>c(gj^oJmP+zoTaCRLFjK9gz#u)2Qq??TNMO{HNg-+=l+iU#Vf;u z(TA+c*M-5jUBO;s`gxR$L$KE(1Ysi1WQieUc!yw3X9(6QhvMb%P*~0g#o?u)78G){%#lomZb^z}^Z4nvnJ{X9pQ@4Fq2UdG|f zQH4XDN2VJr~ zX(U8}(lf&@M`jd8OGS~D9fj3XqVVuQ6qMrG?UEA($&_fEc@Pb4qiD>S5d+6> zF&IQo&lGxj##F{&Xip6Ngt0igB9e)|<)2ARFUbCO@m=v2q|Y39Fc+qi?7Cnh3|Pi*WI&2-a^!m>Vs^(MAy- z^>A<4F9F*%67cg&0@@@A{QFCQb#($>$|k~NO(OG%iAcDZh|#)<2=`|8t~e2MmnX5~ zEeY8tk|2MTTuF82>VkOUldv?8%=MBa&PYk9Y)V3jd@{~lPR22VWOzCxLoty2^;Ic2 z&zyp=F$Hzp8}2`v0uN>t^o>$*lpddy^{G&}kP5kbWMt@bKbVt>^;M}TT$+YE+tT2C zDh<=mrjd)4My6pJ8mF+MaR%qH`JBh@^DswOK30NfTgk!LE64N0g5G3Nj`e$WT(sfGj;bf5^Z%G9d#Kw`8D%yTP-^ zGN{wBm+=ul-(_H1Vg?z+8IYQqiHXxPv3g@B{Un+A&RyXb^-Q?&70 zg|i_|ouv1AvMG1xK%Xb)U=I9^b5PDbp^+7T&z1Q*`h0$8#_laa)OkJ3Vmg{*K%Kf2WpU3$ipZ`wf!*Fpv{8#0pVq-qyZ{=f& zaX$K*=i_21cZiAk&{8SDAKe11Y$R8`H#->zF|%;H5XWeW=k^8=}CFCHN;OqVp+*Tvsf_b|L%@V}umEf=6B@kJ24`^RPzHA9f)0kzD zl)#+(!7j3z6lRv3c1W$be+!)wl6FJF{Fn%t&ty)vX5m*KN(8HPub zd68TO3(jH}xD#AGmkcLn7W5L!u{o(6@AAsgz#ZTKg$k73slfPe6*v%C0TXi7AI_nkd;U6J{Oz?*Q)3+_^K0Je>HP18t~Nb zM1(IZxNS(yEQU-7agPDQS-$~-dC>qNLqt>vJSHk+8i@*dF`~l1i^YUp&&7m$+nBRI zL$<>Uabby;xbP@XT*#FhC@eobQ1Ey&P_XeGC>)C$C~T0B5VW~jDA&u)!Z(A3%SnTU z(-VgXn$|;vGxLWED@%t8hXaQRvnz)QgG(ia{r?RYJZnb?t2{>u-q%M9emBMlgImW5 z#+xSyx$Dm1(YMR!lD~#xDJ6KNKgGNs+{fAU8fM<_v8MYoWS**0SN@7qN#F2E`8z%> z`2p((Klr_&j?C5Sm^$+p%&z{z%7N?|Drn%~DouK^nuwmK1^r0W6PyUlW_81v(m&*{6X@*dpXoTjgM!eC<3cYEJ^Pk97*kyupbra}xnqb2Q zQ!+nI@lwhRnUBm7Oy6;vhy_BFEbukf0y!%zaofxiPuwlBThVf z4$9~4G5U%^8aL z%^v1N4!UCHO;?EBr3ZS?6|2i#Au-(z-cQ`%s^bQ;4mTJ`x+80hJEEk>>*PM4-${4; z`rwW+A?`3qaYuiJJ2b_~<6Pwd$-~TjP)C!XH#(C#n#MR!?!bBC$y6__`Ns=E7rfB_ zlx*@ZWNv!#W-IlAM~@fY(I=ftpY+u~-WdPX8z$D?XteQ$jF&e#x<2r>@IiFD56mQe zp}vY7@_&7?*T5IqWQt8l@J09_KP(yT$84(~`^tV0+wKR=r+(NQ?T63Fe#lzxkI6gz zp?i+`jNATbQt?N&DKp%){!r-lN0bCJ+am&?Cmn!Y`vM?!EdbJd5BaoT1z>@50P4a5 zaQ06CJ^Mfy?G8lI!9bY34#YmAK=ikfK|X-o%=tl(7lPougIoqZ-goxoY9_!Fb7A@6T4=bo+zhbt4$fAA-@sd#^|@7-r_d_$N3Rp&7yWuq*@}?2Be9 zh9K@r2!4GELH#_kG?y@^eIyj2*FzyqrkR>WC?q^X@t}-%-+(aWZ41MNV`1cHg+cjQ z7~Gz7oBLfDa_Q?f>TuJKymIThFx(j$j)>9W7(FK(3#FOo-W?9y3P-CsGar6rmB;fn zhr_lr9Pu0J`yPtGTV?KdGYj%fJp%du5vUp+i9O?(?Oqs39VQYX&m)mfHhGT~b0FT4 z?8I@?PmXsU^B^-cqflZQ1tqU2jEawfk7P8Cu4KM@V>A+(|G2Y18sW@-9Es(h4rDx-qmnFx6`H?9FaX8!@$KTD_U%e3z zl?-~z#qm%alz{4;36M}sfbb{*W1f-CK!3UHV*;R>fJ;9UaK$_U>%Ex&X3w>>l$+p_ z6QMCT5$~iEacWT_4y;c^JN;!HC2D@^iC966uRlGJ*|#Lf4@<&4-jjjL=r8X~g5<6w z^z+@hoP>;9N!b282_JrPE8HOoN$yGL^(Kq0BnhH}lc6Y;jO`1P@lGZgksG)HxHB2w z_9inA&l~i4GPBUUKgoDF$Sg}Xd2LD54WExl!SwYh{F%yZ%dr%!J(+@*k7S%{rl2=I z1$+KWg~CI6)Jmxs{VWwTeN*vgP#T`IC;M(JJF<7upiiG#E;bFlg=rWp!`(mT#~p&x z5nGgwVWJuE9F&3eTQeZ@Fax6Wp)1JoOzGzxIyMuSK~BRL-kYYGnBh*Qr(Y&s(vLnl zJ_|GElGk}63$mxOsI6sj6EX|WRhb#r$inR3_xyV%+m#*5=90JlwHlew<#kWNRL74a$ev?tGYj%*S*3(HX}1 z(D2LWwmd!Q%zRwPCD)-bA88T=+`}%wz7yo8T`OQ0h#B%P1qk|40839Y*8B>vJEVXN z_5#FJ7mx)*KKiRdW^}m|u3Ct7)Bv~XaW~wQ-CF8^%WDd8ySoqz#f$LnI5)x{6d}c% z{&h$Zp2ZeHjr%_DrWB)*Ph?gxrqAY9pk^`NYZv3HcnO@wl%RYZw|(Z7Aa+3sG?uVe zyQu{3)C&*qD#5l7B{=&gH3lC zde@fY%JFjS`mY>L7UlTSUJgZ3ZUstKVAi7w1U#?6Rzot_L#YLpRj_AO3CLGsa4UDf zN7HwftAdk46`mSb!DLu9(j>`%APZt~6`2sjYf!kT1}EifQ1GY*;UC$N&8~s{^jaKS zP>Xu-uB677cA*v(k80tiRf`|K%(PSI3s_wTcY4YlYIU&Hs6&G;wLRN9oV2aSqTqV$ zD4{=G$*$}C2Dq+kKqv3ZSIyM$t~J8pQzLe8k9%A?ecp;DywGgL;^sDdTHB68Z`&bN z)sE0R9XMOxfu;+cuvQ^+J*yMxWu55ct+#z%7Y;~sZ)Yog*MGZk%(RQlF>dfQaf@dL z`=ABnk%{%fz`qx>Km5T~vwqy2BO*L_B_d4zDI%yBaBq;h*{FUoL1EKC!O&Min7v_; zuta^Z(Ct4|nBFr?xOr@ZaBjeBGNoVR&cHX=e(?d-ns^zL0U{r;g})b;S7p#F@Llu-xz$`Wk*=dyxiIjWzLbl@?T#*a1qzufNdoYJthB@5o&3PAEaFfvzLylNskpXuI*~R;3 zr!_RcTBBLV8UqH|z+o5tL}q>)9^268vLWYzyUBCxko+&RyS8@N&y4QnY4-db$sRTI z6F=;<$Kh}G>|;4VWf8k|9~|(*&H+m+9k6&lcaZsHWgIcX`6I53?BSOs?I~ThkW7+>W=xt-jotRFKkv?OQzAGk0xng9VD}8h~ z)O~k@y^|X|)v6*buO=MKu^1xVc4{i;5;8m{&#H2j& zWhJ#O6;JXcJuz=Cy~cxHFuUc2IacIOjr2y}d~d{?u=_UA2M*hOAbQXThmUZF@Td=z zPx`?BmJhbR^}+BDJ~(CLgU}NCjm)~<*v%V>_fl^f8LS1qI8f$`MbyDY?e@d9@AMok zm~oBw<2JoN{EqtLyn;V&Rr%wmBKHNa1Yq0y0381q0DBegve*W|(k%cV*kvo{J#<(@Zs4h}?jA{kXFfhbK2#M1mgY;6dHXMZ3rQxiMHPTT&;K^VR_2q$g? z;f+oZx+R0Le|s<{?g_@_gTdJNl^!HJYlIshk$2A)dXTR)La@{z1mi~2b6mu&;x*hD zyb+35^d49I3B`XC!f8B9^di@khf`Y*hyK6_+#4H#>dg^k$wgq%lL%P6jzErn1iXwQ@HCsZ z(D_IlSBu2BpX7@hL?V%TmwZ7a?8viHSwfaLZ=QJatAdY2;Vb#C^_}drZHk7Nd^Daa zM`I~(oRauxh$ltEqMG}~qsa*;pQ@EzwpQLZVc{{@RUd<)yk}N#ip8>1v8b_$#aQ=P z*i!5IG$jrXn2jq^jDr#H7xhnZ)av6f%ZGOieaP;zI2@OV$Bkj}I6NwzSsZF#)WT}7 zaidr%o(v!UGcu>zP9$LBN!~GM60nr)sSCH+Z@bSs$0h+%b_vi7O28}XV^ew0T=<-b z2U_g1>Cv+^NyHczvSX8(rAtl3-m*ljqi>lyF$sOslkn;k+2Xg!@3@nM%P)E7m?fbj zm^+4fN#y5nyRbb84=0f`K0O(+E0f{3n|c}Vm7ly*c91*%nfrv($e<-acK%>Nu=Vhzfz&9$V?qQ&cV#)D%PhW=L~&K#Wd_PBG-f7W`RQ*E=Hzd zd3+kq4NAv`G2CLAoDTPC>1bJ>j#DSn(f%o&?5T9zAxri_Lpn@bxLvqB1F~m%w>a`8 zWk4w>gIUoGNR7;d`Ke5tW&drYGP890^gE3*ag_IrQZ-p0(pjLR1=XWj*v4C?`YAOv z@}_QxlNmdneYcs}P+rD|YfTr;cGs%(0ugf@m8tN$JDU}$f+n`U#$Qerc(2I%T2*ZvOcn@cl8uP zL9~cFP(>K~AGsbEirC31!h8=hr>cu!B|&D#%wjZbB75p*F&=3Y z1YOL^ZOtx$bbAS!mX&e`xfEA!m2#t@6s>xt5YsP3l}#xwd6i;|WEp>VD8pT&GPIkN z;i@yauR-K@l$0TONI4)|4%dU_uzkRLLy0-LXXWg{m!s2;I#~yIj62JbySV~ixcg%A zq5{VBBHL1Ue{@!0{gg^-NtN(iRf*|GD)Cma60(mf(fERXWC^=-<(0T9S_R_?WWFA( z!sAO-{9U#RZ}^^CS5d>O!mPF`#Ev54mAu!1E2^REynlwfy;5i`)IRP-o9hgqgSMC3Ud5&AuIPlW>vnn(*f@h$i?w*~Ln)r;Rx#?{3ZXlS*d zn11BWsBbBDK!LflLA-w|yE?FD0q-6q_V_$H zA?-~@Ml`o*@;j+*b>aL!T^K+wQpd3i^9#DrIIXn1!+C( z%J$-#c`tYBdvP_P7eo7ck$$ibif-huj-~EJztMJyh;V$lh){4=L@<6PBA9*=5mpD$ zf1Eo&FqR!4ydl@(dJ1(h@+o?DhzT2CiwO@#hzpg=#Dz=O#D(G1zcx9E3;Pnqg-+(b zUQdt^_R2~KZ}Wx<`<4t7UTz&GsAmonB zs{XHNxWio?Y4-DusH4V`ANg!c)DhLC&VJ8N_?`cSxa%6Q$kzb+G||3L3nnUBWE^W@ zo~$!Z^Ebw#*E=L!UWEvxgYr01Sbla19LKk zY=QdemKe9)63u%ok$=Dv z;_obRz||76ftKu`TH!ATD-5SjHsyphR+?M0%VLehgEp{Nw4tA9gR$>z&~0dgM~Tdj zwbWdxxP3>?i#EyI2c6c<(9{e-_L*{~fi~}6<9FWoJfD_V=*t5I{e z@_u}I%oCr~J#it(6ZK=gU^Wd?LeIawe zm;R^Tk0T}fc^|!@= zShpq+n=S<6fnm%`R9ismwxLM?%J%r8HV4QVK}ZA217G?t<7Q7M#8x*7moj! ziCcG&TsMty3>1$-*5D|_EsBEViYWYC%kIVTD9obnHe8Kd1$B0>%%f0iL;a20 z1XEnu$LOW!+7|^&>Td_%P=~W3V<9yf9*xmtv&7)z^cb9&7Xxv!7LLlsz+xX0w`XF| zM;%V~VGLZTzqwR#D_b%a^G5O^7W&HE&89!QiJF^9MJ#udn2XyO2g46>co+#l!i1JpHZv-lNIfGcYD&Ay1RrHoiIB z^&o%YFZMS0=CIRI#x9!SnOywC4Bbt4-l9#p5X8AXFeVQvGuh!_hAwhW9!~QnE&h^+ z4L*5jh|0r}v^->#Fhf_J2kj1aI+&wVTf_Tw9eW+*qN?#mEq+2KDm~l0oP6@_^D(+D zA6gp=m?0@ZSzrOA>k9C&vjC;yWG?6yqPmGY*e!+lFs}&zEaIKIzX+EO7vcQ-BG^$A z`zccl(PP{oxKs>V?&7?+EXH!TVweOJV`OkKmayZ|QCp0E4X65E`e?X=b zb2jl#WoB+mNhxOBE`!vQGTgB%gMMlm`U=Y+R#b-BTgq{i8rUD@a_sp~4vC-TxLr|> z7v!Vfm|B7BGuYjjT>-g$6;OM{+mu>ZH{bEF3bLqqlMbrF$njMuq87IMF?$$axItK2 z1x>$d?5O3YfFg5kS~aNiticO0RM z4K=tdqQE{rZSOK+r$apv4{PSa%< zRF6ML^f0wW54-qA|Nrf!rH|Fg`Y7SEoMV8Xg$8gdHo&C;@p^KW_JRd7quj0M- zuQjshb4n^&W9~I}0937U(aIWE{Mq3TqFy)B1|DN=FlVC;cYJN2eANbaZ8kV^+ZMB* z*uqT1mi!IwAbNAhIoKB4$>(L$&wJDX zB}>@_*y+H#(*a?>9k9%tUHdQx6!9tialoMAj{JG)NbZ#*_Mddbmurp)33Wsi-?A!4 z81HaGDK$N-WO}B}PUs))4Dn;m+y`~Wle^9sdEXhYG`aI^?F_RuE{NR2PQeWq#D8;v zpFO$r!7gxMzdpCv1;(OeWleE~l#DB0u64yaP!GJr-T^&S&n{Ooz1KZ)dZUAnk!@X5n`5 z#tZ!6f$iEJ$Z+?-OX`K2E_)*2jwc$()skmFVWO`mbVl&TgBKJ$yx<=}zE-jqnhW@f zn30q4#>dHI)PMGdCVkbMIB)1I_QCGW|LZ6mH}t_aNA5p+`e1vK4<_VOH{@-%eYP)d zzwpJGH@@`DeKChO-N;N|%q_%=awJBM)>3I zZhxd4^2evE{y6fQUTc;=7WHxKNroGRr+CZV;7zB;EFJa4NZxeXnatNU2H@SkKvZ4~ zgrr6wb~R=%#kPN|y(IJ?=AO!IT_-BVg@Z?WsdLm_fG z6ic6kqFW~vw|(fbhJ~W5pPs8o7;L4(xB<$}`~mJneF&p(!d#s-w+h*%m+=aNHeYjL z7-rF5RUI783=2E7r^B)3V>o8=){}M#hh%U#zo*%;ub>Xu5f1wS5x6B00oCylkYW!4 zha(X8H3G@@5g6BJuoaBy9PTUvP`cD-wr;_%*u` zn{vt3r|){Hk^jy%Zciyhp++$Zhb~3o#jPmpP>Mq6$0+tyqM&ORg$KL^`$MD12V_5& zx8KoL?o^GBffl{js4e7a@%B^DilL_+!`xI1Zac;xusMcX*xaW&9}6q)8J@cri%TzJ zaYmKTHWv2-V$n(O^0|@2chNI96|RR; zA@`L13WHRHC8femlsDk|G(5VH#$HPr3YF5}9g&9XiOldFOvmgq=@@z|9ivs#>4TI zAlv>(7Jfa?!a?0EYzof8n5ZmVj?2RLEc&R$SvXe5e`j+xoKIw9Fgv}%rEKh=XF7m> z>7kd|xc!COhmpJi+4J32nvF#jWD`s0pk{Rr0@vigbUi!2%<-9@%)!RXIqdS2!Npe_ znS-(HSft(0#ebi3;iJPXtNdJ~l;mQ38Sljfc^F}nhls*Fd?+V}xFHYzciEpX%SR{Q zk#Pm^eqVqgb_Fm=#f3$MhsHO$O0BUie<8aF>x z<3(IG>>t;VbD@vJOiY*TY+vyxkM@8&Y#+8Qy~01FcxTu?@qGJ8+}015pP%5oXs3Ircj* zj_QIgbN=Dp+;_gzjsHr!Q6kd^%?M`37XHE7LH*PLM+h^QT|}RZ5-k5#!XeSekh}gE zRc~KIZSH&SuBgCoj0$xY6&&JywO-{5@~pm~)lC()@41nq_Z5fBzag~#8&n^DNA@Un z*mS7l`KsUITicH1JAC11T9Am~u)JmJXWO-l2spdD`fx;{9|=2lv;I z>C~c&vvc+E*q`hsHSXdh>f_l21C#|BprXSNgO(T}Y|U@#lw>F+{Kgf&(`LrFFvSGd zu9)Bg^I0c1nbQ9#ClVFMRCzc3&Z3+D^max^e z#D7+n*pOj~f-zS3#e3#ht`(W4WPWP!UWu@VeUUZ(R9U05!6FPp7^<(0MExt~8UdK*jmlNi)(`Y)F z_sCMRDyTK(?smrHgU%TB%NcF?&hV^sCf~~$3;sAmRMiE6yidG5UC>!TZ*o4l_(iUm zUFV9$(r&0fNzM=XK65UU=@U%dsn!j(^dKL%GQTy-0~?on(DU(t6*F6M?>%tG!UJm( zJYY^ga?nt2;Vkn+{zgx1+UW`H@1B@!!93R+FKkrt!Zvj;>?-ksXcILh5pVV&$mz`V zhSne-)bge3`5^zd4~ACwAVQWreRdx0&ikV8yDx6JQx{5S9^c6imHvKM!;DrynjeCA zH}p36!G4)PbuNFbJjM;2=l<|k^~W1Ce^>?hqbAXxEOmd#*7>7_x=`Lc=C_%ALD9Vub>okbG!HhpKb(7_;dd#89Rmr>_`rbMEUqg*hockAA?)QhspfE7l}#! zk&s9sS2I15xs6El^5_0p$tVooLM>=t6t?kaf82{GEb*c46BmV!`Y0$5kH+E&>_Tpj z#=5i7P`geINGlo=WNR+m9D^BmWBBuqKj+i;-YGcll>*(Q6ogf!kY|>PGgDJxk5q`<|KDwJwnvOFg(qT@{hxq+;?0muAWnwzi)6yZknc4kA zxc)K&RyrBjXGLu&Bm?Cc%=l9a+QsKOE)!pOX5!R=Ocun zS~BSYW#I@j{wo!-$Ux76J2O=cY0UTM^3Ry-uX~maKl5yC3Cc!FNH#Z%vXRfe<(tVl zxcOHO=1tAP5_*+Gs1fON7su{?4m#fCK-QRSkZ5itGgp<{&wXV1T;v=h2jodE5O`*Ulx1i;)PExq|$ORX$0AvBlBSUHV;ar~g{rGbSgoLGnE*KUxxJ$*_otwx$ajP4g{7#Q^J1jE-+oHJGuq1_|eCaOocRl|S${ zp>A}kqXuhE*J6`aEweMVcrmgLb578AyjX|aS9OroCQrk^4r5dQ*LmDV5Aw~Gdc=9x z(_3o5kDU#$yTxqOU1slpH{eZD1LkdRWPYp(ua`H$TfPb9|21I~^Y&)OO>l|jhGcRR z65E@gH;lgHxMp}!XL?iK4BcY(9;dfLa#kxY9c+c7dMoDaZ^NVqZBTsIhF3q@U}DyW z6;^GqKG}}T&F#20t^=1Ibl{0{CoaZyGMn3l%T3Hwt?kBu6W!P>(*xztJ@}~Aizh4l zaG2UtMbIB|!TNFEuOG3^{oJ4z5w@|bxM+rmFzKU+aBC9T_vZ!(uYL{?+~Wrb|0I)# ze^OMa^%51N;zWhNcZ&%xu90s)T3mShL|kYeH&Ez5H%JJ)KSa1SaJaB==_tXg;0omT zDPvabYc!RA!1@8Ih&ZPTt#_*IPN-o*!&j{6{Ra89%(RD)o!P931znoBp{I=pyo*K* z(1n_%E-a}XU0kS-{VDpmdDQ?}j|^aS#1IX;jM!H(;(sH*A%E*POs5%h8^RbSt4-kZ z#RS`3O>n-#1gqzpVqSzPg6+)^8f=C)%viaPvOvrv3!KulfKQnvv>uYT_`({WdGAb2 zvW888HSSbUvl&KSgQN|57TF-3e%ruS`fysd@YS`2TQ5B~_5vk(>`?K?4kq;1?AO_2 zjjlbWg^<;7&H>T>4%ih)rgcmu)9ZlaHyycCM?4I3#42(N%_N-Jv2eyd z-Z3E;oiSF)8Q1Kc$w+p=r8w^S&2Ytv*{<;4OE$SO^Hl1t_$!dvs#I5izFERJ`eQzB zESCl&;HB0JR+S%sb$+|17>c){zi7xERnF#o<6vU%U!TknlU7roJY z%^Tle^7D`0xHE)Vs~z0FmiIvkxeg|#K3LQ5gGVELk^YcO!S}u}r~lQ`=!>UfX{qca_R!4$AG}4(E&%Bj0VwJSz{Zh*!2Cc+EeV9rnLwO< z5{SWbg0RbnOu)h*Gz<$y{b+9c&81JZjhQKOy{4WEhJ{)%&ao@#PW{7wdI-ceboJ^L(9QFa{ zN8#ZP_61KxVfvXUG8m&E9m$u#f5y(BZXUB$lX%0tjE2;wXaqU@j~Tc>BpT+)o zYBI@Wa=67kBmvD*2^dQKBA2ghO9J*EAq(O|0{nCm(3DR$Sy2LtesPyul6u4%=DRm0 zVd{<~T-=|80P`ez+DTYIt>Po^5}_#xr~mM4`DA9DlA(P!8HeMNVVp}oL}fB&^d;l5 zXbL+rDcpunf!pO2v@0<`^_<;8>KE2dDR6I0!G7^nY@C=1^#!T0X5KrKTYha`sW6Np zD@|n~%+PKmECl!xZr$O&}8m4_tL$Z1rhI%poJvbew7o_9vp>zztk&Zy>8HSI@ z5vQIZ)t-)PUFmQom#l@})t~#!QMqT}YhDJkF1%w#XW}sVhx4{)LTpDS45($;MrC4D zYbLT6aD#hC7P2p7A%k93wO$q`8E0XQSr%6LWmgjZxC zab-5Fwq`?WPd3IM%EpXS+0fwg*P*7to5s938<*R%d0SG`I7MIULk`~k&f&gJ4o)WJ zARWwGUC$+hDVM%!E*vd#afkQHNZu>b%!B*x$wQxF9&g(`98>1TAA5?Gb%&%?T~ zJZ>lFVPjPu+UJoYz9Apx+qk#=h8gj<`N+}zUmvk6iTSHy-ZWcy!?2I|>;<>~z89di zu>i_rnDHK0$bC`fzh9I8pu)W+%R(sUQwtfzEY_GJ6iz8Z=))rJh8LmwO%YQ4ipZIz zKGIx-bwjwPG`|?H>3iK)AZPq$F-Gvti7#X~kqom#D@)jWE`gUq2|PVZus^8;<{2f} zx33h^WJ63~AJOhpDP&DcnXM&r+^!TigG+IE7BgA1%aFIcjJ&%t{yjjB+0`=4R4&6? zi!yAtEyK4qem#T@CuC3t$-`Lhz?~HSkhL(zvsERy}S~&D=P6)wi5nV zD!DsdNiB`HP(~$^dHYn3sKUkRRq&l#1@Wy_^nS^NU?xk^z6!1*tMPt9HB@g_W8JbE z*vi&G_I?fKuCGPrKg@uCuZ2x>Ez;R9e7d0y&-jkY)!{AuvVqZcNbIb`9m#rZl&weh zrFux*u7@?B-AC>RiZWyMZv#GLH8A7QfTLoKkXze`bG&yfD;m+&(})I}Cj4F1ghf-D zQ9Y*_OL^POl5NJbL(N!yxtY9E_6l3Mg*~YSOSiO;iORjAD=qjipcOOan5#P6iYez? zu^^!p^(n3B$!|sJ80O2jwIOH+dxrnAU-*?gFz=Axp#wwr_ z%`wbZeeXh1Ah!dBhO1Z1!DW=Fpua>^h;tJa?oSdEmc8P}w4Iocw2GTPcH)BQ+JVB-h=IcUO%lTW zpAy331_>eN{2)P9l^m`1LBcxaA;QG;A;S8aAwtQ&lEMscNn!1m;eu_!Na4|~(SrZ4 zF~Wk&=dhsc0X~m?g1$dbus`G#9!-ADJ+IFYef$NgT{(Mjz|><8K47Y#7|tpUzD7@*R{0Daqxa4XXY zmnZy2mho@w8D@-if-(NRVT@!&6ZFx0+frhJBa=;W&CC>YB29rJem<3qZGoQKD03X2 zWsXV9%`x?k1@p%i=-*_CN&i@)>$fFN?y%xbZUwW`*8D!U#>}ZU*s;nV@Jvk-;C_A$I}j)%(Y)lw8N+`_Ncbzu1u>v9(D6gbs+1{0r?N8FTHob z5ibXf&viiJXlhKhj+mbB$Q&|RS%;i(?xPdpOsO|TIYBhe31ga^a8{99F|N)S=k5%( zMtWY%Eq$TiwO`%^eMeoefV_r!Qx_a^bHR}q7aSkpN-xnBtLC|4>>5|>Uhm2doGTVm zXF9*q4c1%SaL~{VyS&}d66J>R(agc~wXg(sf=@q{D1VE)Dn zrYt`;jBhjC`OUGMQO28_K(F z0rz1-sY`|XLtKqxh+6|DhKCYM#MIS`qA*OmoAa}Bp!jt8M% zDF~}xGWX7VuKnL&43!TiZ#Nin=Yw(NelQMy=UqoH>#uP31nZekqAqoNNC^M8LeRW8 zggZqcs8$WZ#BU*xH|Lhih)^{B6^fVI+;BDwg^6V-#@L6VCo~jUC85YGqkq*-E<`8u z@?y*)4Gx3Gx-e|HME~jzy{f0-_{%gLzs$ptOYf=hBt54q5iqzD0W0cIgQFubV*@*W z%pI-#7>Q%Wk<=Nu-_pVzXEAQKOdxk_dKBbm@+MnIt!Wj#C2CK*4l?t8G73`9qwt;j zQ_4{0idOOIk@svDjlXlEacyD@+W(3H?!>@TmAN7(>QDVKn7BTcyHI3pnKMfi6pN5r z-erGcA;q^)A`W5W$bF`d)P9P&B4*!j>c)`^6^D(~naak+!Cyxb5Eg{|?>$ztZH zJ05i-60mT20;E?YK!&=~)hh`|V9wnmf_hSF0`_#16+1W)Ylo1(MK9?ZJA1wt67l}8MlIxv0_;YR$NO#fC@Wy zswsHxnS#cG6exD4a0il1E^0&Dcz4C^PsMF|M~&xG(c+nkQEB|xz#MybDpYuPH4aaM z1GOU6Wz>nb(n~s;hDL=n1o)?6LozvDWW|b>vYS_)2GfQ##LY@)Pd6PCPNw7A2l5~6 z(lIw69X$!@kj!BAotdHw)#>=SI)nFR1|%*rJ47wXSv`XpI`XRfGGN3U(a9N^C|i(; z0mm}o{VWrKzcVq#K9l^COpNc%L^StZGXCcMMTXb6&spdT%7QI#GKI0()DW||Pe^ac zJ)6v|Y%G|P!!B$NyUNsursYC;33n@RFi(`23#%gLh{$=kF2=0-#ypJKoCh0~JhXV_ zp^LephZT8vAW!PW#KPJS*!R1&ttsDW|vh?sM#}@qxZZuV3Qd|Y5?5N}pMkSU% zs6={UC2xUBOc1HUoRw8*T~&oLYP_9H zhQo{+_)|yf*;Rvm+BK*ytHHFpwP<`?3q74$gf-QoiqEy97ON%e@bB6>C||BazH%Mf zE$g6~M{dK)di;`McKuyF@(oqMdh?-py9r z`_zip&8?WwP0via4F-GLPvYJ^YUl3b@OId(Y{#-qWT{?ghXp%)X8P^8l-G_a z(j6GLq5~HTIuO#!k5@X8%iC)8i!Qi&bs=JEH%{*CMtEO0YI#Eq``Sa#ya$P7HAHUe zh2>c?Qor<)t*fM7I5R0udLDrAwB<)BVpR=%i^JWot8 z3=jO~j|uxzslVvS5NxXT0y%uIzHHpPQSrrg;uBWuYF>(4SP)oMnjwmELw z@_utMM^J=0>f-)?FU10()X+TYEl_vM5}z(wq40qfcHXtd!AxsxX<|RI(;DNtxn1(f z1}AlGxKCq?v)^sG8^LXBPg`7$=6%;?i$$V#?El!|?*Kat3*)W`b54J!+QG2M4hdr& zARKnU(@)H)hdAH?*$*2>Iq`R7XLLSr#>3~%2=#WxUy1zK*Xa490@3E=%UYIz^o7=(O_`^Fa!qOXWm_KSB?}IxN zd@!1K*ypW2=s)ZOxo1A~sD0RV_F*Q|2SzRQ4u|>TtrUI3wZ2$)+838U_+p&8FGlG2 z!orRjrFdU{@A+a=KKK5ZM-rs`@bictJ9~b3@{Ak^bw3Qy^1~QYKgdk+hv!;b9)RB`1JL{_0Q-If;EHYl4kiX*P(}bb&;b}nO|G^(01ATwF>X594{PWx zt|$M2+^-$S0%3ZEeMhZ8v=*`JI4%gvTY2NX=WX{P2wCMpINeL1u|Eh^D}!M#1Vfv* zUK!u|*VONHf+0RF1dn!wAZ8Eu`401zyB2~O&qMH0kIV;a?t+K$<{M7!ZfYp@U!&*v zIuuuU-vwud^7aZtu{eE2b{~zT!cZ$8hRS>7uHFyB8C~X&0>f~h`yT(4gke~D7<=~Q z`>zVelB3}Wwh2d;9oeh);jo}?*EcBwuV+SJ)hhD**GAy^aKBpz3rC%sZN#Wycv(CJMq}_BOb+m@mN(IkLh*sa2U+Y(9{Gd zPp41FyKgjo!~>TS@aH~NA^M*eZsHLlA&Oe z42yXFS#C1C%aZYQRtgdqr{L@ca=sK&PMnq0DJ3R3NvEsr7#te>2)cd1zY zF%{?1Q!&0G6}xLvkzSvQ9lQhG52WGf6KZhg%#zbLJUoou#YyQ{dpI4wmzg7a!q1qD+|(!*=YNo4S6l|950OmEYT^zV&;de9Sg|o zDZsq?0@(Ky(1S0;uYU@$l-qnh{$zKNqcA9}5CPm5-jBnIC(z( zLpkP*E|tLhWeGGVQ%B>T-+voR;Y1%%Z%-+_kCkF^SQ*CDltH?u4DUCRp(>Q4(zF~_ zyz9is{JK(Bj`Qv1I6R^P?>1H-^F#%{{h)U!UkRUkmGI@AXZN8Jfhm>PkX4CSt(8!m zU4^0Rs}Qxn3dL91TMVeet=cMV5a<)$sm4NPmNe)WM(9;TzoVK=!y4?E%`HHBhTf-Y zu>Ecg?uFAY99IkXmRfYJB9H$Ddx?CO%$fVxa$nq+Y_1-9g_G*xa;F|eD)o5Z#(R%< zUfS3OIG=045cLLJv1!0t=LVb%Z-8NQ1AGTGf{iq&|I>)&y!C$FZ^S~wMj*72{P0E` zyxfG`hfOHqJNJYir!}L`su}OSn{i&G1&hzO;HgY2ypFbF*SA)TQE0~&r*^E7>VT(Q z2e$flz-D+SYN(H8tm(ogbLOM0x=?(AS{QTb%0dtBUg<%{;~wtE_F!9ZFD7R6B5_t9 z40iNk_WeFQ%lZSksD8{iCL*XfhzKT0B7(Tw073oZ0HI4-RM1->D%gA%6U=+rDdgs0 z=dpo8aOyxIPeei(B0ES3IXXyqqc%v8nlwbf{EbRWp6M74O@n`KX#Has)?SkpQ#P2SK91cYs24!Thgi8R%C(XDhsGhwuGV~pOGc=Lzd83ZH1@Ltnkc{+ta>Qi2lbKOLkb}%nxgJ zJFKxli@t2AHJ+vLwv@Jk`UxA%yK6&Efem-#ZSc&Q`Lu9;O|SNbm@R*va*KMlEe_7N z#mjZJFh6gLb@y#CL(3N04z}FbvqeR>Eo3F_pr~Sp#;;$=d zC+HV3lRwuP3X7fb>!UL&KRY9vS$q{{@ed_A<5RCQqDHx(fqt?1d}i>^x*++63-=>k z@HW~7(y1=^Q|LlJ!WCO@F_Zt$6(=6MV!MqiHs*2rZX0)RH9>_fCfxl0CK$rgU`9u$LWIYi^@A&s|Pq+m$W1skcB%O0~pXuAg z;|4`i+qSpeV(xlt+qP}nN!uiCj7E)Z+qP{xY2JCy?~nU@ljfY=JvrUyx$bLbKEs`4 ziwg{;T+!9j6>G-20{^CQ&7tnRH@U-Yr3dcPKljbrgUkpI z94qj^Dy6Au;0JmXm~nqIltUbt|N zGw)+B1ibXZJ$mYXfAKVKb&uPEG7mrW-;t2DWnW^;Z zm$S#N#TORy{pj=dgV;yL*jv(aj41(sRAlOmo zK0uAT_D2xHV}sz^Js6)x1jBYlFiu|%#?@QF5IVtFXb=opcH;f04~Dr?2o8>B|JCCV zD5>$&n*3Ym5X{MEXF+2KRCFib4to>U^7UZ!yLofMAOYj`%&heuCdnPLPY21Ou!S_JuD5g2te0^)oG z!l+;CJf(*HHUg9A&s*Xdfs?@z2uX~<8v5=((|0eI5{aDFNcgUb!l&0!7#7G~X9_y8gq;N!$^G!>%p4jG`PgXONsLBF1Lx@0XdIZxdDxZSdT+8o{A2j{E(Rev^xbhr zo;D>GdW&N5=wU2UKhRV6F&2xtC;c53izMpT)(PZ+B*$Vvek^X&OIKh-pFQ>Kap`eL z9TJZ^d@kkz&0Pl&^r|GGyjz~ZsYSad43G4|?#N%a& zNP5K`sBt33GHbcbn=FvxL^L#z1;QD*U@Uw3UL+ybEeVpL+=0e1YnhjXm6FNuna%F3 zWz?sSCqwyCGE5#Mb8kSFnO!ofnvxMUg*(p`De!p4-RN_6TpOmqiCM~doQYR)@7e!$ z8ZJ{~ei+65XL=fxOVfbAY0wbljwR6>*Pf1^iW$%y!QE#xKdB>+qjp^MHWP7?nb>qJ z3**AFaASKmULDJZ&ZTVhm(IbUVL9y8$U)4~92~ij1MS;6kfN_`(W4ydx;Y5^l7o>N zIhd!LL+@!0D%fuoy(|~A4>Aj>MsNJ@T*!3sd+sn}x8y-pmHM!I9^OjlquaiGBy(1E zeVLEpmig$k&Btx}-hLO5+p)I*>Brd-@RS;`V*z#u`s5P}A-jc~ujnGIZYzReuVP3{ zW&d>G9bS~y!wv9UFz&Y(8TQzI`}?Q4^C?OP}3z(y+3PTf;k?|pdV@vec*cLcrwTWzIxkpF8B`vZ>9V!rD@CwD3IBK`1j#@QNY99E!il}J5{ z8kT3Z3xW?(yE^TPf9N%GrIw`{$ajUJ8$4&Yp>VDnR?&N;f7=b_Zq%{pHCjiVO3H}Y z`fLx3%i()qCE1pq%vTTwaeTI43QI+J4xdVO3TJ%A$I3L{LGyBd`U!2$XMR=|+=5SxJ@(uY*{(dNl_QT|2KC7ui zHI52E40~S|-wJ@sZD!x^F)#Hq0LlY`kTfRd{!Qe4??%5AjFIhX1dh*`3eOdDNWZUWXz{H58i$hvD()FccjK!>l)9=oS+O{~O_$@hBXAYV`W( zgySUl3OBy9vt<$Y3VilGs)}SUQ6zgVnKR-Jp>|9ZQpZNYbUr;i+oNzJAPNu4qYzdT z1=$VE7gjoxuWhrV=>5y`Eq)P zi|YA&>!NpuCwf#Ig675{=ASrd)3-C?M;to+$T+Hu!*FJcR@0|*hCZEypY#pe$0N2R z9MChCyqMiAThs7>`682%>5v7#f`p$AWZ3 z^~u1XwHcVPIRiS|GuT_0fwL#b13#UC*S9k;TRj8&?J^)2%RNFJ^F@vH4>xCEWdBS! zIgkrZ&1fw(qgd_>o-=nu4&%%}IqYhoC#RG-^Y$FP9h!?{!*Vfg4tEDHbCIRa zZWdei2HNFfazrjnGjkDCkc-`X?p|J($1bZpj69o%2+cfr73X0 zsrOy>2}ZD=wIUxI+Ve4Z81pcmxw`%^`b zKU0Lcmy0l+JfzDXi%`b>K^qy2aw^5lRuv<1RWX__71QTbjBt8zB>xs;K`xo#B_*i8 zQwk{)dTcUFv9Glh1D2JcXnh&}>XwmrT!vdYWl-!Y!=#?{68EOJcw9OBmy|IKBq^cGbY-L=ECE*D!xvi?iEnG3Eq&Sa@n4*W$qQT0~P@ zYK^W%?zlSmy{bcvF+2Ob>kt!3k8)cbG!^Qxxo15duC2$4YxUS~Kpv7+J;s&RqpZ0e zr}i`;i<;A!yA2qj)qo$m4d|xdKz~95%wILaL8}pUo{iMC>E)4cLbgH^9L6=_9QCJt z+*!yUWp~T*CVaX<4eCx4uBbPmT5IV z?3&@q9ml)DEm*Xy1=m-#V0l{$>Luy_>E4RENv%*nM!o7%E0mcFyWiA`u0`aF-)qBj z?myO>wPA!s8|?eHq5^FT`2so3wEkq*!7EjF^*mI zMs#87cnQ&Kp@eW?Fi$2MJyYBt*|z zcFFMEm!=k`#NA6jNpWb9r0||DDRll}XW(*4arT;|xP6!0^Lvt_njL|$A(Fy9M^ZGk zOA0MA(2ZrJgt>y0NSZ4p#zIO+J(3cKpR+qKSV|NmONmKEQsRPyv{Qzdk>M>Z68)q_H+Bdbwn>YuaWX=Ff{ZYSjOeQ`BfgU#_0yhxg03d$braFG-NcO^@bk(Mlq?cTZuY*h{$VR2DOj zDhtoiD&pTIy~Vcmy@gpwZ*hBjAEEi8k61XquNbv`kjTF`NYsoOBFH!E@Ss z%sccR6VyK8$UIf#NvmT;FAZ$nuL+k|KjD$CjjyB0H!#)1>&5z5!`(@(oe`$7rzZcl zF=8&5z%bYZmfy{A{k}Oex3PETBzc}+Ea(@vfZloXoTiZD^wScqzbrAnrxgaCu)>~0 z)@Xgi%s>52xjNSL)7W6xWzJw-y!T5vVB9fg?gu#H*%C*{wmV`=PbV}FcEa&QCs;8z zzuLwb(SgobAMcEF(_PReT%c~?0=W(sdU$v@zvGG}!`x83p1FAHO}`|_#hgXYPl5-0 zsUHO#^u)*W^#9!SL=3$@x3kC-f5*G?8E&3;w-VP&Hq26ZP#XpTQVuVfzmu|H0H z@@Ef?Kh9?OL`wx@5;dEXOey%y;wz)9h6&(dRYB1X+nbW=$jbLWB49ucYpZ!jKvDhRnhd>te6=Gzcq32 z+Qk0lBXQ)`vSTGV4i~$}Lv1)`sp<4M|I4}RU_8c~$D@`qrBO8;kXG{vTcaqDamV(<}>|#zx zfvrj^`RDAuTb+ulo7lxnzw>*Z-#*k>Qh7L2OUgh1gcRJqBWgdG;I<%J3 zQ?rQ~?9J);d?FnaZl|+vhkxdtjunCF7}1`N%Y8FYGdTkX=(jn$l%03<+=ShtpV=q_ z=S(voNndjiHJ9wA)L(cXeYYtSItMaQ^E?x!@5$%U<4i>zW=d8jwziSy^E3;KerI7& zUKWlVrrvUyo|>j?Y-`TOY?U0`nwSIQ9XZ%}ActIl9Q>fSCW-gdTALg^NXWr7`f8N- z=ED6{F6=($LRLQ)lT6vC!ntXp8#7wWb&q1MdvjGTe$UQ>`hR)Y!aJ+09hp7kNvag& z;l!wX1WzKXXKFq&W^#5~kPqcm>{B_Ik7pP1;SrgSIeGb5I;jAEcuc1jz`vvbnKcC% zF@_nf<%LLGTZsNA3-Rk0J-tBUaB zSTS&>7*|V}=c+E|y|n~Gc;|K8Q-TNYOR&eZ1OpsOm^Wfpd{QZHO)iB3^_amcOVNLI zDOUeiidmQGY5r1*AsVIF$*z@kex+!Qpl;Jqip7#;=)t@49Nv```| zk73p597oU2Eq1O@*EvvKjeQC=s8Ft99=irz3u_QYACKAE8c5u)L9!wH@2KnaudTuJ z)wR@~YEiAh4i=|cjN|O}lH8#I%5~^IrVg)n)nU!kI`*yAAxNVRGUj!VcBo@UwGQPe zbv1Ng9$qc=(B?dL>M^@ozBgd`&j#u`4cOVI z5k3POabRa7hM#7Z>~bTsfQ``PjOJ?GNM2?WUiWXpqIpe-+1iAQdz+|jG-0%B6OR04 z@A86XxNU95qFv4Sl+{e$R5J#&HS-(JlCu+k%wR7WRR)KxJSn zx@~O5)J?5q!L;I)cPj?Qwqj07D@x1A0&in`TvyO7%0g&#{ff31=bZKoxK<3$Ofdrd+d;T&fE zPC`tk2Pu(9HAq6tk!26_P)V_bOwEm}B*m`vl49@H3nwhW?}$X6BA& zONvG%De-oYlo&ooO1$|;N*Jt=5>Bh6#3&mn@yL$7e?IK^3zZT_5~M^Q&Rk1wN{dK# zH8;JK7IvScg{8i<@G_^T$wFF`1W1d`anhn9Ls~p7l@{^}GGgB-8L@hjj2J-8?b>M> zq0ByK#cMKR)^izgew(Z?F66A$%s%HfS&>M;`KH5iVyUK_*lZ;y=EkwlhTiihSGoyh zuWsUkcQ>)Cr@YYS{Nyu7Ud*(U7bBz<#D)J9#FG;W;_)2?;asR7EEg(@JEs+es-dDN z&QKJJGsy+jR1#OTl|+M{l2FoD5{2w_ZtztS3fbMoIQbr;U;my$qqwKoJxoQ&OzbTZ zZ}k>U&-#jY^ZSWOs|JdGRs+TCaf5~Bm!ZPy&IIAP;wVgIu9Gu#9p2(4c76W9S^Wbp zhJD3FM^&=3)iA6~4Q&?RaDwbA)wk-L0W@HntAQvo7Q#C<$nw;LuE9^V9wh%W`WLJg zX<=%2ZS;Cg7N~|c-nwdI=3i~BI-~kr5VH8ewe~b+V<#X!AA3u#LZAlKdMD z)xRMv&8*=`6KEQn;Fy^SepH*FXoD%HyfTGWj44hlo3Z!A3^U)G;R_j{6THnZM#UUU zVFCa97C4?o&gTT`RWqntz50XK{(mszjU~dCS>eC0Rye0;g^c0sBAjiF^f}hlrmb~8l)U?ZLcF#UOFNw*^#rFBP7cl@wCYiz56=h{SGJIvz<`j zO-@s(6GEDups!30=TK)zZFa^R`lhCSa>lC)>Pd}cHw~s{w2FMD5Epz+a>1;27d)1B zh2aS13&(Khx78IP*Xd)w?aG{hD{QP>akA7E9+j@_C30mxllwrtz0?(BSYhq|FV7An&pHPQo~8$76AdSIuX2Q?rMOkM1W!Dl@2ls%ieQFDm%?HP6zKKI9?*ZxrZ;}2tMIp_S@a}&m&HTz>`yFb=+k-MN60PCFr_{q%T zsLS*}eWJFb%1=G^O#kNo(If!>(FY}48GsQoff!9s`{egI^8wl&B zKzOw9Ywjp_o(O^tJ?{5hgK(*Xx=#OKxQ`=uX*M|v8o}t#`8kqV#neLf8&;CL$+N8? z7{57BOOUha`zwUKM3U&J4?be1uYY%&;kC3%_KNMbnLdo?HCBrrp z{%N5|uM0)O^)QTOPI2nDFgz*`L!u=2l7qtWoY_RP>)~YQhGTHQ2pG+dfXR{w*q_q}P)Dkd>`_c0n?5y!fp#LEO zhienCOfC^edM4tjNMw&oBJ6qA?MuY^v+S<1rZ$w6$bRoc?5jwmpM`yi^vWxLC6mc6 z37XW0D#DX6o%+!JR5BQ5B;&`w^h_;HrskOp@g*6FYROQ|NG69n1@=c$FxDgm6|vNZ z;!=>w*?!0{>Ow10VYiOnDbDssx259UMea#&k-6|FmAl1MY|%)?3CmPSTczS5cc)1s z)9`&&8s?o$gUr)3#Hz7V(K!tTZfTH;Ps4UHn2vuYb3rQ|>y6Wqz&YPPC>`giD-}P^ zKtn_ZKB;7)ZDl5g9%erxee|VYGU05N$-NLgR56)AekP7eu`7{#(a&VV?^&CLn0;Az za5f98uVg`0m&{GF6`sw@hWmnSIPJ(rDd+sh@3K+(Ih#B8Y>eQp)Za84ea*75z$Y8= z)S&`YbKs$tgL67$F1Y33FL$I7<8rBA=c17ws};;C4%NtomOa@`1M(nwfjrLJdGLLe zhd%UF*>Sdal+8ybef08!@?o$cA73|;9e;{D(P!i~6;Ye&RRD*61;`p&z>FciRaXme zP^SPk83icnU5F3E*kS#l5YOTZ@guts{?bLbV^21`PZ9J&ig1L!s+_M=Tn$1u)ZpMXS%o|Q@!Ls{g ze!g!(u5Jq&%v<0T!aU;oRy^C>iq>~+Wc#*ZSGRU--qDT`IqmrGH~H=3$YMCyiHNa( z(K44F_`QFz_5EMmp4EkaIg{R9E+N|Kx7zYTLL3a25HGh%iC8cC-d9PB=VPh!{gM^4 z{>lpPJ#u2+DLGMOBPUv2<-}2$ZlcGWZesA&ZsM|gH&MMtUNoD@iy%uy@up{Ykv^=4 zm~-p|vOb z=_^d$e1%)YS46#1#oga(*fRDT7ASqkg45q|?(GlccKv|PLwciz{)GAZpSTkK6MY{1 zf^73IdXluTK|u@E_q1>}QwukDYokL~8}se8$!{SObe9gsR_S1EicsJ)EAR z4=){kT-Vb_pL~6sOx%9>$ws{Yg`@CQXstWr~S2%&<3+UiK0* zY?y40PF-_c&}V;PjyXmbn!{|e1rCLg;m*!vq7@WE=p|ZgO^>b(VtUz-qi%zI`i0J)wn6n*8%T_!&*z*ia&NI$ z$K96x1zX&Wu*Kj?TfX1u@!4#L_jPt?Q?$pgk@k3W-yVB@*kcMkKJP|5Aa$ApdMLrI&}Zm@8+o=3rO+3a2jH#|>lpxuNn7 z^;phfuC;FXvf3S$C*5H{##IZ?21j=B(6e);fJe#$G7CNMiD&Ry4`{5XU;Qh;kMzLO ztDdlX~L6?s+^dSalA7o5j>p=`MqcC6%lX8pgpZ0CRM+_~V5^1Jk; zn|kAr1apN-7r#ESdr-|6e|YwG^FzWg zW)11HX^{1&H`*Tu2GeuC+8-Z|`QzOSe@x;`woR3}LKA<^@cy`!z@K#sfYKcLYk0PA z3Bc4F0rWHm;P-cW&@BSsS`~o(&E%cW4umZk66N&QG(05(le{aR40aJp(o-WFgzNo+ zV74s?F6V>L{)qk>`fB7Af{`&H7!}jlA$^H{^Pj=YS_VVSE*Nhss1Hkr;8LFuG)@b_ zy;&j1&MzP4vz@zZn79kO*9e zjlgEbNK9nj@ZS}Y82CC8-fzf$_!Nl+&XEZ4j6_UsBy@|INo8?m#VKVN3SIZO2%|hsL5uMJ(=<0WqREmUDL;R;!RjH6RZEh&Yt~io=)`>dX~! z*wjBBlUK(><0#LWcnJE?5RYIk|u*{8(Eb+ zH;!?Jy2#z=?L=yii7-*;jx>TS^LnzX#wX!DdF2QHqt3j6jB?c^cp4bn*Mz6X|8eXkS z!zy*|OtsSRCNvF02C{#8Vmj=}tHMe$sm`Qh>;H4AeA40O$6cy_I<8fu<75x!5f@}& z*p&>JJj}o`iwtD_$-wRoav&yW;=IU2&OLVOc(9K;AQQvOGs%R{#PbgF%?D)BLzo2< z@*cKq&qAMfS%~f}P&ka@%$&P|K+FcsX1?#{!85O(~{F2M8c1qeG=fP{+$ zcx_*Rf@Ov9x?Ttky+X8m7oxSc5Cdix;nQYzA86dWwab{REar0#*^R!j?!`)=`26py&S7D+OdFCssA@Q9v z(Wn|M8&g9LcnzBO*D$kGgY}bYF=<^b2Hmekl~ye(b!#DOSc_OcYQHn<@Md=%KAx+? zRf9TQ<)|~uc7)eHBw_qQ6D2I#toDb?o*}2 z?psn~kE)dLHHHn{U`&sfpgHKcV>k7p|)R z!X-=kxR>Z)*a;o#DmuJ7=&(~#7xkQbcB|@R={y72d^UvkdLu|ZH0Dmr7|~nVy}bK3 zQr&(d*u?~rw@jh@ls;+tpTE%C?DNbLgTGrMJK7TM53T60v&JV?Yvj%3jIq!LH!N&W zYHb6rP#gRsZ;NszTezGe2cg&&H;36FnKQ<(TXrxUV2{Nf_Lvbw-a(W-BovvKBR_Na zct?0$bcCEE=ZP9ed}J zohz36x?);}D+;Pz;V_2$f!A)>?C6H0KHT4McQa}fIhgmTHP-Ue-IHBMo+vN%#Mu{K zxWPp?!%MRj`-s~0dMgucuj*`B}=llGM1K;U^zHklpg%kBfSDsPS7@v>@ z_2;f1u4&LetL2AOZ$H@6x8*#|A6qy1m?Q_^~cQyw7*(>6$OaB-5BVq0_ z*y9reSHBpf@GOasL0w4<+z!UV*NWY}*0HEfV)q?AUwh_rKeCJOXFYmcng1Geg8Pn& zc=T#{j-d!&0z=+5N5k zQ(?7-@61!F_;i~ZBXyPP}(vS`@HiJIn4CeAP z;Jh>w#+S%Sa?V7V7yVkzmCj8{RZ~caVL>s)@3CjsBpW5x+0e{q_ilAIen`^CMQ!rbacYqC z&pLmke=9Ku4(T~4D$Ice_YymOauF4m3uom#xC~`?E(>tII_Oztz=&zyhn zh+_D~79%yInEjPk6q$Dx#$z26CN5#AnoaZ}vF#pu;XeoYO;7_lVBJxQo{MAZ% zS7FAVS^wS-%P{Im8IJ!b!?-x+|KrPWvZ)L+rOGjydkk=gVN6Xi`(-&A>F1iHSb@Q@ z6_9SPK*F?2wDaBkd{rd|uCB!EW6Xx#t;7g7=KjMgVfb$q4t=Y_e2pp;bBB=^Qia$E zGM4hIFs_AJu{LrgCFt|*Rt>WO)!4MV8fOy8!%ksOiBt`I2U17;QG;OS!fgN03mj5| zDJeBjF0X<7+**vDUrVjGmVE-X(78~H(7Uzh{-hRqKiO&0Pz&o7?6zA=P4P$_v~Tc} z{;m&S>M*Uc4udAtqyNNuxKcMf=E-;biES7~Z`S%%ZLr$jh8_Fb@E`R+?+0xd zAKr!sa_w-WPb+j2HNf5NkiXoHN1xj9Q>`7+^lSC$(}BDZ9SED)fmGcNBxiS!0p5YT zmGp3R??ln+PE1_aiC3REKj@G}pxcQ<3G`22_>1%>e=*qfFZ%fY#fKwZ9;EOlN8bclENoY zQrsw!6c4H;#egxk&Nmi6T zk`>p!$O>mOS#eiJP8cf5iK%_$g!2SBab$s<7`uYrDb6f~&T?Y$Ki$N8Z8i#LbrV0E zyNSEK<%QyKc`G2@3Uu7RgT)&K(nrmncx(TmYkC3JL6yr^v;%ekGyr1_HgX&*nLDC!kU;iE> z^}oQt`z!vWenni@SDgP>jatz+*!zA*VBvSfcaqgUMjgXa)Dc>w4mTNg-dv!bI7buD zYBlMX{D~P_KXJ|cCsvce;Q0C%cLrLJ7^DS<%UamSeb&At9lScOi;7@fG(Fd2H=iC( z-P31pumQG?GQ{pqLwvbtgpeCX*!|fEFVl_49x}q4xt!&e{Kmn!-{?>_fz22b?DG0w zKCVdG6uO3{C?Ti)*c&q(H86u`wi(t=Gsoiv<_NfH4hI)=*mE~EvbzOd@>tEaz?>}> zsJ}?Z?GO4t{)2e!KNv*bg8eB=?0#U0Q-+o>u(8CT0!s`TX+ zWfsL2S8HtXb%q_{4%k8En;oQ)>scf9>(lZg$A|TIn(ragO!3i22FKG*#dX+MBQ<6pF6T2xnrK1JD&FOK=mLG ztl`{RaM%Mq$X)0#_kg;WCwee@u$8CvuP1g=OB^)L3&o4PAl1_wlH6IvZXnO}pf|!! zkjqZLT;n@$$TgAEF7Jb@<9*Py*9Xmk?Cy*A!Ef@}uU_}X*IT~Oy+c3UBVTlx`@+`G z7rsHhkPq>NY%lio@yt^3Lq&hG+Xwi;W{)4zPts3!&kwPm=(GDyj+UMu4p{l2k0J6w@G}a-c>6Hi@nG(OS%>kJVK8qBW4A{*?rtPQVShMEPK4tk z_gkut?CSFj$I-xWsFAT?^EVtLPehot8H&Nml`)WA9YZ}T1`~MvHDfTCI%9fr4CgO07V5bJlZi!0?^rUfVlkTY z^TwyK=xG*<{oHr$agN2q#8|B4EIlzL7Mr>E>Pg*kY+Ed3$H!sIggEq>!hSw-7rNbx z!?dtCXi#Ik{52jsnWea>9napbcxGhc@wbyXGHQuhlM?8=-6#itXH{hk`5Ljsm? z=H5R%5yQtNB3&mDLCjHTkf|lhEX7uhB=pfu!ZNZHV(Q7zl1j$zLCLs0CmFZ7-}>`U zG8#4}dE-7oeX=+WGIFuV^|iM?G?$GU!RQgZOI5B_w>R8_VcNx zAR?RneUnq6xHA>!cco(4{!~<{lF6=~3UltZWUW&X9+*m>WGcFvQu#Y3YvFSmHu&r!vs-Gy{5O8Dv9{-R?!6dvpf-8ZvMqIRpBc z8O#G_;HGpYN_g_dWI|zMCg$A9gyN4(>@?5BSUdLqHD)3}GKABbf_$sh_TDcJV&kN!ByAXw@g|P8s=ij#?Bz!M| zLSzvda*8nHC^Ht!Rw(Ke!^E%{(%hRRgdB;Rfk>|>#!w;^KxPx0_nf=kgUf_?!_Xz*JI7~ z28?Debn^2C45inuv3Db$ZEZ%6C(U@;NWa|q7MN(ZK+C@c+DfhL1Z{0(LI2m} z8}!aUl^5e{l5jZCT`Wvc5fPbeU|;zHM_OLvBi_Js>RTw~y~W$!@9@gz zJwCqp2xr5ua9*bhiFs;pKB|VuZ)(`2{~fE3t0O-~9ZnxLaOWS+Os{^@2cm_COSNEE zsD)q;ZRF-^W9UvDENIrDXIvMrF6%-iM-NA;^r$E4BYcoPO8?PE?L&Qpmg=LZK_3mA zqc%Ak;I5A$I>QZ-Eo}tn2}ZEXFoMBhV-!mK#-+i(p>6mZ&6`bd_YwI53TEsdHN(RH z%uuz#9I{)@VaSvC${cGZknP@L0sG89Sp3=&`_e5jyulKaFIwU4Ei3fbutLXDYZPCx zhO!BHqB1t97-Iw5B{uW~+hF}V8}^^uK>D!_Iomck$D{wn2F)tAIOlAOw{EtGXtzZV z!S1}Xc6gm{hyKIt5wPAKSC8I&bHg*BlIzBKr_~iD?9`p`* z;^kUT=-PSWMu{gnIR_1s@PgVrFI-*g1=H(Z$o}kw$JFtLxN$y;@M5n7`|ygH#|L@u zg57tYy-^$Bjqzp7XAbj$>NFo*S>;22y${TJ{Iz|sSI-ANVLmvQ;Dek(9~h40Tr`J! zKx1G0b@s(<4`1|<^h4e%vf*#jYxUZX?>}V+{4Z(a6psvq9XX+MKL)brk(qhFK(qt~LOP6G zDfu95>k))wLxS*b7-ylS>@!|QHas(+-wcDG5fB91%pk}VFhgG*gv@2Z_)E6vEoK#b zPxJE(xuVyD5qzJT-xucSnOmrd55{+97CIM(VAu-!te%G8u38A*G5_fs$t*)_2n?8C zNTBZb?QJNm4B2aJ9!maZD4s12!+)E?@OMWTUfm3XA!nnmxG=a%gu`QGILdd2<0mr? z54FOv(}Kr39Mf&XA@36oqu6jv>kP*oW*%1bjDX_62zc+0z&!39+xkSJet0CVPKdVLD&M!|_1;BNBZvn<(fR}uxS zk?b!fhpNYpXdJml2K;OCN6Ddz=iGEIIvTzi%t)+?LB)X>Y`MxVyIV1^f5AD)G=`kG z7+CiscWOZ_mM$h2{%R~c!edcw9E*JBA*LzBL1%Fsd^ra_yd8%n-{UaPKMuVEc+7N($An07;S1ss#eF2G4Yqbmz`;HV=sh9Q~37A`vfMA}jt>nd1Bdq_Ch(}(DxXk_J+#%Ee7tk~HfLRICB#a1Q zhjB&{;uMo%)IAw)%E`<-CX+#(j7W!M*yOU?t|S@i15)tPl#Eg8eNkyC=tFI9|D{yC zxSNW$*JO-p(o6N5T{R)8Fw04WN@XfLjngo3eH!*1NW;++1fnU=iI=2sYg2GE7Kt}gS|BKGnnt?`TuMwb-zMY_R;7vW15ixTlq|! zreA6XyJz%{XW?;97A~~2cSboI%eZsQe31=b=0@t*`9*=+-rK7=c+5P+>^C_uaUhp!NG|$Y=Q78bi#qNfTP5-^xt=5QaGd?*i3Zs*bSmWKnJX|C(!VQDY++HK3n46;TI{qk`$F(2mSjjqYg zhjCFp6ch{4#N#!x0Lk+Tu=_Ig!uthC^(^2Hh#Awe0*vlih$BY}5pkgqr!=^av?)Z6 zE%%ddh1liGeq(0-CpD2p`iQ;8+)uVrFTAg`9;{@( zr4obRSHdy466&p$@E=-*vN2USy{HP+?R(yTi ziY>;i80y%H^#QGTG`tOdQ`(Tcpbd`H2{Wk^j`~BLu(1t~RmgESYDa``JM{e9k=4?U zJ+nKoSE3UIA39MM-wD0MPHa2)7X$MCV#M+;cpUA*RmU#WWzciQ9p4ptuIx{8x5r%N z#8HwWf0CrIVE2u}12V42bN>`0C8W8dTSZ^~mQUQpY0HSskuoANQ$}1jlohqiEGl!y zrZAzKNE$3J?2Hw}&kKs;ppuekI-w*^uIeG?zV0c^Hf)BfWBh#g6pizrA!^riJpcY2$I4!ko$w0w zyk0d&9W>zVuYq4B8kjpz z6BWBPQU6;Lvq$~Jok>4&;UpPaoH0**{t3PBKT*dc>HZU#`-}VGUnuXXMYgpTlK<1f zv<+JDJ*kCKGc7EM)WV4@E%fNE4XLNv^t5W@dX6^Q+O%Q$hk1b2x;T7T7p{+WalTj= zMUA>}JjtHwuX^Zds>kQD9x~hX&_h}uClvK@dZIqG=If*Hc6Jh-*T=;h`uL%(kC)c^ zXmV$MAWk0%8Tyzx(*V24&ktK`0P|x8*#6W2Z_Eu~R&IdnGYqkFr6H!@G=y=jA*x0h zVc{$zIL$ZWtY-v`RYnLuWrT8i+wT4_!mmF@&~rC}d=s+=)QIO88pGL$Jx8|2I2&V( z8`O-G7X3!;=HF;i`wg|U-&k79T!Iw2t@0)qJrJruxCtgdHbG#x3FM@Ukf~0%51|{p3Cf5dS(I57Z!*# zvw%;s1?<`^P&w%jwk`Stza4+@_9*w4FaF??!5_>XXo>xkE$NA~WERR2Gt$_PEn@{) zGP}kPU`FJPZJt?_!EHBM}@COg3z9k;EqvB4U# z(l#)nA1|0Xx$yxTOh048yMPV$e71p_26b~y8?>8p?hm#>IT>Cz8f~!Y4*6V0wm55M zi@vV57@lm4`%COFXDOK|-|e82Xa|oXJMLxeQ89!a*}Lqq^|?I~4D4~h$sTnv_MBaq z*)6a~8RvaZ?mZt2b%5j)2dM0HfW;>V_!>As!PNmN@eb%!%e`o;1J+C?tM-K>q;nim z(&&i0vQB94&JKYoPV6XP_Y-p%#`m4zYU_k29!|*fb%INl6SZGw>>KWkUgN3PPbbG~ zE_<|BJHze^`?EEjnU!)zk%|j8^moBi<}_G*vu#NN&99XR8O4HrGh3-ZMDFYJ{vV=QBUho*|jaSRPaqW~hCTn;j!QLA|?cVq<=YzBT z=_4HMgIb9swG-jg5{l1bC?ojk9UAVhFqdyw~%J*`1_%KL~W&+(_h$bJ`$ zqu+w@k5MoV`v#+xCp>^`ntXO4a8LV1F9a?bA;=vT3Kx1AQpmimIT?z!M($?i!_aR{ z7)EXhL)?`x*sF&j<98UY)`j85_;6VL8;((1!(o3X9Pv-X+1DA4hK_LT=@!9mDEbcR zYZ&bpfox@Ft9joTcZC^``($&yjYPi>k$CbW65Wg0cQBXx*R4_T@Qs2?c@)0$KA|!_ znq5E9ShXw~XPEIgawQrP<iS&9T;vrcypYJDOw?h)M zP06_D!nr>(8CQ$w0W43(P3084ADx1|t5dM>C4GN?QsDl7ES-0F&G#Sn?Y(JBt2K&J zR8hOUPVHIL-l}Tv(OO09y-A3Oh(vZ|@14jVvJ+Xd#TLZ&+`s2}{`g$yd5)N zKll5-U-z9%`O34Xw8G(1qI!F^EypO+VK z=k$H;eD33Bc^~ul_lfGQJ|z>)-^wZ6e_cq`TCrQ?zYCgCg8d|AgEy3lWl&1<5oJt1 zQpVi(Wt{7*8F*GXgY3)c-%k5@JLP;2i^UpL!QjmgmG=@qC0Ch%;g1=;=rLb)eS)`g zx07?9@WYU&3>51mqv|OSHJ8@?^*LAmt>i#urT$&MB`LAGVRhnqY7E*&og3R(pS_)K z&3Dqj_J4eF=szsS?BPatBXu_!@kiZ#>aIFS)HjFtq2pn0)jxvqj3anRdNVGu)a?>JuP+hUQ2WLn##pW~X4x)dI-WBoaiB7c z(@og3(nQ`S6PBMhVV3J<#(G?)anxm6H@?Ck>1w$nt`If-3N<%fAtUVy&W%l}bJ>(w zZ&ObAnrenO!?TAOPyR6D#0E1`YF;CEul#1M%o(L)dDxsM$IX>5HmBDubIiZIPK&|U zDNy$D*P!c|hlw5ENcvkV^-hnpAYhCIt@m5tEl zdRp@RLQBG(Ea{zWNmbLEc(=Go&2MjFG2$jOcHg9Q?@jsRt;qexnxbyj%=^ijF1@Vj z?`KVVnl%HyyM_D0TQo4b#n%mO=rY`fz!5gsnb=_LV8eSy%_Hw@h}0<`rkvymTYedB zOPzDJd@i%ax6+o;FKuc4+E#O)9m!qnxH;C2=y7%g-w-=A%8tK_?D*@E9q;Sd6Q!N$ zzK-@{4cn78+@68c?D1P-Pt)!8?pLuyfEq zs{@-|9T?Hqk>z5DZawWtP?RGPiH>}m;mGDJM~pO|)c!_$)wWKU_s}tO!lBTK^|PEA zzt5S5WzLvMU%c4hHvY|Sb9u0I#fi7EJ11tSW{<&^w>jowC%%{NTcY zp4z)!aG|btuOE|K$oa;V4kMK5+^npCxcC((T&a`qijQ`#R@>dUd|aK_7o;~{a^v&= zv_{PzJ_XVuU%1g+vq$|-?!5Tfo&Wl}B(*uH(tLZ#)^$!IN)?sCV+Za-a^Ln0R?|pwyE(ZN0?f@lsyji(MPU554MzZ@Cvn zO}yzk)|=vq-jvVwCP?#z!@u4P-0Mxd6W%mC@6ARtZ(3G}qc0xZif)BZ={?Yw zhJX5U-P0GF1oc(s`r=TaeXq35E$#e>=;X)#o__2xsJC*L9}5rpfw4NfulcF>K)Pp{ zxT7!pIR8=|-i`fn?d(srIsVi)@#n1O6gTOhKb1)b?IgDTkN|m*1L(alfM53qu+Ckt zy#i@8K9C`Q1TyBYKxS`LSGRP~1zQ4nb3BkAq=){c{_bmyf@u4L7^CBY_-$qog>!=R z|6~x2cL&knNDyw$L9CE2+W2k|MHxXXejY^kuYxgC=HZAOqS>eGB4 z&LN$_@AO&i2-dcgZ?ax+ zM2VFU#fmdg4EBuTK}?jGGs>l2i{_X`G>fH??h1^i>*r|N^@PLE{>WT zrI{X#!_`cE;L&kZmBvxMTs!7ZaTIEg+~;@sF{j33v@4#~%1zj1is|1Vfq3nb%P%Eh zYni|>>7zf27t`vg-d9VcbNxg{d?!70WFmHp6LF7Bq$oz*{#0dGvlFTHO1tJ^@?y?Q zqSg}amM54QAC>7R{yOXGZ|Ud&JGVN!lH!bxmgA+%S_7-)FM1M?1FZ+6Qy0| zYQFhxNe;vR&Edy`IV?BLVW5pN3o$wL{+NUB*j(OE%O%f5v(5u`Rd&l`)8Bc_`B(i( zVR?K}D29HMd^+~b=YzD&SHI<}3oV~;+k70#@;Uow0qe^P_@bhK-P*6VTz{WSVfV#x z602oqAzMugX*|A&E#^h+wl30rw3y%4D35uv7}M${80wZVvVRGKCzY_^uM%n=Ev1W} zat0w~*fi1Jb5A)D`^(AEF4X3i2i!gT05i7-ViQy_<34RW8LO=dj7np(E2_9%=*Aw#MGGnkqM7J^5RrA!_46|3_EMgR90*@ z^SiI$(swIavv@TwU#;Oo(OORUtf$wT^~~|!K*7F^^xe3XKi_V}dBQg4wBO0fSO4+P zqTT9_*~^)WMy%?*5Bu5ss5rEr0EhjUY&<~Lq=Q`g=@5_m9_F~|VUjx>;nJuhxc+dI z^V^TIugNh!Z#l*SyJL)&wskq{7`|1G`K={OJtuFPvpfjdL__dyZ*S&#`RQIrHvhxQ_G>RwXk>?QJ7 z8MAM#F(3Xj#_W{1t1-snJsV4RmZtTK30L}=@ar%WB8Quhw%vq#2TbT3VZxMj6UyJ2 zaA){seh#@z>HW*(wbbl4v>^3rN10(?B8L3gYYez>jb?V&7~^_Pzfaew zQ|CH0>T3pUp;>T;bg$&=jC-Ko?WPu_eQCj7g9XUnmbK49bCm_xEG;E3&OZxv|$)GcqVz?{E+3F_8 z+uh{+9?grEH>EY*r1ITOVrq*q|BV%%U9EW4+lqdJ#Ml^Yh23^5CMnnXU6vJpKDQ#S zp*2%BTjRG=y^~*vJOAY^*7v!^PU&2&G*>oUrdP{bT(_3~^{owum)kIAl?|PZY}hXz z$E3qHINz{Q{@(_#4>rmO+Y&j$7K^2}Y+q@M^%`4NUa-YUjQS-nY&q6e^JE7*zTara z%GY*G9AQt3RrVY^W>4ltdz_8!F=W}(zf?1%_RY^aIv{`}zwUBipQQuUZaQ$$#(}AB z4h->kpi8g=141119qqvTujSF}=*aUfj^g+_%D?N#rA>|$ZgZrgcGknf6&Mw!Up5V%^DX!F9?27R@SI)SJ5ghMINuDc5K5AC2bG}G3=$~%<_OBa*L)^%HDjvy5cd^;riQ3@KJsmF>cLuph2h%JXl){J@H`c5)Zy^>B+n=J=rJr-O{d}#1Hi(=yx&jCVMhL9h>Lk zJ;}?KpYWw8xvjl8KE;a}OT6$i@#5<%UVL#|^J|2cyk1^}zx5)jx;MZ7=uKu%Wn6lB zv!ahTBk?BvH*cyYd1JQNo6|?V`OnRpkN%ol1HEYy=FM7ZW+Oj(bE>rup32i6?&O1C zKh3Jr%)*xYP`F09m~}qP-6b#LZt+fZtfigJcJv`x`q|f|($ebs%9rg+zb|~nZ}R2! zkG>rK#h1bfzC2mrOBcc=Ke&A8d>YOgPjvQD+Z{un0u%b)1atROK{Lj1|Q=TF^oe?sd? zZ~HcYNNI3Ie*}=YB!Dju1z>VMfNtIa6iajKmmGjop0Yay0X(l4sC;1{hYtth927`K zLLg}!gBYk8cV+J&^8X6LBqE5Mf*?A+2%^bb&9gd7YX#FnoRuHH25-CPv3;|&5#KG7!`rdGf*I zL$|Oy;?HTmZ5>Hn<$E2+s|S98X4Nf`xE+qf%0YX6&q(aEG{=^UU#NZmcSEDtGd7Cd zYn2;197P>V^}$<3k?a%24DkyaPl#s8ifB4SXqJ5&O{}`#v)jZlpluArJz}^xAVxeq zvGeU>xU8ABO-77zlQCq`DB#A&xbR#cNf=Ln%da@;*#j3D+hE=^XGS4GwDA?nc07{*nT#f&8@`? zG~6TSl{g~pbFl7_gX3R0?5HL0-d5#wX5>-*yL|Ce3fQ(+UF6XPG+cR~GkIbie^iIk z3hk)BDZ;|6NbHJoY;TqGT)jm77e8QB{sX3*dC2pRVgwI<%(KtW3A~}48aCm)<=QGWj zA)Ut7_J-KcRtz0y&7?!Ocyrf=*!{K~nqnu7!j8?w+OdpuVE0xBW-b-8;E^NSTS}vN z?SxbDZT22_Ve~Z@T;=Uvn(U%%y>tYBSE{GGGPJK7(Vx|a+*sN`KX;}+aVLJH2ZP#r zlG)Fble0YOvsC*B?H1--@=`v{OPv7nW*2$!{-8Iue%?%|B`?x%KE#jlW$-*-W-LLEO-H$QL!lVrn0v{A36LZt9tPtA1hKcW*h1zcg2w{Y&AD(tUJIjR^YpkUl;j z0>Nb&7q2di^a$kyBB(5gVE%S-+mGI%tulRXTO*Y*h$KZG<V?$SujOCaHjEW4UN|x zJAuvm9ensey>8DEaQ%=#>7+!G<|NX4X(BEriL?`2^n#P}^&yFb%Xi#UH#&*Mnerfax~m%R z8J@{pR5wLjQ8G)B#p%SQr!%Bl25lB+ zF!L|{9&gN`_m&K5-^dUnD}zVU%V#{wpyYW5J;fFMc2FjVhi2k4HxuW-GWE}$$>+RG zmKSHzxla~Lwq$8%nMHTwEbi;P3CO}SUOjGIj=1f91N7J*ZUrT(^ukW!@chbLyNMqll_h;_$r_()7WZa`;_B~}i?lHD{4$FSX zpU9%;Xt%&*2B%OM5QP#c5kE{SWDWdPUuCHo5G$n~OtnF8@^J;{QvY zy2$c4`ClHZkLF=xmd9sH-CJ+w;S`!jX?7ldWqIm#RX(tJKCvD0nK3+{nq%@cSLHM7 zg6_25^5m4{llmc_J?#rLgBLJiXaOG5_FwwQH~d&U(l73_TYRG6uDZ|uexG|&)sbO+ zA8+sb{CrQD`xp0l@`Ez=PxamQeGzL%7BS^y5suMCOiL}I&eI|$JS)PXb}_Cxj`fSV zIjC5Fu9!c@7js@-;vQ3r#RD$J%27QUkBXUDyM+36%u_NX%Zb>=H8_~`-O zEi35sMFn)IQ0BOTfJqhjE){#!SPW9H3ijkxV9`Ttqsd~6ifdGF^+Sf~Ufwz8A=XBZ z`2OM}G0q-g{!khIMvvvcc#OCBMsue>=I)Zm95Z^%#Ql$z8F$~R{Alq;pH`Cix)Q(oFG-#ClJeazvAO$_w49gh`RWyu zyT4*euUCxi^Ge<`YUuoa>ni_wkWrny~V7}TQ;40%fYK}Ip^^IeMA28SKoQd+SIqaek9iD3o+q4y%Q(*9Z&neBW8#E zLkHh6D&-vypS;7j+k5mTu?1?Ae^ThAHcIfgI1;6L3NAG#lMlAS=A1IjrLH^SZ zw6*y_Zq5fH+JB_qwvW^_`G{q~M>1=FVn)4BH1G0>yxyNUE8mgDrcaDK^ojj8pLi1R z3EP*Sn6l$D0nVS<^x`v{KYyk}^J<2*?W-AbrdBiLiA8#Gqu2#!#UQn+X1MyWn&D<` zew60?~Ftx71e0E*K z=zr=OuAZxF=-i>6!Qra-Q_=Md$%XX{y$01c1pZpz;61m#p}%s3H?Gw;JnP-SaI{N9 zL(Hy*hP_1%4Zl=0G+b!Z$gpC3Bg5vIjSSr;G&Xd4)z}c&wTYp4SQA5^;Y|##TbdXe ziL)@eelvq#*Jg%UeVQ49qMI2)>NYoYE^KaiYSzMV_Meu9z-uiHUVbeNCGM>ZPL5v~ znoVzQ;HNf*;VmY!cJ;?4yN9b>%F2NN=-sgC9*m*so9j8%s#EE;u%Ia5tJ7iY@PW6dZJxu(tp z?bgPcbHn;NYwWL+FI^;C`>~0smg))6o@~rbaaC^8>+((BxZLEQ;+r(PZcWL~TU^{~ z!}$Mf=zYeP-qr1BG{g?B+hK50KTm``gB!J4>Q8>=$j(YfKD~G3 zNKN@*CO8o@+etdP6P793JJoS!cS~pW)jD%>k2BXDu|(xldFkiHwL0ExmPaKp$eXrn)v5f*hyNO>LwS@h z7j!4Cy5P&JaejOk=f~S}KZaNOVe_^6fzJCAI5B`B4gnmy6Cl35xKcF(x$tKotJei$ zZyQLUT_AZ$f%It_L`l~m2B(QJl^(>sydb`R7bIOGmMsjrwoa+;r#;1;THPv)TJoVc z8y?1lKg7Ea57VrzJ8E$l7vx3%y+t^~hiZ=!5zeYe@jg?+3F;QX`27(CT#exT^$7jH zc!xf^bB^hAhojT)5axPEd!0M7zLQQax?*lKY>CrrC%Xrmc+OMU?g-=`qa7ixG<~mN(s$sgr-a zrE+yXM`JO%7^^%>EM6(94AaQ$6CbkO!I7mM)5eF z(Y-P@UW_316l<2f`fmd7jm4!9FaJqF0*h-W65U=*sNwQ&%u1x!a^>hYCn}#RR;Ono zTY@#;eoW-(H2K1hCJ}l;cT7|5J>10Q%u2%XMUwhr@8YJ}w#}BiIBdJirek-hc|-S3 z4|zK(?(*{WU5Z;K6WK%D{7LFvo0H72f0F66Gg&@abq-HT!S#F!4xTBD_)?iTdA{FH zOC?A%?S>tx46;bYIysf-25DSvlE$7PX(WtJ3w>f%=E)r4#v;Jm2zy zSMQdN)4+7LPfI6Q=g9ta_86y=t-I>(t{E)tskyed82W?MyY^cKGx~kqa|4dG%=pI`ozsRRdQif&`I#VpD)5_0TYM%9{l&XjlaJTkeC-0|L2pz*osI=e?xB0R?&dC23RpLzfJWkP zHojdz503&W1N3Je6tMDzI*nS~C+S<=*){9_c;r5@IrsVH$$cKbyU)04g`BBT$WOHj zQXGvNwNB3i|ISLm_aec9M?IVQOu0uVtSPn zGlmj+k1gTRT;=cnE#bz_650fo&{^~G*@2}@7_Iqudnx_pMW3cDULQ-%%B7`Tc_7Z` z(^C99m0{#q#<-|5@-oX9ohx3y_*J$Y%K5!}In$+0g!z|ay!`>e@eepxsyX=40}fX{ zAXHq>%kLg=qDBR2|5PY5T0x?71v8&juyfi&%!40_gCUMpw@3We;}Ls%K4QVvN7QkB zM9}j`d~ErcSN$GiJM}R;B|{u^;TtG%Ff_zNyhe?hmEFIYWJTFJ^v64q8ScYP&YFICb!q>{4*m0WyN ziDB?drVZ7Mdj2JSTf8Dkv+210udtl-iVw41Q8f1zal2p9*Yg#brLXYr@LC?`*EHYt zTD>Z-d3*RZ(*iY{roZN5uPU1URYjoA->a*{J*=W_n>Q@m`-U;TZ)hL)hOYPCGNaEs zR`q?ST<|---|~)E|B7RE<{gVJyj+x#=Zcg3Qp_{_FfpLsmEnqjc|g6jWK&G5bYg03A@ zXV5iu2ib~urLLfP--~TEwYuT@g6f8Vfa-?)=<0@XV$t_&Tf;DAYz;%3#Wf7Mel-l$ zs%jWMsk3=p(^`h}hFXT7uGKOeaIR$tn=gLE!PRdL_Rn8VTO{?Of6FTsX3l?6=G2f^%5KSZ@oX*dm?JjnS_@8pbAv-2Eg5q_`NSiZ zEWK^1t_e%tw!XVjGi`3?t1KZiKw37`N`q(fw#)kAUw%EvT-RY_=EkbOWnP5v^Q#-sS z*wJ>X9Y4>rBW9r;KX`}};iI`R)sETPPjwt_&vNagZoai=TMO-)j2svr<-qF%2kC!~ zZ2U>=Fw}?qm-zArG)vxexFsd#9eyoOMUNjrq0P)Y6Wr?^AUp*L16j&(g)y;7aWW*!vT zd(gnkgKss5E*EEF^#kpbD&(s+crru#BD)2i{Jzwa?a7{uO!ee;x+k5-ddc(S#cNA1 zCWR|=saf=uc0$hbHTB=;P1YH2R^@9q^vIjJ9mGl<;X}wo?RDf|a`;b7)p#F9rTWmQ zwJ*=L%UPgq%lZR-sWZ%%--r8Z-crx;EMFQd^JVB7eYVz@edfO4>&wr6zFf)kRhOI} z*+2SWm?9R%Mn87>Y1i}FkD5RD^RmA`2UjVhxyPT;PX45i3n0@gfM|K4D<1{Wu}>hj z1J#T9cOZXU4x}_SkhhNlne-x%k`MAq%WE{rF$goAh>##UHIc9RpnT1TN1REhe172+P!>jDaNXHFX5q)+W}5afCCXp6k0I}e7#5FqBuH8$mSQfO3WkdT|<%=~x_lczkv2@%iO<3ovo3SLl6{|wNqU`B$)SDm2 zHz(q7F^^MkjC@2sab(F`bmC(it=h#CJ5oJV|HRYazj&;y;u#$f&);{oyGe_u#yjyV z`iOtOJc0jKXkW7_foGc&7?_$s|JI58{6iwgyDEb@IFTk3#jg+(f3Q&^dvuKb#DbH4 z>{3l^w@yiVE^EFXF3s37NjhW_8)LN3seP9o;!{KzYp!;=%dVumyt;drYSohI&`kR9 z%4BY@5|dTFX)EVsItC{5uY5)WyURE2oPyc&6kfd+-@c`ClS?M~^fw zAHH>ug<;Z#l=H5s^x6Rr;L-z{R6O9rzzPb|#cON(P~X)Lv6%9ZEi)dH zWc-ldZa(D2sz=gg9<#wxKJwznL_86T?Yk$`ke}So)&ge4E2$Q%EZug#zF z>$T_n>w@2S`_Ut1qPP$#Y+J@i9)HeLKvyS0od0j)G*+fEzFIBE? z9fSVd#j-hjc@)aMi{{y5Fa&8Nj>I>YDt zXZZB+3})tMnb`FlM~%+$+u8F3_r5^It_zIae}T=HFHquok@xQ|a`Br>Y>T}_--1hg z@vSil1;+aCZ$kbA6N>knFu~7+>#yW#`dL2pEtk2lSAL~oS4en#g~U0gc)OeOMW?Hn z%YU>u)QrxlnztHX6DR2!D_WT2q8Tg8(;TA}*D1fF+3MbPwp6nqv5y5Y%PnY-U_q~x z8*GrzJaDxoC&%3+Zt+c)q~GMk8+nB~TCsbz6-O@02lUuVy*$>K7g@9IgSc8m)^TOurbtu%wh+IHgaT@ygKIxNq3v-$XRi<7GyeNUfY?-fzB*j z;*5vRacOOx+?;WK;>?=bxB0t?m=R)0Y@B(Uuk3Ei8+BV<4KB1D>cZoZF09?*!fk68 z62e^Q8ttOFMU42yuJ|R0(OT(>ZEZJPzjC8#8#hjVE0#oe^=<#^M*4qlOuXu*%)UDh zzIJD?*sV*axU*36Q1O0u_8oM`JIo!QdLFd<*@IW}#B(+AU|^;Pp5MzG^MfbuCZ0H3 zd6J?tFxXSGil@4LrMXFe>wip~31cq?T6po<-;3pGUd;SiS+Tz~?=1HwL0PfU+Vfs@ z6m#pdw|aoo<@37_{OQBV-9EC7`7p;p94^f{O?Uc|wa1rd*1q`J`tl~ym#;JA)6Vkc zXs$0GzwyJdhadA+OT*ja$4d`CKJ@e_c)dSE|MBPfDdoeg{7Kb}v*w{cM_&8$tX6=y zRRPKZ%Y#x9z<^Hy6m$=yz5Lco!o}f||0JnAkcO|7`)U~O}*vyv*m%7MNSVwFZPXGR1A7DmuzO$4d(U;kkdL9sYq zhSCV8ypP~g`#X4cx`R{SJNWLpL%BRGTdm}`c2jn&W+ZzXMarub$!48hyCX?7iDX1f zBo%q`u{@LyyNj}5j#2E(R#*9xDCO#+2<)LApo!6p*37hfu6lnqX;!)rO~ggbN1B_e zE=My#Gt;G)>MwsCjdcfkuRF!Cet3*>Q883qieZ0n411+N8ni#3Ed8;CcIczJ$6_)t zmT#v?i<}!v?DklC?~xXHA(m`gbqM+CHBx@<;#hI0;`n-S9Kkc=#5#}TptQ&&lQ_;? zl~>y`jzF6@@`BXw85Ku2%~X5G#j|QfJPV~y`k9GmZxPQkKe6oPd&!YkyWfj=vNThz zY@9%?9tjj3N#KaQ+k0;&(9=i#o-vxMk`mY{4KwR=0_*!H@>YFAr+!bQtMX`xhZ8Y! zPGnM4BIY?_-c zyFJZm*HW49ol3ZWDxWh_3F($bn+0jaY96yqPov8h>B=*uYbI5;%`u%}b=0NaNV8S{ z46%4J_-=Cs4^7kqdRg9ZX_~$j>RYd$Nxd$aj2fLu-WYYPkI$rHs(!Z5#3C zou5f*Ste1hGs&o(MevX;YOKhj&)zIr9Li#V_WA8@Wzo_vi=uG#g{JDS%R^rLM*g0! zvbp?oHizeBvtN7tU9Q>O*Y5u1U-zi)a1YC%dkmZ<9`EcNF*0(w-z%51GyWgD>rJ0L zvNz=M!|pu7_vG>TU>@cV#JBI8kGIaR@{EuCE1&NJ^68YF&zyYi-y0N=@y>SA#< z)L#9k`}mBz&y5xL*{;2Mf9=&<-M`O*SK@OuFO-j|kf3LUoRWSxH?)W-dB``K6jRr% z7=vpuZM=%v`KFj7+L`ZfSR!9b2|xRm;u={>?}w!XN$cC9`Kr;VGHPE|mydOsI;Yij z?pDr;lyZ(|mow~JF>vQT5bsBQoNpd*q<;k&5fzxesNn4D3P!m-q(P=wx0Mh1wVJwl zx;*05&*E%Nd&I^SkC?gf5mPtm{Zn;mj@NG6=`qc-9&4}mgl;FENQ-z%w*$}Qu~SCR zq#)v@ILWW@yz+|L4PO)A`879s zzviX(#l~M%an!m>Jswq9ys0AmlXk;$U)eIy1S2uhdqW;F98iodS>KH2S)io?l z?oRtB3;6QZ8tu#0WBTVt(qcFAX3{1m+5f}V=i9mXvk|Raji?AU;_nFM2@8yv`q79V z7Vc;F(*5+v+Ry5H2Uz#lL1LdCWZc9_0;440+x?w3mE;l%^k#()!zD9PW4=D`gXpe{(`w<_U&& zKgspkCpmiTB*k@4vHjOmyf-?Q4wpGH^fF8TzD)RP z>2jMdvqag(;cqX~uk965#B+Exx?vqMqcOOv+KNkf1S7T zy!$S&p!q5baqV(+V-q7X~M(k-czWyv{lVLg5j)1l{0)^EE>p8&C8>sgWiixoSUSrM|vO1+6z zEON8rUby%UGpzYzqcvSktf^&gO^k>9?#fPnH}Dp9Cf_1v&Mm&3e~W^xw}{_ zTt|+TI8yzEBR^F+vgVy5hZZ^Ua)}c+mWuKAuM<~y>3ub4(rRihZ{2jO+$(rNm-6nMTZAP8G&1jR`%nwk8 zGfZ=R_-#%+Ql_(p3$;3m7176qkYO&;fn2z>)P*y9T=ZS%Lc~=U)?Am*Qkv&idtB*v zz?H7YT&ZpCiix)?4-#ET$dP~kCpUV|(7t`P8z#H7b3f}wJ>@tD_^G3z)Q!DWZanPh z&ikL;`7}To&Li%u_H(C4j62U$-LcGb=V^sIw?Dd5v$h9k=6Fz|+23)U2hMvu$UUO@ z-&NXYum{s3J=m%HK!Zmf`Y!aqAn*LOKRhv)w{x76CsW+yrFZuvTBk{gCn1&Uf#~E# zmu_A(pWwy!V$s#{@?x9LW!(`54EEN}TnxI+-gGzeCR7^fl_v6lexrVf_C7q)IX%Y* z$HhJbZ1iEJsSoRQPw=!CL!wCipA|m*)!mnplVa65sbeZwe$fbDEDC)&EuZ`9`hLiW zK+;t|;zPyf4fkWhYd_q+l=k_RKmFt-HD9X^d+DJbI+@=7WCZ)u;JrWHJBl08Q~jTl z0yv}NFhx44&ZE!(<$waP&>dp_ivTvg4UnHckO@BqvTH;jEys!{F*}f7<^?iqMIeh0 z1@bsFQ0#V{`ax8-3u2CM5ZwZUsFo+jZ|7jX>Kn}Yb-_&77A)p*Fgp{2+44;Yb3?)|Qq3#TASBElqO(;8! zLb+)h%KO`)IAw*>s8Bj;voPd9BV&rV5(~pvygH0nvoL~cg|luzI4}PQ=l#NPOm>Hh zeW_e$h<=VzPBS^2whwe)cpOgHcM&w79KnVq5yY>Mj;cF@jj{5Zt`XFCQx`_HI~1J0 z!>Xt|bS@JAw@N+(-4DK$*T67e42i!Y8S5R%@c2khl}2LmA(CcaL@{Vo6pbfGu}5dj ze|o(Vh0m=hHaV-uO1^{ameFkaGMfAu(ahT%O~doiyfBaEE2n5SMC-l~A5Hz-Xl4$J zp=3r3_8~F+7%irRvYFfF#Byk@SQ6)A_4kgYeuBL1nc_?67<&#(*QGkE+9h&cJc*E_>c8l$K8r!pKxZT| z@ZTgh8zreHB8d*_r|Msr#1PHpeQ%4wE1mOkMKZmYr?6Nj^`d$_jZ<(dOksPIRB=;M z$8bM3$s^k`jb{VX7`-Hotv{w?GC!TKlB8dLN@rYm>62IFZM~MkPD{Nv z(_HK*KWjoJF}<@GwjhhsCRsH3QoR&@+0>Osb>qN$d|{y-e#kwRe4WFCA9Lt*F-Lv? z?b=)9Gvr}Dt9lh+)J!b3USc@(*B<*{?X({hs;9k(pAX5``KpK^HH+yvx0s`U71Kv^ zsCB3`z~o}id{FP?fl_{MP)5krGWPW^r{TMDn$M}AT4DveVjr>awLFH^E5(zp6i2zX z!8&3gDchIosqi;*ivH8H--v4s4pQ&kVR>PXvZd8=I@%uRtD@uV+H-=vekb`PKUU}L z(_*}wVOZK3T!)>-D&{Q1KAq*D>v>Msxx_+)cv5RGaXRc0EwmFnC7!|&b7OYZGa<37 z34e^c%y#Kgi+#m&Z(+*!>rC;uZc5U;tMp%YRo@F&S##kk*K@D(=EYSW_BA8rUo%YK znPEKh8hdVDV|wZ}>W?<3-E4DA#$3mE(sd@fUMD~1I=9AIC{ty@r%(&N&bMIfX-f{q zTVhn}CL^>5{MjJ(yEH4GjWsyo2$C`pk)~sD*tv!;u3R~IodVnpz4;AY`?4@Ak(&nzSrLUPS;YqfdgKdev zD)xb$`U~CdX!qGp{2O~h7AuRk$DXan>}lg*&*7$G8T{lx!?6yWU7{|-;|{p#T*`33 zySegZt+f-Jpl9o5MA*#hyK6Gb9P40Ctn|o2J5*rPMt>c zefep%FN?M*i>34Dc3&PjD6bXh%fftLOkev_DoxAoZ$JI}(DQ1GAL)1el-(5bsE3|a z%3GQ4@Fzq(fiF${xtH%xNu|1~s|DcxQvlmH2QX9|fr62NEYkDl$h|;vp9SLCFo;z@ z1o7K1(yT0l#Qh9X-+3?t4+L{KPCm8jA&jmY!oL&5CE69jyDaHc_d^J4A4<2*q4MK} zlKW#Q-~S%UR-KXZrd6y6rRu*>&P9nI9~;V|_)u{p|HqjR<5IgY_Vx;+gS=|JHizMM zC5(aQVHgE!cXv6b!)Z1;oM+Oe+-|B5v2g?)+C|W-Lj>u=BFGsT zL9OKxT-*@BiQ^HhmY1!qWdwb^BdCbb>)QxQnu-N5@(y?ZxI>-4?y!5^9VVOU{R;I` zn?=$&E>fPFNR~B^!m)D{DlsZp17zh#gE^j+{R%& zU(Uy}N}jiAHI>g8r+wVcI4sY^F<9Ip&!6JyH8h^@#>A7jIbNL1c+Bf3(E7^+Hi(1T z{j&CNsp_RJOu(;wBFzRR5~C9}G?C!3iPV{rNWcvF-PVa;v^SCO4kfB%F_BIu6Y+FW z?mtm_QgI?3J|yDvZ4#;dk~lh6{Y#sa?+;AkLag}l@kzY<@-7MO#6lf?mpXbbRca^q z%iFuyG)iX7;AFNBO(uUtGHx4_i4`Ap5KBJQOyQ~tNX5$bA6Or`2WDoHidSkyX=uz6{`+MGs1BlS!Bq_HF-jit(F zB`;4`<}RJBw&}{Ls;kK>T})bKJLY9D*C~Uk-eLpX$>4lO28SAF5g;l`!IM2`PqB z;zpG+Yr1k63rm?;QObw;WjGjXHz!@`XSXu$Rg}?9Ig6(?%GqDHocHz1N$gn8-a+Lg zPf<_ef^yD^X)xPVebKJvv`H=J?w1cZu=D}vmp#B}+XHMbJs{fT0X6SEU|p>Wj<(VH zEFG%$Ln`Y&B&6O$Vm3ci7s^8>jFZP~;v;%qc|^moNBA_B_SE$;@xvZdwD>Wz;vN&8 zp;P*ppnXp`-{dK?ZJu&Vxrkk9&+u#ToUId{GiS?lMqGZ5o%eJ0RX%6UWpzZCzF_3| zO0imA^38;oVrRVMxOA!dzrW(k?XMZ^{FTunwwe=oeYj@GPaKVRdm$LxFmImNctlNIg`?uqd;RVPI}u zgY)ychDY=285ZO>FdY1LU8JaK~PYbTVSJjqvwPjUReGtBRCmYNgJvM}qMbkhrL z?0k`r2QM=2r3sxjU1sp9%XAoLN||(-f0vum&*&;sGQ}5|F24J!Yszz8qe;OvR*p1h zLdWYg-*%nusn?lMQ+)SLV!NALP&e3u`2%inX0as=N^a8m?M;S%y2-XBRy5inp5^b> z)V^q~{z+@b+gP*ng*DDoZn4he7OTeCNMo_pZqrUX2Rj<~w`clzdw$Eb=g(gpXgbY- z)!QA|vd@82uN;{D*@5*#9BFx4j3yID3|@}ROm}4JC;8`VI`KmvCr)m0qVS{>BW^lr zuOyGtzs@9PIa8AF%!&$U&bHD0ewG-Jhi|jvh`JO{-lmuC_1mVoP-y4EIxiQ#&6R(? z%!PYTT$tX@l_8^apC98&@eWsZM(U1V%Z(wU-DoQo(zWI8dG}erhFk8F0xteR1c+qvGUe|b0 z?}|7}Fm4*T(or@Tjr%8`HL&-Mxa zTtA?^xVu>JN&d>=`Ab_;fAi8nTCEIZ;I=@P2Ly8buDnNi%54_@uNNnXQQ{nhZwlfM zBe53G2XWd*3`S)+v)|}*-I?o*4(7D>ZwFS3|DG1iqbI@K(f(~gix9DsL)bAsgl7vw z{!d9~85ZT*etl3v?Cx%`yIr@5-P_uBC%O%6u^Ukk5fBjs>F(|v8ir=*?rwP3|9w7O z#|(oX_OU&@=e*9f)^9;qJplVn0^m&#((CvDxT^i0nCWe4K=q9B+r4?@3x=qd6F zfq-s$kTvC0kL2cVYU94n613@xd_33WiQqFq|8LF|iNlku@O*`Vb0e zQ5&lZCD)#_$ha`foJ9Z7v@pz=&OGS3_=41;W282vJ2wX22WEWJb{ zUBa;~F&x2zBQSPW1hVEv;LW@Uavvig&@)tdg&EPik+^j)5<{Ovq9H62QNuZBa26@1 z-es*U0?n2PzH+Ks>fa#*<4Kk1;b6uyIKOKJnBY zOhD(e1Po;U^k!uOYN`_Ouq6Td%M)>Fb0QudOT@Wr^btKLtLQ}{DyeA&{z}9;heRy( zN#u-|2rtYquCmGRZhPC>gnvlVQGw?@uP(W0u=hqc z9gc?7yUh8yWjb}{bSQ_X<7Y1Ot6k}sDxHCzvooN-i~ToyGGKh2y*QkI_JjPS~u)X47Bqx4^6HpVLCFhiJw?VKCISqZpS*v*YGs3Gy{cU_i#x=9Us@E0khBeNC5t@ztgj=bGue zA4B&0sxrJ0%8+rh4Dol$u-~8z%{(Uz%iuM<9JvR}aoMmOgT2ZzC88X*+2uH}y8?;F zD#$dgz;KNUY}2X0ck2pF_pN{zbEkitD={p;5}|TcSU0>1dq$D_uE?zE;wn73SOr|K z;`dq=;#{iW?N)_vLCm7|s=@coHJJNv4T3p8m1Nc+tEC27TWj&{b}gD-)iU?WIf}li z7SNx+uMYhK=+T#}$HHFqNFH8~aMgPD&(@=GPy_l-Yry?I4fNVKz{9)&y#pIy9oc}M zl?@m*uMv4i8`-DQge<)#m=%)~zqT2E3Cx5RHRI^;7Br-{pnYO1h97Cg9o1I!e8w!N zs0|nBt1lhhfh|)y@Q-XKtf}>lsp}$Ro4LxGZVX|E<7{Rr1@<>;G}9~3Eaj0E(t=H~ zw4lO_5M_#ZUrXaW+RuCFi3=?KAxCqguYY2Iz z0@dO7;QrzPng1`yoqd7p|GmLP>$mWUev96=?;t(nJvLr^j~eUu*c|+x9{vwV+4vFL z4tyjF;v=??RL9rJ^uIq-N3EGUvvQxII6?!j{?S0&Ee#wU`vrD2U&ylmiaz@^v9D1R z8iE$iFZzaY=fB}!<8Nro`v&nSZKNw|V>O-CY~A$83DSdSs~&>(>q9qM9~uTfp?2H= zD$(RcMgPL5QHF4|F@%+~Av(Ja@on^PxW1uo=0mMa)`;B>Mp)!%gaIP%z*_$2jhEeF zjD`z;U~5RedCVV-jsJrK7A837V}g#&rWkX?6g|G0(j#w*XU3-FJecBzD)(M~f8o~s z7yB2O;l*-u)Qip87jJD1SfdtFPUYg%I4cQTo{_e#tr=c0_f z3!Y(`L?Jzevxxql;%iSyz?6V8pb^ zHchv|N&4cKuO}medehW@Z19X{;xQYPUS*eJstv}r(jz~{7Fo-=4>Pw#S*|TU_P4|B zVRp!+mK3tn4#(fyVaE?U`jwfzthFNt*B%iI$rj&5zUM#WiXXN|&Jk)#rsRB9*)yB& zz&xe{4z)WlGvcb>F^Zn|DK)Opks>dA0Ow2kyq;dCx69U@U4P^r#glhbNN#7nJ4SP^ z+_%yLCwY$i-yBKpM(K7 zGyU;=jX!dW$s4Nl$H^LhM2!l-$6W!Kc{TuVuLR)n6Z*Q|2S9=TuKq^sX0!;vgsK4e zc9HWfLycxqAhftkTfc(cLZIL4Guhq|f#?-Ut)_Pnv?c{{jts&T4erstlkpuBgn+TZ z@R%5kX-dIpJrs8X4f)Nu-e><6(eRqXmDt+xn)L$wdhEO}Ae@iz6uk^?*F$%#w zj}Y7{AX~FfC|NsXXRZpxxi#z?+7^oQJ3}EkPqz2vP;_5mCy{R`9wddLtU45$bHgxY zRT%r~!^m3&*q`x9Y^JI@aD3t_N+9)`4F`n#&a=vfZOY3|pm7l$MMU^oW+ zA`2xf94ee;7tfDCg?a=$=zH%dh`^EZ2!ysqU@iUc=T9;}O7FXzP9zrpj)dsk(>;n;^AB%9FzO7I1*vY6t?|T!whj8pVqo4?A=Ws_@O9R9Erk}lTpxo zO0MSHD3s|&;j$zOKE+X(K0F$Sc(!tuz4tO2lS8AK<%>pFf3m@s#z1yi43am-z-f&hA33}Jj6qNKH}2t?U>t*BQ)W@EVxZ+mrV8g?`LGzA;ht{Zuvom>7YogU zWQ&upqQ~9b$QR6|eu;&;7TKGYv8d&&+ozK|x{=($jc4B@{qkA!=#k$V2bXhkP<#^y z4VO4Pag77~s`*P+R0e1n~Y;d$@D9wz(FwuqJ1e?c0L7R zH&fv8A_aM`Q@FE9!HXj9>^j)jI3^Xvi&HUUT`Df^Ooi+}sYp6Szx=sW=&GgS%=c8( z`ljM+Tq<0uQjuPh3h7;GFyuU|WRnIXaT*jyv72Zix#bVj;rTKhL$&Csw@YUqW;&kc zq*IG!XHhG=i#V^&l+B?2l7Y4@WUSoCK=D0xHuBg!$bjfe2Eq+8n9ry7Rgr=C`V44G zWx_;}{H8hVSYMTi=6#v);%qzjbS8AjU>O~iiAkB6s43^q@>#gY{oUEPEbMCM%sWE@ zHO{w-vm|J#W>4edZ1mcmjf`{In0221d(O1_w%M5OkPQjF_uG50mvL|opVv9KG9d@{ zJb5#6AjSM?S8fiDHsqiudy7t}({ulmTHBvI#F*u=w~3k3;hago=d%kYA7SLA1UnU= ztalM)`xap^JzY;xjN|urdaM8dj@|3DDd`UI@c2=Y61T&X!s&VIN4ZfVI#YyhOT4$3Dvc3+& z(K_fptAm0{J@(!w`$LCpke~H1cdLie=mvOOHDJ?5_JJoglE2o3jZ)3fe#1U+_Ih8j zZzBh;4eRK^(p6)hp;0>uUUVSEfSo+@ov8S?6PGhOp=H^H(qt*2F1d#gyGu^cyC5gr zaK4EX%{LKIb{osy-9^wWRg7tTie}^ISTg1<45Hp)QRW9sOe2q7m!5W&FNod$mE5VX z@b9Tf|D`5w*=RAROD5+`0ry0|;K)uGbvZ+5R~ll@PkPjoe?xI3yI*D*;Q|?# zPtF))&~S1c*3h4B_XlgOOt7fK1j%bmnF}z5*XX}^x9u+s_5Q+9*$m&fyBeBGFM6>V zu#o)8sTSBkkIf1l3z&?tM8yY71Xj>XLvEy1trZqU1`azX7M7u<_;L2!Rp>{oKd?!~U$&A6g3GjtkPTmcPN95Qx= zlDR7`M)Eb^758@2H*(Sqn@!xHZQ+Jl*=`W$(@)ad9mfv3BlZrN;TrDv_0=8GmhRZV zY@K$aJ1&nS`}?*B2E6d#tnYz{267dTQ;T~_4uqyB9&=aqi@Mw47Egq7)>t}`+`|=K zShvy(51(<)(DcHZYAH)2M4BZzw}kGtOJ;p>e}LEe1!cw;A-hwqpAK)iu| z@_&8!|Bmd4k3P^6`(XMWG7Y)cYJJ9imdF<>vA*QX`69Z3rx$(Yef=0P{Km5Kdo+i&`N3J_N#hT@dc>V6XH3ApB;I&g)hX#zX`m zw=D?ivx4z=docd&2}TCz41@E*7<4BXkM0IT`w{tuhV+UX1tV&72(qZl4fz&ApEl8ilE{H(R@(4V*5rKw#5wL$9 z0Ym!4-M+K$SvLYp!Xxm9{*R2b2;AVDF{~s4+MN-orC260iP5 z;wHOB*4jnlx_2a;DLp-L?d_P@wY?2N;;gYh`^J|26u<1sxc9)%+ku#o!Q=`#sPq{rLB zD1mqJ1RU;8fccU{jNHxs!c*)cxsix#|0Uws2lBj46Jbm3PLh>~IX#k4HGw;^kDMEf zl9;BWff4-P@9} zVqY@ytdeoRHW`P9q@ZP73cK1;u#j^@S$hgrO-aSOnW@;diyrPHsc1P*AGby-7HFly zh%>_Z;#7R*e6X7Ail!-PnEsSqhxBq^_?1THTpC(S)9`j(I`*ASXU-xWpDohy%qkt5 z#LV3F%0NON^1tbMad^-Cokj*!c|tTZusbOOf0?-(I*_c2A(^ySzM$u~Svbg8c`Y#*C_S|)KWuua_fC6WMzDhaF;ZU{4wjN7{w**E zqq}ktFgq9KTXGTak;^=HE)tW;VWSsk*MmGnM&`j`S3cwqQp>xN5A(53%0*u^PfS=6jg}-E{VQ~TX9tCipP>3PR*j1=p$mc>K9-L%v^X)=t zK4!lmIcs|t72(ciwh9IHg`uS%?!RAOyKC8q5n7oxBV*{xMrPwv_J$<%48dD3Ix>}4`P=|_Jb(m*VhjtgT&UnJ|>M*vsj`~48 z27e?sBDEe0QyO5iqX9cJ8n8UG0sf*!h=w*HKcMNa)8QFysvC2VZF4p zpgdJtPz#h6#)p#gCD%icXD6X?Ll2>4aZe#3pr`O-ri`#tpBx7nS>eGTIbi@ho;_wL z2uJ=Z2v08b7OJlH77Rc16S~d&35S*Y3$L$Vz~Hk_(0A@Llum!nY|3*8(_f&z`z40x zzrl2!k9bc1)%ua2@FDLjHq~g+`|%Ci=V@ch*zZs}p#v9syDL(DK)FN@%i8p?&_*90 zm;b~p&QRIE+4rpT3sM%GndA*2aWH};(+HEYjPUrDF|^f<=~Xj<#R^kIdz<2)pMP<~ z^e@&)%#gL)916|mIN44gva$s}bH{Tm*aEBQXF3~b2^)JWxLmWwCv$83Hn)L}BWECa z@~)@YVw|@v^xABp-A+H!aIyyuvFpvy4rAy$`q$bX_u~Jr&&>gm=??59a)i`8a;E1y zu{+8M1Ie}0vv7i|Co^=1oH2BxnbToH{BaA)_9}klQ%~FB@5Nc8>bV! zaW$V^XnlO}cbE?~YacirW$sOrooE(5NDlV_;(V}gt&^?rD0Oa}8ve-!WIp5v!K3f!1)UIDIYXl(1 zEdciu1E|dfAY*eN##{@;t8alAWXgSqI1qB3^rOxR!dC7$hBBk3@|RgPCw42#1>^1b zU|i?iapr0;dRWj8l+0Xz?+{ETgEbu?a5%vC#v$+z3_*1!^|tO1b_j)H(1=i+m=cO> zYM~fz5(DCgP1eNw(;TcnG%lebC}~l z%skpfvRc*2ZPgFQB9m}fIfO$*J?^w99BJ+0s8oqS)f>(roHaIkMj*0^o}f{YP+S*@ z*&iaYQ8N<3F`O;>aef#pLjEoh`tBBS7b3!%pCZJ%iZCObnKbDr>=_t^*^8sd?TSLH zNfboQWMQ@QwJ&+f)ZYe5Glx%oP33hA?pwqlFd+u5i80tmPteWs7${f8prwtw3TE;T zQd?6JI1lWN#lMeZk^J$0Y}c26W4W_scE5tBJ{FJq#lidrXM~721Q*0%PGKBw$k6*U zARfP;$D>O<9#0J8p^^>xgodx|2I0;jm!4L`3}(ZM z>2d8#&g)R-^2dtV-77{F=ZC4+l5v*%hP~9rvdO$^;MwKFyjU~6LEXvZR;D1~zZ4XZ zca^K2f>-nf?I>g>Uz*wb{;4n=n~HH0xM!G~iUafq)i1?iZqosQ9N>BuAJN>t9C@upgQG0`Fc~@MCY<*zshvaz;>Dl7(3|Ss0(21=*S` zd>$mh>WLD(ohE^dq6F@9B-lxgtMOq8_EIa8Jt|?Zr3Cl&B~USwV1Kd%a|04r0C_ala<+iIcCY`C>w-C9Fh13bj zr|MUPDgBGEcX1Jv$+Aw?l)j@zebdgZ-nt|@~f8f z^;{#qK5RrE=SKXEYs89-Mm(C|1fzpZFu2?VIfEwZl}(s$O)V|537w)QL@sNl($S2E z%FUSfs~Iw0&2aWYSaX6#xz2bk{t0L5b; z@SpAnbd3GTzD{;RfBu9D^H2EL{RxKG)Ui%Q9b0tS32mp&&hO7iKl&MOH9upwf(EK4 zYM}k52H8&<&(3u>xbsM`GvGlpx!C{PP4azQfk+J4CtP`QNC6`L^V+`s-kKx(;4RbP$!T!)KcgblD%R^WX=(7wN)Y&_%~l zU95Vb3v(x3B=Z;))4O&{5BBzYi1gM&Qh^?Rbm`&0`TF>JQ=i#CeYo1|!`b;K9&jJn zxZVKqDh4ptH-N`azIu@Fdg~Y7y!eH$&c9$6^NXF8zu;W>3&s5mvG9o@BA**#={G~T zn;T*{wNkA@LuyIC;W&<6Ium}Qao=w!(F=Dz`!|Xj*uB`_2s0Oxd9l_AcbPT3rp5l~ zRwF2N8R5|=W3rizab}h=MlCXi>O*5Felf;2S8~UyjA69m4=i*4;H#nueyuUV@WUq9 zf7Ap+&X~a9qY1`;HzEJV1Z`#}P>UgFmOeWjX=V%On_`w=3g?ZcSh|Hd!`-IrcQM7v zSEktNY6{zMQ)p$Hq9ljED`y_@FgqGO|KeC5GsM3)!^Dqfn4x2aHfuA?E-=HP5_U+J zbEny5hW#7NF>;$ZYL2o`$IP6)bmqL)XCrr{n=BEv%@W23EFp8p z5?v21@#71;tmGO&+`FRv4>e1tSkDTu8IRUmI)8jIl;` zoHZ1ytudK9)enj`C}zI!BsE?&?pNJaZD8<+J=5Oo+wr%-PzgPQ9X9mB+v4HBwz!{Q zONO;AZf~@sCS-@9kL;ks6Z6&%M=kB}$;l4Gs_Zbf(GIT_sq-q^BlmHZsGx)D7D!3E={LB@v-?69Xiz{aFjy3j=D`xPXb+XzORlH{%?{vk2-rNT(x#7`r zH>7P~USp#hROx$k`s#*EBR3rIb%RzKdwRGtww&vZ4~yLCm2-y&J&(n^-Lda2a~#Zb zeB&MMZiG9oNw`}sWzVFP2jt~FP^aL*?p_aOuh>(4-2-M1Jdn*!p0n?nPqbh+kG}_w z@&5K+$`jhRJu#A5#7|;R3@Y|SeUm4yc5=_$!wYW~c%e75h{0Z7_?pVB2T#dBW)Hbn zUa{I6M^Aafly|v)#?-aBU!E1?4ZT`#Z0B80xi6U|{e6(M#s^!@`#|p@Gl%bdV4>lI z3E#;yN%z6<93On#=!;LAe9`Rb3;h^hIC0;sJdAg_Nz5Bg^JDJF50X}D+U-3taw7mcRRhq^H30rGWEN+VRh$z5)%Af`rNRy!tw7E$fk+Oc zcAXN43g!v@c&GDM31WtX8IMJ*hzgU9J6oo z{k?Fgaksp6R0LEeML_i$y_M9c!#_u03ps5aWfABt!=9W$)Thr!Lh5rQYK$WhVi}3k zu94^>Z^VC~2pZ!>s8bT*)(#PJdEYbL%g#r85u~^;UR^4}zD^NBM@M1CJaXFhN8u86 z>Pi)A(oeWEeiy}TZ4^}dN8|H|XlR~^#@rv#cxn(0+rcp?6=LvyTMQI5V#o!K#XaiF z#zwJtV9fV{u~?iDi_4j@aNih*)GN##rpLiLKMu|0fD~Me$E0`hDEb31ZLSUnjPlT#oZNkQq|6asR*h53nF)NJqYc+*pukbyo^ zvas`#1dr}W(CQ{(eoaCSMK)BAW#f%^F4+xvC=m*&^AzH3S|L8?6rycOF^tv}qu;J# z=zcDSi){&}PhuB?UOD>YR^p!tRdC`C(95|MXYTb7d~#1B_}Ue0ELXu2<^LdEtcsW| zuMsx+EhLWbP~7~UJ1EW}rr*%;>N|pevVU(Czsm<3pnkald_Mgm6Y>{K=Nhty)DUHt ze?yI4bl(~yn7(9gpBa8N4`bx!7-KN=d-qqFK=+0Ttny5u)@I86k}3U$>@42$7a93~ zfi-4$2s7@p%n@?doY`4(B=I}#bhiaucF>dVK>tycCFUtId*^F~_q~~~+sQ2ZVQcJh zvqoC8H9{9L%N}Bb*G)E9ZEuTOX3@L(UGZ=%cUMR45pvHSKluI7nrx4P8xCa9k`Hyv z5lKHBF)Y#%LsJ})GtvpO7dqkVQf9;7I$cK7_?v)9lKDk3+GNOB{m z)D4X-ZrCU7j(IZfXd27cQ}mTvvmedT9fN}0@i@#Kw|dc6KFtH?6g{xS!~@spDc{gS zPf$;0;|4LuJ=znN^ab4w_r$6iPyA#y?$i->l;yFjtd!5q3NM%p@y5ax-uy1}#($RH zcuO75hp}umSF&rz3Cl~4uFQ3d#}O(tmSU&-;aTqXBmk14uSNy zF&kG92q&H|d=xH&rIce*_wP?0 zWfMDwhVvPBE|fcTGAKpNu_=V%*y=FUZ4bjH`fv7L4})`}q0En+VgwX#uB%s)k8=ff!McoBt+t|)w- z8jb87(HMDvY)bme&F_+(q#BL+=FvDB!F*gEpNj*Twf-FgE9)4{^yG7pew(Rs?6qaa zEv_Y&+B&<5CdHv-<^Q;oE=Feo@yt5WN6zPAJiX-4sgLdEj?3Mj z*|*|&Opzhay-x!2xZ9e>z1Eu_^plGd5OO3DlZ_HFhFLc6Lh54m%xL#Zg4*aL_>M`! z$w^7bUX_H-z2v;BCPDvg5-ffsp~X50jrK`U_D{m|nk2Y&^D{XyTDOQfZ;6?c7GtC~ zxsC+~e+GA>R{#*O)WzOGDW7Lhscz5MKOGA^Cwb2lX!>vNLP zr#>0e6{wl5PeG?@3Rae+AgPMkxcU_A8%QtBRI(`Pxj9Q6?UZgR&goNAvrgrHC>3{a zlk@(WM~k~KD>9G5shusPUbZnE4x7@^`%*d@@n7g#*F#jVENmL_q?%nG*P{V%P32a+8ipsHaJw=_P?F{o=hMCD55n zj^(m!+*?U6%@%sc$y4fil^mrKzOT%N&Kh>DttZpnFb99ksj1oI;7tU%mYF#)EMyL@ zJBM9yx!5u}7fTM%FMgP;%P)K_%*74mJba?(#*Lnvp;mdgw?7~0C-bq*lHEnN+=n^m zW2|RBYI&|i=A(>zvLfnhx+@Flp)0^uYHF)%3vh8mAv)9w@jbE-v&#zcw5||mx(o4b zVi9^RC_>GuB7EOogn>tkaQjRV&Oa~0vhPK>6T<$Y)FRYp7vXqm5l(f`H$K0ZY@uS< zA1KC@GxYL2r&s)4F}Bqfqe7|#JtmZ}CztG|g*<0V5FAi~*Yu4m^ejcckENL6PA+Ch zDMGj(8$?xz_wN~6Ho=Wv?_RvuEx}Z z)i777#)?1H%x_fVMmN1h$7^u$ZVhrD)FAd#4d!tFC2d=S|H5i;vY-b2W^(^^pcYZw zfvIcMVvsGFQ2O=w6;uzu&U%D(*F$l719uz^(2Z+A|C9z4AVhC6p) z(YqVrd$1AKryBXa$j`-%NGxbXLRllsH#I?N2lc!Y+LVM0a|Gq_FIRo;ZejAoqK z)QZOkTCw_cD^5;p!}hi81A5nnU2bi-*4c*Ri`!AQydD3pX~(P`?NEQyj%^?KvoX7X z3fpl(t^+3(Ixyl&2S)05V8!naILz+E(A}NrzR(GUr=2Jn(8V5?E{FrT1B>cHc}^GF z2X@2ta5wuaq=W%xQi5uTl<=fNN=TKG7A$s33lC4Szvr^FF!!ajpyekm*u_W-CVzSe zBiY?^v$cn?X--dJ{kxt*pifVs|412Oo2iV@xmH%Vy-rpbWX0WAuB;H#AS>jJmlLw) z$qD9rxFVYs~xC8r`ba_~L5~*F84Su(#p8))qt8+rs^tE&mPhzJAIMYcJXX z;q*vl+u@14J$})XEx*hjS)1+gly~&)xP%}+)<|D4t*VWJTP$Qo`(0;0C${>^vbA;fcrs_QQ_!f(w04GfsG66SI0RK6{~Gycb5vc%%Pp zZ*17)jUBvy24?e4N{^G@1Rp3lveVVs2P-{z=S=WHYn2cDYJHd$@Wtrcz7T&UZ{Vjd z63aMiE%L+kRes2!pISk~4{bmEsGqPWhcniFXFqI9@Nc~ zv+UtG_>nwI&Rc30)D_*x1V2dM(ti5RHhydpF!s%&_qNh2F9LgTixWIFN zLo`wk(O+~d8ebkoqt=`GypU)z)}v9_7!COgF<5me1|I$~7(pFzFa1U0#u!YQAB*iv zWASWTEY2V2JavYAf$&%;r}18yABzvw?9FLlSL>)aOqm^rH>zT$>kRB1n2G+>6o+l0zIZMZU3W4e^*R&xzw!i< zJK0LTQ7#Mn=VW2uqAWCT;0&eBj-0?OZ0{|>)_)}U>n_1X`jJ<5OE8}-pQhQ_Sg)Fm z>wekH8fBv}j(TByHhQFFBTqU9ic`r7P-L!7B?ot`bI|0SgU*l~#FpjI50Qi6BXglV zCKr!6Hz_X3MgQfwShy({8}8=f%{%&&%5upC%wzvw9`=mL!?t&M_-LDlgv2~*SIp6^ z$;Us`82enxhyG2n!&UR~&Nv@3q51HR=I7F!p@tQp&pPUi$H^TyO_um&`jj*Za3#M0 z3uFt?M4j>J-a^cKPquh&A)?730KLzS!;3KacoFv8DuVI1BJA`of?H$}tR#GGFT&}r zBK8W>AAPJCn;sTJ|4}h|kx{V2yBM!him`ZV2_~N?frS3(L!uI-4=RQ4et&YRbzIQFR&VZTe^7gmZ(L(6c3dAsMnW$c|UgHAJjO~cDM&z3{hi?b9@e{ngi zE6Opku^cKpI5*i<;2N1n)r%@&u(1+ZiIsRlPt%%>Rj5#|g5K#WY`Im1$)Bsx{G|%_ zG^;S4`eJTE6_m1>xvQhTc)l8sEvlKptwuy{HCC;xLCU=vj2m8y)5~k|IMPc!z8YsQAz%`js|@AH;sJUiHoOA}fUc(Vn?>MeNZ(1NqkEojMTK~29_%-Y_H z+N-UoG-|{7gf?`xw;^g5SqW#`k*?7WN0)YX954A+a4$Rn0&-jxLe&=>T#;ODE zwH+AvPbWUvcH+qHF6xtA>;mtG>f~;0+}VwEzi!0QJ5`$|C3H`c7F3m}B~nLJI4UjF zy^t10bEfKwmlh5eNDG@6^bn3M{a+uBWq1!Et+0nMzqp66jh%=Q#+WQ2hl zGD4W9jNs!SBg}4-5hg!pE-+YDxJXS=O*9eQ-w!C*Cc376Ub5>58g2M0XWcEE>l2P{Z* zz=lc(NBu4wM>h8tmS=*fMRoT58QIa|F^cgH9_cYOXspVmNf4mOfS zzQu#x5*`>#_JQ*hPyF5Ni90(z;q36gIm&U87s}pvLFB$N&@6|-dhg5f`z`}l3aWDt;P z@F^IN$8#se`RL%Q5R6U=!OQdz#L0xB-&(T1_lDx}@lc4Lgks14dAjAokU4}q_BG@! zaYyy+6faJ=6Xj;Z7xoZ-I8#WWnh zqQkK?H=OL?a7^qKf%DwU_g@@=4;LbE|0;L#mSlj>i-bHGOpI@)jaZ?=;!LYkG+PE5^={Q5xVq)pXAQ#>Ea}6$Vuc~B_ZTg z5{5rdLa0p=7I-H?K0XPT63KX)BSsz1-fi4#eG+4Gl^AFhL#>Ow-Jl12BllVllF{)# z8E^HH@!>Do2ocGcpO}o0O(__&IfXpt6wJSsf-j~ixNnt$CB7->L;k_wgQ-w@l8QG! zQ*qfO6*hjUcovk3gm`Ai5>s)MJ}(Dlb{FnV!;xcYxP2`Rqd2n-eVB$K$22^bONT5y zUn@q^_eG!hg}3QA$vxLWC+5gP(qR;uPBvNwJ=x?t&CWm%&SvJjGcbsr@v_7W+|AB_ zMc+&uUdcYgv9!z4?)e5_8UMrJ0BtNDuhLECepig4t@$YRBmlznz5) zt1K+3&qD8s5{TwYxU-dD-9`ysUY0=Zn*<)eB{*s)!5?ZPM^Yt7%$K08P=dQvoaM&Q zKh8OB+*a+4S0GLpL)Ua?0F&(L1jEDF+2SH`H^m&pHQVGIH=w8*^n7 za;X94LT@^G;yZKEdLtKCU*_U#LN1Ps&m#{q5B=BV!SY5PR(j>ZhbMjjXSKumxN(Z! zu(SF2sgVyg&TZc_^5HDc-tG|vc&uE2%+CdA=v|1U1@w6>V7qN!=2alC$)G*PJ)4VEj%h~VO&)U zs{wVmH?$7NCNpbC-@5IAI`n&0huV*I@O7-iXh|I``qjgJeLartpg&Bj9tWfwa8RcK zVQ(9u-Q9?vQtT`2+k_v(ny~e36SD6#;kglc>7$yVFu55uGn?^sMGKy7Zb9D@E%;&i zzpm`l`&yB5kb5unR!D`DnO@zBQ_Zblq7oNW+tBp14eveM@S~*-6GpUSFPT=xD(!f} zEZ_RK?I^Bm$I#K_ENtq)7jK^UF61rghJJ4;Azo2R7`H%5czRh%_&ATk_X8OzT@SE0h#hbKjmJ6@;3!x*u36?<%!-xzoyxZWN2UE{{dm` B$}0c> diff --git a/benchmarks/perf-tool/knn-perf-tool.py b/benchmarks/perf-tool/knn-perf-tool.py deleted file mode 100644 index 48eedc427..000000000 --- a/benchmarks/perf-tool/knn-perf-tool.py +++ /dev/null @@ -1,10 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# -# The OpenSearch Contributors require contributions made to -# this file be licensed under the Apache-2.0 license or a -# compatible open source license. -"""Script for user to run the testing tool.""" - -import okpt.main - -okpt.main.main() diff --git a/benchmarks/perf-tool/okpt/__init__.py b/benchmarks/perf-tool/okpt/__init__.py deleted file mode 100644 index c3bffc54c..000000000 --- a/benchmarks/perf-tool/okpt/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# -# The OpenSearch Contributors require contributions made to -# this file be licensed under the Apache-2.0 license or a -# compatible open source license. - diff --git a/benchmarks/perf-tool/okpt/diff/diff.py b/benchmarks/perf-tool/okpt/diff/diff.py deleted file mode 100644 index 23f424ab9..000000000 --- a/benchmarks/perf-tool/okpt/diff/diff.py +++ /dev/null @@ -1,142 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# -# The OpenSearch Contributors require contributions made to -# this file be licensed under the Apache-2.0 license or a -# compatible open source license. - -"""Provides the Diff class.""" - -from enum import Enum -from typing import Any, Dict, Tuple - - -class InvalidTestResultsError(Exception): - """Exception raised when the test results are invalid. - - The results can be invalid if they have different fields, non-numeric - values, or if they don't follow the standard result format. - """ - def __init__(self, msg: str): - self.message = msg - super().__init__(self.message) - - -def _is_numeric(a) -> bool: - return isinstance(a, (int, float)) - - -class TestResultFields(str, Enum): - METADATA = 'metadata' - RESULTS = 'results' - TEST_PARAMETERS = 'test_parameters' - - -class TestResultNames(str, Enum): - BASE = 'base_result' - CHANGED = 'changed_result' - - -class Diff: - """Diff class for validating and diffing two test result files. - - Methods: - diff: Returns the diff between two test results. (changed - base) - """ - def __init__( - self, - base_result: Dict[str, - Any], - changed_result: Dict[str, - Any], - metadata: bool - ): - """Initializes test results and validate them.""" - self.base_result = base_result - self.changed_result = changed_result - self.metadata = metadata - - # make sure results have proper test result fields - is_valid, key, result = self._validate_keys() - if not is_valid: - raise InvalidTestResultsError( - f'{result} has a missing or invalid key `{key}`.' - ) - - self.base_results = self.base_result[TestResultFields.RESULTS] - self.changed_results = self.changed_result[TestResultFields.RESULTS] - - # make sure results have the same fields - is_valid, key, result = self._validate_structure() - if not is_valid: - raise InvalidTestResultsError( - f'key `{key}` is not present in {result}.' - ) - - # make sure results have numeric values - is_valid, key, result = self._validate_types() - if not is_valid: - raise InvalidTestResultsError( - f'key `{key}` in {result} points to a non-numeric value.' - ) - - def _validate_keys(self) -> Tuple[bool, str, str]: - """Ensure both test results have `metadata` and `results` keys.""" - check_keydict = lambda key, res: key in res and isinstance( - res[key], dict) - - # check if results have a `metadata` field and if `metadata` is a dict - if self.metadata: - if not check_keydict(TestResultFields.METADATA, self.base_result): - return (False, TestResultFields.METADATA, TestResultNames.BASE) - if not check_keydict(TestResultFields.METADATA, - self.changed_result): - return ( - False, - TestResultFields.METADATA, - TestResultNames.CHANGED - ) - # check if results have a `results` field and `results` is a dict - if not check_keydict(TestResultFields.RESULTS, self.base_result): - return (False, TestResultFields.RESULTS, TestResultNames.BASE) - if not check_keydict(TestResultFields.RESULTS, self.changed_result): - return (False, TestResultFields.RESULTS, TestResultNames.CHANGED) - return (True, '', '') - - def _validate_structure(self) -> Tuple[bool, str, str]: - """Ensure both test results have the same keys.""" - for k in self.base_results: - if not k in self.changed_results: - return (False, k, TestResultNames.CHANGED) - for k in self.changed_results: - if not k in self.base_results: - return (False, k, TestResultNames.BASE) - return (True, '', '') - - def _validate_types(self) -> Tuple[bool, str, str]: - """Ensure both test results have numeric values.""" - for k, v in self.base_results.items(): - if not _is_numeric(v): - return (False, k, TestResultNames.BASE) - for k, v in self.changed_results.items(): - if not _is_numeric(v): - return (False, k, TestResultNames.BASE) - return (True, '', '') - - def diff(self) -> Dict[str, Any]: - """Return the diff between the two test results. (changed - base)""" - results_diff = { - key: self.changed_results[key] - self.base_results[key] - for key in self.base_results - } - - # add metadata if specified - if self.metadata: - return { - f'{TestResultNames.BASE}_{TestResultFields.METADATA}': - self.base_result[TestResultFields.METADATA], - f'{TestResultNames.CHANGED}_{TestResultFields.METADATA}': - self.changed_result[TestResultFields.METADATA], - 'diff': - results_diff - } - return results_diff diff --git a/benchmarks/perf-tool/okpt/io/args.py b/benchmarks/perf-tool/okpt/io/args.py deleted file mode 100644 index f8c5d8809..000000000 --- a/benchmarks/perf-tool/okpt/io/args.py +++ /dev/null @@ -1,178 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# -# The OpenSearch Contributors require contributions made to -# this file be licensed under the Apache-2.0 license or a -# compatible open source license. - -"""Parses and defines command line arguments for the program. - -Defines the subcommands `test` and `diff` and the corresponding -files that are required by each command. - -Functions: - define_args(): Define the command line arguments. - get_args(): Returns a dictionary of the command line args. -""" - -import argparse -import sys -from dataclasses import dataclass -from io import TextIOWrapper -from typing import Union - -_read_type = argparse.FileType('r') -_write_type = argparse.FileType('w') - - -def _add_config(parser, name, **kwargs): - """"Add configuration file path argument.""" - opts = { - 'type': _read_type, - 'help': 'Path of configuration file.', - 'metavar': 'config_path', - **kwargs, - } - parser.add_argument(name, **opts) - - -def _add_result(parser, name, **kwargs): - """"Add results files paths argument.""" - opts = { - 'type': _read_type, - 'help': 'Path of one result file.', - 'metavar': 'result_path', - **kwargs, - } - parser.add_argument(name, **opts) - - -def _add_results(parser, name, **kwargs): - """"Add results files paths argument.""" - opts = { - 'nargs': '+', - 'type': _read_type, - 'help': 'Paths of result files.', - 'metavar': 'result_paths', - **kwargs, - } - parser.add_argument(name, **opts) - - -def _add_output(parser, name, **kwargs): - """"Add output file path argument.""" - opts = { - 'type': _write_type, - 'help': 'Path of output file.', - 'metavar': 'output_path', - **kwargs, - } - parser.add_argument(name, **opts) - - -def _add_metadata(parser, name, **kwargs): - opts = { - 'action': 'store_true', - **kwargs, - } - parser.add_argument(name, **opts) - - -def _add_test_cmd(subparsers): - test_parser = subparsers.add_parser('test') - _add_config(test_parser, 'config') - _add_output(test_parser, 'output') - - -def _add_diff_cmd(subparsers): - diff_parser = subparsers.add_parser('diff') - _add_metadata(diff_parser, '--metadata') - _add_result( - diff_parser, - 'base_result', - help='Base test result.', - metavar='base_result' - ) - _add_result( - diff_parser, - 'changed_result', - help='Changed test result.', - metavar='changed_result' - ) - _add_output(diff_parser, '--output', default=sys.stdout) - - -@dataclass -class TestArgs: - log: str - command: str - config: TextIOWrapper - output: TextIOWrapper - - -@dataclass -class DiffArgs: - log: str - command: str - metadata: bool - base_result: TextIOWrapper - changed_result: TextIOWrapper - output: TextIOWrapper - - -def get_args() -> Union[TestArgs, DiffArgs]: - """Define, parse and return command line args. - - Returns: - A dict containing the command line args. - """ - parser = argparse.ArgumentParser( - description= - 'Run performance tests against the OpenSearch plugin and various ANN ' - 'libaries.' - ) - - def define_args(): - """Define tool commands.""" - - # add log level arg - parser.add_argument( - '--log', - default='info', - type=str, - choices=['debug', - 'info', - 'warning', - 'error', - 'critical'], - help='Log level of the tool.' - ) - - subparsers = parser.add_subparsers( - title='commands', - dest='command', - help='sub-command help' - ) - subparsers.required = True - - # add subcommands - _add_test_cmd(subparsers) - _add_diff_cmd(subparsers) - - define_args() - args = parser.parse_args() - if args.command == 'test': - return TestArgs( - log=args.log, - command=args.command, - config=args.config, - output=args.output - ) - else: - return DiffArgs( - log=args.log, - command=args.command, - metadata=args.metadata, - base_result=args.base_result, - changed_result=args.changed_result, - output=args.output - ) diff --git a/benchmarks/perf-tool/okpt/io/config/parsers/base.py b/benchmarks/perf-tool/okpt/io/config/parsers/base.py deleted file mode 100644 index 795aab1b2..000000000 --- a/benchmarks/perf-tool/okpt/io/config/parsers/base.py +++ /dev/null @@ -1,67 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# -# The OpenSearch Contributors require contributions made to -# this file be licensed under the Apache-2.0 license or a -# compatible open source license. - -"""Base Parser class. - -Classes: - BaseParser: Base class for config parsers. - -Exceptions: - ConfigurationError: An error in the configuration syntax. -""" - -import os -from io import TextIOWrapper - -import cerberus - -from okpt.io.utils import reader - - -class ConfigurationError(Exception): - """Exception raised for errors in the tool configuration. - - Attributes: - message -- explanation of the error - """ - - def __init__(self, message: str): - self.message = f'{message}' - super().__init__(self.message) - - -def _get_validator_from_schema_name(schema_name: str): - """Get the corresponding Cerberus validator from a schema name.""" - curr_file_dir = os.path.dirname(os.path.abspath(__file__)) - schemas_dir = os.path.join(os.path.dirname(curr_file_dir), 'schemas') - schema_file_path = os.path.join(schemas_dir, f'{schema_name}.yml') - schema_obj = reader.parse_yaml_from_path(schema_file_path) - return cerberus.Validator(schema_obj) - - -class BaseParser: - """Base class for config parsers. - - Attributes: - validator: Cerberus validator for a particular schema - errors: Cerberus validation errors (if any are found during validation) - - Methods: - parse: Parse config. - """ - - def __init__(self, schema_name: str): - self.validator = _get_validator_from_schema_name(schema_name) - self.errors = '' - - def parse(self, file_obj: TextIOWrapper): - """Convert file object to dict, while validating against config schema.""" - config_obj = reader.parse_yaml(file_obj) - is_config_valid = self.validator.validate(config_obj) - if not is_config_valid: - raise ConfigurationError(self.validator.errors) - - return self.validator.document diff --git a/benchmarks/perf-tool/okpt/io/config/parsers/test.py b/benchmarks/perf-tool/okpt/io/config/parsers/test.py deleted file mode 100644 index c47e30ecc..000000000 --- a/benchmarks/perf-tool/okpt/io/config/parsers/test.py +++ /dev/null @@ -1,81 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# -# The OpenSearch Contributors require contributions made to -# this file be licensed under the Apache-2.0 license or a -# compatible open source license. - -"""Provides ToolParser. - -Classes: - ToolParser: Tool config parser. -""" -from dataclasses import dataclass -from io import TextIOWrapper -from typing import List - -from okpt.io.config.parsers import base -from okpt.test.steps.base import Step, StepConfig -from okpt.test.steps.factory import create_step - - -@dataclass -class TestConfig: - test_name: str - test_id: str - endpoint: str - port: int - timeout: int - num_runs: int - show_runs: bool - setup: List[Step] - steps: List[Step] - cleanup: List[Step] - - -class TestParser(base.BaseParser): - """Parser for Test config. - - Methods: - parse: Parse and validate the Test config. - """ - - def __init__(self): - super().__init__('test') - - def parse(self, file_obj: TextIOWrapper) -> TestConfig: - """See base class.""" - config_obj = super().parse(file_obj) - - implicit_step_config = dict() - if 'endpoint' in config_obj: - implicit_step_config['endpoint'] = config_obj['endpoint'] - - if 'port' in config_obj: - implicit_step_config['port'] = config_obj['port'] - - # Each step should have its own parse - take the config object and check if its valid - setup = [] - if 'setup' in config_obj: - setup = [create_step(StepConfig(step["name"], step, implicit_step_config)) for step in config_obj['setup']] - - steps = [create_step(StepConfig(step["name"], step, implicit_step_config)) for step in config_obj['steps']] - - cleanup = [] - if 'cleanup' in config_obj: - cleanup = [create_step(StepConfig(step["name"], step, implicit_step_config)) for step - in config_obj['cleanup']] - - test_config = TestConfig( - endpoint=config_obj['endpoint'], - port=config_obj['port'], - timeout=config_obj['timeout'], - test_name=config_obj['test_name'], - test_id=config_obj['test_id'], - num_runs=config_obj['num_runs'], - show_runs=config_obj['show_runs'], - setup=setup, - steps=steps, - cleanup=cleanup - ) - - return test_config diff --git a/benchmarks/perf-tool/okpt/io/config/parsers/util.py b/benchmarks/perf-tool/okpt/io/config/parsers/util.py deleted file mode 100644 index 454fec5a0..000000000 --- a/benchmarks/perf-tool/okpt/io/config/parsers/util.py +++ /dev/null @@ -1,116 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# -# The OpenSearch Contributors require contributions made to -# this file be licensed under the Apache-2.0 license or a -# compatible open source license. - -"""Utility functions for parsing""" - - -from okpt.io.config.parsers.base import ConfigurationError -from okpt.io.dataset import HDF5DataSet, BigANNNeighborDataSet, \ - BigANNVectorDataSet, DataSet, Context - - -def parse_dataset(dataset_format: str, dataset_path: str, - context: Context, custom_context=None) -> DataSet: - if dataset_format == 'hdf5': - return HDF5DataSet(dataset_path, context, custom_context) - - if dataset_format == 'bigann' and context == Context.NEIGHBORS: - return BigANNNeighborDataSet(dataset_path) - - if dataset_format == 'bigann': - return BigANNVectorDataSet(dataset_path) - - raise Exception("Unsupported data-set format") - - -def parse_string_param(key: str, first_map, second_map, default) -> str: - value = first_map.get(key) - if value is not None: - if type(value) is str: - return value - raise ConfigurationError("Invalid type for {}".format(key)) - - value = second_map.get(key) - if value is not None: - if type(value) is str: - return value - raise ConfigurationError("Invalid type for {}".format(key)) - - if default is None: - raise ConfigurationError("{} must be set".format(key)) - return default - - -def parse_int_param(key: str, first_map, second_map, default) -> int: - value = first_map.get(key) - if value is not None: - if type(value) is int: - return value - raise ConfigurationError("Invalid type for {}".format(key)) - - value = second_map.get(key) - if value is not None: - if type(value) is int: - return value - raise ConfigurationError("Invalid type for {}".format(key)) - - if default is None: - raise ConfigurationError("{} must be set".format(key)) - return default - - -def parse_bool_param(key: str, first_map, second_map, default) -> bool: - value = first_map.get(key) - if value is not None: - if type(value) is bool: - return value - raise ConfigurationError("Invalid type for {}".format(key)) - - value = second_map.get(key) - if value is not None: - if type(value) is bool: - return value - raise ConfigurationError("Invalid type for {}".format(key)) - - if default is None: - raise ConfigurationError("{} must be set".format(key)) - return default - - -def parse_dict_param(key: str, first_map, second_map, default) -> dict: - value = first_map.get(key) - if value is not None: - if type(value) is dict: - return value - raise ConfigurationError("Invalid type for {}".format(key)) - - value = second_map.get(key) - if value is not None: - if type(value) is dict: - return value - raise ConfigurationError("Invalid type for {}".format(key)) - - if default is None: - raise ConfigurationError("{} must be set".format(key)) - return default - - -def parse_list_param(key: str, first_map, second_map, default) -> list: - value = first_map.get(key) - if value is not None: - if type(value) is list: - return value - raise ConfigurationError("Invalid type for {}".format(key)) - - value = second_map.get(key) - if value is not None: - if type(value) is list: - return value - raise ConfigurationError("Invalid type for {}".format(key)) - - if default is None: - raise ConfigurationError("{} must be set".format(key)) - return default diff --git a/benchmarks/perf-tool/okpt/io/config/schemas/test.yml b/benchmarks/perf-tool/okpt/io/config/schemas/test.yml deleted file mode 100644 index 4d5c21a15..000000000 --- a/benchmarks/perf-tool/okpt/io/config/schemas/test.yml +++ /dev/null @@ -1,35 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# -# The OpenSearch Contributors require contributions made to -# this file be licensed under the Apache-2.0 license or a -# compatible open source license. - -# defined using the cerberus validation API -# https://docs.python-cerberus.org/en/stable/index.html -endpoint: - type: string - default: "localhost" -port: - type: integer - default: 9200 -timeout: - type: integer - default: 60 -test_name: - type: string -test_id: - type: string -num_runs: - type: integer - default: 1 - min: 1 - max: 10000 -show_runs: - type: boolean - default: false -setup: - type: list -steps: - type: list -cleanup: - type: list diff --git a/benchmarks/perf-tool/okpt/io/dataset.py b/benchmarks/perf-tool/okpt/io/dataset.py deleted file mode 100644 index 001563bab..000000000 --- a/benchmarks/perf-tool/okpt/io/dataset.py +++ /dev/null @@ -1,222 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# -# The OpenSearch Contributors require contributions made to -# this file be licensed under the Apache-2.0 license or a -# compatible open source license. - -"""Defines DataSet interface and implements particular formats - -A DataSet is the basic functionality that it can be read in chunks, or -read completely and reset to the start. - -Currently, we support HDF5 formats from ann-benchmarks and big-ann-benchmarks -datasets. - -Classes: - HDF5DataSet: Format used in ann-benchmarks - BigANNNeighborDataSet: Neighbor format for big-ann-benchmarks - BigANNVectorDataSet: Vector format for big-ann-benchmarks -""" -import os -from abc import ABC, ABCMeta, abstractmethod -from enum import Enum -from typing import cast -import h5py -import numpy as np - -import struct - - -class Context(Enum): - """DataSet context enum. Can be used to add additional context for how a - data-set should be interpreted. - """ - INDEX = 1 - QUERY = 2 - NEIGHBORS = 3 - CUSTOM = 4 - - -class DataSet(ABC): - """DataSet interface. Used for reading data-sets from files. - - Methods: - read: Read a chunk of data from the data-set - size: Gets the number of items in the data-set - reset: Resets internal state of data-set to beginning - """ - __metaclass__ = ABCMeta - - @abstractmethod - def read(self, chunk_size: int): - pass - - @abstractmethod - def size(self): - pass - - @abstractmethod - def reset(self): - pass - - -class HDF5DataSet(DataSet): - """ Data-set format corresponding to `ANN Benchmarks - `_ - """ - - def __init__(self, dataset_path: str, context: Context, custom_context=None): - file = h5py.File(dataset_path) - self.data = cast(h5py.Dataset, file[self._parse_context(context, custom_context)]) - self.current = 0 - - def read(self, chunk_size: int): - if self.current >= self.size(): - return None - - end_i = self.current + chunk_size - if end_i > self.size(): - end_i = self.size() - - v = cast(np.ndarray, self.data[self.current:end_i]) - self.current = end_i - return v - - def size(self): - return self.data.len() - - def reset(self): - self.current = 0 - - @staticmethod - def _parse_context(context: Context, custom_context=None) -> str: - if context == Context.NEIGHBORS: - return "neighbors" - - if context == Context.INDEX: - return "train" - - if context == Context.QUERY: - return "test" - - if context == Context.CUSTOM: - return custom_context - - raise Exception("Unsupported context") - - -class BigANNNeighborDataSet(DataSet): - """ Data-set format for neighbor data-sets for `Big ANN Benchmarks - `_""" - - def __init__(self, dataset_path: str): - self.file = open(dataset_path, 'rb') - self.file.seek(0, os.SEEK_END) - num_bytes = self.file.tell() - self.file.seek(0) - - if num_bytes < 8: - raise Exception("File is invalid") - - self.num_queries = int.from_bytes(self.file.read(4), "little") - self.k = int.from_bytes(self.file.read(4), "little") - - # According to the website, the number of bytes that will follow will - # be: num_queries X K x sizeof(uint32_t) bytes + num_queries X K x - # sizeof(float) - if (num_bytes - 8) != 2 * (self.num_queries * self.k * 4): - raise Exception("File is invalid") - - self.current = 0 - - def read(self, chunk_size: int): - if self.current >= self.size(): - return None - - end_i = self.current + chunk_size - if end_i > self.size(): - end_i = self.size() - - v = [[int.from_bytes(self.file.read(4), "little") for _ in - range(self.k)] for _ in range(end_i - self.current)] - - self.current = end_i - return v - - def size(self): - return self.num_queries - - def reset(self): - self.file.seek(8) - self.current = 0 - - -class BigANNVectorDataSet(DataSet): - """ Data-set format for vector data-sets for `Big ANN Benchmarks - `_ - """ - - def __init__(self, dataset_path: str): - self.file = open(dataset_path, 'rb') - self.file.seek(0, os.SEEK_END) - num_bytes = self.file.tell() - self.file.seek(0) - - if num_bytes < 8: - raise Exception("File is invalid") - - self.num_points = int.from_bytes(self.file.read(4), "little") - self.dimension = int.from_bytes(self.file.read(4), "little") - bytes_per_num = self._get_data_size(dataset_path) - - if (num_bytes - 8) != self.num_points * self.dimension * bytes_per_num: - raise Exception("File is invalid") - - self.reader = self._value_reader(dataset_path) - self.current = 0 - - def read(self, chunk_size: int): - if self.current >= self.size(): - return None - - end_i = self.current + chunk_size - if end_i > self.size(): - end_i = self.size() - - v = np.asarray([self._read_vector() for _ in - range(end_i - self.current)]) - self.current = end_i - return v - - def _read_vector(self): - return np.asarray([self.reader(self.file) for _ in - range(self.dimension)]) - - def size(self): - return self.num_points - - def reset(self): - self.file.seek(8) # Seek to 8 bytes to skip re-reading metadata - self.current = 0 - - @staticmethod - def _get_data_size(file_name): - ext = file_name.split('.')[-1] - if ext == "u8bin": - return 1 - - if ext == "fbin": - return 4 - - raise Exception("Unknown extension") - - @staticmethod - def _value_reader(file_name): - ext = file_name.split('.')[-1] - if ext == "u8bin": - return lambda file: float(int.from_bytes(file.read(1), "little")) - - if ext == "fbin": - return lambda file: struct.unpack(' TextIOWrapper: - """Given a file path, get a readable file object. - - Args: - file path - - Returns: - Writeable file object - """ - return open(path, 'r', encoding='UTF-8') - - -def parse_yaml(file: TextIOWrapper) -> Dict[str, Any]: - """Parses YAML file from file object. - - Args: - file: file object to parse - - Returns: - A dict representing the YAML file. - """ - return yaml.load(file, Loader=yaml.SafeLoader) - - -def parse_yaml_from_path(path: str) -> Dict[str, Any]: - """Parses YAML file from file path. - - Args: - path: file path to parse - - Returns: - A dict representing the YAML file. - """ - file = reader.get_file_obj(path) - return parse_yaml(file) - - -def parse_json(file: TextIOWrapper) -> Dict[str, Any]: - """Parses JSON file from file object. - - Args: - file: file object to parse - - Returns: - A dict representing the JSON file. - """ - return json.load(file) - - -def parse_json_from_path(path: str) -> Dict[str, Any]: - """Parses JSON file from file path. - - Args: - path: file path to parse - - Returns: - A dict representing the JSON file. - """ - file = reader.get_file_obj(path) - return json.load(file) diff --git a/benchmarks/perf-tool/okpt/io/utils/writer.py b/benchmarks/perf-tool/okpt/io/utils/writer.py deleted file mode 100644 index 1f14bfd94..000000000 --- a/benchmarks/perf-tool/okpt/io/utils/writer.py +++ /dev/null @@ -1,40 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# -# The OpenSearch Contributors require contributions made to -# this file be licensed under the Apache-2.0 license or a -# compatible open source license. -"""Provides functions for writing to file. - -Functions: - get_file_obj(): Get a writeable file object. - write_json(): Writes a python dictionary to a JSON file -""" - -import json -from io import TextIOWrapper -from typing import Any, Dict, TextIO, Union - - -def get_file_obj(path: str) -> TextIOWrapper: - """Get a writeable file object from a file path. - - Args: - file path - - Returns: - Writeable file object - """ - return open(path, 'w', encoding='UTF-8') - - -def write_json(data: Dict[str, Any], - file: Union[TextIOWrapper, TextIO], - pretty=False): - """Writes a dictionary to a JSON file. - - Args: - data: A dict to write to JSON. - file: Path of output file. - """ - indent = 2 if pretty else 0 - json.dump(data, file, indent=indent) diff --git a/benchmarks/perf-tool/okpt/main.py b/benchmarks/perf-tool/okpt/main.py deleted file mode 100644 index 3e6e022d4..000000000 --- a/benchmarks/perf-tool/okpt/main.py +++ /dev/null @@ -1,55 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# -# The OpenSearch Contributors require contributions made to -# this file be licensed under the Apache-2.0 license or a -# compatible open source license. - -""" Runner script that serves as the main controller of the testing tool.""" - -import logging -import sys -from typing import cast - -from okpt.diff import diff -from okpt.io import args -from okpt.io.config.parsers import test -from okpt.io.utils import reader, writer -from okpt.test import runner - - -def main(): - """Main function of entry module.""" - cli_args = args.get_args() - output = cli_args.output - if cli_args.log: - log_level = getattr(logging, cli_args.log.upper()) - logging.basicConfig(level=log_level) - - if cli_args.command == 'test': - cli_args = cast(args.TestArgs, cli_args) - - # parse config - parser = test.TestParser() - test_config = parser.parse(cli_args.config) - logging.info('Configs are valid.') - - # run tests - test_runner = runner.TestRunner(test_config=test_config) - test_result = test_runner.execute() - - # write test results - logging.debug( - f'Test Result:\n {writer.write_json(test_result, sys.stdout, pretty=True)}' - ) - writer.write_json(test_result, output, pretty=True) - elif cli_args.command == 'diff': - cli_args = cast(args.DiffArgs, cli_args) - - # parse test results - base_result = reader.parse_json(cli_args.base_result) - changed_result = reader.parse_json(cli_args.changed_result) - - # get diff - diff_result = diff.Diff(base_result, changed_result, - cli_args.metadata).diff() - writer.write_json(data=diff_result, file=output, pretty=True) diff --git a/benchmarks/perf-tool/okpt/test/__init__.py b/benchmarks/perf-tool/okpt/test/__init__.py deleted file mode 100644 index ff4fd04d1..000000000 --- a/benchmarks/perf-tool/okpt/test/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# -# The OpenSearch Contributors require contributions made to -# this file be licensed under the Apache-2.0 license or a -# compatible open source license. diff --git a/benchmarks/perf-tool/okpt/test/profile.py b/benchmarks/perf-tool/okpt/test/profile.py deleted file mode 100644 index d96860f9a..000000000 --- a/benchmarks/perf-tool/okpt/test/profile.py +++ /dev/null @@ -1,86 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# -# The OpenSearch Contributors require contributions made to -# this file be licensed under the Apache-2.0 license or a -# compatible open source license. - -"""Provides decorators to profile functions. - -The decorators work by adding a `measureable` (time, memory, etc) field to a -dictionary returned by the wrapped function. So the wrapped functions must -return a dictionary in order to be profiled. -""" -import functools -import time -from typing import Callable - - -class TimerStoppedWithoutStartingError(Exception): - """Error raised when Timer is stopped without having been started.""" - - def __init__(self): - super().__init__() - self.message = 'Timer must call start() before calling end().' - - -class _Timer(): - """Timer class for timing. - - Methods: - start: Starts the timer. - end: Stops the timer and returns the time elapsed since start. - - Raises: - TimerStoppedWithoutStartingError: Timer must start before ending. - """ - - def __init__(self): - self.start_time = None - - def start(self): - """Starts the timer.""" - self.start_time = time.perf_counter() - - def end(self) -> float: - """Stops the timer. - - Returns: - The time elapsed in milliseconds. - """ - # ensure timer has started before ending - if self.start_time is None: - raise TimerStoppedWithoutStartingError() - - elapsed = (time.perf_counter() - self.start_time) * 1000 - self.start_time = None - return elapsed - - -def took(f: Callable): - """Profiles a functions execution time. - - Args: - f: Function to profile. - - Returns: - A function that wraps the passed in function and adds a time took field - to the return value. - """ - - @functools.wraps(f) - def wrapper(*args, **kwargs): - """Wrapper function.""" - timer = _Timer() - timer.start() - result = f(*args, **kwargs) - time_took = timer.end() - - # if result already has a `took` field, don't modify the result - if isinstance(result, dict) and 'took' in result: - return result - # `result` may not be a dictionary, so it may not be unpackable - elif isinstance(result, dict): - return {**result, 'took': time_took} - return {'took': time_took} - - return wrapper diff --git a/benchmarks/perf-tool/okpt/test/runner.py b/benchmarks/perf-tool/okpt/test/runner.py deleted file mode 100644 index 150154691..000000000 --- a/benchmarks/perf-tool/okpt/test/runner.py +++ /dev/null @@ -1,107 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# -# The OpenSearch Contributors require contributions made to -# this file be licensed under the Apache-2.0 license or a -# compatible open source license. - -"""Provides a test runner class.""" -import logging -import platform -import sys -from datetime import datetime -from typing import Any, Dict, List - -import psutil - -from okpt.io.config.parsers import test -from okpt.test.test import Test, get_avg - - -def _aggregate_runs(runs: List[Dict[str, Any]]): - """Aggregates and averages a list of test results. - - Args: - results: A list of test results. - num_runs: Number of times the tests were ran. - - Returns: - A dictionary containing the averages of the test results. - """ - aggregate: Dict[str, Any] = {} - for run in runs: - for key, value in run.items(): - if key in aggregate: - aggregate[key].append(value) - else: - aggregate[key] = [value] - - aggregate = {key: get_avg(value) for key, value in aggregate.items()} - return aggregate - - -class TestRunner: - """Test runner class for running tests and aggregating the results. - - Methods: - execute: Run the tests and aggregate the results. - """ - - def __init__(self, test_config: test.TestConfig): - """"Initializes test state.""" - self.test_config = test_config - self.test = Test(test_config) - - def _get_metadata(self): - """"Retrieves the test metadata.""" - svmem = psutil.virtual_memory() - return { - 'test_name': - self.test_config.test_name, - 'test_id': - self.test_config.test_id, - 'date': - datetime.now().strftime('%m/%d/%Y %H:%M:%S'), - 'python_version': - sys.version, - 'os_version': - platform.platform(), - 'processor': - platform.processor() + ', ' + - str(psutil.cpu_count(logical=True)) + ' cores', - 'memory': - str(svmem.used) + ' (used) / ' + str(svmem.available) + - ' (available) / ' + str(svmem.total) + ' (total)', - } - - def execute(self) -> Dict[str, Any]: - """Runs the tests and aggregates the results. - - Returns: - A dictionary containing the aggregate of test results. - """ - logging.info('Setting up tests.') - self.test.setup() - logging.info('Beginning to run tests.') - runs = [] - for i in range(self.test_config.num_runs): - logging.info( - f'Running test {i + 1} of {self.test_config.num_runs}' - ) - runs.append(self.test.execute()) - - logging.info('Finished running tests.') - aggregate = _aggregate_runs(runs) - - # add metadata to test results - test_result = { - 'metadata': - self._get_metadata(), - 'results': - aggregate - } - - # include info about all test runs if specified in config - if self.test_config.show_runs: - test_result['runs'] = runs - - return test_result diff --git a/benchmarks/perf-tool/okpt/test/steps/base.py b/benchmarks/perf-tool/okpt/test/steps/base.py deleted file mode 100644 index 829980421..000000000 --- a/benchmarks/perf-tool/okpt/test/steps/base.py +++ /dev/null @@ -1,60 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# -# The OpenSearch Contributors require contributions made to -# this file be licensed under the Apache-2.0 license or a -# compatible open source license. -"""Provides base Step interface.""" - -from dataclasses import dataclass -from typing import Any, Dict, List - -from okpt.test import profile - - -@dataclass -class StepConfig: - step_name: str - config: Dict[str, object] - implicit_config: Dict[str, object] - - -class Step: - """Test step interface. - - Attributes: - label: Name of the step. - - Methods: - execute: Run the step and return a step response with the label and - corresponding measures. - """ - - label = 'base_step' - - def __init__(self, step_config: StepConfig): - self.step_config = step_config - - def _action(self): - """Step logic/behavior to be executed and profiled.""" - pass - - def _get_measures(self) -> List[str]: - """Gets the measures for a particular test""" - pass - - def execute(self) -> List[Dict[str, Any]]: - """Execute step logic while profiling various measures. - - Returns: - Dict containing step label and various step measures. - """ - action = self._action - - # profile the action with measure decorators - add if necessary - action = getattr(profile, 'took')(action) - - result = action() - if isinstance(result, dict): - return [{'label': self.label, **result}] - - raise ValueError('Invalid return by a step') diff --git a/benchmarks/perf-tool/okpt/test/steps/factory.py b/benchmarks/perf-tool/okpt/test/steps/factory.py deleted file mode 100644 index 2033f2672..000000000 --- a/benchmarks/perf-tool/okpt/test/steps/factory.py +++ /dev/null @@ -1,50 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# -# The OpenSearch Contributors require contributions made to -# this file be licensed under the Apache-2.0 license or a -# compatible open source license. -"""Factory for creating steps.""" - -from okpt.io.config.parsers.base import ConfigurationError -from okpt.test.steps.base import Step, StepConfig - -from okpt.test.steps.steps import CreateIndexStep, DisableRefreshStep, RefreshIndexStep, DeleteIndexStep, \ - TrainModelStep, DeleteModelStep, ForceMergeStep, ClearCacheStep, IngestStep, IngestMultiFieldStep, \ - IngestNestedFieldStep, QueryStep, QueryWithFilterStep, QueryNestedFieldStep, GetStatsStep, WarmupStep - - -def create_step(step_config: StepConfig) -> Step: - if step_config.step_name == CreateIndexStep.label: - return CreateIndexStep(step_config) - elif step_config.step_name == DisableRefreshStep.label: - return DisableRefreshStep(step_config) - elif step_config.step_name == RefreshIndexStep.label: - return RefreshIndexStep(step_config) - elif step_config.step_name == TrainModelStep.label: - return TrainModelStep(step_config) - elif step_config.step_name == DeleteModelStep.label: - return DeleteModelStep(step_config) - elif step_config.step_name == DeleteIndexStep.label: - return DeleteIndexStep(step_config) - elif step_config.step_name == IngestStep.label: - return IngestStep(step_config) - elif step_config.step_name == IngestMultiFieldStep.label: - return IngestMultiFieldStep(step_config) - elif step_config.step_name == IngestNestedFieldStep.label: - return IngestNestedFieldStep(step_config) - elif step_config.step_name == QueryStep.label: - return QueryStep(step_config) - elif step_config.step_name == QueryWithFilterStep.label: - return QueryWithFilterStep(step_config) - elif step_config.step_name == QueryNestedFieldStep.label: - return QueryNestedFieldStep(step_config) - elif step_config.step_name == ForceMergeStep.label: - return ForceMergeStep(step_config) - elif step_config.step_name == ClearCacheStep.label: - return ClearCacheStep(step_config) - elif step_config.step_name == GetStatsStep.label: - return GetStatsStep(step_config) - elif step_config.step_name == WarmupStep.label: - return WarmupStep(step_config) - - raise ConfigurationError(f'Invalid step {step_config.step_name}') diff --git a/benchmarks/perf-tool/okpt/test/steps/steps.py b/benchmarks/perf-tool/okpt/test/steps/steps.py deleted file mode 100644 index 99b2728dc..000000000 --- a/benchmarks/perf-tool/okpt/test/steps/steps.py +++ /dev/null @@ -1,987 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# -# The OpenSearch Contributors require contributions made to -# this file be licensed under the Apache-2.0 license or a -# compatible open source license. -"""Provides steps for OpenSearch tests. - -Some OpenSearch operations return a `took` field in the response body, -so the profiling decorators aren't needed for some functions. -""" -import json -from abc import abstractmethod -from typing import Any, Dict, List - -import numpy as np -import requests -import time - -from opensearchpy import OpenSearch, RequestsHttpConnection - -from okpt.io.config.parsers.base import ConfigurationError -from okpt.io.config.parsers.util import parse_string_param, parse_int_param, parse_dataset, parse_bool_param, \ - parse_list_param -from okpt.io.dataset import Context -from okpt.io.utils.reader import parse_json_from_path -from okpt.test.steps import base -from okpt.test.steps.base import StepConfig - - -class OpenSearchStep(base.Step): - """See base class.""" - - def __init__(self, step_config: StepConfig): - super().__init__(step_config) - self.endpoint = parse_string_param('endpoint', step_config.config, - step_config.implicit_config, - 'localhost') - default_port = 9200 if self.endpoint == 'localhost' else 80 - self.port = parse_int_param('port', step_config.config, - step_config.implicit_config, default_port) - self.timeout = parse_int_param('timeout', step_config.config, {}, 60) - self.opensearch = get_opensearch_client(str(self.endpoint), - int(self.port), int(self.timeout)) - - -class CreateIndexStep(OpenSearchStep): - """See base class.""" - - label = 'create_index' - - def __init__(self, step_config: StepConfig): - super().__init__(step_config) - self.index_name = parse_string_param('index_name', step_config.config, - {}, None) - index_spec = parse_string_param('index_spec', step_config.config, {}, - None) - self.body = parse_json_from_path(index_spec) - if self.body is None: - raise ConfigurationError('Index body must be passed in') - - def _action(self): - """Creates an OpenSearch index, applying the index settings/mappings. - - Returns: - An OpenSearch index creation response body. - """ - self.opensearch.indices.create(index=self.index_name, body=self.body) - return {} - - def _get_measures(self) -> List[str]: - return ['took'] - - -class DisableRefreshStep(OpenSearchStep): - """See base class.""" - - label = 'disable_refresh' - - def _action(self): - """Disables the refresh interval for an OpenSearch index. - - Returns: - An OpenSearch index settings update response body. - """ - self.opensearch.indices.put_settings( - body={'index': { - 'refresh_interval': -1 - }}) - - return {} - - def _get_measures(self) -> List[str]: - return ['took'] - - -class RefreshIndexStep(OpenSearchStep): - """See base class.""" - - label = 'refresh_index' - - def __init__(self, step_config: StepConfig): - super().__init__(step_config) - self.index_name = parse_string_param('index_name', step_config.config, - {}, None) - - def _action(self): - while True: - try: - self.opensearch.indices.refresh(index=self.index_name) - return {'store_kb': get_index_size_in_kb(self.opensearch, - self.index_name)} - except: - pass - - def _get_measures(self) -> List[str]: - return ['took', 'store_kb'] - - -class ForceMergeStep(OpenSearchStep): - """See base class.""" - - label = 'force_merge' - - def __init__(self, step_config: StepConfig): - super().__init__(step_config) - self.index_name = parse_string_param('index_name', step_config.config, - {}, None) - self.max_num_segments = parse_int_param('max_num_segments', - step_config.config, {}, None) - - def _action(self): - while True: - try: - self.opensearch.indices.forcemerge( - index=self.index_name, - max_num_segments=self.max_num_segments) - return {} - except: - pass - - def _get_measures(self) -> List[str]: - return ['took'] - -class ClearCacheStep(OpenSearchStep): - """See base class.""" - - label = 'clear_cache' - - def __init__(self, step_config: StepConfig): - super().__init__(step_config) - self.index_name = parse_string_param('index_name', step_config.config, - {}, None) - - def _action(self): - while True: - try: - self.opensearch.indices.clear_cache( - index=self.index_name) - return {} - except: - pass - - def _get_measures(self) -> List[str]: - return ['took'] - - -class WarmupStep(OpenSearchStep): - """See base class.""" - - label = 'warmup_operation' - - def __init__(self, step_config: StepConfig): - super().__init__(step_config) - self.index_name = parse_string_param('index_name', step_config.config, {}, - None) - - def _action(self): - """Performs warmup operation on an index.""" - warmup_operation(self.endpoint, self.port, self.index_name) - return {} - - def _get_measures(self) -> List[str]: - return ['took'] - - -class TrainModelStep(OpenSearchStep): - """See base class.""" - - label = 'train_model' - - def __init__(self, step_config: StepConfig): - super().__init__(step_config) - - self.model_id = parse_string_param('model_id', step_config.config, {}, - 'Test') - self.train_index_name = parse_string_param('train_index', - step_config.config, {}, None) - self.train_index_field = parse_string_param('train_field', - step_config.config, {}, - None) - self.dimension = parse_int_param('dimension', step_config.config, {}, - None) - self.description = parse_string_param('description', step_config.config, - {}, 'Default') - self.max_training_vector_count = parse_int_param( - 'max_training_vector_count', step_config.config, {}, 10000000000000) - - method_spec = parse_string_param('method_spec', step_config.config, {}, - None) - self.method = parse_json_from_path(method_spec) - if self.method is None: - raise ConfigurationError('method must be passed in') - - def _action(self): - """Train a model for an index. - - Returns: - The trained model - """ - - # Build body - body = { - 'training_index': self.train_index_name, - 'training_field': self.train_index_field, - 'description': self.description, - 'dimension': self.dimension, - 'method': self.method, - 'max_training_vector_count': self.max_training_vector_count - } - - # So, we trained the model. Now we need to wait until we have to wait - # until the model is created. Poll every - # 1/10 second - requests.post('http://' + self.endpoint + ':' + str(self.port) + - '/_plugins/_knn/models/' + str(self.model_id) + '/_train', - json.dumps(body), - headers={'content-type': 'application/json'}) - - sleep_time = 0.1 - timeout = 100000 - i = 0 - while i < timeout: - time.sleep(sleep_time) - model_response = get_model(self.endpoint, self.port, self.model_id) - if 'state' in model_response.keys() and model_response['state'] == \ - 'created': - return {} - i += 1 - - raise TimeoutError('Failed to create model') - - def _get_measures(self) -> List[str]: - return ['took'] - - -class DeleteModelStep(OpenSearchStep): - """See base class.""" - - label = 'delete_model' - - def __init__(self, step_config: StepConfig): - super().__init__(step_config) - - self.model_id = parse_string_param('model_id', step_config.config, {}, - 'Test') - - def _action(self): - """Train a model for an index. - - Returns: - The trained model - """ - delete_model(self.endpoint, self.port, self.model_id) - return {} - - def _get_measures(self) -> List[str]: - return ['took'] - - -class DeleteIndexStep(OpenSearchStep): - """See base class.""" - - label = 'delete_index' - - def __init__(self, step_config: StepConfig): - super().__init__(step_config) - - self.index_name = parse_string_param('index_name', step_config.config, - {}, None) - - def _action(self): - """Delete the index - - Returns: - An empty dict - """ - delete_index(self.opensearch, self.index_name) - return {} - - def _get_measures(self) -> List[str]: - return ['took'] - - -class BaseIngestStep(OpenSearchStep): - """See base class.""" - def __init__(self, step_config: StepConfig): - super().__init__(step_config) - self.index_name = parse_string_param('index_name', step_config.config, - {}, None) - self.field_name = parse_string_param('field_name', step_config.config, - {}, None) - self.bulk_size = parse_int_param('bulk_size', step_config.config, {}, - 300) - self.implicit_config = step_config.implicit_config - dataset_format = parse_string_param('dataset_format', - step_config.config, {}, 'hdf5') - dataset_path = parse_string_param('dataset_path', step_config.config, - {}, None) - self.dataset = parse_dataset(dataset_format, dataset_path, - Context.INDEX) - - self.input_doc_count = parse_int_param('doc_count', step_config.config, {}, - self.dataset.size()) - self.doc_count = min(self.input_doc_count, self.dataset.size()) - - def _action(self): - - def action(doc_id): - return {'index': {'_index': self.index_name, '_id': doc_id}} - - # Maintain minimal state outside of this loop. For large data sets, too - # much state may cause out of memory failure - for i in range(0, self.doc_count, self.bulk_size): - partition = self.dataset.read(self.bulk_size) - self._handle_data_bulk(partition, action, i) - self.dataset.reset() - - return {} - - def _get_measures(self) -> List[str]: - return ['took'] - - @abstractmethod - def _handle_data_bulk(self, partition, action, i): - pass - - -class IngestStep(BaseIngestStep): - """See base class.""" - - label = 'ingest' - - def _handle_data_bulk(self, partition, action, i): - if partition is None: - return - body = bulk_transform(partition, self.field_name, action, i) - bulk_index(self.opensearch, self.index_name, body) - - -class IngestMultiFieldStep(BaseIngestStep): - """See base class.""" - - label = 'ingest_multi_field' - - def __init__(self, step_config: StepConfig): - super().__init__(step_config) - - dataset_path = parse_string_param('dataset_path', step_config.config, - {}, None) - - self.attributes_dataset_name = parse_string_param('attributes_dataset_name', - step_config.config, {}, None) - - self.attributes_dataset = parse_dataset('hdf5', dataset_path, - Context.CUSTOM, self.attributes_dataset_name) - - self.attribute_spec = parse_list_param('attribute_spec', - step_config.config, {}, []) - - self.partition_attr = self.attributes_dataset.read(self.doc_count) - self.action_buffer = None - - def _handle_data_bulk(self, partition, action, i): - if partition is None: - return - body = self.bulk_transform_with_attributes(partition, self.partition_attr, self.field_name, - action, i, self.attribute_spec) - bulk_index(self.opensearch, self.index_name, body) - - def bulk_transform_with_attributes(self, partition: np.ndarray, partition_attr, field_name: str, - action, offset: int, attributes_def) -> List[Dict[str, Any]]: - """Partitions and transforms a list of vectors into OpenSearch's bulk - injection format. - Args: - partition: An array of vectors to transform. - partition_attr: dictionary of additional data to transform - field_name: field name for action - action: Bulk API action. - offset: to start counting from - attributes_def: definition of additional doc fields - Returns: - An array of transformed vectors in bulk format. - """ - actions = [] - _ = [ - actions.extend([action(i + offset), None]) - for i in range(len(partition)) - ] - idx = 1 - part_list = partition.tolist() - for i in range(len(partition)): - actions[idx] = {field_name: part_list[i]} - attr_idx = i + offset - attr_def_idx = 0 - for attribute in attributes_def: - attr_def_name = attribute['name'] - attr_def_type = attribute['type'] - - if attr_def_type == 'str': - val = partition_attr[attr_idx][attr_def_idx].decode() - if val != 'None': - actions[idx][attr_def_name] = val - elif attr_def_type == 'int': - val = int(partition_attr[attr_idx][attr_def_idx].decode()) - actions[idx][attr_def_name] = val - attr_def_idx += 1 - idx += 2 - - return actions - - -class IngestNestedFieldStep(BaseIngestStep): - """See base class.""" - - label = 'ingest_nested_field' - - def __init__(self, step_config: StepConfig): - super().__init__(step_config) - - dataset_path = parse_string_param('dataset_path', step_config.config, - {}, None) - - self.attributes_dataset_name = parse_string_param('attributes_dataset_name', - step_config.config, {}, None) - - self.attributes_dataset = parse_dataset('hdf5', dataset_path, - Context.CUSTOM, self.attributes_dataset_name) - - self.attribute_spec = parse_list_param('attribute_spec', - step_config.config, {}, []) - - self.partition_attr = self.attributes_dataset.read(self.doc_count) - - if self.dataset.size() != self.doc_count: - raise ValueError("custom doc_count is not supported for nested field") - self.action_buffer = None - self.action_parent_id = None - self.count = 0 - - def _handle_data_bulk(self, partition, action, i): - if partition is None: - return - body = self.bulk_transform_with_nested(partition, self.partition_attr, self.field_name, - action, i, self.attribute_spec) - if len(body) > 0: - bulk_index(self.opensearch, self.index_name, body) - - def bulk_transform_with_nested(self, partition: np.ndarray, partition_attr, field_name: str, - action, offset: int, attributes_def) -> List[Dict[str, Any]]: - """Partitions and transforms a list of vectors into OpenSearch's bulk - injection format. - Args: - partition: An array of vectors to transform. - partition_attr: dictionary of additional data to transform - field_name: field name for action - action: Bulk API action. - offset: to start counting from - attributes_def: definition of additional doc fields - Returns: - An array of transformed vectors in bulk format. - """ - # offset is index of start row. We need number of parent doc - 1. - # The number of parent document can be calculated by using partition_attr data. - # We need to keep the last parent doc aside so that additional data can be added later. - parent_id_idx = next((index for (index, d) in enumerate(attributes_def) if d.get('name') == 'parent_id'), None) - if parent_id_idx is None: - raise ValueError("parent_id should be provided as attribute spec") - if attributes_def[parent_id_idx]['type'] != 'int': - raise ValueError("parent_id should be int type") - - first_index = offset - last_index = offset + len(partition) - 1 - num_of_actions = int(partition_attr[last_index][parent_id_idx].decode()) - int(partition_attr[first_index][parent_id_idx].decode()) - if self.action_buffer is None: - self.action_buffer = {"nested_field": []} - self.action_parent_id = int(partition_attr[first_index][parent_id_idx].decode()) - - actions = [] - _ = [ - actions.extend([action(i + self.action_parent_id), None]) - for i in range(num_of_actions) - ] - - idx = 1 - part_list = partition.tolist() - for i in range(len(partition)): - self.count += 1 - nested = {field_name: part_list[i]} - attr_idx = i + offset - attr_def_idx = 0 - current_parent_id = None - for attribute in attributes_def: - attr_def_name = attribute['name'] - attr_def_type = attribute['type'] - if attr_def_name == "parent_id": - current_parent_id = int(partition_attr[attr_idx][attr_def_idx].decode()) - attr_def_idx += 1 - continue - - if attr_def_type == 'str': - val = partition_attr[attr_idx][attr_def_idx].decode() - if val != 'None': - nested[attr_def_name] = val - elif attr_def_type == 'int': - val = int(partition_attr[attr_idx][attr_def_idx].decode()) - nested[attr_def_name] = val - attr_def_idx += 1 - - if self.action_parent_id == current_parent_id: - self.action_buffer["nested_field"].append(nested) - else: - actions.extend([action(self.action_parent_id), self.action_buffer]) - self.action_buffer = {"nested_field": []} - self.action_buffer["nested_field"].append(nested) - self.action_parent_id = current_parent_id - idx += 2 - - if self.count == self.doc_count: - actions.extend([action(self.action_parent_id), self.action_buffer]) - - return actions - - -class BaseQueryStep(OpenSearchStep): - """See base class.""" - - def __init__(self, step_config: StepConfig): - super().__init__(step_config) - self.k = parse_int_param('k', step_config.config, {}, 100) - self.r = parse_int_param('r', step_config.config, {}, 1) - self.index_name = parse_string_param('index_name', step_config.config, - {}, None) - self.field_name = parse_string_param('field_name', step_config.config, - {}, None) - self.calculate_recall = parse_bool_param('calculate_recall', - step_config.config, {}, False) - dataset_format = parse_string_param('dataset_format', - step_config.config, {}, 'hdf5') - dataset_path = parse_string_param('dataset_path', - step_config.config, {}, None) - self.dataset = parse_dataset(dataset_format, dataset_path, - Context.QUERY) - - input_query_count = parse_int_param('query_count', - step_config.config, {}, - self.dataset.size()) - self.query_count = min(input_query_count, self.dataset.size()) - - self.neighbors_format = parse_string_param('neighbors_format', - step_config.config, {}, 'hdf5') - self.neighbors_path = parse_string_param('neighbors_path', - step_config.config, {}, None) - - def _action(self): - - results = {} - query_responses = [] - for _ in range(self.query_count): - query = self.dataset.read(1) - if query is None: - break - query_responses.append( - query_index(self.opensearch, self.index_name, - self.get_body(query[0]) , self.get_exclude_fields())) - - results['took'] = [ - float(query_response['took']) for query_response in query_responses - ] - results['client_time'] = [ - float(query_response['client_time']) for query_response in query_responses - ] - results['memory_kb'] = get_cache_size_in_kb(self.endpoint, self.port) - - if self.calculate_recall: - ids = [[int(hit['_id']) - for hit in query_response['hits']['hits']] - for query_response in query_responses] - results['recall@K'] = recall_at_r(ids, self.neighbors, - self.k, self.k, self.query_count) - self.neighbors.reset() - results[f'recall@{str(self.r)}'] = recall_at_r( - ids, self.neighbors, self.r, self.k, self.query_count) - self.neighbors.reset() - - self.dataset.reset() - - return results - - def _get_measures(self) -> List[str]: - measures = ['took', 'memory_kb', 'client_time'] - - if self.calculate_recall: - measures.extend(['recall@K', f'recall@{str(self.r)}']) - - return measures - - @abstractmethod - def get_body(self, vec): - pass - - def get_exclude_fields(self): - return [self.field_name] - -class QueryStep(BaseQueryStep): - """See base class.""" - - label = 'query' - - def __init__(self, step_config: StepConfig): - super().__init__(step_config) - self.neighbors = parse_dataset(self.neighbors_format, self.neighbors_path, - Context.NEIGHBORS) - self.implicit_config = step_config.implicit_config - - def get_body(self, vec): - return { - 'size': self.k, - 'query': { - 'knn': { - self.field_name: { - 'vector': vec, - 'k': self.k - } - } - } - } - - -class QueryWithFilterStep(BaseQueryStep): - """See base class.""" - - label = 'query_with_filter' - - def __init__(self, step_config: StepConfig): - super().__init__(step_config) - - neighbors_dataset = parse_string_param('neighbors_dataset', - step_config.config, {}, None) - - self.neighbors = parse_dataset(self.neighbors_format, self.neighbors_path, - Context.CUSTOM, neighbors_dataset) - - self.filter_type = parse_string_param('filter_type', step_config.config, {}, 'SCRIPT') - self.filter_spec = parse_string_param('filter_spec', step_config.config, {}, None) - self.score_script_similarity = parse_string_param('score_script_similarity', step_config.config, {}, 'l2') - - self.implicit_config = step_config.implicit_config - - def get_body(self, vec): - filter_json = json.load(open(self.filter_spec)) - if self.filter_type == 'FILTER': - return { - 'size': self.k, - 'query': { - 'knn': { - self.field_name: { - 'vector': vec, - 'k': self.k, - 'filter': filter_json - } - } - } - } - elif self.filter_type == 'SCRIPT': - return { - 'size': self.k, - 'query': { - 'script_score': { - 'query': { - 'bool': { - 'filter': filter_json - } - }, - 'script': { - 'source': 'knn_score', - 'lang': 'knn', - 'params': { - 'field': self.field_name, - 'query_value': vec, - 'space_type': self.score_script_similarity - } - } - } - } - } - elif self.filter_type == 'BOOL_POST_FILTER': - return { - 'size': self.k, - 'query': { - 'bool': { - 'filter': filter_json, - 'must': [ - { - 'knn': { - self.field_name: { - 'vector': vec, - 'k': self.k - } - } - } - ] - } - } - } - else: - raise ConfigurationError('Not supported filter type {}'.format(self.filter_type)) - -class QueryNestedFieldStep(BaseQueryStep): - """See base class.""" - - label = 'query_nested_field' - - def __init__(self, step_config: StepConfig): - super().__init__(step_config) - - neighbors_dataset = parse_string_param('neighbors_dataset', - step_config.config, {}, None) - - self.neighbors = parse_dataset(self.neighbors_format, self.neighbors_path, - Context.CUSTOM, neighbors_dataset) - - self.implicit_config = step_config.implicit_config - - def get_body(self, vec): - return { - 'size': self.k, - 'query': { - 'nested': { - 'path': 'nested_field', - 'query': { - 'knn': { - 'nested_field.' + self.field_name: { - 'vector': vec, - 'k': self.k - } - } - } - } - } - } - -class GetStatsStep(OpenSearchStep): - """See base class.""" - - label = 'get_stats' - - def __init__(self, step_config: StepConfig): - super().__init__(step_config) - - self.index_name = parse_string_param('index_name', step_config.config, - {}, None) - - def _action(self): - """Get stats for cluster/index etc. - - Returns: - Stats with following info: - - number of committed and search segments in the index - """ - results = {} - segment_stats = get_segment_stats(self.opensearch, self.index_name) - shards = segment_stats["indices"][self.index_name]["shards"] - num_of_committed_segments = 0 - num_of_search_segments = 0; - for shard_key in shards.keys(): - for segment in shards[shard_key]: - num_of_committed_segments += segment["num_committed_segments"] - num_of_search_segments += segment["num_search_segments"] - - results['committed_segments'] = num_of_committed_segments - results['search_segments'] = num_of_search_segments - return results - - def _get_measures(self) -> List[str]: - return ['committed_segments', 'search_segments'] - -# Helper functions - (AKA not steps) -def bulk_transform(partition: np.ndarray, field_name: str, action, - offset: int) -> List[Dict[str, Any]]: - """Partitions and transforms a list of vectors into OpenSearch's bulk - injection format. - Args: - offset: to start counting from - partition: An array of vectors to transform. - field_name: field name for action - action: Bulk API action. - Returns: - An array of transformed vectors in bulk format. - """ - actions = [] - _ = [ - actions.extend([action(i + offset), None]) - for i in range(len(partition)) - ] - actions[1::2] = [{field_name: vec} for vec in partition.tolist()] - return actions - - -def delete_index(opensearch: OpenSearch, index_name: str): - """Deletes an OpenSearch index. - - Args: - opensearch: An OpenSearch client. - index_name: Name of the OpenSearch index to be deleted. - """ - opensearch.indices.delete(index=index_name, ignore=[400, 404]) - - -def get_model(endpoint, port, model_id): - """ - Retrieve a model from an OpenSearch cluster - Args: - endpoint: Endpoint OpenSearch is running on - port: Port OpenSearch is running on - model_id: ID of model to be deleted - Returns: - Get model response - """ - response = requests.get('http://' + endpoint + ':' + str(port) + - '/_plugins/_knn/models/' + model_id, - headers={'content-type': 'application/json'}) - return response.json() - - -def delete_model(endpoint, port, model_id): - """ - Deletes a model from OpenSearch cluster - Args: - endpoint: Endpoint OpenSearch is running on - port: Port OpenSearch is running on - model_id: ID of model to be deleted - Returns: - Deleted model response - """ - response = requests.delete('http://' + endpoint + ':' + str(port) + - '/_plugins/_knn/models/' + model_id, - headers={'content-type': 'application/json'}) - return response.json() - - -def warmup_operation(endpoint, port, index): - """ - Performs warmup operation on index to load native library files - of that index to reduce query latencies. - Args: - endpoint: Endpoint OpenSearch is running on - port: Port OpenSearch is running on - index: index name - Returns: - number of shards the plugin succeeded and failed to warm up. - """ - response = requests.get('http://' + endpoint + ':' + str(port) + - '/_plugins/_knn/warmup/' + index, - headers={'content-type': 'application/json'}) - return response.json() - - -def get_opensearch_client(endpoint: str, port: int, timeout=60): - """ - Get an opensearch client from an endpoint and port - Args: - endpoint: Endpoint OpenSearch is running on - port: Port OpenSearch is running on - timeout: timeout for OpenSearch client, default value 60 - Returns: - OpenSearch client - - """ - # TODO: fix for security in the future - return OpenSearch( - hosts=[{ - 'host': endpoint, - 'port': port - }], - use_ssl=False, - verify_certs=False, - connection_class=RequestsHttpConnection, - timeout=timeout, - ) - - -def recall_at_r(results, neighbor_dataset, r, k, query_count): - """ - Calculates the recall@R for a set of queries against a ground truth nearest - neighbor set - Args: - results: 2D list containing ids of results returned by OpenSearch. - results[i][j] i refers to query, j refers to - result in the query - neighbor_dataset: 2D dataset containing ids of the true nearest - neighbors for a set of queries - r: number of top results to check if they are in the ground truth k-NN - set. - k: k value for the query - query_count: number of queries - Returns: - Recall at R - """ - correct = 0.0 - total_num_of_results = 0 - for query in range(query_count): - true_neighbors = neighbor_dataset.read(1) - if true_neighbors is None: - break - true_neighbors_set = set(true_neighbors[0][:k]) - true_neighbors_set.discard(-1) - min_r = min(r, len(true_neighbors_set)) - total_num_of_results += min_r - for j in range(min_r): - if results[query][j] in true_neighbors_set: - correct += 1.0 - - return correct / total_num_of_results - - -def get_index_size_in_kb(opensearch, index_name): - """ - Gets the size of an index in kilobytes - Args: - opensearch: opensearch client - index_name: name of index to look up - Returns: - size of index in kilobytes - """ - return int( - opensearch.indices.stats(index_name, metric='store')['indices'] - [index_name]['total']['store']['size_in_bytes']) / 1024 - - -def get_cache_size_in_kb(endpoint, port): - """ - Gets the size of the k-NN cache in kilobytes - Args: - endpoint: endpoint of OpenSearch cluster - port: port of endpoint OpenSearch is running on - Returns: - size of cache in kilobytes - """ - response = requests.get('http://' + endpoint + ':' + str(port) + - '/_plugins/_knn/stats', - headers={'content-type': 'application/json'}) - stats = response.json() - - keys = stats['nodes'].keys() - - total_used = 0 - for key in keys: - total_used += int(stats['nodes'][key]['graph_memory_usage']) - return total_used - - -def query_index(opensearch: OpenSearch, index_name: str, body: dict, - excluded_fields: list): - start_time = round(time.time()*1000) - queryResponse = opensearch.search(index=index_name, - body=body, - _source_excludes=excluded_fields) - end_time = round(time.time() * 1000) - queryResponse['client_time'] = end_time - start_time - return queryResponse - - -def bulk_index(opensearch: OpenSearch, index_name: str, body: List): - return opensearch.bulk(index=index_name, body=body) - -def get_segment_stats(opensearch: OpenSearch, index_name: str): - return opensearch.indices.segments(index=index_name) diff --git a/benchmarks/perf-tool/okpt/test/test.py b/benchmarks/perf-tool/okpt/test/test.py deleted file mode 100644 index c947545ad..000000000 --- a/benchmarks/perf-tool/okpt/test/test.py +++ /dev/null @@ -1,188 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# -# The OpenSearch Contributors require contributions made to -# this file be licensed under the Apache-2.0 license or a -# compatible open source license. - -"""Provides a base Test class.""" -from math import floor -from typing import Any, Dict, List - -from okpt.io.config.parsers.test import TestConfig -from okpt.test.steps.base import Step - - -def get_avg(values: List[Any]): - """Get average value of a list. - - Args: - values: A list of values. - - Returns: - The average value in the list. - """ - valid_total = len(values) - running_sum = 0.0 - - for value in values: - if value == -1: - valid_total -= 1 - continue - running_sum += value - - if valid_total == 0: - return -1 - return running_sum / valid_total - - -def _pxx(values: List[Any], p: float): - """Calculates the pXX statistics for a given list. - - Args: - values: List of values. - p: Percentile (between 0 and 1). - - Returns: - The corresponding pXX metric. - """ - lowest_percentile = 1 / len(values) - highest_percentile = (len(values) - 1) / len(values) - - # return -1 if p is out of range or if the list doesn't have enough elements - # to support the specified percentile - if p < 0 or p > 1: - return -1.0 - elif p < lowest_percentile or p > highest_percentile: - if p == 1.0 and len(values) > 1: - return float(values[len(values) - 1]) - return -1.0 - else: - return float(values[floor(len(values) * p)]) - - -def _aggregate_steps(step_results: List[Dict[str, Any]], - measure_labels=None): - """Aggregates the steps for a given Test. - - The aggregation process extracts the measures from each step and calculates - the total time spent performing each step measure, including the - percentile metrics, if possible. - - The aggregation process also extracts the test measures by simply summing - up the respective step measures. - - A step measure is formatted as `{step_name}_{measure_name}`, for example, - {bulk_index}_{took} or {query_index}_{memory}. The braces are not included - in the actual key string. - - Percentile/Total step measures are give as - `{step_name}_{measure_name}_{percentile|total}`. - - Test measures are just step measure sums so they just given as - `test_{measure_name}`. - - Args: - steps: List of test steps to be aggregated. - measures: List of step metrics to account for. - - Returns: - A complete test result. - """ - if measure_labels is None: - measure_labels = ['took'] - test_measures = { - f'test_{measure_label}': 0 - for measure_label in measure_labels - } - step_measures: Dict[str, Any] = {} - - # iterate over all test steps - for step in step_results: - step_label = step['label'] - - step_measure_labels = list(step.keys()) - step_measure_labels.remove('label') - - # iterate over all measures in each test step - for measure_label in step_measure_labels: - - step_measure = step[measure_label] - step_measure_label = f'{measure_label}' if step_label == 'get_stats' else f'{step_label}_{measure_label}' - - # Add cumulative test measures from steps to test measures - if measure_label in measure_labels: - test_measures[f'test_{measure_label}'] += sum(step_measure) if \ - isinstance(step_measure, list) else step_measure - - if step_measure_label in step_measures: - _ = step_measures[step_measure_label].extend(step_measure) \ - if isinstance(step_measure, list) else \ - step_measures[step_measure_label].append(step_measure) - else: - step_measures[step_measure_label] = step_measure if \ - isinstance(step_measure, list) else [step_measure] - - aggregate = {**test_measures} - # calculate the totals and percentile statistics for each step measure - # where relevant - for step_measure_label, step_measure in step_measures.items(): - step_measure.sort() - - aggregate[step_measure_label + '_total'] = float(sum(step_measure)) - - p50 = _pxx(step_measure, 0.50) - if p50 != -1: - aggregate[step_measure_label + '_p50'] = p50 - p90 = _pxx(step_measure, 0.90) - if p90 != -1: - aggregate[step_measure_label + '_p90'] = p90 - p99 = _pxx(step_measure, 0.99) - if p99 != -1: - aggregate[step_measure_label + '_p99'] = p99 - p99_9 = _pxx(step_measure, 0.999) - if p99_9 != -1: - aggregate[step_measure_label + '_p99.9'] = p99_9 - p100 = _pxx(step_measure, 1.00) - if p100 != -1: - aggregate[step_measure_label + '_p100'] = p100 - - return aggregate - - -class Test: - """A base Test class, representing a collection of steps to profiled and - aggregated. - - Methods: - setup: Performs test setup. Usually for steps not intended to be - profiled. - run_steps: Runs the test steps, aggregating the results into the - `step_results` instance field. - cleanup: Perform test cleanup. Useful for clearing the state of a - persistent process like OpenSearch. Cleanup steps are executed after - each run. - execute: Runs steps, cleans up, and aggregates the test result. - """ - def __init__(self, test_config: TestConfig): - """Initializes the test state. - """ - self.test_config = test_config - self.setup_steps: List[Step] = test_config.setup - self.test_steps: List[Step] = test_config.steps - self.cleanup_steps: List[Step] = test_config.cleanup - - def setup(self): - _ = [step.execute() for step in self.setup_steps] - - def _run_steps(self): - step_results = [] - _ = [step_results.extend(step.execute()) for step in self.test_steps] - return step_results - - def _cleanup(self): - _ = [step.execute() for step in self.cleanup_steps] - - def execute(self): - results = self._run_steps() - self._cleanup() - return _aggregate_steps(results) diff --git a/benchmarks/perf-tool/release-configs/faiss-hnsw/filtering/relaxed-filter/index.json b/benchmarks/perf-tool/release-configs/faiss-hnsw/filtering/relaxed-filter/index.json deleted file mode 100644 index 7e8ddda8e..000000000 --- a/benchmarks/perf-tool/release-configs/faiss-hnsw/filtering/relaxed-filter/index.json +++ /dev/null @@ -1,27 +0,0 @@ -{ - "settings": { - "index": { - "knn": true, - "number_of_shards": 24, - "number_of_replicas": 1, - "knn.algo_param.ef_search": 100 - } - }, - "mappings": { - "properties": { - "target_field": { - "type": "knn_vector", - "dimension": 128, - "method": { - "name": "hnsw", - "space_type": "l2", - "engine": "faiss", - "parameters": { - "ef_construction": 256, - "m": 16 - } - } - } - } - } -} diff --git a/benchmarks/perf-tool/release-configs/faiss-hnsw/filtering/relaxed-filter/relaxed-filter-spec.json b/benchmarks/perf-tool/release-configs/faiss-hnsw/filtering/relaxed-filter/relaxed-filter-spec.json deleted file mode 100644 index 3e04d12c4..000000000 --- a/benchmarks/perf-tool/release-configs/faiss-hnsw/filtering/relaxed-filter/relaxed-filter-spec.json +++ /dev/null @@ -1,42 +0,0 @@ -{ - "bool": - { - "should": - [ - { - "range": - { - "age": - { - "gte": 30, - "lte": 70 - } - } - }, - { - "term": - { - "color": "green" - } - }, - { - "term": - { - "color": "blue" - } - }, - { - "term": - { - "color": "yellow" - } - }, - { - "term": - { - "taste": "sweet" - } - } - ] - } -} diff --git a/benchmarks/perf-tool/release-configs/faiss-hnsw/filtering/relaxed-filter/relaxed-filter-test.yml b/benchmarks/perf-tool/release-configs/faiss-hnsw/filtering/relaxed-filter/relaxed-filter-test.yml deleted file mode 100644 index ba8850e1d..000000000 --- a/benchmarks/perf-tool/release-configs/faiss-hnsw/filtering/relaxed-filter/relaxed-filter-test.yml +++ /dev/null @@ -1,40 +0,0 @@ -endpoint: [ENDPOINT] -port: [PORT] -test_name: "Faiss HNSW Relaxed Filter Test" -test_id: "Faiss HNSW Relaxed Filter Test" -num_runs: 3 -show_runs: false -steps: - - name: delete_index - index_name: target_index - - name: create_index - index_name: target_index - index_spec: release-configs/faiss-hnsw/filtering/relaxed-filter/index.json - - name: ingest_multi_field - index_name: target_index - field_name: target_field - bulk_size: 500 - dataset_format: hdf5 - dataset_path: dataset/sift-128-euclidean-with-attr.hdf5 - attributes_dataset_name: attributes - attribute_spec: [ { name: 'color', type: 'str' }, { name: 'taste', type: 'str' }, { name: 'age', type: 'int' } ] - - name: refresh_index - index_name: target_index - - name: force_merge - index_name: target_index - max_num_segments: 1 - - name: warmup_operation - index_name: target_index - - name: query_with_filter - k: 100 - r: 1 - calculate_recall: true - index_name: target_index - field_name: target_field - dataset_format: hdf5 - dataset_path: dataset/sift-128-euclidean-with-attr.hdf5 - neighbors_format: hdf5 - neighbors_path: dataset/sift-128-euclidean-with-relaxed-filters.hdf5 - neighbors_dataset: neighbors_filter_5 - filter_spec: release-configs/faiss-hnsw/filtering/relaxed-filter/relaxed-filter-spec.json - filter_type: FILTER diff --git a/benchmarks/perf-tool/release-configs/faiss-hnsw/filtering/restrictive-filter/index.json b/benchmarks/perf-tool/release-configs/faiss-hnsw/filtering/restrictive-filter/index.json deleted file mode 100644 index 7e8ddda8e..000000000 --- a/benchmarks/perf-tool/release-configs/faiss-hnsw/filtering/restrictive-filter/index.json +++ /dev/null @@ -1,27 +0,0 @@ -{ - "settings": { - "index": { - "knn": true, - "number_of_shards": 24, - "number_of_replicas": 1, - "knn.algo_param.ef_search": 100 - } - }, - "mappings": { - "properties": { - "target_field": { - "type": "knn_vector", - "dimension": 128, - "method": { - "name": "hnsw", - "space_type": "l2", - "engine": "faiss", - "parameters": { - "ef_construction": 256, - "m": 16 - } - } - } - } - } -} diff --git a/benchmarks/perf-tool/release-configs/faiss-hnsw/filtering/restrictive-filter/restrictive-filter-spec.json b/benchmarks/perf-tool/release-configs/faiss-hnsw/filtering/restrictive-filter/restrictive-filter-spec.json deleted file mode 100644 index 9e6356f1c..000000000 --- a/benchmarks/perf-tool/release-configs/faiss-hnsw/filtering/restrictive-filter/restrictive-filter-spec.json +++ /dev/null @@ -1,44 +0,0 @@ -{ - "bool": - { - "must": - [ - { - "range": - { - "age": - { - "gte": 30, - "lte": 60 - } - } - }, - { - "term": - { - "taste": "bitter" - } - }, - { - "bool": - { - "should": - [ - { - "term": - { - "color": "blue" - } - }, - { - "term": - { - "color": "green" - } - } - ] - } - } - ] - } -} \ No newline at end of file diff --git a/benchmarks/perf-tool/release-configs/faiss-hnsw/filtering/restrictive-filter/restrictive-filter-test.yml b/benchmarks/perf-tool/release-configs/faiss-hnsw/filtering/restrictive-filter/restrictive-filter-test.yml deleted file mode 100644 index 94f4073c7..000000000 --- a/benchmarks/perf-tool/release-configs/faiss-hnsw/filtering/restrictive-filter/restrictive-filter-test.yml +++ /dev/null @@ -1,40 +0,0 @@ -endpoint: [ENDPOINT] -port: [PORT] -test_name: "Faiss HNSW Restrictive Filter Test" -test_id: "Faiss HNSW Restrictive Filter Test" -num_runs: 3 -show_runs: false -steps: - - name: delete_index - index_name: target_index - - name: create_index - index_name: target_index - index_spec: release-configs/faiss-hnsw/filtering/restrictive-filter/index.json - - name: ingest_multi_field - index_name: target_index - field_name: target_field - bulk_size: 500 - dataset_format: hdf5 - dataset_path: dataset/sift-128-euclidean-with-attr.hdf5 - attributes_dataset_name: attributes - attribute_spec: [ { name: 'color', type: 'str' }, { name: 'taste', type: 'str' }, { name: 'age', type: 'int' } ] - - name: refresh_index - index_name: target_index - - name: force_merge - index_name: target_index - max_num_segments: 1 - - name: warmup_operation - index_name: target_index - - name: query_with_filter - k: 100 - r: 1 - calculate_recall: true - index_name: target_index - field_name: target_field - dataset_format: hdf5 - dataset_path: dataset/sift-128-euclidean-with-attr.hdf5 - neighbors_format: hdf5 - neighbors_path: dataset/sift-128-euclidean-with-restrictive-filters.hdf5 - neighbors_dataset: neighbors_filter_4 - filter_spec: release-configs/faiss-hnsw/filtering/restrictive-filter/restrictive-filter-spec.json - filter_type: FILTER diff --git a/benchmarks/perf-tool/release-configs/faiss-hnsw/index.json b/benchmarks/perf-tool/release-configs/faiss-hnsw/index.json deleted file mode 100644 index 7e8ddda8e..000000000 --- a/benchmarks/perf-tool/release-configs/faiss-hnsw/index.json +++ /dev/null @@ -1,27 +0,0 @@ -{ - "settings": { - "index": { - "knn": true, - "number_of_shards": 24, - "number_of_replicas": 1, - "knn.algo_param.ef_search": 100 - } - }, - "mappings": { - "properties": { - "target_field": { - "type": "knn_vector", - "dimension": 128, - "method": { - "name": "hnsw", - "space_type": "l2", - "engine": "faiss", - "parameters": { - "ef_construction": 256, - "m": 16 - } - } - } - } - } -} diff --git a/benchmarks/perf-tool/release-configs/faiss-hnsw/nested/simple/index.json b/benchmarks/perf-tool/release-configs/faiss-hnsw/nested/simple/index.json deleted file mode 100644 index 338ceb1f4..000000000 --- a/benchmarks/perf-tool/release-configs/faiss-hnsw/nested/simple/index.json +++ /dev/null @@ -1,35 +0,0 @@ -{ - "settings": { - "index": { - "knn": true, - "number_of_shards": 24, - "number_of_replicas": 1, - "knn.algo_param.ef_search": 100 - } - }, - "mappings": { - "_source": { - "excludes": ["nested_field"] - }, - "properties": { - "nested_field": { - "type": "nested", - "properties": { - "target_field": { - "type": "knn_vector", - "dimension": 128, - "method": { - "name": "hnsw", - "space_type": "l2", - "engine": "faiss", - "parameters": { - "ef_construction": 256, - "m": 16 - } - } - } - } - } - } - } -} diff --git a/benchmarks/perf-tool/release-configs/faiss-hnsw/nested/simple/simple-nested-test.yml b/benchmarks/perf-tool/release-configs/faiss-hnsw/nested/simple/simple-nested-test.yml deleted file mode 100644 index 151b2014d..000000000 --- a/benchmarks/perf-tool/release-configs/faiss-hnsw/nested/simple/simple-nested-test.yml +++ /dev/null @@ -1,37 +0,0 @@ -endpoint: [ENDPOINT] -port: [PORT] -test_name: "Faiss HNSW Nested Field Test" -test_id: "Faiss HNSW Nested Field Test" -num_runs: 3 -show_runs: false -steps: - - name: delete_index - index_name: target_index - - name: create_index - index_name: target_index - index_spec: release-configs/faiss-hnsw/nested/simple/index.json - - name: ingest_nested_field - index_name: target_index - field_name: target_field - dataset_format: hdf5 - dataset_path: dataset/sift-128-euclidean-nested.hdf5 - attributes_dataset_name: attributes - attribute_spec: [ { name: 'color', type: 'str' }, { name: 'taste', type: 'str' }, { name: 'age', type: 'int' }, { name: 'parent_id', type: 'int'} ] - - name: refresh_index - index_name: target_index - - name: force_merge - index_name: target_index - max_num_segments: 1 - - name: warmup_operation - index_name: target_index - - name: query_nested_field - k: 100 - r: 1 - calculate_recall: true - index_name: target_index - field_name: target_field - dataset_format: hdf5 - dataset_path: dataset/sift-128-euclidean-nested.hdf5 - neighbors_format: hdf5 - neighbors_path: dataset/sift-128-euclidean-nested.hdf5 - neighbors_dataset: neighbour_nested \ No newline at end of file diff --git a/benchmarks/perf-tool/release-configs/faiss-hnsw/test.yml b/benchmarks/perf-tool/release-configs/faiss-hnsw/test.yml deleted file mode 100644 index c4740acf5..000000000 --- a/benchmarks/perf-tool/release-configs/faiss-hnsw/test.yml +++ /dev/null @@ -1,35 +0,0 @@ -endpoint: [ENDPOINT] -port: [PORT] -test_name: "Faiss HNSW Test" -test_id: "Faiss HNSW Test" -num_runs: 3 -show_runs: false -steps: - - name: delete_index - index_name: target_index - - name: create_index - index_name: target_index - index_spec: release-configs/faiss-hnsw/index.json - - name: ingest - index_name: target_index - field_name: target_field - bulk_size: 500 - dataset_format: hdf5 - dataset_path: dataset/sift-128-euclidean.hdf5 - - name: refresh_index - index_name: target_index - - name: force_merge - index_name: target_index - max_num_segments: 1 - - name: warmup_operation - index_name: target_index - - name: query - k: 100 - r: 1 - calculate_recall: true - index_name: target_index - field_name: target_field - dataset_format: hdf5 - dataset_path: dataset/sift-128-euclidean.hdf5 - neighbors_format: hdf5 - neighbors_path: dataset/sift-128-euclidean.hdf5 diff --git a/benchmarks/perf-tool/release-configs/faiss-hnswpq/index.json b/benchmarks/perf-tool/release-configs/faiss-hnswpq/index.json deleted file mode 100644 index 479703412..000000000 --- a/benchmarks/perf-tool/release-configs/faiss-hnswpq/index.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "settings": { - "index": { - "knn": true, - "number_of_shards": 24, - "number_of_replicas": 1 - } - }, - "mappings": { - "properties": { - "target_field": { - "type": "knn_vector", - "model_id": "test-model" - } - } - } -} diff --git a/benchmarks/perf-tool/release-configs/faiss-hnswpq/method-spec.json b/benchmarks/perf-tool/release-configs/faiss-hnswpq/method-spec.json deleted file mode 100644 index 2d67bf2df..000000000 --- a/benchmarks/perf-tool/release-configs/faiss-hnswpq/method-spec.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "name":"hnsw", - "engine":"faiss", - "space_type": "l2", - "parameters":{ - "ef_construction": 256, - "m": 16, - "encoder": { - "name": "pq", - "parameters": { - "m": 16 - } - } - } -} diff --git a/benchmarks/perf-tool/release-configs/faiss-hnswpq/test.yml b/benchmarks/perf-tool/release-configs/faiss-hnswpq/test.yml deleted file mode 100644 index f573ede9c..000000000 --- a/benchmarks/perf-tool/release-configs/faiss-hnswpq/test.yml +++ /dev/null @@ -1,59 +0,0 @@ -endpoint: [ENDPOINT] -port: [PORT] -test_name: "Faiss HNSW PQ Test" -test_id: "Faiss HNSW PQ Test" -num_runs: 3 -show_runs: false -setup: - - name: delete_index - index_name: train_index - - name: create_index - index_name: train_index - index_spec: release-configs/faiss-hnswpq/train-index-spec.json - - name: ingest - index_name: train_index - field_name: train_field - bulk_size: 500 - dataset_format: hdf5 - dataset_path: dataset/sift-128-euclidean.hdf5 - doc_count: 50000 - - name: refresh_index - index_name: train_index -steps: - - name: delete_model - model_id: test-model - - name: delete_index - index_name: target_index - - name: train_model - model_id: test-model - train_index: train_index - train_field: train_field - dimension: 128 - method_spec: release-configs/faiss-hnswpq/method-spec.json - max_training_vector_count: 50000 - - name: create_index - index_name: target_index - index_spec: release-configs/faiss-hnswpq/index.json - - name: ingest - index_name: target_index - field_name: target_field - bulk_size: 500 - dataset_format: hdf5 - dataset_path: dataset/sift-128-euclidean.hdf5 - - name: refresh_index - index_name: target_index - - name: force_merge - index_name: target_index - max_num_segments: 1 - - name: warmup_operation - index_name: target_index - - name: query - k: 100 - r: 1 - calculate_recall: true - index_name: target_index - field_name: target_field - dataset_format: hdf5 - dataset_path: dataset/sift-128-euclidean.hdf5 - neighbors_format: hdf5 - neighbors_path: dataset/sift-128-euclidean.hdf5 diff --git a/benchmarks/perf-tool/release-configs/faiss-hnswpq/train-index-spec.json b/benchmarks/perf-tool/release-configs/faiss-hnswpq/train-index-spec.json deleted file mode 100644 index 804a5707e..000000000 --- a/benchmarks/perf-tool/release-configs/faiss-hnswpq/train-index-spec.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "settings": { - "index": { - "number_of_shards": 24, - "number_of_replicas": 0 - } - }, - "mappings": { - "properties": { - "train_field": { - "type": "knn_vector", - "dimension": 128 - } - } - } -} diff --git a/benchmarks/perf-tool/release-configs/faiss-ivf/filtering/relaxed-filter/index.json b/benchmarks/perf-tool/release-configs/faiss-ivf/filtering/relaxed-filter/index.json deleted file mode 100644 index ade7fa377..000000000 --- a/benchmarks/perf-tool/release-configs/faiss-ivf/filtering/relaxed-filter/index.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "settings": { - "index": { - "knn": true, - "number_of_shards": 24, - "number_of_replicas": 1 - } - }, - "mappings": { - "properties": { - "target_field": { - "type": "knn_vector", - "model_id": "test-model" - } - } - } -} diff --git a/benchmarks/perf-tool/release-configs/faiss-ivf/filtering/relaxed-filter/method-spec.json b/benchmarks/perf-tool/release-configs/faiss-ivf/filtering/relaxed-filter/method-spec.json deleted file mode 100644 index 51ae89877..000000000 --- a/benchmarks/perf-tool/release-configs/faiss-ivf/filtering/relaxed-filter/method-spec.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "name":"ivf", - "engine":"faiss", - "space_type": "l2", - "parameters":{ - "nlist": 128, - "nprobes": 8 - } -} diff --git a/benchmarks/perf-tool/release-configs/faiss-ivf/filtering/relaxed-filter/relaxed-filter-spec.json b/benchmarks/perf-tool/release-configs/faiss-ivf/filtering/relaxed-filter/relaxed-filter-spec.json deleted file mode 100644 index 3e04d12c4..000000000 --- a/benchmarks/perf-tool/release-configs/faiss-ivf/filtering/relaxed-filter/relaxed-filter-spec.json +++ /dev/null @@ -1,42 +0,0 @@ -{ - "bool": - { - "should": - [ - { - "range": - { - "age": - { - "gte": 30, - "lte": 70 - } - } - }, - { - "term": - { - "color": "green" - } - }, - { - "term": - { - "color": "blue" - } - }, - { - "term": - { - "color": "yellow" - } - }, - { - "term": - { - "taste": "sweet" - } - } - ] - } -} diff --git a/benchmarks/perf-tool/release-configs/faiss-ivf/filtering/relaxed-filter/relaxed-filter-test.yml b/benchmarks/perf-tool/release-configs/faiss-ivf/filtering/relaxed-filter/relaxed-filter-test.yml deleted file mode 100644 index adb25a04d..000000000 --- a/benchmarks/perf-tool/release-configs/faiss-ivf/filtering/relaxed-filter/relaxed-filter-test.yml +++ /dev/null @@ -1,64 +0,0 @@ -endpoint: [ENDPOINT] -port: [PORT] -test_name: "Faiss IVF Relaxed Filter Test" -test_id: "Faiss IVF Relaxed Filter Test" -num_runs: 3 -show_runs: false -setup: - - name: delete_index - index_name: train_index - - name: create_index - index_name: train_index - index_spec: release-configs/faiss-ivf/filtering/relaxed-filter/train-index-spec.json - - name: ingest - index_name: train_index - field_name: train_field - bulk_size: 500 - dataset_format: hdf5 - dataset_path: dataset/sift-128-euclidean.hdf5 - doc_count: 50000 - - name: refresh_index - index_name: train_index -steps: - - name: delete_model - model_id: test-model - - name: delete_index - index_name: target_index - - name: train_model - model_id: test-model - train_index: train_index - train_field: train_field - dimension: 128 - method_spec: release-configs/faiss-ivf/filtering/relaxed-filter/method-spec.json - max_training_vector_count: 50000 - - name: create_index - index_name: target_index - index_spec: release-configs/faiss-ivf/filtering/relaxed-filter/index.json - - name: ingest_multi_field - index_name: target_index - field_name: target_field - bulk_size: 500 - dataset_format: hdf5 - dataset_path: dataset/sift-128-euclidean-with-attr.hdf5 - attributes_dataset_name: attributes - attribute_spec: [ { name: 'color', type: 'str' }, { name: 'taste', type: 'str' }, { name: 'age', type: 'int' } ] - - name: refresh_index - index_name: target_index - - name: force_merge - index_name: target_index - max_num_segments: 1 - - name: warmup_operation - index_name: target_index - - name: query_with_filter - k: 100 - r: 1 - calculate_recall: true - index_name: target_index - field_name: target_field - dataset_format: hdf5 - dataset_path: dataset/sift-128-euclidean-with-attr.hdf5 - neighbors_format: hdf5 - neighbors_path: dataset/sift-128-euclidean-with-relaxed-filters.hdf5 - neighbors_dataset: neighbors_filter_5 - filter_spec: release-configs/faiss-ivf/filtering/relaxed-filter/relaxed-filter-spec.json - filter_type: FILTER diff --git a/benchmarks/perf-tool/release-configs/faiss-ivf/filtering/relaxed-filter/train-index-spec.json b/benchmarks/perf-tool/release-configs/faiss-ivf/filtering/relaxed-filter/train-index-spec.json deleted file mode 100644 index 137fac9d8..000000000 --- a/benchmarks/perf-tool/release-configs/faiss-ivf/filtering/relaxed-filter/train-index-spec.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "settings": { - "index": { - "number_of_shards": 24, - "number_of_replicas": 1 - } - }, - "mappings": { - "properties": { - "train_field": { - "type": "knn_vector", - "dimension": 128 - } - } - } -} diff --git a/benchmarks/perf-tool/release-configs/faiss-ivf/filtering/restrictive-filter/index.json b/benchmarks/perf-tool/release-configs/faiss-ivf/filtering/restrictive-filter/index.json deleted file mode 100644 index ade7fa377..000000000 --- a/benchmarks/perf-tool/release-configs/faiss-ivf/filtering/restrictive-filter/index.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "settings": { - "index": { - "knn": true, - "number_of_shards": 24, - "number_of_replicas": 1 - } - }, - "mappings": { - "properties": { - "target_field": { - "type": "knn_vector", - "model_id": "test-model" - } - } - } -} diff --git a/benchmarks/perf-tool/release-configs/faiss-ivf/filtering/restrictive-filter/method-spec.json b/benchmarks/perf-tool/release-configs/faiss-ivf/filtering/restrictive-filter/method-spec.json deleted file mode 100644 index 51ae89877..000000000 --- a/benchmarks/perf-tool/release-configs/faiss-ivf/filtering/restrictive-filter/method-spec.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "name":"ivf", - "engine":"faiss", - "space_type": "l2", - "parameters":{ - "nlist": 128, - "nprobes": 8 - } -} diff --git a/benchmarks/perf-tool/release-configs/faiss-ivf/filtering/restrictive-filter/restrictive-filter-spec.json b/benchmarks/perf-tool/release-configs/faiss-ivf/filtering/restrictive-filter/restrictive-filter-spec.json deleted file mode 100644 index 9e6356f1c..000000000 --- a/benchmarks/perf-tool/release-configs/faiss-ivf/filtering/restrictive-filter/restrictive-filter-spec.json +++ /dev/null @@ -1,44 +0,0 @@ -{ - "bool": - { - "must": - [ - { - "range": - { - "age": - { - "gte": 30, - "lte": 60 - } - } - }, - { - "term": - { - "taste": "bitter" - } - }, - { - "bool": - { - "should": - [ - { - "term": - { - "color": "blue" - } - }, - { - "term": - { - "color": "green" - } - } - ] - } - } - ] - } -} \ No newline at end of file diff --git a/benchmarks/perf-tool/release-configs/faiss-ivf/filtering/restrictive-filter/restrictive-filter-test.yml b/benchmarks/perf-tool/release-configs/faiss-ivf/filtering/restrictive-filter/restrictive-filter-test.yml deleted file mode 100644 index bad047eab..000000000 --- a/benchmarks/perf-tool/release-configs/faiss-ivf/filtering/restrictive-filter/restrictive-filter-test.yml +++ /dev/null @@ -1,64 +0,0 @@ -endpoint: [ENDPOINT] -port: [PORT] -test_name: "Faiss IVF restrictive Filter Test" -test_id: "Faiss IVF restrictive Filter Test" -num_runs: 3 -show_runs: false -setup: - - name: delete_index - index_name: train_index - - name: create_index - index_name: train_index - index_spec: release-configs/faiss-ivf/filtering/restrictive-filter/train-index-spec.json - - name: ingest - index_name: train_index - field_name: train_field - bulk_size: 500 - dataset_format: hdf5 - dataset_path: dataset/sift-128-euclidean.hdf5 - doc_count: 50000 - - name: refresh_index - index_name: train_index -steps: - - name: delete_model - model_id: test-model - - name: delete_index - index_name: target_index - - name: train_model - model_id: test-model - train_index: train_index - train_field: train_field - dimension: 128 - method_spec: release-configs/faiss-ivf/filtering/restrictive-filter/method-spec.json - max_training_vector_count: 50000 - - name: create_index - index_name: target_index - index_spec: release-configs/faiss-ivf/filtering/restrictive-filter/index.json - - name: ingest_multi_field - index_name: target_index - field_name: target_field - bulk_size: 500 - dataset_format: hdf5 - dataset_path: dataset/sift-128-euclidean-with-attr.hdf5 - attributes_dataset_name: attributes - attribute_spec: [ { name: 'color', type: 'str' }, { name: 'taste', type: 'str' }, { name: 'age', type: 'int' } ] - - name: refresh_index - index_name: target_index - - name: force_merge - index_name: target_index - max_num_segments: 1 - - name: warmup_operation - index_name: target_index - - name: query_with_filter - k: 100 - r: 1 - calculate_recall: true - index_name: target_index - field_name: target_field - dataset_format: hdf5 - dataset_path: dataset/sift-128-euclidean-with-attr.hdf5 - neighbors_format: hdf5 - neighbors_path: dataset/sift-128-euclidean-with-restrictive-filters.hdf5 - neighbors_dataset: neighbors_filter_4 - filter_spec: release-configs/faiss-ivf/filtering/restrictive-filter/restrictive-filter-spec.json - filter_type: FILTER diff --git a/benchmarks/perf-tool/release-configs/faiss-ivf/filtering/restrictive-filter/train-index-spec.json b/benchmarks/perf-tool/release-configs/faiss-ivf/filtering/restrictive-filter/train-index-spec.json deleted file mode 100644 index 804a5707e..000000000 --- a/benchmarks/perf-tool/release-configs/faiss-ivf/filtering/restrictive-filter/train-index-spec.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "settings": { - "index": { - "number_of_shards": 24, - "number_of_replicas": 0 - } - }, - "mappings": { - "properties": { - "train_field": { - "type": "knn_vector", - "dimension": 128 - } - } - } -} diff --git a/benchmarks/perf-tool/release-configs/faiss-ivf/index.json b/benchmarks/perf-tool/release-configs/faiss-ivf/index.json deleted file mode 100644 index 479703412..000000000 --- a/benchmarks/perf-tool/release-configs/faiss-ivf/index.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "settings": { - "index": { - "knn": true, - "number_of_shards": 24, - "number_of_replicas": 1 - } - }, - "mappings": { - "properties": { - "target_field": { - "type": "knn_vector", - "model_id": "test-model" - } - } - } -} diff --git a/benchmarks/perf-tool/release-configs/faiss-ivf/method-spec.json b/benchmarks/perf-tool/release-configs/faiss-ivf/method-spec.json deleted file mode 100644 index 51ae89877..000000000 --- a/benchmarks/perf-tool/release-configs/faiss-ivf/method-spec.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "name":"ivf", - "engine":"faiss", - "space_type": "l2", - "parameters":{ - "nlist": 128, - "nprobes": 8 - } -} diff --git a/benchmarks/perf-tool/release-configs/faiss-ivf/test.yml b/benchmarks/perf-tool/release-configs/faiss-ivf/test.yml deleted file mode 100644 index 367c42594..000000000 --- a/benchmarks/perf-tool/release-configs/faiss-ivf/test.yml +++ /dev/null @@ -1,59 +0,0 @@ -endpoint: [ENDPOINT] -port: [PORT] -test_name: "Faiss IVF" -test_id: "Faiss IVF" -num_runs: 3 -show_runs: false -setup: - - name: delete_index - index_name: train_index - - name: create_index - index_name: train_index - index_spec: release-configs/faiss-ivf/train-index-spec.json - - name: ingest - index_name: train_index - field_name: train_field - bulk_size: 500 - dataset_format: hdf5 - dataset_path: dataset/sift-128-euclidean.hdf5 - doc_count: 50000 - - name: refresh_index - index_name: train_index -steps: - - name: delete_model - model_id: test-model - - name: delete_index - index_name: target_index - - name: train_model - model_id: test-model - train_index: train_index - train_field: train_field - dimension: 128 - method_spec: release-configs/faiss-ivf/method-spec.json - max_training_vector_count: 50000 - - name: create_index - index_name: target_index - index_spec: release-configs/faiss-ivf/index.json - - name: ingest - index_name: target_index - field_name: target_field - bulk_size: 500 - dataset_format: hdf5 - dataset_path: dataset/sift-128-euclidean.hdf5 - - name: refresh_index - index_name: target_index - - name: force_merge - index_name: target_index - max_num_segments: 1 - - name: warmup_operation - index_name: target_index - - name: query - k: 100 - r: 1 - calculate_recall: true - index_name: target_index - field_name: target_field - dataset_format: hdf5 - dataset_path: dataset/sift-128-euclidean.hdf5 - neighbors_format: hdf5 - neighbors_path: dataset/sift-128-euclidean.hdf5 diff --git a/benchmarks/perf-tool/release-configs/faiss-ivf/train-index-spec.json b/benchmarks/perf-tool/release-configs/faiss-ivf/train-index-spec.json deleted file mode 100644 index 804a5707e..000000000 --- a/benchmarks/perf-tool/release-configs/faiss-ivf/train-index-spec.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "settings": { - "index": { - "number_of_shards": 24, - "number_of_replicas": 0 - } - }, - "mappings": { - "properties": { - "train_field": { - "type": "knn_vector", - "dimension": 128 - } - } - } -} diff --git a/benchmarks/perf-tool/release-configs/faiss-ivfpq/index.json b/benchmarks/perf-tool/release-configs/faiss-ivfpq/index.json deleted file mode 100644 index 479703412..000000000 --- a/benchmarks/perf-tool/release-configs/faiss-ivfpq/index.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "settings": { - "index": { - "knn": true, - "number_of_shards": 24, - "number_of_replicas": 1 - } - }, - "mappings": { - "properties": { - "target_field": { - "type": "knn_vector", - "model_id": "test-model" - } - } - } -} diff --git a/benchmarks/perf-tool/release-configs/faiss-ivfpq/method-spec.json b/benchmarks/perf-tool/release-configs/faiss-ivfpq/method-spec.json deleted file mode 100644 index 204b0a653..000000000 --- a/benchmarks/perf-tool/release-configs/faiss-ivfpq/method-spec.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "name":"ivf", - "engine":"faiss", - "space_type": "l2", - "parameters":{ - "nlist": 128, - "nprobes": 8, - "encoder": { - "name": "pq", - "parameters": { - "m": 16, - "code_size": 8 - } - } - } -} diff --git a/benchmarks/perf-tool/release-configs/faiss-ivfpq/test.yml b/benchmarks/perf-tool/release-configs/faiss-ivfpq/test.yml deleted file mode 100644 index c3f63348b..000000000 --- a/benchmarks/perf-tool/release-configs/faiss-ivfpq/test.yml +++ /dev/null @@ -1,59 +0,0 @@ -endpoint: [ENDPOINT] -port: [PORT] -test_name: "Faiss IVF PQ Test" -test_id: "Faiss IVF PQ Test" -num_runs: 3 -show_runs: false -setup: - - name: delete_index - index_name: train_index - - name: create_index - index_name: train_index - index_spec: release-configs/faiss-ivfpq/train-index-spec.json - - name: ingest - index_name: train_index - field_name: train_field - bulk_size: 500 - dataset_format: hdf5 - dataset_path: dataset/sift-128-euclidean.hdf5 - doc_count: 50000 - - name: refresh_index - index_name: train_index -steps: - - name: delete_model - model_id: test-model - - name: delete_index - index_name: target_index - - name: train_model - model_id: test-model - train_index: train_index - train_field: train_field - dimension: 128 - method_spec: release-configs/faiss-ivfpq/method-spec.json - max_training_vector_count: 50000 - - name: create_index - index_name: target_index - index_spec: release-configs/faiss-ivfpq/index.json - - name: ingest - index_name: target_index - field_name: target_field - bulk_size: 500 - dataset_format: hdf5 - dataset_path: dataset/sift-128-euclidean.hdf5 - - name: refresh_index - index_name: target_index - - name: force_merge - index_name: target_index - max_num_segments: 1 - - name: warmup_operation - index_name: target_index - - name: query - k: 100 - r: 1 - calculate_recall: true - index_name: target_index - field_name: target_field - dataset_format: hdf5 - dataset_path: dataset/sift-128-euclidean.hdf5 - neighbors_format: hdf5 - neighbors_path: dataset/sift-128-euclidean.hdf5 diff --git a/benchmarks/perf-tool/release-configs/faiss-ivfpq/train-index-spec.json b/benchmarks/perf-tool/release-configs/faiss-ivfpq/train-index-spec.json deleted file mode 100644 index 804a5707e..000000000 --- a/benchmarks/perf-tool/release-configs/faiss-ivfpq/train-index-spec.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "settings": { - "index": { - "number_of_shards": 24, - "number_of_replicas": 0 - } - }, - "mappings": { - "properties": { - "train_field": { - "type": "knn_vector", - "dimension": 128 - } - } - } -} diff --git a/benchmarks/perf-tool/release-configs/lucene-hnsw/filtering/relaxed-filter/index.json b/benchmarks/perf-tool/release-configs/lucene-hnsw/filtering/relaxed-filter/index.json deleted file mode 100644 index 7a9ff2890..000000000 --- a/benchmarks/perf-tool/release-configs/lucene-hnsw/filtering/relaxed-filter/index.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "settings": { - "index": { - "knn": true, - "number_of_shards": 24, - "number_of_replicas": 1 - } - }, - "mappings": { - "properties": { - "target_field": { - "type": "knn_vector", - "dimension": 128, - "method": { - "name": "hnsw", - "space_type": "l2", - "engine": "lucene", - "parameters": { - "ef_construction": 256, - "m": 16 - } - } - } - } - } -} diff --git a/benchmarks/perf-tool/release-configs/lucene-hnsw/filtering/relaxed-filter/relaxed-filter-spec.json b/benchmarks/perf-tool/release-configs/lucene-hnsw/filtering/relaxed-filter/relaxed-filter-spec.json deleted file mode 100644 index 3e04d12c4..000000000 --- a/benchmarks/perf-tool/release-configs/lucene-hnsw/filtering/relaxed-filter/relaxed-filter-spec.json +++ /dev/null @@ -1,42 +0,0 @@ -{ - "bool": - { - "should": - [ - { - "range": - { - "age": - { - "gte": 30, - "lte": 70 - } - } - }, - { - "term": - { - "color": "green" - } - }, - { - "term": - { - "color": "blue" - } - }, - { - "term": - { - "color": "yellow" - } - }, - { - "term": - { - "taste": "sweet" - } - } - ] - } -} diff --git a/benchmarks/perf-tool/release-configs/lucene-hnsw/filtering/relaxed-filter/relaxed-filter-test.yml b/benchmarks/perf-tool/release-configs/lucene-hnsw/filtering/relaxed-filter/relaxed-filter-test.yml deleted file mode 100644 index 3bbb99a0f..000000000 --- a/benchmarks/perf-tool/release-configs/lucene-hnsw/filtering/relaxed-filter/relaxed-filter-test.yml +++ /dev/null @@ -1,38 +0,0 @@ -endpoint: [ENDPOINT] -port: [PORT] -test_name: "Lucene HNSW Relaxed Filter Test" -test_id: "Lucene HNSW Relaxed Filter Test" -num_runs: 3 -show_runs: false -steps: - - name: delete_index - index_name: target_index - - name: create_index - index_name: target_index - index_spec: release-configs/lucene-hnsw/filtering/relaxed-filter/index.json - - name: ingest_multi_field - index_name: target_index - field_name: target_field - bulk_size: 500 - dataset_format: hdf5 - dataset_path: dataset/sift-128-euclidean-with-attr.hdf5 - attributes_dataset_name: attributes - attribute_spec: [ { name: 'color', type: 'str' }, { name: 'taste', type: 'str' }, { name: 'age', type: 'int' } ] - - name: refresh_index - index_name: target_index - - name: force_merge - index_name: target_index - max_num_segments: 1 - - name: query_with_filter - k: 100 - r: 1 - calculate_recall: true - index_name: target_index - field_name: target_field - dataset_format: hdf5 - dataset_path: dataset/sift-128-euclidean-with-attr.hdf5 - neighbors_format: hdf5 - neighbors_path: dataset/sift-128-euclidean-with-relaxed-filters.hdf5 - neighbors_dataset: neighbors_filter_5 - filter_spec: release-configs/lucene-hnsw/filtering/relaxed-filter/relaxed-filter-spec.json - filter_type: FILTER diff --git a/benchmarks/perf-tool/release-configs/lucene-hnsw/filtering/restrictive-filter/index.json b/benchmarks/perf-tool/release-configs/lucene-hnsw/filtering/restrictive-filter/index.json deleted file mode 100644 index 7a9ff2890..000000000 --- a/benchmarks/perf-tool/release-configs/lucene-hnsw/filtering/restrictive-filter/index.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "settings": { - "index": { - "knn": true, - "number_of_shards": 24, - "number_of_replicas": 1 - } - }, - "mappings": { - "properties": { - "target_field": { - "type": "knn_vector", - "dimension": 128, - "method": { - "name": "hnsw", - "space_type": "l2", - "engine": "lucene", - "parameters": { - "ef_construction": 256, - "m": 16 - } - } - } - } - } -} diff --git a/benchmarks/perf-tool/release-configs/lucene-hnsw/filtering/restrictive-filter/restrictive-filter-spec.json b/benchmarks/perf-tool/release-configs/lucene-hnsw/filtering/restrictive-filter/restrictive-filter-spec.json deleted file mode 100644 index 9e6356f1c..000000000 --- a/benchmarks/perf-tool/release-configs/lucene-hnsw/filtering/restrictive-filter/restrictive-filter-spec.json +++ /dev/null @@ -1,44 +0,0 @@ -{ - "bool": - { - "must": - [ - { - "range": - { - "age": - { - "gte": 30, - "lte": 60 - } - } - }, - { - "term": - { - "taste": "bitter" - } - }, - { - "bool": - { - "should": - [ - { - "term": - { - "color": "blue" - } - }, - { - "term": - { - "color": "green" - } - } - ] - } - } - ] - } -} \ No newline at end of file diff --git a/benchmarks/perf-tool/release-configs/lucene-hnsw/filtering/restrictive-filter/restrictive-filter-test.yml b/benchmarks/perf-tool/release-configs/lucene-hnsw/filtering/restrictive-filter/restrictive-filter-test.yml deleted file mode 100644 index aa4c5193f..000000000 --- a/benchmarks/perf-tool/release-configs/lucene-hnsw/filtering/restrictive-filter/restrictive-filter-test.yml +++ /dev/null @@ -1,38 +0,0 @@ -endpoint: [ENDPOINT] -port: [PORT] -test_name: "Lucene HNSW Restrictive Filter Test" -test_id: "Lucene HNSW Restrictive Filter Test" -num_runs: 3 -show_runs: false -steps: - - name: delete_index - index_name: target_index - - name: create_index - index_name: target_index - index_spec: release-configs/lucene-hnsw/filtering/restrictive-filter/index.json - - name: ingest_multi_field - index_name: target_index - field_name: target_field - bulk_size: 500 - dataset_format: hdf5 - dataset_path: dataset/sift-128-euclidean-with-attr.hdf5 - attributes_dataset_name: attributes - attribute_spec: [ { name: 'color', type: 'str' }, { name: 'taste', type: 'str' }, { name: 'age', type: 'int' } ] - - name: refresh_index - index_name: target_index - - name: force_merge - index_name: target_index - max_num_segments: 1 - - name: query_with_filter - k: 100 - r: 1 - calculate_recall: true - index_name: target_index - field_name: target_field - dataset_format: hdf5 - dataset_path: dataset/sift-128-euclidean-with-attr.hdf5 - neighbors_format: hdf5 - neighbors_path: dataset/sift-128-euclidean-with-restrictive-filters.hdf5 - neighbors_dataset: neighbors_filter_4 - filter_spec: release-configs/lucene-hnsw/filtering/restrictive-filter/restrictive-filter-spec.json - filter_type: FILTER diff --git a/benchmarks/perf-tool/release-configs/lucene-hnsw/index.json b/benchmarks/perf-tool/release-configs/lucene-hnsw/index.json deleted file mode 100644 index 7a9ff2890..000000000 --- a/benchmarks/perf-tool/release-configs/lucene-hnsw/index.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "settings": { - "index": { - "knn": true, - "number_of_shards": 24, - "number_of_replicas": 1 - } - }, - "mappings": { - "properties": { - "target_field": { - "type": "knn_vector", - "dimension": 128, - "method": { - "name": "hnsw", - "space_type": "l2", - "engine": "lucene", - "parameters": { - "ef_construction": 256, - "m": 16 - } - } - } - } - } -} diff --git a/benchmarks/perf-tool/release-configs/lucene-hnsw/nested/simple/index.json b/benchmarks/perf-tool/release-configs/lucene-hnsw/nested/simple/index.json deleted file mode 100644 index b41b51c77..000000000 --- a/benchmarks/perf-tool/release-configs/lucene-hnsw/nested/simple/index.json +++ /dev/null @@ -1,34 +0,0 @@ -{ - "settings": { - "index": { - "knn": true, - "number_of_shards": 24, - "number_of_replicas": 1 - } - }, - "mappings": { - "_source": { - "excludes": ["nested_field"] - }, - "properties": { - "nested_field": { - "type": "nested", - "properties": { - "target_field": { - "type": "knn_vector", - "dimension": 128, - "method": { - "name": "hnsw", - "space_type": "l2", - "engine": "lucene", - "parameters": { - "ef_construction": 256, - "m": 16 - } - } - } - } - } - } - } -} diff --git a/benchmarks/perf-tool/release-configs/lucene-hnsw/nested/simple/simple-nested-test.yml b/benchmarks/perf-tool/release-configs/lucene-hnsw/nested/simple/simple-nested-test.yml deleted file mode 100644 index be825487a..000000000 --- a/benchmarks/perf-tool/release-configs/lucene-hnsw/nested/simple/simple-nested-test.yml +++ /dev/null @@ -1,37 +0,0 @@ -endpoint: [ENDPOINT] -port: [PORT] -test_name: "Lucene HNSW Nested Field Test" -test_id: "Lucene HNSW Nested Field Test" -num_runs: 3 -show_runs: false -steps: - - name: delete_index - index_name: target_index - - name: create_index - index_name: target_index - index_spec: release-configs/lucene-hnsw/nested/simple/index.json - - name: ingest_nested_field - index_name: target_index - field_name: target_field - dataset_format: hdf5 - dataset_path: dataset/sift-128-euclidean-nested.hdf5 - attributes_dataset_name: attributes - attribute_spec: [ { name: 'color', type: 'str' }, { name: 'taste', type: 'str' }, { name: 'age', type: 'int' }, { name: 'parent_id', type: 'int'} ] - - name: refresh_index - index_name: target_index - - name: force_merge - index_name: target_index - max_num_segments: 1 - - name: warmup_operation - index_name: target_index - - name: query_nested_field - k: 100 - r: 1 - calculate_recall: true - index_name: target_index - field_name: target_field - dataset_format: hdf5 - dataset_path: dataset/sift-128-euclidean-nested.hdf5 - neighbors_format: hdf5 - neighbors_path: dataset/sift-128-euclidean-nested.hdf5 - neighbors_dataset: neighbour_nested \ No newline at end of file diff --git a/benchmarks/perf-tool/release-configs/lucene-hnsw/test.yml b/benchmarks/perf-tool/release-configs/lucene-hnsw/test.yml deleted file mode 100644 index b253ee08e..000000000 --- a/benchmarks/perf-tool/release-configs/lucene-hnsw/test.yml +++ /dev/null @@ -1,33 +0,0 @@ -endpoint: [ENDPOINT] -port: [PORT] -test_name: "Lucene HNSW" -test_id: "Lucene HNSW" -num_runs: 3 -show_runs: false -steps: - - name: delete_index - index_name: target_index - - name: create_index - index_name: target_index - index_spec: release-configs/lucene-hnsw/index.json - - name: ingest - index_name: target_index - field_name: target_field - bulk_size: 500 - dataset_format: hdf5 - dataset_path: dataset/sift-128-euclidean.hdf5 - - name: refresh_index - index_name: target_index - - name: force_merge - index_name: target_index - max_num_segments: 1 - - name: query - k: 100 - r: 1 - calculate_recall: true - index_name: target_index - field_name: target_field - dataset_format: hdf5 - dataset_path: dataset/sift-128-euclidean.hdf5 - neighbors_format: hdf5 - neighbors_path: dataset/sift-128-euclidean.hdf5 diff --git a/benchmarks/perf-tool/release-configs/nmslib-hnsw/index.json b/benchmarks/perf-tool/release-configs/nmslib-hnsw/index.json deleted file mode 100644 index eb714c5c8..000000000 --- a/benchmarks/perf-tool/release-configs/nmslib-hnsw/index.json +++ /dev/null @@ -1,27 +0,0 @@ -{ - "settings": { - "index": { - "knn": true, - "number_of_shards": 24, - "number_of_replicas": 1, - "knn.algo_param.ef_search": 100 - } - }, - "mappings": { - "properties": { - "target_field": { - "type": "knn_vector", - "dimension": 128, - "method": { - "name": "hnsw", - "space_type": "l2", - "engine": "nmslib", - "parameters": { - "ef_construction": 256, - "m": 16 - } - } - } - } - } -} diff --git a/benchmarks/perf-tool/release-configs/nmslib-hnsw/test.yml b/benchmarks/perf-tool/release-configs/nmslib-hnsw/test.yml deleted file mode 100644 index 94ad9b131..000000000 --- a/benchmarks/perf-tool/release-configs/nmslib-hnsw/test.yml +++ /dev/null @@ -1,35 +0,0 @@ -endpoint: [ENDPOINT] -port: [PORT] -test_name: "Nmslib HNSW Test" -test_id: "Nmslib HNSW Test" -num_runs: 3 -show_runs: false -steps: - - name: delete_index - index_name: target_index - - name: create_index - index_name: target_index - index_spec: release-configs/nmslib-hnsw/index.json - - name: ingest - index_name: target_index - field_name: target_field - bulk_size: 500 - dataset_format: hdf5 - dataset_path: dataset/sift-128-euclidean.hdf5 - - name: refresh_index - index_name: target_index - - name: force_merge - index_name: target_index - max_num_segments: 1 - - name: warmup_operation - index_name: target_index - - name: query - k: 100 - r: 1 - calculate_recall: true - index_name: target_index - field_name: target_field - dataset_format: hdf5 - dataset_path: dataset/sift-128-euclidean.hdf5 - neighbors_format: hdf5 - neighbors_path: dataset/sift-128-euclidean.hdf5 diff --git a/benchmarks/perf-tool/release-configs/run_all_tests.sh b/benchmarks/perf-tool/release-configs/run_all_tests.sh deleted file mode 100755 index e65d5b5c4..000000000 --- a/benchmarks/perf-tool/release-configs/run_all_tests.sh +++ /dev/null @@ -1,102 +0,0 @@ -#!/bin/bash -set -e - -# Description: -# Run a performance test for release -# Dataset should be available in perf-tool/dataset before running this script -# -# Example: -# ./run-test.sh --endpoint localhost -# -# Usage: -# ./run-test.sh \ -# --endpoint -# --port 80 \ -# --num-runs 3 \ -# --outputs ~/outputs - -while [ "$1" != "" ]; do - case $1 in - -url | --endpoint ) shift - ENDPOINT=$1 - ;; - -p | --port ) shift - PORT=$1 - ;; - -n | --num-runs ) shift - NUM_RUNS=$1 - ;; - -o | --outputs ) shift - OUTPUTS=$1 - ;; - * ) echo "Unknown parameter" - echo $1 - exit 1 - ;; - esac - shift -done - -if [ ! -n "$ENDPOINT" ]; then - echo "--endpoint should be specified" - exit -fi - -if [ ! -n "$PORT" ]; then - PORT=80 - echo "--port is not specified. Using default values $PORT" -fi - -if [ ! -n "$NUM_RUNS" ]; then - NUM_RUNS=3 - echo "--num-runs is not specified. Using default values $NUM_RUNS" -fi - -if [ ! -n "$OUTPUTS" ]; then - OUTPUTS="$HOME/outputs" - echo "--outputs is not specified. Using default values $OUTPUTS" -fi - - -curl -X PUT "http://$ENDPOINT:$PORT/_cluster/settings?pretty" -H 'Content-Type: application/json' -d' -{ - "persistent" : { - "knn.algo_param.index_thread_qty" : 4 - } -} -' - -TESTS="./release-configs/faiss-hnsw/filtering/relaxed-filter/relaxed-filter-test.yml -./release-configs/faiss-hnsw/filtering/restrictive-filter/restrictive-filter-test.yml -./release-configs/faiss-hnsw/nested/simple/simple-nested-test.yml -./release-configs/faiss-hnsw/test.yml -./release-configs/faiss-hnswpq/test.yml -./release-configs/faiss-ivf/filtering/relaxed-filter/relaxed-filter-test.yml -./release-configs/faiss-ivf/filtering/restrictive-filter/restrictive-filter-test.yml -./release-configs/faiss-ivf/test.yml -./release-configs/faiss-ivfpq/test.yml -./release-configs/lucene-hnsw/filtering/relaxed-filter/relaxed-filter-test.yml -./release-configs/lucene-hnsw/filtering/restrictive-filter/restrictive-filter-test.yml -./release-configs/lucene-hnsw/nested/simple/simple-nested-test.yml -./release-configs/lucene-hnsw/test.yml -./release-configs/nmslib-hnsw/test.yml" - -if [ ! -d $OUTPUTS ] -then - mkdir $OUTPUTS -fi - -for TEST in $TESTS -do - ORG_FILE=$TEST - NEW_FILE="$ORG_FILE.tmp" - OUT_FILE=$(grep test_id $ORG_FILE | cut -d':' -f2 | sed -r 's/^ "|"$//g' | sed 's/ /_/g') - echo "cp $ORG_FILE $NEW_FILE" - cp $ORG_FILE $NEW_FILE - sed -i "/^endpoint:/c\endpoint: $ENDPOINT" $NEW_FILE - sed -i "/^port:/c\port: $PORT" $NEW_FILE - sed -i "/^num_runs:/c\num_runs: $NUM_RUNS" $NEW_FILE - python3 knn-perf-tool.py test $NEW_FILE $OUTPUTS/$OUT_FILE - #Sleep for 1 min to cool down cpu from the previous run - sleep 60 -done diff --git a/benchmarks/perf-tool/requirements.in b/benchmarks/perf-tool/requirements.in deleted file mode 100644 index fd3555aab..000000000 --- a/benchmarks/perf-tool/requirements.in +++ /dev/null @@ -1,7 +0,0 @@ -Cerberus -opensearch-py -PyYAML -numpy -h5py -requests -psutil diff --git a/benchmarks/perf-tool/requirements.txt b/benchmarks/perf-tool/requirements.txt deleted file mode 100644 index 46cec00ed..000000000 --- a/benchmarks/perf-tool/requirements.txt +++ /dev/null @@ -1,37 +0,0 @@ -# -# This file is autogenerated by pip-compile with python 3.9 -# To update, run: -# -# pip-compile -# -cerberus==1.3.4 - # via -r requirements.in -certifi==2023.7.22 - # via - # opensearch-py - # requests -charset-normalizer==2.0.4 - # via requests -h5py==3.3.0 - # via -r requirements.in -idna==3.7 - # via requests -numpy==1.24.2 - # via - # -r requirements.in - # h5py -opensearch-py==1.0.0 - # via -r requirements.in -psutil==5.8.0 - # via -r requirements.in -pyyaml==5.4.1 - # via -r requirements.in -requests==2.31.0 - # via -r requirements.in -urllib3==1.26.18 - # via - # opensearch-py - # requests - -# The following packages are considered to be unsafe in a requirements file: -# setuptools diff --git a/benchmarks/perf-tool/sample-configs/faiss-sift-ivf/index-spec.json b/benchmarks/perf-tool/sample-configs/faiss-sift-ivf/index-spec.json deleted file mode 100644 index 5542ef387..000000000 --- a/benchmarks/perf-tool/sample-configs/faiss-sift-ivf/index-spec.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "settings": { - "index": { - "knn": true, - "number_of_shards": 3, - "number_of_replicas": 0 - } - }, - "mappings": { - "properties": { - "target_field": { - "type": "knn_vector", - "model_id": "test-model" - } - } - } -} diff --git a/benchmarks/perf-tool/sample-configs/faiss-sift-ivf/method-spec.json b/benchmarks/perf-tool/sample-configs/faiss-sift-ivf/method-spec.json deleted file mode 100644 index 1aa7f809f..000000000 --- a/benchmarks/perf-tool/sample-configs/faiss-sift-ivf/method-spec.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "name":"ivf", - "engine":"faiss", - "parameters":{ - "nlist":16, - "nprobes": 4 - } -} diff --git a/benchmarks/perf-tool/sample-configs/faiss-sift-ivf/test.yml b/benchmarks/perf-tool/sample-configs/faiss-sift-ivf/test.yml deleted file mode 100644 index 027ba8683..000000000 --- a/benchmarks/perf-tool/sample-configs/faiss-sift-ivf/test.yml +++ /dev/null @@ -1,62 +0,0 @@ -endpoint: localhost -test_name: faiss_sift_ivf -test_id: "Test workflow for faiss ivf" -num_runs: 3 -show_runs: true -setup: - - name: delete_model - model_id: test-model - - name: delete_index - index_name: target_index - - name: delete_index - index_name: train_index - - name: create_index - index_name: train_index - index_spec: sample-configs/faiss-sift-ivf/train-index-spec.json - - name: ingest - index_name: train_index - field_name: train_field - bulk_size: 500 - dataset_format: hdf5 - dataset_path: ../dataset/sift-128-euclidean.hdf5 - - name: refresh_index - index_name: train_index -steps: - - name: train_model - model_id: test-model - train_index: train_index - train_field: train_field - dimension: 128 - method_spec: sample-configs/faiss-sift-ivf/method-spec.json - max_training_vector_count: 1000000000 - - name: create_index - index_name: target_index - index_spec: sample-configs/faiss-sift-ivf/index-spec.json - - name: ingest - index_name: target_index - field_name: target_field - bulk_size: 500 - dataset_format: hdf5 - dataset_path: ../dataset/sift-128-euclidean.hdf5 - - name: refresh_index - index_name: target_index - - name: force_merge - index_name: target_index - max_num_segments: 10 - - name: warmup_operation - index_name: target_index - - name: query - k: 100 - r: 1 - calculate_recall: true - index_name: target_index - field_name: target_field - dataset_format: hdf5 - dataset_path: ../dataset/sift-128-euclidean.hdf5 - neighbors_format: hdf5 - neighbors_path: ../dataset/sift-128-euclidean.hdf5 -cleanup: - - name: delete_model - model_id: test-model - - name: delete_index - index_name: target_index diff --git a/benchmarks/perf-tool/sample-configs/faiss-sift-ivf/train-index-spec.json b/benchmarks/perf-tool/sample-configs/faiss-sift-ivf/train-index-spec.json deleted file mode 100644 index 00a418e4f..000000000 --- a/benchmarks/perf-tool/sample-configs/faiss-sift-ivf/train-index-spec.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "settings": { - "index": { - "number_of_shards": 3, - "number_of_replicas": 0 - } - }, - "mappings": { - "properties": { - "train_field": { - "type": "knn_vector", - "dimension": 128 - } - } - } -} diff --git a/benchmarks/perf-tool/sample-configs/filter-spec/filter-1-spec.json b/benchmarks/perf-tool/sample-configs/filter-spec/filter-1-spec.json deleted file mode 100644 index f529de4fe..000000000 --- a/benchmarks/perf-tool/sample-configs/filter-spec/filter-1-spec.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "bool": - { - "must": - [ - { - "range": - { - "age": - { - "gte": 20, - "lte": 100 - } - } - }, - { - "term": - { - "color": "red" - } - } - ] - } -} \ No newline at end of file diff --git a/benchmarks/perf-tool/sample-configs/filter-spec/filter-2-spec.json b/benchmarks/perf-tool/sample-configs/filter-spec/filter-2-spec.json deleted file mode 100644 index 9d4514e62..000000000 --- a/benchmarks/perf-tool/sample-configs/filter-spec/filter-2-spec.json +++ /dev/null @@ -1,40 +0,0 @@ -{ - "bool": - { - "must": - [ - { - "term": - { - "taste": "salty" - } - }, - { - "bool": - { - "should": - [ - { - "bool": - { - "must_not": - { - "exists": - { - "field": "color" - } - } - } - }, - { - "term": - { - "color": "blue" - } - } - ] - } - } - ] - } -} \ No newline at end of file diff --git a/benchmarks/perf-tool/sample-configs/filter-spec/filter-3-spec.json b/benchmarks/perf-tool/sample-configs/filter-spec/filter-3-spec.json deleted file mode 100644 index d69f8768e..000000000 --- a/benchmarks/perf-tool/sample-configs/filter-spec/filter-3-spec.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "bool": - { - "must": - [ - { - "range": - { - "age": - { - "gte": 20, - "lte": 80 - } - } - }, - { - "exists": - { - "field": "color" - } - }, - { - "exists": - { - "field": "taste" - } - } - ] - } -} \ No newline at end of file diff --git a/benchmarks/perf-tool/sample-configs/filter-spec/filter-4-spec.json b/benchmarks/perf-tool/sample-configs/filter-spec/filter-4-spec.json deleted file mode 100644 index 822d63b37..000000000 --- a/benchmarks/perf-tool/sample-configs/filter-spec/filter-4-spec.json +++ /dev/null @@ -1,44 +0,0 @@ -{ - "bool": - { - "must": - [ - { - "range": - { - "age": - { - "gte": 30, - "lte": 60 - } - } - }, - { - "term": - { - "taste": "bitter" - } - }, - { - "bool": - { - "should": - [ - { - "term": - { - "color": "blue" - } - }, - { - "term": - { - "color": "green" - } - } - ] - } - } - ] - } -} diff --git a/benchmarks/perf-tool/sample-configs/filter-spec/filter-5-spec.json b/benchmarks/perf-tool/sample-configs/filter-spec/filter-5-spec.json deleted file mode 100644 index 3e04d12c4..000000000 --- a/benchmarks/perf-tool/sample-configs/filter-spec/filter-5-spec.json +++ /dev/null @@ -1,42 +0,0 @@ -{ - "bool": - { - "should": - [ - { - "range": - { - "age": - { - "gte": 30, - "lte": 70 - } - } - }, - { - "term": - { - "color": "green" - } - }, - { - "term": - { - "color": "blue" - } - }, - { - "term": - { - "color": "yellow" - } - }, - { - "term": - { - "taste": "sweet" - } - } - ] - } -} diff --git a/benchmarks/perf-tool/sample-configs/lucene-sift-hnsw-filter/index-spec.json b/benchmarks/perf-tool/sample-configs/lucene-sift-hnsw-filter/index-spec.json deleted file mode 100644 index 83ea79b15..000000000 --- a/benchmarks/perf-tool/sample-configs/lucene-sift-hnsw-filter/index-spec.json +++ /dev/null @@ -1,27 +0,0 @@ -{ - "settings": { - "index": { - "knn": true, - "refresh_interval": "10s", - "number_of_shards": 30, - "number_of_replicas": 0 - } - }, - "mappings": { - "properties": { - "target_field": { - "type": "knn_vector", - "dimension": 128, - "method": { - "name": "hnsw", - "space_type": "l2", - "engine": "lucene", - "parameters": { - "ef_construction": 100, - "m": 16 - } - } - } - } - } -} diff --git a/benchmarks/perf-tool/sample-configs/lucene-sift-hnsw-filter/test.yml b/benchmarks/perf-tool/sample-configs/lucene-sift-hnsw-filter/test.yml deleted file mode 100644 index aa2ee6389..000000000 --- a/benchmarks/perf-tool/sample-configs/lucene-sift-hnsw-filter/test.yml +++ /dev/null @@ -1,41 +0,0 @@ -endpoint: localhost -test_name: lucene_sift_hnsw -test_id: "Test workflow for lucene hnsw" -num_runs: 1 -show_runs: false -setup: - - name: delete_index - index_name: target_index -steps: - - name: create_index - index_name: target_index - index_spec: sample-configs/lucene-sift-hnsw-filter/index-spec.json - - name: ingest_multi_field - index_name: target_index - field_name: target_field - bulk_size: 500 - dataset_format: hdf5 - dataset_path: ../dataset/sift-128-euclidean-with-attr.hdf5 - attributes_dataset_name: attributes - attribute_spec: [ { name: 'color', type: 'str' }, { name: 'taste', type: 'str' }, { name: 'age', type: 'int' } ] - - name: refresh_index - index_name: target_index - - name: force_merge - index_name: target_index - max_num_segments: 10 - - name: query_with_filter - k: 10 - r: 1 - calculate_recall: true - index_name: target_index - field_name: target_field - dataset_format: hdf5 - dataset_path: ../dataset/sift-128-euclidean-with-attr.hdf5 - neighbors_format: hdf5 - neighbors_path: ../dataset/sift-128-euclidean-with-attr-with-filters.hdf5 - neighbors_dataset: neighbors_filter_1 - filter_spec: sample-configs/filter-spec/filter-1-spec.json - query_count: 100 -cleanup: - - name: delete_index - index_name: target_index \ No newline at end of file diff --git a/benchmarks/perf-tool/sample-configs/nmslib-sift-hnsw/index-spec.json b/benchmarks/perf-tool/sample-configs/nmslib-sift-hnsw/index-spec.json deleted file mode 100644 index 75abe7baa..000000000 --- a/benchmarks/perf-tool/sample-configs/nmslib-sift-hnsw/index-spec.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "settings": { - "index": { - "knn": true, - "knn.algo_param.ef_search": 512, - "refresh_interval": "10s", - "number_of_shards": 1, - "number_of_replicas": 0 - } - }, - "mappings": { - "properties": { - "target_field": { - "type": "knn_vector", - "dimension": 128, - "method": { - "name": "hnsw", - "space_type": "l2", - "engine": "nmslib", - "parameters": { - "ef_construction": 512, - "m": 16 - } - } - } - } - } -} diff --git a/benchmarks/perf-tool/sample-configs/nmslib-sift-hnsw/test.yml b/benchmarks/perf-tool/sample-configs/nmslib-sift-hnsw/test.yml deleted file mode 100644 index 6d96bf80c..000000000 --- a/benchmarks/perf-tool/sample-configs/nmslib-sift-hnsw/test.yml +++ /dev/null @@ -1,38 +0,0 @@ -endpoint: localhost -test_name: nmslib_sift_hnsw -test_id: "Test workflow for nmslib hnsw" -num_runs: 2 -show_runs: false -setup: - - name: delete_index - index_name: target_index -steps: - - name: create_index - index_name: target_index - index_spec: sample-configs/nmslib-sift-hnsw/index-spec.json - - name: ingest - index_name: target_index - field_name: target_field - bulk_size: 500 - dataset_format: hdf5 - dataset_path: ../dataset/sift-128-euclidean.hdf5 - - name: refresh_index - index_name: target_index - - name: force_merge - index_name: target_index - max_num_segments: 10 - - name: warmup_operation - index_name: target_index - - name: query - k: 100 - r: 1 - calculate_recall: true - index_name: target_index - field_name: target_field - dataset_format: hdf5 - dataset_path: ../dataset/sift-128-euclidean.hdf5 - neighbors_format: hdf5 - neighbors_path: ../dataset/sift-128-euclidean.hdf5 -cleanup: - - name: delete_index - index_name: target_index From 222737b203996fc95fefede72304773b9ea0ed05 Mon Sep 17 00:00:00 2001 From: Martin Gaievski Date: Thu, 25 Jul 2024 09:42:09 -0700 Subject: [PATCH 304/416] Corrected search logic for scenario with non-existent fields in filter (#1874) (#1881) * Return empty results for non-existent filter fields (cherry picked from commit 43b072745d5b08d10d135b693645c2343a3a605f) Signed-off-by: Martin Gaievski --- CHANGELOG.md | 1 + .../knn/index/query/KNNQueryBuilder.java | 10 +++ .../org/opensearch/knn/index/FaissIT.java | 81 +++++++++++++++++++ .../opensearch/knn/index/LuceneEngineIT.java | 77 +++++++++++++++++- .../knn/index/query/KNNQueryBuilderTests.java | 45 +++++++++++ 5 files changed, 213 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d10752c01..09b4c47f9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Features ### Enhancements ### Bug Fixes +* Corrected search logic for scenario with non-existent fields in filter [#1874](https://github.com/opensearch-project/k-NN/pull/1874) ### Infrastructure ### Documentation ### Maintenance diff --git a/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java b/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java index e319fa388..63d05540e 100644 --- a/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java +++ b/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java @@ -24,6 +24,7 @@ import org.opensearch.index.mapper.NumberFieldMapper; import org.opensearch.index.query.AbstractQueryBuilder; import org.opensearch.index.query.QueryBuilder; +import org.opensearch.index.query.QueryRewriteContext; import org.opensearch.index.query.QueryShardContext; import org.opensearch.knn.common.KNNConstants; import org.opensearch.knn.index.IndexUtil; @@ -739,4 +740,13 @@ protected int doHashCode() { public String getWriteableName() { return NAME; } + + @Override + protected QueryBuilder doRewrite(QueryRewriteContext queryShardContext) throws IOException { + // rewrite filter query if it exists to avoid runtime errors in next steps of query phase + if (Objects.nonNull(filter)) { + filter = filter.rewrite(queryShardContext); + } + return super.doRewrite(queryShardContext); + } } diff --git a/src/test/java/org/opensearch/knn/index/FaissIT.java b/src/test/java/org/opensearch/knn/index/FaissIT.java index 33a37a4a4..db225b52d 100644 --- a/src/test/java/org/opensearch/knn/index/FaissIT.java +++ b/src/test/java/org/opensearch/knn/index/FaissIT.java @@ -23,6 +23,7 @@ import org.opensearch.common.settings.Settings; import org.opensearch.client.ResponseException; import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.index.query.QueryBuilder; import org.opensearch.index.query.QueryBuilders; import org.opensearch.index.query.TermQueryBuilder; import org.opensearch.knn.KNNRestTestCase; @@ -83,6 +84,15 @@ public class FaissIT extends KNNRestTestCase { private static final String COLOR_FIELD_NAME = "color"; private static final String TASTE_FIELD_NAME = "taste"; + private static final String DIMENSION_FIELD_NAME = "dimension"; + private static final int VECTOR_DIMENSION = 3; + private static final String KNN_VECTOR_TYPE = "knn_vector"; + private static final String PROPERTIES_FIELD_NAME = "properties"; + private static final String TYPE_FIELD_NAME = "type"; + private static final String INTEGER_FIELD_NAME = "int_field"; + private static final String FILED_TYPE_INTEGER = "integer"; + private static final String NON_EXISTENT_INTEGER_FIELD_NAME = "nonexistent_int_field"; + static TestUtils.TestData testData; @BeforeClass @@ -1710,6 +1720,77 @@ public void testIVF_whenBinaryFormat_whenIVF_thenSuccess() { validateGraphEviction(); } + @SneakyThrows + public void testQueryWithFilter_whenNonExistingFieldUsedInFilter_thenSuccessful() { + XContentBuilder builder = XContentFactory.jsonBuilder() + .startObject() + .startObject(PROPERTIES_FIELD_NAME) + .startObject(FIELD_NAME) + .field(TYPE_FIELD_NAME, KNN_VECTOR_TYPE) + .field(DIMENSION_FIELD_NAME, VECTOR_DIMENSION) + .startObject(KNNConstants.KNN_METHOD) + .field(KNNConstants.NAME, KNNEngine.FAISS.getMethod(METHOD_HNSW).getMethodComponent().getName()) + .field(KNNConstants.METHOD_PARAMETER_SPACE_TYPE, SpaceType.L2.getValue()) + .field(KNNConstants.KNN_ENGINE, KNNEngine.LUCENE.getName()) + .endObject() + .endObject() + .startObject(INTEGER_FIELD_NAME) + .field(TYPE_FIELD_NAME, FILED_TYPE_INTEGER) + .endObject() + .endObject() + .endObject(); + Map mappingMap = xContentBuilderToMap(builder); + String mapping = builder.toString(); + + createKnnIndex(INDEX_NAME, mapping); + + Float[] vector = new Float[] { 2.0f, 4.5f, 6.5f }; + + String documentAsString = XContentFactory.jsonBuilder() + .startObject() + .field(INTEGER_FIELD_NAME, 5) + .field(FIELD_NAME, vector) + .endObject() + .toString(); + + addKnnDoc(INDEX_NAME, DOC_ID_1, documentAsString); + + refreshIndex(INDEX_NAME); + assertEquals(1, getDocCount(INDEX_NAME)); + + float[] searchVector = new float[] { 1.0f, 2.1f, 3.9f }; + int k = 10; + + // use filter where nonexistent field is must, we should have no results + QueryBuilder filterWithRequiredNonExistentField = QueryBuilders.boolQuery() + .must(QueryBuilders.rangeQuery(NON_EXISTENT_INTEGER_FIELD_NAME).gte(1)); + Response searchWithRequiredNonExistentFiledInFilterResponse = searchKNNIndex( + INDEX_NAME, + new KNNQueryBuilder(FIELD_NAME, searchVector, k, filterWithRequiredNonExistentField), + k + ); + List resultsQuery1 = parseSearchResponse( + EntityUtils.toString(searchWithRequiredNonExistentFiledInFilterResponse.getEntity()), + FIELD_NAME + ); + assertTrue(resultsQuery1.isEmpty()); + + // use filter with non existent field as optional, we should have some results + QueryBuilder filterWithOptionalNonExistentField = QueryBuilders.boolQuery() + .should(QueryBuilders.rangeQuery(NON_EXISTENT_INTEGER_FIELD_NAME).gte(1)) + .must(QueryBuilders.rangeQuery(INTEGER_FIELD_NAME).gte(1)); + Response searchWithOptionalNonExistentFiledInFilterResponse = searchKNNIndex( + INDEX_NAME, + new KNNQueryBuilder(FIELD_NAME, searchVector, k, filterWithOptionalNonExistentField), + k + ); + List resultsQuery2 = parseSearchResponse( + EntityUtils.toString(searchWithOptionalNonExistentFiledInFilterResponse.getEntity()), + FIELD_NAME + ); + assertEquals(1, resultsQuery2.size()); + } + protected void setupKNNIndexForFilterQuery() throws Exception { // Create Mappings XContentBuilder builder = XContentFactory.jsonBuilder() diff --git a/src/test/java/org/opensearch/knn/index/LuceneEngineIT.java b/src/test/java/org/opensearch/knn/index/LuceneEngineIT.java index 1a047ac95..26f22ff84 100644 --- a/src/test/java/org/opensearch/knn/index/LuceneEngineIT.java +++ b/src/test/java/org/opensearch/knn/index/LuceneEngineIT.java @@ -15,8 +15,9 @@ import org.opensearch.client.Response; import org.opensearch.client.ResponseException; import org.opensearch.common.Nullable; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.index.query.QueryBuilder; import org.opensearch.index.query.QueryBuilders; import org.opensearch.knn.KNNRestTestCase; import org.opensearch.knn.KNNResult; @@ -86,6 +87,9 @@ public class LuceneEngineIT extends KNNRestTestCase { private static final String KNN_VECTOR_TYPE = "knn_vector"; private static final String PROPERTIES_FIELD_NAME = "properties"; private static final String TYPE_FIELD_NAME = "type"; + private static final String INTEGER_FIELD_NAME = "int_field"; + private static final String FILED_TYPE_INTEGER = "integer"; + private static final String NON_EXISTENT_INTEGER_FIELD_NAME = "nonexistent_int_field"; @After public final void cleanUp() throws IOException { @@ -278,6 +282,77 @@ public void testQueryWithFilterUsingByteVectorDataType() { validateQueryResultsWithFilters(searchVector, 5, 1, expectedDocIdsKGreaterThanFilterResult, expectedDocIdsKLimitsFilterResult); } + @SneakyThrows + public void testQueryWithFilter_whenNonExistingFieldUsedInFilter_thenSuccessful() { + XContentBuilder builder = XContentFactory.jsonBuilder() + .startObject() + .startObject(PROPERTIES_FIELD_NAME) + .startObject(FIELD_NAME) + .field(TYPE_FIELD_NAME, KNN_VECTOR_TYPE) + .field(DIMENSION_FIELD_NAME, DIMENSION) + .startObject(KNNConstants.KNN_METHOD) + .field(KNNConstants.NAME, KNNEngine.LUCENE.getMethod(METHOD_HNSW).getMethodComponent().getName()) + .field(KNNConstants.METHOD_PARAMETER_SPACE_TYPE, SpaceType.L2.getValue()) + .field(KNNConstants.KNN_ENGINE, KNNEngine.LUCENE.getName()) + .endObject() + .endObject() + .startObject(INTEGER_FIELD_NAME) + .field(TYPE_FIELD_NAME, FILED_TYPE_INTEGER) + .endObject() + .endObject() + .endObject(); + Map mappingMap = xContentBuilderToMap(builder); + String mapping = builder.toString(); + + createKnnIndex(INDEX_NAME, mapping); + + Float[] vector = new Float[] { 2.0f, 4.5f, 6.5f }; + + String documentAsString = XContentFactory.jsonBuilder() + .startObject() + .field(INTEGER_FIELD_NAME, 5) + .field(FIELD_NAME, vector) + .endObject() + .toString(); + + addKnnDoc(INDEX_NAME, DOC_ID, documentAsString); + + refreshIndex(INDEX_NAME); + assertEquals(1, getDocCount(INDEX_NAME)); + + float[] searchVector = new float[] { 1.0f, 2.1f, 3.9f }; + int k = 10; + + // use filter where nonexistent field is must, we should have no results + QueryBuilder filterWithRequiredNonExistentField = QueryBuilders.boolQuery() + .must(QueryBuilders.rangeQuery(NON_EXISTENT_INTEGER_FIELD_NAME).gte(1)); + Response searchWithRequiredNonExistentFiledInFilterResponse = searchKNNIndex( + INDEX_NAME, + new KNNQueryBuilder(FIELD_NAME, searchVector, k, filterWithRequiredNonExistentField), + k + ); + List resultsQuery1 = parseSearchResponse( + EntityUtils.toString(searchWithRequiredNonExistentFiledInFilterResponse.getEntity()), + FIELD_NAME + ); + assertTrue(resultsQuery1.isEmpty()); + + // use filter with non existent field as optional, we should have some results + QueryBuilder filterWithOptionalNonExistentField = QueryBuilders.boolQuery() + .should(QueryBuilders.rangeQuery(NON_EXISTENT_INTEGER_FIELD_NAME).gte(1)) + .must(QueryBuilders.rangeQuery(INTEGER_FIELD_NAME).gte(1)); + Response searchWithOptionalNonExistentFiledInFilterResponse = searchKNNIndex( + INDEX_NAME, + new KNNQueryBuilder(FIELD_NAME, searchVector, k, filterWithOptionalNonExistentField), + k + ); + List resultsQuery2 = parseSearchResponse( + EntityUtils.toString(searchWithOptionalNonExistentFiledInFilterResponse.getEntity()), + FIELD_NAME + ); + assertEquals(1, resultsQuery2.size()); + } + public void testQuery_filterWithNonLuceneEngine() throws Exception { XContentBuilder builder = XContentFactory.jsonBuilder() .startObject() diff --git a/src/test/java/org/opensearch/knn/index/query/KNNQueryBuilderTests.java b/src/test/java/org/opensearch/knn/index/query/KNNQueryBuilderTests.java index 18e2b914a..5a6caf64d 100644 --- a/src/test/java/org/opensearch/knn/index/query/KNNQueryBuilderTests.java +++ b/src/test/java/org/opensearch/knn/index/query/KNNQueryBuilderTests.java @@ -6,6 +6,7 @@ package org.opensearch.knn.index.query; import com.google.common.collect.ImmutableMap; +import lombok.SneakyThrows; import org.apache.lucene.search.FloatVectorSimilarityQuery; import org.apache.lucene.search.KnnFloatVectorQuery; import org.apache.lucene.search.MatchNoDocsQuery; @@ -26,6 +27,7 @@ import org.opensearch.index.mapper.NumberFieldMapper; import org.opensearch.index.query.QueryBuilder; import org.opensearch.index.query.QueryBuilders; +import org.opensearch.index.query.QueryRewriteContext; import org.opensearch.index.query.QueryShardContext; import org.opensearch.index.query.TermQueryBuilder; import org.opensearch.knn.KNNTestCase; @@ -69,6 +71,8 @@ public class KNNQueryBuilderTests extends KNNTestCase { private static final Float MIN_SCORE = 0.5f; private static final TermQueryBuilder TERM_QUERY = QueryBuilders.termQuery("field", "value"); private static final float[] QUERY_VECTOR = new float[] { 1.0f, 2.0f, 3.0f, 4.0f }; + protected static final String TEXT_FIELD_NAME = "some_field"; + protected static final String TEXT_VALUE = "some_value"; public void testInvalidK() { float[] queryVector = { 1.0f, 1.0f }; @@ -512,6 +516,7 @@ public void testDoToQuery_Normal() throws Exception { assertEquals(knnQueryBuilder.vector(), query.getQueryVector()); } + @SneakyThrows public void testDoToQuery_whenNormal_whenDoRadiusSearch_whenDistanceThreshold_thenSucceed() { float[] queryVector = { 1.0f, 2.0f, 3.0f, 4.0f }; KNNQueryBuilder knnQueryBuilder = KNNQueryBuilder.builder() @@ -545,6 +550,7 @@ public void testDoToQuery_whenNormal_whenDoRadiusSearch_whenDistanceThreshold_th ); } + @SneakyThrows public void testDoToQuery_whenNormal_whenDoRadiusSearch_whenScoreThreshold_thenSucceed() { float[] queryVector = { 1.0f, 2.0f, 3.0f, 4.0f }; @@ -567,6 +573,7 @@ public void testDoToQuery_whenNormal_whenDoRadiusSearch_whenScoreThreshold_thenS assertTrue(query.toString().contains("resultSimilarity=" + 0.5f)); } + @SneakyThrows public void testDoToQuery_whenDoRadiusSearch_whenPassNegativeDistance_whenSupportedSpaceType_thenSucceed() { float[] queryVector = { 1.0f, 2.0f, 3.0f, 4.0f }; float negativeDistance = -1.0f; @@ -629,6 +636,7 @@ public void testDoToQuery_whenDoRadiusSearch_whenPassNegativeDistance_whenUnSupp expectThrows(IllegalArgumentException.class, () -> knnQueryBuilder.doToQuery(mockQueryShardContext)); } + @SneakyThrows public void testDoToQuery_whenDoRadiusSearch_whenPassScoreMoreThanOne_whenSupportedSpaceType_thenSucceed() { float[] queryVector = { 1.0f, 2.0f, 3.0f, 4.0f }; float score = 5f; @@ -682,6 +690,7 @@ public void testDoToQuery_whenDoRadiusSearch_whenPassScoreMoreThanOne_whenUnsupp expectThrows(IllegalArgumentException.class, () -> knnQueryBuilder.doToQuery(mockQueryShardContext)); } + @SneakyThrows public void testDoToQuery_whenPassNegativeDistance_whenSupportedSpaceType_thenSucceed() { float[] queryVector = { 1.0f, 2.0f, 3.0f, 4.0f }; float negativeDistance = -1.0f; @@ -801,6 +810,7 @@ public void testDoToQuery_KnnQueryWithFilter_Lucene() throws Exception { assertTrue(query.getClass().isAssignableFrom(KnnFloatVectorQuery.class)); } + @SneakyThrows public void testDoToQuery_whenDoRadiusSearch_whenDistanceThreshold_whenFilter_thenSucceed() { float[] queryVector = { 1.0f, 2.0f, 3.0f, 4.0f }; @@ -829,6 +839,7 @@ public void testDoToQuery_whenDoRadiusSearch_whenDistanceThreshold_whenFilter_th assertTrue(query.getClass().isAssignableFrom(FloatVectorSimilarityQuery.class)); } + @SneakyThrows public void testDoToQuery_whenDoRadiusSearch_whenScoreThreshold_whenFilter_thenSucceed() { float[] queryVector = { 1.0f, 2.0f, 3.0f, 4.0f }; KNNQueryBuilder knnQueryBuilder = KNNQueryBuilder.builder() @@ -855,6 +866,7 @@ public void testDoToQuery_whenDoRadiusSearch_whenScoreThreshold_whenFilter_thenS assertTrue(query.getClass().isAssignableFrom(FloatVectorSimilarityQuery.class)); } + @SneakyThrows public void testDoToQuery_WhenknnQueryWithFilterAndFaissEngine_thenSuccess() { // Given float[] queryVector = { 1.0f, 2.0f, 3.0f, 4.0f }; @@ -931,6 +943,7 @@ public void testDoToQuery_whenknnQueryWithFilterAndNmsLibEngine_thenException() expectThrows(IllegalArgumentException.class, () -> knnQueryBuilder.doToQuery(mockQueryShardContext)); } + @SneakyThrows public void testDoToQuery_FromModel() { float[] queryVector = { 1.0f, 2.0f, 3.0f, 4.0f }; KNNQueryBuilder knnQueryBuilder = new KNNQueryBuilder(FIELD_NAME, queryVector, K); @@ -965,6 +978,7 @@ public void testDoToQuery_FromModel() { assertEquals(knnQueryBuilder.vector(), query.getQueryVector()); } + @SneakyThrows public void testDoToQuery_whenFromModel_whenDoRadiusSearch_whenDistanceThreshold_thenSucceed() { float[] queryVector = { 1.0f, 2.0f, 3.0f, 4.0f }; @@ -1006,6 +1020,7 @@ public void testDoToQuery_whenFromModel_whenDoRadiusSearch_whenDistanceThreshold assertEquals(knnQueryBuilder.vector(), query.getQueryVector()); } + @SneakyThrows public void testDoToQuery_whenFromModel_whenDoRadiusSearch_whenScoreThreshold_thenSucceed() { float[] queryVector = { 1.0f, 2.0f, 3.0f, 4.0f }; @@ -1260,6 +1275,7 @@ public void testRadialSearch_whenEfSearchIsSet_whenLuceneEngine_thenThrowExcepti expectThrows(IllegalArgumentException.class, () -> knnQueryBuilder.doToQuery(mockQueryShardContext)); } + @SneakyThrows public void testRadialSearch_whenEfSearchIsSet_whenFaissEngine_thenSuccess() { KNNMethodContext knnMethodContext = new KNNMethodContext( KNNEngine.FAISS, @@ -1320,4 +1336,33 @@ public void testDoToQuery_whenBinaryWithInvalidDimension_thenException() throws Exception ex = expectThrows(IllegalArgumentException.class, () -> knnQueryBuilder.doToQuery(mockQueryShardContext)); assertTrue(ex.getMessage(), ex.getMessage().contains("invalid dimension")); } + + @SneakyThrows + public void testDoRewrite_whenNoFilter_thenSuccessful() { + KNNQueryBuilder knnQueryBuilder = new KNNQueryBuilder(FIELD_NAME, QUERY_VECTOR, K); + QueryBuilder rewritten = knnQueryBuilder.rewrite(mock(QueryRewriteContext.class)); + assertEquals(knnQueryBuilder, rewritten); + } + + @SneakyThrows + public void testDoRewrite_whenFilterSet_thenSuccessful() { + // Given + QueryBuilder filter = mock(QueryBuilder.class); + QueryBuilder rewrittenFilter = mock(QueryBuilder.class); + QueryRewriteContext context = mock(QueryRewriteContext.class); + when(filter.rewrite(context)).thenReturn(rewrittenFilter); + KNNQueryBuilder expected = KNNQueryBuilder.builder() + .fieldName(FIELD_NAME) + .vector(QUERY_VECTOR) + .filter(rewrittenFilter) + .k(K) + .build(); + // When + KNNQueryBuilder knnQueryBuilder = KNNQueryBuilder.builder().fieldName(FIELD_NAME).vector(QUERY_VECTOR).filter(filter).k(K).build(); + + QueryBuilder actual = knnQueryBuilder.rewrite(context); + + // Then + assertEquals(expected, actual); + } } From 59bb71a9c76117a19fb5ce338c97960569207ef4 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Thu, 25 Jul 2024 12:28:22 -0700 Subject: [PATCH 305/416] Apply https://github.com/opensearch-project/k-NN/pull/1804 (#1882) (#1886) Signed-off-by: Heemin Kim (cherry picked from commit 161a60ad04c09fb41c1c39865c59a8fb5a007459) Co-authored-by: Heemin Kim --- .../knn/index/codec/transfer/VectorTransferByte.java | 10 ++++++---- .../knn/index/codec/transfer/VectorTransferFloat.java | 10 ++++++---- .../index/codec/transfer/VectorTransferByteTests.java | 2 +- .../index/codec/transfer/VectorTransferFloatTests.java | 2 +- 4 files changed, 14 insertions(+), 10 deletions(-) diff --git a/src/main/java/org/opensearch/knn/index/codec/transfer/VectorTransferByte.java b/src/main/java/org/opensearch/knn/index/codec/transfer/VectorTransferByte.java index 5e9831708..e81ac35fc 100644 --- a/src/main/java/org/opensearch/knn/index/codec/transfer/VectorTransferByte.java +++ b/src/main/java/org/opensearch/knn/index/codec/transfer/VectorTransferByte.java @@ -34,11 +34,13 @@ public void init(final long totalLiveDocs) { public void transfer(final BytesRef bytesRef) { dimension = bytesRef.length * 8; if (vectorsPerTransfer == Integer.MIN_VALUE) { - vectorsPerTransfer = (bytesRef.length * totalLiveDocs) / vectorsStreamingMemoryLimit; - // This condition comes if vectorsStreamingMemoryLimit is higher than total number floats to transfer - // Doing this will reduce 1 extra trip to JNI layer. + // if vectorsStreamingMemoryLimit is 100 bytes and we have 50 vectors with length of 5, then per + // transfer we have to send 100/5 => 20 vectors. + vectorsPerTransfer = vectorsStreamingMemoryLimit / bytesRef.length; + // If vectorsPerTransfer comes out to be 0, then we set number of vectors per transfer to 1, to ensure that + // we are sending minimum number of vectors. if (vectorsPerTransfer == 0) { - vectorsPerTransfer = totalLiveDocs; + vectorsPerTransfer = 1; } } diff --git a/src/main/java/org/opensearch/knn/index/codec/transfer/VectorTransferFloat.java b/src/main/java/org/opensearch/knn/index/codec/transfer/VectorTransferFloat.java index af6d9490e..a9c792398 100644 --- a/src/main/java/org/opensearch/knn/index/codec/transfer/VectorTransferFloat.java +++ b/src/main/java/org/opensearch/knn/index/codec/transfer/VectorTransferFloat.java @@ -38,11 +38,13 @@ public void transfer(final BytesRef bytesRef) { dimension = vector.length; if (vectorsPerTransfer == Integer.MIN_VALUE) { - vectorsPerTransfer = (dimension * Float.BYTES * totalLiveDocs) / vectorsStreamingMemoryLimit; - // This condition comes if vectorsStreamingMemoryLimit is higher than total number floats to transfer - // Doing this will reduce 1 extra trip to JNI layer. + // if vectorsStreamingMemoryLimit is 100 bytes and we have 50 vectors with 5 dimension, then per + // transfer we have to send 100/(5 * 4) => 5 vectors. + vectorsPerTransfer = vectorsStreamingMemoryLimit / ((long) dimension * Float.BYTES); + // If vectorsPerTransfer comes out to be 0, then we set number of vectors per transfer to 1, to ensure that + // we are sending minimum number of vectors. if (vectorsPerTransfer == 0) { - vectorsPerTransfer = totalLiveDocs; + vectorsPerTransfer = 1; } } diff --git a/src/test/java/org/opensearch/knn/index/codec/transfer/VectorTransferByteTests.java b/src/test/java/org/opensearch/knn/index/codec/transfer/VectorTransferByteTests.java index 7e837fbf2..2f091a035 100644 --- a/src/test/java/org/opensearch/knn/index/codec/transfer/VectorTransferByteTests.java +++ b/src/test/java/org/opensearch/knn/index/codec/transfer/VectorTransferByteTests.java @@ -21,7 +21,7 @@ public class VectorTransferByteTests extends TestCase { public void testTransfer_whenCalled_thenAdded() { final BytesRef bytesRef1 = getByteArrayOfVectors(20); final BytesRef bytesRef2 = getByteArrayOfVectors(20); - VectorTransferByte vectorTransfer = new VectorTransferByte(1000); + VectorTransferByte vectorTransfer = new VectorTransferByte(40); try { vectorTransfer.init(2); diff --git a/src/test/java/org/opensearch/knn/index/codec/transfer/VectorTransferFloatTests.java b/src/test/java/org/opensearch/knn/index/codec/transfer/VectorTransferFloatTests.java index 1f36f320d..620fd7c65 100644 --- a/src/test/java/org/opensearch/knn/index/codec/transfer/VectorTransferFloatTests.java +++ b/src/test/java/org/opensearch/knn/index/codec/transfer/VectorTransferFloatTests.java @@ -24,7 +24,7 @@ public class VectorTransferFloatTests extends TestCase { public void testTransfer_whenCalled_thenAdded() { final BytesRef bytesRef1 = getByteArrayOfVectors(20); final BytesRef bytesRef2 = getByteArrayOfVectors(20); - VectorTransferFloat vectorTransfer = new VectorTransferFloat(1000); + VectorTransferFloat vectorTransfer = new VectorTransferFloat(160); try { vectorTransfer.init(2); From c1450cd69736f5636475f028d8cac77ebce0ab6d Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Fri, 26 Jul 2024 22:25:17 -0700 Subject: [PATCH 306/416] Introduce NativeEngineKNNVectorsFormat as a KNNVectorsFormat for Native engines (#1855) (#1887) Signed-off-by: Navneet Verma (cherry picked from commit 1e03e59136e792f95546662fcfd1078b12d3e52a) Co-authored-by: Navneet Verma --- .../NativeEngineFieldVectorsWriter.java | 107 +++++++++ .../NativeEngines990KnnVectorsFormat.java | 68 ++++++ .../NativeEngines990KnnVectorsReader.java | 162 ++++++++++++++ .../NativeEngines990KnnVectorsWriter.java | 115 ++++++++++ .../codec/KNN990Codec/UnitTestCodec.java | 37 ++++ .../services/org.apache.lucene.codecs.Codec | 3 +- .../org.apache.lucene.codecs.KnnVectorsFormat | 12 + .../NativeEngineFieldVectorsWriterTests.java | 98 ++++++++ ...NativeEngines990KnnVectorsFormatTests.java | 209 ++++++++++++++++++ 9 files changed, 810 insertions(+), 1 deletion(-) create mode 100644 src/main/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngineFieldVectorsWriter.java create mode 100644 src/main/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsFormat.java create mode 100644 src/main/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsReader.java create mode 100644 src/main/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsWriter.java create mode 100644 src/main/java/org/opensearch/knn/index/codec/KNN990Codec/UnitTestCodec.java create mode 100644 src/main/resources/META-INF/services/org.apache.lucene.codecs.KnnVectorsFormat create mode 100644 src/test/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngineFieldVectorsWriterTests.java create mode 100644 src/test/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsFormatTests.java diff --git a/src/main/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngineFieldVectorsWriter.java b/src/main/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngineFieldVectorsWriter.java new file mode 100644 index 000000000..26171eece --- /dev/null +++ b/src/main/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngineFieldVectorsWriter.java @@ -0,0 +1,107 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.knn.index.codec.KNN990Codec; + +import lombok.Getter; +import org.apache.lucene.codecs.KnnFieldVectorsWriter; +import org.apache.lucene.index.DocsWithFieldSet; +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.util.InfoStream; +import org.apache.lucene.util.RamUsageEstimator; + +import java.util.HashMap; +import java.util.Map; + +/** + * NativeEngineVectorFieldsWriter is a class that will be used to accumulate all the vectors during ingestion before + * lucene does a flush. This class ensures that KNNVectorWriter is free from generics and this class can encapsulate + * all the details related to vectors types and docIds. + * + * @param float[] or byte[] + */ +class NativeEngineFieldVectorsWriter extends KnnFieldVectorsWriter { + private static final long SHALLOW_SIZE = RamUsageEstimator.shallowSizeOfInstance(NativeEngineFieldVectorsWriter.class); + private final FieldInfo fieldInfo; + /** + * We are using a map here instead of list, because for sampler interface for quantization we have to advance the iterator + * to a specific docId, there a list cannot be useful because a docId != index of the vector in the list. Similar + * thing is true when we have vector field in child document. There doc Ids will not be consistent. Hence, we need to + * use the map here. + */ + @Getter + private final Map vectors; + private int lastDocID = -1; + @Getter + private final DocsWithFieldSet docsWithField; + private final InfoStream infoStream; + + static NativeEngineFieldVectorsWriter create(final FieldInfo fieldInfo, final InfoStream infoStream) { + switch (fieldInfo.getVectorEncoding()) { + case FLOAT32: + return new NativeEngineFieldVectorsWriter(fieldInfo, infoStream); + case BYTE: + return new NativeEngineFieldVectorsWriter(fieldInfo, infoStream); + } + throw new IllegalStateException("Unsupported Vector encoding : " + fieldInfo.getVectorEncoding()); + } + + NativeEngineFieldVectorsWriter(final FieldInfo fieldInfo, final InfoStream infoStream) { + this.fieldInfo = fieldInfo; + this.infoStream = infoStream; + vectors = new HashMap<>(); + this.docsWithField = new DocsWithFieldSet(); + } + + /** + * Add new docID with its vector value to the given field for indexing. Doc IDs must be added in + * increasing order. + * + * @param docID int + * @param vectorValue T + */ + @Override + public void addValue(int docID, T vectorValue) { + if (docID == lastDocID) { + throw new IllegalArgumentException( + "[NativeEngineKNNVectorWriter]VectorValuesField \"" + + fieldInfo.name + + "\" appears more than once in this document (only one value is allowed per field)" + ); + } + assert docID > lastDocID; + vectors.put(docID, vectorValue); + docsWithField.add(docID); + lastDocID = docID; + } + + /** + * Used to copy values being indexed to internal storage. + * + * @param vectorValue an array containing the vector value to add + * @return a copy of the value; a new array + */ + @Override + public T copyValue(T vectorValue) { + throw new UnsupportedOperationException("NativeEngineVectorFieldsWriter doesn't support copyValue operation"); + } + + /** + * Return the memory usage of this object in bytes. Negative values are illegal. + */ + @Override + public long ramBytesUsed() { + return SHALLOW_SIZE + docsWithField.ramBytesUsed() + (long) this.vectors.size() * (long) (RamUsageEstimator.NUM_BYTES_OBJECT_REF + + RamUsageEstimator.NUM_BYTES_ARRAY_HEADER) + (long) this.vectors.size() * RamUsageEstimator.shallowSizeOfInstance( + Integer.class + ) + (long) vectors.size() * fieldInfo.getVectorDimension() * fieldInfo.getVectorEncoding().byteSize; + } +} diff --git a/src/main/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsFormat.java b/src/main/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsFormat.java new file mode 100644 index 000000000..6582cdd1f --- /dev/null +++ b/src/main/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsFormat.java @@ -0,0 +1,68 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.knn.index.codec.KNN990Codec; + +import org.apache.lucene.codecs.KnnVectorsFormat; +import org.apache.lucene.codecs.KnnVectorsReader; +import org.apache.lucene.codecs.KnnVectorsWriter; +import org.apache.lucene.codecs.hnsw.DefaultFlatVectorScorer; +import org.apache.lucene.codecs.hnsw.FlatVectorsFormat; +import org.apache.lucene.codecs.lucene99.Lucene99FlatVectorsFormat; +import org.apache.lucene.index.SegmentReadState; +import org.apache.lucene.index.SegmentWriteState; + +import java.io.IOException; + +/** + * This is a Vector format that will be used for Native engines like Faiss and Nmslib for reading and writing vector + * related data structures. + */ +public class NativeEngines990KnnVectorsFormat extends KnnVectorsFormat { + /** The format for storing, reading, merging vectors on disk */ + private static FlatVectorsFormat flatVectorsFormat; + private static final String FORMAT_NAME = "NativeEngines99KnnVectorsFormat"; + + public NativeEngines990KnnVectorsFormat() { + super(FORMAT_NAME); + flatVectorsFormat = new Lucene99FlatVectorsFormat(new DefaultFlatVectorScorer()); + } + + public NativeEngines990KnnVectorsFormat(final FlatVectorsFormat lucene99FlatVectorsFormat) { + super(FORMAT_NAME); + flatVectorsFormat = lucene99FlatVectorsFormat; + } + + /** + * Returns a {@link KnnVectorsWriter} to write the vectors to the index. + * + * @param state {@link SegmentWriteState} + */ + @Override + public KnnVectorsWriter fieldsWriter(final SegmentWriteState state) throws IOException { + return new NativeEngines990KnnVectorsWriter(state, flatVectorsFormat.fieldsWriter(state)); + } + + /** + * Returns a {@link KnnVectorsReader} to read the vectors from the index. + * + * @param state {@link SegmentReadState} + */ + @Override + public KnnVectorsReader fieldsReader(final SegmentReadState state) throws IOException { + return new NativeEngines990KnnVectorsReader(state, flatVectorsFormat.fieldsReader(state)); + } + + @Override + public String toString() { + return "NativeEngines99KnnVectorsFormat(name=" + this.getClass().getSimpleName() + ", flatVectorsFormat=" + flatVectorsFormat + ")"; + } +} diff --git a/src/main/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsReader.java b/src/main/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsReader.java new file mode 100644 index 000000000..74b158fa5 --- /dev/null +++ b/src/main/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsReader.java @@ -0,0 +1,162 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.knn.index.codec.KNN990Codec; + +import org.apache.lucene.codecs.KnnVectorsReader; +import org.apache.lucene.codecs.hnsw.FlatVectorsReader; +import org.apache.lucene.index.ByteVectorValues; +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.FloatVectorValues; +import org.apache.lucene.index.SegmentReadState; +import org.apache.lucene.search.KnnCollector; +import org.apache.lucene.search.ScoreDoc; +import org.apache.lucene.search.TopDocs; +import org.apache.lucene.search.TotalHits; +import org.apache.lucene.util.Bits; +import org.apache.lucene.util.IOUtils; + +import java.io.IOException; + +/** + * Vectors reader class for reading the flat vectors for native engines. The class provides methods for iterating + * over the vectors and retrieving their values. + */ +public class NativeEngines990KnnVectorsReader extends KnnVectorsReader { + + private final FlatVectorsReader flatVectorsReader; + + public NativeEngines990KnnVectorsReader(final SegmentReadState state, final FlatVectorsReader flatVectorsReader) { + this.flatVectorsReader = flatVectorsReader; + } + + /** + * Checks consistency of this reader. + * + *

    Note that this may be costly in terms of I/O, e.g. may involve computing a checksum value + * against large data files. + * + */ + @Override + public void checkIntegrity() throws IOException { + flatVectorsReader.checkIntegrity(); + } + + /** + * Returns the {@link FloatVectorValues} for the given {@code field}. The behavior is undefined if + * the given field doesn't have KNN vectors enabled on its {@link FieldInfo}. The return value is + * never {@code null}. + * + * @param field {@link String} + */ + @Override + public FloatVectorValues getFloatVectorValues(final String field) throws IOException { + return flatVectorsReader.getFloatVectorValues(field); + } + + /** + * Returns the {@link ByteVectorValues} for the given {@code field}. The behavior is undefined if + * the given field doesn't have KNN vectors enabled on its {@link FieldInfo}. The return value is + * never {@code null}. + * + * @param field {@link String} + */ + @Override + public ByteVectorValues getByteVectorValues(final String field) throws IOException { + return flatVectorsReader.getByteVectorValues(field); + } + + /** + * Return the k nearest neighbor documents as determined by comparison of their vector values for + * this field, to the given vector, by the field's similarity function. The score of each document + * is derived from the vector similarity in a way that ensures scores are positive and that a + * larger score corresponds to a higher ranking. + * + *

    The search is allowed to be approximate, meaning the results are not guaranteed to be the + * true k closest neighbors. For large values of k (for example when k is close to the total + * number of documents), the search may also retrieve fewer than k documents. + * + *

    The returned {@link TopDocs} will contain a {@link ScoreDoc} for each nearest neighbor, in + * order of their similarity to the query vector (decreasing scores). The {@link TotalHits} + * contains the number of documents visited during the search. If the search stopped early because + * it hit {@code visitedLimit}, it is indicated through the relation {@code + * TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO}. + * + *

    The behavior is undefined if the given field doesn't have KNN vectors enabled on its {@link + * FieldInfo}. The return value is never {@code null}. + * + * @param field the vector field to search + * @param target the vector-valued query + * @param knnCollector a KnnResults collector and relevant settings for gathering vector results + * @param acceptDocs {@link Bits} that represents the allowed documents to match, or {@code null} + * if they are all allowed to match. + */ + @Override + public void search(String field, float[] target, KnnCollector knnCollector, Bits acceptDocs) throws IOException { + throw new UnsupportedOperationException("Search functionality using codec is not supported with Native Engine Reader"); + } + + /** + * Return the k nearest neighbor documents as determined by comparison of their vector values for + * this field, to the given vector, by the field's similarity function. The score of each document + * is derived from the vector similarity in a way that ensures scores are positive and that a + * larger score corresponds to a higher ranking. + * + *

    The search is allowed to be approximate, meaning the results are not guaranteed to be the + * true k closest neighbors. For large values of k (for example when k is close to the total + * number of documents), the search may also retrieve fewer than k documents. + * + *

    The returned {@link TopDocs} will contain a {@link ScoreDoc} for each nearest neighbor, in + * order of their similarity to the query vector (decreasing scores). The {@link TotalHits} + * contains the number of documents visited during the search. If the search stopped early because + * it hit {@code visitedLimit}, it is indicated through the relation {@code + * TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO}. + * + *

    The behavior is undefined if the given field doesn't have KNN vectors enabled on its {@link + * FieldInfo}. The return value is never {@code null}. + * + * @param field the vector field to search + * @param target the vector-valued query + * @param knnCollector a KnnResults collector and relevant settings for gathering vector results + * @param acceptDocs {@link Bits} that represents the allowed documents to match, or {@code null} + * if they are all allowed to match. + */ + @Override + public void search(String field, byte[] target, KnnCollector knnCollector, Bits acceptDocs) throws IOException { + throw new UnsupportedOperationException("Search functionality using codec is not supported with Native Engine Reader"); + } + + /** + * Closes this stream and releases any system resources associated + * with it. If the stream is already closed then invoking this + * method has no effect. + * + *

    As noted in {@link AutoCloseable#close()}, cases where the + * close may fail require careful attention. It is strongly advised + * to relinquish the underlying resources and to internally + * mark the {@code Closeable} as closed, prior to throwing + * the {@code IOException}. + * + * @throws IOException if an I/O error occurs + */ + @Override + public void close() throws IOException { + IOUtils.close(flatVectorsReader); + } + + /** + * Return the memory usage of this object in bytes. Negative values are illegal. + */ + @Override + public long ramBytesUsed() { + return flatVectorsReader.ramBytesUsed(); + } +} diff --git a/src/main/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsWriter.java b/src/main/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsWriter.java new file mode 100644 index 000000000..b81ec9789 --- /dev/null +++ b/src/main/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsWriter.java @@ -0,0 +1,115 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.knn.index.codec.KNN990Codec; + +import lombok.RequiredArgsConstructor; +import org.apache.lucene.codecs.KnnFieldVectorsWriter; +import org.apache.lucene.codecs.KnnVectorsWriter; +import org.apache.lucene.codecs.hnsw.FlatVectorsWriter; +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.MergeState; +import org.apache.lucene.index.SegmentWriteState; +import org.apache.lucene.index.Sorter; +import org.apache.lucene.util.IOUtils; +import org.apache.lucene.util.RamUsageEstimator; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +/** + * A KNNVectorsWriter class for writing the vector data strcutures and flat vectors for Native Engines. + */ +@RequiredArgsConstructor +public class NativeEngines990KnnVectorsWriter extends KnnVectorsWriter { + private static final long SHALLOW_SIZE = RamUsageEstimator.shallowSizeOfInstance(NativeEngines990KnnVectorsWriter.class); + private final SegmentWriteState segmentWriteState; + private final FlatVectorsWriter flatVectorsWriter; + private final List> fields = new ArrayList<>(); + private boolean finished; + + /** + * Add new field for indexing. + * In Lucene, we use single file for all the vector fields so here we need to see how we are going to make things + * work. + * @param fieldInfo {@link FieldInfo} + */ + @Override + public KnnFieldVectorsWriter addField(final FieldInfo fieldInfo) throws IOException { + final NativeEngineFieldVectorsWriter newField = NativeEngineFieldVectorsWriter.create(fieldInfo, segmentWriteState.infoStream); + // TODO: we can build the graph here too iteratively. but right now I am skipping that as we need iterative + // graph build support on the JNI layer. + fields.add(newField); + return flatVectorsWriter.addField(fieldInfo, newField); + } + + /** + * Flush all buffered data on disk. This is not fsync. This is lucene flush. + * + * @param maxDoc int + * @param sortMap {@link Sorter.DocMap} + */ + @Override + public void flush(int maxDoc, final Sorter.DocMap sortMap) throws IOException { + // simply write data in the flat file + flatVectorsWriter.flush(maxDoc, sortMap); + // TODO: add code for creating Vector datastructures during lucene flush operation + } + + @Override + public void mergeOneField(final FieldInfo fieldInfo, final MergeState mergeState) throws IOException { + // This will ensure that we are merging the FlatIndex during force merge. + flatVectorsWriter.mergeOneField(fieldInfo, mergeState); + // TODO: add code for creating Vector datastructures during merge operation + } + + /** + * Called once at the end before close + */ + @Override + public void finish() throws IOException { + if (finished) { + throw new IllegalStateException("NativeEnginesKNNVectorsWriter is already finished"); + } + finished = true; + flatVectorsWriter.finish(); + } + + /** + * Closes this stream and releases any system resources associated + * with it. If the stream is already closed then invoking this + * method has no effect. + * + *

    As noted in {@link AutoCloseable#close()}, cases where the + * close may fail require careful attention. It is strongly advised + * to relinquish the underlying resources and to internally + * mark the {@code Closeable} as closed, prior to throwing + * the {@code IOException}. + * + * @throws IOException if an I/O error occurs + */ + @Override + public void close() throws IOException { + IOUtils.close(flatVectorsWriter); + } + + /** + * Return the memory usage of this object in bytes. Negative values are illegal. + */ + @Override + public long ramBytesUsed() { + return SHALLOW_SIZE + flatVectorsWriter.ramBytesUsed() + fields.stream() + .mapToLong(NativeEngineFieldVectorsWriter::ramBytesUsed) + .sum(); + } + +} diff --git a/src/main/java/org/opensearch/knn/index/codec/KNN990Codec/UnitTestCodec.java b/src/main/java/org/opensearch/knn/index/codec/KNN990Codec/UnitTestCodec.java new file mode 100644 index 000000000..6998efbc3 --- /dev/null +++ b/src/main/java/org/opensearch/knn/index/codec/KNN990Codec/UnitTestCodec.java @@ -0,0 +1,37 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.knn.index.codec.KNN990Codec; + +import org.apache.lucene.codecs.FilterCodec; +import org.apache.lucene.codecs.KnnVectorsFormat; +import org.apache.lucene.codecs.perfield.PerFieldKnnVectorsFormat; +import org.opensearch.knn.index.codec.KNNCodecVersion; + +/** + * This codec is for testing. The reason for putting this codec here is SPI is only scanning the src folder and not + * able to pick this class if its in test folder. Don't use this codec outside of testing + */ +public class UnitTestCodec extends FilterCodec { + public UnitTestCodec() { + super("UnitTestCodec", KNNCodecVersion.current().getDefaultKnnCodecSupplier().get()); + } + + @Override + public KnnVectorsFormat knnVectorsFormat() { + return new PerFieldKnnVectorsFormat() { + @Override + public KnnVectorsFormat getKnnVectorsFormatForField(String field) { + return new NativeEngines990KnnVectorsFormat(); + } + }; + } +} diff --git a/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec b/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec index 308b37967..401b094b2 100644 --- a/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec +++ b/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec @@ -6,4 +6,5 @@ org.opensearch.knn.index.codec.KNN910Codec.KNN910Codec org.opensearch.knn.index.codec.KNN920Codec.KNN920Codec org.opensearch.knn.index.codec.KNN940Codec.KNN940Codec org.opensearch.knn.index.codec.KNN950Codec.KNN950Codec -org.opensearch.knn.index.codec.KNN990Codec.KNN990Codec \ No newline at end of file +org.opensearch.knn.index.codec.KNN990Codec.KNN990Codec +org.opensearch.knn.index.codec.KNN990Codec.UnitTestCodec diff --git a/src/main/resources/META-INF/services/org.apache.lucene.codecs.KnnVectorsFormat b/src/main/resources/META-INF/services/org.apache.lucene.codecs.KnnVectorsFormat new file mode 100644 index 000000000..d799c3869 --- /dev/null +++ b/src/main/resources/META-INF/services/org.apache.lucene.codecs.KnnVectorsFormat @@ -0,0 +1,12 @@ +# +# SPDX-License-Identifier: Apache-2.0 +# +# The OpenSearch Contributors require contributions made to +# this file be licensed under the Apache-2.0 license or a +# compatible open source license. +# +# Modifications Copyright OpenSearch Contributors. See +# GitHub history for details. +# + +org.opensearch.knn.index.codec.KNN990Codec.NativeEngines990KnnVectorsFormat diff --git a/src/test/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngineFieldVectorsWriterTests.java b/src/test/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngineFieldVectorsWriterTests.java new file mode 100644 index 000000000..16d6b20da --- /dev/null +++ b/src/test/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngineFieldVectorsWriterTests.java @@ -0,0 +1,98 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.knn.index.codec.KNN990Codec; + +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.VectorEncoding; +import org.apache.lucene.util.InfoStream; +import org.junit.Assert; +import org.mockito.Mockito; +import org.opensearch.knn.index.codec.KNNCodecTestCase; + +public class NativeEngineFieldVectorsWriterTests extends KNNCodecTestCase { + + @SuppressWarnings("unchecked") + public void testCreate_ForDifferentInputs_thenSuccess() { + final FieldInfo fieldInfo = Mockito.mock(FieldInfo.class); + Mockito.when(fieldInfo.getVectorEncoding()).thenReturn(VectorEncoding.FLOAT32); + NativeEngineFieldVectorsWriter floatWriter = (NativeEngineFieldVectorsWriter) NativeEngineFieldVectorsWriter + .create(fieldInfo, InfoStream.getDefault()); + floatWriter.addValue(1, new float[] { 1.0f, 2.0f }); + + Mockito.verify(fieldInfo).getVectorEncoding(); + + Mockito.when(fieldInfo.getVectorEncoding()).thenReturn(VectorEncoding.BYTE); + NativeEngineFieldVectorsWriter byteWriter = (NativeEngineFieldVectorsWriter) NativeEngineFieldVectorsWriter.create( + fieldInfo, + InfoStream.getDefault() + ); + Assert.assertNotNull(byteWriter); + Mockito.verify(fieldInfo, Mockito.times(2)).getVectorEncoding(); + byteWriter.addValue(1, new byte[] { 1, 2 }); + } + + public void testAddValue_ForDifferentInputs_thenSuccess() { + final FieldInfo fieldInfo = Mockito.mock(FieldInfo.class); + final NativeEngineFieldVectorsWriter floatWriter = new NativeEngineFieldVectorsWriter<>( + fieldInfo, + InfoStream.getDefault() + ); + final float[] vec1 = new float[] { 1.0f, 2.0f }; + final float[] vec2 = new float[] { 2.0f, 2.0f }; + floatWriter.addValue(1, vec1); + floatWriter.addValue(2, vec2); + + Assert.assertEquals(vec1, floatWriter.getVectors().get(1)); + Assert.assertEquals(vec2, floatWriter.getVectors().get(2)); + Mockito.verify(fieldInfo, Mockito.never()).getVectorEncoding(); + + final NativeEngineFieldVectorsWriter byteWriter = new NativeEngineFieldVectorsWriter<>(fieldInfo, InfoStream.getDefault()); + final byte[] bvec1 = new byte[] { 1, 2 }; + final byte[] bvec2 = new byte[] { 2, 2 }; + byteWriter.addValue(1, bvec1); + byteWriter.addValue(2, bvec2); + + Assert.assertEquals(bvec1, byteWriter.getVectors().get(1)); + Assert.assertEquals(bvec2, byteWriter.getVectors().get(2)); + Mockito.verify(fieldInfo, Mockito.never()).getVectorEncoding(); + } + + public void testCopyValue_whenValidInput_thenException() { + final FieldInfo fieldInfo = Mockito.mock(FieldInfo.class); + final NativeEngineFieldVectorsWriter floatWriter = new NativeEngineFieldVectorsWriter<>( + fieldInfo, + InfoStream.getDefault() + ); + expectThrows(UnsupportedOperationException.class, () -> floatWriter.copyValue(new float[3])); + + final NativeEngineFieldVectorsWriter byteWriter = new NativeEngineFieldVectorsWriter<>(fieldInfo, InfoStream.getDefault()); + expectThrows(UnsupportedOperationException.class, () -> byteWriter.copyValue(new byte[3])); + } + + public void testRamByteUsed_whenValidInput_thenSuccess() { + final FieldInfo fieldInfo = Mockito.mock(FieldInfo.class); + Mockito.when(fieldInfo.getVectorEncoding()).thenReturn(VectorEncoding.FLOAT32); + Mockito.when(fieldInfo.getVectorDimension()).thenReturn(2); + final NativeEngineFieldVectorsWriter floatWriter = new NativeEngineFieldVectorsWriter<>( + fieldInfo, + InfoStream.getDefault() + ); + // testing for value > 0 as we don't have a concrete way to find out expected bytes. This can OS dependent too. + Assert.assertTrue(floatWriter.ramBytesUsed() > 0); + + Mockito.when(fieldInfo.getVectorEncoding()).thenReturn(VectorEncoding.BYTE); + final NativeEngineFieldVectorsWriter byteWriter = new NativeEngineFieldVectorsWriter<>(fieldInfo, InfoStream.getDefault()); + // testing for value > 0 as we don't have a concrete way to find out expected bytes. This can OS dependent too. + Assert.assertTrue(byteWriter.ramBytesUsed() > 0); + + } +} diff --git a/src/test/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsFormatTests.java b/src/test/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsFormatTests.java new file mode 100644 index 000000000..f9f6a7ed9 --- /dev/null +++ b/src/test/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsFormatTests.java @@ -0,0 +1,209 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.knn.index.codec.KNN990Codec; + +import lombok.SneakyThrows; +import lombok.extern.log4j.Log4j2; +import org.apache.lucene.codecs.Codec; +import org.apache.lucene.codecs.hnsw.FlatVectorsReader; +import org.apache.lucene.codecs.hnsw.FlatVectorsWriter; +import org.apache.lucene.codecs.lucene99.Lucene99FlatVectorsFormat; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.FieldType; +import org.apache.lucene.document.KnnByteVectorField; +import org.apache.lucene.document.KnnFloatVectorField; +import org.apache.lucene.index.ByteVectorValues; +import org.apache.lucene.index.FloatVectorValues; +import org.apache.lucene.index.IndexOptions; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.LeafReader; +import org.apache.lucene.index.NoMergePolicy; +import org.apache.lucene.index.SegmentReadState; +import org.apache.lucene.index.SegmentReader; +import org.apache.lucene.index.SegmentWriteState; +import org.apache.lucene.index.SerialMergeScheduler; +import org.apache.lucene.index.VectorEncoding; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.store.Directory; +import org.apache.lucene.tests.index.RandomIndexWriter; +import org.apache.lucene.tests.store.BaseDirectoryWrapper; +import org.apache.lucene.util.Bits; +import org.junit.After; +import org.junit.Assert; +import org.mockito.Mockito; +import org.opensearch.common.lucene.Lucene; +import org.opensearch.knn.KNNTestCase; +import org.opensearch.knn.common.KNNConstants; +import org.opensearch.knn.index.SpaceType; +import org.opensearch.knn.index.mapper.KNNVectorFieldMapper; +import org.opensearch.knn.index.util.KNNEngine; + +import java.io.IOException; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; + +@Log4j2 +public class NativeEngines990KnnVectorsFormatTests extends KNNTestCase { + private static final Codec TESTING_CODEC = new UnitTestCodec(); + private static final String FLAT_VECTOR_FILE_EXT = ".vec"; + private static final String HNSW_FILE_EXT = ".hnsw"; + private static final String FLOAT_VECTOR_FIELD = "float_field"; + private static final String BYTE_VECTOR_FIELD = "byte_field"; + private Directory dir; + private RandomIndexWriter indexWriter; + + @After + public void tearDown() throws Exception { + if (dir != null) { + dir.close(); + } + super.tearDown(); + } + + @SneakyThrows + public void testReaderAndWriter_whenValidInput_thenSuccess() { + final Lucene99FlatVectorsFormat mockedFlatVectorsFormat = Mockito.mock(Lucene99FlatVectorsFormat.class); + final SegmentWriteState mockedSegmentWriteState = Mockito.mock(SegmentWriteState.class); + final SegmentReadState mockedSegmentReadState = Mockito.mock(SegmentReadState.class); + + Mockito.when(mockedFlatVectorsFormat.fieldsReader(mockedSegmentReadState)).thenReturn(Mockito.mock(FlatVectorsReader.class)); + Mockito.when(mockedFlatVectorsFormat.fieldsWriter(mockedSegmentWriteState)).thenReturn(Mockito.mock(FlatVectorsWriter.class)); + + final NativeEngines990KnnVectorsFormat nativeEngines990KnnVectorsFormat = new NativeEngines990KnnVectorsFormat( + mockedFlatVectorsFormat + ); + Assert.assertTrue( + nativeEngines990KnnVectorsFormat.fieldsReader(mockedSegmentReadState) instanceof NativeEngines990KnnVectorsReader + ); + Assert.assertTrue( + nativeEngines990KnnVectorsFormat.fieldsWriter(mockedSegmentWriteState) instanceof NativeEngines990KnnVectorsWriter + ); + } + + @SneakyThrows + public void testNativeEngineVectorFormat_whenMultipleVectorFieldIndexed_thenSuccess() { + setup(); + float[] floatVector = { 1.0f, 3.0f, 4.0f }; + byte[] byteVector = { 6, 14 }; + + addFieldToIndex( + new KnnFloatVectorField(FLOAT_VECTOR_FIELD, floatVector, createVectorField(3, VectorEncoding.FLOAT32)), + indexWriter + ); + addFieldToIndex(new KnnByteVectorField(BYTE_VECTOR_FIELD, byteVector, createVectorField(2, VectorEncoding.BYTE)), indexWriter); + final IndexReader indexReader = indexWriter.getReader(); + // ensuring segments are created + indexWriter.flush(); + indexWriter.commit(); + indexWriter.close(); + + // Validate to see if correct values are returned, assumption here is only 1 segment is getting created + IndexSearcher searcher = new IndexSearcher(indexReader); + final LeafReader leafReader = searcher.getLeafContexts().get(0).reader(); + SegmentReader segmentReader = Lucene.segmentReader(leafReader); + final List hnswfiles = getFilesFromSegment(dir, HNSW_FILE_EXT); + // 0 hnsw files for now as we have not integrated graph creation here. + assertEquals(0, hnswfiles.size()); + assertEquals(hnswfiles.stream().filter(x -> x.contains(FLOAT_VECTOR_FIELD)).count(), 0); + assertEquals(hnswfiles.stream().filter(x -> x.contains(BYTE_VECTOR_FIELD)).count(), 0); + + // Even setting IWC to not use compound file it still uses compound file, hence ensuring we don't check .vec + // file in case segment uses compound format. use this seed once we fix this to validate everything is + // working or not. -Dtests.seed=CAAE1B8D573EEB7E + if (segmentReader.getSegmentInfo().info.getUseCompoundFile() == false) { + final List vecfiles = getFilesFromSegment(dir, FLAT_VECTOR_FILE_EXT); + // 2 .vec files will be created as we are using per field vectors format. + assertEquals(2, vecfiles.size()); + } + + final FloatVectorValues floatVectorValues = leafReader.getFloatVectorValues(FLOAT_VECTOR_FIELD); + floatVectorValues.nextDoc(); + assertArrayEquals(floatVector, floatVectorValues.vectorValue(), 0.0f); + assertEquals(1, floatVectorValues.size()); + assertEquals(3, floatVectorValues.dimension()); + + final ByteVectorValues byteVectorValues = leafReader.getByteVectorValues(BYTE_VECTOR_FIELD); + byteVectorValues.nextDoc(); + assertArrayEquals(byteVector, byteVectorValues.vectorValue()); + assertEquals(1, byteVectorValues.size()); + assertEquals(2, byteVectorValues.dimension()); + + Assert.assertThrows( + UnsupportedOperationException.class, + () -> leafReader.searchNearestVectors(FLOAT_VECTOR_FIELD, floatVector, 10, new Bits.MatchAllBits(1), 10) + ); + + Assert.assertThrows( + UnsupportedOperationException.class, + () -> leafReader.searchNearestVectors(BYTE_VECTOR_FIELD, byteVector, 10, new Bits.MatchAllBits(1), 10) + ); + // do it at the end so that all search is completed + indexReader.close(); + } + + private List getFilesFromSegment(Directory dir, String fileFormat) throws IOException { + return Arrays.stream(dir.listAll()).filter(x -> x.contains(fileFormat)).collect(Collectors.toList()); + } + + /** + * This should have been annotated with @Before, but somehow when I annotate with @Before apart from running + * before tests, it is also running independently and failing. Need to figure this out. + * @throws IOException + */ + private void setup() throws IOException { + dir = newFSDirectory(createTempDir()); + // on the mock directory Lucene goes ahead and does a search on different fields. We want to avoid that as of + // now. Given we have not implemented search for the native engine format using codec, the dir.close fails + // with exception. Hence, marking this as false. + ((BaseDirectoryWrapper) dir).setCheckIndexOnClose(false); + indexWriter = createIndexWriter(dir); + } + + private RandomIndexWriter createIndexWriter(final Directory dir) throws IOException { + final IndexWriterConfig iwc = newIndexWriterConfig(); + iwc.setMergeScheduler(new SerialMergeScheduler()); + iwc.setCodec(TESTING_CODEC); + iwc.setUseCompoundFile(false); + // Set merge policy to no merges so that we create a predictable number of segments. + iwc.setMergePolicy(NoMergePolicy.INSTANCE); + return new RandomIndexWriter(random(), dir, iwc); + } + + private void addFieldToIndex(final Field vectorField, final RandomIndexWriter indexWriter) throws IOException { + final Document doc1 = new Document(); + doc1.add(vectorField); + indexWriter.addDocument(doc1); + } + + private FieldType createVectorField(int dimension, VectorEncoding vectorEncoding) { + FieldType nativeVectorField = new FieldType(); + // TODO: Replace this with the default field which will be created in mapper for Native Engines with KNNVectorsFormat + nativeVectorField.setTokenized(false); + nativeVectorField.setIndexOptions(IndexOptions.NONE); + nativeVectorField.putAttribute(KNNVectorFieldMapper.KNN_FIELD, "true"); + nativeVectorField.putAttribute(KNNConstants.KNN_METHOD, KNNConstants.METHOD_HNSW); + nativeVectorField.putAttribute(KNNConstants.KNN_ENGINE, KNNEngine.NMSLIB.getName()); + nativeVectorField.putAttribute(KNNConstants.SPACE_TYPE, SpaceType.L2.getValue()); + nativeVectorField.putAttribute(KNNConstants.HNSW_ALGO_M, "32"); + nativeVectorField.putAttribute(KNNConstants.HNSW_ALGO_EF_CONSTRUCTION, "512"); + nativeVectorField.setVectorAttributes( + dimension, + vectorEncoding, + SpaceType.L2.getKnnVectorSimilarityFunction().getVectorSimilarityFunction() + ); + nativeVectorField.freeze(); + return nativeVectorField; + } +} From 29dcadd90c9549095fa25def0ca35c568521310a Mon Sep 17 00:00:00 2001 From: John Mazanec Date: Mon, 29 Jul 2024 18:54:59 -0400 Subject: [PATCH 307/416] Refactor parsing of KNNQueryBuilder (#1895) Refactors parsing of KNNQueryBuilder. First, it moves parsing logic to a separate class. Next, it uses ObjectParser instead of parsing manually by hand. Lastly, it also moves out the streaming. To test, it contains a simple jmh benchmark for testing as well as new unit tests. Signed-off-by: John Mazanec (cherry picked from commit 52636c47c03ca16da3b2c46860d782b46f192118) --- .idea/copyright/SPDX_ALv2.xml | 2 +- CHANGELOG.md | 3 +- .../knn/QueryParsingBenchmarks.java | 109 ++++ .../knn/index/query/KNNQueryBuilder.java | 212 +------- .../query/parser/KNNQueryBuilderParser.java | 266 ++++++++++ .../org/opensearch/knn/plugin/KNNPlugin.java | 3 +- .../knn/index/VectorDataTypeIT.java | 4 +- .../knn/index/query/KNNQueryBuilderTests.java | 335 ------------ .../query/parser/KNNQueryParserTests.java | 486 ++++++++++++++++++ 9 files changed, 882 insertions(+), 538 deletions(-) create mode 100644 micro-benchmarks/src/main/java/org/opensearch/knn/QueryParsingBenchmarks.java create mode 100644 src/main/java/org/opensearch/knn/index/query/parser/KNNQueryBuilderParser.java create mode 100644 src/test/java/org/opensearch/knn/index/query/parser/KNNQueryParserTests.java diff --git a/.idea/copyright/SPDX_ALv2.xml b/.idea/copyright/SPDX_ALv2.xml index a2485beef..3475d1512 100644 --- a/.idea/copyright/SPDX_ALv2.xml +++ b/.idea/copyright/SPDX_ALv2.xml @@ -1,6 +1,6 @@ - \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index 09b4c47f9..c90ea1621 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,4 +20,5 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Infrastructure ### Documentation ### Maintenance -### Refactoring \ No newline at end of file +### Refactoring +* Clean up parsing for query [#1824](https://github.com/opensearch-project/k-NN/pull/1824) diff --git a/micro-benchmarks/src/main/java/org/opensearch/knn/QueryParsingBenchmarks.java b/micro-benchmarks/src/main/java/org/opensearch/knn/QueryParsingBenchmarks.java new file mode 100644 index 000000000..1c5a3b875 --- /dev/null +++ b/micro-benchmarks/src/main/java/org/opensearch/knn/QueryParsingBenchmarks.java @@ -0,0 +1,109 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn; + +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.infra.Blackhole; +import org.opensearch.cluster.ClusterModule; +import org.opensearch.common.xcontent.LoggingDeprecationHandler; +import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.common.xcontent.json.JsonXContent; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.index.query.QueryBuilder; +import org.opensearch.index.query.QueryBuilders; +import org.opensearch.index.query.TermQueryBuilder; +import org.opensearch.knn.index.query.KNNQueryBuilder; +import org.opensearch.knn.index.query.parser.KNNQueryBuilderParser; +import org.opensearch.plugins.SearchPlugin; + +import java.io.IOException; +import java.util.Arrays; +import java.util.List; + +/** + * Benchmarks for impact of changes around query parsing + */ +@Warmup(iterations = 5, time = 10) +@Measurement(iterations = 3, time = 10) +@Fork(3) +@State(Scope.Benchmark) +public class QueryParsingBenchmarks { + private static final TermQueryBuilder TERM_QUERY = QueryBuilders.termQuery("field", "value"); + private static final NamedXContentRegistry NAMED_X_CONTENT_REGISTRY = xContentRegistry(); + + @Param({ "128", "1024" }) + private int dimension; + @Param({ "basic", "filter" }) + private String type; + + private BytesReference bytesReference; + + @Setup + public void setup() throws IOException { + XContentBuilder builder = XContentFactory.jsonBuilder(); + builder.startObject(); + builder.startObject("test"); + builder.field(KNNQueryBuilder.VECTOR_FIELD.getPreferredName(), generateVectorWithOnes(dimension)); + builder.field(KNNQueryBuilder.K_FIELD.getPreferredName(), 1); + if (type.equals("filter")) { + builder.field(KNNQueryBuilder.FILTER_FIELD.getPreferredName(), TERM_QUERY); + } + builder.endObject(); + builder.endObject(); + bytesReference = BytesReference.bytes(builder); + } + + @Benchmark + public void fromXContent(final Blackhole bh) throws IOException { + XContentParser xContentParser = createParser(); + bh.consume(KNNQueryBuilderParser.fromXContent(xContentParser)); + } + + private XContentParser createParser() throws IOException { + XContentParser contentParser = createParser(bytesReference); + contentParser.nextToken(); + return contentParser; + } + + private float[] generateVectorWithOnes(final int dimensions) { + float[] vector = new float[dimensions]; + Arrays.fill(vector, (float) 1); + return vector; + } + + private XContentParser createParser(final BytesReference data) throws IOException { + BytesArray array = (BytesArray) data; + return JsonXContent.jsonXContent.createParser( + NAMED_X_CONTENT_REGISTRY, + LoggingDeprecationHandler.INSTANCE, + array.array(), + array.offset(), + array.length() + ); + } + + private static NamedXContentRegistry xContentRegistry() { + List list = ClusterModule.getNamedXWriteables(); + SearchPlugin.QuerySpec spec = new SearchPlugin.QuerySpec<>( + TermQueryBuilder.NAME, + TermQueryBuilder::new, + TermQueryBuilder::fromXContent + ); + list.add(new NamedXContentRegistry.Entry(QueryBuilder.class, spec.getName(), (p, c) -> spec.getParser().fromXContent(p))); + return new NamedXContentRegistry(list); + } +} diff --git a/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java b/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java index 63d05540e..69039e7c3 100644 --- a/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java +++ b/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java @@ -14,19 +14,15 @@ import org.apache.lucene.search.Query; import org.opensearch.common.ValidationException; import org.opensearch.core.ParseField; -import org.opensearch.core.common.ParsingException; import org.opensearch.core.common.Strings; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.core.xcontent.XContentParser; import org.opensearch.index.mapper.MappedFieldType; -import org.opensearch.index.mapper.NumberFieldMapper; import org.opensearch.index.query.AbstractQueryBuilder; import org.opensearch.index.query.QueryBuilder; import org.opensearch.index.query.QueryRewriteContext; import org.opensearch.index.query.QueryShardContext; -import org.opensearch.knn.common.KNNConstants; import org.opensearch.knn.index.IndexUtil; import org.opensearch.knn.index.KNNMethodContext; import org.opensearch.knn.index.MethodComponentContext; @@ -34,7 +30,7 @@ import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.VectorQueryType; import org.opensearch.knn.index.mapper.KNNVectorFieldMapper; -import org.opensearch.knn.index.query.parser.MethodParametersParser; +import org.opensearch.knn.index.query.parser.KNNQueryBuilderParser; import org.opensearch.knn.index.util.EngineSpecificMethodContext; import org.opensearch.knn.index.util.KNNEngine; import org.opensearch.knn.index.util.QueryContext; @@ -44,7 +40,6 @@ import java.io.IOException; import java.util.Arrays; -import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Objects; @@ -55,8 +50,6 @@ import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_NPROBES; import static org.opensearch.knn.common.KNNConstants.MIN_SCORE; import static org.opensearch.knn.common.KNNValidationUtil.validateByteVectorValue; -import static org.opensearch.knn.index.IndexUtil.isClusterOnOrAfterMinRequiredVersion; -import static org.opensearch.knn.index.IndexUtil.minimalRequiredVersionMap; import static org.opensearch.knn.index.query.parser.MethodParametersParser.validateMethodParameters; import static org.opensearch.knn.index.util.KNNEngine.ENGINES_SUPPORTING_RADIAL_SEARCH; import static org.opensearch.knn.validation.ParameterValidator.validateParameters; @@ -79,6 +72,7 @@ public class KNNQueryBuilder extends AbstractQueryBuilder { public static final ParseField EF_SEARCH_FIELD = new ParseField(METHOD_PARAMETER_EF_SEARCH); public static final ParseField NPROBE_FIELD = new ParseField(METHOD_PARAMETER_NPROBES); public static final ParseField METHOD_PARAMS_FIELD = new ParseField(METHOD_PARAMETER); + public static final int K_MAX = 10000; /** * The name for the knn query @@ -142,7 +136,7 @@ public static class Builder { private String queryName; private float boost = DEFAULT_BOOST; - private Builder() {} + public Builder() {} public Builder fieldName(String fieldName) { this.fieldName = fieldName; @@ -295,182 +289,26 @@ public static void initialize(ModelDao modelDao) { KNNQueryBuilder.modelDao = modelDao; } - private static float[] ObjectsToFloats(List objs) { - if (Objects.isNull(objs) || objs.isEmpty()) { - throw new IllegalArgumentException( - String.format(Locale.ROOT, "[%s] field 'vector' requires to be non-null and non-empty", NAME) - ); - } - float[] vec = new float[objs.size()]; - for (int i = 0; i < objs.size(); i++) { - if ((objs.get(i) instanceof Number) == false) { - throw new IllegalArgumentException( - String.format(Locale.ROOT, "[%s] field 'vector' requires to be an array of numbers", NAME) - ); - } - vec[i] = ((Number) objs.get(i)).floatValue(); - } - return vec; - } - /** * @param in Reads from stream * @throws IOException Throws IO Exception */ public KNNQueryBuilder(StreamInput in) throws IOException { super(in); - try { - fieldName = in.readString(); - vector = in.readFloatArray(); - k = in.readInt(); - // We're checking if all cluster nodes has at least that version or higher. This check is required - // to avoid issues with cluster upgrade - if (isClusterOnOrAfterMinRequiredVersion("filter")) { - filter = in.readOptionalNamedWriteable(QueryBuilder.class); - } - if (isClusterOnOrAfterMinRequiredVersion("ignore_unmapped")) { - ignoreUnmapped = in.readOptionalBoolean(); - } - if (isClusterOnOrAfterMinRequiredVersion(KNNConstants.RADIAL_SEARCH_KEY)) { - maxDistance = in.readOptionalFloat(); - } - if (isClusterOnOrAfterMinRequiredVersion(KNNConstants.RADIAL_SEARCH_KEY)) { - minScore = in.readOptionalFloat(); - } - if (isClusterOnOrAfterMinRequiredVersion(METHOD_PARAMETER)) { - methodParameters = MethodParametersParser.streamInput(in, IndexUtil::isClusterOnOrAfterMinRequiredVersion); - } - - } catch (IOException ex) { - throw new RuntimeException("[KNN] Unable to create KNNQueryBuilder", ex); - } - } - - public static KNNQueryBuilder fromXContent(XContentParser parser) throws IOException { - String fieldName = null; - List vector = null; - float boost = AbstractQueryBuilder.DEFAULT_BOOST; - Integer k = null; - Float maxDistance = null; - Float minScore = null; - QueryBuilder filter = null; - String queryName = null; - String currentFieldName = null; - boolean ignoreUnmapped = false; - Map methodParameters = null; - XContentParser.Token token; - while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { - if (token == XContentParser.Token.FIELD_NAME) { - currentFieldName = parser.currentName(); - } else if (token == XContentParser.Token.START_OBJECT) { - throwParsingExceptionOnMultipleFields(NAME, parser.getTokenLocation(), fieldName, currentFieldName); - fieldName = currentFieldName; - while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { - if (token == XContentParser.Token.FIELD_NAME) { - currentFieldName = parser.currentName(); - } else if (token.isValue() || token == XContentParser.Token.START_ARRAY) { - if (VECTOR_FIELD.match(currentFieldName, parser.getDeprecationHandler())) { - vector = parser.list(); - } else if (AbstractQueryBuilder.BOOST_FIELD.match(currentFieldName, parser.getDeprecationHandler())) { - boost = parser.floatValue(); - } else if (K_FIELD.match(currentFieldName, parser.getDeprecationHandler())) { - k = (Integer) NumberFieldMapper.NumberType.INTEGER.parse(parser.objectBytes(), false); - } else if (IGNORE_UNMAPPED_FIELD.getPreferredName().equals(currentFieldName)) { - if (isClusterOnOrAfterMinRequiredVersion("ignore_unmapped")) { - ignoreUnmapped = parser.booleanValue(); - } - } else if (AbstractQueryBuilder.NAME_FIELD.match(currentFieldName, parser.getDeprecationHandler())) { - queryName = parser.text(); - } else if (MAX_DISTANCE_FIELD.match(currentFieldName, parser.getDeprecationHandler())) { - maxDistance = (Float) NumberFieldMapper.NumberType.FLOAT.parse(parser.objectBytes(), false); - } else if (MIN_SCORE_FIELD.match(currentFieldName, parser.getDeprecationHandler())) { - minScore = (Float) NumberFieldMapper.NumberType.FLOAT.parse(parser.objectBytes(), false); - } else { - throw new ParsingException( - parser.getTokenLocation(), - "[" + NAME + "] query does not support [" + currentFieldName + "]" - ); - } - } else if (token == XContentParser.Token.START_OBJECT) { - String tokenName = parser.currentName(); - if (FILTER_FIELD.getPreferredName().equals(tokenName)) { - log.debug(String.format("Start parsing filter for field [%s]", fieldName)); - // Query filters are supported starting from a certain k-NN version only, exact version is defined by - // MINIMAL_SUPPORTED_VERSION_FOR_LUCENE_HNSW_FILTER variable. - // Here we're checking if all cluster nodes has at least that version or higher. This check is required - // to avoid issues with rolling cluster upgrade - if (isClusterOnOrAfterMinRequiredVersion("filter")) { - filter = parseInnerQueryBuilder(parser); - } else { - log.debug( - String.format( - "This version of k-NN doesn't support [filter] field, minimal required version is [%s]", - minimalRequiredVersionMap.get("filter") - ) - ); - throw new IllegalArgumentException( - String.format( - "%s field is supported from version %s", - FILTER_FIELD.getPreferredName(), - minimalRequiredVersionMap.get("filter") - ) - ); - } - } else if (METHOD_PARAMS_FIELD.match(currentFieldName, parser.getDeprecationHandler())) { - methodParameters = MethodParametersParser.fromXContent(parser); - } else { - throw new ParsingException(parser.getTokenLocation(), "[" + NAME + "] unknown token [" + token + "]"); - } - } else { - throw new ParsingException( - parser.getTokenLocation(), - "[" + NAME + "] unknown token [" + token + "] after [" + currentFieldName + "]" - ); - } - } - } else { - throwParsingExceptionOnMultipleFields(NAME, parser.getTokenLocation(), fieldName, parser.currentName()); - fieldName = parser.currentName(); - vector = parser.list(); - } - } - - return KNNQueryBuilder.builder() - .queryName(queryName) - .boost(boost) - .fieldName(fieldName) - .vector(ObjectsToFloats(vector)) - .k(k) - .maxDistance(maxDistance) - .minScore(minScore) - .methodParameters(methodParameters) - .ignoreUnmapped(ignoreUnmapped) - .filter(filter) - .build(); + KNNQueryBuilder.Builder builder = KNNQueryBuilderParser.streamInput(in, IndexUtil::isClusterOnOrAfterMinRequiredVersion); + fieldName = builder.fieldName; + vector = builder.vector; + k = builder.k; + filter = builder.filter; + ignoreUnmapped = builder.ignoreUnmapped; + maxDistance = builder.maxDistance; + minScore = builder.minScore; + methodParameters = builder.methodParameters; } @Override protected void doWriteTo(StreamOutput out) throws IOException { - out.writeString(fieldName); - out.writeFloatArray(vector); - out.writeInt(k); - // We're checking if all cluster nodes has at least that version or higher. This check is required - // to avoid issues with cluster upgrade - if (isClusterOnOrAfterMinRequiredVersion("filter")) { - out.writeOptionalNamedWriteable(filter); - } - if (isClusterOnOrAfterMinRequiredVersion("ignore_unmapped")) { - out.writeOptionalBoolean(ignoreUnmapped); - } - if (isClusterOnOrAfterMinRequiredVersion(KNNConstants.RADIAL_SEARCH_KEY)) { - out.writeOptionalFloat(maxDistance); - } - if (isClusterOnOrAfterMinRequiredVersion(KNNConstants.RADIAL_SEARCH_KEY)) { - out.writeOptionalFloat(minScore); - } - if (isClusterOnOrAfterMinRequiredVersion(METHOD_PARAMETER)) { - MethodParametersParser.streamOutput(out, methodParameters, IndexUtil::isClusterOnOrAfterMinRequiredVersion); - } + KNNQueryBuilderParser.streamOutput(out, this, IndexUtil::isClusterOnOrAfterMinRequiredVersion); } /** @@ -489,29 +327,7 @@ public Object vector() { @Override public void doXContent(XContentBuilder builder, Params params) throws IOException { - builder.startObject(NAME); - builder.startObject(fieldName); - - builder.field(VECTOR_FIELD.getPreferredName(), vector); - builder.field(K_FIELD.getPreferredName(), k); - if (filter != null) { - builder.field(FILTER_FIELD.getPreferredName(), filter); - } - if (maxDistance != null) { - builder.field(MAX_DISTANCE_FIELD.getPreferredName(), maxDistance); - } - if (ignoreUnmapped) { - builder.field(IGNORE_UNMAPPED_FIELD.getPreferredName(), ignoreUnmapped); - } - if (minScore != null) { - builder.field(MIN_SCORE_FIELD.getPreferredName(), minScore); - } - if (methodParameters != null) { - MethodParametersParser.doXContent(builder, methodParameters); - } - printBoostAndQueryName(builder); - builder.endObject(); - builder.endObject(); + KNNQueryBuilderParser.toXContent(builder, params, this); } @Override diff --git a/src/main/java/org/opensearch/knn/index/query/parser/KNNQueryBuilderParser.java b/src/main/java/org/opensearch/knn/index/query/parser/KNNQueryBuilderParser.java new file mode 100644 index 000000000..3cb536372 --- /dev/null +++ b/src/main/java/org/opensearch/knn/index/query/parser/KNNQueryBuilderParser.java @@ -0,0 +1,266 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.opensearch.knn.index.query.parser; + +import lombok.extern.log4j.Log4j2; +import org.opensearch.core.common.ParsingException; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.xcontent.ObjectParser; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentLocation; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.index.query.QueryBuilder; +import org.opensearch.knn.common.KNNConstants; +import org.opensearch.knn.index.IndexUtil; +import org.opensearch.knn.index.query.KNNQueryBuilder; + +import java.io.IOException; +import java.util.List; +import java.util.Locale; +import java.util.Objects; +import java.util.function.Function; + +import static org.opensearch.index.query.AbstractQueryBuilder.BOOST_FIELD; +import static org.opensearch.index.query.AbstractQueryBuilder.NAME_FIELD; +import static org.opensearch.index.query.AbstractQueryBuilder.parseInnerQueryBuilder; +import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER; +import static org.opensearch.knn.index.IndexUtil.isClusterOnOrAfterMinRequiredVersion; +import static org.opensearch.knn.index.query.KNNQueryBuilder.FILTER_FIELD; +import static org.opensearch.knn.index.query.KNNQueryBuilder.IGNORE_UNMAPPED_FIELD; +import static org.opensearch.knn.index.query.KNNQueryBuilder.K_FIELD; +import static org.opensearch.knn.index.query.KNNQueryBuilder.MAX_DISTANCE_FIELD; +import static org.opensearch.knn.index.query.KNNQueryBuilder.METHOD_PARAMS_FIELD; +import static org.opensearch.knn.index.query.KNNQueryBuilder.MIN_SCORE_FIELD; +import static org.opensearch.knn.index.query.KNNQueryBuilder.NAME; +import static org.opensearch.knn.index.query.KNNQueryBuilder.VECTOR_FIELD; + +/** + * Helper class responsible for parsing and reverse parsing KNNQueryBuilder's + */ +@Log4j2 +public final class KNNQueryBuilderParser { + + private static final ObjectParser INTERNAL_PARSER = createInternalObjectParser(); + + /** + * For a k-NN query, we need to parse roughly the following structure into a KNNQueryBuilder: + * "my_vector2": { + * "vector": [2, 3, 5, 6], + * "k": 2, + * ... + * } + * to simplify the parsing process, we can define an object parser that will the internal structure after the + * field name. We cannot unfortunately also parse the field name because it ends up in the same structure + * as the nested portion. So we need to do that separately. + */ + private static ObjectParser createInternalObjectParser() { + ObjectParser internalParser = new ObjectParser<>(NAME, KNNQueryBuilder.Builder::new); + internalParser.declareFloat(KNNQueryBuilder.Builder::boost, BOOST_FIELD); + internalParser.declareString(KNNQueryBuilder.Builder::queryName, NAME_FIELD); + internalParser.declareFloatArray((b, v) -> b.vector(floatListToFloatArray(v)), VECTOR_FIELD); + internalParser.declareInt(KNNQueryBuilder.Builder::k, K_FIELD); + internalParser.declareBoolean((b, v) -> { + if (isClusterOnOrAfterMinRequiredVersion("ignore_unmapped")) { + b.ignoreUnmapped(v); + } + }, IGNORE_UNMAPPED_FIELD); + internalParser.declareFloat(KNNQueryBuilder.Builder::maxDistance, MAX_DISTANCE_FIELD); + internalParser.declareFloat(KNNQueryBuilder.Builder::minScore, MIN_SCORE_FIELD); + + internalParser.declareObject( + KNNQueryBuilder.Builder::methodParameters, + (p, v) -> MethodParametersParser.fromXContent(p), + METHOD_PARAMS_FIELD + ); + internalParser.declareObject(KNNQueryBuilder.Builder::filter, (p, v) -> parseInnerQueryBuilder(p), FILTER_FIELD); + + return internalParser; + } + + /** + * Stream input for KNNQueryBuilder + * + * @param in stream out + * @param minClusterVersionCheck function to check min version + * @return KNNQueryBuilder.Builder class + * @throws IOException on stream failure + */ + public static KNNQueryBuilder.Builder streamInput(StreamInput in, Function minClusterVersionCheck) throws IOException { + KNNQueryBuilder.Builder builder = new KNNQueryBuilder.Builder(); + builder.fieldName(in.readString()); + builder.vector(in.readFloatArray()); + builder.k(in.readInt()); + // We're checking if all cluster nodes has at least that version or higher. This check is required + // to avoid issues with cluster upgrade + if (isClusterOnOrAfterMinRequiredVersion("filter")) { + builder.filter(in.readOptionalNamedWriteable(QueryBuilder.class)); + } + if (minClusterVersionCheck.apply("ignore_unmapped")) { + builder.ignoreUnmapped(in.readOptionalBoolean()); + } + if (minClusterVersionCheck.apply(KNNConstants.RADIAL_SEARCH_KEY)) { + builder.maxDistance(in.readOptionalFloat()); + } + if (minClusterVersionCheck.apply(KNNConstants.RADIAL_SEARCH_KEY)) { + builder.minScore(in.readOptionalFloat()); + } + if (minClusterVersionCheck.apply(METHOD_PARAMETER)) { + builder.methodParameters(MethodParametersParser.streamInput(in, IndexUtil::isClusterOnOrAfterMinRequiredVersion)); + } + + return builder; + } + + /** + * Stream output for KNNQueryBuilder + * + * @param out stream out + * @param builder KNNQueryBuilder to stream + * @param minClusterVersionCheck function to check min version + * @throws IOException on stream failure + */ + public static void streamOutput(StreamOutput out, KNNQueryBuilder builder, Function minClusterVersionCheck) + throws IOException { + out.writeString(builder.fieldName()); + out.writeFloatArray((float[]) builder.vector()); + out.writeInt(builder.getK()); + // We're checking if all cluster nodes has at least that version or higher. This check is required + // to avoid issues with cluster upgrade + if (isClusterOnOrAfterMinRequiredVersion("filter")) { + out.writeOptionalNamedWriteable(builder.getFilter()); + } + if (minClusterVersionCheck.apply("ignore_unmapped")) { + out.writeOptionalBoolean(builder.isIgnoreUnmapped()); + } + if (minClusterVersionCheck.apply(KNNConstants.RADIAL_SEARCH_KEY)) { + out.writeOptionalFloat(builder.getMaxDistance()); + } + if (minClusterVersionCheck.apply(KNNConstants.RADIAL_SEARCH_KEY)) { + out.writeOptionalFloat(builder.getMinScore()); + } + if (minClusterVersionCheck.apply(METHOD_PARAMETER)) { + MethodParametersParser.streamOutput(out, builder.getMethodParameters(), IndexUtil::isClusterOnOrAfterMinRequiredVersion); + } + } + + /** + * Convert XContent to KNNQueryBuilder + * + * @param parser input parser + * @return KNNQueryBuilder + * @throws IOException on parsing failure + */ + public static KNNQueryBuilder fromXContent(XContentParser parser) throws IOException { + String fieldName = null; + String currentFieldName = null; + XContentParser.Token token; + KNNQueryBuilder.Builder builder = null; + List vector = null; + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + if (token == XContentParser.Token.FIELD_NAME) { + currentFieldName = parser.currentName(); + } else if (token == XContentParser.Token.START_OBJECT) { + throwParsingExceptionOnMultipleFields(parser.getTokenLocation(), fieldName, currentFieldName); + fieldName = currentFieldName; + builder = INTERNAL_PARSER.apply(parser, null); + } else { + throwParsingExceptionOnMultipleFields(parser.getTokenLocation(), fieldName, parser.currentName()); + fieldName = parser.currentName(); + vector = parser.list(); + } + } + + if (builder == null) { + builder = KNNQueryBuilder.builder().vector(objectsToFloats(vector)); + } + builder.fieldName(fieldName); + return builder.build(); + } + + /** + * Convert KNNQueryBuilder to XContent + * + * @param builder xcontent builder to add KNNQueryBuilder + * @param params ToXContent params + * @param knnQueryBuilder KNNQueryBuilder to convert + * @throws IOException on conversion failure + */ + public static void toXContent(XContentBuilder builder, ToXContent.Params params, KNNQueryBuilder knnQueryBuilder) throws IOException { + builder.startObject(NAME); + builder.startObject(knnQueryBuilder.fieldName()); + + builder.field(VECTOR_FIELD.getPreferredName(), knnQueryBuilder.vector()); + builder.field(K_FIELD.getPreferredName(), knnQueryBuilder.getK()); + if (knnQueryBuilder.getFilter() != null) { + builder.field(FILTER_FIELD.getPreferredName(), knnQueryBuilder.getFilter()); + } + if (knnQueryBuilder.getMaxDistance() != null) { + builder.field(MAX_DISTANCE_FIELD.getPreferredName(), knnQueryBuilder.getMaxDistance()); + } + if (knnQueryBuilder.isIgnoreUnmapped()) { + builder.field(IGNORE_UNMAPPED_FIELD.getPreferredName(), knnQueryBuilder.isIgnoreUnmapped()); + } + if (knnQueryBuilder.getMinScore() != null) { + builder.field(MIN_SCORE_FIELD.getPreferredName(), knnQueryBuilder.getMinScore()); + } + if (knnQueryBuilder.getMethodParameters() != null) { + MethodParametersParser.doXContent(builder, knnQueryBuilder.getMethodParameters()); + } + + builder.field(BOOST_FIELD.getPreferredName(), knnQueryBuilder.boost()); + if (knnQueryBuilder.queryName() != null) { + builder.field(NAME_FIELD.getPreferredName(), knnQueryBuilder.queryName()); + } + + builder.endObject(); + builder.endObject(); + } + + private static float[] floatListToFloatArray(List floats) { + if (Objects.isNull(floats) || floats.isEmpty()) { + throw new IllegalArgumentException(String.format("[%s] field 'vector' requires to be non-null and non-empty", NAME)); + } + float[] vec = new float[floats.size()]; + for (int i = 0; i < floats.size(); i++) { + vec[i] = floats.get(i); + } + return vec; + } + + private static float[] objectsToFloats(List objs) { + if (Objects.isNull(objs) || objs.isEmpty()) { + throw new IllegalArgumentException( + String.format(Locale.ROOT, "[%s] field 'vector' requires to be non-null and non-empty", NAME) + ); + } + float[] vec = new float[objs.size()]; + for (int i = 0; i < objs.size(); i++) { + if ((objs.get(i) instanceof Number) == false) { + throw new IllegalArgumentException( + String.format(Locale.ROOT, "[%s] field 'vector' requires to be an array of numbers", NAME) + ); + } + vec[i] = ((Number) objs.get(i)).floatValue(); + } + return vec; + } + + private static void throwParsingExceptionOnMultipleFields( + XContentLocation contentLocation, + String processedFieldName, + String currentFieldName + ) { + if (processedFieldName != null) { + throw new ParsingException( + contentLocation, + "[" + NAME + "] query doesn't support multiple fields, found [" + processedFieldName + "] and [" + currentFieldName + "]" + ); + } + } +} diff --git a/src/main/java/org/opensearch/knn/plugin/KNNPlugin.java b/src/main/java/org/opensearch/knn/plugin/KNNPlugin.java index f898b622e..5301b6e4e 100644 --- a/src/main/java/org/opensearch/knn/plugin/KNNPlugin.java +++ b/src/main/java/org/opensearch/knn/plugin/KNNPlugin.java @@ -19,6 +19,7 @@ import org.opensearch.knn.index.KNNSettings; import org.opensearch.knn.index.mapper.KNNVectorFieldMapper; +import org.opensearch.knn.index.query.parser.KNNQueryBuilderParser; import org.opensearch.knn.index.query.KNNWeight; import org.opensearch.knn.index.codec.KNNCodecService; import org.opensearch.knn.index.memory.NativeMemoryLoadStrategy; @@ -173,7 +174,7 @@ public Map getMappers() { @Override public List> getQueries() { - return singletonList(new QuerySpec<>(KNNQueryBuilder.NAME, KNNQueryBuilder::new, KNNQueryBuilder::fromXContent)); + return singletonList(new QuerySpec<>(KNNQueryBuilder.NAME, KNNQueryBuilder::new, KNNQueryBuilderParser::fromXContent)); } @Override diff --git a/src/test/java/org/opensearch/knn/index/VectorDataTypeIT.java b/src/test/java/org/opensearch/knn/index/VectorDataTypeIT.java index 2d6ffb1e4..a0f78f327 100644 --- a/src/test/java/org/opensearch/knn/index/VectorDataTypeIT.java +++ b/src/test/java/org/opensearch/knn/index/VectorDataTypeIT.java @@ -452,7 +452,7 @@ public void testSearchWithInvalidSearchVectorType() { ResponseException ex = expectThrows(ResponseException.class, () -> client().performRequest(request)); assertEquals(400, ex.getResponse().getStatusLine().getStatusCode()); - assertTrue(ex.getMessage().contains("[knn] field 'vector' requires to be an array of numbers")); + assertTrue(ex.getMessage(), ex.getMessage().contains("[knn] failed to parse field [vector]")); } @SneakyThrows @@ -474,7 +474,7 @@ public void testSearchWithMissingQueryVector() { ResponseException ex = expectThrows(ResponseException.class, () -> client().performRequest(request)); assertEquals(400, ex.getResponse().getStatusLine().getStatusCode()); - assertTrue(ex.getMessage().contains("[knn] field 'vector' requires to be non-null and non-empty")); + assertTrue(ex.getMessage().contains("[knn] requires query vector")); } @SneakyThrows diff --git a/src/test/java/org/opensearch/knn/index/query/KNNQueryBuilderTests.java b/src/test/java/org/opensearch/knn/index/query/KNNQueryBuilderTests.java index 5a6caf64d..d3c17d383 100644 --- a/src/test/java/org/opensearch/knn/index/query/KNNQueryBuilderTests.java +++ b/src/test/java/org/opensearch/knn/index/query/KNNQueryBuilderTests.java @@ -15,14 +15,10 @@ import org.opensearch.cluster.ClusterModule; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.io.stream.BytesStreamOutput; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.common.io.stream.NamedWriteableAwareStreamInput; import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.index.Index; -import org.opensearch.core.xcontent.NamedXContentRegistry; -import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.core.xcontent.XContentParser; import org.opensearch.index.IndexSettings; import org.opensearch.index.mapper.NumberFieldMapper; import org.opensearch.index.query.QueryBuilder; @@ -41,10 +37,8 @@ import org.opensearch.knn.indices.ModelDao; import org.opensearch.knn.indices.ModelMetadata; import org.opensearch.knn.indices.ModelState; -import org.opensearch.plugins.SearchPlugin; import java.io.IOException; -import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.Locale; @@ -58,7 +52,6 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; import static org.opensearch.knn.index.KNNClusterTestUtils.mockClusterService; -import static org.opensearch.knn.index.query.KNNQueryBuilder.EF_SEARCH_FIELD; import static org.opensearch.knn.index.util.KNNEngine.ENGINES_SUPPORTING_RADIAL_SEARCH; public class KNNQueryBuilderTests extends KNNTestCase { @@ -163,334 +156,6 @@ public void testEmptyVector() { ); } - public void testFromXContent() throws Exception { - float[] queryVector = { 1.0f, 2.0f, 3.0f, 4.0f }; - KNNQueryBuilder knnQueryBuilder = KNNQueryBuilder.builder().fieldName(FIELD_NAME).vector(queryVector).k(K).build(); - XContentBuilder builder = XContentFactory.jsonBuilder(); - builder.startObject(); - builder.startObject(knnQueryBuilder.fieldName()); - builder.field(KNNQueryBuilder.VECTOR_FIELD.getPreferredName(), knnQueryBuilder.vector()); - builder.field(KNNQueryBuilder.K_FIELD.getPreferredName(), knnQueryBuilder.getK()); - builder.endObject(); - builder.endObject(); - XContentParser contentParser = createParser(builder); - contentParser.nextToken(); - KNNQueryBuilder actualBuilder = KNNQueryBuilder.fromXContent(contentParser); - assertEquals(knnQueryBuilder, actualBuilder); - } - - public void testFromXContent_KnnWithMethodParameters() throws Exception { - float[] queryVector = { 1.0f, 2.0f, 3.0f, 4.0f }; - KNNQueryBuilder knnQueryBuilder = KNNQueryBuilder.builder() - .fieldName(FIELD_NAME) - .vector(queryVector) - .k(K) - .methodParameters(HNSW_METHOD_PARAMS) - .build(); - XContentBuilder builder = XContentFactory.jsonBuilder(); - builder.startObject(); - builder.startObject(knnQueryBuilder.fieldName()); - builder.field(KNNQueryBuilder.VECTOR_FIELD.getPreferredName(), knnQueryBuilder.vector()); - builder.field(KNNQueryBuilder.K_FIELD.getPreferredName(), knnQueryBuilder.getK()); - builder.startObject(org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER); - builder.field(EF_SEARCH_FIELD.getPreferredName(), EF_SEARCH); - builder.endObject(); - builder.endObject(); - builder.endObject(); - XContentParser contentParser = createParser(builder); - contentParser.nextToken(); - KNNQueryBuilder actualBuilder = KNNQueryBuilder.fromXContent(contentParser); - assertEquals(knnQueryBuilder, actualBuilder); - } - - public void testFromXContent_whenDoRadiusSearch_whenDistanceThreshold_whenMethodParameter_thenSucceed() throws Exception { - float[] queryVector = { 1.0f, 2.0f, 3.0f, 4.0f }; - KNNQueryBuilder knnQueryBuilder = KNNQueryBuilder.builder() - .fieldName(FIELD_NAME) - .vector(queryVector) - .maxDistance(MAX_DISTANCE) - .methodParameters(HNSW_METHOD_PARAMS) - .build(); - XContentBuilder builder = XContentFactory.jsonBuilder(); - builder.startObject(); - builder.startObject(knnQueryBuilder.fieldName()); - builder.field(KNNQueryBuilder.VECTOR_FIELD.getPreferredName(), knnQueryBuilder.vector()); - builder.field(KNNQueryBuilder.MAX_DISTANCE_FIELD.getPreferredName(), knnQueryBuilder.getMaxDistance()); - builder.startObject(org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER); - builder.field(EF_SEARCH_FIELD.getPreferredName(), EF_SEARCH); - builder.endObject(); - builder.endObject(); - builder.endObject(); - XContentParser contentParser = createParser(builder); - contentParser.nextToken(); - KNNQueryBuilder actualBuilder = KNNQueryBuilder.fromXContent(contentParser); - assertEquals(knnQueryBuilder, actualBuilder); - } - - public void testFromXContent_whenDoRadiusSearch_whenScoreThreshold_whenMethodParameter_thenSucceed() throws Exception { - float[] queryVector = { 1.0f, 2.0f, 3.0f, 4.0f }; - KNNQueryBuilder knnQueryBuilder = KNNQueryBuilder.builder() - .fieldName(FIELD_NAME) - .vector(queryVector) - .minScore(MAX_DISTANCE) - .methodParameters(HNSW_METHOD_PARAMS) - .build(); - XContentBuilder builder = XContentFactory.jsonBuilder(); - builder.startObject(); - builder.startObject(knnQueryBuilder.fieldName()); - builder.field(KNNQueryBuilder.VECTOR_FIELD.getPreferredName(), knnQueryBuilder.vector()); - builder.field(KNNQueryBuilder.MIN_SCORE_FIELD.getPreferredName(), knnQueryBuilder.getMinScore()); - builder.startObject(org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER); - builder.field(EF_SEARCH_FIELD.getPreferredName(), EF_SEARCH); - builder.endObject(); - builder.endObject(); - builder.endObject(); - XContentParser contentParser = createParser(builder); - contentParser.nextToken(); - KNNQueryBuilder actualBuilder = KNNQueryBuilder.fromXContent(contentParser); - assertEquals(knnQueryBuilder, actualBuilder); - } - - public void testFromXContent_withFilter() throws Exception { - final ClusterService clusterService = mockClusterService(Version.CURRENT); - - final KNNClusterUtil knnClusterUtil = KNNClusterUtil.instance(); - knnClusterUtil.initialize(clusterService); - - float[] queryVector = { 1.0f, 2.0f, 3.0f, 4.0f }; - KNNQueryBuilder knnQueryBuilder = KNNQueryBuilder.builder() - .fieldName(FIELD_NAME) - .vector(queryVector) - .k(K) - .filter(TERM_QUERY) - .build(); - XContentBuilder builder = XContentFactory.jsonBuilder(); - builder.startObject(); - builder.startObject(knnQueryBuilder.fieldName()); - builder.field(KNNQueryBuilder.VECTOR_FIELD.getPreferredName(), knnQueryBuilder.vector()); - builder.field(KNNQueryBuilder.K_FIELD.getPreferredName(), knnQueryBuilder.getK()); - builder.field(KNNQueryBuilder.FILTER_FIELD.getPreferredName(), knnQueryBuilder.getFilter()); - builder.endObject(); - builder.endObject(); - XContentParser contentParser = createParser(builder); - contentParser.nextToken(); - KNNQueryBuilder actualBuilder = KNNQueryBuilder.fromXContent(contentParser); - assertEquals(knnQueryBuilder, actualBuilder); - } - - public void testFromXContent_KnnWithEfSearch_withFilter() throws Exception { - final ClusterService clusterService = mockClusterService(Version.CURRENT); - - final KNNClusterUtil knnClusterUtil = KNNClusterUtil.instance(); - knnClusterUtil.initialize(clusterService); - - float[] queryVector = { 1.0f, 2.0f, 3.0f, 4.0f }; - KNNQueryBuilder knnQueryBuilder = KNNQueryBuilder.builder() - .fieldName(FIELD_NAME) - .vector(queryVector) - .k(K) - .filter(TERM_QUERY) - .methodParameters(HNSW_METHOD_PARAMS) - .build(); - XContentBuilder builder = XContentFactory.jsonBuilder(); - builder.startObject(); - builder.startObject(knnQueryBuilder.fieldName()); - builder.field(KNNQueryBuilder.VECTOR_FIELD.getPreferredName(), knnQueryBuilder.vector()); - builder.field(KNNQueryBuilder.K_FIELD.getPreferredName(), knnQueryBuilder.getK()); - builder.startObject(org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER); - builder.field(EF_SEARCH_FIELD.getPreferredName(), EF_SEARCH); - builder.endObject(); - builder.field(KNNQueryBuilder.FILTER_FIELD.getPreferredName(), knnQueryBuilder.getFilter()); - builder.endObject(); - builder.endObject(); - XContentParser contentParser = createParser(builder); - contentParser.nextToken(); - KNNQueryBuilder actualBuilder = KNNQueryBuilder.fromXContent(contentParser); - assertEquals(knnQueryBuilder, actualBuilder); - } - - public void testFromXcontent_WithFilter_UnsupportedClusterVersion() throws Exception { - final ClusterService clusterService = mockClusterService(Version.V_2_3_0); - - final KNNClusterUtil knnClusterUtil = KNNClusterUtil.instance(); - knnClusterUtil.initialize(clusterService); - - float[] queryVector = { 1.0f, 2.0f, 3.0f, 4.0f }; - final KNNQueryBuilder knnQueryBuilder = new KNNQueryBuilder(FIELD_NAME, queryVector, K, TERM_QUERY); - final XContentBuilder builder = XContentFactory.jsonBuilder(); - builder.startObject(); - builder.startObject(knnQueryBuilder.fieldName()); - builder.field(KNNQueryBuilder.VECTOR_FIELD.getPreferredName(), knnQueryBuilder.vector()); - builder.field(KNNQueryBuilder.K_FIELD.getPreferredName(), knnQueryBuilder.getK()); - builder.field(KNNQueryBuilder.FILTER_FIELD.getPreferredName(), knnQueryBuilder.getFilter()); - builder.endObject(); - builder.endObject(); - final XContentParser contentParser = createParser(builder); - contentParser.nextToken(); - - expectThrows(IllegalArgumentException.class, () -> KNNQueryBuilder.fromXContent(contentParser)); - } - - public void testFromXContent_wenDoRadiusSearch_whenDistanceThreshold_whenFilter_thenSucceed() throws Exception { - final ClusterService clusterService = mockClusterService(Version.CURRENT); - - final KNNClusterUtil knnClusterUtil = KNNClusterUtil.instance(); - knnClusterUtil.initialize(clusterService); - - float[] queryVector = { 1.0f, 2.0f, 3.0f, 4.0f }; - KNNQueryBuilder knnQueryBuilder = KNNQueryBuilder.builder() - .fieldName(FIELD_NAME) - .vector(queryVector) - .maxDistance(MAX_DISTANCE) - .filter(TERM_QUERY) - .build(); - - XContentBuilder builder = XContentFactory.jsonBuilder(); - builder.startObject(); - builder.startObject(knnQueryBuilder.fieldName()); - builder.field(KNNQueryBuilder.VECTOR_FIELD.getPreferredName(), knnQueryBuilder.vector()); - builder.field(KNNQueryBuilder.MAX_DISTANCE_FIELD.getPreferredName(), knnQueryBuilder.getMaxDistance()); - builder.field(KNNQueryBuilder.FILTER_FIELD.getPreferredName(), knnQueryBuilder.getFilter()); - builder.endObject(); - builder.endObject(); - XContentParser contentParser = createParser(builder); - contentParser.nextToken(); - KNNQueryBuilder actualBuilder = KNNQueryBuilder.fromXContent(contentParser); - assertEquals(knnQueryBuilder, actualBuilder); - } - - public void testFromXContent_wenDoRadiusSearch_whenScoreThreshold_whenFilter_thenSucceed() throws Exception { - final ClusterService clusterService = mockClusterService(Version.CURRENT); - - final KNNClusterUtil knnClusterUtil = KNNClusterUtil.instance(); - knnClusterUtil.initialize(clusterService); - - float[] queryVector = { 1.0f, 2.0f, 3.0f, 4.0f }; - KNNQueryBuilder knnQueryBuilder = KNNQueryBuilder.builder() - .fieldName(FIELD_NAME) - .vector(queryVector) - .minScore(MIN_SCORE) - .filter(TERM_QUERY) - .build(); - XContentBuilder builder = XContentFactory.jsonBuilder(); - builder.startObject(); - builder.startObject(knnQueryBuilder.fieldName()); - builder.field(KNNQueryBuilder.VECTOR_FIELD.getPreferredName(), knnQueryBuilder.vector()); - builder.field(KNNQueryBuilder.MIN_SCORE_FIELD.getPreferredName(), knnQueryBuilder.getMinScore()); - builder.field(KNNQueryBuilder.FILTER_FIELD.getPreferredName(), knnQueryBuilder.getFilter()); - builder.endObject(); - builder.endObject(); - XContentParser contentParser = createParser(builder); - contentParser.nextToken(); - KNNQueryBuilder actualBuilder = KNNQueryBuilder.fromXContent(contentParser); - assertEquals(knnQueryBuilder, actualBuilder); - } - - public void testFromXContent_InvalidQueryVectorType() throws Exception { - final ClusterService clusterService = mockClusterService(Version.CURRENT); - - final KNNClusterUtil knnClusterUtil = KNNClusterUtil.instance(); - knnClusterUtil.initialize(clusterService); - - List invalidTypeQueryVector = new ArrayList<>(); - invalidTypeQueryVector.add(1.5); - invalidTypeQueryVector.add(2.5); - invalidTypeQueryVector.add("a"); - invalidTypeQueryVector.add(null); - - XContentBuilder builder = XContentFactory.jsonBuilder(); - builder.startObject(); - builder.startObject(FIELD_NAME); - builder.field(KNNQueryBuilder.VECTOR_FIELD.getPreferredName(), invalidTypeQueryVector); - builder.field(KNNQueryBuilder.K_FIELD.getPreferredName(), K); - builder.endObject(); - builder.endObject(); - XContentParser contentParser = createParser(builder); - contentParser.nextToken(); - IllegalArgumentException exception = expectThrows( - IllegalArgumentException.class, - () -> KNNQueryBuilder.fromXContent(contentParser) - ); - assertTrue(exception.getMessage().contains("[knn] field 'vector' requires to be an array of numbers")); - } - - public void testFromXContent_whenDoRadiusSearch_whenInputInvalidQueryVectorType_thenException() throws Exception { - final ClusterService clusterService = mockClusterService(Version.CURRENT); - - final KNNClusterUtil knnClusterUtil = KNNClusterUtil.instance(); - knnClusterUtil.initialize(clusterService); - - List invalidTypeQueryVector = new ArrayList<>(); - invalidTypeQueryVector.add(1.5); - invalidTypeQueryVector.add(2.5); - invalidTypeQueryVector.add("a"); - invalidTypeQueryVector.add(null); - - XContentBuilder builder = XContentFactory.jsonBuilder(); - builder.startObject(); - builder.startObject(FIELD_NAME); - builder.field(KNNQueryBuilder.VECTOR_FIELD.getPreferredName(), invalidTypeQueryVector); - builder.field(KNNQueryBuilder.MAX_DISTANCE_FIELD.getPreferredName(), MAX_DISTANCE); - builder.endObject(); - builder.endObject(); - XContentParser contentParser = createParser(builder); - contentParser.nextToken(); - IllegalArgumentException exception = expectThrows( - IllegalArgumentException.class, - () -> KNNQueryBuilder.fromXContent(contentParser) - ); - assertTrue(exception.getMessage().contains("[knn] field 'vector' requires to be an array of numbers")); - } - - public void testFromXContent_missingQueryVector() throws Exception { - final ClusterService clusterService = mockClusterService(Version.CURRENT); - - final KNNClusterUtil knnClusterUtil = KNNClusterUtil.instance(); - knnClusterUtil.initialize(clusterService); - - // Test without vector field - XContentBuilder builderWithoutVectorField = XContentFactory.jsonBuilder(); - builderWithoutVectorField.startObject(); - builderWithoutVectorField.startObject(FIELD_NAME); - builderWithoutVectorField.field(KNNQueryBuilder.K_FIELD.getPreferredName(), K); - builderWithoutVectorField.endObject(); - builderWithoutVectorField.endObject(); - XContentParser contentParserWithoutVectorField = createParser(builderWithoutVectorField); - contentParserWithoutVectorField.nextToken(); - IllegalArgumentException exception = expectThrows( - IllegalArgumentException.class, - () -> KNNQueryBuilder.fromXContent(contentParserWithoutVectorField) - ); - assertTrue(exception.getMessage().contains("[knn] field 'vector' requires to be non-null and non-empty")); - - // Test empty vector field - List emptyQueryVector = new ArrayList<>(); - XContentBuilder builderWithEmptyVector = XContentFactory.jsonBuilder(); - builderWithEmptyVector.startObject(); - builderWithEmptyVector.startObject(FIELD_NAME); - builderWithEmptyVector.field(KNNQueryBuilder.VECTOR_FIELD.getPreferredName(), emptyQueryVector); - builderWithEmptyVector.field(KNNQueryBuilder.K_FIELD.getPreferredName(), K); - builderWithEmptyVector.endObject(); - builderWithEmptyVector.endObject(); - XContentParser contentParserWithEmptyVector = createParser(builderWithEmptyVector); - contentParserWithEmptyVector.nextToken(); - exception = expectThrows(IllegalArgumentException.class, () -> KNNQueryBuilder.fromXContent(contentParserWithEmptyVector)); - assertTrue(exception.getMessage().contains("[knn] field 'vector' requires to be non-null and non-empty")); - } - - @Override - protected NamedXContentRegistry xContentRegistry() { - List list = ClusterModule.getNamedXWriteables(); - SearchPlugin.QuerySpec spec = new SearchPlugin.QuerySpec<>( - TermQueryBuilder.NAME, - TermQueryBuilder::new, - TermQueryBuilder::fromXContent - ); - list.add(new NamedXContentRegistry.Entry(QueryBuilder.class, spec.getName(), (p, c) -> spec.getParser().fromXContent(p))); - NamedXContentRegistry registry = new NamedXContentRegistry(list); - return registry; - } - @Override protected NamedWriteableRegistry writableRegistry() { final List entries = ClusterModule.getNamedWriteables(); diff --git a/src/test/java/org/opensearch/knn/index/query/parser/KNNQueryParserTests.java b/src/test/java/org/opensearch/knn/index/query/parser/KNNQueryParserTests.java new file mode 100644 index 000000000..8a1fadf89 --- /dev/null +++ b/src/test/java/org/opensearch/knn/index/query/parser/KNNQueryParserTests.java @@ -0,0 +1,486 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.opensearch.knn.index.query.parser; + +import org.opensearch.Version; +import org.opensearch.cluster.ClusterModule; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.core.common.ParsingException; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.index.query.QueryBuilder; +import org.opensearch.index.query.QueryBuilders; +import org.opensearch.index.query.TermQueryBuilder; +import org.opensearch.knn.KNNTestCase; +import org.opensearch.knn.index.KNNClusterUtil; +import org.opensearch.knn.index.query.KNNQueryBuilder; +import org.opensearch.plugins.SearchPlugin; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import static org.opensearch.core.xcontent.ToXContent.EMPTY_PARAMS; +import static org.opensearch.index.query.AbstractQueryBuilder.BOOST_FIELD; +import static org.opensearch.knn.index.KNNClusterTestUtils.mockClusterService; +import static org.opensearch.knn.index.query.KNNQueryBuilder.NAME; +import static org.opensearch.knn.index.query.KNNQueryBuilder.EF_SEARCH_FIELD; + +public class KNNQueryParserTests extends KNNTestCase { + + private static final String FIELD_NAME = "myvector"; + private static final int K = 1; + private static final int EF_SEARCH = 10; + private static final Map HNSW_METHOD_PARAMS = Map.of("ef_search", EF_SEARCH); + private static final Float MAX_DISTANCE = 1.0f; + private static final Float MIN_SCORE = 0.5f; + private static final Float BOOST = 10.5f; + private static final TermQueryBuilder TERM_QUERY = QueryBuilders.termQuery("field", "value"); + + public void testFromXContent() throws Exception { + float[] queryVector = { 1.0f, 2.0f, 3.0f, 4.0f }; + KNNQueryBuilder knnQueryBuilder = KNNQueryBuilder.builder().fieldName(FIELD_NAME).vector(queryVector).k(K).build(); + XContentBuilder builder = XContentFactory.jsonBuilder(); + builder.startObject(); + builder.startObject(knnQueryBuilder.fieldName()); + builder.field(KNNQueryBuilder.VECTOR_FIELD.getPreferredName(), knnQueryBuilder.vector()); + builder.field(KNNQueryBuilder.K_FIELD.getPreferredName(), knnQueryBuilder.getK()); + builder.endObject(); + builder.endObject(); + XContentParser contentParser = createParser(builder); + contentParser.nextToken(); + KNNQueryBuilder actualBuilder = KNNQueryBuilderParser.fromXContent(contentParser); + assertEquals(knnQueryBuilder, actualBuilder); + } + + public void testFromXContent_KnnWithMethodParameters() throws Exception { + float[] queryVector = { 1.0f, 2.0f, 3.0f, 4.0f }; + KNNQueryBuilder knnQueryBuilder = KNNQueryBuilder.builder() + .fieldName(FIELD_NAME) + .vector(queryVector) + .k(K) + .methodParameters(HNSW_METHOD_PARAMS) + .build(); + XContentBuilder builder = XContentFactory.jsonBuilder(); + builder.startObject(); + builder.startObject(knnQueryBuilder.fieldName()); + builder.field(KNNQueryBuilder.VECTOR_FIELD.getPreferredName(), knnQueryBuilder.vector()); + builder.field(KNNQueryBuilder.K_FIELD.getPreferredName(), knnQueryBuilder.getK()); + builder.startObject(org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER); + builder.field(EF_SEARCH_FIELD.getPreferredName(), EF_SEARCH); + builder.endObject(); + builder.endObject(); + builder.endObject(); + XContentParser contentParser = createParser(builder); + contentParser.nextToken(); + KNNQueryBuilder actualBuilder = KNNQueryBuilderParser.fromXContent(contentParser); + assertEquals(knnQueryBuilder, actualBuilder); + } + + public void testFromXContent_whenDoRadiusSearch_whenDistanceThreshold_whenMethodParameter_thenSucceed() throws Exception { + float[] queryVector = { 1.0f, 2.0f, 3.0f, 4.0f }; + KNNQueryBuilder knnQueryBuilder = KNNQueryBuilder.builder() + .fieldName(FIELD_NAME) + .vector(queryVector) + .maxDistance(MAX_DISTANCE) + .methodParameters(HNSW_METHOD_PARAMS) + .build(); + XContentBuilder builder = XContentFactory.jsonBuilder(); + builder.startObject(); + builder.startObject(knnQueryBuilder.fieldName()); + builder.field(KNNQueryBuilder.VECTOR_FIELD.getPreferredName(), knnQueryBuilder.vector()); + builder.field(KNNQueryBuilder.MAX_DISTANCE_FIELD.getPreferredName(), knnQueryBuilder.getMaxDistance()); + builder.startObject(org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER); + builder.field(EF_SEARCH_FIELD.getPreferredName(), EF_SEARCH); + builder.endObject(); + builder.endObject(); + builder.endObject(); + XContentParser contentParser = createParser(builder); + contentParser.nextToken(); + KNNQueryBuilder actualBuilder = KNNQueryBuilderParser.fromXContent(contentParser); + assertEquals(knnQueryBuilder, actualBuilder); + } + + public void testFromXContent_whenDoRadiusSearch_whenScoreThreshold_whenMethodParameter_thenSucceed() throws Exception { + float[] queryVector = { 1.0f, 2.0f, 3.0f, 4.0f }; + KNNQueryBuilder knnQueryBuilder = KNNQueryBuilder.builder() + .fieldName(FIELD_NAME) + .vector(queryVector) + .minScore(MAX_DISTANCE) + .methodParameters(HNSW_METHOD_PARAMS) + .build(); + XContentBuilder builder = XContentFactory.jsonBuilder(); + builder.startObject(); + builder.startObject(knnQueryBuilder.fieldName()); + builder.field(KNNQueryBuilder.VECTOR_FIELD.getPreferredName(), knnQueryBuilder.vector()); + builder.field(KNNQueryBuilder.MIN_SCORE_FIELD.getPreferredName(), knnQueryBuilder.getMinScore()); + builder.startObject(org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER); + builder.field(EF_SEARCH_FIELD.getPreferredName(), EF_SEARCH); + builder.endObject(); + builder.endObject(); + builder.endObject(); + XContentParser contentParser = createParser(builder); + contentParser.nextToken(); + KNNQueryBuilder actualBuilder = KNNQueryBuilderParser.fromXContent(contentParser); + assertEquals(knnQueryBuilder, actualBuilder); + } + + public void testFromXContent_withFilter() throws Exception { + final ClusterService clusterService = mockClusterService(Version.CURRENT); + + final KNNClusterUtil knnClusterUtil = KNNClusterUtil.instance(); + knnClusterUtil.initialize(clusterService); + + float[] queryVector = { 1.0f, 2.0f, 3.0f, 4.0f }; + KNNQueryBuilder knnQueryBuilder = KNNQueryBuilder.builder() + .fieldName(FIELD_NAME) + .vector(queryVector) + .k(K) + .filter(TERM_QUERY) + .build(); + XContentBuilder builder = XContentFactory.jsonBuilder(); + builder.startObject(); + builder.startObject(knnQueryBuilder.fieldName()); + builder.field(KNNQueryBuilder.VECTOR_FIELD.getPreferredName(), knnQueryBuilder.vector()); + builder.field(KNNQueryBuilder.K_FIELD.getPreferredName(), knnQueryBuilder.getK()); + builder.field(KNNQueryBuilder.FILTER_FIELD.getPreferredName(), knnQueryBuilder.getFilter()); + builder.endObject(); + builder.endObject(); + XContentParser contentParser = createParser(builder); + contentParser.nextToken(); + KNNQueryBuilder actualBuilder = KNNQueryBuilderParser.fromXContent(contentParser); + assertEquals(knnQueryBuilder, actualBuilder); + } + + public void testFromXContent_KnnWithEfSearch_withFilter() throws Exception { + float[] queryVector = { 1.0f, 2.0f, 3.0f, 4.0f }; + KNNQueryBuilder knnQueryBuilder = KNNQueryBuilder.builder() + .fieldName(FIELD_NAME) + .vector(queryVector) + .k(K) + .filter(TERM_QUERY) + .methodParameters(HNSW_METHOD_PARAMS) + .build(); + XContentBuilder builder = XContentFactory.jsonBuilder(); + builder.startObject(); + builder.startObject(knnQueryBuilder.fieldName()); + builder.field(KNNQueryBuilder.VECTOR_FIELD.getPreferredName(), knnQueryBuilder.vector()); + builder.field(KNNQueryBuilder.K_FIELD.getPreferredName(), knnQueryBuilder.getK()); + builder.startObject(org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER); + builder.field(EF_SEARCH_FIELD.getPreferredName(), EF_SEARCH); + builder.endObject(); + builder.field(KNNQueryBuilder.FILTER_FIELD.getPreferredName(), knnQueryBuilder.getFilter()); + builder.endObject(); + builder.endObject(); + XContentParser contentParser = createParser(builder); + contentParser.nextToken(); + KNNQueryBuilder actualBuilder = KNNQueryBuilderParser.fromXContent(contentParser); + assertEquals(knnQueryBuilder, actualBuilder); + } + + public void testFromXContent_whenDoRadiusSearch_whenDistanceThreshold_whenFilter_thenSucceed() throws Exception { + final ClusterService clusterService = mockClusterService(Version.CURRENT); + + final KNNClusterUtil knnClusterUtil = KNNClusterUtil.instance(); + knnClusterUtil.initialize(clusterService); + + float[] queryVector = { 1.0f, 2.0f, 3.0f, 4.0f }; + KNNQueryBuilder knnQueryBuilder = KNNQueryBuilder.builder() + .fieldName(FIELD_NAME) + .vector(queryVector) + .maxDistance(MAX_DISTANCE) + .filter(TERM_QUERY) + .build(); + + XContentBuilder builder = XContentFactory.jsonBuilder(); + builder.startObject(); + builder.startObject(knnQueryBuilder.fieldName()); + builder.field(KNNQueryBuilder.VECTOR_FIELD.getPreferredName(), knnQueryBuilder.vector()); + builder.field(KNNQueryBuilder.MAX_DISTANCE_FIELD.getPreferredName(), knnQueryBuilder.getMaxDistance()); + builder.field(KNNQueryBuilder.FILTER_FIELD.getPreferredName(), knnQueryBuilder.getFilter()); + builder.endObject(); + builder.endObject(); + XContentParser contentParser = createParser(builder); + contentParser.nextToken(); + KNNQueryBuilder actualBuilder = KNNQueryBuilderParser.fromXContent(contentParser); + assertEquals(knnQueryBuilder, actualBuilder); + } + + public void testFromXContent_whenDoRadiusSearch_whenScoreThreshold_whenFilter_thenSucceed() throws Exception { + final ClusterService clusterService = mockClusterService(Version.CURRENT); + + final KNNClusterUtil knnClusterUtil = KNNClusterUtil.instance(); + knnClusterUtil.initialize(clusterService); + + float[] queryVector = { 1.0f, 2.0f, 3.0f, 4.0f }; + KNNQueryBuilder knnQueryBuilder = KNNQueryBuilder.builder() + .fieldName(FIELD_NAME) + .vector(queryVector) + .minScore(MIN_SCORE) + .filter(TERM_QUERY) + .build(); + XContentBuilder builder = XContentFactory.jsonBuilder(); + builder.startObject(); + builder.startObject(knnQueryBuilder.fieldName()); + builder.field(KNNQueryBuilder.VECTOR_FIELD.getPreferredName(), knnQueryBuilder.vector()); + builder.field(KNNQueryBuilder.MIN_SCORE_FIELD.getPreferredName(), knnQueryBuilder.getMinScore()); + builder.field(KNNQueryBuilder.FILTER_FIELD.getPreferredName(), knnQueryBuilder.getFilter()); + builder.endObject(); + builder.endObject(); + XContentParser contentParser = createParser(builder); + contentParser.nextToken(); + KNNQueryBuilder actualBuilder = KNNQueryBuilderParser.fromXContent(contentParser); + assertEquals(knnQueryBuilder, actualBuilder); + } + + public void testFromXContent_InvalidQueryVectorType() throws Exception { + final ClusterService clusterService = mockClusterService(Version.CURRENT); + + final KNNClusterUtil knnClusterUtil = KNNClusterUtil.instance(); + knnClusterUtil.initialize(clusterService); + + List invalidTypeQueryVector = new ArrayList<>(); + invalidTypeQueryVector.add(1.5); + invalidTypeQueryVector.add(2.5); + invalidTypeQueryVector.add("a"); + invalidTypeQueryVector.add(null); + + XContentBuilder builder = XContentFactory.jsonBuilder(); + builder.startObject(); + builder.startObject(FIELD_NAME); + builder.field(KNNQueryBuilder.VECTOR_FIELD.getPreferredName(), invalidTypeQueryVector); + builder.field(KNNQueryBuilder.K_FIELD.getPreferredName(), K); + builder.endObject(); + builder.endObject(); + XContentParser contentParser = createParser(builder); + contentParser.nextToken(); + IllegalArgumentException exception = expectThrows( + IllegalArgumentException.class, + () -> KNNQueryBuilderParser.fromXContent(contentParser) + ); + assertTrue(exception.getMessage(), exception.getMessage().contains("[knn] failed to parse field [vector]")); + } + + public void testFromXContent_whenDoRadiusSearch_whenInputInvalidQueryVectorType_thenException() throws Exception { + final ClusterService clusterService = mockClusterService(Version.CURRENT); + + final KNNClusterUtil knnClusterUtil = KNNClusterUtil.instance(); + knnClusterUtil.initialize(clusterService); + + List invalidTypeQueryVector = new ArrayList<>(); + invalidTypeQueryVector.add(1.5); + invalidTypeQueryVector.add(2.5); + invalidTypeQueryVector.add("a"); + invalidTypeQueryVector.add(null); + + XContentBuilder builder = XContentFactory.jsonBuilder(); + builder.startObject(); + builder.startObject(FIELD_NAME); + builder.field(KNNQueryBuilder.VECTOR_FIELD.getPreferredName(), invalidTypeQueryVector); + builder.field(KNNQueryBuilder.MAX_DISTANCE_FIELD.getPreferredName(), MAX_DISTANCE); + builder.endObject(); + builder.endObject(); + XContentParser contentParser = createParser(builder); + contentParser.nextToken(); + IllegalArgumentException exception = expectThrows( + IllegalArgumentException.class, + () -> KNNQueryBuilderParser.fromXContent(contentParser) + ); + assertTrue(exception.getMessage(), exception.getMessage().contains("[knn] failed to parse field [vector]")); + } + + public void testFromXContent_missingQueryVector() throws Exception { + final ClusterService clusterService = mockClusterService(Version.CURRENT); + + final KNNClusterUtil knnClusterUtil = KNNClusterUtil.instance(); + knnClusterUtil.initialize(clusterService); + + // Test without vector field + XContentBuilder builderWithoutVectorField = XContentFactory.jsonBuilder(); + builderWithoutVectorField.startObject(); + builderWithoutVectorField.startObject(FIELD_NAME); + builderWithoutVectorField.field(KNNQueryBuilder.K_FIELD.getPreferredName(), K); + builderWithoutVectorField.endObject(); + builderWithoutVectorField.endObject(); + XContentParser contentParserWithoutVectorField = createParser(builderWithoutVectorField); + contentParserWithoutVectorField.nextToken(); + IllegalArgumentException exception = expectThrows( + IllegalArgumentException.class, + () -> KNNQueryBuilderParser.fromXContent(contentParserWithoutVectorField) + ); + assertTrue(exception.getMessage(), exception.getMessage().contains("[knn] requires query vector")); + + // Test empty vector field + List emptyQueryVector = new ArrayList<>(); + XContentBuilder builderWithEmptyVector = XContentFactory.jsonBuilder(); + builderWithEmptyVector.startObject(); + builderWithEmptyVector.startObject(FIELD_NAME); + builderWithEmptyVector.field(KNNQueryBuilder.VECTOR_FIELD.getPreferredName(), emptyQueryVector); + builderWithEmptyVector.field(KNNQueryBuilder.K_FIELD.getPreferredName(), K); + builderWithEmptyVector.endObject(); + builderWithEmptyVector.endObject(); + XContentParser contentParserWithEmptyVector = createParser(builderWithEmptyVector); + contentParserWithEmptyVector.nextToken(); + exception = expectThrows(IllegalArgumentException.class, () -> KNNQueryBuilderParser.fromXContent(contentParserWithEmptyVector)); + assertTrue(exception.getMessage(), exception.getMessage().contains("[knn] failed to parse field [vector]")); + } + + public void testFromXContent_whenFlat_thenException() throws Exception { + float[] queryVector = { 1.0f, 2.0f, 3.0f, 4.0f }; + XContentBuilder builder = XContentFactory.jsonBuilder(); + builder.startObject(); + builder.field(FIELD_NAME, queryVector); + builder.endObject(); + XContentParser contentParser = createParser(builder); + contentParser.nextToken(); + Exception exception = expectThrows(IllegalArgumentException.class, () -> KNNQueryBuilderParser.fromXContent(contentParser)); + assertTrue(exception.getMessage(), exception.getMessage().contains("[knn] requires exactly one of k, distance or score to be set")); + } + + public void testFromXContent_whenMultiFields_thenException() throws Exception { + float[] queryVector = { 1.0f, 2.0f, 3.0f, 4.0f }; + XContentBuilder builder = XContentFactory.jsonBuilder(); + builder.startObject(); + builder.startObject(FIELD_NAME + "1"); + builder.field(KNNQueryBuilder.VECTOR_FIELD.getPreferredName(), queryVector); + builder.field(KNNQueryBuilder.K_FIELD.getPreferredName(), K); + builder.endObject(); + builder.startObject(FIELD_NAME + "2"); + builder.field(KNNQueryBuilder.VECTOR_FIELD.getPreferredName(), queryVector); + builder.field(KNNQueryBuilder.K_FIELD.getPreferredName(), K); + builder.endObject(); + builder.endObject(); + XContentParser contentParser = createParser(builder); + contentParser.nextToken(); + Exception exception = expectThrows(ParsingException.class, () -> KNNQueryBuilderParser.fromXContent(contentParser)); + assertTrue(exception.getMessage(), exception.getMessage().contains("[knn] query doesn't support multiple fields")); + } + + public void testToXContent_whenParamsVectorBoostK_thenSucceed() throws IOException { + float[] queryVector = { 1.0f, 2.0f, 3.0f, 4.0f }; + XContentBuilder builder = XContentFactory.jsonBuilder(); + builder.startObject(); + builder.startObject(NAME); + builder.startObject(FIELD_NAME); + builder.field(KNNQueryBuilder.VECTOR_FIELD.getPreferredName(), queryVector); + builder.field(KNNQueryBuilder.K_FIELD.getPreferredName(), K); + builder.field(BOOST_FIELD.getPreferredName(), BOOST); + builder.endObject(); + builder.endObject(); + builder.endObject(); + + KNNQueryBuilder knnQueryBuilder = KNNQueryBuilder.builder().fieldName(FIELD_NAME).vector(queryVector).k(K).boost(BOOST).build(); + XContentBuilder testBuilder = XContentFactory.jsonBuilder(); + testBuilder.startObject(); + KNNQueryBuilderParser.toXContent(testBuilder, EMPTY_PARAMS, knnQueryBuilder); + testBuilder.endObject(); + assertEquals(builder.toString(), testBuilder.toString()); + } + + public void testToXContent_whenFilter_thenSucceed() throws IOException { + float[] queryVector = { 1.0f, 2.0f, 3.0f, 4.0f }; + XContentBuilder builder = XContentFactory.jsonBuilder(); + builder.startObject(); + builder.startObject(NAME); + builder.startObject(FIELD_NAME); + builder.field(KNNQueryBuilder.VECTOR_FIELD.getPreferredName(), queryVector); + builder.field(KNNQueryBuilder.K_FIELD.getPreferredName(), K); + builder.field(KNNQueryBuilder.FILTER_FIELD.getPreferredName(), TERM_QUERY); + builder.field(BOOST_FIELD.getPreferredName(), BOOST); + builder.endObject(); + builder.endObject(); + builder.endObject(); + + KNNQueryBuilder knnQueryBuilder = KNNQueryBuilder.builder() + .fieldName(FIELD_NAME) + .vector(queryVector) + .k(K) + .boost(BOOST) + .filter(TERM_QUERY) + .build(); + XContentBuilder testBuilder = XContentFactory.jsonBuilder(); + testBuilder.startObject(); + KNNQueryBuilderParser.toXContent(testBuilder, EMPTY_PARAMS, knnQueryBuilder); + testBuilder.endObject(); + assertEquals(builder.toString(), testBuilder.toString()); + } + + public void testToXContent_whenMaxDistance_thenSucceed() throws IOException { + float[] queryVector = { 1.0f, 2.0f, 3.0f, 4.0f }; + XContentBuilder builder = XContentFactory.jsonBuilder(); + builder.startObject(); + builder.startObject(NAME); + builder.startObject(FIELD_NAME); + builder.field(KNNQueryBuilder.VECTOR_FIELD.getPreferredName(), queryVector); + builder.field(KNNQueryBuilder.K_FIELD.getPreferredName(), 0); + builder.field(KNNQueryBuilder.MAX_DISTANCE_FIELD.getPreferredName(), MAX_DISTANCE); + builder.field(BOOST_FIELD.getPreferredName(), BOOST); + builder.endObject(); + builder.endObject(); + builder.endObject(); + + KNNQueryBuilder knnQueryBuilder = KNNQueryBuilder.builder() + .fieldName(FIELD_NAME) + .vector(queryVector) + .boost(BOOST) + .maxDistance(MAX_DISTANCE) + .build(); + XContentBuilder testBuilder = XContentFactory.jsonBuilder(); + testBuilder.startObject(); + KNNQueryBuilderParser.toXContent(testBuilder, EMPTY_PARAMS, knnQueryBuilder); + testBuilder.endObject(); + assertEquals(builder.toString(), testBuilder.toString()); + } + + public void testToXContent_whenMethodParams_thenSucceed() throws IOException { + float[] queryVector = { 1.0f, 2.0f, 3.0f, 4.0f }; + XContentBuilder builder = XContentFactory.jsonBuilder(); + builder.startObject(); + builder.startObject(NAME); + builder.startObject(FIELD_NAME); + builder.field(KNNQueryBuilder.VECTOR_FIELD.getPreferredName(), queryVector); + builder.field(KNNQueryBuilder.K_FIELD.getPreferredName(), K); + builder.startObject(org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER); + builder.field(EF_SEARCH_FIELD.getPreferredName(), EF_SEARCH); + builder.endObject(); + builder.field(BOOST_FIELD.getPreferredName(), BOOST); + builder.endObject(); + builder.endObject(); + builder.endObject(); + + KNNQueryBuilder knnQueryBuilder = KNNQueryBuilder.builder() + .fieldName(FIELD_NAME) + .vector(queryVector) + .boost(BOOST) + .k(K) + .methodParameters(HNSW_METHOD_PARAMS) + .build(); + XContentBuilder testBuilder = XContentFactory.jsonBuilder(); + testBuilder.startObject(); + KNNQueryBuilderParser.toXContent(testBuilder, EMPTY_PARAMS, knnQueryBuilder); + testBuilder.endObject(); + logger.info(builder.toString()); + logger.info(testBuilder.toString()); + assertEquals(builder.toString(), testBuilder.toString()); + } + + @Override + protected NamedXContentRegistry xContentRegistry() { + List list = ClusterModule.getNamedXWriteables(); + SearchPlugin.QuerySpec spec = new SearchPlugin.QuerySpec<>( + TermQueryBuilder.NAME, + TermQueryBuilder::new, + TermQueryBuilder::fromXContent + ); + list.add(new NamedXContentRegistry.Entry(QueryBuilder.class, spec.getName(), (p, c) -> spec.getParser().fromXContent(p))); + NamedXContentRegistry registry = new NamedXContentRegistry(list); + return registry; + } +} From 812255d3fe58b6ebc908a50d552b958558390500 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Tue, 30 Jul 2024 07:57:08 -0700 Subject: [PATCH 308/416] Fix idea xml file (#1902) Signed-off-by: John Mazanec (cherry picked from commit ad166f4f471a6493ab4e1118c6b5adb27fc5069b) --- .idea/copyright/SPDX_ALv2.xml | 2 +- .../knn/index/query/parser/KNNQueryBuilderParser.java | 1 - ...NNQueryParserTests.java => KNNQueryBuilderParserTests.java} | 3 +-- 3 files changed, 2 insertions(+), 4 deletions(-) rename src/test/java/org/opensearch/knn/index/query/parser/{KNNQueryParserTests.java => KNNQueryBuilderParserTests.java} (99%) diff --git a/.idea/copyright/SPDX_ALv2.xml b/.idea/copyright/SPDX_ALv2.xml index 3475d1512..3c6f9c768 100644 --- a/.idea/copyright/SPDX_ALv2.xml +++ b/.idea/copyright/SPDX_ALv2.xml @@ -1,6 +1,6 @@ - \ No newline at end of file diff --git a/src/main/java/org/opensearch/knn/index/query/parser/KNNQueryBuilderParser.java b/src/main/java/org/opensearch/knn/index/query/parser/KNNQueryBuilderParser.java index 3cb536372..39ca915c7 100644 --- a/src/main/java/org/opensearch/knn/index/query/parser/KNNQueryBuilderParser.java +++ b/src/main/java/org/opensearch/knn/index/query/parser/KNNQueryBuilderParser.java @@ -1,7 +1,6 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * */ package org.opensearch.knn.index.query.parser; diff --git a/src/test/java/org/opensearch/knn/index/query/parser/KNNQueryParserTests.java b/src/test/java/org/opensearch/knn/index/query/parser/KNNQueryBuilderParserTests.java similarity index 99% rename from src/test/java/org/opensearch/knn/index/query/parser/KNNQueryParserTests.java rename to src/test/java/org/opensearch/knn/index/query/parser/KNNQueryBuilderParserTests.java index 8a1fadf89..746057f1a 100644 --- a/src/test/java/org/opensearch/knn/index/query/parser/KNNQueryParserTests.java +++ b/src/test/java/org/opensearch/knn/index/query/parser/KNNQueryBuilderParserTests.java @@ -1,7 +1,6 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * */ package org.opensearch.knn.index.query.parser; @@ -33,7 +32,7 @@ import static org.opensearch.knn.index.query.KNNQueryBuilder.NAME; import static org.opensearch.knn.index.query.KNNQueryBuilder.EF_SEARCH_FIELD; -public class KNNQueryParserTests extends KNNTestCase { +public class KNNQueryBuilderParserTests extends KNNTestCase { private static final String FIELD_NAME = "myvector"; private static final int K = 1; From d5b9fab58068867a08f574a79f76ad7784292881 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Tue, 30 Jul 2024 08:49:32 -0700 Subject: [PATCH 309/416] Add triage.md based off of security (#1904) Signed-off-by: John Mazanec (cherry picked from commit 7981406cae59e1a69c51f2c54e66b5a88cd6ed6b) --- TRIAGING.md | 89 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 89 insertions(+) create mode 100644 TRIAGING.md diff --git a/TRIAGING.md b/TRIAGING.md new file mode 100644 index 000000000..9949b3390 --- /dev/null +++ b/TRIAGING.md @@ -0,0 +1,89 @@ + + +The maintainers of the k-NN/neural-search Repo's seek to promote an inclusive and engaged community of contributors. In +order to facilitate this, bi-weekly triage meetings are open-to-all and attendance is encouraged for anyone who hopes to +contribute, discuss an issue, or learn more about the project. To learn more about contributing to the +k-NN/neural-search Repo visit the [Contributing](./CONTRIBUTING.md) documentation. + +### Do I need to attend for my issue to be addressed/triaged? + +Attendance is not required for your issue to be triaged or addressed. All new issues are triaged bi-weekly. + +### What happens if my issue does not get covered this time? + +Each meeting we seek to address all new issues. However, should we run out of time before your issue is discussed, you +are always welcome to attend the next meeting or to follow up on the issue post itself. + +### How do I join the Backlog & Triage meeting? + +Meetings are hosted regularly at 5 PM Pacific Time on Tuesdays bi-weekly and can be joined via the links posted on the +[OpenSearch Meetup Group](https://www.meetup.com/opensearch/events/) list of events. The event will be titled +`Development Backlog & Triage Meeting - k-NN/neural-search`. + +After joining the Chime meeting, you can enable your video / voice to join the discussion. If you do not have a webcam +or microphone available, you can still join in via the text chat. + +If you have an issue you'd like to bring forth please consider getting a link to the issue so it can be presented to +everyone in the meeting. + +### Is there an agenda for each week? + +Meetings are 60 minutes and structured as follows: + +1. Initial Gathering: As we gather, feel free to turn on video and engage in informal and open-to-all conversation. After a bit a volunteer will share their screen and proceed with the agenda. +2. Announcements: If there are any announcements to be made they will happen at the start of the meeting. +3. Review of New Issues: The meetings always start with reviewing all untriaged/recent issues for the k-NN and neural-search repositories. +4. Member Requests: Opportunity for any meeting member to ask for consideration of an issue or pull request. +5. Pull Request Discussion: Then, we review the status of outstanding pull requests from the k-NN and neural-search repositories. +6. Open Discussion: Allow for members of the meeting to surface any topics without issues filed or pull request created. + + +There is no specific ordering within each category. + +If you have an issue you would like to discuss but do not have the ability to attend the entire meeting please attend when is best for you and signal that you have an issue to discuss when you arrive. + +### Do I need to have already contributed to the project to attend a triage meeting? + +No, all are welcome and encouraged to attend. Attending the Backlog & Triage meetings is a great way for a new contributor to learn about the project as well as explore different avenues of contribution. + +### What if I have an issue that is almost a duplicate, should I open a new one to be triaged? + +You can always open an issue including one that you think may be a duplicate. However, in cases where you believe there +is an important distinction to be made between an existing issue and your newly created one, you are encouraged to +attend the triaging meeting to explain. + +### What if I have follow-up questions on an issue? + +If you have an existing issue you would like to discuss, you can always comment on the issue itself. Alternatively, you +are welcome to come to the triage meeting to discuss. + +### Is this meeting a good place to get help setting up k-NN/neural-search features on my OpenSearch instance? + +While we are always happy to help the community, the best resource for implementation questions is [the OpenSearch forum](https://forum.opensearch.org/c/plugins/k-nn/48). + +There you can find answers to many common questions as well as speak with implementation experts. + +### What are the issue labels associated with triaging? + +Yes, there are several labels that are used to identify the 'state' of issues filed in OpenSearch and the Security Plugin. + +| Label | When applied | Meaning | +| ----- | ------------ | ------- | +| Untriaged | When issues are created or re-opened. | Issues labeled as 'Untriaged' require the attention of the repository maintainers and may need to be prioritized for quicker resolution. It's crucial to keep the count of 'Untriaged' labels low to ensure all potential security issues are addressed in a timely manner. See [SECURITY.md](https://github.com/opensearch-project/security/blob/main/SECURITY.md) for more details on handling these issues. | +| Triaged | During triage meetings. | Issues labeled as 'Triaged' have been reviewed and are deemed actionable. Opening a pull request for an issue with the 'Triaged' label has a higher likelihood of approval from the project maintainers, particularly in novel areas. | +| Neither Label | During triage meetings. | This category is for issues that lack sufficient details to formulate a potential solution. Until more details are provided, it's difficult to ascertain if a proposed solution would be acceptable. When dealing with an 'Untriaged' issue that falls into this category, the triage team should provide further insights so the issue can be appropriately closed or labeled as 'Triaged'. Issues in this state are reviewed during every triage meeting. | +| Help Wanted | Anytime. | Issues marked as 'Help Wanted' signal that they are actionable and not the current focus of the project maintainers. Community contributions are especially encouraged for these issues. | +| Good First Issue | Anytime. | Issues labeled as 'Good First Issue' are small in scope and can be resolved with a single pull request. These are recommended starting points for newcomers looking to make their first contributions. | + + +### What if my issue is critical to OpenSearch operations, do I have to wait for the bi-weekly meeting for it to be addressed? + +All new issues for the [k-NN](https://github.com/opensearch-project/k-NN/issues?q=is%3Aissue+is%3Aopen+label%3Auntriaged) repo and [neural-search](https://github.com/opensearch-project/neural-search/issues?q=is%3Aissue+is%3Aopen+-label%3Atriaged) repo are reviewed daily to check for critical issues which require immediate triaging. If an issue relates to a severe concern for OpenSearch operation, it will be triaged by a maintainer mid-week. You can still come to discuss an issue at the following meeting even if it has already been triaged during the week. + +### Is this where I should bring up potential security vulnerabilities? + +Due to the sensitive nature of security vulnerabilities, please report all potential vulnerabilities directly by following the steps outlined on the [SECURITY.md](https://github.com/opensearch-project/k-NN/blob/main/SECURITY.md) document. + +### Who should I contact if I have further questions? + +You can always file an issue for any question you have about the project. From 4ff98498a0ee5ab27860b1512a2a9b978fdc8535 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Tue, 30 Jul 2024 09:43:01 -0700 Subject: [PATCH 310/416] Limit tests run based on changed path (#1905) Signed-off-by: John Mazanec (cherry picked from commit f10e3972da093f3c9d9e58bcc4f6061ae7028edd) --- .github/workflows/CI.yml | 20 +++++++++++++++++++ ...backwards_compatibility_tests_workflow.yml | 20 +++++++++++++++++++ .github/workflows/test_security.yml | 18 +++++++++++++++++ 3 files changed, 58 insertions(+) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index e6dec4daf..3e04f3724 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -6,10 +6,30 @@ on: branches: - "*" - "feature/**" + paths: + - 'build.gradle' + - 'settings.gradle' + - 'src/**' + - 'build-tools/**' + - 'buildSrc/**' + - 'gradle/**' + - 'jni/**' + - 'micro-benchmarks/**' + - '.github/workflows/CI.yml' pull_request: branches: - "*" - "feature/**" + paths: + - 'build.gradle' + - 'settings.gradle' + - 'src/**' + - 'build-tools/**' + - 'buildSrc/**' + - 'gradle/**' + - 'jni/**' + - 'micro-benchmarks/**' + - '.github/workflows/CI.yml' env: ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true diff --git a/.github/workflows/backwards_compatibility_tests_workflow.yml b/.github/workflows/backwards_compatibility_tests_workflow.yml index 8946ed2c1..6b8a71bde 100644 --- a/.github/workflows/backwards_compatibility_tests_workflow.yml +++ b/.github/workflows/backwards_compatibility_tests_workflow.yml @@ -4,10 +4,30 @@ on: branches: - "*" - "feature/**" + paths: + - 'build.gradle' + - 'settings.gradle' + - 'src/**' + - 'build-tools/**' + - 'buildSrc/**' + - 'gradle/**' + - 'jni/**' + - 'qa/**' + - '.github/workflows/backwards_compatibility_tests_workflow.yml' pull_request: branches: - "*" - "feature/**" + paths: + - 'build.gradle' + - 'settings.gradle' + - 'src/**' + - 'build-tools/**' + - 'buildSrc/**' + - 'gradle/**' + - 'jni/**' + - 'qa/**' + - '.github/workflows/backwards_compatibility_tests_workflow.yml' jobs: Restart-Upgrade-BWCTests-k-NN: diff --git a/.github/workflows/test_security.yml b/.github/workflows/test_security.yml index 77b726a69..6a0fe72d0 100644 --- a/.github/workflows/test_security.yml +++ b/.github/workflows/test_security.yml @@ -6,10 +6,28 @@ on: branches: - "*" - "feature/**" + paths: + - 'build.gradle' + - 'settings.gradle' + - 'src/**' + - 'build-tools/**' + - 'buildSrc/**' + - 'gradle/**' + - 'jni/**' + - '.github/workflows/test_security.yml' pull_request: branches: - "*" - "feature/**" + paths: + - 'build.gradle' + - 'settings.gradle' + - 'src/**' + - 'build-tools/**' + - 'buildSrc/**' + - 'gradle/**' + - 'jni/**' + - '.github/workflows/test_security.yml' env: ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true From a22184cb5b7e660120bfa668c8835c2d1dd1f4ae Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Tue, 30 Jul 2024 14:16:06 -0700 Subject: [PATCH 311/416] Adding integration tests for binary index (#1876) (#1910) Signed-off-by: Heemin Kim (cherry picked from commit 245995a6abe64b93bc44fef61e9d13df53696bcd) Co-authored-by: Heemin Kim --- .../knn/index/query/KNNQueryBuilder.java | 19 +- .../knn/plugin/script/KNNScoringSpace.java | 6 +- .../opensearch/knn/integ/BinaryIndexIT.java | 182 +++ .../integ/BinaryIndexInvalidMappingIT.java | 100 ++ .../knn/integ/FilteredSearchBinaryIT.java | 104 ++ .../script => integ}/KNNScriptScoringIT.java | 229 +++- .../knn/integ/NestedSearchBinaryIT.java | 156 +++ .../knn/{index => integ}/NestedSearchIT.java | 4 +- .../script => integ}/PainlessScriptIT.java | 90 +- .../plugin/script/KNNScoringSpaceTests.java | 12 +- src/test/resources/data/README.md | 12 + .../data/test_queries_binary_100x128.csv | 100 ++ .../data/test_vectors_binary_1000x128.json | 1000 +++++++++++++++++ .../knn/KNNJsonIndexMappingsBuilder.java | 132 +++ .../opensearch/knn/KNNJsonQueryBuilder.java | 67 ++ .../org/opensearch/knn/KNNRestTestCase.java | 61 +- .../java/org/opensearch/knn/TestUtils.java | 34 +- 17 files changed, 2241 insertions(+), 67 deletions(-) create mode 100644 src/test/java/org/opensearch/knn/integ/BinaryIndexIT.java create mode 100644 src/test/java/org/opensearch/knn/integ/BinaryIndexInvalidMappingIT.java create mode 100644 src/test/java/org/opensearch/knn/integ/FilteredSearchBinaryIT.java rename src/test/java/org/opensearch/knn/{plugin/script => integ}/KNNScriptScoringIT.java (79%) create mode 100644 src/test/java/org/opensearch/knn/integ/NestedSearchBinaryIT.java rename src/test/java/org/opensearch/knn/{index => integ}/NestedSearchIT.java (99%) rename src/test/java/org/opensearch/knn/{plugin/script => integ}/PainlessScriptIT.java (87%) create mode 100644 src/test/resources/data/test_queries_binary_100x128.csv create mode 100644 src/test/resources/data/test_vectors_binary_1000x128.json create mode 100644 src/testFixtures/java/org/opensearch/knn/KNNJsonIndexMappingsBuilder.java create mode 100644 src/testFixtures/java/org/opensearch/knn/KNNJsonQueryBuilder.java diff --git a/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java b/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java index 69039e7c3..24037a2a4 100644 --- a/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java +++ b/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java @@ -393,6 +393,17 @@ protected Query doToQuery(QueryShardContext context) { } } + if (this.maxDistance != null || this.minScore != null) { + if (!ENGINES_SUPPORTING_RADIAL_SEARCH.contains(knnEngine)) { + throw new UnsupportedOperationException( + String.format(Locale.ROOT, "Engine [%s] does not support radial search", knnEngine) + ); + } + if (vectorDataType == VectorDataType.BINARY) { + throw new UnsupportedOperationException(String.format(Locale.ROOT, "Binary data type does not support radial search")); + } + } + // Currently, k-NN supports distance and score types radial search // We need transform distance/score to right type of engine required radius. Float radius = null; @@ -464,14 +475,6 @@ protected Query doToQuery(QueryShardContext context) { return KNNQueryFactory.create(createQueryRequest); } if (radius != null) { - if (!ENGINES_SUPPORTING_RADIAL_SEARCH.contains(knnEngine)) { - throw new UnsupportedOperationException( - String.format(Locale.ROOT, "Engine [%s] does not support radial search", knnEngine) - ); - } - if (vectorDataType == VectorDataType.BINARY) { - throw new UnsupportedOperationException(String.format(Locale.ROOT, "Binary data type does not support radial search")); - } RNNQueryFactory.CreateQueryRequest createQueryRequest = RNNQueryFactory.CreateQueryRequest.builder() .knnEngine(knnEngine) .indexName(indexName) diff --git a/src/main/java/org/opensearch/knn/plugin/script/KNNScoringSpace.java b/src/main/java/org/opensearch/knn/plugin/script/KNNScoringSpace.java index 2aa203b12..850bff1ab 100644 --- a/src/main/java/org/opensearch/knn/plugin/script/KNNScoringSpace.java +++ b/src/main/java/org/opensearch/knn/plugin/script/KNNScoringSpace.java @@ -5,6 +5,7 @@ package org.opensearch.knn.plugin.script; +import lombok.Getter; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.search.IndexSearcher; import org.opensearch.index.mapper.MappedFieldType; @@ -52,8 +53,9 @@ ScoreScript getScoreScript(Map params, String field, SearchLooku abstract class KNNFieldSpace implements KNNScoringSpace { public static final Set DATA_TYPES_DEFAULT = Set.of(VectorDataType.FLOAT, VectorDataType.BYTE); - float[] processedQuery; - BiFunction scoringMethod; + private float[] processedQuery; + @Getter + private BiFunction scoringMethod; public KNNFieldSpace(final Object query, final MappedFieldType fieldType, final String spaceName) { this(query, fieldType, spaceName, DATA_TYPES_DEFAULT); diff --git a/src/test/java/org/opensearch/knn/integ/BinaryIndexIT.java b/src/test/java/org/opensearch/knn/integ/BinaryIndexIT.java new file mode 100644 index 000000000..2b6cd3e56 --- /dev/null +++ b/src/test/java/org/opensearch/knn/integ/BinaryIndexIT.java @@ -0,0 +1,182 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.integ; + +import com.google.common.primitives.Floats; +import lombok.SneakyThrows; +import lombok.extern.log4j.Log4j2; +import org.apache.commons.lang.ArrayUtils; +import org.apache.http.util.EntityUtils; +import org.junit.After; +import org.junit.BeforeClass; +import org.opensearch.client.Response; +import org.opensearch.knn.KNNJsonIndexMappingsBuilder; +import org.opensearch.knn.KNNJsonQueryBuilder; +import org.opensearch.knn.KNNRestTestCase; +import org.opensearch.knn.KNNResult; +import org.opensearch.knn.TestUtils; +import org.opensearch.knn.index.VectorDataType; +import org.opensearch.knn.index.util.KNNEngine; + +import java.io.IOException; +import java.net.URL; +import java.util.List; + +import static org.opensearch.knn.common.KNNConstants.METHOD_HNSW; + +/** + * This class contains integration tests for binary index with HNSW in Faiss + */ +@Log4j2 +public class BinaryIndexIT extends KNNRestTestCase { + private static TestUtils.TestData testData; + + @BeforeClass + public static void setUpClass() throws IOException { + if (BinaryIndexIT.class.getClassLoader() == null) { + throw new IllegalStateException("ClassLoader of BinaryIndexIT Class is null"); + } + URL testIndexVectors = BinaryIndexIT.class.getClassLoader().getResource("data/test_vectors_binary_1000x128.json"); + URL testQueries = BinaryIndexIT.class.getClassLoader().getResource("data/test_queries_binary_100x128.csv"); + assert testIndexVectors != null; + assert testQueries != null; + testData = new TestUtils.TestData(testIndexVectors.getPath(), testQueries.getPath()); + } + + @After + public void cleanUp() { + try { + deleteKNNIndex(INDEX_NAME); + } catch (Exception e) { + log.error(e); + } + } + + @SneakyThrows + public void testFaissHnswBinary_whenSmallDataSet_thenCreateIngestQueryWorks() { + // Create Index + createKnnHnswBinaryIndex(KNNEngine.FAISS, INDEX_NAME, FIELD_NAME, 16); + + // Ingest + Byte[] vector1 = { 0b00000001, 0b00000001 }; + Byte[] vector2 = { 0b00000011, 0b00000001 }; + Byte[] vector3 = { 0b00000111, 0b00000001 }; + Byte[] vector4 = { 0b00001111, 0b00000001 }; + addKnnDoc(INDEX_NAME, "1", FIELD_NAME, vector1); + addKnnDoc(INDEX_NAME, "2", FIELD_NAME, vector2); + addKnnDoc(INDEX_NAME, "3", FIELD_NAME, vector3); + addKnnDoc(INDEX_NAME, "4", FIELD_NAME, vector4); + + // Query + float[] queryVector = { (byte) 0b10001111, (byte) 0b10000000 }; + int k = 4; + List results = runKnnQuery(INDEX_NAME, FIELD_NAME, queryVector, k); + + // Validate + assertEquals(k, results.size()); + for (int i = 0; i < k; i++) { + assertEquals(k - i, Integer.parseInt(results.get(i).getDocId())); + } + } + + @SneakyThrows + public void testFaissHnswBinary_when1000Data_thenCreateIngestQueryWorks() { + // Create Index + createKnnHnswBinaryIndex(KNNEngine.FAISS, INDEX_NAME, FIELD_NAME, 128); + ingestTestData(INDEX_NAME, FIELD_NAME); + + int k = 10; + for (int i = 0; i < testData.queries.length; i++) { + // Query + List knnResults = runKnnQuery(INDEX_NAME, FIELD_NAME, testData.queries[i], k); + + // Validate + assertEquals(k, knnResults.size()); + } + } + + @SneakyThrows + public void testFaissHnswBinary_whenRadialSearch_thenThrowException() { + // Create Index + createKnnHnswBinaryIndex(KNNEngine.FAISS, INDEX_NAME, FIELD_NAME, 16); + + // Query + float[] queryVector = { (byte) 0b10001111, (byte) 0b10000000 }; + Exception e = expectThrows(Exception.class, () -> runRnnQuery(INDEX_NAME, FIELD_NAME, queryVector, 1, 4)); + assertTrue(e.getMessage(), e.getMessage().contains("Binary data type does not support radial search")); + } + + private List runRnnQuery( + final String indexName, + final String fieldName, + final float[] queryVector, + final float minScore, + final int size + ) throws Exception { + String query = KNNJsonQueryBuilder.builder() + .fieldName(fieldName) + .vector(ArrayUtils.toObject(queryVector)) + .minScore(minScore) + .build() + .getQueryString(); + Response response = searchKNNIndex(indexName, query, size); + return parseSearchResponse(EntityUtils.toString(response.getEntity()), fieldName); + } + + private List runKnnQuery(final String indexName, final String fieldName, final float[] queryVector, final int k) + throws Exception { + String query = KNNJsonQueryBuilder.builder() + .fieldName(fieldName) + .vector(ArrayUtils.toObject(queryVector)) + .k(k) + .build() + .getQueryString(); + Response response = searchKNNIndex(indexName, query, k); + return parseSearchResponse(EntityUtils.toString(response.getEntity()), fieldName); + } + + private void ingestTestData(final String indexName, final String fieldName) throws Exception { + // Index the test data + for (int i = 0; i < testData.indexData.docs.length; i++) { + addKnnDoc( + indexName, + Integer.toString(testData.indexData.docs[i]), + fieldName, + Floats.asList(testData.indexData.vectors[i]).toArray() + ); + } + + // Assert we have the right number of documents in the index + refreshAllIndices(); + assertEquals(testData.indexData.docs.length, getDocCount(indexName)); + } + + private void createKnnHnswBinaryIndex(final KNNEngine knnEngine, final String indexName, final String fieldName, final int dimension) + throws IOException { + KNNJsonIndexMappingsBuilder.Method method = KNNJsonIndexMappingsBuilder.Method.builder() + .methodName(METHOD_HNSW) + .engine(knnEngine.getName()) + .build(); + + String knnIndexMapping = KNNJsonIndexMappingsBuilder.builder() + .fieldName(fieldName) + .dimension(dimension) + .vectorDataType(VectorDataType.BINARY.getValue()) + .method(method) + .build() + .getIndexMapping(); + + createKnnIndex(indexName, knnIndexMapping); + } + + private byte[] toByte(final float[] vector) { + byte[] bytes = new byte[vector.length]; + for (int i = 0; i < vector.length; i++) { + bytes[i] = (byte) vector[i]; + } + return bytes; + } +} diff --git a/src/test/java/org/opensearch/knn/integ/BinaryIndexInvalidMappingIT.java b/src/test/java/org/opensearch/knn/integ/BinaryIndexInvalidMappingIT.java new file mode 100644 index 000000000..647860262 --- /dev/null +++ b/src/test/java/org/opensearch/knn/integ/BinaryIndexInvalidMappingIT.java @@ -0,0 +1,100 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.integ; + +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import lombok.AllArgsConstructor; +import lombok.extern.log4j.Log4j2; +import org.junit.After; +import org.opensearch.knn.KNNJsonIndexMappingsBuilder; +import org.opensearch.knn.KNNRestTestCase; +import org.opensearch.knn.index.VectorDataType; +import org.opensearch.knn.index.util.KNNEngine; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Collection; + +import static com.carrotsearch.randomizedtesting.RandomizedTest.$; +import static com.carrotsearch.randomizedtesting.RandomizedTest.$$; +import static org.opensearch.knn.common.KNNConstants.ENCODER_SQ; +import static org.opensearch.knn.common.KNNConstants.METHOD_HNSW; + +/** + * This class contains integration tests for binary index with invalid mapping + */ +@Log4j2 +@AllArgsConstructor +public class BinaryIndexInvalidMappingIT extends KNNRestTestCase { + @After + public void cleanUp() { + try { + deleteKNNIndex(INDEX_NAME); + } catch (Exception e) { + log.error(e); + } + } + + private String description; + private String indexMapping; + private String expectedExceptionMessage; + + @ParametersFactory(argumentFormatting = "description:%1$s; indexMapping:%2$s, expectedExceptionMessage:%3$s") + public static Collection parameters() throws IOException { + return Arrays.asList( + $$( + $( + "Creation of binary index with lucene engine should fail", + createKnnHnswBinaryIndexMapping(KNNEngine.LUCENE, FIELD_NAME, 16, null), + "only supported for [faiss] engine" + ), + $( + "Creation of binary index with nmslib engine should fail", + createKnnHnswBinaryIndexMapping(KNNEngine.NMSLIB, FIELD_NAME, 16, null), + "only supported for [faiss] engine" + ), + $( + "Creation of binary index with encoder should fail", + createKnnHnswBinaryIndexMapping(KNNEngine.FAISS, FIELD_NAME, 16, ENCODER_SQ), + "does not support sq encoder" + ) + ) + ); + } + + public void testBinaryIndexCreation_whenInvalid_thenThrowException() { + Exception e = expectThrows(Exception.class, () -> createKnnIndex(INDEX_NAME, indexMapping)); + assertTrue(e.getMessage(), e.getMessage().contains(expectedExceptionMessage)); + } + + private static String createKnnHnswBinaryIndexMapping( + final KNNEngine knnEngine, + final String fieldName, + final int dimension, + final String encoderName + ) throws IOException { + KNNJsonIndexMappingsBuilder.Method.Parameters.Encoder encoder; + KNNJsonIndexMappingsBuilder.Method.Parameters parameters = null; + if (encoderName != null) { + encoder = KNNJsonIndexMappingsBuilder.Method.Parameters.Encoder.builder().name(encoderName).build(); + parameters = KNNJsonIndexMappingsBuilder.Method.Parameters.builder().encoder(encoder).build(); + } + + KNNJsonIndexMappingsBuilder.Method method = KNNJsonIndexMappingsBuilder.Method.builder() + .methodName(METHOD_HNSW) + .engine(knnEngine.getName()) + .parameters(parameters) + .build(); + + return KNNJsonIndexMappingsBuilder.builder() + .fieldName(fieldName) + .dimension(dimension) + .vectorDataType(VectorDataType.BINARY.getValue()) + .method(method) + .build() + .getIndexMapping(); + } +} diff --git a/src/test/java/org/opensearch/knn/integ/FilteredSearchBinaryIT.java b/src/test/java/org/opensearch/knn/integ/FilteredSearchBinaryIT.java new file mode 100644 index 000000000..0ca5d792e --- /dev/null +++ b/src/test/java/org/opensearch/knn/integ/FilteredSearchBinaryIT.java @@ -0,0 +1,104 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.integ; + +import com.google.common.collect.ImmutableMap; +import lombok.SneakyThrows; +import lombok.extern.log4j.Log4j2; +import org.apache.http.util.EntityUtils; +import org.junit.After; +import org.opensearch.client.Response; +import org.opensearch.common.settings.Settings; +import org.opensearch.knn.KNNJsonIndexMappingsBuilder; +import org.opensearch.knn.KNNJsonQueryBuilder; +import org.opensearch.knn.KNNRestTestCase; +import org.opensearch.knn.index.KNNSettings; +import org.opensearch.knn.index.VectorDataType; +import org.opensearch.knn.index.util.KNNEngine; + +import java.util.List; + +import static org.opensearch.knn.common.KNNConstants.METHOD_HNSW; + +@Log4j2 +public class FilteredSearchBinaryIT extends KNNRestTestCase { + @After + public void cleanUp() { + try { + deleteKNNIndex(INDEX_NAME); + } catch (Exception e) { + log.error(e); + } + } + + @SneakyThrows + public void testFilteredSearchWithFaissHnswBinary_whenDoingApproximateSearch_thenReturnCorrectResults() { + validateFilteredSearchWithFaissHnswBinary(INDEX_NAME, false); + } + + @SneakyThrows + public void testFilteredSearchWithFaissHnswBinary_whenDoingExactSearch_thenReturnCorrectResults() { + validateFilteredSearchWithFaissHnswBinary(INDEX_NAME, true); + } + + private void validateFilteredSearchWithFaissHnswBinary(final String indexName, final boolean doExactSearch) throws Exception { + String filterFieldName = "parking"; + createKnnBinaryIndex(indexName, FIELD_NAME, 24, KNNEngine.FAISS); + + for (byte i = 1; i < 4; i++) { + addKnnDocWithAttributes( + indexName, + Integer.toString(i), + FIELD_NAME, + new float[] { i, i, i }, + ImmutableMap.of(filterFieldName, i % 2 == 1 ? "true" : "false") + ); + } + refreshIndex(indexName); + forceMergeKnnIndex(indexName); + + // Set it as 0 for approximate search and 100(larger than number of filtered id) for exact search + updateIndexSettings( + indexName, + Settings.builder().put(KNNSettings.ADVANCED_FILTERED_EXACT_SEARCH_THRESHOLD, doExactSearch ? 100 : 0) + ); + + Float[] queryVector = { 3f, 3f, 3f }; + String query = KNNJsonQueryBuilder.builder() + .fieldName(FIELD_NAME) + .vector(queryVector) + .k(3) + .filterFieldName(filterFieldName) + .filterValue("true") + .build() + .getQueryString(); + Response response = searchKNNIndex(indexName, query, 3); + String entity = EntityUtils.toString(response.getEntity()); + List docIds = parseIds(entity); + assertEquals(2, docIds.size()); + assertEquals("3", docIds.get(0)); + assertEquals("1", docIds.get(1)); + assertEquals(2, parseTotalSearchHits(entity)); + } + + private void createKnnBinaryIndex(final String indexName, final String fieldName, final int dimension, final KNNEngine knnEngine) + throws Exception { + KNNJsonIndexMappingsBuilder.Method method = KNNJsonIndexMappingsBuilder.Method.builder() + .methodName(METHOD_HNSW) + .engine(knnEngine.getName()) + .build(); + + String knnIndexMapping = KNNJsonIndexMappingsBuilder.builder() + .fieldName(fieldName) + .dimension(dimension) + .vectorDataType(VectorDataType.BINARY.getValue()) + .method(method) + .build() + .getIndexMapping(); + + createKnnIndex(indexName, knnIndexMapping); + } +} diff --git a/src/test/java/org/opensearch/knn/plugin/script/KNNScriptScoringIT.java b/src/test/java/org/opensearch/knn/integ/KNNScriptScoringIT.java similarity index 79% rename from src/test/java/org/opensearch/knn/plugin/script/KNNScriptScoringIT.java rename to src/test/java/org/opensearch/knn/integ/KNNScriptScoringIT.java index 0d6248830..d8252ad0f 100644 --- a/src/test/java/org/opensearch/knn/plugin/script/KNNScriptScoringIT.java +++ b/src/test/java/org/opensearch/knn/integ/KNNScriptScoringIT.java @@ -3,10 +3,13 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.knn.plugin.script; +package org.opensearch.knn.integ; +import java.util.Set; import java.util.function.BiFunction; import java.util.function.Function; + +import lombok.SneakyThrows; import org.opensearch.ExceptionsHelper; import org.opensearch.knn.KNNRestTestCase; import org.opensearch.knn.KNNResult; @@ -29,6 +32,9 @@ import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.mapper.KNNVectorFieldMapper; import org.opensearch.knn.index.util.KNNEngine; +import org.opensearch.knn.plugin.script.KNNScoringScriptEngine; +import org.opensearch.knn.plugin.script.KNNScoringSpace; +import org.opensearch.knn.plugin.script.KNNScoringSpaceFactory; import org.opensearch.script.Script; import java.util.ArrayList; @@ -74,6 +80,58 @@ public void testKNNCosineScriptScore() throws Exception { testKNNScriptScore(SpaceType.COSINESIMIL); } + @SneakyThrows + public void testKNNHammingScriptScore() { + testKNNScriptScoreOnBinaryIndex(SpaceType.HAMMING); + } + + @SneakyThrows + public void testKNNHammingScriptScore_whenNonBinary_thenException() { + final int dims = randomIntBetween(2, 10) * 8; + final float[] queryVector = randomVector(dims, VectorDataType.BYTE); + final BiFunction scoreFunction = getScoreFunction(SpaceType.HAMMING, queryVector); + List nonBinary = List.of(VectorDataType.FLOAT, VectorDataType.BYTE); + for (VectorDataType vectorDataType : nonBinary) { + Exception e = expectThrows( + Exception.class, + () -> createIndexAndAssertScriptScore( + createKnnIndexMapping(FIELD_NAME, dims, vectorDataType), + SpaceType.HAMMING, + scoreFunction, + dims, + queryVector, + true, + false, + vectorDataType + ) + ); + assertTrue(e.getMessage(), e.getMessage().contains("data type should be [BINARY]")); + } + } + + public void testKNNNonHammingScriptScore_whenBinary_thenException() { + final int dims = randomIntBetween(2, 10) * 8; + final float[] queryVector = randomVector(dims, VectorDataType.BINARY); + final BiFunction scoreFunction = getScoreFunction(SpaceType.HAMMING, queryVector); + Set spaceTypeToExclude = Set.of(SpaceType.UNDEFINED, SpaceType.HAMMING); + Arrays.stream(SpaceType.values()).filter(s -> spaceTypeToExclude.contains(s) == false).forEach(s -> { + Exception e = expectThrows( + Exception.class, + () -> createIndexAndAssertScriptScore( + createKnnIndexMapping(FIELD_NAME, dims, VectorDataType.BINARY), + s, + scoreFunction, + dims, + queryVector, + true, + false, + VectorDataType.BINARY + ) + ); + assertTrue(e.getMessage(), e.getMessage().contains("Incompatible field_type")); + }); + } + public void testKNNInvalidSourceScript() throws Exception { /* * Create knn index and populate data @@ -619,10 +677,38 @@ private List createMappers(int dimensions) throws Exception { ); } - private float[] randomVector(int dimensions) { - final float[] vector = new float[dimensions]; - for (int i = 0; i < dimensions; i++) { - vector[i] = randomFloat(); + private List createBinaryIndexMappers(int dimensions) throws Exception { + return List.of( + createKnnIndexMapping( + FIELD_NAME, + dimensions, + KNNConstants.METHOD_HNSW, + KNNEngine.FAISS.getName(), + SpaceType.DEFAULT_BINARY.getValue(), + true, + VectorDataType.BINARY + ), + createKnnIndexMapping( + FIELD_NAME, + dimensions, + KNNConstants.METHOD_HNSW, + KNNEngine.FAISS.getName(), + SpaceType.DEFAULT_BINARY.getValue(), + false, + VectorDataType.BINARY + ) + ); + } + + private float[] randomVector(final int dimensions) { + return randomVector(dimensions, VectorDataType.FLOAT); + } + + private float[] randomVector(final int dimensions, final VectorDataType vectorDataType) { + int size = VectorDataType.BINARY == vectorDataType ? dimensions / 8 : dimensions; + final float[] vector = new float[size]; + for (int i = 0; i < size; i++) { + vector[i] = VectorDataType.FLOAT == vectorDataType ? randomFloat() : randomByte(); } return vector; } @@ -631,7 +717,8 @@ private Map createDataset( Function scoreFunction, int dimensions, int numDocsWithField, - boolean dense + boolean dense, + VectorDataType vectorDataType ) { final Map dataset = new HashMap<>(dense ? numDocsWithField : numDocsWithField * 3); int id = 0; @@ -640,7 +727,7 @@ private Map createDataset( for (int j = 0; j < dummyDocs; j++) { dataset.put(Integer.toString(id++), null); } - final float[] vector = randomVector(dimensions); + final float[] vector = randomVector(dimensions, vectorDataType); final float score = scoreFunction.apply(vector); dataset.put(Integer.toString(id), new KNNResult(Integer.toString(id++), vector, score)); } @@ -651,8 +738,8 @@ private BiFunction getScoreFunction(SpaceType spaceType KNNVectorFieldMapper.KNNVectorFieldType knnVectorFieldType = new KNNVectorFieldMapper.KNNVectorFieldType( FIELD_NAME, Collections.emptyMap(), - queryVector.length, - VectorDataType.FLOAT, + SpaceType.HAMMING == spaceType ? queryVector.length * 8 : queryVector.length, + SpaceType.HAMMING == spaceType ? VectorDataType.BINARY : VectorDataType.FLOAT, null ); List target = new ArrayList<>(queryVector.length); @@ -662,15 +749,12 @@ private BiFunction getScoreFunction(SpaceType spaceType KNNScoringSpace knnScoringSpace = KNNScoringSpaceFactory.create(spaceType.getValue(), target, knnVectorFieldType); switch (spaceType) { case L1: - return ((KNNScoringSpace.L1) knnScoringSpace).scoringMethod; case L2: - return ((KNNScoringSpace.L2) knnScoringSpace).scoringMethod; case LINF: - return ((KNNScoringSpace.LInf) knnScoringSpace).scoringMethod; case COSINESIMIL: - return ((KNNScoringSpace.CosineSimilarity) knnScoringSpace).scoringMethod; case INNER_PRODUCT: - return ((KNNScoringSpace.InnerProd) knnScoringSpace).scoringMethod; + case HAMMING: + return ((KNNScoringSpace.KNNFieldSpace) knnScoringSpace).getScoringMethod(); default: throw new IllegalArgumentException(); } @@ -686,6 +770,40 @@ private void testKNNScriptScore(SpaceType spaceType) throws Exception { } } + private void testKNNScriptScoreOnBinaryIndex(SpaceType spaceType) throws Exception { + final int dims = randomIntBetween(2, 10) * 8; + final float[] queryVector = randomVector(dims, VectorDataType.BINARY); + final BiFunction scoreFunction = getScoreFunction(spaceType, queryVector); + + // Test when knn is enabled and engine is Faiss + for (String mapper : createBinaryIndexMappers(dims)) { + createIndexAndAssertScriptScore(mapper, spaceType, scoreFunction, dims, queryVector, true, true, VectorDataType.BINARY); + createIndexAndAssertScriptScore(mapper, spaceType, scoreFunction, dims, queryVector, false, true, VectorDataType.BINARY); + } + + // Test when knn is disabled and engine is default(Nmslib) + createIndexAndAssertScriptScore( + createKnnIndexMapping(FIELD_NAME, dims, VectorDataType.BINARY), + spaceType, + scoreFunction, + dims, + queryVector, + true, + false, + VectorDataType.BINARY + ); + createIndexAndAssertScriptScore( + createKnnIndexMapping(FIELD_NAME, dims, VectorDataType.BINARY), + spaceType, + scoreFunction, + dims, + queryVector, + false, + false, + VectorDataType.BINARY + ); + } + private void createIndexAndAssertScriptScore( String mapper, SpaceType spaceType, @@ -694,38 +812,61 @@ private void createIndexAndAssertScriptScore( float[] queryVector, boolean dense ) throws Exception { - /* - * Create knn index and populate data - */ - createKnnIndex(INDEX_NAME, mapper); - final int numDocsWithField = randomIntBetween(4, 10); - Map dataset = createDataset(v -> scoreFunction.apply(queryVector, v), dimensions, numDocsWithField, dense); - final float[] dummyVector = new float[1]; - dataset.forEach((k, v) -> { - final float[] vector = (v != null) ? v.getVector() : dummyVector; - ExceptionsHelper.catchAsRuntimeException(() -> addKnnDoc(INDEX_NAME, k, (v != null) ? FIELD_NAME : "dummy", vector)); - }); + createIndexAndAssertScriptScore(mapper, spaceType, scoreFunction, dimensions, queryVector, dense, true, VectorDataType.FLOAT); + } - /** - * Construct Search Request - */ - QueryBuilder qb = new MatchAllQueryBuilder(); - Map params = new HashMap<>(); + private void createIndexAndAssertScriptScore( + String mapper, + SpaceType spaceType, + BiFunction scoreFunction, + int dimensions, + float[] queryVector, + boolean dense, + boolean enableKnn, + VectorDataType vectorDataType + ) throws Exception { /* - * params": { - * "field": FIELD_NAME, - * "vector": queryVector - * } + * Create knn index and populate data */ - params.put("field", FIELD_NAME); - params.put("query_value", queryVector); - params.put("space_type", spaceType.getValue()); - Request request = constructKNNScriptQueryRequest(INDEX_NAME, qb, params, numDocsWithField); - Response response = client().performRequest(request); - assertEquals(request.getEndpoint() + ": failed", RestStatus.OK, RestStatus.fromCode(response.getStatusLine().getStatusCode())); - - List results = parseSearchResponse(EntityUtils.toString(response.getEntity()), FIELD_NAME); - assertTrue(results.stream().allMatch(r -> dataset.get(r.getDocId()).equals(r))); - deleteKNNIndex(INDEX_NAME); + Settings settings = Settings.builder().put("number_of_shards", 1).put("number_of_replicas", 0).put("index.knn", enableKnn).build(); + createKnnIndex(INDEX_NAME, settings, mapper); + try { + final int numDocsWithField = randomIntBetween(4, 10); + Map dataset = createDataset( + v -> scoreFunction.apply(queryVector, v), + dimensions, + numDocsWithField, + dense, + vectorDataType + ); + final float[] dummyVector = new float[1]; + dataset.forEach((k, v) -> { + final float[] vector = (v != null) ? v.getVector() : dummyVector; + ExceptionsHelper.catchAsRuntimeException(() -> addKnnDoc(INDEX_NAME, k, (v != null) ? FIELD_NAME : "dummy", vector)); + }); + + /** + * Construct Search Request + */ + QueryBuilder qb = new MatchAllQueryBuilder(); + Map params = new HashMap<>(); + /* + * params": { + * "field": FIELD_NAME, + * "vector": queryVector + * } + */ + params.put("field", FIELD_NAME); + params.put("query_value", queryVector); + params.put("space_type", spaceType.getValue()); + Request request = constructKNNScriptQueryRequest(INDEX_NAME, qb, params, numDocsWithField); + Response response = client().performRequest(request); + assertEquals(request.getEndpoint() + ": failed", RestStatus.OK, RestStatus.fromCode(response.getStatusLine().getStatusCode())); + + List results = parseSearchResponse(EntityUtils.toString(response.getEntity()), FIELD_NAME); + assertTrue(results.stream().allMatch(r -> dataset.get(r.getDocId()).equals(r))); + } finally { + deleteKNNIndex(INDEX_NAME); + } } } diff --git a/src/test/java/org/opensearch/knn/integ/NestedSearchBinaryIT.java b/src/test/java/org/opensearch/knn/integ/NestedSearchBinaryIT.java new file mode 100644 index 000000000..bff5588b6 --- /dev/null +++ b/src/test/java/org/opensearch/knn/integ/NestedSearchBinaryIT.java @@ -0,0 +1,156 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.integ; + +import lombok.SneakyThrows; +import lombok.extern.log4j.Log4j2; +import org.apache.http.util.EntityUtils; +import org.junit.After; +import org.opensearch.client.Response; +import org.opensearch.common.settings.Settings; +import org.opensearch.knn.KNNJsonIndexMappingsBuilder; +import org.opensearch.knn.KNNJsonQueryBuilder; +import org.opensearch.knn.KNNRestTestCase; +import org.opensearch.knn.NestedKnnDocBuilder; +import org.opensearch.knn.index.KNNSettings; +import org.opensearch.knn.index.VectorDataType; +import org.opensearch.knn.index.util.KNNEngine; + +import java.util.List; + +import static org.opensearch.knn.common.KNNConstants.METHOD_HNSW; + +@Log4j2 +public class NestedSearchBinaryIT extends KNNRestTestCase { + @After + public void cleanUp() { + try { + deleteKNNIndex(INDEX_NAME); + } catch (Exception e) { + log.error(e); + } + } + + @SneakyThrows + public void testNestedSearchWithFaissHnswBinary_whenKIsTwo_thenReturnTwoResults() { + String nestedFieldName = "nested"; + createKnnBinaryIndexWithNestedField(INDEX_NAME, nestedFieldName, FIELD_NAME, 16, KNNEngine.FAISS); + + int totalDocCount = 15; + for (byte i = 0; i < totalDocCount; i++) { + String doc = NestedKnnDocBuilder.create(nestedFieldName) + .addVectors(FIELD_NAME, new Byte[] { i, i }, new Byte[] { i, i }) + .build(); + addKnnDoc(INDEX_NAME, String.valueOf(i), doc); + } + + refreshIndex(INDEX_NAME); + forceMergeKnnIndex(INDEX_NAME); + + Float[] queryVector = { 14f, 14f }; + String query = KNNJsonQueryBuilder.builder() + .nestedFieldName(nestedFieldName) + .fieldName(FIELD_NAME) + .vector(queryVector) + .k(2) + .build() + .getQueryString(); + Response response = searchKNNIndex(INDEX_NAME, query, 2); + String entity = EntityUtils.toString(response.getEntity()); + + assertEquals(2, parseHits(entity)); + assertEquals(2, parseTotalSearchHits(entity)); + assertEquals("14", parseIds(entity).get(0)); + assertNotEquals("14", parseIds(entity).get(1)); + } + + /** + * { + * "query": { + * "nested": { + * "path": "test_nested", + * "query": { + * "knn": { + * "test_nested.test_vector": { + * "vector": [ + * 1, 1, 1 + * ], + * "k": 3, + * "filter": { + * "term": { + * "parking": "true" + * } + * } + * } + * } + * } + * } + * } + * } + * + */ + @SneakyThrows + public void testNestedSearchWithFaissHnswBinary_whenDoingExactSearch_thenReturnCorrectResults() { + String nestedFieldName = "nested"; + String filterFieldName = "parking"; + createKnnBinaryIndexWithNestedField(INDEX_NAME, nestedFieldName, FIELD_NAME, 24, KNNEngine.FAISS); + + for (byte i = 1; i < 4; i++) { + String doc = NestedKnnDocBuilder.create(nestedFieldName) + .addVectors(FIELD_NAME, new Byte[] { i, i, i }, new Byte[] { i, i, i }, new Byte[] { i, i, i }) + .addTopLevelField(filterFieldName, i % 2 == 1 ? "true" : "false") + .build(); + addKnnDoc(INDEX_NAME, String.valueOf(i), doc); + } + refreshIndex(INDEX_NAME); + forceMergeKnnIndex(INDEX_NAME); + + // Make it as an exact search by setting the threshold larger than size of filteredIds(6) + updateIndexSettings(INDEX_NAME, Settings.builder().put(KNNSettings.ADVANCED_FILTERED_EXACT_SEARCH_THRESHOLD, 100)); + + Float[] queryVector = { 3f, 3f, 3f }; + String query = KNNJsonQueryBuilder.builder() + .nestedFieldName(nestedFieldName) + .fieldName(FIELD_NAME) + .vector(queryVector) + .k(3) + .filterFieldName(filterFieldName) + .filterValue("true") + .build() + .getQueryString(); + Response response = searchKNNIndex(INDEX_NAME, query, 3); + String entity = EntityUtils.toString(response.getEntity()); + List docIds = parseIds(entity); + assertEquals(2, docIds.size()); + assertEquals("3", docIds.get(0)); + assertEquals("1", docIds.get(1)); + assertEquals(2, parseTotalSearchHits(entity)); + } + + private void createKnnBinaryIndexWithNestedField( + final String indexName, + final String nestedFieldName, + final String fieldName, + final int dimension, + final KNNEngine knnEngine + ) throws Exception { + KNNJsonIndexMappingsBuilder.Method method = KNNJsonIndexMappingsBuilder.Method.builder() + .methodName(METHOD_HNSW) + .engine(knnEngine.getName()) + .build(); + + String knnIndexMapping = KNNJsonIndexMappingsBuilder.builder() + .nestedFieldName(nestedFieldName) + .fieldName(fieldName) + .dimension(dimension) + .vectorDataType(VectorDataType.BINARY.getValue()) + .method(method) + .build() + .getIndexMapping(); + + createKnnIndex(indexName, knnIndexMapping); + } +} diff --git a/src/test/java/org/opensearch/knn/index/NestedSearchIT.java b/src/test/java/org/opensearch/knn/integ/NestedSearchIT.java similarity index 99% rename from src/test/java/org/opensearch/knn/index/NestedSearchIT.java rename to src/test/java/org/opensearch/knn/integ/NestedSearchIT.java index 22d8ff68f..1bf6037dc 100644 --- a/src/test/java/org/opensearch/knn/index/NestedSearchIT.java +++ b/src/test/java/org/opensearch/knn/integ/NestedSearchIT.java @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.knn.index; +package org.opensearch.knn.integ; import lombok.SneakyThrows; import org.apache.http.util.EntityUtils; @@ -16,6 +16,8 @@ import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.knn.KNNRestTestCase; import org.opensearch.knn.NestedKnnDocBuilder; +import org.opensearch.knn.index.KNNSettings; +import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.util.KNNEngine; import java.io.IOException; diff --git a/src/test/java/org/opensearch/knn/plugin/script/PainlessScriptIT.java b/src/test/java/org/opensearch/knn/integ/PainlessScriptIT.java similarity index 87% rename from src/test/java/org/opensearch/knn/plugin/script/PainlessScriptIT.java rename to src/test/java/org/opensearch/knn/integ/PainlessScriptIT.java index 47e914d04..03273bffa 100644 --- a/src/test/java/org/opensearch/knn/plugin/script/PainlessScriptIT.java +++ b/src/test/java/org/opensearch/knn/integ/PainlessScriptIT.java @@ -3,8 +3,10 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.knn.plugin.script; +package org.opensearch.knn.integ; +import lombok.SneakyThrows; +import org.opensearch.common.settings.Settings; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.knn.KNNRestTestCase; import org.opensearch.knn.KNNResult; @@ -12,6 +14,7 @@ import org.opensearch.knn.index.KNNMethodContext; import org.opensearch.knn.index.MethodComponentContext; import org.opensearch.knn.index.SpaceType; +import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.mapper.KNNVectorFieldMapper; import org.apache.http.util.EntityUtils; import org.opensearch.client.Request; @@ -27,9 +30,11 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Objects; @@ -584,6 +589,89 @@ public void testL2ScriptingWithLuceneBackedIndex() throws Exception { deleteKNNIndex(INDEX_NAME); } + @SneakyThrows + public void testHammingPainlessScript_whenBinary_thenSuccess() { + int dimensions = 16; + String mappingForKnnDisabled = createKnnIndexMapping(FIELD_NAME, dimensions, VectorDataType.BINARY); + + // 0b00000001, 0b00000001 + String source = String.format("1/(1 + hamming([1.0f, 1.0f], doc['%s']))", FIELD_NAME); + + Map data = new HashMap<>(); + data.put("1", new Float[] { (float) 0b00000001, (float) 0b00000001 });// Hamming distance 0 + data.put("2", new Float[] { (float) 0b01101111, (float) 0b00000010 });// Hamming distance 6 + data.put("3", new Float[] { (float) 0b01100010, (float) 0b00000011 });// Hamming distance 5 + data.put("4", new Float[] { (float) 0b00000001, (float) 0b01001100 });// Hamming distance 4 + + Response response = buildIndexAndRunPainlessScript(source, 4, data, mappingForKnnDisabled, false); + List results = parseSearchResponse(EntityUtils.toString(response.getEntity()), FIELD_NAME); + assertEquals(4, results.size()); + + String[] expectedDocIDs = { "1", "4", "3", "2" }; + for (int i = 0; i < results.size(); i++) { + assertEquals(expectedDocIDs[i], results.get(i).getDocId()); + } + } + + @SneakyThrows + public void testPainlessScript_whenNonBinary_thenException() { + int dimensions = 2; + String mapping = createKnnIndexMapping(FIELD_NAME, dimensions); + String source = String.format("1/(1 + hamming([1.0f, 1.0f], doc['%s']))", FIELD_NAME); + Exception e = expectThrows( + ResponseException.class, + () -> buildIndexAndRunPainlessScript(source, 3, getKnnVectorTestData(), mapping, false) + ); + assertTrue(e.getMessage(), e.getMessage().contains("The data type should be binary")); + } + + @SneakyThrows + public void testNonPainlessScript_whenBinary_thenException() { + List functions = Arrays.asList("l2Squared", "lInfNorm", "l1Norm", "innerProduct", "cosineSimilarity"); + int dimensions = 16; + String mapping = createKnnIndexMapping(FIELD_NAME, dimensions, VectorDataType.BINARY); + for (String function : functions) { + String source = String.format(Locale.ROOT, "%s([1.0f, 1.0f], doc['%s'])", function, FIELD_NAME); + Exception e = expectThrows( + ResponseException.class, + () -> buildIndexAndRunPainlessScript(source, 3, getKnnVectorTestData(), mapping, false) + ); + assertTrue(e.getMessage(), e.getMessage().contains("The data type should be either float or byte")); + } + } + + private Response buildIndexAndRunPainlessScript( + final String source, + final int size, + Map documents, + final String mapper, + final boolean enableKnn + ) throws Exception { + /* + * Create knn index and populate data + */ + Settings settings = Settings.builder().put("number_of_shards", 1).put("number_of_replicas", 0).put("index.knn", enableKnn).build(); + createKnnIndex(INDEX_NAME, settings, mapper); + try { + for (Map.Entry data : documents.entrySet()) { + addKnnDoc(INDEX_NAME, data.getKey(), FIELD_NAME, data.getValue()); + } + QueryBuilder qb = new MatchAllQueryBuilder(); + Request request = constructScriptScoreContextSearchRequest( + INDEX_NAME, + qb, + Collections.emptyMap(), + Script.DEFAULT_SCRIPT_LANG, + source, + size, + Collections.emptyMap() + ); + return client().performRequest(request); + } finally { + deleteKNNIndex(INDEX_NAME); + } + } + static class MappingProperty { private final String name; diff --git a/src/test/java/org/opensearch/knn/plugin/script/KNNScoringSpaceTests.java b/src/test/java/org/opensearch/knn/plugin/script/KNNScoringSpaceTests.java index 5a86deb7e..aa0261657 100644 --- a/src/test/java/org/opensearch/knn/plugin/script/KNNScoringSpaceTests.java +++ b/src/test/java/org/opensearch/knn/plugin/script/KNNScoringSpaceTests.java @@ -65,7 +65,7 @@ public void testL2_whenValid_thenSucceed() { knnMethodContext ); KNNScoringSpace.L2 l2 = new KNNScoringSpace.L2(arrayListQueryObject, fieldType); - assertEquals(1F, l2.scoringMethod.apply(arrayFloat, arrayFloat), 0.1F); + assertEquals(1F, l2.getScoringMethod().apply(arrayFloat, arrayFloat), 0.1F); } @SneakyThrows @@ -87,7 +87,7 @@ public void testCosineSimilarity_whenValid_thenSucceed() { knnMethodContext ); KNNScoringSpace.CosineSimilarity cosineSimilarity = new KNNScoringSpace.CosineSimilarity(arrayListQueryObject, fieldType); - assertEquals(2F, cosineSimilarity.scoringMethod.apply(arrayFloat2, arrayFloat), 0.1F); + assertEquals(2F, cosineSimilarity.getScoringMethod().apply(arrayFloat2, arrayFloat), 0.1F); // invalid zero vector final List queryZeroVector = List.of(0.0f, 0.0f, 0.0f); @@ -141,7 +141,7 @@ public void testInnerProd_whenValid_thenSucceed() { ); KNNScoringSpace.InnerProd innerProd = new KNNScoringSpace.InnerProd(arrayListQueryObject_case1, fieldType); - assertEquals(7.0F, innerProd.scoringMethod.apply(arrayFloat_case1, arrayFloat2_case1), 0.001F); + assertEquals(7.0F, innerProd.getScoringMethod().apply(arrayFloat_case1, arrayFloat2_case1), 0.001F); float[] arrayFloat_case2 = new float[] { 100_000.0f, 200_000.0f, 300_000.0f }; List arrayListQueryObject_case2 = new ArrayList<>(Arrays.asList(100_000.0, 200_000.0, 300_000.0)); @@ -149,7 +149,7 @@ public void testInnerProd_whenValid_thenSucceed() { innerProd = new KNNScoringSpace.InnerProd(arrayListQueryObject_case2, fieldType); - assertEquals(7.142857143E-12F, innerProd.scoringMethod.apply(arrayFloat_case2, arrayFloat2_case2), 1.0E-11F); + assertEquals(7.142857143E-12F, innerProd.getScoringMethod().apply(arrayFloat_case2, arrayFloat2_case2), 1.0E-11F); float[] arrayFloat_case3 = new float[] { 100_000.0f, 200_000.0f, 300_000.0f }; List arrayListQueryObject_case3 = new ArrayList<>(Arrays.asList(100_000.0, 200_000.0, 300_000.0)); @@ -157,7 +157,7 @@ public void testInnerProd_whenValid_thenSucceed() { innerProd = new KNNScoringSpace.InnerProd(arrayListQueryObject_case3, fieldType); - assertEquals(140_000_000_001F, innerProd.scoringMethod.apply(arrayFloat_case3, arrayFloat2_case3), 0.01F); + assertEquals(140_000_000_001F, innerProd.getScoringMethod().apply(arrayFloat_case3, arrayFloat2_case3), 0.01F); } @SneakyThrows @@ -214,7 +214,7 @@ public void testHamming_whenKNNFieldType_thenSucceed() { KNNScoringSpace.Hamming hamming = new KNNScoringSpace.Hamming(arrayListQueryObject, fieldType); float[] arrayFloat = new float[] { 1.0f, 2.0f, 3.0f }; - assertEquals(1F, hamming.scoringMethod.apply(arrayFloat, arrayFloat), 0.1F); + assertEquals(1F, hamming.getScoringMethod().apply(arrayFloat, arrayFloat), 0.1F); } public void testHamming_whenNonBinaryVectorDataType_thenException() { diff --git a/src/test/resources/data/README.md b/src/test/resources/data/README.md index 8d62d361d..f3711eae3 100644 --- a/src/test/resources/data/README.md +++ b/src/test/resources/data/README.md @@ -1,3 +1,15 @@ +# test_vectors_binary_1000x128.json +The file contains a simulated data to represent binary vector. The data is generated by quantizing data from +test_vectors_1000x128.json and packing 8 bits to 1 byte which ends up with 16 length of vector per document. +For quantization technique, we calculated the median(49935.95941056451) of all values and converted it as 0 +if it is less than the median and 1 if it is equal to or larger than the median. + +# test_queries_binary_100x128.csv +The file contains a simulated query data in binary vector format. The data is generated by quantizing data from +test_queries_100x128.csv and packing 8 bits to 1 byte with ends up with 16 length of vector per query. +For quantization technique, we calculated the median(49935.95941056451) of all values in test_vectors_1000x128.json +and converted it as 0 if it is less than the median and 1 if it is equal to or larger than the median. + # test_vectors_nested_1000x128.json The file contains a simulated data to represent nested field. Consecutive ids are assigned for data from same parent document. diff --git a/src/test/resources/data/test_queries_binary_100x128.csv b/src/test/resources/data/test_queries_binary_100x128.csv new file mode 100644 index 000000000..0740c6fd4 --- /dev/null +++ b/src/test/resources/data/test_queries_binary_100x128.csv @@ -0,0 +1,100 @@ +-53,97,109,87,117,-42,116,-90,-17,-5,62,-66,2,109,-78,-52 +9,65,-21,87,-58,-75,-29,1,-73,124,12,-102,-86,18,83,91 +-72,76,-21,-100,66,-83,-65,61,-71,39,-81,106,27,5,-125,99 +34,-54,5,-3,127,45,13,29,24,-60,-90,-40,69,30,-23,-100 +-91,-51,43,78,81,120,-59,-100,-35,-68,-56,-69,-40,16,-110,-28 +-123,-123,-15,-25,-16,17,-52,2,86,12,-98,-73,33,78,110,5 +-77,-115,-13,-84,4,-13,-32,75,-128,24,-17,-78,28,-57,111,-20 +79,-71,-5,127,-46,-72,118,123,-7,33,-85,95,-61,-121,95,-61 +-52,40,-5,11,112,-69,11,-36,-26,78,-16,-33,59,-105,102,-51 +43,48,70,85,-111,-21,34,14,112,98,0,-126,73,127,46,-52 +84,47,-109,-29,72,68,-34,84,-105,64,-73,-71,-111,-35,-79,89 +-57,21,69,33,-65,-42,78,106,-18,92,51,29,-109,60,101,100 +-20,-36,-126,-22,-122,110,50,47,38,-42,99,32,21,-13,61,-38 +-83,36,27,5,127,-115,-87,127,-12,-60,-105,108,60,112,-50,109 +113,34,-57,105,127,-122,70,-86,-111,-95,-112,-75,-72,127,-33,124 +121,-53,-111,113,-87,-106,87,-30,-9,-128,-62,-91,-91,75,-61,83 +-4,-110,55,-119,-114,123,95,-9,-28,-8,-55,-117,-77,-96,88,83 +63,-109,107,82,125,24,-72,-89,-1,64,38,70,117,72,1,-27 +38,95,-87,-87,-49,-104,-35,-126,57,-95,-30,-35,-114,-41,-33,-46 +-95,6,30,71,-16,-112,83,-90,-118,87,-29,53,-96,-68,125,-73 +58,-47,25,-44,-4,-125,-30,5,-16,-42,107,-42,-68,-97,17,-51 +-125,-113,108,-29,57,-48,102,-76,-34,0,-81,-21,113,-13,95,65 +-11,-73,38,47,-6,-80,91,39,-4,-26,-43,30,63,12,30,52 +-76,124,85,126,-106,-23,52,104,-81,-87,-66,-68,81,40,-99,7 +-108,47,-76,-47,110,-28,-49,46,70,25,72,-73,-122,-41,-47,117 +19,27,48,74,-70,-85,-87,-107,-30,-44,92,11,113,-7,42,-16 +35,114,75,-30,-69,115,-94,-116,22,81,-41,120,-64,-115,97,-43 +-111,8,83,-62,0,15,-10,-83,63,-96,92,-44,-23,85,78,123 +110,78,-46,105,-40,-63,40,122,-69,-96,23,-90,81,46,122,51 +21,25,-2,51,-115,28,-111,-8,-125,-126,-79,4,99,48,78,-17 +125,-2,57,82,-66,77,-125,-49,16,33,-88,-6,-74,-75,23,1 +82,28,-38,28,23,-26,-9,-1,-68,85,-92,78,100,-118,-120,-9 +90,55,72,67,-50,20,-9,-57,-66,9,-74,-128,81,125,11,37 +109,49,-23,61,89,-12,7,70,64,-111,110,-89,-21,-95,95,118 +31,55,-44,118,50,87,-90,-91,-2,-44,117,109,6,-96,-105,45 +82,13,109,3,-83,-99,36,-115,55,-124,-36,-126,-10,-55,-47,96 +102,-20,84,-41,-59,-112,-54,117,-70,52,31,-118,-70,-108,-80,-27 +-42,96,3,6,-17,-56,-113,31,86,2,25,25,91,58,-48,-84 +-120,-121,87,-44,121,-27,95,-46,-79,-51,-86,-20,-100,70,51,-88 +118,111,-82,115,48,-108,122,16,66,37,-93,52,106,-59,73,83 +21,-10,101,-68,-73,-42,-25,88,-40,-35,16,62,126,6,17,-109 +-49,68,-10,93,107,88,91,-116,-101,-102,77,-82,117,-119,1,102 +-92,92,70,-67,17,59,-9,-111,19,-21,-60,-111,-88,-2,61,111 +63,14,92,-76,71,101,87,11,24,69,-116,71,22,-22,-94,114 +122,72,-32,-54,91,14,-59,-87,-97,-45,46,99,-118,-21,124,-82 +44,-101,91,15,-25,-17,109,63,-86,50,-76,-32,-86,-46,-91,70 +89,114,101,-96,57,-27,-71,8,-1,-112,98,-90,90,-111,-50,-110 +58,16,-55,-88,83,-59,-26,-83,92,54,-97,79,108,120,32,68 +-39,69,0,109,-39,6,-57,85,-94,-35,107,-127,-54,-43,112,9 +90,88,-25,-58,-14,-23,-46,-37,101,-3,107,81,-52,3,61,-63 +125,9,-4,-55,-56,69,-7,-113,27,69,72,8,-93,97,49,32 +-112,20,-68,-14,-68,101,-19,45,28,66,-10,-68,6,6,-8,-105 +-89,2,-12,112,48,2,-45,0,-1,25,4,-37,35,-54,16,-44 +-92,-85,7,65,-94,121,-26,-53,-23,75,-104,86,-88,74,-83,18 +-9,-34,89,-55,41,86,-30,-110,-1,37,67,99,-42,-63,-63,29 +58,-70,81,80,57,-43,-59,6,-22,-36,6,-96,51,-64,-95,54 +-24,113,-83,-25,29,-44,89,32,36,7,81,121,75,-10,23,-83 +-99,-27,-78,-35,101,-112,0,78,75,-38,-34,-111,-65,110,27,8 +87,-98,45,91,6,64,100,19,-7,51,-78,51,25,13,-78,92 +-126,54,-96,-38,-105,91,-93,-61,118,95,60,28,123,0,105,10 +-56,95,-60,78,2,65,-28,10,-19,-6,-49,101,-85,-80,98,-124 +-62,50,-39,-48,31,-50,-117,-46,-82,-81,34,16,-15,-83,-114,-117 +98,-75,121,117,-127,70,82,113,97,99,119,-56,-52,5,45,58 +3,-12,-33,-41,77,22,-92,120,-119,-112,-85,41,-107,123,53,-11 +-121,90,-55,48,-32,-12,-91,108,23,18,10,-92,-3,26,58,-126 +89,-77,-33,110,115,41,99,-68,-3,37,-52,-43,-39,-76,-107,31 +-68,-72,-2,63,91,-113,-109,41,-11,65,-71,-37,-1,82,70,-88 +-70,-47,-11,18,53,-49,-63,-10,66,-65,-125,65,118,-70,-34,-24 +-63,105,-112,34,91,-119,45,127,76,35,11,121,39,-52,-118,-7 +97,32,-102,-96,42,-94,106,77,-30,69,-92,-100,104,29,127,-31 +-119,35,81,124,50,-8,-53,60,-99,118,-128,29,-31,-123,-27,76 +39,-108,-12,78,-100,-21,10,-12,21,-27,-56,124,113,-30,89,-76 +-39,30,-21,-77,107,-95,-18,-44,115,-32,108,100,-84,0,-45,85 +34,4,50,-58,-20,74,-60,-77,-117,66,-36,115,24,34,-88,32 +63,4,-56,19,104,105,21,18,108,-104,63,50,-61,50,-11,21 +-105,-122,-67,-53,-7,73,11,-61,-71,24,57,-120,82,-31,55,-81 +-66,112,86,-2,48,-9,55,-49,-58,67,109,82,-84,-47,-15,117 +-8,32,-101,83,103,-10,-125,-69,32,-93,-51,8,6,-12,-10,-39 +-70,20,-52,48,-91,-108,26,-116,-117,8,123,-127,69,-35,56,76 +114,90,-80,38,5,43,67,115,-36,103,-56,49,-88,58,-45,-115 +4,-43,-33,114,20,113,-76,83,26,118,-106,-57,123,126,50,-82 +-72,-51,-53,-41,113,-126,-35,-63,65,-94,43,-123,-67,-59,-13,18 +30,93,-126,52,-114,-124,45,125,19,93,-64,-112,36,-34,-52,-100 +-45,30,102,-60,22,-82,74,115,-78,-15,-116,114,1,54,99,-26 +117,-117,-90,-74,35,-81,-85,-16,115,-71,112,102,-105,26,50,41 +119,126,76,-75,85,-74,86,-63,39,-14,113,-25,-17,61,69,-125 +48,64,-118,-97,-101,-79,105,-103,7,6,44,75,-128,-72,-27,-20 +-126,55,67,70,12,-59,-117,-10,-43,-126,75,58,20,-107,65,-15 +47,-37,32,32,-93,26,80,32,39,-1,-60,28,104,126,-22,-128 +53,-83,26,67,-17,-56,33,69,48,31,-60,-112,106,-8,8,109 +-65,-34,36,96,-49,110,38,59,20,-9,-104,111,81,46,-125,98 +-128,88,-124,-56,-47,3,58,-50,-38,-117,42,-88,-44,-128,103,-85 +-106,-93,29,-112,-13,-93,-12,25,73,-8,10,88,122,27,25,30 +16,-107,36,-12,90,-11,91,-103,103,126,122,-45,120,69,-64,-82 +-36,-5,58,80,-40,99,-73,-29,121,118,-39,109,111,7,83,-27 +-26,43,32,-76,30,-110,18,-103,-79,-26,-101,1,-36,-82,65,-15 +85,-117,83,53,-111,-96,-79,52,125,-112,100,118,82,56,-31,0 +65,73,-12,1,57,103,47,-100,-38,-27,48,-108,-2,-114,-68,-67 +26,-120,-37,-126,-101,14,24,-2,124,-31,-84,73,74,-77,-123,-75 +-34,100,-44,-5,-102,-98,57,90,29,98,15,-90,110,114,93,0 diff --git a/src/test/resources/data/test_vectors_binary_1000x128.json b/src/test/resources/data/test_vectors_binary_1000x128.json new file mode 100644 index 000000000..a9eab4098 --- /dev/null +++ b/src/test/resources/data/test_vectors_binary_1000x128.json @@ -0,0 +1,1000 @@ +{"id": 91240048, "vector": [-61, 47, -14, 41, -112, -101, -49, -58, -7, 127, -21, 126, -22, 121, 54, -58]} +{"id": 69136837, "vector": [18, 67, 100, -107, 79, 69, -50, -41, -119, 67, 35, -120, -35, -46, 38, 68]} +{"id": 48088883, "vector": [46, 41, -43, -102, 88, 120, 47, -120, -64, 19, -31, -84, 63, -16, -104, -64]} +{"id": 8729146, "vector": [10, -77, -121, -117, 32, 2, -127, -128, -105, -37, 15, -127, -51, -37, 2, -62]} +{"id": 4204661, "vector": [-121, -103, -42, -88, -98, -48, 56, 41, 82, -103, 126, 108, 96, -43, -45, 81]} +{"id": 83210802, "vector": [-64, -77, -7, 110, 125, 65, 78, 123, 99, -31, -54, 57, 74, 8, 8, 96]} +{"id": 3233569, "vector": [-83, 9, -29, -21, 68, 58, 124, 64, 118, -31, -120, -111, 36, 40, -16, 65]} +{"id": 53666583, "vector": [82, 93, -120, -66, -78, 58, 27, -94, 106, 124, 114, 99, 5, -41, -60, -33]} +{"id": 26275734, "vector": [113, -82, -57, 113, -12, -24, 22, 95, -51, -99, -50, -115, -97, -68, 97, -71]} +{"id": 89804152, "vector": [114, -98, 36, -55, 15, -55, 125, 51, 62, -105, 80, -107, 71, -48, 31, -15]} +{"id": 92803766, "vector": [-69, 126, 10, 30, -60, -10, -6, 117, -109, 107, -73, 30, -48, 101, 83, -27]} +{"id": 97057187, "vector": [59, 79, 51, 51, 20, 40, 37, -87, 61, 97, 20, -112, -111, -3, 114, -107]} +{"id": 76170907, "vector": [-81, -114, 30, 113, -127, 89, 58, 58, 118, 68, 24, -29, 38, 20, -28, -99]} +{"id": 65851721, "vector": [-113, -14, -93, -42, 125, 90, 1, 66, 7, -115, 56, 25, 73, 82, -126, 0]} +{"id": 168541, "vector": [57, 24, 50, -56, -78, -8, -31, 23, 102, 69, 60, 70, 114, -99, 98, -25]} +{"id": 73031054, "vector": [126, -38, 1, 6, -45, -31, 93, -33, -46, -23, -19, 20, -38, 83, 26, -2]} +{"id": 67793644, "vector": [63, -123, -109, -3, 10, 49, 125, -75, -45, 8, 86, 97, 16, -17, -96, 48]} +{"id": 99971982, "vector": [-128, -93, -49, -125, 64, -92, 65, 34, -91, -99, -45, -67, -98, -27, -103, 31]} +{"id": 12571310, "vector": [-7, 96, -110, -10, -26, -77, 117, 47, 76, -32, -78, -85, 23, 103, -30, -89]} +{"id": 75135448, "vector": [11, 36, -27, 23, 127, -73, 84, 122, -22, 22, -31, 81, -98, 121, -97, 33]} +{"id": 72793444, "vector": [21, 77, -107, 55, 112, -86, -125, -17, 0, -84, -79, -101, -24, 8, 127, -90]} +{"id": 98653983, "vector": [75, -39, -107, -111, 2, 91, -128, 117, 53, -99, 4, 108, -62, -53, 127, 124]} +{"id": 40677414, "vector": [-108, 30, 35, -45, -59, 78, -40, 105, -9, -82, 11, -2, 5, 48, -90, 33]} +{"id": 17058722, "vector": [-1, -48, -75, 27, -17, 40, -31, 60, -65, -93, 77, 71, 93, 17, -96, 124]} +{"id": 59109400, "vector": [38, 38, 17, 13, -44, -8, -32, 63, 2, 125, 75, -23, -14, 1, -117, 101]} +{"id": 58180653, "vector": [-119, 125, -125, -57, -5, 92, -25, -36, -117, 30, -51, 69, 86, 23, 92, -42]} +{"id": 90310261, "vector": [-101, 22, 56, 123, -95, -63, 50, 103, -114, -27, -3, -127, -47, 45, 122, -77]} +{"id": 42199455, "vector": [-120, 110, 30, -103, 63, -81, 76, 118, -1, 81, -123, 52, -80, 49, -115, 15]} +{"id": 38645117, "vector": [9, 103, -81, 98, 94, 53, 55, 50, -128, -59, 111, -3, 80, -18, -50, 62]} +{"id": 61897582, "vector": [82, -1, -33, -18, 86, 21, 84, -13, 103, 127, -67, 106, -104, 27, 86, 48]} +{"id": 59197747, "vector": [77, -9, -97, -74, 96, 29, -43, 123, 87, 42, 70, 73, -71, -38, -64, -45]} +{"id": 19486173, "vector": [22, 104, -98, 104, 118, 64, -22, 29, -28, -13, 63, 22, 4, 111, 79, 54]} +{"id": 24826575, "vector": [98, 113, 35, -118, -90, -82, -3, -2, 71, -50, -11, -97, -61, 65, 70, -120]} +{"id": 48360198, "vector": [92, -37, 32, -126, -5, 17, -77, -43, 40, 6, -86, 9, 118, 72, 13, 36]} +{"id": 55753905, "vector": [-103, -6, -6, -29, 29, -76, -19, -26, 48, 54, 0, 89, 19, 84, -64, 123]} +{"id": 50806615, "vector": [-24, 22, 111, -125, -77, -2, 41, -112, -11, 89, 58, 35, 33, 4, -41, 27]} +{"id": 97641116, "vector": [-41, 18, 36, -100, 67, 22, -70, 70, -87, 85, -3, 123, -114, 93, -127, -26]} +{"id": 30163921, "vector": [17, 123, 55, 109, 46, 109, -32, -96, -31, -11, -62, 124, 66, -62, 12, 54]} +{"id": 60176618, "vector": [-121, -23, 112, 70, -79, 91, -30, 35, -53, 23, -126, -93, -126, 90, -108, 113]} +{"id": 13470059, "vector": [6, -128, 102, 103, -116, -125, -96, 82, 127, -4, 107, 68, -86, 98, 4, -87]} +{"id": 55189057, "vector": [-63, 59, 49, 126, 72, -37, 27, 95, 49, -47, 122, 29, 104, -6, 100, -22]} +{"id": 82052050, "vector": [17, -75, -39, -44, -13, 63, -84, 69, -98, -62, 73, -104, 44, 59, 83, 94]} +{"id": 90158785, "vector": [-77, -82, -50, -89, -9, -37, -52, 23, -128, 122, -57, -102, 104, 19, 43, -80]} +{"id": 40197395, "vector": [76, 94, 85, 32, 15, 113, 114, -82, 12, 76, -77, -12, 33, -69, 90, -65]} +{"id": 23432750, "vector": [56, -25, 126, 60, 78, -5, -89, -78, 53, -66, -112, -32, 127, -86, 3, -28]} +{"id": 76825057, "vector": [-121, -87, -10, -38, -33, -122, 13, -45, 34, -97, -59, -19, -98, -126, 82, 63]} +{"id": 8115266, "vector": [-32, 9, -111, 85, -15, -19, -19, -117, -109, -115, 29, -116, 77, 59, 110, -89]} +{"id": 65880522, "vector": [-125, -21, -9, 50, -53, 83, -19, 103, 8, -54, 21, 12, -85, 101, 15, -127]} +{"id": 66832478, "vector": [-104, -1, -44, 50, -127, -16, -81, -94, -16, -119, 81, 40, -47, 67, 78, 93]} +{"id": 64602895, "vector": [-119, -11, -31, 81, 58, 66, -97, 19, -8, 80, 96, -9, -77, -44, 30, 20]} +{"id": 5482538, "vector": [84, 118, -48, 54, 122, -24, -119, 65, 72, 96, 70, 83, -100, -10, 95, 36]} +{"id": 53888755, "vector": [127, -80, 104, 111, 39, 67, 16, -22, 100, 89, -93, 36, 70, 2, 118, -70]} +{"id": 54987042, "vector": [-99, -30, -49, 68, -48, -49, -73, 58, -103, -9, -97, 46, -48, -25, -73, -108]} +{"id": 76434144, "vector": [17, 123, 103, 71, 1, 98, 17, 0, 126, -11, 10, 78, -32, 3, -59, 113]} +{"id": 88130087, "vector": [-28, 62, -75, -54, -48, -101, 49, -32, 82, 59, -107, 55, -6, -112, -16, 19]} +{"id": 78589145, "vector": [-64, -118, -64, 3, -2, -100, -27, -20, -95, 50, -59, -82, -126, 86, 0, 50]} +{"id": 64157906, "vector": [65, 119, -85, 30, -55, 75, 73, 112, 110, 6, 24, -86, -83, 69, 98, 121]} +{"id": 67031644, "vector": [-44, 59, 52, -69, 110, -120, 41, -23, 127, 106, -89, 6, -4, -87, -27, 40]} +{"id": 72278539, "vector": [20, 69, -4, 107, 74, 43, 27, 54, -59, 30, 98, -26, 91, 124, -2, 104]} +{"id": 13348726, "vector": [-64, -30, 93, 61, -126, -32, 124, -7, -62, -106, -36, 30, -80, 64, -61, -28]} +{"id": 44983451, "vector": [-103, 102, 93, -97, -67, 41, -68, -69, 15, -123, 79, 50, 101, -58, 119, 28]} +{"id": 91255408, "vector": [38, -67, 97, -26, 89, 64, 26, -13, -66, 33, -3, 93, 122, 43, 77, 56]} +{"id": 30090481, "vector": [10, -32, 38, -2, 52, 112, 67, -56, 86, -17, -21, -114, -57, -5, 19, -113]} +{"id": 669105, "vector": [-69, 112, -112, -103, 40, -53, 33, 105, -17, -68, -116, 41, -117, -112, 71, -15]} +{"id": 86821229, "vector": [-105, -102, 96, -88, -41, 108, 69, 114, -36, 102, 103, -30, 22, 60, -79, -123]} +{"id": 30543215, "vector": [-55, 68, 35, -59, -1, 121, -91, 46, -98, -59, -85, 77, -118, 3, -41, 26]} +{"id": 72725103, "vector": [-33, -74, 31, -114, -1, 71, -116, -107, 21, -47, -66, 29, 56, 127, 40, -97]} +{"id": 36685023, "vector": [79, 111, 111, -17, -10, 70, -128, -119, 52, 72, -126, 87, -2, 20, 28, 25]} +{"id": 4787945, "vector": [30, 95, 104, -32, 49, -31, -53, 42, 93, 4, -115, 15, 76, -116, -48, 47]} +{"id": 83651211, "vector": [108, 126, 80, 105, 116, 125, 39, -14, -50, -107, 98, -105, 106, -61, -106, -77]} +{"id": 76671482, "vector": [-119, -79, 79, -113, -76, -102, 63, 46, 90, -28, 93, -66, 45, 42, -83, 99]} +{"id": 66271566, "vector": [21, 123, -5, -38, 112, -60, 24, -69, 86, 27, -36, -114, -67, -119, 22, -61]} +{"id": 35512853, "vector": [-106, 1, 50, 120, 37, 29, 78, -67, -19, -11, -32, -121, -87, -96, 47, 71]} +{"id": 11543098, "vector": [-60, 101, 100, -64, 75, -60, 36, 33, -54, 2, -21, -38, -63, 31, -49, -38]} +{"id": 52293995, "vector": [87, -128, 127, 116, -94, -19, 93, 120, -26, -15, 35, 59, 93, 10, 100, -116]} +{"id": 51588015, "vector": [-50, 110, 86, -84, 83, 107, -32, 108, 20, 33, 86, -110, -10, 119, 26, -39]} +{"id": 99549373, "vector": [84, 120, 6, -119, 105, -125, -27, 122, 75, -63, 79, -85, -63, 60, 72, 57]} +{"id": 4806458, "vector": [-112, 67, 4, -82, 16, -95, -9, 97, 72, -62, -40, 1, -30, -112, -104, 124]} +{"id": 51149804, "vector": [60, -15, -93, -109, 73, -78, -48, 59, -64, -100, 76, -52, -77, -94, -126, -122]} +{"id": 45407418, "vector": [-6, -16, 91, -12, 122, -2, -67, 105, -127, -62, 39, -49, -28, -55, 95, -16]} +{"id": 85117092, "vector": [55, -19, -101, 46, -18, 33, 71, 32, -16, -45, 79, -122, -82, -28, 47, 53]} +{"id": 8913721, "vector": [-80, -4, 2, 89, 14, 96, 115, -124, -7, -86, 105, -38, 98, 15, -124, -49]} +{"id": 92787493, "vector": [-97, 107, 53, 103, -82, -106, -5, -29, -84, 66, 37, 43, 19, -91, 31, 97]} +{"id": 98648327, "vector": [113, 50, -114, -77, -28, 110, -62, 27, -54, -98, 95, -102, 53, -62, 32, 31]} +{"id": 4985896, "vector": [-82, -60, 26, -119, 99, -40, -115, 15, -70, -66, 56, 60, -18, 123, 67, -81]} +{"id": 22721500, "vector": [-73, -56, -90, -112, 61, 124, 68, 11, -60, 27, 101, 49, 66, -83, -51, -20]} +{"id": 14220886, "vector": [67, -94, 94, -13, -116, -74, 59, -110, 56, 10, -29, -95, -23, -10, -82, 112]} +{"id": 2717150, "vector": [2, 109, 38, 18, -10, 53, -23, 94, -114, -66, -20, -11, 68, -42, 57, 116]} +{"id": 6521313, "vector": [-19, -55, -108, 47, -10, -59, 25, 51, -112, 122, -103, 56, -48, 14, 121, -75]} +{"id": 68000591, "vector": [-17, -27, -4, 96, 56, 34, 31, -93, -18, 2, 83, -122, -46, -19, 88, -9]} +{"id": 77377183, "vector": [5, 111, -77, -104, -64, -86, 118, -112, -24, 127, 12, -90, -22, 53, -47, -3]} +{"id": 90090964, "vector": [-30, 35, 75, 105, -93, 1, -1, -97, -82, -89, 0, -79, -97, -105, 14, 104]} +{"id": 17764950, "vector": [-85, 64, 75, 95, 11, -79, -58, -20, 18, 43, 75, -67, -32, -28, 85, -112]} +{"id": 41092172, "vector": [83, -20, 109, -73, 13, -104, 62, 55, -4, 115, -46, -40, -16, -87, -124, 95]} +{"id": 20642888, "vector": [77, -54, -114, -19, -50, 28, 108, -29, -77, 102, -120, -65, 52, 44, 60, 57]} +{"id": 42307449, "vector": [93, -19, 61, 105, -47, -98, -104, -22, -15, 30, -68, -2, -127, 38, -83, 75]} +{"id": 34698463, "vector": [2, -82, -11, -1, -83, 120, -126, -84, -27, 127, -74, -44, -106, -87, -76, -61]} +{"id": 76703609, "vector": [25, 1, 109, 62, -111, -23, 32, -91, 108, 89, -34, -43, -76, 45, 35, -2]} +{"id": 22405120, "vector": [96, -101, 98, 113, -31, 0, -59, -88, 34, 95, 119, 37, 37, -110, 63, -95]} +{"id": 99729357, "vector": [68, 108, -2, 23, -122, 14, 78, -123, 33, -97, -24, 67, -53, 118, 85, 11]} +{"id": 48260151, "vector": [80, -33, -9, -40, -124, 103, -107, -77, -80, 21, 122, 45, 34, -89, 37, 91]} +{"id": 22108665, "vector": [-54, -65, -88, -36, 95, -118, 124, -122, 88, 101, -83, 43, -117, -5, -104, 108]} +{"id": 74743862, "vector": [-9, 55, -8, -23, 125, -49, -9, 41, -36, 78, 14, 51, -88, -71, -119, -102]} +{"id": 44550764, "vector": [-32, -62, -100, -119, 25, -86, -108, 107, -65, -21, 28, -88, 8, 14, -114, -28]} +{"id": 61928316, "vector": [-56, -119, 57, 52, -46, 24, -104, -78, -56, 4, -24, -107, 111, -95, 95, -20]} +{"id": 62552524, "vector": [88, 78, -75, 83, 41, -80, 12, 64, -108, -103, 11, 49, -52, 83, 13, 22]} +{"id": 45990383, "vector": [-121, -97, -106, 30, 15, 18, -36, 66, 53, -95, 6, -1, -37, 19, -5, -4]} +{"id": 11365791, "vector": [-46, -99, 97, -90, 56, 3, -26, 11, -74, -16, -96, -34, 56, 2, -93, -128]} +{"id": 82897371, "vector": [88, 17, -121, 120, 93, -45, -112, 69, 79, -10, -59, 121, -31, 102, 115, -37]} +{"id": 9575954, "vector": [-41, -71, 121, -57, -28, 122, 3, 12, -98, -94, 106, -50, 34, -1, -44, -56]} +{"id": 26063929, "vector": [-93, 52, 81, -27, 117, -127, 80, 83, 121, -20, -105, 46, 110, 73, 69, -73]} +{"id": 16405341, "vector": [-25, -57, 50, 83, -11, 103, 91, -93, 49, -56, -28, 23, 112, -27, 96, 98]} +{"id": 9058407, "vector": [-44, 101, 102, -21, -24, 57, -48, -77, -61, -35, 15, -7, 21, 16, -81, -71]} +{"id": 63506281, "vector": [-38, -84, 31, -107, -44, 61, -33, -45, -18, 103, 41, 26, -127, -73, 5, -19]} +{"id": 16019925, "vector": [23, -117, 121, 24, 35, -40, 85, 48, 65, -90, -91, -12, -40, -10, 40, -113]} +{"id": 48673079, "vector": [-15, -105, -93, 74, 87, -46, 35, -7, -1, -108, 60, 66, -93, -16, -113, -73]} +{"id": 4095116, "vector": [-65, 120, 39, -42, -25, -28, -16, -33, -44, 35, -37, -86, 79, -12, -122, 30]} +{"id": 76540481, "vector": [-70, 12, 79, 126, 84, 90, 12, 43, 39, -116, -89, -57, 45, 45, -35, -105]} +{"id": 70800879, "vector": [-122, -16, 26, 124, -56, 97, -82, -82, -43, 77, 58, 22, -27, 22, -23, 19]} +{"id": 92353856, "vector": [90, -95, 2, -118, 104, 115, -65, -8, -61, 99, -45, -109, 115, -115, 46, -16]} +{"id": 44784505, "vector": [-73, -30, -69, -13, -66, -110, -31, 26, 57, 18, -85, 52, 82, -77, 111, 10]} +{"id": 48893685, "vector": [-49, -62, 97, -18, -83, -73, 122, -105, 35, 42, -30, -95, -38, 4, 65, 107]} +{"id": 3235882, "vector": [-65, -41, 99, -86, -19, 75, 90, 119, -11, 69, 22, 100, 33, 84, 110, -117]} +{"id": 52613508, "vector": [-99, 125, 70, 95, 58, 55, -81, -79, -29, 6, 24, -44, -100, -127, -32, 21]} +{"id": 47213183, "vector": [66, -39, -8, -112, -32, 101, -28, 45, 30, -104, -13, -118, -14, -25, 104, -3]} +{"id": 69697787, "vector": [-71, 23, -100, -5, 58, 56, -84, 34, -127, 72, -63, -27, 119, 68, 6, -19]} +{"id": 8696647, "vector": [68, -104, -83, 35, 71, 75, -45, 73, 115, -68, -102, -27, -4, 41, 47, -98]} +{"id": 78561158, "vector": [-79, -49, -21, -82, -69, -4, -14, -76, -31, 18, -104, 49, -35, -94, 81, -52]} +{"id": 26102057, "vector": [81, -115, 94, -120, 94, -52, 89, -51, -67, -113, 94, 61, 52, 53, -24, 2]} +{"id": 66322959, "vector": [-45, 106, -117, -93, -28, 40, 30, 30, -32, 70, -87, 59, -31, 1, 90, -82]} +{"id": 54427233, "vector": [-24, -51, -95, -87, -15, -47, -13, -106, -33, 8, 126, 32, -33, -87, -33, 33]} +{"id": 51507425, "vector": [13, 28, -30, -102, -49, -5, -112, 102, -55, 5, -23, -53, 31, -11, -87, 58]} +{"id": 27411561, "vector": [110, 99, 4, -87, 92, 104, 27, 75, -92, 3, -55, 120, 9, 116, -28, 37]} +{"id": 85242963, "vector": [-26, -121, 79, -72, -19, -85, 126, -119, -21, -10, -54, -87, 82, -12, 64, 58]} +{"id": 874791, "vector": [-79, 7, 123, 110, -58, -45, -124, -105, 63, 71, -20, -30, 104, 14, 70, 5]} +{"id": 13468268, "vector": [-67, 31, -25, 103, -49, -33, 65, -112, -72, -91, -114, -21, -77, -19, 106, 102]} +{"id": 83533741, "vector": [-57, 66, -85, -77, 87, 8, 60, 104, -47, 33, 31, -6, -19, 89, 106, -25]} +{"id": 92071990, "vector": [90, 65, -73, -41, 26, -11, -101, -68, -104, -63, -53, -52, -30, 89, -62, 2]} +{"id": 68694897, "vector": [-34, 87, -75, -108, 68, -126, 5, -116, -54, 105, 94, 100, 50, -1, -38, 107]} +{"id": 69255765, "vector": [76, -50, -44, 50, 68, 49, -36, 70, -27, 65, 99, 81, 27, -109, 14, 127]} +{"id": 2917920, "vector": [95, -91, 101, 108, -5, -122, -9, 5, 54, 7, 106, -33, -85, 15, -39, -28]} +{"id": 75407347, "vector": [-15, -72, 95, 13, -100, 100, 121, 86, -4, -80, -77, -120, 0, 33, -125, -116]} +{"id": 8129978, "vector": [-97, 63, -17, 12, 123, 16, 103, 84, -110, 5, 33, -51, 86, 92, 61, 76]} +{"id": 30569392, "vector": [-33, 121, 6, -54, 85, 31, -35, 80, -23, -48, 50, -44, 61, 60, 104, 99]} +{"id": 66116458, "vector": [-10, -18, -70, -93, 46, -124, 13, 124, -4, 15, -49, -75, 119, -61, -50, -113]} +{"id": 86301513, "vector": [111, 102, -102, -26, 78, -113, 29, -16, 110, 70, 15, -42, 8, -17, -88, 18]} +{"id": 79942022, "vector": [79, -69, 77, -20, 16, -4, -127, -127, -120, 122, 58, 38, 116, -6, 39, -13]} +{"id": 80014588, "vector": [-105, -66, 105, -13, -84, -46, -5, 38, -55, 31, -36, 33, 24, 80, -89, 18]} +{"id": 79880247, "vector": [-44, 91, -31, 40, -115, 85, 77, 62, 59, 28, -52, -68, -88, 42, 7, -100]} +{"id": 35996293, "vector": [-62, -36, -41, 116, -74, -68, 91, 91, -122, 99, 127, 73, 68, -116, -89, -28]} +{"id": 84293052, "vector": [127, 91, 69, 88, -48, 74, 101, 49, -24, 99, -35, -118, -56, -63, 43, -90]} +{"id": 94516935, "vector": [71, -90, 32, -127, -28, 92, -94, 98, 118, -41, 45, -96, 118, 61, -93, 124]} +{"id": 14947650, "vector": [-47, 56, 23, -71, -117, -12, -80, -47, -95, -22, 75, -43, 69, 118, 55, 70]} +{"id": 73617245, "vector": [44, 54, 100, 23, 56, -111, -47, -1, -83, 2, -123, 23, 111, 105, -67, 73]} +{"id": 97940276, "vector": [46, 13, 99, 88, 119, -103, 47, 99, -34, -77, 96, -38, -5, 54, -16, 11]} +{"id": 45306070, "vector": [-98, 30, -123, -126, -56, -98, -1, -37, -121, 31, 52, -30, -100, 18, -56, 95]} +{"id": 67124282, "vector": [46, -114, -69, -14, -120, -48, 31, -30, 87, 101, -55, 26, -32, 43, -23, 23]} +{"id": 56515456, "vector": [-104, -76, 113, 21, -55, -27, -56, 106, -83, 109, 27, 2, -12, 88, -17, 110]} +{"id": 5111370, "vector": [75, -127, -13, -10, 36, 60, 119, -79, 94, 49, 105, 98, 14, -79, 121, 46]} +{"id": 62693428, "vector": [-58, 56, 93, 31, -80, -16, 118, 103, -110, -69, 95, 57, 63, -86, -52, -52]} +{"id": 56531125, "vector": [2, 98, 25, 103, 68, -70, -23, -42, -78, 23, 77, -95, -17, -124, -119, 91]} +{"id": 321665, "vector": [-83, -10, 105, -39, 55, 19, -114, -63, -46, -45, -25, 41, 86, 112, 98, -72]} +{"id": 9398733, "vector": [-119, -125, -75, -127, -90, 121, 86, -53, -47, 12, -74, 36, 16, 12, -98, -122]} +{"id": 89637706, "vector": [0, -74, -13, -67, -21, 85, -87, -18, 54, -14, 9, 104, 31, 118, 7, 35]} +{"id": 94911072, "vector": [12, 123, 121, 11, -31, -9, -25, -72, -35, 52, -36, -25, 85, -54, -105, -94]} +{"id": 62452034, "vector": [-115, -16, -49, -115, -63, 83, 62, 99, -44, 109, 35, -30, 13, -89, 47, 36]} +{"id": 63059024, "vector": [-66, 99, 120, 104, -19, -114, -113, -127, -121, -93, -11, 53, 78, -95, -41, 98]} +{"id": 7423788, "vector": [-114, 20, -121, -43, 105, -53, 12, -46, -30, 36, -33, -70, 72, 116, 117, -63]} +{"id": 43045786, "vector": [103, 70, -65, -102, -63, 41, -93, -88, 22, -89, 116, 22, -78, -7, -23, 15]} +{"id": 5640302, "vector": [48, 5, 115, -68, 80, 112, -104, 123, 77, 43, 61, -89, -128, -18, -22, -55]} +{"id": 68875490, "vector": [-7, -117, 45, -111, 114, 41, 9, -126, -36, 43, 88, -24, 103, 100, -42, -68]} +{"id": 10649306, "vector": [74, -28, 3, -124, -66, -102, -95, 13, -120, -86, 111, -57, -123, 83, -73, 93]} +{"id": 65017137, "vector": [-8, -96, -12, -24, -20, 18, 0, 113, -86, -20, -46, 109, -63, -112, 2, 90]} +{"id": 66137019, "vector": [31, -2, -39, 36, -108, -103, -57, -5, -106, -126, -93, -60, -113, -128, 6, 80]} +{"id": 84904436, "vector": [-18, 20, 115, 114, -50, -5, 66, -89, -42, 98, 110, 116, -40, -43, -52, -18]} +{"id": 55615885, "vector": [-24, 23, -77, -54, -82, -89, 45, 32, 101, -100, -112, 110, -54, 71, 78, 103]} +{"id": 30653863, "vector": [-114, -128, 0, -77, 48, 45, 78, -82, -43, 9, 65, 116, 115, 26, 15, 105]} +{"id": 91727510, "vector": [-37, 45, -27, 85, 60, 37, 100, -72, -120, -127, -74, 0, -85, -128, -87, -27]} +{"id": 2891150, "vector": [-72, 8, 11, 113, 55, -99, -104, 64, 28, 9, -53, 89, 28, 90, -65, -110]} +{"id": 75543508, "vector": [59, -62, -5, -6, 50, -110, 78, -50, 19, 62, 19, -78, 114, 47, 41, 86]} +{"id": 26139110, "vector": [96, 94, 41, -39, -127, 17, -122, -80, -5, 18, -29, 13, 52, 98, -48, 58]} +{"id": 65081429, "vector": [-62, -128, -32, -35, -118, -100, 74, 100, 85, 66, -127, 68, 0, 106, 107, -2]} +{"id": 98739783, "vector": [90, 24, 89, -60, -75, 122, 103, -106, -43, -81, 15, -39, 73, -72, 105, 81]} +{"id": 72358170, "vector": [108, -38, 53, 6, -70, 14, 113, 123, 27, 30, 4, -35, -112, 39, -9, -24]} +{"id": 99125126, "vector": [50, -1, -72, -14, -84, 11, 59, 125, 76, -51, -60, -41, 103, 85, 87, 114]} +{"id": 9886593, "vector": [66, 108, -37, -15, -127, 3, -21, 77, 112, -104, 78, 117, 119, 16, -126, -37]} +{"id": 44846932, "vector": [41, 18, -24, -100, -29, -17, -98, 52, -35, -66, 23, 40, 13, -22, -23, 82]} +{"id": 88904910, "vector": [-65, -93, -74, 35, -35, -15, -73, 34, 113, -18, 44, -93, -93, 119, -75, -118]} +{"id": 61141874, "vector": [79, -32, -19, 57, 86, 82, -113, -32, -59, 109, 19, 77, 112, 3, -128, -16]} +{"id": 39801351, "vector": [71, -65, 32, -66, 4, 100, 68, 83, -82, -47, 30, 102, 39, -107, 57, -122]} +{"id": 95395112, "vector": [-98, 125, -38, 10, 67, 19, 90, 90, 40, -124, 3, -111, 62, 57, 48, -109]} +{"id": 2331773, "vector": [18, -80, -94, -105, -104, -94, 86, -105, -29, -99, 66, -112, -88, -83, 101, 58]} +{"id": 59371804, "vector": [-7, 112, -24, -56, -36, -56, 26, -123, 27, -27, -127, -72, 54, 41, 84, -34]} +{"id": 80316608, "vector": [49, 75, 79, -36, -10, 87, -25, -65, 58, 112, 6, 14, -40, -94, -3, 114]} +{"id": 84002370, "vector": [68, -26, 14, 80, 54, -128, -45, 85, 74, -71, -63, -99, 16, -104, -13, -28]} +{"id": 22435353, "vector": [93, 17, -16, 70, 62, 76, -20, 95, -40, -119, -16, 11, 108, 82, 38, 89]} +{"id": 45790169, "vector": [-57, -18, 79, 62, 79, 123, -36, -125, 8, 108, 51, 73, 28, -31, 62, 11]} +{"id": 93053405, "vector": [59, 113, -126, -83, -56, 12, -109, -8, -5, -76, 97, -40, -15, -126, -88, -17]} +{"id": 4434662, "vector": [7, 1, -113, -48, 11, -128, 125, 60, 63, -69, -44, -90, -30, 31, 41, -42]} +{"id": 46851987, "vector": [-117, -120, 78, -36, 90, -108, -68, 92, -106, -87, -96, 23, 29, 56, -78, 51]} +{"id": 33797252, "vector": [-20, 48, -95, -17, 78, -17, 115, -59, 75, -120, -77, 0, 24, -12, 31, -111]} +{"id": 49598724, "vector": [25, 99, -12, 20, -22, -126, -5, -34, -61, -70, -20, 54, -80, 81, -72, 76]} +{"id": 68041839, "vector": [101, 72, 39, 84, -49, 28, -127, -9, -75, 87, -128, 41, -110, -77, 45, -102]} +{"id": 33336148, "vector": [-28, -91, 34, 106, 116, -127, -8, 107, -59, 36, 94, -26, -23, 24, -33, -57]} +{"id": 26493119, "vector": [-72, -105, 7, -74, 84, 79, -79, -10, 15, 81, 70, 124, 98, 58, -97, -100]} +{"id": 45667668, "vector": [-18, 85, 43, -93, -64, 96, 50, -109, 36, -84, 47, -10, 111, -56, -37, -13]} +{"id": 26507214, "vector": [76, -102, -16, -116, 55, -111, 97, 10, -27, -27, 99, -82, 0, -73, -23, -2]} +{"id": 4798568, "vector": [12, -56, 92, -83, -16, 1, 36, 7, 105, 3, -30, 104, -49, 89, 41, -121]} +{"id": 62740044, "vector": [-44, -83, -57, -121, -68, -35, -20, -28, -22, -37, -65, 54, 10, 107, 52, 45]} +{"id": 13819358, "vector": [38, -52, 37, -110, 45, 7, -113, -47, -80, 67, 25, -128, -96, -70, 62, 61]} +{"id": 91281584, "vector": [-4, -42, 61, -4, 86, 67, 86, -41, 27, 21, 46, 26, -23, 56, -68, 73]} +{"id": 92554120, "vector": [-122, -77, 109, 60, 24, 126, -23, 49, 121, -57, -44, 52, 122, -2, -1, -91]} +{"id": 44627776, "vector": [-116, 10, 32, 102, -12, 10, 56, -74, -73, -37, -62, 79, -28, 92, -57, 11]} +{"id": 36812683, "vector": [27, -42, 83, -82, 40, -87, -40, 98, 25, 121, 116, -55, 0, 111, -101, 124]} +{"id": 20765474, "vector": [103, 4, 94, -94, 66, -56, -1, 77, -118, 82, 88, -68, -123, -64, -93, -11]} +{"id": 29065964, "vector": [30, -20, -99, -77, -71, 16, -60, 77, -28, -45, -127, 105, -126, -44, 16, -9]} +{"id": 19457589, "vector": [96, 33, 46, 34, 6, 115, -12, -124, -37, -68, -18, -18, -67, -63, 95, -122]} +{"id": 42644903, "vector": [-47, -63, -41, 127, -113, 98, -57, 4, -38, -94, 69, -78, -3, 23, 82, -5]} +{"id": 36753250, "vector": [60, 11, -91, 103, -92, -56, 124, -65, -12, -28, 62, -55, -30, -83, 46, 119]} +{"id": 65275241, "vector": [-80, 26, -15, -1, 50, 113, 8, -6, -105, -128, -69, 100, -50, -128, 112, -99]} +{"id": 45049308, "vector": [66, -54, 49, 93, -116, 81, 45, 79, 18, -40, 126, 61, 17, 103, 49, -51]} +{"id": 16387575, "vector": [100, -86, -63, -37, -14, 4, 86, 102, -19, 112, 53, -22, -27, 89, 18, -84]} +{"id": 526217, "vector": [-57, -112, 63, -115, -72, 5, -18, 79, 114, -54, 67, -69, 96, -76, -49, 36]} +{"id": 38341669, "vector": [-2, -97, 30, 94, 18, 66, -31, -108, -27, -44, -12, -104, 32, 58, -58, -105]} +{"id": 31161687, "vector": [-5, -116, -6, -124, 52, -58, -27, 61, 20, 1, -105, -97, -92, -112, -86, -28]} +{"id": 57020481, "vector": [122, -57, 84, 54, -62, 79, 126, -121, 27, -71, -121, 75, -57, -53, -100, -101]} +{"id": 77300457, "vector": [-34, -121, -107, -98, -83, 127, 47, -124, 20, 74, 57, 108, -26, 72, -98, -86]} +{"id": 72614359, "vector": [57, -42, 80, 54, 124, -111, -91, 99, 35, 66, 85, -50, -58, -29, 47, -95]} +{"id": 45617087, "vector": [-107, -12, 17, -27, 56, -57, 87, -86, -64, 103, -64, 19, -44, 102, 47, 107]} +{"id": 97561745, "vector": [-18, 116, 15, 94, 70, 1, 43, 82, -1, 83, -59, 113, 13, 34, 12, 6]} +{"id": 97281847, "vector": [-98, -88, 28, -121, -77, 85, -14, -96, 37, -77, 114, 76, -79, 17, -81, -104]} +{"id": 62936963, "vector": [-80, 64, 58, 8, -78, -70, 0, 65, 59, -87, 0, -77, 40, 9, 8, 84]} +{"id": 44847298, "vector": [13, 19, 24, -41, 60, -97, 27, -107, 94, -117, -53, 5, 26, 114, -94, 0]} +{"id": 18001617, "vector": [4, -97, -114, 111, -100, -3, 66, 35, 58, -27, -95, -114, 69, 122, 31, -94]} +{"id": 36845587, "vector": [-100, 92, 78, 53, -83, -28, 44, -53, -84, 124, -53, -90, 27, 106, 71, -2]} +{"id": 80555751, "vector": [79, 12, -84, -118, -41, -89, 94, 1, -66, -24, 108, -79, 126, 109, 99, -69]} +{"id": 78909561, "vector": [-71, 106, -52, 101, 38, 25, 37, -39, 82, 115, 3, -123, -23, 93, -121, 83]} +{"id": 90272749, "vector": [74, -22, 97, -74, -76, 122, 79, 9, 33, -31, -124, 92, 58, -37, -42, 111]} +{"id": 30139692, "vector": [81, -21, -102, 18, -6, 70, 42, -32, 114, -14, 61, 104, 59, -112, -56, 17]} +{"id": 29516592, "vector": [49, -51, 43, 25, -49, -10, -24, 24, 6, 22, 116, 32, -115, -24, -24, 89]} +{"id": 36505482, "vector": [124, 105, -61, 66, -111, -122, -116, -11, 104, -102, -63, 80, 82, 48, 82, 18]} +{"id": 44473167, "vector": [-57, -46, 76, 89, -112, -42, -99, -56, 6, -18, -95, -71, 68, -117, 66, 34]} +{"id": 37739481, "vector": [-70, -82, -53, 70, -74, 105, 11, 119, 75, -81, 32, 61, -54, -62, -88, 20]} +{"id": 8373289, "vector": [-32, -46, -67, -59, 61, -63, -122, -66, -117, 6, 106, 82, -87, 10, -22, 55]} +{"id": 35192533, "vector": [65, -30, 21, -117, -9, -94, 127, 28, -108, -35, 7, 100, -65, -15, -80, -60]} +{"id": 62051033, "vector": [-123, 63, 61, 40, 76, 2, -3, 71, -9, 89, 68, -57, 119, -121, -82, -114]} +{"id": 92867155, "vector": [80, 17, -24, 113, 54, -119, 12, -38, 105, -111, 53, -35, 51, 39, -68, 92]} +{"id": 62803858, "vector": [66, 28, -109, 126, -48, -34, -108, 15, -87, 90, 99, 74, -124, 122, 85, 59]} +{"id": 95251277, "vector": [63, -104, 1, -108, -119, 76, -46, -92, 79, 107, -124, -90, 31, 96, -121, 18]} +{"id": 65038678, "vector": [-37, -6, -71, -7, -53, 71, 70, 90, 78, -116, 1, -26, 90, -101, -31, -4]} +{"id": 73168565, "vector": [65, -18, -10, 92, -22, 9, -5, 29, 122, 109, 85, 107, 37, 13, -75, -6]} +{"id": 75777973, "vector": [-89, -54, 50, -24, -76, 69, -23, 14, -89, 121, -1, 14, -53, -25, 113, 115]} +{"id": 18833224, "vector": [33, -23, 41, -110, 50, -85, -32, 82, 19, 10, 91, -103, 5, -40, 85, 116]} +{"id": 43506672, "vector": [-23, 110, 113, -36, -121, 123, 4, -114, 45, -33, -12, 71, 122, 116, -88, 118]} +{"id": 61815223, "vector": [-34, -38, 49, 59, 71, 26, -37, -110, -56, -95, -2, 21, -115, 86, 104, 33]} +{"id": 30463802, "vector": [123, -106, 86, 117, -29, 23, -19, -49, -102, 124, -110, -39, 33, 121, -40, 83]} +{"id": 8634541, "vector": [122, 67, 75, 55, -115, 81, 58, 97, 96, -19, -38, 78, 78, -119, -90, 111]} +{"id": 57241521, "vector": [-3, -3, 116, 39, -81, -125, 37, -67, -79, 50, -98, -65, 40, -61, 123, -126]} +{"id": 7955293, "vector": [-20, -11, -115, 47, -24, 56, 68, 96, 9, 127, 102, 88, -107, -11, 38, -28]} +{"id": 33431960, "vector": [16, 44, -64, -110, 111, 70, -108, 44, 84, 114, -72, 97, -48, 85, -117, -126]} +{"id": 94090109, "vector": [-13, -5, -52, 72, 122, -92, 80, -63, -123, -121, 72, 45, 66, -128, 110, -21]} +{"id": 19101477, "vector": [-47, 80, 16, -9, 40, -41, 2, -81, 24, 92, -73, -1, 69, 16, 67, 46]} +{"id": 61741594, "vector": [-82, 38, 6, 88, -45, 46, -83, 95, 34, 89, 54, 61, 89, 78, -18, -79]} +{"id": 43501211, "vector": [30, 12, 96, 69, -81, -67, -5, -89, -30, -122, -67, 71, -76, -42, 24, -126]} +{"id": 61728685, "vector": [126, -100, -122, -124, 104, -19, 85, 9, -27, 56, 65, -100, -73, -35, -114, -103]} +{"id": 29932657, "vector": [0, -65, -19, 20, 41, 89, 118, 103, 109, -31, 50, -93, -24, 24, 28, -108]} +{"id": 55850790, "vector": [115, -118, -58, 107, -107, 66, -101, -73, -110, 72, 36, -111, 43, -58, 11, 127]} +{"id": 74614639, "vector": [-116, -97, 79, -104, -124, -66, 55, 3, -79, 11, -38, -28, -110, 22, 90, 122]} +{"id": 61960400, "vector": [-62, -102, -62, -10, 73, 70, -121, -75, 32, 81, 75, -54, 11, -68, 35, 16]} +{"id": 71083565, "vector": [-52, -102, 37, 38, 28, 78, 50, -54, 101, 67, -106, -79, 127, -3, -75, -12]} +{"id": 60581278, "vector": [63, 19, 30, -128, 31, -126, -70, 6, -78, 67, -106, 57, -5, 90, 47, 84]} +{"id": 54517921, "vector": [-2, -101, -123, -13, 25, -78, -115, -30, 5, 92, -87, -91, 42, -93, 60, 9]} +{"id": 48892201, "vector": [91, -71, 64, -34, 11, 32, -89, -36, 40, 100, 112, -106, -127, 17, -89, -120]} +{"id": 80251430, "vector": [86, 32, 0, 73, 126, 116, -125, 3, -83, 116, -71, 80, -18, 24, 104, 98]} +{"id": 13173644, "vector": [124, -72, -85, 85, 39, -110, -42, 125, -60, 19, -120, 12, -108, 21, 69, -34]} +{"id": 415901, "vector": [76, 100, 77, 51, 77, 48, 62, -19, 81, -38, 57, -23, 41, -108, -23, -15]} +{"id": 78218845, "vector": [-89, -78, -30, -27, -86, 1, 30, -76, -62, -64, -11, -49, 20, 57, -10, -77]} +{"id": 1239555, "vector": [22, -98, -28, -112, -5, 105, -85, -46, -60, 73, 84, 22, -85, 125, 29, 22]} +{"id": 83948335, "vector": [-26, 9, 89, -50, -75, -1, 52, 120, -126, -24, -48, -83, -83, 70, -112, 9]} +{"id": 38365584, "vector": [-2, 9, 33, -117, -32, -64, 105, 66, -87, -59, -2, -99, 104, 8, 8, 44]} +{"id": 91957544, "vector": [66, 93, 108, -40, -124, 115, -17, 7, 36, -62, 87, 9, 21, -79, -99, -84]} +{"id": 32274392, "vector": [-94, 66, -30, -59, -86, -125, -103, 60, -68, -8, -6, 6, 20, -88, 52, -22]} +{"id": 45075471, "vector": [-19, -93, 76, 86, 31, -111, -27, 73, -22, -18, 98, -3, -81, 89, 55, -35]} +{"id": 39553046, "vector": [57, 97, -57, 42, -86, 10, -6, -114, 35, 126, 66, -102, -108, 55, 16, 92]} +{"id": 21289531, "vector": [-87, 70, 1, 41, 45, -30, -6, 9, -68, 35, -37, 127, 32, 106, 38, -25]} +{"id": 82886381, "vector": [-30, -42, -119, -114, 63, -25, 11, -118, -25, 58, 72, 88, -110, -6, -90, -90]} +{"id": 83368048, "vector": [-77, -46, -9, 39, -107, -74, -38, 101, 91, -89, -27, 91, -118, 19, 35, 71]} +{"id": 33123618, "vector": [-73, -62, -113, -105, -62, 106, -2, 124, -29, 76, 49, -106, -2, 44, 71, 93]} +{"id": 79191827, "vector": [-125, 36, 11, -52, 25, 114, -93, 105, -22, 93, -83, 95, 84, -5, 23, 82]} +{"id": 92530431, "vector": [-13, -12, 24, -1, 13, -69, -37, -88, 109, -111, 13, -126, 59, -94, -82, -106]} +{"id": 19891772, "vector": [22, 63, 36, 62, 66, -120, -86, 123, -95, -11, 126, 111, -125, -13, -102, 67]} +{"id": 32590267, "vector": [77, -24, -8, -86, 75, 85, 42, 46, -105, 65, -58, 115, -116, 52, 15, 33]} +{"id": 29510992, "vector": [-125, -51, -55, -15, 123, 88, -7, 124, 60, 2, -69, -110, -44, -66, 4, 14]} +{"id": 49597667, "vector": [-122, -96, -116, -80, -32, 108, 76, 59, 38, 77, 66, -25, 44, 25, 8, -100]} +{"id": 73124510, "vector": [56, -128, -40, -57, 17, 48, 32, 65, -30, 124, 47, 77, -78, -41, -16, 83]} +{"id": 69579137, "vector": [118, -71, 11, 32, -17, -71, -100, -58, -71, 56, -117, -96, 37, -26, 72, -51]} +{"id": 37560164, "vector": [5, -50, -71, 37, 62, -40, -39, 50, -99, 119, 64, -69, 49, 78, 36, -48]} +{"id": 41380093, "vector": [-18, -90, -5, -81, 16, 75, -54, 104, -57, -105, 115, 108, 126, 116, 35, 62]} +{"id": 92692283, "vector": [-19, 101, -43, 43, 89, 121, -123, 49, -36, -41, -54, 15, -4, 70, 115, -83]} +{"id": 31904591, "vector": [-8, -118, -11, -101, -66, 6, -9, -120, 53, -79, 119, -85, -44, -55, -91, -72]} +{"id": 47887470, "vector": [112, -119, -92, -24, -12, 59, -81, -9, 98, -117, 4, -126, -97, -65, 75, -60]} +{"id": 17016156, "vector": [58, 119, 44, -123, -21, 68, 123, 35, -50, -7, -36, -62, -82, 26, 24, 51]} +{"id": 93566986, "vector": [-66, 92, -92, -33, 21, -89, -61, -31, -1, 11, -9, -125, -21, 98, -103, 49]} +{"id": 29959549, "vector": [112, 0, 7, -85, -31, -92, -78, 3, 75, -18, 43, 102, 115, -22, 43, -102]} +{"id": 99917599, "vector": [-29, -57, -97, 13, -21, 27, -118, -60, -81, -83, -62, -108, -126, -63, 84, 81]} +{"id": 77301523, "vector": [74, 92, 73, 11, -13, 40, -71, 85, 61, 8, 44, -106, -2, 83, -92, -108]} +{"id": 1022677, "vector": [-117, -119, 51, -90, -47, 51, 40, -91, -3, 109, -80, 4, 50, 18, -10, 46]} +{"id": 61263068, "vector": [-128, 65, -71, -103, 117, -111, 55, 52, 77, -125, 116, -48, -18, 89, 16, -12]} +{"id": 72738685, "vector": [43, 67, 77, -74, 17, 17, -44, 54, 53, 17, 79, 64, 88, 56, -78, 57]} +{"id": 77620120, "vector": [116, 29, -53, 14, 63, 119, -108, -6, 58, 116, -77, 53, 7, -64, -8, -114]} +{"id": 23110625, "vector": [-19, -119, 30, -20, 84, 59, -57, -80, -67, -46, 64, 59, 86, 68, -21, 106]} +{"id": 92541302, "vector": [-28, 61, 6, 106, 122, -100, 12, -24, -31, 126, -42, -32, -53, 43, 66, 88]} +{"id": 47361209, "vector": [-77, 13, 84, 54, -84, -108, -59, -117, 55, -57, -64, -71, 77, 78, 19, -57]} +{"id": 17539625, "vector": [79, 60, -50, -117, 90, 8, 109, -83, 125, -45, 111, 87, 114, 119, -22, -57]} +{"id": 75986488, "vector": [126, -9, 119, -46, -105, 55, -91, -20, 67, 87, -27, 47, 6, 33, 7, 93]} +{"id": 34295794, "vector": [-43, -71, -95, 39, -68, -54, -87, 75, 113, 94, -29, 90, -105, -90, -118, -18]} +{"id": 15535065, "vector": [76, -70, 74, 23, 122, 5, 72, -64, 112, 21, 112, -122, -13, 52, -104, 101]} +{"id": 37675718, "vector": [-86, -40, 77, 21, -15, 55, 91, 14, -36, -107, -109, 72, 73, 96, -57, 36]} +{"id": 77312810, "vector": [-23, 78, 75, 63, -37, 113, -91, 10, 30, 126, 67, -61, -7, 75, -61, 72]} +{"id": 569864, "vector": [-80, 2, -3, -111, -51, 3, -94, -80, 98, -18, -58, -107, -19, 93, -61, -83]} +{"id": 89217461, "vector": [-22, 68, -63, -97, -13, 19, -47, 55, 22, 24, 32, -59, 46, 123, -104, -45]} +{"id": 82339363, "vector": [-105, 7, 60, -11, 91, -88, 123, 20, 14, 35, 38, -108, -105, 90, 66, -4]} +{"id": 68824981, "vector": [-10, -47, 65, 12, -51, 25, 110, -44, -65, 10, 107, 96, 86, 28, 50, 89]} +{"id": 33553959, "vector": [-110, 84, -20, -48, 112, 111, -70, 105, -88, -23, 41, 118, 92, -70, 73, -47]} +{"id": 82979980, "vector": [-80, -76, -73, 2, -43, 82, 76, -45, -60, -14, -34, 113, -61, 95, -58, -69]} +{"id": 44481640, "vector": [103, 57, -38, -12, -88, -57, 70, 83, 74, -62, 103, 70, -5, -105, 115, 48]} +{"id": 21533347, "vector": [103, -81, -41, 95, 54, -52, -105, 78, -11, 36, -3, -85, 21, 98, 40, -62]} +{"id": 58208470, "vector": [34, 56, 34, -5, -78, 78, -110, -90, 21, 46, 83, -85, 118, 76, 27, -117]} +{"id": 92215320, "vector": [53, 81, -122, -94, 32, -100, -9, -83, -44, 90, -43, -50, -13, 52, -66, -78]} +{"id": 13231279, "vector": [118, 66, -17, 52, 79, 125, -85, 118, 13, 111, -52, -4, 106, 53, -44, -102]} +{"id": 45428665, "vector": [102, -82, 35, 35, 97, -47, -58, -5, 53, 56, -16, 117, 8, -9, -35, -77]} +{"id": 57961282, "vector": [123, 124, -45, 34, 89, -113, 68, 99, 85, -49, 124, -55, 44, 29, 47, -70]} +{"id": 29188588, "vector": [-64, 99, 87, 48, 75, -49, -69, -102, 57, -11, -76, 95, 48, -15, -98, -123]} +{"id": 81855445, "vector": [-70, -58, -112, -100, 97, 0, -13, 26, -10, -81, 110, -41, -44, -113, -94, -29]} +{"id": 39847321, "vector": [-90, 30, -67, 55, 116, -82, -4, 38, 98, 32, 21, 21, -17, 76, -56, -52]} +{"id": 37183543, "vector": [-44, -31, 52, -123, -102, 39, -1, -87, 93, 43, 116, -74, -12, 101, -26, -72]} +{"id": 32161669, "vector": [48, -40, 96, 25, 52, -30, 12, 42, -70, -74, -7, -3, 48, 24, -65, 10]} +{"id": 93933709, "vector": [75, -23, 113, -63, 93, -76, -109, -19, 78, 119, -1, 113, -9, 87, -81, 2]} +{"id": 23134715, "vector": [102, -36, 20, 34, -46, 94, -4, 53, -87, -67, 80, -123, -14, -29, -15, -78]} +{"id": 42967683, "vector": [72, -64, 121, -76, -108, -118, 23, -100, 117, -63, -75, 96, 22, -3, -64, 105]} +{"id": 15148031, "vector": [68, -66, 99, -16, -127, -53, 8, 107, -48, 81, -23, -31, -80, 104, -29, -91]} +{"id": 24733232, "vector": [-118, 60, 27, 82, -89, 42, -53, 121, -80, -65, -91, 100, -36, -126, -19, 64]} +{"id": 26664538, "vector": [-53, 44, -57, -83, -65, 81, 65, -73, 112, 69, 103, -100, 74, -28, 66, -20]} +{"id": 86543538, "vector": [55, -65, -112, -89, 15, -39, 126, -49, 91, -14, -94, 56, 88, -54, -37, 67]} +{"id": 6793819, "vector": [23, -22, -43, -84, 104, -24, 112, -99, -107, -25, 14, -24, 44, 23, 54, -92]} +{"id": 29635537, "vector": [-92, 43, 39, -38, -120, -9, -70, -75, 69, 86, 61, -111, 112, -120, 78, -125]} +{"id": 3509435, "vector": [-92, 100, 29, 75, 118, -107, 3, 92, 26, 22, -59, -37, 42, -64, 69, 12]} +{"id": 75820087, "vector": [-110, -73, -92, 8, 116, 120, -84, -65, 20, 92, -111, -109, -114, -15, -90, -29]} +{"id": 57163802, "vector": [51, -91, -33, 52, -73, -51, -109, 57, 41, 42, -89, -40, -126, -1, 33, 101]} +{"id": 9175338, "vector": [-100, -57, 118, -63, 53, -70, -108, 119, 18, 4, -84, 38, -30, 81, -25, 58]} +{"id": 44348328, "vector": [64, 100, 79, 77, 109, 125, -92, 101, 87, 65, 48, 43, -33, -74, 24, 28]} +{"id": 85116755, "vector": [-12, 98, -82, -16, 40, 29, 16, -42, -89, -61, -50, 101, 14, 4, 91, -4]} +{"id": 69641513, "vector": [36, 48, 9, 110, 112, -26, 82, 108, 18, 80, 35, -28, -87, -108, 74, 3]} +{"id": 6111563, "vector": [105, -36, 38, -112, -81, -54, -76, 78, 102, -43, 16, 24, 1, -32, 104, -89]} +{"id": 16097038, "vector": [40, 112, -16, -43, 83, -88, 88, 14, 70, -65, 68, -79, 49, 116, 29, -21]} +{"id": 94360702, "vector": [110, 126, 63, -39, 22, -86, 94, 11, 2, 90, -18, 99, 123, -61, -13, -79]} +{"id": 61859581, "vector": [103, 31, 110, 127, -31, -119, 23, 49, 17, -122, -118, -75, 30, -40, -35, -93]} +{"id": 82327024, "vector": [98, -120, 123, -23, 103, 82, -97, -111, 23, -123, -114, -21, 103, -89, -126, 124]} +{"id": 44842615, "vector": [60, -119, -122, -3, -119, 64, 72, -47, 119, -110, -25, 1, -3, -94, 100, -14]} +{"id": 9599614, "vector": [-104, -16, 78, 19, -49, 106, -45, -92, -26, -30, 38, 7, 83, -14, 21, 23]} +{"id": 43933006, "vector": [-14, 45, -31, -106, 78, 78, -76, 47, -17, -6, -103, 50, 74, -62, 104, 64]} +{"id": 93057697, "vector": [79, 48, 56, -78, 75, 29, -122, -70, -98, -7, 90, -75, -28, 6, 12, 76]} +{"id": 9188443, "vector": [-28, 96, 73, 24, 81, 63, 12, -71, -121, 50, -108, -102, 23, -92, 77, -13]} +{"id": 84684495, "vector": [67, 33, -36, 61, 125, 76, -29, -101, 39, -43, -61, 42, -71, -97, -77, -57]} +{"id": 96852321, "vector": [2, 74, 13, 18, 14, 21, 93, -69, -128, -79, -39, -5, 35, -124, 31, 6]} +{"id": 98462867, "vector": [-25, -71, 67, 96, 65, 52, -24, 107, -39, -65, -101, 16, 126, 24, 72, -20]} +{"id": 16099750, "vector": [-115, 127, -1, 45, 35, -50, -9, 6, 74, -81, -92, 30, 72, -69, 31, -98]} +{"id": 18411915, "vector": [-110, -11, -118, -83, -125, 110, 69, 93, -57, 92, 111, 58, 33, -7, 66, -69]} +{"id": 98371444, "vector": [66, -3, 81, 119, 103, -13, -104, 91, -68, 85, -96, 16, -87, 93, 38, -45]} +{"id": 88698958, "vector": [-40, -25, 115, 121, -49, 50, 22, 59, 17, -23, -28, -30, 106, 92, 29, -82]} +{"id": 48658605, "vector": [-33, -123, 9, -106, 78, -17, -110, -112, -14, -25, 94, 98, -122, 42, 76, 65]} +{"id": 23848565, "vector": [-55, -9, 116, -106, -63, 48, -116, 114, 118, -24, 63, -85, 65, 33, -62, -2]} +{"id": 39171895, "vector": [58, -125, 92, 127, 91, 105, -44, -10, -17, -9, -59, -9, -2, 56, -61, 68]} +{"id": 5267545, "vector": [109, 45, 65, -43, -26, -81, 39, -27, 44, 84, 31, -124, -93, 64, 79, 99]} +{"id": 84349107, "vector": [-17, -92, -103, 77, 58, 98, 51, 9, -2, 71, 85, 3, 23, -16, 104, 120]} +{"id": 36139942, "vector": [85, 61, 60, 65, -111, -92, -2, 32, -74, 117, -28, -110, -82, 48, 20, 80]} +{"id": 7300047, "vector": [-3, -116, -111, -75, 24, -105, -116, 97, -126, 37, 28, -78, -29, 30, -42, 100]} +{"id": 60102965, "vector": [25, -28, -67, -18, -84, -14, -21, -37, -30, -77, 67, 41, 23, -70, -87, -31]} +{"id": 19376156, "vector": [-109, 33, 125, 43, -32, 32, -56, -34, -89, -56, -66, -55, -125, 29, 3, 97]} +{"id": 10366309, "vector": [118, -61, 1, 80, 107, -101, -63, 70, -121, -103, -63, -40, -105, 36, 39, -123]} +{"id": 26863229, "vector": [-117, -37, -79, -8, -36, -78, 121, -98, 99, -102, 67, 120, -40, -97, -42, -92]} +{"id": 7685448, "vector": [-114, -101, -35, -61, 65, 119, 125, -113, 35, 119, 16, -124, 26, 23, -124, 56]} +{"id": 86001008, "vector": [48, -50, 105, -1, -46, -14, 16, -47, 110, -49, 48, -43, -114, -30, 76, 70]} +{"id": 51047803, "vector": [-5, 19, 18, -8, -11, -65, -28, 54, -90, -92, 13, 47, -84, 127, -56, 2]} +{"id": 71300104, "vector": [113, 40, -24, 104, -14, -42, 86, -92, -32, 70, -117, 79, 116, 56, 57, 53]} +{"id": 89214616, "vector": [-46, -114, 94, -41, 60, -37, -66, 4, 23, 21, 120, -56, -79, -108, -109, 11]} +{"id": 92604458, "vector": [-9, 76, -2, -6, -113, -126, -125, 44, -57, 105, -125, -24, 9, -23, -99, 39]} +{"id": 96906410, "vector": [67, -122, -102, -21, 88, 83, -49, 64, -123, 80, 81, 23, -14, -119, 93, 50]} +{"id": 54868730, "vector": [70, -99, 2, -125, -42, -45, -88, 52, -12, 31, -17, -104, -42, -26, -96, 27]} +{"id": 99690194, "vector": [41, 125, 99, -101, 7, 3, -104, -3, 62, -58, 23, -2, 107, -61, 103, 91]} +{"id": 19939935, "vector": [72, -33, -110, -114, 109, -59, 116, 4, 104, 99, 22, -27, 61, -75, 55, -104]} +{"id": 3233084, "vector": [-122, 116, 51, 59, 70, 9, -114, 50, -77, -15, -121, -39, 30, 75, -49, 72]} +{"id": 25842078, "vector": [-116, -4, 33, 110, 5, -64, -96, 60, -63, -18, 66, 123, 61, 30, 57, 16]} +{"id": 69786756, "vector": [-17, -26, -90, 67, -53, -38, 87, 79, 127, -5, 85, -9, 77, 73, -60, 5]} +{"id": 17957593, "vector": [45, -17, 109, 80, -44, -70, 20, -36, 20, 51, -91, -47, 93, 50, 106, -53]} +{"id": 83133790, "vector": [-66, -39, 94, 75, 39, 6, -109, -109, -128, -44, -74, -114, -39, 12, -6, -109]} +{"id": 71965942, "vector": [51, 3, 88, 91, -111, -111, 60, 88, 32, 57, -44, 81, -4, -85, -8, -76]} +{"id": 70372191, "vector": [-125, 28, 99, -58, 41, -96, -22, -52, -4, 71, 120, -30, 38, -12, -91, -127]} +{"id": 59177718, "vector": [124, 126, -73, -37, -28, -126, -67, 42, 26, -29, -9, 2, 47, -123, 30, 86]} +{"id": 38510840, "vector": [-8, 67, 69, -90, -36, 125, 38, -104, -115, -123, -12, -14, 73, -66, 116, 32]} +{"id": 21001913, "vector": [-52, 12, -13, -92, 55, 31, 88, -57, -79, 121, 20, 4, 116, 72, 122, 44]} +{"id": 32426752, "vector": [-8, 66, 9, -26, 116, 60, -76, -36, 106, -29, 119, -23, -33, -15, -35, -71]} +{"id": 27185644, "vector": [23, -59, 39, -114, 29, -14, 0, 47, 92, -16, 104, 92, -69, 107, -122, 33]} +{"id": 42073124, "vector": [16, 83, 74, -61, -85, -24, 87, 41, -105, -85, 99, -97, 16, -60, 98, -66]} +{"id": 90013093, "vector": [60, 9, 0, -64, 49, -96, 74, 48, 97, 73, -93, -80, -27, 57, -4, 19]} +{"id": 33304202, "vector": [-38, -27, -32, 57, -25, 5, -115, 92, -15, -41, -112, -21, -15, 106, 60, -87]} +{"id": 112651, "vector": [36, 20, -57, 61, -23, 43, -10, 86, 119, -20, 53, 13, 108, 118, -29, -82]} +{"id": 35092039, "vector": [0, -120, -7, -48, -92, -61, 69, 85, 30, -69, -79, -121, 59, 84, 64, 81]} +{"id": 63967300, "vector": [-1, -103, -93, 81, 109, -67, -21, 74, -67, -87, -100, -114, 77, -66, 47, -106]} +{"id": 72274002, "vector": [100, 47, -45, -50, -68, 96, -10, -79, 54, -43, 43, -80, -71, 118, -113, -111]} +{"id": 33249630, "vector": [-25, -86, -57, 24, -111, -89, -2, 102, -79, -39, 83, -85, 44, 8, 12, 125]} +{"id": 92998591, "vector": [-3, -46, 120, -78, -78, -39, 44, 123, 117, -128, -18, -35, -100, 123, -111, -66]} +{"id": 12303248, "vector": [-19, -119, 76, 69, 107, 12, -9, 84, -117, 86, 7, -42, 107, -22, -35, 101]} +{"id": 22766820, "vector": [20, -44, -60, 97, -24, -105, 101, 16, 51, 113, -8, -118, -127, -63, 78, -82]} +{"id": 17894977, "vector": [105, -116, -126, 70, 73, -105, -87, -123, -42, 60, -23, 33, -115, 34, -25, 111]} +{"id": 43322743, "vector": [-98, 55, -92, -100, -46, 86, -91, 59, -34, -61, 81, 79, -10, 62, -7, -68]} +{"id": 42692881, "vector": [89, 60, 12, 95, 105, 71, 93, -33, 17, 21, -32, -127, 53, -110, -89, -64]} +{"id": 17068582, "vector": [99, -1, -84, -28, 105, 93, 108, -101, -58, 46, -38, 57, -7, -1, 7, -5]} +{"id": 43357947, "vector": [51, 4, 62, 94, -70, 105, 117, 46, 30, 11, -48, -6, -52, -53, -119, -117]} +{"id": 76315420, "vector": [-1, 51, -126, 65, -67, -91, -91, 36, 53, 113, -122, 121, -105, -24, -85, -42]} +{"id": 8099994, "vector": [-40, -65, 66, 10, 104, 47, -20, -65, 25, -43, -33, -62, -29, 68, 119, 84]} +{"id": 66885828, "vector": [-48, 51, -126, 115, -101, 1, 50, -72, -40, 40, -114, -46, 52, -118, 105, -17]} +{"id": 5073754, "vector": [115, -65, -94, 58, -84, 14, -40, -34, 46, 32, -71, -6, 51, -8, 85, 35]} +{"id": 16445503, "vector": [-82, 99, 127, -70, -128, 33, -122, 36, 0, -67, -57, -79, -92, 81, -78, 100]} +{"id": 77694700, "vector": [-13, -82, 85, 7, 12, -127, -37, 29, -62, -87, 115, 72, 32, 39, 27, -83]} +{"id": 90457870, "vector": [-94, -111, -122, -67, 56, -67, -96, -27, -95, -47, 73, 111, 0, -55, 31, -2]} +{"id": 27187213, "vector": [123, -46, -7, 38, 49, 18, -90, 2, -78, -79, -27, -1, -125, 70, 42, 49]} +{"id": 51715482, "vector": [-51, 108, -90, -114, 0, 68, 57, 65, 124, 42, -55, 95, -105, 14, -70, -9]} +{"id": 22450468, "vector": [-95, 37, -105, -20, 81, -81, 79, -72, 92, 40, -20, -55, -107, 10, 45, 43]} +{"id": 93359396, "vector": [28, 54, 56, 84, -107, -61, -104, -128, -79, 76, -88, 125, 57, -7, -107, -112]} +{"id": 87160386, "vector": [-37, -102, 114, 37, 82, -49, 88, 3, 78, 46, 0, 85, -41, 102, -77, -94]} +{"id": 12348118, "vector": [-76, 24, -119, -69, 86, -24, -112, 41, -67, 109, -40, 3, 33, -26, -38, 84]} +{"id": 74724075, "vector": [35, -52, 16, -32, 53, -45, -46, -36, -93, -75, -103, -65, -70, 122, 20, -22]} +{"id": 72019362, "vector": [24, 122, 108, -14, 68, -113, 64, 40, 61, 32, 6, -118, 121, -27, 124, -118]} +{"id": 29029316, "vector": [-122, -13, 50, -22, 89, -27, -22, 52, -54, -12, 116, -75, 42, -112, -58, -23]} +{"id": 26998766, "vector": [-65, -74, 26, 35, -107, 38, 95, -34, -115, 120, -42, 63, -23, 43, -65, -122]} +{"id": 44889423, "vector": [118, 103, 120, 10, 17, 5, -11, -96, 13, -120, -113, 104, -123, 3, 30, -87]} +{"id": 78785507, "vector": [-103, 47, 108, -47, 90, -92, -123, 29, 84, -126, 46, -44, -21, 1, -47, -55]} +{"id": 84840549, "vector": [-54, 58, -109, -73, -18, -12, 102, -94, -73, 88, -11, -101, -55, 40, -77, 125]} +{"id": 44245960, "vector": [-6, -75, 26, -4, -114, 45, -94, -12, -70, 57, -94, 27, 121, 42, -72, 60]} +{"id": 49882705, "vector": [-42, -79, -105, 99, 115, 86, 1, 92, 12, -24, -61, -70, -105, 0, -91, 28]} +{"id": 47708850, "vector": [-52, -100, 0, -70, 18, -63, -25, 97, 27, -81, -95, -100, 31, 97, -80, -23]} +{"id": 68102437, "vector": [78, 78, 15, 33, -107, 83, -41, 11, 75, -36, -70, 122, 97, -44, -86, 78]} +{"id": 70367851, "vector": [70, -75, 60, -24, -50, 116, -88, 86, -31, 63, -13, -123, -2, 105, -99, -51]} +{"id": 71522968, "vector": [-4, -50, -97, -85, -125, -32, 46, 73, -62, 107, 47, 43, -26, 15, 123, -27]} +{"id": 81172706, "vector": [-1, -71, -60, -51, -121, 60, -114, -80, -20, 62, -48, -96, 121, -74, -23, -109]} +{"id": 88251446, "vector": [4, -90, -58, 30, 62, 70, -93, -120, -112, 14, 15, -75, -106, -27, -20, 43]} +{"id": 18663507, "vector": [-93, 32, -13, 17, -41, 3, 74, -38, 0, -5, -72, 17, -124, -125, 51, 13]} +{"id": 2204165, "vector": [-51, 67, -94, 98, 34, -2, -102, -17, 6, -104, 45, -35, -49, 106, 98, 113]} +{"id": 7646095, "vector": [-62, -62, 102, 84, 107, 70, 35, -75, -10, -128, -54, -2, 24, -98, 108, 45]} +{"id": 71469330, "vector": [-19, 105, 99, 118, 78, -27, 16, -96, 56, -22, 35, 69, 3, 98, -85, -15]} +{"id": 99658235, "vector": [45, 95, 14, -90, 62, 21, -79, 23, 37, 88, 36, 108, -113, -33, 1, -122]} +{"id": 37620363, "vector": [120, 33, -12, 49, 29, 69, -126, -23, -39, -72, -122, -39, -43, -63, 121, 0]} +{"id": 28928175, "vector": [-46, 54, -20, 31, 44, 0, 67, -20, -38, 43, -54, -28, 17, -95, -33, -85]} +{"id": 37501808, "vector": [-3, -5, -124, 5, -31, 77, 10, 42, -62, 5, -127, -59, -4, 103, -60, -120]} +{"id": 40865610, "vector": [108, 7, -19, 116, 57, -35, 18, -32, 95, -92, 4, 44, -36, -127, 115, 92]} +{"id": 93562257, "vector": [-98, 98, 38, -98, -58, -5, -62, 42, -45, 119, 6, -47, -54, 57, -20, -110]} +{"id": 28550822, "vector": [-80, -103, -57, 41, -77, 48, 43, -15, -12, -92, -122, 36, 126, -81, 69, -67]} +{"id": 73109997, "vector": [93, 41, 11, -102, 108, -31, 12, 53, -116, -53, -105, -45, 87, 37, 64, -84]} +{"id": 63152504, "vector": [73, -13, 92, 82, 82, -2, -96, 122, 50, -29, -78, 91, 6, 43, -118, 41]} +{"id": 36808486, "vector": [-88, -101, 101, 78, 19, -41, 45, 97, 14, 70, 35, -27, -19, 71, 15, 123]} +{"id": 6505939, "vector": [110, -99, 92, -113, 8, 107, 84, -56, 36, 55, -95, -2, -25, -38, 4, 86]} +{"id": 61623915, "vector": [-12, -94, -20, -99, 82, -76, 33, -6, 23, 10, -93, 74, -88, 24, -32, -12]} +{"id": 75153252, "vector": [87, -38, -63, 84, 124, -88, 109, 11, 25, 53, -30, 111, 31, 1, -33, -32]} +{"id": 79136082, "vector": [127, 88, -84, 119, -126, -36, 46, -60, -5, 44, 72, -106, 2, 22, -22, -26]} +{"id": 53084256, "vector": [79, -121, 71, 5, 89, -127, 26, 126, -39, -124, 29, -21, -7, 74, -53, 96]} +{"id": 51464002, "vector": [-22, -1, 4, 31, 41, 82, -91, 114, 34, 22, -88, -39, 47, -22, -88, -43]} +{"id": 75128745, "vector": [91, -31, 65, 1, -88, 71, 93, 66, -98, 41, -53, 34, -45, -24, -48, -46]} +{"id": 3487592, "vector": [-93, -69, -73, -41, 92, -100, -43, -127, 59, 44, -12, -47, 36, 126, -82, -25]} +{"id": 83747892, "vector": [-100, -63, 18, -75, -84, 31, 31, -120, 43, 11, 93, -117, -127, 113, -90, -93]} +{"id": 6808825, "vector": [91, -8, -87, 41, 52, -91, -56, -32, -84, 72, -5, -38, -39, 93, 54, 10]} +{"id": 18504197, "vector": [-65, -55, 2, -7, -21, 114, -108, 82, 67, 91, 6, -91, -118, -46, 109, 113]} +{"id": 10961421, "vector": [-48, -59, -71, -5, 22, 104, -107, 77, -65, -108, 10, 90, -116, 44, -11, 57]} +{"id": 41245325, "vector": [72, 107, -53, -5, -124, 100, -9, -79, -86, 61, 3, 25, 105, 99, 21, 39]} +{"id": 70541760, "vector": [127, -87, -24, 49, -60, -23, 34, 58, -72, 2, -43, 40, -59, 93, 88, -93]} +{"id": 34698428, "vector": [-46, -61, 66, -72, -69, 1, 9, 44, 40, -23, 61, 2, -20, 26, -73, -59]} +{"id": 11923835, "vector": [34, 112, 8, -29, -52, 81, -85, -102, -14, -85, -37, 118, 24, 28, 110, -95]} +{"id": 91141395, "vector": [67, 83, 106, -2, -87, -72, 79, 74, 56, -5, 111, -72, -81, 94, -82, -111]} +{"id": 58749, "vector": [99, 120, -50, -75, -108, 24, 118, -77, -82, -122, 70, -122, 76, -20, -82, -105]} +{"id": 67281495, "vector": [58, 70, -44, -64, -21, -41, -26, 106, 117, -40, -75, 35, 122, -127, 58, 27]} +{"id": 54762643, "vector": [-91, -7, 85, 15, 83, -9, -125, 75, 12, 58, -104, -56, 15, 37, 29, 78]} +{"id": 63372756, "vector": [-7, 112, -106, -31, -97, -37, -27, -16, 127, -62, 117, -115, -67, 34, 82, -53]} +{"id": 65271999, "vector": [31, -74, 77, 35, -38, -101, -45, -40, -44, 55, -30, -35, 107, 106, -4, -5]} +{"id": 88653118, "vector": [62, 114, -37, 7, 10, 127, 4, 97, 95, -93, 114, -59, -124, 68, 25, 7]} +{"id": 6221471, "vector": [49, -76, -101, 32, -120, 105, 42, -38, -42, -123, 3, -82, 25, -44, -89, 59]} +{"id": 23740167, "vector": [23, -87, 117, -89, -46, 99, 125, 17, 105, 30, -49, -44, -112, 76, -115, -32]} +{"id": 66611759, "vector": [86, 29, -71, 32, 89, 63, -19, -119, -41, 93, -14, -34, -17, -47, -46, -95]} +{"id": 48395186, "vector": [24, 109, 84, -95, -31, -67, 105, 127, -81, 31, 31, 82, -99, 65, 65, 2]} +{"id": 51426311, "vector": [125, -119, 83, -100, 55, -50, 54, 112, 78, 26, -29, 15, -23, 90, 61, 66]} +{"id": 55960386, "vector": [-30, -104, 76, -4, 81, -100, -10, 62, -32, -116, 89, -35, 101, -47, -24, -61]} +{"id": 62357986, "vector": [-108, -66, 51, 103, -2, -15, 44, 83, -33, 14, -99, -2, -123, 74, -47, 123]} +{"id": 7104732, "vector": [-108, -78, -71, -37, 59, -102, 29, 91, 105, 54, 30, 86, 74, 104, 39, 26]} +{"id": 99965001, "vector": [-16, 52, 34, 73, 36, -69, -55, -22, -4, -42, -37, -16, -70, -86, -1, 121]} +{"id": 18783702, "vector": [-58, 25, -43, 3, -52, -13, -53, 10, -17, 103, 111, 82, -30, 97, 115, 63]} +{"id": 58224549, "vector": [-99, 58, 115, -45, -6, -121, 88, -63, -122, -55, 4, -11, -62, -12, 39, -127]} +{"id": 7066775, "vector": [-99, -8, -121, 23, 10, -124, -123, -92, -96, 111, 52, 94, -99, -82, -118, -105]} +{"id": 8055981, "vector": [-41, -32, 4, 34, -91, 13, -114, 76, 37, -10, 119, 80, -4, -101, -8, -120]} +{"id": 73235980, "vector": [-88, 56, -92, 76, 39, -124, -67, -110, 37, 106, -114, -39, -77, -10, -69, -98]} +{"id": 86118021, "vector": [-26, -26, -18, 49, 62, 2, 55, -74, -115, 18, 34, -40, -14, -95, 16, 126]} +{"id": 50007421, "vector": [67, 31, 117, -39, -52, -126, 117, 79, -27, 97, -15, 110, -37, -27, 66, -85]} +{"id": 9623492, "vector": [115, -38, -121, -106, 117, -87, 32, -115, -111, -27, 27, -99, -58, -84, 56, 11]} +{"id": 4099191, "vector": [-72, -86, 30, -12, -79, 86, -93, 46, -2, -9, 35, 116, -30, -56, 23, -116]} +{"id": 56461322, "vector": [-69, 54, -51, -55, 65, 65, -1, 46, 61, 3, -71, 7, -81, -64, -29, 97]} +{"id": 56473732, "vector": [84, 4, 100, -100, -45, -86, 65, -117, -4, -105, -56, 26, -51, 6, 42, 69]} +{"id": 44664587, "vector": [45, -69, -64, 49, 93, 62, -37, -83, -109, -21, 63, 117, 50, -93, 49, 113]} +{"id": 45237957, "vector": [-102, 70, -76, -7, 76, -100, -98, -128, 52, -89, -98, 117, 107, -110, -38, 114]} +{"id": 47090124, "vector": [26, -87, 86, -72, 60, -108, -41, 121, -44, -58, -40, 23, -52, 104, 50, -12]} +{"id": 7011964, "vector": [-66, -19, 24, -67, 41, -14, -73, 31, -14, -92, -1, -128, -108, 108, -18, -22]} +{"id": 33699435, "vector": [-11, -18, 66, 109, 110, -110, -61, -52, 56, 7, 44, 73, 16, 105, 9, 24]} +{"id": 25636669, "vector": [85, 117, 15, 22, 51, 78, -125, -86, -113, -63, -19, 81, -121, -67, -121, -28]} +{"id": 8791066, "vector": [-50, 70, 94, 106, -123, 118, -127, 59, -124, 121, 51, -4, -30, 56, 39, -64]} +{"id": 4508700, "vector": [-34, 100, 18, 43, -59, -35, 94, -40, 7, 106, 22, -8, 28, -45, 3, 47]} +{"id": 93270084, "vector": [-77, -91, 91, 98, 23, -80, -96, 6, -23, 9, -109, 97, 106, -82, 54, 125]} +{"id": 3088684, "vector": [103, 38, 47, -31, 126, 21, -21, -43, 84, 122, 99, -36, -51, -109, -81, 36]} +{"id": 17146629, "vector": [-124, 69, 64, 68, -23, 76, 72, -10, -5, 108, -58, 76, -33, 57, -124, 109]} +{"id": 99297164, "vector": [-19, -25, -62, -3, 110, -125, -104, -38, -94, 54, -125, 127, 26, 0, 24, -106]} +{"id": 41442762, "vector": [-72, 87, -58, 118, -33, -2, -108, 19, -46, -103, 67, -85, -46, -94, 28, -109]} +{"id": 66250369, "vector": [-124, 90, -112, 104, 72, -32, 67, 15, 110, 4, 34, -96, -14, -48, 39, 77]} +{"id": 58751351, "vector": [-78, 19, -63, 116, 93, -91, 103, -61, -101, 18, 113, 57, -56, -81, 28, 100]} +{"id": 66667729, "vector": [117, 39, -87, 96, -71, -77, 35, 35, -109, -37, 68, -65, 10, -15, -83, 61]} +{"id": 36312813, "vector": [118, 107, 121, 36, -33, 13, 71, 6, -18, 90, 72, -48, 99, -60, -81, 113]} +{"id": 39373729, "vector": [-63, -93, 125, 60, 7, 77, 114, 100, -122, 50, -15, -71, -102, -62, 50, -74]} +{"id": 68702632, "vector": [123, -25, -53, 26, 101, 117, -63, 78, 18, -105, 30, 10, -71, 26, -108, -42]} +{"id": 34044787, "vector": [-18, 49, 30, -13, -13, 46, -9, 69, 38, -106, 95, -19, 65, -39, -66, -98]} +{"id": 33237508, "vector": [-17, -4, 93, 57, 30, 127, -97, 6, -71, 65, -67, 0, -105, -12, -14, -88]} +{"id": 95581843, "vector": [-23, -73, 75, -57, -17, 92, 82, -41, -14, 116, 41, 33, 67, -80, -28, 122]} +{"id": 89811711, "vector": [-20, -99, -92, -30, -83, -34, -75, -56, 12, 39, -76, 95, 92, 25, -26, 111]} +{"id": 22113133, "vector": [59, -10, -84, -123, 57, -43, 11, 84, 37, 29, 69, 49, -93, -38, -7, -26]} +{"id": 68644627, "vector": [-4, -8, 48, -4, 26, 73, 32, -114, -69, -57, 82, 3, 26, 21, 105, 99]} +{"id": 59329511, "vector": [27, 41, 20, 38, 111, 106, 65, 52, 34, 31, 103, 1, -33, -64, -75, 9]} +{"id": 28787861, "vector": [-17, -68, 36, 105, 35, -14, -6, 16, 104, -118, -23, -58, 60, 100, -112, -124]} +{"id": 28796059, "vector": [49, -6, 121, 79, -22, 55, 98, 99, 57, -11, -122, 52, -119, 106, -76, -40]} +{"id": 71316369, "vector": [-107, 74, -101, 72, -42, 0, -72, 48, -31, 104, -120, -127, 35, 23, -109, -92]} +{"id": 5970607, "vector": [66, 16, -59, -119, -51, 37, 88, -86, -121, 71, 55, 40, 118, -23, -83, 118]} +{"id": 77787724, "vector": [99, -16, 67, 53, 124, 47, -56, 78, 19, -45, -80, 96, -44, -17, 4, 37]} +{"id": 54199160, "vector": [31, -93, -67, -103, -114, 69, 3, -24, -5, -54, 79, 41, 6, -51, -127, 49]} +{"id": 33895336, "vector": [56, -32, -1, 15, 120, -8, -79, 24, 112, 45, 99, 15, 8, 66, 68, -97]} +{"id": 3880712, "vector": [-107, -86, 84, 47, -46, -50, 18, 51, 32, 83, -80, -119, 8, 27, 74, -60]} +{"id": 38022834, "vector": [39, 113, -124, 14, -98, 92, -61, -49, 65, -8, -44, -108, -31, -83, -57, 65]} +{"id": 95957797, "vector": [-26, -75, 74, -84, 54, 66, 54, -51, -103, 97, 47, -118, -56, 127, 25, 12]} +{"id": 79657802, "vector": [54, -26, -12, 111, 39, -19, 47, -5, -116, -105, 122, -124, -58, 18, 115, -22]} +{"id": 53842979, "vector": [-64, -102, 125, -68, 62, -125, 30, -45, -8, 124, -103, -98, 31, 12, 63, 79]} +{"id": 71333116, "vector": [-108, -104, -107, 30, 102, 60, 10, 80, -61, 116, -60, 109, -50, -52, -35, -99]} +{"id": 26776922, "vector": [-9, 100, 14, -96, -75, 12, -73, -72, 32, -78, -1, 33, 31, 19, -128, -27]} +{"id": 66045587, "vector": [-55, -75, -105, 1, -10, -122, -13, -81, -25, 88, 106, -49, 80, -62, 123, 32]} +{"id": 29466406, "vector": [31, 95, -22, 118, 101, 106, -87, -80, 32, 84, -15, -115, -15, -12, -127, 97]} +{"id": 29834512, "vector": [-89, 23, 74, 68, 32, -10, -75, -8, 88, -55, 41, 45, -96, -38, 6, 90]} +{"id": 30998561, "vector": [-48, -115, -14, -63, 38, -127, -3, -122, 78, -3, 75, -86, -125, 93, -6, -52]} +{"id": 16971929, "vector": [22, -33, -81, 82, 113, 54, -18, 54, -20, -6, 75, 61, 21, 33, 86, -72]} +{"id": 49328608, "vector": [-6, -49, -2, 10, 109, 78, -73, 81, 35, -111, -96, -61, -74, -99, 65, -10]} +{"id": 96735716, "vector": [-79, 43, 84, 1, -71, -33, 60, -62, 66, 6, -1, -41, 83, -4, 93, 70]} +{"id": 508198, "vector": [59, -99, 29, -14, 36, -101, -5, 109, 75, -65, -71, 93, -112, 122, 121, -52]} +{"id": 52204879, "vector": [-123, -18, 92, 25, 50, 60, -53, 13, 34, 93, 114, 3, 43, -5, 112, -103]} +{"id": 82427263, "vector": [-127, -6, 19, -8, 117, 57, 36, -85, -77, -98, 4, 4, -50, 36, -51, -128]} +{"id": 2208785, "vector": [-15, 82, -89, -121, 40, 73, 96, -33, 74, -41, -10, 47, 28, 85, 27, -101]} +{"id": 27041967, "vector": [118, -116, 32, -33, -114, -53, 6, -81, 89, -33, -123, -95, 123, -35, -123, 124]} +{"id": 4515343, "vector": [3, 106, 82, -127, 126, -29, 17, 104, -113, -51, 120, 94, -64, -119, -91, -41]} +{"id": 18131876, "vector": [-113, 3, -29, 93, -119, -91, 18, -27, 16, 61, 93, 61, 54, -90, 32, 53]} +{"id": 2607799, "vector": [2, -47, -58, 7, 100, 90, -79, -26, -62, -55, 47, -10, -128, -68, -5, -42]} +{"id": 42782652, "vector": [70, -42, -75, 87, -50, -102, -104, 70, -28, -65, -109, 20, -12, -109, 103, -113]} +{"id": 43376279, "vector": [-110, -21, -43, 124, 16, -127, 33, -30, -126, -35, -44, -122, -12, -36, 18, 70]} +{"id": 96420429, "vector": [42, 80, 5, -85, -67, 101, 11, -108, -69, 38, -9, -97, -96, 31, 86, -59]} +{"id": 74357852, "vector": [19, -41, -37, 105, -111, -127, -86, 62, -108, 96, -56, -24, 66, 58, -113, 83]} +{"id": 91664334, "vector": [-119, 89, -96, -125, -104, -95, 93, 20, -117, -96, 66, -108, 68, 22, -60, 116]} +{"id": 96726697, "vector": [107, -112, -31, 3, 29, 98, -9, 43, 23, 35, 22, 119, -104, -109, 34, -25]} +{"id": 60430369, "vector": [117, -113, -10, -51, 72, -57, -88, 47, -43, 64, 54, 88, -96, -107, -22, 47]} +{"id": 91802888, "vector": [1, 81, -99, -89, 51, 63, 21, -101, -73, -96, 86, 19, 53, 59, 101, 52]} +{"id": 57248122, "vector": [37, -36, -76, 104, 120, 79, 22, -34, -13, -17, 29, 45, 10, 66, 93, 90]} +{"id": 79922758, "vector": [-103, -94, 15, 125, 49, 70, 2, 53, -101, -34, -112, 22, 102, 63, -45, -107]} +{"id": 55602660, "vector": [-128, -115, 73, -24, -57, 95, -11, 62, -17, -69, 81, -97, -17, -96, -99, -95]} +{"id": 7229550, "vector": [41, -99, -70, -117, -18, 2, -86, 36, 67, -89, -100, 37, 53, -83, -114, -101]} +{"id": 78766358, "vector": [-71, 83, -103, -94, 104, -32, -125, -118, 99, -120, 109, -54, 98, 64, 29, 91]} +{"id": 7182642, "vector": [-18, -96, 39, 59, -59, -6, 126, -32, 4, -61, 2, -10, 7, 86, 77, -56]} +{"id": 6038457, "vector": [-53, 8, 108, 5, 57, 15, -83, -39, 30, 106, 64, -80, -41, -37, -49, -111]} +{"id": 31126490, "vector": [121, 117, 91, -77, 56, -22, -21, -110, 65, -6, -60, 32, 102, -10, 120, -104]} +{"id": 88047921, "vector": [31, -58, -75, 8, 3, -114, -2, -85, 56, -11, -95, -96, -110, 66, 31, -63]} +{"id": 3183975, "vector": [-26, 0, 8, -17, 77, 121, 73, 12, -99, -90, 71, -104, 7, 72, -7, 91]} +{"id": 62115552, "vector": [57, -63, 118, -27, 43, -87, -49, -85, -118, 62, 120, 19, 14, 20, -106, -62]} +{"id": 85711894, "vector": [24, -110, 98, 56, -76, 94, -17, 58, -43, -4, 40, 13, 95, 104, 76, 103]} +{"id": 14045383, "vector": [36, -66, 69, -19, -17, 119, 25, 11, 84, 34, -35, 68, -22, 75, -102, -73]} +{"id": 78602717, "vector": [105, -95, 71, -38, 70, 69, 65, -82, 57, -121, -38, 65, -62, -13, 98, 74]} +{"id": 81805959, "vector": [-74, 123, -110, 59, 100, -10, -63, 16, 113, 112, -50, -57, 116, 41, 2, -24]} +{"id": 26114953, "vector": [48, -96, -19, 55, -110, -86, 6, -72, 83, 102, -76, 42, -111, -91, -8, 10]} +{"id": 55143724, "vector": [-87, -116, -77, -12, 7, -115, 77, 106, -10, -25, -111, 114, 71, -42, -116, -95]} +{"id": 78549759, "vector": [-115, 68, -60, -116, 7, 67, -90, -105, 89, 90, -128, -82, 58, 5, -122, 7]} +{"id": 16054533, "vector": [-57, 106, -19, -112, 74, -46, 66, -14, 107, -105, -92, -14, -67, -46, 80, 69]} +{"id": 74137926, "vector": [-103, -51, -44, -79, 50, 87, 80, -91, -41, -31, -22, -24, -122, -76, -93, -67]} +{"id": 23569917, "vector": [4, -22, -20, 75, -105, -28, -61, 31, -93, 108, -111, 46, -21, 61, 89, -59]} +{"id": 33628349, "vector": [7, -4, -78, -106, 74, 81, -13, -43, -70, 10, 34, 3, 75, 2, 76, -58]} +{"id": 59910624, "vector": [-21, -2, 54, -56, 66, -81, -42, -9, -4, 95, -29, -21, 103, 51, 64, 28]} +{"id": 97379790, "vector": [-60, -34, -89, -28, 80, 2, -3, 6, 100, 1, 89, -31, -103, -42, 117, -61]} +{"id": 36135, "vector": [-107, -90, 70, -6, 79, 8, 92, 103, 6, -70, 34, 94, 36, 65, 92, 43]} +{"id": 16424199, "vector": [31, 103, -51, -64, 67, 4, 35, -119, 124, 104, 91, 120, 62, -79, -36, 96]} +{"id": 30694952, "vector": [82, 3, 33, 68, -86, -76, -44, -62, -115, 58, 90, 26, 120, -98, 44, -34]} +{"id": 49641577, "vector": [40, 48, -94, 126, -65, -83, 94, 75, -116, 44, 79, -27, -93, -17, -80, -76]} +{"id": 41481685, "vector": [-105, 72, -28, 0, 93, 55, -94, -52, -8, 107, -71, 19, -74, 41, 78, -30]} +{"id": 32058615, "vector": [124, -68, 15, 67, 45, -49, 111, 82, -40, 8, 9, 106, -6, 103, -45, -104]} +{"id": 81774825, "vector": [-104, 114, 59, -83, 108, 59, 47, 51, 93, -111, 111, -60, 70, -64, 100, 6]} +{"id": 68316156, "vector": [-31, 51, -70, -25, -92, -87, -121, -42, -7, 105, 111, 7, 73, 74, -64, -93]} +{"id": 7517032, "vector": [89, -106, 65, -14, 34, 108, -126, -115, -21, -81, 95, -3, 15, -68, 19, 95]} +{"id": 77898274, "vector": [-22, -98, 44, 6, -61, 103, -56, 104, -126, -37, 28, 113, -56, -50, 80, -2]} +{"id": 51315460, "vector": [67, -43, -20, 119, 24, 57, 85, 29, 0, -114, 2, 98, 126, -120, 109, -57]} +{"id": 75052463, "vector": [16, -93, 50, 16, 20, 120, -106, 86, 51, -77, 1, 55, 32, 14, -95, 0]} +{"id": 9259676, "vector": [77, -35, -77, -95, 89, -128, -108, 120, 71, 4, 56, -61, -105, 10, 120, -24]} +{"id": 20568363, "vector": [5, -11, 80, -18, 96, -83, -28, -79, -74, 25, -41, -71, -94, -25, 12, -40]} +{"id": 61859143, "vector": [-14, 56, 44, 110, 90, -86, -92, 115, 111, 53, -41, -97, 92, 42, 16, 63]} +{"id": 38494874, "vector": [6, -96, -58, -66, -107, 111, 10, -40, -4, -60, 2, 47, 99, 61, -110, -78]} +{"id": 8651647, "vector": [121, -68, -22, 0, 117, 98, 20, 104, -42, -1, 85, -16, -26, -84, -95, 90]} +{"id": 98943869, "vector": [-61, 81, -46, 31, -120, 53, 102, 36, -93, 31, 25, -117, 73, -70, 72, -56]} +{"id": 1936762, "vector": [-15, -16, 0, -49, -125, -86, 90, 68, -61, -73, -33, -91, -78, -59, -49, -112]} +{"id": 52060076, "vector": [0, 53, 65, 127, 63, 111, 89, -110, -96, 106, 30, 121, 120, 52, 9, -30]} +{"id": 57359924, "vector": [-83, -107, 80, -15, -76, -42, -32, -16, 111, 103, 46, -8, 85, 42, 70, -16]} +{"id": 90004325, "vector": [-38, -48, 40, -51, 95, -100, -2, -91, 68, -14, 58, -124, 69, 119, 111, -102]} +{"id": 82651278, "vector": [-117, -122, 44, 33, -117, -101, -114, 107, 25, 19, 19, 62, -20, -32, -69, -121]} +{"id": 4091162, "vector": [-30, -111, 36, 25, 25, -110, -65, -71, 94, -60, 65, 102, 18, -10, 46, 65]} +{"id": 66903004, "vector": [33, 71, -106, 46, -86, -108, -63, -81, 40, -117, -40, 95, 95, -31, -127, 52]} +{"id": 20122224, "vector": [-96, -49, 28, -55, 95, -33, -12, 13, 82, -119, -55, 30, 54, 38, -46, 1]} +{"id": 9860968, "vector": [-93, 74, -20, 70, -11, -52, -11, -79, 56, 118, -112, -84, 13, 62, -114, 3]} +{"id": 24915585, "vector": [-127, 10, 57, 62, -57, 46, -33, 70, -125, 89, -109, -72, -127, 99, -109, 28]} +{"id": 40781449, "vector": [-99, 75, -65, -114, 9, -124, -47, 106, 0, -6, 113, 52, 90, -49, -94, 123]} +{"id": 90654836, "vector": [41, -20, 13, 104, 127, 11, -69, 111, 99, -7, 48, -98, 44, 69, 6, -77]} +{"id": 30811010, "vector": [-119, 87, 56, 91, -82, -35, 34, -69, -26, -102, -59, -123, -104, 7, 81, -18]} +{"id": 91831487, "vector": [30, -14, -106, 35, 63, 22, -24, 127, -127, -40, 30, -69, -3, -66, -7, -61]} +{"id": 54232247, "vector": [28, 0, 69, 116, 112, 7, -49, -2, -92, 96, -94, 68, 124, 83, 126, -97]} +{"id": 81274566, "vector": [89, -126, -88, 124, -63, -118, -35, -22, -59, 86, -120, 107, -29, 76, 2, -1]} +{"id": 68816413, "vector": [-100, 124, -58, 81, -53, 81, 27, -45, -9, -115, 88, -17, -13, -75, 64, -48]} +{"id": 28851716, "vector": [-72, -57, 62, 27, -114, 85, 69, 66, 76, 110, -85, 46, 13, 40, 27, -85]} +{"id": 27625735, "vector": [-115, 67, -105, -41, -78, -23, -40, -82, 19, -61, -19, -54, 71, -85, 31, 32]} +{"id": 72732205, "vector": [23, 95, 26, -127, 76, 44, 72, -83, -120, 8, 73, -7, -5, 120, -93, 115]} +{"id": 85023028, "vector": [-80, -62, -55, -126, -22, 106, -50, -113, 111, 106, 39, -17, -32, 82, -79, -68]} +{"id": 37659250, "vector": [-69, -109, -4, 111, -25, 23, -53, -30, -95, 96, -66, 68, 55, -2, 32, 100]} +{"id": 4119747, "vector": [-106, 43, 50, -128, 18, -99, -107, -54, -66, -26, 54, 77, 59, 104, -119, 82]} +{"id": 60106217, "vector": [-63, -87, 41, 125, -27, 106, 110, 16, -26, 121, 21, -89, 101, 106, -79, 20]} +{"id": 89419466, "vector": [-124, -13, 33, 29, 111, 59, 50, 25, -124, -72, -28, 120, -57, 55, 50, -3]} +{"id": 92692978, "vector": [-23, -59, -43, 51, -58, -120, -8, 53, 19, -116, -102, 68, 112, -2, -118, 3]} +{"id": 59981773, "vector": [61, 108, 87, 17, -89, 2, -85, -99, 106, 104, -46, 90, 36, 124, -116, 111]} +{"id": 69355476, "vector": [-101, 42, 48, 13, -79, -94, 48, 124, -104, 82, 46, 27, 29, -105, 64, 37]} +{"id": 71920426, "vector": [-38, -128, 125, 6, -120, -55, -38, -86, 18, 34, -87, 89, 39, -32, -49, -41]} +{"id": 42947632, "vector": [63, -87, 73, 56, 9, -80, -122, -14, -119, -42, 57, 22, 11, 106, -125, -3]} +{"id": 74110882, "vector": [51, -64, 107, -56, 15, -51, -59, -62, 110, 91, -77, 59, -111, -125, 86, 126]} +{"id": 82532312, "vector": [28, -59, 126, 56, 68, -95, -116, -127, 113, -66, -98, 4, 1, -95, -15, -9]} +{"id": 27665211, "vector": [-112, -12, -11, -104, 14, -90, -64, -128, 92, 32, -56, -108, -122, 41, -101, 85]} +{"id": 14093520, "vector": [32, -64, 42, 59, -110, -110, -102, -105, -27, -61, -87, 56, 84, -87, 34, -78]} +{"id": 47792865, "vector": [-66, 28, 69, 112, 28, -72, -77, 31, -109, 27, -38, 120, 80, -43, -80, 71]} +{"id": 55470718, "vector": [118, 44, -62, 33, -75, 22, -57, -44, 103, -109, 4, -89, -74, 17, -52, -123]} +{"id": 75229462, "vector": [36, 18, -30, -22, 125, -27, 49, 52, -9, 88, 71, -82, 61, -38, -14, -27]} +{"id": 79806380, "vector": [-2, -18, -22, -37, 98, 74, 120, 76, 59, 42, 93, -99, 94, -115, -70, -48]} +{"id": 24314885, "vector": [-112, 125, -67, -43, 125, 10, -26, 2, 81, 50, 58, 120, -31, 63, -101, 75]} +{"id": 18699206, "vector": [-28, -74, 80, -72, 20, -64, 54, 42, 96, 117, -51, 4, -62, -45, -104, 24]} +{"id": 33935899, "vector": [54, 9, 36, 48, -10, -50, 60, 86, 47, 32, -121, -1, 17, 91, 67, -39]} +{"id": 14326617, "vector": [-114, -62, 102, -13, 73, 5, -98, 41, 18, 127, 12, 19, -109, 53, -40, 42]} +{"id": 83150534, "vector": [89, -23, -37, -78, 9, 36, -61, -1, 46, 112, -54, -110, 52, -22, -89, 127]} +{"id": 91938191, "vector": [-42, -21, 58, -32, 75, 122, -66, -107, -69, -60, 20, -125, -10, -34, -40, -108]} +{"id": 49271185, "vector": [-1, 29, 70, -45, 124, 96, -125, 108, 47, 25, -39, 113, 12, -39, 90, 99]} +{"id": 7687278, "vector": [22, -93, 122, 66, -8, -120, 61, 5, -5, 106, 51, -18, 2, -67, -112, -61]} +{"id": 1204161, "vector": [-60, -110, -98, 105, -15, -45, 22, 56, 125, 93, -42, 32, -92, 88, -61, 6]} +{"id": 65047700, "vector": [-42, 99, 27, -105, -38, -63, -65, 58, -99, 98, -124, 24, 25, 24, 100, -36]} +{"id": 21919959, "vector": [-3, 49, -88, -63, 11, 55, 113, 98, -22, 102, -117, -82, 79, -56, 97, -95]} +{"id": 19272365, "vector": [102, 89, 31, 47, 73, -17, 11, 41, -85, 72, 91, 38, 101, -43, 41, 47]} +{"id": 86798033, "vector": [120, -84, 50, 121, -105, -114, -40, -58, 19, 46, -65, 39, -96, 16, 79, -92]} +{"id": 66428795, "vector": [-55, -95, 109, 115, 55, 106, 113, -31, 86, -48, 3, 102, -61, 108, -45, -66]} +{"id": 92033260, "vector": [-108, 99, -75, -57, -11, -105, -9, -9, 61, -78, 86, -97, 127, -59, -43, -38]} +{"id": 65074535, "vector": [-91, 126, 89, -32, -91, 10, -31, 19, 74, 124, 43, 83, -73, -68, -127, -108]} +{"id": 63015256, "vector": [-115, -115, 57, -50, -35, -69, -117, 32, 56, -99, -40, 33, 51, 5, 118, 104]} +{"id": 76960303, "vector": [121, 123, -3, -41, -71, -10, -19, -83, 29, -6, 76, -81, 34, 64, -78, -121]} +{"id": 14363867, "vector": [46, 114, -73, -103, -99, -46, -82, 56, 82, 65, -34, 117, -98, -86, -5, 107]} +{"id": 4069912, "vector": [-71, 85, -123, 14, -34, -43, -99, -80, -111, -128, 76, 43, 56, 5, 6, 6]} +{"id": 95290172, "vector": [10, 92, -99, -20, 76, -49, -121, 24, 5, -58, 117, -111, -96, -82, -38, 99]} +{"id": 70782102, "vector": [63, -127, 96, 52, 119, 67, -43, 19, -115, 56, -8, 106, 124, -21, 74, -122]} +{"id": 61271144, "vector": [-93, 116, 121, 24, -46, -36, 101, 20, 93, 109, 32, -16, -68, 117, -70, 91]} +{"id": 95010552, "vector": [111, -77, 56, 60, -34, -69, -95, -44, -111, 101, 64, -97, -113, 37, 124, -123]} +{"id": 36933038, "vector": [101, 98, -28, 101, -35, -99, 32, 1, -61, -27, 106, -123, 102, -12, 107, 31]} +{"id": 36468541, "vector": [-95, 44, 44, -122, -32, 117, 62, -17, -68, -118, -123, -35, -35, -119, 38, -47]} +{"id": 16684829, "vector": [-75, -29, -126, -105, 42, 63, -94, -123, 51, 111, 101, -69, -12, 88, 118, 14]} +{"id": 50188404, "vector": [90, 12, 43, 85, -66, 27, 62, -33, -20, 39, -48, 27, -82, -40, -84, -26]} +{"id": 88444207, "vector": [-70, -101, -67, -2, -81, -114, -83, 16, 71, 45, 66, 8, 112, 33, 6, 102]} +{"id": 44479073, "vector": [77, -77, -121, -56, -10, -42, -48, -5, -56, -86, -125, -51, 48, -102, -27, 79]} +{"id": 56153345, "vector": [-43, -61, -63, 6, -76, 27, 88, -51, -40, -85, -122, 91, -76, -17, -61, 68]} +{"id": 1788101, "vector": [123, 43, -45, 7, -73, 66, -60, -104, -2, -37, -84, 72, -54, 22, 102, 67]} +{"id": 99021067, "vector": [111, -58, 69, 91, -29, -9, -3, 11, 60, 31, -46, -62, -52, 103, 0, -75]} +{"id": 26397786, "vector": [124, -77, 48, -41, -30, -89, 72, 22, -83, -15, 89, -48, -103, -64, -53, 19]} +{"id": 87720882, "vector": [109, 74, 113, -58, 102, 14, -16, 10, -103, 27, -7, -13, 80, 61, 46, -126]} +{"id": 26392416, "vector": [10, 28, -44, -39, -25, -64, 48, 100, 99, 30, 69, -107, -66, -58, 77, -60]} +{"id": 67084351, "vector": [114, 116, 39, 2, -68, 115, 27, -33, -100, -36, -111, 82, -25, -106, -108, 67]} +{"id": 50668599, "vector": [23, 121, -55, 115, 126, 37, 26, -27, -81, 53, 17, 63, 118, -74, -104, -84]} +{"id": 68204242, "vector": [-41, 76, 109, 4, 82, -24, -42, 40, -77, 71, 1, 9, 80, -99, 68, 114]} +{"id": 42237907, "vector": [-95, -116, -74, -61, 43, -5, 38, 77, 44, -117, -38, -108, -27, 62, -58, 1]} +{"id": 23194618, "vector": [-53, -104, 22, 12, 79, 4, -11, -72, -73, 54, -110, -38, -107, 87, 88, 76]} +{"id": 29401781, "vector": [-123, -74, 104, 14, 24, 93, -66, -29, 80, 25, 91, -121, -127, 26, -114, 26]} +{"id": 36580610, "vector": [22, -60, 94, 69, 119, -11, -18, 105, -17, 39, 22, 5, -81, 108, -43, -84]} +{"id": 72777973, "vector": [-8, 123, 103, -62, -89, 15, -114, -108, -113, 15, 102, 123, -53, -64, 4, 43]} +{"id": 72238278, "vector": [-22, 117, -59, 29, -120, -89, -61, -68, 85, 63, -7, -31, 5, -81, -49, 120]} +{"id": 9257405, "vector": [109, -23, -32, 120, -91, 54, -42, 26, -58, 2, 113, -41, 40, 118, 45, 127]} +{"id": 91990218, "vector": [-21, -20, 0, -91, 107, 82, 61, 0, -52, 105, 81, -104, -69, -91, -70, -28]} +{"id": 72373496, "vector": [-1, 32, 45, 22, 0, 6, 98, -55, 27, 13, -53, -120, 94, -39, -108, 77]} +{"id": 85571389, "vector": [-123, -98, -85, 80, -123, 71, 53, -63, -43, 80, -49, 21, -115, 48, -4, 73]} +{"id": 29994197, "vector": [100, -102, -4, 115, -111, 27, -116, 62, 126, -61, 63, -48, 52, -12, 67, 79]} +{"id": 57393458, "vector": [112, 56, -124, -20, 35, -98, 73, -34, -105, -59, 52, -53, -88, -103, -53, -112]} +{"id": 39201414, "vector": [45, 92, -101, -10, 61, 89, 110, -3, 59, -50, 14, 57, 101, -72, 108, -15]} +{"id": 30366150, "vector": [-27, 106, 30, -77, -84, 17, 3, 95, 100, -18, 101, 122, -63, -126, 21, 37]} +{"id": 73786944, "vector": [124, 116, -76, 38, 105, -50, -5, 107, 66, -39, 22, 37, -54, 87, -13, -38]} +{"id": 54014062, "vector": [56, -17, -87, 27, 69, 29, -13, -30, -12, 33, -79, -28, -64, -29, 24, -84]} +{"id": 13862149, "vector": [-124, 14, 52, 12, 12, 21, 106, 79, 30, 66, 55, 38, -90, -45, 118, 44]} +{"id": 40356877, "vector": [74, -88, -117, 2, -30, -29, 108, 43, 21, 49, -64, 14, -63, -122, -114, 55]} +{"id": 45919976, "vector": [-30, -101, -5, -65, -117, 90, -84, -75, 97, -13, -86, -40, 92, 21, 73, 121]} +{"id": 10264691, "vector": [64, 110, -90, -81, 93, -125, 94, 60, -62, 90, -63, 82, 54, 15, 78, -66]} +{"id": 10358899, "vector": [103, 53, 74, 118, 102, -50, 111, 79, -104, -24, 7, -48, 125, -98, 26, 103]} +{"id": 16567550, "vector": [-25, 40, -102, -8, 64, 85, -122, 107, -79, -59, 124, -122, -27, 31, -47, 24]} +{"id": 63628376, "vector": [57, 45, -126, -68, 28, 24, -116, -51, 93, 126, -105, 22, -78, 119, 1, -3]} +{"id": 95726235, "vector": [58, -119, -5, -79, -79, -48, -108, 70, 93, -83, 106, 7, -73, -80, 99, -2]} +{"id": 51466049, "vector": [-5, 45, -123, -91, 108, -71, 115, 10, -61, -42, -108, 13, -23, 43, -44, -24]} +{"id": 18806856, "vector": [25, 50, -97, -107, -10, 14, 24, -58, 109, -61, 65, -107, 5, 124, 51, -123]} +{"id": 84406788, "vector": [-126, 119, -108, -88, -85, 10, -27, -68, 73, -55, -40, -72, 17, -73, -91, -32]} +{"id": 83302115, "vector": [-110, 66, -15, 117, 84, -67, -89, -96, -64, -26, 111, -38, 96, -115, -59, -83]} +{"id": 94711842, "vector": [123, -109, 121, -2, 24, 58, 124, -107, -2, 115, 62, 12, -80, 67, -107, -114]} +{"id": 12024238, "vector": [-70, -113, 19, -19, 80, -96, -3, 102, 63, 102, -116, -98, 40, 112, -82, 24]} +{"id": 49236559, "vector": [-21, -119, 36, -126, 80, 100, 51, -94, -100, -21, -77, 36, -34, -80, -51, 22]} +{"id": 60393039, "vector": [72, 77, -79, 51, 55, -121, -64, 102, 4, 115, 56, -61, -16, -102, 14, -62]} +{"id": 8733068, "vector": [-48, -21, -114, 66, -32, -56, 102, -123, 26, 123, 79, -127, 72, 115, -111, -61]} +{"id": 37280276, "vector": [37, -66, 14, -121, 63, 101, -28, 64, 45, -99, 85, 10, 111, -77, 115, 82]} +{"id": 22200378, "vector": [93, -58, -29, -19, -97, 31, -49, 59, 15, 79, 114, -10, 32, -90, 70, 53]} +{"id": 64848072, "vector": [68, 101, -95, 60, -107, 12, 53, -38, -110, 112, -102, 117, -9, -93, 95, 62]} +{"id": 47298834, "vector": [100, -128, 112, -94, -25, -28, -33, -63, -31, 120, -51, 126, -79, 69, -124, -58]} +{"id": 31491938, "vector": [127, -115, -82, -86, -60, -68, 90, 101, 96, 27, -98, -25, 101, 92, 72, -8]} +{"id": 9829782, "vector": [-66, -72, 3, 78, -95, -127, 49, -40, 101, 125, 60, 64, 12, -113, 74, -16]} +{"id": 1375023, "vector": [-125, 126, 102, 62, 62, 99, -70, -59, -79, -20, 100, -15, -33, 21, -72, -77]} +{"id": 47893286, "vector": [117, -35, 12, 117, 106, 2, 13, -109, 88, 97, -117, 55, -5, -26, -67, -117]} +{"id": 749283, "vector": [13, -12, -15, -71, 28, 107, 17, -84, 50, 57, -95, 106, 18, -46, 46, 11]} +{"id": 99524975, "vector": [-25, 17, 66, 24, -67, -8, 118, 78, -117, -80, 121, 15, 121, -112, 71, 112]} +{"id": 43152977, "vector": [32, -49, 6, -103, 104, -82, 52, -27, 111, -22, -75, -29, -70, -97, 99, -71]} +{"id": 51472275, "vector": [-19, 32, -57, -49, -16, 25, 117, -107, -95, -57, 54, 72, -52, -68, -101, -85]} +{"id": 36189527, "vector": [20, 127, -15, 38, 81, 0, 76, 27, 105, 73, -123, -124, 0, 53, -23, 76]} +{"id": 48675329, "vector": [-115, 5, -81, 36, 37, 125, 83, 93, -28, 2, -96, -88, 46, -109, -21, -53]} +{"id": 56484900, "vector": [7, -45, -5, -105, 85, 22, -57, 77, 14, 45, -71, -82, 41, 39, 54, -107]} +{"id": 70036438, "vector": [121, -48, -54, -34, 81, -34, 54, 90, 102, -53, -17, 20, -20, -56, -49, -18]} +{"id": 37435892, "vector": [-128, 101, -82, -22, -17, -99, -124, -74, 117, -76, -66, -4, 34, 57, -74, 12]} +{"id": 55247455, "vector": [-118, -38, 2, 53, 25, -87, 68, -53, 87, 41, 3, 62, 33, -63, -73, 67]} +{"id": 20645197, "vector": [9, 1, 87, -86, 34, 73, 6, -58, -112, 38, 82, -125, 53, 73, -108, 46]} +{"id": 4199704, "vector": [-87, -99, -29, 82, -21, 11, 66, 62, 84, 22, 21, -116, -55, -88, -65, -96]} +{"id": 24953498, "vector": [56, 101, 105, 48, -82, 124, -15, 37, -75, -112, -105, 70, 81, -32, 115, -53]} +{"id": 44915235, "vector": [41, 119, 127, 39, -1, 121, 41, 96, 106, 110, 64, 58, 27, -48, 101, -35]} +{"id": 6497830, "vector": [-1, 16, 14, -26, -30, 125, -112, -113, 121, -111, -45, -16, 13, -22, -88, 127]} +{"id": 1431742, "vector": [28, 2, -50, 61, 30, 78, 6, -87, 111, -57, -67, -67, 59, 81, -111, 85]} +{"id": 74441448, "vector": [127, -16, -58, 73, -66, 81, 24, 39, 28, 36, 50, 106, -49, 25, 13, 117]} +{"id": 93515664, "vector": [-51, -80, -48, 106, 73, -41, -6, -30, -116, -5, -115, -24, -45, 7, -65, -87]} +{"id": 64055960, "vector": [-63, 120, -11, 108, 22, -80, 114, 80, -117, 122, 25, -51, 85, -84, 104, -74]} +{"id": 28734791, "vector": [19, 62, -40, -6, -52, -51, -64, -104, -124, 82, 15, 6, 86, -62, -9, 106]} +{"id": 67451935, "vector": [-127, 124, -13, 98, 91, -96, -105, -71, 24, -2, 94, -17, 67, -38, 101, -3]} +{"id": 37957788, "vector": [116, -104, 112, 76, -61, -5, 84, -24, 119, 37, 31, 127, 95, -74, 25, -62]} +{"id": 4978543, "vector": [73, -64, -31, -28, -22, 89, 77, 100, -116, -33, 79, 48, 79, 124, 37, -115]} +{"id": 78300864, "vector": [-29, -15, 31, 40, -52, -100, -82, 90, 31, 61, 100, -120, 64, 82, -83, -49]} +{"id": 17976208, "vector": [17, -111, 33, -39, 68, -31, -8, 84, 1, 30, -48, -127, -87, 71, -47, 88]} +{"id": 83083131, "vector": [10, 34, -42, 105, -10, 97, 28, 40, -54, 116, -61, -76, 5, 27, -102, -71]} +{"id": 60955663, "vector": [9, 87, -126, -66, -28, -4, 95, 8, 76, -53, -81, 31, 30, -108, 62, 79]} +{"id": 86242799, "vector": [37, 91, 100, -70, 98, -120, -53, -77, -14, -53, -20, 3, 22, -5, 103, -23]} +{"id": 15075176, "vector": [-92, -72, -117, -66, -75, 73, 98, -12, 121, -31, -64, 123, 99, -48, -67, -19]} +{"id": 9860195, "vector": [-64, 63, -20, 81, -105, -29, 109, 56, 125, 29, 45, 20, -107, 110, -19, 72]} +{"id": 59405277, "vector": [44, 86, 18, -125, 55, 85, -40, -75, -34, -127, -49, -5, -35, -92, 36, 35]} +{"id": 6157724, "vector": [-84, 52, -25, 72, 13, -73, 106, 64, -2, -45, 49, -2, -17, 78, 88, 126]} +{"id": 4668450, "vector": [-49, 41, 88, -114, -87, 7, -123, 92, -125, 81, 71, -52, -1, 22, 57, -60]} +{"id": 14731700, "vector": [9, 44, 103, 89, -55, -121, -55, -24, -12, 98, 84, -88, 94, -17, 67, 11]} +{"id": 20002147, "vector": [99, 26, -5, 63, 108, -105, 102, -68, -20, 73, 68, -32, 0, 124, 97, -80]} +{"id": 40781854, "vector": [-14, -40, -107, 75, -85, -121, 31, -116, 13, -48, -114, -17, 0, -12, -93, -127]} +{"id": 6819644, "vector": [34, 88, 65, 73, 97, 75, 17, 119, -8, 69, -50, -57, -87, 123, -102, -32]} +{"id": 66319530, "vector": [1, 75, 26, 0, 11, -69, -105, 60, 102, -5, 5, 42, 127, 41, 54, -55]} +{"id": 65454636, "vector": [-35, 12, -128, -61, -42, 11, -65, -53, 111, 80, -43, -70, 60, 26, -82, -102]} +{"id": 11161731, "vector": [-48, 56, -67, 10, -59, 27, -61, -12, -93, -122, -87, -37, -3, 42, -6, -22]} +{"id": 67030811, "vector": [-98, -85, 108, -97, 94, 57, 90, -7, -98, 101, 114, -21, 19, 11, 80, 48]} +{"id": 92398073, "vector": [-53, 82, 70, -56, -90, 93, -37, -110, 23, -16, 106, 75, 118, -58, -71, -116]} +{"id": 20885148, "vector": [7, -13, 60, -37, 100, -49, 32, -52, 63, 25, -74, 82, -49, -128, -85, 93]} +{"id": 77284619, "vector": [11, 111, -43, 94, -29, 110, 77, 3, 71, 11, -77, 112, -45, -105, -58, -79]} +{"id": 10309525, "vector": [-32, -22, 62, -54, 42, -92, 35, 89, 85, 17, 64, 35, 18, -117, 67, -19]} +{"id": 67474219, "vector": [-103, -8, -121, 10, -18, 112, -115, 118, -10, 35, 118, 82, -111, -43, 125, -52]} +{"id": 53802686, "vector": [-126, -13, -99, 111, 112, 69, 120, 118, -105, 56, 24, -48, -25, -54, -75, 74]} +{"id": 15948937, "vector": [-38, -33, -76, 108, 112, 65, -51, -109, -17, 94, 7, -6, 104, 43, -121, 117]} +{"id": 98948034, "vector": [-71, -103, -52, -58, -107, 28, -63, 24, 77, -104, -85, 82, 73, -54, 60, 2]} +{"id": 18466635, "vector": [110, 67, -45, 65, -44, 31, -46, -127, -85, 96, -56, 26, -68, -107, 126, -14]} +{"id": 34667286, "vector": [-119, -121, 27, -65, 10, 31, -91, -106, 83, 26, -9, -37, 110, -58, 60, -88]} +{"id": 30218878, "vector": [71, 90, -85, -84, 19, -113, -113, 79, -119, 83, 56, 64, 72, 77, -125, -56]} +{"id": 32250045, "vector": [17, 104, 77, 47, 76, -71, -30, -80, -56, -6, -85, -34, -42, 66, -125, -44]} +{"id": 73021291, "vector": [-31, 7, 41, 80, -36, -117, 15, 10, -8, -100, -116, 24, -87, -1, -124, 28]} +{"id": 27325428, "vector": [-122, 61, -125, 19, -10, -118, -122, -43, 3, 3, -126, -20, 80, 6, -47, -12]} +{"id": 90061527, "vector": [-12, -99, 54, 94, -51, 61, -85, 45, -115, 67, -57, -21, -104, 95, 82, 13]} +{"id": 51590803, "vector": [16, 47, 110, 104, -86, -102, -44, 87, 11, 60, 54, 16, -84, -83, -83, -30]} +{"id": 3294781, "vector": [42, 69, -123, 0, -23, -61, 36, 65, -81, -90, -27, -66, -62, -87, 90, -31]} +{"id": 59501487, "vector": [-4, 0, 78, -57, -109, 67, 69, 52, 51, 78, 111, -104, -45, 75, 43, 54]} +{"id": 72357096, "vector": [-86, 91, 38, -31, -32, 2, -3, -65, 32, -77, 36, -91, 71, 63, -26, -122]} +{"id": 17365188, "vector": [8, -86, 89, 112, 79, 26, 48, 4, 96, 8, 61, -104, 120, 95, 102, 42]} +{"id": 72495719, "vector": [94, -119, -90, -2, 94, -72, -24, 119, -103, -4, 77, -21, -79, 4, 2, 7]} +{"id": 77272628, "vector": [85, -21, -28, 81, 35, 116, 96, -78, 84, 113, 1, -94, -67, 24, -67, -91]} +{"id": 20140249, "vector": [-1, 42, -116, -78, -66, 123, 4, -85, 1, 91, -21, 41, -107, -35, -115, -105]} +{"id": 824872, "vector": [56, -82, 19, 104, 127, -64, -21, -118, -52, 5, 23, -104, -12, -93, -68, -101]} +{"id": 62496012, "vector": [-35, 119, -103, 7, -114, 28, -35, -110, -87, -19, 51, 0, 75, 46, -40, 115]} +{"id": 56424103, "vector": [-30, -119, 89, -32, 3, 46, 27, -85, -90, -9, 117, 46, 23, 46, 39, -2]} +{"id": 97011160, "vector": [-67, 33, -25, -117, -84, 77, 98, -5, -30, -21, -14, -92, 121, 61, -32, 12]} +{"id": 5822202, "vector": [-16, -111, 81, -127, 111, 47, -98, -14, -61, -5, 76, -103, 14, -76, -127, -69]} +{"id": 81677380, "vector": [85, 20, -128, 58, 78, 45, -31, -31, -101, -38, -101, -86, -95, 9, -49, -102]} +{"id": 22997281, "vector": [72, 102, -128, 0, 111, -111, -49, -16, -64, 115, 68, -122, 95, 14, -38, 64]} +{"id": 99226875, "vector": [-88, -49, -114, -104, 64, -115, -109, 98, -50, 68, -2, -87, 62, -56, 125, -109]} +{"id": 66663942, "vector": [-9, 121, -3, -45, 95, 57, -116, 113, -38, -29, 121, 56, 80, -42, 7, -32]} +{"id": 70420215, "vector": [-11, -83, 55, -35, 68, 7, 24, 117, 34, 108, 82, 77, -76, 95, 5, -51]} +{"id": 61982238, "vector": [18, -96, 45, -15, 111, -71, 42, -7, -47, -54, -32, 48, -32, -52, -36, -112]} +{"id": 32159704, "vector": [63, -17, -87, 5, 121, -36, -83, -14, -15, 3, 6, -91, 81, 59, 59, -34]} +{"id": 96193415, "vector": [-25, 80, -53, -97, 53, -60, 54, -40, 81, 81, -72, -55, 105, 74, -85, 65]} +{"id": 34497327, "vector": [-80, 109, 76, 16, 40, -111, -45, 0, -1, -15, -78, 46, -12, 102, 69, -111]} +{"id": 68128438, "vector": [-57, -120, 39, 36, -96, -42, -127, -6, 83, 6, -19, 70, 22, 14, -65, -123]} +{"id": 34493392, "vector": [-103, -106, -31, 43, 104, 25, 18, 72, -74, -92, -119, 84, -119, -84, 28, 115]} +{"id": 14626618, "vector": [48, -59, -37, -62, 93, -112, -21, -103, -6, 120, -10, -78, -106, -21, -39, 116]} +{"id": 62430984, "vector": [120, -63, 76, -18, -70, -33, -92, 92, -118, -117, 16, -5, -1, -20, -26, 1]} +{"id": 15163258, "vector": [42, -66, 63, -94, 63, -83, -70, -78, -109, -25, 82, -60, 41, 69, 80, 127]} +{"id": 94539824, "vector": [-33, 95, 105, -13, -11, 57, -119, 112, -3, -125, -9, -77, -24, 46, 126, -106]} +{"id": 11757872, "vector": [-95, -42, 7, 112, 89, 0, 75, -31, 26, -39, -16, -90, 5, -75, 38, 75]} +{"id": 77072625, "vector": [-39, 84, -40, -3, -55, -67, 29, -90, -120, -57, 68, -45, -5, 110, -44, 110]} +{"id": 57658654, "vector": [-26, -39, 72, -89, -21, 40, 39, -72, 0, 20, 118, -72, -66, -119, -80, 123]} +{"id": 30764367, "vector": [64, 9, -103, -5, -50, -47, -14, 59, 11, 90, -127, -10, -2, -10, -9, 112]} +{"id": 59614746, "vector": [-105, 11, -42, 108, 74, 39, 61, -94, 49, -51, -76, 112, 14, -56, 54, 48]} +{"id": 9603598, "vector": [117, 119, -1, 112, 73, 27, 30, 79, 127, 116, -98, -69, -30, 114, -88, -45]} +{"id": 84166196, "vector": [22, 43, -68, 122, -52, -9, -31, -41, -50, 83, 104, -87, -19, -105, -24, -67]} +{"id": 36930650, "vector": [42, -48, -87, 1, 39, -23, 39, -30, -9, -39, 89, -56, -86, 3, 82, 18]} +{"id": 89996536, "vector": [-116, -49, 84, -42, -37, 48, 102, -1, -119, -109, -40, 67, 103, -65, -71, -75]} +{"id": 34236719, "vector": [-77, 10, 67, -14, 96, -97, 109, 38, -73, 99, 3, 46, -54, -91, -33, -104]} +{"id": 79322415, "vector": [9, -14, 38, 27, -1, -20, -103, -120, 36, 62, -122, 83, 2, 61, 2, -54]} +{"id": 38008118, "vector": [-107, -76, 29, -94, 66, 92, 9, 64, -127, -34, -40, 25, 78, 28, -111, 89]} +{"id": 54663246, "vector": [-100, -50, 113, 102, 17, 90, -57, 43, 22, 68, 106, -68, 20, 69, -7, -11]} +{"id": 64098930, "vector": [104, 7, -74, -97, -90, 61, -3, -37, 96, 26, 114, -35, -114, 62, -104, 84]} +{"id": 6525948, "vector": [-102, -23, -34, -96, 14, 91, -89, 84, -106, 12, 32, 51, -98, -9, -69, -108]} +{"id": 62762857, "vector": [22, -90, -66, 117, 9, 50, -78, -38, 110, -101, 48, 112, -30, 34, 81, 101]} +{"id": 14723410, "vector": [125, 73, 0, 96, 91, 63, 7, 100, 45, 24, -122, 123, -36, -86, 126, -77]} +{"id": 69848388, "vector": [119, -126, 48, -109, -56, 78, -105, 74, 38, -72, 48, 125, -22, -98, -122, 80]} +{"id": 97783876, "vector": [-103, 85, 12, -28, -117, 75, 86, 40, 41, -27, -84, -26, 40, 50, 81, 1]} +{"id": 98130363, "vector": [50, 71, 23, -116, 125, -32, -23, -16, 28, -84, -1, 23, 36, -104, 69, -86]} +{"id": 1197320, "vector": [11, 73, 32, -53, -5, 99, 29, -112, 85, -108, -35, -89, 36, -51, 64, 43]} +{"id": 26734892, "vector": [-15, -98, 76, -29, -59, 125, -26, 36, 61, 52, 9, 85, -75, -117, -126, -88]} +{"id": 17857111, "vector": [55, -115, -61, -82, -84, 76, 25, 98, 105, 96, 32, -21, -54, -31, -23, -78]} +{"id": 74452589, "vector": [37, -17, -37, 98, -10, 117, 123, 22, -51, -21, 11, 41, 113, 103, -97, 74]} +{"id": 7919588, "vector": [22, 39, -107, 28, 66, 71, 34, -96, 110, 39, 109, -74, 57, -16, -32, 56]} +{"id": 20836893, "vector": [-13, -54, 0, -111, 92, -36, 45, -3, 52, 54, 50, -25, 87, 38, -87, -107]} +{"id": 77187825, "vector": [23, 96, 117, -92, -64, 93, 85, -76, 79, 82, 64, -38, -106, -104, -18, 54]} +{"id": 81853704, "vector": [15, -18, -5, 70, 65, -49, 41, -20, 55, 4, 51, 114, -56, 81, 86, -123]} +{"id": 55669657, "vector": [-50, -91, 32, 54, 22, 18, -44, 118, 15, 65, -57, 3, 95, -10, -84, -38]} +{"id": 38256849, "vector": [-84, -28, 78, 67, -2, 20, -114, -10, 109, 94, -11, -92, -60, -110, -118, 99]} +{"id": 41092102, "vector": [66, 48, -30, 104, 24, -87, -107, 92, -88, 15, 9, 93, -116, -42, -8, 57]} +{"id": 24591705, "vector": [81, -43, 37, 95, -36, 53, -40, -37, 93, -4, 49, 19, 127, 64, -109, -24]} +{"id": 67227442, "vector": [124, 117, 60, 28, 34, -98, -13, 7, 24, -17, 9, 8, 103, -37, 70, -35]} +{"id": 54606384, "vector": [-3, -76, 0, 27, 74, 99, -54, -20, 59, 108, 26, 64, -65, -114, -9, 83]} +{"id": 61712234, "vector": [-8, -121, 102, -128, 6, 99, -31, -9, 114, 48, 34, -111, 98, -49, 41, -77]} +{"id": 15015906, "vector": [5, 96, -77, -15, -11, 114, 59, -78, -51, -128, -121, 12, -16, 90, -23, 67]} +{"id": 24168411, "vector": [-81, 98, -12, -25, 12, -31, 10, -53, -37, -101, 33, 123, 57, 79, -60, -59]} +{"id": 86022504, "vector": [10, -93, -85, -25, 27, -91, -101, -10, -99, 67, -91, -41, 70, -107, -118, -19]} +{"id": 87710366, "vector": [-51, 108, -19, -34, 80, 53, -128, 109, -112, 102, 72, -85, 86, 40, -98, -9]} +{"id": 53547802, "vector": [109, 114, -8, 35, 112, -63, 53, 32, 4, 34, -7, 125, -60, -27, 65, -6]} +{"id": 20681921, "vector": [-108, -102, 55, -23, -97, 17, 58, -83, 66, 77, -75, 19, 31, -101, -120, -118]} +{"id": 16595436, "vector": [-127, 127, 15, -106, -70, -51, 34, 68, -33, -86, -51, 62, 112, 78, -42, 25]} +{"id": 65715134, "vector": [-38, 48, 92, -128, 123, -43, -97, -3, 54, 60, 48, 116, -46, -88, 34, 89]} +{"id": 3633375, "vector": [80, 10, -69, 22, 28, -98, -53, 100, 89, -63, -118, 40, 48, 52, -108, -96]} +{"id": 32699744, "vector": [-100, -39, 27, -127, -77, -26, -2, -87, 56, -49, -22, 4, 83, -49, 36, -86]} +{"id": 24058273, "vector": [119, 93, 70, 96, 44, -10, -69, -123, 114, 65, 29, 63, -39, -103, 23, 113]} +{"id": 33201905, "vector": [106, 59, 50, 18, -18, -40, 79, 5, -124, 52, -5, 19, 125, 33, -71, 76]} +{"id": 37166608, "vector": [61, -106, -33, -76, 98, 16, -10, 42, -47, -27, 38, -37, -63, -106, -97, 111]} +{"id": 36780454, "vector": [-67, -45, -107, 45, -107, -90, -62, 99, 82, 107, -27, 53, -81, 60, -36, 114]} +{"id": 65338021, "vector": [-108, 38, 119, -23, 48, -107, -13, 74, 73, 76, 29, -77, -107, -18, 2, -128]} +{"id": 6153406, "vector": [-6, -2, -88, -21, 34, -42, -16, -65, 113, -82, -89, -96, 2, 88, -111, -10]} +{"id": 45794415, "vector": [-46, -62, 46, 121, 82, -59, 111, 44, 7, -111, 123, -12, -7, -35, -22, -82]} +{"id": 53393358, "vector": [119, 11, -13, 45, -115, 116, 76, -62, 66, -87, 12, 89, -17, 89, -128, -66]} +{"id": 70596786, "vector": [47, 91, -78, -80, -39, 114, 48, 31, -100, -4, -60, -26, 115, 53, -23, 30]} +{"id": 19898053, "vector": [64, 49, -87, 34, -85, 21, -9, -45, 127, 111, -50, 70, -57, -102, -44, 35]} +{"id": 73392814, "vector": [-7, 15, 84, -76, -79, -72, -14, 22, -86, 31, -51, -48, -21, 12, -110, -21]} +{"id": 83269727, "vector": [-36, -48, 66, -91, -65, -54, -66, -76, 4, -118, -80, 8, 126, 121, 127, 93]} +{"id": 89699445, "vector": [46, -120, -25, -61, -76, 61, 25, 42, -66, 45, 112, 95, 0, -106, -35, -126]} +{"id": 78884452, "vector": [-71, 11, -126, 56, 45, -97, 80, -97, -76, -100, 91, -55, -78, 114, 91, 45]} +{"id": 62428472, "vector": [65, 81, -94, -72, 0, -88, -13, 122, -7, -84, -90, 116, 61, 21, -17, -101]} +{"id": 55770687, "vector": [1, 47, 8, -2, -5, -92, -66, 4, 100, -52, 37, 83, -36, 113, 57, -95]} +{"id": 11581181, "vector": [111, -83, 25, -32, 52, -13, 123, -121, 57, 22, -113, -122, -24, -25, 35, -28]} +{"id": 89499542, "vector": [-94, -78, 89, -96, 13, 123, 21, -41, -41, 26, 118, -79, 100, -125, -70, -74]} +{"id": 99333375, "vector": [78, 105, 7, -17, -119, -53, -43, 111, 91, -8, -91, 26, -82, 91, -119, -12]} +{"id": 69605283, "vector": [122, 34, -48, 85, -105, -37, -46, 74, -55, 106, -123, 122, -19, -87, 75, -88]} +{"id": 31727379, "vector": [108, -79, -54, 120, 69, 27, 75, -116, -25, 116, -120, -70, -101, 106, 38, -2]} +{"id": 33565483, "vector": [-74, -73, 117, 97, -67, 62, -17, -6, 86, 108, -100, -26, 44, 47, -103, -115]} +{"id": 17037369, "vector": [-46, 95, -78, 67, 24, -7, -91, -20, -91, -83, 36, 40, -63, 117, 20, -122]} +{"id": 84187166, "vector": [-92, 117, 50, 20, 101, -32, 62, 109, 72, 20, -39, -51, 60, 70, 55, 51]} +{"id": 21070110, "vector": [58, -62, -1, 18, 18, 50, 23, 37, -31, 19, 39, 2, 72, 100, 96, 67]} +{"id": 85695762, "vector": [81, 90, -95, 57, -61, 25, 108, 26, 7, -86, 14, 97, -111, -32, -36, 97]} +{"id": 21993752, "vector": [19, -128, 75, -38, -99, -111, 20, -64, -3, 34, -110, 15, -99, -19, -42, 72]} +{"id": 38658347, "vector": [-69, -93, -28, 103, 52, -120, 26, -103, -69, -83, 52, 50, -103, 93, -9, 37]} +{"id": 68939068, "vector": [-42, -33, 25, 53, -39, -13, 69, 106, 91, -114, -101, 40, 53, 107, -89, 117]} +{"id": 20486294, "vector": [-48, 10, -76, 98, -22, 40, -95, -22, 115, -74, 36, -27, -128, -24, -128, 104]} +{"id": 35456853, "vector": [-24, 96, 104, -8, 21, 62, 97, 99, 44, -50, 40, -85, -36, 23, -89, -113]} +{"id": 26744917, "vector": [108, -122, 15, 35, -56, 70, 119, 0, 15, 91, -101, 88, 98, -41, 25, 76]} +{"id": 50567636, "vector": [29, -87, 24, -123, -66, -79, -56, -74, -101, -64, 83, 58, -18, 81, -112, -53]} +{"id": 67513640, "vector": [-6, 123, 68, 44, -124, -64, -28, -9, 83, -2, 73, 93, 42, -53, -13, -32]} +{"id": 53648154, "vector": [123, -42, 78, -20, -1, 90, -1, 99, 68, 80, 62, 55, -25, -99, 110, -56]} +{"id": 39986008, "vector": [119, 72, -77, -79, -35, 15, 45, 32, -63, 114, -5, -40, -50, -71, -5, -83]} +{"id": 73222868, "vector": [45, -62, -27, 35, 106, 20, -96, -79, -67, 103, -77, -103, -72, -94, -113, 24]} +{"id": 56955985, "vector": [-120, -95, -10, -49, 124, 71, -22, -18, 36, 125, -22, -88, 99, -79, 75, 107]} +{"id": 82178706, "vector": [-85, 78, 40, 49, -20, 102, 24, -6, 40, -113, 119, 73, -28, 123, 84, 107]} +{"id": 37192445, "vector": [3, 19, 125, 61, 54, -64, 105, 110, 120, 112, 111, 117, 81, -79, 122, 62]} +{"id": 22879907, "vector": [-7, 12, 50, -20, 22, -114, 64, 117, -75, 125, 35, -34, -90, -68, 112, 71]} +{"id": 45848907, "vector": [73, -102, 49, 74, 33, 56, -98, 58, -81, 0, 55, 44, 110, 114, -67, -34]} +{"id": 62232211, "vector": [-110, 78, 121, 32, 90, -103, 29, -43, -57, -51, -37, 59, 65, 116, -53, 47]} +{"id": 45381876, "vector": [-83, -63, -73, 1, 117, -125, -17, -44, -29, -97, 74, 57, 61, -9, 78, 75]} +{"id": 81898046, "vector": [-46, -89, -126, 117, 90, 12, -23, 24, 30, 80, 9, -74, 8, -78, 10, 114]} +{"id": 22942635, "vector": [40, 79, 61, 102, 34, 124, -13, 81, -93, -109, -113, -33, 112, 81, 14, -34]} +{"id": 30891921, "vector": [-33, 69, 105, -84, 104, -30, -13, -35, 83, -108, 79, 16, 1, -85, -75, 123]} +{"id": 44844121, "vector": [41, 47, -104, -22, -85, 54, 71, -51, -118, 114, -84, 17, -121, 78, 112, 44]} +{"id": 62490109, "vector": [2, 39, 106, -27, -50, -93, -5, 7, 76, -66, 53, 4, 8, 95, 37, -126]} +{"id": 21673260, "vector": [51, 1, -124, -3, 20, -97, 72, 106, -110, 52, -59, 68, 122, 39, 97, 67]} +{"id": 45996863, "vector": [33, 32, -22, -84, -90, 57, 11, -66, -89, -85, 77, -35, 102, 12, 64, -114]} +{"id": 8128637, "vector": [111, 55, 93, 50, 91, 47, 100, -118, -45, -97, 124, 104, 86, -89, 10, -32]} +{"id": 12664567, "vector": [22, -10, 24, 105, 41, 70, 93, -18, 22, -44, 17, -95, -98, 93, 77, 12]} +{"id": 15536795, "vector": [118, -20, 59, -33, -89, 0, -27, 62, -121, 124, 119, 112, 66, 78, -7, 74]} +{"id": 77413012, "vector": [103, -85, -74, -16, -115, -48, 119, -15, -70, 36, -69, -2, 46, 38, -89, -74]} +{"id": 26292919, "vector": [-118, -38, 11, 121, -17, -127, -27, -123, 82, -102, -114, -30, -5, 100, 66, -21]} +{"id": 80246713, "vector": [-125, -22, -14, 2, 41, -110, 97, 125, 51, -26, -26, 100, -38, 32, 13, 53]} +{"id": 89078848, "vector": [-98, -76, -110, -52, -10, 50, -49, -53, 70, -86, 111, -42, -7, -19, 61, 7]} +{"id": 15902805, "vector": [59, 105, -118, 41, -49, -109, 18, 73, 93, 90, -4, -65, 31, -48, -58, 87]} +{"id": 8807940, "vector": [-64, 47, -39, -20, 32, 120, 51, -19, 21, -94, 83, -27, -52, -46, 51, 108]} +{"id": 36396314, "vector": [-59, -49, -86, 90, 82, 15, 89, 49, -106, 73, -77, 86, 80, 53, 99, -35]} +{"id": 53632373, "vector": [49, -121, -113, 92, -8, 94, 77, -26, -117, 126, 44, 15, 45, -48, -70, 106]} +{"id": 84127901, "vector": [-83, 17, -5, 48, -18, 83, 30, -120, 61, 71, 116, -17, 51, 123, 77, -52]} +{"id": 57240218, "vector": [-86, 52, -122, -67, -88, -69, 109, -120, 54, -97, -122, 108, 111, -77, -62, -32]} +{"id": 40984766, "vector": [28, 49, 7, -14, 61, -126, 0, 118, -70, 60, 48, 19, -53, 119, 121, -2]} +{"id": 57803235, "vector": [-127, -78, 31, 89, 48, -84, -64, 18, -2, -66, -67, -74, 63, 68, 110, -44]} +{"id": 6986898, "vector": [-1, -59, -83, -35, 93, -91, 20, -95, -2, -126, 80, 47, 48, -74, -101, 94]} +{"id": 50702367, "vector": [-111, 29, -41, -111, -83, 71, -10, 114, -4, -31, -25, 51, -128, 69, 60, 55]} +{"id": 17386996, "vector": [-55, -17, 12, 92, -80, 37, -67, -51, 72, -49, -54, 9, 18, -105, -106, 64]} +{"id": 40686254, "vector": [34, 120, -26, 57, 106, 91, -26, 90, -122, 100, -21, -27, 66, 64, -118, -120]} +{"id": 83155442, "vector": [-127, -116, 8, -126, 70, 99, 121, -45, -81, -124, 76, 114, 6, 98, 71, 19]} +{"id": 17081350, "vector": [15, 19, -28, 73, -10, 94, -17, -58, -34, 123, -29, -71, 3, -31, -53, -128]} +{"id": 96709982, "vector": [119, 47, 26, 48, -77, -71, 49, -101, -112, -27, 126, -108, 89, -28, -90, -84]} +{"id": 82726008, "vector": [-23, -9, -48, 84, 47, 41, -38, -22, 74, 80, 48, 99, 34, 11, 31, -3]} +{"id": 68110330, "vector": [101, 85, 42, -71, -53, 1, -98, -50, 10, -63, -46, -21, 70, 93, -113, -98]} +{"id": 17385531, "vector": [48, -7, -120, -4, -65, -37, 24, 12, -27, 60, 118, -66, -115, -28, 23, 79]} +{"id": 1250437, "vector": [19, -32, 32, 49, 96, -87, -67, -88, 69, -123, 114, -64, -125, 32, -90, 47]} +{"id": 99515901, "vector": [-27, 75, 77, 113, 44, -16, 92, -14, -91, -36, 90, -128, -115, -64, 64, -106]} +{"id": 38009615, "vector": [32, 40, 33, -106, -57, -65, -73, -59, -91, 63, 112, 111, -91, -29, -2, -125]} +{"id": 52261574, "vector": [55, -66, -62, 42, 121, -76, 24, -54, 93, -105, 14, -49, 2, -127, -10, 63]} +{"id": 12416768, "vector": [6, 74, -29, 41, -13, -31, 67, 26, 106, -64, 63, 41, -9, 63, 23, 120]} +{"id": 76330843, "vector": [-34, 118, -88, 58, -122, -78, 84, -36, -49, -60, 67, 67, -61, 93, 107, 65]} +{"id": 22129328, "vector": [-24, -20, 13, 13, -55, 51, 12, 116, -76, -88, -2, 82, -9, -83, -86, 108]} +{"id": 86460488, "vector": [100, 7, 45, -32, -11, -60, -116, -34, -6, -127, -55, -100, -83, 44, -48, -22]} +{"id": 14349098, "vector": [-90, -52, 36, 87, 86, 108, 85, 82, 52, -26, -37, -121, 53, 109, 43, 122]} +{"id": 46870723, "vector": [115, -41, 127, 107, -15, -25, 12, 46, -73, 126, 72, 15, 38, 72, 99, -9]} +{"id": 99224392, "vector": [-67, 25, 8, -96, 22, 10, 108, 38, 95, -86, -6, -3, -127, 40, -90, -113]} +{"id": 64087743, "vector": [85, -34, -25, 61, 3, 102, -62, 90, -110, 61, -35, -116, -121, 34, -114, -40]} +{"id": 99604946, "vector": [123, -103, 7, 12, 43, 65, 71, -2, -28, -31, 46, -103, 116, -109, -25, -104]} +{"id": 4064751, "vector": [41, -5, 66, -79, -99, -121, 110, 6, -45, 27, -124, 116, 96, 79, 71, -19]} +{"id": 38061439, "vector": [-51, -123, 8, -46, -13, -119, -77, -81, -121, -35, 37, 13, -67, 17, 26, -2]} +{"id": 33061250, "vector": [11, 123, 14, 126, -33, -40, 0, 10, -82, 94, 66, -55, -89, 97, -77, 28]} +{"id": 3773993, "vector": [22, -59, 92, 74, 87, 98, 100, 3, -25, -51, -87, 116, 109, -28, 60, -55]} +{"id": 48774913, "vector": [23, 1, 33, 96, -25, -4, -107, 118, -66, -2, 59, -49, 56, -34, 9, 94]} +{"id": 17727650, "vector": [2, 80, -96, 104, -66, -107, 47, -48, -66, 113, -57, 123, 76, 32, 94, 9]} +{"id": 30395570, "vector": [88, 118, 9, 102, 38, -37, 75, 122, -93, -92, 17, -96, 18, 122, -112, 124]} +{"id": 47738185, "vector": [101, 124, -73, -94, -122, 45, 90, -123, -46, 20, -49, -4, -18, -109, 31, 88]} +{"id": 82779622, "vector": [3, 30, 99, 45, -48, 73, 120, 35, 91, 39, -92, 114, 102, 79, -70, 31]} +{"id": 24619760, "vector": [11, 8, -115, -5, -88, -102, -6, -23, -82, -2, 5, 65, 117, 70, -60, 31]} +{"id": 44177011, "vector": [87, -128, 108, 67, -101, -50, 81, -51, 99, -92, -94, 30, -23, -38, -84, -37]} +{"id": 89046466, "vector": [-89, -98, 94, 88, -45, 121, 16, -32, 118, 84, -52, -104, 41, -50, -92, 22]} +{"id": 21397057, "vector": [103, 117, 22, 103, -113, 92, -92, 65, 85, -103, 90, -57, -80, 41, -75, 60]} +{"id": 44060493, "vector": [-101, 45, 51, 39, 22, 6, 56, -48, 108, 9, -125, -2, -111, 40, -53, -13]} +{"id": 30787683, "vector": [57, 105, -71, 82, -126, -103, 76, -31, 27, 39, 56, 58, 17, 58, 85, -65]} +{"id": 54263880, "vector": [109, -69, -104, 43, -46, -32, -88, 21, 99, -21, -126, -109, 119, -79, -77, 13]} +{"id": 29819940, "vector": [-126, -45, -94, 43, -54, -27, 10, 19, -3, -94, 37, -18, -55, -112, 46, -51]} +{"id": 61373987, "vector": [19, 4, 105, 50, -80, 72, 69, 68, -44, 89, -61, -32, -125, 24, -63, -68]} +{"id": 5832946, "vector": [-69, 23, -103, -6, -86, 37, -124, 61, 122, 44, 47, -91, 110, -107, -69, -35]} +{"id": 20867149, "vector": [-30, 3, 30, -8, 27, 14, -120, 122, 95, 2, 77, 92, -121, 18, -2, -44]} +{"id": 1569515, "vector": [-47, -54, -40, 85, -9, 118, -48, 74, 127, 72, 86, 121, 32, 19, -47, 31]} +{"id": 30771409, "vector": [-92, 49, -83, -27, -51, 107, -74, -18, 38, -71, -125, -108, 106, 103, -105, -80]} +{"id": 7502255, "vector": [106, 90, 0, -86, 22, -117, -126, -30, -67, -5, -122, 73, -11, 22, 95, -105]} +{"id": 71657078, "vector": [127, -92, -68, -28, 115, -71, 92, 0, 48, -5, 0, 120, 15, -24, -95, 127]} +{"id": 46540998, "vector": [-63, 38, -110, -29, 54, 44, 32, 95, 111, 55, -26, -23, -93, 106, 109, -126]} +{"id": 70004753, "vector": [40, 90, 97, -91, -106, -37, -98, -114, -14, -19, 96, -18, -2, 63, 121, 39]} +{"id": 34946859, "vector": [109, 47, 9, -13, -113, 15, 99, -118, -34, 69, 107, 48, -37, 41, -99, 3]} +{"id": 176257, "vector": [72, 51, 39, 9, 40, -17, 86, -75, -14, -48, 117, 116, 77, -61, -77, -50]} +{"id": 54058788, "vector": [-20, 68, -113, -85, -2, -31, -20, 126, 19, 115, 60, -13, 45, -36, -78, 60]} +{"id": 83789391, "vector": [61, 63, -7, 10, 30, -117, -72, -121, -61, -69, 26, 111, -11, 104, 47, -45]} +{"id": 70727211, "vector": [44, -101, -105, 6, 77, -88, 124, -55, -124, 81, 66, 26, 76, -122, -22, -48]} +{"id": 87598888, "vector": [-21, -123, 20, 112, -96, 16, -124, -97, 9, -116, 7, 29, -114, -9, 19, 80]} +{"id": 31733363, "vector": [34, -60, 29, -124, -117, -54, -116, 79, 91, -54, -54, -84, -15, -37, -47, 60]} +{"id": 99861373, "vector": [-58, -106, -63, -117, -65, 87, 3, -22, 41, 98, 67, 118, -91, 49, -45, -60]} +{"id": 40534591, "vector": [59, -94, -22, -116, -100, 18, 38, -125, -30, -75, -108, -121, 9, -52, 115, 73]} +{"id": 10453030, "vector": [-106, -84, 46, -78, 48, -35, -87, 91, 14, -117, 49, -46, 105, 127, 25, 7]} +{"id": 32151165, "vector": [-36, -69, 95, -128, 101, -5, 3, -25, -38, 16, -118, -78, 115, -40, -37, 19]} +{"id": 79821897, "vector": [-9, 122, 24, -53, -22, -76, 55, -72, -83, -9, -30, 40, -22, -56, 17, -62]} +{"id": 96318094, "vector": [-30, 38, 29, -8, -6, -15, -52, -87, -123, -60, 59, -123, -42, 93, -63, -96]} +{"id": 59318837, "vector": [-100, 103, -103, -1, -65, 33, -127, -71, -11, -110, 99, 64, 42, -74, 45, 98]} +{"id": 6871053, "vector": [62, -117, 112, -126, 61, -96, 94, 38, -104, 100, -95, 66, 115, -76, -94, 89]} +{"id": 40152546, "vector": [-6, 85, 10, 109, 85, 36, -51, -124, -92, 8, -52, 22, -74, 6, -22, 19]} +{"id": 79738755, "vector": [112, -73, -45, -23, 107, -125, -35, 91, 81, 63, 91, -2, -108, -86, -116, -64]} +{"id": 93790285, "vector": [38, 62, 53, 83, -101, 2, 109, -22, 112, 117, 96, 94, -13, -76, -80, -88]} +{"id": 45995325, "vector": [-72, -107, -41, -52, 67, 29, -14, 23, -31, -86, 113, 60, 34, 67, 86, -104]} +{"id": 40027975, "vector": [41, -43, -21, 125, -14, 54, 4, -121, 114, 63, 120, -119, 82, 81, -82, 115]} +{"id": 15064655, "vector": [-48, 4, 78, -44, 127, 7, 61, -87, -31, -82, 123, 22, 29, 4, 11, 3]} +{"id": 37891451, "vector": [-105, -60, -38, 77, -117, 114, -10, -12, -72, 16, 1, 9, 124, -81, -99, 45]} +{"id": 247198, "vector": [-102, -79, 52, 36, -117, -106, -123, 39, 45, -19, -114, -106, 42, 107, 19, -5]} +{"id": 94076128, "vector": [-53, 68, -32, -19, -47, -49, 112, 47, 97, 28, -39, -118, -82, -79, -9, -18]} +{"id": 37224844, "vector": [119, 111, -127, 48, 26, 116, -80, 52, -68, 83, 2, -38, -119, 41, 96, 82]} +{"id": 16380211, "vector": [54, -22, 127, -93, 7, -19, 58, 109, -53, -55, 113, 51, -122, 5, -8, 104]} +{"id": 10597197, "vector": [-27, -110, -38, 37, 99, -7, -29, 29, -127, 127, 19, 11, 6, 116, 83, 83]} +{"id": 34432810, "vector": [30, -94, 79, 91, 96, 1, -118, -119, -19, 116, 119, 59, 94, 120, 89, -58]} +{"id": 94595668, "vector": [-110, 23, -60, -79, 31, -108, -46, -78, -57, -98, 37, 18, 60, 97, 57, 101]} diff --git a/src/testFixtures/java/org/opensearch/knn/KNNJsonIndexMappingsBuilder.java b/src/testFixtures/java/org/opensearch/knn/KNNJsonIndexMappingsBuilder.java new file mode 100644 index 000000000..165dfaf64 --- /dev/null +++ b/src/testFixtures/java/org/opensearch/knn/KNNJsonIndexMappingsBuilder.java @@ -0,0 +1,132 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn; + +import lombok.Builder; +import lombok.NonNull; +import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.core.xcontent.XContentBuilder; + +import java.io.IOException; + +/** + * Helper method to create knn index mapping in json string + * Here, we don't use any of the internal class so that it can mimic user request more closely. + */ +@Builder +public class KNNJsonIndexMappingsBuilder { + @NonNull + private String fieldName; + @NonNull + private Integer dimension; + private String nestedFieldName; + private String vectorDataType; + private Method method; + + public String getIndexMapping() throws IOException { + if (nestedFieldName != null) { + XContentBuilder xContentBuilder = XContentFactory.jsonBuilder() + .startObject() + .startObject("properties") + .startObject(nestedFieldName) + .field("type", "nested") + .startObject("properties") + .startObject(fieldName) + .field("type", "knn_vector") + .field("dimension", dimension); + addVectorDataType(xContentBuilder); + addMethod(xContentBuilder); + xContentBuilder.endObject().endObject().endObject().endObject().endObject(); + return xContentBuilder.toString(); + } else { + XContentBuilder xContentBuilder = XContentFactory.jsonBuilder() + .startObject() + .startObject("properties") + .startObject(fieldName) + .field("type", "knn_vector") + .field("dimension", dimension); + addVectorDataType(xContentBuilder); + addMethod(xContentBuilder); + xContentBuilder.endObject().endObject().endObject(); + return xContentBuilder.toString(); + } + } + + private void addVectorDataType(final XContentBuilder xContentBuilder) throws IOException { + if (vectorDataType == null) { + return; + } + xContentBuilder.field("data_type", vectorDataType); + } + + private void addMethod(final XContentBuilder xContentBuilder) throws IOException { + if (method == null) { + return; + } + method.addTo(xContentBuilder); + } + + @Builder + public static class Method { + @NonNull + private String methodName; + @NonNull + private String engine; + private String spaceType; + private Parameters parameters; + + private void addTo(final XContentBuilder xContentBuilder) throws IOException { + xContentBuilder.startObject("method").field("name", methodName).field("engine", engine); + addSpaceType(xContentBuilder); + addParameters(xContentBuilder); + xContentBuilder.endObject(); + } + + private void addSpaceType(final XContentBuilder xContentBuilder) throws IOException { + if (spaceType == null) { + return; + } + xContentBuilder.field("space_type", spaceType); + } + + private void addParameters(final XContentBuilder xContentBuilder) throws IOException { + if (parameters == null) { + return; + } + parameters.addTo(xContentBuilder); + } + + @Builder + public static class Parameters { + private Encoder encoder; + + private void addTo(final XContentBuilder xContentBuilder) throws IOException { + xContentBuilder.startObject("parameters"); + addEncoder(xContentBuilder); + xContentBuilder.endObject(); + } + + private void addEncoder(final XContentBuilder xContentBuilder) throws IOException { + if (encoder == null) { + return; + } + encoder.addTo(xContentBuilder); + } + + @Builder + public static class Encoder { + @NonNull + private String name; + + private void addTo(final XContentBuilder xContentBuilder) throws IOException { + xContentBuilder.startObject("encoder"); + xContentBuilder.field("name", name); + xContentBuilder.endObject(); + } + } + } + } +} diff --git a/src/testFixtures/java/org/opensearch/knn/KNNJsonQueryBuilder.java b/src/testFixtures/java/org/opensearch/knn/KNNJsonQueryBuilder.java new file mode 100644 index 000000000..c8cd8e6bd --- /dev/null +++ b/src/testFixtures/java/org/opensearch/knn/KNNJsonQueryBuilder.java @@ -0,0 +1,67 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn; + +import lombok.Builder; +import lombok.NonNull; +import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.core.xcontent.XContentBuilder; + +import java.io.IOException; + +/** + * Helper method to create knn query in json string + * Here, we don't use any of the internal class so that it can mimic user request more closely. + */ +@Builder +public class KNNJsonQueryBuilder { + @NonNull + private String fieldName; + @NonNull + private Object[] vector; + private Integer k; + private Float minScore; + private String nestedFieldName; + private String filterFieldName; + private String filterValue; + + public String getQueryString() throws IOException { + XContentBuilder builder = XContentFactory.jsonBuilder().startObject().startObject("query"); + if (nestedFieldName != null) { + builder.startObject("nested"); + builder.field("path", nestedFieldName); + builder.startObject("query"); + builder.startObject("knn"); + builder.startObject(nestedFieldName + "." + fieldName); + } else { + builder.startObject("knn"); + builder.startObject(fieldName); + } + + builder.field("vector", vector); + if (k != null) { + builder.field("k", k); + } + if (minScore != null) { + builder.field("min_score", minScore); + } + + if (filterFieldName != null && filterValue != null) { + builder.startObject("filter"); + builder.startObject("term"); + builder.field(filterFieldName, filterValue); + builder.endObject(); + builder.endObject(); + } + + builder.endObject().endObject().endObject().endObject(); + if (nestedFieldName != null) { + builder.endObject().endObject(); + } + + return builder.toString(); + } +} diff --git a/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java b/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java index df45fe2cc..32932ba9e 100644 --- a/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java +++ b/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java @@ -20,6 +20,7 @@ import org.opensearch.core.xcontent.XContentParser; import org.opensearch.index.query.MatchAllQueryBuilder; import org.opensearch.knn.common.KNNConstants; +import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.query.KNNQueryBuilder; import org.opensearch.knn.index.KNNSettings; import org.opensearch.knn.index.SpaceType; @@ -97,6 +98,7 @@ import static org.opensearch.knn.TestUtils.QUERY_VALUE; import static org.opensearch.knn.TestUtils.computeGroundTruthValues; +import static org.opensearch.knn.common.KNNConstants.VECTOR_DATA_TYPE_FIELD; import static org.opensearch.knn.index.SpaceType.L2; import static org.opensearch.knn.index.memory.NativeMemoryCacheManager.GRAPH_COUNT; import static org.opensearch.knn.index.util.KNNEngine.FAISS; @@ -188,8 +190,10 @@ protected void createBasicKnnIndex(String index, String fieldName, int dimension } /** - * Run KNN Search on Index + * Deprecated + * To better simulate user request, use {@link #searchKNNIndex(String, XContentBuilder, int)} instead */ + @Deprecated protected Response searchKNNIndex(String index, KNNQueryBuilder knnQueryBuilder, int resultSize) throws IOException { XContentBuilder builder = XContentFactory.jsonBuilder().startObject().startObject("query"); knnQueryBuilder.doXContent(builder, ToXContent.EMPTY_PARAMS); @@ -201,12 +205,20 @@ protected Response searchKNNIndex(String index, KNNQueryBuilder knnQueryBuilder, * Run KNN Search on Index with XContentBuilder query */ protected Response searchKNNIndex(String index, XContentBuilder xContentBuilder, int resultSize) throws IOException { + return searchKNNIndex(index, xContentBuilder.toString(), resultSize); + } + + /** + * Run KNN Search on Index with json string query + */ + protected Response searchKNNIndex(String index, String query, int resultSize) throws IOException { Request request = new Request("POST", "/" + index + "/_search"); - request.setJsonEntity(xContentBuilder.toString()); + request.setJsonEntity(query); request.addParameter("size", Integer.toString(resultSize)); - request.addParameter("explain", Boolean.toString(true)); request.addParameter("search_type", "query_then_fetch"); + // Nested field does not support explain parameter and the request is rejected if we set explain parameter + // request.addParameter("explain", Boolean.toString(true)); Response response = client().performRequest(request); assertEquals(request.getEndpoint() + ": failed", RestStatus.OK, RestStatus.fromCode(response.getStatusLine().getStatusCode())); @@ -347,6 +359,21 @@ protected String createKnnIndexMapping(String fieldName, Integer dimensions) thr .toString(); } + protected String createKnnIndexMapping(final String fieldName, final Integer dimensions, final VectorDataType vectorDataType) + throws IOException { + return XContentFactory.jsonBuilder() + .startObject() + .startObject("properties") + .startObject(fieldName) + .field("type", "knn_vector") + .field("dimension", dimensions.toString()) + .field(VECTOR_DATA_TYPE_FIELD, vectorDataType.getValue()) + .endObject() + .endObject() + .endObject() + .toString(); + } + /** * Utility to create a Knn Index Mapping with specific algorithm and engine */ @@ -391,6 +418,34 @@ protected String createKnnIndexMapping( .toString(); } + protected String createKnnIndexMapping( + String fieldName, + Integer dimensions, + String algoName, + String knnEngine, + String spaceType, + boolean docValues, + VectorDataType vectorDataType + ) throws IOException { + return XContentFactory.jsonBuilder() + .startObject() + .startObject("properties") + .startObject(fieldName) + .field(KNNConstants.TYPE, KNNConstants.TYPE_KNN_VECTOR) + .field(KNNConstants.DIMENSION, dimensions.toString()) + .field(VECTOR_DATA_TYPE_FIELD, vectorDataType.getValue()) + .field("doc_values", docValues) + .startObject(KNNConstants.KNN_METHOD) + .field(KNNConstants.NAME, algoName) + .field(KNNConstants.METHOD_PARAMETER_SPACE_TYPE, spaceType) + .field(KNNConstants.KNN_ENGINE, knnEngine) + .endObject() + .endObject() + .endObject() + .endObject() + .toString(); + } + /** * Utility to create a Knn Index Mapping with multiple k-NN fields */ diff --git a/src/testFixtures/java/org/opensearch/knn/TestUtils.java b/src/testFixtures/java/org/opensearch/knn/TestUtils.java index d41bbc0fd..28a60aecb 100644 --- a/src/testFixtures/java/org/opensearch/knn/TestUtils.java +++ b/src/testFixtures/java/org/opensearch/knn/TestUtils.java @@ -254,6 +254,7 @@ public static class TestData { public Pair indexData; public byte[][] indexBinaryData; public float[][] queries; + public String[][] groundTruthValues; public byte[][] binaryQueries; public TestData(String testIndexVectorsPath, String testQueriesPath) throws IOException { @@ -262,6 +263,13 @@ public TestData(String testIndexVectorsPath, String testQueriesPath) throws IOEx initBinaryData(); } + public TestData(String testIndexVectorsPath, String testQueriesPath, String groundTruthValuesPath) throws IOException { + indexData = readIndexData(testIndexVectorsPath); + queries = readQueries(testQueriesPath); + groundTruthValues = readGroundTruthValues(groundTruthValuesPath); + initBinaryData(); + } + private Pair readIndexData(String path) throws IOException { List idsList = new ArrayList<>(); List vectorsList = new ArrayList<>(); @@ -275,10 +283,10 @@ private Pair readIndexData(String path) throws IOException { idsList.add((Integer) doc.get("id")); @SuppressWarnings("unchecked") - ArrayList vector = (ArrayList) doc.get("vector"); + ArrayList vector = (ArrayList) doc.get("vector"); Float[] floatArray = new Float[vector.size()]; for (int i = 0; i < vector.size(); i++) { - floatArray[i] = vector.get(i).floatValue(); + floatArray[i] = Float.valueOf(vector.get(i).toString()); } vectorsList.add(floatArray); @@ -327,6 +335,28 @@ private float[][] readQueries(String path) throws IOException { return queryArray; } + private String[][] readGroundTruthValues(String path) throws IOException { + BufferedReader reader = new BufferedReader(new FileReader(path)); + String line = reader.readLine(); + + List stringList = new ArrayList<>(); + + while (line != null) { + String[] intStrings = line.split(","); + stringList.add(intStrings); + line = reader.readLine(); + } + reader.close(); + + String[][] docIdArray = new String[stringList.size()][stringList.get(0).length]; + for (int i = 0; i < docIdArray.length; i++) { + for (int j = 0; j < docIdArray[i].length; j++) { + docIdArray[i][j] = stringList.get(i)[j].trim(); + } + } + return docIdArray; + } + private void initBinaryData() { // Find medium value List flattenedVectors = new ArrayList<>(indexData.vectors.length * indexData.vectors[0].length); From 570320657bdb6607e1ff88c6eb875803f3d78f75 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Wed, 31 Jul 2024 14:39:53 -0700 Subject: [PATCH 312/416] [Backport 2.x] Refactor engine package structure (#1915) Refactors package structure for engines. First, it renames the package from util to engine. Next, it moves actual utils to the util package. Then, it puts the engines in separate sub-packages. Purpose for this is to separate between engines. Signed-off-by: John Mazanec Co-authored-by: John Mazanec --- CHANGELOG.md | 1 + .../org/opensearch/knn/bwc/FaissSQIT.java | 4 ++-- .../opensearch/knn/index/KNNIndexShard.java | 4 ++-- .../codec/BasePerFieldKnnVectorsFormat.java | 2 +- .../codec/KNN80Codec/KNN80CompoundFormat.java | 2 +- .../KNN80Codec/KNN80DocValuesConsumer.java | 6 ++--- .../KNN950PerFieldKnnVectorsFormat.java | 2 +- .../KNN990PerFieldKnnVectorsFormat.java | 2 +- ...KNNScalarQuantizedVectorsFormatParams.java | 2 +- .../{util => engine}/AbstractKNNLibrary.java | 4 +--- .../{util => engine}/DefaultHnswContext.java | 12 +++------- .../{util => engine}/DefaultIVFContext.java | 12 +++------- .../EngineSpecificMethodContext.java | 12 +++------- .../index/{util => engine}/JVMLibrary.java | 7 ++---- .../knn/index/{util => engine}/KNNEngine.java | 7 +++--- .../index/{util => engine}/KNNLibrary.java | 12 ++-------- .../knn/index/{ => engine}/KNNMethod.java | 11 +++------ .../index/{ => engine}/KNNMethodContext.java | 12 +++------- .../index/{ => engine}/MethodComponent.java | 12 +++------- .../{ => engine}/MethodComponentContext.java | 10 ++------ .../index/{util => engine}/NativeLibrary.java | 8 +++---- .../knn/index/{ => engine}/Parameter.java | 10 ++------ .../index/{util => engine/faiss}/Faiss.java | 15 +++++++----- .../index/{util => engine/lucene}/Lucene.java | 13 ++++++----- .../lucene}/LuceneHNSWContext.java | 16 +++++-------- .../knn/index/engine/model/QueryContext.java | 19 +++++++++++++++ .../index/{util => engine/nmslib}/Nmslib.java | 16 +++++++------ .../validation/ParameterValidator.java | 12 +++------- .../index/mapper/KNNVectorFieldMapper.java | 6 ++--- .../mapper/KNNVectorFieldMapperUtil.java | 4 ++-- .../knn/index/mapper/LegacyFieldMapper.java | 2 +- .../knn/index/mapper/LuceneFieldMapper.java | 4 ++-- .../knn/index/mapper/MethodFieldMapper.java | 4 ++-- .../index/memory/NativeMemoryAllocation.java | 4 ++-- .../memory/NativeMemoryEntryContext.java | 2 +- .../memory/NativeMemoryLoadStrategy.java | 4 ++-- .../knn/index/memory/SharedIndexState.java | 2 +- .../index/memory/SharedIndexStateManager.java | 2 +- .../knn/index/query/BaseQueryFactory.java | 2 +- .../knn/index/query/KNNQueryBuilder.java | 16 ++++++------- .../knn/index/query/KNNQueryFactory.java | 2 +- .../opensearch/knn/index/query/KNNWeight.java | 4 ++-- .../knn/index/query/RNNQueryFactory.java | 2 +- .../query/parser/KNNQueryBuilderParser.java | 4 ++-- .../knn/index/{ => util}/IndexUtil.java | 17 +++++++------- .../knn/index/{ => util}/KNNClusterUtil.java | 2 +- .../knn/index/util/QueryContext.java | 23 ------------------- .../org/opensearch/knn/indices/ModelDao.java | 2 +- .../opensearch/knn/indices/ModelMetadata.java | 6 ++--- .../org/opensearch/knn/jni/FaissService.java | 2 +- .../org/opensearch/knn/jni/JNIService.java | 4 ++-- .../org/opensearch/knn/jni/NmslibService.java | 2 +- .../org/opensearch/knn/plugin/KNNPlugin.java | 4 ++-- .../plugin/rest/RestTrainModelHandler.java | 2 +- .../opensearch/knn/plugin/stats/KNNStats.java | 2 +- .../suppliers/LibraryInitializedSupplier.java | 2 +- .../transport/TrainingModelRequest.java | 4 ++-- .../opensearch/knn/training/TrainingJob.java | 6 ++--- .../opensearch/knn/training/VectorReader.java | 2 +- .../index/AdvancedFilteringUseCasesIT.java | 2 +- .../knn/index/FaissHNSWFlatE2EIT.java | 3 ++- .../org/opensearch/knn/index/FaissIT.java | 3 ++- .../index/KNNCreateIndexFromModelTests.java | 3 ++- .../knn/index/KNNMapperSearcherIT.java | 2 +- .../knn/index/KNNMethodContextTests.java | 4 +++- .../opensearch/knn/index/LuceneEngineIT.java | 2 +- .../index/MethodComponentContextTests.java | 1 + .../org/opensearch/knn/index/NmslibIT.java | 3 ++- .../opensearch/knn/index/OpenSearchIT.java | 3 ++- .../opensearch/knn/index/SpaceTypeTests.java | 2 +- .../knn/index/VectorDataTypeIT.java | 2 +- .../KNN80Codec/KNN80CompoundFormatTests.java | 2 +- .../KNN80DocValuesConsumerTests.java | 6 ++--- ...NativeEngines990KnnVectorsFormatTests.java | 2 +- .../knn/index/codec/KNNCodecTestCase.java | 6 ++--- .../knn/index/codec/KNNCodecTestUtil.java | 2 +- ...alarQuantizedVectorsFormatParamsTests.java | 2 +- .../AbstractKNNLibraryTests.java | 3 ++- .../{util => engine}/KNNEngineTests.java | 5 +++- .../index/{ => engine}/KNNMethodTests.java | 12 +++------- .../{ => engine}/MethodComponentTests.java | 10 ++------ .../{util => engine}/NativeLibraryTests.java | 3 +-- .../index/{ => engine}/ParameterTests.java | 16 ++++--------- .../{util => engine/faiss}/FaissTests.java | 10 ++++---- .../{util => engine/lucene}/LuceneTests.java | 6 ++--- .../mapper/KNNVectorFieldMapperTests.java | 6 ++--- .../mapper/KNNVectorFieldMapperUtilTests.java | 6 ++--- .../index/mapper/MethodFieldMapperTests.java | 2 +- .../memory/NativeMemoryAllocationTests.java | 4 ++-- .../memory/NativeMemoryEntryContextTests.java | 4 ++-- .../memory/NativeMemoryLoadStrategyTests.java | 2 +- .../memory/SharedIndexStateManagerTests.java | 2 +- .../index/memory/SharedIndexStateTests.java | 2 +- .../knn/index/query/KNNQueryBuilderTests.java | 10 ++++---- .../knn/index/query/KNNQueryFactoryTests.java | 2 +- .../knn/index/query/KNNWeightTests.java | 4 ++-- .../knn/index/query/RNNQueryFactoryTests.java | 2 +- .../parser/KNNQueryBuilderParserTests.java | 2 +- .../knn/index/{ => util}/IndexUtilTests.java | 19 ++++++++------- .../index/{ => util}/KNNClusterUtilTests.java | 2 +- .../knn/indices/ModelCacheTests.java | 4 ++-- .../opensearch/knn/indices/ModelDaoTests.java | 4 ++-- .../knn/indices/ModelMetadataTests.java | 4 ++-- .../opensearch/knn/indices/ModelTests.java | 4 ++-- .../opensearch/knn/integ/BinaryIndexIT.java | 2 +- .../integ/BinaryIndexInvalidMappingIT.java | 2 +- .../knn/integ/FilteredSearchBinaryIT.java | 2 +- .../knn/integ/KNNScriptScoringIT.java | 2 +- .../knn/integ/NestedSearchBinaryIT.java | 2 +- .../opensearch/knn/integ/NestedSearchIT.java | 2 +- .../knn/integ/PainlessScriptIT.java | 6 ++--- .../opensearch/knn/jni/JNIServiceTests.java | 6 ++--- .../plugin/action/RestGetModelHandlerIT.java | 2 +- .../action/RestSearchModelHandlerIT.java | 2 +- .../plugin/script/KNNScoringSpaceTests.java | 2 +- .../LibraryInitializedSupplierTests.java | 8 +++---- .../transport/GetModelResponseTests.java | 6 ++--- ...oveModelFromCacheTransportActionTests.java | 4 ++-- ...TrainingJobRouterTransportActionTests.java | 2 +- .../transport/TrainingModelRequestTests.java | 6 ++--- .../TrainingModelTransportActionTests.java | 4 ++-- ...ateModelGraveyardTransportActionTests.java | 4 ++-- .../UpdateModelMetadataRequestTests.java | 4 ++-- ...dateModelMetadataTransportActionTests.java | 4 ++-- .../opensearch/knn/recall/RecallTestsIT.java | 2 +- .../knn/training/TrainingJobTests.java | 6 ++--- .../org/opensearch/knn/KNNRestTestCase.java | 2 +- 127 files changed, 297 insertions(+), 374 deletions(-) rename src/main/java/org/opensearch/knn/index/{util => engine}/AbstractKNNLibrary.java (95%) rename src/main/java/org/opensearch/knn/index/{util => engine}/DefaultHnswContext.java (70%) rename src/main/java/org/opensearch/knn/index/{util => engine}/DefaultIVFContext.java (67%) rename src/main/java/org/opensearch/knn/index/{util => engine}/EngineSpecificMethodContext.java (63%) rename src/main/java/org/opensearch/knn/index/{util => engine}/JVMLibrary.java (77%) rename src/main/java/org/opensearch/knn/index/{util => engine}/KNNEngine.java (96%) rename src/main/java/org/opensearch/knn/index/{util => engine}/KNNLibrary.java (93%) rename src/main/java/org/opensearch/knn/index/{ => engine}/KNNMethod.java (95%) rename src/main/java/org/opensearch/knn/index/{ => engine}/KNNMethodContext.java (96%) rename src/main/java/org/opensearch/knn/index/{ => engine}/MethodComponent.java (97%) rename src/main/java/org/opensearch/knn/index/{ => engine}/MethodComponentContext.java (98%) rename src/main/java/org/opensearch/knn/index/{util => engine}/NativeLibrary.java (92%) rename src/main/java/org/opensearch/knn/index/{ => engine}/Parameter.java (98%) rename src/main/java/org/opensearch/knn/index/{util => engine/faiss}/Faiss.java (97%) rename src/main/java/org/opensearch/knn/index/{util => engine/lucene}/Lucene.java (93%) rename src/main/java/org/opensearch/knn/index/{util => engine/lucene}/LuceneHNSWContext.java (70%) create mode 100644 src/main/java/org/opensearch/knn/index/engine/model/QueryContext.java rename src/main/java/org/opensearch/knn/index/{util => engine/nmslib}/Nmslib.java (83%) rename src/main/java/org/opensearch/knn/{ => index/engine}/validation/ParameterValidator.java (84%) rename src/main/java/org/opensearch/knn/index/{ => util}/IndexUtil.java (97%) rename src/main/java/org/opensearch/knn/index/{ => util}/KNNClusterUtil.java (97%) delete mode 100644 src/main/java/org/opensearch/knn/index/util/QueryContext.java rename src/test/java/org/opensearch/knn/index/{util => engine}/AbstractKNNLibraryTests.java (98%) rename src/test/java/org/opensearch/knn/index/{util => engine}/KNNEngineTests.java (92%) rename src/test/java/org/opensearch/knn/index/{ => engine}/KNNMethodTests.java (95%) rename src/test/java/org/opensearch/knn/index/{ => engine}/MethodComponentTests.java (96%) rename src/test/java/org/opensearch/knn/index/{util => engine}/NativeLibraryTests.java (97%) rename src/test/java/org/opensearch/knn/index/{ => engine}/ParameterTests.java (95%) rename src/test/java/org/opensearch/knn/index/{util => engine/faiss}/FaissTests.java (97%) rename src/test/java/org/opensearch/knn/index/{util => engine/lucene}/LuceneTests.java (97%) rename src/test/java/org/opensearch/knn/index/{ => util}/IndexUtilTests.java (97%) rename src/test/java/org/opensearch/knn/index/{ => util}/KNNClusterUtilTests.java (97%) diff --git a/CHANGELOG.md b/CHANGELOG.md index c90ea1621..69dd6a2e3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,3 +22,4 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Maintenance ### Refactoring * Clean up parsing for query [#1824](https://github.com/opensearch-project/k-NN/pull/1824) +* Refactor engine package structure [#1913](https://github.com/opensearch-project/k-NN/pull/1913) diff --git a/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/FaissSQIT.java b/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/FaissSQIT.java index 83c358cad..cc4cbc043 100644 --- a/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/FaissSQIT.java +++ b/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/FaissSQIT.java @@ -19,10 +19,10 @@ import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.knn.KNNResult; import org.opensearch.knn.common.KNNConstants; -import org.opensearch.knn.index.KNNMethod; +import org.opensearch.knn.index.engine.KNNMethod; import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.query.KNNQueryBuilder; -import org.opensearch.knn.index.util.KNNEngine; +import org.opensearch.knn.index.engine.KNNEngine; import java.io.IOException; import java.util.Arrays; diff --git a/src/main/java/org/opensearch/knn/index/KNNIndexShard.java b/src/main/java/org/opensearch/knn/index/KNNIndexShard.java index 431c0849c..1e0040fe8 100644 --- a/src/main/java/org/opensearch/knn/index/KNNIndexShard.java +++ b/src/main/java/org/opensearch/knn/index/KNNIndexShard.java @@ -23,7 +23,7 @@ import org.opensearch.knn.index.memory.NativeMemoryCacheManager; import org.opensearch.knn.index.memory.NativeMemoryEntryContext; import org.opensearch.knn.index.memory.NativeMemoryLoadStrategy; -import org.opensearch.knn.index.util.KNNEngine; +import org.opensearch.knn.index.engine.KNNEngine; import java.io.IOException; import java.nio.file.Path; @@ -37,7 +37,7 @@ import static org.opensearch.knn.common.KNNConstants.MODEL_ID; import static org.opensearch.knn.common.KNNConstants.SPACE_TYPE; import static org.opensearch.knn.common.KNNConstants.VECTOR_DATA_TYPE_FIELD; -import static org.opensearch.knn.index.IndexUtil.getParametersAtLoading; +import static org.opensearch.knn.index.util.IndexUtil.getParametersAtLoading; import static org.opensearch.knn.index.codec.util.KNNCodecUtil.buildEngineFilePrefix; import static org.opensearch.knn.index.codec.util.KNNCodecUtil.buildEngineFileSuffix; diff --git a/src/main/java/org/opensearch/knn/index/codec/BasePerFieldKnnVectorsFormat.java b/src/main/java/org/opensearch/knn/index/codec/BasePerFieldKnnVectorsFormat.java index f3738452a..2d423c26e 100644 --- a/src/main/java/org/opensearch/knn/index/codec/BasePerFieldKnnVectorsFormat.java +++ b/src/main/java/org/opensearch/knn/index/codec/BasePerFieldKnnVectorsFormat.java @@ -13,7 +13,7 @@ import org.opensearch.knn.index.codec.params.KNNScalarQuantizedVectorsFormatParams; import org.opensearch.knn.index.codec.params.KNNVectorsFormatParams; import org.opensearch.knn.index.mapper.KNNVectorFieldMapper; -import org.opensearch.knn.index.util.KNNEngine; +import org.opensearch.knn.index.engine.KNNEngine; import java.util.Optional; import java.util.function.Function; diff --git a/src/main/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80CompoundFormat.java b/src/main/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80CompoundFormat.java index cbe4991f0..0f51bdcd5 100644 --- a/src/main/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80CompoundFormat.java +++ b/src/main/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80CompoundFormat.java @@ -12,7 +12,7 @@ import org.apache.lucene.index.SegmentInfo; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; -import org.opensearch.knn.index.util.KNNEngine; +import org.opensearch.knn.index.engine.KNNEngine; import java.io.IOException; import java.util.HashSet; diff --git a/src/main/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumer.java b/src/main/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumer.java index d0008b7a0..31862c25d 100644 --- a/src/main/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumer.java +++ b/src/main/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumer.java @@ -12,7 +12,7 @@ import org.opensearch.common.StopWatch; import org.opensearch.core.xcontent.DeprecationHandler; import org.opensearch.core.xcontent.NamedXContentRegistry; -import org.opensearch.knn.index.IndexUtil; +import org.opensearch.knn.index.util.IndexUtil; import org.opensearch.knn.index.KNNSettings; import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.codec.transfer.VectorTransfer; @@ -21,7 +21,7 @@ import org.opensearch.knn.jni.JNIService; import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.codec.util.KNNCodecUtil; -import org.opensearch.knn.index.util.KNNEngine; +import org.opensearch.knn.index.engine.KNNEngine; import org.opensearch.knn.indices.Model; import org.opensearch.knn.indices.ModelCache; import org.opensearch.knn.plugin.stats.KNNCounter; @@ -58,7 +58,7 @@ import static org.opensearch.knn.common.KNNConstants.PARAMETERS; import static org.opensearch.knn.index.codec.util.KNNCodecUtil.buildEngineFileName; import static org.opensearch.knn.index.codec.util.KNNCodecUtil.calculateArraySize; -import static org.opensearch.knn.index.util.Faiss.FAISS_BINARY_INDEX_DESCRIPTION_PREFIX; +import static org.opensearch.knn.index.engine.faiss.Faiss.FAISS_BINARY_INDEX_DESCRIPTION_PREFIX; /** * This class writes the KNN docvalues to the segments diff --git a/src/main/java/org/opensearch/knn/index/codec/KNN950Codec/KNN950PerFieldKnnVectorsFormat.java b/src/main/java/org/opensearch/knn/index/codec/KNN950Codec/KNN950PerFieldKnnVectorsFormat.java index 978b22003..7a1458057 100644 --- a/src/main/java/org/opensearch/knn/index/codec/KNN950Codec/KNN950PerFieldKnnVectorsFormat.java +++ b/src/main/java/org/opensearch/knn/index/codec/KNN950Codec/KNN950PerFieldKnnVectorsFormat.java @@ -8,7 +8,7 @@ import org.apache.lucene.backward_codecs.lucene95.Lucene95HnswVectorsFormat; import org.opensearch.index.mapper.MapperService; import org.opensearch.knn.index.codec.BasePerFieldKnnVectorsFormat; -import org.opensearch.knn.index.util.KNNEngine; +import org.opensearch.knn.index.engine.KNNEngine; import java.util.Optional; diff --git a/src/main/java/org/opensearch/knn/index/codec/KNN990Codec/KNN990PerFieldKnnVectorsFormat.java b/src/main/java/org/opensearch/knn/index/codec/KNN990Codec/KNN990PerFieldKnnVectorsFormat.java index e8ecfad18..f565dfe5b 100644 --- a/src/main/java/org/opensearch/knn/index/codec/KNN990Codec/KNN990PerFieldKnnVectorsFormat.java +++ b/src/main/java/org/opensearch/knn/index/codec/KNN990Codec/KNN990PerFieldKnnVectorsFormat.java @@ -9,7 +9,7 @@ import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat; import org.opensearch.index.mapper.MapperService; import org.opensearch.knn.index.codec.BasePerFieldKnnVectorsFormat; -import org.opensearch.knn.index.util.KNNEngine; +import org.opensearch.knn.index.engine.KNNEngine; import java.util.Optional; diff --git a/src/main/java/org/opensearch/knn/index/codec/params/KNNScalarQuantizedVectorsFormatParams.java b/src/main/java/org/opensearch/knn/index/codec/params/KNNScalarQuantizedVectorsFormatParams.java index 79bf1cbdb..e2d31183b 100644 --- a/src/main/java/org/opensearch/knn/index/codec/params/KNNScalarQuantizedVectorsFormatParams.java +++ b/src/main/java/org/opensearch/knn/index/codec/params/KNNScalarQuantizedVectorsFormatParams.java @@ -6,7 +6,7 @@ package org.opensearch.knn.index.codec.params; import lombok.Getter; -import org.opensearch.knn.index.MethodComponentContext; +import org.opensearch.knn.index.engine.MethodComponentContext; import java.util.Map; diff --git a/src/main/java/org/opensearch/knn/index/util/AbstractKNNLibrary.java b/src/main/java/org/opensearch/knn/index/engine/AbstractKNNLibrary.java similarity index 95% rename from src/main/java/org/opensearch/knn/index/util/AbstractKNNLibrary.java rename to src/main/java/org/opensearch/knn/index/engine/AbstractKNNLibrary.java index 0e0c56128..05cfb09af 100644 --- a/src/main/java/org/opensearch/knn/index/util/AbstractKNNLibrary.java +++ b/src/main/java/org/opensearch/knn/index/engine/AbstractKNNLibrary.java @@ -3,14 +3,12 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.knn.index.util; +package org.opensearch.knn.index.engine; import lombok.AccessLevel; import lombok.AllArgsConstructor; import lombok.Getter; import org.opensearch.common.ValidationException; -import org.opensearch.knn.index.KNNMethod; -import org.opensearch.knn.index.KNNMethodContext; import org.opensearch.knn.training.VectorSpaceInfo; import java.util.Map; diff --git a/src/main/java/org/opensearch/knn/index/util/DefaultHnswContext.java b/src/main/java/org/opensearch/knn/index/engine/DefaultHnswContext.java similarity index 70% rename from src/main/java/org/opensearch/knn/index/util/DefaultHnswContext.java rename to src/main/java/org/opensearch/knn/index/engine/DefaultHnswContext.java index c2bbb9e6f..ef1d960d4 100644 --- a/src/main/java/org/opensearch/knn/index/util/DefaultHnswContext.java +++ b/src/main/java/org/opensearch/knn/index/engine/DefaultHnswContext.java @@ -1,18 +1,12 @@ /* + * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - * - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. */ -package org.opensearch.knn.index.util; +package org.opensearch.knn.index.engine; import com.google.common.collect.ImmutableMap; -import org.opensearch.knn.index.Parameter; +import org.opensearch.knn.index.engine.model.QueryContext; import org.opensearch.knn.index.query.request.MethodParameter; import java.util.Map; diff --git a/src/main/java/org/opensearch/knn/index/util/DefaultIVFContext.java b/src/main/java/org/opensearch/knn/index/engine/DefaultIVFContext.java similarity index 67% rename from src/main/java/org/opensearch/knn/index/util/DefaultIVFContext.java rename to src/main/java/org/opensearch/knn/index/engine/DefaultIVFContext.java index dbccbe9cb..a1a474420 100644 --- a/src/main/java/org/opensearch/knn/index/util/DefaultIVFContext.java +++ b/src/main/java/org/opensearch/knn/index/engine/DefaultIVFContext.java @@ -1,18 +1,12 @@ /* + * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - * - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. */ -package org.opensearch.knn.index.util; +package org.opensearch.knn.index.engine; import com.google.common.collect.ImmutableMap; -import org.opensearch.knn.index.Parameter; +import org.opensearch.knn.index.engine.model.QueryContext; import org.opensearch.knn.index.query.request.MethodParameter; import java.util.Map; diff --git a/src/main/java/org/opensearch/knn/index/util/EngineSpecificMethodContext.java b/src/main/java/org/opensearch/knn/index/engine/EngineSpecificMethodContext.java similarity index 63% rename from src/main/java/org/opensearch/knn/index/util/EngineSpecificMethodContext.java rename to src/main/java/org/opensearch/knn/index/engine/EngineSpecificMethodContext.java index edb8e830a..a043bd9cd 100644 --- a/src/main/java/org/opensearch/knn/index/util/EngineSpecificMethodContext.java +++ b/src/main/java/org/opensearch/knn/index/engine/EngineSpecificMethodContext.java @@ -1,17 +1,11 @@ /* + * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - * - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. */ -package org.opensearch.knn.index.util; +package org.opensearch.knn.index.engine; -import org.opensearch.knn.index.Parameter; +import org.opensearch.knn.index.engine.model.QueryContext; import java.util.Collections; import java.util.Map; diff --git a/src/main/java/org/opensearch/knn/index/util/JVMLibrary.java b/src/main/java/org/opensearch/knn/index/engine/JVMLibrary.java similarity index 77% rename from src/main/java/org/opensearch/knn/index/util/JVMLibrary.java rename to src/main/java/org/opensearch/knn/index/engine/JVMLibrary.java index 850679dc2..781fa9c7d 100644 --- a/src/main/java/org/opensearch/knn/index/util/JVMLibrary.java +++ b/src/main/java/org/opensearch/knn/index/engine/JVMLibrary.java @@ -3,10 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.knn.index.util; - -import org.opensearch.knn.index.KNNMethod; -import org.opensearch.knn.index.KNNMethodContext; +package org.opensearch.knn.index.engine; import java.util.Map; @@ -23,7 +20,7 @@ public abstract class JVMLibrary extends AbstractKNNLibrary { * @param methods Map of k-NN methods that the library supports * @param version String representing version of library */ - JVMLibrary(Map methods, Map engineMethodMetadataMap, String version) { + public JVMLibrary(Map methods, Map engineMethodMetadataMap, String version) { super(methods, engineMethodMetadataMap, version); } diff --git a/src/main/java/org/opensearch/knn/index/util/KNNEngine.java b/src/main/java/org/opensearch/knn/index/engine/KNNEngine.java similarity index 96% rename from src/main/java/org/opensearch/knn/index/util/KNNEngine.java rename to src/main/java/org/opensearch/knn/index/engine/KNNEngine.java index ee8be9c5c..a552b2b57 100644 --- a/src/main/java/org/opensearch/knn/index/util/KNNEngine.java +++ b/src/main/java/org/opensearch/knn/index/engine/KNNEngine.java @@ -3,13 +3,14 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.knn.index.util; +package org.opensearch.knn.index.engine; import com.google.common.collect.ImmutableSet; import org.opensearch.common.ValidationException; -import org.opensearch.knn.index.KNNMethod; -import org.opensearch.knn.index.KNNMethodContext; import org.opensearch.knn.index.SpaceType; +import org.opensearch.knn.index.engine.faiss.Faiss; +import org.opensearch.knn.index.engine.lucene.Lucene; +import org.opensearch.knn.index.engine.nmslib.Nmslib; import org.opensearch.knn.training.VectorSpaceInfo; import java.util.List; diff --git a/src/main/java/org/opensearch/knn/index/util/KNNLibrary.java b/src/main/java/org/opensearch/knn/index/engine/KNNLibrary.java similarity index 93% rename from src/main/java/org/opensearch/knn/index/util/KNNLibrary.java rename to src/main/java/org/opensearch/knn/index/engine/KNNLibrary.java index f9d8429d3..3989c4609 100644 --- a/src/main/java/org/opensearch/knn/index/util/KNNLibrary.java +++ b/src/main/java/org/opensearch/knn/index/engine/KNNLibrary.java @@ -1,19 +1,11 @@ /* + * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - * - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. */ -package org.opensearch.knn.index.util; +package org.opensearch.knn.index.engine; import org.opensearch.common.ValidationException; -import org.opensearch.knn.index.KNNMethod; -import org.opensearch.knn.index.KNNMethodContext; import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.training.VectorSpaceInfo; diff --git a/src/main/java/org/opensearch/knn/index/KNNMethod.java b/src/main/java/org/opensearch/knn/index/engine/KNNMethod.java similarity index 95% rename from src/main/java/org/opensearch/knn/index/KNNMethod.java rename to src/main/java/org/opensearch/knn/index/engine/KNNMethod.java index d456256f5..c3dad21b2 100644 --- a/src/main/java/org/opensearch/knn/index/KNNMethod.java +++ b/src/main/java/org/opensearch/knn/index/engine/KNNMethod.java @@ -1,20 +1,15 @@ /* + * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - * - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. */ -package org.opensearch.knn.index; +package org.opensearch.knn.index.engine; import lombok.AllArgsConstructor; import lombok.Getter; import org.opensearch.common.ValidationException; import org.opensearch.knn.common.KNNConstants; +import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.training.VectorSpaceInfo; import java.util.ArrayList; diff --git a/src/main/java/org/opensearch/knn/index/KNNMethodContext.java b/src/main/java/org/opensearch/knn/index/engine/KNNMethodContext.java similarity index 96% rename from src/main/java/org/opensearch/knn/index/KNNMethodContext.java rename to src/main/java/org/opensearch/knn/index/engine/KNNMethodContext.java index ba7a7509c..7885761b9 100644 --- a/src/main/java/org/opensearch/knn/index/KNNMethodContext.java +++ b/src/main/java/org/opensearch/knn/index/engine/KNNMethodContext.java @@ -1,15 +1,9 @@ /* + * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - * - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. */ -package org.opensearch.knn.index; +package org.opensearch.knn.index.engine; import lombok.AllArgsConstructor; import lombok.Getter; @@ -20,7 +14,7 @@ import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.io.stream.Writeable; -import org.opensearch.knn.index.util.KNNEngine; +import org.opensearch.knn.index.SpaceType; import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.index.mapper.MapperParsingException; diff --git a/src/main/java/org/opensearch/knn/index/MethodComponent.java b/src/main/java/org/opensearch/knn/index/engine/MethodComponent.java similarity index 97% rename from src/main/java/org/opensearch/knn/index/MethodComponent.java rename to src/main/java/org/opensearch/knn/index/engine/MethodComponent.java index b344772f7..cd9377ef1 100644 --- a/src/main/java/org/opensearch/knn/index/MethodComponent.java +++ b/src/main/java/org/opensearch/knn/index/engine/MethodComponent.java @@ -1,15 +1,9 @@ /* + * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - * - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. */ -package org.opensearch.knn.index; +package org.opensearch.knn.index.engine; import lombok.Getter; import org.opensearch.Version; @@ -25,7 +19,7 @@ import java.util.List; import java.util.ArrayList; -import static org.opensearch.knn.validation.ParameterValidator.validateParameters; +import static org.opensearch.knn.index.engine.validation.ParameterValidator.validateParameters; /** * MethodComponent defines the structure of an individual component that can make up an index diff --git a/src/main/java/org/opensearch/knn/index/MethodComponentContext.java b/src/main/java/org/opensearch/knn/index/engine/MethodComponentContext.java similarity index 98% rename from src/main/java/org/opensearch/knn/index/MethodComponentContext.java rename to src/main/java/org/opensearch/knn/index/engine/MethodComponentContext.java index b9fd56b72..fb4327487 100644 --- a/src/main/java/org/opensearch/knn/index/MethodComponentContext.java +++ b/src/main/java/org/opensearch/knn/index/engine/MethodComponentContext.java @@ -1,15 +1,9 @@ /* + * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - * - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. */ -package org.opensearch.knn.index; +package org.opensearch.knn.index.engine; import lombok.AllArgsConstructor; import lombok.Getter; diff --git a/src/main/java/org/opensearch/knn/index/util/NativeLibrary.java b/src/main/java/org/opensearch/knn/index/engine/NativeLibrary.java similarity index 92% rename from src/main/java/org/opensearch/knn/index/util/NativeLibrary.java rename to src/main/java/org/opensearch/knn/index/engine/NativeLibrary.java index 99d4aeeb9..dbbadf4f4 100644 --- a/src/main/java/org/opensearch/knn/index/util/NativeLibrary.java +++ b/src/main/java/org/opensearch/knn/index/engine/NativeLibrary.java @@ -3,12 +3,10 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.knn.index.util; +package org.opensearch.knn.index.engine; import lombok.Getter; import org.opensearch.knn.common.KNNConstants; -import org.opensearch.knn.index.KNNMethod; -import org.opensearch.knn.index.KNNMethodContext; import org.opensearch.knn.index.SpaceType; import java.util.Map; @@ -20,7 +18,7 @@ * Abstract implementation of KNNLibrary. It contains several default methods and fields that * are common across different underlying libraries. */ -abstract class NativeLibrary extends AbstractKNNLibrary { +public abstract class NativeLibrary extends AbstractKNNLibrary { private final Map> scoreTranslation; @Getter private final String extension; @@ -34,7 +32,7 @@ abstract class NativeLibrary extends AbstractKNNLibrary { * @param version String representation of version of the library * @param extension String representing the extension that library files should use */ - NativeLibrary( + public NativeLibrary( Map methods, Map engineMethods, Map> scoreTranslation, diff --git a/src/main/java/org/opensearch/knn/index/Parameter.java b/src/main/java/org/opensearch/knn/index/engine/Parameter.java similarity index 98% rename from src/main/java/org/opensearch/knn/index/Parameter.java rename to src/main/java/org/opensearch/knn/index/engine/Parameter.java index 50d792ebd..fbd3ae692 100644 --- a/src/main/java/org/opensearch/knn/index/Parameter.java +++ b/src/main/java/org/opensearch/knn/index/engine/Parameter.java @@ -1,15 +1,9 @@ /* + * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - * - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. */ -package org.opensearch.knn.index; +package org.opensearch.knn.index.engine; import org.opensearch.common.ValidationException; import org.opensearch.knn.training.VectorSpaceInfo; diff --git a/src/main/java/org/opensearch/knn/index/util/Faiss.java b/src/main/java/org/opensearch/knn/index/engine/faiss/Faiss.java similarity index 97% rename from src/main/java/org/opensearch/knn/index/util/Faiss.java rename to src/main/java/org/opensearch/knn/index/engine/faiss/Faiss.java index 7e33db30c..a665a4136 100644 --- a/src/main/java/org/opensearch/knn/index/util/Faiss.java +++ b/src/main/java/org/opensearch/knn/index/engine/faiss/Faiss.java @@ -3,17 +3,20 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.knn.index.util; +package org.opensearch.knn.index.engine.faiss; import com.google.common.collect.ImmutableMap; import lombok.AllArgsConstructor; import org.opensearch.knn.common.KNNConstants; -import org.opensearch.knn.index.KNNMethod; import org.opensearch.knn.index.KNNSettings; -import org.opensearch.knn.index.MethodComponent; -import org.opensearch.knn.index.MethodComponentContext; -import org.opensearch.knn.index.Parameter; import org.opensearch.knn.index.SpaceType; +import org.opensearch.knn.index.engine.DefaultHnswContext; +import org.opensearch.knn.index.engine.DefaultIVFContext; +import org.opensearch.knn.index.engine.KNNMethod; +import org.opensearch.knn.index.engine.MethodComponent; +import org.opensearch.knn.index.engine.MethodComponentContext; +import org.opensearch.knn.index.engine.NativeLibrary; +import org.opensearch.knn.index.engine.Parameter; import java.util.Collections; import java.util.HashMap; @@ -308,7 +311,7 @@ public class Faiss extends NativeLibrary { ).addSpaces(SpaceType.UNDEFINED, SpaceType.L2, SpaceType.INNER_PRODUCT, SpaceType.HAMMING).build() ); - final static Faiss INSTANCE = new Faiss( + public final static Faiss INSTANCE = new Faiss( METHODS, SCORE_TRANSLATIONS, CURRENT_VERSION, diff --git a/src/main/java/org/opensearch/knn/index/util/Lucene.java b/src/main/java/org/opensearch/knn/index/engine/lucene/Lucene.java similarity index 93% rename from src/main/java/org/opensearch/knn/index/util/Lucene.java rename to src/main/java/org/opensearch/knn/index/engine/lucene/Lucene.java index caf4200cb..4199cbb19 100644 --- a/src/main/java/org/opensearch/knn/index/util/Lucene.java +++ b/src/main/java/org/opensearch/knn/index/engine/lucene/Lucene.java @@ -3,17 +3,18 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.knn.index.util; +package org.opensearch.knn.index.engine.lucene; import com.google.common.collect.ImmutableMap; import org.apache.lucene.util.Version; import org.opensearch.knn.common.KNNConstants; -import org.opensearch.knn.index.KNNMethod; import org.opensearch.knn.index.KNNSettings; -import org.opensearch.knn.index.MethodComponent; -import org.opensearch.knn.index.MethodComponentContext; -import org.opensearch.knn.index.Parameter; import org.opensearch.knn.index.SpaceType; +import org.opensearch.knn.index.engine.JVMLibrary; +import org.opensearch.knn.index.engine.KNNMethod; +import org.opensearch.knn.index.engine.MethodComponent; +import org.opensearch.knn.index.engine.MethodComponentContext; +import org.opensearch.knn.index.engine.Parameter; import java.util.Collections; import java.util.List; @@ -96,7 +97,7 @@ public class Lucene extends JVMLibrary { .put(SpaceType.INNER_PRODUCT, distance -> distance <= 0 ? 1 / (1 - distance) : distance + 1) .build(); - final static Lucene INSTANCE = new Lucene(METHODS, Version.LATEST.toString(), DISTANCE_TRANSLATIONS); + public final static Lucene INSTANCE = new Lucene(METHODS, Version.LATEST.toString(), DISTANCE_TRANSLATIONS); /** * Constructor diff --git a/src/main/java/org/opensearch/knn/index/util/LuceneHNSWContext.java b/src/main/java/org/opensearch/knn/index/engine/lucene/LuceneHNSWContext.java similarity index 70% rename from src/main/java/org/opensearch/knn/index/util/LuceneHNSWContext.java rename to src/main/java/org/opensearch/knn/index/engine/lucene/LuceneHNSWContext.java index d9b6ba1c3..808aa66b0 100644 --- a/src/main/java/org/opensearch/knn/index/util/LuceneHNSWContext.java +++ b/src/main/java/org/opensearch/knn/index/engine/lucene/LuceneHNSWContext.java @@ -1,18 +1,14 @@ /* + * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - * - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. */ -package org.opensearch.knn.index.util; +package org.opensearch.knn.index.engine.lucene; import com.google.common.collect.ImmutableMap; -import org.opensearch.knn.index.Parameter; +import org.opensearch.knn.index.engine.EngineSpecificMethodContext; +import org.opensearch.knn.index.engine.Parameter; +import org.opensearch.knn.index.engine.model.QueryContext; import org.opensearch.knn.index.query.request.MethodParameter; import java.util.Collections; @@ -26,7 +22,7 @@ public class LuceneHNSWContext implements EngineSpecificMethodContext { @Override public Map> supportedMethodParameters(QueryContext ctx) { - if (ctx.queryType.isRadialSearch()) { + if (ctx.getQueryType().isRadialSearch()) { // return empty map if radial search is true return Collections.emptyMap(); } diff --git a/src/main/java/org/opensearch/knn/index/engine/model/QueryContext.java b/src/main/java/org/opensearch/knn/index/engine/model/QueryContext.java new file mode 100644 index 000000000..21182e4aa --- /dev/null +++ b/src/main/java/org/opensearch/knn/index/engine/model/QueryContext.java @@ -0,0 +1,19 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.engine.model; + +import lombok.AllArgsConstructor; +import lombok.Getter; +import org.opensearch.knn.index.VectorQueryType; + +/** + * Context class for query-specific information. + */ +@AllArgsConstructor +@Getter +public class QueryContext { + VectorQueryType queryType; +} diff --git a/src/main/java/org/opensearch/knn/index/util/Nmslib.java b/src/main/java/org/opensearch/knn/index/engine/nmslib/Nmslib.java similarity index 83% rename from src/main/java/org/opensearch/knn/index/util/Nmslib.java rename to src/main/java/org/opensearch/knn/index/engine/nmslib/Nmslib.java index a068901d3..2138c1310 100644 --- a/src/main/java/org/opensearch/knn/index/util/Nmslib.java +++ b/src/main/java/org/opensearch/knn/index/engine/nmslib/Nmslib.java @@ -3,14 +3,16 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.knn.index.util; +package org.opensearch.knn.index.engine.nmslib; import com.google.common.collect.ImmutableMap; -import org.opensearch.knn.index.KNNMethod; import org.opensearch.knn.index.KNNSettings; -import org.opensearch.knn.index.MethodComponent; -import org.opensearch.knn.index.Parameter; import org.opensearch.knn.index.SpaceType; +import org.opensearch.knn.index.engine.DefaultHnswContext; +import org.opensearch.knn.index.engine.KNNMethod; +import org.opensearch.knn.index.engine.MethodComponent; +import org.opensearch.knn.index.engine.NativeLibrary; +import org.opensearch.knn.index.engine.Parameter; import java.util.Collections; import java.util.Map; @@ -23,10 +25,10 @@ /** * Implements NativeLibrary for the nmslib native library */ -class Nmslib extends NativeLibrary { +public class Nmslib extends NativeLibrary { // Extension to be used for Nmslib files. It is ".hnsw" and not ".nmslib" for legacy purposes. - final static String EXTENSION = ".hnsw"; + public final static String EXTENSION = ".hnsw"; final static String CURRENT_VERSION = "2011"; @@ -50,7 +52,7 @@ class Nmslib extends NativeLibrary { ).addSpaces(SpaceType.UNDEFINED, SpaceType.L2, SpaceType.L1, SpaceType.LINF, SpaceType.COSINESIMIL, SpaceType.INNER_PRODUCT).build() ); - final static Nmslib INSTANCE = new Nmslib(METHODS, Collections.emptyMap(), CURRENT_VERSION, EXTENSION); + public final static Nmslib INSTANCE = new Nmslib(METHODS, Collections.emptyMap(), CURRENT_VERSION, EXTENSION); /** * Constructor for Nmslib diff --git a/src/main/java/org/opensearch/knn/validation/ParameterValidator.java b/src/main/java/org/opensearch/knn/index/engine/validation/ParameterValidator.java similarity index 84% rename from src/main/java/org/opensearch/knn/validation/ParameterValidator.java rename to src/main/java/org/opensearch/knn/index/engine/validation/ParameterValidator.java index 15925fffa..6a16b48a5 100644 --- a/src/main/java/org/opensearch/knn/validation/ParameterValidator.java +++ b/src/main/java/org/opensearch/knn/index/engine/validation/ParameterValidator.java @@ -1,19 +1,13 @@ /* + * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - * - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. */ -package org.opensearch.knn.validation; +package org.opensearch.knn.index.engine.validation; import org.opensearch.common.Nullable; import org.opensearch.common.ValidationException; -import org.opensearch.knn.index.Parameter; +import org.opensearch.knn.index.engine.Parameter; import java.util.ArrayList; import java.util.List; diff --git a/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapper.java b/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapper.java index 026e9f469..14596189a 100644 --- a/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapper.java +++ b/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapper.java @@ -44,14 +44,14 @@ import org.opensearch.index.query.QueryShardException; import org.opensearch.knn.common.KNNConstants; import org.opensearch.knn.index.KnnCircuitBreakerException; -import org.opensearch.knn.index.KNNMethodContext; +import org.opensearch.knn.index.engine.KNNMethodContext; import org.opensearch.knn.index.KNNSettings; import org.opensearch.knn.index.KNNVectorIndexFieldData; import org.opensearch.knn.index.SpaceType; -import org.opensearch.knn.index.MethodComponentContext; +import org.opensearch.knn.index.engine.MethodComponentContext; import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.VectorField; -import org.opensearch.knn.index.util.KNNEngine; +import org.opensearch.knn.index.engine.KNNEngine; import org.opensearch.knn.indices.ModelDao; import org.opensearch.search.aggregations.support.CoreValuesSourceType; import org.opensearch.search.lookup.SearchLookup; diff --git a/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperUtil.java b/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperUtil.java index af482813f..07bf4fc2d 100644 --- a/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperUtil.java +++ b/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperUtil.java @@ -18,10 +18,10 @@ import org.apache.lucene.index.DocValuesType; import org.apache.lucene.util.BytesRef; import org.opensearch.index.mapper.ParametrizedFieldMapper; -import org.opensearch.knn.index.KNNMethodContext; +import org.opensearch.knn.index.engine.KNNMethodContext; import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.codec.util.KNNVectorSerializerFactory; -import org.opensearch.knn.index.util.KNNEngine; +import org.opensearch.knn.index.engine.KNNEngine; import org.opensearch.knn.indices.ModelDao; import org.opensearch.knn.indices.ModelMetadata; import org.opensearch.knn.indices.ModelUtil; diff --git a/src/main/java/org/opensearch/knn/index/mapper/LegacyFieldMapper.java b/src/main/java/org/opensearch/knn/index/mapper/LegacyFieldMapper.java index 4959e7bc0..cf5ec933a 100644 --- a/src/main/java/org/opensearch/knn/index/mapper/LegacyFieldMapper.java +++ b/src/main/java/org/opensearch/knn/index/mapper/LegacyFieldMapper.java @@ -14,7 +14,7 @@ import org.opensearch.knn.index.KNNSettings; import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.util.IndexHyperParametersUtil; -import org.opensearch.knn.index.util.KNNEngine; +import org.opensearch.knn.index.engine.KNNEngine; import static org.opensearch.knn.common.KNNConstants.DIMENSION; import static org.opensearch.knn.common.KNNConstants.HNSW_ALGO_EF_CONSTRUCTION; diff --git a/src/main/java/org/opensearch/knn/index/mapper/LuceneFieldMapper.java b/src/main/java/org/opensearch/knn/index/mapper/LuceneFieldMapper.java index b8ba688ff..c82afb9e7 100644 --- a/src/main/java/org/opensearch/knn/index/mapper/LuceneFieldMapper.java +++ b/src/main/java/org/opensearch/knn/index/mapper/LuceneFieldMapper.java @@ -17,10 +17,10 @@ import org.apache.lucene.document.KnnVectorField; import org.apache.lucene.index.VectorSimilarityFunction; import org.opensearch.common.Explicit; -import org.opensearch.knn.index.KNNMethodContext; +import org.opensearch.knn.index.engine.KNNMethodContext; import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.VectorField; -import org.opensearch.knn.index.util.KNNEngine; +import org.opensearch.knn.index.engine.KNNEngine; import static org.opensearch.knn.index.mapper.KNNVectorFieldMapperUtil.createStoredFieldForByteVector; import static org.opensearch.knn.index.mapper.KNNVectorFieldMapperUtil.createStoredFieldForFloatVector; diff --git a/src/main/java/org/opensearch/knn/index/mapper/MethodFieldMapper.java b/src/main/java/org/opensearch/knn/index/mapper/MethodFieldMapper.java index f09ac1b4c..dba37c927 100644 --- a/src/main/java/org/opensearch/knn/index/mapper/MethodFieldMapper.java +++ b/src/main/java/org/opensearch/knn/index/mapper/MethodFieldMapper.java @@ -8,8 +8,8 @@ import org.apache.lucene.document.FieldType; import org.opensearch.common.Explicit; import org.opensearch.common.xcontent.XContentFactory; -import org.opensearch.knn.index.KNNMethodContext; -import org.opensearch.knn.index.util.KNNEngine; +import org.opensearch.knn.index.engine.KNNMethodContext; +import org.opensearch.knn.index.engine.KNNEngine; import java.io.IOException; diff --git a/src/main/java/org/opensearch/knn/index/memory/NativeMemoryAllocation.java b/src/main/java/org/opensearch/knn/index/memory/NativeMemoryAllocation.java index 0b0f1e615..4f92a9c4b 100644 --- a/src/main/java/org/opensearch/knn/index/memory/NativeMemoryAllocation.java +++ b/src/main/java/org/opensearch/knn/index/memory/NativeMemoryAllocation.java @@ -13,12 +13,12 @@ import lombok.Getter; import org.apache.lucene.index.LeafReaderContext; -import org.opensearch.knn.index.IndexUtil; +import org.opensearch.knn.index.util.IndexUtil; import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.query.KNNWeight; import org.opensearch.knn.jni.JNICommons; import org.opensearch.knn.jni.JNIService; -import org.opensearch.knn.index.util.KNNEngine; +import org.opensearch.knn.index.engine.KNNEngine; import org.opensearch.watcher.FileWatcher; import org.opensearch.watcher.WatcherHandle; diff --git a/src/main/java/org/opensearch/knn/index/memory/NativeMemoryEntryContext.java b/src/main/java/org/opensearch/knn/index/memory/NativeMemoryEntryContext.java index b5ddff1e2..2dfc5fafb 100644 --- a/src/main/java/org/opensearch/knn/index/memory/NativeMemoryEntryContext.java +++ b/src/main/java/org/opensearch/knn/index/memory/NativeMemoryEntryContext.java @@ -13,7 +13,7 @@ import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.Nullable; -import org.opensearch.knn.index.IndexUtil; +import org.opensearch.knn.index.util.IndexUtil; import org.opensearch.knn.index.VectorDataType; import java.io.IOException; diff --git a/src/main/java/org/opensearch/knn/index/memory/NativeMemoryLoadStrategy.java b/src/main/java/org/opensearch/knn/index/memory/NativeMemoryLoadStrategy.java index cc46c44cc..6723c2ed0 100644 --- a/src/main/java/org/opensearch/knn/index/memory/NativeMemoryLoadStrategy.java +++ b/src/main/java/org/opensearch/knn/index/memory/NativeMemoryLoadStrategy.java @@ -13,10 +13,10 @@ import lombok.extern.log4j.Log4j2; import org.opensearch.core.action.ActionListener; -import org.opensearch.knn.index.IndexUtil; +import org.opensearch.knn.index.util.IndexUtil; import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.jni.JNIService; -import org.opensearch.knn.index.util.KNNEngine; +import org.opensearch.knn.index.engine.KNNEngine; import org.opensearch.knn.training.ByteTrainingDataConsumer; import org.opensearch.knn.training.FloatTrainingDataConsumer; import org.opensearch.knn.training.TrainingDataConsumer; diff --git a/src/main/java/org/opensearch/knn/index/memory/SharedIndexState.java b/src/main/java/org/opensearch/knn/index/memory/SharedIndexState.java index 2ffadb22e..ba936309b 100644 --- a/src/main/java/org/opensearch/knn/index/memory/SharedIndexState.java +++ b/src/main/java/org/opensearch/knn/index/memory/SharedIndexState.java @@ -7,7 +7,7 @@ import lombok.Getter; import lombok.RequiredArgsConstructor; -import org.opensearch.knn.index.util.KNNEngine; +import org.opensearch.knn.index.engine.KNNEngine; /** * Class stores information about the shared memory allocations between loaded native indices. diff --git a/src/main/java/org/opensearch/knn/index/memory/SharedIndexStateManager.java b/src/main/java/org/opensearch/knn/index/memory/SharedIndexStateManager.java index 896113834..82c5b0a28 100644 --- a/src/main/java/org/opensearch/knn/index/memory/SharedIndexStateManager.java +++ b/src/main/java/org/opensearch/knn/index/memory/SharedIndexStateManager.java @@ -8,7 +8,7 @@ import com.google.common.annotations.VisibleForTesting; import lombok.Getter; import lombok.extern.log4j.Log4j2; -import org.opensearch.knn.index.util.KNNEngine; +import org.opensearch.knn.index.engine.KNNEngine; import org.opensearch.knn.jni.JNIService; import java.util.concurrent.ConcurrentHashMap; diff --git a/src/main/java/org/opensearch/knn/index/query/BaseQueryFactory.java b/src/main/java/org/opensearch/knn/index/query/BaseQueryFactory.java index a02c090b1..931634862 100644 --- a/src/main/java/org/opensearch/knn/index/query/BaseQueryFactory.java +++ b/src/main/java/org/opensearch/knn/index/query/BaseQueryFactory.java @@ -17,7 +17,7 @@ import org.opensearch.index.query.QueryShardContext; import org.opensearch.index.search.NestedHelper; import org.opensearch.knn.index.VectorDataType; -import org.opensearch.knn.index.util.KNNEngine; +import org.opensearch.knn.index.engine.KNNEngine; import java.io.IOException; import java.util.Map; diff --git a/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java b/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java index 24037a2a4..f860b282b 100644 --- a/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java +++ b/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java @@ -23,17 +23,17 @@ import org.opensearch.index.query.QueryBuilder; import org.opensearch.index.query.QueryRewriteContext; import org.opensearch.index.query.QueryShardContext; -import org.opensearch.knn.index.IndexUtil; -import org.opensearch.knn.index.KNNMethodContext; -import org.opensearch.knn.index.MethodComponentContext; +import org.opensearch.knn.index.engine.model.QueryContext; +import org.opensearch.knn.index.util.IndexUtil; +import org.opensearch.knn.index.engine.KNNMethodContext; +import org.opensearch.knn.index.engine.MethodComponentContext; import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.VectorQueryType; import org.opensearch.knn.index.mapper.KNNVectorFieldMapper; import org.opensearch.knn.index.query.parser.KNNQueryBuilderParser; -import org.opensearch.knn.index.util.EngineSpecificMethodContext; -import org.opensearch.knn.index.util.KNNEngine; -import org.opensearch.knn.index.util.QueryContext; +import org.opensearch.knn.index.engine.EngineSpecificMethodContext; +import org.opensearch.knn.index.engine.KNNEngine; import org.opensearch.knn.indices.ModelDao; import org.opensearch.knn.indices.ModelMetadata; import org.opensearch.knn.indices.ModelUtil; @@ -51,8 +51,8 @@ import static org.opensearch.knn.common.KNNConstants.MIN_SCORE; import static org.opensearch.knn.common.KNNValidationUtil.validateByteVectorValue; import static org.opensearch.knn.index.query.parser.MethodParametersParser.validateMethodParameters; -import static org.opensearch.knn.index.util.KNNEngine.ENGINES_SUPPORTING_RADIAL_SEARCH; -import static org.opensearch.knn.validation.ParameterValidator.validateParameters; +import static org.opensearch.knn.index.engine.KNNEngine.ENGINES_SUPPORTING_RADIAL_SEARCH; +import static org.opensearch.knn.index.engine.validation.ParameterValidator.validateParameters; /** * Helper class to build the KNN query diff --git a/src/main/java/org/opensearch/knn/index/query/KNNQueryFactory.java b/src/main/java/org/opensearch/knn/index/query/KNNQueryFactory.java index af7dad026..ee9a12a41 100644 --- a/src/main/java/org/opensearch/knn/index/query/KNNQueryFactory.java +++ b/src/main/java/org/opensearch/knn/index/query/KNNQueryFactory.java @@ -15,7 +15,7 @@ import org.apache.lucene.search.join.DiversifyingChildrenFloatKnnVectorQuery; import org.opensearch.index.query.QueryShardContext; import org.opensearch.knn.index.VectorDataType; -import org.opensearch.knn.index.util.KNNEngine; +import org.opensearch.knn.index.engine.KNNEngine; import java.util.Locale; import java.util.Map; diff --git a/src/main/java/org/opensearch/knn/index/query/KNNWeight.java b/src/main/java/org/opensearch/knn/index/query/KNNWeight.java index 4e5dec89f..f54d8328e 100644 --- a/src/main/java/org/opensearch/knn/index/query/KNNWeight.java +++ b/src/main/java/org/opensearch/knn/index/query/KNNWeight.java @@ -42,7 +42,7 @@ import org.opensearch.knn.index.query.filtered.KNNIterator; import org.opensearch.knn.index.query.filtered.NestedFilteredIdsKNNByteIterator; import org.opensearch.knn.index.query.filtered.NestedFilteredIdsKNNIterator; -import org.opensearch.knn.index.util.KNNEngine; +import org.opensearch.knn.index.engine.KNNEngine; import org.opensearch.knn.indices.ModelDao; import org.opensearch.knn.indices.ModelMetadata; import org.opensearch.knn.indices.ModelUtil; @@ -65,7 +65,7 @@ import static org.opensearch.knn.common.KNNConstants.MODEL_ID; import static org.opensearch.knn.common.KNNConstants.SPACE_TYPE; import static org.opensearch.knn.common.KNNConstants.VECTOR_DATA_TYPE_FIELD; -import static org.opensearch.knn.index.IndexUtil.getParametersAtLoading; +import static org.opensearch.knn.index.util.IndexUtil.getParametersAtLoading; import static org.opensearch.knn.plugin.stats.KNNCounter.GRAPH_QUERY_ERRORS; /** diff --git a/src/main/java/org/opensearch/knn/index/query/RNNQueryFactory.java b/src/main/java/org/opensearch/knn/index/query/RNNQueryFactory.java index dd5efc93f..99152ef6b 100644 --- a/src/main/java/org/opensearch/knn/index/query/RNNQueryFactory.java +++ b/src/main/java/org/opensearch/knn/index/query/RNNQueryFactory.java @@ -20,7 +20,7 @@ import org.opensearch.index.IndexSettings; import org.opensearch.index.query.QueryShardContext; import org.opensearch.knn.index.VectorDataType; -import org.opensearch.knn.index.util.KNNEngine; +import org.opensearch.knn.index.engine.KNNEngine; /** * Class to create radius nearest neighbor queries diff --git a/src/main/java/org/opensearch/knn/index/query/parser/KNNQueryBuilderParser.java b/src/main/java/org/opensearch/knn/index/query/parser/KNNQueryBuilderParser.java index 39ca915c7..217166aa9 100644 --- a/src/main/java/org/opensearch/knn/index/query/parser/KNNQueryBuilderParser.java +++ b/src/main/java/org/opensearch/knn/index/query/parser/KNNQueryBuilderParser.java @@ -16,7 +16,7 @@ import org.opensearch.core.xcontent.XContentParser; import org.opensearch.index.query.QueryBuilder; import org.opensearch.knn.common.KNNConstants; -import org.opensearch.knn.index.IndexUtil; +import org.opensearch.knn.index.util.IndexUtil; import org.opensearch.knn.index.query.KNNQueryBuilder; import java.io.IOException; @@ -29,7 +29,7 @@ import static org.opensearch.index.query.AbstractQueryBuilder.NAME_FIELD; import static org.opensearch.index.query.AbstractQueryBuilder.parseInnerQueryBuilder; import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER; -import static org.opensearch.knn.index.IndexUtil.isClusterOnOrAfterMinRequiredVersion; +import static org.opensearch.knn.index.util.IndexUtil.isClusterOnOrAfterMinRequiredVersion; import static org.opensearch.knn.index.query.KNNQueryBuilder.FILTER_FIELD; import static org.opensearch.knn.index.query.KNNQueryBuilder.IGNORE_UNMAPPED_FIELD; import static org.opensearch.knn.index.query.KNNQueryBuilder.K_FIELD; diff --git a/src/main/java/org/opensearch/knn/index/IndexUtil.java b/src/main/java/org/opensearch/knn/index/util/IndexUtil.java similarity index 97% rename from src/main/java/org/opensearch/knn/index/IndexUtil.java rename to src/main/java/org/opensearch/knn/index/util/IndexUtil.java index 524c9267e..924f4ab19 100644 --- a/src/main/java/org/opensearch/knn/index/IndexUtil.java +++ b/src/main/java/org/opensearch/knn/index/util/IndexUtil.java @@ -1,15 +1,9 @@ /* + * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - * - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. */ -package org.opensearch.knn.index; +package org.opensearch.knn.index.util; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Maps; @@ -19,9 +13,14 @@ import org.opensearch.cluster.metadata.MappingMetadata; import org.opensearch.common.ValidationException; import org.opensearch.knn.common.KNNConstants; +import org.opensearch.knn.index.engine.KNNMethodContext; +import org.opensearch.knn.index.KNNSettings; +import org.opensearch.knn.index.engine.MethodComponentContext; +import org.opensearch.knn.index.SpaceType; +import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.mapper.KNNVectorFieldMapper; import org.opensearch.knn.index.query.request.MethodParameter; -import org.opensearch.knn.index.util.KNNEngine; +import org.opensearch.knn.index.engine.KNNEngine; import org.opensearch.knn.indices.ModelDao; import org.opensearch.knn.indices.ModelMetadata; import org.opensearch.knn.indices.ModelUtil; diff --git a/src/main/java/org/opensearch/knn/index/KNNClusterUtil.java b/src/main/java/org/opensearch/knn/index/util/KNNClusterUtil.java similarity index 97% rename from src/main/java/org/opensearch/knn/index/KNNClusterUtil.java rename to src/main/java/org/opensearch/knn/index/util/KNNClusterUtil.java index 63a49f095..c9c96bea5 100644 --- a/src/main/java/org/opensearch/knn/index/KNNClusterUtil.java +++ b/src/main/java/org/opensearch/knn/index/util/KNNClusterUtil.java @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.knn.index; +package org.opensearch.knn.index.util; import lombok.AccessLevel; import lombok.NoArgsConstructor; diff --git a/src/main/java/org/opensearch/knn/index/util/QueryContext.java b/src/main/java/org/opensearch/knn/index/util/QueryContext.java deleted file mode 100644 index 6bb495814..000000000 --- a/src/main/java/org/opensearch/knn/index/util/QueryContext.java +++ /dev/null @@ -1,23 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - * - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. - */ - -package org.opensearch.knn.index.util; - -import lombok.AllArgsConstructor; -import org.opensearch.knn.index.VectorQueryType; - -/** - * Context class for query-specific information. - */ -@AllArgsConstructor -public class QueryContext { - VectorQueryType queryType; -} diff --git a/src/main/java/org/opensearch/knn/indices/ModelDao.java b/src/main/java/org/opensearch/knn/indices/ModelDao.java index 37edcd3ae..e95596699 100644 --- a/src/main/java/org/opensearch/knn/indices/ModelDao.java +++ b/src/main/java/org/opensearch/knn/indices/ModelDao.java @@ -50,7 +50,7 @@ import org.opensearch.index.IndexNotFoundException; import org.opensearch.knn.common.KNNConstants; import org.opensearch.knn.common.exception.DeleteModelException; -import org.opensearch.knn.index.MethodComponentContext; +import org.opensearch.knn.index.engine.MethodComponentContext; import org.opensearch.knn.plugin.transport.DeleteModelResponse; import org.opensearch.knn.plugin.transport.GetModelResponse; import org.opensearch.knn.plugin.transport.RemoveModelFromCacheAction; diff --git a/src/main/java/org/opensearch/knn/indices/ModelMetadata.java b/src/main/java/org/opensearch/knn/indices/ModelMetadata.java index 6bfb3aaf2..60301e244 100644 --- a/src/main/java/org/opensearch/knn/indices/ModelMetadata.java +++ b/src/main/java/org/opensearch/knn/indices/ModelMetadata.java @@ -23,11 +23,11 @@ import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.knn.common.KNNConstants; -import org.opensearch.knn.index.IndexUtil; -import org.opensearch.knn.index.MethodComponentContext; +import org.opensearch.knn.index.util.IndexUtil; +import org.opensearch.knn.index.engine.MethodComponentContext; import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.VectorDataType; -import org.opensearch.knn.index.util.KNNEngine; +import org.opensearch.knn.index.engine.KNNEngine; import java.io.IOException; import java.util.Map; diff --git a/src/main/java/org/opensearch/knn/jni/FaissService.java b/src/main/java/org/opensearch/knn/jni/FaissService.java index 1f23f6fcd..4f57b616a 100644 --- a/src/main/java/org/opensearch/knn/jni/FaissService.java +++ b/src/main/java/org/opensearch/knn/jni/FaissService.java @@ -13,7 +13,7 @@ import org.opensearch.knn.common.KNNConstants; import org.opensearch.knn.index.query.KNNQueryResult; -import org.opensearch.knn.index.util.KNNEngine; +import org.opensearch.knn.index.engine.KNNEngine; import java.security.AccessController; import java.security.PrivilegedAction; diff --git a/src/main/java/org/opensearch/knn/jni/JNIService.java b/src/main/java/org/opensearch/knn/jni/JNIService.java index 2a8d3ea8f..de696b5ce 100644 --- a/src/main/java/org/opensearch/knn/jni/JNIService.java +++ b/src/main/java/org/opensearch/knn/jni/JNIService.java @@ -13,9 +13,9 @@ import org.apache.commons.lang.ArrayUtils; import org.opensearch.common.Nullable; -import org.opensearch.knn.index.IndexUtil; +import org.opensearch.knn.index.util.IndexUtil; import org.opensearch.knn.index.query.KNNQueryResult; -import org.opensearch.knn.index.util.KNNEngine; +import org.opensearch.knn.index.engine.KNNEngine; import java.util.Map; diff --git a/src/main/java/org/opensearch/knn/jni/NmslibService.java b/src/main/java/org/opensearch/knn/jni/NmslibService.java index 294c5a208..fc72673a8 100644 --- a/src/main/java/org/opensearch/knn/jni/NmslibService.java +++ b/src/main/java/org/opensearch/knn/jni/NmslibService.java @@ -13,7 +13,7 @@ import org.opensearch.knn.common.KNNConstants; import org.opensearch.knn.index.query.KNNQueryResult; -import org.opensearch.knn.index.util.KNNEngine; +import org.opensearch.knn.index.engine.KNNEngine; import java.security.AccessController; import java.security.PrivilegedAction; diff --git a/src/main/java/org/opensearch/knn/plugin/KNNPlugin.java b/src/main/java/org/opensearch/knn/plugin/KNNPlugin.java index 5301b6e4e..e237ac537 100644 --- a/src/main/java/org/opensearch/knn/plugin/KNNPlugin.java +++ b/src/main/java/org/opensearch/knn/plugin/KNNPlugin.java @@ -13,7 +13,7 @@ import org.opensearch.index.engine.EngineFactory; import org.opensearch.indices.SystemIndexDescriptor; import org.opensearch.knn.index.KNNCircuitBreaker; -import org.opensearch.knn.index.KNNClusterUtil; +import org.opensearch.knn.index.util.KNNClusterUtil; import org.opensearch.knn.index.mapper.KNNVectorFieldMapperUtil; import org.opensearch.knn.index.query.KNNQueryBuilder; import org.opensearch.knn.index.KNNSettings; @@ -23,7 +23,7 @@ import org.opensearch.knn.index.query.KNNWeight; import org.opensearch.knn.index.codec.KNNCodecService; import org.opensearch.knn.index.memory.NativeMemoryLoadStrategy; -import org.opensearch.knn.index.util.KNNEngine; +import org.opensearch.knn.index.engine.KNNEngine; import org.opensearch.knn.indices.ModelGraveyard; import org.opensearch.knn.indices.ModelCache; import org.opensearch.knn.indices.ModelDao; diff --git a/src/main/java/org/opensearch/knn/plugin/rest/RestTrainModelHandler.java b/src/main/java/org/opensearch/knn/plugin/rest/RestTrainModelHandler.java index e0b94ec76..58bcd1ebf 100644 --- a/src/main/java/org/opensearch/knn/plugin/rest/RestTrainModelHandler.java +++ b/src/main/java/org/opensearch/knn/plugin/rest/RestTrainModelHandler.java @@ -15,7 +15,7 @@ import org.opensearch.client.node.NodeClient; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.index.mapper.NumberFieldMapper; -import org.opensearch.knn.index.KNNMethodContext; +import org.opensearch.knn.index.engine.KNNMethodContext; import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.indices.ModelUtil; diff --git a/src/main/java/org/opensearch/knn/plugin/stats/KNNStats.java b/src/main/java/org/opensearch/knn/plugin/stats/KNNStats.java index 3ddc8d4b4..bcd419ea6 100644 --- a/src/main/java/org/opensearch/knn/plugin/stats/KNNStats.java +++ b/src/main/java/org/opensearch/knn/plugin/stats/KNNStats.java @@ -9,7 +9,7 @@ import com.google.common.collect.ImmutableMap; import org.opensearch.knn.common.KNNConstants; import org.opensearch.knn.index.memory.NativeMemoryCacheManager; -import org.opensearch.knn.index.util.KNNEngine; +import org.opensearch.knn.index.engine.KNNEngine; import org.opensearch.knn.indices.ModelCache; import org.opensearch.knn.indices.ModelDao; import org.opensearch.knn.plugin.stats.suppliers.EventOccurredWithinThresholdSupplier; diff --git a/src/main/java/org/opensearch/knn/plugin/stats/suppliers/LibraryInitializedSupplier.java b/src/main/java/org/opensearch/knn/plugin/stats/suppliers/LibraryInitializedSupplier.java index 800c1a827..efed0ac7c 100644 --- a/src/main/java/org/opensearch/knn/plugin/stats/suppliers/LibraryInitializedSupplier.java +++ b/src/main/java/org/opensearch/knn/plugin/stats/suppliers/LibraryInitializedSupplier.java @@ -11,7 +11,7 @@ package org.opensearch.knn.plugin.stats.suppliers; -import org.opensearch.knn.index.util.KNNLibrary; +import org.opensearch.knn.index.engine.KNNLibrary; import java.util.function.Supplier; diff --git a/src/main/java/org/opensearch/knn/plugin/transport/TrainingModelRequest.java b/src/main/java/org/opensearch/knn/plugin/transport/TrainingModelRequest.java index f7ad997b2..9464ea806 100644 --- a/src/main/java/org/opensearch/knn/plugin/transport/TrainingModelRequest.java +++ b/src/main/java/org/opensearch/knn/plugin/transport/TrainingModelRequest.java @@ -19,8 +19,8 @@ import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.knn.common.KNNConstants; -import org.opensearch.knn.index.IndexUtil; -import org.opensearch.knn.index.KNNMethodContext; +import org.opensearch.knn.index.util.IndexUtil; +import org.opensearch.knn.index.engine.KNNMethodContext; import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.indices.ModelDao; import org.opensearch.knn.training.VectorSpaceInfo; diff --git a/src/main/java/org/opensearch/knn/training/TrainingJob.java b/src/main/java/org/opensearch/knn/training/TrainingJob.java index 928396289..c0cfb72dc 100644 --- a/src/main/java/org/opensearch/knn/training/TrainingJob.java +++ b/src/main/java/org/opensearch/knn/training/TrainingJob.java @@ -16,11 +16,11 @@ import org.apache.logging.log4j.Logger; import org.opensearch.common.UUIDs; import org.opensearch.knn.common.KNNConstants; -import org.opensearch.knn.index.IndexUtil; +import org.opensearch.knn.index.util.IndexUtil; import org.opensearch.knn.index.KNNSettings; import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.jni.JNIService; -import org.opensearch.knn.index.KNNMethodContext; +import org.opensearch.knn.index.engine.KNNMethodContext; import org.opensearch.knn.index.memory.NativeMemoryAllocation; import org.opensearch.knn.index.memory.NativeMemoryCacheManager; import org.opensearch.knn.index.memory.NativeMemoryEntryContext; @@ -34,7 +34,7 @@ import java.util.Map; import java.util.Objects; -import static org.opensearch.knn.index.util.Faiss.FAISS_BINARY_INDEX_DESCRIPTION_PREFIX; +import static org.opensearch.knn.index.engine.faiss.Faiss.FAISS_BINARY_INDEX_DESCRIPTION_PREFIX; /** * Encapsulates all information required to generate and train a model. diff --git a/src/main/java/org/opensearch/knn/training/VectorReader.java b/src/main/java/org/opensearch/knn/training/VectorReader.java index e94d037bd..3935ee956 100644 --- a/src/main/java/org/opensearch/knn/training/VectorReader.java +++ b/src/main/java/org/opensearch/knn/training/VectorReader.java @@ -23,7 +23,7 @@ import org.opensearch.common.ValidationException; import org.opensearch.common.unit.TimeValue; import org.opensearch.index.query.ExistsQueryBuilder; -import org.opensearch.knn.index.IndexUtil; +import org.opensearch.knn.index.util.IndexUtil; import org.opensearch.search.SearchHit; import org.opensearch.search.sort.SortOrder; diff --git a/src/test/java/org/opensearch/knn/index/AdvancedFilteringUseCasesIT.java b/src/test/java/org/opensearch/knn/index/AdvancedFilteringUseCasesIT.java index 5380dae90..c32b179f9 100644 --- a/src/test/java/org/opensearch/knn/index/AdvancedFilteringUseCasesIT.java +++ b/src/test/java/org/opensearch/knn/index/AdvancedFilteringUseCasesIT.java @@ -20,7 +20,7 @@ import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.knn.KNNRestTestCase; import org.opensearch.knn.NestedKnnDocBuilder; -import org.opensearch.knn.index.util.KNNEngine; +import org.opensearch.knn.index.engine.KNNEngine; import java.io.IOException; import java.util.List; diff --git a/src/test/java/org/opensearch/knn/index/FaissHNSWFlatE2EIT.java b/src/test/java/org/opensearch/knn/index/FaissHNSWFlatE2EIT.java index 5b7a99ce9..f616248c3 100644 --- a/src/test/java/org/opensearch/knn/index/FaissHNSWFlatE2EIT.java +++ b/src/test/java/org/opensearch/knn/index/FaissHNSWFlatE2EIT.java @@ -25,8 +25,9 @@ import org.opensearch.knn.KNNResult; import org.opensearch.knn.TestUtils; import org.opensearch.knn.common.KNNConstants; +import org.opensearch.knn.index.engine.KNNMethod; import org.opensearch.knn.index.query.KNNQueryBuilder; -import org.opensearch.knn.index.util.KNNEngine; +import org.opensearch.knn.index.engine.KNNEngine; import org.opensearch.knn.plugin.script.KNNScoringUtil; import java.io.IOException; diff --git a/src/test/java/org/opensearch/knn/index/FaissIT.java b/src/test/java/org/opensearch/knn/index/FaissIT.java index db225b52d..b4057bec4 100644 --- a/src/test/java/org/opensearch/knn/index/FaissIT.java +++ b/src/test/java/org/opensearch/knn/index/FaissIT.java @@ -30,8 +30,9 @@ import org.opensearch.knn.KNNResult; import org.opensearch.knn.TestUtils; import org.opensearch.knn.common.KNNConstants; +import org.opensearch.knn.index.engine.KNNMethod; import org.opensearch.knn.index.query.KNNQueryBuilder; -import org.opensearch.knn.index.util.KNNEngine; +import org.opensearch.knn.index.engine.KNNEngine; import org.opensearch.knn.plugin.script.KNNScoringUtil; import java.io.IOException; diff --git a/src/test/java/org/opensearch/knn/index/KNNCreateIndexFromModelTests.java b/src/test/java/org/opensearch/knn/index/KNNCreateIndexFromModelTests.java index e9b78e7ec..f0e60ca98 100644 --- a/src/test/java/org/opensearch/knn/index/KNNCreateIndexFromModelTests.java +++ b/src/test/java/org/opensearch/knn/index/KNNCreateIndexFromModelTests.java @@ -17,8 +17,9 @@ import org.opensearch.common.settings.Settings; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.knn.KNNSingleNodeTestCase; +import org.opensearch.knn.index.engine.MethodComponentContext; import org.opensearch.knn.jni.JNIService; -import org.opensearch.knn.index.util.KNNEngine; +import org.opensearch.knn.index.engine.KNNEngine; import org.opensearch.knn.indices.Model; import org.opensearch.knn.indices.ModelDao; import org.opensearch.knn.indices.ModelMetadata; diff --git a/src/test/java/org/opensearch/knn/index/KNNMapperSearcherIT.java b/src/test/java/org/opensearch/knn/index/KNNMapperSearcherIT.java index a50691e4b..62bd15721 100644 --- a/src/test/java/org/opensearch/knn/index/KNNMapperSearcherIT.java +++ b/src/test/java/org/opensearch/knn/index/KNNMapperSearcherIT.java @@ -13,7 +13,7 @@ import org.apache.http.util.EntityUtils; import org.opensearch.client.Response; import org.opensearch.knn.index.query.KNNQueryBuilder; -import org.opensearch.knn.index.util.KNNEngine; +import org.opensearch.knn.index.engine.KNNEngine; import java.util.ArrayList; import java.util.Arrays; diff --git a/src/test/java/org/opensearch/knn/index/KNNMethodContextTests.java b/src/test/java/org/opensearch/knn/index/KNNMethodContextTests.java index cb294bc3d..9a867c58d 100644 --- a/src/test/java/org/opensearch/knn/index/KNNMethodContextTests.java +++ b/src/test/java/org/opensearch/knn/index/KNNMethodContextTests.java @@ -13,12 +13,14 @@ import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.knn.KNNTestCase; -import org.opensearch.knn.index.util.KNNEngine; +import org.opensearch.knn.index.engine.KNNEngine; import com.google.common.collect.ImmutableMap; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.index.mapper.MapperParsingException; +import org.opensearch.knn.index.engine.KNNMethodContext; +import org.opensearch.knn.index.engine.MethodComponentContext; import java.io.IOException; import java.util.Collections; diff --git a/src/test/java/org/opensearch/knn/index/LuceneEngineIT.java b/src/test/java/org/opensearch/knn/index/LuceneEngineIT.java index 26f22ff84..a39327b8f 100644 --- a/src/test/java/org/opensearch/knn/index/LuceneEngineIT.java +++ b/src/test/java/org/opensearch/knn/index/LuceneEngineIT.java @@ -24,7 +24,7 @@ import org.opensearch.knn.TestUtils; import org.opensearch.knn.common.KNNConstants; import org.opensearch.knn.index.query.KNNQueryBuilder; -import org.opensearch.knn.index.util.KNNEngine; +import org.opensearch.knn.index.engine.KNNEngine; import java.io.IOException; import java.util.Arrays; diff --git a/src/test/java/org/opensearch/knn/index/MethodComponentContextTests.java b/src/test/java/org/opensearch/knn/index/MethodComponentContextTests.java index 5ce1a76ac..719c32610 100644 --- a/src/test/java/org/opensearch/knn/index/MethodComponentContextTests.java +++ b/src/test/java/org/opensearch/knn/index/MethodComponentContextTests.java @@ -18,6 +18,7 @@ import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.index.mapper.MapperParsingException; +import org.opensearch.knn.index.engine.MethodComponentContext; import java.io.IOException; import java.util.Collections; diff --git a/src/test/java/org/opensearch/knn/index/NmslibIT.java b/src/test/java/org/opensearch/knn/index/NmslibIT.java index 22168b3e4..31272d89c 100644 --- a/src/test/java/org/opensearch/knn/index/NmslibIT.java +++ b/src/test/java/org/opensearch/knn/index/NmslibIT.java @@ -25,8 +25,9 @@ import org.opensearch.knn.KNNResult; import org.opensearch.knn.TestUtils; import org.opensearch.knn.common.KNNConstants; +import org.opensearch.knn.index.engine.KNNMethod; import org.opensearch.knn.index.query.KNNQueryBuilder; -import org.opensearch.knn.index.util.KNNEngine; +import org.opensearch.knn.index.engine.KNNEngine; import org.opensearch.knn.plugin.script.KNNScoringUtil; import java.io.IOException; diff --git a/src/test/java/org/opensearch/knn/index/OpenSearchIT.java b/src/test/java/org/opensearch/knn/index/OpenSearchIT.java index a751754fd..f7c262e3c 100644 --- a/src/test/java/org/opensearch/knn/index/OpenSearchIT.java +++ b/src/test/java/org/opensearch/knn/index/OpenSearchIT.java @@ -28,8 +28,9 @@ import org.opensearch.index.query.ExistsQueryBuilder; import org.opensearch.knn.TestUtils; import org.opensearch.knn.common.KNNConstants; +import org.opensearch.knn.index.engine.KNNMethod; import org.opensearch.knn.index.query.KNNQueryBuilder; -import org.opensearch.knn.index.util.KNNEngine; +import org.opensearch.knn.index.engine.KNNEngine; import org.opensearch.knn.plugin.script.KNNScoringUtil; import org.opensearch.core.rest.RestStatus; diff --git a/src/test/java/org/opensearch/knn/index/SpaceTypeTests.java b/src/test/java/org/opensearch/knn/index/SpaceTypeTests.java index d10cf5819..b0a6c1375 100644 --- a/src/test/java/org/opensearch/knn/index/SpaceTypeTests.java +++ b/src/test/java/org/opensearch/knn/index/SpaceTypeTests.java @@ -13,7 +13,7 @@ import org.apache.lucene.index.VectorSimilarityFunction; import org.opensearch.knn.KNNTestCase; -import org.opensearch.knn.index.util.KNNEngine; +import org.opensearch.knn.index.engine.KNNEngine; import java.util.Arrays; import java.util.Collections; diff --git a/src/test/java/org/opensearch/knn/index/VectorDataTypeIT.java b/src/test/java/org/opensearch/knn/index/VectorDataTypeIT.java index a0f78f327..ec2d49de0 100644 --- a/src/test/java/org/opensearch/knn/index/VectorDataTypeIT.java +++ b/src/test/java/org/opensearch/knn/index/VectorDataTypeIT.java @@ -20,7 +20,7 @@ import org.opensearch.knn.KNNResult; import org.opensearch.knn.common.KNNConstants; import org.opensearch.knn.index.query.KNNQueryBuilder; -import org.opensearch.knn.index.util.KNNEngine; +import org.opensearch.knn.index.engine.KNNEngine; import org.opensearch.core.rest.RestStatus; import org.opensearch.script.Script; diff --git a/src/test/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80CompoundFormatTests.java b/src/test/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80CompoundFormatTests.java index c370755ac..0ecabcce6 100644 --- a/src/test/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80CompoundFormatTests.java +++ b/src/test/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80CompoundFormatTests.java @@ -18,7 +18,7 @@ import org.opensearch.knn.KNNTestCase; import org.opensearch.knn.index.codec.KNN87Codec.KNN87Codec; import org.opensearch.knn.index.codec.KNNCodecTestUtil; -import org.opensearch.knn.index.util.KNNEngine; +import org.opensearch.knn.index.engine.KNNEngine; import java.io.IOException; import java.util.Arrays; diff --git a/src/test/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumerTests.java b/src/test/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumerTests.java index 6837a5ce5..aabfc2d9f 100644 --- a/src/test/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumerTests.java +++ b/src/test/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumerTests.java @@ -25,15 +25,15 @@ import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.knn.KNNTestCase; import org.opensearch.knn.common.KNNConstants; -import org.opensearch.knn.index.KNNMethodContext; +import org.opensearch.knn.index.engine.KNNMethodContext; import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.mapper.KNNVectorFieldMapper; -import org.opensearch.knn.index.MethodComponentContext; +import org.opensearch.knn.index.engine.MethodComponentContext; import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.codec.KNN87Codec.KNN87Codec; import org.opensearch.knn.index.codec.KNNCodecTestUtil; import org.opensearch.knn.index.codec.util.KNNCodecUtil; -import org.opensearch.knn.index.util.KNNEngine; +import org.opensearch.knn.index.engine.KNNEngine; import org.opensearch.knn.indices.Model; import org.opensearch.knn.indices.ModelCache; import org.opensearch.knn.indices.ModelDao; diff --git a/src/test/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsFormatTests.java b/src/test/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsFormatTests.java index f9f6a7ed9..322b714f2 100644 --- a/src/test/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsFormatTests.java +++ b/src/test/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsFormatTests.java @@ -47,7 +47,7 @@ import org.opensearch.knn.common.KNNConstants; import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.mapper.KNNVectorFieldMapper; -import org.opensearch.knn.index.util.KNNEngine; +import org.opensearch.knn.index.engine.KNNEngine; import java.io.IOException; import java.util.Arrays; diff --git a/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestCase.java b/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestCase.java index 66fe9770d..05e244fd4 100644 --- a/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestCase.java +++ b/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestCase.java @@ -19,9 +19,9 @@ import org.opensearch.index.mapper.MapperService; import org.opensearch.knn.KNNTestCase; import org.opensearch.knn.common.KNNConstants; -import org.opensearch.knn.index.KNNMethodContext; +import org.opensearch.knn.index.engine.KNNMethodContext; import org.opensearch.knn.index.KNNSettings; -import org.opensearch.knn.index.MethodComponentContext; +import org.opensearch.knn.index.engine.MethodComponentContext; import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.VectorField; @@ -42,7 +42,7 @@ import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.settings.Settings; import org.opensearch.knn.index.memory.NativeMemoryLoadStrategy; -import org.opensearch.knn.index.util.KNNEngine; +import org.opensearch.knn.index.engine.KNNEngine; import org.opensearch.knn.indices.Model; import org.opensearch.knn.indices.ModelCache; import org.opensearch.knn.indices.ModelDao; diff --git a/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestUtil.java b/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestUtil.java index 6acdfec5d..345de0eee 100644 --- a/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestUtil.java +++ b/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestUtil.java @@ -39,7 +39,7 @@ import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.codec.util.KNNVectorSerializer; import org.opensearch.knn.index.codec.util.KNNVectorSerializerFactory; -import org.opensearch.knn.index.util.KNNEngine; +import org.opensearch.knn.index.engine.KNNEngine; import org.opensearch.knn.jni.JNIService; import java.io.IOException; diff --git a/src/test/java/org/opensearch/knn/index/codec/params/KNNScalarQuantizedVectorsFormatParamsTests.java b/src/test/java/org/opensearch/knn/index/codec/params/KNNScalarQuantizedVectorsFormatParamsTests.java index bcba8ebbd..573e826e0 100644 --- a/src/test/java/org/opensearch/knn/index/codec/params/KNNScalarQuantizedVectorsFormatParamsTests.java +++ b/src/test/java/org/opensearch/knn/index/codec/params/KNNScalarQuantizedVectorsFormatParamsTests.java @@ -12,7 +12,7 @@ package org.opensearch.knn.index.codec.params; import junit.framework.TestCase; -import org.opensearch.knn.index.MethodComponentContext; +import org.opensearch.knn.index.engine.MethodComponentContext; import java.util.HashMap; import java.util.Map; diff --git a/src/test/java/org/opensearch/knn/index/util/AbstractKNNLibraryTests.java b/src/test/java/org/opensearch/knn/index/engine/AbstractKNNLibraryTests.java similarity index 98% rename from src/test/java/org/opensearch/knn/index/util/AbstractKNNLibraryTests.java rename to src/test/java/org/opensearch/knn/index/engine/AbstractKNNLibraryTests.java index 8e5ae24f9..9a3c8f4df 100644 --- a/src/test/java/org/opensearch/knn/index/util/AbstractKNNLibraryTests.java +++ b/src/test/java/org/opensearch/knn/index/engine/AbstractKNNLibraryTests.java @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.knn.index.util; +package org.opensearch.knn.index.engine; import com.google.common.collect.ImmutableMap; import org.opensearch.common.ValidationException; @@ -12,6 +12,7 @@ import org.opensearch.knn.KNNTestCase; import org.opensearch.knn.common.KNNConstants; import org.opensearch.knn.index.*; +import org.opensearch.knn.index.engine.model.QueryContext; import java.io.IOException; import java.util.Collections; diff --git a/src/test/java/org/opensearch/knn/index/util/KNNEngineTests.java b/src/test/java/org/opensearch/knn/index/engine/KNNEngineTests.java similarity index 92% rename from src/test/java/org/opensearch/knn/index/util/KNNEngineTests.java rename to src/test/java/org/opensearch/knn/index/engine/KNNEngineTests.java index bed0b7908..5a6ed8e52 100644 --- a/src/test/java/org/opensearch/knn/index/util/KNNEngineTests.java +++ b/src/test/java/org/opensearch/knn/index/engine/KNNEngineTests.java @@ -3,10 +3,13 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.knn.index.util; +package org.opensearch.knn.index.engine; import org.opensearch.knn.KNNTestCase; import org.opensearch.knn.common.KNNConstants; +import org.opensearch.knn.index.engine.faiss.Faiss; +import org.opensearch.knn.index.engine.lucene.Lucene; +import org.opensearch.knn.index.engine.nmslib.Nmslib; import java.util.Arrays; import java.util.List; diff --git a/src/test/java/org/opensearch/knn/index/KNNMethodTests.java b/src/test/java/org/opensearch/knn/index/engine/KNNMethodTests.java similarity index 95% rename from src/test/java/org/opensearch/knn/index/KNNMethodTests.java rename to src/test/java/org/opensearch/knn/index/engine/KNNMethodTests.java index 607ca849e..020033f21 100644 --- a/src/test/java/org/opensearch/knn/index/KNNMethodTests.java +++ b/src/test/java/org/opensearch/knn/index/engine/KNNMethodTests.java @@ -1,22 +1,16 @@ /* + * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - * - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. */ -package org.opensearch.knn.index; +package org.opensearch.knn.index.engine; import com.google.common.collect.ImmutableMap; import org.opensearch.knn.KNNTestCase; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.knn.common.KNNConstants; -import org.opensearch.knn.index.util.KNNEngine; +import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.training.VectorSpaceInfo; import java.io.IOException; diff --git a/src/test/java/org/opensearch/knn/index/MethodComponentTests.java b/src/test/java/org/opensearch/knn/index/engine/MethodComponentTests.java similarity index 96% rename from src/test/java/org/opensearch/knn/index/MethodComponentTests.java rename to src/test/java/org/opensearch/knn/index/engine/MethodComponentTests.java index 55ff963a2..c72f59be4 100644 --- a/src/test/java/org/opensearch/knn/index/MethodComponentTests.java +++ b/src/test/java/org/opensearch/knn/index/engine/MethodComponentTests.java @@ -1,15 +1,9 @@ /* + * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - * - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. */ -package org.opensearch.knn.index; +package org.opensearch.knn.index.engine; import com.google.common.collect.ImmutableMap; import org.opensearch.knn.KNNTestCase; diff --git a/src/test/java/org/opensearch/knn/index/util/NativeLibraryTests.java b/src/test/java/org/opensearch/knn/index/engine/NativeLibraryTests.java similarity index 97% rename from src/test/java/org/opensearch/knn/index/util/NativeLibraryTests.java rename to src/test/java/org/opensearch/knn/index/engine/NativeLibraryTests.java index 814712560..02322c4b0 100644 --- a/src/test/java/org/opensearch/knn/index/util/NativeLibraryTests.java +++ b/src/test/java/org/opensearch/knn/index/engine/NativeLibraryTests.java @@ -3,11 +3,10 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.knn.index.util; +package org.opensearch.knn.index.engine; import com.google.common.collect.ImmutableMap; import org.opensearch.knn.KNNTestCase; -import org.opensearch.knn.index.KNNMethod; import org.opensearch.knn.index.SpaceType; import java.util.Collections; diff --git a/src/test/java/org/opensearch/knn/index/ParameterTests.java b/src/test/java/org/opensearch/knn/index/engine/ParameterTests.java similarity index 95% rename from src/test/java/org/opensearch/knn/index/ParameterTests.java rename to src/test/java/org/opensearch/knn/index/engine/ParameterTests.java index 18b892499..7224e2824 100644 --- a/src/test/java/org/opensearch/knn/index/ParameterTests.java +++ b/src/test/java/org/opensearch/knn/index/engine/ParameterTests.java @@ -1,22 +1,16 @@ /* + * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - * - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. */ -package org.opensearch.knn.index; +package org.opensearch.knn.index.engine; import com.google.common.collect.ImmutableMap; import org.opensearch.knn.KNNTestCase; import org.opensearch.common.ValidationException; -import org.opensearch.knn.index.Parameter.IntegerParameter; -import org.opensearch.knn.index.Parameter.StringParameter; -import org.opensearch.knn.index.Parameter.MethodComponentContextParameter; +import org.opensearch.knn.index.engine.Parameter.IntegerParameter; +import org.opensearch.knn.index.engine.Parameter.StringParameter; +import org.opensearch.knn.index.engine.Parameter.MethodComponentContextParameter; import org.opensearch.knn.training.VectorSpaceInfo; import java.util.Map; diff --git a/src/test/java/org/opensearch/knn/index/util/FaissTests.java b/src/test/java/org/opensearch/knn/index/engine/faiss/FaissTests.java similarity index 97% rename from src/test/java/org/opensearch/knn/index/util/FaissTests.java rename to src/test/java/org/opensearch/knn/index/engine/faiss/FaissTests.java index 5dc348a29..e9033007b 100644 --- a/src/test/java/org/opensearch/knn/index/util/FaissTests.java +++ b/src/test/java/org/opensearch/knn/index/engine/faiss/FaissTests.java @@ -3,17 +3,17 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.knn.index.util; +package org.opensearch.knn.index.engine.faiss; import lombok.SneakyThrows; import org.opensearch.Version; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.knn.KNNTestCase; -import org.opensearch.knn.index.KNNMethodContext; -import org.opensearch.knn.index.MethodComponent; -import org.opensearch.knn.index.MethodComponentContext; -import org.opensearch.knn.index.Parameter; +import org.opensearch.knn.index.engine.KNNMethodContext; +import org.opensearch.knn.index.engine.MethodComponent; +import org.opensearch.knn.index.engine.MethodComponentContext; +import org.opensearch.knn.index.engine.Parameter; import java.io.IOException; import java.util.HashMap; diff --git a/src/test/java/org/opensearch/knn/index/util/LuceneTests.java b/src/test/java/org/opensearch/knn/index/engine/lucene/LuceneTests.java similarity index 97% rename from src/test/java/org/opensearch/knn/index/util/LuceneTests.java rename to src/test/java/org/opensearch/knn/index/engine/lucene/LuceneTests.java index c9ffd13b2..976bb4238 100644 --- a/src/test/java/org/opensearch/knn/index/util/LuceneTests.java +++ b/src/test/java/org/opensearch/knn/index/engine/lucene/LuceneTests.java @@ -3,14 +3,14 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.knn.index.util; +package org.opensearch.knn.index.engine.lucene; import org.apache.lucene.util.Version; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.knn.KNNTestCase; -import org.opensearch.knn.index.KNNMethod; -import org.opensearch.knn.index.KNNMethodContext; +import org.opensearch.knn.index.engine.KNNMethod; +import org.opensearch.knn.index.engine.KNNMethodContext; import org.opensearch.knn.index.SpaceType; import java.io.IOException; diff --git a/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java b/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java index 0d3c6ac8a..e68d34e88 100644 --- a/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java +++ b/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java @@ -27,14 +27,14 @@ import org.opensearch.index.mapper.MapperService; import org.opensearch.index.mapper.ParseContext; import org.opensearch.knn.KNNTestCase; -import org.opensearch.knn.index.KNNMethodContext; +import org.opensearch.knn.index.engine.KNNMethodContext; import org.opensearch.knn.index.KNNSettings; -import org.opensearch.knn.index.MethodComponentContext; +import org.opensearch.knn.index.engine.MethodComponentContext; import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.VectorField; import org.opensearch.knn.index.codec.util.KNNVectorSerializerFactory; -import org.opensearch.knn.index.util.KNNEngine; +import org.opensearch.knn.index.engine.KNNEngine; import org.opensearch.knn.indices.ModelDao; import org.opensearch.knn.indices.ModelMetadata; import org.opensearch.knn.indices.ModelState; diff --git a/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperUtilTests.java b/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperUtilTests.java index 7eebc140e..a4d597a41 100644 --- a/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperUtilTests.java +++ b/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperUtilTests.java @@ -15,12 +15,12 @@ import org.apache.lucene.util.BytesRef; import org.opensearch.knn.KNNTestCase; import org.opensearch.knn.common.KNNConstants; -import org.opensearch.knn.index.KNNMethodContext; -import org.opensearch.knn.index.MethodComponentContext; +import org.opensearch.knn.index.engine.KNNMethodContext; +import org.opensearch.knn.index.engine.MethodComponentContext; import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.codec.util.KNNVectorSerializerFactory; -import org.opensearch.knn.index.util.KNNEngine; +import org.opensearch.knn.index.engine.KNNEngine; import org.opensearch.knn.indices.ModelDao; import org.opensearch.knn.indices.ModelMetadata; import org.opensearch.knn.indices.ModelState; diff --git a/src/test/java/org/opensearch/knn/index/mapper/MethodFieldMapperTests.java b/src/test/java/org/opensearch/knn/index/mapper/MethodFieldMapperTests.java index 04fc7cf06..9ae2fad9c 100644 --- a/src/test/java/org/opensearch/knn/index/mapper/MethodFieldMapperTests.java +++ b/src/test/java/org/opensearch/knn/index/mapper/MethodFieldMapperTests.java @@ -7,7 +7,7 @@ import junit.framework.TestCase; import org.opensearch.index.mapper.FieldMapper; -import org.opensearch.knn.index.KNNMethodContext; +import org.opensearch.knn.index.engine.KNNMethodContext; import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.VectorDataType; diff --git a/src/test/java/org/opensearch/knn/index/memory/NativeMemoryAllocationTests.java b/src/test/java/org/opensearch/knn/index/memory/NativeMemoryAllocationTests.java index a98e07182..dc9a97fbf 100644 --- a/src/test/java/org/opensearch/knn/index/memory/NativeMemoryAllocationTests.java +++ b/src/test/java/org/opensearch/knn/index/memory/NativeMemoryAllocationTests.java @@ -15,12 +15,12 @@ import lombok.SneakyThrows; import org.opensearch.knn.KNNTestCase; import org.opensearch.knn.common.KNNConstants; -import org.opensearch.knn.index.IndexUtil; +import org.opensearch.knn.index.util.IndexUtil; import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.jni.JNICommons; import org.opensearch.knn.jni.JNIService; import org.opensearch.knn.index.SpaceType; -import org.opensearch.knn.index.util.KNNEngine; +import org.opensearch.knn.index.engine.KNNEngine; import org.opensearch.watcher.FileWatcher; import org.opensearch.watcher.WatcherHandle; diff --git a/src/test/java/org/opensearch/knn/index/memory/NativeMemoryEntryContextTests.java b/src/test/java/org/opensearch/knn/index/memory/NativeMemoryEntryContextTests.java index f87a069a2..385572cb4 100644 --- a/src/test/java/org/opensearch/knn/index/memory/NativeMemoryEntryContextTests.java +++ b/src/test/java/org/opensearch/knn/index/memory/NativeMemoryEntryContextTests.java @@ -14,9 +14,9 @@ import com.google.common.collect.ImmutableMap; import org.opensearch.cluster.service.ClusterService; import org.opensearch.knn.KNNTestCase; -import org.opensearch.knn.index.IndexUtil; +import org.opensearch.knn.index.util.IndexUtil; import org.opensearch.knn.index.VectorDataType; -import org.opensearch.knn.index.util.KNNEngine; +import org.opensearch.knn.index.engine.KNNEngine; import java.io.BufferedOutputStream; import java.io.IOException; diff --git a/src/test/java/org/opensearch/knn/index/memory/NativeMemoryLoadStrategyTests.java b/src/test/java/org/opensearch/knn/index/memory/NativeMemoryLoadStrategyTests.java index 7fac05271..51f95d29a 100644 --- a/src/test/java/org/opensearch/knn/index/memory/NativeMemoryLoadStrategyTests.java +++ b/src/test/java/org/opensearch/knn/index/memory/NativeMemoryLoadStrategyTests.java @@ -21,7 +21,7 @@ import org.opensearch.knn.jni.JNIService; import org.opensearch.knn.index.query.KNNQueryResult; import org.opensearch.knn.index.SpaceType; -import org.opensearch.knn.index.util.KNNEngine; +import org.opensearch.knn.index.engine.KNNEngine; import org.opensearch.knn.training.FloatTrainingDataConsumer; import org.opensearch.knn.training.VectorReader; import org.opensearch.watcher.ResourceWatcherService; diff --git a/src/test/java/org/opensearch/knn/index/memory/SharedIndexStateManagerTests.java b/src/test/java/org/opensearch/knn/index/memory/SharedIndexStateManagerTests.java index daf02c611..7e4b41730 100644 --- a/src/test/java/org/opensearch/knn/index/memory/SharedIndexStateManagerTests.java +++ b/src/test/java/org/opensearch/knn/index/memory/SharedIndexStateManagerTests.java @@ -15,7 +15,7 @@ import org.mockito.MockedStatic; import org.mockito.Mockito; import org.opensearch.knn.KNNTestCase; -import org.opensearch.knn.index.util.KNNEngine; +import org.opensearch.knn.index.engine.KNNEngine; import org.opensearch.knn.jni.JNIService; import static org.mockito.Mockito.mockStatic; diff --git a/src/test/java/org/opensearch/knn/index/memory/SharedIndexStateTests.java b/src/test/java/org/opensearch/knn/index/memory/SharedIndexStateTests.java index cddbec5c0..40c19f0b2 100644 --- a/src/test/java/org/opensearch/knn/index/memory/SharedIndexStateTests.java +++ b/src/test/java/org/opensearch/knn/index/memory/SharedIndexStateTests.java @@ -12,7 +12,7 @@ package org.opensearch.knn.index.memory; import org.opensearch.knn.KNNTestCase; -import org.opensearch.knn.index.util.KNNEngine; +import org.opensearch.knn.index.engine.KNNEngine; public class SharedIndexStateTests extends KNNTestCase { diff --git a/src/test/java/org/opensearch/knn/index/query/KNNQueryBuilderTests.java b/src/test/java/org/opensearch/knn/index/query/KNNQueryBuilderTests.java index d3c17d383..63d9a6c30 100644 --- a/src/test/java/org/opensearch/knn/index/query/KNNQueryBuilderTests.java +++ b/src/test/java/org/opensearch/knn/index/query/KNNQueryBuilderTests.java @@ -27,13 +27,13 @@ import org.opensearch.index.query.QueryShardContext; import org.opensearch.index.query.TermQueryBuilder; import org.opensearch.knn.KNNTestCase; -import org.opensearch.knn.index.KNNClusterUtil; -import org.opensearch.knn.index.KNNMethodContext; -import org.opensearch.knn.index.MethodComponentContext; +import org.opensearch.knn.index.util.KNNClusterUtil; +import org.opensearch.knn.index.engine.KNNMethodContext; +import org.opensearch.knn.index.engine.MethodComponentContext; import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.mapper.KNNVectorFieldMapper; -import org.opensearch.knn.index.util.KNNEngine; +import org.opensearch.knn.index.engine.KNNEngine; import org.opensearch.knn.indices.ModelDao; import org.opensearch.knn.indices.ModelMetadata; import org.opensearch.knn.indices.ModelState; @@ -52,7 +52,7 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; import static org.opensearch.knn.index.KNNClusterTestUtils.mockClusterService; -import static org.opensearch.knn.index.util.KNNEngine.ENGINES_SUPPORTING_RADIAL_SEARCH; +import static org.opensearch.knn.index.engine.KNNEngine.ENGINES_SUPPORTING_RADIAL_SEARCH; public class KNNQueryBuilderTests extends KNNTestCase { diff --git a/src/test/java/org/opensearch/knn/index/query/KNNQueryFactoryTests.java b/src/test/java/org/opensearch/knn/index/query/KNNQueryFactoryTests.java index 02b64cba5..c74a79946 100644 --- a/src/test/java/org/opensearch/knn/index/query/KNNQueryFactoryTests.java +++ b/src/test/java/org/opensearch/knn/index/query/KNNQueryFactoryTests.java @@ -24,7 +24,7 @@ import org.opensearch.index.search.NestedHelper; import org.opensearch.knn.KNNTestCase; import org.opensearch.knn.index.VectorDataType; -import org.opensearch.knn.index.util.KNNEngine; +import org.opensearch.knn.index.engine.KNNEngine; import java.util.Arrays; import java.util.List; diff --git a/src/test/java/org/opensearch/knn/index/query/KNNWeightTests.java b/src/test/java/org/opensearch/knn/index/query/KNNWeightTests.java index d08f7e0ce..c7077eace 100644 --- a/src/test/java/org/opensearch/knn/index/query/KNNWeightTests.java +++ b/src/test/java/org/opensearch/knn/index/query/KNNWeightTests.java @@ -38,14 +38,14 @@ import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.knn.KNNTestCase; import org.opensearch.knn.index.KNNSettings; -import org.opensearch.knn.index.MethodComponentContext; +import org.opensearch.knn.index.engine.MethodComponentContext; import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.codec.KNNCodecVersion; import org.opensearch.knn.index.codec.util.KNNVectorAsArraySerializer; import org.opensearch.knn.index.memory.NativeMemoryAllocation; import org.opensearch.knn.index.memory.NativeMemoryCacheManager; -import org.opensearch.knn.index.util.KNNEngine; +import org.opensearch.knn.index.engine.KNNEngine; import org.opensearch.knn.indices.ModelDao; import org.opensearch.knn.indices.ModelMetadata; import org.opensearch.knn.indices.ModelState; diff --git a/src/test/java/org/opensearch/knn/index/query/RNNQueryFactoryTests.java b/src/test/java/org/opensearch/knn/index/query/RNNQueryFactoryTests.java index af415f9c5..c41b8fbab 100644 --- a/src/test/java/org/opensearch/knn/index/query/RNNQueryFactoryTests.java +++ b/src/test/java/org/opensearch/knn/index/query/RNNQueryFactoryTests.java @@ -27,7 +27,7 @@ import org.opensearch.index.query.TermQueryBuilder; import org.opensearch.knn.KNNTestCase; import org.opensearch.knn.index.VectorDataType; -import org.opensearch.knn.index.util.KNNEngine; +import org.opensearch.knn.index.engine.KNNEngine; public class RNNQueryFactoryTests extends KNNTestCase { private static final String FILTER_FILED_NAME = "foo"; diff --git a/src/test/java/org/opensearch/knn/index/query/parser/KNNQueryBuilderParserTests.java b/src/test/java/org/opensearch/knn/index/query/parser/KNNQueryBuilderParserTests.java index 746057f1a..713e532f9 100644 --- a/src/test/java/org/opensearch/knn/index/query/parser/KNNQueryBuilderParserTests.java +++ b/src/test/java/org/opensearch/knn/index/query/parser/KNNQueryBuilderParserTests.java @@ -17,7 +17,7 @@ import org.opensearch.index.query.QueryBuilders; import org.opensearch.index.query.TermQueryBuilder; import org.opensearch.knn.KNNTestCase; -import org.opensearch.knn.index.KNNClusterUtil; +import org.opensearch.knn.index.util.KNNClusterUtil; import org.opensearch.knn.index.query.KNNQueryBuilder; import org.opensearch.plugins.SearchPlugin; diff --git a/src/test/java/org/opensearch/knn/index/IndexUtilTests.java b/src/test/java/org/opensearch/knn/index/util/IndexUtilTests.java similarity index 97% rename from src/test/java/org/opensearch/knn/index/IndexUtilTests.java rename to src/test/java/org/opensearch/knn/index/util/IndexUtilTests.java index 809c7d930..f2e85b1ad 100644 --- a/src/test/java/org/opensearch/knn/index/IndexUtilTests.java +++ b/src/test/java/org/opensearch/knn/index/util/IndexUtilTests.java @@ -1,15 +1,9 @@ /* + * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - * - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. */ -package org.opensearch.knn.index; +package org.opensearch.knn.index.util; import com.google.common.collect.ImmutableMap; import org.junit.BeforeClass; @@ -23,7 +17,12 @@ import org.opensearch.common.ValidationException; import org.opensearch.common.settings.Settings; import org.opensearch.knn.KNNTestCase; -import org.opensearch.knn.index.util.KNNEngine; +import org.opensearch.knn.index.KNNSettings; +import org.opensearch.knn.index.SpaceType; +import org.opensearch.knn.index.VectorDataType; +import org.opensearch.knn.index.engine.KNNEngine; +import org.opensearch.knn.index.engine.KNNMethodContext; +import org.opensearch.knn.index.engine.MethodComponentContext; import org.opensearch.knn.indices.ModelDao; import org.opensearch.knn.indices.ModelMetadata; import org.opensearch.knn.jni.JNIService; @@ -45,7 +44,7 @@ import static org.opensearch.knn.common.KNNConstants.METHOD_IVF; import static org.opensearch.knn.common.KNNConstants.SPACE_TYPE; import static org.opensearch.knn.common.KNNConstants.VECTOR_DATA_TYPE_FIELD; -import static org.opensearch.knn.index.IndexUtil.getParametersAtLoading; +import static org.opensearch.knn.index.util.IndexUtil.getParametersAtLoading; import static org.opensearch.knn.index.KNNSettings.KNN_ALGO_PARAM_EF_SEARCH; public class IndexUtilTests extends KNNTestCase { diff --git a/src/test/java/org/opensearch/knn/index/KNNClusterUtilTests.java b/src/test/java/org/opensearch/knn/index/util/KNNClusterUtilTests.java similarity index 97% rename from src/test/java/org/opensearch/knn/index/KNNClusterUtilTests.java rename to src/test/java/org/opensearch/knn/index/util/KNNClusterUtilTests.java index 0e00a7f75..f04db5c70 100644 --- a/src/test/java/org/opensearch/knn/index/KNNClusterUtilTests.java +++ b/src/test/java/org/opensearch/knn/index/util/KNNClusterUtilTests.java @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.knn.index; +package org.opensearch.knn.index.util; import org.opensearch.Version; import org.opensearch.cluster.service.ClusterService; diff --git a/src/test/java/org/opensearch/knn/indices/ModelCacheTests.java b/src/test/java/org/opensearch/knn/indices/ModelCacheTests.java index e0111204d..88f78e716 100644 --- a/src/test/java/org/opensearch/knn/indices/ModelCacheTests.java +++ b/src/test/java/org/opensearch/knn/indices/ModelCacheTests.java @@ -17,10 +17,10 @@ import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; import org.opensearch.knn.KNNTestCase; -import org.opensearch.knn.index.MethodComponentContext; +import org.opensearch.knn.index.engine.MethodComponentContext; import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.VectorDataType; -import org.opensearch.knn.index.util.KNNEngine; +import org.opensearch.knn.index.engine.KNNEngine; import java.time.ZoneOffset; import java.time.ZonedDateTime; diff --git a/src/test/java/org/opensearch/knn/indices/ModelDaoTests.java b/src/test/java/org/opensearch/knn/indices/ModelDaoTests.java index b18a7259e..d9dab081c 100644 --- a/src/test/java/org/opensearch/knn/indices/ModelDaoTests.java +++ b/src/test/java/org/opensearch/knn/indices/ModelDaoTests.java @@ -33,10 +33,10 @@ import org.opensearch.index.engine.VersionConflictEngineException; import org.opensearch.knn.KNNSingleNodeTestCase; import org.opensearch.knn.common.exception.DeleteModelException; -import org.opensearch.knn.index.MethodComponentContext; +import org.opensearch.knn.index.engine.MethodComponentContext; import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.VectorDataType; -import org.opensearch.knn.index.util.KNNEngine; +import org.opensearch.knn.index.engine.KNNEngine; import org.opensearch.knn.plugin.transport.DeleteModelResponse; import org.opensearch.knn.plugin.transport.GetModelResponse; import org.opensearch.knn.plugin.transport.RemoveModelFromCacheAction; diff --git a/src/test/java/org/opensearch/knn/indices/ModelMetadataTests.java b/src/test/java/org/opensearch/knn/indices/ModelMetadataTests.java index c23b7e2fd..04fa50262 100644 --- a/src/test/java/org/opensearch/knn/indices/ModelMetadataTests.java +++ b/src/test/java/org/opensearch/knn/indices/ModelMetadataTests.java @@ -17,10 +17,10 @@ import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.knn.KNNTestCase; import org.opensearch.knn.common.KNNConstants; -import org.opensearch.knn.index.MethodComponentContext; +import org.opensearch.knn.index.engine.MethodComponentContext; import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.VectorDataType; -import org.opensearch.knn.index.util.KNNEngine; +import org.opensearch.knn.index.engine.KNNEngine; import java.io.IOException; import java.time.ZoneId; diff --git a/src/test/java/org/opensearch/knn/indices/ModelTests.java b/src/test/java/org/opensearch/knn/indices/ModelTests.java index 59bfe035f..45e8b05f1 100644 --- a/src/test/java/org/opensearch/knn/indices/ModelTests.java +++ b/src/test/java/org/opensearch/knn/indices/ModelTests.java @@ -13,10 +13,10 @@ import org.opensearch.knn.KNNTestCase; import org.opensearch.knn.common.KNNConstants; -import org.opensearch.knn.index.MethodComponentContext; +import org.opensearch.knn.index.engine.MethodComponentContext; import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.VectorDataType; -import org.opensearch.knn.index.util.KNNEngine; +import org.opensearch.knn.index.engine.KNNEngine; import java.time.ZoneOffset; import java.time.ZonedDateTime; diff --git a/src/test/java/org/opensearch/knn/integ/BinaryIndexIT.java b/src/test/java/org/opensearch/knn/integ/BinaryIndexIT.java index 2b6cd3e56..f75cf7b75 100644 --- a/src/test/java/org/opensearch/knn/integ/BinaryIndexIT.java +++ b/src/test/java/org/opensearch/knn/integ/BinaryIndexIT.java @@ -19,7 +19,7 @@ import org.opensearch.knn.KNNResult; import org.opensearch.knn.TestUtils; import org.opensearch.knn.index.VectorDataType; -import org.opensearch.knn.index.util.KNNEngine; +import org.opensearch.knn.index.engine.KNNEngine; import java.io.IOException; import java.net.URL; diff --git a/src/test/java/org/opensearch/knn/integ/BinaryIndexInvalidMappingIT.java b/src/test/java/org/opensearch/knn/integ/BinaryIndexInvalidMappingIT.java index 647860262..5832f2718 100644 --- a/src/test/java/org/opensearch/knn/integ/BinaryIndexInvalidMappingIT.java +++ b/src/test/java/org/opensearch/knn/integ/BinaryIndexInvalidMappingIT.java @@ -12,7 +12,7 @@ import org.opensearch.knn.KNNJsonIndexMappingsBuilder; import org.opensearch.knn.KNNRestTestCase; import org.opensearch.knn.index.VectorDataType; -import org.opensearch.knn.index.util.KNNEngine; +import org.opensearch.knn.index.engine.KNNEngine; import java.io.IOException; import java.util.Arrays; diff --git a/src/test/java/org/opensearch/knn/integ/FilteredSearchBinaryIT.java b/src/test/java/org/opensearch/knn/integ/FilteredSearchBinaryIT.java index 0ca5d792e..6d0c8fde4 100644 --- a/src/test/java/org/opensearch/knn/integ/FilteredSearchBinaryIT.java +++ b/src/test/java/org/opensearch/knn/integ/FilteredSearchBinaryIT.java @@ -17,7 +17,7 @@ import org.opensearch.knn.KNNRestTestCase; import org.opensearch.knn.index.KNNSettings; import org.opensearch.knn.index.VectorDataType; -import org.opensearch.knn.index.util.KNNEngine; +import org.opensearch.knn.index.engine.KNNEngine; import java.util.List; diff --git a/src/test/java/org/opensearch/knn/integ/KNNScriptScoringIT.java b/src/test/java/org/opensearch/knn/integ/KNNScriptScoringIT.java index d8252ad0f..a1a6e3aa6 100644 --- a/src/test/java/org/opensearch/knn/integ/KNNScriptScoringIT.java +++ b/src/test/java/org/opensearch/knn/integ/KNNScriptScoringIT.java @@ -31,7 +31,7 @@ import org.opensearch.core.rest.RestStatus; import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.mapper.KNNVectorFieldMapper; -import org.opensearch.knn.index.util.KNNEngine; +import org.opensearch.knn.index.engine.KNNEngine; import org.opensearch.knn.plugin.script.KNNScoringScriptEngine; import org.opensearch.knn.plugin.script.KNNScoringSpace; import org.opensearch.knn.plugin.script.KNNScoringSpaceFactory; diff --git a/src/test/java/org/opensearch/knn/integ/NestedSearchBinaryIT.java b/src/test/java/org/opensearch/knn/integ/NestedSearchBinaryIT.java index bff5588b6..bb3767370 100644 --- a/src/test/java/org/opensearch/knn/integ/NestedSearchBinaryIT.java +++ b/src/test/java/org/opensearch/knn/integ/NestedSearchBinaryIT.java @@ -17,7 +17,7 @@ import org.opensearch.knn.NestedKnnDocBuilder; import org.opensearch.knn.index.KNNSettings; import org.opensearch.knn.index.VectorDataType; -import org.opensearch.knn.index.util.KNNEngine; +import org.opensearch.knn.index.engine.KNNEngine; import java.util.List; diff --git a/src/test/java/org/opensearch/knn/integ/NestedSearchIT.java b/src/test/java/org/opensearch/knn/integ/NestedSearchIT.java index 1bf6037dc..2f32021f4 100644 --- a/src/test/java/org/opensearch/knn/integ/NestedSearchIT.java +++ b/src/test/java/org/opensearch/knn/integ/NestedSearchIT.java @@ -18,7 +18,7 @@ import org.opensearch.knn.NestedKnnDocBuilder; import org.opensearch.knn.index.KNNSettings; import org.opensearch.knn.index.SpaceType; -import org.opensearch.knn.index.util.KNNEngine; +import org.opensearch.knn.index.engine.KNNEngine; import java.io.IOException; import java.util.List; diff --git a/src/test/java/org/opensearch/knn/integ/PainlessScriptIT.java b/src/test/java/org/opensearch/knn/integ/PainlessScriptIT.java index 03273bffa..2fed9fc25 100644 --- a/src/test/java/org/opensearch/knn/integ/PainlessScriptIT.java +++ b/src/test/java/org/opensearch/knn/integ/PainlessScriptIT.java @@ -11,8 +11,8 @@ import org.opensearch.knn.KNNRestTestCase; import org.opensearch.knn.KNNResult; import org.opensearch.knn.common.KNNConstants; -import org.opensearch.knn.index.KNNMethodContext; -import org.opensearch.knn.index.MethodComponentContext; +import org.opensearch.knn.index.engine.KNNMethodContext; +import org.opensearch.knn.index.engine.MethodComponentContext; import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.mapper.KNNVectorFieldMapper; @@ -24,7 +24,7 @@ import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.index.query.MatchAllQueryBuilder; import org.opensearch.index.query.QueryBuilder; -import org.opensearch.knn.index.util.KNNEngine; +import org.opensearch.knn.index.engine.KNNEngine; import org.opensearch.core.rest.RestStatus; import org.opensearch.script.Script; diff --git a/src/test/java/org/opensearch/knn/jni/JNIServiceTests.java b/src/test/java/org/opensearch/knn/jni/JNIServiceTests.java index 78b878b90..de3a3fcb2 100644 --- a/src/test/java/org/opensearch/knn/jni/JNIServiceTests.java +++ b/src/test/java/org/opensearch/knn/jni/JNIServiceTests.java @@ -21,12 +21,12 @@ import org.opensearch.knn.KNNTestCase; import org.opensearch.knn.TestUtils; import org.opensearch.knn.common.KNNConstants; -import org.opensearch.knn.index.KNNMethodContext; +import org.opensearch.knn.index.engine.KNNMethodContext; import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.query.KNNQueryResult; -import org.opensearch.knn.index.MethodComponentContext; +import org.opensearch.knn.index.engine.MethodComponentContext; import org.opensearch.knn.index.SpaceType; -import org.opensearch.knn.index.util.KNNEngine; +import org.opensearch.knn.index.engine.KNNEngine; import java.io.IOException; import java.net.URL; diff --git a/src/test/java/org/opensearch/knn/plugin/action/RestGetModelHandlerIT.java b/src/test/java/org/opensearch/knn/plugin/action/RestGetModelHandlerIT.java index 91402b444..21369f9ff 100644 --- a/src/test/java/org/opensearch/knn/plugin/action/RestGetModelHandlerIT.java +++ b/src/test/java/org/opensearch/knn/plugin/action/RestGetModelHandlerIT.java @@ -36,7 +36,7 @@ import static org.opensearch.knn.common.KNNConstants.MODEL_STATE; import static org.opensearch.knn.common.KNNConstants.MODEL_TIMESTAMP; import static org.opensearch.knn.index.SpaceType.L2; -import static org.opensearch.knn.index.util.KNNEngine.FAISS; +import static org.opensearch.knn.index.engine.KNNEngine.FAISS; /** * Integration tests to check the correctness of {@link org.opensearch.knn.plugin.rest.RestGetModelHandler} diff --git a/src/test/java/org/opensearch/knn/plugin/action/RestSearchModelHandlerIT.java b/src/test/java/org/opensearch/knn/plugin/action/RestSearchModelHandlerIT.java index f4ab352ea..5d5f6d3bf 100644 --- a/src/test/java/org/opensearch/knn/plugin/action/RestSearchModelHandlerIT.java +++ b/src/test/java/org/opensearch/knn/plugin/action/RestSearchModelHandlerIT.java @@ -36,7 +36,7 @@ import static org.opensearch.knn.common.KNNConstants.SEARCH_MODEL_MAX_SIZE; import static org.opensearch.knn.common.KNNConstants.SEARCH_MODEL_MIN_SIZE; import static org.opensearch.knn.index.SpaceType.L2; -import static org.opensearch.knn.index.util.KNNEngine.FAISS; +import static org.opensearch.knn.index.engine.KNNEngine.FAISS; /** * Integration tests to check the correctness of {@link org.opensearch.knn.plugin.rest.RestSearchModelHandler} diff --git a/src/test/java/org/opensearch/knn/plugin/script/KNNScoringSpaceTests.java b/src/test/java/org/opensearch/knn/plugin/script/KNNScoringSpaceTests.java index aa0261657..07385e55b 100644 --- a/src/test/java/org/opensearch/knn/plugin/script/KNNScoringSpaceTests.java +++ b/src/test/java/org/opensearch/knn/plugin/script/KNNScoringSpaceTests.java @@ -12,7 +12,7 @@ import lombok.SneakyThrows; import org.opensearch.index.mapper.MappedFieldType; import org.opensearch.knn.KNNTestCase; -import org.opensearch.knn.index.KNNMethodContext; +import org.opensearch.knn.index.engine.KNNMethodContext; import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.mapper.KNNVectorFieldMapper; diff --git a/src/test/java/org/opensearch/knn/plugin/stats/suppliers/LibraryInitializedSupplierTests.java b/src/test/java/org/opensearch/knn/plugin/stats/suppliers/LibraryInitializedSupplierTests.java index d66241376..3a413c595 100644 --- a/src/test/java/org/opensearch/knn/plugin/stats/suppliers/LibraryInitializedSupplierTests.java +++ b/src/test/java/org/opensearch/knn/plugin/stats/suppliers/LibraryInitializedSupplierTests.java @@ -12,11 +12,11 @@ package org.opensearch.knn.plugin.stats.suppliers; import org.opensearch.common.ValidationException; -import org.opensearch.knn.index.util.EngineSpecificMethodContext; -import org.opensearch.knn.index.KNNMethod; -import org.opensearch.knn.index.KNNMethodContext; +import org.opensearch.knn.index.engine.EngineSpecificMethodContext; +import org.opensearch.knn.index.engine.KNNMethod; +import org.opensearch.knn.index.engine.KNNMethodContext; import org.opensearch.knn.index.SpaceType; -import org.opensearch.knn.index.util.KNNLibrary; +import org.opensearch.knn.index.engine.KNNLibrary; import org.opensearch.knn.training.VectorSpaceInfo; import org.opensearch.test.OpenSearchTestCase; diff --git a/src/test/java/org/opensearch/knn/plugin/transport/GetModelResponseTests.java b/src/test/java/org/opensearch/knn/plugin/transport/GetModelResponseTests.java index 06cebff7b..71e8f15a6 100644 --- a/src/test/java/org/opensearch/knn/plugin/transport/GetModelResponseTests.java +++ b/src/test/java/org/opensearch/knn/plugin/transport/GetModelResponseTests.java @@ -18,11 +18,11 @@ import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentType; import org.opensearch.knn.KNNTestCase; -import org.opensearch.knn.index.KNNClusterUtil; -import org.opensearch.knn.index.MethodComponentContext; +import org.opensearch.knn.index.util.KNNClusterUtil; +import org.opensearch.knn.index.engine.MethodComponentContext; import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.VectorDataType; -import org.opensearch.knn.index.util.KNNEngine; +import org.opensearch.knn.index.engine.KNNEngine; import org.opensearch.knn.indices.Model; import org.opensearch.knn.indices.ModelMetadata; import org.opensearch.knn.indices.ModelState; diff --git a/src/test/java/org/opensearch/knn/plugin/transport/RemoveModelFromCacheTransportActionTests.java b/src/test/java/org/opensearch/knn/plugin/transport/RemoveModelFromCacheTransportActionTests.java index 381831fc7..8fdccdac0 100644 --- a/src/test/java/org/opensearch/knn/plugin/transport/RemoveModelFromCacheTransportActionTests.java +++ b/src/test/java/org/opensearch/knn/plugin/transport/RemoveModelFromCacheTransportActionTests.java @@ -17,10 +17,10 @@ import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; import org.opensearch.knn.KNNSingleNodeTestCase; -import org.opensearch.knn.index.MethodComponentContext; +import org.opensearch.knn.index.engine.MethodComponentContext; import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.VectorDataType; -import org.opensearch.knn.index.util.KNNEngine; +import org.opensearch.knn.index.engine.KNNEngine; import org.opensearch.knn.indices.Model; import org.opensearch.knn.indices.ModelCache; import org.opensearch.knn.indices.ModelDao; diff --git a/src/test/java/org/opensearch/knn/plugin/transport/TrainingJobRouterTransportActionTests.java b/src/test/java/org/opensearch/knn/plugin/transport/TrainingJobRouterTransportActionTests.java index 63c770a26..3515c690d 100644 --- a/src/test/java/org/opensearch/knn/plugin/transport/TrainingJobRouterTransportActionTests.java +++ b/src/test/java/org/opensearch/knn/plugin/transport/TrainingJobRouterTransportActionTests.java @@ -23,7 +23,7 @@ import org.opensearch.cluster.node.DiscoveryNodes; import org.opensearch.cluster.service.ClusterService; import org.opensearch.knn.KNNTestCase; -import org.opensearch.knn.index.KNNMethodContext; +import org.opensearch.knn.index.engine.KNNMethodContext; import org.opensearch.knn.index.VectorDataType; import org.opensearch.search.SearchHit; import org.opensearch.search.SearchHits; diff --git a/src/test/java/org/opensearch/knn/plugin/transport/TrainingModelRequestTests.java b/src/test/java/org/opensearch/knn/plugin/transport/TrainingModelRequestTests.java index 0fb478c83..53e59129e 100644 --- a/src/test/java/org/opensearch/knn/plugin/transport/TrainingModelRequestTests.java +++ b/src/test/java/org/opensearch/knn/plugin/transport/TrainingModelRequestTests.java @@ -23,12 +23,12 @@ import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.knn.KNNTestCase; import org.opensearch.knn.common.KNNConstants; -import org.opensearch.knn.index.KNNMethodContext; -import org.opensearch.knn.index.MethodComponentContext; +import org.opensearch.knn.index.engine.KNNMethodContext; +import org.opensearch.knn.index.engine.MethodComponentContext; import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.mapper.KNNVectorFieldMapper; import org.opensearch.knn.index.SpaceType; -import org.opensearch.knn.index.util.KNNEngine; +import org.opensearch.knn.index.engine.KNNEngine; import org.opensearch.knn.indices.ModelDao; import org.opensearch.knn.indices.ModelMetadata; import org.opensearch.knn.indices.ModelState; diff --git a/src/test/java/org/opensearch/knn/plugin/transport/TrainingModelTransportActionTests.java b/src/test/java/org/opensearch/knn/plugin/transport/TrainingModelTransportActionTests.java index 9ca790350..aea0e0b16 100644 --- a/src/test/java/org/opensearch/knn/plugin/transport/TrainingModelTransportActionTests.java +++ b/src/test/java/org/opensearch/knn/plugin/transport/TrainingModelTransportActionTests.java @@ -15,10 +15,10 @@ import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.knn.KNNSingleNodeTestCase; -import org.opensearch.knn.index.KNNMethodContext; +import org.opensearch.knn.index.engine.KNNMethodContext; import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.VectorDataType; -import org.opensearch.knn.index.util.KNNEngine; +import org.opensearch.knn.index.engine.KNNEngine; import org.opensearch.knn.indices.ModelDao; import java.io.IOException; diff --git a/src/test/java/org/opensearch/knn/plugin/transport/UpdateModelGraveyardTransportActionTests.java b/src/test/java/org/opensearch/knn/plugin/transport/UpdateModelGraveyardTransportActionTests.java index bad8d368b..bc6e098f3 100644 --- a/src/test/java/org/opensearch/knn/plugin/transport/UpdateModelGraveyardTransportActionTests.java +++ b/src/test/java/org/opensearch/knn/plugin/transport/UpdateModelGraveyardTransportActionTests.java @@ -15,10 +15,10 @@ import org.opensearch.knn.KNNSingleNodeTestCase; import org.opensearch.knn.TestUtils; import org.opensearch.knn.common.exception.DeleteModelException; -import org.opensearch.knn.index.MethodComponentContext; +import org.opensearch.knn.index.engine.MethodComponentContext; import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.VectorDataType; -import org.opensearch.knn.index.util.KNNEngine; +import org.opensearch.knn.index.engine.KNNEngine; import org.opensearch.knn.indices.Model; import org.opensearch.knn.indices.ModelGraveyard; import org.opensearch.knn.indices.ModelMetadata; diff --git a/src/test/java/org/opensearch/knn/plugin/transport/UpdateModelMetadataRequestTests.java b/src/test/java/org/opensearch/knn/plugin/transport/UpdateModelMetadataRequestTests.java index d2291c4ea..238fc5e45 100644 --- a/src/test/java/org/opensearch/knn/plugin/transport/UpdateModelMetadataRequestTests.java +++ b/src/test/java/org/opensearch/knn/plugin/transport/UpdateModelMetadataRequestTests.java @@ -13,10 +13,10 @@ import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.knn.KNNTestCase; -import org.opensearch.knn.index.MethodComponentContext; +import org.opensearch.knn.index.engine.MethodComponentContext; import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.VectorDataType; -import org.opensearch.knn.index.util.KNNEngine; +import org.opensearch.knn.index.engine.KNNEngine; import org.opensearch.knn.indices.ModelMetadata; import org.opensearch.knn.indices.ModelState; diff --git a/src/test/java/org/opensearch/knn/plugin/transport/UpdateModelMetadataTransportActionTests.java b/src/test/java/org/opensearch/knn/plugin/transport/UpdateModelMetadataTransportActionTests.java index c35c7effb..e5dcb2257 100644 --- a/src/test/java/org/opensearch/knn/plugin/transport/UpdateModelMetadataTransportActionTests.java +++ b/src/test/java/org/opensearch/knn/plugin/transport/UpdateModelMetadataTransportActionTests.java @@ -17,10 +17,10 @@ import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.knn.KNNSingleNodeTestCase; -import org.opensearch.knn.index.MethodComponentContext; +import org.opensearch.knn.index.engine.MethodComponentContext; import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.VectorDataType; -import org.opensearch.knn.index.util.KNNEngine; +import org.opensearch.knn.index.engine.KNNEngine; import org.opensearch.knn.indices.ModelMetadata; import org.opensearch.knn.indices.ModelState; import org.opensearch.threadpool.ThreadPool; diff --git a/src/test/java/org/opensearch/knn/recall/RecallTestsIT.java b/src/test/java/org/opensearch/knn/recall/RecallTestsIT.java index 656b95c2e..3dfb67415 100644 --- a/src/test/java/org/opensearch/knn/recall/RecallTestsIT.java +++ b/src/test/java/org/opensearch/knn/recall/RecallTestsIT.java @@ -19,7 +19,7 @@ import org.opensearch.knn.KNNRestTestCase; import org.opensearch.knn.TestUtils; import org.opensearch.knn.index.SpaceType; -import org.opensearch.knn.index.util.KNNEngine; +import org.opensearch.knn.index.engine.KNNEngine; import java.util.List; import java.util.Map; diff --git a/src/test/java/org/opensearch/knn/training/TrainingJobTests.java b/src/test/java/org/opensearch/knn/training/TrainingJobTests.java index 0852c39de..b6d76c68e 100644 --- a/src/test/java/org/opensearch/knn/training/TrainingJobTests.java +++ b/src/test/java/org/opensearch/knn/training/TrainingJobTests.java @@ -15,14 +15,14 @@ import org.opensearch.Version; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.knn.KNNTestCase; -import org.opensearch.knn.index.KNNMethodContext; -import org.opensearch.knn.index.MethodComponentContext; +import org.opensearch.knn.index.engine.KNNMethodContext; +import org.opensearch.knn.index.engine.MethodComponentContext; import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.memory.NativeMemoryAllocation; import org.opensearch.knn.index.memory.NativeMemoryCacheManager; import org.opensearch.knn.index.memory.NativeMemoryEntryContext; -import org.opensearch.knn.index.util.KNNEngine; +import org.opensearch.knn.index.engine.KNNEngine; import org.opensearch.knn.indices.Model; import org.opensearch.knn.indices.ModelMetadata; import org.opensearch.knn.indices.ModelState; diff --git a/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java b/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java index 32932ba9e..50b149830 100644 --- a/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java +++ b/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java @@ -101,7 +101,7 @@ import static org.opensearch.knn.common.KNNConstants.VECTOR_DATA_TYPE_FIELD; import static org.opensearch.knn.index.SpaceType.L2; import static org.opensearch.knn.index.memory.NativeMemoryCacheManager.GRAPH_COUNT; -import static org.opensearch.knn.index.util.KNNEngine.FAISS; +import static org.opensearch.knn.index.engine.KNNEngine.FAISS; import static org.opensearch.knn.plugin.stats.StatNames.INDICES_IN_CACHE; /** From f9de1765f6154512f401d3e346d9ba87166a344b Mon Sep 17 00:00:00 2001 From: Navneet Verma Date: Thu, 1 Aug 2024 12:32:12 -0700 Subject: [PATCH 313/416] Introduced KNNVectorValues interface to iterate on different types of Vector values during indexing and search (#1897) (#1919) Signed-off-by: Navneet Verma --- CHANGELOG.md | 1 + .../NativeEngineFieldVectorsWriter.java | 2 +- .../knn/index/codec/util/KNNCodecUtil.java | 2 +- .../vectorvalues/KNNBinaryVectorValues.java | 41 ++ .../vectorvalues/KNNByteVectorValues.java | 31 ++ .../vectorvalues/KNNFloatVectorValues.java | 29 ++ .../index/vectorvalues/KNNVectorValues.java | 85 ++++ .../vectorvalues/KNNVectorValuesFactory.java | 60 +++ .../vectorvalues/KNNVectorValuesIterator.java | 188 +++++++++ .../VectorValueExtractorStrategy.java | 126 ++++++ .../KNN80Codec/KNN80BinaryDocValuesTests.java | 6 +- .../KNN80DocValuesConsumerTests.java | 35 +- .../NativeEngineFieldVectorsWriterTests.java | 38 +- .../knn/index/codec/KNNCodecTestUtil.java | 149 +------ .../codec/util/BinaryDocValuesSubTests.java | 6 +- .../KNNVectorValuesFactoryTests.java | 61 +++ .../vectorvalues/KNNVectorValuesTests.java | 145 +++++++ .../index/vectorvalues/TestVectorValues.java | 363 ++++++++++++++++++ .../VectorValueExtractorStrategyTests.java | 29 ++ 19 files changed, 1215 insertions(+), 182 deletions(-) create mode 100644 src/main/java/org/opensearch/knn/index/vectorvalues/KNNBinaryVectorValues.java create mode 100644 src/main/java/org/opensearch/knn/index/vectorvalues/KNNByteVectorValues.java create mode 100644 src/main/java/org/opensearch/knn/index/vectorvalues/KNNFloatVectorValues.java create mode 100644 src/main/java/org/opensearch/knn/index/vectorvalues/KNNVectorValues.java create mode 100644 src/main/java/org/opensearch/knn/index/vectorvalues/KNNVectorValuesFactory.java create mode 100644 src/main/java/org/opensearch/knn/index/vectorvalues/KNNVectorValuesIterator.java create mode 100644 src/main/java/org/opensearch/knn/index/vectorvalues/VectorValueExtractorStrategy.java create mode 100644 src/test/java/org/opensearch/knn/index/vectorvalues/KNNVectorValuesFactoryTests.java create mode 100644 src/test/java/org/opensearch/knn/index/vectorvalues/KNNVectorValuesTests.java create mode 100644 src/test/java/org/opensearch/knn/index/vectorvalues/TestVectorValues.java create mode 100644 src/test/java/org/opensearch/knn/index/vectorvalues/VectorValueExtractorStrategyTests.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 69dd6a2e3..0623e6f98 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,5 +21,6 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Documentation ### Maintenance ### Refactoring +* Introduce KNNVectorValues interface to iterate on different types of Vector values during indexing and search [#1897](https://github.com/opensearch-project/k-NN/pull/1897) * Clean up parsing for query [#1824](https://github.com/opensearch-project/k-NN/pull/1824) * Refactor engine package structure [#1913](https://github.com/opensearch-project/k-NN/pull/1913) diff --git a/src/main/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngineFieldVectorsWriter.java b/src/main/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngineFieldVectorsWriter.java index 26171eece..e4860af31 100644 --- a/src/main/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngineFieldVectorsWriter.java +++ b/src/main/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngineFieldVectorsWriter.java @@ -54,7 +54,7 @@ static NativeEngineFieldVectorsWriter create(final FieldInfo fieldInfo, final throw new IllegalStateException("Unsupported Vector encoding : " + fieldInfo.getVectorEncoding()); } - NativeEngineFieldVectorsWriter(final FieldInfo fieldInfo, final InfoStream infoStream) { + private NativeEngineFieldVectorsWriter(final FieldInfo fieldInfo, final InfoStream infoStream) { this.fieldInfo = fieldInfo; this.infoStream = infoStream; vectors = new HashMap<>(); diff --git a/src/main/java/org/opensearch/knn/index/codec/util/KNNCodecUtil.java b/src/main/java/org/opensearch/knn/index/codec/util/KNNCodecUtil.java index 04aeb337f..d208d8179 100644 --- a/src/main/java/org/opensearch/knn/index/codec/util/KNNCodecUtil.java +++ b/src/main/java/org/opensearch/knn/index/codec/util/KNNCodecUtil.java @@ -115,7 +115,7 @@ public static String buildEngineFileSuffix(String fieldName, String extension) { return String.format("_%s%s", fieldName, extension); } - private static long getTotalLiveDocsCount(final BinaryDocValues binaryDocValues) { + public static long getTotalLiveDocsCount(final BinaryDocValues binaryDocValues) { long totalLiveDocs; if (binaryDocValues instanceof KNN80BinaryDocValues) { totalLiveDocs = ((KNN80BinaryDocValues) binaryDocValues).getTotalLiveDocs(); diff --git a/src/main/java/org/opensearch/knn/index/vectorvalues/KNNBinaryVectorValues.java b/src/main/java/org/opensearch/knn/index/vectorvalues/KNNBinaryVectorValues.java new file mode 100644 index 000000000..f38099b74 --- /dev/null +++ b/src/main/java/org/opensearch/knn/index/vectorvalues/KNNBinaryVectorValues.java @@ -0,0 +1,41 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.vectorvalues; + +import lombok.ToString; +import org.apache.lucene.codecs.KnnFieldVectorsWriter; +import org.apache.lucene.index.BinaryDocValues; +import org.apache.lucene.index.ByteVectorValues; + +import java.io.IOException; + +/** + * Concrete implementation of {@link KNNVectorValues} that returns byte[] as vector where binary vector is stored and + * provides an abstraction over {@link BinaryDocValues}, {@link ByteVectorValues}, {@link KnnFieldVectorsWriter} etc. + */ +@ToString(callSuper = true) +public class KNNBinaryVectorValues extends KNNVectorValues { + KNNBinaryVectorValues(KNNVectorValuesIterator vectorValuesIterator) { + super(vectorValuesIterator); + } + + @Override + public byte[] getVector() throws IOException { + final byte[] vector = VectorValueExtractorStrategy.extractBinaryVector(vectorValuesIterator); + this.dimension = vector.length; + return vector; + } + + /** + * Binary Vector values gets stored as byte[], hence for dimension of the binary vector we have to multiply the + * byte[] size with {@link Byte#SIZE} + * @return int + */ + @Override + public int dimension() { + return super.dimension() * Byte.SIZE; + } +} diff --git a/src/main/java/org/opensearch/knn/index/vectorvalues/KNNByteVectorValues.java b/src/main/java/org/opensearch/knn/index/vectorvalues/KNNByteVectorValues.java new file mode 100644 index 000000000..ccbbfab77 --- /dev/null +++ b/src/main/java/org/opensearch/knn/index/vectorvalues/KNNByteVectorValues.java @@ -0,0 +1,31 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.vectorvalues; + +import lombok.ToString; +import org.apache.lucene.codecs.KnnFieldVectorsWriter; +import org.apache.lucene.index.BinaryDocValues; +import org.apache.lucene.index.ByteVectorValues; + +import java.io.IOException; + +/** + * Concrete implementation of {@link KNNVectorValues} that returns float[] as vector and provides an abstraction over + * {@link BinaryDocValues}, {@link ByteVectorValues}, {@link KnnFieldVectorsWriter} etc. + */ +@ToString(callSuper = true) +public class KNNByteVectorValues extends KNNVectorValues { + KNNByteVectorValues(KNNVectorValuesIterator vectorValuesIterator) { + super(vectorValuesIterator); + } + + @Override + public byte[] getVector() throws IOException { + final byte[] vector = VectorValueExtractorStrategy.extractByteVector(vectorValuesIterator); + this.dimension = vector.length; + return vector; + } +} diff --git a/src/main/java/org/opensearch/knn/index/vectorvalues/KNNFloatVectorValues.java b/src/main/java/org/opensearch/knn/index/vectorvalues/KNNFloatVectorValues.java new file mode 100644 index 000000000..174f3a89e --- /dev/null +++ b/src/main/java/org/opensearch/knn/index/vectorvalues/KNNFloatVectorValues.java @@ -0,0 +1,29 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.vectorvalues; + +import org.apache.lucene.codecs.KnnFieldVectorsWriter; +import org.apache.lucene.index.BinaryDocValues; +import org.apache.lucene.index.FloatVectorValues; + +import java.io.IOException; + +/** + * Concrete implementation of {@link KNNVectorValues} that returns float[] as vector and provides an abstraction over + * {@link BinaryDocValues}, {@link FloatVectorValues}, {@link KnnFieldVectorsWriter} etc. + */ +public class KNNFloatVectorValues extends KNNVectorValues { + KNNFloatVectorValues(final KNNVectorValuesIterator vectorValuesIterator) { + super(vectorValuesIterator); + } + + @Override + public float[] getVector() throws IOException { + final float[] vector = VectorValueExtractorStrategy.extractFloatVector(vectorValuesIterator); + this.dimension = vector.length; + return vector; + } +} diff --git a/src/main/java/org/opensearch/knn/index/vectorvalues/KNNVectorValues.java b/src/main/java/org/opensearch/knn/index/vectorvalues/KNNVectorValues.java new file mode 100644 index 000000000..c4ed64bc2 --- /dev/null +++ b/src/main/java/org/opensearch/knn/index/vectorvalues/KNNVectorValues.java @@ -0,0 +1,85 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.vectorvalues; + +import lombok.ToString; +import org.apache.lucene.codecs.KnnFieldVectorsWriter; +import org.apache.lucene.index.BinaryDocValues; +import org.apache.lucene.index.ByteVectorValues; +import org.apache.lucene.index.FloatVectorValues; + +import java.io.IOException; + +/** + * An abstract class to iterate over KNNVectors, as KNNVectors are stored as different representation like + * {@link BinaryDocValues}, {@link FloatVectorValues}, {@link ByteVectorValues}, {@link KnnFieldVectorsWriter} etc. + * @param + */ +@ToString +public abstract class KNNVectorValues { + + protected final KNNVectorValuesIterator vectorValuesIterator; + protected int dimension; + + protected KNNVectorValues(final KNNVectorValuesIterator vectorValuesIterator) { + this.vectorValuesIterator = vectorValuesIterator; + } + + /** + * Return a vector reference. If you are adding this address in a List/Map ensure that you are copying the vector first. + * This is to ensure that we keep the heap and latency in check by reducing the copies of vectors. + * + * @return T an array of byte[], float[] + * @throws IOException if we are not able to get the vector + */ + public abstract T getVector() throws IOException; + + /** + * Dimension of vector is returned. Do call getVector function first before calling this function otherwise you will get 0 value. + * @return int + */ + public int dimension() { + assert docId() != -1 && dimension != 0 : "Cannot get dimension before we retrieve a vector from KNNVectorValues"; + return dimension; + } + + /** + * Returns the total live docs for KNNVectorValues. + * @return long + */ + public long totalLiveDocs() { + return vectorValuesIterator.liveDocs(); + } + + /** + * Returns the current docId where the iterator is pointing to. + * @return int + */ + public int docId() { + return vectorValuesIterator.docId(); + } + + /** + * Advances to a specific docId. Ensure that the passed docId is greater than current docId where Iterator is + * pointing to, otherwise + * {@link IOException} will be thrown + * @return int + * @throws IOException if we are not able to move to the passed docId. + */ + public int advance(int docId) throws IOException { + return vectorValuesIterator.advance(docId); + } + + /** + * Move to nextDocId. + * @return int + * @throws IOException if we cannot move to next docId + */ + public int nextDoc() throws IOException { + return vectorValuesIterator.nextDoc(); + } + +} diff --git a/src/main/java/org/opensearch/knn/index/vectorvalues/KNNVectorValuesFactory.java b/src/main/java/org/opensearch/knn/index/vectorvalues/KNNVectorValuesFactory.java new file mode 100644 index 000000000..5b6558f32 --- /dev/null +++ b/src/main/java/org/opensearch/knn/index/vectorvalues/KNNVectorValuesFactory.java @@ -0,0 +1,60 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.vectorvalues; + +import org.apache.lucene.index.DocsWithFieldSet; +import org.apache.lucene.search.DocIdSetIterator; +import org.opensearch.knn.index.VectorDataType; + +import java.util.Map; + +/** + * A factory class that provides various methods to create the {@link KNNVectorValues}. + */ +public final class KNNVectorValuesFactory { + + /** + * Returns a {@link KNNVectorValues} for the given {@link DocIdSetIterator} and {@link VectorDataType} + * + * @param vectorDataType {@link VectorDataType} + * @param docIdSetIterator {@link DocIdSetIterator} + * @return {@link KNNVectorValues} of type float[] + */ + public static KNNVectorValues getVectorValues(final VectorDataType vectorDataType, final DocIdSetIterator docIdSetIterator) { + return getVectorValues(vectorDataType, new KNNVectorValuesIterator.DocIdsIteratorValues(docIdSetIterator)); + } + + /** + * Returns a {@link KNNVectorValues} for the given {@link DocIdSetIterator} and a Map of docId and vectors. + * + * @param vectorDataType {@link VectorDataType} + * @param docIdWithFieldSet {@link DocsWithFieldSet} + * @return {@link KNNVectorValues} of type float[] + */ + public static KNNVectorValues getVectorValues( + final VectorDataType vectorDataType, + final DocsWithFieldSet docIdWithFieldSet, + final Map vectors + ) { + return getVectorValues(vectorDataType, new KNNVectorValuesIterator.FieldWriterIteratorValues(docIdWithFieldSet, vectors)); + } + + @SuppressWarnings("unchecked") + private static KNNVectorValues getVectorValues( + final VectorDataType vectorDataType, + final KNNVectorValuesIterator knnVectorValuesIterator + ) { + switch (vectorDataType) { + case FLOAT: + return (KNNVectorValues) new KNNFloatVectorValues(knnVectorValuesIterator); + case BYTE: + return (KNNVectorValues) new KNNByteVectorValues(knnVectorValuesIterator); + case BINARY: + return (KNNVectorValues) new KNNBinaryVectorValues(knnVectorValuesIterator); + } + throw new IllegalArgumentException("Invalid Vector data type provided, hence cannot return VectorValues"); + } +} diff --git a/src/main/java/org/opensearch/knn/index/vectorvalues/KNNVectorValuesIterator.java b/src/main/java/org/opensearch/knn/index/vectorvalues/KNNVectorValuesIterator.java new file mode 100644 index 000000000..4f1445c1c --- /dev/null +++ b/src/main/java/org/opensearch/knn/index/vectorvalues/KNNVectorValuesIterator.java @@ -0,0 +1,188 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.vectorvalues; + +import lombok.NonNull; +import org.apache.lucene.codecs.KnnFieldVectorsWriter; +import org.apache.lucene.index.BinaryDocValues; +import org.apache.lucene.index.ByteVectorValues; +import org.apache.lucene.index.DocsWithFieldSet; +import org.apache.lucene.index.FloatVectorValues; +import org.apache.lucene.search.DocIdSetIterator; +import org.opensearch.knn.index.codec.util.KNNCodecUtil; + +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.function.Function; + +/** + * An abstract class that provides an iterator to iterate over KNNVectors, as KNNVectors are stored as different + * representation like {@link BinaryDocValues}, {@link FloatVectorValues}, FieldWriter etc. How to iterate using this + * iterator please refer {@link DocIdsIteratorValues} java docs. + */ +public interface KNNVectorValuesIterator { + + /** + * Returns the current docId where the iterator is pointing to. + * @return int + */ + int docId(); + + /** + * Advances to a specific docId. Ensure that the passed docId is greater than current docId where Iterator is + * pointing to, otherwise + * {@link IOException} will be thrown + * @return int + * @throws IOException if we are not able to move to the passed docId. + */ + int advance(int docId) throws IOException; + + /** + * Move to nextDocId. If no more docs are present then {@link DocIdSetIterator#NO_MORE_DOCS} will be returned. + * @return int + * @throws IOException if we cannot move to next docId + */ + int nextDoc() throws IOException; + + /** + * Return a {@link DocIdSetIterator} + * @return {@link DocIdSetIterator} + */ + DocIdSetIterator getDocIdSetIterator(); + + /** + * Total number of live doc which will the iterator will iterate upon. + * @return long: total number of live docs + */ + long liveDocs(); + + /** + * Returns the {@link VectorValueExtractorStrategy} to extract the vector from the iterator. + * @return VectorValueExtractorStrategy + */ + VectorValueExtractorStrategy getVectorExtractorStrategy(); + + /** + * A DocIdsIteratorValues provides a common iteration logic for all Values that implements + * {@link DocIdSetIterator} interface. Example: {@link BinaryDocValues}, {@link FloatVectorValues} etc. + */ + class DocIdsIteratorValues implements KNNVectorValuesIterator { + protected DocIdSetIterator docIdSetIterator; + private static final List> VALID_ITERATOR_INSTANCE = List.of( + (itr) -> itr instanceof BinaryDocValues, + (itr) -> itr instanceof FloatVectorValues, + (itr) -> itr instanceof ByteVectorValues + ); + + DocIdsIteratorValues(@NonNull final DocIdSetIterator docIdSetIterator) { + validateIteratorType(docIdSetIterator); + this.docIdSetIterator = docIdSetIterator; + } + + @Override + public int docId() { + return docIdSetIterator.docID(); + } + + @Override + public int advance(int docId) throws IOException { + return docIdSetIterator.advance(docId); + } + + @Override + public int nextDoc() throws IOException { + return docIdSetIterator.nextDoc(); + } + + @Override + public DocIdSetIterator getDocIdSetIterator() { + return docIdSetIterator; + } + + @Override + public long liveDocs() { + if (docIdSetIterator instanceof BinaryDocValues) { + return KNNCodecUtil.getTotalLiveDocsCount((BinaryDocValues) docIdSetIterator); + } else if (docIdSetIterator instanceof FloatVectorValues || docIdSetIterator instanceof ByteVectorValues) { + return docIdSetIterator.cost(); + } + throw new IllegalArgumentException( + "DocIdSetIterator present is not of valid type. Valid types are: BinaryDocValues, FloatVectorValues and ByteVectorValues" + ); + } + + @Override + public VectorValueExtractorStrategy getVectorExtractorStrategy() { + return new VectorValueExtractorStrategy.DISIVectorExtractor(); + } + + private void validateIteratorType(final DocIdSetIterator docIdSetIterator) { + VALID_ITERATOR_INSTANCE.stream() + .map(v -> v.apply(docIdSetIterator)) + .filter(Boolean::booleanValue) + .findFirst() + .orElseThrow( + () -> new IllegalArgumentException( + "DocIdSetIterator present is not of valid type. Valid types are: BinaryDocValues, FloatVectorValues and ByteVectorValues" + ) + ); + } + } + + /** + * A FieldWriterIteratorValues is mainly used when Vectors are stored in {@link KnnFieldVectorsWriter} interface. + */ + class FieldWriterIteratorValues implements KNNVectorValuesIterator { + private final DocIdSetIterator docIdSetIterator; + private final Map vectors; + + FieldWriterIteratorValues(@NonNull final DocsWithFieldSet docsWithFieldSet, @NonNull final Map vectors) { + assert docsWithFieldSet.iterator().cost() == vectors.size(); + this.vectors = vectors; + this.docIdSetIterator = docsWithFieldSet.iterator(); + } + + @Override + public int docId() { + return docIdSetIterator.docID(); + } + + @Override + public int advance(int docId) throws IOException { + return docIdSetIterator.advance(docId); + } + + @Override + public int nextDoc() throws IOException { + return docIdSetIterator.nextDoc(); + } + + /** + * Returns a Map of docId and vector. + * @return {@link Map} + */ + public T vectorsValue() { + return vectors.get(docId()); + } + + @Override + public DocIdSetIterator getDocIdSetIterator() { + return docIdSetIterator; + } + + @Override + public long liveDocs() { + return docIdSetIterator.cost(); + } + + @Override + public VectorValueExtractorStrategy getVectorExtractorStrategy() { + return new VectorValueExtractorStrategy.FieldWriterIteratorVectorExtractor(); + } + } + +} diff --git a/src/main/java/org/opensearch/knn/index/vectorvalues/VectorValueExtractorStrategy.java b/src/main/java/org/opensearch/knn/index/vectorvalues/VectorValueExtractorStrategy.java new file mode 100644 index 000000000..07db4e7f6 --- /dev/null +++ b/src/main/java/org/opensearch/knn/index/vectorvalues/VectorValueExtractorStrategy.java @@ -0,0 +1,126 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.vectorvalues; + +import org.apache.lucene.index.BinaryDocValues; +import org.apache.lucene.index.ByteVectorValues; +import org.apache.lucene.index.FloatVectorValues; +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.util.ArrayUtil; +import org.apache.lucene.util.BytesRef; +import org.opensearch.knn.index.VectorDataType; +import org.opensearch.knn.index.codec.util.KNNVectorSerializer; +import org.opensearch.knn.index.codec.util.KNNVectorSerializerFactory; + +import java.io.IOException; + +/** + * Provides different strategies to extract the vectors from different {@link KNNVectorValuesIterator} + */ +interface VectorValueExtractorStrategy { + + /** + * Extract a float vector from KNNVectorValuesIterator. + * @param iterator {@link KNNVectorValuesIterator} + * @return float[] + * @throws IOException exception while retrieving the vectors + */ + static float[] extractFloatVector(final KNNVectorValuesIterator iterator) throws IOException { + return iterator.getVectorExtractorStrategy().extract(VectorDataType.FLOAT, iterator); + } + + /** + * Extract a byte vector from KNNVectorValuesIterator. + * @param iterator {@link KNNVectorValuesIterator} + * @return byte[] + * @throws IOException exception while retrieving the vectors + */ + static byte[] extractByteVector(final KNNVectorValuesIterator iterator) throws IOException { + return iterator.getVectorExtractorStrategy().extract(VectorDataType.BYTE, iterator); + } + + /** + * Extract a binary vector which is represented as byte[] from KNNVectorValuesIterator. + * @param iterator {@link KNNVectorValuesIterator} + * @return byte[] + * @throws IOException exception while retrieving the vectors + */ + static byte[] extractBinaryVector(final KNNVectorValuesIterator iterator) throws IOException { + return iterator.getVectorExtractorStrategy().extract(VectorDataType.BINARY, iterator); + } + + /** + * Extract Vector based on the vector datatype and vector values iterator. + * @param vectorDataType {@link VectorDataType} + * @param vectorValuesIterator {@link KNNVectorValuesIterator} + * @return vector + * @param could be of type float[], byte[] + * @throws IOException exception during extracting the vectors + */ + T extract(VectorDataType vectorDataType, KNNVectorValuesIterator vectorValuesIterator) throws IOException; + + /** + * Strategy to extract the vector from {@link KNNVectorValuesIterator.DocIdsIteratorValues} + */ + class DISIVectorExtractor implements VectorValueExtractorStrategy { + @Override + public T extract(final VectorDataType vectorDataType, final KNNVectorValuesIterator vectorValuesIterator) throws IOException { + final DocIdSetIterator docIdSetIterator = vectorValuesIterator.getDocIdSetIterator(); + switch (vectorDataType) { + case FLOAT: + if (docIdSetIterator instanceof BinaryDocValues) { + final BinaryDocValues values = (BinaryDocValues) docIdSetIterator; + return (T) getFloatVectorFromByteRef(values.binaryValue()); + } else if (docIdSetIterator instanceof FloatVectorValues) { + return (T) ((FloatVectorValues) docIdSetIterator).vectorValue(); + } + throw new IllegalArgumentException( + "VectorValuesIterator is not of a valid type. Valid Types are: BinaryDocValues and FloatVectorValues" + ); + case BYTE: + case BINARY: + if (docIdSetIterator instanceof BinaryDocValues) { + final BinaryDocValues values = (BinaryDocValues) docIdSetIterator; + final BytesRef bytesRef = values.binaryValue(); + return (T) ArrayUtil.copyOfSubArray(bytesRef.bytes, bytesRef.offset, bytesRef.offset + bytesRef.length); + } else if (docIdSetIterator instanceof ByteVectorValues) { + return (T) ((ByteVectorValues) docIdSetIterator).vectorValue(); + } + throw new IllegalArgumentException( + "VectorValuesIterator is not of a valid type. Valid Types are: BinaryDocValues and ByteVectorValues" + ); + } + throw new IllegalArgumentException("Valid Vector data type not passed to extract vector from DISIVectorExtractor strategy"); + } + + private float[] getFloatVectorFromByteRef(final BytesRef bytesRef) { + final KNNVectorSerializer vectorSerializer = KNNVectorSerializerFactory.getSerializerByBytesRef(bytesRef); + return vectorSerializer.byteToFloatArray(bytesRef); + } + } + + /** + * Strategy to extract the vector from {@link KNNVectorValuesIterator.FieldWriterIteratorValues} + */ + class FieldWriterIteratorVectorExtractor implements VectorValueExtractorStrategy { + + @SuppressWarnings("unchecked") + @Override + public T extract(final VectorDataType vectorDataType, final KNNVectorValuesIterator vectorValuesIterator) throws IOException { + switch (vectorDataType) { + case FLOAT: + return (T) ((KNNVectorValuesIterator.FieldWriterIteratorValues) vectorValuesIterator).vectorsValue(); + case BYTE: + case BINARY: + return (T) ((KNNVectorValuesIterator.FieldWriterIteratorValues) vectorValuesIterator).vectorsValue(); + } + throw new IllegalArgumentException( + "Valid Vector data type not passed to extract vector from FieldWriterIteratorVectorExtractor strategy" + ); + } + } + +} diff --git a/src/test/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80BinaryDocValuesTests.java b/src/test/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80BinaryDocValuesTests.java index 620559867..727cb8e6e 100644 --- a/src/test/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80BinaryDocValuesTests.java +++ b/src/test/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80BinaryDocValuesTests.java @@ -10,7 +10,7 @@ import org.apache.lucene.index.DocIDMerger; import org.apache.lucene.index.MergeState; import org.opensearch.knn.KNNTestCase; -import org.opensearch.knn.index.codec.KNNCodecTestUtil; +import org.opensearch.knn.index.vectorvalues.TestVectorValues; import org.opensearch.knn.index.codec.util.BinaryDocValuesSub; import java.io.IOException; @@ -30,7 +30,7 @@ public void testNextDoc() throws IOException { public int get(int docID) { return expectedDoc; } - }, new KNNCodecTestUtil.ConstantVectorBinaryDocValues(10, 128, 1.0f)); + }, new TestVectorValues.ConstantVectorBinaryDocValues(10, 128, 1.0f)); DocIDMerger docIDMerger = DocIDMerger.of(ImmutableList.of(sub), false); KNN80BinaryDocValues knn80BinaryDocValues = new KNN80BinaryDocValues(docIDMerger); @@ -53,7 +53,7 @@ public void testCost() { } public void testBinaryValue() throws IOException { - BinaryDocValues binaryDocValues = new KNNCodecTestUtil.ConstantVectorBinaryDocValues(10, 128, 1.0f); + BinaryDocValues binaryDocValues = new TestVectorValues.ConstantVectorBinaryDocValues(10, 128, 1.0f); BinaryDocValuesSub sub = new BinaryDocValuesSub(new MergeState.DocMap() { @Override public int get(int docID) { diff --git a/src/test/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumerTests.java b/src/test/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumerTests.java index aabfc2d9f..ce8fad384 100644 --- a/src/test/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumerTests.java +++ b/src/test/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumerTests.java @@ -27,6 +27,7 @@ import org.opensearch.knn.common.KNNConstants; import org.opensearch.knn.index.engine.KNNMethodContext; import org.opensearch.knn.index.VectorDataType; +import org.opensearch.knn.index.vectorvalues.TestVectorValues; import org.opensearch.knn.index.mapper.KNNVectorFieldMapper; import org.opensearch.knn.index.engine.MethodComponentContext; import org.opensearch.knn.index.SpaceType; @@ -69,8 +70,6 @@ import static org.opensearch.knn.index.codec.KNNCodecTestUtil.assertFileInCorrectLocation; import static org.opensearch.knn.index.codec.KNNCodecTestUtil.assertLoadableByEngine; import static org.opensearch.knn.index.codec.KNNCodecTestUtil.assertValidFooter; -import static org.opensearch.knn.index.codec.KNNCodecTestUtil.getRandomVectors; -import static org.opensearch.knn.index.codec.KNNCodecTestUtil.RandomVectorDocValuesProducer; public class KNN80DocValuesConsumerTests extends KNNTestCase { @@ -155,7 +154,10 @@ public void addKNNBinaryField(FieldInfo field, DocValuesProducer valuesProducer, public void testAddKNNBinaryField_noVectors() throws IOException { // When there are no new vectors, no more graph index requests should be added - RandomVectorDocValuesProducer randomVectorDocValuesProducer = new RandomVectorDocValuesProducer(0, 128); + TestVectorValues.RandomVectorDocValuesProducer randomVectorDocValuesProducer = new TestVectorValues.RandomVectorDocValuesProducer( + 0, + 128 + ); Long initialGraphIndexRequests = KNNCounter.GRAPH_INDEX_REQUESTS.getCount(); Long initialRefreshOperations = KNNGraphValue.REFRESH_TOTAL_OPERATIONS.getValue(); Long initialMergeOperations = KNNGraphValue.MERGE_TOTAL_OPERATIONS.getValue(); @@ -224,7 +226,10 @@ public void testAddKNNBinaryField_fromScratch_nmslibCurrent() throws IOException // Add documents to the field KNN80DocValuesConsumer knn80DocValuesConsumer = new KNN80DocValuesConsumer(null, state); - RandomVectorDocValuesProducer randomVectorDocValuesProducer = new RandomVectorDocValuesProducer(docsInSegment, dimension); + TestVectorValues.RandomVectorDocValuesProducer randomVectorDocValuesProducer = new TestVectorValues.RandomVectorDocValuesProducer( + docsInSegment, + dimension + ); knn80DocValuesConsumer.addKNNBinaryField(fieldInfoArray[0], randomVectorDocValuesProducer, true, true); // The document should be created in the correct location @@ -277,7 +282,10 @@ public void testAddKNNBinaryField_fromScratch_nmslibLegacy() throws IOException // Add documents to the field KNN80DocValuesConsumer knn80DocValuesConsumer = new KNN80DocValuesConsumer(null, state); - RandomVectorDocValuesProducer randomVectorDocValuesProducer = new RandomVectorDocValuesProducer(docsInSegment, dimension); + TestVectorValues.RandomVectorDocValuesProducer randomVectorDocValuesProducer = new TestVectorValues.RandomVectorDocValuesProducer( + docsInSegment, + dimension + ); knn80DocValuesConsumer.addKNNBinaryField(fieldInfoArray[0], randomVectorDocValuesProducer, true, true); // The document should be created in the correct location @@ -338,7 +346,10 @@ public void testAddKNNBinaryField_fromScratch_faissCurrent() throws IOException // Add documents to the field KNN80DocValuesConsumer knn80DocValuesConsumer = new KNN80DocValuesConsumer(null, state); - RandomVectorDocValuesProducer randomVectorDocValuesProducer = new RandomVectorDocValuesProducer(docsInSegment, dimension); + TestVectorValues.RandomVectorDocValuesProducer randomVectorDocValuesProducer = new TestVectorValues.RandomVectorDocValuesProducer( + docsInSegment, + dimension + ); knn80DocValuesConsumer.addKNNBinaryField(fieldInfoArray[0], randomVectorDocValuesProducer, true, true); // The document should be created in the correct location @@ -401,7 +412,10 @@ public void testAddKNNBinaryField_whenFaissBinary_thenAdded() throws IOException // Add documents to the field KNN80DocValuesConsumer knn80DocValuesConsumer = new KNN80DocValuesConsumer(null, state); - RandomVectorDocValuesProducer randomVectorDocValuesProducer = new RandomVectorDocValuesProducer(docsInSegment, dimension); + TestVectorValues.RandomVectorDocValuesProducer randomVectorDocValuesProducer = new TestVectorValues.RandomVectorDocValuesProducer( + docsInSegment, + dimension + ); knn80DocValuesConsumer.addKNNBinaryField(fieldInfoArray[0], randomVectorDocValuesProducer, true, true); // The document should be created in the correct location @@ -428,7 +442,7 @@ public void testAddKNNBinaryField_fromModel_faiss() throws IOException, Executio int dimension = 16; String modelId = "test-model-id"; - float[][] trainingData = getRandomVectors(200, dimension); + float[][] trainingData = TestVectorValues.getRandomVectors(200, dimension); long trainingPtr = JNIService.transferVectors(0, trainingData); Map parameters = ImmutableMap.of( @@ -497,7 +511,10 @@ public void testAddKNNBinaryField_fromModel_faiss() throws IOException, Executio // Add documents to the field KNN80DocValuesConsumer knn80DocValuesConsumer = new KNN80DocValuesConsumer(null, state); - RandomVectorDocValuesProducer randomVectorDocValuesProducer = new RandomVectorDocValuesProducer(docsInSegment, dimension); + TestVectorValues.RandomVectorDocValuesProducer randomVectorDocValuesProducer = new TestVectorValues.RandomVectorDocValuesProducer( + docsInSegment, + dimension + ); knn80DocValuesConsumer.addKNNBinaryField(fieldInfoArray[0], randomVectorDocValuesProducer, true, true); // The document should be created in the correct location diff --git a/src/test/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngineFieldVectorsWriterTests.java b/src/test/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngineFieldVectorsWriterTests.java index 16d6b20da..29e3531cf 100644 --- a/src/test/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngineFieldVectorsWriterTests.java +++ b/src/test/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngineFieldVectorsWriterTests.java @@ -40,12 +40,12 @@ public void testCreate_ForDifferentInputs_thenSuccess() { byteWriter.addValue(1, new byte[] { 1, 2 }); } + @SuppressWarnings("unchecked") public void testAddValue_ForDifferentInputs_thenSuccess() { final FieldInfo fieldInfo = Mockito.mock(FieldInfo.class); - final NativeEngineFieldVectorsWriter floatWriter = new NativeEngineFieldVectorsWriter<>( - fieldInfo, - InfoStream.getDefault() - ); + Mockito.when(fieldInfo.getVectorEncoding()).thenReturn(VectorEncoding.FLOAT32); + final NativeEngineFieldVectorsWriter floatWriter = (NativeEngineFieldVectorsWriter) NativeEngineFieldVectorsWriter + .create(fieldInfo, InfoStream.getDefault()); final float[] vec1 = new float[] { 1.0f, 2.0f }; final float[] vec2 = new float[] { 2.0f, 2.0f }; floatWriter.addValue(1, vec1); @@ -53,9 +53,11 @@ public void testAddValue_ForDifferentInputs_thenSuccess() { Assert.assertEquals(vec1, floatWriter.getVectors().get(1)); Assert.assertEquals(vec2, floatWriter.getVectors().get(2)); - Mockito.verify(fieldInfo, Mockito.never()).getVectorEncoding(); + Mockito.verify(fieldInfo).getVectorEncoding(); - final NativeEngineFieldVectorsWriter byteWriter = new NativeEngineFieldVectorsWriter<>(fieldInfo, InfoStream.getDefault()); + Mockito.when(fieldInfo.getVectorEncoding()).thenReturn(VectorEncoding.BYTE); + final NativeEngineFieldVectorsWriter byteWriter = (NativeEngineFieldVectorsWriter) NativeEngineFieldVectorsWriter + .create(fieldInfo, InfoStream.getDefault()); final byte[] bvec1 = new byte[] { 1, 2 }; final byte[] bvec2 = new byte[] { 2, 2 }; byteWriter.addValue(1, bvec1); @@ -63,34 +65,36 @@ public void testAddValue_ForDifferentInputs_thenSuccess() { Assert.assertEquals(bvec1, byteWriter.getVectors().get(1)); Assert.assertEquals(bvec2, byteWriter.getVectors().get(2)); - Mockito.verify(fieldInfo, Mockito.never()).getVectorEncoding(); + Mockito.verify(fieldInfo, Mockito.times(2)).getVectorEncoding(); } + @SuppressWarnings("unchecked") public void testCopyValue_whenValidInput_thenException() { final FieldInfo fieldInfo = Mockito.mock(FieldInfo.class); - final NativeEngineFieldVectorsWriter floatWriter = new NativeEngineFieldVectorsWriter<>( - fieldInfo, - InfoStream.getDefault() - ); + Mockito.when(fieldInfo.getVectorEncoding()).thenReturn(VectorEncoding.FLOAT32); + final NativeEngineFieldVectorsWriter floatWriter = (NativeEngineFieldVectorsWriter) NativeEngineFieldVectorsWriter + .create(fieldInfo, InfoStream.getDefault()); expectThrows(UnsupportedOperationException.class, () -> floatWriter.copyValue(new float[3])); - final NativeEngineFieldVectorsWriter byteWriter = new NativeEngineFieldVectorsWriter<>(fieldInfo, InfoStream.getDefault()); + Mockito.when(fieldInfo.getVectorEncoding()).thenReturn(VectorEncoding.BYTE); + final NativeEngineFieldVectorsWriter byteWriter = (NativeEngineFieldVectorsWriter) NativeEngineFieldVectorsWriter + .create(fieldInfo, InfoStream.getDefault()); expectThrows(UnsupportedOperationException.class, () -> byteWriter.copyValue(new byte[3])); } + @SuppressWarnings("unchecked") public void testRamByteUsed_whenValidInput_thenSuccess() { final FieldInfo fieldInfo = Mockito.mock(FieldInfo.class); Mockito.when(fieldInfo.getVectorEncoding()).thenReturn(VectorEncoding.FLOAT32); Mockito.when(fieldInfo.getVectorDimension()).thenReturn(2); - final NativeEngineFieldVectorsWriter floatWriter = new NativeEngineFieldVectorsWriter<>( - fieldInfo, - InfoStream.getDefault() - ); + final NativeEngineFieldVectorsWriter floatWriter = (NativeEngineFieldVectorsWriter) NativeEngineFieldVectorsWriter + .create(fieldInfo, InfoStream.getDefault()); // testing for value > 0 as we don't have a concrete way to find out expected bytes. This can OS dependent too. Assert.assertTrue(floatWriter.ramBytesUsed() > 0); Mockito.when(fieldInfo.getVectorEncoding()).thenReturn(VectorEncoding.BYTE); - final NativeEngineFieldVectorsWriter byteWriter = new NativeEngineFieldVectorsWriter<>(fieldInfo, InfoStream.getDefault()); + final NativeEngineFieldVectorsWriter byteWriter = (NativeEngineFieldVectorsWriter) NativeEngineFieldVectorsWriter + .create(fieldInfo, InfoStream.getDefault()); // testing for value > 0 as we don't have a concrete way to find out expected bytes. This can OS dependent too. Assert.assertTrue(byteWriter.ramBytesUsed() > 0); diff --git a/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestUtil.java b/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestUtil.java index 345de0eee..2afd86a04 100644 --- a/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestUtil.java +++ b/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestUtil.java @@ -10,26 +10,19 @@ import lombok.Builder; import org.apache.lucene.codecs.Codec; import org.apache.lucene.codecs.CodecUtil; -import org.apache.lucene.codecs.DocValuesProducer; -import org.apache.lucene.index.BinaryDocValues; import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.IndexOptions; -import org.apache.lucene.index.NumericDocValues; import org.apache.lucene.index.SegmentInfo; import org.apache.lucene.index.SegmentWriteState; -import org.apache.lucene.index.SortedDocValues; -import org.apache.lucene.index.SortedNumericDocValues; -import org.apache.lucene.index.SortedSetDocValues; -import org.apache.lucene.index.VectorSimilarityFunction; import org.apache.lucene.index.VectorEncoding; +import org.apache.lucene.index.VectorSimilarityFunction; import org.apache.lucene.search.Sort; import org.apache.lucene.store.ChecksumIndexInput; import org.apache.lucene.store.Directory; import org.apache.lucene.store.FSDirectory; import org.apache.lucene.store.FilterDirectory; import org.apache.lucene.store.IOContext; -import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.StringHelper; import org.apache.lucene.util.Version; import java.util.Set; @@ -37,19 +30,15 @@ import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.query.KNNQueryResult; import org.opensearch.knn.index.SpaceType; -import org.opensearch.knn.index.codec.util.KNNVectorSerializer; -import org.opensearch.knn.index.codec.util.KNNVectorSerializerFactory; import org.opensearch.knn.index.engine.KNNEngine; import org.opensearch.knn.jni.JNIService; import java.io.IOException; import java.nio.file.Paths; -import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.Map; -import static com.carrotsearch.randomizedtesting.RandomizedTest.randomFloat; import static org.junit.Assert.assertTrue; import static org.opensearch.knn.common.KNNConstants.INDEX_DESCRIPTION_PARAMETER; import static org.opensearch.knn.common.KNNConstants.SPACE_TYPE; @@ -198,126 +187,6 @@ public FieldInfo build() { } } - public static abstract class VectorDocValues extends BinaryDocValues { - - final int count; - final int dimension; - int current; - KNNVectorSerializer knnVectorSerializer; - - public VectorDocValues(int count, int dimension) { - this.count = count; - this.dimension = dimension; - this.current = -1; - this.knnVectorSerializer = KNNVectorSerializerFactory.getDefaultSerializer(); - } - - @Override - public boolean advanceExact(int target) throws IOException { - return false; - } - - @Override - public int docID() { - if (this.current > this.count) { - return BinaryDocValues.NO_MORE_DOCS; - } - return this.current; - } - - @Override - public int nextDoc() throws IOException { - return advance(current + 1); - } - - @Override - public int advance(int target) throws IOException { - current = target; - if (current >= count) { - current = NO_MORE_DOCS; - } - return current; - } - - @Override - public long cost() { - return 0; - } - } - - public static class ConstantVectorBinaryDocValues extends VectorDocValues { - - private final BytesRef value; - - public ConstantVectorBinaryDocValues(int count, int dimension, float value) { - super(count, dimension); - float[] array = new float[dimension]; - Arrays.fill(array, value); - this.value = new BytesRef(knnVectorSerializer.floatToByteArray(array)); - } - - @Override - public BytesRef binaryValue() throws IOException { - return value; - } - } - - public static class RandomVectorBinaryDocValues extends VectorDocValues { - - public RandomVectorBinaryDocValues(int count, int dimension) { - super(count, dimension); - } - - @Override - public BytesRef binaryValue() throws IOException { - return new BytesRef(knnVectorSerializer.floatToByteArray(getRandomVector(dimension))); - } - } - - public static class RandomVectorDocValuesProducer extends DocValuesProducer { - - final RandomVectorBinaryDocValues randomBinaryDocValues; - - public RandomVectorDocValuesProducer(int count, int dimension) { - this.randomBinaryDocValues = new RandomVectorBinaryDocValues(count, dimension); - } - - @Override - public NumericDocValues getNumeric(FieldInfo field) { - return null; - } - - @Override - public BinaryDocValues getBinary(FieldInfo field) throws IOException { - return randomBinaryDocValues; - } - - @Override - public SortedDocValues getSorted(FieldInfo field) { - return null; - } - - @Override - public SortedNumericDocValues getSortedNumeric(FieldInfo field) { - return null; - } - - @Override - public SortedSetDocValues getSortedSet(FieldInfo field) { - return null; - } - - @Override - public void checkIntegrity() { - - } - - @Override - public void close() throws IOException { - - } - } - public static void assertFileInCorrectLocation(SegmentWriteState state, String expectedFile) throws IOException { assertTrue(Set.of(state.directory.listAll()).contains(expectedFile)); } @@ -378,22 +247,6 @@ public static void assertBinaryIndexLoadableByEngine( JNIService.free(indexPtr, knnEngine); } - public static float[][] getRandomVectors(int count, int dimension) { - float[][] data = new float[count][dimension]; - for (int i = 0; i < count; i++) { - data[i] = getRandomVector(dimension); - } - return data; - } - - public static float[] getRandomVector(int dimension) { - float[] data = new float[dimension]; - for (int i = 0; i < dimension; i++) { - data[i] = randomFloat(); - } - return data; - } - @Builder(builderMethodName = "segmentInfoBuilder") public static SegmentInfo newSegmentInfo(final Directory directory, final String segmentName, int docsInSegment, final Codec codec) { return new SegmentInfo( diff --git a/src/test/java/org/opensearch/knn/index/codec/util/BinaryDocValuesSubTests.java b/src/test/java/org/opensearch/knn/index/codec/util/BinaryDocValuesSubTests.java index a2105af3a..757930dcd 100644 --- a/src/test/java/org/opensearch/knn/index/codec/util/BinaryDocValuesSubTests.java +++ b/src/test/java/org/opensearch/knn/index/codec/util/BinaryDocValuesSubTests.java @@ -7,14 +7,14 @@ import org.apache.lucene.index.BinaryDocValues; import org.apache.lucene.index.MergeState; import org.opensearch.knn.KNNTestCase; -import org.opensearch.knn.index.codec.KNNCodecTestUtil; +import org.opensearch.knn.index.vectorvalues.TestVectorValues; import java.io.IOException; public class BinaryDocValuesSubTests extends KNNTestCase { public void testNextDoc() throws IOException { - BinaryDocValues binaryDocValues = new KNNCodecTestUtil.ConstantVectorBinaryDocValues(10, 128, 2.0f); + BinaryDocValues binaryDocValues = new TestVectorValues.ConstantVectorBinaryDocValues(10, 128, 2.0f); MergeState.DocMap docMap = new MergeState.DocMap() { @Override public int get(int docID) { @@ -28,7 +28,7 @@ public int get(int docID) { } public void testGetValues() { - BinaryDocValues binaryDocValues = new KNNCodecTestUtil.ConstantVectorBinaryDocValues(10, 128, 2.0f); + BinaryDocValues binaryDocValues = new TestVectorValues.ConstantVectorBinaryDocValues(10, 128, 2.0f); MergeState.DocMap docMap = new MergeState.DocMap() { @Override public int get(int docID) { diff --git a/src/test/java/org/opensearch/knn/index/vectorvalues/KNNVectorValuesFactoryTests.java b/src/test/java/org/opensearch/knn/index/vectorvalues/KNNVectorValuesFactoryTests.java new file mode 100644 index 000000000..9827cb03b --- /dev/null +++ b/src/test/java/org/opensearch/knn/index/vectorvalues/KNNVectorValuesFactoryTests.java @@ -0,0 +1,61 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.vectorvalues; + +import org.apache.lucene.index.BinaryDocValues; +import org.apache.lucene.index.DocsWithFieldSet; +import org.junit.Assert; +import org.opensearch.knn.KNNTestCase; +import org.opensearch.knn.index.VectorDataType; + +import java.util.Map; + +public class KNNVectorValuesFactoryTests extends KNNTestCase { + private static final int COUNT = 10; + private static final int DIMENSION = 10; + + public void testGetVectorValuesFromDISI_whenValidInput_thenSuccess() { + final BinaryDocValues binaryDocValues = new TestVectorValues.RandomVectorBinaryDocValues(COUNT, DIMENSION); + final KNNVectorValues floatVectorValues = KNNVectorValuesFactory.getVectorValues(VectorDataType.FLOAT, binaryDocValues); + Assert.assertNotNull(floatVectorValues); + + final KNNVectorValues byteVectorValues = KNNVectorValuesFactory.getVectorValues(VectorDataType.BYTE, binaryDocValues); + Assert.assertNotNull(byteVectorValues); + + final KNNVectorValues binaryVectorValues = KNNVectorValuesFactory.getVectorValues(VectorDataType.BINARY, binaryDocValues); + Assert.assertNotNull(binaryVectorValues); + } + + public void testGetVectorValuesUsingDocWithFieldSet_whenValidInput_thenSuccess() { + final DocsWithFieldSet docsWithFieldSet = new DocsWithFieldSet(); + docsWithFieldSet.add(0); + docsWithFieldSet.add(1); + final Map floatVectorMap = Map.of(0, new float[] { 1, 2 }, 1, new float[] { 2, 3 }); + final KNNVectorValues floatVectorValues = KNNVectorValuesFactory.getVectorValues( + VectorDataType.FLOAT, + docsWithFieldSet, + floatVectorMap + ); + Assert.assertNotNull(floatVectorValues); + + final Map byteVectorMap = Map.of(0, new byte[] { 4, 5 }, 1, new byte[] { 6, 7 }); + + final KNNVectorValues byteVectorValues = KNNVectorValuesFactory.getVectorValues( + VectorDataType.BYTE, + docsWithFieldSet, + byteVectorMap + ); + Assert.assertNotNull(byteVectorValues); + + final KNNVectorValues binaryVectorValues = KNNVectorValuesFactory.getVectorValues( + VectorDataType.BINARY, + docsWithFieldSet, + byteVectorMap + ); + Assert.assertNotNull(binaryVectorValues); + } + +} diff --git a/src/test/java/org/opensearch/knn/index/vectorvalues/KNNVectorValuesTests.java b/src/test/java/org/opensearch/knn/index/vectorvalues/KNNVectorValuesTests.java new file mode 100644 index 000000000..f5a1351ae --- /dev/null +++ b/src/test/java/org/opensearch/knn/index/vectorvalues/KNNVectorValuesTests.java @@ -0,0 +1,145 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.vectorvalues; + +import lombok.SneakyThrows; +import org.apache.lucene.index.DocsWithFieldSet; +import org.apache.lucene.search.DocIdSetIterator; +import org.junit.Assert; +import org.opensearch.knn.KNNTestCase; +import org.opensearch.knn.index.VectorDataType; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +public class KNNVectorValuesTests extends KNNTestCase { + + @SneakyThrows + public void testFloatVectorValues_whenValidInput_thenSuccess() { + final List floatArray = List.of(new float[] { 1, 2 }, new float[] { 2, 3 }); + final int dimension = floatArray.get(0).length; + final TestVectorValues.PreDefinedFloatVectorValues randomVectorValues = new TestVectorValues.PreDefinedFloatVectorValues( + floatArray + ); + final KNNVectorValues knnVectorValues = KNNVectorValuesFactory.getVectorValues(VectorDataType.FLOAT, randomVectorValues); + new CompareVectorValues().validateVectorValues(knnVectorValues, floatArray, dimension, true); + + final DocsWithFieldSet docsWithFieldSet = getDocIdSetIterator(floatArray.size()); + + final Map vectorsMap = Map.of(0, floatArray.get(0), 1, floatArray.get(1)); + final KNNVectorValues knnVectorValuesForFieldWriter = KNNVectorValuesFactory.getVectorValues( + VectorDataType.FLOAT, + docsWithFieldSet, + vectorsMap + ); + new CompareVectorValues().validateVectorValues(knnVectorValuesForFieldWriter, floatArray, dimension, false); + + final TestVectorValues.PredefinedFloatVectorBinaryDocValues preDefinedFloatVectorValues = + new TestVectorValues.PredefinedFloatVectorBinaryDocValues(floatArray); + final KNNVectorValues knnFloatVectorValuesBinaryDocValues = KNNVectorValuesFactory.getVectorValues( + VectorDataType.FLOAT, + preDefinedFloatVectorValues + ); + new CompareVectorValues().validateVectorValues(knnFloatVectorValuesBinaryDocValues, floatArray, dimension, false); + } + + @SneakyThrows + public void testByteVectorValues_whenValidInput_thenSuccess() { + final List byteArray = List.of(new byte[] { 4, 5 }, new byte[] { 6, 7 }); + final int dimension = byteArray.get(0).length; + final TestVectorValues.PreDefinedByteVectorValues randomVectorValues = new TestVectorValues.PreDefinedByteVectorValues(byteArray); + final KNNVectorValues knnVectorValues = KNNVectorValuesFactory.getVectorValues(VectorDataType.BYTE, randomVectorValues); + new CompareVectorValues().validateVectorValues(knnVectorValues, byteArray, dimension, true); + + final DocsWithFieldSet docsWithFieldSet = getDocIdSetIterator(byteArray.size()); + final Map vectorsMap = Map.of(0, byteArray.get(0), 1, byteArray.get(1)); + final KNNVectorValues knnVectorValuesForFieldWriter = KNNVectorValuesFactory.getVectorValues( + VectorDataType.BYTE, + docsWithFieldSet, + vectorsMap + ); + new CompareVectorValues().validateVectorValues(knnVectorValuesForFieldWriter, byteArray, dimension, false); + + final TestVectorValues.PredefinedByteVectorBinaryDocValues preDefinedByteVectorValues = + new TestVectorValues.PredefinedByteVectorBinaryDocValues(byteArray); + final KNNVectorValues knnBinaryVectorValuesBinaryDocValues = KNNVectorValuesFactory.getVectorValues( + VectorDataType.BYTE, + preDefinedByteVectorValues + ); + new CompareVectorValues().validateVectorValues(knnBinaryVectorValuesBinaryDocValues, byteArray, dimension, false); + } + + @SneakyThrows + public void testBinaryVectorValues_whenValidInput_thenSuccess() { + final List byteArray = List.of(new byte[] { 1, 5, 8 }, new byte[] { 6, 7, 9 }); + int dimension = byteArray.get(0).length * 8; + final TestVectorValues.PreDefinedBinaryVectorValues randomVectorValues = new TestVectorValues.PreDefinedBinaryVectorValues( + byteArray + ); + final KNNVectorValues knnVectorValues = KNNVectorValuesFactory.getVectorValues(VectorDataType.BINARY, randomVectorValues); + new CompareVectorValues().validateVectorValues(knnVectorValues, byteArray, dimension, true); + + final DocsWithFieldSet docsWithFieldSet = getDocIdSetIterator(byteArray.size()); + final Map vectorsMap = Map.of(0, byteArray.get(0), 1, byteArray.get(1)); + final KNNBinaryVectorValues knnVectorValuesForFieldWriter = (KNNBinaryVectorValues) KNNVectorValuesFactory.getVectorValues( + VectorDataType.BINARY, + docsWithFieldSet, + vectorsMap + ); + new CompareVectorValues().validateVectorValues(knnVectorValuesForFieldWriter, byteArray, dimension, false); + + final TestVectorValues.PredefinedByteVectorBinaryDocValues preDefinedByteVectorValues = + new TestVectorValues.PredefinedByteVectorBinaryDocValues(byteArray); + final KNNVectorValues knnBinaryVectorValuesBinaryDocValues = KNNVectorValuesFactory.getVectorValues( + VectorDataType.BINARY, + preDefinedByteVectorValues + ); + new CompareVectorValues().validateVectorValues(knnBinaryVectorValuesBinaryDocValues, byteArray, dimension, false); + } + + public void testDocIdsIteratorValues_whenInvalidDisi_thenThrowException() { + Assert.assertThrows( + IllegalArgumentException.class, + () -> new KNNVectorValuesIterator.DocIdsIteratorValues(new TestVectorValues.NotBinaryDocValues()) + ); + } + + private DocsWithFieldSet getDocIdSetIterator(int numberOfDocIds) { + final DocsWithFieldSet docsWithFieldSet = new DocsWithFieldSet(); + for (int i = 0; i < numberOfDocIds; i++) { + docsWithFieldSet.add(i); + } + return docsWithFieldSet; + } + + private class CompareVectorValues { + void validateVectorValues(KNNVectorValues vectorValues, List vectors, int dimension, boolean validateAddress) + throws IOException { + Assert.assertEquals(vectorValues.totalLiveDocs(), vectors.size()); + int docId, i = 0; + T oldActual = null; + int oldDocId = -1; + final KNNVectorValuesIterator iterator = vectorValues.vectorValuesIterator; + for (docId = iterator.nextDoc(); docId != DocIdSetIterator.NO_MORE_DOCS && i < vectors.size(); docId = iterator.nextDoc()) { + T actual = vectorValues.getVector(); + T expected = vectors.get(i); + Assert.assertNotEquals(oldDocId, docId); + Assert.assertEquals(dimension, vectorValues.dimension()); + // this will check if reference is correct for the vectors. This is mainly required because for + // VectorValues of Lucene when reading vectors put the vector at same reference + if (oldActual != null && validateAddress) { + Assert.assertSame(actual, oldActual); + } + oldActual = actual; + // this will do the deep equals + Assert.assertArrayEquals(new Object[] { actual }, new Object[] { expected }); + i++; + } + } + } + +} diff --git a/src/test/java/org/opensearch/knn/index/vectorvalues/TestVectorValues.java b/src/test/java/org/opensearch/knn/index/vectorvalues/TestVectorValues.java new file mode 100644 index 000000000..3bf79b004 --- /dev/null +++ b/src/test/java/org/opensearch/knn/index/vectorvalues/TestVectorValues.java @@ -0,0 +1,363 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.knn.index.vectorvalues; + +import org.apache.lucene.codecs.DocValuesProducer; +import org.apache.lucene.index.BinaryDocValues; +import org.apache.lucene.index.ByteVectorValues; +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.FloatVectorValues; +import org.apache.lucene.index.NumericDocValues; +import org.apache.lucene.index.SortedDocValues; +import org.apache.lucene.index.SortedNumericDocValues; +import org.apache.lucene.index.SortedSetDocValues; +import org.apache.lucene.search.VectorScorer; +import org.apache.lucene.util.BytesRef; +import org.opensearch.knn.index.codec.util.KNNVectorSerializer; +import org.opensearch.knn.index.codec.util.KNNVectorSerializerFactory; + +import java.io.IOException; +import java.util.Arrays; +import java.util.List; + +import static com.carrotsearch.randomizedtesting.RandomizedTest.randomFloat; + +public class TestVectorValues { + + public static class RandomVectorBinaryDocValues extends VectorDocValues { + + public RandomVectorBinaryDocValues(int count, int dimension) { + super(count, dimension); + } + + @Override + public BytesRef binaryValue() throws IOException { + return new BytesRef(knnVectorSerializer.floatToByteArray(getRandomVector(dimension))); + } + } + + public static class ConstantVectorBinaryDocValues extends VectorDocValues { + + private final BytesRef value; + + public ConstantVectorBinaryDocValues(int count, int dimension, float value) { + super(count, dimension); + float[] array = new float[dimension]; + Arrays.fill(array, value); + this.value = new BytesRef(knnVectorSerializer.floatToByteArray(array)); + } + + @Override + public BytesRef binaryValue() throws IOException { + return value; + } + } + + public static class PredefinedFloatVectorBinaryDocValues extends VectorDocValues { + private final List vectors; + + public PredefinedFloatVectorBinaryDocValues(final List vectors) { + super(vectors.size(), vectors.get(0).length); + this.vectors = vectors; + } + + @Override + public BytesRef binaryValue() throws IOException { + return new BytesRef(knnVectorSerializer.floatToByteArray(vectors.get(docID()))); + } + } + + public static class PredefinedByteVectorBinaryDocValues extends VectorDocValues { + private final List vectors; + + public PredefinedByteVectorBinaryDocValues(final List vectors) { + super(vectors.size(), vectors.get(0).length); + this.vectors = vectors; + } + + @Override + public BytesRef binaryValue() throws IOException { + return new BytesRef(vectors.get(docID())); + } + } + + public static class RandomVectorDocValuesProducer extends DocValuesProducer { + + final RandomVectorBinaryDocValues randomBinaryDocValues; + + public RandomVectorDocValuesProducer(int count, int dimension) { + this.randomBinaryDocValues = new RandomVectorBinaryDocValues(count, dimension); + } + + @Override + public NumericDocValues getNumeric(FieldInfo field) { + return null; + } + + @Override + public BinaryDocValues getBinary(FieldInfo field) throws IOException { + return randomBinaryDocValues; + } + + @Override + public SortedDocValues getSorted(FieldInfo field) { + return null; + } + + @Override + public SortedNumericDocValues getSortedNumeric(FieldInfo field) { + return null; + } + + @Override + public SortedSetDocValues getSortedSet(FieldInfo field) { + return null; + } + + @Override + public void checkIntegrity() { + + } + + @Override + public void close() throws IOException { + + } + } + + static abstract class VectorDocValues extends BinaryDocValues { + + final int count; + final int dimension; + int current; + KNNVectorSerializer knnVectorSerializer; + + public VectorDocValues(int count, int dimension) { + this.count = count; + this.dimension = dimension; + this.current = -1; + this.knnVectorSerializer = KNNVectorSerializerFactory.getDefaultSerializer(); + } + + @Override + public boolean advanceExact(int target) throws IOException { + return false; + } + + @Override + public int docID() { + if (this.current > this.count) { + return BinaryDocValues.NO_MORE_DOCS; + } + return this.current; + } + + @Override + public int nextDoc() throws IOException { + return advance(current + 1); + } + + @Override + public int advance(int target) throws IOException { + current = target; + if (current >= count) { + current = NO_MORE_DOCS; + } + return current; + } + + @Override + public long cost() { + return count; + } + } + + public static class PreDefinedFloatVectorValues extends FloatVectorValues { + final int count; + final int dimension; + final List vectors; + int current; + float[] vector; + + public PreDefinedFloatVectorValues(final List vectors) { + super(); + this.count = vectors.size(); + this.dimension = vectors.get(0).length; + this.vectors = vectors; + this.current = -1; + vector = new float[dimension]; + } + + @Override + public int dimension() { + return dimension; + } + + @Override + public int size() { + return count; + } + + @Override + public float[] vectorValue() throws IOException { + // since in FloatVectorValues the reference to returned vector doesn't change. This code ensure that we + // are replicating the behavior so that if someone uses this RandomFloatVectorValues they get an + // experience similar to what we get in prod. + System.arraycopy(vectors.get(docID()), 0, vector, 0, dimension); + return vector; + } + + @Override + public VectorScorer scorer(float[] query) throws IOException { + throw new UnsupportedOperationException("scorer not supported with PreDefinedFloatVectorValues"); + } + + @Override + public int docID() { + if (this.current > this.count) { + return FloatVectorValues.NO_MORE_DOCS; + } + return this.current; + } + + @Override + public int nextDoc() throws IOException { + return advance(current + 1); + } + + @Override + public int advance(int target) throws IOException { + current = target; + if (current >= count) { + current = NO_MORE_DOCS; + } + return current; + } + } + + public static class PreDefinedByteVectorValues extends ByteVectorValues { + private final int count; + private final int dimension; + private final List vectors; + private int current; + private final byte[] vector; + + public PreDefinedByteVectorValues(final List vectors) { + super(); + this.count = vectors.size(); + this.dimension = vectors.get(0).length; + this.vectors = vectors; + this.current = -1; + vector = new byte[dimension]; + } + + @Override + public int dimension() { + return dimension; + } + + @Override + public int size() { + return count; + } + + @Override + public byte[] vectorValue() throws IOException { + // since in FloatVectorValues the reference to returned vector doesn't change. This code ensure that we + // are replicating the behavior so that if someone uses this RandomFloatVectorValues they get an + // experience similar to what we get in prod. + System.arraycopy(vectors.get(docID()), 0, vector, 0, dimension); + return vector; + } + + @Override + public VectorScorer scorer(byte[] query) throws IOException { + throw new UnsupportedOperationException("scorer not supported with PreDefinedFloatVectorValues"); + } + + @Override + public int docID() { + if (this.current > this.count) { + return FloatVectorValues.NO_MORE_DOCS; + } + return this.current; + } + + @Override + public int nextDoc() throws IOException { + return advance(current + 1); + } + + @Override + public int advance(int target) throws IOException { + current = target; + if (current >= count) { + current = NO_MORE_DOCS; + } + return current; + } + } + + public static class PreDefinedBinaryVectorValues extends PreDefinedByteVectorValues { + + public PreDefinedBinaryVectorValues(List vectors) { + super(vectors); + } + + @Override + public int dimension() { + return super.dimension() * Byte.SIZE; + } + } + + public static class NotBinaryDocValues extends NumericDocValues { + + @Override + public long longValue() throws IOException { + return 0; + } + + @Override + public boolean advanceExact(int target) throws IOException { + return false; + } + + @Override + public int docID() { + return 0; + } + + @Override + public int nextDoc() throws IOException { + return 0; + } + + @Override + public int advance(int target) throws IOException { + return 0; + } + + @Override + public long cost() { + return 0; + } + } + + public static float[][] getRandomVectors(int count, int dimension) { + float[][] data = new float[count][dimension]; + for (int i = 0; i < count; i++) { + data[i] = getRandomVector(dimension); + } + return data; + } + + public static float[] getRandomVector(int dimension) { + float[] data = new float[dimension]; + for (int i = 0; i < dimension; i++) { + data[i] = randomFloat(); + } + return data; + } +} diff --git a/src/test/java/org/opensearch/knn/index/vectorvalues/VectorValueExtractorStrategyTests.java b/src/test/java/org/opensearch/knn/index/vectorvalues/VectorValueExtractorStrategyTests.java new file mode 100644 index 000000000..68a49a54c --- /dev/null +++ b/src/test/java/org/opensearch/knn/index/vectorvalues/VectorValueExtractorStrategyTests.java @@ -0,0 +1,29 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.vectorvalues; + +import lombok.SneakyThrows; +import org.junit.Assert; +import org.mockito.Mockito; +import org.opensearch.knn.KNNTestCase; +import org.opensearch.knn.index.VectorDataType; + +/** + * To avoid unit test duplication, tests for exception is added here. For non exception cases tests are present in + * {@link KNNVectorValuesTests} + */ +public class VectorValueExtractorStrategyTests extends KNNTestCase { + + @SneakyThrows + public void testExtractWithDISI_whenInvalidIterator_thenException() { + final VectorValueExtractorStrategy disiStrategy = new VectorValueExtractorStrategy.DISIVectorExtractor(); + final KNNVectorValuesIterator vectorValuesIterator = Mockito.mock(KNNVectorValuesIterator.DocIdsIteratorValues.class); + Mockito.when(vectorValuesIterator.getDocIdSetIterator()).thenReturn(new TestVectorValues.NotBinaryDocValues()); + Assert.assertThrows(IllegalArgumentException.class, () -> disiStrategy.extract(VectorDataType.FLOAT, vectorValuesIterator)); + Assert.assertThrows(IllegalArgumentException.class, () -> disiStrategy.extract(VectorDataType.BINARY, vectorValuesIterator)); + Assert.assertThrows(IllegalArgumentException.class, () -> disiStrategy.extract(VectorDataType.BYTE, vectorValuesIterator)); + } +} From 4e7c56d1f8a4a3a0c68c94e6380d4320ec0fc7b1 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Thu, 1 Aug 2024 18:32:57 -0700 Subject: [PATCH 314/416] Refactor method structure and definitions (#1923) * Remove getMethod from KNNLibrary interface Removes the getMethod from KNNLibrary interface. Users should not directly retrieve methods from the engines as this creates additional complexity. Instead, they should get everything they need about a method from the KNNLibrary itself. This change will allow us to better maintain the different methods. * Move EngineSpecificMethodContext into KNNMethod Moving the EngineSpecificMethodContext into the KNNMethod class. EngineSpecificMethodContext is mainly used during search to provide parameter validation/configure dynamic updates. Moving into KNNMethod will encapsulate the structure of the method into one class. * Change KNNMethod to interface Changes KNNMethod to interface and creates methods per engine/algo combination. For one, this will make code much more maintainable as we wont have to deal with big maps and builders. For the other, we can implement more complex logic between engines. Signed-off-by: John Mazanec * Add simple encoder interface Adds simple encoder interface to clean up definitions of the encoders. --------- Signed-off-by: John Mazanec (cherry picked from commit adaf150d08416cdff3670317bfb82ecb1df369bc) --- CHANGELOG.md | 1 + .../org/opensearch/knn/bwc/FaissSQIT.java | 8 +- .../knn/index/engine/AbstractKNNLibrary.java | 43 +-- .../knn/index/engine/AbstractKNNMethod.java | 119 ++++++ .../opensearch/knn/index/engine/Encoder.java | 27 ++ .../knn/index/engine/JVMLibrary.java | 4 +- .../knn/index/engine/KNNEngine.java | 5 - .../knn/index/engine/KNNLibrary.java | 9 - .../knn/index/engine/KNNMethod.java | 144 +------ .../knn/index/engine/NativeLibrary.java | 5 +- .../knn/index/engine/faiss/Faiss.java | 352 +----------------- .../index/engine/faiss/FaissFlatEncoder.java | 32 ++ .../index/engine/faiss/FaissHNSWMethod.java | 93 +++++ .../engine/faiss/FaissHNSWPQEncoder.java | 65 ++++ .../index/engine/faiss/FaissIVFMethod.java | 123 ++++++ .../index/engine/faiss/FaissIVFPQEncoder.java | 84 +++++ .../index/engine/faiss/FaissSQEncoder.java | 41 ++ .../engine/faiss/MethodAsMapBuilder.java | 95 +++++ .../knn/index/engine/lucene/Lucene.java | 68 +--- .../index/engine/lucene/LuceneHNSWMethod.java | 80 ++++ .../index/engine/lucene/LuceneSQEncoder.java | 46 +++ .../knn/index/engine/nmslib/Nmslib.java | 30 +- .../index/engine/nmslib/NmslibHNSWMethod.java | 61 +++ .../java/org/opensearch/knn/KNNTestCase.java | 3 + .../knn/index/FaissHNSWFlatE2EIT.java | 5 +- .../org/opensearch/knn/index/FaissIT.java | 39 +- .../opensearch/knn/index/LuceneEngineIT.java | 10 +- .../org/opensearch/knn/index/NmslibIT.java | 8 +- .../opensearch/knn/index/OpenSearchIT.java | 8 +- .../index/engine/AbstractKNNLibraryTests.java | 123 +++--- ...Tests.java => AbstractKNNMethodTests.java} | 65 ++-- .../knn/index/engine/NativeLibraryTests.java | 2 +- .../knn/index/engine/faiss/FaissTests.java | 2 +- .../knn/index/engine/lucene/LuceneTests.java | 15 +- .../opensearch/knn/jni/JNIServiceTests.java | 5 +- .../LibraryInitializedSupplierTests.java | 6 - 36 files changed, 1031 insertions(+), 795 deletions(-) create mode 100644 src/main/java/org/opensearch/knn/index/engine/AbstractKNNMethod.java create mode 100644 src/main/java/org/opensearch/knn/index/engine/Encoder.java create mode 100644 src/main/java/org/opensearch/knn/index/engine/faiss/FaissFlatEncoder.java create mode 100644 src/main/java/org/opensearch/knn/index/engine/faiss/FaissHNSWMethod.java create mode 100644 src/main/java/org/opensearch/knn/index/engine/faiss/FaissHNSWPQEncoder.java create mode 100644 src/main/java/org/opensearch/knn/index/engine/faiss/FaissIVFMethod.java create mode 100644 src/main/java/org/opensearch/knn/index/engine/faiss/FaissIVFPQEncoder.java create mode 100644 src/main/java/org/opensearch/knn/index/engine/faiss/FaissSQEncoder.java create mode 100644 src/main/java/org/opensearch/knn/index/engine/faiss/MethodAsMapBuilder.java create mode 100644 src/main/java/org/opensearch/knn/index/engine/lucene/LuceneHNSWMethod.java create mode 100644 src/main/java/org/opensearch/knn/index/engine/lucene/LuceneSQEncoder.java create mode 100644 src/main/java/org/opensearch/knn/index/engine/nmslib/NmslibHNSWMethod.java rename src/test/java/org/opensearch/knn/index/engine/{KNNMethodTests.java => AbstractKNNMethodTests.java} (76%) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0623e6f98..575d5b6bd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,3 +24,4 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), * Introduce KNNVectorValues interface to iterate on different types of Vector values during indexing and search [#1897](https://github.com/opensearch-project/k-NN/pull/1897) * Clean up parsing for query [#1824](https://github.com/opensearch-project/k-NN/pull/1824) * Refactor engine package structure [#1913](https://github.com/opensearch-project/k-NN/pull/1913) +* Refactor method structure and definitions [#1920](https://github.com/opensearch-project/k-NN/pull/1920) diff --git a/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/FaissSQIT.java b/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/FaissSQIT.java index cc4cbc043..c9a74418f 100644 --- a/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/FaissSQIT.java +++ b/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/FaissSQIT.java @@ -19,7 +19,6 @@ import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.knn.KNNResult; import org.opensearch.knn.common.KNNConstants; -import org.opensearch.knn.index.engine.KNNMethod; import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.query.KNNQueryBuilder; import org.opensearch.knn.index.engine.KNNEngine; @@ -55,7 +54,6 @@ public class FaissSQIT extends AbstractRestartUpgradeTestCase { public void testHNSWSQFP16_onUpgradeWhenIndexedAndQueried_thenSucceed() throws Exception { if (!isRunningAgainstOldCluster()) { - KNNMethod hnswMethod = KNNEngine.FAISS.getMethod(KNNConstants.METHOD_HNSW); SpaceType[] spaceTypes = { SpaceType.L2, SpaceType.INNER_PRODUCT }; Random random = new Random(); SpaceType spaceType = spaceTypes[random.nextInt(spaceTypes.length)]; @@ -96,7 +94,7 @@ public void testHNSWSQFP16_onUpgradeWhenIndexedAndQueried_thenSucceed() throws E .field("type", "knn_vector") .field("dimension", DIMENSION) .startObject(KNNConstants.KNN_METHOD) - .field(KNNConstants.NAME, hnswMethod.getMethodComponent().getName()) + .field(KNNConstants.NAME, KNNConstants.METHOD_HNSW) .field(KNNConstants.METHOD_PARAMETER_SPACE_TYPE, spaceType.getValue()) .field(KNNConstants.KNN_ENGINE, KNNEngine.FAISS.getName()) .startObject(KNNConstants.PARAMETERS) @@ -132,8 +130,6 @@ public void testHNSWSQFP16_onUpgradeWhenIndexedAndQueried_thenSucceed() throws E public void testHNSWSQFP16_onUpgradeWhenClipToFp16isTrueAndIndexedWithOutOfFP16Range_thenSucceed() throws Exception { if (!isRunningAgainstOldCluster()) { - KNNMethod hnswMethod = KNNEngine.FAISS.getMethod(KNNConstants.METHOD_HNSW); - new Random(); List mValues = ImmutableList.of(16, 32, 64, 128); List efConstructionValues = ImmutableList.of(16, 32, 64, 128); @@ -174,7 +170,7 @@ public void testHNSWSQFP16_onUpgradeWhenClipToFp16isTrueAndIndexedWithOutOfFP16R .field("type", "knn_vector") .field("dimension", dimension) .startObject(KNNConstants.KNN_METHOD) - .field(KNNConstants.NAME, hnswMethod.getMethodComponent().getName()) + .field(KNNConstants.NAME, KNNConstants.METHOD_HNSW) .field(KNNConstants.METHOD_PARAMETER_SPACE_TYPE, SpaceType.L2.getValue()) .field(KNNConstants.KNN_ENGINE, KNNEngine.FAISS.getName()) .startObject(PARAMETERS) diff --git a/src/main/java/org/opensearch/knn/index/engine/AbstractKNNLibrary.java b/src/main/java/org/opensearch/knn/index/engine/AbstractKNNLibrary.java index 05cfb09af..9d83f42a8 100644 --- a/src/main/java/org/opensearch/knn/index/engine/AbstractKNNLibrary.java +++ b/src/main/java/org/opensearch/knn/index/engine/AbstractKNNLibrary.java @@ -20,56 +20,49 @@ public abstract class AbstractKNNLibrary implements KNNLibrary { protected final Map methods; - protected final Map engineMethods; @Getter protected final String version; - @Override - public KNNMethod getMethod(String methodName) { - KNNMethod method = methods.get(methodName); - if (method == null) { - throw new IllegalArgumentException(String.format("Invalid method name: %s", methodName)); - } - return method; - } - @Override public EngineSpecificMethodContext getMethodContext(String methodName) { - EngineSpecificMethodContext method = engineMethods.get(methodName); - if (method == null) { - throw new IllegalArgumentException(String.format("Invalid method name: %s", methodName)); - } - return method; + validateMethodExists(methodName); + KNNMethod method = methods.get(methodName); + return method.getEngineSpecificMethodContext(); } @Override public ValidationException validateMethod(KNNMethodContext knnMethodContext) { String methodName = knnMethodContext.getMethodComponentContext().getName(); - return getMethod(methodName).validate(knnMethodContext); + validateMethodExists(methodName); + return methods.get(methodName).validate(knnMethodContext); } @Override public ValidationException validateMethodWithData(KNNMethodContext knnMethodContext, VectorSpaceInfo vectorSpaceInfo) { String methodName = knnMethodContext.getMethodComponentContext().getName(); - return getMethod(methodName).validateWithData(knnMethodContext, vectorSpaceInfo); + validateMethodExists(methodName); + return methods.get(methodName).validateWithData(knnMethodContext, vectorSpaceInfo); } @Override public boolean isTrainingRequired(KNNMethodContext knnMethodContext) { String methodName = knnMethodContext.getMethodComponentContext().getName(); - return getMethod(methodName).isTrainingRequired(knnMethodContext); + validateMethodExists(methodName); + return methods.get(methodName).isTrainingRequired(knnMethodContext); } @Override public Map getMethodAsMap(KNNMethodContext knnMethodContext) { - KNNMethod knnMethod = methods.get(knnMethodContext.getMethodComponentContext().getName()); + String method = knnMethodContext.getMethodComponentContext().getName(); + validateMethodExists(method); + KNNMethod knnMethod = methods.get(method); + return knnMethod.getAsMap(knnMethodContext); + } - if (knnMethod == null) { - throw new IllegalArgumentException( - String.format("Invalid method name: %s", knnMethodContext.getMethodComponentContext().getName()) - ); + private void validateMethodExists(String methodName) { + KNNMethod method = methods.get(methodName); + if (method == null) { + throw new IllegalArgumentException(String.format("Invalid method name: %s", methodName)); } - - return knnMethod.getAsMap(knnMethodContext); } } diff --git a/src/main/java/org/opensearch/knn/index/engine/AbstractKNNMethod.java b/src/main/java/org/opensearch/knn/index/engine/AbstractKNNMethod.java new file mode 100644 index 000000000..cbed5fe40 --- /dev/null +++ b/src/main/java/org/opensearch/knn/index/engine/AbstractKNNMethod.java @@ -0,0 +1,119 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.engine; + +import lombok.AllArgsConstructor; +import org.opensearch.common.ValidationException; +import org.opensearch.knn.common.KNNConstants; +import org.opensearch.knn.index.SpaceType; +import org.opensearch.knn.training.VectorSpaceInfo; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; + +/** + * Abstract class for KNN methods. This class provides the common functionality for all KNN methods. + * It defines the common attributes and methods that all KNN methods should implement. + */ +@AllArgsConstructor +public abstract class AbstractKNNMethod implements KNNMethod { + + protected final MethodComponent methodComponent; + protected final Set spaces; + protected final EngineSpecificMethodContext engineSpecificMethodContext; + + @Override + public boolean isSpaceTypeSupported(SpaceType space) { + return spaces.contains(space); + } + + @Override + public ValidationException validate(KNNMethodContext knnMethodContext) { + List errorMessages = new ArrayList<>(); + if (!isSpaceTypeSupported(knnMethodContext.getSpaceType())) { + errorMessages.add( + String.format( + Locale.ROOT, + "\"%s\" with \"%s\" configuration does not support space type: " + "\"%s\".", + this.methodComponent.getName(), + knnMethodContext.getKnnEngine().getName().toLowerCase(Locale.ROOT), + knnMethodContext.getSpaceType().getValue() + ) + ); + } + + ValidationException methodValidation = methodComponent.validate(knnMethodContext.getMethodComponentContext()); + if (methodValidation != null) { + errorMessages.addAll(methodValidation.validationErrors()); + } + + if (errorMessages.isEmpty()) { + return null; + } + + ValidationException validationException = new ValidationException(); + validationException.addValidationErrors(errorMessages); + return validationException; + } + + @Override + public ValidationException validateWithData(KNNMethodContext knnMethodContext, VectorSpaceInfo vectorSpaceInfo) { + List errorMessages = new ArrayList<>(); + if (!isSpaceTypeSupported(knnMethodContext.getSpaceType())) { + errorMessages.add( + String.format( + Locale.ROOT, + "\"%s\" with \"%s\" configuration does not support space type: " + "\"%s\".", + this.methodComponent.getName(), + knnMethodContext.getKnnEngine().getName().toLowerCase(Locale.ROOT), + knnMethodContext.getSpaceType().getValue() + ) + ); + } + + ValidationException methodValidation = methodComponent.validateWithData( + knnMethodContext.getMethodComponentContext(), + vectorSpaceInfo + ); + if (methodValidation != null) { + errorMessages.addAll(methodValidation.validationErrors()); + } + + if (errorMessages.isEmpty()) { + return null; + } + + ValidationException validationException = new ValidationException(); + validationException.addValidationErrors(errorMessages); + return validationException; + } + + @Override + public boolean isTrainingRequired(KNNMethodContext knnMethodContext) { + return methodComponent.isTrainingRequired(knnMethodContext.getMethodComponentContext()); + } + + @Override + public int estimateOverheadInKB(KNNMethodContext knnMethodContext, int dimension) { + return methodComponent.estimateOverheadInKB(knnMethodContext.getMethodComponentContext(), dimension); + } + + @Override + public Map getAsMap(KNNMethodContext knnMethodContext) { + Map parameterMap = new HashMap<>(methodComponent.getAsMap(knnMethodContext.getMethodComponentContext())); + parameterMap.put(KNNConstants.SPACE_TYPE, knnMethodContext.getSpaceType().getValue()); + return parameterMap; + } + + @Override + public EngineSpecificMethodContext getEngineSpecificMethodContext() { + return engineSpecificMethodContext; + } +} diff --git a/src/main/java/org/opensearch/knn/index/engine/Encoder.java b/src/main/java/org/opensearch/knn/index/engine/Encoder.java new file mode 100644 index 000000000..7e22145eb --- /dev/null +++ b/src/main/java/org/opensearch/knn/index/engine/Encoder.java @@ -0,0 +1,27 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.engine; + +/** + * Interface representing an encoder. An encoder generally refers to a vector quantizer. + */ +public interface Encoder { + /** + * The name of the encoder does not have to be unique. Howevwer, when using within a method, there cannot be + * 2 encoders with the same name. + * + * @return Name of the encoder + */ + default String getName() { + return getMethodComponent().getName(); + } + + /** + * + * @return Method component associated with the encoder + */ + MethodComponent getMethodComponent(); +} diff --git a/src/main/java/org/opensearch/knn/index/engine/JVMLibrary.java b/src/main/java/org/opensearch/knn/index/engine/JVMLibrary.java index 781fa9c7d..762966567 100644 --- a/src/main/java/org/opensearch/knn/index/engine/JVMLibrary.java +++ b/src/main/java/org/opensearch/knn/index/engine/JVMLibrary.java @@ -20,8 +20,8 @@ public abstract class JVMLibrary extends AbstractKNNLibrary { * @param methods Map of k-NN methods that the library supports * @param version String representing version of library */ - public JVMLibrary(Map methods, Map engineMethodMetadataMap, String version) { - super(methods, engineMethodMetadataMap, version); + public JVMLibrary(Map methods, String version) { + super(methods, version); } @Override diff --git a/src/main/java/org/opensearch/knn/index/engine/KNNEngine.java b/src/main/java/org/opensearch/knn/index/engine/KNNEngine.java index a552b2b57..f2f2bab35 100644 --- a/src/main/java/org/opensearch/knn/index/engine/KNNEngine.java +++ b/src/main/java/org/opensearch/knn/index/engine/KNNEngine.java @@ -145,11 +145,6 @@ public String getCompoundExtension() { return knnLibrary.getCompoundExtension(); } - @Override - public KNNMethod getMethod(String methodName) { - return knnLibrary.getMethod(methodName); - } - @Override public EngineSpecificMethodContext getMethodContext(String methodName) { return knnLibrary.getMethodContext(methodName); diff --git a/src/main/java/org/opensearch/knn/index/engine/KNNLibrary.java b/src/main/java/org/opensearch/knn/index/engine/KNNLibrary.java index 3989c4609..c47f39e03 100644 --- a/src/main/java/org/opensearch/knn/index/engine/KNNLibrary.java +++ b/src/main/java/org/opensearch/knn/index/engine/KNNLibrary.java @@ -41,15 +41,6 @@ public interface KNNLibrary { */ String getCompoundExtension(); - /** - * Gets a particular KNN method that the library supports. This should throw an exception if the method is not - * supported by the library. - * - * @param methodName name of the method to be looked up - * @return KNNMethod in the library corresponding to the method name - */ - KNNMethod getMethod(String methodName); - /** * Gets metadata related to methods supported by the library * @param methodName diff --git a/src/main/java/org/opensearch/knn/index/engine/KNNMethod.java b/src/main/java/org/opensearch/knn/index/engine/KNNMethod.java index c3dad21b2..ea556e8bf 100644 --- a/src/main/java/org/opensearch/knn/index/engine/KNNMethod.java +++ b/src/main/java/org/opensearch/knn/index/engine/KNNMethod.java @@ -5,32 +5,18 @@ package org.opensearch.knn.index.engine; -import lombok.AllArgsConstructor; -import lombok.Getter; import org.opensearch.common.ValidationException; -import org.opensearch.knn.common.KNNConstants; import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.training.VectorSpaceInfo; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Locale; import java.util.Map; -import java.util.Set; /** - * KNNMethod is used to define the structure of a method supported by a particular k-NN library. It is used to validate - * the KNNMethodContext passed in by the user. It is also used to provide superficial string translations. + * KNNMethod defines the structure of a method supported by a particular k-NN library. It is used to validate + * the KNNMethodContext passed in by the user, where the KNNMethodContext provides the configuration that the user may + * want. Then, it provides the information necessary to build and search engine knn indices. */ -@AllArgsConstructor -@Getter -public class KNNMethod { - - private final MethodComponent methodComponent; - private final Set spaces; +public interface KNNMethod { /** * Determines whether the provided space is supported for this method @@ -38,9 +24,7 @@ public class KNNMethod { * @param space to be checked * @return true if the space is supported; false otherwise */ - public boolean isSpaceTypeSupported(SpaceType space) { - return spaces.contains(space); - } + boolean isSpaceTypeSupported(SpaceType space); /** * Validate that the configured KNNMethodContext is valid for this method @@ -48,33 +32,7 @@ public boolean isSpaceTypeSupported(SpaceType space) { * @param knnMethodContext to be validated * @return ValidationException produced by validation errors; null if no validations errors. */ - public ValidationException validate(KNNMethodContext knnMethodContext) { - List errorMessages = new ArrayList<>(); - if (!isSpaceTypeSupported(knnMethodContext.getSpaceType())) { - errorMessages.add( - String.format( - Locale.ROOT, - "\"%s\" with \"%s\" configuration does not support space type: " + "\"%s\".", - this.methodComponent.getName(), - knnMethodContext.getKnnEngine().getName().toLowerCase(Locale.ROOT), - knnMethodContext.getSpaceType().getValue() - ) - ); - } - - ValidationException methodValidation = methodComponent.validate(knnMethodContext.getMethodComponentContext()); - if (methodValidation != null) { - errorMessages.addAll(methodValidation.validationErrors()); - } - - if (errorMessages.isEmpty()) { - return null; - } - - ValidationException validationException = new ValidationException(); - validationException.addValidationErrors(errorMessages); - return validationException; - } + ValidationException validate(KNNMethodContext knnMethodContext); /** * Validate that the configured KNNMethodContext is valid for this method, using additional data not present in the method context @@ -83,36 +41,7 @@ public ValidationException validate(KNNMethodContext knnMethodContext) { * @param vectorSpaceInfo additional data not present in the method context * @return ValidationException produced by validation errors; null if no validations errors. */ - public ValidationException validateWithData(KNNMethodContext knnMethodContext, VectorSpaceInfo vectorSpaceInfo) { - List errorMessages = new ArrayList<>(); - if (!isSpaceTypeSupported(knnMethodContext.getSpaceType())) { - errorMessages.add( - String.format( - Locale.ROOT, - "\"%s\" with \"%s\" configuration does not support space type: " + "\"%s\".", - this.methodComponent.getName(), - knnMethodContext.getKnnEngine().getName().toLowerCase(Locale.ROOT), - knnMethodContext.getSpaceType().getValue() - ) - ); - } - - ValidationException methodValidation = methodComponent.validateWithData( - knnMethodContext.getMethodComponentContext(), - vectorSpaceInfo - ); - if (methodValidation != null) { - errorMessages.addAll(methodValidation.validationErrors()); - } - - if (errorMessages.isEmpty()) { - return null; - } - - ValidationException validationException = new ValidationException(); - validationException.addValidationErrors(errorMessages); - return validationException; - } + ValidationException validateWithData(KNNMethodContext knnMethodContext, VectorSpaceInfo vectorSpaceInfo); /** * returns whether training is required or not @@ -120,9 +49,7 @@ public ValidationException validateWithData(KNNMethodContext knnMethodContext, V * @param knnMethodContext context to check if training is required on * @return true if training is required; false otherwise */ - public boolean isTrainingRequired(KNNMethodContext knnMethodContext) { - return methodComponent.isTrainingRequired(knnMethodContext.getMethodComponentContext()); - } + boolean isTrainingRequired(KNNMethodContext knnMethodContext); /** * Returns the estimated overhead of the method in KB @@ -131,9 +58,7 @@ public boolean isTrainingRequired(KNNMethodContext knnMethodContext) { * @param dimension dimension to make estimate with * @return estimate overhead in KB */ - public int estimateOverheadInKB(KNNMethodContext knnMethodContext, int dimension) { - return methodComponent.estimateOverheadInKB(knnMethodContext.getMethodComponentContext(), dimension); - } + int estimateOverheadInKB(KNNMethodContext knnMethodContext, int dimension); /** * Parse knnMethodContext into a map that the library can use to configure the index @@ -141,53 +66,12 @@ public int estimateOverheadInKB(KNNMethodContext knnMethodContext, int dimension * @param knnMethodContext from which to generate map * @return KNNMethod as a map */ - public Map getAsMap(KNNMethodContext knnMethodContext) { - Map parameterMap = new HashMap<>(methodComponent.getAsMap(knnMethodContext.getMethodComponentContext())); - parameterMap.put(KNNConstants.SPACE_TYPE, knnMethodContext.getSpaceType().getValue()); - return parameterMap; - } + Map getAsMap(KNNMethodContext knnMethodContext); /** - * Builder for KNNMethod + * Get the method context for a particular method + * + * @return EngineSpecificMethodContext for the method */ - public static class Builder { - - private MethodComponent methodComponent; - private Set spaces; - - /** - * Method to get a Builder instance - * - * @param methodComponent top level method component for the method - * @return Builder instance - */ - public static Builder builder(MethodComponent methodComponent) { - return new Builder(methodComponent); - } - - private Builder(MethodComponent methodComponent) { - this.methodComponent = methodComponent; - this.spaces = new HashSet<>(); - } - - /** - * Add spaces to KNNMethod - * - * @param spaceTypes to be added - * @return Builder - */ - public Builder addSpaces(SpaceType... spaceTypes) { - spaces.addAll(Arrays.asList(spaceTypes)); - return this; - } - - /** - * Build KNNMethod from builder - * - * @return KNNMethod initialized from builder - */ - public KNNMethod build() { - return new KNNMethod(methodComponent, spaces); - } - } + EngineSpecificMethodContext getEngineSpecificMethodContext(); } diff --git a/src/main/java/org/opensearch/knn/index/engine/NativeLibrary.java b/src/main/java/org/opensearch/knn/index/engine/NativeLibrary.java index dbbadf4f4..1e34cc380 100644 --- a/src/main/java/org/opensearch/knn/index/engine/NativeLibrary.java +++ b/src/main/java/org/opensearch/knn/index/engine/NativeLibrary.java @@ -34,12 +34,11 @@ public abstract class NativeLibrary extends AbstractKNNLibrary { */ public NativeLibrary( Map methods, - Map engineMethods, Map> scoreTranslation, String version, String extension ) { - super(methods, engineMethods, version); + super(methods, version); this.scoreTranslation = scoreTranslation; this.extension = extension; this.initialized = new AtomicBoolean(false); @@ -62,7 +61,7 @@ public float score(float rawScore, SpaceType spaceType) { @Override public int estimateOverheadInKB(KNNMethodContext knnMethodContext, int dimension) { String methodName = knnMethodContext.getMethodComponentContext().getName(); - return getMethod(methodName).estimateOverheadInKB(knnMethodContext, dimension); + return methods.get(methodName).estimateOverheadInKB(knnMethodContext, dimension); } @Override diff --git a/src/main/java/org/opensearch/knn/index/engine/faiss/Faiss.java b/src/main/java/org/opensearch/knn/index/engine/faiss/Faiss.java index a665a4136..329acbdb8 100644 --- a/src/main/java/org/opensearch/knn/index/engine/faiss/Faiss.java +++ b/src/main/java/org/opensearch/knn/index/engine/faiss/Faiss.java @@ -6,54 +6,16 @@ package org.opensearch.knn.index.engine.faiss; import com.google.common.collect.ImmutableMap; -import lombok.AllArgsConstructor; import org.opensearch.knn.common.KNNConstants; -import org.opensearch.knn.index.KNNSettings; import org.opensearch.knn.index.SpaceType; -import org.opensearch.knn.index.engine.DefaultHnswContext; -import org.opensearch.knn.index.engine.DefaultIVFContext; import org.opensearch.knn.index.engine.KNNMethod; -import org.opensearch.knn.index.engine.MethodComponent; -import org.opensearch.knn.index.engine.MethodComponentContext; import org.opensearch.knn.index.engine.NativeLibrary; -import org.opensearch.knn.index.engine.Parameter; -import java.util.Collections; -import java.util.HashMap; import java.util.Map; -import java.util.Objects; import java.util.function.Function; -import static org.opensearch.knn.common.KNNConstants.BYTES_PER_KILOBYTES; -import static org.opensearch.knn.common.KNNConstants.ENCODER_PARAMETER_PQ_CODE_COUNT_DEFAULT; -import static org.opensearch.knn.common.KNNConstants.ENCODER_PARAMETER_PQ_CODE_COUNT_LIMIT; -import static org.opensearch.knn.common.KNNConstants.ENCODER_PARAMETER_PQ_CODE_SIZE; -import static org.opensearch.knn.common.KNNConstants.ENCODER_PARAMETER_PQ_CODE_SIZE_DEFAULT; -import static org.opensearch.knn.common.KNNConstants.ENCODER_PARAMETER_PQ_CODE_SIZE_LIMIT; -import static org.opensearch.knn.common.KNNConstants.ENCODER_PARAMETER_PQ_M; -import static org.opensearch.knn.common.KNNConstants.ENCODER_SQ; -import static org.opensearch.knn.common.KNNConstants.FAISS_HNSW_DESCRIPTION; -import static org.opensearch.knn.common.KNNConstants.FAISS_IVF_DESCRIPTION; -import static org.opensearch.knn.common.KNNConstants.FAISS_PQ_DESCRIPTION; -import static org.opensearch.knn.common.KNNConstants.FAISS_SQ_CLIP; -import static org.opensearch.knn.common.KNNConstants.FAISS_SQ_DESCRIPTION; -import static org.opensearch.knn.common.KNNConstants.FAISS_SQ_ENCODER_FP16; -import static org.opensearch.knn.common.KNNConstants.FAISS_SQ_ENCODER_TYPES; -import static org.opensearch.knn.common.KNNConstants.FAISS_SQ_TYPE; -import static org.opensearch.knn.common.KNNConstants.METHOD_ENCODER_PARAMETER; import static org.opensearch.knn.common.KNNConstants.METHOD_HNSW; import static org.opensearch.knn.common.KNNConstants.METHOD_IVF; -import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_EF_CONSTRUCTION; -import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_EF_SEARCH; -import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_M; -import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_NLIST; -import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_NLIST_DEFAULT; -import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_NLIST_LIMIT; -import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_NPROBES; -import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_NPROBES_DEFAULT; -import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_NPROBES_LIMIT; -import static org.opensearch.knn.common.KNNConstants.NAME; -import static org.opensearch.knn.common.KNNConstants.PARAMETERS; /** * Implements NativeLibrary for the faiss native library @@ -79,236 +41,11 @@ public class Faiss extends NativeLibrary { SpaceType, Function>builder().put(SpaceType.INNER_PRODUCT, score -> score > 1 ? 1 - score : 1 / score - 1).build(); - // Define encoders supported by faiss - private final static MethodComponentContext ENCODER_DEFAULT = new MethodComponentContext( - KNNConstants.ENCODER_FLAT, - Collections.emptyMap() - ); - - private final static Map COMMON_ENCODERS = ImmutableMap.of( - KNNConstants.ENCODER_FLAT, - MethodComponent.Builder.builder(KNNConstants.ENCODER_FLAT) - .setMapGenerator( - ((methodComponent, methodComponentContext) -> MethodAsMapBuilder.builder( - KNNConstants.FAISS_FLAT_DESCRIPTION, - methodComponent, - methodComponentContext - ).build()) - ) - .build(), - ENCODER_SQ, - MethodComponent.Builder.builder(ENCODER_SQ) - .addParameter( - FAISS_SQ_TYPE, - new Parameter.StringParameter(FAISS_SQ_TYPE, FAISS_SQ_ENCODER_FP16, FAISS_SQ_ENCODER_TYPES::contains) - ) - .addParameter(FAISS_SQ_CLIP, new Parameter.BooleanParameter(FAISS_SQ_CLIP, false, Objects::nonNull)) - .setMapGenerator( - ((methodComponent, methodComponentContext) -> MethodAsMapBuilder.builder( - FAISS_SQ_DESCRIPTION, - methodComponent, - methodComponentContext - ).addParameter(FAISS_SQ_TYPE, "", "").build()) - ) - .build() - ); - - private final static Map HNSW_ENCODERS = ImmutableMap.builder() - .putAll( - ImmutableMap.of( - KNNConstants.ENCODER_PQ, - MethodComponent.Builder.builder(KNNConstants.ENCODER_PQ) - .addParameter( - ENCODER_PARAMETER_PQ_M, - new Parameter.IntegerParameter( - ENCODER_PARAMETER_PQ_M, - ENCODER_PARAMETER_PQ_CODE_COUNT_DEFAULT, - v -> v > 0 && v < ENCODER_PARAMETER_PQ_CODE_COUNT_LIMIT, - (v, vectorSpaceInfo) -> vectorSpaceInfo.getDimension() % v == 0 - ) - ) - .addParameter( - ENCODER_PARAMETER_PQ_CODE_SIZE, - new Parameter.IntegerParameter( - ENCODER_PARAMETER_PQ_CODE_SIZE, - ENCODER_PARAMETER_PQ_CODE_SIZE_DEFAULT, - v -> Objects.equals(v, ENCODER_PARAMETER_PQ_CODE_SIZE_DEFAULT) - ) - ) - .setRequiresTraining(true) - .setMapGenerator( - ((methodComponent, methodComponentContext) -> MethodAsMapBuilder.builder( - FAISS_PQ_DESCRIPTION, - methodComponent, - methodComponentContext - ).addParameter(ENCODER_PARAMETER_PQ_M, "", "").build()) - ) - .setOverheadInKBEstimator((methodComponent, methodComponentContext, dimension) -> { - int codeSize = ENCODER_PARAMETER_PQ_CODE_SIZE_DEFAULT; - return ((4L * (1L << codeSize) * dimension) / BYTES_PER_KILOBYTES) + 1; - }) - .build() - ) - ) - .putAll(COMMON_ENCODERS) - .build(); - - private final static Map IVF_ENCODERS = ImmutableMap.builder() - .putAll( - ImmutableMap.of( - KNNConstants.ENCODER_PQ, - MethodComponent.Builder.builder(KNNConstants.ENCODER_PQ) - .addParameter( - ENCODER_PARAMETER_PQ_M, - new Parameter.IntegerParameter( - ENCODER_PARAMETER_PQ_M, - ENCODER_PARAMETER_PQ_CODE_COUNT_DEFAULT, - v -> v > 0 && v < ENCODER_PARAMETER_PQ_CODE_COUNT_LIMIT, - (v, vectorSpaceInfo) -> vectorSpaceInfo.getDimension() % v == 0 - ) - ) - .addParameter( - ENCODER_PARAMETER_PQ_CODE_SIZE, - new Parameter.IntegerParameter( - ENCODER_PARAMETER_PQ_CODE_SIZE, - ENCODER_PARAMETER_PQ_CODE_SIZE_DEFAULT, - v -> v > 0 && v < ENCODER_PARAMETER_PQ_CODE_SIZE_LIMIT - ) - ) - .setRequiresTraining(true) - .setMapGenerator( - ((methodComponent, methodComponentContext) -> MethodAsMapBuilder.builder( - FAISS_PQ_DESCRIPTION, - methodComponent, - methodComponentContext - ).addParameter(ENCODER_PARAMETER_PQ_M, "", "").addParameter(ENCODER_PARAMETER_PQ_CODE_SIZE, "x", "").build()) - ) - .setOverheadInKBEstimator((methodComponent, methodComponentContext, dimension) -> { - // Size estimate formula: (4 * d * 2^code_size) / 1024 + 1 - - // Get value of code size passed in by user - Object codeSizeObject = methodComponentContext.getParameters().get(ENCODER_PARAMETER_PQ_CODE_SIZE); - - // If not specified, get default value of code size - if (codeSizeObject == null) { - Parameter codeSizeParameter = methodComponent.getParameters().get(ENCODER_PARAMETER_PQ_CODE_SIZE); - if (codeSizeParameter == null) { - throw new IllegalStateException( - String.format("%s is not a valid parameter. This is a bug.", ENCODER_PARAMETER_PQ_CODE_SIZE) - ); - } - - codeSizeObject = codeSizeParameter.getDefaultValue(); - } - - if (!(codeSizeObject instanceof Integer)) { - throw new IllegalStateException(String.format("%s must be an integer.", ENCODER_PARAMETER_PQ_CODE_SIZE)); - } - - int codeSize = (Integer) codeSizeObject; - return ((4L * (1L << codeSize) * dimension) / BYTES_PER_KILOBYTES) + 1; - }) - .build() - ) - ) - .putAll(COMMON_ENCODERS) - .build(); - private final static Map METHODS = ImmutableMap.of( METHOD_HNSW, - KNNMethod.Builder.builder( - MethodComponent.Builder.builder(METHOD_HNSW) - .addParameter( - METHOD_PARAMETER_M, - new Parameter.IntegerParameter(METHOD_PARAMETER_M, KNNSettings.INDEX_KNN_DEFAULT_ALGO_PARAM_M, v -> v > 0) - ) - .addParameter( - METHOD_PARAMETER_EF_CONSTRUCTION, - new Parameter.IntegerParameter( - METHOD_PARAMETER_EF_CONSTRUCTION, - KNNSettings.INDEX_KNN_DEFAULT_ALGO_PARAM_EF_CONSTRUCTION, - v -> v > 0 - ) - ) - .addParameter( - METHOD_PARAMETER_EF_SEARCH, - new Parameter.IntegerParameter( - METHOD_PARAMETER_EF_SEARCH, - KNNSettings.INDEX_KNN_DEFAULT_ALGO_PARAM_EF_SEARCH, - v -> v > 0 - ) - ) - .addParameter( - METHOD_ENCODER_PARAMETER, - new Parameter.MethodComponentContextParameter(METHOD_ENCODER_PARAMETER, ENCODER_DEFAULT, HNSW_ENCODERS) - ) - .setMapGenerator( - ((methodComponent, methodComponentContext) -> MethodAsMapBuilder.builder( - FAISS_HNSW_DESCRIPTION, - methodComponent, - methodComponentContext - ).addParameter(METHOD_PARAMETER_M, "", "").addParameter(METHOD_ENCODER_PARAMETER, ",", "").build()) - ) - .build() - ).addSpaces(SpaceType.UNDEFINED, SpaceType.HAMMING, SpaceType.L2, SpaceType.INNER_PRODUCT).build(), + new FaissHNSWMethod(), METHOD_IVF, - KNNMethod.Builder.builder( - MethodComponent.Builder.builder(METHOD_IVF) - .addParameter( - METHOD_PARAMETER_NPROBES, - new Parameter.IntegerParameter( - METHOD_PARAMETER_NPROBES, - METHOD_PARAMETER_NPROBES_DEFAULT, - v -> v > 0 && v < METHOD_PARAMETER_NPROBES_LIMIT - ) - ) - .addParameter( - METHOD_PARAMETER_NLIST, - new Parameter.IntegerParameter( - METHOD_PARAMETER_NLIST, - METHOD_PARAMETER_NLIST_DEFAULT, - v -> v > 0 && v < METHOD_PARAMETER_NLIST_LIMIT - ) - ) - .addParameter( - METHOD_ENCODER_PARAMETER, - new Parameter.MethodComponentContextParameter(METHOD_ENCODER_PARAMETER, ENCODER_DEFAULT, IVF_ENCODERS) - ) - .setRequiresTraining(true) - .setMapGenerator( - ((methodComponent, methodComponentContext) -> MethodAsMapBuilder.builder( - FAISS_IVF_DESCRIPTION, - methodComponent, - methodComponentContext - ).addParameter(METHOD_PARAMETER_NLIST, "", "").addParameter(METHOD_ENCODER_PARAMETER, ",", "").build()) - ) - .setOverheadInKBEstimator((methodComponent, methodComponentContext, dimension) -> { - // Size estimate formula: (4 * nlists * d) / 1024 + 1 - - // Get value of nlists passed in by user - Object nlistObject = methodComponentContext.getParameters().get(METHOD_PARAMETER_NLIST); - - // If not specified, get default value of nlist - if (nlistObject == null) { - Parameter nlistParameter = methodComponent.getParameters().get(METHOD_PARAMETER_NLIST); - if (nlistParameter == null) { - throw new IllegalStateException( - String.format("%s is not a valid parameter. This is a bug.", METHOD_PARAMETER_NLIST) - ); - } - - nlistObject = nlistParameter.getDefaultValue(); - } - - if (!(nlistObject instanceof Integer)) { - throw new IllegalStateException(String.format("%s must be an integer.", METHOD_PARAMETER_NLIST)); - } - - int centroids = (Integer) nlistObject; - return ((4L * centroids * dimension) / BYTES_PER_KILOBYTES) + 1; - }) - .build() - ).addSpaces(SpaceType.UNDEFINED, SpaceType.L2, SpaceType.INNER_PRODUCT, SpaceType.HAMMING).build() + new FaissIVFMethod() ); public final static Faiss INSTANCE = new Faiss( @@ -334,13 +71,7 @@ private Faiss( String extension, Map> scoreTransform ) { - super( - methods, - Map.of(METHOD_HNSW, new DefaultHnswContext(), METHOD_IVF, new DefaultIVFContext()), - scoreTranslation, - currentVersion, - extension - ); + super(methods, scoreTranslation, currentVersion, extension); this.scoreTransform = scoreTransform; } @@ -358,81 +89,4 @@ public Float scoreToRadialThreshold(Float score, SpaceType spaceType) { } return spaceType.scoreToDistanceTranslation(score); } - - /** - * MethodAsMap builder is used to create the map that will be passed to the jni to create the faiss index. - * Faiss's index factory takes an "index description" that it uses to build the index. In this description, - * some parameters of the index can be configured; others need to be manually set. MethodMap builder creates - * the index description from a set of parameters and removes them from the map. On build, it sets the index - * description in the map and returns the processed map - */ - @AllArgsConstructor - static class MethodAsMapBuilder { - String indexDescription; - MethodComponent methodComponent; - Map methodAsMap; - - /** - * Add a parameter that will be used in the index description for the given method component - * - * @param parameterName name of the parameter - * @param prefix to append to the index description before the parameter - * @param suffix to append to the index description after the parameter - * @return this builder - */ - @SuppressWarnings("unchecked") - MethodAsMapBuilder addParameter(String parameterName, String prefix, String suffix) { - indexDescription += prefix; - - // When we add a parameter, what we are doing is taking it from the methods parameter and building it - // into the index description string faiss uses to create the index. - Map methodParameters = (Map) methodAsMap.get(PARAMETERS); - Parameter parameter = methodComponent.getParameters().get(parameterName); - Object value = methodParameters.containsKey(parameterName) ? methodParameters.get(parameterName) : parameter.getDefaultValue(); - - // Recursion is needed if the parameter is a method component context itself. - if (parameter instanceof Parameter.MethodComponentContextParameter) { - MethodComponentContext subMethodComponentContext = (MethodComponentContext) value; - MethodComponent subMethodComponent = ((Parameter.MethodComponentContextParameter) parameter).getMethodComponent( - subMethodComponentContext.getName() - ); - - Map subMethodAsMap = subMethodComponent.getAsMap(subMethodComponentContext); - indexDescription += subMethodAsMap.get(KNNConstants.INDEX_DESCRIPTION_PARAMETER); - subMethodAsMap.remove(KNNConstants.INDEX_DESCRIPTION_PARAMETER); - - // We replace parameterName with the map that contains only parameters that are not included in - // the method description - methodParameters.put(parameterName, subMethodAsMap); - } else { - // Just add the value to the method description and remove from map - indexDescription += value; - methodParameters.remove(parameterName); - } - - indexDescription += suffix; - return this; - } - - /** - * Build - * - * @return Method as a map - */ - Map build() { - methodAsMap.put(KNNConstants.INDEX_DESCRIPTION_PARAMETER, indexDescription); - return methodAsMap; - } - - static MethodAsMapBuilder builder( - String baseDescription, - MethodComponent methodComponent, - MethodComponentContext methodComponentContext - ) { - Map initialMap = new HashMap<>(); - initialMap.put(NAME, methodComponent.getName()); - initialMap.put(PARAMETERS, MethodComponent.getParameterMapWithDefaultsAdded(methodComponentContext, methodComponent)); - return new MethodAsMapBuilder(baseDescription, methodComponent, initialMap); - } - } } diff --git a/src/main/java/org/opensearch/knn/index/engine/faiss/FaissFlatEncoder.java b/src/main/java/org/opensearch/knn/index/engine/faiss/FaissFlatEncoder.java new file mode 100644 index 000000000..aea3bf51a --- /dev/null +++ b/src/main/java/org/opensearch/knn/index/engine/faiss/FaissFlatEncoder.java @@ -0,0 +1,32 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.engine.faiss; + +import org.opensearch.knn.common.KNNConstants; +import org.opensearch.knn.index.engine.Encoder; +import org.opensearch.knn.index.engine.MethodComponent; + +/** + * Flat faiss encoder. Flat encoding means that it does nothing. It needs an encoder, though, because it + * is used in generating the index description. + */ +public class FaissFlatEncoder implements Encoder { + + private final static MethodComponent METHOD_COMPONENT = MethodComponent.Builder.builder(KNNConstants.ENCODER_FLAT) + .setMapGenerator( + ((methodComponent, methodComponentContext) -> MethodAsMapBuilder.builder( + KNNConstants.FAISS_FLAT_DESCRIPTION, + methodComponent, + methodComponentContext + ).build()) + ) + .build(); + + @Override + public MethodComponent getMethodComponent() { + return METHOD_COMPONENT; + } +} diff --git a/src/main/java/org/opensearch/knn/index/engine/faiss/FaissHNSWMethod.java b/src/main/java/org/opensearch/knn/index/engine/faiss/FaissHNSWMethod.java new file mode 100644 index 000000000..ac05afc33 --- /dev/null +++ b/src/main/java/org/opensearch/knn/index/engine/faiss/FaissHNSWMethod.java @@ -0,0 +1,93 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.engine.faiss; + +import org.opensearch.knn.common.KNNConstants; +import org.opensearch.knn.index.KNNSettings; +import org.opensearch.knn.index.SpaceType; +import org.opensearch.knn.index.engine.AbstractKNNMethod; +import org.opensearch.knn.index.engine.DefaultHnswContext; +import org.opensearch.knn.index.engine.Encoder; +import org.opensearch.knn.index.engine.MethodComponent; +import org.opensearch.knn.index.engine.MethodComponentContext; +import org.opensearch.knn.index.engine.Parameter; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + +import static org.opensearch.knn.common.KNNConstants.FAISS_HNSW_DESCRIPTION; +import static org.opensearch.knn.common.KNNConstants.METHOD_ENCODER_PARAMETER; +import static org.opensearch.knn.common.KNNConstants.METHOD_HNSW; +import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_EF_CONSTRUCTION; +import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_EF_SEARCH; +import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_M; + +/** + * Faiss HNSW method implementation + */ +public class FaissHNSWMethod extends AbstractKNNMethod { + public final static List SUPPORTED_SPACES = Arrays.asList( + SpaceType.UNDEFINED, + SpaceType.HAMMING, + SpaceType.L2, + SpaceType.INNER_PRODUCT + ); + + private final static MethodComponentContext DEFAULT_ENCODER_CONTEXT = new MethodComponentContext( + KNNConstants.ENCODER_FLAT, + Collections.emptyMap() + ); + private final static List SUPPORTED_ENCODERS = List.of(new FaissFlatEncoder(), new FaissSQEncoder(), new FaissHNSWPQEncoder()); + + /** + * Constructor for FaissHNSWMethod + * + * @see AbstractKNNMethod + */ + public FaissHNSWMethod() { + super(initMethodComponent(), Set.copyOf(SUPPORTED_SPACES), new DefaultHnswContext()); + } + + private static MethodComponent initMethodComponent() { + return MethodComponent.Builder.builder(METHOD_HNSW) + .addParameter( + METHOD_PARAMETER_M, + new Parameter.IntegerParameter(METHOD_PARAMETER_M, KNNSettings.INDEX_KNN_DEFAULT_ALGO_PARAM_M, v -> v > 0) + ) + .addParameter( + METHOD_PARAMETER_EF_CONSTRUCTION, + new Parameter.IntegerParameter( + METHOD_PARAMETER_EF_CONSTRUCTION, + KNNSettings.INDEX_KNN_DEFAULT_ALGO_PARAM_EF_CONSTRUCTION, + v -> v > 0 + ) + ) + .addParameter( + METHOD_PARAMETER_EF_SEARCH, + new Parameter.IntegerParameter(METHOD_PARAMETER_EF_SEARCH, KNNSettings.INDEX_KNN_DEFAULT_ALGO_PARAM_EF_SEARCH, v -> v > 0) + ) + .addParameter(METHOD_ENCODER_PARAMETER, initEncoderParameter()) + .setMapGenerator( + ((methodComponent, methodComponentContext) -> MethodAsMapBuilder.builder( + FAISS_HNSW_DESCRIPTION, + methodComponent, + methodComponentContext + ).addParameter(METHOD_PARAMETER_M, "", "").addParameter(METHOD_ENCODER_PARAMETER, ",", "").build()) + ) + .build(); + } + + private static Parameter.MethodComponentContextParameter initEncoderParameter() { + return new Parameter.MethodComponentContextParameter( + METHOD_ENCODER_PARAMETER, + DEFAULT_ENCODER_CONTEXT, + SUPPORTED_ENCODERS.stream().collect(Collectors.toMap(Encoder::getName, Encoder::getMethodComponent)) + ); + } +} diff --git a/src/main/java/org/opensearch/knn/index/engine/faiss/FaissHNSWPQEncoder.java b/src/main/java/org/opensearch/knn/index/engine/faiss/FaissHNSWPQEncoder.java new file mode 100644 index 000000000..9880b2cd9 --- /dev/null +++ b/src/main/java/org/opensearch/knn/index/engine/faiss/FaissHNSWPQEncoder.java @@ -0,0 +1,65 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.engine.faiss; + +import org.opensearch.knn.common.KNNConstants; +import org.opensearch.knn.index.engine.Encoder; +import org.opensearch.knn.index.engine.MethodComponent; +import org.opensearch.knn.index.engine.Parameter; + +import java.util.Objects; + +import static org.opensearch.knn.common.KNNConstants.BYTES_PER_KILOBYTES; +import static org.opensearch.knn.common.KNNConstants.ENCODER_PARAMETER_PQ_CODE_COUNT_DEFAULT; +import static org.opensearch.knn.common.KNNConstants.ENCODER_PARAMETER_PQ_CODE_COUNT_LIMIT; +import static org.opensearch.knn.common.KNNConstants.ENCODER_PARAMETER_PQ_CODE_SIZE; +import static org.opensearch.knn.common.KNNConstants.ENCODER_PARAMETER_PQ_CODE_SIZE_DEFAULT; +import static org.opensearch.knn.common.KNNConstants.ENCODER_PARAMETER_PQ_M; +import static org.opensearch.knn.common.KNNConstants.FAISS_PQ_DESCRIPTION; + +/** + * Faiss HNSW PQ encoder. Right now, the implementations are slightly different during validation between this an + * {@link FaissIVFPQEncoder}. Hence, they are separate classes. + */ +public class FaissHNSWPQEncoder implements Encoder { + + private final static MethodComponent METHOD_COMPONENT = MethodComponent.Builder.builder(KNNConstants.ENCODER_PQ) + .addParameter( + ENCODER_PARAMETER_PQ_M, + new Parameter.IntegerParameter( + ENCODER_PARAMETER_PQ_M, + ENCODER_PARAMETER_PQ_CODE_COUNT_DEFAULT, + v -> v > 0 && v < ENCODER_PARAMETER_PQ_CODE_COUNT_LIMIT, + (v, vectorSpaceInfo) -> vectorSpaceInfo.getDimension() % v == 0 + ) + ) + .addParameter( + ENCODER_PARAMETER_PQ_CODE_SIZE, + new Parameter.IntegerParameter( + ENCODER_PARAMETER_PQ_CODE_SIZE, + ENCODER_PARAMETER_PQ_CODE_SIZE_DEFAULT, + v -> Objects.equals(v, ENCODER_PARAMETER_PQ_CODE_SIZE_DEFAULT) + ) + ) + .setRequiresTraining(true) + .setMapGenerator( + ((methodComponent, methodComponentContext) -> MethodAsMapBuilder.builder( + FAISS_PQ_DESCRIPTION, + methodComponent, + methodComponentContext + ).addParameter(ENCODER_PARAMETER_PQ_M, "", "").build()) + ) + .setOverheadInKBEstimator((methodComponent, methodComponentContext, dimension) -> { + int codeSize = ENCODER_PARAMETER_PQ_CODE_SIZE_DEFAULT; + return ((4L * (1L << codeSize) * dimension) / BYTES_PER_KILOBYTES) + 1; + }) + .build(); + + @Override + public MethodComponent getMethodComponent() { + return METHOD_COMPONENT; + } +} diff --git a/src/main/java/org/opensearch/knn/index/engine/faiss/FaissIVFMethod.java b/src/main/java/org/opensearch/knn/index/engine/faiss/FaissIVFMethod.java new file mode 100644 index 000000000..813cb6e9e --- /dev/null +++ b/src/main/java/org/opensearch/knn/index/engine/faiss/FaissIVFMethod.java @@ -0,0 +1,123 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.engine.faiss; + +import org.opensearch.knn.common.KNNConstants; +import org.opensearch.knn.index.SpaceType; +import org.opensearch.knn.index.engine.AbstractKNNMethod; +import org.opensearch.knn.index.engine.DefaultIVFContext; +import org.opensearch.knn.index.engine.Encoder; +import org.opensearch.knn.index.engine.MethodComponent; +import org.opensearch.knn.index.engine.MethodComponentContext; +import org.opensearch.knn.index.engine.Parameter; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + +import static org.opensearch.knn.common.KNNConstants.BYTES_PER_KILOBYTES; +import static org.opensearch.knn.common.KNNConstants.FAISS_IVF_DESCRIPTION; +import static org.opensearch.knn.common.KNNConstants.METHOD_ENCODER_PARAMETER; +import static org.opensearch.knn.common.KNNConstants.METHOD_IVF; +import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_NLIST; +import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_NLIST_DEFAULT; +import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_NLIST_LIMIT; +import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_NPROBES; +import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_NPROBES_DEFAULT; +import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_NPROBES_LIMIT; + +/** + * Faiss ivf implementation + */ +public class FaissIVFMethod extends AbstractKNNMethod { + + public final static List SUPPORTED_SPACES = Arrays.asList( + SpaceType.UNDEFINED, + SpaceType.L2, + SpaceType.INNER_PRODUCT, + SpaceType.HAMMING + ); + + private final static MethodComponentContext DEFAULT_ENCODER_CONTEXT = new MethodComponentContext( + KNNConstants.ENCODER_FLAT, + Collections.emptyMap() + ); + private final static List SUPPORTED_ENCODERS = List.of(new FaissFlatEncoder(), new FaissSQEncoder(), new FaissIVFPQEncoder()); + + /** + * Constructor for FaissIVFMethod + * + * @see AbstractKNNMethod + */ + public FaissIVFMethod() { + super(initMethodComponent(), Set.copyOf(SUPPORTED_SPACES), new DefaultIVFContext()); + } + + private static MethodComponent initMethodComponent() { + return MethodComponent.Builder.builder(METHOD_IVF) + .addParameter( + METHOD_PARAMETER_NPROBES, + new Parameter.IntegerParameter( + METHOD_PARAMETER_NPROBES, + METHOD_PARAMETER_NPROBES_DEFAULT, + v -> v > 0 && v < METHOD_PARAMETER_NPROBES_LIMIT + ) + ) + .addParameter( + METHOD_PARAMETER_NLIST, + new Parameter.IntegerParameter( + METHOD_PARAMETER_NLIST, + METHOD_PARAMETER_NLIST_DEFAULT, + v -> v > 0 && v < METHOD_PARAMETER_NLIST_LIMIT + ) + ) + .addParameter(METHOD_ENCODER_PARAMETER, initEncoderParameter()) + .setRequiresTraining(true) + .setMapGenerator( + ((methodComponent, methodComponentContext) -> MethodAsMapBuilder.builder( + FAISS_IVF_DESCRIPTION, + methodComponent, + methodComponentContext + ).addParameter(METHOD_PARAMETER_NLIST, "", "").addParameter(METHOD_ENCODER_PARAMETER, ",", "").build()) + ) + .setOverheadInKBEstimator((methodComponent, methodComponentContext, dimension) -> { + // Size estimate formula: (4 * nlists * d) / 1024 + 1 + + // Get value of nlists passed in by user + Object nlistObject = methodComponentContext.getParameters().get(METHOD_PARAMETER_NLIST); + + // If not specified, get default value of nlist + if (nlistObject == null) { + Parameter nlistParameter = methodComponent.getParameters().get(METHOD_PARAMETER_NLIST); + if (nlistParameter == null) { + throw new IllegalStateException( + String.format("%s is not a valid parameter. This is a bug.", METHOD_PARAMETER_NLIST) + ); + } + + nlistObject = nlistParameter.getDefaultValue(); + } + + if (!(nlistObject instanceof Integer)) { + throw new IllegalStateException(String.format("%s must be an integer.", METHOD_PARAMETER_NLIST)); + } + + int centroids = (Integer) nlistObject; + return ((4L * centroids * dimension) / BYTES_PER_KILOBYTES) + 1; + }) + .build(); + } + + private static Parameter.MethodComponentContextParameter initEncoderParameter() { + return new Parameter.MethodComponentContextParameter( + METHOD_ENCODER_PARAMETER, + DEFAULT_ENCODER_CONTEXT, + SUPPORTED_ENCODERS.stream().collect(Collectors.toMap(Encoder::getName, Encoder::getMethodComponent)) + ); + } +} diff --git a/src/main/java/org/opensearch/knn/index/engine/faiss/FaissIVFPQEncoder.java b/src/main/java/org/opensearch/knn/index/engine/faiss/FaissIVFPQEncoder.java new file mode 100644 index 000000000..b9632004d --- /dev/null +++ b/src/main/java/org/opensearch/knn/index/engine/faiss/FaissIVFPQEncoder.java @@ -0,0 +1,84 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.engine.faiss; + +import org.opensearch.knn.common.KNNConstants; +import org.opensearch.knn.index.engine.Encoder; +import org.opensearch.knn.index.engine.MethodComponent; +import org.opensearch.knn.index.engine.Parameter; + +import static org.opensearch.knn.common.KNNConstants.BYTES_PER_KILOBYTES; +import static org.opensearch.knn.common.KNNConstants.ENCODER_PARAMETER_PQ_CODE_COUNT_DEFAULT; +import static org.opensearch.knn.common.KNNConstants.ENCODER_PARAMETER_PQ_CODE_COUNT_LIMIT; +import static org.opensearch.knn.common.KNNConstants.ENCODER_PARAMETER_PQ_CODE_SIZE; +import static org.opensearch.knn.common.KNNConstants.ENCODER_PARAMETER_PQ_CODE_SIZE_DEFAULT; +import static org.opensearch.knn.common.KNNConstants.ENCODER_PARAMETER_PQ_CODE_SIZE_LIMIT; +import static org.opensearch.knn.common.KNNConstants.ENCODER_PARAMETER_PQ_M; +import static org.opensearch.knn.common.KNNConstants.FAISS_PQ_DESCRIPTION; + +/** + * Faiss IVF PQ encoder. Right now, the implementations are slightly different during validation between this an + * {@link FaissHNSWPQEncoder}. Hence, they are separate classes. + */ +public class FaissIVFPQEncoder implements Encoder { + private final static MethodComponent METHOD_COMPONENT = MethodComponent.Builder.builder(KNNConstants.ENCODER_PQ) + .addParameter( + ENCODER_PARAMETER_PQ_M, + new Parameter.IntegerParameter( + ENCODER_PARAMETER_PQ_M, + ENCODER_PARAMETER_PQ_CODE_COUNT_DEFAULT, + v -> v > 0 && v < ENCODER_PARAMETER_PQ_CODE_COUNT_LIMIT, + (v, vectorSpaceInfo) -> vectorSpaceInfo.getDimension() % v == 0 + ) + ) + .addParameter( + ENCODER_PARAMETER_PQ_CODE_SIZE, + new Parameter.IntegerParameter( + ENCODER_PARAMETER_PQ_CODE_SIZE, + ENCODER_PARAMETER_PQ_CODE_SIZE_DEFAULT, + v -> v > 0 && v < ENCODER_PARAMETER_PQ_CODE_SIZE_LIMIT + ) + ) + .setRequiresTraining(true) + .setMapGenerator( + ((methodComponent, methodComponentContext) -> MethodAsMapBuilder.builder( + FAISS_PQ_DESCRIPTION, + methodComponent, + methodComponentContext + ).addParameter(ENCODER_PARAMETER_PQ_M, "", "").addParameter(ENCODER_PARAMETER_PQ_CODE_SIZE, "x", "").build()) + ) + .setOverheadInKBEstimator((methodComponent, methodComponentContext, dimension) -> { + // Size estimate formula: (4 * d * 2^code_size) / 1024 + 1 + + // Get value of code size passed in by user + Object codeSizeObject = methodComponentContext.getParameters().get(ENCODER_PARAMETER_PQ_CODE_SIZE); + + // If not specified, get default value of code size + if (codeSizeObject == null) { + Parameter codeSizeParameter = methodComponent.getParameters().get(ENCODER_PARAMETER_PQ_CODE_SIZE); + if (codeSizeParameter == null) { + throw new IllegalStateException( + String.format("%s is not a valid parameter. This is a bug.", ENCODER_PARAMETER_PQ_CODE_SIZE) + ); + } + + codeSizeObject = codeSizeParameter.getDefaultValue(); + } + + if (!(codeSizeObject instanceof Integer)) { + throw new IllegalStateException(String.format("%s must be an integer.", ENCODER_PARAMETER_PQ_CODE_SIZE)); + } + + int codeSize = (Integer) codeSizeObject; + return ((4L * (1L << codeSize) * dimension) / BYTES_PER_KILOBYTES) + 1; + }) + .build(); + + @Override + public MethodComponent getMethodComponent() { + return METHOD_COMPONENT; + } +} diff --git a/src/main/java/org/opensearch/knn/index/engine/faiss/FaissSQEncoder.java b/src/main/java/org/opensearch/knn/index/engine/faiss/FaissSQEncoder.java new file mode 100644 index 000000000..eb0af9c38 --- /dev/null +++ b/src/main/java/org/opensearch/knn/index/engine/faiss/FaissSQEncoder.java @@ -0,0 +1,41 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.engine.faiss; + +import org.opensearch.knn.index.engine.Encoder; +import org.opensearch.knn.index.engine.MethodComponent; +import org.opensearch.knn.index.engine.Parameter; + +import java.util.Objects; + +import static org.opensearch.knn.common.KNNConstants.ENCODER_SQ; +import static org.opensearch.knn.common.KNNConstants.FAISS_SQ_CLIP; +import static org.opensearch.knn.common.KNNConstants.FAISS_SQ_DESCRIPTION; +import static org.opensearch.knn.common.KNNConstants.FAISS_SQ_ENCODER_FP16; +import static org.opensearch.knn.common.KNNConstants.FAISS_SQ_ENCODER_TYPES; +import static org.opensearch.knn.common.KNNConstants.FAISS_SQ_TYPE; + +/** + * Faiss SQ encoder + */ +public class FaissSQEncoder implements Encoder { + private final static MethodComponent METHOD_COMPONENT = MethodComponent.Builder.builder(ENCODER_SQ) + .addParameter(FAISS_SQ_TYPE, new Parameter.StringParameter(FAISS_SQ_TYPE, FAISS_SQ_ENCODER_FP16, FAISS_SQ_ENCODER_TYPES::contains)) + .addParameter(FAISS_SQ_CLIP, new Parameter.BooleanParameter(FAISS_SQ_CLIP, false, Objects::nonNull)) + .setMapGenerator( + ((methodComponent, methodComponentContext) -> MethodAsMapBuilder.builder( + FAISS_SQ_DESCRIPTION, + methodComponent, + methodComponentContext + ).addParameter(FAISS_SQ_TYPE, "", "").build()) + ) + .build(); + + @Override + public MethodComponent getMethodComponent() { + return METHOD_COMPONENT; + } +} diff --git a/src/main/java/org/opensearch/knn/index/engine/faiss/MethodAsMapBuilder.java b/src/main/java/org/opensearch/knn/index/engine/faiss/MethodAsMapBuilder.java new file mode 100644 index 000000000..445abfdd8 --- /dev/null +++ b/src/main/java/org/opensearch/knn/index/engine/faiss/MethodAsMapBuilder.java @@ -0,0 +1,95 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.engine.faiss; + +import lombok.AllArgsConstructor; +import org.opensearch.knn.common.KNNConstants; +import org.opensearch.knn.index.engine.MethodComponent; +import org.opensearch.knn.index.engine.MethodComponentContext; +import org.opensearch.knn.index.engine.Parameter; + +import java.util.HashMap; +import java.util.Map; + +import static org.opensearch.knn.common.KNNConstants.NAME; +import static org.opensearch.knn.common.KNNConstants.PARAMETERS; + +/** + * MethodAsMap builder is used to create the map that will be passed to the jni to create the faiss index. + * Faiss's index factory takes an "index description" that it uses to build the index. In this description, + * some parameters of the index can be configured; others need to be manually set. MethodMap builder creates + * the index description from a set of parameters and removes them from the map. On build, it sets the index + * description in the map and returns the processed map + */ +@AllArgsConstructor +class MethodAsMapBuilder { + String indexDescription; + MethodComponent methodComponent; + Map methodAsMap; + + /** + * Add a parameter that will be used in the index description for the given method component + * + * @param parameterName name of the parameter + * @param prefix to append to the index description before the parameter + * @param suffix to append to the index description after the parameter + * @return this builder + */ + @SuppressWarnings("unchecked") + MethodAsMapBuilder addParameter(String parameterName, String prefix, String suffix) { + indexDescription += prefix; + + // When we add a parameter, what we are doing is taking it from the methods parameter and building it + // into the index description string faiss uses to create the index. + Map methodParameters = (Map) methodAsMap.get(PARAMETERS); + Parameter parameter = methodComponent.getParameters().get(parameterName); + Object value = methodParameters.containsKey(parameterName) ? methodParameters.get(parameterName) : parameter.getDefaultValue(); + + // Recursion is needed if the parameter is a method component context itself. + if (parameter instanceof Parameter.MethodComponentContextParameter) { + MethodComponentContext subMethodComponentContext = (MethodComponentContext) value; + MethodComponent subMethodComponent = ((Parameter.MethodComponentContextParameter) parameter).getMethodComponent( + subMethodComponentContext.getName() + ); + + Map subMethodAsMap = subMethodComponent.getAsMap(subMethodComponentContext); + indexDescription += subMethodAsMap.get(KNNConstants.INDEX_DESCRIPTION_PARAMETER); + subMethodAsMap.remove(KNNConstants.INDEX_DESCRIPTION_PARAMETER); + + // We replace parameterName with the map that contains only parameters that are not included in + // the method description + methodParameters.put(parameterName, subMethodAsMap); + } else { + // Just add the value to the method description and remove from map + indexDescription += value; + methodParameters.remove(parameterName); + } + + indexDescription += suffix; + return this; + } + + /** + * Build + * + * @return Method as a map + */ + Map build() { + methodAsMap.put(KNNConstants.INDEX_DESCRIPTION_PARAMETER, indexDescription); + return methodAsMap; + } + + static MethodAsMapBuilder builder( + String baseDescription, + MethodComponent methodComponent, + MethodComponentContext methodComponentContext + ) { + Map initialMap = new HashMap<>(); + initialMap.put(NAME, methodComponent.getName()); + initialMap.put(PARAMETERS, MethodComponent.getParameterMapWithDefaultsAdded(methodComponentContext, methodComponent)); + return new MethodAsMapBuilder(baseDescription, methodComponent, initialMap); + } +} diff --git a/src/main/java/org/opensearch/knn/index/engine/lucene/Lucene.java b/src/main/java/org/opensearch/knn/index/engine/lucene/Lucene.java index 4199cbb19..986380897 100644 --- a/src/main/java/org/opensearch/knn/index/engine/lucene/Lucene.java +++ b/src/main/java/org/opensearch/knn/index/engine/lucene/Lucene.java @@ -7,31 +7,15 @@ import com.google.common.collect.ImmutableMap; import org.apache.lucene.util.Version; -import org.opensearch.knn.common.KNNConstants; -import org.opensearch.knn.index.KNNSettings; import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.engine.JVMLibrary; import org.opensearch.knn.index.engine.KNNMethod; -import org.opensearch.knn.index.engine.MethodComponent; -import org.opensearch.knn.index.engine.MethodComponentContext; -import org.opensearch.knn.index.engine.Parameter; -import java.util.Collections; import java.util.List; import java.util.Map; import java.util.function.Function; -import static org.opensearch.knn.common.KNNConstants.DYNAMIC_CONFIDENCE_INTERVAL; -import static org.opensearch.knn.common.KNNConstants.ENCODER_SQ; -import static org.opensearch.knn.common.KNNConstants.LUCENE_SQ_BITS; -import static org.opensearch.knn.common.KNNConstants.LUCENE_SQ_CONFIDENCE_INTERVAL; -import static org.opensearch.knn.common.KNNConstants.LUCENE_SQ_DEFAULT_BITS; -import static org.opensearch.knn.common.KNNConstants.MAXIMUM_CONFIDENCE_INTERVAL; -import static org.opensearch.knn.common.KNNConstants.METHOD_ENCODER_PARAMETER; import static org.opensearch.knn.common.KNNConstants.METHOD_HNSW; -import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_EF_CONSTRUCTION; -import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_M; -import static org.opensearch.knn.common.KNNConstants.MINIMUM_CONFIDENCE_INTERVAL; /** * KNN Library for Lucene @@ -39,54 +23,8 @@ public class Lucene extends JVMLibrary { Map> distanceTransform; - private static final List LUCENE_SQ_BITS_SUPPORTED = List.of(7); - - private final static MethodComponentContext ENCODER_DEFAULT = new MethodComponentContext( - KNNConstants.ENCODER_FLAT, - Collections.emptyMap() - ); - - private final static Map HNSW_ENCODERS = ImmutableMap.of( - ENCODER_SQ, - MethodComponent.Builder.builder(ENCODER_SQ) - .addParameter( - LUCENE_SQ_CONFIDENCE_INTERVAL, - new Parameter.DoubleParameter( - LUCENE_SQ_CONFIDENCE_INTERVAL, - null, - v -> v == DYNAMIC_CONFIDENCE_INTERVAL || (v >= MINIMUM_CONFIDENCE_INTERVAL && v <= MAXIMUM_CONFIDENCE_INTERVAL) - ) - ) - .addParameter( - LUCENE_SQ_BITS, - new Parameter.IntegerParameter(LUCENE_SQ_BITS, LUCENE_SQ_DEFAULT_BITS, LUCENE_SQ_BITS_SUPPORTED::contains) - ) - .build() - ); - - final static Map METHODS = ImmutableMap.of( - METHOD_HNSW, - KNNMethod.Builder.builder( - MethodComponent.Builder.builder(METHOD_HNSW) - .addParameter( - METHOD_PARAMETER_M, - new Parameter.IntegerParameter(METHOD_PARAMETER_M, KNNSettings.INDEX_KNN_DEFAULT_ALGO_PARAM_M, v -> v > 0) - ) - .addParameter( - METHOD_PARAMETER_EF_CONSTRUCTION, - new Parameter.IntegerParameter( - METHOD_PARAMETER_EF_CONSTRUCTION, - KNNSettings.INDEX_KNN_DEFAULT_ALGO_PARAM_EF_CONSTRUCTION, - v -> v > 0 - ) - ) - .addParameter( - METHOD_ENCODER_PARAMETER, - new Parameter.MethodComponentContextParameter(METHOD_ENCODER_PARAMETER, ENCODER_DEFAULT, HNSW_ENCODERS) - ) - .build() - ).addSpaces(SpaceType.UNDEFINED, SpaceType.L2, SpaceType.COSINESIMIL, SpaceType.INNER_PRODUCT).build() - ); + + final static Map METHODS = ImmutableMap.of(METHOD_HNSW, new LuceneHNSWMethod()); // Map that overrides the default distance translations for Lucene, check more details in knn documentation: // https://opensearch.org/docs/latest/search-plugins/knn/approximate-knn/#spaces @@ -107,7 +45,7 @@ public class Lucene extends JVMLibrary { * @param distanceTransform Map of space type to distance transformation function */ Lucene(Map methods, String version, Map> distanceTransform) { - super(methods, Map.of(METHOD_HNSW, new LuceneHNSWContext()), version); + super(methods, version); this.distanceTransform = distanceTransform; } diff --git a/src/main/java/org/opensearch/knn/index/engine/lucene/LuceneHNSWMethod.java b/src/main/java/org/opensearch/knn/index/engine/lucene/LuceneHNSWMethod.java new file mode 100644 index 000000000..0419a5440 --- /dev/null +++ b/src/main/java/org/opensearch/knn/index/engine/lucene/LuceneHNSWMethod.java @@ -0,0 +1,80 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.engine.lucene; + +import org.opensearch.knn.common.KNNConstants; +import org.opensearch.knn.index.KNNSettings; +import org.opensearch.knn.index.SpaceType; +import org.opensearch.knn.index.engine.AbstractKNNMethod; +import org.opensearch.knn.index.engine.Encoder; +import org.opensearch.knn.index.engine.MethodComponent; +import org.opensearch.knn.index.engine.MethodComponentContext; +import org.opensearch.knn.index.engine.Parameter; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + +import static org.opensearch.knn.common.KNNConstants.METHOD_ENCODER_PARAMETER; +import static org.opensearch.knn.common.KNNConstants.METHOD_HNSW; +import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_EF_CONSTRUCTION; +import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_M; + +/** + * Lucene HNSW implementation + */ +public class LuceneHNSWMethod extends AbstractKNNMethod { + + public final static List SUPPORTED_SPACES = Arrays.asList( + SpaceType.UNDEFINED, + SpaceType.L2, + SpaceType.COSINESIMIL, + SpaceType.INNER_PRODUCT + ); + + private final static MethodComponentContext DEFAULT_ENCODER_CONTEXT = new MethodComponentContext( + KNNConstants.ENCODER_FLAT, + Collections.emptyMap() + ); + private final static List SUPPORTED_ENCODERS = List.of(new LuceneSQEncoder()); + + /** + * Constructor for LuceneHNSWMethod + * + * @see AbstractKNNMethod + */ + public LuceneHNSWMethod() { + super(initMethodComponent(), Set.copyOf(SUPPORTED_SPACES), new LuceneHNSWContext()); + } + + private static MethodComponent initMethodComponent() { + return MethodComponent.Builder.builder(METHOD_HNSW) + .addParameter( + METHOD_PARAMETER_M, + new Parameter.IntegerParameter(METHOD_PARAMETER_M, KNNSettings.INDEX_KNN_DEFAULT_ALGO_PARAM_M, v -> v > 0) + ) + .addParameter( + METHOD_PARAMETER_EF_CONSTRUCTION, + new Parameter.IntegerParameter( + METHOD_PARAMETER_EF_CONSTRUCTION, + KNNSettings.INDEX_KNN_DEFAULT_ALGO_PARAM_EF_CONSTRUCTION, + v -> v > 0 + ) + ) + .addParameter(METHOD_ENCODER_PARAMETER, initEncoderParameter()) + .build(); + } + + private static Parameter.MethodComponentContextParameter initEncoderParameter() { + return new Parameter.MethodComponentContextParameter( + METHOD_ENCODER_PARAMETER, + DEFAULT_ENCODER_CONTEXT, + SUPPORTED_ENCODERS.stream().collect(Collectors.toMap(Encoder::getName, Encoder::getMethodComponent)) + ); + } +} diff --git a/src/main/java/org/opensearch/knn/index/engine/lucene/LuceneSQEncoder.java b/src/main/java/org/opensearch/knn/index/engine/lucene/LuceneSQEncoder.java new file mode 100644 index 000000000..fac851ea1 --- /dev/null +++ b/src/main/java/org/opensearch/knn/index/engine/lucene/LuceneSQEncoder.java @@ -0,0 +1,46 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.engine.lucene; + +import org.opensearch.knn.index.engine.Encoder; +import org.opensearch.knn.index.engine.MethodComponent; +import org.opensearch.knn.index.engine.Parameter; + +import java.util.List; + +import static org.opensearch.knn.common.KNNConstants.DYNAMIC_CONFIDENCE_INTERVAL; +import static org.opensearch.knn.common.KNNConstants.ENCODER_SQ; +import static org.opensearch.knn.common.KNNConstants.LUCENE_SQ_BITS; +import static org.opensearch.knn.common.KNNConstants.LUCENE_SQ_CONFIDENCE_INTERVAL; +import static org.opensearch.knn.common.KNNConstants.LUCENE_SQ_DEFAULT_BITS; +import static org.opensearch.knn.common.KNNConstants.MAXIMUM_CONFIDENCE_INTERVAL; +import static org.opensearch.knn.common.KNNConstants.MINIMUM_CONFIDENCE_INTERVAL; + +/** + * Lucene scalar quantization encoder + */ +public class LuceneSQEncoder implements Encoder { + private final static List LUCENE_SQ_BITS_SUPPORTED = List.of(7); + private final static MethodComponent METHOD_COMPONENT = MethodComponent.Builder.builder(ENCODER_SQ) + .addParameter( + LUCENE_SQ_CONFIDENCE_INTERVAL, + new Parameter.DoubleParameter( + LUCENE_SQ_CONFIDENCE_INTERVAL, + null, + v -> v == DYNAMIC_CONFIDENCE_INTERVAL || (v >= MINIMUM_CONFIDENCE_INTERVAL && v <= MAXIMUM_CONFIDENCE_INTERVAL) + ) + ) + .addParameter( + LUCENE_SQ_BITS, + new Parameter.IntegerParameter(LUCENE_SQ_BITS, LUCENE_SQ_DEFAULT_BITS, LUCENE_SQ_BITS_SUPPORTED::contains) + ) + .build(); + + @Override + public MethodComponent getMethodComponent() { + return METHOD_COMPONENT; + } +} diff --git a/src/main/java/org/opensearch/knn/index/engine/nmslib/Nmslib.java b/src/main/java/org/opensearch/knn/index/engine/nmslib/Nmslib.java index 2138c1310..d35cc5f6c 100644 --- a/src/main/java/org/opensearch/knn/index/engine/nmslib/Nmslib.java +++ b/src/main/java/org/opensearch/knn/index/engine/nmslib/Nmslib.java @@ -6,51 +6,25 @@ package org.opensearch.knn.index.engine.nmslib; import com.google.common.collect.ImmutableMap; -import org.opensearch.knn.index.KNNSettings; import org.opensearch.knn.index.SpaceType; -import org.opensearch.knn.index.engine.DefaultHnswContext; import org.opensearch.knn.index.engine.KNNMethod; -import org.opensearch.knn.index.engine.MethodComponent; import org.opensearch.knn.index.engine.NativeLibrary; -import org.opensearch.knn.index.engine.Parameter; import java.util.Collections; import java.util.Map; import java.util.function.Function; import static org.opensearch.knn.common.KNNConstants.METHOD_HNSW; -import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_EF_CONSTRUCTION; -import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_M; /** * Implements NativeLibrary for the nmslib native library */ public class Nmslib extends NativeLibrary { - // Extension to be used for Nmslib files. It is ".hnsw" and not ".nmslib" for legacy purposes. public final static String EXTENSION = ".hnsw"; - final static String CURRENT_VERSION = "2011"; - final static Map METHODS = ImmutableMap.of( - METHOD_HNSW, - KNNMethod.Builder.builder( - MethodComponent.Builder.builder(METHOD_HNSW) - .addParameter( - METHOD_PARAMETER_M, - new Parameter.IntegerParameter(METHOD_PARAMETER_M, KNNSettings.INDEX_KNN_DEFAULT_ALGO_PARAM_M, v -> v > 0) - ) - .addParameter( - METHOD_PARAMETER_EF_CONSTRUCTION, - new Parameter.IntegerParameter( - METHOD_PARAMETER_EF_CONSTRUCTION, - KNNSettings.INDEX_KNN_DEFAULT_ALGO_PARAM_EF_CONSTRUCTION, - v -> v > 0 - ) - ) - .build() - ).addSpaces(SpaceType.UNDEFINED, SpaceType.L2, SpaceType.L1, SpaceType.LINF, SpaceType.COSINESIMIL, SpaceType.INNER_PRODUCT).build() - ); + final static Map METHODS = ImmutableMap.of(METHOD_HNSW, new NmslibHNSWMethod()); public final static Nmslib INSTANCE = new Nmslib(METHODS, Collections.emptyMap(), CURRENT_VERSION, EXTENSION); @@ -68,7 +42,7 @@ private Nmslib( String currentVersion, String extension ) { - super(methods, Map.of(METHOD_HNSW, new DefaultHnswContext()), scoreTranslation, currentVersion, extension); + super(methods, scoreTranslation, currentVersion, extension); } @Override diff --git a/src/main/java/org/opensearch/knn/index/engine/nmslib/NmslibHNSWMethod.java b/src/main/java/org/opensearch/knn/index/engine/nmslib/NmslibHNSWMethod.java new file mode 100644 index 000000000..39c5d5f24 --- /dev/null +++ b/src/main/java/org/opensearch/knn/index/engine/nmslib/NmslibHNSWMethod.java @@ -0,0 +1,61 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.engine.nmslib; + +import org.opensearch.knn.index.KNNSettings; +import org.opensearch.knn.index.SpaceType; +import org.opensearch.knn.index.engine.AbstractKNNMethod; +import org.opensearch.knn.index.engine.DefaultHnswContext; +import org.opensearch.knn.index.engine.MethodComponent; +import org.opensearch.knn.index.engine.Parameter; + +import java.util.Arrays; +import java.util.List; +import java.util.Set; + +import static org.opensearch.knn.common.KNNConstants.METHOD_HNSW; +import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_EF_CONSTRUCTION; +import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_M; + +/** + * Nmslib's HNSW implementation + */ +public class NmslibHNSWMethod extends AbstractKNNMethod { + + public final static List SUPPORTED_SPACES = Arrays.asList( + SpaceType.UNDEFINED, + SpaceType.L2, + SpaceType.L1, + SpaceType.LINF, + SpaceType.COSINESIMIL, + SpaceType.INNER_PRODUCT + ); + + /** + * Constructor. Builds the method with the default parameters and supported spaces. + * @see AbstractKNNMethod + */ + public NmslibHNSWMethod() { + super(initMethodComponent(), Set.copyOf(SUPPORTED_SPACES), new DefaultHnswContext()); + } + + private static MethodComponent initMethodComponent() { + return MethodComponent.Builder.builder(METHOD_HNSW) + .addParameter( + METHOD_PARAMETER_M, + new Parameter.IntegerParameter(METHOD_PARAMETER_M, KNNSettings.INDEX_KNN_DEFAULT_ALGO_PARAM_M, v -> v > 0) + ) + .addParameter( + METHOD_PARAMETER_EF_CONSTRUCTION, + new Parameter.IntegerParameter( + METHOD_PARAMETER_EF_CONSTRUCTION, + KNNSettings.INDEX_KNN_DEFAULT_ALGO_PARAM_EF_CONSTRUCTION, + v -> v > 0 + ) + ) + .build(); + } +} diff --git a/src/test/java/org/opensearch/knn/KNNTestCase.java b/src/test/java/org/opensearch/knn/KNNTestCase.java index 565feb398..2d7bb6d2c 100644 --- a/src/test/java/org/opensearch/knn/KNNTestCase.java +++ b/src/test/java/org/opensearch/knn/KNNTestCase.java @@ -12,6 +12,7 @@ import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; import org.opensearch.knn.index.KNNSettings; +import org.opensearch.knn.index.engine.EngineSpecificMethodContext; import org.opensearch.knn.index.memory.NativeMemoryCacheManager; import org.opensearch.knn.plugin.stats.KNNCounter; import org.opensearch.core.common.bytes.BytesReference; @@ -31,6 +32,8 @@ */ public class KNNTestCase extends OpenSearchTestCase { + protected static final EngineSpecificMethodContext EMPTY_ENGINE_SPECIFIC_CONTEXT = ctx -> Map.of(); + @Mock protected ClusterService clusterService; private AutoCloseable openMocks; diff --git a/src/test/java/org/opensearch/knn/index/FaissHNSWFlatE2EIT.java b/src/test/java/org/opensearch/knn/index/FaissHNSWFlatE2EIT.java index f616248c3..db982566c 100644 --- a/src/test/java/org/opensearch/knn/index/FaissHNSWFlatE2EIT.java +++ b/src/test/java/org/opensearch/knn/index/FaissHNSWFlatE2EIT.java @@ -25,7 +25,6 @@ import org.opensearch.knn.KNNResult; import org.opensearch.knn.TestUtils; import org.opensearch.knn.common.KNNConstants; -import org.opensearch.knn.index.engine.KNNMethod; import org.opensearch.knn.index.query.KNNQueryBuilder; import org.opensearch.knn.index.engine.KNNEngine; import org.opensearch.knn.plugin.script.KNNScoringUtil; @@ -87,8 +86,6 @@ public static Collection parameters() { public void testEndToEnd_whenMethodIsHNSWFlat_thenSucceed() { String indexName = "test-index-1"; String fieldName = "test-field-1"; - - KNNMethod hnswMethod = KNNEngine.FAISS.getMethod(METHOD_HNSW); SpaceType spaceType = SpaceType.L2; List mValues = ImmutableList.of(16, 32, 64, 128); @@ -105,7 +102,7 @@ public void testEndToEnd_whenMethodIsHNSWFlat_thenSucceed() { .field("type", "knn_vector") .field("dimension", dimension) .startObject(KNNConstants.KNN_METHOD) - .field(NAME, hnswMethod.getMethodComponent().getName()) + .field(NAME, METHOD_HNSW) .field(METHOD_PARAMETER_SPACE_TYPE, spaceType.getValue()) .field(KNN_ENGINE, KNNEngine.FAISS.getName()) .startObject(PARAMETERS) diff --git a/src/test/java/org/opensearch/knn/index/FaissIT.java b/src/test/java/org/opensearch/knn/index/FaissIT.java index b4057bec4..72c0b01f4 100644 --- a/src/test/java/org/opensearch/knn/index/FaissIT.java +++ b/src/test/java/org/opensearch/knn/index/FaissIT.java @@ -30,7 +30,6 @@ import org.opensearch.knn.KNNResult; import org.opensearch.knn.TestUtils; import org.opensearch.knn.common.KNNConstants; -import org.opensearch.knn.index.engine.KNNMethod; import org.opensearch.knn.index.query.KNNQueryBuilder; import org.opensearch.knn.index.engine.KNNEngine; import org.opensearch.knn.plugin.script.KNNScoringUtil; @@ -110,7 +109,6 @@ public static void setUpClass() throws IOException { @SneakyThrows public void testEndToEnd_whenDoRadiusSearch_whenDistanceThreshold_whenMethodIsHNSWFlat_thenSucceed() { - KNNMethod hnswMethod = KNNEngine.FAISS.getMethod(METHOD_HNSW); SpaceType spaceType = SpaceType.L2; List mValues = ImmutableList.of(16, 32, 64, 128); @@ -127,7 +125,7 @@ public void testEndToEnd_whenDoRadiusSearch_whenDistanceThreshold_whenMethodIsHN .field("type", "knn_vector") .field("dimension", dimension) .startObject(KNN_METHOD) - .field(NAME, hnswMethod.getMethodComponent().getName()) + .field(NAME, METHOD_HNSW) .field(METHOD_PARAMETER_SPACE_TYPE, spaceType.getValue()) .field(KNN_ENGINE, KNNEngine.FAISS.getName()) .startObject(PARAMETERS) @@ -169,7 +167,6 @@ public void testEndToEnd_whenDoRadiusSearch_whenDistanceThreshold_whenMethodIsHN @SneakyThrows public void testEndToEnd_whenDoRadiusSearch_whenScoreThreshold_whenMethodIsHNSWFlat_thenSucceed() { - KNNMethod hnswMethod = KNNEngine.FAISS.getMethod(METHOD_HNSW); SpaceType spaceType = SpaceType.L2; List mValues = ImmutableList.of(16, 32, 64, 128); @@ -186,7 +183,7 @@ public void testEndToEnd_whenDoRadiusSearch_whenScoreThreshold_whenMethodIsHNSWF .field("type", "knn_vector") .field("dimension", dimension) .startObject(KNN_METHOD) - .field(NAME, hnswMethod.getMethodComponent().getName()) + .field(NAME, METHOD_HNSW) .field(METHOD_PARAMETER_SPACE_TYPE, spaceType.getValue()) .field(KNN_ENGINE, KNNEngine.FAISS.getName()) .startObject(PARAMETERS) @@ -229,7 +226,6 @@ public void testEndToEnd_whenDoRadiusSearch_whenScoreThreshold_whenMethodIsHNSWF @SneakyThrows public void testEndToEnd_whenDoRadiusSearch_whenMoreThanOneScoreThreshold_whenMethodIsHNSWFlat_thenSucceed() { - KNNMethod hnswMethod = KNNEngine.FAISS.getMethod(METHOD_HNSW); SpaceType spaceType = SpaceType.INNER_PRODUCT; List mValues = ImmutableList.of(16, 32, 64, 128); @@ -246,7 +242,7 @@ public void testEndToEnd_whenDoRadiusSearch_whenMoreThanOneScoreThreshold_whenMe .field("type", "knn_vector") .field("dimension", dimension) .startObject(KNN_METHOD) - .field(NAME, hnswMethod.getMethodComponent().getName()) + .field(NAME, METHOD_HNSW) .field(METHOD_PARAMETER_SPACE_TYPE, spaceType.getValue()) .field(KNN_ENGINE, KNNEngine.FAISS.getName()) .startObject(PARAMETERS) @@ -530,8 +526,6 @@ public void testEndToEnd_whenMethodIsHNSWPQ_thenSucceed() { public void testHNSWSQFP16_whenIndexedAndQueried_thenSucceed() { String indexName = "test-index-hnsw-sqfp16"; String fieldName = "test-field-hnsw-sqfp16"; - - KNNMethod hnswMethod = KNNEngine.FAISS.getMethod(METHOD_HNSW); SpaceType[] spaceTypes = { SpaceType.L2, SpaceType.INNER_PRODUCT }; Random random = new Random(); SpaceType spaceType = spaceTypes[random.nextInt(spaceTypes.length)]; @@ -551,7 +545,7 @@ public void testHNSWSQFP16_whenIndexedAndQueried_thenSucceed() { .field("type", "knn_vector") .field("dimension", dimension) .startObject(KNN_METHOD) - .field(NAME, hnswMethod.getMethodComponent().getName()) + .field(NAME, METHOD_HNSW) .field(METHOD_PARAMETER_SPACE_TYPE, spaceType.getValue()) .field(KNN_ENGINE, KNNEngine.FAISS.getName()) .startObject(PARAMETERS) @@ -645,8 +639,6 @@ public void testIVFSQFP16_whenIndexedAndQueried_thenSucceed() { public void testHNSWSQFP16_whenIndexedWithOutOfFP16Range_thenThrowException() { String indexName = "test-index-sqfp16"; String fieldName = "test-field-sqfp16"; - - KNNMethod hnswMethod = KNNEngine.FAISS.getMethod(METHOD_HNSW); SpaceType[] spaceTypes = { SpaceType.L2, SpaceType.INNER_PRODUCT }; Random random = new Random(); SpaceType spaceType = spaceTypes[random.nextInt(spaceTypes.length)]; @@ -665,7 +657,7 @@ public void testHNSWSQFP16_whenIndexedWithOutOfFP16Range_thenThrowException() { .field("type", "knn_vector") .field("dimension", dimension) .startObject(KNN_METHOD) - .field(NAME, hnswMethod.getMethodComponent().getName()) + .field(NAME, METHOD_HNSW) .field(METHOD_PARAMETER_SPACE_TYPE, spaceType.getValue()) .field(KNN_ENGINE, KNNEngine.FAISS.getName()) .startObject(PARAMETERS) @@ -748,9 +740,6 @@ public void testHNSWSQFP16_whenClipToFp16isTrueAndIndexedWithOutOfFP16Range_then String indexName = "test-index-sqfp16-clip-fp16"; String fieldName = "test-field-sqfp16"; - KNNMethod hnswMethod = KNNEngine.FAISS.getMethod(METHOD_HNSW); - Random random = new Random(); - List mValues = ImmutableList.of(16, 32, 64, 128); List efConstructionValues = ImmutableList.of(16, 32, 64, 128); List efSearchValues = ImmutableList.of(16, 32, 64, 128); @@ -765,7 +754,7 @@ public void testHNSWSQFP16_whenClipToFp16isTrueAndIndexedWithOutOfFP16Range_then .field("type", "knn_vector") .field("dimension", dimension) .startObject(KNN_METHOD) - .field(NAME, hnswMethod.getMethodComponent().getName()) + .field(NAME, METHOD_HNSW) .field(METHOD_PARAMETER_SPACE_TYPE, SpaceType.L2.getValue()) .field(KNN_ENGINE, KNNEngine.FAISS.getName()) .startObject(PARAMETERS) @@ -1213,8 +1202,6 @@ public void testDocUpdate() throws IOException { String indexName = "test-index-1"; String fieldName = "test-field-1"; Integer dimension = 2; - - KNNMethod hnswMethod = KNNEngine.FAISS.getMethod(METHOD_HNSW); SpaceType spaceType = SpaceType.L2; // Create an index @@ -1225,7 +1212,7 @@ public void testDocUpdate() throws IOException { .field("type", "knn_vector") .field("dimension", dimension) .startObject(KNN_METHOD) - .field(NAME, hnswMethod.getMethodComponent().getName()) + .field(NAME, METHOD_HNSW) .field(METHOD_PARAMETER_SPACE_TYPE, spaceType.getValue()) .field(KNN_ENGINE, KNNEngine.FAISS.getName()) .endObject() @@ -1249,8 +1236,6 @@ public void testDocDeletion() throws IOException { String indexName = "test-index-1"; String fieldName = "test-field-1"; Integer dimension = 2; - - KNNMethod hnswMethod = KNNEngine.FAISS.getMethod(METHOD_HNSW); SpaceType spaceType = SpaceType.L2; // Create an index @@ -1261,7 +1246,7 @@ public void testDocDeletion() throws IOException { .field("type", "knn_vector") .field("dimension", dimension) .startObject(KNN_METHOD) - .field(NAME, hnswMethod.getMethodComponent().getName()) + .field(NAME, METHOD_HNSW) .field(METHOD_PARAMETER_SPACE_TYPE, spaceType.getValue()) .field(KNN_ENGINE, KNNEngine.FAISS.getName()) .endObject() @@ -1437,7 +1422,7 @@ public void testFiltering_whenUsingFaissExactSearchWithIP_thenMatchExpectedScore .field("type", "knn_vector") .field("dimension", 2) .startObject(KNN_METHOD) - .field(NAME, KNNEngine.FAISS.getMethod(METHOD_HNSW).getMethodComponent().getName()) + .field(NAME, METHOD_HNSW) .field(METHOD_PARAMETER_SPACE_TYPE, SpaceType.INNER_PRODUCT.getValue()) .field(KNN_ENGINE, KNNEngine.FAISS.getName()) .endObject() @@ -1627,7 +1612,7 @@ public void testIVF_whenBinaryFormat_whenIVF_thenSuccess() { .field("dimension", dimension) .field("data_type", VectorDataType.BINARY.getValue()) .startObject(KNN_METHOD) - .field(NAME, KNNEngine.FAISS.getMethod(METHOD_HNSW).getMethodComponent().getName()) + .field(NAME, METHOD_HNSW) .field(METHOD_PARAMETER_SPACE_TYPE, SpaceType.HAMMING.getValue()) .field(KNN_ENGINE, KNNEngine.FAISS.getName()) .startObject(PARAMETERS) @@ -1730,7 +1715,7 @@ public void testQueryWithFilter_whenNonExistingFieldUsedInFilter_thenSuccessful( .field(TYPE_FIELD_NAME, KNN_VECTOR_TYPE) .field(DIMENSION_FIELD_NAME, VECTOR_DIMENSION) .startObject(KNNConstants.KNN_METHOD) - .field(KNNConstants.NAME, KNNEngine.FAISS.getMethod(METHOD_HNSW).getMethodComponent().getName()) + .field(KNNConstants.NAME, METHOD_HNSW) .field(KNNConstants.METHOD_PARAMETER_SPACE_TYPE, SpaceType.L2.getValue()) .field(KNNConstants.KNN_ENGINE, KNNEngine.LUCENE.getName()) .endObject() @@ -1801,7 +1786,7 @@ protected void setupKNNIndexForFilterQuery() throws Exception { .field("type", "knn_vector") .field("dimension", 3) .startObject(KNN_METHOD) - .field(NAME, KNNEngine.FAISS.getMethod(METHOD_HNSW).getMethodComponent().getName()) + .field(NAME, METHOD_HNSW) .field(METHOD_PARAMETER_SPACE_TYPE, SpaceType.L2) .field(KNN_ENGINE, KNNEngine.FAISS.getName()) .endObject() diff --git a/src/test/java/org/opensearch/knn/index/LuceneEngineIT.java b/src/test/java/org/opensearch/knn/index/LuceneEngineIT.java index a39327b8f..64e5af1e7 100644 --- a/src/test/java/org/opensearch/knn/index/LuceneEngineIT.java +++ b/src/test/java/org/opensearch/knn/index/LuceneEngineIT.java @@ -199,7 +199,7 @@ public void testAddDoc() throws Exception { .field(TYPE_FIELD_NAME, KNN_VECTOR_TYPE) .field(DIMENSION_FIELD_NAME, DIMENSION) .startObject(KNNConstants.KNN_METHOD) - .field(KNNConstants.NAME, KNNEngine.LUCENE.getMethod(METHOD_HNSW).getMethodComponent().getName()) + .field(KNNConstants.NAME, METHOD_HNSW) .field(KNNConstants.METHOD_PARAMETER_SPACE_TYPE, SpaceType.L2.getValue()) .field(KNNConstants.KNN_ENGINE, KNNEngine.LUCENE.getName()) .startObject(KNNConstants.PARAMETERS) @@ -291,7 +291,7 @@ public void testQueryWithFilter_whenNonExistingFieldUsedInFilter_thenSuccessful( .field(TYPE_FIELD_NAME, KNN_VECTOR_TYPE) .field(DIMENSION_FIELD_NAME, DIMENSION) .startObject(KNNConstants.KNN_METHOD) - .field(KNNConstants.NAME, KNNEngine.LUCENE.getMethod(METHOD_HNSW).getMethodComponent().getName()) + .field(KNNConstants.NAME, METHOD_HNSW) .field(KNNConstants.METHOD_PARAMETER_SPACE_TYPE, SpaceType.L2.getValue()) .field(KNNConstants.KNN_ENGINE, KNNEngine.LUCENE.getName()) .endObject() @@ -718,7 +718,7 @@ private void createKnnIndexMappingWithLuceneEngineAndSQEncoder( .field(DIMENSION_FIELD_NAME, dimension) .field(VECTOR_DATA_TYPE_FIELD, vectorDataType) .startObject(KNNConstants.KNN_METHOD) - .field(KNNConstants.NAME, KNNEngine.LUCENE.getMethod(METHOD_HNSW).getMethodComponent().getName()) + .field(KNNConstants.NAME, METHOD_HNSW) .field(KNNConstants.METHOD_PARAMETER_SPACE_TYPE, spaceType.getValue()) .field(KNNConstants.KNN_ENGINE, KNNEngine.LUCENE.getName()) .startObject(KNNConstants.PARAMETERS) @@ -750,7 +750,7 @@ private void createKnnIndexMappingWithLuceneEngine(int dimension, SpaceType spac .field(DIMENSION_FIELD_NAME, dimension) .field(VECTOR_DATA_TYPE_FIELD, vectorDataType) .startObject(KNNConstants.KNN_METHOD) - .field(KNNConstants.NAME, KNNEngine.LUCENE.getMethod(METHOD_HNSW).getMethodComponent().getName()) + .field(KNNConstants.NAME, METHOD_HNSW) .field(KNNConstants.METHOD_PARAMETER_SPACE_TYPE, spaceType.getValue()) .field(KNNConstants.KNN_ENGINE, KNNEngine.LUCENE.getName()) .startObject(KNNConstants.PARAMETERS) @@ -856,7 +856,7 @@ public void test_whenUsingIP_thenSuccess() { .field("type", "knn_vector") .field("dimension", 2) .startObject(KNNConstants.KNN_METHOD) - .field(KNNConstants.NAME, KNNEngine.LUCENE.getMethod(KNNConstants.METHOD_HNSW).getMethodComponent().getName()) + .field(KNNConstants.NAME, METHOD_HNSW) .field(KNNConstants.METHOD_PARAMETER_SPACE_TYPE, SpaceType.INNER_PRODUCT.getValue()) .field(KNNConstants.KNN_ENGINE, KNNEngine.LUCENE) .endObject() diff --git a/src/test/java/org/opensearch/knn/index/NmslibIT.java b/src/test/java/org/opensearch/knn/index/NmslibIT.java index 31272d89c..113f84b3a 100644 --- a/src/test/java/org/opensearch/knn/index/NmslibIT.java +++ b/src/test/java/org/opensearch/knn/index/NmslibIT.java @@ -25,7 +25,6 @@ import org.opensearch.knn.KNNResult; import org.opensearch.knn.TestUtils; import org.opensearch.knn.common.KNNConstants; -import org.opensearch.knn.index.engine.KNNMethod; import org.opensearch.knn.index.query.KNNQueryBuilder; import org.opensearch.knn.index.engine.KNNEngine; import org.opensearch.knn.plugin.script.KNNScoringUtil; @@ -58,7 +57,6 @@ public void testInvalidMethodParameters() throws Exception { String indexName = "test-index-1"; String fieldName = "test-field-1"; Integer dimension = testData.indexData.vectors[0].length; - KNNMethod hnswMethod = KNNEngine.NMSLIB.getMethod(KNNConstants.METHOD_HNSW); SpaceType spaceType = SpaceType.L1; // Create an index @@ -69,7 +67,7 @@ public void testInvalidMethodParameters() throws Exception { .field("type", "knn_vector") .field("dimension", dimension) .startObject(KNNConstants.KNN_METHOD) - .field(KNNConstants.NAME, hnswMethod.getMethodComponent().getName()) + .field(KNNConstants.NAME, KNNConstants.METHOD_HNSW) .field(KNNConstants.METHOD_PARAMETER_SPACE_TYPE, spaceType.getValue()) .field(KNNConstants.KNN_ENGINE, KNNEngine.NMSLIB.getName()) .startObject(KNNConstants.PARAMETERS) @@ -127,8 +125,6 @@ public void testInvalidMethodParameters() throws Exception { public void testEndToEnd() throws Exception { String indexName = "test-index-1"; String fieldName = "test-field-1"; - - KNNMethod hnswMethod = KNNEngine.NMSLIB.getMethod(KNNConstants.METHOD_HNSW); SpaceType spaceType = SpaceType.L1; List mValues = ImmutableList.of(16, 32, 64, 128); @@ -144,7 +140,7 @@ public void testEndToEnd() throws Exception { .field("type", "knn_vector") .field("dimension", dimension) .startObject(KNNConstants.KNN_METHOD) - .field(KNNConstants.NAME, hnswMethod.getMethodComponent().getName()) + .field(KNNConstants.NAME, KNNConstants.METHOD_HNSW) .field(KNNConstants.METHOD_PARAMETER_SPACE_TYPE, spaceType.getValue()) .field(KNNConstants.KNN_ENGINE, KNNEngine.NMSLIB.getName()) .startObject(KNNConstants.PARAMETERS) diff --git a/src/test/java/org/opensearch/knn/index/OpenSearchIT.java b/src/test/java/org/opensearch/knn/index/OpenSearchIT.java index f7c262e3c..81a5ab142 100644 --- a/src/test/java/org/opensearch/knn/index/OpenSearchIT.java +++ b/src/test/java/org/opensearch/knn/index/OpenSearchIT.java @@ -28,7 +28,6 @@ import org.opensearch.index.query.ExistsQueryBuilder; import org.opensearch.knn.TestUtils; import org.opensearch.knn.common.KNNConstants; -import org.opensearch.knn.index.engine.KNNMethod; import org.opensearch.knn.index.query.KNNQueryBuilder; import org.opensearch.knn.index.engine.KNNEngine; import org.opensearch.knn.plugin.script.KNNScoringUtil; @@ -65,9 +64,6 @@ public void testEndToEnd() throws Exception { KNNEngine knnEngine2 = KNNEngine.FAISS; String fieldName1 = "test-field-1"; String fieldName2 = "test-field-2"; - - KNNMethod method1 = knnEngine1.getMethod(KNNConstants.METHOD_HNSW); - KNNMethod method2 = knnEngine2.getMethod(KNNConstants.METHOD_HNSW); SpaceType spaceType1 = SpaceType.COSINESIMIL; SpaceType spaceType2 = SpaceType.L2; @@ -85,7 +81,7 @@ public void testEndToEnd() throws Exception { .field("type", "knn_vector") .field("dimension", dimension) .startObject(KNNConstants.KNN_METHOD) - .field(KNNConstants.NAME, method1.getMethodComponent().getName()) + .field(KNNConstants.NAME, KNNConstants.METHOD_HNSW) .field(KNNConstants.METHOD_PARAMETER_SPACE_TYPE, spaceType1.getValue()) .field(KNNConstants.KNN_ENGINE, knnEngine1.getName()) .startObject(KNNConstants.PARAMETERS) @@ -98,7 +94,7 @@ public void testEndToEnd() throws Exception { .field("type", "knn_vector") .field("dimension", dimension) .startObject(KNNConstants.KNN_METHOD) - .field(KNNConstants.NAME, method2.getMethodComponent().getName()) + .field(KNNConstants.NAME, KNNConstants.METHOD_HNSW) .field(KNNConstants.METHOD_PARAMETER_SPACE_TYPE, spaceType2.getValue()) .field(KNNConstants.KNN_ENGINE, knnEngine2.getName()) .startObject(KNNConstants.PARAMETERS) diff --git a/src/test/java/org/opensearch/knn/index/engine/AbstractKNNLibraryTests.java b/src/test/java/org/opensearch/knn/index/engine/AbstractKNNLibraryTests.java index 9a3c8f4df..5dd185112 100644 --- a/src/test/java/org/opensearch/knn/index/engine/AbstractKNNLibraryTests.java +++ b/src/test/java/org/opensearch/knn/index/engine/AbstractKNNLibraryTests.java @@ -18,125 +18,94 @@ import java.util.Collections; import java.util.HashMap; import java.util.Map; +import java.util.Set; import static org.opensearch.knn.common.KNNConstants.NAME; public class AbstractKNNLibraryTests extends KNNTestCase { - public void testGetVersion() { - String testVersion = "test-version"; - TestAbstractKNNLibrary testAbstractKNNLibrary = new TestAbstractKNNLibrary(Collections.emptyMap(), testVersion); - assertEquals(testVersion, testAbstractKNNLibrary.getVersion()); - } - - public void testGetMethod() { - String methodName1 = "test-method-1"; - KNNMethod knnMethod1 = KNNMethod.Builder.builder(MethodComponent.Builder.builder(methodName1).build()).build(); - - String methodName2 = "test-method-2"; - KNNMethod knnMethod2 = KNNMethod.Builder.builder(MethodComponent.Builder.builder(methodName2).build()).build(); - - Map knnMethodMap = ImmutableMap.of(methodName1, knnMethod1, methodName2, knnMethod2); + private final static String CURRENT_VERSION = "test-version"; + private final static String INVALID_METHOD_THROWS_VALIDATION_NAME = "test-method-1"; + private final static KNNMethod INVALID_METHOD_THROWS_VALIDATION = new AbstractKNNMethod( + MethodComponent.Builder.builder(INVALID_METHOD_THROWS_VALIDATION_NAME).build(), + Set.of(SpaceType.DEFAULT), + new DefaultHnswContext() + ) { + @Override + public ValidationException validate(KNNMethodContext knnMethodContext) { + return new ValidationException(); + } + }; + private final static String VALID_METHOD_NAME = "test-method-2"; + private final static EngineSpecificMethodContext VALID_METHOD_CONTEXT = ctx -> ImmutableMap.of( + "myparameter", + new Parameter.BooleanParameter("myparameter", null, value -> true) + ); + private final static Map VALID_EXPECTED_MAP = ImmutableMap.of("test-key", "test-param"); + private final static KNNMethod VALID_METHOD = new AbstractKNNMethod( + MethodComponent.Builder.builder(VALID_METHOD_NAME) + .setMapGenerator((methodComponent, methodComponentContext) -> VALID_EXPECTED_MAP) + .build(), + Set.of(SpaceType.DEFAULT), + VALID_METHOD_CONTEXT + ) { + }; + private final static AbstractKNNLibrary TEST_LIBRARY = new TestAbstractKNNLibrary( + ImmutableMap.of(INVALID_METHOD_THROWS_VALIDATION_NAME, INVALID_METHOD_THROWS_VALIDATION, VALID_METHOD_NAME, VALID_METHOD), + CURRENT_VERSION + ); - TestAbstractKNNLibrary testAbstractKNNLibrary = new TestAbstractKNNLibrary(knnMethodMap, ""); - assertEquals(knnMethod1, testAbstractKNNLibrary.getMethod(methodName1)); - assertEquals(knnMethod2, testAbstractKNNLibrary.getMethod(methodName2)); - expectThrows(IllegalArgumentException.class, () -> testAbstractKNNLibrary.getMethod("invalid")); + public void testGetVersion() { + assertEquals(CURRENT_VERSION, TEST_LIBRARY.getVersion()); } public void testValidateMethod() throws IOException { // Invalid - method not supported - String methodName1 = "test-method-1"; - KNNMethod knnMethod1 = KNNMethod.Builder.builder(MethodComponent.Builder.builder(methodName1).build()).build(); - - Map methodMap = ImmutableMap.of(methodName1, knnMethod1); - TestAbstractKNNLibrary testAbstractKNNLibrary1 = new TestAbstractKNNLibrary(methodMap, ""); - XContentBuilder xContentBuilder = XContentFactory.jsonBuilder().startObject().field(NAME, "invalid").endObject(); Map in = xContentBuilderToMap(xContentBuilder); KNNMethodContext knnMethodContext1 = KNNMethodContext.parse(in); - expectThrows(IllegalArgumentException.class, () -> testAbstractKNNLibrary1.validateMethod(knnMethodContext1)); + expectThrows(IllegalArgumentException.class, () -> TEST_LIBRARY.validateMethod(knnMethodContext1)); // Invalid - method validation - String methodName2 = "test-method-2"; - KNNMethod knnMethod2 = new KNNMethod(MethodComponent.Builder.builder(methodName2).build(), Collections.emptySet()) { - @Override - public ValidationException validate(KNNMethodContext knnMethodContext) { - return new ValidationException(); - } - }; - - methodMap = ImmutableMap.of(methodName2, knnMethod2); - TestAbstractKNNLibrary testAbstractKNNLibrary2 = new TestAbstractKNNLibrary(methodMap, ""); - xContentBuilder = XContentFactory.jsonBuilder().startObject().field(NAME, methodName2).endObject(); + xContentBuilder = XContentFactory.jsonBuilder().startObject().field(NAME, INVALID_METHOD_THROWS_VALIDATION_NAME).endObject(); in = xContentBuilderToMap(xContentBuilder); KNNMethodContext knnMethodContext2 = KNNMethodContext.parse(in); - assertNotNull(testAbstractKNNLibrary2.validateMethod(knnMethodContext2)); + assertNotNull(TEST_LIBRARY.validateMethod(knnMethodContext2)); } public void testEngineSpecificMethods() { - String methodName1 = "test-method-1"; QueryContext engineSpecificMethodContext = new QueryContext(VectorQueryType.K); - EngineSpecificMethodContext context = ctx -> ImmutableMap.of( - "myparameter", - new Parameter.BooleanParameter("myparameter", null, value -> true) - ); - - TestAbstractKNNLibrary testAbstractKNNLibrary1 = new TestAbstractKNNLibrary( - Collections.emptyMap(), - Map.of(methodName1, context), - "" - ); - - assertNotNull(testAbstractKNNLibrary1.getMethodContext(methodName1)); + assertNotNull(TEST_LIBRARY.getMethodContext(VALID_METHOD_NAME)); assertTrue( - testAbstractKNNLibrary1.getMethodContext(methodName1) + TEST_LIBRARY.getMethodContext(VALID_METHOD_NAME) .supportedMethodParameters(engineSpecificMethodContext) .containsKey("myparameter") ); } public void testGetMethodAsMap() { - String methodName = "test-method-1"; - SpaceType spaceType = SpaceType.DEFAULT; - Map generatedMap = ImmutableMap.of("test-key", "test-param"); - MethodComponent methodComponent = MethodComponent.Builder.builder(methodName) - .setMapGenerator(((methodComponent1, methodComponentContext) -> generatedMap)) - .build(); - KNNMethod knnMethod = KNNMethod.Builder.builder(methodComponent).build(); - - TestAbstractKNNLibrary testAbstractKNNLibrary = new TestAbstractKNNLibrary(ImmutableMap.of(methodName, knnMethod), ""); - // Check that map is expected - Map expectedMap = new HashMap<>(generatedMap); - expectedMap.put(KNNConstants.SPACE_TYPE, spaceType.getValue()); + Map expectedMap = new HashMap<>(VALID_EXPECTED_MAP); + expectedMap.put(KNNConstants.SPACE_TYPE, SpaceType.DEFAULT.getValue()); KNNMethodContext knnMethodContext = new KNNMethodContext( KNNEngine.DEFAULT, - spaceType, - new MethodComponentContext(methodName, Collections.emptyMap()) + SpaceType.DEFAULT, + new MethodComponentContext(VALID_METHOD_NAME, Collections.emptyMap()) ); - assertEquals(expectedMap, testAbstractKNNLibrary.getMethodAsMap(knnMethodContext)); + assertEquals(expectedMap, TEST_LIBRARY.getMethodAsMap(knnMethodContext)); // Check when invalid method is passed in KNNMethodContext invalidKnnMethodContext = new KNNMethodContext( KNNEngine.DEFAULT, - spaceType, + SpaceType.DEFAULT, new MethodComponentContext("invalid", Collections.emptyMap()) ); - expectThrows(IllegalArgumentException.class, () -> testAbstractKNNLibrary.getMethodAsMap(invalidKnnMethodContext)); + expectThrows(IllegalArgumentException.class, () -> TEST_LIBRARY.getMethodAsMap(invalidKnnMethodContext)); } private static class TestAbstractKNNLibrary extends AbstractKNNLibrary { public TestAbstractKNNLibrary(Map methods, String currentVersion) { - super(methods, Collections.emptyMap(), currentVersion); - } - - public TestAbstractKNNLibrary( - Map methods, - Map engineSpecificMethodContextMap, - String currentVersion - ) { - super(methods, engineSpecificMethodContextMap, currentVersion); + super(methods, currentVersion); } @Override diff --git a/src/test/java/org/opensearch/knn/index/engine/KNNMethodTests.java b/src/test/java/org/opensearch/knn/index/engine/AbstractKNNMethodTests.java similarity index 76% rename from src/test/java/org/opensearch/knn/index/engine/KNNMethodTests.java rename to src/test/java/org/opensearch/knn/index/engine/AbstractKNNMethodTests.java index 020033f21..981024e6c 100644 --- a/src/test/java/org/opensearch/knn/index/engine/KNNMethodTests.java +++ b/src/test/java/org/opensearch/knn/index/engine/AbstractKNNMethodTests.java @@ -16,19 +16,22 @@ import java.io.IOException; import java.util.HashMap; import java.util.Map; +import java.util.Set; import static org.opensearch.knn.common.KNNConstants.NAME; import static org.opensearch.knn.common.KNNConstants.PARAMETERS; import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_SPACE_TYPE; -public class KNNMethodTests extends KNNTestCase { - /** - * Test KNNMethod method component getter - */ - public void testGetMethodComponent() { - String name = "test"; - KNNMethod knnMethod = KNNMethod.Builder.builder(MethodComponent.Builder.builder(name).build()).build(); - assertEquals(name, knnMethod.getMethodComponent().getName()); +public class AbstractKNNMethodTests extends KNNTestCase { + + private static class TestKNNMethod extends AbstractKNNMethod { + public TestKNNMethod( + MethodComponent methodComponent, + Set spaces, + EngineSpecificMethodContext engineSpecificMethodContext + ) { + super(methodComponent, spaces, engineSpecificMethodContext); + } } /** @@ -36,9 +39,11 @@ public void testGetMethodComponent() { */ public void testHasSpace() { String name = "test"; - KNNMethod knnMethod = KNNMethod.Builder.builder(MethodComponent.Builder.builder(name).build()) - .addSpaces(SpaceType.L2, SpaceType.COSINESIMIL) - .build(); + KNNMethod knnMethod = new TestKNNMethod( + MethodComponent.Builder.builder(name).build(), + Set.of(SpaceType.L2, SpaceType.COSINESIMIL), + EMPTY_ENGINE_SPECIFIC_CONTEXT + ); assertTrue(knnMethod.isSpaceTypeSupported(SpaceType.L2)); assertTrue(knnMethod.isSpaceTypeSupported(SpaceType.COSINESIMIL)); assertFalse(knnMethod.isSpaceTypeSupported(SpaceType.INNER_PRODUCT)); @@ -49,9 +54,11 @@ public void testHasSpace() { */ public void testValidate() throws IOException { String methodName = "test-method"; - KNNMethod knnMethod = KNNMethod.Builder.builder(MethodComponent.Builder.builder(methodName).build()) - .addSpaces(SpaceType.L2) - .build(); + KNNMethod knnMethod = new TestKNNMethod( + MethodComponent.Builder.builder(methodName).build(), + Set.of(SpaceType.L2), + EMPTY_ENGINE_SPECIFIC_CONTEXT + ); // Invalid space XContentBuilder xContentBuilder = XContentFactory.jsonBuilder() @@ -93,9 +100,11 @@ public void testValidate() throws IOException { */ public void testValidateWithData() throws IOException { String methodName = "test-method"; - KNNMethod knnMethod = KNNMethod.Builder.builder(MethodComponent.Builder.builder(methodName).build()) - .addSpaces(SpaceType.L2) - .build(); + KNNMethod knnMethod = new TestKNNMethod( + MethodComponent.Builder.builder(methodName).build(), + Set.of(SpaceType.L2), + EMPTY_ENGINE_SPECIFIC_CONTEXT + ); VectorSpaceInfo testVectorSpaceInfo = new VectorSpaceInfo(4); @@ -142,7 +151,7 @@ public void testGetAsMap() { .setMapGenerator(((methodComponent1, methodComponentContext) -> methodComponentContext.getParameters())) .build(); - KNNMethod knnMethod = KNNMethod.Builder.builder(methodComponent).build(); + KNNMethod knnMethod = new TestKNNMethod(methodComponent, Set.of(SpaceType.L2), EMPTY_ENGINE_SPECIFIC_CONTEXT); Map expectedMap = new HashMap<>(generatedMap); expectedMap.put(KNNConstants.SPACE_TYPE, spaceType.getValue()); @@ -153,16 +162,14 @@ public void testGetAsMap() { ); } - public void testBuilder() { - String name = "test"; - KNNMethod.Builder builder = KNNMethod.Builder.builder(MethodComponent.Builder.builder(name).build()); - KNNMethod knnMethod = builder.build(); - - assertEquals(name, knnMethod.getMethodComponent().getName()); - - builder.addSpaces(SpaceType.L2); - knnMethod = builder.build(); - - assertTrue(knnMethod.isSpaceTypeSupported(SpaceType.L2)); + public void testGetEngineSpecificMethodContext() { + String methodName = "test-method"; + EngineSpecificMethodContext engineSpecificMethodContext = new DefaultHnswContext(); + KNNMethod knnMethod = new TestKNNMethod( + MethodComponent.Builder.builder(methodName).build(), + Set.of(SpaceType.L2), + engineSpecificMethodContext + ); + assertEquals(engineSpecificMethodContext, knnMethod.getEngineSpecificMethodContext()); } } diff --git a/src/test/java/org/opensearch/knn/index/engine/NativeLibraryTests.java b/src/test/java/org/opensearch/knn/index/engine/NativeLibraryTests.java index 02322c4b0..243e9a3c1 100644 --- a/src/test/java/org/opensearch/knn/index/engine/NativeLibraryTests.java +++ b/src/test/java/org/opensearch/knn/index/engine/NativeLibraryTests.java @@ -61,7 +61,7 @@ public TestNativeLibrary( String currentVersion, String extension ) { - super(methods, Collections.emptyMap(), scoreTranslation, currentVersion, extension); + super(methods, scoreTranslation, currentVersion, extension); } @Override diff --git a/src/test/java/org/opensearch/knn/index/engine/faiss/FaissTests.java b/src/test/java/org/opensearch/knn/index/engine/faiss/FaissTests.java index e9033007b..366f4ec77 100644 --- a/src/test/java/org/opensearch/knn/index/engine/faiss/FaissTests.java +++ b/src/test/java/org/opensearch/knn/index/engine/faiss/FaissTests.java @@ -234,7 +234,7 @@ public void testMethodAsMapBuilder() throws IOException { expectedMap.put(NAME, methodName); expectedMap.put(INDEX_DESCRIPTION_PARAMETER, methodDescription + value1); - Map methodAsMap = Faiss.MethodAsMapBuilder.builder(methodDescription, methodComponent, methodComponentContext) + Map methodAsMap = MethodAsMapBuilder.builder(methodDescription, methodComponent, methodComponentContext) .addParameter(parameter1, "", "") .build(); diff --git a/src/test/java/org/opensearch/knn/index/engine/lucene/LuceneTests.java b/src/test/java/org/opensearch/knn/index/engine/lucene/LuceneTests.java index 976bb4238..d149f4b3b 100644 --- a/src/test/java/org/opensearch/knn/index/engine/lucene/LuceneTests.java +++ b/src/test/java/org/opensearch/knn/index/engine/lucene/LuceneTests.java @@ -9,7 +9,7 @@ import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.knn.KNNTestCase; -import org.opensearch.knn.index.engine.KNNMethod; +import org.opensearch.knn.index.engine.KNNEngine; import org.opensearch.knn.index.engine.KNNMethodContext; import org.opensearch.knn.index.SpaceType; @@ -28,9 +28,6 @@ public class LuceneTests extends KNNTestCase { public void testLucenHNSWMethod() throws IOException { - KNNMethod luceneHNSW = Lucene.INSTANCE.getMethod(METHOD_HNSW); - assertNotNull(luceneHNSW); - int efConstruction = 100; int m = 17; XContentBuilder xContentBuilder = XContentFactory.jsonBuilder() @@ -44,7 +41,7 @@ public void testLucenHNSWMethod() throws IOException { .endObject(); Map in = xContentBuilderToMap(xContentBuilder); KNNMethodContext knnMethodContext1 = KNNMethodContext.parse(in); - assertNull(luceneHNSW.validate(knnMethodContext1)); + assertNull(KNNEngine.LUCENE.validateMethod(knnMethodContext1)); // Invalid parameter String invalidParameter = "invalid"; @@ -57,7 +54,7 @@ public void testLucenHNSWMethod() throws IOException { .endObject(); in = xContentBuilderToMap(xContentBuilder); KNNMethodContext knnMethodContext2 = KNNMethodContext.parse(in); - assertNotNull(luceneHNSW.validate(knnMethodContext2)); + assertNotNull(KNNEngine.LUCENE.validateMethod(knnMethodContext2)); // Valid parameter, invalid value int invalidEfConstruction = -1; @@ -70,7 +67,7 @@ public void testLucenHNSWMethod() throws IOException { .endObject(); in = xContentBuilderToMap(xContentBuilder); KNNMethodContext knnMethodContext3 = KNNMethodContext.parse(in); - assertNotNull(luceneHNSW.validate(knnMethodContext3)); + assertNotNull(KNNEngine.LUCENE.validateMethod(knnMethodContext3)); // Unsupported space type SpaceType invalidSpaceType = SpaceType.LINF; // Not currently supported @@ -81,7 +78,7 @@ public void testLucenHNSWMethod() throws IOException { .endObject(); in = xContentBuilderToMap(xContentBuilder); KNNMethodContext knnMethodContext4 = KNNMethodContext.parse(in); - assertNotNull(luceneHNSW.validate(knnMethodContext4)); + assertNotNull(KNNEngine.LUCENE.validateMethod(knnMethodContext4)); // Check INNER_PRODUCT is supported with Lucene Engine xContentBuilder = XContentFactory.jsonBuilder() @@ -95,7 +92,7 @@ public void testLucenHNSWMethod() throws IOException { .endObject(); in = xContentBuilderToMap(xContentBuilder); KNNMethodContext knnMethodContext5 = KNNMethodContext.parse(in); - assertNull(luceneHNSW.validate(knnMethodContext5)); + assertNull(KNNEngine.LUCENE.validateMethod(knnMethodContext5)); } public void testGetExtension() { diff --git a/src/test/java/org/opensearch/knn/jni/JNIServiceTests.java b/src/test/java/org/opensearch/knn/jni/JNIServiceTests.java index de3a3fcb2..12ae1d444 100644 --- a/src/test/java/org/opensearch/knn/jni/JNIServiceTests.java +++ b/src/test/java/org/opensearch/knn/jni/JNIServiceTests.java @@ -23,6 +23,7 @@ import org.opensearch.knn.common.KNNConstants; import org.opensearch.knn.index.engine.KNNMethodContext; import org.opensearch.knn.index.VectorDataType; +import org.opensearch.knn.index.engine.nmslib.NmslibHNSWMethod; import org.opensearch.knn.index.query.KNNQueryResult; import org.opensearch.knn.index.engine.MethodComponentContext; import org.opensearch.knn.index.SpaceType; @@ -265,7 +266,7 @@ public void testCreateIndex_nmslib_invalid_badParameterType() throws IOException public void testCreateIndex_nmslib_valid() throws IOException { - for (SpaceType spaceType : KNNEngine.NMSLIB.getMethod(KNNConstants.METHOD_HNSW).getSpaces()) { + for (SpaceType spaceType : NmslibHNSWMethod.SUPPORTED_SPACES) { if (SpaceType.UNDEFINED == spaceType) { continue; } @@ -814,7 +815,7 @@ public void testQueryIndex_nmslib_invalid_nullQueryVector() throws IOException { public void testQueryIndex_nmslib_valid() throws IOException { int k = 50; - for (SpaceType spaceType : KNNEngine.NMSLIB.getMethod(KNNConstants.METHOD_HNSW).getSpaces()) { + for (SpaceType spaceType : NmslibHNSWMethod.SUPPORTED_SPACES) { if (SpaceType.UNDEFINED == spaceType) { continue; } diff --git a/src/test/java/org/opensearch/knn/plugin/stats/suppliers/LibraryInitializedSupplierTests.java b/src/test/java/org/opensearch/knn/plugin/stats/suppliers/LibraryInitializedSupplierTests.java index 3a413c595..8dd4de81e 100644 --- a/src/test/java/org/opensearch/knn/plugin/stats/suppliers/LibraryInitializedSupplierTests.java +++ b/src/test/java/org/opensearch/knn/plugin/stats/suppliers/LibraryInitializedSupplierTests.java @@ -13,7 +13,6 @@ import org.opensearch.common.ValidationException; import org.opensearch.knn.index.engine.EngineSpecificMethodContext; -import org.opensearch.knn.index.engine.KNNMethod; import org.opensearch.knn.index.engine.KNNMethodContext; import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.engine.KNNLibrary; @@ -55,11 +54,6 @@ public String getCompoundExtension() { return null; } - @Override - public KNNMethod getMethod(String methodName) { - return null; - } - @Override public EngineSpecificMethodContext getMethodContext(String methodName) { return null; From 0d2945e435f1e6a14380e949a68f748e0ab8fe29 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Fri, 2 Aug 2024 11:07:56 -0700 Subject: [PATCH 315/416] Fix a flaky unit test:testMultiFieldsKnnIndex, which was failing due to inconsistent merge behaviors (#1924) (#1926) Signed-off-by: Navneet Verma (cherry picked from commit ea8a6352d622a007c563b539da66929f67a842f6) Co-authored-by: Navneet Verma --- CHANGELOG.md | 1 + .../opensearch/knn/index/codec/KNNCodecTestCase.java | 11 +++++------ 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 575d5b6bd..ef3addee5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Infrastructure ### Documentation ### Maintenance +* Fix a flaky unit test:testMultiFieldsKnnIndex, which was failing due to inconsistent merge behaviors [#1924](https://github.com/opensearch-project/k-NN/pull/1924) ### Refactoring * Introduce KNNVectorValues interface to iterate on different types of Vector values during indexing and search [#1897](https://github.com/opensearch-project/k-NN/pull/1897) * Clean up parsing for query [#1824](https://github.com/opensearch-project/k-NN/pull/1824) diff --git a/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestCase.java b/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestCase.java index 05e244fd4..70b055df4 100644 --- a/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestCase.java +++ b/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestCase.java @@ -137,20 +137,19 @@ public void testMultiFieldsKnnIndex(Codec codec) throws Exception { Document doc = new Document(); doc.add(vectorField); writer.addDocument(doc); - writer.close(); + // ensuring the refresh happens, to create the segment and hnsw file + writer.flush(); /** * Add doc with field "my_vector" */ - IndexWriterConfig iwc1 = newIndexWriterConfig(); - iwc1.setMergeScheduler(new SerialMergeScheduler()); - iwc1.setCodec(ACTUAL_CODEC); - writer = new RandomIndexWriter(random(), dir, iwc1); float[] array1 = { 6.0f, 14.0f }; VectorField vectorField1 = new VectorField("my_vector", array1, sampleFieldType); Document doc1 = new Document(); doc1.add(vectorField1); writer.addDocument(doc1); + // ensuring the refresh happens, to create the segment and hnsw file + writer.flush(); IndexReader reader = writer.getReader(); writer.close(); ResourceWatcherService resourceWatcherService = createDisabledResourceWatcherService(); @@ -158,7 +157,7 @@ public void testMultiFieldsKnnIndex(Codec codec) throws Exception { List hnswfiles = Arrays.stream(dir.listAll()).filter(x -> x.contains("hnsw")).collect(Collectors.toList()); // there should be 2 hnsw index files created. one for test_vector and one for my_vector - assertEquals(hnswfiles.size(), 2); + assertEquals(2, hnswfiles.size()); assertEquals(hnswfiles.stream().filter(x -> x.contains("test_vector")).collect(Collectors.toList()).size(), 1); assertEquals(hnswfiles.stream().filter(x -> x.contains("my_vector")).collect(Collectors.toList()).size(), 1); From ec9fe9fe31fffa8b13afe432bd076264c9822e43 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Fri, 2 Aug 2024 14:12:46 -0700 Subject: [PATCH 316/416] Generalize lib interface to return context objects (#1927) Generalizes the KNNLibrary to return an object for both search and indexing so that the plugin can search/index against them. This will help properly pass information that does not need to be sent to the JNI for search and index builds. Signed-off-by: John Mazanec (cherry picked from commit a16b8aa965f66268799c001756c78174c7baba19) --- CHANGELOG.md | 1 + .../knn/index/engine/AbstractKNNLibrary.java | 20 +++++++------- .../knn/index/engine/AbstractKNNMethod.java | 10 +++---- ...ext.java => DefaultHnswSearchContext.java} | 2 +- ...text.java => DefaultIVFSearchContext.java} | 2 +- .../engine/EngineSpecificMethodContext.java | 25 ----------------- .../knn/index/engine/KNNEngine.java | 14 +++++----- .../knn/index/engine/KNNLibrary.java | 22 +++++++-------- .../engine/KNNLibraryIndexingContext.java | 23 ++++++++++++++++ .../engine/KNNLibraryIndexingContextImpl.java | 24 +++++++++++++++++ .../index/engine/KNNLibrarySearchContext.java | 27 +++++++++++++++++++ .../knn/index/engine/KNNMethod.java | 16 +++++------ .../index/engine/faiss/FaissHNSWMethod.java | 4 +-- .../index/engine/faiss/FaissIVFMethod.java | 4 +-- .../index/engine/lucene/LuceneHNSWMethod.java | 2 +- ...text.java => LuceneHNSWSearchContext.java} | 4 +-- .../index/engine/nmslib/NmslibHNSWMethod.java | 4 +-- .../knn/index/mapper/MethodFieldMapper.java | 7 +++-- .../knn/index/query/KNNQueryBuilder.java | 4 +-- .../opensearch/knn/training/TrainingJob.java | 5 +++- .../java/org/opensearch/knn/KNNTestCase.java | 4 +-- .../KNN80DocValuesConsumerTests.java | 12 ++++++--- .../index/engine/AbstractKNNLibraryTests.java | 14 +++++----- .../index/engine/AbstractKNNMethodTests.java | 20 +++++++------- .../knn/index/engine/faiss/FaissTests.java | 12 ++++----- .../opensearch/knn/jni/JNIServiceTests.java | 20 +++++++++----- .../LibraryInitializedSupplierTests.java | 9 +++---- 27 files changed, 186 insertions(+), 125 deletions(-) rename src/main/java/org/opensearch/knn/index/engine/{DefaultHnswContext.java => DefaultHnswSearchContext.java} (91%) rename src/main/java/org/opensearch/knn/index/engine/{DefaultIVFContext.java => DefaultIVFSearchContext.java} (90%) delete mode 100644 src/main/java/org/opensearch/knn/index/engine/EngineSpecificMethodContext.java create mode 100644 src/main/java/org/opensearch/knn/index/engine/KNNLibraryIndexingContext.java create mode 100644 src/main/java/org/opensearch/knn/index/engine/KNNLibraryIndexingContextImpl.java create mode 100644 src/main/java/org/opensearch/knn/index/engine/KNNLibrarySearchContext.java rename src/main/java/org/opensearch/knn/index/engine/lucene/{LuceneHNSWContext.java => LuceneHNSWSearchContext.java} (88%) diff --git a/CHANGELOG.md b/CHANGELOG.md index ef3addee5..f7431e9b7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,3 +26,4 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), * Clean up parsing for query [#1824](https://github.com/opensearch-project/k-NN/pull/1824) * Refactor engine package structure [#1913](https://github.com/opensearch-project/k-NN/pull/1913) * Refactor method structure and definitions [#1920](https://github.com/opensearch-project/k-NN/pull/1920) +* Generalize lib interface to return context objects [#1925](https://github.com/opensearch-project/k-NN/pull/1925) \ No newline at end of file diff --git a/src/main/java/org/opensearch/knn/index/engine/AbstractKNNLibrary.java b/src/main/java/org/opensearch/knn/index/engine/AbstractKNNLibrary.java index 9d83f42a8..92e34be7c 100644 --- a/src/main/java/org/opensearch/knn/index/engine/AbstractKNNLibrary.java +++ b/src/main/java/org/opensearch/knn/index/engine/AbstractKNNLibrary.java @@ -24,10 +24,18 @@ public abstract class AbstractKNNLibrary implements KNNLibrary { protected final String version; @Override - public EngineSpecificMethodContext getMethodContext(String methodName) { + public KNNLibrarySearchContext getKNNLibrarySearchContext(String methodName) { validateMethodExists(methodName); KNNMethod method = methods.get(methodName); - return method.getEngineSpecificMethodContext(); + return method.getKNNLibrarySearchContext(); + } + + @Override + public KNNLibraryIndexingContext getKNNLibraryIndexingContext(KNNMethodContext knnMethodContext) { + String method = knnMethodContext.getMethodComponentContext().getName(); + validateMethodExists(method); + KNNMethod knnMethod = methods.get(method); + return knnMethod.getKNNLibraryIndexingContext(knnMethodContext); } @Override @@ -51,14 +59,6 @@ public boolean isTrainingRequired(KNNMethodContext knnMethodContext) { return methods.get(methodName).isTrainingRequired(knnMethodContext); } - @Override - public Map getMethodAsMap(KNNMethodContext knnMethodContext) { - String method = knnMethodContext.getMethodComponentContext().getName(); - validateMethodExists(method); - KNNMethod knnMethod = methods.get(method); - return knnMethod.getAsMap(knnMethodContext); - } - private void validateMethodExists(String methodName) { KNNMethod method = methods.get(methodName); if (method == null) { diff --git a/src/main/java/org/opensearch/knn/index/engine/AbstractKNNMethod.java b/src/main/java/org/opensearch/knn/index/engine/AbstractKNNMethod.java index cbed5fe40..6e57e6913 100644 --- a/src/main/java/org/opensearch/knn/index/engine/AbstractKNNMethod.java +++ b/src/main/java/org/opensearch/knn/index/engine/AbstractKNNMethod.java @@ -27,7 +27,7 @@ public abstract class AbstractKNNMethod implements KNNMethod { protected final MethodComponent methodComponent; protected final Set spaces; - protected final EngineSpecificMethodContext engineSpecificMethodContext; + protected final KNNLibrarySearchContext knnLibrarySearchContext; @Override public boolean isSpaceTypeSupported(SpaceType space) { @@ -106,14 +106,14 @@ public int estimateOverheadInKB(KNNMethodContext knnMethodContext, int dimension } @Override - public Map getAsMap(KNNMethodContext knnMethodContext) { + public KNNLibraryIndexingContext getKNNLibraryIndexingContext(KNNMethodContext knnMethodContext) { Map parameterMap = new HashMap<>(methodComponent.getAsMap(knnMethodContext.getMethodComponentContext())); parameterMap.put(KNNConstants.SPACE_TYPE, knnMethodContext.getSpaceType().getValue()); - return parameterMap; + return KNNLibraryIndexingContextImpl.builder().parameters(parameterMap).build(); } @Override - public EngineSpecificMethodContext getEngineSpecificMethodContext() { - return engineSpecificMethodContext; + public KNNLibrarySearchContext getKNNLibrarySearchContext() { + return knnLibrarySearchContext; } } diff --git a/src/main/java/org/opensearch/knn/index/engine/DefaultHnswContext.java b/src/main/java/org/opensearch/knn/index/engine/DefaultHnswSearchContext.java similarity index 91% rename from src/main/java/org/opensearch/knn/index/engine/DefaultHnswContext.java rename to src/main/java/org/opensearch/knn/index/engine/DefaultHnswSearchContext.java index ef1d960d4..ecc11f338 100644 --- a/src/main/java/org/opensearch/knn/index/engine/DefaultHnswContext.java +++ b/src/main/java/org/opensearch/knn/index/engine/DefaultHnswSearchContext.java @@ -14,7 +14,7 @@ /** * Default HNSW context for all engines. Have a different implementation if engine context differs. */ -public final class DefaultHnswContext implements EngineSpecificMethodContext { +public final class DefaultHnswSearchContext implements KNNLibrarySearchContext { private final Map> supportedMethodParameters = ImmutableMap.>builder() .put(MethodParameter.EF_SEARCH.getName(), new Parameter.IntegerParameter(MethodParameter.EF_SEARCH.getName(), null, value -> true)) diff --git a/src/main/java/org/opensearch/knn/index/engine/DefaultIVFContext.java b/src/main/java/org/opensearch/knn/index/engine/DefaultIVFSearchContext.java similarity index 90% rename from src/main/java/org/opensearch/knn/index/engine/DefaultIVFContext.java rename to src/main/java/org/opensearch/knn/index/engine/DefaultIVFSearchContext.java index a1a474420..cc612bf8c 100644 --- a/src/main/java/org/opensearch/knn/index/engine/DefaultIVFContext.java +++ b/src/main/java/org/opensearch/knn/index/engine/DefaultIVFSearchContext.java @@ -11,7 +11,7 @@ import java.util.Map; -public final class DefaultIVFContext implements EngineSpecificMethodContext { +public final class DefaultIVFSearchContext implements KNNLibrarySearchContext { private final Map> supportedMethodParameters = ImmutableMap.>builder() .put(MethodParameter.NPROBE.getName(), new Parameter.IntegerParameter(MethodParameter.NPROBE.getName(), null, value -> true)) diff --git a/src/main/java/org/opensearch/knn/index/engine/EngineSpecificMethodContext.java b/src/main/java/org/opensearch/knn/index/engine/EngineSpecificMethodContext.java deleted file mode 100644 index a043bd9cd..000000000 --- a/src/main/java/org/opensearch/knn/index/engine/EngineSpecificMethodContext.java +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.knn.index.engine; - -import org.opensearch.knn.index.engine.model.QueryContext; - -import java.util.Collections; -import java.util.Map; - -/** - * Holds context related to a method for a particular engine - * Each engine can have a specific set of parameters that it supports during index and build time. This context holds - * the information for each engine method combination. - * - * TODO: Move KnnMethod in here - */ -public interface EngineSpecificMethodContext { - - Map> supportedMethodParameters(QueryContext ctx); - - EngineSpecificMethodContext EMPTY = ctx -> Collections.emptyMap(); -} diff --git a/src/main/java/org/opensearch/knn/index/engine/KNNEngine.java b/src/main/java/org/opensearch/knn/index/engine/KNNEngine.java index f2f2bab35..c7b271783 100644 --- a/src/main/java/org/opensearch/knn/index/engine/KNNEngine.java +++ b/src/main/java/org/opensearch/knn/index/engine/KNNEngine.java @@ -145,11 +145,6 @@ public String getCompoundExtension() { return knnLibrary.getCompoundExtension(); } - @Override - public EngineSpecificMethodContext getMethodContext(String methodName) { - return knnLibrary.getMethodContext(methodName); - } - @Override public float score(float rawScore, SpaceType spaceType) { return knnLibrary.score(rawScore, spaceType); @@ -181,8 +176,13 @@ public boolean isTrainingRequired(KNNMethodContext knnMethodContext) { } @Override - public Map getMethodAsMap(KNNMethodContext knnMethodContext) { - return knnLibrary.getMethodAsMap(knnMethodContext); + public KNNLibraryIndexingContext getKNNLibraryIndexingContext(KNNMethodContext knnMethodContext) { + return knnLibrary.getKNNLibraryIndexingContext(knnMethodContext); + } + + @Override + public KNNLibrarySearchContext getKNNLibrarySearchContext(String methodName) { + return knnLibrary.getKNNLibrarySearchContext(methodName); } @Override diff --git a/src/main/java/org/opensearch/knn/index/engine/KNNLibrary.java b/src/main/java/org/opensearch/knn/index/engine/KNNLibrary.java index c47f39e03..96d492307 100644 --- a/src/main/java/org/opensearch/knn/index/engine/KNNLibrary.java +++ b/src/main/java/org/opensearch/knn/index/engine/KNNLibrary.java @@ -11,7 +11,6 @@ import java.util.Collections; import java.util.List; -import java.util.Map; /** * KNNLibrary is an interface that helps the plugin communicate with k-NN libraries @@ -41,13 +40,6 @@ public interface KNNLibrary { */ String getCompoundExtension(); - /** - * Gets metadata related to methods supported by the library - * @param methodName - * @return - */ - EngineSpecificMethodContext getMethodContext(String methodName); - /** * Generate the Lucene score from the rawScore returned by the library. With k-NN, often times the library * will return a score where the lower the score, the better the result. This is the opposite of how Lucene scores @@ -116,12 +108,20 @@ public interface KNNLibrary { int estimateOverheadInKB(KNNMethodContext knnMethodContext, int dimension); /** - * Generate method as map that can be used to configure the knn index from the jni + * Get the context from the library needed to build the index. * - * @param knnMethodContext to generate parameter map from + * @param knnMethodContext to get build context for * @return parameter map */ - Map getMethodAsMap(KNNMethodContext knnMethodContext); + KNNLibraryIndexingContext getKNNLibraryIndexingContext(KNNMethodContext knnMethodContext); + + /** + * Gets metadata related to methods supported by the library + * + * @param methodName name of method + * @return KNNLibrarySearchContext + */ + KNNLibrarySearchContext getKNNLibrarySearchContext(String methodName); /** * Getter for initialized diff --git a/src/main/java/org/opensearch/knn/index/engine/KNNLibraryIndexingContext.java b/src/main/java/org/opensearch/knn/index/engine/KNNLibraryIndexingContext.java new file mode 100644 index 000000000..d00b7c436 --- /dev/null +++ b/src/main/java/org/opensearch/knn/index/engine/KNNLibraryIndexingContext.java @@ -0,0 +1,23 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.engine; + +import java.util.Collections; +import java.util.Map; + +/** + * Context a library gives to build one of its indices + */ +public interface KNNLibraryIndexingContext { + /** + * Get map of parameters that get passed to the library to build the index + * + * @return Map of parameters + */ + Map getLibraryParameters(); + + KNNLibraryIndexingContext EMPTY = Collections::emptyMap; +} diff --git a/src/main/java/org/opensearch/knn/index/engine/KNNLibraryIndexingContextImpl.java b/src/main/java/org/opensearch/knn/index/engine/KNNLibraryIndexingContextImpl.java new file mode 100644 index 000000000..b7c775261 --- /dev/null +++ b/src/main/java/org/opensearch/knn/index/engine/KNNLibraryIndexingContextImpl.java @@ -0,0 +1,24 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.engine; + +import lombok.Builder; + +import java.util.Map; + +/** + * Simple implementation of {@link KNNLibraryIndexingContext} + */ +@Builder +public class KNNLibraryIndexingContextImpl implements KNNLibraryIndexingContext { + + private Map parameters; + + @Override + public Map getLibraryParameters() { + return parameters; + } +} diff --git a/src/main/java/org/opensearch/knn/index/engine/KNNLibrarySearchContext.java b/src/main/java/org/opensearch/knn/index/engine/KNNLibrarySearchContext.java new file mode 100644 index 000000000..b769745f6 --- /dev/null +++ b/src/main/java/org/opensearch/knn/index/engine/KNNLibrarySearchContext.java @@ -0,0 +1,27 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.engine; + +import org.opensearch.knn.index.engine.model.QueryContext; + +import java.util.Collections; +import java.util.Map; + +/** + * Holds the context needed to search a knn library. + */ +public interface KNNLibrarySearchContext { + + /** + * Returns supported parameters for the library. + * + * @param ctx QueryContext + * @return parameters supported by the library + */ + Map> supportedMethodParameters(QueryContext ctx); + + KNNLibrarySearchContext EMPTY = ctx -> Collections.emptyMap(); +} diff --git a/src/main/java/org/opensearch/knn/index/engine/KNNMethod.java b/src/main/java/org/opensearch/knn/index/engine/KNNMethod.java index ea556e8bf..326e5c1e0 100644 --- a/src/main/java/org/opensearch/knn/index/engine/KNNMethod.java +++ b/src/main/java/org/opensearch/knn/index/engine/KNNMethod.java @@ -9,8 +9,6 @@ import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.training.VectorSpaceInfo; -import java.util.Map; - /** * KNNMethod defines the structure of a method supported by a particular k-NN library. It is used to validate * the KNNMethodContext passed in by the user, where the KNNMethodContext provides the configuration that the user may @@ -61,17 +59,17 @@ public interface KNNMethod { int estimateOverheadInKB(KNNMethodContext knnMethodContext, int dimension); /** - * Parse knnMethodContext into a map that the library can use to configure the index + * Parse knnMethodContext into context that the library can use to build the index * - * @param knnMethodContext from which to generate map - * @return KNNMethod as a map + * @param knnMethodContext to generate the context for + * @return KNNLibraryIndexingContext */ - Map getAsMap(KNNMethodContext knnMethodContext); + KNNLibraryIndexingContext getKNNLibraryIndexingContext(KNNMethodContext knnMethodContext); /** - * Get the method context for a particular method + * Get the search context for a particular method * - * @return EngineSpecificMethodContext for the method + * @return KNNLibrarySearchContext */ - EngineSpecificMethodContext getEngineSpecificMethodContext(); + KNNLibrarySearchContext getKNNLibrarySearchContext(); } diff --git a/src/main/java/org/opensearch/knn/index/engine/faiss/FaissHNSWMethod.java b/src/main/java/org/opensearch/knn/index/engine/faiss/FaissHNSWMethod.java index ac05afc33..382a71741 100644 --- a/src/main/java/org/opensearch/knn/index/engine/faiss/FaissHNSWMethod.java +++ b/src/main/java/org/opensearch/knn/index/engine/faiss/FaissHNSWMethod.java @@ -9,7 +9,7 @@ import org.opensearch.knn.index.KNNSettings; import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.engine.AbstractKNNMethod; -import org.opensearch.knn.index.engine.DefaultHnswContext; +import org.opensearch.knn.index.engine.DefaultHnswSearchContext; import org.opensearch.knn.index.engine.Encoder; import org.opensearch.knn.index.engine.MethodComponent; import org.opensearch.knn.index.engine.MethodComponentContext; @@ -51,7 +51,7 @@ public class FaissHNSWMethod extends AbstractKNNMethod { * @see AbstractKNNMethod */ public FaissHNSWMethod() { - super(initMethodComponent(), Set.copyOf(SUPPORTED_SPACES), new DefaultHnswContext()); + super(initMethodComponent(), Set.copyOf(SUPPORTED_SPACES), new DefaultHnswSearchContext()); } private static MethodComponent initMethodComponent() { diff --git a/src/main/java/org/opensearch/knn/index/engine/faiss/FaissIVFMethod.java b/src/main/java/org/opensearch/knn/index/engine/faiss/FaissIVFMethod.java index 813cb6e9e..aa05e8c87 100644 --- a/src/main/java/org/opensearch/knn/index/engine/faiss/FaissIVFMethod.java +++ b/src/main/java/org/opensearch/knn/index/engine/faiss/FaissIVFMethod.java @@ -8,7 +8,7 @@ import org.opensearch.knn.common.KNNConstants; import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.engine.AbstractKNNMethod; -import org.opensearch.knn.index.engine.DefaultIVFContext; +import org.opensearch.knn.index.engine.DefaultIVFSearchContext; import org.opensearch.knn.index.engine.Encoder; import org.opensearch.knn.index.engine.MethodComponent; import org.opensearch.knn.index.engine.MethodComponentContext; @@ -55,7 +55,7 @@ public class FaissIVFMethod extends AbstractKNNMethod { * @see AbstractKNNMethod */ public FaissIVFMethod() { - super(initMethodComponent(), Set.copyOf(SUPPORTED_SPACES), new DefaultIVFContext()); + super(initMethodComponent(), Set.copyOf(SUPPORTED_SPACES), new DefaultIVFSearchContext()); } private static MethodComponent initMethodComponent() { diff --git a/src/main/java/org/opensearch/knn/index/engine/lucene/LuceneHNSWMethod.java b/src/main/java/org/opensearch/knn/index/engine/lucene/LuceneHNSWMethod.java index 0419a5440..c6fcdb7c4 100644 --- a/src/main/java/org/opensearch/knn/index/engine/lucene/LuceneHNSWMethod.java +++ b/src/main/java/org/opensearch/knn/index/engine/lucene/LuceneHNSWMethod.java @@ -49,7 +49,7 @@ public class LuceneHNSWMethod extends AbstractKNNMethod { * @see AbstractKNNMethod */ public LuceneHNSWMethod() { - super(initMethodComponent(), Set.copyOf(SUPPORTED_SPACES), new LuceneHNSWContext()); + super(initMethodComponent(), Set.copyOf(SUPPORTED_SPACES), new LuceneHNSWSearchContext()); } private static MethodComponent initMethodComponent() { diff --git a/src/main/java/org/opensearch/knn/index/engine/lucene/LuceneHNSWContext.java b/src/main/java/org/opensearch/knn/index/engine/lucene/LuceneHNSWSearchContext.java similarity index 88% rename from src/main/java/org/opensearch/knn/index/engine/lucene/LuceneHNSWContext.java rename to src/main/java/org/opensearch/knn/index/engine/lucene/LuceneHNSWSearchContext.java index 808aa66b0..2c4da27df 100644 --- a/src/main/java/org/opensearch/knn/index/engine/lucene/LuceneHNSWContext.java +++ b/src/main/java/org/opensearch/knn/index/engine/lucene/LuceneHNSWSearchContext.java @@ -6,7 +6,7 @@ package org.opensearch.knn.index.engine.lucene; import com.google.common.collect.ImmutableMap; -import org.opensearch.knn.index.engine.EngineSpecificMethodContext; +import org.opensearch.knn.index.engine.KNNLibrarySearchContext; import org.opensearch.knn.index.engine.Parameter; import org.opensearch.knn.index.engine.model.QueryContext; import org.opensearch.knn.index.query.request.MethodParameter; @@ -14,7 +14,7 @@ import java.util.Collections; import java.util.Map; -public class LuceneHNSWContext implements EngineSpecificMethodContext { +public class LuceneHNSWSearchContext implements KNNLibrarySearchContext { private final Map> supportedMethodParameters = ImmutableMap.>builder() .put(MethodParameter.EF_SEARCH.getName(), new Parameter.IntegerParameter(MethodParameter.EF_SEARCH.getName(), null, value -> true)) diff --git a/src/main/java/org/opensearch/knn/index/engine/nmslib/NmslibHNSWMethod.java b/src/main/java/org/opensearch/knn/index/engine/nmslib/NmslibHNSWMethod.java index 39c5d5f24..e8e27bcd6 100644 --- a/src/main/java/org/opensearch/knn/index/engine/nmslib/NmslibHNSWMethod.java +++ b/src/main/java/org/opensearch/knn/index/engine/nmslib/NmslibHNSWMethod.java @@ -8,7 +8,7 @@ import org.opensearch.knn.index.KNNSettings; import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.engine.AbstractKNNMethod; -import org.opensearch.knn.index.engine.DefaultHnswContext; +import org.opensearch.knn.index.engine.DefaultHnswSearchContext; import org.opensearch.knn.index.engine.MethodComponent; import org.opensearch.knn.index.engine.Parameter; @@ -39,7 +39,7 @@ public class NmslibHNSWMethod extends AbstractKNNMethod { * @see AbstractKNNMethod */ public NmslibHNSWMethod() { - super(initMethodComponent(), Set.copyOf(SUPPORTED_SPACES), new DefaultHnswContext()); + super(initMethodComponent(), Set.copyOf(SUPPORTED_SPACES), new DefaultHnswSearchContext()); } private static MethodComponent initMethodComponent() { diff --git a/src/main/java/org/opensearch/knn/index/mapper/MethodFieldMapper.java b/src/main/java/org/opensearch/knn/index/mapper/MethodFieldMapper.java index dba37c927..b15ab1489 100644 --- a/src/main/java/org/opensearch/knn/index/mapper/MethodFieldMapper.java +++ b/src/main/java/org/opensearch/knn/index/mapper/MethodFieldMapper.java @@ -12,6 +12,7 @@ import org.opensearch.knn.index.engine.KNNEngine; import java.io.IOException; +import java.util.Map; import static org.opensearch.knn.common.KNNConstants.DIMENSION; import static org.opensearch.knn.common.KNNConstants.KNN_ENGINE; @@ -58,10 +59,8 @@ public class MethodFieldMapper extends KNNVectorFieldMapper { this.fieldType.putAttribute(KNN_ENGINE, knnEngine.getName()); try { - this.fieldType.putAttribute( - PARAMETERS, - XContentFactory.jsonBuilder().map(knnEngine.getMethodAsMap(knnMethodContext)).toString() - ); + Map libParams = knnEngine.getKNNLibraryIndexingContext(knnMethodContext).getLibraryParameters(); + this.fieldType.putAttribute(PARAMETERS, XContentFactory.jsonBuilder().map(libParams).toString()); } catch (IOException ioe) { throw new RuntimeException(String.format("Unable to create KNNVectorFieldMapper: %s", ioe)); } diff --git a/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java b/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java index f860b282b..b05938c4a 100644 --- a/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java +++ b/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java @@ -32,7 +32,7 @@ import org.opensearch.knn.index.VectorQueryType; import org.opensearch.knn.index.mapper.KNNVectorFieldMapper; import org.opensearch.knn.index.query.parser.KNNQueryBuilderParser; -import org.opensearch.knn.index.engine.EngineSpecificMethodContext; +import org.opensearch.knn.index.engine.KNNLibrarySearchContext; import org.opensearch.knn.index.engine.KNNEngine; import org.opensearch.knn.indices.ModelDao; import org.opensearch.knn.indices.ModelMetadata; @@ -373,7 +373,7 @@ protected Query doToQuery(QueryShardContext context) { final String method = methodComponentContext != null ? methodComponentContext.getName() : null; if (StringUtils.isNotBlank(method)) { - final EngineSpecificMethodContext engineSpecificMethodContext = knnEngine.getMethodContext(method); + final KNNLibrarySearchContext engineSpecificMethodContext = knnEngine.getKNNLibrarySearchContext(method); QueryContext queryContext = new QueryContext(vectorQueryType); ValidationException validationException = validateParameters( engineSpecificMethodContext.supportedMethodParameters(queryContext), diff --git a/src/main/java/org/opensearch/knn/training/TrainingJob.java b/src/main/java/org/opensearch/knn/training/TrainingJob.java index c0cfb72dc..3bdb50ad0 100644 --- a/src/main/java/org/opensearch/knn/training/TrainingJob.java +++ b/src/main/java/org/opensearch/knn/training/TrainingJob.java @@ -182,7 +182,10 @@ public void run() { throw new RuntimeException("Unable to load training data into memory: allocation is already closed"); } setVersionInKnnMethodContext(); - Map trainParameters = model.getModelMetadata().getKnnEngine().getMethodAsMap(knnMethodContext); + Map trainParameters = model.getModelMetadata() + .getKnnEngine() + .getKNNLibraryIndexingContext(knnMethodContext) + .getLibraryParameters(); trainParameters.put( KNNConstants.INDEX_THREAD_QTY, KNNSettings.state().getSettingValue(KNNSettings.KNN_ALGO_PARAM_INDEX_THREAD_QTY) diff --git a/src/test/java/org/opensearch/knn/KNNTestCase.java b/src/test/java/org/opensearch/knn/KNNTestCase.java index 2d7bb6d2c..56c129546 100644 --- a/src/test/java/org/opensearch/knn/KNNTestCase.java +++ b/src/test/java/org/opensearch/knn/KNNTestCase.java @@ -12,7 +12,7 @@ import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; import org.opensearch.knn.index.KNNSettings; -import org.opensearch.knn.index.engine.EngineSpecificMethodContext; +import org.opensearch.knn.index.engine.KNNLibrarySearchContext; import org.opensearch.knn.index.memory.NativeMemoryCacheManager; import org.opensearch.knn.plugin.stats.KNNCounter; import org.opensearch.core.common.bytes.BytesReference; @@ -32,7 +32,7 @@ */ public class KNNTestCase extends OpenSearchTestCase { - protected static final EngineSpecificMethodContext EMPTY_ENGINE_SPECIFIC_CONTEXT = ctx -> Map.of(); + protected static final KNNLibrarySearchContext EMPTY_ENGINE_SPECIFIC_CONTEXT = ctx -> Map.of(); @Mock protected ClusterService clusterService; diff --git a/src/test/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumerTests.java b/src/test/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumerTests.java index ce8fad384..e1ebc5708 100644 --- a/src/test/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumerTests.java +++ b/src/test/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumerTests.java @@ -208,7 +208,9 @@ public void testAddKNNBinaryField_fromScratch_nmslibCurrent() throws IOException new MethodComponentContext(METHOD_HNSW, ImmutableMap.of(METHOD_PARAMETER_M, 16, METHOD_PARAMETER_EF_CONSTRUCTION, 512)) ); - String parameterString = XContentFactory.jsonBuilder().map(knnEngine.getMethodAsMap(knnMethodContext)).toString(); + String parameterString = XContentFactory.jsonBuilder() + .map(knnEngine.getKNNLibraryIndexingContext(knnMethodContext).getLibraryParameters()) + .toString(); FieldInfo[] fieldInfoArray = new FieldInfo[] { KNNCodecTestUtil.FieldInfoBuilder.builder(fieldName) @@ -328,7 +330,9 @@ public void testAddKNNBinaryField_fromScratch_faissCurrent() throws IOException ); knnMethodContext.getMethodComponentContext().setIndexVersion(Version.CURRENT); - String parameterString = XContentFactory.jsonBuilder().map(knnEngine.getMethodAsMap(knnMethodContext)).toString(); + String parameterString = XContentFactory.jsonBuilder() + .map(knnEngine.getKNNLibraryIndexingContext(knnMethodContext).getLibraryParameters()) + .toString(); FieldInfo[] fieldInfoArray = new FieldInfo[] { KNNCodecTestUtil.FieldInfoBuilder.builder(fieldName) @@ -393,7 +397,9 @@ public void testAddKNNBinaryField_whenFaissBinary_thenAdded() throws IOException ); knnMethodContext.getMethodComponentContext().setIndexVersion(Version.CURRENT); - String parameterString = XContentFactory.jsonBuilder().map(knnEngine.getMethodAsMap(knnMethodContext)).toString(); + String parameterString = XContentFactory.jsonBuilder() + .map(knnEngine.getKNNLibraryIndexingContext(knnMethodContext).getLibraryParameters()) + .toString(); FieldInfo[] fieldInfoArray = new FieldInfo[] { KNNCodecTestUtil.FieldInfoBuilder.builder(fieldName) diff --git a/src/test/java/org/opensearch/knn/index/engine/AbstractKNNLibraryTests.java b/src/test/java/org/opensearch/knn/index/engine/AbstractKNNLibraryTests.java index 5dd185112..c6ab9ccdb 100644 --- a/src/test/java/org/opensearch/knn/index/engine/AbstractKNNLibraryTests.java +++ b/src/test/java/org/opensearch/knn/index/engine/AbstractKNNLibraryTests.java @@ -29,7 +29,7 @@ public class AbstractKNNLibraryTests extends KNNTestCase { private final static KNNMethod INVALID_METHOD_THROWS_VALIDATION = new AbstractKNNMethod( MethodComponent.Builder.builder(INVALID_METHOD_THROWS_VALIDATION_NAME).build(), Set.of(SpaceType.DEFAULT), - new DefaultHnswContext() + new DefaultHnswSearchContext() ) { @Override public ValidationException validate(KNNMethodContext knnMethodContext) { @@ -37,7 +37,7 @@ public ValidationException validate(KNNMethodContext knnMethodContext) { } }; private final static String VALID_METHOD_NAME = "test-method-2"; - private final static EngineSpecificMethodContext VALID_METHOD_CONTEXT = ctx -> ImmutableMap.of( + private final static KNNLibrarySearchContext VALID_METHOD_CONTEXT = ctx -> ImmutableMap.of( "myparameter", new Parameter.BooleanParameter("myparameter", null, value -> true) ); @@ -75,15 +75,15 @@ public void testValidateMethod() throws IOException { public void testEngineSpecificMethods() { QueryContext engineSpecificMethodContext = new QueryContext(VectorQueryType.K); - assertNotNull(TEST_LIBRARY.getMethodContext(VALID_METHOD_NAME)); + assertNotNull(TEST_LIBRARY.getKNNLibrarySearchContext(VALID_METHOD_NAME)); assertTrue( - TEST_LIBRARY.getMethodContext(VALID_METHOD_NAME) + TEST_LIBRARY.getKNNLibrarySearchContext(VALID_METHOD_NAME) .supportedMethodParameters(engineSpecificMethodContext) .containsKey("myparameter") ); } - public void testGetMethodAsMap() { + public void testGetKNNLibraryIndexingContext() { // Check that map is expected Map expectedMap = new HashMap<>(VALID_EXPECTED_MAP); expectedMap.put(KNNConstants.SPACE_TYPE, SpaceType.DEFAULT.getValue()); @@ -92,7 +92,7 @@ public void testGetMethodAsMap() { SpaceType.DEFAULT, new MethodComponentContext(VALID_METHOD_NAME, Collections.emptyMap()) ); - assertEquals(expectedMap, TEST_LIBRARY.getMethodAsMap(knnMethodContext)); + assertEquals(expectedMap, TEST_LIBRARY.getKNNLibraryIndexingContext(knnMethodContext).getLibraryParameters()); // Check when invalid method is passed in KNNMethodContext invalidKnnMethodContext = new KNNMethodContext( @@ -100,7 +100,7 @@ public void testGetMethodAsMap() { SpaceType.DEFAULT, new MethodComponentContext("invalid", Collections.emptyMap()) ); - expectThrows(IllegalArgumentException.class, () -> TEST_LIBRARY.getMethodAsMap(invalidKnnMethodContext)); + expectThrows(IllegalArgumentException.class, () -> TEST_LIBRARY.getKNNLibraryIndexingContext(invalidKnnMethodContext)); } private static class TestAbstractKNNLibrary extends AbstractKNNLibrary { diff --git a/src/test/java/org/opensearch/knn/index/engine/AbstractKNNMethodTests.java b/src/test/java/org/opensearch/knn/index/engine/AbstractKNNMethodTests.java index 981024e6c..2c739c6f7 100644 --- a/src/test/java/org/opensearch/knn/index/engine/AbstractKNNMethodTests.java +++ b/src/test/java/org/opensearch/knn/index/engine/AbstractKNNMethodTests.java @@ -25,11 +25,7 @@ public class AbstractKNNMethodTests extends KNNTestCase { private static class TestKNNMethod extends AbstractKNNMethod { - public TestKNNMethod( - MethodComponent methodComponent, - Set spaces, - EngineSpecificMethodContext engineSpecificMethodContext - ) { + public TestKNNMethod(MethodComponent methodComponent, Set spaces, KNNLibrarySearchContext engineSpecificMethodContext) { super(methodComponent, spaces, engineSpecificMethodContext); } } @@ -143,7 +139,7 @@ public void testValidateWithData() throws IOException { assertNull(knnMethod.validateWithData(knnMethodContext3, testVectorSpaceInfo)); } - public void testGetAsMap() { + public void testGetKNNLibraryIndexingContext() { SpaceType spaceType = SpaceType.DEFAULT; String methodName = "test-method"; Map generatedMap = ImmutableMap.of("test-key", "test-value"); @@ -158,18 +154,20 @@ public void testGetAsMap() { assertEquals( expectedMap, - knnMethod.getAsMap(new KNNMethodContext(KNNEngine.DEFAULT, spaceType, new MethodComponentContext(methodName, generatedMap))) + knnMethod.getKNNLibraryIndexingContext( + new KNNMethodContext(KNNEngine.DEFAULT, spaceType, new MethodComponentContext(methodName, generatedMap)) + ).getLibraryParameters() ); } - public void testGetEngineSpecificMethodContext() { + public void testGetKNNLibrarySearchContext() { String methodName = "test-method"; - EngineSpecificMethodContext engineSpecificMethodContext = new DefaultHnswContext(); + KNNLibrarySearchContext knnLibrarySearchContext = new DefaultHnswSearchContext(); KNNMethod knnMethod = new TestKNNMethod( MethodComponent.Builder.builder(methodName).build(), Set.of(SpaceType.L2), - engineSpecificMethodContext + knnLibrarySearchContext ); - assertEquals(engineSpecificMethodContext, knnMethod.getEngineSpecificMethodContext()); + assertEquals(knnLibrarySearchContext, knnMethod.getKNNLibrarySearchContext()); } } diff --git a/src/test/java/org/opensearch/knn/index/engine/faiss/FaissTests.java b/src/test/java/org/opensearch/knn/index/engine/faiss/FaissTests.java index 366f4ec77..af5086491 100644 --- a/src/test/java/org/opensearch/knn/index/engine/faiss/FaissTests.java +++ b/src/test/java/org/opensearch/knn/index/engine/faiss/FaissTests.java @@ -55,7 +55,7 @@ public void testGetMethodAsMap_whenMethodIsHNSWFlat_thenCreateCorrectIndexDescri KNNMethodContext knnMethodContext = KNNMethodContext.parse(in); knnMethodContext.getMethodComponentContext().setIndexVersion(Version.CURRENT); - Map map = Faiss.INSTANCE.getMethodAsMap(knnMethodContext); + Map map = Faiss.INSTANCE.getKNNLibraryIndexingContext(knnMethodContext).getLibraryParameters(); assertTrue(map.containsKey(INDEX_DESCRIPTION_PARAMETER)); assertEquals(expectedIndexDescription, map.get(INDEX_DESCRIPTION_PARAMETER)); @@ -84,7 +84,7 @@ public void testGetMethodAsMap_whenMethodIsHNSWPQ_thenCreateCorrectIndexDescript KNNMethodContext knnMethodContext = KNNMethodContext.parse(in); knnMethodContext.getMethodComponentContext().setIndexVersion(Version.CURRENT); - Map map = Faiss.INSTANCE.getMethodAsMap(knnMethodContext); + Map map = Faiss.INSTANCE.getKNNLibraryIndexingContext(knnMethodContext).getLibraryParameters(); assertTrue(map.containsKey(INDEX_DESCRIPTION_PARAMETER)); assertEquals(expectedIndexDescription, map.get(INDEX_DESCRIPTION_PARAMETER)); @@ -113,7 +113,7 @@ public void testGetMethodAsMap_whenMethodIsHNSWSQFP16_thenCreateCorrectIndexDesc KNNMethodContext knnMethodContext = KNNMethodContext.parse(in); knnMethodContext.getMethodComponentContext().setIndexVersion(Version.CURRENT); - Map map = Faiss.INSTANCE.getMethodAsMap(knnMethodContext); + Map map = Faiss.INSTANCE.getKNNLibraryIndexingContext(knnMethodContext).getLibraryParameters(); assertTrue(map.containsKey(INDEX_DESCRIPTION_PARAMETER)); assertEquals(expectedIndexDescription, map.get(INDEX_DESCRIPTION_PARAMETER)); @@ -134,7 +134,7 @@ public void testGetMethodAsMap_whenMethodIsIVFFlat_thenCreateCorrectIndexDescrip Map in = xContentBuilderToMap(xContentBuilder); KNNMethodContext knnMethodContext = KNNMethodContext.parse(in); - Map map = Faiss.INSTANCE.getMethodAsMap(knnMethodContext); + Map map = Faiss.INSTANCE.getKNNLibraryIndexingContext(knnMethodContext).getLibraryParameters(); assertTrue(map.containsKey(INDEX_DESCRIPTION_PARAMETER)); assertEquals(expectedIndexDescription, map.get(INDEX_DESCRIPTION_PARAMETER)); @@ -164,7 +164,7 @@ public void testGetMethodAsMap_whenMethodIsIVFPQ_thenCreateCorrectIndexDescripti Map in = xContentBuilderToMap(xContentBuilder); KNNMethodContext knnMethodContext = KNNMethodContext.parse(in); - Map map = Faiss.INSTANCE.getMethodAsMap(knnMethodContext); + Map map = Faiss.INSTANCE.getKNNLibraryIndexingContext(knnMethodContext).getLibraryParameters(); assertTrue(map.containsKey(INDEX_DESCRIPTION_PARAMETER)); assertEquals(expectedIndexDescription, map.get(INDEX_DESCRIPTION_PARAMETER)); @@ -192,7 +192,7 @@ public void testGetMethodAsMap_whenMethodIsIVFSQFP16_thenCreateCorrectIndexDescr Map in = xContentBuilderToMap(xContentBuilder); KNNMethodContext knnMethodContext = KNNMethodContext.parse(in); - Map map = Faiss.INSTANCE.getMethodAsMap(knnMethodContext); + Map map = Faiss.INSTANCE.getKNNLibraryIndexingContext(knnMethodContext).getLibraryParameters(); assertTrue(map.containsKey(INDEX_DESCRIPTION_PARAMETER)); assertEquals(expectedIndexDescription, map.get(INDEX_DESCRIPTION_PARAMETER)); diff --git a/src/test/java/org/opensearch/knn/jni/JNIServiceTests.java b/src/test/java/org/opensearch/knn/jni/JNIServiceTests.java index 12ae1d444..ae9ad7106 100644 --- a/src/test/java/org/opensearch/knn/jni/JNIServiceTests.java +++ b/src/test/java/org/opensearch/knn/jni/JNIServiceTests.java @@ -611,7 +611,7 @@ public void testTrain_whenConfigurationIsIVFSQFP16_thenSucceed() { .endObject(); Map in = xContentBuilderToMap(xContentBuilder); KNNMethodContext knnMethodContext = KNNMethodContext.parse(in); - Map parameters = KNNEngine.FAISS.getMethodAsMap(knnMethodContext); + Map parameters = KNNEngine.FAISS.getKNNLibraryIndexingContext(knnMethodContext).getLibraryParameters(); byte[] faissIndex = JNIService.trainIndex(parameters, 128, trainPointer, KNNEngine.FAISS); @@ -1131,7 +1131,7 @@ public void testTrain_whenConfigurationIsIVFFlat_thenSucceed() throws IOExceptio .endObject(); Map in = xContentBuilderToMap(xContentBuilder); KNNMethodContext knnMethodContext = KNNMethodContext.parse(in); - Map parameters = KNNEngine.FAISS.getMethodAsMap(knnMethodContext); + Map parameters = KNNEngine.FAISS.getKNNLibraryIndexingContext(knnMethodContext).getLibraryParameters(); byte[] faissIndex = JNIService.trainIndex(parameters, 128, trainPointer, KNNEngine.FAISS); @@ -1162,7 +1162,7 @@ public void testTrain_whenConfigurationIsIVFPQ_thenSucceed() throws IOException .endObject(); Map in = xContentBuilderToMap(xContentBuilder); KNNMethodContext knnMethodContext = KNNMethodContext.parse(in); - Map parameters = KNNEngine.FAISS.getMethodAsMap(knnMethodContext); + Map parameters = KNNEngine.FAISS.getKNNLibraryIndexingContext(knnMethodContext).getLibraryParameters(); byte[] faissIndex = JNIService.trainIndex(parameters, 128, trainPointer, KNNEngine.FAISS); @@ -1190,7 +1190,7 @@ public void testTrain_whenConfigurationIsHNSWPQ_thenSucceed() throws IOException Map in = xContentBuilderToMap(xContentBuilder); KNNMethodContext knnMethodContext = KNNMethodContext.parse(in); knnMethodContext.getMethodComponentContext().setIndexVersion(Version.CURRENT); - Map parameters = KNNEngine.FAISS.getMethodAsMap(knnMethodContext); + Map parameters = KNNEngine.FAISS.getKNNLibraryIndexingContext(knnMethodContext).getLibraryParameters(); byte[] faissIndex = JNIService.trainIndex(parameters, 128, trainPointer, KNNEngine.FAISS); @@ -1237,7 +1237,11 @@ public void testCreateIndexFromTemplate() throws IOException { ) ); - String description = knnMethodContext.getKnnEngine().getMethodAsMap(knnMethodContext).get(INDEX_DESCRIPTION_PARAMETER).toString(); + String description = knnMethodContext.getKnnEngine() + .getKNNLibraryIndexingContext(knnMethodContext) + .getLibraryParameters() + .get(INDEX_DESCRIPTION_PARAMETER) + .toString(); assertEquals("IVF16,PQ16x8", description); Map parameters = ImmutableMap.of( @@ -1375,7 +1379,11 @@ private String createFaissIVFPQIndex(int ivfNlist, int pqM, int pqCodeSize, Spac ) ); - String description = knnMethodContext.getKnnEngine().getMethodAsMap(knnMethodContext).get(INDEX_DESCRIPTION_PARAMETER).toString(); + String description = knnMethodContext.getKnnEngine() + .getKNNLibraryIndexingContext(knnMethodContext) + .getLibraryParameters() + .get(INDEX_DESCRIPTION_PARAMETER) + .toString(); Map parameters = ImmutableMap.of( INDEX_DESCRIPTION_PARAMETER, description, diff --git a/src/test/java/org/opensearch/knn/plugin/stats/suppliers/LibraryInitializedSupplierTests.java b/src/test/java/org/opensearch/knn/plugin/stats/suppliers/LibraryInitializedSupplierTests.java index 8dd4de81e..7fa0d3bca 100644 --- a/src/test/java/org/opensearch/knn/plugin/stats/suppliers/LibraryInitializedSupplierTests.java +++ b/src/test/java/org/opensearch/knn/plugin/stats/suppliers/LibraryInitializedSupplierTests.java @@ -12,15 +12,14 @@ package org.opensearch.knn.plugin.stats.suppliers; import org.opensearch.common.ValidationException; -import org.opensearch.knn.index.engine.EngineSpecificMethodContext; +import org.opensearch.knn.index.engine.KNNLibraryIndexingContext; +import org.opensearch.knn.index.engine.KNNLibrarySearchContext; import org.opensearch.knn.index.engine.KNNMethodContext; import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.engine.KNNLibrary; import org.opensearch.knn.training.VectorSpaceInfo; import org.opensearch.test.OpenSearchTestCase; -import java.util.Map; - public class LibraryInitializedSupplierTests extends OpenSearchTestCase { public void testEngineInitialized() { @@ -55,7 +54,7 @@ public String getCompoundExtension() { } @Override - public EngineSpecificMethodContext getMethodContext(String methodName) { + public KNNLibrarySearchContext getKNNLibrarySearchContext(String methodName) { return null; } @@ -95,7 +94,7 @@ public int estimateOverheadInKB(KNNMethodContext knnMethodContext, int dimension } @Override - public Map getMethodAsMap(KNNMethodContext knnMethodContext) { + public KNNLibraryIndexingContext getKNNLibraryIndexingContext(KNNMethodContext knnMethodContext) { return null; } From 39061a012780cdaeea7de859d94475ca19a89cf7 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Mon, 5 Aug 2024 16:30:02 -0700 Subject: [PATCH 317/416] Refactor KNNVectorFieldType from KNNVectorFieldMapper to a separate class for better readability. (#1931) (#1935) Signed-off-by: Navneet Verma (cherry picked from commit 967b21129121535f1f0b5d7268d2810b5d82fc47) Co-authored-by: Navneet Verma --- CHANGELOG.md | 3 +- .../codec/BasePerFieldKnnVectorsFormat.java | 6 +- .../index/mapper/KNNVectorFieldMapper.java | 103 ---------------- .../mapper/KNNVectorFieldMapperUtil.java | 4 +- .../knn/index/mapper/KNNVectorFieldType.java | 116 ++++++++++++++++++ .../knn/index/query/KNNQueryBuilder.java | 8 +- .../knn/plugin/script/KNNScoringSpace.java | 14 +-- .../plugin/script/KNNScoringSpaceUtil.java | 6 +- .../knn/index/codec/KNNCodecTestCase.java | 15 +-- .../mapper/KNNVectorFieldMapperTests.java | 2 +- .../mapper/KNNVectorFieldMapperUtilTests.java | 8 +- .../index/mapper/MethodFieldMapperTests.java | 2 +- .../knn/index/query/KNNQueryBuilderTests.java | 56 ++++----- .../knn/integ/KNNScriptScoringIT.java | 4 +- .../script/KNNScoringSpaceFactoryTests.java | 10 +- .../plugin/script/KNNScoringSpaceTests.java | 36 ++---- .../script/KNNScoringSpaceUtilTests.java | 10 +- 17 files changed, 192 insertions(+), 211 deletions(-) create mode 100644 src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldType.java diff --git a/CHANGELOG.md b/CHANGELOG.md index f7431e9b7..6c761d27f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,4 +26,5 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), * Clean up parsing for query [#1824](https://github.com/opensearch-project/k-NN/pull/1824) * Refactor engine package structure [#1913](https://github.com/opensearch-project/k-NN/pull/1913) * Refactor method structure and definitions [#1920](https://github.com/opensearch-project/k-NN/pull/1920) -* Generalize lib interface to return context objects [#1925](https://github.com/opensearch-project/k-NN/pull/1925) \ No newline at end of file +* Refactor KNNVectorFieldType from KNNVectorFieldMapper to a separate class for better readability. [#1931](https://github.com/opensearch-project/k-NN/pull/1931) +* Generalize lib interface to return context objects [#1925](https://github.com/opensearch-project/k-NN/pull/1925) diff --git a/src/main/java/org/opensearch/knn/index/codec/BasePerFieldKnnVectorsFormat.java b/src/main/java/org/opensearch/knn/index/codec/BasePerFieldKnnVectorsFormat.java index 2d423c26e..2a3732d7e 100644 --- a/src/main/java/org/opensearch/knn/index/codec/BasePerFieldKnnVectorsFormat.java +++ b/src/main/java/org/opensearch/knn/index/codec/BasePerFieldKnnVectorsFormat.java @@ -12,8 +12,8 @@ import org.opensearch.index.mapper.MapperService; import org.opensearch.knn.index.codec.params.KNNScalarQuantizedVectorsFormatParams; import org.opensearch.knn.index.codec.params.KNNVectorsFormatParams; -import org.opensearch.knn.index.mapper.KNNVectorFieldMapper; import org.opensearch.knn.index.engine.KNNEngine; +import org.opensearch.knn.index.mapper.KNNVectorFieldType; import java.util.Optional; import java.util.function.Function; @@ -66,7 +66,7 @@ public KnnVectorsFormat getKnnVectorsFormatForField(final String field) { ); return defaultFormatSupplier.get(); } - var type = (KNNVectorFieldMapper.KNNVectorFieldType) mapperService.orElseThrow( + var type = (KNNVectorFieldType) mapperService.orElseThrow( () -> new IllegalStateException( String.format("Cannot read field type for field [%s] because mapper service is not available", field) ) @@ -117,6 +117,6 @@ public int getMaxDimensions(String fieldName) { } private boolean isKnnVectorFieldType(final String field) { - return mapperService.isPresent() && mapperService.get().fieldType(field) instanceof KNNVectorFieldMapper.KNNVectorFieldType; + return mapperService.isPresent() && mapperService.get().fieldType(field) instanceof KNNVectorFieldType; } } diff --git a/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapper.java b/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapper.java index 14596189a..3b9487645 100644 --- a/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapper.java +++ b/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapper.java @@ -14,47 +14,33 @@ import java.util.Objects; import java.util.Optional; import java.util.function.Supplier; -import lombok.Getter; import lombok.extern.log4j.Log4j2; import org.apache.lucene.document.Field; import org.apache.lucene.document.FieldType; import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.IndexOptions; -import org.apache.lucene.search.DocValuesFieldExistsQuery; -import org.apache.lucene.search.Query; -import org.apache.lucene.util.BytesRef; import org.opensearch.Version; import org.opensearch.common.Explicit; -import org.opensearch.common.Nullable; import org.opensearch.common.ValidationException; import org.opensearch.common.xcontent.support.XContentMapValues; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; -import org.opensearch.index.fielddata.IndexFieldData; import org.opensearch.index.mapper.FieldMapper; -import org.opensearch.index.mapper.MappedFieldType; import org.opensearch.index.mapper.Mapper; import org.opensearch.index.mapper.MapperParsingException; import org.opensearch.index.mapper.ParametrizedFieldMapper; import org.opensearch.index.mapper.ParseContext; -import org.opensearch.index.mapper.TextSearchInfo; -import org.opensearch.index.mapper.ValueFetcher; -import org.opensearch.index.query.QueryShardContext; -import org.opensearch.index.query.QueryShardException; import org.opensearch.knn.common.KNNConstants; import org.opensearch.knn.index.KnnCircuitBreakerException; import org.opensearch.knn.index.engine.KNNMethodContext; import org.opensearch.knn.index.KNNSettings; -import org.opensearch.knn.index.KNNVectorIndexFieldData; import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.engine.MethodComponentContext; import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.VectorField; import org.opensearch.knn.index.engine.KNNEngine; import org.opensearch.knn.indices.ModelDao; -import org.opensearch.search.aggregations.support.CoreValuesSourceType; -import org.opensearch.search.lookup.SearchLookup; import static org.opensearch.knn.common.KNNConstants.DEFAULT_VECTOR_DATA_TYPE_FIELD; import static org.opensearch.knn.common.KNNConstants.ENCODER_FLAT; @@ -72,7 +58,6 @@ import static org.opensearch.knn.index.mapper.KNNVectorFieldMapperUtil.createStoredFieldForByteVector; import static org.opensearch.knn.index.mapper.KNNVectorFieldMapperUtil.createStoredFieldForFloatVector; import static org.opensearch.knn.index.mapper.KNNVectorFieldMapperUtil.clipVectorValueToFP16Range; -import static org.opensearch.knn.index.mapper.KNNVectorFieldMapperUtil.deserializeStoredVector; import static org.opensearch.knn.index.mapper.KNNVectorFieldMapperUtil.validateFP16VectorValue; import static org.opensearch.knn.index.mapper.KNNVectorFieldMapperUtil.validateVectorDataType; import static org.opensearch.knn.index.mapper.KNNVectorFieldMapperUtil.validateVectorDataTypeWithKnnIndexSetting; @@ -467,94 +452,6 @@ public Mapper.Builder parse(String name, Map node, ParserCont } } - @Getter - public static class KNNVectorFieldType extends MappedFieldType { - int dimension; - String modelId; - KNNMethodContext knnMethodContext; - VectorDataType vectorDataType; - SpaceType spaceType; - - public KNNVectorFieldType( - String name, - Map meta, - int dimension, - VectorDataType vectorDataType, - SpaceType spaceType - ) { - this(name, meta, dimension, null, null, vectorDataType, spaceType); - } - - public KNNVectorFieldType(String name, Map meta, int dimension, KNNMethodContext knnMethodContext) { - this(name, meta, dimension, knnMethodContext, null, DEFAULT_VECTOR_DATA_TYPE_FIELD, knnMethodContext.getSpaceType()); - } - - public KNNVectorFieldType(String name, Map meta, int dimension, KNNMethodContext knnMethodContext, String modelId) { - this(name, meta, dimension, knnMethodContext, modelId, DEFAULT_VECTOR_DATA_TYPE_FIELD, null); - } - - public KNNVectorFieldType( - String name, - Map meta, - int dimension, - KNNMethodContext knnMethodContext, - VectorDataType vectorDataType - ) { - this(name, meta, dimension, knnMethodContext, null, vectorDataType, knnMethodContext.getSpaceType()); - } - - public KNNVectorFieldType( - String name, - Map meta, - int dimension, - @Nullable KNNMethodContext knnMethodContext, - @Nullable String modelId, - VectorDataType vectorDataType, - @Nullable SpaceType spaceType - ) { - super(name, false, false, true, TextSearchInfo.NONE, meta); - this.dimension = dimension; - this.modelId = modelId; - this.knnMethodContext = knnMethodContext; - this.vectorDataType = vectorDataType; - this.spaceType = spaceType; - } - - @Override - public ValueFetcher valueFetcher(QueryShardContext context, SearchLookup searchLookup, String format) { - throw new UnsupportedOperationException("KNN Vector do not support fields search"); - } - - @Override - public String typeName() { - return CONTENT_TYPE; - } - - @Override - public Query existsQuery(QueryShardContext context) { - return new DocValuesFieldExistsQuery(name()); - } - - @Override - public Query termQuery(Object value, QueryShardContext context) { - throw new QueryShardException( - context, - String.format(Locale.ROOT, "KNN vector do not support exact searching, use KNN queries instead: [%s]", name()) - ); - } - - @Override - public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier searchLookup) { - failIfNoDocValues(); - return new KNNVectorIndexFieldData.Builder(name(), CoreValuesSourceType.BYTES, this.vectorDataType); - } - - @Override - public Object valueForDisplay(Object value) { - return deserializeStoredVector((BytesRef) value, vectorDataType); - } - } - protected Explicit ignoreMalformed; protected boolean stored; protected boolean hasDocValues; diff --git a/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperUtil.java b/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperUtil.java index 07bf4fc2d..2adbbb695 100644 --- a/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperUtil.java +++ b/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperUtil.java @@ -236,7 +236,7 @@ public static Object deserializeStoredVector(BytesRef storedVector, VectorDataTy * @param knnVectorFieldType knn vector field type * @return expected vector length */ - public static int getExpectedVectorLength(final KNNVectorFieldMapper.KNNVectorFieldType knnVectorFieldType) { + public static int getExpectedVectorLength(final KNNVectorFieldType knnVectorFieldType) { int expectedDimensions = knnVectorFieldType.getDimension(); if (isModelBasedIndex(expectedDimensions)) { ModelMetadata modelMetadata = getModelMetadataForField(knnVectorFieldType); @@ -255,7 +255,7 @@ private static boolean isModelBasedIndex(int expectedDimensions) { * @param knnVectorField knn vector field * @return the model metadata from knnVectorField */ - private static ModelMetadata getModelMetadataForField(final KNNVectorFieldMapper.KNNVectorFieldType knnVectorField) { + private static ModelMetadata getModelMetadataForField(final KNNVectorFieldType knnVectorField) { String modelId = knnVectorField.getModelId(); if (modelId == null) { diff --git a/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldType.java b/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldType.java new file mode 100644 index 000000000..8c3815c5f --- /dev/null +++ b/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldType.java @@ -0,0 +1,116 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.mapper; + +import lombok.Getter; +import org.apache.lucene.search.DocValuesFieldExistsQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.util.BytesRef; +import org.opensearch.common.Nullable; +import org.opensearch.index.fielddata.IndexFieldData; +import org.opensearch.index.mapper.MappedFieldType; +import org.opensearch.index.mapper.TextSearchInfo; +import org.opensearch.index.mapper.ValueFetcher; +import org.opensearch.index.query.QueryShardContext; +import org.opensearch.index.query.QueryShardException; +import org.opensearch.knn.index.KNNVectorIndexFieldData; +import org.opensearch.knn.index.SpaceType; +import org.opensearch.knn.index.VectorDataType; +import org.opensearch.knn.index.engine.KNNMethodContext; +import org.opensearch.search.aggregations.support.CoreValuesSourceType; +import org.opensearch.search.lookup.SearchLookup; + +import java.util.Locale; +import java.util.Map; +import java.util.function.Supplier; + +import static org.opensearch.knn.common.KNNConstants.DEFAULT_VECTOR_DATA_TYPE_FIELD; +import static org.opensearch.knn.index.mapper.KNNVectorFieldMapperUtil.deserializeStoredVector; + +/** + * A KNNVector field type to represent the vector field in Opensearch + */ +@Getter +public class KNNVectorFieldType extends MappedFieldType { + int dimension; + String modelId; + KNNMethodContext knnMethodContext; + VectorDataType vectorDataType; + SpaceType spaceType; + + public KNNVectorFieldType(String name, Map meta, int dimension, VectorDataType vectorDataType, SpaceType spaceType) { + this(name, meta, dimension, null, null, vectorDataType, spaceType); + } + + public KNNVectorFieldType(String name, Map meta, int dimension, KNNMethodContext knnMethodContext) { + this(name, meta, dimension, knnMethodContext, null, DEFAULT_VECTOR_DATA_TYPE_FIELD, knnMethodContext.getSpaceType()); + } + + public KNNVectorFieldType(String name, Map meta, int dimension, KNNMethodContext knnMethodContext, String modelId) { + this(name, meta, dimension, knnMethodContext, modelId, DEFAULT_VECTOR_DATA_TYPE_FIELD, null); + } + + public KNNVectorFieldType( + String name, + Map meta, + int dimension, + KNNMethodContext knnMethodContext, + VectorDataType vectorDataType + ) { + this(name, meta, dimension, knnMethodContext, null, vectorDataType, knnMethodContext.getSpaceType()); + } + + public KNNVectorFieldType( + String name, + Map meta, + int dimension, + @Nullable KNNMethodContext knnMethodContext, + @Nullable String modelId, + VectorDataType vectorDataType, + @Nullable SpaceType spaceType + ) { + super(name, false, false, true, TextSearchInfo.NONE, meta); + this.dimension = dimension; + this.modelId = modelId; + this.knnMethodContext = knnMethodContext; + this.vectorDataType = vectorDataType; + this.spaceType = spaceType; + } + + @Override + public ValueFetcher valueFetcher(QueryShardContext context, SearchLookup searchLookup, String format) { + throw new UnsupportedOperationException("KNN Vector do not support fields search"); + } + + @Override + public String typeName() { + return KNNVectorFieldMapper.CONTENT_TYPE; + } + + @Override + public Query existsQuery(QueryShardContext context) { + return new DocValuesFieldExistsQuery(name()); + } + + @Override + public Query termQuery(Object value, QueryShardContext context) { + throw new QueryShardException( + context, + String.format(Locale.ROOT, "KNN vector do not support exact searching, use KNN queries instead: [%s]", name()) + ); + } + + @Override + public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier searchLookup) { + failIfNoDocValues(); + return new KNNVectorIndexFieldData.Builder(name(), CoreValuesSourceType.BYTES, this.vectorDataType); + } + + @Override + public Object valueForDisplay(Object value) { + return deserializeStoredVector((BytesRef) value, vectorDataType); + } +} diff --git a/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java b/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java index b05938c4a..6d57cb2dd 100644 --- a/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java +++ b/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java @@ -24,13 +24,13 @@ import org.opensearch.index.query.QueryRewriteContext; import org.opensearch.index.query.QueryShardContext; import org.opensearch.knn.index.engine.model.QueryContext; +import org.opensearch.knn.index.mapper.KNNVectorFieldType; import org.opensearch.knn.index.util.IndexUtil; import org.opensearch.knn.index.engine.KNNMethodContext; import org.opensearch.knn.index.engine.MethodComponentContext; import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.VectorQueryType; -import org.opensearch.knn.index.mapper.KNNVectorFieldMapper; import org.opensearch.knn.index.query.parser.KNNQueryBuilderParser; import org.opensearch.knn.index.engine.KNNLibrarySearchContext; import org.opensearch.knn.index.engine.KNNEngine; @@ -338,11 +338,11 @@ protected Query doToQuery(QueryShardContext context) { return new MatchNoDocsQuery(); } - if (!(mappedFieldType instanceof KNNVectorFieldMapper.KNNVectorFieldType)) { + if (!(mappedFieldType instanceof KNNVectorFieldType)) { throw new IllegalArgumentException(String.format(Locale.ROOT, "Field '%s' is not knn_vector type.", this.fieldName)); } - KNNVectorFieldMapper.KNNVectorFieldType knnVectorFieldType = (KNNVectorFieldMapper.KNNVectorFieldType) mappedFieldType; + KNNVectorFieldType knnVectorFieldType = (KNNVectorFieldType) mappedFieldType; int fieldDimension = knnVectorFieldType.getDimension(); KNNMethodContext knnMethodContext = knnVectorFieldType.getKnnMethodContext(); MethodComponentContext methodComponentContext = null; @@ -492,7 +492,7 @@ protected Query doToQuery(QueryShardContext context) { throw new IllegalArgumentException(String.format(Locale.ROOT, "[%s] requires k or distance or score to be set", NAME)); } - private ModelMetadata getModelMetadataForField(KNNVectorFieldMapper.KNNVectorFieldType knnVectorField) { + private ModelMetadata getModelMetadataForField(KNNVectorFieldType knnVectorField) { String modelId = knnVectorField.getModelId(); if (modelId == null) { diff --git a/src/main/java/org/opensearch/knn/plugin/script/KNNScoringSpace.java b/src/main/java/org/opensearch/knn/plugin/script/KNNScoringSpace.java index 850bff1ab..71616c9fd 100644 --- a/src/main/java/org/opensearch/knn/plugin/script/KNNScoringSpace.java +++ b/src/main/java/org/opensearch/knn/plugin/script/KNNScoringSpace.java @@ -11,8 +11,8 @@ import org.opensearch.index.mapper.MappedFieldType; import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.VectorDataType; -import org.opensearch.knn.index.mapper.KNNVectorFieldMapper; import org.opensearch.knn.index.mapper.KNNVectorFieldMapperUtil; +import org.opensearch.knn.index.mapper.KNNVectorFieldType; import org.opensearch.knn.index.query.KNNWeight; import org.opensearch.script.ScoreScript; import org.opensearch.search.lookup.SearchLookup; @@ -67,11 +67,7 @@ public KNNFieldSpace( final String spaceName, final Set supportingVectorDataTypes ) { - KNNVectorFieldMapper.KNNVectorFieldType knnVectorFieldType = toKNNVectorFieldType( - fieldType, - spaceName, - supportingVectorDataTypes - ); + KNNVectorFieldType knnVectorFieldType = toKNNVectorFieldType(fieldType, spaceName, supportingVectorDataTypes); this.processedQuery = getProcessedQuery(query, knnVectorFieldType); this.scoringMethod = getScoringMethod(this.processedQuery); } @@ -86,7 +82,7 @@ public ScoreScript getScoreScript( return new KNNScoreScript.KNNVectorType(params, this.processedQuery, field, this.scoringMethod, lookup, ctx, searcher); } - private KNNVectorFieldMapper.KNNVectorFieldType toKNNVectorFieldType( + private KNNVectorFieldType toKNNVectorFieldType( final MappedFieldType fieldType, final String spaceName, final Set supportingVectorDataTypes @@ -97,7 +93,7 @@ private KNNVectorFieldMapper.KNNVectorFieldType toKNNVectorFieldType( ); } - KNNVectorFieldMapper.KNNVectorFieldType knnVectorFieldType = (KNNVectorFieldMapper.KNNVectorFieldType) fieldType; + KNNVectorFieldType knnVectorFieldType = (KNNVectorFieldType) fieldType; VectorDataType vectorDataType = knnVectorFieldType.getVectorDataType() == null ? VectorDataType.FLOAT : knnVectorFieldType.getVectorDataType(); @@ -116,7 +112,7 @@ private KNNVectorFieldMapper.KNNVectorFieldType toKNNVectorFieldType( return knnVectorFieldType; } - protected float[] getProcessedQuery(final Object query, final KNNVectorFieldMapper.KNNVectorFieldType knnVectorFieldType) { + protected float[] getProcessedQuery(final Object query, final KNNVectorFieldType knnVectorFieldType) { return parseToFloatArray( query, KNNVectorFieldMapperUtil.getExpectedVectorLength(knnVectorFieldType), diff --git a/src/main/java/org/opensearch/knn/plugin/script/KNNScoringSpaceUtil.java b/src/main/java/org/opensearch/knn/plugin/script/KNNScoringSpaceUtil.java index e2bade320..7a97fdb05 100644 --- a/src/main/java/org/opensearch/knn/plugin/script/KNNScoringSpaceUtil.java +++ b/src/main/java/org/opensearch/knn/plugin/script/KNNScoringSpaceUtil.java @@ -7,7 +7,7 @@ import java.util.List; import org.opensearch.knn.index.VectorDataType; -import org.opensearch.knn.index.mapper.KNNVectorFieldMapper; +import org.opensearch.knn.index.mapper.KNNVectorFieldType; import org.opensearch.knn.plugin.stats.KNNCounter; import org.opensearch.index.mapper.BinaryFieldMapper; import org.opensearch.index.mapper.MappedFieldType; @@ -52,7 +52,7 @@ public static boolean isBinaryFieldType(MappedFieldType fieldType) { * @return true if fieldType is of type KNNVectorFieldType; false otherwise */ public static boolean isKNNVectorFieldType(MappedFieldType fieldType) { - return fieldType instanceof KNNVectorFieldMapper.KNNVectorFieldType; + return fieldType instanceof KNNVectorFieldType; } /** @@ -61,7 +61,7 @@ public static boolean isKNNVectorFieldType(MappedFieldType fieldType) { * @param fieldType KNN vector field type * @return true if the KNN field type is a binary vector data type */ - public static boolean isBinaryVectorDataType(final KNNVectorFieldMapper.KNNVectorFieldType fieldType) { + public static boolean isBinaryVectorDataType(final KNNVectorFieldType fieldType) { return VectorDataType.BINARY == fieldType.getVectorDataType(); } diff --git a/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestCase.java b/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestCase.java index 70b055df4..a0b9b32d0 100644 --- a/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestCase.java +++ b/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestCase.java @@ -25,6 +25,7 @@ import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.VectorField; +import org.opensearch.knn.index.mapper.KNNVectorFieldType; import org.opensearch.knn.index.query.KNNQueryFactory; import org.opensearch.knn.jni.JNIService; import org.opensearch.knn.index.query.KNNQuery; @@ -308,18 +309,8 @@ public void testKnnVectorIndex( SpaceType.L2, new MethodComponentContext(METHOD_HNSW, Map.of(HNSW_ALGO_M, 16, HNSW_ALGO_EF_CONSTRUCTION, 256)) ); - final KNNVectorFieldMapper.KNNVectorFieldType mappedFieldType1 = new KNNVectorFieldMapper.KNNVectorFieldType( - FIELD_NAME_ONE, - Map.of(), - 3, - knnMethodContext - ); - final KNNVectorFieldMapper.KNNVectorFieldType mappedFieldType2 = new KNNVectorFieldMapper.KNNVectorFieldType( - FIELD_NAME_TWO, - Map.of(), - 2, - knnMethodContext - ); + final KNNVectorFieldType mappedFieldType1 = new KNNVectorFieldType(FIELD_NAME_ONE, Map.of(), 3, knnMethodContext); + final KNNVectorFieldType mappedFieldType2 = new KNNVectorFieldType(FIELD_NAME_TWO, Map.of(), 2, knnMethodContext); when(mapperService.fieldType(eq(FIELD_NAME_ONE))).thenReturn(mappedFieldType1); when(mapperService.fieldType(eq(FIELD_NAME_TWO))).thenReturn(mappedFieldType2); diff --git a/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java b/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java index e68d34e88..c95568be2 100644 --- a/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java +++ b/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java @@ -1058,7 +1058,7 @@ private LuceneFieldMapper.CreateLuceneFieldMapperInput.CreateLuceneFieldMapperIn new MethodComponentContext(METHOD_HNSW, Collections.emptyMap()) ); - KNNVectorFieldMapper.KNNVectorFieldType knnVectorFieldType = new KNNVectorFieldMapper.KNNVectorFieldType( + KNNVectorFieldType knnVectorFieldType = new KNNVectorFieldType( TEST_FIELD_NAME, Collections.emptyMap(), TEST_DIMENSION, diff --git a/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperUtilTests.java b/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperUtilTests.java index a4d597a41..31da12d66 100644 --- a/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperUtilTests.java +++ b/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperUtilTests.java @@ -60,14 +60,14 @@ public void testStoredFields_whenVectorIsFloatType_thenSucceed() { } public void testGetExpectedVectorLengthSuccess() { - KNNVectorFieldMapper.KNNVectorFieldType knnVectorFieldType = mock(KNNVectorFieldMapper.KNNVectorFieldType.class); + KNNVectorFieldType knnVectorFieldType = mock(KNNVectorFieldType.class); when(knnVectorFieldType.getDimension()).thenReturn(3); - KNNVectorFieldMapper.KNNVectorFieldType knnVectorFieldTypeBinary = mock(KNNVectorFieldMapper.KNNVectorFieldType.class); + KNNVectorFieldType knnVectorFieldTypeBinary = mock(KNNVectorFieldType.class); when(knnVectorFieldTypeBinary.getDimension()).thenReturn(8); when(knnVectorFieldTypeBinary.getVectorDataType()).thenReturn(VectorDataType.BINARY); - KNNVectorFieldMapper.KNNVectorFieldType knnVectorFieldTypeModelBased = mock(KNNVectorFieldMapper.KNNVectorFieldType.class); + KNNVectorFieldType knnVectorFieldTypeModelBased = mock(KNNVectorFieldType.class); when(knnVectorFieldTypeModelBased.getDimension()).thenReturn(-1); String modelId = "test-model"; when(knnVectorFieldTypeModelBased.getModelId()).thenReturn(modelId); @@ -86,7 +86,7 @@ public void testGetExpectedVectorLengthSuccess() { } public void testGetExpectedVectorLengthFailure() { - KNNVectorFieldMapper.KNNVectorFieldType knnVectorFieldTypeModelBased = mock(KNNVectorFieldMapper.KNNVectorFieldType.class); + KNNVectorFieldType knnVectorFieldTypeModelBased = mock(KNNVectorFieldType.class); when(knnVectorFieldTypeModelBased.getDimension()).thenReturn(-1); String modelId = "test-model"; when(knnVectorFieldTypeModelBased.getModelId()).thenReturn(modelId); diff --git a/src/test/java/org/opensearch/knn/index/mapper/MethodFieldMapperTests.java b/src/test/java/org/opensearch/knn/index/mapper/MethodFieldMapperTests.java index 9ae2fad9c..dcd255740 100644 --- a/src/test/java/org/opensearch/knn/index/mapper/MethodFieldMapperTests.java +++ b/src/test/java/org/opensearch/knn/index/mapper/MethodFieldMapperTests.java @@ -15,7 +15,7 @@ public class MethodFieldMapperTests extends TestCase { public void testMethodFieldMapper_whenVectorDataTypeIsGiven_thenSetItInFieldType() { - KNNVectorFieldMapper.KNNVectorFieldType mappedFieldType = new KNNVectorFieldMapper.KNNVectorFieldType( + KNNVectorFieldType mappedFieldType = new KNNVectorFieldType( "testField", Collections.emptyMap(), 1, diff --git a/src/test/java/org/opensearch/knn/index/query/KNNQueryBuilderTests.java b/src/test/java/org/opensearch/knn/index/query/KNNQueryBuilderTests.java index 63d9a6c30..0241a9afb 100644 --- a/src/test/java/org/opensearch/knn/index/query/KNNQueryBuilderTests.java +++ b/src/test/java/org/opensearch/knn/index/query/KNNQueryBuilderTests.java @@ -27,12 +27,12 @@ import org.opensearch.index.query.QueryShardContext; import org.opensearch.index.query.TermQueryBuilder; import org.opensearch.knn.KNNTestCase; +import org.opensearch.knn.index.mapper.KNNVectorFieldType; import org.opensearch.knn.index.util.KNNClusterUtil; import org.opensearch.knn.index.engine.KNNMethodContext; import org.opensearch.knn.index.engine.MethodComponentContext; import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.VectorDataType; -import org.opensearch.knn.index.mapper.KNNVectorFieldMapper; import org.opensearch.knn.index.engine.KNNEngine; import org.opensearch.knn.indices.ModelDao; import org.opensearch.knn.indices.ModelMetadata; @@ -169,7 +169,7 @@ public void testDoToQuery_Normal() throws Exception { KNNQueryBuilder knnQueryBuilder = new KNNQueryBuilder(FIELD_NAME, queryVector, K); Index dummyIndex = new Index("dummy", "dummy"); QueryShardContext mockQueryShardContext = mock(QueryShardContext.class); - KNNVectorFieldMapper.KNNVectorFieldType mockKNNVectorField = mock(KNNVectorFieldMapper.KNNVectorFieldType.class); + KNNVectorFieldType mockKNNVectorField = mock(KNNVectorFieldType.class); when(mockQueryShardContext.index()).thenReturn(dummyIndex); when(mockKNNVectorField.getDimension()).thenReturn(4); when(mockKNNVectorField.getVectorDataType()).thenReturn(VectorDataType.FLOAT); @@ -191,7 +191,7 @@ public void testDoToQuery_whenNormal_whenDoRadiusSearch_whenDistanceThreshold_th .build(); Index dummyIndex = new Index("dummy", "dummy"); QueryShardContext mockQueryShardContext = mock(QueryShardContext.class); - KNNVectorFieldMapper.KNNVectorFieldType mockKNNVectorField = mock(KNNVectorFieldMapper.KNNVectorFieldType.class); + KNNVectorFieldType mockKNNVectorField = mock(KNNVectorFieldType.class); when(mockQueryShardContext.index()).thenReturn(dummyIndex); when(mockKNNVectorField.getDimension()).thenReturn(4); when(mockKNNVectorField.getVectorDataType()).thenReturn(VectorDataType.FLOAT); @@ -223,7 +223,7 @@ public void testDoToQuery_whenNormal_whenDoRadiusSearch_whenScoreThreshold_thenS Index dummyIndex = new Index("dummy", "dummy"); QueryShardContext mockQueryShardContext = mock(QueryShardContext.class); - KNNVectorFieldMapper.KNNVectorFieldType mockKNNVectorField = mock(KNNVectorFieldMapper.KNNVectorFieldType.class); + KNNVectorFieldType mockKNNVectorField = mock(KNNVectorFieldType.class); when(mockQueryShardContext.index()).thenReturn(dummyIndex); when(mockKNNVectorField.getDimension()).thenReturn(4); when(mockKNNVectorField.getVectorDataType()).thenReturn(VectorDataType.FLOAT); @@ -250,7 +250,7 @@ public void testDoToQuery_whenDoRadiusSearch_whenPassNegativeDistance_whenSuppor .build(); Index dummyIndex = new Index("dummy", "dummy"); QueryShardContext mockQueryShardContext = mock(QueryShardContext.class); - KNNVectorFieldMapper.KNNVectorFieldType mockKNNVectorField = mock(KNNVectorFieldMapper.KNNVectorFieldType.class); + KNNVectorFieldType mockKNNVectorField = mock(KNNVectorFieldType.class); when(mockQueryShardContext.index()).thenReturn(dummyIndex); when(mockKNNVectorField.getDimension()).thenReturn(4); when(mockKNNVectorField.getVectorDataType()).thenReturn(VectorDataType.FLOAT); @@ -282,7 +282,7 @@ public void testDoToQuery_whenDoRadiusSearch_whenPassNegativeDistance_whenUnSupp .build(); Index dummyIndex = new Index("dummy", "dummy"); QueryShardContext mockQueryShardContext = mock(QueryShardContext.class); - KNNVectorFieldMapper.KNNVectorFieldType mockKNNVectorField = mock(KNNVectorFieldMapper.KNNVectorFieldType.class); + KNNVectorFieldType mockKNNVectorField = mock(KNNVectorFieldType.class); when(mockQueryShardContext.index()).thenReturn(dummyIndex); when(mockKNNVectorField.getDimension()).thenReturn(4); when(mockKNNVectorField.getVectorDataType()).thenReturn(VectorDataType.FLOAT); @@ -309,7 +309,7 @@ public void testDoToQuery_whenDoRadiusSearch_whenPassScoreMoreThanOne_whenSuppor KNNQueryBuilder knnQueryBuilder = KNNQueryBuilder.builder().fieldName(FIELD_NAME).vector(queryVector).minScore(score).build(); Index dummyIndex = new Index("dummy", "dummy"); QueryShardContext mockQueryShardContext = mock(QueryShardContext.class); - KNNVectorFieldMapper.KNNVectorFieldType mockKNNVectorField = mock(KNNVectorFieldMapper.KNNVectorFieldType.class); + KNNVectorFieldType mockKNNVectorField = mock(KNNVectorFieldType.class); when(mockQueryShardContext.index()).thenReturn(dummyIndex); when(mockKNNVectorField.getDimension()).thenReturn(4); when(mockKNNVectorField.getVectorDataType()).thenReturn(VectorDataType.FLOAT); @@ -336,7 +336,7 @@ public void testDoToQuery_whenDoRadiusSearch_whenPassScoreMoreThanOne_whenUnsupp KNNQueryBuilder knnQueryBuilder = KNNQueryBuilder.builder().fieldName(FIELD_NAME).vector(queryVector).minScore(score).build(); Index dummyIndex = new Index("dummy", "dummy"); QueryShardContext mockQueryShardContext = mock(QueryShardContext.class); - KNNVectorFieldMapper.KNNVectorFieldType mockKNNVectorField = mock(KNNVectorFieldMapper.KNNVectorFieldType.class); + KNNVectorFieldType mockKNNVectorField = mock(KNNVectorFieldType.class); when(mockQueryShardContext.index()).thenReturn(dummyIndex); when(mockKNNVectorField.getDimension()).thenReturn(4); when(mockKNNVectorField.getVectorDataType()).thenReturn(VectorDataType.FLOAT); @@ -367,7 +367,7 @@ public void testDoToQuery_whenPassNegativeDistance_whenSupportedSpaceType_thenSu Index dummyIndex = new Index("dummy", "dummy"); QueryShardContext mockQueryShardContext = mock(QueryShardContext.class); - KNNVectorFieldMapper.KNNVectorFieldType mockKNNVectorField = mock(KNNVectorFieldMapper.KNNVectorFieldType.class); + KNNVectorFieldType mockKNNVectorField = mock(KNNVectorFieldType.class); when(mockQueryShardContext.index()).thenReturn(dummyIndex); when(mockKNNVectorField.getDimension()).thenReturn(4); when(mockKNNVectorField.getVectorDataType()).thenReturn(VectorDataType.FLOAT); @@ -400,7 +400,7 @@ public void testDoToQuery_whenPassNegativeDistance_whenUnSupportedSpaceType_then Index dummyIndex = new Index("dummy", "dummy"); QueryShardContext mockQueryShardContext = mock(QueryShardContext.class); - KNNVectorFieldMapper.KNNVectorFieldType mockKNNVectorField = mock(KNNVectorFieldMapper.KNNVectorFieldType.class); + KNNVectorFieldType mockKNNVectorField = mock(KNNVectorFieldType.class); when(mockQueryShardContext.index()).thenReturn(dummyIndex); when(mockKNNVectorField.getDimension()).thenReturn(4); when(mockKNNVectorField.getVectorDataType()).thenReturn(VectorDataType.FLOAT); @@ -428,7 +428,7 @@ public void testDoToQuery_whenRadialSearchOnBinaryIndex_thenException() { .build(); Index dummyIndex = new Index("dummy", "dummy"); QueryShardContext mockQueryShardContext = mock(QueryShardContext.class); - KNNVectorFieldMapper.KNNVectorFieldType mockKNNVectorField = mock(KNNVectorFieldMapper.KNNVectorFieldType.class); + KNNVectorFieldType mockKNNVectorField = mock(KNNVectorFieldType.class); when(mockQueryShardContext.index()).thenReturn(dummyIndex); when(mockKNNVectorField.getDimension()).thenReturn(8); when(mockKNNVectorField.getVectorDataType()).thenReturn(VectorDataType.BINARY); @@ -454,7 +454,7 @@ public void testDoToQuery_KnnQueryWithFilter_Lucene() throws Exception { .build(); Index dummyIndex = new Index("dummy", "dummy"); QueryShardContext mockQueryShardContext = mock(QueryShardContext.class); - KNNVectorFieldMapper.KNNVectorFieldType mockKNNVectorField = mock(KNNVectorFieldMapper.KNNVectorFieldType.class); + KNNVectorFieldType mockKNNVectorField = mock(KNNVectorFieldType.class); when(mockQueryShardContext.index()).thenReturn(dummyIndex); when(mockKNNVectorField.getDimension()).thenReturn(4); when(mockKNNVectorField.getVectorDataType()).thenReturn(VectorDataType.FLOAT); @@ -488,7 +488,7 @@ public void testDoToQuery_whenDoRadiusSearch_whenDistanceThreshold_whenFilter_th Index dummyIndex = new Index("dummy", "dummy"); QueryShardContext mockQueryShardContext = mock(QueryShardContext.class); - KNNVectorFieldMapper.KNNVectorFieldType mockKNNVectorField = mock(KNNVectorFieldMapper.KNNVectorFieldType.class); + KNNVectorFieldType mockKNNVectorField = mock(KNNVectorFieldType.class); when(mockQueryShardContext.index()).thenReturn(dummyIndex); when(mockKNNVectorField.getDimension()).thenReturn(4); when(mockKNNVectorField.getVectorDataType()).thenReturn(VectorDataType.FLOAT); @@ -515,7 +515,7 @@ public void testDoToQuery_whenDoRadiusSearch_whenScoreThreshold_whenFilter_thenS .build(); Index dummyIndex = new Index("dummy", "dummy"); QueryShardContext mockQueryShardContext = mock(QueryShardContext.class); - KNNVectorFieldMapper.KNNVectorFieldType mockKNNVectorField = mock(KNNVectorFieldMapper.KNNVectorFieldType.class); + KNNVectorFieldType mockKNNVectorField = mock(KNNVectorFieldType.class); when(mockQueryShardContext.index()).thenReturn(dummyIndex); when(mockKNNVectorField.getDimension()).thenReturn(4); when(mockKNNVectorField.getVectorDataType()).thenReturn(VectorDataType.FLOAT); @@ -537,7 +537,7 @@ public void testDoToQuery_WhenknnQueryWithFilterAndFaissEngine_thenSuccess() { float[] queryVector = { 1.0f, 2.0f, 3.0f, 4.0f }; Index dummyIndex = new Index("dummy", "dummy"); QueryShardContext mockQueryShardContext = mock(QueryShardContext.class); - KNNVectorFieldMapper.KNNVectorFieldType mockKNNVectorField = mock(KNNVectorFieldMapper.KNNVectorFieldType.class); + KNNVectorFieldType mockKNNVectorField = mock(KNNVectorFieldType.class); when(mockQueryShardContext.index()).thenReturn(dummyIndex); when(mockKNNVectorField.getDimension()).thenReturn(4); when(mockKNNVectorField.getSpaceType()).thenReturn(SpaceType.L2); @@ -570,7 +570,7 @@ public void testDoToQuery_WhenknnQueryWithFilterAndFaissEngine_thenSuccess() { public void testDoToQuery_ThrowsIllegalArgumentExceptionForUnknownMethodParameter() { QueryShardContext mockQueryShardContext = mock(QueryShardContext.class); - KNNVectorFieldMapper.KNNVectorFieldType mockKNNVectorField = mock(KNNVectorFieldMapper.KNNVectorFieldType.class); + KNNVectorFieldType mockKNNVectorField = mock(KNNVectorFieldType.class); when(mockQueryShardContext.fieldMapper(anyString())).thenReturn(mockKNNVectorField); when(mockKNNVectorField.getDimension()).thenReturn(4); when(mockKNNVectorField.getKnnMethodContext()).thenReturn( @@ -593,7 +593,7 @@ public void testDoToQuery_whenknnQueryWithFilterAndNmsLibEngine_thenException() KNNQueryBuilder knnQueryBuilder = new KNNQueryBuilder(FIELD_NAME, queryVector, K, TERM_QUERY); Index dummyIndex = new Index("dummy", "dummy"); QueryShardContext mockQueryShardContext = mock(QueryShardContext.class); - KNNVectorFieldMapper.KNNVectorFieldType mockKNNVectorField = mock(KNNVectorFieldMapper.KNNVectorFieldType.class); + KNNVectorFieldType mockKNNVectorField = mock(KNNVectorFieldType.class); when(mockQueryShardContext.index()).thenReturn(dummyIndex); when(mockKNNVectorField.getDimension()).thenReturn(4); when(mockKNNVectorField.getSpaceType()).thenReturn(SpaceType.L2); @@ -614,7 +614,7 @@ public void testDoToQuery_FromModel() { KNNQueryBuilder knnQueryBuilder = new KNNQueryBuilder(FIELD_NAME, queryVector, K); Index dummyIndex = new Index("dummy", "dummy"); QueryShardContext mockQueryShardContext = mock(QueryShardContext.class); - KNNVectorFieldMapper.KNNVectorFieldType mockKNNVectorField = mock(KNNVectorFieldMapper.KNNVectorFieldType.class); + KNNVectorFieldType mockKNNVectorField = mock(KNNVectorFieldType.class); when(mockQueryShardContext.index()).thenReturn(dummyIndex); // Dimension is -1. In this case, model metadata will need to provide dimension @@ -655,7 +655,7 @@ public void testDoToQuery_whenFromModel_whenDoRadiusSearch_whenDistanceThreshold Index dummyIndex = new Index("dummy", "dummy"); QueryShardContext mockQueryShardContext = mock(QueryShardContext.class); - KNNVectorFieldMapper.KNNVectorFieldType mockKNNVectorField = mock(KNNVectorFieldMapper.KNNVectorFieldType.class); + KNNVectorFieldType mockKNNVectorField = mock(KNNVectorFieldType.class); when(mockQueryShardContext.index()).thenReturn(dummyIndex); when(mockKNNVectorField.getDimension()).thenReturn(-K); @@ -693,7 +693,7 @@ public void testDoToQuery_whenFromModel_whenDoRadiusSearch_whenScoreThreshold_th Index dummyIndex = new Index("dummy", "dummy"); QueryShardContext mockQueryShardContext = mock(QueryShardContext.class); - KNNVectorFieldMapper.KNNVectorFieldType mockKNNVectorField = mock(KNNVectorFieldMapper.KNNVectorFieldType.class); + KNNVectorFieldType mockKNNVectorField = mock(KNNVectorFieldType.class); when(mockQueryShardContext.index()).thenReturn(dummyIndex); when(mockKNNVectorField.getDimension()).thenReturn(-K); @@ -728,7 +728,7 @@ public void testDoToQuery_InvalidDimensions() { KNNQueryBuilder knnQueryBuilder = new KNNQueryBuilder(FIELD_NAME, queryVector, K); Index dummyIndex = new Index("dummy", "dummy"); QueryShardContext mockQueryShardContext = mock(QueryShardContext.class); - KNNVectorFieldMapper.KNNVectorFieldType mockKNNVectorField = mock(KNNVectorFieldMapper.KNNVectorFieldType.class); + KNNVectorFieldType mockKNNVectorField = mock(KNNVectorFieldType.class); when(mockQueryShardContext.index()).thenReturn(dummyIndex); when(mockKNNVectorField.getDimension()).thenReturn(400); when(mockQueryShardContext.fieldMapper(anyString())).thenReturn(mockKNNVectorField); @@ -753,7 +753,7 @@ public void testDoToQuery_InvalidZeroFloatVector() { KNNQueryBuilder knnQueryBuilder = new KNNQueryBuilder(FIELD_NAME, queryVector, K); Index dummyIndex = new Index("dummy", "dummy"); QueryShardContext mockQueryShardContext = mock(QueryShardContext.class); - KNNVectorFieldMapper.KNNVectorFieldType mockKNNVectorField = mock(KNNVectorFieldMapper.KNNVectorFieldType.class); + KNNVectorFieldType mockKNNVectorField = mock(KNNVectorFieldType.class); when(mockQueryShardContext.index()).thenReturn(dummyIndex); when(mockKNNVectorField.getDimension()).thenReturn(4); when(mockKNNVectorField.getVectorDataType()).thenReturn(VectorDataType.FLOAT); @@ -774,7 +774,7 @@ public void testDoToQuery_InvalidZeroByteVector() { KNNQueryBuilder knnQueryBuilder = new KNNQueryBuilder(FIELD_NAME, queryVector, K); Index dummyIndex = new Index("dummy", "dummy"); QueryShardContext mockQueryShardContext = mock(QueryShardContext.class); - KNNVectorFieldMapper.KNNVectorFieldType mockKNNVectorField = mock(KNNVectorFieldMapper.KNNVectorFieldType.class); + KNNVectorFieldType mockKNNVectorField = mock(KNNVectorFieldType.class); when(mockQueryShardContext.index()).thenReturn(dummyIndex); when(mockKNNVectorField.getDimension()).thenReturn(4); when(mockKNNVectorField.getVectorDataType()).thenReturn(VectorDataType.BYTE); @@ -902,7 +902,7 @@ public void testRadialSearch_whenUnsupportedEngine_thenThrowException() { .maxDistance(MAX_DISTANCE) .build(); - KNNVectorFieldMapper.KNNVectorFieldType mockKNNVectorField = mock(KNNVectorFieldMapper.KNNVectorFieldType.class); + KNNVectorFieldType mockKNNVectorField = mock(KNNVectorFieldType.class); QueryShardContext mockQueryShardContext = mock(QueryShardContext.class); Index dummyIndex = new Index("dummy", "dummy"); when(mockKNNVectorField.getKnnMethodContext()).thenReturn(knnMethodContext); @@ -929,7 +929,7 @@ public void testRadialSearch_whenEfSearchIsSet_whenLuceneEngine_thenThrowExcepti .methodParameters(Map.of("ef_search", EF_SEARCH)) .build(); - KNNVectorFieldMapper.KNNVectorFieldType mockKNNVectorField = mock(KNNVectorFieldMapper.KNNVectorFieldType.class); + KNNVectorFieldType mockKNNVectorField = mock(KNNVectorFieldType.class); QueryShardContext mockQueryShardContext = mock(QueryShardContext.class); Index dummyIndex = new Index("dummy", "dummy"); when(mockKNNVectorField.getKnnMethodContext()).thenReturn(knnMethodContext); @@ -955,7 +955,7 @@ public void testRadialSearch_whenEfSearchIsSet_whenFaissEngine_thenSuccess() { .methodParameters(Map.of("ef_search", EF_SEARCH)) .build(); - KNNVectorFieldMapper.KNNVectorFieldType mockKNNVectorField = mock(KNNVectorFieldMapper.KNNVectorFieldType.class); + KNNVectorFieldType mockKNNVectorField = mock(KNNVectorFieldType.class); QueryShardContext mockQueryShardContext = mock(QueryShardContext.class); Index dummyIndex = new Index("dummy", "dummy"); when(mockKNNVectorField.getKnnMethodContext()).thenReturn(knnMethodContext); @@ -976,7 +976,7 @@ public void testDoToQuery_whenBinary_thenValid() throws Exception { KNNQueryBuilder knnQueryBuilder = new KNNQueryBuilder(FIELD_NAME, queryVector, K); Index dummyIndex = new Index("dummy", "dummy"); QueryShardContext mockQueryShardContext = mock(QueryShardContext.class); - KNNVectorFieldMapper.KNNVectorFieldType mockKNNVectorField = mock(KNNVectorFieldMapper.KNNVectorFieldType.class); + KNNVectorFieldType mockKNNVectorField = mock(KNNVectorFieldType.class); when(mockQueryShardContext.index()).thenReturn(dummyIndex); when(mockKNNVectorField.getDimension()).thenReturn(32); when(mockKNNVectorField.getVectorDataType()).thenReturn(VectorDataType.BINARY); @@ -992,7 +992,7 @@ public void testDoToQuery_whenBinaryWithInvalidDimension_thenException() throws KNNQueryBuilder knnQueryBuilder = new KNNQueryBuilder(FIELD_NAME, queryVector, K); Index dummyIndex = new Index("dummy", "dummy"); QueryShardContext mockQueryShardContext = mock(QueryShardContext.class); - KNNVectorFieldMapper.KNNVectorFieldType mockKNNVectorField = mock(KNNVectorFieldMapper.KNNVectorFieldType.class); + KNNVectorFieldType mockKNNVectorField = mock(KNNVectorFieldType.class); when(mockQueryShardContext.index()).thenReturn(dummyIndex); when(mockKNNVectorField.getDimension()).thenReturn(8); when(mockKNNVectorField.getVectorDataType()).thenReturn(VectorDataType.BINARY); diff --git a/src/test/java/org/opensearch/knn/integ/KNNScriptScoringIT.java b/src/test/java/org/opensearch/knn/integ/KNNScriptScoringIT.java index a1a6e3aa6..e67ad40bc 100644 --- a/src/test/java/org/opensearch/knn/integ/KNNScriptScoringIT.java +++ b/src/test/java/org/opensearch/knn/integ/KNNScriptScoringIT.java @@ -30,8 +30,8 @@ import org.opensearch.index.query.functionscore.ScriptScoreQueryBuilder; import org.opensearch.core.rest.RestStatus; import org.opensearch.knn.index.VectorDataType; -import org.opensearch.knn.index.mapper.KNNVectorFieldMapper; import org.opensearch.knn.index.engine.KNNEngine; +import org.opensearch.knn.index.mapper.KNNVectorFieldType; import org.opensearch.knn.plugin.script.KNNScoringScriptEngine; import org.opensearch.knn.plugin.script.KNNScoringSpace; import org.opensearch.knn.plugin.script.KNNScoringSpaceFactory; @@ -735,7 +735,7 @@ private Map createDataset( } private BiFunction getScoreFunction(SpaceType spaceType, float[] queryVector) { - KNNVectorFieldMapper.KNNVectorFieldType knnVectorFieldType = new KNNVectorFieldMapper.KNNVectorFieldType( + KNNVectorFieldType knnVectorFieldType = new KNNVectorFieldType( FIELD_NAME, Collections.emptyMap(), SpaceType.HAMMING == spaceType ? queryVector.length * 8 : queryVector.length, diff --git a/src/test/java/org/opensearch/knn/plugin/script/KNNScoringSpaceFactoryTests.java b/src/test/java/org/opensearch/knn/plugin/script/KNNScoringSpaceFactoryTests.java index e24acc483..823d21080 100644 --- a/src/test/java/org/opensearch/knn/plugin/script/KNNScoringSpaceFactoryTests.java +++ b/src/test/java/org/opensearch/knn/plugin/script/KNNScoringSpaceFactoryTests.java @@ -7,9 +7,9 @@ import org.opensearch.knn.KNNTestCase; import org.opensearch.knn.index.VectorDataType; -import org.opensearch.knn.index.mapper.KNNVectorFieldMapper; import org.opensearch.knn.index.SpaceType; import org.opensearch.index.mapper.NumberFieldMapper; +import org.opensearch.knn.index.mapper.KNNVectorFieldType; import java.util.List; @@ -18,9 +18,9 @@ public class KNNScoringSpaceFactoryTests extends KNNTestCase { public void testValidSpaces() { - KNNVectorFieldMapper.KNNVectorFieldType knnVectorFieldType = mock(KNNVectorFieldMapper.KNNVectorFieldType.class); + KNNVectorFieldType knnVectorFieldType = mock(KNNVectorFieldType.class); when(knnVectorFieldType.getDimension()).thenReturn(3); - KNNVectorFieldMapper.KNNVectorFieldType knnVectorFieldTypeBinary = mock(KNNVectorFieldMapper.KNNVectorFieldType.class); + KNNVectorFieldType knnVectorFieldTypeBinary = mock(KNNVectorFieldType.class); when(knnVectorFieldTypeBinary.getDimension()).thenReturn(24); when(knnVectorFieldTypeBinary.getVectorDataType()).thenReturn(VectorDataType.BINARY); NumberFieldMapper.NumberFieldType numberFieldType = new NumberFieldMapper.NumberFieldType( @@ -65,9 +65,9 @@ public void testValidSpaces() { public void testInvalidSpace() { List floatQueryObject = List.of(1.0f, 1.0f, 1.0f); - KNNVectorFieldMapper.KNNVectorFieldType knnVectorFieldType = mock(KNNVectorFieldMapper.KNNVectorFieldType.class); + KNNVectorFieldType knnVectorFieldType = mock(KNNVectorFieldType.class); when(knnVectorFieldType.getDimension()).thenReturn(3); - KNNVectorFieldMapper.KNNVectorFieldType knnVectorFieldTypeBinary = mock(KNNVectorFieldMapper.KNNVectorFieldType.class); + KNNVectorFieldType knnVectorFieldTypeBinary = mock(KNNVectorFieldType.class); when(knnVectorFieldTypeBinary.getDimension()).thenReturn(24); when(knnVectorFieldTypeBinary.getVectorDataType()).thenReturn(VectorDataType.BINARY); diff --git a/src/test/java/org/opensearch/knn/plugin/script/KNNScoringSpaceTests.java b/src/test/java/org/opensearch/knn/plugin/script/KNNScoringSpaceTests.java index 07385e55b..6c557c8dd 100644 --- a/src/test/java/org/opensearch/knn/plugin/script/KNNScoringSpaceTests.java +++ b/src/test/java/org/opensearch/knn/plugin/script/KNNScoringSpaceTests.java @@ -15,9 +15,9 @@ import org.opensearch.knn.index.engine.KNNMethodContext; import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.VectorDataType; -import org.opensearch.knn.index.mapper.KNNVectorFieldMapper; import org.opensearch.index.mapper.BinaryFieldMapper; import org.opensearch.index.mapper.NumberFieldMapper; +import org.opensearch.knn.index.mapper.KNNVectorFieldType; import java.math.BigInteger; import java.util.ArrayList; @@ -43,7 +43,7 @@ private void expectThrowsExceptionWithNonKNNField(Class clazz) throws NoSuchMeth private void expectThrowsExceptionWithKNNFieldWithBinaryDataType(Class clazz) throws NoSuchMethodException { Constructor constructor = clazz.getConstructor(Object.class, MappedFieldType.class); - KNNVectorFieldMapper.KNNVectorFieldType invalidFieldType = mock(KNNVectorFieldMapper.KNNVectorFieldType.class); + KNNVectorFieldType invalidFieldType = mock(KNNVectorFieldType.class); when(invalidFieldType.getVectorDataType()).thenReturn(VectorDataType.BINARY); Exception e = expectThrows(InvocationTargetException.class, () -> constructor.newInstance(null, invalidFieldType)); assertTrue(e.getCause() instanceof IllegalArgumentException); @@ -58,12 +58,7 @@ public void testL2_whenValid_thenSucceed() { float[] arrayFloat = new float[] { 1.0f, 2.0f, 3.0f }; List arrayListQueryObject = new ArrayList<>(Arrays.asList(1.0, 2.0, 3.0)); KNNMethodContext knnMethodContext = KNNMethodContext.getDefault(); - KNNVectorFieldMapper.KNNVectorFieldType fieldType = new KNNVectorFieldMapper.KNNVectorFieldType( - "test", - Collections.emptyMap(), - 3, - knnMethodContext - ); + KNNVectorFieldType fieldType = new KNNVectorFieldType("test", Collections.emptyMap(), 3, knnMethodContext); KNNScoringSpace.L2 l2 = new KNNScoringSpace.L2(arrayListQueryObject, fieldType); assertEquals(1F, l2.getScoringMethod().apply(arrayFloat, arrayFloat), 0.1F); } @@ -80,12 +75,7 @@ public void testCosineSimilarity_whenValid_thenSucceed() { float[] arrayFloat2 = new float[] { 2.0f, 4.0f, 6.0f }; KNNMethodContext knnMethodContext = KNNMethodContext.getDefault(); - KNNVectorFieldMapper.KNNVectorFieldType fieldType = new KNNVectorFieldMapper.KNNVectorFieldType( - "test", - Collections.emptyMap(), - 3, - knnMethodContext - ); + KNNVectorFieldType fieldType = new KNNVectorFieldType("test", Collections.emptyMap(), 3, knnMethodContext); KNNScoringSpace.CosineSimilarity cosineSimilarity = new KNNScoringSpace.CosineSimilarity(arrayListQueryObject, fieldType); assertEquals(2F, cosineSimilarity.getScoringMethod().apply(arrayFloat2, arrayFloat), 0.1F); @@ -103,12 +93,7 @@ public void testCosineSimilarity_whenValid_thenSucceed() { public void testCosineSimilarity_whenZeroVector_thenException() { KNNMethodContext knnMethodContext = KNNMethodContext.getDefault(); - KNNVectorFieldMapper.KNNVectorFieldType fieldType = new KNNVectorFieldMapper.KNNVectorFieldType( - "test", - Collections.emptyMap(), - 3, - knnMethodContext - ); + KNNVectorFieldType fieldType = new KNNVectorFieldType("test", Collections.emptyMap(), 3, knnMethodContext); final List queryZeroVector = List.of(0.0f, 0.0f, 0.0f); IllegalArgumentException exception1 = expectThrows( @@ -133,12 +118,7 @@ public void testInnerProd_whenValid_thenSucceed() { float[] arrayFloat2_case1 = new float[] { 1.0f, 1.0f, 1.0f }; KNNMethodContext knnMethodContext = KNNMethodContext.getDefault(); - KNNVectorFieldMapper.KNNVectorFieldType fieldType = new KNNVectorFieldMapper.KNNVectorFieldType( - "test", - Collections.emptyMap(), - 3, - knnMethodContext - ); + KNNVectorFieldType fieldType = new KNNVectorFieldType("test", Collections.emptyMap(), 3, knnMethodContext); KNNScoringSpace.InnerProd innerProd = new KNNScoringSpace.InnerProd(arrayListQueryObject_case1, fieldType); assertEquals(7.0F, innerProd.getScoringMethod().apply(arrayFloat_case1, arrayFloat2_case1), 0.001F); @@ -204,7 +184,7 @@ public void testHammingBit_Base64() { public void testHamming_whenKNNFieldType_thenSucceed() { List arrayListQueryObject = new ArrayList<>(Arrays.asList(1.0, 2.0, 3.0)); KNNMethodContext knnMethodContext = KNNMethodContext.getDefault(); - KNNVectorFieldMapper.KNNVectorFieldType fieldType = new KNNVectorFieldMapper.KNNVectorFieldType( + KNNVectorFieldType fieldType = new KNNVectorFieldType( "test", Collections.emptyMap(), 8 * arrayListQueryObject.size(), @@ -218,7 +198,7 @@ public void testHamming_whenKNNFieldType_thenSucceed() { } public void testHamming_whenNonBinaryVectorDataType_thenException() { - KNNVectorFieldMapper.KNNVectorFieldType invalidFieldType = mock(KNNVectorFieldMapper.KNNVectorFieldType.class); + KNNVectorFieldType invalidFieldType = mock(KNNVectorFieldType.class); when(invalidFieldType.getVectorDataType()).thenReturn(randomInt() % 2 == 0 ? VectorDataType.FLOAT : VectorDataType.BYTE); Exception e = expectThrows(IllegalArgumentException.class, () -> new KNNScoringSpace.Hamming(null, invalidFieldType)); assertTrue(e.getMessage(), e.getMessage().contains("The data type should be [BINARY]")); diff --git a/src/test/java/org/opensearch/knn/plugin/script/KNNScoringSpaceUtilTests.java b/src/test/java/org/opensearch/knn/plugin/script/KNNScoringSpaceUtilTests.java index ace3dabc8..781ed2350 100644 --- a/src/test/java/org/opensearch/knn/plugin/script/KNNScoringSpaceUtilTests.java +++ b/src/test/java/org/opensearch/knn/plugin/script/KNNScoringSpaceUtilTests.java @@ -7,9 +7,9 @@ import org.opensearch.knn.KNNTestCase; import org.opensearch.knn.index.VectorDataType; -import org.opensearch.knn.index.mapper.KNNVectorFieldMapper; import org.opensearch.index.mapper.BinaryFieldMapper; import org.opensearch.index.mapper.NumberFieldMapper; +import org.opensearch.knn.index.mapper.KNNVectorFieldType; import java.math.BigInteger; import java.util.ArrayList; @@ -32,7 +32,7 @@ public void testFieldTypeCheck() { KNNScoringSpaceUtil.isBinaryFieldType(new NumberFieldMapper.NumberFieldType("field", NumberFieldMapper.NumberType.INTEGER)) ); - assertTrue(KNNScoringSpaceUtil.isKNNVectorFieldType(mock(KNNVectorFieldMapper.KNNVectorFieldType.class))); + assertTrue(KNNScoringSpaceUtil.isKNNVectorFieldType(mock(KNNVectorFieldType.class))); assertFalse(KNNScoringSpaceUtil.isKNNVectorFieldType(new BinaryFieldMapper.BinaryFieldType("test"))); } @@ -62,7 +62,7 @@ public void testParseKNNVectorQuery() { float[] arrayFloat = new float[] { 1.0f, 2.0f, 3.0f }; List arrayListQueryObject = new ArrayList<>(Arrays.asList(1.0, 2.0, 3.0)); - KNNVectorFieldMapper.KNNVectorFieldType fieldType = mock(KNNVectorFieldMapper.KNNVectorFieldType.class); + KNNVectorFieldType fieldType = mock(KNNVectorFieldType.class); when(fieldType.getDimension()).thenReturn(3); assertArrayEquals(arrayFloat, KNNScoringSpaceUtil.parseToFloatArray(arrayListQueryObject, 3, VectorDataType.FLOAT), 0.1f); @@ -77,13 +77,13 @@ public void testParseKNNVectorQuery() { } public void testIsBinaryVectorDataType_whenBinary_thenReturnTrue() { - KNNVectorFieldMapper.KNNVectorFieldType fieldType = mock(KNNVectorFieldMapper.KNNVectorFieldType.class); + KNNVectorFieldType fieldType = mock(KNNVectorFieldType.class); when(fieldType.getVectorDataType()).thenReturn(VectorDataType.BINARY); assertTrue(KNNScoringSpaceUtil.isBinaryVectorDataType(fieldType)); } public void testIsBinaryVectorDataType_whenNonBinary_thenReturnFalse() { - KNNVectorFieldMapper.KNNVectorFieldType fieldType = mock(KNNVectorFieldMapper.KNNVectorFieldType.class); + KNNVectorFieldType fieldType = mock(KNNVectorFieldType.class); when(fieldType.getVectorDataType()).thenReturn(randomInt() % 2 == 0 ? VectorDataType.FLOAT : VectorDataType.BYTE); assertFalse(KNNScoringSpaceUtil.isBinaryVectorDataType(fieldType)); } From 4b5e2107855e4ab337439b2b35823784e88fd149 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Thu, 8 Aug 2024 09:59:50 -0700 Subject: [PATCH 318/416] Increment version to 2.17.0-SNAPSHOT (#1922) Signed-off-by: opensearch-ci-bot Co-authored-by: opensearch-ci-bot --- .../workflows/backwards_compatibility_tests_workflow.yml | 8 ++++---- build.gradle | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/backwards_compatibility_tests_workflow.yml b/.github/workflows/backwards_compatibility_tests_workflow.yml index 6b8a71bde..e8702820e 100644 --- a/.github/workflows/backwards_compatibility_tests_workflow.yml +++ b/.github/workflows/backwards_compatibility_tests_workflow.yml @@ -34,8 +34,8 @@ jobs: strategy: matrix: java: [ 11, 17 ] - bwc_version : [ "1.1.0", "1.2.4", "1.3.8", "2.0.1", "2.1.0", "2.2.1", "2.3.0", "2.4.1", "2.5.0", "2.6.0", "2.7.0", "2.8.0", "2.9.0", "2.10.0", "2.11.0", "2.12.0", "2.13.0", "2.14.0", "2.15.0" ] - opensearch_version : [ "2.16.0-SNAPSHOT" ] + bwc_version : [ "1.1.0", "1.2.4", "1.3.8", "2.0.1", "2.1.0", "2.2.1", "2.3.0", "2.4.1", "2.5.0", "2.6.0", "2.7.0", "2.8.0", "2.9.0", "2.10.0", "2.11.0", "2.12.0", "2.13.0", "2.14.0", "2.15.0", "2.16.0" ] + opensearch_version : [ "2.17.0-SNAPSHOT" ] name: k-NN Restart-Upgrade BWC Tests runs-on: ubuntu-latest @@ -72,8 +72,8 @@ jobs: strategy: matrix: java: [ 11, 17 ] - bwc_version: [ "1.3.8", "2.0.1", "2.1.0", "2.2.1", "2.3.0", "2.4.1", "2.5.0", "2.6.0", "2.7.0", "2.8.0", "2.9.0", "2.10.0", "2.11.0", "2.12.0", "2.13.0", "2.14.0", "2.15.0"] - opensearch_version: [ "2.16.0-SNAPSHOT" ] + bwc_version: [ "1.3.8", "2.0.1", "2.1.0", "2.2.1", "2.3.0", "2.4.1", "2.5.0", "2.6.0", "2.7.0", "2.8.0", "2.9.0", "2.10.0", "2.11.0", "2.12.0", "2.13.0", "2.14.0", "2.15.0", "2.16.0"] + opensearch_version: [ "2.17.0-SNAPSHOT" ] name: k-NN Rolling-Upgrade BWC Tests runs-on: ubuntu-latest diff --git a/build.gradle b/build.gradle index 73f11933b..58226a38d 100644 --- a/build.gradle +++ b/build.gradle @@ -13,7 +13,7 @@ buildscript { ext { // build.version_qualifier parameter applies to knn plugin artifacts only. OpenSearch version must be set // explicitly as 'opensearch.version' property, for instance opensearch.version=2.0.0-rc1-SNAPSHOT - opensearch_version = System.getProperty("opensearch.version", "2.16.0-SNAPSHOT") + opensearch_version = System.getProperty("opensearch.version", "2.17.0-SNAPSHOT") version_qualifier = System.getProperty("build.version_qualifier", "") opensearch_group = "org.opensearch" isSnapshot = "true" == System.getProperty("build.snapshot", "true") From 05e9779a1c59a40e62f74974bdfbaab5cfc83aa7 Mon Sep 17 00:00:00 2001 From: Ryan Bogan Date: Fri, 9 Aug 2024 12:03:34 -0700 Subject: [PATCH 319/416] Fix graph merge stats size calculation (#1844) (#1941) * Fix graph merge stats size calculation Signed-off-by: Ryan Bogan * Add changelog entry Signed-off-by: Ryan Bogan * Add javadocs Signed-off-by: Ryan Bogan * Make calculations easier to read Signed-off-by: Ryan Bogan * Remove java overhead from calculations Signed-off-by: Ryan Bogan * Change from serialization mode to vector data type for calculations Signed-off-by: Ryan Bogan * Minor change to if statements Signed-off-by: Ryan Bogan --------- Signed-off-by: Ryan Bogan (cherry picked from commit e3158f990d058b02568da617688fd4857d0d521b) --- CHANGELOG.md | 1 + .../KNN80Codec/KNN80DocValuesConsumer.java | 7 +-- .../knn/index/codec/util/KNNCodecUtil.java | 54 ++++++------------- .../index/codec/util/KNNCodecUtilTests.java | 19 +++++++ 4 files changed, 40 insertions(+), 41 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6c761d27f..f9d715823 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Enhancements ### Bug Fixes * Corrected search logic for scenario with non-existent fields in filter [#1874](https://github.com/opensearch-project/k-NN/pull/1874) +* Fix graph merge stats size calculation [#1844](https://github.com/opensearch-project/k-NN/pull/1844) ### Infrastructure ### Documentation ### Maintenance diff --git a/src/main/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumer.java b/src/main/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumer.java index 31862c25d..7dad3a8fd 100644 --- a/src/main/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumer.java +++ b/src/main/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumer.java @@ -129,6 +129,7 @@ public void addKNNBinaryField(FieldInfo field, DocValuesProducer valuesProducer, NativeIndexCreator indexCreator; KNNCodecUtil.Pair pair; Map fieldAttributes = field.attributes(); + VectorDataType vectorDataType; if (fieldAttributes.containsKey(MODEL_ID)) { String modelId = fieldAttributes.get(MODEL_ID); @@ -136,12 +137,12 @@ public void addKNNBinaryField(FieldInfo field, DocValuesProducer valuesProducer, if (model.getModelBlob() == null) { throw new RuntimeException(String.format("There is no trained model with id \"%s\"", modelId)); } - VectorDataType vectorDataType = model.getModelMetadata().getVectorDataType(); + vectorDataType = model.getModelMetadata().getVectorDataType(); pair = KNNCodecUtil.getPair(values, getVectorTransfer(vectorDataType)); indexCreator = () -> createKNNIndexFromTemplate(model, pair, knnEngine, indexPath); } else { // get vector data type from field attributes or provide default value - VectorDataType vectorDataType = VectorDataType.get( + vectorDataType = VectorDataType.get( fieldAttributes.getOrDefault(KNNConstants.VECTOR_DATA_TYPE_FIELD, VectorDataType.DEFAULT.getValue()) ); pair = KNNCodecUtil.getPair(values, getVectorTransfer(vectorDataType)); @@ -154,7 +155,7 @@ public void addKNNBinaryField(FieldInfo field, DocValuesProducer valuesProducer, return; } - long arraySize = calculateArraySize(pair.docs.length, pair.getDimension(), pair.serializationMode); + long arraySize = calculateArraySize(pair.docs.length, pair.getDimension(), vectorDataType); if (isMerge) { KNNGraphValue.MERGE_CURRENT_OPERATIONS.increment(); diff --git a/src/main/java/org/opensearch/knn/index/codec/util/KNNCodecUtil.java b/src/main/java/org/opensearch/knn/index/codec/util/KNNCodecUtil.java index d208d8179..ea14fe883 100644 --- a/src/main/java/org/opensearch/knn/index/codec/util/KNNCodecUtil.java +++ b/src/main/java/org/opensearch/knn/index/codec/util/KNNCodecUtil.java @@ -11,6 +11,7 @@ import org.apache.lucene.index.BinaryDocValues; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.util.BytesRef; +import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.codec.KNN80Codec.KNN80BinaryDocValues; import org.opensearch.knn.index.codec.transfer.VectorTransfer; @@ -21,12 +22,6 @@ public class KNNCodecUtil { // Floats are 4 bytes in size public static final int FLOAT_BYTE_SIZE = 4; - // References to objects are 4 bytes in size - public static final int JAVA_REFERENCE_SIZE = 4; - // Each array in Java has a header that is 12 bytes - public static final int JAVA_ARRAY_HEADER_SIZE = 12; - // Java rounds each array size up to multiples of 8 bytes - public static final int JAVA_ROUNDING_NUMBER = 8; @AllArgsConstructor public static final class Pair { @@ -67,39 +62,22 @@ public static KNNCodecUtil.Pair getPair(final BinaryDocValues values, final Vect ); } - public static long calculateArraySize(int numVectors, int vectorLength, SerializationMode serializationMode) { - if (serializationMode == SerializationMode.ARRAY) { - int vectorSize = vectorLength * FLOAT_BYTE_SIZE + JAVA_ARRAY_HEADER_SIZE; - if (vectorSize % JAVA_ROUNDING_NUMBER != 0) { - vectorSize += vectorSize % JAVA_ROUNDING_NUMBER; - } - int vectorsSize = numVectors * (vectorSize + JAVA_REFERENCE_SIZE) + JAVA_ARRAY_HEADER_SIZE; - if (vectorsSize % JAVA_ROUNDING_NUMBER != 0) { - vectorsSize += vectorsSize % JAVA_ROUNDING_NUMBER; - } - return vectorsSize; - } else if (serializationMode == SerializationMode.COLLECTION_OF_FLOATS) { - int vectorSize = vectorLength * FLOAT_BYTE_SIZE; - if (vectorSize % JAVA_ROUNDING_NUMBER != 0) { - vectorSize += vectorSize % JAVA_ROUNDING_NUMBER; - } - int vectorsSize = numVectors * (vectorSize + JAVA_REFERENCE_SIZE); - if (vectorsSize % JAVA_ROUNDING_NUMBER != 0) { - vectorsSize += vectorsSize % JAVA_ROUNDING_NUMBER; - } - return vectorsSize; - } else if (serializationMode == SerializationMode.COLLECTIONS_OF_BYTES) { - int vectorSize = vectorLength; - if (vectorSize % JAVA_ROUNDING_NUMBER != 0) { - vectorSize += vectorSize % JAVA_ROUNDING_NUMBER; - } - int vectorsSize = numVectors * (vectorSize + JAVA_REFERENCE_SIZE); - if (vectorsSize % JAVA_ROUNDING_NUMBER != 0) { - vectorsSize += vectorsSize % JAVA_ROUNDING_NUMBER; - } - return vectorsSize; + /** + * This method provides a rough estimate of the number of bytes used for storing an array with the given parameters. + * @param numVectors number of vectors in the array + * @param vectorLength the length of each vector + * @param vectorDataType type of data stored in each vector + * @return rough estimate of number of bytes used to store an array with the given parameters + */ + public static long calculateArraySize(int numVectors, int vectorLength, VectorDataType vectorDataType) { + if (vectorDataType == VectorDataType.FLOAT) { + return numVectors * vectorLength * FLOAT_BYTE_SIZE; + } else if (vectorDataType == VectorDataType.BINARY || vectorDataType == VectorDataType.BYTE) { + return numVectors * vectorLength; } else { - throw new IllegalStateException("Unreachable code"); + throw new IllegalArgumentException( + "Float, binary, and byte are the only supported vector data types for array size calculation." + ); } } diff --git a/src/test/java/org/opensearch/knn/index/codec/util/KNNCodecUtilTests.java b/src/test/java/org/opensearch/knn/index/codec/util/KNNCodecUtilTests.java index 2ff0f08e5..47dd1dda9 100644 --- a/src/test/java/org/opensearch/knn/index/codec/util/KNNCodecUtilTests.java +++ b/src/test/java/org/opensearch/knn/index/codec/util/KNNCodecUtilTests.java @@ -9,6 +9,7 @@ import lombok.SneakyThrows; import org.apache.lucene.index.BinaryDocValues; import org.apache.lucene.util.BytesRef; +import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.codec.transfer.VectorTransfer; import java.util.Arrays; @@ -18,6 +19,7 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import static org.opensearch.knn.index.codec.util.KNNCodecUtil.calculateArraySize; public class KNNCodecUtilTests extends TestCase { @SneakyThrows @@ -52,4 +54,21 @@ public void testGetPair_whenCalled_thenReturn() { assertEquals(dimension, pair.getDimension()); assertEquals(SerializationMode.COLLECTIONS_OF_BYTES, pair.serializationMode); } + + public void testCalculateArraySize() { + int numVectors = 4; + int vectorLength = 10; + + // Float data type + VectorDataType vectorDataType = VectorDataType.FLOAT; + assertEquals(160, calculateArraySize(numVectors, vectorLength, vectorDataType)); + + // Byte data type + vectorDataType = VectorDataType.BYTE; + assertEquals(40, calculateArraySize(numVectors, vectorLength, vectorDataType)); + + // Binary data type + vectorDataType = VectorDataType.BINARY; + assertEquals(40, calculateArraySize(numVectors, vectorLength, vectorDataType)); + } } From 20039d0ad5ee99f8e21ae2042cf1b97ece80f6bd Mon Sep 17 00:00:00 2001 From: Tejas Shah Date: Fri, 9 Aug 2024 12:48:38 -0700 Subject: [PATCH 320/416] Introduces NativeEngineKNNQuery which executes ANN on rewrite (#1877) (#1943) Signed-off-by: Tejas Shah (cherry picked from commit df7627c6e580843beb2361f5c2ec3519efd52280) --- CHANGELOG.md | 1 + .../common/featureflags/KNNFeatureFlags.java | 46 +++++ .../org/opensearch/knn/index/KNNSettings.java | 20 +- .../knn/index/query/KNNQueryFactory.java | 9 +- .../opensearch/knn/index/query/KNNScorer.java | 7 + .../opensearch/knn/index/query/KNNWeight.java | 24 ++- .../query/nativelib/DocAndScoreQuery.java | 185 +++++++++++++++++ .../nativelib/NativeEngineKnnVectorQuery.java | 140 +++++++++++++ .../featureflags/KNNFeatureFlagsTests.java | 34 ++++ .../knn/index/query/KNNQueryBuilderTests.java | 14 ++ .../knn/index/query/KNNQueryFactoryTests.java | 34 ++++ .../knn/index/query/KNNWeightTests.java | 4 +- .../nativelib/DocAndScoreQueryTests.java | 99 +++++++++ .../NativeEngineKNNVectorQueryIT.java | 190 ++++++++++++++++++ .../NativeEngineKNNVectorQueryTests.java | 156 ++++++++++++++ 15 files changed, 950 insertions(+), 13 deletions(-) create mode 100644 src/main/java/org/opensearch/knn/common/featureflags/KNNFeatureFlags.java create mode 100644 src/main/java/org/opensearch/knn/index/query/nativelib/DocAndScoreQuery.java create mode 100644 src/main/java/org/opensearch/knn/index/query/nativelib/NativeEngineKnnVectorQuery.java create mode 100644 src/test/java/org/opensearch/knn/common/featureflags/KNNFeatureFlagsTests.java create mode 100644 src/test/java/org/opensearch/knn/index/query/nativelib/DocAndScoreQueryTests.java create mode 100644 src/test/java/org/opensearch/knn/index/query/nativelib/NativeEngineKNNVectorQueryIT.java create mode 100644 src/test/java/org/opensearch/knn/index/query/nativelib/NativeEngineKNNVectorQueryTests.java diff --git a/CHANGELOG.md b/CHANGELOG.md index f9d715823..e10f3e065 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,3 +29,4 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), * Refactor method structure and definitions [#1920](https://github.com/opensearch-project/k-NN/pull/1920) * Refactor KNNVectorFieldType from KNNVectorFieldMapper to a separate class for better readability. [#1931](https://github.com/opensearch-project/k-NN/pull/1931) * Generalize lib interface to return context objects [#1925](https://github.com/opensearch-project/k-NN/pull/1925) +* Move k search k-NN query to re-write phase of vector search query for Native Engines [#1877](https://github.com/opensearch-project/k-NN/pull/1877) \ No newline at end of file diff --git a/src/main/java/org/opensearch/knn/common/featureflags/KNNFeatureFlags.java b/src/main/java/org/opensearch/knn/common/featureflags/KNNFeatureFlags.java new file mode 100644 index 000000000..21160fc2d --- /dev/null +++ b/src/main/java/org/opensearch/knn/common/featureflags/KNNFeatureFlags.java @@ -0,0 +1,46 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.opensearch.knn.common.featureflags; + +import com.google.common.annotations.VisibleForTesting; +import lombok.experimental.UtilityClass; +import org.opensearch.common.settings.Setting; +import org.opensearch.knn.index.KNNSettings; + +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.opensearch.common.settings.Setting.Property.Dynamic; +import static org.opensearch.common.settings.Setting.Property.NodeScope; + +/** + * Class to manage KNN feature flags + */ +@UtilityClass +public class KNNFeatureFlags { + + // Feature flags + private static final String KNN_LAUNCH_QUERY_REWRITE_ENABLED = "knn.feature.query.rewrite.enabled"; + private static final boolean KNN_LAUNCH_QUERY_REWRITE_ENABLED_DEFAULT = true; + + @VisibleForTesting + public static final Setting KNN_LAUNCH_QUERY_REWRITE_ENABLED_SETTING = Setting.boolSetting( + KNN_LAUNCH_QUERY_REWRITE_ENABLED, + KNN_LAUNCH_QUERY_REWRITE_ENABLED_DEFAULT, + NodeScope, + Dynamic + ); + + public static List> getFeatureFlags() { + return Stream.of(KNN_LAUNCH_QUERY_REWRITE_ENABLED_SETTING).collect(Collectors.toUnmodifiableList()); + } + + public static boolean isKnnQueryRewriteEnabled() { + return Boolean.parseBoolean(KNNSettings.state().getSettingValue(KNN_LAUNCH_QUERY_REWRITE_ENABLED).toString()); + } +} diff --git a/src/main/java/org/opensearch/knn/index/KNNSettings.java b/src/main/java/org/opensearch/knn/index/KNNSettings.java index 3279e74bc..33c7ff410 100644 --- a/src/main/java/org/opensearch/knn/index/KNNSettings.java +++ b/src/main/java/org/opensearch/knn/index/KNNSettings.java @@ -9,15 +9,15 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.opensearch.OpenSearchParseException; -import org.opensearch.cluster.metadata.IndexMetadata; -import org.opensearch.core.action.ActionListener; import org.opensearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest; import org.opensearch.action.admin.cluster.settings.ClusterUpdateSettingsResponse; import org.opensearch.client.Client; +import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; +import org.opensearch.core.action.ActionListener; import org.opensearch.core.common.unit.ByteSizeUnit; import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.index.IndexModule; @@ -28,20 +28,22 @@ import org.opensearch.monitor.os.OsProbe; import java.security.InvalidParameterException; -import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.Stream; +import static java.util.stream.Collectors.toUnmodifiableMap; import static org.opensearch.common.settings.Setting.Property.Dynamic; import static org.opensearch.common.settings.Setting.Property.IndexScope; import static org.opensearch.common.settings.Setting.Property.NodeScope; import static org.opensearch.common.unit.MemorySizeValue.parseBytesSizeValueOrHeapRatio; import static org.opensearch.core.common.unit.ByteSizeValue.parseBytesSizeValue; +import static org.opensearch.knn.common.featureflags.KNNFeatureFlags.getFeatureFlags; /** * This class defines @@ -289,6 +291,9 @@ public class KNNSettings { } }; + private final static Map> FEATURE_FLAGS = getFeatureFlags().stream() + .collect(toUnmodifiableMap(Setting::getKey, Function.identity())); + private ClusterService clusterService; private Client client; @@ -326,7 +331,7 @@ private void setSettingsUpdateConsumers() { ); NativeMemoryCacheManager.getInstance().rebuildCache(builder.build()); - }, new ArrayList<>(dynamicCacheSettings.values())); + }, Stream.concat(dynamicCacheSettings.values().stream(), FEATURE_FLAGS.values().stream()).collect(Collectors.toUnmodifiableList())); } /** @@ -346,6 +351,10 @@ private Setting getSetting(String key) { return dynamicCacheSettings.get(key); } + if (FEATURE_FLAGS.containsKey(key)) { + return FEATURE_FLAGS.get(key); + } + if (KNN_CIRCUIT_BREAKER_TRIGGERED.equals(key)) { return KNN_CIRCUIT_BREAKER_TRIGGERED_SETTING; } @@ -390,7 +399,8 @@ public List> getSettings() { KNN_FAISS_AVX2_DISABLED_SETTING, KNN_VECTOR_STREAMING_MEMORY_LIMIT_PCT_SETTING ); - return Stream.concat(settings.stream(), dynamicCacheSettings.values().stream()).collect(Collectors.toList()); + return Stream.concat(settings.stream(), Stream.concat(getFeatureFlags().stream(), dynamicCacheSettings.values().stream())) + .collect(Collectors.toList()); } public static boolean isKNNPluginEnabled() { diff --git a/src/main/java/org/opensearch/knn/index/query/KNNQueryFactory.java b/src/main/java/org/opensearch/knn/index/query/KNNQueryFactory.java index ee9a12a41..f3161b2db 100644 --- a/src/main/java/org/opensearch/knn/index/query/KNNQueryFactory.java +++ b/src/main/java/org/opensearch/knn/index/query/KNNQueryFactory.java @@ -16,12 +16,14 @@ import org.opensearch.index.query.QueryShardContext; import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.engine.KNNEngine; +import org.opensearch.knn.index.query.nativelib.NativeEngineKnnVectorQuery; import java.util.Locale; import java.util.Map; import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_EF_SEARCH; import static org.opensearch.knn.common.KNNConstants.VECTOR_DATA_TYPE_FIELD; +import static org.opensearch.knn.common.featureflags.KNNFeatureFlags.isKnnQueryRewriteEnabled; import static org.opensearch.knn.index.VectorDataType.SUPPORTED_VECTOR_DATA_TYPES; /** @@ -98,9 +100,10 @@ public static Query create(CreateQueryRequest createQueryRequest) { methodParameters ); + KNNQuery knnQuery = null; switch (vectorDataType) { case BINARY: - return KNNQuery.builder() + knnQuery = KNNQuery.builder() .field(fieldName) .byteQueryVector(byteVector) .indexName(indexName) @@ -110,8 +113,9 @@ public static Query create(CreateQueryRequest createQueryRequest) { .filterQuery(validatedFilterQuery) .vectorDataType(vectorDataType) .build(); + break; default: - return KNNQuery.builder() + knnQuery = KNNQuery.builder() .field(fieldName) .queryVector(vector) .indexName(indexName) @@ -122,6 +126,7 @@ public static Query create(CreateQueryRequest createQueryRequest) { .vectorDataType(vectorDataType) .build(); } + return isKnnQueryRewriteEnabled() ? new NativeEngineKnnVectorQuery(knnQuery) : knnQuery; } Integer requestEfSearch = null; diff --git a/src/main/java/org/opensearch/knn/index/query/KNNScorer.java b/src/main/java/org/opensearch/knn/index/query/KNNScorer.java index 02dc86e80..99962d307 100644 --- a/src/main/java/org/opensearch/knn/index/query/KNNScorer.java +++ b/src/main/java/org/opensearch/knn/index/query/KNNScorer.java @@ -87,6 +87,13 @@ public float score() throws IOException { public int docID() { return docIdsIter.docID(); } + + @Override + public boolean equals(Object obj) { + if (!(obj instanceof Scorer)) return false; + return getWeight().equals(((Scorer) obj).getWeight()); + } }; + } } diff --git a/src/main/java/org/opensearch/knn/index/query/KNNWeight.java b/src/main/java/org/opensearch/knn/index/query/KNNWeight.java index f54d8328e..f88652525 100644 --- a/src/main/java/org/opensearch/knn/index/query/KNNWeight.java +++ b/src/main/java/org/opensearch/knn/index/query/KNNWeight.java @@ -108,6 +108,22 @@ public Explanation explain(LeafReaderContext context, int doc) { @Override public Scorer scorer(LeafReaderContext context) throws IOException { + final Map docIdToScoreMap = searchLeaf(context); + if (docIdToScoreMap.isEmpty()) { + return KNNScorer.emptyScorer(this); + } + + return convertSearchResponseToScorer(docIdToScoreMap); + } + + /** + * Executes k nearest neighbor search for a segment to get the top K results + * This is made public purely to be able to be reused in {@link org.opensearch.knn.index.query.nativelib.NativeEngineKnnVectorQuery} + * + * @param context LeafReaderContext + * @return A Map of docId to scores for top k results + */ + public Map searchLeaf(LeafReaderContext context) throws IOException { final BitSet filterBitSet = getFilteredDocsBitSet(context); int cardinality = filterBitSet.cardinality(); @@ -115,7 +131,7 @@ public Scorer scorer(LeafReaderContext context) throws IOException { // We should give this condition a deeper look that where it should be placed. For now I feel this is a good // place, if (filterWeight != null && cardinality == 0) { - return KNNScorer.emptyScorer(this); + return Collections.emptyMap(); } final Map docIdsToScoreMap = new HashMap<>(); @@ -129,7 +145,7 @@ public Scorer scorer(LeafReaderContext context) throws IOException { } else { Map annResults = doANNSearch(context, filterBitSet, cardinality); if (annResults == null) { - return null; + return Collections.emptyMap(); } if (canDoExactSearchAfterANNSearch(cardinality, annResults.size())) { log.debug( @@ -144,9 +160,9 @@ public Scorer scorer(LeafReaderContext context) throws IOException { docIdsToScoreMap.putAll(annResults); } if (docIdsToScoreMap.isEmpty()) { - return KNNScorer.emptyScorer(this); + return Collections.emptyMap(); } - return convertSearchResponseToScorer(docIdsToScoreMap); + return docIdsToScoreMap; } private BitSet getFilteredDocsBitSet(final LeafReaderContext ctx) throws IOException { diff --git a/src/main/java/org/opensearch/knn/index/query/nativelib/DocAndScoreQuery.java b/src/main/java/org/opensearch/knn/index/query/nativelib/DocAndScoreQuery.java new file mode 100644 index 000000000..f1a91d878 --- /dev/null +++ b/src/main/java/org/opensearch/knn/index/query/nativelib/DocAndScoreQuery.java @@ -0,0 +1,185 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.query.nativelib; + +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.search.Explanation; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.QueryVisitor; +import org.apache.lucene.search.ScoreMode; +import org.apache.lucene.search.Scorer; +import org.apache.lucene.search.Weight; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Objects; + +import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS; + +/** + * This is the same as {@link org.apache.lucene.search.AbstractKnnVectorQuery.DocAndScoreQuery} + */ +final class DocAndScoreQuery extends Query { + + private final int k; + private final int[] docs; + private final float[] scores; + private final int[] segmentStarts; + private final Object contextIdentity; + + DocAndScoreQuery(int k, int[] docs, float[] scores, int[] segmentStarts, Object contextIdentity) { + this.k = k; + this.docs = docs; + this.scores = scores; + this.segmentStarts = segmentStarts; + this.contextIdentity = contextIdentity; + } + + @Override + public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) { + if (searcher.getIndexReader().getContext().id() != contextIdentity) { + throw new IllegalStateException("This DocAndScore query was created by a different reader"); + } + return new Weight(this) { + @Override + public Explanation explain(LeafReaderContext context, int doc) { + int found = Arrays.binarySearch(docs, doc + context.docBase); + if (found < 0) { + return Explanation.noMatch("not in top " + k); + } + return Explanation.match(scores[found] * boost, "within top " + k); + } + + @Override + public int count(LeafReaderContext context) { + return segmentStarts[context.ord + 1] - segmentStarts[context.ord]; + } + + @Override + public Scorer scorer(LeafReaderContext context) { + if (segmentStarts[context.ord] == segmentStarts[context.ord + 1]) { + return null; + } + return new Scorer(this) { + final int lower = segmentStarts[context.ord]; + final int upper = segmentStarts[context.ord + 1]; + int upTo = -1; + + @Override + public DocIdSetIterator iterator() { + return new DocIdSetIterator() { + @Override + public int docID() { + return docIdNoShadow(); + } + + @Override + public int nextDoc() { + if (upTo == -1) { + upTo = lower; + } else { + ++upTo; + } + return docIdNoShadow(); + } + + @Override + public int advance(int target) throws IOException { + return slowAdvance(target); + } + + @Override + public long cost() { + return upper - lower; + } + }; + } + + @Override + public float getMaxScore(int docId) { + docId += context.docBase; + float maxScore = 0; + for (int idx = Math.max(0, upTo); idx < upper && docs[idx] <= docId; idx++) { + maxScore = Math.max(maxScore, scores[idx]); + } + return maxScore * boost; + } + + @Override + public float score() { + return scores[upTo] * boost; + } + + @Override + public int advanceShallow(int docid) { + int start = Math.max(upTo, lower); + int docidIndex = Arrays.binarySearch(docs, start, upper, docid + context.docBase); + if (docidIndex < 0) { + docidIndex = -1 - docidIndex; + } + if (docidIndex >= upper) { + return NO_MORE_DOCS; + } + return docs[docidIndex]; + } + + /** + * move the implementation of docID() into a differently-named method so we can call it + * from DocIDSetIterator.docID() even though this class is anonymous + * + * @return the current docid + */ + private int docIdNoShadow() { + if (upTo == -1) { + return -1; + } + if (upTo >= upper) { + return NO_MORE_DOCS; + } + return docs[upTo] - context.docBase; + } + + @Override + public int docID() { + return docIdNoShadow(); + } + }; + } + + @Override + public boolean isCacheable(LeafReaderContext ctx) { + return true; + } + }; + } + + @Override + public String toString(String field) { + return "DocAndScore[" + k + "][docs:" + Arrays.toString(docs) + ", scores:" + Arrays.toString(scores) + "]"; + } + + @Override + public void visit(QueryVisitor visitor) { + visitor.visitLeaf(this); + } + + @Override + public boolean equals(Object obj) { + if (!sameClassAs(obj)) { + return false; + } + return contextIdentity == ((DocAndScoreQuery) obj).contextIdentity + && Arrays.equals(docs, ((DocAndScoreQuery) obj).docs) + && Arrays.equals(scores, ((DocAndScoreQuery) obj).scores); + } + + @Override + public int hashCode() { + return Objects.hash(classHash(), contextIdentity, Arrays.hashCode(docs), Arrays.hashCode(scores)); + } +} diff --git a/src/main/java/org/opensearch/knn/index/query/nativelib/NativeEngineKnnVectorQuery.java b/src/main/java/org/opensearch/knn/index/query/nativelib/NativeEngineKnnVectorQuery.java new file mode 100644 index 000000000..6b9a40a9c --- /dev/null +++ b/src/main/java/org/opensearch/knn/index/query/nativelib/NativeEngineKnnVectorQuery.java @@ -0,0 +1,140 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.query.nativelib; + +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.MatchNoDocsQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.QueryVisitor; +import org.apache.lucene.search.ScoreDoc; +import org.apache.lucene.search.ScoreMode; +import org.apache.lucene.search.TopDocs; +import org.apache.lucene.search.TotalHits; +import org.apache.lucene.util.Bits; +import org.opensearch.knn.index.query.KNNQuery; +import org.opensearch.knn.index.query.KNNWeight; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Comparator; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.Callable; + +/** + * {@link KNNQuery} executes approximate nearest neighbor search (ANN) on a segment level. + * {@link NativeEngineKnnVectorQuery} executes approximate nearest neighbor search but gives + * us the control to combine the top k results in each leaf and post process the results just + * for k-NN query if required. This is done by overriding rewrite method to execute ANN on each leaf + * {@link KNNQuery} does not give the ability to post process segment results. + */ +@Getter +@RequiredArgsConstructor +public class NativeEngineKnnVectorQuery extends Query { + + private final KNNQuery knnQuery; + + @Override + public Query rewrite(final IndexSearcher indexSearcher) throws IOException { + final IndexReader reader = indexSearcher.getIndexReader(); + final KNNWeight knnWeight = (KNNWeight) knnQuery.createWeight(indexSearcher, ScoreMode.COMPLETE, 1); + List leafReaderContexts = reader.leaves(); + + List> tasks = new ArrayList<>(leafReaderContexts.size()); + for (LeafReaderContext leafReaderContext : leafReaderContexts) { + tasks.add(() -> searchLeaf(leafReaderContext, knnWeight)); + } + TopDocs[] perLeafResults = indexSearcher.getTaskExecutor().invokeAll(tasks).toArray(TopDocs[]::new); + // TopDocs.merge requires perLeafResults to be sorted in descending order. + TopDocs topK = TopDocs.merge(knnQuery.getK(), perLeafResults); + if (topK.scoreDocs.length == 0) { + return new MatchNoDocsQuery(); + } + return createRewrittenQuery(reader, topK); + } + + private Query createRewrittenQuery(IndexReader reader, TopDocs topK) { + int len = topK.scoreDocs.length; + Arrays.sort(topK.scoreDocs, Comparator.comparingInt(a -> a.doc)); + int[] docs = new int[len]; + float[] scores = new float[len]; + for (int i = 0; i < len; i++) { + docs[i] = topK.scoreDocs[i].doc; + scores[i] = topK.scoreDocs[i].score; + } + int[] segmentStarts = findSegmentStarts(reader, docs); + return new DocAndScoreQuery(knnQuery.getK(), docs, scores, segmentStarts, reader.getContext().id()); + } + + private static int[] findSegmentStarts(IndexReader reader, int[] docs) { + int[] starts = new int[reader.leaves().size() + 1]; + starts[starts.length - 1] = docs.length; + if (starts.length == 2) { + return starts; + } + int resultIndex = 0; + for (int i = 1; i < starts.length - 1; i++) { + int upper = reader.leaves().get(i).docBase; + resultIndex = Arrays.binarySearch(docs, resultIndex, docs.length, upper); + if (resultIndex < 0) { + resultIndex = -1 - resultIndex; + } + starts[i] = resultIndex; + } + return starts; + } + + private TopDocs searchLeaf(LeafReaderContext ctx, KNNWeight queryWeight) throws IOException { + int totalHits = 0; + final Map leafDocScores = queryWeight.searchLeaf(ctx); + final List scoreDocs = new ArrayList<>(); + final Bits liveDocs = ctx.reader().getLiveDocs(); + + if (!leafDocScores.isEmpty()) { + final List> topScores = new ArrayList<>(leafDocScores.entrySet()); + topScores.sort(Map.Entry.comparingByValue().reversed()); + + for (Map.Entry entry : topScores) { + if (liveDocs == null || liveDocs.get(entry.getKey())) { + ScoreDoc scoreDoc = new ScoreDoc(entry.getKey() + ctx.docBase, entry.getValue()); + scoreDocs.add(scoreDoc); + totalHits++; + } + } + } + + return new TopDocs(new TotalHits(totalHits, TotalHits.Relation.EQUAL_TO), scoreDocs.toArray(ScoreDoc[]::new)); + } + + @Override + public String toString(String field) { + return this.getClass().getSimpleName() + "[" + field + "]..." + KNNQuery.class.getSimpleName() + "[" + knnQuery.toString() + "]"; + } + + @Override + public void visit(QueryVisitor visitor) { + visitor.visitLeaf(this); + } + + @Override + public boolean equals(Object obj) { + if (!sameClassAs(obj)) { + return false; + } + return knnQuery == ((NativeEngineKnnVectorQuery) obj).knnQuery; + } + + @Override + public int hashCode() { + return Objects.hash(classHash(), knnQuery.hashCode()); + } +} diff --git a/src/test/java/org/opensearch/knn/common/featureflags/KNNFeatureFlagsTests.java b/src/test/java/org/opensearch/knn/common/featureflags/KNNFeatureFlagsTests.java new file mode 100644 index 000000000..c3a8a1615 --- /dev/null +++ b/src/test/java/org/opensearch/knn/common/featureflags/KNNFeatureFlagsTests.java @@ -0,0 +1,34 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.common.featureflags; + +import org.mockito.Mock; +import org.opensearch.common.settings.ClusterSettings; +import org.opensearch.knn.KNNTestCase; +import org.opensearch.knn.index.KNNSettings; + +import static org.mockito.Mockito.when; +import static org.opensearch.knn.common.featureflags.KNNFeatureFlags.KNN_LAUNCH_QUERY_REWRITE_ENABLED_SETTING; +import static org.opensearch.knn.common.featureflags.KNNFeatureFlags.isKnnQueryRewriteEnabled; + +public class KNNFeatureFlagsTests extends KNNTestCase { + + @Mock + ClusterSettings clusterSettings; + + public void setUp() throws Exception { + super.setUp(); + when(clusterService.getClusterSettings()).thenReturn(clusterSettings); + KNNSettings.state().setClusterService(clusterService); + } + + public void testIsFeatureEnabled() throws Exception { + when(clusterSettings.get(KNN_LAUNCH_QUERY_REWRITE_ENABLED_SETTING)).thenReturn(false); + assertFalse(isKnnQueryRewriteEnabled()); + when(clusterSettings.get(KNN_LAUNCH_QUERY_REWRITE_ENABLED_SETTING)).thenReturn(true); + assertTrue(isKnnQueryRewriteEnabled()); + } +} diff --git a/src/test/java/org/opensearch/knn/index/query/KNNQueryBuilderTests.java b/src/test/java/org/opensearch/knn/index/query/KNNQueryBuilderTests.java index 0241a9afb..0b918bd9e 100644 --- a/src/test/java/org/opensearch/knn/index/query/KNNQueryBuilderTests.java +++ b/src/test/java/org/opensearch/knn/index/query/KNNQueryBuilderTests.java @@ -11,10 +11,12 @@ import org.apache.lucene.search.KnnFloatVectorQuery; import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.Query; +import org.junit.Before; import org.opensearch.Version; import org.opensearch.cluster.ClusterModule; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.common.settings.ClusterSettings; import org.opensearch.core.common.io.stream.NamedWriteableAwareStreamInput; import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.core.common.io.stream.StreamInput; @@ -31,6 +33,7 @@ import org.opensearch.knn.index.util.KNNClusterUtil; import org.opensearch.knn.index.engine.KNNMethodContext; import org.opensearch.knn.index.engine.MethodComponentContext; +import org.opensearch.knn.index.KNNSettings; import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.engine.KNNEngine; @@ -51,6 +54,7 @@ import static org.mockito.Mockito.anyString; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; +import static org.opensearch.knn.common.featureflags.KNNFeatureFlags.KNN_LAUNCH_QUERY_REWRITE_ENABLED_SETTING; import static org.opensearch.knn.index.KNNClusterTestUtils.mockClusterService; import static org.opensearch.knn.index.engine.KNNEngine.ENGINES_SUPPORTING_RADIAL_SEARCH; @@ -67,6 +71,16 @@ public class KNNQueryBuilderTests extends KNNTestCase { protected static final String TEXT_FIELD_NAME = "some_field"; protected static final String TEXT_VALUE = "some_value"; + @Before + @Override + public void setUp() throws Exception { + super.setUp(); + ClusterSettings clusterSettings = mock(ClusterSettings.class); + when(clusterService.getClusterSettings()).thenReturn(clusterSettings); + when(clusterSettings.get(KNN_LAUNCH_QUERY_REWRITE_ENABLED_SETTING)).thenReturn(false); + KNNSettings.state().setClusterService(clusterService); + } + public void testInvalidK() { float[] queryVector = { 1.0f, 1.0f }; diff --git a/src/test/java/org/opensearch/knn/index/query/KNNQueryFactoryTests.java b/src/test/java/org/opensearch/knn/index/query/KNNQueryFactoryTests.java index c74a79946..7bacc7d10 100644 --- a/src/test/java/org/opensearch/knn/index/query/KNNQueryFactoryTests.java +++ b/src/test/java/org/opensearch/knn/index/query/KNNQueryFactoryTests.java @@ -14,8 +14,11 @@ import org.apache.lucene.search.join.DiversifyingChildrenByteKnnVectorQuery; import org.apache.lucene.search.join.DiversifyingChildrenFloatKnnVectorQuery; import org.apache.lucene.search.join.ToChildBlockJoinQuery; +import org.junit.Before; +import org.mockito.Mock; import org.mockito.MockedConstruction; import org.mockito.Mockito; +import org.opensearch.common.settings.ClusterSettings; import org.opensearch.index.mapper.MappedFieldType; import org.opensearch.index.mapper.MapperService; import org.opensearch.index.query.QueryBuilder; @@ -23,8 +26,10 @@ import org.opensearch.index.query.TermQueryBuilder; import org.opensearch.index.search.NestedHelper; import org.opensearch.knn.KNNTestCase; +import org.opensearch.knn.index.KNNSettings; import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.engine.KNNEngine; +import org.opensearch.knn.index.query.nativelib.NativeEngineKnnVectorQuery; import java.util.Arrays; import java.util.List; @@ -36,6 +41,7 @@ import static org.mockito.Mockito.when; import static org.opensearch.knn.common.KNNConstants.DEFAULT_VECTOR_DATA_TYPE_FIELD; import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_EF_SEARCH; +import static org.opensearch.knn.common.featureflags.KNNFeatureFlags.KNN_LAUNCH_QUERY_REWRITE_ENABLED_SETTING; public class KNNQueryFactoryTests extends KNNTestCase { private static final String FILTER_FILED_NAME = "foo"; @@ -50,8 +56,21 @@ public class KNNQueryFactoryTests extends KNNTestCase { private final int testK = 10; private final Map methodParameters = Map.of(METHOD_PARAMETER_EF_SEARCH, 100); + @Mock + ClusterSettings clusterSettings; + + @Before + @Override + public void setUp() throws Exception { + super.setUp(); + when(clusterService.getClusterSettings()).thenReturn(clusterSettings); + when(clusterSettings.get(KNN_LAUNCH_QUERY_REWRITE_ENABLED_SETTING)).thenReturn(false); + KNNSettings.state().setClusterService(clusterService); + } + public void testCreateCustomKNNQuery() { for (KNNEngine knnEngine : KNNEngine.getEnginesThatCreateCustomSegmentFiles()) { + when(clusterSettings.get(KNN_LAUNCH_QUERY_REWRITE_ENABLED_SETTING)).thenReturn(false); Query query = KNNQueryFactory.create( knnEngine, testIndexName, @@ -61,6 +80,15 @@ public void testCreateCustomKNNQuery() { DEFAULT_VECTOR_DATA_TYPE_FIELD ); assertTrue(query instanceof KNNQuery); + assertEquals(testIndexName, ((KNNQuery) query).getIndexName()); + assertEquals(testFieldName, ((KNNQuery) query).getField()); + assertEquals(testQueryVector, ((KNNQuery) query).getQueryVector()); + assertEquals(testK, ((KNNQuery) query).getK()); + + when(clusterSettings.get(KNN_LAUNCH_QUERY_REWRITE_ENABLED_SETTING)).thenReturn(true); + query = KNNQueryFactory.create(knnEngine, testIndexName, testFieldName, testQueryVector, testK, DEFAULT_VECTOR_DATA_TYPE_FIELD); + assertTrue(query instanceof NativeEngineKnnVectorQuery); + query = ((NativeEngineKnnVectorQuery) query).getKnnQuery(); assertEquals(testIndexName, ((KNNQuery) query).getIndexName()); assertEquals(testFieldName, ((KNNQuery) query).getField()); @@ -392,6 +420,7 @@ public void testCreate_whenBinary_thenSuccess() { when(mockQueryShardContext.fieldMapper(any())).thenReturn(testMapper); BitSetProducer parentFilter = mock(BitSetProducer.class); when(mockQueryShardContext.getParentFilter()).thenReturn(parentFilter); + final KNNQueryFactory.CreateQueryRequest createQueryRequest = KNNQueryFactory.CreateQueryRequest.builder() .knnEngine(KNNEngine.FAISS) .indexName(testIndexName) @@ -407,5 +436,10 @@ public void testCreate_whenBinary_thenSuccess() { assertTrue(query instanceof KNNQuery); assertNotNull(((KNNQuery) query).getByteQueryVector()); assertNull(((KNNQuery) query).getQueryVector()); + + when(clusterSettings.get(KNN_LAUNCH_QUERY_REWRITE_ENABLED_SETTING)).thenReturn(true); + query = KNNQueryFactory.create(createQueryRequest); + assertTrue(query instanceof NativeEngineKnnVectorQuery); } + } diff --git a/src/test/java/org/opensearch/knn/index/query/KNNWeightTests.java b/src/test/java/org/opensearch/knn/index/query/KNNWeightTests.java index c7077eace..c5abc964d 100644 --- a/src/test/java/org/opensearch/knn/index/query/KNNWeightTests.java +++ b/src/test/java/org/opensearch/knn/index/query/KNNWeightTests.java @@ -350,7 +350,7 @@ public void testShardWithoutFiles() { when(fieldInfos.fieldInfo(any())).thenReturn(fieldInfo); final Scorer knnScorer = knnWeight.scorer(leafReaderContext); - assertNull(knnScorer); + assertEquals(KNNScorer.emptyScorer(knnWeight), knnScorer); } @SneakyThrows @@ -394,7 +394,7 @@ public void testEmptyQueryResults() { when(fieldInfos.fieldInfo(any())).thenReturn(fieldInfo); final Scorer knnScorer = knnWeight.scorer(leafReaderContext); - assertNull(knnScorer); + assertEquals(KNNScorer.emptyScorer(knnWeight), knnScorer); } @SneakyThrows diff --git a/src/test/java/org/opensearch/knn/index/query/nativelib/DocAndScoreQueryTests.java b/src/test/java/org/opensearch/knn/index/query/nativelib/DocAndScoreQueryTests.java new file mode 100644 index 000000000..185cb5d47 --- /dev/null +++ b/src/test/java/org/opensearch/knn/index/query/nativelib/DocAndScoreQueryTests.java @@ -0,0 +1,99 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.query.nativelib; + +import lombok.SneakyThrows; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.IndexReaderContext; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.search.Explanation; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.ScoreMode; +import org.apache.lucene.search.Scorer; +import org.apache.lucene.search.Weight; +import org.mockito.Mock; +import org.opensearch.test.OpenSearchTestCase; + +import static org.mockito.Mockito.when; +import static org.mockito.MockitoAnnotations.openMocks; + +public class DocAndScoreQueryTests extends OpenSearchTestCase { + + @Mock + private LeafReaderContext leaf1; + @Mock + private IndexSearcher indexSearcher; + @Mock + private IndexReader reader; + @Mock + private IndexReaderContext readerContext; + + private DocAndScoreQuery objectUnderTest; + + @Override + public void setUp() throws Exception { + super.setUp(); + openMocks(this); + + when(indexSearcher.getIndexReader()).thenReturn(reader); + when(reader.getContext()).thenReturn(readerContext); + when(readerContext.id()).thenReturn(1); + } + + // Note: cannot test with multi leaf as there LeafReaderContext is readonly with no getters for some fields to mock + public void testScorer() throws Exception { + // Given + int[] expectedDocs = { 0, 1, 2, 3, 4 }; + float[] expectedScores = { 0.1f, 1.2f, 2.3f, 5.1f, 3.4f }; + int[] findSegments = { 0, 2, 5 }; + objectUnderTest = new DocAndScoreQuery(4, expectedDocs, expectedScores, findSegments, 1); + + // When + Scorer scorer1 = objectUnderTest.createWeight(indexSearcher, ScoreMode.COMPLETE, 1).scorer(leaf1); + DocIdSetIterator iterator1 = scorer1.iterator(); + Scorer scorer2 = objectUnderTest.createWeight(indexSearcher, ScoreMode.COMPLETE, 1).scorer(leaf1); + DocIdSetIterator iterator2 = scorer2.iterator(); + + int[] actualDocs = new int[2]; + float[] actualScores = new float[2]; + int index = 0; + while (iterator1.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) { + actualDocs[index] = iterator1.docID(); + actualScores[index] = scorer1.score(); + ++index; + } + + // Then + assertEquals(2, iterator1.cost()); + assertArrayEquals(new int[] { 0, 1 }, actualDocs); + assertArrayEquals(new float[] { 0.1f, 1.2f }, actualScores, 0.0001f); + + assertEquals(1.2f, scorer2.getMaxScore(1), 0.0001f); + assertEquals(iterator2.advance(1), 1); + } + + @SneakyThrows + public void testWeight() { + // Given + int[] expectedDocs = { 0, 1, 2, 3, 4 }; + float[] expectedScores = { 0.1f, 1.2f, 2.3f, 5.1f, 3.4f }; + int[] findSegments = { 0, 2, 5 }; + Explanation expectedExplanation = Explanation.match(1.2f, "within top 4"); + + // When + objectUnderTest = new DocAndScoreQuery(4, expectedDocs, expectedScores, findSegments, 1); + Weight weight = objectUnderTest.createWeight(indexSearcher, ScoreMode.COMPLETE, 1); + Explanation explanation = weight.explain(leaf1, 1); + + // Then + assertEquals(objectUnderTest, weight.getQuery()); + assertTrue(weight.isCacheable(leaf1)); + assertEquals(2, weight.count(leaf1)); + assertEquals(expectedExplanation, explanation); + assertEquals(Explanation.noMatch("not in top 4"), weight.explain(leaf1, 9)); + } +} diff --git a/src/test/java/org/opensearch/knn/index/query/nativelib/NativeEngineKNNVectorQueryIT.java b/src/test/java/org/opensearch/knn/index/query/nativelib/NativeEngineKNNVectorQueryIT.java new file mode 100644 index 000000000..1d84fcb48 --- /dev/null +++ b/src/test/java/org/opensearch/knn/index/query/nativelib/NativeEngineKNNVectorQueryIT.java @@ -0,0 +1,190 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.query.nativelib; + +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import com.google.common.primitives.Floats; +import lombok.AllArgsConstructor; +import lombok.SneakyThrows; +import org.apache.http.util.EntityUtils; +import org.junit.BeforeClass; +import org.opensearch.client.Response; +import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.knn.KNNRestTestCase; +import org.opensearch.knn.KNNResult; +import org.opensearch.knn.TestUtils; +import org.opensearch.knn.common.KNNConstants; +import org.opensearch.knn.index.FaissHNSWFlatE2EIT; +import org.opensearch.knn.index.SpaceType; +import org.opensearch.knn.index.engine.KNNEngine; +import org.opensearch.knn.index.query.KNNQueryBuilder; +import org.opensearch.knn.plugin.script.KNNScoringUtil; + +import java.io.IOException; +import java.net.URL; +import java.util.Arrays; +import java.util.Collection; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; +import java.util.concurrent.ThreadLocalRandom; + +import static com.carrotsearch.randomizedtesting.RandomizedTest.$; +import static com.carrotsearch.randomizedtesting.RandomizedTest.$$; +import static org.opensearch.knn.common.KNNConstants.KNN_ENGINE; +import static org.opensearch.knn.common.KNNConstants.METHOD_HNSW; +import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_EF_SEARCH; +import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_SPACE_TYPE; +import static org.opensearch.knn.common.KNNConstants.NAME; +import static org.opensearch.knn.common.KNNConstants.PARAMETERS; + +@AllArgsConstructor +public class NativeEngineKNNVectorQueryIT extends KNNRestTestCase { + + private String description; + private int k; + private Map methodParameters; + private boolean deleteRandomDocs; + + static TestUtils.TestData testData; + + @BeforeClass + public static void setUpClass() throws IOException { + if (FaissHNSWFlatE2EIT.class.getClassLoader() == null) { + throw new IllegalStateException("ClassLoader of FaissIT Class is null"); + } + URL testIndexVectors = FaissHNSWFlatE2EIT.class.getClassLoader().getResource("data/test_vectors_1000x128.json"); + URL testQueries = FaissHNSWFlatE2EIT.class.getClassLoader().getResource("data/test_queries_100x128.csv"); + assert testIndexVectors != null; + assert testQueries != null; + testData = new TestUtils.TestData(testIndexVectors.getPath(), testQueries.getPath()); + } + + @ParametersFactory(argumentFormatting = "description:%1$s; k:%2$s; efSearch:%3$s, deleteDocs:%4$s") + public static Collection parameters() { + return Arrays.asList( + $$( + $("test without deletedocs", 10, Map.of(METHOD_PARAMETER_EF_SEARCH, 300), false), + $("test with deletedocs", 10, Map.of(METHOD_PARAMETER_EF_SEARCH, 300), true) + ) + ); + } + + @SneakyThrows + public void testResultComparisonSanity() { + String indexName = "test-index-1"; + String fieldName = "test-field-1"; + + SpaceType spaceType = SpaceType.L2; + + Integer dimension = testData.indexData.vectors[0].length; + + // Create an index + XContentBuilder builder = XContentFactory.jsonBuilder() + .startObject() + .startObject("properties") + .startObject(fieldName) + .field("type", "knn_vector") + .field("dimension", dimension) + .startObject(KNNConstants.KNN_METHOD) + .field(NAME, METHOD_HNSW) + .field(METHOD_PARAMETER_SPACE_TYPE, spaceType.getValue()) + .field(KNN_ENGINE, KNNEngine.FAISS.getName()) + .startObject(PARAMETERS) + .field(KNNConstants.METHOD_PARAMETER_M, 16) + .field(KNNConstants.METHOD_PARAMETER_EF_CONSTRUCTION, 32) + .field(KNNConstants.METHOD_PARAMETER_EF_SEARCH, 32) + .endObject() + .endObject() + .endObject() + .endObject() + .endObject(); + + Map mappingMap = xContentBuilderToMap(builder); + String mapping = builder.toString(); + + createKnnIndex(indexName, mapping); + assertEquals(new TreeMap<>(mappingMap), new TreeMap<>(getIndexMappingAsMap(indexName))); + + // Index the test data + for (int i = 0; i < testData.indexData.docs.length; i++) { + addKnnDoc( + indexName, + Integer.toString(testData.indexData.docs[i]), + fieldName, + Floats.asList(testData.indexData.vectors[i]).toArray() + ); + } + + // Assert we have the right number of documents in the index + refreshAllNonSystemIndices(); + assertEquals(testData.indexData.docs.length, getDocCount(indexName)); + + // Delete few Docs + if (deleteRandomDocs) { + final Set docIdsToBeDeleted = new HashSet<>(); + while (docIdsToBeDeleted.size() < 10) { + docIdsToBeDeleted.add(randomInt(testData.indexData.docs.length - 1)); + } + + for (Integer id : docIdsToBeDeleted) { + deleteKnnDoc(indexName, Integer.toString(testData.indexData.docs[id])); + } + refreshAllNonSystemIndices(); + forceMergeKnnIndex(indexName, 3); + + assertEquals(testData.indexData.docs.length - 10, getDocCount(indexName)); + } + + int queryIndex = ThreadLocalRandom.current().nextInt(testData.queries.length); + // Test search queries + final KNNQueryBuilder queryBuilder = KNNQueryBuilder.builder() + .fieldName(fieldName) + .vector(testData.queries[queryIndex]) + .k(k) + .methodParameters(methodParameters) + .build(); + Response nativeEngineResponse = searchKNNIndex(indexName, queryBuilder, k); + String responseBody = EntityUtils.toString(nativeEngineResponse.getEntity()); + List nativeEngineKnnResults = parseSearchResponse(responseBody, fieldName); + assertEquals(k, nativeEngineKnnResults.size()); + + List actualScores = parseSearchResponseScore(responseBody, fieldName); + for (int j = 0; j < k; j++) { + float[] primitiveArray = nativeEngineKnnResults.get(j).getVector(); + assertEquals( + KNNEngine.FAISS.score(KNNScoringUtil.l2Squared(testData.queries[queryIndex], primitiveArray), spaceType), + actualScores.get(j), + 0.0001 + ); + } + + updateClusterSettings("knn.feature.query.rewrite.enabled", false); + Response launchControlDisabledResponse = searchKNNIndex(indexName, queryBuilder, k); + String launchControlDisabledResponseString = EntityUtils.toString(launchControlDisabledResponse.getEntity()); + List knnResults = parseSearchResponse(launchControlDisabledResponseString, fieldName); + assertEquals(k, knnResults.size()); + + assertEquals(nativeEngineKnnResults, knnResults); + + // Delete index + deleteKNNIndex(indexName); + + // Search every 5 seconds 14 times to confirm graph gets evicted + int intervals = 14; + for (int i = 0; i < intervals; i++) { + if (getTotalGraphsInCache() == 0) { + return; + } + Thread.sleep(5 * 1000); + } + + fail("Graphs are not getting evicted"); + } +} diff --git a/src/test/java/org/opensearch/knn/index/query/nativelib/NativeEngineKNNVectorQueryTests.java b/src/test/java/org/opensearch/knn/index/query/nativelib/NativeEngineKNNVectorQueryTests.java new file mode 100644 index 000000000..1e4b11a12 --- /dev/null +++ b/src/test/java/org/opensearch/knn/index/query/nativelib/NativeEngineKNNVectorQueryTests.java @@ -0,0 +1,156 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.query.nativelib; + +import lombok.SneakyThrows; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.IndexReaderContext; +import org.apache.lucene.index.LeafReader; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.MatchNoDocsQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.ScoreMode; +import org.apache.lucene.search.TaskExecutor; +import org.apache.lucene.search.TopDocs; +import org.apache.lucene.util.Bits; +import org.mockito.ArgumentMatchers; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.opensearch.knn.index.query.KNNQuery; +import org.opensearch.knn.index.query.KNNWeight; +import org.opensearch.test.OpenSearchTestCase; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.concurrent.Callable; + +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import static org.mockito.MockitoAnnotations.openMocks; + +public class NativeEngineKNNVectorQueryTests extends OpenSearchTestCase { + + @Mock + private IndexSearcher searcher; + @Mock + private IndexReader reader; + @Mock + private KNNQuery knnQuery; + @Mock + private KNNWeight knnWeight; + @Mock + private TaskExecutor taskExecutor; + @Mock + private IndexReaderContext indexReaderContext; + @Mock + private LeafReaderContext leaf1; + @Mock + private LeafReaderContext leaf2; + @Mock + private LeafReader leafReader1; + @Mock + private LeafReader leafReader2; + + @InjectMocks + private NativeEngineKnnVectorQuery objectUnderTest; + + @Override + public void setUp() throws Exception { + super.setUp(); + openMocks(this); + + when(leaf1.reader()).thenReturn(leafReader1); + when(leaf2.reader()).thenReturn(leafReader2); + + when(searcher.getIndexReader()).thenReturn(reader); + when(knnQuery.createWeight(searcher, ScoreMode.COMPLETE, 1)).thenReturn(knnWeight); + + when(searcher.getTaskExecutor()).thenReturn(taskExecutor); + when(taskExecutor.invokeAll(ArgumentMatchers.>anyList())).thenAnswer(invocationOnMock -> { + List> callables = invocationOnMock.getArgument(0); + List topDocs = new ArrayList<>(); + for (Callable callable : callables) { + topDocs.add(callable.call()); + } + return topDocs; + }); + + when(reader.getContext()).thenReturn(indexReaderContext); + } + + @SneakyThrows + public void testMultiLeaf() { + // Given + List leaves = List.of(leaf1, leaf2); + when(reader.leaves()).thenReturn(leaves); + + when(knnWeight.searchLeaf(leaf1)).thenReturn(Map.of(0, 1.2f, 1, 5.1f, 2, 2.2f)); + when(knnWeight.searchLeaf(leaf2)).thenReturn(Map.of(4, 3.4f, 3, 5.1f)); + + // Making sure there is deleted docs in one of the segments + Bits liveDocs = mock(Bits.class); + when(leafReader1.getLiveDocs()).thenReturn(liveDocs); + when(leafReader2.getLiveDocs()).thenReturn(null); + + when(liveDocs.get(anyInt())).thenReturn(true); + when(liveDocs.get(2)).thenReturn(false); + when(liveDocs.get(1)).thenReturn(false); + + // k=4 to make sure we get topk results even if docs are deleted/less in one of the leaves + when(knnQuery.getK()).thenReturn(4); + + when(indexReaderContext.id()).thenReturn(1); + int[] expectedDocs = { 0, 3, 4 }; + float[] expectedScores = { 1.2f, 5.1f, 3.4f }; + int[] findSegments = { 0, 1, 3 }; + DocAndScoreQuery expected = new DocAndScoreQuery(4, expectedDocs, expectedScores, findSegments, 1); + + // When + Query actual = objectUnderTest.rewrite(searcher); + + // Then + assertEquals(expected, actual); + } + + @SneakyThrows + public void testSingleLeaf() { + // Given + List leaves = List.of(leaf1); + when(reader.leaves()).thenReturn(leaves); + when(knnWeight.searchLeaf(leaf1)).thenReturn(Map.of(0, 1.2f, 1, 5.1f, 2, 2.2f)); + when(knnQuery.getK()).thenReturn(4); + + when(indexReaderContext.id()).thenReturn(1); + int[] expectedDocs = { 0, 1, 2 }; + float[] expectedScores = { 1.2f, 5.1f, 2.2f }; + int[] findSegments = { 0, 3 }; + DocAndScoreQuery expected = new DocAndScoreQuery(4, expectedDocs, expectedScores, findSegments, 1); + + // When + Query actual = objectUnderTest.rewrite(searcher); + + // Then + assertEquals(expected, actual); + } + + @SneakyThrows + public void testNoMatch() { + // Given + List leaves = List.of(leaf1); + when(reader.leaves()).thenReturn(leaves); + when(knnWeight.searchLeaf(leaf1)).thenReturn(Collections.emptyMap()); + when(knnQuery.getK()).thenReturn(4); + // When + Query actual = objectUnderTest.rewrite(searcher); + + // Then + assertEquals(new MatchNoDocsQuery(), actual); + } +} From ede3665a93ae0467979e51bc299420954e0d28da Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Sat, 10 Aug 2024 13:32:00 -0700 Subject: [PATCH 321/416] Refactor Around Mapper and Mapping (#1947) Refactors FieldMapper logic. It removes the LegacyFieldMapper and replaces it with a FlatFieldMapper. The FlatFieldMapper's role is to create fields that do not build ANN indices. Additionally, it puts dimension, model_id, and knn_method_context in a new KNNMappingConfig class and adds some safety checks around accessing them. This should make calling logic easier to handle. Lastly, it cleans up the parsing so that there isnt encoder parsing directly in the KNNVectorFieldMapper. Signed-off-by: John Mazanec (cherry picked from commit 2cd57e88665ef9ea3406fb90b49a9fc843901b10) --- CHANGELOG.md | 3 +- .../org/opensearch/knn/bwc/IndexingIT.java | 15 + .../codec/BasePerFieldKnnVectorsFormat.java | 15 +- .../KNN80Codec/KNN80DocValuesConsumer.java | 5 +- .../knn/index/engine/KNNMethodContext.java | 18 - .../index/mapper/FlatVectorFieldMapper.java | 91 ++++ .../knn/index/mapper/KNNMappingConfig.java | 38 ++ .../index/mapper/KNNVectorFieldMapper.java | 479 ++++++++---------- .../mapper/KNNVectorFieldMapperUtil.java | 236 ++++++--- .../knn/index/mapper/KNNVectorFieldType.java | 56 +- .../knn/index/mapper/LegacyFieldMapper.java | 130 ----- .../knn/index/mapper/LuceneFieldMapper.java | 96 +++- .../knn/index/mapper/MethodFieldMapper.java | 121 ++++- .../knn/index/mapper/ModelFieldMapper.java | 183 ++++++- .../index/mapper/PerDimensionProcessor.java | 51 ++ .../index/mapper/PerDimensionValidator.java | 80 +++ .../index/mapper/SpaceVectorValidator.java | 28 + .../knn/index/mapper/VectorValidator.java | 28 + .../knn/index/query/KNNQueryBuilder.java | 83 +-- .../org/opensearch/knn/plugin/KNNPlugin.java | 2 - .../java/org/opensearch/knn/KNNTestCase.java | 55 ++ .../knn/index/KNNMethodContextTests.java | 2 +- .../KNN80DocValuesConsumerTests.java | 56 -- .../knn/index/codec/KNNCodecTestCase.java | 46 +- .../mapper/KNNVectorFieldMapperTests.java | 211 ++++---- .../mapper/KNNVectorFieldMapperUtilTests.java | 59 +-- .../index/mapper/MethodFieldMapperTests.java | 46 +- .../knn/index/query/KNNQueryBuilderTests.java | 119 ++--- .../knn/integ/KNNScriptScoringIT.java | 19 +- .../script/KNNScoringSpaceFactoryTests.java | 12 +- .../plugin/script/KNNScoringSpaceTests.java | 45 +- .../script/KNNScoringSpaceUtilTests.java | 2 +- ...TrainingJobRouterTransportActionTests.java | 7 +- .../transport/TrainingModelRequestTests.java | 4 +- 34 files changed, 1465 insertions(+), 976 deletions(-) create mode 100644 src/main/java/org/opensearch/knn/index/mapper/FlatVectorFieldMapper.java create mode 100644 src/main/java/org/opensearch/knn/index/mapper/KNNMappingConfig.java delete mode 100644 src/main/java/org/opensearch/knn/index/mapper/LegacyFieldMapper.java create mode 100644 src/main/java/org/opensearch/knn/index/mapper/PerDimensionProcessor.java create mode 100644 src/main/java/org/opensearch/knn/index/mapper/PerDimensionValidator.java create mode 100644 src/main/java/org/opensearch/knn/index/mapper/SpaceVectorValidator.java create mode 100644 src/main/java/org/opensearch/knn/index/mapper/VectorValidator.java diff --git a/CHANGELOG.md b/CHANGELOG.md index e10f3e065..879a50e1c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,4 +29,5 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), * Refactor method structure and definitions [#1920](https://github.com/opensearch-project/k-NN/pull/1920) * Refactor KNNVectorFieldType from KNNVectorFieldMapper to a separate class for better readability. [#1931](https://github.com/opensearch-project/k-NN/pull/1931) * Generalize lib interface to return context objects [#1925](https://github.com/opensearch-project/k-NN/pull/1925) -* Move k search k-NN query to re-write phase of vector search query for Native Engines [#1877](https://github.com/opensearch-project/k-NN/pull/1877) \ No newline at end of file +* Move k search k-NN query to re-write phase of vector search query for Native Engines [#1877](https://github.com/opensearch-project/k-NN/pull/1877) +* Restructure mappers to better handle null cases and avoid branching in parsing [#1939](https://github.com/opensearch-project/k-NN/pull/1939) \ No newline at end of file diff --git a/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/IndexingIT.java b/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/IndexingIT.java index 2df79a3a2..1531dd0da 100644 --- a/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/IndexingIT.java +++ b/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/IndexingIT.java @@ -53,6 +53,21 @@ public void testKNNIndexDefaultLegacyFieldMapping() throws Exception { } } + // Ensure that when segments created with old mapping are forcemerged in new cluster, they + // succeed + public void testKNNIndexDefaultLegacyFieldMappingForceMerge() throws Exception { + waitForClusterHealthGreen(NODES_BWC_CLUSTER); + + if (isRunningAgainstOldCluster()) { + createKnnIndex(testIndex, getKNNDefaultIndexSettings(), createKnnIndexMapping(TEST_FIELD, DIMENSIONS)); + addKNNDocs(testIndex, TEST_FIELD, DIMENSIONS, DOC_ID, 100); + // Flush to ensure that index is not re-indexed when node comes back up + flush(testIndex, true); + } else { + forceMergeKnnIndex(testIndex); + } + } + // Custom Legacy Field Mapping // space_type : "linf", engine : "nmslib", m : 2, ef_construction : 2 public void testKNNIndexCustomLegacyFieldMapping() throws Exception { diff --git a/src/main/java/org/opensearch/knn/index/codec/BasePerFieldKnnVectorsFormat.java b/src/main/java/org/opensearch/knn/index/codec/BasePerFieldKnnVectorsFormat.java index 2a3732d7e..69229036e 100644 --- a/src/main/java/org/opensearch/knn/index/codec/BasePerFieldKnnVectorsFormat.java +++ b/src/main/java/org/opensearch/knn/index/codec/BasePerFieldKnnVectorsFormat.java @@ -13,6 +13,8 @@ import org.opensearch.knn.index.codec.params.KNNScalarQuantizedVectorsFormatParams; import org.opensearch.knn.index.codec.params.KNNVectorsFormatParams; import org.opensearch.knn.index.engine.KNNEngine; +import org.opensearch.knn.index.engine.KNNMethodContext; +import org.opensearch.knn.index.mapper.KNNMappingConfig; import org.opensearch.knn.index.mapper.KNNVectorFieldType; import java.util.Optional; @@ -66,16 +68,19 @@ public KnnVectorsFormat getKnnVectorsFormatForField(final String field) { ); return defaultFormatSupplier.get(); } - var type = (KNNVectorFieldType) mapperService.orElseThrow( + KNNVectorFieldType mappedFieldType = (KNNVectorFieldType) mapperService.orElseThrow( () -> new IllegalStateException( String.format("Cannot read field type for field [%s] because mapper service is not available", field) ) ).fieldType(field); - var params = type.getKnnMethodContext().getMethodComponentContext().getParameters(); - if (type.getKnnMethodContext().getKnnEngine() == KNNEngine.LUCENE - && params != null - && params.containsKey(METHOD_ENCODER_PARAMETER)) { + KNNMappingConfig knnMappingConfig = mappedFieldType.getKnnMappingConfig(); + KNNMethodContext knnMethodContext = knnMappingConfig.getKnnMethodContext() + .orElseThrow(() -> new IllegalArgumentException("KNN method context cannot be empty")); + + var params = knnMethodContext.getMethodComponentContext().getParameters(); + + if (knnMethodContext.getKnnEngine() == KNNEngine.LUCENE && params != null && params.containsKey(METHOD_ENCODER_PARAMETER)) { KNNScalarQuantizedVectorsFormatParams knnScalarQuantizedVectorsFormatParams = new KNNScalarQuantizedVectorsFormatParams( params, defaultMaxConnections, diff --git a/src/main/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumer.java b/src/main/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumer.java index 7dad3a8fd..5874eaded 100644 --- a/src/main/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumer.java +++ b/src/main/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumer.java @@ -12,6 +12,7 @@ import org.opensearch.common.StopWatch; import org.opensearch.core.xcontent.DeprecationHandler; import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.util.IndexUtil; import org.opensearch.knn.index.KNNSettings; import org.opensearch.knn.index.VectorDataType; @@ -19,7 +20,6 @@ import org.opensearch.knn.index.codec.transfer.VectorTransferByte; import org.opensearch.knn.index.codec.transfer.VectorTransferFloat; import org.opensearch.knn.jni.JNIService; -import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.codec.util.KNNCodecUtil; import org.opensearch.knn.index.engine.KNNEngine; import org.opensearch.knn.indices.Model; @@ -214,8 +214,7 @@ private void createKNNIndexFromScratch(FieldInfo fieldInfo, KNNCodecUtil.Pair pa throws IOException { Map parameters = new HashMap<>(); Map fieldAttributes = fieldInfo.attributes(); - String parametersString = fieldAttributes.get(KNNConstants.PARAMETERS); - + String parametersString = fieldAttributes.get(PARAMETERS); // parametersString will be null when legacy mapper is used if (parametersString == null) { parameters.put(KNNConstants.SPACE_TYPE, fieldAttributes.getOrDefault(KNNConstants.SPACE_TYPE, SpaceType.DEFAULT.getValue())); diff --git a/src/main/java/org/opensearch/knn/index/engine/KNNMethodContext.java b/src/main/java/org/opensearch/knn/index/engine/KNNMethodContext.java index 7885761b9..d210483e6 100644 --- a/src/main/java/org/opensearch/knn/index/engine/KNNMethodContext.java +++ b/src/main/java/org/opensearch/knn/index/engine/KNNMethodContext.java @@ -9,7 +9,6 @@ import lombok.Getter; import lombok.NonNull; import lombok.Setter; -import org.opensearch.Version; import org.opensearch.common.ValidationException; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; @@ -20,7 +19,6 @@ import org.opensearch.index.mapper.MapperParsingException; import java.io.IOException; -import java.util.Collections; import java.util.HashMap; import java.util.Map; import java.util.stream.Collectors; @@ -29,7 +27,6 @@ import org.opensearch.knn.training.VectorSpaceInfo; import static org.opensearch.knn.common.KNNConstants.KNN_ENGINE; -import static org.opensearch.knn.common.KNNConstants.METHOD_HNSW; import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_SPACE_TYPE; import static org.opensearch.knn.common.KNNConstants.NAME; import static org.opensearch.knn.common.KNNConstants.PARAMETERS; @@ -42,21 +39,6 @@ @Getter public class KNNMethodContext implements ToXContentFragment, Writeable { - private static KNNMethodContext defaultInstance = null; - - /** - * This is used only for testing - * @return default KNNMethodContext for testing - */ - public static synchronized KNNMethodContext getDefault() { - if (defaultInstance == null) { - MethodComponentContext methodComponentContext = new MethodComponentContext(METHOD_HNSW, Collections.emptyMap()); - methodComponentContext.setIndexVersion(Version.CURRENT); - defaultInstance = new KNNMethodContext(KNNEngine.DEFAULT, SpaceType.DEFAULT, methodComponentContext); - } - return defaultInstance; - } - @NonNull private final KNNEngine knnEngine; @NonNull diff --git a/src/main/java/org/opensearch/knn/index/mapper/FlatVectorFieldMapper.java b/src/main/java/org/opensearch/knn/index/mapper/FlatVectorFieldMapper.java new file mode 100644 index 000000000..fffff30f4 --- /dev/null +++ b/src/main/java/org/opensearch/knn/index/mapper/FlatVectorFieldMapper.java @@ -0,0 +1,91 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.mapper; + +import org.apache.lucene.document.FieldType; +import org.opensearch.Version; +import org.opensearch.common.Explicit; +import org.opensearch.knn.index.VectorDataType; + +import java.util.Map; + +/** + * Mapper used when you dont want to build an underlying KNN struct - you just want to + * store vectors as doc values + */ +public class FlatVectorFieldMapper extends KNNVectorFieldMapper { + + private final PerDimensionValidator perDimensionValidator; + + public static FlatVectorFieldMapper createFieldMapper( + String fullname, + String simpleName, + Map metaValue, + VectorDataType vectorDataType, + Integer dimension, + MultiFields multiFields, + CopyTo copyTo, + Explicit ignoreMalformed, + boolean stored, + boolean hasDocValues, + Version indexCreatedVersion + ) { + final KNNVectorFieldType mappedFieldType = new KNNVectorFieldType(fullname, metaValue, vectorDataType, () -> dimension); + return new FlatVectorFieldMapper( + simpleName, + mappedFieldType, + multiFields, + copyTo, + ignoreMalformed, + stored, + hasDocValues, + indexCreatedVersion + ); + } + + private FlatVectorFieldMapper( + String simpleName, + KNNVectorFieldType mappedFieldType, + MultiFields multiFields, + CopyTo copyTo, + Explicit ignoreMalformed, + boolean stored, + boolean hasDocValues, + Version indexCreatedVersion + ) { + super(simpleName, mappedFieldType, multiFields, copyTo, ignoreMalformed, stored, hasDocValues, indexCreatedVersion, null); + this.perDimensionValidator = selectPerDimensionValidator(vectorDataType); + this.fieldType = new FieldType(KNNVectorFieldMapper.Defaults.FIELD_TYPE); + this.fieldType.freeze(); + } + + private PerDimensionValidator selectPerDimensionValidator(VectorDataType vectorDataType) { + if (VectorDataType.BINARY == vectorDataType) { + return PerDimensionValidator.DEFAULT_BIT_VALIDATOR; + } + + if (VectorDataType.BYTE == vectorDataType) { + return PerDimensionValidator.DEFAULT_BYTE_VALIDATOR; + } + + return PerDimensionValidator.DEFAULT_FLOAT_VALIDATOR; + } + + @Override + protected VectorValidator getVectorValidator() { + return VectorValidator.NOOP_VECTOR_VALIDATOR; + } + + @Override + protected PerDimensionValidator getPerDimensionValidator() { + return perDimensionValidator; + } + + @Override + protected PerDimensionProcessor getPerDimensionProcessor() { + return PerDimensionProcessor.NOOP_PROCESSOR; + } +} diff --git a/src/main/java/org/opensearch/knn/index/mapper/KNNMappingConfig.java b/src/main/java/org/opensearch/knn/index/mapper/KNNMappingConfig.java new file mode 100644 index 000000000..4fcd6e1bc --- /dev/null +++ b/src/main/java/org/opensearch/knn/index/mapper/KNNMappingConfig.java @@ -0,0 +1,38 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.mapper; + +import org.opensearch.knn.index.engine.KNNMethodContext; + +import java.util.Optional; + +/** + * Class holds information about how the ANN indices are created. The design of this class ensures that we do not + * accidentally configure an index that has multiple ways it can be created. This class is immutable. + */ +public interface KNNMappingConfig { + /** + * + * @return Optional containing the modelId if created from model, otherwise empty + */ + default Optional getModelId() { + return Optional.empty(); + } + + /** + * + * @return Optional containing the KNNMethodContext if created from method, otherwise empty + */ + default Optional getKnnMethodContext() { + return Optional.empty(); + } + + /** + * + * @return the dimension of the index; for model based indices, it will be null + */ + int getDimension(); +} diff --git a/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapper.java b/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapper.java index 3b9487645..40eaa12ae 100644 --- a/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapper.java +++ b/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapper.java @@ -11,7 +11,6 @@ import java.util.List; import java.util.Locale; import java.util.Map; -import java.util.Objects; import java.util.Optional; import java.util.function.Supplier; import lombok.extern.log4j.Log4j2; @@ -32,9 +31,8 @@ import org.opensearch.index.mapper.ParametrizedFieldMapper; import org.opensearch.index.mapper.ParseContext; import org.opensearch.knn.common.KNNConstants; -import org.opensearch.knn.index.KnnCircuitBreakerException; -import org.opensearch.knn.index.engine.KNNMethodContext; import org.opensearch.knn.index.KNNSettings; +import org.opensearch.knn.index.engine.KNNMethodContext; import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.engine.MethodComponentContext; import org.opensearch.knn.index.VectorDataType; @@ -44,23 +42,17 @@ import static org.opensearch.knn.common.KNNConstants.DEFAULT_VECTOR_DATA_TYPE_FIELD; import static org.opensearch.knn.common.KNNConstants.ENCODER_FLAT; -import static org.opensearch.knn.common.KNNConstants.ENCODER_SQ; -import static org.opensearch.knn.common.KNNConstants.FAISS_SQ_CLIP; -import static org.opensearch.knn.common.KNNConstants.FAISS_SQ_ENCODER_FP16; -import static org.opensearch.knn.common.KNNConstants.FAISS_SQ_TYPE; import static org.opensearch.knn.common.KNNConstants.KNN_METHOD; import static org.opensearch.knn.common.KNNConstants.METHOD_ENCODER_PARAMETER; import static org.opensearch.knn.common.KNNConstants.VECTOR_DATA_TYPE_FIELD; -import static org.opensearch.knn.common.KNNValidationUtil.validateByteVectorValue; -import static org.opensearch.knn.common.KNNValidationUtil.validateFloatVectorValue; import static org.opensearch.knn.common.KNNValidationUtil.validateVectorDimension; -import static org.opensearch.knn.index.KNNSettings.KNN_INDEX; +import static org.opensearch.knn.index.mapper.KNNVectorFieldMapperUtil.createKNNMethodContextFromLegacy; import static org.opensearch.knn.index.mapper.KNNVectorFieldMapperUtil.createStoredFieldForByteVector; import static org.opensearch.knn.index.mapper.KNNVectorFieldMapperUtil.createStoredFieldForFloatVector; -import static org.opensearch.knn.index.mapper.KNNVectorFieldMapperUtil.clipVectorValueToFP16Range; -import static org.opensearch.knn.index.mapper.KNNVectorFieldMapperUtil.validateFP16VectorValue; +import static org.opensearch.knn.index.mapper.KNNVectorFieldMapperUtil.validateIfCircuitBreakerIsNotTriggered; +import static org.opensearch.knn.index.mapper.KNNVectorFieldMapperUtil.validateIfKNNPluginEnabled; import static org.opensearch.knn.index.mapper.KNNVectorFieldMapperUtil.validateVectorDataType; -import static org.opensearch.knn.index.mapper.KNNVectorFieldMapperUtil.validateVectorDataTypeWithKnnIndexSetting; +import static org.opensearch.knn.index.mapper.ModelFieldMapper.UNSET_MODEL_DIMENSION_IDENTIFIER; /** * Field Mapper for KNN vector type. Implementations of this class define what needs to be stored in Lucene's fieldType. @@ -76,10 +68,6 @@ private static KNNVectorFieldMapper toType(FieldMapper in) { return (KNNVectorFieldMapper) in; } - // We store the version of the index with the mapper as different version of Opensearch has different default - // values of KNN engine Algorithms hyperparameters. - protected Version indexCreatedVersion; - /** * Builder for KNNVectorFieldMapper. This class defines the set of parameters that can be applied to the knn_vector * field type @@ -89,25 +77,37 @@ public static class Builder extends ParametrizedFieldMapper.Builder { protected final Parameter stored = Parameter.storeParam(m -> toType(m).stored, false); protected final Parameter hasDocValues = Parameter.docValuesParam(m -> toType(m).hasDocValues, true); - protected final Parameter dimension = new Parameter<>(KNNConstants.DIMENSION, false, () -> -1, (n, c, o) -> { - if (o == null) { - throw new IllegalArgumentException("Dimension cannot be null"); - } - int value; - try { - value = XContentMapValues.nodeIntegerValue(o); - } catch (Exception exception) { - throw new IllegalArgumentException( - String.format(Locale.ROOT, "Unable to parse [dimension] from provided value [%s] for vector [%s]", o, name) - ); - } - if (value <= 0) { - throw new IllegalArgumentException( - String.format(Locale.ROOT, "Dimension value must be greater than 0 for vector: %s", name) - ); + protected final Parameter dimension = new Parameter<>( + KNNConstants.DIMENSION, + false, + () -> UNSET_MODEL_DIMENSION_IDENTIFIER, + (n, c, o) -> { + if (o == null) { + throw new IllegalArgumentException("Dimension cannot be null"); + } + int value; + try { + value = XContentMapValues.nodeIntegerValue(o); + } catch (Exception exception) { + throw new IllegalArgumentException( + String.format(Locale.ROOT, "Unable to parse [dimension] from provided value [%s] for vector [%s]", o, name) + ); + } + if (value <= 0) { + throw new IllegalArgumentException( + String.format(Locale.ROOT, "Dimension value must be greater than 0 for vector: %s", name) + ); + } + return value; + }, + m -> { + KNNMappingConfig knnMappingConfig = toType(m).fieldType().getKnnMappingConfig(); + if (knnMappingConfig.getModelId().isPresent()) { + return UNSET_MODEL_DIMENSION_IDENTIFIER; + } + return knnMappingConfig.getDimension(); } - return value; - }, m -> toType(m).dimension); + ); /** * data_type which defines the datatype of the vector values. This is an optional parameter and @@ -126,7 +126,12 @@ public static class Builder extends ParametrizedFieldMapper.Builder { * model template index. If this parameter is set, it will take precedence. This parameter is only relevant for * library indices that require training. */ - protected final Parameter modelId = Parameter.stringParam(KNNConstants.MODEL_ID, false, m -> toType(m).modelId, null); + protected final Parameter modelId = Parameter.stringParam( + KNNConstants.MODEL_ID, + false, + m -> toType(m).fieldType().getKnnMappingConfig().getModelId().orElse(null), + null + ); /** * knnMethodContext parameter allows a user to define their k-NN library index configuration. Defaults to an L2 @@ -137,7 +142,7 @@ public static class Builder extends ParametrizedFieldMapper.Builder { false, () -> null, (n, c, o) -> KNNMethodContext.parse(o), - m -> toType(m).knnMethod + m -> toType(m).originalKNNMethodContext ).setSerializer(((b, n, v) -> { b.startObject(n); v.toXContent(b, ToXContent.EMPTY_PARAMS); @@ -164,35 +169,30 @@ public static class Builder extends ParametrizedFieldMapper.Builder { protected final Parameter> meta = Parameter.metaParam(); - protected String spaceType; - protected String m; - protected String efConstruction; - protected ModelDao modelDao; - protected Version indexCreatedVersion; - - public Builder(String name, ModelDao modelDao, Version indexCreatedVersion) { + // KNNMethodContext that allows us to properly configure a KNNVectorFieldMapper from another + // KNNVectorFieldMapper. To support our legacy field mapping, on parsing, if index.knn=true and no method is + // passed, we build a KNNMethodContext using the space type, ef_construction and m that are set in the index + // settings. However, for fieldmappers for merging, we need to be able to initialize one field mapper from + // another (see + // https://github.com/opensearch-project/OpenSearch/blob/2.16.0/server/src/main/java/org/opensearch/index/mapper/ParametrizedFieldMapper.java#L98). + // The problem is that in this case, the settings are set to empty so we cannot properly resolve the KNNMethodContext. + // (see + // https://github.com/opensearch-project/OpenSearch/blob/2.16.0/server/src/main/java/org/opensearch/index/mapper/ParametrizedFieldMapper.java#L130). + // While we could override the KNNMethodContext parameter initializer to set the knnMethodContext based on the + // constructed KNNMethodContext from the other field mapper, this can result in merge conflict/serialization + // exceptions. See + // (https://github.com/opensearch-project/OpenSearch/blob/2.16.0/server/src/main/java/org/opensearch/index/mapper/ParametrizedFieldMapper.java#L322-L324). + // So, what we do is pass in a "resolvedKNNMethodContext" that will either be null or be set via the merge builder + // constructor. A similar approach was taken for https://github.com/opendistro-for-elasticsearch/k-NN/issues/288 + private KNNMethodContext resolvedKNNMethodContext; + + public Builder(String name, ModelDao modelDao, Version indexCreatedVersion, KNNMethodContext resolvedKNNMethodContext) { super(name); this.modelDao = modelDao; this.indexCreatedVersion = indexCreatedVersion; - } - - /** - * This constructor is for legacy purposes. - * Checkout ODFE PR 288 - * - * @param name field name - * @param spaceType Spacetype of field - * @param m m value of field - * @param efConstruction efConstruction value of field - */ - public Builder(String name, String spaceType, String m, String efConstruction, Version indexCreatedVersion) { - super(name); - this.spaceType = spaceType; - this.m = m; - this.efConstruction = efConstruction; - this.indexCreatedVersion = indexCreatedVersion; + this.resolvedKNNMethodContext = resolvedKNNMethodContext; } @Override @@ -210,121 +210,117 @@ protected Explicit ignoreMalformed(BuilderContext context) { return KNNVectorFieldMapper.Defaults.IGNORE_MALFORMED; } + private void validateFlatMapper() { + if (modelId.get() != null || knnMethodContext.get() != null) { + throw new IllegalArgumentException("Cannot set modelId or method parameters when index.knn setting is false"); + } + } + @Override public KNNVectorFieldMapper build(BuilderContext context) { - // Originally, a user would use index settings to set the spaceType, efConstruction and m hnsw - // parameters. Upon further review, it makes sense to set these parameters in the mapping of a - // particular field. However, because users migrating from older versions will still use the index - // settings to set these parameters, we will need to provide backwards compatibilty. In order to - // handle this, we first check if the mapping is set, and, if so use it. If not, we check if the model is - // set. If not, we fall back to the parameters set in the index settings. This means that if a user sets - // the mappings, setting the index settings will have no impact. - - final KNNMethodContext knnMethodContext = this.knnMethodContext.getValue(); - setDefaultSpaceType(knnMethodContext, vectorDataType.getValue()); - validateSpaceType(knnMethodContext, vectorDataType.getValue()); - validateDimensions(knnMethodContext, vectorDataType.getValue()); - validateEncoder(knnMethodContext, vectorDataType.getValue()); final MultiFields multiFieldsBuilder = this.multiFieldsBuilder.build(this, context); final CopyTo copyToBuilder = copyTo.build(); final Explicit ignoreMalformed = ignoreMalformed(context); final Map metaValue = meta.getValue(); - if (knnMethodContext != null) { - validateVectorDataType(knnMethodContext, vectorDataType.getValue()); - knnMethodContext.getMethodComponentContext().setIndexVersion(indexCreatedVersion); - final KNNVectorFieldType mappedFieldType = new KNNVectorFieldType( + // Index is being created from model + String modelIdAsString = this.modelId.get(); + if (modelIdAsString != null) { + return ModelFieldMapper.createFieldMapper( buildFullName(context), - metaValue, - dimension.getValue(), - knnMethodContext, - vectorDataType.getValue() - ); - if (knnMethodContext.getKnnEngine() == KNNEngine.LUCENE) { - log.debug(String.format(Locale.ROOT, "Use [LuceneFieldMapper] mapper for field [%s]", name)); - LuceneFieldMapper.CreateLuceneFieldMapperInput createLuceneFieldMapperInput = - LuceneFieldMapper.CreateLuceneFieldMapperInput.builder() - .name(name) - .mappedFieldType(mappedFieldType) - .multiFields(multiFieldsBuilder) - .copyTo(copyToBuilder) - .ignoreMalformed(ignoreMalformed) - .stored(stored.get()) - .hasDocValues(hasDocValues.get()) - .vectorDataType(vectorDataType.getValue()) - .knnMethodContext(knnMethodContext) - .build(); - return new LuceneFieldMapper(createLuceneFieldMapperInput); - } - - return new MethodFieldMapper( name, - mappedFieldType, + metaValue, + vectorDataType.getValue(), + modelIdAsString, multiFieldsBuilder, copyToBuilder, ignoreMalformed, stored.get(), hasDocValues.get(), - knnMethodContext + modelDao, + indexCreatedVersion ); } - String modelIdAsString = this.modelId.get(); - if (modelIdAsString != null) { - // Because model information is stored in cluster metadata, we are unable to get it here. This is - // because to get the cluster metadata, you need access to the cluster state. Because this code is - // sometimes used to initialize the cluster state/update cluster state, we cannot get the state here - // safely. So, we are unable to validate the model. The model gets validated during ingestion. - - return new ModelFieldMapper( + // If the field mapper is using the legacy context and being constructed from another field mapper, + // the settings will be empty. See https://github.com/opendistro-for-elasticsearch/k-NN/issues/288. In this + // case, the input resolvedKNNMethodContext will be null and the settings wont exist (so flat mapper should + // be used). Otherwise, we need to check the setting. + boolean isResolvedNull = resolvedKNNMethodContext == null; + boolean isSettingPresent = KNNSettings.IS_KNN_INDEX_SETTING.exists(context.indexSettings()); + boolean isKnnSettingNotPresentOrFalse = !isSettingPresent || !KNNSettings.IS_KNN_INDEX_SETTING.get(context.indexSettings()); + if (isResolvedNull && isKnnSettingNotPresentOrFalse) { + validateFlatMapper(); + return FlatVectorFieldMapper.createFieldMapper( + buildFullName(context), name, - new KNNVectorFieldType(buildFullName(context), metaValue, -1, knnMethodContext, modelIdAsString), + metaValue, + vectorDataType.getValue(), + dimension.getValue(), multiFieldsBuilder, copyToBuilder, ignoreMalformed, stored.get(), hasDocValues.get(), - modelDao, - modelIdAsString, indexCreatedVersion ); } - // Build legacy - if (this.spaceType == null) { - this.spaceType = LegacyFieldMapper.getSpaceType(context.indexSettings(), vectorDataType.getValue()); - } - - if (this.m == null) { - this.m = LegacyFieldMapper.getM(context.indexSettings()); - } - - if (this.efConstruction == null) { - this.efConstruction = LegacyFieldMapper.getEfConstruction(context.indexSettings(), indexCreatedVersion); - } - - // Validates and throws exception if index.knn is set to true in the index settings - // using any VectorDataType (other than float, which is default) because we are using NMSLIB engine for LegacyFieldMapper - // and it only supports float VectorDataType - validateVectorDataTypeWithKnnIndexSetting(context.indexSettings().getAsBoolean(KNN_INDEX, false), vectorDataType); - - return new LegacyFieldMapper( - name, - new KNNVectorFieldType( + // See resolvedKNNMethodContext definition for explanation + if (isResolvedNull) { + resolvedKNNMethodContext = this.knnMethodContext.getValue(); + setDefaultSpaceType(resolvedKNNMethodContext, vectorDataType.getValue()); + validateSpaceType(resolvedKNNMethodContext, vectorDataType.getValue()); + validateDimensions(resolvedKNNMethodContext, vectorDataType.getValue()); + validateEncoder(resolvedKNNMethodContext, vectorDataType.getValue()); + } + + // If the knnMethodContext is null at this point, that means user built the index with the legacy k-NN + // settings to specify algo params. We need to convert this here to a KNNMethodContext so that we can + // properly configure the rest of the index + if (resolvedKNNMethodContext == null) { + resolvedKNNMethodContext = createKNNMethodContextFromLegacy(context, vectorDataType.getValue(), indexCreatedVersion); + } + + validateVectorDataType(resolvedKNNMethodContext, vectorDataType.getValue()); + resolvedKNNMethodContext.getMethodComponentContext().setIndexVersion(indexCreatedVersion); + if (resolvedKNNMethodContext.getKnnEngine() == KNNEngine.LUCENE) { + log.debug(String.format(Locale.ROOT, "Use [LuceneFieldMapper] mapper for field [%s]", name)); + LuceneFieldMapper.CreateLuceneFieldMapperInput createLuceneFieldMapperInput = LuceneFieldMapper.CreateLuceneFieldMapperInput + .builder() + .name(name) + .multiFields(multiFieldsBuilder) + .copyTo(copyToBuilder) + .ignoreMalformed(ignoreMalformed) + .stored(stored.getValue()) + .hasDocValues(hasDocValues.getValue()) + .vectorDataType(vectorDataType.getValue()) + .indexVersion(indexCreatedVersion) + .originalKnnMethodContext(knnMethodContext.get()) + .build(); + return LuceneFieldMapper.createFieldMapper( buildFullName(context), metaValue, - dimension.getValue(), vectorDataType.getValue(), - SpaceType.getSpace(spaceType) - ), + dimension.getValue(), + resolvedKNNMethodContext, + createLuceneFieldMapperInput + ); + } + + return MethodFieldMapper.createFieldMapper( + buildFullName(context), + name, + metaValue, + vectorDataType.getValue(), + dimension.getValue(), + resolvedKNNMethodContext, + knnMethodContext.get(), multiFieldsBuilder, copyToBuilder, ignoreMalformed, - stored.get(), - hasDocValues.get(), - spaceType, - m, - efConstruction, + stored.getValue(), + hasDocValues.getValue(), indexCreatedVersion ); } @@ -430,7 +426,7 @@ public TypeParser(Supplier modelDaoSupplier) { @Override public Mapper.Builder parse(String name, Map node, ParserContext parserContext) throws MapperParsingException { - Builder builder = new KNNVectorFieldMapper.Builder(name, modelDaoSupplier.get(), parserContext.indexVersionCreated()); + Builder builder = new KNNVectorFieldMapper.Builder(name, modelDaoSupplier.get(), parserContext.indexVersionCreated(), null); builder.parse(name, parserContext, node); // All parse(String name, Map node, ParserCont } // Dimension should not be null unless modelId is used - if (builder.dimension.getValue() == -1 && builder.modelId.get() == null) { + if (builder.dimension.getValue() == UNSET_MODEL_DIMENSION_IDENTIFIER && builder.modelId.get() == null) { throw new IllegalArgumentException(String.format(Locale.ROOT, "Dimension value missing for vector: %s", name)); } @@ -452,18 +448,19 @@ public Mapper.Builder parse(String name, Map node, ParserCont } } + // We store the version of the index with the mapper as different version of Opensearch has different default + // values of KNN engine Algorithms hyperparameters. + protected Version indexCreatedVersion; protected Explicit ignoreMalformed; protected boolean stored; protected boolean hasDocValues; - protected Integer dimension; protected VectorDataType vectorDataType; protected ModelDao modelDao; - // These members map to parameters in the builder. They need to be declared in the abstract class due to the - // "toType" function used in the builder. So, when adding a parameter, it needs to be added here, but set in a - // subclass (if it is unique). - protected KNNMethodContext knnMethod; - protected String modelId; + // We need to ensure that the original KNNMethodContext as parsed is stored to initialize the + // Builder for serialization. So, we need to store it here. This is mainly to ensure that the legacy field mapper + // can use KNNMethodContext without messing up serialization on mapper merge + protected KNNMethodContext originalKNNMethodContext; public KNNVectorFieldMapper( String simpleName, @@ -473,16 +470,17 @@ public KNNVectorFieldMapper( Explicit ignoreMalformed, boolean stored, boolean hasDocValues, - Version indexCreatedVersion + Version indexCreatedVersion, + KNNMethodContext originalKNNMethodContext ) { super(simpleName, mappedFieldType, multiFields, copyTo); this.ignoreMalformed = ignoreMalformed; this.stored = stored; this.hasDocValues = hasDocValues; - this.dimension = mappedFieldType.getDimension(); this.vectorDataType = mappedFieldType.getVectorDataType(); updateEngineStats(); this.indexCreatedVersion = indexCreatedVersion; + this.originalKNNMethodContext = originalKNNMethodContext; } public KNNVectorFieldMapper clone() { @@ -496,20 +494,7 @@ protected String contentType() { @Override protected void parseCreateField(ParseContext context) throws IOException { - parseCreateField( - context, - fieldType().getDimension(), - fieldType().getSpaceType(), - getMethodComponentContext(fieldType().getKnnMethodContext()), - fieldType().getVectorDataType() - ); - } - - private MethodComponentContext getMethodComponentContext(KNNMethodContext knnMethodContext) { - if (Objects.isNull(knnMethodContext)) { - return null; - } - return knnMethodContext.getMethodComponentContext(); + parseCreateField(context, fieldType().getKnnMappingConfig().getDimension(), fieldType().getVectorDataType()); } /** @@ -544,17 +529,37 @@ protected List getFieldsForByteVector(final byte[] array, final FieldType return fields; } - protected void parseCreateField( - ParseContext context, - int dimension, - SpaceType spaceType, - MethodComponentContext methodComponentContext, - VectorDataType vectorDataType - ) throws IOException { - + /** + * Validation checks before parsing of doc begins + */ + protected void validatePreparse() { validateIfKNNPluginEnabled(); validateIfCircuitBreakerIsNotTriggered(); - spaceType.validateVectorDataType(vectorDataType); + } + + /** + * Getter for vector validator after vector parsing + * + * @return VectorValidator + */ + protected abstract VectorValidator getVectorValidator(); + + /** + * Getter for per dimension validator during vector parsing + * + * @return PerDimensionValidator + */ + protected abstract PerDimensionValidator getPerDimensionValidator(); + + /** + * Getter for per dimension processor during vector parsing + * + * @return PerDimensionProcessor + */ + protected abstract PerDimensionProcessor getPerDimensionProcessor(); + + protected void parseCreateField(ParseContext context, int dimension, VectorDataType vectorDataType) throws IOException { + validatePreparse(); if (VectorDataType.BINARY == vectorDataType) { Optional bytesArrayOptional = getBytesFromContext(context, dimension, vectorDataType); @@ -563,7 +568,7 @@ protected void parseCreateField( return; } final byte[] array = bytesArrayOptional.get(); - spaceType.validateVector(array); + getVectorValidator().validateVector(array); context.doc().addAll(getFieldsForByteVector(array, fieldType)); } else if (VectorDataType.BYTE == vectorDataType) { Optional bytesArrayOptional = getBytesFromContext(context, dimension, vectorDataType); @@ -572,16 +577,16 @@ protected void parseCreateField( return; } final byte[] array = bytesArrayOptional.get(); - spaceType.validateVector(array); + getVectorValidator().validateVector(array); context.doc().addAll(getFieldsForByteVector(array, fieldType)); } else if (VectorDataType.FLOAT == vectorDataType) { - Optional floatsArrayOptional = getFloatsFromContext(context, dimension, methodComponentContext); + Optional floatsArrayOptional = getFloatsFromContext(context, dimension); if (floatsArrayOptional.isEmpty()) { return; } final float[] array = floatsArrayOptional.get(); - spaceType.validateVector(array); + getVectorValidator().validateVector(array); context.doc().addAll(getFieldsForFloatVector(array, fieldType)); } else { throw new IllegalArgumentException( @@ -592,80 +597,28 @@ protected void parseCreateField( context.path().remove(); } - // Verify mapping and return true if it is a "faiss" Index using "sq" encoder of type "fp16" - protected boolean isFaissSQfp16(MethodComponentContext methodComponentContext) { - if (Objects.isNull(methodComponentContext)) { - return false; - } - - if (methodComponentContext.getParameters().size() == 0) { - return false; - } - - Map methodComponentParams = methodComponentContext.getParameters(); - - // The method component parameters should have an encoder - if (!methodComponentParams.containsKey(METHOD_ENCODER_PARAMETER)) { - return false; - } - - // Validate if the object is of type MethodComponentContext before casting it later - if (!(methodComponentParams.get(METHOD_ENCODER_PARAMETER) instanceof MethodComponentContext)) { - return false; - } - - MethodComponentContext encoderMethodComponentContext = (MethodComponentContext) methodComponentParams.get(METHOD_ENCODER_PARAMETER); - - // returns true if encoder name is "sq" and type is "fp16" - return ENCODER_SQ.equals(encoderMethodComponentContext.getName()) - && FAISS_SQ_ENCODER_FP16.equals( - encoderMethodComponentContext.getParameters().getOrDefault(FAISS_SQ_TYPE, FAISS_SQ_ENCODER_FP16) - ); - - } - - // Verify mapping and return the value of "clip" parameter(default false) for a "faiss" Index - // using "sq" encoder of type "fp16". - protected boolean isFaissSQClipToFP16RangeEnabled(MethodComponentContext methodComponentContext) { - if (Objects.nonNull(methodComponentContext)) { - return (boolean) methodComponentContext.getParameters().getOrDefault(FAISS_SQ_CLIP, false); - } - return false; - } - - void validateIfCircuitBreakerIsNotTriggered() { - if (KNNSettings.isCircuitBreakerTriggered()) { - throw new KnnCircuitBreakerException( - "Parsing the created knn vector fields prior to indexing has failed as the circuit breaker triggered. This indicates that the cluster is low on memory resources and cannot index more documents at the moment. Check _plugins/_knn/stats for the circuit breaker status." - ); - } - } - - void validateIfKNNPluginEnabled() { - if (!KNNSettings.isKNNPluginEnabled()) { - throw new IllegalStateException("KNN plugin is disabled. To enable update knn.plugin.enabled setting to true"); - } - } - // Returns an optional array of byte values where each value in the vector is parsed as a float and validated // if it is a finite number without any decimals and within the byte range of [-128 to 127]. Optional getBytesFromContext(ParseContext context, int dimension, VectorDataType dataType) throws IOException { context.path().add(simpleName()); + PerDimensionValidator perDimensionValidator = getPerDimensionValidator(); + PerDimensionProcessor perDimensionProcessor = getPerDimensionProcessor(); + ArrayList vector = new ArrayList<>(); XContentParser.Token token = context.parser().currentToken(); if (token == XContentParser.Token.START_ARRAY) { token = context.parser().nextToken(); while (token != XContentParser.Token.END_ARRAY) { - float value = context.parser().floatValue(); - validateByteVectorValue(value, dataType); + float value = perDimensionProcessor.processByte(context.parser().floatValue()); + perDimensionValidator.validateByte(value); vector.add((byte) value); token = context.parser().nextToken(); } } else if (token == XContentParser.Token.VALUE_NUMBER) { - float value = context.parser().floatValue(); - validateByteVectorValue(value, dataType); + float value = perDimensionProcessor.processByte(context.parser().floatValue()); + perDimensionValidator.validateByte(value); vector.add((byte) value); context.parser().nextToken(); } else if (token == XContentParser.Token.VALUE_NULL) { @@ -681,21 +634,11 @@ Optional getBytesFromContext(ParseContext context, int dimension, Vector return Optional.of(array); } - Optional getFloatsFromContext(ParseContext context, int dimension, MethodComponentContext methodComponentContext) - throws IOException { + Optional getFloatsFromContext(ParseContext context, int dimension) throws IOException { context.path().add(simpleName()); - // Returns an optional array of float values where each value in the vector is parsed as a float and validated - // if it is a finite number and within the fp16 range of [-65504 to 65504] by default if Faiss encoder is SQ and type is 'fp16'. - // If the encoder parameter, "clip" is set to True, if the vector value is outside the FP16 range then it will be - // clipped to FP16 range. - boolean isFaissSQfp16Flag = isFaissSQfp16(methodComponentContext); - boolean clipVectorValueToFP16RangeFlag = false; - if (isFaissSQfp16Flag) { - clipVectorValueToFP16RangeFlag = isFaissSQClipToFP16RangeEnabled( - (MethodComponentContext) methodComponentContext.getParameters().get(METHOD_ENCODER_PARAMETER) - ); - } + PerDimensionValidator perDimensionValidator = getPerDimensionValidator(); + PerDimensionProcessor perDimensionProcessor = getPerDimensionProcessor(); ArrayList vector = new ArrayList<>(); XContentParser.Token token = context.parser().currentToken(); @@ -703,31 +646,14 @@ Optional getFloatsFromContext(ParseContext context, int dimension, Meth if (token == XContentParser.Token.START_ARRAY) { token = context.parser().nextToken(); while (token != XContentParser.Token.END_ARRAY) { - value = context.parser().floatValue(); - if (isFaissSQfp16Flag) { - if (clipVectorValueToFP16RangeFlag) { - value = clipVectorValueToFP16Range(value); - } else { - validateFP16VectorValue(value); - } - } else { - validateFloatVectorValue(value); - } - + value = perDimensionProcessor.process(context.parser().floatValue()); + perDimensionValidator.validate(value); vector.add(value); token = context.parser().nextToken(); } } else if (token == XContentParser.Token.VALUE_NUMBER) { - value = context.parser().floatValue(); - if (isFaissSQfp16Flag) { - if (clipVectorValueToFP16RangeFlag) { - value = clipVectorValueToFP16Range(value); - } else { - validateFP16VectorValue(value); - } - } else { - validateFloatVectorValue(value); - } + value = perDimensionProcessor.process(context.parser().floatValue()); + perDimensionValidator.validate(value); vector.add(value); context.parser().nextToken(); } else if (token == XContentParser.Token.VALUE_NULL) { @@ -746,7 +672,12 @@ Optional getFloatsFromContext(ParseContext context, int dimension, Meth @Override public ParametrizedFieldMapper.Builder getMergeBuilder() { - return new KNNVectorFieldMapper.Builder(simpleName(), modelDao, indexCreatedVersion).init(this); + return new KNNVectorFieldMapper.Builder( + simpleName(), + modelDao, + indexCreatedVersion, + fieldType().getKnnMappingConfig().getKnnMethodContext().orElse(null) + ).init(this); } @Override diff --git a/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperUtil.java b/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperUtil.java index 2adbbb695..0caaf80ab 100644 --- a/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperUtil.java +++ b/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperUtil.java @@ -13,30 +13,45 @@ import lombok.AccessLevel; import lombok.NoArgsConstructor; +import lombok.extern.log4j.Log4j2; import org.apache.lucene.document.FieldType; import org.apache.lucene.document.StoredField; import org.apache.lucene.index.DocValuesType; import org.apache.lucene.util.BytesRef; -import org.opensearch.index.mapper.ParametrizedFieldMapper; +import org.opensearch.Version; +import org.opensearch.common.settings.Settings; +import org.opensearch.index.mapper.Mapper; +import org.opensearch.knn.index.KNNSettings; +import org.opensearch.knn.index.KnnCircuitBreakerException; +import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.engine.KNNMethodContext; import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.codec.util.KNNVectorSerializerFactory; import org.opensearch.knn.index.engine.KNNEngine; -import org.opensearch.knn.indices.ModelDao; -import org.opensearch.knn.indices.ModelMetadata; -import org.opensearch.knn.indices.ModelUtil; +import org.opensearch.knn.index.engine.MethodComponentContext; +import org.opensearch.knn.index.util.IndexHyperParametersUtil; import java.util.Arrays; import java.util.Locale; +import java.util.Map; +import java.util.Objects; import static org.opensearch.knn.common.KNNConstants.ENCODER_SQ; import static org.opensearch.knn.common.KNNConstants.FAISS_NAME; +import static org.opensearch.knn.common.KNNConstants.FAISS_SQ_CLIP; import static org.opensearch.knn.common.KNNConstants.FAISS_SQ_ENCODER_FP16; +import static org.opensearch.knn.common.KNNConstants.FAISS_SQ_TYPE; import static org.opensearch.knn.common.KNNConstants.FP16_MAX_VALUE; import static org.opensearch.knn.common.KNNConstants.FP16_MIN_VALUE; +import static org.opensearch.knn.common.KNNConstants.HNSW_ALGO_EF_CONSTRUCTION; +import static org.opensearch.knn.common.KNNConstants.HNSW_ALGO_M; import static org.opensearch.knn.common.KNNConstants.KNN_ENGINE; import static org.opensearch.knn.common.KNNConstants.LUCENE_NAME; +import static org.opensearch.knn.common.KNNConstants.METHOD_ENCODER_PARAMETER; import static org.opensearch.knn.common.KNNConstants.METHOD_HNSW; +import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_EF_CONSTRUCTION; +import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_M; +import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_SPACE_TYPE; import static org.opensearch.knn.common.KNNConstants.NMSLIB_NAME; import static org.opensearch.knn.common.KNNConstants.VECTOR_DATA_TYPE_FIELD; import static org.opensearch.knn.common.KNNValidationUtil.validateFloatVectorValue; @@ -44,19 +59,10 @@ /** * Utility class for KNNVectorFieldMapper */ +@Log4j2 @NoArgsConstructor(access = AccessLevel.PRIVATE) public class KNNVectorFieldMapperUtil { - private static ModelDao modelDao; - - /** - * Initializes static instance variables - * @param modelDao ModelDao object - */ - public static void initialize(final ModelDao modelDao) { - KNNVectorFieldMapperUtil.modelDao = modelDao; - } - /** * Validate the float vector value and throw exception if it is not a number or not in the finite range * or is not within the FP16 range of [-65504 to 65504]. @@ -150,35 +156,6 @@ public static void validateVectorDataType(KNNMethodContext methodContext, Vector throw new IllegalArgumentException("This line should not be reached"); } - /** - * Validates and throws exception if index.knn is set to true in the index settings - * using any VectorDataType (other than float, which is default) because we are using NMSLIB engine - * for LegacyFieldMapper, and it only supports float VectorDataType - * - * @param knnIndexSetting index.knn setting in the index settings - * @param vectorDataType VectorDataType Parameter - */ - public static void validateVectorDataTypeWithKnnIndexSetting( - boolean knnIndexSetting, - ParametrizedFieldMapper.Parameter vectorDataType - ) { - - if (VectorDataType.FLOAT == vectorDataType.getValue()) { - return; - } - if (knnIndexSetting) { - throw new IllegalArgumentException( - String.format( - Locale.ROOT, - "[%s] field with value [%s] is not supported for [%s] engine", - VECTOR_DATA_TYPE_FIELD, - vectorDataType.getValue().getValue(), - NMSLIB_NAME - ) - ); - } - } - /** * @param knnEngine KNNEngine * @return DocValues FieldType of type Binary @@ -237,37 +214,172 @@ public static Object deserializeStoredVector(BytesRef storedVector, VectorDataTy * @return expected vector length */ public static int getExpectedVectorLength(final KNNVectorFieldType knnVectorFieldType) { - int expectedDimensions = knnVectorFieldType.getDimension(); - if (isModelBasedIndex(expectedDimensions)) { - ModelMetadata modelMetadata = getModelMetadataForField(knnVectorFieldType); - expectedDimensions = modelMetadata.getDimension(); - } + int expectedDimensions = knnVectorFieldType.getKnnMappingConfig().getDimension(); return VectorDataType.BINARY == knnVectorFieldType.getVectorDataType() ? expectedDimensions / 8 : expectedDimensions; } - private static boolean isModelBasedIndex(int expectedDimensions) { - return expectedDimensions == -1; + /** + * Validate if the circuit breaker is triggered + */ + static void validateIfCircuitBreakerIsNotTriggered() { + if (KNNSettings.isCircuitBreakerTriggered()) { + throw new KnnCircuitBreakerException( + "Parsing the created knn vector fields prior to indexing has failed as the circuit breaker triggered. This indicates that the cluster is low on memory resources and cannot index more documents at the moment. Check _plugins/_knn/stats for the circuit breaker status." + ); + } } /** - * Returns the model metadata for a specified knn vector field + * Validate if plugin is enabled + */ + static void validateIfKNNPluginEnabled() { + if (!KNNSettings.isKNNPluginEnabled()) { + throw new IllegalStateException("KNN plugin is disabled. To enable update knn.plugin.enabled setting to true"); + } + } + + private static SpaceType getSpaceType(final Settings indexSettings, final VectorDataType vectorDataType) { + String spaceType = indexSettings.get(KNNSettings.INDEX_KNN_SPACE_TYPE.getKey()); + if (spaceType == null) { + spaceType = VectorDataType.BINARY == vectorDataType + ? KNNSettings.INDEX_KNN_DEFAULT_SPACE_TYPE_FOR_BINARY + : KNNSettings.INDEX_KNN_DEFAULT_SPACE_TYPE; + log.info( + String.format( + "[KNN] The setting \"%s\" was not set for the index. Likely caused by recent version upgrade. Setting the setting to the default value=%s", + METHOD_PARAMETER_SPACE_TYPE, + spaceType + ) + ); + } + return SpaceType.getSpace(spaceType); + } + + private static int getM(Settings indexSettings) { + String m = indexSettings.get(KNNSettings.INDEX_KNN_ALGO_PARAM_M_SETTING.getKey()); + if (m == null) { + log.info( + String.format( + "[KNN] The setting \"%s\" was not set for the index. Likely caused by recent version upgrade. Setting the setting to the default value=%s", + HNSW_ALGO_M, + KNNSettings.INDEX_KNN_DEFAULT_ALGO_PARAM_M + ) + ); + return KNNSettings.INDEX_KNN_DEFAULT_ALGO_PARAM_M; + } + return Integer.parseInt(m); + } + + private static int getEfConstruction(Settings indexSettings, Version indexVersion) { + final String efConstruction = indexSettings.get(KNNSettings.INDEX_KNN_ALGO_PARAM_EF_CONSTRUCTION_SETTING.getKey()); + if (efConstruction == null) { + final int defaultEFConstructionValue = IndexHyperParametersUtil.getHNSWEFConstructionValue(indexVersion); + log.info( + String.format( + "[KNN] The setting \"%s\" was not set for the index. Likely caused by recent version upgrade. " + + "Picking up default value for the index =%s", + HNSW_ALGO_EF_CONSTRUCTION, + defaultEFConstructionValue + ) + ); + return defaultEFConstructionValue; + } + return Integer.parseInt(efConstruction); + } + + /** + * Verify mapping and return true if it is a "faiss" Index using "sq" encoder of type "fp16" * - * @param knnVectorField knn vector field - * @return the model metadata from knnVectorField + * @param methodComponentContext MethodComponentContext + * @return true if it is a "faiss" Index using "sq" encoder of type "fp16" */ - private static ModelMetadata getModelMetadataForField(final KNNVectorFieldType knnVectorField) { - String modelId = knnVectorField.getModelId(); + static boolean isFaissSQfp16(MethodComponentContext methodComponentContext) { + if (Objects.isNull(methodComponentContext)) { + return false; + } - if (modelId == null) { - throw new IllegalArgumentException( - String.format("Field '%s' does not have model.", knnVectorField.getKnnMethodContext().getMethodComponentContext().getName()) + if (methodComponentContext.getParameters().size() == 0) { + return false; + } + + Map methodComponentParams = methodComponentContext.getParameters(); + + // The method component parameters should have an encoder + if (!methodComponentParams.containsKey(METHOD_ENCODER_PARAMETER)) { + return false; + } + + // Validate if the object is of type MethodComponentContext before casting it later + if (!(methodComponentParams.get(METHOD_ENCODER_PARAMETER) instanceof MethodComponentContext)) { + return false; + } + + MethodComponentContext encoderMethodComponentContext = (MethodComponentContext) methodComponentParams.get(METHOD_ENCODER_PARAMETER); + + // returns true if encoder name is "sq" and type is "fp16" + return ENCODER_SQ.equals(encoderMethodComponentContext.getName()) + && FAISS_SQ_ENCODER_FP16.equals( + encoderMethodComponentContext.getParameters().getOrDefault(FAISS_SQ_TYPE, FAISS_SQ_ENCODER_FP16) ); + + } + + /** + * Verify mapping and return the value of "clip" parameter(default false) for a "faiss" Index + * using "sq" encoder of type "fp16". + * + * @param methodComponentContext MethodComponentContext + * @return boolean value of "clip" parameter + */ + static boolean isFaissSQClipToFP16RangeEnabled(MethodComponentContext methodComponentContext) { + if (Objects.nonNull(methodComponentContext)) { + return (boolean) methodComponentContext.getParameters().getOrDefault(FAISS_SQ_CLIP, false); } + return false; + } - ModelMetadata modelMetadata = modelDao.getMetadata(modelId); - if (!ModelUtil.isModelCreated(modelMetadata)) { - throw new IllegalArgumentException(String.format("Model ID '%s' is not created.", modelId)); + /** + * Extract MethodComponentContext from KNNMethodContext + * + * @param knnMethodContext KNNMethodContext + * @return MethodComponentContext + */ + static MethodComponentContext getMethodComponentContext(KNNMethodContext knnMethodContext) { + if (Objects.isNull(knnMethodContext)) { + return null; } - return modelMetadata; + return knnMethodContext.getMethodComponentContext(); + } + + static KNNMethodContext createKNNMethodContextFromLegacy( + Mapper.BuilderContext context, + VectorDataType vectorDataType, + Version indexCreatedVersion + ) { + if (VectorDataType.FLOAT != vectorDataType) { + throw new IllegalArgumentException( + String.format( + Locale.ROOT, + "[%s] field with value [%s] is not supported for [%s] engine", + VECTOR_DATA_TYPE_FIELD, + vectorDataType.getValue(), + NMSLIB_NAME + ) + ); + } + + return new KNNMethodContext( + KNNEngine.NMSLIB, + KNNVectorFieldMapperUtil.getSpaceType(context.indexSettings(), vectorDataType), + new MethodComponentContext( + METHOD_HNSW, + Map.of( + METHOD_PARAMETER_M, + KNNVectorFieldMapperUtil.getM(context.indexSettings()), + METHOD_PARAMETER_EF_CONSTRUCTION, + KNNVectorFieldMapperUtil.getEfConstruction(context.indexSettings(), indexCreatedVersion) + ) + ) + ); } } diff --git a/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldType.java b/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldType.java index 8c3815c5f..0fbc569f7 100644 --- a/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldType.java +++ b/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldType.java @@ -9,7 +9,6 @@ import org.apache.lucene.search.DocValuesFieldExistsQuery; import org.apache.lucene.search.Query; import org.apache.lucene.util.BytesRef; -import org.opensearch.common.Nullable; import org.opensearch.index.fielddata.IndexFieldData; import org.opensearch.index.mapper.MappedFieldType; import org.opensearch.index.mapper.TextSearchInfo; @@ -17,9 +16,7 @@ import org.opensearch.index.query.QueryShardContext; import org.opensearch.index.query.QueryShardException; import org.opensearch.knn.index.KNNVectorIndexFieldData; -import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.VectorDataType; -import org.opensearch.knn.index.engine.KNNMethodContext; import org.opensearch.search.aggregations.support.CoreValuesSourceType; import org.opensearch.search.lookup.SearchLookup; @@ -27,7 +24,6 @@ import java.util.Map; import java.util.function.Supplier; -import static org.opensearch.knn.common.KNNConstants.DEFAULT_VECTOR_DATA_TYPE_FIELD; import static org.opensearch.knn.index.mapper.KNNVectorFieldMapperUtil.deserializeStoredVector; /** @@ -35,49 +31,21 @@ */ @Getter public class KNNVectorFieldType extends MappedFieldType { - int dimension; - String modelId; - KNNMethodContext knnMethodContext; + KNNMappingConfig knnMappingConfig; VectorDataType vectorDataType; - SpaceType spaceType; - public KNNVectorFieldType(String name, Map meta, int dimension, VectorDataType vectorDataType, SpaceType spaceType) { - this(name, meta, dimension, null, null, vectorDataType, spaceType); - } - - public KNNVectorFieldType(String name, Map meta, int dimension, KNNMethodContext knnMethodContext) { - this(name, meta, dimension, knnMethodContext, null, DEFAULT_VECTOR_DATA_TYPE_FIELD, knnMethodContext.getSpaceType()); - } - - public KNNVectorFieldType(String name, Map meta, int dimension, KNNMethodContext knnMethodContext, String modelId) { - this(name, meta, dimension, knnMethodContext, modelId, DEFAULT_VECTOR_DATA_TYPE_FIELD, null); - } - - public KNNVectorFieldType( - String name, - Map meta, - int dimension, - KNNMethodContext knnMethodContext, - VectorDataType vectorDataType - ) { - this(name, meta, dimension, knnMethodContext, null, vectorDataType, knnMethodContext.getSpaceType()); - } - - public KNNVectorFieldType( - String name, - Map meta, - int dimension, - @Nullable KNNMethodContext knnMethodContext, - @Nullable String modelId, - VectorDataType vectorDataType, - @Nullable SpaceType spaceType - ) { - super(name, false, false, true, TextSearchInfo.NONE, meta); - this.dimension = dimension; - this.modelId = modelId; - this.knnMethodContext = knnMethodContext; + /** + * Constructor for KNNVectorFieldType. + * + * @param name name of the field + * @param metadata metadata of the field + * @param vectorDataType data type of the vector + * @param annConfig configuration context for the ANN index + */ + public KNNVectorFieldType(String name, Map metadata, VectorDataType vectorDataType, KNNMappingConfig annConfig) { + super(name, false, false, true, TextSearchInfo.NONE, metadata); this.vectorDataType = vectorDataType; - this.spaceType = spaceType; + this.knnMappingConfig = annConfig; } @Override diff --git a/src/main/java/org/opensearch/knn/index/mapper/LegacyFieldMapper.java b/src/main/java/org/opensearch/knn/index/mapper/LegacyFieldMapper.java deleted file mode 100644 index cf5ec933a..000000000 --- a/src/main/java/org/opensearch/knn/index/mapper/LegacyFieldMapper.java +++ /dev/null @@ -1,130 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.knn.index.mapper; - -import lombok.extern.log4j.Log4j2; -import org.apache.lucene.document.FieldType; -import org.opensearch.Version; -import org.opensearch.common.Explicit; -import org.opensearch.common.settings.Settings; -import org.opensearch.index.mapper.ParametrizedFieldMapper; -import org.opensearch.knn.index.KNNSettings; -import org.opensearch.knn.index.VectorDataType; -import org.opensearch.knn.index.util.IndexHyperParametersUtil; -import org.opensearch.knn.index.engine.KNNEngine; - -import static org.opensearch.knn.common.KNNConstants.DIMENSION; -import static org.opensearch.knn.common.KNNConstants.HNSW_ALGO_EF_CONSTRUCTION; -import static org.opensearch.knn.common.KNNConstants.HNSW_ALGO_M; -import static org.opensearch.knn.common.KNNConstants.KNN_ENGINE; -import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_SPACE_TYPE; -import static org.opensearch.knn.common.KNNConstants.SPACE_TYPE; - -/** - * Field mapper for original implementation. It defaults to using nmslib as the engine and retrieves parameters from index settings. - * - * Example of this mapper output: - * - * { - * "type": "knn_vector", - * "dimension": 128 - * } - */ -@Log4j2 -public class LegacyFieldMapper extends KNNVectorFieldMapper { - - protected String spaceType; - protected String m; - protected String efConstruction; - - LegacyFieldMapper( - String simpleName, - KNNVectorFieldType mappedFieldType, - MultiFields multiFields, - CopyTo copyTo, - Explicit ignoreMalformed, - boolean stored, - boolean hasDocValues, - String spaceType, - String m, - String efConstruction, - Version indexCreatedVersion - ) { - super(simpleName, mappedFieldType, multiFields, copyTo, ignoreMalformed, stored, hasDocValues, indexCreatedVersion); - - this.spaceType = spaceType; - this.m = m; - this.efConstruction = efConstruction; - - this.fieldType = new FieldType(KNNVectorFieldMapper.Defaults.FIELD_TYPE); - - this.fieldType.putAttribute(DIMENSION, String.valueOf(dimension)); - this.fieldType.putAttribute(SPACE_TYPE, spaceType); - this.fieldType.putAttribute(KNN_ENGINE, KNNEngine.NMSLIB.getName()); - - // These are extra just for legacy - this.fieldType.putAttribute(HNSW_ALGO_M, m); - this.fieldType.putAttribute(HNSW_ALGO_EF_CONSTRUCTION, efConstruction); - - this.fieldType.freeze(); - } - - @Override - public ParametrizedFieldMapper.Builder getMergeBuilder() { - return new KNNVectorFieldMapper.Builder(simpleName(), this.spaceType, this.m, this.efConstruction, this.indexCreatedVersion).init( - this - ); - } - - static String getSpaceType(final Settings indexSettings, final VectorDataType vectorDataType) { - String spaceType = indexSettings.get(KNNSettings.INDEX_KNN_SPACE_TYPE.getKey()); - if (spaceType == null) { - spaceType = VectorDataType.BINARY == vectorDataType - ? KNNSettings.INDEX_KNN_DEFAULT_SPACE_TYPE_FOR_BINARY - : KNNSettings.INDEX_KNN_DEFAULT_SPACE_TYPE; - log.info( - String.format( - "[KNN] The setting \"%s\" was not set for the index. Likely caused by recent version upgrade. Setting the setting to the default value=%s", - METHOD_PARAMETER_SPACE_TYPE, - spaceType - ) - ); - } - return spaceType; - } - - static String getM(Settings indexSettings) { - String m = indexSettings.get(KNNSettings.INDEX_KNN_ALGO_PARAM_M_SETTING.getKey()); - if (m == null) { - log.info( - String.format( - "[KNN] The setting \"%s\" was not set for the index. Likely caused by recent version upgrade. Setting the setting to the default value=%s", - HNSW_ALGO_M, - KNNSettings.INDEX_KNN_DEFAULT_ALGO_PARAM_M - ) - ); - return String.valueOf(KNNSettings.INDEX_KNN_DEFAULT_ALGO_PARAM_M); - } - return m; - } - - static String getEfConstruction(Settings indexSettings, Version indexVersion) { - final String efConstruction = indexSettings.get(KNNSettings.INDEX_KNN_ALGO_PARAM_EF_CONSTRUCTION_SETTING.getKey()); - if (efConstruction == null) { - final String defaultEFConstructionValue = String.valueOf(IndexHyperParametersUtil.getHNSWEFConstructionValue(indexVersion)); - log.info( - String.format( - "[KNN] The setting \"%s\" was not set for the index. Likely caused by recent version upgrade. " - + "Picking up default value for the index =%s", - HNSW_ALGO_EF_CONSTRUCTION, - defaultEFConstructionValue - ) - ); - return defaultEFConstructionValue; - } - return efConstruction; - } -} diff --git a/src/main/java/org/opensearch/knn/index/mapper/LuceneFieldMapper.java b/src/main/java/org/opensearch/knn/index/mapper/LuceneFieldMapper.java index c82afb9e7..665c35f6e 100644 --- a/src/main/java/org/opensearch/knn/index/mapper/LuceneFieldMapper.java +++ b/src/main/java/org/opensearch/knn/index/mapper/LuceneFieldMapper.java @@ -8,6 +8,9 @@ import java.util.ArrayList; import java.util.List; import java.util.Locale; +import java.util.Map; +import java.util.Optional; + import lombok.AllArgsConstructor; import lombok.Getter; import lombok.NonNull; @@ -16,11 +19,12 @@ import org.apache.lucene.document.KnnByteVectorField; import org.apache.lucene.document.KnnVectorField; import org.apache.lucene.index.VectorSimilarityFunction; +import org.opensearch.Version; import org.opensearch.common.Explicit; -import org.opensearch.knn.index.engine.KNNMethodContext; import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.VectorField; import org.opensearch.knn.index.engine.KNNEngine; +import org.opensearch.knn.index.engine.KNNMethodContext; import static org.opensearch.knn.index.mapper.KNNVectorFieldMapperUtil.createStoredFieldForByteVector; import static org.opensearch.knn.index.mapper.KNNVectorFieldMapperUtil.createStoredFieldForFloatVector; @@ -35,44 +39,76 @@ public class LuceneFieldMapper extends KNNVectorFieldMapper { private final FieldType vectorFieldType; private final VectorDataType vectorDataType; - LuceneFieldMapper(final CreateLuceneFieldMapperInput input) { + private PerDimensionProcessor perDimensionProcessor; + private PerDimensionValidator perDimensionValidator; + private VectorValidator vectorValidator; + + static LuceneFieldMapper createFieldMapper( + String fullname, + Map metaValue, + VectorDataType vectorDataType, + Integer dimension, + KNNMethodContext knnMethodContext, + CreateLuceneFieldMapperInput createLuceneFieldMapperInput + ) { + final KNNVectorFieldType mappedFieldType = new KNNVectorFieldType(fullname, metaValue, vectorDataType, new KNNMappingConfig() { + @Override + public Optional getKnnMethodContext() { + return Optional.of(knnMethodContext); + } + + @Override + public int getDimension() { + return dimension; + } + }); + + return new LuceneFieldMapper(mappedFieldType, createLuceneFieldMapperInput); + } + + private LuceneFieldMapper(final KNNVectorFieldType mappedFieldType, final CreateLuceneFieldMapperInput input) { super( input.getName(), - input.getMappedFieldType(), + mappedFieldType, input.getMultiFields(), input.getCopyTo(), input.getIgnoreMalformed(), input.isStored(), input.isHasDocValues(), - input.getKnnMethodContext().getMethodComponentContext().getIndexVersion() + input.getIndexVersion(), + mappedFieldType.knnMappingConfig.getKnnMethodContext().orElse(null) ); - + KNNMappingConfig knnMappingConfig = mappedFieldType.getKnnMappingConfig(); + KNNMethodContext knnMethodContext = knnMappingConfig.getKnnMethodContext() + .orElseThrow(() -> new IllegalArgumentException("KNN method context is missing")); vectorDataType = input.getVectorDataType(); - this.knnMethod = input.getKnnMethodContext(); - final VectorSimilarityFunction vectorSimilarityFunction = this.knnMethod.getSpaceType() + + final VectorSimilarityFunction vectorSimilarityFunction = knnMethodContext.getSpaceType() .getKnnVectorSimilarityFunction() .getVectorSimilarityFunction(); - final int dimension = input.getMappedFieldType().getDimension(); - if (dimension > KNNEngine.getMaxDimensionByEngine(KNNEngine.LUCENE)) { + if (knnMappingConfig.getDimension() > KNNEngine.getMaxDimensionByEngine(KNNEngine.LUCENE)) { throw new IllegalArgumentException( String.format( Locale.ROOT, "Dimension value cannot be greater than [%s] but got [%s] for vector [%s]", KNNEngine.getMaxDimensionByEngine(KNNEngine.LUCENE), - dimension, + knnMappingConfig.getDimension(), input.getName() ) ); } - this.fieldType = vectorDataType.createKnnVectorFieldType(dimension, vectorSimilarityFunction); + this.fieldType = vectorDataType.createKnnVectorFieldType(knnMappingConfig.getDimension(), vectorSimilarityFunction); if (this.hasDocValues) { - this.vectorFieldType = buildDocValuesFieldType(this.knnMethod.getKnnEngine()); + this.vectorFieldType = buildDocValuesFieldType(knnMethodContext.getKnnEngine()); } else { this.vectorFieldType = null; } + + initValidatorsAndProcessors(knnMethodContext); + knnMethodContext.getSpaceType().validateVectorDataType(vectorDataType); } @Override @@ -105,6 +141,36 @@ protected List getFieldsForByteVector(final byte[] array, final FieldType return fieldsToBeAdded; } + private void initValidatorsAndProcessors(KNNMethodContext knnMethodContext) { + this.vectorValidator = new SpaceVectorValidator(knnMethodContext.getSpaceType()); + this.perDimensionProcessor = PerDimensionProcessor.NOOP_PROCESSOR; + if (VectorDataType.BINARY == vectorDataType) { + this.perDimensionValidator = PerDimensionValidator.DEFAULT_BIT_VALIDATOR; + return; + } + + if (VectorDataType.BYTE == vectorDataType) { + this.perDimensionValidator = PerDimensionValidator.DEFAULT_BYTE_VALIDATOR; + return; + } + this.perDimensionValidator = PerDimensionValidator.DEFAULT_FLOAT_VALIDATOR; + } + + @Override + protected VectorValidator getVectorValidator() { + return vectorValidator; + } + + @Override + protected PerDimensionValidator getPerDimensionValidator() { + return perDimensionValidator; + } + + @Override + protected PerDimensionProcessor getPerDimensionProcessor() { + return perDimensionProcessor; + } + @Override void updateEngineStats() { KNNEngine.LUCENE.setInitialized(true); @@ -117,8 +183,6 @@ static class CreateLuceneFieldMapperInput { @NonNull String name; @NonNull - KNNVectorFieldType mappedFieldType; - @NonNull MultiFields multiFields; @NonNull CopyTo copyTo; @@ -127,7 +191,7 @@ static class CreateLuceneFieldMapperInput { boolean stored; boolean hasDocValues; VectorDataType vectorDataType; - @NonNull - KNNMethodContext knnMethodContext; + Version indexVersion; + KNNMethodContext originalKnnMethodContext; } } diff --git a/src/main/java/org/opensearch/knn/index/mapper/MethodFieldMapper.java b/src/main/java/org/opensearch/knn/index/mapper/MethodFieldMapper.java index b15ab1489..7a69c941b 100644 --- a/src/main/java/org/opensearch/knn/index/mapper/MethodFieldMapper.java +++ b/src/main/java/org/opensearch/knn/index/mapper/MethodFieldMapper.java @@ -6,37 +6,64 @@ package org.opensearch.knn.index.mapper; import org.apache.lucene.document.FieldType; +import org.opensearch.Version; import org.opensearch.common.Explicit; import org.opensearch.common.xcontent.XContentFactory; -import org.opensearch.knn.index.engine.KNNMethodContext; +import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.engine.KNNEngine; +import org.opensearch.knn.index.engine.KNNMethodContext; +import org.opensearch.knn.index.engine.MethodComponentContext; import java.io.IOException; import java.util.Map; +import java.util.Optional; import static org.opensearch.knn.common.KNNConstants.DIMENSION; import static org.opensearch.knn.common.KNNConstants.KNN_ENGINE; +import static org.opensearch.knn.common.KNNConstants.METHOD_ENCODER_PARAMETER; import static org.opensearch.knn.common.KNNConstants.PARAMETERS; import static org.opensearch.knn.common.KNNConstants.SPACE_TYPE; import static org.opensearch.knn.common.KNNConstants.VECTOR_DATA_TYPE_FIELD; +import static org.opensearch.knn.index.mapper.KNNVectorFieldMapperUtil.getMethodComponentContext; +import static org.opensearch.knn.index.mapper.KNNVectorFieldMapperUtil.isFaissSQClipToFP16RangeEnabled; +import static org.opensearch.knn.index.mapper.KNNVectorFieldMapperUtil.isFaissSQfp16; /** * Field mapper for method definition in mapping */ public class MethodFieldMapper extends KNNVectorFieldMapper { - MethodFieldMapper( + private PerDimensionProcessor perDimensionProcessor; + private PerDimensionValidator perDimensionValidator; + private VectorValidator vectorValidator; + + public static MethodFieldMapper createFieldMapper( + String fullname, String simpleName, - KNNVectorFieldType mappedFieldType, + Map metaValue, + VectorDataType vectorDataType, + Integer dimension, + KNNMethodContext knnMethodContext, + KNNMethodContext originalKNNMethodContext, MultiFields multiFields, CopyTo copyTo, Explicit ignoreMalformed, boolean stored, boolean hasDocValues, - KNNMethodContext knnMethodContext + Version indexCreatedVersion ) { + final KNNVectorFieldType mappedFieldType = new KNNVectorFieldType(fullname, metaValue, vectorDataType, new KNNMappingConfig() { + @Override + public Optional getKnnMethodContext() { + return Optional.of(knnMethodContext); + } - super( + @Override + public int getDimension() { + return dimension; + } + }); + return new MethodFieldMapper( simpleName, mappedFieldType, multiFields, @@ -44,14 +71,40 @@ public class MethodFieldMapper extends KNNVectorFieldMapper { ignoreMalformed, stored, hasDocValues, - knnMethodContext.getMethodComponentContext().getIndexVersion() + indexCreatedVersion, + originalKNNMethodContext ); + } - this.knnMethod = knnMethodContext; + private MethodFieldMapper( + String simpleName, + KNNVectorFieldType mappedFieldType, + MultiFields multiFields, + CopyTo copyTo, + Explicit ignoreMalformed, + boolean stored, + boolean hasDocValues, + Version indexVerision, + KNNMethodContext originalKNNMethodContext + ) { + super( + simpleName, + mappedFieldType, + multiFields, + copyTo, + ignoreMalformed, + stored, + hasDocValues, + indexVerision, + originalKNNMethodContext + ); + KNNMappingConfig annConfig = mappedFieldType.getKnnMappingConfig(); + KNNMethodContext knnMethodContext = annConfig.getKnnMethodContext() + .orElseThrow(() -> new IllegalArgumentException("KNN method context cannot be empty")); this.fieldType = new FieldType(KNNVectorFieldMapper.Defaults.FIELD_TYPE); - this.fieldType.putAttribute(DIMENSION, String.valueOf(dimension)); + this.fieldType.putAttribute(DIMENSION, String.valueOf(annConfig.getDimension())); this.fieldType.putAttribute(SPACE_TYPE, knnMethodContext.getSpaceType().getValue()); this.fieldType.putAttribute(VECTOR_DATA_TYPE_FIELD, vectorDataType.getValue()); @@ -66,5 +119,57 @@ public class MethodFieldMapper extends KNNVectorFieldMapper { } this.fieldType.freeze(); + initValidatorsAndProcessors(knnMethodContext); + knnMethodContext.getSpaceType().validateVectorDataType(vectorDataType); + } + + private void initValidatorsAndProcessors(KNNMethodContext knnMethodContext) { + this.vectorValidator = new SpaceVectorValidator(knnMethodContext.getSpaceType()); + + if (VectorDataType.BINARY == vectorDataType) { + this.perDimensionValidator = PerDimensionValidator.DEFAULT_BIT_VALIDATOR; + this.perDimensionProcessor = PerDimensionProcessor.NOOP_PROCESSOR; + return; + } + + if (VectorDataType.BYTE == vectorDataType) { + this.perDimensionValidator = PerDimensionValidator.DEFAULT_BYTE_VALIDATOR; + this.perDimensionProcessor = PerDimensionProcessor.NOOP_PROCESSOR; + return; + } + + MethodComponentContext methodComponentContext = getMethodComponentContext(knnMethodContext); + if (!isFaissSQfp16(methodComponentContext)) { + // Normal float and byte processor + this.perDimensionValidator = PerDimensionValidator.DEFAULT_FLOAT_VALIDATOR; + this.perDimensionProcessor = PerDimensionProcessor.NOOP_PROCESSOR; + return; + } + + this.perDimensionValidator = PerDimensionValidator.DEFAULT_FP16_VALIDATOR; + + if (!isFaissSQClipToFP16RangeEnabled( + (MethodComponentContext) methodComponentContext.getParameters().get(METHOD_ENCODER_PARAMETER) + )) { + this.perDimensionProcessor = PerDimensionProcessor.NOOP_PROCESSOR; + return; + } + + this.perDimensionProcessor = PerDimensionProcessor.CLIP_TO_FP16_PROCESSOR; + } + + @Override + protected VectorValidator getVectorValidator() { + return vectorValidator; + } + + @Override + protected PerDimensionValidator getPerDimensionValidator() { + return perDimensionValidator; + } + + @Override + protected PerDimensionProcessor getPerDimensionProcessor() { + return perDimensionProcessor; } } diff --git a/src/main/java/org/opensearch/knn/index/mapper/ModelFieldMapper.java b/src/main/java/org/opensearch/knn/index/mapper/ModelFieldMapper.java index adaaef28e..a21a01a5d 100644 --- a/src/main/java/org/opensearch/knn/index/mapper/ModelFieldMapper.java +++ b/src/main/java/org/opensearch/knn/index/mapper/ModelFieldMapper.java @@ -9,65 +9,198 @@ import org.opensearch.Version; import org.opensearch.common.Explicit; import org.opensearch.index.mapper.ParseContext; +import org.opensearch.knn.index.VectorDataType; +import org.opensearch.knn.index.engine.MethodComponentContext; import org.opensearch.knn.indices.ModelDao; import org.opensearch.knn.indices.ModelMetadata; import org.opensearch.knn.indices.ModelUtil; import java.io.IOException; +import java.util.Map; +import java.util.Optional; +import static org.opensearch.knn.common.KNNConstants.METHOD_ENCODER_PARAMETER; import static org.opensearch.knn.common.KNNConstants.MODEL_ID; +import static org.opensearch.knn.index.mapper.KNNVectorFieldMapperUtil.isFaissSQClipToFP16RangeEnabled; +import static org.opensearch.knn.index.mapper.KNNVectorFieldMapperUtil.isFaissSQfp16; /** * Field mapper for model in mapping */ public class ModelFieldMapper extends KNNVectorFieldMapper { - ModelFieldMapper( + // If the dimension has not yet been set because we do not have access to model metadata, it will be -1 + public static final int UNSET_MODEL_DIMENSION_IDENTIFIER = -1; + + private PerDimensionProcessor perDimensionProcessor; + private PerDimensionValidator perDimensionValidator; + private VectorValidator vectorValidator; + + private final String modelId; + + public static ModelFieldMapper createFieldMapper( + String fullname, String simpleName, - KNNVectorFieldType mappedFieldType, + Map metaValue, + VectorDataType vectorDataType, + String modelId, MultiFields multiFields, CopyTo copyTo, Explicit ignoreMalformed, boolean stored, boolean hasDocValues, ModelDao modelDao, - String modelId, Version indexCreatedVersion ) { - super(simpleName, mappedFieldType, multiFields, copyTo, ignoreMalformed, stored, hasDocValues, indexCreatedVersion); - this.modelId = modelId; + final KNNVectorFieldType mappedFieldType = new KNNVectorFieldType(fullname, metaValue, vectorDataType, new KNNMappingConfig() { + @Override + public Optional getModelId() { + return Optional.of(modelId); + } + + @Override + public int getDimension() { + return getModelMetadata(modelDao, modelId).getDimension(); + } + }); + return new ModelFieldMapper( + simpleName, + mappedFieldType, + multiFields, + copyTo, + ignoreMalformed, + stored, + hasDocValues, + modelDao, + indexCreatedVersion + ); + } + + private ModelFieldMapper( + String simpleName, + KNNVectorFieldType mappedFieldType, + MultiFields multiFields, + CopyTo copyTo, + Explicit ignoreMalformed, + boolean stored, + boolean hasDocValues, + ModelDao modelDao, + Version indexCreatedVersion + ) { + super(simpleName, mappedFieldType, multiFields, copyTo, ignoreMalformed, stored, hasDocValues, indexCreatedVersion, null); + KNNMappingConfig annConfig = mappedFieldType.getKnnMappingConfig(); + modelId = annConfig.getModelId().orElseThrow(() -> new IllegalArgumentException("KNN method context cannot be empty")); this.modelDao = modelDao; + // For the model field mapper, we cannot validate the model during index creation due to + // an issue with reading cluster state during mapper creation. So, we need to validate the + // model when ingestion starts. We do this as lazily as we can + this.perDimensionProcessor = null; + this.perDimensionValidator = null; + this.vectorValidator = null; + this.fieldType = new FieldType(KNNVectorFieldMapper.Defaults.FIELD_TYPE); this.fieldType.putAttribute(MODEL_ID, modelId); this.fieldType.freeze(); } + @Override + protected VectorValidator getVectorValidator() { + initVectorValidator(); + return vectorValidator; + } + + @Override + protected PerDimensionValidator getPerDimensionValidator() { + initPerDimensionValidator(); + return perDimensionValidator; + } + + @Override + protected PerDimensionProcessor getPerDimensionProcessor() { + initPerDimensionProcessor(); + return perDimensionProcessor; + } + + private void initVectorValidator() { + if (vectorValidator != null) { + return; + } + ModelMetadata modelMetadata = getModelMetadata(modelDao, modelId); + vectorValidator = new SpaceVectorValidator(modelMetadata.getSpaceType()); + } + + private void initPerDimensionValidator() { + if (perDimensionValidator != null) { + return; + } + ModelMetadata modelMetadata = getModelMetadata(modelDao, modelId); + MethodComponentContext methodComponentContext = modelMetadata.getMethodComponentContext(); + VectorDataType dataType = modelMetadata.getVectorDataType(); + + if (VectorDataType.BINARY == dataType) { + perDimensionValidator = PerDimensionValidator.DEFAULT_BIT_VALIDATOR; + return; + } + + if (VectorDataType.BYTE == dataType) { + perDimensionValidator = PerDimensionValidator.DEFAULT_BYTE_VALIDATOR; + return; + } + + if (!isFaissSQfp16(methodComponentContext)) { + perDimensionValidator = PerDimensionValidator.DEFAULT_FLOAT_VALIDATOR; + return; + } + + perDimensionValidator = PerDimensionValidator.DEFAULT_FP16_VALIDATOR; + } + + private void initPerDimensionProcessor() { + if (perDimensionProcessor != null) { + return; + } + ModelMetadata modelMetadata = getModelMetadata(modelDao, modelId); + MethodComponentContext methodComponentContext = modelMetadata.getMethodComponentContext(); + VectorDataType dataType = modelMetadata.getVectorDataType(); + + if (VectorDataType.BINARY == dataType) { + perDimensionProcessor = PerDimensionProcessor.NOOP_PROCESSOR; + return; + } + + if (VectorDataType.BYTE == dataType) { + perDimensionProcessor = PerDimensionProcessor.NOOP_PROCESSOR; + return; + } + + if (!isFaissSQfp16(methodComponentContext)) { + perDimensionProcessor = PerDimensionProcessor.NOOP_PROCESSOR; + return; + } + + if (!isFaissSQClipToFP16RangeEnabled( + (MethodComponentContext) methodComponentContext.getParameters().get(METHOD_ENCODER_PARAMETER) + )) { + perDimensionProcessor = PerDimensionProcessor.NOOP_PROCESSOR; + return; + } + perDimensionProcessor = PerDimensionProcessor.CLIP_TO_FP16_PROCESSOR; + } + @Override protected void parseCreateField(ParseContext context) throws IOException { - // For the model field mapper, we cannot validate the model during index creation due to - // an issue with reading cluster state during mapper creation. So, we need to validate the - // model when ingestion starts. - ModelMetadata modelMetadata = this.modelDao.getMetadata(modelId); + validatePreparse(); + ModelMetadata modelMetadata = getModelMetadata(modelDao, modelId); + parseCreateField(context, modelMetadata.getDimension(), modelMetadata.getVectorDataType()); + } + private static ModelMetadata getModelMetadata(ModelDao modelDao, String modelId) { + ModelMetadata modelMetadata = modelDao.getMetadata(modelId); if (!ModelUtil.isModelCreated(modelMetadata)) { - throw new IllegalStateException( - String.format( - "Model \"%s\" from %s's mapping is not created. Because the \"%s\" parameter is not updatable, this index will need to be recreated with a valid model.", - modelId, - context.mapperService().index().getName(), - MODEL_ID - ) - ); + throw new IllegalStateException(String.format("Model ID '%s' is not created.", modelId)); } - - parseCreateField( - context, - modelMetadata.getDimension(), - modelMetadata.getSpaceType(), - modelMetadata.getMethodComponentContext(), - modelMetadata.getVectorDataType() - ); + return modelMetadata; } } diff --git a/src/main/java/org/opensearch/knn/index/mapper/PerDimensionProcessor.java b/src/main/java/org/opensearch/knn/index/mapper/PerDimensionProcessor.java new file mode 100644 index 000000000..21139f2ad --- /dev/null +++ b/src/main/java/org/opensearch/knn/index/mapper/PerDimensionProcessor.java @@ -0,0 +1,51 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.mapper; + +import static org.opensearch.knn.index.mapper.KNNVectorFieldMapperUtil.clipVectorValueToFP16Range; + +/** + * Process values per dimension. Good to have if we want to do some kind of cleanup on data as it is coming in. + */ +public interface PerDimensionProcessor { + + /** + * Process float value per dimension. + * + * @param value value to process + * @return processed value + */ + default float process(float value) { + return value; + } + + /** + * Process byte as float value per dimension. + * + * @param value value to process + * @return processed value + */ + default float processByte(float value) { + return value; + } + + PerDimensionProcessor NOOP_PROCESSOR = new PerDimensionProcessor() { + }; + + // If the encoder parameter, "clip" is set to True, if the vector value is outside the FP16 range then it will be + // clipped to FP16 range. + PerDimensionProcessor CLIP_TO_FP16_PROCESSOR = new PerDimensionProcessor() { + @Override + public float process(float value) { + return clipVectorValueToFP16Range(value); + } + + @Override + public float processByte(float value) { + throw new IllegalStateException("CLIP_TO_FP16_PROCESSOR should not be called with byte type"); + } + }; +} diff --git a/src/main/java/org/opensearch/knn/index/mapper/PerDimensionValidator.java b/src/main/java/org/opensearch/knn/index/mapper/PerDimensionValidator.java new file mode 100644 index 000000000..2ca0761c0 --- /dev/null +++ b/src/main/java/org/opensearch/knn/index/mapper/PerDimensionValidator.java @@ -0,0 +1,80 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.mapper; + +import org.opensearch.knn.index.VectorDataType; + +import static org.opensearch.knn.common.KNNValidationUtil.validateByteVectorValue; +import static org.opensearch.knn.common.KNNValidationUtil.validateFloatVectorValue; +import static org.opensearch.knn.index.mapper.KNNVectorFieldMapperUtil.validateFP16VectorValue; + +/** + * Validates per dimension fields + */ +public interface PerDimensionValidator { + /** + * Validates the given float is valid for the configuration + * + * @param value to validate + */ + default void validate(float value) {} + + /** + * Validates the given float as a byte is valid for the configuration. + * + * @param value to validate + */ + default void validateByte(float value) {} + + PerDimensionValidator DEFAULT_FLOAT_VALIDATOR = new PerDimensionValidator() { + @Override + public void validate(float value) { + validateFloatVectorValue(value); + } + + @Override + public void validateByte(float value) { + throw new IllegalStateException("DEFAULT_FLOAT_VALIDATOR should only be used for float vectors"); + } + }; + + // Validates if it is a finite number and within the fp16 range of [-65504 to 65504]. + PerDimensionValidator DEFAULT_FP16_VALIDATOR = new PerDimensionValidator() { + @Override + public void validate(float value) { + validateFP16VectorValue(value); + } + + @Override + public void validateByte(float value) { + throw new IllegalStateException("DEFAULT_FP16_VALIDATOR should only be used for float vectors"); + } + }; + + PerDimensionValidator DEFAULT_BYTE_VALIDATOR = new PerDimensionValidator() { + @Override + public void validate(float value) { + throw new IllegalStateException("DEFAULT_BYTE_VALIDATOR should only be used for byte values"); + } + + @Override + public void validateByte(float value) { + validateByteVectorValue(value, VectorDataType.BYTE); + } + }; + + PerDimensionValidator DEFAULT_BIT_VALIDATOR = new PerDimensionValidator() { + @Override + public void validate(float value) { + throw new IllegalStateException("DEFAULT_BIT_VALIDATOR should only be used for byte values"); + } + + @Override + public void validateByte(float value) { + validateByteVectorValue(value, VectorDataType.BINARY); + } + }; +} diff --git a/src/main/java/org/opensearch/knn/index/mapper/SpaceVectorValidator.java b/src/main/java/org/opensearch/knn/index/mapper/SpaceVectorValidator.java new file mode 100644 index 000000000..6ff088604 --- /dev/null +++ b/src/main/java/org/opensearch/knn/index/mapper/SpaceVectorValidator.java @@ -0,0 +1,28 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.mapper; + +import lombok.AllArgsConstructor; +import org.opensearch.knn.index.SpaceType; + +/** + * Confirms that a given vector is valid for the provided space type + */ +@AllArgsConstructor +public class SpaceVectorValidator implements VectorValidator { + + private final SpaceType spaceType; + + @Override + public void validateVector(byte[] vector) { + spaceType.validateVector(vector); + } + + @Override + public void validateVector(float[] vector) { + spaceType.validateVector(vector); + } +} diff --git a/src/main/java/org/opensearch/knn/index/mapper/VectorValidator.java b/src/main/java/org/opensearch/knn/index/mapper/VectorValidator.java new file mode 100644 index 000000000..f4253ae37 --- /dev/null +++ b/src/main/java/org/opensearch/knn/index/mapper/VectorValidator.java @@ -0,0 +1,28 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.mapper; + +/** + * Class validates vector after it has been parsed + */ +public interface VectorValidator { + /** + * Validate if the given byte vector is supported + * + * @param vector the given vector + */ + default void validateVector(byte[] vector) {} + + /** + * Validate if the given float vector is supported + * + * @param vector the given vector + */ + default void validateVector(float[] vector) {} + + VectorValidator NOOP_VECTOR_VALIDATOR = new VectorValidator() { + }; +} diff --git a/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java b/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java index 6d57cb2dd..80833751e 100644 --- a/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java +++ b/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java @@ -24,9 +24,9 @@ import org.opensearch.index.query.QueryRewriteContext; import org.opensearch.index.query.QueryShardContext; import org.opensearch.knn.index.engine.model.QueryContext; +import org.opensearch.knn.index.mapper.KNNMappingConfig; import org.opensearch.knn.index.mapper.KNNVectorFieldType; import org.opensearch.knn.index.util.IndexUtil; -import org.opensearch.knn.index.engine.KNNMethodContext; import org.opensearch.knn.index.engine.MethodComponentContext; import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.VectorDataType; @@ -43,6 +43,7 @@ import java.util.Locale; import java.util.Map; import java.util.Objects; +import java.util.concurrent.atomic.AtomicReference; import static org.opensearch.knn.common.KNNConstants.MAX_DISTANCE; import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER; @@ -341,36 +342,47 @@ protected Query doToQuery(QueryShardContext context) { if (!(mappedFieldType instanceof KNNVectorFieldType)) { throw new IllegalArgumentException(String.format(Locale.ROOT, "Field '%s' is not knn_vector type.", this.fieldName)); } - KNNVectorFieldType knnVectorFieldType = (KNNVectorFieldType) mappedFieldType; - int fieldDimension = knnVectorFieldType.getDimension(); - KNNMethodContext knnMethodContext = knnVectorFieldType.getKnnMethodContext(); - MethodComponentContext methodComponentContext = null; - KNNEngine knnEngine = KNNEngine.DEFAULT; - VectorDataType vectorDataType = knnVectorFieldType.getVectorDataType(); - SpaceType spaceType = knnVectorFieldType.getSpaceType(); + KNNMappingConfig knnMappingConfig = knnVectorFieldType.getKnnMappingConfig(); + final AtomicReference queryConfigFromMapping = new AtomicReference<>(); + int fieldDimension = knnMappingConfig.getDimension(); + knnMappingConfig.getKnnMethodContext() + .ifPresentOrElse( + knnMethodContext -> queryConfigFromMapping.set( + new QueryConfigFromMapping( + knnMethodContext.getKnnEngine(), + knnMethodContext.getMethodComponentContext(), + knnMethodContext.getSpaceType(), + knnVectorFieldType.getVectorDataType() + ) + ), + () -> knnMappingConfig.getModelId().ifPresentOrElse(modelId -> { + ModelMetadata modelMetadata = getModelMetadataForField(modelId); + queryConfigFromMapping.set( + new QueryConfigFromMapping( + modelMetadata.getKnnEngine(), + modelMetadata.getMethodComponentContext(), + modelMetadata.getSpaceType(), + modelMetadata.getVectorDataType() + ) + ); + }, + () -> { + throw new IllegalArgumentException( + String.format(Locale.ROOT, "Field '%s' is not built for ANN search.", this.fieldName) + ); + } + ) + ); + KNNEngine knnEngine = queryConfigFromMapping.get().getKnnEngine(); + MethodComponentContext methodComponentContext = queryConfigFromMapping.get().getMethodComponentContext(); + SpaceType spaceType = queryConfigFromMapping.get().getSpaceType(); + VectorDataType vectorDataType = queryConfigFromMapping.get().getVectorDataType(); + VectorQueryType vectorQueryType = getVectorQueryType(k, maxDistance, minScore); updateQueryStats(vectorQueryType); - if (fieldDimension == -1) { - if (spaceType != null) { - throw new IllegalStateException("Space type should be null when the field uses a model"); - } - // If dimension is not set, the field uses a model and the information needs to be retrieved from there - ModelMetadata modelMetadata = getModelMetadataForField(knnVectorFieldType); - fieldDimension = modelMetadata.getDimension(); - knnEngine = modelMetadata.getKnnEngine(); - spaceType = modelMetadata.getSpaceType(); - methodComponentContext = modelMetadata.getMethodComponentContext(); - vectorDataType = modelMetadata.getVectorDataType(); - - } else if (knnMethodContext != null) { - // If the dimension is set but the knnMethodContext is not then the field is using the legacy mapping - knnEngine = knnMethodContext.getKnnEngine(); - spaceType = knnMethodContext.getSpaceType(); - methodComponentContext = knnMethodContext.getMethodComponentContext(); - } - + // This could be null in the case of when a model did not have serialized methodComponent information final String method = methodComponentContext != null ? methodComponentContext.getName() : null; if (StringUtils.isNotBlank(method)) { final KNNLibrarySearchContext engineSpecificMethodContext = knnEngine.getKNNLibrarySearchContext(method); @@ -492,13 +504,7 @@ protected Query doToQuery(QueryShardContext context) { throw new IllegalArgumentException(String.format(Locale.ROOT, "[%s] requires k or distance or score to be set", NAME)); } - private ModelMetadata getModelMetadataForField(KNNVectorFieldType knnVectorField) { - String modelId = knnVectorField.getModelId(); - - if (modelId == null) { - throw new IllegalArgumentException(String.format(Locale.ROOT, "Field '%s' does not have model.", this.fieldName)); - } - + private ModelMetadata getModelMetadataForField(String modelId) { ModelMetadata modelMetadata = modelDao.getMetadata(modelId); if (!ModelUtil.isModelCreated(modelMetadata)) { throw new IllegalArgumentException(String.format(Locale.ROOT, "Model ID '%s' is not created.", modelId)); @@ -568,4 +574,13 @@ protected QueryBuilder doRewrite(QueryRewriteContext queryShardContext) throws I } return super.doRewrite(queryShardContext); } + + @Getter + @AllArgsConstructor + private static class QueryConfigFromMapping { + private final KNNEngine knnEngine; + private final MethodComponentContext methodComponentContext; + private final SpaceType spaceType; + private final VectorDataType vectorDataType; + } } diff --git a/src/main/java/org/opensearch/knn/plugin/KNNPlugin.java b/src/main/java/org/opensearch/knn/plugin/KNNPlugin.java index e237ac537..6bfcb7eee 100644 --- a/src/main/java/org/opensearch/knn/plugin/KNNPlugin.java +++ b/src/main/java/org/opensearch/knn/plugin/KNNPlugin.java @@ -14,7 +14,6 @@ import org.opensearch.indices.SystemIndexDescriptor; import org.opensearch.knn.index.KNNCircuitBreaker; import org.opensearch.knn.index.util.KNNClusterUtil; -import org.opensearch.knn.index.mapper.KNNVectorFieldMapperUtil; import org.opensearch.knn.index.query.KNNQueryBuilder; import org.opensearch.knn.index.KNNSettings; import org.opensearch.knn.index.mapper.KNNVectorFieldMapper; @@ -206,7 +205,6 @@ public Collection createComponents( TrainingJobClusterStateListener.initialize(threadPool, ModelDao.OpenSearchKNNModelDao.getInstance(), clusterService); KNNCircuitBreaker.getInstance().initialize(threadPool, clusterService, client); KNNQueryBuilder.initialize(ModelDao.OpenSearchKNNModelDao.getInstance()); - KNNVectorFieldMapperUtil.initialize(ModelDao.OpenSearchKNNModelDao.getInstance()); KNNWeight.initialize(ModelDao.OpenSearchKNNModelDao.getInstance()); TrainingModelRequest.initialize(ModelDao.OpenSearchKNNModelDao.getInstance(), clusterService); diff --git a/src/test/java/org/opensearch/knn/KNNTestCase.java b/src/test/java/org/opensearch/knn/KNNTestCase.java index 56c129546..fb09fb30b 100644 --- a/src/test/java/org/opensearch/knn/KNNTestCase.java +++ b/src/test/java/org/opensearch/knn/KNNTestCase.java @@ -7,12 +7,18 @@ import org.mockito.Mock; import org.mockito.MockitoAnnotations; +import org.opensearch.Version; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; import org.opensearch.knn.index.KNNSettings; +import org.opensearch.knn.index.SpaceType; +import org.opensearch.knn.index.engine.KNNEngine; import org.opensearch.knn.index.engine.KNNLibrarySearchContext; +import org.opensearch.knn.index.engine.KNNMethodContext; +import org.opensearch.knn.index.engine.MethodComponentContext; +import org.opensearch.knn.index.mapper.KNNMappingConfig; import org.opensearch.knn.index.memory.NativeMemoryCacheManager; import org.opensearch.knn.plugin.stats.KNNCounter; import org.opensearch.core.common.bytes.BytesReference; @@ -20,12 +26,15 @@ import org.opensearch.common.xcontent.XContentHelper; import org.opensearch.test.OpenSearchTestCase; +import java.util.Collections; import java.util.HashSet; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; import static org.mockito.Mockito.when; +import static org.opensearch.knn.common.KNNConstants.METHOD_HNSW; /** * Base class for integration tests for KNN plugin. Contains several methods for testing KNN ES functionality. @@ -91,4 +100,50 @@ private void initKNNSettings() { public Map xContentBuilderToMap(XContentBuilder xContentBuilder) { return XContentHelper.convertToMap(BytesReference.bytes(xContentBuilder), true, xContentBuilder.contentType()).v2(); } + + public static KNNMethodContext getDefaultKNNMethodContext() { + MethodComponentContext methodComponentContext = new MethodComponentContext(METHOD_HNSW, Collections.emptyMap()); + KNNMethodContext defaultInstance = new KNNMethodContext(KNNEngine.DEFAULT, SpaceType.DEFAULT, methodComponentContext); + methodComponentContext.setIndexVersion(Version.CURRENT); + return defaultInstance; + } + + public static KNNMethodContext getDefaultBinaryKNNMethodContext() { + MethodComponentContext methodComponentContext = new MethodComponentContext(METHOD_HNSW, Collections.emptyMap()); + KNNMethodContext defaultInstance = new KNNMethodContext(KNNEngine.DEFAULT, SpaceType.DEFAULT_BINARY, methodComponentContext); + methodComponentContext.setIndexVersion(Version.CURRENT); + return defaultInstance; + } + + public static KNNMappingConfig getMappingConfigForMethodMapping(KNNMethodContext knnMethodContext, int dimension) { + return new KNNMappingConfig() { + @Override + public Optional getKnnMethodContext() { + return Optional.of(knnMethodContext); + } + + @Override + public int getDimension() { + return dimension; + } + }; + } + + public static KNNMappingConfig getMappingConfigForFlatMapping(int dimension) { + return () -> dimension; + } + + public static KNNMappingConfig getMappingConfigForModelMapping(String modelId, int dimension) { + return new KNNMappingConfig() { + @Override + public Optional getModelId() { + return Optional.of(modelId); + } + + @Override + public int getDimension() { + return dimension; + } + }; + } } diff --git a/src/test/java/org/opensearch/knn/index/KNNMethodContextTests.java b/src/test/java/org/opensearch/knn/index/KNNMethodContextTests.java index 9a867c58d..f71fbaae0 100644 --- a/src/test/java/org/opensearch/knn/index/KNNMethodContextTests.java +++ b/src/test/java/org/opensearch/knn/index/KNNMethodContextTests.java @@ -94,7 +94,7 @@ public void testGetSpaceType() { */ public void testValidate() { // Check valid default - this should not throw any exception - assertNull(KNNMethodContext.getDefault().validate()); + assertNull(getDefaultKNNMethodContext().validate()); // Check a valid nmslib method MethodComponentContext hnswMethod = new MethodComponentContext(METHOD_HNSW, Collections.emptyMap()); diff --git a/src/test/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumerTests.java b/src/test/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumerTests.java index e1ebc5708..e87531561 100644 --- a/src/test/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumerTests.java +++ b/src/test/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumerTests.java @@ -251,62 +251,6 @@ public void testAddKNNBinaryField_fromScratch_nmslibCurrent() throws IOException assertNotEquals(0, (long) KNNGraphValue.MERGE_TOTAL_SIZE_IN_BYTES.getValue()); } - public void testAddKNNBinaryField_fromScratch_nmslibLegacy() throws IOException { - // Set information about the segment and the fields - String segmentName = String.format("test_segment%s", randomAlphaOfLength(4)); - int docsInSegment = 100; - String fieldName = String.format("test_field%s", randomAlphaOfLength(4)); - - KNNEngine knnEngine = KNNEngine.NMSLIB; - SpaceType spaceType = SpaceType.COSINESIMIL; - int dimension = 16; - - SegmentInfo segmentInfo = KNNCodecTestUtil.segmentInfoBuilder() - .directory(directory) - .segmentName(segmentName) - .docsInSegment(docsInSegment) - .codec(codec) - .build(); - - FieldInfo[] fieldInfoArray = new FieldInfo[] { - KNNCodecTestUtil.FieldInfoBuilder.builder(fieldName) - .addAttribute(KNNVectorFieldMapper.KNN_FIELD, "true") - .addAttribute(KNNConstants.HNSW_ALGO_EF_CONSTRUCTION, "512") - .addAttribute(KNNConstants.HNSW_ALGO_M, "16") - .addAttribute(KNNConstants.SPACE_TYPE, spaceType.getValue()) - .build() }; - - FieldInfos fieldInfos = new FieldInfos(fieldInfoArray); - SegmentWriteState state = new SegmentWriteState(null, directory, segmentInfo, fieldInfos, null, IOContext.DEFAULT); - - long initialRefreshOperations = KNNGraphValue.REFRESH_TOTAL_OPERATIONS.getValue(); - long initialMergeOperations = KNNGraphValue.MERGE_TOTAL_OPERATIONS.getValue(); - - // Add documents to the field - KNN80DocValuesConsumer knn80DocValuesConsumer = new KNN80DocValuesConsumer(null, state); - TestVectorValues.RandomVectorDocValuesProducer randomVectorDocValuesProducer = new TestVectorValues.RandomVectorDocValuesProducer( - docsInSegment, - dimension - ); - knn80DocValuesConsumer.addKNNBinaryField(fieldInfoArray[0], randomVectorDocValuesProducer, true, true); - - // The document should be created in the correct location - String expectedFile = KNNCodecUtil.buildEngineFileName(segmentName, knnEngine.getVersion(), fieldName, knnEngine.getExtension()); - assertFileInCorrectLocation(state, expectedFile); - - // The footer should be valid - assertValidFooter(state.directory, expectedFile); - - // The document should be readable by nmslib - assertLoadableByEngine(null, state, expectedFile, knnEngine, spaceType, dimension); - - // The graph creation statistics should be updated - assertEquals(1 + initialRefreshOperations, (long) KNNGraphValue.REFRESH_TOTAL_OPERATIONS.getValue()); - assertEquals(1 + initialMergeOperations, (long) KNNGraphValue.MERGE_TOTAL_OPERATIONS.getValue()); - assertNotEquals(0, (long) KNNGraphValue.MERGE_TOTAL_DOCS.getValue()); - assertNotEquals(0, (long) KNNGraphValue.MERGE_TOTAL_SIZE_IN_BYTES.getValue()); - } - public void testAddKNNBinaryField_fromScratch_faissCurrent() throws IOException { String segmentName = String.format("test_segment%s", randomAlphaOfLength(4)); int docsInSegment = 100; diff --git a/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestCase.java b/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestCase.java index a0b9b32d0..00cc2b167 100644 --- a/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestCase.java +++ b/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestCase.java @@ -14,8 +14,10 @@ import org.apache.lucene.search.Query; import org.apache.lucene.search.TopDocs; import org.apache.lucene.search.join.BitSetProducer; +import org.opensearch.Version; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Setting; +import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.index.mapper.MapperService; import org.opensearch.knn.KNNTestCase; import org.opensearch.knn.common.KNNConstants; @@ -56,6 +58,7 @@ import java.time.ZoneOffset; import java.time.ZonedDateTime; import java.util.Arrays; +import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -77,6 +80,8 @@ import static org.opensearch.knn.common.KNNConstants.HNSW_ALGO_M; import static org.opensearch.knn.common.KNNConstants.INDEX_DESCRIPTION_PARAMETER; import static org.opensearch.knn.common.KNNConstants.METHOD_HNSW; +import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_EF_CONSTRUCTION; +import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_M; import static org.opensearch.knn.common.KNNConstants.SPACE_TYPE; import static org.opensearch.knn.index.KNNSettings.MODEL_CACHE_SIZE_LIMIT_SETTING; @@ -86,14 +91,28 @@ public class KNNCodecTestCase extends KNNTestCase { private static final Codec ACTUAL_CODEC = KNNCodecVersion.current().getDefaultKnnCodecSupplier().get(); - private static FieldType sampleFieldType; + private static final FieldType sampleFieldType; static { + KNNMethodContext knnMethodContext = new KNNMethodContext( + KNNEngine.DEFAULT, + SpaceType.DEFAULT, + new MethodComponentContext(METHOD_HNSW, ImmutableMap.of(METHOD_PARAMETER_M, 16, METHOD_PARAMETER_EF_CONSTRUCTION, 512)) + ); + knnMethodContext.getMethodComponentContext().setIndexVersion(Version.CURRENT); + String parameterString; + try { + parameterString = XContentFactory.jsonBuilder() + .map(knnMethodContext.getKnnEngine().getKNNLibraryIndexingContext(knnMethodContext).getLibraryParameters()) + .toString(); + } catch (IOException e) { + throw new RuntimeException(e); + } + sampleFieldType = new FieldType(KNNVectorFieldMapper.Defaults.FIELD_TYPE); - sampleFieldType.putAttribute(KNNConstants.KNN_METHOD, KNNConstants.METHOD_HNSW); - sampleFieldType.putAttribute(KNNConstants.KNN_ENGINE, KNNEngine.NMSLIB.getName()); - sampleFieldType.putAttribute(KNNConstants.SPACE_TYPE, SpaceType.L2.getValue()); - sampleFieldType.putAttribute(KNNConstants.HNSW_ALGO_M, "32"); - sampleFieldType.putAttribute(KNNConstants.HNSW_ALGO_EF_CONSTRUCTION, "512"); + sampleFieldType.putAttribute(KNNVectorFieldMapper.KNN_FIELD, "true"); + sampleFieldType.putAttribute(KNNConstants.KNN_ENGINE, knnMethodContext.getKnnEngine().getName()); + sampleFieldType.putAttribute(KNNConstants.SPACE_TYPE, knnMethodContext.getSpaceType().getValue()); + sampleFieldType.putAttribute(KNNConstants.PARAMETERS, parameterString); sampleFieldType.freeze(); } private static final String FIELD_NAME_ONE = "test_vector_one"; @@ -309,8 +328,19 @@ public void testKnnVectorIndex( SpaceType.L2, new MethodComponentContext(METHOD_HNSW, Map.of(HNSW_ALGO_M, 16, HNSW_ALGO_EF_CONSTRUCTION, 256)) ); - final KNNVectorFieldType mappedFieldType1 = new KNNVectorFieldType(FIELD_NAME_ONE, Map.of(), 3, knnMethodContext); - final KNNVectorFieldType mappedFieldType2 = new KNNVectorFieldType(FIELD_NAME_TWO, Map.of(), 2, knnMethodContext); + + final KNNVectorFieldType mappedFieldType1 = new KNNVectorFieldType( + "test", + Collections.emptyMap(), + VectorDataType.FLOAT, + getMappingConfigForMethodMapping(knnMethodContext, 3) + ); + final KNNVectorFieldType mappedFieldType2 = new KNNVectorFieldType( + "test", + Collections.emptyMap(), + VectorDataType.FLOAT, + getMappingConfigForMethodMapping(knnMethodContext, 2) + ); when(mapperService.fieldType(eq(FIELD_NAME_ONE))).thenReturn(mappedFieldType1); when(mapperService.fieldType(eq(FIELD_NAME_TWO))).thenReturn(mappedFieldType2); diff --git a/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java b/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java index c95568be2..f06ff7935 100644 --- a/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java +++ b/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java @@ -103,7 +103,7 @@ public class KNNVectorFieldMapperTests extends KNNTestCase { public void testBuilder_getParameters() { String fieldName = "test-field-name"; ModelDao modelDao = mock(ModelDao.class); - KNNVectorFieldMapper.Builder builder = new KNNVectorFieldMapper.Builder(fieldName, modelDao, CURRENT); + KNNVectorFieldMapper.Builder builder = new KNNVectorFieldMapper.Builder(fieldName, modelDao, CURRENT, null); assertEquals(7, builder.getParameters().size()); List actualParams = builder.getParameters().stream().map(a -> a.name).collect(Collectors.toList()); @@ -114,7 +114,7 @@ public void testBuilder_getParameters() { public void testBuilder_build_fromKnnMethodContext() { // Check that knnMethodContext takes precedent over both model and legacy ModelDao modelDao = mock(ModelDao.class); - KNNVectorFieldMapper.Builder builder = new KNNVectorFieldMapper.Builder("test-field-name-1", modelDao, CURRENT); + KNNVectorFieldMapper.Builder builder = new KNNVectorFieldMapper.Builder("test-field-name-1", modelDao, CURRENT, null); SpaceType spaceType = SpaceType.COSINESIMIL; int m = 17; @@ -126,6 +126,7 @@ public void testBuilder_build_fromKnnMethodContext() { .put(KNNSettings.KNN_SPACE_TYPE, spaceType.getValue()) .put(KNNSettings.KNN_ALGO_PARAM_M, m) .put(KNNSettings.KNN_ALGO_PARAM_EF_CONSTRUCTION, efConstruction) + .put(KNN_INDEX, true) .build(); builder.knnMethodContext.setValue( @@ -139,19 +140,17 @@ public void testBuilder_build_fromKnnMethodContext() { ) ); - builder.modelId.setValue("Random modelId"); - Mapper.BuilderContext builderContext = new Mapper.BuilderContext(settings, new ContentPath()); KNNVectorFieldMapper knnVectorFieldMapper = builder.build(builderContext); assertTrue(knnVectorFieldMapper instanceof MethodFieldMapper); - assertNotNull(knnVectorFieldMapper.knnMethod); - assertNull(knnVectorFieldMapper.modelId); + assertTrue(knnVectorFieldMapper.fieldType().getKnnMappingConfig().getKnnMethodContext().isPresent()); + assertTrue(knnVectorFieldMapper.fieldType().getKnnMappingConfig().getModelId().isEmpty()); } public void testBuilder_build_fromModel() { // Check that modelContext takes precedent over legacy ModelDao modelDao = mock(ModelDao.class); - KNNVectorFieldMapper.Builder builder = new KNNVectorFieldMapper.Builder("test-field-name-1", modelDao, CURRENT); + KNNVectorFieldMapper.Builder builder = new KNNVectorFieldMapper.Builder("test-field-name-1", modelDao, CURRENT, null); SpaceType spaceType = SpaceType.COSINESIMIL; int m = 17; @@ -163,6 +162,7 @@ public void testBuilder_build_fromModel() { .put(KNNSettings.KNN_SPACE_TYPE, spaceType.getValue()) .put(KNNSettings.KNN_ALGO_PARAM_M, m) .put(KNNSettings.KNN_ALGO_PARAM_EF_CONSTRUCTION, efConstruction) + .put(KNN_INDEX, true) .build(); String modelId = "Random modelId"; @@ -184,14 +184,14 @@ public void testBuilder_build_fromModel() { when(modelDao.getMetadata(modelId)).thenReturn(mockedModelMetadata); KNNVectorFieldMapper knnVectorFieldMapper = builder.build(builderContext); assertTrue(knnVectorFieldMapper instanceof ModelFieldMapper); - assertNotNull(knnVectorFieldMapper.modelId); - assertNull(knnVectorFieldMapper.knnMethod); + assertTrue(knnVectorFieldMapper.fieldType().getKnnMappingConfig().getModelId().isPresent()); + assertTrue(knnVectorFieldMapper.fieldType().getKnnMappingConfig().getKnnMethodContext().isEmpty()); } public void testBuilder_build_fromLegacy() { // Check legacy is picked up if model context and method context are not set ModelDao modelDao = mock(ModelDao.class); - KNNVectorFieldMapper.Builder builder = new KNNVectorFieldMapper.Builder("test-field-name-1", modelDao, CURRENT); + KNNVectorFieldMapper.Builder builder = new KNNVectorFieldMapper.Builder("test-field-name-1", modelDao, CURRENT, null); int m = 17; int efConstruction = 17; @@ -201,37 +201,22 @@ public void testBuilder_build_fromLegacy() { .put(settings(CURRENT).build()) .put(KNNSettings.KNN_ALGO_PARAM_M, m) .put(KNNSettings.KNN_ALGO_PARAM_EF_CONSTRUCTION, efConstruction) + .put(KNN_INDEX, true) .build(); Mapper.BuilderContext builderContext = new Mapper.BuilderContext(settings, new ContentPath()); KNNVectorFieldMapper knnVectorFieldMapper = builder.build(builderContext); - assertTrue(knnVectorFieldMapper instanceof LegacyFieldMapper); - assertNull(knnVectorFieldMapper.modelId); - assertNull(knnVectorFieldMapper.knnMethod); - assertEquals(SpaceType.L2.getValue(), ((LegacyFieldMapper) knnVectorFieldMapper).spaceType); - } - - public void testBuilder_whenKnnFalseWithBinary_thenSetHammingAsDefault() { - // Check legacy is picked up if model context and method context are not set - ModelDao modelDao = mock(ModelDao.class); - KNNVectorFieldMapper.Builder builder = new KNNVectorFieldMapper.Builder("test-field-name-1", modelDao, CURRENT); - builder.vectorDataType.setValue(VectorDataType.BINARY); - builder.dimension.setValue(8); - - // Setup settings - Settings settings = Settings.builder().put(settings(CURRENT).build()).build(); - - Mapper.BuilderContext builderContext = new Mapper.BuilderContext(settings, new ContentPath()); - KNNVectorFieldMapper knnVectorFieldMapper = builder.build(builderContext); - assertTrue(knnVectorFieldMapper instanceof LegacyFieldMapper); - assertEquals(SpaceType.HAMMING.getValue(), ((LegacyFieldMapper) knnVectorFieldMapper).spaceType); + assertTrue(knnVectorFieldMapper instanceof MethodFieldMapper); + assertTrue(knnVectorFieldMapper.fieldType().getKnnMappingConfig().getKnnMethodContext().isPresent()); + assertTrue(knnVectorFieldMapper.fieldType().getKnnMappingConfig().getModelId().isEmpty()); + assertEquals(SpaceType.L2, knnVectorFieldMapper.fieldType().getKnnMappingConfig().getKnnMethodContext().get().getSpaceType()); } public void testBuilder_parse_fromKnnMethodContext_luceneEngine() throws IOException { String fieldName = "test-field-name"; String indexName = "test-index-name"; - Settings settings = Settings.builder().put(settings(CURRENT).build()).build(); + Settings settings = Settings.builder().put(settings(CURRENT).build()).put(KNN_INDEX, true).build(); ModelDao modelDao = mock(ModelDao.class); KNNVectorFieldMapper.TypeParser typeParser = new KNNVectorFieldMapper.TypeParser(() -> modelDao); @@ -317,7 +302,7 @@ public void testTypeParser_parse_fromKnnMethodContext_invalidDimension() throws String fieldName = "test-field-name"; String indexName = "test-index-name"; - Settings settings = Settings.builder().put(settings(CURRENT).build()).build(); + Settings settings = Settings.builder().put(settings(CURRENT).put(KNN_INDEX, true).build()).build(); ModelDao modelDao = mock(ModelDao.class); KNNVectorFieldMapper.TypeParser typeParser = new KNNVectorFieldMapper.TypeParser(() -> modelDao); @@ -616,7 +601,7 @@ public void testKNNVectorFieldMapper_merge_fromKnnMethodContext() throws IOExcep String fieldName = "test-field-name"; String indexName = "test-index-name"; - Settings settings = Settings.builder().put(settings(CURRENT).build()).build(); + Settings settings = Settings.builder().put(settings(CURRENT).build()).put(KNN_INDEX, true).build(); ModelDao modelDao = mock(ModelDao.class); KNNVectorFieldMapper.TypeParser typeParser = new KNNVectorFieldMapper.TypeParser(() -> modelDao); @@ -646,12 +631,18 @@ public void testKNNVectorFieldMapper_merge_fromKnnMethodContext() throws IOExcep // merge with itself - should be successful KNNVectorFieldMapper knnVectorFieldMapperMerge1 = (KNNVectorFieldMapper) knnVectorFieldMapper1.merge(knnVectorFieldMapper1); - assertEquals(knnVectorFieldMapper1.knnMethod, knnVectorFieldMapperMerge1.knnMethod); + assertEquals( + knnVectorFieldMapper1.fieldType().getKnnMappingConfig().getKnnMethodContext().get(), + knnVectorFieldMapperMerge1.fieldType().getKnnMappingConfig().getKnnMethodContext().get() + ); // merge with another mapper of the same field with same context KNNVectorFieldMapper knnVectorFieldMapper2 = builder.build(builderContext); KNNVectorFieldMapper knnVectorFieldMapperMerge2 = (KNNVectorFieldMapper) knnVectorFieldMapper1.merge(knnVectorFieldMapper2); - assertEquals(knnVectorFieldMapper1.knnMethod, knnVectorFieldMapperMerge2.knnMethod); + assertEquals( + knnVectorFieldMapper1.fieldType().getKnnMappingConfig().getKnnMethodContext().get(), + knnVectorFieldMapperMerge2.fieldType().getKnnMappingConfig().getKnnMethodContext().get() + ); // merge with another mapper of the same field with different context xContentBuilder = XContentFactory.jsonBuilder() @@ -676,7 +667,7 @@ public void testKNNVectorFieldMapper_merge_fromModel() throws IOException { String fieldName = "test-field-name"; String indexName = "test-index-name"; - Settings settings = Settings.builder().put(settings(CURRENT).build()).build(); + Settings settings = Settings.builder().put(settings(CURRENT).build()).put(KNN_INDEX, true).build(); String modelId = "test-id"; int dimension = 133; @@ -715,12 +706,18 @@ public void testKNNVectorFieldMapper_merge_fromModel() throws IOException { // merge with itself - should be successful KNNVectorFieldMapper knnVectorFieldMapperMerge1 = (KNNVectorFieldMapper) knnVectorFieldMapper1.merge(knnVectorFieldMapper1); - assertEquals(knnVectorFieldMapper1.modelId, knnVectorFieldMapperMerge1.modelId); + assertEquals( + knnVectorFieldMapper1.fieldType().getKnnMappingConfig().getModelId().get(), + knnVectorFieldMapperMerge1.fieldType().getKnnMappingConfig().getModelId().get() + ); // merge with another mapper of the same field with same context KNNVectorFieldMapper knnVectorFieldMapper2 = builder.build(builderContext); KNNVectorFieldMapper knnVectorFieldMapperMerge2 = (KNNVectorFieldMapper) knnVectorFieldMapper1.merge(knnVectorFieldMapper2); - assertEquals(knnVectorFieldMapper1.modelId, knnVectorFieldMapperMerge2.modelId); + assertEquals( + knnVectorFieldMapper1.fieldType().getKnnMappingConfig().getModelId().get(), + knnVectorFieldMapperMerge2.fieldType().getKnnMappingConfig().getModelId().get() + ); // merge with another mapper of the same field with different context xContentBuilder = XContentFactory.jsonBuilder() @@ -754,18 +751,19 @@ public void testLuceneFieldMapper_parseCreateField_docValues_withFloats() { when(parseContext.doc()).thenReturn(document); when(parseContext.path()).thenReturn(contentPath); - LuceneFieldMapper luceneFieldMapper = Mockito.spy(new LuceneFieldMapper(inputBuilder.build())); - doReturn(Optional.of(TEST_VECTOR)).when(luceneFieldMapper) - .getFloatsFromContext(parseContext, TEST_DIMENSION, new MethodComponentContext(METHOD_HNSW, Collections.emptyMap())); - doNothing().when(luceneFieldMapper).validateIfCircuitBreakerIsNotTriggered(); - doNothing().when(luceneFieldMapper).validateIfKNNPluginEnabled(); - luceneFieldMapper.parseCreateField( - parseContext, - TEST_DIMENSION, - luceneFieldMapper.fieldType().spaceType, - luceneFieldMapper.fieldType().knnMethodContext.getMethodComponentContext(), - VectorDataType.FLOAT + LuceneFieldMapper luceneFieldMapper = Mockito.spy( + LuceneFieldMapper.createFieldMapper( + TEST_FIELD_NAME, + Collections.emptyMap(), + VectorDataType.FLOAT, + TEST_DIMENSION, + getDefaultKNNMethodContext(), + inputBuilder.build() + ) ); + doReturn(Optional.of(TEST_VECTOR)).when(luceneFieldMapper).getFloatsFromContext(parseContext, TEST_DIMENSION); + doNothing().when(luceneFieldMapper).validatePreparse(); + luceneFieldMapper.parseCreateField(parseContext, TEST_DIMENSION, VectorDataType.FLOAT); // Document should have 2 fields: one for VectorField (binary doc values) and one for KnnVectorField List fields = document.getFields(); @@ -798,19 +796,26 @@ public void testLuceneFieldMapper_parseCreateField_docValues_withFloats() { when(parseContext.path()).thenReturn(contentPath); inputBuilder.hasDocValues(false); - luceneFieldMapper = Mockito.spy(new LuceneFieldMapper(inputBuilder.build())); - doReturn(Optional.of(TEST_VECTOR)).when(luceneFieldMapper) - .getFloatsFromContext(parseContext, TEST_DIMENSION, new MethodComponentContext(METHOD_HNSW, Collections.emptyMap())); - doNothing().when(luceneFieldMapper).validateIfCircuitBreakerIsNotTriggered(); - doNothing().when(luceneFieldMapper).validateIfKNNPluginEnabled(); - - luceneFieldMapper.parseCreateField( - parseContext, - TEST_DIMENSION, - luceneFieldMapper.fieldType().spaceType, - luceneFieldMapper.fieldType().knnMethodContext.getMethodComponentContext(), - VectorDataType.FLOAT + + KNNMethodContext knnMethodContext = new KNNMethodContext( + KNNEngine.LUCENE, + SpaceType.DEFAULT, + new MethodComponentContext(METHOD_HNSW, Collections.emptyMap()) ); + luceneFieldMapper = Mockito.spy( + LuceneFieldMapper.createFieldMapper( + TEST_FIELD_NAME, + Collections.emptyMap(), + VectorDataType.FLOAT, + TEST_DIMENSION, + knnMethodContext, + inputBuilder.build() + ) + ); + doReturn(Optional.of(TEST_VECTOR)).when(luceneFieldMapper).getFloatsFromContext(parseContext, TEST_DIMENSION); + doNothing().when(luceneFieldMapper).validatePreparse(); + + luceneFieldMapper.parseCreateField(parseContext, TEST_DIMENSION, VectorDataType.FLOAT); // Document should have 1 field: one for KnnVectorField fields = document.getFields(); @@ -834,19 +839,21 @@ public void testLuceneFieldMapper_parseCreateField_docValues_withBytes() { when(parseContext.doc()).thenReturn(document); when(parseContext.path()).thenReturn(contentPath); - LuceneFieldMapper luceneFieldMapper = Mockito.spy(new LuceneFieldMapper(inputBuilder.build())); + LuceneFieldMapper luceneFieldMapper = Mockito.spy( + LuceneFieldMapper.createFieldMapper( + TEST_FIELD_NAME, + Collections.emptyMap(), + VectorDataType.BYTE, + TEST_DIMENSION, + getDefaultKNNMethodContext(), + inputBuilder.build() + ) + ); doReturn(Optional.of(TEST_BYTE_VECTOR)).when(luceneFieldMapper) .getBytesFromContext(parseContext, TEST_DIMENSION, VectorDataType.BYTE); - doNothing().when(luceneFieldMapper).validateIfCircuitBreakerIsNotTriggered(); - doNothing().when(luceneFieldMapper).validateIfKNNPluginEnabled(); - - luceneFieldMapper.parseCreateField( - parseContext, - TEST_DIMENSION, - luceneFieldMapper.fieldType().spaceType, - luceneFieldMapper.fieldType().knnMethodContext.getMethodComponentContext(), - VectorDataType.BYTE - ); + doNothing().when(luceneFieldMapper).validatePreparse(); + + luceneFieldMapper.parseCreateField(parseContext, TEST_DIMENSION, VectorDataType.BYTE); // Document should have 2 fields: one for VectorField (binary doc values) and one for KnnByteVectorField List fields = document.getFields(); @@ -878,19 +885,21 @@ public void testLuceneFieldMapper_parseCreateField_docValues_withBytes() { when(parseContext.path()).thenReturn(contentPath); inputBuilder.hasDocValues(false); - luceneFieldMapper = Mockito.spy(new LuceneFieldMapper(inputBuilder.build())); + luceneFieldMapper = Mockito.spy( + LuceneFieldMapper.createFieldMapper( + TEST_FIELD_NAME, + Collections.emptyMap(), + VectorDataType.BYTE, + TEST_DIMENSION, + getDefaultKNNMethodContext(), + inputBuilder.build() + ) + ); doReturn(Optional.of(TEST_BYTE_VECTOR)).when(luceneFieldMapper) .getBytesFromContext(parseContext, TEST_DIMENSION, VectorDataType.BYTE); - doNothing().when(luceneFieldMapper).validateIfCircuitBreakerIsNotTriggered(); - doNothing().when(luceneFieldMapper).validateIfKNNPluginEnabled(); - - luceneFieldMapper.parseCreateField( - parseContext, - TEST_DIMENSION, - luceneFieldMapper.fieldType().spaceType, - luceneFieldMapper.fieldType().knnMethodContext.getMethodComponentContext(), - VectorDataType.BYTE - ); + doNothing().when(luceneFieldMapper).validatePreparse(); + + luceneFieldMapper.parseCreateField(parseContext, TEST_DIMENSION, VectorDataType.BYTE); // Document should have 1 field: one for KnnByteVectorField fields = document.getFields(); @@ -970,10 +979,10 @@ private void testBuilderWithBinaryDataType( String expectedErrMsg ) { ModelDao modelDao = mock(ModelDao.class); - KNNVectorFieldMapper.Builder builder = new KNNVectorFieldMapper.Builder("test-field-name-1", modelDao, CURRENT); + KNNVectorFieldMapper.Builder builder = new KNNVectorFieldMapper.Builder("test-field-name-1", modelDao, CURRENT, null); // Setup settings - Settings settings = Settings.builder().put(settings(CURRENT).build()).build(); + Settings settings = Settings.builder().put(settings(CURRENT).build()).put(KNN_INDEX, true).build(); builder.knnMethodContext.setValue( new KNNMethodContext(knnEngine, spaceType, new MethodComponentContext(method, Collections.emptyMap())) @@ -986,7 +995,10 @@ private void testBuilderWithBinaryDataType( KNNVectorFieldMapper knnVectorFieldMapper = builder.build(builderContext); assertTrue(knnVectorFieldMapper instanceof MethodFieldMapper); if (SpaceType.UNDEFINED == spaceType) { - assertEquals(SpaceType.HAMMING, knnVectorFieldMapper.fieldType().spaceType); + assertEquals( + SpaceType.HAMMING, + knnVectorFieldMapper.fieldType().getKnnMappingConfig().getKnnMethodContext().get().getSpaceType() + ); } } else { Exception ex = expectThrows(Exception.class, () -> builder.build(builderContext)); @@ -996,10 +1008,10 @@ private void testBuilderWithBinaryDataType( public void testBuilder_whenBinaryFaissHNSWWithSQ_thenException() { ModelDao modelDao = mock(ModelDao.class); - KNNVectorFieldMapper.Builder builder = new KNNVectorFieldMapper.Builder("test-field-name-1", modelDao, CURRENT); + KNNVectorFieldMapper.Builder builder = new KNNVectorFieldMapper.Builder("test-field-name-1", modelDao, CURRENT, null); // Setup settings - Settings settings = Settings.builder().put(settings(CURRENT).build()).build(); + Settings settings = Settings.builder().put(settings(CURRENT).build()).put(KNN_INDEX, true).build(); builder.knnMethodContext.setValue( new KNNMethodContext( @@ -1022,7 +1034,7 @@ public void testBuilder_whenBinaryFaissHNSWWithSQ_thenException() { public void testBuilder_whenBinaryWithLegacyKNNDisabled_thenValid() { // Check legacy is picked up if model context and method context are not set ModelDao modelDao = mock(ModelDao.class); - KNNVectorFieldMapper.Builder builder = new KNNVectorFieldMapper.Builder("test-field-name-1", modelDao, CURRENT); + KNNVectorFieldMapper.Builder builder = new KNNVectorFieldMapper.Builder("test-field-name-1", modelDao, CURRENT, null); builder.vectorDataType.setValue(VectorDataType.BINARY); builder.dimension.setValue(8); @@ -1031,13 +1043,13 @@ public void testBuilder_whenBinaryWithLegacyKNNDisabled_thenValid() { Mapper.BuilderContext builderContext = new Mapper.BuilderContext(settings, new ContentPath()); KNNVectorFieldMapper knnVectorFieldMapper = builder.build(builderContext); - assertTrue(knnVectorFieldMapper instanceof LegacyFieldMapper); + assertTrue(knnVectorFieldMapper instanceof FlatVectorFieldMapper); } public void testBuilder_whenBinaryWithLegacyKNNEnabled_thenException() { // Check legacy is picked up if model context and method context are not set ModelDao modelDao = mock(ModelDao.class); - KNNVectorFieldMapper.Builder builder = new KNNVectorFieldMapper.Builder("test-field-name-1", modelDao, CURRENT); + KNNVectorFieldMapper.Builder builder = new KNNVectorFieldMapper.Builder("test-field-name-1", modelDao, CURRENT, null); builder.vectorDataType.setValue(VectorDataType.BINARY); builder.dimension.setValue(8); @@ -1052,29 +1064,14 @@ public void testBuilder_whenBinaryWithLegacyKNNEnabled_thenException() { private LuceneFieldMapper.CreateLuceneFieldMapperInput.CreateLuceneFieldMapperInputBuilder createLuceneFieldMapperInputBuilder( VectorDataType vectorDataType ) { - KNNMethodContext knnMethodContext = new KNNMethodContext( - KNNEngine.LUCENE, - SpaceType.DEFAULT, - new MethodComponentContext(METHOD_HNSW, Collections.emptyMap()) - ); - - KNNVectorFieldType knnVectorFieldType = new KNNVectorFieldType( - TEST_FIELD_NAME, - Collections.emptyMap(), - TEST_DIMENSION, - knnMethodContext, - vectorDataType - ); - return LuceneFieldMapper.CreateLuceneFieldMapperInput.builder() .name(TEST_FIELD_NAME) - .mappedFieldType(knnVectorFieldType) .multiFields(FieldMapper.MultiFields.empty()) .copyTo(FieldMapper.CopyTo.empty()) .hasDocValues(true) .vectorDataType(vectorDataType) .ignoreMalformed(new Explicit<>(true, true)) - .knnMethodContext(knnMethodContext); + .originalKnnMethodContext(getDefaultKNNMethodContext()); } private static float[] createInitializedFloatArray(int dimension, float value) { diff --git a/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperUtilTests.java b/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperUtilTests.java index 31da12d66..8ace5557e 100644 --- a/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperUtilTests.java +++ b/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperUtilTests.java @@ -21,9 +21,6 @@ import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.codec.util.KNNVectorSerializerFactory; import org.opensearch.knn.index.engine.KNNEngine; -import org.opensearch.knn.indices.ModelDao; -import org.opensearch.knn.indices.ModelMetadata; -import org.opensearch.knn.indices.ModelState; import java.util.Arrays; import java.util.Collections; @@ -61,64 +58,24 @@ public void testStoredFields_whenVectorIsFloatType_thenSucceed() { public void testGetExpectedVectorLengthSuccess() { KNNVectorFieldType knnVectorFieldType = mock(KNNVectorFieldType.class); - when(knnVectorFieldType.getDimension()).thenReturn(3); - + when(knnVectorFieldType.getKnnMappingConfig()).thenReturn(getMappingConfigForMethodMapping(getDefaultKNNMethodContext(), 3)); KNNVectorFieldType knnVectorFieldTypeBinary = mock(KNNVectorFieldType.class); - when(knnVectorFieldTypeBinary.getDimension()).thenReturn(8); + when(knnVectorFieldTypeBinary.getKnnMappingConfig()).thenReturn( + getMappingConfigForMethodMapping(getDefaultBinaryKNNMethodContext(), 8) + ); when(knnVectorFieldTypeBinary.getVectorDataType()).thenReturn(VectorDataType.BINARY); KNNVectorFieldType knnVectorFieldTypeModelBased = mock(KNNVectorFieldType.class); - when(knnVectorFieldTypeModelBased.getDimension()).thenReturn(-1); + when(knnVectorFieldTypeModelBased.getKnnMappingConfig()).thenReturn( + getMappingConfigForMethodMapping(getDefaultBinaryKNNMethodContext(), 8) + ); String modelId = "test-model"; - when(knnVectorFieldTypeModelBased.getModelId()).thenReturn(modelId); - - ModelDao modelDao = mock(ModelDao.class); - ModelMetadata modelMetadata = mock(ModelMetadata.class); - when(modelMetadata.getState()).thenReturn(ModelState.CREATED); - when(modelMetadata.getDimension()).thenReturn(4); - when(modelDao.getMetadata(modelId)).thenReturn(modelMetadata); - - KNNVectorFieldMapperUtil.initialize(modelDao); - + when(knnVectorFieldTypeModelBased.getKnnMappingConfig()).thenReturn(getMappingConfigForModelMapping(modelId, 4)); assertEquals(3, KNNVectorFieldMapperUtil.getExpectedVectorLength(knnVectorFieldType)); assertEquals(1, KNNVectorFieldMapperUtil.getExpectedVectorLength(knnVectorFieldTypeBinary)); assertEquals(4, KNNVectorFieldMapperUtil.getExpectedVectorLength(knnVectorFieldTypeModelBased)); } - public void testGetExpectedVectorLengthFailure() { - KNNVectorFieldType knnVectorFieldTypeModelBased = mock(KNNVectorFieldType.class); - when(knnVectorFieldTypeModelBased.getDimension()).thenReturn(-1); - String modelId = "test-model"; - when(knnVectorFieldTypeModelBased.getModelId()).thenReturn(modelId); - - ModelDao modelDao = mock(ModelDao.class); - ModelMetadata modelMetadata = mock(ModelMetadata.class); - when(modelMetadata.getState()).thenReturn(ModelState.TRAINING); - when(modelDao.getMetadata(modelId)).thenReturn(modelMetadata); - - KNNVectorFieldMapperUtil.initialize(modelDao); - - IllegalArgumentException e = expectThrows( - IllegalArgumentException.class, - () -> KNNVectorFieldMapperUtil.getExpectedVectorLength(knnVectorFieldTypeModelBased) - ); - assertEquals(String.format("Model ID '%s' is not created.", modelId), e.getMessage()); - - when(knnVectorFieldTypeModelBased.getModelId()).thenReturn(null); - KNNMethodContext knnMethodContext = mock(KNNMethodContext.class); - MethodComponentContext methodComponentContext = mock(MethodComponentContext.class); - String fieldName = "test-field"; - when(methodComponentContext.getName()).thenReturn(fieldName); - when(knnMethodContext.getMethodComponentContext()).thenReturn(methodComponentContext); - when(knnVectorFieldTypeModelBased.getKnnMethodContext()).thenReturn(knnMethodContext); - - e = expectThrows( - IllegalArgumentException.class, - () -> KNNVectorFieldMapperUtil.getExpectedVectorLength(knnVectorFieldTypeModelBased) - ); - assertEquals(String.format("Field '%s' does not have model.", fieldName), e.getMessage()); - } - public void testValidateVectorDataType_whenBinaryFaissHNSW_thenValid() { validateValidateVectorDataType(KNNEngine.FAISS, KNNConstants.METHOD_HNSW, VectorDataType.BINARY, null); } diff --git a/src/test/java/org/opensearch/knn/index/mapper/MethodFieldMapperTests.java b/src/test/java/org/opensearch/knn/index/mapper/MethodFieldMapperTests.java index dcd255740..faae3e35d 100644 --- a/src/test/java/org/opensearch/knn/index/mapper/MethodFieldMapperTests.java +++ b/src/test/java/org/opensearch/knn/index/mapper/MethodFieldMapperTests.java @@ -5,33 +5,35 @@ package org.opensearch.knn.index.mapper; -import junit.framework.TestCase; +import org.opensearch.Version; import org.opensearch.index.mapper.FieldMapper; -import org.opensearch.knn.index.engine.KNNMethodContext; -import org.opensearch.knn.index.SpaceType; +import org.opensearch.knn.KNNTestCase; import org.opensearch.knn.index.VectorDataType; +import org.opensearch.knn.index.engine.KNNMethodContext; import java.util.Collections; -public class MethodFieldMapperTests extends TestCase { - public void testMethodFieldMapper_whenVectorDataTypeIsGiven_thenSetItInFieldType() { - KNNVectorFieldType mappedFieldType = new KNNVectorFieldType( - "testField", - Collections.emptyMap(), - 1, - VectorDataType.BINARY, - SpaceType.HAMMING - ); - MethodFieldMapper mappers = new MethodFieldMapper( - "simpleName", - mappedFieldType, - null, - new FieldMapper.CopyTo.Builder().build(), - KNNVectorFieldMapper.Defaults.IGNORE_MALFORMED, - true, - true, - KNNMethodContext.getDefault() +public class MethodFieldMapperTests extends KNNTestCase { + public void testMethodFieldMapper_whenVectorDataTypeAndContextMismatch_thenThrow() { + // Expect that we cannot create the mapper with an invalid field type + KNNMethodContext knnMethodContext = getDefaultKNNMethodContext(); + expectThrows( + IllegalArgumentException.class, + () -> MethodFieldMapper.createFieldMapper( + "testField", + "simpleName", + Collections.emptyMap(), + VectorDataType.BINARY, + 1, + knnMethodContext, + knnMethodContext, + null, + new FieldMapper.CopyTo.Builder().build(), + KNNVectorFieldMapper.Defaults.IGNORE_MALFORMED, + true, + true, + Version.CURRENT + ) ); - assertEquals(VectorDataType.BINARY, mappers.fieldType().vectorDataType); } } diff --git a/src/test/java/org/opensearch/knn/index/query/KNNQueryBuilderTests.java b/src/test/java/org/opensearch/knn/index/query/KNNQueryBuilderTests.java index 0b918bd9e..25982fb7d 100644 --- a/src/test/java/org/opensearch/knn/index/query/KNNQueryBuilderTests.java +++ b/src/test/java/org/opensearch/knn/index/query/KNNQueryBuilderTests.java @@ -185,9 +185,8 @@ public void testDoToQuery_Normal() throws Exception { QueryShardContext mockQueryShardContext = mock(QueryShardContext.class); KNNVectorFieldType mockKNNVectorField = mock(KNNVectorFieldType.class); when(mockQueryShardContext.index()).thenReturn(dummyIndex); - when(mockKNNVectorField.getDimension()).thenReturn(4); when(mockKNNVectorField.getVectorDataType()).thenReturn(VectorDataType.FLOAT); - when(mockKNNVectorField.getSpaceType()).thenReturn(SpaceType.L2); + when(mockKNNVectorField.getKnnMappingConfig()).thenReturn(getMappingConfigForMethodMapping(getDefaultKNNMethodContext(), 4)); when(mockQueryShardContext.fieldMapper(anyString())).thenReturn(mockKNNVectorField); KNNQuery query = (KNNQuery) knnQueryBuilder.doToQuery(mockQueryShardContext); assertEquals(knnQueryBuilder.getK(), query.getK()); @@ -207,7 +206,6 @@ public void testDoToQuery_whenNormal_whenDoRadiusSearch_whenDistanceThreshold_th QueryShardContext mockQueryShardContext = mock(QueryShardContext.class); KNNVectorFieldType mockKNNVectorField = mock(KNNVectorFieldType.class); when(mockQueryShardContext.index()).thenReturn(dummyIndex); - when(mockKNNVectorField.getDimension()).thenReturn(4); when(mockKNNVectorField.getVectorDataType()).thenReturn(VectorDataType.FLOAT); when(mockQueryShardContext.fieldMapper(anyString())).thenReturn(mockKNNVectorField); MethodComponentContext methodComponentContext = new MethodComponentContext( @@ -215,7 +213,7 @@ public void testDoToQuery_whenNormal_whenDoRadiusSearch_whenDistanceThreshold_th ImmutableMap.of() ); KNNMethodContext knnMethodContext = new KNNMethodContext(KNNEngine.LUCENE, SpaceType.L2, methodComponentContext); - when(mockKNNVectorField.getKnnMethodContext()).thenReturn(knnMethodContext); + when(mockKNNVectorField.getKnnMappingConfig()).thenReturn(getMappingConfigForMethodMapping(knnMethodContext, 4)); FloatVectorSimilarityQuery query = (FloatVectorSimilarityQuery) knnQueryBuilder.doToQuery(mockQueryShardContext); float resultSimilarity = KNNEngine.LUCENE.distanceToRadialThreshold(MAX_DISTANCE, SpaceType.L2); @@ -239,7 +237,6 @@ public void testDoToQuery_whenNormal_whenDoRadiusSearch_whenScoreThreshold_thenS QueryShardContext mockQueryShardContext = mock(QueryShardContext.class); KNNVectorFieldType mockKNNVectorField = mock(KNNVectorFieldType.class); when(mockQueryShardContext.index()).thenReturn(dummyIndex); - when(mockKNNVectorField.getDimension()).thenReturn(4); when(mockKNNVectorField.getVectorDataType()).thenReturn(VectorDataType.FLOAT); when(mockQueryShardContext.fieldMapper(anyString())).thenReturn(mockKNNVectorField); MethodComponentContext methodComponentContext = new MethodComponentContext( @@ -247,7 +244,7 @@ public void testDoToQuery_whenNormal_whenDoRadiusSearch_whenScoreThreshold_thenS ImmutableMap.of() ); KNNMethodContext knnMethodContext = new KNNMethodContext(KNNEngine.LUCENE, SpaceType.L2, methodComponentContext); - when(mockKNNVectorField.getKnnMethodContext()).thenReturn(knnMethodContext); + when(mockKNNVectorField.getKnnMappingConfig()).thenReturn(getMappingConfigForMethodMapping(knnMethodContext, 4)); FloatVectorSimilarityQuery query = (FloatVectorSimilarityQuery) knnQueryBuilder.doToQuery(mockQueryShardContext); assertTrue(query.toString().contains("resultSimilarity=" + 0.5f)); } @@ -266,16 +263,14 @@ public void testDoToQuery_whenDoRadiusSearch_whenPassNegativeDistance_whenSuppor QueryShardContext mockQueryShardContext = mock(QueryShardContext.class); KNNVectorFieldType mockKNNVectorField = mock(KNNVectorFieldType.class); when(mockQueryShardContext.index()).thenReturn(dummyIndex); - when(mockKNNVectorField.getDimension()).thenReturn(4); when(mockKNNVectorField.getVectorDataType()).thenReturn(VectorDataType.FLOAT); when(mockQueryShardContext.fieldMapper(anyString())).thenReturn(mockKNNVectorField); MethodComponentContext methodComponentContext = new MethodComponentContext( org.opensearch.knn.common.KNNConstants.METHOD_HNSW, ImmutableMap.of() ); - when(mockKNNVectorField.getKnnMethodContext()).thenReturn( - new KNNMethodContext(KNNEngine.FAISS, SpaceType.INNER_PRODUCT, methodComponentContext) - ); + KNNMethodContext knnMethodContext = new KNNMethodContext(KNNEngine.FAISS, SpaceType.INNER_PRODUCT, methodComponentContext); + when(mockKNNVectorField.getKnnMappingConfig()).thenReturn(getMappingConfigForMethodMapping(knnMethodContext, 4)); IndexSettings indexSettings = mock(IndexSettings.class); when(mockQueryShardContext.getIndexSettings()).thenReturn(indexSettings); when(indexSettings.getMaxResultWindow()).thenReturn(1000); @@ -298,16 +293,14 @@ public void testDoToQuery_whenDoRadiusSearch_whenPassNegativeDistance_whenUnSupp QueryShardContext mockQueryShardContext = mock(QueryShardContext.class); KNNVectorFieldType mockKNNVectorField = mock(KNNVectorFieldType.class); when(mockQueryShardContext.index()).thenReturn(dummyIndex); - when(mockKNNVectorField.getDimension()).thenReturn(4); when(mockKNNVectorField.getVectorDataType()).thenReturn(VectorDataType.FLOAT); when(mockQueryShardContext.fieldMapper(anyString())).thenReturn(mockKNNVectorField); MethodComponentContext methodComponentContext = new MethodComponentContext( org.opensearch.knn.common.KNNConstants.METHOD_HNSW, ImmutableMap.of() ); - when(mockKNNVectorField.getKnnMethodContext()).thenReturn( - new KNNMethodContext(KNNEngine.FAISS, SpaceType.L2, methodComponentContext) - ); + KNNMethodContext knnMethodContext = new KNNMethodContext(KNNEngine.FAISS, SpaceType.L2, methodComponentContext); + when(mockKNNVectorField.getKnnMappingConfig()).thenReturn(getMappingConfigForMethodMapping(knnMethodContext, 4)); IndexSettings indexSettings = mock(IndexSettings.class); when(mockQueryShardContext.getIndexSettings()).thenReturn(indexSettings); when(indexSettings.getMaxResultWindow()).thenReturn(1000); @@ -325,16 +318,14 @@ public void testDoToQuery_whenDoRadiusSearch_whenPassScoreMoreThanOne_whenSuppor QueryShardContext mockQueryShardContext = mock(QueryShardContext.class); KNNVectorFieldType mockKNNVectorField = mock(KNNVectorFieldType.class); when(mockQueryShardContext.index()).thenReturn(dummyIndex); - when(mockKNNVectorField.getDimension()).thenReturn(4); when(mockKNNVectorField.getVectorDataType()).thenReturn(VectorDataType.FLOAT); when(mockQueryShardContext.fieldMapper(anyString())).thenReturn(mockKNNVectorField); MethodComponentContext methodComponentContext = new MethodComponentContext( org.opensearch.knn.common.KNNConstants.METHOD_HNSW, ImmutableMap.of() ); - when(mockKNNVectorField.getKnnMethodContext()).thenReturn( - new KNNMethodContext(KNNEngine.FAISS, SpaceType.INNER_PRODUCT, methodComponentContext) - ); + KNNMethodContext knnMethodContext = new KNNMethodContext(KNNEngine.FAISS, SpaceType.INNER_PRODUCT, methodComponentContext); + when(mockKNNVectorField.getKnnMappingConfig()).thenReturn(getMappingConfigForMethodMapping(knnMethodContext, 4)); IndexSettings indexSettings = mock(IndexSettings.class); when(mockQueryShardContext.getIndexSettings()).thenReturn(indexSettings); when(indexSettings.getMaxResultWindow()).thenReturn(1000); @@ -352,16 +343,14 @@ public void testDoToQuery_whenDoRadiusSearch_whenPassScoreMoreThanOne_whenUnsupp QueryShardContext mockQueryShardContext = mock(QueryShardContext.class); KNNVectorFieldType mockKNNVectorField = mock(KNNVectorFieldType.class); when(mockQueryShardContext.index()).thenReturn(dummyIndex); - when(mockKNNVectorField.getDimension()).thenReturn(4); when(mockKNNVectorField.getVectorDataType()).thenReturn(VectorDataType.FLOAT); when(mockQueryShardContext.fieldMapper(anyString())).thenReturn(mockKNNVectorField); MethodComponentContext methodComponentContext = new MethodComponentContext( org.opensearch.knn.common.KNNConstants.METHOD_HNSW, ImmutableMap.of() ); - when(mockKNNVectorField.getKnnMethodContext()).thenReturn( - new KNNMethodContext(KNNEngine.FAISS, SpaceType.L2, methodComponentContext) - ); + KNNMethodContext knnMethodContext = new KNNMethodContext(KNNEngine.FAISS, SpaceType.L2, methodComponentContext); + when(mockKNNVectorField.getKnnMappingConfig()).thenReturn(getMappingConfigForMethodMapping(knnMethodContext, 4)); IndexSettings indexSettings = mock(IndexSettings.class); when(mockQueryShardContext.getIndexSettings()).thenReturn(indexSettings); when(indexSettings.getMaxResultWindow()).thenReturn(1000); @@ -383,16 +372,14 @@ public void testDoToQuery_whenPassNegativeDistance_whenSupportedSpaceType_thenSu QueryShardContext mockQueryShardContext = mock(QueryShardContext.class); KNNVectorFieldType mockKNNVectorField = mock(KNNVectorFieldType.class); when(mockQueryShardContext.index()).thenReturn(dummyIndex); - when(mockKNNVectorField.getDimension()).thenReturn(4); when(mockKNNVectorField.getVectorDataType()).thenReturn(VectorDataType.FLOAT); when(mockQueryShardContext.fieldMapper(anyString())).thenReturn(mockKNNVectorField); MethodComponentContext methodComponentContext = new MethodComponentContext( org.opensearch.knn.common.KNNConstants.METHOD_HNSW, ImmutableMap.of() ); - when(mockKNNVectorField.getKnnMethodContext()).thenReturn( - new KNNMethodContext(KNNEngine.FAISS, SpaceType.INNER_PRODUCT, methodComponentContext) - ); + KNNMethodContext knnMethodContext = new KNNMethodContext(KNNEngine.FAISS, SpaceType.INNER_PRODUCT, methodComponentContext); + when(mockKNNVectorField.getKnnMappingConfig()).thenReturn(getMappingConfigForMethodMapping(knnMethodContext, 4)); IndexSettings indexSettings = mock(IndexSettings.class); when(mockQueryShardContext.getIndexSettings()).thenReturn(indexSettings); when(indexSettings.getMaxResultWindow()).thenReturn(1000); @@ -416,16 +403,14 @@ public void testDoToQuery_whenPassNegativeDistance_whenUnSupportedSpaceType_then QueryShardContext mockQueryShardContext = mock(QueryShardContext.class); KNNVectorFieldType mockKNNVectorField = mock(KNNVectorFieldType.class); when(mockQueryShardContext.index()).thenReturn(dummyIndex); - when(mockKNNVectorField.getDimension()).thenReturn(4); when(mockKNNVectorField.getVectorDataType()).thenReturn(VectorDataType.FLOAT); when(mockQueryShardContext.fieldMapper(anyString())).thenReturn(mockKNNVectorField); MethodComponentContext methodComponentContext = new MethodComponentContext( org.opensearch.knn.common.KNNConstants.METHOD_HNSW, ImmutableMap.of() ); - when(mockKNNVectorField.getKnnMethodContext()).thenReturn( - new KNNMethodContext(KNNEngine.FAISS, SpaceType.L2, methodComponentContext) - ); + KNNMethodContext knnMethodContext = new KNNMethodContext(KNNEngine.FAISS, SpaceType.L2, methodComponentContext); + when(mockKNNVectorField.getKnnMappingConfig()).thenReturn(getMappingConfigForMethodMapping(knnMethodContext, 4)); IndexSettings indexSettings = mock(IndexSettings.class); when(mockQueryShardContext.getIndexSettings()).thenReturn(indexSettings); when(indexSettings.getMaxResultWindow()).thenReturn(1000); @@ -444,7 +429,6 @@ public void testDoToQuery_whenRadialSearchOnBinaryIndex_thenException() { QueryShardContext mockQueryShardContext = mock(QueryShardContext.class); KNNVectorFieldType mockKNNVectorField = mock(KNNVectorFieldType.class); when(mockQueryShardContext.index()).thenReturn(dummyIndex); - when(mockKNNVectorField.getDimension()).thenReturn(8); when(mockKNNVectorField.getVectorDataType()).thenReturn(VectorDataType.BINARY); when(mockQueryShardContext.fieldMapper(anyString())).thenReturn(mockKNNVectorField); MethodComponentContext methodComponentContext = new MethodComponentContext( @@ -452,7 +436,7 @@ public void testDoToQuery_whenRadialSearchOnBinaryIndex_thenException() { ImmutableMap.of() ); KNNMethodContext knnMethodContext = new KNNMethodContext(KNNEngine.FAISS, SpaceType.HAMMING, methodComponentContext); - when(mockKNNVectorField.getKnnMethodContext()).thenReturn(knnMethodContext); + when(mockKNNVectorField.getKnnMappingConfig()).thenReturn(getMappingConfigForMethodMapping(knnMethodContext, 8)); Exception e = expectThrows(UnsupportedOperationException.class, () -> knnQueryBuilder.doToQuery(mockQueryShardContext)); assertTrue(e.getMessage().contains("Binary data type does not support radial search")); } @@ -470,15 +454,13 @@ public void testDoToQuery_KnnQueryWithFilter_Lucene() throws Exception { QueryShardContext mockQueryShardContext = mock(QueryShardContext.class); KNNVectorFieldType mockKNNVectorField = mock(KNNVectorFieldType.class); when(mockQueryShardContext.index()).thenReturn(dummyIndex); - when(mockKNNVectorField.getDimension()).thenReturn(4); when(mockKNNVectorField.getVectorDataType()).thenReturn(VectorDataType.FLOAT); - when(mockKNNVectorField.getSpaceType()).thenReturn(SpaceType.L2); MethodComponentContext methodComponentContext = new MethodComponentContext( org.opensearch.knn.common.KNNConstants.METHOD_HNSW, ImmutableMap.of() ); KNNMethodContext knnMethodContext = new KNNMethodContext(KNNEngine.LUCENE, SpaceType.L2, methodComponentContext); - when(mockKNNVectorField.getKnnMethodContext()).thenReturn(knnMethodContext); + when(mockKNNVectorField.getKnnMappingConfig()).thenReturn(getMappingConfigForMethodMapping(knnMethodContext, 4)); when(mockQueryShardContext.fieldMapper(anyString())).thenReturn(mockKNNVectorField); // When @@ -504,14 +486,13 @@ public void testDoToQuery_whenDoRadiusSearch_whenDistanceThreshold_whenFilter_th QueryShardContext mockQueryShardContext = mock(QueryShardContext.class); KNNVectorFieldType mockKNNVectorField = mock(KNNVectorFieldType.class); when(mockQueryShardContext.index()).thenReturn(dummyIndex); - when(mockKNNVectorField.getDimension()).thenReturn(4); when(mockKNNVectorField.getVectorDataType()).thenReturn(VectorDataType.FLOAT); MethodComponentContext methodComponentContext = new MethodComponentContext( org.opensearch.knn.common.KNNConstants.METHOD_HNSW, ImmutableMap.of() ); KNNMethodContext knnMethodContext = new KNNMethodContext(KNNEngine.LUCENE, SpaceType.L2, methodComponentContext); - when(mockKNNVectorField.getKnnMethodContext()).thenReturn(knnMethodContext); + when(mockKNNVectorField.getKnnMappingConfig()).thenReturn(getMappingConfigForMethodMapping(knnMethodContext, 4)); when(mockQueryShardContext.fieldMapper(anyString())).thenReturn(mockKNNVectorField); Query query = knnQueryBuilder.doToQuery(mockQueryShardContext); assertNotNull(query); @@ -531,14 +512,13 @@ public void testDoToQuery_whenDoRadiusSearch_whenScoreThreshold_whenFilter_thenS QueryShardContext mockQueryShardContext = mock(QueryShardContext.class); KNNVectorFieldType mockKNNVectorField = mock(KNNVectorFieldType.class); when(mockQueryShardContext.index()).thenReturn(dummyIndex); - when(mockKNNVectorField.getDimension()).thenReturn(4); when(mockKNNVectorField.getVectorDataType()).thenReturn(VectorDataType.FLOAT); MethodComponentContext methodComponentContext = new MethodComponentContext( org.opensearch.knn.common.KNNConstants.METHOD_HNSW, ImmutableMap.of() ); KNNMethodContext knnMethodContext = new KNNMethodContext(KNNEngine.LUCENE, SpaceType.L2, methodComponentContext); - when(mockKNNVectorField.getKnnMethodContext()).thenReturn(knnMethodContext); + when(mockKNNVectorField.getKnnMappingConfig()).thenReturn(getMappingConfigForMethodMapping(knnMethodContext, 4)); when(mockQueryShardContext.fieldMapper(anyString())).thenReturn(mockKNNVectorField); Query query = knnQueryBuilder.doToQuery(mockQueryShardContext); assertNotNull(query); @@ -553,15 +533,13 @@ public void testDoToQuery_WhenknnQueryWithFilterAndFaissEngine_thenSuccess() { QueryShardContext mockQueryShardContext = mock(QueryShardContext.class); KNNVectorFieldType mockKNNVectorField = mock(KNNVectorFieldType.class); when(mockQueryShardContext.index()).thenReturn(dummyIndex); - when(mockKNNVectorField.getDimension()).thenReturn(4); - when(mockKNNVectorField.getSpaceType()).thenReturn(SpaceType.L2); MethodComponentContext methodComponentContext = new MethodComponentContext( org.opensearch.knn.common.KNNConstants.METHOD_HNSW, ImmutableMap.of() ); when(mockKNNVectorField.getVectorDataType()).thenReturn(VectorDataType.FLOAT); KNNMethodContext knnMethodContext = new KNNMethodContext(KNNEngine.FAISS, SpaceType.L2, methodComponentContext); - when(mockKNNVectorField.getKnnMethodContext()).thenReturn(knnMethodContext); + when(mockKNNVectorField.getKnnMappingConfig()).thenReturn(getMappingConfigForMethodMapping(knnMethodContext, 4)); when(mockQueryShardContext.fieldMapper(anyString())).thenReturn(mockKNNVectorField); // When @@ -586,10 +564,12 @@ public void testDoToQuery_ThrowsIllegalArgumentExceptionForUnknownMethodParamete QueryShardContext mockQueryShardContext = mock(QueryShardContext.class); KNNVectorFieldType mockKNNVectorField = mock(KNNVectorFieldType.class); when(mockQueryShardContext.fieldMapper(anyString())).thenReturn(mockKNNVectorField); - when(mockKNNVectorField.getDimension()).thenReturn(4); - when(mockKNNVectorField.getKnnMethodContext()).thenReturn( - new KNNMethodContext(KNNEngine.LUCENE, SpaceType.COSINESIMIL, new MethodComponentContext("hnsw", Map.of())) + KNNMethodContext knnMethodContext = new KNNMethodContext( + KNNEngine.LUCENE, + SpaceType.COSINESIMIL, + new MethodComponentContext("hnsw", Map.of()) ); + when(mockKNNVectorField.getKnnMappingConfig()).thenReturn(getMappingConfigForMethodMapping(knnMethodContext, 4)); float[] queryVector = { 1.0f, 2.0f, 3.0f, 4.0f }; KNNQueryBuilder knnQueryBuilder = KNNQueryBuilder.builder() @@ -609,15 +589,13 @@ public void testDoToQuery_whenknnQueryWithFilterAndNmsLibEngine_thenException() QueryShardContext mockQueryShardContext = mock(QueryShardContext.class); KNNVectorFieldType mockKNNVectorField = mock(KNNVectorFieldType.class); when(mockQueryShardContext.index()).thenReturn(dummyIndex); - when(mockKNNVectorField.getDimension()).thenReturn(4); - when(mockKNNVectorField.getSpaceType()).thenReturn(SpaceType.L2); MethodComponentContext methodComponentContext = new MethodComponentContext( org.opensearch.knn.common.KNNConstants.METHOD_HNSW, ImmutableMap.of() ); when(mockKNNVectorField.getVectorDataType()).thenReturn(VectorDataType.FLOAT); KNNMethodContext knnMethodContext = new KNNMethodContext(KNNEngine.NMSLIB, SpaceType.L2, methodComponentContext); - when(mockKNNVectorField.getKnnMethodContext()).thenReturn(knnMethodContext); + when(mockKNNVectorField.getKnnMappingConfig()).thenReturn(getMappingConfigForMethodMapping(knnMethodContext, 4)); when(mockQueryShardContext.fieldMapper(anyString())).thenReturn(mockKNNVectorField); expectThrows(IllegalArgumentException.class, () -> knnQueryBuilder.doToQuery(mockQueryShardContext)); } @@ -632,15 +610,12 @@ public void testDoToQuery_FromModel() { when(mockQueryShardContext.index()).thenReturn(dummyIndex); // Dimension is -1. In this case, model metadata will need to provide dimension - when(mockKNNVectorField.getDimension()).thenReturn(-K); when(mockKNNVectorField.getVectorDataType()).thenReturn(VectorDataType.FLOAT); - when(mockKNNVectorField.getKnnMethodContext()).thenReturn(null); String modelId = "test-model-id"; - when(mockKNNVectorField.getModelId()).thenReturn(modelId); + when(mockKNNVectorField.getKnnMappingConfig()).thenReturn(getMappingConfigForModelMapping(modelId, 4)); // Mock the modelDao to return mocked modelMetadata ModelMetadata modelMetadata = mock(ModelMetadata.class); - when(modelMetadata.getDimension()).thenReturn(4); when(modelMetadata.getKnnEngine()).thenReturn(KNNEngine.FAISS); when(modelMetadata.getSpaceType()).thenReturn(SpaceType.COSINESIMIL); when(modelMetadata.getState()).thenReturn(ModelState.CREATED); @@ -672,14 +647,11 @@ public void testDoToQuery_whenFromModel_whenDoRadiusSearch_whenDistanceThreshold KNNVectorFieldType mockKNNVectorField = mock(KNNVectorFieldType.class); when(mockQueryShardContext.index()).thenReturn(dummyIndex); - when(mockKNNVectorField.getDimension()).thenReturn(-K); when(mockKNNVectorField.getVectorDataType()).thenReturn(VectorDataType.FLOAT); - when(mockKNNVectorField.getKnnMethodContext()).thenReturn(null); String modelId = "test-model-id"; - when(mockKNNVectorField.getModelId()).thenReturn(modelId); + when(mockKNNVectorField.getKnnMappingConfig()).thenReturn(getMappingConfigForModelMapping(modelId, 4)); ModelMetadata modelMetadata = mock(ModelMetadata.class); - when(modelMetadata.getDimension()).thenReturn(4); when(modelMetadata.getKnnEngine()).thenReturn(KNNEngine.FAISS); when(modelMetadata.getSpaceType()).thenReturn(SpaceType.L2); when(modelMetadata.getState()).thenReturn(ModelState.CREATED); @@ -709,15 +681,11 @@ public void testDoToQuery_whenFromModel_whenDoRadiusSearch_whenScoreThreshold_th QueryShardContext mockQueryShardContext = mock(QueryShardContext.class); KNNVectorFieldType mockKNNVectorField = mock(KNNVectorFieldType.class); when(mockQueryShardContext.index()).thenReturn(dummyIndex); - - when(mockKNNVectorField.getDimension()).thenReturn(-K); when(mockKNNVectorField.getVectorDataType()).thenReturn(VectorDataType.FLOAT); - when(mockKNNVectorField.getKnnMethodContext()).thenReturn(null); String modelId = "test-model-id"; - when(mockKNNVectorField.getModelId()).thenReturn(modelId); + when(mockKNNVectorField.getKnnMappingConfig()).thenReturn(getMappingConfigForModelMapping(modelId, 4)); ModelMetadata modelMetadata = mock(ModelMetadata.class); - when(modelMetadata.getDimension()).thenReturn(4); when(modelMetadata.getKnnEngine()).thenReturn(KNNEngine.FAISS); when(modelMetadata.getSpaceType()).thenReturn(SpaceType.L2); when(modelMetadata.getState()).thenReturn(ModelState.CREATED); @@ -744,10 +712,10 @@ public void testDoToQuery_InvalidDimensions() { QueryShardContext mockQueryShardContext = mock(QueryShardContext.class); KNNVectorFieldType mockKNNVectorField = mock(KNNVectorFieldType.class); when(mockQueryShardContext.index()).thenReturn(dummyIndex); - when(mockKNNVectorField.getDimension()).thenReturn(400); + when(mockKNNVectorField.getKnnMappingConfig()).thenReturn(getMappingConfigForMethodMapping(getDefaultKNNMethodContext(), 400)); when(mockQueryShardContext.fieldMapper(anyString())).thenReturn(mockKNNVectorField); expectThrows(IllegalArgumentException.class, () -> knnQueryBuilder.doToQuery(mockQueryShardContext)); - when(mockKNNVectorField.getDimension()).thenReturn(K); + when(mockKNNVectorField.getKnnMappingConfig()).thenReturn(getMappingConfigForMethodMapping(getDefaultKNNMethodContext(), K)); expectThrows(IllegalArgumentException.class, () -> knnQueryBuilder.doToQuery(mockQueryShardContext)); } @@ -769,9 +737,10 @@ public void testDoToQuery_InvalidZeroFloatVector() { QueryShardContext mockQueryShardContext = mock(QueryShardContext.class); KNNVectorFieldType mockKNNVectorField = mock(KNNVectorFieldType.class); when(mockQueryShardContext.index()).thenReturn(dummyIndex); - when(mockKNNVectorField.getDimension()).thenReturn(4); when(mockKNNVectorField.getVectorDataType()).thenReturn(VectorDataType.FLOAT); - when(mockKNNVectorField.getSpaceType()).thenReturn(SpaceType.COSINESIMIL); + KNNMethodContext knnMethodContext = mock(KNNMethodContext.class); + when(knnMethodContext.getSpaceType()).thenReturn(SpaceType.COSINESIMIL); + when(mockKNNVectorField.getKnnMappingConfig()).thenReturn(getMappingConfigForMethodMapping(knnMethodContext, 4)); when(mockQueryShardContext.fieldMapper(anyString())).thenReturn(mockKNNVectorField); IllegalArgumentException exception = expectThrows( IllegalArgumentException.class, @@ -790,9 +759,10 @@ public void testDoToQuery_InvalidZeroByteVector() { QueryShardContext mockQueryShardContext = mock(QueryShardContext.class); KNNVectorFieldType mockKNNVectorField = mock(KNNVectorFieldType.class); when(mockQueryShardContext.index()).thenReturn(dummyIndex); - when(mockKNNVectorField.getDimension()).thenReturn(4); when(mockKNNVectorField.getVectorDataType()).thenReturn(VectorDataType.BYTE); - when(mockKNNVectorField.getSpaceType()).thenReturn(SpaceType.COSINESIMIL); + KNNMethodContext knnMethodContext = mock(KNNMethodContext.class); + when(knnMethodContext.getSpaceType()).thenReturn(SpaceType.COSINESIMIL); + when(mockKNNVectorField.getKnnMappingConfig()).thenReturn(getMappingConfigForMethodMapping(knnMethodContext, 4)); when(mockQueryShardContext.fieldMapper(anyString())).thenReturn(mockKNNVectorField); IllegalArgumentException exception = expectThrows( IllegalArgumentException.class, @@ -919,9 +889,8 @@ public void testRadialSearch_whenUnsupportedEngine_thenThrowException() { KNNVectorFieldType mockKNNVectorField = mock(KNNVectorFieldType.class); QueryShardContext mockQueryShardContext = mock(QueryShardContext.class); Index dummyIndex = new Index("dummy", "dummy"); - when(mockKNNVectorField.getKnnMethodContext()).thenReturn(knnMethodContext); + when(mockKNNVectorField.getKnnMappingConfig()).thenReturn(getMappingConfigForMethodMapping(knnMethodContext, 4)); when(mockQueryShardContext.index()).thenReturn(dummyIndex); - when(mockKNNVectorField.getDimension()).thenReturn(4); when(mockKNNVectorField.getVectorDataType()).thenReturn(VectorDataType.FLOAT); when(mockQueryShardContext.fieldMapper(anyString())).thenReturn(mockKNNVectorField); @@ -946,9 +915,8 @@ public void testRadialSearch_whenEfSearchIsSet_whenLuceneEngine_thenThrowExcepti KNNVectorFieldType mockKNNVectorField = mock(KNNVectorFieldType.class); QueryShardContext mockQueryShardContext = mock(QueryShardContext.class); Index dummyIndex = new Index("dummy", "dummy"); - when(mockKNNVectorField.getKnnMethodContext()).thenReturn(knnMethodContext); + when(mockKNNVectorField.getKnnMappingConfig()).thenReturn(getMappingConfigForMethodMapping(knnMethodContext, 4)); when(mockQueryShardContext.index()).thenReturn(dummyIndex); - when(mockKNNVectorField.getDimension()).thenReturn(4); when(mockQueryShardContext.fieldMapper(anyString())).thenReturn(mockKNNVectorField); expectThrows(IllegalArgumentException.class, () -> knnQueryBuilder.doToQuery(mockQueryShardContext)); @@ -972,9 +940,8 @@ public void testRadialSearch_whenEfSearchIsSet_whenFaissEngine_thenSuccess() { KNNVectorFieldType mockKNNVectorField = mock(KNNVectorFieldType.class); QueryShardContext mockQueryShardContext = mock(QueryShardContext.class); Index dummyIndex = new Index("dummy", "dummy"); - when(mockKNNVectorField.getKnnMethodContext()).thenReturn(knnMethodContext); + when(mockKNNVectorField.getKnnMappingConfig()).thenReturn(getMappingConfigForMethodMapping(knnMethodContext, 4)); when(mockQueryShardContext.index()).thenReturn(dummyIndex); - when(mockKNNVectorField.getDimension()).thenReturn(4); when(mockQueryShardContext.fieldMapper(anyString())).thenReturn(mockKNNVectorField); IndexSettings indexSettings = mock(IndexSettings.class); when(mockQueryShardContext.getIndexSettings()).thenReturn(indexSettings); @@ -992,9 +959,8 @@ public void testDoToQuery_whenBinary_thenValid() throws Exception { QueryShardContext mockQueryShardContext = mock(QueryShardContext.class); KNNVectorFieldType mockKNNVectorField = mock(KNNVectorFieldType.class); when(mockQueryShardContext.index()).thenReturn(dummyIndex); - when(mockKNNVectorField.getDimension()).thenReturn(32); when(mockKNNVectorField.getVectorDataType()).thenReturn(VectorDataType.BINARY); - when(mockKNNVectorField.getSpaceType()).thenReturn(SpaceType.HAMMING); + when(mockKNNVectorField.getKnnMappingConfig()).thenReturn(getMappingConfigForMethodMapping(getDefaultBinaryKNNMethodContext(), 32)); when(mockQueryShardContext.fieldMapper(anyString())).thenReturn(mockKNNVectorField); KNNQuery query = (KNNQuery) knnQueryBuilder.doToQuery(mockQueryShardContext); assertArrayEquals(expectedQueryVector, query.getByteQueryVector()); @@ -1008,9 +974,8 @@ public void testDoToQuery_whenBinaryWithInvalidDimension_thenException() throws QueryShardContext mockQueryShardContext = mock(QueryShardContext.class); KNNVectorFieldType mockKNNVectorField = mock(KNNVectorFieldType.class); when(mockQueryShardContext.index()).thenReturn(dummyIndex); - when(mockKNNVectorField.getDimension()).thenReturn(8); when(mockKNNVectorField.getVectorDataType()).thenReturn(VectorDataType.BINARY); - when(mockKNNVectorField.getSpaceType()).thenReturn(SpaceType.HAMMING); + when(mockKNNVectorField.getKnnMappingConfig()).thenReturn(getMappingConfigForMethodMapping(getDefaultBinaryKNNMethodContext(), 8)); when(mockQueryShardContext.fieldMapper(anyString())).thenReturn(mockKNNVectorField); Exception ex = expectThrows(IllegalArgumentException.class, () -> knnQueryBuilder.doToQuery(mockQueryShardContext)); assertTrue(ex.getMessage(), ex.getMessage().contains("invalid dimension")); diff --git a/src/test/java/org/opensearch/knn/integ/KNNScriptScoringIT.java b/src/test/java/org/opensearch/knn/integ/KNNScriptScoringIT.java index e67ad40bc..ecd425c6c 100644 --- a/src/test/java/org/opensearch/knn/integ/KNNScriptScoringIT.java +++ b/src/test/java/org/opensearch/knn/integ/KNNScriptScoringIT.java @@ -47,6 +47,7 @@ import java.util.stream.Collectors; import static org.hamcrest.Matchers.containsString; +import static org.opensearch.knn.KNNTestCase.getMappingConfigForFlatMapping; import static org.opensearch.knn.common.KNNConstants.FAISS_NAME; import static org.opensearch.knn.common.KNNConstants.KNN_ENGINE; import static org.opensearch.knn.common.KNNConstants.METHOD_IVF; @@ -735,18 +736,20 @@ private Map createDataset( } private BiFunction getScoreFunction(SpaceType spaceType, float[] queryVector) { - KNNVectorFieldType knnVectorFieldType = new KNNVectorFieldType( - FIELD_NAME, - Collections.emptyMap(), - SpaceType.HAMMING == spaceType ? queryVector.length * 8 : queryVector.length, - SpaceType.HAMMING == spaceType ? VectorDataType.BINARY : VectorDataType.FLOAT, - null - ); List target = new ArrayList<>(queryVector.length); for (float f : queryVector) { target.add(f); } - KNNScoringSpace knnScoringSpace = KNNScoringSpaceFactory.create(spaceType.getValue(), target, knnVectorFieldType); + KNNScoringSpace knnScoringSpace = KNNScoringSpaceFactory.create( + spaceType.getValue(), + target, + new KNNVectorFieldType( + FIELD_NAME, + Collections.emptyMap(), + SpaceType.HAMMING == spaceType ? VectorDataType.BINARY : VectorDataType.FLOAT, + getMappingConfigForFlatMapping(SpaceType.HAMMING == spaceType ? queryVector.length * 8 : queryVector.length) + ) + ); switch (spaceType) { case L1: case L2: diff --git a/src/test/java/org/opensearch/knn/plugin/script/KNNScoringSpaceFactoryTests.java b/src/test/java/org/opensearch/knn/plugin/script/KNNScoringSpaceFactoryTests.java index 823d21080..c41e9763b 100644 --- a/src/test/java/org/opensearch/knn/plugin/script/KNNScoringSpaceFactoryTests.java +++ b/src/test/java/org/opensearch/knn/plugin/script/KNNScoringSpaceFactoryTests.java @@ -19,9 +19,11 @@ public class KNNScoringSpaceFactoryTests extends KNNTestCase { public void testValidSpaces() { KNNVectorFieldType knnVectorFieldType = mock(KNNVectorFieldType.class); - when(knnVectorFieldType.getDimension()).thenReturn(3); + when(knnVectorFieldType.getKnnMappingConfig()).thenReturn(getMappingConfigForMethodMapping(getDefaultKNNMethodContext(), 3)); KNNVectorFieldType knnVectorFieldTypeBinary = mock(KNNVectorFieldType.class); - when(knnVectorFieldTypeBinary.getDimension()).thenReturn(24); + when(knnVectorFieldTypeBinary.getKnnMappingConfig()).thenReturn( + getMappingConfigForMethodMapping(getDefaultBinaryKNNMethodContext(), 24) + ); when(knnVectorFieldTypeBinary.getVectorDataType()).thenReturn(VectorDataType.BINARY); NumberFieldMapper.NumberFieldType numberFieldType = new NumberFieldMapper.NumberFieldType( "field", @@ -66,9 +68,11 @@ public void testValidSpaces() { public void testInvalidSpace() { List floatQueryObject = List.of(1.0f, 1.0f, 1.0f); KNNVectorFieldType knnVectorFieldType = mock(KNNVectorFieldType.class); - when(knnVectorFieldType.getDimension()).thenReturn(3); + when(knnVectorFieldType.getKnnMappingConfig()).thenReturn(getMappingConfigForMethodMapping(getDefaultKNNMethodContext(), 3)); KNNVectorFieldType knnVectorFieldTypeBinary = mock(KNNVectorFieldType.class); - when(knnVectorFieldTypeBinary.getDimension()).thenReturn(24); + when(knnVectorFieldTypeBinary.getKnnMappingConfig()).thenReturn( + getMappingConfigForMethodMapping(getDefaultBinaryKNNMethodContext(), 24) + ); when(knnVectorFieldTypeBinary.getVectorDataType()).thenReturn(VectorDataType.BINARY); // Verify diff --git a/src/test/java/org/opensearch/knn/plugin/script/KNNScoringSpaceTests.java b/src/test/java/org/opensearch/knn/plugin/script/KNNScoringSpaceTests.java index 6c557c8dd..4fc549d6b 100644 --- a/src/test/java/org/opensearch/knn/plugin/script/KNNScoringSpaceTests.java +++ b/src/test/java/org/opensearch/knn/plugin/script/KNNScoringSpaceTests.java @@ -57,8 +57,13 @@ private void expectThrowsExceptionWithKNNFieldWithBinaryDataType(Class clazz) th public void testL2_whenValid_thenSucceed() { float[] arrayFloat = new float[] { 1.0f, 2.0f, 3.0f }; List arrayListQueryObject = new ArrayList<>(Arrays.asList(1.0, 2.0, 3.0)); - KNNMethodContext knnMethodContext = KNNMethodContext.getDefault(); - KNNVectorFieldType fieldType = new KNNVectorFieldType("test", Collections.emptyMap(), 3, knnMethodContext); + KNNMethodContext knnMethodContext = getDefaultKNNMethodContext(); + KNNVectorFieldType fieldType = new KNNVectorFieldType( + "test", + Collections.emptyMap(), + VectorDataType.FLOAT, + getMappingConfigForMethodMapping(knnMethodContext, 3) + ); KNNScoringSpace.L2 l2 = new KNNScoringSpace.L2(arrayListQueryObject, fieldType); assertEquals(1F, l2.getScoringMethod().apply(arrayFloat, arrayFloat), 0.1F); } @@ -73,9 +78,13 @@ public void testCosineSimilarity_whenValid_thenSucceed() { float[] arrayFloat = new float[] { 1.0f, 2.0f, 3.0f }; List arrayListQueryObject = new ArrayList<>(Arrays.asList(2.0, 4.0, 6.0)); float[] arrayFloat2 = new float[] { 2.0f, 4.0f, 6.0f }; - KNNMethodContext knnMethodContext = KNNMethodContext.getDefault(); - - KNNVectorFieldType fieldType = new KNNVectorFieldType("test", Collections.emptyMap(), 3, knnMethodContext); + KNNMethodContext knnMethodContext = getDefaultKNNMethodContext(); + KNNVectorFieldType fieldType = new KNNVectorFieldType( + "test", + Collections.emptyMap(), + VectorDataType.FLOAT, + getMappingConfigForMethodMapping(knnMethodContext, 3) + ); KNNScoringSpace.CosineSimilarity cosineSimilarity = new KNNScoringSpace.CosineSimilarity(arrayListQueryObject, fieldType); assertEquals(2F, cosineSimilarity.getScoringMethod().apply(arrayFloat2, arrayFloat), 0.1F); @@ -92,8 +101,13 @@ public void testCosineSimilarity_whenValid_thenSucceed() { } public void testCosineSimilarity_whenZeroVector_thenException() { - KNNMethodContext knnMethodContext = KNNMethodContext.getDefault(); - KNNVectorFieldType fieldType = new KNNVectorFieldType("test", Collections.emptyMap(), 3, knnMethodContext); + KNNMethodContext knnMethodContext = getDefaultKNNMethodContext(); + KNNVectorFieldType fieldType = new KNNVectorFieldType( + "test", + Collections.emptyMap(), + VectorDataType.FLOAT, + getMappingConfigForMethodMapping(knnMethodContext, 3) + ); final List queryZeroVector = List.of(0.0f, 0.0f, 0.0f); IllegalArgumentException exception1 = expectThrows( @@ -116,9 +130,14 @@ public void testInnerProd_whenValid_thenSucceed() { float[] arrayFloat_case1 = new float[] { 1.0f, 2.0f, 3.0f }; List arrayListQueryObject_case1 = new ArrayList<>(Arrays.asList(1.0, 2.0, 3.0)); float[] arrayFloat2_case1 = new float[] { 1.0f, 1.0f, 1.0f }; - KNNMethodContext knnMethodContext = KNNMethodContext.getDefault(); + KNNMethodContext knnMethodContext = getDefaultKNNMethodContext(); - KNNVectorFieldType fieldType = new KNNVectorFieldType("test", Collections.emptyMap(), 3, knnMethodContext); + KNNVectorFieldType fieldType = new KNNVectorFieldType( + "test", + Collections.emptyMap(), + VectorDataType.FLOAT, + getMappingConfigForMethodMapping(knnMethodContext, 3) + ); KNNScoringSpace.InnerProd innerProd = new KNNScoringSpace.InnerProd(arrayListQueryObject_case1, fieldType); assertEquals(7.0F, innerProd.getScoringMethod().apply(arrayFloat_case1, arrayFloat2_case1), 0.001F); @@ -183,14 +202,14 @@ public void testHammingBit_Base64() { public void testHamming_whenKNNFieldType_thenSucceed() { List arrayListQueryObject = new ArrayList<>(Arrays.asList(1.0, 2.0, 3.0)); - KNNMethodContext knnMethodContext = KNNMethodContext.getDefault(); + KNNMethodContext knnMethodContext = getDefaultKNNMethodContext(); KNNVectorFieldType fieldType = new KNNVectorFieldType( "test", Collections.emptyMap(), - 8 * arrayListQueryObject.size(), - knnMethodContext, - VectorDataType.BINARY + VectorDataType.BINARY, + getMappingConfigForMethodMapping(knnMethodContext, 8 * arrayListQueryObject.size()) ); + KNNScoringSpace.Hamming hamming = new KNNScoringSpace.Hamming(arrayListQueryObject, fieldType); float[] arrayFloat = new float[] { 1.0f, 2.0f, 3.0f }; diff --git a/src/test/java/org/opensearch/knn/plugin/script/KNNScoringSpaceUtilTests.java b/src/test/java/org/opensearch/knn/plugin/script/KNNScoringSpaceUtilTests.java index 781ed2350..2374e4f7b 100644 --- a/src/test/java/org/opensearch/knn/plugin/script/KNNScoringSpaceUtilTests.java +++ b/src/test/java/org/opensearch/knn/plugin/script/KNNScoringSpaceUtilTests.java @@ -64,7 +64,7 @@ public void testParseKNNVectorQuery() { KNNVectorFieldType fieldType = mock(KNNVectorFieldType.class); - when(fieldType.getDimension()).thenReturn(3); + when(fieldType.getKnnMappingConfig()).thenReturn(getMappingConfigForMethodMapping(getDefaultKNNMethodContext(), 3)); assertArrayEquals(arrayFloat, KNNScoringSpaceUtil.parseToFloatArray(arrayListQueryObject, 3, VectorDataType.FLOAT), 0.1f); expectThrows( diff --git a/src/test/java/org/opensearch/knn/plugin/transport/TrainingJobRouterTransportActionTests.java b/src/test/java/org/opensearch/knn/plugin/transport/TrainingJobRouterTransportActionTests.java index 3515c690d..8cff4dfa1 100644 --- a/src/test/java/org/opensearch/knn/plugin/transport/TrainingJobRouterTransportActionTests.java +++ b/src/test/java/org/opensearch/knn/plugin/transport/TrainingJobRouterTransportActionTests.java @@ -23,7 +23,6 @@ import org.opensearch.cluster.node.DiscoveryNodes; import org.opensearch.cluster.service.ClusterService; import org.opensearch.knn.KNNTestCase; -import org.opensearch.knn.index.engine.KNNMethodContext; import org.opensearch.knn.index.VectorDataType; import org.opensearch.search.SearchHit; import org.opensearch.search.SearchHits; @@ -303,7 +302,7 @@ public void testTrainingIndexSize() { // Setup the request TrainingModelRequest trainingModelRequest = new TrainingModelRequest( null, - KNNMethodContext.getDefault(), + getDefaultKNNMethodContext(), dimension, trainingIndexName, "training-field", @@ -350,7 +349,7 @@ public void testTrainIndexSize_whenDataTypeIsBinary() { // Setup the request TrainingModelRequest trainingModelRequest = new TrainingModelRequest( null, - KNNMethodContext.getDefault(), + getDefaultKNNMethodContext(), dimension, trainingIndexName, "training-field", @@ -398,7 +397,7 @@ public void testTrainIndexSize_whenDataTypeIsByte() { // Setup the request TrainingModelRequest trainingModelRequest = new TrainingModelRequest( null, - KNNMethodContext.getDefault(), + getDefaultKNNMethodContext(), dimension, trainingIndexName, "training-field", diff --git a/src/test/java/org/opensearch/knn/plugin/transport/TrainingModelRequestTests.java b/src/test/java/org/opensearch/knn/plugin/transport/TrainingModelRequestTests.java index 53e59129e..83d39cfdc 100644 --- a/src/test/java/org/opensearch/knn/plugin/transport/TrainingModelRequestTests.java +++ b/src/test/java/org/opensearch/knn/plugin/transport/TrainingModelRequestTests.java @@ -48,7 +48,7 @@ public class TrainingModelRequestTests extends KNNTestCase { public void testStreams() throws IOException { String modelId = "test-model-id"; - KNNMethodContext knnMethodContext = KNNMethodContext.getDefault(); + KNNMethodContext knnMethodContext = getDefaultKNNMethodContext(); int dimension = 10; String trainingIndex = "test-training-index"; String trainingField = "test-training-field"; @@ -105,7 +105,7 @@ public void testStreams() throws IOException { public void testGetters() { String modelId = "test-model-id"; - KNNMethodContext knnMethodContext = KNNMethodContext.getDefault(); + KNNMethodContext knnMethodContext = getDefaultKNNMethodContext(); int dimension = 10; String trainingIndex = "test-training-index"; String trainingField = "test-training-field"; From 1527f11b487465abf61f27f3c75323dfe41311e3 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Mon, 12 Aug 2024 15:30:00 -0700 Subject: [PATCH 322/416] Integrate Lucene Vector field with native engines to use KNNVectorFormat during segment creation (#1945) (#1948) Signed-off-by: Navneet Verma (cherry picked from commit 5a5351ff79059f48518a45bde3af89b8e970ff43) Co-authored-by: Navneet Verma --- CHANGELOG.md | 3 +- .../org/opensearch/knn/index/KNNSettings.java | 33 ++++- .../index/mapper/FlatVectorFieldMapper.java | 4 + .../index/mapper/KNNVectorFieldMapper.java | 45 +++--- .../mapper/KNNVectorFieldMapperUtil.java | 16 +++ .../knn/index/mapper/LuceneFieldMapper.java | 8 +- .../knn/index/mapper/MethodFieldMapper.java | 20 +++ .../knn/index/mapper/ModelFieldMapper.java | 20 ++- .../knn/index/codec/KNNCodecTestCase.java | 10 +- .../mapper/KNNVectorFieldMapperTests.java | 128 ++++++++++++++++-- .../mapper/KNNVectorFieldMapperUtilTests.java | 21 +++ 11 files changed, 267 insertions(+), 41 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 879a50e1c..5e3060530 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Bug Fixes * Corrected search logic for scenario with non-existent fields in filter [#1874](https://github.com/opensearch-project/k-NN/pull/1874) * Fix graph merge stats size calculation [#1844](https://github.com/opensearch-project/k-NN/pull/1844) +* Integrate Lucene Vector field with native engines to use KNNVectorFormat during segment creation [#1945](https://github.com/opensearch-project/k-NN/pull/1945) ### Infrastructure ### Documentation ### Maintenance @@ -30,4 +31,4 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), * Refactor KNNVectorFieldType from KNNVectorFieldMapper to a separate class for better readability. [#1931](https://github.com/opensearch-project/k-NN/pull/1931) * Generalize lib interface to return context objects [#1925](https://github.com/opensearch-project/k-NN/pull/1925) * Move k search k-NN query to re-write phase of vector search query for Native Engines [#1877](https://github.com/opensearch-project/k-NN/pull/1877) -* Restructure mappers to better handle null cases and avoid branching in parsing [#1939](https://github.com/opensearch-project/k-NN/pull/1939) \ No newline at end of file +* Restructure mappers to better handle null cases and avoid branching in parsing [#1939](https://github.com/opensearch-project/k-NN/pull/1939) diff --git a/src/main/java/org/opensearch/knn/index/KNNSettings.java b/src/main/java/org/opensearch/knn/index/KNNSettings.java index 33c7ff410..4ced38b38 100644 --- a/src/main/java/org/opensearch/knn/index/KNNSettings.java +++ b/src/main/java/org/opensearch/knn/index/KNNSettings.java @@ -82,6 +82,12 @@ public class KNNSettings { public static final String MODEL_CACHE_SIZE_LIMIT = "knn.model.cache.size.limit"; public static final String ADVANCED_FILTERED_EXACT_SEARCH_THRESHOLD = "index.knn.advanced.filtered_exact_search_threshold"; public static final String KNN_FAISS_AVX2_DISABLED = "knn.faiss.avx2.disabled"; + /** + * TODO: This setting is only added to ensure that main branch of k_NN plugin doesn't break till other parts of the + * code is getting ready. Will remove this setting once all changes related to integration of KNNVectorsFormat is added + * for native engines. + */ + public static final String KNN_USE_LUCENE_VECTOR_FORMAT_ENABLED = "knn.use.format.enabled"; /** * Default setting values @@ -255,6 +261,17 @@ public class KNNSettings { NodeScope ); + /** + * TODO: This setting is only added to ensure that main branch of k_NN plugin doesn't break till other parts of the + * code is getting ready. Will remove this setting once all changes related to integration of KNNVectorsFormat is added + * for native engines. + */ + public static final Setting KNN_USE_LUCENE_VECTOR_FORMAT_ENABLED_SETTING = Setting.boolSetting( + KNN_USE_LUCENE_VECTOR_FORMAT_ENABLED, + false, + NodeScope + ); + /** * Dynamic settings */ @@ -379,6 +396,10 @@ private Setting getSetting(String key) { return KNN_VECTOR_STREAMING_MEMORY_LIMIT_PCT_SETTING; } + if (KNN_USE_LUCENE_VECTOR_FORMAT_ENABLED.equals(key)) { + return KNN_USE_LUCENE_VECTOR_FORMAT_ENABLED_SETTING; + } + throw new IllegalArgumentException("Cannot find setting by key [" + key + "]"); } @@ -397,7 +418,8 @@ public List> getSettings() { MODEL_CACHE_SIZE_LIMIT_SETTING, ADVANCED_FILTERED_EXACT_SEARCH_THRESHOLD_SETTING, KNN_FAISS_AVX2_DISABLED_SETTING, - KNN_VECTOR_STREAMING_MEMORY_LIMIT_PCT_SETTING + KNN_VECTOR_STREAMING_MEMORY_LIMIT_PCT_SETTING, + KNN_USE_LUCENE_VECTOR_FORMAT_ENABLED_SETTING ); return Stream.concat(settings.stream(), Stream.concat(getFeatureFlags().stream(), dynamicCacheSettings.values().stream())) .collect(Collectors.toList()); @@ -443,6 +465,15 @@ public static Integer getFilteredExactSearchThreshold(final String indexName) { .getAsInt(ADVANCED_FILTERED_EXACT_SEARCH_THRESHOLD, ADVANCED_FILTERED_EXACT_SEARCH_THRESHOLD_DEFAULT_VALUE); } + /** + * TODO: This setting is only added to ensure that main branch of k_NN plugin doesn't break till other parts of the + * code is getting ready. Will remove this setting once all changes related to integration of KNNVectorsFormat is added + * for native engines. + */ + public static boolean getIsLuceneVectorFormatEnabled() { + return KNNSettings.state().getSettingValue(KNNSettings.KNN_USE_LUCENE_VECTOR_FORMAT_ENABLED); + } + public void initialize(Client client, ClusterService clusterService) { this.client = client; this.clusterService = clusterService; diff --git a/src/main/java/org/opensearch/knn/index/mapper/FlatVectorFieldMapper.java b/src/main/java/org/opensearch/knn/index/mapper/FlatVectorFieldMapper.java index fffff30f4..146b5132f 100644 --- a/src/main/java/org/opensearch/knn/index/mapper/FlatVectorFieldMapper.java +++ b/src/main/java/org/opensearch/knn/index/mapper/FlatVectorFieldMapper.java @@ -6,6 +6,7 @@ package org.opensearch.knn.index.mapper; import org.apache.lucene.document.FieldType; +import org.apache.lucene.index.DocValuesType; import org.opensearch.Version; import org.opensearch.common.Explicit; import org.opensearch.knn.index.VectorDataType; @@ -57,8 +58,11 @@ private FlatVectorFieldMapper( Version indexCreatedVersion ) { super(simpleName, mappedFieldType, multiFields, copyTo, ignoreMalformed, stored, hasDocValues, indexCreatedVersion, null); + // setting it explicitly false here to ensure that when flatmapper is used Lucene based Vector field is not created. + this.useLuceneBasedVectorField = false; this.perDimensionValidator = selectPerDimensionValidator(vectorDataType); this.fieldType = new FieldType(KNNVectorFieldMapper.Defaults.FIELD_TYPE); + this.fieldType.setDocValuesType(DocValuesType.BINARY); this.fieldType.freeze(); } diff --git a/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapper.java b/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapper.java index 40eaa12ae..5d4d3ca58 100644 --- a/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapper.java +++ b/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapper.java @@ -16,7 +16,8 @@ import lombok.extern.log4j.Log4j2; import org.apache.lucene.document.Field; import org.apache.lucene.document.FieldType; -import org.apache.lucene.index.DocValuesType; +import org.apache.lucene.document.KnnByteVectorField; +import org.apache.lucene.document.KnnFloatVectorField; import org.apache.lucene.index.IndexOptions; import org.opensearch.Version; import org.opensearch.common.Explicit; @@ -456,6 +457,7 @@ public Mapper.Builder parse(String name, Map node, ParserCont protected boolean hasDocValues; protected VectorDataType vectorDataType; protected ModelDao modelDao; + protected boolean useLuceneBasedVectorField; // We need to ensure that the original KNNMethodContext as parsed is stored to initialize the // Builder for serialization. So, we need to store it here. This is mainly to ensure that the legacy field mapper @@ -497,16 +499,29 @@ protected void parseCreateField(ParseContext context) throws IOException { parseCreateField(context, fieldType().getKnnMappingConfig().getDimension(), fieldType().getVectorDataType()); } + private Field createVectorField(float[] vectorValue) { + if (useLuceneBasedVectorField) { + return new KnnFloatVectorField(name(), vectorValue, fieldType); + } + return new VectorField(name(), vectorValue, fieldType); + } + + private Field createVectorField(byte[] vectorValue) { + if (useLuceneBasedVectorField) { + return new KnnByteVectorField(name(), vectorValue, fieldType); + } + return new VectorField(name(), vectorValue, fieldType); + } + /** * Function returns a list of fields to be indexed when the vector is float type. * * @param array array of floats - * @param fieldType {@link FieldType} * @return {@link List} of {@link Field} */ - protected List getFieldsForFloatVector(final float[] array, final FieldType fieldType) { + protected List getFieldsForFloatVector(final float[] array) { final List fields = new ArrayList<>(); - fields.add(new VectorField(name(), array, fieldType)); + fields.add(createVectorField(array)); if (this.stored) { fields.add(createStoredFieldForFloatVector(name(), array)); } @@ -517,12 +532,11 @@ protected List getFieldsForFloatVector(final float[] array, final FieldTy * Function returns a list of fields to be indexed when the vector is byte type. * * @param array array of bytes - * @param fieldType {@link FieldType} * @return {@link List} of {@link Field} */ - protected List getFieldsForByteVector(final byte[] array, final FieldType fieldType) { + protected List getFieldsForByteVector(final byte[] array) { final List fields = new ArrayList<>(); - fields.add(new VectorField(name(), array, fieldType)); + fields.add(createVectorField(array)); if (this.stored) { fields.add(createStoredFieldForByteVector(name(), array)); } @@ -561,24 +575,14 @@ protected void validatePreparse() { protected void parseCreateField(ParseContext context, int dimension, VectorDataType vectorDataType) throws IOException { validatePreparse(); - if (VectorDataType.BINARY == vectorDataType) { - Optional bytesArrayOptional = getBytesFromContext(context, dimension, vectorDataType); - - if (bytesArrayOptional.isEmpty()) { - return; - } - final byte[] array = bytesArrayOptional.get(); - getVectorValidator().validateVector(array); - context.doc().addAll(getFieldsForByteVector(array, fieldType)); - } else if (VectorDataType.BYTE == vectorDataType) { + if (VectorDataType.BINARY == vectorDataType || VectorDataType.BYTE == vectorDataType) { Optional bytesArrayOptional = getBytesFromContext(context, dimension, vectorDataType); - if (bytesArrayOptional.isEmpty()) { return; } final byte[] array = bytesArrayOptional.get(); getVectorValidator().validateVector(array); - context.doc().addAll(getFieldsForByteVector(array, fieldType)); + context.doc().addAll(getFieldsForByteVector(array)); } else if (VectorDataType.FLOAT == vectorDataType) { Optional floatsArrayOptional = getFloatsFromContext(context, dimension); @@ -587,7 +591,7 @@ protected void parseCreateField(ParseContext context, int dimension, VectorDataT } final float[] array = floatsArrayOptional.get(); getVectorValidator().validateVector(array); - context.doc().addAll(getFieldsForFloatVector(array, fieldType)); + context.doc().addAll(getFieldsForFloatVector(array)); } else { throw new IllegalArgumentException( String.format(Locale.ROOT, "Cannot parse context for unsupported values provided for field [%s]", VECTOR_DATA_TYPE_FIELD) @@ -714,7 +718,6 @@ public static class Defaults { static { FIELD_TYPE.setTokenized(false); FIELD_TYPE.setIndexOptions(IndexOptions.NONE); - FIELD_TYPE.setDocValuesType(DocValuesType.BINARY); FIELD_TYPE.putAttribute(KNN_FIELD, "true"); // This attribute helps to determine knn field type FIELD_TYPE.freeze(); } diff --git a/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperUtil.java b/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperUtil.java index 0caaf80ab..9cd6bb467 100644 --- a/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperUtil.java +++ b/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperUtil.java @@ -238,6 +238,22 @@ static void validateIfKNNPluginEnabled() { } } + /** + * Prerequisite: Index should a knn index which is validated via index settings index.knn setting. This function + * assumes that caller has already validated that index is a KNN index. + * We will use LuceneKNNVectorsFormat when these below condition satisfy: + *
      + *
    1. Index is created with Version of opensearch >= 2.17
      2. + *
    2. Cluster setting is enabled to use Lucene KNNVectors format. This condition is temporary condition and will be + * removed before release.
      3. + *
    + * @param indexCreatedVersion {@link Version} + * @return true if vector field should use KNNVectorsFormat + */ + static boolean useLuceneKNNVectorsFormat(final Version indexCreatedVersion) { + return indexCreatedVersion.onOrAfter(Version.V_2_17_0) && KNNSettings.getIsLuceneVectorFormatEnabled(); + } + private static SpaceType getSpaceType(final Settings indexSettings, final VectorDataType vectorDataType) { String spaceType = indexSettings.get(KNNSettings.INDEX_KNN_SPACE_TYPE.getKey()); if (spaceType == null) { diff --git a/src/main/java/org/opensearch/knn/index/mapper/LuceneFieldMapper.java b/src/main/java/org/opensearch/knn/index/mapper/LuceneFieldMapper.java index 665c35f6e..7c3d942b6 100644 --- a/src/main/java/org/opensearch/knn/index/mapper/LuceneFieldMapper.java +++ b/src/main/java/org/opensearch/knn/index/mapper/LuceneFieldMapper.java @@ -17,7 +17,7 @@ import org.apache.lucene.document.Field; import org.apache.lucene.document.FieldType; import org.apache.lucene.document.KnnByteVectorField; -import org.apache.lucene.document.KnnVectorField; +import org.apache.lucene.document.KnnFloatVectorField; import org.apache.lucene.index.VectorSimilarityFunction; import org.opensearch.Version; import org.opensearch.common.Explicit; @@ -112,9 +112,9 @@ private LuceneFieldMapper(final KNNVectorFieldType mappedFieldType, final Create } @Override - protected List getFieldsForFloatVector(final float[] array, final FieldType fieldType) { + protected List getFieldsForFloatVector(final float[] array) { final List fieldsToBeAdded = new ArrayList<>(); - fieldsToBeAdded.add(new KnnVectorField(name(), array, fieldType)); + fieldsToBeAdded.add(new KnnFloatVectorField(name(), array, fieldType)); if (hasDocValues && vectorFieldType != null) { fieldsToBeAdded.add(new VectorField(name(), array, vectorFieldType)); @@ -127,7 +127,7 @@ protected List getFieldsForFloatVector(final float[] array, final FieldTy } @Override - protected List getFieldsForByteVector(final byte[] array, final FieldType fieldType) { + protected List getFieldsForByteVector(final byte[] array) { final List fieldsToBeAdded = new ArrayList<>(); fieldsToBeAdded.add(new KnnByteVectorField(name(), array, fieldType)); diff --git a/src/main/java/org/opensearch/knn/index/mapper/MethodFieldMapper.java b/src/main/java/org/opensearch/knn/index/mapper/MethodFieldMapper.java index 7a69c941b..cc2c43386 100644 --- a/src/main/java/org/opensearch/knn/index/mapper/MethodFieldMapper.java +++ b/src/main/java/org/opensearch/knn/index/mapper/MethodFieldMapper.java @@ -6,9 +6,12 @@ package org.opensearch.knn.index.mapper; import org.apache.lucene.document.FieldType; +import org.apache.lucene.index.DocValuesType; +import org.apache.lucene.index.VectorEncoding; import org.opensearch.Version; import org.opensearch.common.Explicit; import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.engine.KNNEngine; import org.opensearch.knn.index.engine.KNNMethodContext; @@ -99,6 +102,7 @@ private MethodFieldMapper( indexVerision, originalKNNMethodContext ); + this.useLuceneBasedVectorField = KNNVectorFieldMapperUtil.useLuceneKNNVectorsFormat(indexCreatedVersion); KNNMappingConfig annConfig = mappedFieldType.getKnnMappingConfig(); KNNMethodContext knnMethodContext = annConfig.getKnnMethodContext() .orElseThrow(() -> new IllegalArgumentException("KNN method context cannot be empty")); @@ -118,6 +122,22 @@ private MethodFieldMapper( throw new RuntimeException(String.format("Unable to create KNNVectorFieldMapper: %s", ioe)); } + if (useLuceneBasedVectorField) { + int adjustedDimension = mappedFieldType.vectorDataType == VectorDataType.BINARY + ? annConfig.getDimension() / 8 + : annConfig.getDimension(); + final VectorEncoding encoding = mappedFieldType.vectorDataType == VectorDataType.FLOAT + ? VectorEncoding.FLOAT32 + : VectorEncoding.BYTE; + fieldType.setVectorAttributes( + adjustedDimension, + encoding, + SpaceType.DEFAULT.getKnnVectorSimilarityFunction().getVectorSimilarityFunction() + ); + } else { + fieldType.setDocValuesType(DocValuesType.BINARY); + } + this.fieldType.freeze(); initValidatorsAndProcessors(knnMethodContext); knnMethodContext.getSpaceType().validateVectorDataType(vectorDataType); diff --git a/src/main/java/org/opensearch/knn/index/mapper/ModelFieldMapper.java b/src/main/java/org/opensearch/knn/index/mapper/ModelFieldMapper.java index a21a01a5d..6c7e45e7e 100644 --- a/src/main/java/org/opensearch/knn/index/mapper/ModelFieldMapper.java +++ b/src/main/java/org/opensearch/knn/index/mapper/ModelFieldMapper.java @@ -6,9 +6,12 @@ package org.opensearch.knn.index.mapper; import org.apache.lucene.document.FieldType; +import org.apache.lucene.index.DocValuesType; +import org.apache.lucene.index.VectorEncoding; import org.opensearch.Version; import org.opensearch.common.Explicit; import org.opensearch.index.mapper.ParseContext; +import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.engine.MethodComponentContext; import org.opensearch.knn.indices.ModelDao; @@ -102,7 +105,7 @@ private ModelFieldMapper( this.fieldType = new FieldType(KNNVectorFieldMapper.Defaults.FIELD_TYPE); this.fieldType.putAttribute(MODEL_ID, modelId); - this.fieldType.freeze(); + this.useLuceneBasedVectorField = KNNVectorFieldMapperUtil.useLuceneKNNVectorsFormat(this.indexCreatedVersion); } @Override @@ -193,6 +196,21 @@ private void initPerDimensionProcessor() { protected void parseCreateField(ParseContext context) throws IOException { validatePreparse(); ModelMetadata modelMetadata = getModelMetadata(modelDao, modelId); + if (useLuceneBasedVectorField) { + int adjustedDimension = modelMetadata.getVectorDataType() == VectorDataType.BINARY + ? modelMetadata.getDimension() + : modelMetadata.getDimension() / 8; + final VectorEncoding encoding = modelMetadata.getVectorDataType() == VectorDataType.FLOAT + ? VectorEncoding.FLOAT32 + : VectorEncoding.BYTE; + fieldType.setVectorAttributes( + adjustedDimension, + encoding, + SpaceType.DEFAULT.getKnnVectorSimilarityFunction().getVectorSimilarityFunction() + ); + } else { + fieldType.setDocValuesType(DocValuesType.BINARY); + } parseCreateField(context, modelMetadata.getDimension(), modelMetadata.getVectorDataType()); } diff --git a/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestCase.java b/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestCase.java index 00cc2b167..bf2c33bf9 100644 --- a/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestCase.java +++ b/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestCase.java @@ -8,7 +8,9 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import org.apache.lucene.codecs.perfield.PerFieldKnnVectorsFormat; +import org.apache.lucene.document.KnnFloatVectorField; import org.apache.lucene.document.KnnVectorField; +import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.NoMergePolicy; import org.apache.lucene.index.VectorSimilarityFunction; import org.apache.lucene.search.Query; @@ -89,8 +91,6 @@ * Test used for testing Codecs */ public class KNNCodecTestCase extends KNNTestCase { - - private static final Codec ACTUAL_CODEC = KNNCodecVersion.current().getDefaultKnnCodecSupplier().get(); private static final FieldType sampleFieldType; static { KNNMethodContext knnMethodContext = new KNNMethodContext( @@ -109,6 +109,7 @@ public class KNNCodecTestCase extends KNNTestCase { } sampleFieldType = new FieldType(KNNVectorFieldMapper.Defaults.FIELD_TYPE); + sampleFieldType.setDocValuesType(DocValuesType.BINARY); sampleFieldType.putAttribute(KNNVectorFieldMapper.KNN_FIELD, "true"); sampleFieldType.putAttribute(KNNConstants.KNN_ENGINE, knnMethodContext.getKnnEngine().getName()); sampleFieldType.putAttribute(KNNConstants.SPACE_TYPE, knnMethodContext.getSpaceType().getValue()); @@ -259,6 +260,7 @@ public void testBuildFromModelTemplate(Codec codec) throws IOException, Executio iwc.setCodec(codec); FieldType fieldType = new FieldType(KNNVectorFieldMapper.Defaults.FIELD_TYPE); + fieldType.setDocValuesType(DocValuesType.BINARY); fieldType.putAttribute(KNNConstants.MODEL_ID, modelId); fieldType.freeze(); @@ -356,9 +358,9 @@ public void testKnnVectorIndex( /** * Add doc with field "test_vector_one" */ - final FieldType luceneFieldType = KnnVectorField.createFieldType(3, VectorSimilarityFunction.EUCLIDEAN); + final FieldType luceneFieldType = KnnFloatVectorField.createFieldType(3, VectorSimilarityFunction.EUCLIDEAN); float[] array = { 1.0f, 3.0f, 4.0f }; - KnnVectorField vectorField = new KnnVectorField(FIELD_NAME_ONE, array, luceneFieldType); + KnnFloatVectorField vectorField = new KnnFloatVectorField(FIELD_NAME_ONE, array, luceneFieldType); RandomIndexWriter writer = new RandomIndexWriter(random(), dir, iwc); Document doc = new Document(); doc.add(vectorField); diff --git a/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java b/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java index f06ff7935..e1d842112 100644 --- a/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java +++ b/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java @@ -7,11 +7,14 @@ import com.google.common.collect.ImmutableMap; import lombok.SneakyThrows; +import lombok.extern.log4j.Log4j2; import org.apache.lucene.document.KnnByteVectorField; +import org.apache.lucene.document.KnnFloatVectorField; import org.apache.lucene.document.KnnVectorField; import org.apache.lucene.index.IndexableField; import org.apache.lucene.index.VectorEncoding; import org.apache.lucene.util.BytesRef; +import org.mockito.MockedStatic; import org.mockito.Mockito; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.Explicit; @@ -27,14 +30,14 @@ import org.opensearch.index.mapper.MapperService; import org.opensearch.index.mapper.ParseContext; import org.opensearch.knn.KNNTestCase; -import org.opensearch.knn.index.engine.KNNMethodContext; import org.opensearch.knn.index.KNNSettings; -import org.opensearch.knn.index.engine.MethodComponentContext; import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.VectorField; import org.opensearch.knn.index.codec.util.KNNVectorSerializerFactory; import org.opensearch.knn.index.engine.KNNEngine; +import org.opensearch.knn.index.engine.KNNMethodContext; +import org.opensearch.knn.index.engine.MethodComponentContext; import org.opensearch.knn.indices.ModelDao; import org.opensearch.knn.indices.ModelMetadata; import org.opensearch.knn.indices.ModelState; @@ -79,6 +82,7 @@ import static org.opensearch.knn.index.mapper.KNNVectorFieldMapperUtil.clipVectorValueToFP16Range; import static org.opensearch.knn.index.mapper.KNNVectorFieldMapperUtil.validateFP16VectorValue; +@Log4j2 public class KNNVectorFieldMapperTests extends KNNTestCase { private static final String TEST_FIELD_NAME = "test-field-name"; @@ -739,6 +743,112 @@ public void testKNNVectorFieldMapper_merge_fromModel() throws IOException { expectThrows(IllegalArgumentException.class, () -> knnVectorFieldMapper1.merge(knnVectorFieldMapper3)); } + @SneakyThrows + public void testMethodFieldMapperParseCreateField_validInput_thenDifferentFieldTypes() { + MockedStatic utilMockedStatic = Mockito.mockStatic(KNNVectorFieldMapperUtil.class); + for (VectorDataType dataType : VectorDataType.values()) { + log.info("Vector Data Type is : {}", dataType); + int dimension = dataType == VectorDataType.BINARY ? TEST_DIMENSION * 8 : TEST_DIMENSION; + final MethodComponentContext methodComponentContext = new MethodComponentContext(METHOD_HNSW, Collections.emptyMap()); + methodComponentContext.setIndexVersion(CURRENT); + SpaceType spaceType = VectorDataType.BINARY == dataType ? SpaceType.DEFAULT_BINARY : SpaceType.INNER_PRODUCT; + final KNNMethodContext knnMethodContext = new KNNMethodContext(KNNEngine.FAISS, spaceType, methodComponentContext); + + ParseContext.Document document = new ParseContext.Document(); + ContentPath contentPath = new ContentPath(); + ParseContext parseContext = mock(ParseContext.class); + when(parseContext.doc()).thenReturn(document); + when(parseContext.path()).thenReturn(contentPath); + + utilMockedStatic.when(() -> KNNVectorFieldMapperUtil.useLuceneKNNVectorsFormat(Mockito.any())).thenReturn(true); + MethodFieldMapper methodFieldMapper = Mockito.spy( + MethodFieldMapper.createFieldMapper( + TEST_FIELD_NAME, + TEST_FIELD_NAME, + Collections.emptyMap(), + dataType, + dimension, + knnMethodContext, + knnMethodContext, + FieldMapper.MultiFields.empty(), + FieldMapper.CopyTo.empty(), + new Explicit<>(true, true), + false, + false, + CURRENT + ) + ); + + if (dataType == VectorDataType.BINARY) { + doReturn(Optional.of(TEST_BYTE_VECTOR)).when(methodFieldMapper) + .getBytesFromContext(parseContext, TEST_DIMENSION * 8, dataType); + } else if (dataType == VectorDataType.BYTE) { + doReturn(Optional.of(TEST_BYTE_VECTOR)).when(methodFieldMapper).getBytesFromContext(parseContext, TEST_DIMENSION, dataType); + } else { + doReturn(Optional.of(TEST_VECTOR)).when(methodFieldMapper).getFloatsFromContext(parseContext, TEST_DIMENSION); + } + + methodFieldMapper.parseCreateField(parseContext, dimension, dataType); + + List fields = document.getFields(); + assertEquals(1, fields.size()); + IndexableField field1 = fields.get(0); + if (dataType == VectorDataType.FLOAT) { + assertTrue(field1 instanceof KnnFloatVectorField); + assertEquals(field1.fieldType().vectorEncoding(), VectorEncoding.FLOAT32); + } else { + assertTrue(field1 instanceof KnnByteVectorField); + assertEquals(field1.fieldType().vectorEncoding(), VectorEncoding.BYTE); + } + + assertEquals(field1.fieldType().vectorDimension(), TEST_DIMENSION); + assertEquals( + field1.fieldType().vectorSimilarityFunction(), + SpaceType.DEFAULT.getKnnVectorSimilarityFunction().getVectorSimilarityFunction() + ); + + utilMockedStatic.when(() -> KNNVectorFieldMapperUtil.useLuceneKNNVectorsFormat(Mockito.any())).thenReturn(false); + + document = new ParseContext.Document(); + contentPath = new ContentPath(); + when(parseContext.doc()).thenReturn(document); + when(parseContext.path()).thenReturn(contentPath); + methodFieldMapper = Mockito.spy( + MethodFieldMapper.createFieldMapper( + TEST_FIELD_NAME, + TEST_FIELD_NAME, + Collections.emptyMap(), + dataType, + dimension, + knnMethodContext, + knnMethodContext, + FieldMapper.MultiFields.empty(), + FieldMapper.CopyTo.empty(), + new Explicit<>(true, true), + false, + false, + CURRENT + ) + ); + + if (dataType == VectorDataType.FLOAT) { + doReturn(Optional.of(TEST_VECTOR)).when(methodFieldMapper).getFloatsFromContext(parseContext, TEST_DIMENSION); + } else { + doReturn(Optional.of(TEST_BYTE_VECTOR)).when(methodFieldMapper) + .getBytesFromContext(parseContext, dataType == VectorDataType.BINARY ? TEST_DIMENSION * 8 : TEST_DIMENSION, dataType); + } + + methodFieldMapper.parseCreateField(parseContext, dimension, dataType); + fields = document.getFields(); + assertEquals(1, fields.size()); + field1 = fields.get(0); + assertTrue(field1 instanceof VectorField); + } + // making sure to close the static mock to ensure that for tests running on this thread are not impacted by + // this mocking + utilMockedStatic.close(); + } + @SneakyThrows public void testLuceneFieldMapper_parseCreateField_docValues_withFloats() { // Create a lucene field mapper that creates a binary doc values field as well as KnnVectorField @@ -765,22 +875,22 @@ public void testLuceneFieldMapper_parseCreateField_docValues_withFloats() { doNothing().when(luceneFieldMapper).validatePreparse(); luceneFieldMapper.parseCreateField(parseContext, TEST_DIMENSION, VectorDataType.FLOAT); - // Document should have 2 fields: one for VectorField (binary doc values) and one for KnnVectorField + // Document should have 2 fields: one for VectorField (binary doc values) and one for KnnFloatVectorField List fields = document.getFields(); assertEquals(2, fields.size()); IndexableField field1 = fields.get(0); IndexableField field2 = fields.get(1); VectorField vectorField; - KnnVectorField knnVectorField; + KnnFloatVectorField knnVectorField; if (field1 instanceof VectorField) { assertTrue(field2 instanceof KnnVectorField); vectorField = (VectorField) field1; - knnVectorField = (KnnVectorField) field2; + knnVectorField = (KnnFloatVectorField) field2; } else { - assertTrue(field1 instanceof KnnVectorField); + assertTrue(field1 instanceof KnnFloatVectorField); assertTrue(field2 instanceof VectorField); - knnVectorField = (KnnVectorField) field1; + knnVectorField = (KnnFloatVectorField) field1; vectorField = (VectorField) field2; } @@ -821,8 +931,8 @@ public void testLuceneFieldMapper_parseCreateField_docValues_withFloats() { fields = document.getFields(); assertEquals(1, fields.size()); IndexableField field = fields.get(0); - assertTrue(field instanceof KnnVectorField); - knnVectorField = (KnnVectorField) field; + assertTrue(field instanceof KnnFloatVectorField); + knnVectorField = (KnnFloatVectorField) field; assertArrayEquals(TEST_VECTOR, knnVectorField.vectorValue(), 0.001f); } diff --git a/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperUtilTests.java b/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperUtilTests.java index 8ace5557e..ad041e47e 100644 --- a/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperUtilTests.java +++ b/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperUtilTests.java @@ -13,8 +13,13 @@ import org.apache.lucene.document.StoredField; import org.apache.lucene.util.BytesRef; +import org.junit.Assert; +import org.mockito.MockedStatic; +import org.mockito.Mockito; +import org.opensearch.Version; import org.opensearch.knn.KNNTestCase; import org.opensearch.knn.common.KNNConstants; +import org.opensearch.knn.index.KNNSettings; import org.opensearch.knn.index.engine.KNNMethodContext; import org.opensearch.knn.index.engine.MethodComponentContext; import org.opensearch.knn.index.SpaceType; @@ -105,6 +110,22 @@ public void testValidateVectorDataType_whenFloat_thenValid() { validateValidateVectorDataType(KNNEngine.NMSLIB, KNNConstants.METHOD_HNSW, VectorDataType.FLOAT, null); } + public void testUseLuceneKNNVectorsFormat_withDifferentInputs_thenSuccess() { + final KNNSettings knnSettings = mock(KNNSettings.class); + final MockedStatic mockedStatic = Mockito.mockStatic(KNNSettings.class); + mockedStatic.when(KNNSettings::state).thenReturn(knnSettings); + + mockedStatic.when(KNNSettings::getIsLuceneVectorFormatEnabled).thenReturn(false); + Assert.assertFalse(KNNVectorFieldMapperUtil.useLuceneKNNVectorsFormat(Version.V_2_16_0)); + Assert.assertFalse(KNNVectorFieldMapperUtil.useLuceneKNNVectorsFormat(Version.V_2_17_0)); + + mockedStatic.when(KNNSettings::getIsLuceneVectorFormatEnabled).thenReturn(true); + Assert.assertTrue(KNNVectorFieldMapperUtil.useLuceneKNNVectorsFormat(Version.V_2_17_0)); + // making sure to close the static mock to ensure that for tests running on this thread are not impacted by + // this mocking + mockedStatic.close(); + } + private void validateValidateVectorDataType( final KNNEngine knnEngine, final String methodName, From fc64e307de2798d730cedd9cd24417fda156a8b8 Mon Sep 17 00:00:00 2001 From: Vikasht34 Date: Tue, 13 Aug 2024 15:11:54 -0700 Subject: [PATCH 323/416] [Backport 2.x] Quantization Framework Implementation with 1bit and MultiBit Binary Quantizer (#1955) * Quantization Framework Implementation with 1bit and MultiBit Binary Quantizer (#1929) * Quantization Framework Implementation with 1bit and MultiBit Binary Quantizer Signed-off-by: VIKASH TIWARI * Quantization Framework Implementation with 1bit and MultiBit Binary Quantizer Signed-off-by: VIKASH TIWARI * Implemented Serlization using Writable Signed-off-by: VIKASH TIWARI --------- Signed-off-by: VIKASH TIWARI Signed-off-by: Vikasht34 * Quantization Framework Implementation with 1bit and MultiBit Binary Quantizer Signed-off-by: VIKASH TIWARI --------- Signed-off-by: VIKASH TIWARI Signed-off-by: Vikasht34 --- CHANGELOG.md | 1 + .../enums/ScalarQuantizationType.java | 62 ++++++ .../factory/QuantizerFactory.java | 54 +++++ .../factory/QuantizerRegistrar.java | 46 +++++ .../factory/QuantizerRegistry.java | 59 ++++++ .../BinaryQuantizationOutput.java | 67 +++++++ .../QuantizationOutput.java | 28 +++ .../QuantizationParams.java | 27 +++ .../ScalarQuantizationParams.java | 77 ++++++++ .../DefaultQuantizationState.java | 67 +++++++ .../MultiBitScalarQuantizationState.java | 127 ++++++++++++ .../OneBitScalarQuantizationState.java | 110 +++++++++++ .../quantizationState/QuantizationState.java | 32 +++ .../QuantizationStateSerializer.java | 56 ++++++ .../models/requests/TrainingRequest.java | 31 +++ .../knn/quantization/quantizer/BitPacker.java | 143 ++++++++++++++ .../quantizer/MultiBitScalarQuantizer.java | 186 ++++++++++++++++++ .../quantizer/OneBitScalarQuantizer.java | 100 ++++++++++ .../knn/quantization/quantizer/Quantizer.java | 40 ++++ .../quantizer/QuantizerHelper.java | 84 ++++++++ .../sampler/ReservoirSampler.java | 90 +++++++++ .../knn/quantization/sampler/Sampler.java | 25 +++ .../knn/quantization/sampler/SamplerType.java | 14 ++ .../quantization/sampler/SamplingFactory.java | 34 ++++ .../enums/ScalarQuantizationTypeTests.java | 35 ++++ .../factory/QuantizerFactoryTests.java | 63 ++++++ .../factory/QuantizerRegistryTests.java | 84 ++++++++ .../QuantizationStateSerializerTests.java | 46 +++++ .../QuantizationStateTests.java | 68 +++++++ .../MultiBitScalarQuantizerTests.java | 107 ++++++++++ .../quantizer/OneBitScalarQuantizerTests.java | 136 +++++++++++++ .../sampler/ReservoirSamplerTests.java | 63 ++++++ .../sampler/SamplingFactoryTests.java | 19 ++ 33 files changed, 2181 insertions(+) create mode 100644 src/main/java/org/opensearch/knn/quantization/enums/ScalarQuantizationType.java create mode 100644 src/main/java/org/opensearch/knn/quantization/factory/QuantizerFactory.java create mode 100644 src/main/java/org/opensearch/knn/quantization/factory/QuantizerRegistrar.java create mode 100644 src/main/java/org/opensearch/knn/quantization/factory/QuantizerRegistry.java create mode 100644 src/main/java/org/opensearch/knn/quantization/models/quantizationOutput/BinaryQuantizationOutput.java create mode 100644 src/main/java/org/opensearch/knn/quantization/models/quantizationOutput/QuantizationOutput.java create mode 100644 src/main/java/org/opensearch/knn/quantization/models/quantizationParams/QuantizationParams.java create mode 100644 src/main/java/org/opensearch/knn/quantization/models/quantizationParams/ScalarQuantizationParams.java create mode 100644 src/main/java/org/opensearch/knn/quantization/models/quantizationState/DefaultQuantizationState.java create mode 100644 src/main/java/org/opensearch/knn/quantization/models/quantizationState/MultiBitScalarQuantizationState.java create mode 100644 src/main/java/org/opensearch/knn/quantization/models/quantizationState/OneBitScalarQuantizationState.java create mode 100644 src/main/java/org/opensearch/knn/quantization/models/quantizationState/QuantizationState.java create mode 100644 src/main/java/org/opensearch/knn/quantization/models/quantizationState/QuantizationStateSerializer.java create mode 100644 src/main/java/org/opensearch/knn/quantization/models/requests/TrainingRequest.java create mode 100644 src/main/java/org/opensearch/knn/quantization/quantizer/BitPacker.java create mode 100644 src/main/java/org/opensearch/knn/quantization/quantizer/MultiBitScalarQuantizer.java create mode 100644 src/main/java/org/opensearch/knn/quantization/quantizer/OneBitScalarQuantizer.java create mode 100644 src/main/java/org/opensearch/knn/quantization/quantizer/Quantizer.java create mode 100644 src/main/java/org/opensearch/knn/quantization/quantizer/QuantizerHelper.java create mode 100644 src/main/java/org/opensearch/knn/quantization/sampler/ReservoirSampler.java create mode 100644 src/main/java/org/opensearch/knn/quantization/sampler/Sampler.java create mode 100644 src/main/java/org/opensearch/knn/quantization/sampler/SamplerType.java create mode 100644 src/main/java/org/opensearch/knn/quantization/sampler/SamplingFactory.java create mode 100644 src/test/java/org/opensearch/knn/quantization/enums/ScalarQuantizationTypeTests.java create mode 100644 src/test/java/org/opensearch/knn/quantization/factory/QuantizerFactoryTests.java create mode 100644 src/test/java/org/opensearch/knn/quantization/factory/QuantizerRegistryTests.java create mode 100644 src/test/java/org/opensearch/knn/quantization/quantizationState/QuantizationStateSerializerTests.java create mode 100644 src/test/java/org/opensearch/knn/quantization/quantizationState/QuantizationStateTests.java create mode 100644 src/test/java/org/opensearch/knn/quantization/quantizer/MultiBitScalarQuantizerTests.java create mode 100644 src/test/java/org/opensearch/knn/quantization/quantizer/OneBitScalarQuantizerTests.java create mode 100644 src/test/java/org/opensearch/knn/quantization/sampler/ReservoirSamplerTests.java create mode 100644 src/test/java/org/opensearch/knn/quantization/sampler/SamplingFactoryTests.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 5e3060530..f53b32a1b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,3 +32,4 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), * Generalize lib interface to return context objects [#1925](https://github.com/opensearch-project/k-NN/pull/1925) * Move k search k-NN query to re-write phase of vector search query for Native Engines [#1877](https://github.com/opensearch-project/k-NN/pull/1877) * Restructure mappers to better handle null cases and avoid branching in parsing [#1939](https://github.com/opensearch-project/k-NN/pull/1939) +* Added Quantization Framework and implemented 1Bit and multibit quantizer[#1889](https://github.com/opensearch-project/k-NN/issues/1889) diff --git a/src/main/java/org/opensearch/knn/quantization/enums/ScalarQuantizationType.java b/src/main/java/org/opensearch/knn/quantization/enums/ScalarQuantizationType.java new file mode 100644 index 000000000..40347ad93 --- /dev/null +++ b/src/main/java/org/opensearch/knn/quantization/enums/ScalarQuantizationType.java @@ -0,0 +1,62 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.quantization.enums; + +import lombok.Getter; + +/** + * The ScalarQuantizationType enum defines the various scalar quantization types that can be used + * for vector quantization. Each type corresponds to a different bit-width representation of the quantized values. + * + *

    + * Future Developers: If you change the name of any enum constant, do not change its associated value. + * Serialization and deserialization depend on these values to maintain compatibility. + *

    + */ +@Getter +public enum ScalarQuantizationType { + /** + * ONE_BIT quantization uses a single bit per coordinate. + */ + ONE_BIT(1), + + /** + * TWO_BIT quantization uses two bits per coordinate. + */ + TWO_BIT(2), + + /** + * FOUR_BIT quantization uses four bits per coordinate. + */ + FOUR_BIT(4); + + private final int id; + + /** + * Constructs a ScalarQuantizationType with the specified ID. + * + * @param id the ID representing the quantization type. + */ + ScalarQuantizationType(int id) { + this.id = id; + } + + /** + * Returns the ScalarQuantizationType associated with the given ID. + * + * @param id the ID of the quantization type. + * @return the corresponding ScalarQuantizationType. + * @throws IllegalArgumentException if the ID does not correspond to any ScalarQuantizationType. + */ + public static ScalarQuantizationType fromId(int id) { + for (ScalarQuantizationType type : ScalarQuantizationType.values()) { + if (type.getId() == id) { + return type; + } + } + throw new IllegalArgumentException("Unknown ScalarQuantizationType ID: " + id); + } +} diff --git a/src/main/java/org/opensearch/knn/quantization/factory/QuantizerFactory.java b/src/main/java/org/opensearch/knn/quantization/factory/QuantizerFactory.java new file mode 100644 index 000000000..b99f6ebdc --- /dev/null +++ b/src/main/java/org/opensearch/knn/quantization/factory/QuantizerFactory.java @@ -0,0 +1,54 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.quantization.factory; + +import lombok.AccessLevel; +import lombok.NoArgsConstructor; +import org.opensearch.knn.quantization.models.quantizationParams.QuantizationParams; +import org.opensearch.knn.quantization.quantizer.Quantizer; + +import java.util.concurrent.atomic.AtomicBoolean; + +/** + * The QuantizerFactory class is responsible for creating instances of {@link Quantizer} + * based on the provided {@link QuantizationParams}. It uses a registry to look up the + * appropriate quantizer implementation for the given quantization parameters. + */ +@NoArgsConstructor(access = AccessLevel.PRIVATE) +public final class QuantizerFactory { + private static final AtomicBoolean isRegistered = new AtomicBoolean(false); + + /** + * Ensures that default quantizers are registered. + */ + private static void ensureRegistered() { + if (!isRegistered.get()) { + synchronized (QuantizerFactory.class) { + if (!isRegistered.get()) { + QuantizerRegistrar.registerDefaultQuantizers(); + isRegistered.set(true); + } + } + } + } + + /** + * Retrieves a quantizer instance based on the provided quantization parameters. + * + * @param params the quantization parameters used to determine the appropriate quantizer + * @param

    the type of quantization parameters, extending {@link QuantizationParams} + * @param the type of the quantized output + * @return an instance of {@link Quantizer} corresponding to the provided parameters + */ + public static

    Quantizer getQuantizer(final P params) { + if (params == null) { + throw new IllegalArgumentException("Quantization parameters must not be null."); + } + // Lazy Registration instead of static block as class level; + ensureRegistered(); + return QuantizerRegistry.getQuantizer(params); + } +} diff --git a/src/main/java/org/opensearch/knn/quantization/factory/QuantizerRegistrar.java b/src/main/java/org/opensearch/knn/quantization/factory/QuantizerRegistrar.java new file mode 100644 index 000000000..7b542aea0 --- /dev/null +++ b/src/main/java/org/opensearch/knn/quantization/factory/QuantizerRegistrar.java @@ -0,0 +1,46 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.quantization.factory; + +import lombok.AccessLevel; +import lombok.NoArgsConstructor; +import org.opensearch.knn.quantization.enums.ScalarQuantizationType; +import org.opensearch.knn.quantization.models.quantizationParams.ScalarQuantizationParams; +import org.opensearch.knn.quantization.quantizer.MultiBitScalarQuantizer; +import org.opensearch.knn.quantization.quantizer.OneBitScalarQuantizer; + +/** + * The QuantizerRegistrar class is responsible for registering default quantizers. + * This class ensures that the registration happens only once in a thread-safe manner. + */ +@NoArgsConstructor(access = AccessLevel.PRIVATE) +final class QuantizerRegistrar { + + /** + * Registers default quantizers + *

    + * This method is synchronized to ensure that registration occurs only once, + * even in a multi-threaded environment. + *

    + */ + static synchronized void registerDefaultQuantizers() { + // Register OneBitScalarQuantizer for SQParams with VALUE_QUANTIZATION and SQTypes.ONE_BIT + QuantizerRegistry.register( + ScalarQuantizationParams.generateTypeIdentifier(ScalarQuantizationType.ONE_BIT), + new OneBitScalarQuantizer() + ); + // Register MultiBitScalarQuantizer for SQParams with VALUE_QUANTIZATION with bit per co-ordinate = 2 + QuantizerRegistry.register( + ScalarQuantizationParams.generateTypeIdentifier(ScalarQuantizationType.TWO_BIT), + new MultiBitScalarQuantizer(2) + ); + // Register MultiBitScalarQuantizer for SQParams with VALUE_QUANTIZATION with bit per co-ordinate = 4 + QuantizerRegistry.register( + ScalarQuantizationParams.generateTypeIdentifier(ScalarQuantizationType.FOUR_BIT), + new MultiBitScalarQuantizer(4) + ); + } +} diff --git a/src/main/java/org/opensearch/knn/quantization/factory/QuantizerRegistry.java b/src/main/java/org/opensearch/knn/quantization/factory/QuantizerRegistry.java new file mode 100644 index 000000000..ac266f547 --- /dev/null +++ b/src/main/java/org/opensearch/knn/quantization/factory/QuantizerRegistry.java @@ -0,0 +1,59 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.quantization.factory; + +import lombok.AccessLevel; +import lombok.NoArgsConstructor; +import org.opensearch.knn.quantization.models.quantizationParams.QuantizationParams; +import org.opensearch.knn.quantization.quantizer.Quantizer; + +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +/** + * The QuantizerRegistry class is responsible for managing the registration and retrieval + * of quantizer instances. Quantizers are registered with specific quantization parameters + * and type identifiers, allowing for efficient lookup and instantiation. + */ +@NoArgsConstructor(access = AccessLevel.PRIVATE) +final class QuantizerRegistry { + // ConcurrentHashMap for thread-safe access + private static final Map> registry = new ConcurrentHashMap<>(); + + /** + * Registers a quantizer with the registry. + * + * @param paramIdentifier the unique identifier for the quantization parameters + * @param quantizer an instance of the quantizer + */ + static void register(final String paramIdentifier, final Quantizer quantizer) { + // Check if the quantizer is already registered for the given identifier + if (registry.putIfAbsent(paramIdentifier, quantizer) != null) { + // Throw an exception if a quantizer is already registered + throw new IllegalArgumentException("Quantizer already registered for identifier: " + paramIdentifier); + } + } + + /** + * Retrieves a quantizer instance based on the provided quantization parameters. + * + * @param params the quantization parameters used to determine the appropriate quantizer + * @param

    the type of quantization parameters + * @param the type of the quantized output + * @return an instance of {@link Quantizer} corresponding to the provided parameters + * @throws IllegalArgumentException if no quantizer is registered for the given parameters + */ + static

    Quantizer getQuantizer(final P params) { + String identifier = params.getTypeIdentifier(); + Quantizer quantizer = registry.get(identifier); + if (quantizer == null) { + throw new IllegalArgumentException("No quantizer registered for type identifier: " + identifier); + } + @SuppressWarnings("unchecked") + Quantizer typedQuantizer = (Quantizer) quantizer; + return typedQuantizer; + } +} diff --git a/src/main/java/org/opensearch/knn/quantization/models/quantizationOutput/BinaryQuantizationOutput.java b/src/main/java/org/opensearch/knn/quantization/models/quantizationOutput/BinaryQuantizationOutput.java new file mode 100644 index 000000000..95592fcb9 --- /dev/null +++ b/src/main/java/org/opensearch/knn/quantization/models/quantizationOutput/BinaryQuantizationOutput.java @@ -0,0 +1,67 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.quantization.models.quantizationOutput; + +import lombok.Getter; +import lombok.NoArgsConstructor; + +import java.util.Arrays; + +/** + * The BinaryQuantizationOutput class represents the output of a quantization process in binary format. + * It implements the QuantizationOutput interface to handle byte arrays specifically. + */ +@NoArgsConstructor +public class BinaryQuantizationOutput implements QuantizationOutput { + @Getter + private byte[] quantizedVector; + + /** + * Prepares the quantized vector array based on the provided parameters and returns it for direct modification. + * This method ensures that the internal byte array is appropriately sized and cleared before being used. + * The method accepts two parameters: + *

      + *
    • bitsPerCoordinate: The number of bits used per coordinate. This determines the granularity of the quantization.
      • + *
    • vectorLength: The length of the original vector that needs to be quantized. This helps in calculating the required byte array size.
      • + *
    + * If the existing quantized vector is either null or not the same size as the required byte array, + * a new byte array is allocated. Otherwise, the existing array is cleared (i.e., all bytes are set to zero). + * This method is designed to be used in conjunction with a bit-packing utility that writes quantized values directly + * into the returned byte array. + * @param params an array of parameters, where the first parameter is the number of bits per coordinate (int), + * and the second parameter is the length of the vector (int). + * @return the prepared and writable quantized vector as a byte array. + * @throws IllegalArgumentException if the parameters are not as expected (e.g., missing or not integers). + */ + @Override + public byte[] prepareAndGetWritableQuantizedVector(Object... params) { + if (params.length != 2 || !(params[0] instanceof Integer) || !(params[1] instanceof Integer)) { + throw new IllegalArgumentException("Expected two integer parameters: bitsPerCoordinate and vectorLength"); + } + int bitsPerCoordinate = (int) params[0]; + int vectorLength = (int) params[1]; + int totalBits = bitsPerCoordinate * vectorLength; + int byteLength = (totalBits + 7) >> 3; + + if (this.quantizedVector == null || this.quantizedVector.length != byteLength) { + this.quantizedVector = new byte[byteLength]; + } else { + Arrays.fill(this.quantizedVector, (byte) 0); + } + + return this.quantizedVector; + } + + /** + * Returns the quantized vector. + * + * @return the quantized vector byte array. + */ + @Override + public byte[] getQuantizedVector() { + return quantizedVector; + } +} diff --git a/src/main/java/org/opensearch/knn/quantization/models/quantizationOutput/QuantizationOutput.java b/src/main/java/org/opensearch/knn/quantization/models/quantizationOutput/QuantizationOutput.java new file mode 100644 index 000000000..aa81a8821 --- /dev/null +++ b/src/main/java/org/opensearch/knn/quantization/models/quantizationOutput/QuantizationOutput.java @@ -0,0 +1,28 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.quantization.models.quantizationOutput; + +/** + * The QuantizationOutput interface defines the contract for quantization output data. + * + * @param The type of the quantized data. + */ +public interface QuantizationOutput { + /** + * Returns the quantized vector. + * + * @return the quantized data. + */ + T getQuantizedVector(); + + /** + * Prepares and returns the writable quantized vector for direct modification. + * + * @param params the parameters needed for preparing the quantized vector. + * @return the prepared and writable quantized vector. + */ + T prepareAndGetWritableQuantizedVector(Object... params); +} diff --git a/src/main/java/org/opensearch/knn/quantization/models/quantizationParams/QuantizationParams.java b/src/main/java/org/opensearch/knn/quantization/models/quantizationParams/QuantizationParams.java new file mode 100644 index 000000000..4f2ee36c5 --- /dev/null +++ b/src/main/java/org/opensearch/knn/quantization/models/quantizationParams/QuantizationParams.java @@ -0,0 +1,27 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.quantization.models.quantizationParams; + +import org.opensearch.core.common.io.stream.Writeable; + +/** + * Interface for quantization parameters. + * This interface defines the basic contract for all quantization parameter types. + * It provides methods to retrieve the quantization type and a unique type identifier. + * Implementations of this interface are expected to provide specific configurations + * for various quantization strategies. + */ +public interface QuantizationParams extends Writeable { + /** + * Provides a unique identifier for the quantization parameters. + * This identifier is typically a combination of the quantization type + * and additional specifics, and it serves to distinguish between different + * configurations or modes of quantization. + * + * @return a string representing the unique type identifier. + */ + String getTypeIdentifier(); +} diff --git a/src/main/java/org/opensearch/knn/quantization/models/quantizationParams/ScalarQuantizationParams.java b/src/main/java/org/opensearch/knn/quantization/models/quantizationParams/ScalarQuantizationParams.java new file mode 100644 index 000000000..4e7a53892 --- /dev/null +++ b/src/main/java/org/opensearch/knn/quantization/models/quantizationParams/ScalarQuantizationParams.java @@ -0,0 +1,77 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.quantization.models.quantizationParams; + +import lombok.AllArgsConstructor; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.NoArgsConstructor; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.knn.quantization.enums.ScalarQuantizationType; + +import java.io.IOException; + +/** + * The ScalarQuantizationParams class represents the parameters specific to scalar quantization (SQ). + * This class implements the QuantizationParams interface and includes the type of scalar quantization. + */ +@Getter +@AllArgsConstructor +@NoArgsConstructor // No-argument constructor for deserialization +@EqualsAndHashCode +public class ScalarQuantizationParams implements QuantizationParams { + private ScalarQuantizationType sqType; + + /** + * Static method to generate type identifier based on ScalarQuantizationType. + * + * @param sqType the scalar quantization type. + * @return A string representing the unique type identifier. + */ + public static String generateTypeIdentifier(ScalarQuantizationType sqType) { + return generateIdentifier(sqType.getId()); + } + + /** + * Provides a unique type identifier for the ScalarQuantizationParams, combining the SQ type. + * This identifier is useful for distinguishing between different configurations of scalar quantization parameters. + * + * @return A string representing the unique type identifier. + */ + @Override + public String getTypeIdentifier() { + return generateIdentifier(sqType.getId()); + } + + private static String generateIdentifier(int id) { + return "ScalarQuantizationParams_" + id; + } + + /** + * Writes the object to the output stream. + * This method is part of the Writeable interface and is used to serialize the object. + * + * @param out the output stream to write the object to. + * @throws IOException if an I/O error occurs. + */ + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeVInt(sqType.getId()); + } + + /** + * Reads the object from the input stream. + * This method is part of the Writeable interface and is used to deserialize the object. + * + * @param in the input stream to read the object from. + * @throws IOException if an I/O error occurs. + */ + public ScalarQuantizationParams(StreamInput in, int version) throws IOException { + int typeId = in.readVInt(); + this.sqType = ScalarQuantizationType.fromId(typeId); + } +} diff --git a/src/main/java/org/opensearch/knn/quantization/models/quantizationState/DefaultQuantizationState.java b/src/main/java/org/opensearch/knn/quantization/models/quantizationState/DefaultQuantizationState.java new file mode 100644 index 000000000..33e775cad --- /dev/null +++ b/src/main/java/org/opensearch/knn/quantization/models/quantizationState/DefaultQuantizationState.java @@ -0,0 +1,67 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.quantization.models.quantizationState; + +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.NoArgsConstructor; +import org.opensearch.Version; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.knn.quantization.models.quantizationParams.QuantizationParams; +import org.opensearch.knn.quantization.models.quantizationParams.ScalarQuantizationParams; + +import java.io.IOException; + +/** + * DefaultQuantizationState is used as a fallback state when no training is required or if training fails. + * It can be utilized by any quantizer to represent a default state. + */ +@Getter +@NoArgsConstructor // No-argument constructor for deserialization +@AllArgsConstructor +public class DefaultQuantizationState implements QuantizationState { + private QuantizationParams params; + + @Override + public QuantizationParams getQuantizationParams() { + return params; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeInt(Version.CURRENT.id); // Write the version + params.writeTo(out); + } + + public DefaultQuantizationState(StreamInput in) throws IOException { + int version = in.readInt(); // Read the version + this.params = new ScalarQuantizationParams(in, version); + } + + /** + * Serializes the quantization state to a byte array. + * + * @return a byte array representing the serialized state. + * @throws IOException if an I/O error occurs during serialization. + */ + @Override + public byte[] toByteArray() throws IOException { + return QuantizationStateSerializer.serialize(this); + } + + /** + * Deserializes a DefaultQuantizationState from a byte array. + * + * @param bytes the byte array containing the serialized state. + * @return the deserialized DefaultQuantizationState. + * @throws IOException if an I/O error occurs during deserialization. + * @throws ClassNotFoundException if the class of the serialized object cannot be found. + */ + public static DefaultQuantizationState fromByteArray(final byte[] bytes) throws IOException, ClassNotFoundException { + return (DefaultQuantizationState) QuantizationStateSerializer.deserialize(bytes, DefaultQuantizationState::new); + } +} diff --git a/src/main/java/org/opensearch/knn/quantization/models/quantizationState/MultiBitScalarQuantizationState.java b/src/main/java/org/opensearch/knn/quantization/models/quantizationState/MultiBitScalarQuantizationState.java new file mode 100644 index 000000000..2778a6cf4 --- /dev/null +++ b/src/main/java/org/opensearch/knn/quantization/models/quantizationState/MultiBitScalarQuantizationState.java @@ -0,0 +1,127 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.quantization.models.quantizationState; + +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.NoArgsConstructor; +import org.opensearch.Version; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.knn.quantization.models.quantizationParams.ScalarQuantizationParams; + +import java.io.IOException; + +/** + * MultiBitScalarQuantizationState represents the state of multi-bit scalar quantization, + * including the thresholds used for quantization. + */ +@Getter +@NoArgsConstructor // No-argument constructor for deserialization +@AllArgsConstructor +public final class MultiBitScalarQuantizationState implements QuantizationState { + private ScalarQuantizationParams quantizationParams; + /** + * The threshold values for multi-bit quantization, organized as a 2D array + * where each row corresponds to a different bit level. + * + * For example: + * - For 2-bit quantization: + * thresholds[0] {0.5f, 1.5f, 2.5f} // Thresholds for the first bit level + * thresholds[1] {1.0f, 2.0f, 3.0f} // Thresholds for the second bit level + * - For 4-bit quantization: + * thresholds[0] {0.1f, 0.2f, 0.3f} + * thresholds[1] {0.4f, 0.5f, 0.6f} + * thresholds[2] {0.7f, 0.8f, 0.9f} + * thresholds[3] {1.0f, 1.1f, 1.2f} + * + * Each column represents the threshold for a specific dimension in the vector space. + */ + private float[][] thresholds; + + @Override + public ScalarQuantizationParams getQuantizationParams() { + return quantizationParams; + } + + /** + * This method is responsible for writing the state of the MultiBitScalarQuantizationState object to an external output. + * It includes versioning information to ensure compatibility between different versions of the serialized object. + * + * @param out the StreamOutput to write the object to. + * @throws IOException if an I/O error occurs during serialization. + */ + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeVInt(Version.CURRENT.id); // Write the version + quantizationParams.writeTo(out); + out.writeVInt(thresholds.length); // Number of rows + for (float[] row : thresholds) { + out.writeFloatArray(row); // Write each row as a float array + } + } + + /** + * This method is responsible for reading the state of the MultiBitScalarQuantizationState object from an external input. + * It includes versioning information to ensure compatibility between different versions of the serialized object. + * + * @param in the StreamInput to read the object from. + * @throws IOException if an I/O error occurs during deserialization. + */ + public MultiBitScalarQuantizationState(StreamInput in) throws IOException { + int version = in.readVInt(); // Read the version + this.quantizationParams = new ScalarQuantizationParams(in, version); + int rows = in.readVInt(); // Read the number of rows + this.thresholds = new float[rows][]; + for (int i = 0; i < rows; i++) { + this.thresholds[i] = in.readFloatArray(); // Read each row as a float array + } + } + + /** + * Serializes the current state of this MultiBitScalarQuantizationState object into a byte array. + * This method uses the QuantizationStateSerializer to handle the serialization process. + * + *

    The serialized byte array includes all necessary state information, such as the thresholds + * and quantization parameters, ensuring that the object can be fully reconstructed from the byte array.

    + * + * 
    +     * {@code
+     * MultiBitScalarQuantizationState state = new MultiBitScalarQuantizationState(params, thresholds);
+     * byte[] serializedState = state.toByteArray();
+     * }
+     *
    + * + * @return a byte array representing the serialized state of this object. + * @throws IOException if an I/O error occurs during serialization. + */ + @Override + public byte[] toByteArray() throws IOException { + return QuantizationStateSerializer.serialize(this); + } + + /** + * Deserializes a MultiBitScalarQuantizationState object from a byte array. + * This method uses the QuantizationStateSerializer to handle the deserialization process. + * + *

    The byte array should contain serialized state information, including the thresholds + * and quantization parameters, which are necessary to reconstruct the MultiBitScalarQuantizationState object.

    + * + * 
    +     * {@code
+     * byte[] serializedState = ...; // obtain the byte array from some source
+     * MultiBitScalarQuantizationState state = MultiBitScalarQuantizationState.fromByteArray(serializedState);
+     * }
+     *
    + * + * @param bytes the byte array containing the serialized state. + * @return the deserialized MultiBitScalarQuantizationState object. + * @throws IOException if an I/O error occurs during deserialization. + */ + public static MultiBitScalarQuantizationState fromByteArray(final byte[] bytes) throws IOException { + return (MultiBitScalarQuantizationState) QuantizationStateSerializer.deserialize(bytes, MultiBitScalarQuantizationState::new); + } +} diff --git a/src/main/java/org/opensearch/knn/quantization/models/quantizationState/OneBitScalarQuantizationState.java b/src/main/java/org/opensearch/knn/quantization/models/quantizationState/OneBitScalarQuantizationState.java new file mode 100644 index 000000000..9998b87e8 --- /dev/null +++ b/src/main/java/org/opensearch/knn/quantization/models/quantizationState/OneBitScalarQuantizationState.java @@ -0,0 +1,110 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.quantization.models.quantizationState; + +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.NoArgsConstructor; +import org.opensearch.Version; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.knn.quantization.models.quantizationParams.ScalarQuantizationParams; + +import java.io.IOException; + +/** + * OneBitScalarQuantizationState represents the state of one-bit scalar quantization, + * including the mean values used for quantization. + */ +@Getter +@NoArgsConstructor // No-argument constructor for deserialization +@AllArgsConstructor +public final class OneBitScalarQuantizationState implements QuantizationState { + private ScalarQuantizationParams quantizationParams; + /** + * Mean thresholds used in the quantization process. + * Each threshold value corresponds to a dimension of the vector being quantized. + * + * Example: + * If we have a vector [1.2, 3.4, 5.6] and mean thresholds [2.0, 3.0, 4.0], + * The quantized vector will be [0, 1, 1]. + */ + private float[] meanThresholds; + + @Override + public ScalarQuantizationParams getQuantizationParams() { + return quantizationParams; + } + + /** + * This method is responsible for writing the state of the OneBitScalarQuantizationState object to an external output. + * It includes versioning information to ensure compatibility between different versions of the serialized object. + * @param out the StreamOutput to write the object to. + * @throws IOException if an I/O error occurs during serialization. + */ + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeVInt(Version.CURRENT.id); // Write the version + quantizationParams.writeTo(out); + out.writeFloatArray(meanThresholds); + } + + /** + * This method is responsible for reading the state of the OneBitScalarQuantizationState object from an external input. + * It includes versioning information to ensure compatibility between different versions of the serialized object. + * @param in the StreamInput to read the object from. + * @throws IOException if an I/O error occurs during deserialization. + */ + public OneBitScalarQuantizationState(StreamInput in) throws IOException { + int version = in.readVInt(); // Read the version + this.quantizationParams = new ScalarQuantizationParams(in, version); + this.meanThresholds = in.readFloatArray(); + } + + /** + * Serializes the current state of this OneBitScalarQuantizationState object into a byte array. + * This method uses the QuantizationStateSerializer to handle the serialization process. + * + *

    The serialized byte array includes all necessary state information, such as the mean thresholds + * and quantization parameters, ensuring that the object can be fully reconstructed from the byte array.

    + * + * 
    +     * {@code
+     * OneBitScalarQuantizationState state = new OneBitScalarQuantizationState(params, meanThresholds);
+     * byte[] serializedState = state.toByteArray();
+     * }
+     *
    + * + * @return a byte array representing the serialized state of this object. + * @throws IOException if an I/O error occurs during serialization. + */ + @Override + public byte[] toByteArray() throws IOException { + return QuantizationStateSerializer.serialize(this); + } + + /** + * Deserializes a OneBitScalarQuantizationState object from a byte array. + * This method uses the QuantizationStateSerializer to handle the deserialization process. + * + *

    The byte array should contain serialized state information, including the mean thresholds + * and quantization parameters, which are necessary to reconstruct the OneBitScalarQuantizationState object.

    + * + * 
    +     * {@code
+     * byte[] serializedState = ...; // obtain the byte array from some source
+     * OneBitScalarQuantizationState state = OneBitScalarQuantizationState.fromByteArray(serializedState);
+     * }
+     *
    + * + * @param bytes the byte array containing the serialized state. + * @return the deserialized OneBitScalarQuantizationState object. + * @throws IOException if an I/O error occurs during deserialization. + */ + public static OneBitScalarQuantizationState fromByteArray(final byte[] bytes) throws IOException { + return (OneBitScalarQuantizationState) QuantizationStateSerializer.deserialize(bytes, OneBitScalarQuantizationState::new); + } +} diff --git a/src/main/java/org/opensearch/knn/quantization/models/quantizationState/QuantizationState.java b/src/main/java/org/opensearch/knn/quantization/models/quantizationState/QuantizationState.java new file mode 100644 index 000000000..e32df8bc3 --- /dev/null +++ b/src/main/java/org/opensearch/knn/quantization/models/quantizationState/QuantizationState.java @@ -0,0 +1,32 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.quantization.models.quantizationState; + +import org.opensearch.core.common.io.stream.Writeable; +import org.opensearch.knn.quantization.models.quantizationParams.QuantizationParams; + +import java.io.IOException; + +/** + * QuantizationState interface represents the state of a quantization process, including the parameters used. + * This interface provides methods for serializing and deserializing the state. + */ +public interface QuantizationState extends Writeable { + /** + * Returns the quantization parameters associated with this state. + * + * @return the quantization parameters. + */ + QuantizationParams getQuantizationParams(); + + /** + * Serializes the quantization state to a byte array. + * + * @return a byte array representing the serialized state. + * @throws IOException if an I/O error occurs during serialization. + */ + byte[] toByteArray() throws IOException; +} diff --git a/src/main/java/org/opensearch/knn/quantization/models/quantizationState/QuantizationStateSerializer.java b/src/main/java/org/opensearch/knn/quantization/models/quantizationState/QuantizationStateSerializer.java new file mode 100644 index 000000000..1f378e0dc --- /dev/null +++ b/src/main/java/org/opensearch/knn/quantization/models/quantizationState/QuantizationStateSerializer.java @@ -0,0 +1,56 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.quantization.models.quantizationState; + +import lombok.experimental.UtilityClass; +import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; + +import java.io.IOException; + +/** + * QuantizationStateSerializer is a utility class that provides methods for serializing and deserializing + * QuantizationState objects along with their specific data. + */ +@UtilityClass +class QuantizationStateSerializer { + + /** + * A functional interface for deserializing specific data associated with a QuantizationState. + */ + @FunctionalInterface + interface SerializableDeserializer { + QuantizationState deserialize(StreamInput in) throws IOException; + } + + /** + * Serializes the QuantizationState and specific data into a byte array. + * + * @param state The QuantizationState to serialize. + * @return A byte array representing the serialized state and specific data. + * @throws IOException If an I/O error occurs during serialization. + */ + static byte[] serialize(QuantizationState state) throws IOException { + try (BytesStreamOutput out = new BytesStreamOutput()) { + state.writeTo(out); + return out.bytes().toBytesRef().bytes; + } + } + + /** + * Deserializes a QuantizationState and its specific data from a byte array. + * + * @param bytes The byte array containing the serialized data. + * @param deserializer The deserializer for the specific data associated with the state. + * @return The deserialized QuantizationState including its specific data. + * @throws IOException If an I/O error occurs during deserialization. + */ + static QuantizationState deserialize(byte[] bytes, SerializableDeserializer deserializer) throws IOException { + try (StreamInput in = StreamInput.wrap(bytes)) { + return deserializer.deserialize(in); + } + } +} diff --git a/src/main/java/org/opensearch/knn/quantization/models/requests/TrainingRequest.java b/src/main/java/org/opensearch/knn/quantization/models/requests/TrainingRequest.java new file mode 100644 index 000000000..54ebe311c --- /dev/null +++ b/src/main/java/org/opensearch/knn/quantization/models/requests/TrainingRequest.java @@ -0,0 +1,31 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.quantization.models.requests; + +import lombok.AllArgsConstructor; +import lombok.Getter; + +/** + * TrainingRequest represents a request for training a quantizer. + * + * @param the type of vectors to be trained. + */ +@Getter +@AllArgsConstructor +public abstract class TrainingRequest { + /** + * The total number of vectors in one segment. + */ + private final int totalNumberOfVectors; + + /** + * Returns the vector corresponding to the specified document ID. + * + * @param docId the document ID. + * @return the vector corresponding to the specified document ID. + */ + public abstract T getVectorByDocId(int docId); +} diff --git a/src/main/java/org/opensearch/knn/quantization/quantizer/BitPacker.java b/src/main/java/org/opensearch/knn/quantization/quantizer/BitPacker.java new file mode 100644 index 000000000..fe470ed74 --- /dev/null +++ b/src/main/java/org/opensearch/knn/quantization/quantizer/BitPacker.java @@ -0,0 +1,143 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.quantization.quantizer; + +import lombok.experimental.UtilityClass; + +/** + * The BitPacker class provides utility methods for quantizing floating-point vectors and packing the resulting bits + * into a pre-allocated byte array. This class supports both single-bit and multi-bit quantization scenarios, + * enabling efficient storage and transmission of quantized vectors. + * + *

    + * The methods in this class are designed to be used by quantizers that need to convert floating-point vectors + * into compact binary representations by comparing them against quantization thresholds. + *

    + * + *

    + * This class is marked as a utility class using Lombok's {@link lombok.experimental.UtilityClass} annotation, + * making it a singleton and preventing instantiation. + *

    + */ +@UtilityClass +class BitPacker { + + /** + * Quantizes a given floating-point vector and packs the resulting quantized bits into a provided byte array. + * This method operates by comparing each element of the input vector against corresponding thresholds + * and encoding the results into a compact binary format using the specified number of bits per coordinate. + * + *

    + * The method supports multi-bit quantization where each coordinate of the input vector can be represented + * by multiple bits. For example, with 2-bit quantization, each coordinate is encoded into 2 bits, allowing + * for four distinct levels of quantization per coordinate. + *

    + * + *

    + * Example: + *

    + *

    + * Consider a vector with 3 coordinates: [1.2, 3.4, 5.6] and thresholds: + *

    + * 
    +     * thresholds = {
+     *     {1.0, 3.0, 5.0},  // First bit thresholds
+     *     {1.5, 3.5, 5.5}   // Second bit thresholds
+     * };
+     *
    + *

    + * If the number of bits per coordinate is 2, the quantization process will proceed as follows: + *

    + *
      + *
    • First bit comparison: + *
        + *
      • 1.2 > 1.0 -> 1
        • + *
      • 3.4 > 3.0 -> 1
        • + *
      • 5.6 > 5.0 -> 1
        • + *
      + *
      • + *
    • Second bit comparison: + *
        + *
      • 1.2 <= 1.5 -> 0
        • + *
      • 3.4 <= 3.5 -> 0
        • + *
      • 5.6 > 5.5 -> 1
        • + *
      + *
      • + *
    + *

    + * The resulting quantized bits will be 11 10 11, which is packed into the provided byte array. + * If there are fewer than 8 bits, the remaining bits in the byte are set to 0. + *

    + * + *

    + * Packing Process: + * The quantized bits are packed into the byte array. The first coordinate's bits are stored in the most + * significant positions of the first byte, followed by the second coordinate, and so on. In the example + * above, the resulting byte array will have the following binary representation: + *

    + * 
    +     * packedBits = [11011000] // Only the first 6 bits are used, and the last two are set to 0.
+     *
    + * + *

    Bitwise Operations Explanation:

    + *
      + *
    • byteIndex: This is calculated using byteIndex = bitPosition >> 3, which is equivalent to bitPosition / 8. It determines which byte in the byte array the current bit should be placed in.
      • + *
    • bitIndex: This is calculated using bitIndex = 7 - (bitPosition & 7), which is equivalent to 7 - (bitPosition % 8). It determines the exact bit position within the byte.
      • + *
    • Setting the bit: The bit is set using packedBits[byteIndex] |= (1 << bitIndex). This shifts a 1 into the correct bit position and ORs it with the existing byte value to set the bit.
      • + *
    + * + * @param vector the floating-point vector to be quantized. + * @param thresholds a 2D array representing the quantization thresholds. The first dimension corresponds to the number of bits per coordinate, and the second dimension corresponds to the vector's length. + * @param bitsPerCoordinate the number of bits used per coordinate, determining the granularity of the quantization. + * @param packedBits the byte array where the quantized bits will be packed. + */ + void quantizeAndPackBits(final float[] vector, final float[][] thresholds, final int bitsPerCoordinate, byte[] packedBits) { + int vectorLength = vector.length; + + for (int i = 0; i < bitsPerCoordinate; i++) { + for (int j = 0; j < vectorLength; j++) { + if (vector[j] > thresholds[i][j]) { + int bitPosition = i * vectorLength + j; + // Calculate the index of the byte in the packedBits array. + int byteIndex = bitPosition >> 3; // Equivalent to bitPosition / 8 + // Calculate the bit index within the byte. + int bitIndex = 7 - (bitPosition & 7); // Equivalent to 7 - (bitPosition % 8) + // Set the bit at the calculated position. + packedBits[byteIndex] |= (1 << bitIndex); // Set the bit at bitIndex + } + } + } + } + + /** + * Overloaded method to quantize a vector using single-bit quantization and pack the results into a provided byte array. + * + *

    + * This method is specifically designed for one-bit quantization scenarios, where each coordinate of the + * vector is represented by a single bit indicating whether the value is above or below the threshold. + *

    + * + *

    Example:

    + *

    + * If we have a vector [1.2, 3.4, 5.6] and thresholds [2.0, 3.0, 4.0], the quantization process will be: + *

    + *
      + *
    • 1.2 < 2.0 -> 0
      • + *
    • 3.4 > 3.0 -> 1
      • + *
    • 5.6 > 4.0 -> 1
      • + *
    + *

    + * The quantized vector will be [0, 1, 1]. + *

    + * + * @param vector the vector to quantize. + * @param thresholds the thresholds for quantization, where each element represents the threshold for a corresponding coordinate. + * @param packedBits the byte array where the quantized bits will be packed. + */ + void quantizeAndPackBits(final float[] vector, final float[] thresholds, byte[] packedBits) { + quantizeAndPackBits(vector, new float[][] { thresholds }, 1, packedBits); + } +} diff --git a/src/main/java/org/opensearch/knn/quantization/quantizer/MultiBitScalarQuantizer.java b/src/main/java/org/opensearch/knn/quantization/quantizer/MultiBitScalarQuantizer.java new file mode 100644 index 000000000..dcf825a6a --- /dev/null +++ b/src/main/java/org/opensearch/knn/quantization/quantizer/MultiBitScalarQuantizer.java @@ -0,0 +1,186 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.opensearch.knn.quantization.quantizer; + +import org.opensearch.knn.quantization.enums.ScalarQuantizationType; +import org.opensearch.knn.quantization.models.quantizationOutput.QuantizationOutput; +import org.opensearch.knn.quantization.models.quantizationParams.ScalarQuantizationParams; +import org.opensearch.knn.quantization.models.quantizationState.MultiBitScalarQuantizationState; +import org.opensearch.knn.quantization.models.quantizationState.QuantizationState; +import org.opensearch.knn.quantization.models.requests.TrainingRequest; +import org.opensearch.knn.quantization.sampler.Sampler; +import org.opensearch.knn.quantization.sampler.SamplerType; +import org.opensearch.knn.quantization.sampler.SamplingFactory; + +/** + * MultiBitScalarQuantizer is responsible for quantizing vectors into multi-bit representations per dimension. + * Unlike the OneBitScalarQuantizer, which uses a single bit per dimension to represent whether a value is above + * or below a mean threshold, the MultiBitScalarQuantizer allows for multiple bits per dimension, enabling more + * granular and precise quantization. + * + *

    + * In a OneBitScalarQuantizer, each dimension of a vector is compared to a single threshold (the mean), and a single + * bit is used to indicate whether the value is above or below that threshold. This results in a very coarse + * representation where each dimension is either "on" or "off." + *

    + * + *

    + * The MultiBitScalarQuantizer, on the other hand, uses multiple thresholds per dimension. For example, in a 2-bit + * quantization scheme, three thresholds are used to divide each dimension into four possible regions. Each region + * is represented by a unique 2-bit value. This allows for a much finer representation of the data, capturing more + * nuances in the variation of each dimension. + *

    + * + *

    + * The thresholds in MultiBitScalarQuantizer are calculated based on the mean and standard deviation of the sampled + * vectors for each dimension. Here's how it works: + *

    + * + *
      + *
    • First, the mean and standard deviation are computed for each dimension across the sampled vectors.
      • + *
    • For each bit used in the quantization (e.g., 2 bits per coordinate), the thresholds are calculated + * using a linear combination of the mean and the standard deviation. The combination coefficients are + * determined by the number of bits, allowing the thresholds to split the data into equal probability regions. + *
      • + *
    • For example, in a 2-bit quantization (which divides data into four regions), the thresholds might be + * set at points corresponding to -1 standard deviation, 0 standard deviations (mean), and +1 standard deviation. + * This ensures that the data is evenly split into four regions, each represented by a 2-bit value. + *
      • + *
    + * + *

    + * The number of bits per coordinate is determined by the type of scalar quantization being applied, such as 2-bit + * or 4-bit quantization. The increased number of bits per coordinate in MultiBitScalarQuantizer allows for better + * preservation of information during the quantization process, making it more suitable for tasks where precision + * is crucial. However, this comes at the cost of increased storage and computational complexity compared to the + * simpler OneBitScalarQuantizer. + *

    + */ +public class MultiBitScalarQuantizer implements Quantizer { + private final int bitsPerCoordinate; // Number of bits used to quantize each dimension + private final int samplingSize; // Sampling size for training + private final Sampler sampler; // Sampler for training + private static final boolean IS_TRAINING_REQUIRED = true; + // Currently Lucene has sampling size as + // 25000 for segment level training , Keeping same + // to having consistent, Will revisit + // if this requires change + private static final int DEFAULT_SAMPLE_SIZE = 25000; + + /** + * Constructs a MultiBitScalarQuantizer with a specified number of bits per coordinate. + * + * @param bitsPerCoordinate the number of bits used per coordinate for quantization. + */ + public MultiBitScalarQuantizer(final int bitsPerCoordinate) { + this(bitsPerCoordinate, DEFAULT_SAMPLE_SIZE, SamplingFactory.getSampler(SamplerType.RESERVOIR)); + } + + /** + * Constructs a MultiBitScalarQuantizer with a specified number of bits per coordinate and sampling size. + * + * @param bitsPerCoordinate the number of bits used per coordinate for quantization. + * @param samplingSize the number of samples to use for training. + * @param sampler the sampler to use for training. + */ + public MultiBitScalarQuantizer(final int bitsPerCoordinate, final int samplingSize, final Sampler sampler) { + if (bitsPerCoordinate < 2) { + throw new IllegalArgumentException("bitsPerCoordinate must be greater than or equal to 2 for multibit quantizer."); + } + this.bitsPerCoordinate = bitsPerCoordinate; + this.samplingSize = samplingSize; + this.sampler = sampler; + } + + /** + * Trains the quantizer based on the provided training request, which should be of type SamplingTrainingRequest. + * The training process calculates the mean and standard deviation for each dimension and then determines + * threshold values for quantization based on these statistics. + * + * @param trainingRequest the request containing the data and parameters for training. + * @return a MultiBitScalarQuantizationState containing the computed thresholds. + */ + @Override + public QuantizationState train(final TrainingRequest trainingRequest) { + int[] sampledIndices = sampler.sample(trainingRequest.getTotalNumberOfVectors(), samplingSize); + int dimension = trainingRequest.getVectorByDocId(sampledIndices[0]).length; + float[] meanArray = new float[dimension]; + float[] stdDevArray = new float[dimension]; + // Calculate sum, mean, and standard deviation in one pass + QuantizerHelper.calculateMeanAndStdDev(trainingRequest, sampledIndices, meanArray, stdDevArray); + float[][] thresholds = calculateThresholds(meanArray, stdDevArray, dimension); + ScalarQuantizationParams params = (bitsPerCoordinate == 2) + ? new ScalarQuantizationParams(ScalarQuantizationType.TWO_BIT) + : new ScalarQuantizationParams(ScalarQuantizationType.FOUR_BIT); + return new MultiBitScalarQuantizationState(params, thresholds); + } + + /** + * Quantizes the provided vector using the provided quantization state, producing a quantized output. + * The vector is quantized based on the thresholds in the quantization state. + * + * @param vector the vector to quantize. + * @param state the quantization state containing threshold information. + * @param output the QuantizationOutput object to store the quantized representation of the vector. + */ + @Override + public void quantize(final float[] vector, final QuantizationState state, final QuantizationOutput output) { + if (vector == null) { + throw new IllegalArgumentException("Vector to quantize must not be null."); + } + validateState(state); + MultiBitScalarQuantizationState multiBitState = (MultiBitScalarQuantizationState) state; + float[][] thresholds = multiBitState.getThresholds(); + if (thresholds == null || thresholds[0].length != vector.length) { + throw new IllegalArgumentException("Thresholds must not be null and must match the dimension of the vector."); + } + // Prepare and get the writable array + byte[] writableArray = output.prepareAndGetWritableQuantizedVector(bitsPerCoordinate, vector.length); + BitPacker.quantizeAndPackBits(vector, thresholds, bitsPerCoordinate, writableArray); + } + + /** + * Calculates the thresholds for quantization based on mean and standard deviation. + * + * @param meanArray the mean for each dimension. + * @param stdDevArray the standard deviation for each dimension. + * @param dimension the number of dimensions in the vectors. + * @return the thresholds for quantization. + */ + private float[][] calculateThresholds(final float[] meanArray, final float[] stdDevArray, final int dimension) { + float[][] thresholds = new float[bitsPerCoordinate][dimension]; + float coef = bitsPerCoordinate + 1; + for (int i = 0; i < bitsPerCoordinate; i++) { + float iCoef = -1 + 2 * (i + 1) / coef; + for (int j = 0; j < dimension; j++) { + thresholds[i][j] = meanArray[j] + iCoef * stdDevArray[j]; + } + } + return thresholds; + } + + /** + * Validates the quantization state to ensure it is of the expected type. + * + * @param state the quantization state to validate. + * @throws IllegalArgumentException if the state is not an instance of MultiBitScalarQuantizationState. + */ + private void validateState(final QuantizationState state) { + if (!(state instanceof MultiBitScalarQuantizationState)) { + throw new IllegalArgumentException("Quantization state must be of type MultiBitScalarQuantizationState."); + } + } + + /** + * Returns the number of bits per coordinate used by this quantizer. + * + * @return the number of bits per coordinate. + */ + public int getBitsPerCoordinate() { + return bitsPerCoordinate; + } +} diff --git a/src/main/java/org/opensearch/knn/quantization/quantizer/OneBitScalarQuantizer.java b/src/main/java/org/opensearch/knn/quantization/quantizer/OneBitScalarQuantizer.java new file mode 100644 index 000000000..41602dfd2 --- /dev/null +++ b/src/main/java/org/opensearch/knn/quantization/quantizer/OneBitScalarQuantizer.java @@ -0,0 +1,100 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.quantization.quantizer; + +import org.opensearch.knn.quantization.enums.ScalarQuantizationType; +import org.opensearch.knn.quantization.models.quantizationOutput.QuantizationOutput; +import org.opensearch.knn.quantization.models.quantizationParams.ScalarQuantizationParams; +import org.opensearch.knn.quantization.models.quantizationState.OneBitScalarQuantizationState; +import org.opensearch.knn.quantization.models.quantizationState.QuantizationState; +import org.opensearch.knn.quantization.models.requests.TrainingRequest; +import org.opensearch.knn.quantization.sampler.Sampler; +import org.opensearch.knn.quantization.sampler.SamplerType; +import org.opensearch.knn.quantization.sampler.SamplingFactory; + +/** + * OneBitScalarQuantizer is responsible for quantizing vectors using a single bit per dimension. + * It computes the mean of each dimension during training and then uses these means as thresholds + * for quantizing the vectors. + */ +public class OneBitScalarQuantizer implements Quantizer { + private final int samplingSize; // Sampling size for training + private static final boolean IS_TRAINING_REQUIRED = true; + private final Sampler sampler; // Sampler for training + // Currently Lucene has sampling size as + // 25000 for segment level training , Keeping same + // to having consistent, Will revisit + // if this requires change + private static final int DEFAULT_SAMPLE_SIZE = 25000; + + /** + * Constructs a OneBitScalarQuantizer with a default sampling size of 25000. + */ + public OneBitScalarQuantizer() { + this(DEFAULT_SAMPLE_SIZE, SamplingFactory.getSampler(SamplerType.RESERVOIR)); + } + + /** + * Constructs a OneBitScalarQuantizer with a specified sampling size. + * + * @param samplingSize the number of samples to use for training. + */ + public OneBitScalarQuantizer(final int samplingSize, final Sampler sampler) { + + this.samplingSize = samplingSize; + this.sampler = sampler; + } + + /** + * Trains the quantizer by calculating the mean of each dimension from the sampled vectors. + * These means are used as thresholds in the quantization process. + * + * @param trainingRequest the request containing the data and parameters for training. + * @return a OneBitScalarQuantizationState containing the calculated means. + */ + @Override + public QuantizationState train(final TrainingRequest trainingRequest) { + int[] sampledDocIds = sampler.sample(trainingRequest.getTotalNumberOfVectors(), samplingSize); + float[] meanThresholds = QuantizerHelper.calculateMeanThresholds(trainingRequest, sampledDocIds); + return new OneBitScalarQuantizationState(new ScalarQuantizationParams(ScalarQuantizationType.ONE_BIT), meanThresholds); + } + + /** + * Quantizes the provided vector using the given quantization state. + * It compares each dimension of the vector against the corresponding mean (threshold) to determine the quantized value. + * + * @param vector the vector to quantize. + * @param state the quantization state containing the means for each dimension. + * @param output the QuantizationOutput object to store the quantized representation of the vector. + */ + @Override + public void quantize(final float[] vector, final QuantizationState state, final QuantizationOutput output) { + if (vector == null) { + throw new IllegalArgumentException("Vector to quantize must not be null."); + } + validateState(state); + OneBitScalarQuantizationState binaryState = (OneBitScalarQuantizationState) state; + float[] thresholds = binaryState.getMeanThresholds(); + if (thresholds == null || thresholds.length != vector.length) { + throw new IllegalArgumentException("Thresholds must not be null and must match the dimension of the vector."); + } + // Prepare and get the writable array + byte[] writableArray = output.prepareAndGetWritableQuantizedVector(1, vector.length); + BitPacker.quantizeAndPackBits(vector, thresholds, writableArray); + } + + /** + * Validates the quantization state to ensure it is of the expected type. + * + * @param state the quantization state to validate. + * @throws IllegalArgumentException if the state is not an instance of OneBitScalarQuantizationState. + */ + private void validateState(final QuantizationState state) { + if (!(state instanceof OneBitScalarQuantizationState)) { + throw new IllegalArgumentException("Quantization state must be of type OneBitScalarQuantizationState."); + } + } +} diff --git a/src/main/java/org/opensearch/knn/quantization/quantizer/Quantizer.java b/src/main/java/org/opensearch/knn/quantization/quantizer/Quantizer.java new file mode 100644 index 000000000..c0b297f5d --- /dev/null +++ b/src/main/java/org/opensearch/knn/quantization/quantizer/Quantizer.java @@ -0,0 +1,40 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.quantization.quantizer; + +import org.opensearch.knn.quantization.models.quantizationOutput.QuantizationOutput; +import org.opensearch.knn.quantization.models.quantizationState.QuantizationState; +import org.opensearch.knn.quantization.models.requests.TrainingRequest; + +/** + * The Quantizer interface defines the methods required for training and quantizing vectors + * in the context of K-Nearest Neighbors (KNN) and similar machine learning tasks. + * It supports training to determine quantization parameters and quantizing data vectors + * based on these parameters. + * + * @param The type of the vector or data to be quantized. + * @param The type of the quantized output, typically a compressed or encoded representation. + */ +public interface Quantizer { + + /** + * Trains the quantizer based on the provided training request. The training process typically + * involves learning parameters that can be used to quantize vectors. + * + * @param trainingRequest the request containing data and parameters for training. + * @return a QuantizationState containing the learned parameters. + */ + QuantizationState train(TrainingRequest trainingRequest); + + /** + * Quantizes the provided vector using the specified quantization state. + * + * @param vector the vector to quantize. + * @param state the quantization state containing parameters for quantization. + * @param output the QuantizationOutput object to store the quantized representation of the vector. + */ + void quantize(T vector, QuantizationState state, QuantizationOutput output); +} diff --git a/src/main/java/org/opensearch/knn/quantization/quantizer/QuantizerHelper.java b/src/main/java/org/opensearch/knn/quantization/quantizer/QuantizerHelper.java new file mode 100644 index 000000000..16f969973 --- /dev/null +++ b/src/main/java/org/opensearch/knn/quantization/quantizer/QuantizerHelper.java @@ -0,0 +1,84 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.quantization.quantizer; + +import org.opensearch.knn.quantization.models.requests.TrainingRequest; +import lombok.experimental.UtilityClass; + +/** + * Utility class providing common methods for quantizer operations, such as parameter validation and + * extraction. This class is designed to be used with various quantizer implementations that require + * consistent handling of training requests and sampled indices. + */ +@UtilityClass +class QuantizerHelper { + /** + * Calculates the mean vector from a set of sampled vectors. + * + * @param samplingRequest The {@link TrainingRequest} containing the dataset and methods to access vectors by their indices. + * @param sampledIndices An array of indices representing the sampled vectors to be used for mean calculation. + * @return A float array representing the mean vector of the sampled vectors. + * @throws IllegalArgumentException If any of the vectors at the sampled indices are null. + * @throws IllegalStateException If the mean array is unexpectedly null after processing the vectors. + */ + static float[] calculateMeanThresholds(TrainingRequest samplingRequest, int[] sampledIndices) { + int totalSamples = sampledIndices.length; + float[] mean = null; + for (int docId : sampledIndices) { + float[] vector = samplingRequest.getVectorByDocId(docId); + if (vector == null) { + throw new IllegalArgumentException("Vector at sampled index " + docId + " is null."); + } + if (mean == null) { + mean = new float[vector.length]; + } + for (int j = 0; j < vector.length; j++) { + mean[j] += vector[j]; + } + } + if (mean == null) { + throw new IllegalStateException("Mean array should not be null after processing vectors."); + } + for (int j = 0; j < mean.length; j++) { + mean[j] /= totalSamples; + } + return mean; + } + + /** + * Calculates the mean and StdDev per dimension for sampled vectors. + * + * @param trainingRequest the request containing the data and parameters for training. + * @param sampledIndices the indices of the sampled vectors. + * @param meanArray the array to store the sum and then the mean of each dimension. + * @param stdDevArray the array to store the sum of squares and then the standard deviation of each dimension. + */ + static void calculateMeanAndStdDev( + TrainingRequest trainingRequest, + int[] sampledIndices, + float[] meanArray, + float[] stdDevArray + ) { + int totalSamples = sampledIndices.length; + int dimension = meanArray.length; + for (int docId : sampledIndices) { + float[] vector = trainingRequest.getVectorByDocId(docId); + if (vector == null) { + throw new IllegalArgumentException("Vector at sampled index " + docId + " is null."); + } + for (int j = 0; j < dimension; j++) { + meanArray[j] += vector[j]; + stdDevArray[j] += vector[j] * vector[j]; + } + } + + // Calculate mean and standard deviation in one pass + for (int j = 0; j < dimension; j++) { + meanArray[j] = meanArray[j] / totalSamples; + stdDevArray[j] = (float) Math.sqrt((stdDevArray[j] / totalSamples) - (meanArray[j] * meanArray[j])); + } + } +} diff --git a/src/main/java/org/opensearch/knn/quantization/sampler/ReservoirSampler.java b/src/main/java/org/opensearch/knn/quantization/sampler/ReservoirSampler.java new file mode 100644 index 000000000..020efe54f --- /dev/null +++ b/src/main/java/org/opensearch/knn/quantization/sampler/ReservoirSampler.java @@ -0,0 +1,90 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.quantization.sampler; + +import lombok.NoArgsConstructor; + +import java.util.Arrays; +import java.util.concurrent.ThreadLocalRandom; +import java.util.stream.IntStream; + +/** + * ReservoirSampler implements the Sampler interface and provides a method for sampling + * a specified number of indices from a total number of vectors using the reservoir sampling algorithm. + * This algorithm is particularly useful for randomly sampling a subset of data from a larger set + * when the total size of the dataset is unknown or very large. + */ +@NoArgsConstructor +final class ReservoirSampler implements Sampler { + /** + * Singleton instance holder. + */ + private static ReservoirSampler instance; + + /** + * Provides the singleton instance of ReservoirSampler. + * + * @return the singleton instance of ReservoirSampler. + */ + public static synchronized ReservoirSampler getInstance() { + if (instance == null) { + instance = new ReservoirSampler(); + } + return instance; + } + + /** + * Samples indices from the range [0, totalNumberOfVectors). + * If the total number of vectors is less than or equal to the sample size, it returns all indices. + * Otherwise, it uses the reservoir sampling algorithm to select a random subset. + * + * @param totalNumberOfVectors the total number of vectors to sample from. + * @param sampleSize the number of indices to sample. + * @return an array of sampled indices. + */ + @Override + public int[] sample(final int totalNumberOfVectors, final int sampleSize) { + if (totalNumberOfVectors <= sampleSize) { + return IntStream.range(0, totalNumberOfVectors).toArray(); + } + return reservoirSampleIndices(totalNumberOfVectors, sampleSize); + } + + /** + * Applies the reservoir sampling algorithm to select a random sample of indices. + * This method ensures that each index in the range [0, numVectors) has an equal probability + * of being included in the sample. + * + * Reservoir sampling is particularly useful for selecting a random sample from a large or unknown-sized dataset. + * For more information on the algorithm, see the following link: + * Reservoir Sampling - Wikipedia + * + * @param numVectors the total number of vectors. + * @param sampleSize the number of indices to sample. + * @return an array of sampled indices. + */ + private int[] reservoirSampleIndices(final int numVectors, final int sampleSize) { + int[] indices = new int[sampleSize]; + + // Initialize the reservoir with the first sampleSize elements + for (int i = 0; i < sampleSize; i++) { + indices[i] = i; + } + + // Replace elements with gradually decreasing probability + for (int i = sampleSize; i < numVectors; i++) { + int j = ThreadLocalRandom.current().nextInt(i + 1); + if (j < sampleSize) { + indices[j] = i; + } + } + + // Sort the sampled indices + Arrays.sort(indices); + + return indices; + } +} diff --git a/src/main/java/org/opensearch/knn/quantization/sampler/Sampler.java b/src/main/java/org/opensearch/knn/quantization/sampler/Sampler.java new file mode 100644 index 000000000..5ff385972 --- /dev/null +++ b/src/main/java/org/opensearch/knn/quantization/sampler/Sampler.java @@ -0,0 +1,25 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.quantization.sampler; + +/** + * The Sampler interface defines the contract for sampling strategies + * used in various quantization processes. Implementations of this + * interface should provide specific strategies for selecting a sample + * from a given set of vectors. + */ +public interface Sampler { + + /** + * Samples a subset of indices from the total number of vectors. + * + * @param totalNumberOfVectors the total number of vectors available. + * @param sampleSize the number of vectors to be sampled. + * @return an array of integers representing the indices of the sampled vectors. + * @throws IllegalArgumentException if the sample size is greater than the total number of vectors. + */ + int[] sample(int totalNumberOfVectors, int sampleSize); +} diff --git a/src/main/java/org/opensearch/knn/quantization/sampler/SamplerType.java b/src/main/java/org/opensearch/knn/quantization/sampler/SamplerType.java new file mode 100644 index 000000000..cd9b301df --- /dev/null +++ b/src/main/java/org/opensearch/knn/quantization/sampler/SamplerType.java @@ -0,0 +1,14 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.quantization.sampler; + +/** + * SamplerType is an enumeration of the different types of samplers that can be created by the factory. + */ +public enum SamplerType { + RESERVOIR, // Represents a reservoir sampling strategy + // Add more enum values here for additional sampler types +} diff --git a/src/main/java/org/opensearch/knn/quantization/sampler/SamplingFactory.java b/src/main/java/org/opensearch/knn/quantization/sampler/SamplingFactory.java new file mode 100644 index 000000000..80fe5bdae --- /dev/null +++ b/src/main/java/org/opensearch/knn/quantization/sampler/SamplingFactory.java @@ -0,0 +1,34 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.quantization.sampler; + +import lombok.AccessLevel; +import lombok.NoArgsConstructor; + +/** + * SamplingFactory is a factory class for creating instances of Sampler. + * It uses the factory design pattern to encapsulate the creation logic for different types of samplers. + */ +@NoArgsConstructor(access = AccessLevel.PRIVATE) +public final class SamplingFactory { + + /** + * Creates and returns a Sampler instance based on the specified SamplerType. + * + * @param samplerType the type of sampler to create. + * @return a Sampler instance. + * @throws IllegalArgumentException if the sampler type is not supported. + */ + public static Sampler getSampler(final SamplerType samplerType) { + switch (samplerType) { + case RESERVOIR: + return ReservoirSampler.getInstance(); + // Add more cases for different samplers here + default: + throw new IllegalArgumentException("Unsupported sampler type: " + samplerType); + } + } +} diff --git a/src/test/java/org/opensearch/knn/quantization/enums/ScalarQuantizationTypeTests.java b/src/test/java/org/opensearch/knn/quantization/enums/ScalarQuantizationTypeTests.java new file mode 100644 index 000000000..99621a0e5 --- /dev/null +++ b/src/test/java/org/opensearch/knn/quantization/enums/ScalarQuantizationTypeTests.java @@ -0,0 +1,35 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.quantization.enums; + +import org.opensearch.knn.KNNTestCase; + +import java.util.HashSet; +import java.util.Set; + +public class ScalarQuantizationTypeTests extends KNNTestCase { + public void testSQTypesValues() { + ScalarQuantizationType[] expectedValues = { + ScalarQuantizationType.ONE_BIT, + ScalarQuantizationType.TWO_BIT, + ScalarQuantizationType.FOUR_BIT }; + assertArrayEquals(expectedValues, ScalarQuantizationType.values()); + } + + public void testSQTypesValueOf() { + assertEquals(ScalarQuantizationType.ONE_BIT, ScalarQuantizationType.valueOf("ONE_BIT")); + assertEquals(ScalarQuantizationType.TWO_BIT, ScalarQuantizationType.valueOf("TWO_BIT")); + assertEquals(ScalarQuantizationType.FOUR_BIT, ScalarQuantizationType.valueOf("FOUR_BIT")); + } + + public void testUniqueSQTypeValues() { + Set uniqueIds = new HashSet<>(); + for (ScalarQuantizationType type : ScalarQuantizationType.values()) { + boolean added = uniqueIds.add(type.getId()); + assertTrue("Duplicate value found: " + type.getId(), added); + } + } +} diff --git a/src/test/java/org/opensearch/knn/quantization/factory/QuantizerFactoryTests.java b/src/test/java/org/opensearch/knn/quantization/factory/QuantizerFactoryTests.java new file mode 100644 index 000000000..3474b7ec9 --- /dev/null +++ b/src/test/java/org/opensearch/knn/quantization/factory/QuantizerFactoryTests.java @@ -0,0 +1,63 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.quantization.factory; + +import org.junit.Before; +import org.opensearch.knn.KNNTestCase; +import org.opensearch.knn.quantization.enums.ScalarQuantizationType; +import org.opensearch.knn.quantization.models.quantizationParams.ScalarQuantizationParams; +import org.opensearch.knn.quantization.quantizer.MultiBitScalarQuantizer; +import org.opensearch.knn.quantization.quantizer.OneBitScalarQuantizer; +import org.opensearch.knn.quantization.quantizer.Quantizer; + +import java.lang.reflect.Field; +import java.util.concurrent.atomic.AtomicBoolean; + +public class QuantizerFactoryTests extends KNNTestCase { + + @Before + public void resetIsRegisteredFlag() throws NoSuchFieldException, IllegalAccessException { + Field isRegisteredField = QuantizerFactory.class.getDeclaredField("isRegistered"); + isRegisteredField.setAccessible(true); + AtomicBoolean isRegistered = (AtomicBoolean) isRegisteredField.get(null); + isRegistered.set(false); + } + + public void test_Lazy_Registration() { + ScalarQuantizationParams params = new ScalarQuantizationParams(ScalarQuantizationType.ONE_BIT); + ScalarQuantizationParams paramsTwoBit = new ScalarQuantizationParams(ScalarQuantizationType.TWO_BIT); + ScalarQuantizationParams paramsFourBit = new ScalarQuantizationParams(ScalarQuantizationType.FOUR_BIT); + assertFalse(isRegisteredFieldAccessible()); + Quantizer quantizer = QuantizerFactory.getQuantizer(params); + Quantizer quantizerTwoBit = QuantizerFactory.getQuantizer(paramsTwoBit); + Quantizer quantizerFourBit = QuantizerFactory.getQuantizer(paramsFourBit); + assertTrue(quantizerFourBit instanceof MultiBitScalarQuantizer); + assertTrue(quantizerTwoBit instanceof MultiBitScalarQuantizer); + assertTrue(quantizer instanceof OneBitScalarQuantizer); + assertTrue(isRegisteredFieldAccessible()); + } + + public void testGetQuantizer_withNullParams() { + try { + QuantizerFactory.getQuantizer(null); + fail("Expected IllegalArgumentException"); + } catch (IllegalArgumentException e) { + assertEquals("Quantization parameters must not be null.", e.getMessage()); + } + } + + private boolean isRegisteredFieldAccessible() { + try { + Field isRegisteredField = QuantizerFactory.class.getDeclaredField("isRegistered"); + isRegisteredField.setAccessible(true); + AtomicBoolean isRegistered = (AtomicBoolean) isRegisteredField.get(null); + return isRegistered.get(); + } catch (NoSuchFieldException | IllegalAccessException e) { + fail("Failed to access isRegistered field."); + return false; + } + } +} diff --git a/src/test/java/org/opensearch/knn/quantization/factory/QuantizerRegistryTests.java b/src/test/java/org/opensearch/knn/quantization/factory/QuantizerRegistryTests.java new file mode 100644 index 000000000..dec34e632 --- /dev/null +++ b/src/test/java/org/opensearch/knn/quantization/factory/QuantizerRegistryTests.java @@ -0,0 +1,84 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.quantization.factory; + +import org.junit.BeforeClass; +import org.opensearch.knn.KNNTestCase; +import org.opensearch.knn.quantization.enums.ScalarQuantizationType; +import org.opensearch.knn.quantization.models.quantizationParams.ScalarQuantizationParams; +import org.opensearch.knn.quantization.quantizer.MultiBitScalarQuantizer; +import org.opensearch.knn.quantization.quantizer.OneBitScalarQuantizer; +import org.opensearch.knn.quantization.quantizer.Quantizer; + +public class QuantizerRegistryTests extends KNNTestCase { + + @BeforeClass + public static void setup() { + QuantizerRegistry.register( + ScalarQuantizationParams.generateTypeIdentifier(ScalarQuantizationType.ONE_BIT), + new OneBitScalarQuantizer() + ); + QuantizerRegistry.register( + ScalarQuantizationParams.generateTypeIdentifier(ScalarQuantizationType.TWO_BIT), + new MultiBitScalarQuantizer(2) + ); + QuantizerRegistry.register( + ScalarQuantizationParams.generateTypeIdentifier(ScalarQuantizationType.FOUR_BIT), + new MultiBitScalarQuantizer(4) + ); + } + + public void testRegisterAndGetQuantizer() { + // Test for OneBitScalarQuantizer + ScalarQuantizationParams oneBitParams = new ScalarQuantizationParams(ScalarQuantizationType.ONE_BIT); + Quantizer oneBitQuantizer = QuantizerRegistry.getQuantizer(oneBitParams); + assertTrue(oneBitQuantizer instanceof OneBitScalarQuantizer); + + // Test for MultiBitScalarQuantizer (2-bit) + ScalarQuantizationParams twoBitParams = new ScalarQuantizationParams(ScalarQuantizationType.TWO_BIT); + Quantizer twoBitQuantizer = QuantizerRegistry.getQuantizer(twoBitParams); + assertTrue(twoBitQuantizer instanceof MultiBitScalarQuantizer); + assertEquals(2, ((MultiBitScalarQuantizer) twoBitQuantizer).getBitsPerCoordinate()); + + // Test for MultiBitScalarQuantizer (4-bit) + ScalarQuantizationParams fourBitParams = new ScalarQuantizationParams(ScalarQuantizationType.FOUR_BIT); + Quantizer fourBitQuantizer = QuantizerRegistry.getQuantizer(fourBitParams); + assertTrue(fourBitQuantizer instanceof MultiBitScalarQuantizer); + assertEquals(4, ((MultiBitScalarQuantizer) fourBitQuantizer).getBitsPerCoordinate()); + } + + public void testQuantizerRegistryIsSingleton() { + // Ensure the same instance is returned for the same type identifier + ScalarQuantizationParams oneBitParams = new ScalarQuantizationParams(ScalarQuantizationType.ONE_BIT); + Quantizer firstOneBitQuantizer = QuantizerRegistry.getQuantizer(oneBitParams); + Quantizer secondOneBitQuantizer = QuantizerRegistry.getQuantizer(oneBitParams); + assertSame(firstOneBitQuantizer, secondOneBitQuantizer); + + // Ensure the same instance is returned for the same type identifier (2-bit) + ScalarQuantizationParams twoBitParams = new ScalarQuantizationParams(ScalarQuantizationType.TWO_BIT); + Quantizer firstTwoBitQuantizer = QuantizerRegistry.getQuantizer(twoBitParams); + Quantizer secondTwoBitQuantizer = QuantizerRegistry.getQuantizer(twoBitParams); + assertSame(firstTwoBitQuantizer, secondTwoBitQuantizer); + + // Ensure the same instance is returned for the same type identifier (4-bit) + ScalarQuantizationParams fourBitParams = new ScalarQuantizationParams(ScalarQuantizationType.FOUR_BIT); + Quantizer firstFourBitQuantizer = QuantizerRegistry.getQuantizer(fourBitParams); + Quantizer secondFourBitQuantizer = QuantizerRegistry.getQuantizer(fourBitParams); + assertSame(firstFourBitQuantizer, secondFourBitQuantizer); + } + + public void testRegisterQuantizerThrowsExceptionWhenAlreadyRegistered() { + ScalarQuantizationParams oneBitParams = new ScalarQuantizationParams(ScalarQuantizationType.ONE_BIT); + + // Attempt to register the same quantizer again should throw an exception + assertThrows(IllegalArgumentException.class, () -> { + QuantizerRegistry.register( + ScalarQuantizationParams.generateTypeIdentifier(ScalarQuantizationType.ONE_BIT), + new OneBitScalarQuantizer() + ); + }); + } +} diff --git a/src/test/java/org/opensearch/knn/quantization/quantizationState/QuantizationStateSerializerTests.java b/src/test/java/org/opensearch/knn/quantization/quantizationState/QuantizationStateSerializerTests.java new file mode 100644 index 000000000..fa25e8e80 --- /dev/null +++ b/src/test/java/org/opensearch/knn/quantization/quantizationState/QuantizationStateSerializerTests.java @@ -0,0 +1,46 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.quantization.quantizationState; + +import org.opensearch.knn.KNNTestCase; +import org.opensearch.knn.quantization.enums.ScalarQuantizationType; +import org.opensearch.knn.quantization.models.quantizationParams.ScalarQuantizationParams; +import org.opensearch.knn.quantization.models.quantizationState.MultiBitScalarQuantizationState; +import org.opensearch.knn.quantization.models.quantizationState.OneBitScalarQuantizationState; + +import java.io.IOException; + +public class QuantizationStateSerializerTests extends KNNTestCase { + + public void testSerializeAndDeserializeOneBitScalarQuantizationState() throws IOException { + ScalarQuantizationParams params = new ScalarQuantizationParams(ScalarQuantizationType.ONE_BIT); + float[] mean = new float[] { 0.1f, 0.2f, 0.3f }; + OneBitScalarQuantizationState state = new OneBitScalarQuantizationState(params, mean); + + // Serialize + byte[] serialized = state.toByteArray(); + + OneBitScalarQuantizationState deserialized = OneBitScalarQuantizationState.fromByteArray(serialized); + + assertArrayEquals(mean, deserialized.getMeanThresholds(), 0.0f); + assertEquals(params, deserialized.getQuantizationParams()); + } + + public void testSerializeAndDeserializeMultiBitScalarQuantizationState() throws IOException { + ScalarQuantizationParams params = new ScalarQuantizationParams(ScalarQuantizationType.TWO_BIT); + float[][] thresholds = new float[][] { { 0.1f, 0.2f, 0.3f }, { 0.4f, 0.5f, 0.6f } }; + MultiBitScalarQuantizationState state = new MultiBitScalarQuantizationState(params, thresholds); + + // Serialize + byte[] serialized = state.toByteArray(); + MultiBitScalarQuantizationState deserialized = MultiBitScalarQuantizationState.fromByteArray(serialized); + + for (int i = 0; i < thresholds.length; i++) { + assertArrayEquals(thresholds[i], deserialized.getThresholds()[i], 0.0f); + } + assertEquals(params, deserialized.getQuantizationParams()); + } +} diff --git a/src/test/java/org/opensearch/knn/quantization/quantizationState/QuantizationStateTests.java b/src/test/java/org/opensearch/knn/quantization/quantizationState/QuantizationStateTests.java new file mode 100644 index 000000000..35edf49e2 --- /dev/null +++ b/src/test/java/org/opensearch/knn/quantization/quantizationState/QuantizationStateTests.java @@ -0,0 +1,68 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.quantization.quantizationState; + +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.knn.KNNTestCase; +import org.opensearch.knn.quantization.enums.ScalarQuantizationType; +import org.opensearch.knn.quantization.models.quantizationParams.ScalarQuantizationParams; +import org.opensearch.knn.quantization.models.quantizationState.MultiBitScalarQuantizationState; +import org.opensearch.knn.quantization.models.quantizationState.OneBitScalarQuantizationState; + +import java.io.IOException; + +public class QuantizationStateTests extends KNNTestCase { + + public void testOneBitScalarQuantizationStateSerialization() throws IOException { + ScalarQuantizationParams params = new ScalarQuantizationParams(ScalarQuantizationType.ONE_BIT); + float[] mean = { 1.0f, 2.0f, 3.0f }; + + OneBitScalarQuantizationState state = new OneBitScalarQuantizationState(params, mean); + + // Serialize + byte[] serializedState = state.toByteArray(); + + // Deserialize + StreamInput in = StreamInput.wrap(serializedState); + OneBitScalarQuantizationState deserializedState = new OneBitScalarQuantizationState(in); + + float delta = 0.0001f; + assertArrayEquals(mean, deserializedState.getMeanThresholds(), delta); + assertEquals(params.getSqType(), deserializedState.getQuantizationParams().getSqType()); + } + + public void testMultiBitScalarQuantizationStateSerialization() throws IOException { + ScalarQuantizationParams params = new ScalarQuantizationParams(ScalarQuantizationType.TWO_BIT); + float[][] thresholds = { { 0.5f, 1.5f, 2.5f }, { 1.0f, 2.0f, 3.0f } }; + + MultiBitScalarQuantizationState state = new MultiBitScalarQuantizationState(params, thresholds); + byte[] serializedState = state.toByteArray(); + + // Deserialize + StreamInput in = StreamInput.wrap(serializedState); + MultiBitScalarQuantizationState deserializedState = new MultiBitScalarQuantizationState(in); + + float delta = 0.0001f; + for (int i = 0; i < thresholds.length; i++) { + assertArrayEquals(thresholds[i], deserializedState.getThresholds()[i], delta); + } + assertEquals(params.getSqType(), deserializedState.getQuantizationParams().getSqType()); + } + + public void testSerializationWithDifferentVersions() throws IOException { + ScalarQuantizationParams params = new ScalarQuantizationParams(ScalarQuantizationType.ONE_BIT); + float[] mean = { 1.0f, 2.0f, 3.0f }; + + OneBitScalarQuantizationState state = new OneBitScalarQuantizationState(params, mean); + byte[] serializedState = state.toByteArray(); + StreamInput in = StreamInput.wrap(serializedState); + OneBitScalarQuantizationState deserializedState = new OneBitScalarQuantizationState(in); + + float delta = 0.0001f; + assertArrayEquals(mean, deserializedState.getMeanThresholds(), delta); + assertEquals(params.getSqType(), deserializedState.getQuantizationParams().getSqType()); + } +} diff --git a/src/test/java/org/opensearch/knn/quantization/quantizer/MultiBitScalarQuantizerTests.java b/src/test/java/org/opensearch/knn/quantization/quantizer/MultiBitScalarQuantizerTests.java new file mode 100644 index 000000000..ad6a44686 --- /dev/null +++ b/src/test/java/org/opensearch/knn/quantization/quantizer/MultiBitScalarQuantizerTests.java @@ -0,0 +1,107 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.quantization.quantizer; + +import org.opensearch.knn.KNNTestCase; +import org.opensearch.knn.quantization.enums.ScalarQuantizationType; +import org.opensearch.knn.quantization.models.quantizationOutput.BinaryQuantizationOutput; +import org.opensearch.knn.quantization.models.quantizationParams.ScalarQuantizationParams; +import org.opensearch.knn.quantization.models.quantizationState.MultiBitScalarQuantizationState; +import org.opensearch.knn.quantization.models.quantizationState.QuantizationState; +import org.opensearch.knn.quantization.models.requests.TrainingRequest; + +import java.io.IOException; + +public class MultiBitScalarQuantizerTests extends KNNTestCase { + + public void testTrain_twoBit() { + float[][] vectors = { + { 0.5f, 1.5f, 2.5f, 3.5f, 4.5f, 5.5f, 6.5f, 7.5f }, + { 1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f, 7.0f, 8.0f }, + { 1.5f, 2.5f, 3.5f, 4.5f, 5.5f, 6.5f, 7.5f, 8.5f } }; + MultiBitScalarQuantizer twoBitQuantizer = new MultiBitScalarQuantizer(2); + ScalarQuantizationParams params = new ScalarQuantizationParams(ScalarQuantizationType.TWO_BIT); + TrainingRequest request = new MockTrainingRequest(params, vectors); + QuantizationState state = twoBitQuantizer.train(request); + + assertTrue(state instanceof MultiBitScalarQuantizationState); + MultiBitScalarQuantizationState mbState = (MultiBitScalarQuantizationState) state; + assertNotNull(mbState.getThresholds()); + assertEquals(2, mbState.getThresholds().length); // 2-bit quantization should have 2 thresholds + } + + public void testTrain_fourBit() { + MultiBitScalarQuantizer fourBitQuantizer = new MultiBitScalarQuantizer(4); + float[][] vectors = { + { 0.5f, 1.5f, 2.5f, 3.5f, 4.5f, 5.5f, 6.5f, 7.5f }, + { 1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f, 7.0f, 8.0f }, + { 1.5f, 2.5f, 3.5f, 4.5f, 5.5f, 6.5f, 7.5f, 8.5f } }; + ScalarQuantizationParams params = new ScalarQuantizationParams(ScalarQuantizationType.FOUR_BIT); + TrainingRequest request = new MockTrainingRequest(params, vectors); + QuantizationState state = fourBitQuantizer.train(request); + + assertTrue(state instanceof MultiBitScalarQuantizationState); + MultiBitScalarQuantizationState mbState = (MultiBitScalarQuantizationState) state; + assertNotNull(mbState.getThresholds()); + assertEquals(4, mbState.getThresholds().length); // 4-bit quantization should have 4 thresholds + } + + public void testQuantize_twoBit() throws IOException { + MultiBitScalarQuantizer twoBitQuantizer = new MultiBitScalarQuantizer(2); + float[] vector = { 1.3f, 2.2f, 3.3f, 4.1f, 5.6f, 6.7f, 7.4f, 8.1f }; + float[][] thresholds = { { 1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f, 7.0f, 8.0f }, { 1.5f, 2.5f, 3.5f, 4.5f, 5.5f, 6.5f, 7.5f, 8.5f } }; + ScalarQuantizationParams params = new ScalarQuantizationParams(ScalarQuantizationType.TWO_BIT); + MultiBitScalarQuantizationState state = new MultiBitScalarQuantizationState(params, thresholds); + + BinaryQuantizationOutput output = new BinaryQuantizationOutput(); + twoBitQuantizer.quantize(vector, state, output); + assertNotNull(output.getQuantizedVector()); + } + + public void testQuantize_fourBit() throws IOException { + MultiBitScalarQuantizer fourBitQuantizer = new MultiBitScalarQuantizer(4); + float[] vector = { 1.3f, 2.2f, 3.3f, 4.1f, 5.6f, 6.7f, 7.4f, 8.1f }; + float[][] thresholds = { + { 1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f, 7.0f, 8.0f }, + { 1.1f, 2.1f, 3.1f, 4.1f, 5.1f, 6.1f, 7.1f, 8.1f }, + { 1.2f, 2.2f, 3.2f, 4.2f, 5.2f, 6.2f, 7.2f, 8.2f }, + { 1.3f, 2.3f, 3.3f, 4.3f, 5.3f, 6.3f, 7.3f, 8.3f } }; + ScalarQuantizationParams params = new ScalarQuantizationParams(ScalarQuantizationType.FOUR_BIT); + MultiBitScalarQuantizationState state = new MultiBitScalarQuantizationState(params, thresholds); + + BinaryQuantizationOutput output = new BinaryQuantizationOutput(); + fourBitQuantizer.quantize(vector, state, output); + assertNotNull(output.getQuantizedVector()); + } + + public void testQuantize_withNullVector() throws IOException { + MultiBitScalarQuantizer twoBitQuantizer = new MultiBitScalarQuantizer(2); + BinaryQuantizationOutput output = new BinaryQuantizationOutput(); + expectThrows( + IllegalArgumentException.class, + () -> twoBitQuantizer.quantize( + null, + new MultiBitScalarQuantizationState(new ScalarQuantizationParams(ScalarQuantizationType.TWO_BIT), new float[2][8]), + output + ) + ); + } + + // Mock classes for testing + private static class MockTrainingRequest extends TrainingRequest { + private final float[][] vectors; + + public MockTrainingRequest(ScalarQuantizationParams params, float[][] vectors) { + super(vectors.length); + this.vectors = vectors; + } + + @Override + public float[] getVectorByDocId(int docId) { + return vectors[docId]; + } + } +} diff --git a/src/test/java/org/opensearch/knn/quantization/quantizer/OneBitScalarQuantizerTests.java b/src/test/java/org/opensearch/knn/quantization/quantizer/OneBitScalarQuantizerTests.java new file mode 100644 index 000000000..28be260d7 --- /dev/null +++ b/src/test/java/org/opensearch/knn/quantization/quantizer/OneBitScalarQuantizerTests.java @@ -0,0 +1,136 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.quantization.quantizer; + +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.knn.KNNTestCase; +import org.opensearch.knn.quantization.enums.ScalarQuantizationType; +import org.opensearch.knn.quantization.models.quantizationOutput.BinaryQuantizationOutput; +import org.opensearch.knn.quantization.models.quantizationParams.ScalarQuantizationParams; +import org.opensearch.knn.quantization.models.quantizationState.OneBitScalarQuantizationState; +import org.opensearch.knn.quantization.models.quantizationState.QuantizationState; +import org.opensearch.knn.quantization.models.requests.TrainingRequest; +import org.opensearch.knn.quantization.sampler.Sampler; +import org.opensearch.knn.quantization.sampler.SamplerType; +import org.opensearch.knn.quantization.sampler.SamplingFactory; + +import java.io.IOException; + +public class OneBitScalarQuantizerTests extends KNNTestCase { + + public void testTrain_withTrainingRequired() { + float[][] vectors = { { 1.0f, 2.0f, 3.0f }, { 4.0f, 5.0f, 6.0f }, { 7.0f, 8.0f, 9.0f } }; + + ScalarQuantizationParams params = new ScalarQuantizationParams(ScalarQuantizationType.ONE_BIT); + TrainingRequest originalRequest = new TrainingRequest(vectors.length) { + @Override + public float[] getVectorByDocId(int docId) { + return vectors[docId]; + } + }; + OneBitScalarQuantizer quantizer = new OneBitScalarQuantizer(); + QuantizationState state = quantizer.train(originalRequest); + + assertTrue(state instanceof OneBitScalarQuantizationState); + float[] meanThresholds = ((OneBitScalarQuantizationState) state).getMeanThresholds(); + assertArrayEquals(new float[] { 4.0f, 5.0f, 6.0f }, meanThresholds, 0.001f); + } + + public void testQuantize_withState() throws IOException { + float[] vector = { 3.0f, 6.0f, 9.0f }; + float[] thresholds = { 4.0f, 5.0f, 6.0f }; + OneBitScalarQuantizationState state = new OneBitScalarQuantizationState( + new ScalarQuantizationParams(ScalarQuantizationType.ONE_BIT), + thresholds + ); + + OneBitScalarQuantizer quantizer = new OneBitScalarQuantizer(); + BinaryQuantizationOutput output = new BinaryQuantizationOutput(); + quantizer.quantize(vector, state, output); + + assertNotNull(output); + byte[] expectedPackedBits = new byte[] { 0b01100000 }; // or 96 in decimal + assertArrayEquals(expectedPackedBits, output.getQuantizedVector()); + } + + public void testQuantize_withNullVector() throws IOException { + OneBitScalarQuantizer quantizer = new OneBitScalarQuantizer(); + OneBitScalarQuantizationState state = new OneBitScalarQuantizationState( + new ScalarQuantizationParams(ScalarQuantizationType.ONE_BIT), + new float[] { 0.0f } + ); + BinaryQuantizationOutput output = new BinaryQuantizationOutput(); + expectThrows(IllegalArgumentException.class, () -> quantizer.quantize(null, state, output)); + } + + public void testQuantize_withInvalidState() throws IOException { + OneBitScalarQuantizer quantizer = new OneBitScalarQuantizer(); + float[] vector = { 1.0f, 2.0f, 3.0f }; + QuantizationState invalidState = new QuantizationState() { + @Override + public ScalarQuantizationParams getQuantizationParams() { + return new ScalarQuantizationParams(ScalarQuantizationType.ONE_BIT); + } + + @Override + public byte[] toByteArray() { + return new byte[0]; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + // Empty implementation for test + } + }; + BinaryQuantizationOutput output = new BinaryQuantizationOutput(); + expectThrows(IllegalArgumentException.class, () -> quantizer.quantize(vector, invalidState, output)); + } + + public void testQuantize_withMismatchedDimensions() throws IOException { + OneBitScalarQuantizer quantizer = new OneBitScalarQuantizer(); + float[] vector = { 1.0f, 2.0f, 3.0f }; + float[] thresholds = { 4.0f, 5.0f }; + OneBitScalarQuantizationState state = new OneBitScalarQuantizationState( + new ScalarQuantizationParams(ScalarQuantizationType.ONE_BIT), + thresholds + ); + BinaryQuantizationOutput output = new BinaryQuantizationOutput(); + expectThrows(IllegalArgumentException.class, () -> quantizer.quantize(vector, state, output)); + } + + public void testCalculateMean() { + float[][] vectors = { { 1.0f, 2.0f, 3.0f }, { 4.0f, 5.0f, 6.0f }, { 7.0f, 8.0f, 9.0f } }; + + ScalarQuantizationParams params = new ScalarQuantizationParams(ScalarQuantizationType.ONE_BIT); + TrainingRequest samplingRequest = new TrainingRequest(vectors.length) { + @Override + public float[] getVectorByDocId(int docId) { + return vectors[docId]; + } + }; + + Sampler sampler = SamplingFactory.getSampler(SamplerType.RESERVOIR); + int[] sampledIndices = sampler.sample(vectors.length, 3); + float[] meanThresholds = QuantizerHelper.calculateMeanThresholds(samplingRequest, sampledIndices); + assertArrayEquals(new float[] { 4.0f, 5.0f, 6.0f }, meanThresholds, 0.001f); + } + + public void testCalculateMean_withNullVector() { + float[][] vectors = { { 1.0f, 2.0f, 3.0f }, null, { 7.0f, 8.0f, 9.0f } }; + + ScalarQuantizationParams params = new ScalarQuantizationParams(ScalarQuantizationType.ONE_BIT); + TrainingRequest samplingRequest = new TrainingRequest(vectors.length) { + @Override + public float[] getVectorByDocId(int docId) { + return vectors[docId]; + } + }; + + Sampler sampler = SamplingFactory.getSampler(SamplerType.RESERVOIR); + int[] sampledIndices = sampler.sample(vectors.length, 3); + expectThrows(IllegalArgumentException.class, () -> QuantizerHelper.calculateMeanThresholds(samplingRequest, sampledIndices)); + } +} diff --git a/src/test/java/org/opensearch/knn/quantization/sampler/ReservoirSamplerTests.java b/src/test/java/org/opensearch/knn/quantization/sampler/ReservoirSamplerTests.java new file mode 100644 index 000000000..59952eb10 --- /dev/null +++ b/src/test/java/org/opensearch/knn/quantization/sampler/ReservoirSamplerTests.java @@ -0,0 +1,63 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.quantization.sampler; + +import org.opensearch.knn.KNNTestCase; + +import java.util.Arrays; +import java.util.stream.IntStream; + +public class ReservoirSamplerTests extends KNNTestCase { + + public void testSampleLessThanSampleSize() { + ReservoirSampler sampler = ReservoirSampler.getInstance(); + int totalNumberOfVectors = 5; + int sampleSize = 10; + int[] sampledIndices = sampler.sample(totalNumberOfVectors, sampleSize); + int[] expectedIndices = IntStream.range(0, totalNumberOfVectors).toArray(); + assertArrayEquals("Sampled indices should include all available indices.", expectedIndices, sampledIndices); + } + + public void testSampleEqualToSampleSize() { + ReservoirSampler sampler = ReservoirSampler.getInstance(); + int totalNumberOfVectors = 10; + int sampleSize = 10; + int[] sampledIndices = sampler.sample(totalNumberOfVectors, sampleSize); + int[] expectedIndices = IntStream.range(0, totalNumberOfVectors).toArray(); + assertArrayEquals("Sampled indices should include all available indices.", expectedIndices, sampledIndices); + } + + public void testSampleRandomness() { + ReservoirSampler sampler1 = ReservoirSampler.getInstance(); + ReservoirSampler sampler2 = ReservoirSampler.getInstance(); + int totalNumberOfVectors = 100; + int sampleSize = 10; + + int[] sampledIndices1 = sampler1.sample(totalNumberOfVectors, sampleSize); + int[] sampledIndices2 = sampler2.sample(totalNumberOfVectors, sampleSize); + + // It's unlikely but possible for the two samples to be equal, so we just check they are sorted correctly + Arrays.sort(sampledIndices1); + Arrays.sort(sampledIndices2); + assertFalse("Sampled indices should be different", Arrays.equals(sampledIndices1, sampledIndices2)); + } + + public void testEdgeCaseZeroVectors() { + ReservoirSampler sampler = ReservoirSampler.getInstance(); + int totalNumberOfVectors = 0; + int sampleSize = 10; + int[] sampledIndices = sampler.sample(totalNumberOfVectors, sampleSize); + assertEquals("Sampled indices should be empty when there are zero vectors.", 0, sampledIndices.length); + } + + public void testEdgeCaseZeroSampleSize() { + ReservoirSampler sampler = ReservoirSampler.getInstance(); + int totalNumberOfVectors = 10; + int sampleSize = 0; + int[] sampledIndices = sampler.sample(totalNumberOfVectors, sampleSize); + assertEquals("Sampled indices should be empty when sample size is zero.", 0, sampledIndices.length); + } +} diff --git a/src/test/java/org/opensearch/knn/quantization/sampler/SamplingFactoryTests.java b/src/test/java/org/opensearch/knn/quantization/sampler/SamplingFactoryTests.java new file mode 100644 index 000000000..db8772b70 --- /dev/null +++ b/src/test/java/org/opensearch/knn/quantization/sampler/SamplingFactoryTests.java @@ -0,0 +1,19 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.quantization.sampler; + +import org.opensearch.knn.KNNTestCase; + +public class SamplingFactoryTests extends KNNTestCase { + public void testGetSampler_withReservoir() { + Sampler sampler = SamplingFactory.getSampler(SamplerType.RESERVOIR); + assertTrue(sampler instanceof ReservoirSampler); + } + + public void testGetSampler_withUnsupportedType() { + expectThrows(NullPointerException.class, () -> SamplingFactory.getSampler(null)); // This should throw an exception + } +} From 1ca7e5e058da74a4ea18dbdbb8d37005a5e224ff Mon Sep 17 00:00:00 2001 From: Navneet Verma Date: Wed, 14 Aug 2024 13:25:46 -0700 Subject: [PATCH 324/416] Integrate KNNVectorValues with vector ANN Search flow (#1952) (#1958) Signed-off-by: Navneet Verma --- CHANGELOG.md | 5 + .../knn/common/FieldInfoExtractor.java | 37 ++++++++ .../opensearch/knn/index/query/KNNWeight.java | 22 +++-- .../filtered/FilteredIdsKNNByteIterator.java | 16 ++-- .../filtered/FilteredIdsKNNIterator.java | 17 ++-- .../NestedFilteredIdsKNNByteIterator.java | 8 +- .../NestedFilteredIdsKNNIterator.java | 8 +- .../vectorvalues/KNNVectorValuesFactory.java | 34 ++++++- .../org/opensearch/knn/indices/ModelUtil.java | 21 +++++ .../knn/common/FieldInfoExtractorTests.java | 44 +++++++++ .../knn/index/query/KNNWeightTests.java | 34 ++++--- .../FilteredIdsKNNByteIteratorTests.java | 8 +- .../filtered/FilteredIdsKNNIteratorTests.java | 11 +-- ...NestedFilteredIdsKNNByteIteratorTests.java | 8 +- .../NestedFilteredIdsKNNIteratorTests.java | 15 ++- .../KNNVectorValuesFactoryTests.java | 91 +++++++++++++++++++ .../knn/indices/ModelUtilTests.java | 36 ++++++++ 17 files changed, 339 insertions(+), 76 deletions(-) create mode 100644 src/main/java/org/opensearch/knn/common/FieldInfoExtractor.java create mode 100644 src/test/java/org/opensearch/knn/common/FieldInfoExtractorTests.java create mode 100644 src/test/java/org/opensearch/knn/indices/ModelUtilTests.java diff --git a/CHANGELOG.md b/CHANGELOG.md index f53b32a1b..86defd59e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,17 +14,22 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ## [Unreleased 2.x](https://github.com/opensearch-project/k-NN/compare/2.16...2.x) ### Features +* Integrate Lucene Vector field with native engines to use KNNVectorFormat during segment creation [#1945](https://github.com/opensearch-project/k-NN/pull/1945) ### Enhancements ### Bug Fixes * Corrected search logic for scenario with non-existent fields in filter [#1874](https://github.com/opensearch-project/k-NN/pull/1874) * Fix graph merge stats size calculation [#1844](https://github.com/opensearch-project/k-NN/pull/1844) * Integrate Lucene Vector field with native engines to use KNNVectorFormat during segment creation [#1945](https://github.com/opensearch-project/k-NN/pull/1945) +* Add script_fields context to KNNAllowlist [#1917] (https://github.com/opensearch-project/k-NN/pull/1917) +* Fix graph merge stats size calculation [#1844](https://github.com/opensearch-project/k-NN/pull/1844) +* Disallow a vector field to have an invalid character for a physical file name. [#1936](https://github.com/opensearch-project/k-NN/pull/1936) ### Infrastructure ### Documentation ### Maintenance * Fix a flaky unit test:testMultiFieldsKnnIndex, which was failing due to inconsistent merge behaviors [#1924](https://github.com/opensearch-project/k-NN/pull/1924) ### Refactoring * Introduce KNNVectorValues interface to iterate on different types of Vector values during indexing and search [#1897](https://github.com/opensearch-project/k-NN/pull/1897) +* Integrate KNNVectorValues with vector ANN Search flow [#1952](https://github.com/opensearch-project/k-NN/pull/1952) * Clean up parsing for query [#1824](https://github.com/opensearch-project/k-NN/pull/1824) * Refactor engine package structure [#1913](https://github.com/opensearch-project/k-NN/pull/1913) * Refactor method structure and definitions [#1920](https://github.com/opensearch-project/k-NN/pull/1920) diff --git a/src/main/java/org/opensearch/knn/common/FieldInfoExtractor.java b/src/main/java/org/opensearch/knn/common/FieldInfoExtractor.java new file mode 100644 index 000000000..591f16735 --- /dev/null +++ b/src/main/java/org/opensearch/knn/common/FieldInfoExtractor.java @@ -0,0 +1,37 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.common; + +import lombok.experimental.UtilityClass; +import org.apache.commons.lang.StringUtils; +import org.apache.lucene.index.FieldInfo; +import org.opensearch.knn.index.VectorDataType; +import org.opensearch.knn.indices.ModelMetadata; +import org.opensearch.knn.indices.ModelUtil; + +/** + * A utility class to extract information from FieldInfo. + */ +@UtilityClass +public class FieldInfoExtractor { + + /** + * Extract vector data type from fieldInfo + * @param fieldInfo {@link FieldInfo} + * @return {@link VectorDataType} + */ + public static VectorDataType extractVectorDataType(final FieldInfo fieldInfo) { + String vectorDataTypeString = fieldInfo.getAttribute(KNNConstants.VECTOR_DATA_TYPE_FIELD); + if (StringUtils.isEmpty(vectorDataTypeString)) { + final ModelMetadata modelMetadata = ModelUtil.getModelMetadata(fieldInfo.getAttribute(KNNConstants.MODEL_ID)); + if (modelMetadata != null) { + VectorDataType vectorDataType = modelMetadata.getVectorDataType(); + vectorDataTypeString = vectorDataType == null ? null : vectorDataType.getValue(); + } + } + return StringUtils.isNotEmpty(vectorDataTypeString) ? VectorDataType.get(vectorDataTypeString) : VectorDataType.DEFAULT; + } +} diff --git a/src/main/java/org/opensearch/knn/index/query/KNNWeight.java b/src/main/java/org/opensearch/knn/index/query/KNNWeight.java index f88652525..df40f6850 100644 --- a/src/main/java/org/opensearch/knn/index/query/KNNWeight.java +++ b/src/main/java/org/opensearch/knn/index/query/KNNWeight.java @@ -8,8 +8,6 @@ import com.google.common.annotations.VisibleForTesting; import lombok.extern.log4j.Log4j2; import org.apache.commons.lang.StringUtils; -import org.apache.lucene.index.BinaryDocValues; -import org.apache.lucene.index.DocValues; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.SegmentReader; @@ -43,6 +41,10 @@ import org.opensearch.knn.index.query.filtered.NestedFilteredIdsKNNByteIterator; import org.opensearch.knn.index.query.filtered.NestedFilteredIdsKNNIterator; import org.opensearch.knn.index.engine.KNNEngine; +import org.opensearch.knn.index.vectorvalues.KNNBinaryVectorValues; +import org.opensearch.knn.index.vectorvalues.KNNFloatVectorValues; +import org.opensearch.knn.index.vectorvalues.KNNVectorValues; +import org.opensearch.knn.index.vectorvalues.KNNVectorValuesFactory; import org.opensearch.knn.indices.ModelDao; import org.opensearch.knn.indices.ModelMetadata; import org.opensearch.knn.indices.ModelUtil; @@ -412,25 +414,31 @@ private Map doExactSearch(final LeafReaderContext leafReaderCont private KNNIterator getFilteredKNNIterator(final LeafReaderContext leafReaderContext, final BitSet filterIdsBitSet) throws IOException { final SegmentReader reader = Lucene.segmentReader(leafReaderContext.reader()); final FieldInfo fieldInfo = reader.getFieldInfos().fieldInfo(knnQuery.getField()); - final BinaryDocValues values = DocValues.getBinary(leafReaderContext.reader(), fieldInfo.getName()); final SpaceType spaceType = getSpaceType(fieldInfo); if (VectorDataType.BINARY == knnQuery.getVectorDataType()) { + final KNNVectorValues vectorValues = KNNVectorValuesFactory.getVectorValues(fieldInfo, leafReaderContext.reader()); return knnQuery.getParentsFilter() == null - ? new FilteredIdsKNNByteIterator(filterIdsBitSet, knnQuery.getByteQueryVector(), values, spaceType) + ? new FilteredIdsKNNByteIterator( + filterIdsBitSet, + knnQuery.getByteQueryVector(), + (KNNBinaryVectorValues) vectorValues, + spaceType + ) : new NestedFilteredIdsKNNByteIterator( filterIdsBitSet, knnQuery.getByteQueryVector(), - values, + (KNNBinaryVectorValues) vectorValues, spaceType, knnQuery.getParentsFilter().getBitSet(leafReaderContext) ); } else { + final KNNVectorValues vectorValues = KNNVectorValuesFactory.getVectorValues(fieldInfo, leafReaderContext.reader()); return knnQuery.getParentsFilter() == null - ? new FilteredIdsKNNIterator(filterIdsBitSet, knnQuery.getQueryVector(), values, spaceType) + ? new FilteredIdsKNNIterator(filterIdsBitSet, knnQuery.getQueryVector(), (KNNFloatVectorValues) vectorValues, spaceType) : new NestedFilteredIdsKNNIterator( filterIdsBitSet, knnQuery.getQueryVector(), - values, + (KNNFloatVectorValues) vectorValues, spaceType, knnQuery.getParentsFilter().getBitSet(leafReaderContext) ); diff --git a/src/main/java/org/opensearch/knn/index/query/filtered/FilteredIdsKNNByteIterator.java b/src/main/java/org/opensearch/knn/index/query/filtered/FilteredIdsKNNByteIterator.java index 815e621f6..ccfe626a0 100644 --- a/src/main/java/org/opensearch/knn/index/query/filtered/FilteredIdsKNNByteIterator.java +++ b/src/main/java/org/opensearch/knn/index/query/filtered/FilteredIdsKNNByteIterator.java @@ -5,14 +5,12 @@ package org.opensearch.knn.index.query.filtered; -import org.apache.lucene.index.BinaryDocValues; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.util.BitSet; import org.apache.lucene.util.BitSetIterator; -import org.apache.lucene.util.BytesRef; import org.opensearch.knn.index.SpaceType; +import org.opensearch.knn.index.vectorvalues.KNNBinaryVectorValues; -import java.io.ByteArrayInputStream; import java.io.IOException; /** @@ -26,7 +24,7 @@ public class FilteredIdsKNNByteIterator implements KNNIterator { protected final BitSet filterIdsBitSet; protected final BitSetIterator bitSetIterator; protected final byte[] queryVector; - protected final BinaryDocValues binaryDocValues; + protected final KNNBinaryVectorValues binaryVectorValues; protected final SpaceType spaceType; protected float currentScore = Float.NEGATIVE_INFINITY; protected int docId; @@ -34,13 +32,13 @@ public class FilteredIdsKNNByteIterator implements KNNIterator { public FilteredIdsKNNByteIterator( final BitSet filterIdsBitSet, final byte[] queryVector, - final BinaryDocValues binaryDocValues, + final KNNBinaryVectorValues binaryVectorValues, final SpaceType spaceType ) { this.filterIdsBitSet = filterIdsBitSet; this.bitSetIterator = new BitSetIterator(filterIdsBitSet, filterIdsBitSet.length()); this.queryVector = queryVector; - this.binaryDocValues = binaryDocValues; + this.binaryVectorValues = binaryVectorValues; this.spaceType = spaceType; this.docId = bitSetIterator.nextDoc(); } @@ -57,7 +55,7 @@ public int nextDoc() throws IOException { if (docId == DocIdSetIterator.NO_MORE_DOCS) { return DocIdSetIterator.NO_MORE_DOCS; } - int doc = binaryDocValues.advance(docId); + int doc = binaryVectorValues.advance(docId); currentScore = computeScore(); docId = bitSetIterator.nextDoc(); return doc; @@ -69,9 +67,7 @@ public float score() { } protected float computeScore() throws IOException { - final BytesRef value = binaryDocValues.binaryValue(); - final ByteArrayInputStream byteStream = new ByteArrayInputStream(value.bytes, value.offset, value.length); - final byte[] vector = byteStream.readAllBytes(); + final byte[] vector = binaryVectorValues.getVector(); // Calculates a similarity score between the two vectors with a specified function. Higher similarity // scores correspond to closer vectors. return spaceType.getKnnVectorSimilarityFunction().compare(queryVector, vector); diff --git a/src/main/java/org/opensearch/knn/index/query/filtered/FilteredIdsKNNIterator.java b/src/main/java/org/opensearch/knn/index/query/filtered/FilteredIdsKNNIterator.java index 7e554fb7d..a0d7694c9 100644 --- a/src/main/java/org/opensearch/knn/index/query/filtered/FilteredIdsKNNIterator.java +++ b/src/main/java/org/opensearch/knn/index/query/filtered/FilteredIdsKNNIterator.java @@ -5,14 +5,11 @@ package org.opensearch.knn.index.query.filtered; -import org.apache.lucene.index.BinaryDocValues; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.util.BitSet; import org.apache.lucene.util.BitSetIterator; -import org.apache.lucene.util.BytesRef; import org.opensearch.knn.index.SpaceType; -import org.opensearch.knn.index.codec.util.KNNVectorSerializer; -import org.opensearch.knn.index.codec.util.KNNVectorSerializerFactory; +import org.opensearch.knn.index.vectorvalues.KNNFloatVectorValues; import java.io.IOException; @@ -27,7 +24,7 @@ public class FilteredIdsKNNIterator implements KNNIterator { protected final BitSet filterIdsBitSet; protected final BitSetIterator bitSetIterator; protected final float[] queryVector; - protected final BinaryDocValues binaryDocValues; + protected final KNNFloatVectorValues knnFloatVectorValues; protected final SpaceType spaceType; protected float currentScore = Float.NEGATIVE_INFINITY; protected int docId; @@ -35,13 +32,13 @@ public class FilteredIdsKNNIterator implements KNNIterator { public FilteredIdsKNNIterator( final BitSet filterIdsBitSet, final float[] queryVector, - final BinaryDocValues binaryDocValues, + final KNNFloatVectorValues knnFloatVectorValues, final SpaceType spaceType ) { this.filterIdsBitSet = filterIdsBitSet; this.bitSetIterator = new BitSetIterator(filterIdsBitSet, filterIdsBitSet.length()); this.queryVector = queryVector; - this.binaryDocValues = binaryDocValues; + this.knnFloatVectorValues = knnFloatVectorValues; this.spaceType = spaceType; this.docId = bitSetIterator.nextDoc(); } @@ -58,7 +55,7 @@ public int nextDoc() throws IOException { if (docId == DocIdSetIterator.NO_MORE_DOCS) { return DocIdSetIterator.NO_MORE_DOCS; } - int doc = binaryDocValues.advance(docId); + int doc = knnFloatVectorValues.advance(docId); currentScore = computeScore(); docId = bitSetIterator.nextDoc(); return doc; @@ -70,9 +67,7 @@ public float score() { } protected float computeScore() throws IOException { - final BytesRef value = binaryDocValues.binaryValue(); - final KNNVectorSerializer vectorSerializer = KNNVectorSerializerFactory.getSerializerByBytesRef(value); - final float[] vector = vectorSerializer.byteToFloatArray(value); + final float[] vector = knnFloatVectorValues.getVector(); // Calculates a similarity score between the two vectors with a specified function. Higher similarity // scores correspond to closer vectors. return spaceType.getKnnVectorSimilarityFunction().compare(queryVector, vector); diff --git a/src/main/java/org/opensearch/knn/index/query/filtered/NestedFilteredIdsKNNByteIterator.java b/src/main/java/org/opensearch/knn/index/query/filtered/NestedFilteredIdsKNNByteIterator.java index 80fba1e41..b69a90518 100644 --- a/src/main/java/org/opensearch/knn/index/query/filtered/NestedFilteredIdsKNNByteIterator.java +++ b/src/main/java/org/opensearch/knn/index/query/filtered/NestedFilteredIdsKNNByteIterator.java @@ -5,10 +5,10 @@ package org.opensearch.knn.index.query.filtered; -import org.apache.lucene.index.BinaryDocValues; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.util.BitSet; import org.opensearch.knn.index.SpaceType; +import org.opensearch.knn.index.vectorvalues.KNNBinaryVectorValues; import java.io.IOException; @@ -22,11 +22,11 @@ public class NestedFilteredIdsKNNByteIterator extends FilteredIdsKNNByteIterator public NestedFilteredIdsKNNByteIterator( final BitSet filterIdsArray, final byte[] queryVector, - final BinaryDocValues values, + final KNNBinaryVectorValues binaryVectorValues, final SpaceType spaceType, final BitSet parentBitSet ) { - super(filterIdsArray, queryVector, values, spaceType); + super(filterIdsArray, queryVector, binaryVectorValues, spaceType); this.parentBitSet = parentBitSet; } @@ -47,7 +47,7 @@ public int nextDoc() throws IOException { int bestChild = -1; while (docId != DocIdSetIterator.NO_MORE_DOCS && docId < currentParent) { - binaryDocValues.advance(docId); + binaryVectorValues.advance(docId); float score = computeScore(); if (score > currentScore) { bestChild = docId; diff --git a/src/main/java/org/opensearch/knn/index/query/filtered/NestedFilteredIdsKNNIterator.java b/src/main/java/org/opensearch/knn/index/query/filtered/NestedFilteredIdsKNNIterator.java index 9776ebbe9..259b004f8 100644 --- a/src/main/java/org/opensearch/knn/index/query/filtered/NestedFilteredIdsKNNIterator.java +++ b/src/main/java/org/opensearch/knn/index/query/filtered/NestedFilteredIdsKNNIterator.java @@ -5,10 +5,10 @@ package org.opensearch.knn.index.query.filtered; -import org.apache.lucene.index.BinaryDocValues; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.util.BitSet; import org.opensearch.knn.index.SpaceType; +import org.opensearch.knn.index.vectorvalues.KNNFloatVectorValues; import java.io.IOException; @@ -22,11 +22,11 @@ public class NestedFilteredIdsKNNIterator extends FilteredIdsKNNIterator { public NestedFilteredIdsKNNIterator( final BitSet filterIdsArray, final float[] queryVector, - final BinaryDocValues values, + final KNNFloatVectorValues knnFloatVectorValues, final SpaceType spaceType, final BitSet parentBitSet ) { - super(filterIdsArray, queryVector, values, spaceType); + super(filterIdsArray, queryVector, knnFloatVectorValues, spaceType); this.parentBitSet = parentBitSet; } @@ -47,7 +47,7 @@ public int nextDoc() throws IOException { int bestChild = -1; while (docId != DocIdSetIterator.NO_MORE_DOCS && docId < currentParent) { - binaryDocValues.advance(docId); + knnFloatVectorValues.advance(docId); float score = computeScore(); if (score > currentScore) { bestChild = docId; diff --git a/src/main/java/org/opensearch/knn/index/vectorvalues/KNNVectorValuesFactory.java b/src/main/java/org/opensearch/knn/index/vectorvalues/KNNVectorValuesFactory.java index 5b6558f32..41408e217 100644 --- a/src/main/java/org/opensearch/knn/index/vectorvalues/KNNVectorValuesFactory.java +++ b/src/main/java/org/opensearch/knn/index/vectorvalues/KNNVectorValuesFactory.java @@ -5,10 +5,16 @@ package org.opensearch.knn.index.vectorvalues; +import org.apache.lucene.index.DocValues; import org.apache.lucene.index.DocsWithFieldSet; +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.LeafReader; +import org.apache.lucene.index.VectorEncoding; import org.apache.lucene.search.DocIdSetIterator; +import org.opensearch.knn.common.FieldInfoExtractor; import org.opensearch.knn.index.VectorDataType; +import java.io.IOException; import java.util.Map; /** @@ -21,7 +27,7 @@ public final class KNNVectorValuesFactory { * * @param vectorDataType {@link VectorDataType} * @param docIdSetIterator {@link DocIdSetIterator} - * @return {@link KNNVectorValues} of type float[] + * @return {@link KNNVectorValues} */ public static KNNVectorValues getVectorValues(final VectorDataType vectorDataType, final DocIdSetIterator docIdSetIterator) { return getVectorValues(vectorDataType, new KNNVectorValuesIterator.DocIdsIteratorValues(docIdSetIterator)); @@ -32,7 +38,7 @@ public static KNNVectorValues getVectorValues(final VectorDataType vector * * @param vectorDataType {@link VectorDataType} * @param docIdWithFieldSet {@link DocsWithFieldSet} - * @return {@link KNNVectorValues} of type float[] + * @return {@link KNNVectorValues} */ public static KNNVectorValues getVectorValues( final VectorDataType vectorDataType, @@ -42,6 +48,30 @@ public static KNNVectorValues getVectorValues( return getVectorValues(vectorDataType, new KNNVectorValuesIterator.FieldWriterIteratorValues(docIdWithFieldSet, vectors)); } + /** + * Returns a {@link KNNVectorValues} for the given {@link FieldInfo} and {@link LeafReader} + * + * @param fieldInfo {@link FieldInfo} + * @param leafReader {@link LeafReader} + * @return {@link KNNVectorValues} + */ + public static KNNVectorValues getVectorValues(final FieldInfo fieldInfo, final LeafReader leafReader) throws IOException { + final DocIdSetIterator docIdSetIterator; + if (fieldInfo.hasVectorValues()) { + if (fieldInfo.getVectorEncoding() == VectorEncoding.BYTE) { + docIdSetIterator = leafReader.getByteVectorValues(fieldInfo.getName()); + } else if (fieldInfo.getVectorEncoding() == VectorEncoding.FLOAT32) { + docIdSetIterator = leafReader.getFloatVectorValues(fieldInfo.getName()); + } else { + throw new IllegalArgumentException("Invalid Vector encoding provided, hence cannot return VectorValues"); + } + } else { + docIdSetIterator = DocValues.getBinary(leafReader, fieldInfo.getName()); + } + final KNNVectorValuesIterator vectorValuesIterator = new KNNVectorValuesIterator.DocIdsIteratorValues(docIdSetIterator); + return getVectorValues(FieldInfoExtractor.extractVectorDataType(fieldInfo), vectorValuesIterator); + } + @SuppressWarnings("unchecked") private static KNNVectorValues getVectorValues( final VectorDataType vectorDataType, diff --git a/src/main/java/org/opensearch/knn/indices/ModelUtil.java b/src/main/java/org/opensearch/knn/indices/ModelUtil.java index 4c6230a46..0f5a049fc 100644 --- a/src/main/java/org/opensearch/knn/indices/ModelUtil.java +++ b/src/main/java/org/opensearch/knn/indices/ModelUtil.java @@ -11,6 +11,10 @@ package org.opensearch.knn.indices; +import org.apache.commons.lang.StringUtils; + +import java.util.Locale; + /** * A utility class for models. */ @@ -33,4 +37,21 @@ public static boolean isModelCreated(ModelMetadata modelMetadata) { return modelMetadata.getState().equals(ModelState.CREATED); } + /** + * Gets Model Metadata from a given model id. + * @param modelId {@link String} + * @return {@link ModelMetadata} + */ + public static ModelMetadata getModelMetadata(final String modelId) { + if (StringUtils.isEmpty(modelId)) { + return null; + } + final Model model = ModelCache.getInstance().get(modelId); + final ModelMetadata modelMetadata = model.getModelMetadata(); + if (ModelUtil.isModelCreated(modelMetadata) == false) { + throw new IllegalArgumentException(String.format(Locale.ROOT, "Model ID '%s' is not created.", modelId)); + } + return modelMetadata; + } + } diff --git a/src/test/java/org/opensearch/knn/common/FieldInfoExtractorTests.java b/src/test/java/org/opensearch/knn/common/FieldInfoExtractorTests.java new file mode 100644 index 000000000..e86a153d3 --- /dev/null +++ b/src/test/java/org/opensearch/knn/common/FieldInfoExtractorTests.java @@ -0,0 +1,44 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.common; + +import org.apache.lucene.index.FieldInfo; +import org.junit.Assert; +import org.mockito.MockedStatic; +import org.mockito.Mockito; +import org.opensearch.knn.KNNTestCase; +import org.opensearch.knn.index.VectorDataType; +import org.opensearch.knn.indices.ModelMetadata; +import org.opensearch.knn.indices.ModelUtil; + +public class FieldInfoExtractorTests extends KNNTestCase { + + private static final String MODEL_ID = "model_id"; + + public void testExtractVectorDataType_whenDifferentConditions_thenSuccess() { + FieldInfo fieldInfo = Mockito.mock(FieldInfo.class); + MockedStatic modelUtilMockedStatic = Mockito.mockStatic(ModelUtil.class); + + // default case + Mockito.when(fieldInfo.getAttribute(KNNConstants.VECTOR_DATA_TYPE_FIELD)).thenReturn(null); + Mockito.when(fieldInfo.getAttribute(KNNConstants.MODEL_ID)).thenReturn(MODEL_ID); + modelUtilMockedStatic.when(() -> ModelUtil.getModelMetadata(MODEL_ID)).thenReturn(null); + Assert.assertEquals(VectorDataType.DEFAULT, FieldInfoExtractor.extractVectorDataType(fieldInfo)); + + // VectorDataType present in fieldInfo + Mockito.when(fieldInfo.getAttribute(KNNConstants.VECTOR_DATA_TYPE_FIELD)).thenReturn(VectorDataType.BINARY.getValue()); + Assert.assertEquals(VectorDataType.BINARY, FieldInfoExtractor.extractVectorDataType(fieldInfo)); + + // VectorDataType present in ModelMetadata + ModelMetadata modelMetadata = Mockito.mock(ModelMetadata.class); + Mockito.when(fieldInfo.getAttribute(KNNConstants.VECTOR_DATA_TYPE_FIELD)).thenReturn(null); + modelUtilMockedStatic.when(() -> ModelUtil.getModelMetadata(MODEL_ID)).thenReturn(modelMetadata); + Mockito.when(modelMetadata.getVectorDataType()).thenReturn(VectorDataType.BYTE); + Assert.assertEquals(VectorDataType.BYTE, FieldInfoExtractor.extractVectorDataType(fieldInfo)); + + modelUtilMockedStatic.close(); + } +} diff --git a/src/test/java/org/opensearch/knn/index/query/KNNWeightTests.java b/src/test/java/org/opensearch/knn/index/query/KNNWeightTests.java index c5abc964d..15402a148 100644 --- a/src/test/java/org/opensearch/knn/index/query/KNNWeightTests.java +++ b/src/test/java/org/opensearch/knn/index/query/KNNWeightTests.java @@ -33,6 +33,7 @@ import org.junit.Before; import org.junit.BeforeClass; import org.mockito.MockedStatic; +import org.mockito.Mockito; import org.opensearch.common.io.PathUtils; import org.opensearch.common.unit.TimeValue; import org.opensearch.core.common.unit.ByteSizeValue; @@ -46,6 +47,9 @@ import org.opensearch.knn.index.memory.NativeMemoryAllocation; import org.opensearch.knn.index.memory.NativeMemoryCacheManager; import org.opensearch.knn.index.engine.KNNEngine; +import org.opensearch.knn.index.vectorvalues.KNNBinaryVectorValues; +import org.opensearch.knn.index.vectorvalues.KNNFloatVectorValues; +import org.opensearch.knn.index.vectorvalues.KNNVectorValuesFactory; import org.opensearch.knn.indices.ModelDao; import org.opensearch.knn.indices.ModelMetadata; import org.opensearch.knn.indices.ModelState; @@ -702,7 +706,8 @@ public void validateANNWithFilterQuery_whenExactSearch_thenSuccess(final boolean ); final FieldInfos fieldInfos = mock(FieldInfos.class); final FieldInfo fieldInfo = mock(FieldInfo.class); - final BinaryDocValues binaryDocValues = mock(BinaryDocValues.class); + final KNNFloatVectorValues floatVectorValues = mock(KNNFloatVectorValues.class); + final KNNBinaryVectorValues binaryVectorValues = mock(KNNBinaryVectorValues.class); when(reader.getFieldInfos()).thenReturn(fieldInfos); when(fieldInfos.fieldInfo(any())).thenReturn(fieldInfo); when(fieldInfo.attributes()).thenReturn(attributesMap); @@ -712,14 +717,15 @@ public void validateANNWithFilterQuery_whenExactSearch_thenSuccess(final boolean when(fieldInfo.getAttribute(SPACE_TYPE)).thenReturn(SpaceType.L2.getValue()); } when(fieldInfo.getName()).thenReturn(FIELD_NAME); - when(reader.getBinaryDocValues(FIELD_NAME)).thenReturn(binaryDocValues); - when(binaryDocValues.advance(filterDocId)).thenReturn(filterDocId); - BytesRef vectorByteRef = new BytesRef(new KNNVectorAsArraySerializer().floatToByteArray(vector)); - + MockedStatic valuesFactoryMockedStatic = Mockito.mockStatic(KNNVectorValuesFactory.class); if (isBinary) { - when(binaryDocValues.binaryValue()).thenReturn(new BytesRef(byteVector)); + valuesFactoryMockedStatic.when(() -> KNNVectorValuesFactory.getVectorValues(fieldInfo, reader)).thenReturn(binaryVectorValues); + when(binaryVectorValues.advance(filterDocId)).thenReturn(filterDocId); + Mockito.when(binaryVectorValues.getVector()).thenReturn(byteVector); } else { - when(binaryDocValues.binaryValue()).thenReturn(vectorByteRef); + valuesFactoryMockedStatic.when(() -> KNNVectorValuesFactory.getVectorValues(fieldInfo, reader)).thenReturn(floatVectorValues); + when(floatVectorValues.advance(filterDocId)).thenReturn(filterDocId); + Mockito.when(floatVectorValues.getVector()).thenReturn(vector); } final KNNScorer knnScorer = (KNNScorer) knnWeight.scorer(leafReaderContext); @@ -739,6 +745,7 @@ public void validateANNWithFilterQuery_whenExactSearch_thenSuccess(final boolean } assertEquals(docIdSetIterator.cost(), actualDocIds.size()); assertTrue(Comparators.isInOrder(actualDocIds, Comparator.naturalOrder())); + valuesFactoryMockedStatic.close(); } @SneakyThrows @@ -909,16 +916,18 @@ public void testANNWithFilterQuery_whenExactSearchViaThresholdSettingOnBinaryInd ); final FieldInfos fieldInfos = mock(FieldInfos.class); final FieldInfo fieldInfo = mock(FieldInfo.class); - final BinaryDocValues binaryDocValues = mock(BinaryDocValues.class); when(reader.getFieldInfos()).thenReturn(fieldInfos); when(fieldInfos.fieldInfo(any())).thenReturn(fieldInfo); when(fieldInfo.attributes()).thenReturn(attributesMap); when(fieldInfo.getAttribute(SPACE_TYPE)).thenReturn(SpaceType.HAMMING.getValue()); when(fieldInfo.getName()).thenReturn(FIELD_NAME); - when(reader.getBinaryDocValues(FIELD_NAME)).thenReturn(binaryDocValues); - when(binaryDocValues.advance(0)).thenReturn(0); - BytesRef vectorByteRef = new BytesRef(vector); - when(binaryDocValues.binaryValue()).thenReturn(vectorByteRef); + + KNNBinaryVectorValues knnBinaryVectorValues = mock(KNNBinaryVectorValues.class); + MockedStatic vectorValuesFactoryMockedStatic = Mockito.mockStatic(KNNVectorValuesFactory.class); + vectorValuesFactoryMockedStatic.when(() -> KNNVectorValuesFactory.getVectorValues(fieldInfo, reader)) + .thenReturn(knnBinaryVectorValues); + when(knnBinaryVectorValues.advance(0)).thenReturn(0); + when(knnBinaryVectorValues.getVector()).thenReturn(vector); final KNNScorer knnScorer = (KNNScorer) knnWeight.scorer(leafReaderContext); assertNotNull(knnScorer); @@ -933,6 +942,7 @@ public void testANNWithFilterQuery_whenExactSearchViaThresholdSettingOnBinaryInd } assertEquals(docIdSetIterator.cost(), actualDocIds.size()); assertTrue(Comparators.isInOrder(actualDocIds, Comparator.naturalOrder())); + vectorValuesFactoryMockedStatic.close(); } @SneakyThrows diff --git a/src/test/java/org/opensearch/knn/index/query/filtered/FilteredIdsKNNByteIteratorTests.java b/src/test/java/org/opensearch/knn/index/query/filtered/FilteredIdsKNNByteIteratorTests.java index 7583f50bc..c52798c05 100644 --- a/src/test/java/org/opensearch/knn/index/query/filtered/FilteredIdsKNNByteIteratorTests.java +++ b/src/test/java/org/opensearch/knn/index/query/filtered/FilteredIdsKNNByteIteratorTests.java @@ -7,11 +7,10 @@ import junit.framework.TestCase; import lombok.SneakyThrows; -import org.apache.lucene.index.BinaryDocValues; import org.apache.lucene.search.DocIdSetIterator; -import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.FixedBitSet; import org.opensearch.knn.index.SpaceType; +import org.opensearch.knn.index.vectorvalues.KNNBinaryVectorValues; import java.util.Arrays; import java.util.List; @@ -31,9 +30,8 @@ public void testNextDoc_whenCalled_IterateAllDocs() { .map(vector -> spaceType.getKnnVectorSimilarityFunction().compare(queryVector, vector)) .collect(Collectors.toList()); - BinaryDocValues values = mock(BinaryDocValues.class); - final List byteRefs = dataVectors.stream().map(vector -> new BytesRef(vector)).collect(Collectors.toList()); - when(values.binaryValue()).thenReturn(byteRefs.get(0), byteRefs.get(1), byteRefs.get(2)); + KNNBinaryVectorValues values = mock(KNNBinaryVectorValues.class); + when(values.getVector()).thenReturn(dataVectors.get(0), dataVectors.get(1), dataVectors.get(2)); FixedBitSet filterBitSet = new FixedBitSet(4); for (int id : filterIds) { diff --git a/src/test/java/org/opensearch/knn/index/query/filtered/FilteredIdsKNNIteratorTests.java b/src/test/java/org/opensearch/knn/index/query/filtered/FilteredIdsKNNIteratorTests.java index cf8582a05..731eed2cc 100644 --- a/src/test/java/org/opensearch/knn/index/query/filtered/FilteredIdsKNNIteratorTests.java +++ b/src/test/java/org/opensearch/knn/index/query/filtered/FilteredIdsKNNIteratorTests.java @@ -6,13 +6,11 @@ package org.opensearch.knn.index.query.filtered; import lombok.SneakyThrows; -import org.apache.lucene.index.BinaryDocValues; import org.apache.lucene.search.DocIdSetIterator; -import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.FixedBitSet; import org.opensearch.knn.KNNTestCase; import org.opensearch.knn.index.SpaceType; -import org.opensearch.knn.index.codec.util.KNNVectorAsArraySerializer; +import org.opensearch.knn.index.vectorvalues.KNNFloatVectorValues; import java.util.Arrays; import java.util.List; @@ -36,11 +34,8 @@ public void testNextDoc_whenCalled_IterateAllDocs() { .map(vector -> spaceType.getKnnVectorSimilarityFunction().compare(queryVector, vector)) .collect(Collectors.toList()); - BinaryDocValues values = mock(BinaryDocValues.class); - final List byteRefs = dataVectors.stream() - .map(vector -> new BytesRef(new KNNVectorAsArraySerializer().floatToByteArray(vector))) - .collect(Collectors.toList()); - when(values.binaryValue()).thenReturn(byteRefs.get(0), byteRefs.get(1), byteRefs.get(2)); + KNNFloatVectorValues values = mock(KNNFloatVectorValues.class); + when(values.getVector()).thenReturn(dataVectors.get(0), dataVectors.get(1), dataVectors.get(2)); FixedBitSet filterBitSet = new FixedBitSet(4); for (int id : filterIds) { diff --git a/src/test/java/org/opensearch/knn/index/query/filtered/NestedFilteredIdsKNNByteIteratorTests.java b/src/test/java/org/opensearch/knn/index/query/filtered/NestedFilteredIdsKNNByteIteratorTests.java index c4b7859d0..1940ffe12 100644 --- a/src/test/java/org/opensearch/knn/index/query/filtered/NestedFilteredIdsKNNByteIteratorTests.java +++ b/src/test/java/org/opensearch/knn/index/query/filtered/NestedFilteredIdsKNNByteIteratorTests.java @@ -7,12 +7,11 @@ import junit.framework.TestCase; import lombok.SneakyThrows; -import org.apache.lucene.index.BinaryDocValues; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.util.BitSet; -import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.FixedBitSet; import org.opensearch.knn.index.SpaceType; +import org.opensearch.knn.index.vectorvalues.KNNBinaryVectorValues; import java.util.Arrays; import java.util.List; @@ -36,9 +35,8 @@ public void testNextDoc_whenIterate_ReturnBestChildDocsPerParent() { .map(vector -> spaceType.getKnnVectorSimilarityFunction().compare(queryVector, vector)) .collect(Collectors.toList()); - BinaryDocValues values = mock(BinaryDocValues.class); - final List byteRefs = dataVectors.stream().map(vector -> new BytesRef(vector)).collect(Collectors.toList()); - when(values.binaryValue()).thenReturn(byteRefs.get(0), byteRefs.get(1), byteRefs.get(2)); + KNNBinaryVectorValues values = mock(KNNBinaryVectorValues.class); + when(values.getVector()).thenReturn(dataVectors.get(0), dataVectors.get(1), dataVectors.get(2)); FixedBitSet filterBitSet = new FixedBitSet(4); for (int id : filterIds) { diff --git a/src/test/java/org/opensearch/knn/index/query/filtered/NestedFilteredIdsKNNIteratorTests.java b/src/test/java/org/opensearch/knn/index/query/filtered/NestedFilteredIdsKNNIteratorTests.java index 508b0d3d6..cca789a4d 100644 --- a/src/test/java/org/opensearch/knn/index/query/filtered/NestedFilteredIdsKNNIteratorTests.java +++ b/src/test/java/org/opensearch/knn/index/query/filtered/NestedFilteredIdsKNNIteratorTests.java @@ -7,13 +7,11 @@ import junit.framework.TestCase; import lombok.SneakyThrows; -import org.apache.lucene.index.BinaryDocValues; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.util.BitSet; -import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.FixedBitSet; import org.opensearch.knn.index.SpaceType; -import org.opensearch.knn.index.codec.util.KNNVectorAsArraySerializer; +import org.opensearch.knn.index.vectorvalues.KNNFloatVectorValues; import java.util.Arrays; import java.util.List; @@ -41,11 +39,12 @@ public void testNextDoc_whenIterate_ReturnBestChildDocsPerParent() { .map(vector -> spaceType.getKnnVectorSimilarityFunction().compare(queryVector, vector)) .collect(Collectors.toList()); - BinaryDocValues values = mock(BinaryDocValues.class); - final List byteRefs = dataVectors.stream() - .map(vector -> new BytesRef(new KNNVectorAsArraySerializer().floatToByteArray(vector))) - .collect(Collectors.toList()); - when(values.binaryValue()).thenReturn(byteRefs.get(0), byteRefs.get(1), byteRefs.get(2)); + KNNFloatVectorValues values = mock(KNNFloatVectorValues.class); + when(values.getVector()).thenReturn(dataVectors.get(0), dataVectors.get(1), dataVectors.get(2)); + // final List byteRefs = dataVectors.stream() + // .map(vector -> new BytesRef(new KNNVectorAsArraySerializer().floatToByteArray(vector))) + // .collect(Collectors.toList()); + // when(values.binaryValue()).thenReturn(byteRefs.get(0), byteRefs.get(1), byteRefs.get(2)); FixedBitSet filterBitSet = new FixedBitSet(4); for (int id : filterIds) { diff --git a/src/test/java/org/opensearch/knn/index/vectorvalues/KNNVectorValuesFactoryTests.java b/src/test/java/org/opensearch/knn/index/vectorvalues/KNNVectorValuesFactoryTests.java index 9827cb03b..a717aa2c2 100644 --- a/src/test/java/org/opensearch/knn/index/vectorvalues/KNNVectorValuesFactoryTests.java +++ b/src/test/java/org/opensearch/knn/index/vectorvalues/KNNVectorValuesFactoryTests.java @@ -5,12 +5,19 @@ package org.opensearch.knn.index.vectorvalues; +import lombok.SneakyThrows; import org.apache.lucene.index.BinaryDocValues; import org.apache.lucene.index.DocsWithFieldSet; +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.SegmentReader; +import org.apache.lucene.index.VectorEncoding; import org.junit.Assert; +import org.mockito.Mockito; import org.opensearch.knn.KNNTestCase; +import org.opensearch.knn.common.KNNConstants; import org.opensearch.knn.index.VectorDataType; +import java.util.List; import java.util.Map; public class KNNVectorValuesFactoryTests extends KNNTestCase { @@ -58,4 +65,88 @@ public void testGetVectorValuesUsingDocWithFieldSet_whenValidInput_thenSuccess() Assert.assertNotNull(binaryVectorValues); } + @SneakyThrows + public void testGetVectorValuesFromFieldInfo_whenVectorDimIsNotZero_thenSuccess() { + final List byteArrayList = List.of(new byte[] { 1, 2, 3 }); + final List floatArrayList = List.of(new float[] { 1.3f, 2.2f, 3.2f }); + final List binaryArrayList = List.of(new byte[] { 3, 2, 3 }); + final FieldInfo fieldInfo = Mockito.mock(FieldInfo.class); + final SegmentReader reader = Mockito.mock(SegmentReader.class); + Mockito.when(fieldInfo.hasVectorValues()).thenReturn(true); + Mockito.when(fieldInfo.getName()).thenReturn("test_field"); + + // Checking for ByteVectorValues + Mockito.when(fieldInfo.getVectorEncoding()).thenReturn(VectorEncoding.BYTE); + Mockito.when(fieldInfo.getAttribute(KNNConstants.VECTOR_DATA_TYPE_FIELD)).thenReturn(VectorDataType.BYTE.getValue()); + Mockito.when(reader.getByteVectorValues("test_field")).thenReturn(new TestVectorValues.PreDefinedByteVectorValues(byteArrayList)); + final KNNVectorValues byteVectorValues = KNNVectorValuesFactory.getVectorValues(fieldInfo, reader); + byteVectorValues.nextDoc(); + Assert.assertArrayEquals(byteArrayList.get(0), byteVectorValues.getVector()); + Assert.assertNotNull(byteVectorValues); + + // Checking for FloatVectorValues + Mockito.when(fieldInfo.getVectorEncoding()).thenReturn(VectorEncoding.FLOAT32); + Mockito.when(fieldInfo.getAttribute(KNNConstants.VECTOR_DATA_TYPE_FIELD)).thenReturn(VectorDataType.FLOAT.getValue()); + Mockito.when(reader.getFloatVectorValues("test_field")) + .thenReturn(new TestVectorValues.PreDefinedFloatVectorValues(floatArrayList)); + final KNNVectorValues floatVectorValues = KNNVectorValuesFactory.getVectorValues(fieldInfo, reader); + floatVectorValues.nextDoc(); + Assert.assertArrayEquals(floatArrayList.get(0), floatVectorValues.getVector(), 0.0f); + Assert.assertNotNull(floatVectorValues); + + // Checking for BinaryVectorValues + Mockito.when(fieldInfo.getVectorEncoding()).thenReturn(VectorEncoding.BYTE); + Mockito.when(fieldInfo.getAttribute(KNNConstants.VECTOR_DATA_TYPE_FIELD)).thenReturn(VectorDataType.BINARY.getValue()); + Mockito.when(reader.getByteVectorValues("test_field")) + .thenReturn(new TestVectorValues.PreDefinedBinaryVectorValues(binaryArrayList)); + final KNNVectorValues binaryVectorValues = KNNVectorValuesFactory.getVectorValues(fieldInfo, reader); + binaryVectorValues.nextDoc(); + Assert.assertArrayEquals(binaryArrayList.get(0), binaryVectorValues.getVector()); + Assert.assertNotNull(binaryVectorValues); + + } + + @SneakyThrows + public void testGetVectorValuesFromFieldInfo_whenVectorDimIsZero_thenSuccess() { + final List byteArrayList = List.of(new byte[] { 1, 2, 3 }); + final List floatArrayList = List.of(new float[] { 1.3f, 2.2f, 3.2f }); + final List binaryArrayList = List.of(new byte[] { 3, 2, 3 }); + final FieldInfo fieldInfo = Mockito.mock(FieldInfo.class); + final SegmentReader reader = Mockito.mock(SegmentReader.class); + Mockito.when(fieldInfo.hasVectorValues()).thenReturn(false); + Mockito.when(fieldInfo.getName()).thenReturn("test_field"); + + // Checking for ByteVectorValues + Mockito.when(fieldInfo.getAttribute(KNNConstants.VECTOR_DATA_TYPE_FIELD)).thenReturn(VectorDataType.BYTE.getValue()); + Mockito.when(reader.getBinaryDocValues("test_field")) + .thenReturn(new TestVectorValues.PredefinedByteVectorBinaryDocValues(byteArrayList)); + + final KNNVectorValues byteVectorValues = KNNVectorValuesFactory.getVectorValues(fieldInfo, reader); + byteVectorValues.nextDoc(); + Assert.assertArrayEquals(byteArrayList.get(0), byteVectorValues.getVector()); + Assert.assertNotNull(byteVectorValues); + + // Checking for Floats with BinaryDocValues + Mockito.when(fieldInfo.getAttribute(KNNConstants.VECTOR_DATA_TYPE_FIELD)).thenReturn(VectorDataType.FLOAT.getValue()); + Mockito.when(reader.getBinaryDocValues("test_field")) + .thenReturn(new TestVectorValues.PredefinedFloatVectorBinaryDocValues(floatArrayList)); + + final KNNVectorValues floatVectorValues = KNNVectorValuesFactory.getVectorValues(fieldInfo, reader); + floatVectorValues.nextDoc(); + Assert.assertArrayEquals(floatArrayList.get(0), floatVectorValues.getVector(), 0.0f); + Assert.assertNotNull(floatVectorValues); + + // Checking for BinaryVectorValues + Mockito.when(fieldInfo.getAttribute(KNNConstants.VECTOR_DATA_TYPE_FIELD)).thenReturn(VectorDataType.BINARY.getValue()); + Mockito.when(reader.getBinaryDocValues("test_field")) + .thenReturn(new TestVectorValues.PredefinedByteVectorBinaryDocValues(binaryArrayList)); + + final KNNVectorValues binaryVectorValues = KNNVectorValuesFactory.getVectorValues(fieldInfo, reader); + binaryVectorValues.nextDoc(); + Assert.assertArrayEquals(binaryArrayList.get(0), binaryVectorValues.getVector()); + Assert.assertNotNull(binaryVectorValues); + + Mockito.verify(fieldInfo, Mockito.times(0)).getVectorEncoding(); + } + } diff --git a/src/test/java/org/opensearch/knn/indices/ModelUtilTests.java b/src/test/java/org/opensearch/knn/indices/ModelUtilTests.java new file mode 100644 index 000000000..edefd10ee --- /dev/null +++ b/src/test/java/org/opensearch/knn/indices/ModelUtilTests.java @@ -0,0 +1,36 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.indices; + +import org.junit.Assert; +import org.mockito.MockedStatic; +import org.mockito.Mockito; +import org.opensearch.knn.KNNTestCase; + +public class ModelUtilTests extends KNNTestCase { + private static final String MODEL_ID = "test-model"; + + public void testGetModelMetadata_whenVariousInputs_thenSuccess() { + Assert.assertNull(ModelUtil.getModelMetadata(null)); + Assert.assertNull(ModelUtil.getModelMetadata("")); + + ModelCache modelCache = Mockito.mock(ModelCache.class); + Model model = Mockito.mock(Model.class); + ModelMetadata modelMetadata = Mockito.mock(ModelMetadata.class); + MockedStatic modelCacheMockedStatic = Mockito.mockStatic(ModelCache.class); + + modelCacheMockedStatic.when(ModelCache::getInstance).thenReturn(modelCache); + + Mockito.when(modelCache.get(MODEL_ID)).thenReturn(model); + Mockito.when(model.getModelMetadata()).thenReturn(null); + Assert.assertThrows(IllegalArgumentException.class, () -> ModelUtil.getModelMetadata(MODEL_ID)); + + Mockito.when(model.getModelMetadata()).thenReturn(modelMetadata); + Mockito.when(modelMetadata.getState()).thenReturn(ModelState.CREATED); + Assert.assertNotNull(ModelUtil.getModelMetadata(MODEL_ID)); + modelCacheMockedStatic.close(); + } +} From 3e9ceccfc31f41f381d84357e46618629369772c Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Thu, 15 Aug 2024 12:19:07 -0700 Subject: [PATCH 325/416] BackPort Java Doc Fix with Code Improvements (#1959) (#1961) (cherry picked from commit 3557d79ab940cfd2d8439a86d3189e0817d63e33) Signed-off-by: Ryan Bogan Co-authored-by: Vikasht34 --- .../factory/QuantizerFactory.java | 35 ++++++++++--------- .../factory/QuantizerRegistry.java | 9 +++-- .../ScalarQuantizationParams.java | 21 ++++++++--- .../factory/QuantizerFactoryTests.java | 12 +++---- .../factory/QuantizerRegistryTests.java | 26 +++++++------- 5 files changed, 57 insertions(+), 46 deletions(-) diff --git a/src/main/java/org/opensearch/knn/quantization/factory/QuantizerFactory.java b/src/main/java/org/opensearch/knn/quantization/factory/QuantizerFactory.java index b99f6ebdc..6705c7688 100644 --- a/src/main/java/org/opensearch/knn/quantization/factory/QuantizerFactory.java +++ b/src/main/java/org/opensearch/knn/quantization/factory/QuantizerFactory.java @@ -21,34 +21,35 @@ public final class QuantizerFactory { private static final AtomicBoolean isRegistered = new AtomicBoolean(false); - /** - * Ensures that default quantizers are registered. - */ - private static void ensureRegistered() { - if (!isRegistered.get()) { - synchronized (QuantizerFactory.class) { - if (!isRegistered.get()) { - QuantizerRegistrar.registerDefaultQuantizers(); - isRegistered.set(true); - } - } - } - } - /** * Retrieves a quantizer instance based on the provided quantization parameters. * * @param params the quantization parameters used to determine the appropriate quantizer * @param

    the type of quantization parameters, extending {@link QuantizationParams} - * @param the type of the quantized output + * @param the type of the input vector to be quantized + * @param the type of the output after quantization * @return an instance of {@link Quantizer} corresponding to the provided parameters */ - public static

    Quantizer getQuantizer(final P params) { + public static

    Quantizer getQuantizer(final P params) { if (params == null) { throw new IllegalArgumentException("Quantization parameters must not be null."); } // Lazy Registration instead of static block as class level; ensureRegistered(); - return QuantizerRegistry.getQuantizer(params); + return (Quantizer) QuantizerRegistry.getQuantizer(params); + } + + /** + * Ensures that default quantizers are registered. + */ + private static void ensureRegistered() { + if (!isRegistered.get()) { + synchronized (QuantizerFactory.class) { + if (!isRegistered.get()) { + QuantizerRegistrar.registerDefaultQuantizers(); + isRegistered.set(true); + } + } + } } } diff --git a/src/main/java/org/opensearch/knn/quantization/factory/QuantizerRegistry.java b/src/main/java/org/opensearch/knn/quantization/factory/QuantizerRegistry.java index ac266f547..2da830f3b 100644 --- a/src/main/java/org/opensearch/knn/quantization/factory/QuantizerRegistry.java +++ b/src/main/java/org/opensearch/knn/quantization/factory/QuantizerRegistry.java @@ -42,18 +42,17 @@ static void register(final String paramIdentifier, final Quantizer quantiz * * @param params the quantization parameters used to determine the appropriate quantizer * @param

    the type of quantization parameters - * @param the type of the quantized output + * @param the type of the input vector to be quantized + * @param the type of the output after quantization * @return an instance of {@link Quantizer} corresponding to the provided parameters * @throws IllegalArgumentException if no quantizer is registered for the given parameters */ - static

    Quantizer getQuantizer(final P params) { + static

    Quantizer getQuantizer(final P params) { String identifier = params.getTypeIdentifier(); Quantizer quantizer = registry.get(identifier); if (quantizer == null) { throw new IllegalArgumentException("No quantizer registered for type identifier: " + identifier); } - @SuppressWarnings("unchecked") - Quantizer typedQuantizer = (Quantizer) quantizer; - return typedQuantizer; + return (Quantizer) quantizer; } } diff --git a/src/main/java/org/opensearch/knn/quantization/models/quantizationParams/ScalarQuantizationParams.java b/src/main/java/org/opensearch/knn/quantization/models/quantizationParams/ScalarQuantizationParams.java index 4e7a53892..881c2132d 100644 --- a/src/main/java/org/opensearch/knn/quantization/models/quantizationParams/ScalarQuantizationParams.java +++ b/src/main/java/org/opensearch/knn/quantization/models/quantizationParams/ScalarQuantizationParams.java @@ -47,10 +47,6 @@ public String getTypeIdentifier() { return generateIdentifier(sqType.getId()); } - private static String generateIdentifier(int id) { - return "ScalarQuantizationParams_" + id; - } - /** * Writes the object to the output stream. * This method is part of the Writeable interface and is used to serialize the object. @@ -74,4 +70,21 @@ public ScalarQuantizationParams(StreamInput in, int version) throws IOException int typeId = in.readVInt(); this.sqType = ScalarQuantizationType.fromId(typeId); } + + /** + * Generates a unique identifier for Scalar Quantization Parameters. + * + *

    + * This method constructs an identifier string by prefixing the given integer ID + * with "ScalarQuantizationParams_". The resulting string can be used to uniquely + * identify specific quantization parameter instances, especially when registering + * or retrieving them in a registry or similar structure. + *

    + * + * @param id the integer ID to be used in generating the unique identifier. + * @return a string representing the unique identifier for the quantization parameters. + */ + private static String generateIdentifier(int id) { + return "ScalarQuantizationParams_" + id; + } } diff --git a/src/test/java/org/opensearch/knn/quantization/factory/QuantizerFactoryTests.java b/src/test/java/org/opensearch/knn/quantization/factory/QuantizerFactoryTests.java index 3474b7ec9..b95123e21 100644 --- a/src/test/java/org/opensearch/knn/quantization/factory/QuantizerFactoryTests.java +++ b/src/test/java/org/opensearch/knn/quantization/factory/QuantizerFactoryTests.java @@ -31,12 +31,12 @@ public void test_Lazy_Registration() { ScalarQuantizationParams paramsTwoBit = new ScalarQuantizationParams(ScalarQuantizationType.TWO_BIT); ScalarQuantizationParams paramsFourBit = new ScalarQuantizationParams(ScalarQuantizationType.FOUR_BIT); assertFalse(isRegisteredFieldAccessible()); - Quantizer quantizer = QuantizerFactory.getQuantizer(params); - Quantizer quantizerTwoBit = QuantizerFactory.getQuantizer(paramsTwoBit); - Quantizer quantizerFourBit = QuantizerFactory.getQuantizer(paramsFourBit); - assertTrue(quantizerFourBit instanceof MultiBitScalarQuantizer); - assertTrue(quantizerTwoBit instanceof MultiBitScalarQuantizer); - assertTrue(quantizer instanceof OneBitScalarQuantizer); + Quantizer oneBitQuantizer = QuantizerFactory.getQuantizer(params); + Quantizer quantizerTwoBit = QuantizerFactory.getQuantizer(paramsTwoBit); + Quantizer quantizerFourBit = QuantizerFactory.getQuantizer(paramsFourBit); + assertEquals(quantizerFourBit.getClass(), MultiBitScalarQuantizer.class); + assertEquals(quantizerTwoBit.getClass(), MultiBitScalarQuantizer.class); + assertEquals(oneBitQuantizer.getClass(), OneBitScalarQuantizer.class); assertTrue(isRegisteredFieldAccessible()); } diff --git a/src/test/java/org/opensearch/knn/quantization/factory/QuantizerRegistryTests.java b/src/test/java/org/opensearch/knn/quantization/factory/QuantizerRegistryTests.java index dec34e632..62d31ab61 100644 --- a/src/test/java/org/opensearch/knn/quantization/factory/QuantizerRegistryTests.java +++ b/src/test/java/org/opensearch/knn/quantization/factory/QuantizerRegistryTests.java @@ -34,39 +34,37 @@ public static void setup() { public void testRegisterAndGetQuantizer() { // Test for OneBitScalarQuantizer ScalarQuantizationParams oneBitParams = new ScalarQuantizationParams(ScalarQuantizationType.ONE_BIT); - Quantizer oneBitQuantizer = QuantizerRegistry.getQuantizer(oneBitParams); - assertTrue(oneBitQuantizer instanceof OneBitScalarQuantizer); + Quantizer oneBitQuantizer = QuantizerRegistry.getQuantizer(oneBitParams); + assertEquals(oneBitQuantizer.getClass(), OneBitScalarQuantizer.class); // Test for MultiBitScalarQuantizer (2-bit) ScalarQuantizationParams twoBitParams = new ScalarQuantizationParams(ScalarQuantizationType.TWO_BIT); - Quantizer twoBitQuantizer = QuantizerRegistry.getQuantizer(twoBitParams); - assertTrue(twoBitQuantizer instanceof MultiBitScalarQuantizer); - assertEquals(2, ((MultiBitScalarQuantizer) twoBitQuantizer).getBitsPerCoordinate()); + Quantizer twoBitQuantizer = QuantizerRegistry.getQuantizer(twoBitParams); + assertEquals(twoBitQuantizer.getClass(), MultiBitScalarQuantizer.class); // Test for MultiBitScalarQuantizer (4-bit) ScalarQuantizationParams fourBitParams = new ScalarQuantizationParams(ScalarQuantizationType.FOUR_BIT); - Quantizer fourBitQuantizer = QuantizerRegistry.getQuantizer(fourBitParams); - assertTrue(fourBitQuantizer instanceof MultiBitScalarQuantizer); - assertEquals(4, ((MultiBitScalarQuantizer) fourBitQuantizer).getBitsPerCoordinate()); + Quantizer fourBitQuantizer = QuantizerRegistry.getQuantizer(fourBitParams); + assertEquals(fourBitQuantizer.getClass(), MultiBitScalarQuantizer.class); } public void testQuantizerRegistryIsSingleton() { // Ensure the same instance is returned for the same type identifier ScalarQuantizationParams oneBitParams = new ScalarQuantizationParams(ScalarQuantizationType.ONE_BIT); - Quantizer firstOneBitQuantizer = QuantizerRegistry.getQuantizer(oneBitParams); - Quantizer secondOneBitQuantizer = QuantizerRegistry.getQuantizer(oneBitParams); + Quantizer firstOneBitQuantizer = QuantizerRegistry.getQuantizer(oneBitParams); + Quantizer secondOneBitQuantizer = QuantizerRegistry.getQuantizer(oneBitParams); assertSame(firstOneBitQuantizer, secondOneBitQuantizer); // Ensure the same instance is returned for the same type identifier (2-bit) ScalarQuantizationParams twoBitParams = new ScalarQuantizationParams(ScalarQuantizationType.TWO_BIT); - Quantizer firstTwoBitQuantizer = QuantizerRegistry.getQuantizer(twoBitParams); - Quantizer secondTwoBitQuantizer = QuantizerRegistry.getQuantizer(twoBitParams); + Quantizer firstTwoBitQuantizer = QuantizerRegistry.getQuantizer(twoBitParams); + Quantizer secondTwoBitQuantizer = QuantizerRegistry.getQuantizer(twoBitParams); assertSame(firstTwoBitQuantizer, secondTwoBitQuantizer); // Ensure the same instance is returned for the same type identifier (4-bit) ScalarQuantizationParams fourBitParams = new ScalarQuantizationParams(ScalarQuantizationType.FOUR_BIT); - Quantizer firstFourBitQuantizer = QuantizerRegistry.getQuantizer(fourBitParams); - Quantizer secondFourBitQuantizer = QuantizerRegistry.getQuantizer(fourBitParams); + Quantizer firstFourBitQuantizer = QuantizerRegistry.getQuantizer(fourBitParams); + Quantizer secondFourBitQuantizer = QuantizerRegistry.getQuantizer(fourBitParams); assertSame(firstFourBitQuantizer, secondFourBitQuantizer); } From d7833007c6412693481bcee5b1999fe60227ea26 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Thu, 15 Aug 2024 12:19:14 -0700 Subject: [PATCH 326/416] Quantization Framework Code Structure Improvement (#1967) (#1968) * BackPort Java Doc Fix with Code Improvements * Quantization Framework Code Structure Improvement --------- (cherry picked from commit b56675320cbc9786bb6d8d0f3066c32b071f5a14) Signed-off-by: VIKASH TIWARI Signed-off-by: Ryan Bogan Co-authored-by: Vikasht34 --- .../BinaryQuantizationOutput.java | 53 ++++---- .../QuantizationOutput.java | 16 ++- .../quantizer/MultiBitScalarQuantizer.java | 6 +- .../quantizer/OneBitScalarQuantizer.java | 8 +- .../MultiBitScalarQuantizerTests.java | 126 +++++++++++++++++- .../quantizer/OneBitScalarQuantizerTests.java | 113 +++++++++++++++- 6 files changed, 277 insertions(+), 45 deletions(-) diff --git a/src/main/java/org/opensearch/knn/quantization/models/quantizationOutput/BinaryQuantizationOutput.java b/src/main/java/org/opensearch/knn/quantization/models/quantizationOutput/BinaryQuantizationOutput.java index 95592fcb9..388fd9e94 100644 --- a/src/main/java/org/opensearch/knn/quantization/models/quantizationOutput/BinaryQuantizationOutput.java +++ b/src/main/java/org/opensearch/knn/quantization/models/quantizationOutput/BinaryQuantizationOutput.java @@ -6,53 +6,52 @@ package org.opensearch.knn.quantization.models.quantizationOutput; import lombok.Getter; -import lombok.NoArgsConstructor; import java.util.Arrays; +import lombok.RequiredArgsConstructor; /** * The BinaryQuantizationOutput class represents the output of a quantization process in binary format. * It implements the QuantizationOutput interface to handle byte arrays specifically. */ -@NoArgsConstructor +@Getter +@RequiredArgsConstructor public class BinaryQuantizationOutput implements QuantizationOutput { - @Getter private byte[] quantizedVector; + private final int bitsPerCoordinate; + private int currentVectorLength = -1; // Indicates uninitialized state /** - * Prepares the quantized vector array based on the provided parameters and returns it for direct modification. - * This method ensures that the internal byte array is appropriately sized and cleared before being used. - * The method accepts two parameters: - *
      - *
    • bitsPerCoordinate: The number of bits used per coordinate. This determines the granularity of the quantization.
      • - *
    • vectorLength: The length of the original vector that needs to be quantized. This helps in calculating the required byte array size.
      • - *
    - * If the existing quantized vector is either null or not the same size as the required byte array, - * a new byte array is allocated. Otherwise, the existing array is cleared (i.e., all bytes are set to zero). - * This method is designed to be used in conjunction with a bit-packing utility that writes quantized values directly - * into the returned byte array. - * @param params an array of parameters, where the first parameter is the number of bits per coordinate (int), - * and the second parameter is the length of the vector (int). - * @return the prepared and writable quantized vector as a byte array. - * @throws IllegalArgumentException if the parameters are not as expected (e.g., missing or not integers). + * Prepares the quantized vector based on the vector length. + * This includes initializing or resetting the quantized vector. + * + * @param vectorLength The length of the vector to be quantized. */ @Override - public byte[] prepareAndGetWritableQuantizedVector(Object... params) { - if (params.length != 2 || !(params[0] instanceof Integer) || !(params[1] instanceof Integer)) { - throw new IllegalArgumentException("Expected two integer parameters: bitsPerCoordinate and vectorLength"); + public void prepareQuantizedVector(int vectorLength) { + if (vectorLength <= 0) { + throw new IllegalArgumentException("Vector length must be greater than zero."); } - int bitsPerCoordinate = (int) params[0]; - int vectorLength = (int) params[1]; - int totalBits = bitsPerCoordinate * vectorLength; - int byteLength = (totalBits + 7) >> 3; - if (this.quantizedVector == null || this.quantizedVector.length != byteLength) { + if (vectorLength != currentVectorLength) { + int totalBits = bitsPerCoordinate * vectorLength; + int byteLength = (totalBits + 7) >> 3; this.quantizedVector = new byte[byteLength]; + this.currentVectorLength = vectorLength; } else { Arrays.fill(this.quantizedVector, (byte) 0); } + } - return this.quantizedVector; + /** + * Checks if the quantized vector has already been prepared for the given vector length. + * + * @param vectorLength The length of the vector to be quantized. + * @return true if the quantized vector is already prepared, false otherwise. + */ + @Override + public boolean isPrepared(int vectorLength) { + return vectorLength == currentVectorLength && quantizedVector != null; } /** diff --git a/src/main/java/org/opensearch/knn/quantization/models/quantizationOutput/QuantizationOutput.java b/src/main/java/org/opensearch/knn/quantization/models/quantizationOutput/QuantizationOutput.java index aa81a8821..4d088f91f 100644 --- a/src/main/java/org/opensearch/knn/quantization/models/quantizationOutput/QuantizationOutput.java +++ b/src/main/java/org/opensearch/knn/quantization/models/quantizationOutput/QuantizationOutput.java @@ -19,10 +19,18 @@ public interface QuantizationOutput { T getQuantizedVector(); /** - * Prepares and returns the writable quantized vector for direct modification. + * Prepares the quantized vector based on the vector length. + * This includes initializing or resetting the quantized vector. * - * @param params the parameters needed for preparing the quantized vector. - * @return the prepared and writable quantized vector. + * @param vectorLength The length of the vector to be quantized. */ - T prepareAndGetWritableQuantizedVector(Object... params); + void prepareQuantizedVector(int vectorLength); + + /** + * Checks if the quantized vector has already been prepared for the given vector length. + * + * @param vectorLength The length of the vector to be quantized. + * @return true if the quantized vector is already prepared, false otherwise. + */ + boolean isPrepared(int vectorLength); } diff --git a/src/main/java/org/opensearch/knn/quantization/quantizer/MultiBitScalarQuantizer.java b/src/main/java/org/opensearch/knn/quantization/quantizer/MultiBitScalarQuantizer.java index dcf825a6a..12a5d1013 100644 --- a/src/main/java/org/opensearch/knn/quantization/quantizer/MultiBitScalarQuantizer.java +++ b/src/main/java/org/opensearch/knn/quantization/quantizer/MultiBitScalarQuantizer.java @@ -133,14 +133,14 @@ public void quantize(final float[] vector, final QuantizationState state, final throw new IllegalArgumentException("Vector to quantize must not be null."); } validateState(state); + int vectorLength = vector.length; MultiBitScalarQuantizationState multiBitState = (MultiBitScalarQuantizationState) state; float[][] thresholds = multiBitState.getThresholds(); if (thresholds == null || thresholds[0].length != vector.length) { throw new IllegalArgumentException("Thresholds must not be null and must match the dimension of the vector."); } - // Prepare and get the writable array - byte[] writableArray = output.prepareAndGetWritableQuantizedVector(bitsPerCoordinate, vector.length); - BitPacker.quantizeAndPackBits(vector, thresholds, bitsPerCoordinate, writableArray); + if (!output.isPrepared(vectorLength)) output.prepareQuantizedVector(vectorLength); + BitPacker.quantizeAndPackBits(vector, thresholds, bitsPerCoordinate, output.getQuantizedVector()); } /** diff --git a/src/main/java/org/opensearch/knn/quantization/quantizer/OneBitScalarQuantizer.java b/src/main/java/org/opensearch/knn/quantization/quantizer/OneBitScalarQuantizer.java index 41602dfd2..a0f6a26b4 100644 --- a/src/main/java/org/opensearch/knn/quantization/quantizer/OneBitScalarQuantizer.java +++ b/src/main/java/org/opensearch/knn/quantization/quantizer/OneBitScalarQuantizer.java @@ -76,14 +76,14 @@ public void quantize(final float[] vector, final QuantizationState state, final throw new IllegalArgumentException("Vector to quantize must not be null."); } validateState(state); + int vectorLength = vector.length; OneBitScalarQuantizationState binaryState = (OneBitScalarQuantizationState) state; float[] thresholds = binaryState.getMeanThresholds(); - if (thresholds == null || thresholds.length != vector.length) { + if (thresholds == null || thresholds.length != vectorLength) { throw new IllegalArgumentException("Thresholds must not be null and must match the dimension of the vector."); } - // Prepare and get the writable array - byte[] writableArray = output.prepareAndGetWritableQuantizedVector(1, vector.length); - BitPacker.quantizeAndPackBits(vector, thresholds, writableArray); + if (!output.isPrepared(vectorLength)) output.prepareQuantizedVector(vectorLength); + BitPacker.quantizeAndPackBits(vector, thresholds, output.getQuantizedVector()); } /** diff --git a/src/test/java/org/opensearch/knn/quantization/quantizer/MultiBitScalarQuantizerTests.java b/src/test/java/org/opensearch/knn/quantization/quantizer/MultiBitScalarQuantizerTests.java index ad6a44686..45acaf357 100644 --- a/src/test/java/org/opensearch/knn/quantization/quantizer/MultiBitScalarQuantizerTests.java +++ b/src/test/java/org/opensearch/knn/quantization/quantizer/MultiBitScalarQuantizerTests.java @@ -56,8 +56,9 @@ public void testQuantize_twoBit() throws IOException { ScalarQuantizationParams params = new ScalarQuantizationParams(ScalarQuantizationType.TWO_BIT); MultiBitScalarQuantizationState state = new MultiBitScalarQuantizationState(params, thresholds); - BinaryQuantizationOutput output = new BinaryQuantizationOutput(); + BinaryQuantizationOutput output = new BinaryQuantizationOutput(2); twoBitQuantizer.quantize(vector, state, output); + assertNotNull(output.getQuantizedVector()); } @@ -72,14 +73,16 @@ public void testQuantize_fourBit() throws IOException { ScalarQuantizationParams params = new ScalarQuantizationParams(ScalarQuantizationType.FOUR_BIT); MultiBitScalarQuantizationState state = new MultiBitScalarQuantizationState(params, thresholds); - BinaryQuantizationOutput output = new BinaryQuantizationOutput(); + BinaryQuantizationOutput output = new BinaryQuantizationOutput(4); fourBitQuantizer.quantize(vector, state, output); + assertNotNull(output.getQuantizedVector()); } public void testQuantize_withNullVector() throws IOException { MultiBitScalarQuantizer twoBitQuantizer = new MultiBitScalarQuantizer(2); - BinaryQuantizationOutput output = new BinaryQuantizationOutput(); + BinaryQuantizationOutput output = new BinaryQuantizationOutput(2); + output.prepareQuantizedVector(8); // Example length expectThrows( IllegalArgumentException.class, () -> twoBitQuantizer.quantize( @@ -90,6 +93,123 @@ public void testQuantize_withNullVector() throws IOException { ); } + public void testQuantize_twoBit_multiple_times() throws IOException { + MultiBitScalarQuantizer twoBitQuantizer = new MultiBitScalarQuantizer(2); + float[] vector = { -2.5f, 1.5f, -0.5f, 4.0f, 6.5f, -3.5f, 0.0f, 7.2f }; + float[][] thresholds = { + { -3.0f, 1.0f, -1.0f, 3.5f, 5.0f, -4.0f, 0.5f, 7.0f }, + { -2.0f, 2.0f, 0.0f, 4.5f, 6.0f, -2.5f, -0.5f, 8.0f } }; + ScalarQuantizationParams params = new ScalarQuantizationParams(ScalarQuantizationType.TWO_BIT); + MultiBitScalarQuantizationState state = new MultiBitScalarQuantizationState(params, thresholds); + + BinaryQuantizationOutput output = new BinaryQuantizationOutput(2); + + // First quantization + twoBitQuantizer.quantize(vector, state, output); + assertNotNull(output.getQuantizedVector()); + + // Save the reference to the byte array + byte[] firstByteArray = output.getQuantizedVector(); + + // Expected output after the first quantization + byte[] expectedPackedBits = new byte[] { (byte) 0b11111101, (byte) 0b00001010 }; + + // Check the output value after the first quantization + assertArrayEquals(expectedPackedBits, output.getQuantizedVector()); + + // Modify vector for a second quantization call + vector = new float[] { -2.5f, 1.5f, -0.5f, 4.0f, 6.5f, -3.5f, 0.0f, 7.2f }; + + // Second quantization + twoBitQuantizer.quantize(vector, state, output); + + // Assert that the same byte array reference is used + assertSame(firstByteArray, output.getQuantizedVector()); + + // Expected output after the second quantization (based on updated vector) + assertArrayEquals(expectedPackedBits, output.getQuantizedVector()); + } + + public void testQuantize_ReuseByteArray_forMultiBitQuantizer() throws IOException { + MultiBitScalarQuantizer twoBitQuantizer = new MultiBitScalarQuantizer(2); + float[] vector = { -2.5f, 1.5f, -0.5f, 4.0f, 6.5f, -3.5f, 0.0f, 7.2f }; + float[][] thresholds = { + { -3.0f, 1.0f, -1.0f, 3.5f, 5.0f, -4.0f, 0.5f, 7.0f }, + { -2.0f, 2.0f, 0.0f, 4.5f, 6.0f, -2.5f, -0.5f, 8.0f } }; + ScalarQuantizationParams params = new ScalarQuantizationParams(ScalarQuantizationType.TWO_BIT); + MultiBitScalarQuantizationState state = new MultiBitScalarQuantizationState(params, thresholds); + + BinaryQuantizationOutput output = new BinaryQuantizationOutput(2); + + // First quantization + twoBitQuantizer.quantize(vector, state, output); + byte[] firstByteArray = output.getQuantizedVector(); + + // Expected output after the first quantization + byte[] expectedPackedBits = new byte[] { (byte) 0b11111101, (byte) 0b00001010 }; + + // Check the output value after the first quantization + assertArrayEquals(expectedPackedBits, output.getQuantizedVector()); + + // Second quantization with the same vector length + twoBitQuantizer.quantize(vector, state, output); + byte[] secondByteArray = output.getQuantizedVector(); + + // Assert that the same byte array reference is used + assertSame(firstByteArray, secondByteArray); + + // Check the output value after the second quantization + assertArrayEquals(expectedPackedBits, output.getQuantizedVector()); + + // Third quantization with the same vector length + twoBitQuantizer.quantize(vector, state, output); + byte[] thirdByteArray = output.getQuantizedVector(); + + // Assert that the same byte array reference is still used + assertSame(firstByteArray, thirdByteArray); + + // Check the output value after the third quantization + assertArrayEquals(expectedPackedBits, output.getQuantizedVector()); + } + + public void testQuantize_withMultipleVectors_inLoop() throws IOException { + MultiBitScalarQuantizer twoBitQuantizer = new MultiBitScalarQuantizer(2); + float[][] vectors = { + { -2.5f, 1.5f, -0.5f, 4.0f, 6.5f, -3.5f, 0.0f, 7.2f }, + { 2.0f, -1.0f, 3.5f, 0.0f, 5.5f, -2.5f, 1.5f, 6.0f }, + { -4.0f, 2.0f, -1.5f, 3.5f, -0.5f, 1.0f, 2.5f, -3.0f } }; + float[][] thresholds = { + { -3.0f, 1.0f, -1.0f, 3.5f, 5.0f, -4.0f, 0.5f, 7.0f }, + { -2.0f, 2.0f, 0.0f, 4.5f, 6.0f, -2.5f, -0.5f, 8.0f } }; + ScalarQuantizationParams params = new ScalarQuantizationParams(ScalarQuantizationType.TWO_BIT); + MultiBitScalarQuantizationState state = new MultiBitScalarQuantizationState(params, thresholds); + + BinaryQuantizationOutput output = new BinaryQuantizationOutput(2); + + byte[] previousByteArray = null; + for (float[] vector : vectors) { + // Check if output is already prepared before quantization + boolean wasPrepared = output.isPrepared(vector.length); + + // Prepare the output for the new vector length + output.prepareQuantizedVector(vector.length); + + // Ensure that if it was prepared, it stays the same reference + if (wasPrepared) { + assertSame(previousByteArray, output.getQuantizedVector()); + } + + // Perform the quantization + twoBitQuantizer.quantize(vector, state, output); + + // Save the reference to the byte array after quantization + previousByteArray = output.getQuantizedVector(); + + // Check that the output vector is correctly prepared + assertTrue(output.isPrepared(vector.length)); + } + } + // Mock classes for testing private static class MockTrainingRequest extends TrainingRequest { private final float[][] vectors; diff --git a/src/test/java/org/opensearch/knn/quantization/quantizer/OneBitScalarQuantizerTests.java b/src/test/java/org/opensearch/knn/quantization/quantizer/OneBitScalarQuantizerTests.java index 28be260d7..6f8c2de87 100644 --- a/src/test/java/org/opensearch/knn/quantization/quantizer/OneBitScalarQuantizerTests.java +++ b/src/test/java/org/opensearch/knn/quantization/quantizer/OneBitScalarQuantizerTests.java @@ -9,6 +9,7 @@ import org.opensearch.knn.KNNTestCase; import org.opensearch.knn.quantization.enums.ScalarQuantizationType; import org.opensearch.knn.quantization.models.quantizationOutput.BinaryQuantizationOutput; +import org.opensearch.knn.quantization.models.quantizationOutput.QuantizationOutput; import org.opensearch.knn.quantization.models.quantizationParams.ScalarQuantizationParams; import org.opensearch.knn.quantization.models.quantizationState.OneBitScalarQuantizationState; import org.opensearch.knn.quantization.models.quantizationState.QuantizationState; @@ -48,7 +49,8 @@ public void testQuantize_withState() throws IOException { ); OneBitScalarQuantizer quantizer = new OneBitScalarQuantizer(); - BinaryQuantizationOutput output = new BinaryQuantizationOutput(); + BinaryQuantizationOutput output = new BinaryQuantizationOutput(1); + quantizer.quantize(vector, state, output); assertNotNull(output); @@ -62,7 +64,7 @@ public void testQuantize_withNullVector() throws IOException { new ScalarQuantizationParams(ScalarQuantizationType.ONE_BIT), new float[] { 0.0f } ); - BinaryQuantizationOutput output = new BinaryQuantizationOutput(); + BinaryQuantizationOutput output = new BinaryQuantizationOutput(1); expectThrows(IllegalArgumentException.class, () -> quantizer.quantize(null, state, output)); } @@ -85,7 +87,7 @@ public void writeTo(StreamOutput out) throws IOException { // Empty implementation for test } }; - BinaryQuantizationOutput output = new BinaryQuantizationOutput(); + BinaryQuantizationOutput output = new BinaryQuantizationOutput(1); expectThrows(IllegalArgumentException.class, () -> quantizer.quantize(vector, invalidState, output)); } @@ -97,7 +99,7 @@ public void testQuantize_withMismatchedDimensions() throws IOException { new ScalarQuantizationParams(ScalarQuantizationType.ONE_BIT), thresholds ); - BinaryQuantizationOutput output = new BinaryQuantizationOutput(); + QuantizationOutput output = new BinaryQuantizationOutput(1); expectThrows(IllegalArgumentException.class, () -> quantizer.quantize(vector, state, output)); } @@ -133,4 +135,107 @@ public float[] getVectorByDocId(int docId) { int[] sampledIndices = sampler.sample(vectors.length, 3); expectThrows(IllegalArgumentException.class, () -> QuantizerHelper.calculateMeanThresholds(samplingRequest, sampledIndices)); } + + public void testQuantize_withState_multiple_times() throws IOException { + float[] vector = { 3.0f, 6.0f, 9.0f }; + float[] thresholds = { 4.0f, 5.0f, 6.0f }; + OneBitScalarQuantizationState state = new OneBitScalarQuantizationState( + new ScalarQuantizationParams(ScalarQuantizationType.ONE_BIT), + thresholds + ); + + OneBitScalarQuantizer quantizer = new OneBitScalarQuantizer(); + BinaryQuantizationOutput output = new BinaryQuantizationOutput(1); + + // First quantization + quantizer.quantize(vector, state, output); + assertNotNull(output); + byte[] expectedPackedBits = new byte[] { 0b01100000 }; // or 96 in decimal + assertArrayEquals(expectedPackedBits, output.getQuantizedVector()); + + // Save the reference to the byte array + byte[] firstByteArray = output.getQuantizedVector(); + + // Modify vector and thresholds for a second quantization call + vector = new float[] { 7.0f, 8.0f, 9.0f }; + thresholds = new float[] { 6.0f, 7.0f, 8.0f }; + state = new OneBitScalarQuantizationState(new ScalarQuantizationParams(ScalarQuantizationType.ONE_BIT), thresholds); + + // Second quantization + output.prepareQuantizedVector(vector.length); // Ensure it is prepared for the new vector + quantizer.quantize(vector, state, output); + + // Assert that the same byte array reference is used + assertSame(firstByteArray, output.getQuantizedVector()); + + // Check the new output + expectedPackedBits = new byte[] { (byte) 0b11100000 }; // or 224 in decimal + assertArrayEquals(expectedPackedBits, output.getQuantizedVector()); + } + + public void testQuantize_ReuseByteArray() throws IOException { + float[] vector = { 3.0f, 6.0f, 9.0f }; + float[] thresholds = { 4.0f, 5.0f, 6.0f }; + OneBitScalarQuantizationState state = new OneBitScalarQuantizationState( + new ScalarQuantizationParams(ScalarQuantizationType.ONE_BIT), + thresholds + ); + + OneBitScalarQuantizer quantizer = new OneBitScalarQuantizer(); + BinaryQuantizationOutput output = new BinaryQuantizationOutput(1); + output.prepareQuantizedVector(vector.length); + + // First quantization + quantizer.quantize(vector, state, output); + byte[] firstByteArray = output.getQuantizedVector(); + + // Second quantization with the same vector length + output.prepareQuantizedVector(vector.length); // Reuse the prepared output + byte[] secondByteArray = output.getQuantizedVector(); + + // Assert that the same byte array reference is used + assertSame(firstByteArray, secondByteArray); + + // Third quantization with the same vector length + output.prepareQuantizedVector(vector.length); // Reuse the prepared output again + quantizer.quantize(vector, state, output); + byte[] thirdByteArray = output.getQuantizedVector(); + + // Assert that the same byte array reference is still used + assertSame(firstByteArray, thirdByteArray); + } + + public void testQuantize_withMultipleVectors_inLoop() throws IOException { + OneBitScalarQuantizer oneBitQuantizer = new OneBitScalarQuantizer(); + float[][] vectors = { { 1.0f, 2.0f, 3.0f, 4.0f }, { 2.0f, 3.0f, 4.0f, 5.0f }, { 1.5f, 2.5f, 3.5f, 4.5f } }; + float[] thresholds = { 1.5f, 2.5f, 3.5f, 4.5f }; + + ScalarQuantizationParams params = new ScalarQuantizationParams(ScalarQuantizationType.ONE_BIT); + OneBitScalarQuantizationState state = new OneBitScalarQuantizationState(params, thresholds); + + BinaryQuantizationOutput output = new BinaryQuantizationOutput(1); + + byte[] previousByteArray = null; + for (float[] vector : vectors) { + // Check if output is already prepared before quantization + boolean wasPrepared = output.isPrepared(vector.length); + + // Prepare the output for the new vector length + output.prepareQuantizedVector(vector.length); + + // Ensure that if it was prepared, it stays the same reference + if (wasPrepared) { + assertSame(previousByteArray, output.getQuantizedVector()); + } + + // Perform the quantization + oneBitQuantizer.quantize(vector, state, output); + + // Save the reference to the byte array after quantization + previousByteArray = output.getQuantizedVector(); + + // Check that the output vector is correctly prepared + assertTrue(output.isPrepared(vector.length)); + } + } } From bf93ce7962598ee737e26b037e72ba9cb7dc7982 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Thu, 15 Aug 2024 15:54:57 -0700 Subject: [PATCH 327/416] [Backport 2.x] Adds rescore parameter to KNNQuery (#1974) * Adds rescore parameter to KNNQuery (#1969) Adds rescore parameter to knn query. With this commit, the rescore is a no-op. The functionality and validation will be added in a later commit. Signed-off-by: John Mazanec (cherry picked from commit 9db70580ed0562445c9045dc9ba7f500e430a936) * fix version Signed-off-by: John Mazanec --------- Signed-off-by: John Mazanec Co-authored-by: John Mazanec --- .../knn/index/query/KNNQueryBuilder.java | 38 ++++- .../query/parser/KNNQueryBuilderParser.java | 24 +++ .../knn/index/query/parser/RescoreParser.java | 131 ++++++++++++++++ .../index/query/rescore/RescoreContext.java | 33 ++++ .../opensearch/knn/index/util/IndexUtil.java | 3 + .../KNNQueryBuilderInvalidParamsTests.java | 12 +- .../knn/index/query/KNNQueryBuilderTests.java | 41 +++-- .../KNNQueryBuilderValidParamsTests.java | 20 ++- .../parser/KNNQueryBuilderParserTests.java | 35 +++++ .../query/parser/RescoreParserTests.java | 97 ++++++++++++ .../query/parser/RescoreValidationTests.java | 45 ++++++ .../opensearch/knn/integ/QueryParseIT.java | 148 ++++++++++++++++++ 12 files changed, 611 insertions(+), 16 deletions(-) create mode 100644 src/main/java/org/opensearch/knn/index/query/parser/RescoreParser.java create mode 100644 src/main/java/org/opensearch/knn/index/query/rescore/RescoreContext.java create mode 100644 src/test/java/org/opensearch/knn/index/query/parser/RescoreParserTests.java create mode 100644 src/test/java/org/opensearch/knn/index/query/parser/RescoreValidationTests.java create mode 100644 src/test/java/org/opensearch/knn/integ/QueryParseIT.java diff --git a/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java b/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java index 80833751e..af8e410d4 100644 --- a/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java +++ b/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java @@ -26,6 +26,8 @@ import org.opensearch.knn.index.engine.model.QueryContext; import org.opensearch.knn.index.mapper.KNNMappingConfig; import org.opensearch.knn.index.mapper.KNNVectorFieldType; +import org.opensearch.knn.index.query.parser.RescoreParser; +import org.opensearch.knn.index.query.rescore.RescoreContext; import org.opensearch.knn.index.util.IndexUtil; import org.opensearch.knn.index.engine.MethodComponentContext; import org.opensearch.knn.index.SpaceType; @@ -54,6 +56,8 @@ import static org.opensearch.knn.index.query.parser.MethodParametersParser.validateMethodParameters; import static org.opensearch.knn.index.engine.KNNEngine.ENGINES_SUPPORTING_RADIAL_SEARCH; import static org.opensearch.knn.index.engine.validation.ParameterValidator.validateParameters; +import static org.opensearch.knn.index.query.parser.RescoreParser.RESCORE_OVERSAMPLE_PARAMETER; +import static org.opensearch.knn.index.query.parser.RescoreParser.RESCORE_PARAMETER; /** * Helper class to build the KNN query @@ -73,6 +77,8 @@ public class KNNQueryBuilder extends AbstractQueryBuilder { public static final ParseField EF_SEARCH_FIELD = new ParseField(METHOD_PARAMETER_EF_SEARCH); public static final ParseField NPROBE_FIELD = new ParseField(METHOD_PARAMETER_NPROBES); public static final ParseField METHOD_PARAMS_FIELD = new ParseField(METHOD_PARAMETER); + public static final ParseField RESCORE_FIELD = new ParseField(RESCORE_PARAMETER); + public static final ParseField RESCORE_OVERSAMPLE_FIELD = new ParseField(RESCORE_OVERSAMPLE_PARAMETER); public static final int K_MAX = 10000; /** @@ -96,6 +102,8 @@ public class KNNQueryBuilder extends AbstractQueryBuilder { private QueryBuilder filter; @Getter private boolean ignoreUnmapped; + @Getter + private RescoreContext rescoreContext; /** * Constructs a new query with the given field name and vector @@ -136,6 +144,7 @@ public static class Builder { private boolean ignoreUnmapped; private String queryName; private float boost = DEFAULT_BOOST; + private RescoreContext rescoreContext; public Builder() {} @@ -189,11 +198,25 @@ public Builder boost(float boost) { return this; } + public Builder rescoreContext(RescoreContext rescoreContext) { + this.rescoreContext = rescoreContext; + return this; + } + public KNNQueryBuilder build() { validate(); int k = this.k == null ? 0 : this.k; - return new KNNQueryBuilder(fieldName, vector, k, maxDistance, minScore, methodParameters, filter, ignoreUnmapped).boost(boost) - .queryName(queryName); + return new KNNQueryBuilder( + fieldName, + vector, + k, + maxDistance, + minScore, + methodParameters, + filter, + ignoreUnmapped, + rescoreContext + ).boost(boost).queryName(queryName); } private void validate() { @@ -240,6 +263,15 @@ private void validate() { ); } } + + if (rescoreContext != null) { + ValidationException validationException = RescoreParser.validate(rescoreContext); + if (validationException != null) { + throw new IllegalArgumentException( + String.format(Locale.ROOT, "[%s] errors in rescore parameter [%s]", NAME, validationException.getMessage()) + ); + } + } } } @@ -284,6 +316,7 @@ public KNNQueryBuilder(String fieldName, float[] vector, int k, QueryBuilder fil this.ignoreUnmapped = false; this.maxDistance = null; this.minScore = null; + this.rescoreContext = null; } public static void initialize(ModelDao modelDao) { @@ -305,6 +338,7 @@ public KNNQueryBuilder(StreamInput in) throws IOException { maxDistance = builder.maxDistance; minScore = builder.minScore; methodParameters = builder.methodParameters; + rescoreContext = builder.rescoreContext; } @Override diff --git a/src/main/java/org/opensearch/knn/index/query/parser/KNNQueryBuilderParser.java b/src/main/java/org/opensearch/knn/index/query/parser/KNNQueryBuilderParser.java index 217166aa9..159480b72 100644 --- a/src/main/java/org/opensearch/knn/index/query/parser/KNNQueryBuilderParser.java +++ b/src/main/java/org/opensearch/knn/index/query/parser/KNNQueryBuilderParser.java @@ -16,6 +16,7 @@ import org.opensearch.core.xcontent.XContentParser; import org.opensearch.index.query.QueryBuilder; import org.opensearch.knn.common.KNNConstants; +import org.opensearch.knn.index.query.rescore.RescoreContext; import org.opensearch.knn.index.util.IndexUtil; import org.opensearch.knn.index.query.KNNQueryBuilder; @@ -29,6 +30,8 @@ import static org.opensearch.index.query.AbstractQueryBuilder.NAME_FIELD; import static org.opensearch.index.query.AbstractQueryBuilder.parseInnerQueryBuilder; import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER; +import static org.opensearch.knn.index.query.KNNQueryBuilder.RESCORE_FIELD; +import static org.opensearch.knn.index.query.parser.RescoreParser.RESCORE_PARAMETER; import static org.opensearch.knn.index.util.IndexUtil.isClusterOnOrAfterMinRequiredVersion; import static org.opensearch.knn.index.query.KNNQueryBuilder.FILTER_FIELD; import static org.opensearch.knn.index.query.KNNQueryBuilder.IGNORE_UNMAPPED_FIELD; @@ -79,6 +82,17 @@ private static ObjectParser createInternalObjectP ); internalParser.declareObject(KNNQueryBuilder.Builder::filter, (p, v) -> parseInnerQueryBuilder(p), FILTER_FIELD); + internalParser.declareObjectOrDefault( + KNNQueryBuilder.Builder::rescoreContext, + (p, v) -> RescoreParser.fromXContent(p), + RescoreContext::getDefault, + RESCORE_FIELD + ); + + // Declare fields that cannot be set at the same time. Right now, rescore and radial is not supported + internalParser.declareExclusiveFieldSet(RESCORE_FIELD.getPreferredName(), MAX_DISTANCE_FIELD.getPreferredName()); + internalParser.declareExclusiveFieldSet(RESCORE_FIELD.getPreferredName(), MIN_SCORE_FIELD.getPreferredName()); + return internalParser; } @@ -113,6 +127,10 @@ public static KNNQueryBuilder.Builder streamInput(StreamInput in, Function INTERNAL_PARSER = createInternalObjectParser(); + + private static ObjectParser createInternalObjectParser() { + ObjectParser internalParser = new ObjectParser<>( + RESCORE_PARAMETER, + RescoreContext::builder + ); + internalParser.declareFloat(RescoreContext.RescoreContextBuilder::oversampleFactor, RESCORE_OVERSAMPLE_FIELD); + return internalParser; + } + + /** + * Validate the rescore context + * + * @return ValidationException if validation fails, null otherwise + */ + public static ValidationException validate(RescoreContext rescoreContext) { + if (rescoreContext.getOversampleFactor() < RescoreContext.MIN_OVERSAMPLE_FACTOR) { + ValidationException validationException = new ValidationException(); + validationException.addValidationError( + String.format( + Locale.ROOT, + "Oversample factor [%f] cannot be less than [%f]", + rescoreContext.getOversampleFactor(), + RescoreContext.MIN_OVERSAMPLE_FACTOR + ) + ); + return validationException; + } + + if (rescoreContext.getOversampleFactor() > RescoreContext.MAX_OVERSAMPLE_FACTOR) { + ValidationException validationException = new ValidationException(); + validationException.addValidationError( + String.format( + Locale.ROOT, + "Oversample factor [%f] cannot be more than [%f]", + rescoreContext.getOversampleFactor(), + RescoreContext.MAX_OVERSAMPLE_FACTOR + ) + ); + return validationException; + } + return null; + } + + /** + * + * @param in stream input + * @return RescoreContext + * @throws IOException on stream failure + */ + public static RescoreContext streamInput(StreamInput in) throws IOException { + if (!IndexUtil.isVersionOnOrAfterMinRequiredVersion(in.getVersion(), RESCORE_PARAMETER)) { + return null; + } + Float oversample = in.readOptionalFloat(); + if (oversample == null) { + return null; + } + return RescoreContext.builder().oversampleFactor(oversample).build(); + } + + /** + * + * @param out stream output + * @param rescoreContext RescoreContext + * @throws IOException on stream failure + */ + public static void streamOutput(StreamOutput out, RescoreContext rescoreContext) throws IOException { + if (!IndexUtil.isVersionOnOrAfterMinRequiredVersion(out.getVersion(), RESCORE_PARAMETER)) { + return; + } + out.writeOptionalFloat(rescoreContext == null ? null : rescoreContext.getOversampleFactor()); + } + + /** + * + * @param builder XContentBuilder + * @param rescoreContext RescoreContext + * @throws IOException on XContent failure + */ + public static void doXContent(final XContentBuilder builder, final RescoreContext rescoreContext) throws IOException { + builder.startObject(RESCORE_PARAMETER); + builder.field(RESCORE_OVERSAMPLE_PARAMETER, rescoreContext.getOversampleFactor()); + builder.endObject(); + } + + /** + * + * @param parser input parser + * @return RescoreContext + */ + public static RescoreContext fromXContent(final XContentParser parser) { + return INTERNAL_PARSER.apply(parser, null).build(); + } +} diff --git a/src/main/java/org/opensearch/knn/index/query/rescore/RescoreContext.java b/src/main/java/org/opensearch/knn/index/query/rescore/RescoreContext.java new file mode 100644 index 000000000..82b09807a --- /dev/null +++ b/src/main/java/org/opensearch/knn/index/query/rescore/RescoreContext.java @@ -0,0 +1,33 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.query.rescore; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.EqualsAndHashCode; +import lombok.Getter; + +@Getter +@AllArgsConstructor +@Builder +@EqualsAndHashCode +public final class RescoreContext { + + public static final float DEFAULT_OVERSAMPLE_FACTOR = 1.0f; + public static final float MAX_OVERSAMPLE_FACTOR = 100.0f; + public static final float MIN_OVERSAMPLE_FACTOR = 0.0f; + + @Builder.Default + private float oversampleFactor = DEFAULT_OVERSAMPLE_FACTOR; + + /** + * + * @return default RescoreContext + */ + public static RescoreContext getDefault() { + return RescoreContext.builder().build(); + } +} diff --git a/src/main/java/org/opensearch/knn/index/util/IndexUtil.java b/src/main/java/org/opensearch/knn/index/util/IndexUtil.java index 924f4ab19..15e1959f8 100644 --- a/src/main/java/org/opensearch/knn/index/util/IndexUtil.java +++ b/src/main/java/org/opensearch/knn/index/util/IndexUtil.java @@ -36,6 +36,7 @@ import static org.opensearch.knn.common.KNNConstants.HNSW_ALGO_EF_SEARCH; import static org.opensearch.knn.common.KNNConstants.SPACE_TYPE; import static org.opensearch.knn.common.KNNConstants.VECTOR_DATA_TYPE_FIELD; +import static org.opensearch.knn.index.query.parser.RescoreParser.RESCORE_PARAMETER; public class IndexUtil { @@ -49,6 +50,7 @@ public class IndexUtil { private static final Version MINIMAL_SUPPORTED_VERSION_FOR_RADIAL_SEARCH = Version.V_2_14_0; private static final Version MINIMAL_SUPPORTED_VERSION_FOR_METHOD_PARAMETERS = Version.V_2_16_0; private static final Version MINIMAL_SUPPORTED_VERSION_FOR_MODEL_VECTOR_DATA_TYPE = Version.V_2_16_0; + private static final Version MINIMAL_RESCORE_FEATURE = Version.V_2_17_0; // public so neural search can access it public static final Map minimalRequiredVersionMap = initializeMinimalRequiredVersionMap(); @@ -404,6 +406,7 @@ private static Map initializeMinimalRequiredVersionMap() { put(KNNConstants.RADIAL_SEARCH_KEY, MINIMAL_SUPPORTED_VERSION_FOR_RADIAL_SEARCH); put(KNNConstants.METHOD_PARAMETER, MINIMAL_SUPPORTED_VERSION_FOR_METHOD_PARAMETERS); put(KNNConstants.MODEL_VECTOR_DATA_TYPE_KEY, MINIMAL_SUPPORTED_VERSION_FOR_MODEL_VECTOR_DATA_TYPE); + put(RESCORE_PARAMETER, MINIMAL_RESCORE_FEATURE); } }; diff --git a/src/test/java/org/opensearch/knn/index/query/KNNQueryBuilderInvalidParamsTests.java b/src/test/java/org/opensearch/knn/index/query/KNNQueryBuilderInvalidParamsTests.java index 74c1cca58..29f2d2368 100644 --- a/src/test/java/org/opensearch/knn/index/query/KNNQueryBuilderInvalidParamsTests.java +++ b/src/test/java/org/opensearch/knn/index/query/KNNQueryBuilderInvalidParamsTests.java @@ -8,6 +8,7 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; import lombok.AllArgsConstructor; import org.opensearch.knn.KNNTestCase; +import org.opensearch.knn.index.query.rescore.RescoreContext; import java.util.Arrays; import java.util.Collection; @@ -86,6 +87,15 @@ public static Collection invalidParameters() { "min score less than 0", "[knn] requires minScore to be greater than 0", KNNQueryBuilder.builder().fieldName(FIELD_NAME).vector(QUERY_VECTOR).minScore(-1f) + ), + $( + "Rescore context", + " cannot be less than", + KNNQueryBuilder.builder() + .rescoreContext(RescoreContext.builder().oversampleFactor(RescoreContext.MIN_OVERSAMPLE_FACTOR - 1).build()) + .fieldName(FIELD_NAME) + .vector(QUERY_VECTOR) + .k(1) ) ) ); @@ -93,6 +103,6 @@ public static Collection invalidParameters() { public void testInvalidBuilder() { Throwable exception = expectThrows(IllegalArgumentException.class, () -> knnQueryBuilderBuilder.build()); - assertEquals(expectedMessage, expectedMessage, exception.getMessage()); + assertTrue(exception.getMessage(), exception.getMessage().contains(expectedMessage)); } } diff --git a/src/test/java/org/opensearch/knn/index/query/KNNQueryBuilderTests.java b/src/test/java/org/opensearch/knn/index/query/KNNQueryBuilderTests.java index 25982fb7d..762a36227 100644 --- a/src/test/java/org/opensearch/knn/index/query/KNNQueryBuilderTests.java +++ b/src/test/java/org/opensearch/knn/index/query/KNNQueryBuilderTests.java @@ -30,6 +30,7 @@ import org.opensearch.index.query.TermQueryBuilder; import org.opensearch.knn.KNNTestCase; import org.opensearch.knn.index.mapper.KNNVectorFieldType; +import org.opensearch.knn.index.query.rescore.RescoreContext; import org.opensearch.knn.index.util.KNNClusterUtil; import org.opensearch.knn.index.engine.KNNMethodContext; import org.opensearch.knn.index.engine.MethodComponentContext; @@ -776,19 +777,23 @@ public void testDoToQuery_InvalidZeroByteVector() { public void testSerialization() throws Exception { // For k-NN search - assertSerialization(Version.CURRENT, Optional.empty(), K, null, null, null); - assertSerialization(Version.CURRENT, Optional.empty(), K, Map.of("ef_search", EF_SEARCH), null, null); - assertSerialization(Version.CURRENT, Optional.of(TERM_QUERY), K, Map.of("ef_search", EF_SEARCH), null, null); - assertSerialization(Version.V_2_3_0, Optional.empty(), K, Map.of("ef_search", EF_SEARCH), null, null); - assertSerialization(Version.V_2_3_0, Optional.empty(), K, null, null, null); + assertSerialization(Version.CURRENT, Optional.empty(), K, null, null, null, null); + assertSerialization(Version.CURRENT, Optional.empty(), K, Map.of("ef_search", EF_SEARCH), null, null, null); + assertSerialization(Version.CURRENT, Optional.of(TERM_QUERY), K, Map.of("ef_search", EF_SEARCH), null, null, null); + assertSerialization(Version.V_2_3_0, Optional.empty(), K, Map.of("ef_search", EF_SEARCH), null, null, null); + assertSerialization(Version.V_2_3_0, Optional.empty(), K, null, null, null, null); // For distance threshold search - assertSerialization(Version.CURRENT, Optional.empty(), null, null, null, MAX_DISTANCE); - assertSerialization(Version.CURRENT, Optional.of(TERM_QUERY), null, null, null, MAX_DISTANCE); + assertSerialization(Version.CURRENT, Optional.empty(), null, null, null, MAX_DISTANCE, null); + assertSerialization(Version.CURRENT, Optional.of(TERM_QUERY), null, null, null, MAX_DISTANCE, null); // For score threshold search - assertSerialization(Version.CURRENT, Optional.empty(), null, null, null, MIN_SCORE); - assertSerialization(Version.CURRENT, Optional.of(TERM_QUERY), null, null, null, MIN_SCORE); + assertSerialization(Version.CURRENT, Optional.empty(), null, null, null, MIN_SCORE, null); + assertSerialization(Version.CURRENT, Optional.of(TERM_QUERY), null, null, null, MIN_SCORE, null); + + // Test rescore + assertSerialization(Version.V_2_3_0, Optional.empty(), K, null, null, null, RescoreContext.getDefault()); + assertSerialization(Version.CURRENT, Optional.empty(), K, null, null, null, RescoreContext.getDefault()); } private void assertSerialization( @@ -797,7 +802,8 @@ private void assertSerialization( Integer k, Map methodParameters, Float distance, - Float score + Float score, + RescoreContext rescoreContext ) throws Exception { final KNNQueryBuilder knnQueryBuilder = KNNQueryBuilder.builder() .fieldName(FIELD_NAME) @@ -807,6 +813,7 @@ private void assertSerialization( .k(k) .methodParameters(methodParameters) .filter(queryBuilderOptional.orElse(null)) + .rescoreContext(rescoreContext) .build(); final ClusterService clusterService = mockClusterService(version); @@ -818,7 +825,7 @@ private void assertSerialization( output.writeNamedWriteable(knnQueryBuilder); try (StreamInput in = new NamedWriteableAwareStreamInput(output.bytes().streamInput(), writableRegistry())) { - in.setVersion(Version.CURRENT); + in.setVersion(version); final QueryBuilder deserializedQuery = in.readNamedWriteable(QueryBuilder.class); assertNotNull(deserializedQuery); @@ -840,6 +847,7 @@ private void assertSerialization( assertNull(deserializedKnnQueryBuilder.getFilter()); } assertMethodParameters(version, methodParameters, deserializedKnnQueryBuilder.getMethodParameters()); + assertRescore(version, rescoreContext, deserializedKnnQueryBuilder.getRescoreContext()); } } } @@ -854,6 +862,17 @@ private void assertMethodParameters(Version version, Map expectedMeth } } + private void assertRescore(Version version, RescoreContext expectedRescoreContext, RescoreContext actualRescoreContext) { + if (!version.onOrAfter(Version.V_2_17_0)) { + assertNull(actualRescoreContext); + return; + } + + if (expectedRescoreContext != null) { + assertEquals(expectedRescoreContext, actualRescoreContext); + } + } + public void testIgnoreUnmapped() throws IOException { float[] queryVector = { 1.0f, 2.0f, 3.0f, 4.0f }; KNNQueryBuilder.Builder knnQueryBuilder = KNNQueryBuilder.builder() diff --git a/src/test/java/org/opensearch/knn/index/query/KNNQueryBuilderValidParamsTests.java b/src/test/java/org/opensearch/knn/index/query/KNNQueryBuilderValidParamsTests.java index 4b97df4b4..7b5e59224 100644 --- a/src/test/java/org/opensearch/knn/index/query/KNNQueryBuilderValidParamsTests.java +++ b/src/test/java/org/opensearch/knn/index/query/KNNQueryBuilderValidParamsTests.java @@ -8,6 +8,7 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; import lombok.AllArgsConstructor; import org.opensearch.knn.KNNTestCase; +import org.opensearch.knn.index.query.rescore.RescoreContext; import java.util.Arrays; import java.util.Collection; @@ -28,8 +29,9 @@ public class KNNQueryBuilderValidParamsTests extends KNNTestCase { private Map methodParameters; private Float maxDistance; private Float minScore; + private RescoreContext rescoreContext; - @ParametersFactory(argumentFormatting = "description:%1$s; k:%3$s, efSearch:%4$s, maxDist:%5$s, minScore:%6$s") + @ParametersFactory(argumentFormatting = "description:%1$s; k:%3$s, efSearch:%4$s, maxDist:%5$s, minScore:%6$s, rescoreContext:%6$s") public static Collection validParameters() { return Arrays.asList( $$( @@ -39,6 +41,7 @@ public static Collection validParameters() { 10, null, null, + null, null ), $( @@ -52,6 +55,7 @@ public static Collection validParameters() { 10, Map.of("ef_search", 12), null, + null, null ), $( @@ -60,6 +64,7 @@ public static Collection validParameters() { null, null, 10.0f, + null, null ), $( @@ -68,7 +73,17 @@ public static Collection validParameters() { null, null, null, - 10.0f + 10.0f, + null + ), + $( + "valid knn with rescore", + KNNQueryBuilder.builder().fieldName(FIELD_NAME).vector(QUERY_VECTOR).minScore(10.0f).build(), + null, + null, + null, + 10.0f, + RescoreContext.getDefault() ) ) ); @@ -84,6 +99,7 @@ public void testValidBuilder() { .methodParameters(methodParameters) .maxDistance(maxDistance) .minScore(minScore) + .rescoreContext(rescoreContext) .build() ); } diff --git a/src/test/java/org/opensearch/knn/index/query/parser/KNNQueryBuilderParserTests.java b/src/test/java/org/opensearch/knn/index/query/parser/KNNQueryBuilderParserTests.java index 713e532f9..6cac5580b 100644 --- a/src/test/java/org/opensearch/knn/index/query/parser/KNNQueryBuilderParserTests.java +++ b/src/test/java/org/opensearch/knn/index/query/parser/KNNQueryBuilderParserTests.java @@ -17,6 +17,7 @@ import org.opensearch.index.query.QueryBuilders; import org.opensearch.index.query.TermQueryBuilder; import org.opensearch.knn.KNNTestCase; +import org.opensearch.knn.index.query.rescore.RescoreContext; import org.opensearch.knn.index.util.KNNClusterUtil; import org.opensearch.knn.index.query.KNNQueryBuilder; import org.opensearch.plugins.SearchPlugin; @@ -31,6 +32,8 @@ import static org.opensearch.knn.index.KNNClusterTestUtils.mockClusterService; import static org.opensearch.knn.index.query.KNNQueryBuilder.NAME; import static org.opensearch.knn.index.query.KNNQueryBuilder.EF_SEARCH_FIELD; +import static org.opensearch.knn.index.query.parser.RescoreParser.RESCORE_OVERSAMPLE_PARAMETER; +import static org.opensearch.knn.index.query.parser.RescoreParser.RESCORE_PARAMETER; public class KNNQueryBuilderParserTests extends KNNTestCase { @@ -470,6 +473,38 @@ public void testToXContent_whenMethodParams_thenSucceed() throws IOException { assertEquals(builder.toString(), testBuilder.toString()); } + public void testToXContent_whenRescore_thenSucceed() throws IOException { + float[] queryVector = { 1.0f, 2.0f, 3.0f, 4.0f }; + float oversample = 1.0f; + XContentBuilder builderFromObject = XContentFactory.jsonBuilder() + .startObject() + .startObject(NAME) + .startObject(FIELD_NAME) + .field(KNNQueryBuilder.VECTOR_FIELD.getPreferredName(), queryVector) + .field(KNNQueryBuilder.K_FIELD.getPreferredName(), K) + .startObject(RESCORE_PARAMETER) + .field(RESCORE_OVERSAMPLE_PARAMETER, oversample) + .endObject() + .field(BOOST_FIELD.getPreferredName(), BOOST) + .endObject() + .endObject() + .endObject(); + + KNNQueryBuilder knnQueryBuilderFromObject = KNNQueryBuilder.builder() + .fieldName(FIELD_NAME) + .vector(queryVector) + .boost(BOOST) + .k(K) + .rescoreContext(RescoreContext.builder().oversampleFactor(oversample).build()) + .build(); + + XContentBuilder testBuilder = XContentFactory.jsonBuilder(); + testBuilder.startObject(); + KNNQueryBuilderParser.toXContent(testBuilder, EMPTY_PARAMS, knnQueryBuilderFromObject); + testBuilder.endObject(); + assertEquals(builderFromObject.toString(), testBuilder.toString()); + } + @Override protected NamedXContentRegistry xContentRegistry() { List list = ClusterModule.getNamedXWriteables(); diff --git a/src/test/java/org/opensearch/knn/index/query/parser/RescoreParserTests.java b/src/test/java/org/opensearch/knn/index/query/parser/RescoreParserTests.java new file mode 100644 index 000000000..2bb1f89fc --- /dev/null +++ b/src/test/java/org/opensearch/knn/index/query/parser/RescoreParserTests.java @@ -0,0 +1,97 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.query.parser; + +import lombok.SneakyThrows; +import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.core.common.io.stream.NamedWriteableAwareStreamInput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.knn.KNNTestCase; +import org.opensearch.knn.index.query.rescore.RescoreContext; + +import java.io.IOException; + +import static org.opensearch.knn.index.query.parser.RescoreParser.RESCORE_OVERSAMPLE_PARAMETER; +import static org.opensearch.knn.index.query.parser.RescoreParser.RESCORE_PARAMETER; + +public class RescoreParserTests extends KNNTestCase { + + @SneakyThrows + public void testStreams() { + RescoreContext rescoreContext = RescoreContext.builder().oversampleFactor(RescoreContext.DEFAULT_OVERSAMPLE_FACTOR).build(); + validateStreams(rescoreContext); + validateStreams(null); + } + + private void validateStreams(RescoreContext rescoreContext) throws IOException { + try (BytesStreamOutput output = new BytesStreamOutput()) { + RescoreParser.streamOutput(output, rescoreContext); + + try (StreamInput in = new NamedWriteableAwareStreamInput(output.bytes().streamInput(), writableRegistry())) { + RescoreContext parsedRescoreContext = RescoreParser.streamInput(in); + assertEquals(rescoreContext, parsedRescoreContext); + } + } + } + + @SneakyThrows + public void testDoXContent() { + float oversample = RescoreContext.MAX_OVERSAMPLE_FACTOR - 1; + XContentBuilder expectedBuilder = XContentFactory.jsonBuilder() + .startObject() + .startObject(RESCORE_PARAMETER) + .field(RESCORE_OVERSAMPLE_PARAMETER, oversample) + .endObject() + .endObject(); + + XContentBuilder builder = XContentFactory.jsonBuilder().startObject(); + RescoreParser.doXContent(builder, RescoreContext.builder().oversampleFactor(oversample).build()); + builder.endObject(); + assertEquals(expectedBuilder.toString(), builder.toString()); + } + + @SneakyThrows + public void testFromXContent_whenValid_thenSucceed() { + float oversample1 = RescoreContext.MAX_OVERSAMPLE_FACTOR - 1; + XContentBuilder builder1 = XContentFactory.jsonBuilder().startObject().field(RESCORE_OVERSAMPLE_PARAMETER, oversample1).endObject(); + validateOversample(oversample1, builder1); + XContentBuilder builder2 = XContentFactory.jsonBuilder().startObject().endObject(); + validateOversample(RescoreContext.DEFAULT_OVERSAMPLE_FACTOR, builder2); + } + + @SneakyThrows + public void testFromXContent_whenInvalid_thenFail() { + XContentBuilder invalidParamBuilder = XContentFactory.jsonBuilder().startObject().field("invalid", 0).endObject(); + expectValidationException(invalidParamBuilder); + + XContentBuilder invalidParamValueBuilder = XContentFactory.jsonBuilder() + .startObject() + .field(RESCORE_OVERSAMPLE_PARAMETER, "c") + .endObject(); + expectValidationException(invalidParamValueBuilder); + + XContentBuilder extraParamBuilder = XContentFactory.jsonBuilder() + .startObject() + .field(RESCORE_OVERSAMPLE_PARAMETER, RescoreContext.MAX_OVERSAMPLE_FACTOR - 1) + .field("invalid", 0) + .endObject(); + expectValidationException(extraParamBuilder); + } + + private void validateOversample(float expectedOversample, XContentBuilder builder) throws IOException { + XContentParser parser = createParser(builder); + RescoreContext rescoreContext = RescoreParser.fromXContent(parser); + assertEquals(expectedOversample, rescoreContext.getOversampleFactor(), 0.0001); + } + + private void expectValidationException(XContentBuilder builder) throws IOException { + XContentParser parser = createParser(builder); + expectThrows(IllegalArgumentException.class, () -> RescoreParser.fromXContent(parser)); + } +} diff --git a/src/test/java/org/opensearch/knn/index/query/parser/RescoreValidationTests.java b/src/test/java/org/opensearch/knn/index/query/parser/RescoreValidationTests.java new file mode 100644 index 000000000..a23a5c757 --- /dev/null +++ b/src/test/java/org/opensearch/knn/index/query/parser/RescoreValidationTests.java @@ -0,0 +1,45 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.query.parser; + +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import lombok.AllArgsConstructor; +import org.opensearch.knn.KNNTestCase; +import org.opensearch.knn.index.query.rescore.RescoreContext; + +import java.util.Arrays; +import java.util.Collection; + +import static com.carrotsearch.randomizedtesting.RandomizedTest.$; +import static com.carrotsearch.randomizedtesting.RandomizedTest.$$; + +@AllArgsConstructor +public class RescoreValidationTests extends KNNTestCase { + + private boolean isValid; + private RescoreContext rescoreContext; + + @ParametersFactory(argumentFormatting = "isValid:%1$s; rescoreContext:%2$s") + public static Collection validParams() { + return Arrays.asList( + $$( + $(true, RescoreContext.builder().build()), + $(true, RescoreContext.getDefault()), + $(true, RescoreContext.builder().oversampleFactor(RescoreContext.MAX_OVERSAMPLE_FACTOR - 1).build()), + $(false, RescoreContext.builder().oversampleFactor(RescoreContext.MAX_OVERSAMPLE_FACTOR + 1).build()), + $(false, RescoreContext.builder().oversampleFactor(RescoreContext.MIN_OVERSAMPLE_FACTOR - 1).build()) + ) + ); + } + + public void testValidate() { + if (isValid) { + assertNull(RescoreParser.validate(rescoreContext)); + } else { + assertNotNull(RescoreParser.validate(rescoreContext)); + } + } +} diff --git a/src/test/java/org/opensearch/knn/integ/QueryParseIT.java b/src/test/java/org/opensearch/knn/integ/QueryParseIT.java new file mode 100644 index 000000000..bcaf6be12 --- /dev/null +++ b/src/test/java/org/opensearch/knn/integ/QueryParseIT.java @@ -0,0 +1,148 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.integ; + +import lombok.SneakyThrows; +import org.opensearch.client.Request; +import org.opensearch.client.ResponseException; +import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.knn.KNNRestTestCase; + +import java.io.IOException; +import java.util.Locale; + +import static org.opensearch.knn.index.query.parser.RescoreParser.RESCORE_OVERSAMPLE_PARAMETER; +import static org.opensearch.knn.index.query.parser.RescoreParser.RESCORE_PARAMETER; + +public class QueryParseIT extends KNNRestTestCase { + + private final static float[] TEST_VECTOR = new float[] { 1.0f, 2.0f }; + private final static int DIMENSION = 2; + private final static int K = 1; + + @SneakyThrows + public void testRescore() { + createTestIndex(); + assertValid( + buildRequest( + closeQueryXContentBuilder( + setupQueryXContentBuilder().field("vector", TEST_VECTOR).field("k", K).startObject("rescore").endObject() + ) + ) + ); + + assertValid( + buildRequest( + closeQueryXContentBuilder( + setupQueryXContentBuilder().field("vector", TEST_VECTOR) + .field("k", K) + .startObject(RESCORE_PARAMETER) + .field(RESCORE_OVERSAMPLE_PARAMETER, 2) + .endObject() + ) + ) + ); + + assertValid( + buildRequest( + closeQueryXContentBuilder( + setupQueryXContentBuilder().field("vector", TEST_VECTOR).field("k", K).startObject(RESCORE_PARAMETER).endObject() + ) + ) + ); + + assertValid( + buildRequest( + closeQueryXContentBuilder( + setupQueryXContentBuilder().field("vector", TEST_VECTOR).field("k", K).field(RESCORE_PARAMETER, true) + ) + ) + ); + + assertValid( + buildRequest( + closeQueryXContentBuilder( + setupQueryXContentBuilder().field("vector", TEST_VECTOR).field("k", K).field(RESCORE_PARAMETER, false) + ) + ) + ); + + // Invalid value for rescore + assertInvalid( + buildRequest( + closeQueryXContentBuilder( + setupQueryXContentBuilder().field("vector", TEST_VECTOR).field("k", K).field(RESCORE_PARAMETER, "invalid") + ) + ) + ); + + // Invalid rescore param + assertInvalid( + buildRequest( + closeQueryXContentBuilder( + setupQueryXContentBuilder().field("vector", TEST_VECTOR) + .field("k", K) + .startObject(RESCORE_OVERSAMPLE_PARAMETER) + .field("invalid_param", "invalid") + .endObject() + ) + ) + ); + + // Invalid rescore param value + assertInvalid( + buildRequest( + closeQueryXContentBuilder( + setupQueryXContentBuilder().field("vector", TEST_VECTOR) + .field("k", K) + .startObject(RESCORE_PARAMETER) + .field(RESCORE_OVERSAMPLE_PARAMETER, "invalid") + .endObject() + ) + ) + ); + } + + private XContentBuilder setupQueryXContentBuilder() throws IOException { + return XContentFactory.jsonBuilder().startObject().startObject("query").startObject("knn").startObject(FIELD_NAME); + } + + private XContentBuilder closeQueryXContentBuilder(XContentBuilder xContentBuilder) throws IOException { + return xContentBuilder.endObject().endObject().endObject().endObject(); + } + + private void assertValid(Request request) throws IOException { + assertOK(client().performRequest(request)); + } + + private void assertInvalid(Request request) { + expectThrows(ResponseException.class, () -> client().performRequest(request)); + } + + private Request buildRequest(XContentBuilder xContentBuilder) { + Request request = new Request("POST", String.format(Locale.ROOT, "/%s/_search", INDEX_NAME)); + request.addParameter("size", Integer.toString(10)); + request.addParameter("explain", Boolean.toString(true)); + request.setJsonEntity(xContentBuilder.toString()); + return request; + } + + private void createTestIndex() throws IOException { + XContentBuilder builder = XContentFactory.jsonBuilder() + .startObject() + .startObject("properties") + .startObject(FIELD_NAME) + .field("type", "knn_vector") + .field("dimension", DIMENSION) + .endObject() + .endObject() + .endObject(); + + String mapping = builder.toString(); + createKnnIndex(INDEX_NAME, mapping); + } +} From f3e644c9316653c15e5e03ce8e5aed88f3a5e54f Mon Sep 17 00:00:00 2001 From: Doo Yong Kim <0ctopus13prime@gmail.com> Date: Fri, 16 Aug 2024 13:41:56 -0700 Subject: [PATCH 328/416] Block a vector field to have invalid characters for a physical file name. (#1982) Signed-off-by: Dooyong Kim Co-authored-by: Dooyong Kim --- .../index/mapper/KNNVectorFieldMapper.java | 31 +++++++++++++++++++ .../mapper/KNNVectorFieldMapperTests.java | 18 +++++++++++ 2 files changed, 49 insertions(+) diff --git a/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapper.java b/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapper.java index 5d4d3ca58..94756f595 100644 --- a/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapper.java +++ b/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapper.java @@ -13,6 +13,8 @@ import java.util.Map; import java.util.Optional; import java.util.function.Supplier; +import java.util.stream.Collectors; + import lombok.extern.log4j.Log4j2; import org.apache.lucene.document.Field; import org.apache.lucene.document.FieldType; @@ -23,6 +25,7 @@ import org.opensearch.common.Explicit; import org.opensearch.common.ValidationException; import org.opensearch.common.xcontent.support.XContentMapValues; +import org.opensearch.core.common.Strings; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; @@ -219,6 +222,8 @@ private void validateFlatMapper() { @Override public KNNVectorFieldMapper build(BuilderContext context) { + validateFullFieldName(context); + final MultiFields multiFieldsBuilder = this.multiFieldsBuilder.build(this, context); final CopyTo copyToBuilder = copyTo.build(); final Explicit ignoreMalformed = ignoreMalformed(context); @@ -413,6 +418,32 @@ private KNNEngine validateDimensions(final KNNMethodContext knnMethodContext, fi } return knnEngine; } + + /** + * Validate whether provided full field name contain any invalid characters for physical file name. + * At the moment, we use a field name as a part of file name while we throw an exception + * if a physical file name contains any invalid characters when creating snapshot. + * To prevent from this happening, we restrict vector field name and make sure generated file to have a valid name. + * + * @param context : Builder context to have field name info. + */ + private void validateFullFieldName(final BuilderContext context) { + final String fullFieldName = buildFullName(context); + for (char ch : fullFieldName.toCharArray()) { + if (Strings.INVALID_FILENAME_CHARS.contains(ch)) { + throw new IllegalArgumentException( + String.format( + Locale.ROOT, + "Vector field name must not include invalid characters of %s. " + + "Provided field name=[%s] had a disallowed character [%c]", + Strings.INVALID_FILENAME_CHARS.stream().map(c -> "'" + c + "'").collect(Collectors.toList()), + fullFieldName, + ch + ) + ); + } + } + } } public static class TypeParser implements Mapper.TypeParser { diff --git a/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java b/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java index e1d842112..b3139fa5c 100644 --- a/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java +++ b/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java @@ -22,6 +22,7 @@ import org.opensearch.common.settings.IndexScopedSettings; import org.opensearch.common.settings.Settings; import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.core.common.Strings; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.index.IndexSettings; import org.opensearch.index.mapper.ContentPath; @@ -1171,6 +1172,23 @@ public void testBuilder_whenBinaryWithLegacyKNNEnabled_thenException() { assertTrue(ex.getMessage(), ex.getMessage().contains("is not supported for")); } + public void testBuild_whenInvalidCharsInFieldName_thenThrowException() { + for (char disallowChar : Strings.INVALID_FILENAME_CHARS) { + // When an invalid vector field name was given. + final String invalidVectorFieldName = "fieldname" + disallowChar; + + // Prepare context. + Settings settings = Settings.builder().put(settings(CURRENT).build()).put(KNN_INDEX, true).build(); + Mapper.BuilderContext builderContext = new Mapper.BuilderContext(settings, new ContentPath()); + + // IllegalArgumentException should be thrown. + Exception e = assertThrows(IllegalArgumentException.class, () -> { + new KNNVectorFieldMapper.Builder(invalidVectorFieldName, null, CURRENT, null).build(builderContext); + }); + assertTrue(e.getMessage(), e.getMessage().contains("Vector field name must not include")); + } + } + private LuceneFieldMapper.CreateLuceneFieldMapperInput.CreateLuceneFieldMapperInputBuilder createLuceneFieldMapperInputBuilder( VectorDataType vectorDataType ) { From ee534fb00ec228c076cb7292685e0ab2713c23e1 Mon Sep 17 00:00:00 2001 From: John Mazanec Date: Fri, 16 Aug 2024 18:02:15 -0400 Subject: [PATCH 329/416] Encapsulate dimension, vector data type validation/processing inside Library (#1983) Introduces a new configuration object KNNMethodConfigContext. The KNNMethodContext contains the user provided information for want their index to be built like. However, it is missing a few pieces that are defined outside of it. These pieces are needed to validate the config and actually build the config. This change Integrates validation of KNNMethodContexts with KNNMethodConfigContext and better encapsulates KNNLibrary config info in the engine package Signed-off-by: John Mazanec --- CHANGELOG.md | 1 + qa/restart-upgrade/build.gradle | 46 +++ .../bwc/AbstractRestartUpgradeTestCase.java | 1 - .../org/opensearch/knn/bwc/IndexingIT.java | 29 ++ .../KNN80Codec/KNN80DocValuesConsumer.java | 41 ++- .../knn/index/engine/AbstractKNNLibrary.java | 83 ++++- .../knn/index/engine/AbstractKNNMethod.java | 87 +++-- .../engine/DefaultHnswSearchContext.java | 5 +- .../index/engine/DefaultIVFSearchContext.java | 5 +- .../knn/index/engine/JVMLibrary.java | 2 +- .../knn/index/engine/KNNEngine.java | 21 +- .../knn/index/engine/KNNLibrary.java | 24 +- .../engine/KNNLibraryIndexingContext.java | 23 +- .../engine/KNNLibraryIndexingContextImpl.java | 21 ++ .../knn/index/engine/KNNMethod.java | 23 +- .../index/engine/KNNMethodConfigContext.java | 51 +++ .../knn/index/engine/KNNMethodContext.java | 24 +- .../knn/index/engine/MethodComponent.java | 108 +++--- .../index/engine/MethodComponentContext.java | 6 - .../knn/index/engine/NativeLibrary.java | 4 +- .../knn/index/engine/Parameter.java | 261 ++------------ .../engine/faiss/AbstractFaissMethod.java | 84 +++++ .../knn/index/engine/faiss/FaissFP16Util.java | 145 ++++++++ .../index/engine/faiss/FaissFlatEncoder.java | 16 +- .../index/engine/faiss/FaissHNSWMethod.java | 37 +- .../engine/faiss/FaissHNSWPQEncoder.java | 25 +- .../index/engine/faiss/FaissIVFMethod.java | 30 +- .../index/engine/faiss/FaissIVFPQEncoder.java | 35 +- .../index/engine/faiss/FaissSQEncoder.java | 19 +- .../engine/faiss/MethodAsMapBuilder.java | 14 +- .../index/engine/lucene/LuceneHNSWMethod.java | 9 +- .../lucene/LuceneHNSWSearchContext.java | 5 +- .../index/engine/lucene/LuceneSQEncoder.java | 10 +- .../index/engine/nmslib/NmslibHNSWMethod.java | 9 +- .../engine/validation/ParameterValidator.java | 11 +- .../index/mapper/FlatVectorFieldMapper.java | 16 +- .../index/mapper/KNNVectorFieldMapper.java | 284 +++++++-------- .../mapper/KNNVectorFieldMapperUtil.java | 202 +---------- .../knn/index/mapper/LuceneFieldMapper.java | 87 ++--- .../knn/index/mapper/MethodFieldMapper.java | 105 ++---- .../knn/index/mapper/ModelFieldMapper.java | 95 +++-- .../index/mapper/PerDimensionProcessor.java | 16 - .../index/mapper/PerDimensionValidator.java | 14 - .../knn/index/query/KNNQueryBuilder.java | 4 +- .../opensearch/knn/index/util/IndexUtil.java | 2 +- .../transport/TrainingModelRequest.java | 122 +------ .../TrainingModelTransportAction.java | 16 +- .../opensearch/knn/training/TrainingJob.java | 58 +-- .../knn/training/VectorSpaceInfo.java | 26 -- .../java/org/opensearch/knn/KNNTestCase.java | 14 +- .../opensearch/knn/index/OpenSearchIT.java | 4 +- .../knn/index/VectorDataTypeIT.java | 28 +- .../KNN80DocValuesConsumerTests.java | 21 +- .../knn/index/codec/KNNCodecTestCase.java | 13 +- .../index/engine/AbstractKNNLibraryTests.java | 36 +- .../index/engine/AbstractKNNMethodTests.java | 44 ++- .../{ => engine}/KNNMethodContextTests.java | 124 +++++-- .../index/engine/MethodComponentTests.java | 69 ++-- .../knn/index/engine/ParameterTests.java | 157 ++++---- .../engine/faiss/FaissFP16UtilTests.java | 60 ++++ .../knn/index/engine/faiss/FaissTests.java | 69 +++- .../knn/index/engine/lucene/LuceneTests.java | 19 +- .../mapper/KNNVectorFieldMapperTests.java | 339 +++++++++--------- .../mapper/KNNVectorFieldMapperUtilTests.java | 54 --- .../index/mapper/MethodFieldMapperTests.java | 39 -- .../integ/BinaryIndexInvalidMappingIT.java | 6 +- .../opensearch/knn/jni/JNIServiceTests.java | 49 ++- .../LibraryInitializedSupplierTests.java | 16 +- .../transport/TrainingModelRequestTests.java | 23 +- .../knn/training/TrainingJobTests.java | 65 ++-- .../org/opensearch/knn/KNNRestTestCase.java | 8 + 71 files changed, 1890 insertions(+), 1729 deletions(-) create mode 100644 src/main/java/org/opensearch/knn/index/engine/KNNMethodConfigContext.java create mode 100644 src/main/java/org/opensearch/knn/index/engine/faiss/AbstractFaissMethod.java create mode 100644 src/main/java/org/opensearch/knn/index/engine/faiss/FaissFP16Util.java delete mode 100644 src/main/java/org/opensearch/knn/training/VectorSpaceInfo.java rename src/test/java/org/opensearch/knn/index/{ => engine}/KNNMethodContextTests.java (80%) create mode 100644 src/test/java/org/opensearch/knn/index/engine/faiss/FaissFP16UtilTests.java delete mode 100644 src/test/java/org/opensearch/knn/index/mapper/MethodFieldMapperTests.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 86defd59e..ccac8d7c3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -38,3 +38,4 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), * Move k search k-NN query to re-write phase of vector search query for Native Engines [#1877](https://github.com/opensearch-project/k-NN/pull/1877) * Restructure mappers to better handle null cases and avoid branching in parsing [#1939](https://github.com/opensearch-project/k-NN/pull/1939) * Added Quantization Framework and implemented 1Bit and multibit quantizer[#1889](https://github.com/opensearch-project/k-NN/issues/1889) +* Encapsulate dimension, vector data type validation/processing inside Library [#1957](https://github.com/opensearch-project/k-NN/pull/1957) diff --git a/qa/restart-upgrade/build.gradle b/qa/restart-upgrade/build.gradle index 6bae754a2..a629e87ff 100644 --- a/qa/restart-upgrade/build.gradle +++ b/qa/restart-upgrade/build.gradle @@ -58,6 +58,29 @@ testClusters { excludeTestsMatching "org.opensearch.knn.bwc.IndexingIT.testEmptyParametersOnUpgrade" } } + + if (knn_bwc_version.startsWith("1.") || + knn_bwc_version.startsWith("2.0.") || + knn_bwc_version.startsWith("2.1.") || + knn_bwc_version.startsWith("2.2.") || + knn_bwc_version.startsWith("2.3.") || + knn_bwc_version.startsWith("2.4") || + knn_bwc_version.startsWith("2.5.") || + knn_bwc_version.startsWith("2.6.") || + knn_bwc_version.startsWith("2.7.") || + knn_bwc_version.startsWith("2.8.") || + knn_bwc_version.startsWith("2.9.") || + knn_bwc_version.startsWith("2.10.") || + knn_bwc_version.startsWith("2.11.") || + knn_bwc_version.startsWith("2.12.") || + knn_bwc_version.startsWith("2.13.") || + knn_bwc_version.startsWith("2.14.") || + knn_bwc_version.startsWith("2.15.")) { + filter { + excludeTestsMatching "org.opensearch.knn.bwc.IndexingIT.testKNNIndexBinaryForceMerge" + } + } + nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}".allHttpSocketURI.join(",")}") nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}".getName()}") systemProperty 'tests.security.manager', 'false' @@ -101,6 +124,29 @@ testClusters { excludeTestsMatching "org.opensearch.knn.bwc.IndexingIT.testEmptyParametersOnUpgrade" } } + + if (knn_bwc_version.startsWith("1.") || + knn_bwc_version.startsWith("2.0.") || + knn_bwc_version.startsWith("2.1.") || + knn_bwc_version.startsWith("2.2.") || + knn_bwc_version.startsWith("2.3.") || + knn_bwc_version.startsWith("2.4") || + knn_bwc_version.startsWith("2.5.") || + knn_bwc_version.startsWith("2.6.") || + knn_bwc_version.startsWith("2.7.") || + knn_bwc_version.startsWith("2.8.") || + knn_bwc_version.startsWith("2.9.") || + knn_bwc_version.startsWith("2.10.") || + knn_bwc_version.startsWith("2.11.") || + knn_bwc_version.startsWith("2.12.") || + knn_bwc_version.startsWith("2.13.") || + knn_bwc_version.startsWith("2.14.") || + knn_bwc_version.startsWith("2.15.")) { + filter { + excludeTestsMatching "org.opensearch.knn.bwc.IndexingIT.testKNNIndexBinaryForceMerge" + } + } + nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}".allHttpSocketURI.join(",")}") nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}".getName()}") systemProperty 'tests.security.manager', 'false' diff --git a/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/AbstractRestartUpgradeTestCase.java b/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/AbstractRestartUpgradeTestCase.java index ed11ca9d8..667a39d16 100644 --- a/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/AbstractRestartUpgradeTestCase.java +++ b/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/AbstractRestartUpgradeTestCase.java @@ -58,5 +58,4 @@ protected static final boolean isRunningAgainstOldCluster() { protected final Optional getBWCVersion() { return Optional.ofNullable(System.getProperty(BWC_VERSION, null)); } - } diff --git a/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/IndexingIT.java b/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/IndexingIT.java index 1531dd0da..9c1dfb018 100644 --- a/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/IndexingIT.java +++ b/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/IndexingIT.java @@ -8,6 +8,8 @@ import org.junit.Assert; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.knn.index.SpaceType; +import org.opensearch.knn.index.VectorDataType; +import org.opensearch.knn.index.engine.KNNEngine; import java.util.Map; @@ -68,6 +70,33 @@ public void testKNNIndexDefaultLegacyFieldMappingForceMerge() throws Exception { } } + // Ensure bwc works for binary force merge + public void testKNNIndexBinaryForceMerge() throws Exception { + int dimension = 40; + + waitForClusterHealthGreen(NODES_BWC_CLUSTER); + if (isRunningAgainstOldCluster()) { + createKnnIndex( + testIndex, + getKNNDefaultIndexSettings(), + createKnnIndexMapping( + TEST_FIELD, + dimension, + METHOD_HNSW, + KNNEngine.FAISS.getName(), + SpaceType.HAMMING.getValue(), + true, + VectorDataType.BINARY + ) + ); + addKNNByteDocs(testIndex, TEST_FIELD, dimension / 8, DOC_ID, 100); + // Flush to ensure that index is not re-indexed when node comes back up + flush(testIndex, true); + } else { + forceMergeKnnIndex(testIndex); + } + } + // Custom Legacy Field Mapping // space_type : "linf", engine : "nmslib", m : 2, ef_construction : 2 public void testKNNIndexCustomLegacyFieldMapping() throws Exception { diff --git a/src/main/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumer.java b/src/main/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumer.java index 5874eaded..f8bd0b3f7 100644 --- a/src/main/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumer.java +++ b/src/main/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumer.java @@ -238,17 +238,11 @@ private void createKNNIndexFromScratch(FieldInfo fieldInfo, KNNCodecUtil.Pair pa ); } - // Update index description of Faiss for binary data type - if (KNNEngine.FAISS == knnEngine - && VectorDataType.BINARY.getValue() - .equals(fieldAttributes.getOrDefault(KNNConstants.VECTOR_DATA_TYPE_FIELD, VectorDataType.DEFAULT.getValue())) - && parameters.get(KNNConstants.INDEX_DESCRIPTION_PARAMETER) != null) { - parameters.put( - KNNConstants.INDEX_DESCRIPTION_PARAMETER, - FAISS_BINARY_INDEX_DESCRIPTION_PREFIX + parameters.get(KNNConstants.INDEX_DESCRIPTION_PARAMETER).toString() - ); - IndexUtil.updateVectorDataTypeToParameters(parameters, VectorDataType.BINARY); - } + // In OpenSearch 2.16, we added the prefix for binary indices in the index description in the codec logic. + // After 2.16, we added the binary prefix in the faiss library code. However, to ensure backwards compatibility, + // we need to ensure that if the description does not contain the prefix but the type is binary, we add the + // description. + maybeAddBinaryPrefixForFaissBWC(knnEngine, parameters, fieldAttributes); // Used to determine how many threads to use when indexing parameters.put(KNNConstants.INDEX_THREAD_QTY, KNNSettings.state().getSettingValue(KNNSettings.KNN_ALGO_PARAM_INDEX_THREAD_QTY)); @@ -260,6 +254,31 @@ private void createKNNIndexFromScratch(FieldInfo fieldInfo, KNNCodecUtil.Pair pa }); } + private void maybeAddBinaryPrefixForFaissBWC(KNNEngine knnEngine, Map parameters, Map fieldAttributes) { + if (KNNEngine.FAISS != knnEngine) { + return; + } + + if (!VectorDataType.BINARY.getValue() + .equals(fieldAttributes.getOrDefault(KNNConstants.VECTOR_DATA_TYPE_FIELD, VectorDataType.DEFAULT.getValue()))) { + return; + } + + if (parameters.get(KNNConstants.INDEX_DESCRIPTION_PARAMETER) == null) { + return; + } + + if (parameters.get(KNNConstants.INDEX_DESCRIPTION_PARAMETER).toString().startsWith(FAISS_BINARY_INDEX_DESCRIPTION_PREFIX)) { + return; + } + + parameters.put( + KNNConstants.INDEX_DESCRIPTION_PARAMETER, + FAISS_BINARY_INDEX_DESCRIPTION_PREFIX + parameters.get(KNNConstants.INDEX_DESCRIPTION_PARAMETER).toString() + ); + IndexUtil.updateVectorDataTypeToParameters(parameters, VectorDataType.BINARY); + } + /** * Merges in the fields from the readers in mergeState * diff --git a/src/main/java/org/opensearch/knn/index/engine/AbstractKNNLibrary.java b/src/main/java/org/opensearch/knn/index/engine/AbstractKNNLibrary.java index 92e34be7c..9b38b1b6b 100644 --- a/src/main/java/org/opensearch/knn/index/engine/AbstractKNNLibrary.java +++ b/src/main/java/org/opensearch/knn/index/engine/AbstractKNNLibrary.java @@ -9,8 +9,9 @@ import lombok.AllArgsConstructor; import lombok.Getter; import org.opensearch.common.ValidationException; -import org.opensearch.knn.training.VectorSpaceInfo; +import org.opensearch.knn.index.VectorDataType; +import java.util.Locale; import java.util.Map; /** @@ -25,44 +26,94 @@ public abstract class AbstractKNNLibrary implements KNNLibrary { @Override public KNNLibrarySearchContext getKNNLibrarySearchContext(String methodName) { - validateMethodExists(methodName); + throwIllegalArgOnNonNull(validateMethodExists(methodName)); KNNMethod method = methods.get(methodName); return method.getKNNLibrarySearchContext(); } @Override - public KNNLibraryIndexingContext getKNNLibraryIndexingContext(KNNMethodContext knnMethodContext) { + public KNNLibraryIndexingContext getKNNLibraryIndexingContext( + KNNMethodContext knnMethodContext, + KNNMethodConfigContext knnMethodConfigContext + ) { String method = knnMethodContext.getMethodComponentContext().getName(); - validateMethodExists(method); + throwIllegalArgOnNonNull(validateMethodExists(method)); KNNMethod knnMethod = methods.get(method); - return knnMethod.getKNNLibraryIndexingContext(knnMethodContext); + return knnMethod.getKNNLibraryIndexingContext(knnMethodContext, knnMethodConfigContext); } @Override - public ValidationException validateMethod(KNNMethodContext knnMethodContext) { + public ValidationException validateMethod(KNNMethodContext knnMethodContext, KNNMethodConfigContext knnMethodConfigContext) { String methodName = knnMethodContext.getMethodComponentContext().getName(); - validateMethodExists(methodName); - return methods.get(methodName).validate(knnMethodContext); + ValidationException validationException = null; + String invalidErrorMessage = validateMethodExists(methodName); + if (invalidErrorMessage != null) { + validationException = new ValidationException(); + validationException.addValidationError(invalidErrorMessage); + return validationException; + } + invalidErrorMessage = validateDimension(knnMethodContext, knnMethodConfigContext); + if (invalidErrorMessage != null) { + validationException = new ValidationException(); + validationException.addValidationError(invalidErrorMessage); + } + + validateSpaceType(knnMethodContext, knnMethodConfigContext); + ValidationException methodValidation = methods.get(methodName).validate(knnMethodContext, knnMethodConfigContext); + if (methodValidation != null) { + validationException = validationException == null ? new ValidationException() : validationException; + validationException.addValidationErrors(methodValidation.validationErrors()); + } + + return validationException; } - @Override - public ValidationException validateMethodWithData(KNNMethodContext knnMethodContext, VectorSpaceInfo vectorSpaceInfo) { - String methodName = knnMethodContext.getMethodComponentContext().getName(); - validateMethodExists(methodName); - return methods.get(methodName).validateWithData(knnMethodContext, vectorSpaceInfo); + private void validateSpaceType(final KNNMethodContext knnMethodContext, KNNMethodConfigContext knnMethodConfigContext) { + if (knnMethodContext == null) { + return; + } + knnMethodContext.getSpaceType().validateVectorDataType(knnMethodConfigContext.getVectorDataType()); + } + + private String validateDimension(final KNNMethodContext knnMethodContext, KNNMethodConfigContext knnMethodConfigContext) { + if (knnMethodContext == null) { + return null; + } + int dimension = knnMethodConfigContext.getDimension(); + if (dimension > KNNEngine.getMaxDimensionByEngine(knnMethodContext.getKnnEngine())) { + return String.format( + Locale.ROOT, + "Dimension value cannot be greater than %s for vector with engine: %s", + KNNEngine.getMaxDimensionByEngine(knnMethodContext.getKnnEngine()), + knnMethodContext.getKnnEngine().getName() + ); + } + + if (VectorDataType.BINARY == knnMethodConfigContext.getVectorDataType() && dimension % 8 != 0) { + return "Dimension should be multiply of 8 for binary vector data type"; + } + + return null; } @Override public boolean isTrainingRequired(KNNMethodContext knnMethodContext) { String methodName = knnMethodContext.getMethodComponentContext().getName(); - validateMethodExists(methodName); + throwIllegalArgOnNonNull(validateMethodExists(methodName)); return methods.get(methodName).isTrainingRequired(knnMethodContext); } - private void validateMethodExists(String methodName) { + private String validateMethodExists(String methodName) { KNNMethod method = methods.get(methodName); if (method == null) { - throw new IllegalArgumentException(String.format("Invalid method name: %s", methodName)); + return String.format("Invalid method name: %s", methodName); + } + return null; + } + + private void throwIllegalArgOnNonNull(String errorMessage) { + if (errorMessage != null) { + throw new IllegalArgumentException(errorMessage); } } } diff --git a/src/main/java/org/opensearch/knn/index/engine/AbstractKNNMethod.java b/src/main/java/org/opensearch/knn/index/engine/AbstractKNNMethod.java index 6e57e6913..52cc79129 100644 --- a/src/main/java/org/opensearch/knn/index/engine/AbstractKNNMethod.java +++ b/src/main/java/org/opensearch/knn/index/engine/AbstractKNNMethod.java @@ -9,7 +9,11 @@ import org.opensearch.common.ValidationException; import org.opensearch.knn.common.KNNConstants; import org.opensearch.knn.index.SpaceType; -import org.opensearch.knn.training.VectorSpaceInfo; +import org.opensearch.knn.index.VectorDataType; +import org.opensearch.knn.index.mapper.PerDimensionProcessor; +import org.opensearch.knn.index.mapper.PerDimensionValidator; +import org.opensearch.knn.index.mapper.SpaceVectorValidator; +import org.opensearch.knn.index.mapper.VectorValidator; import java.util.ArrayList; import java.util.HashMap; @@ -35,7 +39,7 @@ public boolean isSpaceTypeSupported(SpaceType space) { } @Override - public ValidationException validate(KNNMethodContext knnMethodContext) { + public ValidationException validate(KNNMethodContext knnMethodContext, KNNMethodConfigContext knnMethodConfigContext) { List errorMessages = new ArrayList<>(); if (!isSpaceTypeSupported(knnMethodContext.getSpaceType())) { errorMessages.add( @@ -49,7 +53,10 @@ public ValidationException validate(KNNMethodContext knnMethodContext) { ); } - ValidationException methodValidation = methodComponent.validate(knnMethodContext.getMethodComponentContext()); + ValidationException methodValidation = methodComponent.validate( + knnMethodContext.getMethodComponentContext(), + knnMethodConfigContext + ); if (methodValidation != null) { errorMessages.addAll(methodValidation.validationErrors()); } @@ -64,52 +71,58 @@ public ValidationException validate(KNNMethodContext knnMethodContext) { } @Override - public ValidationException validateWithData(KNNMethodContext knnMethodContext, VectorSpaceInfo vectorSpaceInfo) { - List errorMessages = new ArrayList<>(); - if (!isSpaceTypeSupported(knnMethodContext.getSpaceType())) { - errorMessages.add( - String.format( - Locale.ROOT, - "\"%s\" with \"%s\" configuration does not support space type: " + "\"%s\".", - this.methodComponent.getName(), - knnMethodContext.getKnnEngine().getName().toLowerCase(Locale.ROOT), - knnMethodContext.getSpaceType().getValue() - ) - ); - } + public boolean isTrainingRequired(KNNMethodContext knnMethodContext) { + return methodComponent.isTrainingRequired(knnMethodContext.getMethodComponentContext()); + } - ValidationException methodValidation = methodComponent.validateWithData( - knnMethodContext.getMethodComponentContext(), - vectorSpaceInfo - ); - if (methodValidation != null) { - errorMessages.addAll(methodValidation.validationErrors()); - } + @Override + public int estimateOverheadInKB(KNNMethodContext knnMethodContext, KNNMethodConfigContext knnMethodConfigContext) { + return methodComponent.estimateOverheadInKB(knnMethodContext.getMethodComponentContext(), knnMethodConfigContext.getDimension()); + } - if (errorMessages.isEmpty()) { - return null; + protected PerDimensionValidator doGetPerDimensionValidator( + KNNMethodContext knnMethodContext, + KNNMethodConfigContext knnMethodConfigContext + ) { + VectorDataType vectorDataType = knnMethodConfigContext.getVectorDataType(); + + if (VectorDataType.BINARY == vectorDataType) { + return PerDimensionValidator.DEFAULT_BIT_VALIDATOR; } - ValidationException validationException = new ValidationException(); - validationException.addValidationErrors(errorMessages); - return validationException; + if (VectorDataType.BYTE == vectorDataType) { + return PerDimensionValidator.DEFAULT_BYTE_VALIDATOR; + } + return PerDimensionValidator.DEFAULT_FLOAT_VALIDATOR; } - @Override - public boolean isTrainingRequired(KNNMethodContext knnMethodContext) { - return methodComponent.isTrainingRequired(knnMethodContext.getMethodComponentContext()); + protected VectorValidator doGetVectorValidator(KNNMethodContext knnMethodContext, KNNMethodConfigContext knnMethodConfigContext) { + return new SpaceVectorValidator(knnMethodContext.getSpaceType()); } - @Override - public int estimateOverheadInKB(KNNMethodContext knnMethodContext, int dimension) { - return methodComponent.estimateOverheadInKB(knnMethodContext.getMethodComponentContext(), dimension); + protected PerDimensionProcessor doGetPerDimensionProcessor( + KNNMethodContext knnMethodContext, + KNNMethodConfigContext knnMethodConfigContext + ) { + return PerDimensionProcessor.NOOP_PROCESSOR; } @Override - public KNNLibraryIndexingContext getKNNLibraryIndexingContext(KNNMethodContext knnMethodContext) { - Map parameterMap = new HashMap<>(methodComponent.getAsMap(knnMethodContext.getMethodComponentContext())); + public KNNLibraryIndexingContext getKNNLibraryIndexingContext( + KNNMethodContext knnMethodContext, + KNNMethodConfigContext knnMethodConfigContext + ) { + Map parameterMap = new HashMap<>( + methodComponent.getAsMap(knnMethodContext.getMethodComponentContext(), knnMethodConfigContext) + ); parameterMap.put(KNNConstants.SPACE_TYPE, knnMethodContext.getSpaceType().getValue()); - return KNNLibraryIndexingContextImpl.builder().parameters(parameterMap).build(); + parameterMap.put(KNNConstants.VECTOR_DATA_TYPE_FIELD, knnMethodConfigContext.getVectorDataType().getValue()); + return KNNLibraryIndexingContextImpl.builder() + .parameters(parameterMap) + .vectorValidator(doGetVectorValidator(knnMethodContext, knnMethodConfigContext)) + .perDimensionValidator(doGetPerDimensionValidator(knnMethodContext, knnMethodConfigContext)) + .perDimensionProcessor(doGetPerDimensionProcessor(knnMethodContext, knnMethodConfigContext)) + .build(); } @Override diff --git a/src/main/java/org/opensearch/knn/index/engine/DefaultHnswSearchContext.java b/src/main/java/org/opensearch/knn/index/engine/DefaultHnswSearchContext.java index ecc11f338..884657442 100644 --- a/src/main/java/org/opensearch/knn/index/engine/DefaultHnswSearchContext.java +++ b/src/main/java/org/opensearch/knn/index/engine/DefaultHnswSearchContext.java @@ -17,7 +17,10 @@ public final class DefaultHnswSearchContext implements KNNLibrarySearchContext { private final Map> supportedMethodParameters = ImmutableMap.>builder() - .put(MethodParameter.EF_SEARCH.getName(), new Parameter.IntegerParameter(MethodParameter.EF_SEARCH.getName(), null, value -> true)) + .put( + MethodParameter.EF_SEARCH.getName(), + new Parameter.IntegerParameter(MethodParameter.EF_SEARCH.getName(), null, (value, context) -> true) + ) .build(); @Override diff --git a/src/main/java/org/opensearch/knn/index/engine/DefaultIVFSearchContext.java b/src/main/java/org/opensearch/knn/index/engine/DefaultIVFSearchContext.java index cc612bf8c..16e3f67d8 100644 --- a/src/main/java/org/opensearch/knn/index/engine/DefaultIVFSearchContext.java +++ b/src/main/java/org/opensearch/knn/index/engine/DefaultIVFSearchContext.java @@ -14,7 +14,10 @@ public final class DefaultIVFSearchContext implements KNNLibrarySearchContext { private final Map> supportedMethodParameters = ImmutableMap.>builder() - .put(MethodParameter.NPROBE.getName(), new Parameter.IntegerParameter(MethodParameter.NPROBE.getName(), null, value -> true)) + .put( + MethodParameter.NPROBE.getName(), + new Parameter.IntegerParameter(MethodParameter.NPROBE.getName(), null, (value, context) -> true) + ) .build(); @Override diff --git a/src/main/java/org/opensearch/knn/index/engine/JVMLibrary.java b/src/main/java/org/opensearch/knn/index/engine/JVMLibrary.java index 762966567..bfb25c7c6 100644 --- a/src/main/java/org/opensearch/knn/index/engine/JVMLibrary.java +++ b/src/main/java/org/opensearch/knn/index/engine/JVMLibrary.java @@ -25,7 +25,7 @@ public JVMLibrary(Map methods, String version) { } @Override - public int estimateOverheadInKB(KNNMethodContext knnMethodContext, int dimension) { + public int estimateOverheadInKB(KNNMethodContext knnMethodContext, KNNMethodConfigContext knnMethodConfigContext) { throw new UnsupportedOperationException("Estimating overhead is not supported for JVM based libraries."); } diff --git a/src/main/java/org/opensearch/knn/index/engine/KNNEngine.java b/src/main/java/org/opensearch/knn/index/engine/KNNEngine.java index c7b271783..2f3cb3430 100644 --- a/src/main/java/org/opensearch/knn/index/engine/KNNEngine.java +++ b/src/main/java/org/opensearch/knn/index/engine/KNNEngine.java @@ -11,7 +11,6 @@ import org.opensearch.knn.index.engine.faiss.Faiss; import org.opensearch.knn.index.engine.lucene.Lucene; import org.opensearch.knn.index.engine.nmslib.Nmslib; -import org.opensearch.knn.training.VectorSpaceInfo; import java.util.List; import java.util.Map; @@ -161,13 +160,8 @@ public Float scoreToRadialThreshold(Float score, SpaceType spaceType) { } @Override - public ValidationException validateMethod(KNNMethodContext knnMethodContext) { - return knnLibrary.validateMethod(knnMethodContext); - } - - @Override - public ValidationException validateMethodWithData(KNNMethodContext knnMethodContext, VectorSpaceInfo vectorSpaceInfo) { - return knnLibrary.validateMethodWithData(knnMethodContext, vectorSpaceInfo); + public ValidationException validateMethod(KNNMethodContext knnMethodContext, KNNMethodConfigContext knnMethodConfigContext) { + return knnLibrary.validateMethod(knnMethodContext, knnMethodConfigContext); } @Override @@ -176,8 +170,11 @@ public boolean isTrainingRequired(KNNMethodContext knnMethodContext) { } @Override - public KNNLibraryIndexingContext getKNNLibraryIndexingContext(KNNMethodContext knnMethodContext) { - return knnLibrary.getKNNLibraryIndexingContext(knnMethodContext); + public KNNLibraryIndexingContext getKNNLibraryIndexingContext( + KNNMethodContext knnMethodContext, + KNNMethodConfigContext knnMethodConfigContext + ) { + return knnLibrary.getKNNLibraryIndexingContext(knnMethodContext, knnMethodConfigContext); } @Override @@ -186,8 +183,8 @@ public KNNLibrarySearchContext getKNNLibrarySearchContext(String methodName) { } @Override - public int estimateOverheadInKB(KNNMethodContext knnMethodContext, int dimension) { - return knnLibrary.estimateOverheadInKB(knnMethodContext, dimension); + public int estimateOverheadInKB(KNNMethodContext knnMethodContext, KNNMethodConfigContext knnMethodConfigContext) { + return knnLibrary.estimateOverheadInKB(knnMethodContext, knnMethodConfigContext); } @Override diff --git a/src/main/java/org/opensearch/knn/index/engine/KNNLibrary.java b/src/main/java/org/opensearch/knn/index/engine/KNNLibrary.java index 96d492307..14085243f 100644 --- a/src/main/java/org/opensearch/knn/index/engine/KNNLibrary.java +++ b/src/main/java/org/opensearch/knn/index/engine/KNNLibrary.java @@ -7,7 +7,6 @@ import org.opensearch.common.ValidationException; import org.opensearch.knn.index.SpaceType; -import org.opensearch.knn.training.VectorSpaceInfo; import java.util.Collections; import java.util.List; @@ -76,19 +75,10 @@ public interface KNNLibrary { * deemed invalid. * * @param knnMethodContext to be validated + * @param knnMethodConfigContext configuration context for the method * @return ValidationException produced by validation errors; null if no validations errors. */ - ValidationException validateMethod(KNNMethodContext knnMethodContext); - - /** - * Validate the knnMethodContext for the given library, using additional data not present in the method context. A ValidationException should be thrown if the method is - * deemed invalid. - * - * @param knnMethodContext to be validated - * @param vectorSpaceInfo additional data not present in the method context - * @return ValidationException produced by validation errors; null if no validations errors. - */ - ValidationException validateMethodWithData(KNNMethodContext knnMethodContext, VectorSpaceInfo vectorSpaceInfo); + ValidationException validateMethod(KNNMethodContext knnMethodContext, KNNMethodConfigContext knnMethodConfigContext); /** * Returns whether training is required or not from knnMethodContext for the given library. @@ -102,18 +92,22 @@ public interface KNNLibrary { * Estimate overhead of KNNMethodContext in Kilobytes. * * @param knnMethodContext to estimate size for - * @param dimension to estimate size for + * @param knnMethodConfigContext configuration context for the method * @return size overhead estimate in KB */ - int estimateOverheadInKB(KNNMethodContext knnMethodContext, int dimension); + int estimateOverheadInKB(KNNMethodContext knnMethodContext, KNNMethodConfigContext knnMethodConfigContext); /** * Get the context from the library needed to build the index. * * @param knnMethodContext to get build context for + * @param knnMethodConfigContext configuration context for the method * @return parameter map */ - KNNLibraryIndexingContext getKNNLibraryIndexingContext(KNNMethodContext knnMethodContext); + KNNLibraryIndexingContext getKNNLibraryIndexingContext( + KNNMethodContext knnMethodContext, + KNNMethodConfigContext knnMethodConfigContext + ); /** * Gets metadata related to methods supported by the library diff --git a/src/main/java/org/opensearch/knn/index/engine/KNNLibraryIndexingContext.java b/src/main/java/org/opensearch/knn/index/engine/KNNLibraryIndexingContext.java index d00b7c436..20285471e 100644 --- a/src/main/java/org/opensearch/knn/index/engine/KNNLibraryIndexingContext.java +++ b/src/main/java/org/opensearch/knn/index/engine/KNNLibraryIndexingContext.java @@ -5,7 +5,10 @@ package org.opensearch.knn.index.engine; -import java.util.Collections; +import org.opensearch.knn.index.mapper.PerDimensionProcessor; +import org.opensearch.knn.index.mapper.PerDimensionValidator; +import org.opensearch.knn.index.mapper.VectorValidator; + import java.util.Map; /** @@ -19,5 +22,21 @@ public interface KNNLibraryIndexingContext { */ Map getLibraryParameters(); - KNNLibraryIndexingContext EMPTY = Collections::emptyMap; + /** + * + * @return Get the vector validator + */ + VectorValidator getVectorValidator(); + + /** + * + * @return Get the per dimension validator + */ + PerDimensionValidator getPerDimensionValidator(); + + /** + * + * @return Get the per dimension processor + */ + PerDimensionProcessor getPerDimensionProcessor(); } diff --git a/src/main/java/org/opensearch/knn/index/engine/KNNLibraryIndexingContextImpl.java b/src/main/java/org/opensearch/knn/index/engine/KNNLibraryIndexingContextImpl.java index b7c775261..51b60d9e5 100644 --- a/src/main/java/org/opensearch/knn/index/engine/KNNLibraryIndexingContextImpl.java +++ b/src/main/java/org/opensearch/knn/index/engine/KNNLibraryIndexingContextImpl.java @@ -6,6 +6,9 @@ package org.opensearch.knn.index.engine; import lombok.Builder; +import org.opensearch.knn.index.mapper.PerDimensionProcessor; +import org.opensearch.knn.index.mapper.PerDimensionValidator; +import org.opensearch.knn.index.mapper.VectorValidator; import java.util.Map; @@ -15,10 +18,28 @@ @Builder public class KNNLibraryIndexingContextImpl implements KNNLibraryIndexingContext { + private VectorValidator vectorValidator; + private PerDimensionValidator perDimensionValidator; + private PerDimensionProcessor perDimensionProcessor; private Map parameters; @Override public Map getLibraryParameters() { return parameters; } + + @Override + public VectorValidator getVectorValidator() { + return vectorValidator; + } + + @Override + public PerDimensionValidator getPerDimensionValidator() { + return perDimensionValidator; + } + + @Override + public PerDimensionProcessor getPerDimensionProcessor() { + return perDimensionProcessor; + } } diff --git a/src/main/java/org/opensearch/knn/index/engine/KNNMethod.java b/src/main/java/org/opensearch/knn/index/engine/KNNMethod.java index 326e5c1e0..0bcccacf0 100644 --- a/src/main/java/org/opensearch/knn/index/engine/KNNMethod.java +++ b/src/main/java/org/opensearch/knn/index/engine/KNNMethod.java @@ -7,7 +7,6 @@ import org.opensearch.common.ValidationException; import org.opensearch.knn.index.SpaceType; -import org.opensearch.knn.training.VectorSpaceInfo; /** * KNNMethod defines the structure of a method supported by a particular k-NN library. It is used to validate @@ -28,18 +27,10 @@ public interface KNNMethod { * Validate that the configured KNNMethodContext is valid for this method * * @param knnMethodContext to be validated + * @param knnMethodConfigContext to be validated * @return ValidationException produced by validation errors; null if no validations errors. */ - ValidationException validate(KNNMethodContext knnMethodContext); - - /** - * Validate that the configured KNNMethodContext is valid for this method, using additional data not present in the method context - * - * @param knnMethodContext to be validated - * @param vectorSpaceInfo additional data not present in the method context - * @return ValidationException produced by validation errors; null if no validations errors. - */ - ValidationException validateWithData(KNNMethodContext knnMethodContext, VectorSpaceInfo vectorSpaceInfo); + ValidationException validate(KNNMethodContext knnMethodContext, KNNMethodConfigContext knnMethodConfigContext); /** * returns whether training is required or not @@ -53,18 +44,22 @@ public interface KNNMethod { * Returns the estimated overhead of the method in KB * * @param knnMethodContext context to estimate overhead - * @param dimension dimension to make estimate with + * @param knnMethodConfigContext config context to estimate overhead * @return estimate overhead in KB */ - int estimateOverheadInKB(KNNMethodContext knnMethodContext, int dimension); + int estimateOverheadInKB(KNNMethodContext knnMethodContext, KNNMethodConfigContext knnMethodConfigContext); /** * Parse knnMethodContext into context that the library can use to build the index * * @param knnMethodContext to generate the context for + * @param knnMethodConfigContext to generate the context for * @return KNNLibraryIndexingContext */ - KNNLibraryIndexingContext getKNNLibraryIndexingContext(KNNMethodContext knnMethodContext); + KNNLibraryIndexingContext getKNNLibraryIndexingContext( + KNNMethodContext knnMethodContext, + KNNMethodConfigContext knnMethodConfigContext + ); /** * Get the search context for a particular method diff --git a/src/main/java/org/opensearch/knn/index/engine/KNNMethodConfigContext.java b/src/main/java/org/opensearch/knn/index/engine/KNNMethodConfigContext.java new file mode 100644 index 000000000..731085f0b --- /dev/null +++ b/src/main/java/org/opensearch/knn/index/engine/KNNMethodConfigContext.java @@ -0,0 +1,51 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.engine; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Getter; +import lombok.Setter; +import org.apache.commons.lang.builder.EqualsBuilder; +import org.apache.commons.lang.builder.HashCodeBuilder; +import org.opensearch.Version; +import org.opensearch.knn.index.VectorDataType; + +/** + * This object provides additional context that the user does not provide when {@link KNNMethodContext} is + * created via parsing. The values in this object need to be dynamically set and calling code needs to handle + * the possibility that the values have not been set. + */ +@Setter +@Getter +@Builder +@AllArgsConstructor +public final class KNNMethodConfigContext { + private VectorDataType vectorDataType; + private Integer dimension; + private Version versionCreated; + + @Override + public boolean equals(Object obj) { + if (this == obj) return true; + if (obj == null || getClass() != obj.getClass()) return false; + KNNMethodConfigContext other = (KNNMethodConfigContext) obj; + + EqualsBuilder equalsBuilder = new EqualsBuilder(); + equalsBuilder.append(vectorDataType, other.vectorDataType); + equalsBuilder.append(dimension, other.dimension); + equalsBuilder.append(versionCreated, other.versionCreated); + + return equalsBuilder.isEquals(); + } + + @Override + public int hashCode() { + return new HashCodeBuilder().append(vectorDataType).append(dimension).append(versionCreated).toHashCode(); + } + + public static final KNNMethodConfigContext EMPTY = KNNMethodConfigContext.builder().build(); +} diff --git a/src/main/java/org/opensearch/knn/index/engine/KNNMethodContext.java b/src/main/java/org/opensearch/knn/index/engine/KNNMethodContext.java index d210483e6..8b2f00f74 100644 --- a/src/main/java/org/opensearch/knn/index/engine/KNNMethodContext.java +++ b/src/main/java/org/opensearch/knn/index/engine/KNNMethodContext.java @@ -24,7 +24,6 @@ import java.util.stream.Collectors; import org.apache.commons.lang.builder.EqualsBuilder; import org.apache.commons.lang.builder.HashCodeBuilder; -import org.opensearch.knn.training.VectorSpaceInfo; import static org.opensearch.knn.common.KNNConstants.KNN_ENGINE; import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_SPACE_TYPE; @@ -60,22 +59,13 @@ public KNNMethodContext(StreamInput in) throws IOException { } /** - * This method uses the knnEngine to validate that the method is compatible with the engine + * This method uses the knnEngine to validate that the method is compatible with the engine. * + * @param knnMethodConfigContext context to validate against * @return ValidationException produced by validation errors; null if no validations errors. */ - public ValidationException validate() { - return knnEngine.validateMethod(this); - } - - /** - * This method uses the knnEngine to validate that the method is compatible with the engine, using additional data not present in the method context - * - * @param vectorSpaceInfo additional data not present in the method context - * @return ValidationException produced by validation errors; null if no validations errors. - */ - public ValidationException validateWithData(VectorSpaceInfo vectorSpaceInfo) { - return knnEngine.validateMethodWithData(this, vectorSpaceInfo); + public ValidationException validate(KNNMethodConfigContext knnMethodConfigContext) { + return knnEngine.validateMethod(this, knnMethodConfigContext); } /** @@ -90,11 +80,11 @@ public boolean isTrainingRequired() { /** * This method estimates the overhead the knn method adds irrespective of the number of vectors * - * @param dimension dimension to make estimate with + * @param knnMethodConfigContext context to estimate overhead * @return size in Kilobytes */ - public int estimateOverheadInKB(int dimension) { - return knnEngine.estimateOverheadInKB(this, dimension); + public int estimateOverheadInKB(KNNMethodConfigContext knnMethodConfigContext) { + return knnEngine.estimateOverheadInKB(this, knnMethodConfigContext); } /** diff --git a/src/main/java/org/opensearch/knn/index/engine/MethodComponent.java b/src/main/java/org/opensearch/knn/index/engine/MethodComponent.java index cd9377ef1..988812e61 100644 --- a/src/main/java/org/opensearch/knn/index/engine/MethodComponent.java +++ b/src/main/java/org/opensearch/knn/index/engine/MethodComponent.java @@ -10,14 +10,14 @@ import org.opensearch.common.TriFunction; import org.opensearch.common.ValidationException; import org.opensearch.knn.common.KNNConstants; +import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.util.IndexHyperParametersUtil; -import org.opensearch.knn.training.VectorSpaceInfo; import java.util.HashMap; +import java.util.HashSet; +import java.util.Locale; import java.util.Map; -import java.util.function.BiFunction; -import java.util.List; -import java.util.ArrayList; +import java.util.Set; import static org.opensearch.knn.index.engine.validation.ParameterValidator.validateParameters; @@ -27,12 +27,13 @@ public class MethodComponent { @Getter - private String name; + private final String name; @Getter - private Map> parameters; - private BiFunction> mapGenerator; - private TriFunction overheadInKBEstimator; - final private boolean requiresTraining; + private final Map> parameters; + private final TriFunction> mapGenerator; + private final TriFunction overheadInKBEstimator; + private final boolean requiresTraining; + private final Set supportedVectorDataTypes; /** * Constructor @@ -45,6 +46,7 @@ private MethodComponent(Builder builder) { this.mapGenerator = builder.mapGenerator; this.overheadInKBEstimator = builder.overheadInKBEstimator; this.requiresTraining = builder.requiresTraining; + this.supportedVectorDataTypes = builder.supportedDataTypes; } /** @@ -53,61 +55,49 @@ private MethodComponent(Builder builder) { * @param methodComponentContext from which to generate map * @return Method component as a map */ - public Map getAsMap(MethodComponentContext methodComponentContext) { + public Map getAsMap(MethodComponentContext methodComponentContext, KNNMethodConfigContext knnMethodConfigContext) { if (mapGenerator == null) { Map parameterMap = new HashMap<>(); parameterMap.put(KNNConstants.NAME, methodComponentContext.getName()); - parameterMap.put(KNNConstants.PARAMETERS, getParameterMapWithDefaultsAdded(methodComponentContext, this)); + parameterMap.put( + KNNConstants.PARAMETERS, + getParameterMapWithDefaultsAdded(methodComponentContext, this, knnMethodConfigContext) + ); return parameterMap; } - return mapGenerator.apply(this, methodComponentContext); + return mapGenerator.apply(this, methodComponentContext, knnMethodConfigContext); } /** * Validate that the methodComponentContext is a valid configuration for this methodComponent * * @param methodComponentContext to be validated + * @param knnMethodConfigContext context for the method configuration * @return ValidationException produced by validation errors; null if no validations errors. */ - public ValidationException validate(MethodComponentContext methodComponentContext) { + public ValidationException validate(MethodComponentContext methodComponentContext, KNNMethodConfigContext knnMethodConfigContext) { Map providedParameters = methodComponentContext.getParameters(); - return validateParameters(parameters, providedParameters); - } - - /** - * Validate that the methodComponentContext is a valid configuration for this methodComponent, using additional data not present in the method component context - * - * @param methodComponentContext to be validated - * @param vectorSpaceInfo additional data not present in the method component context - * @return ValidationException produced by validation errors; null if no validations errors. - */ - public ValidationException validateWithData(MethodComponentContext methodComponentContext, VectorSpaceInfo vectorSpaceInfo) { - Map providedParameters = methodComponentContext.getParameters(); - List errorMessages = new ArrayList<>(); - if (providedParameters == null) { - return null; + ValidationException validationException = null; + if (!supportedVectorDataTypes.contains(knnMethodConfigContext.getVectorDataType())) { + validationException = new ValidationException(); + validationException.addValidationError( + String.format( + Locale.ROOT, + "Method \"%s\" is not supported for vector data type \"%s\".", + name, + knnMethodConfigContext.getVectorDataType() + ) + ); } - ValidationException parameterValidation; - for (Map.Entry parameter : providedParameters.entrySet()) { - if (!parameters.containsKey(parameter.getKey())) { - errorMessages.add(String.format("Invalid parameter for method \"%s\".", getName())); - continue; - } - - parameterValidation = parameters.get(parameter.getKey()).validateWithData(parameter.getValue(), vectorSpaceInfo); - if (parameterValidation != null) { - errorMessages.addAll(parameterValidation.validationErrors()); - } - } + ValidationException methodValidationException = validateParameters(parameters, providedParameters, knnMethodConfigContext); - if (errorMessages.isEmpty()) { - return null; + if (methodValidationException != null) { + validationException = validationException == null ? new ValidationException() : validationException; + validationException.addValidationErrors(methodValidationException.validationErrors()); } - ValidationException validationException = new ValidationException(); - validationException.addValidationErrors(errorMessages); return validationException; } @@ -217,11 +207,12 @@ public int estimateOverheadInKB(MethodComponentContext methodComponentContext, i */ public static class Builder { - private String name; - private Map> parameters; - private BiFunction> mapGenerator; + private final String name; + private final Map> parameters; + private TriFunction> mapGenerator; private TriFunction overheadInKBEstimator; private boolean requiresTraining; + private final Set supportedDataTypes; /** * Method to get a Builder instance @@ -230,7 +221,7 @@ public static class Builder { * @return Builder instance */ public static Builder builder(String name) { - return new MethodComponent.Builder(name); + return new Builder(name); } private Builder(String name) { @@ -238,6 +229,7 @@ private Builder(String name) { this.parameters = new HashMap<>(); this.mapGenerator = null; this.overheadInKBEstimator = (mc, mcc, d) -> 0L; + this.supportedDataTypes = new HashSet<>(); } /** @@ -258,7 +250,9 @@ public Builder addParameter(String parameterName, Parameter parameter) { * @param mapGenerator function to parse a MethodComponentContext as a map * @return this builder */ - public Builder setMapGenerator(BiFunction> mapGenerator) { + public Builder setMapGenerator( + TriFunction> mapGenerator + ) { this.mapGenerator = mapGenerator; return this; } @@ -284,6 +278,17 @@ public Builder setOverheadInKBEstimator(TriFunction dataTypeSet) { + supportedDataTypes.addAll(dataTypeSet); + return this; + } + /** * Build MethodComponent * @@ -303,11 +308,12 @@ public MethodComponent build() { */ public static Map getParameterMapWithDefaultsAdded( MethodComponentContext methodComponentContext, - MethodComponent methodComponent + MethodComponent methodComponent, + KNNMethodConfigContext knnMethodConfigContext ) { Map parametersWithDefaultsMap = new HashMap<>(); Map userProvidedParametersMap = methodComponentContext.getParameters(); - Version indexCreationVersion = methodComponentContext.getIndexVersion(); + Version indexCreationVersion = knnMethodConfigContext.getVersionCreated(); for (Parameter parameter : methodComponent.getParameters().values()) { if (methodComponentContext.getParameters().containsKey(parameter.getName())) { parametersWithDefaultsMap.put(parameter.getName(), userProvidedParametersMap.get(parameter.getName())); diff --git a/src/main/java/org/opensearch/knn/index/engine/MethodComponentContext.java b/src/main/java/org/opensearch/knn/index/engine/MethodComponentContext.java index fb4327487..586cc338f 100644 --- a/src/main/java/org/opensearch/knn/index/engine/MethodComponentContext.java +++ b/src/main/java/org/opensearch/knn/index/engine/MethodComponentContext.java @@ -8,9 +8,7 @@ import lombok.AllArgsConstructor; import lombok.Getter; import lombok.RequiredArgsConstructor; -import lombok.Setter; import org.apache.commons.lang.math.NumberUtils; -import org.opensearch.Version; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.io.stream.Writeable; @@ -51,10 +49,6 @@ public class MethodComponentContext implements ToXContentFragment, Writeable { private final String name; private final Map parameters; - @Getter - @Setter - private Version indexVersion; - /** * Constructor from stream. * diff --git a/src/main/java/org/opensearch/knn/index/engine/NativeLibrary.java b/src/main/java/org/opensearch/knn/index/engine/NativeLibrary.java index 1e34cc380..c3c61292a 100644 --- a/src/main/java/org/opensearch/knn/index/engine/NativeLibrary.java +++ b/src/main/java/org/opensearch/knn/index/engine/NativeLibrary.java @@ -59,9 +59,9 @@ public float score(float rawScore, SpaceType spaceType) { } @Override - public int estimateOverheadInKB(KNNMethodContext knnMethodContext, int dimension) { + public int estimateOverheadInKB(KNNMethodContext knnMethodContext, KNNMethodConfigContext knnMethodConfigContext) { String methodName = knnMethodContext.getMethodComponentContext().getName(); - return methods.get(methodName).estimateOverheadInKB(knnMethodContext, dimension); + return methods.get(methodName).estimateOverheadInKB(knnMethodContext, knnMethodConfigContext); } @Override diff --git a/src/main/java/org/opensearch/knn/index/engine/Parameter.java b/src/main/java/org/opensearch/knn/index/engine/Parameter.java index fbd3ae692..4dd6b9c33 100644 --- a/src/main/java/org/opensearch/knn/index/engine/Parameter.java +++ b/src/main/java/org/opensearch/knn/index/engine/Parameter.java @@ -5,14 +5,13 @@ package org.opensearch.knn.index.engine; +import lombok.Getter; import org.opensearch.common.ValidationException; -import org.opensearch.knn.training.VectorSpaceInfo; import java.util.Locale; import java.util.Map; import java.util.Objects; import java.util.function.BiFunction; -import java.util.function.Predicate; /** * Parameter that can be set for a method component @@ -21,10 +20,11 @@ */ public abstract class Parameter { - private String name; - private T defaultValue; - protected Predicate validator; - protected BiFunction validatorWithData; + @Getter + private final String name; + @Getter + private final T defaultValue; + protected BiFunction validator; /** * Constructor @@ -33,74 +33,31 @@ public abstract class Parameter { * @param defaultValue of the parameter * @param validator used to validate a parameter value passed */ - public Parameter(String name, T defaultValue, Predicate validator) { + public Parameter(String name, T defaultValue, BiFunction validator) { this.name = name; this.defaultValue = defaultValue; this.validator = validator; - this.validatorWithData = null; - } - - public Parameter(String name, T defaultValue, Predicate validator, BiFunction validatorWithData) { - this.name = name; - this.defaultValue = defaultValue; - this.validator = validator; - this.validatorWithData = validatorWithData; - } - - /** - * Getter for parameter name - * - * @return parameter name - */ - public String getName() { - return name; - } - - /** - * Get default value for parameter - * - * @return default value of the parameter - */ - public T getDefaultValue() { - return defaultValue; } /** * Check if the value passed in is valid * * @param value to be checked + * @param knnMethodConfigContext context for the validation * @return ValidationException produced by validation errors; null if no validations errors. */ - public abstract ValidationException validate(Object value); - - /** - * Check if the value passed in is valid, using additional data not present in the value - * - * @param value to be checked - * @param vectorSpaceInfo additional data not present in the value - * @return ValidationException produced by validation errors; null if no validations errors. - */ - public abstract ValidationException validateWithData(Object value, VectorSpaceInfo vectorSpaceInfo); + public abstract ValidationException validate(Object value, KNNMethodConfigContext knnMethodConfigContext); /** * Boolean method parameter */ public static class BooleanParameter extends Parameter { - public BooleanParameter(String name, Boolean defaultValue, Predicate validator) { + public BooleanParameter(String name, Boolean defaultValue, BiFunction validator) { super(name, defaultValue, validator); } - public BooleanParameter( - String name, - Boolean defaultValue, - Predicate validator, - BiFunction validatorWithData - ) { - super(name, defaultValue, validator, validatorWithData); - } - @Override - public ValidationException validate(Object value) { + public ValidationException validate(Object value, KNNMethodConfigContext knnMethodConfigContext) { ValidationException validationException = null; if (!(value instanceof Boolean)) { validationException = new ValidationException(); @@ -110,74 +67,24 @@ public ValidationException validate(Object value) { return validationException; } - if (!validator.test((Boolean) value)) { + if (!validator.apply((Boolean) value, knnMethodConfigContext)) { validationException = new ValidationException(); validationException.addValidationError(String.format("parameter validation failed for Boolean parameter [%s].", getName())); } return validationException; } - - @Override - public ValidationException validateWithData(Object value, VectorSpaceInfo vectorSpaceInfo) { - ValidationException validationException = null; - if (!(value instanceof Boolean)) { - validationException = new ValidationException(); - validationException.addValidationError(String.format("value not of type Boolean for Boolean parameter [%s].", getName())); - return validationException; - } - - if (validatorWithData == null) { - return null; - } - - if (!validatorWithData.apply((Boolean) value, vectorSpaceInfo)) { - validationException = new ValidationException(); - validationException.addValidationError(String.format("parameter validation failed for Boolean parameter [%s].", getName())); - } - - return validationException; - } } /** * Integer method parameter */ public static class IntegerParameter extends Parameter { - public IntegerParameter(String name, Integer defaultValue, Predicate validator) { + public IntegerParameter(String name, Integer defaultValue, BiFunction validator) { super(name, defaultValue, validator); } - public IntegerParameter( - String name, - Integer defaultValue, - Predicate validator, - BiFunction validatorWithData - ) { - super(name, defaultValue, validator, validatorWithData); - } - @Override - public ValidationException validate(Object value) { - ValidationException validationException = null; - if (!(value instanceof Integer)) { - validationException = new ValidationException(); - validationException.addValidationError( - String.format("Value not of type Integer for Integer " + "parameter \"%s\".", getName()) - ); - return validationException; - } - - if (!validator.test((Integer) value)) { - validationException = new ValidationException(); - validationException.addValidationError( - String.format("Parameter validation failed for Integer " + "parameter \"%s\".", getName()) - ); - } - return validationException; - } - - @Override - public ValidationException validateWithData(Object value, VectorSpaceInfo vectorSpaceInfo) { + public ValidationException validate(Object value, KNNMethodConfigContext knnMethodConfigContext) { ValidationException validationException = null; if (!(value instanceof Integer)) { validationException = new ValidationException(); @@ -187,11 +94,7 @@ public ValidationException validateWithData(Object value, VectorSpaceInfo vector return validationException; } - if (validatorWithData == null) { - return null; - } - - if (!validatorWithData.apply((Integer) value, vectorSpaceInfo)) { + if (!validator.apply((Integer) value, knnMethodConfigContext)) { validationException = new ValidationException(); validationException.addValidationError(String.format("parameter validation failed for Integer parameter [%s].", getName())); } @@ -204,53 +107,18 @@ public ValidationException validateWithData(Object value, VectorSpaceInfo vector * Double method parameter */ public static class DoubleParameter extends Parameter { - public DoubleParameter(String name, Double defaultValue, Predicate validator) { + public DoubleParameter(String name, Double defaultValue, BiFunction validator) { super(name, defaultValue, validator); } - public DoubleParameter( - String name, - Double defaultValue, - Predicate validator, - BiFunction validatorWithData - ) { - super(name, defaultValue, validator, validatorWithData); - } - @Override - public ValidationException validate(Object value) { + public ValidationException validate(Object value, KNNMethodConfigContext knnMethodConfigContext) { if (Objects.isNull(value)) { String validationErrorMsg = String.format(Locale.ROOT, "Null value provided for Double " + "parameter \"%s\".", getName()); return getValidationException(validationErrorMsg); } - if (value.equals(0)) value = 0.0; - if (!(value instanceof Double)) { - String validationErrorMsg = String.format( - Locale.ROOT, - "Value not of type Double for Double " + "parameter \"%s\".", - getName() - ); - return getValidationException(validationErrorMsg); - } - - if (!validator.test((Double) value)) { - String validationErrorMsg = String.format( - Locale.ROOT, - "Parameter validation failed for Double " + "parameter \"%s\".", - getName() - ); - return getValidationException(validationErrorMsg); - } - return null; - } - - @Override - public ValidationException validateWithData(Object value, VectorSpaceInfo vectorSpaceInfo) { - if (Objects.isNull(value)) { - String validationErrorMsg = String.format(Locale.ROOT, "Null value provided for Double " + "parameter \"%s\".", getName()); - return getValidationException(validationErrorMsg); - } + if (value.equals(0)) value = 0.0; if (!(value instanceof Double)) { String validationErrorMsg = String.format( @@ -261,11 +129,7 @@ public ValidationException validateWithData(Object value, VectorSpaceInfo vector return getValidationException(validationErrorMsg); } - if (validatorWithData == null) { - return null; - } - - if (!validatorWithData.apply((Double) value, vectorSpaceInfo)) { + if (!validator.apply((Double) value, knnMethodConfigContext)) { String validationErrorMsg = String.format(Locale.ROOT, "parameter validation failed for Double parameter [%s].", getName()); return getValidationException(validationErrorMsg); } @@ -291,47 +155,12 @@ public static class StringParameter extends Parameter { * @param defaultValue value to assign if the parameter is not set * @param validator used to validate the parameter value passed */ - public StringParameter(String name, String defaultValue, Predicate validator) { + public StringParameter(String name, String defaultValue, BiFunction validator) { super(name, defaultValue, validator); } - public StringParameter( - String name, - String defaultValue, - Predicate validator, - BiFunction validatorWithData - ) { - super(name, defaultValue, validator, validatorWithData); - } - - /** - * Check if the value passed in is valid - * - * @param value to be checked - * @return ValidationException produced by validation errors; null if no validations errors. - */ - @Override - public ValidationException validate(Object value) { - ValidationException validationException = null; - if (!(value instanceof String)) { - validationException = new ValidationException(); - validationException.addValidationError( - String.format("Value not of type String for String " + "parameter \"%s\".", getName()) - ); - return validationException; - } - - if (!validator.test((String) value)) { - validationException = new ValidationException(); - validationException.addValidationError( - String.format("Parameter validation failed for String " + "parameter \"%s\".", getName()) - ); - } - return validationException; - } - @Override - public ValidationException validateWithData(Object value, VectorSpaceInfo vectorSpaceInfo) { + public ValidationException validate(Object value, KNNMethodConfigContext knnMethodConfigContext) { ValidationException validationException = null; if (!(value instanceof String)) { validationException = new ValidationException(); @@ -341,11 +170,7 @@ public ValidationException validateWithData(Object value, VectorSpaceInfo vector return validationException; } - if (validatorWithData == null) { - return null; - } - - if (!validatorWithData.apply((String) value, vectorSpaceInfo)) { + if (!validator.apply((String) value, knnMethodConfigContext)) { validationException = new ValidationException(); validationException.addValidationError(String.format("parameter validation failed for String parameter [%s].", getName())); } @@ -361,7 +186,7 @@ public ValidationException validateWithData(Object value, VectorSpaceInfo vector */ public static class MethodComponentContextParameter extends Parameter { - private Map methodComponents; + private final Map methodComponents; /** * Constructor @@ -375,46 +200,18 @@ public MethodComponentContextParameter( MethodComponentContext defaultValue, Map methodComponents ) { - super(name, defaultValue, methodComponentContext -> { - if (!methodComponents.containsKey(methodComponentContext.getName())) { - return false; - } - - return methodComponents.get(methodComponentContext.getName()).validate(methodComponentContext) == null; - }, (methodComponentContext, vectorSpaceInfo) -> { + super(name, defaultValue, (methodComponentContext, knnMethodConfigContext) -> { if (!methodComponents.containsKey(methodComponentContext.getName())) { return false; } return methodComponents.get(methodComponentContext.getName()) - .validateWithData(methodComponentContext, vectorSpaceInfo) == null; + .validate(methodComponentContext, knnMethodConfigContext) == null; }); this.methodComponents = methodComponents; } @Override - public ValidationException validate(Object value) { - ValidationException validationException = null; - if (!(value instanceof MethodComponentContext)) { - validationException = new ValidationException(); - validationException.addValidationError( - String.format("Value not of type MethodComponentContext for" + " MethodComponentContext parameter \"%s\".", getName()) - ); - return validationException; - } - - if (!validator.test((MethodComponentContext) value)) { - validationException = new ValidationException(); - validationException.addValidationError("Parameter validation failed."); - validationException.addValidationError( - String.format("Parameter validation failed for " + "MethodComponentContext parameter \"%s\".", getName()) - ); - } - - return validationException; - } - - @Override - public ValidationException validateWithData(Object value, VectorSpaceInfo vectorSpaceInfo) { + public ValidationException validate(Object value, KNNMethodConfigContext knnMethodConfigContext) { ValidationException validationException = null; if (!(value instanceof MethodComponentContext)) { validationException = new ValidationException(); @@ -424,11 +221,7 @@ public ValidationException validateWithData(Object value, VectorSpaceInfo vector return validationException; } - if (validatorWithData == null) { - return null; - } - - if (!validatorWithData.apply((MethodComponentContext) value, vectorSpaceInfo)) { + if (!validator.apply((MethodComponentContext) value, knnMethodConfigContext)) { validationException = new ValidationException(); validationException.addValidationError( String.format("parameter validation failed for MethodComponentContext parameter [%s].", getName()) diff --git a/src/main/java/org/opensearch/knn/index/engine/faiss/AbstractFaissMethod.java b/src/main/java/org/opensearch/knn/index/engine/faiss/AbstractFaissMethod.java new file mode 100644 index 000000000..52b7efe73 --- /dev/null +++ b/src/main/java/org/opensearch/knn/index/engine/faiss/AbstractFaissMethod.java @@ -0,0 +1,84 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.engine.faiss; + +import org.opensearch.knn.index.SpaceType; +import org.opensearch.knn.index.VectorDataType; +import org.opensearch.knn.index.engine.AbstractKNNMethod; +import org.opensearch.knn.index.engine.KNNLibrarySearchContext; +import org.opensearch.knn.index.engine.KNNMethodConfigContext; +import org.opensearch.knn.index.engine.KNNMethodContext; +import org.opensearch.knn.index.engine.MethodComponent; +import org.opensearch.knn.index.mapper.PerDimensionProcessor; +import org.opensearch.knn.index.mapper.PerDimensionValidator; + +import java.util.Set; + +import static org.opensearch.knn.index.engine.faiss.FaissFP16Util.isFaissSQClipToFP16RangeEnabled; +import static org.opensearch.knn.index.engine.faiss.FaissFP16Util.isFaissSQfp16; + +public abstract class AbstractFaissMethod extends AbstractKNNMethod { + + /** + * Constructor for the AbstractFaissMethod class. + * + * @param methodComponent The method component used to create the method + * @param spaces The set of spaces supported by the method + * @param knnLibrarySearchContext The KNN library search context + */ + public AbstractFaissMethod(MethodComponent methodComponent, Set spaces, KNNLibrarySearchContext knnLibrarySearchContext) { + super(methodComponent, spaces, knnLibrarySearchContext); + } + + @Override + protected PerDimensionValidator doGetPerDimensionValidator( + KNNMethodContext knnMethodContext, + KNNMethodConfigContext knnMethodConfigContext + ) { + VectorDataType vectorDataType = knnMethodConfigContext.getVectorDataType(); + if (VectorDataType.BINARY == vectorDataType) { + return PerDimensionValidator.DEFAULT_BIT_VALIDATOR; + } + + if (VectorDataType.BYTE == vectorDataType) { + return PerDimensionValidator.DEFAULT_BYTE_VALIDATOR; + } + + if (VectorDataType.FLOAT == vectorDataType) { + if (isFaissSQfp16(knnMethodContext.getMethodComponentContext())) { + return FaissFP16Util.FP16_VALIDATOR; + } + return PerDimensionValidator.DEFAULT_FLOAT_VALIDATOR; + } + + throw new IllegalStateException("Unsupported vector data type " + vectorDataType); + } + + @Override + protected PerDimensionProcessor doGetPerDimensionProcessor( + KNNMethodContext knnMethodContext, + KNNMethodConfigContext knnMethodConfigContext + ) { + VectorDataType vectorDataType = knnMethodConfigContext.getVectorDataType(); + + if (VectorDataType.BINARY == vectorDataType) { + return PerDimensionProcessor.NOOP_PROCESSOR; + } + + if (VectorDataType.BYTE == vectorDataType) { + return PerDimensionProcessor.NOOP_PROCESSOR; + } + + if (VectorDataType.FLOAT == vectorDataType) { + if (isFaissSQClipToFP16RangeEnabled(knnMethodContext.getMethodComponentContext())) { + return FaissFP16Util.CLIP_TO_FP16_PROCESSOR; + } + return PerDimensionProcessor.NOOP_PROCESSOR; + } + + throw new IllegalStateException("Unsupported vector data type " + vectorDataType); + } +} diff --git a/src/main/java/org/opensearch/knn/index/engine/faiss/FaissFP16Util.java b/src/main/java/org/opensearch/knn/index/engine/faiss/FaissFP16Util.java new file mode 100644 index 000000000..8e76ca0fb --- /dev/null +++ b/src/main/java/org/opensearch/knn/index/engine/faiss/FaissFP16Util.java @@ -0,0 +1,145 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.engine.faiss; + +import org.opensearch.knn.index.engine.MethodComponentContext; +import org.opensearch.knn.index.mapper.PerDimensionProcessor; +import org.opensearch.knn.index.mapper.PerDimensionValidator; + +import java.util.Locale; +import java.util.Map; +import java.util.Objects; + +import static org.opensearch.knn.common.KNNConstants.ENCODER_SQ; +import static org.opensearch.knn.common.KNNConstants.FAISS_SQ_CLIP; +import static org.opensearch.knn.common.KNNConstants.FAISS_SQ_ENCODER_FP16; +import static org.opensearch.knn.common.KNNConstants.FAISS_SQ_TYPE; +import static org.opensearch.knn.common.KNNConstants.FP16_MAX_VALUE; +import static org.opensearch.knn.common.KNNConstants.FP16_MIN_VALUE; +import static org.opensearch.knn.common.KNNConstants.METHOD_ENCODER_PARAMETER; +import static org.opensearch.knn.common.KNNValidationUtil.validateFloatVectorValue; + +public class FaissFP16Util { + + // Validates if it is a finite number and within the fp16 range of [-65504 to 65504]. + static PerDimensionValidator FP16_VALIDATOR = new PerDimensionValidator() { + @Override + public void validate(float value) { + validateFP16VectorValue(value); + } + + @Override + public void validateByte(float value) { + throw new IllegalStateException("DEFAULT_FP16_VALIDATOR should only be used for float vectors"); + } + }; + + // If the encoder parameter, "clip" is set to True, if the vector value is outside the FP16 range then it will be + // clipped to FP16 range. + static PerDimensionProcessor CLIP_TO_FP16_PROCESSOR = new PerDimensionProcessor() { + @Override + public float process(float value) { + return clipVectorValueToFP16Range(value); + } + + @Override + public float processByte(float value) { + throw new IllegalStateException("CLIP_TO_FP16_PROCESSOR should not be called with byte type"); + } + }; + + /** + * Validate the float vector value and if it is outside FP16 range, + * then it will be clipped to FP16 range of [-65504 to 65504]. + * + * @param value float vector value + * @return vector value clipped to FP16 range + */ + public static float clipVectorValueToFP16Range(float value) { + validateFloatVectorValue(value); + if (value < FP16_MIN_VALUE) return FP16_MIN_VALUE; + if (value > FP16_MAX_VALUE) return FP16_MAX_VALUE; + return value; + } + + /** + * Validate the float vector value and throw exception if it is not a number or not in the finite range + * or is not within the FP16 range of [-65504 to 65504]. + * + * @param value float vector value + */ + public static void validateFP16VectorValue(float value) { + validateFloatVectorValue(value); + if (value < FP16_MIN_VALUE || value > FP16_MAX_VALUE) { + throw new IllegalArgumentException( + String.format( + Locale.ROOT, + "encoder name is set as [%s] and type is set as [%s] in index mapping. But, KNN vector values are not within in the FP16 range [%f, %f]", + ENCODER_SQ, + FAISS_SQ_ENCODER_FP16, + FP16_MIN_VALUE, + FP16_MAX_VALUE + ) + ); + } + } + + /** + * Verify mapping and return true if it is a "faiss" Index using "sq" encoder of type "fp16" + * + * @param methodComponentContext MethodComponentContext + * @return true if it is a "faiss" Index using "sq" encoder of type "fp16" + */ + static boolean isFaissSQfp16(MethodComponentContext methodComponentContext) { + MethodComponentContext encoderContext = extractEncoderMethodComponentContext(methodComponentContext); + if (encoderContext == null) { + return false; + } + + // returns true if encoder name is "sq" and type is "fp16" + return ENCODER_SQ.equals(encoderContext.getName()) + && FAISS_SQ_ENCODER_FP16.equals(encoderContext.getParameters().getOrDefault(FAISS_SQ_TYPE, FAISS_SQ_ENCODER_FP16)); + } + + /** + * Verify mapping and return the value of "clip" parameter(default false) for a "faiss" Index + * using "sq" encoder of type "fp16". + * + * @param methodComponentContext MethodComponentContext + * @return boolean value of "clip" parameter + */ + static boolean isFaissSQClipToFP16RangeEnabled(MethodComponentContext methodComponentContext) { + MethodComponentContext encoderContext = extractEncoderMethodComponentContext(methodComponentContext); + if (encoderContext == null) { + return false; + } + return (boolean) encoderContext.getParameters().getOrDefault(FAISS_SQ_CLIP, false); + } + + static MethodComponentContext extractEncoderMethodComponentContext(MethodComponentContext methodComponentContext) { + if (Objects.isNull(methodComponentContext)) { + return null; + } + + if (methodComponentContext.getParameters().isEmpty()) { + return null; + } + + Map methodComponentParams = methodComponentContext.getParameters(); + + // The method component parameters should have an encoder + if (!methodComponentParams.containsKey(METHOD_ENCODER_PARAMETER)) { + return null; + } + + // Validate if the object is of type MethodComponentContext before casting it later + if (!(methodComponentParams.get(METHOD_ENCODER_PARAMETER) instanceof MethodComponentContext)) { + return null; + } + + return (MethodComponentContext) methodComponentParams.get(METHOD_ENCODER_PARAMETER); + } +} diff --git a/src/main/java/org/opensearch/knn/index/engine/faiss/FaissFlatEncoder.java b/src/main/java/org/opensearch/knn/index/engine/faiss/FaissFlatEncoder.java index aea3bf51a..5e6e4060f 100644 --- a/src/main/java/org/opensearch/knn/index/engine/faiss/FaissFlatEncoder.java +++ b/src/main/java/org/opensearch/knn/index/engine/faiss/FaissFlatEncoder.java @@ -5,24 +5,36 @@ package org.opensearch.knn.index.engine.faiss; +import com.google.common.collect.ImmutableSet; import org.opensearch.knn.common.KNNConstants; +import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.engine.Encoder; import org.opensearch.knn.index.engine.MethodComponent; +import java.util.Set; + /** * Flat faiss encoder. Flat encoding means that it does nothing. It needs an encoder, though, because it * is used in generating the index description. */ public class FaissFlatEncoder implements Encoder { + private static final Set SUPPORTED_DATA_TYPES = ImmutableSet.of( + VectorDataType.FLOAT, + VectorDataType.BYTE, + VectorDataType.BINARY + ); + private final static MethodComponent METHOD_COMPONENT = MethodComponent.Builder.builder(KNNConstants.ENCODER_FLAT) .setMapGenerator( - ((methodComponent, methodComponentContext) -> MethodAsMapBuilder.builder( + ((methodComponent, methodComponentContext, knnMethodConfigContext) -> MethodAsMapBuilder.builder( KNNConstants.FAISS_FLAT_DESCRIPTION, methodComponent, - methodComponentContext + methodComponentContext, + knnMethodConfigContext ).build()) ) + .addSupportedDataTypes(SUPPORTED_DATA_TYPES) .build(); @Override diff --git a/src/main/java/org/opensearch/knn/index/engine/faiss/FaissHNSWMethod.java b/src/main/java/org/opensearch/knn/index/engine/faiss/FaissHNSWMethod.java index 382a71741..ee6a4f101 100644 --- a/src/main/java/org/opensearch/knn/index/engine/faiss/FaissHNSWMethod.java +++ b/src/main/java/org/opensearch/knn/index/engine/faiss/FaissHNSWMethod.java @@ -5,9 +5,11 @@ package org.opensearch.knn.index.engine.faiss; +import com.google.common.collect.ImmutableSet; import org.opensearch.knn.common.KNNConstants; import org.opensearch.knn.index.KNNSettings; import org.opensearch.knn.index.SpaceType; +import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.engine.AbstractKNNMethod; import org.opensearch.knn.index.engine.DefaultHnswSearchContext; import org.opensearch.knn.index.engine.Encoder; @@ -27,11 +29,15 @@ import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_EF_CONSTRUCTION; import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_EF_SEARCH; import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_M; +import static org.opensearch.knn.index.engine.faiss.Faiss.FAISS_BINARY_INDEX_DESCRIPTION_PREFIX; /** * Faiss HNSW method implementation */ -public class FaissHNSWMethod extends AbstractKNNMethod { +public class FaissHNSWMethod extends AbstractFaissMethod { + + private static final Set SUPPORTED_DATA_TYPES = ImmutableSet.of(VectorDataType.FLOAT, VectorDataType.BINARY); + public final static List SUPPORTED_SPACES = Arrays.asList( SpaceType.UNDEFINED, SpaceType.HAMMING, @@ -56,30 +62,41 @@ public FaissHNSWMethod() { private static MethodComponent initMethodComponent() { return MethodComponent.Builder.builder(METHOD_HNSW) + .addSupportedDataTypes(SUPPORTED_DATA_TYPES) .addParameter( METHOD_PARAMETER_M, - new Parameter.IntegerParameter(METHOD_PARAMETER_M, KNNSettings.INDEX_KNN_DEFAULT_ALGO_PARAM_M, v -> v > 0) + new Parameter.IntegerParameter(METHOD_PARAMETER_M, KNNSettings.INDEX_KNN_DEFAULT_ALGO_PARAM_M, (v, context) -> v > 0) ) .addParameter( METHOD_PARAMETER_EF_CONSTRUCTION, new Parameter.IntegerParameter( METHOD_PARAMETER_EF_CONSTRUCTION, KNNSettings.INDEX_KNN_DEFAULT_ALGO_PARAM_EF_CONSTRUCTION, - v -> v > 0 + (v, context) -> v > 0 ) ) .addParameter( METHOD_PARAMETER_EF_SEARCH, - new Parameter.IntegerParameter(METHOD_PARAMETER_EF_SEARCH, KNNSettings.INDEX_KNN_DEFAULT_ALGO_PARAM_EF_SEARCH, v -> v > 0) + new Parameter.IntegerParameter( + METHOD_PARAMETER_EF_SEARCH, + KNNSettings.INDEX_KNN_DEFAULT_ALGO_PARAM_EF_SEARCH, + (v, context) -> v > 0 + ) ) .addParameter(METHOD_ENCODER_PARAMETER, initEncoderParameter()) - .setMapGenerator( - ((methodComponent, methodComponentContext) -> MethodAsMapBuilder.builder( - FAISS_HNSW_DESCRIPTION, + .setMapGenerator(((methodComponent, methodComponentContext, knnMethodConfigContext) -> { + String prefix = ""; + if (knnMethodConfigContext.getVectorDataType() == VectorDataType.BINARY) { + prefix = FAISS_BINARY_INDEX_DESCRIPTION_PREFIX; + } + + return MethodAsMapBuilder.builder( + prefix + FAISS_HNSW_DESCRIPTION, methodComponent, - methodComponentContext - ).addParameter(METHOD_PARAMETER_M, "", "").addParameter(METHOD_ENCODER_PARAMETER, ",", "").build()) - ) + methodComponentContext, + knnMethodConfigContext + ).addParameter(METHOD_PARAMETER_M, "", "").addParameter(METHOD_ENCODER_PARAMETER, ",", "").build(); + })) .build(); } diff --git a/src/main/java/org/opensearch/knn/index/engine/faiss/FaissHNSWPQEncoder.java b/src/main/java/org/opensearch/knn/index/engine/faiss/FaissHNSWPQEncoder.java index 9880b2cd9..8d53f3c0a 100644 --- a/src/main/java/org/opensearch/knn/index/engine/faiss/FaissHNSWPQEncoder.java +++ b/src/main/java/org/opensearch/knn/index/engine/faiss/FaissHNSWPQEncoder.java @@ -5,12 +5,15 @@ package org.opensearch.knn.index.engine.faiss; +import com.google.common.collect.ImmutableSet; import org.opensearch.knn.common.KNNConstants; +import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.engine.Encoder; import org.opensearch.knn.index.engine.MethodComponent; import org.opensearch.knn.index.engine.Parameter; import java.util.Objects; +import java.util.Set; import static org.opensearch.knn.common.KNNConstants.BYTES_PER_KILOBYTES; import static org.opensearch.knn.common.KNNConstants.ENCODER_PARAMETER_PQ_CODE_COUNT_DEFAULT; @@ -26,30 +29,34 @@ */ public class FaissHNSWPQEncoder implements Encoder { + private static final Set SUPPORTED_DATA_TYPES = ImmutableSet.of(VectorDataType.FLOAT); + private final static MethodComponent METHOD_COMPONENT = MethodComponent.Builder.builder(KNNConstants.ENCODER_PQ) + .addSupportedDataTypes(SUPPORTED_DATA_TYPES) .addParameter( ENCODER_PARAMETER_PQ_M, - new Parameter.IntegerParameter( - ENCODER_PARAMETER_PQ_M, - ENCODER_PARAMETER_PQ_CODE_COUNT_DEFAULT, - v -> v > 0 && v < ENCODER_PARAMETER_PQ_CODE_COUNT_LIMIT, - (v, vectorSpaceInfo) -> vectorSpaceInfo.getDimension() % v == 0 - ) + new Parameter.IntegerParameter(ENCODER_PARAMETER_PQ_M, ENCODER_PARAMETER_PQ_CODE_COUNT_DEFAULT, (v, context) -> { + boolean isValueGreaterThan0 = v > 0; + boolean isValueLessThanCodeCountLimit = v < ENCODER_PARAMETER_PQ_CODE_COUNT_LIMIT; + boolean isDimensionDivisibleByValue = context.getDimension() % v == 0; + return isValueGreaterThan0 && isValueLessThanCodeCountLimit && isDimensionDivisibleByValue; + }) ) .addParameter( ENCODER_PARAMETER_PQ_CODE_SIZE, new Parameter.IntegerParameter( ENCODER_PARAMETER_PQ_CODE_SIZE, ENCODER_PARAMETER_PQ_CODE_SIZE_DEFAULT, - v -> Objects.equals(v, ENCODER_PARAMETER_PQ_CODE_SIZE_DEFAULT) + (v, context) -> Objects.equals(v, ENCODER_PARAMETER_PQ_CODE_SIZE_DEFAULT) ) ) .setRequiresTraining(true) .setMapGenerator( - ((methodComponent, methodComponentContext) -> MethodAsMapBuilder.builder( + ((methodComponent, methodComponentContext, knnMethodConfigContext) -> MethodAsMapBuilder.builder( FAISS_PQ_DESCRIPTION, methodComponent, - methodComponentContext + methodComponentContext, + knnMethodConfigContext ).addParameter(ENCODER_PARAMETER_PQ_M, "", "").build()) ) .setOverheadInKBEstimator((methodComponent, methodComponentContext, dimension) -> { diff --git a/src/main/java/org/opensearch/knn/index/engine/faiss/FaissIVFMethod.java b/src/main/java/org/opensearch/knn/index/engine/faiss/FaissIVFMethod.java index aa05e8c87..a21810b50 100644 --- a/src/main/java/org/opensearch/knn/index/engine/faiss/FaissIVFMethod.java +++ b/src/main/java/org/opensearch/knn/index/engine/faiss/FaissIVFMethod.java @@ -5,8 +5,10 @@ package org.opensearch.knn.index.engine.faiss; +import com.google.common.collect.ImmutableSet; import org.opensearch.knn.common.KNNConstants; import org.opensearch.knn.index.SpaceType; +import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.engine.AbstractKNNMethod; import org.opensearch.knn.index.engine.DefaultIVFSearchContext; import org.opensearch.knn.index.engine.Encoder; @@ -30,11 +32,14 @@ import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_NPROBES; import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_NPROBES_DEFAULT; import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_NPROBES_LIMIT; +import static org.opensearch.knn.index.engine.faiss.Faiss.FAISS_BINARY_INDEX_DESCRIPTION_PREFIX; /** * Faiss ivf implementation */ -public class FaissIVFMethod extends AbstractKNNMethod { +public class FaissIVFMethod extends AbstractFaissMethod { + + private static final Set SUPPORTED_DATA_TYPES = ImmutableSet.of(VectorDataType.FLOAT, VectorDataType.BINARY); public final static List SUPPORTED_SPACES = Arrays.asList( SpaceType.UNDEFINED, @@ -60,12 +65,13 @@ public FaissIVFMethod() { private static MethodComponent initMethodComponent() { return MethodComponent.Builder.builder(METHOD_IVF) + .addSupportedDataTypes(SUPPORTED_DATA_TYPES) .addParameter( METHOD_PARAMETER_NPROBES, new Parameter.IntegerParameter( METHOD_PARAMETER_NPROBES, METHOD_PARAMETER_NPROBES_DEFAULT, - v -> v > 0 && v < METHOD_PARAMETER_NPROBES_LIMIT + (v, context) -> v > 0 && v < METHOD_PARAMETER_NPROBES_LIMIT ) ) .addParameter( @@ -73,18 +79,24 @@ private static MethodComponent initMethodComponent() { new Parameter.IntegerParameter( METHOD_PARAMETER_NLIST, METHOD_PARAMETER_NLIST_DEFAULT, - v -> v > 0 && v < METHOD_PARAMETER_NLIST_LIMIT + (v, context) -> v > 0 && v < METHOD_PARAMETER_NLIST_LIMIT ) ) .addParameter(METHOD_ENCODER_PARAMETER, initEncoderParameter()) .setRequiresTraining(true) - .setMapGenerator( - ((methodComponent, methodComponentContext) -> MethodAsMapBuilder.builder( - FAISS_IVF_DESCRIPTION, + .setMapGenerator(((methodComponent, methodComponentContext, knnMethodConfigContext) -> { + String prefix = ""; + if (knnMethodConfigContext.getVectorDataType() == VectorDataType.BINARY) { + prefix = FAISS_BINARY_INDEX_DESCRIPTION_PREFIX; + } + + return MethodAsMapBuilder.builder( + prefix + FAISS_IVF_DESCRIPTION, methodComponent, - methodComponentContext - ).addParameter(METHOD_PARAMETER_NLIST, "", "").addParameter(METHOD_ENCODER_PARAMETER, ",", "").build()) - ) + methodComponentContext, + knnMethodConfigContext + ).addParameter(METHOD_PARAMETER_NLIST, "", "").addParameter(METHOD_ENCODER_PARAMETER, ",", "").build(); + })) .setOverheadInKBEstimator((methodComponent, methodComponentContext, dimension) -> { // Size estimate formula: (4 * nlists * d) / 1024 + 1 diff --git a/src/main/java/org/opensearch/knn/index/engine/faiss/FaissIVFPQEncoder.java b/src/main/java/org/opensearch/knn/index/engine/faiss/FaissIVFPQEncoder.java index b9632004d..b38f5c816 100644 --- a/src/main/java/org/opensearch/knn/index/engine/faiss/FaissIVFPQEncoder.java +++ b/src/main/java/org/opensearch/knn/index/engine/faiss/FaissIVFPQEncoder.java @@ -5,11 +5,15 @@ package org.opensearch.knn.index.engine.faiss; +import com.google.common.collect.ImmutableSet; import org.opensearch.knn.common.KNNConstants; +import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.engine.Encoder; import org.opensearch.knn.index.engine.MethodComponent; import org.opensearch.knn.index.engine.Parameter; +import java.util.Set; + import static org.opensearch.knn.common.KNNConstants.BYTES_PER_KILOBYTES; import static org.opensearch.knn.common.KNNConstants.ENCODER_PARAMETER_PQ_CODE_COUNT_DEFAULT; import static org.opensearch.knn.common.KNNConstants.ENCODER_PARAMETER_PQ_CODE_COUNT_LIMIT; @@ -24,30 +28,35 @@ * {@link FaissHNSWPQEncoder}. Hence, they are separate classes. */ public class FaissIVFPQEncoder implements Encoder { + + private static final Set SUPPORTED_DATA_TYPES = ImmutableSet.of(VectorDataType.FLOAT); + private final static MethodComponent METHOD_COMPONENT = MethodComponent.Builder.builder(KNNConstants.ENCODER_PQ) + .addSupportedDataTypes(SUPPORTED_DATA_TYPES) .addParameter( ENCODER_PARAMETER_PQ_M, - new Parameter.IntegerParameter( - ENCODER_PARAMETER_PQ_M, - ENCODER_PARAMETER_PQ_CODE_COUNT_DEFAULT, - v -> v > 0 && v < ENCODER_PARAMETER_PQ_CODE_COUNT_LIMIT, - (v, vectorSpaceInfo) -> vectorSpaceInfo.getDimension() % v == 0 - ) + new Parameter.IntegerParameter(ENCODER_PARAMETER_PQ_M, ENCODER_PARAMETER_PQ_CODE_COUNT_DEFAULT, (v, context) -> { + boolean isValueGreaterThan0 = v > 0; + boolean isValueLessThanCodeCountLimit = v < ENCODER_PARAMETER_PQ_CODE_COUNT_LIMIT; + boolean isDimensionDivisibleByValue = context.getDimension() % v == 0; + return isValueGreaterThan0 && isValueLessThanCodeCountLimit && isDimensionDivisibleByValue; + }) ) .addParameter( ENCODER_PARAMETER_PQ_CODE_SIZE, - new Parameter.IntegerParameter( - ENCODER_PARAMETER_PQ_CODE_SIZE, - ENCODER_PARAMETER_PQ_CODE_SIZE_DEFAULT, - v -> v > 0 && v < ENCODER_PARAMETER_PQ_CODE_SIZE_LIMIT - ) + new Parameter.IntegerParameter(ENCODER_PARAMETER_PQ_CODE_SIZE, ENCODER_PARAMETER_PQ_CODE_SIZE_DEFAULT, (v, context) -> { + boolean isValueGreaterThan0 = v > 0; + boolean isValueLessThanCodeSizeLimit = v < ENCODER_PARAMETER_PQ_CODE_SIZE_LIMIT; + return isValueGreaterThan0 && isValueLessThanCodeSizeLimit; + }) ) .setRequiresTraining(true) .setMapGenerator( - ((methodComponent, methodComponentContext) -> MethodAsMapBuilder.builder( + ((methodComponent, methodComponentContext, knnMethodConfigContext) -> MethodAsMapBuilder.builder( FAISS_PQ_DESCRIPTION, methodComponent, - methodComponentContext + methodComponentContext, + knnMethodConfigContext ).addParameter(ENCODER_PARAMETER_PQ_M, "", "").addParameter(ENCODER_PARAMETER_PQ_CODE_SIZE, "x", "").build()) ) .setOverheadInKBEstimator((methodComponent, methodComponentContext, dimension) -> { diff --git a/src/main/java/org/opensearch/knn/index/engine/faiss/FaissSQEncoder.java b/src/main/java/org/opensearch/knn/index/engine/faiss/FaissSQEncoder.java index eb0af9c38..2d0d184ca 100644 --- a/src/main/java/org/opensearch/knn/index/engine/faiss/FaissSQEncoder.java +++ b/src/main/java/org/opensearch/knn/index/engine/faiss/FaissSQEncoder.java @@ -5,11 +5,14 @@ package org.opensearch.knn.index.engine.faiss; +import com.google.common.collect.ImmutableSet; +import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.engine.Encoder; import org.opensearch.knn.index.engine.MethodComponent; import org.opensearch.knn.index.engine.Parameter; import java.util.Objects; +import java.util.Set; import static org.opensearch.knn.common.KNNConstants.ENCODER_SQ; import static org.opensearch.knn.common.KNNConstants.FAISS_SQ_CLIP; @@ -22,14 +25,22 @@ * Faiss SQ encoder */ public class FaissSQEncoder implements Encoder { + + private static final Set SUPPORTED_DATA_TYPES = ImmutableSet.of(VectorDataType.FLOAT); + private final static MethodComponent METHOD_COMPONENT = MethodComponent.Builder.builder(ENCODER_SQ) - .addParameter(FAISS_SQ_TYPE, new Parameter.StringParameter(FAISS_SQ_TYPE, FAISS_SQ_ENCODER_FP16, FAISS_SQ_ENCODER_TYPES::contains)) - .addParameter(FAISS_SQ_CLIP, new Parameter.BooleanParameter(FAISS_SQ_CLIP, false, Objects::nonNull)) + .addSupportedDataTypes(SUPPORTED_DATA_TYPES) + .addParameter( + FAISS_SQ_TYPE, + new Parameter.StringParameter(FAISS_SQ_TYPE, FAISS_SQ_ENCODER_FP16, (v, context) -> FAISS_SQ_ENCODER_TYPES.contains(v)) + ) + .addParameter(FAISS_SQ_CLIP, new Parameter.BooleanParameter(FAISS_SQ_CLIP, false, (v, context) -> Objects.nonNull(v))) .setMapGenerator( - ((methodComponent, methodComponentContext) -> MethodAsMapBuilder.builder( + ((methodComponent, methodComponentContext, knnMethodConfigContext) -> MethodAsMapBuilder.builder( FAISS_SQ_DESCRIPTION, methodComponent, - methodComponentContext + methodComponentContext, + knnMethodConfigContext ).addParameter(FAISS_SQ_TYPE, "", "").build()) ) .build(); diff --git a/src/main/java/org/opensearch/knn/index/engine/faiss/MethodAsMapBuilder.java b/src/main/java/org/opensearch/knn/index/engine/faiss/MethodAsMapBuilder.java index 445abfdd8..abb3d08c9 100644 --- a/src/main/java/org/opensearch/knn/index/engine/faiss/MethodAsMapBuilder.java +++ b/src/main/java/org/opensearch/knn/index/engine/faiss/MethodAsMapBuilder.java @@ -7,6 +7,7 @@ import lombok.AllArgsConstructor; import org.opensearch.knn.common.KNNConstants; +import org.opensearch.knn.index.engine.KNNMethodConfigContext; import org.opensearch.knn.index.engine.MethodComponent; import org.opensearch.knn.index.engine.MethodComponentContext; import org.opensearch.knn.index.engine.Parameter; @@ -29,6 +30,7 @@ class MethodAsMapBuilder { String indexDescription; MethodComponent methodComponent; Map methodAsMap; + KNNMethodConfigContext knnMethodConfigContext; /** * Add a parameter that will be used in the index description for the given method component @@ -55,7 +57,7 @@ MethodAsMapBuilder addParameter(String parameterName, String prefix, String suff subMethodComponentContext.getName() ); - Map subMethodAsMap = subMethodComponent.getAsMap(subMethodComponentContext); + Map subMethodAsMap = subMethodComponent.getAsMap(subMethodComponentContext, knnMethodConfigContext); indexDescription += subMethodAsMap.get(KNNConstants.INDEX_DESCRIPTION_PARAMETER); subMethodAsMap.remove(KNNConstants.INDEX_DESCRIPTION_PARAMETER); @@ -85,11 +87,15 @@ Map build() { static MethodAsMapBuilder builder( String baseDescription, MethodComponent methodComponent, - MethodComponentContext methodComponentContext + MethodComponentContext methodComponentContext, + KNNMethodConfigContext knnMethodConfigContext ) { Map initialMap = new HashMap<>(); initialMap.put(NAME, methodComponent.getName()); - initialMap.put(PARAMETERS, MethodComponent.getParameterMapWithDefaultsAdded(methodComponentContext, methodComponent)); - return new MethodAsMapBuilder(baseDescription, methodComponent, initialMap); + initialMap.put( + PARAMETERS, + MethodComponent.getParameterMapWithDefaultsAdded(methodComponentContext, methodComponent, knnMethodConfigContext) + ); + return new MethodAsMapBuilder(baseDescription, methodComponent, initialMap, knnMethodConfigContext); } } diff --git a/src/main/java/org/opensearch/knn/index/engine/lucene/LuceneHNSWMethod.java b/src/main/java/org/opensearch/knn/index/engine/lucene/LuceneHNSWMethod.java index c6fcdb7c4..317f67c10 100644 --- a/src/main/java/org/opensearch/knn/index/engine/lucene/LuceneHNSWMethod.java +++ b/src/main/java/org/opensearch/knn/index/engine/lucene/LuceneHNSWMethod.java @@ -5,9 +5,11 @@ package org.opensearch.knn.index.engine.lucene; +import com.google.common.collect.ImmutableSet; import org.opensearch.knn.common.KNNConstants; import org.opensearch.knn.index.KNNSettings; import org.opensearch.knn.index.SpaceType; +import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.engine.AbstractKNNMethod; import org.opensearch.knn.index.engine.Encoder; import org.opensearch.knn.index.engine.MethodComponent; @@ -30,6 +32,8 @@ */ public class LuceneHNSWMethod extends AbstractKNNMethod { + private static final Set SUPPORTED_DATA_TYPES = ImmutableSet.of(VectorDataType.FLOAT, VectorDataType.BYTE); + public final static List SUPPORTED_SPACES = Arrays.asList( SpaceType.UNDEFINED, SpaceType.L2, @@ -54,16 +58,17 @@ public LuceneHNSWMethod() { private static MethodComponent initMethodComponent() { return MethodComponent.Builder.builder(METHOD_HNSW) + .addSupportedDataTypes(SUPPORTED_DATA_TYPES) .addParameter( METHOD_PARAMETER_M, - new Parameter.IntegerParameter(METHOD_PARAMETER_M, KNNSettings.INDEX_KNN_DEFAULT_ALGO_PARAM_M, v -> v > 0) + new Parameter.IntegerParameter(METHOD_PARAMETER_M, KNNSettings.INDEX_KNN_DEFAULT_ALGO_PARAM_M, (v, context) -> v > 0) ) .addParameter( METHOD_PARAMETER_EF_CONSTRUCTION, new Parameter.IntegerParameter( METHOD_PARAMETER_EF_CONSTRUCTION, KNNSettings.INDEX_KNN_DEFAULT_ALGO_PARAM_EF_CONSTRUCTION, - v -> v > 0 + (v, context) -> v > 0 ) ) .addParameter(METHOD_ENCODER_PARAMETER, initEncoderParameter()) diff --git a/src/main/java/org/opensearch/knn/index/engine/lucene/LuceneHNSWSearchContext.java b/src/main/java/org/opensearch/knn/index/engine/lucene/LuceneHNSWSearchContext.java index 2c4da27df..bcc1c9af0 100644 --- a/src/main/java/org/opensearch/knn/index/engine/lucene/LuceneHNSWSearchContext.java +++ b/src/main/java/org/opensearch/knn/index/engine/lucene/LuceneHNSWSearchContext.java @@ -17,7 +17,10 @@ public class LuceneHNSWSearchContext implements KNNLibrarySearchContext { private final Map> supportedMethodParameters = ImmutableMap.>builder() - .put(MethodParameter.EF_SEARCH.getName(), new Parameter.IntegerParameter(MethodParameter.EF_SEARCH.getName(), null, value -> true)) + .put( + MethodParameter.EF_SEARCH.getName(), + new Parameter.IntegerParameter(MethodParameter.EF_SEARCH.getName(), null, (v, context) -> true) + ) .build(); @Override diff --git a/src/main/java/org/opensearch/knn/index/engine/lucene/LuceneSQEncoder.java b/src/main/java/org/opensearch/knn/index/engine/lucene/LuceneSQEncoder.java index fac851ea1..0ec43db41 100644 --- a/src/main/java/org/opensearch/knn/index/engine/lucene/LuceneSQEncoder.java +++ b/src/main/java/org/opensearch/knn/index/engine/lucene/LuceneSQEncoder.java @@ -5,11 +5,14 @@ package org.opensearch.knn.index.engine.lucene; +import com.google.common.collect.ImmutableSet; +import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.engine.Encoder; import org.opensearch.knn.index.engine.MethodComponent; import org.opensearch.knn.index.engine.Parameter; import java.util.List; +import java.util.Set; import static org.opensearch.knn.common.KNNConstants.DYNAMIC_CONFIDENCE_INTERVAL; import static org.opensearch.knn.common.KNNConstants.ENCODER_SQ; @@ -23,19 +26,22 @@ * Lucene scalar quantization encoder */ public class LuceneSQEncoder implements Encoder { + private static final Set SUPPORTED_DATA_TYPES = ImmutableSet.of(VectorDataType.FLOAT); + private final static List LUCENE_SQ_BITS_SUPPORTED = List.of(7); private final static MethodComponent METHOD_COMPONENT = MethodComponent.Builder.builder(ENCODER_SQ) + .addSupportedDataTypes(SUPPORTED_DATA_TYPES) .addParameter( LUCENE_SQ_CONFIDENCE_INTERVAL, new Parameter.DoubleParameter( LUCENE_SQ_CONFIDENCE_INTERVAL, null, - v -> v == DYNAMIC_CONFIDENCE_INTERVAL || (v >= MINIMUM_CONFIDENCE_INTERVAL && v <= MAXIMUM_CONFIDENCE_INTERVAL) + (v, context) -> v == DYNAMIC_CONFIDENCE_INTERVAL || (v >= MINIMUM_CONFIDENCE_INTERVAL && v <= MAXIMUM_CONFIDENCE_INTERVAL) ) ) .addParameter( LUCENE_SQ_BITS, - new Parameter.IntegerParameter(LUCENE_SQ_BITS, LUCENE_SQ_DEFAULT_BITS, LUCENE_SQ_BITS_SUPPORTED::contains) + new Parameter.IntegerParameter(LUCENE_SQ_BITS, LUCENE_SQ_DEFAULT_BITS, (v, context) -> LUCENE_SQ_BITS_SUPPORTED.contains(v)) ) .build(); diff --git a/src/main/java/org/opensearch/knn/index/engine/nmslib/NmslibHNSWMethod.java b/src/main/java/org/opensearch/knn/index/engine/nmslib/NmslibHNSWMethod.java index e8e27bcd6..779c16cd3 100644 --- a/src/main/java/org/opensearch/knn/index/engine/nmslib/NmslibHNSWMethod.java +++ b/src/main/java/org/opensearch/knn/index/engine/nmslib/NmslibHNSWMethod.java @@ -5,8 +5,10 @@ package org.opensearch.knn.index.engine.nmslib; +import com.google.common.collect.ImmutableSet; import org.opensearch.knn.index.KNNSettings; import org.opensearch.knn.index.SpaceType; +import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.engine.AbstractKNNMethod; import org.opensearch.knn.index.engine.DefaultHnswSearchContext; import org.opensearch.knn.index.engine.MethodComponent; @@ -25,6 +27,8 @@ */ public class NmslibHNSWMethod extends AbstractKNNMethod { + private static final Set SUPPORTED_DATA_TYPES = ImmutableSet.of(VectorDataType.FLOAT); + public final static List SUPPORTED_SPACES = Arrays.asList( SpaceType.UNDEFINED, SpaceType.L2, @@ -44,16 +48,17 @@ public NmslibHNSWMethod() { private static MethodComponent initMethodComponent() { return MethodComponent.Builder.builder(METHOD_HNSW) + .addSupportedDataTypes(SUPPORTED_DATA_TYPES) .addParameter( METHOD_PARAMETER_M, - new Parameter.IntegerParameter(METHOD_PARAMETER_M, KNNSettings.INDEX_KNN_DEFAULT_ALGO_PARAM_M, v -> v > 0) + new Parameter.IntegerParameter(METHOD_PARAMETER_M, KNNSettings.INDEX_KNN_DEFAULT_ALGO_PARAM_M, (v, context) -> v > 0) ) .addParameter( METHOD_PARAMETER_EF_CONSTRUCTION, new Parameter.IntegerParameter( METHOD_PARAMETER_EF_CONSTRUCTION, KNNSettings.INDEX_KNN_DEFAULT_ALGO_PARAM_EF_CONSTRUCTION, - v -> v > 0 + (v, context) -> v > 0 ) ) .build(); diff --git a/src/main/java/org/opensearch/knn/index/engine/validation/ParameterValidator.java b/src/main/java/org/opensearch/knn/index/engine/validation/ParameterValidator.java index 6a16b48a5..c79778503 100644 --- a/src/main/java/org/opensearch/knn/index/engine/validation/ParameterValidator.java +++ b/src/main/java/org/opensearch/knn/index/engine/validation/ParameterValidator.java @@ -7,6 +7,7 @@ import org.opensearch.common.Nullable; import org.opensearch.common.ValidationException; +import org.opensearch.knn.index.engine.KNNMethodConfigContext; import org.opensearch.knn.index.engine.Parameter; import java.util.ArrayList; @@ -17,14 +18,17 @@ public final class ParameterValidator { /** * A function which validates request parameters. + * * @param validParameters A set of valid parameters that can be requestParameters can be validated against * @param requestParameters parameters from the request - * @return + * @param knnMethodConfigContext context of the knn method + * @return ValidationException if there are any validation errors, null otherwise */ @Nullable public static ValidationException validateParameters( final Map> validParameters, - final Map requestParameters + final Map requestParameters, + KNNMethodConfigContext knnMethodConfigContext ) { if (validParameters == null) { @@ -38,7 +42,8 @@ public static ValidationException validateParameters( final List errorMessages = new ArrayList<>(); for (Map.Entry parameter : requestParameters.entrySet()) { if (validParameters.containsKey(parameter.getKey())) { - final ValidationException parameterValidation = validParameters.get(parameter.getKey()).validate(parameter.getValue()); + final ValidationException parameterValidation = validParameters.get(parameter.getKey()) + .validate(parameter.getValue(), knnMethodConfigContext); if (parameterValidation != null) { errorMessages.addAll(parameterValidation.validationErrors()); } diff --git a/src/main/java/org/opensearch/knn/index/mapper/FlatVectorFieldMapper.java b/src/main/java/org/opensearch/knn/index/mapper/FlatVectorFieldMapper.java index 146b5132f..d37ab9b86 100644 --- a/src/main/java/org/opensearch/knn/index/mapper/FlatVectorFieldMapper.java +++ b/src/main/java/org/opensearch/knn/index/mapper/FlatVectorFieldMapper.java @@ -10,6 +10,7 @@ import org.opensearch.Version; import org.opensearch.common.Explicit; import org.opensearch.knn.index.VectorDataType; +import org.opensearch.knn.index.engine.KNNMethodConfigContext; import java.util.Map; @@ -25,16 +26,19 @@ public static FlatVectorFieldMapper createFieldMapper( String fullname, String simpleName, Map metaValue, - VectorDataType vectorDataType, - Integer dimension, + KNNMethodConfigContext knnMethodConfigContext, MultiFields multiFields, CopyTo copyTo, Explicit ignoreMalformed, boolean stored, - boolean hasDocValues, - Version indexCreatedVersion + boolean hasDocValues ) { - final KNNVectorFieldType mappedFieldType = new KNNVectorFieldType(fullname, metaValue, vectorDataType, () -> dimension); + final KNNVectorFieldType mappedFieldType = new KNNVectorFieldType( + fullname, + metaValue, + knnMethodConfigContext.getVectorDataType(), + knnMethodConfigContext::getDimension + ); return new FlatVectorFieldMapper( simpleName, mappedFieldType, @@ -43,7 +47,7 @@ public static FlatVectorFieldMapper createFieldMapper( ignoreMalformed, stored, hasDocValues, - indexCreatedVersion + knnMethodConfigContext.getVersionCreated() ); } diff --git a/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapper.java b/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapper.java index 94756f595..65c3cfb66 100644 --- a/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapper.java +++ b/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapper.java @@ -15,6 +15,8 @@ import java.util.function.Supplier; import java.util.stream.Collectors; +import lombok.Getter; +import lombok.Setter; import lombok.extern.log4j.Log4j2; import org.apache.lucene.document.Field; import org.apache.lucene.document.FieldType; @@ -24,6 +26,7 @@ import org.opensearch.Version; import org.opensearch.common.Explicit; import org.opensearch.common.ValidationException; +import org.opensearch.common.settings.Settings; import org.opensearch.common.xcontent.support.XContentMapValues; import org.opensearch.core.common.Strings; import org.opensearch.core.xcontent.ToXContent; @@ -36,18 +39,16 @@ import org.opensearch.index.mapper.ParseContext; import org.opensearch.knn.common.KNNConstants; import org.opensearch.knn.index.KNNSettings; +import org.opensearch.knn.index.engine.KNNMethodConfigContext; import org.opensearch.knn.index.engine.KNNMethodContext; import org.opensearch.knn.index.SpaceType; -import org.opensearch.knn.index.engine.MethodComponentContext; import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.VectorField; import org.opensearch.knn.index.engine.KNNEngine; import org.opensearch.knn.indices.ModelDao; import static org.opensearch.knn.common.KNNConstants.DEFAULT_VECTOR_DATA_TYPE_FIELD; -import static org.opensearch.knn.common.KNNConstants.ENCODER_FLAT; import static org.opensearch.knn.common.KNNConstants.KNN_METHOD; -import static org.opensearch.knn.common.KNNConstants.METHOD_ENCODER_PARAMETER; import static org.opensearch.knn.common.KNNConstants.VECTOR_DATA_TYPE_FIELD; import static org.opensearch.knn.common.KNNValidationUtil.validateVectorDimension; import static org.opensearch.knn.index.mapper.KNNVectorFieldMapperUtil.createKNNMethodContextFromLegacy; @@ -55,7 +56,6 @@ import static org.opensearch.knn.index.mapper.KNNVectorFieldMapperUtil.createStoredFieldForFloatVector; import static org.opensearch.knn.index.mapper.KNNVectorFieldMapperUtil.validateIfCircuitBreakerIsNotTriggered; import static org.opensearch.knn.index.mapper.KNNVectorFieldMapperUtil.validateIfKNNPluginEnabled; -import static org.opensearch.knn.index.mapper.KNNVectorFieldMapperUtil.validateVectorDataType; import static org.opensearch.knn.index.mapper.ModelFieldMapper.UNSET_MODEL_DIMENSION_IDENTIFIER; /** @@ -154,19 +154,10 @@ public static class Builder extends ParametrizedFieldMapper.Builder { }), m -> m.getMethodComponentContext().getName()).setValidator(v -> { if (v == null) return; - ValidationException validationException = null; + ValidationException validationException; if (v.isTrainingRequired()) { validationException = new ValidationException(); validationException.addValidationError(String.format(Locale.ROOT, "\"%s\" requires training.", KNN_METHOD)); - } - - ValidationException methodValidation = v.validate(); - if (methodValidation != null) { - validationException = validationException == null ? new ValidationException() : validationException; - validationException.addValidationErrors(methodValidation.validationErrors()); - } - - if (validationException != null) { throw validationException; } }); @@ -190,13 +181,24 @@ public static class Builder extends ParametrizedFieldMapper.Builder { // (https://github.com/opensearch-project/OpenSearch/blob/2.16.0/server/src/main/java/org/opensearch/index/mapper/ParametrizedFieldMapper.java#L322-L324). // So, what we do is pass in a "resolvedKNNMethodContext" that will either be null or be set via the merge builder // constructor. A similar approach was taken for https://github.com/opendistro-for-elasticsearch/k-NN/issues/288 + @Setter + @Getter private KNNMethodContext resolvedKNNMethodContext; - - public Builder(String name, ModelDao modelDao, Version indexCreatedVersion, KNNMethodContext resolvedKNNMethodContext) { + @Setter + private KNNMethodConfigContext knnMethodConfigContext; + + public Builder( + String name, + ModelDao modelDao, + Version indexCreatedVersion, + KNNMethodContext resolvedKNNMethodContext, + KNNMethodConfigContext knnMethodConfigContext + ) { super(name); this.modelDao = modelDao; this.indexCreatedVersion = indexCreatedVersion; this.resolvedKNNMethodContext = resolvedKNNMethodContext; + this.knnMethodConfigContext = knnMethodConfigContext; } @Override @@ -214,12 +216,6 @@ protected Explicit ignoreMalformed(BuilderContext context) { return KNNVectorFieldMapper.Defaults.IGNORE_MALFORMED; } - private void validateFlatMapper() { - if (modelId.get() != null || knnMethodContext.get() != null) { - throw new IllegalArgumentException("Cannot set modelId or method parameters when index.knn setting is false"); - } - } - @Override public KNNVectorFieldMapper build(BuilderContext context) { validateFullFieldName(context); @@ -229,15 +225,13 @@ public KNNVectorFieldMapper build(BuilderContext context) { final Explicit ignoreMalformed = ignoreMalformed(context); final Map metaValue = meta.getValue(); - // Index is being created from model - String modelIdAsString = this.modelId.get(); - if (modelIdAsString != null) { + if (modelId.get() != null) { return ModelFieldMapper.createFieldMapper( buildFullName(context), name, metaValue, vectorDataType.getValue(), - modelIdAsString, + modelId.get(), multiFieldsBuilder, copyToBuilder, ignoreMalformed, @@ -248,48 +242,24 @@ public KNNVectorFieldMapper build(BuilderContext context) { ); } - // If the field mapper is using the legacy context and being constructed from another field mapper, - // the settings will be empty. See https://github.com/opendistro-for-elasticsearch/k-NN/issues/288. In this - // case, the input resolvedKNNMethodContext will be null and the settings wont exist (so flat mapper should - // be used). Otherwise, we need to check the setting. - boolean isResolvedNull = resolvedKNNMethodContext == null; - boolean isSettingPresent = KNNSettings.IS_KNN_INDEX_SETTING.exists(context.indexSettings()); - boolean isKnnSettingNotPresentOrFalse = !isSettingPresent || !KNNSettings.IS_KNN_INDEX_SETTING.get(context.indexSettings()); - if (isResolvedNull && isKnnSettingNotPresentOrFalse) { - validateFlatMapper(); + if (resolvedKNNMethodContext == null) { return FlatVectorFieldMapper.createFieldMapper( buildFullName(context), name, metaValue, - vectorDataType.getValue(), - dimension.getValue(), + KNNMethodConfigContext.builder() + .vectorDataType(vectorDataType.getValue()) + .versionCreated(indexCreatedVersion) + .dimension(dimension.getValue()) + .build(), multiFieldsBuilder, copyToBuilder, ignoreMalformed, stored.get(), - hasDocValues.get(), - indexCreatedVersion + hasDocValues.get() ); } - // See resolvedKNNMethodContext definition for explanation - if (isResolvedNull) { - resolvedKNNMethodContext = this.knnMethodContext.getValue(); - setDefaultSpaceType(resolvedKNNMethodContext, vectorDataType.getValue()); - validateSpaceType(resolvedKNNMethodContext, vectorDataType.getValue()); - validateDimensions(resolvedKNNMethodContext, vectorDataType.getValue()); - validateEncoder(resolvedKNNMethodContext, vectorDataType.getValue()); - } - - // If the knnMethodContext is null at this point, that means user built the index with the legacy k-NN - // settings to specify algo params. We need to convert this here to a KNNMethodContext so that we can - // properly configure the rest of the index - if (resolvedKNNMethodContext == null) { - resolvedKNNMethodContext = createKNNMethodContextFromLegacy(context, vectorDataType.getValue(), indexCreatedVersion); - } - - validateVectorDataType(resolvedKNNMethodContext, vectorDataType.getValue()); - resolvedKNNMethodContext.getMethodComponentContext().setIndexVersion(indexCreatedVersion); if (resolvedKNNMethodContext.getKnnEngine() == KNNEngine.LUCENE) { log.debug(String.format(Locale.ROOT, "Use [LuceneFieldMapper] mapper for field [%s]", name)); LuceneFieldMapper.CreateLuceneFieldMapperInput createLuceneFieldMapperInput = LuceneFieldMapper.CreateLuceneFieldMapperInput @@ -300,16 +270,13 @@ public KNNVectorFieldMapper build(BuilderContext context) { .ignoreMalformed(ignoreMalformed) .stored(stored.getValue()) .hasDocValues(hasDocValues.getValue()) - .vectorDataType(vectorDataType.getValue()) - .indexVersion(indexCreatedVersion) .originalKnnMethodContext(knnMethodContext.get()) .build(); return LuceneFieldMapper.createFieldMapper( buildFullName(context), metaValue, - vectorDataType.getValue(), - dimension.getValue(), resolvedKNNMethodContext, + knnMethodConfigContext, createLuceneFieldMapperInput ); } @@ -318,107 +285,17 @@ public KNNVectorFieldMapper build(BuilderContext context) { buildFullName(context), name, metaValue, - vectorDataType.getValue(), - dimension.getValue(), resolvedKNNMethodContext, + knnMethodConfigContext, knnMethodContext.get(), multiFieldsBuilder, copyToBuilder, ignoreMalformed, stored.getValue(), - hasDocValues.getValue(), - indexCreatedVersion + hasDocValues.getValue() ); } - private void validateEncoder(final KNNMethodContext knnMethodContext, final VectorDataType vectorDataType) { - if (knnMethodContext == null) { - return; - } - - if (VectorDataType.FLOAT == vectorDataType) { - return; - } - - if (knnMethodContext.getMethodComponentContext() == null) { - return; - } - - if (knnMethodContext.getMethodComponentContext().getParameters() == null) { - return; - } - - if (knnMethodContext.getMethodComponentContext().getParameters().get(METHOD_ENCODER_PARAMETER) == null) { - return; - } - - if (knnMethodContext.getMethodComponentContext() - .getParameters() - .get(METHOD_ENCODER_PARAMETER) instanceof MethodComponentContext == false) { - return; - } - - MethodComponentContext encoderMethodComponentContext = (MethodComponentContext) knnMethodContext.getMethodComponentContext() - .getParameters() - .get(METHOD_ENCODER_PARAMETER); - - if (ENCODER_FLAT.equals(encoderMethodComponentContext.getName()) == false) { - throw new IllegalArgumentException( - String.format( - Locale.ROOT, - "%s data type does not support %s encoder", - vectorDataType.getValue(), - encoderMethodComponentContext.getName() - ) - ); - } - } - - private void setDefaultSpaceType(final KNNMethodContext knnMethodContext, final VectorDataType vectorDataType) { - if (knnMethodContext == null) { - return; - } - - if (SpaceType.UNDEFINED == knnMethodContext.getSpaceType()) { - if (VectorDataType.BINARY == vectorDataType) { - knnMethodContext.setSpaceType(SpaceType.DEFAULT_BINARY); - } else { - knnMethodContext.setSpaceType(SpaceType.DEFAULT); - } - } - } - - private void validateSpaceType(final KNNMethodContext knnMethodContext, final VectorDataType vectorDataType) { - if (knnMethodContext == null) { - return; - } - - knnMethodContext.getSpaceType().validateVectorDataType(vectorDataType); - } - - private KNNEngine validateDimensions(final KNNMethodContext knnMethodContext, final VectorDataType dataType) { - final KNNEngine knnEngine; - if (knnMethodContext != null) { - knnEngine = knnMethodContext.getKnnEngine(); - } else { - knnEngine = KNNEngine.DEFAULT; - } - if (dimension.getValue() > KNNEngine.getMaxDimensionByEngine(knnEngine)) { - throw new IllegalArgumentException( - String.format( - Locale.ROOT, - "Dimension value cannot be greater than %s for vector: %s", - KNNEngine.getMaxDimensionByEngine(knnEngine), - name - ) - ); - } - if (VectorDataType.BINARY == dataType && dimension.getValue() % 8 != 0) { - throw new IllegalArgumentException("Dimension should be multiply of 8 for binary vector data type"); - } - return knnEngine; - } - /** * Validate whether provided full field name contain any invalid characters for physical file name. * At the moment, we use a field name as a part of file name while we throw an exception @@ -458,7 +335,13 @@ public TypeParser(Supplier modelDaoSupplier) { @Override public Mapper.Builder parse(String name, Map node, ParserContext parserContext) throws MapperParsingException { - Builder builder = new KNNVectorFieldMapper.Builder(name, modelDaoSupplier.get(), parserContext.indexVersionCreated(), null); + Builder builder = new KNNVectorFieldMapper.Builder( + name, + modelDaoSupplier.get(), + parserContext.indexVersionCreated(), + null, + null + ); builder.parse(name, parserContext, node); // All parse(String name, Map node, ParserCont ); } + // Check for flat configuration + if (isKNNDisabled(parserContext.getSettings())) { + validateFromFlat(builder); + } else if (builder.modelId.get() != null) { + validateFromModel(builder); + } else { + resolveKNNMethodComponents(builder, parserContext); + validateFromKNNMethod(builder); + } + + return builder; + } + + private void validateFromFlat(KNNVectorFieldMapper.Builder builder) { + if (builder.modelId.get() != null || builder.knnMethodContext.get() != null) { + throw new IllegalArgumentException("Cannot set modelId or method parameters when index.knn setting is false"); + } + validateDimensionSet(builder); + } + + private void validateFromModel(KNNVectorFieldMapper.Builder builder) { // Dimension should not be null unless modelId is used if (builder.dimension.getValue() == UNSET_MODEL_DIMENSION_IDENTIFIER && builder.modelId.get() == null) { - throw new IllegalArgumentException(String.format(Locale.ROOT, "Dimension value missing for vector: %s", name)); + throw new IllegalArgumentException(String.format(Locale.ROOT, "Dimension value missing for vector: %s", builder.name())); } + } - return builder; + private void validateFromKNNMethod(KNNVectorFieldMapper.Builder builder) { + if (builder.resolvedKNNMethodContext != null) { + ValidationException validationException = builder.resolvedKNNMethodContext.validate(builder.knnMethodConfigContext); + if (validationException != null) { + throw validationException; + } + } + validateDimensionSet(builder); + } + + private void validateDimensionSet(KNNVectorFieldMapper.Builder builder) { + if (builder.dimension.getValue() == UNSET_MODEL_DIMENSION_IDENTIFIER) { + throw new IllegalArgumentException(String.format(Locale.ROOT, "Dimension value missing for vector: %s", builder.name())); + } + } + + private void resolveKNNMethodComponents(KNNVectorFieldMapper.Builder builder, ParserContext parserContext) { + builder.setKnnMethodConfigContext( + KNNMethodConfigContext.builder() + .vectorDataType(builder.vectorDataType.getValue()) + .versionCreated(parserContext.indexVersionCreated()) + .dimension(builder.dimension.getValue()) + .build() + ); + + // Configure method from map or legacy + builder.setResolvedKNNMethodContext( + builder.knnMethodContext.getValue() != null + ? builder.knnMethodContext.getValue() + : createKNNMethodContextFromLegacy(parserContext.getSettings(), parserContext.indexVersionCreated()) + ); + // TODO: We should remove this and set it based on the KNNMethodContext + setDefaultSpaceType(builder.resolvedKNNMethodContext, builder.vectorDataType.getValue()); + } + + private boolean isKNNDisabled(Settings settings) { + boolean isSettingPresent = KNNSettings.IS_KNN_INDEX_SETTING.exists(settings); + return !isSettingPresent || !KNNSettings.IS_KNN_INDEX_SETTING.get(settings); + } + + private void setDefaultSpaceType(final KNNMethodContext knnMethodContext, final VectorDataType vectorDataType) { + if (knnMethodContext == null) { + return; + } + + if (SpaceType.UNDEFINED == knnMethodContext.getSpaceType()) { + if (VectorDataType.BINARY == vectorDataType) { + knnMethodContext.setSpaceType(SpaceType.DEFAULT_BINARY); + } else { + knnMethodContext.setSpaceType(SpaceType.DEFAULT); + } + } } } @@ -707,11 +663,25 @@ Optional getFloatsFromContext(ParseContext context, int dimension) thro @Override public ParametrizedFieldMapper.Builder getMergeBuilder() { + // We cannot get the dimension from the model based indices at this field because the + // cluster state may not be available. So, we need to set it to null. + KNNMethodConfigContext knnMethodConfigContext; + if (fieldType().getKnnMappingConfig().getModelId().isPresent()) { + knnMethodConfigContext = null; + } else { + knnMethodConfigContext = KNNMethodConfigContext.builder() + .vectorDataType(vectorDataType) + .versionCreated(indexCreatedVersion) + .dimension(fieldType().getKnnMappingConfig().getDimension()) + .build(); + } + return new KNNVectorFieldMapper.Builder( simpleName(), modelDao, indexCreatedVersion, - fieldType().getKnnMappingConfig().getKnnMethodContext().orElse(null) + fieldType().getKnnMappingConfig().getKnnMethodContext().orElse(null), + knnMethodConfigContext ).init(this); } diff --git a/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperUtil.java b/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperUtil.java index 9cd6bb467..57a4dd062 100644 --- a/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperUtil.java +++ b/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperUtil.java @@ -20,7 +20,6 @@ import org.apache.lucene.util.BytesRef; import org.opensearch.Version; import org.opensearch.common.settings.Settings; -import org.opensearch.index.mapper.Mapper; import org.opensearch.knn.index.KNNSettings; import org.opensearch.knn.index.KnnCircuitBreakerException; import org.opensearch.knn.index.SpaceType; @@ -32,29 +31,15 @@ import org.opensearch.knn.index.util.IndexHyperParametersUtil; import java.util.Arrays; -import java.util.Locale; import java.util.Map; -import java.util.Objects; -import static org.opensearch.knn.common.KNNConstants.ENCODER_SQ; -import static org.opensearch.knn.common.KNNConstants.FAISS_NAME; -import static org.opensearch.knn.common.KNNConstants.FAISS_SQ_CLIP; -import static org.opensearch.knn.common.KNNConstants.FAISS_SQ_ENCODER_FP16; -import static org.opensearch.knn.common.KNNConstants.FAISS_SQ_TYPE; -import static org.opensearch.knn.common.KNNConstants.FP16_MAX_VALUE; -import static org.opensearch.knn.common.KNNConstants.FP16_MIN_VALUE; import static org.opensearch.knn.common.KNNConstants.HNSW_ALGO_EF_CONSTRUCTION; import static org.opensearch.knn.common.KNNConstants.HNSW_ALGO_M; import static org.opensearch.knn.common.KNNConstants.KNN_ENGINE; -import static org.opensearch.knn.common.KNNConstants.LUCENE_NAME; -import static org.opensearch.knn.common.KNNConstants.METHOD_ENCODER_PARAMETER; import static org.opensearch.knn.common.KNNConstants.METHOD_HNSW; import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_EF_CONSTRUCTION; import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_M; import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_SPACE_TYPE; -import static org.opensearch.knn.common.KNNConstants.NMSLIB_NAME; -import static org.opensearch.knn.common.KNNConstants.VECTOR_DATA_TYPE_FIELD; -import static org.opensearch.knn.common.KNNValidationUtil.validateFloatVectorValue; /** * Utility class for KNNVectorFieldMapper @@ -63,99 +48,6 @@ @NoArgsConstructor(access = AccessLevel.PRIVATE) public class KNNVectorFieldMapperUtil { - /** - * Validate the float vector value and throw exception if it is not a number or not in the finite range - * or is not within the FP16 range of [-65504 to 65504]. - * - * @param value float vector value - */ - public static void validateFP16VectorValue(float value) { - validateFloatVectorValue(value); - if (value < FP16_MIN_VALUE || value > FP16_MAX_VALUE) { - throw new IllegalArgumentException( - String.format( - Locale.ROOT, - "encoder name is set as [%s] and type is set as [%s] in index mapping. But, KNN vector values are not within in the FP16 range [%f, %f]", - ENCODER_SQ, - FAISS_SQ_ENCODER_FP16, - FP16_MIN_VALUE, - FP16_MAX_VALUE - ) - ); - } - } - - /** - * Validate the float vector value and if it is outside FP16 range, - * then it will be clipped to FP16 range of [-65504 to 65504]. - * - * @param value float vector value - * @return vector value clipped to FP16 range - */ - public static float clipVectorValueToFP16Range(float value) { - validateFloatVectorValue(value); - if (value < FP16_MIN_VALUE) return FP16_MIN_VALUE; - if (value > FP16_MAX_VALUE) return FP16_MAX_VALUE; - return value; - } - - /** - * Validates if the vector data type is supported with given method context - * - * @param methodContext methodContext - * @param vectorDataType vector data type - */ - public static void validateVectorDataType(KNNMethodContext methodContext, VectorDataType vectorDataType) { - if (VectorDataType.FLOAT == vectorDataType) { - return; - } - - if (VectorDataType.BYTE == vectorDataType) { - if (KNNEngine.LUCENE == methodContext.getKnnEngine()) { - return; - } else { - throw new IllegalArgumentException( - String.format( - Locale.ROOT, - "[%s] field with value [%s] is only supported for [%s] engine", - VECTOR_DATA_TYPE_FIELD, - vectorDataType.getValue(), - LUCENE_NAME - ) - ); - } - } - - if (VectorDataType.BINARY == vectorDataType) { - if (KNNEngine.FAISS == methodContext.getKnnEngine()) { - if (METHOD_HNSW.equals(methodContext.getMethodComponentContext().getName())) { - return; - } else { - throw new IllegalArgumentException( - String.format( - Locale.ROOT, - "[%s] field with value [%s] is only supported for [%s] method", - VECTOR_DATA_TYPE_FIELD, - vectorDataType.getValue(), - METHOD_HNSW - ) - ); - } - } else { - throw new IllegalArgumentException( - String.format( - Locale.ROOT, - "[%s] field with value [%s] is only supported for [%s] engine", - VECTOR_DATA_TYPE_FIELD, - vectorDataType.getValue(), - FAISS_NAME - ) - ); - } - } - throw new IllegalArgumentException("This line should not be reached"); - } - /** * @param knnEngine KNNEngine * @return DocValues FieldType of type Binary @@ -254,12 +146,10 @@ static boolean useLuceneKNNVectorsFormat(final Version indexCreatedVersion) { return indexCreatedVersion.onOrAfter(Version.V_2_17_0) && KNNSettings.getIsLuceneVectorFormatEnabled(); } - private static SpaceType getSpaceType(final Settings indexSettings, final VectorDataType vectorDataType) { + private static SpaceType getSpaceType(final Settings indexSettings) { String spaceType = indexSettings.get(KNNSettings.INDEX_KNN_SPACE_TYPE.getKey()); if (spaceType == null) { - spaceType = VectorDataType.BINARY == vectorDataType - ? KNNSettings.INDEX_KNN_DEFAULT_SPACE_TYPE_FOR_BINARY - : KNNSettings.INDEX_KNN_DEFAULT_SPACE_TYPE; + spaceType = KNNSettings.INDEX_KNN_DEFAULT_SPACE_TYPE; log.info( String.format( "[KNN] The setting \"%s\" was not set for the index. Likely caused by recent version upgrade. Setting the setting to the default value=%s", @@ -303,97 +193,17 @@ private static int getEfConstruction(Settings indexSettings, Version indexVersio return Integer.parseInt(efConstruction); } - /** - * Verify mapping and return true if it is a "faiss" Index using "sq" encoder of type "fp16" - * - * @param methodComponentContext MethodComponentContext - * @return true if it is a "faiss" Index using "sq" encoder of type "fp16" - */ - static boolean isFaissSQfp16(MethodComponentContext methodComponentContext) { - if (Objects.isNull(methodComponentContext)) { - return false; - } - - if (methodComponentContext.getParameters().size() == 0) { - return false; - } - - Map methodComponentParams = methodComponentContext.getParameters(); - - // The method component parameters should have an encoder - if (!methodComponentParams.containsKey(METHOD_ENCODER_PARAMETER)) { - return false; - } - - // Validate if the object is of type MethodComponentContext before casting it later - if (!(methodComponentParams.get(METHOD_ENCODER_PARAMETER) instanceof MethodComponentContext)) { - return false; - } - - MethodComponentContext encoderMethodComponentContext = (MethodComponentContext) methodComponentParams.get(METHOD_ENCODER_PARAMETER); - - // returns true if encoder name is "sq" and type is "fp16" - return ENCODER_SQ.equals(encoderMethodComponentContext.getName()) - && FAISS_SQ_ENCODER_FP16.equals( - encoderMethodComponentContext.getParameters().getOrDefault(FAISS_SQ_TYPE, FAISS_SQ_ENCODER_FP16) - ); - - } - - /** - * Verify mapping and return the value of "clip" parameter(default false) for a "faiss" Index - * using "sq" encoder of type "fp16". - * - * @param methodComponentContext MethodComponentContext - * @return boolean value of "clip" parameter - */ - static boolean isFaissSQClipToFP16RangeEnabled(MethodComponentContext methodComponentContext) { - if (Objects.nonNull(methodComponentContext)) { - return (boolean) methodComponentContext.getParameters().getOrDefault(FAISS_SQ_CLIP, false); - } - return false; - } - - /** - * Extract MethodComponentContext from KNNMethodContext - * - * @param knnMethodContext KNNMethodContext - * @return MethodComponentContext - */ - static MethodComponentContext getMethodComponentContext(KNNMethodContext knnMethodContext) { - if (Objects.isNull(knnMethodContext)) { - return null; - } - return knnMethodContext.getMethodComponentContext(); - } - - static KNNMethodContext createKNNMethodContextFromLegacy( - Mapper.BuilderContext context, - VectorDataType vectorDataType, - Version indexCreatedVersion - ) { - if (VectorDataType.FLOAT != vectorDataType) { - throw new IllegalArgumentException( - String.format( - Locale.ROOT, - "[%s] field with value [%s] is not supported for [%s] engine", - VECTOR_DATA_TYPE_FIELD, - vectorDataType.getValue(), - NMSLIB_NAME - ) - ); - } - + static KNNMethodContext createKNNMethodContextFromLegacy(Settings indexSettings, Version indexCreatedVersion) { return new KNNMethodContext( KNNEngine.NMSLIB, - KNNVectorFieldMapperUtil.getSpaceType(context.indexSettings(), vectorDataType), + KNNVectorFieldMapperUtil.getSpaceType(indexSettings), new MethodComponentContext( METHOD_HNSW, Map.of( METHOD_PARAMETER_M, - KNNVectorFieldMapperUtil.getM(context.indexSettings()), + KNNVectorFieldMapperUtil.getM(indexSettings), METHOD_PARAMETER_EF_CONSTRUCTION, - KNNVectorFieldMapperUtil.getEfConstruction(context.indexSettings(), indexCreatedVersion) + KNNVectorFieldMapperUtil.getEfConstruction(indexSettings, indexCreatedVersion) ) ) ); diff --git a/src/main/java/org/opensearch/knn/index/mapper/LuceneFieldMapper.java b/src/main/java/org/opensearch/knn/index/mapper/LuceneFieldMapper.java index 7c3d942b6..744ba4bd5 100644 --- a/src/main/java/org/opensearch/knn/index/mapper/LuceneFieldMapper.java +++ b/src/main/java/org/opensearch/knn/index/mapper/LuceneFieldMapper.java @@ -7,7 +7,6 @@ import java.util.ArrayList; import java.util.List; -import java.util.Locale; import java.util.Map; import java.util.Optional; @@ -19,11 +18,12 @@ import org.apache.lucene.document.KnnByteVectorField; import org.apache.lucene.document.KnnFloatVectorField; import org.apache.lucene.index.VectorSimilarityFunction; -import org.opensearch.Version; import org.opensearch.common.Explicit; import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.VectorField; import org.opensearch.knn.index.engine.KNNEngine; +import org.opensearch.knn.index.engine.KNNLibraryIndexingContext; +import org.opensearch.knn.index.engine.KNNMethodConfigContext; import org.opensearch.knn.index.engine.KNNMethodContext; import static org.opensearch.knn.index.mapper.KNNVectorFieldMapperUtil.createStoredFieldForByteVector; @@ -37,36 +37,43 @@ public class LuceneFieldMapper extends KNNVectorFieldMapper { /** FieldType used for initializing VectorField, which is used for creating binary doc values. **/ private final FieldType vectorFieldType; - private final VectorDataType vectorDataType; - private PerDimensionProcessor perDimensionProcessor; - private PerDimensionValidator perDimensionValidator; - private VectorValidator vectorValidator; + private final PerDimensionProcessor perDimensionProcessor; + private final PerDimensionValidator perDimensionValidator; + private final VectorValidator vectorValidator; static LuceneFieldMapper createFieldMapper( String fullname, Map metaValue, - VectorDataType vectorDataType, - Integer dimension, KNNMethodContext knnMethodContext, + KNNMethodConfigContext knnMethodConfigContext, CreateLuceneFieldMapperInput createLuceneFieldMapperInput ) { - final KNNVectorFieldType mappedFieldType = new KNNVectorFieldType(fullname, metaValue, vectorDataType, new KNNMappingConfig() { - @Override - public Optional getKnnMethodContext() { - return Optional.of(knnMethodContext); + final KNNVectorFieldType mappedFieldType = new KNNVectorFieldType( + fullname, + metaValue, + knnMethodConfigContext.getVectorDataType(), + new KNNMappingConfig() { + @Override + public Optional getKnnMethodContext() { + return Optional.of(knnMethodContext); + } + + @Override + public int getDimension() { + return knnMethodConfigContext.getDimension(); + } } + ); - @Override - public int getDimension() { - return dimension; - } - }); - - return new LuceneFieldMapper(mappedFieldType, createLuceneFieldMapperInput); + return new LuceneFieldMapper(mappedFieldType, createLuceneFieldMapperInput, knnMethodConfigContext); } - private LuceneFieldMapper(final KNNVectorFieldType mappedFieldType, final CreateLuceneFieldMapperInput input) { + private LuceneFieldMapper( + final KNNVectorFieldType mappedFieldType, + final CreateLuceneFieldMapperInput input, + KNNMethodConfigContext knnMethodConfigContext + ) { super( input.getName(), mappedFieldType, @@ -75,30 +82,18 @@ private LuceneFieldMapper(final KNNVectorFieldType mappedFieldType, final Create input.getIgnoreMalformed(), input.isStored(), input.isHasDocValues(), - input.getIndexVersion(), + knnMethodConfigContext.getVersionCreated(), mappedFieldType.knnMappingConfig.getKnnMethodContext().orElse(null) ); KNNMappingConfig knnMappingConfig = mappedFieldType.getKnnMappingConfig(); KNNMethodContext knnMethodContext = knnMappingConfig.getKnnMethodContext() .orElseThrow(() -> new IllegalArgumentException("KNN method context is missing")); - vectorDataType = input.getVectorDataType(); + VectorDataType vectorDataType = mappedFieldType.getVectorDataType(); final VectorSimilarityFunction vectorSimilarityFunction = knnMethodContext.getSpaceType() .getKnnVectorSimilarityFunction() .getVectorSimilarityFunction(); - if (knnMappingConfig.getDimension() > KNNEngine.getMaxDimensionByEngine(KNNEngine.LUCENE)) { - throw new IllegalArgumentException( - String.format( - Locale.ROOT, - "Dimension value cannot be greater than [%s] but got [%s] for vector [%s]", - KNNEngine.getMaxDimensionByEngine(KNNEngine.LUCENE), - knnMappingConfig.getDimension(), - input.getName() - ) - ); - } - this.fieldType = vectorDataType.createKnnVectorFieldType(knnMappingConfig.getDimension(), vectorSimilarityFunction); if (this.hasDocValues) { @@ -107,8 +102,11 @@ private LuceneFieldMapper(final KNNVectorFieldType mappedFieldType, final Create this.vectorFieldType = null; } - initValidatorsAndProcessors(knnMethodContext); - knnMethodContext.getSpaceType().validateVectorDataType(vectorDataType); + KNNLibraryIndexingContext knnLibraryIndexingContext = knnMethodContext.getKnnEngine() + .getKNNLibraryIndexingContext(knnMethodContext, knnMethodConfigContext); + this.perDimensionProcessor = knnLibraryIndexingContext.getPerDimensionProcessor(); + this.perDimensionValidator = knnLibraryIndexingContext.getPerDimensionValidator(); + this.vectorValidator = knnLibraryIndexingContext.getVectorValidator(); } @Override @@ -141,21 +139,6 @@ protected List getFieldsForByteVector(final byte[] array) { return fieldsToBeAdded; } - private void initValidatorsAndProcessors(KNNMethodContext knnMethodContext) { - this.vectorValidator = new SpaceVectorValidator(knnMethodContext.getSpaceType()); - this.perDimensionProcessor = PerDimensionProcessor.NOOP_PROCESSOR; - if (VectorDataType.BINARY == vectorDataType) { - this.perDimensionValidator = PerDimensionValidator.DEFAULT_BIT_VALIDATOR; - return; - } - - if (VectorDataType.BYTE == vectorDataType) { - this.perDimensionValidator = PerDimensionValidator.DEFAULT_BYTE_VALIDATOR; - return; - } - this.perDimensionValidator = PerDimensionValidator.DEFAULT_FLOAT_VALIDATOR; - } - @Override protected VectorValidator getVectorValidator() { return vectorValidator; @@ -190,8 +173,6 @@ static class CreateLuceneFieldMapperInput { Explicit ignoreMalformed; boolean stored; boolean hasDocValues; - VectorDataType vectorDataType; - Version indexVersion; KNNMethodContext originalKnnMethodContext; } } diff --git a/src/main/java/org/opensearch/knn/index/mapper/MethodFieldMapper.java b/src/main/java/org/opensearch/knn/index/mapper/MethodFieldMapper.java index cc2c43386..c602b53fb 100644 --- a/src/main/java/org/opensearch/knn/index/mapper/MethodFieldMapper.java +++ b/src/main/java/org/opensearch/knn/index/mapper/MethodFieldMapper.java @@ -8,14 +8,14 @@ import org.apache.lucene.document.FieldType; import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.VectorEncoding; -import org.opensearch.Version; import org.opensearch.common.Explicit; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.engine.KNNEngine; +import org.opensearch.knn.index.engine.KNNLibraryIndexingContext; +import org.opensearch.knn.index.engine.KNNMethodConfigContext; import org.opensearch.knn.index.engine.KNNMethodContext; -import org.opensearch.knn.index.engine.MethodComponentContext; import java.io.IOException; import java.util.Map; @@ -23,49 +23,48 @@ import static org.opensearch.knn.common.KNNConstants.DIMENSION; import static org.opensearch.knn.common.KNNConstants.KNN_ENGINE; -import static org.opensearch.knn.common.KNNConstants.METHOD_ENCODER_PARAMETER; import static org.opensearch.knn.common.KNNConstants.PARAMETERS; import static org.opensearch.knn.common.KNNConstants.SPACE_TYPE; import static org.opensearch.knn.common.KNNConstants.VECTOR_DATA_TYPE_FIELD; -import static org.opensearch.knn.index.mapper.KNNVectorFieldMapperUtil.getMethodComponentContext; -import static org.opensearch.knn.index.mapper.KNNVectorFieldMapperUtil.isFaissSQClipToFP16RangeEnabled; -import static org.opensearch.knn.index.mapper.KNNVectorFieldMapperUtil.isFaissSQfp16; /** * Field mapper for method definition in mapping */ public class MethodFieldMapper extends KNNVectorFieldMapper { - private PerDimensionProcessor perDimensionProcessor; - private PerDimensionValidator perDimensionValidator; - private VectorValidator vectorValidator; + private final PerDimensionProcessor perDimensionProcessor; + private final PerDimensionValidator perDimensionValidator; + private final VectorValidator vectorValidator; public static MethodFieldMapper createFieldMapper( String fullname, String simpleName, Map metaValue, - VectorDataType vectorDataType, - Integer dimension, KNNMethodContext knnMethodContext, + KNNMethodConfigContext knnMethodConfigContext, KNNMethodContext originalKNNMethodContext, MultiFields multiFields, CopyTo copyTo, Explicit ignoreMalformed, boolean stored, - boolean hasDocValues, - Version indexCreatedVersion + boolean hasDocValues ) { - final KNNVectorFieldType mappedFieldType = new KNNVectorFieldType(fullname, metaValue, vectorDataType, new KNNMappingConfig() { - @Override - public Optional getKnnMethodContext() { - return Optional.of(knnMethodContext); - } - - @Override - public int getDimension() { - return dimension; + final KNNVectorFieldType mappedFieldType = new KNNVectorFieldType( + fullname, + metaValue, + knnMethodConfigContext.getVectorDataType(), + new KNNMappingConfig() { + @Override + public Optional getKnnMethodContext() { + return Optional.of(knnMethodContext); + } + + @Override + public int getDimension() { + return knnMethodConfigContext.getDimension(); + } } - }); + ); return new MethodFieldMapper( simpleName, mappedFieldType, @@ -74,8 +73,8 @@ public int getDimension() { ignoreMalformed, stored, hasDocValues, - indexCreatedVersion, - originalKNNMethodContext + originalKNNMethodContext, + knnMethodConfigContext ); } @@ -87,8 +86,8 @@ private MethodFieldMapper( Explicit ignoreMalformed, boolean stored, boolean hasDocValues, - Version indexVerision, - KNNMethodContext originalKNNMethodContext + KNNMethodContext originalKNNMethodContext, + KNNMethodConfigContext knnMethodConfigContext ) { super( @@ -99,7 +98,7 @@ private MethodFieldMapper( ignoreMalformed, stored, hasDocValues, - indexVerision, + knnMethodConfigContext.getVersionCreated(), originalKNNMethodContext ); this.useLuceneBasedVectorField = KNNVectorFieldMapperUtil.useLuceneKNNVectorsFormat(indexCreatedVersion); @@ -115,9 +114,15 @@ private MethodFieldMapper( KNNEngine knnEngine = knnMethodContext.getKnnEngine(); this.fieldType.putAttribute(KNN_ENGINE, knnEngine.getName()); + KNNLibraryIndexingContext knnLibraryIndexingContext = knnEngine.getKNNLibraryIndexingContext( + knnMethodContext, + knnMethodConfigContext + ); try { - Map libParams = knnEngine.getKNNLibraryIndexingContext(knnMethodContext).getLibraryParameters(); - this.fieldType.putAttribute(PARAMETERS, XContentFactory.jsonBuilder().map(libParams).toString()); + this.fieldType.putAttribute( + PARAMETERS, + XContentFactory.jsonBuilder().map(knnLibraryIndexingContext.getLibraryParameters()).toString() + ); } catch (IOException ioe) { throw new RuntimeException(String.format("Unable to create KNNVectorFieldMapper: %s", ioe)); } @@ -139,43 +144,9 @@ private MethodFieldMapper( } this.fieldType.freeze(); - initValidatorsAndProcessors(knnMethodContext); - knnMethodContext.getSpaceType().validateVectorDataType(vectorDataType); - } - - private void initValidatorsAndProcessors(KNNMethodContext knnMethodContext) { - this.vectorValidator = new SpaceVectorValidator(knnMethodContext.getSpaceType()); - - if (VectorDataType.BINARY == vectorDataType) { - this.perDimensionValidator = PerDimensionValidator.DEFAULT_BIT_VALIDATOR; - this.perDimensionProcessor = PerDimensionProcessor.NOOP_PROCESSOR; - return; - } - - if (VectorDataType.BYTE == vectorDataType) { - this.perDimensionValidator = PerDimensionValidator.DEFAULT_BYTE_VALIDATOR; - this.perDimensionProcessor = PerDimensionProcessor.NOOP_PROCESSOR; - return; - } - - MethodComponentContext methodComponentContext = getMethodComponentContext(knnMethodContext); - if (!isFaissSQfp16(methodComponentContext)) { - // Normal float and byte processor - this.perDimensionValidator = PerDimensionValidator.DEFAULT_FLOAT_VALIDATOR; - this.perDimensionProcessor = PerDimensionProcessor.NOOP_PROCESSOR; - return; - } - - this.perDimensionValidator = PerDimensionValidator.DEFAULT_FP16_VALIDATOR; - - if (!isFaissSQClipToFP16RangeEnabled( - (MethodComponentContext) methodComponentContext.getParameters().get(METHOD_ENCODER_PARAMETER) - )) { - this.perDimensionProcessor = PerDimensionProcessor.NOOP_PROCESSOR; - return; - } - - this.perDimensionProcessor = PerDimensionProcessor.CLIP_TO_FP16_PROCESSOR; + this.perDimensionProcessor = knnLibraryIndexingContext.getPerDimensionProcessor(); + this.perDimensionValidator = knnLibraryIndexingContext.getPerDimensionValidator(); + this.vectorValidator = knnLibraryIndexingContext.getVectorValidator(); } @Override diff --git a/src/main/java/org/opensearch/knn/index/mapper/ModelFieldMapper.java b/src/main/java/org/opensearch/knn/index/mapper/ModelFieldMapper.java index 6c7e45e7e..954d6addf 100644 --- a/src/main/java/org/opensearch/knn/index/mapper/ModelFieldMapper.java +++ b/src/main/java/org/opensearch/knn/index/mapper/ModelFieldMapper.java @@ -13,6 +13,9 @@ import org.opensearch.index.mapper.ParseContext; import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.VectorDataType; +import org.opensearch.knn.index.engine.KNNLibraryIndexingContext; +import org.opensearch.knn.index.engine.KNNMethodConfigContext; +import org.opensearch.knn.index.engine.KNNMethodContext; import org.opensearch.knn.index.engine.MethodComponentContext; import org.opensearch.knn.indices.ModelDao; import org.opensearch.knn.indices.ModelMetadata; @@ -22,10 +25,7 @@ import java.util.Map; import java.util.Optional; -import static org.opensearch.knn.common.KNNConstants.METHOD_ENCODER_PARAMETER; import static org.opensearch.knn.common.KNNConstants.MODEL_ID; -import static org.opensearch.knn.index.mapper.KNNVectorFieldMapperUtil.isFaissSQClipToFP16RangeEnabled; -import static org.opensearch.knn.index.mapper.KNNVectorFieldMapperUtil.isFaissSQfp16; /** * Field mapper for model in mapping @@ -131,7 +131,18 @@ private void initVectorValidator() { return; } ModelMetadata modelMetadata = getModelMetadata(modelDao, modelId); - vectorValidator = new SpaceVectorValidator(modelMetadata.getSpaceType()); + + KNNMethodContext knnMethodContext = getKNNMethodContextFromModelMetadata(modelMetadata); + KNNMethodConfigContext knnMethodConfigContext = getKNNMethodConfigContextFromModelMetadata(modelMetadata); + // Need to handle BWC case + if (knnMethodContext == null || knnMethodConfigContext == null) { + vectorValidator = new SpaceVectorValidator(modelMetadata.getSpaceType()); + return; + } + + KNNLibraryIndexingContext knnLibraryIndexingContext = knnMethodContext.getKnnEngine() + .getKNNLibraryIndexingContext(knnMethodContext, knnMethodConfigContext); + vectorValidator = knnLibraryIndexingContext.getVectorValidator(); } private void initPerDimensionValidator() { @@ -139,25 +150,25 @@ private void initPerDimensionValidator() { return; } ModelMetadata modelMetadata = getModelMetadata(modelDao, modelId); - MethodComponentContext methodComponentContext = modelMetadata.getMethodComponentContext(); - VectorDataType dataType = modelMetadata.getVectorDataType(); - - if (VectorDataType.BINARY == dataType) { - perDimensionValidator = PerDimensionValidator.DEFAULT_BIT_VALIDATOR; - return; - } - if (VectorDataType.BYTE == dataType) { - perDimensionValidator = PerDimensionValidator.DEFAULT_BYTE_VALIDATOR; - return; - } + KNNMethodContext knnMethodContext = getKNNMethodContextFromModelMetadata(modelMetadata); + KNNMethodConfigContext knnMethodConfigContext = getKNNMethodConfigContextFromModelMetadata(modelMetadata); + // Need to handle BWC case + if (knnMethodContext == null || knnMethodConfigContext == null) { + if (modelMetadata.getVectorDataType() == VectorDataType.BINARY) { + perDimensionValidator = PerDimensionValidator.DEFAULT_BIT_VALIDATOR; + } else if (modelMetadata.getVectorDataType() == VectorDataType.BYTE) { + perDimensionValidator = PerDimensionValidator.DEFAULT_BYTE_VALIDATOR; + } else { + perDimensionValidator = PerDimensionValidator.DEFAULT_FLOAT_VALIDATOR; + } - if (!isFaissSQfp16(methodComponentContext)) { - perDimensionValidator = PerDimensionValidator.DEFAULT_FLOAT_VALIDATOR; return; } - perDimensionValidator = PerDimensionValidator.DEFAULT_FP16_VALIDATOR; + KNNLibraryIndexingContext knnLibraryIndexingContext = knnMethodContext.getKnnEngine() + .getKNNLibraryIndexingContext(knnMethodContext, knnMethodConfigContext); + perDimensionValidator = knnLibraryIndexingContext.getPerDimensionValidator(); } private void initPerDimensionProcessor() { @@ -165,31 +176,18 @@ private void initPerDimensionProcessor() { return; } ModelMetadata modelMetadata = getModelMetadata(modelDao, modelId); - MethodComponentContext methodComponentContext = modelMetadata.getMethodComponentContext(); - VectorDataType dataType = modelMetadata.getVectorDataType(); - - if (VectorDataType.BINARY == dataType) { - perDimensionProcessor = PerDimensionProcessor.NOOP_PROCESSOR; - return; - } - if (VectorDataType.BYTE == dataType) { + KNNMethodContext knnMethodContext = getKNNMethodContextFromModelMetadata(modelMetadata); + KNNMethodConfigContext knnMethodConfigContext = getKNNMethodConfigContextFromModelMetadata(modelMetadata); + // Need to handle BWC case + if (knnMethodContext == null || knnMethodConfigContext == null) { perDimensionProcessor = PerDimensionProcessor.NOOP_PROCESSOR; return; } - if (!isFaissSQfp16(methodComponentContext)) { - perDimensionProcessor = PerDimensionProcessor.NOOP_PROCESSOR; - return; - } - - if (!isFaissSQClipToFP16RangeEnabled( - (MethodComponentContext) methodComponentContext.getParameters().get(METHOD_ENCODER_PARAMETER) - )) { - perDimensionProcessor = PerDimensionProcessor.NOOP_PROCESSOR; - return; - } - perDimensionProcessor = PerDimensionProcessor.CLIP_TO_FP16_PROCESSOR; + KNNLibraryIndexingContext knnLibraryIndexingContext = knnMethodContext.getKnnEngine() + .getKNNLibraryIndexingContext(knnMethodContext, knnMethodConfigContext); + perDimensionProcessor = knnLibraryIndexingContext.getPerDimensionProcessor(); } @Override @@ -214,6 +212,27 @@ protected void parseCreateField(ParseContext context) throws IOException { parseCreateField(context, modelMetadata.getDimension(), modelMetadata.getVectorDataType()); } + private static KNNMethodContext getKNNMethodContextFromModelMetadata(ModelMetadata modelMetadata) { + MethodComponentContext methodComponentContext = modelMetadata.getMethodComponentContext(); + if (methodComponentContext == MethodComponentContext.EMPTY) { + return null; + } + return new KNNMethodContext(modelMetadata.getKnnEngine(), modelMetadata.getSpaceType(), methodComponentContext); + } + + private static KNNMethodConfigContext getKNNMethodConfigContextFromModelMetadata(ModelMetadata modelMetadata) { + MethodComponentContext methodComponentContext = modelMetadata.getMethodComponentContext(); + if (methodComponentContext == MethodComponentContext.EMPTY) { + return null; + } + // TODO: Need to fix this version check by serializing the model + return KNNMethodConfigContext.builder() + .vectorDataType(modelMetadata.getVectorDataType()) + .dimension(modelMetadata.getDimension()) + .versionCreated(Version.V_2_14_0) + .build(); + } + private static ModelMetadata getModelMetadata(ModelDao modelDao, String modelId) { ModelMetadata modelMetadata = modelDao.getMetadata(modelId); if (!ModelUtil.isModelCreated(modelMetadata)) { diff --git a/src/main/java/org/opensearch/knn/index/mapper/PerDimensionProcessor.java b/src/main/java/org/opensearch/knn/index/mapper/PerDimensionProcessor.java index 21139f2ad..9a3bbfb6b 100644 --- a/src/main/java/org/opensearch/knn/index/mapper/PerDimensionProcessor.java +++ b/src/main/java/org/opensearch/knn/index/mapper/PerDimensionProcessor.java @@ -5,8 +5,6 @@ package org.opensearch.knn.index.mapper; -import static org.opensearch.knn.index.mapper.KNNVectorFieldMapperUtil.clipVectorValueToFP16Range; - /** * Process values per dimension. Good to have if we want to do some kind of cleanup on data as it is coming in. */ @@ -34,18 +32,4 @@ default float processByte(float value) { PerDimensionProcessor NOOP_PROCESSOR = new PerDimensionProcessor() { }; - - // If the encoder parameter, "clip" is set to True, if the vector value is outside the FP16 range then it will be - // clipped to FP16 range. - PerDimensionProcessor CLIP_TO_FP16_PROCESSOR = new PerDimensionProcessor() { - @Override - public float process(float value) { - return clipVectorValueToFP16Range(value); - } - - @Override - public float processByte(float value) { - throw new IllegalStateException("CLIP_TO_FP16_PROCESSOR should not be called with byte type"); - } - }; } diff --git a/src/main/java/org/opensearch/knn/index/mapper/PerDimensionValidator.java b/src/main/java/org/opensearch/knn/index/mapper/PerDimensionValidator.java index 2ca0761c0..60d8540c6 100644 --- a/src/main/java/org/opensearch/knn/index/mapper/PerDimensionValidator.java +++ b/src/main/java/org/opensearch/knn/index/mapper/PerDimensionValidator.java @@ -9,7 +9,6 @@ import static org.opensearch.knn.common.KNNValidationUtil.validateByteVectorValue; import static org.opensearch.knn.common.KNNValidationUtil.validateFloatVectorValue; -import static org.opensearch.knn.index.mapper.KNNVectorFieldMapperUtil.validateFP16VectorValue; /** * Validates per dimension fields @@ -41,19 +40,6 @@ public void validateByte(float value) { } }; - // Validates if it is a finite number and within the fp16 range of [-65504 to 65504]. - PerDimensionValidator DEFAULT_FP16_VALIDATOR = new PerDimensionValidator() { - @Override - public void validate(float value) { - validateFP16VectorValue(value); - } - - @Override - public void validateByte(float value) { - throw new IllegalStateException("DEFAULT_FP16_VALIDATOR should only be used for float vectors"); - } - }; - PerDimensionValidator DEFAULT_BYTE_VALIDATOR = new PerDimensionValidator() { @Override public void validate(float value) { diff --git a/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java b/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java index af8e410d4..61dba45c8 100644 --- a/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java +++ b/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java @@ -23,6 +23,7 @@ import org.opensearch.index.query.QueryBuilder; import org.opensearch.index.query.QueryRewriteContext; import org.opensearch.index.query.QueryShardContext; +import org.opensearch.knn.index.engine.KNNMethodConfigContext; import org.opensearch.knn.index.engine.model.QueryContext; import org.opensearch.knn.index.mapper.KNNMappingConfig; import org.opensearch.knn.index.mapper.KNNVectorFieldType; @@ -423,7 +424,8 @@ protected Query doToQuery(QueryShardContext context) { QueryContext queryContext = new QueryContext(vectorQueryType); ValidationException validationException = validateParameters( engineSpecificMethodContext.supportedMethodParameters(queryContext), - (Map) methodParameters + (Map) methodParameters, + KNNMethodConfigContext.EMPTY ); if (validationException != null) { throw new IllegalArgumentException( diff --git a/src/main/java/org/opensearch/knn/index/util/IndexUtil.java b/src/main/java/org/opensearch/knn/index/util/IndexUtil.java index 15e1959f8..853b5237a 100644 --- a/src/main/java/org/opensearch/knn/index/util/IndexUtil.java +++ b/src/main/java/org/opensearch/knn/index/util/IndexUtil.java @@ -344,7 +344,7 @@ public static boolean isBinaryIndex(VectorDataType vectorDataType) { */ public static void updateVectorDataTypeToParameters(Map parameters, VectorDataType vectorDataType) { if (VectorDataType.BINARY == vectorDataType) { - parameters.put(VECTOR_DATA_TYPE_FIELD, VectorDataType.BINARY.getValue()); + parameters.put(VECTOR_DATA_TYPE_FIELD, vectorDataType.getValue()); } } diff --git a/src/main/java/org/opensearch/knn/plugin/transport/TrainingModelRequest.java b/src/main/java/org/opensearch/knn/plugin/transport/TrainingModelRequest.java index 9464ea806..3634d13f0 100644 --- a/src/main/java/org/opensearch/knn/plugin/transport/TrainingModelRequest.java +++ b/src/main/java/org/opensearch/knn/plugin/transport/TrainingModelRequest.java @@ -11,6 +11,8 @@ package org.opensearch.knn.plugin.transport; +import lombok.Getter; +import org.opensearch.Version; import org.opensearch.action.ActionRequest; import org.opensearch.action.ActionRequestValidationException; import org.opensearch.cluster.metadata.IndexMetadata; @@ -19,17 +21,18 @@ import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.knn.common.KNNConstants; +import org.opensearch.knn.index.engine.KNNMethodConfigContext; import org.opensearch.knn.index.util.IndexUtil; import org.opensearch.knn.index.engine.KNNMethodContext; import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.indices.ModelDao; -import org.opensearch.knn.training.VectorSpaceInfo; import java.io.IOException; /** * Request to train and serialize a model */ +@Getter public class TrainingModelRequest extends ActionRequest { private static ClusterService clusterService; @@ -37,16 +40,15 @@ public class TrainingModelRequest extends ActionRequest { private final String modelId; private final KNNMethodContext knnMethodContext; + private final KNNMethodConfigContext knnMethodConfigContext; private final int dimension; private final String trainingIndex; private final String trainingField; private final String preferredNodeId; private final String description; private final VectorDataType vectorDataType; - private int maximumVectorCount; private int searchSize; - private int trainingDataSizeInKB; /** @@ -87,6 +89,11 @@ public TrainingModelRequest( // Training data size in kilobytes. By default, this is invalid (it cant have negative kb). It eventually gets // calculated in transit. A user cannot set this value directly. this.trainingDataSizeInKB = -1; + this.knnMethodConfigContext = KNNMethodConfigContext.builder() + .vectorDataType(vectorDataType) + .dimension(dimension) + .versionCreated(Version.CURRENT) + .build(); } /** @@ -112,6 +119,11 @@ public TrainingModelRequest(StreamInput in) throws IOException { } else { this.vectorDataType = VectorDataType.DEFAULT; } + this.knnMethodConfigContext = KNNMethodConfigContext.builder() + .vectorDataType(vectorDataType) + .dimension(dimension) + .versionCreated(in.getVersion()) + .build(); } /** @@ -125,79 +137,6 @@ public static void initialize(ModelDao modelDao, ClusterService clusterService) TrainingModelRequest.clusterService = clusterService; } - /** - * Getter for modelId - * - * @return modelId - */ - public String getModelId() { - return modelId; - } - - /** - * Getter for knnMethodContext - * - * @return knnMethodContext - */ - public KNNMethodContext getKnnMethodContext() { - return knnMethodContext; - } - - /** - * Getter for dimension - * - * @return dimension - */ - public int getDimension() { - return dimension; - } - - /** - * Getter for trainingIndex - * - * @return trainingIndex - */ - public String getTrainingIndex() { - return trainingIndex; - } - - /** - * Getter for trainingField - * - * @return trainingField - */ - public String getTrainingField() { - return trainingField; - } - - /** - * Getter for preferredNodeId - * - * @return preferredNodeId - */ - public String getPreferredNodeId() { - return preferredNodeId; - } - - /** - * Getter description of the model - * - * @return description - */ - public String getDescription() { - return description; - } - - /** - * Getter for maximum vector count. This corresponds to the maximum number of vectors from the training index - * a user wants to use for training. - * - * @return maximumVectorCount - */ - public int getMaximumVectorCount() { - return maximumVectorCount; - } - /** * Setter for maximum vector count * @@ -212,20 +151,6 @@ public void setMaximumVectorCount(int maximumVectorCount) { this.maximumVectorCount = maximumVectorCount; } - /** - * Getter for search size. This value corresponds to how many vectors are pulled from the training index per - * search request - * - * @return searchSize - */ - public int getSearchSize() { - return searchSize; - } - - public VectorDataType getVectorDataType() { - return vectorDataType; - } - /** * Setter for search size. * @@ -240,15 +165,6 @@ public void setSearchSize(int searchSize) { this.searchSize = searchSize; } - /** - * Getter for training data size in kilobytes. - * - * @return trainingDataSizeInKB - */ - public int getTrainingDataSizeInKB() { - return trainingDataSizeInKB; - } - /** * Setter for trainingDataSizeInKB. Package private to prevent users from changing this value directly. * @@ -289,13 +205,7 @@ public ActionRequestValidationException validate() { } // Confirm that the passed in knnMethodContext is valid and requires training - ValidationException validationException = this.knnMethodContext.validate(); - if (validationException != null) { - exception = new ActionRequestValidationException(); - exception.addValidationErrors(validationException.validationErrors()); - } - - validationException = this.knnMethodContext.validateWithData(new VectorSpaceInfo(dimension)); + ValidationException validationException = this.knnMethodContext.validate(knnMethodConfigContext); if (validationException != null) { exception = new ActionRequestValidationException(); exception.addValidationErrors(validationException.validationErrors()); diff --git a/src/main/java/org/opensearch/knn/plugin/transport/TrainingModelTransportAction.java b/src/main/java/org/opensearch/knn/plugin/transport/TrainingModelTransportAction.java index a9eca609d..963142c1f 100644 --- a/src/main/java/org/opensearch/knn/plugin/transport/TrainingModelTransportAction.java +++ b/src/main/java/org/opensearch/knn/plugin/transport/TrainingModelTransportAction.java @@ -11,11 +11,13 @@ package org.opensearch.knn.plugin.transport; +import org.opensearch.Version; import org.opensearch.core.action.ActionListener; import org.opensearch.action.support.ActionFilters; import org.opensearch.action.support.HandledTransportAction; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Inject; +import org.opensearch.knn.index.engine.KNNMethodConfigContext; import org.opensearch.knn.index.memory.NativeMemoryCacheManager; import org.opensearch.knn.index.memory.NativeMemoryEntryContext; import org.opensearch.knn.index.memory.NativeMemoryLoadStrategy; @@ -57,7 +59,14 @@ protected void doExecute(Task task, TrainingModelRequest request, ActionListener // Allocation representing size model will occupy in memory during training NativeMemoryEntryContext.AnonymousEntryContext modelAnonymousEntryContext = new NativeMemoryEntryContext.AnonymousEntryContext( - request.getKnnMethodContext().estimateOverheadInKB(request.getDimension()), + request.getKnnMethodContext() + .estimateOverheadInKB( + KNNMethodConfigContext.builder() + .dimension(request.getDimension()) + .vectorDataType(request.getVectorDataType()) + .versionCreated(Version.CURRENT) + .build() + ), NativeMemoryLoadStrategy.AnonymousLoadStrategy.getInstance() ); @@ -67,10 +76,9 @@ protected void doExecute(Task task, TrainingModelRequest request, ActionListener NativeMemoryCacheManager.getInstance(), trainingDataEntryContext, modelAnonymousEntryContext, - request.getDimension(), + request.getKnnMethodConfigContext(), request.getDescription(), - clusterService.localNode().getEphemeralId(), - request.getVectorDataType() + clusterService.localNode().getEphemeralId() ); KNNCounter.TRAINING_REQUESTS.increment(); diff --git a/src/main/java/org/opensearch/knn/training/TrainingJob.java b/src/main/java/org/opensearch/knn/training/TrainingJob.java index 3bdb50ad0..e30d860db 100644 --- a/src/main/java/org/opensearch/knn/training/TrainingJob.java +++ b/src/main/java/org/opensearch/knn/training/TrainingJob.java @@ -11,14 +11,14 @@ package org.opensearch.knn.training; +import lombok.Getter; import org.apache.commons.lang.StringUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.opensearch.common.UUIDs; import org.opensearch.knn.common.KNNConstants; -import org.opensearch.knn.index.util.IndexUtil; import org.opensearch.knn.index.KNNSettings; -import org.opensearch.knn.index.VectorDataType; +import org.opensearch.knn.index.engine.KNNMethodConfigContext; import org.opensearch.knn.jni.JNIService; import org.opensearch.knn.index.engine.KNNMethodContext; import org.opensearch.knn.index.memory.NativeMemoryAllocation; @@ -34,8 +34,6 @@ import java.util.Map; import java.util.Objects; -import static org.opensearch.knn.index.engine.faiss.Faiss.FAISS_BINARY_INDEX_DESCRIPTION_PREFIX; - /** * Encapsulates all information required to generate and train a model. */ @@ -44,12 +42,15 @@ public class TrainingJob implements Runnable { public static Logger logger = LogManager.getLogger(TrainingJob.class); private final KNNMethodContext knnMethodContext; + private final KNNMethodConfigContext knnMethodConfigContext; private final NativeMemoryCacheManager nativeMemoryCacheManager; private final NativeMemoryEntryContext.TrainingDataEntryContext trainingDataEntryContext; private final NativeMemoryEntryContext.AnonymousEntryContext modelAnonymousEntryContext; + @Getter private final Model model; - private String modelId; + @Getter + private final String modelId; /** * Constructor. @@ -59,7 +60,6 @@ public class TrainingJob implements Runnable { * @param nativeMemoryCacheManager Cache manager loads training data into native memory. * @param trainingDataEntryContext Training data configuration * @param modelAnonymousEntryContext Model allocation context - * @param dimension model's dimension * @param description user provided description of the model. */ public TrainingJob( @@ -68,14 +68,14 @@ public TrainingJob( NativeMemoryCacheManager nativeMemoryCacheManager, NativeMemoryEntryContext.TrainingDataEntryContext trainingDataEntryContext, NativeMemoryEntryContext.AnonymousEntryContext modelAnonymousEntryContext, - int dimension, + KNNMethodConfigContext knnMethodConfigContext, String description, - String nodeAssignment, - VectorDataType vectorDataType + String nodeAssignment ) { // Generate random base64 string if one is not provided this.modelId = StringUtils.isNotBlank(modelId) ? modelId : UUIDs.randomBase64UUID(); this.knnMethodContext = Objects.requireNonNull(knnMethodContext, "MethodContext cannot be null."); + this.knnMethodConfigContext = knnMethodConfigContext; this.nativeMemoryCacheManager = Objects.requireNonNull(nativeMemoryCacheManager, "NativeMemoryCacheManager cannot be null."); this.trainingDataEntryContext = Objects.requireNonNull(trainingDataEntryContext, "TrainingDataEntryContext cannot be null."); this.modelAnonymousEntryContext = Objects.requireNonNull(modelAnonymousEntryContext, "AnonymousEntryContext cannot be null."); @@ -83,38 +83,20 @@ public TrainingJob( new ModelMetadata( knnMethodContext.getKnnEngine(), knnMethodContext.getSpaceType(), - dimension, + knnMethodConfigContext.getDimension(), ModelState.TRAINING, ZonedDateTime.now(ZoneOffset.UTC).toString(), description, "", nodeAssignment, knnMethodContext.getMethodComponentContext(), - vectorDataType + knnMethodConfigContext.getVectorDataType() ), null, this.modelId ); } - /** - * Getter for model id. - * - * @return modelId - */ - public String getModelId() { - return modelId; - } - - /** - * Getter for model - * - * @return model - */ - public Model getModel() { - return model; - } - @Override public void run() { NativeMemoryAllocation trainingDataAllocation = null; @@ -181,25 +163,15 @@ public void run() { if (trainingDataAllocation.isClosed()) { throw new RuntimeException("Unable to load training data into memory: allocation is already closed"); } - setVersionInKnnMethodContext(); Map trainParameters = model.getModelMetadata() .getKnnEngine() - .getKNNLibraryIndexingContext(knnMethodContext) + .getKNNLibraryIndexingContext(knnMethodContext, knnMethodConfigContext) .getLibraryParameters(); trainParameters.put( KNNConstants.INDEX_THREAD_QTY, KNNSettings.state().getSettingValue(KNNSettings.KNN_ALGO_PARAM_INDEX_THREAD_QTY) ); - if (VectorDataType.BINARY == model.getModelMetadata().getVectorDataType()) { - trainParameters.put( - KNNConstants.INDEX_DESCRIPTION_PARAMETER, - FAISS_BINARY_INDEX_DESCRIPTION_PREFIX + trainParameters.get(KNNConstants.INDEX_DESCRIPTION_PARAMETER).toString() - ); - } - - IndexUtil.updateVectorDataTypeToParameters(trainParameters, model.getModelMetadata().getVectorDataType()); - byte[] modelBlob = JNIService.trainIndex( trainParameters, model.getModelMetadata().getDimension(), @@ -227,10 +199,4 @@ public void run() { nativeMemoryCacheManager.invalidate(modelAnonymousEntryContext.getKey()); } } - - private void setVersionInKnnMethodContext() { - // We are picking up the node version here. For more details why we did this please check below conversation - // Ref: https://github.com/opensearch-project/k-NN/pull/1353#discussion_r1434428542 - knnMethodContext.getMethodComponentContext().setIndexVersion(trainingDataEntryContext.getClusterService().localNode().getVersion()); - } } diff --git a/src/main/java/org/opensearch/knn/training/VectorSpaceInfo.java b/src/main/java/org/opensearch/knn/training/VectorSpaceInfo.java deleted file mode 100644 index 13843486d..000000000 --- a/src/main/java/org/opensearch/knn/training/VectorSpaceInfo.java +++ /dev/null @@ -1,26 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - * - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. - */ - -package org.opensearch.knn.training; - -import lombok.AllArgsConstructor; -import lombok.Getter; -import lombok.Setter; - -/** - * A data spec containing relevant information for validation. - */ -@Getter -@Setter -@AllArgsConstructor -public class VectorSpaceInfo { - private int dimension; -} diff --git a/src/test/java/org/opensearch/knn/KNNTestCase.java b/src/test/java/org/opensearch/knn/KNNTestCase.java index fb09fb30b..2aa11a247 100644 --- a/src/test/java/org/opensearch/knn/KNNTestCase.java +++ b/src/test/java/org/opensearch/knn/KNNTestCase.java @@ -7,7 +7,6 @@ import org.mockito.Mock; import org.mockito.MockitoAnnotations; -import org.opensearch.Version; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Setting; @@ -103,16 +102,17 @@ public Map xContentBuilderToMap(XContentBuilder xContentBuilder) public static KNNMethodContext getDefaultKNNMethodContext() { MethodComponentContext methodComponentContext = new MethodComponentContext(METHOD_HNSW, Collections.emptyMap()); - KNNMethodContext defaultInstance = new KNNMethodContext(KNNEngine.DEFAULT, SpaceType.DEFAULT, methodComponentContext); - methodComponentContext.setIndexVersion(Version.CURRENT); - return defaultInstance; + return new KNNMethodContext(KNNEngine.DEFAULT, SpaceType.DEFAULT, methodComponentContext); + } + + public static KNNMethodContext getDefaultByteKNNMethodContext() { + MethodComponentContext methodComponentContext = new MethodComponentContext(METHOD_HNSW, Collections.emptyMap()); + return new KNNMethodContext(KNNEngine.DEFAULT, SpaceType.DEFAULT, methodComponentContext); } public static KNNMethodContext getDefaultBinaryKNNMethodContext() { MethodComponentContext methodComponentContext = new MethodComponentContext(METHOD_HNSW, Collections.emptyMap()); - KNNMethodContext defaultInstance = new KNNMethodContext(KNNEngine.DEFAULT, SpaceType.DEFAULT_BINARY, methodComponentContext); - methodComponentContext.setIndexVersion(Version.CURRENT); - return defaultInstance; + return new KNNMethodContext(KNNEngine.DEFAULT, SpaceType.DEFAULT_BINARY, methodComponentContext); } public static KNNMappingConfig getMappingConfigForMethodMapping(KNNMethodContext knnMethodContext, int dimension) { diff --git a/src/test/java/org/opensearch/knn/index/OpenSearchIT.java b/src/test/java/org/opensearch/knn/index/OpenSearchIT.java index 81a5ab142..c1d3b47c3 100644 --- a/src/test/java/org/opensearch/knn/index/OpenSearchIT.java +++ b/src/test/java/org/opensearch/knn/index/OpenSearchIT.java @@ -326,8 +326,8 @@ public void testVectorMappingValidation_invalidDimension() { containsString( "Dimension value cannot be greater than " + KNNEngine.getMaxDimensionByEngine(KNNEngine.DEFAULT) - + " for vector: " - + FIELD_NAME + + " for vector with engine: " + + KNNEngine.DEFAULT.getName() ) ); } diff --git a/src/test/java/org/opensearch/knn/index/VectorDataTypeIT.java b/src/test/java/org/opensearch/knn/index/VectorDataTypeIT.java index ec2d49de0..0b045e848 100644 --- a/src/test/java/org/opensearch/knn/index/VectorDataTypeIT.java +++ b/src/test/java/org/opensearch/knn/index/VectorDataTypeIT.java @@ -32,9 +32,7 @@ import java.util.Map; import static org.opensearch.knn.common.KNNConstants.DIMENSION; -import static org.opensearch.knn.common.KNNConstants.LUCENE_NAME; import static org.opensearch.knn.common.KNNConstants.METHOD_HNSW; -import static org.opensearch.knn.common.KNNConstants.NMSLIB_NAME; import static org.opensearch.knn.common.KNNConstants.VECTOR_DATA_TYPE_FIELD; import static org.opensearch.knn.index.VectorDataType.SUPPORTED_VECTOR_DATA_TYPES; @@ -245,18 +243,7 @@ public void testByteVectorDataTypeWithNmslibEngine() { ResponseException.class, () -> createKnnIndexMappingWithNmslibEngine(2, SpaceType.L2, VectorDataType.BYTE.getValue()) ); - assertTrue( - ex.getMessage() - .contains( - String.format( - Locale.ROOT, - "[%s] field with value [%s] is only supported for [%s] engine", - VECTOR_DATA_TYPE_FIELD, - VectorDataType.BYTE.getValue(), - LUCENE_NAME - ) - ) - ); + assertTrue(ex.getMessage().contains("is not supported for vector data type")); } @SneakyThrows @@ -278,18 +265,7 @@ public void testByteVectorDataTypeWithLegacyFieldMapperKnnIndexSetting() { String mapping = builder.toString(); ResponseException ex = expectThrows(ResponseException.class, () -> createKnnIndex(INDEX_NAME, mapping)); - assertTrue( - ex.getMessage() - .contains( - String.format( - Locale.ROOT, - "[%s] field with value [%s] is not supported for [%s] engine", - VECTOR_DATA_TYPE_FIELD, - VectorDataType.BYTE.getValue(), - NMSLIB_NAME - ) - ) - ); + assertTrue(ex.getMessage(), ex.getMessage().contains("is not supported for vector data type")); } diff --git a/src/test/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumerTests.java b/src/test/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumerTests.java index e87531561..4c235a896 100644 --- a/src/test/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumerTests.java +++ b/src/test/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumerTests.java @@ -25,6 +25,7 @@ import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.knn.KNNTestCase; import org.opensearch.knn.common.KNNConstants; +import org.opensearch.knn.index.engine.KNNMethodConfigContext; import org.opensearch.knn.index.engine.KNNMethodContext; import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.vectorvalues.TestVectorValues; @@ -202,6 +203,10 @@ public void testAddKNNBinaryField_fromScratch_nmslibCurrent() throws IOException .codec(codec) .build(); + KNNMethodConfigContext knnMethodConfigContext = KNNMethodConfigContext.builder() + .vectorDataType(VectorDataType.FLOAT) + .versionCreated(Version.CURRENT) + .build(); KNNMethodContext knnMethodContext = new KNNMethodContext( knnEngine, spaceType, @@ -209,7 +214,7 @@ public void testAddKNNBinaryField_fromScratch_nmslibCurrent() throws IOException ); String parameterString = XContentFactory.jsonBuilder() - .map(knnEngine.getKNNLibraryIndexingContext(knnMethodContext).getLibraryParameters()) + .map(knnEngine.getKNNLibraryIndexingContext(knnMethodContext, knnMethodConfigContext).getLibraryParameters()) .toString(); FieldInfo[] fieldInfoArray = new FieldInfo[] { @@ -267,15 +272,18 @@ public void testAddKNNBinaryField_fromScratch_faissCurrent() throws IOException .codec(codec) .build(); + KNNMethodConfigContext knnMethodConfigContext = KNNMethodConfigContext.builder() + .vectorDataType(VectorDataType.FLOAT) + .versionCreated(Version.CURRENT) + .build(); KNNMethodContext knnMethodContext = new KNNMethodContext( knnEngine, spaceType, new MethodComponentContext(METHOD_HNSW, ImmutableMap.of(METHOD_PARAMETER_M, 16, METHOD_PARAMETER_EF_CONSTRUCTION, 512)) ); - knnMethodContext.getMethodComponentContext().setIndexVersion(Version.CURRENT); String parameterString = XContentFactory.jsonBuilder() - .map(knnEngine.getKNNLibraryIndexingContext(knnMethodContext).getLibraryParameters()) + .map(knnEngine.getKNNLibraryIndexingContext(knnMethodContext, knnMethodConfigContext).getLibraryParameters()) .toString(); FieldInfo[] fieldInfoArray = new FieldInfo[] { @@ -334,15 +342,18 @@ public void testAddKNNBinaryField_whenFaissBinary_thenAdded() throws IOException .codec(codec) .build(); + KNNMethodConfigContext knnMethodConfigContext = KNNMethodConfigContext.builder() + .vectorDataType(VectorDataType.BINARY) + .versionCreated(Version.CURRENT) + .build(); KNNMethodContext knnMethodContext = new KNNMethodContext( knnEngine, spaceType, new MethodComponentContext(METHOD_HNSW, ImmutableMap.of(METHOD_PARAMETER_M, 16, METHOD_PARAMETER_EF_CONSTRUCTION, 512)) ); - knnMethodContext.getMethodComponentContext().setIndexVersion(Version.CURRENT); String parameterString = XContentFactory.jsonBuilder() - .map(knnEngine.getKNNLibraryIndexingContext(knnMethodContext).getLibraryParameters()) + .map(knnEngine.getKNNLibraryIndexingContext(knnMethodContext, knnMethodConfigContext).getLibraryParameters()) .toString(); FieldInfo[] fieldInfoArray = new FieldInfo[] { diff --git a/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestCase.java b/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestCase.java index bf2c33bf9..1158d3ebb 100644 --- a/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestCase.java +++ b/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestCase.java @@ -16,13 +16,13 @@ import org.apache.lucene.search.Query; import org.apache.lucene.search.TopDocs; import org.apache.lucene.search.join.BitSetProducer; -import org.opensearch.Version; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Setting; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.index.mapper.MapperService; import org.opensearch.knn.KNNTestCase; import org.opensearch.knn.common.KNNConstants; +import org.opensearch.knn.index.engine.KNNMethodConfigContext; import org.opensearch.knn.index.engine.KNNMethodContext; import org.opensearch.knn.index.KNNSettings; import org.opensearch.knn.index.engine.MethodComponentContext; @@ -93,16 +93,23 @@ public class KNNCodecTestCase extends KNNTestCase { private static final FieldType sampleFieldType; static { + KNNMethodConfigContext knnMethodConfigContext = KNNMethodConfigContext.builder() + .versionCreated(CURRENT) + .vectorDataType(VectorDataType.DEFAULT) + .build(); KNNMethodContext knnMethodContext = new KNNMethodContext( KNNEngine.DEFAULT, SpaceType.DEFAULT, new MethodComponentContext(METHOD_HNSW, ImmutableMap.of(METHOD_PARAMETER_M, 16, METHOD_PARAMETER_EF_CONSTRUCTION, 512)) ); - knnMethodContext.getMethodComponentContext().setIndexVersion(Version.CURRENT); String parameterString; try { parameterString = XContentFactory.jsonBuilder() - .map(knnMethodContext.getKnnEngine().getKNNLibraryIndexingContext(knnMethodContext).getLibraryParameters()) + .map( + knnMethodContext.getKnnEngine() + .getKNNLibraryIndexingContext(knnMethodContext, knnMethodConfigContext) + .getLibraryParameters() + ) .toString(); } catch (IOException e) { throw new RuntimeException(e); diff --git a/src/test/java/org/opensearch/knn/index/engine/AbstractKNNLibraryTests.java b/src/test/java/org/opensearch/knn/index/engine/AbstractKNNLibraryTests.java index c6ab9ccdb..6f8f9afe5 100644 --- a/src/test/java/org/opensearch/knn/index/engine/AbstractKNNLibraryTests.java +++ b/src/test/java/org/opensearch/knn/index/engine/AbstractKNNLibraryTests.java @@ -27,24 +27,25 @@ public class AbstractKNNLibraryTests extends KNNTestCase { private final static String CURRENT_VERSION = "test-version"; private final static String INVALID_METHOD_THROWS_VALIDATION_NAME = "test-method-1"; private final static KNNMethod INVALID_METHOD_THROWS_VALIDATION = new AbstractKNNMethod( - MethodComponent.Builder.builder(INVALID_METHOD_THROWS_VALIDATION_NAME).build(), + MethodComponent.Builder.builder(INVALID_METHOD_THROWS_VALIDATION_NAME).addSupportedDataTypes(Set.of(VectorDataType.FLOAT)).build(), Set.of(SpaceType.DEFAULT), new DefaultHnswSearchContext() ) { @Override - public ValidationException validate(KNNMethodContext knnMethodContext) { + public ValidationException validate(KNNMethodContext knnMethodContext, KNNMethodConfigContext knnMethodConfigContext) { return new ValidationException(); } }; private final static String VALID_METHOD_NAME = "test-method-2"; private final static KNNLibrarySearchContext VALID_METHOD_CONTEXT = ctx -> ImmutableMap.of( "myparameter", - new Parameter.BooleanParameter("myparameter", null, value -> true) + new Parameter.BooleanParameter("myparameter", null, (v, context) -> true) ); private final static Map VALID_EXPECTED_MAP = ImmutableMap.of("test-key", "test-param"); private final static KNNMethod VALID_METHOD = new AbstractKNNMethod( MethodComponent.Builder.builder(VALID_METHOD_NAME) - .setMapGenerator((methodComponent, methodComponentContext) -> VALID_EXPECTED_MAP) + .setMapGenerator((methodComponent, methodComponentContext, knnMethodConfigContext) -> VALID_EXPECTED_MAP) + .addSupportedDataTypes(Set.of(VectorDataType.FLOAT)) .build(), Set.of(SpaceType.DEFAULT), VALID_METHOD_CONTEXT @@ -60,17 +61,22 @@ public void testGetVersion() { } public void testValidateMethod() throws IOException { + KNNMethodConfigContext knnMethodConfigContext = KNNMethodConfigContext.builder() + .versionCreated(org.opensearch.Version.CURRENT) + .dimension(10) + .vectorDataType(VectorDataType.FLOAT) + .build(); // Invalid - method not supported XContentBuilder xContentBuilder = XContentFactory.jsonBuilder().startObject().field(NAME, "invalid").endObject(); Map in = xContentBuilderToMap(xContentBuilder); KNNMethodContext knnMethodContext1 = KNNMethodContext.parse(in); - expectThrows(IllegalArgumentException.class, () -> TEST_LIBRARY.validateMethod(knnMethodContext1)); + assertNotNull(TEST_LIBRARY.validateMethod(knnMethodContext1, knnMethodConfigContext)); // Invalid - method validation xContentBuilder = XContentFactory.jsonBuilder().startObject().field(NAME, INVALID_METHOD_THROWS_VALIDATION_NAME).endObject(); in = xContentBuilderToMap(xContentBuilder); KNNMethodContext knnMethodContext2 = KNNMethodContext.parse(in); - assertNotNull(TEST_LIBRARY.validateMethod(knnMethodContext2)); + expectThrows(IllegalStateException.class, () -> TEST_LIBRARY.validateMethod(knnMethodContext2, knnMethodConfigContext)); } public void testEngineSpecificMethods() { @@ -84,15 +90,24 @@ public void testEngineSpecificMethods() { } public void testGetKNNLibraryIndexingContext() { + KNNMethodConfigContext knnMethodConfigContext = KNNMethodConfigContext.builder() + .versionCreated(org.opensearch.Version.CURRENT) + .dimension(10) + .vectorDataType(VectorDataType.FLOAT) + .build(); // Check that map is expected Map expectedMap = new HashMap<>(VALID_EXPECTED_MAP); expectedMap.put(KNNConstants.SPACE_TYPE, SpaceType.DEFAULT.getValue()); + expectedMap.put(KNNConstants.VECTOR_DATA_TYPE_FIELD, VectorDataType.FLOAT.getValue()); KNNMethodContext knnMethodContext = new KNNMethodContext( KNNEngine.DEFAULT, SpaceType.DEFAULT, new MethodComponentContext(VALID_METHOD_NAME, Collections.emptyMap()) ); - assertEquals(expectedMap, TEST_LIBRARY.getKNNLibraryIndexingContext(knnMethodContext).getLibraryParameters()); + assertEquals( + expectedMap, + TEST_LIBRARY.getKNNLibraryIndexingContext(knnMethodContext, knnMethodConfigContext).getLibraryParameters() + ); // Check when invalid method is passed in KNNMethodContext invalidKnnMethodContext = new KNNMethodContext( @@ -100,7 +115,10 @@ public void testGetKNNLibraryIndexingContext() { SpaceType.DEFAULT, new MethodComponentContext("invalid", Collections.emptyMap()) ); - expectThrows(IllegalArgumentException.class, () -> TEST_LIBRARY.getKNNLibraryIndexingContext(invalidKnnMethodContext)); + expectThrows( + IllegalArgumentException.class, + () -> TEST_LIBRARY.getKNNLibraryIndexingContext(invalidKnnMethodContext, knnMethodConfigContext) + ); } private static class TestAbstractKNNLibrary extends AbstractKNNLibrary { @@ -133,7 +151,7 @@ public Float scoreToRadialThreshold(Float score, SpaceType spaceType) { } @Override - public int estimateOverheadInKB(KNNMethodContext knnMethodContext, int dimension) { + public int estimateOverheadInKB(KNNMethodContext knnMethodContext, KNNMethodConfigContext knnMethodConfigContext) { return 0; } diff --git a/src/test/java/org/opensearch/knn/index/engine/AbstractKNNMethodTests.java b/src/test/java/org/opensearch/knn/index/engine/AbstractKNNMethodTests.java index 2c739c6f7..4d743c42a 100644 --- a/src/test/java/org/opensearch/knn/index/engine/AbstractKNNMethodTests.java +++ b/src/test/java/org/opensearch/knn/index/engine/AbstractKNNMethodTests.java @@ -11,7 +11,7 @@ import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.knn.common.KNNConstants; import org.opensearch.knn.index.SpaceType; -import org.opensearch.knn.training.VectorSpaceInfo; +import org.opensearch.knn.index.VectorDataType; import java.io.IOException; import java.util.HashMap; @@ -49,9 +49,14 @@ public void testHasSpace() { * Test KNNMethod validate */ public void testValidate() throws IOException { + KNNMethodConfigContext knnMethodConfigContext = KNNMethodConfigContext.builder() + .versionCreated(org.opensearch.Version.CURRENT) + .dimension(10) + .vectorDataType(VectorDataType.FLOAT) + .build(); String methodName = "test-method"; KNNMethod knnMethod = new TestKNNMethod( - MethodComponent.Builder.builder(methodName).build(), + MethodComponent.Builder.builder(methodName).addSupportedDataTypes(Set.of(VectorDataType.FLOAT)).build(), Set.of(SpaceType.L2), EMPTY_ENGINE_SPECIFIC_CONTEXT ); @@ -64,7 +69,7 @@ public void testValidate() throws IOException { .endObject(); Map in = xContentBuilderToMap(xContentBuilder); KNNMethodContext knnMethodContext1 = KNNMethodContext.parse(in); - assertNotNull(knnMethod.validate(knnMethodContext1)); + assertNotNull(knnMethod.validate(knnMethodContext1, knnMethodConfigContext)); // Invalid methodComponent xContentBuilder = XContentFactory.jsonBuilder() @@ -78,7 +83,7 @@ public void testValidate() throws IOException { in = xContentBuilderToMap(xContentBuilder); KNNMethodContext knnMethodContext2 = KNNMethodContext.parse(in); - assertNotNull(knnMethod.validate(knnMethodContext2)); + assertNotNull(knnMethod.validate(knnMethodContext2, knnMethodConfigContext)); // Valid everything xContentBuilder = XContentFactory.jsonBuilder() @@ -88,22 +93,25 @@ public void testValidate() throws IOException { .endObject(); in = xContentBuilderToMap(xContentBuilder); KNNMethodContext knnMethodContext3 = KNNMethodContext.parse(in); - assertNull(knnMethod.validate(knnMethodContext3)); + assertNull(knnMethod.validate(knnMethodContext3, knnMethodConfigContext)); } /** * Test KNNMethod validateWithData */ - public void testValidateWithData() throws IOException { + public void testValidateWithContext() throws IOException { + KNNMethodConfigContext knnMethodConfigContext = KNNMethodConfigContext.builder() + .versionCreated(org.opensearch.Version.CURRENT) + .dimension(4) + .vectorDataType(VectorDataType.FLOAT) + .build(); String methodName = "test-method"; KNNMethod knnMethod = new TestKNNMethod( - MethodComponent.Builder.builder(methodName).build(), + MethodComponent.Builder.builder(methodName).addSupportedDataTypes(Set.of(VectorDataType.FLOAT)).build(), Set.of(SpaceType.L2), EMPTY_ENGINE_SPECIFIC_CONTEXT ); - VectorSpaceInfo testVectorSpaceInfo = new VectorSpaceInfo(4); - // Invalid space XContentBuilder xContentBuilder = XContentFactory.jsonBuilder() .startObject() @@ -112,7 +120,7 @@ public void testValidateWithData() throws IOException { .endObject(); Map in = xContentBuilderToMap(xContentBuilder); KNNMethodContext knnMethodContext1 = KNNMethodContext.parse(in); - assertNotNull(knnMethod.validateWithData(knnMethodContext1, testVectorSpaceInfo)); + assertNotNull(knnMethod.validate(knnMethodContext1, knnMethodConfigContext)); // Invalid methodComponent xContentBuilder = XContentFactory.jsonBuilder() @@ -125,8 +133,7 @@ public void testValidateWithData() throws IOException { .endObject(); in = xContentBuilderToMap(xContentBuilder); KNNMethodContext knnMethodContext2 = KNNMethodContext.parse(in); - - assertNotNull(knnMethod.validateWithData(knnMethodContext2, testVectorSpaceInfo)); + assertNotNull(knnMethod.validate(knnMethodContext2, knnMethodConfigContext)); // Valid everything xContentBuilder = XContentFactory.jsonBuilder() @@ -136,26 +143,33 @@ public void testValidateWithData() throws IOException { .endObject(); in = xContentBuilderToMap(xContentBuilder); KNNMethodContext knnMethodContext3 = KNNMethodContext.parse(in); - assertNull(knnMethod.validateWithData(knnMethodContext3, testVectorSpaceInfo)); + assertNull(knnMethod.validate(knnMethodContext3, knnMethodConfigContext)); } public void testGetKNNLibraryIndexingContext() { + KNNMethodConfigContext knnMethodConfigContext = KNNMethodConfigContext.builder() + .versionCreated(org.opensearch.Version.CURRENT) + .dimension(4) + .vectorDataType(VectorDataType.FLOAT) + .build(); SpaceType spaceType = SpaceType.DEFAULT; String methodName = "test-method"; Map generatedMap = ImmutableMap.of("test-key", "test-value"); MethodComponent methodComponent = MethodComponent.Builder.builder(methodName) - .setMapGenerator(((methodComponent1, methodComponentContext) -> methodComponentContext.getParameters())) + .setMapGenerator(((methodComponent1, methodComponentContext, methodConfigContext) -> methodComponentContext.getParameters())) .build(); KNNMethod knnMethod = new TestKNNMethod(methodComponent, Set.of(SpaceType.L2), EMPTY_ENGINE_SPECIFIC_CONTEXT); Map expectedMap = new HashMap<>(generatedMap); expectedMap.put(KNNConstants.SPACE_TYPE, spaceType.getValue()); + expectedMap.put(KNNConstants.VECTOR_DATA_TYPE_FIELD, VectorDataType.FLOAT.getValue()); assertEquals( expectedMap, knnMethod.getKNNLibraryIndexingContext( - new KNNMethodContext(KNNEngine.DEFAULT, spaceType, new MethodComponentContext(methodName, generatedMap)) + new KNNMethodContext(KNNEngine.DEFAULT, spaceType, new MethodComponentContext(methodName, generatedMap)), + knnMethodConfigContext ).getLibraryParameters() ); } diff --git a/src/test/java/org/opensearch/knn/index/KNNMethodContextTests.java b/src/test/java/org/opensearch/knn/index/engine/KNNMethodContextTests.java similarity index 80% rename from src/test/java/org/opensearch/knn/index/KNNMethodContextTests.java rename to src/test/java/org/opensearch/knn/index/engine/KNNMethodContextTests.java index f71fbaae0..6defa4c50 100644 --- a/src/test/java/org/opensearch/knn/index/KNNMethodContextTests.java +++ b/src/test/java/org/opensearch/knn/index/engine/KNNMethodContextTests.java @@ -1,26 +1,21 @@ /* + * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - * - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. */ -package org.opensearch.knn.index; +package org.opensearch.knn.index.engine; +import org.opensearch.Version; import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.knn.KNNTestCase; -import org.opensearch.knn.index.engine.KNNEngine; +import org.opensearch.knn.common.KNNConstants; +import org.opensearch.knn.index.SpaceType; +import org.opensearch.knn.index.VectorDataType; import com.google.common.collect.ImmutableMap; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.index.mapper.MapperParsingException; -import org.opensearch.knn.index.engine.KNNMethodContext; -import org.opensearch.knn.index.engine.MethodComponentContext; import java.io.IOException; import java.util.Collections; @@ -93,23 +88,25 @@ public void testGetSpaceType() { * Test KNNMethodContext validation */ public void testValidate() { - // Check valid default - this should not throw any exception - assertNull(getDefaultKNNMethodContext().validate()); - // Check a valid nmslib method MethodComponentContext hnswMethod = new MethodComponentContext(METHOD_HNSW, Collections.emptyMap()); + KNNMethodConfigContext knnMethodConfigContext = KNNMethodConfigContext.builder() + .vectorDataType(VectorDataType.FLOAT) + .dimension(2) + .versionCreated(Version.CURRENT) + .build(); KNNMethodContext knnMethodContext = new KNNMethodContext(KNNEngine.NMSLIB, SpaceType.L2, hnswMethod); - assertNull(knnMethodContext.validate()); + assertNull(knnMethodContext.validate(knnMethodConfigContext)); // Check invalid parameter nmslib hnswMethod = new MethodComponentContext(METHOD_HNSW, ImmutableMap.of("invalid", 111)); KNNMethodContext knnMethodContext1 = new KNNMethodContext(KNNEngine.NMSLIB, SpaceType.L2, hnswMethod); - assertNotNull(knnMethodContext1.validate()); + assertNotNull(knnMethodContext1.validate(knnMethodConfigContext)); // Check invalid method nmslib MethodComponentContext invalidMethod = new MethodComponentContext("invalid", Collections.emptyMap()); KNNMethodContext knnMethodContext2 = new KNNMethodContext(KNNEngine.NMSLIB, SpaceType.L2, invalidMethod); - expectThrows(IllegalArgumentException.class, knnMethodContext2::validate); + assertNotNull(knnMethodContext2.validate(knnMethodConfigContext)); } /** @@ -146,16 +143,26 @@ public void testRequiresTraining() { public void testEstimateOverheadInKB_whenMethodIsHNSWFlatNmslib_thenSizeIsExpectedValue() { // For HNSW no encoding we expect 0 MethodComponentContext hnswMethod = new MethodComponentContext(METHOD_HNSW, Collections.emptyMap()); + KNNMethodConfigContext knnMethodConfigContext = KNNMethodConfigContext.builder() + .vectorDataType(VectorDataType.FLOAT) + .dimension(2) + .versionCreated(Version.CURRENT) + .build(); KNNMethodContext knnMethodContext = new KNNMethodContext(KNNEngine.NMSLIB, SpaceType.L2, hnswMethod); - assertEquals(0, knnMethodContext.estimateOverheadInKB(1000)); + assertEquals(0, knnMethodContext.estimateOverheadInKB(knnMethodConfigContext)); } public void testEstimateOverheadInKB_whenMethodIsHNSWFlatFaiss_thenSizeIsExpectedValue() { // For HNSW no encoding we expect 0 MethodComponentContext hnswMethod = new MethodComponentContext(METHOD_HNSW, Collections.emptyMap()); + KNNMethodConfigContext knnMethodConfigContext = KNNMethodConfigContext.builder() + .vectorDataType(VectorDataType.FLOAT) + .dimension(168) + .versionCreated(Version.CURRENT) + .build(); KNNMethodContext knnMethodContext = new KNNMethodContext(KNNEngine.FAISS, SpaceType.INNER_PRODUCT, hnswMethod); - assertEquals(0, knnMethodContext.estimateOverheadInKB(168)); + assertEquals(0, knnMethodContext.estimateOverheadInKB(knnMethodConfigContext)); } @@ -172,8 +179,13 @@ public void testEstimateOverheadInKB_whenMethodIsHNSWPQFaiss_thenSizeIsExpectedV METHOD_HNSW, ImmutableMap.of(METHOD_ENCODER_PARAMETER, pqMethodContext) ); + KNNMethodConfigContext knnMethodConfigContext = KNNMethodConfigContext.builder() + .vectorDataType(VectorDataType.FLOAT) + .dimension(dimension) + .versionCreated(Version.CURRENT) + .build(); KNNMethodContext knnMethodContext = new KNNMethodContext(KNNEngine.FAISS, SpaceType.L2, hnswMethodPq); - assertEquals(expectedHnswPq, knnMethodContext.estimateOverheadInKB(dimension)); + assertEquals(expectedHnswPq, knnMethodContext.estimateOverheadInKB(knnMethodConfigContext)); } public void testEstimateOverheadInKB_whenMethodIsIVFFlatFaiss_thenSizeIsExpectedValue() { @@ -183,8 +195,13 @@ public void testEstimateOverheadInKB_whenMethodIsIVFFlatFaiss_thenSizeIsExpected int expectedIvf = 4 * nlists * dimension / BYTES_PER_KILOBYTES + 1; MethodComponentContext ivfMethod = new MethodComponentContext(METHOD_IVF, ImmutableMap.of(METHOD_PARAMETER_NLIST, nlists)); + KNNMethodConfigContext knnMethodConfigContext = KNNMethodConfigContext.builder() + .vectorDataType(VectorDataType.FLOAT) + .dimension(dimension) + .versionCreated(Version.CURRENT) + .build(); KNNMethodContext knnMethodContext = new KNNMethodContext(KNNEngine.FAISS, SpaceType.L2, ivfMethod); - assertEquals(expectedIvf, knnMethodContext.estimateOverheadInKB(dimension)); + assertEquals(expectedIvf, knnMethodContext.estimateOverheadInKB(knnMethodConfigContext)); } public void testEstimateOverheadInKB_whenMethodIsIVFPQFaiss_thenSizeIsExpectedValue() { @@ -206,8 +223,13 @@ public void testEstimateOverheadInKB_whenMethodIsIVFPQFaiss_thenSizeIsExpectedVa METHOD_IVF, ImmutableMap.of(METHOD_PARAMETER_NLIST, nlists, METHOD_ENCODER_PARAMETER, pqMethodContext) ); + KNNMethodConfigContext knnMethodConfigContext = KNNMethodConfigContext.builder() + .vectorDataType(VectorDataType.FLOAT) + .dimension(dimension) + .versionCreated(Version.CURRENT) + .build(); KNNMethodContext knnMethodContext = new KNNMethodContext(KNNEngine.FAISS, SpaceType.L2, ivfMethodPq); - assertEquals(expectedIvfPq, knnMethodContext.estimateOverheadInKB(dimension)); + assertEquals(expectedIvfPq, knnMethodContext.estimateOverheadInKB(knnMethodConfigContext)); } /** @@ -411,4 +433,62 @@ public void testHashCode() { assertNotEquals(methodContext1.hashCode(), methodContext4.hashCode()); assertNotEquals(methodContext1.hashCode(), methodContext5.hashCode()); } + + public void testValidateVectorDataType_whenBinaryFaissHNSW_thenValid() { + validateValidateVectorDataType(KNNEngine.FAISS, KNNConstants.METHOD_HNSW, VectorDataType.BINARY, SpaceType.HAMMING, null); + } + + public void testValidateVectorDataType_whenBinaryNonFaiss_thenException() { + validateValidateVectorDataType( + KNNEngine.LUCENE, + KNNConstants.METHOD_HNSW, + VectorDataType.BINARY, + SpaceType.HAMMING, + "UnsupportedMethod" + ); + validateValidateVectorDataType( + KNNEngine.NMSLIB, + KNNConstants.METHOD_HNSW, + VectorDataType.BINARY, + SpaceType.HAMMING, + "UnsupportedMethod" + ); + } + + public void testValidateVectorDataType_whenByteLucene_thenValid() { + validateValidateVectorDataType(KNNEngine.LUCENE, KNNConstants.METHOD_HNSW, VectorDataType.BYTE, SpaceType.L2, null); + } + + public void testValidateVectorDataType_whenByteNonLucene_thenException() { + validateValidateVectorDataType(KNNEngine.FAISS, KNNConstants.METHOD_HNSW, VectorDataType.BYTE, SpaceType.L2, "UnsupportedMethod"); + validateValidateVectorDataType(KNNEngine.NMSLIB, KNNConstants.METHOD_IVF, VectorDataType.BYTE, SpaceType.L2, "UnsupportedMethod"); + } + + public void testValidateVectorDataType_whenFloat_thenValid() { + validateValidateVectorDataType(KNNEngine.FAISS, KNNConstants.METHOD_HNSW, VectorDataType.FLOAT, SpaceType.L2, null); + validateValidateVectorDataType(KNNEngine.LUCENE, KNNConstants.METHOD_HNSW, VectorDataType.FLOAT, SpaceType.L2, null); + validateValidateVectorDataType(KNNEngine.NMSLIB, KNNConstants.METHOD_HNSW, VectorDataType.FLOAT, SpaceType.L2, null); + } + + private void validateValidateVectorDataType( + final KNNEngine knnEngine, + final String methodName, + final VectorDataType vectorDataType, + final SpaceType spaceType, + final String expectedErrMsg + ) { + MethodComponentContext methodComponentContext = new MethodComponentContext(methodName, Collections.emptyMap()); + KNNMethodContext methodContext = new KNNMethodContext(knnEngine, spaceType, methodComponentContext); + KNNMethodConfigContext knnMethodConfigContext = KNNMethodConfigContext.builder() + .vectorDataType(vectorDataType) + .dimension(8) + .versionCreated(Version.CURRENT) + .build(); + if (expectedErrMsg == null) { + assertNull(methodContext.validate(knnMethodConfigContext)); + } else { + assertNotNull(methodContext.validate(knnMethodConfigContext)); + } + } + } diff --git a/src/test/java/org/opensearch/knn/index/engine/MethodComponentTests.java b/src/test/java/org/opensearch/knn/index/engine/MethodComponentTests.java index c72f59be4..a5c72f5ee 100644 --- a/src/test/java/org/opensearch/knn/index/engine/MethodComponentTests.java +++ b/src/test/java/org/opensearch/knn/index/engine/MethodComponentTests.java @@ -6,12 +6,15 @@ package org.opensearch.knn.index.engine; import com.google.common.collect.ImmutableMap; +import org.opensearch.Version; import org.opensearch.knn.KNNTestCase; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.knn.index.VectorDataType; import java.io.IOException; import java.util.Map; +import java.util.Set; import static org.opensearch.knn.common.KNNConstants.NAME; import static org.opensearch.knn.common.KNNConstants.PARAMETERS; @@ -33,7 +36,7 @@ public void testGetParameters() { String name = "test"; String paramKey = "key"; MethodComponent methodComponent = MethodComponent.Builder.builder(name) - .addParameter(paramKey, new Parameter.IntegerParameter(paramKey, 1, v -> v > 0)) + .addParameter(paramKey, new Parameter.IntegerParameter(paramKey, 1, (v, context) -> v > 0)) .build(); assertEquals(1, methodComponent.getParameters().size()); assertTrue(methodComponent.getParameters().containsKey(paramKey)); @@ -52,11 +55,18 @@ public void testValidate() throws IOException { .field("invalid", "invalid") .endObject() .endObject(); + KNNMethodConfigContext knnMethodConfigContext = KNNMethodConfigContext.builder() + .dimension(1) + .versionCreated(Version.CURRENT) + .vectorDataType(VectorDataType.FLOAT) + .build(); Map in = xContentBuilderToMap(xContentBuilder); MethodComponentContext componentContext1 = MethodComponentContext.parse(in); - MethodComponent methodComponent1 = MethodComponent.Builder.builder(methodName).build(); - assertNotNull(methodComponent1.validate(componentContext1)); + MethodComponent methodComponent1 = MethodComponent.Builder.builder(methodName) + .addSupportedDataTypes(Set.of(VectorDataType.FLOAT)) + .build(); + assertNotNull(methodComponent1.validate(componentContext1, knnMethodConfigContext)); // Invalid parameter type xContentBuilder = XContentFactory.jsonBuilder() @@ -70,9 +80,10 @@ public void testValidate() throws IOException { MethodComponentContext componentContext2 = MethodComponentContext.parse(in); MethodComponent methodComponent2 = MethodComponent.Builder.builder(methodName) - .addParameter("valid", new Parameter.IntegerParameter("valid", 1, v -> v > 0)) + .addSupportedDataTypes(Set.of(VectorDataType.FLOAT)) + .addParameter("valid", new Parameter.IntegerParameter("valid", 1, (v, context) -> v > 0)) .build(); - assertNotNull(methodComponent2.validate(componentContext2)); + assertNotNull(methodComponent2.validate(componentContext2, knnMethodConfigContext)); // valid configuration xContentBuilder = XContentFactory.jsonBuilder() @@ -87,10 +98,11 @@ public void testValidate() throws IOException { MethodComponentContext componentContext3 = MethodComponentContext.parse(in); MethodComponent methodComponent3 = MethodComponent.Builder.builder(methodName) - .addParameter("valid1", new Parameter.IntegerParameter("valid1", 1, v -> v > 0)) - .addParameter("valid2", new Parameter.IntegerParameter("valid2", 1, v -> v > 0)) + .addSupportedDataTypes(Set.of(VectorDataType.FLOAT)) + .addParameter("valid1", new Parameter.IntegerParameter("valid1", 1, (v, context) -> v > 0)) + .addParameter("valid2", new Parameter.IntegerParameter("valid2", 1, (v, context) -> v > 0)) .build(); - assertNull(methodComponent3.validate(componentContext3)); + assertNull(methodComponent3.validate(componentContext3, knnMethodConfigContext)); // valid configuration - empty parameters xContentBuilder = XContentFactory.jsonBuilder().startObject().field(NAME, methodName).endObject(); @@ -98,10 +110,11 @@ public void testValidate() throws IOException { MethodComponentContext componentContext4 = MethodComponentContext.parse(in); MethodComponent methodComponent4 = MethodComponent.Builder.builder(methodName) - .addParameter("valid1", new Parameter.IntegerParameter("valid1", 1, v -> v > 0)) - .addParameter("valid2", new Parameter.IntegerParameter("valid2", 1, v -> v > 0)) + .addSupportedDataTypes(Set.of(VectorDataType.FLOAT)) + .addParameter("valid1", new Parameter.IntegerParameter("valid1", 1, (v, context) -> v > 0)) + .addParameter("valid2", new Parameter.IntegerParameter("valid2", 1, (v, context) -> v > 0)) .build(); - assertNull(methodComponent4.validate(componentContext4)); + assertNull(methodComponent4.validate(componentContext4, knnMethodConfigContext)); } @SuppressWarnings("unchecked") @@ -113,8 +126,8 @@ public void testGetAsMap_withoutGenerator() throws IOException { int default2 = 5; MethodComponent methodComponent = MethodComponent.Builder.builder(methodName) - .addParameter(parameterName1, new Parameter.IntegerParameter(parameterName1, default1, v -> v > 0)) - .addParameter(parameterName2, new Parameter.IntegerParameter(parameterName2, default2, v -> v > 0)) + .addParameter(parameterName1, new Parameter.IntegerParameter(parameterName1, default1, (v, context) -> v > 0)) + .addParameter(parameterName2, new Parameter.IntegerParameter(parameterName2, default2, (v, context) -> v > 0)) .build(); XContentBuilder xContentBuilder = XContentFactory.jsonBuilder() @@ -128,13 +141,19 @@ public void testGetAsMap_withoutGenerator() throws IOException { Map in = xContentBuilderToMap(xContentBuilder); MethodComponentContext methodComponentContext = MethodComponentContext.parse(in); - assertEquals(in, methodComponent.getAsMap(methodComponentContext)); + assertEquals( + in, + methodComponent.getAsMap(methodComponentContext, KNNMethodConfigContext.builder().versionCreated(Version.CURRENT).build()) + ); xContentBuilder = XContentFactory.jsonBuilder().startObject().field(NAME, methodName).endObject(); in = xContentBuilderToMap(xContentBuilder); methodComponentContext = MethodComponentContext.parse(in); - Map methodAsMap = methodComponent.getAsMap(methodComponentContext); + Map methodAsMap = methodComponent.getAsMap( + methodComponentContext, + KNNMethodConfigContext.builder().versionCreated(Version.CURRENT).build() + ); assertEquals(default1, ((Map) methodAsMap.get(PARAMETERS)).get(parameterName1)); assertEquals(default2, ((Map) methodAsMap.get(PARAMETERS)).get(parameterName2)); } @@ -143,16 +162,19 @@ public void testGetAsMap_withGenerator() throws IOException { String methodName = "test-method"; Map generatedMap = ImmutableMap.of("test-key", "test-value"); MethodComponent methodComponent = MethodComponent.Builder.builder(methodName) - .addParameter("valid1", new Parameter.IntegerParameter("valid1", 1, v -> v > 0)) - .addParameter("valid2", new Parameter.IntegerParameter("valid2", 1, v -> v > 0)) - .setMapGenerator((methodComponent1, methodComponentContext) -> generatedMap) + .addParameter("valid1", new Parameter.IntegerParameter("valid1", 1, (v, context) -> v > 0)) + .addParameter("valid2", new Parameter.IntegerParameter("valid2", 1, (v, context) -> v > 0)) + .setMapGenerator((methodComponent1, methodComponentContext, knnMethodConfigContext) -> generatedMap) .build(); XContentBuilder xContentBuilder = XContentFactory.jsonBuilder().startObject().field(NAME, methodName).endObject(); Map in = xContentBuilderToMap(xContentBuilder); MethodComponentContext methodComponentContext = MethodComponentContext.parse(in); - assertEquals(generatedMap, methodComponent.getAsMap(methodComponentContext)); + assertEquals( + generatedMap, + methodComponent.getAsMap(methodComponentContext, KNNMethodConfigContext.builder().versionCreated(Version.CURRENT).build()) + ); } public void testBuilder() { @@ -163,15 +185,18 @@ public void testBuilder() { assertEquals(0, methodComponent.getParameters().size()); assertEquals(name, methodComponent.getName()); - builder.addParameter("test", new Parameter.IntegerParameter("test", 1, v -> v > 0)); + builder.addParameter("test", new Parameter.IntegerParameter("test", 1, (v, context) -> v > 0)); methodComponent = builder.build(); assertEquals(1, methodComponent.getParameters().size()); Map generatedMap = ImmutableMap.of("test-key", "test-value"); - builder.setMapGenerator((methodComponent1, methodComponentContext) -> generatedMap); + builder.setMapGenerator((methodComponent1, methodComponentContext, knnMethodConfigContext) -> generatedMap); methodComponent = builder.build(); - assertEquals(generatedMap, methodComponent.getAsMap(null)); + assertEquals( + generatedMap, + methodComponent.getAsMap(null, KNNMethodConfigContext.builder().versionCreated(Version.CURRENT).build()) + ); } } diff --git a/src/test/java/org/opensearch/knn/index/engine/ParameterTests.java b/src/test/java/org/opensearch/knn/index/engine/ParameterTests.java index 7224e2824..9f3979314 100644 --- a/src/test/java/org/opensearch/knn/index/engine/ParameterTests.java +++ b/src/test/java/org/opensearch/knn/index/engine/ParameterTests.java @@ -6,14 +6,16 @@ package org.opensearch.knn.index.engine; import com.google.common.collect.ImmutableMap; +import org.opensearch.Version; import org.opensearch.knn.KNNTestCase; import org.opensearch.common.ValidationException; +import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.engine.Parameter.IntegerParameter; import org.opensearch.knn.index.engine.Parameter.StringParameter; import org.opensearch.knn.index.engine.Parameter.MethodComponentContextParameter; -import org.opensearch.knn.training.VectorSpaceInfo; import java.util.Map; +import java.util.Set; public class ParameterTests extends KNNTestCase { /** @@ -21,17 +23,11 @@ public class ParameterTests extends KNNTestCase { */ public void testGetDefaultValue() { String defaultValue = "test-default"; - Parameter parameter = new Parameter("test", defaultValue, v -> true) { + Parameter parameter = new Parameter("test", defaultValue, (v, context) -> true) { @Override - public ValidationException validate(Object value) { + public ValidationException validate(Object value, KNNMethodConfigContext context) { return null; } - - @Override - public ValidationException validateWithData(Object value, VectorSpaceInfo vectorSpaceInfo) { - return null; - } - }; assertEquals(defaultValue, parameter.getDefaultValue()); @@ -41,95 +37,97 @@ public ValidationException validateWithData(Object value, VectorSpaceInfo vector * Test integer parameter validate */ public void testIntegerParameter_validate() { - final IntegerParameter parameter = new IntegerParameter("test", 1, v -> v > 0); - + final IntegerParameter parameter = new IntegerParameter("test", 1, (v, context) -> v > 0); + KNNMethodConfigContext knnMethodConfigContext = KNNMethodConfigContext.builder() + .dimension(1) + .versionCreated(Version.CURRENT) + .vectorDataType(VectorDataType.FLOAT) + .build(); // Invalid type - assertNotNull(parameter.validate("String")); + assertNotNull(parameter.validate("String", knnMethodConfigContext)); // Invalid value - assertNotNull(parameter.validate(-1)); + assertNotNull(parameter.validate(-1, knnMethodConfigContext)); // valid value - assertNull(parameter.validate(12)); + assertNull(parameter.validate(12, knnMethodConfigContext)); } /** * Test integer parameter validate */ - public void testIntegerParameter_validateWithData() { - final IntegerParameter parameter = new IntegerParameter( - "test", - 1, - v -> v > 0, - (v, vectorSpaceInfo) -> v > vectorSpaceInfo.getDimension() - ); + public void testIntegerParameter_validateWithContext() { + final IntegerParameter parameter = new IntegerParameter("test", 1, (v, context) -> v > 0 && v > context.getDimension()); - VectorSpaceInfo testVectorSpaceInfo = new VectorSpaceInfo(0); + KNNMethodConfigContext knnMethodConfigContext = KNNMethodConfigContext.builder().dimension(0).build(); // Invalid type - assertNotNull(parameter.validateWithData("String", testVectorSpaceInfo)); + assertNotNull(parameter.validate("String", knnMethodConfigContext)); // Invalid value - assertNotNull(parameter.validateWithData(-1, testVectorSpaceInfo)); + assertNotNull(parameter.validate(-1, knnMethodConfigContext)); // valid value - assertNull(parameter.validateWithData(12, testVectorSpaceInfo)); + assertNull(parameter.validate(12, knnMethodConfigContext)); } public void testStringParameter_validate() { - final StringParameter parameter = new StringParameter("test_parameter", "default_value", v -> "test".equals(v)); - + final StringParameter parameter = new StringParameter("test_parameter", "default_value", (v, context) -> "test".equals(v)); + KNNMethodConfigContext knnMethodConfigContext = KNNMethodConfigContext.builder() + .dimension(1) + .versionCreated(Version.CURRENT) + .vectorDataType(VectorDataType.FLOAT) + .build(); // Invalid type - assertNotNull(parameter.validate(5)); + assertNotNull(parameter.validate(5, knnMethodConfigContext)); // null - assertNotNull(parameter.validate(null)); + assertNotNull(parameter.validate(null, knnMethodConfigContext)); // valid value - assertNull(parameter.validate("test")); + assertNull(parameter.validate("test", knnMethodConfigContext)); } public void testStringParameter_validateWithData() { - final StringParameter parameter = new StringParameter( - "test_parameter", - "default_value", - v -> "test".equals(v), - (v, vectorSpaceInfo) -> { - if (vectorSpaceInfo.getDimension() > 0) { - return "test".equals(v); - } - return false; + final StringParameter parameter = new StringParameter("test_parameter", "default_value", (v, context) -> { + if (context.getDimension() > 0) { + return "test".equals(v); } - ); + return false; + }); - VectorSpaceInfo testVectorSpaceInfo = new VectorSpaceInfo(1); + KNNMethodConfigContext knnMethodConfigContext = KNNMethodConfigContext.builder().dimension(1).build(); // Invalid type - assertNotNull(parameter.validateWithData(5, testVectorSpaceInfo)); + assertNotNull(parameter.validate(5, knnMethodConfigContext)); // null - assertNotNull(parameter.validateWithData(null, testVectorSpaceInfo)); + assertNotNull(parameter.validate(null, knnMethodConfigContext)); // valid value - assertNull(parameter.validateWithData("test", testVectorSpaceInfo)); + assertNull(parameter.validate("test", knnMethodConfigContext)); - testVectorSpaceInfo.setDimension(0); + knnMethodConfigContext.setDimension(0); // invalid value - assertNotNull(parameter.validateWithData("test", testVectorSpaceInfo)); + assertNotNull(parameter.validate("test", knnMethodConfigContext)); } public void testDoubleParameter_validate() { - final Parameter.DoubleParameter parameter = new Parameter.DoubleParameter("test_parameter", 1.0, v -> v >= 0); - + final Parameter.DoubleParameter parameter = new Parameter.DoubleParameter("test_parameter", 1.0, (v, context) -> v >= 0); + KNNMethodConfigContext knnMethodConfigContext = KNNMethodConfigContext.builder() + .dimension(1) + .versionCreated(Version.CURRENT) + .vectorDataType(VectorDataType.FLOAT) + .build(); // valid value - assertNull(parameter.validate(0.9)); + assertNull(parameter.validate(0.9, knnMethodConfigContext)); // Invalid type - assertNotNull(parameter.validate(true)); + assertNotNull(parameter.validate(true, knnMethodConfigContext)); // Invalid type - assertNotNull(parameter.validate(-1)); + assertNotNull(parameter.validate(-1, knnMethodConfigContext)); } @@ -137,20 +135,19 @@ public void testDoubleParameter_validateWithData() { final Parameter.DoubleParameter parameter = new Parameter.DoubleParameter( "test", 1.0, - v -> v > 0, - (v, vectorSpaceInfo) -> v > vectorSpaceInfo.getDimension() + (v, context) -> v > 0 && v > context.getDimension() ); - VectorSpaceInfo testVectorSpaceInfo = new VectorSpaceInfo(0); + KNNMethodConfigContext knnMethodConfigContext = KNNMethodConfigContext.builder().dimension(0).build(); // Invalid type - assertNotNull(parameter.validateWithData("String", testVectorSpaceInfo)); + assertNotNull(parameter.validate("String", knnMethodConfigContext)); // Invalid value - assertNotNull(parameter.validateWithData(-1, testVectorSpaceInfo)); + assertNotNull(parameter.validate(-1, knnMethodConfigContext)); // valid value - assertNull(parameter.validateWithData(1.2, testVectorSpaceInfo)); + assertNull(parameter.validate(1.2, knnMethodConfigContext)); } public void testMethodComponentContextParameter_validate() { @@ -161,10 +158,17 @@ public void testMethodComponentContextParameter_validate() { Map defaultParameterMap = ImmutableMap.of(parameterKey1, parameterValue1); MethodComponentContext methodComponentContext = new MethodComponentContext(methodComponentName1, defaultParameterMap); + KNNMethodConfigContext knnMethodConfigContext = KNNMethodConfigContext.builder() + .dimension(1) + .versionCreated(Version.CURRENT) + .vectorDataType(VectorDataType.FLOAT) + .build(); + Map methodComponentMap = ImmutableMap.of( methodComponentName1, MethodComponent.Builder.builder(parameterKey1) - .addParameter(parameterKey1, new IntegerParameter(parameterKey1, 1, v -> v > 0)) + .addSupportedDataTypes(Set.of(VectorDataType.FLOAT)) + .addParameter(parameterKey1, new IntegerParameter(parameterKey1, 1, (v, context) -> v > 0)) .build() ); @@ -175,26 +179,26 @@ public void testMethodComponentContextParameter_validate() { ); // Invalid type - assertNotNull(parameter.validate(17)); - assertNotNull(parameter.validate("invalid-value")); + assertNotNull(parameter.validate(17, knnMethodConfigContext)); + assertNotNull(parameter.validate("invalid-value", knnMethodConfigContext)); // Invalid value String invalidMethodComponentName = "invalid-method"; MethodComponentContext invalidMethodComponentContext1 = new MethodComponentContext(invalidMethodComponentName, defaultParameterMap); - assertNotNull(parameter.validate(invalidMethodComponentContext1)); + assertNotNull(parameter.validate(invalidMethodComponentContext1, knnMethodConfigContext)); String invalidParameterKey = "invalid-parameter"; Map invalidParameterMap1 = ImmutableMap.of(invalidParameterKey, parameterValue1); MethodComponentContext invalidMethodComponentContext2 = new MethodComponentContext(methodComponentName1, invalidParameterMap1); - assertNotNull(parameter.validate(invalidMethodComponentContext2)); + assertNotNull(parameter.validate(invalidMethodComponentContext2, knnMethodConfigContext)); String invalidParameterValue = "invalid-value"; Map invalidParameterMap2 = ImmutableMap.of(parameterKey1, invalidParameterValue); MethodComponentContext invalidMethodComponentContext3 = new MethodComponentContext(methodComponentName1, invalidParameterMap2); - assertNotNull(parameter.validate(invalidMethodComponentContext3)); + assertNotNull(parameter.validate(invalidMethodComponentContext3, knnMethodConfigContext)); // valid value - assertNull(parameter.validate(methodComponentContext)); + assertNull(parameter.validate(methodComponentContext, knnMethodConfigContext)); } public void testMethodComponentContextParameter_validateWithData() { @@ -208,10 +212,8 @@ public void testMethodComponentContextParameter_validateWithData() { Map methodComponentMap = ImmutableMap.of( methodComponentName1, MethodComponent.Builder.builder(parameterKey1) - .addParameter( - parameterKey1, - new IntegerParameter(parameterKey1, 1, v -> v > 0, (v, vectorSpaceInfo) -> v > vectorSpaceInfo.getDimension()) - ) + .addSupportedDataTypes(Set.of(VectorDataType.FLOAT)) + .addParameter(parameterKey1, new IntegerParameter(parameterKey1, 1, (v, context) -> v > 0 && v > context.getDimension())) .build() ); @@ -221,29 +223,32 @@ public void testMethodComponentContextParameter_validateWithData() { methodComponentMap ); - VectorSpaceInfo testVectorSpaceInfo = new VectorSpaceInfo(0); + KNNMethodConfigContext knnMethodConfigContext = KNNMethodConfigContext.builder() + .dimension(0) + .vectorDataType(VectorDataType.FLOAT) + .versionCreated(Version.CURRENT) + .build(); // Invalid type - assertNotNull(parameter.validateWithData(17, testVectorSpaceInfo)); - assertNotNull(parameter.validateWithData("invalid-value", testVectorSpaceInfo)); + assertNotNull(parameter.validate("invalid-value", knnMethodConfigContext)); // Invalid value String invalidMethodComponentName = "invalid-method"; MethodComponentContext invalidMethodComponentContext1 = new MethodComponentContext(invalidMethodComponentName, defaultParameterMap); - assertNotNull(parameter.validateWithData(invalidMethodComponentContext1, testVectorSpaceInfo)); + assertNotNull(parameter.validate(invalidMethodComponentContext1, knnMethodConfigContext)); String invalidParameterKey = "invalid-parameter"; Map invalidParameterMap1 = ImmutableMap.of(invalidParameterKey, parameterValue1); MethodComponentContext invalidMethodComponentContext2 = new MethodComponentContext(methodComponentName1, invalidParameterMap1); - assertNotNull(parameter.validateWithData(invalidMethodComponentContext2, testVectorSpaceInfo)); + assertNotNull(parameter.validate(invalidMethodComponentContext2, knnMethodConfigContext)); String invalidParameterValue = "invalid-value"; Map invalidParameterMap2 = ImmutableMap.of(parameterKey1, invalidParameterValue); MethodComponentContext invalidMethodComponentContext3 = new MethodComponentContext(methodComponentName1, invalidParameterMap2); - assertNotNull(parameter.validateWithData(invalidMethodComponentContext3, testVectorSpaceInfo)); + assertNotNull(parameter.validate(invalidMethodComponentContext3, knnMethodConfigContext)); // valid value - assertNull(parameter.validateWithData(methodComponentContext, testVectorSpaceInfo)); + assertNull(parameter.validate(methodComponentContext, knnMethodConfigContext)); } public void testMethodComponentContextParameter_getMethodComponent() { @@ -257,7 +262,7 @@ public void testMethodComponentContextParameter_getMethodComponent() { Map methodComponentMap = ImmutableMap.of( methodComponentName1, MethodComponent.Builder.builder(parameterKey1) - .addParameter(parameterKey1, new IntegerParameter(parameterKey1, 1, v -> v > 0)) + .addParameter(parameterKey1, new IntegerParameter(parameterKey1, 1, (v, context) -> v > 0)) .build() ); diff --git a/src/test/java/org/opensearch/knn/index/engine/faiss/FaissFP16UtilTests.java b/src/test/java/org/opensearch/knn/index/engine/faiss/FaissFP16UtilTests.java new file mode 100644 index 000000000..81afef877 --- /dev/null +++ b/src/test/java/org/opensearch/knn/index/engine/faiss/FaissFP16UtilTests.java @@ -0,0 +1,60 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.engine.faiss; + +import org.opensearch.knn.KNNTestCase; + +import java.util.Locale; + +import static org.opensearch.knn.common.KNNConstants.ENCODER_SQ; +import static org.opensearch.knn.common.KNNConstants.FAISS_SQ_ENCODER_FP16; +import static org.opensearch.knn.common.KNNConstants.FP16_MAX_VALUE; +import static org.opensearch.knn.common.KNNConstants.FP16_MIN_VALUE; +import static org.opensearch.knn.index.engine.faiss.FaissFP16Util.clipVectorValueToFP16Range; +import static org.opensearch.knn.index.engine.faiss.FaissFP16Util.validateFP16VectorValue; + +public class FaissFP16UtilTests extends KNNTestCase { + + public void testValidateFp16VectorValue_outOfRange_throwsException() { + IllegalArgumentException ex = expectThrows(IllegalArgumentException.class, () -> validateFP16VectorValue(65505.25f)); + assertTrue( + ex.getMessage() + .contains( + String.format( + Locale.ROOT, + "encoder name is set as [%s] and type is set as [%s] in index mapping. But, KNN vector values are not within in the FP16 range [%f, %f]", + ENCODER_SQ, + FAISS_SQ_ENCODER_FP16, + FP16_MIN_VALUE, + FP16_MAX_VALUE + ) + ) + ); + + IllegalArgumentException ex1 = expectThrows(IllegalArgumentException.class, () -> validateFP16VectorValue(-65525.65f)); + assertTrue( + ex1.getMessage() + .contains( + String.format( + Locale.ROOT, + "encoder name is set as [%s] and type is set as [%s] in index mapping. But, KNN vector values are not within in the FP16 range [%f, %f]", + ENCODER_SQ, + FAISS_SQ_ENCODER_FP16, + FP16_MIN_VALUE, + FP16_MAX_VALUE + ) + ) + ); + } + + public void testClipVectorValuetoFP16Range_succeed() { + assertEquals(65504.0f, clipVectorValueToFP16Range(65504.10f), 0.0f); + assertEquals(65504.0f, clipVectorValueToFP16Range(1000000.89f), 0.0f); + assertEquals(-65504.0f, clipVectorValueToFP16Range(-65504.10f), 0.0f); + assertEquals(-65504.0f, clipVectorValueToFP16Range(-1000000.89f), 0.0f); + } + +} diff --git a/src/test/java/org/opensearch/knn/index/engine/faiss/FaissTests.java b/src/test/java/org/opensearch/knn/index/engine/faiss/FaissTests.java index af5086491..5dfd6c58c 100644 --- a/src/test/java/org/opensearch/knn/index/engine/faiss/FaissTests.java +++ b/src/test/java/org/opensearch/knn/index/engine/faiss/FaissTests.java @@ -10,6 +10,8 @@ import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.knn.KNNTestCase; +import org.opensearch.knn.index.VectorDataType; +import org.opensearch.knn.index.engine.KNNMethodConfigContext; import org.opensearch.knn.index.engine.KNNMethodContext; import org.opensearch.knn.index.engine.MethodComponent; import org.opensearch.knn.index.engine.MethodComponentContext; @@ -40,6 +42,12 @@ public class FaissTests extends KNNTestCase { public void testGetMethodAsMap_whenMethodIsHNSWFlat_thenCreateCorrectIndexDescription() throws IOException { + KNNMethodConfigContext knnMethodConfigContext = KNNMethodConfigContext.builder() + .versionCreated(org.opensearch.Version.CURRENT) + .dimension(4) + .vectorDataType(VectorDataType.FLOAT) + .build(); + int mParam = 65; String expectedIndexDescription = String.format(Locale.ROOT, "HNSW%d,Flat", mParam); @@ -53,15 +61,20 @@ public void testGetMethodAsMap_whenMethodIsHNSWFlat_thenCreateCorrectIndexDescri .endObject(); Map in = xContentBuilderToMap(xContentBuilder); KNNMethodContext knnMethodContext = KNNMethodContext.parse(in); - knnMethodContext.getMethodComponentContext().setIndexVersion(Version.CURRENT); - Map map = Faiss.INSTANCE.getKNNLibraryIndexingContext(knnMethodContext).getLibraryParameters(); + Map map = Faiss.INSTANCE.getKNNLibraryIndexingContext(knnMethodContext, knnMethodConfigContext) + .getLibraryParameters(); assertTrue(map.containsKey(INDEX_DESCRIPTION_PARAMETER)); assertEquals(expectedIndexDescription, map.get(INDEX_DESCRIPTION_PARAMETER)); } public void testGetMethodAsMap_whenMethodIsHNSWPQ_thenCreateCorrectIndexDescription() throws IOException { + KNNMethodConfigContext knnMethodConfigContext = KNNMethodConfigContext.builder() + .versionCreated(org.opensearch.Version.CURRENT) + .dimension(4) + .vectorDataType(VectorDataType.FLOAT) + .build(); int hnswMParam = 65; int pqMParam = 17; String expectedIndexDescription = String.format(Locale.ROOT, "HNSW%d,PQ%d", hnswMParam, pqMParam); @@ -82,9 +95,9 @@ public void testGetMethodAsMap_whenMethodIsHNSWPQ_thenCreateCorrectIndexDescript .endObject(); Map in = xContentBuilderToMap(xContentBuilder); KNNMethodContext knnMethodContext = KNNMethodContext.parse(in); - knnMethodContext.getMethodComponentContext().setIndexVersion(Version.CURRENT); - Map map = Faiss.INSTANCE.getKNNLibraryIndexingContext(knnMethodContext).getLibraryParameters(); + Map map = Faiss.INSTANCE.getKNNLibraryIndexingContext(knnMethodContext, knnMethodConfigContext) + .getLibraryParameters(); assertTrue(map.containsKey(INDEX_DESCRIPTION_PARAMETER)); assertEquals(expectedIndexDescription, map.get(INDEX_DESCRIPTION_PARAMETER)); @@ -92,6 +105,11 @@ public void testGetMethodAsMap_whenMethodIsHNSWPQ_thenCreateCorrectIndexDescript @SneakyThrows public void testGetMethodAsMap_whenMethodIsHNSWSQFP16_thenCreateCorrectIndexDescription() { + KNNMethodConfigContext knnMethodConfigContext = KNNMethodConfigContext.builder() + .versionCreated(org.opensearch.Version.CURRENT) + .dimension(4) + .vectorDataType(VectorDataType.FLOAT) + .build(); int hnswMParam = 65; String expectedIndexDescription = String.format(Locale.ROOT, "HNSW%d,SQfp16", hnswMParam); @@ -111,15 +129,20 @@ public void testGetMethodAsMap_whenMethodIsHNSWSQFP16_thenCreateCorrectIndexDesc .endObject(); Map in = xContentBuilderToMap(xContentBuilder); KNNMethodContext knnMethodContext = KNNMethodContext.parse(in); - knnMethodContext.getMethodComponentContext().setIndexVersion(Version.CURRENT); - Map map = Faiss.INSTANCE.getKNNLibraryIndexingContext(knnMethodContext).getLibraryParameters(); + Map map = Faiss.INSTANCE.getKNNLibraryIndexingContext(knnMethodContext, knnMethodConfigContext) + .getLibraryParameters(); assertTrue(map.containsKey(INDEX_DESCRIPTION_PARAMETER)); assertEquals(expectedIndexDescription, map.get(INDEX_DESCRIPTION_PARAMETER)); } public void testGetMethodAsMap_whenMethodIsIVFFlat_thenCreateCorrectIndexDescription() throws IOException { + KNNMethodConfigContext knnMethodConfigContext = KNNMethodConfigContext.builder() + .versionCreated(org.opensearch.Version.CURRENT) + .dimension(4) + .vectorDataType(VectorDataType.FLOAT) + .build(); int nlists = 88; String expectedIndexDescription = String.format(Locale.ROOT, "IVF%d,Flat", nlists); @@ -134,13 +157,19 @@ public void testGetMethodAsMap_whenMethodIsIVFFlat_thenCreateCorrectIndexDescrip Map in = xContentBuilderToMap(xContentBuilder); KNNMethodContext knnMethodContext = KNNMethodContext.parse(in); - Map map = Faiss.INSTANCE.getKNNLibraryIndexingContext(knnMethodContext).getLibraryParameters(); + Map map = Faiss.INSTANCE.getKNNLibraryIndexingContext(knnMethodContext, knnMethodConfigContext) + .getLibraryParameters(); assertTrue(map.containsKey(INDEX_DESCRIPTION_PARAMETER)); assertEquals(expectedIndexDescription, map.get(INDEX_DESCRIPTION_PARAMETER)); } public void testGetMethodAsMap_whenMethodIsIVFPQ_thenCreateCorrectIndexDescription() throws IOException { + KNNMethodConfigContext knnMethodConfigContext = KNNMethodConfigContext.builder() + .versionCreated(org.opensearch.Version.CURRENT) + .dimension(4) + .vectorDataType(VectorDataType.FLOAT) + .build(); int ivfNlistsParam = 88; int pqMParam = 17; int pqCodeSizeParam = 53; @@ -164,7 +193,8 @@ public void testGetMethodAsMap_whenMethodIsIVFPQ_thenCreateCorrectIndexDescripti Map in = xContentBuilderToMap(xContentBuilder); KNNMethodContext knnMethodContext = KNNMethodContext.parse(in); - Map map = Faiss.INSTANCE.getKNNLibraryIndexingContext(knnMethodContext).getLibraryParameters(); + Map map = Faiss.INSTANCE.getKNNLibraryIndexingContext(knnMethodContext, knnMethodConfigContext) + .getLibraryParameters(); assertTrue(map.containsKey(INDEX_DESCRIPTION_PARAMETER)); assertEquals(expectedIndexDescription, map.get(INDEX_DESCRIPTION_PARAMETER)); @@ -172,6 +202,11 @@ public void testGetMethodAsMap_whenMethodIsIVFPQ_thenCreateCorrectIndexDescripti @SneakyThrows public void testGetMethodAsMap_whenMethodIsIVFSQFP16_thenCreateCorrectIndexDescription() { + KNNMethodConfigContext knnMethodConfigContext = KNNMethodConfigContext.builder() + .versionCreated(org.opensearch.Version.CURRENT) + .dimension(4) + .vectorDataType(VectorDataType.FLOAT) + .build(); int nlists = 88; String expectedIndexDescription = String.format(Locale.ROOT, "IVF%d,SQfp16", nlists); @@ -192,7 +227,8 @@ public void testGetMethodAsMap_whenMethodIsIVFSQFP16_thenCreateCorrectIndexDescr Map in = xContentBuilderToMap(xContentBuilder); KNNMethodContext knnMethodContext = KNNMethodContext.parse(in); - Map map = Faiss.INSTANCE.getKNNLibraryIndexingContext(knnMethodContext).getLibraryParameters(); + Map map = Faiss.INSTANCE.getKNNLibraryIndexingContext(knnMethodContext, knnMethodConfigContext) + .getLibraryParameters(); assertTrue(map.containsKey(INDEX_DESCRIPTION_PARAMETER)); assertEquals(expectedIndexDescription, map.get(INDEX_DESCRIPTION_PARAMETER)); @@ -210,9 +246,9 @@ public void testMethodAsMapBuilder() throws IOException { String parameter3 = "test-parameter-3"; Integer defaultValue3 = 3; MethodComponent methodComponent = MethodComponent.Builder.builder(methodName) - .addParameter(parameter1, new Parameter.IntegerParameter(parameter1, defaultValue1, value -> value > 0)) - .addParameter(parameter2, new Parameter.IntegerParameter(parameter2, defaultValue2, value -> value > 0)) - .addParameter(parameter3, new Parameter.IntegerParameter(parameter3, defaultValue3, value -> value > 0)) + .addParameter(parameter1, new Parameter.IntegerParameter(parameter1, defaultValue1, (value, context) -> value > 0)) + .addParameter(parameter2, new Parameter.IntegerParameter(parameter2, defaultValue2, (value, context) -> value > 0)) + .addParameter(parameter3, new Parameter.IntegerParameter(parameter3, defaultValue3, (value, context) -> value > 0)) .build(); XContentBuilder xContentBuilder = XContentFactory.jsonBuilder() @@ -234,9 +270,12 @@ public void testMethodAsMapBuilder() throws IOException { expectedMap.put(NAME, methodName); expectedMap.put(INDEX_DESCRIPTION_PARAMETER, methodDescription + value1); - Map methodAsMap = MethodAsMapBuilder.builder(methodDescription, methodComponent, methodComponentContext) - .addParameter(parameter1, "", "") - .build(); + Map methodAsMap = MethodAsMapBuilder.builder( + methodDescription, + methodComponent, + methodComponentContext, + KNNMethodConfigContext.builder().versionCreated(Version.CURRENT).build() + ).addParameter(parameter1, "", "").build(); assertEquals(expectedMap, methodAsMap); } diff --git a/src/test/java/org/opensearch/knn/index/engine/lucene/LuceneTests.java b/src/test/java/org/opensearch/knn/index/engine/lucene/LuceneTests.java index d149f4b3b..2d2025d49 100644 --- a/src/test/java/org/opensearch/knn/index/engine/lucene/LuceneTests.java +++ b/src/test/java/org/opensearch/knn/index/engine/lucene/LuceneTests.java @@ -9,7 +9,9 @@ import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.knn.KNNTestCase; +import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.engine.KNNEngine; +import org.opensearch.knn.index.engine.KNNMethodConfigContext; import org.opensearch.knn.index.engine.KNNMethodContext; import org.opensearch.knn.index.SpaceType; @@ -28,6 +30,11 @@ public class LuceneTests extends KNNTestCase { public void testLucenHNSWMethod() throws IOException { + KNNMethodConfigContext knnMethodConfigContext = KNNMethodConfigContext.builder() + .versionCreated(org.opensearch.Version.CURRENT) + .dimension(10) + .vectorDataType(VectorDataType.FLOAT) + .build(); int efConstruction = 100; int m = 17; XContentBuilder xContentBuilder = XContentFactory.jsonBuilder() @@ -41,7 +48,7 @@ public void testLucenHNSWMethod() throws IOException { .endObject(); Map in = xContentBuilderToMap(xContentBuilder); KNNMethodContext knnMethodContext1 = KNNMethodContext.parse(in); - assertNull(KNNEngine.LUCENE.validateMethod(knnMethodContext1)); + assertNull(KNNEngine.LUCENE.validateMethod(knnMethodContext1, knnMethodConfigContext)); // Invalid parameter String invalidParameter = "invalid"; @@ -54,7 +61,8 @@ public void testLucenHNSWMethod() throws IOException { .endObject(); in = xContentBuilderToMap(xContentBuilder); KNNMethodContext knnMethodContext2 = KNNMethodContext.parse(in); - assertNotNull(KNNEngine.LUCENE.validateMethod(knnMethodContext2)); + knnMethodContext2.setSpaceType(SpaceType.L2); + assertNotNull(KNNEngine.LUCENE.validateMethod(knnMethodContext2, knnMethodConfigContext)); // Valid parameter, invalid value int invalidEfConstruction = -1; @@ -67,7 +75,8 @@ public void testLucenHNSWMethod() throws IOException { .endObject(); in = xContentBuilderToMap(xContentBuilder); KNNMethodContext knnMethodContext3 = KNNMethodContext.parse(in); - assertNotNull(KNNEngine.LUCENE.validateMethod(knnMethodContext3)); + knnMethodContext3.setSpaceType(SpaceType.L2); + assertNotNull(KNNEngine.LUCENE.validateMethod(knnMethodContext3, knnMethodConfigContext)); // Unsupported space type SpaceType invalidSpaceType = SpaceType.LINF; // Not currently supported @@ -78,7 +87,7 @@ public void testLucenHNSWMethod() throws IOException { .endObject(); in = xContentBuilderToMap(xContentBuilder); KNNMethodContext knnMethodContext4 = KNNMethodContext.parse(in); - assertNotNull(KNNEngine.LUCENE.validateMethod(knnMethodContext4)); + assertNotNull(KNNEngine.LUCENE.validateMethod(knnMethodContext4, knnMethodConfigContext)); // Check INNER_PRODUCT is supported with Lucene Engine xContentBuilder = XContentFactory.jsonBuilder() @@ -92,7 +101,7 @@ public void testLucenHNSWMethod() throws IOException { .endObject(); in = xContentBuilderToMap(xContentBuilder); KNNMethodContext knnMethodContext5 = KNNMethodContext.parse(in); - assertNull(KNNEngine.LUCENE.validateMethod(knnMethodContext5)); + assertNull(KNNEngine.LUCENE.validateMethod(knnMethodContext5, knnMethodConfigContext)); } public void testGetExtension() { diff --git a/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java b/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java index b3139fa5c..4034e4cb6 100644 --- a/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java +++ b/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java @@ -5,7 +5,6 @@ package org.opensearch.knn.index.mapper; -import com.google.common.collect.ImmutableMap; import lombok.SneakyThrows; import lombok.extern.log4j.Log4j2; import org.apache.lucene.document.KnnByteVectorField; @@ -37,6 +36,7 @@ import org.opensearch.knn.index.VectorField; import org.opensearch.knn.index.codec.util.KNNVectorSerializerFactory; import org.opensearch.knn.index.engine.KNNEngine; +import org.opensearch.knn.index.engine.KNNMethodConfigContext; import org.opensearch.knn.index.engine.KNNMethodContext; import org.opensearch.knn.index.engine.MethodComponentContext; import org.opensearch.knn.indices.ModelDao; @@ -51,7 +51,6 @@ import java.util.HashSet; import java.util.List; import java.util.Locale; -import java.util.Map; import java.util.Optional; import java.util.stream.Collectors; @@ -62,9 +61,6 @@ import static org.opensearch.Version.CURRENT; import static org.opensearch.knn.common.KNNConstants.DIMENSION; import static org.opensearch.knn.common.KNNConstants.ENCODER_SQ; -import static org.opensearch.knn.common.KNNConstants.FAISS_SQ_ENCODER_FP16; -import static org.opensearch.knn.common.KNNConstants.FP16_MAX_VALUE; -import static org.opensearch.knn.common.KNNConstants.FP16_MIN_VALUE; import static org.opensearch.knn.common.KNNConstants.KNN_ENGINE; import static org.opensearch.knn.common.KNNConstants.KNN_METHOD; import static org.opensearch.knn.common.KNNConstants.LUCENE_NAME; @@ -80,8 +76,6 @@ import static org.opensearch.knn.common.KNNConstants.VECTOR_DATA_TYPE_FIELD; import static org.opensearch.knn.index.KNNSettings.KNN_INDEX; import static org.opensearch.knn.index.VectorDataType.SUPPORTED_VECTOR_DATA_TYPES; -import static org.opensearch.knn.index.mapper.KNNVectorFieldMapperUtil.clipVectorValueToFP16Range; -import static org.opensearch.knn.index.mapper.KNNVectorFieldMapperUtil.validateFP16VectorValue; @Log4j2 public class KNNVectorFieldMapperTests extends KNNTestCase { @@ -108,7 +102,7 @@ public class KNNVectorFieldMapperTests extends KNNTestCase { public void testBuilder_getParameters() { String fieldName = "test-field-name"; ModelDao modelDao = mock(ModelDao.class); - KNNVectorFieldMapper.Builder builder = new KNNVectorFieldMapper.Builder(fieldName, modelDao, CURRENT, null); + KNNVectorFieldMapper.Builder builder = new KNNVectorFieldMapper.Builder(fieldName, modelDao, CURRENT, null, null); assertEquals(7, builder.getParameters().size()); List actualParams = builder.getParameters().stream().map(a -> a.name).collect(Collectors.toList()); @@ -116,46 +110,64 @@ public void testBuilder_getParameters() { assertEquals(expectedParams, actualParams); } - public void testBuilder_build_fromKnnMethodContext() { + public void testTypeParser_build_fromKnnMethodContext() throws IOException { // Check that knnMethodContext takes precedent over both model and legacy ModelDao modelDao = mock(ModelDao.class); - KNNVectorFieldMapper.Builder builder = new KNNVectorFieldMapper.Builder("test-field-name-1", modelDao, CURRENT, null); SpaceType spaceType = SpaceType.COSINESIMIL; - int m = 17; - int efConstruction = 17; + int mRight = 17; + int mWrong = 71; + + KNNVectorFieldMapper.TypeParser typeParser = new KNNVectorFieldMapper.TypeParser(() -> modelDao); + + XContentBuilder xContentBuilder = XContentFactory.jsonBuilder() + .startObject() + .field(TYPE_FIELD_NAME, KNN_VECTOR_TYPE) + .field(DIMENSION_FIELD_NAME, TEST_DIMENSION) + .startObject(KNN_METHOD) + .field(NAME, METHOD_HNSW) + .field(METHOD_PARAMETER_SPACE_TYPE, spaceType) + .startObject(PARAMETERS) + .field(METHOD_PARAMETER_M, mRight) + .endObject() + .endObject() + .endObject(); // Setup settings Settings settings = Settings.builder() .put(settings(CURRENT).build()) - .put(KNNSettings.KNN_SPACE_TYPE, spaceType.getValue()) - .put(KNNSettings.KNN_ALGO_PARAM_M, m) - .put(KNNSettings.KNN_ALGO_PARAM_EF_CONSTRUCTION, efConstruction) + .put(KNNSettings.KNN_ALGO_PARAM_M, mWrong) .put(KNN_INDEX, true) .build(); - builder.knnMethodContext.setValue( - new KNNMethodContext( - KNNEngine.DEFAULT, - spaceType, - new MethodComponentContext( - METHOD_HNSW, - ImmutableMap.of(METHOD_PARAMETER_M, m, METHOD_PARAMETER_EF_CONSTRUCTION, efConstruction) - ) - ) + KNNVectorFieldMapper.Builder builder = (KNNVectorFieldMapper.Builder) typeParser.parse( + "test-field-name-1", + xContentBuilderToMap(xContentBuilder), + buildParserContext("test", settings) ); Mapper.BuilderContext builderContext = new Mapper.BuilderContext(settings, new ContentPath()); KNNVectorFieldMapper knnVectorFieldMapper = builder.build(builderContext); assertTrue(knnVectorFieldMapper instanceof MethodFieldMapper); assertTrue(knnVectorFieldMapper.fieldType().getKnnMappingConfig().getKnnMethodContext().isPresent()); + assertEquals(spaceType, knnVectorFieldMapper.fieldType().getKnnMappingConfig().getKnnMethodContext().get().getSpaceType()); + assertEquals( + mRight, + knnVectorFieldMapper.fieldType() + .getKnnMappingConfig() + .getKnnMethodContext() + .get() + .getMethodComponentContext() + .getParameters() + .get(METHOD_PARAMETER_M) + ); assertTrue(knnVectorFieldMapper.fieldType().getKnnMappingConfig().getModelId().isEmpty()); } public void testBuilder_build_fromModel() { // Check that modelContext takes precedent over legacy ModelDao modelDao = mock(ModelDao.class); - KNNVectorFieldMapper.Builder builder = new KNNVectorFieldMapper.Builder("test-field-name-1", modelDao, CURRENT, null); + KNNVectorFieldMapper.Builder builder = new KNNVectorFieldMapper.Builder("test-field-name-1", modelDao, CURRENT, null, null); SpaceType spaceType = SpaceType.COSINESIMIL; int m = 17; @@ -193,15 +205,19 @@ public void testBuilder_build_fromModel() { assertTrue(knnVectorFieldMapper.fieldType().getKnnMappingConfig().getKnnMethodContext().isEmpty()); } - public void testBuilder_build_fromLegacy() { + public void testBuilder_build_fromLegacy() throws IOException { // Check legacy is picked up if model context and method context are not set ModelDao modelDao = mock(ModelDao.class); - KNNVectorFieldMapper.Builder builder = new KNNVectorFieldMapper.Builder("test-field-name-1", modelDao, CURRENT, null); - int m = 17; int efConstruction = 17; + KNNVectorFieldMapper.TypeParser typeParser = new KNNVectorFieldMapper.TypeParser(() -> modelDao); + + XContentBuilder xContentBuilder = XContentFactory.jsonBuilder() + .startObject() + .field(TYPE_FIELD_NAME, KNN_VECTOR_TYPE) + .field(DIMENSION_FIELD_NAME, 12) + .endObject(); - // Setup settings Settings settings = Settings.builder() .put(settings(CURRENT).build()) .put(KNNSettings.KNN_ALGO_PARAM_M, m) @@ -209,6 +225,13 @@ public void testBuilder_build_fromLegacy() { .put(KNN_INDEX, true) .build(); + KNNVectorFieldMapper.Builder builder = (KNNVectorFieldMapper.Builder) typeParser.parse( + "test-field-name-1", + xContentBuilderToMap(xContentBuilder), + buildParserContext("test", settings) + ); + + // Setup settings Mapper.BuilderContext builderContext = new Mapper.BuilderContext(settings, new ContentPath()); KNNVectorFieldMapper knnVectorFieldMapper = builder.build(builderContext); assertTrue(knnVectorFieldMapper instanceof MethodFieldMapper); @@ -327,17 +350,16 @@ public void testTypeParser_parse_fromKnnMethodContext_invalidDimension() throws .endObject() .endObject() .endObject(); - KNNVectorFieldMapper.Builder builderOverMaxDimension = (KNNVectorFieldMapper.Builder) typeParser.parse( - fieldName, - xContentBuilderToMap(xContentBuilderOverMaxDimension), - buildParserContext(indexName, settings) - ); IllegalArgumentException ex = expectThrows( IllegalArgumentException.class, - () -> builderOverMaxDimension.build(new Mapper.BuilderContext(settings, new ContentPath())) + () -> typeParser.parse( + fieldName, + xContentBuilderToMap(xContentBuilderOverMaxDimension), + buildParserContext(indexName, settings) + ) ); - assertEquals("Dimension value cannot be greater than 16000 for vector: test-field-name", ex.getMessage()); + assertTrue(ex.getMessage().contains("Dimension value cannot be greater than 16000 for vector with engine: lucene")); XContentBuilder xContentBuilderInvalidDimension = XContentFactory.jsonBuilder() .startObject() @@ -417,7 +439,7 @@ public void testTypeParser_parse_fromKnnMethodContext_invalidSpaceType() throws String fieldName = "test-field-name"; String indexName = "test-index-name"; - Settings settings = Settings.builder().put(settings(CURRENT).build()).build(); + Settings settings = Settings.builder().put(settings(CURRENT).build()).put(KNN_INDEX, true).build(); ModelDao modelDao = mock(ModelDao.class); KNNVectorFieldMapper.TypeParser typeParser = new KNNVectorFieldMapper.TypeParser(() -> modelDao); @@ -449,7 +471,7 @@ public void testTypeParser_parse_fromKnnMethodContext() throws IOException { String fieldName = "test-field-name"; String indexName = "test-index-name"; - Settings settings = Settings.builder().put(settings(CURRENT).build()).build(); + Settings settings = Settings.builder().put(settings(CURRENT).build()).put(KNN_INDEX, true).build(); ModelDao modelDao = mock(ModelDao.class); KNNVectorFieldMapper.TypeParser typeParser = new KNNVectorFieldMapper.TypeParser(() -> modelDao); @@ -546,7 +568,7 @@ public void testTypeParser_parse_fromModel() throws IOException { String fieldName = "test-field-name"; String indexName = "test-index-name"; - Settings settings = Settings.builder().put(settings(CURRENT).build()).build(); + Settings settings = Settings.builder().put(settings(CURRENT).build()).put(KNN_INDEX, true).build(); ModelDao modelDao = mock(ModelDao.class); KNNVectorFieldMapper.TypeParser typeParser = new KNNVectorFieldMapper.TypeParser(() -> modelDao); @@ -751,8 +773,12 @@ public void testMethodFieldMapperParseCreateField_validInput_thenDifferentFieldT log.info("Vector Data Type is : {}", dataType); int dimension = dataType == VectorDataType.BINARY ? TEST_DIMENSION * 8 : TEST_DIMENSION; final MethodComponentContext methodComponentContext = new MethodComponentContext(METHOD_HNSW, Collections.emptyMap()); - methodComponentContext.setIndexVersion(CURRENT); SpaceType spaceType = VectorDataType.BINARY == dataType ? SpaceType.DEFAULT_BINARY : SpaceType.INNER_PRODUCT; + KNNMethodConfigContext knnMethodConfigContext = KNNMethodConfigContext.builder() + .vectorDataType(dataType) + .versionCreated(CURRENT) + .dimension(dimension) + .build(); final KNNMethodContext knnMethodContext = new KNNMethodContext(KNNEngine.FAISS, spaceType, methodComponentContext); ParseContext.Document document = new ParseContext.Document(); @@ -767,16 +793,14 @@ public void testMethodFieldMapperParseCreateField_validInput_thenDifferentFieldT TEST_FIELD_NAME, TEST_FIELD_NAME, Collections.emptyMap(), - dataType, - dimension, knnMethodContext, + knnMethodConfigContext, knnMethodContext, FieldMapper.MultiFields.empty(), FieldMapper.CopyTo.empty(), new Explicit<>(true, true), false, - false, - CURRENT + false ) ); @@ -819,16 +843,14 @@ public void testMethodFieldMapperParseCreateField_validInput_thenDifferentFieldT TEST_FIELD_NAME, TEST_FIELD_NAME, Collections.emptyMap(), - dataType, - dimension, knnMethodContext, + knnMethodConfigContext, knnMethodContext, FieldMapper.MultiFields.empty(), FieldMapper.CopyTo.empty(), new Explicit<>(true, true), false, - false, - CURRENT + false ) ); @@ -854,21 +876,24 @@ public void testMethodFieldMapperParseCreateField_validInput_thenDifferentFieldT public void testLuceneFieldMapper_parseCreateField_docValues_withFloats() { // Create a lucene field mapper that creates a binary doc values field as well as KnnVectorField LuceneFieldMapper.CreateLuceneFieldMapperInput.CreateLuceneFieldMapperInputBuilder inputBuilder = - createLuceneFieldMapperInputBuilder(VectorDataType.FLOAT); + createLuceneFieldMapperInputBuilder(); ParseContext.Document document = new ParseContext.Document(); ContentPath contentPath = new ContentPath(); ParseContext parseContext = mock(ParseContext.class); when(parseContext.doc()).thenReturn(document); when(parseContext.path()).thenReturn(contentPath); - + KNNMethodConfigContext knnMethodConfigContext = KNNMethodConfigContext.builder() + .vectorDataType(VectorDataType.FLOAT) + .versionCreated(CURRENT) + .dimension(TEST_DIMENSION) + .build(); LuceneFieldMapper luceneFieldMapper = Mockito.spy( LuceneFieldMapper.createFieldMapper( TEST_FIELD_NAME, Collections.emptyMap(), - VectorDataType.FLOAT, - TEST_DIMENSION, getDefaultKNNMethodContext(), + knnMethodConfigContext, inputBuilder.build() ) ); @@ -908,18 +933,19 @@ public void testLuceneFieldMapper_parseCreateField_docValues_withFloats() { inputBuilder.hasDocValues(false); - KNNMethodContext knnMethodContext = new KNNMethodContext( - KNNEngine.LUCENE, - SpaceType.DEFAULT, - new MethodComponentContext(METHOD_HNSW, Collections.emptyMap()) - ); + knnMethodConfigContext = KNNMethodConfigContext.builder() + .vectorDataType(VectorDataType.FLOAT) + .versionCreated(CURRENT) + .dimension(TEST_DIMENSION) + .build(); + MethodComponentContext methodComponentContext = new MethodComponentContext(METHOD_HNSW, Collections.emptyMap()); + KNNMethodContext knnMethodContext = new KNNMethodContext(KNNEngine.LUCENE, SpaceType.DEFAULT, methodComponentContext); luceneFieldMapper = Mockito.spy( LuceneFieldMapper.createFieldMapper( TEST_FIELD_NAME, Collections.emptyMap(), - VectorDataType.FLOAT, - TEST_DIMENSION, knnMethodContext, + knnMethodConfigContext, inputBuilder.build() ) ); @@ -942,7 +968,7 @@ public void testLuceneFieldMapper_parseCreateField_docValues_withBytes() { // Create a lucene field mapper that creates a binary doc values field as well as KnnByteVectorField LuceneFieldMapper.CreateLuceneFieldMapperInput.CreateLuceneFieldMapperInputBuilder inputBuilder = - createLuceneFieldMapperInputBuilder(VectorDataType.BYTE); + createLuceneFieldMapperInputBuilder(); ParseContext.Document document = new ParseContext.Document(); ContentPath contentPath = new ContentPath(); @@ -954,9 +980,12 @@ public void testLuceneFieldMapper_parseCreateField_docValues_withBytes() { LuceneFieldMapper.createFieldMapper( TEST_FIELD_NAME, Collections.emptyMap(), - VectorDataType.BYTE, - TEST_DIMENSION, - getDefaultKNNMethodContext(), + getDefaultByteKNNMethodContext(), + KNNMethodConfigContext.builder() + .vectorDataType(VectorDataType.BYTE) + .versionCreated(CURRENT) + .dimension(TEST_DIMENSION) + .build(), inputBuilder.build() ) ); @@ -1000,9 +1029,12 @@ public void testLuceneFieldMapper_parseCreateField_docValues_withBytes() { LuceneFieldMapper.createFieldMapper( TEST_FIELD_NAME, Collections.emptyMap(), - VectorDataType.BYTE, - TEST_DIMENSION, - getDefaultKNNMethodContext(), + getDefaultByteKNNMethodContext(), + KNNMethodConfigContext.builder() + .vectorDataType(VectorDataType.BYTE) + .versionCreated(CURRENT) + .dimension(TEST_DIMENSION) + .build(), inputBuilder.build() ) ); @@ -1021,131 +1053,105 @@ public void testLuceneFieldMapper_parseCreateField_docValues_withBytes() { assertArrayEquals(TEST_BYTE_VECTOR, knnByteVectorField.vectorValue()); } - public void testValidateFp16VectorValue_outOfRange_throwsException() { - IllegalArgumentException ex = expectThrows(IllegalArgumentException.class, () -> validateFP16VectorValue(65505.25f)); - assertTrue( - ex.getMessage() - .contains( - String.format( - Locale.ROOT, - "encoder name is set as [%s] and type is set as [%s] in index mapping. But, KNN vector values are not within in the FP16 range [%f, %f]", - ENCODER_SQ, - FAISS_SQ_ENCODER_FP16, - FP16_MIN_VALUE, - FP16_MAX_VALUE - ) - ) - ); - - IllegalArgumentException ex1 = expectThrows(IllegalArgumentException.class, () -> validateFP16VectorValue(-65525.65f)); - assertTrue( - ex1.getMessage() - .contains( - String.format( - Locale.ROOT, - "encoder name is set as [%s] and type is set as [%s] in index mapping. But, KNN vector values are not within in the FP16 range [%f, %f]", - ENCODER_SQ, - FAISS_SQ_ENCODER_FP16, - FP16_MIN_VALUE, - FP16_MAX_VALUE - ) - ) - ); - } - - public void testClipVectorValuetoFP16Range_succeed() { - assertEquals(65504.0f, clipVectorValueToFP16Range(65504.10f), 0.0f); - assertEquals(65504.0f, clipVectorValueToFP16Range(1000000.89f), 0.0f); - assertEquals(-65504.0f, clipVectorValueToFP16Range(-65504.10f), 0.0f); - assertEquals(-65504.0f, clipVectorValueToFP16Range(-1000000.89f), 0.0f); + public void testTypeParser_whenBinaryFaissHNSW_thenValid() throws IOException { + testTypeParserWithBinaryDataType(KNNEngine.FAISS, SpaceType.HAMMING, METHOD_HNSW, 8, null); } - public void testBuilder_whenBinaryFaissHNSW_thenValid() { - testBuilderWithBinaryDataType(KNNEngine.FAISS, SpaceType.UNDEFINED, METHOD_HNSW, 8, null); + public void testTypeParser_whenBinaryWithInvalidDimension_thenException() throws IOException { + testTypeParserWithBinaryDataType(KNNEngine.FAISS, SpaceType.UNDEFINED, METHOD_HNSW, 4, "should be multiply of 8"); } - public void testBuilder_whenBinaryWithInvalidDimension_thenException() { - testBuilderWithBinaryDataType(KNNEngine.FAISS, SpaceType.UNDEFINED, METHOD_HNSW, 4, "should be multiply of 8"); - } - - public void testBuilder_whenBinaryFaissHNSWWithInvalidSpaceType_thenException() { + public void testTypeParser_whenBinaryFaissHNSWWithInvalidSpaceType_thenException() throws IOException { for (SpaceType spaceType : SpaceType.values()) { if (SpaceType.UNDEFINED == spaceType || SpaceType.HAMMING == spaceType) { continue; } - testBuilderWithBinaryDataType(KNNEngine.FAISS, spaceType, METHOD_HNSW, 8, "is not supported"); + testTypeParserWithBinaryDataType(KNNEngine.FAISS, spaceType, METHOD_HNSW, 8, "is not supported with"); } } - public void testBuilder_whenBinaryNonFaiss_thenException() { - testBuilderWithBinaryDataType(KNNEngine.LUCENE, SpaceType.UNDEFINED, METHOD_HNSW, 8, "is only supported for"); - testBuilderWithBinaryDataType(KNNEngine.NMSLIB, SpaceType.UNDEFINED, METHOD_HNSW, 8, "is only supported for"); + public void testTypeParser_whenBinaryNonFaiss_thenException() throws IOException { + testTypeParserWithBinaryDataType(KNNEngine.LUCENE, SpaceType.UNDEFINED, METHOD_HNSW, 8, "is not supported for vector data type"); + testTypeParserWithBinaryDataType(KNNEngine.NMSLIB, SpaceType.UNDEFINED, METHOD_HNSW, 8, "is not supported for vector data type"); } - private void testBuilderWithBinaryDataType( + private void testTypeParserWithBinaryDataType( KNNEngine knnEngine, SpaceType spaceType, String method, int dimension, String expectedErrMsg - ) { + ) throws IOException { + // Check legacy is picked up if model context and method context are not set ModelDao modelDao = mock(ModelDao.class); - KNNVectorFieldMapper.Builder builder = new KNNVectorFieldMapper.Builder("test-field-name-1", modelDao, CURRENT, null); + KNNVectorFieldMapper.TypeParser typeParser = new KNNVectorFieldMapper.TypeParser(() -> modelDao); + String fieldName = "test-field-name-1"; + String indexName = "test-index"; // Setup settings Settings settings = Settings.builder().put(settings(CURRENT).build()).put(KNN_INDEX, true).build(); - builder.knnMethodContext.setValue( - new KNNMethodContext(knnEngine, spaceType, new MethodComponentContext(method, Collections.emptyMap())) - ); - builder.vectorDataType.setValue(VectorDataType.BINARY); - builder.dimension.setValue(dimension); + XContentBuilder xContentBuilder = XContentFactory.jsonBuilder() + .startObject() + .field(TYPE_FIELD_NAME, KNN_VECTOR_TYPE) + .field(DIMENSION_FIELD_NAME, dimension) + .field(VECTOR_DATA_TYPE_FIELD, VectorDataType.BINARY.getValue()) + .startObject(KNN_METHOD) + .field(NAME, METHOD_HNSW) + .field(METHOD_PARAMETER_SPACE_TYPE, spaceType.getValue()) + .field(KNN_ENGINE, knnEngine.getName()) + .endObject() + .endObject(); - Mapper.BuilderContext builderContext = new Mapper.BuilderContext(settings, new ContentPath()); if (expectedErrMsg == null) { - KNNVectorFieldMapper knnVectorFieldMapper = builder.build(builderContext); - assertTrue(knnVectorFieldMapper instanceof MethodFieldMapper); - if (SpaceType.UNDEFINED == spaceType) { - assertEquals( - SpaceType.HAMMING, - knnVectorFieldMapper.fieldType().getKnnMappingConfig().getKnnMethodContext().get().getSpaceType() - ); - } + KNNVectorFieldMapper.Builder builder = (KNNVectorFieldMapper.Builder) typeParser.parse( + fieldName, + xContentBuilderToMap(xContentBuilder), + buildParserContext(indexName, settings) + ); + + assertEquals(spaceType, builder.getResolvedKNNMethodContext().getSpaceType()); } else { - Exception ex = expectThrows(Exception.class, () -> builder.build(builderContext)); + Exception ex = expectThrows(Exception.class, () -> { + typeParser.parse(fieldName, xContentBuilderToMap(xContentBuilder), buildParserContext(indexName, settings)); + }); assertTrue(ex.getMessage(), ex.getMessage().contains(expectedErrMsg)); } } - public void testBuilder_whenBinaryFaissHNSWWithSQ_thenException() { + public void testTypeParser_whenBinaryFaissHNSWWithSQ_thenException() throws IOException { ModelDao modelDao = mock(ModelDao.class); - KNNVectorFieldMapper.Builder builder = new KNNVectorFieldMapper.Builder("test-field-name-1", modelDao, CURRENT, null); - + KNNVectorFieldMapper.TypeParser typeParser = new KNNVectorFieldMapper.TypeParser(() -> modelDao); // Setup settings Settings settings = Settings.builder().put(settings(CURRENT).build()).put(KNN_INDEX, true).build(); - builder.knnMethodContext.setValue( - new KNNMethodContext( - KNNEngine.FAISS, - SpaceType.HAMMING, - new MethodComponentContext( - METHOD_HNSW, - Map.of(METHOD_ENCODER_PARAMETER, new MethodComponentContext(ENCODER_SQ, Collections.emptyMap())) - ) - ) - ); - builder.vectorDataType.setValue(VectorDataType.BINARY); - builder.dimension.setValue(8); + XContentBuilder xContentBuilder = XContentFactory.jsonBuilder() + .startObject() + .field(TYPE_FIELD_NAME, KNN_VECTOR_TYPE) + .field(DIMENSION_FIELD_NAME, 8) + .field(VECTOR_DATA_TYPE_FIELD, VectorDataType.BINARY.getValue()) + .startObject(KNN_METHOD) + .field(NAME, METHOD_HNSW) + .field(KNN_ENGINE, KNNEngine.FAISS.getName()) + .startObject(PARAMETERS) + .startObject(METHOD_ENCODER_PARAMETER) + .field(NAME, ENCODER_SQ) + .endObject() + .endObject() + .endObject() + .endObject(); - Mapper.BuilderContext builderContext = new Mapper.BuilderContext(settings, new ContentPath()); - Exception ex = expectThrows(Exception.class, () -> builder.build(builderContext)); - assertTrue(ex.getMessage(), ex.getMessage().contains("data type does not support")); + Exception ex = expectThrows( + Exception.class, + () -> typeParser.parse("test", xContentBuilderToMap(xContentBuilder), buildParserContext("test", settings)) + ); + assertTrue(ex.getMessage(), ex.getMessage().contains("parameter validation failed for MethodComponentContext parameter [encoder]")); } public void testBuilder_whenBinaryWithLegacyKNNDisabled_thenValid() { // Check legacy is picked up if model context and method context are not set ModelDao modelDao = mock(ModelDao.class); - KNNVectorFieldMapper.Builder builder = new KNNVectorFieldMapper.Builder("test-field-name-1", modelDao, CURRENT, null); + KNNVectorFieldMapper.Builder builder = new KNNVectorFieldMapper.Builder("test-field-name-1", modelDao, CURRENT, null, null); builder.vectorDataType.setValue(VectorDataType.BINARY); builder.dimension.setValue(8); @@ -1157,19 +1163,28 @@ public void testBuilder_whenBinaryWithLegacyKNNDisabled_thenValid() { assertTrue(knnVectorFieldMapper instanceof FlatVectorFieldMapper); } - public void testBuilder_whenBinaryWithLegacyKNNEnabled_thenException() { + public void testTypeParser_whenBinaryWithLegacyKNNEnabled_thenException() throws IOException { // Check legacy is picked up if model context and method context are not set ModelDao modelDao = mock(ModelDao.class); - KNNVectorFieldMapper.Builder builder = new KNNVectorFieldMapper.Builder("test-field-name-1", modelDao, CURRENT, null); - builder.vectorDataType.setValue(VectorDataType.BINARY); - builder.dimension.setValue(8); + KNNVectorFieldMapper.TypeParser typeParser = new KNNVectorFieldMapper.TypeParser(() -> modelDao); + String fieldName = "test-field-name-1"; + String indexName = "test-index"; // Setup settings Settings settings = Settings.builder().put(settings(CURRENT).build()).put(KNN_INDEX, true).build(); - Mapper.BuilderContext builderContext = new Mapper.BuilderContext(settings, new ContentPath()); - Exception ex = expectThrows(Exception.class, () -> builder.build(builderContext)); - assertTrue(ex.getMessage(), ex.getMessage().contains("is not supported for")); + XContentBuilder xContentBuilder = XContentFactory.jsonBuilder() + .startObject() + .field(TYPE_FIELD_NAME, KNN_VECTOR_TYPE) + .field(DIMENSION_FIELD_NAME, 8) + .field(VECTOR_DATA_TYPE_FIELD, VectorDataType.BINARY.getValue()) + .endObject(); + + Exception ex = expectThrows(Exception.class, () -> { + typeParser.parse(fieldName, xContentBuilderToMap(xContentBuilder), buildParserContext(indexName, settings)); + }); + + assertTrue(ex.getMessage(), ex.getMessage().contains("is not supported with")); } public void testBuild_whenInvalidCharsInFieldName_thenThrowException() { @@ -1183,21 +1198,18 @@ public void testBuild_whenInvalidCharsInFieldName_thenThrowException() { // IllegalArgumentException should be thrown. Exception e = assertThrows(IllegalArgumentException.class, () -> { - new KNNVectorFieldMapper.Builder(invalidVectorFieldName, null, CURRENT, null).build(builderContext); + new KNNVectorFieldMapper.Builder(invalidVectorFieldName, null, CURRENT, null, null).build(builderContext); }); assertTrue(e.getMessage(), e.getMessage().contains("Vector field name must not include")); } } - private LuceneFieldMapper.CreateLuceneFieldMapperInput.CreateLuceneFieldMapperInputBuilder createLuceneFieldMapperInputBuilder( - VectorDataType vectorDataType - ) { + private LuceneFieldMapper.CreateLuceneFieldMapperInput.CreateLuceneFieldMapperInputBuilder createLuceneFieldMapperInputBuilder() { return LuceneFieldMapper.CreateLuceneFieldMapperInput.builder() .name(TEST_FIELD_NAME) .multiFields(FieldMapper.MultiFields.empty()) .copyTo(FieldMapper.CopyTo.empty()) .hasDocValues(true) - .vectorDataType(vectorDataType) .ignoreMalformed(new Explicit<>(true, true)) .originalKnnMethodContext(getDefaultKNNMethodContext()); } @@ -1247,6 +1259,5 @@ public Mapper.TypeParser.ParserContext buildParserContext(String indexName, Sett null, null ); - } } diff --git a/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperUtilTests.java b/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperUtilTests.java index ad041e47e..740d75206 100644 --- a/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperUtilTests.java +++ b/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperUtilTests.java @@ -18,17 +18,11 @@ import org.mockito.Mockito; import org.opensearch.Version; import org.opensearch.knn.KNNTestCase; -import org.opensearch.knn.common.KNNConstants; import org.opensearch.knn.index.KNNSettings; -import org.opensearch.knn.index.engine.KNNMethodContext; -import org.opensearch.knn.index.engine.MethodComponentContext; -import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.codec.util.KNNVectorSerializerFactory; -import org.opensearch.knn.index.engine.KNNEngine; import java.util.Arrays; -import java.util.Collections; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -81,35 +75,6 @@ public void testGetExpectedVectorLengthSuccess() { assertEquals(4, KNNVectorFieldMapperUtil.getExpectedVectorLength(knnVectorFieldTypeModelBased)); } - public void testValidateVectorDataType_whenBinaryFaissHNSW_thenValid() { - validateValidateVectorDataType(KNNEngine.FAISS, KNNConstants.METHOD_HNSW, VectorDataType.BINARY, null); - } - - public void testValidateVectorDataType_whenBinaryNonFaiss_thenException() { - validateValidateVectorDataType(KNNEngine.LUCENE, KNNConstants.METHOD_HNSW, VectorDataType.BINARY, "only supported"); - validateValidateVectorDataType(KNNEngine.NMSLIB, KNNConstants.METHOD_HNSW, VectorDataType.BINARY, "only supported"); - } - - public void testValidateVectorDataType_whenBinaryFaissIVF_thenException() { - validateValidateVectorDataType(KNNEngine.FAISS, KNNConstants.METHOD_IVF, VectorDataType.BINARY, "only supported"); - } - - public void testValidateVectorDataType_whenByteLucene_thenValid() { - validateValidateVectorDataType(KNNEngine.LUCENE, KNNConstants.METHOD_HNSW, VectorDataType.BYTE, null); - validateValidateVectorDataType(KNNEngine.LUCENE, KNNConstants.METHOD_IVF, VectorDataType.BYTE, null); - } - - public void testValidateVectorDataType_whenByteNonLucene_thenException() { - validateValidateVectorDataType(KNNEngine.FAISS, KNNConstants.METHOD_HNSW, VectorDataType.BYTE, "only supported"); - validateValidateVectorDataType(KNNEngine.NMSLIB, KNNConstants.METHOD_IVF, VectorDataType.BYTE, "only supported"); - } - - public void testValidateVectorDataType_whenFloat_thenValid() { - validateValidateVectorDataType(KNNEngine.FAISS, KNNConstants.METHOD_HNSW, VectorDataType.FLOAT, null); - validateValidateVectorDataType(KNNEngine.LUCENE, KNNConstants.METHOD_HNSW, VectorDataType.FLOAT, null); - validateValidateVectorDataType(KNNEngine.NMSLIB, KNNConstants.METHOD_HNSW, VectorDataType.FLOAT, null); - } - public void testUseLuceneKNNVectorsFormat_withDifferentInputs_thenSuccess() { final KNNSettings knnSettings = mock(KNNSettings.class); final MockedStatic mockedStatic = Mockito.mockStatic(KNNSettings.class); @@ -125,23 +90,4 @@ public void testUseLuceneKNNVectorsFormat_withDifferentInputs_thenSuccess() { // this mocking mockedStatic.close(); } - - private void validateValidateVectorDataType( - final KNNEngine knnEngine, - final String methodName, - final VectorDataType vectorDataType, - final String expectedErrMsg - ) { - MethodComponentContext methodComponentContext = new MethodComponentContext(methodName, Collections.emptyMap()); - KNNMethodContext methodContext = new KNNMethodContext(knnEngine, SpaceType.UNDEFINED, methodComponentContext); - if (expectedErrMsg == null) { - KNNVectorFieldMapperUtil.validateVectorDataType(methodContext, vectorDataType); - } else { - Exception ex = expectThrows( - IllegalArgumentException.class, - () -> KNNVectorFieldMapperUtil.validateVectorDataType(methodContext, vectorDataType) - ); - assertTrue(ex.getMessage().contains(expectedErrMsg)); - } - } } diff --git a/src/test/java/org/opensearch/knn/index/mapper/MethodFieldMapperTests.java b/src/test/java/org/opensearch/knn/index/mapper/MethodFieldMapperTests.java deleted file mode 100644 index faae3e35d..000000000 --- a/src/test/java/org/opensearch/knn/index/mapper/MethodFieldMapperTests.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.knn.index.mapper; - -import org.opensearch.Version; -import org.opensearch.index.mapper.FieldMapper; -import org.opensearch.knn.KNNTestCase; -import org.opensearch.knn.index.VectorDataType; -import org.opensearch.knn.index.engine.KNNMethodContext; - -import java.util.Collections; - -public class MethodFieldMapperTests extends KNNTestCase { - public void testMethodFieldMapper_whenVectorDataTypeAndContextMismatch_thenThrow() { - // Expect that we cannot create the mapper with an invalid field type - KNNMethodContext knnMethodContext = getDefaultKNNMethodContext(); - expectThrows( - IllegalArgumentException.class, - () -> MethodFieldMapper.createFieldMapper( - "testField", - "simpleName", - Collections.emptyMap(), - VectorDataType.BINARY, - 1, - knnMethodContext, - knnMethodContext, - null, - new FieldMapper.CopyTo.Builder().build(), - KNNVectorFieldMapper.Defaults.IGNORE_MALFORMED, - true, - true, - Version.CURRENT - ) - ); - } -} diff --git a/src/test/java/org/opensearch/knn/integ/BinaryIndexInvalidMappingIT.java b/src/test/java/org/opensearch/knn/integ/BinaryIndexInvalidMappingIT.java index 5832f2718..29e710ec1 100644 --- a/src/test/java/org/opensearch/knn/integ/BinaryIndexInvalidMappingIT.java +++ b/src/test/java/org/opensearch/knn/integ/BinaryIndexInvalidMappingIT.java @@ -49,17 +49,17 @@ public static Collection parameters() throws IOException { $( "Creation of binary index with lucene engine should fail", createKnnHnswBinaryIndexMapping(KNNEngine.LUCENE, FIELD_NAME, 16, null), - "only supported for [faiss] engine" + "Validation Failed" ), $( "Creation of binary index with nmslib engine should fail", createKnnHnswBinaryIndexMapping(KNNEngine.NMSLIB, FIELD_NAME, 16, null), - "only supported for [faiss] engine" + "Validation Failed" ), $( "Creation of binary index with encoder should fail", createKnnHnswBinaryIndexMapping(KNNEngine.FAISS, FIELD_NAME, 16, ENCODER_SQ), - "does not support sq encoder" + "Validation Failed" ) ) ); diff --git a/src/test/java/org/opensearch/knn/jni/JNIServiceTests.java b/src/test/java/org/opensearch/knn/jni/JNIServiceTests.java index ae9ad7106..53245cc62 100644 --- a/src/test/java/org/opensearch/knn/jni/JNIServiceTests.java +++ b/src/test/java/org/opensearch/knn/jni/JNIServiceTests.java @@ -21,6 +21,7 @@ import org.opensearch.knn.KNNTestCase; import org.opensearch.knn.TestUtils; import org.opensearch.knn.common.KNNConstants; +import org.opensearch.knn.index.engine.KNNMethodConfigContext; import org.opensearch.knn.index.engine.KNNMethodContext; import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.engine.nmslib.NmslibHNSWMethod; @@ -611,7 +612,13 @@ public void testTrain_whenConfigurationIsIVFSQFP16_thenSucceed() { .endObject(); Map in = xContentBuilderToMap(xContentBuilder); KNNMethodContext knnMethodContext = KNNMethodContext.parse(in); - Map parameters = KNNEngine.FAISS.getKNNLibraryIndexingContext(knnMethodContext).getLibraryParameters(); + KNNMethodConfigContext knnMethodConfigContext = KNNMethodConfigContext.builder() + .versionCreated(Version.CURRENT) + .dimension(128) + .vectorDataType(VectorDataType.FLOAT) + .build(); + Map parameters = KNNEngine.FAISS.getKNNLibraryIndexingContext(knnMethodContext, knnMethodConfigContext) + .getLibraryParameters(); byte[] faissIndex = JNIService.trainIndex(parameters, 128, trainPointer, KNNEngine.FAISS); @@ -1131,7 +1138,13 @@ public void testTrain_whenConfigurationIsIVFFlat_thenSucceed() throws IOExceptio .endObject(); Map in = xContentBuilderToMap(xContentBuilder); KNNMethodContext knnMethodContext = KNNMethodContext.parse(in); - Map parameters = KNNEngine.FAISS.getKNNLibraryIndexingContext(knnMethodContext).getLibraryParameters(); + KNNMethodConfigContext knnMethodConfigContext = KNNMethodConfigContext.builder() + .vectorDataType(VectorDataType.FLOAT) + .dimension(testData.indexData.getDimension()) + .versionCreated(Version.CURRENT) + .build(); + Map parameters = KNNEngine.FAISS.getKNNLibraryIndexingContext(knnMethodContext, knnMethodConfigContext) + .getLibraryParameters(); byte[] faissIndex = JNIService.trainIndex(parameters, 128, trainPointer, KNNEngine.FAISS); @@ -1162,7 +1175,13 @@ public void testTrain_whenConfigurationIsIVFPQ_thenSucceed() throws IOException .endObject(); Map in = xContentBuilderToMap(xContentBuilder); KNNMethodContext knnMethodContext = KNNMethodContext.parse(in); - Map parameters = KNNEngine.FAISS.getKNNLibraryIndexingContext(knnMethodContext).getLibraryParameters(); + KNNMethodConfigContext knnMethodConfigContext = KNNMethodConfigContext.builder() + .versionCreated(Version.CURRENT) + .dimension(128) + .vectorDataType(VectorDataType.FLOAT) + .build(); + Map parameters = KNNEngine.FAISS.getKNNLibraryIndexingContext(knnMethodContext, knnMethodConfigContext) + .getLibraryParameters(); byte[] faissIndex = JNIService.trainIndex(parameters, 128, trainPointer, KNNEngine.FAISS); @@ -1189,8 +1208,13 @@ public void testTrain_whenConfigurationIsHNSWPQ_thenSucceed() throws IOException .endObject(); Map in = xContentBuilderToMap(xContentBuilder); KNNMethodContext knnMethodContext = KNNMethodContext.parse(in); - knnMethodContext.getMethodComponentContext().setIndexVersion(Version.CURRENT); - Map parameters = KNNEngine.FAISS.getKNNLibraryIndexingContext(knnMethodContext).getLibraryParameters(); + KNNMethodConfigContext knnMethodConfigContext = KNNMethodConfigContext.builder() + .vectorDataType(VectorDataType.FLOAT) + .dimension(testData.indexData.getDimension()) + .versionCreated(Version.CURRENT) + .build(); + Map parameters = KNNEngine.FAISS.getKNNLibraryIndexingContext(knnMethodContext, knnMethodConfigContext) + .getLibraryParameters(); byte[] faissIndex = JNIService.trainIndex(parameters, 128, trainPointer, KNNEngine.FAISS); @@ -1223,6 +1247,11 @@ public void testCreateIndexFromTemplate() throws IOException { } SpaceType spaceType = SpaceType.L2; + KNNMethodConfigContext knnMethodConfigContext = KNNMethodConfigContext.builder() + .versionCreated(Version.CURRENT) + .dimension(128) + .vectorDataType(VectorDataType.FLOAT) + .build(); KNNMethodContext knnMethodContext = new KNNMethodContext( KNNEngine.FAISS, spaceType, @@ -1238,7 +1267,7 @@ public void testCreateIndexFromTemplate() throws IOException { ); String description = knnMethodContext.getKnnEngine() - .getKNNLibraryIndexingContext(knnMethodContext) + .getKNNLibraryIndexingContext(knnMethodContext, knnMethodConfigContext) .getLibraryParameters() .get(INDEX_DESCRIPTION_PARAMETER) .toString(); @@ -1361,7 +1390,11 @@ private void assertQueryResultsMatch(float[][] testQueries, int k, List in private String createFaissIVFPQIndex(int ivfNlist, int pqM, int pqCodeSize, SpaceType spaceType) throws IOException { long trainPointer = JNIService.transferVectors(0, testData.indexData.vectors); assertNotEquals(0, trainPointer); - + KNNMethodConfigContext knnMethodConfigContext = KNNMethodConfigContext.builder() + .versionCreated(Version.CURRENT) + .dimension(128) + .vectorDataType(VectorDataType.FLOAT) + .build(); KNNMethodContext knnMethodContext = new KNNMethodContext( KNNEngine.FAISS, spaceType, @@ -1380,7 +1413,7 @@ private String createFaissIVFPQIndex(int ivfNlist, int pqM, int pqCodeSize, Spac ); String description = knnMethodContext.getKnnEngine() - .getKNNLibraryIndexingContext(knnMethodContext) + .getKNNLibraryIndexingContext(knnMethodContext, knnMethodConfigContext) .getLibraryParameters() .get(INDEX_DESCRIPTION_PARAMETER) .toString(); diff --git a/src/test/java/org/opensearch/knn/plugin/stats/suppliers/LibraryInitializedSupplierTests.java b/src/test/java/org/opensearch/knn/plugin/stats/suppliers/LibraryInitializedSupplierTests.java index 7fa0d3bca..4399b3318 100644 --- a/src/test/java/org/opensearch/knn/plugin/stats/suppliers/LibraryInitializedSupplierTests.java +++ b/src/test/java/org/opensearch/knn/plugin/stats/suppliers/LibraryInitializedSupplierTests.java @@ -14,10 +14,10 @@ import org.opensearch.common.ValidationException; import org.opensearch.knn.index.engine.KNNLibraryIndexingContext; import org.opensearch.knn.index.engine.KNNLibrarySearchContext; +import org.opensearch.knn.index.engine.KNNMethodConfigContext; import org.opensearch.knn.index.engine.KNNMethodContext; import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.engine.KNNLibrary; -import org.opensearch.knn.training.VectorSpaceInfo; import org.opensearch.test.OpenSearchTestCase; public class LibraryInitializedSupplierTests extends OpenSearchTestCase { @@ -74,12 +74,7 @@ public Float scoreToRadialThreshold(Float score, SpaceType spaceType) { } @Override - public ValidationException validateMethod(KNNMethodContext knnMethodContext) { - return null; - } - - @Override - public ValidationException validateMethodWithData(KNNMethodContext knnMethodContext, VectorSpaceInfo vectorSpaceInfo) { + public ValidationException validateMethod(KNNMethodContext knnMethodContext, KNNMethodConfigContext knnMethodConfigContext) { return null; } @@ -89,12 +84,15 @@ public boolean isTrainingRequired(KNNMethodContext knnMethodContext) { } @Override - public int estimateOverheadInKB(KNNMethodContext knnMethodContext, int dimension) { + public int estimateOverheadInKB(KNNMethodContext knnMethodContext, KNNMethodConfigContext knnMethodConfigContext) { return 0; } @Override - public KNNLibraryIndexingContext getKNNLibraryIndexingContext(KNNMethodContext knnMethodContext) { + public KNNLibraryIndexingContext getKNNLibraryIndexingContext( + KNNMethodContext knnMethodContext, + KNNMethodConfigContext knnMethodConfigContext + ) { return null; } diff --git a/src/test/java/org/opensearch/knn/plugin/transport/TrainingModelRequestTests.java b/src/test/java/org/opensearch/knn/plugin/transport/TrainingModelRequestTests.java index 83d39cfdc..d7920d987 100644 --- a/src/test/java/org/opensearch/knn/plugin/transport/TrainingModelRequestTests.java +++ b/src/test/java/org/opensearch/knn/plugin/transport/TrainingModelRequestTests.java @@ -41,6 +41,7 @@ import java.util.List; import java.util.Map; +import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -149,7 +150,7 @@ public void testValidation_invalid_modelIdAlreadyExists() { // Setup the training request String modelId = "test-model-id"; KNNMethodContext knnMethodContext = mock(KNNMethodContext.class); - when(knnMethodContext.validate()).thenReturn(null); + when(knnMethodContext.validate(any())).thenReturn(null); when(knnMethodContext.isTrainingRequired()).thenReturn(true); int dimension = 10; String trainingIndex = "test-training-index"; @@ -206,7 +207,7 @@ public void testValidation_blocked_modelId() { // Setup the training request String modelId = "test-model-id"; KNNMethodContext knnMethodContext = mock(KNNMethodContext.class); - when(knnMethodContext.validate()).thenReturn(null); + when(knnMethodContext.validate(any())).thenReturn(null); when(knnMethodContext.isTrainingRequired()).thenReturn(true); int dimension = 10; String trainingIndex = "test-training-index"; @@ -252,7 +253,7 @@ public void testValidation_invalid_invalidMethodContext() { String validationExceptionMessage = "knn method invalid"; ValidationException validationException = new ValidationException(); validationException.addValidationError(validationExceptionMessage); - when(knnMethodContext.validate()).thenReturn(validationException); + when(knnMethodContext.validate(any())).thenReturn(validationException); when(knnMethodContext.isTrainingRequired()).thenReturn(false); when(knnMethodContext.getMethodComponentContext()).thenReturn(MethodComponentContext.EMPTY); @@ -297,7 +298,7 @@ public void testValidation_invalid_trainingIndexDoesNotExist() { String modelId = "test-model-id"; KNNMethodContext knnMethodContext = mock(KNNMethodContext.class); - when(knnMethodContext.validate()).thenReturn(null); + when(knnMethodContext.validate(any())).thenReturn(null); when(knnMethodContext.isTrainingRequired()).thenReturn(true); int dimension = 10; @@ -344,7 +345,7 @@ public void testValidation_invalid_trainingFieldDoesNotExist() { String modelId = "test-model-id"; KNNMethodContext knnMethodContext = mock(KNNMethodContext.class); - when(knnMethodContext.validate()).thenReturn(null); + when(knnMethodContext.validate(any())).thenReturn(null); when(knnMethodContext.isTrainingRequired()).thenReturn(true); int dimension = 10; @@ -396,7 +397,7 @@ public void testValidation_invalid_trainingFieldNotKnnVector() { String modelId = "test-model-id"; KNNMethodContext knnMethodContext = mock(KNNMethodContext.class); - when(knnMethodContext.validate()).thenReturn(null); + when(knnMethodContext.validate(any())).thenReturn(null); when(knnMethodContext.isTrainingRequired()).thenReturn(true); int dimension = 10; @@ -452,7 +453,7 @@ public void testValidation_invalid_dimensionDoesNotMatch() { String modelId = "test-model-id"; KNNMethodContext knnMethodContext = mock(KNNMethodContext.class); - when(knnMethodContext.validate()).thenReturn(null); + when(knnMethodContext.validate(any())).thenReturn(null); when(knnMethodContext.isTrainingRequired()).thenReturn(true); when(knnMethodContext.getMethodComponentContext()).thenReturn(MethodComponentContext.EMPTY); @@ -511,7 +512,7 @@ public void testValidation_invalid_preferredNodeDoesNotExist() { // Setup the training request String modelId = "test-model-id"; KNNMethodContext knnMethodContext = mock(KNNMethodContext.class); - when(knnMethodContext.validate()).thenReturn(null); + when(knnMethodContext.validate(any())).thenReturn(null); when(knnMethodContext.isTrainingRequired()).thenReturn(true); when(knnMethodContext.getMethodComponentContext()).thenReturn(MethodComponentContext.EMPTY); int dimension = 10; @@ -574,7 +575,7 @@ public void testValidation_invalid_descriptionToLong() { // Setup the training request String modelId = "test-model-id"; KNNMethodContext knnMethodContext = mock(KNNMethodContext.class); - when(knnMethodContext.validate()).thenReturn(null); + when(knnMethodContext.validate(any())).thenReturn(null); when(knnMethodContext.isTrainingRequired()).thenReturn(true); when(knnMethodContext.getMethodComponentContext()).thenReturn(MethodComponentContext.EMPTY); int dimension = 10; @@ -625,7 +626,7 @@ public void testValidation_valid_trainingIndexBuiltFromMethod() { // Setup the training request String modelId = "test-model-id"; KNNMethodContext knnMethodContext = mock(KNNMethodContext.class); - when(knnMethodContext.validate()).thenReturn(null); + when(knnMethodContext.validate(any())).thenReturn(null); when(knnMethodContext.isTrainingRequired()).thenReturn(true); when(knnMethodContext.getMethodComponentContext()).thenReturn(MethodComponentContext.EMPTY); int dimension = 10; @@ -663,7 +664,7 @@ public void testValidation_valid_trainingIndexBuiltFromModel() { // Setup the training request String modelId = "test-model-id"; KNNMethodContext knnMethodContext = mock(KNNMethodContext.class); - when(knnMethodContext.validate()).thenReturn(null); + when(knnMethodContext.validate(any())).thenReturn(null); when(knnMethodContext.isTrainingRequired()).thenReturn(true); when(knnMethodContext.getMethodComponentContext()).thenReturn(MethodComponentContext.EMPTY); int dimension = 10; diff --git a/src/test/java/org/opensearch/knn/training/TrainingJobTests.java b/src/test/java/org/opensearch/knn/training/TrainingJobTests.java index b6d76c68e..adecca43a 100644 --- a/src/test/java/org/opensearch/knn/training/TrainingJobTests.java +++ b/src/test/java/org/opensearch/knn/training/TrainingJobTests.java @@ -15,6 +15,7 @@ import org.opensearch.Version; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.knn.KNNTestCase; +import org.opensearch.knn.index.engine.KNNMethodConfigContext; import org.opensearch.knn.index.engine.KNNMethodContext; import org.opensearch.knn.index.engine.MethodComponentContext; import org.opensearch.knn.index.SpaceType; @@ -66,10 +67,9 @@ public void testGetModelId() { mock(NativeMemoryCacheManager.class), mock(NativeMemoryEntryContext.TrainingDataEntryContext.class), mock(NativeMemoryEntryContext.AnonymousEntryContext.class), - 10, + KNNMethodConfigContext.builder().vectorDataType(VectorDataType.DEFAULT).dimension(10).versionCreated(Version.CURRENT).build(), "", - "test-node", - VectorDataType.DEFAULT + "test-node" ); assertEquals(modelId, trainingJob.getModelId()); @@ -96,10 +96,13 @@ public void testGetModel() { mock(NativeMemoryCacheManager.class), mock(NativeMemoryEntryContext.TrainingDataEntryContext.class), mock(NativeMemoryEntryContext.AnonymousEntryContext.class), - dimension, + KNNMethodConfigContext.builder() + .vectorDataType(VectorDataType.FLOAT) + .dimension(dimension) + .versionCreated(Version.CURRENT) + .build(), description, - nodeAssignment, - VectorDataType.DEFAULT + nodeAssignment ); Model model = new Model( @@ -130,6 +133,11 @@ public void testRun_success() throws IOException, ExecutionException { int nlists = 5; int dimension = 16; KNNEngine knnEngine = KNNEngine.FAISS; + KNNMethodConfigContext knnMethodConfigContext = KNNMethodConfigContext.builder() + .vectorDataType(VectorDataType.FLOAT) + .dimension(dimension) + .versionCreated(Version.CURRENT) + .build(); KNNMethodContext knnMethodContext = new KNNMethodContext( knnEngine, SpaceType.INNER_PRODUCT, @@ -185,10 +193,9 @@ public void testRun_success() throws IOException, ExecutionException { nativeMemoryCacheManager, trainingDataEntryContext, modelContext, - dimension, + knnMethodConfigContext, "", - "test-node", - VectorDataType.DEFAULT + "test-node" ); trainingJob.run(); @@ -225,6 +232,11 @@ public void testRun_failure_onGetTrainingDataAllocation() throws ExecutionExcept int nlists = 5; int dimension = 16; KNNEngine knnEngine = KNNEngine.FAISS; + KNNMethodConfigContext knnMethodConfigContext = KNNMethodConfigContext.builder() + .vectorDataType(VectorDataType.FLOAT) + .dimension(dimension) + .versionCreated(Version.CURRENT) + .build(); KNNMethodContext knnMethodContext = new KNNMethodContext( knnEngine, SpaceType.INNER_PRODUCT, @@ -264,10 +276,9 @@ public void testRun_failure_onGetTrainingDataAllocation() throws ExecutionExcept nativeMemoryCacheManager, trainingDataEntryContext, modelContext, - dimension, + knnMethodConfigContext, "", - "test-node", - VectorDataType.DEFAULT + "test-node" ); trainingJob.run(); @@ -287,6 +298,11 @@ public void testRun_failure_onGetModelAnonymousAllocation() throws ExecutionExce int nlists = 5; int dimension = 16; KNNEngine knnEngine = KNNEngine.FAISS; + KNNMethodConfigContext knnMethodConfigContext = KNNMethodConfigContext.builder() + .vectorDataType(VectorDataType.FLOAT) + .dimension(dimension) + .versionCreated(Version.CURRENT) + .build(); KNNMethodContext knnMethodContext = new KNNMethodContext( knnEngine, SpaceType.INNER_PRODUCT, @@ -332,10 +348,9 @@ public void testRun_failure_onGetModelAnonymousAllocation() throws ExecutionExce nativeMemoryCacheManager, trainingDataEntryContext, modelContext, - dimension, + knnMethodConfigContext, "", - "test-node", - VectorDataType.DEFAULT + "test-node" ); trainingJob.run(); @@ -355,6 +370,11 @@ public void testRun_failure_closedTrainingDataAllocation() throws ExecutionExcep int nlists = 5; int dimension = 16; KNNEngine knnEngine = KNNEngine.FAISS; + KNNMethodConfigContext knnMethodConfigContext = KNNMethodConfigContext.builder() + .vectorDataType(VectorDataType.FLOAT) + .dimension(dimension) + .versionCreated(Version.CURRENT) + .build(); KNNMethodContext knnMethodContext = new KNNMethodContext( knnEngine, SpaceType.INNER_PRODUCT, @@ -399,10 +419,9 @@ public void testRun_failure_closedTrainingDataAllocation() throws ExecutionExcep nativeMemoryCacheManager, trainingDataEntryContext, mock(NativeMemoryEntryContext.AnonymousEntryContext.class), - dimension, + knnMethodConfigContext, "", - "test-node", - VectorDataType.DEFAULT + "test-node" ); trainingJob.run(); @@ -420,6 +439,11 @@ public void testRun_failure_notEnoughTrainingData() throws ExecutionException { int nlists = 1024; // setting this to 1024 will cause training to fail when there is only 2 data points int dimension = 16; KNNEngine knnEngine = KNNEngine.FAISS; + KNNMethodConfigContext knnMethodConfigContext = KNNMethodConfigContext.builder() + .vectorDataType(VectorDataType.FLOAT) + .dimension(dimension) + .versionCreated(Version.CURRENT) + .build(); KNNMethodContext knnMethodContext = new KNNMethodContext( knnEngine, SpaceType.INNER_PRODUCT, @@ -473,10 +497,9 @@ public void testRun_failure_notEnoughTrainingData() throws ExecutionException { nativeMemoryCacheManager, trainingDataEntryContext, modelContext, - dimension, + knnMethodConfigContext, "", - "test-node", - VectorDataType.DEFAULT + "test-node" ); trainingJob.run(); diff --git a/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java b/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java index 50b149830..a90935869 100644 --- a/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java +++ b/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java @@ -1222,6 +1222,14 @@ public void addKNNDocs(String testIndex, String testField, int dimension, int fi } } + public void addKNNByteDocs(String testIndex, String testField, int dimension, int firstDocID, int numDocs) throws IOException { + for (int i = firstDocID; i < firstDocID + numDocs; i++) { + Byte[] indexVector = new Byte[dimension]; + Arrays.fill(indexVector, (byte) i); + addKnnDoc(testIndex, Integer.toString(i), testField, indexVector); + } + } + public void validateKNNSearch(String testIndex, String testField, int dimension, int numDocs, int k) throws Exception { validateKNNSearch(testIndex, testField, dimension, numDocs, k, null); } From 7ad7d2faf18194d79c1885de0f948830529faccc Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Mon, 19 Aug 2024 11:45:05 -0700 Subject: [PATCH 330/416] Fixing the dimension for the vector when using Lucene field in ModelFieldMapper (#1980) (#1986) Signed-off-by: Navneet Verma (cherry picked from commit 3ca52e401b3741562b83a8e444a8ba674a3513d3) Co-authored-by: Navneet Verma --- .../knn/index/mapper/ModelFieldMapper.java | 4 +- .../java/org/opensearch/knn/KNNTestCase.java | 22 ++ .../knn/common/FieldInfoExtractorTests.java | 38 ++- .../mapper/KNNVectorFieldMapperTests.java | 298 ++++++++++++------ .../knn/index/query/KNNWeightTests.java | 236 +++++++------- 5 files changed, 360 insertions(+), 238 deletions(-) diff --git a/src/main/java/org/opensearch/knn/index/mapper/ModelFieldMapper.java b/src/main/java/org/opensearch/knn/index/mapper/ModelFieldMapper.java index 954d6addf..a7dc17288 100644 --- a/src/main/java/org/opensearch/knn/index/mapper/ModelFieldMapper.java +++ b/src/main/java/org/opensearch/knn/index/mapper/ModelFieldMapper.java @@ -196,8 +196,8 @@ protected void parseCreateField(ParseContext context) throws IOException { ModelMetadata modelMetadata = getModelMetadata(modelDao, modelId); if (useLuceneBasedVectorField) { int adjustedDimension = modelMetadata.getVectorDataType() == VectorDataType.BINARY - ? modelMetadata.getDimension() - : modelMetadata.getDimension() / 8; + ? modelMetadata.getDimension() / Byte.SIZE + : modelMetadata.getDimension(); final VectorEncoding encoding = modelMetadata.getVectorDataType() == VectorDataType.FLOAT ? VectorEncoding.FLOAT32 : VectorEncoding.BYTE; diff --git a/src/test/java/org/opensearch/knn/KNNTestCase.java b/src/test/java/org/opensearch/knn/KNNTestCase.java index 2aa11a247..6ef7373d2 100644 --- a/src/test/java/org/opensearch/knn/KNNTestCase.java +++ b/src/test/java/org/opensearch/knn/KNNTestCase.java @@ -13,6 +13,7 @@ import org.opensearch.common.settings.Settings; import org.opensearch.knn.index.KNNSettings; import org.opensearch.knn.index.SpaceType; +import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.engine.KNNEngine; import org.opensearch.knn.index.engine.KNNLibrarySearchContext; import org.opensearch.knn.index.engine.KNNMethodContext; @@ -146,4 +147,25 @@ public int getDimension() { } }; } + + /** + * Adjust the provided dimension based on {@link VectorDataType} during ingestion. + * @param dimension int + * @param vectorDataType {@link VectorDataType} + * @return int + */ + protected int adjustDimensionForIndexing(final int dimension, final VectorDataType vectorDataType) { + return VectorDataType.BINARY == vectorDataType ? dimension * Byte.SIZE : dimension; + } + + /** + * Adjust the provided dimension based on {@link VectorDataType} for search. + * + * @param dimension int + * @param vectorDataType {@link VectorDataType} + * @return int + */ + protected int adjustDimensionForSearch(final int dimension, final VectorDataType vectorDataType) { + return VectorDataType.BINARY == vectorDataType ? dimension / Byte.SIZE : dimension; + } } diff --git a/src/test/java/org/opensearch/knn/common/FieldInfoExtractorTests.java b/src/test/java/org/opensearch/knn/common/FieldInfoExtractorTests.java index e86a153d3..df529bb47 100644 --- a/src/test/java/org/opensearch/knn/common/FieldInfoExtractorTests.java +++ b/src/test/java/org/opensearch/knn/common/FieldInfoExtractorTests.java @@ -20,25 +20,23 @@ public class FieldInfoExtractorTests extends KNNTestCase { public void testExtractVectorDataType_whenDifferentConditions_thenSuccess() { FieldInfo fieldInfo = Mockito.mock(FieldInfo.class); - MockedStatic modelUtilMockedStatic = Mockito.mockStatic(ModelUtil.class); - - // default case - Mockito.when(fieldInfo.getAttribute(KNNConstants.VECTOR_DATA_TYPE_FIELD)).thenReturn(null); - Mockito.when(fieldInfo.getAttribute(KNNConstants.MODEL_ID)).thenReturn(MODEL_ID); - modelUtilMockedStatic.when(() -> ModelUtil.getModelMetadata(MODEL_ID)).thenReturn(null); - Assert.assertEquals(VectorDataType.DEFAULT, FieldInfoExtractor.extractVectorDataType(fieldInfo)); - - // VectorDataType present in fieldInfo - Mockito.when(fieldInfo.getAttribute(KNNConstants.VECTOR_DATA_TYPE_FIELD)).thenReturn(VectorDataType.BINARY.getValue()); - Assert.assertEquals(VectorDataType.BINARY, FieldInfoExtractor.extractVectorDataType(fieldInfo)); - - // VectorDataType present in ModelMetadata - ModelMetadata modelMetadata = Mockito.mock(ModelMetadata.class); - Mockito.when(fieldInfo.getAttribute(KNNConstants.VECTOR_DATA_TYPE_FIELD)).thenReturn(null); - modelUtilMockedStatic.when(() -> ModelUtil.getModelMetadata(MODEL_ID)).thenReturn(modelMetadata); - Mockito.when(modelMetadata.getVectorDataType()).thenReturn(VectorDataType.BYTE); - Assert.assertEquals(VectorDataType.BYTE, FieldInfoExtractor.extractVectorDataType(fieldInfo)); - - modelUtilMockedStatic.close(); + try (MockedStatic modelUtilMockedStatic = Mockito.mockStatic(ModelUtil.class)) { + // default case + Mockito.when(fieldInfo.getAttribute(KNNConstants.VECTOR_DATA_TYPE_FIELD)).thenReturn(null); + Mockito.when(fieldInfo.getAttribute(KNNConstants.MODEL_ID)).thenReturn(MODEL_ID); + modelUtilMockedStatic.when(() -> ModelUtil.getModelMetadata(MODEL_ID)).thenReturn(null); + Assert.assertEquals(VectorDataType.DEFAULT, FieldInfoExtractor.extractVectorDataType(fieldInfo)); + + // VectorDataType present in fieldInfo + Mockito.when(fieldInfo.getAttribute(KNNConstants.VECTOR_DATA_TYPE_FIELD)).thenReturn(VectorDataType.BINARY.getValue()); + Assert.assertEquals(VectorDataType.BINARY, FieldInfoExtractor.extractVectorDataType(fieldInfo)); + + // VectorDataType present in ModelMetadata + ModelMetadata modelMetadata = Mockito.mock(ModelMetadata.class); + Mockito.when(fieldInfo.getAttribute(KNNConstants.VECTOR_DATA_TYPE_FIELD)).thenReturn(null); + modelUtilMockedStatic.when(() -> ModelUtil.getModelMetadata(MODEL_ID)).thenReturn(modelMetadata); + Mockito.when(modelMetadata.getVectorDataType()).thenReturn(VectorDataType.BYTE); + Assert.assertEquals(VectorDataType.BYTE, FieldInfoExtractor.extractVectorDataType(fieldInfo)); + } } } diff --git a/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java b/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java index 4034e4cb6..369f38cf9 100644 --- a/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java +++ b/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java @@ -9,7 +9,6 @@ import lombok.extern.log4j.Log4j2; import org.apache.lucene.document.KnnByteVectorField; import org.apache.lucene.document.KnnFloatVectorField; -import org.apache.lucene.document.KnnVectorField; import org.apache.lucene.index.IndexableField; import org.apache.lucene.index.VectorEncoding; import org.apache.lucene.util.BytesRef; @@ -20,9 +19,15 @@ import org.opensearch.common.ValidationException; import org.opensearch.common.settings.IndexScopedSettings; import org.opensearch.common.settings.Settings; +import org.opensearch.common.xcontent.LoggingDeprecationHandler; import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.common.xcontent.XContentHelper; import org.opensearch.core.common.Strings; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.xcontent.MediaTypeRegistry; +import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; import org.opensearch.index.IndexSettings; import org.opensearch.index.mapper.ContentPath; import org.opensearch.index.mapper.FieldMapper; @@ -42,6 +47,7 @@ import org.opensearch.knn.indices.ModelDao; import org.opensearch.knn.indices.ModelMetadata; import org.opensearch.knn.indices.ModelState; +import org.opensearch.knn.indices.ModelUtil; import java.io.IOException; import java.time.ZoneOffset; @@ -66,6 +72,7 @@ import static org.opensearch.knn.common.KNNConstants.LUCENE_NAME; import static org.opensearch.knn.common.KNNConstants.METHOD_ENCODER_PARAMETER; import static org.opensearch.knn.common.KNNConstants.METHOD_HNSW; +import static org.opensearch.knn.common.KNNConstants.METHOD_IVF; import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_EF_CONSTRUCTION; import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_M; import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_SPACE_TYPE; @@ -768,28 +775,28 @@ public void testKNNVectorFieldMapper_merge_fromModel() throws IOException { @SneakyThrows public void testMethodFieldMapperParseCreateField_validInput_thenDifferentFieldTypes() { - MockedStatic utilMockedStatic = Mockito.mockStatic(KNNVectorFieldMapperUtil.class); - for (VectorDataType dataType : VectorDataType.values()) { - log.info("Vector Data Type is : {}", dataType); - int dimension = dataType == VectorDataType.BINARY ? TEST_DIMENSION * 8 : TEST_DIMENSION; - final MethodComponentContext methodComponentContext = new MethodComponentContext(METHOD_HNSW, Collections.emptyMap()); - SpaceType spaceType = VectorDataType.BINARY == dataType ? SpaceType.DEFAULT_BINARY : SpaceType.INNER_PRODUCT; - KNNMethodConfigContext knnMethodConfigContext = KNNMethodConfigContext.builder() - .vectorDataType(dataType) - .versionCreated(CURRENT) - .dimension(dimension) - .build(); - final KNNMethodContext knnMethodContext = new KNNMethodContext(KNNEngine.FAISS, spaceType, methodComponentContext); - - ParseContext.Document document = new ParseContext.Document(); - ContentPath contentPath = new ContentPath(); - ParseContext parseContext = mock(ParseContext.class); - when(parseContext.doc()).thenReturn(document); - when(parseContext.path()).thenReturn(contentPath); - - utilMockedStatic.when(() -> KNNVectorFieldMapperUtil.useLuceneKNNVectorsFormat(Mockito.any())).thenReturn(true); - MethodFieldMapper methodFieldMapper = Mockito.spy( - MethodFieldMapper.createFieldMapper( + try (MockedStatic utilMockedStatic = Mockito.mockStatic(KNNVectorFieldMapperUtil.class)) { + for (VectorDataType dataType : VectorDataType.values()) { + log.info("Vector Data Type is : {}", dataType); + int dimension = adjustDimensionForIndexing(TEST_DIMENSION, dataType); + final MethodComponentContext methodComponentContext = new MethodComponentContext(METHOD_HNSW, Collections.emptyMap()); + SpaceType spaceType = VectorDataType.BINARY == dataType ? SpaceType.DEFAULT_BINARY : SpaceType.INNER_PRODUCT; + KNNMethodConfigContext knnMethodConfigContext = KNNMethodConfigContext.builder() + .vectorDataType(dataType) + .versionCreated(CURRENT) + .dimension(dimension) + .build(); + final KNNMethodContext knnMethodContext = new KNNMethodContext(KNNEngine.FAISS, spaceType, methodComponentContext); + + ParseContext.Document document = new ParseContext.Document(); + ContentPath contentPath = new ContentPath(); + ParseContext parseContext = mock(ParseContext.class); + when(parseContext.doc()).thenReturn(document); + when(parseContext.path()).thenReturn(contentPath); + when(parseContext.parser()).thenReturn(createXContentParser(dataType)); + + utilMockedStatic.when(() -> KNNVectorFieldMapperUtil.useLuceneKNNVectorsFormat(Mockito.any())).thenReturn(true); + MethodFieldMapper methodFieldMapper = MethodFieldMapper.createFieldMapper( TEST_FIELD_NAME, TEST_FIELD_NAME, Collections.emptyMap(), @@ -801,45 +808,35 @@ public void testMethodFieldMapperParseCreateField_validInput_thenDifferentFieldT new Explicit<>(true, true), false, false - ) - ); - - if (dataType == VectorDataType.BINARY) { - doReturn(Optional.of(TEST_BYTE_VECTOR)).when(methodFieldMapper) - .getBytesFromContext(parseContext, TEST_DIMENSION * 8, dataType); - } else if (dataType == VectorDataType.BYTE) { - doReturn(Optional.of(TEST_BYTE_VECTOR)).when(methodFieldMapper).getBytesFromContext(parseContext, TEST_DIMENSION, dataType); - } else { - doReturn(Optional.of(TEST_VECTOR)).when(methodFieldMapper).getFloatsFromContext(parseContext, TEST_DIMENSION); - } - - methodFieldMapper.parseCreateField(parseContext, dimension, dataType); - - List fields = document.getFields(); - assertEquals(1, fields.size()); - IndexableField field1 = fields.get(0); - if (dataType == VectorDataType.FLOAT) { - assertTrue(field1 instanceof KnnFloatVectorField); - assertEquals(field1.fieldType().vectorEncoding(), VectorEncoding.FLOAT32); - } else { - assertTrue(field1 instanceof KnnByteVectorField); - assertEquals(field1.fieldType().vectorEncoding(), VectorEncoding.BYTE); - } - - assertEquals(field1.fieldType().vectorDimension(), TEST_DIMENSION); - assertEquals( - field1.fieldType().vectorSimilarityFunction(), - SpaceType.DEFAULT.getKnnVectorSimilarityFunction().getVectorSimilarityFunction() - ); - - utilMockedStatic.when(() -> KNNVectorFieldMapperUtil.useLuceneKNNVectorsFormat(Mockito.any())).thenReturn(false); - - document = new ParseContext.Document(); - contentPath = new ContentPath(); - when(parseContext.doc()).thenReturn(document); - when(parseContext.path()).thenReturn(contentPath); - methodFieldMapper = Mockito.spy( - MethodFieldMapper.createFieldMapper( + ); + methodFieldMapper.parseCreateField(parseContext, dimension, dataType); + + List fields = document.getFields(); + assertEquals(1, fields.size()); + IndexableField field1 = fields.get(0); + if (dataType == VectorDataType.FLOAT) { + assertTrue(field1 instanceof KnnFloatVectorField); + assertEquals(field1.fieldType().vectorEncoding(), VectorEncoding.FLOAT32); + } else { + assertTrue(field1 instanceof KnnByteVectorField); + assertEquals(field1.fieldType().vectorEncoding(), VectorEncoding.BYTE); + } + + assertEquals(field1.fieldType().vectorDimension(), adjustDimensionForSearch(dimension, dataType)); + assertEquals(Integer.parseInt(field1.fieldType().getAttributes().get(DIMENSION_FIELD_NAME)), dimension); + assertEquals( + field1.fieldType().vectorSimilarityFunction(), + SpaceType.DEFAULT.getKnnVectorSimilarityFunction().getVectorSimilarityFunction() + ); + + utilMockedStatic.when(() -> KNNVectorFieldMapperUtil.useLuceneKNNVectorsFormat(Mockito.any())).thenReturn(false); + + document = new ParseContext.Document(); + contentPath = new ContentPath(); + when(parseContext.doc()).thenReturn(document); + when(parseContext.path()).thenReturn(contentPath); + when(parseContext.parser()).thenReturn(createXContentParser(dataType)); + methodFieldMapper = MethodFieldMapper.createFieldMapper( TEST_FIELD_NAME, TEST_FIELD_NAME, Collections.emptyMap(), @@ -851,25 +848,110 @@ public void testMethodFieldMapperParseCreateField_validInput_thenDifferentFieldT new Explicit<>(true, true), false, false - ) - ); - - if (dataType == VectorDataType.FLOAT) { - doReturn(Optional.of(TEST_VECTOR)).when(methodFieldMapper).getFloatsFromContext(parseContext, TEST_DIMENSION); - } else { - doReturn(Optional.of(TEST_BYTE_VECTOR)).when(methodFieldMapper) - .getBytesFromContext(parseContext, dataType == VectorDataType.BINARY ? TEST_DIMENSION * 8 : TEST_DIMENSION, dataType); + ); + + methodFieldMapper.parseCreateField(parseContext, dimension, dataType); + fields = document.getFields(); + assertEquals(1, fields.size()); + field1 = fields.get(0); + assertTrue(field1 instanceof VectorField); + assertEquals(Integer.parseInt(field1.fieldType().getAttributes().get(DIMENSION_FIELD_NAME)), dimension); } + } + } - methodFieldMapper.parseCreateField(parseContext, dimension, dataType); - fields = document.getFields(); - assertEquals(1, fields.size()); - field1 = fields.get(0); - assertTrue(field1 instanceof VectorField); + @SneakyThrows + public void testModelFieldMapperParseCreateField_validInput_thenDifferentFieldTypes() { + ModelDao modelDao = mock(ModelDao.class); + ModelMetadata modelMetadata = mock(ModelMetadata.class); + final MethodComponentContext methodComponentContext = new MethodComponentContext(METHOD_IVF, Collections.emptyMap()); + try ( + MockedStatic utilMockedStatic = Mockito.mockStatic(KNNVectorFieldMapperUtil.class); + MockedStatic modelUtilMockedStatic = Mockito.mockStatic(ModelUtil.class) + ) { + for (VectorDataType dataType : VectorDataType.values()) { + log.info("Vector Data Type is : {}", dataType); + SpaceType spaceType = VectorDataType.BINARY == dataType ? SpaceType.DEFAULT_BINARY : SpaceType.INNER_PRODUCT; + int dimension = adjustDimensionForIndexing(TEST_DIMENSION, dataType); + when(modelDao.getMetadata(MODEL_ID)).thenReturn(modelMetadata); + modelUtilMockedStatic.when(() -> ModelUtil.isModelCreated(modelMetadata)).thenReturn(true); + when(modelMetadata.getDimension()).thenReturn(dimension); + when(modelMetadata.getVectorDataType()).thenReturn(dataType); + when(modelMetadata.getSpaceType()).thenReturn(spaceType); + when(modelMetadata.getKnnEngine()).thenReturn(KNNEngine.FAISS); + when(modelMetadata.getMethodComponentContext()).thenReturn(methodComponentContext); + + ParseContext.Document document = new ParseContext.Document(); + ContentPath contentPath = new ContentPath(); + ParseContext parseContext = mock(ParseContext.class); + when(parseContext.doc()).thenReturn(document); + when(parseContext.path()).thenReturn(contentPath); + when(parseContext.parser()).thenReturn(createXContentParser(dataType)); + + utilMockedStatic.when(() -> KNNVectorFieldMapperUtil.useLuceneKNNVectorsFormat(Mockito.any())).thenReturn(true); + ModelFieldMapper modelFieldMapper = ModelFieldMapper.createFieldMapper( + TEST_FIELD_NAME, + TEST_FIELD_NAME, + Collections.emptyMap(), + dataType, + MODEL_ID, + FieldMapper.MultiFields.empty(), + FieldMapper.CopyTo.empty(), + new Explicit<>(true, true), + false, + false, + modelDao, + CURRENT + ); + + modelFieldMapper.parseCreateField(parseContext); + + List fields = document.getFields(); + assertEquals(1, fields.size()); + IndexableField field1 = fields.get(0); + if (dataType == VectorDataType.FLOAT) { + assertTrue(field1 instanceof KnnFloatVectorField); + assertEquals(field1.fieldType().vectorEncoding(), VectorEncoding.FLOAT32); + } else { + assertTrue(field1 instanceof KnnByteVectorField); + assertEquals(field1.fieldType().vectorEncoding(), VectorEncoding.BYTE); + } + + assertEquals(field1.fieldType().vectorDimension(), adjustDimensionForSearch(dimension, dataType)); + assertEquals( + field1.fieldType().vectorSimilarityFunction(), + SpaceType.DEFAULT.getKnnVectorSimilarityFunction().getVectorSimilarityFunction() + ); + + utilMockedStatic.when(() -> KNNVectorFieldMapperUtil.useLuceneKNNVectorsFormat(Mockito.any())).thenReturn(false); + + document = new ParseContext.Document(); + contentPath = new ContentPath(); + when(parseContext.doc()).thenReturn(document); + when(parseContext.path()).thenReturn(contentPath); + when(parseContext.parser()).thenReturn(createXContentParser(dataType)); + modelFieldMapper = ModelFieldMapper.createFieldMapper( + TEST_FIELD_NAME, + TEST_FIELD_NAME, + Collections.emptyMap(), + dataType, + MODEL_ID, + FieldMapper.MultiFields.empty(), + FieldMapper.CopyTo.empty(), + new Explicit<>(true, true), + false, + false, + modelDao, + CURRENT + ); + + modelFieldMapper.parseCreateField(parseContext); + fields = document.getFields(); + assertEquals(1, fields.size()); + field1 = fields.get(0); + assertTrue(field1 instanceof VectorField); + } } - // making sure to close the static mock to ensure that for tests running on this thread are not impacted by - // this mocking - utilMockedStatic.close(); } @SneakyThrows @@ -883,22 +965,19 @@ public void testLuceneFieldMapper_parseCreateField_docValues_withFloats() { ParseContext parseContext = mock(ParseContext.class); when(parseContext.doc()).thenReturn(document); when(parseContext.path()).thenReturn(contentPath); + when(parseContext.parser()).thenReturn(createXContentParser(VectorDataType.FLOAT)); KNNMethodConfigContext knnMethodConfigContext = KNNMethodConfigContext.builder() .vectorDataType(VectorDataType.FLOAT) .versionCreated(CURRENT) .dimension(TEST_DIMENSION) .build(); - LuceneFieldMapper luceneFieldMapper = Mockito.spy( - LuceneFieldMapper.createFieldMapper( - TEST_FIELD_NAME, - Collections.emptyMap(), - getDefaultKNNMethodContext(), - knnMethodConfigContext, - inputBuilder.build() - ) + LuceneFieldMapper luceneFieldMapper = LuceneFieldMapper.createFieldMapper( + TEST_FIELD_NAME, + Collections.emptyMap(), + getDefaultKNNMethodContext(), + knnMethodConfigContext, + inputBuilder.build() ); - doReturn(Optional.of(TEST_VECTOR)).when(luceneFieldMapper).getFloatsFromContext(parseContext, TEST_DIMENSION); - doNothing().when(luceneFieldMapper).validatePreparse(); luceneFieldMapper.parseCreateField(parseContext, TEST_DIMENSION, VectorDataType.FLOAT); // Document should have 2 fields: one for VectorField (binary doc values) and one for KnnFloatVectorField @@ -910,7 +989,7 @@ public void testLuceneFieldMapper_parseCreateField_docValues_withFloats() { VectorField vectorField; KnnFloatVectorField knnVectorField; if (field1 instanceof VectorField) { - assertTrue(field2 instanceof KnnVectorField); + assertTrue(field2 instanceof KnnFloatVectorField); vectorField = (VectorField) field1; knnVectorField = (KnnFloatVectorField) field2; } else { @@ -930,6 +1009,7 @@ public void testLuceneFieldMapper_parseCreateField_docValues_withFloats() { parseContext = mock(ParseContext.class); when(parseContext.doc()).thenReturn(document); when(parseContext.path()).thenReturn(contentPath); + when(parseContext.parser()).thenReturn(createXContentParser(VectorDataType.FLOAT)); inputBuilder.hasDocValues(false); @@ -940,18 +1020,13 @@ public void testLuceneFieldMapper_parseCreateField_docValues_withFloats() { .build(); MethodComponentContext methodComponentContext = new MethodComponentContext(METHOD_HNSW, Collections.emptyMap()); KNNMethodContext knnMethodContext = new KNNMethodContext(KNNEngine.LUCENE, SpaceType.DEFAULT, methodComponentContext); - luceneFieldMapper = Mockito.spy( - LuceneFieldMapper.createFieldMapper( - TEST_FIELD_NAME, - Collections.emptyMap(), - knnMethodContext, - knnMethodConfigContext, - inputBuilder.build() - ) + luceneFieldMapper = LuceneFieldMapper.createFieldMapper( + TEST_FIELD_NAME, + Collections.emptyMap(), + knnMethodContext, + knnMethodConfigContext, + inputBuilder.build() ); - doReturn(Optional.of(TEST_VECTOR)).when(luceneFieldMapper).getFloatsFromContext(parseContext, TEST_DIMENSION); - doNothing().when(luceneFieldMapper).validatePreparse(); - luceneFieldMapper.parseCreateField(parseContext, TEST_DIMENSION, VectorDataType.FLOAT); // Document should have 1 field: one for KnnVectorField @@ -1226,6 +1301,29 @@ private static byte[] createInitializedByteArray(int dimension, byte value) { return array; } + private XContentParser createXContentParser(final VectorDataType dataType) throws IOException { + final String vectorString; + if (dataType == VectorDataType.FLOAT) { + vectorString = Arrays.toString(TEST_VECTOR); + } else { + vectorString = Arrays.toString(TEST_BYTE_VECTOR); + } + + XContentParser parser = XContentHelper.createParser( + NamedXContentRegistry.EMPTY, + LoggingDeprecationHandler.INSTANCE, + new BytesArray(new BytesArray("{\"" + TEST_FIELD_NAME + "\":" + vectorString + "}").toBytesRef()), + MediaTypeRegistry.JSON + ); + // We need to move to 3rd token, as at start XContentParser is at null, first nextToken call will move the + // parser to { aka START_OBJECT, the next call will move it to FIELD_NAME and after that it will move to [ + // aka START_ARRAY which is what we get when we parse the vectors. + parser.nextToken(); + parser.nextToken(); + parser.nextToken(); + return parser; + } + public IndexMetadata buildIndexMetaData(String indexName, Settings settings) { return IndexMetadata.builder(indexName) .settings(settings) diff --git a/src/test/java/org/opensearch/knn/index/query/KNNWeightTests.java b/src/test/java/org/opensearch/knn/index/query/KNNWeightTests.java index 15402a148..fa4ec8001 100644 --- a/src/test/java/org/opensearch/knn/index/query/KNNWeightTests.java +++ b/src/test/java/org/opensearch/knn/index/query/KNNWeightTests.java @@ -676,76 +676,79 @@ public void testANNWithFilterQuery_whenExactSearchBinary_thenSuccess() { } public void validateANNWithFilterQuery_whenExactSearch_thenSuccess(final boolean isBinary) throws IOException { - float[] vector = new float[] { 0.1f, 0.3f }; - byte[] byteVector = new byte[] { 1, 3 }; - int filterDocId = 0; - final LeafReaderContext leafReaderContext = mock(LeafReaderContext.class); - final SegmentReader reader = mock(SegmentReader.class); - when(leafReaderContext.reader()).thenReturn(reader); - - final KNNQuery query = isBinary - ? new KNNQuery(FIELD_NAME, BYTE_QUERY_VECTOR, K, INDEX_NAME, FILTER_QUERY, null, VectorDataType.BINARY) - : new KNNQuery(FIELD_NAME, QUERY_VECTOR, K, INDEX_NAME, FILTER_QUERY, null); - final Weight filterQueryWeight = mock(Weight.class); - final Scorer filterScorer = mock(Scorer.class); - when(filterQueryWeight.scorer(leafReaderContext)).thenReturn(filterScorer); - // scorer will return 2 documents - when(filterScorer.iterator()).thenReturn(DocIdSetIterator.all(1)); - when(reader.maxDoc()).thenReturn(1); - final Bits liveDocsBits = mock(Bits.class); - when(reader.getLiveDocs()).thenReturn(liveDocsBits); - when(liveDocsBits.get(filterDocId)).thenReturn(true); - - final float boost = (float) randomDoubleBetween(0, 10, true); - final KNNWeight knnWeight = new KNNWeight(query, boost, filterQueryWeight); - final Map attributesMap = ImmutableMap.of( - KNN_ENGINE, - KNNEngine.FAISS.getName(), - SPACE_TYPE, - isBinary ? SpaceType.HAMMING.getValue() : SpaceType.L2.getValue() - ); - final FieldInfos fieldInfos = mock(FieldInfos.class); - final FieldInfo fieldInfo = mock(FieldInfo.class); - final KNNFloatVectorValues floatVectorValues = mock(KNNFloatVectorValues.class); - final KNNBinaryVectorValues binaryVectorValues = mock(KNNBinaryVectorValues.class); - when(reader.getFieldInfos()).thenReturn(fieldInfos); - when(fieldInfos.fieldInfo(any())).thenReturn(fieldInfo); - when(fieldInfo.attributes()).thenReturn(attributesMap); - if (isBinary) { - when(fieldInfo.getAttribute(SPACE_TYPE)).thenReturn(SpaceType.HAMMING.getValue()); - } else { - when(fieldInfo.getAttribute(SPACE_TYPE)).thenReturn(SpaceType.L2.getValue()); - } - when(fieldInfo.getName()).thenReturn(FIELD_NAME); - MockedStatic valuesFactoryMockedStatic = Mockito.mockStatic(KNNVectorValuesFactory.class); - if (isBinary) { - valuesFactoryMockedStatic.when(() -> KNNVectorValuesFactory.getVectorValues(fieldInfo, reader)).thenReturn(binaryVectorValues); - when(binaryVectorValues.advance(filterDocId)).thenReturn(filterDocId); - Mockito.when(binaryVectorValues.getVector()).thenReturn(byteVector); - } else { - valuesFactoryMockedStatic.when(() -> KNNVectorValuesFactory.getVectorValues(fieldInfo, reader)).thenReturn(floatVectorValues); - when(floatVectorValues.advance(filterDocId)).thenReturn(filterDocId); - Mockito.when(floatVectorValues.getVector()).thenReturn(vector); - } + try (MockedStatic valuesFactoryMockedStatic = Mockito.mockStatic(KNNVectorValuesFactory.class)) { + float[] vector = new float[] { 0.1f, 0.3f }; + byte[] byteVector = new byte[] { 1, 3 }; + int filterDocId = 0; + final LeafReaderContext leafReaderContext = mock(LeafReaderContext.class); + final SegmentReader reader = mock(SegmentReader.class); + when(leafReaderContext.reader()).thenReturn(reader); + + final KNNQuery query = isBinary + ? new KNNQuery(FIELD_NAME, BYTE_QUERY_VECTOR, K, INDEX_NAME, FILTER_QUERY, null, VectorDataType.BINARY) + : new KNNQuery(FIELD_NAME, QUERY_VECTOR, K, INDEX_NAME, FILTER_QUERY, null); + final Weight filterQueryWeight = mock(Weight.class); + final Scorer filterScorer = mock(Scorer.class); + when(filterQueryWeight.scorer(leafReaderContext)).thenReturn(filterScorer); + // scorer will return 2 documents + when(filterScorer.iterator()).thenReturn(DocIdSetIterator.all(1)); + when(reader.maxDoc()).thenReturn(1); + final Bits liveDocsBits = mock(Bits.class); + when(reader.getLiveDocs()).thenReturn(liveDocsBits); + when(liveDocsBits.get(filterDocId)).thenReturn(true); - final KNNScorer knnScorer = (KNNScorer) knnWeight.scorer(leafReaderContext); - assertNotNull(knnScorer); - final DocIdSetIterator docIdSetIterator = knnScorer.iterator(); - assertNotNull(docIdSetIterator); - assertEquals(1, docIdSetIterator.cost()); + final float boost = (float) randomDoubleBetween(0, 10, true); + final KNNWeight knnWeight = new KNNWeight(query, boost, filterQueryWeight); + final Map attributesMap = ImmutableMap.of( + KNN_ENGINE, + KNNEngine.FAISS.getName(), + SPACE_TYPE, + isBinary ? SpaceType.HAMMING.getValue() : SpaceType.L2.getValue() + ); + final FieldInfos fieldInfos = mock(FieldInfos.class); + final FieldInfo fieldInfo = mock(FieldInfo.class); + final KNNFloatVectorValues floatVectorValues = mock(KNNFloatVectorValues.class); + final KNNBinaryVectorValues binaryVectorValues = mock(KNNBinaryVectorValues.class); + when(reader.getFieldInfos()).thenReturn(fieldInfos); + when(fieldInfos.fieldInfo(any())).thenReturn(fieldInfo); + when(fieldInfo.attributes()).thenReturn(attributesMap); + if (isBinary) { + when(fieldInfo.getAttribute(SPACE_TYPE)).thenReturn(SpaceType.HAMMING.getValue()); + } else { + when(fieldInfo.getAttribute(SPACE_TYPE)).thenReturn(SpaceType.L2.getValue()); + } + when(fieldInfo.getName()).thenReturn(FIELD_NAME); - final List actualDocIds = new ArrayList<>(); - for (int docId = docIdSetIterator.nextDoc(); docId != NO_MORE_DOCS; docId = docIdSetIterator.nextDoc()) { - actualDocIds.add(docId); if (isBinary) { - assertEquals(BINARY_EXACT_SEARCH_DOC_ID_TO_SCORES.get(docId) * boost, knnScorer.score(), 0.01f); + valuesFactoryMockedStatic.when(() -> KNNVectorValuesFactory.getVectorValues(fieldInfo, reader)) + .thenReturn(binaryVectorValues); + when(binaryVectorValues.advance(filterDocId)).thenReturn(filterDocId); + Mockito.when(binaryVectorValues.getVector()).thenReturn(byteVector); } else { - assertEquals(EXACT_SEARCH_DOC_ID_TO_SCORES.get(docId) * boost, knnScorer.score(), 0.01f); + valuesFactoryMockedStatic.when(() -> KNNVectorValuesFactory.getVectorValues(fieldInfo, reader)) + .thenReturn(floatVectorValues); + when(floatVectorValues.advance(filterDocId)).thenReturn(filterDocId); + Mockito.when(floatVectorValues.getVector()).thenReturn(vector); + } + + final KNNScorer knnScorer = (KNNScorer) knnWeight.scorer(leafReaderContext); + assertNotNull(knnScorer); + final DocIdSetIterator docIdSetIterator = knnScorer.iterator(); + assertNotNull(docIdSetIterator); + assertEquals(1, docIdSetIterator.cost()); + + final List actualDocIds = new ArrayList<>(); + for (int docId = docIdSetIterator.nextDoc(); docId != NO_MORE_DOCS; docId = docIdSetIterator.nextDoc()) { + actualDocIds.add(docId); + if (isBinary) { + assertEquals(BINARY_EXACT_SEARCH_DOC_ID_TO_SCORES.get(docId) * boost, knnScorer.score(), 0.01f); + } else { + assertEquals(EXACT_SEARCH_DOC_ID_TO_SCORES.get(docId) * boost, knnScorer.score(), 0.01f); + } } + assertEquals(docIdSetIterator.cost(), actualDocIds.size()); + assertTrue(Comparators.isInOrder(actualDocIds, Comparator.naturalOrder())); } - assertEquals(docIdSetIterator.cost(), actualDocIds.size()); - assertTrue(Comparators.isInOrder(actualDocIds, Comparator.naturalOrder())); - valuesFactoryMockedStatic.close(); } @SneakyThrows @@ -886,63 +889,64 @@ public void testANNWithFilterQuery_whenExactSearchViaThresholdSetting_thenSucces */ @SneakyThrows public void testANNWithFilterQuery_whenExactSearchViaThresholdSettingOnBinaryIndex_thenSuccess() { - knnSettingsMockedStatic.when(() -> KNNSettings.getFilteredExactSearchThreshold(INDEX_NAME)).thenReturn(10); - byte[] vector = new byte[] { 1, 3 }; - int k = 1; - final int[] filterDocIds = new int[] { 0, 1, 2, 3, 4, 5 }; - - final LeafReaderContext leafReaderContext = mock(LeafReaderContext.class); - final SegmentReader reader = mock(SegmentReader.class); - when(leafReaderContext.reader()).thenReturn(reader); - when(reader.maxDoc()).thenReturn(100); - when(reader.getLiveDocs()).thenReturn(null); - final Weight filterQueryWeight = mock(Weight.class); - final Scorer filterScorer = mock(Scorer.class); - when(filterQueryWeight.scorer(leafReaderContext)).thenReturn(filterScorer); - - when(filterScorer.iterator()).thenReturn(DocIdSetIterator.all(filterDocIds.length)); - - final KNNQuery query = new KNNQuery(FIELD_NAME, BYTE_QUERY_VECTOR, k, INDEX_NAME, FILTER_QUERY, null, VectorDataType.BINARY); + try (MockedStatic vectorValuesFactoryMockedStatic = Mockito.mockStatic(KNNVectorValuesFactory.class)) { + knnSettingsMockedStatic.when(() -> KNNSettings.getFilteredExactSearchThreshold(INDEX_NAME)).thenReturn(10); + byte[] vector = new byte[] { 1, 3 }; + int k = 1; + final int[] filterDocIds = new int[] { 0, 1, 2, 3, 4, 5 }; + + final LeafReaderContext leafReaderContext = mock(LeafReaderContext.class); + final SegmentReader reader = mock(SegmentReader.class); + when(leafReaderContext.reader()).thenReturn(reader); + when(reader.maxDoc()).thenReturn(100); + when(reader.getLiveDocs()).thenReturn(null); + final Weight filterQueryWeight = mock(Weight.class); + final Scorer filterScorer = mock(Scorer.class); + when(filterQueryWeight.scorer(leafReaderContext)).thenReturn(filterScorer); + + when(filterScorer.iterator()).thenReturn(DocIdSetIterator.all(filterDocIds.length)); + + final KNNQuery query = new KNNQuery(FIELD_NAME, BYTE_QUERY_VECTOR, k, INDEX_NAME, FILTER_QUERY, null, VectorDataType.BINARY); + + final float boost = (float) randomDoubleBetween(0, 10, true); + final KNNWeight knnWeight = new KNNWeight(query, boost, filterQueryWeight); + final Map attributesMap = ImmutableMap.of( + KNN_ENGINE, + KNNEngine.FAISS.getName(), + SPACE_TYPE, + SpaceType.HAMMING.name(), + PARAMETERS, + String.format(Locale.ROOT, "{\"%s\":\"%s\"}", INDEX_DESCRIPTION_PARAMETER, "BHNSW32") + ); + final FieldInfos fieldInfos = mock(FieldInfos.class); + final FieldInfo fieldInfo = mock(FieldInfo.class); + when(reader.getFieldInfos()).thenReturn(fieldInfos); + when(fieldInfos.fieldInfo(any())).thenReturn(fieldInfo); + when(fieldInfo.attributes()).thenReturn(attributesMap); + when(fieldInfo.getAttribute(SPACE_TYPE)).thenReturn(SpaceType.HAMMING.getValue()); + when(fieldInfo.getName()).thenReturn(FIELD_NAME); - final float boost = (float) randomDoubleBetween(0, 10, true); - final KNNWeight knnWeight = new KNNWeight(query, boost, filterQueryWeight); - final Map attributesMap = ImmutableMap.of( - KNN_ENGINE, - KNNEngine.FAISS.getName(), - SPACE_TYPE, - SpaceType.HAMMING.name(), - PARAMETERS, - String.format(Locale.ROOT, "{\"%s\":\"%s\"}", INDEX_DESCRIPTION_PARAMETER, "BHNSW32") - ); - final FieldInfos fieldInfos = mock(FieldInfos.class); - final FieldInfo fieldInfo = mock(FieldInfo.class); - when(reader.getFieldInfos()).thenReturn(fieldInfos); - when(fieldInfos.fieldInfo(any())).thenReturn(fieldInfo); - when(fieldInfo.attributes()).thenReturn(attributesMap); - when(fieldInfo.getAttribute(SPACE_TYPE)).thenReturn(SpaceType.HAMMING.getValue()); - when(fieldInfo.getName()).thenReturn(FIELD_NAME); + KNNBinaryVectorValues knnBinaryVectorValues = mock(KNNBinaryVectorValues.class); - KNNBinaryVectorValues knnBinaryVectorValues = mock(KNNBinaryVectorValues.class); - MockedStatic vectorValuesFactoryMockedStatic = Mockito.mockStatic(KNNVectorValuesFactory.class); - vectorValuesFactoryMockedStatic.when(() -> KNNVectorValuesFactory.getVectorValues(fieldInfo, reader)) - .thenReturn(knnBinaryVectorValues); - when(knnBinaryVectorValues.advance(0)).thenReturn(0); - when(knnBinaryVectorValues.getVector()).thenReturn(vector); + vectorValuesFactoryMockedStatic.when(() -> KNNVectorValuesFactory.getVectorValues(fieldInfo, reader)) + .thenReturn(knnBinaryVectorValues); + when(knnBinaryVectorValues.advance(0)).thenReturn(0); + when(knnBinaryVectorValues.getVector()).thenReturn(vector); - final KNNScorer knnScorer = (KNNScorer) knnWeight.scorer(leafReaderContext); - assertNotNull(knnScorer); - final DocIdSetIterator docIdSetIterator = knnScorer.iterator(); - assertNotNull(docIdSetIterator); - assertEquals(EXACT_SEARCH_DOC_ID_TO_SCORES.size(), docIdSetIterator.cost()); + final KNNScorer knnScorer = (KNNScorer) knnWeight.scorer(leafReaderContext); + assertNotNull(knnScorer); + final DocIdSetIterator docIdSetIterator = knnScorer.iterator(); + assertNotNull(docIdSetIterator); + assertEquals(EXACT_SEARCH_DOC_ID_TO_SCORES.size(), docIdSetIterator.cost()); - final List actualDocIds = new ArrayList<>(); - for (int docId = docIdSetIterator.nextDoc(); docId != NO_MORE_DOCS; docId = docIdSetIterator.nextDoc()) { - actualDocIds.add(docId); - assertEquals(BINARY_EXACT_SEARCH_DOC_ID_TO_SCORES.get(docId) * boost, knnScorer.score(), 0.01f); + final List actualDocIds = new ArrayList<>(); + for (int docId = docIdSetIterator.nextDoc(); docId != NO_MORE_DOCS; docId = docIdSetIterator.nextDoc()) { + actualDocIds.add(docId); + assertEquals(BINARY_EXACT_SEARCH_DOC_ID_TO_SCORES.get(docId) * boost, knnScorer.score(), 0.01f); + } + assertEquals(docIdSetIterator.cost(), actualDocIds.size()); + assertTrue(Comparators.isInOrder(actualDocIds, Comparator.naturalOrder())); } - assertEquals(docIdSetIterator.cost(), actualDocIds.size()); - assertTrue(Comparators.isInOrder(actualDocIds, Comparator.naturalOrder())); - vectorValuesFactoryMockedStatic.close(); } @SneakyThrows From 3a0363f56721e411781df467142edea2c4cf70d7 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Mon, 19 Aug 2024 19:49:59 -0700 Subject: [PATCH 331/416] Disables rewrite code path to debug latency issues (#1987) (#1988) Signed-off-by: Tejas Shah (cherry picked from commit 19d5c9fb262f2a467d2460821383375544734511) --- .../org/opensearch/knn/common/featureflags/KNNFeatureFlags.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/opensearch/knn/common/featureflags/KNNFeatureFlags.java b/src/main/java/org/opensearch/knn/common/featureflags/KNNFeatureFlags.java index 21160fc2d..9b4a5ba7e 100644 --- a/src/main/java/org/opensearch/knn/common/featureflags/KNNFeatureFlags.java +++ b/src/main/java/org/opensearch/knn/common/featureflags/KNNFeatureFlags.java @@ -26,7 +26,7 @@ public class KNNFeatureFlags { // Feature flags private static final String KNN_LAUNCH_QUERY_REWRITE_ENABLED = "knn.feature.query.rewrite.enabled"; - private static final boolean KNN_LAUNCH_QUERY_REWRITE_ENABLED_DEFAULT = true; + private static final boolean KNN_LAUNCH_QUERY_REWRITE_ENABLED_DEFAULT = false; @VisibleForTesting public static final Setting KNN_LAUNCH_QUERY_REWRITE_ENABLED_SETTING = Setting.boolSetting( From 81e9587176aacd92c9dc75f85caa10e3f75711f1 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Tue, 20 Aug 2024 13:25:40 -0700 Subject: [PATCH 332/416] Add quantization state cache (#1960) (#1990) * Add quantization state cache Signed-off-by: Ryan Bogan * Fix compile on tests and add changelog Signed-off-by: Ryan Bogan * Add javadocs Signed-off-by: Ryan Bogan * Change cache implementation to use Cache class Signed-off-by: Ryan Bogan * Make add method synchronized Signed-off-by: Ryan Bogan * Make get instance method synchronized Signed-off-by: Ryan Bogan * Fix compile Signed-off-by: Ryan Bogan * Remove lock Signed-off-by: Ryan Bogan * Add more unit tests Signed-off-by: Ryan Bogan * Add removal listener to update when cache is full Signed-off-by: Ryan Bogan * Add removal listener, weigher, and fix tests Signed-off-by: Ryan Bogan * Decouple from cluster service Signed-off-by: Ryan Bogan * Add testing Signed-off-by: Ryan Bogan * Change to SneakyThrows Signed-off-by: Ryan Bogan * Fix flaky tests Signed-off-by: Ryan Bogan * Fix flaky tests Signed-off-by: Ryan Bogan * Move test file to same package as main class to fix flaky test Signed-off-by: Ryan Bogan * Fix spotless Signed-off-by: Ryan Bogan --------- Signed-off-by: Ryan Bogan (cherry picked from commit 433cd70694af3ff0616891e2aa7e5585409dfd61) Co-authored-by: Ryan Bogan --- CHANGELOG.md | 1 + .../org/opensearch/knn/index/KNNSettings.java | 65 ++- .../QuantizationStateCache.java | 125 +++++ .../QuantizationStateCacheTests.java | 450 ++++++++++++++++++ 4 files changed, 640 insertions(+), 1 deletion(-) create mode 100644 src/main/java/org/opensearch/knn/quantization/models/quantizationState/QuantizationStateCache.java create mode 100644 src/test/java/org/opensearch/knn/quantization/models/quantizationState/QuantizationStateCacheTests.java diff --git a/CHANGELOG.md b/CHANGELOG.md index ccac8d7c3..e60ce2667 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -39,3 +39,4 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), * Restructure mappers to better handle null cases and avoid branching in parsing [#1939](https://github.com/opensearch-project/k-NN/pull/1939) * Added Quantization Framework and implemented 1Bit and multibit quantizer[#1889](https://github.com/opensearch-project/k-NN/issues/1889) * Encapsulate dimension, vector data type validation/processing inside Library [#1957](https://github.com/opensearch-project/k-NN/pull/1957) +* Add quantization state cache [#1960](https://github.com/opensearch-project/k-NN/pull/1960) diff --git a/src/main/java/org/opensearch/knn/index/KNNSettings.java b/src/main/java/org/opensearch/knn/index/KNNSettings.java index 4ced38b38..73f43d3d1 100644 --- a/src/main/java/org/opensearch/knn/index/KNNSettings.java +++ b/src/main/java/org/opensearch/knn/index/KNNSettings.java @@ -24,6 +24,7 @@ import org.opensearch.knn.index.memory.NativeMemoryCacheManager; import org.opensearch.knn.index.memory.NativeMemoryCacheManagerDto; import org.opensearch.knn.index.util.IndexHyperParametersUtil; +import org.opensearch.knn.quantization.models.quantizationState.QuantizationStateCache; import org.opensearch.monitor.jvm.JvmInfo; import org.opensearch.monitor.os.OsProbe; @@ -88,6 +89,8 @@ public class KNNSettings { * for native engines. */ public static final String KNN_USE_LUCENE_VECTOR_FORMAT_ENABLED = "knn.use.format.enabled"; + public static final String QUANTIZATION_STATE_CACHE_SIZE_LIMIT = "knn.quantization.cache.size.limit"; + public static final String QUANTIZATION_STATE_CACHE_EXPIRY_TIME_MINUTES = "knn.quantization.cache.expiry.minutes"; /** * Default setting values @@ -106,6 +109,11 @@ public class KNNSettings { public static final String KNN_DEFAULT_VECTOR_STREAMING_MEMORY_LIMIT_PCT = "1%"; public static final Integer ADVANCED_FILTERED_EXACT_SEARCH_THRESHOLD_DEFAULT_VALUE = -1; + public static final Integer KNN_DEFAULT_QUANTIZATION_STATE_CACHE_SIZE_LIMIT_PERCENTAGE = 5; // By default, set aside 5% of the JVM for + // the limit + public static final Integer KNN_MAX_QUANTIZATION_STATE_CACHE_SIZE_LIMIT_PERCENTAGE = 10; // Quantization state cache limit cannot exceed + // 10% of the JVM heap + public static final Integer KNN_DEFAULT_QUANTIZATION_STATE_CACHE_EXPIRY_TIME_MINUTES = 60; /** * Settings Definition @@ -272,6 +280,44 @@ public class KNNSettings { NodeScope ); + /* + * Quantization state cache settings + */ + public static final Setting QUANTIZATION_STATE_CACHE_SIZE_LIMIT_SETTING = new Setting( + QUANTIZATION_STATE_CACHE_SIZE_LIMIT, + percentageAsString(KNN_DEFAULT_QUANTIZATION_STATE_CACHE_SIZE_LIMIT_PERCENTAGE), + (s) -> { + ByteSizeValue userDefinedLimit = parseBytesSizeValueOrHeapRatio(s, QUANTIZATION_STATE_CACHE_SIZE_LIMIT); + + // parseBytesSizeValueOrHeapRatio will make sure that the value entered falls between 0 and 100% of the + // JVM heap. However, we want the maximum percentage of the heap to be much smaller. So, we add + // some additional validation here before returning + ByteSizeValue jvmHeapSize = JvmInfo.jvmInfo().getMem().getHeapMax(); + if ((userDefinedLimit.getKbFrac() / jvmHeapSize.getKbFrac()) > percentageAsFraction( + KNN_MAX_QUANTIZATION_STATE_CACHE_SIZE_LIMIT_PERCENTAGE + )) { + throw new OpenSearchParseException( + "{} ({} KB) cannot exceed {}% of the heap ({} KB).", + QUANTIZATION_STATE_CACHE_SIZE_LIMIT, + userDefinedLimit.getKb(), + KNN_MAX_QUANTIZATION_STATE_CACHE_SIZE_LIMIT_PERCENTAGE, + jvmHeapSize.getKb() + ); + } + + return userDefinedLimit; + }, + NodeScope, + Dynamic + ); + + public static final Setting QUANTIZATION_STATE_CACHE_EXPIRY_TIME_MINUTES_SETTING = Setting.positiveTimeSetting( + QUANTIZATION_STATE_CACHE_EXPIRY_TIME_MINUTES, + TimeValue.timeValueMinutes(KNN_DEFAULT_QUANTIZATION_STATE_CACHE_EXPIRY_TIME_MINUTES), + NodeScope, + Dynamic + ); + /** * Dynamic settings */ @@ -349,6 +395,13 @@ private void setSettingsUpdateConsumers() { NativeMemoryCacheManager.getInstance().rebuildCache(builder.build()); }, Stream.concat(dynamicCacheSettings.values().stream(), FEATURE_FLAGS.values().stream()).collect(Collectors.toUnmodifiableList())); + clusterService.getClusterSettings().addSettingsUpdateConsumer(QUANTIZATION_STATE_CACHE_SIZE_LIMIT_SETTING, it -> { + QuantizationStateCache.getInstance().setMaxCacheSizeInKB(it.getKb()); + QuantizationStateCache.getInstance().rebuildCache(); + }); + clusterService.getClusterSettings().addSettingsUpdateConsumer(QUANTIZATION_STATE_CACHE_EXPIRY_TIME_MINUTES_SETTING, it -> { + QuantizationStateCache.getInstance().rebuildCache(); + }); } /** @@ -400,6 +453,14 @@ private Setting getSetting(String key) { return KNN_USE_LUCENE_VECTOR_FORMAT_ENABLED_SETTING; } + if (QUANTIZATION_STATE_CACHE_SIZE_LIMIT.equals(key)) { + return QUANTIZATION_STATE_CACHE_SIZE_LIMIT_SETTING; + } + + if (QUANTIZATION_STATE_CACHE_EXPIRY_TIME_MINUTES.equals(key)) { + return QUANTIZATION_STATE_CACHE_EXPIRY_TIME_MINUTES_SETTING; + } + throw new IllegalArgumentException("Cannot find setting by key [" + key + "]"); } @@ -419,7 +480,9 @@ public List> getSettings() { ADVANCED_FILTERED_EXACT_SEARCH_THRESHOLD_SETTING, KNN_FAISS_AVX2_DISABLED_SETTING, KNN_VECTOR_STREAMING_MEMORY_LIMIT_PCT_SETTING, - KNN_USE_LUCENE_VECTOR_FORMAT_ENABLED_SETTING + KNN_USE_LUCENE_VECTOR_FORMAT_ENABLED_SETTING, + QUANTIZATION_STATE_CACHE_SIZE_LIMIT_SETTING, + QUANTIZATION_STATE_CACHE_EXPIRY_TIME_MINUTES_SETTING ); return Stream.concat(settings.stream(), Stream.concat(getFeatureFlags().stream(), dynamicCacheSettings.values().stream())) .collect(Collectors.toList()); diff --git a/src/main/java/org/opensearch/knn/quantization/models/quantizationState/QuantizationStateCache.java b/src/main/java/org/opensearch/knn/quantization/models/quantizationState/QuantizationStateCache.java new file mode 100644 index 000000000..ba26d517d --- /dev/null +++ b/src/main/java/org/opensearch/knn/quantization/models/quantizationState/QuantizationStateCache.java @@ -0,0 +1,125 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.quantization.models.quantizationState; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.cache.Cache; +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.RemovalCause; +import com.google.common.cache.RemovalNotification; +import lombok.Getter; +import lombok.Setter; +import lombok.extern.log4j.Log4j2; +import org.opensearch.common.unit.TimeValue; +import org.opensearch.core.common.unit.ByteSizeValue; +import org.opensearch.knn.index.KNNSettings; + +import java.io.IOException; +import java.time.Instant; +import java.util.concurrent.TimeUnit; + +import static org.opensearch.knn.index.KNNSettings.QUANTIZATION_STATE_CACHE_EXPIRY_TIME_MINUTES; +import static org.opensearch.knn.index.KNNSettings.QUANTIZATION_STATE_CACHE_SIZE_LIMIT; + +/** + * A thread-safe singleton cache that contains quantization states. + */ +@Log4j2 +public class QuantizationStateCache { + + private static volatile QuantizationStateCache instance; + private Cache cache; + @Getter + @Setter + private long maxCacheSizeInKB; + @Getter + private Instant evictedDueToSizeAt; + + @VisibleForTesting + QuantizationStateCache() { + maxCacheSizeInKB = ((ByteSizeValue) KNNSettings.state().getSettingValue(QUANTIZATION_STATE_CACHE_SIZE_LIMIT)).getKb(); + buildCache(); + } + + /** + * Gets the singleton instance of the cache. + * @return QuantizationStateCache + */ + public static QuantizationStateCache getInstance() { + if (instance == null) { + synchronized (QuantizationStateCache.class) { + if (instance == null) { + instance = new QuantizationStateCache(); + } + } + } + return instance; + } + + private void buildCache() { + this.cache = CacheBuilder.newBuilder().concurrencyLevel(1).maximumWeight(maxCacheSizeInKB).weigher((k, v) -> { + try { + return ((QuantizationState) v).toByteArray().length; + } catch (IOException e) { + throw new RuntimeException(e); + } + }) + .expireAfterAccess( + ((TimeValue) KNNSettings.state().getSettingValue(QUANTIZATION_STATE_CACHE_EXPIRY_TIME_MINUTES)).getMinutes(), + TimeUnit.MINUTES + ) + .removalListener(this::onRemoval) + .build(); + } + + public synchronized void rebuildCache() { + clear(); + buildCache(); + } + + /** + * Retrieves the quantization state associated with a given field name. + * @param fieldName The name of the field. + * @return The associated QuantizationState, or null if not present. + */ + public QuantizationState getQuantizationState(String fieldName) { + return cache.getIfPresent(fieldName); + } + + /** + * Adds or updates a quantization state in the cache. + * @param fieldName The name of the field. + * @param quantizationState The quantization state to store. + */ + public void addQuantizationState(String fieldName, QuantizationState quantizationState) { + cache.put(fieldName, quantizationState); + } + + /** + * Removes the quantization state associated with a given field name. + * @param fieldName The name of the field. + */ + public void evict(String fieldName) { + cache.invalidate(fieldName); + } + + private void onRemoval(RemovalNotification removalNotification) { + if (RemovalCause.SIZE == removalNotification.getCause()) { + updateEvictedDueToSizeAt(); + } + } + + private void updateEvictedDueToSizeAt() { + evictedDueToSizeAt = Instant.now(); + } + + /** + * Clears all entries from the cache. + */ + public void clear() { + cache.invalidateAll(); + } +} diff --git a/src/test/java/org/opensearch/knn/quantization/models/quantizationState/QuantizationStateCacheTests.java b/src/test/java/org/opensearch/knn/quantization/models/quantizationState/QuantizationStateCacheTests.java new file mode 100644 index 000000000..e5381aec7 --- /dev/null +++ b/src/test/java/org/opensearch/knn/quantization/models/quantizationState/QuantizationStateCacheTests.java @@ -0,0 +1,450 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.quantization.models.quantizationState; + +import com.google.common.collect.ImmutableSet; +import lombok.SneakyThrows; +import org.opensearch.client.Client; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.settings.ClusterSettings; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.unit.TimeValue; +import org.opensearch.knn.KNNTestCase; +import org.opensearch.knn.index.KNNSettings; +import org.opensearch.knn.quantization.models.quantizationParams.ScalarQuantizationParams; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import static org.opensearch.knn.index.KNNSettings.QUANTIZATION_STATE_CACHE_EXPIRY_TIME_MINUTES_SETTING; +import static org.opensearch.knn.index.KNNSettings.QUANTIZATION_STATE_CACHE_SIZE_LIMIT_SETTING; +import static org.opensearch.knn.quantization.enums.ScalarQuantizationType.ONE_BIT; + +public class QuantizationStateCacheTests extends KNNTestCase { + + @SneakyThrows + public void testSingleThreadedAddAndRetrieve() { + String fieldName = "singleThreadField"; + QuantizationState state = new OneBitScalarQuantizationState( + new ScalarQuantizationParams(ONE_BIT), + new float[] { 1.2f, 2.3f, 3.4f } + ); + + String cacheSize = "10%"; + TimeValue expiry = TimeValue.timeValueMinutes(30); + + Settings settings = Settings.builder() + .put(QUANTIZATION_STATE_CACHE_SIZE_LIMIT_SETTING.getKey(), cacheSize) + .put(QUANTIZATION_STATE_CACHE_EXPIRY_TIME_MINUTES_SETTING.getKey(), expiry) + .build(); + ClusterSettings clusterSettings = new ClusterSettings( + settings, + ImmutableSet.of(QUANTIZATION_STATE_CACHE_SIZE_LIMIT_SETTING, QUANTIZATION_STATE_CACHE_EXPIRY_TIME_MINUTES_SETTING) + ); + ClusterService clusterService = mock(ClusterService.class); + when(clusterService.getClusterSettings()).thenReturn(clusterSettings); + when(clusterService.getSettings()).thenReturn(settings); + + QuantizationStateCache cache = QuantizationStateCache.getInstance(); + clusterService.getClusterSettings().applySettings(settings); + + // Add state + cache.addQuantizationState(fieldName, state); + + QuantizationState retrievedState = cache.getQuantizationState(fieldName); + assertNotNull("State should be retrieved successfully", retrievedState); + assertSame("Retrieved state should be the same instance as the one added", state, retrievedState); + } + + @SneakyThrows + public void testMultiThreadedAddAndRetrieve() { + int threadCount = 10; + ExecutorService executorService = Executors.newFixedThreadPool(threadCount); + CountDownLatch latch = new CountDownLatch(threadCount); + String fieldName = "multiThreadField"; + QuantizationState state = new OneBitScalarQuantizationState( + new ScalarQuantizationParams(ONE_BIT), + new float[] { 1.2f, 2.3f, 3.4f } + ); + String cacheSize = "10%"; + TimeValue expiry = TimeValue.timeValueMinutes(30); + + Settings settings = Settings.builder() + .put(QUANTIZATION_STATE_CACHE_SIZE_LIMIT_SETTING.getKey(), cacheSize) + .put(QUANTIZATION_STATE_CACHE_EXPIRY_TIME_MINUTES_SETTING.getKey(), expiry) + .build(); + ClusterSettings clusterSettings = new ClusterSettings( + settings, + ImmutableSet.of(QUANTIZATION_STATE_CACHE_SIZE_LIMIT_SETTING, QUANTIZATION_STATE_CACHE_EXPIRY_TIME_MINUTES_SETTING) + ); + ClusterService clusterService = mock(ClusterService.class); + when(clusterService.getClusterSettings()).thenReturn(clusterSettings); + when(clusterService.getSettings()).thenReturn(settings); + + QuantizationStateCache cache = QuantizationStateCache.getInstance(); + clusterService.getClusterSettings().applySettings(settings); + + // Add state from multiple threads + for (int i = 0; i < threadCount; i++) { + executorService.submit(() -> { + try { + cache.addQuantizationState(fieldName, state); + } finally { + latch.countDown(); + } + }); + } + + // Wait for all threads to finish + latch.await(); + executorService.shutdown(); + + QuantizationState retrievedState = cache.getQuantizationState(fieldName); + assertNotNull("State should be retrieved successfully", retrievedState); + assertSame("Retrieved state should be the same instance as the one added", state, retrievedState); + } + + @SneakyThrows + public void testMultiThreadedEvict() { + int threadCount = 10; + ExecutorService executorService = Executors.newFixedThreadPool(threadCount); + CountDownLatch latch = new CountDownLatch(threadCount); + String fieldName = "multiThreadEvictField"; + QuantizationState state = new OneBitScalarQuantizationState( + new ScalarQuantizationParams(ONE_BIT), + new float[] { 1.2f, 2.3f, 3.4f } + ); + String cacheSize = "10%"; + TimeValue expiry = TimeValue.timeValueMinutes(30); + + Settings settings = Settings.builder() + .put(QUANTIZATION_STATE_CACHE_SIZE_LIMIT_SETTING.getKey(), cacheSize) + .put(QUANTIZATION_STATE_CACHE_EXPIRY_TIME_MINUTES_SETTING.getKey(), expiry) + .build(); + ClusterSettings clusterSettings = new ClusterSettings( + settings, + ImmutableSet.of(QUANTIZATION_STATE_CACHE_SIZE_LIMIT_SETTING, QUANTIZATION_STATE_CACHE_EXPIRY_TIME_MINUTES_SETTING) + ); + ClusterService clusterService = mock(ClusterService.class); + when(clusterService.getClusterSettings()).thenReturn(clusterSettings); + when(clusterService.getSettings()).thenReturn(settings); + + QuantizationStateCache cache = QuantizationStateCache.getInstance(); + + clusterService.getClusterSettings().applySettings(settings); + + cache.addQuantizationState(fieldName, state); + + // Evict state from multiple threads + for (int i = 0; i < threadCount; i++) { + executorService.submit(() -> { + try { + cache.evict(fieldName); + } finally { + latch.countDown(); + } + }); + } + + // Wait for all threads to finish + latch.await(); + executorService.shutdown(); + + QuantizationState retrievedState = cache.getQuantizationState(fieldName); + assertNull("State should be null", retrievedState); + } + + @SneakyThrows + public void testConcurrentAddAndEvict() { + int threadCount = 10; + ExecutorService executorService = Executors.newFixedThreadPool(threadCount); + CountDownLatch latch = new CountDownLatch(threadCount); + String fieldName = "concurrentAddEvictField"; + QuantizationState state = new OneBitScalarQuantizationState( + new ScalarQuantizationParams(ONE_BIT), + new float[] { 1.2f, 2.3f, 3.4f } + ); + String cacheSize = "10%"; + TimeValue expiry = TimeValue.timeValueMinutes(30); + + Settings settings = Settings.builder() + .put(QUANTIZATION_STATE_CACHE_SIZE_LIMIT_SETTING.getKey(), cacheSize) + .put(QUANTIZATION_STATE_CACHE_EXPIRY_TIME_MINUTES_SETTING.getKey(), expiry) + .build(); + ClusterSettings clusterSettings = new ClusterSettings( + settings, + ImmutableSet.of(QUANTIZATION_STATE_CACHE_SIZE_LIMIT_SETTING, QUANTIZATION_STATE_CACHE_EXPIRY_TIME_MINUTES_SETTING) + ); + ClusterService clusterService = mock(ClusterService.class); + when(clusterService.getClusterSettings()).thenReturn(clusterSettings); + when(clusterService.getSettings()).thenReturn(settings); + + QuantizationStateCache cache = QuantizationStateCache.getInstance(); + clusterService.getClusterSettings().applySettings(settings); + + // Concurrently add and evict state from multiple threads + for (int i = 0; i < threadCount; i++) { + if (i % 2 == 0) { + executorService.submit(() -> { + try { + cache.addQuantizationState(fieldName, state); + } finally { + latch.countDown(); + } + }); + } else { + executorService.submit(() -> { + try { + cache.evict(fieldName); + } finally { + latch.countDown(); + } + }); + } + + } + + // Wait for all threads to finish + latch.await(); + executorService.shutdown(); + + // Since operations are concurrent, we can't be sure of the final state, but we can assert that the cache handles it gracefully + QuantizationState retrievedState = cache.getQuantizationState(fieldName); + assertTrue("Final state should be either null or the added state", retrievedState == null || retrievedState == state); + } + + @SneakyThrows + public void testMultipleThreadedCacheClear() { + int threadCount = 10; + ExecutorService executorService = Executors.newFixedThreadPool(threadCount); + CountDownLatch latch = new CountDownLatch(threadCount); + String fieldName = "multiThreadField"; + QuantizationState state = new OneBitScalarQuantizationState( + new ScalarQuantizationParams(ONE_BIT), + new float[] { 1.2f, 2.3f, 3.4f } + ); + String cacheSize = "10%"; + TimeValue expiry = TimeValue.timeValueMinutes(30); + + Settings settings = Settings.builder() + .put(QUANTIZATION_STATE_CACHE_SIZE_LIMIT_SETTING.getKey(), cacheSize) + .put(QUANTIZATION_STATE_CACHE_EXPIRY_TIME_MINUTES_SETTING.getKey(), expiry) + .build(); + ClusterSettings clusterSettings = new ClusterSettings( + settings, + ImmutableSet.of(QUANTIZATION_STATE_CACHE_SIZE_LIMIT_SETTING, QUANTIZATION_STATE_CACHE_EXPIRY_TIME_MINUTES_SETTING) + ); + ClusterService clusterService = mock(ClusterService.class); + when(clusterService.getClusterSettings()).thenReturn(clusterSettings); + when(clusterService.getSettings()).thenReturn(settings); + + QuantizationStateCache cache = QuantizationStateCache.getInstance(); + clusterService.getClusterSettings().applySettings(settings); + cache.addQuantizationState(fieldName, state); + + // Clear cache from multiple threads + for (int i = 0; i < threadCount; i++) { + executorService.submit(() -> { + try { + cache.clear(); + } finally { + latch.countDown(); + } + }); + } + + // Wait for all threads to finish + latch.await(); + executorService.shutdown(); + + QuantizationState retrievedState = cache.getQuantizationState(fieldName); + assertNull("State should be null", retrievedState); + } + + @SneakyThrows + public void testRebuild() { + int threadCount = 10; + ExecutorService executorService = Executors.newFixedThreadPool(threadCount); + CountDownLatch latch = new CountDownLatch(threadCount); + String fieldName = "rebuildField"; + QuantizationState state = new OneBitScalarQuantizationState( + new ScalarQuantizationParams(ONE_BIT), + new float[] { 1.2f, 2.3f, 3.4f } + ); + String cacheSize = "10%"; + TimeValue expiry = TimeValue.timeValueMinutes(30); + + Settings settings = Settings.builder() + .put(QUANTIZATION_STATE_CACHE_SIZE_LIMIT_SETTING.getKey(), cacheSize) + .put(QUANTIZATION_STATE_CACHE_EXPIRY_TIME_MINUTES_SETTING.getKey(), expiry) + .build(); + ClusterSettings clusterSettings = new ClusterSettings( + settings, + ImmutableSet.of(QUANTIZATION_STATE_CACHE_SIZE_LIMIT_SETTING, QUANTIZATION_STATE_CACHE_EXPIRY_TIME_MINUTES_SETTING) + ); + ClusterService clusterService = mock(ClusterService.class); + when(clusterService.getClusterSettings()).thenReturn(clusterSettings); + when(clusterService.getSettings()).thenReturn(settings); + + QuantizationStateCache cache = QuantizationStateCache.getInstance(); + cache.addQuantizationState(fieldName, state); + + // Rebuild cache from multiple threads + for (int i = 0; i < threadCount; i++) { + executorService.submit(() -> { + try { + cache.rebuildCache(); + } finally { + latch.countDown(); + } + }); + } + + // Wait for all threads to finish + latch.await(); + executorService.shutdown(); + + QuantizationState retrievedState = cache.getQuantizationState(fieldName); + assertNull("State should be null", retrievedState); + } + + @SneakyThrows + public void testRebuildOnCacheSizeSettingsChange() { + int threadCount = 10; + ExecutorService executorService = Executors.newFixedThreadPool(threadCount); + CountDownLatch latch = new CountDownLatch(threadCount); + String fieldName = "rebuildField"; + QuantizationState state = new OneBitScalarQuantizationState( + new ScalarQuantizationParams(ONE_BIT), + new float[] { 1.2f, 2.3f, 3.4f } + ); + + Settings settings = Settings.builder().build(); + ClusterSettings clusterSettings = new ClusterSettings( + settings, + ImmutableSet.of(QUANTIZATION_STATE_CACHE_SIZE_LIMIT_SETTING, QUANTIZATION_STATE_CACHE_EXPIRY_TIME_MINUTES_SETTING) + ); + ClusterService clusterService = mock(ClusterService.class); + when(clusterService.getClusterSettings()).thenReturn(clusterSettings); + when(clusterService.getSettings()).thenReturn(settings); + + Client client = mock(Client.class); + + KNNSettings.state().initialize(client, clusterService); + + QuantizationStateCache cache = QuantizationStateCache.getInstance(); + cache.rebuildCache(); + long maxCacheSizeInKB = cache.getMaxCacheSizeInKB(); + cache.addQuantizationState(fieldName, state); + + String newCacheSize = "10%"; + + Settings newSettings = Settings.builder().put(QUANTIZATION_STATE_CACHE_SIZE_LIMIT_SETTING.getKey(), newCacheSize).build(); + + // Rebuild cache from multiple threads + for (int i = 0; i < threadCount; i++) { + executorService.submit(() -> { + try { + clusterService.getClusterSettings().applySettings(newSettings); + } finally { + latch.countDown(); + } + }); + } + + // Wait for all threads to finish + latch.await(); + executorService.shutdown(); + + QuantizationState retrievedState = cache.getQuantizationState(fieldName); + assertNull("State should be null", retrievedState); + assertNotEquals(maxCacheSizeInKB, cache.getMaxCacheSizeInKB()); + } + + @SneakyThrows + public void testRebuildOnTimeExpirySettingsChange() { + int threadCount = 10; + ExecutorService executorService = Executors.newFixedThreadPool(threadCount); + CountDownLatch latch = new CountDownLatch(threadCount); + String fieldName = "rebuildField"; + QuantizationState state = new OneBitScalarQuantizationState( + new ScalarQuantizationParams(ONE_BIT), + new float[] { 1.2f, 2.3f, 3.4f } + ); + + Settings settings = Settings.builder().build(); + ClusterSettings clusterSettings = new ClusterSettings( + settings, + ImmutableSet.of(QUANTIZATION_STATE_CACHE_SIZE_LIMIT_SETTING, QUANTIZATION_STATE_CACHE_EXPIRY_TIME_MINUTES_SETTING) + ); + ClusterService clusterService = mock(ClusterService.class); + when(clusterService.getClusterSettings()).thenReturn(clusterSettings); + when(clusterService.getSettings()).thenReturn(settings); + + Client client = mock(Client.class); + + KNNSettings.state().initialize(client, clusterService); + + QuantizationStateCache cache = QuantizationStateCache.getInstance(); + cache.addQuantizationState(fieldName, state); + + TimeValue newExpiry = TimeValue.timeValueMinutes(30); + + Settings newSettings = Settings.builder().put(QUANTIZATION_STATE_CACHE_EXPIRY_TIME_MINUTES_SETTING.getKey(), newExpiry).build(); + + // Rebuild cache from multiple threads + for (int i = 0; i < threadCount; i++) { + executorService.submit(() -> { + try { + clusterService.getClusterSettings().applySettings(newSettings); + } finally { + latch.countDown(); + } + }); + } + + // Wait for all threads to finish + latch.await(); + executorService.shutdown(); + + QuantizationState retrievedState = cache.getQuantizationState(fieldName); + assertNull("State should be null", retrievedState); + } + + public void testCacheEvictionDueToSize() { + String fieldName = "evictionField"; + // States have size of slightly over 500 bytes so that adding two will reach the max size of 1 kb for the cache + int arrayLength = 112; + float[] arr = new float[arrayLength]; + float[] arr2 = new float[arrayLength]; + for (int i = 0; i < arrayLength; i++) { + arr[i] = i; + arr[i] = i + 1; + } + QuantizationState state = new OneBitScalarQuantizationState(new ScalarQuantizationParams(ONE_BIT), arr); + QuantizationState state2 = new OneBitScalarQuantizationState(new ScalarQuantizationParams(ONE_BIT), arr2); + long cacheSize = 1; + Settings settings = Settings.builder().build(); + ClusterSettings clusterSettings = new ClusterSettings( + settings, + ImmutableSet.of(QUANTIZATION_STATE_CACHE_SIZE_LIMIT_SETTING, QUANTIZATION_STATE_CACHE_EXPIRY_TIME_MINUTES_SETTING) + ); + ClusterService clusterService = mock(ClusterService.class); + when(clusterService.getClusterSettings()).thenReturn(clusterSettings); + when(clusterService.getSettings()).thenReturn(settings); + + QuantizationStateCache cache = new QuantizationStateCache(); + cache.setMaxCacheSizeInKB(cacheSize); + cache.rebuildCache(); + cache.addQuantizationState(fieldName, state); + cache.addQuantizationState(fieldName, state2); + cache.clear(); + assertNotNull(cache.getEvictedDueToSizeAt()); + } +} From c633d3fac8a4bfb5918206ed00c5f04c836e5037 Mon Sep 17 00:00:00 2001 From: Vijayan Balasubramanian Date: Tue, 20 Aug 2024 13:44:03 -0700 Subject: [PATCH 333/416] Add script_fields context to KNNAllowlist (#1917) (#1966) Include script_fields context to existing supported context for knn methods. Added test cases for method and doc values. Signed-off-by: Vijayan Balasubramanian --- CHANGELOG.md | 1 + .../plugin/script/KNNAllowlistExtension.java | 3 + .../knn/integ/PainlessScriptFieldsIT.java | 131 ++++++++++++++++++ .../knn/integ/PainlessScriptHelper.java | 58 ++++++++ ...riptIT.java => PainlessScriptScoreIT.java} | 98 ++----------- .../org/opensearch/knn/KNNRestTestCase.java | 56 ++++++++ 6 files changed, 259 insertions(+), 88 deletions(-) create mode 100644 src/test/java/org/opensearch/knn/integ/PainlessScriptFieldsIT.java create mode 100644 src/test/java/org/opensearch/knn/integ/PainlessScriptHelper.java rename src/test/java/org/opensearch/knn/integ/{PainlessScriptIT.java => PainlessScriptScoreIT.java} (91%) diff --git a/CHANGELOG.md b/CHANGELOG.md index e60ce2667..3ab1861c8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), * Add script_fields context to KNNAllowlist [#1917] (https://github.com/opensearch-project/k-NN/pull/1917) * Fix graph merge stats size calculation [#1844](https://github.com/opensearch-project/k-NN/pull/1844) * Disallow a vector field to have an invalid character for a physical file name. [#1936](https://github.com/opensearch-project/k-NN/pull/1936) +* Add script_fields context to KNNAllowlist [#1917] (https://github.com/opensearch-project/k-NN/pull/1917) ### Infrastructure ### Documentation ### Maintenance diff --git a/src/main/java/org/opensearch/knn/plugin/script/KNNAllowlistExtension.java b/src/main/java/org/opensearch/knn/plugin/script/KNNAllowlistExtension.java index 959063d61..47b99af17 100644 --- a/src/main/java/org/opensearch/knn/plugin/script/KNNAllowlistExtension.java +++ b/src/main/java/org/opensearch/knn/plugin/script/KNNAllowlistExtension.java @@ -9,6 +9,7 @@ import org.opensearch.painless.spi.PainlessExtension; import org.opensearch.painless.spi.Whitelist; import org.opensearch.painless.spi.WhitelistLoader; +import org.opensearch.script.FieldScript; import org.opensearch.script.ScoreScript; import org.opensearch.script.ScriptContext; import org.opensearch.script.ScriptedMetricAggContexts; @@ -33,6 +34,8 @@ public Map, List> getContextWhitelists() { ScriptedMetricAggContexts.CombineScript.CONTEXT, allowLists, ScriptedMetricAggContexts.ReduceScript.CONTEXT, + allowLists, + FieldScript.CONTEXT, allowLists ); } diff --git a/src/test/java/org/opensearch/knn/integ/PainlessScriptFieldsIT.java b/src/test/java/org/opensearch/knn/integ/PainlessScriptFieldsIT.java new file mode 100644 index 000000000..85f980a25 --- /dev/null +++ b/src/test/java/org/opensearch/knn/integ/PainlessScriptFieldsIT.java @@ -0,0 +1,131 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.integ; + +import lombok.SneakyThrows; +import org.apache.http.util.EntityUtils; +import org.opensearch.client.Request; +import org.opensearch.client.Response; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.knn.KNNRestTestCase; +import org.opensearch.knn.KNNResult; +import org.opensearch.knn.index.mapper.KNNVectorFieldMapper; +import org.opensearch.knn.integ.PainlessScriptHelper.MappingProperty; +import org.opensearch.script.Script; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; + +import static org.opensearch.knn.integ.PainlessScriptHelper.createMapping; + +// PainlesScriptScoreIT already tests every similarity methods with different field type. Hence, +// we don't have to recreate all tests for script_fields. From implementation point of view, +// it is clear if similarity method is supported by script_score, then same is applicable for script_fields +// provided script_fields context is supported. Hence, we test for one similarity method to verify that script_fields +// context is supported by this plugin. +public final class PainlessScriptFieldsIT extends KNNRestTestCase { + + private static final String NUMERIC_INDEX_FIELD_NAME = "price"; + + private void buildTestIndex(final Map knnDocuments) throws Exception { + List properties = buildMappingProperties(); + buildTestIndex(knnDocuments, properties); + } + + private void buildTestIndex(final Map knnDocuments, final List properties) throws Exception { + createKnnIndex(INDEX_NAME, createMapping(properties)); + for (Map.Entry data : knnDocuments.entrySet()) { + addKnnDoc(INDEX_NAME, data.getKey(), FIELD_NAME, data.getValue()); + } + } + + private Map getKnnVectorTestData() { + Map data = new HashMap<>(); + data.put("1", new Float[] { 100.0f, 1.0f }); + data.put("2", new Float[] { 99.0f, 2.0f }); + data.put("3", new Float[] { 97.0f, 3.0f }); + data.put("4", new Float[] { 98.0f, 4.0f }); + return data; + } + + private Map getCosineTestData() { + Map data = new HashMap<>(); + data.put("0", new Float[] { 1.0f, -1.0f }); + data.put("2", new Float[] { 1.0f, 1.0f }); + data.put("1", new Float[] { 1.0f, 0.0f }); + return data; + } + + /* + The doc['field'] will throw an error if field is missing from the mappings. + */ + private List buildMappingProperties() { + List properties = new ArrayList<>(); + properties.add(MappingProperty.builder().name(FIELD_NAME).type(KNNVectorFieldMapper.CONTENT_TYPE).dimension("2").build()); + properties.add(MappingProperty.builder().name(NUMERIC_INDEX_FIELD_NAME).type("integer").build()); + return properties; + } + + @SneakyThrows + public void testCosineSimilarity_whenUsedInScriptFields_thenExecutesScript() { + String source = String.format(Locale.ROOT, "1 + cosineSimilarity([2.0f, -2.0f], doc['%s'])", FIELD_NAME); + String scriptFieldName = "similarity"; + Request request = buildPainlessScriptFieldsRequest(source, 3, getCosineTestData(), scriptFieldName); + Response response = client().performRequest(request); + assertEquals(request.getEndpoint() + ": failed", RestStatus.OK, RestStatus.fromCode(response.getStatusLine().getStatusCode())); + + List results = parseSearchResponseScriptFields(EntityUtils.toString(response.getEntity()), scriptFieldName); + assertEquals(3, results.size()); + + String[] expectedDocIDs = { "0", "1", "2" }; + for (int i = 0; i < results.size(); i++) { + assertEquals(expectedDocIDs[i], results.get(i).getDocId()); + } + deleteKNNIndex(INDEX_NAME); + } + + @SneakyThrows + public void testGetValue_whenUsedInScriptFields_thenReturnsDocValues() { + String source = String.format(Locale.ROOT, "doc['%s'].value[0]", FIELD_NAME); + String scriptFieldName = "doc_value_field"; + Map testData = getKnnVectorTestData(); + Request request = buildPainlessScriptFieldsRequest(source, testData.size(), testData, scriptFieldName); + + Response response = client().performRequest(request); + assertEquals(request.getEndpoint() + ": failed", RestStatus.OK, RestStatus.fromCode(response.getStatusLine().getStatusCode())); + + List results = parseSearchResponseScriptFields(EntityUtils.toString(response.getEntity()), scriptFieldName); + assertEquals(testData.size(), results.size()); + + String[] expectedDocIDs = { "1", "2", "3", "4" }; + for (int i = 0; i < results.size(); i++) { + assertEquals(expectedDocIDs[i], results.get(i).getDocId()); + } + deleteKNNIndex(INDEX_NAME); + } + + private Request buildPainlessScriptFieldsRequest( + final String source, + final int size, + final Map documents, + final String scriptFieldName + ) throws Exception { + buildTestIndex(documents); + return constructScriptFieldsContextSearchRequest( + INDEX_NAME, + scriptFieldName, + Collections.emptyMap(), + Script.DEFAULT_SCRIPT_LANG, + source, + size, + Collections.emptyMap() + ); + } +} diff --git a/src/test/java/org/opensearch/knn/integ/PainlessScriptHelper.java b/src/test/java/org/opensearch/knn/integ/PainlessScriptHelper.java new file mode 100644 index 000000000..e53fc41e4 --- /dev/null +++ b/src/test/java/org/opensearch/knn/integ/PainlessScriptHelper.java @@ -0,0 +1,58 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.integ; + +import lombok.Builder; +import lombok.Getter; +import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.knn.common.KNNConstants; +import org.opensearch.knn.index.engine.KNNMethodContext; + +import java.io.IOException; +import java.util.List; +import java.util.Objects; + +public final class PainlessScriptHelper { + /** + * Utility to create a Index Mapping with multiple fields + */ + public static String createMapping(final List properties) throws IOException { + Objects.requireNonNull(properties); + XContentBuilder xContentBuilder = XContentFactory.jsonBuilder().startObject().startObject("properties"); + for (MappingProperty property : properties) { + XContentBuilder builder = xContentBuilder.startObject(property.getName()).field("type", property.getType()); + if (property.getDimension() != null) { + builder.field("dimension", property.getDimension()); + } + + if (property.getDocValues() != null) { + builder.field("doc_values", property.getDocValues()); + } + + if (property.getKnnMethodContext() != null) { + builder.startObject(KNNConstants.KNN_METHOD); + property.getKnnMethodContext().toXContent(builder, ToXContent.EMPTY_PARAMS); + builder.endObject(); + } + + builder.endObject(); + } + xContentBuilder.endObject().endObject(); + return xContentBuilder.toString(); + } + + @Getter + @Builder + final static class MappingProperty { + private final String name; + private final String type; + private String dimension; + private KNNMethodContext knnMethodContext; + private Boolean docValues; + } +} diff --git a/src/test/java/org/opensearch/knn/integ/PainlessScriptIT.java b/src/test/java/org/opensearch/knn/integ/PainlessScriptScoreIT.java similarity index 91% rename from src/test/java/org/opensearch/knn/integ/PainlessScriptIT.java rename to src/test/java/org/opensearch/knn/integ/PainlessScriptScoreIT.java index 2fed9fc25..c8835f70f 100644 --- a/src/test/java/org/opensearch/knn/integ/PainlessScriptIT.java +++ b/src/test/java/org/opensearch/knn/integ/PainlessScriptScoreIT.java @@ -7,10 +7,8 @@ import lombok.SneakyThrows; import org.opensearch.common.settings.Settings; -import org.opensearch.core.xcontent.ToXContent; import org.opensearch.knn.KNNRestTestCase; import org.opensearch.knn.KNNResult; -import org.opensearch.knn.common.KNNConstants; import org.opensearch.knn.index.engine.KNNMethodContext; import org.opensearch.knn.index.engine.MethodComponentContext; import org.opensearch.knn.index.SpaceType; @@ -20,15 +18,13 @@ import org.opensearch.client.Request; import org.opensearch.client.Response; import org.opensearch.client.ResponseException; -import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.index.query.MatchAllQueryBuilder; import org.opensearch.index.query.QueryBuilder; import org.opensearch.knn.index.engine.KNNEngine; import org.opensearch.core.rest.RestStatus; +import org.opensearch.knn.integ.PainlessScriptHelper.MappingProperty; import org.opensearch.script.Script; -import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -36,44 +32,16 @@ import java.util.List; import java.util.Locale; import java.util.Map; -import java.util.Objects; import static org.opensearch.knn.common.KNNConstants.METHOD_HNSW; +import static org.opensearch.knn.integ.PainlessScriptHelper.createMapping; -public class PainlessScriptIT extends KNNRestTestCase { +public final class PainlessScriptScoreIT extends KNNRestTestCase { public static final int AGGREGATION_FIELD_NAME_MIN_LENGTH = 2; public static final int AGGREGATION_FIELD_NAME_MAX_LENGTH = 5; private static final String NUMERIC_INDEX_FIELD_NAME = "price"; - /** - * Utility to create a Index Mapping with multiple fields - */ - protected String createMapping(List properties) throws IOException { - Objects.requireNonNull(properties); - XContentBuilder xContentBuilder = XContentFactory.jsonBuilder().startObject().startObject("properties"); - for (MappingProperty property : properties) { - XContentBuilder builder = xContentBuilder.startObject(property.getName()).field("type", property.getType()); - if (property.getDimension() != null) { - builder.field("dimension", property.getDimension()); - } - - if (property.getDocValues() != null) { - builder.field("doc_values", property.getDocValues()); - } - - if (property.getKnnMethodContext() != null) { - builder.startObject(KNNConstants.KNN_METHOD); - property.getKnnMethodContext().toXContent(builder, ToXContent.EMPTY_PARAMS); - builder.endObject(); - } - - builder.endObject(); - } - xContentBuilder.endObject().endObject(); - return xContentBuilder.toString(); - } - /* creates KnnIndex based on properties, we add single non-knn vector documents to verify whether actions works on non-knn vector documents as well @@ -148,8 +116,8 @@ private Map getCosineTestData() { */ private List buildMappingProperties() { List properties = new ArrayList<>(); - properties.add(new MappingProperty(FIELD_NAME, KNNVectorFieldMapper.CONTENT_TYPE).dimension("2")); - properties.add(new MappingProperty(NUMERIC_INDEX_FIELD_NAME, "integer")); + properties.add(MappingProperty.builder().name(FIELD_NAME).type(KNNVectorFieldMapper.CONTENT_TYPE).dimension("2").build()); + properties.add(MappingProperty.builder().name(NUMERIC_INDEX_FIELD_NAME).type("integer").build()); return properties; } @@ -568,9 +536,13 @@ public void testL2ScriptingWithLuceneBackedIndex() throws Exception { new MethodComponentContext(METHOD_HNSW, Collections.emptyMap()) ); properties.add( - new MappingProperty(FIELD_NAME, KNNVectorFieldMapper.CONTENT_TYPE).dimension("2") + MappingProperty.builder() + .name(FIELD_NAME) + .type(KNNVectorFieldMapper.CONTENT_TYPE) + .dimension("2") .knnMethodContext(knnMethodContext) .docValues(randomBoolean()) + .build() ); String source = String.format("1/(1 + l2Squared([1.0f, 1.0f], doc['%s']))", FIELD_NAME); @@ -671,54 +643,4 @@ private Response buildIndexAndRunPainlessScript( deleteKNNIndex(INDEX_NAME); } } - - static class MappingProperty { - - private final String name; - private final String type; - private String dimension; - - private KNNMethodContext knnMethodContext; - private Boolean docValues; - - MappingProperty(String name, String type) { - this.name = name; - this.type = type; - } - - MappingProperty dimension(String dimension) { - this.dimension = dimension; - return this; - } - - MappingProperty knnMethodContext(KNNMethodContext knnMethodContext) { - this.knnMethodContext = knnMethodContext; - return this; - } - - MappingProperty docValues(boolean docValues) { - this.docValues = docValues; - return this; - } - - KNNMethodContext getKnnMethodContext() { - return knnMethodContext; - } - - String getDimension() { - return dimension; - } - - String getName() { - return name; - } - - String getType() { - return type; - } - - Boolean getDocValues() { - return docValues; - } - } } diff --git a/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java b/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java index a90935869..0c1dbb2ce 100644 --- a/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java +++ b/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java @@ -303,6 +303,31 @@ protected List parseSearchResponseScore(String responseBody, String field return knnSearchResponses; } + protected List parseSearchResponseScriptFields(final String responseBody, final String scriptFieldName) throws IOException { + @SuppressWarnings("unchecked") + List hits = (List) ((Map) createParser( + MediaTypeRegistry.getDefaultMediaType().xContent(), + responseBody + ).map().get("hits")).get("hits"); + + @SuppressWarnings("unchecked") + List knnSearchResponses = hits.stream().map(hit -> { + @SuppressWarnings("unchecked") + final float[] vector = Floats.toArray( + Arrays.stream( + ((ArrayList) ((Map) ((Map) hit).get("fields")).get(scriptFieldName)).toArray() + ).map(Object::toString).map(Float::valueOf).collect(Collectors.toList()) + ); + return new KNNResult( + (String) ((Map) hit).get("_id"), + vector, + ((Double) ((Map) hit).get("_score")).floatValue() + ); + }).collect(Collectors.toList()); + + return knnSearchResponses; + } + /** * Parse the response of Aggregation to extract the value */ @@ -1002,6 +1027,37 @@ protected Request constructScriptedMetricAggregationSearchRequest( return request; } + protected Request constructScriptFieldsContextSearchRequest( + final String indexName, + final String fieldName, + final Map scriptParams, + final String language, + final String source, + final int size, + final Map searchParams + ) throws Exception { + Script script = buildScript(source, language, scriptParams); + XContentBuilder builder = XContentFactory.jsonBuilder().startObject().field("size", size).startObject("query"); + builder.startObject("match_all"); + builder.endObject(); + builder.endObject(); + builder.startObject("script_fields"); + builder.startObject(fieldName); + builder.field("script", script); + builder.endObject(); + builder.endObject(); + builder.endObject(); + URIBuilder uriBuilder = new URIBuilder("/" + indexName + "/_search"); + if (Objects.nonNull(searchParams)) { + for (Map.Entry entry : searchParams.entrySet()) { + uriBuilder.addParameter(entry.getKey(), entry.getValue().toString()); + } + } + Request request = new Request("POST", uriBuilder.toString()); + request.setJsonEntity(builder.toString()); + return request; + } + protected Request constructScriptScoreContextSearchRequest( String indexName, QueryBuilder qb, From 8dffc2b4f8d49fcc350c98fb33d879f73158a7aa Mon Sep 17 00:00:00 2001 From: John Mazanec Date: Wed, 21 Aug 2024 18:50:04 -0400 Subject: [PATCH 334/416] Add support for qframework from interface (#1998) Adds support for parsing qframework for faiss in the interface for IVF and HNSW. Does not yet integrate them with the framework. Signed-off-by: John Mazanec (cherry picked from commit 45f8aef17fc565fc49d98e6e821eacd1c276f242) --- .../knn/common/FieldInfoExtractor.java | 19 +++ .../opensearch/knn/common/KNNConstants.java | 2 + .../knn/index/engine/AbstractKNNMethod.java | 8 +- .../engine/KNNLibraryIndexingContext.java | 8 ++ .../engine/KNNLibraryIndexingContextImpl.java | 12 +- .../knn/index/engine/MethodComponent.java | 38 ++++-- .../engine/faiss/AbstractFaissMethod.java | 38 ++++++ .../index/engine/faiss/FaissFlatEncoder.java | 2 +- .../index/engine/faiss/FaissHNSWMethod.java | 22 ++-- .../engine/faiss/FaissHNSWPQEncoder.java | 2 +- .../index/engine/faiss/FaissIVFMethod.java | 22 ++-- .../index/engine/faiss/FaissIVFPQEncoder.java | 2 +- .../index/engine/faiss/FaissSQEncoder.java | 2 +- .../engine/faiss/MethodAsMapBuilder.java | 28 +++- .../index/engine/faiss/QFrameBitEncoder.java | 78 +++++++++++ .../engine/qframe/QuantizationConfig.java | 23 ++++ .../qframe/QuantizationConfigParser.java | 85 ++++++++++++ .../knn/index/mapper/MethodFieldMapper.java | 22 +++- .../knn/index/mapper/ModelFieldMapper.java | 16 +++ .../opensearch/knn/index/query/KNNWeight.java | 5 + .../index/engine/AbstractKNNLibraryTests.java | 6 +- .../index/engine/AbstractKNNMethodTests.java | 8 +- .../index/engine/MethodComponentTests.java | 31 +++-- .../knn/index/engine/faiss/FaissTests.java | 103 +++++++++++++-- .../engine/faiss/QFrameBitEncoderTests.java | 124 ++++++++++++++++++ .../qframe/QuantizationConfigParserTests.java | 91 +++++++++++++ .../opensearch/knn/integ/QFrameworkIT.java | 77 +++++++++++ 27 files changed, 800 insertions(+), 74 deletions(-) create mode 100644 src/main/java/org/opensearch/knn/index/engine/faiss/QFrameBitEncoder.java create mode 100644 src/main/java/org/opensearch/knn/index/engine/qframe/QuantizationConfig.java create mode 100644 src/main/java/org/opensearch/knn/index/engine/qframe/QuantizationConfigParser.java create mode 100644 src/test/java/org/opensearch/knn/index/engine/faiss/QFrameBitEncoderTests.java create mode 100644 src/test/java/org/opensearch/knn/index/engine/qframe/QuantizationConfigParserTests.java create mode 100644 src/test/java/org/opensearch/knn/integ/QFrameworkIT.java diff --git a/src/main/java/org/opensearch/knn/common/FieldInfoExtractor.java b/src/main/java/org/opensearch/knn/common/FieldInfoExtractor.java index 591f16735..724632a46 100644 --- a/src/main/java/org/opensearch/knn/common/FieldInfoExtractor.java +++ b/src/main/java/org/opensearch/knn/common/FieldInfoExtractor.java @@ -12,6 +12,11 @@ import org.opensearch.knn.indices.ModelMetadata; import org.opensearch.knn.indices.ModelUtil; +import org.opensearch.knn.index.engine.qframe.QuantizationConfig; +import org.opensearch.knn.index.engine.qframe.QuantizationConfigParser; + +import static org.opensearch.knn.common.KNNConstants.QFRAMEWORK_CONFIG; + /** * A utility class to extract information from FieldInfo. */ @@ -34,4 +39,18 @@ public static VectorDataType extractVectorDataType(final FieldInfo fieldInfo) { } return StringUtils.isNotEmpty(vectorDataTypeString) ? VectorDataType.get(vectorDataTypeString) : VectorDataType.DEFAULT; } + + /** + * Extract quantization config from fieldInfo + * + * @param fieldInfo {@link FieldInfo} + * @return {@link QuantizationConfig} + */ + public static QuantizationConfig extractQuantizationConfig(final FieldInfo fieldInfo) { + String quantizationConfigString = fieldInfo.getAttribute(QFRAMEWORK_CONFIG); + if (StringUtils.isEmpty(quantizationConfigString)) { + return QuantizationConfig.EMPTY; + } + return QuantizationConfigParser.fromCsv(quantizationConfigString); + } } diff --git a/src/main/java/org/opensearch/knn/common/KNNConstants.java b/src/main/java/org/opensearch/knn/common/KNNConstants.java index de46cdaa8..4d935407a 100644 --- a/src/main/java/org/opensearch/knn/common/KNNConstants.java +++ b/src/main/java/org/opensearch/knn/common/KNNConstants.java @@ -66,6 +66,8 @@ public class KNNConstants { public static final String MAX_VECTOR_COUNT_PARAMETER = "max_training_vector_count"; public static final String SEARCH_SIZE_PARAMETER = "search_size"; + public static final String QFRAMEWORK_CONFIG = "qframe_config"; + public static final String VECTOR_DATA_TYPE_FIELD = "data_type"; public static final String MODEL_VECTOR_DATA_TYPE_KEY = VECTOR_DATA_TYPE_FIELD; public static final VectorDataType DEFAULT_VECTOR_DATA_TYPE_FIELD = VectorDataType.FLOAT; diff --git a/src/main/java/org/opensearch/knn/index/engine/AbstractKNNMethod.java b/src/main/java/org/opensearch/knn/index/engine/AbstractKNNMethod.java index 52cc79129..f53655136 100644 --- a/src/main/java/org/opensearch/knn/index/engine/AbstractKNNMethod.java +++ b/src/main/java/org/opensearch/knn/index/engine/AbstractKNNMethod.java @@ -16,7 +16,6 @@ import org.opensearch.knn.index.mapper.VectorValidator; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; import java.util.Locale; import java.util.Map; @@ -112,12 +111,15 @@ public KNNLibraryIndexingContext getKNNLibraryIndexingContext( KNNMethodContext knnMethodContext, KNNMethodConfigContext knnMethodConfigContext ) { - Map parameterMap = new HashMap<>( - methodComponent.getAsMap(knnMethodContext.getMethodComponentContext(), knnMethodConfigContext) + KNNLibraryIndexingContext knnLibraryIndexingContext = methodComponent.getKNNLibraryIndexingContext( + knnMethodContext.getMethodComponentContext(), + knnMethodConfigContext ); + Map parameterMap = knnLibraryIndexingContext.getLibraryParameters(); parameterMap.put(KNNConstants.SPACE_TYPE, knnMethodContext.getSpaceType().getValue()); parameterMap.put(KNNConstants.VECTOR_DATA_TYPE_FIELD, knnMethodConfigContext.getVectorDataType().getValue()); return KNNLibraryIndexingContextImpl.builder() + .quantizationConfig(knnLibraryIndexingContext.getQuantizationConfig()) .parameters(parameterMap) .vectorValidator(doGetVectorValidator(knnMethodContext, knnMethodConfigContext)) .perDimensionValidator(doGetPerDimensionValidator(knnMethodContext, knnMethodConfigContext)) diff --git a/src/main/java/org/opensearch/knn/index/engine/KNNLibraryIndexingContext.java b/src/main/java/org/opensearch/knn/index/engine/KNNLibraryIndexingContext.java index 20285471e..9208661af 100644 --- a/src/main/java/org/opensearch/knn/index/engine/KNNLibraryIndexingContext.java +++ b/src/main/java/org/opensearch/knn/index/engine/KNNLibraryIndexingContext.java @@ -5,6 +5,7 @@ package org.opensearch.knn.index.engine; +import org.opensearch.knn.index.engine.qframe.QuantizationConfig; import org.opensearch.knn.index.mapper.PerDimensionProcessor; import org.opensearch.knn.index.mapper.PerDimensionValidator; import org.opensearch.knn.index.mapper.VectorValidator; @@ -22,6 +23,13 @@ public interface KNNLibraryIndexingContext { */ Map getLibraryParameters(); + /** + * Get map of parameters that get passed to the quantization framework + * + * @return Map of parameters + */ + QuantizationConfig getQuantizationConfig(); + /** * * @return Get the vector validator diff --git a/src/main/java/org/opensearch/knn/index/engine/KNNLibraryIndexingContextImpl.java b/src/main/java/org/opensearch/knn/index/engine/KNNLibraryIndexingContextImpl.java index 51b60d9e5..f5329fc31 100644 --- a/src/main/java/org/opensearch/knn/index/engine/KNNLibraryIndexingContextImpl.java +++ b/src/main/java/org/opensearch/knn/index/engine/KNNLibraryIndexingContextImpl.java @@ -6,10 +6,12 @@ package org.opensearch.knn.index.engine; import lombok.Builder; +import org.opensearch.knn.index.engine.qframe.QuantizationConfig; import org.opensearch.knn.index.mapper.PerDimensionProcessor; import org.opensearch.knn.index.mapper.PerDimensionValidator; import org.opensearch.knn.index.mapper.VectorValidator; +import java.util.Collections; import java.util.Map; /** @@ -21,13 +23,21 @@ public class KNNLibraryIndexingContextImpl implements KNNLibraryIndexingContext private VectorValidator vectorValidator; private PerDimensionValidator perDimensionValidator; private PerDimensionProcessor perDimensionProcessor; - private Map parameters; + @Builder.Default + private Map parameters = Collections.emptyMap(); + @Builder.Default + private QuantizationConfig quantizationConfig = QuantizationConfig.EMPTY; @Override public Map getLibraryParameters() { return parameters; } + @Override + public QuantizationConfig getQuantizationConfig() { + return quantizationConfig; + } + @Override public VectorValidator getVectorValidator() { return vectorValidator; diff --git a/src/main/java/org/opensearch/knn/index/engine/MethodComponent.java b/src/main/java/org/opensearch/knn/index/engine/MethodComponent.java index 988812e61..2579063e9 100644 --- a/src/main/java/org/opensearch/knn/index/engine/MethodComponent.java +++ b/src/main/java/org/opensearch/knn/index/engine/MethodComponent.java @@ -30,7 +30,11 @@ public class MethodComponent { private final String name; @Getter private final Map> parameters; - private final TriFunction> mapGenerator; + private final TriFunction< + MethodComponent, + MethodComponentContext, + KNNMethodConfigContext, + KNNLibraryIndexingContext> knnLibraryIndexingContextGenerator; private final TriFunction overheadInKBEstimator; private final boolean requiresTraining; private final Set supportedVectorDataTypes; @@ -43,7 +47,7 @@ public class MethodComponent { private MethodComponent(Builder builder) { this.name = builder.name; this.parameters = builder.parameters; - this.mapGenerator = builder.mapGenerator; + this.knnLibraryIndexingContextGenerator = builder.knnLibraryIndexingContextGenerator; this.overheadInKBEstimator = builder.overheadInKBEstimator; this.requiresTraining = builder.requiresTraining; this.supportedVectorDataTypes = builder.supportedDataTypes; @@ -55,17 +59,20 @@ private MethodComponent(Builder builder) { * @param methodComponentContext from which to generate map * @return Method component as a map */ - public Map getAsMap(MethodComponentContext methodComponentContext, KNNMethodConfigContext knnMethodConfigContext) { - if (mapGenerator == null) { + public KNNLibraryIndexingContext getKNNLibraryIndexingContext( + MethodComponentContext methodComponentContext, + KNNMethodConfigContext knnMethodConfigContext + ) { + if (knnLibraryIndexingContextGenerator == null) { Map parameterMap = new HashMap<>(); parameterMap.put(KNNConstants.NAME, methodComponentContext.getName()); parameterMap.put( KNNConstants.PARAMETERS, getParameterMapWithDefaultsAdded(methodComponentContext, this, knnMethodConfigContext) ); - return parameterMap; + return KNNLibraryIndexingContextImpl.builder().parameters(parameterMap).build(); } - return mapGenerator.apply(this, methodComponentContext, knnMethodConfigContext); + return knnLibraryIndexingContextGenerator.apply(this, methodComponentContext, knnMethodConfigContext); } /** @@ -209,7 +216,11 @@ public static class Builder { private final String name; private final Map> parameters; - private TriFunction> mapGenerator; + private TriFunction< + MethodComponent, + MethodComponentContext, + KNNMethodConfigContext, + KNNLibraryIndexingContext> knnLibraryIndexingContextGenerator; private TriFunction overheadInKBEstimator; private boolean requiresTraining; private final Set supportedDataTypes; @@ -227,7 +238,6 @@ public static Builder builder(String name) { private Builder(String name) { this.name = name; this.parameters = new HashMap<>(); - this.mapGenerator = null; this.overheadInKBEstimator = (mc, mcc, d) -> 0L; this.supportedDataTypes = new HashSet<>(); } @@ -247,13 +257,17 @@ public Builder addParameter(String parameterName, Parameter parameter) { /** * Set the function used to parse a MethodComponentContext as a map * - * @param mapGenerator function to parse a MethodComponentContext as a map + * @param knnLibraryIndexingContextGenerator function to parse a MethodComponentContext as a knnLibraryIndexingContext * @return this builder */ - public Builder setMapGenerator( - TriFunction> mapGenerator + public Builder setKnnLibraryIndexingContextGenerator( + TriFunction< + MethodComponent, + MethodComponentContext, + KNNMethodConfigContext, + KNNLibraryIndexingContext> knnLibraryIndexingContextGenerator ) { - this.mapGenerator = mapGenerator; + this.knnLibraryIndexingContextGenerator = knnLibraryIndexingContextGenerator; return this; } diff --git a/src/main/java/org/opensearch/knn/index/engine/faiss/AbstractFaissMethod.java b/src/main/java/org/opensearch/knn/index/engine/faiss/AbstractFaissMethod.java index 52b7efe73..908671a21 100644 --- a/src/main/java/org/opensearch/knn/index/engine/faiss/AbstractFaissMethod.java +++ b/src/main/java/org/opensearch/knn/index/engine/faiss/AbstractFaissMethod.java @@ -8,15 +8,20 @@ import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.engine.AbstractKNNMethod; +import org.opensearch.knn.index.engine.KNNLibraryIndexingContext; import org.opensearch.knn.index.engine.KNNLibrarySearchContext; import org.opensearch.knn.index.engine.KNNMethodConfigContext; import org.opensearch.knn.index.engine.KNNMethodContext; import org.opensearch.knn.index.engine.MethodComponent; +import org.opensearch.knn.index.engine.MethodComponentContext; import org.opensearch.knn.index.mapper.PerDimensionProcessor; import org.opensearch.knn.index.mapper.PerDimensionValidator; +import java.util.Objects; import java.util.Set; +import static org.opensearch.knn.common.KNNConstants.METHOD_ENCODER_PARAMETER; +import static org.opensearch.knn.index.engine.faiss.Faiss.FAISS_BINARY_INDEX_DESCRIPTION_PREFIX; import static org.opensearch.knn.index.engine.faiss.FaissFP16Util.isFaissSQClipToFP16RangeEnabled; import static org.opensearch.knn.index.engine.faiss.FaissFP16Util.isFaissSQfp16; @@ -81,4 +86,37 @@ protected PerDimensionProcessor doGetPerDimensionProcessor( throw new IllegalStateException("Unsupported vector data type " + vectorDataType); } + + static KNNLibraryIndexingContext adjustPrefix( + MethodAsMapBuilder methodAsMapBuilder, + MethodComponentContext methodComponentContext, + KNNMethodConfigContext knnMethodConfigContext + ) { + String prefix = ""; + MethodComponentContext encoderContext = getEncoderMethodComponent(methodComponentContext); + // We need to update the prefix used to create the faiss index if we are using the quantization + // framework + if (encoderContext != null && Objects.equals(encoderContext.getName(), QFrameBitEncoder.NAME)) { + // TODO: Uncomment to use Quantization framework + // leaving commented now just so it wont fail creating faiss indices. + // prefix = FAISS_BINARY_INDEX_DESCRIPTION_PREFIX; + } + + if (knnMethodConfigContext.getVectorDataType() == VectorDataType.BINARY) { + prefix = FAISS_BINARY_INDEX_DESCRIPTION_PREFIX; + } + methodAsMapBuilder.indexDescription = prefix + methodAsMapBuilder.indexDescription; + return methodAsMapBuilder.build(); + } + + static MethodComponentContext getEncoderMethodComponent(MethodComponentContext methodComponentContext) { + if (!methodComponentContext.getParameters().containsKey(METHOD_ENCODER_PARAMETER)) { + return null; + } + Object object = methodComponentContext.getParameters().get(METHOD_ENCODER_PARAMETER); + if (!(object instanceof MethodComponentContext)) { + return null; + } + return (MethodComponentContext) object; + } } diff --git a/src/main/java/org/opensearch/knn/index/engine/faiss/FaissFlatEncoder.java b/src/main/java/org/opensearch/knn/index/engine/faiss/FaissFlatEncoder.java index 5e6e4060f..bd7598d84 100644 --- a/src/main/java/org/opensearch/knn/index/engine/faiss/FaissFlatEncoder.java +++ b/src/main/java/org/opensearch/knn/index/engine/faiss/FaissFlatEncoder.java @@ -26,7 +26,7 @@ public class FaissFlatEncoder implements Encoder { ); private final static MethodComponent METHOD_COMPONENT = MethodComponent.Builder.builder(KNNConstants.ENCODER_FLAT) - .setMapGenerator( + .setKnnLibraryIndexingContextGenerator( ((methodComponent, methodComponentContext, knnMethodConfigContext) -> MethodAsMapBuilder.builder( KNNConstants.FAISS_FLAT_DESCRIPTION, methodComponent, diff --git a/src/main/java/org/opensearch/knn/index/engine/faiss/FaissHNSWMethod.java b/src/main/java/org/opensearch/knn/index/engine/faiss/FaissHNSWMethod.java index ee6a4f101..15e43cdd5 100644 --- a/src/main/java/org/opensearch/knn/index/engine/faiss/FaissHNSWMethod.java +++ b/src/main/java/org/opensearch/knn/index/engine/faiss/FaissHNSWMethod.java @@ -29,7 +29,6 @@ import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_EF_CONSTRUCTION; import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_EF_SEARCH; import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_M; -import static org.opensearch.knn.index.engine.faiss.Faiss.FAISS_BINARY_INDEX_DESCRIPTION_PREFIX; /** * Faiss HNSW method implementation @@ -49,7 +48,12 @@ public class FaissHNSWMethod extends AbstractFaissMethod { KNNConstants.ENCODER_FLAT, Collections.emptyMap() ); - private final static List SUPPORTED_ENCODERS = List.of(new FaissFlatEncoder(), new FaissSQEncoder(), new FaissHNSWPQEncoder()); + private final static List SUPPORTED_ENCODERS = List.of( + new FaissFlatEncoder(), + new FaissSQEncoder(), + new FaissHNSWPQEncoder(), + new QFrameBitEncoder() + ); /** * Constructor for FaissHNSWMethod @@ -84,18 +88,14 @@ private static MethodComponent initMethodComponent() { ) ) .addParameter(METHOD_ENCODER_PARAMETER, initEncoderParameter()) - .setMapGenerator(((methodComponent, methodComponentContext, knnMethodConfigContext) -> { - String prefix = ""; - if (knnMethodConfigContext.getVectorDataType() == VectorDataType.BINARY) { - prefix = FAISS_BINARY_INDEX_DESCRIPTION_PREFIX; - } - - return MethodAsMapBuilder.builder( - prefix + FAISS_HNSW_DESCRIPTION, + .setKnnLibraryIndexingContextGenerator(((methodComponent, methodComponentContext, knnMethodConfigContext) -> { + MethodAsMapBuilder methodAsMapBuilder = MethodAsMapBuilder.builder( + FAISS_HNSW_DESCRIPTION, methodComponent, methodComponentContext, knnMethodConfigContext - ).addParameter(METHOD_PARAMETER_M, "", "").addParameter(METHOD_ENCODER_PARAMETER, ",", "").build(); + ).addParameter(METHOD_PARAMETER_M, "", "").addParameter(METHOD_ENCODER_PARAMETER, ",", ""); + return adjustPrefix(methodAsMapBuilder, methodComponentContext, knnMethodConfigContext); })) .build(); } diff --git a/src/main/java/org/opensearch/knn/index/engine/faiss/FaissHNSWPQEncoder.java b/src/main/java/org/opensearch/knn/index/engine/faiss/FaissHNSWPQEncoder.java index 8d53f3c0a..9bebf5b4d 100644 --- a/src/main/java/org/opensearch/knn/index/engine/faiss/FaissHNSWPQEncoder.java +++ b/src/main/java/org/opensearch/knn/index/engine/faiss/FaissHNSWPQEncoder.java @@ -51,7 +51,7 @@ public class FaissHNSWPQEncoder implements Encoder { ) ) .setRequiresTraining(true) - .setMapGenerator( + .setKnnLibraryIndexingContextGenerator( ((methodComponent, methodComponentContext, knnMethodConfigContext) -> MethodAsMapBuilder.builder( FAISS_PQ_DESCRIPTION, methodComponent, diff --git a/src/main/java/org/opensearch/knn/index/engine/faiss/FaissIVFMethod.java b/src/main/java/org/opensearch/knn/index/engine/faiss/FaissIVFMethod.java index a21810b50..bc30e372c 100644 --- a/src/main/java/org/opensearch/knn/index/engine/faiss/FaissIVFMethod.java +++ b/src/main/java/org/opensearch/knn/index/engine/faiss/FaissIVFMethod.java @@ -32,7 +32,6 @@ import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_NPROBES; import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_NPROBES_DEFAULT; import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_NPROBES_LIMIT; -import static org.opensearch.knn.index.engine.faiss.Faiss.FAISS_BINARY_INDEX_DESCRIPTION_PREFIX; /** * Faiss ivf implementation @@ -52,7 +51,12 @@ public class FaissIVFMethod extends AbstractFaissMethod { KNNConstants.ENCODER_FLAT, Collections.emptyMap() ); - private final static List SUPPORTED_ENCODERS = List.of(new FaissFlatEncoder(), new FaissSQEncoder(), new FaissIVFPQEncoder()); + private final static List SUPPORTED_ENCODERS = List.of( + new FaissFlatEncoder(), + new FaissSQEncoder(), + new FaissIVFPQEncoder(), + new QFrameBitEncoder() + ); /** * Constructor for FaissIVFMethod @@ -84,18 +88,14 @@ private static MethodComponent initMethodComponent() { ) .addParameter(METHOD_ENCODER_PARAMETER, initEncoderParameter()) .setRequiresTraining(true) - .setMapGenerator(((methodComponent, methodComponentContext, knnMethodConfigContext) -> { - String prefix = ""; - if (knnMethodConfigContext.getVectorDataType() == VectorDataType.BINARY) { - prefix = FAISS_BINARY_INDEX_DESCRIPTION_PREFIX; - } - - return MethodAsMapBuilder.builder( - prefix + FAISS_IVF_DESCRIPTION, + .setKnnLibraryIndexingContextGenerator(((methodComponent, methodComponentContext, knnMethodConfigContext) -> { + MethodAsMapBuilder methodAsMapBuilder = MethodAsMapBuilder.builder( + FAISS_IVF_DESCRIPTION, methodComponent, methodComponentContext, knnMethodConfigContext - ).addParameter(METHOD_PARAMETER_NLIST, "", "").addParameter(METHOD_ENCODER_PARAMETER, ",", "").build(); + ).addParameter(METHOD_PARAMETER_NLIST, "", "").addParameter(METHOD_ENCODER_PARAMETER, ",", ""); + return adjustPrefix(methodAsMapBuilder, methodComponentContext, knnMethodConfigContext); })) .setOverheadInKBEstimator((methodComponent, methodComponentContext, dimension) -> { // Size estimate formula: (4 * nlists * d) / 1024 + 1 diff --git a/src/main/java/org/opensearch/knn/index/engine/faiss/FaissIVFPQEncoder.java b/src/main/java/org/opensearch/knn/index/engine/faiss/FaissIVFPQEncoder.java index b38f5c816..bb6623600 100644 --- a/src/main/java/org/opensearch/knn/index/engine/faiss/FaissIVFPQEncoder.java +++ b/src/main/java/org/opensearch/knn/index/engine/faiss/FaissIVFPQEncoder.java @@ -51,7 +51,7 @@ public class FaissIVFPQEncoder implements Encoder { }) ) .setRequiresTraining(true) - .setMapGenerator( + .setKnnLibraryIndexingContextGenerator( ((methodComponent, methodComponentContext, knnMethodConfigContext) -> MethodAsMapBuilder.builder( FAISS_PQ_DESCRIPTION, methodComponent, diff --git a/src/main/java/org/opensearch/knn/index/engine/faiss/FaissSQEncoder.java b/src/main/java/org/opensearch/knn/index/engine/faiss/FaissSQEncoder.java index 2d0d184ca..6d57aef2f 100644 --- a/src/main/java/org/opensearch/knn/index/engine/faiss/FaissSQEncoder.java +++ b/src/main/java/org/opensearch/knn/index/engine/faiss/FaissSQEncoder.java @@ -35,7 +35,7 @@ public class FaissSQEncoder implements Encoder { new Parameter.StringParameter(FAISS_SQ_TYPE, FAISS_SQ_ENCODER_FP16, (v, context) -> FAISS_SQ_ENCODER_TYPES.contains(v)) ) .addParameter(FAISS_SQ_CLIP, new Parameter.BooleanParameter(FAISS_SQ_CLIP, false, (v, context) -> Objects.nonNull(v))) - .setMapGenerator( + .setKnnLibraryIndexingContextGenerator( ((methodComponent, methodComponentContext, knnMethodConfigContext) -> MethodAsMapBuilder.builder( FAISS_SQ_DESCRIPTION, methodComponent, diff --git a/src/main/java/org/opensearch/knn/index/engine/faiss/MethodAsMapBuilder.java b/src/main/java/org/opensearch/knn/index/engine/faiss/MethodAsMapBuilder.java index abb3d08c9..8b688cbcc 100644 --- a/src/main/java/org/opensearch/knn/index/engine/faiss/MethodAsMapBuilder.java +++ b/src/main/java/org/opensearch/knn/index/engine/faiss/MethodAsMapBuilder.java @@ -7,10 +7,13 @@ import lombok.AllArgsConstructor; import org.opensearch.knn.common.KNNConstants; +import org.opensearch.knn.index.engine.KNNLibraryIndexingContext; +import org.opensearch.knn.index.engine.KNNLibraryIndexingContextImpl; import org.opensearch.knn.index.engine.KNNMethodConfigContext; import org.opensearch.knn.index.engine.MethodComponent; import org.opensearch.knn.index.engine.MethodComponentContext; import org.opensearch.knn.index.engine.Parameter; +import org.opensearch.knn.index.engine.qframe.QuantizationConfig; import java.util.HashMap; import java.util.Map; @@ -31,6 +34,7 @@ class MethodAsMapBuilder { MethodComponent methodComponent; Map methodAsMap; KNNMethodConfigContext knnMethodConfigContext; + QuantizationConfig quantizationConfig; /** * Add a parameter that will be used in the index description for the given method component @@ -57,9 +61,21 @@ MethodAsMapBuilder addParameter(String parameterName, String prefix, String suff subMethodComponentContext.getName() ); - Map subMethodAsMap = subMethodComponent.getAsMap(subMethodComponentContext, knnMethodConfigContext); - indexDescription += subMethodAsMap.get(KNNConstants.INDEX_DESCRIPTION_PARAMETER); - subMethodAsMap.remove(KNNConstants.INDEX_DESCRIPTION_PARAMETER); + KNNLibraryIndexingContext knnLibraryIndexingContext = subMethodComponent.getKNNLibraryIndexingContext( + subMethodComponentContext, + knnMethodConfigContext + ); + Map subMethodAsMap = knnLibraryIndexingContext.getLibraryParameters(); + if (subMethodAsMap != null + && !subMethodAsMap.isEmpty() + && subMethodAsMap.containsKey(KNNConstants.INDEX_DESCRIPTION_PARAMETER)) { + indexDescription += subMethodAsMap.get(KNNConstants.INDEX_DESCRIPTION_PARAMETER); + subMethodAsMap.remove(KNNConstants.INDEX_DESCRIPTION_PARAMETER); + } + + if (quantizationConfig == null || quantizationConfig == QuantizationConfig.EMPTY) { + quantizationConfig = knnLibraryIndexingContext.getQuantizationConfig(); + } // We replace parameterName with the map that contains only parameters that are not included in // the method description @@ -79,9 +95,9 @@ MethodAsMapBuilder addParameter(String parameterName, String prefix, String suff * * @return Method as a map */ - Map build() { + KNNLibraryIndexingContext build() { methodAsMap.put(KNNConstants.INDEX_DESCRIPTION_PARAMETER, indexDescription); - return methodAsMap; + return KNNLibraryIndexingContextImpl.builder().parameters(methodAsMap).quantizationConfig(quantizationConfig).build(); } static MethodAsMapBuilder builder( @@ -96,6 +112,6 @@ static MethodAsMapBuilder builder( PARAMETERS, MethodComponent.getParameterMapWithDefaultsAdded(methodComponentContext, methodComponent, knnMethodConfigContext) ); - return new MethodAsMapBuilder(baseDescription, methodComponent, initialMap, knnMethodConfigContext); + return new MethodAsMapBuilder(baseDescription, methodComponent, initialMap, knnMethodConfigContext, QuantizationConfig.EMPTY); } } diff --git a/src/main/java/org/opensearch/knn/index/engine/faiss/QFrameBitEncoder.java b/src/main/java/org/opensearch/knn/index/engine/faiss/QFrameBitEncoder.java new file mode 100644 index 000000000..e135fa33f --- /dev/null +++ b/src/main/java/org/opensearch/knn/index/engine/faiss/QFrameBitEncoder.java @@ -0,0 +1,78 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.engine.faiss; + +import com.google.common.collect.ImmutableSet; +import org.opensearch.knn.index.VectorDataType; +import org.opensearch.knn.index.engine.Encoder; +import org.opensearch.knn.index.engine.KNNLibraryIndexingContextImpl; +import org.opensearch.knn.index.engine.MethodComponent; +import org.opensearch.knn.index.engine.Parameter; +import org.opensearch.knn.index.engine.qframe.QuantizationConfig; +import org.opensearch.knn.quantization.enums.ScalarQuantizationType; + +import java.util.HashMap; +import java.util.Locale; +import java.util.Set; + +import static org.opensearch.knn.common.KNNConstants.FAISS_FLAT_DESCRIPTION; +import static org.opensearch.knn.common.KNNConstants.INDEX_DESCRIPTION_PARAMETER; + +/** + * Quantization framework binary encoder, + */ +public class QFrameBitEncoder implements Encoder { + + public static final String NAME = "binary"; + public static final String BITCOUNT_PARAM = "bits"; + private static final int DEFAULT_BITS = 1; + private static final Set validBitCounts = ImmutableSet.of(1, 2, 4); + private static final Set SUPPORTED_DATA_TYPES = ImmutableSet.of(VectorDataType.FLOAT); + + /** + * { + * "encoder": { + * "name": "binary", + * "parameters": { + * "bits": 2 + * } + * } + * } + */ + private final static MethodComponent METHOD_COMPONENT = MethodComponent.Builder.builder(NAME) + .addSupportedDataTypes(SUPPORTED_DATA_TYPES) + .addParameter( + BITCOUNT_PARAM, + new Parameter.IntegerParameter(BITCOUNT_PARAM, DEFAULT_BITS, (v, context) -> validBitCounts.contains(v)) + ) + .setKnnLibraryIndexingContextGenerator(((methodComponent, methodComponentContext, knnMethodConfigContext) -> { + QuantizationConfig quantizationConfig; + int bitCount = (int) methodComponentContext.getParameters().getOrDefault(BITCOUNT_PARAM, DEFAULT_BITS); + if (bitCount == 1) { + quantizationConfig = QuantizationConfig.builder().quantizationType(ScalarQuantizationType.ONE_BIT).build(); + } else if (bitCount == 2) { + quantizationConfig = QuantizationConfig.builder().quantizationType(ScalarQuantizationType.TWO_BIT).build(); + } else if (bitCount == 4) { + quantizationConfig = QuantizationConfig.builder().quantizationType(ScalarQuantizationType.FOUR_BIT).build(); + } else { + throw new IllegalArgumentException(String.format(Locale.ROOT, "Invalid bit count: %d", bitCount)); + } + + // We use the flat description because we are doing the quantization + return KNNLibraryIndexingContextImpl.builder().quantizationConfig(quantizationConfig).parameters(new HashMap<>() { + { + put(INDEX_DESCRIPTION_PARAMETER, FAISS_FLAT_DESCRIPTION); + } + }).build(); + })) + .setRequiresTraining(false) + .build(); + + @Override + public MethodComponent getMethodComponent() { + return METHOD_COMPONENT; + } +} diff --git a/src/main/java/org/opensearch/knn/index/engine/qframe/QuantizationConfig.java b/src/main/java/org/opensearch/knn/index/engine/qframe/QuantizationConfig.java new file mode 100644 index 000000000..666247692 --- /dev/null +++ b/src/main/java/org/opensearch/knn/index/engine/qframe/QuantizationConfig.java @@ -0,0 +1,23 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.engine.qframe; + +import lombok.Builder; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import org.opensearch.knn.quantization.enums.ScalarQuantizationType; + +/** + * Configuration for quantization + */ +@Builder +@Getter +@EqualsAndHashCode +public class QuantizationConfig { + @Builder.Default + private ScalarQuantizationType quantizationType = null; + public static final QuantizationConfig EMPTY = QuantizationConfig.builder().build(); +} diff --git a/src/main/java/org/opensearch/knn/index/engine/qframe/QuantizationConfigParser.java b/src/main/java/org/opensearch/knn/index/engine/qframe/QuantizationConfigParser.java new file mode 100644 index 000000000..1f3349d34 --- /dev/null +++ b/src/main/java/org/opensearch/knn/index/engine/qframe/QuantizationConfigParser.java @@ -0,0 +1,85 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.engine.qframe; + +import org.apache.lucene.analysis.util.CSVUtil; +import org.opensearch.knn.index.engine.faiss.QFrameBitEncoder; +import org.opensearch.knn.quantization.enums.ScalarQuantizationType; + +import java.util.Locale; + +/** + * Parse util for quantization config + */ +public class QuantizationConfigParser { + + public static final String SEPARATOR = "="; + public static final String TYPE_NAME = "type"; + public static final String BINARY_TYPE = QFrameBitEncoder.NAME; + public static final String BIT_COUNT_NAME = QFrameBitEncoder.BITCOUNT_PARAM; + + /** + * Parse quantization config to csv format + * Example: type=binary,bits=2 + * @param quantizationConfig Quantization config + * @return Csv format of quantization config + */ + public static String toCsv(QuantizationConfig quantizationConfig) { + if (quantizationConfig == null + || quantizationConfig == QuantizationConfig.EMPTY + || quantizationConfig.getQuantizationType() == null) { + return ""; + } + return String.format( + Locale.ROOT, + "%s=%s,%s=%d", + TYPE_NAME, + BINARY_TYPE, + BIT_COUNT_NAME, + quantizationConfig.getQuantizationType().getId() + ); + } + + /** + * Parse csv format to quantization config + * + * @param csv Csv format of quantization config + * @return Quantization config + */ + public static QuantizationConfig fromCsv(String csv) { + if (csv == null || csv.isEmpty()) { + return QuantizationConfig.EMPTY; + } + + String[] csvArray = CSVUtil.parse(csv); + if (csvArray.length != 2) { + throw new IllegalArgumentException(String.format(Locale.ROOT, "Invalid csv for quantization config: \"%s\"", csv)); + } + + String typeValue = getValueOrThrow(TYPE_NAME, csvArray[0]); + if (!typeValue.equals(BINARY_TYPE)) { + throw new IllegalArgumentException(String.format(Locale.ROOT, "Unsupported quantization type: \"%s\"", typeValue)); + } + + String bitsValue = getValueOrThrow(BIT_COUNT_NAME, csvArray[1]); + int bitCount = Integer.parseInt(bitsValue); + ScalarQuantizationType quantizationType = ScalarQuantizationType.fromId(bitCount); + return QuantizationConfig.builder().quantizationType(quantizationType).build(); + } + + private static String getValueOrThrow(String expectedKey, String keyValue) { + String[] keyValueArr = keyValue.split(SEPARATOR); + if (keyValueArr.length != 2) { + throw new IllegalArgumentException(String.format(Locale.ROOT, "Invalid csv value for quantization config: \"%s\"", keyValue)); + } + + if (!keyValueArr[0].equals(expectedKey)) { + throw new IllegalArgumentException(String.format(Locale.ROOT, "Expected: \"%s\" But got: \"%s\"", expectedKey, keyValue)); + } + + return keyValueArr[1]; + } +} diff --git a/src/main/java/org/opensearch/knn/index/mapper/MethodFieldMapper.java b/src/main/java/org/opensearch/knn/index/mapper/MethodFieldMapper.java index c602b53fb..90d4ca879 100644 --- a/src/main/java/org/opensearch/knn/index/mapper/MethodFieldMapper.java +++ b/src/main/java/org/opensearch/knn/index/mapper/MethodFieldMapper.java @@ -16,6 +16,8 @@ import org.opensearch.knn.index.engine.KNNLibraryIndexingContext; import org.opensearch.knn.index.engine.KNNMethodConfigContext; import org.opensearch.knn.index.engine.KNNMethodContext; +import org.opensearch.knn.index.engine.qframe.QuantizationConfig; +import org.opensearch.knn.index.engine.qframe.QuantizationConfigParser; import java.io.IOException; import java.util.Map; @@ -24,6 +26,7 @@ import static org.opensearch.knn.common.KNNConstants.DIMENSION; import static org.opensearch.knn.common.KNNConstants.KNN_ENGINE; import static org.opensearch.knn.common.KNNConstants.PARAMETERS; +import static org.opensearch.knn.common.KNNConstants.QFRAMEWORK_CONFIG; import static org.opensearch.knn.common.KNNConstants.SPACE_TYPE; import static org.opensearch.knn.common.KNNConstants.VECTOR_DATA_TYPE_FIELD; @@ -105,19 +108,24 @@ private MethodFieldMapper( KNNMappingConfig annConfig = mappedFieldType.getKnnMappingConfig(); KNNMethodContext knnMethodContext = annConfig.getKnnMethodContext() .orElseThrow(() -> new IllegalArgumentException("KNN method context cannot be empty")); - this.fieldType = new FieldType(KNNVectorFieldMapper.Defaults.FIELD_TYPE); + KNNEngine knnEngine = knnMethodContext.getKnnEngine(); + KNNLibraryIndexingContext knnLibraryIndexingContext = knnEngine.getKNNLibraryIndexingContext( + knnMethodContext, + knnMethodConfigContext + ); + QuantizationConfig quantizationConfig = knnLibraryIndexingContext.getQuantizationConfig(); + this.fieldType = new FieldType(KNNVectorFieldMapper.Defaults.FIELD_TYPE); this.fieldType.putAttribute(DIMENSION, String.valueOf(annConfig.getDimension())); this.fieldType.putAttribute(SPACE_TYPE, knnMethodContext.getSpaceType().getValue()); - this.fieldType.putAttribute(VECTOR_DATA_TYPE_FIELD, vectorDataType.getValue()); + // Conditionally add quantization config + if (quantizationConfig != null && quantizationConfig != QuantizationConfig.EMPTY) { + this.fieldType.putAttribute(QFRAMEWORK_CONFIG, QuantizationConfigParser.toCsv(quantizationConfig)); + } - KNNEngine knnEngine = knnMethodContext.getKnnEngine(); + this.fieldType.putAttribute(VECTOR_DATA_TYPE_FIELD, vectorDataType.getValue()); this.fieldType.putAttribute(KNN_ENGINE, knnEngine.getName()); - KNNLibraryIndexingContext knnLibraryIndexingContext = knnEngine.getKNNLibraryIndexingContext( - knnMethodContext, - knnMethodConfigContext - ); try { this.fieldType.putAttribute( PARAMETERS, diff --git a/src/main/java/org/opensearch/knn/index/mapper/ModelFieldMapper.java b/src/main/java/org/opensearch/knn/index/mapper/ModelFieldMapper.java index a7dc17288..b29466eef 100644 --- a/src/main/java/org/opensearch/knn/index/mapper/ModelFieldMapper.java +++ b/src/main/java/org/opensearch/knn/index/mapper/ModelFieldMapper.java @@ -17,6 +17,8 @@ import org.opensearch.knn.index.engine.KNNMethodConfigContext; import org.opensearch.knn.index.engine.KNNMethodContext; import org.opensearch.knn.index.engine.MethodComponentContext; +import org.opensearch.knn.index.engine.qframe.QuantizationConfig; +import org.opensearch.knn.index.engine.qframe.QuantizationConfigParser; import org.opensearch.knn.indices.ModelDao; import org.opensearch.knn.indices.ModelMetadata; import org.opensearch.knn.indices.ModelUtil; @@ -26,6 +28,7 @@ import java.util.Optional; import static org.opensearch.knn.common.KNNConstants.MODEL_ID; +import static org.opensearch.knn.common.KNNConstants.QFRAMEWORK_CONFIG; /** * Field mapper for model in mapping @@ -209,6 +212,19 @@ protected void parseCreateField(ParseContext context) throws IOException { } else { fieldType.setDocValuesType(DocValuesType.BINARY); } + + // Conditionally add quantization config + KNNMethodContext knnMethodContext = getKNNMethodContextFromModelMetadata(modelMetadata); + KNNMethodConfigContext knnMethodConfigContext = getKNNMethodConfigContextFromModelMetadata(modelMetadata); + if (knnMethodContext != null && knnMethodConfigContext != null) { + KNNLibraryIndexingContext knnLibraryIndexingContext = modelMetadata.getKnnEngine() + .getKNNLibraryIndexingContext(knnMethodContext, knnMethodConfigContext); + QuantizationConfig quantizationConfig = knnLibraryIndexingContext.getQuantizationConfig(); + if (quantizationConfig != null && quantizationConfig != QuantizationConfig.EMPTY) { + this.fieldType.putAttribute(QFRAMEWORK_CONFIG, QuantizationConfigParser.toCsv(quantizationConfig)); + } + } + parseCreateField(context, modelMetadata.getDimension(), modelMetadata.getVectorDataType()); } diff --git a/src/main/java/org/opensearch/knn/index/query/KNNWeight.java b/src/main/java/org/opensearch/knn/index/query/KNNWeight.java index df40f6850..1863e0632 100644 --- a/src/main/java/org/opensearch/knn/index/query/KNNWeight.java +++ b/src/main/java/org/opensearch/knn/index/query/KNNWeight.java @@ -27,10 +27,12 @@ import org.apache.lucene.util.FixedBitSet; import org.opensearch.common.io.PathUtils; import org.opensearch.common.lucene.Lucene; +import org.opensearch.knn.common.FieldInfoExtractor; import org.opensearch.knn.common.KNNConstants; import org.opensearch.knn.index.KNNSettings; import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.VectorDataType; +import org.opensearch.knn.index.engine.qframe.QuantizationConfig; import org.opensearch.knn.index.memory.NativeMemoryAllocation; import org.opensearch.knn.index.memory.NativeMemoryCacheManager; import org.opensearch.knn.index.memory.NativeMemoryEntryContext; @@ -231,6 +233,9 @@ private Map doANNSearch(final LeafReaderContext context, final B return null; } + // TODO: Use this to get quantization config + QuantizationConfig quantizationConfig = FieldInfoExtractor.extractQuantizationConfig(fieldInfo); + KNNEngine knnEngine; SpaceType spaceType; VectorDataType vectorDataType; diff --git a/src/test/java/org/opensearch/knn/index/engine/AbstractKNNLibraryTests.java b/src/test/java/org/opensearch/knn/index/engine/AbstractKNNLibraryTests.java index 6f8f9afe5..ccaeb19a5 100644 --- a/src/test/java/org/opensearch/knn/index/engine/AbstractKNNLibraryTests.java +++ b/src/test/java/org/opensearch/knn/index/engine/AbstractKNNLibraryTests.java @@ -44,7 +44,11 @@ public ValidationException validate(KNNMethodContext knnMethodContext, KNNMethod private final static Map VALID_EXPECTED_MAP = ImmutableMap.of("test-key", "test-param"); private final static KNNMethod VALID_METHOD = new AbstractKNNMethod( MethodComponent.Builder.builder(VALID_METHOD_NAME) - .setMapGenerator((methodComponent, methodComponentContext, knnMethodConfigContext) -> VALID_EXPECTED_MAP) + .setKnnLibraryIndexingContextGenerator( + (methodComponent, methodComponentContext, knnMethodConfigContext) -> KNNLibraryIndexingContextImpl.builder() + .parameters(new HashMap<>(VALID_EXPECTED_MAP)) + .build() + ) .addSupportedDataTypes(Set.of(VectorDataType.FLOAT)) .build(), Set.of(SpaceType.DEFAULT), diff --git a/src/test/java/org/opensearch/knn/index/engine/AbstractKNNMethodTests.java b/src/test/java/org/opensearch/knn/index/engine/AbstractKNNMethodTests.java index 4d743c42a..241703d8b 100644 --- a/src/test/java/org/opensearch/knn/index/engine/AbstractKNNMethodTests.java +++ b/src/test/java/org/opensearch/knn/index/engine/AbstractKNNMethodTests.java @@ -154,9 +154,13 @@ public void testGetKNNLibraryIndexingContext() { .build(); SpaceType spaceType = SpaceType.DEFAULT; String methodName = "test-method"; - Map generatedMap = ImmutableMap.of("test-key", "test-value"); + Map generatedMap = new HashMap<>(ImmutableMap.of("test-key", "test-value")); MethodComponent methodComponent = MethodComponent.Builder.builder(methodName) - .setMapGenerator(((methodComponent1, methodComponentContext, methodConfigContext) -> methodComponentContext.getParameters())) + .setKnnLibraryIndexingContextGenerator( + ((methodComponent1, methodComponentContext, methodConfigContext) -> KNNLibraryIndexingContextImpl.builder() + .parameters(methodComponentContext.getParameters()) + .build()) + ) .build(); KNNMethod knnMethod = new TestKNNMethod(methodComponent, Set.of(SpaceType.L2), EMPTY_ENGINE_SPECIFIC_CONTEXT); diff --git a/src/test/java/org/opensearch/knn/index/engine/MethodComponentTests.java b/src/test/java/org/opensearch/knn/index/engine/MethodComponentTests.java index a5c72f5ee..7730095c7 100644 --- a/src/test/java/org/opensearch/knn/index/engine/MethodComponentTests.java +++ b/src/test/java/org/opensearch/knn/index/engine/MethodComponentTests.java @@ -143,19 +143,22 @@ public void testGetAsMap_withoutGenerator() throws IOException { assertEquals( in, - methodComponent.getAsMap(methodComponentContext, KNNMethodConfigContext.builder().versionCreated(Version.CURRENT).build()) + methodComponent.getKNNLibraryIndexingContext( + methodComponentContext, + KNNMethodConfigContext.builder().versionCreated(Version.CURRENT).build() + ).getLibraryParameters() ); xContentBuilder = XContentFactory.jsonBuilder().startObject().field(NAME, methodName).endObject(); in = xContentBuilderToMap(xContentBuilder); methodComponentContext = MethodComponentContext.parse(in); - Map methodAsMap = methodComponent.getAsMap( + KNNLibraryIndexingContext methodAsMap = methodComponent.getKNNLibraryIndexingContext( methodComponentContext, KNNMethodConfigContext.builder().versionCreated(Version.CURRENT).build() ); - assertEquals(default1, ((Map) methodAsMap.get(PARAMETERS)).get(parameterName1)); - assertEquals(default2, ((Map) methodAsMap.get(PARAMETERS)).get(parameterName2)); + assertEquals(default1, ((Map) methodAsMap.getLibraryParameters().get(PARAMETERS)).get(parameterName1)); + assertEquals(default2, ((Map) methodAsMap.getLibraryParameters().get(PARAMETERS)).get(parameterName2)); } public void testGetAsMap_withGenerator() throws IOException { @@ -164,7 +167,11 @@ public void testGetAsMap_withGenerator() throws IOException { MethodComponent methodComponent = MethodComponent.Builder.builder(methodName) .addParameter("valid1", new Parameter.IntegerParameter("valid1", 1, (v, context) -> v > 0)) .addParameter("valid2", new Parameter.IntegerParameter("valid2", 1, (v, context) -> v > 0)) - .setMapGenerator((methodComponent1, methodComponentContext, knnMethodConfigContext) -> generatedMap) + .setKnnLibraryIndexingContextGenerator( + (methodComponent1, methodComponentContext, knnMethodConfigContext) -> KNNLibraryIndexingContextImpl.builder() + .parameters(generatedMap) + .build() + ) .build(); XContentBuilder xContentBuilder = XContentFactory.jsonBuilder().startObject().field(NAME, methodName).endObject(); @@ -173,7 +180,10 @@ public void testGetAsMap_withGenerator() throws IOException { assertEquals( generatedMap, - methodComponent.getAsMap(methodComponentContext, KNNMethodConfigContext.builder().versionCreated(Version.CURRENT).build()) + methodComponent.getKNNLibraryIndexingContext( + methodComponentContext, + KNNMethodConfigContext.builder().versionCreated(Version.CURRENT).build() + ).getLibraryParameters() ); } @@ -191,12 +201,17 @@ public void testBuilder() { assertEquals(1, methodComponent.getParameters().size()); Map generatedMap = ImmutableMap.of("test-key", "test-value"); - builder.setMapGenerator((methodComponent1, methodComponentContext, knnMethodConfigContext) -> generatedMap); + builder.setKnnLibraryIndexingContextGenerator( + (methodComponent1, methodComponentContext, knnMethodConfigContext) -> KNNLibraryIndexingContextImpl.builder() + .parameters(generatedMap) + .build() + ); methodComponent = builder.build(); assertEquals( generatedMap, - methodComponent.getAsMap(null, KNNMethodConfigContext.builder().versionCreated(Version.CURRENT).build()) + methodComponent.getKNNLibraryIndexingContext(null, KNNMethodConfigContext.builder().versionCreated(Version.CURRENT).build()) + .getLibraryParameters() ); } } diff --git a/src/test/java/org/opensearch/knn/index/engine/faiss/FaissTests.java b/src/test/java/org/opensearch/knn/index/engine/faiss/FaissTests.java index 5dfd6c58c..99bc930a6 100644 --- a/src/test/java/org/opensearch/knn/index/engine/faiss/FaissTests.java +++ b/src/test/java/org/opensearch/knn/index/engine/faiss/FaissTests.java @@ -11,11 +11,15 @@ import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.knn.KNNTestCase; import org.opensearch.knn.index.VectorDataType; +import org.opensearch.knn.index.engine.KNNLibraryIndexingContext; +import org.opensearch.knn.index.engine.KNNLibraryIndexingContextImpl; import org.opensearch.knn.index.engine.KNNMethodConfigContext; import org.opensearch.knn.index.engine.KNNMethodContext; import org.opensearch.knn.index.engine.MethodComponent; import org.opensearch.knn.index.engine.MethodComponentContext; import org.opensearch.knn.index.engine.Parameter; +import org.opensearch.knn.index.engine.qframe.QuantizationConfig; +import org.opensearch.knn.quantization.enums.ScalarQuantizationType; import java.io.IOException; import java.util.HashMap; @@ -41,7 +45,7 @@ public class FaissTests extends KNNTestCase { - public void testGetMethodAsMap_whenMethodIsHNSWFlat_thenCreateCorrectIndexDescription() throws IOException { + public void testGetKNNLibraryIndexingContext_whenMethodIsHNSWFlat_thenCreateCorrectIndexDescription() throws IOException { KNNMethodConfigContext knnMethodConfigContext = KNNMethodConfigContext.builder() .versionCreated(org.opensearch.Version.CURRENT) .dimension(4) @@ -69,7 +73,7 @@ public void testGetMethodAsMap_whenMethodIsHNSWFlat_thenCreateCorrectIndexDescri assertEquals(expectedIndexDescription, map.get(INDEX_DESCRIPTION_PARAMETER)); } - public void testGetMethodAsMap_whenMethodIsHNSWPQ_thenCreateCorrectIndexDescription() throws IOException { + public void testGetKNNLibraryIndexingContext_whenMethodIsHNSWPQ_thenCreateCorrectIndexDescription() throws IOException { KNNMethodConfigContext knnMethodConfigContext = KNNMethodConfigContext.builder() .versionCreated(org.opensearch.Version.CURRENT) .dimension(4) @@ -104,7 +108,7 @@ public void testGetMethodAsMap_whenMethodIsHNSWPQ_thenCreateCorrectIndexDescript } @SneakyThrows - public void testGetMethodAsMap_whenMethodIsHNSWSQFP16_thenCreateCorrectIndexDescription() { + public void testGetKNNLibraryIndexingContext_whenMethodIsHNSWSQFP16_thenCreateCorrectIndexDescription() { KNNMethodConfigContext knnMethodConfigContext = KNNMethodConfigContext.builder() .versionCreated(org.opensearch.Version.CURRENT) .dimension(4) @@ -137,7 +141,7 @@ public void testGetMethodAsMap_whenMethodIsHNSWSQFP16_thenCreateCorrectIndexDesc assertEquals(expectedIndexDescription, map.get(INDEX_DESCRIPTION_PARAMETER)); } - public void testGetMethodAsMap_whenMethodIsIVFFlat_thenCreateCorrectIndexDescription() throws IOException { + public void testGetKNNLibraryIndexingContext_whenMethodIsIVFFlat_thenCreateCorrectIndexDescription() throws IOException { KNNMethodConfigContext knnMethodConfigContext = KNNMethodConfigContext.builder() .versionCreated(org.opensearch.Version.CURRENT) .dimension(4) @@ -164,7 +168,7 @@ public void testGetMethodAsMap_whenMethodIsIVFFlat_thenCreateCorrectIndexDescrip assertEquals(expectedIndexDescription, map.get(INDEX_DESCRIPTION_PARAMETER)); } - public void testGetMethodAsMap_whenMethodIsIVFPQ_thenCreateCorrectIndexDescription() throws IOException { + public void testGetKNNLibraryIndexingContext_whenMethodIsIVFPQ_thenCreateCorrectIndexDescription() throws IOException { KNNMethodConfigContext knnMethodConfigContext = KNNMethodConfigContext.builder() .versionCreated(org.opensearch.Version.CURRENT) .dimension(4) @@ -201,7 +205,7 @@ public void testGetMethodAsMap_whenMethodIsIVFPQ_thenCreateCorrectIndexDescripti } @SneakyThrows - public void testGetMethodAsMap_whenMethodIsIVFSQFP16_thenCreateCorrectIndexDescription() { + public void testGetKNNLibraryIndexingContext_whenMethodIsIVFSQFP16_thenCreateCorrectIndexDescription() { KNNMethodConfigContext knnMethodConfigContext = KNNMethodConfigContext.builder() .versionCreated(org.opensearch.Version.CURRENT) .dimension(4) @@ -234,6 +238,84 @@ public void testGetMethodAsMap_whenMethodIsIVFSQFP16_thenCreateCorrectIndexDescr assertEquals(expectedIndexDescription, map.get(INDEX_DESCRIPTION_PARAMETER)); } + @SneakyThrows + public void testGetKNNLibraryIndexingContext_whenMethodIsHNSWWithQFrame_thenCreateCorrectConfig() { + KNNMethodConfigContext knnMethodConfigContext = KNNMethodConfigContext.builder() + .versionCreated(org.opensearch.Version.CURRENT) + .dimension(4) + .vectorDataType(VectorDataType.FLOAT) + .build(); + int m = 88; + String expectedIndexDescription = String.format(Locale.ROOT, "HNSW%d,Flat", m); + XContentBuilder xContentBuilder = XContentFactory.jsonBuilder() + .startObject() + .field(NAME, METHOD_HNSW) + .field(KNN_ENGINE, FAISS_NAME) + .startObject(PARAMETERS) + .field(METHOD_PARAMETER_M, m) + .startObject(METHOD_ENCODER_PARAMETER) + .field(NAME, QFrameBitEncoder.NAME) + .startObject(PARAMETERS) + .field(QFrameBitEncoder.BITCOUNT_PARAM, 4) + .endObject() + .endObject() + .endObject() + .endObject(); + Map in = xContentBuilderToMap(xContentBuilder); + KNNMethodContext knnMethodContext = KNNMethodContext.parse(in); + KNNLibraryIndexingContext knnLibraryIndexingContext = Faiss.INSTANCE.getKNNLibraryIndexingContext( + knnMethodContext, + knnMethodConfigContext + ); + Map map = knnLibraryIndexingContext.getLibraryParameters(); + + assertTrue(map.containsKey(INDEX_DESCRIPTION_PARAMETER)); + assertEquals(expectedIndexDescription, map.get(INDEX_DESCRIPTION_PARAMETER)); + assertEquals( + QuantizationConfig.builder().quantizationType(ScalarQuantizationType.FOUR_BIT).build(), + knnLibraryIndexingContext.getQuantizationConfig() + ); + } + + @SneakyThrows + public void testGetKNNLibraryIndexingContext_whenMethodIsIVFWithQFrame_thenCreateCorrectConfig() { + KNNMethodConfigContext knnMethodConfigContext = KNNMethodConfigContext.builder() + .versionCreated(org.opensearch.Version.CURRENT) + .dimension(4) + .vectorDataType(VectorDataType.FLOAT) + .build(); + int nlist = 88; + String expectedIndexDescription = String.format(Locale.ROOT, "IVF%d,Flat", nlist); + XContentBuilder xContentBuilder = XContentFactory.jsonBuilder() + .startObject() + .field(NAME, METHOD_IVF) + .field(KNN_ENGINE, FAISS_NAME) + .startObject(PARAMETERS) + .field(METHOD_PARAMETER_NLIST, nlist) + .startObject(METHOD_ENCODER_PARAMETER) + .field(NAME, QFrameBitEncoder.NAME) + .startObject(PARAMETERS) + .field(QFrameBitEncoder.BITCOUNT_PARAM, 2) + .endObject() + .endObject() + .endObject() + .endObject(); + Map in = xContentBuilderToMap(xContentBuilder); + KNNMethodContext knnMethodContext = KNNMethodContext.parse(in); + KNNLibraryIndexingContext knnLibraryIndexingContext = Faiss.INSTANCE.getKNNLibraryIndexingContext( + knnMethodContext, + knnMethodConfigContext + ); + Map map = knnLibraryIndexingContext.getLibraryParameters(); + + assertTrue(map.containsKey(INDEX_DESCRIPTION_PARAMETER)); + assertEquals(expectedIndexDescription, map.get(INDEX_DESCRIPTION_PARAMETER)); + assertEquals( + QuantizationConfig.builder().quantizationType(ScalarQuantizationType.TWO_BIT).build(), + knnLibraryIndexingContext.getQuantizationConfig() + ); + } + public void testMethodAsMapBuilder() throws IOException { String methodName = "test-method"; String methodDescription = "test-description"; @@ -269,15 +351,20 @@ public void testMethodAsMapBuilder() throws IOException { expectedMap.put(PARAMETERS, expectedParametersMap); expectedMap.put(NAME, methodName); expectedMap.put(INDEX_DESCRIPTION_PARAMETER, methodDescription + value1); + KNNLibraryIndexingContext expectedKNNMethodContext = KNNLibraryIndexingContextImpl.builder().parameters(expectedMap).build(); - Map methodAsMap = MethodAsMapBuilder.builder( + KNNLibraryIndexingContext actualKNNLibraryIndexingContext = MethodAsMapBuilder.builder( methodDescription, methodComponent, methodComponentContext, KNNMethodConfigContext.builder().versionCreated(Version.CURRENT).build() ).addParameter(parameter1, "", "").build(); - assertEquals(expectedMap, methodAsMap); + assertEquals(expectedKNNMethodContext.getQuantizationConfig(), actualKNNLibraryIndexingContext.getQuantizationConfig()); + assertEquals(expectedKNNMethodContext.getLibraryParameters(), actualKNNLibraryIndexingContext.getLibraryParameters()); + assertEquals(expectedKNNMethodContext.getPerDimensionProcessor(), actualKNNLibraryIndexingContext.getPerDimensionProcessor()); + assertEquals(expectedKNNMethodContext.getPerDimensionValidator(), actualKNNLibraryIndexingContext.getPerDimensionValidator()); + assertEquals(expectedKNNMethodContext.getVectorValidator(), actualKNNLibraryIndexingContext.getVectorValidator()); } } diff --git a/src/test/java/org/opensearch/knn/index/engine/faiss/QFrameBitEncoderTests.java b/src/test/java/org/opensearch/knn/index/engine/faiss/QFrameBitEncoderTests.java new file mode 100644 index 000000000..7457b49aa --- /dev/null +++ b/src/test/java/org/opensearch/knn/index/engine/faiss/QFrameBitEncoderTests.java @@ -0,0 +1,124 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.engine.faiss; + +import com.google.common.collect.ImmutableMap; +import org.opensearch.Version; +import org.opensearch.knn.KNNTestCase; +import org.opensearch.knn.index.VectorDataType; +import org.opensearch.knn.index.engine.KNNLibraryIndexingContext; +import org.opensearch.knn.index.engine.KNNMethodConfigContext; +import org.opensearch.knn.index.engine.MethodComponent; +import org.opensearch.knn.index.engine.MethodComponentContext; +import org.opensearch.knn.index.engine.qframe.QuantizationConfig; +import org.opensearch.knn.quantization.enums.ScalarQuantizationType; + +import static org.opensearch.knn.common.KNNConstants.FAISS_FLAT_DESCRIPTION; +import static org.opensearch.knn.common.KNNConstants.INDEX_DESCRIPTION_PARAMETER; +import static org.opensearch.knn.index.engine.faiss.QFrameBitEncoder.BITCOUNT_PARAM; + +public class QFrameBitEncoderTests extends KNNTestCase { + public void testGetLibraryIndexingContext() { + QFrameBitEncoder qFrameBitEncoder = new QFrameBitEncoder(); + MethodComponent methodComponent = qFrameBitEncoder.getMethodComponent(); + KNNMethodConfigContext knnMethodConfigContext = KNNMethodConfigContext.builder() + .versionCreated(Version.CURRENT) + .vectorDataType(VectorDataType.FLOAT) + .dimension(10) + .build(); + + MethodComponentContext methodComponentContext = new MethodComponentContext( + QFrameBitEncoder.NAME, + ImmutableMap.of(BITCOUNT_PARAM, 4) + ); + + KNNLibraryIndexingContext knnLibraryIndexingContext = methodComponent.getKNNLibraryIndexingContext( + methodComponentContext, + knnMethodConfigContext + ); + assertEquals( + ImmutableMap.of(INDEX_DESCRIPTION_PARAMETER, FAISS_FLAT_DESCRIPTION), + knnLibraryIndexingContext.getLibraryParameters() + ); + assertEquals( + QuantizationConfig.builder().quantizationType(ScalarQuantizationType.FOUR_BIT).build(), + knnLibraryIndexingContext.getQuantizationConfig() + ); + + methodComponentContext = new MethodComponentContext(QFrameBitEncoder.NAME, ImmutableMap.of(BITCOUNT_PARAM, 2)); + knnLibraryIndexingContext = methodComponent.getKNNLibraryIndexingContext(methodComponentContext, knnMethodConfigContext); + assertEquals( + ImmutableMap.of(INDEX_DESCRIPTION_PARAMETER, FAISS_FLAT_DESCRIPTION), + knnLibraryIndexingContext.getLibraryParameters() + ); + assertEquals( + QuantizationConfig.builder().quantizationType(ScalarQuantizationType.TWO_BIT).build(), + knnLibraryIndexingContext.getQuantizationConfig() + ); + } + + public void testValidate() { + QFrameBitEncoder qFrameBitEncoder = new QFrameBitEncoder(); + MethodComponent methodComponent = qFrameBitEncoder.getMethodComponent(); + + // Invalid data type + KNNMethodConfigContext knnMethodConfigContext = KNNMethodConfigContext.builder() + .versionCreated(Version.CURRENT) + .vectorDataType(VectorDataType.BYTE) + .dimension(10) + .build(); + MethodComponentContext methodComponentContext = new MethodComponentContext( + QFrameBitEncoder.NAME, + ImmutableMap.of(BITCOUNT_PARAM, 4) + ); + + assertNotNull(methodComponent.validate(methodComponentContext, knnMethodConfigContext)); + + // Invalid param + knnMethodConfigContext = KNNMethodConfigContext.builder() + .versionCreated(Version.CURRENT) + .vectorDataType(VectorDataType.FLOAT) + .dimension(10) + .build(); + methodComponentContext = new MethodComponentContext(QFrameBitEncoder.NAME, ImmutableMap.of(BITCOUNT_PARAM, 4, "invalid", 4)); + assertNotNull(methodComponent.validate(methodComponentContext, knnMethodConfigContext)); + + // Invalid param type + knnMethodConfigContext = KNNMethodConfigContext.builder() + .versionCreated(Version.CURRENT) + .vectorDataType(VectorDataType.FLOAT) + .dimension(10) + .build(); + methodComponentContext = new MethodComponentContext(QFrameBitEncoder.NAME, ImmutableMap.of(BITCOUNT_PARAM, "invalid")); + assertNotNull(methodComponent.validate(methodComponentContext, knnMethodConfigContext)); + + // Invalid param value + knnMethodConfigContext = KNNMethodConfigContext.builder() + .versionCreated(Version.CURRENT) + .vectorDataType(VectorDataType.FLOAT) + .dimension(10) + .build(); + methodComponentContext = new MethodComponentContext(QFrameBitEncoder.NAME, ImmutableMap.of(BITCOUNT_PARAM, 5)); + assertNotNull(methodComponent.validate(methodComponentContext, knnMethodConfigContext)); + } + + public void testIsTrainingRequired() { + QFrameBitEncoder qFrameBitEncoder = new QFrameBitEncoder(); + assertFalse( + qFrameBitEncoder.getMethodComponent() + .isTrainingRequired(new MethodComponentContext(QFrameBitEncoder.NAME, ImmutableMap.of(BITCOUNT_PARAM, 4))) + ); + } + + public void testEstimateOverheadInKB() { + QFrameBitEncoder qFrameBitEncoder = new QFrameBitEncoder(); + assertEquals( + 0, + qFrameBitEncoder.getMethodComponent() + .estimateOverheadInKB(new MethodComponentContext(QFrameBitEncoder.NAME, ImmutableMap.of(BITCOUNT_PARAM, 4)), 8) + ); + } +} diff --git a/src/test/java/org/opensearch/knn/index/engine/qframe/QuantizationConfigParserTests.java b/src/test/java/org/opensearch/knn/index/engine/qframe/QuantizationConfigParserTests.java new file mode 100644 index 000000000..8d25c578d --- /dev/null +++ b/src/test/java/org/opensearch/knn/index/engine/qframe/QuantizationConfigParserTests.java @@ -0,0 +1,91 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.engine.qframe; + +import org.opensearch.knn.KNNTestCase; +import org.opensearch.knn.quantization.enums.ScalarQuantizationType; + +import java.util.Locale; + +public class QuantizationConfigParserTests extends KNNTestCase { + + public void testFromCsv() { + assertEquals(QuantizationConfig.EMPTY, QuantizationConfigParser.fromCsv("")); + assertEquals(QuantizationConfig.EMPTY, QuantizationConfigParser.fromCsv(null)); + + expectThrows( + IllegalArgumentException.class, + () -> QuantizationConfigParser.fromCsv( + String.format(Locale.ROOT, "%s=%s", QuantizationConfigParser.TYPE_NAME, QuantizationConfigParser.BINARY_TYPE) + ) + ); + + expectThrows( + IllegalArgumentException.class, + () -> QuantizationConfigParser.fromCsv( + String.format( + Locale.ROOT, + "%s=%s,%s=%d", + QuantizationConfigParser.TYPE_NAME, + "invalid", + QuantizationConfigParser.BIT_COUNT_NAME, + 4 + ) + ) + ); + + expectThrows( + IllegalArgumentException.class, + () -> QuantizationConfigParser.fromCsv( + String.format( + Locale.ROOT, + "%s=%s,%s=%d", + QuantizationConfigParser.TYPE_NAME, + QuantizationConfigParser.BINARY_TYPE, + "invalid", + 4 + ) + ) + ); + + expectThrows( + IllegalArgumentException.class, + () -> QuantizationConfigParser.fromCsv( + String.format( + Locale.ROOT, + "%s=%s,%s=%s", + QuantizationConfigParser.TYPE_NAME, + QuantizationConfigParser.BINARY_TYPE, + QuantizationConfigParser.BIT_COUNT_NAME, + "invalid" + ) + ) + ); + + assertEquals( + QuantizationConfig.builder().quantizationType(ScalarQuantizationType.FOUR_BIT).build(), + QuantizationConfigParser.fromCsv( + String.format( + Locale.ROOT, + "%s=%s,%s=%d", + QuantizationConfigParser.TYPE_NAME, + QuantizationConfigParser.BINARY_TYPE, + QuantizationConfigParser.BIT_COUNT_NAME, + 4 + ) + ) + ); + } + + public void testToCsv() { + assertEquals("", QuantizationConfigParser.toCsv(null)); + assertEquals("", QuantizationConfigParser.toCsv(QuantizationConfig.EMPTY)); + assertEquals( + "type=binary,bits=2", + QuantizationConfigParser.toCsv(QuantizationConfig.builder().quantizationType(ScalarQuantizationType.TWO_BIT).build()) + ); + } +} diff --git a/src/test/java/org/opensearch/knn/integ/QFrameworkIT.java b/src/test/java/org/opensearch/knn/integ/QFrameworkIT.java new file mode 100644 index 000000000..e3f8b607a --- /dev/null +++ b/src/test/java/org/opensearch/knn/integ/QFrameworkIT.java @@ -0,0 +1,77 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.integ; + +import org.opensearch.client.Response; +import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.knn.KNNRestTestCase; +import org.opensearch.knn.index.engine.faiss.QFrameBitEncoder; + +import java.io.IOException; + +import static org.opensearch.knn.common.KNNConstants.FAISS_NAME; +import static org.opensearch.knn.common.KNNConstants.KNN_ENGINE; +import static org.opensearch.knn.common.KNNConstants.KNN_METHOD; +import static org.opensearch.knn.common.KNNConstants.METHOD_ENCODER_PARAMETER; +import static org.opensearch.knn.common.KNNConstants.METHOD_HNSW; +import static org.opensearch.knn.common.KNNConstants.NAME; +import static org.opensearch.knn.common.KNNConstants.PARAMETERS; + +public class QFrameworkIT extends KNNRestTestCase { + + private final static float[] TEST_VECTOR = new float[] { 1.0f, 2.0f }; + private final static int DIMENSION = 2; + private final static int K = 1; + + public void testBaseCase() throws IOException { + createTestIndex(4); + addKnnDoc(INDEX_NAME, "1", FIELD_NAME, TEST_VECTOR); + Response response = searchKNNIndex( + INDEX_NAME, + XContentFactory.jsonBuilder() + .startObject() + .startObject("query") + .startObject("knn") + .startObject(FIELD_NAME) + .field("vector", TEST_VECTOR) + .field("k", K) + .endObject() + .endObject() + .endObject() + .endObject(), + 1 + ); + assertOK(response); + } + + private void createTestIndex(int bitCount) throws IOException { + XContentBuilder builder = XContentFactory.jsonBuilder() + .startObject() + .startObject("properties") + .startObject(FIELD_NAME) + .field("type", "knn_vector") + .field("dimension", DIMENSION) + .startObject(KNN_METHOD) + .field(NAME, METHOD_HNSW) + .field(KNN_ENGINE, FAISS_NAME) + .startObject(PARAMETERS) + .startObject(METHOD_ENCODER_PARAMETER) + .field(NAME, QFrameBitEncoder.NAME) + .startObject(PARAMETERS) + .field(QFrameBitEncoder.BITCOUNT_PARAM, bitCount) + .endObject() + .endObject() + .endObject() + .endObject() + .endObject() + .endObject() + .endObject(); + + String mapping = builder.toString(); + createKnnIndex(INDEX_NAME, mapping); + } +} From ebd8a828b260d0e03bce11cdaae32211dc7c7f1b Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Wed, 21 Aug 2024 18:10:31 -0700 Subject: [PATCH 335/416] Fix string for qframe parser key (#2000) Signed-off-by: John Mazanec (cherry picked from commit b739f384d6623ef76dcb53f4b81e3e3076a99c18) --- .../qframe/QuantizationConfigParser.java | 10 +--- .../knn/index/engine/faiss/FaissTests.java | 4 +- .../qframe/QuantizationConfigParserTests.java | 57 +++++++------------ 3 files changed, 24 insertions(+), 47 deletions(-) diff --git a/src/main/java/org/opensearch/knn/index/engine/qframe/QuantizationConfigParser.java b/src/main/java/org/opensearch/knn/index/engine/qframe/QuantizationConfigParser.java index 1f3349d34..f86d7f886 100644 --- a/src/main/java/org/opensearch/knn/index/engine/qframe/QuantizationConfigParser.java +++ b/src/main/java/org/opensearch/knn/index/engine/qframe/QuantizationConfigParser.java @@ -33,14 +33,8 @@ public static String toCsv(QuantizationConfig quantizationConfig) { || quantizationConfig.getQuantizationType() == null) { return ""; } - return String.format( - Locale.ROOT, - "%s=%s,%s=%d", - TYPE_NAME, - BINARY_TYPE, - BIT_COUNT_NAME, - quantizationConfig.getQuantizationType().getId() - ); + + return TYPE_NAME + SEPARATOR + BINARY_TYPE + "," + BIT_COUNT_NAME + SEPARATOR + quantizationConfig.getQuantizationType().getId(); } /** diff --git a/src/test/java/org/opensearch/knn/index/engine/faiss/FaissTests.java b/src/test/java/org/opensearch/knn/index/engine/faiss/FaissTests.java index 99bc930a6..c9ce50f22 100644 --- a/src/test/java/org/opensearch/knn/index/engine/faiss/FaissTests.java +++ b/src/test/java/org/opensearch/knn/index/engine/faiss/FaissTests.java @@ -246,7 +246,7 @@ public void testGetKNNLibraryIndexingContext_whenMethodIsHNSWWithQFrame_thenCrea .vectorDataType(VectorDataType.FLOAT) .build(); int m = 88; - String expectedIndexDescription = String.format(Locale.ROOT, "HNSW%d,Flat", m); + String expectedIndexDescription = "HNSW" + m + ",Flat"; XContentBuilder xContentBuilder = XContentFactory.jsonBuilder() .startObject() .field(NAME, METHOD_HNSW) @@ -285,7 +285,7 @@ public void testGetKNNLibraryIndexingContext_whenMethodIsIVFWithQFrame_thenCreat .vectorDataType(VectorDataType.FLOAT) .build(); int nlist = 88; - String expectedIndexDescription = String.format(Locale.ROOT, "IVF%d,Flat", nlist); + String expectedIndexDescription = "IVF" + nlist + ",Flat"; XContentBuilder xContentBuilder = XContentFactory.jsonBuilder() .startObject() .field(NAME, METHOD_IVF) diff --git a/src/test/java/org/opensearch/knn/index/engine/qframe/QuantizationConfigParserTests.java b/src/test/java/org/opensearch/knn/index/engine/qframe/QuantizationConfigParserTests.java index 8d25c578d..317d40e27 100644 --- a/src/test/java/org/opensearch/knn/index/engine/qframe/QuantizationConfigParserTests.java +++ b/src/test/java/org/opensearch/knn/index/engine/qframe/QuantizationConfigParserTests.java @@ -8,8 +8,6 @@ import org.opensearch.knn.KNNTestCase; import org.opensearch.knn.quantization.enums.ScalarQuantizationType; -import java.util.Locale; - public class QuantizationConfigParserTests extends KNNTestCase { public void testFromCsv() { @@ -18,64 +16,49 @@ public void testFromCsv() { expectThrows( IllegalArgumentException.class, - () -> QuantizationConfigParser.fromCsv( - String.format(Locale.ROOT, "%s=%s", QuantizationConfigParser.TYPE_NAME, QuantizationConfigParser.BINARY_TYPE) - ) + () -> QuantizationConfigParser.fromCsv(QuantizationConfigParser.TYPE_NAME + "=" + QuantizationConfigParser.BINARY_TYPE) ); expectThrows( IllegalArgumentException.class, () -> QuantizationConfigParser.fromCsv( - String.format( - Locale.ROOT, - "%s=%s,%s=%d", - QuantizationConfigParser.TYPE_NAME, - "invalid", - QuantizationConfigParser.BIT_COUNT_NAME, - 4 - ) + QuantizationConfigParser.TYPE_NAME + "=invalid," + QuantizationConfigParser.BIT_COUNT_NAME + "=4" ) ); expectThrows( IllegalArgumentException.class, () -> QuantizationConfigParser.fromCsv( - String.format( - Locale.ROOT, - "%s=%s,%s=%d", - QuantizationConfigParser.TYPE_NAME, - QuantizationConfigParser.BINARY_TYPE, - "invalid", - 4 - ) + QuantizationConfigParser.TYPE_NAME + + "=" + + QuantizationConfigParser.BINARY_TYPE + + ",invalid=4" + + QuantizationConfigParser.BIT_COUNT_NAME + + "=4" ) ); expectThrows( IllegalArgumentException.class, () -> QuantizationConfigParser.fromCsv( - String.format( - Locale.ROOT, - "%s=%s,%s=%s", - QuantizationConfigParser.TYPE_NAME, - QuantizationConfigParser.BINARY_TYPE, - QuantizationConfigParser.BIT_COUNT_NAME, - "invalid" - ) + QuantizationConfigParser.TYPE_NAME + + "=" + + QuantizationConfigParser.BINARY_TYPE + + "," + + QuantizationConfigParser.BIT_COUNT_NAME + + "=invalid" ) ); assertEquals( QuantizationConfig.builder().quantizationType(ScalarQuantizationType.FOUR_BIT).build(), QuantizationConfigParser.fromCsv( - String.format( - Locale.ROOT, - "%s=%s,%s=%d", - QuantizationConfigParser.TYPE_NAME, - QuantizationConfigParser.BINARY_TYPE, - QuantizationConfigParser.BIT_COUNT_NAME, - 4 - ) + QuantizationConfigParser.TYPE_NAME + + "=" + + QuantizationConfigParser.BINARY_TYPE + + "," + + QuantizationConfigParser.BIT_COUNT_NAME + + "=4" ) ); } From c5365de9e59d82d879481552be7b0fcd57d0c0b0 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Wed, 21 Aug 2024 19:30:10 -0700 Subject: [PATCH 336/416] k-NN query rescore support for native engines (#2001) Implements re-scoring. It uses the rescoring context in the query builder to execute a two-phased search. First, it oversamples the ANN index and then reduces the results down to the oversample factor, and then rescores and returns. (cherry picked from commit 33eb45b2cc82f95d257fe57de9c8fc611c474ca4) Signed-off-by: John Mazanec Co-authored-by: John Mazanec --- CHANGELOG.md | 1 + .../knn/common/FieldInfoExtractor.java | 36 +++- .../knn/index/query/BaseQueryFactory.java | 6 + .../knn/index/query/ExactSearcher.java | 154 ++++++++++++++++ .../opensearch/knn/index/query/KNNQuery.java | 22 ++- .../knn/index/query/KNNQueryBuilder.java | 1 + .../knn/index/query/KNNQueryFactory.java | 38 +--- .../opensearch/knn/index/query/KNNWeight.java | 171 +++++------------- .../knn/index/query/ResultUtil.java | 119 ++++++++++++ .../nativelib/NativeEngineKnnVectorQuery.java | 85 ++++++--- .../index/query/rescore/RescoreContext.java | 12 ++ .../knn/index/codec/KNNCodecTestCase.java | 30 +-- .../knn/index/query/KNNQueryFactoryTests.java | 64 +++++-- .../knn/index/query/KNNWeightTests.java | 31 +++- .../knn/index/query/ResultUtilTests.java | 140 ++++++++++++++ .../NativeEngineKNNVectorQueryTests.java | 67 +++++-- .../query/rescore/RescoreContextTests.java | 26 +++ .../opensearch/knn/integ/NestedSearchIT.java | 42 ++++- 18 files changed, 797 insertions(+), 248 deletions(-) create mode 100644 src/main/java/org/opensearch/knn/index/query/ExactSearcher.java create mode 100644 src/main/java/org/opensearch/knn/index/query/ResultUtil.java create mode 100644 src/test/java/org/opensearch/knn/index/query/ResultUtilTests.java create mode 100644 src/test/java/org/opensearch/knn/index/query/rescore/RescoreContextTests.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 3ab1861c8..acb195121 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ## [Unreleased 2.x](https://github.com/opensearch-project/k-NN/compare/2.16...2.x) ### Features * Integrate Lucene Vector field with native engines to use KNNVectorFormat during segment creation [#1945](https://github.com/opensearch-project/k-NN/pull/1945) +* k-NN query rescore support for native engines [#1984](https://github.com/opensearch-project/k-NN/pull/1984) ### Enhancements ### Bug Fixes * Corrected search logic for scenario with non-existent fields in filter [#1874](https://github.com/opensearch-project/k-NN/pull/1874) diff --git a/src/main/java/org/opensearch/knn/common/FieldInfoExtractor.java b/src/main/java/org/opensearch/knn/common/FieldInfoExtractor.java index 724632a46..cf866d840 100644 --- a/src/main/java/org/opensearch/knn/common/FieldInfoExtractor.java +++ b/src/main/java/org/opensearch/knn/common/FieldInfoExtractor.java @@ -8,6 +8,7 @@ import lombok.experimental.UtilityClass; import org.apache.commons.lang.StringUtils; import org.apache.lucene.index.FieldInfo; +import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.indices.ModelMetadata; import org.opensearch.knn.indices.ModelUtil; @@ -15,7 +16,13 @@ import org.opensearch.knn.index.engine.qframe.QuantizationConfig; import org.opensearch.knn.index.engine.qframe.QuantizationConfigParser; +import static org.opensearch.knn.common.KNNConstants.MODEL_ID; import static org.opensearch.knn.common.KNNConstants.QFRAMEWORK_CONFIG; +import org.opensearch.knn.indices.ModelDao; + +import java.util.Locale; + +import static org.opensearch.knn.common.KNNConstants.SPACE_TYPE; /** * A utility class to extract information from FieldInfo. @@ -31,7 +38,7 @@ public class FieldInfoExtractor { public static VectorDataType extractVectorDataType(final FieldInfo fieldInfo) { String vectorDataTypeString = fieldInfo.getAttribute(KNNConstants.VECTOR_DATA_TYPE_FIELD); if (StringUtils.isEmpty(vectorDataTypeString)) { - final ModelMetadata modelMetadata = ModelUtil.getModelMetadata(fieldInfo.getAttribute(KNNConstants.MODEL_ID)); + final ModelMetadata modelMetadata = ModelUtil.getModelMetadata(fieldInfo.getAttribute(MODEL_ID)); if (modelMetadata != null) { VectorDataType vectorDataType = modelMetadata.getVectorDataType(); vectorDataTypeString = vectorDataType == null ? null : vectorDataType.getValue(); @@ -53,4 +60,31 @@ public static QuantizationConfig extractQuantizationConfig(final FieldInfo field } return QuantizationConfigParser.fromCsv(quantizationConfigString); } + + /** + * Get the space type for the given field info. + * + * @param modelDao ModelDao instance to retrieve model metadata + * @param fieldInfo FieldInfo instance to extract space type from + * @return SpaceType for the given field info + */ + public static SpaceType getSpaceType(final ModelDao modelDao, final FieldInfo fieldInfo) { + final String spaceTypeString = fieldInfo.getAttribute(SPACE_TYPE); + if (StringUtils.isNotEmpty(spaceTypeString)) { + return SpaceType.getSpace(spaceTypeString); + } + + final String modelId = fieldInfo.getAttribute(MODEL_ID); + if (StringUtils.isEmpty(modelId)) { + throw new IllegalArgumentException( + String.format(Locale.ROOT, "Unable to find the Space Type from Field Info attribute for field %s", fieldInfo.getName()) + ); + } + + ModelMetadata modelMetadata = modelDao.getMetadata(modelId); + if (modelMetadata == null) { + throw new IllegalArgumentException(String.format(Locale.ROOT, "Unable to find the model metadata for model id %s", modelId)); + } + return modelMetadata.getSpaceType(); + } } diff --git a/src/main/java/org/opensearch/knn/index/query/BaseQueryFactory.java b/src/main/java/org/opensearch/knn/index/query/BaseQueryFactory.java index 931634862..cfb604c18 100644 --- a/src/main/java/org/opensearch/knn/index/query/BaseQueryFactory.java +++ b/src/main/java/org/opensearch/knn/index/query/BaseQueryFactory.java @@ -18,6 +18,7 @@ import org.opensearch.index.search.NestedHelper; import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.engine.KNNEngine; +import org.opensearch.knn.index.query.rescore.RescoreContext; import java.io.IOException; import java.util.Map; @@ -48,6 +49,7 @@ public static class CreateQueryRequest { private Float radius; private QueryBuilder filter; private QueryShardContext context; + private RescoreContext rescoreContext; public Optional getFilter() { return Optional.ofNullable(filter); @@ -56,6 +58,10 @@ public Optional getFilter() { public Optional getContext() { return Optional.ofNullable(context); } + + public Optional getRescoreContext() { + return Optional.ofNullable(rescoreContext); + } } /** diff --git a/src/main/java/org/opensearch/knn/index/query/ExactSearcher.java b/src/main/java/org/opensearch/knn/index/query/ExactSearcher.java new file mode 100644 index 000000000..249c66d03 --- /dev/null +++ b/src/main/java/org/opensearch/knn/index/query/ExactSearcher.java @@ -0,0 +1,154 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.query; + +import lombok.AllArgsConstructor; +import lombok.extern.log4j.Log4j2; +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.SegmentReader; +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.search.HitQueue; +import org.apache.lucene.search.ScoreDoc; +import org.apache.lucene.util.BitSet; +import org.opensearch.common.lucene.Lucene; +import org.opensearch.knn.common.FieldInfoExtractor; +import org.opensearch.knn.index.SpaceType; +import org.opensearch.knn.index.VectorDataType; +import org.opensearch.knn.index.query.filtered.FilteredIdsKNNByteIterator; +import org.opensearch.knn.index.query.filtered.FilteredIdsKNNIterator; +import org.opensearch.knn.index.query.filtered.KNNIterator; +import org.opensearch.knn.index.query.filtered.NestedFilteredIdsKNNByteIterator; +import org.opensearch.knn.index.query.filtered.NestedFilteredIdsKNNIterator; +import org.opensearch.knn.index.vectorvalues.KNNBinaryVectorValues; +import org.opensearch.knn.index.vectorvalues.KNNFloatVectorValues; +import org.opensearch.knn.index.vectorvalues.KNNVectorValues; +import org.opensearch.knn.index.vectorvalues.KNNVectorValuesFactory; +import org.opensearch.knn.indices.ModelDao; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +@Log4j2 +@AllArgsConstructor +public class ExactSearcher { + + private final ModelDao modelDao; + + /** + * Execute an exact search on a subset of documents of a leaf + * + * @param leafReaderContext LeafReaderContext to be searched over + * @param matchedDocs matched documents + * @param knnQuery KNN Query + * @param k number of results to return + * @param isParentHits whether the matchedDocs contains parent ids or child ids. This is relevant in the case of + * filtered nested search where the matchedDocs contain the parent ids and {@link NestedFilteredIdsKNNIterator} + * needs to be used. + * @return Map of re-scored results + */ + public Map searchLeaf( + final LeafReaderContext leafReaderContext, + final BitSet matchedDocs, + final KNNQuery knnQuery, + int k, + boolean isParentHits + ) throws IOException { + KNNIterator iterator = getMatchedKNNIterator(leafReaderContext, matchedDocs, knnQuery, isParentHits); + if (matchedDocs.cardinality() <= k) { + return scoreAllDocs(iterator); + } + return searchTopK(iterator, k); + } + + private Map scoreAllDocs(KNNIterator iterator) throws IOException { + final Map docToScore = new HashMap<>(); + int docId; + while ((docId = iterator.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) { + docToScore.put(docId, iterator.score()); + } + return docToScore; + } + + private Map searchTopK(KNNIterator iterator, int k) throws IOException { + // Creating min heap and init with MAX DocID and Score as -INF. + final HitQueue queue = new HitQueue(k, true); + ScoreDoc topDoc = queue.top(); + final Map docToScore = new HashMap<>(); + int docId; + while ((docId = iterator.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) { + if (iterator.score() > topDoc.score) { + topDoc.score = iterator.score(); + topDoc.doc = docId; + // As the HitQueue is min heap, updating top will bring the doc with -INF score or worst score we + // have seen till now on top. + topDoc = queue.updateTop(); + } + } + + // If scores are negative we will remove them. + // This is done, because there can be negative values in the Heap as we init the heap with Score as -INF. + // If filterIds < k, the some values in heap can have a negative score. + while (queue.size() > 0 && queue.top().score < 0) { + queue.pop(); + } + + while (queue.size() > 0) { + final ScoreDoc doc = queue.pop(); + docToScore.put(doc.doc, doc.score); + } + + return docToScore; + } + + private KNNIterator getMatchedKNNIterator( + final LeafReaderContext leafReaderContext, + final BitSet matchedDocs, + KNNQuery knnQuery, + boolean isParentHits + ) throws IOException { + final SegmentReader reader = Lucene.segmentReader(leafReaderContext.reader()); + final FieldInfo fieldInfo = reader.getFieldInfos().fieldInfo(knnQuery.getField()); + final SpaceType spaceType = FieldInfoExtractor.getSpaceType(modelDao, fieldInfo); + + boolean isNestedRequired = isParentHits && knnQuery.getParentsFilter() != null; + + if (VectorDataType.BINARY == knnQuery.getVectorDataType() && isNestedRequired) { + final KNNVectorValues vectorValues = KNNVectorValuesFactory.getVectorValues(fieldInfo, reader); + return new NestedFilteredIdsKNNByteIterator( + matchedDocs, + knnQuery.getByteQueryVector(), + (KNNBinaryVectorValues) vectorValues, + spaceType, + knnQuery.getParentsFilter().getBitSet(leafReaderContext) + ); + } + + if (VectorDataType.BINARY == knnQuery.getVectorDataType()) { + final KNNVectorValues vectorValues = KNNVectorValuesFactory.getVectorValues(fieldInfo, reader); + return new FilteredIdsKNNByteIterator( + matchedDocs, + knnQuery.getByteQueryVector(), + (KNNBinaryVectorValues) vectorValues, + spaceType + ); + } + + final KNNVectorValues vectorValues = KNNVectorValuesFactory.getVectorValues(fieldInfo, reader); + if (isNestedRequired) { + return new NestedFilteredIdsKNNIterator( + matchedDocs, + knnQuery.getQueryVector(), + (KNNFloatVectorValues) vectorValues, + spaceType, + knnQuery.getParentsFilter().getBitSet(leafReaderContext) + ); + } + + return new FilteredIdsKNNIterator(matchedDocs, knnQuery.getQueryVector(), (KNNFloatVectorValues) vectorValues, spaceType); + } +} diff --git a/src/main/java/org/opensearch/knn/index/query/KNNQuery.java b/src/main/java/org/opensearch/knn/index/query/KNNQuery.java index 1c4ef25e5..04a10143c 100644 --- a/src/main/java/org/opensearch/knn/index/query/KNNQuery.java +++ b/src/main/java/org/opensearch/knn/index/query/KNNQuery.java @@ -21,6 +21,7 @@ import org.apache.lucene.search.join.BitSetProducer; import org.opensearch.knn.index.KNNSettings; import org.opensearch.knn.index.VectorDataType; +import org.opensearch.knn.index.query.rescore.RescoreContext; import java.io.IOException; import java.util.Arrays; @@ -38,13 +39,12 @@ public class KNNQuery extends Query { private final String field; private final float[] queryVector; - @Getter private final byte[] byteQueryVector; private int k; private Map methodParameters; private final String indexName; - @Getter private final VectorDataType vectorDataType; + private final RescoreContext rescoreContext; @Setter private Query filterQuery; @@ -59,7 +59,7 @@ public KNNQuery( final String indexName, final BitSetProducer parentsFilter ) { - this(field, queryVector, null, k, indexName, null, parentsFilter, VectorDataType.FLOAT); + this(field, queryVector, null, k, indexName, null, parentsFilter, VectorDataType.FLOAT, null); } public KNNQuery( @@ -68,9 +68,10 @@ public KNNQuery( final int k, final String indexName, final Query filterQuery, - final BitSetProducer parentsFilter + final BitSetProducer parentsFilter, + final RescoreContext rescoreContext ) { - this(field, queryVector, null, k, indexName, filterQuery, parentsFilter, VectorDataType.FLOAT); + this(field, queryVector, null, k, indexName, filterQuery, parentsFilter, VectorDataType.FLOAT, rescoreContext); } public KNNQuery( @@ -80,9 +81,10 @@ public KNNQuery( final String indexName, final Query filterQuery, final BitSetProducer parentsFilter, - final VectorDataType vectorDataType + final VectorDataType vectorDataType, + final RescoreContext rescoreContext ) { - this(field, null, byteQueryVector, k, indexName, filterQuery, parentsFilter, vectorDataType); + this(field, null, byteQueryVector, k, indexName, filterQuery, parentsFilter, vectorDataType, rescoreContext); } private KNNQuery( @@ -93,7 +95,8 @@ private KNNQuery( final String indexName, final Query filterQuery, final BitSetProducer parentsFilter, - final VectorDataType vectorDataType + final VectorDataType vectorDataType, + final RescoreContext rescoreContext ) { this.field = field; this.queryVector = queryVector; @@ -103,6 +106,7 @@ private KNNQuery( this.filterQuery = filterQuery; this.parentsFilter = parentsFilter; this.vectorDataType = vectorDataType; + this.rescoreContext = rescoreContext; } /** @@ -114,7 +118,7 @@ private KNNQuery( * @param parentsFilter parent filter */ public KNNQuery(String field, float[] queryVector, String indexName, BitSetProducer parentsFilter) { - this(field, queryVector, null, 0, indexName, null, parentsFilter, VectorDataType.FLOAT); + this(field, queryVector, null, 0, indexName, null, parentsFilter, VectorDataType.FLOAT, null); } /** diff --git a/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java b/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java index 61dba45c8..cba5692c9 100644 --- a/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java +++ b/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java @@ -519,6 +519,7 @@ protected Query doToQuery(QueryShardContext context) { .methodParameters(this.methodParameters) .filter(this.filter) .context(context) + .rescoreContext(rescoreContext) .build(); return KNNQueryFactory.create(createQueryRequest); } diff --git a/src/main/java/org/opensearch/knn/index/query/KNNQueryFactory.java b/src/main/java/org/opensearch/knn/index/query/KNNQueryFactory.java index f3161b2db..2af3c35ca 100644 --- a/src/main/java/org/opensearch/knn/index/query/KNNQueryFactory.java +++ b/src/main/java/org/opensearch/knn/index/query/KNNQueryFactory.java @@ -5,7 +5,6 @@ package org.opensearch.knn.index.query; -import com.google.common.annotations.VisibleForTesting; import lombok.extern.log4j.Log4j2; import org.apache.lucene.search.KnnByteVectorQuery; import org.apache.lucene.search.KnnFloatVectorQuery; @@ -17,6 +16,7 @@ import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.engine.KNNEngine; import org.opensearch.knn.index.query.nativelib.NativeEngineKnnVectorQuery; +import org.opensearch.knn.index.query.rescore.RescoreContext; import java.util.Locale; import java.util.Map; @@ -32,39 +32,6 @@ @Log4j2 public class KNNQueryFactory extends BaseQueryFactory { - /** - * Note. This method should be used only for test. - * Should use {@link #create(CreateQueryRequest)} instead. - * - * Creates a Lucene query for a particular engine. - * - * @param knnEngine Engine to create the query for - * @param indexName Name of the OpenSearch index that is being queried - * @param fieldName Name of the field in the OpenSearch index that will be queried - * @param vector The query vector to get the nearest neighbors for - * @param k the number of nearest neighbors to return - * @return Lucene Query - */ - @VisibleForTesting - public static Query create( - KNNEngine knnEngine, - String indexName, - String fieldName, - float[] vector, - int k, - VectorDataType vectorDataType - ) { - final CreateQueryRequest createQueryRequest = CreateQueryRequest.builder() - .knnEngine(knnEngine) - .indexName(indexName) - .fieldName(fieldName) - .vector(vector) - .vectorDataType(vectorDataType) - .k(k) - .build(); - return create(createQueryRequest); - } - /** * Creates a Lucene query for a particular engine. * @param createQueryRequest request object that has all required fields to construct the query @@ -81,6 +48,7 @@ public static Query create(CreateQueryRequest createQueryRequest) { final VectorDataType vectorDataType = createQueryRequest.getVectorDataType(); final Query filterQuery = getFilterQuery(createQueryRequest); final Map methodParameters = createQueryRequest.getMethodParameters(); + final RescoreContext rescoreContext = createQueryRequest.getRescoreContext().orElse(null); BitSetProducer parentFilter = null; if (createQueryRequest.getContext().isPresent()) { @@ -112,6 +80,7 @@ public static Query create(CreateQueryRequest createQueryRequest) { .methodParameters(methodParameters) .filterQuery(validatedFilterQuery) .vectorDataType(vectorDataType) + .rescoreContext(rescoreContext) .build(); break; default: @@ -124,6 +93,7 @@ public static Query create(CreateQueryRequest createQueryRequest) { .methodParameters(methodParameters) .filterQuery(validatedFilterQuery) .vectorDataType(vectorDataType) + .rescoreContext(rescoreContext) .build(); } return isKnnQueryRewriteEnabled() ? new NativeEngineKnnVectorQuery(knnQuery) : knnQuery; diff --git a/src/main/java/org/opensearch/knn/index/query/KNNWeight.java b/src/main/java/org/opensearch/knn/index/query/KNNWeight.java index 1863e0632..6cc110839 100644 --- a/src/main/java/org/opensearch/knn/index/query/KNNWeight.java +++ b/src/main/java/org/opensearch/knn/index/query/KNNWeight.java @@ -7,15 +7,12 @@ import com.google.common.annotations.VisibleForTesting; import lombok.extern.log4j.Log4j2; -import org.apache.commons.lang.StringUtils; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.SegmentReader; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.Explanation; import org.apache.lucene.search.FilteredDocIdSetIterator; -import org.apache.lucene.search.HitQueue; -import org.apache.lucene.search.ScoreDoc; import org.apache.lucene.search.Scorer; import org.apache.lucene.search.Weight; import org.apache.lucene.store.FSDirectory; @@ -23,7 +20,6 @@ import org.apache.lucene.util.BitSet; import org.apache.lucene.util.BitSetIterator; import org.apache.lucene.util.Bits; -import org.apache.lucene.util.DocIdSetBuilder; import org.apache.lucene.util.FixedBitSet; import org.opensearch.common.io.PathUtils; import org.opensearch.common.lucene.Lucene; @@ -37,16 +33,7 @@ import org.opensearch.knn.index.memory.NativeMemoryCacheManager; import org.opensearch.knn.index.memory.NativeMemoryEntryContext; import org.opensearch.knn.index.memory.NativeMemoryLoadStrategy; -import org.opensearch.knn.index.query.filtered.FilteredIdsKNNByteIterator; -import org.opensearch.knn.index.query.filtered.FilteredIdsKNNIterator; -import org.opensearch.knn.index.query.filtered.KNNIterator; -import org.opensearch.knn.index.query.filtered.NestedFilteredIdsKNNByteIterator; -import org.opensearch.knn.index.query.filtered.NestedFilteredIdsKNNIterator; import org.opensearch.knn.index.engine.KNNEngine; -import org.opensearch.knn.index.vectorvalues.KNNBinaryVectorValues; -import org.opensearch.knn.index.vectorvalues.KNNFloatVectorValues; -import org.opensearch.knn.index.vectorvalues.KNNVectorValues; -import org.opensearch.knn.index.vectorvalues.KNNVectorValuesFactory; import org.opensearch.knn.indices.ModelDao; import org.opensearch.knn.indices.ModelMetadata; import org.opensearch.knn.indices.ModelUtil; @@ -58,9 +45,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.Comparator; -import java.util.HashMap; import java.util.List; -import java.util.Locale; import java.util.Map; import java.util.concurrent.ExecutionException; import java.util.stream.Collectors; @@ -84,6 +69,9 @@ public class KNNWeight extends Weight { private final NativeMemoryCacheManager nativeMemoryCacheManager; private final Weight filterWeight; + private final ExactSearcher exactSearcher; + + private static ExactSearcher DEFAULT_EXACT_SEARCHER; public KNNWeight(KNNQuery query, float boost) { super(query); @@ -91,6 +79,7 @@ public KNNWeight(KNNQuery query, float boost) { this.boost = boost; this.nativeMemoryCacheManager = NativeMemoryCacheManager.getInstance(); this.filterWeight = null; + this.exactSearcher = DEFAULT_EXACT_SEARCHER; } public KNNWeight(KNNQuery query, float boost, Weight filterWeight) { @@ -99,10 +88,12 @@ public KNNWeight(KNNQuery query, float boost, Weight filterWeight) { this.boost = boost; this.nativeMemoryCacheManager = NativeMemoryCacheManager.getInstance(); this.filterWeight = filterWeight; + this.exactSearcher = DEFAULT_EXACT_SEARCHER; } public static void initialize(ModelDao modelDao) { KNNWeight.modelDao = modelDao; + KNNWeight.DEFAULT_EXACT_SEARCHER = new ExactSearcher(modelDao); } @Override @@ -112,12 +103,12 @@ public Explanation explain(LeafReaderContext context, int doc) { @Override public Scorer scorer(LeafReaderContext context) throws IOException { - final Map docIdToScoreMap = searchLeaf(context); + final Map docIdToScoreMap = searchLeaf(context, knnQuery.getK()); if (docIdToScoreMap.isEmpty()) { return KNNScorer.emptyScorer(this); } - - return convertSearchResponseToScorer(docIdToScoreMap); + final int maxDoc = Collections.max(docIdToScoreMap.keySet()) + 1; + return new KNNScorer(this, ResultUtil.resultMapToDocIds(docIdToScoreMap, maxDoc), docIdToScoreMap, boost); } /** @@ -125,10 +116,10 @@ public Scorer scorer(LeafReaderContext context) throws IOException { * This is made public purely to be able to be reused in {@link org.opensearch.knn.index.query.nativelib.NativeEngineKnnVectorQuery} * * @param context LeafReaderContext + * @param k Number of results to return * @return A Map of docId to scores for top k results */ - public Map searchLeaf(LeafReaderContext context) throws IOException { - + public Map searchLeaf(LeafReaderContext context, int k) throws IOException { final BitSet filterBitSet = getFilteredDocsBitSet(context); int cardinality = filterBitSet.cardinality(); // We don't need to go to JNI layer if no documents are found which satisfy the filters @@ -137,31 +128,30 @@ public Map searchLeaf(LeafReaderContext context) throws IOExcept if (filterWeight != null && cardinality == 0) { return Collections.emptyMap(); } - final Map docIdsToScoreMap = new HashMap<>(); /* * The idea for this optimization is to get K results, we need to atleast look at K vectors in the HNSW graph * . Hence, if filtered results are less than K and filter query is present we should shift to exact search. * This improves the recall. */ + Map docIdsToScoreMap; if (filterWeight != null && canDoExactSearch(cardinality)) { - docIdsToScoreMap.putAll(doExactSearch(context, filterBitSet, cardinality)); + docIdsToScoreMap = exactSearch(context, filterBitSet, true, k); } else { - Map annResults = doANNSearch(context, filterBitSet, cardinality); - if (annResults == null) { + docIdsToScoreMap = doANNSearch(context, filterBitSet, cardinality, k); + if (docIdsToScoreMap == null) { return Collections.emptyMap(); } - if (canDoExactSearchAfterANNSearch(cardinality, annResults.size())) { + if (canDoExactSearchAfterANNSearch(cardinality, docIdsToScoreMap.size())) { log.debug( "Doing ExactSearch after doing ANNSearch as the number of documents returned are less than " + "K, even when we have more than K filtered Ids. K: {}, ANNResults: {}, filteredIdCount: {}", - knnQuery.getK(), - annResults.size(), + k, + docIdsToScoreMap.size(), cardinality ); - annResults = doExactSearch(context, filterBitSet, cardinality); + docIdsToScoreMap = exactSearch(context, filterBitSet, true, k); } - docIdsToScoreMap.putAll(annResults); } if (docIdsToScoreMap.isEmpty()) { return Collections.emptyMap(); @@ -221,8 +211,12 @@ private int[] bitSetToIntArray(final BitSet bitSet) { return intArray; } - private Map doANNSearch(final LeafReaderContext context, final BitSet filterIdsBitSet, final int cardinality) - throws IOException { + private Map doANNSearch( + final LeafReaderContext context, + final BitSet filterIdsBitSet, + final int cardinality, + final int k + ) throws IOException { final SegmentReader reader = Lucene.segmentReader(context.reader()); String directory = ((FSDirectory) FilterDirectory.unwrap(reader.directory())).getDirectory().toString(); @@ -301,12 +295,12 @@ private Map doANNSearch(final LeafReaderContext context, final B throw new RuntimeException("Index has already been closed"); } int[] parentIds = getParentIdsArray(context); - if (knnQuery.getK() > 0) { + if (k > 0) { if (knnQuery.getVectorDataType() == VectorDataType.BINARY) { results = JNIService.queryBinaryIndex( indexAllocation.getMemoryAddress(), knnQuery.getByteQueryVector(), - knnQuery.getK(), + k, knnQuery.getMethodParameters(), knnEngine, filterIds, @@ -317,7 +311,7 @@ private Map doANNSearch(final LeafReaderContext context, final B results = JNIService.queryIndex( indexAllocation.getMemoryAddress(), knnQuery.getQueryVector(), - knnQuery.getK(), + k, knnQuery.getMethodParameters(), knnEngine, filterIds, @@ -379,86 +373,19 @@ List getEngineFiles(SegmentReader reader, String extension) throws IOExc return engineFiles; } - private Map doExactSearch(final LeafReaderContext leafReaderContext, final BitSet filterIdsBitSet, int cardinality) { - try { - // Creating min heap and init with MAX DocID and Score as -INF. - final HitQueue queue = new HitQueue(Math.min(this.knnQuery.getK(), cardinality), true); - ScoreDoc topDoc = queue.top(); - final Map docToScore = new HashMap<>(); - KNNIterator iterator = getFilteredKNNIterator(leafReaderContext, filterIdsBitSet); - int docId; - while ((docId = iterator.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) { - if (iterator.score() > topDoc.score) { - topDoc.score = iterator.score(); - topDoc.doc = docId; - // As the HitQueue is min heap, updating top will bring the doc with -INF score or worst score we - // have seen till now on top. - topDoc = queue.updateTop(); - } - } - - // If scores are negative we will remove them. - // This is done, because there can be negative values in the Heap as we init the heap with Score as -INF. - // If filterIds < k, the some values in heap can have a negative score. - while (queue.size() > 0 && queue.top().score < 0) { - queue.pop(); - } - - while (queue.size() > 0) { - final ScoreDoc doc = queue.pop(); - docToScore.put(doc.doc, doc.score); - } - - return docToScore; - } catch (Exception e) { - log.error("Error while getting the doc values to do the k-NN Search for query : {}", this.knnQuery, e); - } - return Collections.emptyMap(); - } - - private KNNIterator getFilteredKNNIterator(final LeafReaderContext leafReaderContext, final BitSet filterIdsBitSet) throws IOException { - final SegmentReader reader = Lucene.segmentReader(leafReaderContext.reader()); - final FieldInfo fieldInfo = reader.getFieldInfos().fieldInfo(knnQuery.getField()); - final SpaceType spaceType = getSpaceType(fieldInfo); - if (VectorDataType.BINARY == knnQuery.getVectorDataType()) { - final KNNVectorValues vectorValues = KNNVectorValuesFactory.getVectorValues(fieldInfo, leafReaderContext.reader()); - return knnQuery.getParentsFilter() == null - ? new FilteredIdsKNNByteIterator( - filterIdsBitSet, - knnQuery.getByteQueryVector(), - (KNNBinaryVectorValues) vectorValues, - spaceType - ) - : new NestedFilteredIdsKNNByteIterator( - filterIdsBitSet, - knnQuery.getByteQueryVector(), - (KNNBinaryVectorValues) vectorValues, - spaceType, - knnQuery.getParentsFilter().getBitSet(leafReaderContext) - ); - } else { - final KNNVectorValues vectorValues = KNNVectorValuesFactory.getVectorValues(fieldInfo, leafReaderContext.reader()); - return knnQuery.getParentsFilter() == null - ? new FilteredIdsKNNIterator(filterIdsBitSet, knnQuery.getQueryVector(), (KNNFloatVectorValues) vectorValues, spaceType) - : new NestedFilteredIdsKNNIterator( - filterIdsBitSet, - knnQuery.getQueryVector(), - (KNNFloatVectorValues) vectorValues, - spaceType, - knnQuery.getParentsFilter().getBitSet(leafReaderContext) - ); - } - } - - private Scorer convertSearchResponseToScorer(final Map docsToScore) throws IOException { - final int maxDoc = Collections.max(docsToScore.keySet()) + 1; - final DocIdSetBuilder docIdSetBuilder = new DocIdSetBuilder(maxDoc); - // The docIdSetIterator will contain the docids of the returned results. So, before adding results to - // the builder, we can grow to docsToScore.size() - final DocIdSetBuilder.BulkAdder setAdder = docIdSetBuilder.grow(docsToScore.size()); - docsToScore.keySet().forEach(setAdder::add); - final DocIdSetIterator docIdSetIter = docIdSetBuilder.build().iterator(); - return new KNNScorer(this, docIdSetIter, docsToScore, boost); + /** + * Execute exact search for the given matched doc ids and return the results as a map of docId to score. + * + * @param leafReaderContext The leaf reader context for the current segment. + * @param matchSet The filterIds to search for. + * @param isParentHits Whether the matchedDocs contains parent ids or child ids. + * @param k The number of results to return. + * @return Map of docId to score for the exact search results. + * @throws IOException If an error occurs during the search. + */ + public Map exactSearch(final LeafReaderContext leafReaderContext, final BitSet matchSet, boolean isParentHits, int k) + throws IOException { + return exactSearcher.searchLeaf(leafReaderContext, matchSet, knnQuery, k, isParentHits); } @Override @@ -471,22 +398,6 @@ public static float normalizeScore(float score) { return -score + 1; } - private SpaceType getSpaceType(final FieldInfo fieldInfo) { - final String spaceTypeString = fieldInfo.getAttribute(SPACE_TYPE); - if (StringUtils.isNotEmpty(spaceTypeString)) { - return SpaceType.getSpace(spaceTypeString); - } - - final String modelId = fieldInfo.getAttribute(MODEL_ID); - if (StringUtils.isNotEmpty(modelId)) { - ModelMetadata modelMetadata = modelDao.getMetadata(modelId); - return modelMetadata.getSpaceType(); - } - throw new IllegalArgumentException( - String.format(Locale.ROOT, "Unable to find the Space Type from Field Info attribute for field %s", fieldInfo.getName()) - ); - } - private boolean canDoExactSearch(final int filterIdsCount) { log.debug( "Info for doing exact search filterIdsLength : {}, Threshold value: {}", diff --git a/src/main/java/org/opensearch/knn/index/query/ResultUtil.java b/src/main/java/org/opensearch/knn/index/query/ResultUtil.java new file mode 100644 index 000000000..2ed70b8b3 --- /dev/null +++ b/src/main/java/org/opensearch/knn/index/query/ResultUtil.java @@ -0,0 +1,119 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.query; + +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.search.ScoreDoc; +import org.apache.lucene.search.TopDocs; +import org.apache.lucene.search.TotalHits; +import org.apache.lucene.util.BitSet; +import org.apache.lucene.util.DocIdSetBuilder; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.PriorityQueue; + +/** + * Utility class used for processing results + */ +public final class ResultUtil { + + /** + * Reduce the results to only include the top k results across all leaf results + * + * @param perLeafResults Results from the list + * @param k the number of results across all leaf results to return + */ + public static void reduceToTopK(List> perLeafResults, int k) { + // Iterate over all scores to get min competitive score + PriorityQueue topKMinQueue = new PriorityQueue<>(k); + for (int i = 0; i < k; i++) { + topKMinQueue.add(-Float.MAX_VALUE); + } + + int count = 0; + for (Map perLeafResult : perLeafResults) { + count += perLeafResult.size(); + for (Float score : perLeafResult.values()) { + if (topKMinQueue.peek() != null && score > topKMinQueue.peek()) { + topKMinQueue.poll(); + topKMinQueue.add(score); + } + } + } + + // If there are at most k results across everything, then no need to filter anything out + if (count <= k) { + return; + } + + // Reduce the results based on min competitive score + float minScore = topKMinQueue.peek() == null ? -Float.MAX_VALUE : topKMinQueue.peek(); + perLeafResults.forEach(results -> results.entrySet().removeIf(entry -> entry.getValue() < minScore)); + } + + /** + * Convert map to bit set + * + * @param resultMap Map of results + * @return BitSet of results + * @throws IOException If an error occurs during the search. + */ + public static BitSet resultMapToMatchBitSet(Map resultMap) throws IOException { + if (resultMap.isEmpty()) { + return BitSet.of(DocIdSetIterator.empty(), 0); + } + + final int maxDoc = Collections.max(resultMap.keySet()) + 1; + return BitSet.of(resultMapToDocIds(resultMap, maxDoc), maxDoc); + } + + /** + * Convert map of docs to doc id set iterator + * + * @param resultMap Map of results + * @param maxDoc Max doc id + * @return Doc id set iterator + * @throws IOException If an error occurs during the search. + */ + public static DocIdSetIterator resultMapToDocIds(Map resultMap, final int maxDoc) throws IOException { + if (resultMap.isEmpty()) { + return DocIdSetIterator.empty(); + } + final DocIdSetBuilder docIdSetBuilder = new DocIdSetBuilder(maxDoc); + final DocIdSetBuilder.BulkAdder setAdder = docIdSetBuilder.grow(resultMap.size()); + resultMap.keySet().forEach(setAdder::add); + return docIdSetBuilder.build().iterator(); + } + + /** + * COnvert map of results to top docs. Doc ids have proper offset + * + * @param resultMap map of scores for the leafs + * @param segmentOffset Offset to apply to ids to make them shard ids + * @return Top docs + */ + public static TopDocs resultMapToTopDocs(Map resultMap, int segmentOffset) { + if (resultMap.isEmpty()) { + return new TopDocs(new TotalHits(0, TotalHits.Relation.EQUAL_TO), new ScoreDoc[0]); + } + + int totalHits = 0; + final List scoreDocs = new ArrayList<>(); + final List> topScores = new ArrayList<>(resultMap.entrySet()); + topScores.sort(Map.Entry.comparingByValue().reversed()); + for (Map.Entry entry : topScores) { + ScoreDoc scoreDoc = new ScoreDoc(entry.getKey() + segmentOffset, entry.getValue()); + scoreDocs.add(scoreDoc); + totalHits++; + } + + return new TopDocs(new TotalHits(totalHits, TotalHits.Relation.EQUAL_TO), scoreDocs.toArray(ScoreDoc[]::new)); + } +} diff --git a/src/main/java/org/opensearch/knn/index/query/nativelib/NativeEngineKnnVectorQuery.java b/src/main/java/org/opensearch/knn/index/query/nativelib/NativeEngineKnnVectorQuery.java index 6b9a40a9c..06e5fc577 100644 --- a/src/main/java/org/opensearch/knn/index/query/nativelib/NativeEngineKnnVectorQuery.java +++ b/src/main/java/org/opensearch/knn/index/query/nativelib/NativeEngineKnnVectorQuery.java @@ -13,13 +13,14 @@ import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.QueryVisitor; -import org.apache.lucene.search.ScoreDoc; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.TopDocs; -import org.apache.lucene.search.TotalHits; +import org.apache.lucene.util.BitSet; import org.apache.lucene.util.Bits; import org.opensearch.knn.index.query.KNNQuery; import org.opensearch.knn.index.query.KNNWeight; +import org.opensearch.knn.index.query.ResultUtil; +import org.opensearch.knn.index.query.rescore.RescoreContext; import java.io.IOException; import java.util.ArrayList; @@ -49,19 +50,62 @@ public Query rewrite(final IndexSearcher indexSearcher) throws IOException { final KNNWeight knnWeight = (KNNWeight) knnQuery.createWeight(indexSearcher, ScoreMode.COMPLETE, 1); List leafReaderContexts = reader.leaves(); - List> tasks = new ArrayList<>(leafReaderContexts.size()); - for (LeafReaderContext leafReaderContext : leafReaderContexts) { - tasks.add(() -> searchLeaf(leafReaderContext, knnWeight)); + List> perLeafResults; + RescoreContext rescoreContext = knnQuery.getRescoreContext(); + int finalK = knnQuery.getK(); + if (rescoreContext == null) { + perLeafResults = doSearch(indexSearcher, leafReaderContexts, knnWeight, finalK); + } else { + int firstPassK = rescoreContext.getFirstPassK(finalK); + perLeafResults = doSearch(indexSearcher, leafReaderContexts, knnWeight, firstPassK); + ResultUtil.reduceToTopK(perLeafResults, firstPassK); + perLeafResults = doRescore(indexSearcher, leafReaderContexts, knnWeight, perLeafResults, finalK); + } + ResultUtil.reduceToTopK(perLeafResults, finalK); + TopDocs[] topDocs = new TopDocs[perLeafResults.size()]; + for (int i = 0; i < perLeafResults.size(); i++) { + topDocs[i] = ResultUtil.resultMapToTopDocs(perLeafResults.get(i), leafReaderContexts.get(i).docBase); } - TopDocs[] perLeafResults = indexSearcher.getTaskExecutor().invokeAll(tasks).toArray(TopDocs[]::new); - // TopDocs.merge requires perLeafResults to be sorted in descending order. - TopDocs topK = TopDocs.merge(knnQuery.getK(), perLeafResults); + + TopDocs topK = TopDocs.merge(knnQuery.getK(), topDocs); if (topK.scoreDocs.length == 0) { return new MatchNoDocsQuery(); } return createRewrittenQuery(reader, topK); } + private List> doSearch( + final IndexSearcher indexSearcher, + List leafReaderContexts, + KNNWeight knnWeight, + int k + ) throws IOException { + List>> tasks = new ArrayList<>(leafReaderContexts.size()); + for (LeafReaderContext leafReaderContext : leafReaderContexts) { + tasks.add(() -> searchLeaf(leafReaderContext, knnWeight, k)); + } + return indexSearcher.getTaskExecutor().invokeAll(tasks); + } + + private List> doRescore( + final IndexSearcher indexSearcher, + List leafReaderContexts, + KNNWeight knnWeight, + List> perLeafResults, + int k + ) throws IOException { + List>> rescoreTasks = new ArrayList<>(leafReaderContexts.size()); + for (int i = 0; i < perLeafResults.size(); i++) { + LeafReaderContext leafReaderContext = leafReaderContexts.get(i); + int finalI = i; + rescoreTasks.add(() -> { + BitSet convertedBitSet = ResultUtil.resultMapToMatchBitSet(perLeafResults.get(finalI)); + return knnWeight.exactSearch(leafReaderContext, convertedBitSet, false, k); + }); + } + return indexSearcher.getTaskExecutor().invokeAll(rescoreTasks); + } + private Query createRewrittenQuery(IndexReader reader, TopDocs topK) { int len = topK.scoreDocs.length; Arrays.sort(topK.scoreDocs, Comparator.comparingInt(a -> a.doc)); @@ -75,7 +119,7 @@ private Query createRewrittenQuery(IndexReader reader, TopDocs topK) { return new DocAndScoreQuery(knnQuery.getK(), docs, scores, segmentStarts, reader.getContext().id()); } - private static int[] findSegmentStarts(IndexReader reader, int[] docs) { + static int[] findSegmentStarts(IndexReader reader, int[] docs) { int[] starts = new int[reader.leaves().size() + 1]; starts[starts.length - 1] = docs.length; if (starts.length == 2) { @@ -93,26 +137,13 @@ private static int[] findSegmentStarts(IndexReader reader, int[] docs) { return starts; } - private TopDocs searchLeaf(LeafReaderContext ctx, KNNWeight queryWeight) throws IOException { - int totalHits = 0; - final Map leafDocScores = queryWeight.searchLeaf(ctx); - final List scoreDocs = new ArrayList<>(); + private Map searchLeaf(LeafReaderContext ctx, KNNWeight queryWeight, int k) throws IOException { + final Map leafDocScores = queryWeight.searchLeaf(ctx, k); final Bits liveDocs = ctx.reader().getLiveDocs(); - - if (!leafDocScores.isEmpty()) { - final List> topScores = new ArrayList<>(leafDocScores.entrySet()); - topScores.sort(Map.Entry.comparingByValue().reversed()); - - for (Map.Entry entry : topScores) { - if (liveDocs == null || liveDocs.get(entry.getKey())) { - ScoreDoc scoreDoc = new ScoreDoc(entry.getKey() + ctx.docBase, entry.getValue()); - scoreDocs.add(scoreDoc); - totalHits++; - } - } + if (liveDocs != null) { + leafDocScores.entrySet().removeIf(entry -> liveDocs.get(entry.getKey()) == false); } - - return new TopDocs(new TotalHits(totalHits, TotalHits.Relation.EQUAL_TO), scoreDocs.toArray(ScoreDoc[]::new)); + return leafDocScores; } @Override diff --git a/src/main/java/org/opensearch/knn/index/query/rescore/RescoreContext.java b/src/main/java/org/opensearch/knn/index/query/rescore/RescoreContext.java index 82b09807a..9fe2ddbc5 100644 --- a/src/main/java/org/opensearch/knn/index/query/rescore/RescoreContext.java +++ b/src/main/java/org/opensearch/knn/index/query/rescore/RescoreContext.java @@ -20,6 +20,8 @@ public final class RescoreContext { public static final float MAX_OVERSAMPLE_FACTOR = 100.0f; public static final float MIN_OVERSAMPLE_FACTOR = 0.0f; + public static final int MAX_FIRST_PASS_RESULTS = 10000; + @Builder.Default private float oversampleFactor = DEFAULT_OVERSAMPLE_FACTOR; @@ -30,4 +32,14 @@ public final class RescoreContext { public static RescoreContext getDefault() { return RescoreContext.builder().build(); } + + /** + * Gets the number of results to return for the first pass of rescoring. + * + * @param finalK The final number of results to return for the entire shard + * @return The number of results to return for the first pass of rescoring + */ + public int getFirstPassK(int finalK) { + return Math.min(MAX_FIRST_PASS_RESULTS, (int) Math.ceil(finalK * oversampleFactor)); + } } diff --git a/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestCase.java b/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestCase.java index 1158d3ebb..2a4e26a82 100644 --- a/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestCase.java +++ b/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestCase.java @@ -30,6 +30,7 @@ import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.VectorField; import org.opensearch.knn.index.mapper.KNNVectorFieldType; +import org.opensearch.knn.index.query.BaseQueryFactory; import org.opensearch.knn.index.query.KNNQueryFactory; import org.opensearch.knn.jni.JNIService; import org.opensearch.knn.index.query.KNNQuery; @@ -380,13 +381,16 @@ public void testKnnVectorIndex( verify(perFieldKnnVectorsFormatSpy, atLeastOnce()).getMaxDimensions(eq(FIELD_NAME_ONE)); IndexSearcher searcher = new IndexSearcher(reader); + Query query = KNNQueryFactory.create( - KNNEngine.LUCENE, - "dummy", - FIELD_NAME_ONE, - new float[] { 1.0f, 0.0f, 0.0f }, - 1, - DEFAULT_VECTOR_DATA_TYPE_FIELD + BaseQueryFactory.CreateQueryRequest.builder() + .knnEngine(KNNEngine.LUCENE) + .indexName("dummy") + .fieldName(FIELD_NAME_ONE) + .vector(new float[] { 1.0f, 0.0f, 0.0f }) + .k(1) + .vectorDataType(DEFAULT_VECTOR_DATA_TYPE_FIELD) + .build() ); assertEquals(1, searcher.count(query)); @@ -416,12 +420,14 @@ public void testKnnVectorIndex( IndexSearcher searcher1 = new IndexSearcher(reader1); Query query1 = KNNQueryFactory.create( - KNNEngine.LUCENE, - "dummy", - FIELD_NAME_TWO, - new float[] { 1.0f, 0.0f }, - 1, - DEFAULT_VECTOR_DATA_TYPE_FIELD + BaseQueryFactory.CreateQueryRequest.builder() + .knnEngine(KNNEngine.LUCENE) + .indexName("dummy") + .fieldName(FIELD_NAME_TWO) + .vector(new float[] { 1.0f, 0.0f }) + .k(1) + .vectorDataType(DEFAULT_VECTOR_DATA_TYPE_FIELD) + .build() ); assertEquals(1, searcher1.count(query1)); diff --git a/src/test/java/org/opensearch/knn/index/query/KNNQueryFactoryTests.java b/src/test/java/org/opensearch/knn/index/query/KNNQueryFactoryTests.java index 7bacc7d10..cd7fd035e 100644 --- a/src/test/java/org/opensearch/knn/index/query/KNNQueryFactoryTests.java +++ b/src/test/java/org/opensearch/knn/index/query/KNNQueryFactoryTests.java @@ -30,6 +30,7 @@ import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.engine.KNNEngine; import org.opensearch.knn.index.query.nativelib.NativeEngineKnnVectorQuery; +import org.opensearch.knn.index.query.rescore.RescoreContext; import java.util.Arrays; import java.util.List; @@ -72,12 +73,14 @@ public void testCreateCustomKNNQuery() { for (KNNEngine knnEngine : KNNEngine.getEnginesThatCreateCustomSegmentFiles()) { when(clusterSettings.get(KNN_LAUNCH_QUERY_REWRITE_ENABLED_SETTING)).thenReturn(false); Query query = KNNQueryFactory.create( - knnEngine, - testIndexName, - testFieldName, - testQueryVector, - testK, - DEFAULT_VECTOR_DATA_TYPE_FIELD + BaseQueryFactory.CreateQueryRequest.builder() + .knnEngine(knnEngine) + .indexName(testIndexName) + .fieldName(testFieldName) + .vector(testQueryVector) + .k(testK) + .vectorDataType(DEFAULT_VECTOR_DATA_TYPE_FIELD) + .build() ); assertTrue(query instanceof KNNQuery); assertEquals(testIndexName, ((KNNQuery) query).getIndexName()); @@ -86,7 +89,16 @@ public void testCreateCustomKNNQuery() { assertEquals(testK, ((KNNQuery) query).getK()); when(clusterSettings.get(KNN_LAUNCH_QUERY_REWRITE_ENABLED_SETTING)).thenReturn(true); - query = KNNQueryFactory.create(knnEngine, testIndexName, testFieldName, testQueryVector, testK, DEFAULT_VECTOR_DATA_TYPE_FIELD); + query = KNNQueryFactory.create( + BaseQueryFactory.CreateQueryRequest.builder() + .knnEngine(knnEngine) + .indexName(testIndexName) + .fieldName(testFieldName) + .vector(testQueryVector) + .k(testK) + .vectorDataType(DEFAULT_VECTOR_DATA_TYPE_FIELD) + .build() + ); assertTrue(query instanceof NativeEngineKnnVectorQuery); query = ((NativeEngineKnnVectorQuery) query).getKnnQuery(); @@ -103,12 +115,14 @@ public void testCreateLuceneDefaultQuery() { .collect(Collectors.toList()); for (KNNEngine knnEngine : luceneDefaultQueryEngineList) { Query query = KNNQueryFactory.create( - knnEngine, - testIndexName, - testFieldName, - testQueryVector, - testK, - DEFAULT_VECTOR_DATA_TYPE_FIELD + BaseQueryFactory.CreateQueryRequest.builder() + .knnEngine(knnEngine) + .indexName(testIndexName) + .fieldName(testFieldName) + .vector(testQueryVector) + .k(testK) + .vectorDataType(DEFAULT_VECTOR_DATA_TYPE_FIELD) + .build() ); assertEquals(KnnFloatVectorQuery.class, query.getClass()); } @@ -442,4 +456,28 @@ public void testCreate_whenBinary_thenSuccess() { assertTrue(query instanceof NativeEngineKnnVectorQuery); } + public void testCreate_whenRescoreContextPassed_thenSuccess() { + QueryShardContext mockQueryShardContext = mock(QueryShardContext.class); + MappedFieldType testMapper = mock(MappedFieldType.class); + when(mockQueryShardContext.fieldMapper(any())).thenReturn(testMapper); + BitSetProducer parentFilter = mock(BitSetProducer.class); + when(mockQueryShardContext.getParentFilter()).thenReturn(parentFilter); + + final KNNQueryFactory.CreateQueryRequest createQueryRequest = KNNQueryFactory.CreateQueryRequest.builder() + .knnEngine(KNNEngine.FAISS) + .indexName(testIndexName) + .fieldName(testFieldName) + .vector(testQueryVector) + .byteVector(testByteQueryVector) + .vectorDataType(VectorDataType.BINARY) + .k(testK) + .context(mockQueryShardContext) + .filter(FILTER_QUERY_BUILDER) + .rescoreContext(RescoreContext.getDefault()) + .build(); + Query query = KNNQueryFactory.create(createQueryRequest); + assertTrue(query instanceof KNNQuery); + assertEquals(RescoreContext.getDefault(), ((KNNQuery) query).getRescoreContext()); + } + } diff --git a/src/test/java/org/opensearch/knn/index/query/KNNWeightTests.java b/src/test/java/org/opensearch/knn/index/query/KNNWeightTests.java index fa4ec8001..249ae04ce 100644 --- a/src/test/java/org/opensearch/knn/index/query/KNNWeightTests.java +++ b/src/test/java/org/opensearch/knn/index/query/KNNWeightTests.java @@ -677,6 +677,7 @@ public void testANNWithFilterQuery_whenExactSearchBinary_thenSuccess() { public void validateANNWithFilterQuery_whenExactSearch_thenSuccess(final boolean isBinary) throws IOException { try (MockedStatic valuesFactoryMockedStatic = Mockito.mockStatic(KNNVectorValuesFactory.class)) { + KNNWeight.initialize(null); float[] vector = new float[] { 0.1f, 0.3f }; byte[] byteVector = new byte[] { 1, 3 }; int filterDocId = 0; @@ -685,8 +686,8 @@ public void validateANNWithFilterQuery_whenExactSearch_thenSuccess(final boolean when(leafReaderContext.reader()).thenReturn(reader); final KNNQuery query = isBinary - ? new KNNQuery(FIELD_NAME, BYTE_QUERY_VECTOR, K, INDEX_NAME, FILTER_QUERY, null, VectorDataType.BINARY) - : new KNNQuery(FIELD_NAME, QUERY_VECTOR, K, INDEX_NAME, FILTER_QUERY, null); + ? new KNNQuery(FIELD_NAME, BYTE_QUERY_VECTOR, K, INDEX_NAME, FILTER_QUERY, null, VectorDataType.BINARY, null) + : new KNNQuery(FIELD_NAME, QUERY_VECTOR, K, INDEX_NAME, FILTER_QUERY, null, null); final Weight filterQueryWeight = mock(Weight.class); final Scorer filterScorer = mock(Scorer.class); when(filterQueryWeight.scorer(leafReaderContext)).thenReturn(filterScorer); @@ -753,6 +754,8 @@ public void validateANNWithFilterQuery_whenExactSearch_thenSuccess(final boolean @SneakyThrows public void testANNWithFilterQuery_whenExactSearchAndThresholdComputations_thenSuccess() { + ModelDao modelDao = mock(ModelDao.class); + KNNWeight.initialize(modelDao); knnSettingsMockedStatic.when(() -> KNNSettings.getFilteredExactSearchThreshold(INDEX_NAME)).thenReturn(-1); float[] vector = new float[] { 0.1f, 0.3f }; int filterDocId = 0; @@ -760,7 +763,7 @@ public void testANNWithFilterQuery_whenExactSearchAndThresholdComputations_thenS final SegmentReader reader = mock(SegmentReader.class); when(leafReaderContext.reader()).thenReturn(reader); - final KNNQuery query = new KNNQuery(FIELD_NAME, QUERY_VECTOR, K, INDEX_NAME, FILTER_QUERY, null); + final KNNQuery query = new KNNQuery(FIELD_NAME, QUERY_VECTOR, K, INDEX_NAME, FILTER_QUERY, null, null); final Weight filterQueryWeight = mock(Weight.class); final Scorer filterScorer = mock(Scorer.class); when(filterQueryWeight.scorer(leafReaderContext)).thenReturn(filterScorer); @@ -821,6 +824,8 @@ public void testANNWithFilterQuery_whenExactSearchAndThresholdComputations_thenS */ @SneakyThrows public void testANNWithFilterQuery_whenExactSearchViaThresholdSetting_thenSuccess() { + ModelDao modelDao = mock(ModelDao.class); + KNNWeight.initialize(modelDao); knnSettingsMockedStatic.when(() -> KNNSettings.getFilteredExactSearchThreshold(INDEX_NAME)).thenReturn(10); float[] vector = new float[] { 0.1f, 0.3f }; int k = 1; @@ -837,7 +842,7 @@ public void testANNWithFilterQuery_whenExactSearchViaThresholdSetting_thenSucces when(filterScorer.iterator()).thenReturn(DocIdSetIterator.all(filterDocIds.length)); - final KNNQuery query = new KNNQuery(FIELD_NAME, QUERY_VECTOR, k, INDEX_NAME, FILTER_QUERY, null); + final KNNQuery query = new KNNQuery(FIELD_NAME, QUERY_VECTOR, k, INDEX_NAME, FILTER_QUERY, null, null); final float boost = (float) randomDoubleBetween(0, 10, true); final KNNWeight knnWeight = new KNNWeight(query, boost, filterQueryWeight); @@ -890,6 +895,7 @@ public void testANNWithFilterQuery_whenExactSearchViaThresholdSetting_thenSucces @SneakyThrows public void testANNWithFilterQuery_whenExactSearchViaThresholdSettingOnBinaryIndex_thenSuccess() { try (MockedStatic vectorValuesFactoryMockedStatic = Mockito.mockStatic(KNNVectorValuesFactory.class)) { + KNNWeight.initialize(null); knnSettingsMockedStatic.when(() -> KNNSettings.getFilteredExactSearchThreshold(INDEX_NAME)).thenReturn(10); byte[] vector = new byte[] { 1, 3 }; int k = 1; @@ -906,7 +912,16 @@ public void testANNWithFilterQuery_whenExactSearchViaThresholdSettingOnBinaryInd when(filterScorer.iterator()).thenReturn(DocIdSetIterator.all(filterDocIds.length)); - final KNNQuery query = new KNNQuery(FIELD_NAME, BYTE_QUERY_VECTOR, k, INDEX_NAME, FILTER_QUERY, null, VectorDataType.BINARY); + final KNNQuery query = new KNNQuery( + FIELD_NAME, + BYTE_QUERY_VECTOR, + k, + INDEX_NAME, + FILTER_QUERY, + null, + VectorDataType.BINARY, + null + ); final float boost = (float) randomDoubleBetween(0, 10, true); final KNNWeight knnWeight = new KNNWeight(query, boost, filterQueryWeight); @@ -960,7 +975,7 @@ public void testANNWithFilterQuery_whenEmptyFilterIds_thenReturnEarly() { when(filterQueryWeight.scorer(leafReaderContext)).thenReturn(filterScorer); when(filterScorer.iterator()).thenReturn(DocIdSetIterator.empty()); - final KNNQuery query = new KNNQuery(FIELD_NAME, QUERY_VECTOR, K, INDEX_NAME, FILTER_QUERY, null); + final KNNQuery query = new KNNQuery(FIELD_NAME, QUERY_VECTOR, K, INDEX_NAME, FILTER_QUERY, null, null); final KNNWeight knnWeight = new KNNWeight(query, 0.0f, filterQueryWeight); final FieldInfos fieldInfos = mock(FieldInfos.class); @@ -978,6 +993,8 @@ public void testANNWithFilterQuery_whenEmptyFilterIds_thenReturnEarly() { @SneakyThrows public void testANNWithParentsFilter_whenExactSearch_thenSuccess() { + ModelDao modelDao = mock(ModelDao.class); + KNNWeight.initialize(modelDao); SegmentReader reader = getMockedSegmentReader(); final LeafReaderContext leafReaderContext = mock(LeafReaderContext.class); @@ -1006,7 +1023,7 @@ public void testANNWithParentsFilter_whenExactSearch_thenSuccess() { final Weight filterQueryWeight = mock(Weight.class); when(filterQueryWeight.scorer(leafReaderContext)).thenReturn(filterScorer); - final KNNQuery query = new KNNQuery(FIELD_NAME, QUERY_VECTOR, K, INDEX_NAME, FILTER_QUERY, parentFilter); + final KNNQuery query = new KNNQuery(FIELD_NAME, QUERY_VECTOR, K, INDEX_NAME, FILTER_QUERY, parentFilter, null); final float boost = (float) randomDoubleBetween(0, 10, true); final KNNWeight knnWeight = new KNNWeight(query, boost, filterQueryWeight); diff --git a/src/test/java/org/opensearch/knn/index/query/ResultUtilTests.java b/src/test/java/org/opensearch/knn/index/query/ResultUtilTests.java new file mode 100644 index 000000000..70cb86e02 --- /dev/null +++ b/src/test/java/org/opensearch/knn/index/query/ResultUtilTests.java @@ -0,0 +1,140 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.query; + +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.search.ScoreDoc; +import org.apache.lucene.search.TopDocs; +import org.apache.lucene.util.BitSet; +import org.opensearch.knn.KNNTestCase; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +public class ResultUtilTests extends KNNTestCase { + + public void testReduceToTopK() { + int firstPassK = 20; + int finalK = 10; + int segmentCount = 5; + + List> initialLeafResults = getRandomListOfResults(firstPassK, segmentCount); + List> reducedLeafResults = initialLeafResults.stream().map(HashMap::new).collect(Collectors.toList()); + ResultUtil.reduceToTopK(reducedLeafResults, finalK); + assertTopK(initialLeafResults, reducedLeafResults, finalK); + + firstPassK = 5; + finalK = 20; + segmentCount = 1; + + initialLeafResults = getRandomListOfResults(firstPassK, segmentCount); + reducedLeafResults = initialLeafResults.stream().map(HashMap::new).collect(Collectors.toList()); + ResultUtil.reduceToTopK(reducedLeafResults, finalK); + assertTopK(initialLeafResults, reducedLeafResults, firstPassK); + } + + public void testResultMapToMatchBitSet() throws IOException { + int firstPassK = 35; + Map perLeafResults = getRandomResults(firstPassK); + BitSet resultBitset = ResultUtil.resultMapToMatchBitSet(perLeafResults); + assertResultMapToMatchBitSet(perLeafResults, resultBitset); + } + + public void testResultMapToDocIds() throws IOException { + int firstPassK = 42; + Map perLeafResults = getRandomResults(firstPassK); + final int maxDoc = Collections.max(perLeafResults.keySet()) + 1; + DocIdSetIterator resultDocIdSetIterator = ResultUtil.resultMapToDocIds(perLeafResults, maxDoc); + assertResultMapToDocIdSetIterator(perLeafResults, resultDocIdSetIterator); + } + + public void testResultMapToTopDocs() { + int k = 18; + int offset = 121; + Map perLeafResults = getRandomResults(k); + TopDocs topDocs = ResultUtil.resultMapToTopDocs(perLeafResults, offset); + assertResultMapToTopDocs(perLeafResults, topDocs, k, offset); + } + + private void assertResultMapToTopDocs(Map perLeafResults, TopDocs topDocs, int k, int offset) { + assertEquals(k, topDocs.totalHits.value); + float previousScore = Float.MAX_VALUE; + for (ScoreDoc scoreDoc : topDocs.scoreDocs) { + assertTrue(perLeafResults.containsKey(scoreDoc.doc - offset)); + assertEquals(perLeafResults.get(scoreDoc.doc - offset), scoreDoc.score, 0.0001); + assertTrue(previousScore > scoreDoc.score); + previousScore = scoreDoc.score; + } + } + + private void assertTopK(List> beforeResults, List> reducedResults, int expectedK) { + assertEquals(beforeResults.size(), reducedResults.size()); + assertEquals(expectedK, reducedResults.stream().map(Map::size).reduce(Integer::sum).orElse(-1).intValue()); + float minScore = getMinScore(reducedResults); + int count = 0; + for (Map result : beforeResults) { + for (float score : result.values()) { + if (score >= minScore) { + count++; + } + } + } + assertEquals(expectedK, count); + } + + private void assertResultMapToMatchBitSet(Map resultsMap, BitSet resultBitset) { + assertEquals(resultsMap.size(), resultBitset.cardinality()); + for (Integer docId : resultsMap.keySet()) { + assertTrue(resultBitset.get(docId)); + } + } + + private void assertResultMapToDocIdSetIterator(Map resultsMap, DocIdSetIterator resultDocIdSetIterator) + throws IOException { + int count = 0; + int docId = resultDocIdSetIterator.nextDoc(); + while (docId != DocIdSetIterator.NO_MORE_DOCS) { + assertTrue(resultsMap.containsKey(docId)); + count++; + docId = resultDocIdSetIterator.nextDoc(); + } + assertEquals(resultsMap.size(), count); + } + + private List> getRandomListOfResults(int k, int segments) { + List> perLeafResults = new ArrayList<>(); + for (int i = 0; i < segments; i++) { + perLeafResults.add(getRandomResults(k)); + } + return perLeafResults; + } + + private Map getRandomResults(int k) { + Map results = new HashMap<>(); + for (int i = 0; i < k; i++) { + results.put(i, random().nextFloat()); + } + + return results; + } + + private float getMinScore(List> perLeafResults) { + float minScore = Float.MAX_VALUE; + for (Map result : perLeafResults) { + for (float score : result.values()) { + if (score < minScore) { + minScore = score; + } + } + } + return minScore; + } +} diff --git a/src/test/java/org/opensearch/knn/index/query/nativelib/NativeEngineKNNVectorQueryTests.java b/src/test/java/org/opensearch/knn/index/query/nativelib/NativeEngineKNNVectorQueryTests.java index 1e4b11a12..b6059bd11 100644 --- a/src/test/java/org/opensearch/knn/index/query/nativelib/NativeEngineKNNVectorQueryTests.java +++ b/src/test/java/org/opensearch/knn/index/query/nativelib/NativeEngineKNNVectorQueryTests.java @@ -17,21 +17,29 @@ import org.apache.lucene.search.TaskExecutor; import org.apache.lucene.search.TopDocs; import org.apache.lucene.util.Bits; -import org.mockito.ArgumentMatchers; import org.mockito.InjectMocks; import org.mockito.Mock; +import org.mockito.MockedStatic; +import org.mockito.invocation.InvocationOnMock; import org.opensearch.knn.index.query.KNNQuery; import org.opensearch.knn.index.query.KNNWeight; +import org.opensearch.knn.index.query.ResultUtil; +import org.opensearch.knn.index.query.rescore.RescoreContext; import org.opensearch.test.OpenSearchTestCase; import java.util.ArrayList; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.concurrent.Callable; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.mockStatic; import static org.mockito.Mockito.when; import static org.mockito.MockitoAnnotations.openMocks; @@ -73,13 +81,13 @@ public void setUp() throws Exception { when(knnQuery.createWeight(searcher, ScoreMode.COMPLETE, 1)).thenReturn(knnWeight); when(searcher.getTaskExecutor()).thenReturn(taskExecutor); - when(taskExecutor.invokeAll(ArgumentMatchers.>anyList())).thenAnswer(invocationOnMock -> { - List> callables = invocationOnMock.getArgument(0); - List topDocs = new ArrayList<>(); - for (Callable callable : callables) { - topDocs.add(callable.call()); + when(taskExecutor.invokeAll(any())).thenAnswer(invocationOnMock -> { + List>> callables = invocationOnMock.getArgument(0); + List> results = new ArrayList<>(); + for (Callable> callable : callables) { + results.add(callable.call()); } - return topDocs; + return results; }); when(reader.getContext()).thenReturn(indexReaderContext); @@ -91,8 +99,8 @@ public void testMultiLeaf() { List leaves = List.of(leaf1, leaf2); when(reader.leaves()).thenReturn(leaves); - when(knnWeight.searchLeaf(leaf1)).thenReturn(Map.of(0, 1.2f, 1, 5.1f, 2, 2.2f)); - when(knnWeight.searchLeaf(leaf2)).thenReturn(Map.of(4, 3.4f, 3, 5.1f)); + when(knnWeight.searchLeaf(leaf1, 4)).thenReturn(new HashMap<>(Map.of(0, 1.2f, 1, 5.1f, 2, 2.2f))); + when(knnWeight.searchLeaf(leaf2, 4)).thenReturn(new HashMap<>(Map.of(4, 3.4f, 3, 5.1f))); // Making sure there is deleted docs in one of the segments Bits liveDocs = mock(Bits.class); @@ -124,7 +132,7 @@ public void testSingleLeaf() { // Given List leaves = List.of(leaf1); when(reader.leaves()).thenReturn(leaves); - when(knnWeight.searchLeaf(leaf1)).thenReturn(Map.of(0, 1.2f, 1, 5.1f, 2, 2.2f)); + when(knnWeight.searchLeaf(leaf1, 4)).thenReturn(new HashMap<>(Map.of(0, 1.2f, 1, 5.1f, 2, 2.2f))); when(knnQuery.getK()).thenReturn(4); when(indexReaderContext.id()).thenReturn(1); @@ -145,7 +153,7 @@ public void testNoMatch() { // Given List leaves = List.of(leaf1); when(reader.leaves()).thenReturn(leaves); - when(knnWeight.searchLeaf(leaf1)).thenReturn(Collections.emptyMap()); + when(knnWeight.searchLeaf(leaf1, 4)).thenReturn(Collections.emptyMap()); when(knnQuery.getK()).thenReturn(4); // When Query actual = objectUnderTest.rewrite(searcher); @@ -153,4 +161,41 @@ public void testNoMatch() { // Then assertEquals(new MatchNoDocsQuery(), actual); } + + @SneakyThrows + public void testRescore() { + // Given + List leaves = List.of(leaf1, leaf2); + when(reader.leaves()).thenReturn(leaves); + + int k = 2; + int firstPassK = 3; + Map initialLeaf1Results = new HashMap<>(Map.of(0, 21f, 1, 19f, 2, 17f, 3, 15f)); + Map initialLeaf2Results = new HashMap<>(Map.of(0, 20f, 1, 18f, 2, 16f, 3, 14f)); + Map rescoredLeaf1Results = new HashMap<>(Map.of(0, 18f, 1, 20f)); + Map rescoredLeaf2Results = new HashMap<>(Map.of(0, 21f)); + TopDocs topDocs1 = ResultUtil.resultMapToTopDocs(Map.of(1, 20f), 0); + TopDocs topDocs2 = ResultUtil.resultMapToTopDocs(Map.of(0, 21f), 4); + DocAndScoreQuery expected = new DocAndScoreQuery(2, new int[] { 1, 4 }, new float[] { 20f, 21f }, new int[] { 0, 4, 2 }, 1); + + when(indexReaderContext.id()).thenReturn(1); + when(knnQuery.getRescoreContext()).thenReturn(RescoreContext.builder().oversampleFactor(1.5f).build()); + when(knnQuery.getK()).thenReturn(k); + when(knnWeight.getQuery()).thenReturn(knnQuery); + when(knnWeight.searchLeaf(leaf1, firstPassK)).thenReturn(initialLeaf1Results); + when(knnWeight.searchLeaf(leaf2, firstPassK)).thenReturn(initialLeaf2Results); + when(knnWeight.exactSearch(eq(leaf1), any(), anyBoolean(), anyInt())).thenReturn(rescoredLeaf1Results); + when(knnWeight.exactSearch(eq(leaf2), any(), anyBoolean(), anyInt())).thenReturn(rescoredLeaf2Results); + try (MockedStatic mockedResultUtil = mockStatic(ResultUtil.class)) { + mockedResultUtil.when(() -> ResultUtil.reduceToTopK(any(), anyInt())).thenAnswer(InvocationOnMock::callRealMethod); + mockedResultUtil.when(() -> ResultUtil.resultMapToTopDocs(eq(rescoredLeaf1Results), anyInt())).thenAnswer(t -> topDocs1); + mockedResultUtil.when(() -> ResultUtil.resultMapToTopDocs(eq(rescoredLeaf2Results), anyInt())).thenAnswer(t -> topDocs2); + try (MockedStatic mockedStaticNativeKnnVectorQuery = mockStatic(NativeEngineKnnVectorQuery.class)) { + mockedStaticNativeKnnVectorQuery.when(() -> NativeEngineKnnVectorQuery.findSegmentStarts(any(), any())) + .thenReturn(new int[] { 0, 4, 2 }); + Query actual = objectUnderTest.rewrite(searcher); + assertEquals(expected, actual); + } + } + } } diff --git a/src/test/java/org/opensearch/knn/index/query/rescore/RescoreContextTests.java b/src/test/java/org/opensearch/knn/index/query/rescore/RescoreContextTests.java new file mode 100644 index 000000000..6d872ddb8 --- /dev/null +++ b/src/test/java/org/opensearch/knn/index/query/rescore/RescoreContextTests.java @@ -0,0 +1,26 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.query.rescore; + +import org.opensearch.knn.KNNTestCase; + +import static org.opensearch.knn.index.query.rescore.RescoreContext.MAX_FIRST_PASS_RESULTS; + +public class RescoreContextTests extends KNNTestCase { + + public void testGetFirstPassK() { + float oversample = 2.6f; + RescoreContext rescoreContext = RescoreContext.builder().oversampleFactor(oversample).build(); + int finalK = 100; + assertEquals(260, rescoreContext.getFirstPassK(finalK)); + finalK = 1; + assertEquals(3, rescoreContext.getFirstPassK(finalK)); + finalK = 0; + assertEquals(0, rescoreContext.getFirstPassK(finalK)); + finalK = MAX_FIRST_PASS_RESULTS; + assertEquals(MAX_FIRST_PASS_RESULTS, rescoreContext.getFirstPassK(finalK)); + } +} diff --git a/src/test/java/org/opensearch/knn/integ/NestedSearchIT.java b/src/test/java/org/opensearch/knn/integ/NestedSearchIT.java index 2f32021f4..12504b5d8 100644 --- a/src/test/java/org/opensearch/knn/integ/NestedSearchIT.java +++ b/src/test/java/org/opensearch/knn/integ/NestedSearchIT.java @@ -43,6 +43,8 @@ import static org.opensearch.knn.common.KNNConstants.TYPE_KNN_VECTOR; import static org.opensearch.knn.common.KNNConstants.TYPE_NESTED; import static org.opensearch.knn.common.KNNConstants.VECTOR; +import static org.opensearch.knn.index.query.parser.RescoreParser.RESCORE_OVERSAMPLE_PARAMETER; +import static org.opensearch.knn.index.query.parser.RescoreParser.RESCORE_PARAMETER; public class NestedSearchIT extends KNNRestTestCase { private static final String INDEX_NAME = "test-index-nested-search"; @@ -110,6 +112,31 @@ public void testNestedSearchWithFaiss_whenKIsTwo_thenReturnTwoResults() { assertEquals("13", parseIds(entity).get(1)); } + @SneakyThrows + public void testNestedSearchWithFaiss_whenRescoreEnabled_thenSucceed() { + createKnnIndex(2, KNNEngine.FAISS.getName()); + + int totalDocCount = 15; + for (int i = 0; i < totalDocCount; i++) { + String doc = NestedKnnDocBuilder.create(FIELD_NAME_NESTED) + .addVectors(FIELD_NAME_VECTOR, new Float[] { (float) i, (float) i }, new Float[] { (float) i, (float) i }) + .build(); + addKnnDoc(INDEX_NAME, String.valueOf(i), doc); + } + + refreshIndex(INDEX_NAME); + forceMergeKnnIndex(INDEX_NAME); + + Float[] queryVector = { 14f, 14f }; + Response response = queryNestedField(INDEX_NAME, 2, queryVector, null, null, null, 2.0f); + + String entity = EntityUtils.toString(response.getEntity()); + assertEquals(2, parseHits(entity)); + assertEquals(2, parseTotalSearchHits(entity)); + assertEquals("14", parseIds(entity).get(0)); + assertEquals("13", parseIds(entity).get(1)); + } + /** * { * "query": { @@ -159,7 +186,7 @@ public void testNestedSearchWithFaiss_whenDoingExactSearch_thenReturnCorrectResu updateIndexSettings(INDEX_NAME, Settings.builder().put(KNNSettings.ADVANCED_FILTERED_EXACT_SEARCH_THRESHOLD, 100)); Float[] queryVector = { 3f, 3f, 3f }; - Response response = queryNestedField(INDEX_NAME, 3, queryVector, FIELD_NAME_PARKING, FIELD_VALUE_TRUE, null); + Response response = queryNestedField(INDEX_NAME, 3, queryVector, FIELD_NAME_PARKING, FIELD_VALUE_TRUE, null, null); String entity = EntityUtils.toString(response.getEntity()); List docIds = parseIds(entity); assertEquals(2, docIds.size()); @@ -215,7 +242,7 @@ public void testNestedWithFaiss_whenFilter_whenDoRadialSearch_thenReturnCorrectR Float[] queryVector = { 3f, 3f, 3f }; Float minScore = 0.00001f; - Response response = queryNestedField(INDEX_NAME, null, queryVector, FIELD_NAME_PARKING, FIELD_VALUE_TRUE, minScore); + Response response = queryNestedField(INDEX_NAME, null, queryVector, FIELD_NAME_PARKING, FIELD_VALUE_TRUE, minScore, null); String entity = EntityUtils.toString(response.getEntity()); List docIds = parseIds(entity); @@ -279,7 +306,7 @@ private void createKnnIndex(final int dimension, final String engine) throws Exc } private Response queryNestedField(final String index, final int k, final Object[] vector) throws IOException { - return queryNestedField(index, k, vector, null, null, null); + return queryNestedField(index, k, vector, null, null, null, null); } private Response queryNestedField( @@ -288,7 +315,8 @@ private Response queryNestedField( final Object[] vector, final String filterName, final String filterValue, - final Float minScore + final Float minScore, + final Float oversampleFactor ) throws IOException { XContentBuilder builder = XContentFactory.jsonBuilder().startObject().startObject(QUERY); builder.startObject(TYPE_NESTED); @@ -309,6 +337,12 @@ private Response queryNestedField( builder.endObject(); builder.endObject(); } + if (oversampleFactor != null) { + builder.startObject(RESCORE_PARAMETER); + builder.field(RESCORE_OVERSAMPLE_PARAMETER, oversampleFactor); + builder.endObject(); + } + builder.endObject().endObject().endObject().endObject().endObject().endObject(); Request request = new Request("POST", "/" + index + "/_search"); From 020ab1f671160b5c6c315b77adbdc8db62a7fc1e Mon Sep 17 00:00:00 2001 From: Tejas Shah Date: Wed, 21 Aug 2024 19:31:01 -0700 Subject: [PATCH 337/416] Integrates FAISS iterative builds with NativeEngines990KnnVectorsFormat (#1950) (#1992) * Iterative Vector Insertion (#1840) * Rebased with new version of k-NN Signed-off-by: Andrew Klepchick * Optimized faiss insertion Signed-off-by: Andrew Klepchick * Optimized threadCount logic Signed-off-by: Andrew Klepchick * Removed IDEA files Signed-off-by: Andrew Klepchick * Removed unnecessary cmake file Signed-off-by: Andrew Klepchick * Added comments to new functions Signed-off-by: Andrew Klepchick * Removed createIndex and fixed test cases that use it Signed-off-by: Andrew Klepchick * Removed unused code Signed-off-by: Andrew Klepchick * Explained zero initialization for vector transfer Signed-off-by: Andrew Klepchick * Added locale Signed-off-by: Andrew Klepchick * Spotless Apply Signed-off-by: Andrew Klepchick * Account for zero documents in finished batch Signed-off-by: Andrew Klepchick * Changed where we check for zero docs Signed-off-by: Andrew Klepchick * Changed tip for return Signed-off-by: Andrew Klepchick * Use unique pointers to make sure resources are released on exception Signed-off-by: Andrew Klepchick * Moved createIndex to testUtils Signed-off-by: Andrew Klepchick * Fixed memory management so that the underlying index is not deleted after initialized Signed-off-by: Andrew Klepchick * Created new KNNIndexBuilder graph to make index building more modular Signed-off-by: Andrew Klepchick * Streamlined logic in KNNIndexBuilder. Signed-off-by: Andrew Klepchick * Cleaned up unnecessary code in KNN80DocValuesConsumer Signed-off-by: Andrew Klepchick * Fixed memory management process Signed-off-by: Andrew Klepchick * Added note about index initialization in faiss_index_service Signed-off-by: Andrew Klepchick * Accounted for case where the exception happens after the indexWriter is released. Signed-off-by: Andrew Klepchick * Delete jni/src/.idea/modules.xml Signed-off-by: Andrew Klepchick * Delete jni/src/.idea/vcs.xml Signed-off-by: Andrew Klepchick * Delete jni/src/.idea/workspace.xml Signed-off-by: Andrew Klepchick * Spotless apply and free iterative index on exception Signed-off-by: Andrew Klepchick * Undid hack for checking first document metrics Signed-off-by: Andrew Klepchick * Removed print statements Signed-off-by: Andrew Klepchick * Free Vector Transfer on batch ingestion Signed-off-by: Andrew Klepchick * Undid free Signed-off-by: Andrew Klepchick * Fixed check for transfer ready Signed-off-by: Andrew Klepchick * Don't crash when zero vectors inserted? Signed-off-by: Andrew Klepchick * Reverted to old insertion process? Signed-off-by: Andrew Klepchick * Spotless apply Signed-off-by: Andrew Klepchick * Added back createOutput Signed-off-by: Andrew Klepchick * Removed prior createOutput Signed-off-by: Andrew Klepchick * Test remaking vectorTransfer Signed-off-by: Andrew Klepchick * Test restructuring of insertion Signed-off-by: Andrew Klepchick * Fixed case where vector address is immediately discarded Signed-off-by: Andrew Klepchick * Spotless apply Signed-off-by: Andrew Klepchick * Split Index Builder into multiple classes Signed-off-by: Andrew Klepchick * Fixed descriptions of functions in faiss_index_service Signed-off-by: Andrew Klepchick * Added back copyright files Signed-off-by: Andrew Klepchick * Removed unused builder names Signed-off-by: Andrew Klepchick * Modified tests to work with new insertion methods Signed-off-by: Andrew Klepchick * Track index insertions Signed-off-by: Andrew Klepchick * Tracked insertions for binary indices Signed-off-by: Andrew Klepchick * Added back insertIds Signed-off-by: Andrew Klepchick * Added check for freeVectorData to see if it works with an already deleted address Signed-off-by: Andrew Klepchick * Cleaned up logs and comments in KNNIndexBuilder Signed-off-by: Andrew Klepchick * Restructured the logic for KNNIndexBuilder Signed-off-by: Andrew Klepchick * Changed package name of KNNIndexBuilder Signed-off-by: Andrew Klepchick * Changed all package names and deleted unnecessary headers Signed-off-by: Andrew Klepchick * Fixed for loop Signed-off-by: Andrew Klepchick * Removed createIndex methods for faiss index service Signed-off-by: Andrew Klepchick * Fixed package to fit naming conventions Signed-off-by: Andrew Klepchick * Changed name of index builder Signed-off-by: Andrew Klepchick * Spotless apply Signed-off-by: Andrew Klepchick * Added comments to NativeIndexBuilder and restructured Signed-off-by: Andrew Klepchick * Added deletion for memoryAddress Signed-off-by: Andrew Klepchick * Spotless apply Signed-off-by: Andrew Klepchick * Changed naming of classes to Writer and changed package name to fit conventions Signed-off-by: Andrew Klepchick * Changed NativeIndexInfo and NativeVectorInfo to follow builder pattern Signed-off-by: Andrew Klepchick * Added feature to changelog Signed-off-by: Andrew Klepchick * Added class descriptions to each NativeIndexWriter Signed-off-by: Andrew Klepchick * Changed name to getBytesPerVector Signed-off-by: Andrew Klepchick * Added == false instead of ! for readability Signed-off-by: Andrew Klepchick * Fixed changelog Signed-off-by: Andrew Klepchick * Fixed naming in docvaluesconsumer Signed-off-by: Andrew Klepchick * SpotlessApply Signed-off-by: Andrew Klepchick * Made it so that we don't reuse testValues and removed a foot gun Signed-off-by: Andrew Klepchick * Removed another foot gun in getIndexInfo Signed-off-by: Andrew Klepchick * Fixed javadoc Signed-off-by: Andrew Klepchick * Added deletion on exception cases Signed-off-by: Andrew Klepchick * Removed unnecessary delete (NativeIndexWriter will handle deletion of vectors on exception) Signed-off-by: Andrew Klepchick * Added correct logger and getWriter method to NativeIndexWriter Signed-off-by: Andrew Klepchick * Ensured memory safety on JNI layer so that Java doesn't have to wrap everything in a try catch loop. Signed-off-by: Andrew Klepchick * Refactored NativeIndexWriter and added comments to FaissService Signed-off-by: Andrew Klepchick * Removed free in the JNIExport since index will always be freed in writeIndex. Signed-off-by: Andrew Klepchick * Changed getVectorTransfer back to accept VectorDataType Signed-off-by: Andrew Klepchick * Reverted free since not guaranteed to be IDMap. Signed-off-by: Andrew Klepchick * Added all processes in addKNNBinaryField to NativeIndexWriter.createKNNIndex Signed-off-by: Andrew Klepchick * Fixed javadoc Signed-off-by: Andrew Klepchick * Applied spotless Signed-off-by: Andrew Klepchick * Added back writeFooter Signed-off-by: Andrew Klepchick * Removed threadCount fron writeIndex Signed-off-by: Andrew Klepchick * Removed redundancies in KNN80DocValuesConsumer Signed-off-by: Andrew Klepchick * Removed serializationMode Signed-off-by: Andrew Klepchick * Fixed changelog Signed-off-by: Andrew Klepchick * Fixed changelog Signed-off-by: Andrew Klepchick * Removed double free test as we don't have to worry about that anymore Signed-off-by: Andrew Klepchick * Accounted for HNSWSQ in index service Signed-off-by: Andrew Klepchick * Removed delete in catch Signed-off-by: Andrew Klepchick * Fixed faiss tests to work with writeIndex Signed-off-by: Andrew Klepchick --------- Signed-off-by: Andrew Klepchick * Index Initialization Alloc Method (#1933) * Added methods for allocating memory before inserting vectors to a faiss index Signed-off-by: Andrew Klepchick * Fixed logic that gets type of index Signed-off-by: Andrew Klepchick * Removed print statement Signed-off-by: Andrew Klepchick * Removed unnecessary iostream Signed-off-by: Andrew Klepchick * Removed flat index Signed-off-by: Andrew Klepchick * Fixed flat index case Signed-off-by: Andrew Klepchick * Fixed naming Signed-off-by: Andrew Klepchick * Properly allocate HNSWSQ storage Signed-off-by: Andrew Klepchick * Removed print statements Signed-off-by: Andrew Klepchick * Fixed changelog Signed-off-by: Andrew Klepchick * Removed unnecessary lib Signed-off-by: Andrew Klepchick * Made alloc adaptive to different code sizes Signed-off-by: Andrew Klepchick --------- Signed-off-by: Andrew Klepchick * Integrates FAISS iterative builds with NativeEngines990KnnVectorsFormat Changes include reusing the same vector buffer in the JNI layer Signed-off-by: Tejas Shah --------- Signed-off-by: Andrew Klepchick Signed-off-by: Tejas Shah Co-authored-by: Andrew Klepchick (cherry picked from commit fd59b9adf42b07aa2b2058c4badff6dacf8306a8) Signed-off-by: Tejas Shah --- CHANGELOG.md | 6 +- jni/include/commons.h | 17 +- jni/include/faiss_index_service.h | 87 +++-- jni/include/faiss_wrapper.h | 9 +- .../org_opensearch_knn_jni_FaissService.h | 49 ++- .../org_opensearch_knn_jni_JNICommons.h | 6 +- jni/src/commons.cpp | 14 +- jni/src/faiss_index_service.cpp | 168 +++++++--- jni/src/faiss_wrapper.cpp | 72 +++-- .../org_opensearch_knn_jni_FaissService.cpp | 76 ++++- jni/src/org_opensearch_knn_jni_JNICommons.cpp | 8 +- jni/tests/commons_test.cpp | 116 ++++++- jni/tests/faiss_index_service_test.cpp | 30 +- jni/tests/faiss_wrapper_test.cpp | 94 +++++- jni/tests/mocks/faiss_index_service_mock.h | 25 +- .../knn/common/FieldInfoExtractor.java | 19 +- .../opensearch/knn/common/KNNVectorUtil.java | 35 +- .../codec/BasePerFieldKnnVectorsFormat.java | 69 ++-- .../KNN80Codec/KNN80DocValuesConsumer.java | 279 +--------------- .../NativeEngineFieldVectorsWriter.java | 3 + .../NativeEngines990KnnVectorsWriter.java | 42 ++- .../DefaultIndexBuildStrategy.java | 95 ++++++ .../MemOptimizedNativeIndexBuildStrategy.java | 117 +++++++ .../nativeindex/NativeIndexBuildStrategy.java | 19 ++ .../codec/nativeindex/NativeIndexWriter.java | 298 ++++++++++++++++++ .../nativeindex/model/BuildIndexParams.java | 26 ++ .../transfer/OffHeapBinaryVectorTransfer.java | 32 ++ .../transfer/OffHeapByteVectorTransfer.java | 38 +++ .../transfer/OffHeapFloatVectorTransfer.java | 36 +++ .../codec/transfer/OffHeapVectorTransfer.java | 99 ++++++ .../OffHeapVectorTransferFactory.java | 37 +++ .../index/codec/transfer/VectorTransfer.java | 54 ---- .../codec/transfer/VectorTransferByte.java | 68 ---- .../codec/transfer/VectorTransferFloat.java | 71 ----- .../knn/index/codec/util/KNNCodecUtil.java | 49 --- .../vectorvalues/KNNBinaryVectorValues.java | 17 +- .../vectorvalues/KNNByteVectorValues.java | 12 + .../vectorvalues/KNNFloatVectorValues.java | 11 + .../index/vectorvalues/KNNVectorValues.java | 25 +- .../org/opensearch/knn/indices/ModelUtil.java | 4 +- .../org/opensearch/knn/jni/FaissService.java | 64 +++- .../org/opensearch/knn/jni/JNICommons.java | 76 ++++- .../org/opensearch/knn/jni/JNIService.java | 197 ++++++++---- .../knn/common/FieldInfoExtractorTests.java | 24 ++ .../knn/common/KNNVectorUtilTests.java | 34 ++ .../KNN80DocValuesConsumerTests.java | 82 +++-- ...NativeEngines990KnnVectorsFormatTests.java | 14 +- .../DefaultIndexBuildStrategyTests.java | 167 ++++++++++ ...ptimizedNativeIndexBuildStrategyTests.java | 129 ++++++++ .../OffHeapVectorTransferFactoryTests.java | 26 ++ .../transfer/OffHeapVectorTransferTests.java | 92 ++++++ .../transfer/VectorTransferByteTests.java | 56 ---- .../transfer/VectorTransferFloatTests.java | 66 ---- .../index/codec/util/KNNCodecUtilTests.java | 43 --- .../memory/NativeMemoryAllocationTests.java | 5 +- .../memory/NativeMemoryLoadStrategyTests.java | 7 +- .../vectorvalues/KNNVectorValuesTests.java | 42 ++- .../opensearch/knn/jni/JNIServiceTests.java | 93 +++--- .../java/org/opensearch/knn/TestUtils.java | 17 +- 59 files changed, 2483 insertions(+), 1083 deletions(-) create mode 100644 src/main/java/org/opensearch/knn/index/codec/nativeindex/DefaultIndexBuildStrategy.java create mode 100644 src/main/java/org/opensearch/knn/index/codec/nativeindex/MemOptimizedNativeIndexBuildStrategy.java create mode 100644 src/main/java/org/opensearch/knn/index/codec/nativeindex/NativeIndexBuildStrategy.java create mode 100644 src/main/java/org/opensearch/knn/index/codec/nativeindex/NativeIndexWriter.java create mode 100644 src/main/java/org/opensearch/knn/index/codec/nativeindex/model/BuildIndexParams.java create mode 100644 src/main/java/org/opensearch/knn/index/codec/transfer/OffHeapBinaryVectorTransfer.java create mode 100644 src/main/java/org/opensearch/knn/index/codec/transfer/OffHeapByteVectorTransfer.java create mode 100644 src/main/java/org/opensearch/knn/index/codec/transfer/OffHeapFloatVectorTransfer.java create mode 100644 src/main/java/org/opensearch/knn/index/codec/transfer/OffHeapVectorTransfer.java create mode 100644 src/main/java/org/opensearch/knn/index/codec/transfer/OffHeapVectorTransferFactory.java delete mode 100644 src/main/java/org/opensearch/knn/index/codec/transfer/VectorTransfer.java delete mode 100644 src/main/java/org/opensearch/knn/index/codec/transfer/VectorTransferByte.java delete mode 100644 src/main/java/org/opensearch/knn/index/codec/transfer/VectorTransferFloat.java create mode 100644 src/test/java/org/opensearch/knn/index/codec/nativeindex/DefaultIndexBuildStrategyTests.java create mode 100644 src/test/java/org/opensearch/knn/index/codec/nativeindex/MemOptimizedNativeIndexBuildStrategyTests.java create mode 100644 src/test/java/org/opensearch/knn/index/codec/transfer/OffHeapVectorTransferFactoryTests.java create mode 100644 src/test/java/org/opensearch/knn/index/codec/transfer/OffHeapVectorTransferTests.java delete mode 100644 src/test/java/org/opensearch/knn/index/codec/transfer/VectorTransferByteTests.java delete mode 100644 src/test/java/org/opensearch/knn/index/codec/transfer/VectorTransferFloatTests.java diff --git a/CHANGELOG.md b/CHANGELOG.md index acb195121..f469f4a1f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,4 @@ + # CHANGELOG All notable changes to this project are documented in this file. @@ -6,7 +7,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ## [Unreleased 3.0](https://github.com/opensearch-project/k-NN/compare/2.x...HEAD) ### Features ### Enhancements -### Bug Fixes +### Bug Fixes ### Infrastructure ### Documentation ### Maintenance @@ -17,12 +18,13 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), * Integrate Lucene Vector field with native engines to use KNNVectorFormat during segment creation [#1945](https://github.com/opensearch-project/k-NN/pull/1945) * k-NN query rescore support for native engines [#1984](https://github.com/opensearch-project/k-NN/pull/1984) ### Enhancements +* Adds iterative graph build capability into a faiss index to improve the memory footprint during indexing and Integrates KNNVectorsFormat for native engines[#1950](https://github.com/opensearch-project/k-NN/pull/1950) ### Bug Fixes * Corrected search logic for scenario with non-existent fields in filter [#1874](https://github.com/opensearch-project/k-NN/pull/1874) * Fix graph merge stats size calculation [#1844](https://github.com/opensearch-project/k-NN/pull/1844) * Integrate Lucene Vector field with native engines to use KNNVectorFormat during segment creation [#1945](https://github.com/opensearch-project/k-NN/pull/1945) * Add script_fields context to KNNAllowlist [#1917] (https://github.com/opensearch-project/k-NN/pull/1917) -* Fix graph merge stats size calculation [#1844](https://github.com/opensearch-project/k-NN/pull/1844) +* Fix graph merge stats size calculation [#1844](https://github.com/opensearch-project/k-NN/pull/1844) * Disallow a vector field to have an invalid character for a physical file name. [#1936](https://github.com/opensearch-project/k-NN/pull/1936) * Add script_fields context to KNNAllowlist [#1917] (https://github.com/opensearch-project/k-NN/pull/1917) ### Infrastructure diff --git a/jni/include/commons.h b/jni/include/commons.h index d02439377..4cdaf28fc 100644 --- a/jni/include/commons.h +++ b/jni/include/commons.h @@ -19,12 +19,19 @@ namespace knn_jni { * For subsequent calls you can pass the same memoryAddress. If the data cannot be stored in the memory location * will throw Exception. * + * append tells the method to keep appending to the existing vector. Passing the value as false will clear the vector + * without reallocating new memory. This helps with reducing memory frangmentation and overhead of allocating + * and deallocating when the memory address needs to be reused. + * + * CAUTION: The behavior is undefined if the memory address is deallocated and the method is called + * * @param memoryAddress The address of the memory location where data will be stored. * @param data 2D float array containing data to be stored in native memory. * @param initialCapacity The initial capacity of the memory location. + * @param append whether to append or start from index 0 when called subsequently with the same address * @return memory address of std::vector where the data is stored. */ - jlong storeVectorData(knn_jni::JNIUtilInterface *, JNIEnv *, jlong , jobjectArray, jlong); + jlong storeVectorData(knn_jni::JNIUtilInterface *, JNIEnv *, jlong , jobjectArray, jlong, jboolean); /** * This is utility function that can be used to store data in native memory. This function will allocate memory for @@ -33,12 +40,18 @@ namespace knn_jni { * For subsequent calls you can pass the same memoryAddress. If the data cannot be stored in the memory location * will throw Exception. * + * append tells the method to keep appending to the existing vector. Passing the value as false will clear the vector + * without reallocating new memory. This helps with reducing memory frangmentation and overhead of allocating + * and deallocating when the memory address needs to be reused. + * + * CAUTION: The behavior is undefined if the memory address is deallocated and the method is called + * * @param memoryAddress The address of the memory location where data will be stored. * @param data 2D byte array containing data to be stored in native memory. * @param initialCapacity The initial capacity of the memory location. * @return memory address of std::vector where the data is stored. */ - jlong storeByteVectorData(knn_jni::JNIUtilInterface *, JNIEnv *, jlong , jobjectArray, jlong); + jlong storeByteVectorData(knn_jni::JNIUtilInterface *, JNIEnv *, jlong , jobjectArray, jlong, jboolean); /** * Free up the memory allocated for the data stored in memory address. This function should be used with the memory diff --git a/jni/include/faiss_index_service.h b/jni/include/faiss_index_service.h index 59f15fda9..c57309cfc 100644 --- a/jni/include/faiss_index_service.h +++ b/jni/include/faiss_index_service.h @@ -31,38 +31,41 @@ namespace faiss_wrapper { class IndexService { public: IndexService(std::unique_ptr faissMethods); - //TODO Remove dependency on JNIUtilInterface and JNIEnv - //TODO Reduce the number of parameters - /** - * Create index + * Initialize index * * @param jniUtil jni util * @param env jni environment * @param metric space type for distance calculation * @param indexDescription index description to be used by faiss index factory * @param dim dimension of vectors + * @param numVectors number of vectors + * @param threadCount number of thread count to be used while adding data + * @param parameters parameters to be applied to faiss index + * @return memory address of the native index object + */ + virtual jlong initIndex(knn_jni::JNIUtilInterface *jniUtil, JNIEnv *env, faiss::MetricType metric, std::string indexDescription, int dim, int numVectors, int threadCount, std::unordered_map parameters); + /** + * Add vectors to index + * + * @param dim dimension of vectors * @param numIds number of vectors * @param threadCount number of thread count to be used while adding data * @param vectorsAddress memory address which is holding vector data - * @param ids a list of document ids for corresponding vectors + * @param idMapAddress memory address of the native index object + */ + virtual void insertToIndex(int dim, int numIds, int threadCount, int64_t vectorsAddress, std::vector &ids, jlong idMapAddress); + /** + * Write index to disk + * + * @param threadCount number of thread count to be used while adding data * @param indexPath path to write index - * @param parameters parameters to be applied to faiss index + * @param idMap memory address of the native index object */ - virtual void createIndex( - knn_jni::JNIUtilInterface * jniUtil, - JNIEnv * env, - faiss::MetricType metric, - std::string indexDescription, - int dim, - int numIds, - int threadCount, - int64_t vectorsAddress, - std::vector ids, - std::string indexPath, - std::unordered_map parameters); + virtual void writeIndex(std::string indexPath, jlong idMapAddress); virtual ~IndexService() = default; protected: + virtual void allocIndex(faiss::Index * index, size_t dim, size_t numVectors); std::unique_ptr faissMethods; }; @@ -76,7 +79,21 @@ class BinaryIndexService : public IndexService { //TODO Reduce the number of parameters BinaryIndexService(std::unique_ptr faissMethods); /** - * Create binary index + * Initialize index + * + * @param jniUtil jni util + * @param env jni environment + * @param metric space type for distance calculation + * @param indexDescription index description to be used by faiss index factory + * @param dim dimension of vectors + * @param numVectors number of vectors + * @param threadCount number of thread count to be used while adding data + * @param parameters parameters to be applied to faiss index + * @return memory address of the native index object + */ + virtual jlong initIndex(knn_jni::JNIUtilInterface *jniUtil, JNIEnv *env, faiss::MetricType metric, std::string indexDescription, int dim, int numVectors, int threadCount, std::unordered_map parameters) override; + /** + * Add vectors to index * * @param jniUtil jni util * @param env jni environment @@ -86,28 +103,30 @@ class BinaryIndexService : public IndexService { * @param numIds number of vectors * @param threadCount number of thread count to be used while adding data * @param vectorsAddress memory address which is holding vector data - * @param ids a list of document ids for corresponding vectors + * @param idMap a map of document id and vector id + * @param parameters parameters to be applied to faiss index + */ + virtual void insertToIndex(int dim, int numIds, int threadCount, int64_t vectorsAddress, std::vector &ids, jlong idMapAddress) override; + /** + * Write index to disk + * + * @param jniUtil jni util + * @param env jni environment + * @param metric space type for distance calculation + * @param indexDescription index description to be used by faiss index factory + * @param threadCount number of thread count to be used while adding data * @param indexPath path to write index + * @param idMap a map of document id and vector id * @param parameters parameters to be applied to faiss index */ - virtual void createIndex( - knn_jni::JNIUtilInterface * jniUtil, - JNIEnv * env, - faiss::MetricType metric, - std::string indexDescription, - int dim, - int numIds, - int threadCount, - int64_t vectorsAddress, - std::vector ids, - std::string indexPath, - std::unordered_map parameters - ) override; + virtual void writeIndex(std::string indexPath, jlong idMapAddress) override; virtual ~BinaryIndexService() = default; +protected: + virtual void allocIndex(faiss::Index * index, size_t dim, size_t numVectors) override; }; } } -#endif //OPENSEARCH_KNN_FAISS_INDEX_SERVICE_H +#endif //OPENSEARCH_KNN_FAISS_INDEX_SERVICE_H \ No newline at end of file diff --git a/jni/include/faiss_wrapper.h b/jni/include/faiss_wrapper.h index 5ad0dedc4..574efb6fd 100644 --- a/jni/include/faiss_wrapper.h +++ b/jni/include/faiss_wrapper.h @@ -18,10 +18,11 @@ namespace knn_jni { namespace faiss_wrapper { - // Create an index with ids and vectors. The configuration is defined by values in the Java map, parametersJ. - // The index is serialized to indexPathJ. - void CreateIndex(knn_jni::JNIUtilInterface * jniUtil, JNIEnv * env, jintArray idsJ, jlong vectorsAddressJ, jint dimJ, - jstring indexPathJ, jobject parametersJ, IndexService* indexService); + jlong InitIndex(knn_jni::JNIUtilInterface *jniUtil, JNIEnv *env, jlong numDocs, jint dimJ, jobject parametersJ, IndexService *indexService); + + void InsertToIndex(knn_jni::JNIUtilInterface *jniUtil, JNIEnv *env, jintArray idsJ, jlong vectorsAddressJ, jint dimJ, jlong indexAddr, jint threadCount, IndexService *indexService); + + void WriteIndex(knn_jni::JNIUtilInterface *jniUtil, JNIEnv *env, jstring indexPathJ, jlong indexAddr, IndexService *indexService); // Create an index with ids and vectors. Instead of creating a new index, this function creates the index // based off of the template index passed in. The index is serialized to indexPathJ. diff --git a/jni/include/org_opensearch_knn_jni_FaissService.h b/jni/include/org_opensearch_knn_jni_FaissService.h index 025fb12e8..19e13d402 100644 --- a/jni/include/org_opensearch_knn_jni_FaissService.h +++ b/jni/include/org_opensearch_knn_jni_FaissService.h @@ -18,23 +18,54 @@ #ifdef __cplusplus extern "C" { #endif - /* * Class: org_opensearch_knn_jni_FaissService - * Method: createIndex + * Method: initIndex * Signature: ([IJILjava/lang/String;Ljava/util/Map;)V */ -JNIEXPORT void JNICALL Java_org_opensearch_knn_jni_FaissService_createIndex - (JNIEnv *, jclass, jintArray, jlong, jint, jstring, jobject); - +JNIEXPORT jlong JNICALL Java_org_opensearch_knn_jni_FaissService_initIndex(JNIEnv * env, jclass cls, + jlong numDocs, jint dimJ, + jobject parametersJ); /* * Class: org_opensearch_knn_jni_FaissService - * Method: createBinaryIndex + * Method: initBinaryIndex * Signature: ([IJILjava/lang/String;Ljava/util/Map;)V */ -JNIEXPORT void JNICALL Java_org_opensearch_knn_jni_FaissService_createBinaryIndex - (JNIEnv *, jclass, jintArray, jlong, jint, jstring, jobject); - +JNIEXPORT jlong JNICALL Java_org_opensearch_knn_jni_FaissService_initBinaryIndex(JNIEnv * env, jclass cls, + jlong numDocs, jint dimJ, + jobject parametersJ); +/* + * Class: org_opensearch_knn_jni_FaissService + * Method: insertToIndex + * Signature: ([IJILjava/lang/String;Ljava/util/Map;)V + */ +JNIEXPORT void JNICALL Java_org_opensearch_knn_jni_FaissService_insertToIndex(JNIEnv * env, jclass cls, jintArray idsJ, + jlong vectorsAddressJ, jint dimJ, + jlong indexAddress, jint threadCount); +/* + * Class: org_opensearch_knn_jni_FaissService + * Method: insertToBinaryIndex + * Signature: ([IJILjava/lang/String;Ljava/util/Map;)V + */ +JNIEXPORT void JNICALL Java_org_opensearch_knn_jni_FaissService_insertToBinaryIndex(JNIEnv * env, jclass cls, jintArray idsJ, + jlong vectorsAddressJ, jint dimJ, + jlong indexAddress, jint threadCount); +/* + * Class: org_opensearch_knn_jni_FaissService + * Method: writeIndex + * Signature: ([IJILjava/lang/String;Ljava/util/Map;)V + */ +JNIEXPORT void JNICALL Java_org_opensearch_knn_jni_FaissService_writeIndex(JNIEnv * env, jclass cls, + jlong indexAddress, + jstring indexPathJ); +/* + * Class: org_opensearch_knn_jni_FaissService + * Method: writeBinaryIndex + * Signature: ([IJILjava/lang/String;Ljava/util/Map;)V + */ +JNIEXPORT void JNICALL Java_org_opensearch_knn_jni_FaissService_writeBinaryIndex(JNIEnv * env, jclass cls, + jlong indexAddress, + jstring indexPathJ); /* * Class: org_opensearch_knn_jni_FaissService * Method: createIndexFromTemplate diff --git a/jni/include/org_opensearch_knn_jni_JNICommons.h b/jni/include/org_opensearch_knn_jni_JNICommons.h index 89de76520..03c0d023a 100644 --- a/jni/include/org_opensearch_knn_jni_JNICommons.h +++ b/jni/include/org_opensearch_knn_jni_JNICommons.h @@ -21,10 +21,10 @@ extern "C" { /* * Class: org_opensearch_knn_jni_JNICommons * Method: storeVectorData - * Signature: (J[[FJJ) + * Signature: (J[[FJJJ) */ JNIEXPORT jlong JNICALL Java_org_opensearch_knn_jni_JNICommons_storeVectorData - (JNIEnv *, jclass, jlong, jobjectArray, jlong); + (JNIEnv *, jclass, jlong, jobjectArray, jlong, jboolean); /* * Class: org_opensearch_knn_jni_JNICommons @@ -32,7 +32,7 @@ JNIEXPORT jlong JNICALL Java_org_opensearch_knn_jni_JNICommons_storeVectorData * Signature: (J[[FJJ) */ JNIEXPORT jlong JNICALL Java_org_opensearch_knn_jni_JNICommons_storeByteVectorData - (JNIEnv *, jclass, jlong, jobjectArray, jlong); + (JNIEnv *, jclass, jlong, jobjectArray, jlong, jboolean); /* * Class: org_opensearch_knn_jni_JNICommons diff --git a/jni/src/commons.cpp b/jni/src/commons.cpp index 13f59194e..f9764db73 100644 --- a/jni/src/commons.cpp +++ b/jni/src/commons.cpp @@ -18,7 +18,7 @@ #include "commons.h" jlong knn_jni::commons::storeVectorData(knn_jni::JNIUtilInterface *jniUtil, JNIEnv *env, jlong memoryAddressJ, - jobjectArray dataJ, jlong initialCapacityJ) { + jobjectArray dataJ, jlong initialCapacityJ, jboolean appendJ) { std::vector *vect; if ((long) memoryAddressJ == 0) { vect = new std::vector(); @@ -26,6 +26,11 @@ jlong knn_jni::commons::storeVectorData(knn_jni::JNIUtilInterface *jniUtil, JNIE } else { vect = reinterpret_cast*>(memoryAddressJ); } + + if (appendJ == JNI_FALSE) { + vect->clear(); + } + int dim = jniUtil->GetInnerDimensionOf2dJavaFloatArray(env, dataJ); jniUtil->Convert2dJavaObjectArrayAndStoreToFloatVector(env, dataJ, dim, vect); @@ -33,7 +38,7 @@ jlong knn_jni::commons::storeVectorData(knn_jni::JNIUtilInterface *jniUtil, JNIE } jlong knn_jni::commons::storeByteVectorData(knn_jni::JNIUtilInterface *jniUtil, JNIEnv *env, jlong memoryAddressJ, - jobjectArray dataJ, jlong initialCapacityJ) { + jobjectArray dataJ, jlong initialCapacityJ, jboolean appendJ) { std::vector *vect; if ((long) memoryAddressJ == 0) { vect = new std::vector(); @@ -41,6 +46,11 @@ jlong knn_jni::commons::storeByteVectorData(knn_jni::JNIUtilInterface *jniUtil, } else { vect = reinterpret_cast*>(memoryAddressJ); } + + if (appendJ == JNI_FALSE) { + vect->clear(); + } + int dim = jniUtil->GetInnerDimensionOf2dJavaByteArray(env, dataJ); jniUtil->Convert2dJavaObjectArrayAndStoreToByteVector(env, dataJ, dim, vect); diff --git a/jni/src/faiss_index_service.cpp b/jni/src/faiss_index_service.cpp index 8c5ba36af..f76c54428 100644 --- a/jni/src/faiss_index_service.cpp +++ b/jni/src/faiss_index_service.cpp @@ -57,21 +57,69 @@ void SetExtraParameters(knn_jni::JNIUtilInterface * jniUtil, JNIEnv *env, IndexService::IndexService(std::unique_ptr faissMethods) : faissMethods(std::move(faissMethods)) {} -void IndexService::createIndex( +void IndexService::allocIndex(faiss::Index * index, size_t dim, size_t numVectors) { + if(auto * indexHNSWSQ = dynamic_cast(index)) { + if(auto * indexScalarQuantizer = dynamic_cast(indexHNSWSQ->storage)) { + indexScalarQuantizer->codes.reserve(indexScalarQuantizer->code_size * numVectors); + } + return; + } + if(auto * indexHNSW = dynamic_cast(index)) { + if(auto * indexFlat = dynamic_cast(indexHNSW->storage)) { + indexFlat->codes.reserve(indexFlat->code_size * numVectors); + } + return; + } +} + +jlong IndexService::initIndex( knn_jni::JNIUtilInterface * jniUtil, JNIEnv * env, faiss::MetricType metric, std::string indexDescription, + int dim, + int numVectors, + int threadCount, + std::unordered_map parameters + ) { + // Create index using Faiss factory method + std::unique_ptr index(faissMethods->indexFactory(dim, indexDescription.c_str(), metric)); + + // Set thread count if it is passed in as a parameter. Setting this variable will only impact the current thread + if(threadCount != 0) { + omp_set_num_threads(threadCount); + } + + // Add extra parameters that cant be configured with the index factory + SetExtraParameters(jniUtil, env, parameters, index.get()); + + // Check that the index does not need to be trained + if(!index->is_trained) { + throw std::runtime_error("Index is not trained"); + } + + std::unique_ptr idMap (faissMethods->indexIdMap(index.get())); + //Makes sure the index is deleted when the destructor is called, this cannot be passed in the constructor + idMap->own_fields = true; + + allocIndex(dynamic_cast(idMap->index), dim, numVectors); + + //Release the ownership so as to make sure not delete the underlying index that is created. The index is needed later + //in insert and write operations + index.release(); + return reinterpret_cast(idMap.release()); +} + +void IndexService::insertToIndex( int dim, int numIds, int threadCount, int64_t vectorsAddress, - std::vector ids, - std::string indexPath, - std::unordered_map parameters + std::vector & ids, + jlong idMapAddress ) { // Read vectors from memory address - auto *inputVectors = reinterpret_cast*>(vectorsAddress); + std::vector * inputVectors = reinterpret_cast*>(vectorsAddress); // The number of vectors can be int here because a lucene segment number of total docs never crosses INT_MAX value int numVectors = (int) (inputVectors->size() / (uint64_t) dim); @@ -83,50 +131,89 @@ void IndexService::createIndex( throw std::runtime_error("Number of IDs does not match number of vectors"); } - std::unique_ptr indexWriter(faissMethods->indexFactory(dim, indexDescription.c_str(), metric)); - // Set thread count if it is passed in as a parameter. Setting this variable will only impact the current thread if(threadCount != 0) { omp_set_num_threads(threadCount); } - // Add extra parameters that cant be configured with the index factory - SetExtraParameters(jniUtil, env, parameters, indexWriter.get()); - - // Check that the index does not need to be trained - if(!indexWriter->is_trained) { - throw std::runtime_error("Index is not trained"); - } + faiss::IndexIDMap * idMap = reinterpret_cast (idMapAddress); // Add vectors - std::unique_ptr idMap(faissMethods->indexIdMap(indexWriter.get())); idMap->add_with_ids(numVectors, inputVectors->data(), ids.data()); +} - // Write the index to disk - faissMethods->writeIndex(idMap.get(), indexPath.c_str()); +void IndexService::writeIndex( + std::string indexPath, + jlong idMapAddress + ) { + std::unique_ptr idMap (reinterpret_cast (idMapAddress)); + + try { + // Write the index to disk + faissMethods->writeIndex(idMap.get(), indexPath.c_str()); + } catch(std::exception &e) { + throw std::runtime_error("Failed to write index to disk"); + } } BinaryIndexService::BinaryIndexService(std::unique_ptr faissMethods) : IndexService(std::move(faissMethods)) {} -void BinaryIndexService::createIndex( +void BinaryIndexService::allocIndex(faiss::Index * index, size_t dim, size_t numVectors) { + if(auto * indexBinaryHNSW = dynamic_cast(index)) { + auto * indexBinaryFlat = dynamic_cast(indexBinaryHNSW->storage); + indexBinaryFlat->xb.reserve(dim * numVectors / 8); + return; + } +} + +jlong BinaryIndexService::initIndex( knn_jni::JNIUtilInterface * jniUtil, JNIEnv * env, faiss::MetricType metric, std::string indexDescription, int dim, - int numIds, + int numVectors, int threadCount, - int64_t vectorsAddress, - std::vector ids, - std::string indexPath, std::unordered_map parameters ) { - // Read vectors from memory address - auto *inputVectors = reinterpret_cast*>(vectorsAddress); + // Create index using Faiss factory method + std::unique_ptr index(faissMethods->indexBinaryFactory(dim, indexDescription.c_str())); + // Set thread count if it is passed in as a parameter. Setting this variable will only impact the current thread + if(threadCount != 0) { + omp_set_num_threads(threadCount); + } + + // Add extra parameters that cant be configured with the index factory + SetExtraParameters(jniUtil, env, parameters, index.get()); - if (dim % 8 != 0) { - throw std::runtime_error("Dimensions should be multiply of 8"); + // Check that the index does not need to be trained + if(!index->is_trained) { + throw std::runtime_error("Index is not trained"); } + + std::unique_ptr idMap(faissMethods->indexBinaryIdMap(index.get())); + //Makes sure the index is deleted when the destructor is called + idMap->own_fields = true; + + allocIndex(dynamic_cast(idMap->index), dim, numVectors); + + //Release the ownership so as to make sure not delete the underlying index that is created. The index is needed later + //in insert and write operations + index.release(); + return reinterpret_cast(idMap.release()); +} + +void BinaryIndexService::insertToIndex( + int dim, + int numIds, + int threadCount, + int64_t vectorsAddress, + std::vector & ids, + jlong idMapAddress + ) { + // Read vectors from memory address (unique ptr since we want to remove from memory after use) + std::vector * inputVectors = reinterpret_cast*>(vectorsAddress); + // The number of vectors can be int here because a lucene segment number of total docs never crosses INT_MAX value int numVectors = (int) (inputVectors->size() / (uint64_t) (dim / 8)); if(numVectors == 0) { @@ -137,28 +224,31 @@ void BinaryIndexService::createIndex( throw std::runtime_error("Number of IDs does not match number of vectors"); } - std::unique_ptr indexWriter(faissMethods->indexBinaryFactory(dim, indexDescription.c_str())); - // Set thread count if it is passed in as a parameter. Setting this variable will only impact the current thread if(threadCount != 0) { omp_set_num_threads(threadCount); } - // Add extra parameters that cant be configured with the index factory - SetExtraParameters(jniUtil, env, parameters, indexWriter.get()); - - // Check that the index does not need to be trained - if(!indexWriter->is_trained) { - throw std::runtime_error("Index is not trained"); - } + faiss::IndexBinaryIDMap * idMap = reinterpret_cast (idMapAddress); // Add vectors - std::unique_ptr idMap(faissMethods->indexBinaryIdMap(indexWriter.get())); idMap->add_with_ids(numVectors, inputVectors->data(), ids.data()); +} - // Write the index to disk - faissMethods->writeIndexBinary(idMap.get(), indexPath.c_str()); +void BinaryIndexService::writeIndex( + std::string indexPath, + jlong idMapAddress + ) { + + std::unique_ptr idMap (reinterpret_cast (idMapAddress)); + + try { + // Write the index to disk + faissMethods->writeIndexBinary(idMap.get(), indexPath.c_str()); + } catch(std::exception &e) { + throw std::runtime_error("Failed to write index to disk"); + } } } // namespace faiss_wrapper -} // namesapce knn_jni +} // namesapce knn_jni \ No newline at end of file diff --git a/jni/src/faiss_wrapper.cpp b/jni/src/faiss_wrapper.cpp index 1d4437414..0e1029ecf 100644 --- a/jni/src/faiss_wrapper.cpp +++ b/jni/src/faiss_wrapper.cpp @@ -88,24 +88,13 @@ bool isIndexIVFPQL2(faiss::Index * index); // IndexIDMap which has member that will point to underlying index that stores the data faiss::IndexIVFPQ * extractIVFPQIndex(faiss::Index * index); -void knn_jni::faiss_wrapper::CreateIndex(knn_jni::JNIUtilInterface * jniUtil, JNIEnv * env, jintArray idsJ, jlong vectorsAddressJ, jint dimJ, - jstring indexPathJ, jobject parametersJ, IndexService* indexService) { - if (idsJ == nullptr) { - throw std::runtime_error("IDs cannot be null"); - } - - if (vectorsAddressJ <= 0) { - throw std::runtime_error("VectorsAddress cannot be less than 0"); - } +jlong knn_jni::faiss_wrapper::InitIndex(knn_jni::JNIUtilInterface * jniUtil, JNIEnv * env, jlong numDocs, jint dimJ, + jobject parametersJ, IndexService* indexService) { if(dimJ <= 0) { throw std::runtime_error("Vectors dimensions cannot be less than or equal to 0"); } - if (indexPathJ == nullptr) { - throw std::runtime_error("Index path cannot be null"); - } - if (parametersJ == nullptr) { throw std::runtime_error("Parameters cannot be null"); } @@ -124,8 +113,8 @@ void knn_jni::faiss_wrapper::CreateIndex(knn_jni::JNIUtilInterface * jniUtil, JN // Dimension int dim = (int)dimJ; - // Number of vectors - int numIds = jniUtil->GetJavaIntArrayLength(env, idsJ); + // Number of docs + int docs = (int)numDocs; // Index description jobject indexDescriptionJ = knn_jni::GetJObjectFromMapOrThrow(parametersCpp, knn_jni::INDEX_DESCRIPTION); @@ -138,25 +127,60 @@ void knn_jni::faiss_wrapper::CreateIndex(knn_jni::JNIUtilInterface * jniUtil, JN threadCount = jniUtil->ConvertJavaObjectToCppInteger(env, parametersCpp[knn_jni::INDEX_THREAD_QUANTITY]); } + // Extra parameters + // TODO: parse the entire map and remove jni object + std::unordered_map subParametersCpp; + if(parametersCpp.find(knn_jni::PARAMETERS) != parametersCpp.end()) { + subParametersCpp = jniUtil->ConvertJavaMapToCppMap(env, parametersCpp[knn_jni::PARAMETERS]); + } + // end parameters to pass + + // Create index + return indexService->initIndex(jniUtil, env, metric, indexDescriptionCpp, dim, numDocs, threadCount, subParametersCpp); +} + +void knn_jni::faiss_wrapper::InsertToIndex(knn_jni::JNIUtilInterface * jniUtil, JNIEnv * env, jintArray idsJ, jlong vectorsAddressJ, jint dimJ, + jlong index_ptr, jint threadCount, IndexService* indexService) { + if (idsJ == nullptr) { + throw std::runtime_error("IDs cannot be null"); + } + + if (vectorsAddressJ <= 0) { + throw std::runtime_error("VectorsAddress cannot be less than 0"); + } + + if(dimJ <= 0) { + throw std::runtime_error("Vectors dimensions cannot be less than or equal to 0"); + } + + // Dimension + int dim = (int)dimJ; + + // Number of vectors + int numIds = jniUtil->GetJavaIntArrayLength(env, idsJ); + // Vectors address int64_t vectorsAddress = (int64_t)vectorsAddressJ; // Ids auto ids = jniUtil->ConvertJavaIntArrayToCppIntVector(env, idsJ); - // Index path - std::string indexPathCpp(jniUtil->ConvertJavaStringToCppString(env, indexPathJ)); + // Create index + indexService->insertToIndex(dim, numIds, threadCount, vectorsAddress, ids, index_ptr); +} - // Extra parameters - // TODO: parse the entire map and remove jni object - std::unordered_map subParametersCpp; - if(parametersCpp.find(knn_jni::PARAMETERS) != parametersCpp.end()) { - subParametersCpp = jniUtil->ConvertJavaMapToCppMap(env, parametersCpp[knn_jni::PARAMETERS]); +void knn_jni::faiss_wrapper::WriteIndex(knn_jni::JNIUtilInterface * jniUtil, JNIEnv * env, + jstring indexPathJ, jlong index_ptr, IndexService* indexService) { + + if (indexPathJ == nullptr) { + throw std::runtime_error("Index path cannot be null"); } - // end parameters to pass + + // Index path + std::string indexPathCpp(jniUtil->ConvertJavaStringToCppString(env, indexPathJ)); // Create index - indexService->createIndex(jniUtil, env, metric, indexDescriptionCpp, dim, numIds, threadCount, vectorsAddress, ids, indexPathCpp, subParametersCpp); + indexService->writeIndex(indexPathCpp, index_ptr); } void knn_jni::faiss_wrapper::CreateIndexFromTemplate(knn_jni::JNIUtilInterface * jniUtil, JNIEnv * env, jintArray idsJ, diff --git a/jni/src/org_opensearch_knn_jni_FaissService.cpp b/jni/src/org_opensearch_knn_jni_FaissService.cpp index 2394e2951..663e18457 100644 --- a/jni/src/org_opensearch_knn_jni_FaissService.cpp +++ b/jni/src/org_opensearch_knn_jni_FaissService.cpp @@ -39,37 +39,83 @@ void JNI_OnUnload(JavaVM *vm, void *reserved) { jniUtil.Uninitialize(env); } -JNIEXPORT void JNICALL Java_org_opensearch_knn_jni_FaissService_createIndex(JNIEnv * env, jclass cls, jintArray idsJ, - jlong vectorsAddressJ, jint dimJ, - jstring indexPathJ, jobject parametersJ) +JNIEXPORT jlong JNICALL Java_org_opensearch_knn_jni_FaissService_initIndex(JNIEnv * env, jclass cls, + jlong numDocs, jint dimJ, + jobject parametersJ) { try { std::unique_ptr faissMethods(new knn_jni::faiss_wrapper::FaissMethods()); knn_jni::faiss_wrapper::IndexService indexService(std::move(faissMethods)); - knn_jni::faiss_wrapper::CreateIndex(&jniUtil, env, idsJ, vectorsAddressJ, dimJ, indexPathJ, parametersJ, &indexService); + return knn_jni::faiss_wrapper::InitIndex(&jniUtil, env, numDocs, dimJ, parametersJ, &indexService); + } catch (...) { + jniUtil.CatchCppExceptionAndThrowJava(env); + } + return (jlong)0; +} + +JNIEXPORT jlong JNICALL Java_org_opensearch_knn_jni_FaissService_initBinaryIndex(JNIEnv * env, jclass cls, + jlong numDocs, jint dimJ, + jobject parametersJ) +{ + try { + std::unique_ptr faissMethods(new knn_jni::faiss_wrapper::FaissMethods()); + knn_jni::faiss_wrapper::BinaryIndexService binaryIndexService(std::move(faissMethods)); + return knn_jni::faiss_wrapper::InitIndex(&jniUtil, env, numDocs, dimJ, parametersJ, &binaryIndexService); + } catch (...) { + jniUtil.CatchCppExceptionAndThrowJava(env); + } + return (jlong)0; +} - // Releasing the vectorsAddressJ memory as that is not required once we have created the index. - // This is not the ideal approach, please refer this gh issue for long term solution: - // https://github.com/opensearch-project/k-NN/issues/1600 - delete reinterpret_cast*>(vectorsAddressJ); +JNIEXPORT void JNICALL Java_org_opensearch_knn_jni_FaissService_insertToIndex(JNIEnv * env, jclass cls, jintArray idsJ, + jlong vectorsAddressJ, jint dimJ, + jlong indexAddress, jint threadCount) +{ + try { + std::unique_ptr faissMethods(new knn_jni::faiss_wrapper::FaissMethods()); + knn_jni::faiss_wrapper::IndexService indexService(std::move(faissMethods)); + knn_jni::faiss_wrapper::InsertToIndex(&jniUtil, env, idsJ, vectorsAddressJ, dimJ, indexAddress, threadCount, &indexService); } catch (...) { + // NOTE: ADDING DELETE STATEMENT HERE CAUSES A CRASH! jniUtil.CatchCppExceptionAndThrowJava(env); } } -JNIEXPORT void JNICALL Java_org_opensearch_knn_jni_FaissService_createBinaryIndex(JNIEnv * env, jclass cls, jintArray idsJ, +JNIEXPORT void JNICALL Java_org_opensearch_knn_jni_FaissService_insertToBinaryIndex(JNIEnv * env, jclass cls, jintArray idsJ, jlong vectorsAddressJ, jint dimJ, - jstring indexPathJ, jobject parametersJ) + jlong indexAddress, jint threadCount) { try { std::unique_ptr faissMethods(new knn_jni::faiss_wrapper::FaissMethods()); knn_jni::faiss_wrapper::BinaryIndexService binaryIndexService(std::move(faissMethods)); - knn_jni::faiss_wrapper::CreateIndex(&jniUtil, env, idsJ, vectorsAddressJ, dimJ, indexPathJ, parametersJ, &binaryIndexService); + knn_jni::faiss_wrapper::InsertToIndex(&jniUtil, env, idsJ, vectorsAddressJ, dimJ, indexAddress, threadCount, &binaryIndexService); + } catch (...) { + // NOTE: ADDING DELETE STATEMENT HERE CAUSES A CRASH! + jniUtil.CatchCppExceptionAndThrowJava(env); + } +} + +JNIEXPORT void JNICALL Java_org_opensearch_knn_jni_FaissService_writeIndex(JNIEnv * env, jclass cls, + jlong indexAddress, + jstring indexPathJ) +{ + try { + std::unique_ptr faissMethods(new knn_jni::faiss_wrapper::FaissMethods()); + knn_jni::faiss_wrapper::IndexService indexService(std::move(faissMethods)); + knn_jni::faiss_wrapper::WriteIndex(&jniUtil, env, indexPathJ, indexAddress, &indexService); + } catch (...) { + jniUtil.CatchCppExceptionAndThrowJava(env); + } +} - // Releasing the vectorsAddressJ memory as that is not required once we have created the index. - // This is not the ideal approach, please refer this gh issue for long term solution: - // https://github.com/opensearch-project/k-NN/issues/1600 - delete reinterpret_cast*>(vectorsAddressJ); +JNIEXPORT void JNICALL Java_org_opensearch_knn_jni_FaissService_writeBinaryIndex(JNIEnv * env, jclass cls, + jlong indexAddress, + jstring indexPathJ) +{ + try { + std::unique_ptr faissMethods(new knn_jni::faiss_wrapper::FaissMethods()); + knn_jni::faiss_wrapper::BinaryIndexService binaryIndexService(std::move(faissMethods)); + knn_jni::faiss_wrapper::WriteIndex(&jniUtil, env, indexPathJ, indexAddress, &binaryIndexService); } catch (...) { jniUtil.CatchCppExceptionAndThrowJava(env); } diff --git a/jni/src/org_opensearch_knn_jni_JNICommons.cpp b/jni/src/org_opensearch_knn_jni_JNICommons.cpp index 0bc2e4633..7432c44d3 100644 --- a/jni/src/org_opensearch_knn_jni_JNICommons.cpp +++ b/jni/src/org_opensearch_knn_jni_JNICommons.cpp @@ -38,11 +38,11 @@ void JNI_OnUnload(JavaVM *vm, void *reserved) { JNIEXPORT jlong JNICALL Java_org_opensearch_knn_jni_JNICommons_storeVectorData(JNIEnv * env, jclass cls, -jlong memoryAddressJ, jobjectArray dataJ, jlong initialCapacityJ) +jlong memoryAddressJ, jobjectArray dataJ, jlong initialCapacityJ, jboolean appendJ) { try { - return knn_jni::commons::storeVectorData(&jniUtil, env, memoryAddressJ, dataJ, initialCapacityJ); + return knn_jni::commons::storeVectorData(&jniUtil, env, memoryAddressJ, dataJ, initialCapacityJ, appendJ); } catch (...) { jniUtil.CatchCppExceptionAndThrowJava(env); } @@ -50,11 +50,11 @@ jlong memoryAddressJ, jobjectArray dataJ, jlong initialCapacityJ) } JNIEXPORT jlong JNICALL Java_org_opensearch_knn_jni_JNICommons_storeByteVectorData(JNIEnv * env, jclass cls, -jlong memoryAddressJ, jobjectArray dataJ, jlong initialCapacityJ) +jlong memoryAddressJ, jobjectArray dataJ, jlong initialCapacityJ, jboolean appendJ) { try { - return knn_jni::commons::storeByteVectorData(&jniUtil, env, memoryAddressJ, dataJ, initialCapacityJ); + return knn_jni::commons::storeByteVectorData(&jniUtil, env, memoryAddressJ, dataJ, initialCapacityJ, appendJ); } catch (...) { jniUtil.CatchCppExceptionAndThrowJava(env); } diff --git a/jni/tests/commons_test.cpp b/jni/tests/commons_test.cpp index 630358919..d469fe268 100644 --- a/jni/tests/commons_test.cpp +++ b/jni/tests/commons_test.cpp @@ -33,7 +33,7 @@ TEST(CommonsTests, BasicAssertions) { testing::NiceMock mockJNIUtil; jlong memoryAddress = knn_jni::commons::storeVectorData(&mockJNIUtil, jniEnv, (jlong)0, - reinterpret_cast(&data), (jlong)(totalNumberOfVector * dim)); + reinterpret_cast(&data), (jlong)(totalNumberOfVector * dim), true); ASSERT_NE(memoryAddress, 0); auto *vect = reinterpret_cast*>(memoryAddress); ASSERT_EQ(vect->size(), data.size() * dim); @@ -48,12 +48,13 @@ TEST(CommonsTests, BasicAssertions) { } data2.push_back(vector); memoryAddress = knn_jni::commons::storeVectorData(&mockJNIUtil, jniEnv, memoryAddress, - reinterpret_cast(&data2), (jlong)(totalNumberOfVector * dim)); + reinterpret_cast(&data2), (jlong)(totalNumberOfVector * dim), true); ASSERT_NE(memoryAddress, 0); ASSERT_EQ(memoryAddress, oldMemoryAddress); vect = reinterpret_cast*>(memoryAddress); int currentIndex = 0; - ASSERT_EQ(vect->size(), totalNumberOfVector*dim); + std::cout << vect->size() + "\n"; + ASSERT_EQ(vect->size(), totalNumberOfVector * dim); ASSERT_EQ(vect->capacity(), totalNumberOfVector * dim); // Validate if all vectors data are at correct location @@ -70,6 +71,115 @@ TEST(CommonsTests, BasicAssertions) { currentIndex++; } } + + // test append == true + std::vector> data3; + std::vector vecto3; + for(int j = 0 ; j < dim ; j ++) { + vecto3.push_back((float)j); + } + data3.push_back(vecto3); + memoryAddress = knn_jni::commons::storeVectorData(&mockJNIUtil, jniEnv, memoryAddress, + reinterpret_cast(&data3), (jlong)(totalNumberOfVector * dim), false); + ASSERT_NE(memoryAddress, 0); + ASSERT_EQ(memoryAddress, oldMemoryAddress); + vect = reinterpret_cast*>(memoryAddress); + + ASSERT_EQ(vect->size(), dim); //Since we just added 1 vector + ASSERT_EQ(vect->capacity(), totalNumberOfVector * dim); //This is the initial capacity allocated + + currentIndex = 0; + for(auto & i : data3) { + for(float j : i) { + ASSERT_FLOAT_EQ(vect->at(currentIndex), j); + currentIndex++; + } + } + + // Check that freeing vector data works + knn_jni::commons::freeVectorData(memoryAddress); +} + +TEST(StoreByteVectorTest, BasicAssertions) { + long dim = 3; + long totalNumberOfVector = 5; + std::vector> data; + for(int i = 0 ; i < totalNumberOfVector - 1 ; i++) { + std::vector vector; + for(int j = 0 ; j < dim ; j ++) { + vector.push_back((uint8_t)j); + } + data.push_back(vector); + } + JNIEnv *jniEnv = nullptr; + + testing::NiceMock mockJNIUtil; + + jlong memoryAddress = knn_jni::commons::storeByteVectorData(&mockJNIUtil, jniEnv, (jlong)0, + reinterpret_cast(&data), (jlong)(totalNumberOfVector * dim), true); + ASSERT_NE(memoryAddress, 0); + auto *vect = reinterpret_cast*>(memoryAddress); + ASSERT_EQ(vect->size(), data.size() * dim); + ASSERT_EQ(vect->capacity(), totalNumberOfVector * dim); + + // Check by inserting more vectors at same memory location + jlong oldMemoryAddress = memoryAddress; + std::vector> data2; + std::vector vector; + for(int j = 0 ; j < dim ; j ++) { + vector.push_back((uint8_t)j); + } + data2.push_back(vector); + memoryAddress = knn_jni::commons::storeByteVectorData(&mockJNIUtil, jniEnv, memoryAddress, + reinterpret_cast(&data2), (jlong)(totalNumberOfVector * dim), true); + ASSERT_NE(memoryAddress, 0); + ASSERT_EQ(memoryAddress, oldMemoryAddress); + vect = reinterpret_cast*>(memoryAddress); + int currentIndex = 0; + ASSERT_EQ(vect->size(), totalNumberOfVector*dim); + ASSERT_EQ(vect->capacity(), totalNumberOfVector * dim); + + // Validate if all vectors data are at correct location + for(auto & i : data) { + for(uint8_t j : i) { + ASSERT_EQ(vect->at(currentIndex), j); + currentIndex++; + } + } + + for(auto & i : data2) { + for(uint8_t j : i) { + ASSERT_EQ(vect->at(currentIndex), j); + currentIndex++; + } + } + + // test append == true + std::vector> data3; + std::vector vecto3; + for(int j = 0 ; j < dim ; j ++) { + vecto3.push_back((uint8_t)j); + } + data3.push_back(vecto3); + memoryAddress = knn_jni::commons::storeByteVectorData(&mockJNIUtil, jniEnv, memoryAddress, + reinterpret_cast(&data3), (jlong)(totalNumberOfVector * dim), false); + ASSERT_NE(memoryAddress, 0); + ASSERT_EQ(memoryAddress, oldMemoryAddress); + vect = reinterpret_cast*>(memoryAddress); + + ASSERT_EQ(vect->size(), dim); + ASSERT_EQ(vect->capacity(), totalNumberOfVector * dim); + + currentIndex = 0; + for(auto & i : data3) { + for(uint8_t j : i) { + ASSERT_EQ(vect->at(currentIndex), j); + currentIndex++; + } + } + + // Check that freeing vector data works + knn_jni::commons::freeVectorData(memoryAddress); } TEST(CommonTests, GetIntegerMethodParam) { diff --git a/jni/tests/faiss_index_service_test.cpp b/jni/tests/faiss_index_service_test.cpp index f876edced..1f00f6a1d 100644 --- a/jni/tests/faiss_index_service_test.cpp +++ b/jni/tests/faiss_index_service_test.cpp @@ -64,18 +64,9 @@ TEST(CreateIndexTest, BasicAssertions) { // Create the index knn_jni::faiss_wrapper::IndexService indexService(std::move(mockFaissMethods)); - indexService.createIndex( - &mockJNIUtil, - jniEnv, - metricType, - indexDescription, - dim, - numIds, - threadCount, - (int64_t) &vectors, - ids, - indexPath, - parametersMap); + long indexAddress = indexService.initIndex(&mockJNIUtil, jniEnv, metricType, indexDescription, dim, numIds, threadCount, parametersMap); + indexService.insertToIndex(dim, numIds, threadCount, (int64_t) &vectors, ids, indexAddress); + indexService.writeIndex(indexPath, indexAddress); } TEST(CreateBinaryIndexTest, BasicAssertions) { @@ -119,16 +110,7 @@ TEST(CreateBinaryIndexTest, BasicAssertions) { // Create the index knn_jni::faiss_wrapper::BinaryIndexService indexService(std::move(mockFaissMethods)); - indexService.createIndex( - &mockJNIUtil, - jniEnv, - metricType, - indexDescription, - dim, - numIds, - threadCount, - (int64_t) &vectors, - ids, - indexPath, - parametersMap); + long indexAddress = indexService.initIndex(&mockJNIUtil, jniEnv, metricType, indexDescription, dim, numIds, threadCount, parametersMap); + indexService.insertToIndex(dim, numIds, threadCount, (int64_t) &vectors, ids, indexAddress); + indexService.writeIndex(indexPath, indexAddress); } \ No newline at end of file diff --git a/jni/tests/faiss_wrapper_test.cpp b/jni/tests/faiss_wrapper_test.cpp index 5ae443837..a1839c6ce 100644 --- a/jni/tests/faiss_wrapper_test.cpp +++ b/jni/tests/faiss_wrapper_test.cpp @@ -32,6 +32,70 @@ float rangeSearchRandomDataMin = -50; float rangeSearchRandomDataMax = 50; float rangeSearchRadius = 20000; +void createIndexIteratively( + knn_jni::JNIUtilInterface * JNIUtil, + JNIEnv *jniEnv, + std::vector & ids, + std::vector & vectors, + int dim, + std::string & indexPath, + std::unordered_map parametersMap, + IndexService * indexService, + int insertions = 10 + ) { + long numDocs = ids.size(); + if(numDocs % insertions != 0) { + throw std::invalid_argument("Number of documents should be divisible by number of insertions"); + } + long docsPerInsertion = numDocs / insertions; + long index_ptr = knn_jni::faiss_wrapper::InitIndex(JNIUtil, jniEnv, numDocs, dim, (jobject)¶metersMap, indexService); + for(int i = 0; i < insertions; i++) { + int start_idx = i * docsPerInsertion; + int end_idx = start_idx + docsPerInsertion; + std::vector insertIds; + std::vector insertVecs; + for(int j = start_idx; j < end_idx; j++) { + insertIds.push_back(j); + for(int k = 0; k < dim; k++) { + insertVecs.push_back(vectors[j * dim + k]); + } + } + knn_jni::faiss_wrapper::InsertToIndex(JNIUtil, jniEnv, reinterpret_cast(&insertIds), (jlong)&insertVecs, dim, index_ptr, 0, indexService); + } + knn_jni::faiss_wrapper::WriteIndex(JNIUtil, jniEnv, (jstring)&indexPath, index_ptr, indexService); +} + +void createBinaryIndexIteratively( + knn_jni::JNIUtilInterface * JNIUtil, + JNIEnv *jniEnv, + std::vector & ids, + std::vector & vectors, + int dim, + std::string & indexPath, + std::unordered_map parametersMap, + IndexService * indexService, + int insertions = 10 + ) { + long numDocs = ids.size();; + long index_ptr = knn_jni::faiss_wrapper::InitIndex(JNIUtil, jniEnv, numDocs, dim, (jobject)¶metersMap, indexService); + for(int i = 0; i < insertions; i++) { + int start_idx = numDocs * i / insertions; + int end_idx = numDocs * (i + 1) / insertions; + int docs_to_insert = end_idx - start_idx; + if(docs_to_insert == 0) continue; + std::vector insertIds; + std::vector insertVecs; + for(int j = start_idx; j < end_idx; j++) { + insertIds.push_back(j); + for(int k = 0; k < dim / 8; k++) { + insertVecs.push_back(vectors[j * (dim / 8) + k]); + } + } + knn_jni::faiss_wrapper::InsertToIndex(JNIUtil, jniEnv, reinterpret_cast(&insertIds), (jlong)&insertVecs, dim, index_ptr, 0, indexService); + } + knn_jni::faiss_wrapper::WriteIndex(JNIUtil, jniEnv, (jstring)&indexPath, index_ptr, indexService); +} + TEST(FaissCreateIndexTest, BasicAssertions) { // Define the data faiss::idx_t numIds = 200; @@ -63,13 +127,15 @@ TEST(FaissCreateIndexTest, BasicAssertions) { // Create the index std::unique_ptr faissMethods(new FaissMethods()); NiceMock mockIndexService(std::move(faissMethods)); - EXPECT_CALL(mockIndexService, createIndex(_, _, faiss::METRIC_L2, indexDescription, dim, (int)numIds, 0, (int64_t)&vectors, ids, indexPath, subParametersMap)) + int insertions = 10; + EXPECT_CALL(mockIndexService, initIndex(_, _, faiss::METRIC_L2, indexDescription, dim, (int)numIds, 0, subParametersMap)) + .Times(1); + EXPECT_CALL(mockIndexService, insertToIndex(dim, numIds / insertions, 0, _, _, _)) + .Times(insertions); + EXPECT_CALL(mockIndexService, writeIndex(indexPath, _)) .Times(1); - knn_jni::faiss_wrapper::CreateIndex( - &mockJNIUtil, jniEnv, reinterpret_cast(&ids), - (jlong) &vectors, dim , (jstring)&indexPath, - (jobject)¶metersMap, &mockIndexService); + createIndexIteratively(&mockJNIUtil, jniEnv, ids, vectors, dim, indexPath, parametersMap, &mockIndexService, insertions); } TEST(FaissCreateBinaryIndexTest, BasicAssertions) { @@ -103,14 +169,16 @@ TEST(FaissCreateBinaryIndexTest, BasicAssertions) { // Create the index std::unique_ptr faissMethods(new FaissMethods()); NiceMock mockIndexService(std::move(faissMethods)); - EXPECT_CALL(mockIndexService, createIndex(_, _, faiss::METRIC_L2, indexDescription, dim, (int)numIds, 0, (int64_t)&vectors, ids, indexPath, subParametersMap)) + int insertions = 10; + EXPECT_CALL(mockIndexService, initIndex(_, _, faiss::METRIC_L2, indexDescription, dim, (int)numIds, 0, subParametersMap)) + .Times(1); + EXPECT_CALL(mockIndexService, insertToIndex(dim, numIds / insertions, 0, _, _, _)) + .Times(insertions); + EXPECT_CALL(mockIndexService, writeIndex(indexPath, _)) .Times(1); // This method calls delete vectors at the end - knn_jni::faiss_wrapper::CreateIndex( - &mockJNIUtil, jniEnv, reinterpret_cast(&ids), - (jlong) &vectors, dim , (jstring)&indexPath, - (jobject)¶metersMap, &mockIndexService); + createBinaryIndexIteratively(&mockJNIUtil, jniEnv, ids, vectors, dim, indexPath, parametersMap, &mockIndexService, insertions); } TEST(FaissCreateIndexFromTemplateTest, BasicAssertions) { @@ -683,10 +751,8 @@ TEST(FaissCreateHnswSQfp16IndexTest, BasicAssertions) { // Create the index std::unique_ptr faissMethods(new FaissMethods()); knn_jni::faiss_wrapper::IndexService IndexService(std::move(faissMethods)); - knn_jni::faiss_wrapper::CreateIndex( - &mockJNIUtil, jniEnv, reinterpret_cast(&ids), - (jlong)&vectors, dim, (jstring)&indexPath, - (jobject)¶metersMap, &IndexService); + + createIndexIteratively(&mockJNIUtil, jniEnv, ids, vectors, dim, indexPath, parametersMap, &IndexService); // Make sure index can be loaded std::unique_ptr index(test_util::FaissLoadIndex(indexPath)); diff --git a/jni/tests/mocks/faiss_index_service_mock.h b/jni/tests/mocks/faiss_index_service_mock.h index 7af08c82e..285e34053 100644 --- a/jni/tests/mocks/faiss_index_service_mock.h +++ b/jni/tests/mocks/faiss_index_service_mock.h @@ -23,20 +23,37 @@ class MockIndexService : public IndexService { public: MockIndexService(std::unique_ptr faissMethods) : IndexService(std::move(faissMethods)) {}; MOCK_METHOD( - void, - createIndex, + long, + initIndex, ( knn_jni::JNIUtilInterface * jniUtil, JNIEnv * env, faiss::MetricType metric, std::string indexDescription, + int dim, + int numIds, + int threadCount, + StringToJObjectMap parameters + ), + (override)); + MOCK_METHOD( + void, + insertToIndex, + ( int dim, int numIds, int threadCount, int64_t vectorsAddress, - std::vector ids, + std::vector & ids, + long indexPtr + ), + (override)); + MOCK_METHOD( + void, + writeIndex, + ( std::string indexPath, - StringToJObjectMap parameters + long indexPtr ), (override)); }; diff --git a/src/main/java/org/opensearch/knn/common/FieldInfoExtractor.java b/src/main/java/org/opensearch/knn/common/FieldInfoExtractor.java index cf866d840..4ded68237 100644 --- a/src/main/java/org/opensearch/knn/common/FieldInfoExtractor.java +++ b/src/main/java/org/opensearch/knn/common/FieldInfoExtractor.java @@ -10,9 +10,12 @@ import org.apache.lucene.index.FieldInfo; import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.VectorDataType; +import org.opensearch.knn.index.engine.KNNEngine; import org.opensearch.knn.indices.ModelMetadata; import org.opensearch.knn.indices.ModelUtil; +import static org.opensearch.knn.common.KNNConstants.MODEL_ID; +import static org.opensearch.knn.indices.ModelUtil.getModelMetadata; import org.opensearch.knn.index.engine.qframe.QuantizationConfig; import org.opensearch.knn.index.engine.qframe.QuantizationConfigParser; @@ -31,7 +34,21 @@ public class FieldInfoExtractor { /** - * Extract vector data type from fieldInfo + * Extracts KNNEngine from FieldInfo + * @param field {@link FieldInfo} + * @return {@link KNNEngine} + */ + public static KNNEngine extractKNNEngine(final FieldInfo field) { + final ModelMetadata modelMetadata = getModelMetadata(field.attributes().get(MODEL_ID)); + if (modelMetadata != null) { + return modelMetadata.getKnnEngine(); + } + final String engineName = field.attributes().getOrDefault(KNNConstants.KNN_ENGINE, KNNEngine.DEFAULT.getName()); + return KNNEngine.getEngine(engineName); + } + + /** + * Extracts VectorDataType from FieldInfo * @param fieldInfo {@link FieldInfo} * @return {@link VectorDataType} */ diff --git a/src/main/java/org/opensearch/knn/common/KNNVectorUtil.java b/src/main/java/org/opensearch/knn/common/KNNVectorUtil.java index fd9e5b6c2..778cc164d 100644 --- a/src/main/java/org/opensearch/knn/common/KNNVectorUtil.java +++ b/src/main/java/org/opensearch/knn/common/KNNVectorUtil.java @@ -5,9 +5,13 @@ package org.opensearch.knn.common; -import java.util.Objects; import lombok.AccessLevel; import lombok.NoArgsConstructor; +import org.opensearch.knn.index.vectorvalues.KNNVectorValues; + +import java.io.IOException; +import java.util.List; +import java.util.Objects; @NoArgsConstructor(access = AccessLevel.PRIVATE) public class KNNVectorUtil { @@ -42,4 +46,33 @@ public static boolean isZeroVector(float[] vector) { } return true; } + + /** + * Converts an integer List to and array + * @param integerList + * @return null if list is null or empty, int[] otherwise + */ + public static int[] intListToArray(final List integerList) { + if (integerList == null || integerList.isEmpty()) { + return null; + } + int[] intArray = new int[integerList.size()]; + for (int i = 0; i < integerList.size(); i++) { + intArray[i] = integerList.get(i); + } + return intArray; + } + + /** + * Iterates vector values once if it is not at start of the location, + * Intended to be done to make sure dimension and bytesPerVector are available + * @param vectorValues + * @throws IOException + */ + public static void iterateVectorValuesOnce(final KNNVectorValues vectorValues) throws IOException { + if (vectorValues.docId() == -1) { + vectorValues.nextDoc(); + vectorValues.getVector(); + } + } } diff --git a/src/main/java/org/opensearch/knn/index/codec/BasePerFieldKnnVectorsFormat.java b/src/main/java/org/opensearch/knn/index/codec/BasePerFieldKnnVectorsFormat.java index 69229036e..8beced605 100644 --- a/src/main/java/org/opensearch/knn/index/codec/BasePerFieldKnnVectorsFormat.java +++ b/src/main/java/org/opensearch/knn/index/codec/BasePerFieldKnnVectorsFormat.java @@ -8,8 +8,11 @@ import lombok.AllArgsConstructor; import lombok.extern.log4j.Log4j2; import org.apache.lucene.codecs.KnnVectorsFormat; +import org.apache.lucene.codecs.hnsw.FlatVectorScorerUtil; +import org.apache.lucene.codecs.lucene99.Lucene99FlatVectorsFormat; import org.apache.lucene.codecs.perfield.PerFieldKnnVectorsFormat; import org.opensearch.index.mapper.MapperService; +import org.opensearch.knn.index.codec.KNN990Codec.NativeEngines990KnnVectorsFormat; import org.opensearch.knn.index.codec.params.KNNScalarQuantizedVectorsFormatParams; import org.opensearch.knn.index.codec.params.KNNVectorsFormatParams; import org.opensearch.knn.index.engine.KNNEngine; @@ -17,6 +20,7 @@ import org.opensearch.knn.index.mapper.KNNMappingConfig; import org.opensearch.knn.index.mapper.KNNVectorFieldType; +import java.util.Map; import java.util.Optional; import java.util.function.Function; import java.util.function.Supplier; @@ -78,42 +82,47 @@ public KnnVectorsFormat getKnnVectorsFormatForField(final String field) { KNNMethodContext knnMethodContext = knnMappingConfig.getKnnMethodContext() .orElseThrow(() -> new IllegalArgumentException("KNN method context cannot be empty")); - var params = knnMethodContext.getMethodComponentContext().getParameters(); + final KNNEngine engine = knnMethodContext.getKnnEngine(); + final Map params = knnMethodContext.getMethodComponentContext().getParameters(); - if (knnMethodContext.getKnnEngine() == KNNEngine.LUCENE && params != null && params.containsKey(METHOD_ENCODER_PARAMETER)) { - KNNScalarQuantizedVectorsFormatParams knnScalarQuantizedVectorsFormatParams = new KNNScalarQuantizedVectorsFormatParams( - params, - defaultMaxConnections, - defaultBeamWidth - ); - if (knnScalarQuantizedVectorsFormatParams.validate(params)) { - log.debug( - "Initialize KNN vector format for field [{}] with params [{}] = \"{}\", [{}] = \"{}\", [{}] = \"{}\", [{}] = \"{}\"", - field, - MAX_CONNECTIONS, - knnScalarQuantizedVectorsFormatParams.getMaxConnections(), - BEAM_WIDTH, - knnScalarQuantizedVectorsFormatParams.getBeamWidth(), - LUCENE_SQ_CONFIDENCE_INTERVAL, - knnScalarQuantizedVectorsFormatParams.getConfidenceInterval(), - LUCENE_SQ_BITS, - knnScalarQuantizedVectorsFormatParams.getBits() + if (engine == KNNEngine.LUCENE) { + if (params != null && params.containsKey(METHOD_ENCODER_PARAMETER)) { + KNNScalarQuantizedVectorsFormatParams knnScalarQuantizedVectorsFormatParams = new KNNScalarQuantizedVectorsFormatParams( + params, + defaultMaxConnections, + defaultBeamWidth ); - return scalarQuantizedVectorsFormatSupplier.apply(knnScalarQuantizedVectorsFormatParams); + if (knnScalarQuantizedVectorsFormatParams.validate(params)) { + log.debug( + "Initialize KNN vector format for field [{}] with params [{}] = \"{}\", [{}] = \"{}\", [{}] = \"{}\", [{}] = \"{}\"", + field, + MAX_CONNECTIONS, + knnScalarQuantizedVectorsFormatParams.getMaxConnections(), + BEAM_WIDTH, + knnScalarQuantizedVectorsFormatParams.getBeamWidth(), + LUCENE_SQ_CONFIDENCE_INTERVAL, + knnScalarQuantizedVectorsFormatParams.getConfidenceInterval(), + LUCENE_SQ_BITS, + knnScalarQuantizedVectorsFormatParams.getBits() + ); + return scalarQuantizedVectorsFormatSupplier.apply(knnScalarQuantizedVectorsFormatParams); + } } + KNNVectorsFormatParams knnVectorsFormatParams = new KNNVectorsFormatParams(params, defaultMaxConnections, defaultBeamWidth); + log.debug( + "Initialize KNN vector format for field [{}] with params [{}] = \"{}\" and [{}] = \"{}\"", + field, + MAX_CONNECTIONS, + knnVectorsFormatParams.getMaxConnections(), + BEAM_WIDTH, + knnVectorsFormatParams.getBeamWidth() + ); + return vectorsFormatSupplier.apply(knnVectorsFormatParams); } - KNNVectorsFormatParams knnVectorsFormatParams = new KNNVectorsFormatParams(params, defaultMaxConnections, defaultBeamWidth); - log.debug( - "Initialize KNN vector format for field [{}] with params [{}] = \"{}\" and [{}] = \"{}\"", - field, - MAX_CONNECTIONS, - knnVectorsFormatParams.getMaxConnections(), - BEAM_WIDTH, - knnVectorsFormatParams.getBeamWidth() - ); - return vectorsFormatSupplier.apply(knnVectorsFormatParams); + // All native engines to use NativeEngines990KnnVectorsFormat + return new NativeEngines990KnnVectorsFormat(new Lucene99FlatVectorsFormat(FlatVectorScorerUtil.getLucene99FlatVectorsScorer())); } @Override diff --git a/src/main/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumer.java b/src/main/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumer.java index f8bd0b3f7..218c9d891 100644 --- a/src/main/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumer.java +++ b/src/main/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumer.java @@ -5,74 +5,40 @@ package org.opensearch.knn.index.codec.KNN80Codec; -import lombok.NonNull; import lombok.extern.log4j.Log4j2; -import org.apache.lucene.store.ChecksumIndexInput; -import org.opensearch.common.xcontent.XContentType; import org.opensearch.common.StopWatch; -import org.opensearch.core.xcontent.DeprecationHandler; -import org.opensearch.core.xcontent.NamedXContentRegistry; -import org.opensearch.knn.index.SpaceType; -import org.opensearch.knn.index.util.IndexUtil; -import org.opensearch.knn.index.KNNSettings; import org.opensearch.knn.index.VectorDataType; -import org.opensearch.knn.index.codec.transfer.VectorTransfer; -import org.opensearch.knn.index.codec.transfer.VectorTransferByte; -import org.opensearch.knn.index.codec.transfer.VectorTransferFloat; -import org.opensearch.knn.jni.JNIService; -import org.opensearch.knn.index.codec.util.KNNCodecUtil; import org.opensearch.knn.index.engine.KNNEngine; -import org.opensearch.knn.indices.Model; -import org.opensearch.knn.indices.ModelCache; -import org.opensearch.knn.plugin.stats.KNNCounter; +import org.opensearch.knn.index.vectorvalues.KNNVectorValues; +import org.opensearch.knn.index.vectorvalues.KNNVectorValuesFactory; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.lucene.codecs.DocValuesConsumer; import org.apache.lucene.codecs.DocValuesProducer; -import org.apache.lucene.index.BinaryDocValues; import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.MergeState; import org.apache.lucene.index.SegmentWriteState; -import org.apache.lucene.store.FSDirectory; -import org.apache.lucene.store.FilterDirectory; +import org.opensearch.knn.index.codec.nativeindex.NativeIndexWriter; import org.opensearch.knn.index.mapper.KNNVectorFieldMapper; -import org.opensearch.knn.common.KNNConstants; import org.opensearch.knn.plugin.stats.KNNGraphValue; -import java.io.Closeable; import java.io.IOException; -import java.io.OutputStream; -import java.nio.ByteBuffer; -import java.nio.ByteOrder; -import java.nio.file.Files; -import java.nio.file.Paths; -import java.nio.file.StandardOpenOption; -import java.security.AccessController; -import java.security.PrivilegedAction; -import java.util.HashMap; -import java.util.Map; -import static org.apache.lucene.codecs.CodecUtil.FOOTER_MAGIC; -import static org.opensearch.knn.common.KNNConstants.MODEL_ID; -import static org.opensearch.knn.common.KNNConstants.PARAMETERS; -import static org.opensearch.knn.index.codec.util.KNNCodecUtil.buildEngineFileName; -import static org.opensearch.knn.index.codec.util.KNNCodecUtil.calculateArraySize; -import static org.opensearch.knn.index.engine.faiss.Faiss.FAISS_BINARY_INDEX_DESCRIPTION_PREFIX; +import static org.opensearch.knn.common.FieldInfoExtractor.extractKNNEngine; +import static org.opensearch.knn.common.FieldInfoExtractor.extractVectorDataType; /** * This class writes the KNN docvalues to the segments */ @Log4j2 -class KNN80DocValuesConsumer extends DocValuesConsumer implements Closeable { +class KNN80DocValuesConsumer extends DocValuesConsumer { private final Logger logger = LogManager.getLogger(KNN80DocValuesConsumer.class); private final DocValuesConsumer delegatee; private final SegmentWriteState state; - private static final Long CRC32_CHECKSUM_SANITY = 0xFFFFFFFF00000000L; - KNN80DocValuesConsumer(DocValuesConsumer delegatee, SegmentWriteState state) { this.delegatee = delegatee; this.state = state; @@ -84,7 +50,7 @@ public void addBinaryField(FieldInfo field, DocValuesProducer valuesProducer) th if (isKNNBinaryFieldRequired(field)) { StopWatch stopWatch = new StopWatch(); stopWatch.start(); - addKNNBinaryField(field, valuesProducer, false, true); + addKNNBinaryField(field, valuesProducer, false); stopWatch.stop(); long time_in_millis = stopWatch.totalTime().millis(); KNNGraphValue.REFRESH_TOTAL_TIME_IN_MILLIS.set(KNNGraphValue.REFRESH_TOTAL_TIME_IN_MILLIS.getValue() + time_in_millis); @@ -93,190 +59,21 @@ public void addBinaryField(FieldInfo field, DocValuesProducer valuesProducer) th } private boolean isKNNBinaryFieldRequired(FieldInfo field) { - final KNNEngine knnEngine = getKNNEngine(field); + final KNNEngine knnEngine = extractKNNEngine(field); log.debug(String.format("Read engine [%s] for field [%s]", knnEngine.getName(), field.getName())); return field.attributes().containsKey(KNNVectorFieldMapper.KNN_FIELD) && KNNEngine.getEnginesThatCreateCustomSegmentFiles().stream().anyMatch(engine -> engine == knnEngine); } - private KNNEngine getKNNEngine(@NonNull FieldInfo field) { - final String modelId = field.attributes().get(MODEL_ID); - if (modelId != null) { - var model = ModelCache.getInstance().get(modelId); - return model.getModelMetadata().getKnnEngine(); - } - final String engineName = field.attributes().getOrDefault(KNNConstants.KNN_ENGINE, KNNEngine.DEFAULT.getName()); - return KNNEngine.getEngine(engineName); - } - - public void addKNNBinaryField(FieldInfo field, DocValuesProducer valuesProducer, boolean isMerge, boolean isRefresh) - throws IOException { - // Get values to be indexed - BinaryDocValues values = valuesProducer.getBinary(field); - final KNNEngine knnEngine = getKNNEngine(field); - final String engineFileName = buildEngineFileName( - state.segmentInfo.name, - knnEngine.getVersion(), - field.name, - knnEngine.getExtension() - ); - final String indexPath = Paths.get( - ((FSDirectory) (FilterDirectory.unwrap(state.directory))).getDirectory().toString(), - engineFileName - ).toString(); - - // Determine if we are creating an index from a model or from scratch - NativeIndexCreator indexCreator; - KNNCodecUtil.Pair pair; - Map fieldAttributes = field.attributes(); - VectorDataType vectorDataType; - - if (fieldAttributes.containsKey(MODEL_ID)) { - String modelId = fieldAttributes.get(MODEL_ID); - Model model = ModelCache.getInstance().get(modelId); - if (model.getModelBlob() == null) { - throw new RuntimeException(String.format("There is no trained model with id \"%s\"", modelId)); - } - vectorDataType = model.getModelMetadata().getVectorDataType(); - pair = KNNCodecUtil.getPair(values, getVectorTransfer(vectorDataType)); - indexCreator = () -> createKNNIndexFromTemplate(model, pair, knnEngine, indexPath); - } else { - // get vector data type from field attributes or provide default value - vectorDataType = VectorDataType.get( - fieldAttributes.getOrDefault(KNNConstants.VECTOR_DATA_TYPE_FIELD, VectorDataType.DEFAULT.getValue()) - ); - pair = KNNCodecUtil.getPair(values, getVectorTransfer(vectorDataType)); - indexCreator = () -> createKNNIndexFromScratch(field, pair, knnEngine, indexPath); - } - - // Skip index creation if no vectors or docs in segment - if (pair.getVectorAddress() == 0 || pair.docs.length == 0) { - logger.info("Skipping engine index creation as there are no vectors or docs in the segment"); - return; - } - - long arraySize = calculateArraySize(pair.docs.length, pair.getDimension(), vectorDataType); + public void addKNNBinaryField(FieldInfo field, DocValuesProducer valuesProducer, boolean isMerge) throws IOException { + final VectorDataType vectorDataType = extractVectorDataType(field); + final KNNVectorValues knnVectorValues = KNNVectorValuesFactory.getVectorValues(vectorDataType, valuesProducer.getBinary(field)); if (isMerge) { - KNNGraphValue.MERGE_CURRENT_OPERATIONS.increment(); - KNNGraphValue.MERGE_CURRENT_DOCS.incrementBy(pair.docs.length); - KNNGraphValue.MERGE_CURRENT_SIZE_IN_BYTES.incrementBy(arraySize); - recordMergeStats(pair.docs.length, arraySize); - } - - // Increment counter for number of graph index requests - KNNCounter.GRAPH_INDEX_REQUESTS.increment(); - - if (isRefresh) { - recordRefreshStats(); - } - - // Ensure engineFileName is added to the tracked files by Lucene's TrackingDirectoryWrapper - state.directory.createOutput(engineFileName, state.context).close(); - indexCreator.createIndex(); - writeFooter(indexPath, engineFileName); - } - - private void recordMergeStats(int length, long arraySize) { - KNNGraphValue.MERGE_CURRENT_OPERATIONS.decrement(); - KNNGraphValue.MERGE_CURRENT_DOCS.decrementBy(length); - KNNGraphValue.MERGE_CURRENT_SIZE_IN_BYTES.decrementBy(arraySize); - KNNGraphValue.MERGE_TOTAL_OPERATIONS.increment(); - KNNGraphValue.MERGE_TOTAL_DOCS.incrementBy(length); - KNNGraphValue.MERGE_TOTAL_SIZE_IN_BYTES.incrementBy(arraySize); - } - - private void recordRefreshStats() { - KNNGraphValue.REFRESH_TOTAL_OPERATIONS.increment(); - } - - private void createKNNIndexFromTemplate(Model model, KNNCodecUtil.Pair pair, KNNEngine knnEngine, String indexPath) { - Map parameters = new HashMap<>(); - parameters.put(KNNConstants.INDEX_THREAD_QTY, KNNSettings.state().getSettingValue(KNNSettings.KNN_ALGO_PARAM_INDEX_THREAD_QTY)); - - IndexUtil.updateVectorDataTypeToParameters(parameters, model.getModelMetadata().getVectorDataType()); - - AccessController.doPrivileged((PrivilegedAction) () -> { - JNIService.createIndexFromTemplate( - pair.docs, - pair.getVectorAddress(), - pair.getDimension(), - indexPath, - model.getModelBlob(), - parameters, - knnEngine - ); - return null; - }); - } - - private void createKNNIndexFromScratch(FieldInfo fieldInfo, KNNCodecUtil.Pair pair, KNNEngine knnEngine, String indexPath) - throws IOException { - Map parameters = new HashMap<>(); - Map fieldAttributes = fieldInfo.attributes(); - String parametersString = fieldAttributes.get(PARAMETERS); - // parametersString will be null when legacy mapper is used - if (parametersString == null) { - parameters.put(KNNConstants.SPACE_TYPE, fieldAttributes.getOrDefault(KNNConstants.SPACE_TYPE, SpaceType.DEFAULT.getValue())); - - String efConstruction = fieldAttributes.get(KNNConstants.HNSW_ALGO_EF_CONSTRUCTION); - Map algoParams = new HashMap<>(); - if (efConstruction != null) { - algoParams.put(KNNConstants.METHOD_PARAMETER_EF_CONSTRUCTION, Integer.parseInt(efConstruction)); - } - - String m = fieldAttributes.get(KNNConstants.HNSW_ALGO_M); - if (m != null) { - algoParams.put(KNNConstants.METHOD_PARAMETER_M, Integer.parseInt(m)); - } - parameters.put(PARAMETERS, algoParams); + NativeIndexWriter.getWriter(field, state).mergeIndex(knnVectorValues); } else { - parameters.putAll( - XContentType.JSON.xContent() - .createParser(NamedXContentRegistry.EMPTY, DeprecationHandler.THROW_UNSUPPORTED_OPERATION, parametersString) - .map() - ); - } - - // In OpenSearch 2.16, we added the prefix for binary indices in the index description in the codec logic. - // After 2.16, we added the binary prefix in the faiss library code. However, to ensure backwards compatibility, - // we need to ensure that if the description does not contain the prefix but the type is binary, we add the - // description. - maybeAddBinaryPrefixForFaissBWC(knnEngine, parameters, fieldAttributes); - - // Used to determine how many threads to use when indexing - parameters.put(KNNConstants.INDEX_THREAD_QTY, KNNSettings.state().getSettingValue(KNNSettings.KNN_ALGO_PARAM_INDEX_THREAD_QTY)); - - // Pass the path for the nms library to save the file - AccessController.doPrivileged((PrivilegedAction) () -> { - JNIService.createIndex(pair.docs, pair.getVectorAddress(), pair.getDimension(), indexPath, parameters, knnEngine); - return null; - }); - } - - private void maybeAddBinaryPrefixForFaissBWC(KNNEngine knnEngine, Map parameters, Map fieldAttributes) { - if (KNNEngine.FAISS != knnEngine) { - return; - } - - if (!VectorDataType.BINARY.getValue() - .equals(fieldAttributes.getOrDefault(KNNConstants.VECTOR_DATA_TYPE_FIELD, VectorDataType.DEFAULT.getValue()))) { - return; - } - - if (parameters.get(KNNConstants.INDEX_DESCRIPTION_PARAMETER) == null) { - return; - } - - if (parameters.get(KNNConstants.INDEX_DESCRIPTION_PARAMETER).toString().startsWith(FAISS_BINARY_INDEX_DESCRIPTION_PREFIX)) { - return; + NativeIndexWriter.getWriter(field, state).flushIndex(knnVectorValues); } - - parameters.put( - KNNConstants.INDEX_DESCRIPTION_PARAMETER, - FAISS_BINARY_INDEX_DESCRIPTION_PREFIX + parameters.get(KNNConstants.INDEX_DESCRIPTION_PARAMETER).toString() - ); - IndexUtil.updateVectorDataTypeToParameters(parameters, VectorDataType.BINARY); } /** @@ -295,7 +92,7 @@ public void merge(MergeState mergeState) { if (type == DocValuesType.BINARY && fieldInfo.attributes().containsKey(KNNVectorFieldMapper.KNN_FIELD)) { StopWatch stopWatch = new StopWatch(); stopWatch.start(); - addKNNBinaryField(fieldInfo, new KNN80DocValuesReader(mergeState), true, false); + addKNNBinaryField(fieldInfo, new KNN80DocValuesReader(mergeState), true); stopWatch.stop(); long time_in_millis = stopWatch.totalTime().millis(); KNNGraphValue.MERGE_TOTAL_TIME_IN_MILLIS.set(KNNGraphValue.MERGE_TOTAL_TIME_IN_MILLIS.getValue() + time_in_millis); @@ -331,52 +128,4 @@ public void addNumericField(FieldInfo field, DocValuesProducer valuesProducer) t public void close() throws IOException { delegatee.close(); } - - @FunctionalInterface - private interface NativeIndexCreator { - void createIndex() throws IOException; - } - - private void writeFooter(String indexPath, String engineFileName) throws IOException { - // Opens the engine file that was created and appends a footer to it. The footer consists of - // 1. A Footer magic number (int - 4 bytes) - // 2. A checksum algorithm id (int - 4 bytes) - // 3. A checksum (long - bytes) - // The checksum is computed on all the bytes written to the file up to that point. - // Logic where footer is written in Lucene can be found here: - // https://github.com/apache/lucene/blob/branch_9_0/lucene/core/src/java/org/apache/lucene/codecs/CodecUtil.java#L390-L412 - OutputStream os = Files.newOutputStream(Paths.get(indexPath), StandardOpenOption.APPEND); - ByteBuffer byteBuffer = ByteBuffer.allocate(8).order(ByteOrder.BIG_ENDIAN); - byteBuffer.putInt(FOOTER_MAGIC); - byteBuffer.putInt(0); - os.write(byteBuffer.array()); - os.flush(); - - ChecksumIndexInput checksumIndexInput = state.directory.openChecksumInput(engineFileName, state.context); - checksumIndexInput.seek(checksumIndexInput.length()); - long value = checksumIndexInput.getChecksum(); - checksumIndexInput.close(); - - if (isChecksumValid(value)) { - throw new IllegalStateException("Illegal CRC-32 checksum: " + value + " (resource=" + os + ")"); - } - - // Write the CRC checksum to the end of the OutputStream and close the stream - byteBuffer.putLong(0, value); - os.write(byteBuffer.array()); - os.close(); - } - - private boolean isChecksumValid(long value) { - // Check pulled from - // https://github.com/apache/lucene/blob/branch_9_0/lucene/core/src/java/org/apache/lucene/codecs/CodecUtil.java#L644-L647 - return (value & CRC32_CHECKSUM_SANITY) != 0; - } - - private VectorTransfer getVectorTransfer(VectorDataType vectorDataType) { - if (VectorDataType.BINARY == vectorDataType) { - return new VectorTransferByte(KNNSettings.getVectorStreamingMemoryLimit().getBytes()); - } - return new VectorTransferFloat(KNNSettings.getVectorStreamingMemoryLimit().getBytes()); - } } diff --git a/src/main/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngineFieldVectorsWriter.java b/src/main/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngineFieldVectorsWriter.java index e4860af31..1abb84944 100644 --- a/src/main/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngineFieldVectorsWriter.java +++ b/src/main/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngineFieldVectorsWriter.java @@ -30,6 +30,7 @@ */ class NativeEngineFieldVectorsWriter extends KnnFieldVectorsWriter { private static final long SHALLOW_SIZE = RamUsageEstimator.shallowSizeOfInstance(NativeEngineFieldVectorsWriter.class); + @Getter private final FieldInfo fieldInfo; /** * We are using a map here instead of list, because for sampler interface for quantization we have to advance the iterator @@ -77,6 +78,8 @@ public void addValue(int docID, T vectorValue) { + "\" appears more than once in this document (only one value is allowed per field)" ); } + // TODO: we can build the graph here too iteratively. but right now I am skipping that as we need iterative + // graph build support on the JNI layer. assert docID > lastDocID; vectors.put(docID, vectorValue); docsWithField.add(docID); diff --git a/src/main/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsWriter.java b/src/main/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsWriter.java index b81ec9789..65736a63e 100644 --- a/src/main/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsWriter.java +++ b/src/main/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsWriter.java @@ -12,23 +12,33 @@ package org.opensearch.knn.index.codec.KNN990Codec; import lombok.RequiredArgsConstructor; +import lombok.extern.log4j.Log4j2; import org.apache.lucene.codecs.KnnFieldVectorsWriter; import org.apache.lucene.codecs.KnnVectorsWriter; import org.apache.lucene.codecs.hnsw.FlatVectorsWriter; +import org.apache.lucene.index.ByteVectorValues; import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.FloatVectorValues; import org.apache.lucene.index.MergeState; import org.apache.lucene.index.SegmentWriteState; import org.apache.lucene.index.Sorter; import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.RamUsageEstimator; +import org.opensearch.knn.index.VectorDataType; +import org.opensearch.knn.index.codec.nativeindex.NativeIndexWriter; +import org.opensearch.knn.index.vectorvalues.KNNVectorValues; +import org.opensearch.knn.index.vectorvalues.KNNVectorValuesFactory; import java.io.IOException; import java.util.ArrayList; import java.util.List; +import static org.opensearch.knn.common.FieldInfoExtractor.extractVectorDataType; + /** * A KNNVectorsWriter class for writing the vector data strcutures and flat vectors for Native Engines. */ +@Log4j2 @RequiredArgsConstructor public class NativeEngines990KnnVectorsWriter extends KnnVectorsWriter { private static final long SHALLOW_SIZE = RamUsageEstimator.shallowSizeOfInstance(NativeEngines990KnnVectorsWriter.class); @@ -46,8 +56,6 @@ public class NativeEngines990KnnVectorsWriter extends KnnVectorsWriter { @Override public KnnFieldVectorsWriter addField(final FieldInfo fieldInfo) throws IOException { final NativeEngineFieldVectorsWriter newField = NativeEngineFieldVectorsWriter.create(fieldInfo, segmentWriteState.infoStream); - // TODO: we can build the graph here too iteratively. but right now I am skipping that as we need iterative - // graph build support on the JNI layer. fields.add(newField); return flatVectorsWriter.addField(fieldInfo, newField); } @@ -62,14 +70,40 @@ public KnnFieldVectorsWriter addField(final FieldInfo fieldInfo) throws IOExc public void flush(int maxDoc, final Sorter.DocMap sortMap) throws IOException { // simply write data in the flat file flatVectorsWriter.flush(maxDoc, sortMap); - // TODO: add code for creating Vector datastructures during lucene flush operation + for (final NativeEngineFieldVectorsWriter field : fields) { + final VectorDataType vectorDataType = extractVectorDataType(field.getFieldInfo()); + final KNNVectorValues knnVectorValues = KNNVectorValuesFactory.getVectorValues( + vectorDataType, + field.getDocsWithField(), + field.getVectors() + ); + + NativeIndexWriter.getWriter(field.getFieldInfo(), segmentWriteState).flushIndex(knnVectorValues); + } } @Override public void mergeOneField(final FieldInfo fieldInfo, final MergeState mergeState) throws IOException { // This will ensure that we are merging the FlatIndex during force merge. flatVectorsWriter.mergeOneField(fieldInfo, mergeState); - // TODO: add code for creating Vector datastructures during merge operation + + // For merge, pick values from flat vector and reindex again. This will use the flush operation to create graphs + final VectorDataType vectorDataType = extractVectorDataType(fieldInfo); + final KNNVectorValues knnVectorValues; + switch (fieldInfo.getVectorEncoding()) { + case FLOAT32: + final FloatVectorValues mergedFloats = MergedVectorValues.mergeFloatVectorValues(fieldInfo, mergeState); + knnVectorValues = KNNVectorValuesFactory.getVectorValues(vectorDataType, mergedFloats); + break; + case BYTE: + final ByteVectorValues mergedBytes = MergedVectorValues.mergeByteVectorValues(fieldInfo, mergeState); + knnVectorValues = KNNVectorValuesFactory.getVectorValues(vectorDataType, mergedBytes); + break; + default: + throw new IllegalStateException("Unsupported vector encoding [" + fieldInfo.getVectorEncoding() + "]"); + } + + NativeIndexWriter.getWriter(fieldInfo, segmentWriteState).mergeIndex(knnVectorValues); } /** diff --git a/src/main/java/org/opensearch/knn/index/codec/nativeindex/DefaultIndexBuildStrategy.java b/src/main/java/org/opensearch/knn/index/codec/nativeindex/DefaultIndexBuildStrategy.java new file mode 100644 index 000000000..5787ea76b --- /dev/null +++ b/src/main/java/org/opensearch/knn/index/codec/nativeindex/DefaultIndexBuildStrategy.java @@ -0,0 +1,95 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.codec.nativeindex; + +import lombok.AccessLevel; +import lombok.NoArgsConstructor; +import org.opensearch.knn.common.KNNConstants; +import org.opensearch.knn.index.KNNSettings; +import org.opensearch.knn.index.codec.nativeindex.model.BuildIndexParams; +import org.opensearch.knn.index.codec.transfer.OffHeapVectorTransfer; +import org.opensearch.knn.index.vectorvalues.KNNVectorValues; +import org.opensearch.knn.jni.JNIService; + +import java.io.IOException; +import java.security.AccessController; +import java.security.PrivilegedAction; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS; +import static org.opensearch.knn.common.KNNConstants.MODEL_ID; +import static org.opensearch.knn.common.KNNVectorUtil.intListToArray; +import static org.opensearch.knn.common.KNNVectorUtil.iterateVectorValuesOnce; +import static org.opensearch.knn.index.codec.transfer.OffHeapVectorTransferFactory.getVectorTransfer; + +/** + * Transfers all vectors to off heap and then builds an index + */ +@NoArgsConstructor(access = AccessLevel.PRIVATE) +final class DefaultIndexBuildStrategy implements NativeIndexBuildStrategy { + + private static DefaultIndexBuildStrategy INSTANCE = new DefaultIndexBuildStrategy(); + + public static DefaultIndexBuildStrategy getInstance() { + return INSTANCE; + } + + public void buildAndWriteIndex(final BuildIndexParams indexInfo, final KNNVectorValues knnVectorValues) throws IOException { + iterateVectorValuesOnce(knnVectorValues); // to get bytesPerVector + int transferLimit = (int) Math.max(1, KNNSettings.getVectorStreamingMemoryLimit().getBytes() / knnVectorValues.bytesPerVector()); + try (final OffHeapVectorTransfer vectorTransfer = getVectorTransfer(indexInfo.getVectorDataType(), transferLimit)) { + + final List tranferredDocIds = new ArrayList<>(); + while (knnVectorValues.docId() != NO_MORE_DOCS) { + // append is true here so off heap memory buffer isn't overwritten + vectorTransfer.transfer(knnVectorValues.conditionalCloneVector(), true); + tranferredDocIds.add(knnVectorValues.docId()); + knnVectorValues.nextDoc(); + } + vectorTransfer.flush(true); + + final Map params = indexInfo.getParameters(); + long vectorAddress = vectorTransfer.getVectorAddress(); + // Currently this is if else as there are only two cases, with more cases this will have to be made + // more maintainable + if (params.containsKey(MODEL_ID)) { + AccessController.doPrivileged((PrivilegedAction) () -> { + JNIService.createIndexFromTemplate( + intListToArray(tranferredDocIds), + vectorAddress, + knnVectorValues.dimension(), + indexInfo.getIndexPath(), + (byte[]) params.get(KNNConstants.MODEL_BLOB_PARAMETER), + indexInfo.getParameters(), + indexInfo.getKnnEngine() + ); + return null; + }); + } else { + AccessController.doPrivileged((PrivilegedAction) () -> { + JNIService.createIndex( + intListToArray(tranferredDocIds), + vectorAddress, + knnVectorValues.dimension(), + indexInfo.getIndexPath(), + indexInfo.getParameters(), + indexInfo.getKnnEngine() + ); + return null; + }); + } + // Resetting here as vectors are deleted in JNILayer for non-iterative index builds + vectorTransfer.reset(); + } catch (Exception exception) { + throw new RuntimeException( + "Failed to build index, field name " + indexInfo.getFieldName() + ", parameters " + indexInfo, + exception + ); + } + } +} diff --git a/src/main/java/org/opensearch/knn/index/codec/nativeindex/MemOptimizedNativeIndexBuildStrategy.java b/src/main/java/org/opensearch/knn/index/codec/nativeindex/MemOptimizedNativeIndexBuildStrategy.java new file mode 100644 index 000000000..af80215b6 --- /dev/null +++ b/src/main/java/org/opensearch/knn/index/codec/nativeindex/MemOptimizedNativeIndexBuildStrategy.java @@ -0,0 +1,117 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.codec.nativeindex; + +import lombok.AccessLevel; +import lombok.NoArgsConstructor; +import org.opensearch.knn.index.KNNSettings; +import org.opensearch.knn.index.codec.nativeindex.model.BuildIndexParams; +import org.opensearch.knn.index.codec.transfer.OffHeapVectorTransfer; +import org.opensearch.knn.index.engine.KNNEngine; +import org.opensearch.knn.index.vectorvalues.KNNVectorValues; +import org.opensearch.knn.jni.JNIService; + +import java.io.IOException; +import java.security.AccessController; +import java.security.PrivilegedAction; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS; +import static org.opensearch.knn.common.KNNVectorUtil.intListToArray; +import static org.opensearch.knn.common.KNNVectorUtil.iterateVectorValuesOnce; +import static org.opensearch.knn.index.codec.transfer.OffHeapVectorTransferFactory.getVectorTransfer; + +/** + * Iteratively builds the index. Iterative builds are memory optimized as it does not require all vectors + * to be transferred. It transfers vectors in small batches, builds index and can clear the offheap space where + * the vectors were transferred + */ +@NoArgsConstructor(access = AccessLevel.PRIVATE) +final class MemOptimizedNativeIndexBuildStrategy implements NativeIndexBuildStrategy { + + private static MemOptimizedNativeIndexBuildStrategy INSTANCE = new MemOptimizedNativeIndexBuildStrategy(); + + public static MemOptimizedNativeIndexBuildStrategy getInstance() { + return INSTANCE; + } + + public void buildAndWriteIndex(final BuildIndexParams indexInfo, final KNNVectorValues knnVectorValues) throws IOException { + // Needed to make sure we dont get 0 dimensions while initializing index + iterateVectorValuesOnce(knnVectorValues); + KNNEngine engine = indexInfo.getKnnEngine(); + Map indexParameters = indexInfo.getParameters(); + + // Initialize the index + long indexMemoryAddress = AccessController.doPrivileged( + (PrivilegedAction) () -> JNIService.initIndex( + knnVectorValues.totalLiveDocs(), + knnVectorValues.dimension(), + indexParameters, + engine + ) + ); + + int transferLimit = (int) Math.max(1, KNNSettings.getVectorStreamingMemoryLimit().getBytes() / knnVectorValues.bytesPerVector()); + try (final OffHeapVectorTransfer vectorTransfer = getVectorTransfer(indexInfo.getVectorDataType(), transferLimit)) { + + final List tranferredDocIds = new ArrayList<>(transferLimit); + while (knnVectorValues.docId() != NO_MORE_DOCS) { + // append is false to be able to reuse the memory location + boolean transferred = vectorTransfer.transfer(knnVectorValues.conditionalCloneVector(), false); + tranferredDocIds.add(knnVectorValues.docId()); + if (transferred) { + // Insert vectors + long vectorAddress = vectorTransfer.getVectorAddress(); + AccessController.doPrivileged((PrivilegedAction) () -> { + JNIService.insertToIndex( + intListToArray(tranferredDocIds), + vectorAddress, + knnVectorValues.dimension(), + indexParameters, + indexMemoryAddress, + engine + ); + return null; + }); + tranferredDocIds.clear(); + } + knnVectorValues.nextDoc(); + } + + boolean flush = vectorTransfer.flush(false); + // Need to make sure that the flushed vectors are indexed + if (flush) { + long vectorAddress = vectorTransfer.getVectorAddress(); + AccessController.doPrivileged((PrivilegedAction) () -> { + JNIService.insertToIndex( + intListToArray(tranferredDocIds), + vectorAddress, + knnVectorValues.dimension(), + indexParameters, + indexMemoryAddress, + engine + ); + return null; + }); + tranferredDocIds.clear(); + } + + // Write vector + AccessController.doPrivileged((PrivilegedAction) () -> { + JNIService.writeIndex(indexInfo.getIndexPath(), indexMemoryAddress, engine, indexParameters); + return null; + }); + + } catch (Exception exception) { + throw new RuntimeException( + "Failed to build index, field name [" + indexInfo.getFieldName() + "], parameters " + indexInfo, + exception + ); + } + } +} diff --git a/src/main/java/org/opensearch/knn/index/codec/nativeindex/NativeIndexBuildStrategy.java b/src/main/java/org/opensearch/knn/index/codec/nativeindex/NativeIndexBuildStrategy.java new file mode 100644 index 000000000..19475adfa --- /dev/null +++ b/src/main/java/org/opensearch/knn/index/codec/nativeindex/NativeIndexBuildStrategy.java @@ -0,0 +1,19 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.codec.nativeindex; + +import org.opensearch.knn.index.codec.nativeindex.model.BuildIndexParams; +import org.opensearch.knn.index.vectorvalues.KNNVectorValues; + +import java.io.IOException; + +/** + * Interface which dictates how the index needs to be built + */ +public interface NativeIndexBuildStrategy { + + void buildAndWriteIndex(BuildIndexParams indexInfo, final KNNVectorValues knnVectorValues) throws IOException; +} diff --git a/src/main/java/org/opensearch/knn/index/codec/nativeindex/NativeIndexWriter.java b/src/main/java/org/opensearch/knn/index/codec/nativeindex/NativeIndexWriter.java new file mode 100644 index 000000000..61500371b --- /dev/null +++ b/src/main/java/org/opensearch/knn/index/codec/nativeindex/NativeIndexWriter.java @@ -0,0 +1,298 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.codec.nativeindex; + +import lombok.AllArgsConstructor; +import lombok.extern.log4j.Log4j2; +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.SegmentWriteState; +import org.apache.lucene.store.ChecksumIndexInput; +import org.apache.lucene.store.FSDirectory; +import org.apache.lucene.store.FilterDirectory; +import org.opensearch.common.xcontent.XContentHelper; +import org.opensearch.core.common.bytes.BytesArray; +import org.opensearch.core.xcontent.DeprecationHandler; +import org.opensearch.core.xcontent.MediaTypeRegistry; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.knn.common.KNNConstants; +import org.opensearch.knn.index.KNNSettings; +import org.opensearch.knn.index.SpaceType; +import org.opensearch.knn.index.VectorDataType; +import org.opensearch.knn.index.codec.nativeindex.model.BuildIndexParams; +import org.opensearch.knn.index.engine.KNNEngine; +import org.opensearch.knn.index.util.IndexUtil; +import org.opensearch.knn.index.vectorvalues.KNNVectorValues; +import org.opensearch.knn.indices.Model; +import org.opensearch.knn.indices.ModelCache; +import org.opensearch.knn.plugin.stats.KNNGraphValue; + +import java.io.IOException; +import java.io.OutputStream; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.nio.file.StandardOpenOption; +import java.util.HashMap; +import java.util.Map; + +import static org.apache.lucene.codecs.CodecUtil.FOOTER_MAGIC; +import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS; +import static org.opensearch.knn.common.FieldInfoExtractor.extractKNNEngine; +import static org.opensearch.knn.common.FieldInfoExtractor.extractVectorDataType; +import static org.opensearch.knn.common.KNNConstants.MODEL_ID; +import static org.opensearch.knn.common.KNNConstants.PARAMETERS; +import static org.opensearch.knn.common.KNNVectorUtil.iterateVectorValuesOnce; +import static org.opensearch.knn.index.codec.util.KNNCodecUtil.buildEngineFileName; +import static org.opensearch.knn.index.engine.faiss.Faiss.FAISS_BINARY_INDEX_DESCRIPTION_PREFIX; + +/** + * Writes KNN Index for a field in a segment. This is intended to be used for native engines + */ +@AllArgsConstructor +@Log4j2 +public class NativeIndexWriter { + private static final Long CRC32_CHECKSUM_SANITY = 0xFFFFFFFF00000000L; + + private final SegmentWriteState state; + private final FieldInfo fieldInfo; + private final NativeIndexBuildStrategy indexBuilder; + + /** + * Gets the correct writer type from fieldInfo + * + * @param fieldInfo + * @return correct NativeIndexWriter to make index specified in fieldInfo + */ + public static NativeIndexWriter getWriter(final FieldInfo fieldInfo, SegmentWriteState state) { + final KNNEngine knnEngine = extractKNNEngine(fieldInfo); + boolean isTemplate = fieldInfo.attributes().containsKey(MODEL_ID); + boolean iterative = !isTemplate && KNNEngine.FAISS == knnEngine; + if (iterative) { + return new NativeIndexWriter(state, fieldInfo, MemOptimizedNativeIndexBuildStrategy.getInstance()); + } + return new NativeIndexWriter(state, fieldInfo, DefaultIndexBuildStrategy.getInstance()); + } + + /** + * flushes the index + * + * @param knnVectorValues + * @throws IOException + */ + public void flushIndex(final KNNVectorValues knnVectorValues) throws IOException { + iterateVectorValuesOnce(knnVectorValues); + buildAndWriteIndex(knnVectorValues); + recordRefreshStats(); + } + + /** + * Merges kNN index + * @param knnVectorValues + * @throws IOException + */ + public void mergeIndex(final KNNVectorValues knnVectorValues) throws IOException { + iterateVectorValuesOnce(knnVectorValues); + if (knnVectorValues.docId() == NO_MORE_DOCS) { + // This is in place so we do not add metrics + log.debug("Skipping mergeIndex, vector values are already iterated for {}", fieldInfo.name); + return; + } + + long bytesPerVector = knnVectorValues.bytesPerVector(); + startMergeStats((int) knnVectorValues.totalLiveDocs(), bytesPerVector); + buildAndWriteIndex(knnVectorValues); + endMergeStats((int) knnVectorValues.totalLiveDocs(), bytesPerVector); + } + + private void buildAndWriteIndex(final KNNVectorValues knnVectorValues) throws IOException { + if (knnVectorValues.totalLiveDocs() == 0) { + log.debug("No live docs for field " + fieldInfo.name); + return; + } + + final KNNEngine knnEngine = extractKNNEngine(fieldInfo); + final String engineFileName = buildEngineFileName( + state.segmentInfo.name, + knnEngine.getVersion(), + fieldInfo.name, + knnEngine.getExtension() + ); + final String indexPath = Paths.get( + ((FSDirectory) (FilterDirectory.unwrap(state.directory))).getDirectory().toString(), + engineFileName + ).toString(); + state.directory.createOutput(engineFileName, state.context).close(); + + final BuildIndexParams nativeIndexParams = indexParams(fieldInfo, indexPath, knnEngine); + indexBuilder.buildAndWriteIndex(nativeIndexParams, knnVectorValues); + writeFooter(indexPath, engineFileName, state); + } + + // The logic for building parameters need to be cleaned up. There are various cases handled here + // Currently it falls under two categories - with model and without model. Without model is further divided based on vector data type + // TODO: Refactor this so its scalable. Possibly move it out of this class + private BuildIndexParams indexParams(FieldInfo fieldInfo, String indexPath, KNNEngine knnEngine) throws IOException { + final Map parameters; + final VectorDataType vectorDataType = extractVectorDataType(fieldInfo); + if (fieldInfo.attributes().containsKey(MODEL_ID)) { + Model model = getModel(fieldInfo); + parameters = getTemplateParameters(fieldInfo, model); + } else { + parameters = getParameters(fieldInfo, vectorDataType, knnEngine); + } + + return BuildIndexParams.builder() + .fieldName(fieldInfo.name) + .parameters(parameters) + .vectorDataType(vectorDataType) + .knnEngine(knnEngine) + .indexPath(indexPath) + .build(); + } + + private Map getParameters(FieldInfo fieldInfo, VectorDataType vectorDataType, KNNEngine knnEngine) throws IOException { + Map parameters = new HashMap<>(); + Map fieldAttributes = fieldInfo.attributes(); + String parametersString = fieldAttributes.get(KNNConstants.PARAMETERS); + + // parametersString will be null when legacy mapper is used + if (parametersString == null) { + parameters.put(KNNConstants.SPACE_TYPE, fieldAttributes.getOrDefault(KNNConstants.SPACE_TYPE, SpaceType.DEFAULT.getValue())); + + String efConstruction = fieldAttributes.get(KNNConstants.HNSW_ALGO_EF_CONSTRUCTION); + Map algoParams = new HashMap<>(); + if (efConstruction != null) { + algoParams.put(KNNConstants.METHOD_PARAMETER_EF_CONSTRUCTION, Integer.parseInt(efConstruction)); + } + + String m = fieldAttributes.get(KNNConstants.HNSW_ALGO_M); + if (m != null) { + algoParams.put(KNNConstants.METHOD_PARAMETER_M, Integer.parseInt(m)); + } + parameters.put(PARAMETERS, algoParams); + } else { + parameters.putAll( + XContentHelper.createParser( + NamedXContentRegistry.EMPTY, + DeprecationHandler.THROW_UNSUPPORTED_OPERATION, + new BytesArray(parametersString), + MediaTypeRegistry.getDefaultMediaType() + ).map() + ); + } + + parameters.put(KNNConstants.VECTOR_DATA_TYPE_FIELD, vectorDataType.getValue()); + // In OpenSearch 2.16, we added the prefix for binary indices in the index description in the codec logic. + // After 2.16, we added the binary prefix in the faiss library code. However, to ensure backwards compatibility, + // we need to ensure that if the description does not contain the prefix but the type is binary, we add the + // description. + maybeAddBinaryPrefixForFaissBWC(knnEngine, parameters, fieldAttributes); + + // Used to determine how many threads to use when indexing + parameters.put(KNNConstants.INDEX_THREAD_QTY, KNNSettings.state().getSettingValue(KNNSettings.KNN_ALGO_PARAM_INDEX_THREAD_QTY)); + + return parameters; + } + + private void maybeAddBinaryPrefixForFaissBWC(KNNEngine knnEngine, Map parameters, Map fieldAttributes) { + if (KNNEngine.FAISS != knnEngine) { + return; + } + + if (!VectorDataType.BINARY.getValue() + .equals(fieldAttributes.getOrDefault(KNNConstants.VECTOR_DATA_TYPE_FIELD, VectorDataType.DEFAULT.getValue()))) { + return; + } + + if (parameters.get(KNNConstants.INDEX_DESCRIPTION_PARAMETER) == null) { + return; + } + + if (parameters.get(KNNConstants.INDEX_DESCRIPTION_PARAMETER).toString().startsWith(FAISS_BINARY_INDEX_DESCRIPTION_PREFIX)) { + return; + } + + parameters.put( + KNNConstants.INDEX_DESCRIPTION_PARAMETER, + FAISS_BINARY_INDEX_DESCRIPTION_PREFIX + parameters.get(KNNConstants.INDEX_DESCRIPTION_PARAMETER).toString() + ); + IndexUtil.updateVectorDataTypeToParameters(parameters, VectorDataType.BINARY); + } + + private Map getTemplateParameters(FieldInfo fieldInfo, Model model) throws IOException { + Map parameters = new HashMap<>(); + parameters.put(KNNConstants.INDEX_THREAD_QTY, KNNSettings.state().getSettingValue(KNNSettings.KNN_ALGO_PARAM_INDEX_THREAD_QTY)); + parameters.put(KNNConstants.MODEL_ID, fieldInfo.attributes().get(MODEL_ID)); + parameters.put(KNNConstants.MODEL_BLOB_PARAMETER, model.getModelBlob()); + IndexUtil.updateVectorDataTypeToParameters(parameters, model.getModelMetadata().getVectorDataType()); + return parameters; + } + + private Model getModel(FieldInfo fieldInfo) { + String modelId = fieldInfo.attributes().get(MODEL_ID); + Model model = ModelCache.getInstance().get(modelId); + if (model.getModelBlob() == null) { + throw new RuntimeException(String.format("There is no trained model with id \"%s\"", modelId)); + } + return model; + } + + private void startMergeStats(int numDocs, long bytesPerVector) { + KNNGraphValue.MERGE_CURRENT_OPERATIONS.increment(); + KNNGraphValue.MERGE_CURRENT_DOCS.incrementBy(numDocs); + KNNGraphValue.MERGE_CURRENT_SIZE_IN_BYTES.incrementBy(bytesPerVector); + KNNGraphValue.MERGE_TOTAL_OPERATIONS.increment(); + KNNGraphValue.MERGE_TOTAL_DOCS.incrementBy(numDocs); + KNNGraphValue.MERGE_TOTAL_SIZE_IN_BYTES.incrementBy(bytesPerVector); + } + + private void endMergeStats(int numDocs, long arraySize) { + KNNGraphValue.MERGE_CURRENT_OPERATIONS.decrement(); + KNNGraphValue.MERGE_CURRENT_DOCS.decrementBy(numDocs); + KNNGraphValue.MERGE_CURRENT_SIZE_IN_BYTES.decrementBy(arraySize); + } + + private void recordRefreshStats() { + KNNGraphValue.REFRESH_TOTAL_OPERATIONS.increment(); + } + + private boolean isChecksumValid(long value) { + // Check pulled from + // https://github.com/apache/lucene/blob/branch_9_0/lucene/core/src/java/org/apache/lucene/codecs/CodecUtil.java#L644-L647 + return (value & CRC32_CHECKSUM_SANITY) != 0; + } + + private void writeFooter(String indexPath, String engineFileName, SegmentWriteState state) throws IOException { + // Opens the engine file that was created and appends a footer to it. The footer consists of + // 1. A Footer magic number (int - 4 bytes) + // 2. A checksum algorithm id (int - 4 bytes) + // 3. A checksum (long - bytes) + // The checksum is computed on all the bytes written to the file up to that point. + // Logic where footer is written in Lucene can be found here: + // https://github.com/apache/lucene/blob/branch_9_0/lucene/core/src/java/org/apache/lucene/codecs/CodecUtil.java#L390-L412 + OutputStream os = Files.newOutputStream(Paths.get(indexPath), StandardOpenOption.APPEND); + ByteBuffer byteBuffer = ByteBuffer.allocate(8).order(ByteOrder.BIG_ENDIAN); + byteBuffer.putInt(FOOTER_MAGIC); + byteBuffer.putInt(0); + os.write(byteBuffer.array()); + os.flush(); + + ChecksumIndexInput checksumIndexInput = state.directory.openChecksumInput(engineFileName, state.context); + checksumIndexInput.seek(checksumIndexInput.length()); + long value = checksumIndexInput.getChecksum(); + checksumIndexInput.close(); + + if (isChecksumValid(value)) { + throw new IllegalStateException("Illegal CRC-32 checksum: " + value + " (resource=" + os + ")"); + } + + // Write the CRC checksum to the end of the OutputStream and close the stream + byteBuffer.putLong(0, value); + os.write(byteBuffer.array()); + os.close(); + } +} diff --git a/src/main/java/org/opensearch/knn/index/codec/nativeindex/model/BuildIndexParams.java b/src/main/java/org/opensearch/knn/index/codec/nativeindex/model/BuildIndexParams.java new file mode 100644 index 000000000..af43ff37e --- /dev/null +++ b/src/main/java/org/opensearch/knn/index/codec/nativeindex/model/BuildIndexParams.java @@ -0,0 +1,26 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.codec.nativeindex.model; + +import lombok.Builder; +import lombok.ToString; +import lombok.Value; +import org.opensearch.knn.index.VectorDataType; +import org.opensearch.knn.index.engine.KNNEngine; + +import java.util.Map; + +@Value +@Builder +@ToString +public class BuildIndexParams { + String fieldName; + KNNEngine knnEngine; + String indexPath; + VectorDataType vectorDataType; + Map parameters; + // TODO: Add quantization state as parameter to build index +} diff --git a/src/main/java/org/opensearch/knn/index/codec/transfer/OffHeapBinaryVectorTransfer.java b/src/main/java/org/opensearch/knn/index/codec/transfer/OffHeapBinaryVectorTransfer.java new file mode 100644 index 000000000..c9d4802fe --- /dev/null +++ b/src/main/java/org/opensearch/knn/index/codec/transfer/OffHeapBinaryVectorTransfer.java @@ -0,0 +1,32 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.codec.transfer; + +import java.io.IOException; +import java.util.List; + +/** + * Transfer quantized binary vectors to off heap memory + * The reason this is different from {@link OffHeapByteVectorTransfer} is because of allocation and deallocation + * of memory on JNI layer. Use this if unsigned int is needed on JNI layer + */ +public final class OffHeapBinaryVectorTransfer extends OffHeapVectorTransfer { + + public OffHeapBinaryVectorTransfer(int transferLimit) { + super(transferLimit); + } + + @Override + public void deallocate() { + // TODO: deallocate the memory location + } + + @Override + protected long transfer(List vectorsToTransfer, boolean append) throws IOException { + // TODO: call to JNIService to transfer vector + return 0L; + } +} diff --git a/src/main/java/org/opensearch/knn/index/codec/transfer/OffHeapByteVectorTransfer.java b/src/main/java/org/opensearch/knn/index/codec/transfer/OffHeapByteVectorTransfer.java new file mode 100644 index 000000000..83ebf2fa3 --- /dev/null +++ b/src/main/java/org/opensearch/knn/index/codec/transfer/OffHeapByteVectorTransfer.java @@ -0,0 +1,38 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.codec.transfer; + +import org.opensearch.knn.jni.JNICommons; + +import java.io.IOException; +import java.util.List; + +/** + * Transfer quantized byte vectors to off heap memory. + * The reason this is different from {@link OffHeapBinaryVectorTransfer} is because of allocation and deallocation + * of memory on JNI layer. Use this if signed int is needed on JNI layer + */ +public final class OffHeapByteVectorTransfer extends OffHeapVectorTransfer { + + public OffHeapByteVectorTransfer(int transferLimit) { + super(transferLimit); + } + + @Override + protected long transfer(List batch, boolean append) throws IOException { + return JNICommons.storeByteVectorData( + getVectorAddress(), + batch.toArray(new byte[][] {}), + (long) batch.get(0).length * transferLimit, + append + ); + } + + @Override + public void deallocate() { + JNICommons.freeByteVectorData(getVectorAddress()); + } +} diff --git a/src/main/java/org/opensearch/knn/index/codec/transfer/OffHeapFloatVectorTransfer.java b/src/main/java/org/opensearch/knn/index/codec/transfer/OffHeapFloatVectorTransfer.java new file mode 100644 index 000000000..0eb28d791 --- /dev/null +++ b/src/main/java/org/opensearch/knn/index/codec/transfer/OffHeapFloatVectorTransfer.java @@ -0,0 +1,36 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.codec.transfer; + +import org.opensearch.knn.jni.JNICommons; + +import java.io.IOException; +import java.util.List; + +/** + * Transfer float vectors to off heap memory. + */ +public final class OffHeapFloatVectorTransfer extends OffHeapVectorTransfer { + + public OffHeapFloatVectorTransfer(int transferLimit) { + super(transferLimit); + } + + @Override + protected long transfer(final List vectorsToTransfer, boolean append) throws IOException { + return JNICommons.storeVectorData( + getVectorAddress(), + vectorsToTransfer.toArray(new float[][] {}), + (long) vectorsToTransfer.get(0).length * this.transferLimit, + append + ); + } + + @Override + public void deallocate() { + JNICommons.freeVectorData(getVectorAddress()); + } +} diff --git a/src/main/java/org/opensearch/knn/index/codec/transfer/OffHeapVectorTransfer.java b/src/main/java/org/opensearch/knn/index/codec/transfer/OffHeapVectorTransfer.java new file mode 100644 index 000000000..43c27c8da --- /dev/null +++ b/src/main/java/org/opensearch/knn/index/codec/transfer/OffHeapVectorTransfer.java @@ -0,0 +1,99 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.codec.transfer; + +import lombok.Getter; +import org.opensearch.knn.index.vectorvalues.KNNVectorValues; + +import java.io.Closeable; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +/** + *

    + * The class is intended to transfer {@link KNNVectorValues} to off heap memory. + *

    + *

    + * The class is not thread safe. + *

    + * + * @param byte[] or float[] + */ +public abstract class OffHeapVectorTransfer implements Closeable { + + @Getter + private long vectorAddress; + protected final int transferLimit; + + private final List vectorsToTransfer; + + public OffHeapVectorTransfer(final int transferLimit) { + this.transferLimit = transferLimit; + this.vectorsToTransfer = new ArrayList<>(transferLimit); + this.vectorAddress = 0; + } + + /** + * Transfer vectors to off-heap + * @param vector float[] or byte[] + * @param append This indicates whether to append or rewrite the off-heap buffer + * @return true of the vectors were transferred, false if not + * @throws IOException + */ + public boolean transfer(T vector, boolean append) throws IOException { + vectorsToTransfer.add(vector); + if (vectorsToTransfer.size() == this.transferLimit) { + vectorAddress = transfer(vectorsToTransfer, append); + vectorsToTransfer.clear(); + return true; + } + return false; + } + + /** + * Empties the {@link #vectorsToTransfer} if its not empty. Intended to be used before + * closing the transfer + * + * @param append This indicates whether to append or rewrite the off-heap buffer + * @return true of the vectors were transferred, false if not + * @throws IOException + */ + public boolean flush(boolean append) throws IOException { + // flush before closing + if (!vectorsToTransfer.isEmpty()) { + vectorAddress = transfer(vectorsToTransfer, append); + vectorsToTransfer.clear(); + return true; + } + return false; + } + + @Override + public void close() { + // Remove this if condition once create and write index is separated for nmslib + if (vectorAddress != 0) { + deallocate(); + } + reset(); + } + + /** + * Resets address and vectortoTransfer + * + * DO NOT USE this in the middle of the transfer, The behavior is undefined + * + * TODO: Make it package private once create and write index is separated for nmslib + */ + public void reset() { + vectorAddress = 0; + vectorsToTransfer.clear(); + } + + protected abstract void deallocate(); + + protected abstract long transfer(final List vectorsToTransfer, boolean append) throws IOException; +} diff --git a/src/main/java/org/opensearch/knn/index/codec/transfer/OffHeapVectorTransferFactory.java b/src/main/java/org/opensearch/knn/index/codec/transfer/OffHeapVectorTransferFactory.java new file mode 100644 index 000000000..bfcc13491 --- /dev/null +++ b/src/main/java/org/opensearch/knn/index/codec/transfer/OffHeapVectorTransferFactory.java @@ -0,0 +1,37 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.codec.transfer; + +import lombok.AccessLevel; +import lombok.NoArgsConstructor; +import org.opensearch.knn.index.VectorDataType; + +/** + * Factory to get the right implementation of vector transfer + */ +@NoArgsConstructor(access = AccessLevel.PRIVATE) +public final class OffHeapVectorTransferFactory { + + /** + * Gets the right vector transfer object based on vector data type + * @param vectorDataType {@link VectorDataType} + * @param transferLimit max number of vectors that can be transferred to off heap in one transfer + * @return Correct implementation of {@link OffHeapVectorTransfer} + * @param float[] or byte[] + */ + public static OffHeapVectorTransfer getVectorTransfer(final VectorDataType vectorDataType, final int transferLimit) { + switch (vectorDataType) { + case FLOAT: + return (OffHeapVectorTransfer) new OffHeapFloatVectorTransfer(transferLimit); + case BINARY: + // TODO: Add binary here + case BYTE: + return (OffHeapVectorTransfer) new OffHeapByteVectorTransfer(transferLimit); + default: + throw new IllegalArgumentException("Unsupported vector data type: " + vectorDataType); + } + } +} diff --git a/src/main/java/org/opensearch/knn/index/codec/transfer/VectorTransfer.java b/src/main/java/org/opensearch/knn/index/codec/transfer/VectorTransfer.java deleted file mode 100644 index c23bd4317..000000000 --- a/src/main/java/org/opensearch/knn/index/codec/transfer/VectorTransfer.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.knn.index.codec.transfer; - -import lombok.Data; -import org.apache.lucene.util.BytesRef; -import org.opensearch.knn.index.codec.util.SerializationMode; - -/** - * Abstract class to transfer vector value from Java to native memory - */ -@Data -public abstract class VectorTransfer { - protected final long vectorsStreamingMemoryLimit; - protected long totalLiveDocs; - protected long vectorsPerTransfer; - protected long vectorAddress; - protected int dimension; - - public VectorTransfer(final long vectorsStreamingMemoryLimit) { - this.vectorsStreamingMemoryLimit = vectorsStreamingMemoryLimit; - this.vectorsPerTransfer = Integer.MIN_VALUE; - } - - /** - * Initialize the transfer - * - * @param totalLiveDocs total number of vectors to be transferred - */ - abstract public void init(final long totalLiveDocs); - - /** - * Transfer a single vector - * - * @param bytesRef a vector in bytes format - */ - abstract public void transfer(final BytesRef bytesRef); - - /** - * Close the transfer - */ - abstract public void close(); - - /** - * Get serialization mode of given byte stream - * - * @param bytesRef bytes of a vector - * @return serialization mode - */ - abstract public SerializationMode getSerializationMode(final BytesRef bytesRef); -} diff --git a/src/main/java/org/opensearch/knn/index/codec/transfer/VectorTransferByte.java b/src/main/java/org/opensearch/knn/index/codec/transfer/VectorTransferByte.java deleted file mode 100644 index e81ac35fc..000000000 --- a/src/main/java/org/opensearch/knn/index/codec/transfer/VectorTransferByte.java +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.knn.index.codec.transfer; - -import org.apache.lucene.util.ArrayUtil; -import org.apache.lucene.util.BytesRef; -import org.opensearch.knn.index.codec.util.SerializationMode; -import org.opensearch.knn.jni.JNICommons; - -import java.util.ArrayList; -import java.util.List; - -/** - * Vector transfer for byte - */ -public class VectorTransferByte extends VectorTransfer { - private List vectorList; - - public VectorTransferByte(final long vectorsStreamingMemoryLimit) { - super(vectorsStreamingMemoryLimit); - vectorList = new ArrayList<>(); - } - - @Override - public void init(final long totalLiveDocs) { - this.totalLiveDocs = totalLiveDocs; - vectorList.clear(); - } - - @Override - public void transfer(final BytesRef bytesRef) { - dimension = bytesRef.length * 8; - if (vectorsPerTransfer == Integer.MIN_VALUE) { - // if vectorsStreamingMemoryLimit is 100 bytes and we have 50 vectors with length of 5, then per - // transfer we have to send 100/5 => 20 vectors. - vectorsPerTransfer = vectorsStreamingMemoryLimit / bytesRef.length; - // If vectorsPerTransfer comes out to be 0, then we set number of vectors per transfer to 1, to ensure that - // we are sending minimum number of vectors. - if (vectorsPerTransfer == 0) { - vectorsPerTransfer = 1; - } - } - - vectorList.add(ArrayUtil.copyOfSubArray(bytesRef.bytes, bytesRef.offset, bytesRef.offset + bytesRef.length)); - if (vectorList.size() == vectorsPerTransfer) { - transfer(); - } - } - - @Override - public void close() { - transfer(); - } - - @Override - public SerializationMode getSerializationMode(final BytesRef bytesRef) { - return SerializationMode.COLLECTIONS_OF_BYTES; - } - - private void transfer() { - int lengthOfVector = dimension / 8; - vectorAddress = JNICommons.storeByteVectorData(vectorAddress, vectorList.toArray(new byte[][] {}), totalLiveDocs * lengthOfVector); - vectorList.clear(); - } -} diff --git a/src/main/java/org/opensearch/knn/index/codec/transfer/VectorTransferFloat.java b/src/main/java/org/opensearch/knn/index/codec/transfer/VectorTransferFloat.java deleted file mode 100644 index a9c792398..000000000 --- a/src/main/java/org/opensearch/knn/index/codec/transfer/VectorTransferFloat.java +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.knn.index.codec.transfer; - -import org.apache.lucene.util.BytesRef; -import org.opensearch.knn.index.codec.util.KNNVectorSerializer; -import org.opensearch.knn.index.codec.util.KNNVectorSerializerFactory; -import org.opensearch.knn.index.codec.util.SerializationMode; -import org.opensearch.knn.jni.JNICommons; - -import java.util.ArrayList; -import java.util.List; - -/** - * Vector transfer for float - */ -public class VectorTransferFloat extends VectorTransfer { - private List vectorList; - - public VectorTransferFloat(final long vectorsStreamingMemoryLimit) { - super(vectorsStreamingMemoryLimit); - vectorList = new ArrayList<>(); - } - - @Override - public void init(final long totalLiveDocs) { - this.totalLiveDocs = totalLiveDocs; - vectorList.clear(); - } - - @Override - public void transfer(final BytesRef bytesRef) { - final KNNVectorSerializer vectorSerializer = KNNVectorSerializerFactory.getSerializerByBytesRef(bytesRef); - final float[] vector = vectorSerializer.byteToFloatArray(bytesRef); - dimension = vector.length; - - if (vectorsPerTransfer == Integer.MIN_VALUE) { - // if vectorsStreamingMemoryLimit is 100 bytes and we have 50 vectors with 5 dimension, then per - // transfer we have to send 100/(5 * 4) => 5 vectors. - vectorsPerTransfer = vectorsStreamingMemoryLimit / ((long) dimension * Float.BYTES); - // If vectorsPerTransfer comes out to be 0, then we set number of vectors per transfer to 1, to ensure that - // we are sending minimum number of vectors. - if (vectorsPerTransfer == 0) { - vectorsPerTransfer = 1; - } - } - - vectorList.add(vector); - if (vectorList.size() == vectorsPerTransfer) { - transfer(); - } - } - - @Override - public void close() { - transfer(); - } - - @Override - public SerializationMode getSerializationMode(final BytesRef bytesRef) { - return KNNVectorSerializerFactory.getSerializerModeFromBytesRef(bytesRef); - } - - private void transfer() { - vectorAddress = JNICommons.storeVectorData(vectorAddress, vectorList.toArray(new float[][] {}), totalLiveDocs * dimension); - vectorList.clear(); - } -} diff --git a/src/main/java/org/opensearch/knn/index/codec/util/KNNCodecUtil.java b/src/main/java/org/opensearch/knn/index/codec/util/KNNCodecUtil.java index ea14fe883..51100a1e0 100644 --- a/src/main/java/org/opensearch/knn/index/codec/util/KNNCodecUtil.java +++ b/src/main/java/org/opensearch/knn/index/codec/util/KNNCodecUtil.java @@ -5,63 +5,14 @@ package org.opensearch.knn.index.codec.util; -import lombok.AllArgsConstructor; -import lombok.Getter; -import lombok.Setter; import org.apache.lucene.index.BinaryDocValues; -import org.apache.lucene.search.DocIdSetIterator; -import org.apache.lucene.util.BytesRef; import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.codec.KNN80Codec.KNN80BinaryDocValues; -import org.opensearch.knn.index.codec.transfer.VectorTransfer; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; public class KNNCodecUtil { // Floats are 4 bytes in size public static final int FLOAT_BYTE_SIZE = 4; - @AllArgsConstructor - public static final class Pair { - public int[] docs; - @Getter - @Setter - private long vectorAddress; - @Getter - @Setter - private int dimension; - public SerializationMode serializationMode; - } - - /** - * Extract docIds and vectors from binary doc values. - * - * @param values Binary doc values - * @param vectorTransfer Utility to make transfer - * @return KNNCodecUtil.Pair representing doc ids and corresponding vectors - * @throws IOException thrown when unable to get binary of vectors - */ - public static KNNCodecUtil.Pair getPair(final BinaryDocValues values, final VectorTransfer vectorTransfer) throws IOException { - List docIdList = new ArrayList<>(); - SerializationMode serializationMode = SerializationMode.COLLECTION_OF_FLOATS; - vectorTransfer.init(getTotalLiveDocsCount(values)); - for (int doc = values.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = values.nextDoc()) { - BytesRef bytesref = values.binaryValue(); - serializationMode = vectorTransfer.getSerializationMode(bytesref); - vectorTransfer.transfer(bytesref); - docIdList.add(doc); - } - vectorTransfer.close(); - return new KNNCodecUtil.Pair( - docIdList.stream().mapToInt(Integer::intValue).toArray(), - vectorTransfer.getVectorAddress(), - vectorTransfer.getDimension(), - serializationMode - ); - } - /** * This method provides a rough estimate of the number of bytes used for storing an array with the given parameters. * @param numVectors number of vectors in the array diff --git a/src/main/java/org/opensearch/knn/index/vectorvalues/KNNBinaryVectorValues.java b/src/main/java/org/opensearch/knn/index/vectorvalues/KNNBinaryVectorValues.java index f38099b74..5da093fd5 100644 --- a/src/main/java/org/opensearch/knn/index/vectorvalues/KNNBinaryVectorValues.java +++ b/src/main/java/org/opensearch/knn/index/vectorvalues/KNNBinaryVectorValues.java @@ -11,6 +11,7 @@ import org.apache.lucene.index.ByteVectorValues; import java.io.IOException; +import java.util.Arrays; /** * Concrete implementation of {@link KNNVectorValues} that returns byte[] as vector where binary vector is stored and @@ -25,17 +26,17 @@ public class KNNBinaryVectorValues extends KNNVectorValues { @Override public byte[] getVector() throws IOException { final byte[] vector = VectorValueExtractorStrategy.extractBinaryVector(vectorValuesIterator); - this.dimension = vector.length; + this.dimension = vector.length * Byte.SIZE; + this.bytesPerVector = vector.length; return vector; } - /** - * Binary Vector values gets stored as byte[], hence for dimension of the binary vector we have to multiply the - * byte[] size with {@link Byte#SIZE} - * @return int - */ @Override - public int dimension() { - return super.dimension() * Byte.SIZE; + public byte[] conditionalCloneVector() throws IOException { + byte[] vector = getVector(); + if (vectorValuesIterator.getDocIdSetIterator() instanceof ByteVectorValues) { + return Arrays.copyOf(vector, vector.length); + } + return vector; } } diff --git a/src/main/java/org/opensearch/knn/index/vectorvalues/KNNByteVectorValues.java b/src/main/java/org/opensearch/knn/index/vectorvalues/KNNByteVectorValues.java index ccbbfab77..1ebc50970 100644 --- a/src/main/java/org/opensearch/knn/index/vectorvalues/KNNByteVectorValues.java +++ b/src/main/java/org/opensearch/knn/index/vectorvalues/KNNByteVectorValues.java @@ -11,6 +11,7 @@ import org.apache.lucene.index.ByteVectorValues; import java.io.IOException; +import java.util.Arrays; /** * Concrete implementation of {@link KNNVectorValues} that returns float[] as vector and provides an abstraction over @@ -26,6 +27,17 @@ public class KNNByteVectorValues extends KNNVectorValues { public byte[] getVector() throws IOException { final byte[] vector = VectorValueExtractorStrategy.extractByteVector(vectorValuesIterator); this.dimension = vector.length; + this.bytesPerVector = vector.length; + return vector; + } + + @Override + public byte[] conditionalCloneVector() throws IOException { + byte[] vector = getVector(); + if (vectorValuesIterator.getDocIdSetIterator() instanceof ByteVectorValues) { + return Arrays.copyOf(vector, vector.length); + + } return vector; } } diff --git a/src/main/java/org/opensearch/knn/index/vectorvalues/KNNFloatVectorValues.java b/src/main/java/org/opensearch/knn/index/vectorvalues/KNNFloatVectorValues.java index 174f3a89e..dffdd8f0d 100644 --- a/src/main/java/org/opensearch/knn/index/vectorvalues/KNNFloatVectorValues.java +++ b/src/main/java/org/opensearch/knn/index/vectorvalues/KNNFloatVectorValues.java @@ -10,6 +10,7 @@ import org.apache.lucene.index.FloatVectorValues; import java.io.IOException; +import java.util.Arrays; /** * Concrete implementation of {@link KNNVectorValues} that returns float[] as vector and provides an abstraction over @@ -24,6 +25,16 @@ public class KNNFloatVectorValues extends KNNVectorValues { public float[] getVector() throws IOException { final float[] vector = VectorValueExtractorStrategy.extractFloatVector(vectorValuesIterator); this.dimension = vector.length; + this.bytesPerVector = vector.length * 4; + return vector; + } + + @Override + public float[] conditionalCloneVector() throws IOException { + float[] vector = getVector(); + if (vectorValuesIterator.getDocIdSetIterator() instanceof FloatVectorValues) { + return Arrays.copyOf(vector, vector.length); + } return vector; } } diff --git a/src/main/java/org/opensearch/knn/index/vectorvalues/KNNVectorValues.java b/src/main/java/org/opensearch/knn/index/vectorvalues/KNNVectorValues.java index c4ed64bc2..56ebd208f 100644 --- a/src/main/java/org/opensearch/knn/index/vectorvalues/KNNVectorValues.java +++ b/src/main/java/org/opensearch/knn/index/vectorvalues/KNNVectorValues.java @@ -23,6 +23,7 @@ public abstract class KNNVectorValues { protected final KNNVectorValuesIterator vectorValuesIterator; protected int dimension; + protected int bytesPerVector; protected KNNVectorValues(final KNNVectorValuesIterator vectorValuesIterator) { this.vectorValuesIterator = vectorValuesIterator; @@ -37,6 +38,20 @@ protected KNNVectorValues(final KNNVectorValuesIterator vectorValuesIterator) { */ public abstract T getVector() throws IOException; + /** + * Intended to return a vector reference either after deep copy of the vector obtained from {@code getVector} + * or return the vector itself. + *

    + * This decision to clone depends on the vector returned based on the type of iterator + *

    + * Running this function can incur latency hence should be absolutely used when necessary. + * For most of the cases {@link #getVector()} function should work. + * + * @return T an array of byte[], float[] Or a deep copy of it + * @throws IOException + */ + public abstract T conditionalCloneVector() throws IOException; + /** * Dimension of vector is returned. Do call getVector function first before calling this function otherwise you will get 0 value. * @return int @@ -46,6 +61,15 @@ public int dimension() { return dimension; } + /** + * Size of a vector in bytes is returned. Do call getVector function first before calling this function otherwise you will get 0 value. + * @return int + */ + public int bytesPerVector() { + assert docId() != -1 && bytesPerVector != 0 : "Cannot get bytesPerVector before we retrieve a vector from KNNVectorValues"; + return bytesPerVector; + } + /** * Returns the total live docs for KNNVectorValues. * @return long @@ -81,5 +105,4 @@ public int advance(int docId) throws IOException { public int nextDoc() throws IOException { return vectorValuesIterator.nextDoc(); } - } diff --git a/src/main/java/org/opensearch/knn/indices/ModelUtil.java b/src/main/java/org/opensearch/knn/indices/ModelUtil.java index 0f5a049fc..ac0e4fb79 100644 --- a/src/main/java/org/opensearch/knn/indices/ModelUtil.java +++ b/src/main/java/org/opensearch/knn/indices/ModelUtil.java @@ -11,6 +11,7 @@ package org.opensearch.knn.indices; +import lombok.experimental.UtilityClass; import org.apache.commons.lang.StringUtils; import java.util.Locale; @@ -18,6 +19,7 @@ /** * A utility class for models. */ +@UtilityClass public class ModelUtil { public static void blockCommasInModelDescription(String description) { @@ -48,7 +50,7 @@ public static ModelMetadata getModelMetadata(final String modelId) { } final Model model = ModelCache.getInstance().get(modelId); final ModelMetadata modelMetadata = model.getModelMetadata(); - if (ModelUtil.isModelCreated(modelMetadata) == false) { + if (isModelCreated(modelMetadata) == false) { throw new IllegalArgumentException(String.format(Locale.ROOT, "Model ID '%s' is not created.", modelId)); } return modelMetadata; diff --git a/src/main/java/org/opensearch/knn/jni/FaissService.java b/src/main/java/org/opensearch/knn/jni/FaissService.java index 4f57b616a..a402be1f3 100644 --- a/src/main/java/org/opensearch/knn/jni/FaissService.java +++ b/src/main/java/org/opensearch/knn/jni/FaissService.java @@ -50,32 +50,70 @@ class FaissService { } /** - * Create an index for the native library The memory occupied by the vectorsAddress will be freed up during the + * Initialize an index for the native library. Takes in numDocs to + * allocate the correct amount of memory. + * + * @param numDocs number of documents to be added + * @param dim dimension of the vector to be indexed + * @param parameters parameters to build index + */ + public static native long initIndex(long numDocs, int dim, Map parameters); + + /** + * Initialize an index for the native library. Takes in numDocs to + * allocate the correct amount of memory. + * + * @param numDocs number of documents to be added + * @param dim dimension of the vector to be indexed + * @param parameters parameters to build index + */ + public static native long initBinaryIndex(long numDocs, int dim, Map parameters); + + /** + * Inserts to a faiss index. The memory occupied by the vectorsAddress will be freed up during the * function call. So Java layer doesn't need to free up the memory. This is not an ideal behavior because Java layer - * created the memory address and that should only free up the memory. We are tracking the proper fix for this on this - * issue + * created the memory address and that should only free up the memory. * - * @param ids array of ids mapping to the data passed in + * @param ids ids of documents * @param vectorsAddress address of native memory where vectors are stored * @param dim dimension of the vector to be indexed - * @param indexPath path to save index file to - * @param parameters parameters to build index + * @param indexAddress address of native memory where index is stored + * @param threadCount number of threads to use for insertion */ - public static native void createIndex(int[] ids, long vectorsAddress, int dim, String indexPath, Map parameters); + public static native void insertToIndex(int[] ids, long vectorsAddress, int dim, long indexAddress, int threadCount); /** - * Create a binary index for the native library The memory occupied by the vectorsAddress will be freed up during the + * Inserts to a faiss index. The memory occupied by the vectorsAddress will be freed up during the * function call. So Java layer doesn't need to free up the memory. This is not an ideal behavior because Java layer - * created the memory address and that should only free up the memory. We are tracking the proper fix for this on this - * issue + * created the memory address and that should only free up the memory. * - * @param ids array of ids mapping to the data passed in + * @param ids ids of documents * @param vectorsAddress address of native memory where vectors are stored * @param dim dimension of the vector to be indexed + * @param indexAddress address of native memory where index is stored + * @param threadCount number of threads to use for insertion + */ + public static native void insertToBinaryIndex(int[] ids, long vectorsAddress, int dim, long indexAddress, int threadCount); + + /** + * Writes a faiss index. + * + * NOTE: This will always free the index. Do not call free after this. + * + * @param indexAddress address of native memory where index is stored + * @param indexPath path to save index file to + */ + public static native void writeIndex(long indexAddress, String indexPath); + + /** + * Writes a faiss index. + * + * NOTE: This will always free the index. Do not call free after this. + * + * @param indexAddress address of native memory where index is stored * @param indexPath path to save index file to - * @param parameters parameters to build index */ - public static native void createBinaryIndex(int[] ids, long vectorsAddress, int dim, String indexPath, Map parameters); + public static native void writeBinaryIndex(long indexAddress, String indexPath); /** * Create an index for the native library with a provided template index diff --git a/src/main/java/org/opensearch/knn/jni/JNICommons.java b/src/main/java/org/opensearch/knn/jni/JNICommons.java index 31a8f43cc..c7222738e 100644 --- a/src/main/java/org/opensearch/knn/jni/JNICommons.java +++ b/src/main/java/org/opensearch/knn/jni/JNICommons.java @@ -36,16 +36,59 @@ public class JNICommons { * will throw Exception. * *

    - * The function is not threadsafe. If multiple threads are trying to insert on same memory location, then it can - * lead to data corruption. + * The function is not threadsafe. If multiple threads are trying to insert on same memory location, then it can + * lead to data corruption. *

    * - * @param memoryAddress The address of the memory location where data will be stored. - * @param data 2D float array containing data to be stored in native memory. + * @param memoryAddress The address of the memory location where data will be stored. + * @param data 2D float array containing data to be stored in native memory. * @param initialCapacity The initial capacity of the memory location. * @return memory address where the data is stored. */ - public static native long storeVectorData(long memoryAddress, float[][] data, long initialCapacity); + public static long storeVectorData(long memoryAddress, float[][] data, long initialCapacity) { + return storeVectorData(memoryAddress, data, initialCapacity, true); + } + + /** + * This is utility function that can be used to store data in native memory. This function will allocate memory for + * the data(rows*columns) with initialCapacity and return the memory address where the data is stored. + * If you are using this function for first time use memoryAddress = 0 to ensure that a new memory location is created. + * For subsequent calls you can pass the same memoryAddress. If the data cannot be stored in the memory location + * will throw Exception. + * + *

    + * The function is not threadsafe. If multiple threads are trying to insert on same memory location, then it can + * lead to data corruption. + *

    + * + * @param memoryAddress The address of the memory location where data will be stored. + * @param data 2D float array containing data to be stored in native memory. + * @param initialCapacity The initial capacity of the memory location. + * @param append append the data or rewrite the memory location + * @return memory address where the data is stored. + */ + public static native long storeVectorData(long memoryAddress, float[][] data, long initialCapacity, boolean append); + + /** + * This is utility function that can be used to store data in native memory. This function will allocate memory for + * the data(rows*columns) with initialCapacity and return the memory address where the data is stored. + * If you are using this function for first time use memoryAddress = 0 to ensure that a new memory location is created. + * For subsequent calls you can pass the same memoryAddress. If the data cannot be stored in the memory location + * will throw Exception. + * + *

    + * The function is not threadsafe. If multiple threads are trying to insert on same memory location, then it can + * lead to data corruption. + *

    + * + * @param memoryAddress The address of the memory location where data will be stored. + * @param data 2D byte array containing data to be stored in native memory. + * @param initialCapacity The initial capacity of the memory location. + * @return memory address where the data is stored. + */ + public static long storeByteVectorData(long memoryAddress, byte[][] data, long initialCapacity) { + return storeByteVectorData(memoryAddress, data, initialCapacity, true); + } /** * This is utility function that can be used to store data in native memory. This function will allocate memory for @@ -55,24 +98,25 @@ public class JNICommons { * will throw Exception. * *

    - * The function is not threadsafe. If multiple threads are trying to insert on same memory location, then it can - * lead to data corruption. + * The function is not threadsafe. If multiple threads are trying to insert on same memory location, then it can + * lead to data corruption. *

    * - * @param memoryAddress The address of the memory location where data will be stored. - * @param data 2D byte array containing data to be stored in native memory. + * @param memoryAddress The address of the memory location where data will be stored. + * @param data 2D byte array containing data to be stored in native memory. * @param initialCapacity The initial capacity of the memory location. + * @param append append the data or rewrite the memory location * @return memory address where the data is stored. */ - public static native long storeByteVectorData(long memoryAddress, byte[][] data, long initialCapacity); + public static native long storeByteVectorData(long memoryAddress, byte[][] data, long initialCapacity, boolean append); /** * Free up the memory allocated for the data stored in memory address. This function should be used with the memory - * address returned by {@link JNICommons#storeVectorData(long, float[][], long)} + * address returned by {@link JNICommons#storeVectorData(long, float[][], long, boolean)} * *

    - * The function is not threadsafe. If multiple threads are trying to free up same memory location, then it can - * lead to errors. + * The function is not threadsafe. If multiple threads are trying to free up same memory location, then it can + * lead to errors. *

    * * @param memoryAddress address to be freed. @@ -81,11 +125,11 @@ public class JNICommons { /** * Free up the memory allocated for the byte data stored in memory address. This function should be used with the memory - * address returned by {@link JNICommons#storeVectorData(long, float[][], long)} + * address returned by {@link JNICommons#storeVectorData(long, float[][], long, boolean)} * *

    - * The function is not threadsafe. If multiple threads are trying to free up same memory location, then it can - * lead to errors. + * The function is not threadsafe. If multiple threads are trying to free up same memory location, then it can + * lead to errors. *

    * * @param memoryAddress address to be freed. diff --git a/src/main/java/org/opensearch/knn/jni/JNIService.java b/src/main/java/org/opensearch/knn/jni/JNIService.java index de696b5ce..d1d5f6c11 100644 --- a/src/main/java/org/opensearch/knn/jni/JNIService.java +++ b/src/main/java/org/opensearch/knn/jni/JNIService.java @@ -13,28 +13,110 @@ import org.apache.commons.lang.ArrayUtils; import org.opensearch.common.Nullable; -import org.opensearch.knn.index.util.IndexUtil; -import org.opensearch.knn.index.query.KNNQueryResult; +import org.opensearch.knn.common.KNNConstants; import org.opensearch.knn.index.engine.KNNEngine; +import org.opensearch.knn.index.query.KNNQueryResult; +import org.opensearch.knn.index.util.IndexUtil; +import java.util.Locale; import java.util.Map; /** * Service to distribute requests to the proper engine jni service */ public class JNIService { + /** + * Initialize an index for the native library. Takes in numDocs to + * allocate the correct amount of memory. + * + * @param numDocs number of documents to be added + * @param dim dimension of the vector to be indexed + * @param parameters parameters to build index + * @param knnEngine knn engine + * @return address of the index in memory + */ + public static long initIndex(long numDocs, int dim, Map parameters, KNNEngine knnEngine) { + if (KNNEngine.FAISS == knnEngine) { + if (IndexUtil.isBinaryIndex(knnEngine, parameters)) { + return FaissService.initBinaryIndex(numDocs, dim, parameters); + } else { + return FaissService.initIndex(numDocs, dim, parameters); + } + } + + throw new IllegalArgumentException( + String.format(Locale.ROOT, "initIndexFromScratch not supported for provided engine : %s", knnEngine.getName()) + ); + } + + /** + * Inserts to a faiss index. + * + * @param docs ids of documents + * @param vectorsAddress address of native memory where vectors are stored + * @param dimension dimension of the vector to be indexed + * @param parameters parameters to build index + * @param indexAddress address of native memory where index is stored + * @param knnEngine knn engine + */ + public static void insertToIndex( + int[] docs, + long vectorsAddress, + int dimension, + Map parameters, + long indexAddress, + KNNEngine knnEngine + ) { + int threadCount = (int) parameters.getOrDefault(KNNConstants.INDEX_THREAD_QTY, 0); + if (KNNEngine.FAISS == knnEngine) { + if (IndexUtil.isBinaryIndex(knnEngine, parameters)) { + FaissService.insertToBinaryIndex(docs, vectorsAddress, dimension, indexAddress, threadCount); + } else { + FaissService.insertToIndex(docs, vectorsAddress, dimension, indexAddress, threadCount); + } + return; + } + + throw new IllegalArgumentException( + String.format(Locale.ROOT, "insertToIndex not supported for provided engine : %s", knnEngine.getName()) + ); + } + + /** + * Writes a faiss index to disk. + * + * @param indexPath path to save index to + * @param indexAddress address of native memory where index is stored + * @param knnEngine knn engine + * @param parameters parameters to build index + */ + public static void writeIndex(String indexPath, long indexAddress, KNNEngine knnEngine, Map parameters) { + if (KNNEngine.FAISS == knnEngine) { + if (IndexUtil.isBinaryIndex(knnEngine, parameters)) { + FaissService.writeBinaryIndex(indexAddress, indexPath); + } else { + FaissService.writeIndex(indexAddress, indexPath); + } + return; + } + + throw new IllegalArgumentException( + String.format(Locale.ROOT, "writeIndex not supported for provided engine : %s", knnEngine.getName()) + ); + } + /** * Create an index for the native library. The memory occupied by the vectorsAddress will be freed up during the * function call. So Java layer doesn't need to free up the memory. This is not an ideal behavior because Java layer * created the memory address and that should only free up the memory. We are tracking the proper fix for this on this * issue * - * @param ids array of ids mapping to the data passed in + * @param ids array of ids mapping to the data passed in * @param vectorsAddress address of native memory where vectors are stored - * @param dim dimension of the vector to be indexed - * @param indexPath path to save index file to - * @param parameters parameters to build index - * @param knnEngine engine to build index for + * @param dim dimension of the vector to be indexed + * @param indexPath path to save index file to + * @param parameters parameters to build index + * @param knnEngine engine to build index for */ public static void createIndex( int[] ids, @@ -50,28 +132,21 @@ public static void createIndex( return; } - if (KNNEngine.FAISS == knnEngine) { - if (IndexUtil.isBinaryIndex(knnEngine, parameters)) { - FaissService.createBinaryIndex(ids, vectorsAddress, dim, indexPath, parameters); - } else { - FaissService.createIndex(ids, vectorsAddress, dim, indexPath, parameters); - } - return; - } - - throw new IllegalArgumentException(String.format("CreateIndex not supported for provided engine : %s", knnEngine.getName())); + throw new IllegalArgumentException( + String.format(Locale.ROOT, "CreateIndex not supported for provided engine : %s", knnEngine.getName()) + ); } /** * Create an index for the native library with a provided template index * - * @param ids array of ids mapping to the data passed in + * @param ids array of ids mapping to the data passed in * @param vectorsAddress address of native memory where vectors are stored - * @param dim dimension of vectors to be indexed - * @param indexPath path to save index file to - * @param templateIndex empty template index - * @param parameters parameters to build index - * @param knnEngine engine to build index for + * @param dim dimension of vectors to be indexed + * @param indexPath path to save index file to + * @param templateIndex empty template index + * @param parameters parameters to build index + * @param knnEngine engine to build index for */ public static void createIndexFromTemplate( int[] ids, @@ -93,7 +168,7 @@ public static void createIndexFromTemplate( } throw new IllegalArgumentException( - String.format("CreateIndexFromTemplate not supported for provided engine : %s", knnEngine.getName()) + String.format(Locale.ROOT, "CreateIndexFromTemplate not supported for provided engine : %s", knnEngine.getName()) ); } @@ -118,7 +193,9 @@ public static long loadIndex(String indexPath, Map parameters, K } } - throw new IllegalArgumentException(String.format("LoadIndex not supported for provided engine : %s", knnEngine.getName())); + throw new IllegalArgumentException( + String.format(Locale.ROOT, "LoadIndex not supported for provided engine : %s", knnEngine.getName()) + ); } /** @@ -150,7 +227,7 @@ public static long initSharedIndexState(long indexAddr, KNNEngine knnEngine) { return FaissService.initSharedIndexState(indexAddr); } throw new IllegalArgumentException( - String.format("InitSharedIndexState not supported for provided engine : %s", knnEngine.getName()) + String.format(Locale.ROOT, "InitSharedIndexState not supported for provided engine : %s", knnEngine.getName()) ); } @@ -168,20 +245,20 @@ public static void setSharedIndexState(long indexAddr, long shareIndexStateAddr, } throw new IllegalArgumentException( - String.format("SetSharedIndexState not supported for provided engine : %s", knnEngine.getName()) + String.format(Locale.ROOT, "SetSharedIndexState not supported for provided engine : %s", knnEngine.getName()) ); } /** * Query an index * - * @param indexPointer pointer to index in memory - * @param queryVector vector to be used for query - * @param k neighbors to be returned - * @param methodParameters method parameter - * @param knnEngine engine to query index - * @param filteredIds array of ints on which should be used for search. - * @param filterIdsType how to filter ids: Batch or BitMap + * @param indexPointer pointer to index in memory + * @param queryVector vector to be used for query + * @param k neighbors to be returned + * @param methodParameters method parameter + * @param knnEngine engine to query index + * @param filteredIds array of ints on which should be used for search. + * @param filterIdsType how to filter ids: Batch or BitMap * @return KNNQueryResult array of k neighbors */ public static KNNQueryResult[] queryIndex( @@ -216,19 +293,21 @@ public static KNNQueryResult[] queryIndex( } return FaissService.queryIndex(indexPointer, queryVector, k, methodParameters, parentIds); } - throw new IllegalArgumentException(String.format("QueryIndex not supported for provided engine : %s", knnEngine.getName())); + throw new IllegalArgumentException( + String.format(Locale.ROOT, "QueryIndex not supported for provided engine : %s", knnEngine.getName()) + ); } /** * Query a binary index * - * @param indexPointer pointer to index in memory - * @param queryVector vector to be used for query - * @param k neighbors to be returned - * @param methodParameters method parameter - * @param knnEngine engine to query index - * @param filteredIds array of ints on which should be used for search. - * @param filterIdsType how to filter ids: Batch or BitMap + * @param indexPointer pointer to index in memory + * @param queryVector vector to be used for query + * @param k neighbors to be returned + * @param methodParameters method parameter + * @param knnEngine engine to query index + * @param filteredIds array of ints on which should be used for search. + * @param filterIdsType how to filter ids: Batch or BitMap * @return KNNQueryResult array of k neighbors */ public static KNNQueryResult[] queryBinaryIndex( @@ -252,7 +331,9 @@ public static KNNQueryResult[] queryBinaryIndex( parentIds ); } - throw new IllegalArgumentException(String.format("QueryBinaryIndex not supported for provided engine : %s", knnEngine.getName())); + throw new IllegalArgumentException( + String.format(Locale.ROOT, "QueryBinaryIndex not supported for provided engine : %s", knnEngine.getName()) + ); } /** @@ -283,7 +364,7 @@ public static void free(final long indexPointer, final KNNEngine knnEngine, fina return; } - throw new IllegalArgumentException(String.format("Free not supported for provided engine : %s", knnEngine.getName())); + throw new IllegalArgumentException(String.format(Locale.ROOT, "Free not supported for provided engine : %s", knnEngine.getName())); } /** @@ -298,7 +379,7 @@ public static void freeSharedIndexState(long shareIndexStateAddr, KNNEngine knnE return; } throw new IllegalArgumentException( - String.format("FreeSharedIndexState not supported for provided engine : %s", knnEngine.getName()) + String.format(Locale.ROOT, "FreeSharedIndexState not supported for provided engine : %s", knnEngine.getName()) ); } @@ -319,17 +400,19 @@ public static byte[] trainIndex(Map indexParameters, int dimensi return FaissService.trainIndex(indexParameters, dimension, trainVectorsPointer); } - throw new IllegalArgumentException(String.format("TrainIndex not supported for provided engine : %s", knnEngine.getName())); + throw new IllegalArgumentException( + String.format(Locale.ROOT, "TrainIndex not supported for provided engine : %s", knnEngine.getName()) + ); } /** *

    - * The function is deprecated. Use {@link JNICommons#storeVectorData(long, float[][], long)} + * The function is deprecated. Use {@link JNICommons#storeVectorData(long, float[][], long, boolean)} *

    * Transfer vectors from Java to native * * @param vectorsPointer pointer to vectors in native memory. Should be 0 to create vector as well - * @param trainingData data to be transferred + * @param trainingData data to be transferred * @return pointer to native memory location of training data */ @Deprecated(since = "2.14.0", forRemoval = true) @@ -340,15 +423,15 @@ public static long transferVectors(long vectorsPointer, float[][] trainingData) /** * Range search index for a given query vector * - * @param indexPointer pointer to index in memory - * @param queryVector vector to be used for query - * @param radius search within radius threshold - * @param methodParameters parameters to be used when loading index - * @param knnEngine engine to query index + * @param indexPointer pointer to index in memory + * @param queryVector vector to be used for query + * @param radius search within radius threshold + * @param methodParameters parameters to be used when loading index + * @param knnEngine engine to query index * @param indexMaxResultWindow maximum number of results to return - * @param filteredIds list of doc ids to include in the query result - * @param filterIdsType how to filter ids: Batch or BitMap - * @param parentIds parent ids of the vectors + * @param filteredIds list of doc ids to include in the query result + * @param filterIdsType how to filter ids: Batch or BitMap + * @param parentIds parent ids of the vectors * @return KNNQueryResult array of neighbors within radius */ public static KNNQueryResult[] radiusQueryIndex( @@ -377,6 +460,6 @@ public static KNNQueryResult[] radiusQueryIndex( } return FaissService.rangeSearchIndex(indexPointer, queryVector, radius, methodParameters, indexMaxResultWindow, parentIds); } - throw new IllegalArgumentException("RadiusQueryIndex not supported for provided engine"); + throw new IllegalArgumentException(String.format(Locale.ROOT, "RadiusQueryIndex not supported for provided engine")); } } diff --git a/src/test/java/org/opensearch/knn/common/FieldInfoExtractorTests.java b/src/test/java/org/opensearch/knn/common/FieldInfoExtractorTests.java index df529bb47..27aedd1d0 100644 --- a/src/test/java/org/opensearch/knn/common/FieldInfoExtractorTests.java +++ b/src/test/java/org/opensearch/knn/common/FieldInfoExtractorTests.java @@ -14,6 +14,8 @@ import org.opensearch.knn.indices.ModelMetadata; import org.opensearch.knn.indices.ModelUtil; +import static org.mockito.Mockito.when; + public class FieldInfoExtractorTests extends KNNTestCase { private static final String MODEL_ID = "model_id"; @@ -39,4 +41,26 @@ public void testExtractVectorDataType_whenDifferentConditions_thenSuccess() { Assert.assertEquals(VectorDataType.BYTE, FieldInfoExtractor.extractVectorDataType(fieldInfo)); } } + + public void testExtractVectorDataType() { + FieldInfo fieldInfo = Mockito.mock(FieldInfo.class); + when(fieldInfo.getAttribute("data_type")).thenReturn(VectorDataType.BINARY.getValue()); + + assertEquals(VectorDataType.BINARY, FieldInfoExtractor.extractVectorDataType(fieldInfo)); + when(fieldInfo.getAttribute("data_type")).thenReturn(null); + + when(fieldInfo.getAttribute("model_id")).thenReturn(MODEL_ID); + try (MockedStatic modelUtilMockedStatic = Mockito.mockStatic(ModelUtil.class)) { + ModelMetadata modelMetadata = Mockito.mock(ModelMetadata.class); + modelUtilMockedStatic.when(() -> ModelUtil.getModelMetadata(MODEL_ID)).thenReturn(modelMetadata); + when(modelMetadata.getVectorDataType()).thenReturn(VectorDataType.BYTE); + + assertEquals(VectorDataType.BYTE, FieldInfoExtractor.extractVectorDataType(fieldInfo)); + when(modelMetadata.getVectorDataType()).thenReturn(null); + when(modelMetadata.getVectorDataType()).thenReturn(VectorDataType.DEFAULT); + } + + when(fieldInfo.getAttribute("model_id")).thenReturn(null); + assertEquals(VectorDataType.DEFAULT, FieldInfoExtractor.extractVectorDataType(fieldInfo)); + } } diff --git a/src/test/java/org/opensearch/knn/common/KNNVectorUtilTests.java b/src/test/java/org/opensearch/knn/common/KNNVectorUtilTests.java index 457ea8c5b..d64b73c9a 100644 --- a/src/test/java/org/opensearch/knn/common/KNNVectorUtilTests.java +++ b/src/test/java/org/opensearch/knn/common/KNNVectorUtilTests.java @@ -11,7 +11,16 @@ package org.opensearch.knn.common; +import lombok.SneakyThrows; import org.opensearch.knn.KNNTestCase; +import org.opensearch.knn.index.VectorDataType; +import org.opensearch.knn.index.vectorvalues.KNNVectorValues; +import org.opensearch.knn.index.vectorvalues.KNNVectorValuesFactory; +import org.opensearch.knn.index.vectorvalues.TestVectorValues; + +import java.util.List; + +import static org.opensearch.knn.common.KNNVectorUtil.iterateVectorValuesOnce; public class KNNVectorUtilTests extends KNNTestCase { public void testByteZeroVector() { @@ -23,4 +32,29 @@ public void testFloatZeroVector() { assertTrue(KNNVectorUtil.isZeroVector(new float[] { 0.0f, 0.0f, 0.0f })); assertFalse(KNNVectorUtil.isZeroVector(new float[] { 1.0f, 1.0f, 1.0f })); } + + public void testIntListToArray() { + assertArrayEquals(new int[] { 1, 2, 3 }, KNNVectorUtil.intListToArray(List.of(1, 2, 3))); + assertNull(KNNVectorUtil.intListToArray(List.of())); + assertNull(KNNVectorUtil.intListToArray(null)); + } + + @SneakyThrows + public void testInit() { + // Give + final List floatArray = List.of(new float[] { 1, 2 }, new float[] { 2, 3 }); + final int dimension = floatArray.get(0).length; + final TestVectorValues.PreDefinedFloatVectorValues randomVectorValues = new TestVectorValues.PreDefinedFloatVectorValues( + floatArray + ); + final KNNVectorValues knnVectorValues = KNNVectorValuesFactory.getVectorValues(VectorDataType.FLOAT, randomVectorValues); + + // When + iterateVectorValuesOnce(knnVectorValues); + + // Then + assertNotEquals(-1, knnVectorValues.docId()); + assertArrayEquals(floatArray.get(0), knnVectorValues.getVector(), 0.001f); + assertEquals(dimension, knnVectorValues.dimension()); + } } diff --git a/src/test/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumerTests.java b/src/test/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumerTests.java index 4c235a896..f49587bc5 100644 --- a/src/test/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumerTests.java +++ b/src/test/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumerTests.java @@ -105,7 +105,7 @@ public void testAddBinaryField_withKNN() throws IOException { KNN80DocValuesConsumer knn80DocValuesConsumer = new KNN80DocValuesConsumer(delegate, null) { @Override - public void addKNNBinaryField(FieldInfo field, DocValuesProducer valuesProducer, boolean isMerge, boolean isRefresh) { + public void addKNNBinaryField(FieldInfo field, DocValuesProducer valuesProducer, boolean isMerge) { called[0] = true; } }; @@ -142,7 +142,7 @@ public void testAddBinaryField_withoutKNN() throws IOException { KNN80DocValuesConsumer knn80DocValuesConsumer = new KNN80DocValuesConsumer(delegate, state) { @Override - public void addKNNBinaryField(FieldInfo field, DocValuesProducer valuesProducer, boolean isMerge, boolean isRefresh) { + public void addKNNBinaryField(FieldInfo field, DocValuesProducer valuesProducer, boolean isMerge) { called[0] = true; } }; @@ -160,7 +160,6 @@ public void testAddKNNBinaryField_noVectors() throws IOException { 128 ); Long initialGraphIndexRequests = KNNCounter.GRAPH_INDEX_REQUESTS.getCount(); - Long initialRefreshOperations = KNNGraphValue.REFRESH_TOTAL_OPERATIONS.getValue(); Long initialMergeOperations = KNNGraphValue.MERGE_TOTAL_OPERATIONS.getValue(); Long initialMergeSize = KNNGraphValue.MERGE_TOTAL_SIZE_IN_BYTES.getValue(); Long initialMergeDocs = KNNGraphValue.MERGE_TOTAL_DOCS.getValue(); @@ -178,9 +177,8 @@ public void testAddKNNBinaryField_noVectors() throws IOException { SegmentWriteState state = new SegmentWriteState(null, directory, segmentInfo, fieldInfos, null, IOContext.DEFAULT); KNN80DocValuesConsumer knn80DocValuesConsumer = new KNN80DocValuesConsumer(null, state); FieldInfo fieldInfo = KNNCodecTestUtil.FieldInfoBuilder.builder("test-field").build(); - knn80DocValuesConsumer.addKNNBinaryField(fieldInfo, randomVectorDocValuesProducer, true, true); + knn80DocValuesConsumer.addKNNBinaryField(fieldInfo, randomVectorDocValuesProducer, true); assertEquals(initialGraphIndexRequests, KNNCounter.GRAPH_INDEX_REQUESTS.getCount()); - assertEquals(initialRefreshOperations, KNNGraphValue.REFRESH_TOTAL_OPERATIONS.getValue()); assertEquals(initialMergeOperations, KNNGraphValue.MERGE_TOTAL_OPERATIONS.getValue()); assertEquals(initialMergeSize, KNNGraphValue.MERGE_TOTAL_SIZE_IN_BYTES.getValue()); assertEquals(initialMergeDocs, KNNGraphValue.MERGE_TOTAL_DOCS.getValue()); @@ -228,7 +226,6 @@ public void testAddKNNBinaryField_fromScratch_nmslibCurrent() throws IOException FieldInfos fieldInfos = new FieldInfos(fieldInfoArray); SegmentWriteState state = new SegmentWriteState(null, directory, segmentInfo, fieldInfos, null, IOContext.DEFAULT); - long initialRefreshOperations = KNNGraphValue.REFRESH_TOTAL_OPERATIONS.getValue(); long initialMergeOperations = KNNGraphValue.MERGE_TOTAL_OPERATIONS.getValue(); // Add documents to the field @@ -237,7 +234,61 @@ public void testAddKNNBinaryField_fromScratch_nmslibCurrent() throws IOException docsInSegment, dimension ); - knn80DocValuesConsumer.addKNNBinaryField(fieldInfoArray[0], randomVectorDocValuesProducer, true, true); + knn80DocValuesConsumer.addKNNBinaryField(fieldInfoArray[0], randomVectorDocValuesProducer, true); + + // The document should be created in the correct location + String expectedFile = KNNCodecUtil.buildEngineFileName(segmentName, knnEngine.getVersion(), fieldName, knnEngine.getExtension()); + assertFileInCorrectLocation(state, expectedFile); + + // The footer should be valid + assertValidFooter(state.directory, expectedFile); + + // The document should be readable by nmslib + assertLoadableByEngine(null, state, expectedFile, knnEngine, spaceType, dimension); + + // The graph creation statistics should be updated + assertEquals(1 + initialMergeOperations, (long) KNNGraphValue.MERGE_TOTAL_OPERATIONS.getValue()); + assertNotEquals(0, (long) KNNGraphValue.MERGE_TOTAL_DOCS.getValue()); + assertNotEquals(0, (long) KNNGraphValue.MERGE_TOTAL_SIZE_IN_BYTES.getValue()); + } + + public void testAddKNNBinaryField_fromScratch_nmslibLegacy() throws IOException { + // Set information about the segment and the fields + String segmentName = String.format("test_segment%s", randomAlphaOfLength(4)); + int docsInSegment = 100; + String fieldName = String.format("test_field%s", randomAlphaOfLength(4)); + + KNNEngine knnEngine = KNNEngine.NMSLIB; + SpaceType spaceType = SpaceType.COSINESIMIL; + int dimension = 16; + + SegmentInfo segmentInfo = KNNCodecTestUtil.segmentInfoBuilder() + .directory(directory) + .segmentName(segmentName) + .docsInSegment(docsInSegment) + .codec(codec) + .build(); + + FieldInfo[] fieldInfoArray = new FieldInfo[] { + KNNCodecTestUtil.FieldInfoBuilder.builder(fieldName) + .addAttribute(KNNVectorFieldMapper.KNN_FIELD, "true") + .addAttribute(KNNConstants.HNSW_ALGO_EF_CONSTRUCTION, "512") + .addAttribute(KNNConstants.HNSW_ALGO_M, "16") + .addAttribute(KNNConstants.SPACE_TYPE, spaceType.getValue()) + .build() }; + + FieldInfos fieldInfos = new FieldInfos(fieldInfoArray); + SegmentWriteState state = new SegmentWriteState(null, directory, segmentInfo, fieldInfos, null, IOContext.DEFAULT); + + long initialMergeOperations = KNNGraphValue.MERGE_TOTAL_OPERATIONS.getValue(); + + // Add documents to the field + KNN80DocValuesConsumer knn80DocValuesConsumer = new KNN80DocValuesConsumer(null, state); + TestVectorValues.RandomVectorDocValuesProducer randomVectorDocValuesProducer = new TestVectorValues.RandomVectorDocValuesProducer( + docsInSegment, + dimension + ); + knn80DocValuesConsumer.addKNNBinaryField(fieldInfoArray[0], randomVectorDocValuesProducer, true); // The document should be created in the correct location String expectedFile = KNNCodecUtil.buildEngineFileName(segmentName, knnEngine.getVersion(), fieldName, knnEngine.getExtension()); @@ -250,7 +301,6 @@ public void testAddKNNBinaryField_fromScratch_nmslibCurrent() throws IOException assertLoadableByEngine(null, state, expectedFile, knnEngine, spaceType, dimension); // The graph creation statistics should be updated - assertEquals(1 + initialRefreshOperations, (long) KNNGraphValue.REFRESH_TOTAL_OPERATIONS.getValue()); assertEquals(1 + initialMergeOperations, (long) KNNGraphValue.MERGE_TOTAL_OPERATIONS.getValue()); assertNotEquals(0, (long) KNNGraphValue.MERGE_TOTAL_DOCS.getValue()); assertNotEquals(0, (long) KNNGraphValue.MERGE_TOTAL_SIZE_IN_BYTES.getValue()); @@ -298,7 +348,6 @@ public void testAddKNNBinaryField_fromScratch_faissCurrent() throws IOException SegmentWriteState state = new SegmentWriteState(null, directory, segmentInfo, fieldInfos, null, IOContext.DEFAULT); long initialRefreshOperations = KNNGraphValue.REFRESH_TOTAL_OPERATIONS.getValue(); - long initialMergeOperations = KNNGraphValue.MERGE_TOTAL_OPERATIONS.getValue(); // Add documents to the field KNN80DocValuesConsumer knn80DocValuesConsumer = new KNN80DocValuesConsumer(null, state); @@ -306,7 +355,7 @@ public void testAddKNNBinaryField_fromScratch_faissCurrent() throws IOException docsInSegment, dimension ); - knn80DocValuesConsumer.addKNNBinaryField(fieldInfoArray[0], randomVectorDocValuesProducer, true, true); + knn80DocValuesConsumer.addKNNBinaryField(fieldInfoArray[0], randomVectorDocValuesProducer, false); // The document should be created in the correct location String expectedFile = KNNCodecUtil.buildEngineFileName(segmentName, knnEngine.getVersion(), fieldName, knnEngine.getExtension()); @@ -320,9 +369,6 @@ public void testAddKNNBinaryField_fromScratch_faissCurrent() throws IOException // The graph creation statistics should be updated assertEquals(1 + initialRefreshOperations, (long) KNNGraphValue.REFRESH_TOTAL_OPERATIONS.getValue()); - assertEquals(1 + initialMergeOperations, (long) KNNGraphValue.MERGE_TOTAL_OPERATIONS.getValue()); - assertNotEquals(0, (long) KNNGraphValue.MERGE_TOTAL_DOCS.getValue()); - assertNotEquals(0, (long) KNNGraphValue.MERGE_TOTAL_SIZE_IN_BYTES.getValue()); } public void testAddKNNBinaryField_whenFaissBinary_thenAdded() throws IOException { @@ -368,7 +414,6 @@ public void testAddKNNBinaryField_whenFaissBinary_thenAdded() throws IOException FieldInfos fieldInfos = new FieldInfos(fieldInfoArray); SegmentWriteState state = new SegmentWriteState(null, directory, segmentInfo, fieldInfos, null, IOContext.DEFAULT); - long initialRefreshOperations = KNNGraphValue.REFRESH_TOTAL_OPERATIONS.getValue(); long initialMergeOperations = KNNGraphValue.MERGE_TOTAL_OPERATIONS.getValue(); // Add documents to the field @@ -377,7 +422,7 @@ public void testAddKNNBinaryField_whenFaissBinary_thenAdded() throws IOException docsInSegment, dimension ); - knn80DocValuesConsumer.addKNNBinaryField(fieldInfoArray[0], randomVectorDocValuesProducer, true, true); + knn80DocValuesConsumer.addKNNBinaryField(fieldInfoArray[0], randomVectorDocValuesProducer, true); // The document should be created in the correct location String expectedFile = KNNCodecUtil.buildEngineFileName(segmentName, knnEngine.getVersion(), fieldName, knnEngine.getExtension()); @@ -390,7 +435,6 @@ public void testAddKNNBinaryField_whenFaissBinary_thenAdded() throws IOException assertBinaryIndexLoadableByEngine(state, expectedFile, knnEngine, spaceType, dimension, dataType); // The graph creation statistics should be updated - assertEquals(1 + initialRefreshOperations, (long) KNNGraphValue.REFRESH_TOTAL_OPERATIONS.getValue()); assertEquals(1 + initialMergeOperations, (long) KNNGraphValue.MERGE_TOTAL_OPERATIONS.getValue()); assertNotEquals(0, (long) KNNGraphValue.MERGE_TOTAL_DOCS.getValue()); assertNotEquals(0, (long) KNNGraphValue.MERGE_TOTAL_SIZE_IN_BYTES.getValue()); @@ -467,7 +511,6 @@ public void testAddKNNBinaryField_fromModel_faiss() throws IOException, Executio FieldInfos fieldInfos = new FieldInfos(fieldInfoArray); SegmentWriteState state = new SegmentWriteState(null, directory, segmentInfo, fieldInfos, null, IOContext.DEFAULT); - long initialRefreshOperations = KNNGraphValue.REFRESH_TOTAL_OPERATIONS.getValue(); long initialMergeOperations = KNNGraphValue.MERGE_TOTAL_OPERATIONS.getValue(); // Add documents to the field @@ -476,7 +519,7 @@ public void testAddKNNBinaryField_fromModel_faiss() throws IOException, Executio docsInSegment, dimension ); - knn80DocValuesConsumer.addKNNBinaryField(fieldInfoArray[0], randomVectorDocValuesProducer, true, true); + knn80DocValuesConsumer.addKNNBinaryField(fieldInfoArray[0], randomVectorDocValuesProducer, true); // The document should be created in the correct location String expectedFile = KNNCodecUtil.buildEngineFileName(segmentName, knnEngine.getVersion(), fieldName, knnEngine.getExtension()); @@ -489,7 +532,6 @@ public void testAddKNNBinaryField_fromModel_faiss() throws IOException, Executio assertLoadableByEngine(HNSW_METHODPARAMETERS, state, expectedFile, knnEngine, spaceType, dimension); // The graph creation statistics should be updated - assertEquals(1 + initialRefreshOperations, (long) KNNGraphValue.REFRESH_TOTAL_OPERATIONS.getValue()); assertEquals(1 + initialMergeOperations, (long) KNNGraphValue.MERGE_TOTAL_OPERATIONS.getValue()); assertNotEquals(0, (long) KNNGraphValue.MERGE_TOTAL_DOCS.getValue()); assertNotEquals(0, (long) KNNGraphValue.MERGE_TOTAL_SIZE_IN_BYTES.getValue()); @@ -561,6 +603,6 @@ public void testAddBinaryField_luceneEngine_noInvocations_addKNNBinary() throws knn80DocValuesConsumer.addBinaryField(fieldInfo, docValuesProducer); verify(delegate, times(1)).addBinaryField(fieldInfo, docValuesProducer); - verify(knn80DocValuesConsumer, never()).addKNNBinaryField(any(), any(), eq(false), eq(true)); + verify(knn80DocValuesConsumer, never()).addKNNBinaryField(any(), any(), eq(false)); } } diff --git a/src/test/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsFormatTests.java b/src/test/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsFormatTests.java index 322b714f2..3810d46fd 100644 --- a/src/test/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsFormatTests.java +++ b/src/test/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsFormatTests.java @@ -46,6 +46,7 @@ import org.opensearch.knn.KNNTestCase; import org.opensearch.knn.common.KNNConstants; import org.opensearch.knn.index.SpaceType; +import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.mapper.KNNVectorFieldMapper; import org.opensearch.knn.index.engine.KNNEngine; @@ -99,10 +100,13 @@ public void testNativeEngineVectorFormat_whenMultipleVectorFieldIndexed_thenSucc byte[] byteVector = { 6, 14 }; addFieldToIndex( - new KnnFloatVectorField(FLOAT_VECTOR_FIELD, floatVector, createVectorField(3, VectorEncoding.FLOAT32)), + new KnnFloatVectorField(FLOAT_VECTOR_FIELD, floatVector, createVectorField(3, VectorEncoding.FLOAT32, VectorDataType.FLOAT)), + indexWriter + ); + addFieldToIndex( + new KnnByteVectorField(BYTE_VECTOR_FIELD, byteVector, createVectorField(2, VectorEncoding.BYTE, VectorDataType.BINARY)), indexWriter ); - addFieldToIndex(new KnnByteVectorField(BYTE_VECTOR_FIELD, byteVector, createVectorField(2, VectorEncoding.BYTE)), indexWriter); final IndexReader indexReader = indexWriter.getReader(); // ensuring segments are created indexWriter.flush(); @@ -187,17 +191,19 @@ private void addFieldToIndex(final Field vectorField, final RandomIndexWriter in indexWriter.addDocument(doc1); } - private FieldType createVectorField(int dimension, VectorEncoding vectorEncoding) { + private FieldType createVectorField(int dimension, VectorEncoding vectorEncoding, VectorDataType vectorDataType) { FieldType nativeVectorField = new FieldType(); // TODO: Replace this with the default field which will be created in mapper for Native Engines with KNNVectorsFormat nativeVectorField.setTokenized(false); nativeVectorField.setIndexOptions(IndexOptions.NONE); nativeVectorField.putAttribute(KNNVectorFieldMapper.KNN_FIELD, "true"); nativeVectorField.putAttribute(KNNConstants.KNN_METHOD, KNNConstants.METHOD_HNSW); - nativeVectorField.putAttribute(KNNConstants.KNN_ENGINE, KNNEngine.NMSLIB.getName()); + nativeVectorField.putAttribute(KNNConstants.KNN_ENGINE, KNNEngine.FAISS.getName()); nativeVectorField.putAttribute(KNNConstants.SPACE_TYPE, SpaceType.L2.getValue()); nativeVectorField.putAttribute(KNNConstants.HNSW_ALGO_M, "32"); nativeVectorField.putAttribute(KNNConstants.HNSW_ALGO_EF_CONSTRUCTION, "512"); + nativeVectorField.putAttribute(KNNConstants.VECTOR_DATA_TYPE_FIELD, vectorDataType.getValue()); + nativeVectorField.putAttribute(KNNConstants.PARAMETERS, "{ \"index_description\":\"HNSW16,Flat\", \"spaceType\": \"l2\"}"); nativeVectorField.setVectorAttributes( dimension, vectorEncoding, diff --git a/src/test/java/org/opensearch/knn/index/codec/nativeindex/DefaultIndexBuildStrategyTests.java b/src/test/java/org/opensearch/knn/index/codec/nativeindex/DefaultIndexBuildStrategyTests.java new file mode 100644 index 000000000..34a333471 --- /dev/null +++ b/src/test/java/org/opensearch/knn/index/codec/nativeindex/DefaultIndexBuildStrategyTests.java @@ -0,0 +1,167 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.codec.nativeindex; + +import lombok.SneakyThrows; +import org.apache.lucene.index.DocsWithFieldSet; +import org.junit.Before; +import org.mockito.ArgumentCaptor; +import org.mockito.MockedStatic; +import org.opensearch.core.common.unit.ByteSizeValue; +import org.opensearch.knn.index.KNNSettings; +import org.opensearch.knn.index.VectorDataType; +import org.opensearch.knn.index.codec.nativeindex.model.BuildIndexParams; +import org.opensearch.knn.index.codec.transfer.OffHeapVectorTransfer; +import org.opensearch.knn.index.codec.transfer.OffHeapVectorTransferFactory; +import org.opensearch.knn.index.engine.KNNEngine; +import org.opensearch.knn.index.vectorvalues.KNNFloatVectorValues; +import org.opensearch.knn.index.vectorvalues.KNNVectorValues; +import org.opensearch.knn.index.vectorvalues.KNNVectorValuesFactory; +import org.opensearch.knn.index.vectorvalues.TestVectorValues; +import org.opensearch.knn.jni.JNIService; +import org.opensearch.test.OpenSearchTestCase; + +import java.util.List; +import java.util.Map; + +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.mockStatic; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +public class DefaultIndexBuildStrategyTests extends OpenSearchTestCase { + + ArgumentCaptor vectorTransferCapture = ArgumentCaptor.forClass(float[].class); + + @Before + public void init() { + vectorTransferCapture = ArgumentCaptor.forClass(float[].class); + } + + @SneakyThrows + public void testBuildAndWrite() { + // Given + List vectorValues = List.of(new float[] { 1, 2 }, new float[] { 2, 3 }, new float[] { 3, 4 }); + + final TestVectorValues.PreDefinedFloatVectorValues randomVectorValues = new TestVectorValues.PreDefinedFloatVectorValues( + vectorValues + ); + final KNNVectorValues knnVectorValues = KNNVectorValuesFactory.getVectorValues(VectorDataType.FLOAT, randomVectorValues); + + try ( + MockedStatic mockedKNNSettings = mockStatic(KNNSettings.class); + MockedStatic mockedJNIService = mockStatic(JNIService.class); + MockedStatic mockedOffHeapVectorTransferFactory = mockStatic(OffHeapVectorTransferFactory.class) + ) { + + mockedKNNSettings.when(KNNSettings::getVectorStreamingMemoryLimit).thenReturn(new ByteSizeValue(16)); + OffHeapVectorTransfer offHeapVectorTransfer = mock(OffHeapVectorTransfer.class); + mockedOffHeapVectorTransferFactory.when(() -> OffHeapVectorTransferFactory.getVectorTransfer(VectorDataType.FLOAT, 2)) + .thenReturn(offHeapVectorTransfer); + + when(offHeapVectorTransfer.getVectorAddress()).thenReturn(200L); + + BuildIndexParams buildIndexParams = BuildIndexParams.builder() + .indexPath("indexPath") + .knnEngine(KNNEngine.NMSLIB) + .vectorDataType(VectorDataType.FLOAT) + .parameters(Map.of("index", "param")) + .build(); + + // When + DefaultIndexBuildStrategy.getInstance().buildAndWriteIndex(buildIndexParams, knnVectorValues); + + // Then + mockedJNIService.verify( + () -> JNIService.createIndex( + eq(new int[] { 0, 1, 2 }), + eq(200L), + eq(knnVectorValues.dimension()), + eq("indexPath"), + eq(Map.of("index", "param")), + eq(KNNEngine.NMSLIB) + ) + ); + mockedJNIService.verifyNoMoreInteractions(); + verify(offHeapVectorTransfer).flush(true); + verify(offHeapVectorTransfer, times(3)).transfer(vectorTransferCapture.capture(), eq(true)); + verify(offHeapVectorTransfer).reset(); + + float[] prev = null; + for (float[] vector : vectorTransferCapture.getAllValues()) { + if (prev != null) { + assertNotSame(prev, vector); + } + prev = vector; + } + } + } + + @SneakyThrows + public void testBuildAndWriteWithModel() { + // Given + final Map docs = Map.of(0, new float[] { 1, 2 }, 1, new float[] { 2, 3 }, 2, new float[] { 3, 4 }); + DocsWithFieldSet docsWithFieldSet = new DocsWithFieldSet(); + docs.keySet().stream().sorted().forEach(docsWithFieldSet::add); + + byte[] modelBlob = new byte[] { 1 }; + + KNNFloatVectorValues knnVectorValues = (KNNFloatVectorValues) KNNVectorValuesFactory.getVectorValues( + VectorDataType.FLOAT, + docsWithFieldSet, + docs + ); + try ( + MockedStatic mockedKNNSettings = mockStatic(KNNSettings.class); + MockedStatic mockedJNIService = mockStatic(JNIService.class); + MockedStatic mockedOffHeapVectorTransferFactory = mockStatic(OffHeapVectorTransferFactory.class) + ) { + + mockedKNNSettings.when(KNNSettings::getVectorStreamingMemoryLimit).thenReturn(new ByteSizeValue(16)); + OffHeapVectorTransfer offHeapVectorTransfer = mock(OffHeapVectorTransfer.class); + mockedOffHeapVectorTransferFactory.when(() -> OffHeapVectorTransferFactory.getVectorTransfer(VectorDataType.FLOAT, 2)) + .thenReturn(offHeapVectorTransfer); + + when(offHeapVectorTransfer.getVectorAddress()).thenReturn(200L); + + BuildIndexParams buildIndexParams = BuildIndexParams.builder() + .indexPath("indexPath") + .knnEngine(KNNEngine.NMSLIB) + .vectorDataType(VectorDataType.FLOAT) + .parameters(Map.of("model_id", "id", "model_blob", modelBlob)) + .build(); + + // When + DefaultIndexBuildStrategy.getInstance().buildAndWriteIndex(buildIndexParams, knnVectorValues); + + // Then + mockedJNIService.verify( + () -> JNIService.createIndexFromTemplate( + eq(new int[] { 0, 1, 2 }), + eq(200L), + eq(2), + eq("indexPath"), + eq(modelBlob), + eq(Map.of("model_id", "id", "model_blob", modelBlob)), + eq(KNNEngine.NMSLIB) + ) + ); + mockedJNIService.verifyNoMoreInteractions(); + verify(offHeapVectorTransfer).flush(true); + verify(offHeapVectorTransfer, times(3)).transfer(vectorTransferCapture.capture(), eq(true)); + + float[] prev = null; + for (float[] vector : vectorTransferCapture.getAllValues()) { + if (prev != null) { + assertNotSame(prev, vector); + } + prev = vector; + } + } + } +} diff --git a/src/test/java/org/opensearch/knn/index/codec/nativeindex/MemOptimizedNativeIndexBuildStrategyTests.java b/src/test/java/org/opensearch/knn/index/codec/nativeindex/MemOptimizedNativeIndexBuildStrategyTests.java new file mode 100644 index 000000000..2ecfe9259 --- /dev/null +++ b/src/test/java/org/opensearch/knn/index/codec/nativeindex/MemOptimizedNativeIndexBuildStrategyTests.java @@ -0,0 +1,129 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.codec.nativeindex; + +import lombok.SneakyThrows; +import org.mockito.ArgumentCaptor; +import org.mockito.MockedStatic; +import org.mockito.Mockito; +import org.opensearch.core.common.unit.ByteSizeValue; +import org.opensearch.knn.index.KNNSettings; +import org.opensearch.knn.index.VectorDataType; +import org.opensearch.knn.index.codec.nativeindex.model.BuildIndexParams; +import org.opensearch.knn.index.codec.transfer.OffHeapVectorTransfer; +import org.opensearch.knn.index.codec.transfer.OffHeapVectorTransferFactory; +import org.opensearch.knn.index.engine.KNNEngine; +import org.opensearch.knn.index.vectorvalues.KNNVectorValues; +import org.opensearch.knn.index.vectorvalues.KNNVectorValuesFactory; +import org.opensearch.knn.index.vectorvalues.TestVectorValues; +import org.opensearch.knn.jni.JNIService; +import org.opensearch.test.OpenSearchTestCase; + +import java.util.List; +import java.util.Map; + +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +public class MemOptimizedNativeIndexBuildStrategyTests extends OpenSearchTestCase { + + @SneakyThrows + public void testBuildAndWrite() { + // Given + ArgumentCaptor vectorAddressCaptor = ArgumentCaptor.forClass(Long.class); + ArgumentCaptor vectorTransferCapture = ArgumentCaptor.forClass(float[].class); + + List vectorValues = List.of(new float[] { 1, 2 }, new float[] { 2, 3 }, new float[] { 3, 4 }); + final TestVectorValues.PreDefinedFloatVectorValues randomVectorValues = new TestVectorValues.PreDefinedFloatVectorValues( + vectorValues + ); + final KNNVectorValues knnVectorValues = KNNVectorValuesFactory.getVectorValues(VectorDataType.FLOAT, randomVectorValues); + + try ( + MockedStatic mockedKNNSettings = Mockito.mockStatic(KNNSettings.class); + MockedStatic mockedJNIService = Mockito.mockStatic(JNIService.class); + MockedStatic mockedOffHeapVectorTransferFactory = Mockito.mockStatic( + OffHeapVectorTransferFactory.class + ); + ) { + + // Limits transfer to 2 vectors + mockedKNNSettings.when(KNNSettings::getVectorStreamingMemoryLimit).thenReturn(new ByteSizeValue(16)); + mockedJNIService.when(() -> JNIService.initIndex(3, 2, Map.of("index", "param"), KNNEngine.FAISS)).thenReturn(100L); + + OffHeapVectorTransfer offHeapVectorTransfer = mock(OffHeapVectorTransfer.class); + mockedOffHeapVectorTransferFactory.when(() -> OffHeapVectorTransferFactory.getVectorTransfer(VectorDataType.FLOAT, 2)) + .thenReturn(offHeapVectorTransfer); + + when(offHeapVectorTransfer.transfer(vectorTransferCapture.capture(), eq(false))).thenReturn(false) + .thenReturn(true) + .thenReturn(false); + when(offHeapVectorTransfer.flush(false)).thenReturn(true); + when(offHeapVectorTransfer.getVectorAddress()).thenReturn(200L); + + BuildIndexParams buildIndexParams = BuildIndexParams.builder() + .indexPath("indexPath") + .knnEngine(KNNEngine.FAISS) + .vectorDataType(VectorDataType.FLOAT) + .parameters(Map.of("index", "param")) + .build(); + + // When + MemOptimizedNativeIndexBuildStrategy.getInstance().buildAndWriteIndex(buildIndexParams, knnVectorValues); + + // Then + mockedJNIService.verify( + () -> JNIService.initIndex( + knnVectorValues.totalLiveDocs(), + knnVectorValues.dimension(), + Map.of("index", "param"), + KNNEngine.FAISS + ) + ); + + mockedJNIService.verify( + () -> JNIService.insertToIndex( + eq(new int[] { 0, 1 }), + vectorAddressCaptor.capture(), + eq(knnVectorValues.dimension()), + eq(Map.of("index", "param")), + eq(100L), + eq(KNNEngine.FAISS) + ) + ); + + // For the flush + mockedJNIService.verify( + () -> JNIService.insertToIndex( + eq(new int[] { 2 }), + vectorAddressCaptor.capture(), + eq(knnVectorValues.dimension()), + eq(Map.of("index", "param")), + eq(100L), + eq(KNNEngine.FAISS) + ) + ); + + mockedJNIService.verify( + () -> JNIService.writeIndex(eq("indexPath"), eq(100L), eq(KNNEngine.FAISS), eq(Map.of("index", "param"))) + ); + assertEquals(200L, vectorAddressCaptor.getValue().longValue()); + assertEquals(vectorAddressCaptor.getValue().longValue(), vectorAddressCaptor.getAllValues().get(0).longValue()); + verify(offHeapVectorTransfer, times(0)).reset(); + + float[] prev = null; + for (float[] vector : vectorTransferCapture.getAllValues()) { + if (prev != null) { + assertNotSame(prev, vector); + } + prev = vector; + } + } + } +} diff --git a/src/test/java/org/opensearch/knn/index/codec/transfer/OffHeapVectorTransferFactoryTests.java b/src/test/java/org/opensearch/knn/index/codec/transfer/OffHeapVectorTransferFactoryTests.java new file mode 100644 index 000000000..cef875cfc --- /dev/null +++ b/src/test/java/org/opensearch/knn/index/codec/transfer/OffHeapVectorTransferFactoryTests.java @@ -0,0 +1,26 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.codec.transfer; + +import org.opensearch.knn.index.VectorDataType; +import org.opensearch.test.OpenSearchTestCase; + +public class OffHeapVectorTransferFactoryTests extends OpenSearchTestCase { + + public void testOffHeapVectorTransferFactory() { + var floatVectorTransfer = OffHeapVectorTransferFactory.getVectorTransfer(VectorDataType.FLOAT, 10); + assertEquals(OffHeapFloatVectorTransfer.class, floatVectorTransfer.getClass()); + assertNotSame(floatVectorTransfer, OffHeapVectorTransferFactory.getVectorTransfer(VectorDataType.FLOAT, 10)); + + var byteVectorTransfer = OffHeapVectorTransferFactory.getVectorTransfer(VectorDataType.BYTE, 10); + assertEquals(OffHeapByteVectorTransfer.class, byteVectorTransfer.getClass()); + assertNotSame(byteVectorTransfer, OffHeapVectorTransferFactory.getVectorTransfer(VectorDataType.BYTE, 10)); + + var binaryVectorTransfer = OffHeapVectorTransferFactory.getVectorTransfer(VectorDataType.BINARY, 10); + assertEquals(OffHeapByteVectorTransfer.class, binaryVectorTransfer.getClass()); + assertNotSame(binaryVectorTransfer, OffHeapVectorTransferFactory.getVectorTransfer(VectorDataType.BINARY, 10)); + } +} diff --git a/src/test/java/org/opensearch/knn/index/codec/transfer/OffHeapVectorTransferTests.java b/src/test/java/org/opensearch/knn/index/codec/transfer/OffHeapVectorTransferTests.java new file mode 100644 index 000000000..f1650db8f --- /dev/null +++ b/src/test/java/org/opensearch/knn/index/codec/transfer/OffHeapVectorTransferTests.java @@ -0,0 +1,92 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.codec.transfer; + +import lombok.SneakyThrows; +import org.opensearch.knn.KNNTestCase; + +import java.util.List; + +public class OffHeapVectorTransferTests extends KNNTestCase { + + @SneakyThrows + public void testFloatTransfer() { + List vectors = List.of( + new float[] { 0.1f, 0.2f }, + new float[] { 0.2f, 0.3f }, + new float[] { 0.3f, 0.4f }, + new float[] { 0.3f, 0.4f }, + new float[] { 0.3f, 0.4f } + ); + + OffHeapFloatVectorTransfer vectorTransfer = new OffHeapFloatVectorTransfer(2); + long vectorAddress = 0; + assertFalse(vectorTransfer.transfer(vectors.get(0), false)); + assertEquals(0, vectorTransfer.getVectorAddress()); + assertTrue(vectorTransfer.transfer(vectors.get(1), false)); + vectorAddress = vectorTransfer.getVectorAddress(); + assertFalse(vectorTransfer.transfer(vectors.get(2), false)); + assertEquals(vectorAddress, vectorTransfer.getVectorAddress()); + assertTrue(vectorTransfer.transfer(vectors.get(3), false)); + assertEquals(vectorAddress, vectorTransfer.getVectorAddress()); + assertFalse(vectorTransfer.transfer(vectors.get(4), false)); + assertTrue(vectorTransfer.flush(false)); + vectorTransfer.reset(); + assertEquals(0, vectorTransfer.getVectorAddress()); + vectorTransfer.close(); + } + + @SneakyThrows + public void testByteTransfer() { + List vectors = List.of( + new byte[] { 0, 1 }, + new byte[] { 2, 3 }, + new byte[] { 4, 5 }, + new byte[] { 6, 7 }, + new byte[] { 8, 9 } + ); + + OffHeapByteVectorTransfer vectorTransfer = new OffHeapByteVectorTransfer(2); + long vectorAddress = 0; + assertFalse(vectorTransfer.transfer(vectors.get(0), false)); + assertEquals(0, vectorTransfer.getVectorAddress()); + assertTrue(vectorTransfer.transfer(vectors.get(1), false)); + vectorAddress = vectorTransfer.getVectorAddress(); + assertFalse(vectorTransfer.transfer(vectors.get(2), false)); + assertEquals(vectorAddress, vectorTransfer.getVectorAddress()); + assertTrue(vectorTransfer.transfer(vectors.get(3), false)); + assertEquals(vectorAddress, vectorTransfer.getVectorAddress()); + assertFalse(vectorTransfer.transfer(vectors.get(4), false)); + assertTrue(vectorTransfer.flush(false)); + vectorTransfer.close(); + assertEquals(0, vectorTransfer.getVectorAddress()); + } + + @SneakyThrows + public void testBinaryTransfer() { + List vectors = List.of( + new byte[] { 0, 1 }, + new byte[] { 2, 3 }, + new byte[] { 4, 5 }, + new byte[] { 6, 7 }, + new byte[] { 8, 9 } + ); + + OffHeapBinaryVectorTransfer vectorTransfer = new OffHeapBinaryVectorTransfer(2); + long vectorAddress = 0; + assertFalse(vectorTransfer.transfer(vectors.get(0), false)); + assertEquals(0, vectorTransfer.getVectorAddress()); + assertTrue(vectorTransfer.transfer(vectors.get(1), false)); + vectorAddress = vectorTransfer.getVectorAddress(); + assertFalse(vectorTransfer.transfer(vectors.get(2), false)); + assertEquals(vectorAddress, vectorTransfer.getVectorAddress()); + assertTrue(vectorTransfer.transfer(vectors.get(3), false)); + assertEquals(vectorAddress, vectorTransfer.getVectorAddress()); + assertFalse(vectorTransfer.transfer(vectors.get(4), false)); + assertTrue(vectorTransfer.flush(false)); + vectorTransfer.close(); + } +} diff --git a/src/test/java/org/opensearch/knn/index/codec/transfer/VectorTransferByteTests.java b/src/test/java/org/opensearch/knn/index/codec/transfer/VectorTransferByteTests.java deleted file mode 100644 index 2f091a035..000000000 --- a/src/test/java/org/opensearch/knn/index/codec/transfer/VectorTransferByteTests.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.knn.index.codec.transfer; - -import junit.framework.TestCase; -import lombok.SneakyThrows; -import org.apache.lucene.util.BytesRef; -import org.opensearch.knn.index.codec.util.SerializationMode; -import org.opensearch.knn.jni.JNICommons; - -import java.io.IOException; -import java.util.Random; - -import static org.junit.Assert.assertNotEquals; - -public class VectorTransferByteTests extends TestCase { - @SneakyThrows - public void testTransfer_whenCalled_thenAdded() { - final BytesRef bytesRef1 = getByteArrayOfVectors(20); - final BytesRef bytesRef2 = getByteArrayOfVectors(20); - VectorTransferByte vectorTransfer = new VectorTransferByte(40); - try { - vectorTransfer.init(2); - - vectorTransfer.transfer(bytesRef1); - // flush is not called - assertEquals(0, vectorTransfer.getVectorAddress()); - - vectorTransfer.transfer(bytesRef2); - // flush should be called - assertNotEquals(0, vectorTransfer.getVectorAddress()); - } finally { - if (vectorTransfer.getVectorAddress() != 0) { - JNICommons.freeVectorData(vectorTransfer.getVectorAddress()); - } - } - } - - @SneakyThrows - public void testSerializationMode_whenCalled_thenReturn() { - final BytesRef bytesRef = getByteArrayOfVectors(20); - VectorTransferByte vectorTransfer = new VectorTransferByte(1000); - - // Verify - assertEquals(SerializationMode.COLLECTIONS_OF_BYTES, vectorTransfer.getSerializationMode(bytesRef)); - } - - private BytesRef getByteArrayOfVectors(int vectorLength) throws IOException { - byte[] vector = new byte[vectorLength]; - new Random().nextBytes(vector); - return new BytesRef(vector); - } -} diff --git a/src/test/java/org/opensearch/knn/index/codec/transfer/VectorTransferFloatTests.java b/src/test/java/org/opensearch/knn/index/codec/transfer/VectorTransferFloatTests.java deleted file mode 100644 index 620fd7c65..000000000 --- a/src/test/java/org/opensearch/knn/index/codec/transfer/VectorTransferFloatTests.java +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.knn.index.codec.transfer; - -import junit.framework.TestCase; -import lombok.SneakyThrows; -import org.apache.lucene.util.BytesRef; -import org.opensearch.knn.index.codec.util.KNNVectorSerializerFactory; -import org.opensearch.knn.jni.JNICommons; - -import java.io.ByteArrayOutputStream; -import java.io.DataOutputStream; -import java.io.IOException; -import java.util.Random; -import java.util.stream.IntStream; - -import static org.junit.Assert.assertNotEquals; - -public class VectorTransferFloatTests extends TestCase { - @SneakyThrows - public void testTransfer_whenCalled_thenAdded() { - final BytesRef bytesRef1 = getByteArrayOfVectors(20); - final BytesRef bytesRef2 = getByteArrayOfVectors(20); - VectorTransferFloat vectorTransfer = new VectorTransferFloat(160); - try { - vectorTransfer.init(2); - - vectorTransfer.transfer(bytesRef1); - // flush is not called - assertEquals(0, vectorTransfer.getVectorAddress()); - - vectorTransfer.transfer(bytesRef2); - // flush should be called - assertNotEquals(0, vectorTransfer.getVectorAddress()); - } finally { - if (vectorTransfer.getVectorAddress() != 0) { - JNICommons.freeVectorData(vectorTransfer.getVectorAddress()); - } - } - } - - @SneakyThrows - public void testSerializationMode_whenCalled_thenReturn() { - final BytesRef bytesRef = getByteArrayOfVectors(20); - VectorTransferFloat vectorTransfer = new VectorTransferFloat(1000); - - // Verify - assertEquals(KNNVectorSerializerFactory.getSerializerModeFromBytesRef(bytesRef), vectorTransfer.getSerializationMode(bytesRef)); - } - - private BytesRef getByteArrayOfVectors(int vectorLength) throws IOException { - float[] vector = new float[vectorLength]; - IntStream.range(0, vectorLength).forEach(index -> vector[index] = new Random().nextFloat()); - - final ByteArrayOutputStream bas = new ByteArrayOutputStream(); - final DataOutputStream ds = new DataOutputStream(bas); - for (float f : vector) { - ds.writeFloat(f); - } - final byte[] vectorAsCollectionOfFloats = bas.toByteArray(); - return new BytesRef(vectorAsCollectionOfFloats); - } -} diff --git a/src/test/java/org/opensearch/knn/index/codec/util/KNNCodecUtilTests.java b/src/test/java/org/opensearch/knn/index/codec/util/KNNCodecUtilTests.java index 47dd1dda9..dbea6375b 100644 --- a/src/test/java/org/opensearch/knn/index/codec/util/KNNCodecUtilTests.java +++ b/src/test/java/org/opensearch/knn/index/codec/util/KNNCodecUtilTests.java @@ -6,54 +6,11 @@ package org.opensearch.knn.index.codec.util; import junit.framework.TestCase; -import lombok.SneakyThrows; -import org.apache.lucene.index.BinaryDocValues; -import org.apache.lucene.util.BytesRef; import org.opensearch.knn.index.VectorDataType; -import org.opensearch.knn.index.codec.transfer.VectorTransfer; -import java.util.Arrays; - -import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; import static org.opensearch.knn.index.codec.util.KNNCodecUtil.calculateArraySize; public class KNNCodecUtilTests extends TestCase { - @SneakyThrows - public void testGetPair_whenCalled_thenReturn() { - long liveDocCount = 1l; - int[] docId = { 2 }; - long vectorAddress = 3l; - int dimension = 4; - BytesRef bytesRef = new BytesRef(); - - BinaryDocValues binaryDocValues = mock(BinaryDocValues.class); - when(binaryDocValues.cost()).thenReturn(liveDocCount); - when(binaryDocValues.nextDoc()).thenReturn(docId[0], NO_MORE_DOCS); - when(binaryDocValues.binaryValue()).thenReturn(bytesRef); - - VectorTransfer vectorTransfer = mock(VectorTransfer.class); - when(vectorTransfer.getSerializationMode(any(BytesRef.class))).thenReturn(SerializationMode.COLLECTIONS_OF_BYTES); - when(vectorTransfer.getVectorAddress()).thenReturn(vectorAddress); - when(vectorTransfer.getDimension()).thenReturn(dimension); - - // Run - KNNCodecUtil.Pair pair = KNNCodecUtil.getPair(binaryDocValues, vectorTransfer); - - // Verify - verify(vectorTransfer).init(liveDocCount); - verify(vectorTransfer).getSerializationMode(any(BytesRef.class)); - verify(vectorTransfer).transfer(any(BytesRef.class)); - verify(vectorTransfer).close(); - - assertTrue(Arrays.equals(docId, pair.docs)); - assertEquals(vectorAddress, pair.getVectorAddress()); - assertEquals(dimension, pair.getDimension()); - assertEquals(SerializationMode.COLLECTIONS_OF_BYTES, pair.serializationMode); - } public void testCalculateArraySize() { int numVectors = 4; diff --git a/src/test/java/org/opensearch/knn/index/memory/NativeMemoryAllocationTests.java b/src/test/java/org/opensearch/knn/index/memory/NativeMemoryAllocationTests.java index dc9a97fbf..316582f6c 100644 --- a/src/test/java/org/opensearch/knn/index/memory/NativeMemoryAllocationTests.java +++ b/src/test/java/org/opensearch/knn/index/memory/NativeMemoryAllocationTests.java @@ -14,6 +14,7 @@ import com.google.common.collect.ImmutableMap; import lombok.SneakyThrows; import org.opensearch.knn.KNNTestCase; +import org.opensearch.knn.TestUtils; import org.opensearch.knn.common.KNNConstants; import org.opensearch.knn.index.util.IndexUtil; import org.opensearch.knn.index.VectorDataType; @@ -56,7 +57,7 @@ public void testIndexAllocation_close() throws InterruptedException { } Map parameters = ImmutableMap.of(KNNConstants.SPACE_TYPE, SpaceType.DEFAULT.getValue()); long vectorMemoryAddress = JNICommons.storeVectorData(0, vectors, numVectors * dimension); - JNIService.createIndex(ids, vectorMemoryAddress, dimension, path, parameters, knnEngine); + TestUtils.createIndex(ids, vectorMemoryAddress, dimension, path, parameters, knnEngine); // Load index into memory long memoryAddress = JNIService.loadIndex(path, parameters, knnEngine); @@ -117,7 +118,7 @@ public void testClose_whenBinaryFiass_thenSuccess() { VectorDataType.BINARY.getValue() ); long vectorMemoryAddress = JNICommons.storeByteVectorData(0, vectors, numVectors * dataLength); - JNIService.createIndex(ids, vectorMemoryAddress, dimension, path, parameters, knnEngine); + TestUtils.createIndex(ids, vectorMemoryAddress, dimension, path, parameters, knnEngine); // Load index into memory long memoryAddress = JNIService.loadIndex(path, parameters, knnEngine); diff --git a/src/test/java/org/opensearch/knn/index/memory/NativeMemoryLoadStrategyTests.java b/src/test/java/org/opensearch/knn/index/memory/NativeMemoryLoadStrategyTests.java index 51f95d29a..8a38cadb5 100644 --- a/src/test/java/org/opensearch/knn/index/memory/NativeMemoryLoadStrategyTests.java +++ b/src/test/java/org/opensearch/knn/index/memory/NativeMemoryLoadStrategyTests.java @@ -15,6 +15,7 @@ import org.opensearch.core.action.ActionListener; import org.opensearch.action.search.SearchResponse; import org.opensearch.knn.KNNTestCase; +import org.opensearch.knn.TestUtils; import org.opensearch.knn.common.KNNConstants; import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.jni.JNICommons; @@ -32,6 +33,8 @@ import java.util.Arrays; import java.util.Map; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.any; import static org.mockito.Mockito.eq; import static org.mockito.Mockito.doAnswer; @@ -56,7 +59,7 @@ public void testIndexLoadStrategy_load() throws IOException { } Map parameters = ImmutableMap.of(KNNConstants.SPACE_TYPE, SpaceType.DEFAULT.getValue()); long memoryAddress = JNICommons.storeVectorData(0, vectors, numVectors * dimension); - JNIService.createIndex(ids, memoryAddress, dimension, path, parameters, knnEngine); + TestUtils.createIndex(ids, memoryAddress, dimension, path, parameters, knnEngine); // Setup mock resource manager ResourceWatcherService resourceWatcherService = mock(ResourceWatcherService.class); @@ -104,7 +107,7 @@ public void testLoad_whenFaissBinary_thenSuccess() throws IOException { VectorDataType.BINARY.getValue() ); long memoryAddress = JNICommons.storeByteVectorData(0, vectors, numVectors); - JNIService.createIndex(ids, memoryAddress, dimension, path, parameters, knnEngine); + TestUtils.createIndex(ids, memoryAddress, dimension, path, parameters, knnEngine); // Setup mock resource manager ResourceWatcherService resourceWatcherService = mock(ResourceWatcherService.class); diff --git a/src/test/java/org/opensearch/knn/index/vectorvalues/KNNVectorValuesTests.java b/src/test/java/org/opensearch/knn/index/vectorvalues/KNNVectorValuesTests.java index f5a1351ae..0b631ab41 100644 --- a/src/test/java/org/opensearch/knn/index/vectorvalues/KNNVectorValuesTests.java +++ b/src/test/java/org/opensearch/knn/index/vectorvalues/KNNVectorValuesTests.java @@ -26,7 +26,7 @@ public void testFloatVectorValues_whenValidInput_thenSuccess() { floatArray ); final KNNVectorValues knnVectorValues = KNNVectorValuesFactory.getVectorValues(VectorDataType.FLOAT, randomVectorValues); - new CompareVectorValues().validateVectorValues(knnVectorValues, floatArray, dimension, true); + new CompareVectorValues().validateVectorValues(knnVectorValues, floatArray, 8, dimension, true); final DocsWithFieldSet docsWithFieldSet = getDocIdSetIterator(floatArray.size()); @@ -36,7 +36,7 @@ public void testFloatVectorValues_whenValidInput_thenSuccess() { docsWithFieldSet, vectorsMap ); - new CompareVectorValues().validateVectorValues(knnVectorValuesForFieldWriter, floatArray, dimension, false); + new CompareVectorValues().validateVectorValues(knnVectorValuesForFieldWriter, floatArray, 8, dimension, false); final TestVectorValues.PredefinedFloatVectorBinaryDocValues preDefinedFloatVectorValues = new TestVectorValues.PredefinedFloatVectorBinaryDocValues(floatArray); @@ -44,7 +44,7 @@ public void testFloatVectorValues_whenValidInput_thenSuccess() { VectorDataType.FLOAT, preDefinedFloatVectorValues ); - new CompareVectorValues().validateVectorValues(knnFloatVectorValuesBinaryDocValues, floatArray, dimension, false); + new CompareVectorValues().validateVectorValues(knnFloatVectorValuesBinaryDocValues, floatArray, 8, dimension, false); } @SneakyThrows @@ -53,7 +53,7 @@ public void testByteVectorValues_whenValidInput_thenSuccess() { final int dimension = byteArray.get(0).length; final TestVectorValues.PreDefinedByteVectorValues randomVectorValues = new TestVectorValues.PreDefinedByteVectorValues(byteArray); final KNNVectorValues knnVectorValues = KNNVectorValuesFactory.getVectorValues(VectorDataType.BYTE, randomVectorValues); - new CompareVectorValues().validateVectorValues(knnVectorValues, byteArray, dimension, true); + new CompareVectorValues().validateVectorValues(knnVectorValues, byteArray, 2, dimension, true); final DocsWithFieldSet docsWithFieldSet = getDocIdSetIterator(byteArray.size()); final Map vectorsMap = Map.of(0, byteArray.get(0), 1, byteArray.get(1)); @@ -62,7 +62,7 @@ public void testByteVectorValues_whenValidInput_thenSuccess() { docsWithFieldSet, vectorsMap ); - new CompareVectorValues().validateVectorValues(knnVectorValuesForFieldWriter, byteArray, dimension, false); + new CompareVectorValues().validateVectorValues(knnVectorValuesForFieldWriter, byteArray, 2, dimension, false); final TestVectorValues.PredefinedByteVectorBinaryDocValues preDefinedByteVectorValues = new TestVectorValues.PredefinedByteVectorBinaryDocValues(byteArray); @@ -70,7 +70,7 @@ public void testByteVectorValues_whenValidInput_thenSuccess() { VectorDataType.BYTE, preDefinedByteVectorValues ); - new CompareVectorValues().validateVectorValues(knnBinaryVectorValuesBinaryDocValues, byteArray, dimension, false); + new CompareVectorValues().validateVectorValues(knnBinaryVectorValuesBinaryDocValues, byteArray, 2, dimension, false); } @SneakyThrows @@ -81,7 +81,7 @@ public void testBinaryVectorValues_whenValidInput_thenSuccess() { byteArray ); final KNNVectorValues knnVectorValues = KNNVectorValuesFactory.getVectorValues(VectorDataType.BINARY, randomVectorValues); - new CompareVectorValues().validateVectorValues(knnVectorValues, byteArray, dimension, true); + new CompareVectorValues().validateVectorValues(knnVectorValues, byteArray, 3, dimension, true); final DocsWithFieldSet docsWithFieldSet = getDocIdSetIterator(byteArray.size()); final Map vectorsMap = Map.of(0, byteArray.get(0), 1, byteArray.get(1)); @@ -90,7 +90,7 @@ public void testBinaryVectorValues_whenValidInput_thenSuccess() { docsWithFieldSet, vectorsMap ); - new CompareVectorValues().validateVectorValues(knnVectorValuesForFieldWriter, byteArray, dimension, false); + new CompareVectorValues().validateVectorValues(knnVectorValuesForFieldWriter, byteArray, 3, dimension, false); final TestVectorValues.PredefinedByteVectorBinaryDocValues preDefinedByteVectorValues = new TestVectorValues.PredefinedByteVectorBinaryDocValues(byteArray); @@ -98,7 +98,7 @@ public void testBinaryVectorValues_whenValidInput_thenSuccess() { VectorDataType.BINARY, preDefinedByteVectorValues ); - new CompareVectorValues().validateVectorValues(knnBinaryVectorValuesBinaryDocValues, byteArray, dimension, false); + new CompareVectorValues().validateVectorValues(knnBinaryVectorValuesBinaryDocValues, byteArray, 3, dimension, false); } public void testDocIdsIteratorValues_whenInvalidDisi_thenThrowException() { @@ -117,28 +117,38 @@ private DocsWithFieldSet getDocIdSetIterator(int numberOfDocIds) { } private class CompareVectorValues { - void validateVectorValues(KNNVectorValues vectorValues, List vectors, int dimension, boolean validateAddress) - throws IOException { - Assert.assertEquals(vectorValues.totalLiveDocs(), vectors.size()); + void validateVectorValues( + KNNVectorValues vectorValues, + List vectors, + int bytesPerVector, + int dimension, + boolean validateAddress + ) throws IOException { + assertEquals(vectorValues.totalLiveDocs(), vectors.size()); int docId, i = 0; T oldActual = null; int oldDocId = -1; final KNNVectorValuesIterator iterator = vectorValues.vectorValuesIterator; for (docId = iterator.nextDoc(); docId != DocIdSetIterator.NO_MORE_DOCS && i < vectors.size(); docId = iterator.nextDoc()) { T actual = vectorValues.getVector(); + T clone = vectorValues.conditionalCloneVector(); T expected = vectors.get(i); - Assert.assertNotEquals(oldDocId, docId); - Assert.assertEquals(dimension, vectorValues.dimension()); + assertNotEquals(oldDocId, docId); + assertEquals(dimension, vectorValues.dimension()); // this will check if reference is correct for the vectors. This is mainly required because for // VectorValues of Lucene when reading vectors put the vector at same reference if (oldActual != null && validateAddress) { - Assert.assertSame(actual, oldActual); + assertSame(actual, oldActual); + assertNotSame(clone, oldActual); } + oldActual = actual; // this will do the deep equals - Assert.assertArrayEquals(new Object[] { actual }, new Object[] { expected }); + assertArrayEquals(new Object[] { actual }, new Object[] { expected }); + assertArrayEquals(new Object[] { clone }, new Object[] { expected }); i++; } + assertEquals(bytesPerVector, vectorValues.bytesPerVector); } } diff --git a/src/test/java/org/opensearch/knn/jni/JNIServiceTests.java b/src/test/java/org/opensearch/knn/jni/JNIServiceTests.java index 53245cc62..c78478f4d 100644 --- a/src/test/java/org/opensearch/knn/jni/JNIServiceTests.java +++ b/src/test/java/org/opensearch/knn/jni/JNIServiceTests.java @@ -86,7 +86,7 @@ public static void setUpClass() throws IOException { public void testCreateIndex_invalid_engineNotSupported() { expectThrows( IllegalArgumentException.class, - () -> JNIService.createIndex( + () -> TestUtils.createIndex( new int[] {}, 0, 0, @@ -100,21 +100,14 @@ public void testCreateIndex_invalid_engineNotSupported() { public void testCreateIndex_invalid_engineNull() { expectThrows( Exception.class, - () -> JNIService.createIndex( - new int[] {}, - 0, - 0, - "test", - ImmutableMap.of(KNNConstants.SPACE_TYPE, SpaceType.L2.getValue()), - null - ) + () -> TestUtils.createIndex(new int[] {}, 0, 0, "test", ImmutableMap.of(KNNConstants.SPACE_TYPE, SpaceType.L2.getValue()), null) ); } public void testCreateIndex_nmslib_invalid_noSpaceType() { expectThrows( Exception.class, - () -> JNIService.createIndex( + () -> TestUtils.createIndex( testData.indexData.docs, testData.loadDataToMemoryAddress(), testData.indexData.getDimension(), @@ -133,7 +126,7 @@ public void testCreateIndex_nmslib_invalid_vectorDocIDMismatch() throws IOExcept Path tmpFile1 = createTempFile(); expectThrows( Exception.class, - () -> JNIService.createIndex( + () -> TestUtils.createIndex( docIds, memoryAddress, vectors1[0].length, @@ -149,7 +142,7 @@ public void testCreateIndex_nmslib_invalid_vectorDocIDMismatch() throws IOExcept Path tmpFile2 = createTempFile(); expectThrows( Exception.class, - () -> JNIService.createIndex( + () -> TestUtils.createIndex( docIds, memoryAddress2, vectors2[0].length, @@ -168,7 +161,7 @@ public void testCreateIndex_nmslib_invalid_nullArgument() throws IOException { Path tmpFile = createTempFile(); expectThrows( Exception.class, - () -> JNIService.createIndex( + () -> TestUtils.createIndex( null, memoryAddress, 0, @@ -180,7 +173,7 @@ public void testCreateIndex_nmslib_invalid_nullArgument() throws IOException { expectThrows( Exception.class, - () -> JNIService.createIndex( + () -> TestUtils.createIndex( docIds, 0, 0, @@ -192,7 +185,7 @@ public void testCreateIndex_nmslib_invalid_nullArgument() throws IOException { expectThrows( Exception.class, - () -> JNIService.createIndex( + () -> TestUtils.createIndex( docIds, memoryAddress, 0, @@ -204,12 +197,12 @@ public void testCreateIndex_nmslib_invalid_nullArgument() throws IOException { expectThrows( Exception.class, - () -> JNIService.createIndex(docIds, memoryAddress, 0, tmpFile.toAbsolutePath().toString(), null, KNNEngine.NMSLIB) + () -> TestUtils.createIndex(docIds, memoryAddress, 0, tmpFile.toAbsolutePath().toString(), null, KNNEngine.NMSLIB) ); expectThrows( Exception.class, - () -> JNIService.createIndex( + () -> TestUtils.createIndex( docIds, memoryAddress, 0, @@ -228,7 +221,7 @@ public void testCreateIndex_nmslib_invalid_badSpace() throws IOException { Path tmpFile = createTempFile(); expectThrows( Exception.class, - () -> JNIService.createIndex( + () -> TestUtils.createIndex( docIds, memoryAddress, vectors[0].length, @@ -254,7 +247,7 @@ public void testCreateIndex_nmslib_invalid_badParameterType() throws IOException Path tmpFile = createTempFile(); expectThrows( Exception.class, - () -> JNIService.createIndex( + () -> TestUtils.createIndex( docIds, memoryAddress, vectors[0].length, @@ -274,7 +267,7 @@ public void testCreateIndex_nmslib_valid() throws IOException { Path tmpFile = createTempFile(); - JNIService.createIndex( + TestUtils.createIndex( testData.indexData.docs, testData.loadDataToMemoryAddress(), testData.indexData.getDimension(), @@ -286,7 +279,7 @@ public void testCreateIndex_nmslib_valid() throws IOException { tmpFile = createTempFile(); - JNIService.createIndex( + TestUtils.createIndex( testData.indexData.docs, testData.loadDataToMemoryAddress(), testData.indexData.getDimension(), @@ -310,7 +303,7 @@ public void testCreateIndex_faiss_invalid_noSpaceType() { expectThrows( Exception.class, - () -> JNIService.createIndex( + () -> TestUtils.createIndex( docIds, testData.loadDataToMemoryAddress(), testData.indexData.getDimension(), @@ -329,7 +322,7 @@ public void testCreateIndex_faiss_invalid_vectorDocIDMismatch() throws IOExcepti Path tmpFile1 = createTempFile(); expectThrows( Exception.class, - () -> JNIService.createIndex( + () -> TestUtils.createIndex( docIds, memoryAddress, vectors1[0].length, @@ -344,7 +337,7 @@ public void testCreateIndex_faiss_invalid_vectorDocIDMismatch() throws IOExcepti Path tmpFile2 = createTempFile(); expectThrows( Exception.class, - () -> JNIService.createIndex( + () -> TestUtils.createIndex( docIds, memoryAddress, vectors2[0].length, @@ -364,7 +357,7 @@ public void testCreateIndex_faiss_invalid_null() throws IOException { Path tmpFile = createTempFile(); expectThrows( Exception.class, - () -> JNIService.createIndex( + () -> TestUtils.createIndex( null, memoryAddress, 0, @@ -376,7 +369,7 @@ public void testCreateIndex_faiss_invalid_null() throws IOException { expectThrows( Exception.class, - () -> JNIService.createIndex( + () -> TestUtils.createIndex( docIds, 0, 0, @@ -388,7 +381,7 @@ public void testCreateIndex_faiss_invalid_null() throws IOException { expectThrows( Exception.class, - () -> JNIService.createIndex( + () -> TestUtils.createIndex( docIds, testData.loadDataToMemoryAddress(), testData.indexData.getDimension(), @@ -400,7 +393,7 @@ public void testCreateIndex_faiss_invalid_null() throws IOException { expectThrows( Exception.class, - () -> JNIService.createIndex( + () -> TestUtils.createIndex( docIds, testData.loadDataToMemoryAddress(), testData.indexData.getDimension(), @@ -412,7 +405,7 @@ public void testCreateIndex_faiss_invalid_null() throws IOException { expectThrows( Exception.class, - () -> JNIService.createIndex( + () -> TestUtils.createIndex( docIds, testData.loadDataToMemoryAddress(), testData.indexData.getDimension(), @@ -432,7 +425,7 @@ public void testCreateIndex_faiss_invalid_invalidSpace() throws IOException { Path tmpFile = createTempFile(); expectThrows( Exception.class, - () -> JNIService.createIndex( + () -> TestUtils.createIndex( docIds, memoryAddress, vectors[0].length, @@ -452,7 +445,7 @@ public void testCreateIndex_faiss_invalid_noIndexDescription() throws IOExceptio Path tmpFile = createTempFile(); expectThrows( Exception.class, - () -> JNIService.createIndex( + () -> TestUtils.createIndex( docIds, memoryAddress, vectors[0].length, @@ -470,7 +463,7 @@ public void testCreateIndex_faiss_invalid_invalidIndexDescription() throws IOExc Path tmpFile = createTempFile(); expectThrows( Exception.class, - () -> JNIService.createIndex( + () -> TestUtils.createIndex( docIds, memoryAddress, vectors[0].length, @@ -493,7 +486,7 @@ public void testCreateIndex_faiss_sqfp16_invalidIndexDescription() { Path tmpFile = createTempFile(); expectThrows( Exception.class, - () -> JNIService.createIndex( + () -> TestUtils.createIndex( docIds, memoryAddress, vectors[0].length, @@ -517,7 +510,7 @@ public void testLoadIndex_faiss_sqfp16_valid() { String sqfp16IndexDescription = "HNSW16,SQfp16"; long memoryAddress = JNICommons.storeVectorData(0, vectors, (long) vectors.length * vectors[0].length); Path tmpFile = createTempFile(); - JNIService.createIndex( + TestUtils.createIndex( docIds, memoryAddress, vectors[0].length, @@ -540,7 +533,7 @@ public void testQueryIndex_faiss_sqfp16_valid() { float[][] truncatedVectors = truncateToFp16Range(testData.indexData.vectors); long memoryAddress = JNICommons.storeVectorData(0, truncatedVectors, (long) truncatedVectors.length * truncatedVectors[0].length); Path tmpFile = createTempFile(); - JNIService.createIndex( + TestUtils.createIndex( testData.indexData.docs, memoryAddress, testData.indexData.getDimension(), @@ -634,7 +627,7 @@ public void testCreateIndex_faiss_invalid_invalidParameterType() throws IOExcept Path tmpFile = createTempFile(); expectThrows( Exception.class, - () -> JNIService.createIndex( + () -> TestUtils.createIndex( docIds, testData.loadDataToMemoryAddress(), testData.indexData.getDimension(), @@ -660,7 +653,7 @@ public void testCreateIndex_faiss_valid() throws IOException { for (String method : methods) { for (SpaceType spaceType : spaces) { Path tmpFile1 = createTempFile(); - JNIService.createIndex( + TestUtils.createIndex( testData.indexData.docs, testData.loadDataToMemoryAddress(), testData.indexData.getDimension(), @@ -677,7 +670,7 @@ public void testCreateIndex_faiss_valid() throws IOException { public void testCreateIndex_binary_faiss_valid() { Path tmpFile1 = createTempFile(); long memoryAddr = testData.loadBinaryDataToMemoryAddress(); - JNIService.createIndex( + TestUtils.createIndex( testData.indexData.docs, memoryAddr, testData.indexData.getDimension(), @@ -733,7 +726,7 @@ public void testLoadIndex_nmslib_valid() throws IOException { Path tmpFile = createTempFile(); - JNIService.createIndex( + TestUtils.createIndex( testData.indexData.docs, testData.loadDataToMemoryAddress(), testData.indexData.getDimension(), @@ -769,7 +762,7 @@ public void testLoadIndex_faiss_valid() throws IOException { Path tmpFile = createTempFile(); - JNIService.createIndex( + TestUtils.createIndex( testData.indexData.docs, testData.loadDataToMemoryAddress(), testData.indexData.getDimension(), @@ -799,7 +792,7 @@ public void testQueryIndex_nmslib_invalid_nullQueryVector() throws IOException { Path tmpFile = createTempFile(); - JNIService.createIndex( + TestUtils.createIndex( testData.indexData.docs, testData.loadDataToMemoryAddress(), testData.indexData.getDimension(), @@ -829,7 +822,7 @@ public void testQueryIndex_nmslib_valid() throws IOException { Path tmpFile = createTempFile(); - JNIService.createIndex( + TestUtils.createIndex( testData.indexData.docs, testData.loadDataToMemoryAddress(), testData.indexData.getDimension(), @@ -862,7 +855,7 @@ public void testQueryIndex_faiss_invalid_nullQueryVector() throws IOException { Path tmpFile = createTempFile(); - JNIService.createIndex( + TestUtils.createIndex( testData.indexData.docs, testData.loadDataToMemoryAddress(), testData.indexData.getDimension(), @@ -888,7 +881,7 @@ public void testQueryIndex_faiss_valid() throws IOException { for (String method : methods) { for (SpaceType spaceType : spaces) { Path tmpFile = createTempFile(); - JNIService.createIndex( + TestUtils.createIndex( testData.indexData.docs, testData.loadDataToMemoryAddress(), testData.indexData.getDimension(), @@ -949,7 +942,7 @@ public void testQueryIndex_faiss_parentIds() throws IOException { for (String method : methods) { for (SpaceType spaceType : spaces) { Path tmpFile = createTempFile(); - JNIService.createIndex( + TestUtils.createIndex( testDataNested.indexData.docs, testData.loadDataToMemoryAddress(), testDataNested.indexData.getDimension(), @@ -992,7 +985,7 @@ public void testQueryBinaryIndex_faiss_valid() { for (String method : methods) { Path tmpFile = createTempFile(); long memoryAddr = testData.loadBinaryDataToMemoryAddress(); - JNIService.createIndex( + TestUtils.createIndex( testData.indexData.docs, memoryAddr, testData.indexData.getDimension(), @@ -1071,7 +1064,7 @@ public void testFree_nmslib_valid() throws IOException { Path tmpFile = createTempFile(); - JNIService.createIndex( + TestUtils.createIndex( testData.indexData.docs, testData.loadDataToMemoryAddress(), testData.indexData.getDimension(), @@ -1095,7 +1088,7 @@ public void testFree_faiss_valid() throws IOException { Path tmpFile = createTempFile(); - JNIService.createIndex( + TestUtils.createIndex( testData.indexData.docs, testData.loadDataToMemoryAddress(), testData.indexData.getDimension(), @@ -1235,7 +1228,7 @@ private long transferVectors(int numDuplicates) { return trainPointer1; } - public void testCreateIndexFromTemplate() throws IOException { + public void createIndexFromTemplate() throws IOException { long trainPointer1 = JNIService.transferVectors(0, testData.indexData.vectors); assertNotEquals(0, trainPointer1); @@ -1445,7 +1438,7 @@ private String createFaissIVFPQIndex(int ivfNlist, int pqM, int pqCodeSize, Spac private String createFaissHNSWIndex(SpaceType spaceType) throws IOException { Path tmpFile = createTempFile(); - JNIService.createIndex( + TestUtils.createIndex( testData.indexData.docs, testData.loadDataToMemoryAddress(), testData.indexData.getDimension(), diff --git a/src/testFixtures/java/org/opensearch/knn/TestUtils.java b/src/testFixtures/java/org/opensearch/knn/TestUtils.java index 28a60aecb..afc8f0b92 100644 --- a/src/testFixtures/java/org/opensearch/knn/TestUtils.java +++ b/src/testFixtures/java/org/opensearch/knn/TestUtils.java @@ -17,7 +17,9 @@ import java.io.IOException; import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.codec.util.SerializationMode; +import org.opensearch.knn.index.engine.KNNEngine; import org.opensearch.knn.jni.JNICommons; +import org.opensearch.knn.jni.JNIService; import org.opensearch.knn.plugin.script.KNNScoringUtil; import java.util.Collections; @@ -392,11 +394,11 @@ private void initBinaryData() { } public long loadDataToMemoryAddress() { - return JNICommons.storeVectorData(0, indexData.vectors, (long) indexData.vectors.length * indexData.vectors[0].length); + return JNICommons.storeVectorData(0, indexData.vectors, (long) indexData.vectors.length * indexData.vectors[0].length, true); } public long loadBinaryDataToMemoryAddress() { - return JNICommons.storeByteVectorData(0, indexBinaryData, (long) indexBinaryData.length * indexBinaryData[0].length); + return JNICommons.storeByteVectorData(0, indexBinaryData, (long) indexBinaryData.length * indexBinaryData[0].length, true); } @AllArgsConstructor @@ -409,4 +411,15 @@ public static class Pair { public float[][] vectors; } } + + public static void createIndex(int[] ids, long address, int dimension, String name, Map parameters, KNNEngine engine) { + if (engine != KNNEngine.FAISS) { + JNIService.createIndex(ids, address, dimension, name, parameters, engine); + } else { + // We can initialize numDocs as 0, this will just not reserve anything. + long indexAddress = JNIService.initIndex(0, dimension, parameters, engine); + JNIService.insertToIndex(ids, address, dimension, parameters, indexAddress, engine); + JNIService.writeIndex(name, indexAddress, engine, parameters); + } + } } From abce10e21a2ee87f4f280fbfc5a504b7fd9c41c5 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Mon, 26 Aug 2024 10:50:52 -0500 Subject: [PATCH 338/416] Add HNSW changes to support Faiss byte vector (#1823) (#2003) * Add HNSW changes to support Faiss byte vector Signed-off-by: Naveen Tatikonda * Address Review Comments Signed-off-by: Naveen Tatikonda --------- Signed-off-by: Naveen Tatikonda (cherry picked from commit 59c312bf54f439032c28b2cf0d5c84d8c800071a) Co-authored-by: Naveen Tatikonda --- CHANGELOG.md | 1 + jni/include/commons.h | 26 ++- jni/include/faiss_index_service.h | 57 +++++++ jni/include/jni_util.h | 7 +- .../org_opensearch_knn_jni_FaissService.h | 30 ++++ .../org_opensearch_knn_jni_JNICommons.h | 20 ++- jni/src/commons.cpp | 31 +++- jni/src/faiss_index_service.cpp | 113 +++++++++++++ jni/src/jni_util.cpp | 34 +++- .../org_opensearch_knn_jni_FaissService.cpp | 41 +++++ jni/src/org_opensearch_knn_jni_JNICommons.cpp | 22 +++ jni/tests/faiss_index_service_test.cpp | 44 ++++++ jni/tests/test_util.cpp | 8 +- jni/tests/test_util.h | 4 +- .../opensearch/knn/common/KNNConstants.java | 1 + .../transfer/OffHeapBinaryVectorTransfer.java | 14 +- .../OffHeapVectorTransferFactory.java | 2 +- .../engine/faiss/AbstractFaissMethod.java | 17 +- .../index/engine/faiss/FaissHNSWMethod.java | 8 +- .../index/engine/faiss/FaissIVFMethod.java | 2 +- .../mapper/KNNVectorFieldMapperUtil.java | 2 +- .../index/memory/NativeMemoryAllocation.java | 2 +- .../knn/index/query/KNNQueryBuilder.java | 60 ++++--- .../opensearch/knn/index/util/IndexUtil.java | 12 ++ .../org/opensearch/knn/jni/FaissService.java | 33 ++++ .../org/opensearch/knn/jni/JNICommons.java | 66 +++++++- .../org/opensearch/knn/jni/JNIService.java | 12 +- .../training/ByteTrainingDataConsumer.java | 2 +- .../knn/index/VectorDataTypeIT.java | 148 +++++++++++++++++- .../OffHeapVectorTransferFactoryTests.java | 2 +- .../index/engine/KNNMethodContextTests.java | 6 +- .../memory/NativeMemoryAllocationTests.java | 2 +- .../memory/NativeMemoryLoadStrategyTests.java | 2 +- .../knn/index/query/KNNQueryBuilderTests.java | 1 + .../java/org/opensearch/knn/TestUtils.java | 2 +- 35 files changed, 776 insertions(+), 58 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f469f4a1f..66d9396c6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Features * Integrate Lucene Vector field with native engines to use KNNVectorFormat during segment creation [#1945](https://github.com/opensearch-project/k-NN/pull/1945) * k-NN query rescore support for native engines [#1984](https://github.com/opensearch-project/k-NN/pull/1984) +* Add support for byte vector with Faiss Engine HNSW algorithm [#1823](https://github.com/opensearch-project/k-NN/pull/1823) ### Enhancements * Adds iterative graph build capability into a faiss index to improve the memory footprint during indexing and Integrates KNNVectorsFormat for native engines[#1950](https://github.com/opensearch-project/k-NN/pull/1950) ### Bug Fixes diff --git a/jni/include/commons.h b/jni/include/commons.h index 4cdaf28fc..3f1ee19a3 100644 --- a/jni/include/commons.h +++ b/jni/include/commons.h @@ -47,10 +47,26 @@ namespace knn_jni { * CAUTION: The behavior is undefined if the memory address is deallocated and the method is called * * @param memoryAddress The address of the memory location where data will be stored. - * @param data 2D byte array containing data to be stored in native memory. + * @param data 2D byte array containing binary data to be stored in native memory. * @param initialCapacity The initial capacity of the memory location. + * @param append whether to append or start from index 0 when called subsequently with the same address * @return memory address of std::vector where the data is stored. */ + jlong storeBinaryVectorData(knn_jni::JNIUtilInterface *, JNIEnv *, jlong , jobjectArray, jlong, jboolean); + + /** + * This is utility function that can be used to store signed int8 data in native memory. This function will allocate memory for + * the data(rows*columns) with initialCapacity and return the memory address where the data is stored. + * If you are using this function for first time use memoryAddress = 0 to ensure that a new memory location is created. + * For subsequent calls you can pass the same memoryAddress. If the data cannot be stored in the memory location + * will throw Exception. + * + * @param memoryAddress The address of the memory location where data will be stored. + * @param data 2D byte array containing int8 data to be stored in native memory. + * @param initialCapacity The initial capacity of the memory location. + * @param append whether to append or start from index 0 when called subsequently with the same address + * @return memory address of std::vector where the data is stored. + */ jlong storeByteVectorData(knn_jni::JNIUtilInterface *, JNIEnv *, jlong , jobjectArray, jlong, jboolean); /** @@ -69,6 +85,14 @@ namespace knn_jni { */ void freeByteVectorData(jlong); + /** + * Free up the memory allocated for the data stored in memory address. This function should be used with the memory + * address returned by {@link JNICommons#storeBinaryVectorData(long, byte[][], long, long)} + * + * @param memoryAddress address to be freed. + */ + void freeBinaryVectorData(jlong); + /** * Extracts query time efSearch from method parameters **/ diff --git a/jni/include/faiss_index_service.h b/jni/include/faiss_index_service.h index c57309cfc..29ec90e80 100644 --- a/jni/include/faiss_index_service.h +++ b/jni/include/faiss_index_service.h @@ -125,6 +125,63 @@ class BinaryIndexService : public IndexService { virtual void allocIndex(faiss::Index * index, size_t dim, size_t numVectors) override; }; +/** + * A class to provide operations on index + * This class should evolve to have only cpp object but not jni object + */ +class ByteIndexService : public IndexService { +public: + //TODO Remove dependency on JNIUtilInterface and JNIEnv + //TODO Reduce the number of parameters + ByteIndexService(std::unique_ptr faissMethods); + +/** + * Initialize index + * + * @param jniUtil jni util + * @param env jni environment + * @param metric space type for distance calculation + * @param indexDescription index description to be used by faiss index factory + * @param dim dimension of vectors + * @param numVectors number of vectors + * @param threadCount number of thread count to be used while adding data + * @param parameters parameters to be applied to faiss index + * @return memory address of the native index object + */ + virtual jlong initIndex(knn_jni::JNIUtilInterface *jniUtil, JNIEnv *env, faiss::MetricType metric, std::string indexDescription, int dim, int numVectors, int threadCount, std::unordered_map parameters) override; + /** + * Add vectors to index + * + * @param jniUtil jni util + * @param env jni environment + * @param metric space type for distance calculation + * @param indexDescription index description to be used by faiss index factory + * @param dim dimension of vectors + * @param numIds number of vectors + * @param threadCount number of thread count to be used while adding data + * @param vectorsAddress memory address which is holding vector data + * @param idMap a map of document id and vector id + * @param parameters parameters to be applied to faiss index + */ + virtual void insertToIndex(int dim, int numIds, int threadCount, int64_t vectorsAddress, std::vector &ids, jlong idMapAddress) override; + /** + * Write index to disk + * + * @param jniUtil jni util + * @param env jni environment + * @param metric space type for distance calculation + * @param indexDescription index description to be used by faiss index factory + * @param threadCount number of thread count to be used while adding data + * @param indexPath path to write index + * @param idMap a map of document id and vector id + * @param parameters parameters to be applied to faiss index + */ + virtual void writeIndex(std::string indexPath, jlong idMapAddress) override; + virtual ~ByteIndexService() = default; +protected: + virtual void allocIndex(faiss::Index * index, size_t dim, size_t numVectors) override; +}; + } } diff --git a/jni/include/jni_util.h b/jni/include/jni_util.h index 1579522d0..825471a3c 100644 --- a/jni/include/jni_util.h +++ b/jni/include/jni_util.h @@ -71,8 +71,10 @@ namespace knn_jni { virtual void Convert2dJavaObjectArrayAndStoreToFloatVector(JNIEnv *env, jobjectArray array2dJ, int dim, std::vector *vect ) = 0; - virtual void Convert2dJavaObjectArrayAndStoreToByteVector(JNIEnv *env, jobjectArray array2dJ, + virtual void Convert2dJavaObjectArrayAndStoreToBinaryVector(JNIEnv *env, jobjectArray array2dJ, int dim, std::vector *vect ) = 0; + virtual void Convert2dJavaObjectArrayAndStoreToByteVector(JNIEnv *env, jobjectArray array2dJ, + int dim, std::vector *vect ) = 0; virtual std::vector ConvertJavaIntArrayToCppIntVector(JNIEnv *env, jintArray arrayJ) = 0; @@ -173,7 +175,8 @@ namespace knn_jni { void SetObjectArrayElement(JNIEnv *env, jobjectArray array, jsize index, jobject val); void SetByteArrayRegion(JNIEnv *env, jbyteArray array, jsize start, jsize len, const jbyte * buf); void Convert2dJavaObjectArrayAndStoreToFloatVector(JNIEnv *env, jobjectArray array2dJ, int dim, std::vector *vect); - void Convert2dJavaObjectArrayAndStoreToByteVector(JNIEnv *env, jobjectArray array2dJ, int dim, std::vector *vect); + void Convert2dJavaObjectArrayAndStoreToBinaryVector(JNIEnv *env, jobjectArray array2dJ, int dim, std::vector *vect); + void Convert2dJavaObjectArrayAndStoreToByteVector(JNIEnv *env, jobjectArray array2dJ, int dim, std::vector *vect); private: std::unordered_map cachedClasses; diff --git a/jni/include/org_opensearch_knn_jni_FaissService.h b/jni/include/org_opensearch_knn_jni_FaissService.h index 19e13d402..09f3ec8b7 100644 --- a/jni/include/org_opensearch_knn_jni_FaissService.h +++ b/jni/include/org_opensearch_knn_jni_FaissService.h @@ -34,6 +34,16 @@ JNIEXPORT jlong JNICALL Java_org_opensearch_knn_jni_FaissService_initIndex(JNIEn JNIEXPORT jlong JNICALL Java_org_opensearch_knn_jni_FaissService_initBinaryIndex(JNIEnv * env, jclass cls, jlong numDocs, jint dimJ, jobject parametersJ); + +/* + * Class: org_opensearch_knn_jni_FaissService + * Method: initByteIndex + * Signature: ([IJILjava/lang/String;Ljava/util/Map;)V + */ +JNIEXPORT jlong JNICALL Java_org_opensearch_knn_jni_FaissService_initByteIndex(JNIEnv * env, jclass cls, + jlong numDocs, jint dimJ, + jobject parametersJ); + /* * Class: org_opensearch_knn_jni_FaissService * Method: insertToIndex @@ -50,6 +60,16 @@ JNIEXPORT void JNICALL Java_org_opensearch_knn_jni_FaissService_insertToIndex(JN JNIEXPORT void JNICALL Java_org_opensearch_knn_jni_FaissService_insertToBinaryIndex(JNIEnv * env, jclass cls, jintArray idsJ, jlong vectorsAddressJ, jint dimJ, jlong indexAddress, jint threadCount); + +/* + * Class: org_opensearch_knn_jni_FaissService + * Method: insertToByteIndex + * Signature: ([IJILjava/lang/String;Ljava/util/Map;)V + */ +JNIEXPORT void JNICALL Java_org_opensearch_knn_jni_FaissService_insertToByteIndex(JNIEnv * env, jclass cls, jintArray idsJ, + jlong vectorsAddressJ, jint dimJ, + jlong indexAddress, jint threadCount); + /* * Class: org_opensearch_knn_jni_FaissService * Method: writeIndex @@ -66,6 +86,16 @@ JNIEXPORT void JNICALL Java_org_opensearch_knn_jni_FaissService_writeIndex(JNIEn JNIEXPORT void JNICALL Java_org_opensearch_knn_jni_FaissService_writeBinaryIndex(JNIEnv * env, jclass cls, jlong indexAddress, jstring indexPathJ); + +/* + * Class: org_opensearch_knn_jni_FaissService + * Method: writeByteIndex + * Signature: ([IJILjava/lang/String;Ljava/util/Map;)V + */ +JNIEXPORT void JNICALL Java_org_opensearch_knn_jni_FaissService_writeByteIndex(JNIEnv * env, jclass cls, + jlong indexAddress, + jstring indexPathJ); + /* * Class: org_opensearch_knn_jni_FaissService * Method: createIndexFromTemplate diff --git a/jni/include/org_opensearch_knn_jni_JNICommons.h b/jni/include/org_opensearch_knn_jni_JNICommons.h index 03c0d023a..8bfbcc266 100644 --- a/jni/include/org_opensearch_knn_jni_JNICommons.h +++ b/jni/include/org_opensearch_knn_jni_JNICommons.h @@ -28,7 +28,15 @@ JNIEXPORT jlong JNICALL Java_org_opensearch_knn_jni_JNICommons_storeVectorData /* * Class: org_opensearch_knn_jni_JNICommons - * Method: storeVectorData + * Method: storeBinaryVectorData + * Signature: (J[[FJJ) + */ +JNIEXPORT jlong JNICALL Java_org_opensearch_knn_jni_JNICommons_storeBinaryVectorData + (JNIEnv *, jclass, jlong, jobjectArray, jlong, jboolean); + +/* + * Class: org_opensearch_knn_jni_JNICommons + * Method: storeByteVectorData * Signature: (J[[FJJ) */ JNIEXPORT jlong JNICALL Java_org_opensearch_knn_jni_JNICommons_storeByteVectorData @@ -44,7 +52,15 @@ JNIEXPORT void JNICALL Java_org_opensearch_knn_jni_JNICommons_freeVectorData /* * Class: org_opensearch_knn_jni_JNICommons -* Method: freeVectorData +* Method: freeBinaryVectorData +* Signature: (J)V +*/ +JNIEXPORT void JNICALL Java_org_opensearch_knn_jni_JNICommons_freeBinaryVectorData +(JNIEnv *, jclass, jlong); + +/* +* Class: org_opensearch_knn_jni_JNICommons +* Method: freeByteVectorData * Signature: (J)V */ JNIEXPORT void JNICALL Java_org_opensearch_knn_jni_JNICommons_freeByteVectorData diff --git a/jni/src/commons.cpp b/jni/src/commons.cpp index f9764db73..38e3ac8a4 100644 --- a/jni/src/commons.cpp +++ b/jni/src/commons.cpp @@ -37,7 +37,7 @@ jlong knn_jni::commons::storeVectorData(knn_jni::JNIUtilInterface *jniUtil, JNIE return (jlong) vect; } -jlong knn_jni::commons::storeByteVectorData(knn_jni::JNIUtilInterface *jniUtil, JNIEnv *env, jlong memoryAddressJ, +jlong knn_jni::commons::storeBinaryVectorData(knn_jni::JNIUtilInterface *jniUtil, JNIEnv *env, jlong memoryAddressJ, jobjectArray dataJ, jlong initialCapacityJ, jboolean appendJ) { std::vector *vect; if ((long) memoryAddressJ == 0) { @@ -51,6 +51,26 @@ jlong knn_jni::commons::storeByteVectorData(knn_jni::JNIUtilInterface *jniUtil, vect->clear(); } + int dim = jniUtil->GetInnerDimensionOf2dJavaByteArray(env, dataJ); + jniUtil->Convert2dJavaObjectArrayAndStoreToBinaryVector(env, dataJ, dim, vect); + + return (jlong) vect; +} + +jlong knn_jni::commons::storeByteVectorData(knn_jni::JNIUtilInterface *jniUtil, JNIEnv *env, jlong memoryAddressJ, + jobjectArray dataJ, jlong initialCapacityJ, jboolean appendJ) { + std::vector *vect; + if (memoryAddressJ == 0) { + vect = new std::vector(); + vect->reserve(static_cast(initialCapacityJ)); + } else { + vect = reinterpret_cast*>(memoryAddressJ); + } + + if (appendJ == JNI_FALSE) { + vect->clear(); + } + int dim = jniUtil->GetInnerDimensionOf2dJavaByteArray(env, dataJ); jniUtil->Convert2dJavaObjectArrayAndStoreToByteVector(env, dataJ, dim, vect); @@ -64,13 +84,20 @@ void knn_jni::commons::freeVectorData(jlong memoryAddressJ) { } } -void knn_jni::commons::freeByteVectorData(jlong memoryAddressJ) { +void knn_jni::commons::freeBinaryVectorData(jlong memoryAddressJ) { if (memoryAddressJ != 0) { auto *vect = reinterpret_cast*>(memoryAddressJ); delete vect; } } +void knn_jni::commons::freeByteVectorData(jlong memoryAddressJ) { + if (memoryAddressJ != 0) { + auto *vect = reinterpret_cast*>(memoryAddressJ); + delete vect; + } +} + int knn_jni::commons::getIntegerMethodParameter(JNIEnv * env, knn_jni::JNIUtilInterface * jniUtil, std::unordered_map methodParams, std::string methodParam, int defaultValue) { if (methodParams.empty()) { return defaultValue; diff --git a/jni/src/faiss_index_service.cpp b/jni/src/faiss_index_service.cpp index f76c54428..16ded4bcb 100644 --- a/jni/src/faiss_index_service.cpp +++ b/jni/src/faiss_index_service.cpp @@ -250,5 +250,118 @@ void BinaryIndexService::writeIndex( } } +ByteIndexService::ByteIndexService(std::unique_ptr faissMethods) : IndexService(std::move(faissMethods)) {} + +void ByteIndexService::allocIndex(faiss::Index * index, size_t dim, size_t numVectors) { + if(auto * indexHNSWSQ = dynamic_cast(index)) { + if(auto * indexScalarQuantizer = dynamic_cast(indexHNSWSQ->storage)) { + indexScalarQuantizer->codes.reserve(indexScalarQuantizer->code_size * numVectors); + } + return; + } +} + +jlong ByteIndexService::initIndex( + knn_jni::JNIUtilInterface * jniUtil, + JNIEnv * env, + faiss::MetricType metric, + std::string indexDescription, + int dim, + int numVectors, + int threadCount, + std::unordered_map parameters + ) { + // Create index using Faiss factory method + std::unique_ptr index(faissMethods->indexFactory(dim, indexDescription.c_str(), metric)); + + // Set thread count if it is passed in as a parameter. Setting this variable will only impact the current thread + if(threadCount != 0) { + omp_set_num_threads(threadCount); + } + + // Add extra parameters that cant be configured with the index factory + SetExtraParameters(jniUtil, env, parameters, index.get()); + + // Check that the index does not need to be trained + if(!index->is_trained) { + throw std::runtime_error("Index is not trained"); + } + + std::unique_ptr idMap (faissMethods->indexIdMap(index.get())); + //Makes sure the index is deleted when the destructor is called, this cannot be passed in the constructor + idMap->own_fields = true; + + allocIndex(dynamic_cast(idMap->index), dim, numVectors); + + //Release the ownership so as to make sure not delete the underlying index that is created. The index is needed later + //in insert and write operations + index.release(); + return reinterpret_cast(idMap.release()); +} + +void ByteIndexService::insertToIndex( + int dim, + int numIds, + int threadCount, + int64_t vectorsAddress, + std::vector & ids, + jlong idMapAddress + ) { + // Read vectors from memory address + auto *inputVectors = reinterpret_cast*>(vectorsAddress); + + // The number of vectors can be int here because a lucene segment number of total docs never crosses INT_MAX value + int numVectors = inputVectors->size() / dim; + if(numVectors == 0) { + throw std::runtime_error("Number of vectors cannot be 0"); + } + + if (numIds != numVectors) { + throw std::runtime_error("Number of IDs does not match number of vectors"); + } + + // Set thread count if it is passed in as a parameter. Setting this variable will only impact the current thread + if(threadCount != 0) { + omp_set_num_threads(threadCount); + } + + faiss::IndexIDMap * idMap = reinterpret_cast (idMapAddress); + + // Add vectors in batches by casting int8 vectors into float with a batch size of 1000 to avoid additional memory spike. + // Refer to this github issue for more details https://github.com/opensearch-project/k-NN/issues/1659#issuecomment-2307390255 + int batchSize = 1000; + std::vector inputFloatVectors(batchSize * dim); + std::vector floatVectorsIds(batchSize); + int id = 0; + auto iter = inputVectors->begin(); + + for (int id = 0; id < numVectors; id += batchSize) { + if (numVectors - id < batchSize) { + batchSize = numVectors - id; + } + + for (int i = 0; i < batchSize; ++i) { + floatVectorsIds[i] = ids[id + i]; + for (int j = 0; j < dim; ++j, ++iter) { + inputFloatVectors[i * dim + j] = static_cast(*iter); + } + } + idMap->add_with_ids(batchSize, inputFloatVectors.data(), floatVectorsIds.data()); + } +} + +void ByteIndexService::writeIndex( + std::string indexPath, + jlong idMapAddress + ) { + std::unique_ptr idMap (reinterpret_cast (idMapAddress)); + + try { + // Write the index to disk + faissMethods->writeIndex(idMap.get(), indexPath.c_str()); + } catch(std::exception &e) { + throw std::runtime_error("Failed to write index to disk"); + } +} } // namespace faiss_wrapper } // namesapce knn_jni \ No newline at end of file diff --git a/jni/src/jni_util.cpp b/jni/src/jni_util.cpp index ee4c382b5..82900b5ce 100644 --- a/jni/src/jni_util.cpp +++ b/jni/src/jni_util.cpp @@ -261,7 +261,7 @@ void knn_jni::JNIUtil::Convert2dJavaObjectArrayAndStoreToFloatVector(JNIEnv *env env->DeleteLocalRef(array2dJ); } -void knn_jni::JNIUtil::Convert2dJavaObjectArrayAndStoreToByteVector(JNIEnv *env, jobjectArray array2dJ, +void knn_jni::JNIUtil::Convert2dJavaObjectArrayAndStoreToBinaryVector(JNIEnv *env, jobjectArray array2dJ, int dim, std::vector *vect) { if (array2dJ == nullptr) { @@ -294,6 +294,38 @@ void knn_jni::JNIUtil::Convert2dJavaObjectArrayAndStoreToByteVector(JNIEnv *env, env->DeleteLocalRef(array2dJ); } +void knn_jni::JNIUtil::Convert2dJavaObjectArrayAndStoreToByteVector(JNIEnv *env, jobjectArray array2dJ, + int dim, std::vector *vect) { + + if (array2dJ == nullptr) { + throw std::runtime_error("Array cannot be null"); + } + + int numVectors = env->GetArrayLength(array2dJ); + this->HasExceptionInStack(env, "Unable to get array length"); + + for (int i = 0; i < numVectors; ++i) { + auto vectorArray = static_cast(env->GetObjectArrayElement(array2dJ, i)); + this->HasExceptionInStack(env, "Unable to get object array element"); + + if (dim != env->GetArrayLength(vectorArray)) { + env->DeleteLocalRef(array2dJ); + throw std::runtime_error("Dimension of vectors is inconsistent"); + } + + int8_t* vector = reinterpret_cast(env->GetByteArrayElements(vectorArray, nullptr)); + if (vector == nullptr) { + this->HasExceptionInStack(env); + throw std::runtime_error("Unable to get byte array elements"); + } + + vect->insert(vect->end(), vector, vector + dim); + env->ReleaseByteArrayElements(vectorArray, reinterpret_cast(vector), JNI_ABORT); + } + this->HasExceptionInStack(env); + env->DeleteLocalRef(array2dJ); +} + std::vector knn_jni::JNIUtil::ConvertJavaIntArrayToCppIntVector(JNIEnv *env, jintArray arrayJ) { if (arrayJ == nullptr) { diff --git a/jni/src/org_opensearch_knn_jni_FaissService.cpp b/jni/src/org_opensearch_knn_jni_FaissService.cpp index 663e18457..bcdc4f18b 100644 --- a/jni/src/org_opensearch_knn_jni_FaissService.cpp +++ b/jni/src/org_opensearch_knn_jni_FaissService.cpp @@ -67,6 +67,20 @@ JNIEXPORT jlong JNICALL Java_org_opensearch_knn_jni_FaissService_initBinaryIndex return (jlong)0; } +JNIEXPORT jlong JNICALL Java_org_opensearch_knn_jni_FaissService_initByteIndex(JNIEnv * env, jclass cls, + jlong numDocs, jint dimJ, + jobject parametersJ) +{ + try { + std::unique_ptr faissMethods(new knn_jni::faiss_wrapper::FaissMethods()); + knn_jni::faiss_wrapper::ByteIndexService byteIndexService(std::move(faissMethods)); + return knn_jni::faiss_wrapper::InitIndex(&jniUtil, env, numDocs, dimJ, parametersJ, &byteIndexService); + } catch (...) { + jniUtil.CatchCppExceptionAndThrowJava(env); + } + return (jlong)0; +} + JNIEXPORT void JNICALL Java_org_opensearch_knn_jni_FaissService_insertToIndex(JNIEnv * env, jclass cls, jintArray idsJ, jlong vectorsAddressJ, jint dimJ, jlong indexAddress, jint threadCount) @@ -95,6 +109,20 @@ JNIEXPORT void JNICALL Java_org_opensearch_knn_jni_FaissService_insertToBinaryIn } } +JNIEXPORT void JNICALL Java_org_opensearch_knn_jni_FaissService_insertToByteIndex(JNIEnv * env, jclass cls, jintArray idsJ, + jlong vectorsAddressJ, jint dimJ, + jlong indexAddress, jint threadCount) +{ + try { + std::unique_ptr faissMethods(new knn_jni::faiss_wrapper::FaissMethods()); + knn_jni::faiss_wrapper::ByteIndexService byteIndexService(std::move(faissMethods)); + knn_jni::faiss_wrapper::InsertToIndex(&jniUtil, env, idsJ, vectorsAddressJ, dimJ, indexAddress, threadCount, &byteIndexService); + } catch (...) { + // NOTE: ADDING DELETE STATEMENT HERE CAUSES A CRASH! + jniUtil.CatchCppExceptionAndThrowJava(env); + } +} + JNIEXPORT void JNICALL Java_org_opensearch_knn_jni_FaissService_writeIndex(JNIEnv * env, jclass cls, jlong indexAddress, jstring indexPathJ) @@ -121,6 +149,19 @@ JNIEXPORT void JNICALL Java_org_opensearch_knn_jni_FaissService_writeBinaryIndex } } +JNIEXPORT void JNICALL Java_org_opensearch_knn_jni_FaissService_writeByteIndex(JNIEnv * env, jclass cls, + jlong indexAddress, + jstring indexPathJ) +{ + try { + std::unique_ptr faissMethods(new knn_jni::faiss_wrapper::FaissMethods()); + knn_jni::faiss_wrapper::ByteIndexService byteIndexService(std::move(faissMethods)); + knn_jni::faiss_wrapper::WriteIndex(&jniUtil, env, indexPathJ, indexAddress, &byteIndexService); + } catch (...) { + jniUtil.CatchCppExceptionAndThrowJava(env); + } +} + JNIEXPORT void JNICALL Java_org_opensearch_knn_jni_FaissService_createIndexFromTemplate(JNIEnv * env, jclass cls, jintArray idsJ, jlong vectorsAddressJ, diff --git a/jni/src/org_opensearch_knn_jni_JNICommons.cpp b/jni/src/org_opensearch_knn_jni_JNICommons.cpp index 7432c44d3..906592b2d 100644 --- a/jni/src/org_opensearch_knn_jni_JNICommons.cpp +++ b/jni/src/org_opensearch_knn_jni_JNICommons.cpp @@ -49,6 +49,18 @@ jlong memoryAddressJ, jobjectArray dataJ, jlong initialCapacityJ, jboolean appen return (long)memoryAddressJ; } +JNIEXPORT jlong JNICALL Java_org_opensearch_knn_jni_JNICommons_storeBinaryVectorData(JNIEnv * env, jclass cls, +jlong memoryAddressJ, jobjectArray dataJ, jlong initialCapacityJ, jboolean appendJ) + +{ + try { + return knn_jni::commons::storeBinaryVectorData(&jniUtil, env, memoryAddressJ, dataJ, initialCapacityJ, appendJ); + } catch (...) { + jniUtil.CatchCppExceptionAndThrowJava(env); + } + return (long)memoryAddressJ; +} + JNIEXPORT jlong JNICALL Java_org_opensearch_knn_jni_JNICommons_storeByteVectorData(JNIEnv * env, jclass cls, jlong memoryAddressJ, jobjectArray dataJ, jlong initialCapacityJ, jboolean appendJ) @@ -72,6 +84,16 @@ JNIEXPORT void JNICALL Java_org_opensearch_knn_jni_JNICommons_freeVectorData(JNI } +JNIEXPORT void JNICALL Java_org_opensearch_knn_jni_JNICommons_freeBinaryVectorData(JNIEnv * env, jclass cls, + jlong memoryAddressJ) +{ + try { + return knn_jni::commons::freeBinaryVectorData(memoryAddressJ); + } catch (...) { + jniUtil.CatchCppExceptionAndThrowJava(env); + } +} + JNIEXPORT void JNICALL Java_org_opensearch_knn_jni_JNICommons_freeByteVectorData(JNIEnv * env, jclass cls, jlong memoryAddressJ) { diff --git a/jni/tests/faiss_index_service_test.cpp b/jni/tests/faiss_index_service_test.cpp index 1f00f6a1d..8d9e4bb43 100644 --- a/jni/tests/faiss_index_service_test.cpp +++ b/jni/tests/faiss_index_service_test.cpp @@ -113,4 +113,48 @@ TEST(CreateBinaryIndexTest, BasicAssertions) { long indexAddress = indexService.initIndex(&mockJNIUtil, jniEnv, metricType, indexDescription, dim, numIds, threadCount, parametersMap); indexService.insertToIndex(dim, numIds, threadCount, (int64_t) &vectors, ids, indexAddress); indexService.writeIndex(indexPath, indexAddress); +} + +TEST(CreateByteIndexTest, BasicAssertions) { + // Define the data + faiss::idx_t numIds = 200; + std::vector ids; + std::vector vectors; + int dim = 8; + vectors.reserve(numIds * dim); + for (int64_t i = 0; i < numIds; ++i) { + ids.push_back(i); + for (int j = 0; j < dim; ++j) { + vectors.push_back(test_util::RandomInt(-128, 127)); + } + } + + std::string indexPath = test_util::RandomString(10, "tmp/", ".faiss"); + faiss::MetricType metricType = faiss::METRIC_L2; + std::string indexDescription = "HNSW16,SQ8_direct_signed"; + int threadCount = 1; + std::unordered_map parametersMap; + + // Set up jni + JNIEnv *jniEnv = nullptr; + NiceMock mockJNIUtil; + + // Setup faiss method mock + // This object is handled by unique_ptr inside indexService.createIndex() + MockIndex* index = new MockIndex(); + // This object is handled by unique_ptr inside indexService.createIndex() + faiss::IndexIDMap* indexIdMap = new faiss::IndexIDMap(index); + std::unique_ptr mockFaissMethods(new MockFaissMethods()); + EXPECT_CALL(*mockFaissMethods, indexFactory(dim, ::testing::StrEq(indexDescription.c_str()), metricType)) + .WillOnce(Return(index)); + EXPECT_CALL(*mockFaissMethods, indexIdMap(index)) + .WillOnce(Return(indexIdMap)); + EXPECT_CALL(*mockFaissMethods, writeIndex(indexIdMap, ::testing::StrEq(indexPath.c_str()))) + .Times(1); + + // Create the index + knn_jni::faiss_wrapper::ByteIndexService indexService(std::move(mockFaissMethods)); + long indexAddress = indexService.initIndex(&mockJNIUtil, jniEnv, metricType, indexDescription, dim, numIds, threadCount, parametersMap); + indexService.insertToIndex(dim, numIds, threadCount, (int64_t) &vectors, ids, indexAddress); + indexService.writeIndex(indexPath, indexAddress); } \ No newline at end of file diff --git a/jni/tests/test_util.cpp b/jni/tests/test_util.cpp index 2149f8a1a..4f8bd2c34 100644 --- a/jni/tests/test_util.cpp +++ b/jni/tests/test_util.cpp @@ -51,12 +51,18 @@ test_util::MockJNIUtil::MockJNIUtil() { (*reinterpret_cast> *>(array2dJ))) for (auto item : v) data->push_back(item); }); - ON_CALL(*this, Convert2dJavaObjectArrayAndStoreToByteVector) + ON_CALL(*this, Convert2dJavaObjectArrayAndStoreToBinaryVector) .WillByDefault([this](JNIEnv *env, jobjectArray array2dJ, int dim, std::vector* data) { for (const auto &v : (*reinterpret_cast> *>(array2dJ))) for (auto item : v) data->push_back(item); }); + ON_CALL(*this, Convert2dJavaObjectArrayAndStoreToByteVector) + .WillByDefault([this](JNIEnv *env, jobjectArray array2dJ, int dim, std::vector* data) { + for (const auto &v : + (*reinterpret_cast> *>(array2dJ))) + for (auto item : v) data->push_back(item); + }); // arrayJ is re-interpreted as std::vector * diff --git a/jni/tests/test_util.h b/jni/tests/test_util.h index ba773fad3..a90d45dd9 100644 --- a/jni/tests/test_util.h +++ b/jni/tests/test_util.h @@ -46,8 +46,10 @@ namespace test_util { (JNIEnv * env, jobjectArray array2dJ, int dim)); MOCK_METHOD(void, Convert2dJavaObjectArrayAndStoreToFloatVector, (JNIEnv * env, jobjectArray array2dJ, int dim, std::vector*vect)); - MOCK_METHOD(void, Convert2dJavaObjectArrayAndStoreToByteVector, + MOCK_METHOD(void, Convert2dJavaObjectArrayAndStoreToBinaryVector, (JNIEnv * env, jobjectArray array2dJ, int dim, std::vector*vect)); + MOCK_METHOD(void, Convert2dJavaObjectArrayAndStoreToByteVector, + (JNIEnv * env, jobjectArray array2dJ, int dim, std::vector*vect)); MOCK_METHOD(std::vector, ConvertJavaIntArrayToCppIntVector, (JNIEnv * env, jintArray arrayJ)); MOCK_METHOD2(ConvertJavaMapToCppMap, diff --git a/src/main/java/org/opensearch/knn/common/KNNConstants.java b/src/main/java/org/opensearch/knn/common/KNNConstants.java index 4d935407a..4c1dc6e64 100644 --- a/src/main/java/org/opensearch/knn/common/KNNConstants.java +++ b/src/main/java/org/opensearch/knn/common/KNNConstants.java @@ -111,6 +111,7 @@ public class KNNConstants { public static final String FAISS_SQ_TYPE = "type"; public static final String FAISS_SQ_ENCODER_FP16 = "fp16"; public static final List FAISS_SQ_ENCODER_TYPES = List.of(FAISS_SQ_ENCODER_FP16); + public static final String FAISS_SIGNED_BYTE_SQ = "SQ8_direct_signed"; public static final String FAISS_SQ_CLIP = "clip"; // Parameter defaults/limits diff --git a/src/main/java/org/opensearch/knn/index/codec/transfer/OffHeapBinaryVectorTransfer.java b/src/main/java/org/opensearch/knn/index/codec/transfer/OffHeapBinaryVectorTransfer.java index c9d4802fe..ffa12a231 100644 --- a/src/main/java/org/opensearch/knn/index/codec/transfer/OffHeapBinaryVectorTransfer.java +++ b/src/main/java/org/opensearch/knn/index/codec/transfer/OffHeapBinaryVectorTransfer.java @@ -5,6 +5,8 @@ package org.opensearch.knn.index.codec.transfer; +import org.opensearch.knn.jni.JNICommons; + import java.io.IOException; import java.util.List; @@ -21,12 +23,16 @@ public OffHeapBinaryVectorTransfer(int transferLimit) { @Override public void deallocate() { - // TODO: deallocate the memory location + JNICommons.freeBinaryVectorData(getVectorAddress()); } @Override - protected long transfer(List vectorsToTransfer, boolean append) throws IOException { - // TODO: call to JNIService to transfer vector - return 0L; + protected long transfer(List batch, boolean append) throws IOException { + return JNICommons.storeBinaryVectorData( + getVectorAddress(), + batch.toArray(new byte[][] {}), + (long) batch.get(0).length * transferLimit, + append + ); } } diff --git a/src/main/java/org/opensearch/knn/index/codec/transfer/OffHeapVectorTransferFactory.java b/src/main/java/org/opensearch/knn/index/codec/transfer/OffHeapVectorTransferFactory.java index bfcc13491..446b6ae80 100644 --- a/src/main/java/org/opensearch/knn/index/codec/transfer/OffHeapVectorTransferFactory.java +++ b/src/main/java/org/opensearch/knn/index/codec/transfer/OffHeapVectorTransferFactory.java @@ -27,7 +27,7 @@ public static OffHeapVectorTransfer getVectorTransfer(final VectorDataTyp case FLOAT: return (OffHeapVectorTransfer) new OffHeapFloatVectorTransfer(transferLimit); case BINARY: - // TODO: Add binary here + return (OffHeapVectorTransfer) new OffHeapBinaryVectorTransfer(transferLimit); case BYTE: return (OffHeapVectorTransfer) new OffHeapByteVectorTransfer(transferLimit); default: diff --git a/src/main/java/org/opensearch/knn/index/engine/faiss/AbstractFaissMethod.java b/src/main/java/org/opensearch/knn/index/engine/faiss/AbstractFaissMethod.java index 908671a21..8c2dfc126 100644 --- a/src/main/java/org/opensearch/knn/index/engine/faiss/AbstractFaissMethod.java +++ b/src/main/java/org/opensearch/knn/index/engine/faiss/AbstractFaissMethod.java @@ -5,6 +5,7 @@ package org.opensearch.knn.index.engine.faiss; +import org.apache.commons.lang.StringUtils; import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.engine.AbstractKNNMethod; @@ -20,6 +21,7 @@ import java.util.Objects; import java.util.Set; +import static org.opensearch.knn.common.KNNConstants.FAISS_SIGNED_BYTE_SQ; import static org.opensearch.knn.common.KNNConstants.METHOD_ENCODER_PARAMETER; import static org.opensearch.knn.index.engine.faiss.Faiss.FAISS_BINARY_INDEX_DESCRIPTION_PREFIX; import static org.opensearch.knn.index.engine.faiss.FaissFP16Util.isFaissSQClipToFP16RangeEnabled; @@ -87,7 +89,7 @@ protected PerDimensionProcessor doGetPerDimensionProcessor( throw new IllegalStateException("Unsupported vector data type " + vectorDataType); } - static KNNLibraryIndexingContext adjustPrefix( + static KNNLibraryIndexingContext adjustIndexDescription( MethodAsMapBuilder methodAsMapBuilder, MethodComponentContext methodComponentContext, KNNMethodConfigContext knnMethodConfigContext @@ -105,6 +107,19 @@ static KNNLibraryIndexingContext adjustPrefix( if (knnMethodConfigContext.getVectorDataType() == VectorDataType.BINARY) { prefix = FAISS_BINARY_INDEX_DESCRIPTION_PREFIX; } + if (knnMethodConfigContext.getVectorDataType() == VectorDataType.BYTE) { + + // If VectorDataType is Byte using Faiss engine then manipulate Index Description to use "SQ8_direct_signed" scalar quantizer + // For example, Index Description "HNSW16,Flat" will be updated as "HNSW16,SQ8_direct_signed" + String indexDescription = methodAsMapBuilder.indexDescription; + if (StringUtils.isNotEmpty(indexDescription)) { + StringBuilder indexDescriptionBuilder = new StringBuilder(); + indexDescriptionBuilder.append(indexDescription.split(",")[0]); + indexDescriptionBuilder.append(","); + indexDescriptionBuilder.append(FAISS_SIGNED_BYTE_SQ); + methodAsMapBuilder.indexDescription = indexDescriptionBuilder.toString(); + } + } methodAsMapBuilder.indexDescription = prefix + methodAsMapBuilder.indexDescription; return methodAsMapBuilder.build(); } diff --git a/src/main/java/org/opensearch/knn/index/engine/faiss/FaissHNSWMethod.java b/src/main/java/org/opensearch/knn/index/engine/faiss/FaissHNSWMethod.java index 15e43cdd5..41db777e3 100644 --- a/src/main/java/org/opensearch/knn/index/engine/faiss/FaissHNSWMethod.java +++ b/src/main/java/org/opensearch/knn/index/engine/faiss/FaissHNSWMethod.java @@ -35,7 +35,11 @@ */ public class FaissHNSWMethod extends AbstractFaissMethod { - private static final Set SUPPORTED_DATA_TYPES = ImmutableSet.of(VectorDataType.FLOAT, VectorDataType.BINARY); + private static final Set SUPPORTED_DATA_TYPES = ImmutableSet.of( + VectorDataType.FLOAT, + VectorDataType.BINARY, + VectorDataType.BYTE + ); public final static List SUPPORTED_SPACES = Arrays.asList( SpaceType.UNDEFINED, @@ -95,7 +99,7 @@ private static MethodComponent initMethodComponent() { methodComponentContext, knnMethodConfigContext ).addParameter(METHOD_PARAMETER_M, "", "").addParameter(METHOD_ENCODER_PARAMETER, ",", ""); - return adjustPrefix(methodAsMapBuilder, methodComponentContext, knnMethodConfigContext); + return adjustIndexDescription(methodAsMapBuilder, methodComponentContext, knnMethodConfigContext); })) .build(); } diff --git a/src/main/java/org/opensearch/knn/index/engine/faiss/FaissIVFMethod.java b/src/main/java/org/opensearch/knn/index/engine/faiss/FaissIVFMethod.java index bc30e372c..b3dd12c92 100644 --- a/src/main/java/org/opensearch/knn/index/engine/faiss/FaissIVFMethod.java +++ b/src/main/java/org/opensearch/knn/index/engine/faiss/FaissIVFMethod.java @@ -95,7 +95,7 @@ private static MethodComponent initMethodComponent() { methodComponentContext, knnMethodConfigContext ).addParameter(METHOD_PARAMETER_NLIST, "", "").addParameter(METHOD_ENCODER_PARAMETER, ",", ""); - return adjustPrefix(methodAsMapBuilder, methodComponentContext, knnMethodConfigContext); + return adjustIndexDescription(methodAsMapBuilder, methodComponentContext, knnMethodConfigContext); })) .setOverheadInKBEstimator((methodComponent, methodComponentContext, dimension) -> { // Size estimate formula: (4 * nlists * d) / 1024 + 1 diff --git a/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperUtil.java b/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperUtil.java index 57a4dd062..5ab2dd888 100644 --- a/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperUtil.java +++ b/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperUtil.java @@ -23,10 +23,10 @@ import org.opensearch.knn.index.KNNSettings; import org.opensearch.knn.index.KnnCircuitBreakerException; import org.opensearch.knn.index.SpaceType; -import org.opensearch.knn.index.engine.KNNMethodContext; import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.codec.util.KNNVectorSerializerFactory; import org.opensearch.knn.index.engine.KNNEngine; +import org.opensearch.knn.index.engine.KNNMethodContext; import org.opensearch.knn.index.engine.MethodComponentContext; import org.opensearch.knn.index.util.IndexHyperParametersUtil; diff --git a/src/main/java/org/opensearch/knn/index/memory/NativeMemoryAllocation.java b/src/main/java/org/opensearch/knn/index/memory/NativeMemoryAllocation.java index 4f92a9c4b..755b6b925 100644 --- a/src/main/java/org/opensearch/knn/index/memory/NativeMemoryAllocation.java +++ b/src/main/java/org/opensearch/knn/index/memory/NativeMemoryAllocation.java @@ -300,7 +300,7 @@ private void cleanup() { if (this.memoryAddress != 0) { if (IndexUtil.isBinaryIndex(vectorDataType)) { - JNICommons.freeByteVectorData(this.memoryAddress); + JNICommons.freeBinaryVectorData(this.memoryAddress); } else { JNICommons.freeVectorData(this.memoryAddress); } diff --git a/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java b/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java index cba5692c9..208e075eb 100644 --- a/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java +++ b/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java @@ -481,22 +481,32 @@ protected Query doToQuery(QueryShardContext context) { } byte[] byteVector = new byte[0]; - if (VectorDataType.BINARY == vectorDataType) { - byteVector = new byte[vector.length]; - for (int i = 0; i < vector.length; i++) { - validateByteVectorValue(vector[i], knnVectorFieldType.getVectorDataType()); - byteVector[i] = (byte) vector[i]; - } - spaceType.validateVector(byteVector); - } else if (VectorDataType.BYTE == vectorDataType) { - byteVector = new byte[vector.length]; - for (int i = 0; i < vector.length; i++) { - validateByteVectorValue(vector[i], knnVectorFieldType.getVectorDataType()); - byteVector[i] = (byte) vector[i]; - } - spaceType.validateVector(byteVector); - } else { - spaceType.validateVector(vector); + switch (vectorDataType) { + case BINARY: + byteVector = new byte[vector.length]; + for (int i = 0; i < vector.length; i++) { + validateByteVectorValue(vector[i], knnVectorFieldType.getVectorDataType()); + byteVector[i] = (byte) vector[i]; + } + spaceType.validateVector(byteVector); + break; + case BYTE: + if (KNNEngine.LUCENE == knnEngine) { + byteVector = new byte[vector.length]; + for (int i = 0; i < vector.length; i++) { + validateByteVectorValue(vector[i], knnVectorFieldType.getVectorDataType()); + byteVector[i] = (byte) vector[i]; + } + spaceType.validateVector(byteVector); + } else { + for (float v : vector) { + validateByteVectorValue(v, knnVectorFieldType.getVectorDataType()); + } + spaceType.validateVector(vector); + } + break; + default: + spaceType.validateVector(vector); } if (KNNEngine.getEnginesThatCreateCustomSegmentFiles().contains(knnEngine) @@ -512,8 +522,8 @@ protected Query doToQuery(QueryShardContext context) { .knnEngine(knnEngine) .indexName(indexName) .fieldName(this.fieldName) - .vector(VectorDataType.FLOAT == vectorDataType ? this.vector : null) - .byteVector(VectorDataType.BYTE == vectorDataType || VectorDataType.BINARY == vectorDataType ? byteVector : null) + .vector(getVectorForCreatingQueryRequest(vectorDataType, knnEngine)) + .byteVector(getVectorForCreatingQueryRequest(vectorDataType, knnEngine, byteVector)) .vectorDataType(vectorDataType) .k(this.k) .methodParameters(this.methodParameters) @@ -581,6 +591,20 @@ private void updateQueryStats(VectorQueryType vectorQueryType) { } } + private float[] getVectorForCreatingQueryRequest(VectorDataType vectorDataType, KNNEngine knnEngine) { + if ((VectorDataType.FLOAT == vectorDataType) || (VectorDataType.BYTE == vectorDataType && KNNEngine.FAISS == knnEngine)) { + return this.vector; + } + return null; + } + + private byte[] getVectorForCreatingQueryRequest(VectorDataType vectorDataType, KNNEngine knnEngine, byte[] byteVector) { + if (VectorDataType.BINARY == vectorDataType || (VectorDataType.BYTE == vectorDataType && KNNEngine.LUCENE == knnEngine)) { + return byteVector; + } + return null; + } + @Override protected boolean doEquals(KNNQueryBuilder other) { return Objects.equals(fieldName, other.fieldName) diff --git a/src/main/java/org/opensearch/knn/index/util/IndexUtil.java b/src/main/java/org/opensearch/knn/index/util/IndexUtil.java index 853b5237a..8232f84a2 100644 --- a/src/main/java/org/opensearch/knn/index/util/IndexUtil.java +++ b/src/main/java/org/opensearch/knn/index/util/IndexUtil.java @@ -417,4 +417,16 @@ private static Map initializeMinimalRequiredVersionMap() { } return Collections.unmodifiableMap(versionMap); } + + /** + * Tell if it is byte index or not + * + * @param parameters parameters associated with an index + * @return true if it is binary index + */ + public static boolean isByteIndex(Map parameters) { + return parameters.getOrDefault(VECTOR_DATA_TYPE_FIELD, VectorDataType.DEFAULT.getValue()) + .toString() + .equals(VectorDataType.BYTE.getValue()); + } } diff --git a/src/main/java/org/opensearch/knn/jni/FaissService.java b/src/main/java/org/opensearch/knn/jni/FaissService.java index a402be1f3..26c703eeb 100644 --- a/src/main/java/org/opensearch/knn/jni/FaissService.java +++ b/src/main/java/org/opensearch/knn/jni/FaissService.java @@ -69,6 +69,16 @@ class FaissService { */ public static native long initBinaryIndex(long numDocs, int dim, Map parameters); + /** + * Initialize a byte index for the native library. Takes in numDocs to + * allocate the correct amount of memory. + * + * @param numDocs number of documents to be added + * @param dim dimension of the vector to be indexed + * @param parameters parameters to build index + */ + public static native long initByteIndex(long numDocs, int dim, Map parameters); + /** * Inserts to a faiss index. The memory occupied by the vectorsAddress will be freed up during the * function call. So Java layer doesn't need to free up the memory. This is not an ideal behavior because Java layer @@ -95,6 +105,19 @@ class FaissService { */ public static native void insertToBinaryIndex(int[] ids, long vectorsAddress, int dim, long indexAddress, int threadCount); + /** + * Inserts to a faiss index. The memory occupied by the vectorsAddress will be freed up during the + * function call. So Java layer doesn't need to free up the memory. This is not an ideal behavior because Java layer + * created the memory address and that should only free up the memory. + * + * @param ids ids of documents + * @param vectorsAddress address of native memory where vectors are stored + * @param dim dimension of the vector to be indexed + * @param indexAddress address of native memory where index is stored + * @param threadCount number of threads to use for insertion + */ + public static native void insertToByteIndex(int[] ids, long vectorsAddress, int dim, long indexAddress, int threadCount); + /** * Writes a faiss index. * @@ -115,6 +138,16 @@ class FaissService { */ public static native void writeBinaryIndex(long indexAddress, String indexPath); + /** + * Writes a faiss index. + * + * NOTE: This will always free the index. Do not call free after this. + * + * @param indexAddress address of native memory where index is stored + * @param indexPath path to save index file to + */ + public static native void writeByteIndex(long indexAddress, String indexPath); + /** * Create an index for the native library with a provided template index * diff --git a/src/main/java/org/opensearch/knn/jni/JNICommons.java b/src/main/java/org/opensearch/knn/jni/JNICommons.java index c7222738e..df1024db4 100644 --- a/src/main/java/org/opensearch/knn/jni/JNICommons.java +++ b/src/main/java/org/opensearch/knn/jni/JNICommons.java @@ -70,7 +70,28 @@ public static long storeVectorData(long memoryAddress, float[][] data, long init public static native long storeVectorData(long memoryAddress, float[][] data, long initialCapacity, boolean append); /** - * This is utility function that can be used to store data in native memory. This function will allocate memory for + * This is utility function that can be used to store binary data in native memory. This function will allocate memory for + * the data(rows*columns) with initialCapacity and return the memory address where the data is stored. + * If you are using this function for first time use memoryAddress = 0 to ensure that a new memory location is created. + * For subsequent calls you can pass the same memoryAddress. If the data cannot be stored in the memory location + * will throw Exception. + * + *

    + * The function is not threadsafe. If multiple threads are trying to insert on same memory location, then it can + * lead to data corruption. + *

    + * + * @param memoryAddress The address of the memory location where data will be stored. + * @param data 2D byte array containing binary data to be stored in native memory. + * @param initialCapacity The initial capacity of the memory location. + * @return memory address where the data is stored. + */ + public static long storeBinaryVectorData(long memoryAddress, byte[][] data, long initialCapacity) { + return storeBinaryVectorData(memoryAddress, data, initialCapacity, true); + } + + /** + * This is utility function that can be used to store binary data in native memory. This function will allocate memory for * the data(rows*columns) with initialCapacity and return the memory address where the data is stored. * If you are using this function for first time use memoryAddress = 0 to ensure that a new memory location is created. * For subsequent calls you can pass the same memoryAddress. If the data cannot be stored in the memory location @@ -82,7 +103,27 @@ public static long storeVectorData(long memoryAddress, float[][] data, long init *

    * * @param memoryAddress The address of the memory location where data will be stored. - * @param data 2D byte array containing data to be stored in native memory. + * @param data 2D byte array containing binary data to be stored in native memory. + * @param initialCapacity The initial capacity of the memory location. + * @param append append the data or rewrite the memory location + * @return memory address where the data is stored. + */ + public static native long storeBinaryVectorData(long memoryAddress, byte[][] data, long initialCapacity, boolean append); + + /** + * This is utility function that can be used to store byte data in native memory. This function will allocate memory for + * the data(rows*columns) with initialCapacity and return the memory address where the data is stored. + * If you are using this function for first time use memoryAddress = 0 to ensure that a new memory location is created. + * For subsequent calls you can pass the same memoryAddress. If the data cannot be stored in the memory location + * will throw Exception. + * + *

    + * The function is not threadsafe. If multiple threads are trying to insert on same memory location, then it can + * lead to data corruption. + *

    + * + * @param memoryAddress The address of the memory location where data will be stored. + * @param data 2D byte array containing byte data to be stored in native memory. * @param initialCapacity The initial capacity of the memory location. * @return memory address where the data is stored. */ @@ -91,7 +132,7 @@ public static long storeByteVectorData(long memoryAddress, byte[][] data, long i } /** - * This is utility function that can be used to store data in native memory. This function will allocate memory for + * This is utility function that can be used to store byte data in native memory. This function will allocate memory for * the data(rows*columns) with initialCapacity and return the memory address where the data is stored. * If you are using this function for first time use memoryAddress = 0 to ensure that a new memory location is created. * For subsequent calls you can pass the same memoryAddress. If the data cannot be stored in the memory location @@ -103,7 +144,7 @@ public static long storeByteVectorData(long memoryAddress, byte[][] data, long i *

    * * @param memoryAddress The address of the memory location where data will be stored. - * @param data 2D byte array containing data to be stored in native memory. + * @param data 2D byte array containing byte data to be stored in native memory. * @param initialCapacity The initial capacity of the memory location. * @param append append the data or rewrite the memory location * @return memory address where the data is stored. @@ -124,8 +165,21 @@ public static long storeByteVectorData(long memoryAddress, byte[][] data, long i public static native void freeVectorData(long memoryAddress); /** - * Free up the memory allocated for the byte data stored in memory address. This function should be used with the memory - * address returned by {@link JNICommons#storeVectorData(long, float[][], long, boolean)} + * Free up the memory allocated for the binary data stored in memory address. This function should be used with the memory + * address returned by {@link JNICommons#storeBinaryVectorData(long, byte[][], long)} + * + *

    + * The function is not threadsafe. If multiple threads are trying to free up same memory location, then it can + * lead to errors. + *

    + * + * @param memoryAddress address to be freed. + */ + public static native void freeBinaryVectorData(long memoryAddress); + + /** + * Free up the memory allocated for the binary data stored in memory address. This function should be used with the memory + * address returned by {@link JNICommons#storeBinaryVectorData(long, byte[][], long)} * *

    * The function is not threadsafe. If multiple threads are trying to free up same memory location, then it can diff --git a/src/main/java/org/opensearch/knn/jni/JNIService.java b/src/main/java/org/opensearch/knn/jni/JNIService.java index d1d5f6c11..1177d635e 100644 --- a/src/main/java/org/opensearch/knn/jni/JNIService.java +++ b/src/main/java/org/opensearch/knn/jni/JNIService.java @@ -39,9 +39,13 @@ public static long initIndex(long numDocs, int dim, Map paramete if (KNNEngine.FAISS == knnEngine) { if (IndexUtil.isBinaryIndex(knnEngine, parameters)) { return FaissService.initBinaryIndex(numDocs, dim, parameters); - } else { - return FaissService.initIndex(numDocs, dim, parameters); } + if (IndexUtil.isByteIndex(parameters)) { + return FaissService.initByteIndex(numDocs, dim, parameters); + } + + return FaissService.initIndex(numDocs, dim, parameters); + } throw new IllegalArgumentException( @@ -71,6 +75,8 @@ public static void insertToIndex( if (KNNEngine.FAISS == knnEngine) { if (IndexUtil.isBinaryIndex(knnEngine, parameters)) { FaissService.insertToBinaryIndex(docs, vectorsAddress, dimension, indexAddress, threadCount); + } else if (IndexUtil.isByteIndex(parameters)) { + FaissService.insertToByteIndex(docs, vectorsAddress, dimension, indexAddress, threadCount); } else { FaissService.insertToIndex(docs, vectorsAddress, dimension, indexAddress, threadCount); } @@ -94,6 +100,8 @@ public static void writeIndex(String indexPath, long indexAddress, KNNEngine knn if (KNNEngine.FAISS == knnEngine) { if (IndexUtil.isBinaryIndex(knnEngine, parameters)) { FaissService.writeBinaryIndex(indexAddress, indexPath); + } else if (IndexUtil.isByteIndex(parameters)) { + FaissService.writeByteIndex(indexAddress, indexPath); } else { FaissService.writeIndex(indexAddress, indexPath); } diff --git a/src/main/java/org/opensearch/knn/training/ByteTrainingDataConsumer.java b/src/main/java/org/opensearch/knn/training/ByteTrainingDataConsumer.java index 70cfb4f4c..e838b5214 100644 --- a/src/main/java/org/opensearch/knn/training/ByteTrainingDataConsumer.java +++ b/src/main/java/org/opensearch/knn/training/ByteTrainingDataConsumer.java @@ -39,7 +39,7 @@ public ByteTrainingDataConsumer(NativeMemoryAllocation.TrainingDataAllocation tr @Override public void accept(List byteVectors) { long memoryAddress = trainingDataAllocation.getMemoryAddress(); - memoryAddress = JNICommons.storeByteVectorData(memoryAddress, byteVectors.toArray(new byte[0][0]), byteVectors.size()); + memoryAddress = JNICommons.storeBinaryVectorData(memoryAddress, byteVectors.toArray(new byte[0][0]), byteVectors.size()); trainingDataAllocation.setMemoryAddress(memoryAddress); } diff --git a/src/test/java/org/opensearch/knn/index/VectorDataTypeIT.java b/src/test/java/org/opensearch/knn/index/VectorDataTypeIT.java index 0b045e848..4979514a8 100644 --- a/src/test/java/org/opensearch/knn/index/VectorDataTypeIT.java +++ b/src/test/java/org/opensearch/knn/index/VectorDataTypeIT.java @@ -13,15 +13,15 @@ import org.opensearch.client.ResponseException; import org.opensearch.common.settings.Settings; import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.core.rest.RestStatus; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.index.query.MatchAllQueryBuilder; import org.opensearch.index.query.QueryBuilder; import org.opensearch.knn.KNNRestTestCase; import org.opensearch.knn.KNNResult; import org.opensearch.knn.common.KNNConstants; -import org.opensearch.knn.index.query.KNNQueryBuilder; import org.opensearch.knn.index.engine.KNNEngine; -import org.opensearch.core.rest.RestStatus; +import org.opensearch.knn.index.query.KNNQueryBuilder; import org.opensearch.script.Script; import java.util.ArrayList; @@ -32,7 +32,13 @@ import java.util.Map; import static org.opensearch.knn.common.KNNConstants.DIMENSION; +import static org.opensearch.knn.common.KNNConstants.ENCODER_SQ; +import static org.opensearch.knn.common.KNNConstants.FAISS_SQ_ENCODER_FP16; +import static org.opensearch.knn.common.KNNConstants.FAISS_SQ_TYPE; +import static org.opensearch.knn.common.KNNConstants.METHOD_ENCODER_PARAMETER; import static org.opensearch.knn.common.KNNConstants.METHOD_HNSW; +import static org.opensearch.knn.common.KNNConstants.NAME; +import static org.opensearch.knn.common.KNNConstants.PARAMETERS; import static org.opensearch.knn.common.KNNConstants.VECTOR_DATA_TYPE_FIELD; import static org.opensearch.knn.index.VectorDataType.SUPPORTED_VECTOR_DATA_TYPES; @@ -453,6 +459,138 @@ public void testSearchWithMissingQueryVector() { assertTrue(ex.getMessage().contains("[knn] requires query vector")); } + @SneakyThrows + public void testAddDocWithByteVectorUsingFaissEngine() { + createKnnIndexMappingWithFaissEngine(2, SpaceType.L2, VectorDataType.BYTE.getValue()); + Byte[] vector = { 6, 6 }; + addKnnDoc(INDEX_NAME, DOC_ID, FIELD_NAME, vector); + + refreshAllIndices(); + assertEquals(1, getDocCount(INDEX_NAME)); + } + + @SneakyThrows + public void testUpdateDocWithByteVectorUsingFaissEngine() { + createKnnIndexMappingWithFaissEngine(2, SpaceType.L2, VectorDataType.BYTE.getValue()); + Byte[] vector = { -36, 78 }; + addKnnDoc(INDEX_NAME, DOC_ID, FIELD_NAME, vector); + + Byte[] updatedVector = { 89, -8 }; + updateKnnDoc(INDEX_NAME, DOC_ID, FIELD_NAME, updatedVector); + + refreshAllIndices(); + assertEquals(1, getDocCount(INDEX_NAME)); + } + + @SneakyThrows + public void testDeleteDocWithByteVectorUsingFaissEngine() { + createKnnIndexMappingWithFaissEngine(2, SpaceType.L2, VectorDataType.BYTE.getValue()); + Byte[] vector = { 35, -46 }; + addKnnDoc(INDEX_NAME, DOC_ID, FIELD_NAME, vector); + + deleteKnnDoc(INDEX_NAME, DOC_ID); + refreshAllIndices(); + + assertEquals(0, getDocCount(INDEX_NAME)); + } + + @SneakyThrows + public void testSearchWithByteVectorUsingFaissEngine() { + createKnnIndexMappingWithFaissEngine(2, SpaceType.L2, VectorDataType.BYTE.getValue()); + ingestL2ByteTestData(); + + Byte[] queryVector = { 1, 1 }; + Response response = searchKNNIndex(INDEX_NAME, new KNNQueryBuilder(FIELD_NAME, convertByteToFloatArray(queryVector), 4), 4); + + validateL2SearchResults(response); + } + + @SneakyThrows + public void testInvalidVectorDataUsingFaissEngine() { + createKnnIndexMappingWithFaissEngine(2, SpaceType.L2, VectorDataType.BYTE.getValue()); + Float[] vector = { -10.76f, 15.89f }; + + ResponseException ex = expectThrows(ResponseException.class, () -> addKnnDoc(INDEX_NAME, DOC_ID, FIELD_NAME, vector)); + assertTrue( + ex.getMessage() + .contains( + String.format( + Locale.ROOT, + "[%s] field was set as [%s] in index mapping. But, KNN vector values are floats instead of byte integers", + VECTOR_DATA_TYPE_FIELD, + VectorDataType.BYTE.getValue() + ) + ) + ); + } + + // Create an index with byte vector data_type and add a doc with values out of byte range which should throw exception + @SneakyThrows + public void testInvalidByteVectorRangeUsingFaissEngine() { + createKnnIndexMappingWithFaissEngine(2, SpaceType.L2, VectorDataType.BYTE.getValue()); + Float[] vector = { -1000f, 155f }; + + ResponseException ex = expectThrows(ResponseException.class, () -> addKnnDoc(INDEX_NAME, DOC_ID, FIELD_NAME, vector)); + assertTrue( + ex.getMessage() + .contains( + String.format( + Locale.ROOT, + "[%s] field was set as [%s] in index mapping. But, KNN vector values are not within in the byte range [%d, %d]", + VECTOR_DATA_TYPE_FIELD, + VectorDataType.BYTE.getValue(), + Byte.MIN_VALUE, + Byte.MAX_VALUE + ) + ) + ); + } + + // Create an index with byte vector data_type using faiss engine with an encoder which should throw an exception + @SneakyThrows + public void testByteVectorDataTypeWithFaissEngineUsingEncoderThrowsException() { + XContentBuilder builder = XContentFactory.jsonBuilder() + .startObject() + .startObject(PROPERTIES_FIELD) + .startObject(FIELD_NAME) + .field(TYPE_FIELD_NAME, KNN_VECTOR_TYPE) + .field(DIMENSION, 2) + .field(VECTOR_DATA_TYPE_FIELD, VectorDataType.BYTE.getValue()) + .startObject(KNNConstants.KNN_METHOD) + .field(KNNConstants.NAME, METHOD_HNSW) + .field(KNNConstants.METHOD_PARAMETER_SPACE_TYPE, SpaceType.L2) + .field(KNNConstants.KNN_ENGINE, KNNEngine.FAISS.getName()) + .startObject(PARAMETERS) + .field(KNNConstants.METHOD_PARAMETER_M, M) + .field(KNNConstants.METHOD_PARAMETER_EF_CONSTRUCTION, EF_CONSTRUCTION) + .startObject(METHOD_ENCODER_PARAMETER) + .field(NAME, ENCODER_SQ) + .startObject(PARAMETERS) + .field(FAISS_SQ_TYPE, FAISS_SQ_ENCODER_FP16) + .endObject() + .endObject() + .endObject() + .endObject() + .endObject() + .endObject() + .endObject(); + + String mapping = builder.toString(); + expectThrows(ResponseException.class, () -> createKnnIndex(INDEX_NAME, mapping)); + } + + public void testDocValuesWithByteVectorDataTypeFaissEngine() throws Exception { + createKnnIndexMappingWithFaissEngine(2, SpaceType.L2, VectorDataType.BYTE.getValue()); + ingestL2ByteTestData(); + + Byte[] queryVector = { 1, 1 }; + Request request = createScriptQueryRequest(queryVector, SpaceType.L2.getValue(), MATCH_ALL_QUERY_BUILDER); + Response response = client().performRequest(request); + assertEquals(request.getEndpoint() + ": failed", RestStatus.OK, RestStatus.fromCode(response.getStatusLine().getStatusCode())); + + validateL2SearchResults(response); + } + @SneakyThrows private void ingestL2ByteTestData() { Byte[] b1 = { 6, 6 }; @@ -491,6 +629,10 @@ private void createKnnIndexMappingWithLuceneEngine(int dimension, SpaceType spac createKnnIndexMappingWithCustomEngine(dimension, spaceType, vectorDataType, KNNEngine.LUCENE.getName()); } + private void createKnnIndexMappingWithFaissEngine(int dimension, SpaceType spaceType, String vectorDataType) throws Exception { + createKnnIndexMappingWithCustomEngine(dimension, spaceType, vectorDataType, KNNEngine.FAISS.getName()); + } + private void createKnnIndexMappingWithCustomEngine(int dimension, SpaceType spaceType, String vectorDataType, String engine) throws Exception { XContentBuilder builder = XContentFactory.jsonBuilder() @@ -504,7 +646,7 @@ private void createKnnIndexMappingWithCustomEngine(int dimension, SpaceType spac .field(KNNConstants.NAME, METHOD_HNSW) .field(KNNConstants.METHOD_PARAMETER_SPACE_TYPE, spaceType.getValue()) .field(KNNConstants.KNN_ENGINE, engine) - .startObject(KNNConstants.PARAMETERS) + .startObject(PARAMETERS) .field(KNNConstants.METHOD_PARAMETER_M, M) .field(KNNConstants.METHOD_PARAMETER_EF_CONSTRUCTION, EF_CONSTRUCTION) .endObject() diff --git a/src/test/java/org/opensearch/knn/index/codec/transfer/OffHeapVectorTransferFactoryTests.java b/src/test/java/org/opensearch/knn/index/codec/transfer/OffHeapVectorTransferFactoryTests.java index cef875cfc..39415d811 100644 --- a/src/test/java/org/opensearch/knn/index/codec/transfer/OffHeapVectorTransferFactoryTests.java +++ b/src/test/java/org/opensearch/knn/index/codec/transfer/OffHeapVectorTransferFactoryTests.java @@ -20,7 +20,7 @@ public void testOffHeapVectorTransferFactory() { assertNotSame(byteVectorTransfer, OffHeapVectorTransferFactory.getVectorTransfer(VectorDataType.BYTE, 10)); var binaryVectorTransfer = OffHeapVectorTransferFactory.getVectorTransfer(VectorDataType.BINARY, 10); - assertEquals(OffHeapByteVectorTransfer.class, binaryVectorTransfer.getClass()); + assertEquals(OffHeapBinaryVectorTransfer.class, binaryVectorTransfer.getClass()); assertNotSame(binaryVectorTransfer, OffHeapVectorTransferFactory.getVectorTransfer(VectorDataType.BINARY, 10)); } } diff --git a/src/test/java/org/opensearch/knn/index/engine/KNNMethodContextTests.java b/src/test/java/org/opensearch/knn/index/engine/KNNMethodContextTests.java index 6defa4c50..f142a9770 100644 --- a/src/test/java/org/opensearch/knn/index/engine/KNNMethodContextTests.java +++ b/src/test/java/org/opensearch/knn/index/engine/KNNMethodContextTests.java @@ -455,12 +455,12 @@ public void testValidateVectorDataType_whenBinaryNonFaiss_thenException() { ); } - public void testValidateVectorDataType_whenByteLucene_thenValid() { + public void testValidateVectorDataType_whenByte_thenValid() { validateValidateVectorDataType(KNNEngine.LUCENE, KNNConstants.METHOD_HNSW, VectorDataType.BYTE, SpaceType.L2, null); + validateValidateVectorDataType(KNNEngine.FAISS, KNNConstants.METHOD_HNSW, VectorDataType.BYTE, SpaceType.L2, null); } - public void testValidateVectorDataType_whenByteNonLucene_thenException() { - validateValidateVectorDataType(KNNEngine.FAISS, KNNConstants.METHOD_HNSW, VectorDataType.BYTE, SpaceType.L2, "UnsupportedMethod"); + public void testValidateVectorDataType_whenByte_thenException() { validateValidateVectorDataType(KNNEngine.NMSLIB, KNNConstants.METHOD_IVF, VectorDataType.BYTE, SpaceType.L2, "UnsupportedMethod"); } diff --git a/src/test/java/org/opensearch/knn/index/memory/NativeMemoryAllocationTests.java b/src/test/java/org/opensearch/knn/index/memory/NativeMemoryAllocationTests.java index 316582f6c..1e2134581 100644 --- a/src/test/java/org/opensearch/knn/index/memory/NativeMemoryAllocationTests.java +++ b/src/test/java/org/opensearch/knn/index/memory/NativeMemoryAllocationTests.java @@ -117,7 +117,7 @@ public void testClose_whenBinaryFiass_thenSuccess() { KNNConstants.VECTOR_DATA_TYPE_FIELD, VectorDataType.BINARY.getValue() ); - long vectorMemoryAddress = JNICommons.storeByteVectorData(0, vectors, numVectors * dataLength); + long vectorMemoryAddress = JNICommons.storeBinaryVectorData(0, vectors, numVectors * dataLength); TestUtils.createIndex(ids, vectorMemoryAddress, dimension, path, parameters, knnEngine); // Load index into memory diff --git a/src/test/java/org/opensearch/knn/index/memory/NativeMemoryLoadStrategyTests.java b/src/test/java/org/opensearch/knn/index/memory/NativeMemoryLoadStrategyTests.java index 8a38cadb5..29fbdb978 100644 --- a/src/test/java/org/opensearch/knn/index/memory/NativeMemoryLoadStrategyTests.java +++ b/src/test/java/org/opensearch/knn/index/memory/NativeMemoryLoadStrategyTests.java @@ -106,7 +106,7 @@ public void testLoad_whenFaissBinary_thenSuccess() throws IOException { KNNConstants.VECTOR_DATA_TYPE_FIELD, VectorDataType.BINARY.getValue() ); - long memoryAddress = JNICommons.storeByteVectorData(0, vectors, numVectors); + long memoryAddress = JNICommons.storeBinaryVectorData(0, vectors, numVectors); TestUtils.createIndex(ids, memoryAddress, dimension, path, parameters, knnEngine); // Setup mock resource manager diff --git a/src/test/java/org/opensearch/knn/index/query/KNNQueryBuilderTests.java b/src/test/java/org/opensearch/knn/index/query/KNNQueryBuilderTests.java index 762a36227..b7de89564 100644 --- a/src/test/java/org/opensearch/knn/index/query/KNNQueryBuilderTests.java +++ b/src/test/java/org/opensearch/knn/index/query/KNNQueryBuilderTests.java @@ -961,6 +961,7 @@ public void testRadialSearch_whenEfSearchIsSet_whenFaissEngine_thenSuccess() { Index dummyIndex = new Index("dummy", "dummy"); when(mockKNNVectorField.getKnnMappingConfig()).thenReturn(getMappingConfigForMethodMapping(knnMethodContext, 4)); when(mockQueryShardContext.index()).thenReturn(dummyIndex); + when(mockKNNVectorField.getVectorDataType()).thenReturn(VectorDataType.FLOAT); when(mockQueryShardContext.fieldMapper(anyString())).thenReturn(mockKNNVectorField); IndexSettings indexSettings = mock(IndexSettings.class); when(mockQueryShardContext.getIndexSettings()).thenReturn(indexSettings); diff --git a/src/testFixtures/java/org/opensearch/knn/TestUtils.java b/src/testFixtures/java/org/opensearch/knn/TestUtils.java index afc8f0b92..621688b62 100644 --- a/src/testFixtures/java/org/opensearch/knn/TestUtils.java +++ b/src/testFixtures/java/org/opensearch/knn/TestUtils.java @@ -398,7 +398,7 @@ public long loadDataToMemoryAddress() { } public long loadBinaryDataToMemoryAddress() { - return JNICommons.storeByteVectorData(0, indexBinaryData, (long) indexBinaryData.length * indexBinaryData[0].length, true); + return JNICommons.storeBinaryVectorData(0, indexBinaryData, (long) indexBinaryData.length * indexBinaryData[0].length, true); } @AllArgsConstructor From 9da1a77b746863d57f3fdd24d8796ae41f4ba04c Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Mon, 26 Aug 2024 14:19:59 -0500 Subject: [PATCH 339/416] Integration of Quantization Framework for Binary Quantization with Indexing Flow (#1996) (#2004) * Integration of Quantization Framework for Binary Quantization with Indexing Flow * Integration With Qunatization Config --------- (cherry picked from commit bbaaaf900e79e311dea3d5ff5fca2bf3976e7655) Signed-off-by: VIKASH TIWARI Signed-off-by: Ryan Bogan Co-authored-by: Vikasht34 --- .../NativeEngines990KnnVectorsWriter.java | 132 ++++++++++++--- .../DefaultIndexBuildStrategy.java | 38 +++-- .../codec/nativeindex/IndexBuildSetup.java | 40 +++++ .../MemOptimizedNativeIndexBuildStrategy.java | 39 +++-- .../codec/nativeindex/NativeIndexWriter.java | 65 ++++++- .../nativeindex/QuantizationIndexUtils.java | 68 ++++++++ .../nativeindex/model/BuildIndexParams.java | 8 +- .../engine/faiss/AbstractFaissMethod.java | 4 +- .../KNNVectorQuantizationTrainingRequest.java | 55 ++++++ .../QuantizationService.java | 128 ++++++++++++++ .../DefaultQuantizationState.java | 15 ++ .../MultiBitScalarQuantizationState.java | 46 +++++ .../OneBitScalarQuantizationState.java | 33 ++++ .../quantizationState/QuantizationState.java | 24 +++ .../models/requests/TrainingRequest.java | 6 +- .../quantizer/MultiBitScalarQuantizer.java | 16 +- .../quantizer/OneBitScalarQuantizer.java | 4 +- .../knn/quantization/quantizer/Quantizer.java | 4 +- .../quantizer/QuantizerHelper.java | 60 +++++-- ...NativeEngines990KnnVectorsFormatTests.java | 72 +++++++- .../DefaultIndexBuildStrategyTests.java | 116 +++++++++++++ ...ptimizedNativeIndexBuildStrategyTests.java | 118 +++++++++++++ .../QuantizationIndexUtilsTests.java | 109 ++++++++++++ .../knn/index/engine/faiss/FaissTests.java | 4 +- .../QuantizationServiceTests.java | 159 ++++++++++++++++++ .../opensearch/knn/integ/QFrameworkIT.java | 36 ++-- .../factory/QuantizerFactoryTests.java | 48 ++---- .../factory/QuantizerRegistryTests.java | 28 +-- .../QuantizationStateTests.java | 73 ++++++++ .../MultiBitScalarQuantizerTests.java | 8 +- .../quantizer/OneBitScalarQuantizerTests.java | 31 +++- 31 files changed, 1414 insertions(+), 173 deletions(-) create mode 100644 src/main/java/org/opensearch/knn/index/codec/nativeindex/IndexBuildSetup.java create mode 100644 src/main/java/org/opensearch/knn/index/codec/nativeindex/QuantizationIndexUtils.java create mode 100644 src/main/java/org/opensearch/knn/index/quantizationService/KNNVectorQuantizationTrainingRequest.java create mode 100644 src/main/java/org/opensearch/knn/index/quantizationService/QuantizationService.java create mode 100644 src/test/java/org/opensearch/knn/index/codec/nativeindex/QuantizationIndexUtilsTests.java create mode 100644 src/test/java/org/opensearch/knn/index/quantizationService/QuantizationServiceTests.java diff --git a/src/main/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsWriter.java b/src/main/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsWriter.java index 65736a63e..43f4d7ad6 100644 --- a/src/main/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsWriter.java +++ b/src/main/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsWriter.java @@ -24,10 +24,13 @@ import org.apache.lucene.index.Sorter; import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.RamUsageEstimator; +import org.opensearch.knn.index.quantizationService.QuantizationService; import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.codec.nativeindex.NativeIndexWriter; import org.opensearch.knn.index.vectorvalues.KNNVectorValues; import org.opensearch.knn.index.vectorvalues.KNNVectorValuesFactory; +import org.opensearch.knn.quantization.models.quantizationParams.QuantizationParams; +import org.opensearch.knn.quantization.models.quantizationState.QuantizationState; import java.io.IOException; import java.util.ArrayList; @@ -46,6 +49,7 @@ public class NativeEngines990KnnVectorsWriter extends KnnVectorsWriter { private final FlatVectorsWriter flatVectorsWriter; private final List> fields = new ArrayList<>(); private boolean finished; + private final QuantizationService quantizationService = QuantizationService.getInstance(); /** * Add new field for indexing. @@ -68,17 +72,14 @@ public KnnFieldVectorsWriter addField(final FieldInfo fieldInfo) throws IOExc */ @Override public void flush(int maxDoc, final Sorter.DocMap sortMap) throws IOException { - // simply write data in the flat file flatVectorsWriter.flush(maxDoc, sortMap); for (final NativeEngineFieldVectorsWriter field : fields) { - final VectorDataType vectorDataType = extractVectorDataType(field.getFieldInfo()); - final KNNVectorValues knnVectorValues = KNNVectorValuesFactory.getVectorValues( - vectorDataType, - field.getDocsWithField(), - field.getVectors() + trainAndIndex( + field.getFieldInfo(), + (vectorDataType, fieldInfo, fieldVectorsWriter) -> getKNNVectorValues(vectorDataType, fieldVectorsWriter), + NativeIndexWriter::flushIndex, + field ); - - NativeIndexWriter.getWriter(field.getFieldInfo(), segmentWriteState).flushIndex(knnVectorValues); } } @@ -86,24 +87,9 @@ public void flush(int maxDoc, final Sorter.DocMap sortMap) throws IOException { public void mergeOneField(final FieldInfo fieldInfo, final MergeState mergeState) throws IOException { // This will ensure that we are merging the FlatIndex during force merge. flatVectorsWriter.mergeOneField(fieldInfo, mergeState); - // For merge, pick values from flat vector and reindex again. This will use the flush operation to create graphs - final VectorDataType vectorDataType = extractVectorDataType(fieldInfo); - final KNNVectorValues knnVectorValues; - switch (fieldInfo.getVectorEncoding()) { - case FLOAT32: - final FloatVectorValues mergedFloats = MergedVectorValues.mergeFloatVectorValues(fieldInfo, mergeState); - knnVectorValues = KNNVectorValuesFactory.getVectorValues(vectorDataType, mergedFloats); - break; - case BYTE: - final ByteVectorValues mergedBytes = MergedVectorValues.mergeByteVectorValues(fieldInfo, mergeState); - knnVectorValues = KNNVectorValuesFactory.getVectorValues(vectorDataType, mergedBytes); - break; - default: - throw new IllegalStateException("Unsupported vector encoding [" + fieldInfo.getVectorEncoding() + "]"); - } + trainAndIndex(fieldInfo, this::getKNNVectorValuesForMerge, NativeIndexWriter::mergeIndex, mergeState); - NativeIndexWriter.getWriter(fieldInfo, segmentWriteState).mergeIndex(knnVectorValues); } /** @@ -146,4 +132,102 @@ public long ramBytesUsed() { .sum(); } + /** + * Retrieves the {@link KNNVectorValues} for a specific field based on the vector data type and field writer. + * + * @param vectorDataType The {@link VectorDataType} representing the type of vectors stored. + * @param field The {@link NativeEngineFieldVectorsWriter} representing the field from which to retrieve vectors. + * @param The type of vectors being processed. + * @return The {@link KNNVectorValues} associated with the field. + */ + private KNNVectorValues getKNNVectorValues(final VectorDataType vectorDataType, final NativeEngineFieldVectorsWriter field) { + return (KNNVectorValues) KNNVectorValuesFactory.getVectorValues(vectorDataType, field.getDocsWithField(), field.getVectors()); + } + + /** + * Retrieves the {@link KNNVectorValues} for a specific field during a merge operation, based on the vector data type. + * + * @param vectorDataType The {@link VectorDataType} representing the type of vectors stored. + * @param fieldInfo The {@link FieldInfo} object containing metadata about the field. + * @param mergeState The {@link MergeState} representing the state of the merge operation. + * @param The type of vectors being processed. + * @return The {@link KNNVectorValues} associated with the field during the merge. + * @throws IOException If an I/O error occurs during the retrieval. + */ + private KNNVectorValues getKNNVectorValuesForMerge( + final VectorDataType vectorDataType, + final FieldInfo fieldInfo, + final MergeState mergeState + ) throws IOException { + switch (fieldInfo.getVectorEncoding()) { + case FLOAT32: + FloatVectorValues mergedFloats = MergedVectorValues.mergeFloatVectorValues(fieldInfo, mergeState); + return (KNNVectorValues) KNNVectorValuesFactory.getVectorValues(vectorDataType, mergedFloats); + case BYTE: + ByteVectorValues mergedBytes = MergedVectorValues.mergeByteVectorValues(fieldInfo, mergeState); + return (KNNVectorValues) KNNVectorValuesFactory.getVectorValues(vectorDataType, mergedBytes); + default: + throw new IllegalStateException("Unsupported vector encoding [" + fieldInfo.getVectorEncoding() + "]"); + } + } + + /** + * Functional interface representing an operation that indexes the provided {@link KNNVectorValues}. + * + * @param The type of vectors being processed. + */ + @FunctionalInterface + private interface IndexOperation { + void buildAndWrite(NativeIndexWriter writer, KNNVectorValues knnVectorValues) throws IOException; + } + + /** + * Functional interface representing a method that retrieves {@link KNNVectorValues} based on + * the vector data type, field information, and the merge state. + * + * @param The type of the data representing the vector (e.g., {@link VectorDataType}). + * @param The metadata about the field. + * @param The state of the merge operation. + * @param The result of the retrieval, typically {@link KNNVectorValues}. + */ + @FunctionalInterface + private interface VectorValuesRetriever { + Result apply(DataType vectorDataType, FieldInfo fieldInfo, MergeState mergeState) throws IOException; + } + + /** + * Unified method for processing a field during either the indexing or merge operation. This method retrieves vector values + * based on the provided vector data type and applies the specified index operation, potentially including quantization if needed. + * + * @param fieldInfo The {@link FieldInfo} object containing metadata about the field. + * @param vectorValuesRetriever A functional interface that retrieves {@link KNNVectorValues} based on the vector data type, + * field information, and additional context (e.g., merge state or field writer). + * @param indexOperation A functional interface that performs the indexing operation using the retrieved + * {@link KNNVectorValues}. + * @param VectorProcessingContext The additional context required for retrieving the vector values (e.g., {@link MergeState} or {@link NativeEngineFieldVectorsWriter}). + * From Flush we need NativeFieldWriter which contains total number of vectors while from Merge we need merge state which contains vector information + * @param The type of vectors being processed. + * @param The type of the context needed for retrieving the vector values. + * @throws IOException If an I/O error occurs during the processing. + */ + private void trainAndIndex( + final FieldInfo fieldInfo, + final VectorValuesRetriever> vectorValuesRetriever, + final IndexOperation indexOperation, + final C VectorProcessingContext + ) throws IOException { + final VectorDataType vectorDataType = extractVectorDataType(fieldInfo); + KNNVectorValues knnVectorValues = vectorValuesRetriever.apply(vectorDataType, fieldInfo, VectorProcessingContext); + QuantizationParams quantizationParams = quantizationService.getQuantizationParams(fieldInfo); + QuantizationState quantizationState = null; + if (quantizationParams != null) { + quantizationState = quantizationService.train(quantizationParams, knnVectorValues); + } + NativeIndexWriter writer = (quantizationParams != null) + ? NativeIndexWriter.getWriter(fieldInfo, segmentWriteState, quantizationState) + : NativeIndexWriter.getWriter(fieldInfo, segmentWriteState); + + knnVectorValues = vectorValuesRetriever.apply(vectorDataType, fieldInfo, VectorProcessingContext); + indexOperation.buildAndWrite(writer, knnVectorValues); + } } diff --git a/src/main/java/org/opensearch/knn/index/codec/nativeindex/DefaultIndexBuildStrategy.java b/src/main/java/org/opensearch/knn/index/codec/nativeindex/DefaultIndexBuildStrategy.java index 5787ea76b..d2a6027db 100644 --- a/src/main/java/org/opensearch/knn/index/codec/nativeindex/DefaultIndexBuildStrategy.java +++ b/src/main/java/org/opensearch/knn/index/codec/nativeindex/DefaultIndexBuildStrategy.java @@ -39,16 +39,32 @@ public static DefaultIndexBuildStrategy getInstance() { return INSTANCE; } + /** + * Builds and writes a k-NN index using the provided vector values and index parameters. This method handles both + * quantized and non-quantized vectors, transferring them off-heap before building the index using native JNI services. + * + *

    The method first iterates over the vector values to calculate the necessary bytes per vector. If quantization is + * enabled, the vectors are quantized before being transferred off-heap. Once all vectors are transferred, they are + * flushed and used to build the index. The index is then written to the specified path using JNI calls.

    + * + * @param indexInfo The {@link BuildIndexParams} containing the parameters and configuration for building the index. + * @param knnVectorValues The {@link KNNVectorValues} representing the vectors to be indexed. + * @throws IOException If an I/O error occurs during the process of building and writing the index. + */ public void buildAndWriteIndex(final BuildIndexParams indexInfo, final KNNVectorValues knnVectorValues) throws IOException { - iterateVectorValuesOnce(knnVectorValues); // to get bytesPerVector - int transferLimit = (int) Math.max(1, KNNSettings.getVectorStreamingMemoryLimit().getBytes() / knnVectorValues.bytesPerVector()); + // Needed to make sure we don't get 0 dimensions while initializing index + iterateVectorValuesOnce(knnVectorValues); + IndexBuildSetup indexBuildSetup = QuantizationIndexUtils.prepareIndexBuild(knnVectorValues, indexInfo); + + int transferLimit = (int) Math.max(1, KNNSettings.getVectorStreamingMemoryLimit().getBytes() / indexBuildSetup.getBytesPerVector()); try (final OffHeapVectorTransfer vectorTransfer = getVectorTransfer(indexInfo.getVectorDataType(), transferLimit)) { + final List transferredDocIds = new ArrayList<>((int) knnVectorValues.totalLiveDocs()); - final List tranferredDocIds = new ArrayList<>(); while (knnVectorValues.docId() != NO_MORE_DOCS) { + Object vector = QuantizationIndexUtils.processAndReturnVector(knnVectorValues, indexBuildSetup); // append is true here so off heap memory buffer isn't overwritten - vectorTransfer.transfer(knnVectorValues.conditionalCloneVector(), true); - tranferredDocIds.add(knnVectorValues.docId()); + vectorTransfer.transfer(vector, true); + transferredDocIds.add(knnVectorValues.docId()); knnVectorValues.nextDoc(); } vectorTransfer.flush(true); @@ -60,12 +76,12 @@ public void buildAndWriteIndex(final BuildIndexParams indexInfo, final KNNVector if (params.containsKey(MODEL_ID)) { AccessController.doPrivileged((PrivilegedAction) () -> { JNIService.createIndexFromTemplate( - intListToArray(tranferredDocIds), + intListToArray(transferredDocIds), vectorAddress, - knnVectorValues.dimension(), + indexBuildSetup.getDimensions(), indexInfo.getIndexPath(), (byte[]) params.get(KNNConstants.MODEL_BLOB_PARAMETER), - indexInfo.getParameters(), + params, indexInfo.getKnnEngine() ); return null; @@ -73,11 +89,11 @@ public void buildAndWriteIndex(final BuildIndexParams indexInfo, final KNNVector } else { AccessController.doPrivileged((PrivilegedAction) () -> { JNIService.createIndex( - intListToArray(tranferredDocIds), + intListToArray(transferredDocIds), vectorAddress, - knnVectorValues.dimension(), + indexBuildSetup.getDimensions(), indexInfo.getIndexPath(), - indexInfo.getParameters(), + params, indexInfo.getKnnEngine() ); return null; diff --git a/src/main/java/org/opensearch/knn/index/codec/nativeindex/IndexBuildSetup.java b/src/main/java/org/opensearch/knn/index/codec/nativeindex/IndexBuildSetup.java new file mode 100644 index 000000000..c6c999c07 --- /dev/null +++ b/src/main/java/org/opensearch/knn/index/codec/nativeindex/IndexBuildSetup.java @@ -0,0 +1,40 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.codec.nativeindex; + +import lombok.AllArgsConstructor; +import lombok.Getter; +import org.opensearch.knn.quantization.models.quantizationOutput.QuantizationOutput; +import org.opensearch.knn.quantization.models.quantizationState.QuantizationState; + +/** + * IndexBuildSetup encapsulates the configuration and parameters required for building an index. + * This includes the size of each vector, the dimensions of the vectors, and any quantization-related + * settings such as the output and state of quantization. + */ +@Getter +@AllArgsConstructor +final class IndexBuildSetup { + /** + * The number of bytes per vector. + */ + private final int bytesPerVector; + + /** + * Dimension of Vector for Indexing + */ + private final int dimensions; + + /** + * The quantization output that will hold the quantized vector. + */ + private final QuantizationOutput quantizationOutput; + + /** + * The state of quantization, which may include parameters and trained models. + */ + private final QuantizationState quantizationState; +} diff --git a/src/main/java/org/opensearch/knn/index/codec/nativeindex/MemOptimizedNativeIndexBuildStrategy.java b/src/main/java/org/opensearch/knn/index/codec/nativeindex/MemOptimizedNativeIndexBuildStrategy.java index af80215b6..1115bfe05 100644 --- a/src/main/java/org/opensearch/knn/index/codec/nativeindex/MemOptimizedNativeIndexBuildStrategy.java +++ b/src/main/java/org/opensearch/knn/index/codec/nativeindex/MemOptimizedNativeIndexBuildStrategy.java @@ -40,45 +40,60 @@ public static MemOptimizedNativeIndexBuildStrategy getInstance() { return INSTANCE; } + /** + * Builds and writes a k-NN index using the provided vector values and index parameters. This method handles both + * quantized and non-quantized vectors, transferring them off-heap before building the index using native JNI services. + * + *

    The method first iterates over the vector values to calculate the necessary bytes per vector. If quantization is + * enabled, the vectors are quantized before being transferred off-heap. Once all vectors are transferred, they are + * flushed and used to build the index. The index is then written to the specified path using JNI calls.

    + * + * @param indexInfo The {@link BuildIndexParams} containing the parameters and configuration for building the index. + * @param knnVectorValues The {@link KNNVectorValues} representing the vectors to be indexed. + * @throws IOException If an I/O error occurs during the process of building and writing the index. + */ public void buildAndWriteIndex(final BuildIndexParams indexInfo, final KNNVectorValues knnVectorValues) throws IOException { - // Needed to make sure we dont get 0 dimensions while initializing index + // Needed to make sure we don't get 0 dimensions while initializing index iterateVectorValuesOnce(knnVectorValues); KNNEngine engine = indexInfo.getKnnEngine(); Map indexParameters = indexInfo.getParameters(); + IndexBuildSetup indexBuildSetup = QuantizationIndexUtils.prepareIndexBuild(knnVectorValues, indexInfo); // Initialize the index long indexMemoryAddress = AccessController.doPrivileged( (PrivilegedAction) () -> JNIService.initIndex( knnVectorValues.totalLiveDocs(), - knnVectorValues.dimension(), + indexBuildSetup.getDimensions(), indexParameters, engine ) ); - int transferLimit = (int) Math.max(1, KNNSettings.getVectorStreamingMemoryLimit().getBytes() / knnVectorValues.bytesPerVector()); + int transferLimit = (int) Math.max(1, KNNSettings.getVectorStreamingMemoryLimit().getBytes() / indexBuildSetup.getBytesPerVector()); try (final OffHeapVectorTransfer vectorTransfer = getVectorTransfer(indexInfo.getVectorDataType(), transferLimit)) { - final List tranferredDocIds = new ArrayList<>(transferLimit); + final List transferredDocIds = new ArrayList<>(transferLimit); + while (knnVectorValues.docId() != NO_MORE_DOCS) { + Object vector = QuantizationIndexUtils.processAndReturnVector(knnVectorValues, indexBuildSetup); // append is false to be able to reuse the memory location - boolean transferred = vectorTransfer.transfer(knnVectorValues.conditionalCloneVector(), false); - tranferredDocIds.add(knnVectorValues.docId()); + boolean transferred = vectorTransfer.transfer(vector, false); + transferredDocIds.add(knnVectorValues.docId()); if (transferred) { // Insert vectors long vectorAddress = vectorTransfer.getVectorAddress(); AccessController.doPrivileged((PrivilegedAction) () -> { JNIService.insertToIndex( - intListToArray(tranferredDocIds), + intListToArray(transferredDocIds), vectorAddress, - knnVectorValues.dimension(), + indexBuildSetup.getDimensions(), indexParameters, indexMemoryAddress, engine ); return null; }); - tranferredDocIds.clear(); + transferredDocIds.clear(); } knnVectorValues.nextDoc(); } @@ -89,16 +104,16 @@ public void buildAndWriteIndex(final BuildIndexParams indexInfo, final KNNVector long vectorAddress = vectorTransfer.getVectorAddress(); AccessController.doPrivileged((PrivilegedAction) () -> { JNIService.insertToIndex( - intListToArray(tranferredDocIds), + intListToArray(transferredDocIds), vectorAddress, - knnVectorValues.dimension(), + indexBuildSetup.getDimensions(), indexParameters, indexMemoryAddress, engine ); return null; }); - tranferredDocIds.clear(); + transferredDocIds.clear(); } // Write vector diff --git a/src/main/java/org/opensearch/knn/index/codec/nativeindex/NativeIndexWriter.java b/src/main/java/org/opensearch/knn/index/codec/nativeindex/NativeIndexWriter.java index 61500371b..ed0e8149a 100644 --- a/src/main/java/org/opensearch/knn/index/codec/nativeindex/NativeIndexWriter.java +++ b/src/main/java/org/opensearch/knn/index/codec/nativeindex/NativeIndexWriter.java @@ -12,6 +12,7 @@ import org.apache.lucene.store.ChecksumIndexInput; import org.apache.lucene.store.FSDirectory; import org.apache.lucene.store.FilterDirectory; +import org.opensearch.common.Nullable; import org.opensearch.common.xcontent.XContentHelper; import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.core.xcontent.DeprecationHandler; @@ -23,11 +24,13 @@ import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.codec.nativeindex.model.BuildIndexParams; import org.opensearch.knn.index.engine.KNNEngine; +import org.opensearch.knn.index.quantizationService.QuantizationService; import org.opensearch.knn.index.util.IndexUtil; import org.opensearch.knn.index.vectorvalues.KNNVectorValues; import org.opensearch.knn.indices.Model; import org.opensearch.knn.indices.ModelCache; import org.opensearch.knn.plugin.stats.KNNGraphValue; +import org.opensearch.knn.quantization.models.quantizationState.QuantizationState; import java.io.IOException; import java.io.OutputStream; @@ -60,6 +63,8 @@ public class NativeIndexWriter { private final SegmentWriteState state; private final FieldInfo fieldInfo; private final NativeIndexBuildStrategy indexBuilder; + @Nullable + private final QuantizationState quantizationState; /** * Gets the correct writer type from fieldInfo @@ -68,13 +73,29 @@ public class NativeIndexWriter { * @return correct NativeIndexWriter to make index specified in fieldInfo */ public static NativeIndexWriter getWriter(final FieldInfo fieldInfo, SegmentWriteState state) { - final KNNEngine knnEngine = extractKNNEngine(fieldInfo); - boolean isTemplate = fieldInfo.attributes().containsKey(MODEL_ID); - boolean iterative = !isTemplate && KNNEngine.FAISS == knnEngine; - if (iterative) { - return new NativeIndexWriter(state, fieldInfo, MemOptimizedNativeIndexBuildStrategy.getInstance()); - } - return new NativeIndexWriter(state, fieldInfo, DefaultIndexBuildStrategy.getInstance()); + return createWriter(fieldInfo, state, null); + } + + /** + * Gets the correct writer type for the specified field, using a given QuantizationModel. + * + * This method returns a NativeIndexWriter instance that is tailored to the specific characteristics + * of the field described by the provided FieldInfo. It determines whether to use a template-based + * writer or an iterative approach based on the engine type and whether the field is associated with a template. + * + * If quantization is required, the QuantizationModel is passed to the writer to facilitate the quantization process. + * + * @param fieldInfo The FieldInfo object containing metadata about the field for which the writer is needed. + * @param state The SegmentWriteState representing the current segment's writing context. + * @param quantizationState The QuantizationState that contains quantization state required for quantization + * @return A NativeIndexWriter instance appropriate for the specified field, configured with or without quantization. + */ + public static NativeIndexWriter getWriter( + final FieldInfo fieldInfo, + final SegmentWriteState state, + final QuantizationState quantizationState + ) { + return createWriter(fieldInfo, state, quantizationState); } /** @@ -137,7 +158,12 @@ private void buildAndWriteIndex(final KNNVectorValues knnVectorValues) throws // TODO: Refactor this so its scalable. Possibly move it out of this class private BuildIndexParams indexParams(FieldInfo fieldInfo, String indexPath, KNNEngine knnEngine) throws IOException { final Map parameters; - final VectorDataType vectorDataType = extractVectorDataType(fieldInfo); + VectorDataType vectorDataType; + if (quantizationState != null) { + vectorDataType = QuantizationService.getInstance().getVectorDataTypeForTransfer(fieldInfo); + } else { + vectorDataType = extractVectorDataType(fieldInfo); + } if (fieldInfo.attributes().containsKey(MODEL_ID)) { Model model = getModel(fieldInfo); parameters = getTemplateParameters(fieldInfo, model); @@ -151,6 +177,7 @@ private BuildIndexParams indexParams(FieldInfo fieldInfo, String indexPath, KNNE .vectorDataType(vectorDataType) .knnEngine(knnEngine) .indexPath(indexPath) + .quantizationState(quantizationState) .build(); } @@ -295,4 +322,26 @@ private void writeFooter(String indexPath, String engineFileName, SegmentWriteSt os.write(byteBuffer.array()); os.close(); } + + /** + * Helper method to create the appropriate NativeIndexWriter based on the field info and quantization state. + * + * @param fieldInfo The FieldInfo object containing metadata about the field for which the writer is needed. + * @param state The SegmentWriteState representing the current segment's writing context. + * @param quantizationState The QuantizationState that contains quantization state required for quantization, can be null. + * @return A NativeIndexWriter instance appropriate for the specified field, configured with or without quantization. + */ + private static NativeIndexWriter createWriter( + final FieldInfo fieldInfo, + final SegmentWriteState state, + @Nullable final QuantizationState quantizationState + ) { + final KNNEngine knnEngine = extractKNNEngine(fieldInfo); + boolean isTemplate = fieldInfo.attributes().containsKey(MODEL_ID); + boolean iterative = !isTemplate && KNNEngine.FAISS == knnEngine; + NativeIndexBuildStrategy strategy = iterative + ? MemOptimizedNativeIndexBuildStrategy.getInstance() + : DefaultIndexBuildStrategy.getInstance(); + return new NativeIndexWriter(state, fieldInfo, strategy, quantizationState); + } } diff --git a/src/main/java/org/opensearch/knn/index/codec/nativeindex/QuantizationIndexUtils.java b/src/main/java/org/opensearch/knn/index/codec/nativeindex/QuantizationIndexUtils.java new file mode 100644 index 000000000..8fec1af6d --- /dev/null +++ b/src/main/java/org/opensearch/knn/index/codec/nativeindex/QuantizationIndexUtils.java @@ -0,0 +1,68 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.codec.nativeindex; + +import lombok.experimental.UtilityClass; +import org.opensearch.knn.index.codec.nativeindex.model.BuildIndexParams; +import org.opensearch.knn.index.quantizationService.QuantizationService; +import org.opensearch.knn.index.vectorvalues.KNNVectorValues; +import org.opensearch.knn.quantization.models.quantizationOutput.QuantizationOutput; +import org.opensearch.knn.quantization.models.quantizationState.QuantizationState; + +import java.io.IOException; + +@UtilityClass +class QuantizationIndexUtils { + + /** + * Processes and returns the vector based on whether quantization is applied or not. + * + * @param knnVectorValues the KNN vector values to be processed. + * @param indexBuildSetup the setup containing quantization state and output, along with other parameters. + * @return the processed vector, either quantized or original. + * @throws IOException if an I/O error occurs during processing. + */ + static Object processAndReturnVector(KNNVectorValues knnVectorValues, IndexBuildSetup indexBuildSetup) throws IOException { + QuantizationService quantizationService = QuantizationService.getInstance(); + if (indexBuildSetup.getQuantizationState() != null && indexBuildSetup.getQuantizationOutput() != null) { + quantizationService.quantize( + indexBuildSetup.getQuantizationState(), + knnVectorValues.getVector(), + indexBuildSetup.getQuantizationOutput() + ); + return indexBuildSetup.getQuantizationOutput().getQuantizedVector(); + } else { + return knnVectorValues.conditionalCloneVector(); + } + } + + /** + * Prepares the quantization setup including bytes per vector and dimensions. + * + * @param knnVectorValues the KNN vector values. + * @param indexInfo the index build parameters. + * @return an instance of QuantizationSetup containing relevant information. + */ + static IndexBuildSetup prepareIndexBuild(KNNVectorValues knnVectorValues, BuildIndexParams indexInfo) { + QuantizationState quantizationState = indexInfo.getQuantizationState(); + QuantizationOutput quantizationOutput = null; + QuantizationService quantizationService = QuantizationService.getInstance(); + + int bytesPerVector; + int dimensions; + + if (quantizationState != null) { + bytesPerVector = quantizationState.getBytesPerVector(); + dimensions = quantizationState.getDimensions(); + quantizationOutput = quantizationService.createQuantizationOutput(quantizationState.getQuantizationParams()); + } else { + bytesPerVector = knnVectorValues.bytesPerVector(); + dimensions = knnVectorValues.dimension(); + } + + return new IndexBuildSetup(bytesPerVector, dimensions, quantizationOutput, quantizationState); + } +} diff --git a/src/main/java/org/opensearch/knn/index/codec/nativeindex/model/BuildIndexParams.java b/src/main/java/org/opensearch/knn/index/codec/nativeindex/model/BuildIndexParams.java index af43ff37e..78674c64b 100644 --- a/src/main/java/org/opensearch/knn/index/codec/nativeindex/model/BuildIndexParams.java +++ b/src/main/java/org/opensearch/knn/index/codec/nativeindex/model/BuildIndexParams.java @@ -8,8 +8,10 @@ import lombok.Builder; import lombok.ToString; import lombok.Value; +import org.opensearch.common.Nullable; import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.engine.KNNEngine; +import org.opensearch.knn.quantization.models.quantizationState.QuantizationState; import java.util.Map; @@ -22,5 +24,9 @@ public class BuildIndexParams { String indexPath; VectorDataType vectorDataType; Map parameters; - // TODO: Add quantization state as parameter to build index + /** + * An optional quantization state that contains required information for quantization + */ + @Nullable + QuantizationState quantizationState; } diff --git a/src/main/java/org/opensearch/knn/index/engine/faiss/AbstractFaissMethod.java b/src/main/java/org/opensearch/knn/index/engine/faiss/AbstractFaissMethod.java index 8c2dfc126..7ae403445 100644 --- a/src/main/java/org/opensearch/knn/index/engine/faiss/AbstractFaissMethod.java +++ b/src/main/java/org/opensearch/knn/index/engine/faiss/AbstractFaissMethod.java @@ -99,9 +99,7 @@ static KNNLibraryIndexingContext adjustIndexDescription( // We need to update the prefix used to create the faiss index if we are using the quantization // framework if (encoderContext != null && Objects.equals(encoderContext.getName(), QFrameBitEncoder.NAME)) { - // TODO: Uncomment to use Quantization framework - // leaving commented now just so it wont fail creating faiss indices. - // prefix = FAISS_BINARY_INDEX_DESCRIPTION_PREFIX; + prefix = FAISS_BINARY_INDEX_DESCRIPTION_PREFIX; } if (knnMethodConfigContext.getVectorDataType() == VectorDataType.BINARY) { diff --git a/src/main/java/org/opensearch/knn/index/quantizationService/KNNVectorQuantizationTrainingRequest.java b/src/main/java/org/opensearch/knn/index/quantizationService/KNNVectorQuantizationTrainingRequest.java new file mode 100644 index 000000000..4cf68d16c --- /dev/null +++ b/src/main/java/org/opensearch/knn/index/quantizationService/KNNVectorQuantizationTrainingRequest.java @@ -0,0 +1,55 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.quantizationService; + +import lombok.extern.log4j.Log4j2; +import org.opensearch.knn.index.vectorvalues.KNNVectorValues; +import org.opensearch.knn.quantization.models.requests.TrainingRequest; + +import java.io.IOException; + +import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS; + +/** + * KNNVectorQuantizationTrainingRequest is a concrete implementation of the abstract TrainingRequest class. + * It provides a mechanism to retrieve float vectors from the KNNVectorValues by document ID. + */ +@Log4j2 +final class KNNVectorQuantizationTrainingRequest extends TrainingRequest { + + private final KNNVectorValues knnVectorValues; + private int lastIndex; + + /** + * Constructs a new QuantizationFloatVectorTrainingRequest. + * + * @param knnVectorValues the KNNVectorValues instance containing the vectors. + */ + KNNVectorQuantizationTrainingRequest(KNNVectorValues knnVectorValues) { + super((int) knnVectorValues.totalLiveDocs()); + this.knnVectorValues = knnVectorValues; + this.lastIndex = 0; + } + + /** + * Retrieves the vector associated with the specified document ID. + * + * @param position the document ID. + * @return the float vector corresponding to the specified document ID, or null if the docId is invalid. + */ + @Override + public T getVectorAtThePosition(int position) throws IOException { + while (lastIndex <= position) { + lastIndex++; + if (knnVectorValues.docId() == NO_MORE_DOCS) { + return null; + } + knnVectorValues.nextDoc(); + } + // Return the vector and the updated index + return knnVectorValues.getVector(); + } +} diff --git a/src/main/java/org/opensearch/knn/index/quantizationService/QuantizationService.java b/src/main/java/org/opensearch/knn/index/quantizationService/QuantizationService.java new file mode 100644 index 000000000..a9e3cc715 --- /dev/null +++ b/src/main/java/org/opensearch/knn/index/quantizationService/QuantizationService.java @@ -0,0 +1,128 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.quantizationService; + +import lombok.AccessLevel; +import lombok.NoArgsConstructor; +import org.apache.lucene.index.FieldInfo; +import org.opensearch.knn.index.VectorDataType; +import org.opensearch.knn.index.engine.qframe.QuantizationConfig; +import org.opensearch.knn.index.vectorvalues.KNNVectorValues; +import org.opensearch.knn.quantization.factory.QuantizerFactory; +import org.opensearch.knn.quantization.models.quantizationOutput.BinaryQuantizationOutput; +import org.opensearch.knn.quantization.models.quantizationOutput.QuantizationOutput; +import org.opensearch.knn.quantization.models.quantizationParams.QuantizationParams; +import org.opensearch.knn.quantization.models.quantizationParams.ScalarQuantizationParams; +import org.opensearch.knn.quantization.models.quantizationState.QuantizationState; +import org.opensearch.knn.quantization.quantizer.Quantizer; +import java.io.IOException; + +import static org.opensearch.knn.common.FieldInfoExtractor.extractQuantizationConfig; + +/** + * A singleton class responsible for handling the quantization process, including training a quantizer + * and applying quantization to vectors. This class is designed to be thread-safe. + * + * @param The type of the input vectors to be quantized. + * @param The type of the quantized output vectors. + */ +@NoArgsConstructor(access = AccessLevel.PRIVATE) +public final class QuantizationService { + + /** + * The singleton instance of the {@link QuantizationService} class. + */ + private static final QuantizationService INSTANCE = new QuantizationService<>(); + + /** + * Returns the singleton instance of the {@link QuantizationService} class. + * + * @param The type of the input vectors to be quantized. + * @param The type of the quantized output vectors. + * @return The singleton instance of {@link QuantizationService}. + */ + public static QuantizationService getInstance() { + return (QuantizationService) INSTANCE; + } + + /** + * Trains a quantizer using the provided {@link KNNVectorValues} and returns the resulting + * {@link QuantizationState}. The quantizer is determined based on the given {@link QuantizationParams}. + * + * @param quantizationParams The {@link QuantizationParams} containing the parameters for quantization. + * @param knnVectorValues The {@link KNNVectorValues} representing the vector data to be used for training. + * @return The {@link QuantizationState} containing the state of the trained quantizer. + * @throws IOException If an I/O error occurs during the training process. + */ + public QuantizationState train(final QuantizationParams quantizationParams, final KNNVectorValues knnVectorValues) + throws IOException { + Quantizer quantizer = QuantizerFactory.getQuantizer(quantizationParams); + + // Create the training request from the vector values + KNNVectorQuantizationTrainingRequest trainingRequest = new KNNVectorQuantizationTrainingRequest<>(knnVectorValues); + + // Train the quantizer and return the quantization state + return quantizer.train(trainingRequest); + } + + /** + * Applies quantization to the given vector using the specified {@link QuantizationState} and + * {@link QuantizationOutput}. + * + * @param quantizationState The {@link QuantizationState} containing the state of the trained quantizer. + * @param vector The vector to be quantized. + * @param quantizationOutput The {@link QuantizationOutput} to store the quantized vector. + * @return The quantized vector as an object of type {@code R}. + */ + public R quantize(final QuantizationState quantizationState, final T vector, final QuantizationOutput quantizationOutput) { + Quantizer quantizer = QuantizerFactory.getQuantizer(quantizationState.getQuantizationParams()); + quantizer.quantize(vector, quantizationState, quantizationOutput); + return quantizationOutput.getQuantizedVector(); + } + + /** + * Retrieves quantization parameters from the FieldInfo. + */ + public QuantizationParams getQuantizationParams(final FieldInfo fieldInfo) { + QuantizationConfig quantizationConfig = extractQuantizationConfig(fieldInfo); + if (quantizationConfig != QuantizationConfig.EMPTY && quantizationConfig.getQuantizationType() != null) { + return new ScalarQuantizationParams(quantizationConfig.getQuantizationType()); + } + return null; + } + + /** + * Retrieves the appropriate {@link VectorDataType} to be used during the transfer of vectors for indexing or merging. + * This method is intended to determine the correct vector data type based on the provided {@link FieldInfo}. + * + * @param fieldInfo The {@link FieldInfo} object containing metadata about the field for which the vector data type + * is being determined. + * @return The {@link VectorDataType} to be used during the vector transfer process + */ + public VectorDataType getVectorDataTypeForTransfer(final FieldInfo fieldInfo) { + QuantizationConfig quantizationConfig = extractQuantizationConfig(fieldInfo); + if (quantizationConfig != QuantizationConfig.EMPTY && quantizationConfig.getQuantizationType() != null) { + return VectorDataType.BINARY; + } + return null; + } + + /** + * Creates the appropriate {@link QuantizationOutput} based on the given {@link QuantizationParams}. + * + * @param quantizationParams The {@link QuantizationParams} containing the parameters for quantization. + * @return The {@link QuantizationOutput} corresponding to the provided parameters. + * @throws IllegalArgumentException If the quantization parameters are unsupported. + */ + @SuppressWarnings("unchecked") + public QuantizationOutput createQuantizationOutput(final QuantizationParams quantizationParams) { + if (quantizationParams instanceof ScalarQuantizationParams) { + ScalarQuantizationParams scalarParams = (ScalarQuantizationParams) quantizationParams; + return (QuantizationOutput) new BinaryQuantizationOutput(scalarParams.getSqType().getId()); + } + throw new IllegalArgumentException("Unsupported quantization parameters: " + quantizationParams.getClass().getName()); + } +} diff --git a/src/main/java/org/opensearch/knn/quantization/models/quantizationState/DefaultQuantizationState.java b/src/main/java/org/opensearch/knn/quantization/models/quantizationState/DefaultQuantizationState.java index 33e775cad..531a70851 100644 --- a/src/main/java/org/opensearch/knn/quantization/models/quantizationState/DefaultQuantizationState.java +++ b/src/main/java/org/opensearch/knn/quantization/models/quantizationState/DefaultQuantizationState.java @@ -64,4 +64,19 @@ public byte[] toByteArray() throws IOException { public static DefaultQuantizationState fromByteArray(final byte[] bytes) throws IOException, ClassNotFoundException { return (DefaultQuantizationState) QuantizationStateSerializer.deserialize(bytes, DefaultQuantizationState::new); } + + @Override + public int getBytesPerVector() { + return 0; + } + + @Override + public int getDimensions() { + return 0; + } + + @Override + public long ramBytesUsed() { + return 0; + } } diff --git a/src/main/java/org/opensearch/knn/quantization/models/quantizationState/MultiBitScalarQuantizationState.java b/src/main/java/org/opensearch/knn/quantization/models/quantizationState/MultiBitScalarQuantizationState.java index 2778a6cf4..79ce7b955 100644 --- a/src/main/java/org/opensearch/knn/quantization/models/quantizationState/MultiBitScalarQuantizationState.java +++ b/src/main/java/org/opensearch/knn/quantization/models/quantizationState/MultiBitScalarQuantizationState.java @@ -8,6 +8,7 @@ import lombok.AllArgsConstructor; import lombok.Getter; import lombok.NoArgsConstructor; +import org.apache.lucene.util.RamUsageEstimator; import org.opensearch.Version; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; @@ -124,4 +125,49 @@ public byte[] toByteArray() throws IOException { public static MultiBitScalarQuantizationState fromByteArray(final byte[] bytes) throws IOException { return (MultiBitScalarQuantizationState) QuantizationStateSerializer.deserialize(bytes, MultiBitScalarQuantizationState::new); } + + /** + * Calculates and returns the number of bytes stored per vector after quantization. + * + * @return the number of bytes stored per vector. + */ + @Override + public int getBytesPerVector() { + // Check if thresholds are null or have invalid structure + if (thresholds == null || thresholds.length == 0 || thresholds[0] == null) { + throw new IllegalStateException("Error in getBytesStoredPerVector: The thresholds array is not initialized."); + } + + // Calculate the number of bytes required for multi-bit quantization + return thresholds.length * thresholds[0].length; + } + + @Override + public int getDimensions() { + // For multi-bit quantization, the dimension for indexing is the number of rows * columns in the thresholds array. + // Where number of column reprensents Dimesion of Original vector and number of rows equals to number of bits + // Check if thresholds are null or have invalid structure + if (thresholds == null || thresholds.length == 0 || thresholds[0] == null) { + throw new IllegalStateException("Error in getting Dimension: The thresholds array is not initialized."); + } + return thresholds.length * thresholds[0].length; + } + + /** + * Calculates the memory usage of the MultiBitScalarQuantizationState object in bytes. + * This method computes the shallow size of the instance itself, the shallow size of the + * quantization parameters, and the memory usage of the 2D thresholds array. + * + * @return The estimated memory usage of the MultiBitScalarQuantizationState object in bytes. + */ + @Override + public long ramBytesUsed() { + long size = RamUsageEstimator.shallowSizeOfInstance(MultiBitScalarQuantizationState.class); + size += RamUsageEstimator.shallowSizeOf(quantizationParams); + size += RamUsageEstimator.shallowSizeOf(thresholds); // shallow size of the 2D array (array of references to rows) + for (float[] row : thresholds) { + size += RamUsageEstimator.sizeOf(row); // size of each row in the 2D array + } + return size; + } } diff --git a/src/main/java/org/opensearch/knn/quantization/models/quantizationState/OneBitScalarQuantizationState.java b/src/main/java/org/opensearch/knn/quantization/models/quantizationState/OneBitScalarQuantizationState.java index 9998b87e8..9c4ff7460 100644 --- a/src/main/java/org/opensearch/knn/quantization/models/quantizationState/OneBitScalarQuantizationState.java +++ b/src/main/java/org/opensearch/knn/quantization/models/quantizationState/OneBitScalarQuantizationState.java @@ -8,6 +8,7 @@ import lombok.AllArgsConstructor; import lombok.Getter; import lombok.NoArgsConstructor; +import org.apache.lucene.util.RamUsageEstimator; import org.opensearch.Version; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; @@ -107,4 +108,36 @@ public byte[] toByteArray() throws IOException { public static OneBitScalarQuantizationState fromByteArray(final byte[] bytes) throws IOException { return (OneBitScalarQuantizationState) QuantizationStateSerializer.deserialize(bytes, OneBitScalarQuantizationState::new); } + + /** + * Calculates and returns the number of bytes stored per vector after quantization. + * + * @return the number of bytes stored per vector. + */ + @Override + public int getBytesPerVector() { + // Calculate the number of bytes required for one-bit quantization + return meanThresholds.length; + } + + @Override + public int getDimensions() { + // For one-bit quantization, the dimension for indexing is just the length of the thresholds array. + return meanThresholds.length; + } + + /** + * Calculates the memory usage of the OneBitScalarQuantizationState object in bytes. + * This method computes the shallow size of the instance itself, the shallow size of the + * quantization parameters, and the memory usage of the mean thresholds array. + * + * @return The estimated memory usage of the OneBitScalarQuantizationState object in bytes. + */ + @Override + public long ramBytesUsed() { + long size = RamUsageEstimator.shallowSizeOfInstance(OneBitScalarQuantizationState.class); + size += RamUsageEstimator.shallowSizeOf(quantizationParams); + size += RamUsageEstimator.sizeOf(meanThresholds); + return size; + } } diff --git a/src/main/java/org/opensearch/knn/quantization/models/quantizationState/QuantizationState.java b/src/main/java/org/opensearch/knn/quantization/models/quantizationState/QuantizationState.java index e32df8bc3..18ee813fc 100644 --- a/src/main/java/org/opensearch/knn/quantization/models/quantizationState/QuantizationState.java +++ b/src/main/java/org/opensearch/knn/quantization/models/quantizationState/QuantizationState.java @@ -29,4 +29,28 @@ public interface QuantizationState extends Writeable { * @throws IOException if an I/O error occurs during serialization. */ byte[] toByteArray() throws IOException; + + /** + * Calculates the number of bytes stored per vector after quantization. + * This method can be overridden by implementing classes to provide the specific calculation. + * + * @return the number of bytes stored per vector. + */ + int getBytesPerVector(); + + /** + * Returns the effective dimension used for indexing after quantization. + * For one-bit quantization, this might correspond to the length of thresholds. + * For multi-bit quantization, this might correspond to rows * columns of the thresholds matrix. + * + * @return the effective dimension for indexing. + */ + int getDimensions(); + + /** + * Estimates the memory usage of the quantization state in bytes. + * + * @return the memory usage in bytes. + */ + long ramBytesUsed(); } diff --git a/src/main/java/org/opensearch/knn/quantization/models/requests/TrainingRequest.java b/src/main/java/org/opensearch/knn/quantization/models/requests/TrainingRequest.java index 54ebe311c..d8b0eab10 100644 --- a/src/main/java/org/opensearch/knn/quantization/models/requests/TrainingRequest.java +++ b/src/main/java/org/opensearch/knn/quantization/models/requests/TrainingRequest.java @@ -8,6 +8,8 @@ import lombok.AllArgsConstructor; import lombok.Getter; +import java.io.IOException; + /** * TrainingRequest represents a request for training a quantizer. * @@ -24,8 +26,8 @@ public abstract class TrainingRequest { /** * Returns the vector corresponding to the specified document ID. * - * @param docId the document ID. + * @param position the document position. * @return the vector corresponding to the specified document ID. */ - public abstract T getVectorByDocId(int docId); + public abstract T getVectorAtThePosition(int position) throws IOException; } diff --git a/src/main/java/org/opensearch/knn/quantization/quantizer/MultiBitScalarQuantizer.java b/src/main/java/org/opensearch/knn/quantization/quantizer/MultiBitScalarQuantizer.java index 12a5d1013..a0e6ec402 100644 --- a/src/main/java/org/opensearch/knn/quantization/quantizer/MultiBitScalarQuantizer.java +++ b/src/main/java/org/opensearch/knn/quantization/quantizer/MultiBitScalarQuantizer.java @@ -15,6 +15,9 @@ import org.opensearch.knn.quantization.sampler.Sampler; import org.opensearch.knn.quantization.sampler.SamplerType; import org.opensearch.knn.quantization.sampler.SamplingFactory; +import oshi.util.tuples.Pair; + +import java.io.IOException; /** * MultiBitScalarQuantizer is responsible for quantizing vectors into multi-bit representations per dimension. @@ -105,14 +108,11 @@ public MultiBitScalarQuantizer(final int bitsPerCoordinate, final int samplingSi * @return a MultiBitScalarQuantizationState containing the computed thresholds. */ @Override - public QuantizationState train(final TrainingRequest trainingRequest) { + public QuantizationState train(final TrainingRequest trainingRequest) throws IOException { int[] sampledIndices = sampler.sample(trainingRequest.getTotalNumberOfVectors(), samplingSize); - int dimension = trainingRequest.getVectorByDocId(sampledIndices[0]).length; - float[] meanArray = new float[dimension]; - float[] stdDevArray = new float[dimension]; // Calculate sum, mean, and standard deviation in one pass - QuantizerHelper.calculateMeanAndStdDev(trainingRequest, sampledIndices, meanArray, stdDevArray); - float[][] thresholds = calculateThresholds(meanArray, stdDevArray, dimension); + Pair meanAndStdDev = QuantizerHelper.calculateMeanAndStdDev(trainingRequest, sampledIndices); + float[][] thresholds = calculateThresholds(meanAndStdDev.getA(), meanAndStdDev.getB()); ScalarQuantizationParams params = (bitsPerCoordinate == 2) ? new ScalarQuantizationParams(ScalarQuantizationType.TWO_BIT) : new ScalarQuantizationParams(ScalarQuantizationType.FOUR_BIT); @@ -148,10 +148,10 @@ public void quantize(final float[] vector, final QuantizationState state, final * * @param meanArray the mean for each dimension. * @param stdDevArray the standard deviation for each dimension. - * @param dimension the number of dimensions in the vectors. * @return the thresholds for quantization. */ - private float[][] calculateThresholds(final float[] meanArray, final float[] stdDevArray, final int dimension) { + private float[][] calculateThresholds(final float[] meanArray, final float[] stdDevArray) { + int dimension = meanArray.length; float[][] thresholds = new float[bitsPerCoordinate][dimension]; float coef = bitsPerCoordinate + 1; for (int i = 0; i < bitsPerCoordinate; i++) { diff --git a/src/main/java/org/opensearch/knn/quantization/quantizer/OneBitScalarQuantizer.java b/src/main/java/org/opensearch/knn/quantization/quantizer/OneBitScalarQuantizer.java index a0f6a26b4..ac48a9523 100644 --- a/src/main/java/org/opensearch/knn/quantization/quantizer/OneBitScalarQuantizer.java +++ b/src/main/java/org/opensearch/knn/quantization/quantizer/OneBitScalarQuantizer.java @@ -15,6 +15,8 @@ import org.opensearch.knn.quantization.sampler.SamplerType; import org.opensearch.knn.quantization.sampler.SamplingFactory; +import java.io.IOException; + /** * OneBitScalarQuantizer is responsible for quantizing vectors using a single bit per dimension. * It computes the mean of each dimension during training and then uses these means as thresholds @@ -56,7 +58,7 @@ public OneBitScalarQuantizer(final int samplingSize, final Sampler sampler) { * @return a OneBitScalarQuantizationState containing the calculated means. */ @Override - public QuantizationState train(final TrainingRequest trainingRequest) { + public QuantizationState train(final TrainingRequest trainingRequest) throws IOException { int[] sampledDocIds = sampler.sample(trainingRequest.getTotalNumberOfVectors(), samplingSize); float[] meanThresholds = QuantizerHelper.calculateMeanThresholds(trainingRequest, sampledDocIds); return new OneBitScalarQuantizationState(new ScalarQuantizationParams(ScalarQuantizationType.ONE_BIT), meanThresholds); diff --git a/src/main/java/org/opensearch/knn/quantization/quantizer/Quantizer.java b/src/main/java/org/opensearch/knn/quantization/quantizer/Quantizer.java index c0b297f5d..521863205 100644 --- a/src/main/java/org/opensearch/knn/quantization/quantizer/Quantizer.java +++ b/src/main/java/org/opensearch/knn/quantization/quantizer/Quantizer.java @@ -9,6 +9,8 @@ import org.opensearch.knn.quantization.models.quantizationState.QuantizationState; import org.opensearch.knn.quantization.models.requests.TrainingRequest; +import java.io.IOException; + /** * The Quantizer interface defines the methods required for training and quantizing vectors * in the context of K-Nearest Neighbors (KNN) and similar machine learning tasks. @@ -27,7 +29,7 @@ public interface Quantizer { * @param trainingRequest the request containing data and parameters for training. * @return a QuantizationState containing the learned parameters. */ - QuantizationState train(TrainingRequest trainingRequest); + QuantizationState train(TrainingRequest trainingRequest) throws IOException; /** * Quantizes the provided vector using the specified quantization state. diff --git a/src/main/java/org/opensearch/knn/quantization/quantizer/QuantizerHelper.java b/src/main/java/org/opensearch/knn/quantization/quantizer/QuantizerHelper.java index 16f969973..bac2067c0 100644 --- a/src/main/java/org/opensearch/knn/quantization/quantizer/QuantizerHelper.java +++ b/src/main/java/org/opensearch/knn/quantization/quantizer/QuantizerHelper.java @@ -7,6 +7,9 @@ import org.opensearch.knn.quantization.models.requests.TrainingRequest; import lombok.experimental.UtilityClass; +import oshi.util.tuples.Pair; + +import java.io.IOException; /** * Utility class providing common methods for quantizer operations, such as parameter validation and @@ -19,16 +22,17 @@ class QuantizerHelper { * Calculates the mean vector from a set of sampled vectors. * * @param samplingRequest The {@link TrainingRequest} containing the dataset and methods to access vectors by their indices. - * @param sampledIndices An array of indices representing the sampled vectors to be used for mean calculation. + * @param sampledIndices An array of indices representing the sampled vectors to be used for mean calculation. * @return A float array representing the mean vector of the sampled vectors. * @throws IllegalArgumentException If any of the vectors at the sampled indices are null. - * @throws IllegalStateException If the mean array is unexpectedly null after processing the vectors. + * @throws IllegalStateException If the mean array is unexpectedly null after processing the vectors. */ - static float[] calculateMeanThresholds(TrainingRequest samplingRequest, int[] sampledIndices) { + static float[] calculateMeanThresholds(TrainingRequest samplingRequest, int[] sampledIndices) throws IOException { int totalSamples = sampledIndices.length; float[] mean = null; + int lastIndex = 0; for (int docId : sampledIndices) { - float[] vector = samplingRequest.getVectorByDocId(docId); + float[] vector = samplingRequest.getVectorAtThePosition(docId); if (vector == null) { throw new IllegalArgumentException("Vector at sampled index " + docId + " is null."); } @@ -49,36 +53,56 @@ static float[] calculateMeanThresholds(TrainingRequest samplingRequest, } /** - * Calculates the mean and StdDev per dimension for sampled vectors. + * Calculates the mean and standard deviation for each dimension of the vectors in the training request. + *

    + * This method processes the vectors specified by the sampled indices and calculates both the mean and + * standard deviation in one pass. The results are returned as a pair of arrays: one for the means and + * one for the standard deviations. * - * @param trainingRequest the request containing the data and parameters for training. - * @param sampledIndices the indices of the sampled vectors. - * @param meanArray the array to store the sum and then the mean of each dimension. - * @param stdDevArray the array to store the sum of squares and then the standard deviation of each dimension. + * @param trainingRequest The request containing the data and parameters for training. + * @param sampledIndices An array of document IDs representing the sampled indices to be processed. + * @return A Pair containing two float arrays: the first array represents the mean of each dimension, + * and the second array represents the standard deviation of each dimension. + * @throws IllegalArgumentException if any of the vectors at the sampled indices are null. + * @throws IllegalStateException if the mean or standard deviation arrays are not initialized after processing. */ - static void calculateMeanAndStdDev( - TrainingRequest trainingRequest, - int[] sampledIndices, - float[] meanArray, - float[] stdDevArray - ) { + static Pair calculateMeanAndStdDev(TrainingRequest trainingRequest, int[] sampledIndices) + throws IOException { + float[] meanArray = null; + float[] stdDevArray = null; int totalSamples = sampledIndices.length; - int dimension = meanArray.length; + int lastIndex = 0; for (int docId : sampledIndices) { - float[] vector = trainingRequest.getVectorByDocId(docId); + float[] vector = trainingRequest.getVectorAtThePosition(docId); if (vector == null) { throw new IllegalArgumentException("Vector at sampled index " + docId + " is null."); } + int dimension = vector.length; + + // Initialize meanArray and stdDevArray on the first iteration + if (meanArray == null) { + meanArray = new float[dimension]; + } + if (stdDevArray == null) { + stdDevArray = new float[dimension]; + } + for (int j = 0; j < dimension; j++) { meanArray[j] += vector[j]; stdDevArray[j] += vector[j] * vector[j]; } } + if (meanArray == null || stdDevArray == null) { + throw new IllegalStateException("Mean and StdDev should not be null after processing vectors."); + } // Calculate mean and standard deviation in one pass - for (int j = 0; j < dimension; j++) { + for (int j = 0; j < meanArray.length; j++) { meanArray[j] = meanArray[j] / totalSamples; stdDevArray[j] = (float) Math.sqrt((stdDevArray[j] / totalSamples) - (meanArray[j] * meanArray[j])); } + + // Return both arrays as a Pair + return new Pair<>(meanArray, stdDevArray); } } diff --git a/src/test/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsFormatTests.java b/src/test/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsFormatTests.java index 3810d46fd..85b5d07e6 100644 --- a/src/test/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsFormatTests.java +++ b/src/test/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsFormatTests.java @@ -47,8 +47,11 @@ import org.opensearch.knn.common.KNNConstants; import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.VectorDataType; +import org.opensearch.knn.index.engine.qframe.QuantizationConfig; +import org.opensearch.knn.index.engine.qframe.QuantizationConfigParser; import org.opensearch.knn.index.mapper.KNNVectorFieldMapper; import org.opensearch.knn.index.engine.KNNEngine; +import org.opensearch.knn.quantization.enums.ScalarQuantizationType; import java.io.IOException; import java.util.Arrays; @@ -61,6 +64,7 @@ public class NativeEngines990KnnVectorsFormatTests extends KNNTestCase { private static final String FLAT_VECTOR_FILE_EXT = ".vec"; private static final String HNSW_FILE_EXT = ".hnsw"; private static final String FLOAT_VECTOR_FIELD = "float_field"; + private static final String FLOAT_VECTOR_FIELD_BINARY = "float_field_binary"; private static final String BYTE_VECTOR_FIELD = "byte_field"; private Directory dir; private RandomIndexWriter indexWriter; @@ -99,14 +103,32 @@ public void testNativeEngineVectorFormat_whenMultipleVectorFieldIndexed_thenSucc float[] floatVector = { 1.0f, 3.0f, 4.0f }; byte[] byteVector = { 6, 14 }; + FieldType fieldTypeForFloat = createVectorField(3, VectorEncoding.FLOAT32, VectorDataType.FLOAT); + fieldTypeForFloat.putAttribute(KNNConstants.PARAMETERS, "{ \"index_description\":\"HNSW16,Flat\", \"spaceType\": \"l2\"}"); + fieldTypeForFloat.freeze(); + addFieldToIndex(new KnnFloatVectorField(FLOAT_VECTOR_FIELD, floatVector, fieldTypeForFloat), indexWriter); + FieldType fieldTypeForByte = createVectorField(2, VectorEncoding.BYTE, VectorDataType.BINARY); + fieldTypeForByte.putAttribute(KNNConstants.PARAMETERS, "{ \"index_description\":\"HNSW16,Flat\", \"spaceType\": \"l2\"}"); + fieldTypeForByte.freeze(); + addFieldToIndex(new KnnByteVectorField(BYTE_VECTOR_FIELD, byteVector, fieldTypeForByte), indexWriter); + + float[] floatVectorForBinaryQuantization_1 = { 1.0f, 3.0f, 4.0f, 5.0f, 6.0f, 7.0f, 8.0f, 9.0f }; + float[] floatVectorForBinaryQuantization_2 = { 1.0f, 3.0f, 4.0f, 5.0f, 6.0f, 7.0f, 8.0f, 9.0f }; + FieldType fieldTypeForBinaryQuantization = createVectorField(8, VectorEncoding.FLOAT32, VectorDataType.FLOAT); + fieldTypeForBinaryQuantization.putAttribute(KNNConstants.PARAMETERS, "{ \"index_description\":\"BHNSW32\", \"spaceType\": \"l2\"}"); + QuantizationConfig quantizationConfig = QuantizationConfig.builder().quantizationType(ScalarQuantizationType.ONE_BIT).build(); + fieldTypeForBinaryQuantization.putAttribute(KNNConstants.QFRAMEWORK_CONFIG, QuantizationConfigParser.toCsv(quantizationConfig)); + fieldTypeForBinaryQuantization.freeze(); + addFieldToIndex( - new KnnFloatVectorField(FLOAT_VECTOR_FIELD, floatVector, createVectorField(3, VectorEncoding.FLOAT32, VectorDataType.FLOAT)), + new KnnFloatVectorField(FLOAT_VECTOR_FIELD_BINARY, floatVectorForBinaryQuantization_1, fieldTypeForBinaryQuantization), indexWriter ); addFieldToIndex( - new KnnByteVectorField(BYTE_VECTOR_FIELD, byteVector, createVectorField(2, VectorEncoding.BYTE, VectorDataType.BINARY)), + new KnnFloatVectorField(FLOAT_VECTOR_FIELD_BINARY, floatVectorForBinaryQuantization_2, fieldTypeForBinaryQuantization), indexWriter ); + final IndexReader indexReader = indexWriter.getReader(); // ensuring segments are created indexWriter.flush(); @@ -129,7 +151,7 @@ public void testNativeEngineVectorFormat_whenMultipleVectorFieldIndexed_thenSucc if (segmentReader.getSegmentInfo().info.getUseCompoundFile() == false) { final List vecfiles = getFilesFromSegment(dir, FLAT_VECTOR_FILE_EXT); // 2 .vec files will be created as we are using per field vectors format. - assertEquals(2, vecfiles.size()); + assertEquals(3, vecfiles.size()); } final FloatVectorValues floatVectorValues = leafReader.getFloatVectorValues(FLOAT_VECTOR_FIELD); @@ -144,6 +166,12 @@ public void testNativeEngineVectorFormat_whenMultipleVectorFieldIndexed_thenSucc assertEquals(1, byteVectorValues.size()); assertEquals(2, byteVectorValues.dimension()); + final FloatVectorValues floatVectorValuesForBinaryQuantization = leafReader.getFloatVectorValues(FLOAT_VECTOR_FIELD_BINARY); + floatVectorValuesForBinaryQuantization.nextDoc(); + assertArrayEquals(floatVectorForBinaryQuantization_1, floatVectorValuesForBinaryQuantization.vectorValue(), 0.0f); + assertEquals(2, floatVectorValuesForBinaryQuantization.size()); + assertEquals(8, floatVectorValuesForBinaryQuantization.dimension()); + Assert.assertThrows( UnsupportedOperationException.class, () -> leafReader.searchNearestVectors(FLOAT_VECTOR_FIELD, floatVector, 10, new Bits.MatchAllBits(1), 10) @@ -157,6 +185,42 @@ public void testNativeEngineVectorFormat_whenMultipleVectorFieldIndexed_thenSucc indexReader.close(); } + @SneakyThrows + public void testNativeEngineVectorFormat_whenBinaryQuantizationApplied_thenSuccess() { + setup(); + float[] floatVectorForBinaryQuantization = { 1.0f, 3.0f, 4.0f, 5.0f, 6.0f, 7.0f, 8.0f, 9.0f }; + FieldType fieldTypeForBinaryQuantization = createVectorField(8, VectorEncoding.FLOAT32, VectorDataType.FLOAT); + fieldTypeForBinaryQuantization.putAttribute(KNNConstants.PARAMETERS, "{ \"index_description\":\"BHNSW32\", \"spaceType\": \"l2\"}"); + QuantizationConfig quantizationConfig = QuantizationConfig.builder().quantizationType(ScalarQuantizationType.ONE_BIT).build(); + fieldTypeForBinaryQuantization.putAttribute(KNNConstants.QFRAMEWORK_CONFIG, QuantizationConfigParser.toCsv(quantizationConfig)); + + addFieldToIndex( + new KnnFloatVectorField(FLOAT_VECTOR_FIELD_BINARY, floatVectorForBinaryQuantization, fieldTypeForBinaryQuantization), + indexWriter + ); + + final IndexReader indexReader = indexWriter.getReader(); + // ensuring segments are created + indexWriter.flush(); + indexWriter.commit(); + indexWriter.close(); + IndexSearcher searcher = new IndexSearcher(indexReader); + final LeafReader leafReader = searcher.getLeafContexts().get(0).reader(); + SegmentReader segmentReader = Lucene.segmentReader(leafReader); + if (segmentReader.getSegmentInfo().info.getUseCompoundFile() == false) { + final List vecfiles = getFilesFromSegment(dir, FLAT_VECTOR_FILE_EXT); + // 2 .vec files will be created as we are using per field vectors format. + assertEquals(1, vecfiles.size()); + } + + final FloatVectorValues floatVectorValues = leafReader.getFloatVectorValues(FLOAT_VECTOR_FIELD_BINARY); + floatVectorValues.nextDoc(); + assertArrayEquals(floatVectorForBinaryQuantization, floatVectorValues.vectorValue(), 0.0f); + assertEquals(1, floatVectorValues.size()); + assertEquals(8, floatVectorValues.dimension()); + indexReader.close(); + } + private List getFilesFromSegment(Directory dir, String fileFormat) throws IOException { return Arrays.stream(dir.listAll()).filter(x -> x.contains(fileFormat)).collect(Collectors.toList()); } @@ -203,13 +267,11 @@ private FieldType createVectorField(int dimension, VectorEncoding vectorEncoding nativeVectorField.putAttribute(KNNConstants.HNSW_ALGO_M, "32"); nativeVectorField.putAttribute(KNNConstants.HNSW_ALGO_EF_CONSTRUCTION, "512"); nativeVectorField.putAttribute(KNNConstants.VECTOR_DATA_TYPE_FIELD, vectorDataType.getValue()); - nativeVectorField.putAttribute(KNNConstants.PARAMETERS, "{ \"index_description\":\"HNSW16,Flat\", \"spaceType\": \"l2\"}"); nativeVectorField.setVectorAttributes( dimension, vectorEncoding, SpaceType.L2.getKnnVectorSimilarityFunction().getVectorSimilarityFunction() ); - nativeVectorField.freeze(); return nativeVectorField; } } diff --git a/src/test/java/org/opensearch/knn/index/codec/nativeindex/DefaultIndexBuildStrategyTests.java b/src/test/java/org/opensearch/knn/index/codec/nativeindex/DefaultIndexBuildStrategyTests.java index 34a333471..0b5a06dfc 100644 --- a/src/test/java/org/opensearch/knn/index/codec/nativeindex/DefaultIndexBuildStrategyTests.java +++ b/src/test/java/org/opensearch/knn/index/codec/nativeindex/DefaultIndexBuildStrategyTests.java @@ -17,11 +17,14 @@ import org.opensearch.knn.index.codec.transfer.OffHeapVectorTransfer; import org.opensearch.knn.index.codec.transfer.OffHeapVectorTransferFactory; import org.opensearch.knn.index.engine.KNNEngine; +import org.opensearch.knn.index.quantizationService.QuantizationService; import org.opensearch.knn.index.vectorvalues.KNNFloatVectorValues; import org.opensearch.knn.index.vectorvalues.KNNVectorValues; import org.opensearch.knn.index.vectorvalues.KNNVectorValuesFactory; import org.opensearch.knn.index.vectorvalues.TestVectorValues; import org.opensearch.knn.jni.JNIService; +import org.opensearch.knn.quantization.models.quantizationOutput.QuantizationOutput; +import org.opensearch.knn.quantization.models.quantizationState.QuantizationState; import org.opensearch.test.OpenSearchTestCase; import java.util.List; @@ -102,6 +105,119 @@ public void testBuildAndWrite() { } } + @SneakyThrows + public void testBuildAndWrite_withQuantization() { + // Given + ArgumentCaptor vectorAddressCaptor = ArgumentCaptor.forClass(Long.class); + ArgumentCaptor vectorTransferCapture = ArgumentCaptor.forClass(Object.class); + + List vectorValues = List.of(new float[] { 1, 2 }, new float[] { 2, 3 }, new float[] { 3, 4 }); + final TestVectorValues.PreDefinedFloatVectorValues randomVectorValues = new TestVectorValues.PreDefinedFloatVectorValues( + vectorValues + ); + final KNNVectorValues knnVectorValues = KNNVectorValuesFactory.getVectorValues(VectorDataType.FLOAT, randomVectorValues); + + try ( + MockedStatic mockedKNNSettings = mockStatic(KNNSettings.class); + MockedStatic mockedJNIService = mockStatic(JNIService.class); + MockedStatic mockedOffHeapVectorTransferFactory = mockStatic(OffHeapVectorTransferFactory.class); + MockedStatic mockedQuantizationIntegration = mockStatic(QuantizationService.class) + ) { + + // Limits transfer to 2 vectors + mockedKNNSettings.when(KNNSettings::getVectorStreamingMemoryLimit).thenReturn(new ByteSizeValue(16)); + mockedJNIService.when(() -> JNIService.initIndex(3, 2, Map.of("index", "param"), KNNEngine.FAISS)).thenReturn(100L); + + OffHeapVectorTransfer offHeapVectorTransfer = mock(OffHeapVectorTransfer.class); + mockedOffHeapVectorTransferFactory.when(() -> OffHeapVectorTransferFactory.getVectorTransfer(VectorDataType.FLOAT, 2)) + .thenReturn(offHeapVectorTransfer); + + QuantizationService quantizationService = mock(QuantizationService.class); + mockedQuantizationIntegration.when(QuantizationService::getInstance).thenReturn(quantizationService); + + QuantizationState quantizationState = mock(QuantizationState.class); + ArgumentCaptor vectorCaptor = ArgumentCaptor.forClass(float[].class); + // New: Create QuantizationOutput and mock the quantization process + QuantizationOutput quantizationOutput = mock(QuantizationOutput.class); + when(quantizationOutput.getQuantizedVector()).thenReturn(new byte[] { 1, 2 }); + when(quantizationService.createQuantizationOutput(eq(quantizationState.getQuantizationParams()))).thenReturn( + quantizationOutput + ); + + // Quantize the vector with the quantization output + when(quantizationService.quantize(eq(quantizationState), vectorCaptor.capture(), eq(quantizationOutput))).thenAnswer( + invocation -> { + quantizationOutput.getQuantizedVector(); + return quantizationOutput.getQuantizedVector(); + } + ); + when(quantizationState.getDimensions()).thenReturn(2); + when(quantizationState.getBytesPerVector()).thenReturn(8); + + when(offHeapVectorTransfer.transfer(vectorTransferCapture.capture(), eq(false))).thenReturn(false) + .thenReturn(true) + .thenReturn(false); + when(offHeapVectorTransfer.flush(false)).thenReturn(true); + when(offHeapVectorTransfer.getVectorAddress()).thenReturn(200L); + + BuildIndexParams buildIndexParams = BuildIndexParams.builder() + .indexPath("indexPath") + .knnEngine(KNNEngine.FAISS) + .vectorDataType(VectorDataType.FLOAT) + .parameters(Map.of("index", "param")) + .quantizationState(quantizationState) + .build(); + + // When + MemOptimizedNativeIndexBuildStrategy.getInstance().buildAndWriteIndex(buildIndexParams, knnVectorValues); + + // Then + mockedJNIService.verify( + () -> JNIService.initIndex( + knnVectorValues.totalLiveDocs(), + knnVectorValues.dimension(), + Map.of("index", "param"), + KNNEngine.FAISS + ) + ); + + mockedJNIService.verify( + () -> JNIService.insertToIndex( + eq(new int[] { 0, 1 }), + vectorAddressCaptor.capture(), + eq(knnVectorValues.dimension()), + eq(Map.of("index", "param")), + eq(100L), + eq(KNNEngine.FAISS) + ) + ); + + // For the flush + mockedJNIService.verify( + () -> JNIService.insertToIndex( + eq(new int[] { 2 }), + vectorAddressCaptor.capture(), + eq(knnVectorValues.dimension()), + eq(Map.of("index", "param")), + eq(100L), + eq(KNNEngine.FAISS) + ) + ); + + mockedJNIService.verify( + () -> JNIService.writeIndex(eq("indexPath"), eq(100L), eq(KNNEngine.FAISS), eq(Map.of("index", "param"))) + ); + assertEquals(200L, vectorAddressCaptor.getValue().longValue()); + assertEquals(vectorAddressCaptor.getValue().longValue(), vectorAddressCaptor.getAllValues().get(0).longValue()); + verify(offHeapVectorTransfer, times(0)).reset(); + + for (Object vector : vectorTransferCapture.getAllValues()) { + // Assert that the vector is in byte[] format due to quantization + assertTrue(vector instanceof byte[]); + } + } + } + @SneakyThrows public void testBuildAndWriteWithModel() { // Given diff --git a/src/test/java/org/opensearch/knn/index/codec/nativeindex/MemOptimizedNativeIndexBuildStrategyTests.java b/src/test/java/org/opensearch/knn/index/codec/nativeindex/MemOptimizedNativeIndexBuildStrategyTests.java index 2ecfe9259..3bfec4104 100644 --- a/src/test/java/org/opensearch/knn/index/codec/nativeindex/MemOptimizedNativeIndexBuildStrategyTests.java +++ b/src/test/java/org/opensearch/knn/index/codec/nativeindex/MemOptimizedNativeIndexBuildStrategyTests.java @@ -16,10 +16,13 @@ import org.opensearch.knn.index.codec.transfer.OffHeapVectorTransfer; import org.opensearch.knn.index.codec.transfer.OffHeapVectorTransferFactory; import org.opensearch.knn.index.engine.KNNEngine; +import org.opensearch.knn.index.quantizationService.QuantizationService; import org.opensearch.knn.index.vectorvalues.KNNVectorValues; import org.opensearch.knn.index.vectorvalues.KNNVectorValuesFactory; import org.opensearch.knn.index.vectorvalues.TestVectorValues; import org.opensearch.knn.jni.JNIService; +import org.opensearch.knn.quantization.models.quantizationOutput.QuantizationOutput; +import org.opensearch.knn.quantization.models.quantizationState.QuantizationState; import org.opensearch.test.OpenSearchTestCase; import java.util.List; @@ -126,4 +129,119 @@ public void testBuildAndWrite() { } } } + + @SneakyThrows + public void testBuildAndWrite_withQuantization() { + // Given + ArgumentCaptor vectorAddressCaptor = ArgumentCaptor.forClass(Long.class); + ArgumentCaptor vectorTransferCapture = ArgumentCaptor.forClass(Object.class); + + List vectorValues = List.of(new float[] { 1, 2 }, new float[] { 2, 3 }, new float[] { 3, 4 }); + final TestVectorValues.PreDefinedFloatVectorValues randomVectorValues = new TestVectorValues.PreDefinedFloatVectorValues( + vectorValues + ); + final KNNVectorValues knnVectorValues = KNNVectorValuesFactory.getVectorValues(VectorDataType.FLOAT, randomVectorValues); + + try ( + MockedStatic mockedKNNSettings = Mockito.mockStatic(KNNSettings.class); + MockedStatic mockedJNIService = Mockito.mockStatic(JNIService.class); + MockedStatic mockedOffHeapVectorTransferFactory = Mockito.mockStatic( + OffHeapVectorTransferFactory.class + ); + MockedStatic mockedQuantizationIntegration = Mockito.mockStatic(QuantizationService.class) + ) { + + // Limits transfer to 2 vectors + mockedKNNSettings.when(KNNSettings::getVectorStreamingMemoryLimit).thenReturn(new ByteSizeValue(16)); + mockedJNIService.when(() -> JNIService.initIndex(3, 2, Map.of("index", "param"), KNNEngine.FAISS)).thenReturn(100L); + + OffHeapVectorTransfer offHeapVectorTransfer = mock(OffHeapVectorTransfer.class); + mockedOffHeapVectorTransferFactory.when(() -> OffHeapVectorTransferFactory.getVectorTransfer(VectorDataType.FLOAT, 2)) + .thenReturn(offHeapVectorTransfer); + + QuantizationService quantizationService = mock(QuantizationService.class); + mockedQuantizationIntegration.when(QuantizationService::getInstance).thenReturn(quantizationService); + + QuantizationState quantizationState = mock(QuantizationState.class); + ArgumentCaptor vectorCaptor = ArgumentCaptor.forClass(float[].class); + // New: Create QuantizationOutput and mock the quantization process + QuantizationOutput quantizationOutput = mock(QuantizationOutput.class); + when(quantizationOutput.getQuantizedVector()).thenReturn(new byte[] { 1, 2 }); + when(quantizationService.createQuantizationOutput(eq(quantizationState.getQuantizationParams()))).thenReturn( + quantizationOutput + ); + + // Quantize the vector with the quantization output + when(quantizationService.quantize(eq(quantizationState), vectorCaptor.capture(), eq(quantizationOutput))).thenAnswer( + invocation -> { + quantizationOutput.getQuantizedVector(); + return quantizationOutput.getQuantizedVector(); + } + ); + when(quantizationState.getDimensions()).thenReturn(2); + when(quantizationState.getBytesPerVector()).thenReturn(8); + + when(offHeapVectorTransfer.transfer(vectorTransferCapture.capture(), eq(false))).thenReturn(false) + .thenReturn(true) + .thenReturn(false); + when(offHeapVectorTransfer.flush(false)).thenReturn(true); + when(offHeapVectorTransfer.getVectorAddress()).thenReturn(200L); + + BuildIndexParams buildIndexParams = BuildIndexParams.builder() + .indexPath("indexPath") + .knnEngine(KNNEngine.FAISS) + .vectorDataType(VectorDataType.FLOAT) + .parameters(Map.of("index", "param")) + .quantizationState(quantizationState) + .build(); + + // When + MemOptimizedNativeIndexBuildStrategy.getInstance().buildAndWriteIndex(buildIndexParams, knnVectorValues); + + // Then + mockedJNIService.verify( + () -> JNIService.initIndex( + knnVectorValues.totalLiveDocs(), + knnVectorValues.dimension(), + Map.of("index", "param"), + KNNEngine.FAISS + ) + ); + + mockedJNIService.verify( + () -> JNIService.insertToIndex( + eq(new int[] { 0, 1 }), + vectorAddressCaptor.capture(), + eq(knnVectorValues.dimension()), + eq(Map.of("index", "param")), + eq(100L), + eq(KNNEngine.FAISS) + ) + ); + + // For the flush + mockedJNIService.verify( + () -> JNIService.insertToIndex( + eq(new int[] { 2 }), + vectorAddressCaptor.capture(), + eq(knnVectorValues.dimension()), + eq(Map.of("index", "param")), + eq(100L), + eq(KNNEngine.FAISS) + ) + ); + + mockedJNIService.verify( + () -> JNIService.writeIndex(eq("indexPath"), eq(100L), eq(KNNEngine.FAISS), eq(Map.of("index", "param"))) + ); + assertEquals(200L, vectorAddressCaptor.getValue().longValue()); + assertEquals(vectorAddressCaptor.getValue().longValue(), vectorAddressCaptor.getAllValues().get(0).longValue()); + verify(offHeapVectorTransfer, times(0)).reset(); + + for (Object vector : vectorTransferCapture.getAllValues()) { + // Assert that the vector is in byte[] format due to quantization + assertTrue(vector instanceof byte[]); + } + } + } } diff --git a/src/test/java/org/opensearch/knn/index/codec/nativeindex/QuantizationIndexUtilsTests.java b/src/test/java/org/opensearch/knn/index/codec/nativeindex/QuantizationIndexUtilsTests.java new file mode 100644 index 000000000..30a2098dd --- /dev/null +++ b/src/test/java/org/opensearch/knn/index/codec/nativeindex/QuantizationIndexUtilsTests.java @@ -0,0 +1,109 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.codec.nativeindex; + +import org.junit.Before; +import org.opensearch.knn.KNNTestCase; +import org.opensearch.knn.index.VectorDataType; +import org.opensearch.knn.index.codec.nativeindex.model.BuildIndexParams; +import org.opensearch.knn.index.quantizationService.QuantizationService; +import org.opensearch.knn.index.vectorvalues.KNNVectorValues; +import org.opensearch.knn.index.vectorvalues.KNNVectorValuesFactory; +import org.opensearch.knn.index.vectorvalues.TestVectorValues; +import org.opensearch.knn.quantization.enums.ScalarQuantizationType; +import org.opensearch.knn.quantization.models.quantizationOutput.QuantizationOutput; +import org.opensearch.knn.quantization.models.quantizationParams.ScalarQuantizationParams; +import org.opensearch.knn.quantization.models.quantizationState.OneBitScalarQuantizationState; +import org.opensearch.knn.quantization.models.quantizationState.QuantizationState; + +import java.io.IOException; +import java.util.List; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class QuantizationIndexUtilsTests extends KNNTestCase { + + private KNNVectorValues knnVectorValues; + private BuildIndexParams buildIndexParams; + private QuantizationService quantizationService; + + @Before + public void setUp() throws Exception { + super.setUp(); + quantizationService = mock(QuantizationService.class); + + // Predefined float vectors for testing + List floatVectors = List.of( + new float[] { 1.0f, 2.0f, 3.0f }, + new float[] { 4.0f, 5.0f, 6.0f }, + new float[] { 7.0f, 8.0f, 9.0f } + ); + + // Use the predefined vectors to create KNNVectorValues + knnVectorValues = KNNVectorValuesFactory.getVectorValues( + VectorDataType.FLOAT, + new TestVectorValues.PreDefinedFloatVectorValues(floatVectors) + ); + + // Mocking BuildIndexParams + buildIndexParams = mock(BuildIndexParams.class); + } + + public void testPrepareIndexBuild_withQuantization_success() { + QuantizationState quantizationState = mock(OneBitScalarQuantizationState.class); + QuantizationOutput quantizationOutput = mock(QuantizationOutput.class); + + ScalarQuantizationParams params = new ScalarQuantizationParams(ScalarQuantizationType.ONE_BIT); + when(quantizationOutput.getQuantizedVector()).thenReturn(new byte[] { 0x01 }); + when(quantizationState.getDimensions()).thenReturn(2); + when(quantizationState.getBytesPerVector()).thenReturn(8); + when(quantizationState.getQuantizationParams()).thenReturn(params); + + when(buildIndexParams.getQuantizationState()).thenReturn(quantizationState); + + IndexBuildSetup setup = QuantizationIndexUtils.prepareIndexBuild(knnVectorValues, buildIndexParams); + + assertNotNull(setup.getQuantizationState()); + assertEquals(8, setup.getBytesPerVector()); + assertEquals(2, setup.getDimensions()); + } + + public void testPrepareIndexBuild_withoutQuantization_success() throws IOException { + when(buildIndexParams.getQuantizationState()).thenReturn(null); + knnVectorValues.nextDoc(); + knnVectorValues.getVector(); + IndexBuildSetup setup = QuantizationIndexUtils.prepareIndexBuild(knnVectorValues, buildIndexParams); + assertNull(setup.getQuantizationState()); + assertEquals(knnVectorValues.bytesPerVector(), setup.getBytesPerVector()); + assertEquals(knnVectorValues.dimension(), setup.getDimensions()); + } + + public void testProcessAndReturnVector_withoutQuantization_success() throws IOException { + // Set up the BuildIndexParams to return no quantization + when(buildIndexParams.getQuantizationState()).thenReturn(null); + knnVectorValues.nextDoc(); + knnVectorValues.getVector(); + IndexBuildSetup setup = QuantizationIndexUtils.prepareIndexBuild(knnVectorValues, buildIndexParams); + // Process and return the vector + assertNotNull(QuantizationIndexUtils.processAndReturnVector(knnVectorValues, setup)); + } + + public void testProcessAndReturnVector_withQuantization_success() throws IOException { + // Set up quantization state and output + ScalarQuantizationParams params = new ScalarQuantizationParams(ScalarQuantizationType.ONE_BIT); + float[] mean = { 1.0f, 2.0f, 3.0f }; + knnVectorValues.nextDoc(); + OneBitScalarQuantizationState state = new OneBitScalarQuantizationState(params, mean); + QuantizationOutput quantizationOutput = mock(QuantizationOutput.class); + when(buildIndexParams.getQuantizationState()).thenReturn(state); + IndexBuildSetup setup = QuantizationIndexUtils.prepareIndexBuild(knnVectorValues, buildIndexParams); + // Process and return the vector + Object result = QuantizationIndexUtils.processAndReturnVector(knnVectorValues, setup); + assertTrue(result instanceof byte[]); + assertArrayEquals(new byte[] { 0x00 }, (byte[]) result); + } +} diff --git a/src/test/java/org/opensearch/knn/index/engine/faiss/FaissTests.java b/src/test/java/org/opensearch/knn/index/engine/faiss/FaissTests.java index c9ce50f22..75da6811e 100644 --- a/src/test/java/org/opensearch/knn/index/engine/faiss/FaissTests.java +++ b/src/test/java/org/opensearch/knn/index/engine/faiss/FaissTests.java @@ -246,7 +246,7 @@ public void testGetKNNLibraryIndexingContext_whenMethodIsHNSWWithQFrame_thenCrea .vectorDataType(VectorDataType.FLOAT) .build(); int m = 88; - String expectedIndexDescription = "HNSW" + m + ",Flat"; + String expectedIndexDescription = "BHNSW" + m + ",Flat"; XContentBuilder xContentBuilder = XContentFactory.jsonBuilder() .startObject() .field(NAME, METHOD_HNSW) @@ -285,7 +285,7 @@ public void testGetKNNLibraryIndexingContext_whenMethodIsIVFWithQFrame_thenCreat .vectorDataType(VectorDataType.FLOAT) .build(); int nlist = 88; - String expectedIndexDescription = "IVF" + nlist + ",Flat"; + String expectedIndexDescription = "BIVF" + nlist + ",Flat"; XContentBuilder xContentBuilder = XContentFactory.jsonBuilder() .startObject() .field(NAME, METHOD_IVF) diff --git a/src/test/java/org/opensearch/knn/index/quantizationService/QuantizationServiceTests.java b/src/test/java/org/opensearch/knn/index/quantizationService/QuantizationServiceTests.java new file mode 100644 index 000000000..886dbeabc --- /dev/null +++ b/src/test/java/org/opensearch/knn/index/quantizationService/QuantizationServiceTests.java @@ -0,0 +1,159 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.quantizationService; + +import org.opensearch.knn.KNNTestCase; +import org.junit.Before; + +import org.opensearch.knn.index.VectorDataType; +import org.opensearch.knn.index.vectorvalues.KNNVectorValues; +import org.opensearch.knn.index.vectorvalues.KNNVectorValuesFactory; +import org.opensearch.knn.index.vectorvalues.TestVectorValues; +import org.opensearch.knn.quantization.enums.ScalarQuantizationType; +import org.opensearch.knn.quantization.models.quantizationOutput.QuantizationOutput; +import org.opensearch.knn.quantization.models.quantizationParams.ScalarQuantizationParams; +import org.opensearch.knn.quantization.models.quantizationState.MultiBitScalarQuantizationState; +import org.opensearch.knn.quantization.models.quantizationState.OneBitScalarQuantizationState; +import org.opensearch.knn.quantization.models.quantizationState.QuantizationState; +import java.io.IOException; +import java.util.List; + +public class QuantizationServiceTests extends KNNTestCase { + private QuantizationService quantizationService; + private KNNVectorValues knnVectorValues; + + @Before + public void setUp() throws Exception { + super.setUp(); + quantizationService = QuantizationService.getInstance(); + + // Predefined float vectors for testing + List floatVectors = List.of( + new float[] { 1.0f, 2.0f, 3.0f }, + new float[] { 4.0f, 5.0f, 6.0f }, + new float[] { 7.0f, 8.0f, 9.0f } + ); + + // Use the predefined vectors to create KNNVectorValues + knnVectorValues = KNNVectorValuesFactory.getVectorValues( + VectorDataType.FLOAT, + new TestVectorValues.PreDefinedFloatVectorValues(floatVectors) + ); + } + + public void testTrain_oneBitQuantizer_success() throws IOException { + ScalarQuantizationParams oneBitParams = new ScalarQuantizationParams(ScalarQuantizationType.ONE_BIT); + QuantizationState quantizationState = quantizationService.train(oneBitParams, knnVectorValues); + + assertTrue(quantizationState instanceof OneBitScalarQuantizationState); + OneBitScalarQuantizationState oneBitState = (OneBitScalarQuantizationState) quantizationState; + + // Validate the mean thresholds obtained from the training + float[] thresholds = oneBitState.getMeanThresholds(); + assertNotNull("Thresholds should not be null", thresholds); + assertEquals("Thresholds array length should match the dimension", 3, thresholds.length); + + // Example expected thresholds based on the provided vectors + assertArrayEquals(new float[] { 4.0f, 5.0f, 6.0f }, thresholds, 0.1f); + } + + public void testTrain_twoBitQuantizer_success() throws IOException { + ScalarQuantizationParams twoBitParams = new ScalarQuantizationParams(ScalarQuantizationType.TWO_BIT); + QuantizationState quantizationState = quantizationService.train(twoBitParams, knnVectorValues); + + assertTrue(quantizationState instanceof MultiBitScalarQuantizationState); + MultiBitScalarQuantizationState multiBitState = (MultiBitScalarQuantizationState) quantizationState; + + // Validate the thresholds obtained from the training + float[][] thresholds = multiBitState.getThresholds(); + assertNotNull("Thresholds should not be null", thresholds); + assertEquals("Number of bits should match the number of rows", 2, thresholds.length); + assertEquals("Thresholds array length should match the dimension", 3, thresholds[0].length); + + // // Example expected thresholds for two-bit quantization + float[][] expectedThresholds = { + { 3.1835034f, 4.1835036f, 5.1835036f }, // First bit level + { 4.816497f, 5.816497f, 6.816497f } // Second bit level + }; + for (int i = 0; i < thresholds.length; i++) { + assertArrayEquals(expectedThresholds[i], thresholds[i], 0.1f); + } + } + + public void testTrain_fourBitQuantizer_success() throws IOException { + ScalarQuantizationParams fourBitParams = new ScalarQuantizationParams(ScalarQuantizationType.FOUR_BIT); + QuantizationState quantizationState = quantizationService.train(fourBitParams, knnVectorValues); + + assertTrue(quantizationState instanceof MultiBitScalarQuantizationState); + MultiBitScalarQuantizationState multiBitState = (MultiBitScalarQuantizationState) quantizationState; + + // Validate the thresholds obtained from the training + float[][] thresholds = multiBitState.getThresholds(); + assertNotNull("Thresholds should not be null", thresholds); + assertEquals("Number of bits should match the number of rows", 4, thresholds.length); + assertEquals("Thresholds array length should match the dimension", 3, thresholds[0].length); + + // // Example expected thresholds for four-bit quantization + float[][] expectedThresholds = { + { 2.530306f, 3.530306f, 4.530306f }, // First bit level + { 3.510102f, 4.5101023f, 5.5101023f }, // Second bit level + { 4.489898f, 5.489898f, 6.489898f }, // Third bit level + { 5.469694f, 6.469694f, 7.469694f } // Fourth bit level + }; + for (int i = 0; i < thresholds.length; i++) { + assertArrayEquals(expectedThresholds[i], thresholds[i], 0.1f); + } + } + + public void testQuantize_oneBitQuantizer_success() throws IOException { + ScalarQuantizationParams oneBitParams = new ScalarQuantizationParams(ScalarQuantizationType.ONE_BIT); + QuantizationState quantizationState = quantizationService.train(oneBitParams, knnVectorValues); + + QuantizationOutput quantizationOutput = quantizationService.createQuantizationOutput(oneBitParams); + + byte[] quantizedVector = quantizationService.quantize(quantizationState, new float[] { 1.0f, 2.0f, 3.0f }, quantizationOutput); + + assertNotNull("Quantized vector should not be null", quantizedVector); + + // Expected quantized vector values for one-bit quantization (packed bits) + byte[] expectedQuantizedVector = new byte[] { 0 }; // 00000000 (all bits are 0) + assertArrayEquals(expectedQuantizedVector, quantizedVector); + } + + public void testQuantize_twoBitQuantizer_success() throws IOException { + ScalarQuantizationParams twoBitParams = new ScalarQuantizationParams(ScalarQuantizationType.TWO_BIT); + QuantizationState quantizationState = quantizationService.train(twoBitParams, knnVectorValues); + QuantizationOutput quantizationOutput = quantizationService.createQuantizationOutput(twoBitParams); + byte[] quantizedVector = quantizationService.quantize(quantizationState, new float[] { 4.0f, 5.0f, 6.0f }, quantizationOutput); + + assertNotNull("Quantized vector should not be null", quantizedVector); + + // Expected quantized vector values for two-bit quantization (packed bits) + byte[] expectedQuantizedVector = new byte[] { (byte) 0b11100000 }; + assertArrayEquals(expectedQuantizedVector, quantizedVector); + } + + public void testQuantize_fourBitQuantizer_success() throws IOException { + ScalarQuantizationParams fourBitParams = new ScalarQuantizationParams(ScalarQuantizationType.FOUR_BIT); + QuantizationState quantizationState = quantizationService.train(fourBitParams, knnVectorValues); + QuantizationOutput quantizationOutput = quantizationService.createQuantizationOutput(fourBitParams); + + byte[] quantizedVector = quantizationService.quantize(quantizationState, new float[] { 7.0f, 8.0f, 9.0f }, quantizationOutput); + + assertNotNull("Quantized vector should not be null", quantizedVector); + + // Expected quantized vector values for four-bit quantization (packed bits) + byte[] expectedQuantizedVector = new byte[] { (byte) 0xFF, (byte) 0xF0 }; + assertArrayEquals(expectedQuantizedVector, quantizedVector); + } + + public void testQuantize_whenInvalidInput_thenThrows() throws IOException { + ScalarQuantizationParams oneBitParams = new ScalarQuantizationParams(ScalarQuantizationType.ONE_BIT); + QuantizationState quantizationState = quantizationService.train(oneBitParams, knnVectorValues); + QuantizationOutput quantizationOutput = quantizationService.createQuantizationOutput(oneBitParams); + assertThrows(IllegalArgumentException.class, () -> quantizationService.quantize(quantizationState, null, quantizationOutput)); + } +} diff --git a/src/test/java/org/opensearch/knn/integ/QFrameworkIT.java b/src/test/java/org/opensearch/knn/integ/QFrameworkIT.java index e3f8b607a..38371d8c3 100644 --- a/src/test/java/org/opensearch/knn/integ/QFrameworkIT.java +++ b/src/test/java/org/opensearch/knn/integ/QFrameworkIT.java @@ -5,7 +5,6 @@ package org.opensearch.knn.integ; -import org.opensearch.client.Response; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.knn.KNNRestTestCase; @@ -29,23 +28,24 @@ public class QFrameworkIT extends KNNRestTestCase { public void testBaseCase() throws IOException { createTestIndex(4); - addKnnDoc(INDEX_NAME, "1", FIELD_NAME, TEST_VECTOR); - Response response = searchKNNIndex( - INDEX_NAME, - XContentFactory.jsonBuilder() - .startObject() - .startObject("query") - .startObject("knn") - .startObject(FIELD_NAME) - .field("vector", TEST_VECTOR) - .field("k", K) - .endObject() - .endObject() - .endObject() - .endObject(), - 1 - ); - assertOK(response); + // TODO :- UnComment this once Search is Integrated and KNN_USE_LUCENE_VECTOR_FORMAT_ENABLED_SETTING is enabled + // addKnnDoc(INDEX_NAME, "1", FIELD_NAME, TEST_VECTOR); + // Response response = searchKNNIndex( + // INDEX_NAME, + // XContentFactory.jsonBuilder() + // .startObject() + // .startObject("query") + // .startObject("knn") + // .startObject(FIELD_NAME) + // .field("vector", TEST_VECTOR) + // .field("k", K) + // .endObject() + // .endObject() + // .endObject() + // .endObject(), + // 1 + // ); + // assertOK(response); } private void createTestIndex(int bitCount) throws IOException { diff --git a/src/test/java/org/opensearch/knn/quantization/factory/QuantizerFactoryTests.java b/src/test/java/org/opensearch/knn/quantization/factory/QuantizerFactoryTests.java index b95123e21..f6974aea2 100644 --- a/src/test/java/org/opensearch/knn/quantization/factory/QuantizerFactoryTests.java +++ b/src/test/java/org/opensearch/knn/quantization/factory/QuantizerFactoryTests.java @@ -5,7 +5,6 @@ package org.opensearch.knn.quantization.factory; -import org.junit.Before; import org.opensearch.knn.KNNTestCase; import org.opensearch.knn.quantization.enums.ScalarQuantizationType; import org.opensearch.knn.quantization.models.quantizationParams.ScalarQuantizationParams; @@ -13,31 +12,22 @@ import org.opensearch.knn.quantization.quantizer.OneBitScalarQuantizer; import org.opensearch.knn.quantization.quantizer.Quantizer; -import java.lang.reflect.Field; -import java.util.concurrent.atomic.AtomicBoolean; - public class QuantizerFactoryTests extends KNNTestCase { - @Before - public void resetIsRegisteredFlag() throws NoSuchFieldException, IllegalAccessException { - Field isRegisteredField = QuantizerFactory.class.getDeclaredField("isRegistered"); - isRegisteredField.setAccessible(true); - AtomicBoolean isRegistered = (AtomicBoolean) isRegisteredField.get(null); - isRegistered.set(false); - } - public void test_Lazy_Registration() { - ScalarQuantizationParams params = new ScalarQuantizationParams(ScalarQuantizationType.ONE_BIT); - ScalarQuantizationParams paramsTwoBit = new ScalarQuantizationParams(ScalarQuantizationType.TWO_BIT); - ScalarQuantizationParams paramsFourBit = new ScalarQuantizationParams(ScalarQuantizationType.FOUR_BIT); - assertFalse(isRegisteredFieldAccessible()); - Quantizer oneBitQuantizer = QuantizerFactory.getQuantizer(params); - Quantizer quantizerTwoBit = QuantizerFactory.getQuantizer(paramsTwoBit); - Quantizer quantizerFourBit = QuantizerFactory.getQuantizer(paramsFourBit); - assertEquals(quantizerFourBit.getClass(), MultiBitScalarQuantizer.class); - assertEquals(quantizerTwoBit.getClass(), MultiBitScalarQuantizer.class); - assertEquals(oneBitQuantizer.getClass(), OneBitScalarQuantizer.class); - assertTrue(isRegisteredFieldAccessible()); + try { + ScalarQuantizationParams params = new ScalarQuantizationParams(ScalarQuantizationType.ONE_BIT); + ScalarQuantizationParams paramsTwoBit = new ScalarQuantizationParams(ScalarQuantizationType.TWO_BIT); + ScalarQuantizationParams paramsFourBit = new ScalarQuantizationParams(ScalarQuantizationType.FOUR_BIT); + Quantizer oneBitQuantizer = QuantizerFactory.getQuantizer(params); + Quantizer quantizerTwoBit = QuantizerFactory.getQuantizer(paramsTwoBit); + Quantizer quantizerFourBit = QuantizerFactory.getQuantizer(paramsFourBit); + assertEquals(OneBitScalarQuantizer.class, oneBitQuantizer.getClass()); + assertEquals(MultiBitScalarQuantizer.class, quantizerTwoBit.getClass()); + assertEquals(MultiBitScalarQuantizer.class, quantizerFourBit.getClass()); + } catch (Exception e) { + assertTrue(e.getMessage().contains("already registered")); + } } public void testGetQuantizer_withNullParams() { @@ -48,16 +38,4 @@ public void testGetQuantizer_withNullParams() { assertEquals("Quantization parameters must not be null.", e.getMessage()); } } - - private boolean isRegisteredFieldAccessible() { - try { - Field isRegisteredField = QuantizerFactory.class.getDeclaredField("isRegistered"); - isRegisteredField.setAccessible(true); - AtomicBoolean isRegistered = (AtomicBoolean) isRegisteredField.get(null); - return isRegistered.get(); - } catch (NoSuchFieldException | IllegalAccessException e) { - fail("Failed to access isRegistered field."); - return false; - } - } } diff --git a/src/test/java/org/opensearch/knn/quantization/factory/QuantizerRegistryTests.java b/src/test/java/org/opensearch/knn/quantization/factory/QuantizerRegistryTests.java index 62d31ab61..7c974e517 100644 --- a/src/test/java/org/opensearch/knn/quantization/factory/QuantizerRegistryTests.java +++ b/src/test/java/org/opensearch/knn/quantization/factory/QuantizerRegistryTests.java @@ -17,18 +17,22 @@ public class QuantizerRegistryTests extends KNNTestCase { @BeforeClass public static void setup() { - QuantizerRegistry.register( - ScalarQuantizationParams.generateTypeIdentifier(ScalarQuantizationType.ONE_BIT), - new OneBitScalarQuantizer() - ); - QuantizerRegistry.register( - ScalarQuantizationParams.generateTypeIdentifier(ScalarQuantizationType.TWO_BIT), - new MultiBitScalarQuantizer(2) - ); - QuantizerRegistry.register( - ScalarQuantizationParams.generateTypeIdentifier(ScalarQuantizationType.FOUR_BIT), - new MultiBitScalarQuantizer(4) - ); + try { + QuantizerRegistry.register( + ScalarQuantizationParams.generateTypeIdentifier(ScalarQuantizationType.ONE_BIT), + new OneBitScalarQuantizer() + ); + QuantizerRegistry.register( + ScalarQuantizationParams.generateTypeIdentifier(ScalarQuantizationType.TWO_BIT), + new MultiBitScalarQuantizer(2) + ); + QuantizerRegistry.register( + ScalarQuantizationParams.generateTypeIdentifier(ScalarQuantizationType.FOUR_BIT), + new MultiBitScalarQuantizer(4) + ); + } catch (Exception e) { + assertTrue(e.getMessage().contains("already registered")); + } } public void testRegisterAndGetQuantizer() { diff --git a/src/test/java/org/opensearch/knn/quantization/quantizationState/QuantizationStateTests.java b/src/test/java/org/opensearch/knn/quantization/quantizationState/QuantizationStateTests.java index 35edf49e2..298256127 100644 --- a/src/test/java/org/opensearch/knn/quantization/quantizationState/QuantizationStateTests.java +++ b/src/test/java/org/opensearch/knn/quantization/quantizationState/QuantizationStateTests.java @@ -5,6 +5,7 @@ package org.opensearch.knn.quantization.quantizationState; +import org.apache.lucene.util.RamUsageEstimator; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.knn.KNNTestCase; import org.opensearch.knn.quantization.enums.ScalarQuantizationType; @@ -65,4 +66,76 @@ public void testSerializationWithDifferentVersions() throws IOException { assertArrayEquals(mean, deserializedState.getMeanThresholds(), delta); assertEquals(params.getSqType(), deserializedState.getQuantizationParams().getSqType()); } + + public void testOneBitScalarQuantizationStateRamBytesUsed() throws IOException { + ScalarQuantizationParams params = new ScalarQuantizationParams(ScalarQuantizationType.ONE_BIT); + float[] mean = { 1.0f, 2.0f, 3.0f }; + + OneBitScalarQuantizationState state = new OneBitScalarQuantizationState(params, mean); + + // 1. Manual Calculation of RAM Usage + long manualEstimatedRamBytesUsed = 0L; + + // OneBitScalarQuantizationState object overhead for Object Header + manualEstimatedRamBytesUsed += alignSize(16L); + + // ScalarQuantizationParams object overhead Object Header + manualEstimatedRamBytesUsed += alignSize(16L); + + // Mean array overhead (array header + size of elements) + manualEstimatedRamBytesUsed += alignSize(16L + 4L * mean.length); + + // 3. RAM Usage from RamUsageEstimator + long expectedRamBytesUsed = RamUsageEstimator.shallowSizeOfInstance(OneBitScalarQuantizationState.class) + RamUsageEstimator + .shallowSizeOf(params) + RamUsageEstimator.sizeOf(mean); + + long actualRamBytesUsed = state.ramBytesUsed(); + + // Allow a difference between manual estimation, serialization size, and actual RAM usage + assertTrue( + "The difference between manual and actual RAM usage exceeds 8 bytes", + Math.abs(manualEstimatedRamBytesUsed - actualRamBytesUsed) <= 8 + ); + + assertEquals(expectedRamBytesUsed, actualRamBytesUsed); + } + + public void testMultiBitScalarQuantizationStateRamBytesUsedManualCalculation() throws IOException { + ScalarQuantizationParams params = new ScalarQuantizationParams(ScalarQuantizationType.TWO_BIT); + float[][] thresholds = { { 0.5f, 1.5f, 2.5f }, { 1.0f, 2.0f, 3.0f } }; + + MultiBitScalarQuantizationState state = new MultiBitScalarQuantizationState(params, thresholds); + + // Manually estimate RAM usage with alignment + long manualEstimatedRamBytesUsed = 0L; + + // Estimate for MultiBitScalarQuantizationState object + manualEstimatedRamBytesUsed += alignSize(16L); // Example overhead for object + + // Estimate for ScalarQuantizationParams object + manualEstimatedRamBytesUsed += alignSize(16L); // Overhead for params object (including fields) + + // Estimate for thresholds array + manualEstimatedRamBytesUsed += alignSize(16L + 4L * thresholds.length); // Overhead for array + references to sub-arrays + + for (float[] row : thresholds) { + manualEstimatedRamBytesUsed += alignSize(16L + 4L * row.length); // Overhead for each sub-array + size of each float + } + + long ramEstimatorRamBytesUsed = RamUsageEstimator.shallowSizeOfInstance(MultiBitScalarQuantizationState.class) + RamUsageEstimator + .shallowSizeOf(params) + RamUsageEstimator.shallowSizeOf(thresholds); + + for (float[] row : thresholds) { + ramEstimatorRamBytesUsed += RamUsageEstimator.sizeOf(row); + } + + long difference = Math.abs(manualEstimatedRamBytesUsed - ramEstimatorRamBytesUsed); + assertTrue("The difference between manual and actual RAM usage exceeds 8 bytes", difference <= 8); + assertEquals(ramEstimatorRamBytesUsed, state.ramBytesUsed()); + } + + private long alignSize(long size) { + return (size + 7) & ~7; // Align to 8 bytes boundary + } + } diff --git a/src/test/java/org/opensearch/knn/quantization/quantizer/MultiBitScalarQuantizerTests.java b/src/test/java/org/opensearch/knn/quantization/quantizer/MultiBitScalarQuantizerTests.java index 45acaf357..de815d8ad 100644 --- a/src/test/java/org/opensearch/knn/quantization/quantizer/MultiBitScalarQuantizerTests.java +++ b/src/test/java/org/opensearch/knn/quantization/quantizer/MultiBitScalarQuantizerTests.java @@ -17,7 +17,7 @@ public class MultiBitScalarQuantizerTests extends KNNTestCase { - public void testTrain_twoBit() { + public void testTrain_twoBit() throws IOException { float[][] vectors = { { 0.5f, 1.5f, 2.5f, 3.5f, 4.5f, 5.5f, 6.5f, 7.5f }, { 1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f, 7.0f, 8.0f }, @@ -33,7 +33,7 @@ public void testTrain_twoBit() { assertEquals(2, mbState.getThresholds().length); // 2-bit quantization should have 2 thresholds } - public void testTrain_fourBit() { + public void testTrain_fourBit() throws IOException { MultiBitScalarQuantizer fourBitQuantizer = new MultiBitScalarQuantizer(4); float[][] vectors = { { 0.5f, 1.5f, 2.5f, 3.5f, 4.5f, 5.5f, 6.5f, 7.5f }, @@ -220,8 +220,8 @@ public MockTrainingRequest(ScalarQuantizationParams params, float[][] vectors) { } @Override - public float[] getVectorByDocId(int docId) { - return vectors[docId]; + public float[] getVectorAtThePosition(int position) { + return vectors[position]; } } } diff --git a/src/test/java/org/opensearch/knn/quantization/quantizer/OneBitScalarQuantizerTests.java b/src/test/java/org/opensearch/knn/quantization/quantizer/OneBitScalarQuantizerTests.java index 6f8c2de87..a6b907ccb 100644 --- a/src/test/java/org/opensearch/knn/quantization/quantizer/OneBitScalarQuantizerTests.java +++ b/src/test/java/org/opensearch/knn/quantization/quantizer/OneBitScalarQuantizerTests.java @@ -22,14 +22,14 @@ public class OneBitScalarQuantizerTests extends KNNTestCase { - public void testTrain_withTrainingRequired() { + public void testTrain_withTrainingRequired() throws IOException { float[][] vectors = { { 1.0f, 2.0f, 3.0f }, { 4.0f, 5.0f, 6.0f }, { 7.0f, 8.0f, 9.0f } }; ScalarQuantizationParams params = new ScalarQuantizationParams(ScalarQuantizationType.ONE_BIT); TrainingRequest originalRequest = new TrainingRequest(vectors.length) { @Override - public float[] getVectorByDocId(int docId) { - return vectors[docId]; + public float[] getVectorAtThePosition(int position) { + return vectors[position]; } }; OneBitScalarQuantizer quantizer = new OneBitScalarQuantizer(); @@ -77,6 +77,21 @@ public ScalarQuantizationParams getQuantizationParams() { return new ScalarQuantizationParams(ScalarQuantizationType.ONE_BIT); } + @Override + public int getBytesPerVector() { + return 0; + } + + @Override + public int getDimensions() { + return 0; + } + + @Override + public long ramBytesUsed() { + return 0; + } + @Override public byte[] toByteArray() { return new byte[0]; @@ -103,14 +118,14 @@ public void testQuantize_withMismatchedDimensions() throws IOException { expectThrows(IllegalArgumentException.class, () -> quantizer.quantize(vector, state, output)); } - public void testCalculateMean() { + public void testCalculateMean() throws IOException { float[][] vectors = { { 1.0f, 2.0f, 3.0f }, { 4.0f, 5.0f, 6.0f }, { 7.0f, 8.0f, 9.0f } }; ScalarQuantizationParams params = new ScalarQuantizationParams(ScalarQuantizationType.ONE_BIT); TrainingRequest samplingRequest = new TrainingRequest(vectors.length) { @Override - public float[] getVectorByDocId(int docId) { - return vectors[docId]; + public float[] getVectorAtThePosition(int position) { + return vectors[position]; } }; @@ -126,8 +141,8 @@ public void testCalculateMean_withNullVector() { ScalarQuantizationParams params = new ScalarQuantizationParams(ScalarQuantizationType.ONE_BIT); TrainingRequest samplingRequest = new TrainingRequest(vectors.length) { @Override - public float[] getVectorByDocId(int docId) { - return vectors[docId]; + public float[] getVectorAtThePosition(int position) { + return vectors[position]; } }; From e751cad2bac83c9aeadcba9626e3d9a45631617e Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Wed, 28 Aug 2024 20:18:01 -0700 Subject: [PATCH 340/416] Align dimensions to the nearest multiple of 8 in QuantizationState (#2010) (#2012) * Align dimensions to the nearest multiple of 8 in QuantizationState Signed-off-by: VIKASH TIWARI * Update src/test/java/org/opensearch/knn/quantization/quantizationState/QuantizationStateTests.java Co-authored-by: Navneet Verma Signed-off-by: Vikasht34 * Update QuantizationStateTests.java Signed-off-by: Vikasht34 --------- Signed-off-by: VIKASH TIWARI Signed-off-by: Vikasht34 Co-authored-by: Navneet Verma (cherry picked from commit f089b5bcc5402950c680faab2b2d523fe6dd8213) Co-authored-by: Vikasht34 --- .../NativeEngines990KnnVectorsWriter.java | 2 +- .../codec/nativeindex/NativeIndexWriter.java | 2 +- .../nativeindex/QuantizationIndexUtils.java | 2 +- .../KNNVectorQuantizationTrainingRequest.java | 4 +- .../QuantizationService.java | 2 +- .../MultiBitScalarQuantizationState.java | 9 ++- .../OneBitScalarQuantizationState.java | 3 +- .../DefaultIndexBuildStrategyTests.java | 2 +- ...ptimizedNativeIndexBuildStrategyTests.java | 2 +- .../QuantizationIndexUtilsTests.java | 2 +- .../QuantizationServiceTests.java | 2 +- .../QuantizationStateTests.java | 72 +++++++++++++++++++ 12 files changed, 92 insertions(+), 12 deletions(-) rename src/main/java/org/opensearch/knn/index/{quantizationService => quantizationservice}/KNNVectorQuantizationTrainingRequest.java (94%) rename src/main/java/org/opensearch/knn/index/{quantizationService => quantizationservice}/QuantizationService.java (99%) rename src/test/java/org/opensearch/knn/index/{quantizationService => quantizationservice}/QuantizationServiceTests.java (99%) diff --git a/src/main/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsWriter.java b/src/main/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsWriter.java index 43f4d7ad6..2c110fb79 100644 --- a/src/main/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsWriter.java +++ b/src/main/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsWriter.java @@ -24,7 +24,7 @@ import org.apache.lucene.index.Sorter; import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.RamUsageEstimator; -import org.opensearch.knn.index.quantizationService.QuantizationService; +import org.opensearch.knn.index.quantizationservice.QuantizationService; import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.codec.nativeindex.NativeIndexWriter; import org.opensearch.knn.index.vectorvalues.KNNVectorValues; diff --git a/src/main/java/org/opensearch/knn/index/codec/nativeindex/NativeIndexWriter.java b/src/main/java/org/opensearch/knn/index/codec/nativeindex/NativeIndexWriter.java index ed0e8149a..886c6d93d 100644 --- a/src/main/java/org/opensearch/knn/index/codec/nativeindex/NativeIndexWriter.java +++ b/src/main/java/org/opensearch/knn/index/codec/nativeindex/NativeIndexWriter.java @@ -24,7 +24,7 @@ import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.codec.nativeindex.model.BuildIndexParams; import org.opensearch.knn.index.engine.KNNEngine; -import org.opensearch.knn.index.quantizationService.QuantizationService; +import org.opensearch.knn.index.quantizationservice.QuantizationService; import org.opensearch.knn.index.util.IndexUtil; import org.opensearch.knn.index.vectorvalues.KNNVectorValues; import org.opensearch.knn.indices.Model; diff --git a/src/main/java/org/opensearch/knn/index/codec/nativeindex/QuantizationIndexUtils.java b/src/main/java/org/opensearch/knn/index/codec/nativeindex/QuantizationIndexUtils.java index 8fec1af6d..bebe9e8b0 100644 --- a/src/main/java/org/opensearch/knn/index/codec/nativeindex/QuantizationIndexUtils.java +++ b/src/main/java/org/opensearch/knn/index/codec/nativeindex/QuantizationIndexUtils.java @@ -7,7 +7,7 @@ import lombok.experimental.UtilityClass; import org.opensearch.knn.index.codec.nativeindex.model.BuildIndexParams; -import org.opensearch.knn.index.quantizationService.QuantizationService; +import org.opensearch.knn.index.quantizationservice.QuantizationService; import org.opensearch.knn.index.vectorvalues.KNNVectorValues; import org.opensearch.knn.quantization.models.quantizationOutput.QuantizationOutput; import org.opensearch.knn.quantization.models.quantizationState.QuantizationState; diff --git a/src/main/java/org/opensearch/knn/index/quantizationService/KNNVectorQuantizationTrainingRequest.java b/src/main/java/org/opensearch/knn/index/quantizationservice/KNNVectorQuantizationTrainingRequest.java similarity index 94% rename from src/main/java/org/opensearch/knn/index/quantizationService/KNNVectorQuantizationTrainingRequest.java rename to src/main/java/org/opensearch/knn/index/quantizationservice/KNNVectorQuantizationTrainingRequest.java index 4cf68d16c..d880a4178 100644 --- a/src/main/java/org/opensearch/knn/index/quantizationService/KNNVectorQuantizationTrainingRequest.java +++ b/src/main/java/org/opensearch/knn/index/quantizationservice/KNNVectorQuantizationTrainingRequest.java @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.knn.index.quantizationService; +package org.opensearch.knn.index.quantizationservice; import lombok.extern.log4j.Log4j2; import org.opensearch.knn.index.vectorvalues.KNNVectorValues; @@ -49,7 +49,7 @@ public T getVectorAtThePosition(int position) throws IOException { } knnVectorValues.nextDoc(); } - // Return the vector and the updated index + // Return the vector return knnVectorValues.getVector(); } } diff --git a/src/main/java/org/opensearch/knn/index/quantizationService/QuantizationService.java b/src/main/java/org/opensearch/knn/index/quantizationservice/QuantizationService.java similarity index 99% rename from src/main/java/org/opensearch/knn/index/quantizationService/QuantizationService.java rename to src/main/java/org/opensearch/knn/index/quantizationservice/QuantizationService.java index a9e3cc715..b1c94f993 100644 --- a/src/main/java/org/opensearch/knn/index/quantizationService/QuantizationService.java +++ b/src/main/java/org/opensearch/knn/index/quantizationservice/QuantizationService.java @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.knn.index.quantizationService; +package org.opensearch.knn.index.quantizationservice; import lombok.AccessLevel; import lombok.NoArgsConstructor; diff --git a/src/main/java/org/opensearch/knn/quantization/models/quantizationState/MultiBitScalarQuantizationState.java b/src/main/java/org/opensearch/knn/quantization/models/quantizationState/MultiBitScalarQuantizationState.java index 79ce7b955..dbef3a72a 100644 --- a/src/main/java/org/opensearch/knn/quantization/models/quantizationState/MultiBitScalarQuantizationState.java +++ b/src/main/java/org/opensearch/knn/quantization/models/quantizationState/MultiBitScalarQuantizationState.java @@ -150,7 +150,14 @@ public int getDimensions() { if (thresholds == null || thresholds.length == 0 || thresholds[0] == null) { throw new IllegalStateException("Error in getting Dimension: The thresholds array is not initialized."); } - return thresholds.length * thresholds[0].length; + int originalDimensions = thresholds[0].length; + + // Align the original dimensions to the next multiple of 8 for each bit level + int alignedDimensions = (originalDimensions + 7) & ~7; + + // The final dimension count should consider the bit levels + return thresholds.length * alignedDimensions; + } /** diff --git a/src/main/java/org/opensearch/knn/quantization/models/quantizationState/OneBitScalarQuantizationState.java b/src/main/java/org/opensearch/knn/quantization/models/quantizationState/OneBitScalarQuantizationState.java index 9c4ff7460..0a8c33771 100644 --- a/src/main/java/org/opensearch/knn/quantization/models/quantizationState/OneBitScalarQuantizationState.java +++ b/src/main/java/org/opensearch/knn/quantization/models/quantizationState/OneBitScalarQuantizationState.java @@ -123,7 +123,8 @@ public int getBytesPerVector() { @Override public int getDimensions() { // For one-bit quantization, the dimension for indexing is just the length of the thresholds array. - return meanThresholds.length; + // Align the original dimensions to the next multiple of 8 + return (meanThresholds.length + 7) & ~7; } /** diff --git a/src/test/java/org/opensearch/knn/index/codec/nativeindex/DefaultIndexBuildStrategyTests.java b/src/test/java/org/opensearch/knn/index/codec/nativeindex/DefaultIndexBuildStrategyTests.java index 0b5a06dfc..1a8a832aa 100644 --- a/src/test/java/org/opensearch/knn/index/codec/nativeindex/DefaultIndexBuildStrategyTests.java +++ b/src/test/java/org/opensearch/knn/index/codec/nativeindex/DefaultIndexBuildStrategyTests.java @@ -17,7 +17,7 @@ import org.opensearch.knn.index.codec.transfer.OffHeapVectorTransfer; import org.opensearch.knn.index.codec.transfer.OffHeapVectorTransferFactory; import org.opensearch.knn.index.engine.KNNEngine; -import org.opensearch.knn.index.quantizationService.QuantizationService; +import org.opensearch.knn.index.quantizationservice.QuantizationService; import org.opensearch.knn.index.vectorvalues.KNNFloatVectorValues; import org.opensearch.knn.index.vectorvalues.KNNVectorValues; import org.opensearch.knn.index.vectorvalues.KNNVectorValuesFactory; diff --git a/src/test/java/org/opensearch/knn/index/codec/nativeindex/MemOptimizedNativeIndexBuildStrategyTests.java b/src/test/java/org/opensearch/knn/index/codec/nativeindex/MemOptimizedNativeIndexBuildStrategyTests.java index 3bfec4104..81d490bb4 100644 --- a/src/test/java/org/opensearch/knn/index/codec/nativeindex/MemOptimizedNativeIndexBuildStrategyTests.java +++ b/src/test/java/org/opensearch/knn/index/codec/nativeindex/MemOptimizedNativeIndexBuildStrategyTests.java @@ -16,7 +16,7 @@ import org.opensearch.knn.index.codec.transfer.OffHeapVectorTransfer; import org.opensearch.knn.index.codec.transfer.OffHeapVectorTransferFactory; import org.opensearch.knn.index.engine.KNNEngine; -import org.opensearch.knn.index.quantizationService.QuantizationService; +import org.opensearch.knn.index.quantizationservice.QuantizationService; import org.opensearch.knn.index.vectorvalues.KNNVectorValues; import org.opensearch.knn.index.vectorvalues.KNNVectorValuesFactory; import org.opensearch.knn.index.vectorvalues.TestVectorValues; diff --git a/src/test/java/org/opensearch/knn/index/codec/nativeindex/QuantizationIndexUtilsTests.java b/src/test/java/org/opensearch/knn/index/codec/nativeindex/QuantizationIndexUtilsTests.java index 30a2098dd..61d3d7589 100644 --- a/src/test/java/org/opensearch/knn/index/codec/nativeindex/QuantizationIndexUtilsTests.java +++ b/src/test/java/org/opensearch/knn/index/codec/nativeindex/QuantizationIndexUtilsTests.java @@ -9,7 +9,7 @@ import org.opensearch.knn.KNNTestCase; import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.codec.nativeindex.model.BuildIndexParams; -import org.opensearch.knn.index.quantizationService.QuantizationService; +import org.opensearch.knn.index.quantizationservice.QuantizationService; import org.opensearch.knn.index.vectorvalues.KNNVectorValues; import org.opensearch.knn.index.vectorvalues.KNNVectorValuesFactory; import org.opensearch.knn.index.vectorvalues.TestVectorValues; diff --git a/src/test/java/org/opensearch/knn/index/quantizationService/QuantizationServiceTests.java b/src/test/java/org/opensearch/knn/index/quantizationservice/QuantizationServiceTests.java similarity index 99% rename from src/test/java/org/opensearch/knn/index/quantizationService/QuantizationServiceTests.java rename to src/test/java/org/opensearch/knn/index/quantizationservice/QuantizationServiceTests.java index 886dbeabc..720b67fd5 100644 --- a/src/test/java/org/opensearch/knn/index/quantizationService/QuantizationServiceTests.java +++ b/src/test/java/org/opensearch/knn/index/quantizationservice/QuantizationServiceTests.java @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.knn.index.quantizationService; +package org.opensearch.knn.index.quantizationservice; import org.opensearch.knn.KNNTestCase; import org.junit.Before; diff --git a/src/test/java/org/opensearch/knn/quantization/quantizationState/QuantizationStateTests.java b/src/test/java/org/opensearch/knn/quantization/quantizationState/QuantizationStateTests.java index 298256127..4fd4f40a6 100644 --- a/src/test/java/org/opensearch/knn/quantization/quantizationState/QuantizationStateTests.java +++ b/src/test/java/org/opensearch/knn/quantization/quantizationState/QuantizationStateTests.java @@ -100,6 +100,78 @@ public void testOneBitScalarQuantizationStateRamBytesUsed() throws IOException { assertEquals(expectedRamBytesUsed, actualRamBytesUsed); } + public void testMultiBitScalarQuantizationStateGetDimensions_withDimensionNotMultipleOf8_thenSuccess() { + ScalarQuantizationParams params = new ScalarQuantizationParams(ScalarQuantizationType.TWO_BIT); + + // Case 1: 3 thresholds, each with 2 dimensions + float[][] thresholds1 = { { 0.5f, 1.5f }, { 1.0f, 2.0f }, { 1.5f, 2.5f } }; + MultiBitScalarQuantizationState state1 = new MultiBitScalarQuantizationState(params, thresholds1); + int expectedDimensions1 = 24; // The next multiple of 8 considering all bits + assertEquals(expectedDimensions1, state1.getDimensions()); + + // Case 2: 1 threshold, with 5 dimensions (5 bits, should align to 8) + float[][] thresholds2 = { { 0.5f, 1.5f, 2.5f, 3.5f, 4.5f } }; + MultiBitScalarQuantizationState state2 = new MultiBitScalarQuantizationState(params, thresholds2); + int expectedDimensions2 = 8; // The next multiple of 8 considering all bits + assertEquals(expectedDimensions2, state2.getDimensions()); + + // Case 3: 4 thresholds, each with 7 dimensions (28 bits, should align to 32) + float[][] thresholds3 = { + { 0.5f, 1.5f, 2.5f, 3.5f, 4.5f, 5.5f, 6.5f }, + { 1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f, 7.0f }, + { 1.5f, 2.5f, 3.5f, 4.5f, 5.5f, 6.5f, 7.5f }, + { 2.0f, 3.0f, 4.0f, 5.0f, 6.0f, 7.0f, 8.0f } }; + MultiBitScalarQuantizationState state3 = new MultiBitScalarQuantizationState(params, thresholds3); + int expectedDimensions3 = 32; // The next multiple of 8 considering all bits + assertEquals(expectedDimensions3, state3.getDimensions()); + + // Case 4: 2 thresholds, each with 8 dimensions (16 bits, already aligned) + float[][] thresholds4 = { { 0.5f, 1.5f, 2.5f, 3.5f, 4.5f, 5.5f, 6.5f, 7.5f }, { 1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f, 7.0f, 8.0f } }; + MultiBitScalarQuantizationState state4 = new MultiBitScalarQuantizationState(params, thresholds4); + int expectedDimensions4 = 16; // Already aligned to 8 + assertEquals(expectedDimensions4, state4.getDimensions()); + + // Case 5: 2 thresholds, each with 6 dimensions (12 bits, should align to 16) + float[][] thresholds5 = { { 0.5f, 1.5f, 2.5f, 3.5f, 4.5f, 5.5f }, { 1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f } }; + MultiBitScalarQuantizationState state5 = new MultiBitScalarQuantizationState(params, thresholds5); + int expectedDimensions5 = 16; // The next multiple of 8 considering all bits + assertEquals(expectedDimensions5, state5.getDimensions()); + } + + public void testOneBitScalarQuantizationStateGetDimensions_withDimensionNotMultipleOf8_thenSuccess() { + ScalarQuantizationParams params = new ScalarQuantizationParams(ScalarQuantizationType.ONE_BIT); + + // Case 1: 5 dimensions (should align to 8) + float[] thresholds1 = { 0.5f, 1.5f, 2.5f, 3.5f, 4.5f }; + OneBitScalarQuantizationState state1 = new OneBitScalarQuantizationState(params, thresholds1); + int expectedDimensions1 = 8; // The next multiple of 8 + assertEquals(expectedDimensions1, state1.getDimensions()); + + // Case 2: 7 dimensions (should align to 8) + float[] thresholds2 = { 0.5f, 1.5f, 2.5f, 3.5f, 4.5f, 5.5f, 6.5f }; + OneBitScalarQuantizationState state2 = new OneBitScalarQuantizationState(params, thresholds2); + int expectedDimensions2 = 8; // The next multiple of 8 + assertEquals(expectedDimensions2, state2.getDimensions()); + + // Case 3: 8 dimensions (already aligned to 8) + float[] thresholds3 = { 0.5f, 1.5f, 2.5f, 3.5f, 4.5f, 5.5f, 6.5f, 7.5f }; + OneBitScalarQuantizationState state3 = new OneBitScalarQuantizationState(params, thresholds3); + int expectedDimensions3 = 8; // Already aligned to 8 + assertEquals(expectedDimensions3, state3.getDimensions()); + + // Case 4: 10 dimensions (should align to 16) + float[] thresholds4 = { 0.5f, 1.5f, 2.5f, 3.5f, 4.5f, 5.5f, 6.5f, 7.5f, 8.5f, 9.5f }; + OneBitScalarQuantizationState state4 = new OneBitScalarQuantizationState(params, thresholds4); + int expectedDimensions4 = 16; // The next multiple of 8 + assertEquals(expectedDimensions4, state4.getDimensions()); + + // Case 5: 16 dimensions (already aligned to 16) + float[] thresholds5 = { 0.5f, 1.5f, 2.5f, 3.5f, 4.5f, 5.5f, 6.5f, 7.5f, 8.5f, 9.5f, 10.5f, 11.5f, 12.5f, 13.5f, 14.5f, 15.5f }; + OneBitScalarQuantizationState state5 = new OneBitScalarQuantizationState(params, thresholds5); + int expectedDimensions5 = 16; // Already aligned to 16 + assertEquals(expectedDimensions5, state5.getDimensions()); + } + public void testMultiBitScalarQuantizationStateRamBytesUsedManualCalculation() throws IOException { ScalarQuantizationParams params = new ScalarQuantizationParams(ScalarQuantizationType.TWO_BIT); float[][] thresholds = { { 0.5f, 1.5f, 2.5f }, { 1.0f, 2.0f, 3.0f } }; From 3e19b9d30b9ca3b52978899cd225f350d1ccc1ee Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Wed, 28 Aug 2024 23:08:04 -0500 Subject: [PATCH 341/416] Parallelize make to reduce build time (#2006) (#2008) (cherry picked from commit bf38c2eced348655a53b80306f339661e049fadf) Co-authored-by: Naveen Tatikonda --- .github/workflows/CI.yml | 12 +++++++----- .github/workflows/test_security.yml | 2 +- CHANGELOG.md | 1 + DEVELOPER_GUIDE.md | 20 +++++++++++++++++++- build.gradle | 3 ++- 5 files changed, 30 insertions(+), 8 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 3e04f3724..932ce8022 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -76,10 +76,10 @@ jobs: if lscpu | grep -i avx2 then echo "avx2 available on system" - su `id -un 1000` -c "whoami && java -version && ./gradlew build" + su `id -un 1000` -c "whoami && java -version && ./gradlew build -Dnproc.count=`nproc`" else echo "avx2 not available on system" - su `id -un 1000` -c "whoami && java -version && ./gradlew build -Dsimd.enabled=false" + su `id -un 1000` -c "whoami && java -version && ./gradlew build -Dsimd.enabled=false -Dnproc.count=`nproc`" fi @@ -117,15 +117,16 @@ jobs: brew reinstall gcc export FC=/usr/local/Cellar/gcc/12.2.0/bin/gfortran + # TODO: Detect processor count and set the value of nproc.count - name: Run build run: | if sysctl -n machdep.cpu.features machdep.cpu.leaf7_features | grep -i AVX2 then echo "avx2 available on system" - ./gradlew build + ./gradlew build -Dnproc.count=3 else echo "avx2 not available on system" - ./gradlew build -Dsimd.enabled=false + ./gradlew build -Dsimd.enabled=false -Dnproc.count=3 fi Build-k-NN-Windows: @@ -183,6 +184,7 @@ jobs: rm .\OpenBLAS-0.3.21-x64.zip rm -r .\OpenBLAS\ + # TODO: Detect processor count and set the value of nproc.count - name: Run build run: | - ./gradlew.bat build -D'simd.enabled=false' + ./gradlew.bat build -D'simd.enabled=false' -D'nproc.count=4' diff --git a/.github/workflows/test_security.yml b/.github/workflows/test_security.yml index 6a0fe72d0..2f8df8526 100644 --- a/.github/workflows/test_security.yml +++ b/.github/workflows/test_security.yml @@ -70,4 +70,4 @@ jobs: # switching the user, as OpenSearch cluster can only be started as root/Administrator on linux-deb/linux-rpm/windows-zip. run: | chown -R 1000:1000 `pwd` - su `id -un 1000` -c "whoami && java -version && ./gradlew integTest -Dsecurity.enabled=true -Dsimd.enabled=true" + su `id -un 1000` -c "whoami && java -version && ./gradlew integTest -Dsecurity.enabled=true -Dsimd.enabled=true -Dnproc.count=`nproc`" diff --git a/CHANGELOG.md b/CHANGELOG.md index 66d9396c6..62aa4db36 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,6 +29,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), * Disallow a vector field to have an invalid character for a physical file name. [#1936](https://github.com/opensearch-project/k-NN/pull/1936) * Add script_fields context to KNNAllowlist [#1917] (https://github.com/opensearch-project/k-NN/pull/1917) ### Infrastructure +* Parallelize make to reduce build time [#2006] (https://github.com/opensearch-project/k-NN/pull/2006) ### Documentation ### Maintenance * Fix a flaky unit test:testMultiFieldsKnnIndex, which was failing due to inconsistent merge behaviors [#1924](https://github.com/opensearch-project/k-NN/pull/1924) diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index ecc97eb78..d6eb1d2f6 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -11,6 +11,7 @@ - [Build](#build) - [JNI Library](#jni-library) - [JNI Library Artifacts](#jni-library-artifacts) + - [Parallelize make](#parallelize-make) - [Enable SIMD Optimization](#enable-simd-optimization) - [Run OpenSearch k-NN](#run-opensearch-k-nn) - [Run Single-node Cluster Locally](#run-single-node-cluster-locally) @@ -210,7 +211,7 @@ To build the JNI Library manually, follow these steps: cd jni cmake . -# To build everything, including tests +# To build everything, including tests. If your computer has multiple cores you can speed it up by building in parallel using make -j 2 (or a higher number for more parallelism) make # To just build the libraries @@ -258,6 +259,23 @@ these in your environment, you can disable committing the changes to the library not committed, then the full library build process will run each time `cmake` is invoked. In a development environment, it is recommended to setup the user git configuration to avoid this cost. +### Parallelize make +When we are building the plugin for the first time, it takes some time to build the JNI libraries. We can parallelize make and speed up the build time by setting and passing +this flag to gradle, `nproc.count` if your computer has more number of cores (greater than or equal to 2). +``` +# While building OpenSearch k-NN +./gradlew build -Dnproc.count=4 + +# While running OpenSearch k-NN +./gradlew run -Dnproc.count=4 + +# When building the JNI library manually +cd jni +cmake . +# Pass the processor count with make using `-j` +make -j 4 +``` + ### Enable SIMD Optimization SIMD(Single Instruction/Multiple Data) Optimization is enabled by default on Linux and Mac which boosts the performance by enabling `AVX2` on `x86 architecture` and `NEON` on `ARM64 architecture` while building the Faiss library. But to enable SIMD, the underlying processor diff --git a/build.gradle b/build.gradle index 58226a38d..355d46320 100644 --- a/build.gradle +++ b/build.gradle @@ -18,6 +18,7 @@ buildscript { opensearch_group = "org.opensearch" isSnapshot = "true" == System.getProperty("build.snapshot", "true") simd_enabled = System.getProperty("simd.enabled", "true") + nproc_count = System.getProperty("nproc.count", "1") // This flag determines whether the CMake build system should apply a custom patch. It prevents build failures // when the cmakeJniLib task is run multiple times. If the build.lib.commit_patches is true, the CMake build // system skips applying the patch if the patches have been applied already. If build.lib.commit_patches is @@ -331,7 +332,7 @@ task cmakeJniLib(type:Exec) { task buildJniLib(type:Exec) { dependsOn cmakeJniLib workingDir 'jni' - commandLine 'make', 'opensearchknn_nmslib', 'opensearchknn_faiss', 'opensearchknn_common' + commandLine 'make', 'opensearchknn_nmslib', 'opensearchknn_faiss', 'opensearchknn_common', '-j', "${nproc_count}" } test { From 85af51c8e65cfbae08e6fec5758f308290077ea2 Mon Sep 17 00:00:00 2001 From: Tejas Shah Date: Fri, 30 Aug 2024 11:00:56 -0700 Subject: [PATCH 342/416] Changes NativeEngineKNNQuery to execute search and rescore in (#2014) (#2016) --- .gitignore | 5 + CHANGELOG.md | 3 +- .../common/featureflags/KNNFeatureFlags.java | 46 ----- .../org/opensearch/knn/index/KNNSettings.java | 15 +- .../opensearch/knn/index/query/KNNQuery.java | 6 +- .../knn/index/query/KNNQueryBuilder.java | 15 +- .../knn/index/query/KNNQueryFactory.java | 3 +- .../query/nativelib/DocAndScoreQuery.java | 1 + .../nativelib/NativeEngineKnnVectorQuery.java | 9 +- .../featureflags/KNNFeatureFlagsTests.java | 34 ---- .../knn/index/query/KNNQueryBuilderTests.java | 2 - .../KNNQueryBuilderValidParamsTests.java | 7 +- .../knn/index/query/KNNQueryFactoryTests.java | 30 +-- .../NativeEngineKNNVectorQueryIT.java | 190 ------------------ .../NativeEngineKNNVectorQueryTests.java | 24 ++- 15 files changed, 67 insertions(+), 323 deletions(-) delete mode 100644 src/main/java/org/opensearch/knn/common/featureflags/KNNFeatureFlags.java delete mode 100644 src/test/java/org/opensearch/knn/common/featureflags/KNNFeatureFlagsTests.java delete mode 100644 src/test/java/org/opensearch/knn/index/query/nativelib/NativeEngineKNNVectorQueryIT.java diff --git a/.gitignore b/.gitignore index 7ff764056..34f7ff154 100644 --- a/.gitignore +++ b/.gitignore @@ -29,6 +29,11 @@ jni/lib/ jni/jni_test* jni/googletest* jni/cmake/*.cmake-e +jni/.cmake +jni/.idea +jni/build.ninja +jni/.ninja_deps +jni/.ninja_log benchmarks/perf-tool/okpt/output benchmarks/perf-tool/okpt/dev diff --git a/CHANGELOG.md b/CHANGELOG.md index 62aa4db36..b030333e7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -41,8 +41,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), * Refactor method structure and definitions [#1920](https://github.com/opensearch-project/k-NN/pull/1920) * Refactor KNNVectorFieldType from KNNVectorFieldMapper to a separate class for better readability. [#1931](https://github.com/opensearch-project/k-NN/pull/1931) * Generalize lib interface to return context objects [#1925](https://github.com/opensearch-project/k-NN/pull/1925) -* Move k search k-NN query to re-write phase of vector search query for Native Engines [#1877](https://github.com/opensearch-project/k-NN/pull/1877) * Restructure mappers to better handle null cases and avoid branching in parsing [#1939](https://github.com/opensearch-project/k-NN/pull/1939) * Added Quantization Framework and implemented 1Bit and multibit quantizer[#1889](https://github.com/opensearch-project/k-NN/issues/1889) * Encapsulate dimension, vector data type validation/processing inside Library [#1957](https://github.com/opensearch-project/k-NN/pull/1957) -* Add quantization state cache [#1960](https://github.com/opensearch-project/k-NN/pull/1960) +* Add quantization state cache [#1960](https://github.com/opensearch-project/k-NN/pull/1960) \ No newline at end of file diff --git a/src/main/java/org/opensearch/knn/common/featureflags/KNNFeatureFlags.java b/src/main/java/org/opensearch/knn/common/featureflags/KNNFeatureFlags.java deleted file mode 100644 index 9b4a5ba7e..000000000 --- a/src/main/java/org/opensearch/knn/common/featureflags/KNNFeatureFlags.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - * - */ - -package org.opensearch.knn.common.featureflags; - -import com.google.common.annotations.VisibleForTesting; -import lombok.experimental.UtilityClass; -import org.opensearch.common.settings.Setting; -import org.opensearch.knn.index.KNNSettings; - -import java.util.List; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -import static org.opensearch.common.settings.Setting.Property.Dynamic; -import static org.opensearch.common.settings.Setting.Property.NodeScope; - -/** - * Class to manage KNN feature flags - */ -@UtilityClass -public class KNNFeatureFlags { - - // Feature flags - private static final String KNN_LAUNCH_QUERY_REWRITE_ENABLED = "knn.feature.query.rewrite.enabled"; - private static final boolean KNN_LAUNCH_QUERY_REWRITE_ENABLED_DEFAULT = false; - - @VisibleForTesting - public static final Setting KNN_LAUNCH_QUERY_REWRITE_ENABLED_SETTING = Setting.boolSetting( - KNN_LAUNCH_QUERY_REWRITE_ENABLED, - KNN_LAUNCH_QUERY_REWRITE_ENABLED_DEFAULT, - NodeScope, - Dynamic - ); - - public static List> getFeatureFlags() { - return Stream.of(KNN_LAUNCH_QUERY_REWRITE_ENABLED_SETTING).collect(Collectors.toUnmodifiableList()); - } - - public static boolean isKnnQueryRewriteEnabled() { - return Boolean.parseBoolean(KNNSettings.state().getSettingValue(KNN_LAUNCH_QUERY_REWRITE_ENABLED).toString()); - } -} diff --git a/src/main/java/org/opensearch/knn/index/KNNSettings.java b/src/main/java/org/opensearch/knn/index/KNNSettings.java index 73f43d3d1..0932e3c96 100644 --- a/src/main/java/org/opensearch/knn/index/KNNSettings.java +++ b/src/main/java/org/opensearch/knn/index/KNNSettings.java @@ -34,17 +34,14 @@ import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.Stream; -import static java.util.stream.Collectors.toUnmodifiableMap; import static org.opensearch.common.settings.Setting.Property.Dynamic; import static org.opensearch.common.settings.Setting.Property.IndexScope; import static org.opensearch.common.settings.Setting.Property.NodeScope; import static org.opensearch.common.unit.MemorySizeValue.parseBytesSizeValueOrHeapRatio; import static org.opensearch.core.common.unit.ByteSizeValue.parseBytesSizeValue; -import static org.opensearch.knn.common.featureflags.KNNFeatureFlags.getFeatureFlags; /** * This class defines @@ -354,9 +351,6 @@ public class KNNSettings { } }; - private final static Map> FEATURE_FLAGS = getFeatureFlags().stream() - .collect(toUnmodifiableMap(Setting::getKey, Function.identity())); - private ClusterService clusterService; private Client client; @@ -394,7 +388,7 @@ private void setSettingsUpdateConsumers() { ); NativeMemoryCacheManager.getInstance().rebuildCache(builder.build()); - }, Stream.concat(dynamicCacheSettings.values().stream(), FEATURE_FLAGS.values().stream()).collect(Collectors.toUnmodifiableList())); + }, dynamicCacheSettings.values().stream().collect(Collectors.toUnmodifiableList())); clusterService.getClusterSettings().addSettingsUpdateConsumer(QUANTIZATION_STATE_CACHE_SIZE_LIMIT_SETTING, it -> { QuantizationStateCache.getInstance().setMaxCacheSizeInKB(it.getKb()); QuantizationStateCache.getInstance().rebuildCache(); @@ -421,10 +415,6 @@ private Setting getSetting(String key) { return dynamicCacheSettings.get(key); } - if (FEATURE_FLAGS.containsKey(key)) { - return FEATURE_FLAGS.get(key); - } - if (KNN_CIRCUIT_BREAKER_TRIGGERED.equals(key)) { return KNN_CIRCUIT_BREAKER_TRIGGERED_SETTING; } @@ -484,8 +474,7 @@ public List> getSettings() { QUANTIZATION_STATE_CACHE_SIZE_LIMIT_SETTING, QUANTIZATION_STATE_CACHE_EXPIRY_TIME_MINUTES_SETTING ); - return Stream.concat(settings.stream(), Stream.concat(getFeatureFlags().stream(), dynamicCacheSettings.values().stream())) - .collect(Collectors.toList()); + return Stream.concat(settings.stream(), dynamicCacheSettings.values().stream()).collect(Collectors.toList()); } public static boolean isKNNPluginEnabled() { diff --git a/src/main/java/org/opensearch/knn/index/query/KNNQuery.java b/src/main/java/org/opensearch/knn/index/query/KNNQuery.java index 04a10143c..f5c4d3131 100644 --- a/src/main/java/org/opensearch/knn/index/query/KNNQuery.java +++ b/src/main/java/org/opensearch/knn/index/query/KNNQuery.java @@ -207,7 +207,8 @@ public int hashCode() { context, parentsFilter, radius, - methodParameters + methodParameters, + rescoreContext ); } @@ -227,7 +228,8 @@ private boolean equalsTo(KNNQuery other) { && Objects.equals(context, other.context) && Objects.equals(indexName, other.indexName) && Objects.equals(parentsFilter, other.parentsFilter) - && Objects.equals(filterQuery, other.filterQuery); + && Objects.equals(filterQuery, other.filterQuery) + && Objects.equals(rescoreContext, other.rescoreContext); } /** diff --git a/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java b/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java index 208e075eb..37f159fa2 100644 --- a/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java +++ b/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java @@ -614,12 +614,23 @@ protected boolean doEquals(KNNQueryBuilder other) { && Objects.equals(maxDistance, other.maxDistance) && Objects.equals(methodParameters, other.methodParameters) && Objects.equals(filter, other.filter) - && Objects.equals(ignoreUnmapped, other.ignoreUnmapped); + && Objects.equals(ignoreUnmapped, other.ignoreUnmapped) + && Objects.equals(rescoreContext, other.rescoreContext); } @Override protected int doHashCode() { - return Objects.hash(fieldName, Arrays.hashCode(vector), k, methodParameters, filter, ignoreUnmapped, maxDistance, minScore); + return Objects.hash( + fieldName, + Arrays.hashCode(vector), + k, + methodParameters, + filter, + ignoreUnmapped, + maxDistance, + minScore, + rescoreContext + ); } @Override diff --git a/src/main/java/org/opensearch/knn/index/query/KNNQueryFactory.java b/src/main/java/org/opensearch/knn/index/query/KNNQueryFactory.java index 2af3c35ca..dab2e08c8 100644 --- a/src/main/java/org/opensearch/knn/index/query/KNNQueryFactory.java +++ b/src/main/java/org/opensearch/knn/index/query/KNNQueryFactory.java @@ -23,7 +23,6 @@ import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_EF_SEARCH; import static org.opensearch.knn.common.KNNConstants.VECTOR_DATA_TYPE_FIELD; -import static org.opensearch.knn.common.featureflags.KNNFeatureFlags.isKnnQueryRewriteEnabled; import static org.opensearch.knn.index.VectorDataType.SUPPORTED_VECTOR_DATA_TYPES; /** @@ -96,7 +95,7 @@ public static Query create(CreateQueryRequest createQueryRequest) { .rescoreContext(rescoreContext) .build(); } - return isKnnQueryRewriteEnabled() ? new NativeEngineKnnVectorQuery(knnQuery) : knnQuery; + return createQueryRequest.getRescoreContext().isPresent() ? new NativeEngineKnnVectorQuery(knnQuery) : knnQuery; } Integer requestEfSearch = null; diff --git a/src/main/java/org/opensearch/knn/index/query/nativelib/DocAndScoreQuery.java b/src/main/java/org/opensearch/knn/index/query/nativelib/DocAndScoreQuery.java index f1a91d878..b94264b4d 100644 --- a/src/main/java/org/opensearch/knn/index/query/nativelib/DocAndScoreQuery.java +++ b/src/main/java/org/opensearch/knn/index/query/nativelib/DocAndScoreQuery.java @@ -45,6 +45,7 @@ public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float bo if (searcher.getIndexReader().getContext().id() != contextIdentity) { throw new IllegalStateException("This DocAndScore query was created by a different reader"); } + return new Weight(this) { @Override public Explanation explain(LeafReaderContext context, int doc) { diff --git a/src/main/java/org/opensearch/knn/index/query/nativelib/NativeEngineKnnVectorQuery.java b/src/main/java/org/opensearch/knn/index/query/nativelib/NativeEngineKnnVectorQuery.java index 06e5fc577..ac5a72945 100644 --- a/src/main/java/org/opensearch/knn/index/query/nativelib/NativeEngineKnnVectorQuery.java +++ b/src/main/java/org/opensearch/knn/index/query/nativelib/NativeEngineKnnVectorQuery.java @@ -15,6 +15,7 @@ import org.apache.lucene.search.QueryVisitor; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.TopDocs; +import org.apache.lucene.search.Weight; import org.apache.lucene.util.BitSet; import org.apache.lucene.util.Bits; import org.opensearch.knn.index.query.KNNQuery; @@ -45,7 +46,7 @@ public class NativeEngineKnnVectorQuery extends Query { private final KNNQuery knnQuery; @Override - public Query rewrite(final IndexSearcher indexSearcher) throws IOException { + public Weight createWeight(IndexSearcher indexSearcher, ScoreMode scoreMode, float boost) throws IOException { final IndexReader reader = indexSearcher.getIndexReader(); final KNNWeight knnWeight = (KNNWeight) knnQuery.createWeight(indexSearcher, ScoreMode.COMPLETE, 1); List leafReaderContexts = reader.leaves(); @@ -69,9 +70,9 @@ public Query rewrite(final IndexSearcher indexSearcher) throws IOException { TopDocs topK = TopDocs.merge(knnQuery.getK(), topDocs); if (topK.scoreDocs.length == 0) { - return new MatchNoDocsQuery(); + return new MatchNoDocsQuery().createWeight(indexSearcher, scoreMode, boost); } - return createRewrittenQuery(reader, topK); + return createDocAndScoreQuery(reader, topK).createWeight(indexSearcher, scoreMode, boost); } private List> doSearch( @@ -106,7 +107,7 @@ private List> doRescore( return indexSearcher.getTaskExecutor().invokeAll(rescoreTasks); } - private Query createRewrittenQuery(IndexReader reader, TopDocs topK) { + private Query createDocAndScoreQuery(IndexReader reader, TopDocs topK) { int len = topK.scoreDocs.length; Arrays.sort(topK.scoreDocs, Comparator.comparingInt(a -> a.doc)); int[] docs = new int[len]; diff --git a/src/test/java/org/opensearch/knn/common/featureflags/KNNFeatureFlagsTests.java b/src/test/java/org/opensearch/knn/common/featureflags/KNNFeatureFlagsTests.java deleted file mode 100644 index c3a8a1615..000000000 --- a/src/test/java/org/opensearch/knn/common/featureflags/KNNFeatureFlagsTests.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.knn.common.featureflags; - -import org.mockito.Mock; -import org.opensearch.common.settings.ClusterSettings; -import org.opensearch.knn.KNNTestCase; -import org.opensearch.knn.index.KNNSettings; - -import static org.mockito.Mockito.when; -import static org.opensearch.knn.common.featureflags.KNNFeatureFlags.KNN_LAUNCH_QUERY_REWRITE_ENABLED_SETTING; -import static org.opensearch.knn.common.featureflags.KNNFeatureFlags.isKnnQueryRewriteEnabled; - -public class KNNFeatureFlagsTests extends KNNTestCase { - - @Mock - ClusterSettings clusterSettings; - - public void setUp() throws Exception { - super.setUp(); - when(clusterService.getClusterSettings()).thenReturn(clusterSettings); - KNNSettings.state().setClusterService(clusterService); - } - - public void testIsFeatureEnabled() throws Exception { - when(clusterSettings.get(KNN_LAUNCH_QUERY_REWRITE_ENABLED_SETTING)).thenReturn(false); - assertFalse(isKnnQueryRewriteEnabled()); - when(clusterSettings.get(KNN_LAUNCH_QUERY_REWRITE_ENABLED_SETTING)).thenReturn(true); - assertTrue(isKnnQueryRewriteEnabled()); - } -} diff --git a/src/test/java/org/opensearch/knn/index/query/KNNQueryBuilderTests.java b/src/test/java/org/opensearch/knn/index/query/KNNQueryBuilderTests.java index b7de89564..1c1b6edd9 100644 --- a/src/test/java/org/opensearch/knn/index/query/KNNQueryBuilderTests.java +++ b/src/test/java/org/opensearch/knn/index/query/KNNQueryBuilderTests.java @@ -55,7 +55,6 @@ import static org.mockito.Mockito.anyString; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -import static org.opensearch.knn.common.featureflags.KNNFeatureFlags.KNN_LAUNCH_QUERY_REWRITE_ENABLED_SETTING; import static org.opensearch.knn.index.KNNClusterTestUtils.mockClusterService; import static org.opensearch.knn.index.engine.KNNEngine.ENGINES_SUPPORTING_RADIAL_SEARCH; @@ -78,7 +77,6 @@ public void setUp() throws Exception { super.setUp(); ClusterSettings clusterSettings = mock(ClusterSettings.class); when(clusterService.getClusterSettings()).thenReturn(clusterSettings); - when(clusterSettings.get(KNN_LAUNCH_QUERY_REWRITE_ENABLED_SETTING)).thenReturn(false); KNNSettings.state().setClusterService(clusterService); } diff --git a/src/test/java/org/opensearch/knn/index/query/KNNQueryBuilderValidParamsTests.java b/src/test/java/org/opensearch/knn/index/query/KNNQueryBuilderValidParamsTests.java index 7b5e59224..23278d28a 100644 --- a/src/test/java/org/opensearch/knn/index/query/KNNQueryBuilderValidParamsTests.java +++ b/src/test/java/org/opensearch/knn/index/query/KNNQueryBuilderValidParamsTests.java @@ -78,7 +78,12 @@ public static Collection validParameters() { ), $( "valid knn with rescore", - KNNQueryBuilder.builder().fieldName(FIELD_NAME).vector(QUERY_VECTOR).minScore(10.0f).build(), + KNNQueryBuilder.builder() + .fieldName(FIELD_NAME) + .vector(QUERY_VECTOR) + .rescoreContext(RescoreContext.getDefault()) + .minScore(10.0f) + .build(), null, null, null, diff --git a/src/test/java/org/opensearch/knn/index/query/KNNQueryFactoryTests.java b/src/test/java/org/opensearch/knn/index/query/KNNQueryFactoryTests.java index cd7fd035e..96493acec 100644 --- a/src/test/java/org/opensearch/knn/index/query/KNNQueryFactoryTests.java +++ b/src/test/java/org/opensearch/knn/index/query/KNNQueryFactoryTests.java @@ -42,7 +42,6 @@ import static org.mockito.Mockito.when; import static org.opensearch.knn.common.KNNConstants.DEFAULT_VECTOR_DATA_TYPE_FIELD; import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_EF_SEARCH; -import static org.opensearch.knn.common.featureflags.KNNFeatureFlags.KNN_LAUNCH_QUERY_REWRITE_ENABLED_SETTING; public class KNNQueryFactoryTests extends KNNTestCase { private static final String FILTER_FILED_NAME = "foo"; @@ -65,13 +64,11 @@ public class KNNQueryFactoryTests extends KNNTestCase { public void setUp() throws Exception { super.setUp(); when(clusterService.getClusterSettings()).thenReturn(clusterSettings); - when(clusterSettings.get(KNN_LAUNCH_QUERY_REWRITE_ENABLED_SETTING)).thenReturn(false); KNNSettings.state().setClusterService(clusterService); } public void testCreateCustomKNNQuery() { for (KNNEngine knnEngine : KNNEngine.getEnginesThatCreateCustomSegmentFiles()) { - when(clusterSettings.get(KNN_LAUNCH_QUERY_REWRITE_ENABLED_SETTING)).thenReturn(false); Query query = KNNQueryFactory.create( BaseQueryFactory.CreateQueryRequest.builder() .knnEngine(knnEngine) @@ -88,7 +85,6 @@ public void testCreateCustomKNNQuery() { assertEquals(testQueryVector, ((KNNQuery) query).getQueryVector()); assertEquals(testK, ((KNNQuery) query).getK()); - when(clusterSettings.get(KNN_LAUNCH_QUERY_REWRITE_ENABLED_SETTING)).thenReturn(true); query = KNNQueryFactory.create( BaseQueryFactory.CreateQueryRequest.builder() .knnEngine(knnEngine) @@ -99,9 +95,8 @@ public void testCreateCustomKNNQuery() { .vectorDataType(DEFAULT_VECTOR_DATA_TYPE_FIELD) .build() ); - assertTrue(query instanceof NativeEngineKnnVectorQuery); - query = ((NativeEngineKnnVectorQuery) query).getKnnQuery(); + assertTrue(query instanceof KNNQuery); assertEquals(testIndexName, ((KNNQuery) query).getIndexName()); assertEquals(testFieldName, ((KNNQuery) query).getField()); assertEquals(testQueryVector, ((KNNQuery) query).getQueryVector()); @@ -450,19 +445,27 @@ public void testCreate_whenBinary_thenSuccess() { assertTrue(query instanceof KNNQuery); assertNotNull(((KNNQuery) query).getByteQueryVector()); assertNull(((KNNQuery) query).getQueryVector()); - - when(clusterSettings.get(KNN_LAUNCH_QUERY_REWRITE_ENABLED_SETTING)).thenReturn(true); - query = KNNQueryFactory.create(createQueryRequest); - assertTrue(query instanceof NativeEngineKnnVectorQuery); } public void testCreate_whenRescoreContextPassed_thenSuccess() { + // Given QueryShardContext mockQueryShardContext = mock(QueryShardContext.class); MappedFieldType testMapper = mock(MappedFieldType.class); when(mockQueryShardContext.fieldMapper(any())).thenReturn(testMapper); BitSetProducer parentFilter = mock(BitSetProducer.class); when(mockQueryShardContext.getParentFilter()).thenReturn(parentFilter); + final KNNQuery expected = KNNQuery.builder() + .field(testFieldName) + .indexName(testIndexName) + .byteQueryVector(testByteQueryVector) + .k(testK) + .parentsFilter(parentFilter) + .vectorDataType(VectorDataType.BINARY) + .rescoreContext(RescoreContext.getDefault()) + .build(); + + // When final KNNQueryFactory.CreateQueryRequest createQueryRequest = KNNQueryFactory.CreateQueryRequest.builder() .knnEngine(KNNEngine.FAISS) .indexName(testIndexName) @@ -472,12 +475,11 @@ public void testCreate_whenRescoreContextPassed_thenSuccess() { .vectorDataType(VectorDataType.BINARY) .k(testK) .context(mockQueryShardContext) - .filter(FILTER_QUERY_BUILDER) .rescoreContext(RescoreContext.getDefault()) .build(); Query query = KNNQueryFactory.create(createQueryRequest); - assertTrue(query instanceof KNNQuery); - assertEquals(RescoreContext.getDefault(), ((KNNQuery) query).getRescoreContext()); - } + // Then + assertEquals(expected, ((NativeEngineKnnVectorQuery) query).getKnnQuery()); + } } diff --git a/src/test/java/org/opensearch/knn/index/query/nativelib/NativeEngineKNNVectorQueryIT.java b/src/test/java/org/opensearch/knn/index/query/nativelib/NativeEngineKNNVectorQueryIT.java deleted file mode 100644 index 1d84fcb48..000000000 --- a/src/test/java/org/opensearch/knn/index/query/nativelib/NativeEngineKNNVectorQueryIT.java +++ /dev/null @@ -1,190 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.knn.index.query.nativelib; - -import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; -import com.google.common.primitives.Floats; -import lombok.AllArgsConstructor; -import lombok.SneakyThrows; -import org.apache.http.util.EntityUtils; -import org.junit.BeforeClass; -import org.opensearch.client.Response; -import org.opensearch.common.xcontent.XContentFactory; -import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.knn.KNNRestTestCase; -import org.opensearch.knn.KNNResult; -import org.opensearch.knn.TestUtils; -import org.opensearch.knn.common.KNNConstants; -import org.opensearch.knn.index.FaissHNSWFlatE2EIT; -import org.opensearch.knn.index.SpaceType; -import org.opensearch.knn.index.engine.KNNEngine; -import org.opensearch.knn.index.query.KNNQueryBuilder; -import org.opensearch.knn.plugin.script.KNNScoringUtil; - -import java.io.IOException; -import java.net.URL; -import java.util.Arrays; -import java.util.Collection; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.TreeMap; -import java.util.concurrent.ThreadLocalRandom; - -import static com.carrotsearch.randomizedtesting.RandomizedTest.$; -import static com.carrotsearch.randomizedtesting.RandomizedTest.$$; -import static org.opensearch.knn.common.KNNConstants.KNN_ENGINE; -import static org.opensearch.knn.common.KNNConstants.METHOD_HNSW; -import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_EF_SEARCH; -import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_SPACE_TYPE; -import static org.opensearch.knn.common.KNNConstants.NAME; -import static org.opensearch.knn.common.KNNConstants.PARAMETERS; - -@AllArgsConstructor -public class NativeEngineKNNVectorQueryIT extends KNNRestTestCase { - - private String description; - private int k; - private Map methodParameters; - private boolean deleteRandomDocs; - - static TestUtils.TestData testData; - - @BeforeClass - public static void setUpClass() throws IOException { - if (FaissHNSWFlatE2EIT.class.getClassLoader() == null) { - throw new IllegalStateException("ClassLoader of FaissIT Class is null"); - } - URL testIndexVectors = FaissHNSWFlatE2EIT.class.getClassLoader().getResource("data/test_vectors_1000x128.json"); - URL testQueries = FaissHNSWFlatE2EIT.class.getClassLoader().getResource("data/test_queries_100x128.csv"); - assert testIndexVectors != null; - assert testQueries != null; - testData = new TestUtils.TestData(testIndexVectors.getPath(), testQueries.getPath()); - } - - @ParametersFactory(argumentFormatting = "description:%1$s; k:%2$s; efSearch:%3$s, deleteDocs:%4$s") - public static Collection parameters() { - return Arrays.asList( - $$( - $("test without deletedocs", 10, Map.of(METHOD_PARAMETER_EF_SEARCH, 300), false), - $("test with deletedocs", 10, Map.of(METHOD_PARAMETER_EF_SEARCH, 300), true) - ) - ); - } - - @SneakyThrows - public void testResultComparisonSanity() { - String indexName = "test-index-1"; - String fieldName = "test-field-1"; - - SpaceType spaceType = SpaceType.L2; - - Integer dimension = testData.indexData.vectors[0].length; - - // Create an index - XContentBuilder builder = XContentFactory.jsonBuilder() - .startObject() - .startObject("properties") - .startObject(fieldName) - .field("type", "knn_vector") - .field("dimension", dimension) - .startObject(KNNConstants.KNN_METHOD) - .field(NAME, METHOD_HNSW) - .field(METHOD_PARAMETER_SPACE_TYPE, spaceType.getValue()) - .field(KNN_ENGINE, KNNEngine.FAISS.getName()) - .startObject(PARAMETERS) - .field(KNNConstants.METHOD_PARAMETER_M, 16) - .field(KNNConstants.METHOD_PARAMETER_EF_CONSTRUCTION, 32) - .field(KNNConstants.METHOD_PARAMETER_EF_SEARCH, 32) - .endObject() - .endObject() - .endObject() - .endObject() - .endObject(); - - Map mappingMap = xContentBuilderToMap(builder); - String mapping = builder.toString(); - - createKnnIndex(indexName, mapping); - assertEquals(new TreeMap<>(mappingMap), new TreeMap<>(getIndexMappingAsMap(indexName))); - - // Index the test data - for (int i = 0; i < testData.indexData.docs.length; i++) { - addKnnDoc( - indexName, - Integer.toString(testData.indexData.docs[i]), - fieldName, - Floats.asList(testData.indexData.vectors[i]).toArray() - ); - } - - // Assert we have the right number of documents in the index - refreshAllNonSystemIndices(); - assertEquals(testData.indexData.docs.length, getDocCount(indexName)); - - // Delete few Docs - if (deleteRandomDocs) { - final Set docIdsToBeDeleted = new HashSet<>(); - while (docIdsToBeDeleted.size() < 10) { - docIdsToBeDeleted.add(randomInt(testData.indexData.docs.length - 1)); - } - - for (Integer id : docIdsToBeDeleted) { - deleteKnnDoc(indexName, Integer.toString(testData.indexData.docs[id])); - } - refreshAllNonSystemIndices(); - forceMergeKnnIndex(indexName, 3); - - assertEquals(testData.indexData.docs.length - 10, getDocCount(indexName)); - } - - int queryIndex = ThreadLocalRandom.current().nextInt(testData.queries.length); - // Test search queries - final KNNQueryBuilder queryBuilder = KNNQueryBuilder.builder() - .fieldName(fieldName) - .vector(testData.queries[queryIndex]) - .k(k) - .methodParameters(methodParameters) - .build(); - Response nativeEngineResponse = searchKNNIndex(indexName, queryBuilder, k); - String responseBody = EntityUtils.toString(nativeEngineResponse.getEntity()); - List nativeEngineKnnResults = parseSearchResponse(responseBody, fieldName); - assertEquals(k, nativeEngineKnnResults.size()); - - List actualScores = parseSearchResponseScore(responseBody, fieldName); - for (int j = 0; j < k; j++) { - float[] primitiveArray = nativeEngineKnnResults.get(j).getVector(); - assertEquals( - KNNEngine.FAISS.score(KNNScoringUtil.l2Squared(testData.queries[queryIndex], primitiveArray), spaceType), - actualScores.get(j), - 0.0001 - ); - } - - updateClusterSettings("knn.feature.query.rewrite.enabled", false); - Response launchControlDisabledResponse = searchKNNIndex(indexName, queryBuilder, k); - String launchControlDisabledResponseString = EntityUtils.toString(launchControlDisabledResponse.getEntity()); - List knnResults = parseSearchResponse(launchControlDisabledResponseString, fieldName); - assertEquals(k, knnResults.size()); - - assertEquals(nativeEngineKnnResults, knnResults); - - // Delete index - deleteKNNIndex(indexName); - - // Search every 5 seconds 14 times to confirm graph gets evicted - int intervals = 14; - for (int i = 0; i < intervals; i++) { - if (getTotalGraphsInCache() == 0) { - return; - } - Thread.sleep(5 * 1000); - } - - fail("Graphs are not getting evicted"); - } -} diff --git a/src/test/java/org/opensearch/knn/index/query/nativelib/NativeEngineKNNVectorQueryTests.java b/src/test/java/org/opensearch/knn/index/query/nativelib/NativeEngineKNNVectorQueryTests.java index b6059bd11..ee53818f1 100644 --- a/src/test/java/org/opensearch/knn/index/query/nativelib/NativeEngineKNNVectorQueryTests.java +++ b/src/test/java/org/opensearch/knn/index/query/nativelib/NativeEngineKNNVectorQueryTests.java @@ -16,6 +16,7 @@ import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.TaskExecutor; import org.apache.lucene.search.TopDocs; +import org.apache.lucene.search.Weight; import org.apache.lucene.util.Bits; import org.mockito.InjectMocks; import org.mockito.Mock; @@ -118,13 +119,13 @@ public void testMultiLeaf() { int[] expectedDocs = { 0, 3, 4 }; float[] expectedScores = { 1.2f, 5.1f, 3.4f }; int[] findSegments = { 0, 1, 3 }; - DocAndScoreQuery expected = new DocAndScoreQuery(4, expectedDocs, expectedScores, findSegments, 1); + Query expected = new DocAndScoreQuery(4, expectedDocs, expectedScores, findSegments, 1); // When - Query actual = objectUnderTest.rewrite(searcher); + Weight actual = objectUnderTest.createWeight(searcher, ScoreMode.COMPLETE, 1); // Then - assertEquals(expected, actual); + assertEquals(expected, actual.getQuery()); } @SneakyThrows @@ -139,13 +140,13 @@ public void testSingleLeaf() { int[] expectedDocs = { 0, 1, 2 }; float[] expectedScores = { 1.2f, 5.1f, 2.2f }; int[] findSegments = { 0, 3 }; - DocAndScoreQuery expected = new DocAndScoreQuery(4, expectedDocs, expectedScores, findSegments, 1); + Query expected = new DocAndScoreQuery(4, expectedDocs, expectedScores, findSegments, 1); // When - Query actual = objectUnderTest.rewrite(searcher); + Weight actual = objectUnderTest.createWeight(searcher, ScoreMode.COMPLETE, 1); // Then - assertEquals(expected, actual); + assertEquals(expected, actual.getQuery()); } @SneakyThrows @@ -155,11 +156,12 @@ public void testNoMatch() { when(reader.leaves()).thenReturn(leaves); when(knnWeight.searchLeaf(leaf1, 4)).thenReturn(Collections.emptyMap()); when(knnQuery.getK()).thenReturn(4); + // When - Query actual = objectUnderTest.rewrite(searcher); + Weight actual = objectUnderTest.createWeight(searcher, ScoreMode.COMPLETE, 1); // Then - assertEquals(new MatchNoDocsQuery(), actual); + assertEquals(new MatchNoDocsQuery(), actual.getQuery()); } @SneakyThrows @@ -176,7 +178,7 @@ public void testRescore() { Map rescoredLeaf2Results = new HashMap<>(Map.of(0, 21f)); TopDocs topDocs1 = ResultUtil.resultMapToTopDocs(Map.of(1, 20f), 0); TopDocs topDocs2 = ResultUtil.resultMapToTopDocs(Map.of(0, 21f), 4); - DocAndScoreQuery expected = new DocAndScoreQuery(2, new int[] { 1, 4 }, new float[] { 20f, 21f }, new int[] { 0, 4, 2 }, 1); + Query expected = new DocAndScoreQuery(2, new int[] { 1, 4 }, new float[] { 20f, 21f }, new int[] { 0, 4, 2 }, 1); when(indexReaderContext.id()).thenReturn(1); when(knnQuery.getRescoreContext()).thenReturn(RescoreContext.builder().oversampleFactor(1.5f).build()); @@ -193,8 +195,8 @@ public void testRescore() { try (MockedStatic mockedStaticNativeKnnVectorQuery = mockStatic(NativeEngineKnnVectorQuery.class)) { mockedStaticNativeKnnVectorQuery.when(() -> NativeEngineKnnVectorQuery.findSegmentStarts(any(), any())) .thenReturn(new int[] { 0, 4, 2 }); - Query actual = objectUnderTest.rewrite(searcher); - assertEquals(expected, actual); + Weight actual = objectUnderTest.createWeight(searcher, ScoreMode.COMPLETE, 1); + assertEquals(expected, actual.getQuery()); } } } From 0410e37c85c33a7f7f9a23a3aaffd118af683134 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Fri, 30 Aug 2024 14:38:13 -0500 Subject: [PATCH 343/416] Add IVF changes to support Faiss byte vector (#2002) (#2017) * Add HNSW changes to support Faiss byte vector Signed-off-by: Naveen Tatikonda * Address Review Comments Signed-off-by: Naveen Tatikonda * Add IVF changes to support Faiss byte vector Signed-off-by: Naveen Tatikonda * Address Review Comments Signed-off-by: Naveen Tatikonda --------- Signed-off-by: Naveen Tatikonda (cherry picked from commit 2b303d90d7e47169f5bb1a216c4b9d7207794fba) Co-authored-by: Naveen Tatikonda --- CHANGELOG.md | 1 + jni/include/faiss_wrapper.h | 13 ++ .../org_opensearch_knn_jni_FaissService.h | 16 ++ jni/src/faiss_wrapper.cpp | 157 ++++++++++++++++++ .../org_opensearch_knn_jni_FaissService.cpp | 28 ++++ jni/tests/faiss_wrapper_test.cpp | 81 +++++++++ jni/tests/test_util.cpp | 8 + jni/tests/test_util.h | 2 + .../opensearch/knn/index/VectorDataType.java | 47 ++++++ .../index/engine/faiss/FaissIVFMethod.java | 6 +- .../index/memory/NativeMemoryAllocation.java | 8 +- .../memory/NativeMemoryLoadStrategy.java | 8 +- .../opensearch/knn/index/util/IndexUtil.java | 33 +--- .../org/opensearch/knn/jni/FaissService.java | 29 ++++ .../org/opensearch/knn/jni/JNICommons.java | 8 +- .../org/opensearch/knn/jni/JNIService.java | 12 +- .../training/BinaryTrainingDataConsumer.java | 74 +++++++++ .../training/ByteTrainingDataConsumer.java | 16 +- .../knn/index/VectorDataTypeIT.java | 98 +++++++++++ .../knn/index/util/IndexUtilTests.java | 62 ++++--- .../org/opensearch/knn/KNNRestTestCase.java | 11 ++ 21 files changed, 626 insertions(+), 92 deletions(-) create mode 100644 src/main/java/org/opensearch/knn/training/BinaryTrainingDataConsumer.java diff --git a/CHANGELOG.md b/CHANGELOG.md index b030333e7..58058dbd1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), * Integrate Lucene Vector field with native engines to use KNNVectorFormat during segment creation [#1945](https://github.com/opensearch-project/k-NN/pull/1945) * k-NN query rescore support for native engines [#1984](https://github.com/opensearch-project/k-NN/pull/1984) * Add support for byte vector with Faiss Engine HNSW algorithm [#1823](https://github.com/opensearch-project/k-NN/pull/1823) +* Add support for byte vector with Faiss Engine IVF algorithm [#2002](https://github.com/opensearch-project/k-NN/pull/2002) ### Enhancements * Adds iterative graph build capability into a faiss index to improve the memory footprint during indexing and Integrates KNNVectorsFormat for native engines[#1950](https://github.com/opensearch-project/k-NN/pull/1950) ### Bug Fixes diff --git a/jni/include/faiss_wrapper.h b/jni/include/faiss_wrapper.h index 574efb6fd..d6375653d 100644 --- a/jni/include/faiss_wrapper.h +++ b/jni/include/faiss_wrapper.h @@ -36,6 +36,12 @@ namespace knn_jni { jlong vectorsAddressJ, jint dimJ, jstring indexPathJ, jbyteArray templateIndexJ, jobject parametersJ); + // Create a index with ids and byte vectors. Instead of creating a new index, this function creates the index + // based off of the template index passed in. The index is serialized to indexPathJ. + void CreateByteIndexFromTemplate(knn_jni::JNIUtilInterface * jniUtil, JNIEnv * env, jintArray idsJ, + jlong vectorsAddressJ, jint dimJ, jstring indexPathJ, jbyteArray templateIndexJ, + jobject parametersJ); + // Load an index from indexPathJ into memory. // // Return a pointer to the loaded index @@ -110,6 +116,13 @@ namespace knn_jni { jbyteArray TrainBinaryIndex(knn_jni::JNIUtilInterface * jniUtil, JNIEnv * env, jobject parametersJ, jint dimension, jlong trainVectorsPointerJ); + // Create an empty byte index defined by the values in the Java map, parametersJ. Train the index with + // the byte vectors located at trainVectorsPointerJ. + // + // Return the serialized representation + jbyteArray TrainByteIndex(knn_jni::JNIUtilInterface * jniUtil, JNIEnv * env, jobject parametersJ, jint dimension, + jlong trainVectorsPointerJ); + /* * Perform a range search with filter against the index located in memory at indexPointerJ. * diff --git a/jni/include/org_opensearch_knn_jni_FaissService.h b/jni/include/org_opensearch_knn_jni_FaissService.h index 09f3ec8b7..d42ce197c 100644 --- a/jni/include/org_opensearch_knn_jni_FaissService.h +++ b/jni/include/org_opensearch_knn_jni_FaissService.h @@ -112,6 +112,14 @@ JNIEXPORT void JNICALL Java_org_opensearch_knn_jni_FaissService_createIndexFromT JNIEXPORT void JNICALL Java_org_opensearch_knn_jni_FaissService_createBinaryIndexFromTemplate (JNIEnv *, jclass, jintArray, jlong, jint, jstring, jbyteArray, jobject); +/* + * Class: org_opensearch_knn_jni_FaissService + * Method: createByteIndexFromTemplate + * Signature: ([IJILjava/lang/String;[BLjava/util/Map;)V + */ + JNIEXPORT void JNICALL Java_org_opensearch_knn_jni_FaissService_createByteIndexFromTemplate + (JNIEnv *, jclass, jintArray, jlong, jint, jstring, jbyteArray, jobject); + /* * Class: org_opensearch_knn_jni_FaissService * Method: loadIndex @@ -216,6 +224,14 @@ JNIEXPORT jbyteArray JNICALL Java_org_opensearch_knn_jni_FaissService_trainIndex JNIEXPORT jbyteArray JNICALL Java_org_opensearch_knn_jni_FaissService_trainBinaryIndex (JNIEnv *, jclass, jobject, jint, jlong); +/* + * Class: org_opensearch_knn_jni_FaissService + * Method: trainByteIndex + * Signature: (Ljava/util/Map;IJ)[B + */ +JNIEXPORT jbyteArray JNICALL Java_org_opensearch_knn_jni_FaissService_trainByteIndex + (JNIEnv *, jclass, jobject, jint, jlong); + /* * Class: org_opensearch_knn_jni_FaissService * Method: transferVectors diff --git a/jni/src/faiss_wrapper.cpp b/jni/src/faiss_wrapper.cpp index 0e1029ecf..ba15c3ce7 100644 --- a/jni/src/faiss_wrapper.cpp +++ b/jni/src/faiss_wrapper.cpp @@ -320,6 +320,96 @@ void knn_jni::faiss_wrapper::CreateBinaryIndexFromTemplate(knn_jni::JNIUtilInter faiss::write_index_binary(&idMap, indexPathCpp.c_str()); } +void knn_jni::faiss_wrapper::CreateByteIndexFromTemplate(knn_jni::JNIUtilInterface * jniUtil, JNIEnv * env, jintArray idsJ, + jlong vectorsAddressJ, jint dimJ, jstring indexPathJ, + jbyteArray templateIndexJ, jobject parametersJ) { + if (idsJ == nullptr) { + throw std::runtime_error("IDs cannot be null"); + } + + if (vectorsAddressJ <= 0) { + throw std::runtime_error("VectorsAddress cannot be less than 0"); + } + + if(dimJ <= 0) { + throw std::runtime_error("Vectors dimensions cannot be less than or equal to 0"); + } + + if (indexPathJ == nullptr) { + throw std::runtime_error("Index path cannot be null"); + } + + if (templateIndexJ == nullptr) { + throw std::runtime_error("Template index cannot be null"); + } + + // Set thread count if it is passed in as a parameter. Setting this variable will only impact the current thread + auto parametersCpp = jniUtil->ConvertJavaMapToCppMap(env, parametersJ); + if(parametersCpp.find(knn_jni::INDEX_THREAD_QUANTITY) != parametersCpp.end()) { + auto threadCount = jniUtil->ConvertJavaObjectToCppInteger(env, parametersCpp[knn_jni::INDEX_THREAD_QUANTITY]); + omp_set_num_threads(threadCount); + } + jniUtil->DeleteLocalRef(env, parametersJ); + + // Read data set + // Read vectors from memory address + auto *inputVectors = reinterpret_cast*>(vectorsAddressJ); + int dim = (int)dimJ; + int numVectors = (int) (inputVectors->size() / (uint64_t) dim); + int numIds = jniUtil->GetJavaIntArrayLength(env, idsJ); + + if (numIds != numVectors) { + throw std::runtime_error("Number of IDs does not match number of vectors"); + } + + // Get vector of bytes from jbytearray + int indexBytesCount = jniUtil->GetJavaBytesArrayLength(env, templateIndexJ); + jbyte * indexBytesJ = jniUtil->GetByteArrayElements(env, templateIndexJ, nullptr); + + faiss::VectorIOReader vectorIoReader; + for (int i = 0; i < indexBytesCount; i++) { + vectorIoReader.data.push_back((uint8_t) indexBytesJ[i]); + } + jniUtil->ReleaseByteArrayElements(env, templateIndexJ, indexBytesJ, JNI_ABORT); + + // Create faiss index + std::unique_ptr indexWriter; + indexWriter.reset(faiss::read_index(&vectorIoReader, 0)); + + auto ids = jniUtil->ConvertJavaIntArrayToCppIntVector(env, idsJ); + faiss::IndexIDMap idMap = faiss::IndexIDMap(indexWriter.get()); + + // Add vectors in batches by casting int8 vectors into float with a batch size of 1000 to avoid additional memory spike. + // Refer to this github issue for more details https://github.com/opensearch-project/k-NN/issues/1659#issuecomment-2307390255 + int batchSize = 1000; + std::vector inputFloatVectors(batchSize * dim); + std::vector floatVectorsIds(batchSize); + int id = 0; + auto iter = inputVectors->begin(); + + for (int id = 0; id < numVectors; id += batchSize) { + if (numVectors - id < batchSize) { + batchSize = numVectors - id; + } + + for (int i = 0; i < batchSize; ++i) { + floatVectorsIds[i] = ids[id + i]; + for (int j = 0; j < dim; ++j, ++iter) { + inputFloatVectors[i * dim + j] = static_cast(*iter); + } + } + idMap.add_with_ids(batchSize, inputFloatVectors.data(), floatVectorsIds.data()); + } + + // Releasing the vectorsAddressJ memory as that is not required once we have created the index. + // This is not the ideal approach, please refer this gh issue for long term solution: + // https://github.com/opensearch-project/k-NN/issues/1600 + delete inputVectors; + // Write the index to disk + std::string indexPathCpp(jniUtil->ConvertJavaStringToCppString(env, indexPathJ)); + faiss::write_index(&idMap, indexPathCpp.c_str()); +} + jlong knn_jni::faiss_wrapper::LoadIndex(knn_jni::JNIUtilInterface * jniUtil, JNIEnv * env, jstring indexPathJ) { if (indexPathJ == nullptr) { throw std::runtime_error("Index path cannot be null"); @@ -782,6 +872,73 @@ jbyteArray knn_jni::faiss_wrapper::TrainBinaryIndex(knn_jni::JNIUtilInterface * return ret; } +jbyteArray knn_jni::faiss_wrapper::TrainByteIndex(knn_jni::JNIUtilInterface * jniUtil, JNIEnv * env, jobject parametersJ, + jint dimensionJ, jlong trainVectorsPointerJ) { + // First, we need to build the index + if (parametersJ == nullptr) { + throw std::runtime_error("Parameters cannot be null"); + } + + auto parametersCpp = jniUtil->ConvertJavaMapToCppMap(env, parametersJ); + + jobject spaceTypeJ = knn_jni::GetJObjectFromMapOrThrow(parametersCpp, knn_jni::SPACE_TYPE); + std::string spaceTypeCpp(jniUtil->ConvertJavaObjectToCppString(env, spaceTypeJ)); + faiss::MetricType metric = TranslateSpaceToMetric(spaceTypeCpp); + + // Create faiss index + jobject indexDescriptionJ = knn_jni::GetJObjectFromMapOrThrow(parametersCpp, knn_jni::INDEX_DESCRIPTION); + std::string indexDescriptionCpp(jniUtil->ConvertJavaObjectToCppString(env, indexDescriptionJ)); + + std::unique_ptr indexWriter; + indexWriter.reset(faiss::index_factory((int) dimensionJ, indexDescriptionCpp.c_str(), metric)); + + // Set thread count if it is passed in as a parameter. Setting this variable will only impact the current thread + if(parametersCpp.find(knn_jni::INDEX_THREAD_QUANTITY) != parametersCpp.end()) { + auto threadCount = jniUtil->ConvertJavaObjectToCppInteger(env, parametersCpp[knn_jni::INDEX_THREAD_QUANTITY]); + omp_set_num_threads(threadCount); + } + + // Add extra parameters that cant be configured with the index factory + if(parametersCpp.find(knn_jni::PARAMETERS) != parametersCpp.end()) { + jobject subParametersJ = parametersCpp[knn_jni::PARAMETERS]; + auto subParametersCpp = jniUtil->ConvertJavaMapToCppMap(env, subParametersJ); + SetExtraParameters(jniUtil, env, subParametersCpp, indexWriter.get()); + jniUtil->DeleteLocalRef(env, subParametersJ); + } + + // Train index if needed + auto *trainingVectorsPointerCpp = reinterpret_cast*>(trainVectorsPointerJ); + int numVectors = trainingVectorsPointerCpp->size()/(int) dimensionJ; + + auto iter = trainingVectorsPointerCpp->begin(); + std::vector trainingFloatVectors(numVectors * dimensionJ); + for(int i=0; i < numVectors * dimensionJ; ++i, ++iter) { + trainingFloatVectors[i] = static_cast(*iter); + } + + if(!indexWriter->is_trained) { + InternalTrainIndex(indexWriter.get(), numVectors, trainingFloatVectors.data()); + } + jniUtil->DeleteLocalRef(env, parametersJ); + + // Now that indexWriter is trained, we just load the bytes into an array and return + faiss::VectorIOWriter vectorIoWriter; + faiss::write_index(indexWriter.get(), &vectorIoWriter); + + // Wrap in smart pointer + std::unique_ptr jbytesBuffer; + jbytesBuffer.reset(new jbyte[vectorIoWriter.data.size()]); + int c = 0; + for (auto b : vectorIoWriter.data) { + jbytesBuffer[c++] = (jbyte) b; + } + + jbyteArray ret = jniUtil->NewByteArray(env, vectorIoWriter.data.size()); + jniUtil->SetByteArrayRegion(env, ret, 0, vectorIoWriter.data.size(), jbytesBuffer.get()); + return ret; +} + + faiss::MetricType TranslateSpaceToMetric(const std::string& spaceType) { if (spaceType == knn_jni::L2) { return faiss::METRIC_L2; diff --git a/jni/src/org_opensearch_knn_jni_FaissService.cpp b/jni/src/org_opensearch_knn_jni_FaissService.cpp index bcdc4f18b..70c986b7d 100644 --- a/jni/src/org_opensearch_knn_jni_FaissService.cpp +++ b/jni/src/org_opensearch_knn_jni_FaissService.cpp @@ -192,6 +192,21 @@ JNIEXPORT void JNICALL Java_org_opensearch_knn_jni_FaissService_createBinaryInde } } +JNIEXPORT void JNICALL Java_org_opensearch_knn_jni_FaissService_createByteIndexFromTemplate(JNIEnv * env, jclass cls, + jintArray idsJ, + jlong vectorsAddressJ, + jint dimJ, + jstring indexPathJ, + jbyteArray templateIndexJ, + jobject parametersJ) +{ + try { + knn_jni::faiss_wrapper::CreateByteIndexFromTemplate(&jniUtil, env, idsJ, vectorsAddressJ, dimJ, indexPathJ, templateIndexJ, parametersJ); + } catch (...) { + jniUtil.CatchCppExceptionAndThrowJava(env); + } +} + JNIEXPORT jlong JNICALL Java_org_opensearch_knn_jni_FaissService_loadIndex(JNIEnv * env, jclass cls, jstring indexPathJ) { try { @@ -335,6 +350,19 @@ JNIEXPORT jbyteArray JNICALL Java_org_opensearch_knn_jni_FaissService_trainBinar return nullptr; } +JNIEXPORT jbyteArray JNICALL Java_org_opensearch_knn_jni_FaissService_trainByteIndex(JNIEnv * env, jclass cls, + jobject parametersJ, + jint dimensionJ, + jlong trainVectorsPointerJ) +{ + try { + return knn_jni::faiss_wrapper::TrainByteIndex(&jniUtil, env, parametersJ, dimensionJ, trainVectorsPointerJ); + } catch (...) { + jniUtil.CatchCppExceptionAndThrowJava(env); + } + return nullptr; +} + JNIEXPORT jlong JNICALL Java_org_opensearch_knn_jni_FaissService_transferVectors(JNIEnv * env, jclass cls, jlong vectorsPointerJ, jobjectArray vectorsJ) diff --git a/jni/tests/faiss_wrapper_test.cpp b/jni/tests/faiss_wrapper_test.cpp index a1839c6ce..5f6f83c46 100644 --- a/jni/tests/faiss_wrapper_test.cpp +++ b/jni/tests/faiss_wrapper_test.cpp @@ -230,6 +230,55 @@ TEST(FaissCreateIndexFromTemplateTest, BasicAssertions) { std::remove(indexPath.c_str()); } +TEST(FaissCreateByteIndexFromTemplateTest, BasicAssertions) { + // Define the data + faiss::idx_t numIds = 100; + std::vector ids; + auto *vectors = new std::vector(); + int dim = 8; + vectors->reserve(dim * numIds); + for (int64_t i = 0; i < numIds; ++i) { + ids.push_back(i); + for (int j = 0; j < dim; ++j) { + vectors->push_back(test_util::RandomInt(-128, 127)); + } + } + + std::string indexPath = test_util::RandomString(10, "tmp/", ".faiss"); + faiss::MetricType metricType = faiss::METRIC_L2; + std::string method = "HNSW32,SQ8_direct_signed"; + + std::unique_ptr createdIndex( + test_util::FaissCreateIndex(dim, method, metricType)); + auto vectorIoWriter = test_util::FaissGetSerializedIndex(createdIndex.get()); + + // Setup jni + JNIEnv *jniEnv = nullptr; + NiceMock mockJNIUtil; + + EXPECT_CALL(mockJNIUtil, + GetJavaObjectArrayLength( + jniEnv, reinterpret_cast(&vectors))) + .WillRepeatedly(Return(vectors->size())); + + std::string spaceType = knn_jni::L2; + std::unordered_map parametersMap; + parametersMap[knn_jni::SPACE_TYPE] = (jobject) &spaceType; + + knn_jni::faiss_wrapper::CreateByteIndexFromTemplate( + &mockJNIUtil, jniEnv, reinterpret_cast(&ids), + (jlong)vectors, dim, (jstring)&indexPath, + reinterpret_cast(&(vectorIoWriter.data)), + (jobject) ¶metersMap + ); + + // Make sure index can be loaded + std::unique_ptr index(test_util::FaissLoadIndex(indexPath)); + + // Clean up + std::remove(indexPath.c_str()); +} + TEST(FaissLoadIndexTest, BasicAssertions) { // Define the data faiss::idx_t numIds = 100; @@ -717,6 +766,38 @@ TEST(FaissTrainIndexTest, BasicAssertions) { ASSERT_TRUE(trainedIndex->is_trained); } +TEST(FaissTrainByteIndexTest, BasicAssertions) { + // Define the index configuration + int dim = 2; + std::string spaceType = knn_jni::L2; + std::string index_description = "IVF4,SQ8_direct_signed"; + + std::unordered_map parametersMap; + parametersMap[knn_jni::SPACE_TYPE] = (jobject) &spaceType; + parametersMap[knn_jni::INDEX_DESCRIPTION] = (jobject) &index_description; + + // Define training data + int numTrainingVectors = 256; + std::vector trainingVectors = test_util::RandomByteVectors(dim, numTrainingVectors, -128, 127); + + // Setup jni + JNIEnv *jniEnv = nullptr; + NiceMock mockJNIUtil; + + // Perform training + std::unique_ptr> trainedIndexSerialization( + reinterpret_cast *>( + knn_jni::faiss_wrapper::TrainByteIndex( + &mockJNIUtil, jniEnv, (jobject) ¶metersMap, dim, + reinterpret_cast(&trainingVectors)))); + + std::unique_ptr trainedIndex( + test_util::FaissLoadFromSerializedIndex(trainedIndexSerialization.get())); + + // Confirm that training succeeded + ASSERT_TRUE(trainedIndex->is_trained); +} + TEST(FaissCreateHnswSQfp16IndexTest, BasicAssertions) { // Define the data faiss::idx_t numIds = 200; diff --git a/jni/tests/test_util.cpp b/jni/tests/test_util.cpp index 4f8bd2c34..47d1a7c8e 100644 --- a/jni/tests/test_util.cpp +++ b/jni/tests/test_util.cpp @@ -447,6 +447,14 @@ std::vector test_util::RandomVectors(int dim, int64_t numVectors, float m return vectors; } +std::vector test_util::RandomByteVectors(int dim, int64_t numVectors, int min, int max) { + std::vector vectors(dim*numVectors); + for (int64_t i = 0; i < dim*numVectors; i++) { + vectors[i] = test_util::RandomInt(min, max); + } + return vectors; +} + std::vector test_util::Range(int64_t numElements) { std::vector rangeVector(numElements); for (int64_t i = 0; i < numElements; i++) { diff --git a/jni/tests/test_util.h b/jni/tests/test_util.h index a90d45dd9..ea02da6f2 100644 --- a/jni/tests/test_util.h +++ b/jni/tests/test_util.h @@ -173,6 +173,8 @@ namespace test_util { std::vector RandomVectors(int dim, int64_t numVectors, float min, float max); + std::vector RandomByteVectors(int dim, int64_t numVectors, int min, int max); + std::vector Range(int64_t numElements); // returns the number of 64 bit words it would take to hold numBits diff --git a/src/main/java/org/opensearch/knn/index/VectorDataType.java b/src/main/java/org/opensearch/knn/index/VectorDataType.java index 9283e5ee6..4827a4582 100644 --- a/src/main/java/org/opensearch/knn/index/VectorDataType.java +++ b/src/main/java/org/opensearch/knn/index/VectorDataType.java @@ -14,6 +14,12 @@ import org.apache.lucene.util.BytesRef; import org.opensearch.knn.index.codec.util.KNNVectorSerializer; import org.opensearch.knn.index.codec.util.KNNVectorSerializerFactory; +import org.opensearch.knn.index.memory.NativeMemoryAllocation; +import org.opensearch.knn.jni.JNICommons; +import org.opensearch.knn.training.BinaryTrainingDataConsumer; +import org.opensearch.knn.training.ByteTrainingDataConsumer; +import org.opensearch.knn.training.FloatTrainingDataConsumer; +import org.opensearch.knn.training.TrainingDataConsumer; import java.util.Arrays; import java.util.Locale; @@ -48,6 +54,16 @@ public float[] getVectorFromBytesRef(BytesRef binaryValue) { } return vector; } + + @Override + public TrainingDataConsumer getTrainingDataConsumer(NativeMemoryAllocation.TrainingDataAllocation trainingDataAllocation) { + return new BinaryTrainingDataConsumer(trainingDataAllocation); + } + + @Override + public void freeNativeMemory(long memoryAddress) { + JNICommons.freeBinaryVectorData(memoryAddress); + } }, BYTE("byte") { @@ -67,6 +83,16 @@ public float[] getVectorFromBytesRef(BytesRef binaryValue) { } return vector; } + + @Override + public TrainingDataConsumer getTrainingDataConsumer(NativeMemoryAllocation.TrainingDataAllocation trainingDataAllocation) { + return new ByteTrainingDataConsumer(trainingDataAllocation); + } + + @Override + public void freeNativeMemory(long memoryAddress) { + JNICommons.freeByteVectorData(memoryAddress); + } }, FLOAT("float") { @@ -81,6 +107,16 @@ public float[] getVectorFromBytesRef(BytesRef binaryValue) { return vectorSerializer.byteToFloatArray(binaryValue); } + @Override + public TrainingDataConsumer getTrainingDataConsumer(NativeMemoryAllocation.TrainingDataAllocation trainingDataAllocation) { + return new FloatTrainingDataConsumer(trainingDataAllocation); + } + + @Override + public void freeNativeMemory(long memoryAddress) { + JNICommons.freeVectorData(memoryAddress); + } + }; public static final String SUPPORTED_VECTOR_DATA_TYPES = Arrays.stream(VectorDataType.values()) @@ -107,6 +143,17 @@ public float[] getVectorFromBytesRef(BytesRef binaryValue) { */ public abstract float[] getVectorFromBytesRef(BytesRef binaryValue); + /** + * @param trainingDataAllocation training data that has been allocated in native memory + * @return TrainingDataConsumer which consumes training data + */ + public abstract TrainingDataConsumer getTrainingDataConsumer(NativeMemoryAllocation.TrainingDataAllocation trainingDataAllocation); + + /** + * @param memoryAddress address to be freed + */ + public abstract void freeNativeMemory(long memoryAddress); + /** * Validates if given VectorDataType is in the list of supported data types. * @param vectorDataType VectorDataType diff --git a/src/main/java/org/opensearch/knn/index/engine/faiss/FaissIVFMethod.java b/src/main/java/org/opensearch/knn/index/engine/faiss/FaissIVFMethod.java index b3dd12c92..70ab4222b 100644 --- a/src/main/java/org/opensearch/knn/index/engine/faiss/FaissIVFMethod.java +++ b/src/main/java/org/opensearch/knn/index/engine/faiss/FaissIVFMethod.java @@ -38,7 +38,11 @@ */ public class FaissIVFMethod extends AbstractFaissMethod { - private static final Set SUPPORTED_DATA_TYPES = ImmutableSet.of(VectorDataType.FLOAT, VectorDataType.BINARY); + private static final Set SUPPORTED_DATA_TYPES = ImmutableSet.of( + VectorDataType.FLOAT, + VectorDataType.BINARY, + VectorDataType.BYTE + ); public final static List SUPPORTED_SPACES = Arrays.asList( SpaceType.UNDEFINED, diff --git a/src/main/java/org/opensearch/knn/index/memory/NativeMemoryAllocation.java b/src/main/java/org/opensearch/knn/index/memory/NativeMemoryAllocation.java index 755b6b925..c711f3342 100644 --- a/src/main/java/org/opensearch/knn/index/memory/NativeMemoryAllocation.java +++ b/src/main/java/org/opensearch/knn/index/memory/NativeMemoryAllocation.java @@ -13,10 +13,8 @@ import lombok.Getter; import org.apache.lucene.index.LeafReaderContext; -import org.opensearch.knn.index.util.IndexUtil; import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.query.KNNWeight; -import org.opensearch.knn.jni.JNICommons; import org.opensearch.knn.jni.JNIService; import org.opensearch.knn.index.engine.KNNEngine; import org.opensearch.watcher.FileWatcher; @@ -299,11 +297,7 @@ private void cleanup() { closed = true; if (this.memoryAddress != 0) { - if (IndexUtil.isBinaryIndex(vectorDataType)) { - JNICommons.freeBinaryVectorData(this.memoryAddress); - } else { - JNICommons.freeVectorData(this.memoryAddress); - } + vectorDataType.freeNativeMemory(this.memoryAddress); } } diff --git a/src/main/java/org/opensearch/knn/index/memory/NativeMemoryLoadStrategy.java b/src/main/java/org/opensearch/knn/index/memory/NativeMemoryLoadStrategy.java index 6723c2ed0..8324f2340 100644 --- a/src/main/java/org/opensearch/knn/index/memory/NativeMemoryLoadStrategy.java +++ b/src/main/java/org/opensearch/knn/index/memory/NativeMemoryLoadStrategy.java @@ -14,11 +14,8 @@ import lombok.extern.log4j.Log4j2; import org.opensearch.core.action.ActionListener; import org.opensearch.knn.index.util.IndexUtil; -import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.jni.JNIService; import org.opensearch.knn.index.engine.KNNEngine; -import org.opensearch.knn.training.ByteTrainingDataConsumer; -import org.opensearch.knn.training.FloatTrainingDataConsumer; import org.opensearch.knn.training.TrainingDataConsumer; import org.opensearch.knn.training.VectorReader; import org.opensearch.watcher.FileChangesListener; @@ -174,9 +171,8 @@ public NativeMemoryAllocation.TrainingDataAllocation load( nativeMemoryEntryContext.getVectorDataType() ); - TrainingDataConsumer vectorDataConsumer = nativeMemoryEntryContext.getVectorDataType() == VectorDataType.FLOAT - ? new FloatTrainingDataConsumer(trainingDataAllocation) - : new ByteTrainingDataConsumer(trainingDataAllocation); + TrainingDataConsumer vectorDataConsumer = nativeMemoryEntryContext.getVectorDataType() + .getTrainingDataConsumer(trainingDataAllocation); trainingDataAllocation.writeLock(); diff --git a/src/main/java/org/opensearch/knn/index/util/IndexUtil.java b/src/main/java/org/opensearch/knn/index/util/IndexUtil.java index 8232f84a2..f0d57ddeb 100644 --- a/src/main/java/org/opensearch/knn/index/util/IndexUtil.java +++ b/src/main/java/org/opensearch/knn/index/util/IndexUtil.java @@ -31,6 +31,7 @@ import java.util.HashMap; import java.util.Locale; import java.util.Map; +import java.util.Set; import static org.opensearch.knn.common.KNNConstants.BYTES_PER_KILOBYTES; import static org.opensearch.knn.common.KNNConstants.HNSW_ALGO_EF_SEARCH; @@ -53,6 +54,7 @@ public class IndexUtil { private static final Version MINIMAL_RESCORE_FEATURE = Version.V_2_17_0; // public so neural search can access it public static final Map minimalRequiredVersionMap = initializeMinimalRequiredVersionMap(); + public static final Set VECTOR_DATA_TYPES_NOT_SUPPORTING_ENCODERS = Set.of(VectorDataType.BINARY, VectorDataType.BYTE); /** * Determines the size of a file on disk in kilobytes @@ -146,16 +148,6 @@ public static ValidationException validateKnnField( } if (trainRequestVectorDataType != null) { - if (VectorDataType.BYTE == trainRequestVectorDataType) { - exception.addValidationError( - String.format( - Locale.ROOT, - "vector data type \"%s\" is not supported for training.", - trainRequestVectorDataType.getValue() - ) - ); - return exception; - } VectorDataType trainIndexDataType = getVectorDataTypeFromFieldMapping(fieldMap); if (trainIndexDataType != trainRequestVectorDataType) { @@ -171,20 +163,18 @@ public static ValidationException validateKnnField( return exception; } - // Block binary vector data type for pq encoder + // Block binary and byte vector data type for any encoder if (trainRequestKnnMethodContext != null) { MethodComponentContext methodComponentContext = trainRequestKnnMethodContext.getMethodComponentContext(); Map parameters = methodComponentContext.getParameters(); if (parameters != null && parameters.containsKey(KNNConstants.METHOD_ENCODER_PARAMETER)) { MethodComponentContext encoder = (MethodComponentContext) parameters.get(KNNConstants.METHOD_ENCODER_PARAMETER); - if (encoder != null - && KNNConstants.ENCODER_PQ.equals(encoder.getName()) - && VectorDataType.BINARY == trainRequestVectorDataType) { + if (encoder != null && VECTOR_DATA_TYPES_NOT_SUPPORTING_ENCODERS.contains(trainRequestVectorDataType)) { exception.addValidationError( String.format( Locale.ROOT, - "vector data type \"%s\" is not supported for pq encoder.", + "encoder is not supported for vector data type [%s]", trainRequestVectorDataType.getValue() ) ); @@ -326,16 +316,6 @@ public static boolean isBinaryIndex(KNNEngine knnEngine, Map par && parameters.get(VECTOR_DATA_TYPE_FIELD).toString().equals(VectorDataType.BINARY.getValue()); } - /** - * Tell if it is binary index or not - * - * @param vectorDataType vector data type - * @return true if it is binary index - */ - public static boolean isBinaryIndex(VectorDataType vectorDataType) { - return VectorDataType.BINARY == vectorDataType; - } - /** * Update vector data type into parameters * @@ -346,6 +326,9 @@ public static void updateVectorDataTypeToParameters(Map paramete if (VectorDataType.BINARY == vectorDataType) { parameters.put(VECTOR_DATA_TYPE_FIELD, vectorDataType.getValue()); } + if (VectorDataType.BYTE == vectorDataType) { + parameters.put(VECTOR_DATA_TYPE_FIELD, vectorDataType.getValue()); + } } /** diff --git a/src/main/java/org/opensearch/knn/jni/FaissService.java b/src/main/java/org/opensearch/knn/jni/FaissService.java index 26c703eeb..037171b98 100644 --- a/src/main/java/org/opensearch/knn/jni/FaissService.java +++ b/src/main/java/org/opensearch/knn/jni/FaissService.java @@ -186,6 +186,25 @@ public static native void createBinaryIndexFromTemplate( Map parameters ); + /** + * Create a byte index for the native library with a provided template index + * + * @param ids array of ids mapping to the data passed in + * @param vectorsAddress address of native memory where vectors are stored + * @param dim dimension of the vector to be indexed + * @param indexPath path to save index file to + * @param templateIndex empty template index + * @param parameters additional build time parameters + */ + public static native void createByteIndexFromTemplate( + int[] ids, + long vectorsAddress, + int dim, + String indexPath, + byte[] templateIndex, + Map parameters + ); + /** * Load an index into memory * @@ -349,6 +368,16 @@ public static native KNNQueryResult[] queryBinaryIndexWithFilter( */ public static native byte[] trainBinaryIndex(Map indexParameters, int dimension, long trainVectorsPointer); + /** + * Train an empty byte index + * + * @param indexParameters parameters used to build index + * @param dimension dimension for the index + * @param trainVectorsPointer pointer to where training vectors are stored in native memory + * @return bytes array of trained template index + */ + public static native byte[] trainByteIndex(Map indexParameters, int dimension, long trainVectorsPointer); + /** *

    * The function is deprecated. Use {@link JNICommons#storeVectorData(long, float[][], long)} diff --git a/src/main/java/org/opensearch/knn/jni/JNICommons.java b/src/main/java/org/opensearch/knn/jni/JNICommons.java index df1024db4..df3e551cd 100644 --- a/src/main/java/org/opensearch/knn/jni/JNICommons.java +++ b/src/main/java/org/opensearch/knn/jni/JNICommons.java @@ -178,12 +178,12 @@ public static long storeByteVectorData(long memoryAddress, byte[][] data, long i public static native void freeBinaryVectorData(long memoryAddress); /** - * Free up the memory allocated for the binary data stored in memory address. This function should be used with the memory - * address returned by {@link JNICommons#storeBinaryVectorData(long, byte[][], long)} + * Free up the memory allocated for the byte data stored in memory address. This function should be used with the memory + * address returned by {@link JNICommons#storeByteVectorData(long, byte[][], long)} * *

    - * The function is not threadsafe. If multiple threads are trying to free up same memory location, then it can - * lead to errors. + * The function is not threadsafe. If multiple threads are trying to free up same memory location, then it can + * lead to errors. *

    * * @param memoryAddress address to be freed. diff --git a/src/main/java/org/opensearch/knn/jni/JNIService.java b/src/main/java/org/opensearch/knn/jni/JNIService.java index 1177d635e..94c1ec48e 100644 --- a/src/main/java/org/opensearch/knn/jni/JNIService.java +++ b/src/main/java/org/opensearch/knn/jni/JNIService.java @@ -169,10 +169,15 @@ public static void createIndexFromTemplate( if (IndexUtil.isBinaryIndex(knnEngine, parameters)) { FaissService.createBinaryIndexFromTemplate(ids, vectorsAddress, dim, indexPath, templateIndex, parameters); return; - } else { - FaissService.createIndexFromTemplate(ids, vectorsAddress, dim, indexPath, templateIndex, parameters); + } + if (IndexUtil.isByteIndex(parameters)) { + FaissService.createByteIndexFromTemplate(ids, vectorsAddress, dim, indexPath, templateIndex, parameters); return; } + + FaissService.createIndexFromTemplate(ids, vectorsAddress, dim, indexPath, templateIndex, parameters); + return; + } throw new IllegalArgumentException( @@ -405,6 +410,9 @@ public static byte[] trainIndex(Map indexParameters, int dimensi if (IndexUtil.isBinaryIndex(knnEngine, indexParameters)) { return FaissService.trainBinaryIndex(indexParameters, dimension, trainVectorsPointer); } + if (IndexUtil.isByteIndex(indexParameters)) { + return FaissService.trainByteIndex(indexParameters, dimension, trainVectorsPointer); + } return FaissService.trainIndex(indexParameters, dimension, trainVectorsPointer); } diff --git a/src/main/java/org/opensearch/knn/training/BinaryTrainingDataConsumer.java b/src/main/java/org/opensearch/knn/training/BinaryTrainingDataConsumer.java new file mode 100644 index 000000000..6760afc9f --- /dev/null +++ b/src/main/java/org/opensearch/knn/training/BinaryTrainingDataConsumer.java @@ -0,0 +1,74 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.training; + +import lombok.extern.log4j.Log4j2; +import org.opensearch.action.search.SearchResponse; +import org.opensearch.knn.index.memory.NativeMemoryAllocation; +import org.opensearch.knn.jni.JNICommons; +import org.opensearch.search.SearchHit; + +import java.util.ArrayList; +import java.util.List; + +/** + * Transfers binary vectors from JVM to native memory. + */ +@Log4j2 +public class BinaryTrainingDataConsumer extends TrainingDataConsumer { + + /** + * Constructor + * + * @param trainingDataAllocation NativeMemoryAllocation that contains information about native memory allocation. + */ + public BinaryTrainingDataConsumer(NativeMemoryAllocation.TrainingDataAllocation trainingDataAllocation) { + super(trainingDataAllocation); + } + + @Override + public void accept(List byteVectors) { + long memoryAddress = trainingDataAllocation.getMemoryAddress(); + memoryAddress = JNICommons.storeBinaryVectorData(memoryAddress, byteVectors.toArray(new byte[0][0]), byteVectors.size()); + trainingDataAllocation.setMemoryAddress(memoryAddress); + } + + @Override + public void processTrainingVectors(SearchResponse searchResponse, int vectorsToAdd, String fieldName) { + SearchHit[] hits = searchResponse.getHits().getHits(); + List vectors = new ArrayList<>(); + String[] fieldPath = fieldName.split("\\."); + int nullVectorCount = 0; + + for (int vector = 0; vector < vectorsToAdd; vector++) { + Object fieldValue = extractFieldValue(hits[vector], fieldPath); + if (fieldValue == null) { + nullVectorCount++; + continue; + } + + byte[] byteArray; + if (!(fieldValue instanceof List)) { + continue; + } + List fieldList = (List) fieldValue; + byteArray = new byte[fieldList.size()]; + for (int i = 0; i < fieldList.size(); i++) { + byteArray[i] = fieldList.get(i).byteValue(); + } + + vectors.add(byteArray); + } + + if (nullVectorCount > 0) { + log.warn("Found {} documents with null byte vectors in field {}", nullVectorCount, fieldName); + } + + setTotalVectorsCountAdded(getTotalVectorsCountAdded() + vectors.size()); + + accept(vectors); + } +} diff --git a/src/main/java/org/opensearch/knn/training/ByteTrainingDataConsumer.java b/src/main/java/org/opensearch/knn/training/ByteTrainingDataConsumer.java index e838b5214..c51a96533 100644 --- a/src/main/java/org/opensearch/knn/training/ByteTrainingDataConsumer.java +++ b/src/main/java/org/opensearch/knn/training/ByteTrainingDataConsumer.java @@ -11,11 +11,9 @@ package org.opensearch.knn.training; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; import org.opensearch.action.search.SearchResponse; -import org.opensearch.knn.jni.JNICommons; import org.opensearch.knn.index.memory.NativeMemoryAllocation; +import org.opensearch.knn.jni.JNICommons; import org.opensearch.search.SearchHit; import java.util.ArrayList; @@ -25,7 +23,6 @@ * Transfers byte vectors from JVM to native memory. */ public class ByteTrainingDataConsumer extends TrainingDataConsumer { - private static final Logger logger = LogManager.getLogger(TrainingDataConsumer.class); /** * Constructor @@ -39,7 +36,7 @@ public ByteTrainingDataConsumer(NativeMemoryAllocation.TrainingDataAllocation tr @Override public void accept(List byteVectors) { long memoryAddress = trainingDataAllocation.getMemoryAddress(); - memoryAddress = JNICommons.storeBinaryVectorData(memoryAddress, byteVectors.toArray(new byte[0][0]), byteVectors.size()); + memoryAddress = JNICommons.storeByteVectorData(memoryAddress, byteVectors.toArray(new byte[0][0]), byteVectors.size()); trainingDataAllocation.setMemoryAddress(memoryAddress); } @@ -48,14 +45,9 @@ public void processTrainingVectors(SearchResponse searchResponse, int vectorsToA SearchHit[] hits = searchResponse.getHits().getHits(); List vectors = new ArrayList<>(); String[] fieldPath = fieldName.split("\\."); - int nullVectorCount = 0; for (int vector = 0; vector < vectorsToAdd; vector++) { Object fieldValue = extractFieldValue(hits[vector], fieldPath); - if (fieldValue == null) { - nullVectorCount++; - continue; - } byte[] byteArray; if (!(fieldValue instanceof List)) { @@ -70,10 +62,6 @@ public void processTrainingVectors(SearchResponse searchResponse, int vectorsToA vectors.add(byteArray); } - if (nullVectorCount > 0) { - logger.warn("Found {} documents with null byte vectors in field {}", nullVectorCount, fieldName); - } - setTotalVectorsCountAdded(getTotalVectorsCountAdded() + vectors.size()); accept(vectors); diff --git a/src/test/java/org/opensearch/knn/index/VectorDataTypeIT.java b/src/test/java/org/opensearch/knn/index/VectorDataTypeIT.java index 4979514a8..fcfb192ca 100644 --- a/src/test/java/org/opensearch/knn/index/VectorDataTypeIT.java +++ b/src/test/java/org/opensearch/knn/index/VectorDataTypeIT.java @@ -33,12 +33,23 @@ import static org.opensearch.knn.common.KNNConstants.DIMENSION; import static org.opensearch.knn.common.KNNConstants.ENCODER_SQ; +import static org.opensearch.knn.common.KNNConstants.FAISS_NAME; import static org.opensearch.knn.common.KNNConstants.FAISS_SQ_ENCODER_FP16; import static org.opensearch.knn.common.KNNConstants.FAISS_SQ_TYPE; +import static org.opensearch.knn.common.KNNConstants.KNN_ENGINE; +import static org.opensearch.knn.common.KNNConstants.KNN_METHOD; import static org.opensearch.knn.common.KNNConstants.METHOD_ENCODER_PARAMETER; import static org.opensearch.knn.common.KNNConstants.METHOD_HNSW; +import static org.opensearch.knn.common.KNNConstants.METHOD_IVF; +import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_NLIST; +import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_NPROBES; +import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_SPACE_TYPE; +import static org.opensearch.knn.common.KNNConstants.MODEL_DESCRIPTION; +import static org.opensearch.knn.common.KNNConstants.MODEL_ID; import static org.opensearch.knn.common.KNNConstants.NAME; import static org.opensearch.knn.common.KNNConstants.PARAMETERS; +import static org.opensearch.knn.common.KNNConstants.TRAIN_FIELD_PARAMETER; +import static org.opensearch.knn.common.KNNConstants.TRAIN_INDEX_PARAMETER; import static org.opensearch.knn.common.KNNConstants.VECTOR_DATA_TYPE_FIELD; import static org.opensearch.knn.index.VectorDataType.SUPPORTED_VECTOR_DATA_TYPES; @@ -591,6 +602,93 @@ public void testDocValuesWithByteVectorDataTypeFaissEngine() throws Exception { validateL2SearchResults(response); } + @SneakyThrows + public void testIVFByteVector_whenIndexedAndQueried_thenSucceed() { + + String modelId = "test-model-ivf-byte"; + int dimension = 2; + + // Add training data + String trainIndexMapping = XContentFactory.jsonBuilder() + .startObject() + .startObject("properties") + .startObject(FIELD_NAME) + .field("type", "knn_vector") + .field("dimension", dimension) + .field("data_type", VectorDataType.BYTE.getValue()) + .startObject(KNN_METHOD) + .field(NAME, METHOD_HNSW) + .field(KNN_ENGINE, KNNEngine.FAISS.getName()) + .endObject() + .endObject() + .endObject() + .endObject() + .toString(); + createKnnIndex(INDEX_NAME, trainIndexMapping); + + int trainingDataCount = 100; + bulkIngestRandomByteVectors(INDEX_NAME, FIELD_NAME, trainingDataCount, dimension); + + XContentBuilder trainModelXContentBuilder = XContentFactory.jsonBuilder() + .startObject() + .field(TRAIN_INDEX_PARAMETER, INDEX_NAME) + .field(TRAIN_FIELD_PARAMETER, FIELD_NAME) + .field(DIMENSION, dimension) + .field(MODEL_DESCRIPTION, "My model description") + .field(VECTOR_DATA_TYPE_FIELD, VectorDataType.BYTE.getValue()) + .field( + KNN_METHOD, + Map.of( + NAME, + METHOD_IVF, + KNN_ENGINE, + FAISS_NAME, + METHOD_PARAMETER_SPACE_TYPE, + SpaceType.L2.getValue(), + PARAMETERS, + Map.of(METHOD_PARAMETER_NLIST, 4, METHOD_PARAMETER_NPROBES, 4) + ) + ) + .endObject(); + + trainModel(modelId, trainModelXContentBuilder); + + // Make sure training succeeds after 30 seconds + assertTrainingSucceeds(modelId, 30, 1000); + + // Create knn index from model + String indexName = "test-index-name-ivf-byte"; + String indexMapping = XContentFactory.jsonBuilder() + .startObject() + .startObject("properties") + .startObject(FIELD_NAME) + .field("type", "knn_vector") + .field(MODEL_ID, modelId) + .endObject() + .endObject() + .endObject() + .toString(); + + createKnnIndex(indexName, getKNNDefaultIndexSettings(), indexMapping); + + Byte[] b1 = { 6, 6 }; + addKnnDoc(indexName, "1", FIELD_NAME, b1); + Byte[] b2 = { 2, 2 }; + addKnnDoc(indexName, "2", FIELD_NAME, b2); + Byte[] b3 = { 4, 4 }; + addKnnDoc(indexName, "3", FIELD_NAME, b3); + Byte[] b4 = { 3, 3 }; + addKnnDoc(indexName, "4", FIELD_NAME, b4); + + Byte[] queryVector = { 1, 1 }; + Response response = searchKNNIndex(indexName, new KNNQueryBuilder(FIELD_NAME, convertByteToFloatArray(queryVector), 4), 4); + + validateL2SearchResults(response); + deleteKNNIndex(indexName); + Thread.sleep(45 * 1000); + deleteModel(modelId); + } + @SneakyThrows private void ingestL2ByteTestData() { Byte[] b1 = { 6, 6 }; diff --git a/src/test/java/org/opensearch/knn/index/util/IndexUtilTests.java b/src/test/java/org/opensearch/knn/index/util/IndexUtilTests.java index f2e85b1ad..867028dd8 100644 --- a/src/test/java/org/opensearch/knn/index/util/IndexUtilTests.java +++ b/src/test/java/org/opensearch/knn/index/util/IndexUtilTests.java @@ -29,6 +29,7 @@ import java.util.Collections; import java.util.HashMap; +import java.util.Locale; import java.util.Map; import java.util.Objects; @@ -39,6 +40,7 @@ import static org.mockito.Mockito.mockStatic; import static org.mockito.Mockito.when; import static org.opensearch.knn.common.KNNConstants.ENCODER_PQ; +import static org.opensearch.knn.common.KNNConstants.ENCODER_SQ; import static org.opensearch.knn.common.KNNConstants.HNSW_ALGO_EF_SEARCH; import static org.opensearch.knn.common.KNNConstants.METHOD_ENCODER_PARAMETER; import static org.opensearch.knn.common.KNNConstants.METHOD_IVF; @@ -288,34 +290,33 @@ public void testValidateKnnField_whenTrainModelUseDifferentVectorDataTypeFromTra ); } - public void testValidateKnnField_whenPassByteVectorDataType_thenThrowException() { - Map fieldValues = Map.of("type", "knn_vector", "dimension", 8, "data_type", "byte"); - Map top_level_field = Map.of("top_level_field", fieldValues); - Map properties = Map.of("properties", top_level_field); - String field = "top_level_field"; - int dimension = 8; - - MappingMetadata mappingMetadata = mock(MappingMetadata.class); - when(mappingMetadata.getSourceAsMap()).thenReturn(properties); - IndexMetadata indexMetadata = mock(IndexMetadata.class); - when(indexMetadata.mapping()).thenReturn(mappingMetadata); - ModelDao modelDao = mock(ModelDao.class); - - ValidationException e = IndexUtil.validateKnnField(indexMetadata, field, dimension, modelDao, VectorDataType.BYTE, null); - - assert Objects.requireNonNull(e) - .getMessage() - .matches("Validation Failed: 1: vector data type \"" + VectorDataType.BYTE.getValue() + "\" is not supported for training.;"); - } - public void testUpdateVectorDataTypeToParameters_whenVectorDataTypeIsBinary() { Map indexParams = new HashMap<>(); IndexUtil.updateVectorDataTypeToParameters(indexParams, VectorDataType.BINARY); assertEquals(VectorDataType.BINARY.getValue(), indexParams.get(VECTOR_DATA_TYPE_FIELD)); } - public void testValidateKnnField_whenPassBinaryVectorDataTypeAndPQEncoder_thenThrowException() { - Map fieldValues = Map.of("type", "knn_vector", "dimension", 8, "data_type", "binary", "encoder", "pq"); + public void testValidateKnnField_whenPassBinaryVectorDataTypeAndEncoder_thenThrowException() { + validateKnnField_whenPassVectorDataTypeAndEncoder_thenThrowException(ENCODER_SQ, VectorDataType.BINARY); + validateKnnField_whenPassVectorDataTypeAndEncoder_thenThrowException(ENCODER_PQ, VectorDataType.BINARY); + } + + public void testValidateKnnField_whenPassByteVectorDataTypeAndEncoder_thenThrowException() { + validateKnnField_whenPassVectorDataTypeAndEncoder_thenThrowException(ENCODER_SQ, VectorDataType.BYTE); + validateKnnField_whenPassVectorDataTypeAndEncoder_thenThrowException(ENCODER_PQ, VectorDataType.BYTE); + } + + public void validateKnnField_whenPassVectorDataTypeAndEncoder_thenThrowException(String encoder, VectorDataType vectorDataType) { + Map fieldValues = Map.of( + "type", + "knn_vector", + "dimension", + 8, + "data_type", + vectorDataType.getValue(), + "encoder", + encoder + ); Map top_level_field = Map.of("top_level_field", fieldValues); Map properties = Map.of("properties", top_level_field); String field = "top_level_field"; @@ -326,24 +327,19 @@ public void testValidateKnnField_whenPassBinaryVectorDataTypeAndPQEncoder_thenTh IndexMetadata indexMetadata = mock(IndexMetadata.class); when(indexMetadata.mapping()).thenReturn(mappingMetadata); ModelDao modelDao = mock(ModelDao.class); - MethodComponentContext pq = new MethodComponentContext(ENCODER_PQ, Collections.emptyMap()); KNNMethodContext knnMethodContext = new KNNMethodContext( KNNEngine.FAISS, SpaceType.INNER_PRODUCT, - new MethodComponentContext(METHOD_IVF, ImmutableMap.of(METHOD_ENCODER_PARAMETER, pq)) + new MethodComponentContext( + METHOD_IVF, + ImmutableMap.of(METHOD_ENCODER_PARAMETER, new MethodComponentContext(encoder, Collections.emptyMap())) + ) ); - ValidationException e = IndexUtil.validateKnnField( - indexMetadata, - field, - dimension, - modelDao, - VectorDataType.BINARY, - knnMethodContext - ); + ValidationException e = IndexUtil.validateKnnField(indexMetadata, field, dimension, modelDao, vectorDataType, knnMethodContext); assert Objects.requireNonNull(e) .getMessage() - .matches("Validation Failed: 1: vector data type \"binary\" is not supported for pq encoder.;"); + .contains(String.format(Locale.ROOT, "encoder is not supported for vector data type [%s]", vectorDataType.getValue())); } } diff --git a/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java b/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java index 0c1dbb2ce..a4659f691 100644 --- a/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java +++ b/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java @@ -5,6 +5,7 @@ package org.opensearch.knn; +import com.google.common.primitives.Bytes; import com.google.common.primitives.Floats; import com.google.common.primitives.Ints; import lombok.SneakyThrows; @@ -1167,6 +1168,16 @@ public void bulkIngestRandomBinaryVectors(String indexName, String fieldName, in } } + public void bulkIngestRandomByteVectors(String indexName, String fieldName, int numVectors, int dimension) throws IOException { + for (int i = 0; i < numVectors; i++) { + byte[] vector = new byte[dimension]; + for (int j = 0; j < dimension; j++) { + vector[j] = randomByte(); + } + addKnnDoc(indexName, String.valueOf(i + 1), fieldName, Bytes.asList(vector).toArray()); + } + } + /** * Bulk ingest random vectors with nested field * From 2fb9bb94b885b1a13b93d41c32dfc6c63cf7a3c1 Mon Sep 17 00:00:00 2001 From: Tejas Shah Date: Tue, 3 Sep 2024 14:27:09 -0700 Subject: [PATCH 344/416] Makes NativeEngines990KnnVectorFormat as default for index version (#2020) (#2021) starting 2.17 This removes the feature flag for the same Signed-off-by: Tejas Shah (cherry picked from commit 6b197be39be4dbe96417d9410b858a33f239ca10) --- .../org/opensearch/knn/index/KNNSettings.java | 31 ------------------- .../codec/BasePerFieldKnnVectorsFormat.java | 13 ++++++-- .../mapper/KNNVectorFieldMapperUtil.java | 2 +- .../mapper/KNNVectorFieldMapperUtilTests.java | 15 +-------- 4 files changed, 12 insertions(+), 49 deletions(-) diff --git a/src/main/java/org/opensearch/knn/index/KNNSettings.java b/src/main/java/org/opensearch/knn/index/KNNSettings.java index 0932e3c96..e0123ef8d 100644 --- a/src/main/java/org/opensearch/knn/index/KNNSettings.java +++ b/src/main/java/org/opensearch/knn/index/KNNSettings.java @@ -80,12 +80,6 @@ public class KNNSettings { public static final String MODEL_CACHE_SIZE_LIMIT = "knn.model.cache.size.limit"; public static final String ADVANCED_FILTERED_EXACT_SEARCH_THRESHOLD = "index.knn.advanced.filtered_exact_search_threshold"; public static final String KNN_FAISS_AVX2_DISABLED = "knn.faiss.avx2.disabled"; - /** - * TODO: This setting is only added to ensure that main branch of k_NN plugin doesn't break till other parts of the - * code is getting ready. Will remove this setting once all changes related to integration of KNNVectorsFormat is added - * for native engines. - */ - public static final String KNN_USE_LUCENE_VECTOR_FORMAT_ENABLED = "knn.use.format.enabled"; public static final String QUANTIZATION_STATE_CACHE_SIZE_LIMIT = "knn.quantization.cache.size.limit"; public static final String QUANTIZATION_STATE_CACHE_EXPIRY_TIME_MINUTES = "knn.quantization.cache.expiry.minutes"; @@ -266,17 +260,6 @@ public class KNNSettings { NodeScope ); - /** - * TODO: This setting is only added to ensure that main branch of k_NN plugin doesn't break till other parts of the - * code is getting ready. Will remove this setting once all changes related to integration of KNNVectorsFormat is added - * for native engines. - */ - public static final Setting KNN_USE_LUCENE_VECTOR_FORMAT_ENABLED_SETTING = Setting.boolSetting( - KNN_USE_LUCENE_VECTOR_FORMAT_ENABLED, - false, - NodeScope - ); - /* * Quantization state cache settings */ @@ -439,10 +422,6 @@ private Setting getSetting(String key) { return KNN_VECTOR_STREAMING_MEMORY_LIMIT_PCT_SETTING; } - if (KNN_USE_LUCENE_VECTOR_FORMAT_ENABLED.equals(key)) { - return KNN_USE_LUCENE_VECTOR_FORMAT_ENABLED_SETTING; - } - if (QUANTIZATION_STATE_CACHE_SIZE_LIMIT.equals(key)) { return QUANTIZATION_STATE_CACHE_SIZE_LIMIT_SETTING; } @@ -470,7 +449,6 @@ public List> getSettings() { ADVANCED_FILTERED_EXACT_SEARCH_THRESHOLD_SETTING, KNN_FAISS_AVX2_DISABLED_SETTING, KNN_VECTOR_STREAMING_MEMORY_LIMIT_PCT_SETTING, - KNN_USE_LUCENE_VECTOR_FORMAT_ENABLED_SETTING, QUANTIZATION_STATE_CACHE_SIZE_LIMIT_SETTING, QUANTIZATION_STATE_CACHE_EXPIRY_TIME_MINUTES_SETTING ); @@ -517,15 +495,6 @@ public static Integer getFilteredExactSearchThreshold(final String indexName) { .getAsInt(ADVANCED_FILTERED_EXACT_SEARCH_THRESHOLD, ADVANCED_FILTERED_EXACT_SEARCH_THRESHOLD_DEFAULT_VALUE); } - /** - * TODO: This setting is only added to ensure that main branch of k_NN plugin doesn't break till other parts of the - * code is getting ready. Will remove this setting once all changes related to integration of KNNVectorsFormat is added - * for native engines. - */ - public static boolean getIsLuceneVectorFormatEnabled() { - return KNNSettings.state().getSettingValue(KNNSettings.KNN_USE_LUCENE_VECTOR_FORMAT_ENABLED); - } - public void initialize(Client client, ClusterService clusterService) { this.client = client; this.clusterService = clusterService; diff --git a/src/main/java/org/opensearch/knn/index/codec/BasePerFieldKnnVectorsFormat.java b/src/main/java/org/opensearch/knn/index/codec/BasePerFieldKnnVectorsFormat.java index 8beced605..b06c2a1d8 100644 --- a/src/main/java/org/opensearch/knn/index/codec/BasePerFieldKnnVectorsFormat.java +++ b/src/main/java/org/opensearch/knn/index/codec/BasePerFieldKnnVectorsFormat.java @@ -78,10 +78,13 @@ public KnnVectorsFormat getKnnVectorsFormatForField(final String field) { ) ).fieldType(field); - KNNMappingConfig knnMappingConfig = mappedFieldType.getKnnMappingConfig(); - KNNMethodContext knnMethodContext = knnMappingConfig.getKnnMethodContext() - .orElseThrow(() -> new IllegalArgumentException("KNN method context cannot be empty")); + final KNNMappingConfig knnMappingConfig = mappedFieldType.getKnnMappingConfig(); + if (knnMappingConfig.getModelId().isPresent()) { + return nativeEngineVectorsFormat(); + } + final KNNMethodContext knnMethodContext = knnMappingConfig.getKnnMethodContext() + .orElseThrow(() -> new IllegalArgumentException("KNN method context cannot be empty")); final KNNEngine engine = knnMethodContext.getKnnEngine(); final Map params = knnMethodContext.getMethodComponentContext().getParameters(); @@ -122,6 +125,10 @@ public KnnVectorsFormat getKnnVectorsFormatForField(final String field) { } // All native engines to use NativeEngines990KnnVectorsFormat + return nativeEngineVectorsFormat(); + } + + private NativeEngines990KnnVectorsFormat nativeEngineVectorsFormat() { return new NativeEngines990KnnVectorsFormat(new Lucene99FlatVectorsFormat(FlatVectorScorerUtil.getLucene99FlatVectorsScorer())); } diff --git a/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperUtil.java b/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperUtil.java index 5ab2dd888..551905793 100644 --- a/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperUtil.java +++ b/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperUtil.java @@ -143,7 +143,7 @@ static void validateIfKNNPluginEnabled() { * @return true if vector field should use KNNVectorsFormat */ static boolean useLuceneKNNVectorsFormat(final Version indexCreatedVersion) { - return indexCreatedVersion.onOrAfter(Version.V_2_17_0) && KNNSettings.getIsLuceneVectorFormatEnabled(); + return indexCreatedVersion.onOrAfter(Version.V_2_17_0); } private static SpaceType getSpaceType(final Settings indexSettings) { diff --git a/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperUtilTests.java b/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperUtilTests.java index 740d75206..d0fa150a5 100644 --- a/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperUtilTests.java +++ b/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperUtilTests.java @@ -14,11 +14,8 @@ import org.apache.lucene.document.StoredField; import org.apache.lucene.util.BytesRef; import org.junit.Assert; -import org.mockito.MockedStatic; -import org.mockito.Mockito; import org.opensearch.Version; import org.opensearch.knn.KNNTestCase; -import org.opensearch.knn.index.KNNSettings; import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.codec.util.KNNVectorSerializerFactory; @@ -76,18 +73,8 @@ public void testGetExpectedVectorLengthSuccess() { } public void testUseLuceneKNNVectorsFormat_withDifferentInputs_thenSuccess() { - final KNNSettings knnSettings = mock(KNNSettings.class); - final MockedStatic mockedStatic = Mockito.mockStatic(KNNSettings.class); - mockedStatic.when(KNNSettings::state).thenReturn(knnSettings); - - mockedStatic.when(KNNSettings::getIsLuceneVectorFormatEnabled).thenReturn(false); Assert.assertFalse(KNNVectorFieldMapperUtil.useLuceneKNNVectorsFormat(Version.V_2_16_0)); - Assert.assertFalse(KNNVectorFieldMapperUtil.useLuceneKNNVectorsFormat(Version.V_2_17_0)); - - mockedStatic.when(KNNSettings::getIsLuceneVectorFormatEnabled).thenReturn(true); Assert.assertTrue(KNNVectorFieldMapperUtil.useLuceneKNNVectorsFormat(Version.V_2_17_0)); - // making sure to close the static mock to ensure that for tests running on this thread are not impacted by - // this mocking - mockedStatic.close(); + Assert.assertTrue(KNNVectorFieldMapperUtil.useLuceneKNNVectorsFormat(Version.V_3_0_0)); } } From 2bb26e5cbc0027dfd0c04189dd64d051717eb936 Mon Sep 17 00:00:00 2001 From: Tejas Shah Date: Tue, 3 Sep 2024 18:19:21 -0700 Subject: [PATCH 345/416] Fixes the build (#2024) This was merged https://github.com/opensearch-project/k-NN/pull/2021 Signed-off-by: Tejas Shah --- .../knn/index/mapper/KNNVectorFieldMapperUtilTests.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperUtilTests.java b/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperUtilTests.java index d0fa150a5..3025c9bd1 100644 --- a/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperUtilTests.java +++ b/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperUtilTests.java @@ -75,6 +75,5 @@ public void testGetExpectedVectorLengthSuccess() { public void testUseLuceneKNNVectorsFormat_withDifferentInputs_thenSuccess() { Assert.assertFalse(KNNVectorFieldMapperUtil.useLuceneKNNVectorsFormat(Version.V_2_16_0)); Assert.assertTrue(KNNVectorFieldMapperUtil.useLuceneKNNVectorsFormat(Version.V_2_17_0)); - Assert.assertTrue(KNNVectorFieldMapperUtil.useLuceneKNNVectorsFormat(Version.V_3_0_0)); } } From b04bb3a76885ffb34cd28234eaf7647fb55193c5 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Tue, 3 Sep 2024 18:51:05 -0700 Subject: [PATCH 346/416] Adds graph build time metrics in NativeEngines990KnnVectorsWriter (#2018) (#2022) Signed-off-by: Tejas Shah (cherry picked from commit e6c5953933f38e0f804ad888e0dca2c55e0410ea) Co-authored-by: Tejas Shah --- .../NativeEngines990KnnVectorsWriter.java | 29 +++++++++++++++++-- ...NativeEngines990KnnVectorsFormatTests.java | 5 ++++ 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsWriter.java b/src/main/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsWriter.java index 2c110fb79..1d3ff368a 100644 --- a/src/main/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsWriter.java +++ b/src/main/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsWriter.java @@ -24,11 +24,13 @@ import org.apache.lucene.index.Sorter; import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.RamUsageEstimator; +import org.opensearch.common.StopWatch; import org.opensearch.knn.index.quantizationservice.QuantizationService; import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.codec.nativeindex.NativeIndexWriter; import org.opensearch.knn.index.vectorvalues.KNNVectorValues; import org.opensearch.knn.index.vectorvalues.KNNVectorValuesFactory; +import org.opensearch.knn.plugin.stats.KNNGraphValue; import org.opensearch.knn.quantization.models.quantizationParams.QuantizationParams; import org.opensearch.knn.quantization.models.quantizationState.QuantizationState; @@ -45,6 +47,10 @@ @RequiredArgsConstructor public class NativeEngines990KnnVectorsWriter extends KnnVectorsWriter { private static final long SHALLOW_SIZE = RamUsageEstimator.shallowSizeOfInstance(NativeEngines990KnnVectorsWriter.class); + + private static final String FLUSH_OPERATION = "flush"; + private static final String MERGE_OPERATION = "merge"; + private final SegmentWriteState segmentWriteState; private final FlatVectorsWriter flatVectorsWriter; private final List> fields = new ArrayList<>(); @@ -78,7 +84,9 @@ public void flush(int maxDoc, final Sorter.DocMap sortMap) throws IOException { field.getFieldInfo(), (vectorDataType, fieldInfo, fieldVectorsWriter) -> getKNNVectorValues(vectorDataType, fieldVectorsWriter), NativeIndexWriter::flushIndex, - field + field, + KNNGraphValue.REFRESH_TOTAL_TIME_IN_MILLIS, + FLUSH_OPERATION ); } } @@ -88,7 +96,14 @@ public void mergeOneField(final FieldInfo fieldInfo, final MergeState mergeState // This will ensure that we are merging the FlatIndex during force merge. flatVectorsWriter.mergeOneField(fieldInfo, mergeState); // For merge, pick values from flat vector and reindex again. This will use the flush operation to create graphs - trainAndIndex(fieldInfo, this::getKNNVectorValuesForMerge, NativeIndexWriter::mergeIndex, mergeState); + trainAndIndex( + fieldInfo, + this::getKNNVectorValuesForMerge, + NativeIndexWriter::mergeIndex, + mergeState, + KNNGraphValue.MERGE_TOTAL_TIME_IN_MILLIS, + MERGE_OPERATION + ); } @@ -214,7 +229,9 @@ private void trainAndIndex( final FieldInfo fieldInfo, final VectorValuesRetriever> vectorValuesRetriever, final IndexOperation indexOperation, - final C VectorProcessingContext + final C VectorProcessingContext, + final KNNGraphValue graphBuildTime, + final String operationName ) throws IOException { final VectorDataType vectorDataType = extractVectorDataType(fieldInfo); KNNVectorValues knnVectorValues = vectorValuesRetriever.apply(vectorDataType, fieldInfo, VectorProcessingContext); @@ -228,6 +245,12 @@ private void trainAndIndex( : NativeIndexWriter.getWriter(fieldInfo, segmentWriteState); knnVectorValues = vectorValuesRetriever.apply(vectorDataType, fieldInfo, VectorProcessingContext); + + StopWatch stopWatch = new StopWatch(); + stopWatch.start(); indexOperation.buildAndWrite(writer, knnVectorValues); + long time_in_millis = stopWatch.totalTime().millis(); + graphBuildTime.incrementBy(time_in_millis); + log.warn("Graph build took " + time_in_millis + " ms for " + operationName); } } diff --git a/src/test/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsFormatTests.java b/src/test/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsFormatTests.java index 85b5d07e6..7bf17ec54 100644 --- a/src/test/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsFormatTests.java +++ b/src/test/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsFormatTests.java @@ -51,6 +51,7 @@ import org.opensearch.knn.index.engine.qframe.QuantizationConfigParser; import org.opensearch.knn.index.mapper.KNNVectorFieldMapper; import org.opensearch.knn.index.engine.KNNEngine; +import org.opensearch.knn.plugin.stats.KNNGraphValue; import org.opensearch.knn.quantization.enums.ScalarQuantizationType; import java.io.IOException; @@ -135,6 +136,8 @@ public void testNativeEngineVectorFormat_whenMultipleVectorFieldIndexed_thenSucc indexWriter.commit(); indexWriter.close(); + assertNotEquals(0L, (long) KNNGraphValue.REFRESH_TOTAL_TIME_IN_MILLIS.getValue()); + // Validate to see if correct values are returned, assumption here is only 1 segment is getting created IndexSearcher searcher = new IndexSearcher(indexReader); final LeafReader leafReader = searcher.getLeafContexts().get(0).reader(); @@ -204,6 +207,8 @@ public void testNativeEngineVectorFormat_whenBinaryQuantizationApplied_thenSucce indexWriter.flush(); indexWriter.commit(); indexWriter.close(); + assertNotEquals(0L, (long) KNNGraphValue.REFRESH_TOTAL_TIME_IN_MILLIS.getValue()); + IndexSearcher searcher = new IndexSearcher(indexReader); final LeafReader leafReader = searcher.getLeafContexts().get(0).reader(); SegmentReader segmentReader = Lucene.segmentReader(leafReader); From d48b6edcf3fabcdb0528c5e16c33c89905553474 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Tue, 3 Sep 2024 18:54:18 -0700 Subject: [PATCH 347/416] Fixing the format name for NativeEngines990KnnVectorsFormat (#2025) (#2026) Signed-off-by: Navneet Verma (cherry picked from commit 8a353422ce681a83a05f26af20817f14d42b2b3e) Co-authored-by: Navneet Verma --- .../KNN990Codec/NativeEngines990KnnVectorsFormat.java | 2 +- .../NativeEngines990KnnVectorsFormatTests.java | 11 +++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsFormat.java b/src/main/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsFormat.java index 6582cdd1f..626210f25 100644 --- a/src/main/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsFormat.java +++ b/src/main/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsFormat.java @@ -29,7 +29,7 @@ public class NativeEngines990KnnVectorsFormat extends KnnVectorsFormat { /** The format for storing, reading, merging vectors on disk */ private static FlatVectorsFormat flatVectorsFormat; - private static final String FORMAT_NAME = "NativeEngines99KnnVectorsFormat"; + private static final String FORMAT_NAME = "NativeEngines990KnnVectorsFormat"; public NativeEngines990KnnVectorsFormat() { super(FORMAT_NAME); diff --git a/src/test/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsFormatTests.java b/src/test/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsFormatTests.java index 7bf17ec54..f4f02ba91 100644 --- a/src/test/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsFormatTests.java +++ b/src/test/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsFormatTests.java @@ -14,6 +14,7 @@ import lombok.SneakyThrows; import lombok.extern.log4j.Log4j2; import org.apache.lucene.codecs.Codec; +import org.apache.lucene.codecs.hnsw.FlatVectorScorerUtil; import org.apache.lucene.codecs.hnsw.FlatVectorsReader; import org.apache.lucene.codecs.hnsw.FlatVectorsWriter; import org.apache.lucene.codecs.lucene99.Lucene99FlatVectorsFormat; @@ -226,6 +227,16 @@ public void testNativeEngineVectorFormat_whenBinaryQuantizationApplied_thenSucce indexReader.close(); } + public void testFormatName_withValidInput_thenSuccess() { + final String validFormatName = "NativeEngines990KnnVectorsFormat"; + Assert.assertEquals(validFormatName, new NativeEngines990KnnVectorsFormat().getName()); + Assert.assertEquals( + validFormatName, + new NativeEngines990KnnVectorsFormat(new Lucene99FlatVectorsFormat(FlatVectorScorerUtil.getLucene99FlatVectorsScorer())) + .getName() + ); + } + private List getFilesFromSegment(Directory dir, String fileFormat) throws IOException { return Arrays.stream(dir.listAll()).filter(x -> x.contains(fileFormat)).collect(Collectors.toList()); } From 10c5e29e4eb4658244a3d11f2445687eb239e0a7 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Tue, 3 Sep 2024 19:35:49 -0700 Subject: [PATCH 348/416] Introduce compression and mode mapping parms (#2028) Introduces new params for mapping and training, called compression_level and mode. These parameters are high level parameters that give the plugin a hint as to what the user wants to configure their system like without exposing algorithmic details. This change just adds these parameters to the plugin as noops. In future change, we will add the functionality for parameter resolution. Along with this, I added a class to more easily manage the original parameters that a user passes. This will help ensure our mapper maintains good compatibility. Signed-off-by: John Mazanec (cherry picked from commit 920c8198bf917706579b8f104faaad016d57dc74) --- .../opensearch/knn/common/KNNConstants.java | 4 + .../index/engine/KNNMethodConfigContext.java | 24 +- .../knn/index/mapper/CompressionLevel.java | 89 +++++ .../index/mapper/FlatVectorFieldMapper.java | 21 +- .../index/mapper/KNNVectorFieldMapper.java | 121 +++---- .../knn/index/mapper/LuceneFieldMapper.java | 13 +- .../knn/index/mapper/MethodFieldMapper.java | 17 +- .../org/opensearch/knn/index/mapper/Mode.java | 68 ++++ .../knn/index/mapper/ModelFieldMapper.java | 26 +- .../mapper/OriginalMappingParameters.java | 79 +++++ .../opensearch/knn/index/util/IndexUtil.java | 2 + .../org/opensearch/knn/indices/ModelDao.java | 9 +- .../opensearch/knn/indices/ModelMetadata.java | 64 +++- .../plugin/rest/RestTrainModelHandler.java | 14 +- .../transport/TrainingModelRequest.java | 22 +- .../TrainingModelTransportAction.java | 4 +- .../opensearch/knn/training/TrainingJob.java | 10 +- src/main/resources/mappings/model-index.json | 6 + .../index/KNNCreateIndexFromModelTests.java | 6 +- .../KNN80DocValuesConsumerTests.java | 6 +- .../knn/index/codec/KNNCodecTestCase.java | 6 +- .../index/mapper/CompressionLevelTests.java | 42 +++ .../mapper/KNNVectorFieldMapperTests.java | 194 +++++++++-- .../knn/index/mapper/ModeTests.java | 33 ++ .../OriginalMappingParametersTests.java | 38 +++ .../knn/indices/ModelCacheTests.java | 50 ++- .../opensearch/knn/indices/ModelDaoTests.java | 58 +++- .../knn/indices/ModelMetadataTests.java | 317 +++++++++++++++--- .../opensearch/knn/indices/ModelTests.java | 68 +++- .../knn/integ/ModeAndCompressionIT.java | 155 +++++++++ .../transport/GetModelResponseTests.java | 6 +- ...oveModelFromCacheTransportActionTests.java | 6 +- ...TrainingJobRouterTransportActionTests.java | 14 +- .../transport/TrainingModelRequestTests.java | 91 ++++- .../TrainingModelTransportActionTests.java | 6 +- ...ateModelGraveyardTransportActionTests.java | 6 +- .../UpdateModelMetadataRequestTests.java | 14 +- ...dateModelMetadataTransportActionTests.java | 6 +- .../knn/training/TrainingJobTests.java | 34 +- 39 files changed, 1491 insertions(+), 258 deletions(-) create mode 100644 src/main/java/org/opensearch/knn/index/mapper/CompressionLevel.java create mode 100644 src/main/java/org/opensearch/knn/index/mapper/Mode.java create mode 100644 src/main/java/org/opensearch/knn/index/mapper/OriginalMappingParameters.java create mode 100644 src/test/java/org/opensearch/knn/index/mapper/CompressionLevelTests.java create mode 100644 src/test/java/org/opensearch/knn/index/mapper/ModeTests.java create mode 100644 src/test/java/org/opensearch/knn/index/mapper/OriginalMappingParametersTests.java create mode 100644 src/test/java/org/opensearch/knn/integ/ModeAndCompressionIT.java diff --git a/src/main/java/org/opensearch/knn/common/KNNConstants.java b/src/main/java/org/opensearch/knn/common/KNNConstants.java index 4c1dc6e64..b9319a434 100644 --- a/src/main/java/org/opensearch/knn/common/KNNConstants.java +++ b/src/main/java/org/opensearch/knn/common/KNNConstants.java @@ -71,6 +71,7 @@ public class KNNConstants { public static final String VECTOR_DATA_TYPE_FIELD = "data_type"; public static final String MODEL_VECTOR_DATA_TYPE_KEY = VECTOR_DATA_TYPE_FIELD; public static final VectorDataType DEFAULT_VECTOR_DATA_TYPE_FIELD = VectorDataType.FLOAT; + public static final String MINIMAL_MODE_AND_COMPRESSION_FEATURE = "mode_and_compression_feature"; public static final String RADIAL_SEARCH_KEY = "radial_search"; @@ -149,4 +150,7 @@ public class KNNConstants { public static final Float DEFAULT_LUCENE_RADIAL_SEARCH_TRAVERSAL_SIMILARITY_RATIO = 0.95f; public static final String MIN_SCORE = "min_score"; public static final String MAX_DISTANCE = "max_distance"; + + public static final String MODE_PARAMETER = "mode"; + public static final String COMPRESSION_LEVEL_PARAMETER = "compression_level"; } diff --git a/src/main/java/org/opensearch/knn/index/engine/KNNMethodConfigContext.java b/src/main/java/org/opensearch/knn/index/engine/KNNMethodConfigContext.java index 731085f0b..1ba2777dd 100644 --- a/src/main/java/org/opensearch/knn/index/engine/KNNMethodConfigContext.java +++ b/src/main/java/org/opensearch/knn/index/engine/KNNMethodConfigContext.java @@ -7,10 +7,9 @@ import lombok.AllArgsConstructor; import lombok.Builder; +import lombok.EqualsAndHashCode; import lombok.Getter; import lombok.Setter; -import org.apache.commons.lang.builder.EqualsBuilder; -import org.apache.commons.lang.builder.HashCodeBuilder; import org.opensearch.Version; import org.opensearch.knn.index.VectorDataType; @@ -23,29 +22,10 @@ @Getter @Builder @AllArgsConstructor +@EqualsAndHashCode public final class KNNMethodConfigContext { private VectorDataType vectorDataType; private Integer dimension; private Version versionCreated; - - @Override - public boolean equals(Object obj) { - if (this == obj) return true; - if (obj == null || getClass() != obj.getClass()) return false; - KNNMethodConfigContext other = (KNNMethodConfigContext) obj; - - EqualsBuilder equalsBuilder = new EqualsBuilder(); - equalsBuilder.append(vectorDataType, other.vectorDataType); - equalsBuilder.append(dimension, other.dimension); - equalsBuilder.append(versionCreated, other.versionCreated); - - return equalsBuilder.isEquals(); - } - - @Override - public int hashCode() { - return new HashCodeBuilder().append(vectorDataType).append(dimension).append(versionCreated).toHashCode(); - } - public static final KNNMethodConfigContext EMPTY = KNNMethodConfigContext.builder().build(); } diff --git a/src/main/java/org/opensearch/knn/index/mapper/CompressionLevel.java b/src/main/java/org/opensearch/knn/index/mapper/CompressionLevel.java new file mode 100644 index 000000000..b5ce81af9 --- /dev/null +++ b/src/main/java/org/opensearch/knn/index/mapper/CompressionLevel.java @@ -0,0 +1,89 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.mapper; + +import lombok.AllArgsConstructor; +import lombok.Getter; +import org.opensearch.core.common.Strings; + +import java.util.Arrays; +import java.util.Locale; +import java.util.stream.Collectors; + +/** + * Enum representing the compression level for float vectors. Compression in this sense refers to compressing a + * full precision value into a smaller number of bits. For instance. "16x" compression would mean that 2 bits would + * need to be used to represent a 32-bit floating point number. + */ +@AllArgsConstructor +public enum CompressionLevel { + NOT_CONFIGURED(-1, ""), + x1(1, "1x"), + x2(2, "2x"), + x4(4, "4x"), + x8(8, "8x"), + x16(16, "16x"), + x32(32, "32x"); + + // Internally, an empty string is easier to deal with them null. However, from the mapping, + // we do not want users to pass in the empty string and instead want null. So we make the conversion herex + static final String[] NAMES_ARRAY = Arrays.stream(CompressionLevel.values()) + .map(compressionLevel -> compressionLevel == NOT_CONFIGURED ? null : compressionLevel.getName()) + .collect(Collectors.toList()) + .toArray(new String[0]); + + /** + * Default is set to 1x and is a noop + */ + private static final CompressionLevel DEFAULT = x1; + + /** + * Get the compression level from a string representation. The format for the string should be "Nx", where N is + * the factor by which compression should take place + * + * @param name String representation of the compression level + * @return CompressionLevel enum value + */ + public static CompressionLevel fromName(String name) { + if (Strings.isEmpty(name)) { + return NOT_CONFIGURED; + } + for (CompressionLevel config : CompressionLevel.values()) { + if (config.getName() != null && config.getName().equals(name)) { + return config; + } + } + throw new IllegalArgumentException(String.format(Locale.ROOT, "Invalid compression level: \"[%s]\"", name)); + } + + private final int compressionLevel; + @Getter + private final String name; + + /** + * Gets the number of bits used to represent a float in order to achieve this compression. For instance, for + * 32x compression, each float would need to be encoded in a single bit. + * + * @return number of bits to represent a float at this compression level + */ + public int numBitsForFloat32() { + if (this == NOT_CONFIGURED) { + return DEFAULT.numBitsForFloat32(); + } + + return (Float.BYTES * Byte.SIZE) / compressionLevel; + } + + /** + * Utility method that checks if compression is configured. + * + * @param compressionLevel Compression to check + * @return true if compression is configured, false otherwise + */ + public static boolean isConfigured(CompressionLevel compressionLevel) { + return compressionLevel != null && compressionLevel != NOT_CONFIGURED; + } +} diff --git a/src/main/java/org/opensearch/knn/index/mapper/FlatVectorFieldMapper.java b/src/main/java/org/opensearch/knn/index/mapper/FlatVectorFieldMapper.java index d37ab9b86..8da41aa59 100644 --- a/src/main/java/org/opensearch/knn/index/mapper/FlatVectorFieldMapper.java +++ b/src/main/java/org/opensearch/knn/index/mapper/FlatVectorFieldMapper.java @@ -31,7 +31,8 @@ public static FlatVectorFieldMapper createFieldMapper( CopyTo copyTo, Explicit ignoreMalformed, boolean stored, - boolean hasDocValues + boolean hasDocValues, + OriginalMappingParameters originalMappingParameters ) { final KNNVectorFieldType mappedFieldType = new KNNVectorFieldType( fullname, @@ -47,7 +48,8 @@ public static FlatVectorFieldMapper createFieldMapper( ignoreMalformed, stored, hasDocValues, - knnMethodConfigContext.getVersionCreated() + knnMethodConfigContext.getVersionCreated(), + originalMappingParameters ); } @@ -59,9 +61,20 @@ private FlatVectorFieldMapper( Explicit ignoreMalformed, boolean stored, boolean hasDocValues, - Version indexCreatedVersion + Version indexCreatedVersion, + OriginalMappingParameters originalMappingParameters ) { - super(simpleName, mappedFieldType, multiFields, copyTo, ignoreMalformed, stored, hasDocValues, indexCreatedVersion, null); + super( + simpleName, + mappedFieldType, + multiFields, + copyTo, + ignoreMalformed, + stored, + hasDocValues, + indexCreatedVersion, + originalMappingParameters + ); // setting it explicitly false here to ensure that when flatmapper is used Lucene based Vector field is not created. this.useLuceneBasedVectorField = false; this.perDimensionValidator = selectPerDimensionValidator(vectorDataType); diff --git a/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapper.java b/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapper.java index 65c3cfb66..0eab5a7bb 100644 --- a/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapper.java +++ b/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapper.java @@ -104,13 +104,7 @@ public static class Builder extends ParametrizedFieldMapper.Builder { } return value; }, - m -> { - KNNMappingConfig knnMappingConfig = toType(m).fieldType().getKnnMappingConfig(); - if (knnMappingConfig.getModelId().isPresent()) { - return UNSET_MODEL_DIMENSION_IDENTIFIER; - } - return knnMappingConfig.getDimension(); - } + m -> toType(m).originalMappingParameters.getDimension() ); /** @@ -122,7 +116,7 @@ public static class Builder extends ParametrizedFieldMapper.Builder { false, () -> DEFAULT_VECTOR_DATA_TYPE_FIELD, (n, c, o) -> VectorDataType.get((String) o), - m -> toType(m).vectorDataType + m -> toType(m).originalMappingParameters.getVectorDataType() ); /** @@ -133,7 +127,7 @@ public static class Builder extends ParametrizedFieldMapper.Builder { protected final Parameter modelId = Parameter.stringParam( KNNConstants.MODEL_ID, false, - m -> toType(m).fieldType().getKnnMappingConfig().getModelId().orElse(null), + m -> toType(m).originalMappingParameters.getModelId(), null ); @@ -146,7 +140,7 @@ public static class Builder extends ParametrizedFieldMapper.Builder { false, () -> null, (n, c, o) -> KNNMethodContext.parse(o), - m -> toType(m).originalKNNMethodContext + m -> toType(m).originalMappingParameters.getKnnMethodContext() ).setSerializer(((b, n, v) -> { b.startObject(n); v.toXContent(b, ToXContent.EMPTY_PARAMS); @@ -162,48 +156,47 @@ public static class Builder extends ParametrizedFieldMapper.Builder { } }); + protected final Parameter mode = Parameter.restrictedStringParam( + KNNConstants.MODE_PARAMETER, + false, + m -> toType(m).originalMappingParameters.getMode(), + Mode.NAMES_ARRAY + ).acceptsNull(); + + protected final Parameter compressionLevel = Parameter.restrictedStringParam( + KNNConstants.COMPRESSION_LEVEL_PARAMETER, + false, + m -> toType(m).originalMappingParameters.getCompressionLevel(), + CompressionLevel.NAMES_ARRAY + ).acceptsNull(); + protected final Parameter> meta = Parameter.metaParam(); protected ModelDao modelDao; protected Version indexCreatedVersion; - // KNNMethodContext that allows us to properly configure a KNNVectorFieldMapper from another - // KNNVectorFieldMapper. To support our legacy field mapping, on parsing, if index.knn=true and no method is - // passed, we build a KNNMethodContext using the space type, ef_construction and m that are set in the index - // settings. However, for fieldmappers for merging, we need to be able to initialize one field mapper from - // another (see - // https://github.com/opensearch-project/OpenSearch/blob/2.16.0/server/src/main/java/org/opensearch/index/mapper/ParametrizedFieldMapper.java#L98). - // The problem is that in this case, the settings are set to empty so we cannot properly resolve the KNNMethodContext. - // (see - // https://github.com/opensearch-project/OpenSearch/blob/2.16.0/server/src/main/java/org/opensearch/index/mapper/ParametrizedFieldMapper.java#L130). - // While we could override the KNNMethodContext parameter initializer to set the knnMethodContext based on the - // constructed KNNMethodContext from the other field mapper, this can result in merge conflict/serialization - // exceptions. See - // (https://github.com/opensearch-project/OpenSearch/blob/2.16.0/server/src/main/java/org/opensearch/index/mapper/ParametrizedFieldMapper.java#L322-L324). - // So, what we do is pass in a "resolvedKNNMethodContext" that will either be null or be set via the merge builder - // constructor. A similar approach was taken for https://github.com/opendistro-for-elasticsearch/k-NN/issues/288 - @Setter - @Getter - private KNNMethodContext resolvedKNNMethodContext; @Setter private KNNMethodConfigContext knnMethodConfigContext; + @Setter + @Getter + private OriginalMappingParameters originalParameters; public Builder( String name, ModelDao modelDao, Version indexCreatedVersion, - KNNMethodContext resolvedKNNMethodContext, - KNNMethodConfigContext knnMethodConfigContext + KNNMethodConfigContext knnMethodConfigContext, + OriginalMappingParameters originalParameters ) { super(name); this.modelDao = modelDao; this.indexCreatedVersion = indexCreatedVersion; - this.resolvedKNNMethodContext = resolvedKNNMethodContext; this.knnMethodConfigContext = knnMethodConfigContext; + this.originalParameters = originalParameters; } @Override protected List> getParameters() { - return Arrays.asList(stored, hasDocValues, dimension, vectorDataType, meta, knnMethodContext, modelId); + return Arrays.asList(stored, hasDocValues, dimension, vectorDataType, meta, knnMethodContext, modelId, mode, compressionLevel); } protected Explicit ignoreMalformed(BuilderContext context) { @@ -231,18 +224,18 @@ public KNNVectorFieldMapper build(BuilderContext context) { name, metaValue, vectorDataType.getValue(), - modelId.get(), multiFieldsBuilder, copyToBuilder, ignoreMalformed, stored.get(), hasDocValues.get(), modelDao, - indexCreatedVersion + indexCreatedVersion, + originalParameters ); } - if (resolvedKNNMethodContext == null) { + if (originalParameters.getResolvedKnnMethodContext() == null) { return FlatVectorFieldMapper.createFieldMapper( buildFullName(context), name, @@ -256,11 +249,12 @@ public KNNVectorFieldMapper build(BuilderContext context) { copyToBuilder, ignoreMalformed, stored.get(), - hasDocValues.get() + hasDocValues.get(), + originalParameters ); } - if (resolvedKNNMethodContext.getKnnEngine() == KNNEngine.LUCENE) { + if (originalParameters.getResolvedKnnMethodContext().getKnnEngine() == KNNEngine.LUCENE) { log.debug(String.format(Locale.ROOT, "Use [LuceneFieldMapper] mapper for field [%s]", name)); LuceneFieldMapper.CreateLuceneFieldMapperInput createLuceneFieldMapperInput = LuceneFieldMapper.CreateLuceneFieldMapperInput .builder() @@ -275,9 +269,9 @@ public KNNVectorFieldMapper build(BuilderContext context) { return LuceneFieldMapper.createFieldMapper( buildFullName(context), metaValue, - resolvedKNNMethodContext, knnMethodConfigContext, - createLuceneFieldMapperInput + createLuceneFieldMapperInput, + originalParameters ); } @@ -285,14 +279,13 @@ public KNNVectorFieldMapper build(BuilderContext context) { buildFullName(context), name, metaValue, - resolvedKNNMethodContext, knnMethodConfigContext, - knnMethodContext.get(), multiFieldsBuilder, copyToBuilder, ignoreMalformed, stored.getValue(), - hasDocValues.getValue() + hasDocValues.getValue(), + originalParameters ); } @@ -343,6 +336,7 @@ public Mapper.Builder parse(String name, Map node, ParserCont null ); builder.parse(name, parserContext, node); + builder.setOriginalParameters(new OriginalMappingParameters(builder)); // All parsing @@ -372,6 +366,7 @@ private void validateFromFlat(KNNVectorFieldMapper.Builder builder) { throw new IllegalArgumentException("Cannot set modelId or method parameters when index.knn setting is false"); } validateDimensionSet(builder); + validateCompressionAndModeNotSet(builder, builder.name(), "flat"); } private void validateFromModel(KNNVectorFieldMapper.Builder builder) { @@ -379,11 +374,13 @@ private void validateFromModel(KNNVectorFieldMapper.Builder builder) { if (builder.dimension.getValue() == UNSET_MODEL_DIMENSION_IDENTIFIER && builder.modelId.get() == null) { throw new IllegalArgumentException(String.format(Locale.ROOT, "Dimension value missing for vector: %s", builder.name())); } + validateCompressionAndModeNotSet(builder, builder.name(), "model"); } private void validateFromKNNMethod(KNNVectorFieldMapper.Builder builder) { - if (builder.resolvedKNNMethodContext != null) { - ValidationException validationException = builder.resolvedKNNMethodContext.validate(builder.knnMethodConfigContext); + if (builder.originalParameters.getResolvedKnnMethodContext() != null) { + ValidationException validationException = builder.originalParameters.getResolvedKnnMethodContext() + .validate(builder.knnMethodConfigContext); if (validationException != null) { throw validationException; } @@ -397,6 +394,19 @@ private void validateDimensionSet(KNNVectorFieldMapper.Builder builder) { } } + private void validateCompressionAndModeNotSet(KNNVectorFieldMapper.Builder builder, String name, String context) { + if (builder.mode.isConfigured() || builder.compressionLevel.isConfigured()) { + throw new MapperParsingException( + String.format( + Locale.ROOT, + "Compression and mode can not be specified in a %s mapping configuration for field: %s", + context, + name + ) + ); + } + } + private void resolveKNNMethodComponents(KNNVectorFieldMapper.Builder builder, ParserContext parserContext) { builder.setKnnMethodConfigContext( KNNMethodConfigContext.builder() @@ -407,13 +417,12 @@ private void resolveKNNMethodComponents(KNNVectorFieldMapper.Builder builder, Pa ); // Configure method from map or legacy - builder.setResolvedKNNMethodContext( - builder.knnMethodContext.getValue() != null - ? builder.knnMethodContext.getValue() - : createKNNMethodContextFromLegacy(parserContext.getSettings(), parserContext.indexVersionCreated()) - ); - // TODO: We should remove this and set it based on the KNNMethodContext - setDefaultSpaceType(builder.resolvedKNNMethodContext, builder.vectorDataType.getValue()); + if (builder.originalParameters.isLegacyMapping()) { + builder.originalParameters.setResolvedKnnMethodContext( + createKNNMethodContextFromLegacy(parserContext.getSettings(), parserContext.indexVersionCreated()) + ); + } + setDefaultSpaceType(builder.originalParameters.getResolvedKnnMethodContext(), builder.vectorDataType.getValue()); } private boolean isKNNDisabled(Settings settings) { @@ -449,7 +458,7 @@ private void setDefaultSpaceType(final KNNMethodContext knnMethodContext, final // We need to ensure that the original KNNMethodContext as parsed is stored to initialize the // Builder for serialization. So, we need to store it here. This is mainly to ensure that the legacy field mapper // can use KNNMethodContext without messing up serialization on mapper merge - protected KNNMethodContext originalKNNMethodContext; + protected OriginalMappingParameters originalMappingParameters; public KNNVectorFieldMapper( String simpleName, @@ -460,7 +469,7 @@ public KNNVectorFieldMapper( boolean stored, boolean hasDocValues, Version indexCreatedVersion, - KNNMethodContext originalKNNMethodContext + OriginalMappingParameters originalMappingParameters ) { super(simpleName, mappedFieldType, multiFields, copyTo); this.ignoreMalformed = ignoreMalformed; @@ -469,7 +478,7 @@ public KNNVectorFieldMapper( this.vectorDataType = mappedFieldType.getVectorDataType(); updateEngineStats(); this.indexCreatedVersion = indexCreatedVersion; - this.originalKNNMethodContext = originalKNNMethodContext; + this.originalMappingParameters = originalMappingParameters; } public KNNVectorFieldMapper clone() { @@ -680,8 +689,8 @@ public ParametrizedFieldMapper.Builder getMergeBuilder() { simpleName(), modelDao, indexCreatedVersion, - fieldType().getKnnMappingConfig().getKnnMethodContext().orElse(null), - knnMethodConfigContext + knnMethodConfigContext, + originalMappingParameters ).init(this); } diff --git a/src/main/java/org/opensearch/knn/index/mapper/LuceneFieldMapper.java b/src/main/java/org/opensearch/knn/index/mapper/LuceneFieldMapper.java index 744ba4bd5..3da2745ac 100644 --- a/src/main/java/org/opensearch/knn/index/mapper/LuceneFieldMapper.java +++ b/src/main/java/org/opensearch/knn/index/mapper/LuceneFieldMapper.java @@ -45,9 +45,9 @@ public class LuceneFieldMapper extends KNNVectorFieldMapper { static LuceneFieldMapper createFieldMapper( String fullname, Map metaValue, - KNNMethodContext knnMethodContext, KNNMethodConfigContext knnMethodConfigContext, - CreateLuceneFieldMapperInput createLuceneFieldMapperInput + CreateLuceneFieldMapperInput createLuceneFieldMapperInput, + OriginalMappingParameters originalMappingParameters ) { final KNNVectorFieldType mappedFieldType = new KNNVectorFieldType( fullname, @@ -56,7 +56,7 @@ static LuceneFieldMapper createFieldMapper( new KNNMappingConfig() { @Override public Optional getKnnMethodContext() { - return Optional.of(knnMethodContext); + return Optional.of(originalMappingParameters.getResolvedKnnMethodContext()); } @Override @@ -66,13 +66,14 @@ public int getDimension() { } ); - return new LuceneFieldMapper(mappedFieldType, createLuceneFieldMapperInput, knnMethodConfigContext); + return new LuceneFieldMapper(mappedFieldType, createLuceneFieldMapperInput, knnMethodConfigContext, originalMappingParameters); } private LuceneFieldMapper( final KNNVectorFieldType mappedFieldType, final CreateLuceneFieldMapperInput input, - KNNMethodConfigContext knnMethodConfigContext + KNNMethodConfigContext knnMethodConfigContext, + OriginalMappingParameters originalMappingParameters ) { super( input.getName(), @@ -83,7 +84,7 @@ private LuceneFieldMapper( input.isStored(), input.isHasDocValues(), knnMethodConfigContext.getVersionCreated(), - mappedFieldType.knnMappingConfig.getKnnMethodContext().orElse(null) + originalMappingParameters ); KNNMappingConfig knnMappingConfig = mappedFieldType.getKnnMappingConfig(); KNNMethodContext knnMethodContext = knnMappingConfig.getKnnMethodContext() diff --git a/src/main/java/org/opensearch/knn/index/mapper/MethodFieldMapper.java b/src/main/java/org/opensearch/knn/index/mapper/MethodFieldMapper.java index 90d4ca879..f1a87c64b 100644 --- a/src/main/java/org/opensearch/knn/index/mapper/MethodFieldMapper.java +++ b/src/main/java/org/opensearch/knn/index/mapper/MethodFieldMapper.java @@ -43,14 +43,13 @@ public static MethodFieldMapper createFieldMapper( String fullname, String simpleName, Map metaValue, - KNNMethodContext knnMethodContext, KNNMethodConfigContext knnMethodConfigContext, - KNNMethodContext originalKNNMethodContext, MultiFields multiFields, CopyTo copyTo, Explicit ignoreMalformed, boolean stored, - boolean hasDocValues + boolean hasDocValues, + OriginalMappingParameters originalMappingParameters ) { final KNNVectorFieldType mappedFieldType = new KNNVectorFieldType( fullname, @@ -59,7 +58,7 @@ public static MethodFieldMapper createFieldMapper( new KNNMappingConfig() { @Override public Optional getKnnMethodContext() { - return Optional.of(knnMethodContext); + return Optional.of(originalMappingParameters.getResolvedKnnMethodContext()); } @Override @@ -76,8 +75,8 @@ public int getDimension() { ignoreMalformed, stored, hasDocValues, - originalKNNMethodContext, - knnMethodConfigContext + knnMethodConfigContext, + originalMappingParameters ); } @@ -89,8 +88,8 @@ private MethodFieldMapper( Explicit ignoreMalformed, boolean stored, boolean hasDocValues, - KNNMethodContext originalKNNMethodContext, - KNNMethodConfigContext knnMethodConfigContext + KNNMethodConfigContext knnMethodConfigContext, + OriginalMappingParameters originalMappingParameters ) { super( @@ -102,7 +101,7 @@ private MethodFieldMapper( stored, hasDocValues, knnMethodConfigContext.getVersionCreated(), - originalKNNMethodContext + originalMappingParameters ); this.useLuceneBasedVectorField = KNNVectorFieldMapperUtil.useLuceneKNNVectorsFormat(indexCreatedVersion); KNNMappingConfig annConfig = mappedFieldType.getKnnMappingConfig(); diff --git a/src/main/java/org/opensearch/knn/index/mapper/Mode.java b/src/main/java/org/opensearch/knn/index/mapper/Mode.java new file mode 100644 index 000000000..0798ab941 --- /dev/null +++ b/src/main/java/org/opensearch/knn/index/mapper/Mode.java @@ -0,0 +1,68 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.mapper; + +import lombok.AllArgsConstructor; +import lombok.Getter; +import org.opensearch.core.common.Strings; + +import java.util.Arrays; +import java.util.Locale; +import java.util.stream.Collectors; + +/** + * Enum representing the intended workload optimization a user wants their k-NN system to have. Based on this value, + * default parameter resolution will be determined. + */ +@Getter +@AllArgsConstructor +public enum Mode { + NOT_CONFIGURED(""), + IN_MEMORY("in_memory"), + ON_DISK("on_disk"); + + // Internally, an empty string is easier to deal with them null. However, from the mapping, + // we do not want users to pass in the empty string and instead want null. So we make the conversion herex + static final String[] NAMES_ARRAY = Arrays.stream(Mode.values()) + .map(mode -> mode == NOT_CONFIGURED ? null : mode.getName()) + .collect(Collectors.toList()) + .toArray(new String[0]); + + private static final Mode DEFAULT = IN_MEMORY; + + /** + * Convert a string to a Mode enum value + * + * @param name String value to convert + * @return Mode enum value + */ + public static Mode fromName(String name) { + if (Strings.isEmpty(name)) { + return NOT_CONFIGURED; + } + + if (IN_MEMORY.name.equalsIgnoreCase(name)) { + return IN_MEMORY; + } + + if (ON_DISK.name.equalsIgnoreCase(name)) { + return ON_DISK; + } + throw new IllegalArgumentException(String.format(Locale.ROOT, "Invalid mode: \"[%s]\"", name)); + } + + private final String name; + + /** + * Utility method that checks if mode is configured. + * + * @param mode Mode to check + * @return true if mode is configured, false otherwise + */ + public static boolean isConfigured(Mode mode) { + return mode != null && mode != NOT_CONFIGURED; + } +} diff --git a/src/main/java/org/opensearch/knn/index/mapper/ModelFieldMapper.java b/src/main/java/org/opensearch/knn/index/mapper/ModelFieldMapper.java index b29466eef..bfb188a75 100644 --- a/src/main/java/org/opensearch/knn/index/mapper/ModelFieldMapper.java +++ b/src/main/java/org/opensearch/knn/index/mapper/ModelFieldMapper.java @@ -49,25 +49,25 @@ public static ModelFieldMapper createFieldMapper( String simpleName, Map metaValue, VectorDataType vectorDataType, - String modelId, MultiFields multiFields, CopyTo copyTo, Explicit ignoreMalformed, boolean stored, boolean hasDocValues, ModelDao modelDao, - Version indexCreatedVersion + Version indexCreatedVersion, + OriginalMappingParameters originalMappingParameters ) { final KNNVectorFieldType mappedFieldType = new KNNVectorFieldType(fullname, metaValue, vectorDataType, new KNNMappingConfig() { @Override public Optional getModelId() { - return Optional.of(modelId); + return Optional.of(originalMappingParameters.getModelId()); } @Override public int getDimension() { - return getModelMetadata(modelDao, modelId).getDimension(); + return getModelMetadata(modelDao, originalMappingParameters.getModelId()).getDimension(); } }); return new ModelFieldMapper( @@ -79,7 +79,8 @@ public int getDimension() { stored, hasDocValues, modelDao, - indexCreatedVersion + indexCreatedVersion, + originalMappingParameters ); } @@ -92,9 +93,20 @@ private ModelFieldMapper( boolean stored, boolean hasDocValues, ModelDao modelDao, - Version indexCreatedVersion + Version indexCreatedVersion, + OriginalMappingParameters originalMappingParameters ) { - super(simpleName, mappedFieldType, multiFields, copyTo, ignoreMalformed, stored, hasDocValues, indexCreatedVersion, null); + super( + simpleName, + mappedFieldType, + multiFields, + copyTo, + ignoreMalformed, + stored, + hasDocValues, + indexCreatedVersion, + originalMappingParameters + ); KNNMappingConfig annConfig = mappedFieldType.getKnnMappingConfig(); modelId = annConfig.getModelId().orElseThrow(() -> new IllegalArgumentException("KNN method context cannot be empty")); this.modelDao = modelDao; diff --git a/src/main/java/org/opensearch/knn/index/mapper/OriginalMappingParameters.java b/src/main/java/org/opensearch/knn/index/mapper/OriginalMappingParameters.java new file mode 100644 index 000000000..f7235620f --- /dev/null +++ b/src/main/java/org/opensearch/knn/index/mapper/OriginalMappingParameters.java @@ -0,0 +1,79 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.mapper; + +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import lombok.Setter; +import org.opensearch.core.common.Strings; +import org.opensearch.knn.index.VectorDataType; +import org.opensearch.knn.index.engine.KNNMethodContext; + +/** + * Utility class to store the original mapping parameters for a KNNVectorFieldMapper. These parameters need to be + * kept around for when a {@link KNNVectorFieldMapper} is built from merge + */ +@Getter +@RequiredArgsConstructor +public final class OriginalMappingParameters { + private final VectorDataType vectorDataType; + private final int dimension; + private final KNNMethodContext knnMethodContext; + + // To support our legacy field mapping, on parsing, if index.knn=true and no method is + // passed, we build a KNNMethodContext using the space type, ef_construction and m that are set in the index + // settings. However, for fieldmappers for merging, we need to be able to initialize one field mapper from + // another (see + // https://github.com/opensearch-project/OpenSearch/blob/2.16.0/server/src/main/java/org/opensearch/index/mapper/ParametrizedFieldMapper.java#L98). + // The problem is that in this case, the settings are set to empty so we cannot properly resolve the KNNMethodContext. + // (see + // https://github.com/opensearch-project/OpenSearch/blob/2.16.0/server/src/main/java/org/opensearch/index/mapper/ParametrizedFieldMapper.java#L130). + // While we could override the KNNMethodContext parameter initializer to set the knnMethodContext based on the + // constructed KNNMethodContext from the other field mapper, this can result in merge conflict/serialization + // exceptions. See + // (https://github.com/opensearch-project/OpenSearch/blob/2.16.0/server/src/main/java/org/opensearch/index/mapper/ParametrizedFieldMapper.java#L322-L324). + // So, what we do is pass in a "resolvedKNNMethodContext" to ensure we track this resolveKnnMethodContext. + // A similar approach was taken for https://github.com/opendistro-for-elasticsearch/k-NN/issues/288 + @Setter + private KNNMethodContext resolvedKnnMethodContext; + private final String mode; + private final String compressionLevel; + private final String modelId; + + /** + * Initialize the parameters from the builder + * + * @param builder The builder to initialize from + */ + public OriginalMappingParameters(KNNVectorFieldMapper.Builder builder) { + this.vectorDataType = builder.vectorDataType.get(); + this.knnMethodContext = builder.knnMethodContext.get(); + this.resolvedKnnMethodContext = builder.knnMethodContext.get(); + this.dimension = builder.dimension.get(); + this.mode = builder.mode.get(); + this.compressionLevel = builder.compressionLevel.get(); + this.modelId = builder.modelId.get(); + } + + /** + * Determine if the mapping used the legacy mechanism to setup the index. The legacy mechanism is used if + * the index is created only by specifying the dimension. If this is the case, the constructed parameters + * need to be collected from the index settings + * + * @return true if the mapping used the legacy mechanism, false otherwise + */ + public boolean isLegacyMapping() { + if (knnMethodContext != null) { + return false; + } + + if (modelId != null) { + return false; + } + + return Strings.isEmpty(mode) && Strings.isEmpty(compressionLevel); + } +} diff --git a/src/main/java/org/opensearch/knn/index/util/IndexUtil.java b/src/main/java/org/opensearch/knn/index/util/IndexUtil.java index f0d57ddeb..b548a9fd7 100644 --- a/src/main/java/org/opensearch/knn/index/util/IndexUtil.java +++ b/src/main/java/org/opensearch/knn/index/util/IndexUtil.java @@ -52,6 +52,7 @@ public class IndexUtil { private static final Version MINIMAL_SUPPORTED_VERSION_FOR_METHOD_PARAMETERS = Version.V_2_16_0; private static final Version MINIMAL_SUPPORTED_VERSION_FOR_MODEL_VECTOR_DATA_TYPE = Version.V_2_16_0; private static final Version MINIMAL_RESCORE_FEATURE = Version.V_2_17_0; + private static final Version MINIMAL_MODE_AND_COMPRESSION_FEATURE = Version.V_2_17_0; // public so neural search can access it public static final Map minimalRequiredVersionMap = initializeMinimalRequiredVersionMap(); public static final Set VECTOR_DATA_TYPES_NOT_SUPPORTING_ENCODERS = Set.of(VectorDataType.BINARY, VectorDataType.BYTE); @@ -390,6 +391,7 @@ private static Map initializeMinimalRequiredVersionMap() { put(KNNConstants.METHOD_PARAMETER, MINIMAL_SUPPORTED_VERSION_FOR_METHOD_PARAMETERS); put(KNNConstants.MODEL_VECTOR_DATA_TYPE_KEY, MINIMAL_SUPPORTED_VERSION_FOR_MODEL_VECTOR_DATA_TYPE); put(RESCORE_PARAMETER, MINIMAL_RESCORE_FEATURE); + put(KNNConstants.MINIMAL_MODE_AND_COMPRESSION_FEATURE, MINIMAL_MODE_AND_COMPRESSION_FEATURE); } }; diff --git a/src/main/java/org/opensearch/knn/indices/ModelDao.java b/src/main/java/org/opensearch/knn/indices/ModelDao.java index e95596699..326d595a4 100644 --- a/src/main/java/org/opensearch/knn/indices/ModelDao.java +++ b/src/main/java/org/opensearch/knn/indices/ModelDao.java @@ -51,6 +51,8 @@ import org.opensearch.knn.common.KNNConstants; import org.opensearch.knn.common.exception.DeleteModelException; import org.opensearch.knn.index.engine.MethodComponentContext; +import org.opensearch.knn.index.mapper.CompressionLevel; +import org.opensearch.knn.index.mapper.Mode; import org.opensearch.knn.plugin.transport.DeleteModelResponse; import org.opensearch.knn.plugin.transport.GetModelResponse; import org.opensearch.knn.plugin.transport.RemoveModelFromCacheAction; @@ -293,7 +295,12 @@ private void putInternal(Model model, ActionListener listener, Do put(KNNConstants.MODEL_ERROR, modelMetadata.getError()); put(KNNConstants.MODEL_NODE_ASSIGNMENT, modelMetadata.getNodeAssignment()); put(KNNConstants.VECTOR_DATA_TYPE_FIELD, modelMetadata.getVectorDataType()); - + if (Mode.isConfigured(modelMetadata.getMode())) { + put(KNNConstants.MODE_PARAMETER, modelMetadata.getMode().getName()); + } + if (CompressionLevel.isConfigured(modelMetadata.getCompressionLevel())) { + put(KNNConstants.COMPRESSION_LEVEL_PARAMETER, modelMetadata.getCompressionLevel().getName()); + } MethodComponentContext methodComponentContext = modelMetadata.getMethodComponentContext(); if (!methodComponentContext.getName().isEmpty()) { XContentBuilder builder = XContentFactory.jsonBuilder().startObject(); diff --git a/src/main/java/org/opensearch/knn/indices/ModelMetadata.java b/src/main/java/org/opensearch/knn/indices/ModelMetadata.java index 60301e244..620e520ba 100644 --- a/src/main/java/org/opensearch/knn/indices/ModelMetadata.java +++ b/src/main/java/org/opensearch/knn/indices/ModelMetadata.java @@ -11,6 +11,7 @@ package org.opensearch.knn.indices; +import lombok.Getter; import lombok.extern.log4j.Log4j2; import org.apache.commons.lang.builder.EqualsBuilder; import org.apache.commons.lang.builder.HashCodeBuilder; @@ -23,6 +24,8 @@ import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.knn.common.KNNConstants; +import org.opensearch.knn.index.mapper.CompressionLevel; +import org.opensearch.knn.index.mapper.Mode; import org.opensearch.knn.index.util.IndexUtil; import org.opensearch.knn.index.engine.MethodComponentContext; import org.opensearch.knn.index.SpaceType; @@ -51,7 +54,11 @@ public class ModelMetadata implements Writeable, ToXContentObject { final private String trainingNodeAssignment; final private VectorDataType vectorDataType; private MethodComponentContext methodComponentContext; + @Getter + private final Mode mode; private String error; + @Getter + private final CompressionLevel compressionLevel; /** * Constructor @@ -89,6 +96,15 @@ public ModelMetadata(StreamInput in) throws IOException { } else { this.vectorDataType = VectorDataType.DEFAULT; } + + if (IndexUtil.isVersionOnOrAfterMinRequiredVersion(in.getVersion(), KNNConstants.MINIMAL_MODE_AND_COMPRESSION_FEATURE)) { + this.mode = Mode.fromName(in.readOptionalString()); + this.compressionLevel = CompressionLevel.fromName(in.readOptionalString()); + } else { + this.mode = Mode.NOT_CONFIGURED; + this.compressionLevel = CompressionLevel.NOT_CONFIGURED; + } + } /** @@ -115,7 +131,9 @@ public ModelMetadata( String error, String trainingNodeAssignment, MethodComponentContext methodComponentContext, - VectorDataType vectorDataType + VectorDataType vectorDataType, + Mode mode, + CompressionLevel compressionLevel ) { this.knnEngine = Objects.requireNonNull(knnEngine, "knnEngine must not be null"); this.spaceType = Objects.requireNonNull(spaceType, "spaceType must not be null"); @@ -139,6 +157,8 @@ public ModelMetadata( this.trainingNodeAssignment = Objects.requireNonNull(trainingNodeAssignment, "node assignment must not be null"); this.methodComponentContext = Objects.requireNonNull(methodComponentContext, "method context must not be null"); this.vectorDataType = Objects.requireNonNull(vectorDataType, "vector data type must not be null"); + this.mode = Objects.requireNonNull(mode, "Mode must not be null"); + this.compressionLevel = Objects.requireNonNull(compressionLevel, "Compression level must not be null"); } /** @@ -257,7 +277,9 @@ public String toString() { error, trainingNodeAssignment, methodComponentContext.toClusterStateString(), - vectorDataType.getValue() + vectorDataType.getValue(), + mode.getName(), + compressionLevel.getName() ); } @@ -276,6 +298,8 @@ public boolean equals(Object obj) { equalsBuilder.append(getDescription(), other.getDescription()); equalsBuilder.append(getError(), other.getError()); equalsBuilder.append(getVectorDataType(), other.getVectorDataType()); + equalsBuilder.append(getMode(), other.getMode()); + equalsBuilder.append(getCompressionLevel(), other.getCompressionLevel()); return equalsBuilder.isEquals(); } @@ -291,6 +315,8 @@ public int hashCode() { .append(getError()) .append(getMethodComponentContext()) .append(getVectorDataType()) + .append(getMode()) + .append(getCompressionLevel()) .toHashCode(); } @@ -304,13 +330,14 @@ public static ModelMetadata fromString(String modelMetadataString) { String[] modelMetadataArray = modelMetadataString.split(DELIMITER, -1); int length = modelMetadataArray.length; - if (length < 7 || length > 10) { + if (length < 7 || length > 12) { throw new IllegalArgumentException( "Illegal format for model metadata. Must be of the form " + "\",,,,,,\" or " + "\",,,,,,,\" or " + "\",,,,,,,,\" or " - + "\",,,,,,,,,\"." + + "\",,,,,,,,,\". or " + + "\",,,,,,,,,,,\"." ); } @@ -326,6 +353,10 @@ public static ModelMetadata fromString(String modelMetadataString) { ? MethodComponentContext.fromClusterStateString(modelMetadataArray[8]) : MethodComponentContext.EMPTY; VectorDataType vectorDataType = length > 9 ? VectorDataType.get(modelMetadataArray[9]) : VectorDataType.DEFAULT; + Mode mode = length > 10 ? Mode.fromName(modelMetadataArray[10]) : Mode.NOT_CONFIGURED; + CompressionLevel compressionLevel = length > 11 + ? CompressionLevel.fromName(modelMetadataArray[11]) + : CompressionLevel.NOT_CONFIGURED; log.debug(getLogMessage(length)); @@ -339,7 +370,9 @@ public static ModelMetadata fromString(String modelMetadataString) { error, trainingNodeAssignment, methodComponentContext, - vectorDataType + vectorDataType, + mode, + compressionLevel ); } @@ -353,6 +386,9 @@ private static String getLogMessage(int length) { return "Model metadata contains training node assignment and method context."; case 10: return "Model metadata contains training node assignment, method context and vector data type."; + case 11: + case 12: + return "Model metadata contains mode and compression level"; default: throw new IllegalArgumentException("Unexpected metadata array length: " + length); } @@ -385,6 +421,8 @@ public static ModelMetadata getMetadataFromSourceMap(final Map m Object trainingNodeAssignment = modelSourceMap.get(KNNConstants.MODEL_NODE_ASSIGNMENT); Object methodComponentContext = modelSourceMap.get(KNNConstants.MODEL_METHOD_COMPONENT_CONTEXT); Object vectorDataType = modelSourceMap.get(KNNConstants.VECTOR_DATA_TYPE_FIELD); + Object mode = modelSourceMap.get(KNNConstants.MODE_PARAMETER); + Object compressionLevel = modelSourceMap.get(KNNConstants.COMPRESSION_LEVEL_PARAMETER); if (trainingNodeAssignment == null) { trainingNodeAssignment = ""; @@ -419,7 +457,9 @@ public static ModelMetadata getMetadataFromSourceMap(final Map m objectToString(error), objectToString(trainingNodeAssignment), (MethodComponentContext) methodComponentContext, - VectorDataType.get(objectToString(vectorDataType)) + VectorDataType.get(objectToString(vectorDataType)), + Mode.fromName(objectToString(mode)), + CompressionLevel.fromName(objectToString(compressionLevel)) ); return modelMetadata; } @@ -442,6 +482,10 @@ public void writeTo(StreamOutput out) throws IOException { if (IndexUtil.isVersionOnOrAfterMinRequiredVersion(out.getVersion(), KNNConstants.MODEL_VECTOR_DATA_TYPE_KEY)) { out.writeString(vectorDataType.getValue()); } + if (IndexUtil.isVersionOnOrAfterMinRequiredVersion(out.getVersion(), KNNConstants.MINIMAL_MODE_AND_COMPRESSION_FEATURE)) { + out.writeOptionalString(mode.getName()); + out.writeOptionalString(compressionLevel.getName()); + } } @Override @@ -465,6 +509,14 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws if (IndexUtil.isClusterOnOrAfterMinRequiredVersion(KNNConstants.MODEL_VECTOR_DATA_TYPE_KEY)) { builder.field(KNNConstants.VECTOR_DATA_TYPE_FIELD, vectorDataType.getValue()); } + if (IndexUtil.isClusterOnOrAfterMinRequiredVersion(KNNConstants.MINIMAL_MODE_AND_COMPRESSION_FEATURE)) { + if (Mode.isConfigured(mode)) { + builder.field(KNNConstants.MODE_PARAMETER, mode.getName()); + } + if (CompressionLevel.isConfigured(compressionLevel)) { + builder.field(KNNConstants.COMPRESSION_LEVEL_PARAMETER, compressionLevel.getName()); + } + } return builder; } } diff --git a/src/main/java/org/opensearch/knn/plugin/rest/RestTrainModelHandler.java b/src/main/java/org/opensearch/knn/plugin/rest/RestTrainModelHandler.java index 58bcd1ebf..c9038f0c7 100644 --- a/src/main/java/org/opensearch/knn/plugin/rest/RestTrainModelHandler.java +++ b/src/main/java/org/opensearch/knn/plugin/rest/RestTrainModelHandler.java @@ -15,9 +15,12 @@ import org.opensearch.client.node.NodeClient; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.index.mapper.NumberFieldMapper; +import org.opensearch.knn.common.KNNConstants; import org.opensearch.knn.index.engine.KNNMethodContext; import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.VectorDataType; +import org.opensearch.knn.index.mapper.CompressionLevel; +import org.opensearch.knn.index.mapper.Mode; import org.opensearch.knn.indices.ModelUtil; import org.opensearch.knn.plugin.KNNPlugin; import org.opensearch.knn.plugin.transport.TrainingJobRouterAction; @@ -91,6 +94,9 @@ private TrainingModelRequest createTransportRequest(RestRequest restRequest) thr int maximumVectorCount = DEFAULT_NOT_SET_INT_VALUE; int searchSize = DEFAULT_NOT_SET_INT_VALUE; + String compressionLevel = null; + String mode = null; + while (parser.nextToken() != XContentParser.Token.END_OBJECT) { String fieldName = parser.currentName(); parser.nextToken(); @@ -115,6 +121,10 @@ private TrainingModelRequest createTransportRequest(RestRequest restRequest) thr ModelUtil.blockCommasInModelDescription(description); } else if (VECTOR_DATA_TYPE_FIELD.equals(fieldName) && ensureNotSet(fieldName, vectorDataType)) { vectorDataType = VectorDataType.get(parser.text()); + } else if (KNNConstants.MODE_PARAMETER.equals(fieldName) && ensureNotSet(fieldName, mode)) { + mode = parser.text(); + } else if (KNNConstants.COMPRESSION_LEVEL_PARAMETER.equals(fieldName) && ensureNotSet(fieldName, compressionLevel)) { + compressionLevel = parser.text(); } else { throw new IllegalArgumentException("Unable to parse token. \"" + fieldName + "\" is not a valid " + "parameter."); } @@ -143,7 +153,9 @@ private TrainingModelRequest createTransportRequest(RestRequest restRequest) thr trainingField, preferredNodeId, description, - vectorDataType + vectorDataType, + Mode.fromName(mode), + CompressionLevel.fromName(compressionLevel) ); if (maximumVectorCount != DEFAULT_NOT_SET_INT_VALUE) { diff --git a/src/main/java/org/opensearch/knn/plugin/transport/TrainingModelRequest.java b/src/main/java/org/opensearch/knn/plugin/transport/TrainingModelRequest.java index 3634d13f0..fdc82526d 100644 --- a/src/main/java/org/opensearch/knn/plugin/transport/TrainingModelRequest.java +++ b/src/main/java/org/opensearch/knn/plugin/transport/TrainingModelRequest.java @@ -22,6 +22,8 @@ import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.knn.common.KNNConstants; import org.opensearch.knn.index.engine.KNNMethodConfigContext; +import org.opensearch.knn.index.mapper.CompressionLevel; +import org.opensearch.knn.index.mapper.Mode; import org.opensearch.knn.index.util.IndexUtil; import org.opensearch.knn.index.engine.KNNMethodContext; import org.opensearch.knn.index.VectorDataType; @@ -50,6 +52,8 @@ public class TrainingModelRequest extends ActionRequest { private int maximumVectorCount; private int searchSize; private int trainingDataSizeInKB; + private final Mode mode; + private final CompressionLevel compressionLevel; /** * Constructor. @@ -70,7 +74,9 @@ public TrainingModelRequest( String trainingField, String preferredNodeId, String description, - VectorDataType vectorDataType + VectorDataType vectorDataType, + Mode mode, + CompressionLevel compressionLevel ) { super(); this.modelId = modelId; @@ -94,6 +100,8 @@ public TrainingModelRequest( .dimension(dimension) .versionCreated(Version.CURRENT) .build(); + this.mode = mode; + this.compressionLevel = compressionLevel; } /** @@ -119,6 +127,14 @@ public TrainingModelRequest(StreamInput in) throws IOException { } else { this.vectorDataType = VectorDataType.DEFAULT; } + if (IndexUtil.isVersionOnOrAfterMinRequiredVersion(in.getVersion(), KNNConstants.MINIMAL_MODE_AND_COMPRESSION_FEATURE)) { + this.mode = Mode.fromName(in.readOptionalString()); + this.compressionLevel = CompressionLevel.fromName(in.readOptionalString()); + } else { + this.mode = Mode.NOT_CONFIGURED; + this.compressionLevel = CompressionLevel.NOT_CONFIGURED; + } + this.knnMethodConfigContext = KNNMethodConfigContext.builder() .vectorDataType(vectorDataType) .dimension(dimension) @@ -271,5 +287,9 @@ public void writeTo(StreamOutput out) throws IOException { } else { out.writeString(VectorDataType.DEFAULT.getValue()); } + if (IndexUtil.isVersionOnOrAfterMinRequiredVersion(out.getVersion(), KNNConstants.MINIMAL_MODE_AND_COMPRESSION_FEATURE)) { + out.writeOptionalString(mode.getName()); + out.writeOptionalString(compressionLevel.getName()); + } } } diff --git a/src/main/java/org/opensearch/knn/plugin/transport/TrainingModelTransportAction.java b/src/main/java/org/opensearch/knn/plugin/transport/TrainingModelTransportAction.java index 963142c1f..aa1db8c6a 100644 --- a/src/main/java/org/opensearch/knn/plugin/transport/TrainingModelTransportAction.java +++ b/src/main/java/org/opensearch/knn/plugin/transport/TrainingModelTransportAction.java @@ -78,7 +78,9 @@ protected void doExecute(Task task, TrainingModelRequest request, ActionListener modelAnonymousEntryContext, request.getKnnMethodConfigContext(), request.getDescription(), - clusterService.localNode().getEphemeralId() + clusterService.localNode().getEphemeralId(), + request.getMode(), + request.getCompressionLevel() ); KNNCounter.TRAINING_REQUESTS.increment(); diff --git a/src/main/java/org/opensearch/knn/training/TrainingJob.java b/src/main/java/org/opensearch/knn/training/TrainingJob.java index e30d860db..63df79bde 100644 --- a/src/main/java/org/opensearch/knn/training/TrainingJob.java +++ b/src/main/java/org/opensearch/knn/training/TrainingJob.java @@ -19,6 +19,8 @@ import org.opensearch.knn.common.KNNConstants; import org.opensearch.knn.index.KNNSettings; import org.opensearch.knn.index.engine.KNNMethodConfigContext; +import org.opensearch.knn.index.mapper.CompressionLevel; +import org.opensearch.knn.index.mapper.Mode; import org.opensearch.knn.jni.JNIService; import org.opensearch.knn.index.engine.KNNMethodContext; import org.opensearch.knn.index.memory.NativeMemoryAllocation; @@ -70,7 +72,9 @@ public TrainingJob( NativeMemoryEntryContext.AnonymousEntryContext modelAnonymousEntryContext, KNNMethodConfigContext knnMethodConfigContext, String description, - String nodeAssignment + String nodeAssignment, + Mode mode, + CompressionLevel compressionLevel ) { // Generate random base64 string if one is not provided this.modelId = StringUtils.isNotBlank(modelId) ? modelId : UUIDs.randomBase64UUID(); @@ -90,7 +94,9 @@ public TrainingJob( "", nodeAssignment, knnMethodContext.getMethodComponentContext(), - knnMethodConfigContext.getVectorDataType() + knnMethodConfigContext.getVectorDataType(), + mode, + compressionLevel ), null, this.modelId diff --git a/src/main/resources/mappings/model-index.json b/src/main/resources/mappings/model-index.json index cd2a50839..e7879cced 100644 --- a/src/main/resources/mappings/model-index.json +++ b/src/main/resources/mappings/model-index.json @@ -32,6 +32,12 @@ }, "method_component_context": { "type": "keyword" + }, + "mode": { + "type": "keyword" + }, + "compression_level": { + "type": "keyword" } } } diff --git a/src/test/java/org/opensearch/knn/index/KNNCreateIndexFromModelTests.java b/src/test/java/org/opensearch/knn/index/KNNCreateIndexFromModelTests.java index f0e60ca98..28ef41e04 100644 --- a/src/test/java/org/opensearch/knn/index/KNNCreateIndexFromModelTests.java +++ b/src/test/java/org/opensearch/knn/index/KNNCreateIndexFromModelTests.java @@ -18,6 +18,8 @@ import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.knn.KNNSingleNodeTestCase; import org.opensearch.knn.index.engine.MethodComponentContext; +import org.opensearch.knn.index.mapper.CompressionLevel; +import org.opensearch.knn.index.mapper.Mode; import org.opensearch.knn.jni.JNIService; import org.opensearch.knn.index.engine.KNNEngine; import org.opensearch.knn.indices.Model; @@ -65,7 +67,9 @@ public void testCreateIndexFromModel() throws IOException, InterruptedException "", "test-node", MethodComponentContext.EMPTY, - VectorDataType.FLOAT + VectorDataType.FLOAT, + Mode.NOT_CONFIGURED, + CompressionLevel.NOT_CONFIGURED ); Model model = new Model(modelMetadata, modelBlob, modelId); diff --git a/src/test/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumerTests.java b/src/test/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumerTests.java index f49587bc5..786061af8 100644 --- a/src/test/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumerTests.java +++ b/src/test/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumerTests.java @@ -28,6 +28,8 @@ import org.opensearch.knn.index.engine.KNNMethodConfigContext; import org.opensearch.knn.index.engine.KNNMethodContext; import org.opensearch.knn.index.VectorDataType; +import org.opensearch.knn.index.mapper.CompressionLevel; +import org.opensearch.knn.index.mapper.Mode; import org.opensearch.knn.index.vectorvalues.TestVectorValues; import org.opensearch.knn.index.mapper.KNNVectorFieldMapper; import org.opensearch.knn.index.engine.MethodComponentContext; @@ -469,7 +471,9 @@ public void testAddKNNBinaryField_fromModel_faiss() throws IOException, Executio "", "", MethodComponentContext.EMPTY, - VectorDataType.FLOAT + VectorDataType.FLOAT, + Mode.NOT_CONFIGURED, + CompressionLevel.NOT_CONFIGURED ), modelBytes, modelId diff --git a/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestCase.java b/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestCase.java index 2a4e26a82..174441df8 100644 --- a/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestCase.java +++ b/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestCase.java @@ -29,7 +29,9 @@ import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.VectorField; +import org.opensearch.knn.index.mapper.CompressionLevel; import org.opensearch.knn.index.mapper.KNNVectorFieldType; +import org.opensearch.knn.index.mapper.Mode; import org.opensearch.knn.index.query.BaseQueryFactory; import org.opensearch.knn.index.query.KNNQueryFactory; import org.opensearch.knn.jni.JNIService; @@ -243,7 +245,9 @@ public void testBuildFromModelTemplate(Codec codec) throws IOException, Executio "", "", MethodComponentContext.EMPTY, - VectorDataType.FLOAT + VectorDataType.FLOAT, + Mode.NOT_CONFIGURED, + CompressionLevel.NOT_CONFIGURED ); Model mockModel = new Model(modelMetadata1, modelBlob, modelId); diff --git a/src/test/java/org/opensearch/knn/index/mapper/CompressionLevelTests.java b/src/test/java/org/opensearch/knn/index/mapper/CompressionLevelTests.java new file mode 100644 index 000000000..07475109a --- /dev/null +++ b/src/test/java/org/opensearch/knn/index/mapper/CompressionLevelTests.java @@ -0,0 +1,42 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.mapper; + +import org.opensearch.core.common.Strings; +import org.opensearch.knn.KNNTestCase; + +public class CompressionLevelTests extends KNNTestCase { + + public void testFromName() { + assertEquals(CompressionLevel.NOT_CONFIGURED, CompressionLevel.fromName(null)); + assertEquals(CompressionLevel.NOT_CONFIGURED, CompressionLevel.fromName("")); + assertEquals(CompressionLevel.x1, CompressionLevel.fromName("1x")); + assertEquals(CompressionLevel.x32, CompressionLevel.fromName("32x")); + expectThrows(IllegalArgumentException.class, () -> CompressionLevel.fromName("x1")); + } + + public void testGetName() { + assertTrue(Strings.isEmpty(CompressionLevel.NOT_CONFIGURED.getName())); + assertEquals("4x", CompressionLevel.x4.getName()); + assertEquals("16x", CompressionLevel.x16.getName()); + } + + public void testNumBitsForFloat32() { + assertEquals(1, CompressionLevel.x32.numBitsForFloat32()); + assertEquals(2, CompressionLevel.x16.numBitsForFloat32()); + assertEquals(4, CompressionLevel.x8.numBitsForFloat32()); + assertEquals(8, CompressionLevel.x4.numBitsForFloat32()); + assertEquals(16, CompressionLevel.x2.numBitsForFloat32()); + assertEquals(32, CompressionLevel.x1.numBitsForFloat32()); + assertEquals(32, CompressionLevel.NOT_CONFIGURED.numBitsForFloat32()); + } + + public void testIsConfigured() { + assertFalse(CompressionLevel.isConfigured(CompressionLevel.NOT_CONFIGURED)); + assertFalse(CompressionLevel.isConfigured(null)); + assertTrue(CompressionLevel.isConfigured(CompressionLevel.x1)); + } +} diff --git a/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java b/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java index 369f38cf9..f04c1a4f6 100644 --- a/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java +++ b/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java @@ -32,6 +32,7 @@ import org.opensearch.index.mapper.ContentPath; import org.opensearch.index.mapper.FieldMapper; import org.opensearch.index.mapper.Mapper; +import org.opensearch.index.mapper.MapperParsingException; import org.opensearch.index.mapper.MapperService; import org.opensearch.index.mapper.ParseContext; import org.opensearch.knn.KNNTestCase; @@ -65,6 +66,7 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; import static org.opensearch.Version.CURRENT; +import static org.opensearch.knn.common.KNNConstants.COMPRESSION_LEVEL_PARAMETER; import static org.opensearch.knn.common.KNNConstants.DIMENSION; import static org.opensearch.knn.common.KNNConstants.ENCODER_SQ; import static org.opensearch.knn.common.KNNConstants.KNN_ENGINE; @@ -77,6 +79,7 @@ import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_M; import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_SPACE_TYPE; import static org.opensearch.knn.common.KNNConstants.MODEL_ID; +import static org.opensearch.knn.common.KNNConstants.MODE_PARAMETER; import static org.opensearch.knn.common.KNNConstants.NAME; import static org.opensearch.knn.common.KNNConstants.NMSLIB_NAME; import static org.opensearch.knn.common.KNNConstants.PARAMETERS; @@ -109,11 +112,27 @@ public class KNNVectorFieldMapperTests extends KNNTestCase { public void testBuilder_getParameters() { String fieldName = "test-field-name"; ModelDao modelDao = mock(ModelDao.class); - KNNVectorFieldMapper.Builder builder = new KNNVectorFieldMapper.Builder(fieldName, modelDao, CURRENT, null, null); + KNNVectorFieldMapper.Builder builder = new KNNVectorFieldMapper.Builder( + fieldName, + modelDao, + CURRENT, + null, + new OriginalMappingParameters(VectorDataType.DEFAULT, TEST_DIMENSION, null, null, null, null) + ); - assertEquals(7, builder.getParameters().size()); + assertEquals(9, builder.getParameters().size()); List actualParams = builder.getParameters().stream().map(a -> a.name).collect(Collectors.toList()); - List expectedParams = Arrays.asList("store", "doc_values", DIMENSION, VECTOR_DATA_TYPE_FIELD, "meta", KNN_METHOD, MODEL_ID); + List expectedParams = Arrays.asList( + "store", + "doc_values", + DIMENSION, + VECTOR_DATA_TYPE_FIELD, + "meta", + KNN_METHOD, + MODEL_ID, + MODE_PARAMETER, + COMPRESSION_LEVEL_PARAMETER + ); assertEquals(expectedParams, actualParams); } @@ -200,12 +219,15 @@ public void testBuilder_build_fromModel() { "", "", MethodComponentContext.EMPTY, - VectorDataType.FLOAT + VectorDataType.FLOAT, + Mode.NOT_CONFIGURED, + CompressionLevel.NOT_CONFIGURED ); builder.modelId.setValue(modelId); Mapper.BuilderContext builderContext = new Mapper.BuilderContext(settings, new ContentPath()); when(modelDao.getMetadata(modelId)).thenReturn(mockedModelMetadata); + builder.setOriginalParameters(new OriginalMappingParameters(builder)); KNNVectorFieldMapper knnVectorFieldMapper = builder.build(builderContext); assertTrue(knnVectorFieldMapper instanceof ModelFieldMapper); assertTrue(knnVectorFieldMapper.fieldType().getKnnMappingConfig().getModelId().isPresent()); @@ -396,6 +418,78 @@ public void testTypeParser_parse_fromKnnMethodContext_invalidDimension() throws ); } + @SneakyThrows + public void testTypeParser_parse_compressionAndModeParameter() { + String fieldName = "test-field-name-vec"; + String indexName = "test-index-name-vec"; + + Settings settings = Settings.builder().put(settings(CURRENT).build()).put(KNN_INDEX, true).build(); + + ModelDao modelDao = mock(ModelDao.class); + KNNVectorFieldMapper.TypeParser typeParser = new KNNVectorFieldMapper.TypeParser(() -> modelDao); + + XContentBuilder xContentBuilder1 = XContentFactory.jsonBuilder() + .startObject() + .field(TYPE_FIELD_NAME, KNN_VECTOR_TYPE) + .field(DIMENSION, 10) + .field(VECTOR_DATA_TYPE_FIELD, VectorDataType.DEFAULT.getValue()) + .field(MODE_PARAMETER, Mode.ON_DISK.getName()) + .field(COMPRESSION_LEVEL_PARAMETER, CompressionLevel.x16.getName()) + .endObject(); + + Mapper.Builder builder = typeParser.parse( + fieldName, + xContentBuilderToMap(xContentBuilder1), + buildParserContext(indexName, settings) + ); + + assertTrue(builder instanceof KNNVectorFieldMapper.Builder); + assertEquals(Mode.ON_DISK.getName(), ((KNNVectorFieldMapper.Builder) builder).mode.get()); + assertEquals(CompressionLevel.x16.getName(), ((KNNVectorFieldMapper.Builder) builder).compressionLevel.get()); + + XContentBuilder xContentBuilder2 = XContentFactory.jsonBuilder() + .startObject() + .field(TYPE_FIELD_NAME, KNN_VECTOR_TYPE) + .field(DIMENSION, 10) + .field(VECTOR_DATA_TYPE_FIELD, VectorDataType.DEFAULT.getValue()) + .field(MODE_PARAMETER, "invalid") + .field(COMPRESSION_LEVEL_PARAMETER, CompressionLevel.x16.getName()) + .endObject(); + + expectThrows( + MapperParsingException.class, + () -> typeParser.parse(fieldName, xContentBuilderToMap(xContentBuilder2), buildParserContext(indexName, settings)) + ); + + XContentBuilder xContentBuilder3 = XContentFactory.jsonBuilder() + .startObject() + .field(TYPE_FIELD_NAME, KNN_VECTOR_TYPE) + .field(DIMENSION, 10) + .field(VECTOR_DATA_TYPE_FIELD, VectorDataType.DEFAULT.getValue()) + .field(COMPRESSION_LEVEL_PARAMETER, "invalid") + .endObject(); + + expectThrows( + MapperParsingException.class, + () -> typeParser.parse(fieldName, xContentBuilderToMap(xContentBuilder3), buildParserContext(indexName, settings)) + ); + + XContentBuilder xContentBuilder4 = XContentFactory.jsonBuilder() + .startObject() + .field(TYPE_FIELD_NAME, KNN_VECTOR_TYPE) + .field(DIMENSION, 10) + .field(VECTOR_DATA_TYPE_FIELD, VectorDataType.DEFAULT.getValue()) + .field(MODEL_ID, "test") + .field(MODE_PARAMETER, Mode.ON_DISK.getName()) + .field(COMPRESSION_LEVEL_PARAMETER, CompressionLevel.x16.getName()) + .endObject(); + + expectThrows( + MapperParsingException.class, + () -> typeParser.parse(fieldName, xContentBuilderToMap(xContentBuilder4), buildParserContext(indexName, settings)) + ); + } + // Validate TypeParser parsing invalid vector data_type which throws exception @SneakyThrows public void testTypeParser_parse_invalidVectorDataType() { @@ -717,7 +811,9 @@ public void testKNNVectorFieldMapper_merge_fromModel() throws IOException { "", "", MethodComponentContext.EMPTY, - VectorDataType.FLOAT + VectorDataType.FLOAT, + Mode.NOT_CONFIGURED, + CompressionLevel.NOT_CONFIGURED ); when(mockModelDao.getMetadata(modelId)).thenReturn(mockModelMetadata); @@ -796,18 +892,27 @@ public void testMethodFieldMapperParseCreateField_validInput_thenDifferentFieldT when(parseContext.parser()).thenReturn(createXContentParser(dataType)); utilMockedStatic.when(() -> KNNVectorFieldMapperUtil.useLuceneKNNVectorsFormat(Mockito.any())).thenReturn(true); + + OriginalMappingParameters originalMappingParameters = new OriginalMappingParameters( + dataType, + dimension, + knnMethodContext, + Mode.NOT_CONFIGURED.getName(), + CompressionLevel.NOT_CONFIGURED.getName(), + null + ); + originalMappingParameters.setResolvedKnnMethodContext(knnMethodContext); MethodFieldMapper methodFieldMapper = MethodFieldMapper.createFieldMapper( TEST_FIELD_NAME, TEST_FIELD_NAME, Collections.emptyMap(), - knnMethodContext, knnMethodConfigContext, - knnMethodContext, FieldMapper.MultiFields.empty(), FieldMapper.CopyTo.empty(), new Explicit<>(true, true), false, - false + false, + originalMappingParameters ); methodFieldMapper.parseCreateField(parseContext, dimension, dataType); @@ -840,14 +945,13 @@ public void testMethodFieldMapperParseCreateField_validInput_thenDifferentFieldT TEST_FIELD_NAME, TEST_FIELD_NAME, Collections.emptyMap(), - knnMethodContext, knnMethodConfigContext, - knnMethodContext, FieldMapper.MultiFields.empty(), FieldMapper.CopyTo.empty(), new Explicit<>(true, true), false, - false + false, + originalMappingParameters ); methodFieldMapper.parseCreateField(parseContext, dimension, dataType); @@ -889,19 +993,29 @@ public void testModelFieldMapperParseCreateField_validInput_thenDifferentFieldTy when(parseContext.parser()).thenReturn(createXContentParser(dataType)); utilMockedStatic.when(() -> KNNVectorFieldMapperUtil.useLuceneKNNVectorsFormat(Mockito.any())).thenReturn(true); + + OriginalMappingParameters originalMappingParameters = new OriginalMappingParameters( + VectorDataType.DEFAULT, + -1, + null, + Mode.NOT_CONFIGURED.getName(), + CompressionLevel.NOT_CONFIGURED.getName(), + MODEL_ID + ); + ModelFieldMapper modelFieldMapper = ModelFieldMapper.createFieldMapper( TEST_FIELD_NAME, TEST_FIELD_NAME, Collections.emptyMap(), dataType, - MODEL_ID, FieldMapper.MultiFields.empty(), FieldMapper.CopyTo.empty(), new Explicit<>(true, true), false, false, modelDao, - CURRENT + CURRENT, + originalMappingParameters ); modelFieldMapper.parseCreateField(parseContext); @@ -935,14 +1049,14 @@ public void testModelFieldMapperParseCreateField_validInput_thenDifferentFieldTy TEST_FIELD_NAME, Collections.emptyMap(), dataType, - MODEL_ID, FieldMapper.MultiFields.empty(), FieldMapper.CopyTo.empty(), new Explicit<>(true, true), false, false, modelDao, - CURRENT + CURRENT, + originalMappingParameters ); modelFieldMapper.parseCreateField(parseContext); @@ -971,12 +1085,23 @@ public void testLuceneFieldMapper_parseCreateField_docValues_withFloats() { .versionCreated(CURRENT) .dimension(TEST_DIMENSION) .build(); + + OriginalMappingParameters originalMappingParameters = new OriginalMappingParameters( + VectorDataType.FLOAT, + TEST_DIMENSION, + getDefaultKNNMethodContext(), + Mode.NOT_CONFIGURED.getName(), + CompressionLevel.NOT_CONFIGURED.getName(), + null + ); + originalMappingParameters.setResolvedKnnMethodContext(originalMappingParameters.getKnnMethodContext()); + LuceneFieldMapper luceneFieldMapper = LuceneFieldMapper.createFieldMapper( TEST_FIELD_NAME, Collections.emptyMap(), - getDefaultKNNMethodContext(), knnMethodConfigContext, - inputBuilder.build() + inputBuilder.build(), + originalMappingParameters ); luceneFieldMapper.parseCreateField(parseContext, TEST_DIMENSION, VectorDataType.FLOAT); @@ -1020,12 +1145,21 @@ public void testLuceneFieldMapper_parseCreateField_docValues_withFloats() { .build(); MethodComponentContext methodComponentContext = new MethodComponentContext(METHOD_HNSW, Collections.emptyMap()); KNNMethodContext knnMethodContext = new KNNMethodContext(KNNEngine.LUCENE, SpaceType.DEFAULT, methodComponentContext); + originalMappingParameters = new OriginalMappingParameters( + VectorDataType.FLOAT, + TEST_DIMENSION, + knnMethodContext, + Mode.NOT_CONFIGURED.getName(), + CompressionLevel.NOT_CONFIGURED.getName(), + null + ); + originalMappingParameters.setResolvedKnnMethodContext(originalMappingParameters.getKnnMethodContext()); luceneFieldMapper = LuceneFieldMapper.createFieldMapper( TEST_FIELD_NAME, Collections.emptyMap(), - knnMethodContext, knnMethodConfigContext, - inputBuilder.build() + inputBuilder.build(), + originalMappingParameters ); luceneFieldMapper.parseCreateField(parseContext, TEST_DIMENSION, VectorDataType.FLOAT); @@ -1051,17 +1185,27 @@ public void testLuceneFieldMapper_parseCreateField_docValues_withBytes() { when(parseContext.doc()).thenReturn(document); when(parseContext.path()).thenReturn(contentPath); + OriginalMappingParameters originalMappingParameters = new OriginalMappingParameters( + VectorDataType.BYTE, + TEST_DIMENSION, + getDefaultByteKNNMethodContext(), + Mode.NOT_CONFIGURED.getName(), + CompressionLevel.NOT_CONFIGURED.getName(), + null + ); + originalMappingParameters.setResolvedKnnMethodContext(originalMappingParameters.getKnnMethodContext()); + LuceneFieldMapper luceneFieldMapper = Mockito.spy( LuceneFieldMapper.createFieldMapper( TEST_FIELD_NAME, Collections.emptyMap(), - getDefaultByteKNNMethodContext(), KNNMethodConfigContext.builder() .vectorDataType(VectorDataType.BYTE) .versionCreated(CURRENT) .dimension(TEST_DIMENSION) .build(), - inputBuilder.build() + inputBuilder.build(), + originalMappingParameters ) ); doReturn(Optional.of(TEST_BYTE_VECTOR)).when(luceneFieldMapper) @@ -1100,17 +1244,18 @@ public void testLuceneFieldMapper_parseCreateField_docValues_withBytes() { when(parseContext.path()).thenReturn(contentPath); inputBuilder.hasDocValues(false); + luceneFieldMapper = Mockito.spy( LuceneFieldMapper.createFieldMapper( TEST_FIELD_NAME, Collections.emptyMap(), - getDefaultByteKNNMethodContext(), KNNMethodConfigContext.builder() .vectorDataType(VectorDataType.BYTE) .versionCreated(CURRENT) .dimension(TEST_DIMENSION) .build(), - inputBuilder.build() + inputBuilder.build(), + originalMappingParameters ) ); doReturn(Optional.of(TEST_BYTE_VECTOR)).when(luceneFieldMapper) @@ -1185,7 +1330,7 @@ private void testTypeParserWithBinaryDataType( buildParserContext(indexName, settings) ); - assertEquals(spaceType, builder.getResolvedKNNMethodContext().getSpaceType()); + assertEquals(spaceType, builder.getOriginalParameters().getResolvedKnnMethodContext().getSpaceType()); } else { Exception ex = expectThrows(Exception.class, () -> { typeParser.parse(fieldName, xContentBuilderToMap(xContentBuilder), buildParserContext(indexName, settings)); @@ -1233,6 +1378,7 @@ public void testBuilder_whenBinaryWithLegacyKNNDisabled_thenValid() { // Setup settings Settings settings = Settings.builder().put(settings(CURRENT).build()).put(KNN_INDEX, false).build(); + builder.setOriginalParameters(new OriginalMappingParameters(builder)); Mapper.BuilderContext builderContext = new Mapper.BuilderContext(settings, new ContentPath()); KNNVectorFieldMapper knnVectorFieldMapper = builder.build(builderContext); assertTrue(knnVectorFieldMapper instanceof FlatVectorFieldMapper); diff --git a/src/test/java/org/opensearch/knn/index/mapper/ModeTests.java b/src/test/java/org/opensearch/knn/index/mapper/ModeTests.java new file mode 100644 index 000000000..2035bba80 --- /dev/null +++ b/src/test/java/org/opensearch/knn/index/mapper/ModeTests.java @@ -0,0 +1,33 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.mapper; + +import org.opensearch.core.common.Strings; +import org.opensearch.knn.KNNTestCase; + +public class ModeTests extends KNNTestCase { + + public void testFromName() { + assertEquals(Mode.NOT_CONFIGURED, Mode.fromName(null)); + assertEquals(Mode.NOT_CONFIGURED, Mode.fromName("")); + assertEquals(Mode.ON_DISK, Mode.fromName("on_disk")); + assertEquals(Mode.IN_MEMORY, Mode.fromName("in_memory")); + expectThrows(IllegalArgumentException.class, () -> Mode.fromName("on_disk2")); + } + + public void testGetName() { + assertTrue(Strings.isEmpty(Mode.NOT_CONFIGURED.getName())); + assertEquals("on_disk", Mode.ON_DISK.getName()); + assertEquals("in_memory", Mode.IN_MEMORY.getName()); + } + + public void testIsConfigured() { + assertFalse(Mode.isConfigured(Mode.NOT_CONFIGURED)); + assertFalse(Mode.isConfigured(null)); + assertTrue(Mode.isConfigured(Mode.ON_DISK)); + } + +} diff --git a/src/test/java/org/opensearch/knn/index/mapper/OriginalMappingParametersTests.java b/src/test/java/org/opensearch/knn/index/mapper/OriginalMappingParametersTests.java new file mode 100644 index 000000000..2822a882e --- /dev/null +++ b/src/test/java/org/opensearch/knn/index/mapper/OriginalMappingParametersTests.java @@ -0,0 +1,38 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.mapper; + +import org.opensearch.knn.KNNTestCase; +import org.opensearch.knn.index.SpaceType; +import org.opensearch.knn.index.VectorDataType; +import org.opensearch.knn.index.engine.KNNEngine; +import org.opensearch.knn.index.engine.KNNMethodContext; +import org.opensearch.knn.index.engine.MethodComponentContext; + +import java.util.Collections; + +public class OriginalMappingParametersTests extends KNNTestCase { + + public void testIsLegacy() { + assertTrue(new OriginalMappingParameters(VectorDataType.DEFAULT, 123, null, null, null, null).isLegacyMapping()); + assertFalse(new OriginalMappingParameters(VectorDataType.DEFAULT, 123, null, null, null, "model-id").isLegacyMapping()); + assertFalse(new OriginalMappingParameters(VectorDataType.DEFAULT, 123, null, Mode.ON_DISK.getName(), null, null).isLegacyMapping()); + assertFalse( + new OriginalMappingParameters(VectorDataType.DEFAULT, 123, null, null, CompressionLevel.x2.getName(), null).isLegacyMapping() + ); + assertFalse( + new OriginalMappingParameters( + VectorDataType.DEFAULT, + 123, + new KNNMethodContext(KNNEngine.DEFAULT, SpaceType.L2, new MethodComponentContext(null, Collections.emptyMap())), + null, + null, + null + ).isLegacyMapping() + ); + } + +} diff --git a/src/test/java/org/opensearch/knn/indices/ModelCacheTests.java b/src/test/java/org/opensearch/knn/indices/ModelCacheTests.java index 88f78e716..91bb7d3d9 100644 --- a/src/test/java/org/opensearch/knn/indices/ModelCacheTests.java +++ b/src/test/java/org/opensearch/knn/indices/ModelCacheTests.java @@ -21,6 +21,8 @@ import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.engine.KNNEngine; +import org.opensearch.knn.index.mapper.CompressionLevel; +import org.opensearch.knn.index.mapper.Mode; import java.time.ZoneOffset; import java.time.ZonedDateTime; @@ -47,7 +49,9 @@ public void testGet_normal() throws ExecutionException, InterruptedException { "", "", MethodComponentContext.EMPTY, - VectorDataType.DEFAULT + VectorDataType.DEFAULT, + Mode.NOT_CONFIGURED, + CompressionLevel.NOT_CONFIGURED ), "hello".getBytes(), modelId @@ -85,7 +89,9 @@ public void testGet_modelDoesNotFitInCache() throws ExecutionException, Interrup "", "", MethodComponentContext.EMPTY, - VectorDataType.DEFAULT + VectorDataType.DEFAULT, + Mode.NOT_CONFIGURED, + CompressionLevel.NOT_CONFIGURED ), new byte[BYTES_PER_KILOBYTES + 1], modelId @@ -144,7 +150,9 @@ public void testGetTotalWeight() throws ExecutionException, InterruptedException "", "", MethodComponentContext.EMPTY, - VectorDataType.DEFAULT + VectorDataType.DEFAULT, + Mode.NOT_CONFIGURED, + CompressionLevel.NOT_CONFIGURED ), new byte[size1], modelId1 @@ -161,7 +169,9 @@ public void testGetTotalWeight() throws ExecutionException, InterruptedException "", "", MethodComponentContext.EMPTY, - VectorDataType.DEFAULT + VectorDataType.DEFAULT, + Mode.NOT_CONFIGURED, + CompressionLevel.NOT_CONFIGURED ), new byte[size2], modelId2 @@ -206,7 +216,9 @@ public void testRemove_normal() throws ExecutionException, InterruptedException "", "", MethodComponentContext.EMPTY, - VectorDataType.DEFAULT + VectorDataType.DEFAULT, + Mode.NOT_CONFIGURED, + CompressionLevel.NOT_CONFIGURED ), new byte[size1], modelId1 @@ -223,7 +235,9 @@ public void testRemove_normal() throws ExecutionException, InterruptedException "", "", MethodComponentContext.EMPTY, - VectorDataType.DEFAULT + VectorDataType.DEFAULT, + Mode.NOT_CONFIGURED, + CompressionLevel.NOT_CONFIGURED ), new byte[size2], modelId2 @@ -273,7 +287,9 @@ public void testRebuild_normal() throws ExecutionException, InterruptedException "", "", MethodComponentContext.EMPTY, - VectorDataType.DEFAULT + VectorDataType.DEFAULT, + Mode.NOT_CONFIGURED, + CompressionLevel.NOT_CONFIGURED ), "hello".getBytes(), modelId @@ -320,7 +336,9 @@ public void testRebuild_afterSettingUpdate() throws ExecutionException, Interrup "", "", MethodComponentContext.EMPTY, - VectorDataType.DEFAULT + VectorDataType.DEFAULT, + Mode.NOT_CONFIGURED, + CompressionLevel.NOT_CONFIGURED ), new byte[modelSize], modelId @@ -390,7 +408,9 @@ public void testContains() throws ExecutionException, InterruptedException { "", "", MethodComponentContext.EMPTY, - VectorDataType.DEFAULT + VectorDataType.DEFAULT, + Mode.NOT_CONFIGURED, + CompressionLevel.NOT_CONFIGURED ), new byte[modelSize1], modelId1 @@ -433,7 +453,9 @@ public void testRemoveAll() throws ExecutionException, InterruptedException { "", "", MethodComponentContext.EMPTY, - VectorDataType.DEFAULT + VectorDataType.DEFAULT, + Mode.NOT_CONFIGURED, + CompressionLevel.NOT_CONFIGURED ), new byte[modelSize1], modelId1 @@ -452,7 +474,9 @@ public void testRemoveAll() throws ExecutionException, InterruptedException { "", "", MethodComponentContext.EMPTY, - VectorDataType.DEFAULT + VectorDataType.DEFAULT, + Mode.NOT_CONFIGURED, + CompressionLevel.NOT_CONFIGURED ), new byte[modelSize2], modelId2 @@ -499,7 +523,9 @@ public void testModelCacheEvictionDueToSize() throws ExecutionException, Interru "", "", MethodComponentContext.EMPTY, - VectorDataType.DEFAULT + VectorDataType.DEFAULT, + Mode.NOT_CONFIGURED, + CompressionLevel.NOT_CONFIGURED ), new byte[BYTES_PER_KILOBYTES * 2], modelId diff --git a/src/test/java/org/opensearch/knn/indices/ModelDaoTests.java b/src/test/java/org/opensearch/knn/indices/ModelDaoTests.java index d9dab081c..560ea59b2 100644 --- a/src/test/java/org/opensearch/knn/indices/ModelDaoTests.java +++ b/src/test/java/org/opensearch/knn/indices/ModelDaoTests.java @@ -37,6 +37,8 @@ import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.engine.KNNEngine; +import org.opensearch.knn.index.mapper.CompressionLevel; +import org.opensearch.knn.index.mapper.Mode; import org.opensearch.knn.plugin.transport.DeleteModelResponse; import org.opensearch.knn.plugin.transport.GetModelResponse; import org.opensearch.knn.plugin.transport.RemoveModelFromCacheAction; @@ -141,7 +143,9 @@ public void testModelIndexHealth() throws InterruptedException, ExecutionExcepti "", "", MethodComponentContext.EMPTY, - VectorDataType.DEFAULT + VectorDataType.DEFAULT, + Mode.NOT_CONFIGURED, + CompressionLevel.NOT_CONFIGURED ), modelBlob, modelId @@ -162,7 +166,9 @@ public void testModelIndexHealth() throws InterruptedException, ExecutionExcepti "", "", MethodComponentContext.EMPTY, - VectorDataType.DEFAULT + VectorDataType.DEFAULT, + Mode.NOT_CONFIGURED, + CompressionLevel.NOT_CONFIGURED ), modelBlob, modelId @@ -191,7 +197,9 @@ public void testPut_withId() throws InterruptedException, IOException { "", "", new MethodComponentContext("test", Collections.emptyMap()), - VectorDataType.DEFAULT + VectorDataType.DEFAULT, + Mode.NOT_CONFIGURED, + CompressionLevel.NOT_CONFIGURED ), modelBlob, modelId @@ -253,7 +261,9 @@ public void testPut_withoutModel() throws InterruptedException, IOException { "", "", MethodComponentContext.EMPTY, - VectorDataType.DEFAULT + VectorDataType.DEFAULT, + Mode.NOT_CONFIGURED, + CompressionLevel.NOT_CONFIGURED ), modelBlob, modelId @@ -316,7 +326,9 @@ public void testPut_invalid_badState() { "", "", MethodComponentContext.EMPTY, - VectorDataType.DEFAULT + VectorDataType.DEFAULT, + Mode.NOT_CONFIGURED, + CompressionLevel.NOT_CONFIGURED ), modelBlob, "any-id" @@ -354,7 +366,9 @@ public void testUpdate() throws IOException, InterruptedException { "", "", MethodComponentContext.EMPTY, - VectorDataType.DEFAULT + VectorDataType.DEFAULT, + Mode.NOT_CONFIGURED, + CompressionLevel.NOT_CONFIGURED ), null, modelId @@ -394,7 +408,9 @@ public void testUpdate() throws IOException, InterruptedException { "", "", MethodComponentContext.EMPTY, - VectorDataType.DEFAULT + VectorDataType.DEFAULT, + Mode.NOT_CONFIGURED, + CompressionLevel.NOT_CONFIGURED ), modelBlob, modelId @@ -446,7 +462,9 @@ public void testGet() throws IOException, InterruptedException, ExecutionExcepti "", "", MethodComponentContext.EMPTY, - VectorDataType.DEFAULT + VectorDataType.DEFAULT, + Mode.NOT_CONFIGURED, + CompressionLevel.NOT_CONFIGURED ), modelBlob, modelId @@ -466,7 +484,9 @@ public void testGet() throws IOException, InterruptedException, ExecutionExcepti "", "", MethodComponentContext.EMPTY, - VectorDataType.DEFAULT + VectorDataType.DEFAULT, + Mode.NOT_CONFIGURED, + CompressionLevel.NOT_CONFIGURED ), null, modelId @@ -504,7 +524,9 @@ public void testGetMetadata() throws IOException, InterruptedException { "", "", MethodComponentContext.EMPTY, - VectorDataType.DEFAULT + VectorDataType.DEFAULT, + Mode.NOT_CONFIGURED, + CompressionLevel.NOT_CONFIGURED ); Model model = new Model(modelMetadata, modelBlob, modelId); @@ -582,7 +604,9 @@ public void testDelete() throws IOException, InterruptedException { "", "", MethodComponentContext.EMPTY, - VectorDataType.DEFAULT + VectorDataType.DEFAULT, + Mode.NOT_CONFIGURED, + CompressionLevel.NOT_CONFIGURED ), modelBlob, modelId @@ -617,7 +641,9 @@ public void testDelete() throws IOException, InterruptedException { "", "", MethodComponentContext.EMPTY, - VectorDataType.DEFAULT + VectorDataType.DEFAULT, + Mode.NOT_CONFIGURED, + CompressionLevel.NOT_CONFIGURED ), modelBlob, modelId1 @@ -686,7 +712,9 @@ public void testDeleteModelInTrainingWithStepListeners() throws IOException, Exe "", "", MethodComponentContext.EMPTY, - VectorDataType.DEFAULT + VectorDataType.DEFAULT, + Mode.NOT_CONFIGURED, + CompressionLevel.NOT_CONFIGURED ), modelBlob, modelId @@ -729,7 +757,9 @@ public void testDeleteWithStepListeners() throws IOException, InterruptedExcepti "", "", MethodComponentContext.EMPTY, - VectorDataType.DEFAULT + VectorDataType.DEFAULT, + Mode.NOT_CONFIGURED, + CompressionLevel.NOT_CONFIGURED ), modelBlob, modelId diff --git a/src/test/java/org/opensearch/knn/indices/ModelMetadataTests.java b/src/test/java/org/opensearch/knn/indices/ModelMetadataTests.java index 04fa50262..6f0b49285 100644 --- a/src/test/java/org/opensearch/knn/indices/ModelMetadataTests.java +++ b/src/test/java/org/opensearch/knn/indices/ModelMetadataTests.java @@ -21,6 +21,8 @@ import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.engine.KNNEngine; +import org.opensearch.knn.index.mapper.CompressionLevel; +import org.opensearch.knn.index.mapper.Mode; import java.io.IOException; import java.time.ZoneId; @@ -47,14 +49,33 @@ public void testStreams() throws IOException { "", "", MethodComponentContext.EMPTY, - VectorDataType.DEFAULT + VectorDataType.DEFAULT, + Mode.NOT_CONFIGURED, + CompressionLevel.NOT_CONFIGURED ); BytesStreamOutput streamOutput = new BytesStreamOutput(); modelMetadata.writeTo(streamOutput); - ModelMetadata modelMetadataCopy = new ModelMetadata(streamOutput.bytes().streamInput()); + assertEquals(modelMetadata, modelMetadataCopy); + modelMetadata = new ModelMetadata( + knnEngine, + spaceType, + dimension, + ModelState.CREATED, + ZonedDateTime.now(ZoneOffset.UTC).toString(), + "", + "", + "", + MethodComponentContext.EMPTY, + VectorDataType.DEFAULT, + Mode.ON_DISK, + CompressionLevel.x16 + ); + streamOutput = new BytesStreamOutput(); + modelMetadata.writeTo(streamOutput); + modelMetadataCopy = new ModelMetadata(streamOutput.bytes().streamInput()); assertEquals(modelMetadata, modelMetadataCopy); } @@ -70,7 +91,9 @@ public void testGetKnnEngine() { "", "", MethodComponentContext.EMPTY, - VectorDataType.DEFAULT + VectorDataType.DEFAULT, + Mode.NOT_CONFIGURED, + CompressionLevel.NOT_CONFIGURED ); assertEquals(knnEngine, modelMetadata.getKnnEngine()); @@ -88,7 +111,9 @@ public void testGetSpaceType() { "", "", MethodComponentContext.EMPTY, - VectorDataType.DEFAULT + VectorDataType.DEFAULT, + Mode.NOT_CONFIGURED, + CompressionLevel.NOT_CONFIGURED ); assertEquals(spaceType, modelMetadata.getSpaceType()); @@ -106,7 +131,9 @@ public void testGetDimension() { "", "", MethodComponentContext.EMPTY, - VectorDataType.DEFAULT + VectorDataType.DEFAULT, + Mode.NOT_CONFIGURED, + CompressionLevel.NOT_CONFIGURED ); assertEquals(dimension, modelMetadata.getDimension()); @@ -124,7 +151,9 @@ public void testGetState() { "", "", MethodComponentContext.EMPTY, - VectorDataType.DEFAULT + VectorDataType.DEFAULT, + Mode.NOT_CONFIGURED, + CompressionLevel.NOT_CONFIGURED ); assertEquals(modelState, modelMetadata.getState()); @@ -142,7 +171,9 @@ public void testGetTimestamp() { "", "", MethodComponentContext.EMPTY, - VectorDataType.DEFAULT + VectorDataType.DEFAULT, + Mode.NOT_CONFIGURED, + CompressionLevel.NOT_CONFIGURED ); assertEquals(timeValue, modelMetadata.getTimestamp()); @@ -160,7 +191,9 @@ public void testDescription() { "", "", MethodComponentContext.EMPTY, - VectorDataType.DEFAULT + VectorDataType.DEFAULT, + Mode.NOT_CONFIGURED, + CompressionLevel.NOT_CONFIGURED ); assertEquals(description, modelMetadata.getDescription()); @@ -178,7 +211,9 @@ public void testGetError() { error, "", MethodComponentContext.EMPTY, - VectorDataType.DEFAULT + VectorDataType.DEFAULT, + Mode.NOT_CONFIGURED, + CompressionLevel.NOT_CONFIGURED ); assertEquals(error, modelMetadata.getError()); @@ -196,7 +231,9 @@ public void testGetVectorDataType() { "", "", MethodComponentContext.EMPTY, - vectorDataType + vectorDataType, + Mode.NOT_CONFIGURED, + CompressionLevel.NOT_CONFIGURED ); assertEquals(vectorDataType, modelMetadata.getVectorDataType()); @@ -214,7 +251,9 @@ public void testSetState() { "", "", MethodComponentContext.EMPTY, - VectorDataType.DEFAULT + VectorDataType.DEFAULT, + Mode.NOT_CONFIGURED, + CompressionLevel.NOT_CONFIGURED ); assertEquals(modelState, modelMetadata.getState()); @@ -236,7 +275,9 @@ public void testSetError() { error, "", MethodComponentContext.EMPTY, - VectorDataType.DEFAULT + VectorDataType.DEFAULT, + Mode.NOT_CONFIGURED, + CompressionLevel.NOT_CONFIGURED ); assertEquals(error, modelMetadata.getError()); @@ -275,7 +316,9 @@ public void testToString() { + "," + methodComponentContext.toClusterStateString() + "," - + VectorDataType.DEFAULT.getValue(); + + VectorDataType.DEFAULT.getValue() + + "," + + ","; ModelMetadata modelMetadata = new ModelMetadata( knnEngine, @@ -287,7 +330,50 @@ public void testToString() { error, nodeAssignment, MethodComponentContext.EMPTY, - VectorDataType.DEFAULT + VectorDataType.DEFAULT, + Mode.NOT_CONFIGURED, + CompressionLevel.NOT_CONFIGURED + ); + + assertEquals(expected, modelMetadata.toString()); + + expected = knnEngine.getName() + + "," + + spaceType.getValue() + + "," + + dimension + + "," + + modelState.getName() + + "," + + timestamp + + "," + + description + + "," + + error + + "," + + nodeAssignment + + "," + + methodComponentContext.toClusterStateString() + + "," + + VectorDataType.DEFAULT.getValue() + + "," + + Mode.ON_DISK.getName() + + "," + + CompressionLevel.x32.getName(); + + modelMetadata = new ModelMetadata( + knnEngine, + spaceType, + dimension, + modelState, + timestamp, + description, + error, + nodeAssignment, + MethodComponentContext.EMPTY, + VectorDataType.DEFAULT, + Mode.ON_DISK, + CompressionLevel.x32 ); assertEquals(expected, modelMetadata.toString()); @@ -308,7 +394,9 @@ public void testEquals() { "", "", MethodComponentContext.EMPTY, - VectorDataType.DEFAULT + VectorDataType.DEFAULT, + Mode.NOT_CONFIGURED, + CompressionLevel.NOT_CONFIGURED ); ModelMetadata modelMetadata2 = new ModelMetadata( KNNEngine.FAISS, @@ -320,7 +408,9 @@ public void testEquals() { "", "", MethodComponentContext.EMPTY, - VectorDataType.DEFAULT + VectorDataType.DEFAULT, + Mode.NOT_CONFIGURED, + CompressionLevel.NOT_CONFIGURED ); ModelMetadata modelMetadata3 = new ModelMetadata( @@ -333,7 +423,9 @@ public void testEquals() { "", "", MethodComponentContext.EMPTY, - VectorDataType.DEFAULT + VectorDataType.DEFAULT, + Mode.NOT_CONFIGURED, + CompressionLevel.NOT_CONFIGURED ); ModelMetadata modelMetadata4 = new ModelMetadata( KNNEngine.FAISS, @@ -345,7 +437,9 @@ public void testEquals() { "", "", MethodComponentContext.EMPTY, - VectorDataType.DEFAULT + VectorDataType.DEFAULT, + Mode.NOT_CONFIGURED, + CompressionLevel.NOT_CONFIGURED ); ModelMetadata modelMetadata5 = new ModelMetadata( KNNEngine.FAISS, @@ -357,7 +451,9 @@ public void testEquals() { "", "", MethodComponentContext.EMPTY, - VectorDataType.DEFAULT + VectorDataType.DEFAULT, + Mode.NOT_CONFIGURED, + CompressionLevel.NOT_CONFIGURED ); ModelMetadata modelMetadata6 = new ModelMetadata( KNNEngine.FAISS, @@ -369,7 +465,9 @@ public void testEquals() { "", "", MethodComponentContext.EMPTY, - VectorDataType.DEFAULT + VectorDataType.DEFAULT, + Mode.NOT_CONFIGURED, + CompressionLevel.NOT_CONFIGURED ); ModelMetadata modelMetadata7 = new ModelMetadata( KNNEngine.FAISS, @@ -381,7 +479,9 @@ public void testEquals() { "", "", MethodComponentContext.EMPTY, - VectorDataType.DEFAULT + VectorDataType.DEFAULT, + Mode.NOT_CONFIGURED, + CompressionLevel.NOT_CONFIGURED ); ModelMetadata modelMetadata8 = new ModelMetadata( KNNEngine.FAISS, @@ -393,7 +493,9 @@ public void testEquals() { "", "", MethodComponentContext.EMPTY, - VectorDataType.DEFAULT + VectorDataType.DEFAULT, + Mode.NOT_CONFIGURED, + CompressionLevel.NOT_CONFIGURED ); ModelMetadata modelMetadata9 = new ModelMetadata( KNNEngine.FAISS, @@ -405,7 +507,9 @@ public void testEquals() { "diff error", "", MethodComponentContext.EMPTY, - VectorDataType.DEFAULT + VectorDataType.DEFAULT, + Mode.NOT_CONFIGURED, + CompressionLevel.NOT_CONFIGURED ); ModelMetadata modelMetadata10 = new ModelMetadata( @@ -418,7 +522,9 @@ public void testEquals() { "", "", new MethodComponentContext("test", Collections.emptyMap()), - VectorDataType.DEFAULT + VectorDataType.DEFAULT, + Mode.NOT_CONFIGURED, + CompressionLevel.NOT_CONFIGURED ); assertEquals(modelMetadata1, modelMetadata1); @@ -449,7 +555,9 @@ public void testHashCode() { "", "", MethodComponentContext.EMPTY, - VectorDataType.DEFAULT + VectorDataType.DEFAULT, + Mode.NOT_CONFIGURED, + CompressionLevel.NOT_CONFIGURED ); ModelMetadata modelMetadata2 = new ModelMetadata( KNNEngine.FAISS, @@ -461,7 +569,9 @@ public void testHashCode() { "", "", MethodComponentContext.EMPTY, - VectorDataType.DEFAULT + VectorDataType.DEFAULT, + Mode.NOT_CONFIGURED, + CompressionLevel.NOT_CONFIGURED ); ModelMetadata modelMetadata3 = new ModelMetadata( @@ -474,7 +584,9 @@ public void testHashCode() { "", "", MethodComponentContext.EMPTY, - VectorDataType.DEFAULT + VectorDataType.DEFAULT, + Mode.NOT_CONFIGURED, + CompressionLevel.NOT_CONFIGURED ); ModelMetadata modelMetadata4 = new ModelMetadata( KNNEngine.FAISS, @@ -486,7 +598,9 @@ public void testHashCode() { "", "", MethodComponentContext.EMPTY, - VectorDataType.DEFAULT + VectorDataType.DEFAULT, + Mode.NOT_CONFIGURED, + CompressionLevel.NOT_CONFIGURED ); ModelMetadata modelMetadata5 = new ModelMetadata( KNNEngine.FAISS, @@ -498,7 +612,9 @@ public void testHashCode() { "", "", MethodComponentContext.EMPTY, - VectorDataType.DEFAULT + VectorDataType.DEFAULT, + Mode.NOT_CONFIGURED, + CompressionLevel.NOT_CONFIGURED ); ModelMetadata modelMetadata6 = new ModelMetadata( KNNEngine.FAISS, @@ -510,7 +626,9 @@ public void testHashCode() { "", "", MethodComponentContext.EMPTY, - VectorDataType.DEFAULT + VectorDataType.DEFAULT, + Mode.NOT_CONFIGURED, + CompressionLevel.NOT_CONFIGURED ); ModelMetadata modelMetadata7 = new ModelMetadata( KNNEngine.FAISS, @@ -522,7 +640,9 @@ public void testHashCode() { "", "", MethodComponentContext.EMPTY, - VectorDataType.DEFAULT + VectorDataType.DEFAULT, + Mode.NOT_CONFIGURED, + CompressionLevel.NOT_CONFIGURED ); ModelMetadata modelMetadata8 = new ModelMetadata( KNNEngine.FAISS, @@ -534,7 +654,9 @@ public void testHashCode() { "", "", MethodComponentContext.EMPTY, - VectorDataType.DEFAULT + VectorDataType.DEFAULT, + Mode.NOT_CONFIGURED, + CompressionLevel.NOT_CONFIGURED ); ModelMetadata modelMetadata9 = new ModelMetadata( KNNEngine.FAISS, @@ -546,7 +668,9 @@ public void testHashCode() { "diff error", "", MethodComponentContext.EMPTY, - VectorDataType.DEFAULT + VectorDataType.DEFAULT, + Mode.NOT_CONFIGURED, + CompressionLevel.NOT_CONFIGURED ); ModelMetadata modelMetadata10 = new ModelMetadata( @@ -559,7 +683,9 @@ public void testHashCode() { "", "", new MethodComponentContext("test", Collections.emptyMap()), - VectorDataType.DEFAULT + VectorDataType.DEFAULT, + Mode.NOT_CONFIGURED, + CompressionLevel.NOT_CONFIGURED ); assertEquals(modelMetadata1.hashCode(), modelMetadata1.hashCode()); @@ -622,6 +748,52 @@ public void testFromString() { + "," + VectorDataType.DEFAULT.getValue(); + String stringRep3 = knnEngine.getName() + + "," + + spaceType.getValue() + + "," + + dimension + + "," + + modelState.getName() + + "," + + timestamp + + "," + + description + + "," + + error + + "," + + nodeAssignment + + "," + + methodComponentContext.toClusterStateString() + + "," + + VectorDataType.DEFAULT.getValue() + + "," + + ","; + + String stringRep4 = knnEngine.getName() + + "," + + spaceType.getValue() + + "," + + dimension + + "," + + modelState.getName() + + "," + + timestamp + + "," + + description + + "," + + error + + "," + + nodeAssignment + + "," + + methodComponentContext.toClusterStateString() + + "," + + VectorDataType.DEFAULT.getValue() + + "," + + Mode.ON_DISK.getName() + + "," + + CompressionLevel.x32.getName(); + ModelMetadata expected1 = new ModelMetadata( knnEngine, spaceType, @@ -632,7 +804,9 @@ public void testFromString() { error, nodeAssignment, MethodComponentContext.EMPTY, - VectorDataType.DEFAULT + VectorDataType.DEFAULT, + Mode.NOT_CONFIGURED, + CompressionLevel.NOT_CONFIGURED ); ModelMetadata expected2 = new ModelMetadata( @@ -645,14 +819,35 @@ public void testFromString() { error, "", MethodComponentContext.EMPTY, - VectorDataType.DEFAULT + VectorDataType.DEFAULT, + Mode.NOT_CONFIGURED, + CompressionLevel.NOT_CONFIGURED + ); + + ModelMetadata expected3 = new ModelMetadata( + knnEngine, + spaceType, + dimension, + modelState, + timestamp, + description, + error, + "", + MethodComponentContext.EMPTY, + VectorDataType.DEFAULT, + Mode.ON_DISK, + CompressionLevel.x32 ); ModelMetadata fromString1 = ModelMetadata.fromString(stringRep1); ModelMetadata fromString2 = ModelMetadata.fromString(stringRep2); + ModelMetadata fromString3 = ModelMetadata.fromString(stringRep3); + ModelMetadata fromString4 = ModelMetadata.fromString(stringRep4); assertEquals(expected1, fromString1); assertEquals(expected2, fromString2); + assertEquals(expected2, fromString3); + assertEquals(expected3, fromString4); expectThrows(IllegalArgumentException.class, () -> ModelMetadata.fromString("invalid")); } @@ -679,7 +874,9 @@ public void testFromResponseMap() throws IOException { error, nodeAssignment, methodComponentContext, - VectorDataType.DEFAULT + VectorDataType.DEFAULT, + Mode.NOT_CONFIGURED, + CompressionLevel.NOT_CONFIGURED ); ModelMetadata expected2 = new ModelMetadata( knnEngine, @@ -691,7 +888,39 @@ public void testFromResponseMap() throws IOException { error, "", emptyMethodComponentContext, - VectorDataType.DEFAULT + VectorDataType.DEFAULT, + Mode.NOT_CONFIGURED, + CompressionLevel.NOT_CONFIGURED + ); + + ModelMetadata expected3 = new ModelMetadata( + knnEngine, + spaceType, + dimension, + modelState, + timestamp, + description, + error, + "", + emptyMethodComponentContext, + VectorDataType.DEFAULT, + Mode.ON_DISK, + CompressionLevel.NOT_CONFIGURED + ); + + ModelMetadata expected4 = new ModelMetadata( + knnEngine, + spaceType, + dimension, + modelState, + timestamp, + description, + error, + "", + emptyMethodComponentContext, + VectorDataType.DEFAULT, + Mode.ON_DISK, + CompressionLevel.x16 ); Map metadataAsMap = new HashMap<>(); metadataAsMap.put(KNNConstants.KNN_ENGINE, knnEngine.getName()); @@ -714,6 +943,14 @@ public void testFromResponseMap() throws IOException { metadataAsMap.put(KNNConstants.MODEL_NODE_ASSIGNMENT, null); metadataAsMap.put(KNNConstants.MODEL_METHOD_COMPONENT_CONTEXT, null); assertEquals(expected2, fromMap); + + metadataAsMap.put(KNNConstants.MODE_PARAMETER, Mode.ON_DISK.getName()); + fromMap = ModelMetadata.getMetadataFromSourceMap(metadataAsMap); + assertEquals(expected3, fromMap); + + metadataAsMap.put(KNNConstants.COMPRESSION_LEVEL_PARAMETER, CompressionLevel.x16.getName()); + fromMap = ModelMetadata.getMetadataFromSourceMap(metadataAsMap); + assertEquals(expected4, fromMap); } public void testBlockCommasInDescription() { @@ -739,7 +976,9 @@ public void testBlockCommasInDescription() { error, nodeAssignment, methodComponentContext, - VectorDataType.DEFAULT + VectorDataType.DEFAULT, + Mode.NOT_CONFIGURED, + CompressionLevel.NOT_CONFIGURED ) ); assertEquals("Model description cannot contain any commas: ','", e.getMessage()); diff --git a/src/test/java/org/opensearch/knn/indices/ModelTests.java b/src/test/java/org/opensearch/knn/indices/ModelTests.java index 45e8b05f1..4e666872f 100644 --- a/src/test/java/org/opensearch/knn/indices/ModelTests.java +++ b/src/test/java/org/opensearch/knn/indices/ModelTests.java @@ -17,6 +17,8 @@ import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.engine.KNNEngine; +import org.opensearch.knn.index.mapper.CompressionLevel; +import org.opensearch.knn.index.mapper.Mode; import java.time.ZoneOffset; import java.time.ZonedDateTime; @@ -43,7 +45,9 @@ public void testInvalidConstructor() { "", "", MethodComponentContext.EMPTY, - VectorDataType.DEFAULT + VectorDataType.DEFAULT, + Mode.NOT_CONFIGURED, + CompressionLevel.NOT_CONFIGURED ), null, "test-model" @@ -65,7 +69,9 @@ public void testInvalidDimension() { "", "", MethodComponentContext.EMPTY, - VectorDataType.DEFAULT + VectorDataType.DEFAULT, + Mode.NOT_CONFIGURED, + CompressionLevel.NOT_CONFIGURED ), new byte[16], "test-model" @@ -84,7 +90,9 @@ public void testInvalidDimension() { "", "", MethodComponentContext.EMPTY, - VectorDataType.DEFAULT + VectorDataType.DEFAULT, + Mode.NOT_CONFIGURED, + CompressionLevel.NOT_CONFIGURED ), new byte[16], "test-model" @@ -103,7 +111,9 @@ public void testInvalidDimension() { "", "", MethodComponentContext.EMPTY, - VectorDataType.DEFAULT + VectorDataType.DEFAULT, + Mode.NOT_CONFIGURED, + CompressionLevel.NOT_CONFIGURED ), new byte[16], "test-model" @@ -123,7 +133,9 @@ public void testGetModelMetadata() { "", "", MethodComponentContext.EMPTY, - VectorDataType.DEFAULT + VectorDataType.DEFAULT, + Mode.NOT_CONFIGURED, + CompressionLevel.NOT_CONFIGURED ); Model model = new Model(modelMetadata, new byte[16], "test-model"); assertEquals(modelMetadata, model.getModelMetadata()); @@ -142,7 +154,9 @@ public void testGetModelBlob() { "", "", MethodComponentContext.EMPTY, - VectorDataType.DEFAULT + VectorDataType.DEFAULT, + Mode.NOT_CONFIGURED, + CompressionLevel.NOT_CONFIGURED ), modelBlob, "test-model" @@ -163,7 +177,9 @@ public void testGetLength() { "", "", MethodComponentContext.EMPTY, - VectorDataType.DEFAULT + VectorDataType.DEFAULT, + Mode.NOT_CONFIGURED, + CompressionLevel.NOT_CONFIGURED ), new byte[size], "test-model" @@ -181,7 +197,9 @@ public void testGetLength() { "", "", MethodComponentContext.EMPTY, - VectorDataType.DEFAULT + VectorDataType.DEFAULT, + Mode.NOT_CONFIGURED, + CompressionLevel.NOT_CONFIGURED ), null, "test-model" @@ -202,7 +220,9 @@ public void testSetModelBlob() { "", "", MethodComponentContext.EMPTY, - VectorDataType.DEFAULT + VectorDataType.DEFAULT, + Mode.NOT_CONFIGURED, + CompressionLevel.NOT_CONFIGURED ), blob1, "test-model" @@ -229,7 +249,9 @@ public void testEquals() { "", "", MethodComponentContext.EMPTY, - VectorDataType.DEFAULT + VectorDataType.DEFAULT, + Mode.NOT_CONFIGURED, + CompressionLevel.NOT_CONFIGURED ), new byte[16], "test-model-1" @@ -245,7 +267,9 @@ public void testEquals() { "", "", MethodComponentContext.EMPTY, - VectorDataType.DEFAULT + VectorDataType.DEFAULT, + Mode.NOT_CONFIGURED, + CompressionLevel.NOT_CONFIGURED ), new byte[16], "test-model-1" @@ -261,7 +285,9 @@ public void testEquals() { "", "", MethodComponentContext.EMPTY, - VectorDataType.DEFAULT + VectorDataType.DEFAULT, + Mode.NOT_CONFIGURED, + CompressionLevel.NOT_CONFIGURED ), new byte[16], "test-model-2" @@ -287,7 +313,9 @@ public void testHashCode() { "", "", MethodComponentContext.EMPTY, - VectorDataType.DEFAULT + VectorDataType.DEFAULT, + Mode.NOT_CONFIGURED, + CompressionLevel.NOT_CONFIGURED ), new byte[16], "test-model-1" @@ -303,7 +331,9 @@ public void testHashCode() { "", "", MethodComponentContext.EMPTY, - VectorDataType.DEFAULT + VectorDataType.DEFAULT, + Mode.NOT_CONFIGURED, + CompressionLevel.NOT_CONFIGURED ), new byte[16], "test-model-1" @@ -319,7 +349,9 @@ public void testHashCode() { "", "", MethodComponentContext.EMPTY, - VectorDataType.DEFAULT + VectorDataType.DEFAULT, + Mode.NOT_CONFIGURED, + CompressionLevel.NOT_CONFIGURED ), new byte[16], "test-model-2" @@ -351,7 +383,9 @@ public void testModelFromSourceMap() { error, nodeAssignment, MethodComponentContext.EMPTY, - VectorDataType.DEFAULT + VectorDataType.DEFAULT, + Mode.NOT_CONFIGURED, + CompressionLevel.NOT_CONFIGURED ); Map modelAsMap = new HashMap<>(); modelAsMap.put(KNNConstants.MODEL_ID, modelID); @@ -365,6 +399,8 @@ public void testModelFromSourceMap() { modelAsMap.put(KNNConstants.MODEL_NODE_ASSIGNMENT, nodeAssignment); modelAsMap.put(KNNConstants.MODEL_BLOB_PARAMETER, "aGVsbG8="); modelAsMap.put(KNNConstants.VECTOR_DATA_TYPE_FIELD, VectorDataType.DEFAULT.getValue()); + modelAsMap.put(KNNConstants.MODE_PARAMETER, Mode.NOT_CONFIGURED.getName()); + modelAsMap.put(KNNConstants.COMPRESSION_LEVEL_PARAMETER, CompressionLevel.NOT_CONFIGURED.getName()); byte[] blob1 = "hello".getBytes(); Model expected = new Model(metadata, blob1, modelID); diff --git a/src/test/java/org/opensearch/knn/integ/ModeAndCompressionIT.java b/src/test/java/org/opensearch/knn/integ/ModeAndCompressionIT.java new file mode 100644 index 000000000..bf2e7f0c0 --- /dev/null +++ b/src/test/java/org/opensearch/knn/integ/ModeAndCompressionIT.java @@ -0,0 +1,155 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.integ; + +import lombok.SneakyThrows; +import org.opensearch.client.Response; +import org.opensearch.client.ResponseException; +import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.knn.KNNRestTestCase; +import org.opensearch.knn.common.KNNConstants; +import org.opensearch.knn.index.mapper.CompressionLevel; +import org.opensearch.knn.index.mapper.Mode; + +import java.io.IOException; + +import static org.opensearch.knn.common.KNNConstants.COMPRESSION_LEVEL_PARAMETER; +import static org.opensearch.knn.common.KNNConstants.FAISS_NAME; +import static org.opensearch.knn.common.KNNConstants.KNN_ENGINE; +import static org.opensearch.knn.common.KNNConstants.KNN_METHOD; +import static org.opensearch.knn.common.KNNConstants.METHOD_HNSW; +import static org.opensearch.knn.common.KNNConstants.METHOD_IVF; +import static org.opensearch.knn.common.KNNConstants.MODEL_DESCRIPTION; +import static org.opensearch.knn.common.KNNConstants.MODE_PARAMETER; +import static org.opensearch.knn.common.KNNConstants.NAME; +import static org.opensearch.knn.common.KNNConstants.PARAMETERS; +import static org.opensearch.knn.common.KNNConstants.TRAIN_FIELD_PARAMETER; +import static org.opensearch.knn.common.KNNConstants.TRAIN_INDEX_PARAMETER; + +public class ModeAndCompressionIT extends KNNRestTestCase { + + private static final int DIMENSION = 10; + + public void testIndexCreation() throws IOException { + XContentBuilder builder = XContentFactory.jsonBuilder() + .startObject() + .startObject("properties") + .startObject(FIELD_NAME) + .field("type", "knn_vector") + .field("dimension", DIMENSION) + .field(MODE_PARAMETER, "on_disk") + .field(COMPRESSION_LEVEL_PARAMETER, "16x") + .startObject(KNN_METHOD) + .field(NAME, METHOD_HNSW) + .field(KNN_ENGINE, FAISS_NAME) + .startObject(PARAMETERS) + .endObject() + .endObject() + .endObject() + .endObject() + .endObject(); + String mapping = builder.toString(); + createKnnIndex(INDEX_NAME + "1", mapping); + + builder = XContentFactory.jsonBuilder() + .startObject() + .startObject("properties") + .startObject(FIELD_NAME) + .field("type", "knn_vector") + .field("dimension", DIMENSION) + .field(MODE_PARAMETER, "in_memory") + .field(COMPRESSION_LEVEL_PARAMETER, "32x") + .startObject(KNN_METHOD) + .field(NAME, METHOD_HNSW) + .field(KNN_ENGINE, FAISS_NAME) + .startObject(PARAMETERS) + .endObject() + .endObject() + .endObject() + .endObject() + .endObject(); + mapping = builder.toString(); + createKnnIndex(INDEX_NAME + "2", mapping); + + builder = XContentFactory.jsonBuilder() + .startObject() + .startObject("properties") + .startObject(FIELD_NAME) + .field("type", "knn_vector") + .field("dimension", DIMENSION) + .field(MODE_PARAMETER, "invalid") + .startObject(KNN_METHOD) + .field(NAME, METHOD_HNSW) + .field(KNN_ENGINE, FAISS_NAME) + .startObject(PARAMETERS) + .endObject() + .endObject() + .endObject() + .endObject() + .endObject(); + String finalMapping = builder.toString(); + expectThrows(ResponseException.class, () -> createKnnIndex(INDEX_NAME + "3", finalMapping)); + } + + @SneakyThrows + public void testTraining() { + String trainingIndexName = "training-index"; + String trainingFieldName = "training-field"; + String modelDescription = "test model"; + int dimension = 20; + int trainingDataCount = 256; + createBasicKnnIndex(trainingIndexName, trainingFieldName, dimension); + bulkIngestRandomVectors(trainingIndexName, trainingFieldName, trainingDataCount, dimension); + + String modelId1 = "test-model-1"; + XContentBuilder xContentBuilder = XContentFactory.jsonBuilder() + .startObject() + .field(TRAIN_INDEX_PARAMETER, trainingIndexName) + .field(TRAIN_FIELD_PARAMETER, trainingFieldName) + .field(KNNConstants.DIMENSION, dimension) + .startObject(KNN_METHOD) + .field(NAME, METHOD_IVF) + .field(KNN_ENGINE, FAISS_NAME) + .endObject() + .field(MODEL_DESCRIPTION, modelDescription) + .field(COMPRESSION_LEVEL_PARAMETER, CompressionLevel.x16.getName()) + .field(MODE_PARAMETER, Mode.ON_DISK.getName()) + .endObject(); + Response trainResponse = trainModel(modelId1, xContentBuilder); + assertEquals(RestStatus.OK, RestStatus.fromCode(trainResponse.getStatusLine().getStatusCode())); + assertTrainingSucceeds(modelId1, 360, 1000); + XContentBuilder builder = XContentFactory.jsonBuilder() + .startObject() + .startObject("properties") + .startObject(FIELD_NAME) + .field("type", "knn_vector") + .field("model_id", modelId1) + .endObject() + .endObject() + .endObject(); + String mapping = builder.toString(); + createKnnIndex(INDEX_NAME + "1", mapping); + deleteKNNIndex(INDEX_NAME + "1"); + deleteModel(modelId1); + String modelId2 = "test-model-2"; + XContentBuilder xContentBuilder2 = XContentFactory.jsonBuilder() + .startObject() + .field(TRAIN_INDEX_PARAMETER, trainingIndexName) + .field(TRAIN_FIELD_PARAMETER, trainingFieldName) + .field(KNNConstants.DIMENSION, dimension) + .startObject(KNN_METHOD) + .field(NAME, METHOD_IVF) + .field(KNN_ENGINE, FAISS_NAME) + .endObject() + .field(MODEL_DESCRIPTION, modelDescription) + .field(COMPRESSION_LEVEL_PARAMETER, CompressionLevel.x16.getName()) + .field(MODE_PARAMETER, "invalid") + .endObject(); + expectThrows(ResponseException.class, () -> trainModel(modelId2, xContentBuilder2)); + } +} diff --git a/src/test/java/org/opensearch/knn/plugin/transport/GetModelResponseTests.java b/src/test/java/org/opensearch/knn/plugin/transport/GetModelResponseTests.java index 71e8f15a6..7010dbf43 100644 --- a/src/test/java/org/opensearch/knn/plugin/transport/GetModelResponseTests.java +++ b/src/test/java/org/opensearch/knn/plugin/transport/GetModelResponseTests.java @@ -18,6 +18,8 @@ import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentType; import org.opensearch.knn.KNNTestCase; +import org.opensearch.knn.index.mapper.CompressionLevel; +import org.opensearch.knn.index.mapper.Mode; import org.opensearch.knn.index.util.KNNClusterUtil; import org.opensearch.knn.index.engine.MethodComponentContext; import org.opensearch.knn.index.SpaceType; @@ -46,7 +48,9 @@ private ModelMetadata getModelMetadata(ModelState state) { "", "", MethodComponentContext.EMPTY, - VectorDataType.DEFAULT + VectorDataType.DEFAULT, + Mode.NOT_CONFIGURED, + CompressionLevel.NOT_CONFIGURED ); } diff --git a/src/test/java/org/opensearch/knn/plugin/transport/RemoveModelFromCacheTransportActionTests.java b/src/test/java/org/opensearch/knn/plugin/transport/RemoveModelFromCacheTransportActionTests.java index 8fdccdac0..6252a29ac 100644 --- a/src/test/java/org/opensearch/knn/plugin/transport/RemoveModelFromCacheTransportActionTests.java +++ b/src/test/java/org/opensearch/knn/plugin/transport/RemoveModelFromCacheTransportActionTests.java @@ -21,6 +21,8 @@ import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.engine.KNNEngine; +import org.opensearch.knn.index.mapper.CompressionLevel; +import org.opensearch.knn.index.mapper.Mode; import org.opensearch.knn.indices.Model; import org.opensearch.knn.indices.ModelCache; import org.opensearch.knn.indices.ModelDao; @@ -80,7 +82,9 @@ public void testNodeOperation_modelInCache() throws ExecutionException, Interrup "", "", MethodComponentContext.EMPTY, - VectorDataType.DEFAULT + VectorDataType.DEFAULT, + Mode.NOT_CONFIGURED, + CompressionLevel.NOT_CONFIGURED ), new byte[128], modelId diff --git a/src/test/java/org/opensearch/knn/plugin/transport/TrainingJobRouterTransportActionTests.java b/src/test/java/org/opensearch/knn/plugin/transport/TrainingJobRouterTransportActionTests.java index 8cff4dfa1..151626ef5 100644 --- a/src/test/java/org/opensearch/knn/plugin/transport/TrainingJobRouterTransportActionTests.java +++ b/src/test/java/org/opensearch/knn/plugin/transport/TrainingJobRouterTransportActionTests.java @@ -24,6 +24,8 @@ import org.opensearch.cluster.service.ClusterService; import org.opensearch.knn.KNNTestCase; import org.opensearch.knn.index.VectorDataType; +import org.opensearch.knn.index.mapper.CompressionLevel; +import org.opensearch.knn.index.mapper.Mode; import org.opensearch.search.SearchHit; import org.opensearch.search.SearchHits; import org.opensearch.transport.TransportService; @@ -308,7 +310,9 @@ public void testTrainingIndexSize() { "training-field", null, "description", - VectorDataType.DEFAULT + VectorDataType.DEFAULT, + Mode.NOT_CONFIGURED, + CompressionLevel.NOT_CONFIGURED ); // Mock client to return the right number of docs @@ -355,7 +359,9 @@ public void testTrainIndexSize_whenDataTypeIsBinary() { "training-field", null, "description", - VectorDataType.BINARY + VectorDataType.BINARY, + Mode.NOT_CONFIGURED, + CompressionLevel.NOT_CONFIGURED ); // Mock client to return the right number of docs @@ -403,7 +409,9 @@ public void testTrainIndexSize_whenDataTypeIsByte() { "training-field", null, "description", - VectorDataType.BYTE + VectorDataType.BYTE, + Mode.NOT_CONFIGURED, + CompressionLevel.NOT_CONFIGURED ); // Mock client to return the right number of docs diff --git a/src/test/java/org/opensearch/knn/plugin/transport/TrainingModelRequestTests.java b/src/test/java/org/opensearch/knn/plugin/transport/TrainingModelRequestTests.java index d7920d987..10f35457d 100644 --- a/src/test/java/org/opensearch/knn/plugin/transport/TrainingModelRequestTests.java +++ b/src/test/java/org/opensearch/knn/plugin/transport/TrainingModelRequestTests.java @@ -26,9 +26,11 @@ import org.opensearch.knn.index.engine.KNNMethodContext; import org.opensearch.knn.index.engine.MethodComponentContext; import org.opensearch.knn.index.VectorDataType; +import org.opensearch.knn.index.mapper.CompressionLevel; import org.opensearch.knn.index.mapper.KNNVectorFieldMapper; import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.engine.KNNEngine; +import org.opensearch.knn.index.mapper.Mode; import org.opensearch.knn.indices.ModelDao; import org.opensearch.knn.indices.ModelMetadata; import org.opensearch.knn.indices.ModelState; @@ -64,7 +66,9 @@ public void testStreams() throws IOException { trainingField, preferredNode, description, - VectorDataType.DEFAULT + VectorDataType.DEFAULT, + Mode.NOT_CONFIGURED, + CompressionLevel.NOT_CONFIGURED ); BytesStreamOutput streamOutput = new BytesStreamOutput(); @@ -78,6 +82,8 @@ public void testStreams() throws IOException { assertEquals(original1.getTrainingField(), copy1.getTrainingField()); assertEquals(original1.getPreferredNodeId(), copy1.getPreferredNodeId()); assertEquals(original1.getVectorDataType(), copy1.getVectorDataType()); + assertEquals(original1.getMode(), copy1.getMode()); + assertEquals(original1.getCompressionLevel(), copy1.getCompressionLevel()); // Also, check when preferred node and model id and description are null TrainingModelRequest original2 = new TrainingModelRequest( @@ -88,7 +94,9 @@ public void testStreams() throws IOException { trainingField, null, null, - VectorDataType.DEFAULT + VectorDataType.DEFAULT, + Mode.NOT_CONFIGURED, + CompressionLevel.NOT_CONFIGURED ); streamOutput = new BytesStreamOutput(); @@ -102,6 +110,33 @@ public void testStreams() throws IOException { assertEquals(original2.getTrainingField(), copy2.getTrainingField()); assertEquals(original2.getPreferredNodeId(), copy2.getPreferredNodeId()); assertEquals(original2.getVectorDataType(), copy2.getVectorDataType()); + assertEquals(original2.getMode(), copy2.getMode()); + assertEquals(original2.getCompressionLevel(), copy2.getCompressionLevel()); + + TrainingModelRequest original3 = new TrainingModelRequest( + null, + knnMethodContext, + dimension, + trainingIndex, + trainingField, + null, + null, + VectorDataType.DEFAULT, + Mode.ON_DISK, + CompressionLevel.x32 + ); + + streamOutput = new BytesStreamOutput(); + original3.writeTo(streamOutput); + TrainingModelRequest copy3 = new TrainingModelRequest(streamOutput.bytes().streamInput()); + + assertEquals(original3.getModelId(), copy3.getModelId()); + assertEquals(original3.getKnnMethodContext(), copy3.getKnnMethodContext()); + assertEquals(original3.getDimension(), copy3.getDimension()); + assertEquals(original3.getTrainingIndex(), copy3.getTrainingIndex()); + assertEquals(original3.getTrainingField(), copy3.getTrainingField()); + assertEquals(original3.getMode(), copy3.getMode()); + assertEquals(original3.getCompressionLevel(), copy3.getCompressionLevel()); } public void testGetters() { @@ -124,7 +159,9 @@ public void testGetters() { trainingField, preferredNode, description, - VectorDataType.DEFAULT + VectorDataType.DEFAULT, + Mode.NOT_CONFIGURED, + CompressionLevel.NOT_CONFIGURED ); trainingModelRequest.setMaximumVectorCount(maxVectorCount); @@ -164,7 +201,9 @@ public void testValidation_invalid_modelIdAlreadyExists() { trainingField, null, null, - VectorDataType.DEFAULT + VectorDataType.DEFAULT, + Mode.NOT_CONFIGURED, + CompressionLevel.NOT_CONFIGURED ); // Mock the model dao to return metadata for modelId to recognize it is a duplicate @@ -179,7 +218,9 @@ public void testValidation_invalid_modelIdAlreadyExists() { "", "", MethodComponentContext.EMPTY, - VectorDataType.DEFAULT + VectorDataType.DEFAULT, + Mode.NOT_CONFIGURED, + CompressionLevel.NOT_CONFIGURED ); when(modelDao.getMetadata(modelId)).thenReturn(modelMetadata); @@ -221,7 +262,9 @@ public void testValidation_blocked_modelId() { trainingField, null, null, - VectorDataType.DEFAULT + VectorDataType.DEFAULT, + Mode.NOT_CONFIGURED, + CompressionLevel.NOT_CONFIGURED ); // Mock the model dao to return true to recognize that the modelId is in graveyard @@ -269,7 +312,9 @@ public void testValidation_invalid_invalidMethodContext() { trainingField, null, null, - VectorDataType.DEFAULT + VectorDataType.DEFAULT, + Mode.NOT_CONFIGURED, + CompressionLevel.NOT_CONFIGURED ); // Mock the model dao to return null so that no exception is produced @@ -313,7 +358,9 @@ public void testValidation_invalid_trainingIndexDoesNotExist() { trainingField, null, null, - VectorDataType.DEFAULT + VectorDataType.DEFAULT, + Mode.NOT_CONFIGURED, + CompressionLevel.NOT_CONFIGURED ); // Mock the model dao to return null so that no exception is produced @@ -360,7 +407,9 @@ public void testValidation_invalid_trainingFieldDoesNotExist() { trainingField, null, null, - VectorDataType.DEFAULT + VectorDataType.DEFAULT, + Mode.NOT_CONFIGURED, + CompressionLevel.NOT_CONFIGURED ); // Mock the model dao to return null so that no exception is produced @@ -412,7 +461,9 @@ public void testValidation_invalid_trainingFieldNotKnnVector() { trainingField, null, null, - VectorDataType.DEFAULT + VectorDataType.DEFAULT, + Mode.NOT_CONFIGURED, + CompressionLevel.NOT_CONFIGURED ); // Mock the model dao to return null so that no exception is produced @@ -469,7 +520,9 @@ public void testValidation_invalid_dimensionDoesNotMatch() { trainingField, null, null, - VectorDataType.DEFAULT + VectorDataType.DEFAULT, + Mode.NOT_CONFIGURED, + CompressionLevel.NOT_CONFIGURED ); // Mock the model dao to return null so that no exception is produced @@ -528,7 +581,9 @@ public void testValidation_invalid_preferredNodeDoesNotExist() { trainingField, preferredNode, null, - VectorDataType.DEFAULT + VectorDataType.DEFAULT, + Mode.NOT_CONFIGURED, + CompressionLevel.NOT_CONFIGURED ); // Mock the model dao to return metadata for modelId to recognize it is a duplicate @@ -595,7 +650,9 @@ public void testValidation_invalid_descriptionToLong() { trainingField, null, description, - VectorDataType.DEFAULT + VectorDataType.DEFAULT, + Mode.NOT_CONFIGURED, + CompressionLevel.NOT_CONFIGURED ); // Mock the model dao to return metadata for modelId to recognize it is a duplicate @@ -641,7 +698,9 @@ public void testValidation_valid_trainingIndexBuiltFromMethod() { trainingField, null, null, - VectorDataType.DEFAULT + VectorDataType.DEFAULT, + Mode.NOT_CONFIGURED, + CompressionLevel.NOT_CONFIGURED ); // Mock the model dao to return metadata for modelId to recognize it is a duplicate @@ -680,7 +739,9 @@ public void testValidation_valid_trainingIndexBuiltFromModel() { trainingField, null, null, - VectorDataType.DEFAULT + VectorDataType.DEFAULT, + Mode.NOT_CONFIGURED, + CompressionLevel.NOT_CONFIGURED ); // Mock the model dao to return metadata for modelId to recognize it is a duplicate diff --git a/src/test/java/org/opensearch/knn/plugin/transport/TrainingModelTransportActionTests.java b/src/test/java/org/opensearch/knn/plugin/transport/TrainingModelTransportActionTests.java index aea0e0b16..345f4feb5 100644 --- a/src/test/java/org/opensearch/knn/plugin/transport/TrainingModelTransportActionTests.java +++ b/src/test/java/org/opensearch/knn/plugin/transport/TrainingModelTransportActionTests.java @@ -19,6 +19,8 @@ import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.engine.KNNEngine; +import org.opensearch.knn.index.mapper.CompressionLevel; +import org.opensearch.knn.index.mapper.Mode; import org.opensearch.knn.indices.ModelDao; import java.io.IOException; @@ -74,7 +76,9 @@ public void testDoExecute() throws InterruptedException, ExecutionException, IOE trainingFieldName, null, "test-detector", - VectorDataType.DEFAULT + VectorDataType.DEFAULT, + Mode.NOT_CONFIGURED, + CompressionLevel.NOT_CONFIGURED ); trainingModelRequest.setTrainingDataSizeInKB(estimateVectorSetSizeInKB(trainingDataCount, dimension, VectorDataType.DEFAULT)); diff --git a/src/test/java/org/opensearch/knn/plugin/transport/UpdateModelGraveyardTransportActionTests.java b/src/test/java/org/opensearch/knn/plugin/transport/UpdateModelGraveyardTransportActionTests.java index bc6e098f3..45203dae6 100644 --- a/src/test/java/org/opensearch/knn/plugin/transport/UpdateModelGraveyardTransportActionTests.java +++ b/src/test/java/org/opensearch/knn/plugin/transport/UpdateModelGraveyardTransportActionTests.java @@ -19,6 +19,8 @@ import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.engine.KNNEngine; +import org.opensearch.knn.index.mapper.CompressionLevel; +import org.opensearch.knn.index.mapper.Mode; import org.opensearch.knn.indices.Model; import org.opensearch.knn.indices.ModelGraveyard; import org.opensearch.knn.indices.ModelMetadata; @@ -212,7 +214,9 @@ public void testClusterManagerOperation_GetIndicesUsingModel() throws IOExceptio "", "", MethodComponentContext.EMPTY, - VectorDataType.DEFAULT + VectorDataType.DEFAULT, + Mode.NOT_CONFIGURED, + CompressionLevel.NOT_CONFIGURED ), modelBlob, modelId diff --git a/src/test/java/org/opensearch/knn/plugin/transport/UpdateModelMetadataRequestTests.java b/src/test/java/org/opensearch/knn/plugin/transport/UpdateModelMetadataRequestTests.java index 238fc5e45..d0a83ccc5 100644 --- a/src/test/java/org/opensearch/knn/plugin/transport/UpdateModelMetadataRequestTests.java +++ b/src/test/java/org/opensearch/knn/plugin/transport/UpdateModelMetadataRequestTests.java @@ -17,6 +17,8 @@ import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.engine.KNNEngine; +import org.opensearch.knn.index.mapper.CompressionLevel; +import org.opensearch.knn.index.mapper.Mode; import org.opensearch.knn.indices.ModelMetadata; import org.opensearch.knn.indices.ModelState; @@ -44,7 +46,9 @@ public void testStreams() throws IOException { "", "", MethodComponentContext.EMPTY, - VectorDataType.DEFAULT + VectorDataType.DEFAULT, + Mode.NOT_CONFIGURED, + CompressionLevel.NOT_CONFIGURED ); UpdateModelMetadataRequest updateModelMetadataRequest = new UpdateModelMetadataRequest(modelId, isRemoveRequest, modelMetadata); @@ -70,7 +74,9 @@ public void testValidate() { "", "", MethodComponentContext.EMPTY, - VectorDataType.DEFAULT + VectorDataType.DEFAULT, + Mode.NOT_CONFIGURED, + CompressionLevel.NOT_CONFIGURED ); UpdateModelMetadataRequest updateModelMetadataRequest1 = new UpdateModelMetadataRequest("test", true, null); @@ -111,7 +117,9 @@ public void testGetModelMetadata() { "", "", MethodComponentContext.EMPTY, - VectorDataType.DEFAULT + VectorDataType.DEFAULT, + Mode.NOT_CONFIGURED, + CompressionLevel.NOT_CONFIGURED ); UpdateModelMetadataRequest updateModelMetadataRequest = new UpdateModelMetadataRequest("test", true, modelMetadata); diff --git a/src/test/java/org/opensearch/knn/plugin/transport/UpdateModelMetadataTransportActionTests.java b/src/test/java/org/opensearch/knn/plugin/transport/UpdateModelMetadataTransportActionTests.java index e5dcb2257..d317fa893 100644 --- a/src/test/java/org/opensearch/knn/plugin/transport/UpdateModelMetadataTransportActionTests.java +++ b/src/test/java/org/opensearch/knn/plugin/transport/UpdateModelMetadataTransportActionTests.java @@ -21,6 +21,8 @@ import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.engine.KNNEngine; +import org.opensearch.knn.index.mapper.CompressionLevel; +import org.opensearch.knn.index.mapper.Mode; import org.opensearch.knn.indices.ModelMetadata; import org.opensearch.knn.indices.ModelState; import org.opensearch.threadpool.ThreadPool; @@ -70,7 +72,9 @@ public void testClusterManagerOperation() throws InterruptedException { "", "", MethodComponentContext.EMPTY, - VectorDataType.DEFAULT + VectorDataType.DEFAULT, + Mode.NOT_CONFIGURED, + CompressionLevel.NOT_CONFIGURED ); // Get update transport action diff --git a/src/test/java/org/opensearch/knn/training/TrainingJobTests.java b/src/test/java/org/opensearch/knn/training/TrainingJobTests.java index adecca43a..32794a33b 100644 --- a/src/test/java/org/opensearch/knn/training/TrainingJobTests.java +++ b/src/test/java/org/opensearch/knn/training/TrainingJobTests.java @@ -20,6 +20,8 @@ import org.opensearch.knn.index.engine.MethodComponentContext; import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.VectorDataType; +import org.opensearch.knn.index.mapper.CompressionLevel; +import org.opensearch.knn.index.mapper.Mode; import org.opensearch.knn.index.memory.NativeMemoryAllocation; import org.opensearch.knn.index.memory.NativeMemoryCacheManager; import org.opensearch.knn.index.memory.NativeMemoryEntryContext; @@ -69,7 +71,9 @@ public void testGetModelId() { mock(NativeMemoryEntryContext.AnonymousEntryContext.class), KNNMethodConfigContext.builder().vectorDataType(VectorDataType.DEFAULT).dimension(10).versionCreated(Version.CURRENT).build(), "", - "test-node" + "test-node", + Mode.NOT_CONFIGURED, + CompressionLevel.NOT_CONFIGURED ); assertEquals(modelId, trainingJob.getModelId()); @@ -102,7 +106,9 @@ public void testGetModel() { .versionCreated(Version.CURRENT) .build(), description, - nodeAssignment + nodeAssignment, + Mode.NOT_CONFIGURED, + CompressionLevel.NOT_CONFIGURED ); Model model = new Model( @@ -116,7 +122,9 @@ public void testGetModel() { error, nodeAssignment, MethodComponentContext.EMPTY, - VectorDataType.DEFAULT + VectorDataType.DEFAULT, + Mode.NOT_CONFIGURED, + CompressionLevel.NOT_CONFIGURED ), null, modelID @@ -195,7 +203,9 @@ public void testRun_success() throws IOException, ExecutionException { modelContext, knnMethodConfigContext, "", - "test-node" + "test-node", + Mode.NOT_CONFIGURED, + CompressionLevel.NOT_CONFIGURED ); trainingJob.run(); @@ -278,7 +288,9 @@ public void testRun_failure_onGetTrainingDataAllocation() throws ExecutionExcept modelContext, knnMethodConfigContext, "", - "test-node" + "test-node", + Mode.NOT_CONFIGURED, + CompressionLevel.NOT_CONFIGURED ); trainingJob.run(); @@ -350,7 +362,9 @@ public void testRun_failure_onGetModelAnonymousAllocation() throws ExecutionExce modelContext, knnMethodConfigContext, "", - "test-node" + "test-node", + Mode.NOT_CONFIGURED, + CompressionLevel.NOT_CONFIGURED ); trainingJob.run(); @@ -421,7 +435,9 @@ public void testRun_failure_closedTrainingDataAllocation() throws ExecutionExcep mock(NativeMemoryEntryContext.AnonymousEntryContext.class), knnMethodConfigContext, "", - "test-node" + "test-node", + Mode.NOT_CONFIGURED, + CompressionLevel.NOT_CONFIGURED ); trainingJob.run(); @@ -499,7 +515,9 @@ public void testRun_failure_notEnoughTrainingData() throws ExecutionException { modelContext, knnMethodConfigContext, "", - "test-node" + "test-node", + Mode.NOT_CONFIGURED, + CompressionLevel.NOT_CONFIGURED ); trainingJob.run(); From 5858d1c9d472467184524b80530019a9a839dc07 Mon Sep 17 00:00:00 2001 From: Kunal Kotwani Date: Wed, 4 Sep 2024 11:26:27 -0700 Subject: [PATCH 349/416] Fix memory overflow caused by cache behavior (#2015) (#2032) Signed-off-by: Kunal Kotwani (cherry picked from commit d4af93edfb45878fdb0a442f4ac07e091b5ad14b) --- CHANGELOG.md | 3 +- .../common/featureflags/KNNFeatureFlags.java | 49 ++++++++++ .../org/opensearch/knn/index/KNNSettings.java | 15 ++- .../index/memory/NativeMemoryAllocation.java | 13 ++- .../memory/NativeMemoryCacheManager.java | 54 ++++++++++- .../featureflags/KNNFeatureFlagsTests.java | 34 +++++++ .../memory/NativeMemoryAllocationTests.java | 94 ++++++++++++++++++- 7 files changed, 252 insertions(+), 10 deletions(-) create mode 100644 src/main/java/org/opensearch/knn/common/featureflags/KNNFeatureFlags.java create mode 100644 src/test/java/org/opensearch/knn/common/featureflags/KNNFeatureFlagsTests.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 58058dbd1..060ac75de 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,6 +29,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), * Fix graph merge stats size calculation [#1844](https://github.com/opensearch-project/k-NN/pull/1844) * Disallow a vector field to have an invalid character for a physical file name. [#1936](https://github.com/opensearch-project/k-NN/pull/1936) * Add script_fields context to KNNAllowlist [#1917] (https://github.com/opensearch-project/k-NN/pull/1917) +* Fix memory overflow caused by cache behavior [#2015](https://github.com/opensearch-project/k-NN/pull/2015) ### Infrastructure * Parallelize make to reduce build time [#2006] (https://github.com/opensearch-project/k-NN/pull/2006) ### Documentation @@ -45,4 +46,4 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), * Restructure mappers to better handle null cases and avoid branching in parsing [#1939](https://github.com/opensearch-project/k-NN/pull/1939) * Added Quantization Framework and implemented 1Bit and multibit quantizer[#1889](https://github.com/opensearch-project/k-NN/issues/1889) * Encapsulate dimension, vector data type validation/processing inside Library [#1957](https://github.com/opensearch-project/k-NN/pull/1957) -* Add quantization state cache [#1960](https://github.com/opensearch-project/k-NN/pull/1960) \ No newline at end of file +* Add quantization state cache [#1960](https://github.com/opensearch-project/k-NN/pull/1960) diff --git a/src/main/java/org/opensearch/knn/common/featureflags/KNNFeatureFlags.java b/src/main/java/org/opensearch/knn/common/featureflags/KNNFeatureFlags.java new file mode 100644 index 000000000..bab5b97bb --- /dev/null +++ b/src/main/java/org/opensearch/knn/common/featureflags/KNNFeatureFlags.java @@ -0,0 +1,49 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.opensearch.knn.common.featureflags; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.ImmutableList; +import lombok.experimental.UtilityClass; +import org.opensearch.common.Booleans; +import org.opensearch.common.settings.Setting; +import org.opensearch.knn.index.KNNSettings; + +import java.util.List; + +import static org.opensearch.common.settings.Setting.Property.Dynamic; +import static org.opensearch.common.settings.Setting.Property.NodeScope; + +/** + * Class to manage KNN feature flags + */ +@UtilityClass +public class KNNFeatureFlags { + + // Feature flags + private static final String KNN_FORCE_EVICT_CACHE_ENABLED = "knn.feature.cache.force_evict.enabled"; + + @VisibleForTesting + public static final Setting KNN_FORCE_EVICT_CACHE_ENABLED_SETTING = Setting.boolSetting( + KNN_FORCE_EVICT_CACHE_ENABLED, + false, + NodeScope, + Dynamic + ); + + public static List> getFeatureFlags() { + return ImmutableList.of(KNN_FORCE_EVICT_CACHE_ENABLED_SETTING); + } + + /** + * Checks if force evict for cache is enabled by executing a check against cluster settings + * @return true if force evict setting is set to true + */ + public static boolean isForceEvictCacheEnabled() { + return Booleans.parseBoolean(KNNSettings.state().getSettingValue(KNN_FORCE_EVICT_CACHE_ENABLED).toString(), false); + } +} diff --git a/src/main/java/org/opensearch/knn/index/KNNSettings.java b/src/main/java/org/opensearch/knn/index/KNNSettings.java index e0123ef8d..a70a17d85 100644 --- a/src/main/java/org/opensearch/knn/index/KNNSettings.java +++ b/src/main/java/org/opensearch/knn/index/KNNSettings.java @@ -34,14 +34,17 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.Stream; +import static java.util.stream.Collectors.toUnmodifiableMap; import static org.opensearch.common.settings.Setting.Property.Dynamic; import static org.opensearch.common.settings.Setting.Property.IndexScope; import static org.opensearch.common.settings.Setting.Property.NodeScope; import static org.opensearch.common.unit.MemorySizeValue.parseBytesSizeValueOrHeapRatio; import static org.opensearch.core.common.unit.ByteSizeValue.parseBytesSizeValue; +import static org.opensearch.knn.common.featureflags.KNNFeatureFlags.getFeatureFlags; /** * This class defines @@ -334,6 +337,9 @@ public class KNNSettings { } }; + private final static Map> FEATURE_FLAGS = getFeatureFlags().stream() + .collect(toUnmodifiableMap(Setting::getKey, Function.identity())); + private ClusterService clusterService; private Client client; @@ -371,7 +377,7 @@ private void setSettingsUpdateConsumers() { ); NativeMemoryCacheManager.getInstance().rebuildCache(builder.build()); - }, dynamicCacheSettings.values().stream().collect(Collectors.toUnmodifiableList())); + }, Stream.concat(dynamicCacheSettings.values().stream(), FEATURE_FLAGS.values().stream()).collect(Collectors.toUnmodifiableList())); clusterService.getClusterSettings().addSettingsUpdateConsumer(QUANTIZATION_STATE_CACHE_SIZE_LIMIT_SETTING, it -> { QuantizationStateCache.getInstance().setMaxCacheSizeInKB(it.getKb()); QuantizationStateCache.getInstance().rebuildCache(); @@ -398,6 +404,10 @@ private Setting getSetting(String key) { return dynamicCacheSettings.get(key); } + if (FEATURE_FLAGS.containsKey(key)) { + return FEATURE_FLAGS.get(key); + } + if (KNN_CIRCUIT_BREAKER_TRIGGERED.equals(key)) { return KNN_CIRCUIT_BREAKER_TRIGGERED_SETTING; } @@ -452,7 +462,8 @@ public List> getSettings() { QUANTIZATION_STATE_CACHE_SIZE_LIMIT_SETTING, QUANTIZATION_STATE_CACHE_EXPIRY_TIME_MINUTES_SETTING ); - return Stream.concat(settings.stream(), dynamicCacheSettings.values().stream()).collect(Collectors.toList()); + return Stream.concat(settings.stream(), Stream.concat(getFeatureFlags().stream(), dynamicCacheSettings.values().stream())) + .collect(Collectors.toList()); } public static boolean isKNNPluginEnabled() { diff --git a/src/main/java/org/opensearch/knn/index/memory/NativeMemoryAllocation.java b/src/main/java/org/opensearch/knn/index/memory/NativeMemoryAllocation.java index c711f3342..02b480ed4 100644 --- a/src/main/java/org/opensearch/knn/index/memory/NativeMemoryAllocation.java +++ b/src/main/java/org/opensearch/knn/index/memory/NativeMemoryAllocation.java @@ -13,6 +13,7 @@ import lombok.Getter; import org.apache.lucene.index.LeafReaderContext; +import org.opensearch.knn.common.featureflags.KNNFeatureFlags; import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.query.KNNWeight; import org.opensearch.knn.jni.JNIService; @@ -161,11 +162,19 @@ class IndexAllocation implements NativeMemoryAllocation { @Override public void close() { - executor.execute(() -> { + Runnable onClose = () -> { writeLock(); cleanup(); writeUnlock(); - }); + }; + + // The close operation needs to be blocking to prevent overflow + // This blocks any entry until the close has completed, preventing creation before close scenarios + if (KNNFeatureFlags.isForceEvictCacheEnabled()) { + onClose.run(); + } else { + executor.execute(onClose); + } } private void cleanup() { diff --git a/src/main/java/org/opensearch/knn/index/memory/NativeMemoryCacheManager.java b/src/main/java/org/opensearch/knn/index/memory/NativeMemoryCacheManager.java index 9478e1e00..649fb9774 100644 --- a/src/main/java/org/opensearch/knn/index/memory/NativeMemoryCacheManager.java +++ b/src/main/java/org/opensearch/knn/index/memory/NativeMemoryCacheManager.java @@ -21,13 +21,17 @@ import org.apache.logging.log4j.Logger; import org.opensearch.common.unit.TimeValue; import org.opensearch.knn.common.exception.OutOfNativeMemoryException; +import org.opensearch.knn.common.featureflags.KNNFeatureFlags; import org.opensearch.knn.index.KNNSettings; import org.opensearch.knn.plugin.stats.StatNames; import java.io.Closeable; +import java.util.Deque; import java.util.HashMap; +import java.util.Iterator; import java.util.Map; import java.util.Optional; +import java.util.concurrent.ConcurrentLinkedDeque; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -45,6 +49,7 @@ public class NativeMemoryCacheManager implements Closeable { private static NativeMemoryCacheManager INSTANCE; private Cache cache; + private Deque accessRecencyQueue; private final ExecutorService executor; private AtomicBoolean cacheCapacityReached; private long maxWeight; @@ -97,7 +102,7 @@ private void initialize(NativeMemoryCacheManagerDto nativeMemoryCacheDTO) { } cacheCapacityReached = new AtomicBoolean(false); - + accessRecencyQueue = new ConcurrentLinkedDeque<>(); cache = cacheBuilder.build(); } @@ -301,7 +306,52 @@ public NativeMemoryAllocation get(NativeMemoryEntryContext nativeMemoryEntryC ); } - return cache.get(nativeMemoryEntryContext.getKey(), nativeMemoryEntryContext::load); + if (KNNFeatureFlags.isForceEvictCacheEnabled()) { + // Utilizes a force eviction mechanism to free up memory before the entry can be added to the cache + // In case of a cache hit, the operation just updates the locally maintained recency list + // In case of a cache miss, least recently accessed entries are evicted in a blocking manner + // before the new entry can be added to the cache. + String key = nativeMemoryEntryContext.getKey(); + NativeMemoryAllocation result = cache.getIfPresent(key); + + // Cache Hit + // In case of a cache hit, moving the item to the end of the recency queue adds + // some overhead to the get operation. This can be optimized further to make this operation + // as lightweight as possible. Multiple approaches and their outcomes were documented + // before moving forward with the current solution. + // The details are outlined here: https://github.com/opensearch-project/k-NN/pull/2015#issuecomment-2327064680 + if (result != null) { + accessRecencyQueue.remove(key); + accessRecencyQueue.addLast(key); + return result; + } + + // Cache Miss + // Evict before put + synchronized (this) { + if (getCacheSizeInKilobytes() + nativeMemoryEntryContext.calculateSizeInKB() >= maxWeight) { + Iterator lruIterator = accessRecencyQueue.iterator(); + while (lruIterator.hasNext() + && (getCacheSizeInKilobytes() + nativeMemoryEntryContext.calculateSizeInKB() >= maxWeight)) { + + String keyToRemove = lruIterator.next(); + NativeMemoryAllocation allocationToRemove = cache.getIfPresent(keyToRemove); + if (allocationToRemove != null) { + allocationToRemove.close(); + cache.invalidate(keyToRemove); + } + lruIterator.remove(); + } + } + + result = cache.get(key, nativeMemoryEntryContext::load); + accessRecencyQueue.addLast(key); + + return result; + } + } else { + return cache.get(nativeMemoryEntryContext.getKey(), nativeMemoryEntryContext::load); + } } /** diff --git a/src/test/java/org/opensearch/knn/common/featureflags/KNNFeatureFlagsTests.java b/src/test/java/org/opensearch/knn/common/featureflags/KNNFeatureFlagsTests.java new file mode 100644 index 000000000..f2f74944e --- /dev/null +++ b/src/test/java/org/opensearch/knn/common/featureflags/KNNFeatureFlagsTests.java @@ -0,0 +1,34 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.common.featureflags; + +import org.mockito.Mock; +import org.opensearch.common.settings.ClusterSettings; +import org.opensearch.knn.KNNTestCase; +import org.opensearch.knn.index.KNNSettings; + +import static org.mockito.Mockito.when; +import static org.opensearch.knn.common.featureflags.KNNFeatureFlags.KNN_FORCE_EVICT_CACHE_ENABLED_SETTING; +import static org.opensearch.knn.common.featureflags.KNNFeatureFlags.isForceEvictCacheEnabled; + +public class KNNFeatureFlagsTests extends KNNTestCase { + + @Mock + ClusterSettings clusterSettings; + + public void setUp() throws Exception { + super.setUp(); + when(clusterService.getClusterSettings()).thenReturn(clusterSettings); + KNNSettings.state().setClusterService(clusterService); + } + + public void testIsForceEvictCacheEnabled() throws Exception { + when(clusterSettings.get(KNN_FORCE_EVICT_CACHE_ENABLED_SETTING)).thenReturn(false); + assertFalse(isForceEvictCacheEnabled()); + when(clusterSettings.get(KNN_FORCE_EVICT_CACHE_ENABLED_SETTING)).thenReturn(true); + assertTrue(isForceEvictCacheEnabled()); + } +} diff --git a/src/test/java/org/opensearch/knn/index/memory/NativeMemoryAllocationTests.java b/src/test/java/org/opensearch/knn/index/memory/NativeMemoryAllocationTests.java index 1e2134581..cb5fbaeba 100644 --- a/src/test/java/org/opensearch/knn/index/memory/NativeMemoryAllocationTests.java +++ b/src/test/java/org/opensearch/knn/index/memory/NativeMemoryAllocationTests.java @@ -13,26 +13,36 @@ import com.google.common.collect.ImmutableMap; import lombok.SneakyThrows; +import org.junit.Before; +import org.mockito.Mock; +import org.opensearch.common.settings.ClusterSettings; import org.opensearch.knn.KNNTestCase; import org.opensearch.knn.TestUtils; import org.opensearch.knn.common.KNNConstants; -import org.opensearch.knn.index.util.IndexUtil; +import org.opensearch.knn.index.KNNSettings; +import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.VectorDataType; +import org.opensearch.knn.index.engine.KNNEngine; +import org.opensearch.knn.index.util.IndexUtil; import org.opensearch.knn.jni.JNICommons; import org.opensearch.knn.jni.JNIService; -import org.opensearch.knn.index.SpaceType; -import org.opensearch.knn.index.engine.KNNEngine; import org.opensearch.watcher.FileWatcher; import org.opensearch.watcher.WatcherHandle; import java.nio.file.Path; import java.util.Arrays; import java.util.Map; +import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; import static org.mockito.Mockito.doNothing; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import static org.opensearch.knn.common.featureflags.KNNFeatureFlags.KNN_FORCE_EVICT_CACHE_ENABLED_SETTING; public class NativeMemoryAllocationTests extends KNNTestCase { @@ -41,6 +51,19 @@ public class NativeMemoryAllocationTests extends KNNTestCase { private int testLockValue3; private int testLockValue4; + @Mock + ClusterSettings clusterSettings; + + @Before + @Override + public void setUp() throws Exception { + super.setUp(); + clusterSettings = mock(ClusterSettings.class); + when(clusterService.getClusterSettings()).thenReturn(clusterSettings); + when(clusterSettings.get(KNN_FORCE_EVICT_CACHE_ENABLED_SETTING)).thenReturn(false); + KNNSettings.state().setClusterService(clusterService); + } + public void testIndexAllocation_close() throws InterruptedException { // Create basic nmslib HNSW index Path dir = createTempDir(); @@ -207,6 +230,71 @@ public void testIndexAllocation_readLock() throws InterruptedException { assertEquals(finalValue, testLockValue1); } + public void testIndexAllocation_closeDefault() { + WatcherHandle watcherHandle = (WatcherHandle) mock(WatcherHandle.class); + ExecutorService executorService = Executors.newFixedThreadPool(2); + AtomicReference expectedException = new AtomicReference<>(); + + // Executor based non-blocking close + NativeMemoryAllocation.IndexAllocation nonBlockingIndexAllocation = new NativeMemoryAllocation.IndexAllocation( + mock(ExecutorService.class), + 0, + 0, + null, + "test", + "test", + watcherHandle + ); + + executorService.submit(nonBlockingIndexAllocation::readLock); + Future closingThread = executorService.submit(nonBlockingIndexAllocation::close); + try { + closingThread.get(); + } catch (Exception ex) { + expectedException.set(ex); + } + assertNull(expectedException.get()); + expectedException.set(null); + executorService.shutdown(); + } + + public void testIndexAllocation_closeBlocking() throws InterruptedException, ExecutionException { + WatcherHandle watcherHandle = (WatcherHandle) mock(WatcherHandle.class); + ExecutorService executorService = Executors.newFixedThreadPool(2); + AtomicReference expectedException = new AtomicReference<>(); + + // Blocking close + when(clusterSettings.get(KNN_FORCE_EVICT_CACHE_ENABLED_SETTING)).thenReturn(true); + NativeMemoryAllocation.IndexAllocation blockingIndexAllocation = new NativeMemoryAllocation.IndexAllocation( + mock(ExecutorService.class), + 0, + 0, + null, + "test", + "test", + watcherHandle + ); + + executorService.submit(blockingIndexAllocation::readLock); + Future closingThread = executorService.submit(blockingIndexAllocation::close); + + // Check if thread is currently blocked + try { + closingThread.get(5, TimeUnit.SECONDS); + } catch (Exception e) { + expectedException.set(e); + } + + assertNotNull(expectedException.get()); + + executorService.submit(blockingIndexAllocation::readUnlock); + closingThread.get(); + + // Waits until close + assertTrue(blockingIndexAllocation.isClosed()); + executorService.shutdown(); + } + public void testIndexAllocation_writeLock() throws InterruptedException { // To test the writeLock, we first grab the writeLock in the main thread. Then we start another thread that // grabs the readLock and asserts testLockValue2 has been updated. Next in the main thread, we update the value From 4417428ebf92b1d941cdf5edeed12bcfadd7749a Mon Sep 17 00:00:00 2001 From: Naveen Tatikonda Date: Wed, 4 Sep 2024 20:17:53 -0500 Subject: [PATCH 350/416] Add release notes for 2.17 (#2039) Signed-off-by: Naveen Tatikonda --- CHANGELOG.md | 28 +--------------- .../opensearch-knn.release-notes-2.17.0.0.md | 33 +++++++++++++++++++ 2 files changed, 34 insertions(+), 27 deletions(-) create mode 100644 release-notes/opensearch-knn.release-notes-2.17.0.0.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 060ac75de..2954d898c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,37 +13,11 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Maintenance ### Refactoring -## [Unreleased 2.x](https://github.com/opensearch-project/k-NN/compare/2.16...2.x) +## [Unreleased 2.x](https://github.com/opensearch-project/k-NN/compare/2.17...2.x) ### Features -* Integrate Lucene Vector field with native engines to use KNNVectorFormat during segment creation [#1945](https://github.com/opensearch-project/k-NN/pull/1945) -* k-NN query rescore support for native engines [#1984](https://github.com/opensearch-project/k-NN/pull/1984) -* Add support for byte vector with Faiss Engine HNSW algorithm [#1823](https://github.com/opensearch-project/k-NN/pull/1823) -* Add support for byte vector with Faiss Engine IVF algorithm [#2002](https://github.com/opensearch-project/k-NN/pull/2002) ### Enhancements -* Adds iterative graph build capability into a faiss index to improve the memory footprint during indexing and Integrates KNNVectorsFormat for native engines[#1950](https://github.com/opensearch-project/k-NN/pull/1950) ### Bug Fixes -* Corrected search logic for scenario with non-existent fields in filter [#1874](https://github.com/opensearch-project/k-NN/pull/1874) -* Fix graph merge stats size calculation [#1844](https://github.com/opensearch-project/k-NN/pull/1844) -* Integrate Lucene Vector field with native engines to use KNNVectorFormat during segment creation [#1945](https://github.com/opensearch-project/k-NN/pull/1945) -* Add script_fields context to KNNAllowlist [#1917] (https://github.com/opensearch-project/k-NN/pull/1917) -* Fix graph merge stats size calculation [#1844](https://github.com/opensearch-project/k-NN/pull/1844) -* Disallow a vector field to have an invalid character for a physical file name. [#1936](https://github.com/opensearch-project/k-NN/pull/1936) -* Add script_fields context to KNNAllowlist [#1917] (https://github.com/opensearch-project/k-NN/pull/1917) -* Fix memory overflow caused by cache behavior [#2015](https://github.com/opensearch-project/k-NN/pull/2015) ### Infrastructure -* Parallelize make to reduce build time [#2006] (https://github.com/opensearch-project/k-NN/pull/2006) ### Documentation ### Maintenance -* Fix a flaky unit test:testMultiFieldsKnnIndex, which was failing due to inconsistent merge behaviors [#1924](https://github.com/opensearch-project/k-NN/pull/1924) ### Refactoring -* Introduce KNNVectorValues interface to iterate on different types of Vector values during indexing and search [#1897](https://github.com/opensearch-project/k-NN/pull/1897) -* Integrate KNNVectorValues with vector ANN Search flow [#1952](https://github.com/opensearch-project/k-NN/pull/1952) -* Clean up parsing for query [#1824](https://github.com/opensearch-project/k-NN/pull/1824) -* Refactor engine package structure [#1913](https://github.com/opensearch-project/k-NN/pull/1913) -* Refactor method structure and definitions [#1920](https://github.com/opensearch-project/k-NN/pull/1920) -* Refactor KNNVectorFieldType from KNNVectorFieldMapper to a separate class for better readability. [#1931](https://github.com/opensearch-project/k-NN/pull/1931) -* Generalize lib interface to return context objects [#1925](https://github.com/opensearch-project/k-NN/pull/1925) -* Restructure mappers to better handle null cases and avoid branching in parsing [#1939](https://github.com/opensearch-project/k-NN/pull/1939) -* Added Quantization Framework and implemented 1Bit and multibit quantizer[#1889](https://github.com/opensearch-project/k-NN/issues/1889) -* Encapsulate dimension, vector data type validation/processing inside Library [#1957](https://github.com/opensearch-project/k-NN/pull/1957) -* Add quantization state cache [#1960](https://github.com/opensearch-project/k-NN/pull/1960) diff --git a/release-notes/opensearch-knn.release-notes-2.17.0.0.md b/release-notes/opensearch-knn.release-notes-2.17.0.0.md new file mode 100644 index 000000000..e042bc4e1 --- /dev/null +++ b/release-notes/opensearch-knn.release-notes-2.17.0.0.md @@ -0,0 +1,33 @@ +## Version 2.17.0.0 Release Notes + +Compatible with OpenSearch 2.17.0 + +### Features +* Integrate Lucene Vector field with native engines to use KNNVectorFormat during segment creation [#1945](https://github.com/opensearch-project/k-NN/pull/1945) +* k-NN query rescore support for native engines [#1984](https://github.com/opensearch-project/k-NN/pull/1984) +* Add support for byte vector with Faiss Engine HNSW algorithm [#1823](https://github.com/opensearch-project/k-NN/pull/1823) +* Add support for byte vector with Faiss Engine IVF algorithm [#2002](https://github.com/opensearch-project/k-NN/pull/2002) +### Enhancements +* Adds iterative graph build capability into a faiss index to improve the memory footprint during indexing and Integrates KNNVectorsFormat for native engines[#1950](https://github.com/opensearch-project/k-NN/pull/1950) +### Bug Fixes +* Corrected search logic for scenario with non-existent fields in filter [#1874](https://github.com/opensearch-project/k-NN/pull/1874) +* Add script_fields context to KNNAllowlist [#1917] (https://github.com/opensearch-project/k-NN/pull/1917) +* Fix graph merge stats size calculation [#1844](https://github.com/opensearch-project/k-NN/pull/1844) +* Disallow a vector field to have an invalid character for a physical file name. [#1936](https://github.com/opensearch-project/k-NN/pull/1936) +* Fix memory overflow caused by cache behavior [#2015](https://github.com/opensearch-project/k-NN/pull/2015) +### Infrastructure +* Parallelize make to reduce build time [#2006] (https://github.com/opensearch-project/k-NN/pull/2006) +### Maintenance +* Fix a flaky unit test:testMultiFieldsKnnIndex, which was failing due to inconsistent merge behaviors [#1924](https://github.com/opensearch-project/k-NN/pull/1924) +### Refactoring +* Introduce KNNVectorValues interface to iterate on different types of Vector values during indexing and search [#1897](https://github.com/opensearch-project/k-NN/pull/1897) +* Integrate KNNVectorValues with vector ANN Search flow [#1952](https://github.com/opensearch-project/k-NN/pull/1952) +* Clean up parsing for query [#1824](https://github.com/opensearch-project/k-NN/pull/1824) +* Refactor engine package structure [#1913](https://github.com/opensearch-project/k-NN/pull/1913) +* Refactor method structure and definitions [#1920](https://github.com/opensearch-project/k-NN/pull/1920) +* Refactor KNNVectorFieldType from KNNVectorFieldMapper to a separate class for better readability. [#1931](https://github.com/opensearch-project/k-NN/pull/1931) +* Generalize lib interface to return context objects [#1925](https://github.com/opensearch-project/k-NN/pull/1925) +* Restructure mappers to better handle null cases and avoid branching in parsing [#1939](https://github.com/opensearch-project/k-NN/pull/1939) +* Added Quantization Framework and implemented 1Bit and multibit quantizer[#1889](https://github.com/opensearch-project/k-NN/issues/1889) +* Encapsulate dimension, vector data type validation/processing inside Library [#1957](https://github.com/opensearch-project/k-NN/pull/1957) +* Add quantization state cache [#1960](https://github.com/opensearch-project/k-NN/pull/1960) \ No newline at end of file From cc74ca9b25b30a47f99b62c699c5d55928b4ca37 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Wed, 4 Sep 2024 19:53:24 -0700 Subject: [PATCH 351/416] Add quantization state reader and writer (#1997) (#2041) Add quantization state reader and writer Signed-off-by: Ryan Bogan (cherry picked from commit a58a9dcc7b7c264693dbf25e2ad37b0807c2f724) Co-authored-by: Ryan Bogan --- .../opensearch-knn.release-notes-2.17.0.0.md | 3 +- .../opensearch/knn/common/KNNConstants.java | 1 + .../org/opensearch/knn/index/KNNSettings.java | 9 +- .../KNN990QuantizationStateReader.java | 155 ++++++++++++ .../KNN990QuantizationStateWriter.java | 116 +++++++++ .../NativeEngines990KnnVectorsReader.java | 64 ++++- .../NativeEngines990KnnVectorsWriter.java | 26 +- .../QuantizationConfigKNNCollector.java | 64 +++++ .../opensearch/knn/index/query/KNNWeight.java | 50 +++- .../QuantizationStateCache.java | 14 +- .../QuantizationStateCacheManager.java | 82 ++++++ .../QuantizationStateReadConfig.java | 20 ++ .../KNN990QuantizationStateReaderTests.java | 236 ++++++++++++++++++ .../KNN990QuantizationStateWriterTests.java | 187 ++++++++++++++ ...NativeEngines990KnnVectorsFormatTests.java | 88 ++++++- .../knn/index/query/KNNWeightTests.java | 163 +++++++++++- .../QuantizationStateCacheManagerTests.java | 98 ++++++++ 17 files changed, 1340 insertions(+), 36 deletions(-) create mode 100644 src/main/java/org/opensearch/knn/index/codec/KNN990Codec/KNN990QuantizationStateReader.java create mode 100644 src/main/java/org/opensearch/knn/index/codec/KNN990Codec/KNN990QuantizationStateWriter.java create mode 100644 src/main/java/org/opensearch/knn/index/codec/KNN990Codec/QuantizationConfigKNNCollector.java create mode 100644 src/main/java/org/opensearch/knn/quantization/models/quantizationState/QuantizationStateCacheManager.java create mode 100644 src/main/java/org/opensearch/knn/quantization/models/quantizationState/QuantizationStateReadConfig.java create mode 100644 src/test/java/org/opensearch/knn/index/codec/KNN990Codec/KNN990QuantizationStateReaderTests.java create mode 100644 src/test/java/org/opensearch/knn/index/codec/KNN990Codec/KNN990QuantizationStateWriterTests.java create mode 100644 src/test/java/org/opensearch/knn/quantization/models/quantizationState/QuantizationStateCacheManagerTests.java diff --git a/release-notes/opensearch-knn.release-notes-2.17.0.0.md b/release-notes/opensearch-knn.release-notes-2.17.0.0.md index e042bc4e1..8dea9422b 100644 --- a/release-notes/opensearch-knn.release-notes-2.17.0.0.md +++ b/release-notes/opensearch-knn.release-notes-2.17.0.0.md @@ -30,4 +30,5 @@ Compatible with OpenSearch 2.17.0 * Restructure mappers to better handle null cases and avoid branching in parsing [#1939](https://github.com/opensearch-project/k-NN/pull/1939) * Added Quantization Framework and implemented 1Bit and multibit quantizer[#1889](https://github.com/opensearch-project/k-NN/issues/1889) * Encapsulate dimension, vector data type validation/processing inside Library [#1957](https://github.com/opensearch-project/k-NN/pull/1957) -* Add quantization state cache [#1960](https://github.com/opensearch-project/k-NN/pull/1960) \ No newline at end of file +* Add quantization state cache [#1960](https://github.com/opensearch-project/k-NN/pull/1960) +* Add quantization state reader and writer [#1997](https://github.com/opensearch-project/k-NN/pull/1997) \ No newline at end of file diff --git a/src/main/java/org/opensearch/knn/common/KNNConstants.java b/src/main/java/org/opensearch/knn/common/KNNConstants.java index b9319a434..e707d448a 100644 --- a/src/main/java/org/opensearch/knn/common/KNNConstants.java +++ b/src/main/java/org/opensearch/knn/common/KNNConstants.java @@ -74,6 +74,7 @@ public class KNNConstants { public static final String MINIMAL_MODE_AND_COMPRESSION_FEATURE = "mode_and_compression_feature"; public static final String RADIAL_SEARCH_KEY = "radial_search"; + public static final String QUANTIZATION_STATE_FILE_SUFFIX = "osknnqstate"; // Lucene specific constants public static final String LUCENE_NAME = "lucene"; diff --git a/src/main/java/org/opensearch/knn/index/KNNSettings.java b/src/main/java/org/opensearch/knn/index/KNNSettings.java index a70a17d85..4da11a2ad 100644 --- a/src/main/java/org/opensearch/knn/index/KNNSettings.java +++ b/src/main/java/org/opensearch/knn/index/KNNSettings.java @@ -24,7 +24,7 @@ import org.opensearch.knn.index.memory.NativeMemoryCacheManager; import org.opensearch.knn.index.memory.NativeMemoryCacheManagerDto; import org.opensearch.knn.index.util.IndexHyperParametersUtil; -import org.opensearch.knn.quantization.models.quantizationState.QuantizationStateCache; +import org.opensearch.knn.quantization.models.quantizationState.QuantizationStateCacheManager; import org.opensearch.monitor.jvm.JvmInfo; import org.opensearch.monitor.os.OsProbe; @@ -60,6 +60,7 @@ public class KNNSettings { private static final OsProbe osProbe = OsProbe.getInstance(); private static final int INDEX_THREAD_QTY_MAX = 32; + private static final QuantizationStateCacheManager quantizationStateCacheManager = QuantizationStateCacheManager.getInstance(); /** * Settings name @@ -379,11 +380,11 @@ private void setSettingsUpdateConsumers() { NativeMemoryCacheManager.getInstance().rebuildCache(builder.build()); }, Stream.concat(dynamicCacheSettings.values().stream(), FEATURE_FLAGS.values().stream()).collect(Collectors.toUnmodifiableList())); clusterService.getClusterSettings().addSettingsUpdateConsumer(QUANTIZATION_STATE_CACHE_SIZE_LIMIT_SETTING, it -> { - QuantizationStateCache.getInstance().setMaxCacheSizeInKB(it.getKb()); - QuantizationStateCache.getInstance().rebuildCache(); + quantizationStateCacheManager.setMaxCacheSizeInKB(it.getKb()); + quantizationStateCacheManager.rebuildCache(); }); clusterService.getClusterSettings().addSettingsUpdateConsumer(QUANTIZATION_STATE_CACHE_EXPIRY_TIME_MINUTES_SETTING, it -> { - QuantizationStateCache.getInstance().rebuildCache(); + quantizationStateCacheManager.rebuildCache(); }); } diff --git a/src/main/java/org/opensearch/knn/index/codec/KNN990Codec/KNN990QuantizationStateReader.java b/src/main/java/org/opensearch/knn/index/codec/KNN990Codec/KNN990QuantizationStateReader.java new file mode 100644 index 000000000..5ae4e7b3b --- /dev/null +++ b/src/main/java/org/opensearch/knn/index/codec/KNN990Codec/KNN990QuantizationStateReader.java @@ -0,0 +1,155 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.codec.KNN990Codec; + +import com.google.common.annotations.VisibleForTesting; +import lombok.extern.log4j.Log4j2; +import org.apache.lucene.codecs.CodecUtil; +import org.apache.lucene.index.IndexFileNames; +import org.apache.lucene.index.SegmentReadState; +import org.apache.lucene.store.IOContext; +import org.apache.lucene.store.IndexInput; +import org.opensearch.knn.common.KNNConstants; +import org.opensearch.knn.quantization.enums.ScalarQuantizationType; +import org.opensearch.knn.quantization.models.quantizationParams.ScalarQuantizationParams; +import org.opensearch.knn.quantization.models.quantizationState.MultiBitScalarQuantizationState; +import org.opensearch.knn.quantization.models.quantizationState.OneBitScalarQuantizationState; +import org.opensearch.knn.quantization.models.quantizationState.QuantizationState; +import org.opensearch.knn.quantization.models.quantizationState.QuantizationStateReadConfig; + +import java.io.IOException; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +/** + * Reads quantization states + */ +@Log4j2 +public final class KNN990QuantizationStateReader { + + /** + * Read quantization states and return list of fieldNames and bytes + * File format: + * Header + * QS1 state bytes + * QS2 state bytes + * Number of quantization states + * QS1 field number + * QS1 state bytes length + * QS1 position of state bytes + * QS2 field number + * QS2 state bytes length + * QS2 position of state bytes + * Position of index section (where QS1 field name is located) + * -1 (marker) + * Footer + * + * @param state the read state to read from + */ + public static Map read(SegmentReadState state) throws IOException { + String quantizationStateFileName = getQuantizationStateFileName(state); + Map readQuantizationStateInfos = null; + + try (IndexInput input = state.directory.openInput(quantizationStateFileName, IOContext.READ)) { + CodecUtil.retrieveChecksum(input); + + int numFields = getNumFields(input); + + readQuantizationStateInfos = new HashMap<>(); + + // Read each field's metadata from the index section and then read bytes + for (int i = 0; i < numFields; i++) { + int fieldNumber = input.readInt(); + int length = input.readInt(); + long position = input.readVLong(); + byte[] stateBytes = readStateBytes(input, position, length); + String fieldName = state.fieldInfos.fieldInfo(fieldNumber).getName(); + readQuantizationStateInfos.put(fieldName, stateBytes); + } + } catch (Exception e) { + log.warn(String.format("Unable to read the quantization state file for segment %s", state.segmentInfo.name), e); + return Collections.emptyMap(); + } + return readQuantizationStateInfos; + } + + /** + * Reads an individual quantization state for a given field + * @param readConfig a config class that contains necessary information for reading the state + * @return quantization state + */ + public static QuantizationState read(QuantizationStateReadConfig readConfig) throws IOException { + SegmentReadState segmentReadState = readConfig.getSegmentReadState(); + String field = readConfig.getField(); + String quantizationStateFileName = getQuantizationStateFileName(segmentReadState); + int fieldNumber = segmentReadState.fieldInfos.fieldInfo(field).getFieldNumber(); + + try (IndexInput input = segmentReadState.directory.openInput(quantizationStateFileName, IOContext.READ)) { + CodecUtil.retrieveChecksum(input); + int numFields = getNumFields(input); + + long position = -1; + int length = 0; + + // Read each field's metadata from the index section, break when correct field is found + for (int i = 0; i < numFields; i++) { + int tempFieldNumber = input.readInt(); + int tempLength = input.readInt(); + long tempPosition = input.readVLong(); + if (tempFieldNumber == fieldNumber) { + position = tempPosition; + length = tempLength; + break; + } + } + + if (position == -1 || length == 0) { + throw new IllegalArgumentException(String.format("Field %s not found", field)); + } + + byte[] stateBytes = readStateBytes(input, position, length); + + // Deserialize the byte array to a quantization state object + ScalarQuantizationType scalarQuantizationType = ((ScalarQuantizationParams) readConfig.getQuantizationParams()).getSqType(); + switch (scalarQuantizationType) { + case ONE_BIT: + return OneBitScalarQuantizationState.fromByteArray(stateBytes); + case TWO_BIT: + case FOUR_BIT: + return MultiBitScalarQuantizationState.fromByteArray(stateBytes); + default: + throw new IllegalArgumentException(String.format("Unexpected scalar quantization type: %s", scalarQuantizationType)); + } + } catch (Exception e) { + log.warn(String.format("Unable to read the quantization state file for segment %s", segmentReadState.segmentInfo.name), e); + return null; + } + } + + @VisibleForTesting + static int getNumFields(IndexInput input) throws IOException { + long footerStart = input.length() - CodecUtil.footerLength(); + long markerAndIndexPosition = footerStart - Integer.BYTES - Long.BYTES; + input.seek(markerAndIndexPosition); + long indexStartPosition = input.readLong(); + input.seek(indexStartPosition); + return input.readInt(); + } + + @VisibleForTesting + static byte[] readStateBytes(IndexInput input, long position, int length) throws IOException { + input.seek(position); + byte[] stateBytes = new byte[length]; + input.readBytes(stateBytes, 0, length); + return stateBytes; + } + + @VisibleForTesting + static String getQuantizationStateFileName(SegmentReadState state) { + return IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, KNNConstants.QUANTIZATION_STATE_FILE_SUFFIX); + } +} diff --git a/src/main/java/org/opensearch/knn/index/codec/KNN990Codec/KNN990QuantizationStateWriter.java b/src/main/java/org/opensearch/knn/index/codec/KNN990Codec/KNN990QuantizationStateWriter.java new file mode 100644 index 000000000..49b1819c1 --- /dev/null +++ b/src/main/java/org/opensearch/knn/index/codec/KNN990Codec/KNN990QuantizationStateWriter.java @@ -0,0 +1,116 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.codec.KNN990Codec; + +import lombok.AllArgsConstructor; +import lombok.Setter; +import org.apache.lucene.codecs.CodecUtil; +import org.apache.lucene.index.IndexFileNames; +import org.apache.lucene.index.SegmentWriteState; +import org.apache.lucene.store.IndexOutput; +import org.opensearch.knn.common.KNNConstants; +import org.opensearch.knn.quantization.models.quantizationState.QuantizationState; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +/** + * Writes quantization states to off heap memory + */ +public final class KNN990QuantizationStateWriter { + + private final IndexOutput output; + private List fieldQuantizationStates = new ArrayList<>(); + static final String NATIVE_ENGINES_990_KNN_VECTORS_FORMAT_QS_DATA = "NativeEngines990KnnVectorsFormatQSData"; + + /** + * Constructor + * Overall file format for writer: + * Header + * QS1 state bytes + * QS2 state bytes + * Number of quantization states + * QS1 field number + * QS1 state bytes length + * QS1 position of state bytes + * QS2 field number + * QS2 state bytes length + * QS2 position of state bytes + * Position of index section (where QS1 field name is located) + * -1 (marker) + * Footer + * @param segmentWriteState segment write state containing segment information + * @throws IOException exception could be thrown while creating the output + */ + public KNN990QuantizationStateWriter(SegmentWriteState segmentWriteState) throws IOException { + String quantizationStateFileName = IndexFileNames.segmentFileName( + segmentWriteState.segmentInfo.name, + segmentWriteState.segmentSuffix, + KNNConstants.QUANTIZATION_STATE_FILE_SUFFIX + ); + + output = segmentWriteState.directory.createOutput(quantizationStateFileName, segmentWriteState.context); + } + + /** + * Writes an index header + * @param segmentWriteState state containing segment information + * @throws IOException exception could be thrown while writing header + */ + public void writeHeader(SegmentWriteState segmentWriteState) throws IOException { + CodecUtil.writeIndexHeader( + output, + NATIVE_ENGINES_990_KNN_VECTORS_FORMAT_QS_DATA, + 0, + segmentWriteState.segmentInfo.getId(), + segmentWriteState.segmentSuffix + ); + } + + /** + * Writes a quantization state as bytes + * + * @param fieldNumber field number + * @param quantizationState quantization state + * @throws IOException could be thrown while writing + */ + public void writeState(int fieldNumber, QuantizationState quantizationState) throws IOException { + byte[] stateBytes = quantizationState.toByteArray(); + long position = output.getFilePointer(); + output.writeBytes(stateBytes, stateBytes.length); + fieldQuantizationStates.add(new FieldQuantizationState(fieldNumber, stateBytes, position)); + } + + /** + * Writes index footer and other index information for parsing later + * @throws IOException could be thrown while writing + */ + public void writeFooter() throws IOException { + long indexStartPosition = output.getFilePointer(); + output.writeInt(fieldQuantizationStates.size()); + for (FieldQuantizationState fieldQuantizationState : fieldQuantizationStates) { + output.writeInt(fieldQuantizationState.fieldNumber); + output.writeInt(fieldQuantizationState.stateBytes.length); + output.writeVLong(fieldQuantizationState.position); + } + output.writeLong(indexStartPosition); + output.writeInt(-1); + CodecUtil.writeFooter(output); + } + + @AllArgsConstructor + private static class FieldQuantizationState { + final int fieldNumber; + final byte[] stateBytes; + @Setter + Long position; + } + + public void closeOutput() throws IOException { + output.close(); + } +} diff --git a/src/main/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsReader.java b/src/main/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsReader.java index 74b158fa5..06f705c1f 100644 --- a/src/main/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsReader.java +++ b/src/main/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsReader.java @@ -23,8 +23,19 @@ import org.apache.lucene.search.TotalHits; import org.apache.lucene.util.Bits; import org.apache.lucene.util.IOUtils; +import org.opensearch.common.UUIDs; +import org.opensearch.knn.index.quantizationservice.QuantizationService; +import org.opensearch.knn.quantization.models.quantizationParams.QuantizationParams; +import org.opensearch.knn.quantization.models.quantizationParams.ScalarQuantizationParams; +import org.opensearch.knn.quantization.models.quantizationState.MultiBitScalarQuantizationState; +import org.opensearch.knn.quantization.models.quantizationState.OneBitScalarQuantizationState; +import org.opensearch.knn.quantization.models.quantizationState.QuantizationState; +import org.opensearch.knn.quantization.models.quantizationState.QuantizationStateCacheManager; +import org.opensearch.knn.quantization.models.quantizationState.QuantizationStateReadConfig; import java.io.IOException; +import java.util.HashMap; +import java.util.Map; /** * Vectors reader class for reading the flat vectors for native engines. The class provides methods for iterating @@ -33,8 +44,13 @@ public class NativeEngines990KnnVectorsReader extends KnnVectorsReader { private final FlatVectorsReader flatVectorsReader; + private final SegmentReadState segmentReadState; + private final QuantizationStateCacheManager quantizationStateCacheManager = QuantizationStateCacheManager.getInstance(); + private Map quantizationStateCacheKeyPerField; - public NativeEngines990KnnVectorsReader(final SegmentReadState state, final FlatVectorsReader flatVectorsReader) { + public NativeEngines990KnnVectorsReader(final SegmentReadState state, final FlatVectorsReader flatVectorsReader) throws IOException { + this.segmentReadState = state; + primeQuantizationStateCache(); this.flatVectorsReader = flatVectorsReader; } @@ -101,6 +117,22 @@ public ByteVectorValues getByteVectorValues(final String field) throws IOExcepti */ @Override public void search(String field, float[] target, KnnCollector knnCollector, Bits acceptDocs) throws IOException { + // TODO: This is a temporary hack where we are using KNNCollector to initialize the quantization state. + if (knnCollector instanceof QuantizationConfigKNNCollector) { + String cacheKey = quantizationStateCacheKeyPerField.get(field); + FieldInfo fieldInfo = segmentReadState.fieldInfos.fieldInfo(field); + QuantizationState quantizationState = QuantizationStateCacheManager.getInstance() + .getQuantizationState( + new QuantizationStateReadConfig( + segmentReadState, + QuantizationService.getInstance().getQuantizationParams(fieldInfo), + field, + cacheKey + ) + ); + ((QuantizationConfigKNNCollector) knnCollector).setQuantizationState(quantizationState); + return; + } throw new UnsupportedOperationException("Search functionality using codec is not supported with Native Engine Reader"); } @@ -150,6 +182,9 @@ public void search(String field, byte[] target, KnnCollector knnCollector, Bits @Override public void close() throws IOException { IOUtils.close(flatVectorsReader); + for (String cacheKey : quantizationStateCacheKeyPerField.values()) { + QuantizationStateCacheManager.getInstance().evict(cacheKey); + } } /** @@ -159,4 +194,31 @@ public void close() throws IOException { public long ramBytesUsed() { return flatVectorsReader.ramBytesUsed(); } + + private void primeQuantizationStateCache() throws IOException { + quantizationStateCacheKeyPerField = new HashMap<>(); + Map stateMap = KNN990QuantizationStateReader.read(segmentReadState); + for (Map.Entry entry : stateMap.entrySet()) { + FieldInfo fieldInfo = segmentReadState.fieldInfos.fieldInfo(entry.getKey()); + QuantizationParams quantizationParams = QuantizationService.getInstance().getQuantizationParams(fieldInfo); + if (quantizationParams instanceof ScalarQuantizationParams) { + QuantizationState quantizationState; + ScalarQuantizationParams scalarQuantizationParams = (ScalarQuantizationParams) quantizationParams; + switch (scalarQuantizationParams.getSqType()) { + case ONE_BIT: + quantizationState = OneBitScalarQuantizationState.fromByteArray(entry.getValue()); + break; + case TWO_BIT: + case FOUR_BIT: + quantizationState = MultiBitScalarQuantizationState.fromByteArray(entry.getValue()); + break; + default: + throw new IllegalArgumentException("Unknown Scalar Quantization Type"); + } + String cacheKey = UUIDs.base64UUID(); + quantizationStateCacheKeyPerField.put(entry.getKey(), cacheKey); + quantizationStateCacheManager.addQuantizationState(cacheKey, quantizationState); + } + } + } } diff --git a/src/main/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsWriter.java b/src/main/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsWriter.java index 1d3ff368a..af7f1c576 100644 --- a/src/main/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsWriter.java +++ b/src/main/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsWriter.java @@ -11,7 +11,6 @@ package org.opensearch.knn.index.codec.KNN990Codec; -import lombok.RequiredArgsConstructor; import lombok.extern.log4j.Log4j2; import org.apache.lucene.codecs.KnnFieldVectorsWriter; import org.apache.lucene.codecs.KnnVectorsWriter; @@ -44,7 +43,6 @@ * A KNNVectorsWriter class for writing the vector data strcutures and flat vectors for Native Engines. */ @Log4j2 -@RequiredArgsConstructor public class NativeEngines990KnnVectorsWriter extends KnnVectorsWriter { private static final long SHALLOW_SIZE = RamUsageEstimator.shallowSizeOfInstance(NativeEngines990KnnVectorsWriter.class); @@ -53,10 +51,16 @@ public class NativeEngines990KnnVectorsWriter extends KnnVectorsWriter { private final SegmentWriteState segmentWriteState; private final FlatVectorsWriter flatVectorsWriter; + private KNN990QuantizationStateWriter quantizationStateWriter; private final List> fields = new ArrayList<>(); private boolean finished; private final QuantizationService quantizationService = QuantizationService.getInstance(); + public NativeEngines990KnnVectorsWriter(SegmentWriteState segmentWriteState, FlatVectorsWriter flatVectorsWriter) { + this.segmentWriteState = segmentWriteState; + this.flatVectorsWriter = flatVectorsWriter; + } + /** * Add new field for indexing. * In Lucene, we use single file for all the vector fields so here we need to see how we are going to make things @@ -79,6 +83,7 @@ public KnnFieldVectorsWriter addField(final FieldInfo fieldInfo) throws IOExc @Override public void flush(int maxDoc, final Sorter.DocMap sortMap) throws IOException { flatVectorsWriter.flush(maxDoc, sortMap); + for (final NativeEngineFieldVectorsWriter field : fields) { trainAndIndex( field.getFieldInfo(), @@ -95,6 +100,7 @@ public void flush(int maxDoc, final Sorter.DocMap sortMap) throws IOException { public void mergeOneField(final FieldInfo fieldInfo, final MergeState mergeState) throws IOException { // This will ensure that we are merging the FlatIndex during force merge. flatVectorsWriter.mergeOneField(fieldInfo, mergeState); + // For merge, pick values from flat vector and reindex again. This will use the flush operation to create graphs trainAndIndex( fieldInfo, @@ -104,7 +110,6 @@ public void mergeOneField(final FieldInfo fieldInfo, final MergeState mergeState KNNGraphValue.MERGE_TOTAL_TIME_IN_MILLIS, MERGE_OPERATION ); - } /** @@ -116,6 +121,9 @@ public void finish() throws IOException { throw new IllegalStateException("NativeEnginesKNNVectorsWriter is already finished"); } finished = true; + if (quantizationStateWriter != null) { + quantizationStateWriter.writeFooter(); + } flatVectorsWriter.finish(); } @@ -134,6 +142,9 @@ public void finish() throws IOException { */ @Override public void close() throws IOException { + if (quantizationStateWriter != null) { + quantizationStateWriter.closeOutput(); + } IOUtils.close(flatVectorsWriter); } @@ -238,7 +249,9 @@ private void trainAndIndex( QuantizationParams quantizationParams = quantizationService.getQuantizationParams(fieldInfo); QuantizationState quantizationState = null; if (quantizationParams != null) { + initQuantizationStateWriterIfNecessary(); quantizationState = quantizationService.train(quantizationParams, knnVectorValues); + quantizationStateWriter.writeState(fieldInfo.getFieldNumber(), quantizationState); } NativeIndexWriter writer = (quantizationParams != null) ? NativeIndexWriter.getWriter(fieldInfo, segmentWriteState, quantizationState) @@ -253,4 +266,11 @@ private void trainAndIndex( graphBuildTime.incrementBy(time_in_millis); log.warn("Graph build took " + time_in_millis + " ms for " + operationName); } + + private void initQuantizationStateWriterIfNecessary() throws IOException { + if (quantizationStateWriter == null) { + quantizationStateWriter = new KNN990QuantizationStateWriter(segmentWriteState); + quantizationStateWriter.writeHeader(segmentWriteState); + } + } } diff --git a/src/main/java/org/opensearch/knn/index/codec/KNN990Codec/QuantizationConfigKNNCollector.java b/src/main/java/org/opensearch/knn/index/codec/KNN990Codec/QuantizationConfigKNNCollector.java new file mode 100644 index 000000000..295b0fe58 --- /dev/null +++ b/src/main/java/org/opensearch/knn/index/codec/KNN990Codec/QuantizationConfigKNNCollector.java @@ -0,0 +1,64 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.codec.KNN990Codec; + +import lombok.Getter; +import lombok.Setter; +import org.apache.lucene.search.KnnCollector; +import org.apache.lucene.search.TopDocs; +import org.opensearch.knn.quantization.models.quantizationState.QuantizationState; + +/** + * Collector used for passing the quantization state during query flow. + */ +@Setter +@Getter +public class QuantizationConfigKNNCollector implements KnnCollector { + + private QuantizationState quantizationState; + + private final String NATIVE_ENGINE_SEARCH_ERROR_MESSAGE = "Search functionality using codec is not supported with Native Engine Reader"; + + @Override + public boolean earlyTerminated() { + throw new UnsupportedOperationException(NATIVE_ENGINE_SEARCH_ERROR_MESSAGE); + } + + @Override + public void incVisitedCount(int i) { + throw new UnsupportedOperationException(NATIVE_ENGINE_SEARCH_ERROR_MESSAGE); + } + + @Override + public long visitedCount() { + throw new UnsupportedOperationException(NATIVE_ENGINE_SEARCH_ERROR_MESSAGE); + } + + @Override + public long visitLimit() { + throw new UnsupportedOperationException(NATIVE_ENGINE_SEARCH_ERROR_MESSAGE); + } + + @Override + public int k() { + throw new UnsupportedOperationException(NATIVE_ENGINE_SEARCH_ERROR_MESSAGE); + } + + @Override + public boolean collect(int i, float v) { + throw new UnsupportedOperationException(NATIVE_ENGINE_SEARCH_ERROR_MESSAGE); + } + + @Override + public float minCompetitiveSimilarity() { + throw new UnsupportedOperationException(NATIVE_ENGINE_SEARCH_ERROR_MESSAGE); + } + + @Override + public TopDocs topDocs() { + throw new UnsupportedOperationException(NATIVE_ENGINE_SEARCH_ERROR_MESSAGE); + } +} diff --git a/src/main/java/org/opensearch/knn/index/query/KNNWeight.java b/src/main/java/org/opensearch/knn/index/query/KNNWeight.java index 6cc110839..1769328fe 100644 --- a/src/main/java/org/opensearch/knn/index/query/KNNWeight.java +++ b/src/main/java/org/opensearch/knn/index/query/KNNWeight.java @@ -23,22 +23,24 @@ import org.apache.lucene.util.FixedBitSet; import org.opensearch.common.io.PathUtils; import org.opensearch.common.lucene.Lucene; -import org.opensearch.knn.common.FieldInfoExtractor; import org.opensearch.knn.common.KNNConstants; import org.opensearch.knn.index.KNNSettings; import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.VectorDataType; -import org.opensearch.knn.index.engine.qframe.QuantizationConfig; +import org.opensearch.knn.index.codec.KNN990Codec.QuantizationConfigKNNCollector; import org.opensearch.knn.index.memory.NativeMemoryAllocation; import org.opensearch.knn.index.memory.NativeMemoryCacheManager; import org.opensearch.knn.index.memory.NativeMemoryEntryContext; import org.opensearch.knn.index.memory.NativeMemoryLoadStrategy; import org.opensearch.knn.index.engine.KNNEngine; +import org.opensearch.knn.index.quantizationservice.QuantizationService; import org.opensearch.knn.indices.ModelDao; import org.opensearch.knn.indices.ModelMetadata; import org.opensearch.knn.indices.ModelUtil; import org.opensearch.knn.jni.JNIService; import org.opensearch.knn.plugin.stats.KNNCounter; +import org.opensearch.knn.quantization.models.quantizationOutput.QuantizationOutput; +import org.opensearch.knn.quantization.models.quantizationParams.QuantizationParams; import java.io.IOException; import java.nio.file.Path; @@ -72,6 +74,7 @@ public class KNNWeight extends Weight { private final ExactSearcher exactSearcher; private static ExactSearcher DEFAULT_EXACT_SEARCHER; + private final QuantizationService quantizationService; public KNNWeight(KNNQuery query, float boost) { super(query); @@ -80,6 +83,7 @@ public KNNWeight(KNNQuery query, float boost) { this.nativeMemoryCacheManager = NativeMemoryCacheManager.getInstance(); this.filterWeight = null; this.exactSearcher = DEFAULT_EXACT_SEARCHER; + this.quantizationService = QuantizationService.getInstance(); } public KNNWeight(KNNQuery query, float boost, Weight filterWeight) { @@ -89,6 +93,7 @@ public KNNWeight(KNNQuery query, float boost, Weight filterWeight) { this.nativeMemoryCacheManager = NativeMemoryCacheManager.getInstance(); this.filterWeight = filterWeight; this.exactSearcher = DEFAULT_EXACT_SEARCHER; + this.quantizationService = QuantizationService.getInstance(); } public static void initialize(ModelDao modelDao) { @@ -227,9 +232,6 @@ private Map doANNSearch( return null; } - // TODO: Use this to get quantization config - QuantizationConfig quantizationConfig = FieldInfoExtractor.extractQuantizationConfig(fieldInfo); - KNNEngine knnEngine; SpaceType spaceType; VectorDataType vectorDataType; @@ -256,6 +258,11 @@ private Map doANNSearch( ); } + QuantizationParams quantizationParams = quantizationService.getQuantizationParams(fieldInfo); + + // TODO: Change type of vector once more quantization methods are supported + byte[] quantizedVector = getQuantizedVector(quantizationParams, reader, fieldInfo); + List engineFiles = getEngineFiles(reader, knnEngine.getExtension()); if (engineFiles.isEmpty()) { log.debug("[KNN] No engine index found for field {} for segment {}", knnQuery.getField(), reader.getSegmentName()); @@ -273,7 +280,13 @@ private Map doANNSearch( new NativeMemoryEntryContext.IndexEntryContext( indexPath.toString(), NativeMemoryLoadStrategy.IndexLoadStrategy.getInstance(), - getParametersAtLoading(spaceType, knnEngine, knnQuery.getIndexName(), vectorDataType), + getParametersAtLoading( + spaceType, + knnEngine, + knnQuery.getIndexName(), + // TODO: In the future, more vector data types will be supported with quantization + quantizationParams == null ? vectorDataType : VectorDataType.BINARY + ), knnQuery.getIndexName(), modelId ), @@ -296,10 +309,12 @@ private Map doANNSearch( } int[] parentIds = getParentIdsArray(context); if (k > 0) { - if (knnQuery.getVectorDataType() == VectorDataType.BINARY) { + if (knnQuery.getVectorDataType() == VectorDataType.BINARY + || quantizationParams != null && quantizationService.getVectorDataTypeForTransfer(fieldInfo) == VectorDataType.BINARY) { results = JNIService.queryBinaryIndex( indexAllocation.getMemoryAddress(), - knnQuery.getByteQueryVector(), + // TODO: In the future, quantizedVector can have other data types than byte + quantizationParams == null ? knnQuery.getByteQueryVector() : quantizedVector, k, knnQuery.getMethodParameters(), knnEngine, @@ -447,4 +462,23 @@ private boolean isExactSearchThresholdSettingSet(int filterThresholdValue) { private boolean canDoExactSearchAfterANNSearch(final int filterIdsCount, final int annResultCount) { return filterWeight != null && filterIdsCount >= knnQuery.getK() && knnQuery.getK() > annResultCount; } + + // TODO: this will eventually return more types than just byte + private byte[] getQuantizedVector(QuantizationParams quantizationParams, SegmentReader reader, FieldInfo fieldInfo) throws IOException { + if (quantizationParams != null) { + QuantizationConfigKNNCollector tempCollector = new QuantizationConfigKNNCollector(); + reader.searchNearestVectors(knnQuery.getField(), new float[0], tempCollector, null); + if (tempCollector.getQuantizationState() == null) { + throw new IllegalStateException(String.format("No quantization state found for field %s", fieldInfo.getName())); + } + QuantizationOutput quantizationOutput = quantizationService.createQuantizationOutput(quantizationParams); + // TODO: In the future, byte array will not be the only output type from this method + return (byte[]) quantizationService.quantize( + tempCollector.getQuantizationState(), + knnQuery.getQueryVector(), + quantizationOutput + ); + } + return null; + } } diff --git a/src/main/java/org/opensearch/knn/quantization/models/quantizationState/QuantizationStateCache.java b/src/main/java/org/opensearch/knn/quantization/models/quantizationState/QuantizationStateCache.java index ba26d517d..cc5a34bcd 100644 --- a/src/main/java/org/opensearch/knn/quantization/models/quantizationState/QuantizationStateCache.java +++ b/src/main/java/org/opensearch/knn/quantization/models/quantizationState/QuantizationStateCache.java @@ -11,7 +11,6 @@ import com.google.common.cache.RemovalCause; import com.google.common.cache.RemovalNotification; import lombok.Getter; -import lombok.Setter; import lombok.extern.log4j.Log4j2; import org.opensearch.common.unit.TimeValue; import org.opensearch.core.common.unit.ByteSizeValue; @@ -33,7 +32,6 @@ public class QuantizationStateCache { private static volatile QuantizationStateCache instance; private Cache cache; @Getter - @Setter private long maxCacheSizeInKB; @Getter private Instant evictedDueToSizeAt; @@ -48,7 +46,7 @@ public class QuantizationStateCache { * Gets the singleton instance of the cache. * @return QuantizationStateCache */ - public static QuantizationStateCache getInstance() { + static QuantizationStateCache getInstance() { if (instance == null) { synchronized (QuantizationStateCache.class) { if (instance == null) { @@ -75,7 +73,7 @@ private void buildCache() { .build(); } - public synchronized void rebuildCache() { + synchronized void rebuildCache() { clear(); buildCache(); } @@ -85,7 +83,7 @@ public synchronized void rebuildCache() { * @param fieldName The name of the field. * @return The associated QuantizationState, or null if not present. */ - public QuantizationState getQuantizationState(String fieldName) { + QuantizationState getQuantizationState(String fieldName) { return cache.getIfPresent(fieldName); } @@ -94,7 +92,7 @@ public QuantizationState getQuantizationState(String fieldName) { * @param fieldName The name of the field. * @param quantizationState The quantization state to store. */ - public void addQuantizationState(String fieldName, QuantizationState quantizationState) { + void addQuantizationState(String fieldName, QuantizationState quantizationState) { cache.put(fieldName, quantizationState); } @@ -112,6 +110,10 @@ private void onRemoval(RemovalNotification removalNot } } + void setMaxCacheSizeInKB(long maxCacheSizeInKB) { + this.maxCacheSizeInKB = maxCacheSizeInKB; + } + private void updateEvictedDueToSizeAt() { evictedDueToSizeAt = Instant.now(); } diff --git a/src/main/java/org/opensearch/knn/quantization/models/quantizationState/QuantizationStateCacheManager.java b/src/main/java/org/opensearch/knn/quantization/models/quantizationState/QuantizationStateCacheManager.java new file mode 100644 index 000000000..932d5cde0 --- /dev/null +++ b/src/main/java/org/opensearch/knn/quantization/models/quantizationState/QuantizationStateCacheManager.java @@ -0,0 +1,82 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.quantization.models.quantizationState; + +import lombok.AccessLevel; +import lombok.NoArgsConstructor; +import org.opensearch.knn.index.codec.KNN990Codec.KNN990QuantizationStateReader; + +import java.io.IOException; + +@NoArgsConstructor(access = AccessLevel.PRIVATE) +public final class QuantizationStateCacheManager { + + private static volatile QuantizationStateCacheManager instance; + + /** + * Gets the singleton instance of the cache. + * @return QuantizationStateCache + */ + public static QuantizationStateCacheManager getInstance() { + if (instance == null) { + synchronized (QuantizationStateCacheManager.class) { + if (instance == null) { + instance = new QuantizationStateCacheManager(); + } + } + } + return instance; + } + + public synchronized void rebuildCache() { + QuantizationStateCache.getInstance().rebuildCache(); + } + + /** + * Retrieves the quantization state associated with a given field name. Reads from cache first, then from disk if necessary. + * @param quantizationStateReadConfig information required from reading from off-heap if necessary + * @return The associated QuantizationState + */ + public QuantizationState getQuantizationState(QuantizationStateReadConfig quantizationStateReadConfig) throws IOException { + QuantizationState quantizationState = QuantizationStateCache.getInstance() + .getQuantizationState(quantizationStateReadConfig.getCacheKey()); + if (quantizationState == null) { + quantizationState = KNN990QuantizationStateReader.read(quantizationStateReadConfig); + if (quantizationState != null) { + addQuantizationState(quantizationStateReadConfig.getCacheKey(), quantizationState); + } + } + return quantizationState; + } + + /** + * Adds or updates a quantization state in the cache. + * @param fieldName The name of the field. + * @param quantizationState The quantization state to store. + */ + public void addQuantizationState(String fieldName, QuantizationState quantizationState) { + QuantizationStateCache.getInstance().addQuantizationState(fieldName, quantizationState); + } + + /** + * Removes the quantization state associated with a given field name. + * @param fieldName The name of the field. + */ + public void evict(String fieldName) { + QuantizationStateCache.getInstance().evict(fieldName); + } + + public void setMaxCacheSizeInKB(long maxCacheSizeInKB) { + QuantizationStateCache.getInstance().setMaxCacheSizeInKB(maxCacheSizeInKB); + } + + /** + * Clears all entries from the cache. + */ + public void clear() { + QuantizationStateCache.getInstance().clear(); + } +} diff --git a/src/main/java/org/opensearch/knn/quantization/models/quantizationState/QuantizationStateReadConfig.java b/src/main/java/org/opensearch/knn/quantization/models/quantizationState/QuantizationStateReadConfig.java new file mode 100644 index 000000000..d13e4f3f5 --- /dev/null +++ b/src/main/java/org/opensearch/knn/quantization/models/quantizationState/QuantizationStateReadConfig.java @@ -0,0 +1,20 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.quantization.models.quantizationState; + +import lombok.AllArgsConstructor; +import lombok.Getter; +import org.apache.lucene.index.SegmentReadState; +import org.opensearch.knn.quantization.models.quantizationParams.QuantizationParams; + +@Getter +@AllArgsConstructor +public class QuantizationStateReadConfig { + private SegmentReadState segmentReadState; + private QuantizationParams quantizationParams; + private String field; + private String cacheKey; +} diff --git a/src/test/java/org/opensearch/knn/index/codec/KNN990Codec/KNN990QuantizationStateReaderTests.java b/src/test/java/org/opensearch/knn/index/codec/KNN990Codec/KNN990QuantizationStateReaderTests.java new file mode 100644 index 000000000..280156062 --- /dev/null +++ b/src/test/java/org/opensearch/knn/index/codec/KNN990Codec/KNN990QuantizationStateReaderTests.java @@ -0,0 +1,236 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.codec.KNN990Codec; + +import lombok.SneakyThrows; +import org.apache.lucene.codecs.Codec; +import org.apache.lucene.codecs.CodecUtil; +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.FieldInfos; +import org.apache.lucene.index.IndexFileNames; +import org.apache.lucene.index.SegmentInfo; +import org.apache.lucene.index.SegmentReadState; +import org.apache.lucene.search.Sort; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IOContext; +import org.apache.lucene.store.IndexInput; +import org.apache.lucene.util.Version; +import org.mockito.MockedStatic; +import org.mockito.Mockito; +import org.opensearch.knn.KNNTestCase; +import org.opensearch.knn.common.KNNConstants; +import org.opensearch.knn.quantization.enums.ScalarQuantizationType; +import org.opensearch.knn.quantization.models.quantizationParams.ScalarQuantizationParams; +import org.opensearch.knn.quantization.models.quantizationState.MultiBitScalarQuantizationState; +import org.opensearch.knn.quantization.models.quantizationState.OneBitScalarQuantizationState; +import org.opensearch.knn.quantization.models.quantizationState.QuantizationState; +import org.opensearch.knn.quantization.models.quantizationState.QuantizationStateReadConfig; + +import java.util.Map; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.Mockito.mockStatic; +import static org.mockito.Mockito.times; + +public class KNN990QuantizationStateReaderTests extends KNNTestCase { + + @SneakyThrows + public void testReadFromSegmentReadState() { + final String segmentName = "test-segment-name"; + final String segmentSuffix = "test-segment-suffix"; + + final SegmentInfo segmentInfo = new SegmentInfo( + Mockito.mock(Directory.class), + Mockito.mock(Version.class), + Mockito.mock(Version.class), + segmentName, + 0, + false, + false, + Mockito.mock(Codec.class), + Mockito.mock(Map.class), + new byte[16], + Mockito.mock(Map.class), + Mockito.mock(Sort.class) + ); + + Directory directory = Mockito.mock(Directory.class); + IndexInput input = Mockito.mock(IndexInput.class); + Mockito.when(directory.openInput(any(), any())).thenReturn(input); + + String fieldName = "test-field"; + FieldInfos fieldInfos = Mockito.mock(FieldInfos.class); + FieldInfo fieldInfo = Mockito.mock(FieldInfo.class); + Mockito.when(fieldInfo.getName()).thenReturn(fieldName); + Mockito.when(fieldInfos.fieldInfo(anyInt())).thenReturn(fieldInfo); + + final SegmentReadState segmentReadState = new SegmentReadState( + directory, + segmentInfo, + fieldInfos, + Mockito.mock(IOContext.class), + segmentSuffix + ); + + try (MockedStatic mockedStaticReader = Mockito.mockStatic(KNN990QuantizationStateReader.class)) { + mockedStaticReader.when(() -> KNN990QuantizationStateReader.getNumFields(input)).thenReturn(2); + mockedStaticReader.when(() -> KNN990QuantizationStateReader.read(segmentReadState)).thenCallRealMethod(); + try (MockedStatic mockedStaticCodecUtil = mockStatic(CodecUtil.class)) { + KNN990QuantizationStateReader.read(segmentReadState); + + mockedStaticCodecUtil.verify(() -> CodecUtil.retrieveChecksum(input)); + Mockito.verify(input, times(4)).readInt(); + Mockito.verify(input, times(2)).readVLong(); + } + } + } + + @SneakyThrows + public void testReadFromQuantizationStateReadConfig() { + String fieldName = "test-field"; + int fieldNumber = 4; + FieldInfos fieldInfos = Mockito.mock(FieldInfos.class); + FieldInfo fieldInfo = Mockito.mock(FieldInfo.class); + Mockito.when(fieldInfo.getFieldNumber()).thenReturn(fieldNumber); + Mockito.when(fieldInfos.fieldInfo(fieldName)).thenReturn(fieldInfo); + + final String segmentName = "test-segment-name"; + final String segmentSuffix = "test-segment-suffix"; + + final SegmentInfo segmentInfo = new SegmentInfo( + Mockito.mock(Directory.class), + Mockito.mock(Version.class), + Mockito.mock(Version.class), + segmentName, + 0, + false, + false, + Mockito.mock(Codec.class), + Mockito.mock(Map.class), + new byte[16], + Mockito.mock(Map.class), + Mockito.mock(Sort.class) + ); + + Directory directory = Mockito.mock(Directory.class); + IndexInput input = Mockito.mock(IndexInput.class); + Mockito.when(directory.openInput(any(), any())).thenReturn(input); + + final SegmentReadState segmentReadState = new SegmentReadState( + directory, + segmentInfo, + fieldInfos, + Mockito.mock(IOContext.class), + segmentSuffix + ); + ScalarQuantizationParams scalarQuantizationParams1 = new ScalarQuantizationParams(ScalarQuantizationType.ONE_BIT); + ScalarQuantizationParams scalarQuantizationParams2 = new ScalarQuantizationParams(ScalarQuantizationType.TWO_BIT); + ScalarQuantizationParams scalarQuantizationParams4 = new ScalarQuantizationParams(ScalarQuantizationType.FOUR_BIT); + QuantizationStateReadConfig quantizationStateReadConfig = Mockito.mock(QuantizationStateReadConfig.class); + Mockito.when(quantizationStateReadConfig.getSegmentReadState()).thenReturn(segmentReadState); + Mockito.when(quantizationStateReadConfig.getField()).thenReturn(fieldName); + + try (MockedStatic mockedStaticReader = Mockito.mockStatic(KNN990QuantizationStateReader.class)) { + mockedStaticReader.when(() -> KNN990QuantizationStateReader.getNumFields(input)).thenReturn(2); + mockedStaticReader.when(() -> KNN990QuantizationStateReader.read(quantizationStateReadConfig)).thenCallRealMethod(); + mockedStaticReader.when(() -> KNN990QuantizationStateReader.readStateBytes(any(IndexInput.class), anyLong(), anyInt())) + .thenReturn(new byte[8]); + try (MockedStatic mockedStaticCodecUtil = mockStatic(CodecUtil.class)) { + assertThrows(IllegalArgumentException.class, () -> KNN990QuantizationStateReader.read(quantizationStateReadConfig)); + + mockedStaticCodecUtil.verify(() -> CodecUtil.retrieveChecksum(input)); + Mockito.verify(input, times(4)).readInt(); + Mockito.verify(input, times(2)).readVLong(); + + Mockito.when(input.readInt()).thenReturn(fieldNumber); + + try (MockedStatic mockedStaticOneBit = mockStatic(OneBitScalarQuantizationState.class)) { + Mockito.when(quantizationStateReadConfig.getQuantizationParams()).thenReturn(scalarQuantizationParams1); + OneBitScalarQuantizationState oneBitScalarQuantizationState = Mockito.mock(OneBitScalarQuantizationState.class); + mockedStaticOneBit.when(() -> OneBitScalarQuantizationState.fromByteArray(any(byte[].class))) + .thenReturn(oneBitScalarQuantizationState); + QuantizationState quantizationState = KNN990QuantizationStateReader.read(quantizationStateReadConfig); + assertEquals(oneBitScalarQuantizationState, quantizationState); + } + + try (MockedStatic mockedStaticOneBit = mockStatic(MultiBitScalarQuantizationState.class)) { + MultiBitScalarQuantizationState multiBitScalarQuantizationState = Mockito.mock(MultiBitScalarQuantizationState.class); + mockedStaticOneBit.when(() -> MultiBitScalarQuantizationState.fromByteArray(any(byte[].class))) + .thenReturn(multiBitScalarQuantizationState); + + Mockito.when(quantizationStateReadConfig.getQuantizationParams()).thenReturn(scalarQuantizationParams2); + Mockito.when(quantizationStateReadConfig.getQuantizationParams()).thenReturn(scalarQuantizationParams2); + QuantizationState quantizationState = KNN990QuantizationStateReader.read(quantizationStateReadConfig); + assertEquals(multiBitScalarQuantizationState, quantizationState); + + Mockito.when(quantizationStateReadConfig.getQuantizationParams()).thenReturn(scalarQuantizationParams4); + Mockito.when(quantizationStateReadConfig.getQuantizationParams()).thenReturn(scalarQuantizationParams4); + quantizationState = KNN990QuantizationStateReader.read(quantizationStateReadConfig); + assertEquals(multiBitScalarQuantizationState, quantizationState); + } + } + } + } + + @SneakyThrows + public void testGetNumFields() { + IndexInput input = Mockito.mock(IndexInput.class); + KNN990QuantizationStateReader.getNumFields(input); + + Mockito.verify(input, times(1)).readInt(); + Mockito.verify(input, times(1)).readLong(); + Mockito.verify(input, times(2)).seek(anyLong()); + Mockito.verify(input, times(1)).length(); + } + + @SneakyThrows + public void testReadStateBytes() { + IndexInput input = Mockito.mock(IndexInput.class); + long position = 1; + int length = 2; + byte[] stateBytes = new byte[length]; + KNN990QuantizationStateReader.readStateBytes(input, position, length); + + Mockito.verify(input, times(1)).seek(position); + Mockito.verify(input, times(1)).readBytes(stateBytes, 0, length); + + } + + @SneakyThrows + public void testGetQuantizationStateFileName() { + String segmentName = "test-segment"; + String segmentSuffix = "test-suffix"; + String expectedName = IndexFileNames.segmentFileName(segmentName, segmentSuffix, KNNConstants.QUANTIZATION_STATE_FILE_SUFFIX); + + final SegmentInfo segmentInfo = new SegmentInfo( + Mockito.mock(Directory.class), + Mockito.mock(Version.class), + Mockito.mock(Version.class), + segmentName, + 0, + false, + false, + Mockito.mock(Codec.class), + Mockito.mock(Map.class), + new byte[16], + Mockito.mock(Map.class), + Mockito.mock(Sort.class) + ); + + final SegmentReadState segmentReadState = new SegmentReadState( + Mockito.mock(Directory.class), + segmentInfo, + Mockito.mock(FieldInfos.class), + Mockito.mock(IOContext.class), + segmentSuffix + ); + + assertEquals(expectedName, KNN990QuantizationStateReader.getQuantizationStateFileName(segmentReadState)); + + } +} diff --git a/src/test/java/org/opensearch/knn/index/codec/KNN990Codec/KNN990QuantizationStateWriterTests.java b/src/test/java/org/opensearch/knn/index/codec/KNN990Codec/KNN990QuantizationStateWriterTests.java new file mode 100644 index 000000000..2423a6827 --- /dev/null +++ b/src/test/java/org/opensearch/knn/index/codec/KNN990Codec/KNN990QuantizationStateWriterTests.java @@ -0,0 +1,187 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.codec.KNN990Codec; + +import lombok.SneakyThrows; +import org.apache.lucene.codecs.Codec; +import org.apache.lucene.codecs.CodecUtil; +import org.apache.lucene.index.FieldInfos; +import org.apache.lucene.index.SegmentInfo; +import org.apache.lucene.index.SegmentWriteState; +import org.apache.lucene.search.Sort; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IOContext; +import org.apache.lucene.store.IndexOutput; +import org.apache.lucene.util.InfoStream; +import org.apache.lucene.util.Version; +import org.mockito.MockedStatic; +import org.mockito.Mockito; +import org.mockito.stubbing.Answer; +import org.opensearch.knn.KNNTestCase; +import org.opensearch.knn.quantization.enums.ScalarQuantizationType; +import org.opensearch.knn.quantization.models.quantizationParams.ScalarQuantizationParams; +import org.opensearch.knn.quantization.models.quantizationState.OneBitScalarQuantizationState; +import org.opensearch.knn.quantization.models.quantizationState.QuantizationState; + +import java.util.Map; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mockStatic; +import static org.mockito.Mockito.times; + +public class KNN990QuantizationStateWriterTests extends KNNTestCase { + + @SneakyThrows + public void testWriteHeader() { + final String segmentName = "test-segment-name"; + + final SegmentInfo segmentInfo = new SegmentInfo( + Mockito.mock(Directory.class), + Mockito.mock(Version.class), + Mockito.mock(Version.class), + segmentName, + 0, + false, + false, + Mockito.mock(Codec.class), + Mockito.mock(Map.class), + new byte[16], + Mockito.mock(Map.class), + Mockito.mock(Sort.class) + ); + + Directory directory = Mockito.mock(Directory.class); + IndexOutput output = Mockito.mock(IndexOutput.class); + Mockito.when(directory.createOutput(any(), any())).thenReturn(output); + + final SegmentWriteState segmentWriteState = new SegmentWriteState( + Mockito.mock(InfoStream.class), + directory, + segmentInfo, + Mockito.mock(FieldInfos.class), + null, + Mockito.mock(IOContext.class) + ); + KNN990QuantizationStateWriter quantizationStateWriter = new KNN990QuantizationStateWriter(segmentWriteState); + try (MockedStatic mockedStaticCodecUtil = Mockito.mockStatic(CodecUtil.class)) { + mockedStaticCodecUtil.when( + () -> CodecUtil.writeIndexHeader(any(IndexOutput.class), anyString(), anyInt(), any(byte[].class), anyString()) + ).thenAnswer((Answer) invocation -> null); + quantizationStateWriter.writeHeader(segmentWriteState); + mockedStaticCodecUtil.verify( + () -> CodecUtil.writeIndexHeader( + output, + KNN990QuantizationStateWriter.NATIVE_ENGINES_990_KNN_VECTORS_FORMAT_QS_DATA, + 0, + segmentWriteState.segmentInfo.getId(), + segmentWriteState.segmentSuffix + ) + ); + } + } + + @SneakyThrows + public void testWriteState() { + final String segmentName = "test-segment-name"; + + final SegmentInfo segmentInfo = new SegmentInfo( + Mockito.mock(Directory.class), + Mockito.mock(Version.class), + Mockito.mock(Version.class), + segmentName, + 0, + false, + false, + Mockito.mock(Codec.class), + Mockito.mock(Map.class), + new byte[16], + Mockito.mock(Map.class), + Mockito.mock(Sort.class) + ); + + Directory directory = Mockito.mock(Directory.class); + IndexOutput output = Mockito.mock(IndexOutput.class); + Mockito.when(directory.createOutput(any(), any())).thenReturn(output); + + final SegmentWriteState segmentWriteState = new SegmentWriteState( + Mockito.mock(InfoStream.class), + directory, + segmentInfo, + Mockito.mock(FieldInfos.class), + null, + Mockito.mock(IOContext.class) + ); + KNN990QuantizationStateWriter quantizationStateWriter = new KNN990QuantizationStateWriter(segmentWriteState); + + int fieldNumber = 0; + QuantizationState quantizationState = new OneBitScalarQuantizationState( + new ScalarQuantizationParams(ScalarQuantizationType.ONE_BIT), + new float[] { 1.2f, 2.3f, 3.4f, 4.5f } + ); + quantizationStateWriter.writeState(fieldNumber, quantizationState); + byte[] stateBytes = quantizationState.toByteArray(); + Mockito.verify(output, times(1)).writeBytes(stateBytes, stateBytes.length); + } + + @SneakyThrows + public void testWriteFooter() { + final String segmentName = "test-segment-name"; + + final SegmentInfo segmentInfo = new SegmentInfo( + Mockito.mock(Directory.class), + Mockito.mock(Version.class), + Mockito.mock(Version.class), + segmentName, + 0, + false, + false, + Mockito.mock(Codec.class), + Mockito.mock(Map.class), + new byte[16], + Mockito.mock(Map.class), + Mockito.mock(Sort.class) + ); + + Directory directory = Mockito.mock(Directory.class); + IndexOutput output = Mockito.mock(IndexOutput.class); + Mockito.when(directory.createOutput(any(), any())).thenReturn(output); + + final SegmentWriteState segmentWriteState = new SegmentWriteState( + Mockito.mock(InfoStream.class), + directory, + segmentInfo, + Mockito.mock(FieldInfos.class), + null, + Mockito.mock(IOContext.class) + ); + KNN990QuantizationStateWriter quantizationStateWriter = new KNN990QuantizationStateWriter(segmentWriteState); + + int fieldNumber1 = 1; + int fieldNumber2 = 2; + QuantizationState quantizationState1 = new OneBitScalarQuantizationState( + new ScalarQuantizationParams(ScalarQuantizationType.ONE_BIT), + new float[] { 1.2f, 2.3f, 3.4f, 4.5f } + ); + QuantizationState quantizationState2 = new OneBitScalarQuantizationState( + new ScalarQuantizationParams(ScalarQuantizationType.ONE_BIT), + new float[] { 2.3f, 3.4f, 4.5f, 5.6f } + ); + quantizationStateWriter.writeState(fieldNumber1, quantizationState1); + quantizationStateWriter.writeState(fieldNumber2, quantizationState2); + + try (MockedStatic mockedStaticCodecUtil = mockStatic(CodecUtil.class)) { + quantizationStateWriter.writeFooter(); + + Mockito.verify(output, times(6)).writeInt(anyInt()); + Mockito.verify(output, times(2)).writeVLong(anyLong()); + Mockito.verify(output, times(1)).writeLong(anyLong()); + mockedStaticCodecUtil.verify(() -> CodecUtil.writeFooter(output)); + } + } +} diff --git a/src/test/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsFormatTests.java b/src/test/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsFormatTests.java index f4f02ba91..21bd4c1bd 100644 --- a/src/test/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsFormatTests.java +++ b/src/test/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsFormatTests.java @@ -14,6 +14,7 @@ import lombok.SneakyThrows; import lombok.extern.log4j.Log4j2; import org.apache.lucene.codecs.Codec; +import org.apache.lucene.codecs.CodecUtil; import org.apache.lucene.codecs.hnsw.FlatVectorScorerUtil; import org.apache.lucene.codecs.hnsw.FlatVectorsReader; import org.apache.lucene.codecs.hnsw.FlatVectorsWriter; @@ -24,25 +25,36 @@ import org.apache.lucene.document.KnnByteVectorField; import org.apache.lucene.document.KnnFloatVectorField; import org.apache.lucene.index.ByteVectorValues; +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.FieldInfos; import org.apache.lucene.index.FloatVectorValues; import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.LeafReader; import org.apache.lucene.index.NoMergePolicy; +import org.apache.lucene.index.SegmentInfo; import org.apache.lucene.index.SegmentReadState; import org.apache.lucene.index.SegmentReader; import org.apache.lucene.index.SegmentWriteState; import org.apache.lucene.index.SerialMergeScheduler; import org.apache.lucene.index.VectorEncoding; import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Sort; import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IOContext; +import org.apache.lucene.store.IndexInput; +import org.apache.lucene.store.IndexOutput; import org.apache.lucene.tests.index.RandomIndexWriter; import org.apache.lucene.tests.store.BaseDirectoryWrapper; import org.apache.lucene.util.Bits; +import org.apache.lucene.util.InfoStream; +import org.apache.lucene.util.Version; import org.junit.After; import org.junit.Assert; +import org.mockito.MockedStatic; import org.mockito.Mockito; +import org.mockito.stubbing.Answer; import org.opensearch.common.lucene.Lucene; import org.opensearch.knn.KNNTestCase; import org.opensearch.knn.common.KNNConstants; @@ -52,14 +64,18 @@ import org.opensearch.knn.index.engine.qframe.QuantizationConfigParser; import org.opensearch.knn.index.mapper.KNNVectorFieldMapper; import org.opensearch.knn.index.engine.KNNEngine; -import org.opensearch.knn.plugin.stats.KNNGraphValue; import org.opensearch.knn.quantization.enums.ScalarQuantizationType; import java.io.IOException; import java.util.Arrays; import java.util.List; +import java.util.Map; import java.util.stream.Collectors; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyString; + @Log4j2 public class NativeEngines990KnnVectorsFormatTests extends KNNTestCase { private static final Codec TESTING_CODEC = new UnitTestCodec(); @@ -82,21 +98,72 @@ public void tearDown() throws Exception { @SneakyThrows public void testReaderAndWriter_whenValidInput_thenSuccess() { final Lucene99FlatVectorsFormat mockedFlatVectorsFormat = Mockito.mock(Lucene99FlatVectorsFormat.class); - final SegmentWriteState mockedSegmentWriteState = Mockito.mock(SegmentWriteState.class); - final SegmentReadState mockedSegmentReadState = Mockito.mock(SegmentReadState.class); + final String segmentName = "test-segment-name"; + + final SegmentInfo mockedSegmentInfo = new SegmentInfo( + Mockito.mock(Directory.class), + Mockito.mock(Version.class), + Mockito.mock(Version.class), + segmentName, + 0, + false, + false, + Mockito.mock(Codec.class), + Mockito.mock(Map.class), + new byte[16], + Mockito.mock(Map.class), + Mockito.mock(Sort.class) + ); + + final String segmentSuffix = "test-segment-suffix"; + + Directory directory = Mockito.mock(Directory.class); + IndexInput input = Mockito.mock(IndexInput.class); + Mockito.when(directory.openInput(any(), any())).thenReturn(input); + + String fieldName = "test-field"; + FieldInfos fieldInfos = Mockito.mock(FieldInfos.class); + FieldInfo fieldInfo = Mockito.mock(FieldInfo.class); + Mockito.when(fieldInfo.getName()).thenReturn(fieldName); + Mockito.when(fieldInfos.fieldInfo(anyInt())).thenReturn(fieldInfo); + + final SegmentReadState mockedSegmentReadState = new SegmentReadState( + directory, + mockedSegmentInfo, + fieldInfos, + Mockito.mock(IOContext.class), + segmentSuffix + ); + + final SegmentWriteState mockedSegmentWriteState = new SegmentWriteState( + Mockito.mock(InfoStream.class), + Mockito.mock(Directory.class), + mockedSegmentInfo, + Mockito.mock(FieldInfos.class), + null, + Mockito.mock(IOContext.class) + ); Mockito.when(mockedFlatVectorsFormat.fieldsReader(mockedSegmentReadState)).thenReturn(Mockito.mock(FlatVectorsReader.class)); Mockito.when(mockedFlatVectorsFormat.fieldsWriter(mockedSegmentWriteState)).thenReturn(Mockito.mock(FlatVectorsWriter.class)); final NativeEngines990KnnVectorsFormat nativeEngines990KnnVectorsFormat = new NativeEngines990KnnVectorsFormat( mockedFlatVectorsFormat ); - Assert.assertTrue( - nativeEngines990KnnVectorsFormat.fieldsReader(mockedSegmentReadState) instanceof NativeEngines990KnnVectorsReader - ); - Assert.assertTrue( - nativeEngines990KnnVectorsFormat.fieldsWriter(mockedSegmentWriteState) instanceof NativeEngines990KnnVectorsWriter - ); + try (MockedStatic mockedStaticCodecUtil = Mockito.mockStatic(CodecUtil.class)) { + mockedStaticCodecUtil.when( + () -> CodecUtil.writeIndexHeader(any(IndexOutput.class), anyString(), anyInt(), any(byte[].class), anyString()) + ).thenAnswer((Answer) invocation -> null); + mockedStaticCodecUtil.when(() -> CodecUtil.retrieveChecksum(any(IndexInput.class))) + .thenAnswer((Answer) invocation -> null); + Assert.assertTrue( + nativeEngines990KnnVectorsFormat.fieldsReader(mockedSegmentReadState) instanceof NativeEngines990KnnVectorsReader + ); + + Assert.assertTrue( + nativeEngines990KnnVectorsFormat.fieldsWriter(mockedSegmentWriteState) instanceof NativeEngines990KnnVectorsWriter + ); + } } @SneakyThrows @@ -137,8 +204,6 @@ public void testNativeEngineVectorFormat_whenMultipleVectorFieldIndexed_thenSucc indexWriter.commit(); indexWriter.close(); - assertNotEquals(0L, (long) KNNGraphValue.REFRESH_TOTAL_TIME_IN_MILLIS.getValue()); - // Validate to see if correct values are returned, assumption here is only 1 segment is getting created IndexSearcher searcher = new IndexSearcher(indexReader); final LeafReader leafReader = searcher.getLeafContexts().get(0).reader(); @@ -208,7 +273,6 @@ public void testNativeEngineVectorFormat_whenBinaryQuantizationApplied_thenSucce indexWriter.flush(); indexWriter.commit(); indexWriter.close(); - assertNotEquals(0L, (long) KNNGraphValue.REFRESH_TOTAL_TIME_IN_MILLIS.getValue()); IndexSearcher searcher = new IndexSearcher(indexReader); final LeafReader leafReader = searcher.getLeafContexts().get(0).reader(); diff --git a/src/test/java/org/opensearch/knn/index/query/KNNWeightTests.java b/src/test/java/org/opensearch/knn/index/query/KNNWeightTests.java index 249ae04ce..a2b41804a 100644 --- a/src/test/java/org/opensearch/knn/index/query/KNNWeightTests.java +++ b/src/test/java/org/opensearch/knn/index/query/KNNWeightTests.java @@ -32,6 +32,7 @@ import org.junit.After; import org.junit.Before; import org.junit.BeforeClass; +import org.mockito.MockedConstruction; import org.mockito.MockedStatic; import org.mockito.Mockito; import org.opensearch.common.io.PathUtils; @@ -39,6 +40,7 @@ import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.knn.KNNTestCase; import org.opensearch.knn.index.KNNSettings; +import org.opensearch.knn.index.codec.KNN990Codec.QuantizationConfigKNNCollector; import org.opensearch.knn.index.engine.MethodComponentContext; import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.VectorDataType; @@ -47,6 +49,7 @@ import org.opensearch.knn.index.memory.NativeMemoryAllocation; import org.opensearch.knn.index.memory.NativeMemoryCacheManager; import org.opensearch.knn.index.engine.KNNEngine; +import org.opensearch.knn.index.quantizationservice.QuantizationService; import org.opensearch.knn.index.vectorvalues.KNNBinaryVectorValues; import org.opensearch.knn.index.vectorvalues.KNNFloatVectorValues; import org.opensearch.knn.index.vectorvalues.KNNVectorValuesFactory; @@ -54,6 +57,11 @@ import org.opensearch.knn.indices.ModelMetadata; import org.opensearch.knn.indices.ModelState; import org.opensearch.knn.jni.JNIService; +import org.opensearch.knn.quantization.enums.ScalarQuantizationType; +import org.opensearch.knn.quantization.models.quantizationParams.QuantizationParams; +import org.opensearch.knn.quantization.models.quantizationParams.ScalarQuantizationParams; +import org.opensearch.knn.quantization.models.quantizationState.OneBitScalarQuantizationState; +import org.opensearch.knn.quantization.models.quantizationState.QuantizationState; import java.io.IOException; import java.nio.file.Path; @@ -1064,7 +1072,7 @@ public void testANNWithParentsFilter_whenDoingANN_thenBitSetIsPassedToJNI() { .parentsFilter(bitSetProducer) .build(); - final KNNWeight knnWeight = new KNNWeight(query, 0.0f, null); + final KNNWeight knnWeight = new KNNWeight(query, 0.0f); jniServiceMockedStatic.when( () -> JNIService.queryIndex( @@ -1350,4 +1358,157 @@ private KNNQueryResult[] getFilteredKNNQueryResults() { .collect(Collectors.toList()) .toArray(new KNNQueryResult[0]); } + + @SneakyThrows + public void testANNWithQuantizationParams_whenStateNotFound_thenFail() { + try (MockedStatic quantizationServiceMockedStatic = Mockito.mockStatic(QuantizationService.class)) { + QuantizationService quantizationService = Mockito.mock(QuantizationService.class); + QuantizationParams quantizationParams = new ScalarQuantizationParams(ScalarQuantizationType.ONE_BIT); + Mockito.when(quantizationService.getQuantizationParams(any(FieldInfo.class))).thenReturn(quantizationParams); + quantizationServiceMockedStatic.when(QuantizationService::getInstance).thenReturn(quantizationService); + + // Given + int k = 3; + jniServiceMockedStatic.when( + () -> JNIService.queryIndex(anyLong(), eq(QUERY_VECTOR), eq(k), eq(HNSW_METHOD_PARAMETERS), any(), any(), anyInt(), any()) + ).thenReturn(getFilteredKNNQueryResults()); + + jniServiceMockedStatic.when( + () -> JNIService.queryBinaryIndex( + anyLong(), + eq(BYTE_QUERY_VECTOR), + eq(k), + eq(HNSW_METHOD_PARAMETERS), + any(), + any(), + anyInt(), + any() + ) + ).thenReturn(getFilteredKNNQueryResults()); + final SegmentReader reader = mockSegmentReader(); + final LeafReaderContext leafReaderContext = mock(LeafReaderContext.class); + when(leafReaderContext.reader()).thenReturn(reader); + + final KNNQuery query = KNNQuery.builder() + .field(FIELD_NAME) + .queryVector(QUERY_VECTOR) + .k(k) + .indexName(INDEX_NAME) + .filterQuery(FILTER_QUERY) + .methodParameters(HNSW_METHOD_PARAMETERS) + .vectorDataType(VectorDataType.FLOAT) + .build(); + + final float boost = (float) randomDoubleBetween(0, 10, true); + final KNNWeight knnWeight = new KNNWeight(query, boost); + final FieldInfos fieldInfos = mock(FieldInfos.class); + final FieldInfo fieldInfo = mock(FieldInfo.class); + final Map attributesMap = ImmutableMap.of( + KNN_ENGINE, + KNNEngine.FAISS.getName(), + PARAMETERS, + String.format(Locale.ROOT, "{\"%s\":\"%s\"}", INDEX_DESCRIPTION_PARAMETER, "HNSW32") + ); + + when(reader.getFieldInfos()).thenReturn(fieldInfos); + when(fieldInfos.fieldInfo(any())).thenReturn(fieldInfo); + when(fieldInfo.attributes()).thenReturn(attributesMap); + + expectThrows(IllegalStateException.class, () -> knnWeight.scorer(leafReaderContext)); + } + } + + @SneakyThrows + public void testANNWithQuantizationParams_thenSuccess() { + try (MockedStatic quantizationServiceMockedStatic = Mockito.mockStatic(QuantizationService.class)) { + QuantizationService quantizationService = Mockito.mock(QuantizationService.class); + ScalarQuantizationParams quantizationParams = new ScalarQuantizationParams(ScalarQuantizationType.ONE_BIT); + Mockito.when(quantizationService.getQuantizationParams(any(FieldInfo.class))).thenReturn(quantizationParams); + quantizationServiceMockedStatic.when(QuantizationService::getInstance).thenReturn(quantizationService); + + float[] meanThresholds = new float[] { 1.2f, 2.3f, 3.4f, 4.5f }; + QuantizationState quantizationState = new OneBitScalarQuantizationState(quantizationParams, meanThresholds); + + try ( + MockedConstruction quantizationCollectorMockedConstruction = Mockito.mockConstruction( + QuantizationConfigKNNCollector.class, + (mock, context) -> Mockito.when(mock.getQuantizationState()).thenReturn(quantizationState) + ) + ) { + + // Given + int k = 3; + jniServiceMockedStatic.when( + () -> JNIService.queryIndex( + anyLong(), + eq(QUERY_VECTOR), + eq(k), + eq(HNSW_METHOD_PARAMETERS), + any(), + any(), + anyInt(), + any() + ) + ).thenReturn(getFilteredKNNQueryResults()); + + jniServiceMockedStatic.when( + () -> JNIService.queryBinaryIndex( + anyLong(), + eq(BYTE_QUERY_VECTOR), + eq(k), + eq(HNSW_METHOD_PARAMETERS), + any(), + any(), + anyInt(), + any() + ) + ).thenReturn(getFilteredKNNQueryResults()); + final SegmentReader reader = mockSegmentReader(); + final LeafReaderContext leafReaderContext = mock(LeafReaderContext.class); + when(leafReaderContext.reader()).thenReturn(reader); + + final KNNQuery query = KNNQuery.builder() + .field(FIELD_NAME) + .queryVector(QUERY_VECTOR) + .k(k) + .indexName(INDEX_NAME) + .filterQuery(FILTER_QUERY) + .methodParameters(HNSW_METHOD_PARAMETERS) + .vectorDataType(VectorDataType.FLOAT) + .build(); + + final float boost = (float) randomDoubleBetween(0, 10, true); + final KNNWeight knnWeight = new KNNWeight(query, boost); + final FieldInfos fieldInfos = mock(FieldInfos.class); + final FieldInfo fieldInfo = mock(FieldInfo.class); + final Map attributesMap = ImmutableMap.of( + KNN_ENGINE, + KNNEngine.FAISS.getName(), + PARAMETERS, + String.format(Locale.ROOT, "{\"%s\":\"%s\"}", INDEX_DESCRIPTION_PARAMETER, "HNSW32") + ); + + when(reader.getFieldInfos()).thenReturn(fieldInfos); + when(fieldInfos.fieldInfo(any())).thenReturn(fieldInfo); + when(fieldInfo.attributes()).thenReturn(attributesMap); + + KNNScorer knnScorer = (KNNScorer) knnWeight.scorer(leafReaderContext); + + assertNotNull(knnScorer); + jniServiceMockedStatic.verify( + () -> JNIService.queryIndex( + anyLong(), + eq(QUERY_VECTOR), + eq(k), + eq(HNSW_METHOD_PARAMETERS), + any(), + any(), + anyInt(), + any() + ), + times(1) + ); + } + } + } } diff --git a/src/test/java/org/opensearch/knn/quantization/models/quantizationState/QuantizationStateCacheManagerTests.java b/src/test/java/org/opensearch/knn/quantization/models/quantizationState/QuantizationStateCacheManagerTests.java new file mode 100644 index 000000000..14e55e627 --- /dev/null +++ b/src/test/java/org/opensearch/knn/quantization/models/quantizationState/QuantizationStateCacheManagerTests.java @@ -0,0 +1,98 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.quantization.models.quantizationState; + +import lombok.SneakyThrows; +import org.mockito.MockedStatic; +import org.mockito.Mockito; +import org.opensearch.knn.KNNTestCase; +import org.opensearch.knn.index.codec.KNN990Codec.KNN990QuantizationStateReader; + +import static org.mockito.Mockito.times; + +public class QuantizationStateCacheManagerTests extends KNNTestCase { + + @SneakyThrows + public void testRebuildCache() { + try (MockedStatic mockedStaticCache = Mockito.mockStatic(QuantizationStateCache.class)) { + QuantizationStateCache quantizationStateCache = Mockito.mock(QuantizationStateCache.class); + mockedStaticCache.when(QuantizationStateCache::getInstance).thenReturn(quantizationStateCache); + Mockito.doNothing().when(quantizationStateCache).rebuildCache(); + QuantizationStateCacheManager.getInstance().rebuildCache(); + Mockito.verify(quantizationStateCache, times(1)).rebuildCache(); + } + } + + @SneakyThrows + public void testGetQuantizationState() { + try (MockedStatic mockedStaticCache = Mockito.mockStatic(QuantizationStateCache.class)) { + QuantizationStateReadConfig quantizationStateReadConfig = Mockito.mock(QuantizationStateReadConfig.class); + String cacheKey = "test-key"; + Mockito.when(quantizationStateReadConfig.getCacheKey()).thenReturn(cacheKey); + QuantizationState quantizationState = Mockito.mock(QuantizationState.class); + QuantizationStateCache quantizationStateCache = Mockito.mock(QuantizationStateCache.class); + mockedStaticCache.when(QuantizationStateCache::getInstance).thenReturn(quantizationStateCache); + Mockito.doNothing().when(quantizationStateCache).addQuantizationState(cacheKey, quantizationState); + try (MockedStatic mockedStaticReader = Mockito.mockStatic(KNN990QuantizationStateReader.class)) { + mockedStaticReader.when(() -> KNN990QuantizationStateReader.read(quantizationStateReadConfig)) + .thenReturn(quantizationState); + QuantizationStateCacheManager.getInstance().getQuantizationState(quantizationStateReadConfig); + Mockito.verify(quantizationStateCache, times(1)).addQuantizationState(cacheKey, quantizationState); + } + Mockito.when(quantizationStateCache.getQuantizationState(cacheKey)).thenReturn(quantizationState); + QuantizationStateCacheManager.getInstance().getQuantizationState(quantizationStateReadConfig); + Mockito.verify(quantizationStateCache, times(1)).addQuantizationState(cacheKey, quantizationState); + } + } + + @SneakyThrows + public void testEvict() { + try (MockedStatic mockedStaticCache = Mockito.mockStatic(QuantizationStateCache.class)) { + String field = "test-field"; + QuantizationStateCache quantizationStateCache = Mockito.mock(QuantizationStateCache.class); + mockedStaticCache.when(QuantizationStateCache::getInstance).thenReturn(quantizationStateCache); + Mockito.doNothing().when(quantizationStateCache).evict(field); + QuantizationStateCacheManager.getInstance().evict(field); + Mockito.verify(quantizationStateCache, times(1)).evict(field); + } + } + + @SneakyThrows + public void testAddQuantizationState() { + try (MockedStatic mockedStaticCache = Mockito.mockStatic(QuantizationStateCache.class)) { + String field = "test-field"; + QuantizationState quantizationState = Mockito.mock(QuantizationState.class); + QuantizationStateCache quantizationStateCache = Mockito.mock(QuantizationStateCache.class); + mockedStaticCache.when(QuantizationStateCache::getInstance).thenReturn(quantizationStateCache); + Mockito.doNothing().when(quantizationStateCache).addQuantizationState(field, quantizationState); + QuantizationStateCacheManager.getInstance().addQuantizationState(field, quantizationState); + Mockito.verify(quantizationStateCache, times(1)).addQuantizationState(field, quantizationState); + } + } + + @SneakyThrows + public void testSetMaxCacheSizeInKB() { + try (MockedStatic mockedStaticCache = Mockito.mockStatic(QuantizationStateCache.class)) { + long maxCacheSizeInKB = 1024; + QuantizationStateCache quantizationStateCache = Mockito.mock(QuantizationStateCache.class); + mockedStaticCache.when(QuantizationStateCache::getInstance).thenReturn(quantizationStateCache); + Mockito.doNothing().when(quantizationStateCache).setMaxCacheSizeInKB(maxCacheSizeInKB); + QuantizationStateCacheManager.getInstance().setMaxCacheSizeInKB(1024); + Mockito.verify(quantizationStateCache, times(1)).setMaxCacheSizeInKB(1024); + } + } + + @SneakyThrows + public void testClear() { + try (MockedStatic mockedStaticCache = Mockito.mockStatic(QuantizationStateCache.class)) { + QuantizationStateCache quantizationStateCache = Mockito.mock(QuantizationStateCache.class); + mockedStaticCache.when(QuantizationStateCache::getInstance).thenReturn(quantizationStateCache); + Mockito.doNothing().when(quantizationStateCache).clear(); + QuantizationStateCacheManager.getInstance().clear(); + Mockito.verify(quantizationStateCache, times(1)).clear(); + } + } +} From 91b2858108ed0fd2f2f2b702094d9e728269fbcb Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Thu, 5 Sep 2024 06:38:31 -0700 Subject: [PATCH 352/416] [Backport 2.x] Introduce mode and compression param resolution (#2042) Adds mode and compression based parameter resolution. With this, if a user specifies the mode and/or compression params, we will create a default configuration with the aim of meeting those hints. Currently, it does not contain support for overriding any of the parameters. This will be taken in a future commit. Signed-off-by: John Mazanec --- .../opensearch-knn.release-notes-2.17.0.0.md | 1 + .../opensearch/knn/index/KNNIndexShard.java | 8 +- .../KNN990QuantizationStateReader.java | 68 ++-- .../NativeEngines990KnnVectorsReader.java | 31 +- .../codec/nativeindex/NativeIndexWriter.java | 9 +- .../index/engine/KNNMethodConfigContext.java | 7 + .../knn/index/mapper/CompressionLevel.java | 30 +- .../knn/index/mapper/KNNMappingConfig.java | 18 + .../index/mapper/KNNVectorFieldMapper.java | 61 ++- .../knn/index/mapper/KNNVectorFieldType.java | 17 + .../knn/index/mapper/MethodFieldMapper.java | 10 + .../org/opensearch/knn/index/mapper/Mode.java | 2 +- .../knn/index/mapper/ModeBasedResolver.java | 209 ++++++++++ .../knn/index/mapper/ModelFieldMapper.java | 37 +- .../knn/index/query/KNNQueryBuilder.java | 3 +- .../plugin/rest/RestTrainModelHandler.java | 55 ++- .../transport/TrainingModelRequest.java | 17 +- .../opensearch/knn/training/TrainingJob.java | 17 +- .../KNN990QuantizationStateReaderTests.java | 2 + ...NativeEngines990KnnVectorsFormatTests.java | 2 + .../index/engine/KNNMethodContextTests.java | 160 +------- .../mapper/KNNVectorFieldMapperTests.java | 6 +- .../knn/integ/ModeAndCompressionIT.java | 367 ++++++++++++++---- .../opensearch/knn/integ/QFrameworkIT.java | 28 +- .../transport/TrainingModelRequestTests.java | 83 ++-- 25 files changed, 845 insertions(+), 403 deletions(-) create mode 100644 src/main/java/org/opensearch/knn/index/mapper/ModeBasedResolver.java diff --git a/release-notes/opensearch-knn.release-notes-2.17.0.0.md b/release-notes/opensearch-knn.release-notes-2.17.0.0.md index 8dea9422b..4892d8b69 100644 --- a/release-notes/opensearch-knn.release-notes-2.17.0.0.md +++ b/release-notes/opensearch-knn.release-notes-2.17.0.0.md @@ -7,6 +7,7 @@ Compatible with OpenSearch 2.17.0 * k-NN query rescore support for native engines [#1984](https://github.com/opensearch-project/k-NN/pull/1984) * Add support for byte vector with Faiss Engine HNSW algorithm [#1823](https://github.com/opensearch-project/k-NN/pull/1823) * Add support for byte vector with Faiss Engine IVF algorithm [#2002](https://github.com/opensearch-project/k-NN/pull/2002) +* Add mode/compression configuration support for disk-based vector search [#2034](https://github.com/opensearch-project/k-NN/pull/2034) ### Enhancements * Adds iterative graph build capability into a faiss index to improve the memory footprint during indexing and Integrates KNNVectorsFormat for native engines[#1950](https://github.com/opensearch-project/k-NN/pull/1950) ### Bug Fixes diff --git a/src/main/java/org/opensearch/knn/index/KNNIndexShard.java b/src/main/java/org/opensearch/knn/index/KNNIndexShard.java index 1e0040fe8..cc022b310 100644 --- a/src/main/java/org/opensearch/knn/index/KNNIndexShard.java +++ b/src/main/java/org/opensearch/knn/index/KNNIndexShard.java @@ -18,6 +18,8 @@ import org.opensearch.common.lucene.Lucene; import org.opensearch.index.engine.Engine; import org.opensearch.index.shard.IndexShard; +import org.opensearch.knn.common.FieldInfoExtractor; +import org.opensearch.knn.index.engine.qframe.QuantizationConfig; import org.opensearch.knn.index.mapper.KNNVectorFieldMapper; import org.opensearch.knn.index.memory.NativeMemoryAllocation; import org.opensearch.knn.index.memory.NativeMemoryCacheManager; @@ -182,7 +184,11 @@ List getEngineFileContexts(IndexReader indexReader, KNNEngine shardPath, spaceType, modelId, - VectorDataType.get(fieldInfo.attributes().getOrDefault(VECTOR_DATA_TYPE_FIELD, VectorDataType.FLOAT.getValue())) + FieldInfoExtractor.extractQuantizationConfig(fieldInfo) == QuantizationConfig.EMPTY + ? VectorDataType.get( + fieldInfo.attributes().getOrDefault(VECTOR_DATA_TYPE_FIELD, VectorDataType.FLOAT.getValue()) + ) + : VectorDataType.BINARY ) ); } diff --git a/src/main/java/org/opensearch/knn/index/codec/KNN990Codec/KNN990QuantizationStateReader.java b/src/main/java/org/opensearch/knn/index/codec/KNN990Codec/KNN990QuantizationStateReader.java index 5ae4e7b3b..cea496c5b 100644 --- a/src/main/java/org/opensearch/knn/index/codec/KNN990Codec/KNN990QuantizationStateReader.java +++ b/src/main/java/org/opensearch/knn/index/codec/KNN990Codec/KNN990QuantizationStateReader.java @@ -88,45 +88,41 @@ public static QuantizationState read(QuantizationStateReadConfig readConfig) thr String quantizationStateFileName = getQuantizationStateFileName(segmentReadState); int fieldNumber = segmentReadState.fieldInfos.fieldInfo(field).getFieldNumber(); - try (IndexInput input = segmentReadState.directory.openInput(quantizationStateFileName, IOContext.READ)) { - CodecUtil.retrieveChecksum(input); - int numFields = getNumFields(input); - - long position = -1; - int length = 0; - - // Read each field's metadata from the index section, break when correct field is found - for (int i = 0; i < numFields; i++) { - int tempFieldNumber = input.readInt(); - int tempLength = input.readInt(); - long tempPosition = input.readVLong(); - if (tempFieldNumber == fieldNumber) { - position = tempPosition; - length = tempLength; - break; - } + IndexInput input = segmentReadState.directory.openInput(quantizationStateFileName, IOContext.READ); + CodecUtil.retrieveChecksum(input); + int numFields = getNumFields(input); + + long position = -1; + int length = 0; + + // Read each field's metadata from the index section, break when correct field is found + for (int i = 0; i < numFields; i++) { + int tempFieldNumber = input.readInt(); + int tempLength = input.readInt(); + long tempPosition = input.readVLong(); + if (tempFieldNumber == fieldNumber) { + position = tempPosition; + length = tempLength; + break; } + } - if (position == -1 || length == 0) { - throw new IllegalArgumentException(String.format("Field %s not found", field)); - } + if (position == -1 || length == 0) { + throw new IllegalArgumentException(String.format("Field %s not found", field)); + } - byte[] stateBytes = readStateBytes(input, position, length); - - // Deserialize the byte array to a quantization state object - ScalarQuantizationType scalarQuantizationType = ((ScalarQuantizationParams) readConfig.getQuantizationParams()).getSqType(); - switch (scalarQuantizationType) { - case ONE_BIT: - return OneBitScalarQuantizationState.fromByteArray(stateBytes); - case TWO_BIT: - case FOUR_BIT: - return MultiBitScalarQuantizationState.fromByteArray(stateBytes); - default: - throw new IllegalArgumentException(String.format("Unexpected scalar quantization type: %s", scalarQuantizationType)); - } - } catch (Exception e) { - log.warn(String.format("Unable to read the quantization state file for segment %s", segmentReadState.segmentInfo.name), e); - return null; + byte[] stateBytes = readStateBytes(input, position, length); + + // Deserialize the byte array to a quantization state object + ScalarQuantizationType scalarQuantizationType = ((ScalarQuantizationParams) readConfig.getQuantizationParams()).getSqType(); + switch (scalarQuantizationType) { + case ONE_BIT: + return OneBitScalarQuantizationState.fromByteArray(stateBytes); + case TWO_BIT: + case FOUR_BIT: + return MultiBitScalarQuantizationState.fromByteArray(stateBytes); + default: + throw new IllegalArgumentException(String.format("Unexpected scalar quantization type: %s", scalarQuantizationType)); } } diff --git a/src/main/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsReader.java b/src/main/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsReader.java index 06f705c1f..ae077188a 100644 --- a/src/main/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsReader.java +++ b/src/main/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsReader.java @@ -25,10 +25,6 @@ import org.apache.lucene.util.IOUtils; import org.opensearch.common.UUIDs; import org.opensearch.knn.index.quantizationservice.QuantizationService; -import org.opensearch.knn.quantization.models.quantizationParams.QuantizationParams; -import org.opensearch.knn.quantization.models.quantizationParams.ScalarQuantizationParams; -import org.opensearch.knn.quantization.models.quantizationState.MultiBitScalarQuantizationState; -import org.opensearch.knn.quantization.models.quantizationState.OneBitScalarQuantizationState; import org.opensearch.knn.quantization.models.quantizationState.QuantizationState; import org.opensearch.knn.quantization.models.quantizationState.QuantizationStateCacheManager; import org.opensearch.knn.quantization.models.quantizationState.QuantizationStateReadConfig; @@ -50,8 +46,8 @@ public class NativeEngines990KnnVectorsReader extends KnnVectorsReader { public NativeEngines990KnnVectorsReader(final SegmentReadState state, final FlatVectorsReader flatVectorsReader) throws IOException { this.segmentReadState = state; - primeQuantizationStateCache(); this.flatVectorsReader = flatVectorsReader; + primeQuantizationStateCache(); } /** @@ -197,28 +193,9 @@ public long ramBytesUsed() { private void primeQuantizationStateCache() throws IOException { quantizationStateCacheKeyPerField = new HashMap<>(); - Map stateMap = KNN990QuantizationStateReader.read(segmentReadState); - for (Map.Entry entry : stateMap.entrySet()) { - FieldInfo fieldInfo = segmentReadState.fieldInfos.fieldInfo(entry.getKey()); - QuantizationParams quantizationParams = QuantizationService.getInstance().getQuantizationParams(fieldInfo); - if (quantizationParams instanceof ScalarQuantizationParams) { - QuantizationState quantizationState; - ScalarQuantizationParams scalarQuantizationParams = (ScalarQuantizationParams) quantizationParams; - switch (scalarQuantizationParams.getSqType()) { - case ONE_BIT: - quantizationState = OneBitScalarQuantizationState.fromByteArray(entry.getValue()); - break; - case TWO_BIT: - case FOUR_BIT: - quantizationState = MultiBitScalarQuantizationState.fromByteArray(entry.getValue()); - break; - default: - throw new IllegalArgumentException("Unknown Scalar Quantization Type"); - } - String cacheKey = UUIDs.base64UUID(); - quantizationStateCacheKeyPerField.put(entry.getKey(), cacheKey); - quantizationStateCacheManager.addQuantizationState(cacheKey, quantizationState); - } + for (FieldInfo fieldInfo : segmentReadState.fieldInfos) { + String cacheKey = UUIDs.base64UUID(); + quantizationStateCacheKeyPerField.put(fieldInfo.getName(), cacheKey); } } } diff --git a/src/main/java/org/opensearch/knn/index/codec/nativeindex/NativeIndexWriter.java b/src/main/java/org/opensearch/knn/index/codec/nativeindex/NativeIndexWriter.java index 886c6d93d..087773044 100644 --- a/src/main/java/org/opensearch/knn/index/codec/nativeindex/NativeIndexWriter.java +++ b/src/main/java/org/opensearch/knn/index/codec/nativeindex/NativeIndexWriter.java @@ -18,12 +18,14 @@ import org.opensearch.core.xcontent.DeprecationHandler; import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.knn.common.FieldInfoExtractor; import org.opensearch.knn.common.KNNConstants; import org.opensearch.knn.index.KNNSettings; import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.codec.nativeindex.model.BuildIndexParams; import org.opensearch.knn.index.engine.KNNEngine; +import org.opensearch.knn.index.engine.qframe.QuantizationConfig; import org.opensearch.knn.index.quantizationservice.QuantizationService; import org.opensearch.knn.index.util.IndexUtil; import org.opensearch.knn.index.vectorvalues.KNNVectorValues; @@ -255,7 +257,12 @@ private Map getTemplateParameters(FieldInfo fieldInfo, Model mod parameters.put(KNNConstants.INDEX_THREAD_QTY, KNNSettings.state().getSettingValue(KNNSettings.KNN_ALGO_PARAM_INDEX_THREAD_QTY)); parameters.put(KNNConstants.MODEL_ID, fieldInfo.attributes().get(MODEL_ID)); parameters.put(KNNConstants.MODEL_BLOB_PARAMETER, model.getModelBlob()); - IndexUtil.updateVectorDataTypeToParameters(parameters, model.getModelMetadata().getVectorDataType()); + if (FieldInfoExtractor.extractQuantizationConfig(fieldInfo) != QuantizationConfig.EMPTY) { + IndexUtil.updateVectorDataTypeToParameters(parameters, VectorDataType.BINARY); + } else { + IndexUtil.updateVectorDataTypeToParameters(parameters, model.getModelMetadata().getVectorDataType()); + } + return parameters; } diff --git a/src/main/java/org/opensearch/knn/index/engine/KNNMethodConfigContext.java b/src/main/java/org/opensearch/knn/index/engine/KNNMethodConfigContext.java index 1ba2777dd..ccb427d29 100644 --- a/src/main/java/org/opensearch/knn/index/engine/KNNMethodConfigContext.java +++ b/src/main/java/org/opensearch/knn/index/engine/KNNMethodConfigContext.java @@ -12,6 +12,8 @@ import lombok.Setter; import org.opensearch.Version; import org.opensearch.knn.index.VectorDataType; +import org.opensearch.knn.index.mapper.CompressionLevel; +import org.opensearch.knn.index.mapper.Mode; /** * This object provides additional context that the user does not provide when {@link KNNMethodContext} is @@ -27,5 +29,10 @@ public final class KNNMethodConfigContext { private VectorDataType vectorDataType; private Integer dimension; private Version versionCreated; + @Builder.Default + private Mode mode = Mode.NOT_CONFIGURED; + @Builder.Default + private CompressionLevel compressionLevel = CompressionLevel.NOT_CONFIGURED; + public static final KNNMethodConfigContext EMPTY = KNNMethodConfigContext.builder().build(); } diff --git a/src/main/java/org/opensearch/knn/index/mapper/CompressionLevel.java b/src/main/java/org/opensearch/knn/index/mapper/CompressionLevel.java index b5ce81af9..4b5026598 100644 --- a/src/main/java/org/opensearch/knn/index/mapper/CompressionLevel.java +++ b/src/main/java/org/opensearch/knn/index/mapper/CompressionLevel.java @@ -8,10 +8,9 @@ import lombok.AllArgsConstructor; import lombok.Getter; import org.opensearch.core.common.Strings; +import org.opensearch.knn.index.query.rescore.RescoreContext; -import java.util.Arrays; import java.util.Locale; -import java.util.stream.Collectors; /** * Enum representing the compression level for float vectors. Compression in this sense refers to compressing a @@ -20,20 +19,23 @@ */ @AllArgsConstructor public enum CompressionLevel { - NOT_CONFIGURED(-1, ""), - x1(1, "1x"), - x2(2, "2x"), - x4(4, "4x"), - x8(8, "8x"), - x16(16, "16x"), - x32(32, "32x"); + NOT_CONFIGURED(-1, "", null), + x1(1, "1x", null), + x2(2, "2x", null), + x4(4, "4x", new RescoreContext(1.0f)), + x8(8, "8x", new RescoreContext(1.5f)), + x16(16, "16x", new RescoreContext(2.0f)), + x32(32, "32x", new RescoreContext(2.0f)); // Internally, an empty string is easier to deal with them null. However, from the mapping, // we do not want users to pass in the empty string and instead want null. So we make the conversion herex - static final String[] NAMES_ARRAY = Arrays.stream(CompressionLevel.values()) - .map(compressionLevel -> compressionLevel == NOT_CONFIGURED ? null : compressionLevel.getName()) - .collect(Collectors.toList()) - .toArray(new String[0]); + public static final String[] NAMES_ARRAY = new String[] { + NOT_CONFIGURED.getName(), + x1.getName(), + x2.getName(), + x8.getName(), + x16.getName(), + x32.getName() }; /** * Default is set to 1x and is a noop @@ -62,6 +64,8 @@ public static CompressionLevel fromName(String name) { private final int compressionLevel; @Getter private final String name; + @Getter + private final RescoreContext defaultRescoreContext; /** * Gets the number of bits used to represent a float in order to achieve this compression. For instance, for diff --git a/src/main/java/org/opensearch/knn/index/mapper/KNNMappingConfig.java b/src/main/java/org/opensearch/knn/index/mapper/KNNMappingConfig.java index 4fcd6e1bc..5b1955f23 100644 --- a/src/main/java/org/opensearch/knn/index/mapper/KNNMappingConfig.java +++ b/src/main/java/org/opensearch/knn/index/mapper/KNNMappingConfig.java @@ -30,6 +30,24 @@ default Optional getKnnMethodContext() { return Optional.empty(); } + /** + * Return the mode to be used for this field + * + * @return {@link Mode} + */ + default Mode getMode() { + return Mode.NOT_CONFIGURED; + } + + /** + * Return compression level to be used for this field + * + * @return {@link CompressionLevel} + */ + default CompressionLevel getCompressionLevel() { + return CompressionLevel.NOT_CONFIGURED; + } + /** * * @return the dimension of the index; for model based indices, it will be null diff --git a/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapper.java b/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapper.java index 0eab5a7bb..d2bb8e41a 100644 --- a/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapper.java +++ b/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapper.java @@ -145,16 +145,7 @@ public static class Builder extends ParametrizedFieldMapper.Builder { b.startObject(n); v.toXContent(b, ToXContent.EMPTY_PARAMS); b.endObject(); - }), m -> m.getMethodComponentContext().getName()).setValidator(v -> { - if (v == null) return; - - ValidationException validationException; - if (v.isTrainingRequired()) { - validationException = new ValidationException(); - validationException.addValidationError(String.format(Locale.ROOT, "\"%s\" requires training.", KNN_METHOD)); - throw validationException; - } - }); + }), m -> m.getMethodComponentContext().getName()); protected final Parameter mode = Parameter.restrictedStringParam( KNNConstants.MODE_PARAMETER, @@ -354,6 +345,7 @@ public Mapper.Builder parse(String name, Map node, ParserCont } else if (builder.modelId.get() != null) { validateFromModel(builder); } else { + validateMode(builder); resolveKNNMethodComponents(builder, parserContext); validateFromKNNMethod(builder); } @@ -361,6 +353,26 @@ public Mapper.Builder parse(String name, Map node, ParserCont return builder; } + private void validateMode(KNNVectorFieldMapper.Builder builder) { + boolean isKNNMethodContextConfigured = builder.originalParameters.getKnnMethodContext() != null; + boolean isModeConfigured = builder.mode.isConfigured() || builder.compressionLevel.isConfigured(); + if (isModeConfigured && isKNNMethodContextConfigured) { + throw new MapperParsingException( + String.format( + Locale.ROOT, + "Compression and mode can not be specified in a \"method\" mapping configuration for field: %s", + builder.name + ) + ); + } + + if (isModeConfigured && builder.vectorDataType.getValue() != VectorDataType.FLOAT) { + throw new MapperParsingException( + String.format(Locale.ROOT, "Compression and mode cannot be used for non-float32 data type for field %s", builder.name) + ); + } + } + private void validateFromFlat(KNNVectorFieldMapper.Builder builder) { if (builder.modelId.get() != null || builder.knnMethodContext.get() != null) { throw new IllegalArgumentException("Cannot set modelId or method parameters when index.knn setting is false"); @@ -378,9 +390,15 @@ private void validateFromModel(KNNVectorFieldMapper.Builder builder) { } private void validateFromKNNMethod(KNNVectorFieldMapper.Builder builder) { + ValidationException validationException; + if (builder.originalParameters.getResolvedKnnMethodContext().isTrainingRequired()) { + validationException = new ValidationException(); + validationException.addValidationError(String.format(Locale.ROOT, "\"%s\" requires training.", KNN_METHOD)); + throw validationException; + } + if (builder.originalParameters.getResolvedKnnMethodContext() != null) { - ValidationException validationException = builder.originalParameters.getResolvedKnnMethodContext() - .validate(builder.knnMethodConfigContext); + validationException = builder.originalParameters.getResolvedKnnMethodContext().validate(builder.knnMethodConfigContext); if (validationException != null) { throw validationException; } @@ -410,9 +428,11 @@ private void validateCompressionAndModeNotSet(KNNVectorFieldMapper.Builder build private void resolveKNNMethodComponents(KNNVectorFieldMapper.Builder builder, ParserContext parserContext) { builder.setKnnMethodConfigContext( KNNMethodConfigContext.builder() - .vectorDataType(builder.vectorDataType.getValue()) + .vectorDataType(builder.originalParameters.getVectorDataType()) .versionCreated(parserContext.indexVersionCreated()) - .dimension(builder.dimension.getValue()) + .dimension(builder.originalParameters.getDimension()) + .mode(Mode.fromName(builder.originalParameters.getMode())) + .compressionLevel(CompressionLevel.fromName(builder.originalParameters.getCompressionLevel())) .build() ); @@ -421,8 +441,17 @@ private void resolveKNNMethodComponents(KNNVectorFieldMapper.Builder builder, Pa builder.originalParameters.setResolvedKnnMethodContext( createKNNMethodContextFromLegacy(parserContext.getSettings(), parserContext.indexVersionCreated()) ); - } - setDefaultSpaceType(builder.originalParameters.getResolvedKnnMethodContext(), builder.vectorDataType.getValue()); + } else if (Mode.isConfigured(Mode.fromName(builder.mode.get())) + || CompressionLevel.isConfigured(CompressionLevel.fromName(builder.compressionLevel.get()))) { + builder.originalParameters.setResolvedKnnMethodContext( + ModeBasedResolver.INSTANCE.resolveKNNMethodContext( + builder.knnMethodConfigContext.getMode(), + builder.knnMethodConfigContext.getCompressionLevel(), + false + ) + ); + } + setDefaultSpaceType(builder.originalParameters.getResolvedKnnMethodContext(), builder.originalParameters.getVectorDataType()); } private boolean isKNNDisabled(Settings settings) { diff --git a/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldType.java b/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldType.java index 0fbc569f7..963688d0c 100644 --- a/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldType.java +++ b/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldType.java @@ -17,6 +17,7 @@ import org.opensearch.index.query.QueryShardException; import org.opensearch.knn.index.KNNVectorIndexFieldData; import org.opensearch.knn.index.VectorDataType; +import org.opensearch.knn.index.query.rescore.RescoreContext; import org.opensearch.search.aggregations.support.CoreValuesSourceType; import org.opensearch.search.lookup.SearchLookup; @@ -81,4 +82,20 @@ public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, S public Object valueForDisplay(Object value) { return deserializeStoredVector((BytesRef) value, vectorDataType); } + + /** + * Resolve the rescore context provided for a user based on the field configuration + * + * @param userProvidedContext {@link RescoreContext} user passed; if null, the default should be configured + * @return resolved {@link RescoreContext} + */ + public RescoreContext resolveRescoreContext(RescoreContext userProvidedContext) { + if (userProvidedContext != null) { + return userProvidedContext; + } + return ModeBasedResolver.INSTANCE.resolveRescoreContext( + getKnnMappingConfig().getMode(), + getKnnMappingConfig().getCompressionLevel() + ); + } } diff --git a/src/main/java/org/opensearch/knn/index/mapper/MethodFieldMapper.java b/src/main/java/org/opensearch/knn/index/mapper/MethodFieldMapper.java index f1a87c64b..d479da39c 100644 --- a/src/main/java/org/opensearch/knn/index/mapper/MethodFieldMapper.java +++ b/src/main/java/org/opensearch/knn/index/mapper/MethodFieldMapper.java @@ -65,6 +65,16 @@ public Optional getKnnMethodContext() { public int getDimension() { return knnMethodConfigContext.getDimension(); } + + @Override + public Mode getMode() { + return knnMethodConfigContext.getMode(); + } + + @Override + public CompressionLevel getCompressionLevel() { + return knnMethodConfigContext.getCompressionLevel(); + } } ); return new MethodFieldMapper( diff --git a/src/main/java/org/opensearch/knn/index/mapper/Mode.java b/src/main/java/org/opensearch/knn/index/mapper/Mode.java index 0798ab941..51822cae1 100644 --- a/src/main/java/org/opensearch/knn/index/mapper/Mode.java +++ b/src/main/java/org/opensearch/knn/index/mapper/Mode.java @@ -26,7 +26,7 @@ public enum Mode { // Internally, an empty string is easier to deal with them null. However, from the mapping, // we do not want users to pass in the empty string and instead want null. So we make the conversion herex - static final String[] NAMES_ARRAY = Arrays.stream(Mode.values()) + public static final String[] NAMES_ARRAY = Arrays.stream(Mode.values()) .map(mode -> mode == NOT_CONFIGURED ? null : mode.getName()) .collect(Collectors.toList()) .toArray(new String[0]); diff --git a/src/main/java/org/opensearch/knn/index/mapper/ModeBasedResolver.java b/src/main/java/org/opensearch/knn/index/mapper/ModeBasedResolver.java new file mode 100644 index 000000000..06b34fcd3 --- /dev/null +++ b/src/main/java/org/opensearch/knn/index/mapper/ModeBasedResolver.java @@ -0,0 +1,209 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.mapper; + +import org.opensearch.knn.index.KNNSettings; +import org.opensearch.knn.index.SpaceType; +import org.opensearch.knn.index.engine.KNNEngine; +import org.opensearch.knn.index.engine.KNNMethodContext; +import org.opensearch.knn.index.engine.MethodComponentContext; +import org.opensearch.knn.index.engine.faiss.QFrameBitEncoder; +import org.opensearch.knn.index.query.rescore.RescoreContext; + +import java.util.Locale; +import java.util.Map; +import java.util.Set; + +import static org.opensearch.knn.common.KNNConstants.ENCODER_SQ; +import static org.opensearch.knn.common.KNNConstants.FAISS_SQ_CLIP; +import static org.opensearch.knn.common.KNNConstants.FAISS_SQ_ENCODER_FP16; +import static org.opensearch.knn.common.KNNConstants.FAISS_SQ_TYPE; +import static org.opensearch.knn.common.KNNConstants.METHOD_ENCODER_PARAMETER; +import static org.opensearch.knn.common.KNNConstants.METHOD_HNSW; +import static org.opensearch.knn.common.KNNConstants.METHOD_IVF; +import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_EF_CONSTRUCTION; +import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_EF_SEARCH; +import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_M; +import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_NLIST; +import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_NLIST_DEFAULT; +import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_NPROBES; +import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_NPROBES_DEFAULT; + +/** + * Class contains the logic to make parameter resolutions based on the {@link Mode} and {@link CompressionLevel}. + */ +public final class ModeBasedResolver { + + public static final ModeBasedResolver INSTANCE = new ModeBasedResolver(); + + private static final CompressionLevel DEFAULT_COMPRESSION_FOR_MODE_ON_DISK = CompressionLevel.x32; + private static final CompressionLevel DEFAULT_COMPRESSION_FOR_MODE_IN_MEMORY = CompressionLevel.x1; + public final static Set SUPPORTED_COMPRESSION_LEVELS = Set.of( + CompressionLevel.x1, + CompressionLevel.x2, + CompressionLevel.x8, + CompressionLevel.x16, + CompressionLevel.x32 + ); + + private ModeBasedResolver() {} + + /** + * Based on the provided {@link Mode} and {@link CompressionLevel}, resolve to a {@link KNNMethodContext} + * + * @param mode {@link Mode} + * @param compressionLevel {@link CompressionLevel} + * @param requiresTraining whether config requires trianing + * @return {@link KNNMethodContext} + */ + public KNNMethodContext resolveKNNMethodContext(Mode mode, CompressionLevel compressionLevel, boolean requiresTraining) { + if (requiresTraining) { + return resolveWithTraining(mode, compressionLevel); + } + + return resolveWithoutTraining(mode, compressionLevel); + } + + private KNNMethodContext resolveWithoutTraining(Mode mode, CompressionLevel compressionLevel) { + CompressionLevel resolvedCompressionLevel = resolveCompressionLevel(mode, compressionLevel); + MethodComponentContext encoderContext = resolveEncoder(resolvedCompressionLevel); + + KNNEngine knnEngine = Mode.ON_DISK == mode || encoderContext != null ? KNNEngine.FAISS : KNNEngine.DEFAULT; + + if (encoderContext != null) { + return new KNNMethodContext( + knnEngine, + SpaceType.DEFAULT, + new MethodComponentContext( + METHOD_HNSW, + Map.of( + METHOD_PARAMETER_M, + KNNSettings.INDEX_KNN_DEFAULT_ALGO_PARAM_M, + METHOD_PARAMETER_EF_CONSTRUCTION, + KNNSettings.INDEX_KNN_DEFAULT_ALGO_PARAM_EF_CONSTRUCTION, + METHOD_PARAMETER_EF_SEARCH, + KNNSettings.INDEX_KNN_DEFAULT_ALGO_PARAM_EF_SEARCH, + METHOD_ENCODER_PARAMETER, + encoderContext + ) + ) + ); + } + + if (knnEngine == KNNEngine.FAISS) { + return new KNNMethodContext( + knnEngine, + SpaceType.DEFAULT, + new MethodComponentContext( + METHOD_HNSW, + Map.of( + METHOD_PARAMETER_M, + KNNSettings.INDEX_KNN_DEFAULT_ALGO_PARAM_M, + METHOD_PARAMETER_EF_CONSTRUCTION, + KNNSettings.INDEX_KNN_DEFAULT_ALGO_PARAM_EF_CONSTRUCTION, + METHOD_PARAMETER_EF_SEARCH, + KNNSettings.INDEX_KNN_DEFAULT_ALGO_PARAM_EF_SEARCH + ) + ) + ); + } + + return new KNNMethodContext( + knnEngine, + SpaceType.DEFAULT, + new MethodComponentContext( + METHOD_HNSW, + Map.of( + METHOD_PARAMETER_M, + KNNSettings.INDEX_KNN_DEFAULT_ALGO_PARAM_M, + METHOD_PARAMETER_EF_CONSTRUCTION, + KNNSettings.INDEX_KNN_DEFAULT_ALGO_PARAM_EF_CONSTRUCTION + ) + ) + ); + } + + private KNNMethodContext resolveWithTraining(Mode mode, CompressionLevel compressionLevel) { + CompressionLevel resolvedCompressionLevel = resolveCompressionLevel(mode, compressionLevel); + MethodComponentContext encoderContext = resolveEncoder(resolvedCompressionLevel); + if (encoderContext != null) { + return new KNNMethodContext( + KNNEngine.FAISS, + SpaceType.DEFAULT, + new MethodComponentContext( + METHOD_IVF, + Map.of( + METHOD_PARAMETER_NLIST, + METHOD_PARAMETER_NLIST_DEFAULT, + METHOD_PARAMETER_NPROBES, + METHOD_PARAMETER_NPROBES_DEFAULT, + METHOD_ENCODER_PARAMETER, + encoderContext + ) + ) + ); + } + + return new KNNMethodContext( + KNNEngine.FAISS, + SpaceType.DEFAULT, + new MethodComponentContext( + METHOD_IVF, + Map.of(METHOD_PARAMETER_NLIST, METHOD_PARAMETER_NLIST_DEFAULT, METHOD_PARAMETER_NPROBES, METHOD_PARAMETER_NPROBES_DEFAULT) + ) + ); + } + + /** + * Resolves the rescore context give the {@link Mode} and {@link CompressionLevel} + * + * @param mode {@link Mode} + * @param compressionLevel {@link CompressionLevel} + * @return {@link RescoreContext} + */ + public RescoreContext resolveRescoreContext(Mode mode, CompressionLevel compressionLevel) { + CompressionLevel resolvedCompressionLevel = resolveCompressionLevel(mode, compressionLevel); + return resolvedCompressionLevel.getDefaultRescoreContext(); + } + + private CompressionLevel resolveCompressionLevel(Mode mode, CompressionLevel compressionLevel) { + if (CompressionLevel.isConfigured(compressionLevel)) { + return compressionLevel; + } + + if (mode == Mode.ON_DISK) { + return DEFAULT_COMPRESSION_FOR_MODE_ON_DISK; + } + + return DEFAULT_COMPRESSION_FOR_MODE_IN_MEMORY; + } + + private MethodComponentContext resolveEncoder(CompressionLevel compressionLevel) { + if (CompressionLevel.isConfigured(compressionLevel) == false) { + throw new IllegalStateException("Compression level needs to be configured"); + } + + if (SUPPORTED_COMPRESSION_LEVELS.contains(compressionLevel) == false) { + throw new IllegalArgumentException( + String.format(Locale.ROOT, "Unsupported compression level: \"[%s]\"", compressionLevel.getName()) + ); + } + + if (compressionLevel == CompressionLevel.x1) { + return null; + } + + if (compressionLevel == CompressionLevel.x2) { + return new MethodComponentContext(ENCODER_SQ, Map.of(FAISS_SQ_TYPE, FAISS_SQ_ENCODER_FP16, FAISS_SQ_CLIP, true)); + } + + return new MethodComponentContext( + QFrameBitEncoder.NAME, + Map.of(QFrameBitEncoder.BITCOUNT_PARAM, compressionLevel.numBitsForFloat32()) + ); + } + +} diff --git a/src/main/java/org/opensearch/knn/index/mapper/ModelFieldMapper.java b/src/main/java/org/opensearch/knn/index/mapper/ModelFieldMapper.java index bfb188a75..b7bbc5a0d 100644 --- a/src/main/java/org/opensearch/knn/index/mapper/ModelFieldMapper.java +++ b/src/main/java/org/opensearch/knn/index/mapper/ModelFieldMapper.java @@ -60,6 +60,10 @@ public static ModelFieldMapper createFieldMapper( ) { final KNNVectorFieldType mappedFieldType = new KNNVectorFieldType(fullname, metaValue, vectorDataType, new KNNMappingConfig() { + private Integer dimension = null; + private Mode mode = null; + private CompressionLevel compressionLevel = null; + @Override public Optional getModelId() { return Optional.of(originalMappingParameters.getModelId()); @@ -67,7 +71,36 @@ public Optional getModelId() { @Override public int getDimension() { - return getModelMetadata(modelDao, originalMappingParameters.getModelId()).getDimension(); + if (dimension == null) { + initFromModelMetadata(); + } + + return dimension; + } + + @Override + public Mode getMode() { + if (mode == null) { + initFromModelMetadata(); + } + return mode; + } + + @Override + public CompressionLevel getCompressionLevel() { + if (compressionLevel == null) { + initFromModelMetadata(); + } + return compressionLevel; + } + + // ModelMetadata relies on cluster state which may not be available during field mapper creation. Thus, + // we lazily initialize it. + private void initFromModelMetadata() { + ModelMetadata modelMetadata = getModelMetadata(modelDao, originalMappingParameters.getModelId()); + dimension = modelMetadata.getDimension(); + mode = modelMetadata.getMode(); + compressionLevel = modelMetadata.getCompressionLevel(); } }); return new ModelFieldMapper( @@ -258,6 +291,8 @@ private static KNNMethodConfigContext getKNNMethodConfigContextFromModelMetadata .vectorDataType(modelMetadata.getVectorDataType()) .dimension(modelMetadata.getDimension()) .versionCreated(Version.V_2_14_0) + .mode(modelMetadata.getMode()) + .compressionLevel(modelMetadata.getCompressionLevel()) .build(); } diff --git a/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java b/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java index 37f159fa2..b699a7705 100644 --- a/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java +++ b/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java @@ -413,6 +413,7 @@ protected Query doToQuery(QueryShardContext context) { MethodComponentContext methodComponentContext = queryConfigFromMapping.get().getMethodComponentContext(); SpaceType spaceType = queryConfigFromMapping.get().getSpaceType(); VectorDataType vectorDataType = queryConfigFromMapping.get().getVectorDataType(); + RescoreContext processedRescoreContext = knnVectorFieldType.resolveRescoreContext(rescoreContext); VectorQueryType vectorQueryType = getVectorQueryType(k, maxDistance, minScore); updateQueryStats(vectorQueryType); @@ -529,7 +530,7 @@ protected Query doToQuery(QueryShardContext context) { .methodParameters(this.methodParameters) .filter(this.filter) .context(context) - .rescoreContext(rescoreContext) + .rescoreContext(processedRescoreContext) .build(); return KNNQueryFactory.create(createQueryRequest); } diff --git a/src/main/java/org/opensearch/knn/plugin/rest/RestTrainModelHandler.java b/src/main/java/org/opensearch/knn/plugin/rest/RestTrainModelHandler.java index c9038f0c7..8770449eb 100644 --- a/src/main/java/org/opensearch/knn/plugin/rest/RestTrainModelHandler.java +++ b/src/main/java/org/opensearch/knn/plugin/rest/RestTrainModelHandler.java @@ -34,12 +34,14 @@ import java.util.Locale; import static org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken; +import static org.opensearch.knn.common.KNNConstants.COMPRESSION_LEVEL_PARAMETER; import static org.opensearch.knn.common.KNNConstants.DIMENSION; import static org.opensearch.knn.common.KNNConstants.KNN_METHOD; import static org.opensearch.knn.common.KNNConstants.MAX_VECTOR_COUNT_PARAMETER; import static org.opensearch.knn.common.KNNConstants.MODELS; import static org.opensearch.knn.common.KNNConstants.MODEL_DESCRIPTION; import static org.opensearch.knn.common.KNNConstants.MODEL_ID; +import static org.opensearch.knn.common.KNNConstants.MODE_PARAMETER; import static org.opensearch.knn.common.KNNConstants.PREFERENCE_PARAMETER; import static org.opensearch.knn.common.KNNConstants.SEARCH_SIZE_PARAMETER; import static org.opensearch.knn.common.KNNConstants.TRAIN_FIELD_PARAMETER; @@ -131,7 +133,10 @@ private TrainingModelRequest createTransportRequest(RestRequest restRequest) thr } // Check that these parameters get set - ensureSet(KNN_METHOD, knnMethodContext); + ensureAtleasOneSet(KNN_METHOD, knnMethodContext, MODE_PARAMETER, mode, COMPRESSION_LEVEL_PARAMETER, compressionLevel); + ensureMutualExclusion(KNN_METHOD, knnMethodContext, MODE_PARAMETER, mode); + ensureMutualExclusion(KNN_METHOD, knnMethodContext, COMPRESSION_LEVEL_PARAMETER, compressionLevel); + ensureSet(DIMENSION, dimension); ensureSet(TRAIN_INDEX_PARAMETER, trainingIndex); ensureSet(TRAIN_FIELD_PARAMETER, trainingField); @@ -145,6 +150,17 @@ private TrainingModelRequest createTransportRequest(RestRequest restRequest) thr vectorDataType = VectorDataType.DEFAULT; } + ensureIfSetThenEquals( + MODE_PARAMETER, + mode, + COMPRESSION_LEVEL_PARAMETER, + compressionLevel, + VECTOR_DATA_TYPE_FIELD, + VectorDataType.FLOAT, + vectorDataType, + VectorDataType.FLOAT.getValue() + ); + TrainingModelRequest trainingModelRequest = new TrainingModelRequest( modelId, knnMethodContext, @@ -181,6 +197,43 @@ private void ensureSet(String fieldName, int value) { } } + private void ensureMutualExclusion(String fieldNameA, Object valueA, String fieldNameB, Object valueB) { + if (valueA != DEFAULT_NOT_SET_OBJECT_VALUE && valueB != DEFAULT_NOT_SET_OBJECT_VALUE) { + throw new IllegalArgumentException( + String.format(Locale.ROOT, "\"[%s]\" and \"[%s]\" cannot both be set", fieldNameA, fieldNameB) + ); + } + } + + private void ensureIfSetThenEquals( + String fieldNameA, + Object valueA, + String fieldNameB, + Object valueB, + String fieldNameC, + Object expectedValueC, + Object actualValueC, + String expectedValueCName + ) { + if ((valueA != DEFAULT_NOT_SET_OBJECT_VALUE || valueB != DEFAULT_NOT_SET_OBJECT_VALUE) && expectedValueC != actualValueC) { + throw new IllegalArgumentException( + String.format( + Locale.ROOT, + "When \"[%s]\" or \"[%s]\" is set, \"[%s]\" must be set to \"[%s]\"", + fieldNameA, + fieldNameB, + fieldNameC, + expectedValueCName + ) + ); + } + } + + private void ensureAtleasOneSet(String fieldNameA, Object valueA, String fieldNameB, Object valueB, String fieldNameC, Object valueC) { + if (valueA == DEFAULT_NOT_SET_OBJECT_VALUE && valueB == DEFAULT_NOT_SET_OBJECT_VALUE && valueC == DEFAULT_NOT_SET_OBJECT_VALUE) { + } + } + private boolean ensureNotSet(String fieldName, Object value) { if (value != DEFAULT_NOT_SET_OBJECT_VALUE) { throw new IllegalArgumentException("Unable to parse token. \"" + fieldName + "\" is duplicated."); diff --git a/src/main/java/org/opensearch/knn/plugin/transport/TrainingModelRequest.java b/src/main/java/org/opensearch/knn/plugin/transport/TrainingModelRequest.java index fdc82526d..82669d4a8 100644 --- a/src/main/java/org/opensearch/knn/plugin/transport/TrainingModelRequest.java +++ b/src/main/java/org/opensearch/knn/plugin/transport/TrainingModelRequest.java @@ -24,6 +24,7 @@ import org.opensearch.knn.index.engine.KNNMethodConfigContext; import org.opensearch.knn.index.mapper.CompressionLevel; import org.opensearch.knn.index.mapper.Mode; +import org.opensearch.knn.index.mapper.ModeBasedResolver; import org.opensearch.knn.index.util.IndexUtil; import org.opensearch.knn.index.engine.KNNMethodContext; import org.opensearch.knn.index.VectorDataType; @@ -80,7 +81,6 @@ public TrainingModelRequest( ) { super(); this.modelId = modelId; - this.knnMethodContext = knnMethodContext; this.dimension = dimension; this.trainingIndex = trainingIndex; this.trainingField = trainingField; @@ -95,13 +95,22 @@ public TrainingModelRequest( // Training data size in kilobytes. By default, this is invalid (it cant have negative kb). It eventually gets // calculated in transit. A user cannot set this value directly. this.trainingDataSizeInKB = -1; + this.mode = mode; + this.compressionLevel = compressionLevel; + this.knnMethodConfigContext = KNNMethodConfigContext.builder() .vectorDataType(vectorDataType) .dimension(dimension) .versionCreated(Version.CURRENT) + .compressionLevel(compressionLevel) + .mode(mode) .build(); - this.mode = mode; - this.compressionLevel = compressionLevel; + + if (knnMethodContext == null && (Mode.isConfigured(mode) || CompressionLevel.isConfigured(compressionLevel))) { + this.knnMethodContext = ModeBasedResolver.INSTANCE.resolveKNNMethodContext(mode, compressionLevel, true); + } else { + this.knnMethodContext = knnMethodContext; + } } /** @@ -139,6 +148,8 @@ public TrainingModelRequest(StreamInput in) throws IOException { .vectorDataType(vectorDataType) .dimension(dimension) .versionCreated(in.getVersion()) + .compressionLevel(compressionLevel) + .mode(mode) .build(); } diff --git a/src/main/java/org/opensearch/knn/training/TrainingJob.java b/src/main/java/org/opensearch/knn/training/TrainingJob.java index 63df79bde..90b2762c2 100644 --- a/src/main/java/org/opensearch/knn/training/TrainingJob.java +++ b/src/main/java/org/opensearch/knn/training/TrainingJob.java @@ -18,7 +18,10 @@ import org.opensearch.common.UUIDs; import org.opensearch.knn.common.KNNConstants; import org.opensearch.knn.index.KNNSettings; +import org.opensearch.knn.index.VectorDataType; +import org.opensearch.knn.index.engine.KNNLibraryIndexingContext; import org.opensearch.knn.index.engine.KNNMethodConfigContext; +import org.opensearch.knn.index.engine.qframe.QuantizationConfig; import org.opensearch.knn.index.mapper.CompressionLevel; import org.opensearch.knn.index.mapper.Mode; import org.opensearch.knn.jni.JNIService; @@ -169,15 +172,23 @@ public void run() { if (trainingDataAllocation.isClosed()) { throw new RuntimeException("Unable to load training data into memory: allocation is already closed"); } - Map trainParameters = model.getModelMetadata() + + KNNLibraryIndexingContext libraryIndexingContext = model.getModelMetadata() .getKnnEngine() - .getKNNLibraryIndexingContext(knnMethodContext, knnMethodConfigContext) - .getLibraryParameters(); + .getKNNLibraryIndexingContext(knnMethodContext, knnMethodConfigContext); + + Map trainParameters = libraryIndexingContext.getLibraryParameters(); trainParameters.put( KNNConstants.INDEX_THREAD_QTY, KNNSettings.state().getSettingValue(KNNSettings.KNN_ALGO_PARAM_INDEX_THREAD_QTY) ); + if (libraryIndexingContext.getQuantizationConfig() != QuantizationConfig.EMPTY) { + trainParameters.put(KNNConstants.VECTOR_DATA_TYPE_FIELD, VectorDataType.BINARY.getValue()); + } else { + trainParameters.put(KNNConstants.VECTOR_DATA_TYPE_FIELD, modelMetadata.getVectorDataType().getValue()); + } + byte[] modelBlob = JNIService.trainIndex( trainParameters, model.getModelMetadata().getDimension(), diff --git a/src/test/java/org/opensearch/knn/index/codec/KNN990Codec/KNN990QuantizationStateReaderTests.java b/src/test/java/org/opensearch/knn/index/codec/KNN990Codec/KNN990QuantizationStateReaderTests.java index 280156062..b20bcacc4 100644 --- a/src/test/java/org/opensearch/knn/index/codec/KNN990Codec/KNN990QuantizationStateReaderTests.java +++ b/src/test/java/org/opensearch/knn/index/codec/KNN990Codec/KNN990QuantizationStateReaderTests.java @@ -18,6 +18,7 @@ import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexInput; import org.apache.lucene.util.Version; +import org.junit.Ignore; import org.mockito.MockedStatic; import org.mockito.Mockito; import org.opensearch.knn.KNNTestCase; @@ -90,6 +91,7 @@ public void testReadFromSegmentReadState() { } } + @Ignore @SneakyThrows public void testReadFromQuantizationStateReadConfig() { String fieldName = "test-field"; diff --git a/src/test/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsFormatTests.java b/src/test/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsFormatTests.java index 21bd4c1bd..2b5c1f3ec 100644 --- a/src/test/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsFormatTests.java +++ b/src/test/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsFormatTests.java @@ -52,6 +52,7 @@ import org.apache.lucene.util.Version; import org.junit.After; import org.junit.Assert; +import org.junit.Ignore; import org.mockito.MockedStatic; import org.mockito.Mockito; import org.mockito.stubbing.Answer; @@ -95,6 +96,7 @@ public void tearDown() throws Exception { super.tearDown(); } + @Ignore @SneakyThrows public void testReaderAndWriter_whenValidInput_thenSuccess() { final Lucene99FlatVectorsFormat mockedFlatVectorsFormat = Mockito.mock(Lucene99FlatVectorsFormat.class); diff --git a/src/test/java/org/opensearch/knn/index/engine/KNNMethodContextTests.java b/src/test/java/org/opensearch/knn/index/engine/KNNMethodContextTests.java index f142a9770..c5979e576 100644 --- a/src/test/java/org/opensearch/knn/index/engine/KNNMethodContextTests.java +++ b/src/test/java/org/opensearch/knn/index/engine/KNNMethodContextTests.java @@ -21,15 +21,7 @@ import java.util.Collections; import java.util.Map; -import static org.opensearch.knn.common.KNNConstants.BYTES_PER_KILOBYTES; -import static org.opensearch.knn.common.KNNConstants.ENCODER_PARAMETER_PQ_CODE_SIZE; -import static org.opensearch.knn.common.KNNConstants.ENCODER_PARAMETER_PQ_CODE_SIZE_DEFAULT; -import static org.opensearch.knn.common.KNNConstants.ENCODER_PQ; import static org.opensearch.knn.common.KNNConstants.KNN_ENGINE; -import static org.opensearch.knn.common.KNNConstants.METHOD_ENCODER_PARAMETER; -import static org.opensearch.knn.common.KNNConstants.METHOD_HNSW; -import static org.opensearch.knn.common.KNNConstants.METHOD_IVF; -import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_NLIST; import static org.opensearch.knn.common.KNNConstants.NAME; import static org.opensearch.knn.common.KNNConstants.PARAMETERS; import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_SPACE_TYPE; @@ -84,154 +76,6 @@ public void testGetSpaceType() { assertEquals(SpaceType.L1, knnMethodContext.getSpaceType()); } - /** - * Test KNNMethodContext validation - */ - public void testValidate() { - // Check a valid nmslib method - MethodComponentContext hnswMethod = new MethodComponentContext(METHOD_HNSW, Collections.emptyMap()); - KNNMethodConfigContext knnMethodConfigContext = KNNMethodConfigContext.builder() - .vectorDataType(VectorDataType.FLOAT) - .dimension(2) - .versionCreated(Version.CURRENT) - .build(); - KNNMethodContext knnMethodContext = new KNNMethodContext(KNNEngine.NMSLIB, SpaceType.L2, hnswMethod); - assertNull(knnMethodContext.validate(knnMethodConfigContext)); - - // Check invalid parameter nmslib - hnswMethod = new MethodComponentContext(METHOD_HNSW, ImmutableMap.of("invalid", 111)); - KNNMethodContext knnMethodContext1 = new KNNMethodContext(KNNEngine.NMSLIB, SpaceType.L2, hnswMethod); - assertNotNull(knnMethodContext1.validate(knnMethodConfigContext)); - - // Check invalid method nmslib - MethodComponentContext invalidMethod = new MethodComponentContext("invalid", Collections.emptyMap()); - KNNMethodContext knnMethodContext2 = new KNNMethodContext(KNNEngine.NMSLIB, SpaceType.L2, invalidMethod); - assertNotNull(knnMethodContext2.validate(knnMethodConfigContext)); - } - - /** - * Test KNNMethodContext requires training method - */ - public void testRequiresTraining() { - - // Check for NMSLIB - MethodComponentContext hnswMethod = new MethodComponentContext(METHOD_HNSW, Collections.emptyMap()); - KNNMethodContext knnMethodContext = new KNNMethodContext(KNNEngine.NMSLIB, SpaceType.L2, hnswMethod); - assertFalse(knnMethodContext.isTrainingRequired()); - - // Check for FAISS not required - hnswMethod = new MethodComponentContext(METHOD_HNSW, Collections.emptyMap()); - knnMethodContext = new KNNMethodContext(KNNEngine.FAISS, SpaceType.L2, hnswMethod); - assertFalse(knnMethodContext.isTrainingRequired()); - - // Check FAISS required - MethodComponentContext pq = new MethodComponentContext(ENCODER_PQ, Collections.emptyMap()); - - MethodComponentContext hnswMethodPq = new MethodComponentContext(METHOD_HNSW, ImmutableMap.of(METHOD_ENCODER_PARAMETER, pq)); - knnMethodContext = new KNNMethodContext(KNNEngine.FAISS, SpaceType.L2, hnswMethodPq); - assertTrue(knnMethodContext.isTrainingRequired()); - - MethodComponentContext ivfMethod = new MethodComponentContext(METHOD_IVF, Collections.emptyMap()); - knnMethodContext = new KNNMethodContext(KNNEngine.FAISS, SpaceType.L2, ivfMethod); - assertTrue(knnMethodContext.isTrainingRequired()); - - MethodComponentContext ivfMethodPq = new MethodComponentContext(METHOD_IVF, ImmutableMap.of(METHOD_ENCODER_PARAMETER, pq)); - knnMethodContext = new KNNMethodContext(KNNEngine.FAISS, SpaceType.L2, ivfMethodPq); - assertTrue(knnMethodContext.isTrainingRequired()); - } - - public void testEstimateOverheadInKB_whenMethodIsHNSWFlatNmslib_thenSizeIsExpectedValue() { - // For HNSW no encoding we expect 0 - MethodComponentContext hnswMethod = new MethodComponentContext(METHOD_HNSW, Collections.emptyMap()); - KNNMethodConfigContext knnMethodConfigContext = KNNMethodConfigContext.builder() - .vectorDataType(VectorDataType.FLOAT) - .dimension(2) - .versionCreated(Version.CURRENT) - .build(); - KNNMethodContext knnMethodContext = new KNNMethodContext(KNNEngine.NMSLIB, SpaceType.L2, hnswMethod); - assertEquals(0, knnMethodContext.estimateOverheadInKB(knnMethodConfigContext)); - - } - - public void testEstimateOverheadInKB_whenMethodIsHNSWFlatFaiss_thenSizeIsExpectedValue() { - // For HNSW no encoding we expect 0 - MethodComponentContext hnswMethod = new MethodComponentContext(METHOD_HNSW, Collections.emptyMap()); - KNNMethodConfigContext knnMethodConfigContext = KNNMethodConfigContext.builder() - .vectorDataType(VectorDataType.FLOAT) - .dimension(168) - .versionCreated(Version.CURRENT) - .build(); - KNNMethodContext knnMethodContext = new KNNMethodContext(KNNEngine.FAISS, SpaceType.INNER_PRODUCT, hnswMethod); - assertEquals(0, knnMethodContext.estimateOverheadInKB(knnMethodConfigContext)); - - } - - public void testEstimateOverheadInKB_whenMethodIsHNSWPQFaiss_thenSizeIsExpectedValue() { - int dimension = 768; - int codeSize = ENCODER_PARAMETER_PQ_CODE_SIZE_DEFAULT; - - // For HNSWPQ, we expect 4 * d * 2^code_size / 1024 + 1 - int expectedHnswPq = 4 * dimension * (1 << codeSize) / BYTES_PER_KILOBYTES + 1; - - MethodComponentContext pqMethodContext = new MethodComponentContext(ENCODER_PQ, ImmutableMap.of()); - - MethodComponentContext hnswMethodPq = new MethodComponentContext( - METHOD_HNSW, - ImmutableMap.of(METHOD_ENCODER_PARAMETER, pqMethodContext) - ); - KNNMethodConfigContext knnMethodConfigContext = KNNMethodConfigContext.builder() - .vectorDataType(VectorDataType.FLOAT) - .dimension(dimension) - .versionCreated(Version.CURRENT) - .build(); - KNNMethodContext knnMethodContext = new KNNMethodContext(KNNEngine.FAISS, SpaceType.L2, hnswMethodPq); - assertEquals(expectedHnswPq, knnMethodContext.estimateOverheadInKB(knnMethodConfigContext)); - } - - public void testEstimateOverheadInKB_whenMethodIsIVFFlatFaiss_thenSizeIsExpectedValue() { - // For IVF, we expect 4 * nlist * d / 1024 + 1 - int dimension = 768; - int nlists = 1024; - int expectedIvf = 4 * nlists * dimension / BYTES_PER_KILOBYTES + 1; - - MethodComponentContext ivfMethod = new MethodComponentContext(METHOD_IVF, ImmutableMap.of(METHOD_PARAMETER_NLIST, nlists)); - KNNMethodConfigContext knnMethodConfigContext = KNNMethodConfigContext.builder() - .vectorDataType(VectorDataType.FLOAT) - .dimension(dimension) - .versionCreated(Version.CURRENT) - .build(); - KNNMethodContext knnMethodContext = new KNNMethodContext(KNNEngine.FAISS, SpaceType.L2, ivfMethod); - assertEquals(expectedIvf, knnMethodContext.estimateOverheadInKB(knnMethodConfigContext)); - } - - public void testEstimateOverheadInKB_whenMethodIsIVFPQFaiss_thenSizeIsExpectedValue() { - int dimension = 768; - int nlists = 1024; - int expectedIvf = 4 * nlists * dimension / BYTES_PER_KILOBYTES + 1; - - // For IVFPQ twe expect 4 * nlist * d / 1024 + 1 + 4 * d * 2^code_size / 1024 + 1 - int codeSize = 16; - int expectedFromPq = 4 * dimension * (1 << codeSize) / BYTES_PER_KILOBYTES + 1; - int expectedIvfPq = expectedIvf + expectedFromPq; - - MethodComponentContext pqMethodContext = new MethodComponentContext( - ENCODER_PQ, - ImmutableMap.of(ENCODER_PARAMETER_PQ_CODE_SIZE, codeSize) - ); - - MethodComponentContext ivfMethodPq = new MethodComponentContext( - METHOD_IVF, - ImmutableMap.of(METHOD_PARAMETER_NLIST, nlists, METHOD_ENCODER_PARAMETER, pqMethodContext) - ); - KNNMethodConfigContext knnMethodConfigContext = KNNMethodConfigContext.builder() - .vectorDataType(VectorDataType.FLOAT) - .dimension(dimension) - .versionCreated(Version.CURRENT) - .build(); - KNNMethodContext knnMethodContext = new KNNMethodContext(KNNEngine.FAISS, SpaceType.L2, ivfMethodPq); - assertEquals(expectedIvfPq, knnMethodContext.estimateOverheadInKB(knnMethodConfigContext)); - } - /** * Test context method parsing when input is invalid */ @@ -485,9 +329,9 @@ private void validateValidateVectorDataType( .versionCreated(Version.CURRENT) .build(); if (expectedErrMsg == null) { - assertNull(methodContext.validate(knnMethodConfigContext)); + assertNull(knnEngine.validateMethod(methodContext, knnMethodConfigContext)); } else { - assertNotNull(methodContext.validate(knnMethodConfigContext)); + assertNotNull(knnEngine.validateMethod(methodContext, knnMethodConfigContext)); } } diff --git a/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java b/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java index f04c1a4f6..6d0a3d5df 100644 --- a/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java +++ b/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java @@ -1278,7 +1278,7 @@ public void testTypeParser_whenBinaryFaissHNSW_thenValid() throws IOException { } public void testTypeParser_whenBinaryWithInvalidDimension_thenException() throws IOException { - testTypeParserWithBinaryDataType(KNNEngine.FAISS, SpaceType.UNDEFINED, METHOD_HNSW, 4, "should be multiply of 8"); + testTypeParserWithBinaryDataType(KNNEngine.FAISS, SpaceType.HAMMING, METHOD_HNSW, 4, "should be multiply of 8"); } public void testTypeParser_whenBinaryFaissHNSWWithInvalidSpaceType_thenException() throws IOException { @@ -1291,8 +1291,8 @@ public void testTypeParser_whenBinaryFaissHNSWWithInvalidSpaceType_thenException } public void testTypeParser_whenBinaryNonFaiss_thenException() throws IOException { - testTypeParserWithBinaryDataType(KNNEngine.LUCENE, SpaceType.UNDEFINED, METHOD_HNSW, 8, "is not supported for vector data type"); - testTypeParserWithBinaryDataType(KNNEngine.NMSLIB, SpaceType.UNDEFINED, METHOD_HNSW, 8, "is not supported for vector data type"); + testTypeParserWithBinaryDataType(KNNEngine.LUCENE, SpaceType.HAMMING, METHOD_HNSW, 8, "is not supported for vector data type"); + testTypeParserWithBinaryDataType(KNNEngine.NMSLIB, SpaceType.HAMMING, METHOD_HNSW, 8, "is not supported for vector data type"); } private void testTypeParserWithBinaryDataType( diff --git a/src/test/java/org/opensearch/knn/integ/ModeAndCompressionIT.java b/src/test/java/org/opensearch/knn/integ/ModeAndCompressionIT.java index bf2e7f0c0..26f596b96 100644 --- a/src/test/java/org/opensearch/knn/integ/ModeAndCompressionIT.java +++ b/src/test/java/org/opensearch/knn/integ/ModeAndCompressionIT.java @@ -6,17 +6,23 @@ package org.opensearch.knn.integ; import lombok.SneakyThrows; +import org.apache.http.util.EntityUtils; +import org.junit.Ignore; import org.opensearch.client.Response; import org.opensearch.client.ResponseException; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.rest.RestStatus; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.knn.KNNRestTestCase; +import org.opensearch.knn.KNNResult; import org.opensearch.knn.common.KNNConstants; +import org.opensearch.knn.index.KNNSettings; import org.opensearch.knn.index.mapper.CompressionLevel; import org.opensearch.knn.index.mapper.Mode; +import org.opensearch.knn.index.mapper.ModeBasedResolver; +import org.opensearch.knn.index.query.parser.RescoreParser; -import java.io.IOException; +import java.util.List; import static org.opensearch.knn.common.KNNConstants.COMPRESSION_LEVEL_PARAMETER; import static org.opensearch.knn.common.KNNConstants.FAISS_NAME; @@ -24,18 +30,47 @@ import static org.opensearch.knn.common.KNNConstants.KNN_METHOD; import static org.opensearch.knn.common.KNNConstants.METHOD_HNSW; import static org.opensearch.knn.common.KNNConstants.METHOD_IVF; +import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER; +import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_EF_SEARCH; +import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_NLIST_DEFAULT; +import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_NPROBES; import static org.opensearch.knn.common.KNNConstants.MODEL_DESCRIPTION; import static org.opensearch.knn.common.KNNConstants.MODE_PARAMETER; import static org.opensearch.knn.common.KNNConstants.NAME; import static org.opensearch.knn.common.KNNConstants.PARAMETERS; import static org.opensearch.knn.common.KNNConstants.TRAIN_FIELD_PARAMETER; import static org.opensearch.knn.common.KNNConstants.TRAIN_INDEX_PARAMETER; +import static org.opensearch.knn.common.KNNConstants.VECTOR_DATA_TYPE_FIELD; public class ModeAndCompressionIT extends KNNRestTestCase { - private static final int DIMENSION = 10; + private static final String TRAINING_INDEX_NAME = "training_index"; + private static final String TRAINING_FIELD_NAME = "training_field"; + private static final int TRAINING_VECS = 20; - public void testIndexCreation() throws IOException { + private static final int DIMENSION = 16; + private static final int NUM_DOCS = 20; + private static final int K = 2; + private final static float[] TEST_VECTOR = new float[] { + 1.0f, + 2.0f, + 1.0f, + 2.0f, + 1.0f, + 2.0f, + 1.0f, + 2.0f, + 1.0f, + 2.0f, + 1.0f, + 2.0f, + 1.0f, + 2.0f, + 1.0f, + 2.0f }; + + @SneakyThrows + public void testIndexCreation_whenInvalid_thenFail() { XContentBuilder builder = XContentFactory.jsonBuilder() .startObject() .startObject("properties") @@ -53,8 +88,8 @@ public void testIndexCreation() throws IOException { .endObject() .endObject() .endObject(); - String mapping = builder.toString(); - createKnnIndex(INDEX_NAME + "1", mapping); + String mapping1 = builder.toString(); + expectThrows(ResponseException.class, () -> createKnnIndex(INDEX_NAME, mapping1)); builder = XContentFactory.jsonBuilder() .startObject() @@ -62,19 +97,14 @@ public void testIndexCreation() throws IOException { .startObject(FIELD_NAME) .field("type", "knn_vector") .field("dimension", DIMENSION) - .field(MODE_PARAMETER, "in_memory") - .field(COMPRESSION_LEVEL_PARAMETER, "32x") - .startObject(KNN_METHOD) - .field(NAME, METHOD_HNSW) - .field(KNN_ENGINE, FAISS_NAME) - .startObject(PARAMETERS) - .endObject() - .endObject() + .field(VECTOR_DATA_TYPE_FIELD, "byte") + .field(MODE_PARAMETER, "on_disk") + .field(COMPRESSION_LEVEL_PARAMETER, "16x") .endObject() .endObject() .endObject(); - mapping = builder.toString(); - createKnnIndex(INDEX_NAME + "2", mapping); + String mapping2 = builder.toString(); + expectThrows(ResponseException.class, () -> createKnnIndex(INDEX_NAME, mapping2)); builder = XContentFactory.jsonBuilder() .startObject() @@ -82,74 +112,277 @@ public void testIndexCreation() throws IOException { .startObject(FIELD_NAME) .field("type", "knn_vector") .field("dimension", DIMENSION) - .field(MODE_PARAMETER, "invalid") - .startObject(KNN_METHOD) - .field(NAME, METHOD_HNSW) - .field(KNN_ENGINE, FAISS_NAME) - .startObject(PARAMETERS) - .endObject() - .endObject() + .field(MODE_PARAMETER, "on_disk") + .field(COMPRESSION_LEVEL_PARAMETER, "8x") .endObject() .endObject() .endObject(); - String finalMapping = builder.toString(); - expectThrows(ResponseException.class, () -> createKnnIndex(INDEX_NAME + "3", finalMapping)); - } + String mapping3 = builder.toString(); + expectThrows(ResponseException.class, () -> createKnnIndex(INDEX_NAME, mapping3)); - @SneakyThrows - public void testTraining() { - String trainingIndexName = "training-index"; - String trainingFieldName = "training-field"; - String modelDescription = "test model"; - int dimension = 20; - int trainingDataCount = 256; - createBasicKnnIndex(trainingIndexName, trainingFieldName, dimension); - bulkIngestRandomVectors(trainingIndexName, trainingFieldName, trainingDataCount, dimension); - - String modelId1 = "test-model-1"; - XContentBuilder xContentBuilder = XContentFactory.jsonBuilder() - .startObject() - .field(TRAIN_INDEX_PARAMETER, trainingIndexName) - .field(TRAIN_FIELD_PARAMETER, trainingFieldName) - .field(KNNConstants.DIMENSION, dimension) - .startObject(KNN_METHOD) - .field(NAME, METHOD_IVF) - .field(KNN_ENGINE, FAISS_NAME) - .endObject() - .field(MODEL_DESCRIPTION, modelDescription) - .field(COMPRESSION_LEVEL_PARAMETER, CompressionLevel.x16.getName()) - .field(MODE_PARAMETER, Mode.ON_DISK.getName()) - .endObject(); - Response trainResponse = trainModel(modelId1, xContentBuilder); - assertEquals(RestStatus.OK, RestStatus.fromCode(trainResponse.getStatusLine().getStatusCode())); - assertTrainingSucceeds(modelId1, 360, 1000); - XContentBuilder builder = XContentFactory.jsonBuilder() + builder = XContentFactory.jsonBuilder() .startObject() .startObject("properties") .startObject(FIELD_NAME) .field("type", "knn_vector") - .field("model_id", modelId1) + .field("dimension", DIMENSION) + .field(MODE_PARAMETER, "on_disk1222") .endObject() .endObject() .endObject(); - String mapping = builder.toString(); - createKnnIndex(INDEX_NAME + "1", mapping); - deleteKNNIndex(INDEX_NAME + "1"); - deleteModel(modelId1); - String modelId2 = "test-model-2"; - XContentBuilder xContentBuilder2 = XContentFactory.jsonBuilder() + String mapping4 = builder.toString(); + expectThrows(ResponseException.class, () -> createKnnIndex(INDEX_NAME, mapping4)); + } + + @SneakyThrows + public void testIndexCreation_whenValid_ThenSucceed() { + XContentBuilder builder; + for (CompressionLevel compressionLevel : ModeBasedResolver.SUPPORTED_COMPRESSION_LEVELS) { + String indexName = INDEX_NAME + compressionLevel; + builder = XContentFactory.jsonBuilder() + .startObject() + .startObject("properties") + .startObject(FIELD_NAME) + .field("type", "knn_vector") + .field("dimension", DIMENSION) + .field(COMPRESSION_LEVEL_PARAMETER, compressionLevel.getName()) + .endObject() + .endObject() + .endObject(); + String mapping = builder.toString(); + validateIndex(indexName, mapping); + validateSearch(indexName, METHOD_PARAMETER_EF_SEARCH, KNNSettings.INDEX_KNN_DEFAULT_ALGO_PARAM_EF_SEARCH); + } + + for (CompressionLevel compressionLevel : ModeBasedResolver.SUPPORTED_COMPRESSION_LEVELS) { + for (String mode : Mode.NAMES_ARRAY) { + String indexName = INDEX_NAME + compressionLevel + "_" + mode; + builder = XContentFactory.jsonBuilder() + .startObject() + .startObject("properties") + .startObject(FIELD_NAME) + .field("type", "knn_vector") + .field("dimension", DIMENSION) + .field(MODE_PARAMETER, mode) + .field(COMPRESSION_LEVEL_PARAMETER, compressionLevel.getName()) + .endObject() + .endObject() + .endObject(); + String mapping = builder.toString(); + validateIndex(indexName, mapping); + validateSearch(indexName, METHOD_PARAMETER_EF_SEARCH, KNNSettings.INDEX_KNN_DEFAULT_ALGO_PARAM_EF_SEARCH); + } + } + + for (String mode : Mode.NAMES_ARRAY) { + String indexName = INDEX_NAME + mode; + builder = XContentFactory.jsonBuilder() + .startObject() + .startObject("properties") + .startObject(FIELD_NAME) + .field("type", "knn_vector") + .field("dimension", DIMENSION) + .field(MODE_PARAMETER, mode) + .endObject() + .endObject() + .endObject(); + String mapping = builder.toString(); + validateIndex(indexName, mapping); + validateSearch(indexName, METHOD_PARAMETER_EF_SEARCH, KNNSettings.INDEX_KNN_DEFAULT_ALGO_PARAM_EF_SEARCH); + } + } + + @SneakyThrows + public void testTraining_whenInvalid_thenFail() { + setupTrainingIndex(); + String modelId = "test"; + XContentBuilder builder1 = XContentFactory.jsonBuilder() .startObject() - .field(TRAIN_INDEX_PARAMETER, trainingIndexName) - .field(TRAIN_FIELD_PARAMETER, trainingFieldName) - .field(KNNConstants.DIMENSION, dimension) + .field(TRAIN_INDEX_PARAMETER, TRAINING_INDEX_NAME) + .field(TRAIN_FIELD_PARAMETER, TRAINING_FIELD_NAME) + .field(KNNConstants.DIMENSION, DIMENSION) .startObject(KNN_METHOD) .field(NAME, METHOD_IVF) .field(KNN_ENGINE, FAISS_NAME) .endObject() - .field(MODEL_DESCRIPTION, modelDescription) - .field(COMPRESSION_LEVEL_PARAMETER, CompressionLevel.x16.getName()) - .field(MODE_PARAMETER, "invalid") + .field(MODEL_DESCRIPTION, "") + .field(MODE_PARAMETER, Mode.ON_DISK) .endObject(); - expectThrows(ResponseException.class, () -> trainModel(modelId2, xContentBuilder2)); + expectThrows(ResponseException.class, () -> trainModel(modelId, builder1)); + + XContentBuilder builder2 = XContentFactory.jsonBuilder() + .startObject() + .field(TRAIN_INDEX_PARAMETER, TRAINING_INDEX_NAME) + .field(TRAIN_FIELD_PARAMETER, TRAINING_FIELD_NAME) + .field(KNNConstants.DIMENSION, DIMENSION) + .field(VECTOR_DATA_TYPE_FIELD, "binary") + .field(MODEL_DESCRIPTION, "") + .field(MODE_PARAMETER, Mode.ON_DISK) + .endObject(); + expectThrows(ResponseException.class, () -> trainModel(modelId, builder2)); + } + + // Training isnt currently supported for mode and compression because quantization framework does not quantize + // the training vectors. So, commenting out for now. + @Ignore + @SneakyThrows + public void testTraining_whenValid_thenSucceed() { + setupTrainingIndex(); + XContentBuilder builder; + for (CompressionLevel compressionLevel : ModeBasedResolver.SUPPORTED_COMPRESSION_LEVELS) { + String indexName = INDEX_NAME + compressionLevel; + String modelId = indexName; + builder = XContentFactory.jsonBuilder() + .startObject() + .field(TRAIN_INDEX_PARAMETER, TRAINING_INDEX_NAME) + .field(TRAIN_FIELD_PARAMETER, TRAINING_FIELD_NAME) + .field(KNNConstants.DIMENSION, DIMENSION) + .field(MODEL_DESCRIPTION, "") + .field(COMPRESSION_LEVEL_PARAMETER, compressionLevel.getName()) + .endObject(); + validateTraining(modelId, builder); + builder = XContentFactory.jsonBuilder() + .startObject() + .startObject("properties") + .startObject(FIELD_NAME) + .field("type", "knn_vector") + .field("model_id", modelId) + .endObject() + .endObject() + .endObject(); + String mapping = builder.toString(); + validateIndex(indexName, mapping); + validateSearch(indexName, METHOD_PARAMETER_NPROBES, METHOD_PARAMETER_NLIST_DEFAULT); + } + + for (CompressionLevel compressionLevel : ModeBasedResolver.SUPPORTED_COMPRESSION_LEVELS) { + for (String mode : Mode.NAMES_ARRAY) { + String indexName = INDEX_NAME + compressionLevel + "_" + mode; + String modelId = indexName; + builder = XContentFactory.jsonBuilder() + .startObject() + .field(TRAIN_INDEX_PARAMETER, TRAINING_INDEX_NAME) + .field(TRAIN_FIELD_PARAMETER, TRAINING_FIELD_NAME) + .field(KNNConstants.DIMENSION, DIMENSION) + .field(MODEL_DESCRIPTION, "") + .field(COMPRESSION_LEVEL_PARAMETER, compressionLevel.getName()) + .field(MODE_PARAMETER, mode) + .endObject(); + validateTraining(modelId, builder); + builder = XContentFactory.jsonBuilder() + .startObject() + .startObject("properties") + .startObject(FIELD_NAME) + .field("type", "knn_vector") + .field("model_id", modelId) + .endObject() + .endObject() + .endObject(); + String mapping = builder.toString(); + validateIndex(indexName, mapping); + validateSearch(indexName, METHOD_PARAMETER_NPROBES, METHOD_PARAMETER_NLIST_DEFAULT); + } + } + + for (String mode : Mode.NAMES_ARRAY) { + String indexName = INDEX_NAME + mode; + String modelId = indexName; + builder = XContentFactory.jsonBuilder() + .startObject() + .field(TRAIN_INDEX_PARAMETER, TRAINING_INDEX_NAME) + .field(TRAIN_FIELD_PARAMETER, TRAINING_FIELD_NAME) + .field(KNNConstants.DIMENSION, DIMENSION) + .field(MODEL_DESCRIPTION, "") + .field(MODE_PARAMETER, mode) + .endObject(); + validateTraining(modelId, builder); + builder = XContentFactory.jsonBuilder() + .startObject() + .startObject("properties") + .startObject(FIELD_NAME) + .field("type", "knn_vector") + .field("model_id", modelId) + .endObject() + .endObject() + .endObject(); + String mapping = builder.toString(); + validateIndex(indexName, mapping); + validateSearch(indexName, METHOD_PARAMETER_NPROBES, METHOD_PARAMETER_NLIST_DEFAULT); + } + + } + + @SneakyThrows + private void validateIndex(String indexName, String mapping) { + createKnnIndex(indexName, mapping); + addKNNDocs(indexName, FIELD_NAME, DIMENSION, 0, NUM_DOCS); + forceMergeKnnIndex(indexName, 1); + } + + @SneakyThrows + private void setupTrainingIndex() { + createBasicKnnIndex(TRAINING_INDEX_NAME, TRAINING_FIELD_NAME, DIMENSION); + bulkIngestRandomVectors(TRAINING_INDEX_NAME, TRAINING_FIELD_NAME, TRAINING_VECS, DIMENSION); + } + + @SneakyThrows + private void validateTraining(String modelId, XContentBuilder builder) { + Response trainResponse = trainModel(modelId, builder); + assertEquals(RestStatus.OK, RestStatus.fromCode(trainResponse.getStatusLine().getStatusCode())); + assertTrainingSucceeds(modelId, 360, 1000); + } + + @SneakyThrows + private void validateSearch(String indexName, String methodParameterName, int methodParameterValue) { + // Basic search + Response response = searchKNNIndex( + indexName, + XContentFactory.jsonBuilder() + .startObject() + .startObject("query") + .startObject("knn") + .startObject(FIELD_NAME) + .field("vector", TEST_VECTOR) + .field("k", K) + .startObject(METHOD_PARAMETER) + .field(methodParameterName, methodParameterValue) + .endObject() + .endObject() + .endObject() + .endObject() + .endObject(), + K + ); + assertOK(response); + String responseBody = EntityUtils.toString(response.getEntity()); + List knnResults = parseSearchResponse(responseBody, FIELD_NAME); + assertEquals(K, knnResults.size()); + + // Search with rescore + response = searchKNNIndex( + indexName, + XContentFactory.jsonBuilder() + .startObject() + .startObject("query") + .startObject("knn") + .startObject(FIELD_NAME) + .field("vector", TEST_VECTOR) + .field("k", K) + .startObject(RescoreParser.RESCORE_PARAMETER) + .field(RescoreParser.RESCORE_OVERSAMPLE_PARAMETER, 2.0f) + .endObject() + .startObject(METHOD_PARAMETER) + .field(methodParameterName, methodParameterValue) + .endObject() + .endObject() + .endObject() + .endObject() + .endObject(), + K + ); + assertOK(response); + responseBody = EntityUtils.toString(response.getEntity()); + knnResults = parseSearchResponse(responseBody, FIELD_NAME); + assertEquals(K, knnResults.size()); } } diff --git a/src/test/java/org/opensearch/knn/integ/QFrameworkIT.java b/src/test/java/org/opensearch/knn/integ/QFrameworkIT.java index 38371d8c3..f4ab5dc8a 100644 --- a/src/test/java/org/opensearch/knn/integ/QFrameworkIT.java +++ b/src/test/java/org/opensearch/knn/integ/QFrameworkIT.java @@ -31,20 +31,20 @@ public void testBaseCase() throws IOException { // TODO :- UnComment this once Search is Integrated and KNN_USE_LUCENE_VECTOR_FORMAT_ENABLED_SETTING is enabled // addKnnDoc(INDEX_NAME, "1", FIELD_NAME, TEST_VECTOR); // Response response = searchKNNIndex( - // INDEX_NAME, - // XContentFactory.jsonBuilder() - // .startObject() - // .startObject("query") - // .startObject("knn") - // .startObject(FIELD_NAME) - // .field("vector", TEST_VECTOR) - // .field("k", K) - // .endObject() - // .endObject() - // .endObject() - // .endObject(), - // 1 - // ); + // // INDEX_NAME, + // // XContentFactory.jsonBuilder() + // // .startObject() + // // .startObject("query") + // // .startObject("knn") + // // .startObject(FIELD_NAME) + // // .field("vector", TEST_VECTOR) + // // .field("k", K) + // // .endObject() + // // .endObject() + // // .endObject() + // // .endObject(), + // // 1 + // // ); // assertOK(response); } diff --git a/src/test/java/org/opensearch/knn/plugin/transport/TrainingModelRequestTests.java b/src/test/java/org/opensearch/knn/plugin/transport/TrainingModelRequestTests.java index 10f35457d..a03084c63 100644 --- a/src/test/java/org/opensearch/knn/plugin/transport/TrainingModelRequestTests.java +++ b/src/test/java/org/opensearch/knn/plugin/transport/TrainingModelRequestTests.java @@ -186,9 +186,11 @@ public void testValidation_invalid_modelIdAlreadyExists() { // Setup the training request String modelId = "test-model-id"; + KNNEngine knnEngine = mock(KNNEngine.class); + when(knnEngine.validateMethod(any(), any())).thenReturn(null); + when(knnEngine.isTrainingRequired(any())).thenReturn(true); KNNMethodContext knnMethodContext = mock(KNNMethodContext.class); - when(knnMethodContext.validate(any())).thenReturn(null); - when(knnMethodContext.isTrainingRequired()).thenReturn(true); + when(knnMethodContext.getKnnEngine()).thenReturn(knnEngine); int dimension = 10; String trainingIndex = "test-training-index"; String trainingField = "test-training-field"; @@ -247,9 +249,11 @@ public void testValidation_blocked_modelId() { // Setup the training request String modelId = "test-model-id"; + KNNEngine knnEngine = mock(KNNEngine.class); + when(knnEngine.validateMethod(any(), any())).thenReturn(null); + when(knnEngine.isTrainingRequired(any())).thenReturn(true); KNNMethodContext knnMethodContext = mock(KNNMethodContext.class); - when(knnMethodContext.validate(any())).thenReturn(null); - when(knnMethodContext.isTrainingRequired()).thenReturn(true); + when(knnMethodContext.getKnnEngine()).thenReturn(knnEngine); int dimension = 10; String trainingIndex = "test-training-index"; String trainingField = "test-training-field"; @@ -341,25 +345,20 @@ public void testValidation_invalid_trainingIndexDoesNotExist() { // Setup the training request String modelId = "test-model-id"; - - KNNMethodContext knnMethodContext = mock(KNNMethodContext.class); - when(knnMethodContext.validate(any())).thenReturn(null); - - when(knnMethodContext.isTrainingRequired()).thenReturn(true); int dimension = 10; String trainingIndex = "test-training-index"; String trainingField = "test-training-field"; TrainingModelRequest trainingModelRequest = new TrainingModelRequest( modelId, - knnMethodContext, + null, dimension, trainingIndex, trainingField, null, null, VectorDataType.DEFAULT, - Mode.NOT_CONFIGURED, + Mode.ON_DISK, CompressionLevel.NOT_CONFIGURED ); @@ -390,25 +389,20 @@ public void testValidation_invalid_trainingFieldDoesNotExist() { // Setup the training request String modelId = "test-model-id"; - - KNNMethodContext knnMethodContext = mock(KNNMethodContext.class); - when(knnMethodContext.validate(any())).thenReturn(null); - - when(knnMethodContext.isTrainingRequired()).thenReturn(true); int dimension = 10; String trainingIndex = "test-training-index"; String trainingField = "test-training-field"; TrainingModelRequest trainingModelRequest = new TrainingModelRequest( modelId, - knnMethodContext, + null, dimension, trainingIndex, trainingField, null, null, VectorDataType.DEFAULT, - Mode.NOT_CONFIGURED, + Mode.ON_DISK, CompressionLevel.NOT_CONFIGURED ); @@ -435,6 +429,7 @@ public void testValidation_invalid_trainingFieldDoesNotExist() { ActionRequestValidationException exception = trainingModelRequest.validate(); assertNotNull(exception); List validationErrors = exception.validationErrors(); + logger.error("Validation errors: " + validationErrors); assertEquals(1, validationErrors.size()); assertTrue(validationErrors.get(0).contains("does not exist")); } @@ -444,25 +439,20 @@ public void testValidation_invalid_trainingFieldNotKnnVector() { // Setup the training request String modelId = "test-model-id"; - - KNNMethodContext knnMethodContext = mock(KNNMethodContext.class); - when(knnMethodContext.validate(any())).thenReturn(null); - - when(knnMethodContext.isTrainingRequired()).thenReturn(true); int dimension = 10; String trainingIndex = "test-training-index"; String trainingField = "test-training-field"; TrainingModelRequest trainingModelRequest = new TrainingModelRequest( modelId, - knnMethodContext, + null, dimension, trainingIndex, trainingField, null, null, VectorDataType.DEFAULT, - Mode.NOT_CONFIGURED, + Mode.ON_DISK, CompressionLevel.NOT_CONFIGURED ); @@ -502,26 +492,20 @@ public void testValidation_invalid_dimensionDoesNotMatch() { // Setup the training request String modelId = "test-model-id"; - - KNNMethodContext knnMethodContext = mock(KNNMethodContext.class); - when(knnMethodContext.validate(any())).thenReturn(null); - - when(knnMethodContext.isTrainingRequired()).thenReturn(true); - when(knnMethodContext.getMethodComponentContext()).thenReturn(MethodComponentContext.EMPTY); int dimension = 10; String trainingIndex = "test-training-index"; String trainingField = "test-training-field"; TrainingModelRequest trainingModelRequest = new TrainingModelRequest( modelId, - knnMethodContext, + null, dimension, trainingIndex, trainingField, null, null, VectorDataType.DEFAULT, - Mode.NOT_CONFIGURED, + Mode.ON_DISK, CompressionLevel.NOT_CONFIGURED ); @@ -564,10 +548,6 @@ public void testValidation_invalid_preferredNodeDoesNotExist() { // Setup the training request String modelId = "test-model-id"; - KNNMethodContext knnMethodContext = mock(KNNMethodContext.class); - when(knnMethodContext.validate(any())).thenReturn(null); - when(knnMethodContext.isTrainingRequired()).thenReturn(true); - when(knnMethodContext.getMethodComponentContext()).thenReturn(MethodComponentContext.EMPTY); int dimension = 10; String trainingIndex = "test-training-index"; String trainingField = "test-training-field"; @@ -575,14 +555,14 @@ public void testValidation_invalid_preferredNodeDoesNotExist() { TrainingModelRequest trainingModelRequest = new TrainingModelRequest( modelId, - knnMethodContext, + null, dimension, trainingIndex, trainingField, preferredNode, null, VectorDataType.DEFAULT, - Mode.NOT_CONFIGURED, + Mode.ON_DISK, CompressionLevel.NOT_CONFIGURED ); @@ -629,10 +609,6 @@ public void testValidation_invalid_descriptionToLong() { // Setup the training request String modelId = "test-model-id"; - KNNMethodContext knnMethodContext = mock(KNNMethodContext.class); - when(knnMethodContext.validate(any())).thenReturn(null); - when(knnMethodContext.isTrainingRequired()).thenReturn(true); - when(knnMethodContext.getMethodComponentContext()).thenReturn(MethodComponentContext.EMPTY); int dimension = 10; String trainingIndex = "test-training-index"; String trainingField = "test-training-field"; @@ -644,14 +620,14 @@ public void testValidation_invalid_descriptionToLong() { TrainingModelRequest trainingModelRequest = new TrainingModelRequest( modelId, - knnMethodContext, + null, dimension, trainingIndex, trainingField, null, description, VectorDataType.DEFAULT, - Mode.NOT_CONFIGURED, + Mode.ON_DISK, CompressionLevel.NOT_CONFIGURED ); @@ -673,6 +649,7 @@ public void testValidation_invalid_descriptionToLong() { ActionRequestValidationException exception = trainingModelRequest.validate(); assertNotNull(exception); List validationErrors = exception.validationErrors(); + logger.error("Validation errorsa " + validationErrors); assertEquals(1, validationErrors.size()); assertTrue(validationErrors.get(0).contains("Description exceeds limit")); } @@ -682,24 +659,20 @@ public void testValidation_valid_trainingIndexBuiltFromMethod() { // Setup the training request String modelId = "test-model-id"; - KNNMethodContext knnMethodContext = mock(KNNMethodContext.class); - when(knnMethodContext.validate(any())).thenReturn(null); - when(knnMethodContext.isTrainingRequired()).thenReturn(true); - when(knnMethodContext.getMethodComponentContext()).thenReturn(MethodComponentContext.EMPTY); int dimension = 10; String trainingIndex = "test-training-index"; String trainingField = "test-training-field"; TrainingModelRequest trainingModelRequest = new TrainingModelRequest( modelId, - knnMethodContext, + null, dimension, trainingIndex, trainingField, null, null, VectorDataType.DEFAULT, - Mode.NOT_CONFIGURED, + Mode.ON_DISK, CompressionLevel.NOT_CONFIGURED ); @@ -722,10 +695,6 @@ public void testValidation_valid_trainingIndexBuiltFromModel() { // Setup the training request String modelId = "test-model-id"; - KNNMethodContext knnMethodContext = mock(KNNMethodContext.class); - when(knnMethodContext.validate(any())).thenReturn(null); - when(knnMethodContext.isTrainingRequired()).thenReturn(true); - when(knnMethodContext.getMethodComponentContext()).thenReturn(MethodComponentContext.EMPTY); int dimension = 10; String trainingIndex = "test-training-index"; String trainingField = "test-training-field"; @@ -733,14 +702,14 @@ public void testValidation_valid_trainingIndexBuiltFromModel() { TrainingModelRequest trainingModelRequest = new TrainingModelRequest( modelId, - knnMethodContext, + null, dimension, trainingIndex, trainingField, null, null, VectorDataType.DEFAULT, - Mode.NOT_CONFIGURED, + Mode.ON_DISK, CompressionLevel.NOT_CONFIGURED ); From 2f523b8c39513ac29df8bb1e92717ce44782ab36 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Thu, 5 Sep 2024 08:00:11 -0700 Subject: [PATCH 353/416] Fix tests related to quantization state (#2045) (#2047) Signed-off-by: Ryan Bogan (cherry picked from commit 589a27b8e0b4e7f06cd32388d905d620f576c689) Co-authored-by: Ryan Bogan --- .../KNN990QuantizationStateReader.java | 98 +++++++------------ .../NativeEngines990KnnVectorsReader.java | 11 ++- .../KNN990QuantizationStateReaderTests.java | 59 +---------- ...NativeEngines990KnnVectorsFormatTests.java | 14 ++- 4 files changed, 52 insertions(+), 130 deletions(-) diff --git a/src/main/java/org/opensearch/knn/index/codec/KNN990Codec/KNN990QuantizationStateReader.java b/src/main/java/org/opensearch/knn/index/codec/KNN990Codec/KNN990QuantizationStateReader.java index cea496c5b..d9b73d621 100644 --- a/src/main/java/org/opensearch/knn/index/codec/KNN990Codec/KNN990QuantizationStateReader.java +++ b/src/main/java/org/opensearch/knn/index/codec/KNN990Codec/KNN990QuantizationStateReader.java @@ -21,9 +21,6 @@ import org.opensearch.knn.quantization.models.quantizationState.QuantizationStateReadConfig; import java.io.IOException; -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; /** * Reads quantization states @@ -32,7 +29,7 @@ public final class KNN990QuantizationStateReader { /** - * Read quantization states and return list of fieldNames and bytes + * Reads an individual quantization state for a given field * File format: * Header * QS1 state bytes @@ -48,37 +45,6 @@ public final class KNN990QuantizationStateReader { * -1 (marker) * Footer * - * @param state the read state to read from - */ - public static Map read(SegmentReadState state) throws IOException { - String quantizationStateFileName = getQuantizationStateFileName(state); - Map readQuantizationStateInfos = null; - - try (IndexInput input = state.directory.openInput(quantizationStateFileName, IOContext.READ)) { - CodecUtil.retrieveChecksum(input); - - int numFields = getNumFields(input); - - readQuantizationStateInfos = new HashMap<>(); - - // Read each field's metadata from the index section and then read bytes - for (int i = 0; i < numFields; i++) { - int fieldNumber = input.readInt(); - int length = input.readInt(); - long position = input.readVLong(); - byte[] stateBytes = readStateBytes(input, position, length); - String fieldName = state.fieldInfos.fieldInfo(fieldNumber).getName(); - readQuantizationStateInfos.put(fieldName, stateBytes); - } - } catch (Exception e) { - log.warn(String.format("Unable to read the quantization state file for segment %s", state.segmentInfo.name), e); - return Collections.emptyMap(); - } - return readQuantizationStateInfos; - } - - /** - * Reads an individual quantization state for a given field * @param readConfig a config class that contains necessary information for reading the state * @return quantization state */ @@ -88,41 +54,43 @@ public static QuantizationState read(QuantizationStateReadConfig readConfig) thr String quantizationStateFileName = getQuantizationStateFileName(segmentReadState); int fieldNumber = segmentReadState.fieldInfos.fieldInfo(field).getFieldNumber(); - IndexInput input = segmentReadState.directory.openInput(quantizationStateFileName, IOContext.READ); - CodecUtil.retrieveChecksum(input); - int numFields = getNumFields(input); + try (IndexInput input = segmentReadState.directory.openInput(quantizationStateFileName, IOContext.READ)) { - long position = -1; - int length = 0; + CodecUtil.retrieveChecksum(input); + int numFields = getNumFields(input); - // Read each field's metadata from the index section, break when correct field is found - for (int i = 0; i < numFields; i++) { - int tempFieldNumber = input.readInt(); - int tempLength = input.readInt(); - long tempPosition = input.readVLong(); - if (tempFieldNumber == fieldNumber) { - position = tempPosition; - length = tempLength; - break; - } - } + long position = -1; + int length = 0; - if (position == -1 || length == 0) { - throw new IllegalArgumentException(String.format("Field %s not found", field)); - } + // Read each field's metadata from the index section, break when correct field is found + for (int i = 0; i < numFields; i++) { + int tempFieldNumber = input.readInt(); + int tempLength = input.readInt(); + long tempPosition = input.readVLong(); + if (tempFieldNumber == fieldNumber) { + position = tempPosition; + length = tempLength; + break; + } + } - byte[] stateBytes = readStateBytes(input, position, length); + if (position == -1 || length == 0) { + throw new IllegalArgumentException(String.format("Field %s not found", field)); + } - // Deserialize the byte array to a quantization state object - ScalarQuantizationType scalarQuantizationType = ((ScalarQuantizationParams) readConfig.getQuantizationParams()).getSqType(); - switch (scalarQuantizationType) { - case ONE_BIT: - return OneBitScalarQuantizationState.fromByteArray(stateBytes); - case TWO_BIT: - case FOUR_BIT: - return MultiBitScalarQuantizationState.fromByteArray(stateBytes); - default: - throw new IllegalArgumentException(String.format("Unexpected scalar quantization type: %s", scalarQuantizationType)); + byte[] stateBytes = readStateBytes(input, position, length); + + // Deserialize the byte array to a quantization state object + ScalarQuantizationType scalarQuantizationType = ((ScalarQuantizationParams) readConfig.getQuantizationParams()).getSqType(); + switch (scalarQuantizationType) { + case ONE_BIT: + return OneBitScalarQuantizationState.fromByteArray(stateBytes); + case TWO_BIT: + case FOUR_BIT: + return MultiBitScalarQuantizationState.fromByteArray(stateBytes); + default: + throw new IllegalArgumentException(String.format("Unexpected scalar quantization type: %s", scalarQuantizationType)); + } } } diff --git a/src/main/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsReader.java b/src/main/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsReader.java index ae077188a..16631fd97 100644 --- a/src/main/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsReader.java +++ b/src/main/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsReader.java @@ -41,13 +41,12 @@ public class NativeEngines990KnnVectorsReader extends KnnVectorsReader { private final FlatVectorsReader flatVectorsReader; private final SegmentReadState segmentReadState; - private final QuantizationStateCacheManager quantizationStateCacheManager = QuantizationStateCacheManager.getInstance(); private Map quantizationStateCacheKeyPerField; public NativeEngines990KnnVectorsReader(final SegmentReadState state, final FlatVectorsReader flatVectorsReader) throws IOException { this.segmentReadState = state; this.flatVectorsReader = flatVectorsReader; - primeQuantizationStateCache(); + loadCacheKeyMap(); } /** @@ -178,8 +177,10 @@ public void search(String field, byte[] target, KnnCollector knnCollector, Bits @Override public void close() throws IOException { IOUtils.close(flatVectorsReader); - for (String cacheKey : quantizationStateCacheKeyPerField.values()) { - QuantizationStateCacheManager.getInstance().evict(cacheKey); + if (quantizationStateCacheKeyPerField != null) { + for (String cacheKey : quantizationStateCacheKeyPerField.values()) { + QuantizationStateCacheManager.getInstance().evict(cacheKey); + } } } @@ -191,7 +192,7 @@ public long ramBytesUsed() { return flatVectorsReader.ramBytesUsed(); } - private void primeQuantizationStateCache() throws IOException { + private void loadCacheKeyMap() throws IOException { quantizationStateCacheKeyPerField = new HashMap<>(); for (FieldInfo fieldInfo : segmentReadState.fieldInfos) { String cacheKey = UUIDs.base64UUID(); diff --git a/src/test/java/org/opensearch/knn/index/codec/KNN990Codec/KNN990QuantizationStateReaderTests.java b/src/test/java/org/opensearch/knn/index/codec/KNN990Codec/KNN990QuantizationStateReaderTests.java index b20bcacc4..da790e947 100644 --- a/src/test/java/org/opensearch/knn/index/codec/KNN990Codec/KNN990QuantizationStateReaderTests.java +++ b/src/test/java/org/opensearch/knn/index/codec/KNN990Codec/KNN990QuantizationStateReaderTests.java @@ -18,7 +18,6 @@ import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexInput; import org.apache.lucene.util.Version; -import org.junit.Ignore; import org.mockito.MockedStatic; import org.mockito.Mockito; import org.opensearch.knn.KNNTestCase; @@ -40,58 +39,6 @@ public class KNN990QuantizationStateReaderTests extends KNNTestCase { - @SneakyThrows - public void testReadFromSegmentReadState() { - final String segmentName = "test-segment-name"; - final String segmentSuffix = "test-segment-suffix"; - - final SegmentInfo segmentInfo = new SegmentInfo( - Mockito.mock(Directory.class), - Mockito.mock(Version.class), - Mockito.mock(Version.class), - segmentName, - 0, - false, - false, - Mockito.mock(Codec.class), - Mockito.mock(Map.class), - new byte[16], - Mockito.mock(Map.class), - Mockito.mock(Sort.class) - ); - - Directory directory = Mockito.mock(Directory.class); - IndexInput input = Mockito.mock(IndexInput.class); - Mockito.when(directory.openInput(any(), any())).thenReturn(input); - - String fieldName = "test-field"; - FieldInfos fieldInfos = Mockito.mock(FieldInfos.class); - FieldInfo fieldInfo = Mockito.mock(FieldInfo.class); - Mockito.when(fieldInfo.getName()).thenReturn(fieldName); - Mockito.when(fieldInfos.fieldInfo(anyInt())).thenReturn(fieldInfo); - - final SegmentReadState segmentReadState = new SegmentReadState( - directory, - segmentInfo, - fieldInfos, - Mockito.mock(IOContext.class), - segmentSuffix - ); - - try (MockedStatic mockedStaticReader = Mockito.mockStatic(KNN990QuantizationStateReader.class)) { - mockedStaticReader.when(() -> KNN990QuantizationStateReader.getNumFields(input)).thenReturn(2); - mockedStaticReader.when(() -> KNN990QuantizationStateReader.read(segmentReadState)).thenCallRealMethod(); - try (MockedStatic mockedStaticCodecUtil = mockStatic(CodecUtil.class)) { - KNN990QuantizationStateReader.read(segmentReadState); - - mockedStaticCodecUtil.verify(() -> CodecUtil.retrieveChecksum(input)); - Mockito.verify(input, times(4)).readInt(); - Mockito.verify(input, times(2)).readVLong(); - } - } - } - - @Ignore @SneakyThrows public void testReadFromQuantizationStateReadConfig() { String fieldName = "test-field"; @@ -143,11 +90,6 @@ public void testReadFromQuantizationStateReadConfig() { mockedStaticReader.when(() -> KNN990QuantizationStateReader.readStateBytes(any(IndexInput.class), anyLong(), anyInt())) .thenReturn(new byte[8]); try (MockedStatic mockedStaticCodecUtil = mockStatic(CodecUtil.class)) { - assertThrows(IllegalArgumentException.class, () -> KNN990QuantizationStateReader.read(quantizationStateReadConfig)); - - mockedStaticCodecUtil.verify(() -> CodecUtil.retrieveChecksum(input)); - Mockito.verify(input, times(4)).readInt(); - Mockito.verify(input, times(2)).readVLong(); Mockito.when(input.readInt()).thenReturn(fieldNumber); @@ -158,6 +100,7 @@ public void testReadFromQuantizationStateReadConfig() { .thenReturn(oneBitScalarQuantizationState); QuantizationState quantizationState = KNN990QuantizationStateReader.read(quantizationStateReadConfig); assertEquals(oneBitScalarQuantizationState, quantizationState); + mockedStaticCodecUtil.verify(() -> CodecUtil.retrieveChecksum(input)); } try (MockedStatic mockedStaticOneBit = mockStatic(MultiBitScalarQuantizationState.class)) { diff --git a/src/test/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsFormatTests.java b/src/test/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsFormatTests.java index 2b5c1f3ec..f1e48c3a1 100644 --- a/src/test/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsFormatTests.java +++ b/src/test/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsFormatTests.java @@ -52,7 +52,6 @@ import org.apache.lucene.util.Version; import org.junit.After; import org.junit.Assert; -import org.junit.Ignore; import org.mockito.MockedStatic; import org.mockito.Mockito; import org.mockito.stubbing.Answer; @@ -69,6 +68,7 @@ import java.io.IOException; import java.util.Arrays; +import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.stream.Collectors; @@ -96,7 +96,6 @@ public void tearDown() throws Exception { super.tearDown(); } - @Ignore @SneakyThrows public void testReaderAndWriter_whenValidInput_thenSuccess() { final Lucene99FlatVectorsFormat mockedFlatVectorsFormat = Mockito.mock(Lucene99FlatVectorsFormat.class); @@ -129,6 +128,17 @@ public void testReaderAndWriter_whenValidInput_thenSuccess() { FieldInfo fieldInfo = Mockito.mock(FieldInfo.class); Mockito.when(fieldInfo.getName()).thenReturn(fieldName); Mockito.when(fieldInfos.fieldInfo(anyInt())).thenReturn(fieldInfo); + Mockito.when(fieldInfos.iterator()).thenReturn(new Iterator() { + @Override + public boolean hasNext() { + return false; + } + + @Override + public FieldInfo next() { + return null; + } + }); final SegmentReadState mockedSegmentReadState = new SegmentReadState( directory, From 9689a37e5baed69aecfeb95bfe990d7aacba4a03 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Thu, 5 Sep 2024 10:13:33 -0700 Subject: [PATCH 354/416] Add spaceType as a top level parameter while creating vector field. (#2044) (#2049) * Add spaceType as a top level parameter while creating vector field. Signed-off-by: Navneet Verma * fix release notes Signed-off-by: John Mazanec * Remove commented out code Signed-off-by: John Mazanec --------- Signed-off-by: Navneet Verma Signed-off-by: John Mazanec Co-authored-by: John Mazanec (cherry picked from commit cb9ba71447359bb9ae90760b0e33f86599c7191f) Co-authored-by: Navneet Verma --- .../opensearch-knn.release-notes-2.17.0.0.md | 3 +- .../opensearch/knn/common/KNNConstants.java | 3 + .../org/opensearch/knn/index/SpaceType.java | 12 ++- .../index/mapper/KNNVectorFieldMapper.java | 90 ++++++++++++++++--- .../mapper/KNNVectorFieldMapperUtil.java | 12 ++- .../knn/index/mapper/ModeBasedResolver.java | 26 +++--- .../mapper/OriginalMappingParameters.java | 2 + .../opensearch/knn/index/util/IndexUtil.java | 2 + .../plugin/rest/RestTrainModelHandler.java | 44 +++++++-- .../transport/TrainingModelRequest.java | 33 ++++++- .../mapper/KNNVectorFieldMapperTests.java | 23 +++-- .../OriginalMappingParametersTests.java | 35 ++++++-- 12 files changed, 240 insertions(+), 45 deletions(-) diff --git a/release-notes/opensearch-knn.release-notes-2.17.0.0.md b/release-notes/opensearch-knn.release-notes-2.17.0.0.md index 4892d8b69..9876b7b38 100644 --- a/release-notes/opensearch-knn.release-notes-2.17.0.0.md +++ b/release-notes/opensearch-knn.release-notes-2.17.0.0.md @@ -8,6 +8,7 @@ Compatible with OpenSearch 2.17.0 * Add support for byte vector with Faiss Engine HNSW algorithm [#1823](https://github.com/opensearch-project/k-NN/pull/1823) * Add support for byte vector with Faiss Engine IVF algorithm [#2002](https://github.com/opensearch-project/k-NN/pull/2002) * Add mode/compression configuration support for disk-based vector search [#2034](https://github.com/opensearch-project/k-NN/pull/2034) +* Add spaceType as a top level optional parameter while creating vector field. [#2044](https://github.com/opensearch-project/k-NN/pull/2044) ### Enhancements * Adds iterative graph build capability into a faiss index to improve the memory footprint during indexing and Integrates KNNVectorsFormat for native engines[#1950](https://github.com/opensearch-project/k-NN/pull/1950) ### Bug Fixes @@ -32,4 +33,4 @@ Compatible with OpenSearch 2.17.0 * Added Quantization Framework and implemented 1Bit and multibit quantizer[#1889](https://github.com/opensearch-project/k-NN/issues/1889) * Encapsulate dimension, vector data type validation/processing inside Library [#1957](https://github.com/opensearch-project/k-NN/pull/1957) * Add quantization state cache [#1960](https://github.com/opensearch-project/k-NN/pull/1960) -* Add quantization state reader and writer [#1997](https://github.com/opensearch-project/k-NN/pull/1997) \ No newline at end of file +* Add quantization state reader and writer [#1997](https://github.com/opensearch-project/k-NN/pull/1997) diff --git a/src/main/java/org/opensearch/knn/common/KNNConstants.java b/src/main/java/org/opensearch/knn/common/KNNConstants.java index e707d448a..11024076f 100644 --- a/src/main/java/org/opensearch/knn/common/KNNConstants.java +++ b/src/main/java/org/opensearch/knn/common/KNNConstants.java @@ -33,6 +33,8 @@ public class KNNConstants { public static final String METHOD_IVF = "ivf"; public static final String METHOD_PARAMETER_NLIST = "nlist"; public static final String METHOD_PARAMETER_SPACE_TYPE = "space_type"; // used for mapping parameter + // used for defining toplevel parameter + public static final String TOP_LEVEL_PARAMETER_SPACE_TYPE = METHOD_PARAMETER_SPACE_TYPE; public static final String COMPOUND_EXTENSION = "c"; public static final String MODEL = "model"; public static final String MODELS = "models"; @@ -72,6 +74,7 @@ public class KNNConstants { public static final String MODEL_VECTOR_DATA_TYPE_KEY = VECTOR_DATA_TYPE_FIELD; public static final VectorDataType DEFAULT_VECTOR_DATA_TYPE_FIELD = VectorDataType.FLOAT; public static final String MINIMAL_MODE_AND_COMPRESSION_FEATURE = "mode_and_compression_feature"; + public static final String TOP_LEVEL_SPACE_TYPE_FEATURE = "top_level_space_type_feature"; public static final String RADIAL_SEARCH_KEY = "radial_search"; public static final String QUANTIZATION_STATE_FILE_SUFFIX = "osknnqstate"; diff --git a/src/main/java/org/opensearch/knn/index/SpaceType.java b/src/main/java/org/opensearch/knn/index/SpaceType.java index 43ff45e1d..abe265a01 100644 --- a/src/main/java/org/opensearch/knn/index/SpaceType.java +++ b/src/main/java/org/opensearch/knn/index/SpaceType.java @@ -11,10 +11,12 @@ package org.opensearch.knn.index; +import java.util.Arrays; import java.util.Locale; import java.util.HashSet; import java.util.Set; +import java.util.stream.Collectors; import static org.opensearch.knn.common.KNNVectorUtil.isZeroVector; @@ -149,6 +151,12 @@ public KNNVectorSimilarityFunction getKnnVectorSimilarityFunction() { public static SpaceType DEFAULT = L2; public static SpaceType DEFAULT_BINARY = HAMMING; + private static final String[] VALID_VALUES = Arrays.stream(SpaceType.values()) + .filter(space -> space != SpaceType.UNDEFINED) + .map(SpaceType::getValue) + .collect(Collectors.toList()) + .toArray(new String[0]); + private final String value; SpaceType(String value) { @@ -221,7 +229,9 @@ public static SpaceType getSpace(String spaceTypeName) { return currentSpaceType; } } - throw new IllegalArgumentException("Unable to find space: " + spaceTypeName); + throw new IllegalArgumentException( + String.format(Locale.ROOT, "Unable to find space: %s . Valid values are: %s", spaceTypeName, Arrays.toString(VALID_VALUES)) + ); } /** diff --git a/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapper.java b/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapper.java index d2bb8e41a..10ba0d94d 100644 --- a/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapper.java +++ b/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapper.java @@ -161,6 +161,14 @@ public static class Builder extends ParametrizedFieldMapper.Builder { CompressionLevel.NAMES_ARRAY ).acceptsNull(); + // A top level space Type field. + protected final Parameter topLevelSpaceType = Parameter.stringParam( + KNNConstants.TOP_LEVEL_PARAMETER_SPACE_TYPE, + false, + m -> toType(m).originalMappingParameters.getTopLevelSpaceType(), + SpaceType.UNDEFINED.getValue() + ).setValidator(SpaceType::getSpace); + protected final Parameter> meta = Parameter.metaParam(); protected ModelDao modelDao; @@ -187,7 +195,18 @@ public Builder( @Override protected List> getParameters() { - return Arrays.asList(stored, hasDocValues, dimension, vectorDataType, meta, knnMethodContext, modelId, mode, compressionLevel); + return Arrays.asList( + stored, + hasDocValues, + dimension, + vectorDataType, + meta, + knnMethodContext, + modelId, + mode, + compressionLevel, + topLevelSpaceType + ); } protected Explicit ignoreMalformed(BuilderContext context) { @@ -346,6 +365,7 @@ public Mapper.Builder parse(String name, Map node, ParserCont validateFromModel(builder); } else { validateMode(builder); + validateSpaceType(builder); resolveKNNMethodComponents(builder, parserContext); validateFromKNNMethod(builder); } @@ -353,6 +373,23 @@ public Mapper.Builder parse(String name, Map node, ParserCont return builder; } + private void validateSpaceType(KNNVectorFieldMapper.Builder builder) { + final KNNMethodContext knnMethodContext = builder.knnMethodContext.get(); + // if context is defined + if (knnMethodContext != null) { + // now ensure both space types are same. + final SpaceType knnMethodContextSpaceType = knnMethodContext.getSpaceType(); + final SpaceType topLevelSpaceType = SpaceType.getSpace(builder.topLevelSpaceType.get()); + if (topLevelSpaceType != SpaceType.UNDEFINED + && topLevelSpaceType != knnMethodContextSpaceType + && knnMethodContextSpaceType != SpaceType.UNDEFINED) { + throw new MapperParsingException( + "Space type in \"method\" and top level space type should be same or one of them should be defined" + ); + } + } + } + private void validateMode(KNNVectorFieldMapper.Builder builder) { boolean isKNNMethodContextConfigured = builder.originalParameters.getKnnMethodContext() != null; boolean isModeConfigured = builder.mode.isConfigured() || builder.compressionLevel.isConfigured(); @@ -386,6 +423,11 @@ private void validateFromModel(KNNVectorFieldMapper.Builder builder) { if (builder.dimension.getValue() == UNSET_MODEL_DIMENSION_IDENTIFIER && builder.modelId.get() == null) { throw new IllegalArgumentException(String.format(Locale.ROOT, "Dimension value missing for vector: %s", builder.name())); } + // ensure model and top level spaceType is not defined + if (builder.modelId.get() != null && SpaceType.getSpace(builder.topLevelSpaceType.get()) != SpaceType.UNDEFINED) { + throw new IllegalArgumentException("TopLevel Space type and model can not be both specified in the " + "mapping"); + } + validateCompressionAndModeNotSet(builder, builder.name(), "model"); } @@ -439,19 +481,33 @@ private void resolveKNNMethodComponents(KNNVectorFieldMapper.Builder builder, Pa // Configure method from map or legacy if (builder.originalParameters.isLegacyMapping()) { builder.originalParameters.setResolvedKnnMethodContext( - createKNNMethodContextFromLegacy(parserContext.getSettings(), parserContext.indexVersionCreated()) + createKNNMethodContextFromLegacy( + parserContext.getSettings(), + parserContext.indexVersionCreated(), + SpaceType.getSpace(builder.topLevelSpaceType.get()) + ) ); } else if (Mode.isConfigured(Mode.fromName(builder.mode.get())) || CompressionLevel.isConfigured(CompressionLevel.fromName(builder.compressionLevel.get()))) { + // we need don't need to resolve the space type, whatever default we are using will be passed down to + // while resolving KNNMethodContext for the mode and compression. and then when we resolve the spaceType + // we will set the correct spaceType. builder.originalParameters.setResolvedKnnMethodContext( ModeBasedResolver.INSTANCE.resolveKNNMethodContext( builder.knnMethodConfigContext.getMode(), builder.knnMethodConfigContext.getCompressionLevel(), - false + false, + SpaceType.getSpace(builder.originalParameters.getTopLevelSpaceType()) ) ); } - setDefaultSpaceType(builder.originalParameters.getResolvedKnnMethodContext(), builder.originalParameters.getVectorDataType()); + // this function should now correct the space type for the above resolved context too, if spaceType was + // not provided. + setSpaceType( + builder.originalParameters.getResolvedKnnMethodContext(), + builder.originalParameters.getVectorDataType(), + builder.topLevelSpaceType.get() + ); } private boolean isKNNDisabled(Settings settings) { @@ -459,16 +515,30 @@ private boolean isKNNDisabled(Settings settings) { return !isSettingPresent || !KNNSettings.IS_KNN_INDEX_SETTING.get(settings); } - private void setDefaultSpaceType(final KNNMethodContext knnMethodContext, final VectorDataType vectorDataType) { + private void setSpaceType( + final KNNMethodContext knnMethodContext, + final VectorDataType vectorDataType, + final String topLevelSpaceType + ) { + // Now KNNMethodContext should never be null. Because only case it could be null is flatMapper which is + // already handled if (knnMethodContext == null) { - return; + throw new IllegalArgumentException("KNNMethodContext cannot be null"); } - + final SpaceType topLevelSpaceTypeEnum = SpaceType.getSpace(topLevelSpaceType); + // Now set the spaceSpaceType for KNNMethodContext if (SpaceType.UNDEFINED == knnMethodContext.getSpaceType()) { - if (VectorDataType.BINARY == vectorDataType) { - knnMethodContext.setSpaceType(SpaceType.DEFAULT_BINARY); + // We are handling the case when top level space type is defined but method level spaceType is not + // defined. + if (topLevelSpaceTypeEnum != SpaceType.UNDEFINED) { + knnMethodContext.setSpaceType(topLevelSpaceTypeEnum); } else { - knnMethodContext.setSpaceType(SpaceType.DEFAULT); + // If both spaceTypes are undefined then put the default spaceType based on datatype + if (VectorDataType.BINARY == vectorDataType) { + knnMethodContext.setSpaceType(SpaceType.DEFAULT_BINARY); + } else { + knnMethodContext.setSpaceType(SpaceType.DEFAULT); + } } } } diff --git a/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperUtil.java b/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperUtil.java index 551905793..b3727f2ef 100644 --- a/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperUtil.java +++ b/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperUtil.java @@ -193,10 +193,18 @@ private static int getEfConstruction(Settings indexSettings, Version indexVersio return Integer.parseInt(efConstruction); } - static KNNMethodContext createKNNMethodContextFromLegacy(Settings indexSettings, Version indexCreatedVersion) { + static KNNMethodContext createKNNMethodContextFromLegacy( + Settings indexSettings, + Version indexCreatedVersion, + SpaceType topLevelSpaceType + ) { + // If top level spaceType is set then use that spaceType otherwise default to spaceType from index-settings + final SpaceType finalSpaceToSet = topLevelSpaceType != SpaceType.UNDEFINED + ? topLevelSpaceType + : KNNVectorFieldMapperUtil.getSpaceType(indexSettings); return new KNNMethodContext( KNNEngine.NMSLIB, - KNNVectorFieldMapperUtil.getSpaceType(indexSettings), + finalSpaceToSet, new MethodComponentContext( METHOD_HNSW, Map.of( diff --git a/src/main/java/org/opensearch/knn/index/mapper/ModeBasedResolver.java b/src/main/java/org/opensearch/knn/index/mapper/ModeBasedResolver.java index 06b34fcd3..2a0c8ef46 100644 --- a/src/main/java/org/opensearch/knn/index/mapper/ModeBasedResolver.java +++ b/src/main/java/org/opensearch/knn/index/mapper/ModeBasedResolver.java @@ -59,15 +59,19 @@ private ModeBasedResolver() {} * @param requiresTraining whether config requires trianing * @return {@link KNNMethodContext} */ - public KNNMethodContext resolveKNNMethodContext(Mode mode, CompressionLevel compressionLevel, boolean requiresTraining) { + public KNNMethodContext resolveKNNMethodContext( + Mode mode, + CompressionLevel compressionLevel, + boolean requiresTraining, + SpaceType spaceType + ) { if (requiresTraining) { - return resolveWithTraining(mode, compressionLevel); + return resolveWithTraining(mode, compressionLevel, spaceType); } - - return resolveWithoutTraining(mode, compressionLevel); + return resolveWithoutTraining(mode, compressionLevel, spaceType); } - private KNNMethodContext resolveWithoutTraining(Mode mode, CompressionLevel compressionLevel) { + private KNNMethodContext resolveWithoutTraining(Mode mode, CompressionLevel compressionLevel, final SpaceType spaceType) { CompressionLevel resolvedCompressionLevel = resolveCompressionLevel(mode, compressionLevel); MethodComponentContext encoderContext = resolveEncoder(resolvedCompressionLevel); @@ -76,7 +80,7 @@ private KNNMethodContext resolveWithoutTraining(Mode mode, CompressionLevel comp if (encoderContext != null) { return new KNNMethodContext( knnEngine, - SpaceType.DEFAULT, + spaceType, new MethodComponentContext( METHOD_HNSW, Map.of( @@ -96,7 +100,7 @@ private KNNMethodContext resolveWithoutTraining(Mode mode, CompressionLevel comp if (knnEngine == KNNEngine.FAISS) { return new KNNMethodContext( knnEngine, - SpaceType.DEFAULT, + spaceType, new MethodComponentContext( METHOD_HNSW, Map.of( @@ -113,7 +117,7 @@ private KNNMethodContext resolveWithoutTraining(Mode mode, CompressionLevel comp return new KNNMethodContext( knnEngine, - SpaceType.DEFAULT, + spaceType, new MethodComponentContext( METHOD_HNSW, Map.of( @@ -126,13 +130,13 @@ private KNNMethodContext resolveWithoutTraining(Mode mode, CompressionLevel comp ); } - private KNNMethodContext resolveWithTraining(Mode mode, CompressionLevel compressionLevel) { + private KNNMethodContext resolveWithTraining(Mode mode, CompressionLevel compressionLevel, SpaceType spaceType) { CompressionLevel resolvedCompressionLevel = resolveCompressionLevel(mode, compressionLevel); MethodComponentContext encoderContext = resolveEncoder(resolvedCompressionLevel); if (encoderContext != null) { return new KNNMethodContext( KNNEngine.FAISS, - SpaceType.DEFAULT, + spaceType, new MethodComponentContext( METHOD_IVF, Map.of( @@ -149,7 +153,7 @@ private KNNMethodContext resolveWithTraining(Mode mode, CompressionLevel compres return new KNNMethodContext( KNNEngine.FAISS, - SpaceType.DEFAULT, + spaceType, new MethodComponentContext( METHOD_IVF, Map.of(METHOD_PARAMETER_NLIST, METHOD_PARAMETER_NLIST_DEFAULT, METHOD_PARAMETER_NPROBES, METHOD_PARAMETER_NPROBES_DEFAULT) diff --git a/src/main/java/org/opensearch/knn/index/mapper/OriginalMappingParameters.java b/src/main/java/org/opensearch/knn/index/mapper/OriginalMappingParameters.java index f7235620f..77bf07a90 100644 --- a/src/main/java/org/opensearch/knn/index/mapper/OriginalMappingParameters.java +++ b/src/main/java/org/opensearch/knn/index/mapper/OriginalMappingParameters.java @@ -42,6 +42,7 @@ public final class OriginalMappingParameters { private final String mode; private final String compressionLevel; private final String modelId; + private final String topLevelSpaceType; /** * Initialize the parameters from the builder @@ -56,6 +57,7 @@ public OriginalMappingParameters(KNNVectorFieldMapper.Builder builder) { this.mode = builder.mode.get(); this.compressionLevel = builder.compressionLevel.get(); this.modelId = builder.modelId.get(); + this.topLevelSpaceType = builder.topLevelSpaceType.get(); } /** diff --git a/src/main/java/org/opensearch/knn/index/util/IndexUtil.java b/src/main/java/org/opensearch/knn/index/util/IndexUtil.java index b548a9fd7..4a0118f58 100644 --- a/src/main/java/org/opensearch/knn/index/util/IndexUtil.java +++ b/src/main/java/org/opensearch/knn/index/util/IndexUtil.java @@ -53,6 +53,7 @@ public class IndexUtil { private static final Version MINIMAL_SUPPORTED_VERSION_FOR_MODEL_VECTOR_DATA_TYPE = Version.V_2_16_0; private static final Version MINIMAL_RESCORE_FEATURE = Version.V_2_17_0; private static final Version MINIMAL_MODE_AND_COMPRESSION_FEATURE = Version.V_2_17_0; + private static final Version MINIMAL_TOP_LEVEL_SPACE_TYPE_FEATURE = Version.V_2_17_0; // public so neural search can access it public static final Map minimalRequiredVersionMap = initializeMinimalRequiredVersionMap(); public static final Set VECTOR_DATA_TYPES_NOT_SUPPORTING_ENCODERS = Set.of(VectorDataType.BINARY, VectorDataType.BYTE); @@ -392,6 +393,7 @@ private static Map initializeMinimalRequiredVersionMap() { put(KNNConstants.MODEL_VECTOR_DATA_TYPE_KEY, MINIMAL_SUPPORTED_VERSION_FOR_MODEL_VECTOR_DATA_TYPE); put(RESCORE_PARAMETER, MINIMAL_RESCORE_FEATURE); put(KNNConstants.MINIMAL_MODE_AND_COMPRESSION_FEATURE, MINIMAL_MODE_AND_COMPRESSION_FEATURE); + put(KNNConstants.TOP_LEVEL_SPACE_TYPE_FEATURE, MINIMAL_TOP_LEVEL_SPACE_TYPE_FEATURE); } }; diff --git a/src/main/java/org/opensearch/knn/plugin/rest/RestTrainModelHandler.java b/src/main/java/org/opensearch/knn/plugin/rest/RestTrainModelHandler.java index 8770449eb..0f7e8523b 100644 --- a/src/main/java/org/opensearch/knn/plugin/rest/RestTrainModelHandler.java +++ b/src/main/java/org/opensearch/knn/plugin/rest/RestTrainModelHandler.java @@ -95,6 +95,7 @@ private TrainingModelRequest createTransportRequest(RestRequest restRequest) thr int dimension = DEFAULT_NOT_SET_INT_VALUE; int maximumVectorCount = DEFAULT_NOT_SET_INT_VALUE; int searchSize = DEFAULT_NOT_SET_INT_VALUE; + SpaceType topLevelSpaceType = SpaceType.UNDEFINED; String compressionLevel = null; String mode = null; @@ -109,9 +110,6 @@ private TrainingModelRequest createTransportRequest(RestRequest restRequest) thr trainingField = parser.textOrNull(); } else if (KNN_METHOD.equals(fieldName) && ensureNotSet(fieldName, knnMethodContext)) { knnMethodContext = KNNMethodContext.parse(parser.map()); - if (SpaceType.UNDEFINED == knnMethodContext.getSpaceType()) { - knnMethodContext.setSpaceType(SpaceType.L2); - } } else if (DIMENSION.equals(fieldName) && ensureNotSet(fieldName, dimension)) { dimension = (Integer) NumberFieldMapper.NumberType.INTEGER.parse(parser.objectBytes(), false); } else if (MAX_VECTOR_COUNT_PARAMETER.equals(fieldName) && ensureNotSet(fieldName, maximumVectorCount)) { @@ -127,6 +125,8 @@ private TrainingModelRequest createTransportRequest(RestRequest restRequest) thr mode = parser.text(); } else if (KNNConstants.COMPRESSION_LEVEL_PARAMETER.equals(fieldName) && ensureNotSet(fieldName, compressionLevel)) { compressionLevel = parser.text(); + } else if (KNNConstants.SPACE_TYPE.equals(fieldName) && ensureSpaceTypeNotSet(topLevelSpaceType)) { + topLevelSpaceType = SpaceType.getSpace(parser.text()); } else { throw new IllegalArgumentException("Unable to parse token. \"" + fieldName + "\" is not a valid " + "parameter."); } @@ -160,7 +160,11 @@ private TrainingModelRequest createTransportRequest(RestRequest restRequest) thr vectorDataType, VectorDataType.FLOAT.getValue() ); - + resolveSpaceTypeAndSetInKNNMethodContext(topLevelSpaceType, knnMethodContext); + // if KNNMethodContext was not null then spaceTypes we should fix the space type if it is not set. + if (knnMethodContext == null && topLevelSpaceType == SpaceType.UNDEFINED) { + topLevelSpaceType = SpaceType.DEFAULT; + } TrainingModelRequest trainingModelRequest = new TrainingModelRequest( modelId, knnMethodContext, @@ -171,7 +175,8 @@ private TrainingModelRequest createTransportRequest(RestRequest restRequest) thr description, vectorDataType, Mode.fromName(mode), - CompressionLevel.fromName(compressionLevel) + CompressionLevel.fromName(compressionLevel), + topLevelSpaceType ); if (maximumVectorCount != DEFAULT_NOT_SET_INT_VALUE) { @@ -205,6 +210,35 @@ private void ensureMutualExclusion(String fieldNameA, Object valueA, String fiel } } + private boolean ensureSpaceTypeNotSet(SpaceType spaceType) { + if (spaceType != SpaceType.UNDEFINED) { + throw new IllegalArgumentException("Unable to parse SpaceType as it is duplicated."); + } + return true; + } + + private void resolveSpaceTypeAndSetInKNNMethodContext(SpaceType topLevelSpaceType, KNNMethodContext knnMethodContext) { + // First check if KNNMethodContext is not null as it can be null + if (knnMethodContext != null) { + // if space type is not provided by user then it will undefined + if (knnMethodContext.getSpaceType() == SpaceType.UNDEFINED) { + // fix the top level spaceType if it is undefined + if (topLevelSpaceType == SpaceType.UNDEFINED) { + topLevelSpaceType = SpaceType.DEFAULT; + } + // set the space type now in KNNMethodContext + knnMethodContext.setSpaceType(topLevelSpaceType); + } else { + // if spaceType is set at 2 places lets ensure that we validate those cases and throw error + if (topLevelSpaceType != SpaceType.UNDEFINED) { + throw new IllegalArgumentException( + "Top Level spaceType and space type in method both are set. Set space type at 1 place." + ); + } + } + } + } + private void ensureIfSetThenEquals( String fieldNameA, Object valueA, diff --git a/src/main/java/org/opensearch/knn/plugin/transport/TrainingModelRequest.java b/src/main/java/org/opensearch/knn/plugin/transport/TrainingModelRequest.java index 82669d4a8..df24baf0e 100644 --- a/src/main/java/org/opensearch/knn/plugin/transport/TrainingModelRequest.java +++ b/src/main/java/org/opensearch/knn/plugin/transport/TrainingModelRequest.java @@ -21,6 +21,7 @@ import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.knn.common.KNNConstants; +import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.engine.KNNMethodConfigContext; import org.opensearch.knn.index.mapper.CompressionLevel; import org.opensearch.knn.index.mapper.Mode; @@ -56,6 +57,33 @@ public class TrainingModelRequest extends ActionRequest { private final Mode mode; private final CompressionLevel compressionLevel; + TrainingModelRequest( + String modelId, + KNNMethodContext knnMethodContext, + int dimension, + String trainingIndex, + String trainingField, + String preferredNodeId, + String description, + VectorDataType vectorDataType, + Mode mode, + CompressionLevel compressionLevel + ) { + this( + modelId, + knnMethodContext, + dimension, + trainingIndex, + trainingField, + preferredNodeId, + description, + vectorDataType, + mode, + compressionLevel, + SpaceType.DEFAULT + ); + } + /** * Constructor. * @@ -77,7 +105,8 @@ public TrainingModelRequest( String description, VectorDataType vectorDataType, Mode mode, - CompressionLevel compressionLevel + CompressionLevel compressionLevel, + SpaceType spaceType ) { super(); this.modelId = modelId; @@ -107,7 +136,7 @@ public TrainingModelRequest( .build(); if (knnMethodContext == null && (Mode.isConfigured(mode) || CompressionLevel.isConfigured(compressionLevel))) { - this.knnMethodContext = ModeBasedResolver.INSTANCE.resolveKNNMethodContext(mode, compressionLevel, true); + this.knnMethodContext = ModeBasedResolver.INSTANCE.resolveKNNMethodContext(mode, compressionLevel, true, spaceType); } else { this.knnMethodContext = knnMethodContext; } diff --git a/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java b/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java index 6d0a3d5df..abc4f563e 100644 --- a/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java +++ b/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java @@ -36,6 +36,7 @@ import org.opensearch.index.mapper.MapperService; import org.opensearch.index.mapper.ParseContext; import org.opensearch.knn.KNNTestCase; +import org.opensearch.knn.common.KNNConstants; import org.opensearch.knn.index.KNNSettings; import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.VectorDataType; @@ -117,10 +118,10 @@ public void testBuilder_getParameters() { modelDao, CURRENT, null, - new OriginalMappingParameters(VectorDataType.DEFAULT, TEST_DIMENSION, null, null, null, null) + new OriginalMappingParameters(VectorDataType.DEFAULT, TEST_DIMENSION, null, null, null, null, SpaceType.UNDEFINED.getValue()) ); - assertEquals(9, builder.getParameters().size()); + assertEquals(10, builder.getParameters().size()); List actualParams = builder.getParameters().stream().map(a -> a.name).collect(Collectors.toList()); List expectedParams = Arrays.asList( "store", @@ -131,7 +132,8 @@ public void testBuilder_getParameters() { KNN_METHOD, MODEL_ID, MODE_PARAMETER, - COMPRESSION_LEVEL_PARAMETER + COMPRESSION_LEVEL_PARAMETER, + KNNConstants.TOP_LEVEL_PARAMETER_SPACE_TYPE ); assertEquals(expectedParams, actualParams); } @@ -899,7 +901,8 @@ public void testMethodFieldMapperParseCreateField_validInput_thenDifferentFieldT knnMethodContext, Mode.NOT_CONFIGURED.getName(), CompressionLevel.NOT_CONFIGURED.getName(), - null + null, + SpaceType.UNDEFINED.getValue() ); originalMappingParameters.setResolvedKnnMethodContext(knnMethodContext); MethodFieldMapper methodFieldMapper = MethodFieldMapper.createFieldMapper( @@ -1000,7 +1003,8 @@ public void testModelFieldMapperParseCreateField_validInput_thenDifferentFieldTy null, Mode.NOT_CONFIGURED.getName(), CompressionLevel.NOT_CONFIGURED.getName(), - MODEL_ID + MODEL_ID, + SpaceType.UNDEFINED.getValue() ); ModelFieldMapper modelFieldMapper = ModelFieldMapper.createFieldMapper( @@ -1092,7 +1096,8 @@ public void testLuceneFieldMapper_parseCreateField_docValues_withFloats() { getDefaultKNNMethodContext(), Mode.NOT_CONFIGURED.getName(), CompressionLevel.NOT_CONFIGURED.getName(), - null + null, + SpaceType.UNDEFINED.getValue() ); originalMappingParameters.setResolvedKnnMethodContext(originalMappingParameters.getKnnMethodContext()); @@ -1151,7 +1156,8 @@ public void testLuceneFieldMapper_parseCreateField_docValues_withFloats() { knnMethodContext, Mode.NOT_CONFIGURED.getName(), CompressionLevel.NOT_CONFIGURED.getName(), - null + null, + SpaceType.UNDEFINED.getValue() ); originalMappingParameters.setResolvedKnnMethodContext(originalMappingParameters.getKnnMethodContext()); luceneFieldMapper = LuceneFieldMapper.createFieldMapper( @@ -1191,7 +1197,8 @@ public void testLuceneFieldMapper_parseCreateField_docValues_withBytes() { getDefaultByteKNNMethodContext(), Mode.NOT_CONFIGURED.getName(), CompressionLevel.NOT_CONFIGURED.getName(), - null + null, + SpaceType.UNDEFINED.getValue() ); originalMappingParameters.setResolvedKnnMethodContext(originalMappingParameters.getKnnMethodContext()); diff --git a/src/test/java/org/opensearch/knn/index/mapper/OriginalMappingParametersTests.java b/src/test/java/org/opensearch/knn/index/mapper/OriginalMappingParametersTests.java index 2822a882e..4b089b149 100644 --- a/src/test/java/org/opensearch/knn/index/mapper/OriginalMappingParametersTests.java +++ b/src/test/java/org/opensearch/knn/index/mapper/OriginalMappingParametersTests.java @@ -17,11 +17,35 @@ public class OriginalMappingParametersTests extends KNNTestCase { public void testIsLegacy() { - assertTrue(new OriginalMappingParameters(VectorDataType.DEFAULT, 123, null, null, null, null).isLegacyMapping()); - assertFalse(new OriginalMappingParameters(VectorDataType.DEFAULT, 123, null, null, null, "model-id").isLegacyMapping()); - assertFalse(new OriginalMappingParameters(VectorDataType.DEFAULT, 123, null, Mode.ON_DISK.getName(), null, null).isLegacyMapping()); + assertTrue( + new OriginalMappingParameters(VectorDataType.DEFAULT, 123, null, null, null, null, SpaceType.UNDEFINED.getValue()) + .isLegacyMapping() + ); + assertFalse( + new OriginalMappingParameters(VectorDataType.DEFAULT, 123, null, null, null, "model-id", SpaceType.UNDEFINED.getValue()) + .isLegacyMapping() + ); + assertFalse( + new OriginalMappingParameters( + VectorDataType.DEFAULT, + 123, + null, + Mode.ON_DISK.getName(), + null, + null, + SpaceType.UNDEFINED.getValue() + ).isLegacyMapping() + ); assertFalse( - new OriginalMappingParameters(VectorDataType.DEFAULT, 123, null, null, CompressionLevel.x2.getName(), null).isLegacyMapping() + new OriginalMappingParameters( + VectorDataType.DEFAULT, + 123, + null, + null, + CompressionLevel.x2.getName(), + null, + SpaceType.UNDEFINED.getValue() + ).isLegacyMapping() ); assertFalse( new OriginalMappingParameters( @@ -30,7 +54,8 @@ public void testIsLegacy() { new KNNMethodContext(KNNEngine.DEFAULT, SpaceType.L2, new MethodComponentContext(null, Collections.emptyMap())), null, null, - null + null, + SpaceType.UNDEFINED.getValue() ).isLegacyMapping() ); } From f3d38bc640af712778b1d69c643a22c97cccb6ed Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Thu, 5 Sep 2024 11:06:31 -0700 Subject: [PATCH 355/416] Fix the force merge with Quantization failures when a segment has deleted docs in it (#2046) (#2051) Signed-off-by: Navneet Verma (cherry picked from commit da854c969a85f9cab2b8d84b2433020b2e49e18f) Co-authored-by: Navneet Verma --- .../KNN80Codec/KNN80DocValuesConsumer.java | 6 ++-- .../NativeEngines990KnnVectorsWriter.java | 28 ++++++++++++---- .../DefaultIndexBuildStrategy.java | 6 ++-- .../MemOptimizedNativeIndexBuildStrategy.java | 5 +-- .../nativeindex/NativeIndexBuildStrategy.java | 3 +- .../codec/nativeindex/NativeIndexWriter.java | 30 ++++++++++------- .../nativeindex/model/BuildIndexParams.java | 3 ++ .../KNNVectorQuantizationTrainingRequest.java | 4 +-- .../QuantizationService.java | 9 +++-- .../index/vectorvalues/KNNVectorValues.java | 11 ++++++- .../DefaultIndexBuildStrategyTests.java | 12 +++++-- ...ptimizedNativeIndexBuildStrategyTests.java | 8 +++-- .../QuantizationServiceTests.java | 14 ++++---- .../knn/integ/ModeAndCompressionIT.java | 33 +++++++++++++++++++ 14 files changed, 128 insertions(+), 44 deletions(-) diff --git a/src/main/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumer.java b/src/main/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumer.java index 218c9d891..443b12b9c 100644 --- a/src/main/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumer.java +++ b/src/main/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumer.java @@ -69,10 +69,12 @@ public void addKNNBinaryField(FieldInfo field, DocValuesProducer valuesProducer, final VectorDataType vectorDataType = extractVectorDataType(field); final KNNVectorValues knnVectorValues = KNNVectorValuesFactory.getVectorValues(vectorDataType, valuesProducer.getBinary(field)); + // For BDV it is fine to use knnVectorValues.totalLiveDocs() as we already run the full loop to calculate total + // live docs if (isMerge) { - NativeIndexWriter.getWriter(field, state).mergeIndex(knnVectorValues); + NativeIndexWriter.getWriter(field, state).mergeIndex(knnVectorValues, (int) knnVectorValues.totalLiveDocs()); } else { - NativeIndexWriter.getWriter(field, state).flushIndex(knnVectorValues); + NativeIndexWriter.getWriter(field, state).flushIndex(knnVectorValues, (int) knnVectorValues.totalLiveDocs()); } } diff --git a/src/main/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsWriter.java b/src/main/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsWriter.java index af7f1c576..dba0926ff 100644 --- a/src/main/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsWriter.java +++ b/src/main/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsWriter.java @@ -21,6 +21,7 @@ import org.apache.lucene.index.MergeState; import org.apache.lucene.index.SegmentWriteState; import org.apache.lucene.index.Sorter; +import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.RamUsageEstimator; import org.opensearch.common.StopWatch; @@ -63,8 +64,6 @@ public NativeEngines990KnnVectorsWriter(SegmentWriteState segmentWriteState, Fla /** * Add new field for indexing. - * In Lucene, we use single file for all the vector fields so here we need to see how we are going to make things - * work. * @param fieldInfo {@link FieldInfo} */ @Override @@ -204,7 +203,7 @@ private KNNVectorValues getKNNVectorValuesForMerge( */ @FunctionalInterface private interface IndexOperation { - void buildAndWrite(NativeIndexWriter writer, KNNVectorValues knnVectorValues) throws IOException; + void buildAndWrite(NativeIndexWriter writer, KNNVectorValues knnVectorValues, int totalLiveDocs) throws IOException; } /** @@ -248,9 +247,11 @@ private void trainAndIndex( KNNVectorValues knnVectorValues = vectorValuesRetriever.apply(vectorDataType, fieldInfo, VectorProcessingContext); QuantizationParams quantizationParams = quantizationService.getQuantizationParams(fieldInfo); QuantizationState quantizationState = null; - if (quantizationParams != null) { + // Count the docIds + int totalLiveDocs = getLiveDocs(vectorValuesRetriever.apply(vectorDataType, fieldInfo, VectorProcessingContext)); + if (quantizationParams != null && totalLiveDocs > 0) { initQuantizationStateWriterIfNecessary(); - quantizationState = quantizationService.train(quantizationParams, knnVectorValues); + quantizationState = quantizationService.train(quantizationParams, knnVectorValues, totalLiveDocs); quantizationStateWriter.writeState(fieldInfo.getFieldNumber(), quantizationState); } NativeIndexWriter writer = (quantizationParams != null) @@ -261,12 +262,27 @@ private void trainAndIndex( StopWatch stopWatch = new StopWatch(); stopWatch.start(); - indexOperation.buildAndWrite(writer, knnVectorValues); + indexOperation.buildAndWrite(writer, knnVectorValues, totalLiveDocs); long time_in_millis = stopWatch.totalTime().millis(); graphBuildTime.incrementBy(time_in_millis); log.warn("Graph build took " + time_in_millis + " ms for " + operationName); } + /** + * The {@link KNNVectorValues} will be exhausted after this function run. So make sure that you are not sending the + * vectorsValues object which you plan to use later + */ + private int getLiveDocs(KNNVectorValues vectorValues) throws IOException { + // Count all the live docs as there vectorValues.totalLiveDocs() just gives the cost for the FloatVectorValues, + // and doesn't tell the correct number of docs, if there are deleted docs in the segment. So we are counting + // the total live docs here. + int liveDocs = 0; + while (vectorValues.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) { + liveDocs++; + } + return liveDocs; + } + private void initQuantizationStateWriterIfNecessary() throws IOException { if (quantizationStateWriter == null) { quantizationStateWriter = new KNN990QuantizationStateWriter(segmentWriteState); diff --git a/src/main/java/org/opensearch/knn/index/codec/nativeindex/DefaultIndexBuildStrategy.java b/src/main/java/org/opensearch/knn/index/codec/nativeindex/DefaultIndexBuildStrategy.java index d2a6027db..e68121a7d 100644 --- a/src/main/java/org/opensearch/knn/index/codec/nativeindex/DefaultIndexBuildStrategy.java +++ b/src/main/java/org/opensearch/knn/index/codec/nativeindex/DefaultIndexBuildStrategy.java @@ -48,17 +48,17 @@ public static DefaultIndexBuildStrategy getInstance() { * flushed and used to build the index. The index is then written to the specified path using JNI calls.

    * * @param indexInfo The {@link BuildIndexParams} containing the parameters and configuration for building the index. - * @param knnVectorValues The {@link KNNVectorValues} representing the vectors to be indexed. * @throws IOException If an I/O error occurs during the process of building and writing the index. */ - public void buildAndWriteIndex(final BuildIndexParams indexInfo, final KNNVectorValues knnVectorValues) throws IOException { + public void buildAndWriteIndex(final BuildIndexParams indexInfo) throws IOException { + final KNNVectorValues knnVectorValues = indexInfo.getVectorValues(); // Needed to make sure we don't get 0 dimensions while initializing index iterateVectorValuesOnce(knnVectorValues); IndexBuildSetup indexBuildSetup = QuantizationIndexUtils.prepareIndexBuild(knnVectorValues, indexInfo); int transferLimit = (int) Math.max(1, KNNSettings.getVectorStreamingMemoryLimit().getBytes() / indexBuildSetup.getBytesPerVector()); try (final OffHeapVectorTransfer vectorTransfer = getVectorTransfer(indexInfo.getVectorDataType(), transferLimit)) { - final List transferredDocIds = new ArrayList<>((int) knnVectorValues.totalLiveDocs()); + final List transferredDocIds = new ArrayList<>(indexInfo.getTotalLiveDocs()); while (knnVectorValues.docId() != NO_MORE_DOCS) { Object vector = QuantizationIndexUtils.processAndReturnVector(knnVectorValues, indexBuildSetup); diff --git a/src/main/java/org/opensearch/knn/index/codec/nativeindex/MemOptimizedNativeIndexBuildStrategy.java b/src/main/java/org/opensearch/knn/index/codec/nativeindex/MemOptimizedNativeIndexBuildStrategy.java index 1115bfe05..af3f4777f 100644 --- a/src/main/java/org/opensearch/knn/index/codec/nativeindex/MemOptimizedNativeIndexBuildStrategy.java +++ b/src/main/java/org/opensearch/knn/index/codec/nativeindex/MemOptimizedNativeIndexBuildStrategy.java @@ -52,7 +52,8 @@ public static MemOptimizedNativeIndexBuildStrategy getInstance() { * @param knnVectorValues The {@link KNNVectorValues} representing the vectors to be indexed. * @throws IOException If an I/O error occurs during the process of building and writing the index. */ - public void buildAndWriteIndex(final BuildIndexParams indexInfo, final KNNVectorValues knnVectorValues) throws IOException { + public void buildAndWriteIndex(final BuildIndexParams indexInfo) throws IOException { + final KNNVectorValues knnVectorValues = indexInfo.getVectorValues(); // Needed to make sure we don't get 0 dimensions while initializing index iterateVectorValuesOnce(knnVectorValues); KNNEngine engine = indexInfo.getKnnEngine(); @@ -62,7 +63,7 @@ public void buildAndWriteIndex(final BuildIndexParams indexInfo, final KNNVector // Initialize the index long indexMemoryAddress = AccessController.doPrivileged( (PrivilegedAction) () -> JNIService.initIndex( - knnVectorValues.totalLiveDocs(), + indexInfo.getTotalLiveDocs(), indexBuildSetup.getDimensions(), indexParameters, engine diff --git a/src/main/java/org/opensearch/knn/index/codec/nativeindex/NativeIndexBuildStrategy.java b/src/main/java/org/opensearch/knn/index/codec/nativeindex/NativeIndexBuildStrategy.java index 19475adfa..8c9f6de97 100644 --- a/src/main/java/org/opensearch/knn/index/codec/nativeindex/NativeIndexBuildStrategy.java +++ b/src/main/java/org/opensearch/knn/index/codec/nativeindex/NativeIndexBuildStrategy.java @@ -6,7 +6,6 @@ package org.opensearch.knn.index.codec.nativeindex; import org.opensearch.knn.index.codec.nativeindex.model.BuildIndexParams; -import org.opensearch.knn.index.vectorvalues.KNNVectorValues; import java.io.IOException; @@ -15,5 +14,5 @@ */ public interface NativeIndexBuildStrategy { - void buildAndWriteIndex(BuildIndexParams indexInfo, final KNNVectorValues knnVectorValues) throws IOException; + void buildAndWriteIndex(BuildIndexParams indexInfo) throws IOException; } diff --git a/src/main/java/org/opensearch/knn/index/codec/nativeindex/NativeIndexWriter.java b/src/main/java/org/opensearch/knn/index/codec/nativeindex/NativeIndexWriter.java index 087773044..edc96c9e1 100644 --- a/src/main/java/org/opensearch/knn/index/codec/nativeindex/NativeIndexWriter.java +++ b/src/main/java/org/opensearch/knn/index/codec/nativeindex/NativeIndexWriter.java @@ -106,9 +106,9 @@ public static NativeIndexWriter getWriter( * @param knnVectorValues * @throws IOException */ - public void flushIndex(final KNNVectorValues knnVectorValues) throws IOException { + public void flushIndex(final KNNVectorValues knnVectorValues, int totalLiveDocs) throws IOException { iterateVectorValuesOnce(knnVectorValues); - buildAndWriteIndex(knnVectorValues); + buildAndWriteIndex(knnVectorValues, totalLiveDocs); recordRefreshStats(); } @@ -117,7 +117,7 @@ public void flushIndex(final KNNVectorValues knnVectorValues) throws IOExcept * @param knnVectorValues * @throws IOException */ - public void mergeIndex(final KNNVectorValues knnVectorValues) throws IOException { + public void mergeIndex(final KNNVectorValues knnVectorValues, int totalLiveDocs) throws IOException { iterateVectorValuesOnce(knnVectorValues); if (knnVectorValues.docId() == NO_MORE_DOCS) { // This is in place so we do not add metrics @@ -126,13 +126,13 @@ public void mergeIndex(final KNNVectorValues knnVectorValues) throws IOExcept } long bytesPerVector = knnVectorValues.bytesPerVector(); - startMergeStats((int) knnVectorValues.totalLiveDocs(), bytesPerVector); - buildAndWriteIndex(knnVectorValues); - endMergeStats((int) knnVectorValues.totalLiveDocs(), bytesPerVector); + startMergeStats(totalLiveDocs, bytesPerVector); + buildAndWriteIndex(knnVectorValues, totalLiveDocs); + endMergeStats(totalLiveDocs, bytesPerVector); } - private void buildAndWriteIndex(final KNNVectorValues knnVectorValues) throws IOException { - if (knnVectorValues.totalLiveDocs() == 0) { + private void buildAndWriteIndex(final KNNVectorValues knnVectorValues, int totalLiveDocs) throws IOException { + if (totalLiveDocs == 0) { log.debug("No live docs for field " + fieldInfo.name); return; } @@ -150,15 +150,21 @@ private void buildAndWriteIndex(final KNNVectorValues knnVectorValues) throws ).toString(); state.directory.createOutput(engineFileName, state.context).close(); - final BuildIndexParams nativeIndexParams = indexParams(fieldInfo, indexPath, knnEngine); - indexBuilder.buildAndWriteIndex(nativeIndexParams, knnVectorValues); + final BuildIndexParams nativeIndexParams = indexParams(fieldInfo, indexPath, knnEngine, knnVectorValues, totalLiveDocs); + indexBuilder.buildAndWriteIndex(nativeIndexParams); writeFooter(indexPath, engineFileName, state); } // The logic for building parameters need to be cleaned up. There are various cases handled here // Currently it falls under two categories - with model and without model. Without model is further divided based on vector data type // TODO: Refactor this so its scalable. Possibly move it out of this class - private BuildIndexParams indexParams(FieldInfo fieldInfo, String indexPath, KNNEngine knnEngine) throws IOException { + private BuildIndexParams indexParams( + FieldInfo fieldInfo, + String indexPath, + KNNEngine knnEngine, + KNNVectorValues vectorValues, + int totalLiveDocs + ) throws IOException { final Map parameters; VectorDataType vectorDataType; if (quantizationState != null) { @@ -180,6 +186,8 @@ private BuildIndexParams indexParams(FieldInfo fieldInfo, String indexPath, KNNE .knnEngine(knnEngine) .indexPath(indexPath) .quantizationState(quantizationState) + .vectorValues(vectorValues) + .totalLiveDocs(totalLiveDocs) .build(); } diff --git a/src/main/java/org/opensearch/knn/index/codec/nativeindex/model/BuildIndexParams.java b/src/main/java/org/opensearch/knn/index/codec/nativeindex/model/BuildIndexParams.java index 78674c64b..88507b1fc 100644 --- a/src/main/java/org/opensearch/knn/index/codec/nativeindex/model/BuildIndexParams.java +++ b/src/main/java/org/opensearch/knn/index/codec/nativeindex/model/BuildIndexParams.java @@ -11,6 +11,7 @@ import org.opensearch.common.Nullable; import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.engine.KNNEngine; +import org.opensearch.knn.index.vectorvalues.KNNVectorValues; import org.opensearch.knn.quantization.models.quantizationState.QuantizationState; import java.util.Map; @@ -29,4 +30,6 @@ public class BuildIndexParams { */ @Nullable QuantizationState quantizationState; + KNNVectorValues vectorValues; + int totalLiveDocs; } diff --git a/src/main/java/org/opensearch/knn/index/quantizationservice/KNNVectorQuantizationTrainingRequest.java b/src/main/java/org/opensearch/knn/index/quantizationservice/KNNVectorQuantizationTrainingRequest.java index d880a4178..f7ee12904 100644 --- a/src/main/java/org/opensearch/knn/index/quantizationservice/KNNVectorQuantizationTrainingRequest.java +++ b/src/main/java/org/opensearch/knn/index/quantizationservice/KNNVectorQuantizationTrainingRequest.java @@ -28,8 +28,8 @@ final class KNNVectorQuantizationTrainingRequest extends TrainingRequest { * * @param knnVectorValues the KNNVectorValues instance containing the vectors. */ - KNNVectorQuantizationTrainingRequest(KNNVectorValues knnVectorValues) { - super((int) knnVectorValues.totalLiveDocs()); + KNNVectorQuantizationTrainingRequest(KNNVectorValues knnVectorValues, long liveDocs) { + super((int) liveDocs); this.knnVectorValues = knnVectorValues; this.lastIndex = 0; } diff --git a/src/main/java/org/opensearch/knn/index/quantizationservice/QuantizationService.java b/src/main/java/org/opensearch/knn/index/quantizationservice/QuantizationService.java index b1c94f993..771848730 100644 --- a/src/main/java/org/opensearch/knn/index/quantizationservice/QuantizationService.java +++ b/src/main/java/org/opensearch/knn/index/quantizationservice/QuantizationService.java @@ -57,12 +57,15 @@ public static QuantizationService getInstance() { * @return The {@link QuantizationState} containing the state of the trained quantizer. * @throws IOException If an I/O error occurs during the training process. */ - public QuantizationState train(final QuantizationParams quantizationParams, final KNNVectorValues knnVectorValues) - throws IOException { + public QuantizationState train( + final QuantizationParams quantizationParams, + final KNNVectorValues knnVectorValues, + final long liveDocs + ) throws IOException { Quantizer quantizer = QuantizerFactory.getQuantizer(quantizationParams); // Create the training request from the vector values - KNNVectorQuantizationTrainingRequest trainingRequest = new KNNVectorQuantizationTrainingRequest<>(knnVectorValues); + KNNVectorQuantizationTrainingRequest trainingRequest = new KNNVectorQuantizationTrainingRequest<>(knnVectorValues, liveDocs); // Train the quantizer and return the quantization state return quantizer.train(trainingRequest); diff --git a/src/main/java/org/opensearch/knn/index/vectorvalues/KNNVectorValues.java b/src/main/java/org/opensearch/knn/index/vectorvalues/KNNVectorValues.java index 56ebd208f..b12395185 100644 --- a/src/main/java/org/opensearch/knn/index/vectorvalues/KNNVectorValues.java +++ b/src/main/java/org/opensearch/knn/index/vectorvalues/KNNVectorValues.java @@ -71,9 +71,18 @@ public int bytesPerVector() { } /** - * Returns the total live docs for KNNVectorValues. + * Returns the total live docs for KNNVectorValues. This function is broken and doesn't always give the accurate + * live docs count when iterators are {@link FloatVectorValues}, {@link ByteVectorValues}. Avoid using this iterator, + * rather use a simple function like this: + * 
    +     *     int liveDocs = 0;
+     *     while(vectorValues.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) {
+     *         liveDocs++;
+     *     }
+     *
    * @return long */ + @Deprecated public long totalLiveDocs() { return vectorValuesIterator.liveDocs(); } diff --git a/src/test/java/org/opensearch/knn/index/codec/nativeindex/DefaultIndexBuildStrategyTests.java b/src/test/java/org/opensearch/knn/index/codec/nativeindex/DefaultIndexBuildStrategyTests.java index 1a8a832aa..96af0db19 100644 --- a/src/test/java/org/opensearch/knn/index/codec/nativeindex/DefaultIndexBuildStrategyTests.java +++ b/src/test/java/org/opensearch/knn/index/codec/nativeindex/DefaultIndexBuildStrategyTests.java @@ -74,10 +74,12 @@ public void testBuildAndWrite() { .knnEngine(KNNEngine.NMSLIB) .vectorDataType(VectorDataType.FLOAT) .parameters(Map.of("index", "param")) + .vectorValues(knnVectorValues) + .totalLiveDocs((int) knnVectorValues.totalLiveDocs()) .build(); // When - DefaultIndexBuildStrategy.getInstance().buildAndWriteIndex(buildIndexParams, knnVectorValues); + DefaultIndexBuildStrategy.getInstance().buildAndWriteIndex(buildIndexParams); // Then mockedJNIService.verify( @@ -166,10 +168,12 @@ public void testBuildAndWrite_withQuantization() { .vectorDataType(VectorDataType.FLOAT) .parameters(Map.of("index", "param")) .quantizationState(quantizationState) + .vectorValues(knnVectorValues) + .totalLiveDocs((int) knnVectorValues.totalLiveDocs()) .build(); // When - MemOptimizedNativeIndexBuildStrategy.getInstance().buildAndWriteIndex(buildIndexParams, knnVectorValues); + MemOptimizedNativeIndexBuildStrategy.getInstance().buildAndWriteIndex(buildIndexParams); // Then mockedJNIService.verify( @@ -250,10 +254,12 @@ public void testBuildAndWriteWithModel() { .knnEngine(KNNEngine.NMSLIB) .vectorDataType(VectorDataType.FLOAT) .parameters(Map.of("model_id", "id", "model_blob", modelBlob)) + .vectorValues(knnVectorValues) + .totalLiveDocs((int) knnVectorValues.totalLiveDocs()) .build(); // When - DefaultIndexBuildStrategy.getInstance().buildAndWriteIndex(buildIndexParams, knnVectorValues); + DefaultIndexBuildStrategy.getInstance().buildAndWriteIndex(buildIndexParams); // Then mockedJNIService.verify( diff --git a/src/test/java/org/opensearch/knn/index/codec/nativeindex/MemOptimizedNativeIndexBuildStrategyTests.java b/src/test/java/org/opensearch/knn/index/codec/nativeindex/MemOptimizedNativeIndexBuildStrategyTests.java index 81d490bb4..22f9b2dfd 100644 --- a/src/test/java/org/opensearch/knn/index/codec/nativeindex/MemOptimizedNativeIndexBuildStrategyTests.java +++ b/src/test/java/org/opensearch/knn/index/codec/nativeindex/MemOptimizedNativeIndexBuildStrategyTests.java @@ -75,10 +75,12 @@ public void testBuildAndWrite() { .knnEngine(KNNEngine.FAISS) .vectorDataType(VectorDataType.FLOAT) .parameters(Map.of("index", "param")) + .vectorValues(knnVectorValues) + .totalLiveDocs((int) knnVectorValues.totalLiveDocs()) .build(); // When - MemOptimizedNativeIndexBuildStrategy.getInstance().buildAndWriteIndex(buildIndexParams, knnVectorValues); + MemOptimizedNativeIndexBuildStrategy.getInstance().buildAndWriteIndex(buildIndexParams); // Then mockedJNIService.verify( @@ -193,10 +195,12 @@ public void testBuildAndWrite_withQuantization() { .vectorDataType(VectorDataType.FLOAT) .parameters(Map.of("index", "param")) .quantizationState(quantizationState) + .vectorValues(knnVectorValues) + .totalLiveDocs((int) knnVectorValues.totalLiveDocs()) .build(); // When - MemOptimizedNativeIndexBuildStrategy.getInstance().buildAndWriteIndex(buildIndexParams, knnVectorValues); + MemOptimizedNativeIndexBuildStrategy.getInstance().buildAndWriteIndex(buildIndexParams); // Then mockedJNIService.verify( diff --git a/src/test/java/org/opensearch/knn/index/quantizationservice/QuantizationServiceTests.java b/src/test/java/org/opensearch/knn/index/quantizationservice/QuantizationServiceTests.java index 720b67fd5..690391dbd 100644 --- a/src/test/java/org/opensearch/knn/index/quantizationservice/QuantizationServiceTests.java +++ b/src/test/java/org/opensearch/knn/index/quantizationservice/QuantizationServiceTests.java @@ -46,7 +46,7 @@ public void setUp() throws Exception { public void testTrain_oneBitQuantizer_success() throws IOException { ScalarQuantizationParams oneBitParams = new ScalarQuantizationParams(ScalarQuantizationType.ONE_BIT); - QuantizationState quantizationState = quantizationService.train(oneBitParams, knnVectorValues); + QuantizationState quantizationState = quantizationService.train(oneBitParams, knnVectorValues, knnVectorValues.totalLiveDocs()); assertTrue(quantizationState instanceof OneBitScalarQuantizationState); OneBitScalarQuantizationState oneBitState = (OneBitScalarQuantizationState) quantizationState; @@ -62,7 +62,7 @@ public void testTrain_oneBitQuantizer_success() throws IOException { public void testTrain_twoBitQuantizer_success() throws IOException { ScalarQuantizationParams twoBitParams = new ScalarQuantizationParams(ScalarQuantizationType.TWO_BIT); - QuantizationState quantizationState = quantizationService.train(twoBitParams, knnVectorValues); + QuantizationState quantizationState = quantizationService.train(twoBitParams, knnVectorValues, knnVectorValues.totalLiveDocs()); assertTrue(quantizationState instanceof MultiBitScalarQuantizationState); MultiBitScalarQuantizationState multiBitState = (MultiBitScalarQuantizationState) quantizationState; @@ -85,7 +85,7 @@ public void testTrain_twoBitQuantizer_success() throws IOException { public void testTrain_fourBitQuantizer_success() throws IOException { ScalarQuantizationParams fourBitParams = new ScalarQuantizationParams(ScalarQuantizationType.FOUR_BIT); - QuantizationState quantizationState = quantizationService.train(fourBitParams, knnVectorValues); + QuantizationState quantizationState = quantizationService.train(fourBitParams, knnVectorValues, knnVectorValues.totalLiveDocs()); assertTrue(quantizationState instanceof MultiBitScalarQuantizationState); MultiBitScalarQuantizationState multiBitState = (MultiBitScalarQuantizationState) quantizationState; @@ -110,7 +110,7 @@ public void testTrain_fourBitQuantizer_success() throws IOException { public void testQuantize_oneBitQuantizer_success() throws IOException { ScalarQuantizationParams oneBitParams = new ScalarQuantizationParams(ScalarQuantizationType.ONE_BIT); - QuantizationState quantizationState = quantizationService.train(oneBitParams, knnVectorValues); + QuantizationState quantizationState = quantizationService.train(oneBitParams, knnVectorValues, knnVectorValues.totalLiveDocs()); QuantizationOutput quantizationOutput = quantizationService.createQuantizationOutput(oneBitParams); @@ -125,7 +125,7 @@ public void testQuantize_oneBitQuantizer_success() throws IOException { public void testQuantize_twoBitQuantizer_success() throws IOException { ScalarQuantizationParams twoBitParams = new ScalarQuantizationParams(ScalarQuantizationType.TWO_BIT); - QuantizationState quantizationState = quantizationService.train(twoBitParams, knnVectorValues); + QuantizationState quantizationState = quantizationService.train(twoBitParams, knnVectorValues, knnVectorValues.totalLiveDocs()); QuantizationOutput quantizationOutput = quantizationService.createQuantizationOutput(twoBitParams); byte[] quantizedVector = quantizationService.quantize(quantizationState, new float[] { 4.0f, 5.0f, 6.0f }, quantizationOutput); @@ -138,7 +138,7 @@ public void testQuantize_twoBitQuantizer_success() throws IOException { public void testQuantize_fourBitQuantizer_success() throws IOException { ScalarQuantizationParams fourBitParams = new ScalarQuantizationParams(ScalarQuantizationType.FOUR_BIT); - QuantizationState quantizationState = quantizationService.train(fourBitParams, knnVectorValues); + QuantizationState quantizationState = quantizationService.train(fourBitParams, knnVectorValues, knnVectorValues.totalLiveDocs()); QuantizationOutput quantizationOutput = quantizationService.createQuantizationOutput(fourBitParams); byte[] quantizedVector = quantizationService.quantize(quantizationState, new float[] { 7.0f, 8.0f, 9.0f }, quantizationOutput); @@ -152,7 +152,7 @@ public void testQuantize_fourBitQuantizer_success() throws IOException { public void testQuantize_whenInvalidInput_thenThrows() throws IOException { ScalarQuantizationParams oneBitParams = new ScalarQuantizationParams(ScalarQuantizationType.ONE_BIT); - QuantizationState quantizationState = quantizationService.train(oneBitParams, knnVectorValues); + QuantizationState quantizationState = quantizationService.train(oneBitParams, knnVectorValues, knnVectorValues.totalLiveDocs()); QuantizationOutput quantizationOutput = quantizationService.createQuantizationOutput(oneBitParams); assertThrows(IllegalArgumentException.class, () -> quantizationService.quantize(quantizationState, null, quantizationOutput)); } diff --git a/src/test/java/org/opensearch/knn/integ/ModeAndCompressionIT.java b/src/test/java/org/opensearch/knn/integ/ModeAndCompressionIT.java index 26f596b96..af0480078 100644 --- a/src/test/java/org/opensearch/knn/integ/ModeAndCompressionIT.java +++ b/src/test/java/org/opensearch/knn/integ/ModeAndCompressionIT.java @@ -192,6 +192,27 @@ public void testIndexCreation_whenValid_ThenSucceed() { } } + @SneakyThrows + public void testDeletedDocsWithSegmentMerge_whenValid_ThenSucceed() { + XContentBuilder builder; + CompressionLevel compressionLevel = CompressionLevel.x32; + String indexName = INDEX_NAME + compressionLevel; + builder = XContentFactory.jsonBuilder() + .startObject() + .startObject("properties") + .startObject(FIELD_NAME) + .field("type", "knn_vector") + .field("dimension", DIMENSION) + .field(COMPRESSION_LEVEL_PARAMETER, compressionLevel.getName()) + .field(MODE_PARAMETER, Mode.ON_DISK.getName()) + .endObject() + .endObject() + .endObject(); + String mapping = builder.toString(); + validateIndexWithDeletedDocs(indexName, mapping); + validateSearch(indexName, METHOD_PARAMETER_EF_SEARCH, KNNSettings.INDEX_KNN_DEFAULT_ALGO_PARAM_EF_SEARCH); + } + @SneakyThrows public void testTraining_whenInvalid_thenFail() { setupTrainingIndex(); @@ -319,6 +340,18 @@ private void validateIndex(String indexName, String mapping) { forceMergeKnnIndex(indexName, 1); } + @SneakyThrows + private void validateIndexWithDeletedDocs(String indexName, String mapping) { + createKnnIndex(indexName, mapping); + addKNNDocs(indexName, FIELD_NAME, DIMENSION, 0, NUM_DOCS); + refreshIndex(indexName); + // this will simulate the deletion of the docs + addKNNDocs(indexName, FIELD_NAME, DIMENSION, 0, NUM_DOCS); + refreshIndex(indexName); + forceMergeKnnIndex(indexName, 1); + refreshIndex(indexName); + } + @SneakyThrows private void setupTrainingIndex() { createBasicKnnIndex(TRAINING_INDEX_NAME, TRAINING_FIELD_NAME, DIMENSION); From 4dbc1159bd9b254c49dc8a53a2f70c34a4323678 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Sun, 8 Sep 2024 19:07:15 -0700 Subject: [PATCH 356/416] Bounds the transferLimit in OffheapVectorTransfer (#2070) (#2073) The array list buffer was unnecessarily allocated a large memory irrespective of the vectors to transfer. This change considers total vectors Signed-off-by: Tejas Shah (cherry picked from commit 0492fb350fa3e398871b4a3b477a7eea921818b1) Co-authored-by: Tejas Shah --- .../DefaultIndexBuildStrategy.java | 10 +- .../MemOptimizedNativeIndexBuildStrategy.java | 12 ++- .../transfer/OffHeapBinaryVectorTransfer.java | 4 +- .../transfer/OffHeapByteVectorTransfer.java | 4 +- .../transfer/OffHeapFloatVectorTransfer.java | 4 +- .../codec/transfer/OffHeapVectorTransfer.java | 17 ++- .../OffHeapVectorTransferFactory.java | 15 ++- .../DefaultIndexBuildStrategyTests.java | 11 +- ...ptimizedNativeIndexBuildStrategyTests.java | 12 +-- .../OffHeapVectorTransferFactoryTests.java | 26 +++-- .../transfer/OffHeapVectorTransferTests.java | 101 ++++++++++-------- 11 files changed, 126 insertions(+), 90 deletions(-) diff --git a/src/main/java/org/opensearch/knn/index/codec/nativeindex/DefaultIndexBuildStrategy.java b/src/main/java/org/opensearch/knn/index/codec/nativeindex/DefaultIndexBuildStrategy.java index e68121a7d..476c95b8d 100644 --- a/src/main/java/org/opensearch/knn/index/codec/nativeindex/DefaultIndexBuildStrategy.java +++ b/src/main/java/org/opensearch/knn/index/codec/nativeindex/DefaultIndexBuildStrategy.java @@ -8,7 +8,6 @@ import lombok.AccessLevel; import lombok.NoArgsConstructor; import org.opensearch.knn.common.KNNConstants; -import org.opensearch.knn.index.KNNSettings; import org.opensearch.knn.index.codec.nativeindex.model.BuildIndexParams; import org.opensearch.knn.index.codec.transfer.OffHeapVectorTransfer; import org.opensearch.knn.index.vectorvalues.KNNVectorValues; @@ -56,8 +55,13 @@ public void buildAndWriteIndex(final BuildIndexParams indexInfo) throws IOExcept iterateVectorValuesOnce(knnVectorValues); IndexBuildSetup indexBuildSetup = QuantizationIndexUtils.prepareIndexBuild(knnVectorValues, indexInfo); - int transferLimit = (int) Math.max(1, KNNSettings.getVectorStreamingMemoryLimit().getBytes() / indexBuildSetup.getBytesPerVector()); - try (final OffHeapVectorTransfer vectorTransfer = getVectorTransfer(indexInfo.getVectorDataType(), transferLimit)) { + try ( + final OffHeapVectorTransfer vectorTransfer = getVectorTransfer( + indexInfo.getVectorDataType(), + indexBuildSetup.getBytesPerVector(), + indexInfo.getTotalLiveDocs() + ) + ) { final List transferredDocIds = new ArrayList<>(indexInfo.getTotalLiveDocs()); while (knnVectorValues.docId() != NO_MORE_DOCS) { diff --git a/src/main/java/org/opensearch/knn/index/codec/nativeindex/MemOptimizedNativeIndexBuildStrategy.java b/src/main/java/org/opensearch/knn/index/codec/nativeindex/MemOptimizedNativeIndexBuildStrategy.java index af3f4777f..b7e337081 100644 --- a/src/main/java/org/opensearch/knn/index/codec/nativeindex/MemOptimizedNativeIndexBuildStrategy.java +++ b/src/main/java/org/opensearch/knn/index/codec/nativeindex/MemOptimizedNativeIndexBuildStrategy.java @@ -7,7 +7,6 @@ import lombok.AccessLevel; import lombok.NoArgsConstructor; -import org.opensearch.knn.index.KNNSettings; import org.opensearch.knn.index.codec.nativeindex.model.BuildIndexParams; import org.opensearch.knn.index.codec.transfer.OffHeapVectorTransfer; import org.opensearch.knn.index.engine.KNNEngine; @@ -70,10 +69,15 @@ public void buildAndWriteIndex(final BuildIndexParams indexInfo) throws IOExcept ) ); - int transferLimit = (int) Math.max(1, KNNSettings.getVectorStreamingMemoryLimit().getBytes() / indexBuildSetup.getBytesPerVector()); - try (final OffHeapVectorTransfer vectorTransfer = getVectorTransfer(indexInfo.getVectorDataType(), transferLimit)) { + try ( + final OffHeapVectorTransfer vectorTransfer = getVectorTransfer( + indexInfo.getVectorDataType(), + indexBuildSetup.getBytesPerVector(), + indexInfo.getTotalLiveDocs() + ) + ) { - final List transferredDocIds = new ArrayList<>(transferLimit); + final List transferredDocIds = new ArrayList<>(vectorTransfer.getTransferLimit()); while (knnVectorValues.docId() != NO_MORE_DOCS) { Object vector = QuantizationIndexUtils.processAndReturnVector(knnVectorValues, indexBuildSetup); diff --git a/src/main/java/org/opensearch/knn/index/codec/transfer/OffHeapBinaryVectorTransfer.java b/src/main/java/org/opensearch/knn/index/codec/transfer/OffHeapBinaryVectorTransfer.java index ffa12a231..964007fc0 100644 --- a/src/main/java/org/opensearch/knn/index/codec/transfer/OffHeapBinaryVectorTransfer.java +++ b/src/main/java/org/opensearch/knn/index/codec/transfer/OffHeapBinaryVectorTransfer.java @@ -17,8 +17,8 @@ */ public final class OffHeapBinaryVectorTransfer extends OffHeapVectorTransfer { - public OffHeapBinaryVectorTransfer(int transferLimit) { - super(transferLimit); + public OffHeapBinaryVectorTransfer(int bytesPerVector, int totalVectorsToTransfer) { + super(bytesPerVector, totalVectorsToTransfer); } @Override diff --git a/src/main/java/org/opensearch/knn/index/codec/transfer/OffHeapByteVectorTransfer.java b/src/main/java/org/opensearch/knn/index/codec/transfer/OffHeapByteVectorTransfer.java index 83ebf2fa3..16e333478 100644 --- a/src/main/java/org/opensearch/knn/index/codec/transfer/OffHeapByteVectorTransfer.java +++ b/src/main/java/org/opensearch/knn/index/codec/transfer/OffHeapByteVectorTransfer.java @@ -17,8 +17,8 @@ */ public final class OffHeapByteVectorTransfer extends OffHeapVectorTransfer { - public OffHeapByteVectorTransfer(int transferLimit) { - super(transferLimit); + public OffHeapByteVectorTransfer(int bytesPerVector, int totalVectorsToTransfer) { + super(bytesPerVector, totalVectorsToTransfer); } @Override diff --git a/src/main/java/org/opensearch/knn/index/codec/transfer/OffHeapFloatVectorTransfer.java b/src/main/java/org/opensearch/knn/index/codec/transfer/OffHeapFloatVectorTransfer.java index 0eb28d791..767f57271 100644 --- a/src/main/java/org/opensearch/knn/index/codec/transfer/OffHeapFloatVectorTransfer.java +++ b/src/main/java/org/opensearch/knn/index/codec/transfer/OffHeapFloatVectorTransfer.java @@ -15,8 +15,8 @@ */ public final class OffHeapFloatVectorTransfer extends OffHeapVectorTransfer { - public OffHeapFloatVectorTransfer(int transferLimit) { - super(transferLimit); + public OffHeapFloatVectorTransfer(int bytesPerVector, int totalVectorsToTransfer) { + super(bytesPerVector, totalVectorsToTransfer); } @Override diff --git a/src/main/java/org/opensearch/knn/index/codec/transfer/OffHeapVectorTransfer.java b/src/main/java/org/opensearch/knn/index/codec/transfer/OffHeapVectorTransfer.java index 43c27c8da..8a248e06c 100644 --- a/src/main/java/org/opensearch/knn/index/codec/transfer/OffHeapVectorTransfer.java +++ b/src/main/java/org/opensearch/knn/index/codec/transfer/OffHeapVectorTransfer.java @@ -6,6 +6,7 @@ package org.opensearch.knn.index.codec.transfer; import lombok.Getter; +import org.opensearch.knn.index.KNNSettings; import org.opensearch.knn.index.vectorvalues.KNNVectorValues; import java.io.Closeable; @@ -27,16 +28,22 @@ public abstract class OffHeapVectorTransfer implements Closeable { @Getter private long vectorAddress; + @Getter protected final int transferLimit; - private final List vectorsToTransfer; + private List vectorsToTransfer; - public OffHeapVectorTransfer(final int transferLimit) { - this.transferLimit = transferLimit; - this.vectorsToTransfer = new ArrayList<>(transferLimit); + public OffHeapVectorTransfer(int bytesPerVector, int totalVectorsToTransfer) { + this.transferLimit = computeTransferLimit(bytesPerVector, totalVectorsToTransfer); + this.vectorsToTransfer = new ArrayList<>(this.transferLimit); this.vectorAddress = 0; } + private int computeTransferLimit(int bytesPerVector, int totalVectorsToTransfer) { + int limit = (int) Math.max(1, KNNSettings.getVectorStreamingMemoryLimit().getBytes() / bytesPerVector); + return Math.min(limit, totalVectorsToTransfer); + } + /** * Transfer vectors to off-heap * @param vector float[] or byte[] @@ -90,7 +97,7 @@ public void close() { */ public void reset() { vectorAddress = 0; - vectorsToTransfer.clear(); + vectorsToTransfer = null; } protected abstract void deallocate(); diff --git a/src/main/java/org/opensearch/knn/index/codec/transfer/OffHeapVectorTransferFactory.java b/src/main/java/org/opensearch/knn/index/codec/transfer/OffHeapVectorTransferFactory.java index 446b6ae80..3bc55f7fa 100644 --- a/src/main/java/org/opensearch/knn/index/codec/transfer/OffHeapVectorTransferFactory.java +++ b/src/main/java/org/opensearch/knn/index/codec/transfer/OffHeapVectorTransferFactory.java @@ -18,18 +18,23 @@ public final class OffHeapVectorTransferFactory { /** * Gets the right vector transfer object based on vector data type * @param vectorDataType {@link VectorDataType} - * @param transferLimit max number of vectors that can be transferred to off heap in one transfer + * @param bytesPerVector Bytes used per vector + * @param totalVectorsToTransfer total number of vectors that will be transferred off heap * @return Correct implementation of {@link OffHeapVectorTransfer} * @param float[] or byte[] */ - public static OffHeapVectorTransfer getVectorTransfer(final VectorDataType vectorDataType, final int transferLimit) { + public static OffHeapVectorTransfer getVectorTransfer( + final VectorDataType vectorDataType, + int bytesPerVector, + int totalVectorsToTransfer + ) { switch (vectorDataType) { case FLOAT: - return (OffHeapVectorTransfer) new OffHeapFloatVectorTransfer(transferLimit); + return (OffHeapVectorTransfer) new OffHeapFloatVectorTransfer(bytesPerVector, totalVectorsToTransfer); case BINARY: - return (OffHeapVectorTransfer) new OffHeapBinaryVectorTransfer(transferLimit); + return (OffHeapVectorTransfer) new OffHeapBinaryVectorTransfer(bytesPerVector, totalVectorsToTransfer); case BYTE: - return (OffHeapVectorTransfer) new OffHeapByteVectorTransfer(transferLimit); + return (OffHeapVectorTransfer) new OffHeapByteVectorTransfer(bytesPerVector, totalVectorsToTransfer); default: throw new IllegalArgumentException("Unsupported vector data type: " + vectorDataType); } diff --git a/src/test/java/org/opensearch/knn/index/codec/nativeindex/DefaultIndexBuildStrategyTests.java b/src/test/java/org/opensearch/knn/index/codec/nativeindex/DefaultIndexBuildStrategyTests.java index 96af0db19..9c2e5a4b7 100644 --- a/src/test/java/org/opensearch/knn/index/codec/nativeindex/DefaultIndexBuildStrategyTests.java +++ b/src/test/java/org/opensearch/knn/index/codec/nativeindex/DefaultIndexBuildStrategyTests.java @@ -57,14 +57,11 @@ public void testBuildAndWrite() { final KNNVectorValues knnVectorValues = KNNVectorValuesFactory.getVectorValues(VectorDataType.FLOAT, randomVectorValues); try ( - MockedStatic mockedKNNSettings = mockStatic(KNNSettings.class); MockedStatic mockedJNIService = mockStatic(JNIService.class); MockedStatic mockedOffHeapVectorTransferFactory = mockStatic(OffHeapVectorTransferFactory.class) ) { - - mockedKNNSettings.when(KNNSettings::getVectorStreamingMemoryLimit).thenReturn(new ByteSizeValue(16)); OffHeapVectorTransfer offHeapVectorTransfer = mock(OffHeapVectorTransfer.class); - mockedOffHeapVectorTransferFactory.when(() -> OffHeapVectorTransferFactory.getVectorTransfer(VectorDataType.FLOAT, 2)) + mockedOffHeapVectorTransferFactory.when(() -> OffHeapVectorTransferFactory.getVectorTransfer(VectorDataType.FLOAT, 8, 3)) .thenReturn(offHeapVectorTransfer); when(offHeapVectorTransfer.getVectorAddress()).thenReturn(200L); @@ -131,7 +128,7 @@ public void testBuildAndWrite_withQuantization() { mockedJNIService.when(() -> JNIService.initIndex(3, 2, Map.of("index", "param"), KNNEngine.FAISS)).thenReturn(100L); OffHeapVectorTransfer offHeapVectorTransfer = mock(OffHeapVectorTransfer.class); - mockedOffHeapVectorTransferFactory.when(() -> OffHeapVectorTransferFactory.getVectorTransfer(VectorDataType.FLOAT, 2)) + mockedOffHeapVectorTransferFactory.when(() -> OffHeapVectorTransferFactory.getVectorTransfer(VectorDataType.FLOAT, 8, 3)) .thenReturn(offHeapVectorTransfer); QuantizationService quantizationService = mock(QuantizationService.class); @@ -237,14 +234,12 @@ public void testBuildAndWriteWithModel() { docs ); try ( - MockedStatic mockedKNNSettings = mockStatic(KNNSettings.class); MockedStatic mockedJNIService = mockStatic(JNIService.class); MockedStatic mockedOffHeapVectorTransferFactory = mockStatic(OffHeapVectorTransferFactory.class) ) { - mockedKNNSettings.when(KNNSettings::getVectorStreamingMemoryLimit).thenReturn(new ByteSizeValue(16)); OffHeapVectorTransfer offHeapVectorTransfer = mock(OffHeapVectorTransfer.class); - mockedOffHeapVectorTransferFactory.when(() -> OffHeapVectorTransferFactory.getVectorTransfer(VectorDataType.FLOAT, 2)) + mockedOffHeapVectorTransferFactory.when(() -> OffHeapVectorTransferFactory.getVectorTransfer(VectorDataType.FLOAT, 8, 3)) .thenReturn(offHeapVectorTransfer); when(offHeapVectorTransfer.getVectorAddress()).thenReturn(200L); diff --git a/src/test/java/org/opensearch/knn/index/codec/nativeindex/MemOptimizedNativeIndexBuildStrategyTests.java b/src/test/java/org/opensearch/knn/index/codec/nativeindex/MemOptimizedNativeIndexBuildStrategyTests.java index 22f9b2dfd..62c3b7a71 100644 --- a/src/test/java/org/opensearch/knn/index/codec/nativeindex/MemOptimizedNativeIndexBuildStrategyTests.java +++ b/src/test/java/org/opensearch/knn/index/codec/nativeindex/MemOptimizedNativeIndexBuildStrategyTests.java @@ -9,8 +9,6 @@ import org.mockito.ArgumentCaptor; import org.mockito.MockedStatic; import org.mockito.Mockito; -import org.opensearch.core.common.unit.ByteSizeValue; -import org.opensearch.knn.index.KNNSettings; import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.codec.nativeindex.model.BuildIndexParams; import org.opensearch.knn.index.codec.transfer.OffHeapVectorTransfer; @@ -49,7 +47,6 @@ public void testBuildAndWrite() { final KNNVectorValues knnVectorValues = KNNVectorValuesFactory.getVectorValues(VectorDataType.FLOAT, randomVectorValues); try ( - MockedStatic mockedKNNSettings = Mockito.mockStatic(KNNSettings.class); MockedStatic mockedJNIService = Mockito.mockStatic(JNIService.class); MockedStatic mockedOffHeapVectorTransferFactory = Mockito.mockStatic( OffHeapVectorTransferFactory.class @@ -57,13 +54,13 @@ public void testBuildAndWrite() { ) { // Limits transfer to 2 vectors - mockedKNNSettings.when(KNNSettings::getVectorStreamingMemoryLimit).thenReturn(new ByteSizeValue(16)); mockedJNIService.when(() -> JNIService.initIndex(3, 2, Map.of("index", "param"), KNNEngine.FAISS)).thenReturn(100L); OffHeapVectorTransfer offHeapVectorTransfer = mock(OffHeapVectorTransfer.class); - mockedOffHeapVectorTransferFactory.when(() -> OffHeapVectorTransferFactory.getVectorTransfer(VectorDataType.FLOAT, 2)) + mockedOffHeapVectorTransferFactory.when(() -> OffHeapVectorTransferFactory.getVectorTransfer(VectorDataType.FLOAT, 8, 3)) .thenReturn(offHeapVectorTransfer); + when(offHeapVectorTransfer.getTransferLimit()).thenReturn(2); when(offHeapVectorTransfer.transfer(vectorTransferCapture.capture(), eq(false))).thenReturn(false) .thenReturn(true) .thenReturn(false); @@ -145,7 +142,6 @@ public void testBuildAndWrite_withQuantization() { final KNNVectorValues knnVectorValues = KNNVectorValuesFactory.getVectorValues(VectorDataType.FLOAT, randomVectorValues); try ( - MockedStatic mockedKNNSettings = Mockito.mockStatic(KNNSettings.class); MockedStatic mockedJNIService = Mockito.mockStatic(JNIService.class); MockedStatic mockedOffHeapVectorTransferFactory = Mockito.mockStatic( OffHeapVectorTransferFactory.class @@ -154,11 +150,11 @@ public void testBuildAndWrite_withQuantization() { ) { // Limits transfer to 2 vectors - mockedKNNSettings.when(KNNSettings::getVectorStreamingMemoryLimit).thenReturn(new ByteSizeValue(16)); mockedJNIService.when(() -> JNIService.initIndex(3, 2, Map.of("index", "param"), KNNEngine.FAISS)).thenReturn(100L); OffHeapVectorTransfer offHeapVectorTransfer = mock(OffHeapVectorTransfer.class); - mockedOffHeapVectorTransferFactory.when(() -> OffHeapVectorTransferFactory.getVectorTransfer(VectorDataType.FLOAT, 2)) + when(offHeapVectorTransfer.getTransferLimit()).thenReturn(2); + mockedOffHeapVectorTransferFactory.when(() -> OffHeapVectorTransferFactory.getVectorTransfer(VectorDataType.FLOAT, 8, 3)) .thenReturn(offHeapVectorTransfer); QuantizationService quantizationService = mock(QuantizationService.class); diff --git a/src/test/java/org/opensearch/knn/index/codec/transfer/OffHeapVectorTransferFactoryTests.java b/src/test/java/org/opensearch/knn/index/codec/transfer/OffHeapVectorTransferFactoryTests.java index 39415d811..09984ba46 100644 --- a/src/test/java/org/opensearch/knn/index/codec/transfer/OffHeapVectorTransferFactoryTests.java +++ b/src/test/java/org/opensearch/knn/index/codec/transfer/OffHeapVectorTransferFactoryTests.java @@ -5,22 +5,30 @@ package org.opensearch.knn.index.codec.transfer; +import org.mockito.MockedStatic; +import org.opensearch.core.common.unit.ByteSizeValue; +import org.opensearch.knn.index.KNNSettings; import org.opensearch.knn.index.VectorDataType; import org.opensearch.test.OpenSearchTestCase; +import static org.mockito.Mockito.mockStatic; + public class OffHeapVectorTransferFactoryTests extends OpenSearchTestCase { public void testOffHeapVectorTransferFactory() { - var floatVectorTransfer = OffHeapVectorTransferFactory.getVectorTransfer(VectorDataType.FLOAT, 10); - assertEquals(OffHeapFloatVectorTransfer.class, floatVectorTransfer.getClass()); - assertNotSame(floatVectorTransfer, OffHeapVectorTransferFactory.getVectorTransfer(VectorDataType.FLOAT, 10)); + try (MockedStatic mockedKNNSettings = mockStatic(KNNSettings.class)) { + mockedKNNSettings.when(KNNSettings::getVectorStreamingMemoryLimit).thenReturn(new ByteSizeValue(16)); + var floatVectorTransfer = OffHeapVectorTransferFactory.getVectorTransfer(VectorDataType.FLOAT, 10, 10); + assertEquals(OffHeapFloatVectorTransfer.class, floatVectorTransfer.getClass()); + assertNotSame(floatVectorTransfer, OffHeapVectorTransferFactory.getVectorTransfer(VectorDataType.FLOAT, 10, 10)); - var byteVectorTransfer = OffHeapVectorTransferFactory.getVectorTransfer(VectorDataType.BYTE, 10); - assertEquals(OffHeapByteVectorTransfer.class, byteVectorTransfer.getClass()); - assertNotSame(byteVectorTransfer, OffHeapVectorTransferFactory.getVectorTransfer(VectorDataType.BYTE, 10)); + var byteVectorTransfer = OffHeapVectorTransferFactory.getVectorTransfer(VectorDataType.BYTE, 10, 10); + assertEquals(OffHeapByteVectorTransfer.class, byteVectorTransfer.getClass()); + assertNotSame(byteVectorTransfer, OffHeapVectorTransferFactory.getVectorTransfer(VectorDataType.BYTE, 10, 10)); - var binaryVectorTransfer = OffHeapVectorTransferFactory.getVectorTransfer(VectorDataType.BINARY, 10); - assertEquals(OffHeapBinaryVectorTransfer.class, binaryVectorTransfer.getClass()); - assertNotSame(binaryVectorTransfer, OffHeapVectorTransferFactory.getVectorTransfer(VectorDataType.BINARY, 10)); + var binaryVectorTransfer = OffHeapVectorTransferFactory.getVectorTransfer(VectorDataType.BINARY, 10, 10); + assertEquals(OffHeapBinaryVectorTransfer.class, binaryVectorTransfer.getClass()); + assertNotSame(binaryVectorTransfer, OffHeapVectorTransferFactory.getVectorTransfer(VectorDataType.BINARY, 10, 10)); + } } } diff --git a/src/test/java/org/opensearch/knn/index/codec/transfer/OffHeapVectorTransferTests.java b/src/test/java/org/opensearch/knn/index/codec/transfer/OffHeapVectorTransferTests.java index f1650db8f..fb2ef274e 100644 --- a/src/test/java/org/opensearch/knn/index/codec/transfer/OffHeapVectorTransferTests.java +++ b/src/test/java/org/opensearch/knn/index/codec/transfer/OffHeapVectorTransferTests.java @@ -6,10 +6,15 @@ package org.opensearch.knn.index.codec.transfer; import lombok.SneakyThrows; +import org.mockito.MockedStatic; +import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.knn.KNNTestCase; +import org.opensearch.knn.index.KNNSettings; import java.util.List; +import static org.mockito.Mockito.mockStatic; + public class OffHeapVectorTransferTests extends KNNTestCase { @SneakyThrows @@ -22,21 +27,27 @@ public void testFloatTransfer() { new float[] { 0.3f, 0.4f } ); - OffHeapFloatVectorTransfer vectorTransfer = new OffHeapFloatVectorTransfer(2); - long vectorAddress = 0; - assertFalse(vectorTransfer.transfer(vectors.get(0), false)); - assertEquals(0, vectorTransfer.getVectorAddress()); - assertTrue(vectorTransfer.transfer(vectors.get(1), false)); - vectorAddress = vectorTransfer.getVectorAddress(); - assertFalse(vectorTransfer.transfer(vectors.get(2), false)); - assertEquals(vectorAddress, vectorTransfer.getVectorAddress()); - assertTrue(vectorTransfer.transfer(vectors.get(3), false)); - assertEquals(vectorAddress, vectorTransfer.getVectorAddress()); - assertFalse(vectorTransfer.transfer(vectors.get(4), false)); - assertTrue(vectorTransfer.flush(false)); - vectorTransfer.reset(); - assertEquals(0, vectorTransfer.getVectorAddress()); - vectorTransfer.close(); + try (MockedStatic mockedKNNSettings = mockStatic(KNNSettings.class)) { + mockedKNNSettings.when(KNNSettings::getVectorStreamingMemoryLimit).thenReturn(new ByteSizeValue(16)); + + OffHeapFloatVectorTransfer vectorTransfer = new OffHeapFloatVectorTransfer(8, 5); + long vectorAddress = 0; + assertFalse(vectorTransfer.transfer(vectors.get(0), false)); + assertEquals(0, vectorTransfer.getVectorAddress()); + assertTrue(vectorTransfer.transfer(vectors.get(1), false)); + vectorAddress = vectorTransfer.getVectorAddress(); + assertFalse(vectorTransfer.transfer(vectors.get(2), false)); + assertEquals(vectorAddress, vectorTransfer.getVectorAddress()); + assertTrue(vectorTransfer.transfer(vectors.get(3), false)); + assertEquals(vectorAddress, vectorTransfer.getVectorAddress()); + assertFalse(vectorTransfer.transfer(vectors.get(4), false)); + assertTrue(vectorTransfer.flush(false)); + vectorTransfer.reset(); + assertEquals(0, vectorTransfer.getVectorAddress()); + vectorTransfer.close(); + + } + } @SneakyThrows @@ -49,20 +60,23 @@ public void testByteTransfer() { new byte[] { 8, 9 } ); - OffHeapByteVectorTransfer vectorTransfer = new OffHeapByteVectorTransfer(2); - long vectorAddress = 0; - assertFalse(vectorTransfer.transfer(vectors.get(0), false)); - assertEquals(0, vectorTransfer.getVectorAddress()); - assertTrue(vectorTransfer.transfer(vectors.get(1), false)); - vectorAddress = vectorTransfer.getVectorAddress(); - assertFalse(vectorTransfer.transfer(vectors.get(2), false)); - assertEquals(vectorAddress, vectorTransfer.getVectorAddress()); - assertTrue(vectorTransfer.transfer(vectors.get(3), false)); - assertEquals(vectorAddress, vectorTransfer.getVectorAddress()); - assertFalse(vectorTransfer.transfer(vectors.get(4), false)); - assertTrue(vectorTransfer.flush(false)); - vectorTransfer.close(); - assertEquals(0, vectorTransfer.getVectorAddress()); + try (MockedStatic mockedKNNSettings = mockStatic(KNNSettings.class)) { + mockedKNNSettings.when(KNNSettings::getVectorStreamingMemoryLimit).thenReturn(new ByteSizeValue(4)); + OffHeapByteVectorTransfer vectorTransfer = new OffHeapByteVectorTransfer(2, 5); + long vectorAddress = 0; + assertFalse(vectorTransfer.transfer(vectors.get(0), false)); + assertEquals(0, vectorTransfer.getVectorAddress()); + assertTrue(vectorTransfer.transfer(vectors.get(1), false)); + vectorAddress = vectorTransfer.getVectorAddress(); + assertFalse(vectorTransfer.transfer(vectors.get(2), false)); + assertEquals(vectorAddress, vectorTransfer.getVectorAddress()); + assertTrue(vectorTransfer.transfer(vectors.get(3), false)); + assertEquals(vectorAddress, vectorTransfer.getVectorAddress()); + assertFalse(vectorTransfer.transfer(vectors.get(4), false)); + assertTrue(vectorTransfer.flush(false)); + vectorTransfer.close(); + assertEquals(0, vectorTransfer.getVectorAddress()); + } } @SneakyThrows @@ -75,18 +89,21 @@ public void testBinaryTransfer() { new byte[] { 8, 9 } ); - OffHeapBinaryVectorTransfer vectorTransfer = new OffHeapBinaryVectorTransfer(2); - long vectorAddress = 0; - assertFalse(vectorTransfer.transfer(vectors.get(0), false)); - assertEquals(0, vectorTransfer.getVectorAddress()); - assertTrue(vectorTransfer.transfer(vectors.get(1), false)); - vectorAddress = vectorTransfer.getVectorAddress(); - assertFalse(vectorTransfer.transfer(vectors.get(2), false)); - assertEquals(vectorAddress, vectorTransfer.getVectorAddress()); - assertTrue(vectorTransfer.transfer(vectors.get(3), false)); - assertEquals(vectorAddress, vectorTransfer.getVectorAddress()); - assertFalse(vectorTransfer.transfer(vectors.get(4), false)); - assertTrue(vectorTransfer.flush(false)); - vectorTransfer.close(); + try (MockedStatic mockedKNNSettings = mockStatic(KNNSettings.class)) { + mockedKNNSettings.when(KNNSettings::getVectorStreamingMemoryLimit).thenReturn(new ByteSizeValue(4)); + OffHeapBinaryVectorTransfer vectorTransfer = new OffHeapBinaryVectorTransfer(2, 5); + long vectorAddress = 0; + assertFalse(vectorTransfer.transfer(vectors.get(0), false)); + assertEquals(0, vectorTransfer.getVectorAddress()); + assertTrue(vectorTransfer.transfer(vectors.get(1), false)); + vectorAddress = vectorTransfer.getVectorAddress(); + assertFalse(vectorTransfer.transfer(vectors.get(2), false)); + assertEquals(vectorAddress, vectorTransfer.getVectorAddress()); + assertTrue(vectorTransfer.transfer(vectors.get(3), false)); + assertEquals(vectorAddress, vectorTransfer.getVectorAddress()); + assertFalse(vectorTransfer.transfer(vectors.get(4), false)); + assertTrue(vectorTransfer.flush(false)); + vectorTransfer.close(); + } } } From 9ddfed511c5a354121bd9b035e96d86f4c0dc123 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Sun, 8 Sep 2024 19:14:54 -0700 Subject: [PATCH 357/416] Add tests for top level spaceType parameter with different combinations (#2058) (#2060) Signed-off-by: Navneet Verma (cherry picked from commit 71532c21b3018b02c0c99f2b15f430f30005a9bb) Co-authored-by: Navneet Verma --- .../mapper/KNNVectorFieldMapperTests.java | 180 ++++++++++++++++++ .../integ/TopLevelSpaceTypeParameterIT.java | 106 +++++++++++ 2 files changed, 286 insertions(+) create mode 100644 src/test/java/org/opensearch/knn/integ/TopLevelSpaceTypeParameterIT.java diff --git a/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java b/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java index abc4f563e..35e027666 100644 --- a/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java +++ b/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java @@ -12,6 +12,7 @@ import org.apache.lucene.index.IndexableField; import org.apache.lucene.index.VectorEncoding; import org.apache.lucene.util.BytesRef; +import org.junit.Assert; import org.mockito.MockedStatic; import org.mockito.Mockito; import org.opensearch.cluster.metadata.IndexMetadata; @@ -192,6 +193,156 @@ public void testTypeParser_build_fromKnnMethodContext() throws IOException { assertTrue(knnVectorFieldMapper.fieldType().getKnnMappingConfig().getModelId().isEmpty()); } + public void testTypeParser_withDifferentSpaceTypeCombinations_thenSuccess() throws IOException { + // Check that knnMethodContext takes precedent over both model and legacy + ModelDao modelDao = mock(ModelDao.class); + int mForSetting = 71; + // Setup settings + Settings settings = Settings.builder() + .put(settings(CURRENT).build()) + .put(KNNSettings.KNN_ALGO_PARAM_M, mForSetting) + .put(KNN_INDEX, true) + .build(); + SpaceType methodSpaceType = SpaceType.COSINESIMIL; + SpaceType topLevelSpaceType = SpaceType.INNER_PRODUCT; + KNNVectorFieldMapper.TypeParser typeParser = new KNNVectorFieldMapper.TypeParser(() -> modelDao); + + // space type provided at top level but not in the method + XContentBuilder xContentBuilder = createXContentForFieldMapping(topLevelSpaceType, null, null, null, TEST_DIMENSION); + + KNNVectorFieldMapper.Builder builder = (KNNVectorFieldMapper.Builder) typeParser.parse( + "test-field-name-1", + xContentBuilderToMap(xContentBuilder), + buildParserContext("test", settings) + ); + + Mapper.BuilderContext builderContext = new Mapper.BuilderContext(settings, new ContentPath()); + KNNVectorFieldMapper knnVectorFieldMapper = builder.build(builderContext); + assertTrue(knnVectorFieldMapper instanceof MethodFieldMapper); + assertTrue(knnVectorFieldMapper.fieldType().getKnnMappingConfig().getKnnMethodContext().isPresent()); + assertEquals(topLevelSpaceType, knnVectorFieldMapper.fieldType().getKnnMappingConfig().getKnnMethodContext().get().getSpaceType()); + assertTrue(knnVectorFieldMapper.fieldType().getKnnMappingConfig().getModelId().isEmpty()); + + // not setting any space type + xContentBuilder = createXContentForFieldMapping(null, null, null, null, TEST_DIMENSION); + + builder = (KNNVectorFieldMapper.Builder) typeParser.parse( + "test-field-name-1", + xContentBuilderToMap(xContentBuilder), + buildParserContext("test", settings) + ); + + builderContext = new Mapper.BuilderContext(settings, new ContentPath()); + knnVectorFieldMapper = builder.build(builderContext); + assertTrue(knnVectorFieldMapper instanceof MethodFieldMapper); + assertTrue(knnVectorFieldMapper.fieldType().getKnnMappingConfig().getKnnMethodContext().isPresent()); + assertEquals(SpaceType.DEFAULT, knnVectorFieldMapper.fieldType().getKnnMappingConfig().getKnnMethodContext().get().getSpaceType()); + assertTrue(knnVectorFieldMapper.fieldType().getKnnMappingConfig().getModelId().isEmpty()); + + // if space types are same + xContentBuilder = createXContentForFieldMapping(topLevelSpaceType, topLevelSpaceType, null, null, TEST_DIMENSION); + builder = (KNNVectorFieldMapper.Builder) typeParser.parse( + "test-field-name-1", + xContentBuilderToMap(xContentBuilder), + buildParserContext("test", settings) + ); + + builderContext = new Mapper.BuilderContext(settings, new ContentPath()); + knnVectorFieldMapper = builder.build(builderContext); + assertTrue(knnVectorFieldMapper instanceof MethodFieldMapper); + assertTrue(knnVectorFieldMapper.fieldType().getKnnMappingConfig().getKnnMethodContext().isPresent()); + assertEquals(topLevelSpaceType, knnVectorFieldMapper.fieldType().getKnnMappingConfig().getKnnMethodContext().get().getSpaceType()); + assertTrue(knnVectorFieldMapper.fieldType().getKnnMappingConfig().getModelId().isEmpty()); + + // if space types are not same + xContentBuilder = createXContentForFieldMapping(topLevelSpaceType, methodSpaceType, null, null, TEST_DIMENSION); + + XContentBuilder finalXContentBuilder = xContentBuilder; + Assert.assertThrows( + MapperParsingException.class, + () -> typeParser.parse("test-field-name-1", xContentBuilderToMap(finalXContentBuilder), buildParserContext("test", settings)) + ); + + // if space types not provided and field is binary + xContentBuilder = createXContentForFieldMapping(null, null, KNNEngine.FAISS, VectorDataType.BINARY, 8); + builder = (KNNVectorFieldMapper.Builder) typeParser.parse( + "test-field-name-1", + xContentBuilderToMap(xContentBuilder), + buildParserContext("test", settings) + ); + + builderContext = new Mapper.BuilderContext(settings, new ContentPath()); + knnVectorFieldMapper = builder.build(builderContext); + assertTrue(knnVectorFieldMapper instanceof MethodFieldMapper); + assertTrue(knnVectorFieldMapper.fieldType().getKnnMappingConfig().getKnnMethodContext().isPresent()); + assertEquals( + SpaceType.DEFAULT_BINARY, + knnVectorFieldMapper.fieldType().getKnnMappingConfig().getKnnMethodContext().get().getSpaceType() + ); + assertTrue(knnVectorFieldMapper.fieldType().getKnnMappingConfig().getModelId().isEmpty()); + + // if space type is provided and legacy mappings is hit + xContentBuilder = XContentFactory.jsonBuilder() + .startObject() + .field(TYPE_FIELD_NAME, KNN_VECTOR_TYPE) + .field(DIMENSION_FIELD_NAME, TEST_DIMENSION) + .field(KNNConstants.TOP_LEVEL_PARAMETER_SPACE_TYPE, topLevelSpaceType.getValue()) + .endObject(); + builder = (KNNVectorFieldMapper.Builder) typeParser.parse( + "test-field-name-1", + xContentBuilderToMap(xContentBuilder), + buildParserContext("test", settings) + ); + + builderContext = new Mapper.BuilderContext(settings, new ContentPath()); + knnVectorFieldMapper = builder.build(builderContext); + assertTrue(knnVectorFieldMapper instanceof MethodFieldMapper); + assertTrue(knnVectorFieldMapper.fieldType().getKnnMappingConfig().getKnnMethodContext().isPresent()); + assertEquals(topLevelSpaceType, knnVectorFieldMapper.fieldType().getKnnMappingConfig().getKnnMethodContext().get().getSpaceType()); + // this check ensures that legacy mapping is hit, as in legacy mapping we pick M from index settings + assertEquals( + mForSetting, + knnVectorFieldMapper.fieldType() + .getKnnMappingConfig() + .getKnnMethodContext() + .get() + .getMethodComponentContext() + .getParameters() + .get(METHOD_PARAMETER_M) + ); + assertTrue(knnVectorFieldMapper.fieldType().getKnnMappingConfig().getModelId().isEmpty()); + } + + public void testTypeParser_withSpaceTypeAndMode_thenSuccess() throws IOException { + // Check that knnMethodContext takes precedent over both model and legacy + ModelDao modelDao = mock(ModelDao.class); + // Setup settings + Settings settings = Settings.builder().put(settings(CURRENT).build()).put(KNN_INDEX, true).build(); + + SpaceType topLevelSpaceType = SpaceType.INNER_PRODUCT; + KNNVectorFieldMapper.TypeParser typeParser = new KNNVectorFieldMapper.TypeParser(() -> modelDao); + XContentBuilder xContentBuilder = XContentFactory.jsonBuilder() + .startObject() + .field(TYPE_FIELD_NAME, KNN_VECTOR_TYPE) + .field(DIMENSION, TEST_DIMENSION) + .field(MODE_PARAMETER, Mode.ON_DISK.getName()) + .field(COMPRESSION_LEVEL_PARAMETER, CompressionLevel.x16.getName()) + .field(KNNConstants.TOP_LEVEL_PARAMETER_SPACE_TYPE, topLevelSpaceType.getValue()) + .endObject(); + KNNVectorFieldMapper.Builder builder = (KNNVectorFieldMapper.Builder) typeParser.parse( + "test-field-name-1", + xContentBuilderToMap(xContentBuilder), + buildParserContext("test", settings) + ); + + Mapper.BuilderContext builderContext = new Mapper.BuilderContext(settings, new ContentPath()); + KNNVectorFieldMapper knnVectorFieldMapper = builder.build(builderContext); + assertTrue(knnVectorFieldMapper instanceof MethodFieldMapper); + assertTrue(knnVectorFieldMapper.fieldType().getKnnMappingConfig().getKnnMethodContext().isPresent()); + assertEquals(topLevelSpaceType, knnVectorFieldMapper.fieldType().getKnnMappingConfig().getKnnMethodContext().get().getSpaceType()); + assertTrue(knnVectorFieldMapper.fieldType().getKnnMappingConfig().getModelId().isEmpty()); + } + public void testBuilder_build_fromModel() { // Check that modelContext takes precedent over legacy ModelDao modelDao = mock(ModelDao.class); @@ -1442,6 +1593,35 @@ private LuceneFieldMapper.CreateLuceneFieldMapperInput.CreateLuceneFieldMapperIn .originalKnnMethodContext(getDefaultKNNMethodContext()); } + private XContentBuilder createXContentForFieldMapping( + SpaceType topLevelSpaceType, + SpaceType methodSpaceType, + KNNEngine knnEngine, + VectorDataType vectorDataType, + int dimension + ) throws IOException { + XContentBuilder xContentBuilder = XContentFactory.jsonBuilder() + .startObject() + .field(TYPE_FIELD_NAME, KNN_VECTOR_TYPE) + .field(DIMENSION_FIELD_NAME, dimension); + + if (topLevelSpaceType != null && topLevelSpaceType != SpaceType.UNDEFINED) { + xContentBuilder.field(KNNConstants.TOP_LEVEL_PARAMETER_SPACE_TYPE, topLevelSpaceType.getValue()); + } + if (vectorDataType != null) { + xContentBuilder.field(VECTOR_DATA_TYPE_FIELD, vectorDataType.getValue()); + } + xContentBuilder.startObject(KNN_METHOD).field(NAME, METHOD_HNSW); + if (knnEngine != null) { + xContentBuilder.field(KNN_ENGINE, knnEngine.getName()); + } + if (methodSpaceType != null && methodSpaceType != SpaceType.UNDEFINED) { + xContentBuilder.field(METHOD_PARAMETER_SPACE_TYPE, methodSpaceType.getValue()); + } + xContentBuilder.endObject().endObject(); + return xContentBuilder; + } + private static float[] createInitializedFloatArray(int dimension, float value) { float[] array = new float[dimension]; Arrays.fill(array, value); diff --git a/src/test/java/org/opensearch/knn/integ/TopLevelSpaceTypeParameterIT.java b/src/test/java/org/opensearch/knn/integ/TopLevelSpaceTypeParameterIT.java new file mode 100644 index 000000000..42cfb1491 --- /dev/null +++ b/src/test/java/org/opensearch/knn/integ/TopLevelSpaceTypeParameterIT.java @@ -0,0 +1,106 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.integ; + +import lombok.SneakyThrows; +import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.knn.KNNRestTestCase; +import org.opensearch.knn.common.KNNConstants; +import org.opensearch.knn.index.SpaceType; + +import java.io.IOException; + +import static org.opensearch.knn.common.KNNConstants.FAISS_NAME; +import static org.opensearch.knn.common.KNNConstants.KNN_ENGINE; +import static org.opensearch.knn.common.KNNConstants.KNN_METHOD; +import static org.opensearch.knn.common.KNNConstants.METHOD_HNSW; +import static org.opensearch.knn.common.KNNConstants.NAME; + +public class TopLevelSpaceTypeParameterIT extends KNNRestTestCase { + private final static float[] TEST_VECTOR = new float[] { 1.0f, 2.0f }; + private final static int DIMENSION = 2; + private final static int K = 1; + private static final String INDEX_NAME = "top-level-space-type-index"; + + @SneakyThrows + public void testBaseCase() { + createTestIndexWithTopLevelSpaceTypeOnly(); + addKnnDoc(INDEX_NAME, "0", FIELD_NAME, TEST_VECTOR); + validateKNNSearch(INDEX_NAME, FIELD_NAME, DIMENSION, 1, K); + deleteIndex(INDEX_NAME); + + createTestIndexWithTopLevelSpaceTypeAndMethodSpaceType(); + addKnnDoc(INDEX_NAME, "0", FIELD_NAME, TEST_VECTOR); + validateKNNSearch(INDEX_NAME, FIELD_NAME, DIMENSION, 1, K); + deleteIndex(INDEX_NAME); + + createTestIndexWithNoSpaceType(); + addKnnDoc(INDEX_NAME, "0", FIELD_NAME, TEST_VECTOR); + validateKNNSearch(INDEX_NAME, FIELD_NAME, DIMENSION, 1, K); + deleteIndex(INDEX_NAME); + } + + private void createTestIndexWithTopLevelSpaceTypeOnly() throws IOException { + XContentBuilder builder = XContentFactory.jsonBuilder() + .startObject() + .startObject("properties") + .startObject(FIELD_NAME) + .field("type", "knn_vector") + .field("dimension", DIMENSION) + .field(KNNConstants.TOP_LEVEL_PARAMETER_SPACE_TYPE, SpaceType.INNER_PRODUCT.getValue()) + .startObject(KNN_METHOD) + .field(NAME, METHOD_HNSW) + .field(KNN_ENGINE, FAISS_NAME) + .endObject() + .endObject() + .endObject() + .endObject(); + + String mapping = builder.toString(); + createKnnIndex(INDEX_NAME, mapping); + } + + private void createTestIndexWithTopLevelSpaceTypeAndMethodSpaceType() throws IOException { + XContentBuilder builder = XContentFactory.jsonBuilder() + .startObject() + .startObject("properties") + .startObject(FIELD_NAME) + .field("type", "knn_vector") + .field("dimension", DIMENSION) + .field(KNNConstants.TOP_LEVEL_PARAMETER_SPACE_TYPE, SpaceType.INNER_PRODUCT.getValue()) + .startObject(KNN_METHOD) + .field(NAME, METHOD_HNSW) + .field(KNN_ENGINE, FAISS_NAME) + .field(KNNConstants.METHOD_PARAMETER_SPACE_TYPE, SpaceType.INNER_PRODUCT.getValue()) + .endObject() + .endObject() + .endObject() + .endObject(); + + String mapping = builder.toString(); + createKnnIndex(INDEX_NAME, mapping); + } + + private void createTestIndexWithNoSpaceType() throws IOException { + XContentBuilder builder = XContentFactory.jsonBuilder() + .startObject() + .startObject("properties") + .startObject(FIELD_NAME) + .field("type", "knn_vector") + .field("dimension", DIMENSION) + .startObject(KNN_METHOD) + .field(NAME, METHOD_HNSW) + .field(KNN_ENGINE, FAISS_NAME) + .endObject() + .endObject() + .endObject() + .endObject(); + + String mapping = builder.toString(); + createKnnIndex(INDEX_NAME, mapping); + } +} From 77eb37f4d7173ec47200c86dfd19df25e061d3b5 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Sun, 8 Sep 2024 19:31:26 -0700 Subject: [PATCH 358/416] Throws and exception for radial search when mapping is for on-disk mode (#2055) (#2066) Signed-off-by: Tejas Shah (cherry picked from commit cbc63436cd6fb2f381241e1d4e71fa367a6258d2) Co-authored-by: Tejas Shah --- .../knn/index/mapper/KNNMappingConfig.java | 9 ++++ .../index/mapper/KNNVectorFieldMapper.java | 3 +- .../knn/index/mapper/MethodFieldMapper.java | 11 +++++ .../knn/index/mapper/ModelFieldMapper.java | 15 ++++++- .../knn/index/query/KNNQueryBuilder.java | 5 +++ .../nativelib/NativeEngineKnnVectorQuery.java | 7 +++ .../mapper/KNNVectorFieldMapperTests.java | 12 ++++- .../knn/index/query/KNNQueryBuilderTests.java | 45 +++++++++++++++++++ 8 files changed, 103 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/opensearch/knn/index/mapper/KNNMappingConfig.java b/src/main/java/org/opensearch/knn/index/mapper/KNNMappingConfig.java index 5b1955f23..cd77ebd9a 100644 --- a/src/main/java/org/opensearch/knn/index/mapper/KNNMappingConfig.java +++ b/src/main/java/org/opensearch/knn/index/mapper/KNNMappingConfig.java @@ -6,6 +6,7 @@ package org.opensearch.knn.index.mapper; import org.opensearch.knn.index.engine.KNNMethodContext; +import org.opensearch.knn.index.engine.qframe.QuantizationConfig; import java.util.Optional; @@ -48,6 +49,14 @@ default CompressionLevel getCompressionLevel() { return CompressionLevel.NOT_CONFIGURED; } + /** + * Returns quantization config + * @return + */ + default QuantizationConfig getQuantizationConfig() { + return QuantizationConfig.EMPTY; + } + /** * * @return the dimension of the index; for model based indices, it will be null diff --git a/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapper.java b/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapper.java index 10ba0d94d..265876310 100644 --- a/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapper.java +++ b/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapper.java @@ -241,7 +241,8 @@ public KNNVectorFieldMapper build(BuilderContext context) { hasDocValues.get(), modelDao, indexCreatedVersion, - originalParameters + originalParameters, + knnMethodConfigContext ); } diff --git a/src/main/java/org/opensearch/knn/index/mapper/MethodFieldMapper.java b/src/main/java/org/opensearch/knn/index/mapper/MethodFieldMapper.java index d479da39c..3d11949fe 100644 --- a/src/main/java/org/opensearch/knn/index/mapper/MethodFieldMapper.java +++ b/src/main/java/org/opensearch/knn/index/mapper/MethodFieldMapper.java @@ -51,6 +51,12 @@ public static MethodFieldMapper createFieldMapper( boolean hasDocValues, OriginalMappingParameters originalMappingParameters ) { + + KNNMethodContext knnMethodContext = originalMappingParameters.getResolvedKnnMethodContext(); + QuantizationConfig quantizationConfig = knnMethodContext.getKnnEngine() + .getKNNLibraryIndexingContext(knnMethodContext, knnMethodConfigContext) + .getQuantizationConfig(); + final KNNVectorFieldType mappedFieldType = new KNNVectorFieldType( fullname, metaValue, @@ -75,6 +81,11 @@ public Mode getMode() { public CompressionLevel getCompressionLevel() { return knnMethodConfigContext.getCompressionLevel(); } + + @Override + public QuantizationConfig getQuantizationConfig() { + return quantizationConfig; + } } ); return new MethodFieldMapper( diff --git a/src/main/java/org/opensearch/knn/index/mapper/ModelFieldMapper.java b/src/main/java/org/opensearch/knn/index/mapper/ModelFieldMapper.java index b7bbc5a0d..6b07483c5 100644 --- a/src/main/java/org/opensearch/knn/index/mapper/ModelFieldMapper.java +++ b/src/main/java/org/opensearch/knn/index/mapper/ModelFieldMapper.java @@ -56,9 +56,17 @@ public static ModelFieldMapper createFieldMapper( boolean hasDocValues, ModelDao modelDao, Version indexCreatedVersion, - OriginalMappingParameters originalMappingParameters + OriginalMappingParameters originalMappingParameters, + KNNMethodConfigContext knnMethodConfigContext ) { + final KNNMethodContext knnMethodContext = originalMappingParameters.getKnnMethodContext(); + final QuantizationConfig quantizationConfig = knnMethodContext == null + ? QuantizationConfig.EMPTY + : knnMethodContext.getKnnEngine() + .getKNNLibraryIndexingContext(knnMethodContext, knnMethodConfigContext) + .getQuantizationConfig(); + final KNNVectorFieldType mappedFieldType = new KNNVectorFieldType(fullname, metaValue, vectorDataType, new KNNMappingConfig() { private Integer dimension = null; private Mode mode = null; @@ -94,6 +102,11 @@ public CompressionLevel getCompressionLevel() { return compressionLevel; } + @Override + public QuantizationConfig getQuantizationConfig() { + return quantizationConfig; + } + // ModelMetadata relies on cluster state which may not be available during field mapper creation. Thus, // we lazily initialize it. private void initFromModelMetadata() { diff --git a/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java b/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java index b699a7705..8f7c5a3ff 100644 --- a/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java +++ b/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java @@ -25,6 +25,7 @@ import org.opensearch.index.query.QueryShardContext; import org.opensearch.knn.index.engine.KNNMethodConfigContext; import org.opensearch.knn.index.engine.model.QueryContext; +import org.opensearch.knn.index.engine.qframe.QuantizationConfig; import org.opensearch.knn.index.mapper.KNNMappingConfig; import org.opensearch.knn.index.mapper.KNNVectorFieldType; import org.opensearch.knn.index.query.parser.RescoreParser; @@ -451,6 +452,10 @@ protected Query doToQuery(QueryShardContext context) { if (vectorDataType == VectorDataType.BINARY) { throw new UnsupportedOperationException(String.format(Locale.ROOT, "Binary data type does not support radial search")); } + + if (knnMappingConfig.getQuantizationConfig() != QuantizationConfig.EMPTY) { + throw new UnsupportedOperationException("Radial search is not supported for indices which have quantization enabled"); + } } // Currently, k-NN supports distance and score types radial search diff --git a/src/main/java/org/opensearch/knn/index/query/nativelib/NativeEngineKnnVectorQuery.java b/src/main/java/org/opensearch/knn/index/query/nativelib/NativeEngineKnnVectorQuery.java index ac5a72945..c13d86554 100644 --- a/src/main/java/org/opensearch/knn/index/query/nativelib/NativeEngineKnnVectorQuery.java +++ b/src/main/java/org/opensearch/knn/index/query/nativelib/NativeEngineKnnVectorQuery.java @@ -7,6 +7,7 @@ import lombok.Getter; import lombok.RequiredArgsConstructor; +import lombok.extern.log4j.Log4j2; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.search.IndexSearcher; @@ -18,6 +19,7 @@ import org.apache.lucene.search.Weight; import org.apache.lucene.util.BitSet; import org.apache.lucene.util.Bits; +import org.opensearch.common.StopWatch; import org.opensearch.knn.index.query.KNNQuery; import org.opensearch.knn.index.query.KNNWeight; import org.opensearch.knn.index.query.ResultUtil; @@ -39,6 +41,7 @@ * for k-NN query if required. This is done by overriding rewrite method to execute ANN on each leaf * {@link KNNQuery} does not give the ability to post process segment results. */ +@Log4j2 @Getter @RequiredArgsConstructor public class NativeEngineKnnVectorQuery extends Query { @@ -60,7 +63,11 @@ public Weight createWeight(IndexSearcher indexSearcher, ScoreMode scoreMode, flo int firstPassK = rescoreContext.getFirstPassK(finalK); perLeafResults = doSearch(indexSearcher, leafReaderContexts, knnWeight, firstPassK); ResultUtil.reduceToTopK(perLeafResults, firstPassK); + + StopWatch stopWatch = new StopWatch().start(); perLeafResults = doRescore(indexSearcher, leafReaderContexts, knnWeight, perLeafResults, finalK); + long rescoreTime = stopWatch.stop().totalTime().millis(); + log.debug("Rescoring results took {} ms. oversampled k:{}, segments:{}", rescoreTime, firstPassK, leafReaderContexts.size()); } ResultUtil.reduceToTopK(perLeafResults, finalK); TopDocs[] topDocs = new TopDocs[perLeafResults.size()]; diff --git a/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java b/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java index 35e027666..f43434316 100644 --- a/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java +++ b/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java @@ -1127,6 +1127,12 @@ public void testModelFieldMapperParseCreateField_validInput_thenDifferentFieldTy MockedStatic utilMockedStatic = Mockito.mockStatic(KNNVectorFieldMapperUtil.class); MockedStatic modelUtilMockedStatic = Mockito.mockStatic(ModelUtil.class) ) { + KNNMethodConfigContext knnMethodConfigContext = KNNMethodConfigContext.builder() + .vectorDataType(VectorDataType.FLOAT) + .versionCreated(CURRENT) + .dimension(TEST_DIMENSION) + .build(); + for (VectorDataType dataType : VectorDataType.values()) { log.info("Vector Data Type is : {}", dataType); SpaceType spaceType = VectorDataType.BINARY == dataType ? SpaceType.DEFAULT_BINARY : SpaceType.INNER_PRODUCT; @@ -1170,7 +1176,8 @@ public void testModelFieldMapperParseCreateField_validInput_thenDifferentFieldTy false, modelDao, CURRENT, - originalMappingParameters + originalMappingParameters, + knnMethodConfigContext ); modelFieldMapper.parseCreateField(parseContext); @@ -1211,7 +1218,8 @@ public void testModelFieldMapperParseCreateField_validInput_thenDifferentFieldTy false, modelDao, CURRENT, - originalMappingParameters + originalMappingParameters, + knnMethodConfigContext ); modelFieldMapper.parseCreateField(parseContext); diff --git a/src/test/java/org/opensearch/knn/index/query/KNNQueryBuilderTests.java b/src/test/java/org/opensearch/knn/index/query/KNNQueryBuilderTests.java index 1c1b6edd9..3db03085b 100644 --- a/src/test/java/org/opensearch/knn/index/query/KNNQueryBuilderTests.java +++ b/src/test/java/org/opensearch/knn/index/query/KNNQueryBuilderTests.java @@ -29,7 +29,10 @@ import org.opensearch.index.query.QueryShardContext; import org.opensearch.index.query.TermQueryBuilder; import org.opensearch.knn.KNNTestCase; +import org.opensearch.knn.index.engine.qframe.QuantizationConfig; +import org.opensearch.knn.index.mapper.KNNMappingConfig; import org.opensearch.knn.index.mapper.KNNVectorFieldType; +import org.opensearch.knn.index.mapper.Mode; import org.opensearch.knn.index.query.rescore.RescoreContext; import org.opensearch.knn.index.util.KNNClusterUtil; import org.opensearch.knn.index.engine.KNNMethodContext; @@ -41,6 +44,7 @@ import org.opensearch.knn.indices.ModelDao; import org.opensearch.knn.indices.ModelMetadata; import org.opensearch.knn.indices.ModelState; +import org.opensearch.knn.quantization.enums.ScalarQuantizationType; import java.io.IOException; import java.util.Arrays; @@ -440,6 +444,47 @@ public void testDoToQuery_whenRadialSearchOnBinaryIndex_thenException() { assertTrue(e.getMessage().contains("Binary data type does not support radial search")); } + public void testDoToQuery_whenRadialSearchOnDiskMode_thenException() { + float[] queryVector = { 1.0f }; + KNNQueryBuilder knnQueryBuilder = KNNQueryBuilder.builder() + .fieldName(FIELD_NAME) + .vector(queryVector) + .maxDistance(MAX_DISTANCE) + .build(); + Index dummyIndex = new Index("dummy", "dummy"); + QueryShardContext mockQueryShardContext = mock(QueryShardContext.class); + KNNVectorFieldType mockKNNVectorField = mock(KNNVectorFieldType.class); + when(mockQueryShardContext.index()).thenReturn(dummyIndex); + when(mockKNNVectorField.getVectorDataType()).thenReturn(VectorDataType.FLOAT); + when(mockQueryShardContext.fieldMapper(anyString())).thenReturn(mockKNNVectorField); + MethodComponentContext methodComponentContext = new MethodComponentContext( + org.opensearch.knn.common.KNNConstants.METHOD_HNSW, + ImmutableMap.of() + ); + KNNMethodContext knnMethodContext = new KNNMethodContext(KNNEngine.FAISS, SpaceType.L2, methodComponentContext); + when(mockKNNVectorField.getKnnMappingConfig()).thenReturn(new KNNMappingConfig() { + @Override + public Optional getKnnMethodContext() { + return Optional.of(knnMethodContext); + } + + @Override + public int getDimension() { + return 1; + } + + public Mode getMode() { + return Mode.ON_DISK; + } + + public QuantizationConfig getQuantizationConfig() { + return QuantizationConfig.builder().quantizationType(ScalarQuantizationType.ONE_BIT).build(); + } + }); + Exception e = expectThrows(UnsupportedOperationException.class, () -> knnQueryBuilder.doToQuery(mockQueryShardContext)); + assertEquals("Radial search is not supported for indices which have quantization enabled", e.getMessage()); + } + public void testDoToQuery_KnnQueryWithFilter_Lucene() throws Exception { // Given float[] queryVector = { 1.0f, 2.0f, 3.0f, 4.0f }; From 0eaa5a0086709f98f4154543f3a831020432d505 Mon Sep 17 00:00:00 2001 From: Navneet Verma Date: Mon, 9 Sep 2024 09:02:47 -0700 Subject: [PATCH 359/416] [BugFix] Fixing KNNMethodContext resolution when documents are ingested and a flaky test (#2072) (#2075) Signed-off-by: Navneet Verma (cherry picked from commit 8f6b177986ab89676fcca2b1fdedad580896b8c3) --- .../index/mapper/KNNVectorFieldMapper.java | 2 + .../mapper/KNNVectorFieldMapperTests.java | 59 +++++++++++++++++++ .../knn/integ/ModeAndCompressionIT.java | 58 ++++++++++++++++-- 3 files changed, 114 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapper.java b/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapper.java index 265876310..f149fa1d2 100644 --- a/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapper.java +++ b/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapper.java @@ -782,6 +782,8 @@ public ParametrizedFieldMapper.Builder getMergeBuilder() { .vectorDataType(vectorDataType) .versionCreated(indexCreatedVersion) .dimension(fieldType().getKnnMappingConfig().getDimension()) + .compressionLevel(fieldType().getKnnMappingConfig().getCompressionLevel()) + .mode(fieldType().getKnnMappingConfig().getMode()) .build(); } diff --git a/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java b/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java index f43434316..21a116fd5 100644 --- a/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java +++ b/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java @@ -878,6 +878,65 @@ public void testTypeParser_parse_fromLegacy() throws IOException { assertNull(builder.knnMethodContext.get()); } + public void testKNNVectorFieldMapperMerge_whenModeAndCompressionIsPresent_thenSuccess() throws IOException { + String fieldName = "test-field-name"; + String indexName = "test-index-name"; + + Settings settings = Settings.builder().put(settings(CURRENT).build()).put(KNN_INDEX, true).build(); + ModelDao modelDao = mock(ModelDao.class); + KNNVectorFieldMapper.TypeParser typeParser = new KNNVectorFieldMapper.TypeParser(() -> modelDao); + + int dimension = 133; + XContentBuilder xContentBuilder = XContentFactory.jsonBuilder() + .startObject() + .field(TYPE_FIELD_NAME, KNN_VECTOR_TYPE) + .field(DIMENSION_FIELD_NAME, dimension) + .field(MODE_PARAMETER, Mode.ON_DISK.getName()) + .field(COMPRESSION_LEVEL_PARAMETER, CompressionLevel.x32.getName()) + .endObject(); + + KNNVectorFieldMapper.Builder builder = (KNNVectorFieldMapper.Builder) typeParser.parse( + fieldName, + xContentBuilderToMap(xContentBuilder), + buildParserContext(indexName, settings) + ); + Mapper.BuilderContext builderContext = new Mapper.BuilderContext(settings, new ContentPath()); + KNNVectorFieldMapper knnVectorFieldMapper1 = builder.build(builderContext); + + // merge with itself - should be successful + KNNVectorFieldMapper knnVectorFieldMapperMerge1 = (KNNVectorFieldMapper) knnVectorFieldMapper1.merge(knnVectorFieldMapper1); + assertEquals( + knnVectorFieldMapper1.fieldType().getKnnMappingConfig().getKnnMethodContext().get(), + knnVectorFieldMapperMerge1.fieldType().getKnnMappingConfig().getKnnMethodContext().get() + ); + + assertEquals( + knnVectorFieldMapper1.fieldType().getKnnMappingConfig().getCompressionLevel(), + knnVectorFieldMapperMerge1.fieldType().getKnnMappingConfig().getCompressionLevel() + ); + assertEquals( + knnVectorFieldMapper1.fieldType().getKnnMappingConfig().getMode(), + knnVectorFieldMapperMerge1.fieldType().getKnnMappingConfig().getMode() + ); + + // merge with another mapper of the same field with same context + KNNVectorFieldMapper knnVectorFieldMapper2 = builder.build(builderContext); + KNNVectorFieldMapper knnVectorFieldMapperMerge2 = (KNNVectorFieldMapper) knnVectorFieldMapper1.merge(knnVectorFieldMapper2); + assertEquals( + knnVectorFieldMapper1.fieldType().getKnnMappingConfig().getKnnMethodContext().get(), + knnVectorFieldMapperMerge2.fieldType().getKnnMappingConfig().getKnnMethodContext().get() + ); + + assertEquals( + knnVectorFieldMapper1.fieldType().getKnnMappingConfig().getCompressionLevel(), + knnVectorFieldMapperMerge2.fieldType().getKnnMappingConfig().getCompressionLevel() + ); + assertEquals( + knnVectorFieldMapper1.fieldType().getKnnMappingConfig().getMode(), + knnVectorFieldMapperMerge2.fieldType().getKnnMappingConfig().getMode() + ); + } + public void testKNNVectorFieldMapper_merge_fromKnnMethodContext() throws IOException { String fieldName = "test-field-name"; String indexName = "test-index-name"; diff --git a/src/test/java/org/opensearch/knn/integ/ModeAndCompressionIT.java b/src/test/java/org/opensearch/knn/integ/ModeAndCompressionIT.java index af0480078..925ba0fff 100644 --- a/src/test/java/org/opensearch/knn/integ/ModeAndCompressionIT.java +++ b/src/test/java/org/opensearch/knn/integ/ModeAndCompressionIT.java @@ -7,16 +7,18 @@ import lombok.SneakyThrows; import org.apache.http.util.EntityUtils; +import org.junit.Assert; import org.junit.Ignore; +import org.opensearch.client.Request; import org.opensearch.client.Response; import org.opensearch.client.ResponseException; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.rest.RestStatus; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.knn.KNNRestTestCase; -import org.opensearch.knn.KNNResult; import org.opensearch.knn.common.KNNConstants; import org.opensearch.knn.index.KNNSettings; +import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.mapper.CompressionLevel; import org.opensearch.knn.index.mapper.Mode; import org.opensearch.knn.index.mapper.ModeBasedResolver; @@ -50,7 +52,7 @@ public class ModeAndCompressionIT extends KNNRestTestCase { private static final int DIMENSION = 16; private static final int NUM_DOCS = 20; - private static final int K = 2; + private static final int K = NUM_DOCS; private final static float[] TEST_VECTOR = new float[] { 1.0f, 2.0f, @@ -210,7 +212,7 @@ public void testDeletedDocsWithSegmentMerge_whenValid_ThenSucceed() { .endObject(); String mapping = builder.toString(); validateIndexWithDeletedDocs(indexName, mapping); - validateSearch(indexName, METHOD_PARAMETER_EF_SEARCH, KNNSettings.INDEX_KNN_DEFAULT_ALGO_PARAM_EF_SEARCH); + validateGreenIndex(indexName); } @SneakyThrows @@ -352,6 +354,19 @@ private void validateIndexWithDeletedDocs(String indexName, String mapping) { refreshIndex(indexName); } + @SneakyThrows + private void validateGreenIndex(String indexName) { + Request request = new Request("GET", "/_cat/indices/" + indexName + "?format=csv"); + Response response = client().performRequest(request); + assertOK(response); + assertEquals( + "The status of index " + indexName + " is not green", + "green", + new String(response.getEntity().getContent().readAllBytes()).split("\n")[0].split(" ")[0] + ); + + } + @SneakyThrows private void setupTrainingIndex() { createBasicKnnIndex(TRAINING_INDEX_NAME, TRAINING_FIELD_NAME, DIMENSION); @@ -388,9 +403,41 @@ private void validateSearch(String indexName, String methodParameterName, int me ); assertOK(response); String responseBody = EntityUtils.toString(response.getEntity()); - List knnResults = parseSearchResponse(responseBody, FIELD_NAME); + List knnResults = parseSearchResponseScore(responseBody, FIELD_NAME); assertEquals(K, knnResults.size()); + // Do exact search and gather right scores for the documents + Response exactSearchResponse = searchKNNIndex( + indexName, + XContentFactory.jsonBuilder() + .startObject() + .startObject("query") + .startObject("script_score") + .startObject("query") + .field("match_all") + .startObject() + .endObject() + .endObject() + .startObject("script") + .field("source", "knn_score") + .field("lang", "knn") + .startObject("params") + .field("field", FIELD_NAME) + .field("query_value", TEST_VECTOR) + .field("space_type", SpaceType.L2.getValue()) + .endObject() + .endObject() + .endObject() + .endObject() + .endObject(), + K + ); + assertOK(exactSearchResponse); + String exactSearchResponseBody = EntityUtils.toString(exactSearchResponse.getEntity()); + List exactSearchKnnResults = parseSearchResponseScore(exactSearchResponseBody, FIELD_NAME); + assertEquals(NUM_DOCS, exactSearchKnnResults.size()); + Assert.assertEquals(exactSearchKnnResults, knnResults); + // Search with rescore response = searchKNNIndex( indexName, @@ -415,7 +462,8 @@ private void validateSearch(String indexName, String methodParameterName, int me ); assertOK(response); responseBody = EntityUtils.toString(response.getEntity()); - knnResults = parseSearchResponse(responseBody, FIELD_NAME); + knnResults = parseSearchResponseScore(responseBody, FIELD_NAME); assertEquals(K, knnResults.size()); + Assert.assertEquals(exactSearchKnnResults, knnResults); } } From ed5d7d10ec644ceab75e2e7930cbb24da6d57597 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Mon, 9 Sep 2024 13:44:01 -0500 Subject: [PATCH 360/416] Re-Call Issue Fix with Binary Quantized Vectors (#2071) (#2078) --- .../nativeindex/QuantizationIndexUtils.java | 16 ++- .../BinaryQuantizationOutput.java | 12 ++ .../QuantizationOutput.java | 34 ++++- .../quantizer/MultiBitScalarQuantizer.java | 2 +- .../quantizer/OneBitScalarQuantizer.java | 2 +- .../DefaultIndexBuildStrategyTests.java | 6 +- ...ptimizedNativeIndexBuildStrategyTests.java | 6 +- .../output/BinaryQuantizationOutputTests.java | 121 ++++++++++++++++++ 8 files changed, 184 insertions(+), 15 deletions(-) create mode 100644 src/test/java/org/opensearch/knn/quantization/output/BinaryQuantizationOutputTests.java diff --git a/src/main/java/org/opensearch/knn/index/codec/nativeindex/QuantizationIndexUtils.java b/src/main/java/org/opensearch/knn/index/codec/nativeindex/QuantizationIndexUtils.java index bebe9e8b0..c5994d66b 100644 --- a/src/main/java/org/opensearch/knn/index/codec/nativeindex/QuantizationIndexUtils.java +++ b/src/main/java/org/opensearch/knn/index/codec/nativeindex/QuantizationIndexUtils.java @@ -18,12 +18,12 @@ class QuantizationIndexUtils { /** - * Processes and returns the vector based on whether quantization is applied or not. + * Processes the vector from {@link KNNVectorValues} and returns either a cloned quantized vector or a cloned original vector. * - * @param knnVectorValues the KNN vector values to be processed. - * @param indexBuildSetup the setup containing quantization state and output, along with other parameters. - * @return the processed vector, either quantized or original. - * @throws IOException if an I/O error occurs during processing. + * @param knnVectorValues The KNN vector values containing the original vector. + * @param indexBuildSetup The setup containing the quantization state and output details. + * @return The quantized vector (as a byte array) or the original/cloned vector. + * @throws IOException If an I/O error occurs while processing the vector. */ static Object processAndReturnVector(KNNVectorValues knnVectorValues, IndexBuildSetup indexBuildSetup) throws IOException { QuantizationService quantizationService = QuantizationService.getInstance(); @@ -33,7 +33,11 @@ static Object processAndReturnVector(KNNVectorValues knnVectorValues, IndexBu knnVectorValues.getVector(), indexBuildSetup.getQuantizationOutput() ); - return indexBuildSetup.getQuantizationOutput().getQuantizedVector(); + /** + * Returns a copy of the quantized vector. This is because of during transfer same vectors was getting + * added due to reference. + */ + return indexBuildSetup.getQuantizationOutput().getQuantizedVectorCopy(); } else { return knnVectorValues.conditionalCloneVector(); } diff --git a/src/main/java/org/opensearch/knn/quantization/models/quantizationOutput/BinaryQuantizationOutput.java b/src/main/java/org/opensearch/knn/quantization/models/quantizationOutput/BinaryQuantizationOutput.java index 388fd9e94..dc8634b9d 100644 --- a/src/main/java/org/opensearch/knn/quantization/models/quantizationOutput/BinaryQuantizationOutput.java +++ b/src/main/java/org/opensearch/knn/quantization/models/quantizationOutput/BinaryQuantizationOutput.java @@ -63,4 +63,16 @@ public boolean isPrepared(int vectorLength) { public byte[] getQuantizedVector() { return quantizedVector; } + + /** + * Returns a copy of the quantized vector. + * + * @return a copy of the quantized vector byte array. + */ + @Override + public byte[] getQuantizedVectorCopy() { + byte[] clonedByteArray = new byte[quantizedVector.length]; + System.arraycopy(quantizedVector, 0, clonedByteArray, 0, quantizedVector.length); + return clonedByteArray; + } } diff --git a/src/main/java/org/opensearch/knn/quantization/models/quantizationOutput/QuantizationOutput.java b/src/main/java/org/opensearch/knn/quantization/models/quantizationOutput/QuantizationOutput.java index 4d088f91f..29124c268 100644 --- a/src/main/java/org/opensearch/knn/quantization/models/quantizationOutput/QuantizationOutput.java +++ b/src/main/java/org/opensearch/knn/quantization/models/quantizationOutput/QuantizationOutput.java @@ -14,7 +14,32 @@ public interface QuantizationOutput { /** * Returns the quantized vector. * - * @return the quantized data. + * This method provides access to the quantized data in its current state. + * It returns the same reference to the internal quantized vector on each call, meaning any modifications + * to the returned array will directly affect the internal state of the object. This design is intentional + * to avoid unnecessary copying of data and improve performance, especially in scenarios where frequent + * access to the quantized vector is required. + * + *

    Important: As this method returns a direct reference to the internal array, care must be taken + * when modifying the returned array. If the returned vector is altered, the changes will reflect in the + * quantized vector managed by the object, which could lead to unintended side effects.

    + * + *

    Usage Example:

    + * 
    +     * byte[] quantizedData = quantizationOutput.getQuantizedVector();
+     * // Use or modify quantizedData, but be cautious that changes affect the internal state.
+     *
    + * + * This method does not create a deep copy of the vector to avoid performance overhead in real-time + * or high-frequency operations. If a separate copy of the vector is needed, the caller should manually + * clone or copy the returned array. + * + *

    Example to clone the array:

    + * 
    +     * byte[] clonedData = Arrays.copyOf(quantizationOutput.getQuantizedVector(), quantizationOutput.getQuantizedVector().length);
+     *
    + * + * @return the quantized vector (same reference on each invocation). */ T getQuantizedVector(); @@ -33,4 +58,11 @@ public interface QuantizationOutput { * @return true if the quantized vector is already prepared, false otherwise. */ boolean isPrepared(int vectorLength); + + /** + * Returns a copy of the quantized vector. + * + * @return a copy of the quantized data. + */ + T getQuantizedVectorCopy(); } diff --git a/src/main/java/org/opensearch/knn/quantization/quantizer/MultiBitScalarQuantizer.java b/src/main/java/org/opensearch/knn/quantization/quantizer/MultiBitScalarQuantizer.java index a0e6ec402..0bcc252d1 100644 --- a/src/main/java/org/opensearch/knn/quantization/quantizer/MultiBitScalarQuantizer.java +++ b/src/main/java/org/opensearch/knn/quantization/quantizer/MultiBitScalarQuantizer.java @@ -139,7 +139,7 @@ public void quantize(final float[] vector, final QuantizationState state, final if (thresholds == null || thresholds[0].length != vector.length) { throw new IllegalArgumentException("Thresholds must not be null and must match the dimension of the vector."); } - if (!output.isPrepared(vectorLength)) output.prepareQuantizedVector(vectorLength); + output.prepareQuantizedVector(vectorLength); BitPacker.quantizeAndPackBits(vector, thresholds, bitsPerCoordinate, output.getQuantizedVector()); } diff --git a/src/main/java/org/opensearch/knn/quantization/quantizer/OneBitScalarQuantizer.java b/src/main/java/org/opensearch/knn/quantization/quantizer/OneBitScalarQuantizer.java index ac48a9523..3cba89c39 100644 --- a/src/main/java/org/opensearch/knn/quantization/quantizer/OneBitScalarQuantizer.java +++ b/src/main/java/org/opensearch/knn/quantization/quantizer/OneBitScalarQuantizer.java @@ -84,7 +84,7 @@ public void quantize(final float[] vector, final QuantizationState state, final if (thresholds == null || thresholds.length != vectorLength) { throw new IllegalArgumentException("Thresholds must not be null and must match the dimension of the vector."); } - if (!output.isPrepared(vectorLength)) output.prepareQuantizedVector(vectorLength); + output.prepareQuantizedVector(vectorLength); BitPacker.quantizeAndPackBits(vector, thresholds, output.getQuantizedVector()); } diff --git a/src/test/java/org/opensearch/knn/index/codec/nativeindex/DefaultIndexBuildStrategyTests.java b/src/test/java/org/opensearch/knn/index/codec/nativeindex/DefaultIndexBuildStrategyTests.java index 9c2e5a4b7..abb61ccd9 100644 --- a/src/test/java/org/opensearch/knn/index/codec/nativeindex/DefaultIndexBuildStrategyTests.java +++ b/src/test/java/org/opensearch/knn/index/codec/nativeindex/DefaultIndexBuildStrategyTests.java @@ -138,7 +138,7 @@ public void testBuildAndWrite_withQuantization() { ArgumentCaptor vectorCaptor = ArgumentCaptor.forClass(float[].class); // New: Create QuantizationOutput and mock the quantization process QuantizationOutput quantizationOutput = mock(QuantizationOutput.class); - when(quantizationOutput.getQuantizedVector()).thenReturn(new byte[] { 1, 2 }); + when(quantizationOutput.getQuantizedVectorCopy()).thenReturn(new byte[] { 1, 2 }); when(quantizationService.createQuantizationOutput(eq(quantizationState.getQuantizationParams()))).thenReturn( quantizationOutput ); @@ -146,8 +146,8 @@ public void testBuildAndWrite_withQuantization() { // Quantize the vector with the quantization output when(quantizationService.quantize(eq(quantizationState), vectorCaptor.capture(), eq(quantizationOutput))).thenAnswer( invocation -> { - quantizationOutput.getQuantizedVector(); - return quantizationOutput.getQuantizedVector(); + quantizationOutput.getQuantizedVectorCopy(); + return quantizationOutput.getQuantizedVectorCopy(); } ); when(quantizationState.getDimensions()).thenReturn(2); diff --git a/src/test/java/org/opensearch/knn/index/codec/nativeindex/MemOptimizedNativeIndexBuildStrategyTests.java b/src/test/java/org/opensearch/knn/index/codec/nativeindex/MemOptimizedNativeIndexBuildStrategyTests.java index 62c3b7a71..77abe1cd2 100644 --- a/src/test/java/org/opensearch/knn/index/codec/nativeindex/MemOptimizedNativeIndexBuildStrategyTests.java +++ b/src/test/java/org/opensearch/knn/index/codec/nativeindex/MemOptimizedNativeIndexBuildStrategyTests.java @@ -164,7 +164,7 @@ public void testBuildAndWrite_withQuantization() { ArgumentCaptor vectorCaptor = ArgumentCaptor.forClass(float[].class); // New: Create QuantizationOutput and mock the quantization process QuantizationOutput quantizationOutput = mock(QuantizationOutput.class); - when(quantizationOutput.getQuantizedVector()).thenReturn(new byte[] { 1, 2 }); + when(quantizationOutput.getQuantizedVectorCopy()).thenReturn(new byte[] { 1, 2 }); when(quantizationService.createQuantizationOutput(eq(quantizationState.getQuantizationParams()))).thenReturn( quantizationOutput ); @@ -172,8 +172,8 @@ public void testBuildAndWrite_withQuantization() { // Quantize the vector with the quantization output when(quantizationService.quantize(eq(quantizationState), vectorCaptor.capture(), eq(quantizationOutput))).thenAnswer( invocation -> { - quantizationOutput.getQuantizedVector(); - return quantizationOutput.getQuantizedVector(); + quantizationOutput.getQuantizedVectorCopy(); + return quantizationOutput.getQuantizedVectorCopy(); } ); when(quantizationState.getDimensions()).thenReturn(2); diff --git a/src/test/java/org/opensearch/knn/quantization/output/BinaryQuantizationOutputTests.java b/src/test/java/org/opensearch/knn/quantization/output/BinaryQuantizationOutputTests.java new file mode 100644 index 000000000..8eab5d00c --- /dev/null +++ b/src/test/java/org/opensearch/knn/quantization/output/BinaryQuantizationOutputTests.java @@ -0,0 +1,121 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.quantization.output; + +import org.junit.Before; +import org.opensearch.knn.KNNTestCase; +import org.opensearch.knn.quantization.models.quantizationOutput.BinaryQuantizationOutput; + +public class BinaryQuantizationOutputTests extends KNNTestCase { + + private static final int BITS_PER_COORDINATE = 1; + private BinaryQuantizationOutput quantizationOutput; + + @Before + public void setUp() throws Exception { + super.setUp(); + quantizationOutput = new BinaryQuantizationOutput(BITS_PER_COORDINATE); + } + + public void testPrepareQuantizedVector_ShouldInitializeCorrectly_WhenVectorLengthIsValid() { + // Arrange + int vectorLength = 10; + + // Act + quantizationOutput.prepareQuantizedVector(vectorLength); + + // Assert + assertNotNull(quantizationOutput.getQuantizedVector()); + } + + public void testPrepareQuantizedVector_ShouldThrowException_WhenVectorLengthIsZeroOrNegative() { + // Act and Assert + expectThrows(IllegalArgumentException.class, () -> quantizationOutput.prepareQuantizedVector(0)); + expectThrows(IllegalArgumentException.class, () -> quantizationOutput.prepareQuantizedVector(-1)); + } + + public void testIsPrepared_ShouldReturnTrue_WhenCalledWithSameVectorLength() { + // Arrange + int vectorLength = 8; + quantizationOutput.prepareQuantizedVector(vectorLength); + // Act and Assert + assertTrue(quantizationOutput.isPrepared(vectorLength)); + } + + public void testIsPrepared_ShouldReturnFalse_WhenCalledWithDifferentVectorLength() { + // Arrange + int vectorLength = 8; + quantizationOutput.prepareQuantizedVector(vectorLength); + // Act and Assert + assertFalse(quantizationOutput.isPrepared(vectorLength + 1)); + } + + public void testGetQuantizedVector_ShouldReturnSameReference() { + // Arrange + int vectorLength = 5; + quantizationOutput.prepareQuantizedVector(vectorLength); + // Act + byte[] vector = quantizationOutput.getQuantizedVector(); + // Assert + assertEquals(vector, quantizationOutput.getQuantizedVector()); + } + + public void testGetQuantizedVectorCopy_ShouldReturnCopyOfVector() { + // Arrange + int vectorLength = 5; + quantizationOutput.prepareQuantizedVector(vectorLength); + + // Act + byte[] vectorCopy = quantizationOutput.getQuantizedVectorCopy(); + + // Assert + assertNotSame(vectorCopy, quantizationOutput.getQuantizedVector()); + assertArrayEquals(vectorCopy, quantizationOutput.getQuantizedVector()); + } + + public void testGetQuantizedVectorCopy_ShouldReturnNewCopyOnEachCall() { + // Arrange + int vectorLength = 5; + quantizationOutput.prepareQuantizedVector(vectorLength); + + // Act + byte[] vectorCopy1 = quantizationOutput.getQuantizedVectorCopy(); + byte[] vectorCopy2 = quantizationOutput.getQuantizedVectorCopy(); + + // Assert + assertNotSame(vectorCopy1, vectorCopy2); + } + + public void testPrepareQuantizedVector_ShouldResetQuantizedVector_WhenCalledWithDifferentLength() { + // Arrange + int initialLength = 5; + int newLength = 10; + quantizationOutput.prepareQuantizedVector(initialLength); + byte[] initialVector = quantizationOutput.getQuantizedVector(); + + // Act + quantizationOutput.prepareQuantizedVector(newLength); + byte[] newVector = quantizationOutput.getQuantizedVector(); + + // Assert + assertNotSame(initialVector, newVector); // The array reference should change + assertEquals(newVector.length, (BITS_PER_COORDINATE * newLength + 7) / 8); // Correct size for new vector + } + + public void testPrepareQuantizedVector_ShouldRetainSameArray_WhenCalledWithSameLength() { + // Arrange + int vectorLength = 5; + quantizationOutput.prepareQuantizedVector(vectorLength); + byte[] initialVector = quantizationOutput.getQuantizedVector(); + + // Act + quantizationOutput.prepareQuantizedVector(vectorLength); + byte[] newVector = quantizationOutput.getQuantizedVector(); + + // Assert + assertSame(newVector, initialVector); // The array reference should remain the same + } +} From d69a03867c9d934872bb35f3f008dc983182310a Mon Sep 17 00:00:00 2001 From: Ryan Bogan Date: Mon, 9 Sep 2024 12:39:15 -0700 Subject: [PATCH 361/416] [Backport 2.x] Add model version to model metadata and change model metadata reads to be from cluster metadata (#2063) * Add model version to model metadata and change model metadata reads to be from cluster metadata (#2005) * Add model version to model metadata Signed-off-by: Ryan Bogan * Add model version to model metadata and change model metadata reads to be from cluster metadata Signed-off-by: Ryan Bogan * Add changelog entry Signed-off-by: Ryan Bogan * Set version from config context Signed-off-by: Ryan Bogan * Fix spotless Signed-off-by: Ryan Bogan * Update model index mappings Signed-off-by: Ryan Bogan * Change field mapper to read model version Signed-off-by: Ryan Bogan * Fix tests Signed-off-by: Ryan Bogan * remove println Signed-off-by: John Mazanec --------- Signed-off-by: Ryan Bogan Signed-off-by: John Mazanec Co-authored-by: John Mazanec (cherry picked from commit 6814c8f60707ff8e3be835558ab35ae5a9ea0c1a) * Fix tests Signed-off-by: Ryan Bogan --------- Signed-off-by: Ryan Bogan --- .../opensearch-knn.release-notes-2.17.0.0.md | 1 + .../opensearch/knn/common/KNNConstants.java | 1 + .../knn/index/mapper/ModelFieldMapper.java | 2 +- .../opensearch/knn/index/util/IndexUtil.java | 2 + .../org/opensearch/knn/indices/ModelDao.java | 1 + .../opensearch/knn/indices/ModelMetadata.java | 56 ++++- .../org/opensearch/knn/indices/ModelUtil.java | 4 +- .../opensearch/knn/training/TrainingJob.java | 3 +- .../knn/training/TrainingJobRunner.java | 7 +- src/main/resources/mappings/model-index.json | 3 + .../index/KNNCreateIndexFromModelTests.java | 4 +- .../KNN80DocValuesConsumerTests.java | 161 +++++++------- .../knn/index/codec/KNNCodecTestCase.java | 156 ++++++------- .../mapper/KNNVectorFieldMapperTests.java | 7 +- .../knn/indices/ModelCacheTests.java | 37 +++- .../opensearch/knn/indices/ModelDaoTests.java | 43 ++-- .../knn/indices/ModelMetadataTests.java | 206 +++++++++++------- .../opensearch/knn/indices/ModelTests.java | 50 +++-- .../knn/indices/ModelUtilTests.java | 21 +- .../transport/GetModelResponseTests.java | 15 +- ...oveModelFromCacheTransportActionTests.java | 4 +- .../transport/TrainingModelRequestTests.java | 4 +- ...ateModelGraveyardTransportActionTests.java | 4 +- .../UpdateModelMetadataRequestTests.java | 10 +- ...dateModelMetadataTransportActionTests.java | 4 +- .../knn/training/TrainingJobRunnerTests.java | 2 +- .../knn/training/TrainingJobTests.java | 3 +- 27 files changed, 490 insertions(+), 321 deletions(-) diff --git a/release-notes/opensearch-knn.release-notes-2.17.0.0.md b/release-notes/opensearch-knn.release-notes-2.17.0.0.md index 9876b7b38..8b4aa8e95 100644 --- a/release-notes/opensearch-knn.release-notes-2.17.0.0.md +++ b/release-notes/opensearch-knn.release-notes-2.17.0.0.md @@ -11,6 +11,7 @@ Compatible with OpenSearch 2.17.0 * Add spaceType as a top level optional parameter while creating vector field. [#2044](https://github.com/opensearch-project/k-NN/pull/2044) ### Enhancements * Adds iterative graph build capability into a faiss index to improve the memory footprint during indexing and Integrates KNNVectorsFormat for native engines[#1950](https://github.com/opensearch-project/k-NN/pull/1950) +* Add model version to model metadata and change model metadata reads to be from cluster metadata [#2005](https://github.com/opensearch-project/k-NN/pull/2005) ### Bug Fixes * Corrected search logic for scenario with non-existent fields in filter [#1874](https://github.com/opensearch-project/k-NN/pull/1874) * Add script_fields context to KNNAllowlist [#1917] (https://github.com/opensearch-project/k-NN/pull/1917) diff --git a/src/main/java/org/opensearch/knn/common/KNNConstants.java b/src/main/java/org/opensearch/knn/common/KNNConstants.java index 11024076f..ed21d3005 100644 --- a/src/main/java/org/opensearch/knn/common/KNNConstants.java +++ b/src/main/java/org/opensearch/knn/common/KNNConstants.java @@ -77,6 +77,7 @@ public class KNNConstants { public static final String TOP_LEVEL_SPACE_TYPE_FEATURE = "top_level_space_type_feature"; public static final String RADIAL_SEARCH_KEY = "radial_search"; + public static final String MODEL_VERSION = "model_version"; public static final String QUANTIZATION_STATE_FILE_SUFFIX = "osknnqstate"; // Lucene specific constants diff --git a/src/main/java/org/opensearch/knn/index/mapper/ModelFieldMapper.java b/src/main/java/org/opensearch/knn/index/mapper/ModelFieldMapper.java index 6b07483c5..013cb0c53 100644 --- a/src/main/java/org/opensearch/knn/index/mapper/ModelFieldMapper.java +++ b/src/main/java/org/opensearch/knn/index/mapper/ModelFieldMapper.java @@ -303,7 +303,7 @@ private static KNNMethodConfigContext getKNNMethodConfigContextFromModelMetadata return KNNMethodConfigContext.builder() .vectorDataType(modelMetadata.getVectorDataType()) .dimension(modelMetadata.getDimension()) - .versionCreated(Version.V_2_14_0) + .versionCreated(modelMetadata.getModelVersion()) .mode(modelMetadata.getMode()) .compressionLevel(modelMetadata.getCompressionLevel()) .build(); diff --git a/src/main/java/org/opensearch/knn/index/util/IndexUtil.java b/src/main/java/org/opensearch/knn/index/util/IndexUtil.java index 4a0118f58..02aa1e954 100644 --- a/src/main/java/org/opensearch/knn/index/util/IndexUtil.java +++ b/src/main/java/org/opensearch/knn/index/util/IndexUtil.java @@ -54,6 +54,7 @@ public class IndexUtil { private static final Version MINIMAL_RESCORE_FEATURE = Version.V_2_17_0; private static final Version MINIMAL_MODE_AND_COMPRESSION_FEATURE = Version.V_2_17_0; private static final Version MINIMAL_TOP_LEVEL_SPACE_TYPE_FEATURE = Version.V_2_17_0; + private static final Version MINIMAL_SUPPORTED_VERSION_FOR_MODEL_VERSION = Version.V_2_17_0; // public so neural search can access it public static final Map minimalRequiredVersionMap = initializeMinimalRequiredVersionMap(); public static final Set VECTOR_DATA_TYPES_NOT_SUPPORTING_ENCODERS = Set.of(VectorDataType.BINARY, VectorDataType.BYTE); @@ -394,6 +395,7 @@ private static Map initializeMinimalRequiredVersionMap() { put(RESCORE_PARAMETER, MINIMAL_RESCORE_FEATURE); put(KNNConstants.MINIMAL_MODE_AND_COMPRESSION_FEATURE, MINIMAL_MODE_AND_COMPRESSION_FEATURE); put(KNNConstants.TOP_LEVEL_SPACE_TYPE_FEATURE, MINIMAL_TOP_LEVEL_SPACE_TYPE_FEATURE); + put(KNNConstants.MODEL_VERSION, MINIMAL_SUPPORTED_VERSION_FOR_MODEL_VERSION); } }; diff --git a/src/main/java/org/opensearch/knn/indices/ModelDao.java b/src/main/java/org/opensearch/knn/indices/ModelDao.java index 326d595a4..d0abe8612 100644 --- a/src/main/java/org/opensearch/knn/indices/ModelDao.java +++ b/src/main/java/org/opensearch/knn/indices/ModelDao.java @@ -301,6 +301,7 @@ private void putInternal(Model model, ActionListener listener, Do if (CompressionLevel.isConfigured(modelMetadata.getCompressionLevel())) { put(KNNConstants.COMPRESSION_LEVEL_PARAMETER, modelMetadata.getCompressionLevel().getName()); } + put(KNNConstants.MODEL_VERSION, modelMetadata.getModelVersion()); MethodComponentContext methodComponentContext = modelMetadata.getMethodComponentContext(); if (!methodComponentContext.getName().isEmpty()) { XContentBuilder builder = XContentFactory.jsonBuilder().startObject(); diff --git a/src/main/java/org/opensearch/knn/indices/ModelMetadata.java b/src/main/java/org/opensearch/knn/indices/ModelMetadata.java index 620e520ba..17eed833e 100644 --- a/src/main/java/org/opensearch/knn/indices/ModelMetadata.java +++ b/src/main/java/org/opensearch/knn/indices/ModelMetadata.java @@ -15,6 +15,7 @@ import lombok.extern.log4j.Log4j2; import org.apache.commons.lang.builder.EqualsBuilder; import org.apache.commons.lang.builder.HashCodeBuilder; +import org.opensearch.Version; import org.opensearch.common.xcontent.json.JsonXContent; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; @@ -59,6 +60,7 @@ public class ModelMetadata implements Writeable, ToXContentObject { private String error; @Getter private final CompressionLevel compressionLevel; + private final Version version; /** * Constructor @@ -66,7 +68,6 @@ public class ModelMetadata implements Writeable, ToXContentObject { * @param in Stream input */ public ModelMetadata(StreamInput in) throws IOException { - String tempTrainingNodeAssignment; this.knnEngine = KNNEngine.getEngine(in.readString()); this.spaceType = SpaceType.getSpace(in.readString()); this.dimension = in.readInt(); @@ -96,7 +97,6 @@ public ModelMetadata(StreamInput in) throws IOException { } else { this.vectorDataType = VectorDataType.DEFAULT; } - if (IndexUtil.isVersionOnOrAfterMinRequiredVersion(in.getVersion(), KNNConstants.MINIMAL_MODE_AND_COMPRESSION_FEATURE)) { this.mode = Mode.fromName(in.readOptionalString()); this.compressionLevel = CompressionLevel.fromName(in.readOptionalString()); @@ -105,6 +105,11 @@ public ModelMetadata(StreamInput in) throws IOException { this.compressionLevel = CompressionLevel.NOT_CONFIGURED; } + if (IndexUtil.isVersionOnOrAfterMinRequiredVersion(in.getVersion(), KNNConstants.MODEL_VERSION)) { + this.version = Version.fromString(in.readString()); + } else { + this.version = Version.V_EMPTY; + } } /** @@ -133,7 +138,8 @@ public ModelMetadata( MethodComponentContext methodComponentContext, VectorDataType vectorDataType, Mode mode, - CompressionLevel compressionLevel + CompressionLevel compressionLevel, + Version version ) { this.knnEngine = Objects.requireNonNull(knnEngine, "knnEngine must not be null"); this.spaceType = Objects.requireNonNull(spaceType, "spaceType must not be null"); @@ -159,6 +165,7 @@ public ModelMetadata( this.vectorDataType = Objects.requireNonNull(vectorDataType, "vector data type must not be null"); this.mode = Objects.requireNonNull(mode, "Mode must not be null"); this.compressionLevel = Objects.requireNonNull(compressionLevel, "Compression level must not be null"); + this.version = Objects.requireNonNull(version, "model version must not be null"); } /** @@ -246,6 +253,14 @@ public VectorDataType getVectorDataType() { return vectorDataType; } + /** + * Getter for the model version + * @return version + */ + public Version getModelVersion() { + return version; + } + /** * setter for model's state * @@ -279,7 +294,8 @@ public String toString() { methodComponentContext.toClusterStateString(), vectorDataType.getValue(), mode.getName(), - compressionLevel.getName() + compressionLevel.getName(), + version.toString() ); } @@ -317,6 +333,7 @@ public int hashCode() { .append(getVectorDataType()) .append(getMode()) .append(getCompressionLevel()) + .append(getModelVersion()) .toHashCode(); } @@ -329,15 +346,15 @@ public int hashCode() { public static ModelMetadata fromString(String modelMetadataString) { String[] modelMetadataArray = modelMetadataString.split(DELIMITER, -1); int length = modelMetadataArray.length; - - if (length < 7 || length > 12) { + if (length < 7 || length > 13) { throw new IllegalArgumentException( "Illegal format for model metadata. Must be of the form " + "\",,,,,,\" or " + "\",,,,,,,\" or " + "\",,,,,,,,\" or " + "\",,,,,,,,,\". or " - + "\",,,,,,,,,,,\"." + + "\",,,,,,,,,,,\" or " + + "\",,,,,,,,,,,,\"." ); } @@ -357,6 +374,7 @@ public static ModelMetadata fromString(String modelMetadataString) { CompressionLevel compressionLevel = length > 11 ? CompressionLevel.fromName(modelMetadataArray[11]) : CompressionLevel.NOT_CONFIGURED; + Version version = length > 12 ? Version.fromString(modelMetadataArray[12]) : Version.V_EMPTY; log.debug(getLogMessage(length)); @@ -372,7 +390,8 @@ public static ModelMetadata fromString(String modelMetadataString) { methodComponentContext, vectorDataType, mode, - compressionLevel + compressionLevel, + version ); } @@ -386,9 +405,10 @@ private static String getLogMessage(int length) { return "Model metadata contains training node assignment and method context."; case 10: return "Model metadata contains training node assignment, method context and vector data type."; - case 11: case 12: return "Model metadata contains mode and compression level"; + case 13: + return "Model metadata contains training node assignment, method context, vector data type, and version"; default: throw new IllegalArgumentException("Unexpected metadata array length: " + length); } @@ -423,6 +443,7 @@ public static ModelMetadata getMetadataFromSourceMap(final Map m Object vectorDataType = modelSourceMap.get(KNNConstants.VECTOR_DATA_TYPE_FIELD); Object mode = modelSourceMap.get(KNNConstants.MODE_PARAMETER); Object compressionLevel = modelSourceMap.get(KNNConstants.COMPRESSION_LEVEL_PARAMETER); + Object version = modelSourceMap.get(KNNConstants.MODEL_VERSION); if (trainingNodeAssignment == null) { trainingNodeAssignment = ""; @@ -447,6 +468,10 @@ public static ModelMetadata getMetadataFromSourceMap(final Map m vectorDataType = VectorDataType.DEFAULT.getValue(); } + if (version == null) { + version = Version.V_EMPTY; + } + ModelMetadata modelMetadata = new ModelMetadata( KNNEngine.getEngine(objectToString(engine)), SpaceType.getSpace(objectToString(space)), @@ -459,7 +484,8 @@ public static ModelMetadata getMetadataFromSourceMap(final Map m (MethodComponentContext) methodComponentContext, VectorDataType.get(objectToString(vectorDataType)), Mode.fromName(objectToString(mode)), - CompressionLevel.fromName(objectToString(compressionLevel)) + CompressionLevel.fromName(objectToString(compressionLevel)), + Version.fromString(version.toString()) ); return modelMetadata; } @@ -486,6 +512,9 @@ public void writeTo(StreamOutput out) throws IOException { out.writeOptionalString(mode.getName()); out.writeOptionalString(compressionLevel.getName()); } + if (IndexUtil.isVersionOnOrAfterMinRequiredVersion(out.getVersion(), KNNConstants.MODEL_VERSION)) { + out.writeString(version.toString()); + } } @Override @@ -517,6 +546,13 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field(KNNConstants.COMPRESSION_LEVEL_PARAMETER, compressionLevel.getName()); } } + if (IndexUtil.isClusterOnOrAfterMinRequiredVersion(KNNConstants.MODEL_VERSION)) { + String versionString = "unknown"; + if (version != Version.V_EMPTY) { + versionString = version.toString(); + } + builder.field(KNNConstants.MODEL_VERSION, versionString); + } return builder; } } diff --git a/src/main/java/org/opensearch/knn/indices/ModelUtil.java b/src/main/java/org/opensearch/knn/indices/ModelUtil.java index ac0e4fb79..d63f02b2b 100644 --- a/src/main/java/org/opensearch/knn/indices/ModelUtil.java +++ b/src/main/java/org/opensearch/knn/indices/ModelUtil.java @@ -48,8 +48,8 @@ public static ModelMetadata getModelMetadata(final String modelId) { if (StringUtils.isEmpty(modelId)) { return null; } - final Model model = ModelCache.getInstance().get(modelId); - final ModelMetadata modelMetadata = model.getModelMetadata(); + ModelDao modelDao = ModelDao.OpenSearchKNNModelDao.getInstance(); + final ModelMetadata modelMetadata = modelDao.getMetadata(modelId); if (isModelCreated(modelMetadata) == false) { throw new IllegalArgumentException(String.format(Locale.ROOT, "Model ID '%s' is not created.", modelId)); } diff --git a/src/main/java/org/opensearch/knn/training/TrainingJob.java b/src/main/java/org/opensearch/knn/training/TrainingJob.java index 90b2762c2..b479192e8 100644 --- a/src/main/java/org/opensearch/knn/training/TrainingJob.java +++ b/src/main/java/org/opensearch/knn/training/TrainingJob.java @@ -99,7 +99,8 @@ public TrainingJob( knnMethodContext.getMethodComponentContext(), knnMethodConfigContext.getVectorDataType(), mode, - compressionLevel + compressionLevel, + knnMethodConfigContext.getVersionCreated() ), null, this.modelId diff --git a/src/main/java/org/opensearch/knn/training/TrainingJobRunner.java b/src/main/java/org/opensearch/knn/training/TrainingJobRunner.java index 8884f8102..5b2bb26b6 100644 --- a/src/main/java/org/opensearch/knn/training/TrainingJobRunner.java +++ b/src/main/java/org/opensearch/knn/training/TrainingJobRunner.java @@ -16,7 +16,6 @@ import org.opensearch.core.action.ActionListener; import org.opensearch.action.index.IndexResponse; import org.opensearch.common.ValidationException; -import org.opensearch.knn.indices.Model; import org.opensearch.knn.indices.ModelDao; import org.opensearch.knn.indices.ModelMetadata; import org.opensearch.knn.indices.ModelState; @@ -166,11 +165,11 @@ private void train(TrainingJob trainingJob) { private void serializeModel(TrainingJob trainingJob, ActionListener listener, boolean update) throws IOException, ExecutionException, InterruptedException { if (update) { - Model model = modelDao.get(trainingJob.getModelId()); - if (model.getModelMetadata().getState().equals(ModelState.TRAINING)) { + ModelMetadata modelMetadata = modelDao.getMetadata(trainingJob.getModelId()); + if (modelMetadata.getState().equals(ModelState.TRAINING)) { modelDao.update(trainingJob.getModel(), listener); } else { - logger.info("Model state is {}. Skipping serialization of trained data", model.getModelMetadata().getState()); + logger.info("Model state is {}. Skipping serialization of trained data", modelMetadata.getState()); } } else { modelDao.put(trainingJob.getModel(), listener); diff --git a/src/main/resources/mappings/model-index.json b/src/main/resources/mappings/model-index.json index e7879cced..8d16b98ab 100644 --- a/src/main/resources/mappings/model-index.json +++ b/src/main/resources/mappings/model-index.json @@ -38,6 +38,9 @@ }, "compression_level": { "type": "keyword" + }, + "model_version": { + "type": "keyword" } } } diff --git a/src/test/java/org/opensearch/knn/index/KNNCreateIndexFromModelTests.java b/src/test/java/org/opensearch/knn/index/KNNCreateIndexFromModelTests.java index 28ef41e04..d6ee2d7fd 100644 --- a/src/test/java/org/opensearch/knn/index/KNNCreateIndexFromModelTests.java +++ b/src/test/java/org/opensearch/knn/index/KNNCreateIndexFromModelTests.java @@ -12,6 +12,7 @@ package org.opensearch.knn.index; import com.google.common.collect.ImmutableMap; +import org.opensearch.Version; import org.opensearch.core.action.ActionListener; import org.opensearch.action.admin.indices.create.CreateIndexRequestBuilder; import org.opensearch.common.settings.Settings; @@ -69,7 +70,8 @@ public void testCreateIndexFromModel() throws IOException, InterruptedException MethodComponentContext.EMPTY, VectorDataType.FLOAT, Mode.NOT_CONFIGURED, - CompressionLevel.NOT_CONFIGURED + CompressionLevel.NOT_CONFIGURED, + Version.V_EMPTY ); Model model = new Model(modelMetadata, modelBlob, modelId); diff --git a/src/test/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumerTests.java b/src/test/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumerTests.java index 786061af8..e1f16006d 100644 --- a/src/test/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumerTests.java +++ b/src/test/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumerTests.java @@ -18,6 +18,8 @@ import org.apache.lucene.store.IOContext; import org.junit.AfterClass; import org.junit.BeforeClass; +import org.mockito.MockedStatic; +import org.mockito.Mockito; import org.opensearch.Version; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.settings.ClusterSettings; @@ -460,85 +462,92 @@ public void testAddKNNBinaryField_fromModel_faiss() throws IOException, Executio ); byte[] modelBytes = JNIService.trainIndex(parameters, dimension, trainingPtr, knnEngine); - Model model = new Model( - new ModelMetadata( - knnEngine, - spaceType, - dimension, - ModelState.CREATED, - "timestamp", - "Empty description", - "", - "", - MethodComponentContext.EMPTY, - VectorDataType.FLOAT, - Mode.NOT_CONFIGURED, - CompressionLevel.NOT_CONFIGURED - ), - modelBytes, - modelId + ModelMetadata modelMetadata = new ModelMetadata( + knnEngine, + spaceType, + dimension, + ModelState.CREATED, + "timestamp", + "Empty description", + "", + "", + MethodComponentContext.EMPTY, + VectorDataType.FLOAT, + Mode.NOT_CONFIGURED, + CompressionLevel.NOT_CONFIGURED, + Version.V_EMPTY ); + Model model = new Model(modelMetadata, modelBytes, modelId); JNICommons.freeVectorData(trainingPtr); - // Setup the model cache to return the correct model - ModelDao modelDao = mock(ModelDao.class); - when(modelDao.get(modelId)).thenReturn(model); - ClusterService clusterService = mock(ClusterService.class); - when(clusterService.getSettings()).thenReturn(Settings.EMPTY); - - ClusterSettings clusterSettings = new ClusterSettings( - Settings.builder().put(MODEL_CACHE_SIZE_LIMIT_SETTING.getKey(), "10kb").build(), - ImmutableSet.of(MODEL_CACHE_SIZE_LIMIT_SETTING) - ); - - when(clusterService.getClusterSettings()).thenReturn(clusterSettings); - ModelCache.initialize(modelDao, clusterService); - - // Build the segment and field info - String segmentName = String.format("test_segment%s", randomAlphaOfLength(4)); - int docsInSegment = 100; - String fieldName = String.format("test_field%s", randomAlphaOfLength(4)); - - SegmentInfo segmentInfo = KNNCodecTestUtil.segmentInfoBuilder() - .directory(directory) - .segmentName(segmentName) - .docsInSegment(docsInSegment) - .codec(codec) - .build(); - - FieldInfo[] fieldInfoArray = new FieldInfo[] { - KNNCodecTestUtil.FieldInfoBuilder.builder(fieldName) - .addAttribute(KNNVectorFieldMapper.KNN_FIELD, "true") - .addAttribute(MODEL_ID, modelId) - .build() }; - - FieldInfos fieldInfos = new FieldInfos(fieldInfoArray); - SegmentWriteState state = new SegmentWriteState(null, directory, segmentInfo, fieldInfos, null, IOContext.DEFAULT); - - long initialMergeOperations = KNNGraphValue.MERGE_TOTAL_OPERATIONS.getValue(); - - // Add documents to the field - KNN80DocValuesConsumer knn80DocValuesConsumer = new KNN80DocValuesConsumer(null, state); - TestVectorValues.RandomVectorDocValuesProducer randomVectorDocValuesProducer = new TestVectorValues.RandomVectorDocValuesProducer( - docsInSegment, - dimension - ); - knn80DocValuesConsumer.addKNNBinaryField(fieldInfoArray[0], randomVectorDocValuesProducer, true); - - // The document should be created in the correct location - String expectedFile = KNNCodecUtil.buildEngineFileName(segmentName, knnEngine.getVersion(), fieldName, knnEngine.getExtension()); - assertFileInCorrectLocation(state, expectedFile); - - // The footer should be valid - assertValidFooter(state.directory, expectedFile); - - // The document should be readable by faiss - assertLoadableByEngine(HNSW_METHODPARAMETERS, state, expectedFile, knnEngine, spaceType, dimension); - - // The graph creation statistics should be updated - assertEquals(1 + initialMergeOperations, (long) KNNGraphValue.MERGE_TOTAL_OPERATIONS.getValue()); - assertNotEquals(0, (long) KNNGraphValue.MERGE_TOTAL_DOCS.getValue()); - assertNotEquals(0, (long) KNNGraphValue.MERGE_TOTAL_SIZE_IN_BYTES.getValue()); + try (MockedStatic modelDaoMockedStatic = Mockito.mockStatic(ModelDao.OpenSearchKNNModelDao.class)) { + // Setup the model cache to return the correct model + ModelDao.OpenSearchKNNModelDao modelDao = mock(ModelDao.OpenSearchKNNModelDao.class); + when(modelDao.get(modelId)).thenReturn(model); + when(modelDao.getMetadata(modelId)).thenReturn(modelMetadata); + + modelDaoMockedStatic.when(ModelDao.OpenSearchKNNModelDao::getInstance).thenReturn(modelDao); + + ClusterService clusterService = mock(ClusterService.class); + when(clusterService.getSettings()).thenReturn(Settings.EMPTY); + + ClusterSettings clusterSettings = new ClusterSettings( + Settings.builder().put(MODEL_CACHE_SIZE_LIMIT_SETTING.getKey(), "10kb").build(), + ImmutableSet.of(MODEL_CACHE_SIZE_LIMIT_SETTING) + ); + + when(clusterService.getClusterSettings()).thenReturn(clusterSettings); + ModelCache.initialize(modelDao, clusterService); + + // Build the segment and field info + String segmentName = String.format("test_segment%s", randomAlphaOfLength(4)); + int docsInSegment = 100; + String fieldName = String.format("test_field%s", randomAlphaOfLength(4)); + + SegmentInfo segmentInfo = KNNCodecTestUtil.segmentInfoBuilder() + .directory(directory) + .segmentName(segmentName) + .docsInSegment(docsInSegment) + .codec(codec) + .build(); + + FieldInfo[] fieldInfoArray = new FieldInfo[] { + KNNCodecTestUtil.FieldInfoBuilder.builder(fieldName) + .addAttribute(KNNVectorFieldMapper.KNN_FIELD, "true") + .addAttribute(MODEL_ID, modelId) + .build() }; + + FieldInfos fieldInfos = new FieldInfos(fieldInfoArray); + SegmentWriteState state = new SegmentWriteState(null, directory, segmentInfo, fieldInfos, null, IOContext.DEFAULT); + + long initialMergeOperations = KNNGraphValue.MERGE_TOTAL_OPERATIONS.getValue(); + + // Add documents to the field + KNN80DocValuesConsumer knn80DocValuesConsumer = new KNN80DocValuesConsumer(null, state); + TestVectorValues.RandomVectorDocValuesProducer randomVectorDocValuesProducer = + new TestVectorValues.RandomVectorDocValuesProducer(docsInSegment, dimension); + knn80DocValuesConsumer.addKNNBinaryField(fieldInfoArray[0], randomVectorDocValuesProducer, true); + + // The document should be created in the correct location + String expectedFile = KNNCodecUtil.buildEngineFileName( + segmentName, + knnEngine.getVersion(), + fieldName, + knnEngine.getExtension() + ); + assertFileInCorrectLocation(state, expectedFile); + + // The footer should be valid + assertValidFooter(state.directory, expectedFile); + + // The document should be readable by faiss + assertLoadableByEngine(HNSW_METHODPARAMETERS, state, expectedFile, knnEngine, spaceType, dimension); + + // The graph creation statistics should be updated + assertEquals(1 + initialMergeOperations, (long) KNNGraphValue.MERGE_TOTAL_OPERATIONS.getValue()); + assertNotEquals(0, (long) KNNGraphValue.MERGE_TOTAL_DOCS.getValue()); + assertNotEquals(0, (long) KNNGraphValue.MERGE_TOTAL_SIZE_IN_BYTES.getValue()); + } } diff --git a/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestCase.java b/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestCase.java index 174441df8..3d9969a1e 100644 --- a/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestCase.java +++ b/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestCase.java @@ -16,6 +16,8 @@ import org.apache.lucene.search.Query; import org.apache.lucene.search.TopDocs; import org.apache.lucene.search.join.BitSetProducer; +import org.mockito.MockedStatic; +import org.opensearch.Version; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Setting; import org.opensearch.common.xcontent.XContentFactory; @@ -232,82 +234,86 @@ public void testBuildFromModelTemplate(Codec codec) throws IOException, Executio ); // Setup model cache - ModelDao modelDao = mock(ModelDao.class); - - // Set model state to created - ModelMetadata modelMetadata1 = new ModelMetadata( - knnEngine, - spaceType, - dimension, - ModelState.CREATED, - ZonedDateTime.now(ZoneOffset.UTC).toString(), - "", - "", - "", - MethodComponentContext.EMPTY, - VectorDataType.FLOAT, - Mode.NOT_CONFIGURED, - CompressionLevel.NOT_CONFIGURED - ); - - Model mockModel = new Model(modelMetadata1, modelBlob, modelId); - when(modelDao.get(modelId)).thenReturn(mockModel); - when(modelDao.getMetadata(modelId)).thenReturn(modelMetadata1); - - Settings settings = settings(CURRENT).put(MODEL_CACHE_SIZE_LIMIT_SETTING.getKey(), "10%").build(); - ClusterSettings clusterSettings = new ClusterSettings(settings, ImmutableSet.of(MODEL_CACHE_SIZE_LIMIT_SETTING)); - - ClusterService clusterService = mock(ClusterService.class); - when(clusterService.getSettings()).thenReturn(settings); - when(clusterService.getClusterSettings()).thenReturn(clusterSettings); - - ModelCache.initialize(modelDao, clusterService); - ModelCache.getInstance().removeAll(); - - // Setup Lucene - setUpMockClusterService(); - Directory dir = newFSDirectory(createTempDir()); - IndexWriterConfig iwc = newIndexWriterConfig(); - iwc.setMergeScheduler(new SerialMergeScheduler()); - iwc.setCodec(codec); - - FieldType fieldType = new FieldType(KNNVectorFieldMapper.Defaults.FIELD_TYPE); - fieldType.setDocValuesType(DocValuesType.BINARY); - fieldType.putAttribute(KNNConstants.MODEL_ID, modelId); - fieldType.freeze(); - - // Add the documents to the index - float[][] arrays = { { 1.0f, 3.0f, 4.0f }, { 2.0f, 5.0f, 8.0f }, { 3.0f, 6.0f, 9.0f }, { 4.0f, 7.0f, 10.0f } }; - - RandomIndexWriter writer = new RandomIndexWriter(random(), dir, iwc); - String fieldName = "test_vector"; - for (float[] array : arrays) { - VectorField vectorField = new VectorField(fieldName, array, fieldType); - Document doc = new Document(); - doc.add(vectorField); - writer.addDocument(doc); + try (MockedStatic modelDaoMockedStatic = Mockito.mockStatic(ModelDao.OpenSearchKNNModelDao.class)) { + ModelDao.OpenSearchKNNModelDao modelDao = mock(ModelDao.OpenSearchKNNModelDao.class); + modelDaoMockedStatic.when(ModelDao.OpenSearchKNNModelDao::getInstance).thenReturn(modelDao); + + // Set model state to created + ModelMetadata modelMetadata1 = new ModelMetadata( + knnEngine, + spaceType, + dimension, + ModelState.CREATED, + ZonedDateTime.now(ZoneOffset.UTC).toString(), + "", + "", + "", + MethodComponentContext.EMPTY, + VectorDataType.FLOAT, + Mode.NOT_CONFIGURED, + CompressionLevel.NOT_CONFIGURED, + Version.V_EMPTY + ); + + Model mockModel = new Model(modelMetadata1, modelBlob, modelId); + when(modelDao.get(modelId)).thenReturn(mockModel); + when(modelDao.getMetadata(modelId)).thenReturn(modelMetadata1); + + Settings settings = settings(CURRENT).put(MODEL_CACHE_SIZE_LIMIT_SETTING.getKey(), "10%").build(); + ClusterSettings clusterSettings = new ClusterSettings(settings, ImmutableSet.of(MODEL_CACHE_SIZE_LIMIT_SETTING)); + + ClusterService clusterService = mock(ClusterService.class); + when(clusterService.getSettings()).thenReturn(settings); + when(clusterService.getClusterSettings()).thenReturn(clusterSettings); + + ModelCache.initialize(modelDao, clusterService); + ModelCache.getInstance().removeAll(); + + // Setup Lucene + setUpMockClusterService(); + Directory dir = newFSDirectory(createTempDir()); + IndexWriterConfig iwc = newIndexWriterConfig(); + iwc.setMergeScheduler(new SerialMergeScheduler()); + iwc.setCodec(codec); + + FieldType fieldType = new FieldType(KNNVectorFieldMapper.Defaults.FIELD_TYPE); + fieldType.setDocValuesType(DocValuesType.BINARY); + fieldType.putAttribute(KNNConstants.MODEL_ID, modelId); + fieldType.freeze(); + + // Add the documents to the index + float[][] arrays = { { 1.0f, 3.0f, 4.0f }, { 2.0f, 5.0f, 8.0f }, { 3.0f, 6.0f, 9.0f }, { 4.0f, 7.0f, 10.0f } }; + + RandomIndexWriter writer = new RandomIndexWriter(random(), dir, iwc); + String fieldName = "test_vector"; + for (float[] array : arrays) { + VectorField vectorField = new VectorField(fieldName, array, fieldType); + Document doc = new Document(); + doc.add(vectorField); + writer.addDocument(doc); + } + + IndexReader reader = writer.getReader(); + writer.close(); + + // Make sure that search returns the correct results + KNNWeight.initialize(modelDao); + ResourceWatcherService resourceWatcherService = createDisabledResourceWatcherService(); + NativeMemoryLoadStrategy.IndexLoadStrategy.initialize(resourceWatcherService); + float[] query = { 10.0f, 10.0f, 10.0f }; + IndexSearcher searcher = new IndexSearcher(reader); + TopDocs topDocs = searcher.search(new KNNQuery(fieldName, query, 4, "dummy", (BitSetProducer) null), 10); + + assertEquals(3, topDocs.scoreDocs[0].doc); + assertEquals(2, topDocs.scoreDocs[1].doc); + assertEquals(1, topDocs.scoreDocs[2].doc); + assertEquals(0, topDocs.scoreDocs[3].doc); + + reader.close(); + dir.close(); + resourceWatcherService.close(); + NativeMemoryLoadStrategy.IndexLoadStrategy.getInstance().close(); } - - IndexReader reader = writer.getReader(); - writer.close(); - - // Make sure that search returns the correct results - KNNWeight.initialize(modelDao); - ResourceWatcherService resourceWatcherService = createDisabledResourceWatcherService(); - NativeMemoryLoadStrategy.IndexLoadStrategy.initialize(resourceWatcherService); - float[] query = { 10.0f, 10.0f, 10.0f }; - IndexSearcher searcher = new IndexSearcher(reader); - TopDocs topDocs = searcher.search(new KNNQuery(fieldName, query, 4, "dummy", (BitSetProducer) null), 10); - - assertEquals(3, topDocs.scoreDocs[0].doc); - assertEquals(2, topDocs.scoreDocs[1].doc); - assertEquals(1, topDocs.scoreDocs[2].doc); - assertEquals(0, topDocs.scoreDocs[3].doc); - - reader.close(); - dir.close(); - resourceWatcherService.close(); - NativeMemoryLoadStrategy.IndexLoadStrategy.getInstance().close(); } public void testWriteByOldCodec(Codec codec) throws IOException { diff --git a/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java b/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java index 21a116fd5..84cbf05dc 100644 --- a/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java +++ b/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java @@ -15,6 +15,7 @@ import org.junit.Assert; import org.mockito.MockedStatic; import org.mockito.Mockito; +import org.opensearch.Version; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.Explicit; import org.opensearch.common.ValidationException; @@ -374,7 +375,8 @@ public void testBuilder_build_fromModel() { MethodComponentContext.EMPTY, VectorDataType.FLOAT, Mode.NOT_CONFIGURED, - CompressionLevel.NOT_CONFIGURED + CompressionLevel.NOT_CONFIGURED, + Version.V_EMPTY ); builder.modelId.setValue(modelId); Mapper.BuilderContext builderContext = new Mapper.BuilderContext(settings, new ContentPath()); @@ -1025,7 +1027,8 @@ public void testKNNVectorFieldMapper_merge_fromModel() throws IOException { MethodComponentContext.EMPTY, VectorDataType.FLOAT, Mode.NOT_CONFIGURED, - CompressionLevel.NOT_CONFIGURED + CompressionLevel.NOT_CONFIGURED, + Version.V_EMPTY ); when(mockModelDao.getMetadata(modelId)).thenReturn(mockModelMetadata); diff --git a/src/test/java/org/opensearch/knn/indices/ModelCacheTests.java b/src/test/java/org/opensearch/knn/indices/ModelCacheTests.java index 91bb7d3d9..a31fe4a7d 100644 --- a/src/test/java/org/opensearch/knn/indices/ModelCacheTests.java +++ b/src/test/java/org/opensearch/knn/indices/ModelCacheTests.java @@ -13,6 +13,7 @@ import com.google.common.collect.ImmutableSet; import com.google.common.util.concurrent.UncheckedExecutionException; +import org.opensearch.Version; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; @@ -51,7 +52,8 @@ public void testGet_normal() throws ExecutionException, InterruptedException { MethodComponentContext.EMPTY, VectorDataType.DEFAULT, Mode.NOT_CONFIGURED, - CompressionLevel.NOT_CONFIGURED + CompressionLevel.NOT_CONFIGURED, + Version.V_EMPTY ), "hello".getBytes(), modelId @@ -91,7 +93,8 @@ public void testGet_modelDoesNotFitInCache() throws ExecutionException, Interrup MethodComponentContext.EMPTY, VectorDataType.DEFAULT, Mode.NOT_CONFIGURED, - CompressionLevel.NOT_CONFIGURED + CompressionLevel.NOT_CONFIGURED, + Version.V_EMPTY ), new byte[BYTES_PER_KILOBYTES + 1], modelId @@ -152,7 +155,8 @@ public void testGetTotalWeight() throws ExecutionException, InterruptedException MethodComponentContext.EMPTY, VectorDataType.DEFAULT, Mode.NOT_CONFIGURED, - CompressionLevel.NOT_CONFIGURED + CompressionLevel.NOT_CONFIGURED, + Version.V_EMPTY ), new byte[size1], modelId1 @@ -171,7 +175,8 @@ public void testGetTotalWeight() throws ExecutionException, InterruptedException MethodComponentContext.EMPTY, VectorDataType.DEFAULT, Mode.NOT_CONFIGURED, - CompressionLevel.NOT_CONFIGURED + CompressionLevel.NOT_CONFIGURED, + Version.V_EMPTY ), new byte[size2], modelId2 @@ -218,7 +223,8 @@ public void testRemove_normal() throws ExecutionException, InterruptedException MethodComponentContext.EMPTY, VectorDataType.DEFAULT, Mode.NOT_CONFIGURED, - CompressionLevel.NOT_CONFIGURED + CompressionLevel.NOT_CONFIGURED, + Version.V_EMPTY ), new byte[size1], modelId1 @@ -237,7 +243,8 @@ public void testRemove_normal() throws ExecutionException, InterruptedException MethodComponentContext.EMPTY, VectorDataType.DEFAULT, Mode.NOT_CONFIGURED, - CompressionLevel.NOT_CONFIGURED + CompressionLevel.NOT_CONFIGURED, + Version.V_EMPTY ), new byte[size2], modelId2 @@ -289,7 +296,8 @@ public void testRebuild_normal() throws ExecutionException, InterruptedException MethodComponentContext.EMPTY, VectorDataType.DEFAULT, Mode.NOT_CONFIGURED, - CompressionLevel.NOT_CONFIGURED + CompressionLevel.NOT_CONFIGURED, + Version.V_EMPTY ), "hello".getBytes(), modelId @@ -338,7 +346,8 @@ public void testRebuild_afterSettingUpdate() throws ExecutionException, Interrup MethodComponentContext.EMPTY, VectorDataType.DEFAULT, Mode.NOT_CONFIGURED, - CompressionLevel.NOT_CONFIGURED + CompressionLevel.NOT_CONFIGURED, + Version.V_EMPTY ), new byte[modelSize], modelId @@ -410,7 +419,8 @@ public void testContains() throws ExecutionException, InterruptedException { MethodComponentContext.EMPTY, VectorDataType.DEFAULT, Mode.NOT_CONFIGURED, - CompressionLevel.NOT_CONFIGURED + CompressionLevel.NOT_CONFIGURED, + Version.V_EMPTY ), new byte[modelSize1], modelId1 @@ -455,7 +465,8 @@ public void testRemoveAll() throws ExecutionException, InterruptedException { MethodComponentContext.EMPTY, VectorDataType.DEFAULT, Mode.NOT_CONFIGURED, - CompressionLevel.NOT_CONFIGURED + CompressionLevel.NOT_CONFIGURED, + Version.V_EMPTY ), new byte[modelSize1], modelId1 @@ -476,7 +487,8 @@ public void testRemoveAll() throws ExecutionException, InterruptedException { MethodComponentContext.EMPTY, VectorDataType.DEFAULT, Mode.NOT_CONFIGURED, - CompressionLevel.NOT_CONFIGURED + CompressionLevel.NOT_CONFIGURED, + Version.V_EMPTY ), new byte[modelSize2], modelId2 @@ -525,7 +537,8 @@ public void testModelCacheEvictionDueToSize() throws ExecutionException, Interru MethodComponentContext.EMPTY, VectorDataType.DEFAULT, Mode.NOT_CONFIGURED, - CompressionLevel.NOT_CONFIGURED + CompressionLevel.NOT_CONFIGURED, + Version.V_EMPTY ), new byte[BYTES_PER_KILOBYTES * 2], modelId diff --git a/src/test/java/org/opensearch/knn/indices/ModelDaoTests.java b/src/test/java/org/opensearch/knn/indices/ModelDaoTests.java index 560ea59b2..1edb5cff2 100644 --- a/src/test/java/org/opensearch/knn/indices/ModelDaoTests.java +++ b/src/test/java/org/opensearch/knn/indices/ModelDaoTests.java @@ -18,6 +18,7 @@ import org.opensearch.ExceptionsHelper; import org.opensearch.ResourceAlreadyExistsException; import org.opensearch.ResourceNotFoundException; +import org.opensearch.Version; import org.opensearch.cluster.ClusterChangedEvent; import org.opensearch.core.action.ActionListener; import org.opensearch.action.DocWriteResponse; @@ -145,7 +146,8 @@ public void testModelIndexHealth() throws InterruptedException, ExecutionExcepti MethodComponentContext.EMPTY, VectorDataType.DEFAULT, Mode.NOT_CONFIGURED, - CompressionLevel.NOT_CONFIGURED + CompressionLevel.NOT_CONFIGURED, + Version.CURRENT ), modelBlob, modelId @@ -168,7 +170,8 @@ public void testModelIndexHealth() throws InterruptedException, ExecutionExcepti MethodComponentContext.EMPTY, VectorDataType.DEFAULT, Mode.NOT_CONFIGURED, - CompressionLevel.NOT_CONFIGURED + CompressionLevel.NOT_CONFIGURED, + Version.CURRENT ), modelBlob, modelId @@ -199,7 +202,8 @@ public void testPut_withId() throws InterruptedException, IOException { new MethodComponentContext("test", Collections.emptyMap()), VectorDataType.DEFAULT, Mode.NOT_CONFIGURED, - CompressionLevel.NOT_CONFIGURED + CompressionLevel.NOT_CONFIGURED, + Version.CURRENT ), modelBlob, modelId @@ -263,7 +267,8 @@ public void testPut_withoutModel() throws InterruptedException, IOException { MethodComponentContext.EMPTY, VectorDataType.DEFAULT, Mode.NOT_CONFIGURED, - CompressionLevel.NOT_CONFIGURED + CompressionLevel.NOT_CONFIGURED, + Version.CURRENT ), modelBlob, modelId @@ -328,7 +333,8 @@ public void testPut_invalid_badState() { MethodComponentContext.EMPTY, VectorDataType.DEFAULT, Mode.NOT_CONFIGURED, - CompressionLevel.NOT_CONFIGURED + CompressionLevel.NOT_CONFIGURED, + Version.CURRENT ), modelBlob, "any-id" @@ -368,7 +374,8 @@ public void testUpdate() throws IOException, InterruptedException { MethodComponentContext.EMPTY, VectorDataType.DEFAULT, Mode.NOT_CONFIGURED, - CompressionLevel.NOT_CONFIGURED + CompressionLevel.NOT_CONFIGURED, + Version.CURRENT ), null, modelId @@ -410,7 +417,8 @@ public void testUpdate() throws IOException, InterruptedException { MethodComponentContext.EMPTY, VectorDataType.DEFAULT, Mode.NOT_CONFIGURED, - CompressionLevel.NOT_CONFIGURED + CompressionLevel.NOT_CONFIGURED, + Version.CURRENT ), modelBlob, modelId @@ -464,7 +472,8 @@ public void testGet() throws IOException, InterruptedException, ExecutionExcepti MethodComponentContext.EMPTY, VectorDataType.DEFAULT, Mode.NOT_CONFIGURED, - CompressionLevel.NOT_CONFIGURED + CompressionLevel.NOT_CONFIGURED, + Version.CURRENT ), modelBlob, modelId @@ -486,7 +495,8 @@ public void testGet() throws IOException, InterruptedException, ExecutionExcepti MethodComponentContext.EMPTY, VectorDataType.DEFAULT, Mode.NOT_CONFIGURED, - CompressionLevel.NOT_CONFIGURED + CompressionLevel.NOT_CONFIGURED, + Version.CURRENT ), null, modelId @@ -526,7 +536,8 @@ public void testGetMetadata() throws IOException, InterruptedException { MethodComponentContext.EMPTY, VectorDataType.DEFAULT, Mode.NOT_CONFIGURED, - CompressionLevel.NOT_CONFIGURED + CompressionLevel.NOT_CONFIGURED, + Version.CURRENT ); Model model = new Model(modelMetadata, modelBlob, modelId); @@ -606,7 +617,8 @@ public void testDelete() throws IOException, InterruptedException { MethodComponentContext.EMPTY, VectorDataType.DEFAULT, Mode.NOT_CONFIGURED, - CompressionLevel.NOT_CONFIGURED + CompressionLevel.NOT_CONFIGURED, + Version.CURRENT ), modelBlob, modelId @@ -643,7 +655,8 @@ public void testDelete() throws IOException, InterruptedException { MethodComponentContext.EMPTY, VectorDataType.DEFAULT, Mode.NOT_CONFIGURED, - CompressionLevel.NOT_CONFIGURED + CompressionLevel.NOT_CONFIGURED, + Version.CURRENT ), modelBlob, modelId1 @@ -714,7 +727,8 @@ public void testDeleteModelInTrainingWithStepListeners() throws IOException, Exe MethodComponentContext.EMPTY, VectorDataType.DEFAULT, Mode.NOT_CONFIGURED, - CompressionLevel.NOT_CONFIGURED + CompressionLevel.NOT_CONFIGURED, + Version.CURRENT ), modelBlob, modelId @@ -759,7 +773,8 @@ public void testDeleteWithStepListeners() throws IOException, InterruptedExcepti MethodComponentContext.EMPTY, VectorDataType.DEFAULT, Mode.NOT_CONFIGURED, - CompressionLevel.NOT_CONFIGURED + CompressionLevel.NOT_CONFIGURED, + Version.CURRENT ), modelBlob, modelId diff --git a/src/test/java/org/opensearch/knn/indices/ModelMetadataTests.java b/src/test/java/org/opensearch/knn/indices/ModelMetadataTests.java index 6f0b49285..79340d331 100644 --- a/src/test/java/org/opensearch/knn/indices/ModelMetadataTests.java +++ b/src/test/java/org/opensearch/knn/indices/ModelMetadataTests.java @@ -11,6 +11,7 @@ package org.opensearch.knn.indices; +import org.opensearch.Version; import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.ToXContent; @@ -51,7 +52,8 @@ public void testStreams() throws IOException { MethodComponentContext.EMPTY, VectorDataType.DEFAULT, Mode.NOT_CONFIGURED, - CompressionLevel.NOT_CONFIGURED + CompressionLevel.NOT_CONFIGURED, + Version.CURRENT ); BytesStreamOutput streamOutput = new BytesStreamOutput(); @@ -71,7 +73,8 @@ public void testStreams() throws IOException { MethodComponentContext.EMPTY, VectorDataType.DEFAULT, Mode.ON_DISK, - CompressionLevel.x16 + CompressionLevel.x16, + Version.CURRENT ); streamOutput = new BytesStreamOutput(); modelMetadata.writeTo(streamOutput); @@ -93,7 +96,8 @@ public void testGetKnnEngine() { MethodComponentContext.EMPTY, VectorDataType.DEFAULT, Mode.NOT_CONFIGURED, - CompressionLevel.NOT_CONFIGURED + CompressionLevel.NOT_CONFIGURED, + Version.CURRENT ); assertEquals(knnEngine, modelMetadata.getKnnEngine()); @@ -113,7 +117,8 @@ public void testGetSpaceType() { MethodComponentContext.EMPTY, VectorDataType.DEFAULT, Mode.NOT_CONFIGURED, - CompressionLevel.NOT_CONFIGURED + CompressionLevel.NOT_CONFIGURED, + Version.CURRENT ); assertEquals(spaceType, modelMetadata.getSpaceType()); @@ -133,7 +138,8 @@ public void testGetDimension() { MethodComponentContext.EMPTY, VectorDataType.DEFAULT, Mode.NOT_CONFIGURED, - CompressionLevel.NOT_CONFIGURED + CompressionLevel.NOT_CONFIGURED, + Version.CURRENT ); assertEquals(dimension, modelMetadata.getDimension()); @@ -153,7 +159,8 @@ public void testGetState() { MethodComponentContext.EMPTY, VectorDataType.DEFAULT, Mode.NOT_CONFIGURED, - CompressionLevel.NOT_CONFIGURED + CompressionLevel.NOT_CONFIGURED, + Version.CURRENT ); assertEquals(modelState, modelMetadata.getState()); @@ -173,7 +180,8 @@ public void testGetTimestamp() { MethodComponentContext.EMPTY, VectorDataType.DEFAULT, Mode.NOT_CONFIGURED, - CompressionLevel.NOT_CONFIGURED + CompressionLevel.NOT_CONFIGURED, + Version.CURRENT ); assertEquals(timeValue, modelMetadata.getTimestamp()); @@ -193,7 +201,8 @@ public void testDescription() { MethodComponentContext.EMPTY, VectorDataType.DEFAULT, Mode.NOT_CONFIGURED, - CompressionLevel.NOT_CONFIGURED + CompressionLevel.NOT_CONFIGURED, + Version.CURRENT ); assertEquals(description, modelMetadata.getDescription()); @@ -213,7 +222,8 @@ public void testGetError() { MethodComponentContext.EMPTY, VectorDataType.DEFAULT, Mode.NOT_CONFIGURED, - CompressionLevel.NOT_CONFIGURED + CompressionLevel.NOT_CONFIGURED, + Version.CURRENT ); assertEquals(error, modelMetadata.getError()); @@ -233,12 +243,34 @@ public void testGetVectorDataType() { MethodComponentContext.EMPTY, vectorDataType, Mode.NOT_CONFIGURED, - CompressionLevel.NOT_CONFIGURED + CompressionLevel.NOT_CONFIGURED, + Version.CURRENT ); assertEquals(vectorDataType, modelMetadata.getVectorDataType()); } + public void testGetModelVersion() { + Version version = Version.CURRENT; + ModelMetadata modelMetadata = new ModelMetadata( + KNNEngine.DEFAULT, + SpaceType.L2, + 12, + ModelState.CREATED, + ZonedDateTime.now(ZoneOffset.UTC).toString(), + "", + "", + "", + MethodComponentContext.EMPTY, + VectorDataType.DEFAULT, + Mode.NOT_CONFIGURED, + CompressionLevel.NOT_CONFIGURED, + version + ); + + assertEquals(version, modelMetadata.getModelVersion()); + } + public void testSetState() { ModelState modelState = ModelState.FAILED; ModelMetadata modelMetadata = new ModelMetadata( @@ -253,7 +285,8 @@ public void testSetState() { MethodComponentContext.EMPTY, VectorDataType.DEFAULT, Mode.NOT_CONFIGURED, - CompressionLevel.NOT_CONFIGURED + CompressionLevel.NOT_CONFIGURED, + Version.CURRENT ); assertEquals(modelState, modelMetadata.getState()); @@ -277,7 +310,8 @@ public void testSetError() { MethodComponentContext.EMPTY, VectorDataType.DEFAULT, Mode.NOT_CONFIGURED, - CompressionLevel.NOT_CONFIGURED + CompressionLevel.NOT_CONFIGURED, + Version.CURRENT ); assertEquals(error, modelMetadata.getError()); @@ -297,47 +331,9 @@ public void testToString() { String error = "test-error"; String nodeAssignment = ""; MethodComponentContext methodComponentContext = MethodComponentContext.EMPTY; + Version version = Version.CURRENT; String expected = knnEngine.getName() - + "," - + spaceType.getValue() - + "," - + dimension - + "," - + modelState.getName() - + "," - + timestamp - + "," - + description - + "," - + error - + "," - + nodeAssignment - + "," - + methodComponentContext.toClusterStateString() - + "," - + VectorDataType.DEFAULT.getValue() - + "," - + ","; - - ModelMetadata modelMetadata = new ModelMetadata( - knnEngine, - spaceType, - dimension, - modelState, - timestamp, - description, - error, - nodeAssignment, - MethodComponentContext.EMPTY, - VectorDataType.DEFAULT, - Mode.NOT_CONFIGURED, - CompressionLevel.NOT_CONFIGURED - ); - - assertEquals(expected, modelMetadata.toString()); - - expected = knnEngine.getName() + "," + spaceType.getValue() + "," @@ -359,9 +355,11 @@ public void testToString() { + "," + Mode.ON_DISK.getName() + "," - + CompressionLevel.x32.getName(); + + CompressionLevel.x32.getName() + + "," + + version; - modelMetadata = new ModelMetadata( + ModelMetadata modelMetadata = new ModelMetadata( knnEngine, spaceType, dimension, @@ -373,7 +371,8 @@ public void testToString() { MethodComponentContext.EMPTY, VectorDataType.DEFAULT, Mode.ON_DISK, - CompressionLevel.x32 + CompressionLevel.x32, + Version.CURRENT ); assertEquals(expected, modelMetadata.toString()); @@ -396,7 +395,8 @@ public void testEquals() { MethodComponentContext.EMPTY, VectorDataType.DEFAULT, Mode.NOT_CONFIGURED, - CompressionLevel.NOT_CONFIGURED + CompressionLevel.NOT_CONFIGURED, + Version.CURRENT ); ModelMetadata modelMetadata2 = new ModelMetadata( KNNEngine.FAISS, @@ -410,7 +410,8 @@ public void testEquals() { MethodComponentContext.EMPTY, VectorDataType.DEFAULT, Mode.NOT_CONFIGURED, - CompressionLevel.NOT_CONFIGURED + CompressionLevel.NOT_CONFIGURED, + Version.CURRENT ); ModelMetadata modelMetadata3 = new ModelMetadata( @@ -425,7 +426,8 @@ public void testEquals() { MethodComponentContext.EMPTY, VectorDataType.DEFAULT, Mode.NOT_CONFIGURED, - CompressionLevel.NOT_CONFIGURED + CompressionLevel.NOT_CONFIGURED, + Version.CURRENT ); ModelMetadata modelMetadata4 = new ModelMetadata( KNNEngine.FAISS, @@ -439,7 +441,8 @@ public void testEquals() { MethodComponentContext.EMPTY, VectorDataType.DEFAULT, Mode.NOT_CONFIGURED, - CompressionLevel.NOT_CONFIGURED + CompressionLevel.NOT_CONFIGURED, + Version.CURRENT ); ModelMetadata modelMetadata5 = new ModelMetadata( KNNEngine.FAISS, @@ -453,7 +456,8 @@ public void testEquals() { MethodComponentContext.EMPTY, VectorDataType.DEFAULT, Mode.NOT_CONFIGURED, - CompressionLevel.NOT_CONFIGURED + CompressionLevel.NOT_CONFIGURED, + Version.CURRENT ); ModelMetadata modelMetadata6 = new ModelMetadata( KNNEngine.FAISS, @@ -467,7 +471,8 @@ public void testEquals() { MethodComponentContext.EMPTY, VectorDataType.DEFAULT, Mode.NOT_CONFIGURED, - CompressionLevel.NOT_CONFIGURED + CompressionLevel.NOT_CONFIGURED, + Version.CURRENT ); ModelMetadata modelMetadata7 = new ModelMetadata( KNNEngine.FAISS, @@ -481,7 +486,8 @@ public void testEquals() { MethodComponentContext.EMPTY, VectorDataType.DEFAULT, Mode.NOT_CONFIGURED, - CompressionLevel.NOT_CONFIGURED + CompressionLevel.NOT_CONFIGURED, + Version.CURRENT ); ModelMetadata modelMetadata8 = new ModelMetadata( KNNEngine.FAISS, @@ -495,7 +501,8 @@ public void testEquals() { MethodComponentContext.EMPTY, VectorDataType.DEFAULT, Mode.NOT_CONFIGURED, - CompressionLevel.NOT_CONFIGURED + CompressionLevel.NOT_CONFIGURED, + Version.CURRENT ); ModelMetadata modelMetadata9 = new ModelMetadata( KNNEngine.FAISS, @@ -509,7 +516,8 @@ public void testEquals() { MethodComponentContext.EMPTY, VectorDataType.DEFAULT, Mode.NOT_CONFIGURED, - CompressionLevel.NOT_CONFIGURED + CompressionLevel.NOT_CONFIGURED, + Version.CURRENT ); ModelMetadata modelMetadata10 = new ModelMetadata( @@ -524,7 +532,8 @@ public void testEquals() { new MethodComponentContext("test", Collections.emptyMap()), VectorDataType.DEFAULT, Mode.NOT_CONFIGURED, - CompressionLevel.NOT_CONFIGURED + CompressionLevel.NOT_CONFIGURED, + Version.CURRENT ); assertEquals(modelMetadata1, modelMetadata1); @@ -557,7 +566,8 @@ public void testHashCode() { MethodComponentContext.EMPTY, VectorDataType.DEFAULT, Mode.NOT_CONFIGURED, - CompressionLevel.NOT_CONFIGURED + CompressionLevel.NOT_CONFIGURED, + Version.CURRENT ); ModelMetadata modelMetadata2 = new ModelMetadata( KNNEngine.FAISS, @@ -571,7 +581,8 @@ public void testHashCode() { MethodComponentContext.EMPTY, VectorDataType.DEFAULT, Mode.NOT_CONFIGURED, - CompressionLevel.NOT_CONFIGURED + CompressionLevel.NOT_CONFIGURED, + Version.CURRENT ); ModelMetadata modelMetadata3 = new ModelMetadata( @@ -586,7 +597,8 @@ public void testHashCode() { MethodComponentContext.EMPTY, VectorDataType.DEFAULT, Mode.NOT_CONFIGURED, - CompressionLevel.NOT_CONFIGURED + CompressionLevel.NOT_CONFIGURED, + Version.CURRENT ); ModelMetadata modelMetadata4 = new ModelMetadata( KNNEngine.FAISS, @@ -600,7 +612,8 @@ public void testHashCode() { MethodComponentContext.EMPTY, VectorDataType.DEFAULT, Mode.NOT_CONFIGURED, - CompressionLevel.NOT_CONFIGURED + CompressionLevel.NOT_CONFIGURED, + Version.CURRENT ); ModelMetadata modelMetadata5 = new ModelMetadata( KNNEngine.FAISS, @@ -614,7 +627,8 @@ public void testHashCode() { MethodComponentContext.EMPTY, VectorDataType.DEFAULT, Mode.NOT_CONFIGURED, - CompressionLevel.NOT_CONFIGURED + CompressionLevel.NOT_CONFIGURED, + Version.CURRENT ); ModelMetadata modelMetadata6 = new ModelMetadata( KNNEngine.FAISS, @@ -628,7 +642,8 @@ public void testHashCode() { MethodComponentContext.EMPTY, VectorDataType.DEFAULT, Mode.NOT_CONFIGURED, - CompressionLevel.NOT_CONFIGURED + CompressionLevel.NOT_CONFIGURED, + Version.CURRENT ); ModelMetadata modelMetadata7 = new ModelMetadata( KNNEngine.FAISS, @@ -642,7 +657,8 @@ public void testHashCode() { MethodComponentContext.EMPTY, VectorDataType.DEFAULT, Mode.NOT_CONFIGURED, - CompressionLevel.NOT_CONFIGURED + CompressionLevel.NOT_CONFIGURED, + Version.CURRENT ); ModelMetadata modelMetadata8 = new ModelMetadata( KNNEngine.FAISS, @@ -656,7 +672,8 @@ public void testHashCode() { MethodComponentContext.EMPTY, VectorDataType.DEFAULT, Mode.NOT_CONFIGURED, - CompressionLevel.NOT_CONFIGURED + CompressionLevel.NOT_CONFIGURED, + Version.CURRENT ); ModelMetadata modelMetadata9 = new ModelMetadata( KNNEngine.FAISS, @@ -670,7 +687,8 @@ public void testHashCode() { MethodComponentContext.EMPTY, VectorDataType.DEFAULT, Mode.NOT_CONFIGURED, - CompressionLevel.NOT_CONFIGURED + CompressionLevel.NOT_CONFIGURED, + Version.CURRENT ); ModelMetadata modelMetadata10 = new ModelMetadata( @@ -685,7 +703,8 @@ public void testHashCode() { new MethodComponentContext("test", Collections.emptyMap()), VectorDataType.DEFAULT, Mode.NOT_CONFIGURED, - CompressionLevel.NOT_CONFIGURED + CompressionLevel.NOT_CONFIGURED, + Version.CURRENT ); assertEquals(modelMetadata1.hashCode(), modelMetadata1.hashCode()); @@ -711,6 +730,7 @@ public void testFromString() { String error = "test-error"; String nodeAssignment = "test-node"; MethodComponentContext methodComponentContext = MethodComponentContext.EMPTY; + Version version = Version.CURRENT; String stringRep1 = knnEngine.getName() + "," @@ -730,7 +750,11 @@ public void testFromString() { + "," + methodComponentContext.toClusterStateString() + "," - + VectorDataType.DEFAULT.getValue(); + + VectorDataType.DEFAULT.getValue() + + "," + + "," + + "," + + version.toString(); String stringRep2 = knnEngine.getName() + "," @@ -768,7 +792,9 @@ public void testFromString() { + "," + VectorDataType.DEFAULT.getValue() + "," - + ","; + + "," + + "," + + version; String stringRep4 = knnEngine.getName() + "," @@ -806,7 +832,8 @@ public void testFromString() { MethodComponentContext.EMPTY, VectorDataType.DEFAULT, Mode.NOT_CONFIGURED, - CompressionLevel.NOT_CONFIGURED + CompressionLevel.NOT_CONFIGURED, + Version.V_EMPTY ); ModelMetadata expected2 = new ModelMetadata( @@ -821,7 +848,8 @@ public void testFromString() { MethodComponentContext.EMPTY, VectorDataType.DEFAULT, Mode.NOT_CONFIGURED, - CompressionLevel.NOT_CONFIGURED + CompressionLevel.NOT_CONFIGURED, + Version.V_EMPTY ); ModelMetadata expected3 = new ModelMetadata( @@ -836,7 +864,8 @@ public void testFromString() { MethodComponentContext.EMPTY, VectorDataType.DEFAULT, Mode.ON_DISK, - CompressionLevel.x32 + CompressionLevel.x32, + Version.CURRENT ); ModelMetadata fromString1 = ModelMetadata.fromString(stringRep1); @@ -863,6 +892,8 @@ public void testFromResponseMap() throws IOException { String nodeAssignment = "test-node"; MethodComponentContext methodComponentContext = getMethodComponentContext(); MethodComponentContext emptyMethodComponentContext = MethodComponentContext.EMPTY; + Version version = Version.CURRENT; + Version emptyVersion = Version.V_EMPTY; ModelMetadata expected = new ModelMetadata( knnEngine, @@ -876,7 +907,8 @@ public void testFromResponseMap() throws IOException { methodComponentContext, VectorDataType.DEFAULT, Mode.NOT_CONFIGURED, - CompressionLevel.NOT_CONFIGURED + CompressionLevel.NOT_CONFIGURED, + version ); ModelMetadata expected2 = new ModelMetadata( knnEngine, @@ -890,7 +922,8 @@ public void testFromResponseMap() throws IOException { emptyMethodComponentContext, VectorDataType.DEFAULT, Mode.NOT_CONFIGURED, - CompressionLevel.NOT_CONFIGURED + CompressionLevel.NOT_CONFIGURED, + emptyVersion ); ModelMetadata expected3 = new ModelMetadata( @@ -905,7 +938,8 @@ public void testFromResponseMap() throws IOException { emptyMethodComponentContext, VectorDataType.DEFAULT, Mode.ON_DISK, - CompressionLevel.NOT_CONFIGURED + CompressionLevel.NOT_CONFIGURED, + Version.CURRENT ); ModelMetadata expected4 = new ModelMetadata( @@ -920,7 +954,8 @@ public void testFromResponseMap() throws IOException { emptyMethodComponentContext, VectorDataType.DEFAULT, Mode.ON_DISK, - CompressionLevel.x16 + CompressionLevel.x16, + Version.CURRENT ); Map metadataAsMap = new HashMap<>(); metadataAsMap.put(KNNConstants.KNN_ENGINE, knnEngine.getName()); @@ -936,12 +971,14 @@ public void testFromResponseMap() throws IOException { XContentBuilder builder = XContentFactory.jsonBuilder().startObject(); builder = methodComponentContext.toXContent(builder, ToXContent.EMPTY_PARAMS).endObject(); metadataAsMap.put(KNNConstants.MODEL_METHOD_COMPONENT_CONTEXT, builder.toString()); + metadataAsMap.put(KNNConstants.MODEL_VERSION, version.toString()); ModelMetadata fromMap = ModelMetadata.getMetadataFromSourceMap(metadataAsMap); assertEquals(expected, fromMap); metadataAsMap.put(KNNConstants.MODEL_NODE_ASSIGNMENT, null); metadataAsMap.put(KNNConstants.MODEL_METHOD_COMPONENT_CONTEXT, null); + metadataAsMap.put(KNNConstants.MODEL_VERSION, emptyVersion); assertEquals(expected2, fromMap); metadataAsMap.put(KNNConstants.MODE_PARAMETER, Mode.ON_DISK.getName()); @@ -978,7 +1015,8 @@ public void testBlockCommasInDescription() { methodComponentContext, VectorDataType.DEFAULT, Mode.NOT_CONFIGURED, - CompressionLevel.NOT_CONFIGURED + CompressionLevel.NOT_CONFIGURED, + Version.CURRENT ) ); assertEquals("Model description cannot contain any commas: ','", e.getMessage()); diff --git a/src/test/java/org/opensearch/knn/indices/ModelTests.java b/src/test/java/org/opensearch/knn/indices/ModelTests.java index 4e666872f..59ecd66ec 100644 --- a/src/test/java/org/opensearch/knn/indices/ModelTests.java +++ b/src/test/java/org/opensearch/knn/indices/ModelTests.java @@ -11,6 +11,7 @@ package org.opensearch.knn.indices; +import org.opensearch.Version; import org.opensearch.knn.KNNTestCase; import org.opensearch.knn.common.KNNConstants; import org.opensearch.knn.index.engine.MethodComponentContext; @@ -47,7 +48,8 @@ public void testInvalidConstructor() { MethodComponentContext.EMPTY, VectorDataType.DEFAULT, Mode.NOT_CONFIGURED, - CompressionLevel.NOT_CONFIGURED + CompressionLevel.NOT_CONFIGURED, + Version.CURRENT ), null, "test-model" @@ -71,7 +73,8 @@ public void testInvalidDimension() { MethodComponentContext.EMPTY, VectorDataType.DEFAULT, Mode.NOT_CONFIGURED, - CompressionLevel.NOT_CONFIGURED + CompressionLevel.NOT_CONFIGURED, + Version.CURRENT ), new byte[16], "test-model" @@ -92,7 +95,8 @@ public void testInvalidDimension() { MethodComponentContext.EMPTY, VectorDataType.DEFAULT, Mode.NOT_CONFIGURED, - CompressionLevel.NOT_CONFIGURED + CompressionLevel.NOT_CONFIGURED, + Version.CURRENT ), new byte[16], "test-model" @@ -113,7 +117,8 @@ public void testInvalidDimension() { MethodComponentContext.EMPTY, VectorDataType.DEFAULT, Mode.NOT_CONFIGURED, - CompressionLevel.NOT_CONFIGURED + CompressionLevel.NOT_CONFIGURED, + Version.CURRENT ), new byte[16], "test-model" @@ -135,7 +140,8 @@ public void testGetModelMetadata() { MethodComponentContext.EMPTY, VectorDataType.DEFAULT, Mode.NOT_CONFIGURED, - CompressionLevel.NOT_CONFIGURED + CompressionLevel.NOT_CONFIGURED, + Version.CURRENT ); Model model = new Model(modelMetadata, new byte[16], "test-model"); assertEquals(modelMetadata, model.getModelMetadata()); @@ -156,7 +162,8 @@ public void testGetModelBlob() { MethodComponentContext.EMPTY, VectorDataType.DEFAULT, Mode.NOT_CONFIGURED, - CompressionLevel.NOT_CONFIGURED + CompressionLevel.NOT_CONFIGURED, + Version.CURRENT ), modelBlob, "test-model" @@ -179,7 +186,8 @@ public void testGetLength() { MethodComponentContext.EMPTY, VectorDataType.DEFAULT, Mode.NOT_CONFIGURED, - CompressionLevel.NOT_CONFIGURED + CompressionLevel.NOT_CONFIGURED, + Version.CURRENT ), new byte[size], "test-model" @@ -199,7 +207,8 @@ public void testGetLength() { MethodComponentContext.EMPTY, VectorDataType.DEFAULT, Mode.NOT_CONFIGURED, - CompressionLevel.NOT_CONFIGURED + CompressionLevel.NOT_CONFIGURED, + Version.CURRENT ), null, "test-model" @@ -222,7 +231,8 @@ public void testSetModelBlob() { MethodComponentContext.EMPTY, VectorDataType.DEFAULT, Mode.NOT_CONFIGURED, - CompressionLevel.NOT_CONFIGURED + CompressionLevel.NOT_CONFIGURED, + Version.CURRENT ), blob1, "test-model" @@ -251,7 +261,8 @@ public void testEquals() { MethodComponentContext.EMPTY, VectorDataType.DEFAULT, Mode.NOT_CONFIGURED, - CompressionLevel.NOT_CONFIGURED + CompressionLevel.NOT_CONFIGURED, + Version.CURRENT ), new byte[16], "test-model-1" @@ -269,7 +280,8 @@ public void testEquals() { MethodComponentContext.EMPTY, VectorDataType.DEFAULT, Mode.NOT_CONFIGURED, - CompressionLevel.NOT_CONFIGURED + CompressionLevel.NOT_CONFIGURED, + Version.CURRENT ), new byte[16], "test-model-1" @@ -287,13 +299,13 @@ public void testEquals() { MethodComponentContext.EMPTY, VectorDataType.DEFAULT, Mode.NOT_CONFIGURED, - CompressionLevel.NOT_CONFIGURED + CompressionLevel.NOT_CONFIGURED, + Version.CURRENT ), new byte[16], "test-model-2" ); - assertEquals(model1, model1); assertEquals(model1, model2); assertNotEquals(model1, model3); } @@ -315,7 +327,8 @@ public void testHashCode() { MethodComponentContext.EMPTY, VectorDataType.DEFAULT, Mode.NOT_CONFIGURED, - CompressionLevel.NOT_CONFIGURED + CompressionLevel.NOT_CONFIGURED, + Version.CURRENT ), new byte[16], "test-model-1" @@ -333,7 +346,8 @@ public void testHashCode() { MethodComponentContext.EMPTY, VectorDataType.DEFAULT, Mode.NOT_CONFIGURED, - CompressionLevel.NOT_CONFIGURED + CompressionLevel.NOT_CONFIGURED, + Version.CURRENT ), new byte[16], "test-model-1" @@ -351,7 +365,8 @@ public void testHashCode() { MethodComponentContext.EMPTY, VectorDataType.DEFAULT, Mode.NOT_CONFIGURED, - CompressionLevel.NOT_CONFIGURED + CompressionLevel.NOT_CONFIGURED, + Version.CURRENT ), new byte[16], "test-model-2" @@ -385,7 +400,8 @@ public void testModelFromSourceMap() { MethodComponentContext.EMPTY, VectorDataType.DEFAULT, Mode.NOT_CONFIGURED, - CompressionLevel.NOT_CONFIGURED + CompressionLevel.NOT_CONFIGURED, + Version.CURRENT ); Map modelAsMap = new HashMap<>(); modelAsMap.put(KNNConstants.MODEL_ID, modelID); diff --git a/src/test/java/org/opensearch/knn/indices/ModelUtilTests.java b/src/test/java/org/opensearch/knn/indices/ModelUtilTests.java index edefd10ee..45597b4c9 100644 --- a/src/test/java/org/opensearch/knn/indices/ModelUtilTests.java +++ b/src/test/java/org/opensearch/knn/indices/ModelUtilTests.java @@ -23,14 +23,19 @@ public void testGetModelMetadata_whenVariousInputs_thenSuccess() { MockedStatic modelCacheMockedStatic = Mockito.mockStatic(ModelCache.class); modelCacheMockedStatic.when(ModelCache::getInstance).thenReturn(modelCache); - - Mockito.when(modelCache.get(MODEL_ID)).thenReturn(model); - Mockito.when(model.getModelMetadata()).thenReturn(null); - Assert.assertThrows(IllegalArgumentException.class, () -> ModelUtil.getModelMetadata(MODEL_ID)); - - Mockito.when(model.getModelMetadata()).thenReturn(modelMetadata); - Mockito.when(modelMetadata.getState()).thenReturn(ModelState.CREATED); - Assert.assertNotNull(ModelUtil.getModelMetadata(MODEL_ID)); + try (MockedStatic modelDaoMockedStatic = Mockito.mockStatic(ModelDao.OpenSearchKNNModelDao.class)) { + ModelDao.OpenSearchKNNModelDao modelDao = Mockito.mock(ModelDao.OpenSearchKNNModelDao.class); + Mockito.when(modelDao.getMetadata(MODEL_ID)).thenReturn(modelMetadata); + Mockito.when(modelMetadata.getState()).thenReturn(ModelState.FAILED); + modelDaoMockedStatic.when(ModelDao.OpenSearchKNNModelDao::getInstance).thenReturn(modelDao); + + Mockito.when(modelCache.get(MODEL_ID)).thenReturn(model); + Mockito.when(model.getModelMetadata()).thenReturn(null); + Assert.assertThrows(IllegalArgumentException.class, () -> ModelUtil.getModelMetadata(MODEL_ID)); + + Mockito.when(modelMetadata.getState()).thenReturn(ModelState.CREATED); + Assert.assertNotNull(ModelUtil.getModelMetadata(MODEL_ID)); + } modelCacheMockedStatic.close(); } } diff --git a/src/test/java/org/opensearch/knn/plugin/transport/GetModelResponseTests.java b/src/test/java/org/opensearch/knn/plugin/transport/GetModelResponseTests.java index 7010dbf43..3bf5f302c 100644 --- a/src/test/java/org/opensearch/knn/plugin/transport/GetModelResponseTests.java +++ b/src/test/java/org/opensearch/knn/plugin/transport/GetModelResponseTests.java @@ -50,7 +50,8 @@ private ModelMetadata getModelMetadata(ModelState state) { MethodComponentContext.EMPTY, VectorDataType.DEFAULT, Mode.NOT_CONFIGURED, - CompressionLevel.NOT_CONFIGURED + CompressionLevel.NOT_CONFIGURED, + Version.CURRENT ); } @@ -74,8 +75,10 @@ public void testXContent() throws IOException { byte[] testModelBlob = "hello".getBytes(); Model model = new Model(getModelMetadata(ModelState.CREATED), testModelBlob, modelId); GetModelResponse getModelResponse = new GetModelResponse(model); - String expectedResponseString = - "{\"model_id\":\"test-model\",\"model_blob\":\"aGVsbG8=\",\"state\":\"created\",\"timestamp\":\"2021-03-27 10:15:30 AM +05:30\",\"description\":\"test model\",\"error\":\"\",\"space_type\":\"l2\",\"dimension\":4,\"engine\":\"nmslib\",\"training_node_assignment\":\"\",\"model_definition\":{\"name\":\"\",\"parameters\":{}},\"data_type\":\"float\"}"; + String expectedResponseString = String.format( + "{\"model_id\":\"test-model\",\"model_blob\":\"aGVsbG8=\",\"state\":\"created\",\"timestamp\":\"2021-03-27 10:15:30 AM +05:30\",\"description\":\"test model\",\"error\":\"\",\"space_type\":\"l2\",\"dimension\":4,\"engine\":\"nmslib\",\"training_node_assignment\":\"\",\"model_definition\":{\"name\":\"\",\"parameters\":{}},\"data_type\":\"float\",\"model_version\":\"%s\"}", + Version.CURRENT.toString() + ); XContentBuilder xContentBuilder = MediaTypeRegistry.contentBuilder(XContentType.JSON); getModelResponse.toXContent(xContentBuilder, null); assertEquals(expectedResponseString, xContentBuilder.toString()); @@ -90,8 +93,10 @@ public void testXContentWithNoModelBlob() throws IOException { String modelId = "test-model"; Model model = new Model(getModelMetadata(ModelState.FAILED), null, modelId); GetModelResponse getModelResponse = new GetModelResponse(model); - String expectedResponseString = - "{\"model_id\":\"test-model\",\"model_blob\":\"\",\"state\":\"failed\",\"timestamp\":\"2021-03-27 10:15:30 AM +05:30\",\"description\":\"test model\",\"error\":\"\",\"space_type\":\"l2\",\"dimension\":4,\"engine\":\"nmslib\",\"training_node_assignment\":\"\",\"model_definition\":{\"name\":\"\",\"parameters\":{}},\"data_type\":\"float\"}"; + String expectedResponseString = String.format( + "{\"model_id\":\"test-model\",\"model_blob\":\"\",\"state\":\"failed\",\"timestamp\":\"2021-03-27 10:15:30 AM +05:30\",\"description\":\"test model\",\"error\":\"\",\"space_type\":\"l2\",\"dimension\":4,\"engine\":\"nmslib\",\"training_node_assignment\":\"\",\"model_definition\":{\"name\":\"\",\"parameters\":{}},\"data_type\":\"float\",\"model_version\":\"%s\"}", + Version.CURRENT.toString() + ); XContentBuilder xContentBuilder = MediaTypeRegistry.contentBuilder(XContentType.JSON); getModelResponse.toXContent(xContentBuilder, null); assertEquals(expectedResponseString, xContentBuilder.toString()); diff --git a/src/test/java/org/opensearch/knn/plugin/transport/RemoveModelFromCacheTransportActionTests.java b/src/test/java/org/opensearch/knn/plugin/transport/RemoveModelFromCacheTransportActionTests.java index 6252a29ac..8f4cad112 100644 --- a/src/test/java/org/opensearch/knn/plugin/transport/RemoveModelFromCacheTransportActionTests.java +++ b/src/test/java/org/opensearch/knn/plugin/transport/RemoveModelFromCacheTransportActionTests.java @@ -13,6 +13,7 @@ import com.google.common.collect.ImmutableSet; import org.junit.Ignore; +import org.opensearch.Version; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; @@ -84,7 +85,8 @@ public void testNodeOperation_modelInCache() throws ExecutionException, Interrup MethodComponentContext.EMPTY, VectorDataType.DEFAULT, Mode.NOT_CONFIGURED, - CompressionLevel.NOT_CONFIGURED + CompressionLevel.NOT_CONFIGURED, + Version.CURRENT ), new byte[128], modelId diff --git a/src/test/java/org/opensearch/knn/plugin/transport/TrainingModelRequestTests.java b/src/test/java/org/opensearch/knn/plugin/transport/TrainingModelRequestTests.java index a03084c63..79292fb53 100644 --- a/src/test/java/org/opensearch/knn/plugin/transport/TrainingModelRequestTests.java +++ b/src/test/java/org/opensearch/knn/plugin/transport/TrainingModelRequestTests.java @@ -12,6 +12,7 @@ package org.opensearch.knn.plugin.transport; import com.google.common.collect.ImmutableMap; +import org.opensearch.Version; import org.opensearch.action.ActionRequestValidationException; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.metadata.IndexMetadata; @@ -222,7 +223,8 @@ public void testValidation_invalid_modelIdAlreadyExists() { MethodComponentContext.EMPTY, VectorDataType.DEFAULT, Mode.NOT_CONFIGURED, - CompressionLevel.NOT_CONFIGURED + CompressionLevel.NOT_CONFIGURED, + Version.CURRENT ); when(modelDao.getMetadata(modelId)).thenReturn(modelMetadata); diff --git a/src/test/java/org/opensearch/knn/plugin/transport/UpdateModelGraveyardTransportActionTests.java b/src/test/java/org/opensearch/knn/plugin/transport/UpdateModelGraveyardTransportActionTests.java index 45203dae6..cac5c1b9c 100644 --- a/src/test/java/org/opensearch/knn/plugin/transport/UpdateModelGraveyardTransportActionTests.java +++ b/src/test/java/org/opensearch/knn/plugin/transport/UpdateModelGraveyardTransportActionTests.java @@ -5,6 +5,7 @@ package org.opensearch.knn.plugin.transport; +import org.opensearch.Version; import org.opensearch.action.admin.indices.create.CreateIndexRequestBuilder; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.action.ActionListener; @@ -216,7 +217,8 @@ public void testClusterManagerOperation_GetIndicesUsingModel() throws IOExceptio MethodComponentContext.EMPTY, VectorDataType.DEFAULT, Mode.NOT_CONFIGURED, - CompressionLevel.NOT_CONFIGURED + CompressionLevel.NOT_CONFIGURED, + Version.CURRENT ), modelBlob, modelId diff --git a/src/test/java/org/opensearch/knn/plugin/transport/UpdateModelMetadataRequestTests.java b/src/test/java/org/opensearch/knn/plugin/transport/UpdateModelMetadataRequestTests.java index d0a83ccc5..577762a42 100644 --- a/src/test/java/org/opensearch/knn/plugin/transport/UpdateModelMetadataRequestTests.java +++ b/src/test/java/org/opensearch/knn/plugin/transport/UpdateModelMetadataRequestTests.java @@ -11,6 +11,7 @@ package org.opensearch.knn.plugin.transport; +import org.opensearch.Version; import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.knn.KNNTestCase; import org.opensearch.knn.index.engine.MethodComponentContext; @@ -48,7 +49,8 @@ public void testStreams() throws IOException { MethodComponentContext.EMPTY, VectorDataType.DEFAULT, Mode.NOT_CONFIGURED, - CompressionLevel.NOT_CONFIGURED + CompressionLevel.NOT_CONFIGURED, + Version.CURRENT ); UpdateModelMetadataRequest updateModelMetadataRequest = new UpdateModelMetadataRequest(modelId, isRemoveRequest, modelMetadata); @@ -76,7 +78,8 @@ public void testValidate() { MethodComponentContext.EMPTY, VectorDataType.DEFAULT, Mode.NOT_CONFIGURED, - CompressionLevel.NOT_CONFIGURED + CompressionLevel.NOT_CONFIGURED, + Version.CURRENT ); UpdateModelMetadataRequest updateModelMetadataRequest1 = new UpdateModelMetadataRequest("test", true, null); @@ -119,7 +122,8 @@ public void testGetModelMetadata() { MethodComponentContext.EMPTY, VectorDataType.DEFAULT, Mode.NOT_CONFIGURED, - CompressionLevel.NOT_CONFIGURED + CompressionLevel.NOT_CONFIGURED, + Version.CURRENT ); UpdateModelMetadataRequest updateModelMetadataRequest = new UpdateModelMetadataRequest("test", true, modelMetadata); diff --git a/src/test/java/org/opensearch/knn/plugin/transport/UpdateModelMetadataTransportActionTests.java b/src/test/java/org/opensearch/knn/plugin/transport/UpdateModelMetadataTransportActionTests.java index d317fa893..48b93653f 100644 --- a/src/test/java/org/opensearch/knn/plugin/transport/UpdateModelMetadataTransportActionTests.java +++ b/src/test/java/org/opensearch/knn/plugin/transport/UpdateModelMetadataTransportActionTests.java @@ -11,6 +11,7 @@ package org.opensearch.knn.plugin.transport; +import org.opensearch.Version; import org.opensearch.core.action.ActionListener; import org.opensearch.action.support.master.AcknowledgedResponse; import org.opensearch.cluster.ClusterState; @@ -74,7 +75,8 @@ public void testClusterManagerOperation() throws InterruptedException { MethodComponentContext.EMPTY, VectorDataType.DEFAULT, Mode.NOT_CONFIGURED, - CompressionLevel.NOT_CONFIGURED + CompressionLevel.NOT_CONFIGURED, + Version.CURRENT ); // Get update transport action diff --git a/src/test/java/org/opensearch/knn/training/TrainingJobRunnerTests.java b/src/test/java/org/opensearch/knn/training/TrainingJobRunnerTests.java index 4876b1562..9671b6b5a 100644 --- a/src/test/java/org/opensearch/knn/training/TrainingJobRunnerTests.java +++ b/src/test/java/org/opensearch/knn/training/TrainingJobRunnerTests.java @@ -65,7 +65,7 @@ public void testExecute_success() throws IOException, InterruptedException, Exec // After put finishes, it should call the onResponse function that will call responseListener and then kickoff // training. ModelDao modelDao = mock(ModelDao.class); - when(modelDao.get(modelId)).thenReturn(model); + when(modelDao.getMetadata(modelId)).thenReturn(modelMetadata); doAnswer(invocationOnMock -> { assertEquals(1, trainingJobRunner.getJobCount()); // Make sure job count is correct IndexResponse indexResponse = new IndexResponse(new ShardId(MODEL_INDEX_NAME, "uuid", 0), modelId, 0, 0, 0, true); diff --git a/src/test/java/org/opensearch/knn/training/TrainingJobTests.java b/src/test/java/org/opensearch/knn/training/TrainingJobTests.java index 32794a33b..14308b915 100644 --- a/src/test/java/org/opensearch/knn/training/TrainingJobTests.java +++ b/src/test/java/org/opensearch/knn/training/TrainingJobTests.java @@ -124,7 +124,8 @@ public void testGetModel() { MethodComponentContext.EMPTY, VectorDataType.DEFAULT, Mode.NOT_CONFIGURED, - CompressionLevel.NOT_CONFIGURED + CompressionLevel.NOT_CONFIGURED, + Version.CURRENT ), null, modelID From ff6097e56f40e68b5bb18eb9ba0768ad81217ef8 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Mon, 9 Sep 2024 17:16:28 -0700 Subject: [PATCH 362/416] Fixing filtered vector search when quantization is applied (#2076) (#2081) Signed-off-by: Navneet Verma (cherry picked from commit 524dbd003e908cc7a4a81d6bcc2387e2d398cd48) Co-authored-by: Navneet Verma --- .../knn/index/query/ExactSearcher.java | 86 +++++++++++++------ .../opensearch/knn/index/query/KNNWeight.java | 62 ++++++------- .../query/SegmentLevelQuantizationInfo.java | 46 ++++++++++ .../query/SegmentLevelQuantizationUtil.java | 60 +++++++++++++ .../filtered/FilteredIdsKNNIterator.java | 30 ++++++- .../NestedFilteredIdsKNNIterator.java | 17 +++- .../nativelib/NativeEngineKnnVectorQuery.java | 11 ++- .../knn/index/query/KNNWeightTests.java | 5 +- .../NativeEngineKNNVectorQueryTests.java | 6 +- 9 files changed, 249 insertions(+), 74 deletions(-) create mode 100644 src/main/java/org/opensearch/knn/index/query/SegmentLevelQuantizationInfo.java create mode 100644 src/main/java/org/opensearch/knn/index/query/SegmentLevelQuantizationUtil.java diff --git a/src/main/java/org/opensearch/knn/index/query/ExactSearcher.java b/src/main/java/org/opensearch/knn/index/query/ExactSearcher.java index 249c66d03..5b6029766 100644 --- a/src/main/java/org/opensearch/knn/index/query/ExactSearcher.java +++ b/src/main/java/org/opensearch/knn/index/query/ExactSearcher.java @@ -6,6 +6,8 @@ package org.opensearch.knn.index.query; import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Value; import lombok.extern.log4j.Log4j2; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.LeafReaderContext; @@ -42,27 +44,18 @@ public class ExactSearcher { /** * Execute an exact search on a subset of documents of a leaf * - * @param leafReaderContext LeafReaderContext to be searched over - * @param matchedDocs matched documents - * @param knnQuery KNN Query - * @param k number of results to return - * @param isParentHits whether the matchedDocs contains parent ids or child ids. This is relevant in the case of - * filtered nested search where the matchedDocs contain the parent ids and {@link NestedFilteredIdsKNNIterator} - * needs to be used. + * @param leafReaderContext {@link LeafReaderContext} + * @param exactSearcherContext {@link ExactSearcherContext} * @return Map of re-scored results + * @throws IOException exception during execution of exact search */ - public Map searchLeaf( - final LeafReaderContext leafReaderContext, - final BitSet matchedDocs, - final KNNQuery knnQuery, - int k, - boolean isParentHits - ) throws IOException { - KNNIterator iterator = getMatchedKNNIterator(leafReaderContext, matchedDocs, knnQuery, isParentHits); - if (matchedDocs.cardinality() <= k) { + public Map searchLeaf(final LeafReaderContext leafReaderContext, final ExactSearcherContext exactSearcherContext) + throws IOException { + KNNIterator iterator = getMatchedKNNIterator(leafReaderContext, exactSearcherContext); + if (exactSearcherContext.getMatchedDocs().cardinality() <= exactSearcherContext.getK()) { return scoreAllDocs(iterator); } - return searchTopK(iterator, k); + return searchTopK(iterator, exactSearcherContext.getK()); } private Map scoreAllDocs(KNNIterator iterator) throws IOException { @@ -105,17 +98,15 @@ private Map searchTopK(KNNIterator iterator, int k) throws IOExc return docToScore; } - private KNNIterator getMatchedKNNIterator( - final LeafReaderContext leafReaderContext, - final BitSet matchedDocs, - KNNQuery knnQuery, - boolean isParentHits - ) throws IOException { + private KNNIterator getMatchedKNNIterator(LeafReaderContext leafReaderContext, ExactSearcherContext exactSearcherContext) + throws IOException { + final KNNQuery knnQuery = exactSearcherContext.getKnnQuery(); + final BitSet matchedDocs = exactSearcherContext.getMatchedDocs(); final SegmentReader reader = Lucene.segmentReader(leafReaderContext.reader()); final FieldInfo fieldInfo = reader.getFieldInfos().fieldInfo(knnQuery.getField()); final SpaceType spaceType = FieldInfoExtractor.getSpaceType(modelDao, fieldInfo); - boolean isNestedRequired = isParentHits && knnQuery.getParentsFilter() != null; + boolean isNestedRequired = exactSearcherContext.isParentHits() && knnQuery.getParentsFilter() != null; if (VectorDataType.BINARY == knnQuery.getVectorDataType() && isNestedRequired) { final KNNVectorValues vectorValues = KNNVectorValuesFactory.getVectorValues(fieldInfo, reader); @@ -137,6 +128,17 @@ private KNNIterator getMatchedKNNIterator( spaceType ); } + final byte[] quantizedQueryVector; + final SegmentLevelQuantizationInfo segmentLevelQuantizationInfo; + if (exactSearcherContext.isUseQuantizedVectorsForSearch()) { + // Build Segment Level Quantization info. + segmentLevelQuantizationInfo = SegmentLevelQuantizationInfo.build(reader, fieldInfo, knnQuery.getField()); + // Quantize the Query Vector Once. + quantizedQueryVector = SegmentLevelQuantizationUtil.quantizeVector(knnQuery.getQueryVector(), segmentLevelQuantizationInfo); + } else { + segmentLevelQuantizationInfo = null; + quantizedQueryVector = null; + } final KNNVectorValues vectorValues = KNNVectorValuesFactory.getVectorValues(fieldInfo, reader); if (isNestedRequired) { @@ -145,10 +147,42 @@ private KNNIterator getMatchedKNNIterator( knnQuery.getQueryVector(), (KNNFloatVectorValues) vectorValues, spaceType, - knnQuery.getParentsFilter().getBitSet(leafReaderContext) + knnQuery.getParentsFilter().getBitSet(leafReaderContext), + quantizedQueryVector, + segmentLevelQuantizationInfo ); } - return new FilteredIdsKNNIterator(matchedDocs, knnQuery.getQueryVector(), (KNNFloatVectorValues) vectorValues, spaceType); + return new FilteredIdsKNNIterator( + matchedDocs, + knnQuery.getQueryVector(), + (KNNFloatVectorValues) vectorValues, + spaceType, + quantizedQueryVector, + segmentLevelQuantizationInfo + ); + } + + /** + * Stores the context that is used to do the exact search. This class will help in reducing the explosion of attributes + * for doing exact search. + */ + @Value + @Builder + public static class ExactSearcherContext { + /** + * controls whether we should use Quantized vectors during exact search or not. This is useful because when we do + * re-scoring we need to re-score using full precision vectors and not quantized vectors. + */ + boolean useQuantizedVectorsForSearch; + int k; + BitSet matchedDocs; + KNNQuery knnQuery; + /** + * whether the matchedDocs contains parent ids or child ids. This is relevant in the case of + * filtered nested search where the matchedDocs contain the parent ids and {@link NestedFilteredIdsKNNIterator} + * needs to be used. + */ + boolean isParentHits; } } diff --git a/src/main/java/org/opensearch/knn/index/query/KNNWeight.java b/src/main/java/org/opensearch/knn/index/query/KNNWeight.java index 1769328fe..b1ba9de59 100644 --- a/src/main/java/org/opensearch/knn/index/query/KNNWeight.java +++ b/src/main/java/org/opensearch/knn/index/query/KNNWeight.java @@ -27,7 +27,6 @@ import org.opensearch.knn.index.KNNSettings; import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.VectorDataType; -import org.opensearch.knn.index.codec.KNN990Codec.QuantizationConfigKNNCollector; import org.opensearch.knn.index.memory.NativeMemoryAllocation; import org.opensearch.knn.index.memory.NativeMemoryCacheManager; import org.opensearch.knn.index.memory.NativeMemoryEntryContext; @@ -39,8 +38,6 @@ import org.opensearch.knn.indices.ModelUtil; import org.opensearch.knn.jni.JNIService; import org.opensearch.knn.plugin.stats.KNNCounter; -import org.opensearch.knn.quantization.models.quantizationOutput.QuantizationOutput; -import org.opensearch.knn.quantization.models.quantizationParams.QuantizationParams; import java.io.IOException; import java.nio.file.Path; @@ -140,8 +137,17 @@ public Map searchLeaf(LeafReaderContext context, int k) throws I * This improves the recall. */ Map docIdsToScoreMap; + final ExactSearcher.ExactSearcherContext exactSearcherContext = ExactSearcher.ExactSearcherContext.builder() + .k(k) + .isParentHits(true) + .matchedDocs(filterBitSet) + // setting to true, so that if quantization details are present we want to do search on the quantized + // vectors as this flow is used in first pass of search. + .useQuantizedVectorsForSearch(true) + .knnQuery(knnQuery) + .build(); if (filterWeight != null && canDoExactSearch(cardinality)) { - docIdsToScoreMap = exactSearch(context, filterBitSet, true, k); + docIdsToScoreMap = exactSearch(context, exactSearcherContext); } else { docIdsToScoreMap = doANNSearch(context, filterBitSet, cardinality, k); if (docIdsToScoreMap == null) { @@ -155,7 +161,7 @@ public Map searchLeaf(LeafReaderContext context, int k) throws I docIdsToScoreMap.size(), cardinality ); - docIdsToScoreMap = exactSearch(context, filterBitSet, true, k); + docIdsToScoreMap = exactSearch(context, exactSearcherContext); } } if (docIdsToScoreMap.isEmpty()) { @@ -258,10 +264,13 @@ private Map doANNSearch( ); } - QuantizationParams quantizationParams = quantizationService.getQuantizationParams(fieldInfo); - + final SegmentLevelQuantizationInfo segmentLevelQuantizationInfo = SegmentLevelQuantizationInfo.build( + reader, + fieldInfo, + knnQuery.getField() + ); // TODO: Change type of vector once more quantization methods are supported - byte[] quantizedVector = getQuantizedVector(quantizationParams, reader, fieldInfo); + final byte[] quantizedVector = SegmentLevelQuantizationUtil.quantizeVector(knnQuery.getQueryVector(), segmentLevelQuantizationInfo); List engineFiles = getEngineFiles(reader, knnEngine.getExtension()); if (engineFiles.isEmpty()) { @@ -285,7 +294,7 @@ private Map doANNSearch( knnEngine, knnQuery.getIndexName(), // TODO: In the future, more vector data types will be supported with quantization - quantizationParams == null ? vectorDataType : VectorDataType.BINARY + quantizedVector == null ? vectorDataType : VectorDataType.BINARY ), knnQuery.getIndexName(), modelId @@ -310,11 +319,11 @@ private Map doANNSearch( int[] parentIds = getParentIdsArray(context); if (k > 0) { if (knnQuery.getVectorDataType() == VectorDataType.BINARY - || quantizationParams != null && quantizationService.getVectorDataTypeForTransfer(fieldInfo) == VectorDataType.BINARY) { + || quantizedVector != null && quantizationService.getVectorDataTypeForTransfer(fieldInfo) == VectorDataType.BINARY) { results = JNIService.queryBinaryIndex( indexAllocation.getMemoryAddress(), // TODO: In the future, quantizedVector can have other data types than byte - quantizationParams == null ? knnQuery.getByteQueryVector() : quantizedVector, + quantizedVector == null ? knnQuery.getByteQueryVector() : quantizedVector, k, knnQuery.getMethodParameters(), knnEngine, @@ -391,16 +400,14 @@ List getEngineFiles(SegmentReader reader, String extension) throws IOExc /** * Execute exact search for the given matched doc ids and return the results as a map of docId to score. * - * @param leafReaderContext The leaf reader context for the current segment. - * @param matchSet The filterIds to search for. - * @param isParentHits Whether the matchedDocs contains parent ids or child ids. - * @param k The number of results to return. * @return Map of docId to score for the exact search results. * @throws IOException If an error occurs during the search. */ - public Map exactSearch(final LeafReaderContext leafReaderContext, final BitSet matchSet, boolean isParentHits, int k) - throws IOException { - return exactSearcher.searchLeaf(leafReaderContext, matchSet, knnQuery, k, isParentHits); + public Map exactSearch( + final LeafReaderContext leafReaderContext, + final ExactSearcher.ExactSearcherContext exactSearcherContext + ) throws IOException { + return exactSearcher.searchLeaf(leafReaderContext, exactSearcherContext); } @Override @@ -462,23 +469,4 @@ private boolean isExactSearchThresholdSettingSet(int filterThresholdValue) { private boolean canDoExactSearchAfterANNSearch(final int filterIdsCount, final int annResultCount) { return filterWeight != null && filterIdsCount >= knnQuery.getK() && knnQuery.getK() > annResultCount; } - - // TODO: this will eventually return more types than just byte - private byte[] getQuantizedVector(QuantizationParams quantizationParams, SegmentReader reader, FieldInfo fieldInfo) throws IOException { - if (quantizationParams != null) { - QuantizationConfigKNNCollector tempCollector = new QuantizationConfigKNNCollector(); - reader.searchNearestVectors(knnQuery.getField(), new float[0], tempCollector, null); - if (tempCollector.getQuantizationState() == null) { - throw new IllegalStateException(String.format("No quantization state found for field %s", fieldInfo.getName())); - } - QuantizationOutput quantizationOutput = quantizationService.createQuantizationOutput(quantizationParams); - // TODO: In the future, byte array will not be the only output type from this method - return (byte[]) quantizationService.quantize( - tempCollector.getQuantizationState(), - knnQuery.getQueryVector(), - quantizationOutput - ); - } - return null; - } } diff --git a/src/main/java/org/opensearch/knn/index/query/SegmentLevelQuantizationInfo.java b/src/main/java/org/opensearch/knn/index/query/SegmentLevelQuantizationInfo.java new file mode 100644 index 000000000..d25774cdc --- /dev/null +++ b/src/main/java/org/opensearch/knn/index/query/SegmentLevelQuantizationInfo.java @@ -0,0 +1,46 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.query; + +import lombok.AccessLevel; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.LeafReader; +import org.opensearch.knn.index.quantizationservice.QuantizationService; +import org.opensearch.knn.quantization.models.quantizationParams.QuantizationParams; +import org.opensearch.knn.quantization.models.quantizationState.QuantizationState; + +import java.io.IOException; + +/** + * This class encapsulate the necessary details to do the quantization of the vectors present in a lucene segment. + */ +@Getter +@RequiredArgsConstructor(access = AccessLevel.PRIVATE) +public class SegmentLevelQuantizationInfo { + private final QuantizationParams quantizationParams; + private final QuantizationState quantizationState; + + /** + * A builder like function to build the {@link SegmentLevelQuantizationInfo} + * @param leafReader {@link LeafReader} + * @param fieldInfo {@link FieldInfo} + * @param fieldName {@link String} + * @return {@link SegmentLevelQuantizationInfo} + * @throws IOException exception while creating the {@link SegmentLevelQuantizationInfo} object. + */ + public static SegmentLevelQuantizationInfo build(final LeafReader leafReader, final FieldInfo fieldInfo, final String fieldName) + throws IOException { + final QuantizationParams quantizationParams = QuantizationService.getInstance().getQuantizationParams(fieldInfo); + if (quantizationParams == null) { + return null; + } + final QuantizationState quantizationState = SegmentLevelQuantizationUtil.getQuantizationState(leafReader, fieldName); + return new SegmentLevelQuantizationInfo(quantizationParams, quantizationState); + } + +} diff --git a/src/main/java/org/opensearch/knn/index/query/SegmentLevelQuantizationUtil.java b/src/main/java/org/opensearch/knn/index/query/SegmentLevelQuantizationUtil.java new file mode 100644 index 000000000..46db8bb6b --- /dev/null +++ b/src/main/java/org/opensearch/knn/index/query/SegmentLevelQuantizationUtil.java @@ -0,0 +1,60 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.query; + +import lombok.experimental.UtilityClass; +import org.apache.lucene.index.LeafReader; +import org.opensearch.knn.index.codec.KNN990Codec.QuantizationConfigKNNCollector; +import org.opensearch.knn.index.quantizationservice.QuantizationService; +import org.opensearch.knn.quantization.models.quantizationState.QuantizationState; + +import java.io.IOException; +import java.util.Locale; + +/** + * A utility class for doing Quantization related operation at a segment level. We can move this utility in {@link SegmentLevelQuantizationInfo} + * but I am keeping it thinking that {@link SegmentLevelQuantizationInfo} free from these utility functions to reduce + * the responsibilities of {@link SegmentLevelQuantizationInfo} class. + */ +@UtilityClass +public class SegmentLevelQuantizationUtil { + + /** + * A simple function to convert a vector to a quantized vector for a segment. + * @param vector array of float + * @return array of byte + */ + @SuppressWarnings("unchecked") + public static byte[] quantizeVector(final float[] vector, final SegmentLevelQuantizationInfo segmentLevelQuantizationInfo) { + if (segmentLevelQuantizationInfo == null) { + return null; + } + final QuantizationService quantizationService = QuantizationService.getInstance(); + // TODO: We are converting the output of Quantize to byte array for now. But this needs to be fixed when + // other types of quantized outputs are returned like float[]. + return (byte[]) quantizationService.quantize( + segmentLevelQuantizationInfo.getQuantizationState(), + vector, + quantizationService.createQuantizationOutput(segmentLevelQuantizationInfo.getQuantizationParams()) + ); + } + + /** + * A utility function to get {@link QuantizationState} for a given segment and field. + * @param leafReader {@link LeafReader} + * @param fieldName {@link String} + * @return {@link QuantizationState} + * @throws IOException exception during reading the {@link QuantizationState} + */ + static QuantizationState getQuantizationState(final LeafReader leafReader, String fieldName) throws IOException { + final QuantizationConfigKNNCollector tempCollector = new QuantizationConfigKNNCollector(); + leafReader.searchNearestVectors(fieldName, new float[0], tempCollector, null); + if (tempCollector.getQuantizationState() == null) { + throw new IllegalStateException(String.format(Locale.ROOT, "No quantization state found for field %s", fieldName)); + } + return tempCollector.getQuantizationState(); + } +} diff --git a/src/main/java/org/opensearch/knn/index/query/filtered/FilteredIdsKNNIterator.java b/src/main/java/org/opensearch/knn/index/query/filtered/FilteredIdsKNNIterator.java index a0d7694c9..56d291470 100644 --- a/src/main/java/org/opensearch/knn/index/query/filtered/FilteredIdsKNNIterator.java +++ b/src/main/java/org/opensearch/knn/index/query/filtered/FilteredIdsKNNIterator.java @@ -9,6 +9,8 @@ import org.apache.lucene.util.BitSet; import org.apache.lucene.util.BitSetIterator; import org.opensearch.knn.index.SpaceType; +import org.opensearch.knn.index.query.SegmentLevelQuantizationInfo; +import org.opensearch.knn.index.query.SegmentLevelQuantizationUtil; import org.opensearch.knn.index.vectorvalues.KNNFloatVectorValues; import java.io.IOException; @@ -24,16 +26,29 @@ public class FilteredIdsKNNIterator implements KNNIterator { protected final BitSet filterIdsBitSet; protected final BitSetIterator bitSetIterator; protected final float[] queryVector; + private final byte[] quantizedQueryVector; protected final KNNFloatVectorValues knnFloatVectorValues; protected final SpaceType spaceType; protected float currentScore = Float.NEGATIVE_INFINITY; protected int docId; + private final SegmentLevelQuantizationInfo segmentLevelQuantizationInfo; - public FilteredIdsKNNIterator( + FilteredIdsKNNIterator( final BitSet filterIdsBitSet, final float[] queryVector, final KNNFloatVectorValues knnFloatVectorValues, final SpaceType spaceType + ) { + this(filterIdsBitSet, queryVector, knnFloatVectorValues, spaceType, null, null); + } + + public FilteredIdsKNNIterator( + final BitSet filterIdsBitSet, + final float[] queryVector, + final KNNFloatVectorValues knnFloatVectorValues, + final SpaceType spaceType, + final byte[] quantizedQueryVector, + final SegmentLevelQuantizationInfo segmentLevelQuantizationInfo ) { this.filterIdsBitSet = filterIdsBitSet; this.bitSetIterator = new BitSetIterator(filterIdsBitSet, filterIdsBitSet.length()); @@ -41,6 +56,8 @@ public FilteredIdsKNNIterator( this.knnFloatVectorValues = knnFloatVectorValues; this.spaceType = spaceType; this.docId = bitSetIterator.nextDoc(); + this.quantizedQueryVector = quantizedQueryVector; + this.segmentLevelQuantizationInfo = segmentLevelQuantizationInfo; } /** @@ -68,8 +85,13 @@ public float score() { protected float computeScore() throws IOException { final float[] vector = knnFloatVectorValues.getVector(); - // Calculates a similarity score between the two vectors with a specified function. Higher similarity - // scores correspond to closer vectors. - return spaceType.getKnnVectorSimilarityFunction().compare(queryVector, vector); + if (segmentLevelQuantizationInfo != null && quantizedQueryVector != null) { + byte[] quantizedVector = SegmentLevelQuantizationUtil.quantizeVector(vector, segmentLevelQuantizationInfo); + return SpaceType.HAMMING.getKnnVectorSimilarityFunction().compare(quantizedQueryVector, quantizedVector); + } else { + // Calculates a similarity score between the two vectors with a specified function. Higher similarity + // scores correspond to closer vectors. + return spaceType.getKnnVectorSimilarityFunction().compare(queryVector, vector); + } } } diff --git a/src/main/java/org/opensearch/knn/index/query/filtered/NestedFilteredIdsKNNIterator.java b/src/main/java/org/opensearch/knn/index/query/filtered/NestedFilteredIdsKNNIterator.java index 259b004f8..53ac72882 100644 --- a/src/main/java/org/opensearch/knn/index/query/filtered/NestedFilteredIdsKNNIterator.java +++ b/src/main/java/org/opensearch/knn/index/query/filtered/NestedFilteredIdsKNNIterator.java @@ -8,6 +8,7 @@ import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.util.BitSet; import org.opensearch.knn.index.SpaceType; +import org.opensearch.knn.index.query.SegmentLevelQuantizationInfo; import org.opensearch.knn.index.vectorvalues.KNNFloatVectorValues; import java.io.IOException; @@ -19,14 +20,26 @@ public class NestedFilteredIdsKNNIterator extends FilteredIdsKNNIterator { private final BitSet parentBitSet; - public NestedFilteredIdsKNNIterator( + NestedFilteredIdsKNNIterator( final BitSet filterIdsArray, final float[] queryVector, final KNNFloatVectorValues knnFloatVectorValues, final SpaceType spaceType, final BitSet parentBitSet ) { - super(filterIdsArray, queryVector, knnFloatVectorValues, spaceType); + this(filterIdsArray, queryVector, knnFloatVectorValues, spaceType, parentBitSet, null, null); + } + + public NestedFilteredIdsKNNIterator( + final BitSet filterIdsArray, + final float[] queryVector, + final KNNFloatVectorValues knnFloatVectorValues, + final SpaceType spaceType, + final BitSet parentBitSet, + final byte[] quantizedVector, + final SegmentLevelQuantizationInfo segmentLevelQuantizationInfo + ) { + super(filterIdsArray, queryVector, knnFloatVectorValues, spaceType, quantizedVector, segmentLevelQuantizationInfo); this.parentBitSet = parentBitSet; } diff --git a/src/main/java/org/opensearch/knn/index/query/nativelib/NativeEngineKnnVectorQuery.java b/src/main/java/org/opensearch/knn/index/query/nativelib/NativeEngineKnnVectorQuery.java index c13d86554..945da850a 100644 --- a/src/main/java/org/opensearch/knn/index/query/nativelib/NativeEngineKnnVectorQuery.java +++ b/src/main/java/org/opensearch/knn/index/query/nativelib/NativeEngineKnnVectorQuery.java @@ -20,6 +20,7 @@ import org.apache.lucene.util.BitSet; import org.apache.lucene.util.Bits; import org.opensearch.common.StopWatch; +import org.opensearch.knn.index.query.ExactSearcher; import org.opensearch.knn.index.query.KNNQuery; import org.opensearch.knn.index.query.KNNWeight; import org.opensearch.knn.index.query.ResultUtil; @@ -108,7 +109,15 @@ private List> doRescore( int finalI = i; rescoreTasks.add(() -> { BitSet convertedBitSet = ResultUtil.resultMapToMatchBitSet(perLeafResults.get(finalI)); - return knnWeight.exactSearch(leafReaderContext, convertedBitSet, false, k); + final ExactSearcher.ExactSearcherContext exactSearcherContext = ExactSearcher.ExactSearcherContext.builder() + .matchedDocs(convertedBitSet) + // setting to false because in re-scoring we want to do exact search on full precision vectors + .useQuantizedVectorsForSearch(false) + .k(k) + .isParentHits(false) + .knnQuery(knnQuery) + .build(); + return knnWeight.exactSearch(leafReaderContext, exactSearcherContext); }); } return indexSearcher.getTaskExecutor().invokeAll(rescoreTasks); diff --git a/src/test/java/org/opensearch/knn/index/query/KNNWeightTests.java b/src/test/java/org/opensearch/knn/index/query/KNNWeightTests.java index a2b41804a..810f49c15 100644 --- a/src/test/java/org/opensearch/knn/index/query/KNNWeightTests.java +++ b/src/test/java/org/opensearch/knn/index/query/KNNWeightTests.java @@ -84,6 +84,7 @@ import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.ArgumentMatchers.isNull; +import static org.mockito.Mockito.doNothing; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.mockStatic; import static org.mockito.Mockito.times; @@ -1363,9 +1364,9 @@ private KNNQueryResult[] getFilteredKNNQueryResults() { public void testANNWithQuantizationParams_whenStateNotFound_thenFail() { try (MockedStatic quantizationServiceMockedStatic = Mockito.mockStatic(QuantizationService.class)) { QuantizationService quantizationService = Mockito.mock(QuantizationService.class); + quantizationServiceMockedStatic.when(QuantizationService::getInstance).thenReturn(quantizationService); QuantizationParams quantizationParams = new ScalarQuantizationParams(ScalarQuantizationType.ONE_BIT); Mockito.when(quantizationService.getQuantizationParams(any(FieldInfo.class))).thenReturn(quantizationParams); - quantizationServiceMockedStatic.when(QuantizationService::getInstance).thenReturn(quantizationService); // Given int k = 3; @@ -1413,6 +1414,8 @@ public void testANNWithQuantizationParams_whenStateNotFound_thenFail() { when(reader.getFieldInfos()).thenReturn(fieldInfos); when(fieldInfos.fieldInfo(any())).thenReturn(fieldInfo); when(fieldInfo.attributes()).thenReturn(attributesMap); + // fieldName, new float[0], tempCollector, null) + doNothing().when(reader).searchNearestVectors(any(), eq(new float[0]), any(), any()); expectThrows(IllegalStateException.class, () -> knnWeight.scorer(leafReaderContext)); } diff --git a/src/test/java/org/opensearch/knn/index/query/nativelib/NativeEngineKNNVectorQueryTests.java b/src/test/java/org/opensearch/knn/index/query/nativelib/NativeEngineKNNVectorQueryTests.java index ee53818f1..06350f39c 100644 --- a/src/test/java/org/opensearch/knn/index/query/nativelib/NativeEngineKNNVectorQueryTests.java +++ b/src/test/java/org/opensearch/knn/index/query/nativelib/NativeEngineKNNVectorQueryTests.java @@ -36,7 +36,6 @@ import java.util.concurrent.Callable; import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; @@ -186,8 +185,9 @@ public void testRescore() { when(knnWeight.getQuery()).thenReturn(knnQuery); when(knnWeight.searchLeaf(leaf1, firstPassK)).thenReturn(initialLeaf1Results); when(knnWeight.searchLeaf(leaf2, firstPassK)).thenReturn(initialLeaf2Results); - when(knnWeight.exactSearch(eq(leaf1), any(), anyBoolean(), anyInt())).thenReturn(rescoredLeaf1Results); - when(knnWeight.exactSearch(eq(leaf2), any(), anyBoolean(), anyInt())).thenReturn(rescoredLeaf2Results); + + when(knnWeight.exactSearch(eq(leaf1), any())).thenReturn(rescoredLeaf1Results); + when(knnWeight.exactSearch(eq(leaf2), any())).thenReturn(rescoredLeaf2Results); try (MockedStatic mockedResultUtil = mockStatic(ResultUtil.class)) { mockedResultUtil.when(() -> ResultUtil.reduceToTopK(any(), anyInt())).thenAnswer(InvocationOnMock::callRealMethod); mockedResultUtil.when(() -> ResultUtil.resultMapToTopDocs(eq(rescoredLeaf1Results), anyInt())).thenAnswer(t -> topDocs1); From 107ac0e57441429ea77fb5173408d8099dd0392a Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Tue, 10 Sep 2024 16:03:15 -0700 Subject: [PATCH 363/416] Fix mode/comp params so parameter overrides work (#2084) PR adds capability to override parameters when specifying mode and compression. In order to do this, I add functionality for creating a deep copy of KNNMethodContext and MethodComponentContext so that we wouldnt overwrite user provided config. Then, re-arranged some of the parameter resolution logic. Signed-off-by: John Mazanec (cherry picked from commit 270ac6a52f2ff9cc9cfa58dbd65333b21f925048) --- .../index/engine/AbstractMethodResolver.java | 185 ++++++++++ .../opensearch/knn/index/engine/Encoder.java | 12 + .../knn/index/engine/EngineResolver.java | 62 ++++ .../knn/index/engine/KNNEngine.java | 10 + .../knn/index/engine/KNNLibrary.java | 2 +- .../knn/index/engine/KNNMethodContext.java | 55 ++- .../knn/index/engine/MethodComponent.java | 5 +- .../index/engine/MethodComponentContext.java | 23 ++ .../knn/index/engine/MethodResolver.java | 36 ++ .../index/engine/ResolvedMethodContext.java | 23 ++ .../knn/index/engine/SpaceTypeResolver.java | 96 +++++ .../knn/index/engine/faiss/Faiss.java | 25 +- .../index/engine/faiss/FaissFlatEncoder.java | 11 + .../index/engine/faiss/FaissHNSWMethod.java | 26 +- .../engine/faiss/FaissHNSWPQEncoder.java | 12 + .../index/engine/faiss/FaissIVFMethod.java | 27 +- .../index/engine/faiss/FaissIVFPQEncoder.java | 12 + .../engine/faiss/FaissMethodResolver.java | 159 ++++++++ .../index/engine/faiss/FaissSQEncoder.java | 12 + .../index/engine/faiss/QFrameBitEncoder.java | 35 ++ .../knn/index/engine/lucene/Lucene.java | 17 + .../index/engine/lucene/LuceneHNSWMethod.java | 19 +- .../engine/lucene/LuceneMethodResolver.java | 106 ++++++ .../index/engine/lucene/LuceneSQEncoder.java | 12 + .../knn/index/engine/nmslib/Nmslib.java | 16 + .../index/engine/nmslib/NmslibHNSWMethod.java | 4 +- .../engine/nmslib/NmslibMethodResolver.java | 69 ++++ .../knn/index/mapper/CompressionLevel.java | 26 +- .../index/mapper/KNNVectorFieldMapper.java | 123 +++---- .../knn/index/mapper/KNNVectorFieldType.java | 5 +- .../knn/index/mapper/LuceneFieldMapper.java | 21 +- .../knn/index/mapper/MethodFieldMapper.java | 19 +- .../knn/index/mapper/ModeBasedResolver.java | 213 ----------- .../mapper/OriginalMappingParameters.java | 2 + .../opensearch/knn/index/util/IndexUtil.java | 5 +- .../plugin/rest/RestTrainModelHandler.java | 45 +-- .../transport/TrainingModelRequest.java | 19 +- .../java/org/opensearch/knn/KNNTestCase.java | 6 + .../index/engine/AbstractKNNLibraryTests.java | 10 + .../engine/AbstractMethodResolverTests.java | 158 ++++++++ .../knn/index/engine/EngineResolverTests.java | 152 ++++++++ .../knn/index/engine/NativeLibraryTests.java | 10 + .../index/engine/SpaceTypeResolverTests.java | 99 +++++ .../engine/faiss/FaissHNSWPQEncoderTests.java | 16 + .../engine/faiss/FaissIVFPQEncoderTests.java | 16 + .../faiss/FaissMethodResolverTests.java | 246 +++++++++++++ .../engine/faiss/FaissSQEncoderTests.java | 16 + .../engine/faiss/QFrameBitEncoderTests.java | 43 +++ .../lucene/LuceneMethodResolverTests.java | 212 +++++++++++ .../engine/lucene/LuceneSQEncoderTests.java | 16 + .../nmslib/NmslibMethodResolverTests.java | 106 ++++++ .../mapper/KNNVectorFieldMapperTests.java | 343 +++++++++++++++++- .../knn/integ/ModeAndCompressionIT.java | 107 ++++-- .../LibraryInitializedSupplierTests.java | 11 + ...TrainingJobRouterTransportActionTests.java | 6 +- .../transport/TrainingModelRequestTests.java | 72 +--- 56 files changed, 2715 insertions(+), 479 deletions(-) create mode 100644 src/main/java/org/opensearch/knn/index/engine/AbstractMethodResolver.java create mode 100644 src/main/java/org/opensearch/knn/index/engine/EngineResolver.java create mode 100644 src/main/java/org/opensearch/knn/index/engine/MethodResolver.java create mode 100644 src/main/java/org/opensearch/knn/index/engine/ResolvedMethodContext.java create mode 100644 src/main/java/org/opensearch/knn/index/engine/SpaceTypeResolver.java create mode 100644 src/main/java/org/opensearch/knn/index/engine/faiss/FaissMethodResolver.java create mode 100644 src/main/java/org/opensearch/knn/index/engine/lucene/LuceneMethodResolver.java create mode 100644 src/main/java/org/opensearch/knn/index/engine/nmslib/NmslibMethodResolver.java delete mode 100644 src/main/java/org/opensearch/knn/index/mapper/ModeBasedResolver.java create mode 100644 src/test/java/org/opensearch/knn/index/engine/AbstractMethodResolverTests.java create mode 100644 src/test/java/org/opensearch/knn/index/engine/EngineResolverTests.java create mode 100644 src/test/java/org/opensearch/knn/index/engine/SpaceTypeResolverTests.java create mode 100644 src/test/java/org/opensearch/knn/index/engine/faiss/FaissHNSWPQEncoderTests.java create mode 100644 src/test/java/org/opensearch/knn/index/engine/faiss/FaissIVFPQEncoderTests.java create mode 100644 src/test/java/org/opensearch/knn/index/engine/faiss/FaissMethodResolverTests.java create mode 100644 src/test/java/org/opensearch/knn/index/engine/faiss/FaissSQEncoderTests.java create mode 100644 src/test/java/org/opensearch/knn/index/engine/lucene/LuceneMethodResolverTests.java create mode 100644 src/test/java/org/opensearch/knn/index/engine/lucene/LuceneSQEncoderTests.java create mode 100644 src/test/java/org/opensearch/knn/index/engine/nmslib/NmslibMethodResolverTests.java diff --git a/src/main/java/org/opensearch/knn/index/engine/AbstractMethodResolver.java b/src/main/java/org/opensearch/knn/index/engine/AbstractMethodResolver.java new file mode 100644 index 000000000..8127a041d --- /dev/null +++ b/src/main/java/org/opensearch/knn/index/engine/AbstractMethodResolver.java @@ -0,0 +1,185 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.engine; + +import org.opensearch.common.ValidationException; +import org.opensearch.knn.index.SpaceType; +import org.opensearch.knn.index.VectorDataType; +import org.opensearch.knn.index.mapper.CompressionLevel; +import org.opensearch.knn.index.mapper.Mode; + +import java.util.HashMap; +import java.util.Locale; +import java.util.Map; +import java.util.Set; + +import static org.opensearch.knn.common.KNNConstants.METHOD_ENCODER_PARAMETER; + +/** + * Abstract {@link MethodResolver} with helpful utilitiy functions that can be shared across different + * implementations + */ +public abstract class AbstractMethodResolver implements MethodResolver { + + /** + * Utility method to get the compression level from the context + * + * @param resolvedKnnMethodContext Resolved method context. Should have an encoder set in the params if available + * @return {@link CompressionLevel} Compression level that is configured with the {@link KNNMethodContext} + */ + protected CompressionLevel resolveCompressionLevelFromMethodContext( + KNNMethodContext resolvedKnnMethodContext, + KNNMethodConfigContext knnMethodConfigContext, + Map encoderMap + ) { + // If the context is null, the compression is not configured or the encoder is not defined, return not configured + // because the method context does not contain this info + if (isEncoderSpecified(resolvedKnnMethodContext) == false) { + return CompressionLevel.x1; + } + Encoder encoder = encoderMap.get(getEncoderName(resolvedKnnMethodContext)); + if (encoder == null) { + return CompressionLevel.NOT_CONFIGURED; + } + return encoder.calculateCompressionLevel(getEncoderComponentContext(resolvedKnnMethodContext), knnMethodConfigContext); + } + + protected void resolveMethodParams( + MethodComponentContext methodComponentContext, + KNNMethodConfigContext knnMethodConfigContext, + MethodComponent methodComponent + ) { + Map resolvedParams = MethodComponent.getParameterMapWithDefaultsAdded( + methodComponentContext, + methodComponent, + knnMethodConfigContext + ); + methodComponentContext.getParameters().putAll(resolvedParams); + } + + protected KNNMethodContext initResolvedKNNMethodContext( + KNNMethodContext originalMethodContext, + KNNEngine knnEngine, + SpaceType spaceType, + String methodName + ) { + if (originalMethodContext == null) { + return new KNNMethodContext(knnEngine, spaceType, new MethodComponentContext(methodName, new HashMap<>())); + } + return new KNNMethodContext(originalMethodContext); + } + + protected String getEncoderName(KNNMethodContext knnMethodContext) { + if (isEncoderSpecified(knnMethodContext) == false) { + return null; + } + + MethodComponentContext methodComponentContext = getEncoderComponentContext(knnMethodContext); + if (methodComponentContext == null) { + return null; + } + + return methodComponentContext.getName(); + } + + protected MethodComponentContext getEncoderComponentContext(KNNMethodContext knnMethodContext) { + if (isEncoderSpecified(knnMethodContext) == false) { + return null; + } + + return (MethodComponentContext) knnMethodContext.getMethodComponentContext().getParameters().get(METHOD_ENCODER_PARAMETER); + } + + /** + * Determine if the encoder parameter is specified + * + * @param knnMethodContext {@link KNNMethodContext} + * @return true is the encoder is specified in the structure; false otherwise + */ + protected boolean isEncoderSpecified(KNNMethodContext knnMethodContext) { + return knnMethodContext != null + && knnMethodContext.getMethodComponentContext().getParameters() != null + && knnMethodContext.getMethodComponentContext().getParameters().containsKey(METHOD_ENCODER_PARAMETER); + } + + protected boolean shouldEncoderBeResolved(KNNMethodContext knnMethodContext, KNNMethodConfigContext knnMethodConfigContext) { + // The encoder should not be resolved if: + // 1. The encoder is specified + // 2. The compression is x1 + // 3. The compression is not specified and the mode is not disk-based + if (isEncoderSpecified(knnMethodContext)) { + return false; + } + + if (knnMethodConfigContext.getCompressionLevel() == CompressionLevel.x1) { + return false; + } + + if (CompressionLevel.isConfigured(knnMethodConfigContext.getCompressionLevel()) == false + && Mode.ON_DISK != knnMethodConfigContext.getMode()) { + return false; + } + + if (VectorDataType.FLOAT != knnMethodConfigContext.getVectorDataType()) { + return false; + } + + return true; + } + + protected ValidationException validateNotTrainingContext( + boolean shouldRequireTraining, + KNNEngine knnEngine, + ValidationException validationException + ) { + if (shouldRequireTraining) { + validationException = validationException == null ? new ValidationException() : validationException; + validationException.addValidationError( + String.format(Locale.ROOT, "Cannot use \"%s\" engine from training context", knnEngine.getName()) + ); + } + + return validationException; + } + + protected ValidationException validateCompressionSupported( + CompressionLevel compressionLevel, + Set supportedCompressionLevels, + KNNEngine knnEngine, + ValidationException validationException + ) { + if (CompressionLevel.isConfigured(compressionLevel) && supportedCompressionLevels.contains(compressionLevel) == false) { + validationException = validationException == null ? new ValidationException() : validationException; + validationException.addValidationError( + String.format(Locale.ROOT, "\"%s\" does not support \"%s\" compression", knnEngine.getName(), compressionLevel.getName()) + ); + } + return validationException; + } + + protected ValidationException validateCompressionNotx1WhenOnDisk( + KNNMethodConfigContext knnMethodConfigContext, + ValidationException validationException + ) { + if (knnMethodConfigContext.getCompressionLevel() == CompressionLevel.x1 && knnMethodConfigContext.getMode() == Mode.ON_DISK) { + validationException = validationException == null ? new ValidationException() : validationException; + validationException.addValidationError( + String.format(Locale.ROOT, "Cannot specify \"x1\" compression level when using \"%s\" mode", Mode.ON_DISK.getName()) + ); + } + return validationException; + } + + protected void validateCompressionConflicts(CompressionLevel originalCompressionLevel, CompressionLevel resolvedCompressionLevel) { + if (CompressionLevel.isConfigured(originalCompressionLevel) + && CompressionLevel.isConfigured(resolvedCompressionLevel) + && resolvedCompressionLevel != originalCompressionLevel) { + ValidationException validationException = new ValidationException(); + validationException.addValidationError("Cannot specify an encoder that conflicts with the provided compression level"); + throw validationException; + } + } +} diff --git a/src/main/java/org/opensearch/knn/index/engine/Encoder.java b/src/main/java/org/opensearch/knn/index/engine/Encoder.java index 7e22145eb..f15d0afcf 100644 --- a/src/main/java/org/opensearch/knn/index/engine/Encoder.java +++ b/src/main/java/org/opensearch/knn/index/engine/Encoder.java @@ -5,6 +5,8 @@ package org.opensearch.knn.index.engine; +import org.opensearch.knn.index.mapper.CompressionLevel; + /** * Interface representing an encoder. An encoder generally refers to a vector quantizer. */ @@ -24,4 +26,14 @@ default String getName() { * @return Method component associated with the encoder */ MethodComponent getMethodComponent(); + + /** + * Calculate the compression level for the give params. Assume float32 vectors are used. All parameters should + * be resolved in the encoderContext passed in. + * + * @param encoderContext Context for the encoder to extract params from + * @return Compression level this encoder produces. If the encoder does not support this calculation yet, it will + * return {@link CompressionLevel#NOT_CONFIGURED} + */ + CompressionLevel calculateCompressionLevel(MethodComponentContext encoderContext, KNNMethodConfigContext knnMethodConfigContext); } diff --git a/src/main/java/org/opensearch/knn/index/engine/EngineResolver.java b/src/main/java/org/opensearch/knn/index/engine/EngineResolver.java new file mode 100644 index 000000000..daae361e4 --- /dev/null +++ b/src/main/java/org/opensearch/knn/index/engine/EngineResolver.java @@ -0,0 +1,62 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.engine; + +import org.opensearch.knn.index.mapper.CompressionLevel; +import org.opensearch.knn.index.mapper.Mode; + +/** + * Figures out what {@link KNNEngine} to use based on configuration details + */ +public final class EngineResolver { + + public static final EngineResolver INSTANCE = new EngineResolver(); + + private EngineResolver() {} + + /** + * Based on the provided {@link Mode} and {@link CompressionLevel}, resolve to a {@link KNNEngine}. + * + * @param knnMethodConfigContext configuration context + * @param knnMethodContext KNNMethodContext + * @param requiresTraining whether config requires training + * @return {@link KNNEngine} + */ + public KNNEngine resolveEngine( + KNNMethodConfigContext knnMethodConfigContext, + KNNMethodContext knnMethodContext, + boolean requiresTraining + ) { + // User configuration gets precedence + if (knnMethodContext != null && knnMethodContext.isEngineConfigured()) { + return knnMethodContext.getKnnEngine(); + } + + // Faiss is the only engine that supports training, so we default to faiss here for now + if (requiresTraining) { + return KNNEngine.FAISS; + } + + Mode mode = knnMethodConfigContext.getMode(); + CompressionLevel compressionLevel = knnMethodConfigContext.getCompressionLevel(); + // If both mode and compression are not specified, we can just default + if (Mode.isConfigured(mode) == false && CompressionLevel.isConfigured(compressionLevel) == false) { + return KNNEngine.DEFAULT; + } + + // For 1x, we need to default to faiss if mode is provided and use nmslib otherwise + if (CompressionLevel.isConfigured(compressionLevel) == false || compressionLevel == CompressionLevel.x1) { + return mode == Mode.ON_DISK ? KNNEngine.FAISS : KNNEngine.DEFAULT; + } + + // Lucene is only engine that supports 4x - so we have to default to it here. + if (compressionLevel == CompressionLevel.x4) { + return KNNEngine.LUCENE; + } + + return KNNEngine.FAISS; + } +} diff --git a/src/main/java/org/opensearch/knn/index/engine/KNNEngine.java b/src/main/java/org/opensearch/knn/index/engine/KNNEngine.java index 2f3cb3430..80b9f32a6 100644 --- a/src/main/java/org/opensearch/knn/index/engine/KNNEngine.java +++ b/src/main/java/org/opensearch/knn/index/engine/KNNEngine.java @@ -201,4 +201,14 @@ public void setInitialized(Boolean isInitialized) { public List mmapFileExtensions() { return knnLibrary.mmapFileExtensions(); } + + @Override + public ResolvedMethodContext resolveMethod( + KNNMethodContext knnMethodContext, + KNNMethodConfigContext knnMethodConfigContext, + boolean shouldRequireTraining, + final SpaceType spaceType + ) { + return knnLibrary.resolveMethod(knnMethodContext, knnMethodConfigContext, shouldRequireTraining, spaceType); + } } diff --git a/src/main/java/org/opensearch/knn/index/engine/KNNLibrary.java b/src/main/java/org/opensearch/knn/index/engine/KNNLibrary.java index 14085243f..cf7c4ad82 100644 --- a/src/main/java/org/opensearch/knn/index/engine/KNNLibrary.java +++ b/src/main/java/org/opensearch/knn/index/engine/KNNLibrary.java @@ -14,7 +14,7 @@ /** * KNNLibrary is an interface that helps the plugin communicate with k-NN libraries */ -public interface KNNLibrary { +public interface KNNLibrary extends MethodResolver { /** * Gets the version of the library that is being used. In general, this can be used for ensuring compatibility of diff --git a/src/main/java/org/opensearch/knn/index/engine/KNNMethodContext.java b/src/main/java/org/opensearch/knn/index/engine/KNNMethodContext.java index 8b2f00f74..4a4c2704e 100644 --- a/src/main/java/org/opensearch/knn/index/engine/KNNMethodContext.java +++ b/src/main/java/org/opensearch/knn/index/engine/KNNMethodContext.java @@ -5,6 +5,7 @@ package org.opensearch.knn.index.engine; +import lombok.AccessLevel; import lombok.AllArgsConstructor; import lombok.Getter; import lombok.NonNull; @@ -34,17 +35,48 @@ * KNNMethodContext will contain the information necessary to produce a library index from an Opensearch mapping. * It will encompass all parameters necessary to build the index. */ -@AllArgsConstructor +@AllArgsConstructor(access = AccessLevel.PACKAGE) @Getter public class KNNMethodContext implements ToXContentFragment, Writeable { @NonNull - private final KNNEngine knnEngine; + private KNNEngine knnEngine; @NonNull @Setter private SpaceType spaceType; @NonNull private final MethodComponentContext methodComponentContext; + // Currently, the KNNEngine member variable cannot be null and defaults during parsing to nmslib. However, in order + // to support disk based engine resolution, this value potentially needs to be updated. Thus, this value is used + // to determine if the variable can be overridden or not based on whether the user explicitly set the value during parsing + private boolean isEngineConfigured; + + /** + * Copy constructor. Useful for creating a deep copy of a {@link KNNMethodContext}. Note that the engine and + * space type should be set. + * + * @param knnMethodContext original {@link KNNMethodContext}. Must NOT be null + */ + public KNNMethodContext(KNNMethodContext knnMethodContext) { + if (knnMethodContext == null) { + throw new IllegalArgumentException("KNNMethodContext cannot be null"); + } + + this.knnEngine = knnMethodContext.knnEngine; + this.spaceType = knnMethodContext.spaceType; + this.isEngineConfigured = true; + this.methodComponentContext = new MethodComponentContext(knnMethodContext.methodComponentContext); + } + + /** + * + * @param knnEngine {@link KNNEngine} + * @param spaceType {@link SpaceType} + * @param methodComponentContext {@link MethodComponentContext} + */ + public KNNMethodContext(KNNEngine knnEngine, SpaceType spaceType, MethodComponentContext methodComponentContext) { + this(knnEngine, spaceType, methodComponentContext, true); + } /** * Constructor from stream. @@ -56,6 +88,21 @@ public KNNMethodContext(StreamInput in) throws IOException { this.knnEngine = KNNEngine.getEngine(in.readString()); this.spaceType = SpaceType.getSpace(in.readString()); this.methodComponentContext = new MethodComponentContext(in); + this.isEngineConfigured = true; + } + + /** + * Set the {@link KNNEngine} if it is not configured (i.e. DEFAULT). This is useful for using different engines + * for different configurations - i.e. dynamic defaults + * + * @param knnEngine KNNEngine to set + */ + public void setKnnEngine(KNNEngine knnEngine) { + if (isEngineConfigured) { + throw new IllegalArgumentException("Cannot configure KNNEngine if it has already been configured"); + } + this.knnEngine = knnEngine; + this.isEngineConfigured = true; } /** @@ -101,6 +148,7 @@ public static KNNMethodContext parse(Object in) { @SuppressWarnings("unchecked") Map methodMap = (Map) in; + boolean isEngineConfigured = false; KNNEngine engine = KNNEngine.DEFAULT; // Get or default SpaceType spaceType = SpaceType.UNDEFINED; // Get or default String name = ""; @@ -123,6 +171,7 @@ public static KNNMethodContext parse(Object in) { throw new MapperParsingException("Invalid " + KNN_ENGINE + ": " + value); } } + isEngineConfigured = true; } else if (METHOD_PARAMETER_SPACE_TYPE.equals(key)) { if (value != null && !(value instanceof String)) { throw new MapperParsingException("\"" + METHOD_PARAMETER_SPACE_TYPE + "\" must be a string"); @@ -173,7 +222,7 @@ public static KNNMethodContext parse(Object in) { MethodComponentContext method = new MethodComponentContext(name, parameters); - return new KNNMethodContext(engine, spaceType, method); + return new KNNMethodContext(engine, spaceType, method, isEngineConfigured); } @Override diff --git a/src/main/java/org/opensearch/knn/index/engine/MethodComponent.java b/src/main/java/org/opensearch/knn/index/engine/MethodComponent.java index 2579063e9..75e18a243 100644 --- a/src/main/java/org/opensearch/knn/index/engine/MethodComponent.java +++ b/src/main/java/org/opensearch/knn/index/engine/MethodComponent.java @@ -342,7 +342,10 @@ public static Map getParameterMapWithDefaultsAdded( IndexHyperParametersUtil.getHNSWEFConstructionValue(indexCreationVersion) ); } else { - parametersWithDefaultsMap.put(parameter.getName(), parameter.getDefaultValue()); + Object value = parameter.getDefaultValue(); + if (value != null) { + parametersWithDefaultsMap.put(parameter.getName(), value); + } } } diff --git a/src/main/java/org/opensearch/knn/index/engine/MethodComponentContext.java b/src/main/java/org/opensearch/knn/index/engine/MethodComponentContext.java index 586cc338f..1f0b345e9 100644 --- a/src/main/java/org/opensearch/knn/index/engine/MethodComponentContext.java +++ b/src/main/java/org/opensearch/knn/index/engine/MethodComponentContext.java @@ -49,6 +49,29 @@ public class MethodComponentContext implements ToXContentFragment, Writeable { private final String name; private final Map parameters; + /** + * Copy constructor. Creates a deep copy of a {@link MethodComponentContext} + * + * @param methodComponentContext to be copied. Must NOT be null + */ + public MethodComponentContext(MethodComponentContext methodComponentContext) { + if (methodComponentContext == null) { + throw new IllegalArgumentException("MethodComponentContext cannot be null"); + } + + this.name = methodComponentContext.name; + this.parameters = new HashMap<>(); + if (methodComponentContext.parameters != null) { + for (Map.Entry entry : methodComponentContext.parameters.entrySet()) { + if (entry.getValue() instanceof MethodComponentContext) { + parameters.put(entry.getKey(), new MethodComponentContext((MethodComponentContext) entry.getValue())); + } else { + parameters.put(entry.getKey(), entry.getValue()); + } + } + } + } + /** * Constructor from stream. * diff --git a/src/main/java/org/opensearch/knn/index/engine/MethodResolver.java b/src/main/java/org/opensearch/knn/index/engine/MethodResolver.java new file mode 100644 index 000000000..4df18ad72 --- /dev/null +++ b/src/main/java/org/opensearch/knn/index/engine/MethodResolver.java @@ -0,0 +1,36 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.engine; + +import org.opensearch.knn.index.SpaceType; + +/** + * Interface for resolving the {@link ResolvedMethodContext} for an engine and configuration + */ +public interface MethodResolver { + + /** + * Creates a new {@link ResolvedMethodContext} filling parameters based on other configuration details. A validation + * exception will be thrown if the {@link KNNMethodConfigContext} is not compatible with the + * parameters provided by the user. + * + * @param knnMethodContext User provided information regarding the method context. A new context should be + * constructed. This variable will not be modified. + * @param knnMethodConfigContext Configuration details that can be used for resolving the defaults. Should not be null + * @param shouldRequireTraining Should the provided context require training + * @param spaceType Space type for the method. Cannot be null or undefined + * @return {@link ResolvedMethodContext} with dynamic defaults configured. This will include both the resolved + * compression as well as the completely resolve {@link KNNMethodContext}. + * This is guanteed to be a copy of the user provided context. + * @throws org.opensearch.common.ValidationException on invalid configuration and userprovided context. + */ + ResolvedMethodContext resolveMethod( + KNNMethodContext knnMethodContext, + KNNMethodConfigContext knnMethodConfigContext, + boolean shouldRequireTraining, + final SpaceType spaceType + ); +} diff --git a/src/main/java/org/opensearch/knn/index/engine/ResolvedMethodContext.java b/src/main/java/org/opensearch/knn/index/engine/ResolvedMethodContext.java new file mode 100644 index 000000000..1edc0a98e --- /dev/null +++ b/src/main/java/org/opensearch/knn/index/engine/ResolvedMethodContext.java @@ -0,0 +1,23 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.engine; + +import lombok.Builder; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import org.opensearch.knn.index.mapper.CompressionLevel; + +/** + * Small data class for storing info that gets resolved during resolution process + */ +@RequiredArgsConstructor +@Getter +@Builder +public class ResolvedMethodContext { + private final KNNMethodContext knnMethodContext; + @Builder.Default + private final CompressionLevel compressionLevel = CompressionLevel.NOT_CONFIGURED; +} diff --git a/src/main/java/org/opensearch/knn/index/engine/SpaceTypeResolver.java b/src/main/java/org/opensearch/knn/index/engine/SpaceTypeResolver.java new file mode 100644 index 000000000..a12ffbc7b --- /dev/null +++ b/src/main/java/org/opensearch/knn/index/engine/SpaceTypeResolver.java @@ -0,0 +1,96 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.engine; + +import org.apache.logging.log4j.util.Strings; +import org.opensearch.index.mapper.MapperParsingException; +import org.opensearch.knn.index.SpaceType; +import org.opensearch.knn.index.VectorDataType; + +import java.util.Locale; + +/** + * Class contains the logic to figure out what {@link SpaceType} to use based on configuration + * details. A user can either provide the {@link SpaceType} via the {@link KNNMethodContext} or through a + * top level parameter. This class will take care of this resolution logic (as well as if neither are configured) and + * ensure there are not any contradictions. + */ +public final class SpaceTypeResolver { + + public static final SpaceTypeResolver INSTANCE = new SpaceTypeResolver(); + + private SpaceTypeResolver() {} + + /** + * Resolves space type from configuration details. It is guaranteed not to return either null or + * {@link SpaceType#UNDEFINED} + * + * @param knnMethodContext Method context + * @param vectorDataType Vectordatatype + * @param topLevelSpaceTypeString Alternative top-level space type + * @return {@link SpaceType} for the method + */ + public SpaceType resolveSpaceType( + final KNNMethodContext knnMethodContext, + final VectorDataType vectorDataType, + final String topLevelSpaceTypeString + ) { + SpaceType methodSpaceType = getSpaceTypeFromMethodContext(knnMethodContext); + SpaceType topLevelSpaceType = getSpaceTypeFromString(topLevelSpaceTypeString); + + if (isSpaceTypeConfigured(methodSpaceType) == false && isSpaceTypeConfigured(topLevelSpaceType) == false) { + return getSpaceTypeFromVectorDataType(vectorDataType); + } + + if (isSpaceTypeConfigured(methodSpaceType) == false) { + return topLevelSpaceType; + } + + if (isSpaceTypeConfigured(topLevelSpaceType) == false) { + return methodSpaceType; + } + + if (methodSpaceType == topLevelSpaceType) { + return topLevelSpaceType; + } + + throw new MapperParsingException( + String.format( + Locale.ROOT, + "Cannot specify conflicting space types: \"[%s]\" \"[%s]\"", + methodSpaceType.getValue(), + topLevelSpaceType.getValue() + ) + ); + } + + private SpaceType getSpaceTypeFromMethodContext(final KNNMethodContext knnMethodContext) { + if (knnMethodContext == null) { + return SpaceType.UNDEFINED; + } + + return knnMethodContext.getSpaceType(); + } + + private SpaceType getSpaceTypeFromVectorDataType(final VectorDataType vectorDataType) { + if (vectorDataType == VectorDataType.BINARY) { + return SpaceType.DEFAULT_BINARY; + } + return SpaceType.DEFAULT; + } + + private SpaceType getSpaceTypeFromString(final String spaceType) { + if (Strings.isEmpty(spaceType)) { + return SpaceType.UNDEFINED; + } + + return SpaceType.getSpace(spaceType); + } + + private boolean isSpaceTypeConfigured(final SpaceType spaceType) { + return spaceType != null && spaceType != SpaceType.UNDEFINED; + } +} diff --git a/src/main/java/org/opensearch/knn/index/engine/faiss/Faiss.java b/src/main/java/org/opensearch/knn/index/engine/faiss/Faiss.java index 329acbdb8..a602619a1 100644 --- a/src/main/java/org/opensearch/knn/index/engine/faiss/Faiss.java +++ b/src/main/java/org/opensearch/knn/index/engine/faiss/Faiss.java @@ -9,7 +9,11 @@ import org.opensearch.knn.common.KNNConstants; import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.engine.KNNMethod; +import org.opensearch.knn.index.engine.KNNMethodConfigContext; +import org.opensearch.knn.index.engine.KNNMethodContext; +import org.opensearch.knn.index.engine.MethodResolver; import org.opensearch.knn.index.engine.NativeLibrary; +import org.opensearch.knn.index.engine.ResolvedMethodContext; import java.util.Map; import java.util.function.Function; @@ -41,12 +45,8 @@ public class Faiss extends NativeLibrary { SpaceType, Function>builder().put(SpaceType.INNER_PRODUCT, score -> score > 1 ? 1 - score : 1 / score - 1).build(); - private final static Map METHODS = ImmutableMap.of( - METHOD_HNSW, - new FaissHNSWMethod(), - METHOD_IVF, - new FaissIVFMethod() - ); + // Package private so that the method resolving logic can access the methods + final static Map METHODS = ImmutableMap.of(METHOD_HNSW, new FaissHNSWMethod(), METHOD_IVF, new FaissIVFMethod()); public final static Faiss INSTANCE = new Faiss( METHODS, @@ -56,6 +56,8 @@ public class Faiss extends NativeLibrary { SCORE_TO_DISTANCE_TRANSFORMATIONS ); + private final MethodResolver methodResolver; + /** * Constructor for Faiss * @@ -73,6 +75,7 @@ private Faiss( ) { super(methods, scoreTranslation, currentVersion, extension); this.scoreTransform = scoreTransform; + this.methodResolver = new FaissMethodResolver(); } @Override @@ -89,4 +92,14 @@ public Float scoreToRadialThreshold(Float score, SpaceType spaceType) { } return spaceType.scoreToDistanceTranslation(score); } + + @Override + public ResolvedMethodContext resolveMethod( + KNNMethodContext knnMethodContext, + KNNMethodConfigContext knnMethodConfigContext, + boolean shouldRequireTraining, + final SpaceType spaceType + ) { + return methodResolver.resolveMethod(knnMethodContext, knnMethodConfigContext, shouldRequireTraining, spaceType); + } } diff --git a/src/main/java/org/opensearch/knn/index/engine/faiss/FaissFlatEncoder.java b/src/main/java/org/opensearch/knn/index/engine/faiss/FaissFlatEncoder.java index bd7598d84..f7d4342fc 100644 --- a/src/main/java/org/opensearch/knn/index/engine/faiss/FaissFlatEncoder.java +++ b/src/main/java/org/opensearch/knn/index/engine/faiss/FaissFlatEncoder.java @@ -9,7 +9,10 @@ import org.opensearch.knn.common.KNNConstants; import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.engine.Encoder; +import org.opensearch.knn.index.engine.KNNMethodConfigContext; import org.opensearch.knn.index.engine.MethodComponent; +import org.opensearch.knn.index.engine.MethodComponentContext; +import org.opensearch.knn.index.mapper.CompressionLevel; import java.util.Set; @@ -41,4 +44,12 @@ public class FaissFlatEncoder implements Encoder { public MethodComponent getMethodComponent() { return METHOD_COMPONENT; } + + @Override + public CompressionLevel calculateCompressionLevel( + MethodComponentContext encoderContext, + KNNMethodConfigContext knnMethodConfigContext + ) { + return CompressionLevel.x1; + } } diff --git a/src/main/java/org/opensearch/knn/index/engine/faiss/FaissHNSWMethod.java b/src/main/java/org/opensearch/knn/index/engine/faiss/FaissHNSWMethod.java index 41db777e3..c153a9328 100644 --- a/src/main/java/org/opensearch/knn/index/engine/faiss/FaissHNSWMethod.java +++ b/src/main/java/org/opensearch/knn/index/engine/faiss/FaissHNSWMethod.java @@ -20,6 +20,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.stream.Collectors; @@ -52,12 +53,23 @@ public class FaissHNSWMethod extends AbstractFaissMethod { KNNConstants.ENCODER_FLAT, Collections.emptyMap() ); - private final static List SUPPORTED_ENCODERS = List.of( - new FaissFlatEncoder(), - new FaissSQEncoder(), - new FaissHNSWPQEncoder(), - new QFrameBitEncoder() + + // Package private so that the method resolving logic can access the methods + final static Encoder FLAT_ENCODER = new FaissFlatEncoder(); + final static Encoder SQ_ENCODER = new FaissSQEncoder(); + final static Encoder HNSW_PQ_ENCODER = new FaissHNSWPQEncoder(); + final static Encoder QFRAME_BIT_ENCODER = new QFrameBitEncoder(); + final static Map SUPPORTED_ENCODERS = Map.of( + FLAT_ENCODER.getName(), + FLAT_ENCODER, + SQ_ENCODER.getName(), + SQ_ENCODER, + HNSW_PQ_ENCODER.getName(), + HNSW_PQ_ENCODER, + QFRAME_BIT_ENCODER.getName(), + QFRAME_BIT_ENCODER ); + final static MethodComponent HNSW_COMPONENT = initMethodComponent(); /** * Constructor for FaissHNSWMethod @@ -65,7 +77,7 @@ public class FaissHNSWMethod extends AbstractFaissMethod { * @see AbstractKNNMethod */ public FaissHNSWMethod() { - super(initMethodComponent(), Set.copyOf(SUPPORTED_SPACES), new DefaultHnswSearchContext()); + super(HNSW_COMPONENT, Set.copyOf(SUPPORTED_SPACES), new DefaultHnswSearchContext()); } private static MethodComponent initMethodComponent() { @@ -108,7 +120,7 @@ private static Parameter.MethodComponentContextParameter initEncoderParameter() return new Parameter.MethodComponentContextParameter( METHOD_ENCODER_PARAMETER, DEFAULT_ENCODER_CONTEXT, - SUPPORTED_ENCODERS.stream().collect(Collectors.toMap(Encoder::getName, Encoder::getMethodComponent)) + SUPPORTED_ENCODERS.values().stream().collect(Collectors.toMap(Encoder::getName, Encoder::getMethodComponent)) ); } } diff --git a/src/main/java/org/opensearch/knn/index/engine/faiss/FaissHNSWPQEncoder.java b/src/main/java/org/opensearch/knn/index/engine/faiss/FaissHNSWPQEncoder.java index 9bebf5b4d..6750d84ed 100644 --- a/src/main/java/org/opensearch/knn/index/engine/faiss/FaissHNSWPQEncoder.java +++ b/src/main/java/org/opensearch/knn/index/engine/faiss/FaissHNSWPQEncoder.java @@ -9,8 +9,11 @@ import org.opensearch.knn.common.KNNConstants; import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.engine.Encoder; +import org.opensearch.knn.index.engine.KNNMethodConfigContext; import org.opensearch.knn.index.engine.MethodComponent; +import org.opensearch.knn.index.engine.MethodComponentContext; import org.opensearch.knn.index.engine.Parameter; +import org.opensearch.knn.index.mapper.CompressionLevel; import java.util.Objects; import java.util.Set; @@ -69,4 +72,13 @@ public class FaissHNSWPQEncoder implements Encoder { public MethodComponent getMethodComponent() { return METHOD_COMPONENT; } + + @Override + public CompressionLevel calculateCompressionLevel( + MethodComponentContext methodComponentContext, + KNNMethodConfigContext knnMethodConfigContext + ) { + // TODO: For now, not supported out of the box + return CompressionLevel.NOT_CONFIGURED; + } } diff --git a/src/main/java/org/opensearch/knn/index/engine/faiss/FaissIVFMethod.java b/src/main/java/org/opensearch/knn/index/engine/faiss/FaissIVFMethod.java index 70ab4222b..340c1f4d8 100644 --- a/src/main/java/org/opensearch/knn/index/engine/faiss/FaissIVFMethod.java +++ b/src/main/java/org/opensearch/knn/index/engine/faiss/FaissIVFMethod.java @@ -19,6 +19,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.stream.Collectors; @@ -55,20 +56,32 @@ public class FaissIVFMethod extends AbstractFaissMethod { KNNConstants.ENCODER_FLAT, Collections.emptyMap() ); - private final static List SUPPORTED_ENCODERS = List.of( - new FaissFlatEncoder(), - new FaissSQEncoder(), - new FaissIVFPQEncoder(), - new QFrameBitEncoder() + + // Package private so that the method resolving logic can access the methods + final static Encoder FLAT_ENCODER = new FaissFlatEncoder(); + final static Encoder SQ_ENCODER = new FaissSQEncoder(); + final static Encoder IVF_PQ_ENCODER = new FaissIVFPQEncoder(); + final static Encoder QFRAME_BIT_ENCODER = new QFrameBitEncoder(); + final static Map SUPPORTED_ENCODERS = Map.of( + FLAT_ENCODER.getName(), + FLAT_ENCODER, + SQ_ENCODER.getName(), + SQ_ENCODER, + IVF_PQ_ENCODER.getName(), + IVF_PQ_ENCODER, + QFRAME_BIT_ENCODER.getName(), + QFRAME_BIT_ENCODER ); + final static MethodComponent IVF_COMPONENT = initMethodComponent(); + /** * Constructor for FaissIVFMethod * * @see AbstractKNNMethod */ public FaissIVFMethod() { - super(initMethodComponent(), Set.copyOf(SUPPORTED_SPACES), new DefaultIVFSearchContext()); + super(IVF_COMPONENT, Set.copyOf(SUPPORTED_SPACES), new DefaultIVFSearchContext()); } private static MethodComponent initMethodComponent() { @@ -133,7 +146,7 @@ private static Parameter.MethodComponentContextParameter initEncoderParameter() return new Parameter.MethodComponentContextParameter( METHOD_ENCODER_PARAMETER, DEFAULT_ENCODER_CONTEXT, - SUPPORTED_ENCODERS.stream().collect(Collectors.toMap(Encoder::getName, Encoder::getMethodComponent)) + SUPPORTED_ENCODERS.values().stream().collect(Collectors.toMap(Encoder::getName, Encoder::getMethodComponent)) ); } } diff --git a/src/main/java/org/opensearch/knn/index/engine/faiss/FaissIVFPQEncoder.java b/src/main/java/org/opensearch/knn/index/engine/faiss/FaissIVFPQEncoder.java index bb6623600..8d54548bd 100644 --- a/src/main/java/org/opensearch/knn/index/engine/faiss/FaissIVFPQEncoder.java +++ b/src/main/java/org/opensearch/knn/index/engine/faiss/FaissIVFPQEncoder.java @@ -9,8 +9,11 @@ import org.opensearch.knn.common.KNNConstants; import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.engine.Encoder; +import org.opensearch.knn.index.engine.KNNMethodConfigContext; import org.opensearch.knn.index.engine.MethodComponent; +import org.opensearch.knn.index.engine.MethodComponentContext; import org.opensearch.knn.index.engine.Parameter; +import org.opensearch.knn.index.mapper.CompressionLevel; import java.util.Set; @@ -90,4 +93,13 @@ public class FaissIVFPQEncoder implements Encoder { public MethodComponent getMethodComponent() { return METHOD_COMPONENT; } + + @Override + public CompressionLevel calculateCompressionLevel( + MethodComponentContext methodComponentContext, + KNNMethodConfigContext knnMethodConfigContext + ) { + // TODO: For now, not supported out of the box + return CompressionLevel.NOT_CONFIGURED; + } } diff --git a/src/main/java/org/opensearch/knn/index/engine/faiss/FaissMethodResolver.java b/src/main/java/org/opensearch/knn/index/engine/faiss/FaissMethodResolver.java new file mode 100644 index 000000000..90e938eb3 --- /dev/null +++ b/src/main/java/org/opensearch/knn/index/engine/faiss/FaissMethodResolver.java @@ -0,0 +1,159 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.engine.faiss; + +import org.opensearch.common.ValidationException; +import org.opensearch.knn.index.SpaceType; +import org.opensearch.knn.index.engine.AbstractMethodResolver; +import org.opensearch.knn.index.engine.Encoder; +import org.opensearch.knn.index.engine.KNNEngine; +import org.opensearch.knn.index.engine.KNNMethodConfigContext; +import org.opensearch.knn.index.engine.KNNMethodContext; +import org.opensearch.knn.index.engine.MethodComponent; +import org.opensearch.knn.index.engine.MethodComponentContext; +import org.opensearch.knn.index.engine.ResolvedMethodContext; +import org.opensearch.knn.index.mapper.CompressionLevel; +import org.opensearch.knn.index.mapper.Mode; + +import java.util.HashMap; +import java.util.Map; +import java.util.Set; + +import static org.opensearch.knn.common.KNNConstants.ENCODER_FLAT; +import static org.opensearch.knn.common.KNNConstants.ENCODER_SQ; +import static org.opensearch.knn.common.KNNConstants.FAISS_SQ_ENCODER_FP16; +import static org.opensearch.knn.common.KNNConstants.FAISS_SQ_TYPE; +import static org.opensearch.knn.common.KNNConstants.METHOD_ENCODER_PARAMETER; +import static org.opensearch.knn.common.KNNConstants.METHOD_HNSW; +import static org.opensearch.knn.common.KNNConstants.METHOD_IVF; +import static org.opensearch.knn.index.engine.faiss.FaissHNSWMethod.HNSW_COMPONENT; +import static org.opensearch.knn.index.engine.faiss.FaissIVFMethod.IVF_COMPONENT; + +public class FaissMethodResolver extends AbstractMethodResolver { + + private static final Set SUPPORTED_COMPRESSION_LEVELS = Set.of( + CompressionLevel.x1, + CompressionLevel.x2, + CompressionLevel.x8, + CompressionLevel.x16, + CompressionLevel.x32 + ); + + @Override + public ResolvedMethodContext resolveMethod( + KNNMethodContext knnMethodContext, + KNNMethodConfigContext knnMethodConfigContext, + boolean shouldRequireTraining, + final SpaceType spaceType + ) { + // Initial validation to ensure that there are no contradictions in provided parameters + validateConfig(knnMethodConfigContext); + + KNNMethodContext resolvedKNNMethodContext = initResolvedKNNMethodContext( + knnMethodContext, + KNNEngine.FAISS, + spaceType, + shouldRequireTraining ? METHOD_IVF : METHOD_HNSW + ); + MethodComponent method = METHOD_HNSW.equals(resolvedKNNMethodContext.getMethodComponentContext().getName()) == false + ? IVF_COMPONENT + : HNSW_COMPONENT; + Map encoderMap = method == HNSW_COMPONENT ? FaissHNSWMethod.SUPPORTED_ENCODERS : FaissIVFMethod.SUPPORTED_ENCODERS; + + // Fill in parameters for the encoder and then the method. + resolveEncoder(resolvedKNNMethodContext, knnMethodConfigContext, encoderMap); + resolveMethodParams(resolvedKNNMethodContext.getMethodComponentContext(), knnMethodConfigContext, method); + + // From the resolved method context, get the compression level and validate it against the passed in + // configuration + CompressionLevel resolvedCompressionLevel = resolveCompressionLevelFromMethodContext( + resolvedKNNMethodContext, + knnMethodConfigContext, + encoderMap + ); + + // Validate that resolved compression doesnt have any conflicts + validateCompressionConflicts(knnMethodConfigContext.getCompressionLevel(), resolvedCompressionLevel); + return ResolvedMethodContext.builder() + .knnMethodContext(resolvedKNNMethodContext) + .compressionLevel(resolvedCompressionLevel) + .build(); + } + + private void resolveEncoder( + KNNMethodContext resolvedKNNMethodContext, + KNNMethodConfigContext knnMethodConfigContext, + Map encoderMap + ) { + if (shouldEncoderBeResolved(resolvedKNNMethodContext, knnMethodConfigContext) == false) { + return; + } + + CompressionLevel resolvedCompressionLevel = getDefaultCompressionLevel(knnMethodConfigContext); + if (resolvedCompressionLevel == CompressionLevel.x1) { + return; + } + + MethodComponentContext encoderComponentContext = new MethodComponentContext(ENCODER_FLAT, new HashMap<>()); + Encoder encoder = encoderMap.get(ENCODER_FLAT); + if (CompressionLevel.x2 == resolvedCompressionLevel) { + encoderComponentContext = new MethodComponentContext(ENCODER_SQ, new HashMap<>()); + encoder = encoderMap.get(ENCODER_SQ); + encoderComponentContext.getParameters().put(FAISS_SQ_TYPE, FAISS_SQ_ENCODER_FP16); + } + + if (CompressionLevel.x8 == resolvedCompressionLevel) { + encoderComponentContext = new MethodComponentContext(QFrameBitEncoder.NAME, new HashMap<>()); + encoder = encoderMap.get(QFrameBitEncoder.NAME); + encoderComponentContext.getParameters().put(QFrameBitEncoder.BITCOUNT_PARAM, CompressionLevel.x8.numBitsForFloat32()); + } + + if (CompressionLevel.x16 == resolvedCompressionLevel) { + encoderComponentContext = new MethodComponentContext(QFrameBitEncoder.NAME, new HashMap<>()); + encoder = encoderMap.get(QFrameBitEncoder.NAME); + encoderComponentContext.getParameters().put(QFrameBitEncoder.BITCOUNT_PARAM, CompressionLevel.x16.numBitsForFloat32()); + } + + if (CompressionLevel.x32 == resolvedCompressionLevel) { + encoderComponentContext = new MethodComponentContext(QFrameBitEncoder.NAME, new HashMap<>()); + encoder = encoderMap.get(QFrameBitEncoder.NAME); + encoderComponentContext.getParameters().put(QFrameBitEncoder.BITCOUNT_PARAM, CompressionLevel.x32.numBitsForFloat32()); + } + + Map resolvedParams = MethodComponent.getParameterMapWithDefaultsAdded( + encoderComponentContext, + encoder.getMethodComponent(), + knnMethodConfigContext + ); + encoderComponentContext.getParameters().putAll(resolvedParams); + resolvedKNNMethodContext.getMethodComponentContext().getParameters().put(METHOD_ENCODER_PARAMETER, encoderComponentContext); + } + + // Method validates for explicit contradictions in the config + private void validateConfig(KNNMethodConfigContext knnMethodConfigContext) { + CompressionLevel compressionLevel = knnMethodConfigContext.getCompressionLevel(); + ValidationException validationException = validateCompressionSupported( + compressionLevel, + SUPPORTED_COMPRESSION_LEVELS, + KNNEngine.FAISS, + null + ); + validationException = validateCompressionNotx1WhenOnDisk(knnMethodConfigContext, validationException); + if (validationException != null) { + throw validationException; + } + } + + private CompressionLevel getDefaultCompressionLevel(KNNMethodConfigContext knnMethodConfigContext) { + if (CompressionLevel.isConfigured(knnMethodConfigContext.getCompressionLevel())) { + return knnMethodConfigContext.getCompressionLevel(); + } + if (knnMethodConfigContext.getMode() == Mode.ON_DISK) { + return CompressionLevel.x32; + } + return CompressionLevel.x1; + } +} diff --git a/src/main/java/org/opensearch/knn/index/engine/faiss/FaissSQEncoder.java b/src/main/java/org/opensearch/knn/index/engine/faiss/FaissSQEncoder.java index 6d57aef2f..cd7e1e5f3 100644 --- a/src/main/java/org/opensearch/knn/index/engine/faiss/FaissSQEncoder.java +++ b/src/main/java/org/opensearch/knn/index/engine/faiss/FaissSQEncoder.java @@ -8,8 +8,11 @@ import com.google.common.collect.ImmutableSet; import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.engine.Encoder; +import org.opensearch.knn.index.engine.KNNMethodConfigContext; import org.opensearch.knn.index.engine.MethodComponent; +import org.opensearch.knn.index.engine.MethodComponentContext; import org.opensearch.knn.index.engine.Parameter; +import org.opensearch.knn.index.mapper.CompressionLevel; import java.util.Objects; import java.util.Set; @@ -49,4 +52,13 @@ public class FaissSQEncoder implements Encoder { public MethodComponent getMethodComponent() { return METHOD_COMPONENT; } + + @Override + public CompressionLevel calculateCompressionLevel( + MethodComponentContext methodComponentContext, + KNNMethodConfigContext knnMethodConfigContext + ) { + // TODO: Hard code for now + return CompressionLevel.x2; + } } diff --git a/src/main/java/org/opensearch/knn/index/engine/faiss/QFrameBitEncoder.java b/src/main/java/org/opensearch/knn/index/engine/faiss/QFrameBitEncoder.java index e135fa33f..2292dc3cc 100644 --- a/src/main/java/org/opensearch/knn/index/engine/faiss/QFrameBitEncoder.java +++ b/src/main/java/org/opensearch/knn/index/engine/faiss/QFrameBitEncoder.java @@ -6,12 +6,16 @@ package org.opensearch.knn.index.engine.faiss; import com.google.common.collect.ImmutableSet; +import org.opensearch.common.ValidationException; import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.engine.Encoder; import org.opensearch.knn.index.engine.KNNLibraryIndexingContextImpl; +import org.opensearch.knn.index.engine.KNNMethodConfigContext; import org.opensearch.knn.index.engine.MethodComponent; +import org.opensearch.knn.index.engine.MethodComponentContext; import org.opensearch.knn.index.engine.Parameter; import org.opensearch.knn.index.engine.qframe.QuantizationConfig; +import org.opensearch.knn.index.mapper.CompressionLevel; import org.opensearch.knn.quantization.enums.ScalarQuantizationType; import java.util.HashMap; @@ -75,4 +79,35 @@ public class QFrameBitEncoder implements Encoder { public MethodComponent getMethodComponent() { return METHOD_COMPONENT; } + + @Override + public CompressionLevel calculateCompressionLevel( + MethodComponentContext methodComponentContext, + KNNMethodConfigContext knnMethodConfigContext + ) { + if (methodComponentContext.getParameters().containsKey(BITCOUNT_PARAM) == false) { + return CompressionLevel.NOT_CONFIGURED; + } + + // Map the number of bits passed in, back to the compression level + Object value = methodComponentContext.getParameters().get(BITCOUNT_PARAM); + ValidationException validationException = METHOD_COMPONENT.getParameters() + .get(BITCOUNT_PARAM) + .validate(value, knnMethodConfigContext); + if (validationException != null) { + throw validationException; + } + + Integer bitCount = (Integer) value; + if (bitCount == 1) { + return CompressionLevel.x32; + } + + if (bitCount == 2) { + return CompressionLevel.x16; + } + + // Validation will ensure that only 1 of the supported bit count will be selected. + return CompressionLevel.x8; + } } diff --git a/src/main/java/org/opensearch/knn/index/engine/lucene/Lucene.java b/src/main/java/org/opensearch/knn/index/engine/lucene/Lucene.java index 986380897..db516d309 100644 --- a/src/main/java/org/opensearch/knn/index/engine/lucene/Lucene.java +++ b/src/main/java/org/opensearch/knn/index/engine/lucene/Lucene.java @@ -10,6 +10,10 @@ import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.engine.JVMLibrary; import org.opensearch.knn.index.engine.KNNMethod; +import org.opensearch.knn.index.engine.KNNMethodConfigContext; +import org.opensearch.knn.index.engine.KNNMethodContext; +import org.opensearch.knn.index.engine.MethodResolver; +import org.opensearch.knn.index.engine.ResolvedMethodContext; import java.util.List; import java.util.Map; @@ -37,6 +41,8 @@ public class Lucene extends JVMLibrary { public final static Lucene INSTANCE = new Lucene(METHODS, Version.LATEST.toString(), DISTANCE_TRANSLATIONS); + private final MethodResolver methodResolver; + /** * Constructor * @@ -47,6 +53,7 @@ public class Lucene extends JVMLibrary { Lucene(Map methods, String version, Map> distanceTransform) { super(methods, version); this.distanceTransform = distanceTransform; + this.methodResolver = new LuceneMethodResolver(); } @Override @@ -86,4 +93,14 @@ public Float scoreToRadialThreshold(Float score, SpaceType spaceType) { public List mmapFileExtensions() { return List.of("vec", "vex"); } + + @Override + public ResolvedMethodContext resolveMethod( + KNNMethodContext knnMethodContext, + KNNMethodConfigContext knnMethodConfigContext, + boolean shouldRequireTraining, + final SpaceType spaceType + ) { + return methodResolver.resolveMethod(knnMethodContext, knnMethodConfigContext, shouldRequireTraining, spaceType); + } } diff --git a/src/main/java/org/opensearch/knn/index/engine/lucene/LuceneHNSWMethod.java b/src/main/java/org/opensearch/knn/index/engine/lucene/LuceneHNSWMethod.java index 317f67c10..57cc016a6 100644 --- a/src/main/java/org/opensearch/knn/index/engine/lucene/LuceneHNSWMethod.java +++ b/src/main/java/org/opensearch/knn/index/engine/lucene/LuceneHNSWMethod.java @@ -6,19 +6,17 @@ package org.opensearch.knn.index.engine.lucene; import com.google.common.collect.ImmutableSet; -import org.opensearch.knn.common.KNNConstants; import org.opensearch.knn.index.KNNSettings; import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.engine.AbstractKNNMethod; import org.opensearch.knn.index.engine.Encoder; import org.opensearch.knn.index.engine.MethodComponent; -import org.opensearch.knn.index.engine.MethodComponentContext; import org.opensearch.knn.index.engine.Parameter; import java.util.Arrays; -import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.stream.Collectors; @@ -41,11 +39,10 @@ public class LuceneHNSWMethod extends AbstractKNNMethod { SpaceType.INNER_PRODUCT ); - private final static MethodComponentContext DEFAULT_ENCODER_CONTEXT = new MethodComponentContext( - KNNConstants.ENCODER_FLAT, - Collections.emptyMap() - ); - private final static List SUPPORTED_ENCODERS = List.of(new LuceneSQEncoder()); + final static Encoder SQ_ENCODER = new LuceneSQEncoder(); + final static Map SUPPORTED_ENCODERS = Map.of(SQ_ENCODER.getName(), SQ_ENCODER); + + final static MethodComponent HNSW_METHOD_COMPONENT = initMethodComponent(); /** * Constructor for LuceneHNSWMethod @@ -53,7 +50,7 @@ public class LuceneHNSWMethod extends AbstractKNNMethod { * @see AbstractKNNMethod */ public LuceneHNSWMethod() { - super(initMethodComponent(), Set.copyOf(SUPPORTED_SPACES), new LuceneHNSWSearchContext()); + super(HNSW_METHOD_COMPONENT, Set.copyOf(SUPPORTED_SPACES), new LuceneHNSWSearchContext()); } private static MethodComponent initMethodComponent() { @@ -78,8 +75,8 @@ private static MethodComponent initMethodComponent() { private static Parameter.MethodComponentContextParameter initEncoderParameter() { return new Parameter.MethodComponentContextParameter( METHOD_ENCODER_PARAMETER, - DEFAULT_ENCODER_CONTEXT, - SUPPORTED_ENCODERS.stream().collect(Collectors.toMap(Encoder::getName, Encoder::getMethodComponent)) + null, + SUPPORTED_ENCODERS.values().stream().collect(Collectors.toMap(Encoder::getName, Encoder::getMethodComponent)) ); } } diff --git a/src/main/java/org/opensearch/knn/index/engine/lucene/LuceneMethodResolver.java b/src/main/java/org/opensearch/knn/index/engine/lucene/LuceneMethodResolver.java new file mode 100644 index 000000000..6546d9f93 --- /dev/null +++ b/src/main/java/org/opensearch/knn/index/engine/lucene/LuceneMethodResolver.java @@ -0,0 +1,106 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.engine.lucene; + +import org.opensearch.common.ValidationException; +import org.opensearch.knn.index.SpaceType; +import org.opensearch.knn.index.engine.AbstractMethodResolver; +import org.opensearch.knn.index.engine.KNNEngine; +import org.opensearch.knn.index.engine.KNNMethodConfigContext; +import org.opensearch.knn.index.engine.KNNMethodContext; +import org.opensearch.knn.index.engine.MethodComponent; +import org.opensearch.knn.index.engine.MethodComponentContext; +import org.opensearch.knn.index.engine.ResolvedMethodContext; +import org.opensearch.knn.index.mapper.CompressionLevel; +import org.opensearch.knn.index.mapper.Mode; + +import java.util.HashMap; +import java.util.Map; +import java.util.Set; + +import static org.opensearch.knn.common.KNNConstants.METHOD_ENCODER_PARAMETER; +import static org.opensearch.knn.common.KNNConstants.METHOD_HNSW; +import static org.opensearch.knn.index.engine.lucene.LuceneHNSWMethod.HNSW_METHOD_COMPONENT; +import static org.opensearch.knn.index.engine.lucene.LuceneHNSWMethod.SQ_ENCODER; + +public class LuceneMethodResolver extends AbstractMethodResolver { + + private static final Set SUPPORTED_COMPRESSION_LEVELS = Set.of(CompressionLevel.x1, CompressionLevel.x4); + + @Override + public ResolvedMethodContext resolveMethod( + KNNMethodContext knnMethodContext, + KNNMethodConfigContext knnMethodConfigContext, + boolean shouldRequireTraining, + final SpaceType spaceType + ) { + validateConfig(knnMethodConfigContext, shouldRequireTraining); + KNNMethodContext resolvedKNNMethodContext = initResolvedKNNMethodContext( + knnMethodContext, + KNNEngine.LUCENE, + spaceType, + METHOD_HNSW + ); + resolveEncoder(resolvedKNNMethodContext, knnMethodConfigContext); + resolveMethodParams(resolvedKNNMethodContext.getMethodComponentContext(), knnMethodConfigContext, HNSW_METHOD_COMPONENT); + CompressionLevel resolvedCompressionLevel = resolveCompressionLevelFromMethodContext( + resolvedKNNMethodContext, + knnMethodConfigContext, + LuceneHNSWMethod.SUPPORTED_ENCODERS + ); + validateCompressionConflicts(knnMethodConfigContext.getCompressionLevel(), resolvedCompressionLevel); + return ResolvedMethodContext.builder() + .knnMethodContext(resolvedKNNMethodContext) + .compressionLevel(resolvedCompressionLevel) + .build(); + } + + protected void resolveEncoder(KNNMethodContext resolvedKNNMethodContext, KNNMethodConfigContext knnMethodConfigContext) { + if (shouldEncoderBeResolved(resolvedKNNMethodContext, knnMethodConfigContext) == false) { + return; + } + + CompressionLevel resolvedCompressionLevel = getDefaultCompressionLevel(knnMethodConfigContext); + if (resolvedCompressionLevel == CompressionLevel.x1) { + return; + } + + MethodComponentContext methodComponentContext = resolvedKNNMethodContext.getMethodComponentContext(); + MethodComponentContext encoderComponentContext = new MethodComponentContext(SQ_ENCODER.getName(), new HashMap<>()); + Map resolvedParams = MethodComponent.getParameterMapWithDefaultsAdded( + encoderComponentContext, + SQ_ENCODER.getMethodComponent(), + knnMethodConfigContext + ); + encoderComponentContext.getParameters().putAll(resolvedParams); + methodComponentContext.getParameters().put(METHOD_ENCODER_PARAMETER, encoderComponentContext); + } + + // Method validates for explicit contradictions in the config + private void validateConfig(KNNMethodConfigContext knnMethodConfigContext, boolean shouldRequireTraining) { + ValidationException validationException = validateNotTrainingContext(shouldRequireTraining, KNNEngine.LUCENE, null); + validationException = validateCompressionSupported( + knnMethodConfigContext.getCompressionLevel(), + SUPPORTED_COMPRESSION_LEVELS, + KNNEngine.LUCENE, + validationException + ); + validationException = validateCompressionNotx1WhenOnDisk(knnMethodConfigContext, validationException); + if (validationException != null) { + throw validationException; + } + } + + private CompressionLevel getDefaultCompressionLevel(KNNMethodConfigContext knnMethodConfigContext) { + if (CompressionLevel.isConfigured(knnMethodConfigContext.getCompressionLevel())) { + return knnMethodConfigContext.getCompressionLevel(); + } + if (knnMethodConfigContext.getMode() == Mode.ON_DISK) { + return CompressionLevel.x4; + } + return CompressionLevel.x1; + } +} diff --git a/src/main/java/org/opensearch/knn/index/engine/lucene/LuceneSQEncoder.java b/src/main/java/org/opensearch/knn/index/engine/lucene/LuceneSQEncoder.java index 0ec43db41..6bd16ebee 100644 --- a/src/main/java/org/opensearch/knn/index/engine/lucene/LuceneSQEncoder.java +++ b/src/main/java/org/opensearch/knn/index/engine/lucene/LuceneSQEncoder.java @@ -8,8 +8,11 @@ import com.google.common.collect.ImmutableSet; import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.engine.Encoder; +import org.opensearch.knn.index.engine.KNNMethodConfigContext; import org.opensearch.knn.index.engine.MethodComponent; +import org.opensearch.knn.index.engine.MethodComponentContext; import org.opensearch.knn.index.engine.Parameter; +import org.opensearch.knn.index.mapper.CompressionLevel; import java.util.List; import java.util.Set; @@ -49,4 +52,13 @@ public class LuceneSQEncoder implements Encoder { public MethodComponent getMethodComponent() { return METHOD_COMPONENT; } + + @Override + public CompressionLevel calculateCompressionLevel( + MethodComponentContext methodComponentContext, + KNNMethodConfigContext knnMethodConfigContext + ) { + // Hard coding to 4x for now, given thats all that is supported. + return CompressionLevel.x4; + } } diff --git a/src/main/java/org/opensearch/knn/index/engine/nmslib/Nmslib.java b/src/main/java/org/opensearch/knn/index/engine/nmslib/Nmslib.java index d35cc5f6c..4d7f7f423 100644 --- a/src/main/java/org/opensearch/knn/index/engine/nmslib/Nmslib.java +++ b/src/main/java/org/opensearch/knn/index/engine/nmslib/Nmslib.java @@ -8,7 +8,11 @@ import com.google.common.collect.ImmutableMap; import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.engine.KNNMethod; +import org.opensearch.knn.index.engine.KNNMethodConfigContext; +import org.opensearch.knn.index.engine.KNNMethodContext; +import org.opensearch.knn.index.engine.MethodResolver; import org.opensearch.knn.index.engine.NativeLibrary; +import org.opensearch.knn.index.engine.ResolvedMethodContext; import java.util.Collections; import java.util.Map; @@ -27,6 +31,7 @@ public class Nmslib extends NativeLibrary { final static Map METHODS = ImmutableMap.of(METHOD_HNSW, new NmslibHNSWMethod()); public final static Nmslib INSTANCE = new Nmslib(METHODS, Collections.emptyMap(), CURRENT_VERSION, EXTENSION); + private final MethodResolver methodResolver; /** * Constructor for Nmslib @@ -43,6 +48,7 @@ private Nmslib( String extension ) { super(methods, scoreTranslation, currentVersion, extension); + this.methodResolver = new NmslibMethodResolver(); } @Override @@ -53,4 +59,14 @@ public Float distanceToRadialThreshold(Float distance, SpaceType spaceType) { public Float scoreToRadialThreshold(Float score, SpaceType spaceType) { return score; } + + @Override + public ResolvedMethodContext resolveMethod( + KNNMethodContext knnMethodContext, + KNNMethodConfigContext knnMethodConfigContext, + boolean shouldRequireTraining, + final SpaceType spaceType + ) { + return methodResolver.resolveMethod(knnMethodContext, knnMethodConfigContext, shouldRequireTraining, spaceType); + } } diff --git a/src/main/java/org/opensearch/knn/index/engine/nmslib/NmslibHNSWMethod.java b/src/main/java/org/opensearch/knn/index/engine/nmslib/NmslibHNSWMethod.java index 779c16cd3..d2440926e 100644 --- a/src/main/java/org/opensearch/knn/index/engine/nmslib/NmslibHNSWMethod.java +++ b/src/main/java/org/opensearch/knn/index/engine/nmslib/NmslibHNSWMethod.java @@ -38,12 +38,14 @@ public class NmslibHNSWMethod extends AbstractKNNMethod { SpaceType.INNER_PRODUCT ); + final static MethodComponent HNSW_METHOD_COMPONENT = initMethodComponent(); + /** * Constructor. Builds the method with the default parameters and supported spaces. * @see AbstractKNNMethod */ public NmslibHNSWMethod() { - super(initMethodComponent(), Set.copyOf(SUPPORTED_SPACES), new DefaultHnswSearchContext()); + super(HNSW_METHOD_COMPONENT, Set.copyOf(SUPPORTED_SPACES), new DefaultHnswSearchContext()); } private static MethodComponent initMethodComponent() { diff --git a/src/main/java/org/opensearch/knn/index/engine/nmslib/NmslibMethodResolver.java b/src/main/java/org/opensearch/knn/index/engine/nmslib/NmslibMethodResolver.java new file mode 100644 index 000000000..619a00eda --- /dev/null +++ b/src/main/java/org/opensearch/knn/index/engine/nmslib/NmslibMethodResolver.java @@ -0,0 +1,69 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.engine.nmslib; + +import org.opensearch.common.ValidationException; +import org.opensearch.knn.index.SpaceType; +import org.opensearch.knn.index.engine.AbstractMethodResolver; +import org.opensearch.knn.index.engine.KNNEngine; +import org.opensearch.knn.index.engine.KNNMethodConfigContext; +import org.opensearch.knn.index.engine.KNNMethodContext; +import org.opensearch.knn.index.engine.ResolvedMethodContext; +import org.opensearch.knn.index.mapper.CompressionLevel; +import org.opensearch.knn.index.mapper.Mode; + +import java.util.Set; + +import static org.opensearch.knn.common.KNNConstants.METHOD_HNSW; +import static org.opensearch.knn.index.engine.nmslib.NmslibHNSWMethod.HNSW_METHOD_COMPONENT; + +/** + * Method resolution logic for nmslib. Because nmslib does not support quantization, it is in general a validation + * before returning the original request + */ +public class NmslibMethodResolver extends AbstractMethodResolver { + + private static final Set SUPPORTED_COMPRESSION_LEVELS = Set.of(CompressionLevel.x1); + + @Override + public ResolvedMethodContext resolveMethod( + KNNMethodContext knnMethodContext, + KNNMethodConfigContext knnMethodConfigContext, + boolean shouldRequireTraining, + final SpaceType spaceType + ) { + validateConfig(knnMethodConfigContext, shouldRequireTraining); + KNNMethodContext resolvedKNNMethodContext = initResolvedKNNMethodContext( + knnMethodContext, + KNNEngine.NMSLIB, + spaceType, + METHOD_HNSW + ); + resolveMethodParams(resolvedKNNMethodContext.getMethodComponentContext(), knnMethodConfigContext, HNSW_METHOD_COMPONENT); + return ResolvedMethodContext.builder().knnMethodContext(resolvedKNNMethodContext).compressionLevel(CompressionLevel.x1).build(); + } + + // Method validates for explicit contradictions in the config + private void validateConfig(KNNMethodConfigContext knnMethodConfigContext, boolean shouldRequireTraining) { + ValidationException validationException = validateNotTrainingContext(shouldRequireTraining, KNNEngine.NMSLIB, null); + CompressionLevel compressionLevel = knnMethodConfigContext.getCompressionLevel(); + validationException = validateCompressionSupported( + compressionLevel, + SUPPORTED_COMPRESSION_LEVELS, + KNNEngine.NMSLIB, + validationException + ); + + if (Mode.ON_DISK == knnMethodConfigContext.getMode()) { + validationException = validationException == null ? new ValidationException() : validationException; + validationException.addValidationError("Nmslib engine does not support disk-based search"); + } + + if (validationException != null) { + throw validationException; + } + } +} diff --git a/src/main/java/org/opensearch/knn/index/mapper/CompressionLevel.java b/src/main/java/org/opensearch/knn/index/mapper/CompressionLevel.java index 4b5026598..cc80bb1ed 100644 --- a/src/main/java/org/opensearch/knn/index/mapper/CompressionLevel.java +++ b/src/main/java/org/opensearch/knn/index/mapper/CompressionLevel.java @@ -10,7 +10,9 @@ import org.opensearch.core.common.Strings; import org.opensearch.knn.index.query.rescore.RescoreContext; +import java.util.Collections; import java.util.Locale; +import java.util.Set; /** * Enum representing the compression level for float vectors. Compression in this sense refers to compressing a @@ -19,13 +21,13 @@ */ @AllArgsConstructor public enum CompressionLevel { - NOT_CONFIGURED(-1, "", null), - x1(1, "1x", null), - x2(2, "2x", null), - x4(4, "4x", new RescoreContext(1.0f)), - x8(8, "8x", new RescoreContext(1.5f)), - x16(16, "16x", new RescoreContext(2.0f)), - x32(32, "32x", new RescoreContext(2.0f)); + NOT_CONFIGURED(-1, "", null, Collections.emptySet()), + x1(1, "1x", null, Collections.emptySet()), + x2(2, "2x", null, Collections.emptySet()), + x4(4, "4x", null, Collections.emptySet()), + x8(8, "8x", new RescoreContext(1.5f), Set.of(Mode.ON_DISK)), + x16(16, "16x", new RescoreContext(2.0f), Set.of(Mode.ON_DISK)), + x32(32, "32x", new RescoreContext(2.0f), Set.of(Mode.ON_DISK)); // Internally, an empty string is easier to deal with them null. However, from the mapping, // we do not want users to pass in the empty string and instead want null. So we make the conversion herex @@ -33,6 +35,7 @@ public enum CompressionLevel { NOT_CONFIGURED.getName(), x1.getName(), x2.getName(), + x4.getName(), x8.getName(), x16.getName(), x32.getName() }; @@ -64,8 +67,8 @@ public static CompressionLevel fromName(String name) { private final int compressionLevel; @Getter private final String name; - @Getter private final RescoreContext defaultRescoreContext; + private final Set modesForRescore; /** * Gets the number of bits used to represent a float in order to achieve this compression. For instance, for @@ -90,4 +93,11 @@ public int numBitsForFloat32() { public static boolean isConfigured(CompressionLevel compressionLevel) { return compressionLevel != null && compressionLevel != NOT_CONFIGURED; } + + public RescoreContext getDefaultRescoreContext(Mode mode) { + if (modesForRescore.contains(mode)) { + return defaultRescoreContext; + } + return null; + } } diff --git a/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapper.java b/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapper.java index f149fa1d2..6e5138a56 100644 --- a/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapper.java +++ b/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapper.java @@ -39,12 +39,15 @@ import org.opensearch.index.mapper.ParseContext; import org.opensearch.knn.common.KNNConstants; import org.opensearch.knn.index.KNNSettings; +import org.opensearch.knn.index.engine.EngineResolver; import org.opensearch.knn.index.engine.KNNMethodConfigContext; import org.opensearch.knn.index.engine.KNNMethodContext; import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.VectorField; import org.opensearch.knn.index.engine.KNNEngine; +import org.opensearch.knn.index.engine.ResolvedMethodContext; +import org.opensearch.knn.index.engine.SpaceTypeResolver; import org.opensearch.knn.indices.ModelDao; import static org.opensearch.knn.common.KNNConstants.DEFAULT_VECTOR_DATA_TYPE_FIELD; @@ -174,6 +177,7 @@ public static class Builder extends ParametrizedFieldMapper.Builder { protected ModelDao modelDao; protected Version indexCreatedVersion; @Setter + @Getter private KNNMethodConfigContext knnMethodConfigContext; @Setter @Getter @@ -365,9 +369,20 @@ public Mapper.Builder parse(String name, Map node, ParserCont } else if (builder.modelId.get() != null) { validateFromModel(builder); } else { - validateMode(builder); + // Validate that the mode and compression are not set if data type is not float, as they are not + // supported. + validateModeAndCompressionForDataType(builder); + // If the original knnMethodContext is not null, resolve its space type and engine from the rest of the + // configuration. This is consistent with the existing behavior for space type in 2.16 where we modify the + // parsed value + SpaceType resolvedSpaceType = SpaceTypeResolver.INSTANCE.resolveSpaceType( + builder.originalParameters.getKnnMethodContext(), + builder.vectorDataType.get(), + builder.topLevelSpaceType.get() + ); + setSpaceType(builder.originalParameters.getKnnMethodContext(), resolvedSpaceType); validateSpaceType(builder); - resolveKNNMethodComponents(builder, parserContext); + resolveKNNMethodComponents(builder, parserContext, resolvedSpaceType); validateFromKNNMethod(builder); } @@ -391,20 +406,9 @@ private void validateSpaceType(KNNVectorFieldMapper.Builder builder) { } } - private void validateMode(KNNVectorFieldMapper.Builder builder) { - boolean isKNNMethodContextConfigured = builder.originalParameters.getKnnMethodContext() != null; - boolean isModeConfigured = builder.mode.isConfigured() || builder.compressionLevel.isConfigured(); - if (isModeConfigured && isKNNMethodContextConfigured) { - throw new MapperParsingException( - String.format( - Locale.ROOT, - "Compression and mode can not be specified in a \"method\" mapping configuration for field: %s", - builder.name - ) - ); - } - - if (isModeConfigured && builder.vectorDataType.getValue() != VectorDataType.FLOAT) { + private void validateModeAndCompressionForDataType(KNNVectorFieldMapper.Builder builder) { + boolean isModeOrCompressionConfigured = builder.mode.isConfigured() || builder.compressionLevel.isConfigured(); + if (isModeOrCompressionConfigured && builder.vectorDataType.getValue() != VectorDataType.FLOAT) { throw new MapperParsingException( String.format(Locale.ROOT, "Compression and mode cannot be used for non-float32 data type for field %s", builder.name) ); @@ -468,7 +472,12 @@ private void validateCompressionAndModeNotSet(KNNVectorFieldMapper.Builder build } } - private void resolveKNNMethodComponents(KNNVectorFieldMapper.Builder builder, ParserContext parserContext) { + private void resolveKNNMethodComponents( + KNNVectorFieldMapper.Builder builder, + ParserContext parserContext, + SpaceType resolvedSpaceType + ) { + // Setup the initial configuration that is used to help resolve parameters. builder.setKnnMethodConfigContext( KNNMethodConfigContext.builder() .vectorDataType(builder.originalParameters.getVectorDataType()) @@ -479,36 +488,34 @@ private void resolveKNNMethodComponents(KNNVectorFieldMapper.Builder builder, Pa .build() ); - // Configure method from map or legacy + // If the original parameters are from legacy if (builder.originalParameters.isLegacyMapping()) { + // Then create KNNMethodContext to be used from the legacy index settings builder.originalParameters.setResolvedKnnMethodContext( - createKNNMethodContextFromLegacy( - parserContext.getSettings(), - parserContext.indexVersionCreated(), - SpaceType.getSpace(builder.topLevelSpaceType.get()) - ) + createKNNMethodContextFromLegacy(parserContext.getSettings(), parserContext.indexVersionCreated(), resolvedSpaceType) ); - } else if (Mode.isConfigured(Mode.fromName(builder.mode.get())) - || CompressionLevel.isConfigured(CompressionLevel.fromName(builder.compressionLevel.get()))) { - // we need don't need to resolve the space type, whatever default we are using will be passed down to - // while resolving KNNMethodContext for the mode and compression. and then when we resolve the spaceType - // we will set the correct spaceType. - builder.originalParameters.setResolvedKnnMethodContext( - ModeBasedResolver.INSTANCE.resolveKNNMethodContext( - builder.knnMethodConfigContext.getMode(), - builder.knnMethodConfigContext.getCompressionLevel(), - false, - SpaceType.getSpace(builder.originalParameters.getTopLevelSpaceType()) - ) - ); - } - // this function should now correct the space type for the above resolved context too, if spaceType was - // not provided. - setSpaceType( + } + + // Based on config context, if the user does not set the engine, set it + KNNEngine resolvedKNNEngine = EngineResolver.INSTANCE.resolveEngine( + builder.knnMethodConfigContext, builder.originalParameters.getResolvedKnnMethodContext(), - builder.originalParameters.getVectorDataType(), - builder.topLevelSpaceType.get() + false ); + setEngine(builder.originalParameters.getResolvedKnnMethodContext(), resolvedKNNEngine); + + // Create a copy of the KNNMethodContext and fill in the parameters left blank by configuration context context + ResolvedMethodContext resolvedMethodContext = resolvedKNNEngine.resolveMethod( + builder.originalParameters.getResolvedKnnMethodContext(), + builder.knnMethodConfigContext, + false, + resolvedSpaceType + ); + + // The original parameters stores both the resolveMethodContext as well as the original provided by the + // user. Now that we have resolved, we need to update this in the original parameters. + builder.originalParameters.setResolvedKnnMethodContext(resolvedMethodContext.getKnnMethodContext()); + builder.knnMethodConfigContext.setCompressionLevel(resolvedMethodContext.getCompressionLevel()); } private boolean isKNNDisabled(Settings settings) { @@ -516,32 +523,18 @@ private boolean isKNNDisabled(Settings settings) { return !isSettingPresent || !KNNSettings.IS_KNN_INDEX_SETTING.get(settings); } - private void setSpaceType( - final KNNMethodContext knnMethodContext, - final VectorDataType vectorDataType, - final String topLevelSpaceType - ) { - // Now KNNMethodContext should never be null. Because only case it could be null is flatMapper which is - // already handled + private void setSpaceType(final KNNMethodContext knnMethodContext, final SpaceType spaceType) { if (knnMethodContext == null) { - throw new IllegalArgumentException("KNNMethodContext cannot be null"); + return; } - final SpaceType topLevelSpaceTypeEnum = SpaceType.getSpace(topLevelSpaceType); - // Now set the spaceSpaceType for KNNMethodContext - if (SpaceType.UNDEFINED == knnMethodContext.getSpaceType()) { - // We are handling the case when top level space type is defined but method level spaceType is not - // defined. - if (topLevelSpaceTypeEnum != SpaceType.UNDEFINED) { - knnMethodContext.setSpaceType(topLevelSpaceTypeEnum); - } else { - // If both spaceTypes are undefined then put the default spaceType based on datatype - if (VectorDataType.BINARY == vectorDataType) { - knnMethodContext.setSpaceType(SpaceType.DEFAULT_BINARY); - } else { - knnMethodContext.setSpaceType(SpaceType.DEFAULT); - } - } + knnMethodContext.setSpaceType(spaceType); + } + + private void setEngine(final KNNMethodContext knnMethodContext, KNNEngine knnEngine) { + if (knnMethodContext == null || knnMethodContext.isEngineConfigured()) { + return; } + knnMethodContext.setKnnEngine(knnEngine); } } diff --git a/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldType.java b/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldType.java index 963688d0c..e684ba4f1 100644 --- a/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldType.java +++ b/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldType.java @@ -93,9 +93,6 @@ public RescoreContext resolveRescoreContext(RescoreContext userProvidedContext) if (userProvidedContext != null) { return userProvidedContext; } - return ModeBasedResolver.INSTANCE.resolveRescoreContext( - getKnnMappingConfig().getMode(), - getKnnMappingConfig().getCompressionLevel() - ); + return getKnnMappingConfig().getCompressionLevel().getDefaultRescoreContext(getKnnMappingConfig().getMode()); } } diff --git a/src/main/java/org/opensearch/knn/index/mapper/LuceneFieldMapper.java b/src/main/java/org/opensearch/knn/index/mapper/LuceneFieldMapper.java index 3da2745ac..fcf3aa034 100644 --- a/src/main/java/org/opensearch/knn/index/mapper/LuceneFieldMapper.java +++ b/src/main/java/org/opensearch/knn/index/mapper/LuceneFieldMapper.java @@ -63,6 +63,16 @@ public Optional getKnnMethodContext() { public int getDimension() { return knnMethodConfigContext.getDimension(); } + + @Override + public Mode getMode() { + return knnMethodConfigContext.getMode(); + } + + @Override + public CompressionLevel getCompressionLevel() { + return knnMethodConfigContext.getCompressionLevel(); + } } ); @@ -87,24 +97,23 @@ private LuceneFieldMapper( originalMappingParameters ); KNNMappingConfig knnMappingConfig = mappedFieldType.getKnnMappingConfig(); - KNNMethodContext knnMethodContext = knnMappingConfig.getKnnMethodContext() - .orElseThrow(() -> new IllegalArgumentException("KNN method context is missing")); + KNNMethodContext resolvedKnnMethodContext = originalMappingParameters.getResolvedKnnMethodContext(); VectorDataType vectorDataType = mappedFieldType.getVectorDataType(); - final VectorSimilarityFunction vectorSimilarityFunction = knnMethodContext.getSpaceType() + final VectorSimilarityFunction vectorSimilarityFunction = resolvedKnnMethodContext.getSpaceType() .getKnnVectorSimilarityFunction() .getVectorSimilarityFunction(); this.fieldType = vectorDataType.createKnnVectorFieldType(knnMappingConfig.getDimension(), vectorSimilarityFunction); if (this.hasDocValues) { - this.vectorFieldType = buildDocValuesFieldType(knnMethodContext.getKnnEngine()); + this.vectorFieldType = buildDocValuesFieldType(resolvedKnnMethodContext.getKnnEngine()); } else { this.vectorFieldType = null; } - KNNLibraryIndexingContext knnLibraryIndexingContext = knnMethodContext.getKnnEngine() - .getKNNLibraryIndexingContext(knnMethodContext, knnMethodConfigContext); + KNNLibraryIndexingContext knnLibraryIndexingContext = resolvedKnnMethodContext.getKnnEngine() + .getKNNLibraryIndexingContext(resolvedKnnMethodContext, knnMethodConfigContext); this.perDimensionProcessor = knnLibraryIndexingContext.getPerDimensionProcessor(); this.perDimensionValidator = knnLibraryIndexingContext.getPerDimensionValidator(); this.vectorValidator = knnLibraryIndexingContext.getVectorValidator(); diff --git a/src/main/java/org/opensearch/knn/index/mapper/MethodFieldMapper.java b/src/main/java/org/opensearch/knn/index/mapper/MethodFieldMapper.java index 3d11949fe..bf5bc2b51 100644 --- a/src/main/java/org/opensearch/knn/index/mapper/MethodFieldMapper.java +++ b/src/main/java/org/opensearch/knn/index/mapper/MethodFieldMapper.java @@ -74,7 +74,7 @@ public int getDimension() { @Override public Mode getMode() { - return knnMethodConfigContext.getMode(); + return Mode.fromName(originalMappingParameters.getMode()); } @Override @@ -125,19 +125,18 @@ private MethodFieldMapper( originalMappingParameters ); this.useLuceneBasedVectorField = KNNVectorFieldMapperUtil.useLuceneKNNVectorsFormat(indexCreatedVersion); - KNNMappingConfig annConfig = mappedFieldType.getKnnMappingConfig(); - KNNMethodContext knnMethodContext = annConfig.getKnnMethodContext() - .orElseThrow(() -> new IllegalArgumentException("KNN method context cannot be empty")); - KNNEngine knnEngine = knnMethodContext.getKnnEngine(); + KNNMappingConfig knnMappingConfig = mappedFieldType.getKnnMappingConfig(); + KNNMethodContext resolvedKnnMethodContext = originalMappingParameters.getResolvedKnnMethodContext(); + KNNEngine knnEngine = resolvedKnnMethodContext.getKnnEngine(); KNNLibraryIndexingContext knnLibraryIndexingContext = knnEngine.getKNNLibraryIndexingContext( - knnMethodContext, + resolvedKnnMethodContext, knnMethodConfigContext ); QuantizationConfig quantizationConfig = knnLibraryIndexingContext.getQuantizationConfig(); this.fieldType = new FieldType(KNNVectorFieldMapper.Defaults.FIELD_TYPE); - this.fieldType.putAttribute(DIMENSION, String.valueOf(annConfig.getDimension())); - this.fieldType.putAttribute(SPACE_TYPE, knnMethodContext.getSpaceType().getValue()); + this.fieldType.putAttribute(DIMENSION, String.valueOf(knnMappingConfig.getDimension())); + this.fieldType.putAttribute(SPACE_TYPE, resolvedKnnMethodContext.getSpaceType().getValue()); // Conditionally add quantization config if (quantizationConfig != null && quantizationConfig != QuantizationConfig.EMPTY) { this.fieldType.putAttribute(QFRAMEWORK_CONFIG, QuantizationConfigParser.toCsv(quantizationConfig)); @@ -157,8 +156,8 @@ private MethodFieldMapper( if (useLuceneBasedVectorField) { int adjustedDimension = mappedFieldType.vectorDataType == VectorDataType.BINARY - ? annConfig.getDimension() / 8 - : annConfig.getDimension(); + ? knnMappingConfig.getDimension() / 8 + : knnMappingConfig.getDimension(); final VectorEncoding encoding = mappedFieldType.vectorDataType == VectorDataType.FLOAT ? VectorEncoding.FLOAT32 : VectorEncoding.BYTE; diff --git a/src/main/java/org/opensearch/knn/index/mapper/ModeBasedResolver.java b/src/main/java/org/opensearch/knn/index/mapper/ModeBasedResolver.java deleted file mode 100644 index 2a0c8ef46..000000000 --- a/src/main/java/org/opensearch/knn/index/mapper/ModeBasedResolver.java +++ /dev/null @@ -1,213 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.knn.index.mapper; - -import org.opensearch.knn.index.KNNSettings; -import org.opensearch.knn.index.SpaceType; -import org.opensearch.knn.index.engine.KNNEngine; -import org.opensearch.knn.index.engine.KNNMethodContext; -import org.opensearch.knn.index.engine.MethodComponentContext; -import org.opensearch.knn.index.engine.faiss.QFrameBitEncoder; -import org.opensearch.knn.index.query.rescore.RescoreContext; - -import java.util.Locale; -import java.util.Map; -import java.util.Set; - -import static org.opensearch.knn.common.KNNConstants.ENCODER_SQ; -import static org.opensearch.knn.common.KNNConstants.FAISS_SQ_CLIP; -import static org.opensearch.knn.common.KNNConstants.FAISS_SQ_ENCODER_FP16; -import static org.opensearch.knn.common.KNNConstants.FAISS_SQ_TYPE; -import static org.opensearch.knn.common.KNNConstants.METHOD_ENCODER_PARAMETER; -import static org.opensearch.knn.common.KNNConstants.METHOD_HNSW; -import static org.opensearch.knn.common.KNNConstants.METHOD_IVF; -import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_EF_CONSTRUCTION; -import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_EF_SEARCH; -import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_M; -import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_NLIST; -import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_NLIST_DEFAULT; -import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_NPROBES; -import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_NPROBES_DEFAULT; - -/** - * Class contains the logic to make parameter resolutions based on the {@link Mode} and {@link CompressionLevel}. - */ -public final class ModeBasedResolver { - - public static final ModeBasedResolver INSTANCE = new ModeBasedResolver(); - - private static final CompressionLevel DEFAULT_COMPRESSION_FOR_MODE_ON_DISK = CompressionLevel.x32; - private static final CompressionLevel DEFAULT_COMPRESSION_FOR_MODE_IN_MEMORY = CompressionLevel.x1; - public final static Set SUPPORTED_COMPRESSION_LEVELS = Set.of( - CompressionLevel.x1, - CompressionLevel.x2, - CompressionLevel.x8, - CompressionLevel.x16, - CompressionLevel.x32 - ); - - private ModeBasedResolver() {} - - /** - * Based on the provided {@link Mode} and {@link CompressionLevel}, resolve to a {@link KNNMethodContext} - * - * @param mode {@link Mode} - * @param compressionLevel {@link CompressionLevel} - * @param requiresTraining whether config requires trianing - * @return {@link KNNMethodContext} - */ - public KNNMethodContext resolveKNNMethodContext( - Mode mode, - CompressionLevel compressionLevel, - boolean requiresTraining, - SpaceType spaceType - ) { - if (requiresTraining) { - return resolveWithTraining(mode, compressionLevel, spaceType); - } - return resolveWithoutTraining(mode, compressionLevel, spaceType); - } - - private KNNMethodContext resolveWithoutTraining(Mode mode, CompressionLevel compressionLevel, final SpaceType spaceType) { - CompressionLevel resolvedCompressionLevel = resolveCompressionLevel(mode, compressionLevel); - MethodComponentContext encoderContext = resolveEncoder(resolvedCompressionLevel); - - KNNEngine knnEngine = Mode.ON_DISK == mode || encoderContext != null ? KNNEngine.FAISS : KNNEngine.DEFAULT; - - if (encoderContext != null) { - return new KNNMethodContext( - knnEngine, - spaceType, - new MethodComponentContext( - METHOD_HNSW, - Map.of( - METHOD_PARAMETER_M, - KNNSettings.INDEX_KNN_DEFAULT_ALGO_PARAM_M, - METHOD_PARAMETER_EF_CONSTRUCTION, - KNNSettings.INDEX_KNN_DEFAULT_ALGO_PARAM_EF_CONSTRUCTION, - METHOD_PARAMETER_EF_SEARCH, - KNNSettings.INDEX_KNN_DEFAULT_ALGO_PARAM_EF_SEARCH, - METHOD_ENCODER_PARAMETER, - encoderContext - ) - ) - ); - } - - if (knnEngine == KNNEngine.FAISS) { - return new KNNMethodContext( - knnEngine, - spaceType, - new MethodComponentContext( - METHOD_HNSW, - Map.of( - METHOD_PARAMETER_M, - KNNSettings.INDEX_KNN_DEFAULT_ALGO_PARAM_M, - METHOD_PARAMETER_EF_CONSTRUCTION, - KNNSettings.INDEX_KNN_DEFAULT_ALGO_PARAM_EF_CONSTRUCTION, - METHOD_PARAMETER_EF_SEARCH, - KNNSettings.INDEX_KNN_DEFAULT_ALGO_PARAM_EF_SEARCH - ) - ) - ); - } - - return new KNNMethodContext( - knnEngine, - spaceType, - new MethodComponentContext( - METHOD_HNSW, - Map.of( - METHOD_PARAMETER_M, - KNNSettings.INDEX_KNN_DEFAULT_ALGO_PARAM_M, - METHOD_PARAMETER_EF_CONSTRUCTION, - KNNSettings.INDEX_KNN_DEFAULT_ALGO_PARAM_EF_CONSTRUCTION - ) - ) - ); - } - - private KNNMethodContext resolveWithTraining(Mode mode, CompressionLevel compressionLevel, SpaceType spaceType) { - CompressionLevel resolvedCompressionLevel = resolveCompressionLevel(mode, compressionLevel); - MethodComponentContext encoderContext = resolveEncoder(resolvedCompressionLevel); - if (encoderContext != null) { - return new KNNMethodContext( - KNNEngine.FAISS, - spaceType, - new MethodComponentContext( - METHOD_IVF, - Map.of( - METHOD_PARAMETER_NLIST, - METHOD_PARAMETER_NLIST_DEFAULT, - METHOD_PARAMETER_NPROBES, - METHOD_PARAMETER_NPROBES_DEFAULT, - METHOD_ENCODER_PARAMETER, - encoderContext - ) - ) - ); - } - - return new KNNMethodContext( - KNNEngine.FAISS, - spaceType, - new MethodComponentContext( - METHOD_IVF, - Map.of(METHOD_PARAMETER_NLIST, METHOD_PARAMETER_NLIST_DEFAULT, METHOD_PARAMETER_NPROBES, METHOD_PARAMETER_NPROBES_DEFAULT) - ) - ); - } - - /** - * Resolves the rescore context give the {@link Mode} and {@link CompressionLevel} - * - * @param mode {@link Mode} - * @param compressionLevel {@link CompressionLevel} - * @return {@link RescoreContext} - */ - public RescoreContext resolveRescoreContext(Mode mode, CompressionLevel compressionLevel) { - CompressionLevel resolvedCompressionLevel = resolveCompressionLevel(mode, compressionLevel); - return resolvedCompressionLevel.getDefaultRescoreContext(); - } - - private CompressionLevel resolveCompressionLevel(Mode mode, CompressionLevel compressionLevel) { - if (CompressionLevel.isConfigured(compressionLevel)) { - return compressionLevel; - } - - if (mode == Mode.ON_DISK) { - return DEFAULT_COMPRESSION_FOR_MODE_ON_DISK; - } - - return DEFAULT_COMPRESSION_FOR_MODE_IN_MEMORY; - } - - private MethodComponentContext resolveEncoder(CompressionLevel compressionLevel) { - if (CompressionLevel.isConfigured(compressionLevel) == false) { - throw new IllegalStateException("Compression level needs to be configured"); - } - - if (SUPPORTED_COMPRESSION_LEVELS.contains(compressionLevel) == false) { - throw new IllegalArgumentException( - String.format(Locale.ROOT, "Unsupported compression level: \"[%s]\"", compressionLevel.getName()) - ); - } - - if (compressionLevel == CompressionLevel.x1) { - return null; - } - - if (compressionLevel == CompressionLevel.x2) { - return new MethodComponentContext(ENCODER_SQ, Map.of(FAISS_SQ_TYPE, FAISS_SQ_ENCODER_FP16, FAISS_SQ_CLIP, true)); - } - - return new MethodComponentContext( - QFrameBitEncoder.NAME, - Map.of(QFrameBitEncoder.BITCOUNT_PARAM, compressionLevel.numBitsForFloat32()) - ); - } - -} diff --git a/src/main/java/org/opensearch/knn/index/mapper/OriginalMappingParameters.java b/src/main/java/org/opensearch/knn/index/mapper/OriginalMappingParameters.java index 77bf07a90..340c450ee 100644 --- a/src/main/java/org/opensearch/knn/index/mapper/OriginalMappingParameters.java +++ b/src/main/java/org/opensearch/knn/index/mapper/OriginalMappingParameters.java @@ -37,6 +37,8 @@ public final class OriginalMappingParameters { // (https://github.com/opensearch-project/OpenSearch/blob/2.16.0/server/src/main/java/org/opensearch/index/mapper/ParametrizedFieldMapper.java#L322-L324). // So, what we do is pass in a "resolvedKNNMethodContext" to ensure we track this resolveKnnMethodContext. // A similar approach was taken for https://github.com/opendistro-for-elasticsearch/k-NN/issues/288 + // + // In almost all cases except when dealing with the mapping, the resolved context should be used @Setter private KNNMethodContext resolvedKnnMethodContext; private final String mode; diff --git a/src/main/java/org/opensearch/knn/index/util/IndexUtil.java b/src/main/java/org/opensearch/knn/index/util/IndexUtil.java index 02aa1e954..02f57660b 100644 --- a/src/main/java/org/opensearch/knn/index/util/IndexUtil.java +++ b/src/main/java/org/opensearch/knn/index/util/IndexUtil.java @@ -34,6 +34,7 @@ import java.util.Set; import static org.opensearch.knn.common.KNNConstants.BYTES_PER_KILOBYTES; +import static org.opensearch.knn.common.KNNConstants.ENCODER_FLAT; import static org.opensearch.knn.common.KNNConstants.HNSW_ALGO_EF_SEARCH; import static org.opensearch.knn.common.KNNConstants.SPACE_TYPE; import static org.opensearch.knn.common.KNNConstants.VECTOR_DATA_TYPE_FIELD; @@ -173,7 +174,9 @@ public static ValidationException validateKnnField( if (parameters != null && parameters.containsKey(KNNConstants.METHOD_ENCODER_PARAMETER)) { MethodComponentContext encoder = (MethodComponentContext) parameters.get(KNNConstants.METHOD_ENCODER_PARAMETER); - if (encoder != null && VECTOR_DATA_TYPES_NOT_SUPPORTING_ENCODERS.contains(trainRequestVectorDataType)) { + if (encoder != null + && VECTOR_DATA_TYPES_NOT_SUPPORTING_ENCODERS.contains(trainRequestVectorDataType) + && ENCODER_FLAT.equals(encoder.getName()) == false) { exception.addValidationError( String.format( Locale.ROOT, diff --git a/src/main/java/org/opensearch/knn/plugin/rest/RestTrainModelHandler.java b/src/main/java/org/opensearch/knn/plugin/rest/RestTrainModelHandler.java index 0f7e8523b..71f7201de 100644 --- a/src/main/java/org/opensearch/knn/plugin/rest/RestTrainModelHandler.java +++ b/src/main/java/org/opensearch/knn/plugin/rest/RestTrainModelHandler.java @@ -19,6 +19,7 @@ import org.opensearch.knn.index.engine.KNNMethodContext; import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.VectorDataType; +import org.opensearch.knn.index.engine.SpaceTypeResolver; import org.opensearch.knn.index.mapper.CompressionLevel; import org.opensearch.knn.index.mapper.Mode; import org.opensearch.knn.indices.ModelUtil; @@ -132,8 +133,7 @@ private TrainingModelRequest createTransportRequest(RestRequest restRequest) thr } } - // Check that these parameters get set - ensureAtleasOneSet(KNN_METHOD, knnMethodContext, MODE_PARAMETER, mode, COMPRESSION_LEVEL_PARAMETER, compressionLevel); + ensureAtleastOneSet(KNN_METHOD, knnMethodContext, MODE_PARAMETER, mode, COMPRESSION_LEVEL_PARAMETER, compressionLevel); ensureMutualExclusion(KNN_METHOD, knnMethodContext, MODE_PARAMETER, mode); ensureMutualExclusion(KNN_METHOD, knnMethodContext, COMPRESSION_LEVEL_PARAMETER, compressionLevel); @@ -160,11 +160,12 @@ private TrainingModelRequest createTransportRequest(RestRequest restRequest) thr vectorDataType, VectorDataType.FLOAT.getValue() ); - resolveSpaceTypeAndSetInKNNMethodContext(topLevelSpaceType, knnMethodContext); - // if KNNMethodContext was not null then spaceTypes we should fix the space type if it is not set. - if (knnMethodContext == null && topLevelSpaceType == SpaceType.UNDEFINED) { - topLevelSpaceType = SpaceType.DEFAULT; - } + SpaceType resolvedSpaceType = SpaceTypeResolver.INSTANCE.resolveSpaceType( + knnMethodContext, + vectorDataType, + topLevelSpaceType.getValue() + ); + setSpaceType(knnMethodContext, resolvedSpaceType); TrainingModelRequest trainingModelRequest = new TrainingModelRequest( modelId, knnMethodContext, @@ -176,7 +177,7 @@ private TrainingModelRequest createTransportRequest(RestRequest restRequest) thr vectorDataType, Mode.fromName(mode), CompressionLevel.fromName(compressionLevel), - topLevelSpaceType + resolvedSpaceType ); if (maximumVectorCount != DEFAULT_NOT_SET_INT_VALUE) { @@ -217,26 +218,11 @@ private boolean ensureSpaceTypeNotSet(SpaceType spaceType) { return true; } - private void resolveSpaceTypeAndSetInKNNMethodContext(SpaceType topLevelSpaceType, KNNMethodContext knnMethodContext) { - // First check if KNNMethodContext is not null as it can be null - if (knnMethodContext != null) { - // if space type is not provided by user then it will undefined - if (knnMethodContext.getSpaceType() == SpaceType.UNDEFINED) { - // fix the top level spaceType if it is undefined - if (topLevelSpaceType == SpaceType.UNDEFINED) { - topLevelSpaceType = SpaceType.DEFAULT; - } - // set the space type now in KNNMethodContext - knnMethodContext.setSpaceType(topLevelSpaceType); - } else { - // if spaceType is set at 2 places lets ensure that we validate those cases and throw error - if (topLevelSpaceType != SpaceType.UNDEFINED) { - throw new IllegalArgumentException( - "Top Level spaceType and space type in method both are set. Set space type at 1 place." - ); - } - } + private void setSpaceType(KNNMethodContext knnMethodContext, SpaceType resolvedSpaceType) { + if (knnMethodContext == null) { + return; } + knnMethodContext.setSpaceType(resolvedSpaceType); } private void ensureIfSetThenEquals( @@ -263,8 +249,11 @@ private void ensureIfSetThenEquals( } } - private void ensureAtleasOneSet(String fieldNameA, Object valueA, String fieldNameB, Object valueB, String fieldNameC, Object valueC) { + private void ensureAtleastOneSet(String fieldNameA, Object valueA, String fieldNameB, Object valueB, String fieldNameC, Object valueC) { if (valueA == DEFAULT_NOT_SET_OBJECT_VALUE && valueB == DEFAULT_NOT_SET_OBJECT_VALUE && valueC == DEFAULT_NOT_SET_OBJECT_VALUE) { + throw new IllegalArgumentException( + String.format(Locale.ROOT, "At least \"[%s]\", \"[%s]\" or \"[%s]\" needs to be set", fieldNameA, fieldNameB, fieldNameC) + ); } } diff --git a/src/main/java/org/opensearch/knn/plugin/transport/TrainingModelRequest.java b/src/main/java/org/opensearch/knn/plugin/transport/TrainingModelRequest.java index df24baf0e..9906ab490 100644 --- a/src/main/java/org/opensearch/knn/plugin/transport/TrainingModelRequest.java +++ b/src/main/java/org/opensearch/knn/plugin/transport/TrainingModelRequest.java @@ -22,10 +22,12 @@ import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.knn.common.KNNConstants; import org.opensearch.knn.index.SpaceType; +import org.opensearch.knn.index.engine.KNNEngine; import org.opensearch.knn.index.engine.KNNMethodConfigContext; +import org.opensearch.knn.index.engine.ResolvedMethodContext; import org.opensearch.knn.index.mapper.CompressionLevel; import org.opensearch.knn.index.mapper.Mode; -import org.opensearch.knn.index.mapper.ModeBasedResolver; +import org.opensearch.knn.index.engine.EngineResolver; import org.opensearch.knn.index.util.IndexUtil; import org.opensearch.knn.index.engine.KNNMethodContext; import org.opensearch.knn.index.VectorDataType; @@ -116,6 +118,7 @@ public TrainingModelRequest( this.preferredNodeId = preferredNodeId; this.description = description; this.vectorDataType = vectorDataType; + this.mode = mode; // Set these as defaults initially. If call wants to override them, they can use the setters. this.maximumVectorCount = Integer.MAX_VALUE; // By default, get all vectors in the index @@ -123,10 +126,6 @@ public TrainingModelRequest( // Training data size in kilobytes. By default, this is invalid (it cant have negative kb). It eventually gets // calculated in transit. A user cannot set this value directly. - this.trainingDataSizeInKB = -1; - this.mode = mode; - this.compressionLevel = compressionLevel; - this.knnMethodConfigContext = KNNMethodConfigContext.builder() .vectorDataType(vectorDataType) .dimension(dimension) @@ -135,11 +134,11 @@ public TrainingModelRequest( .mode(mode) .build(); - if (knnMethodContext == null && (Mode.isConfigured(mode) || CompressionLevel.isConfigured(compressionLevel))) { - this.knnMethodContext = ModeBasedResolver.INSTANCE.resolveKNNMethodContext(mode, compressionLevel, true, spaceType); - } else { - this.knnMethodContext = knnMethodContext; - } + KNNEngine knnEngine = EngineResolver.INSTANCE.resolveEngine(knnMethodConfigContext, knnMethodContext, true); + ResolvedMethodContext resolvedMethodContext = knnEngine.resolveMethod(knnMethodContext, knnMethodConfigContext, true, spaceType); + this.knnMethodContext = resolvedMethodContext.getKnnMethodContext(); + this.compressionLevel = resolvedMethodContext.getCompressionLevel(); + this.knnMethodConfigContext.setCompressionLevel(resolvedMethodContext.getCompressionLevel()); } /** diff --git a/src/test/java/org/opensearch/knn/KNNTestCase.java b/src/test/java/org/opensearch/knn/KNNTestCase.java index 6ef7373d2..21b3298be 100644 --- a/src/test/java/org/opensearch/knn/KNNTestCase.java +++ b/src/test/java/org/opensearch/knn/KNNTestCase.java @@ -35,6 +35,7 @@ import static org.mockito.Mockito.when; import static org.opensearch.knn.common.KNNConstants.METHOD_HNSW; +import static org.opensearch.knn.common.KNNConstants.METHOD_IVF; /** * Base class for integration tests for KNN plugin. Contains several methods for testing KNN ES functionality. @@ -106,6 +107,11 @@ public static KNNMethodContext getDefaultKNNMethodContext() { return new KNNMethodContext(KNNEngine.DEFAULT, SpaceType.DEFAULT, methodComponentContext); } + public static KNNMethodContext getDefaultKNNMethodContextForModel() { + MethodComponentContext methodComponentContext = new MethodComponentContext(METHOD_IVF, Collections.emptyMap()); + return new KNNMethodContext(KNNEngine.FAISS, SpaceType.DEFAULT, methodComponentContext); + } + public static KNNMethodContext getDefaultByteKNNMethodContext() { MethodComponentContext methodComponentContext = new MethodComponentContext(METHOD_HNSW, Collections.emptyMap()); return new KNNMethodContext(KNNEngine.DEFAULT, SpaceType.DEFAULT, methodComponentContext); diff --git a/src/test/java/org/opensearch/knn/index/engine/AbstractKNNLibraryTests.java b/src/test/java/org/opensearch/knn/index/engine/AbstractKNNLibraryTests.java index ccaeb19a5..95d4b68a5 100644 --- a/src/test/java/org/opensearch/knn/index/engine/AbstractKNNLibraryTests.java +++ b/src/test/java/org/opensearch/knn/index/engine/AbstractKNNLibraryTests.java @@ -168,5 +168,15 @@ public Boolean isInitialized() { public void setInitialized(Boolean isInitialized) { } + + @Override + public ResolvedMethodContext resolveMethod( + KNNMethodContext knnMethodContext, + KNNMethodConfigContext knnMethodConfigContext, + boolean shouldRequireTraining, + SpaceType spaceType + ) { + return null; + } } } diff --git a/src/test/java/org/opensearch/knn/index/engine/AbstractMethodResolverTests.java b/src/test/java/org/opensearch/knn/index/engine/AbstractMethodResolverTests.java new file mode 100644 index 000000000..f21459246 --- /dev/null +++ b/src/test/java/org/opensearch/knn/index/engine/AbstractMethodResolverTests.java @@ -0,0 +1,158 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.engine; + +import org.opensearch.knn.KNNTestCase; +import org.opensearch.knn.index.SpaceType; +import org.opensearch.knn.index.VectorDataType; +import org.opensearch.knn.index.mapper.CompressionLevel; +import org.opensearch.knn.index.mapper.Mode; + +import java.util.Map; + +import static org.opensearch.knn.common.KNNConstants.METHOD_ENCODER_PARAMETER; +import static org.opensearch.knn.common.KNNConstants.METHOD_HNSW; + +public class AbstractMethodResolverTests extends KNNTestCase { + + private final static String ENCODER_NAME = "test"; + private final static CompressionLevel DEFAULT_COMPRESSION = CompressionLevel.x8; + + private final static AbstractMethodResolver TEST_RESOLVER = new AbstractMethodResolver() { + @Override + public ResolvedMethodContext resolveMethod( + KNNMethodContext knnMethodContext, + KNNMethodConfigContext knnMethodConfigContext, + boolean shouldRequireTraining, + SpaceType spaceType + ) { + return null; + } + }; + + private final static Encoder TEST_ENCODER = new Encoder() { + + @Override + public MethodComponent getMethodComponent() { + return MethodComponent.Builder.builder(ENCODER_NAME).build(); + } + + @Override + public CompressionLevel calculateCompressionLevel( + MethodComponentContext encoderContext, + KNNMethodConfigContext knnMethodConfigContext + ) { + return DEFAULT_COMPRESSION; + } + }; + + private final static Map ENCODER_MAP = Map.of(ENCODER_NAME, TEST_ENCODER); + + public void testResolveCompressionLevelFromMethodContext() { + assertEquals( + CompressionLevel.x1, + TEST_RESOLVER.resolveCompressionLevelFromMethodContext( + new KNNMethodContext(KNNEngine.DEFAULT, SpaceType.DEFAULT, MethodComponentContext.EMPTY), + KNNMethodConfigContext.builder().build(), + ENCODER_MAP + ) + ); + assertEquals( + DEFAULT_COMPRESSION, + TEST_RESOLVER.resolveCompressionLevelFromMethodContext( + new KNNMethodContext( + KNNEngine.DEFAULT, + SpaceType.DEFAULT, + new MethodComponentContext( + METHOD_HNSW, + Map.of(METHOD_ENCODER_PARAMETER, new MethodComponentContext(ENCODER_NAME, Map.of())) + ) + ), + KNNMethodConfigContext.builder().build(), + ENCODER_MAP + ) + ); + } + + public void testIsEncoderSpecified() { + assertFalse(TEST_RESOLVER.isEncoderSpecified(null)); + assertFalse( + TEST_RESOLVER.isEncoderSpecified(new KNNMethodContext(KNNEngine.DEFAULT, SpaceType.DEFAULT, MethodComponentContext.EMPTY)) + ); + assertFalse( + TEST_RESOLVER.isEncoderSpecified( + new KNNMethodContext(KNNEngine.DEFAULT, SpaceType.DEFAULT, new MethodComponentContext(METHOD_HNSW, Map.of())) + ) + ); + assertTrue( + TEST_RESOLVER.isEncoderSpecified( + new KNNMethodContext( + KNNEngine.DEFAULT, + SpaceType.DEFAULT, + new MethodComponentContext(METHOD_HNSW, Map.of(METHOD_ENCODER_PARAMETER, "test")) + ) + ) + ); + } + + public void testShouldEncoderBeResolved() { + assertFalse( + TEST_RESOLVER.shouldEncoderBeResolved( + new KNNMethodContext( + KNNEngine.DEFAULT, + SpaceType.DEFAULT, + new MethodComponentContext(METHOD_HNSW, Map.of(METHOD_ENCODER_PARAMETER, "test")) + ), + KNNMethodConfigContext.builder().build() + ) + ); + assertFalse( + TEST_RESOLVER.shouldEncoderBeResolved(null, KNNMethodConfigContext.builder().compressionLevel(CompressionLevel.x1).build()) + ); + assertFalse( + TEST_RESOLVER.shouldEncoderBeResolved( + null, + KNNMethodConfigContext.builder().compressionLevel(CompressionLevel.x1).mode(Mode.ON_DISK).build() + ) + ); + assertFalse( + TEST_RESOLVER.shouldEncoderBeResolved( + null, + KNNMethodConfigContext.builder().compressionLevel(CompressionLevel.NOT_CONFIGURED).mode(Mode.IN_MEMORY).build() + ) + ); + assertFalse( + TEST_RESOLVER.shouldEncoderBeResolved( + null, + KNNMethodConfigContext.builder() + .compressionLevel(CompressionLevel.NOT_CONFIGURED) + .mode(Mode.ON_DISK) + .vectorDataType(VectorDataType.BINARY) + .build() + ) + ); + assertTrue( + TEST_RESOLVER.shouldEncoderBeResolved( + null, + KNNMethodConfigContext.builder() + .compressionLevel(CompressionLevel.NOT_CONFIGURED) + .mode(Mode.ON_DISK) + .vectorDataType(VectorDataType.FLOAT) + .build() + ) + ); + assertTrue( + TEST_RESOLVER.shouldEncoderBeResolved( + null, + KNNMethodConfigContext.builder() + .compressionLevel(CompressionLevel.x32) + .mode(Mode.ON_DISK) + .vectorDataType(VectorDataType.FLOAT) + .build() + ) + ); + } +} diff --git a/src/test/java/org/opensearch/knn/index/engine/EngineResolverTests.java b/src/test/java/org/opensearch/knn/index/engine/EngineResolverTests.java new file mode 100644 index 000000000..df195883a --- /dev/null +++ b/src/test/java/org/opensearch/knn/index/engine/EngineResolverTests.java @@ -0,0 +1,152 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.engine; + +import org.opensearch.knn.KNNTestCase; +import org.opensearch.knn.index.SpaceType; +import org.opensearch.knn.index.mapper.CompressionLevel; +import org.opensearch.knn.index.mapper.Mode; + +public class EngineResolverTests extends KNNTestCase { + + private static final EngineResolver ENGINE_RESOLVER = EngineResolver.INSTANCE; + + public void testResolveEngine_whenEngineSpecifiedInMethod_thenThatEngine() { + assertEquals( + KNNEngine.LUCENE, + ENGINE_RESOLVER.resolveEngine( + KNNMethodConfigContext.builder().build(), + new KNNMethodContext(KNNEngine.LUCENE, SpaceType.DEFAULT, MethodComponentContext.EMPTY), + false + ) + ); + } + + public void testResolveEngine_whenRequiresTraining_thenFaiss() { + assertEquals(KNNEngine.FAISS, ENGINE_RESOLVER.resolveEngine(KNNMethodConfigContext.builder().build(), null, true)); + } + + public void testResolveEngine_whenModeAndCompressionAreFalse_thenDefault() { + assertEquals(KNNEngine.DEFAULT, ENGINE_RESOLVER.resolveEngine(KNNMethodConfigContext.builder().build(), null, false)); + assertEquals( + KNNEngine.DEFAULT, + ENGINE_RESOLVER.resolveEngine( + KNNMethodConfigContext.builder().build(), + new KNNMethodContext(KNNEngine.DEFAULT, SpaceType.UNDEFINED, MethodComponentContext.EMPTY, false), + false + ) + ); + } + + public void testResolveEngine_whenModeSpecifiedAndCompressionIsNotSpecified_thenDefault() { + assertEquals(KNNEngine.DEFAULT, ENGINE_RESOLVER.resolveEngine(KNNMethodConfigContext.builder().build(), null, false)); + assertEquals( + KNNEngine.DEFAULT, + ENGINE_RESOLVER.resolveEngine( + KNNMethodConfigContext.builder().mode(Mode.IN_MEMORY).build(), + new KNNMethodContext(KNNEngine.DEFAULT, SpaceType.UNDEFINED, MethodComponentContext.EMPTY, false), + false + ) + ); + } + + public void testResolveEngine_whenCompressionIs1x_thenEngineBasedOnMode() { + assertEquals( + KNNEngine.FAISS, + ENGINE_RESOLVER.resolveEngine( + KNNMethodConfigContext.builder().mode(Mode.ON_DISK).compressionLevel(CompressionLevel.x1).build(), + null, + false + ) + ); + assertEquals( + KNNEngine.DEFAULT, + ENGINE_RESOLVER.resolveEngine(KNNMethodConfigContext.builder().compressionLevel(CompressionLevel.x1).build(), null, false) + ); + } + + public void testResolveEngine_whenCompressionIs4x_thenEngineIsLucene() { + assertEquals( + KNNEngine.LUCENE, + ENGINE_RESOLVER.resolveEngine( + KNNMethodConfigContext.builder().mode(Mode.ON_DISK).compressionLevel(CompressionLevel.x4).build(), + null, + false + ) + ); + assertEquals( + KNNEngine.LUCENE, + ENGINE_RESOLVER.resolveEngine(KNNMethodConfigContext.builder().compressionLevel(CompressionLevel.x4).build(), null, false) + ); + } + + public void testResolveEngine_whenConfiguredForBQ_thenEngineIsFaiss() { + assertEquals( + KNNEngine.FAISS, + ENGINE_RESOLVER.resolveEngine( + KNNMethodConfigContext.builder().mode(Mode.ON_DISK).compressionLevel(CompressionLevel.x2).build(), + null, + false + ) + ); + assertEquals( + KNNEngine.FAISS, + ENGINE_RESOLVER.resolveEngine( + KNNMethodConfigContext.builder().mode(Mode.IN_MEMORY).compressionLevel(CompressionLevel.x2).build(), + null, + false + ) + ); + assertEquals( + KNNEngine.FAISS, + ENGINE_RESOLVER.resolveEngine( + KNNMethodConfigContext.builder().mode(Mode.ON_DISK).compressionLevel(CompressionLevel.x8).build(), + null, + false + ) + ); + assertEquals( + KNNEngine.FAISS, + ENGINE_RESOLVER.resolveEngine( + KNNMethodConfigContext.builder().mode(Mode.IN_MEMORY).compressionLevel(CompressionLevel.x8).build(), + null, + false + ) + ); + assertEquals( + KNNEngine.FAISS, + ENGINE_RESOLVER.resolveEngine( + KNNMethodConfigContext.builder().mode(Mode.ON_DISK).compressionLevel(CompressionLevel.x16).build(), + null, + false + ) + ); + assertEquals( + KNNEngine.FAISS, + ENGINE_RESOLVER.resolveEngine( + KNNMethodConfigContext.builder().mode(Mode.IN_MEMORY).compressionLevel(CompressionLevel.x16).build(), + null, + false + ) + ); + assertEquals( + KNNEngine.FAISS, + ENGINE_RESOLVER.resolveEngine( + KNNMethodConfigContext.builder().mode(Mode.ON_DISK).compressionLevel(CompressionLevel.x32).build(), + null, + false + ) + ); + assertEquals( + KNNEngine.FAISS, + ENGINE_RESOLVER.resolveEngine( + KNNMethodConfigContext.builder().mode(Mode.IN_MEMORY).compressionLevel(CompressionLevel.x32).build(), + null, + false + ) + ); + } +} diff --git a/src/test/java/org/opensearch/knn/index/engine/NativeLibraryTests.java b/src/test/java/org/opensearch/knn/index/engine/NativeLibraryTests.java index 243e9a3c1..c1fbe4aa5 100644 --- a/src/test/java/org/opensearch/knn/index/engine/NativeLibraryTests.java +++ b/src/test/java/org/opensearch/knn/index/engine/NativeLibraryTests.java @@ -73,5 +73,15 @@ public Float distanceToRadialThreshold(Float distance, SpaceType spaceType) { public Float scoreToRadialThreshold(Float score, SpaceType spaceType) { return 0.0f; } + + @Override + public ResolvedMethodContext resolveMethod( + KNNMethodContext knnMethodContext, + KNNMethodConfigContext knnMethodConfigContext, + boolean shouldRequireTraining, + SpaceType spaceType + ) { + return null; + } } } diff --git a/src/test/java/org/opensearch/knn/index/engine/SpaceTypeResolverTests.java b/src/test/java/org/opensearch/knn/index/engine/SpaceTypeResolverTests.java new file mode 100644 index 000000000..99fc98c9e --- /dev/null +++ b/src/test/java/org/opensearch/knn/index/engine/SpaceTypeResolverTests.java @@ -0,0 +1,99 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.engine; + +import lombok.SneakyThrows; +import org.opensearch.index.mapper.MapperParsingException; +import org.opensearch.knn.KNNTestCase; +import org.opensearch.knn.index.SpaceType; +import org.opensearch.knn.index.VectorDataType; + +public class SpaceTypeResolverTests extends KNNTestCase { + + private static final SpaceTypeResolver SPACE_TYPE_RESOLVER = SpaceTypeResolver.INSTANCE; + + public void testResolveSpaceType_whenNoConfigProvided_thenFallbackToVectorDataType() { + assertEquals(SpaceType.DEFAULT, SPACE_TYPE_RESOLVER.resolveSpaceType(null, VectorDataType.FLOAT, "")); + assertEquals(SpaceType.DEFAULT, SPACE_TYPE_RESOLVER.resolveSpaceType(null, VectorDataType.BYTE, "")); + assertEquals( + SpaceType.DEFAULT, + SPACE_TYPE_RESOLVER.resolveSpaceType( + new KNNMethodContext(KNNEngine.DEFAULT, SpaceType.UNDEFINED, MethodComponentContext.EMPTY), + VectorDataType.FLOAT, + "" + ) + ); + assertEquals(SpaceType.DEFAULT_BINARY, SPACE_TYPE_RESOLVER.resolveSpaceType(null, VectorDataType.BINARY, "")); + assertEquals( + SpaceType.DEFAULT_BINARY, + SPACE_TYPE_RESOLVER.resolveSpaceType( + new KNNMethodContext(KNNEngine.DEFAULT, SpaceType.UNDEFINED, MethodComponentContext.EMPTY), + VectorDataType.BINARY, + "" + ) + ); + } + + @SneakyThrows + public void testResolveSpaceType_whenMethodSpaceTypeAndTopLevelSpecified_thenThrowIfConflict() { + expectThrows( + MapperParsingException.class, + () -> SPACE_TYPE_RESOLVER.resolveSpaceType( + new KNNMethodContext(KNNEngine.DEFAULT, SpaceType.L2, MethodComponentContext.EMPTY), + VectorDataType.FLOAT, + SpaceType.INNER_PRODUCT.getValue() + ) + ); + assertEquals( + SpaceType.DEFAULT, + SPACE_TYPE_RESOLVER.resolveSpaceType( + new KNNMethodContext(KNNEngine.DEFAULT, SpaceType.DEFAULT, MethodComponentContext.EMPTY), + VectorDataType.FLOAT, + SpaceType.DEFAULT.getValue() + ) + ); + assertEquals( + SpaceType.DEFAULT, + SPACE_TYPE_RESOLVER.resolveSpaceType( + new KNNMethodContext(KNNEngine.DEFAULT, SpaceType.DEFAULT, MethodComponentContext.EMPTY), + VectorDataType.FLOAT, + SpaceType.UNDEFINED.getValue() + ) + ); + assertEquals( + SpaceType.DEFAULT, + SPACE_TYPE_RESOLVER.resolveSpaceType( + new KNNMethodContext(KNNEngine.DEFAULT, SpaceType.UNDEFINED, MethodComponentContext.EMPTY), + VectorDataType.FLOAT, + SpaceType.DEFAULT.getValue() + ) + ); + assertEquals( + SpaceType.DEFAULT, + SPACE_TYPE_RESOLVER.resolveSpaceType( + new KNNMethodContext(KNNEngine.DEFAULT, SpaceType.UNDEFINED, MethodComponentContext.EMPTY), + VectorDataType.FLOAT, + SpaceType.UNDEFINED.getValue() + ) + ); + } + + @SneakyThrows + public void testResolveSpaceType_whenSpaceTypeSpecifiedOnce_thenReturnValue() { + assertEquals( + SpaceType.L1, + SPACE_TYPE_RESOLVER.resolveSpaceType( + new KNNMethodContext(KNNEngine.DEFAULT, SpaceType.L1, MethodComponentContext.EMPTY), + VectorDataType.FLOAT, + "" + ) + ); + assertEquals( + SpaceType.INNER_PRODUCT, + SPACE_TYPE_RESOLVER.resolveSpaceType(null, VectorDataType.FLOAT, SpaceType.INNER_PRODUCT.getValue()) + ); + } +} diff --git a/src/test/java/org/opensearch/knn/index/engine/faiss/FaissHNSWPQEncoderTests.java b/src/test/java/org/opensearch/knn/index/engine/faiss/FaissHNSWPQEncoderTests.java new file mode 100644 index 000000000..3f7dd9dcd --- /dev/null +++ b/src/test/java/org/opensearch/knn/index/engine/faiss/FaissHNSWPQEncoderTests.java @@ -0,0 +1,16 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.engine.faiss; + +import org.opensearch.knn.KNNTestCase; +import org.opensearch.knn.index.mapper.CompressionLevel; + +public class FaissHNSWPQEncoderTests extends KNNTestCase { + public void testCalculateCompressionLevel() { + FaissHNSWPQEncoder encoder = new FaissHNSWPQEncoder(); + assertEquals(CompressionLevel.NOT_CONFIGURED, encoder.calculateCompressionLevel(null, null)); + } +} diff --git a/src/test/java/org/opensearch/knn/index/engine/faiss/FaissIVFPQEncoderTests.java b/src/test/java/org/opensearch/knn/index/engine/faiss/FaissIVFPQEncoderTests.java new file mode 100644 index 000000000..35b7a64ab --- /dev/null +++ b/src/test/java/org/opensearch/knn/index/engine/faiss/FaissIVFPQEncoderTests.java @@ -0,0 +1,16 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.engine.faiss; + +import org.opensearch.knn.KNNTestCase; +import org.opensearch.knn.index.mapper.CompressionLevel; + +public class FaissIVFPQEncoderTests extends KNNTestCase { + public void testCalculateCompressionLevel() { + FaissIVFPQEncoder encoder = new FaissIVFPQEncoder(); + assertEquals(CompressionLevel.NOT_CONFIGURED, encoder.calculateCompressionLevel(null, null)); + } +} diff --git a/src/test/java/org/opensearch/knn/index/engine/faiss/FaissMethodResolverTests.java b/src/test/java/org/opensearch/knn/index/engine/faiss/FaissMethodResolverTests.java new file mode 100644 index 000000000..ad466d4bb --- /dev/null +++ b/src/test/java/org/opensearch/knn/index/engine/faiss/FaissMethodResolverTests.java @@ -0,0 +1,246 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.engine.faiss; + +import org.opensearch.Version; +import org.opensearch.common.ValidationException; +import org.opensearch.knn.KNNTestCase; +import org.opensearch.knn.index.SpaceType; +import org.opensearch.knn.index.VectorDataType; +import org.opensearch.knn.index.engine.KNNEngine; +import org.opensearch.knn.index.engine.KNNMethodConfigContext; +import org.opensearch.knn.index.engine.KNNMethodContext; +import org.opensearch.knn.index.engine.MethodComponentContext; +import org.opensearch.knn.index.engine.MethodResolver; +import org.opensearch.knn.index.engine.ResolvedMethodContext; +import org.opensearch.knn.index.mapper.CompressionLevel; +import org.opensearch.knn.index.mapper.Mode; + +import java.util.Map; + +import static org.opensearch.knn.common.KNNConstants.ENCODER_FLAT; +import static org.opensearch.knn.common.KNNConstants.METHOD_ENCODER_PARAMETER; +import static org.opensearch.knn.common.KNNConstants.METHOD_HNSW; + +public class FaissMethodResolverTests extends KNNTestCase { + + MethodResolver TEST_RESOLVER = new FaissMethodResolver(); + + public void testResolveMethod_whenValid_thenResolve() { + ResolvedMethodContext resolvedMethodContext = TEST_RESOLVER.resolveMethod( + null, + KNNMethodConfigContext.builder().vectorDataType(VectorDataType.FLOAT).versionCreated(Version.CURRENT).build(), + false, + SpaceType.INNER_PRODUCT + ); + validateResolveMethodContext(resolvedMethodContext, CompressionLevel.x1, SpaceType.INNER_PRODUCT, ENCODER_FLAT, false); + + resolvedMethodContext = TEST_RESOLVER.resolveMethod( + null, + KNNMethodConfigContext.builder() + .vectorDataType(VectorDataType.FLOAT) + .mode(Mode.ON_DISK) + .versionCreated(Version.CURRENT) + .build(), + false, + SpaceType.INNER_PRODUCT + ); + validateResolveMethodContext(resolvedMethodContext, CompressionLevel.x32, SpaceType.INNER_PRODUCT, QFrameBitEncoder.NAME, true); + + resolvedMethodContext = TEST_RESOLVER.resolveMethod( + null, + KNNMethodConfigContext.builder() + .vectorDataType(VectorDataType.FLOAT) + .mode(Mode.ON_DISK) + .compressionLevel(CompressionLevel.x16) + .versionCreated(Version.CURRENT) + .build(), + false, + SpaceType.INNER_PRODUCT + ); + validateResolveMethodContext(resolvedMethodContext, CompressionLevel.x16, SpaceType.INNER_PRODUCT, QFrameBitEncoder.NAME, true); + + resolvedMethodContext = TEST_RESOLVER.resolveMethod( + null, + KNNMethodConfigContext.builder() + .vectorDataType(VectorDataType.FLOAT) + .compressionLevel(CompressionLevel.x16) + .versionCreated(Version.CURRENT) + .build(), + false, + SpaceType.INNER_PRODUCT + ); + validateResolveMethodContext(resolvedMethodContext, CompressionLevel.x16, SpaceType.INNER_PRODUCT, QFrameBitEncoder.NAME, true); + + resolvedMethodContext = TEST_RESOLVER.resolveMethod( + new KNNMethodContext( + KNNEngine.FAISS, + SpaceType.L2, + new MethodComponentContext( + METHOD_HNSW, + Map.of( + METHOD_ENCODER_PARAMETER, + new MethodComponentContext( + QFrameBitEncoder.NAME, + Map.of(QFrameBitEncoder.BITCOUNT_PARAM, CompressionLevel.x8.numBitsForFloat32()) + ) + ) + ) + ), + KNNMethodConfigContext.builder() + .vectorDataType(VectorDataType.FLOAT) + .mode(Mode.ON_DISK) + .versionCreated(Version.CURRENT) + .build(), + false, + SpaceType.L2 + ); + validateResolveMethodContext(resolvedMethodContext, CompressionLevel.x8, SpaceType.L2, QFrameBitEncoder.NAME, true); + + resolvedMethodContext = TEST_RESOLVER.resolveMethod( + new KNNMethodContext( + KNNEngine.FAISS, + SpaceType.L2, + new MethodComponentContext( + METHOD_HNSW, + Map.of( + METHOD_ENCODER_PARAMETER, + new MethodComponentContext( + QFrameBitEncoder.NAME, + Map.of(QFrameBitEncoder.BITCOUNT_PARAM, CompressionLevel.x8.numBitsForFloat32()) + ) + ) + ) + ), + KNNMethodConfigContext.builder().vectorDataType(VectorDataType.FLOAT).versionCreated(Version.CURRENT).build(), + false, + SpaceType.L2 + ); + validateResolveMethodContext(resolvedMethodContext, CompressionLevel.x8, SpaceType.L2, QFrameBitEncoder.NAME, true); + + resolvedMethodContext = TEST_RESOLVER.resolveMethod( + new KNNMethodContext(KNNEngine.FAISS, SpaceType.L2, new MethodComponentContext(METHOD_HNSW, Map.of())), + KNNMethodConfigContext.builder().vectorDataType(VectorDataType.FLOAT).versionCreated(Version.CURRENT).build(), + false, + SpaceType.L2 + ); + validateResolveMethodContext(resolvedMethodContext, CompressionLevel.x1, SpaceType.L2, ENCODER_FLAT, false); + + resolvedMethodContext = TEST_RESOLVER.resolveMethod( + new KNNMethodContext(KNNEngine.FAISS, SpaceType.L2, new MethodComponentContext(METHOD_HNSW, Map.of())), + KNNMethodConfigContext.builder().vectorDataType(VectorDataType.BINARY).versionCreated(Version.CURRENT).build(), + false, + SpaceType.L2 + ); + validateResolveMethodContext(resolvedMethodContext, CompressionLevel.x1, SpaceType.L2, ENCODER_FLAT, false); + } + + private void validateResolveMethodContext( + ResolvedMethodContext resolvedMethodContext, + CompressionLevel expectedCompression, + SpaceType expectedSpaceType, + String expectedEncoderName, + boolean checkBitsEncoderParam + ) { + assertEquals(expectedCompression, resolvedMethodContext.getCompressionLevel()); + assertEquals(KNNEngine.FAISS, resolvedMethodContext.getKnnMethodContext().getKnnEngine()); + assertEquals(expectedSpaceType, resolvedMethodContext.getKnnMethodContext().getSpaceType()); + assertEquals( + expectedEncoderName, + ((MethodComponentContext) resolvedMethodContext.getKnnMethodContext() + .getMethodComponentContext() + .getParameters() + .get(METHOD_ENCODER_PARAMETER)).getName() + ); + if (checkBitsEncoderParam) { + assertEquals( + expectedCompression.numBitsForFloat32(), + ((MethodComponentContext) resolvedMethodContext.getKnnMethodContext() + .getMethodComponentContext() + .getParameters() + .get(METHOD_ENCODER_PARAMETER)).getParameters().get(QFrameBitEncoder.BITCOUNT_PARAM) + ); + } + + } + + public void testResolveMethod_whenInvalid_thenThrow() { + // Invalid compression + expectThrows( + ValidationException.class, + () -> TEST_RESOLVER.resolveMethod( + null, + KNNMethodConfigContext.builder() + .vectorDataType(VectorDataType.FLOAT) + .compressionLevel(CompressionLevel.x4) + .versionCreated(Version.CURRENT) + .build(), + false, + SpaceType.L2 + ) + ); + + expectThrows( + ValidationException.class, + () -> TEST_RESOLVER.resolveMethod( + null, + KNNMethodConfigContext.builder() + .vectorDataType(VectorDataType.BINARY) + .compressionLevel(CompressionLevel.x4) + .versionCreated(Version.CURRENT) + .build(), + false, + SpaceType.L2 + ) + ); + + // Invalid spec ondisk and compression is 1 + expectThrows( + ValidationException.class, + () -> TEST_RESOLVER.resolveMethod( + null, + KNNMethodConfigContext.builder() + .vectorDataType(VectorDataType.FLOAT) + .mode(Mode.ON_DISK) + .compressionLevel(CompressionLevel.x1) + .versionCreated(Version.CURRENT) + .build(), + false, + SpaceType.L2 + ) + ); + + // Invalid compression conflict + expectThrows( + ValidationException.class, + () -> TEST_RESOLVER.resolveMethod( + new KNNMethodContext( + KNNEngine.FAISS, + SpaceType.INNER_PRODUCT, + new MethodComponentContext( + METHOD_HNSW, + Map.of( + METHOD_ENCODER_PARAMETER, + new MethodComponentContext( + QFrameBitEncoder.NAME, + Map.of(QFrameBitEncoder.BITCOUNT_PARAM, CompressionLevel.x32.numBitsForFloat32()) + ) + ) + ) + ), + KNNMethodConfigContext.builder() + .vectorDataType(VectorDataType.FLOAT) + .mode(Mode.ON_DISK) + .compressionLevel(CompressionLevel.x8) + .versionCreated(Version.CURRENT) + .build(), + false, + SpaceType.INNER_PRODUCT + ) + + ); + } +} diff --git a/src/test/java/org/opensearch/knn/index/engine/faiss/FaissSQEncoderTests.java b/src/test/java/org/opensearch/knn/index/engine/faiss/FaissSQEncoderTests.java new file mode 100644 index 000000000..3905158a2 --- /dev/null +++ b/src/test/java/org/opensearch/knn/index/engine/faiss/FaissSQEncoderTests.java @@ -0,0 +1,16 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.engine.faiss; + +import org.opensearch.knn.KNNTestCase; +import org.opensearch.knn.index.mapper.CompressionLevel; + +public class FaissSQEncoderTests extends KNNTestCase { + public void testCalculateCompressionLevel() { + FaissSQEncoder encoder = new FaissSQEncoder(); + assertEquals(CompressionLevel.x2, encoder.calculateCompressionLevel(null, null)); + } +} diff --git a/src/test/java/org/opensearch/knn/index/engine/faiss/QFrameBitEncoderTests.java b/src/test/java/org/opensearch/knn/index/engine/faiss/QFrameBitEncoderTests.java index 7457b49aa..e926916af 100644 --- a/src/test/java/org/opensearch/knn/index/engine/faiss/QFrameBitEncoderTests.java +++ b/src/test/java/org/opensearch/knn/index/engine/faiss/QFrameBitEncoderTests.java @@ -7,6 +7,7 @@ import com.google.common.collect.ImmutableMap; import org.opensearch.Version; +import org.opensearch.common.ValidationException; import org.opensearch.knn.KNNTestCase; import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.engine.KNNLibraryIndexingContext; @@ -14,10 +15,16 @@ import org.opensearch.knn.index.engine.MethodComponent; import org.opensearch.knn.index.engine.MethodComponentContext; import org.opensearch.knn.index.engine.qframe.QuantizationConfig; +import org.opensearch.knn.index.mapper.CompressionLevel; import org.opensearch.knn.quantization.enums.ScalarQuantizationType; +import java.util.HashMap; +import java.util.Map; + import static org.opensearch.knn.common.KNNConstants.FAISS_FLAT_DESCRIPTION; import static org.opensearch.knn.common.KNNConstants.INDEX_DESCRIPTION_PARAMETER; +import static org.opensearch.knn.common.KNNConstants.METHOD_ENCODER_PARAMETER; +import static org.opensearch.knn.common.KNNConstants.METHOD_HNSW; import static org.opensearch.knn.index.engine.faiss.QFrameBitEncoder.BITCOUNT_PARAM; public class QFrameBitEncoderTests extends KNNTestCase { @@ -121,4 +128,40 @@ public void testEstimateOverheadInKB() { .estimateOverheadInKB(new MethodComponentContext(QFrameBitEncoder.NAME, ImmutableMap.of(BITCOUNT_PARAM, 4)), 8) ); } + + public void testCalculateCompressionLevel() { + QFrameBitEncoder encoder = new QFrameBitEncoder(); + assertEquals( + CompressionLevel.x32, + encoder.calculateCompressionLevel(generateMethodComponentContext(CompressionLevel.x32.numBitsForFloat32()), null) + ); + assertEquals( + CompressionLevel.x16, + encoder.calculateCompressionLevel(generateMethodComponentContext(CompressionLevel.x16.numBitsForFloat32()), null) + ); + assertEquals( + CompressionLevel.x8, + encoder.calculateCompressionLevel(generateMethodComponentContext(CompressionLevel.x8.numBitsForFloat32()), null) + ); + assertEquals( + CompressionLevel.NOT_CONFIGURED, + encoder.calculateCompressionLevel( + new MethodComponentContext( + METHOD_HNSW, + new HashMap<>(Map.of(METHOD_ENCODER_PARAMETER, new MethodComponentContext(QFrameBitEncoder.NAME, Map.of()))) + ), + null + ) + ); + + expectThrows( + ValidationException.class, + () -> encoder.calculateCompressionLevel(generateMethodComponentContext(CompressionLevel.x4.numBitsForFloat32()), null) + ); + expectThrows(ValidationException.class, () -> encoder.calculateCompressionLevel(generateMethodComponentContext(-1), null)); + } + + private MethodComponentContext generateMethodComponentContext(int bitCount) { + return new MethodComponentContext(QFrameBitEncoder.NAME, Map.of(BITCOUNT_PARAM, bitCount)); + } } diff --git a/src/test/java/org/opensearch/knn/index/engine/lucene/LuceneMethodResolverTests.java b/src/test/java/org/opensearch/knn/index/engine/lucene/LuceneMethodResolverTests.java new file mode 100644 index 000000000..833d83135 --- /dev/null +++ b/src/test/java/org/opensearch/knn/index/engine/lucene/LuceneMethodResolverTests.java @@ -0,0 +1,212 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.engine.lucene; + +import org.opensearch.Version; +import org.opensearch.common.ValidationException; +import org.opensearch.knn.KNNTestCase; +import org.opensearch.knn.index.SpaceType; +import org.opensearch.knn.index.VectorDataType; +import org.opensearch.knn.index.engine.KNNEngine; +import org.opensearch.knn.index.engine.KNNMethodConfigContext; +import org.opensearch.knn.index.engine.KNNMethodContext; +import org.opensearch.knn.index.engine.MethodComponentContext; +import org.opensearch.knn.index.engine.MethodResolver; +import org.opensearch.knn.index.engine.ResolvedMethodContext; +import org.opensearch.knn.index.mapper.CompressionLevel; +import org.opensearch.knn.index.mapper.Mode; + +import java.util.Map; + +import static org.opensearch.knn.common.KNNConstants.ENCODER_SQ; +import static org.opensearch.knn.common.KNNConstants.METHOD_ENCODER_PARAMETER; +import static org.opensearch.knn.common.KNNConstants.METHOD_HNSW; + +public class LuceneMethodResolverTests extends KNNTestCase { + MethodResolver TEST_RESOLVER = new LuceneMethodResolver(); + + public void testResolveMethod_whenValid_thenResolve() { + ResolvedMethodContext resolvedMethodContext = TEST_RESOLVER.resolveMethod( + null, + KNNMethodConfigContext.builder().vectorDataType(VectorDataType.FLOAT).versionCreated(Version.CURRENT).build(), + false, + SpaceType.INNER_PRODUCT + ); + assertEquals(METHOD_HNSW, resolvedMethodContext.getKnnMethodContext().getMethodComponentContext().getName()); + assertFalse(resolvedMethodContext.getKnnMethodContext().getMethodComponentContext().getParameters().isEmpty()); + assertEquals(KNNEngine.LUCENE, resolvedMethodContext.getKnnMethodContext().getKnnEngine()); + assertEquals(SpaceType.INNER_PRODUCT, resolvedMethodContext.getKnnMethodContext().getSpaceType()); + assertEquals(CompressionLevel.x1, resolvedMethodContext.getCompressionLevel()); + + resolvedMethodContext = TEST_RESOLVER.resolveMethod( + null, + KNNMethodConfigContext.builder() + .vectorDataType(VectorDataType.FLOAT) + .versionCreated(Version.CURRENT) + .mode(Mode.ON_DISK) + .build(), + false, + SpaceType.INNER_PRODUCT + ); + assertEquals(METHOD_HNSW, resolvedMethodContext.getKnnMethodContext().getMethodComponentContext().getName()); + assertFalse(resolvedMethodContext.getKnnMethodContext().getMethodComponentContext().getParameters().isEmpty()); + assertTrue( + resolvedMethodContext.getKnnMethodContext().getMethodComponentContext().getParameters().containsKey(METHOD_ENCODER_PARAMETER) + ); + assertEquals(KNNEngine.LUCENE, resolvedMethodContext.getKnnMethodContext().getKnnEngine()); + assertEquals(SpaceType.INNER_PRODUCT, resolvedMethodContext.getKnnMethodContext().getSpaceType()); + assertEquals(CompressionLevel.x4, resolvedMethodContext.getCompressionLevel()); + + resolvedMethodContext = TEST_RESOLVER.resolveMethod( + null, + KNNMethodConfigContext.builder() + .vectorDataType(VectorDataType.FLOAT) + .versionCreated(Version.CURRENT) + .compressionLevel(CompressionLevel.x4) + .build(), + false, + SpaceType.INNER_PRODUCT + ); + assertEquals(METHOD_HNSW, resolvedMethodContext.getKnnMethodContext().getMethodComponentContext().getName()); + assertFalse(resolvedMethodContext.getKnnMethodContext().getMethodComponentContext().getParameters().isEmpty()); + assertTrue( + resolvedMethodContext.getKnnMethodContext().getMethodComponentContext().getParameters().containsKey(METHOD_ENCODER_PARAMETER) + ); + assertEquals(KNNEngine.LUCENE, resolvedMethodContext.getKnnMethodContext().getKnnEngine()); + assertEquals(SpaceType.INNER_PRODUCT, resolvedMethodContext.getKnnMethodContext().getSpaceType()); + assertEquals(CompressionLevel.x4, resolvedMethodContext.getCompressionLevel()); + + KNNMethodContext knnMethodContext = new KNNMethodContext( + KNNEngine.LUCENE, + SpaceType.INNER_PRODUCT, + new MethodComponentContext(METHOD_HNSW, Map.of()) + ); + resolvedMethodContext = TEST_RESOLVER.resolveMethod( + knnMethodContext, + KNNMethodConfigContext.builder() + .vectorDataType(VectorDataType.FLOAT) + .versionCreated(Version.CURRENT) + .mode(Mode.ON_DISK) + .build(), + false, + SpaceType.INNER_PRODUCT + ); + assertEquals(METHOD_HNSW, resolvedMethodContext.getKnnMethodContext().getMethodComponentContext().getName()); + assertFalse(resolvedMethodContext.getKnnMethodContext().getMethodComponentContext().getParameters().isEmpty()); + assertTrue( + resolvedMethodContext.getKnnMethodContext().getMethodComponentContext().getParameters().containsKey(METHOD_ENCODER_PARAMETER) + ); + assertEquals(KNNEngine.LUCENE, resolvedMethodContext.getKnnMethodContext().getKnnEngine()); + assertEquals(SpaceType.INNER_PRODUCT, resolvedMethodContext.getKnnMethodContext().getSpaceType()); + assertEquals(CompressionLevel.x4, resolvedMethodContext.getCompressionLevel()); + assertNotEquals(knnMethodContext, resolvedMethodContext.getKnnMethodContext()); + + knnMethodContext = new KNNMethodContext( + KNNEngine.LUCENE, + SpaceType.INNER_PRODUCT, + new MethodComponentContext(METHOD_HNSW, Map.of()) + ); + resolvedMethodContext = TEST_RESOLVER.resolveMethod( + knnMethodContext, + KNNMethodConfigContext.builder() + .vectorDataType(VectorDataType.FLOAT) + .versionCreated(Version.CURRENT) + .compressionLevel(CompressionLevel.x4) + .build(), + false, + SpaceType.INNER_PRODUCT + ); + assertEquals(METHOD_HNSW, resolvedMethodContext.getKnnMethodContext().getMethodComponentContext().getName()); + assertFalse(resolvedMethodContext.getKnnMethodContext().getMethodComponentContext().getParameters().isEmpty()); + assertTrue( + resolvedMethodContext.getKnnMethodContext().getMethodComponentContext().getParameters().containsKey(METHOD_ENCODER_PARAMETER) + ); + assertEquals(KNNEngine.LUCENE, resolvedMethodContext.getKnnMethodContext().getKnnEngine()); + assertEquals(SpaceType.INNER_PRODUCT, resolvedMethodContext.getKnnMethodContext().getSpaceType()); + assertEquals(CompressionLevel.x4, resolvedMethodContext.getCompressionLevel()); + assertNotEquals(knnMethodContext, resolvedMethodContext.getKnnMethodContext()); + + knnMethodContext = new KNNMethodContext( + KNNEngine.LUCENE, + SpaceType.INNER_PRODUCT, + new MethodComponentContext(METHOD_HNSW, Map.of(METHOD_ENCODER_PARAMETER, new MethodComponentContext(ENCODER_SQ, Map.of()))) + ); + resolvedMethodContext = TEST_RESOLVER.resolveMethod( + knnMethodContext, + KNNMethodConfigContext.builder().vectorDataType(VectorDataType.FLOAT).versionCreated(Version.CURRENT).build(), + false, + SpaceType.INNER_PRODUCT + ); + assertEquals(METHOD_HNSW, resolvedMethodContext.getKnnMethodContext().getMethodComponentContext().getName()); + assertFalse(resolvedMethodContext.getKnnMethodContext().getMethodComponentContext().getParameters().isEmpty()); + assertTrue( + resolvedMethodContext.getKnnMethodContext().getMethodComponentContext().getParameters().containsKey(METHOD_ENCODER_PARAMETER) + ); + assertEquals(KNNEngine.LUCENE, resolvedMethodContext.getKnnMethodContext().getKnnEngine()); + assertEquals(SpaceType.INNER_PRODUCT, resolvedMethodContext.getKnnMethodContext().getSpaceType()); + assertEquals(CompressionLevel.x4, resolvedMethodContext.getCompressionLevel()); + assertNotEquals(knnMethodContext, resolvedMethodContext.getKnnMethodContext()); + + resolvedMethodContext = TEST_RESOLVER.resolveMethod( + null, + KNNMethodConfigContext.builder().vectorDataType(VectorDataType.BYTE).versionCreated(Version.CURRENT).build(), + false, + SpaceType.INNER_PRODUCT + ); + assertEquals(METHOD_HNSW, resolvedMethodContext.getKnnMethodContext().getMethodComponentContext().getName()); + assertFalse(resolvedMethodContext.getKnnMethodContext().getMethodComponentContext().getParameters().isEmpty()); + assertFalse( + resolvedMethodContext.getKnnMethodContext().getMethodComponentContext().getParameters().containsKey(METHOD_ENCODER_PARAMETER) + ); + assertEquals(KNNEngine.LUCENE, resolvedMethodContext.getKnnMethodContext().getKnnEngine()); + assertEquals(SpaceType.INNER_PRODUCT, resolvedMethodContext.getKnnMethodContext().getSpaceType()); + assertEquals(CompressionLevel.x1, resolvedMethodContext.getCompressionLevel()); + } + + public void testResolveMethod_whenInvalid_thenThrow() { + // Invalid training context + expectThrows( + ValidationException.class, + () -> TEST_RESOLVER.resolveMethod( + null, + KNNMethodConfigContext.builder().vectorDataType(VectorDataType.FLOAT).versionCreated(Version.CURRENT).build(), + true, + SpaceType.L2 + ) + ); + + // Invalid compression + expectThrows( + ValidationException.class, + () -> TEST_RESOLVER.resolveMethod( + null, + KNNMethodConfigContext.builder() + .vectorDataType(VectorDataType.FLOAT) + .compressionLevel(CompressionLevel.x32) + .versionCreated(Version.CURRENT) + .build(), + false, + SpaceType.L2 + ) + ); + + // Invalid spec ondisk and compression is 1 + expectThrows( + ValidationException.class, + () -> TEST_RESOLVER.resolveMethod( + null, + KNNMethodConfigContext.builder() + .vectorDataType(VectorDataType.FLOAT) + .mode(Mode.ON_DISK) + .compressionLevel(CompressionLevel.x1) + .versionCreated(Version.CURRENT) + .build(), + false, + SpaceType.L2 + ) + ); + } +} diff --git a/src/test/java/org/opensearch/knn/index/engine/lucene/LuceneSQEncoderTests.java b/src/test/java/org/opensearch/knn/index/engine/lucene/LuceneSQEncoderTests.java new file mode 100644 index 000000000..139f96e8b --- /dev/null +++ b/src/test/java/org/opensearch/knn/index/engine/lucene/LuceneSQEncoderTests.java @@ -0,0 +1,16 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.engine.lucene; + +import org.opensearch.knn.KNNTestCase; +import org.opensearch.knn.index.mapper.CompressionLevel; + +public class LuceneSQEncoderTests extends KNNTestCase { + public void testCalculateCompressionLevel() { + LuceneSQEncoder encoder = new LuceneSQEncoder(); + assertEquals(CompressionLevel.x4, encoder.calculateCompressionLevel(null, null)); + } +} diff --git a/src/test/java/org/opensearch/knn/index/engine/nmslib/NmslibMethodResolverTests.java b/src/test/java/org/opensearch/knn/index/engine/nmslib/NmslibMethodResolverTests.java new file mode 100644 index 000000000..065e0e378 --- /dev/null +++ b/src/test/java/org/opensearch/knn/index/engine/nmslib/NmslibMethodResolverTests.java @@ -0,0 +1,106 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.engine.nmslib; + +import org.opensearch.Version; +import org.opensearch.common.ValidationException; +import org.opensearch.knn.KNNTestCase; +import org.opensearch.knn.index.SpaceType; +import org.opensearch.knn.index.VectorDataType; +import org.opensearch.knn.index.engine.KNNEngine; +import org.opensearch.knn.index.engine.KNNMethodConfigContext; +import org.opensearch.knn.index.engine.KNNMethodContext; +import org.opensearch.knn.index.engine.MethodComponentContext; +import org.opensearch.knn.index.engine.MethodResolver; +import org.opensearch.knn.index.engine.ResolvedMethodContext; +import org.opensearch.knn.index.mapper.CompressionLevel; +import org.opensearch.knn.index.mapper.Mode; + +import java.util.Map; + +import static org.opensearch.knn.common.KNNConstants.METHOD_HNSW; + +public class NmslibMethodResolverTests extends KNNTestCase { + + MethodResolver TEST_RESOLVER = new NmslibMethodResolver(); + + public void testResolveMethod_whenValid_thenResolve() { + // No configuration passed in + ResolvedMethodContext resolvedMethodContext = TEST_RESOLVER.resolveMethod( + null, + KNNMethodConfigContext.builder().vectorDataType(VectorDataType.FLOAT).versionCreated(Version.CURRENT).build(), + false, + SpaceType.INNER_PRODUCT + ); + assertEquals(METHOD_HNSW, resolvedMethodContext.getKnnMethodContext().getMethodComponentContext().getName()); + assertFalse(resolvedMethodContext.getKnnMethodContext().getMethodComponentContext().getParameters().isEmpty()); + assertEquals(KNNEngine.NMSLIB, resolvedMethodContext.getKnnMethodContext().getKnnEngine()); + assertEquals(SpaceType.INNER_PRODUCT, resolvedMethodContext.getKnnMethodContext().getSpaceType()); + assertEquals(CompressionLevel.x1, resolvedMethodContext.getCompressionLevel()); + + KNNMethodContext knnMethodContext = new KNNMethodContext( + KNNEngine.NMSLIB, + SpaceType.INNER_PRODUCT, + new MethodComponentContext(METHOD_HNSW, Map.of()) + ); + + resolvedMethodContext = TEST_RESOLVER.resolveMethod( + knnMethodContext, + KNNMethodConfigContext.builder().vectorDataType(VectorDataType.FLOAT).versionCreated(Version.CURRENT).build(), + false, + SpaceType.INNER_PRODUCT + ); + assertEquals(METHOD_HNSW, resolvedMethodContext.getKnnMethodContext().getMethodComponentContext().getName()); + assertFalse(resolvedMethodContext.getKnnMethodContext().getMethodComponentContext().getParameters().isEmpty()); + assertEquals(KNNEngine.NMSLIB, resolvedMethodContext.getKnnMethodContext().getKnnEngine()); + assertEquals(SpaceType.INNER_PRODUCT, resolvedMethodContext.getKnnMethodContext().getSpaceType()); + assertEquals(CompressionLevel.x1, resolvedMethodContext.getCompressionLevel()); + assertNotEquals(knnMethodContext, resolvedMethodContext.getKnnMethodContext()); + } + + public void testResolveMethod_whenInvalid_thenThrow() { + // Invalid training context + expectThrows( + ValidationException.class, + () -> TEST_RESOLVER.resolveMethod( + null, + KNNMethodConfigContext.builder().vectorDataType(VectorDataType.FLOAT).versionCreated(Version.CURRENT).build(), + true, + SpaceType.L2 + ) + ); + + // Invalid compression + expectThrows( + ValidationException.class, + () -> TEST_RESOLVER.resolveMethod( + null, + KNNMethodConfigContext.builder() + .vectorDataType(VectorDataType.FLOAT) + .compressionLevel(CompressionLevel.x8) + .versionCreated(Version.CURRENT) + .build(), + false, + SpaceType.L2 + ) + ); + + // Invalid mode + expectThrows( + ValidationException.class, + () -> TEST_RESOLVER.resolveMethod( + null, + KNNMethodConfigContext.builder() + .vectorDataType(VectorDataType.FLOAT) + .mode(Mode.ON_DISK) + .versionCreated(Version.CURRENT) + .build(), + false, + SpaceType.L2 + ) + ); + } +} diff --git a/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java b/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java index 84cbf05dc..98bbf42ca 100644 --- a/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java +++ b/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java @@ -48,6 +48,7 @@ import org.opensearch.knn.index.engine.KNNMethodConfigContext; import org.opensearch.knn.index.engine.KNNMethodContext; import org.opensearch.knn.index.engine.MethodComponentContext; +import org.opensearch.knn.index.engine.faiss.QFrameBitEncoder; import org.opensearch.knn.indices.ModelDao; import org.opensearch.knn.indices.ModelMetadata; import org.opensearch.knn.indices.ModelState; @@ -93,6 +94,7 @@ @Log4j2 public class KNNVectorFieldMapperTests extends KNNTestCase { + private static final String TEST_INDEX_NAME = "test-index-name"; private static final String TEST_FIELD_NAME = "test-field-name"; private static final int TEST_DIMENSION = 17; @@ -1633,7 +1635,7 @@ public void testTypeParser_whenBinaryWithLegacyKNNEnabled_thenException() throws typeParser.parse(fieldName, xContentBuilderToMap(xContentBuilder), buildParserContext(indexName, settings)); }); - assertTrue(ex.getMessage(), ex.getMessage().contains("is not supported with")); + assertTrue(ex.getMessage(), ex.getMessage().contains("does not support space type")); } public void testBuild_whenInvalidCharsInFieldName_thenThrowException() { @@ -1653,6 +1655,345 @@ public void testBuild_whenInvalidCharsInFieldName_thenThrowException() { } } + public void testTypeParser_whenModeAndCompressionAreSet_thenHandle() throws IOException { + int dimension = 16; + Settings settings = Settings.builder().put(settings(CURRENT).build()).put(KNN_INDEX, true).build(); + ModelDao modelDao = mock(ModelDao.class); + KNNVectorFieldMapper.TypeParser typeParser = new KNNVectorFieldMapper.TypeParser(() -> modelDao); + + // Default to nmslib and ensure legacy is in use + XContentBuilder xContentBuilder = XContentFactory.jsonBuilder() + .startObject() + .field(TYPE_FIELD_NAME, KNN_VECTOR_TYPE) + .field(DIMENSION_FIELD_NAME, dimension) + .endObject(); + KNNVectorFieldMapper.Builder builder = (KNNVectorFieldMapper.Builder) typeParser.parse( + TEST_FIELD_NAME, + xContentBuilderToMap(xContentBuilder), + buildParserContext(TEST_INDEX_NAME, settings) + ); + assertNull(builder.getOriginalParameters().getKnnMethodContext()); + assertTrue(builder.getOriginalParameters().isLegacyMapping()); + validateBuilderAfterParsing( + builder, + KNNEngine.NMSLIB, + SpaceType.L2, + VectorDataType.FLOAT, + CompressionLevel.x1, + CompressionLevel.NOT_CONFIGURED, + Mode.NOT_CONFIGURED, + false + ); + + // If mode is in memory and 1x compression, again use default legacy + xContentBuilder = XContentFactory.jsonBuilder() + .startObject() + .field(TYPE_FIELD_NAME, KNN_VECTOR_TYPE) + .field(DIMENSION_FIELD_NAME, dimension) + .field(COMPRESSION_LEVEL_PARAMETER, CompressionLevel.x1.getName()) + .field(MODE_PARAMETER, Mode.IN_MEMORY.getName()) + .endObject(); + builder = (KNNVectorFieldMapper.Builder) typeParser.parse( + TEST_FIELD_NAME, + xContentBuilderToMap(xContentBuilder), + buildParserContext(TEST_INDEX_NAME, settings) + ); + assertNull(builder.getOriginalParameters().getKnnMethodContext()); + assertFalse(builder.getOriginalParameters().isLegacyMapping()); + validateBuilderAfterParsing( + builder, + KNNEngine.NMSLIB, + SpaceType.L2, + VectorDataType.FLOAT, + CompressionLevel.x1, + CompressionLevel.x1, + Mode.IN_MEMORY, + false + ); + + // Default on disk is faiss with 32x binary quant + xContentBuilder = XContentFactory.jsonBuilder() + .startObject() + .field(TYPE_FIELD_NAME, KNN_VECTOR_TYPE) + .field(DIMENSION_FIELD_NAME, dimension) + .field(MODE_PARAMETER, Mode.ON_DISK.getName()) + .endObject(); + + builder = (KNNVectorFieldMapper.Builder) typeParser.parse( + TEST_FIELD_NAME, + xContentBuilderToMap(xContentBuilder), + buildParserContext(TEST_INDEX_NAME, settings) + ); + validateBuilderAfterParsing( + builder, + KNNEngine.FAISS, + SpaceType.L2, + VectorDataType.FLOAT, + CompressionLevel.x32, + CompressionLevel.NOT_CONFIGURED, + Mode.ON_DISK, + true + ); + + // Ensure 2x does not use binary quantization + xContentBuilder = XContentFactory.jsonBuilder() + .startObject() + .field(TYPE_FIELD_NAME, KNN_VECTOR_TYPE) + .field(DIMENSION_FIELD_NAME, dimension) + .field(COMPRESSION_LEVEL_PARAMETER, CompressionLevel.x2.getName()) + .endObject(); + + builder = (KNNVectorFieldMapper.Builder) typeParser.parse( + TEST_FIELD_NAME, + xContentBuilderToMap(xContentBuilder), + buildParserContext(TEST_INDEX_NAME, settings) + ); + validateBuilderAfterParsing( + builder, + KNNEngine.FAISS, + SpaceType.L2, + VectorDataType.FLOAT, + CompressionLevel.x2, + CompressionLevel.x2, + Mode.NOT_CONFIGURED, + false + ); + + // For 8x ensure that it does use binary quantization + xContentBuilder = XContentFactory.jsonBuilder() + .startObject() + .field(TYPE_FIELD_NAME, KNN_VECTOR_TYPE) + .field(DIMENSION_FIELD_NAME, dimension) + .field(MODE_PARAMETER, Mode.ON_DISK.getName()) + .field(COMPRESSION_LEVEL_PARAMETER, CompressionLevel.x8.getName()) + .endObject(); + + builder = (KNNVectorFieldMapper.Builder) typeParser.parse( + TEST_FIELD_NAME, + xContentBuilderToMap(xContentBuilder), + buildParserContext(TEST_INDEX_NAME, settings) + ); + validateBuilderAfterParsing( + builder, + KNNEngine.FAISS, + SpaceType.L2, + VectorDataType.FLOAT, + CompressionLevel.x8, + CompressionLevel.x8, + Mode.ON_DISK, + true + ); + + // For 4x compression on disk, use Lucene + xContentBuilder = XContentFactory.jsonBuilder() + .startObject() + .field(TYPE_FIELD_NAME, KNN_VECTOR_TYPE) + .field(DIMENSION_FIELD_NAME, dimension) + .field(MODE_PARAMETER, Mode.ON_DISK.getName()) + .field(COMPRESSION_LEVEL_PARAMETER, CompressionLevel.x4.getName()) + .endObject(); + + builder = (KNNVectorFieldMapper.Builder) typeParser.parse( + TEST_FIELD_NAME, + xContentBuilderToMap(xContentBuilder), + buildParserContext(TEST_INDEX_NAME, settings) + ); + validateBuilderAfterParsing( + builder, + KNNEngine.LUCENE, + SpaceType.L2, + VectorDataType.FLOAT, + CompressionLevel.x4, + CompressionLevel.x4, + Mode.ON_DISK, + false + ); + + // For 4x compression in memory, use Lucene + xContentBuilder = XContentFactory.jsonBuilder() + .startObject() + .field(TYPE_FIELD_NAME, KNN_VECTOR_TYPE) + .field(DIMENSION_FIELD_NAME, dimension) + .field(MODE_PARAMETER, Mode.IN_MEMORY.getName()) + .field(COMPRESSION_LEVEL_PARAMETER, CompressionLevel.x4.getName()) + .endObject(); + + builder = (KNNVectorFieldMapper.Builder) typeParser.parse( + TEST_FIELD_NAME, + xContentBuilderToMap(xContentBuilder), + buildParserContext(TEST_INDEX_NAME, settings) + ); + validateBuilderAfterParsing( + builder, + KNNEngine.LUCENE, + SpaceType.L2, + VectorDataType.FLOAT, + CompressionLevel.x4, + CompressionLevel.x4, + Mode.IN_MEMORY, + false + ); + + // For override, ensure compression is correct + xContentBuilder = XContentFactory.jsonBuilder() + .startObject() + .field(TYPE_FIELD_NAME, KNN_VECTOR_TYPE) + .field(DIMENSION_FIELD_NAME, dimension) + .startObject(KNN_METHOD) + .field(NAME, METHOD_HNSW) + .field(KNN_ENGINE, KNNEngine.FAISS) + .startObject(PARAMETERS) + .startObject(METHOD_ENCODER_PARAMETER) + .field(NAME, QFrameBitEncoder.NAME) + .startObject(PARAMETERS) + .field(QFrameBitEncoder.BITCOUNT_PARAM, CompressionLevel.x16.numBitsForFloat32()) + .endObject() + .endObject() + .endObject() + .endObject() + .endObject(); + + builder = (KNNVectorFieldMapper.Builder) typeParser.parse( + TEST_FIELD_NAME, + xContentBuilderToMap(xContentBuilder), + buildParserContext(TEST_INDEX_NAME, settings) + ); + validateBuilderAfterParsing( + builder, + KNNEngine.FAISS, + SpaceType.L2, + VectorDataType.FLOAT, + CompressionLevel.x16, + CompressionLevel.NOT_CONFIGURED, + Mode.NOT_CONFIGURED, + true + ); + + // Override with conflicting compression levels should fail + XContentBuilder invalidXContentBuilder1 = XContentFactory.jsonBuilder() + .startObject() + .field(TYPE_FIELD_NAME, KNN_VECTOR_TYPE) + .field(DIMENSION_FIELD_NAME, dimension) + .field(COMPRESSION_LEVEL_PARAMETER, CompressionLevel.x4.getName()) + .startObject(KNN_METHOD) + .field(NAME, METHOD_HNSW) + .field(KNN_ENGINE, KNNEngine.FAISS) + .startObject(PARAMETERS) + .startObject(METHOD_ENCODER_PARAMETER) + .field(NAME, QFrameBitEncoder.NAME) + .startObject(PARAMETERS) + .field(QFrameBitEncoder.BITCOUNT_PARAM, CompressionLevel.x16.numBitsForFloat32()) + .endObject() + .endObject() + .endObject() + .endObject() + .endObject(); + + expectThrows( + ValidationException.class, + () -> typeParser.parse( + TEST_FIELD_NAME, + xContentBuilderToMap(invalidXContentBuilder1), + buildParserContext(TEST_INDEX_NAME, settings) + ) + ); + + // Invalid if vector data type is binary + XContentBuilder invalidXContentBuilder2 = XContentFactory.jsonBuilder() + .startObject() + .field(TYPE_FIELD_NAME, KNN_VECTOR_TYPE) + .field(DIMENSION_FIELD_NAME, dimension) + .field(VECTOR_DATA_TYPE_FIELD, VectorDataType.BINARY.getValue()) + .field(MODE_PARAMETER, Mode.IN_MEMORY.getName()) + .endObject(); + + expectThrows( + MapperParsingException.class, + () -> typeParser.parse( + TEST_FIELD_NAME, + xContentBuilderToMap(invalidXContentBuilder2), + buildParserContext(TEST_INDEX_NAME, settings) + ) + ); + + // Invalid if engine doesnt support the compression + XContentBuilder invalidXContentBuilder3 = XContentFactory.jsonBuilder() + .startObject() + .field(TYPE_FIELD_NAME, KNN_VECTOR_TYPE) + .field(DIMENSION_FIELD_NAME, dimension) + .field(COMPRESSION_LEVEL_PARAMETER, CompressionLevel.x4.getName()) + .startObject(KNN_METHOD) + .field(NAME, METHOD_HNSW) + .field(KNN_ENGINE, KNNEngine.FAISS) + .endObject() + .endObject(); + + expectThrows( + ValidationException.class, + () -> typeParser.parse( + TEST_FIELD_NAME, + xContentBuilderToMap(invalidXContentBuilder3), + buildParserContext(TEST_INDEX_NAME, settings) + ) + ); + } + + private void validateBuilderAfterParsing( + KNNVectorFieldMapper.Builder builder, + KNNEngine expectedEngine, + SpaceType expectedSpaceType, + VectorDataType expectedVectorDataType, + CompressionLevel expectedResolvedCompressionLevel, + CompressionLevel expectedOriginalCompressionLevel, + Mode expectedMode, + boolean shouldUsesBinaryQFramework + ) { + assertEquals(expectedEngine, builder.getOriginalParameters().getResolvedKnnMethodContext().getKnnEngine()); + assertEquals(expectedSpaceType, builder.getOriginalParameters().getResolvedKnnMethodContext().getSpaceType()); + assertEquals(expectedVectorDataType, builder.getKnnMethodConfigContext().getVectorDataType()); + + assertEquals(expectedResolvedCompressionLevel, builder.getKnnMethodConfigContext().getCompressionLevel()); + assertEquals(expectedOriginalCompressionLevel, CompressionLevel.fromName(builder.getOriginalParameters().getCompressionLevel())); + assertEquals(expectedMode, Mode.fromName(builder.getOriginalParameters().getMode())); + assertEquals(expectedMode, builder.getKnnMethodConfigContext().getMode()); + assertFalse(builder.getOriginalParameters().getResolvedKnnMethodContext().getMethodComponentContext().getParameters().isEmpty()); + + if (shouldUsesBinaryQFramework) { + assertEquals( + QFrameBitEncoder.NAME, + ((MethodComponentContext) builder.getOriginalParameters() + .getResolvedKnnMethodContext() + .getMethodComponentContext() + .getParameters() + .get(METHOD_ENCODER_PARAMETER)).getName() + ); + assertEquals( + expectedResolvedCompressionLevel.numBitsForFloat32(), + (int) ((MethodComponentContext) builder.getOriginalParameters() + .getResolvedKnnMethodContext() + .getMethodComponentContext() + .getParameters() + .get(METHOD_ENCODER_PARAMETER)).getParameters().get(QFrameBitEncoder.BITCOUNT_PARAM) + ); + } else { + assertTrue( + builder.getOriginalParameters().getResolvedKnnMethodContext().getMethodComponentContext().getParameters().isEmpty() + || builder.getOriginalParameters() + .getResolvedKnnMethodContext() + .getMethodComponentContext() + .getParameters() + .containsKey(METHOD_ENCODER_PARAMETER) == false + || QFrameBitEncoder.NAME.equals( + ((MethodComponentContext) builder.getOriginalParameters() + .getResolvedKnnMethodContext() + .getMethodComponentContext() + .getParameters() + .get(METHOD_ENCODER_PARAMETER)).getName() + ) == false + ); + } + } + private LuceneFieldMapper.CreateLuceneFieldMapperInput.CreateLuceneFieldMapperInputBuilder createLuceneFieldMapperInputBuilder() { return LuceneFieldMapper.CreateLuceneFieldMapperInput.builder() .name(TEST_FIELD_NAME) diff --git a/src/test/java/org/opensearch/knn/integ/ModeAndCompressionIT.java b/src/test/java/org/opensearch/knn/integ/ModeAndCompressionIT.java index 925ba0fff..ea9203196 100644 --- a/src/test/java/org/opensearch/knn/integ/ModeAndCompressionIT.java +++ b/src/test/java/org/opensearch/knn/integ/ModeAndCompressionIT.java @@ -21,7 +21,6 @@ import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.mapper.CompressionLevel; import org.opensearch.knn.index.mapper.Mode; -import org.opensearch.knn.index.mapper.ModeBasedResolver; import org.opensearch.knn.index.query.parser.RescoreParser; import java.util.List; @@ -30,7 +29,6 @@ import static org.opensearch.knn.common.KNNConstants.FAISS_NAME; import static org.opensearch.knn.common.KNNConstants.KNN_ENGINE; import static org.opensearch.knn.common.KNNConstants.KNN_METHOD; -import static org.opensearch.knn.common.KNNConstants.METHOD_HNSW; import static org.opensearch.knn.common.KNNConstants.METHOD_IVF; import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER; import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_EF_SEARCH; @@ -39,7 +37,6 @@ import static org.opensearch.knn.common.KNNConstants.MODEL_DESCRIPTION; import static org.opensearch.knn.common.KNNConstants.MODE_PARAMETER; import static org.opensearch.knn.common.KNNConstants.NAME; -import static org.opensearch.knn.common.KNNConstants.PARAMETERS; import static org.opensearch.knn.common.KNNConstants.TRAIN_FIELD_PARAMETER; import static org.opensearch.knn.common.KNNConstants.TRAIN_INDEX_PARAMETER; import static org.opensearch.knn.common.KNNConstants.VECTOR_DATA_TYPE_FIELD; @@ -71,29 +68,16 @@ public class ModeAndCompressionIT extends KNNRestTestCase { 1.0f, 2.0f }; + private static final String[] COMPRESSION_LEVELS = new String[] { + CompressionLevel.x2.getName(), + CompressionLevel.x4.getName(), + CompressionLevel.x8.getName(), + CompressionLevel.x16.getName(), + CompressionLevel.x32.getName() }; + @SneakyThrows public void testIndexCreation_whenInvalid_thenFail() { XContentBuilder builder = XContentFactory.jsonBuilder() - .startObject() - .startObject("properties") - .startObject(FIELD_NAME) - .field("type", "knn_vector") - .field("dimension", DIMENSION) - .field(MODE_PARAMETER, "on_disk") - .field(COMPRESSION_LEVEL_PARAMETER, "16x") - .startObject(KNN_METHOD) - .field(NAME, METHOD_HNSW) - .field(KNN_ENGINE, FAISS_NAME) - .startObject(PARAMETERS) - .endObject() - .endObject() - .endObject() - .endObject() - .endObject(); - String mapping1 = builder.toString(); - expectThrows(ResponseException.class, () -> createKnnIndex(INDEX_NAME, mapping1)); - - builder = XContentFactory.jsonBuilder() .startObject() .startObject("properties") .startObject(FIELD_NAME) @@ -139,7 +123,7 @@ public void testIndexCreation_whenInvalid_thenFail() { @SneakyThrows public void testIndexCreation_whenValid_ThenSucceed() { XContentBuilder builder; - for (CompressionLevel compressionLevel : ModeBasedResolver.SUPPORTED_COMPRESSION_LEVELS) { + for (String compressionLevel : COMPRESSION_LEVELS) { String indexName = INDEX_NAME + compressionLevel; builder = XContentFactory.jsonBuilder() .startObject() @@ -147,16 +131,23 @@ public void testIndexCreation_whenValid_ThenSucceed() { .startObject(FIELD_NAME) .field("type", "knn_vector") .field("dimension", DIMENSION) - .field(COMPRESSION_LEVEL_PARAMETER, compressionLevel.getName()) + .field(COMPRESSION_LEVEL_PARAMETER, compressionLevel) .endObject() .endObject() .endObject(); String mapping = builder.toString(); validateIndex(indexName, mapping); - validateSearch(indexName, METHOD_PARAMETER_EF_SEARCH, KNNSettings.INDEX_KNN_DEFAULT_ALGO_PARAM_EF_SEARCH); + logger.info("Compression level {}", compressionLevel); + validateSearch( + indexName, + METHOD_PARAMETER_EF_SEARCH, + KNNSettings.INDEX_KNN_DEFAULT_ALGO_PARAM_EF_SEARCH, + compressionLevel, + Mode.NOT_CONFIGURED.getName() + ); } - for (CompressionLevel compressionLevel : ModeBasedResolver.SUPPORTED_COMPRESSION_LEVELS) { + for (String compressionLevel : COMPRESSION_LEVELS) { for (String mode : Mode.NAMES_ARRAY) { String indexName = INDEX_NAME + compressionLevel + "_" + mode; builder = XContentFactory.jsonBuilder() @@ -166,13 +157,20 @@ public void testIndexCreation_whenValid_ThenSucceed() { .field("type", "knn_vector") .field("dimension", DIMENSION) .field(MODE_PARAMETER, mode) - .field(COMPRESSION_LEVEL_PARAMETER, compressionLevel.getName()) + .field(COMPRESSION_LEVEL_PARAMETER, compressionLevel) .endObject() .endObject() .endObject(); String mapping = builder.toString(); validateIndex(indexName, mapping); - validateSearch(indexName, METHOD_PARAMETER_EF_SEARCH, KNNSettings.INDEX_KNN_DEFAULT_ALGO_PARAM_EF_SEARCH); + logger.info("Compression level {}", compressionLevel); + validateSearch( + indexName, + METHOD_PARAMETER_EF_SEARCH, + KNNSettings.INDEX_KNN_DEFAULT_ALGO_PARAM_EF_SEARCH, + compressionLevel, + mode + ); } } @@ -190,7 +188,14 @@ public void testIndexCreation_whenValid_ThenSucceed() { .endObject(); String mapping = builder.toString(); validateIndex(indexName, mapping); - validateSearch(indexName, METHOD_PARAMETER_EF_SEARCH, KNNSettings.INDEX_KNN_DEFAULT_ALGO_PARAM_EF_SEARCH); + logger.info("Compression level {}", CompressionLevel.NOT_CONFIGURED.getName()); + validateSearch( + indexName, + METHOD_PARAMETER_EF_SEARCH, + KNNSettings.INDEX_KNN_DEFAULT_ALGO_PARAM_EF_SEARCH, + CompressionLevel.NOT_CONFIGURED.getName(), + mode + ); } } @@ -252,7 +257,7 @@ public void testTraining_whenInvalid_thenFail() { public void testTraining_whenValid_thenSucceed() { setupTrainingIndex(); XContentBuilder builder; - for (CompressionLevel compressionLevel : ModeBasedResolver.SUPPORTED_COMPRESSION_LEVELS) { + for (String compressionLevel : CompressionLevel.NAMES_ARRAY) { String indexName = INDEX_NAME + compressionLevel; String modelId = indexName; builder = XContentFactory.jsonBuilder() @@ -261,7 +266,7 @@ public void testTraining_whenValid_thenSucceed() { .field(TRAIN_FIELD_PARAMETER, TRAINING_FIELD_NAME) .field(KNNConstants.DIMENSION, DIMENSION) .field(MODEL_DESCRIPTION, "") - .field(COMPRESSION_LEVEL_PARAMETER, compressionLevel.getName()) + .field(COMPRESSION_LEVEL_PARAMETER, compressionLevel) .endObject(); validateTraining(modelId, builder); builder = XContentFactory.jsonBuilder() @@ -275,10 +280,16 @@ public void testTraining_whenValid_thenSucceed() { .endObject(); String mapping = builder.toString(); validateIndex(indexName, mapping); - validateSearch(indexName, METHOD_PARAMETER_NPROBES, METHOD_PARAMETER_NLIST_DEFAULT); + validateSearch( + indexName, + METHOD_PARAMETER_NPROBES, + METHOD_PARAMETER_NLIST_DEFAULT, + compressionLevel, + Mode.NOT_CONFIGURED.getName() + ); } - for (CompressionLevel compressionLevel : ModeBasedResolver.SUPPORTED_COMPRESSION_LEVELS) { + for (String compressionLevel : CompressionLevel.NAMES_ARRAY) { for (String mode : Mode.NAMES_ARRAY) { String indexName = INDEX_NAME + compressionLevel + "_" + mode; String modelId = indexName; @@ -288,7 +299,7 @@ public void testTraining_whenValid_thenSucceed() { .field(TRAIN_FIELD_PARAMETER, TRAINING_FIELD_NAME) .field(KNNConstants.DIMENSION, DIMENSION) .field(MODEL_DESCRIPTION, "") - .field(COMPRESSION_LEVEL_PARAMETER, compressionLevel.getName()) + .field(COMPRESSION_LEVEL_PARAMETER, compressionLevel) .field(MODE_PARAMETER, mode) .endObject(); validateTraining(modelId, builder); @@ -303,7 +314,7 @@ public void testTraining_whenValid_thenSucceed() { .endObject(); String mapping = builder.toString(); validateIndex(indexName, mapping); - validateSearch(indexName, METHOD_PARAMETER_NPROBES, METHOD_PARAMETER_NLIST_DEFAULT); + validateSearch(indexName, METHOD_PARAMETER_NPROBES, METHOD_PARAMETER_NLIST_DEFAULT, compressionLevel, mode); } } @@ -330,7 +341,13 @@ public void testTraining_whenValid_thenSucceed() { .endObject(); String mapping = builder.toString(); validateIndex(indexName, mapping); - validateSearch(indexName, METHOD_PARAMETER_NPROBES, METHOD_PARAMETER_NLIST_DEFAULT); + validateSearch( + indexName, + METHOD_PARAMETER_NPROBES, + METHOD_PARAMETER_NLIST_DEFAULT, + CompressionLevel.NOT_CONFIGURED.getName(), + mode + ); } } @@ -381,7 +398,13 @@ private void validateTraining(String modelId, XContentBuilder builder) { } @SneakyThrows - private void validateSearch(String indexName, String methodParameterName, int methodParameterValue) { + private void validateSearch( + String indexName, + String methodParameterName, + int methodParameterValue, + String compressionLevelString, + String mode + ) { // Basic search Response response = searchKNNIndex( indexName, @@ -436,7 +459,9 @@ private void validateSearch(String indexName, String methodParameterName, int me String exactSearchResponseBody = EntityUtils.toString(exactSearchResponse.getEntity()); List exactSearchKnnResults = parseSearchResponseScore(exactSearchResponseBody, FIELD_NAME); assertEquals(NUM_DOCS, exactSearchKnnResults.size()); - Assert.assertEquals(exactSearchKnnResults, knnResults); + if (CompressionLevel.x4.getName().equals(compressionLevelString) == false && Mode.ON_DISK.getName().equals(mode)) { + Assert.assertEquals(exactSearchKnnResults, knnResults); + } // Search with rescore response = searchKNNIndex( @@ -464,6 +489,8 @@ private void validateSearch(String indexName, String methodParameterName, int me responseBody = EntityUtils.toString(response.getEntity()); knnResults = parseSearchResponseScore(responseBody, FIELD_NAME); assertEquals(K, knnResults.size()); - Assert.assertEquals(exactSearchKnnResults, knnResults); + if (CompressionLevel.x4.getName().equals(compressionLevelString) == false && Mode.ON_DISK.getName().equals(mode)) { + Assert.assertEquals(exactSearchKnnResults, knnResults); + } } } diff --git a/src/test/java/org/opensearch/knn/plugin/stats/suppliers/LibraryInitializedSupplierTests.java b/src/test/java/org/opensearch/knn/plugin/stats/suppliers/LibraryInitializedSupplierTests.java index 4399b3318..5dbd0fc8b 100644 --- a/src/test/java/org/opensearch/knn/plugin/stats/suppliers/LibraryInitializedSupplierTests.java +++ b/src/test/java/org/opensearch/knn/plugin/stats/suppliers/LibraryInitializedSupplierTests.java @@ -18,6 +18,7 @@ import org.opensearch.knn.index.engine.KNNMethodContext; import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.engine.KNNLibrary; +import org.opensearch.knn.index.engine.ResolvedMethodContext; import org.opensearch.test.OpenSearchTestCase; public class LibraryInitializedSupplierTests extends OpenSearchTestCase { @@ -105,5 +106,15 @@ public Boolean isInitialized() { public void setInitialized(Boolean isInitialized) { this.initialized = isInitialized; } + + @Override + public ResolvedMethodContext resolveMethod( + KNNMethodContext knnMethodContext, + KNNMethodConfigContext knnMethodConfigContext, + boolean shouldRequireTraining, + SpaceType spaceType + ) { + return null; + } } } diff --git a/src/test/java/org/opensearch/knn/plugin/transport/TrainingJobRouterTransportActionTests.java b/src/test/java/org/opensearch/knn/plugin/transport/TrainingJobRouterTransportActionTests.java index 151626ef5..30c5d33a1 100644 --- a/src/test/java/org/opensearch/knn/plugin/transport/TrainingJobRouterTransportActionTests.java +++ b/src/test/java/org/opensearch/knn/plugin/transport/TrainingJobRouterTransportActionTests.java @@ -304,7 +304,7 @@ public void testTrainingIndexSize() { // Setup the request TrainingModelRequest trainingModelRequest = new TrainingModelRequest( null, - getDefaultKNNMethodContext(), + getDefaultKNNMethodContextForModel(), dimension, trainingIndexName, "training-field", @@ -353,7 +353,7 @@ public void testTrainIndexSize_whenDataTypeIsBinary() { // Setup the request TrainingModelRequest trainingModelRequest = new TrainingModelRequest( null, - getDefaultKNNMethodContext(), + getDefaultKNNMethodContextForModel(), dimension, trainingIndexName, "training-field", @@ -403,7 +403,7 @@ public void testTrainIndexSize_whenDataTypeIsByte() { // Setup the request TrainingModelRequest trainingModelRequest = new TrainingModelRequest( null, - getDefaultKNNMethodContext(), + getDefaultKNNMethodContextForModel(), dimension, trainingIndexName, "training-field", diff --git a/src/test/java/org/opensearch/knn/plugin/transport/TrainingModelRequestTests.java b/src/test/java/org/opensearch/knn/plugin/transport/TrainingModelRequestTests.java index 79292fb53..2c423e0ef 100644 --- a/src/test/java/org/opensearch/knn/plugin/transport/TrainingModelRequestTests.java +++ b/src/test/java/org/opensearch/knn/plugin/transport/TrainingModelRequestTests.java @@ -44,7 +44,6 @@ import java.util.List; import java.util.Map; -import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -52,7 +51,7 @@ public class TrainingModelRequestTests extends KNNTestCase { public void testStreams() throws IOException { String modelId = "test-model-id"; - KNNMethodContext knnMethodContext = getDefaultKNNMethodContext(); + KNNMethodContext knnMethodContext = getDefaultKNNMethodContextForModel(); int dimension = 10; String trainingIndex = "test-training-index"; String trainingField = "test-training-field"; @@ -142,7 +141,7 @@ public void testStreams() throws IOException { public void testGetters() { String modelId = "test-model-id"; - KNNMethodContext knnMethodContext = getDefaultKNNMethodContext(); + KNNMethodContext knnMethodContext = getDefaultKNNMethodContextForModel(); int dimension = 10; String trainingIndex = "test-training-index"; String trainingField = "test-training-field"; @@ -170,7 +169,6 @@ public void testGetters() { trainingModelRequest.setTrainingDataSizeInKB(trainingSetSizeInKB); assertEquals(modelId, trainingModelRequest.getModelId()); - assertEquals(knnMethodContext, trainingModelRequest.getKnnMethodContext()); assertEquals(dimension, trainingModelRequest.getDimension()); assertEquals(trainingIndex, trainingModelRequest.getTrainingIndex()); assertEquals(trainingField, trainingModelRequest.getTrainingField()); @@ -187,18 +185,13 @@ public void testValidation_invalid_modelIdAlreadyExists() { // Setup the training request String modelId = "test-model-id"; - KNNEngine knnEngine = mock(KNNEngine.class); - when(knnEngine.validateMethod(any(), any())).thenReturn(null); - when(knnEngine.isTrainingRequired(any())).thenReturn(true); - KNNMethodContext knnMethodContext = mock(KNNMethodContext.class); - when(knnMethodContext.getKnnEngine()).thenReturn(knnEngine); int dimension = 10; String trainingIndex = "test-training-index"; String trainingField = "test-training-field"; TrainingModelRequest trainingModelRequest = new TrainingModelRequest( modelId, - knnMethodContext, + getDefaultKNNMethodContextForModel(), dimension, trainingIndex, trainingField, @@ -251,18 +244,13 @@ public void testValidation_blocked_modelId() { // Setup the training request String modelId = "test-model-id"; - KNNEngine knnEngine = mock(KNNEngine.class); - when(knnEngine.validateMethod(any(), any())).thenReturn(null); - when(knnEngine.isTrainingRequired(any())).thenReturn(true); - KNNMethodContext knnMethodContext = mock(KNNMethodContext.class); - when(knnMethodContext.getKnnEngine()).thenReturn(knnEngine); int dimension = 10; String trainingIndex = "test-training-index"; String trainingField = "test-training-field"; TrainingModelRequest trainingModelRequest = new TrainingModelRequest( modelId, - knnMethodContext, + getDefaultKNNMethodContextForModel(), dimension, trainingIndex, trainingField, @@ -298,48 +286,26 @@ public void testValidation_invalid_invalidMethodContext() { String modelId = "test-model-id"; // Mock throwing an exception on validation - KNNMethodContext knnMethodContext = mock(KNNMethodContext.class); - String validationExceptionMessage = "knn method invalid"; - ValidationException validationException = new ValidationException(); - validationException.addValidationError(validationExceptionMessage); - when(knnMethodContext.validate(any())).thenReturn(validationException); - - when(knnMethodContext.isTrainingRequired()).thenReturn(false); - when(knnMethodContext.getMethodComponentContext()).thenReturn(MethodComponentContext.EMPTY); int dimension = 10; String trainingIndex = "test-training-index"; String trainingField = "test-training-field"; - TrainingModelRequest trainingModelRequest = new TrainingModelRequest( - modelId, - knnMethodContext, - dimension, - trainingIndex, - trainingField, - null, - null, - VectorDataType.DEFAULT, - Mode.NOT_CONFIGURED, - CompressionLevel.NOT_CONFIGURED + ValidationException validationException = expectThrows( + ValidationException.class, + () -> new TrainingModelRequest( + modelId, + getDefaultKNNMethodContext(), + dimension, + trainingIndex, + trainingField, + null, + null, + VectorDataType.DEFAULT, + Mode.NOT_CONFIGURED, + CompressionLevel.NOT_CONFIGURED + ) ); - - // Mock the model dao to return null so that no exception is produced - ModelDao modelDao = mock(ModelDao.class); - when(modelDao.getMetadata(modelId)).thenReturn(null); - - // This cluster service will result in no validation exceptions - ClusterService clusterService = getClusterServiceForValidReturns(trainingIndex, trainingField, dimension); - - // Initialize static components with the mocks - TrainingModelRequest.initialize(modelDao, clusterService); - - // Test that validation produces model already exists error message - ActionRequestValidationException exception = trainingModelRequest.validate(); - assertNotNull(exception); - List validationErrors = exception.validationErrors(); - assertEquals(2, validationErrors.size()); - assertTrue(validationErrors.get(0).contains(validationExceptionMessage)); - assertTrue(validationErrors.get(1).contains("Method does not require training.")); + assertTrue(validationException.getMessage().contains("engine from training context")); } public void testValidation_invalid_trainingIndexDoesNotExist() { From 9717e4c223d3fa5cb5000c1de47d5c10e177b19e Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Wed, 11 Sep 2024 12:13:16 -0700 Subject: [PATCH 364/416] Use correct type for binary vector in ivf training (#2086) (#2088) Signed-off-by: Heemin Kim (cherry picked from commit bf11cbb12731f5471f441f92f9360065a6caedd1) Co-authored-by: Heemin Kim --- jni/src/faiss_wrapper.cpp | 16 ++++++++++------ .../opensearch-knn.release-notes-2.17.0.0.md | 1 + 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/jni/src/faiss_wrapper.cpp b/jni/src/faiss_wrapper.cpp index ba15c3ce7..227fcb477 100644 --- a/jni/src/faiss_wrapper.cpp +++ b/jni/src/faiss_wrapper.cpp @@ -71,7 +71,7 @@ void SetExtraParameters(knn_jni::JNIUtilInterface * jniUtil, JNIEnv *env, void InternalTrainIndex(faiss::Index * index, faiss::idx_t n, const float* x); // Train a binary index with data provided -void InternalTrainBinaryIndex(faiss::IndexBinary * index, faiss::idx_t n, const float* x); +void InternalTrainBinaryIndex(faiss::IndexBinary * index, faiss::idx_t n, const uint8_t* x); // Converts the int FilterIds to Faiss ids type array. void convertFilterIdsToFaissIdType(const int* filterIds, int filterIdsLength, faiss::idx_t* convertedFilterIds); @@ -286,7 +286,7 @@ void knn_jni::faiss_wrapper::CreateBinaryIndexFromTemplate(knn_jni::JNIUtilInter auto *inputVectors = reinterpret_cast*>(vectorsAddressJ); int dim = (int)dimJ; if (dim % 8 != 0) { - throw std::runtime_error("Dimensions should be multiply of 8"); + throw std::runtime_error("Dimensions should be multiple of 8"); } int numVectors = (int) (inputVectors->size() / (uint64_t) (dim / 8)); int numIds = jniUtil->GetJavaIntArrayLength(env, idsJ); @@ -848,8 +848,12 @@ jbyteArray knn_jni::faiss_wrapper::TrainBinaryIndex(knn_jni::JNIUtilInterface * } // Train index if needed - auto *trainingVectorsPointerCpp = reinterpret_cast*>(trainVectorsPointerJ); - int numVectors = trainingVectorsPointerCpp->size()/(int) dimensionJ; + int dim = (int)dimensionJ; + if (dim % 8 != 0) { + throw std::runtime_error("Dimensions should be multiple of 8"); + } + auto *trainingVectorsPointerCpp = reinterpret_cast*>(trainVectorsPointerJ); + int numVectors = (int) (trainingVectorsPointerCpp->size() / (dim / 8)); if(!indexWriter->is_trained) { InternalTrainBinaryIndex(indexWriter.get(), numVectors, trainingVectorsPointerCpp->data()); } @@ -997,12 +1001,12 @@ void InternalTrainIndex(faiss::Index * index, faiss::idx_t n, const float* x) { } } -void InternalTrainBinaryIndex(faiss::IndexBinary * index, faiss::idx_t n, const float* x) { +void InternalTrainBinaryIndex(faiss::IndexBinary * index, faiss::idx_t n, const uint8_t* x) { if (auto * indexIvf = dynamic_cast(index)) { indexIvf->make_direct_map(); } if (!index->is_trained) { - index->train(n, reinterpret_cast(x)); + index->train(n, x); } } diff --git a/release-notes/opensearch-knn.release-notes-2.17.0.0.md b/release-notes/opensearch-knn.release-notes-2.17.0.0.md index 8b4aa8e95..1e046be9b 100644 --- a/release-notes/opensearch-knn.release-notes-2.17.0.0.md +++ b/release-notes/opensearch-knn.release-notes-2.17.0.0.md @@ -18,6 +18,7 @@ Compatible with OpenSearch 2.17.0 * Fix graph merge stats size calculation [#1844](https://github.com/opensearch-project/k-NN/pull/1844) * Disallow a vector field to have an invalid character for a physical file name. [#1936](https://github.com/opensearch-project/k-NN/pull/1936) * Fix memory overflow caused by cache behavior [#2015](https://github.com/opensearch-project/k-NN/pull/2015) +* Use correct type for binary vector in ivf training [#2086](https://github.com/opensearch-project/k-NN/pull/2086) ### Infrastructure * Parallelize make to reduce build time [#2006] (https://github.com/opensearch-project/k-NN/pull/2006) ### Maintenance From 3c40b93531d01456ae5c630de459d7312617946d Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Wed, 11 Sep 2024 13:08:49 -0700 Subject: [PATCH 365/416] Switch MINGW32 to MINGW64 (#2090) (#2091) In the CI, we uses MINGW64. With MINGW32, the hamming distance calculation is not correct which result in bad recall for binary vector in window platform. Signed-off-by: Heemin Kim (cherry picked from commit 019bcc3509a6793e93b4801360b0d8e20528259e) Co-authored-by: Heemin Kim --- .../opensearch-knn.release-notes-2.17.0.0.md | 1 + scripts/windowsScript.ps1 | 24 +++++++++---------- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/release-notes/opensearch-knn.release-notes-2.17.0.0.md b/release-notes/opensearch-knn.release-notes-2.17.0.0.md index 1e046be9b..d5bf80319 100644 --- a/release-notes/opensearch-knn.release-notes-2.17.0.0.md +++ b/release-notes/opensearch-knn.release-notes-2.17.0.0.md @@ -19,6 +19,7 @@ Compatible with OpenSearch 2.17.0 * Disallow a vector field to have an invalid character for a physical file name. [#1936](https://github.com/opensearch-project/k-NN/pull/1936) * Fix memory overflow caused by cache behavior [#2015](https://github.com/opensearch-project/k-NN/pull/2015) * Use correct type for binary vector in ivf training [#2086](https://github.com/opensearch-project/k-NN/pull/2086) +* Switch MINGW32 to MINGW64 [#2090](https://github.com/opensearch-project/k-NN/pull/2090) ### Infrastructure * Parallelize make to reduce build time [#2006] (https://github.com/opensearch-project/k-NN/pull/2006) ### Maintenance diff --git a/scripts/windowsScript.ps1 b/scripts/windowsScript.ps1 index 2b0d4f799..e9373223a 100644 --- a/scripts/windowsScript.ps1 +++ b/scripts/windowsScript.ps1 @@ -7,14 +7,14 @@ git submodule update --init -- jni/external/nmslib git submodule update --init -- jni/external/faiss # _MSC_VER is a predefined macro which defines the version of Visual Studio Compiler -# As we are using x86_64-w64-mingw32-gcc compiler we need to replace this macro with __MINGW32__ -(Get-Content jni/external/faiss/faiss/impl/index_read.cpp).replace('_MSC_VER', '__MINGW32__') | Set-Content jni/external/faiss/faiss/impl/index_read.cpp -(Get-Content jni/external/faiss/faiss/impl/index_write.cpp).replace('_MSC_VER', '__MINGW32__') | Set-Content jni/external/faiss/faiss/impl/index_write.cpp -(Get-Content jni/external/faiss/faiss/impl/platform_macros.h).replace('_MSC_VER', '__MINGW32__') | Set-Content jni/external/faiss/faiss/impl/platform_macros.h +# As we are using x86_64-w64-mingw32-gcc compiler we need to replace this macro with __MINGW64__ +(Get-Content jni/external/faiss/faiss/impl/index_read.cpp).replace('_MSC_VER', '__MINGW64__') | Set-Content jni/external/faiss/faiss/impl/index_read.cpp +(Get-Content jni/external/faiss/faiss/impl/index_write.cpp).replace('_MSC_VER', '__MINGW64__') | Set-Content jni/external/faiss/faiss/impl/index_write.cpp +(Get-Content jni/external/faiss/faiss/impl/platform_macros.h).replace('_MSC_VER', '__MINGW64__') | Set-Content jni/external/faiss/faiss/impl/platform_macros.h (Get-Content jni/external/faiss/faiss/impl/platform_macros.h).replace('#define __PRETTY_FUNCTION__ __FUNCSIG__', ' ') | Set-Content jni/external/faiss/faiss/impl/platform_macros.h -(Get-Content jni/external/faiss/faiss/utils/utils.cpp).replace('_MSC_VER', '__MINGW32__') | Set-Content jni/external/faiss/faiss/utils/utils.cpp -(Get-Content jni/external/faiss/faiss/utils/prefetch.h).replace('_MSC_VER', '__MINGW32__') | Set-Content jni/external/faiss/faiss/utils/prefetch.h -(Get-Content jni/external/faiss/faiss/invlists/InvertedListsIOHook.cpp).replace('_MSC_VER', '__MINGW32__') | Set-Content jni/external/faiss/faiss/invlists/InvertedListsIOHook.cpp +(Get-Content jni/external/faiss/faiss/utils/utils.cpp).replace('_MSC_VER', '__MINGW64__') | Set-Content jni/external/faiss/faiss/utils/utils.cpp +(Get-Content jni/external/faiss/faiss/utils/prefetch.h).replace('_MSC_VER', '__MINGW64__') | Set-Content jni/external/faiss/faiss/utils/prefetch.h +(Get-Content jni/external/faiss/faiss/invlists/InvertedListsIOHook.cpp).replace('_MSC_VER', '__MINGW64__') | Set-Content jni/external/faiss/faiss/invlists/InvertedListsIOHook.cpp (Get-Content jni/external/faiss/faiss/AutoTune.cpp).replace('__PRETTY_FUNCTION__', 'NULL') | Set-Content jni/external/faiss/faiss/AutoTune.cpp (Get-Content jni/external/faiss/faiss/utils/distances_simd.cpp).replace('FAISS_PRAGMA_IMPRECISE_FUNCTION_BEGIN', ' ') | Set-Content jni/external/faiss/faiss/utils/distances_simd.cpp (Get-Content jni/external/faiss/faiss/utils/distances_simd.cpp).replace('FAISS_PRAGMA_IMPRECISE_FUNCTION_END', ' ') | Set-Content jni/external/faiss/faiss/utils/distances_simd.cpp @@ -23,17 +23,17 @@ git submodule update --init -- jni/external/faiss # is a Unix header and is not available on Windows. So, adding condition to include it if not running on Windows # Replace '#include ' with -# #ifndef __MINGW32__ +# #ifndef __MINGW64__ # #include # #endif -(Get-Content jni/external/faiss/faiss/invlists/OnDiskInvertedLists.cpp).replace('#include ', "#ifndef __MINGW32__`n#include `n#endif") | Set-Content jni/external/faiss/faiss/invlists/OnDiskInvertedLists.cpp -# intrin.h function like __builtin_ctz, __builtin_clzll is not available in MINGW32. So, adding condition to include it if not running on Windows +(Get-Content jni/external/faiss/faiss/invlists/OnDiskInvertedLists.cpp).replace('#include ', "#ifndef __MINGW64__`n#include `n#endif") | Set-Content jni/external/faiss/faiss/invlists/OnDiskInvertedLists.cpp +# intrin.h function like __builtin_ctz, __builtin_clzll is not available in MINGW64. So, adding condition to include it if not running on Windows # Replace '#include ' with -# #ifndef __MINGW32__ +# #ifndef __MINGW64__ # include # and # Closing the above #ifndef with # #define __builtin_popcountl __popcnt64 # #endif -(Get-Content jni/external/faiss/faiss/impl/platform_macros.h).replace('#include ', "#ifndef __MINGW32__`n#include `n") | Set-Content jni/external/faiss/faiss/impl/platform_macros.h +(Get-Content jni/external/faiss/faiss/impl/platform_macros.h).replace('#include ', "#ifndef __MINGW64__`n#include `n") | Set-Content jni/external/faiss/faiss/impl/platform_macros.h (Get-Content jni/external/faiss/faiss/impl/platform_macros.h).replace('#define __builtin_popcountl __popcnt64', "#define __builtin_popcountl __popcnt64`n#endif`n") | Set-Content jni/external/faiss/faiss/impl/platform_macros.h From e4a591467b0954cd62d0c52848443f20ad6c824b Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Wed, 11 Sep 2024 15:41:20 -0700 Subject: [PATCH 366/416] [Backport 2.x] Add recall test with small dataset (#2093) * Add recall test with small dataset (#2080) Signed-off-by: Heemin Kim (cherry picked from commit 0c79147f2ecd6a4c82109594faa8ff92efc52c0d) * Update IndexIT.java Change package for EntityUtils Signed-off-by: Heemin Kim --------- Signed-off-by: Heemin Kim Co-authored-by: Heemin Kim --- .../opensearch/knn/integ/BinaryIndexIT.java | 40 +++-- .../org/opensearch/knn/integ/IndexIT.java | 139 ++++++++++++++++++ src/test/resources/data/README.md | 8 + .../data/test_ground_truth_binary_100.csv | 100 +++++++++++++ .../data/test_ground_truth_l2_100.csv | 100 +++++++++++++ .../knn/KNNJsonIndexMappingsBuilder.java | 8 + 6 files changed, 380 insertions(+), 15 deletions(-) create mode 100644 src/test/java/org/opensearch/knn/integ/IndexIT.java create mode 100644 src/test/resources/data/test_ground_truth_binary_100.csv create mode 100644 src/test/resources/data/test_ground_truth_l2_100.csv diff --git a/src/test/java/org/opensearch/knn/integ/BinaryIndexIT.java b/src/test/java/org/opensearch/knn/integ/BinaryIndexIT.java index f75cf7b75..86cee6a28 100644 --- a/src/test/java/org/opensearch/knn/integ/BinaryIndexIT.java +++ b/src/test/java/org/opensearch/knn/integ/BinaryIndexIT.java @@ -23,7 +23,10 @@ import java.io.IOException; import java.net.URL; +import java.util.Arrays; import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; import static org.opensearch.knn.common.KNNConstants.METHOD_HNSW; @@ -41,9 +44,11 @@ public static void setUpClass() throws IOException { } URL testIndexVectors = BinaryIndexIT.class.getClassLoader().getResource("data/test_vectors_binary_1000x128.json"); URL testQueries = BinaryIndexIT.class.getClassLoader().getResource("data/test_queries_binary_100x128.csv"); + URL groundTruthValues = BinaryIndexIT.class.getClassLoader().getResource("data/test_ground_truth_binary_100.csv"); assert testIndexVectors != null; assert testQueries != null; - testData = new TestUtils.TestData(testIndexVectors.getPath(), testQueries.getPath()); + assert groundTruthValues != null; + testData = new TestUtils.TestData(testIndexVectors.getPath(), testQueries.getPath(), groundTruthValues.getPath()); } @After @@ -83,18 +88,19 @@ public void testFaissHnswBinary_whenSmallDataSet_thenCreateIngestQueryWorks() { } @SneakyThrows - public void testFaissHnswBinary_when1000Data_thenCreateIngestQueryWorks() { + public void testFaissHnswBinary_when1000Data_thenRecallIsAboveNinePointZero() { // Create Index createKnnHnswBinaryIndex(KNNEngine.FAISS, INDEX_NAME, FIELD_NAME, 128); ingestTestData(INDEX_NAME, FIELD_NAME); - int k = 10; + int k = 100; for (int i = 0; i < testData.queries.length; i++) { - // Query List knnResults = runKnnQuery(INDEX_NAME, FIELD_NAME, testData.queries[i], k); - - // Validate - assertEquals(k, knnResults.size()); + float recall = getRecall( + Set.of(Arrays.copyOf(testData.groundTruthValues[i], k)), + knnResults.stream().map(KNNResult::getDocId).collect(Collectors.toSet()) + ); + assertTrue("Recall: " + recall, recall > 0.1); } } @@ -109,6 +115,18 @@ public void testFaissHnswBinary_whenRadialSearch_thenThrowException() { assertTrue(e.getMessage(), e.getMessage().contains("Binary data type does not support radial search")); } + private float getRecall(final Set truth, final Set result) { + // Count the number of relevant documents retrieved + result.retainAll(truth); + int relevantRetrieved = result.size(); + + // Total number of relevant documents + int totalRelevant = truth.size(); + + // Calculate recall + return (float) relevantRetrieved / totalRelevant; + } + private List runRnnQuery( final String indexName, final String fieldName, @@ -171,12 +189,4 @@ private void createKnnHnswBinaryIndex(final KNNEngine knnEngine, final String in createKnnIndex(indexName, knnIndexMapping); } - - private byte[] toByte(final float[] vector) { - byte[] bytes = new byte[vector.length]; - for (int i = 0; i < vector.length; i++) { - bytes[i] = (byte) vector[i]; - } - return bytes; - } } diff --git a/src/test/java/org/opensearch/knn/integ/IndexIT.java b/src/test/java/org/opensearch/knn/integ/IndexIT.java new file mode 100644 index 000000000..793c17f23 --- /dev/null +++ b/src/test/java/org/opensearch/knn/integ/IndexIT.java @@ -0,0 +1,139 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.integ; + +import com.google.common.primitives.Floats; +import lombok.SneakyThrows; +import lombok.extern.log4j.Log4j2; +import org.apache.commons.lang.ArrayUtils; +import org.apache.http.util.EntityUtils; +import org.junit.After; +import org.junit.BeforeClass; +import org.opensearch.client.Response; +import org.opensearch.knn.KNNJsonIndexMappingsBuilder; +import org.opensearch.knn.KNNJsonQueryBuilder; +import org.opensearch.knn.KNNRestTestCase; +import org.opensearch.knn.KNNResult; +import org.opensearch.knn.TestUtils; +import org.opensearch.knn.index.SpaceType; +import org.opensearch.knn.index.VectorDataType; +import org.opensearch.knn.index.engine.KNNEngine; + +import java.io.IOException; +import java.net.URL; +import java.util.Arrays; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + +import static org.opensearch.knn.common.KNNConstants.METHOD_HNSW; + +/** + * This class contains integration tests for index + */ +@Log4j2 +public class IndexIT extends KNNRestTestCase { + private static TestUtils.TestData testData; + + @BeforeClass + public static void setUpClass() throws IOException { + if (IndexIT.class.getClassLoader() == null) { + throw new IllegalStateException("ClassLoader of IndexIT Class is null"); + } + URL testIndexVectors = IndexIT.class.getClassLoader().getResource("data/test_vectors_1000x128.json"); + URL testQueries = IndexIT.class.getClassLoader().getResource("data/test_queries_100x128.csv"); + URL groundTruthValues = IndexIT.class.getClassLoader().getResource("data/test_ground_truth_l2_100.csv"); + assert testIndexVectors != null; + assert testQueries != null; + assert groundTruthValues != null; + testData = new TestUtils.TestData(testIndexVectors.getPath(), testQueries.getPath(), groundTruthValues.getPath()); + } + + @After + public void cleanUp() { + try { + deleteKNNIndex(INDEX_NAME); + } catch (Exception e) { + log.error(e); + } + } + + @SneakyThrows + public void testFaissHnsw_when1000Data_thenRecallIsAboveNinePointZero() { + // Create Index + createKnnHnswIndex(KNNEngine.FAISS, INDEX_NAME, FIELD_NAME, 128); + ingestTestData(INDEX_NAME, FIELD_NAME); + + int k = 100; + for (int i = 0; i < testData.queries.length; i++) { + List knnResults = runKnnQuery(INDEX_NAME, FIELD_NAME, testData.queries[i], k); + float recall = getRecall( + Set.of(Arrays.copyOf(testData.groundTruthValues[i], k)), + knnResults.stream().map(KNNResult::getDocId).collect(Collectors.toSet()) + ); + assertTrue("Recall: " + recall, recall > 0.9); + } + } + + private float getRecall(final Set truth, final Set result) { + // Count the number of relevant documents retrieved + result.retainAll(truth); + int relevantRetrieved = result.size(); + + // Total number of relevant documents + int totalRelevant = truth.size(); + + // Calculate recall + return (float) relevantRetrieved / totalRelevant; + } + + private List runKnnQuery(final String indexName, final String fieldName, final float[] queryVector, final int k) + throws Exception { + String query = KNNJsonQueryBuilder.builder() + .fieldName(fieldName) + .vector(ArrayUtils.toObject(queryVector)) + .k(k) + .build() + .getQueryString(); + Response response = searchKNNIndex(indexName, query, k); + return parseSearchResponse(EntityUtils.toString(response.getEntity()), fieldName); + } + + private void ingestTestData(final String indexName, final String fieldName) throws Exception { + // Index the test data + for (int i = 0; i < testData.indexData.docs.length; i++) { + addKnnDoc( + indexName, + Integer.toString(testData.indexData.docs[i]), + fieldName, + Floats.asList(testData.indexData.vectors[i]).toArray() + ); + } + + // Assert we have the right number of documents in the index + refreshAllIndices(); + assertEquals(testData.indexData.docs.length, getDocCount(indexName)); + } + + private void createKnnHnswIndex(final KNNEngine knnEngine, final String indexName, final String fieldName, final int dimension) + throws IOException { + KNNJsonIndexMappingsBuilder.Method method = KNNJsonIndexMappingsBuilder.Method.builder() + .methodName(METHOD_HNSW) + .spaceType(SpaceType.L2.getValue()) + .engine(knnEngine.getName()) + .build(); + + String knnIndexMapping = KNNJsonIndexMappingsBuilder.builder() + .fieldName(fieldName) + .dimension(dimension) + .vectorDataType(VectorDataType.FLOAT.getValue()) + .method(method) + .build() + .getIndexMapping(); + + createKnnIndex(indexName, knnIndexMapping); + } +} diff --git a/src/test/resources/data/README.md b/src/test/resources/data/README.md index f3711eae3..df103870f 100644 --- a/src/test/resources/data/README.md +++ b/src/test/resources/data/README.md @@ -10,6 +10,14 @@ test_queries_100x128.csv and packing 8 bits to 1 byte with ends up with 16 lengt For quantization technique, we calculated the median(49935.95941056451) of all values in test_vectors_1000x128.json and converted it as 0 if it is less than the median and 1 if it is equal to or larger than the median. +# test_ground_truth_binary_100.csv +The file contains the ground truth for the query test_queries_binary_100x128.csv against the data +test_vectors_binary_1000x128.json using hamming distance. + +# test_ground_truth_l2_100.csv +The file contains the ground truth for the query test_queries_100x128.csv against the data test_vectors_1000x128.json +using l2 distance + # test_vectors_nested_1000x128.json The file contains a simulated data to represent nested field. Consecutive ids are assigned for data from same parent document. diff --git a/src/test/resources/data/test_ground_truth_binary_100.csv b/src/test/resources/data/test_ground_truth_binary_100.csv new file mode 100644 index 000000000..bafc11a28 --- /dev/null +++ b/src/test/resources/data/test_ground_truth_binary_100.csv @@ -0,0 +1,100 @@ +76960303, 62740044, 2607799, 66428795, 37435892, 30998561, 60106217, 9575954, 43933006, 76703609, 7687278, 92033260, 94711842, 17081350, 54987042, 62693428, 45075471, 23848565, 49598724, 12571310, 61263068, 63152504, 16971929, 90004325, 6986898, 247198, 68694897, 16387575, 92803766, 41092172, 84840549, 79922758, 74137926, 91990218, 32250045, 34236719, 33061250, 22108665, 75407347, 34698463, 42307449, 16099750, 69786756, 71469330, 79136082, 4099191, 7182642, 57359924, 24314885, 18806856, 56484900, 20002147, 3294781, 8128637, 57803235, 46870723, 99224392, 94076128, 44550764, 2917920, 88904910, 45049308, 26664538, 19376156, 91240048, 60176618, 92787493, 4095116, 94516935, 91727510, 4434662, 68102437, 58749, 66045587, 9860968, 61271144, 36580610, 93515664, 20885148, 53802686, 97011160, 32159704, 6153406, 21993752, 38658347, 35456853, 3773993, 176257, 75135448, 24826575, 30090481, 68000591, 92867155, 29932657, 39171895, 76671482, 72738685, 39847321, 7685448, 86118021, 25636669, 8791066, 39373729, 34044787, 24915585, 79806380, 87720882, 50668599, 83302115, 72357096, 79322415, 67513640, 15536795, 53632373, 99515901, 48774913, 30771409, 99971982, 65851721, 50806615, 97641116, 53888755, 61141874, 19457589, 36753250, 43506672, 71083565, 26493119, 97281847, 8634541, 57241521, 17539625, 84684495, 40865610, 75128745, 73235980, 28796059, 30694952, 8651647, 42947632, 1788101, 51472275, 70036438, 4978543, 9860195, 16595436, 45794415, 1569515, 40027975, 13348726, 77377183, 7423788, 5640302, 68875490, 39801351, 1022677, 37183543, 93933709, 42967683, 3233569, 11543098, 84293052, 80555751, 35192533, 7955293, 83948335, 38365584, 75986488, 29188588, 27187213, 7646095, 73109997, 6808825, 18783702, 50007421, 95957797, 96726697, 16054533, 90654836, 37659250, 33935899, 56153345, 72373496, 73786944, 8733068, 1431742, 40781854, 56424103, 96193415, 74452589, 11581181, 99333375, 31727379, 26744917, 56955985, 37192445, 17385531, 22129328, 4064751, 40534591, 54058788, 94595668, 97057187, 61897582, 22721500, 5111370, 44627776, 39553046, 82339363, 9599614, 93057697, 83210802, 38645117, 72725103, 17764950, 16405341, 83533741, 75543508, 59371804, 44348328, 19939935, 21001913, 27185644, 76315420, 72019362, 3487592, 91141395, 58224549, 93270084, 66667729, 28787861, 508198, 4515343, 55602660, 78602717, 78549759, 98943869, 21919959, 70782102, 36933038, 54014062, 6157724, 15948937, 94539824, 24591705, 65715134, 21070110, 53648154, 40984766, 86460488, 30395570, 15064655, 19486173, 65880522, 76434144, 30569392, 86301513, 98739783, 82886381, 21289531, 37675718, 89217461, 16097038, 30543215, 66271566, 52293995, 8913721, 26063929, 63506281, 13468268, 80014588, 73617245, 84904436, 26507214, 44847298, 36845587, 415901, 85116755, 48658605, 51047803, 70372191, 32426752, 42073124, 12303248, 74724075, 18663507, 51464002, 7066775, 17146629, 33237508, 5970607, 43376279, 41481685, 7517032, 52060076, 40781449, 30811010, 74110882, 99021067, 23194618, 51466049, 12024238, 60393039, 24953498, 64055960, 60955663, 14731700, 66319530, 59501487, 66663942, 34497327, 9603598, 36930650, 77187825, 55669657, 86022504, 24168411, 87710366, 33565483, 17037369, 45381876, 84127901, 50702367, 17386996, 14349098, 82779622, 21397057, 44060493, 71657078, 79821897, 45995325, 16380211, 34432810, 44983451, 2717150, 74743862, 11365791, 82897371, 48893685, 9398733, 321665, 94911072, 62452034, 91281584, 95251277, 32274392, 31904591, 68824981, 32161669, 36139942, 40677414, 90310261, 76540481, 79942022, 97940276, 90272749, 44473167, 47361209, 21533347, 13231279, 26863229, 26998766, 78785507, 49882705, 28928175, 54762643, 7104732, 9623492, 44664587, 36312813, 68702632, 95581843, 53842979, 96735716, 2208785, 96420429, 26114953, 20568363, 85023028, 81274566, 89419466, 14093520, 75229462, 49236559, 36189527, 44915235, 92398073, 77272628, 17365188, 14626618, 62430984, 54663246, 7919588, 81853704, 53547802, 73392814, 22879907, 22942635, 62490109, 38009615, 38061439, 44177011, 37224844, 64602895, 78589145, 64157906, 47213183, 92071990, 63059024, 10649306, 92554120, 38341669, 31161687, 62051033, 17016156, 29959549, 82979980, 6111563, 42199455, 54427233, 874791, 84002370, 46851987, 19101477, 37560164, 77620120, 23110625, 92541302, 6793819, 57163802, 9188443, 98371444, 71300104, 63967300, 33249630, 29029316, 44889423, 2204165, 93562257, 23740167, 55960386, 7011964, 4508700, 58751351, 29834512, 31126490, 77898274, 61859143, 20122224, 28851716, 27625735, 65047700, 50188404, 29994197, 95726235, 47298834, 43152977, 55247455, 6819644, 67474219, 77284619, 30218878, 73021291, 62496012, 5822202, 77072625, 38256849, 68939068, 50567636, 39986008, 45848907, 26292919, 77413012, 36396314, 1250437, 86821229, 72278539, 4985896, 56515456, 72358170, 44846932, 36812683, 57020481, 77300457, 83368048, 79191827, 92530431, 99917599, 82327024, 10366309, 5267545, 69136837, 8115266, 13470059, 83651211, 45407418, 20642888, 9058407, 26102057, 85242963, 35996293, 80316608, 45790169, 45667668, 33431960, 94090109, 13173644, 29510992, 81855445, 58208470, 57961282, 98462867, 88698958, 96852321, 86001008, 83133790, 38510840, 16445503, 90457870, 87160386, 63372756, 54199160, 3880712, 62115552, 3183975, 81805959, 23569917, 75052463, 1936762, 47792865, 55470718, 14326617, 36468541, 22997281, 68128438, 1197320, 20836893, 15015906, 67227442, 3633375, 32699744, 65338021, 89499542, 69605283, 30891921, 99604946, 30787683, 61373987, 83789391, 87598888, 99861373, 93790285, 8729146, 45990383, 52613508, 8129978, 20765474, 75777973, 33123618, 77301523, 569864, 98653983, 17058722, 55189057, 82052050, 51149804, 16019925, 79880247, 2331773, 30463802, 48892201, 91957544, 73124510, 92692283, 89214616, 54868730, 99690194, 59177718, 22766820, 43322743, 77694700, 47708850, 70367851, 83747892, 41245325, 67281495, 56461322, 47090124, 89811711, 38022834, 77787724, 18131876, 16424199, 49641577, 36135, 32058615, 4091162, 69355476, 82532312, 27665211, 1204161, 4069912, 44479073, 72777973, 72238278, 9257405, 78300864, 83083131, 67030811, 11757872, 62762857, 97783876, 83269727, 45996863, 40686254, 96709982, 52261574, 82726008, 76330843, 96318094, 40152546, 73031054, 30163921, 6521313, 90090964, 92353856, 29065964, 65038678, 61960400, 54517921, 93566986, 90158785, 35512853, 51588015, 48673079, 14947650, 33797252, 33336148, 8373289, 19891772, 3233084, 25842078, 17957593, 33304202, 35092039, 17068582, 43357947, 51715482, 93359396, 99658235, 61623915, 10961421, 11923835, 48395186, 88653118, 79657802, 26776922, 42782652, 82427263, 57248122, 91802888, 6038457, 38494874, 91831487, 59981773, 83150534, 26392416, 84406788, 10358899, 9829782, 48675329, 6497830, 67451935, 18466635, 34667286, 27325428, 51590803, 89996536, 59614746, 84166196, 64098930, 53393358, 20486294, 73222868, 82178706, 62232211, 44844121, 15902805, 54263880, 70004753, 55753905, 98648327, 61928316, 44784505, 3235882, 78561158, 42644903, 62803858, 77312810, 33553959, 23134715, 7300047, 60102965, 48088883, 4204661, 53666583, 26275734, 58180653, 40197395, 36685023, 66322959, 65081429, 4798568, 72614359, 62936963, 36505482, 37739481, 61741594, 49597667, 41380093, 34295794, 15535065, 9175338, 18411915, 92604458, 112651, 92998591, 37620363, 51426311, 56473732, 99297164, 41442762, 71316369, 59329511, 91664334, 88047921, 59910624, 81774825, 68316156, 82651278, 4119747, 54232247, 92692978, 49271185, 63015256, 16684829, 68204242, 45919976, 10264691, 31491938, 99524975, 86242799, 4668450, 98948034, 61982238, 70420215, 33201905, 89699445, 46540998, 34946859, 32151165, 168541, 56531125, 526217, 18833224, 43501211, 74614639, 84349107, 59109400, 23432750, 99549373, 22435353, 68041839, 45617087, 97561745, 30139692, 29516592, 69579137, 92215320, 75820087, 8099994, 75153252, 53084256, 34698428, 66611759, 62357986, 8055981, 33895336, 60430369, 68816413, 95010552, 88444207, 26397786, 13862149, 63628376, 22200378, 4199704, 37957788, 15075176, 65454636, 11161731, 72495719, 15163258, 30764367, 69848388, 26734892, 24058273, 70596786, 19898053, 55770687, 68110330, 12416768, 24619760, 76170907, 67031644, 43045786, 60581278, 47887470, 86543538, 94360702, 99729357, 48260151, 55615885, 78909561, 61815223, 29635537, 42692881, 12348118, 71522968, 36808486, 18504197, 99965001, 33699435, 22113133, 27041967, 52204879, 78766358, 85711894, 71920426, 18699206, 91938191, 19272365, 86798033, 65074535, 14363867, 29401781, 85571389, 39201414, 16567550, 1375023, 20645197, 20140249, 57658654, 37166608, 12664567, 57240218, 83155442, 17727650, 29819940, 5832946, 10453030, 6871053, 10597197, 67793644, 66832478, 669105, 91255408, 14220886, 8696647, 65017137, 76825057, 22405120, 26139110, 80251430, 78218845, 3509435, 69641513, 71965942, 72274002, 88251446, 65271999, 45237957, 3088684, 49328608, 7229550, 51315460, 9259676, 66903004, 42237907, 57393458, 37280276, 64848072, 47893286, 749283, 28734791, 17976208, 90061527, 81677380, 38008118, 6525948, 61712234, 78884452, 85695762, 80246713, 8807940, 64087743, 89046466, 20867149, 70727211, 31733363, 88130087, 62552524, 65275241, 73168565, 61728685, 55850790, 72793444, 85117092, 30653863, 32590267, 45428665, 66885828, 44245960, 6505939, 70541760, 68644627, 29466406, 55143724, 40356877, 74441448, 36780454, 89078848, 69697787, 45306070, 9886593, 44481640, 61859581, 4787945, 4806458, 70800879, 27411561, 2891150, 18001617, 90013093, 17894977, 95290172, 59405277, 14723410, 98130363, 17857111, 81898046, 21673260, 79738755, 59197747, 69255765, 66116458, 95395112, 13819358, 89804152, 51507425, 5073754, 81172706, 37501808, 97379790, 824872, 99226875, 62428472, 84187166, 37891451, 67124282, 89637706, 15148031, 66250369, 33628349, 67084351, 30366150, 10309525, 1239555, 71333116, 74357852, 14045383, 54606384, 7502255, 59318837, 66137019, 93053405, 22450468, 20681921, 48360198, 72732205, 41092102, 99125126, 24733232, 28550822, 34493392, 96906410, 47738185, 5482538, 6221471, 44842615 +36930650, 18466635, 82052050, 5267545, 36139942, 98943869, 42644903, 73124510, 41245325, 10597197, 80316608, 84840549, 23569917, 30787683, 1569515, 76434144, 77377183, 77312810, 30543215, 79136082, 18783702, 16684829, 31727379, 56531125, 39553046, 97940276, 17894977, 68702632, 48675329, 44915235, 17976208, 30764367, 73222868, 40027975, 17016156, 3233569, 36685023, 99729357, 874791, 80555751, 78766358, 30811010, 76960303, 56484900, 24953498, 14731700, 26292919, 83155442, 47738185, 40152546, 247198, 50806615, 5111370, 74614639, 45075471, 83368048, 13470059, 17764950, 68041839, 80251430, 13231279, 78785507, 10961421, 28796059, 62115552, 14326617, 61271144, 36933038, 1788101, 32250045, 72495719, 81677380, 14626618, 24591705, 19898053, 81898046, 22942635, 15902805, 38061439, 55753905, 78589145, 64157906, 33123618, 1022677, 89217461, 99690194, 63967300, 93562257, 71316369, 18131876, 74357852, 24915585, 83302115, 79322415, 87710366, 56955985, 46870723, 17727650, 48774913, 94076128, 99971982, 669105, 82897371, 48893685, 9886593, 88904910, 20642888, 45667668, 90272749, 29516592, 3509435, 35092039, 62357986, 33895336, 66045587, 30694952, 83150534, 26397786, 37280276, 6157724, 16595436, 22879907, 62490109, 38009615, 33061250, 59197747, 52613508, 92071990, 77301523, 51149804, 79880247, 55615885, 62936963, 30463802, 98371444, 48658605, 22766820, 2204165, 16424199, 82532312, 65047700, 92033260, 4069912, 85571389, 8733068, 12024238, 60393039, 60955663, 73021291, 9603598, 97783876, 37166608, 89699445, 50567636, 45381876, 45996863, 29819940, 40534591, 45995325, 34432810, 97057187, 61897582, 2717150, 11365791, 2917920, 45306070, 56515456, 19457589, 18833224, 99917599, 93566986, 9599614, 19376156, 92803766, 60176618, 8115266, 48673079, 4434662, 49598724, 47361209, 44348328, 84684495, 3233084, 72019362, 99658235, 40865610, 91141395, 48395186, 54762643, 73235980, 66667729, 95581843, 3183975, 33628349, 20568363, 9860968, 89419466, 75229462, 10358899, 65454636, 53802686, 34236719, 81853704, 67227442, 78884452, 21070110, 35456853, 21673260, 64087743, 24619760, 8729146, 30090481, 98648327, 43045786, 62803858, 75777973, 55850790, 16097038, 7300047, 98653983, 58180653, 85117092, 76703609, 63506281, 13468268, 2331773, 36845587, 78909561, 72738685, 18411915, 7685448, 89214616, 42073124, 12303248, 12348118, 18663507, 71469330, 67281495, 7066775, 41442762, 52204879, 91664334, 96726697, 7517032, 28851716, 92692978, 74110882, 27665211, 24314885, 63015256, 50668599, 23194618, 40356877, 16567550, 37435892, 78300864, 66319530, 3294781, 62496012, 59501487, 34497327, 74452589, 54606384, 62428472, 26744917, 15536795, 36396314, 53632373, 6986898, 65851721, 168541, 4985896, 6521313, 67124282, 89637706, 94911072, 44846932, 43501211, 79191827, 38645117, 76825057, 90158785, 23432750, 92787493, 70800879, 75543508, 59371804, 33336148, 18001617, 30139692, 8634541, 57241521, 13173644, 72274002, 3487592, 63372756, 66611759, 58224549, 45237957, 36312813, 30998561, 96420429, 79922758, 91802888, 78602717, 78549759, 81274566, 68816413, 60106217, 95290172, 95010552, 36580610, 22200378, 51472275, 28734791, 67451935, 20885148, 90061527, 17365188, 22997281, 34493392, 38008118, 64098930, 15015906, 24058273, 73392814, 30891921, 12416768, 30395570, 70004753, 37224844, 64602895, 62552524, 9575954, 44784505, 47213183, 78561158, 10649306, 98739783, 38341669, 73168565, 61263068, 569864, 43933006, 72793444, 76671482, 66271566, 8913721, 48260151, 65081429, 46851987, 32590267, 37560164, 23110625, 29635537, 57163802, 9188443, 16099750, 51047803, 96906410, 54868730, 112651, 66885828, 27187213, 29029316, 6808825, 56473732, 8791066, 4508700, 39373729, 38022834, 2607799, 16054533, 68316156, 52060076, 66428795, 99021067, 68204242, 9257405, 72373496, 51466049, 47298834, 1431742, 20002147, 99226875, 99333375, 84127901, 57803235, 17385531, 99515901, 4064751, 5832946, 34946859, 59318837, 30163921, 90090964, 74743862, 8696647, 61141874, 82339363, 23848565, 40677414, 59109400, 9058407, 83533741, 2891150, 22435353, 93053405, 97561745, 92215320, 69641513, 88698958, 96852321, 86001008, 17957593, 83133790, 76315420, 16445503, 61623915, 9623492, 44664587, 66250369, 42782652, 60430369, 81805959, 26114953, 51315460, 85023028, 21919959, 26392416, 63628376, 4978543, 9860195, 11161731, 92398073, 10309525, 15948937, 77272628, 84166196, 6525948, 14723410, 26734892, 55770687, 69605283, 38658347, 44844121, 40984766, 54263880, 44177011, 61373987, 70727211, 75135448, 65880522, 13348726, 45990383, 69255765, 13819358, 29959549, 33553959, 11543098, 42307449, 84904436, 26507214, 72614359, 36505482, 19101477, 48892201, 415901, 92692283, 15535065, 75986488, 9175338, 74724075, 7646095, 36808486, 63152504, 18504197, 99965001, 99297164, 58751351, 22113133, 71333116, 29466406, 29834512, 43376279, 31126490, 88047921, 85711894, 41481685, 32058615, 77898274, 57359924, 20122224, 90654836, 27625735, 72732205, 37659250, 54232247, 7687278, 72777973, 95726235, 94711842, 1375023, 55247455, 86242799, 67030811, 98948034, 5822202, 96193415, 77072625, 57658654, 77187825, 38256849, 86022504, 36780454, 33201905, 17081350, 82779622, 7502255, 54058788, 32151165, 79821897, 94595668, 73031054, 54987042, 72358170, 44627776, 91281584, 36753250, 45049308, 16387575, 61960400, 32274392, 68824981, 24733232, 82327024, 61859581, 93057697, 10366309, 91240048, 69136837, 4787945, 35512853, 4806458, 16405341, 4095116, 79942022, 94516935, 66137019, 26139110, 33797252, 19891772, 98462867, 25842078, 21001913, 51715482, 93359396, 68102437, 75128745, 65271999, 93270084, 3088684, 54199160, 79657802, 53842979, 82427263, 4515343, 55602660, 6038457, 74137926, 91831487, 59981773, 42947632, 70782102, 88444207, 67084351, 30366150, 54014062, 64848072, 9829782, 749283, 4199704, 34667286, 94539824, 69848388, 1197320, 7919588, 61712234, 3633375, 53393358, 70596786, 85695762, 21993752, 89046466, 87598888, 15064655, 76170907, 12571310, 24826575, 88130087, 7423788, 95395112, 20765474, 61728685, 60581278, 82886381, 93933709, 39171895, 17058722, 52293995, 26063929, 66322959, 54427233, 80014588, 84002370, 44847298, 83948335, 38365584, 70372191, 33249630, 44245960, 81172706, 71522968, 37620363, 83747892, 70541760, 16971929, 49641577, 90004325, 40781449, 71920426, 33935899, 49271185, 72238278, 39201414, 73786944, 45919976, 99524975, 43152977, 64055960, 83083131, 4668450, 77284619, 62762857, 41092102, 11581181, 17037369, 67513640, 39986008, 77413012, 57240218, 82726008, 21397057, 44060493, 30771409, 71657078, 66832478, 44983451, 72278539, 68694897, 9398733, 62452034, 65017137, 36812683, 57020481, 95251277, 92530431, 44481640, 26664538, 84349107, 51588015, 41092172, 22405120, 85242963, 14947650, 91727510, 26493119, 44473167, 8373289, 94090109, 21533347, 58208470, 57961282, 38510840, 27185644, 33304202, 43357947, 90457870, 22450468, 88251446, 28550822, 75153252, 53084256, 11923835, 34698428, 88653118, 49328608, 508198, 57248122, 9259676, 75052463, 8651647, 14093520, 57393458, 13862149, 70036438, 93515664, 37957788, 27325428, 15163258, 59614746, 54663246, 65715134, 65338021, 80246713, 3773993, 19486173, 5482538, 61928316, 86301513, 8129978, 63059024, 68875490, 31161687, 29932657, 21289531, 47887470, 37675718, 94360702, 60102965, 4204661, 26275734, 42199455, 55189057, 34698463, 51507425, 84293052, 73617245, 30653863, 37739481, 35192533, 1239555, 91957544, 92541302, 34295794, 29188588, 6793819, 71300104, 92604458, 44889423, 47708850, 23740167, 86118021, 50007421, 56461322, 33699435, 17146629, 7182642, 55143724, 61859143, 4119747, 91938191, 1204161, 19272365, 86798033, 65074535, 14363867, 44479073, 50188404, 56153345, 87720882, 29401781, 31491938, 74441448, 67474219, 30218878, 66663942, 61982238, 70420215, 24168411, 84187166, 37192445, 45848907, 17386996, 40686254, 1250437, 76330843, 14349098, 46540998, 37891451, 97641116, 14220886, 62693428, 321665, 99125126, 29065964, 526217, 65038678, 54517921, 32161669, 83210802, 90310261, 83651211, 45407418, 76540481, 97281847, 29510992, 69579137, 81855445, 26863229, 71965942, 90013093, 17068582, 8099994, 87160386, 58749, 6221471, 7104732, 3880712, 7229550, 38494874, 47792865, 55470718, 36468541, 42237907, 36189527, 51590803, 32159704, 98130363, 53547802, 6153406, 20486294, 62232211, 99604946, 176257, 83789391, 3235882, 75407347, 30569392, 5640302, 39801351, 65275241, 62051033, 92867155, 37183543, 42967683, 86543538, 53666583, 40197395, 62740044, 49597667, 45428665, 39847321, 85116755, 69786756, 59177718, 92998591, 77694700, 51426311, 4099191, 47090124, 34044787, 59329511, 95957797, 14045383, 59910624, 97379790, 36135, 81774825, 82651278, 69355476, 79806380, 29994197, 10264691, 20645197, 6819644, 824872, 56424103, 11757872, 33565483, 50702367, 6871053, 16380211, 67793644, 53888755, 44550764, 77300457, 43506672, 31904591, 44842615, 26102057, 35996293, 45790169, 17539625, 75820087, 19939935, 49882705, 8055981, 2208785, 1936762, 66903004, 49236559, 97011160, 62430984, 17857111, 45794415, 53648154, 8807940, 86460488, 99861373, 48360198, 22108665, 92554120, 82979980, 23134715, 6111563, 16019925, 7955293, 43322743, 42692881, 5073754, 73109997, 6505939, 25636669, 33237508, 68644627, 5970607, 18806856, 40781854, 83269727, 8128637, 89078848, 52261574, 10453030, 91255408, 69697787, 72725103, 99549373, 45617087, 33431960, 26998766, 28928175, 91990218, 47893286, 6497830, 15075176, 59405277, 68128438, 20681921, 82178706, 20867149, 31733363, 93790285, 67031644, 48088883, 89804152, 4798568, 61815223, 61741594, 41380093, 77620120, 32426752, 70367851, 51464002, 55960386, 89811711, 27041967, 4091162, 18699206, 55669657, 22129328, 96318094, 86821229, 78218845, 89996536, 32699744, 68110330, 79738755, 68000591, 37501808, 7011964, 77787724, 72357096, 68939068, 96709982, 99224392, 22721500, 92353856, 15148031, 27411561, 20836893, 89499542, 66116458, 28787861, 26776922, 12664567, 71083565, 96735716, 20140249, 84406788 +15064655, 56461322, 44846932, 92803766, 21070110, 4069912, 68204242, 30218878, 62428472, 54987042, 75153252, 48395186, 78549759, 73168565, 51507425, 57163802, 87710366, 97641116, 74614639, 19891772, 81855445, 78785507, 26776922, 47792865, 90061527, 72495719, 92071990, 33553959, 42967683, 30543215, 63506281, 46851987, 9188443, 99690194, 59177718, 47708850, 68644627, 88047921, 55143724, 82532312, 72777973, 83302115, 24953498, 37166608, 89637706, 83368048, 5267545, 23432750, 13231279, 3233084, 51715482, 45237957, 26114953, 34236719, 6153406, 73392814, 62232211, 12571310, 77377183, 37675718, 43933006, 17058722, 8913721, 48260151, 13468268, 874791, 92604458, 12348118, 71522968, 71469330, 41245325, 7066775, 71316369, 4119747, 72238278, 45919976, 1431742, 67030811, 99226875, 86022504, 17386996, 5832946, 10597197, 669105, 62452034, 93566986, 24733232, 82327024, 40677414, 4095116, 83533741, 97940276, 91727510, 80316608, 33431960, 8099994, 22450468, 61623915, 10961421, 33895336, 62115552, 9860968, 14326617, 36468541, 61271144, 51472275, 48675329, 3633375, 32699744, 30891921, 38009615, 30787683, 55753905, 22108665, 5640302, 31161687, 21289531, 77312810, 94360702, 66322959, 54427233, 55615885, 4798568, 32590267, 6793819, 48658605, 73109997, 70541760, 56473732, 7011964, 99297164, 33237508, 59910624, 7517032, 69355476, 79806380, 7687278, 65047700, 63015256, 12024238, 43152977, 60955663, 66319530, 56424103, 77072625, 97783876, 84187166, 26292919, 36396314, 57240218, 1250437, 22129328, 82779622, 79821897, 6871053, 45995325, 67793644, 168541, 50806615, 72278539, 4985896, 90090964, 44550764, 48893685, 45306070, 95251277, 61960400, 32274392, 68824981, 59109400, 4787945, 45407418, 92787493, 76540481, 26102057, 35996293, 93053405, 45667668, 97561745, 17539625, 96852321, 17894977, 53084256, 6221471, 53842979, 49328608, 60430369, 33628349, 81274566, 14093520, 88444207, 63628376, 749283, 37957788, 10309525, 32250045, 15163258, 20836893, 81853704, 38658347, 22879907, 22942635, 40027975, 37224844, 78589145, 52613508, 78561158, 66116458, 63059024, 33123618, 7300047, 42199455, 11543098, 99729357, 84293052, 36845587, 78909561, 23110625, 72738685, 29188588, 89214616, 70372191, 32426752, 44889423, 44245960, 36808486, 83747892, 50007421, 33699435, 27041967, 74357852, 16424199, 41481685, 81774825, 68316156, 40781449, 28851716, 27625735, 72732205, 92692978, 71920426, 24314885, 33935899, 87720882, 40356877, 47298834, 1375023, 14731700, 6819644, 62496012, 59501487, 66663942, 96193415, 74452589, 54606384, 45996863, 83155442, 44060493, 29819940, 54058788, 59318837, 16380211, 61897582, 86821229, 68694897, 2917920, 56515456, 5111370, 94911072, 9886593, 91281584, 43506672, 43501211, 32161669, 61859581, 19376156, 69136837, 41092172, 85242963, 66137019, 2891150, 26139110, 59371804, 8373289, 8634541, 13173644, 21533347, 35092039, 16445503, 72019362, 28928175, 73235980, 44664587, 54199160, 7229550, 42947632, 21919959, 95290172, 67451935, 59405277, 15948937, 27325428, 81677380, 34493392, 59614746, 67227442, 85695762, 35456853, 73222868, 21673260, 80246713, 70004753, 75135448, 48360198, 64157906, 13348726, 69255765, 75407347, 86301513, 43045786, 10649306, 20765474, 75777973, 29959549, 61263068, 77301523, 93933709, 48088883, 76671482, 66271566, 42307449, 80555751, 62936963, 37739481, 61741594, 415901, 41380093, 75986488, 9175338, 85116755, 18411915, 12303248, 77694700, 79136082, 23740167, 17146629, 95957797, 29466406, 18131876, 96726697, 20122224, 24915585, 90654836, 74110882, 19272365, 50188404, 23194618, 72373496, 37435892, 55247455, 86242799, 40781854, 77284619, 3294781, 34497327, 57658654, 79322415, 41092102, 17037369, 56955985, 15536795, 6986898, 50702367, 40686254, 14349098, 71657078, 40152546, 94076128, 44983451, 11365791, 92353856, 56531125, 36812683, 57020481, 36753250, 79191827, 31904591, 91240048, 38645117, 8115266, 35512853, 48673079, 27411561, 75543508, 4434662, 33797252, 68041839, 90272749, 57241521, 29510992, 92215320, 3509435, 88698958, 25842078, 17957593, 38510840, 33304202, 43357947, 90457870, 99658235, 28550822, 54762643, 7104732, 68702632, 30998561, 2208785, 55602660, 3183975, 74137926, 78602717, 23569917, 66903004, 85023028, 59981773, 36580610, 54014062, 10358899, 22200378, 9829782, 11161731, 22997281, 14626618, 32159704, 26734892, 17857111, 7919588, 16595436, 65715134, 24058273, 45794415, 62490109, 15902805, 12416768, 38061439, 83789391, 59197747, 19486173, 3235882, 8129978, 7423788, 68875490, 95395112, 62803858, 61728685, 47887470, 1022677, 16097038, 60102965, 58180653, 51149804, 85117092, 80014588, 73617245, 30653863, 36505482, 48892201, 73124510, 29635537, 16099750, 7685448, 71300104, 42073124, 63967300, 33249630, 27187213, 93562257, 63152504, 6808825, 67281495, 55960386, 86118021, 25636669, 4508700, 89811711, 5970607, 78766358, 85711894, 97379790, 49641577, 32058615, 61859143, 52060076, 27665211, 86798033, 76960303, 99021067, 16567550, 8733068, 94711842, 31491938, 56484900, 74441448, 67474219, 72357096, 5822202, 36930650, 24168411, 89699445, 11581181, 39986008, 45381876, 77413012, 53632373, 84127901, 17385531, 76330843, 17727650, 47738185, 34946859, 10453030, 37891451, 247198, 34432810, 99971982, 97057187, 73031054, 74743862, 69697787, 9398733, 99125126, 61141874, 19457589, 92530431, 82339363, 23848565, 90158785, 17764950, 9058407, 70800879, 45790169, 49598724, 33336148, 30139692, 29516592, 58208470, 69641513, 84684495, 19939935, 83133790, 72274002, 76315420, 88251446, 11923835, 34698428, 66611759, 88653118, 62357986, 9623492, 79657802, 96420429, 57248122, 51315460, 75052463, 38494874, 1936762, 68816413, 89419466, 75229462, 83150534, 26397786, 49236559, 70036438, 4199704, 36189527, 6497830, 93515664, 28734791, 4978543, 17976208, 15075176, 9860195, 51590803, 68128438, 62430984, 54663246, 6525948, 61712234, 19898053, 55770687, 21993752, 81898046, 8807940, 68110330, 64087743, 79738755, 24826575, 65880522, 76434144, 67031644, 61928316, 44784505, 47213183, 98739783, 42644903, 65275241, 569864, 89217461, 37183543, 86543538, 39171895, 53666583, 26275734, 82052050, 52293995, 76703609, 26063929, 16019925, 84904436, 65081429, 26507214, 62740044, 7955293, 19101477, 91957544, 38365584, 37560164, 92692283, 77620120, 92541302, 34295794, 98371444, 51047803, 54868730, 112651, 92998591, 42692881, 5073754, 81172706, 7646095, 51426311, 99965001, 18783702, 58751351, 16971929, 91664334, 16054533, 82651278, 4091162, 54232247, 49271185, 92033260, 14363867, 16684829, 56153345, 39201414, 10264691, 95726235, 20645197, 20002147, 73021291, 70420215, 9603598, 38256849, 33201905, 31727379, 68939068, 8128637, 96709982, 52261574, 99224392, 48774913, 7502255, 40534591, 32151165, 96318094, 66832478, 91255408, 6521313, 67124282, 321665, 72358170, 16387575, 39553046, 15148031, 26664538, 9599614, 84349107, 83210802, 90310261, 51588015, 99549373, 4806458, 20642888, 94090109, 69579137, 47361209, 57961282, 75820087, 44348328, 98462867, 27185644, 93359396, 87160386, 40865610, 91141395, 93270084, 66250369, 36312813, 28796059, 82427263, 81805959, 98943869, 70782102, 1788101, 91990218, 57393458, 84406788, 47893286, 6157724, 65454636, 18466635, 34667286, 97011160, 89996536, 30764367, 64098930, 14723410, 98130363, 24591705, 20681921, 53547802, 65338021, 70596786, 78884452, 69605283, 20486294, 82178706, 30395570, 24619760, 176257, 64602895, 88130087, 30569392, 92554120, 92867155, 60581278, 82886381, 17016156, 4204661, 3233569, 72793444, 34698463, 79880247, 72614359, 44847298, 35192533, 61815223, 39847321, 22766820, 66885828, 4099191, 41442762, 39373729, 71333116, 29834512, 14045383, 90004325, 30811010, 37659250, 50668599, 29994197, 51466049, 99524975, 78300864, 4668450, 98948034, 20140249, 99333375, 50567636, 67513640, 45848907, 57803235, 46870723, 4064751, 30771409, 46540998, 94595668, 30163921, 22721500, 2717150, 65017137, 88904910, 44627776, 29065964, 77300457, 18833224, 99917599, 44481640, 36139942, 76825057, 14947650, 26493119, 45617087, 97281847, 80251430, 78218845, 26863229, 21001913, 90013093, 17068582, 84840549, 68102437, 75128745, 3487592, 58749, 95581843, 28787861, 3880712, 508198, 4515343, 6038457, 9259676, 20568363, 8651647, 67084351, 42237907, 30366150, 13862149, 64848072, 44915235, 20885148, 17365188, 94539824, 38008118, 1197320, 15015906, 86460488, 54263880, 61373987, 70727211, 31733363, 5482538, 45990383, 13819358, 55850790, 89804152, 98653983, 30463802, 83948335, 15535065, 43322743, 29029316, 2204165, 37620363, 6505939, 22113133, 59329511, 77787724, 52204879, 36135, 57359924, 18699206, 65074535, 29401781, 85571389, 73786944, 64055960, 83083131, 11757872, 77187825, 83269727, 26744917, 37192445, 89078848, 17081350, 82726008, 82897371, 62693428, 45049308, 526217, 65038678, 10366309, 60176618, 83651211, 79942022, 94516935, 22435353, 18001617, 86001008, 3088684, 66045587, 79922758, 91802888, 95010552, 37280276, 92398073, 44844121, 99604946, 20867149, 87598888, 99861373, 76170907, 98648327, 68000591, 62552524, 9575954, 39801351, 62051033, 29932657, 23134715, 6111563, 2331773, 84002370, 1239555, 49597667, 45428665, 69786756, 74724075, 18663507, 37501808, 47090124, 8791066, 7182642, 77898274, 91938191, 1204161, 60393039, 824872, 61982238, 55669657, 36780454, 14220886, 8696647, 45075471, 44842615, 72725103, 22405120, 16405341, 44473167, 63372756, 65271999, 66667729, 26392416, 89046466, 1569515, 8729146, 30090481, 38341669, 82979980, 55189057, 40197395, 36685023, 70367851, 51464002, 18504197, 34044787, 2607799, 43376279, 44479073, 9257405, 18806856, 62762857, 65851721, 71083565, 13470059, 26998766, 58224549, 96735716, 30694952, 55470718, 77272628, 69848388, 53648154, 3773993, 93790285, 66428795, 33565483, 54517921, 93057697, 71965942, 49882705, 8055981, 42782652, 60106217, 36933038, 53802686, 84166196, 53393358, 89499542, 40984766, 33061250, 44177011, 38022834, 31126490, 99515901, 53888755, 91831487, 96906410, 12664567, 21397057 +3183975, 20836893, 61982238, 45049308, 9623492, 96420429, 89419466, 37675718, 4798568, 61741594, 83302115, 83083131, 40781854, 86821229, 82339363, 82327024, 69136837, 2891150, 29510992, 38510840, 22450468, 78785507, 34698428, 3088684, 82427263, 79922758, 57393458, 99604946, 20867149, 31733363, 19101477, 6793819, 7646095, 90004325, 39201414, 39986008, 96318094, 44983451, 2717150, 6521313, 8115266, 90158785, 72725103, 76540481, 8373289, 3509435, 83133790, 49882705, 75153252, 3487592, 10961421, 8055981, 79657802, 91802888, 6038457, 51315460, 4978543, 98130363, 20681921, 70596786, 68110330, 70727211, 13819358, 77301523, 26275734, 30543215, 11543098, 30463802, 48892201, 49597667, 42692881, 73109997, 77787724, 71333116, 91664334, 14045383, 54232247, 33935899, 91938191, 65047700, 45919976, 6819644, 30218878, 73021291, 72357096, 86022504, 89699445, 99333375, 15536795, 22129328, 14349098, 5832946, 97057187, 91281584, 65038678, 61960400, 26664538, 38645117, 70800879, 27411561, 35996293, 91727510, 80316608, 26493119, 18001617, 13231279, 44348328, 17957593, 68102437, 99658235, 91141395, 73235980, 38494874, 9860968, 95290172, 36933038, 51472275, 36189527, 20885148, 45794415, 55770687, 44844121, 12416768, 92071990, 68875490, 10649306, 65275241, 73168565, 47887470, 89804152, 82052050, 52293995, 65081429, 35192533, 57163802, 98371444, 32426752, 112651, 63967300, 51464002, 55960386, 33699435, 33237508, 27041967, 55143724, 81774825, 52060076, 20122224, 99021067, 23194618, 10264691, 16567550, 12024238, 1375023, 43152977, 74441448, 78300864, 20002147, 67030811, 824872, 57658654, 79322415, 26292919, 36396314, 82779622, 7502255, 6871053, 22721500, 321665, 72358170, 99125126, 44846932, 32161669, 61859581, 10366309, 99549373, 45407418, 97940276, 68041839, 90272749, 80251430, 88698958, 96852321, 3233084, 25842078, 33304202, 90013093, 43357947, 28550822, 61623915, 28796059, 53842979, 4515343, 60430369, 91831487, 85023028, 14326617, 42237907, 84406788, 10358899, 47893286, 67451935, 92398073, 32250045, 27325428, 17365188, 89996536, 54663246, 64098930, 14723410, 35456853, 62232211, 86460488, 33061250, 89046466, 61373987, 76170907, 48360198, 30090481, 75407347, 30569392, 86301513, 8129978, 98739783, 92554120, 92867155, 55850790, 61263068, 77312810, 569864, 72793444, 42199455, 40197395, 36685023, 26063929, 16019925, 66322959, 79880247, 46851987, 26507214, 37739481, 61815223, 415901, 39847321, 9188443, 12303248, 66885828, 44245960, 37620363, 36808486, 23740167, 56473732, 7011964, 95957797, 52204879, 7182642, 59910624, 49641577, 24915585, 69355476, 19272365, 65074535, 29994197, 37435892, 4668450, 77284619, 59501487, 96193415, 70420215, 11757872, 41092102, 33565483, 37192445, 12664567, 57240218, 17386996, 40686254, 96709982, 1250437, 17727650, 29819940, 71657078, 46540998, 54058788, 40152546, 65851721, 73031054, 67793644, 30163921, 4985896, 44550764, 11365791, 2917920, 36812683, 57020481, 16387575, 526217, 20642888, 85242963, 83533741, 93053405, 4434662, 29516592, 78218845, 69579137, 47361209, 76315420, 8099994, 72019362, 53084256, 62357986, 45237957, 36312813, 3880712, 74137926, 26114953, 59981773, 30366150, 22200378, 6497830, 15075176, 9860195, 34667286, 90061527, 94539824, 32159704, 59614746, 1197320, 15015906, 89499542, 53648154, 73222868, 81898046, 21673260, 44177011, 1569515, 70004753, 93790285, 37224844, 12571310, 19486173, 5482538, 22108665, 62552524, 45990383, 47213183, 66116458, 7423788, 39801351, 38341669, 29959549, 89217461, 33553959, 93933709, 42967683, 86543538, 94360702, 39171895, 98653983, 17058722, 8913721, 30653863, 84002370, 78909561, 91957544, 38365584, 37560164, 92692283, 77620120, 23110625, 72738685, 99690194, 92998591, 43322743, 77694700, 81172706, 71469330, 7066775, 34044787, 22113133, 5970607, 74357852, 90654836, 27625735, 24314885, 18806856, 98948034, 20140249, 66663942, 56424103, 77072625, 77187825, 37166608, 62428472, 45848907, 45996863, 6986898, 4064751, 48774913, 10453030, 59318837, 16380211, 34432810, 82897371, 9398733, 9886593, 36753250, 77300457, 18833224, 43506672, 32274392, 45075471, 31904591, 68824981, 24733232, 44842615, 4787945, 51588015, 4806458, 41092172, 9058407, 26102057, 45790169, 44473167, 33431960, 21533347, 57961282, 69641513, 84684495, 26863229, 86001008, 19939935, 71965942, 58749, 54762643, 7104732, 66250369, 68702632, 508198, 2208785, 78602717, 23569917, 60106217, 75229462, 95010552, 67084351, 63628376, 37280276, 48675329, 44915235, 17976208, 53802686, 15948937, 72495719, 81677380, 62430984, 15163258, 84166196, 38008118, 17857111, 24591705, 53393358, 38658347, 82178706, 62490109, 40984766, 24619760, 176257, 87598888, 15064655, 67031644, 98648327, 44784505, 3235882, 42644903, 31161687, 62803858, 61728685, 6111563, 7300047, 53666583, 58180653, 55189057, 34698463, 63506281, 80555751, 7955293, 83948335, 92541302, 29188588, 9175338, 85116755, 18411915, 89214616, 12348118, 37501808, 93562257, 6808825, 47090124, 17146629, 58751351, 68644627, 59329511, 2607799, 36135, 72732205, 92692978, 1204161, 66428795, 16684829, 56153345, 50668599, 51466049, 1431742, 60955663, 67474219, 5822202, 99226875, 97783876, 33201905, 83269727, 68939068, 77413012, 57803235, 17385531, 99515901, 47738185, 91255408, 14220886, 90090964, 74743862, 8696647, 56531125, 65017137, 44627776, 5267545, 83210802, 22405120, 26139110, 75543508, 59371804, 33797252, 97561745, 17539625, 81855445, 98462867, 72274002, 17068582, 90457870, 40865610, 33895336, 49328608, 62115552, 9259676, 47792865, 36468541, 70782102, 61271144, 88444207, 36580610, 91990218, 13862149, 64848072, 9829782, 4199704, 59405277, 65454636, 10309525, 22997281, 14626618, 68128438, 34236719, 6525948, 26734892, 3633375, 32699744, 6153406, 78884452, 22879907, 22942635, 64087743, 99861373, 79738755, 55753905, 24826575, 76434144, 13348726, 61928316, 9575954, 52613508, 43045786, 29932657, 60581278, 82886381, 33123618, 1022677, 16097038, 48088883, 3233569, 76671482, 76703609, 13468268, 874791, 84293052, 44847298, 1239555, 34295794, 15535065, 16099750, 7685448, 51047803, 54868730, 42073124, 22766820, 29029316, 71522968, 18663507, 63152504, 6505939, 79136082, 18504197, 70541760, 86118021, 4099191, 99297164, 89811711, 71316369, 38022834, 43376279, 32058615, 7517032, 77898274, 61859143, 4091162, 28851716, 37659250, 4119747, 82532312, 27665211, 14363867, 44479073, 50188404, 87720882, 72238278, 94711842, 31491938, 56484900, 55247455, 64055960, 86242799, 9603598, 24168411, 87710366, 26744917, 53632373, 52261574, 99224392, 34946859, 97641116, 72278539, 48893685, 69697787, 67124282, 89637706, 29065964, 43501211, 71083565, 83368048, 92530431, 44481640, 15148031, 93057697, 19376156, 84349107, 59109400, 76825057, 23432750, 35512853, 17764950, 16405341, 4095116, 79942022, 94516935, 14947650, 22435353, 13173644, 58208470, 21001913, 27185644, 35092039, 16445503, 87160386, 84840549, 88251446, 11923835, 65271999, 66611759, 48395186, 88653118, 54199160, 26776922, 30998561, 96735716, 42782652, 57248122, 81805959, 30694952, 33628349, 66903004, 42947632, 14093520, 1788101, 26397786, 49236559, 749283, 51590803, 77272628, 61712234, 67227442, 16595436, 69605283, 85695762, 80246713, 3773993, 54263880, 30787683, 8729146, 75135448, 59197747, 65880522, 64602895, 78589145, 20765474, 75777973, 17016156, 37183543, 4204661, 85117092, 42307449, 72614359, 36845587, 62936963, 32590267, 29635537, 71300104, 70372191, 59177718, 44889423, 47708850, 51426311, 25636669, 8791066, 29466406, 31126490, 88047921, 82651278, 57359924, 74110882, 79806380, 86798033, 92033260, 72777973, 85571389, 40356877, 24953498, 20645197, 14731700, 3294781, 55669657, 38256849, 11581181, 50567636, 8128637, 84127901, 83155442, 17081350, 82726008, 46870723, 44060493, 32151165, 79821897, 168541, 53888755, 54987042, 45306070, 5111370, 62452034, 61141874, 54517921, 39553046, 79191827, 99917599, 93566986, 23848565, 40677414, 83651211, 66137019, 45617087, 97281847, 8634541, 57241521, 94090109, 75820087, 17894977, 26998766, 58224549, 44664587, 95581843, 28787861, 20568363, 75052463, 83150534, 26392416, 70036438, 28734791, 6157724, 11161731, 97011160, 30764367, 69848388, 81853704, 65715134, 65338021, 21070110, 20486294, 30891921, 15902805, 8807940, 30395570, 40027975, 78561158, 69255765, 5640302, 62051033, 21289531, 82979980, 23134715, 51149804, 99729357, 48260151, 51507425, 73617245, 2331773, 36505482, 73124510, 75986488, 45428665, 48658605, 33249630, 27187213, 83747892, 4508700, 41442762, 16971929, 18131876, 85711894, 97379790, 16424199, 68316156, 18699206, 7687278, 63015256, 76960303, 4069912, 68204242, 9257405, 95726235, 60393039, 62496012, 34497327, 74452589, 54606384, 36780454, 31727379, 17037369, 84187166, 56955985, 76330843, 21397057, 30771409, 247198, 10597197, 50806615, 61897582, 669105, 56515456, 88904910, 95251277, 74614639, 90310261, 33336148, 45667668, 30139692, 92215320, 51715482, 93359396, 6221471, 78549759, 8651647, 98943869, 81274566, 54014062, 37957788, 18466635, 34493392, 7919588, 73392814, 21993752, 38009615, 38061439, 64157906, 95395112, 43933006, 60102965, 54427233, 84904436, 55615885, 92604458, 69786756, 5073754, 2204165, 41245325, 99965001, 78766358, 16054533, 71920426, 72373496, 73786944, 8733068, 47298834, 66319530, 36930650, 89078848, 40534591, 45995325, 37891451, 94076128, 99971982, 92353856, 68694897, 62693428, 94911072, 19457589, 9599614, 13470059, 48673079, 19891772, 28928175, 75128745, 63372756, 66667729, 55602660, 7229550, 1936762, 55470718, 21919959, 53547802, 19898053, 77377183, 63059024, 41380093, 96906410, 74724075, 70367851, 67281495, 50007421, 56461322, 39373729, 29834512, 41481685, 30811010, 29401781, 62762857, 67513640, 45381876, 50702367, 94595668, 92803766, 93270084, 68816413, 24058273, 62740044, 18783702, 96726697, 40781449, 49271185, 99524975, 66832478, 36139942, 60176618, 92787493, 49598724, 66045587, 93515664, 68000591, 80014588, 88130087, 66271566, 91240048, 83789391 +23110625, 78561158, 92692283, 87710366, 94911072, 24591705, 63015256, 98462867, 25842078, 55602660, 38341669, 77312810, 39171895, 58180653, 84002370, 74724075, 56473732, 16971929, 4069912, 31727379, 65851721, 73031054, 19457589, 59109400, 60176618, 27185644, 17894977, 16445503, 66611759, 61271144, 15075176, 32250045, 72495719, 54663246, 77377183, 42644903, 72793444, 37560164, 29029316, 17146629, 71316369, 76960303, 98948034, 17037369, 53632373, 17386996, 4985896, 2717150, 33336148, 75820087, 26863229, 17957593, 22450468, 10961421, 68702632, 62115552, 90061527, 14626618, 38008118, 81853704, 86460488, 61373987, 40027975, 61928316, 45990383, 31161687, 1022677, 3233569, 17058722, 51149804, 8913721, 54427233, 874791, 79880247, 46851987, 62936963, 79136082, 91664334, 20122224, 72732205, 99524975, 37435892, 40152546, 43506672, 91240048, 83210802, 40677414, 23432750, 4787945, 35512853, 17764950, 48673079, 21533347, 17068582, 51715482, 26998766, 49882705, 75153252, 53084256, 3183975, 81805959, 78549759, 81274566, 89419466, 70782102, 91990218, 4199704, 67451935, 11161731, 15948937, 14723410, 26734892, 3633375, 62232211, 22942635, 8807940, 38061439, 76434144, 13348726, 75407347, 68875490, 82886381, 61263068, 77301523, 37675718, 48088883, 30543215, 16019925, 13468268, 84293052, 36505482, 7955293, 83948335, 72738685, 6793819, 12348118, 44889423, 25636669, 29466406, 29834512, 74110882, 91938191, 87720882, 68204242, 23194618, 85571389, 8733068, 12024238, 56484900, 6819644, 73021291, 66663942, 74452589, 77187825, 38256849, 99333375, 57803235, 6986898, 22129328, 99224392, 54987042, 86821229, 22721500, 44550764, 11365791, 18833224, 79191827, 99917599, 68824981, 82327024, 61859581, 10366309, 23848565, 90158785, 4806458, 9058407, 4095116, 97940276, 93053405, 68041839, 29510992, 47361209, 81855445, 92215320, 3509435, 38510840, 43357947, 68102437, 40865610, 3487592, 54762643, 28787861, 33895336, 26776922, 30998561, 23569917, 9259676, 9860968, 85023028, 47792865, 75229462, 36468541, 84406788, 63628376, 48675329, 37957788, 59405277, 10309525, 77272628, 94539824, 32159704, 98130363, 1197320, 16595436, 35456853, 81898046, 89046466, 70727211, 31733363, 64157906, 30090481, 22108665, 9575954, 98739783, 20765474, 29932657, 82979980, 23134715, 66271566, 66322959, 37739481, 48892201, 38365584, 92541302, 45428665, 9188443, 18411915, 69786756, 42692881, 23740167, 55960386, 47090124, 41442762, 39373729, 89811711, 38022834, 97379790, 61859143, 24915585, 79806380, 49271185, 56153345, 72373496, 39201414, 95726235, 47298834, 78300864, 66319530, 57658654, 9603598, 50567636, 45381876, 26292919, 36396314, 99515901, 14349098, 46870723, 21397057, 79821897, 37891451, 94076128, 34432810, 97057187, 72278539, 90090964, 8696647, 61141874, 29065964, 91281584, 45075471, 32161669, 93057697, 36139942, 19376156, 8115266, 83651211, 26102057, 85242963, 83533741, 26139110, 80316608, 22435353, 4434662, 44473167, 44348328, 78785507, 61623915, 93270084, 66250369, 82427263, 30694952, 33628349, 51315460, 38494874, 66903004, 60106217, 42947632, 14093520, 95010552, 88444207, 1788101, 54014062, 17976208, 18466635, 34667286, 27325428, 89996536, 6153406, 53393358, 70596786, 73392814, 78884452, 89499542, 55770687, 21993752, 64087743, 99604946, 87598888, 59197747, 24826575, 64602895, 88130087, 78589145, 8129978, 7423788, 5640302, 92554120, 16097038, 43933006, 26275734, 36685023, 42307449, 76703609, 80014588, 16099750, 70372191, 42073124, 63967300, 47708850, 81172706, 7646095, 99965001, 7011964, 8791066, 27041967, 74357852, 96726697, 85711894, 16424199, 41481685, 32058615, 68316156, 77898274, 52060076, 30811010, 28851716, 82532312, 27665211, 1204161, 65047700, 65074535, 16684829, 50188404, 99021067, 29401781, 45919976, 1375023, 60955663, 3294781, 96193415, 79322415, 37166608, 33201905, 11581181, 84187166, 68939068, 8128637, 15536795, 96709982, 17385531, 52261574, 48774913, 29819940, 10597197, 99971982, 30163921, 61897582, 669105, 6521313, 9398733, 321665, 72358170, 16387575, 526217, 95251277, 39553046, 83368048, 92530431, 92803766, 51588015, 16405341, 79942022, 2891150, 49598724, 45667668, 29516592, 8373289, 33431960, 94090109, 13173644, 13231279, 84684495, 96852321, 83133790, 72274002, 8099994, 11923835, 62357986, 3088684, 95581843, 96420429, 57248122, 79922758, 78602717, 20568363, 75052463, 68816413, 55470718, 67084351, 42237907, 10358899, 49236559, 37280276, 9829782, 51472275, 36189527, 44915235, 68128438, 6525948, 85695762, 38658347, 30891921, 15902805, 33061250, 3773993, 54263880, 30787683, 93790285, 8729146, 55753905, 62552524, 52613508, 92071990, 30569392, 61728685, 33123618, 21289531, 86543538, 7300047, 82052050, 52293995, 84904436, 30653863, 44847298, 35192533, 61815223, 32590267, 77620120, 34295794, 15535065, 29635537, 96906410, 32426752, 92998591, 12303248, 44245960, 73109997, 6505939, 99297164, 58751351, 68644627, 95957797, 71333116, 43376279, 88047921, 55143724, 16054533, 59910624, 40781449, 24314885, 86798033, 72777973, 40356877, 94711842, 55247455, 20645197, 40781854, 4668450, 67030811, 30218878, 59501487, 11757872, 36930650, 97783876, 86022504, 89699445, 67513640, 45996863, 40686254, 83155442, 17081350, 47738185, 44060493, 30771409, 54058788, 32151165, 96318094, 45995325, 67793644, 168541, 50806615, 97641116, 74743862, 48893685, 69697787, 2917920, 67124282, 5111370, 56531125, 9886593, 44627776, 65038678, 74614639, 32274392, 93566986, 82339363, 15148031, 24733232, 44842615, 76825057, 70800879, 59371804, 45790169, 26493119, 45617087, 18001617, 97561745, 57241521, 80251430, 69579137, 17539625, 58208470, 88698958, 86001008, 71965942, 33304202, 35092039, 93359396, 91141395, 6221471, 9623492, 36312813, 79657802, 49328608, 508198, 2208785, 91802888, 6038457, 74137926, 8651647, 98943869, 1936762, 83150534, 95290172, 57393458, 30366150, 64848072, 47893286, 70036438, 6497830, 4978543, 65454636, 62430984, 84166196, 69848388, 17857111, 15015906, 24058273, 65338021, 45794415, 20486294, 73222868, 22879907, 44844121, 80246713, 68110330, 30395570, 44177011, 1569515, 20867149, 79738755, 37224844, 76170907, 48360198, 47213183, 63059024, 43045786, 10649306, 95395112, 65275241, 62051033, 75777973, 37183543, 60102965, 55189057, 76671482, 11543098, 85117092, 34698463, 63506281, 51507425, 55615885, 36845587, 415901, 41380093, 29188588, 9175338, 7685448, 71300104, 89214616, 92604458, 54868730, 112651, 33249630, 27187213, 70367851, 2204165, 93562257, 6808825, 18504197, 41245325, 51426311, 86118021, 4099191, 33699435, 4508700, 22113133, 2607799, 78766358, 7517032, 57359924, 4091162, 90654836, 69355476, 7687278, 66428795, 92033260, 50668599, 51466049, 83302115, 67474219, 77284619, 824872, 34497327, 77072625, 62762857, 41092102, 62428472, 33565483, 39986008, 45848907, 1250437, 76330843, 5832946, 34946859, 66832478, 91255408, 82897371, 92353856, 45306070, 56515456, 88904910, 36753250, 45049308, 26664538, 38645117, 99549373, 41092172, 20642888, 76540481, 66137019, 91727510, 75543508, 90272749, 69641513, 3233084, 90457870, 84840549, 75128745, 34698428, 65271999, 73235980, 66667729, 28796059, 66045587, 26114953, 14326617, 93515664, 92398073, 20885148, 53802686, 81677380, 22997281, 34493392, 64098930, 20836893, 38009615, 12416768, 70004753, 12571310, 75135448, 5482538, 66116458, 62803858, 73168565, 47887470, 89217461, 93933709, 89804152, 99729357, 26507214, 80555751, 19101477, 61741594, 91957544, 49597667, 75986488, 85116755, 98371444, 48658605, 51047803, 22766820, 43322743, 71522968, 18663507, 71469330, 37620363, 51464002, 83747892, 7066775, 56461322, 33237508, 18131876, 31126490, 14045383, 27625735, 44479073, 72238278, 9257405, 16567550, 60393039, 31491938, 43152977, 24953498, 64055960, 1431742, 86242799, 20002147, 62496012, 5822202, 99226875, 61982238, 55669657, 24168411, 26744917, 56955985, 12664567, 77413012, 57240218, 17727650, 71657078, 59318837, 6871053, 16380211, 62693428, 65017137, 44846932, 36812683, 77300457, 61960400, 71083565, 54517921, 31904591, 84349107, 69136837, 13470059, 72725103, 22405120, 27411561, 33797252, 19891772, 21001913, 90013093, 76315420, 87160386, 99658235, 28928175, 28550822, 63372756, 8055981, 45237957, 3880712, 4515343, 60430369, 59981773, 26397786, 36580610, 22200378, 749283, 51590803, 30764367, 34236719, 19898053, 40984766, 83789391, 99861373, 15064655, 68000591, 44784505, 69255765, 39801351, 13819358, 17016156, 569864, 42967683, 6111563, 94360702, 98653983, 42199455, 26063929, 62740044, 4798568, 78909561, 30463802, 73124510, 39847321, 66885828, 5073754, 77694700, 37501808, 70541760, 34044787, 59329511, 52204879, 49641577, 82651278, 4119747, 92692978, 33935899, 19272365, 10264691, 83083131, 14731700, 72357096, 70420215, 54606384, 37192445, 89078848, 82779622, 7502255, 40534591, 10453030, 247198, 44983451, 68694897, 62452034, 99125126, 9599614, 35996293, 94516935, 14947650, 97281847, 78218845, 57961282, 88251446, 88653118, 7104732, 44664587, 53842979, 42782652, 91831487, 21919959, 28734791, 9860195, 6157724, 17365188, 97011160, 59614746, 7919588, 67227442, 32699744, 53547802, 21070110, 176257, 98648327, 55850790, 33553959, 4204661, 53666583, 73617245, 2331773, 57163802, 36808486, 63152504, 18783702, 50007421, 77787724, 7182642, 36135, 81774825, 54232247, 18699206, 29994197, 74441448, 56424103, 36780454, 84127901, 50702367, 82726008, 94595668, 53888755, 89637706, 43501211, 90310261, 45407418, 8634541, 19939935, 72019362, 58749, 48395186, 7229550, 61712234, 20681921, 65715134, 69605283, 53648154, 19486173, 65880522, 3235882, 40197395, 1239555, 99690194, 67281495, 90004325, 71920426, 14363867, 73786944, 20140249, 83269727, 4064751, 46540998, 14220886, 57020481, 5267545, 30139692, 58224549, 54199160, 36933038, 13862149, 15163258, 82178706, 21673260, 62490109, 92867155, 60581278, 48260151, 65081429, 18806856, 92787493, 96735716, 26392416, 24619760, 67031644, 86301513, 29959549, 59177718, 37659250, 44481640, 72614359, 5970607 +7300047, 874791, 9398733, 65338021, 3233569, 56484900, 33336148, 62357986, 22200378, 5640302, 92692978, 63015256, 33565483, 57803235, 526217, 60176618, 8115266, 3487592, 30998561, 20568363, 94539824, 54663246, 1197320, 3235882, 26063929, 62740044, 37560164, 27187213, 23740167, 97379790, 29401781, 12024238, 89078848, 82779622, 11365791, 23848565, 19376156, 9259676, 60106217, 36189527, 4978543, 76170907, 64602895, 7423788, 1022677, 76671482, 42307449, 30653863, 39847321, 56153345, 37435892, 98948034, 68939068, 99224392, 65851721, 67793644, 44983451, 69697787, 62452034, 22405120, 9058407, 70800879, 75543508, 47361209, 21001913, 35092039, 22450468, 11923835, 58224549, 96735716, 91802888, 7229550, 42237907, 15948937, 34667286, 68128438, 34493392, 89046466, 61928316, 82979980, 72793444, 13468268, 54427233, 79880247, 55615885, 38365584, 49597667, 92692283, 9175338, 22766820, 55143724, 57359924, 37659250, 1204161, 86798033, 9257405, 29994197, 20645197, 73021291, 89699445, 45381876, 84127901, 4064751, 40534591, 40152546, 6521313, 2917920, 67124282, 45049308, 18833224, 54517921, 93057697, 83210802, 40677414, 4787945, 20642888, 76540481, 83533741, 22435353, 45617087, 8373289, 29510992, 72274002, 53084256, 93270084, 60430369, 51315460, 13862149, 4199704, 65454636, 53802686, 15015906, 85695762, 38658347, 62232211, 1569515, 40027975, 12571310, 59197747, 65880522, 64157906, 66116458, 65275241, 31161687, 62051033, 569864, 89217461, 16097038, 36685023, 66271566, 85117092, 80014588, 65081429, 30463802, 61815223, 19101477, 32590267, 96906410, 18504197, 91664334, 68316156, 24314885, 4069912, 87720882, 31491938, 9603598, 24168411, 37166608, 50702367, 14349098, 46870723, 46540998, 54058788, 72278539, 56515456, 88904910, 44627776, 36753250, 82327024, 5267545, 13470059, 90158785, 72725103, 16405341, 57241521, 78218845, 21533347, 3509435, 33304202, 17894977, 17068582, 16445503, 68102437, 45237957, 53842979, 42782652, 78549759, 70782102, 36933038, 47893286, 67451935, 17976208, 16595436, 53648154, 81898046, 62490109, 3773993, 61373987, 79738755, 88130087, 62552524, 9575954, 44784505, 47213183, 68875490, 21289531, 77312810, 37183543, 4204661, 82052050, 4798568, 44847298, 1239555, 83948335, 73124510, 98371444, 51047803, 70372191, 112651, 63967300, 66885828, 12348118, 18663507, 83747892, 56473732, 71316369, 18131876, 43376279, 74357852, 14045383, 36135, 20122224, 27625735, 54232247, 71920426, 82532312, 39201414, 95726235, 83302115, 60393039, 47298834, 55247455, 83083131, 60955663, 77284619, 59501487, 70420215, 74452589, 11581181, 56955985, 36396314, 83155442, 17081350, 44060493, 5832946, 10453030, 168541, 50806615, 66832478, 91255408, 2717150, 8696647, 94911072, 99917599, 82339363, 15148031, 61859581, 91240048, 38645117, 45667668, 69579137, 81855445, 69641513, 86001008, 3233084, 38510840, 90013093, 87160386, 88251446, 91141395, 6221471, 66611759, 7104732, 66250369, 36312813, 33895336, 66045587, 96420429, 30694952, 33628349, 75052463, 98943869, 9860968, 81274566, 55470718, 75229462, 14326617, 36580610, 9860195, 59405277, 72495719, 17365188, 14626618, 62430984, 59614746, 81853704, 19898053, 21993752, 21673260, 30787683, 87598888, 5482538, 98648327, 68000591, 45990383, 69255765, 86301513, 39801351, 92554120, 92867155, 75777973, 55850790, 47887470, 37675718, 30543215, 52293995, 66322959, 72614359, 62936963, 61741594, 15535065, 45428665, 29635537, 48658605, 54868730, 69786756, 12303248, 29029316, 44889423, 2204165, 7646095, 37501808, 36808486, 79136082, 18783702, 56461322, 99297164, 52204879, 2607799, 96726697, 49641577, 24915585, 28851716, 27665211, 33935899, 65074535, 76960303, 16684829, 99021067, 85571389, 40356877, 99226875, 96193415, 11757872, 77187825, 55669657, 38256849, 87710366, 36780454, 15536795, 26292919, 1250437, 52261574, 82726008, 48774913, 21397057, 45995325, 99971982, 61897582, 74743862, 68694897, 62693428, 89637706, 9886593, 36812683, 19457589, 91281584, 16387575, 43501211, 26664538, 35512853, 17764950, 94516935, 66137019, 91727510, 2891150, 18001617, 44473167, 58208470, 75820087, 98462867, 25842078, 8099994, 51715482, 26998766, 49882705, 28550822, 75128745, 58749, 10961421, 65271999, 66667729, 28796059, 3880712, 57248122, 62115552, 3183975, 26114953, 1936762, 85023028, 21919959, 36468541, 1788101, 26392416, 63628376, 64848072, 749283, 44915235, 77272628, 97011160, 32159704, 69848388, 98130363, 26734892, 24591705, 20681921, 53393358, 73222868, 44844121, 68110330, 86460488, 83789391, 8729146, 19486173, 48360198, 13348726, 52613508, 8129978, 43045786, 98739783, 13819358, 42644903, 29932657, 60581278, 33123618, 29959549, 93933709, 43933006, 53666583, 58180653, 40197395, 11543098, 51149804, 76703609, 16019925, 73617245, 46851987, 80555751, 7955293, 41380093, 77620120, 34295794, 72738685, 71300104, 89214616, 99690194, 81172706, 71522968, 6808825, 67281495, 55960386, 17146629, 34044787, 89811711, 38022834, 77787724, 71333116, 78766358, 7182642, 85711894, 81774825, 82651278, 4091162, 65047700, 92033260, 44479073, 10264691, 16567550, 51466049, 18806856, 86242799, 40781854, 67474219, 72357096, 34497327, 77072625, 97783876, 31727379, 84187166, 45996863, 12664567, 53632373, 6986898, 40686254, 17385531, 99515901, 22129328, 17727650, 47738185, 29819940, 30771409, 71657078, 32151165, 96318094, 247198, 94595668, 97057187, 86821229, 4985896, 14220886, 44550764, 48893685, 321665, 65017137, 77300457, 45075471, 83368048, 32161669, 10366309, 36139942, 90310261, 59109400, 76825057, 4434662, 49598724, 29516592, 57961282, 71965942, 27185644, 43357947, 93359396, 84840549, 78785507, 48395186, 54762643, 3088684, 68702632, 82427263, 6038457, 74137926, 95290172, 88444207, 26397786, 6497830, 93515664, 37957788, 27325428, 90061527, 81677380, 30764367, 38008118, 64098930, 7919588, 61712234, 73392814, 89499542, 55770687, 20486294, 12416768, 64087743, 24619760, 54263880, 70004753, 31733363, 15064655, 55753905, 24826575, 78589145, 76434144, 30090481, 30569392, 20765474, 38341669, 73168565, 77301523, 86543538, 98653983, 34698463, 48260151, 84904436, 2331773, 26507214, 78909561, 37739481, 23110625, 92541302, 6793819, 16099750, 18411915, 7685448, 59177718, 92998591, 33249630, 5073754, 77694700, 74724075, 6505939, 51464002, 51426311, 99965001, 47090124, 7011964, 4508700, 22113133, 29834512, 16971929, 16054533, 52060076, 90004325, 4119747, 74110882, 49271185, 7687278, 66428795, 14363867, 94711842, 74441448, 64055960, 78300864, 14731700, 20002147, 6819644, 67030811, 62496012, 66663942, 54606384, 62428472, 26744917, 50567636, 37192445, 45848907, 77413012, 57240218, 6871053, 94076128, 53888755, 54987042, 22721500, 90090964, 92353856, 45306070, 5111370, 56531125, 99125126, 44846932, 61141874, 29065964, 95251277, 65038678, 32274392, 92530431, 93566986, 68824981, 44481640, 44842615, 84349107, 83651211, 51588015, 99549373, 92787493, 26102057, 79942022, 35996293, 45790169, 97561745, 90272749, 33431960, 17539625, 92215320, 88698958, 96852321, 26863229, 19939935, 17957593, 83133790, 72019362, 40865610, 61623915, 75153252, 34698428, 88653118, 73235980, 28787861, 508198, 2208785, 79922758, 55602660, 81805959, 23569917, 38494874, 91831487, 68816413, 59981773, 83150534, 95010552, 61271144, 91990218, 30366150, 10358899, 48675329, 28734791, 6157724, 20885148, 18466635, 51590803, 22997281, 89996536, 34236719, 6525948, 14723410, 32699744, 53547802, 45794415, 70596786, 22942635, 30891921, 15902805, 44177011, 20867149, 75135448, 22108665, 78561158, 10649306, 95395112, 61263068, 6111563, 26275734, 55189057, 99729357, 63506281, 36845587, 9188443, 92604458, 42073124, 70367851, 71469330, 37620363, 73109997, 70541760, 7066775, 33699435, 41442762, 58751351, 5970607, 31126490, 41481685, 7517032, 77898274, 69355476, 91938191, 23194618, 72777973, 72238278, 4668450, 3294781, 824872, 61982238, 62762857, 86022504, 83269727, 17386996, 96709982, 76330843, 34946859, 59318837, 10597197, 34432810, 30163921, 82897371, 72358170, 57020481, 43506672, 74614639, 71083565, 4806458, 48673079, 27411561, 97940276, 26139110, 33797252, 26493119, 97281847, 44348328, 84684495, 76315420, 99658235, 28928175, 95581843, 78602717, 66903004, 54014062, 84406788, 9829782, 51472275, 70036438, 15075176, 92398073, 17857111, 20836893, 3633375, 78884452, 22879907, 8807940, 40984766, 38009615, 38061439, 70727211, 99861373, 77377183, 92071990, 61728685, 17016156, 42967683, 94360702, 39171895, 60102965, 35192533, 415901, 75986488, 57163802, 85116755, 43322743, 42692881, 47708850, 93562257, 63152504, 41245325, 4099191, 8791066, 39373729, 68644627, 29466406, 27041967, 88047921, 61859143, 40781449, 30811010, 90654836, 79806380, 18699206, 19272365, 50188404, 50668599, 68204242, 73786944, 24953498, 57658654, 79322415, 41092102, 33201905, 17037369, 8128637, 37891451, 61960400, 39553046, 79191827, 31904591, 69136837, 4095116, 30139692, 8634541, 94090109, 80251430, 63372756, 8055981, 9623492, 44664587, 54199160, 42947632, 47792865, 57393458, 37280276, 11161731, 32250045, 15163258, 84166196, 65715134, 24058273, 69605283, 21070110, 35456853, 80246713, 67031644, 75407347, 62803858, 82886381, 48088883, 89804152, 42199455, 84002370, 48892201, 91957544, 29188588, 44245960, 50007421, 95957797, 16424199, 72732205, 72373496, 8733068, 1431742, 20140249, 56424103, 99333375, 39986008, 669105, 23432750, 41092172, 85242963, 14947650, 59371804, 80316608, 19891772, 90457870, 4515343, 8651647, 89419466, 14093520, 67084351, 49236559, 10309525, 67227442, 6153406, 33061250, 30395570, 93790285, 37224844, 63059024, 33553959, 23134715, 51507425, 84293052, 36505482, 33237508, 59329511, 59910624, 32058615, 45919976, 99524975, 43152977, 5822202, 36930650, 67513640, 7502255, 73031054, 97641116, 24733232, 9599614, 45407418, 13173644, 79657802, 26776922, 82178706, 99604946, 17058722, 25636669, 66319530, 30218878, 92803766, 93053405, 68041839, 13231279, 8913721, 32426752, 1375023, 16380211, 176257, 49328608, 86118021, 79821897 +22721500, 90158785, 5640302, 31161687, 11581181, 7011964, 11365791, 70782102, 28734791, 14626618, 54663246, 6525948, 44784505, 47213183, 34295794, 23740167, 94076128, 62452034, 19457589, 45049308, 98462867, 90457870, 93270084, 60430369, 13862149, 48675329, 34667286, 68128438, 61712234, 65338021, 21673260, 70727211, 78561158, 75777973, 43933006, 85117092, 874791, 84904436, 26507214, 36845587, 71522968, 40781449, 69355476, 74110882, 9398733, 36812683, 18833224, 79191827, 15148031, 26664538, 29516592, 69579137, 26863229, 17894977, 54762643, 30998561, 20568363, 83150534, 17976208, 30764367, 98130363, 89499542, 30891921, 64087743, 10649306, 47887470, 7300047, 60102965, 4204661, 51507425, 62740044, 6793819, 57163802, 18411915, 77694700, 18663507, 6808825, 99965001, 58751351, 39373729, 95957797, 43376279, 87720882, 83302115, 31491938, 24168411, 62428472, 89078848, 50702367, 96709982, 40534591, 45995325, 67793644, 2717150, 56515456, 5111370, 92803766, 59109400, 51588015, 79942022, 35996293, 75543508, 57241521, 75820087, 21001913, 16445503, 88251446, 6221471, 81805959, 30694952, 26392416, 63628376, 70036438, 32250045, 51590803, 84166196, 78884452, 69605283, 15902805, 8807940, 79738755, 12571310, 19486173, 61928316, 45990383, 39801351, 20765474, 42644903, 65275241, 62051033, 55850790, 21289531, 86543538, 94360702, 72793444, 76703609, 48260151, 54427233, 72614359, 91957544, 41380093, 23110625, 45428665, 63967300, 33249630, 27187213, 74724075, 18504197, 2607799, 41481685, 63015256, 56153345, 9257405, 45919976, 10264691, 95726235, 98948034, 56955985, 37192445, 45381876, 77413012, 57803235, 40686254, 83155442, 17385531, 4064751, 47738185, 37891451, 16380211, 94595668, 53888755, 48893685, 69697787, 62693428, 526217, 74614639, 44481640, 91240048, 45407418, 76540481, 14947650, 66137019, 91727510, 45790169, 33336148, 45617087, 47361209, 81855445, 21533347, 27185644, 22450468, 28550822, 75153252, 66667729, 28787861, 53842979, 66045587, 508198, 2208785, 62115552, 33628349, 42947632, 47792865, 9829782, 749283, 36189527, 93515664, 10309525, 18466635, 90061527, 17365188, 17857111, 35456853, 53648154, 22942635, 62490109, 99604946, 3773993, 65880522, 5482538, 13348726, 98648327, 69255765, 75407347, 66116458, 7423788, 62803858, 61728685, 33553959, 82979980, 98653983, 52293995, 26063929, 16019925, 13468268, 80014588, 79880247, 19101477, 38365584, 77620120, 9175338, 98371444, 54868730, 70372191, 37620363, 36808486, 70541760, 50007421, 96726697, 88047921, 7182642, 55143724, 97379790, 77898274, 30811010, 37659250, 27665211, 19272365, 86798033, 72238278, 72373496, 85571389, 12024238, 47298834, 99524975, 56484900, 64055960, 78300864, 86242799, 60955663, 20002147, 67474219, 30218878, 20140249, 99226875, 56424103, 70420215, 55669657, 84187166, 68939068, 67513640, 39986008, 15536795, 44060493, 5832946, 99971982, 168541, 97641116, 66832478, 669105, 44983451, 6521313, 92353856, 65038678, 54517921, 32274392, 83368048, 92530431, 31904591, 24733232, 44842615, 10366309, 69136837, 8115266, 23432750, 92787493, 9058407, 4095116, 2891150, 33797252, 49598724, 45667668, 13173644, 29510992, 57961282, 88698958, 3233084, 71965942, 72274002, 17068582, 3487592, 48395186, 66250369, 79657802, 33895336, 49328608, 96735716, 74137926, 78549759, 51315460, 89419466, 75229462, 42237907, 10358899, 4978543, 15948937, 72495719, 97011160, 59614746, 53393358, 70596786, 73392814, 22879907, 44844121, 87598888, 62552524, 9575954, 1022677, 77312810, 48088883, 26275734, 55189057, 82052050, 11543098, 51149804, 84293052, 55615885, 2331773, 80555751, 62936963, 7955293, 83948335, 32590267, 73124510, 92692283, 92604458, 96906410, 22766820, 66885828, 44889423, 70367851, 73109997, 67281495, 51426311, 18783702, 56473732, 47090124, 4508700, 99297164, 77787724, 27041967, 18131876, 78766358, 31126490, 81774825, 57359924, 20122224, 24915585, 28851716, 71920426, 82532312, 49271185, 4069912, 16567550, 51466049, 24953498, 20645197, 4668450, 3294781, 824872, 66663942, 37166608, 36780454, 33201905, 99333375, 22129328, 82779622, 30771409, 10453030, 32151165, 40152546, 73031054, 30163921, 61897582, 86821229, 14220886, 44550764, 74743862, 68694897, 45306070, 321665, 9886593, 39553046, 32161669, 82327024, 19376156, 38645117, 60176618, 76825057, 13470059, 17764950, 16405341, 80316608, 22435353, 25842078, 17957593, 38510840, 35092039, 43357947, 8099994, 49882705, 68102437, 10961421, 11923835, 91141395, 66611759, 62357986, 3088684, 68702632, 3880712, 82427263, 9259676, 98943869, 1936762, 66903004, 91831487, 21919959, 36468541, 91990218, 54014062, 84406788, 22200378, 6497830, 9860195, 6157724, 65454636, 81677380, 34236719, 7919588, 81853704, 20681921, 32699744, 6153406, 45794415, 21070110, 38658347, 81898046, 80246713, 61373987, 31733363, 8729146, 75135448, 77377183, 3235882, 8129978, 92554120, 82886381, 33123618, 569864, 37183543, 93933709, 3233569, 40197395, 42307449, 63506281, 65081429, 61815223, 37560164, 75986488, 29635537, 16099750, 51047803, 99690194, 92998591, 5073754, 12348118, 29029316, 44245960, 47708850, 2204165, 7646095, 6505939, 33699435, 8791066, 33237508, 89811711, 59329511, 29834512, 74357852, 16054533, 82651278, 79806380, 18699206, 33935899, 65074535, 14363867, 44479073, 16684829, 99021067, 23194618, 40356877, 60393039, 1375023, 55247455, 14731700, 41092102, 87710366, 31727379, 33565483, 8128637, 26292919, 36396314, 57240218, 99515901, 82726008, 29819940, 54058788, 96318094, 247198, 10597197, 97057187, 54987042, 72278539, 91255408, 4985896, 90090964, 8696647, 56531125, 89637706, 88904910, 43506672, 71083565, 68824981, 84349107, 4787945, 22405120, 48673079, 26102057, 85242963, 83533741, 27411561, 94516935, 90272749, 8634541, 94090109, 3509435, 84684495, 86001008, 19939935, 51715482, 87160386, 72019362, 84840549, 99658235, 28928175, 61623915, 58749, 28796059, 26776922, 42782652, 95290172, 95010552, 1788101, 64848072, 44915235, 67451935, 37957788, 11161731, 20885148, 64098930, 24591705, 24058273, 21993752, 62232211, 12416768, 89046466, 176257, 83789391, 24826575, 30090481, 95395112, 60581278, 29959549, 53666583, 58180653, 66271566, 8913721, 34698463, 66322959, 61741594, 48892201, 39847321, 85116755, 9188443, 59177718, 42073124, 51464002, 55960386, 56461322, 41442762, 22113133, 71316369, 68644627, 5970607, 38022834, 71333116, 16971929, 91664334, 59910624, 49641577, 36135, 32058615, 68316156, 27625735, 72732205, 54232247, 29401781, 73786944, 94711842, 43152977, 83083131, 77284619, 73021291, 61982238, 11757872, 74452589, 86022504, 17037369, 45996863, 53632373, 84127901, 17081350, 52261574, 14349098, 46870723, 48774913, 59318837, 82897371, 2917920, 94911072, 72358170, 99125126, 43501211, 61960400, 45075471, 99917599, 93566986, 82339363, 23848565, 5267545, 83210802, 90310261, 35512853, 4806458, 26493119, 18001617, 8373289, 69641513, 96852321, 83133790, 26998766, 53084256, 75128745, 34698428, 36312813, 54199160, 4515343, 91802888, 55602660, 6038457, 3183975, 75052463, 8651647, 14093520, 88444207, 49236559, 37280276, 4199704, 92398073, 27325428, 77272628, 34493392, 62430984, 94539824, 32159704, 14723410, 26734892, 53547802, 85695762, 40984766, 38009615, 70004753, 99861373, 40027975, 15064655, 64602895, 78589145, 64157906, 67031644, 68000591, 22108665, 63059024, 13819358, 38341669, 6111563, 16097038, 36685023, 76671482, 4798568, 49597667, 72738685, 48658605, 7685448, 112651, 42692881, 81172706, 71469330, 37501808, 93562257, 79136082, 29466406, 85711894, 14045383, 16424199, 61859143, 52060076, 90004325, 4091162, 90654836, 92033260, 68204242, 6819644, 72357096, 59501487, 5822202, 96193415, 34497327, 57658654, 62762857, 77187825, 89699445, 50567636, 45848907, 12664567, 17727650, 21397057, 71657078, 46540998, 6871053, 34432810, 65017137, 91281584, 77300457, 61859581, 40677414, 72725103, 93053405, 44473167, 33431960, 19891772, 92215320, 58224549, 8055981, 73235980, 9623492, 7229550, 78602717, 85023028, 60106217, 59981773, 55470718, 61271144, 36933038, 36580610, 30366150, 15075176, 59405277, 53802686, 38008118, 1197320, 20836893, 67227442, 16595436, 3633375, 68110330, 24619760, 30787683, 1569515, 20867149, 37224844, 76170907, 55753905, 30569392, 43045786, 68875490, 98739783, 92867155, 77301523, 37675718, 89217461, 42967683, 89804152, 30543215, 73617245, 30653863, 84002370, 46851987, 37739481, 1239555, 89214616, 83747892, 41245325, 4099191, 25636669, 52204879, 92692978, 24314885, 65047700, 50188404, 29994197, 39201414, 8733068, 40781854, 66319530, 83269727, 26744917, 6986898, 17386996, 76330843, 99224392, 7502255, 65851721, 67124282, 44846932, 29065964, 57020481, 36753250, 95251277, 93057697, 83651211, 99549373, 20642888, 70800879, 97940276, 97561745, 80251430, 78218845, 58208470, 13231279, 90013093, 76315420, 93359396, 78785507, 40865610, 44664587, 57248122, 26114953, 81274566, 26397786, 67084351, 57393458, 47893286, 22997281, 89996536, 15015906, 55770687, 73222868, 82178706, 86460488, 30395570, 54263880, 48360198, 88130087, 52613508, 86301513, 73168565, 29932657, 61263068, 17058722, 78909561, 35192533, 30463802, 92541302, 71300104, 32426752, 12303248, 43322743, 7517032, 91938191, 66428795, 76960303, 74441448, 67030811, 77072625, 36930650, 97783876, 54606384, 1250437, 50806615, 36139942, 41092172, 26139110, 59371804, 4434662, 68041839, 97281847, 17539625, 44348328, 33304202, 63372756, 65271999, 88653118, 7104732, 95581843, 96420429, 23569917, 15163258, 69848388, 20486294, 38061439, 33061250, 59197747, 76434144, 92071990, 23134715, 39171895, 42199455, 99729357, 36505482, 415901, 29188588, 69786756, 63152504, 7066775, 86118021, 4119747, 1204161, 72777973, 37435892, 9603598, 34946859, 61141874, 16387575, 45237957, 79922758, 9860968, 14326617, 51472275, 44177011, 34044787, 50668599, 79322415, 38494874, 17016156, 44847298, 17146629, 18806856, 1431742, 38256849, 44627776, 30139692, 65715134, 93790285, 7687278, 62496012, 79821897, 68816413, 19898053, 15535065, 9599614 +37166608, 37957788, 19891772, 75153252, 61928316, 70541760, 50007421, 40356877, 17539625, 95010552, 44784505, 42307449, 63506281, 89699445, 62428472, 2917920, 5111370, 91240048, 45407418, 83533741, 66137019, 18001617, 13173644, 33628349, 72495719, 66322959, 71522968, 63152504, 79136082, 41245325, 55960386, 68316156, 7687278, 94711842, 55247455, 66663942, 74452589, 86022504, 77413012, 76330843, 91255408, 62693428, 83368048, 44481640, 61859581, 92803766, 83210802, 92787493, 35996293, 21533347, 65271999, 33895336, 64848072, 47893286, 15075176, 19898053, 9575954, 62803858, 86543538, 52293995, 84293052, 34295794, 45428665, 98371444, 63967300, 27187213, 23740167, 51426311, 7182642, 71920426, 44479073, 45919976, 99524975, 64055960, 67030811, 77284619, 37192445, 82779622, 44060493, 29819940, 61897582, 54987042, 67124282, 94911072, 62452034, 57020481, 36753250, 74614639, 17764950, 70800879, 97940276, 33336148, 45667668, 57241521, 3233084, 17957593, 62357986, 53842979, 66045587, 26114953, 23569917, 9259676, 1788101, 10358899, 70036438, 93515664, 18466635, 32250045, 34493392, 94539824, 89996536, 30764367, 53547802, 69605283, 21993752, 54263880, 30787683, 75135448, 59197747, 22108665, 45990383, 69255765, 8129978, 5640302, 39801351, 92867155, 93933709, 94360702, 4204661, 58180653, 55189057, 30543215, 66271566, 11543098, 99729357, 84904436, 4798568, 61815223, 73124510, 57163802, 16099750, 66885828, 12348118, 71469330, 95957797, 74357852, 96726697, 88047921, 27625735, 24314885, 68204242, 95726235, 56484900, 60955663, 96193415, 41092102, 36396314, 17081350, 79821897, 10597197, 73031054, 53888755, 72358170, 61141874, 16387575, 79191827, 24733232, 5267545, 19376156, 83651211, 41092172, 14947650, 93053405, 90272749, 94090109, 69579137, 69641513, 98462867, 88698958, 26863229, 51715482, 87160386, 53084256, 91141395, 88653118, 7104732, 93270084, 28796059, 3880712, 42782652, 51315460, 20568363, 98943869, 81274566, 54014062, 51472275, 27325428, 34236719, 15015906, 81853704, 21070110, 53648154, 22879907, 22942635, 12416768, 44177011, 70727211, 40027975, 12571310, 64602895, 30090481, 68000591, 77377183, 78561158, 92071990, 98739783, 65275241, 3233569, 26275734, 72793444, 36685023, 16019925, 13468268, 51507425, 26507214, 7955293, 48892201, 32590267, 59177718, 92998591, 6505939, 6808825, 18783702, 56473732, 41442762, 71316369, 55143724, 49641577, 61859143, 69355476, 79806380, 33935899, 66428795, 50188404, 87720882, 50668599, 16567550, 31491938, 67474219, 98948034, 62496012, 9603598, 55669657, 11581181, 45848907, 8128637, 89078848, 40686254, 52261574, 47738185, 7502255, 46540998, 34946859, 45995325, 37891451, 168541, 6521313, 92353856, 9398733, 88904910, 29065964, 93057697, 90310261, 38645117, 59109400, 60176618, 35512853, 20642888, 48673079, 76540481, 91727510, 22435353, 33797252, 29510992, 84684495, 96852321, 86001008, 90457870, 84840549, 26998766, 78785507, 28928175, 58749, 66611759, 54762643, 9623492, 44664587, 95581843, 508198, 55602660, 78602717, 42947632, 21919959, 26397786, 749283, 36189527, 67451935, 9860195, 53802686, 68128438, 32159704, 17857111, 32699744, 85695762, 38009615, 64087743, 3773993, 83789391, 79738755, 93790285, 19486173, 24826575, 65880522, 76434144, 13348726, 75407347, 66116458, 73168565, 29932657, 33123618, 1022677, 53666583, 98653983, 76671482, 51149804, 85117092, 8913721, 34698463, 48260151, 26063929, 73617245, 30653863, 65081429, 92692283, 77620120, 23110625, 39847321, 71300104, 96906410, 12303248, 5073754, 70367851, 18663507, 37620363, 56461322, 99297164, 58751351, 33237508, 71333116, 16971929, 59910624, 97379790, 32058615, 81774825, 82651278, 92692978, 18699206, 86798033, 92033260, 14363867, 56153345, 23194618, 29401781, 9257405, 29994197, 83302115, 47298834, 24953498, 74441448, 78300864, 30218878, 61982238, 62762857, 24168411, 87710366, 99333375, 15536795, 50702367, 17727650, 5832946, 40534591, 32151165, 94076128, 16380211, 97057187, 30163921, 66832478, 14220886, 90090964, 48893685, 99125126, 19457589, 91281584, 526217, 43501211, 45075471, 26664538, 23848565, 4095116, 79942022, 75543508, 80316608, 45790169, 97561745, 8634541, 72274002, 43357947, 8099994, 22450468, 72019362, 28550822, 3487592, 10961421, 58224549, 79657802, 82427263, 57248122, 38494874, 1936762, 68816413, 89419466, 14093520, 47792865, 36468541, 9829782, 48675329, 4199704, 11161731, 20885148, 10309525, 14723410, 24591705, 67227442, 73392814, 38658347, 73222868, 21673260, 15902805, 40984766, 99604946, 20867149, 87598888, 55753905, 5482538, 88130087, 3235882, 47213183, 95395112, 92554120, 42644903, 62051033, 75777973, 77301523, 33553959, 23134715, 43933006, 39171895, 60102965, 48088883, 54427233, 55615885, 2331773, 46851987, 72614359, 78909561, 37739481, 415901, 15535065, 9188443, 99690194, 69786756, 42073124, 22766820, 44889423, 44245960, 47708850, 93562257, 36808486, 25636669, 34044787, 68644627, 38022834, 29834512, 2607799, 85711894, 16054533, 41481685, 57359924, 90004325, 30811010, 65047700, 19272365, 63015256, 39201414, 60393039, 83083131, 86242799, 6819644, 20140249, 36930650, 54606384, 36780454, 33201905, 17037369, 26744917, 50567636, 67513640, 39986008, 56955985, 45996863, 84127901, 57803235, 17386996, 96709982, 82726008, 4064751, 30771409, 59318837, 6871053, 34432810, 50806615, 669105, 11365791, 82897371, 69697787, 8696647, 56531125, 65017137, 9886593, 44627776, 65038678, 18833224, 99917599, 15148031, 82327024, 36139942, 76825057, 8115266, 13470059, 4787945, 51588015, 4434662, 68041839, 45617087, 81855445, 58208470, 71965942, 17068582, 61623915, 75128745, 48395186, 36312813, 4515343, 60430369, 62115552, 75052463, 91831487, 70782102, 61271144, 36933038, 26392416, 36580610, 30366150, 22200378, 28734791, 34667286, 90061527, 51590803, 81677380, 64098930, 26734892, 30891921, 8807940, 24619760, 1569515, 176257, 99861373, 37224844, 78589145, 67031644, 7423788, 10649306, 31161687, 61728685, 47887470, 29959549, 37675718, 77312810, 82979980, 42967683, 7300047, 17058722, 76703609, 874791, 36505482, 30463802, 83948335, 37560164, 92541302, 75986488, 29188588, 6793819, 48658605, 51047803, 92604458, 54868730, 43322743, 77694700, 81172706, 2204165, 73109997, 18504197, 7066775, 86118021, 4099191, 47090124, 7011964, 8791066, 39373729, 18131876, 36135, 7517032, 4091162, 24915585, 37659250, 54232247, 74110882, 82532312, 49271185, 72238278, 72373496, 85571389, 51466049, 8733068, 4668450, 20002147, 3294781, 31727379, 84187166, 68939068, 6986898, 17385531, 46870723, 99224392, 48774913, 247198, 99971982, 67793644, 97641116, 86821229, 44983451, 74743862, 56515456, 36812683, 45049308, 54517921, 92530431, 93566986, 82339363, 68824981, 44842615, 9599614, 84349107, 22405120, 16405341, 9058407, 2891150, 97281847, 80251430, 78218845, 57961282, 83133790, 27185644, 90013093, 76315420, 68102437, 40865610, 11923835, 63372756, 6221471, 45237957, 3088684, 28787861, 54199160, 49328608, 96735716, 96420429, 7229550, 81805959, 30694952, 60106217, 83150534, 67084351, 42237907, 49236559, 44915235, 6157724, 17365188, 84166196, 54663246, 69848388, 20681921, 16595436, 3633375, 24058273, 65338021, 6153406, 53393358, 70596786, 55770687, 35456853, 62232211, 81898046, 44844121, 80246713, 68110330, 38061439, 33061250, 70004753, 76170907, 48360198, 62552524, 52613508, 30569392, 86301513, 63059024, 20765474, 55850790, 21289531, 16097038, 89804152, 82052050, 40197395, 84002370, 80555751, 61741594, 38365584, 29635537, 9175338, 18411915, 32426752, 112651, 33249630, 42692881, 37501808, 29466406, 91664334, 78766358, 14045383, 16424199, 52060076, 20122224, 90654836, 28851716, 91938191, 1204161, 76960303, 99021067, 73786944, 10264691, 18806856, 12024238, 1431742, 72357096, 824872, 70420215, 77072625, 79322415, 97783876, 45381876, 26292919, 22129328, 14349098, 71657078, 10453030, 96318094, 65851721, 72278539, 22721500, 89637706, 61960400, 32161669, 40677414, 90158785, 99549373, 85242963, 27411561, 26139110, 59371804, 30139692, 44473167, 47361209, 92215320, 13231279, 44348328, 25842078, 33304202, 35092039, 49882705, 73235980, 66250369, 79922758, 91802888, 9860968, 14326617, 88444207, 63628376, 6497830, 77272628, 14626618, 62430984, 15163258, 61712234, 45794415, 62490109, 86460488, 89046466, 15064655, 64157906, 89217461, 42199455, 62740044, 36845587, 62936963, 19101477, 41380093, 89214616, 70372191, 29029316, 33699435, 89811711, 77787724, 4119747, 27665211, 65074535, 1375023, 14731700, 59501487, 56424103, 34497327, 57658654, 38256849, 33565483, 57240218, 83155442, 1250437, 21397057, 40152546, 4985896, 2717150, 44550764, 68694897, 44846932, 71083565, 31904591, 10366309, 23432750, 26102057, 49598724, 8373289, 75820087, 3509435, 19939935, 38510840, 93359396, 99658235, 30998561, 6038457, 3183975, 74137926, 66903004, 95290172, 91990218, 57393458, 37280276, 17976208, 59405277, 92398073, 15948937, 59614746, 6525948, 20836893, 82178706, 61373987, 8729146, 43045786, 68875490, 38341669, 60581278, 61263068, 37183543, 80014588, 44847298, 35192533, 1239555, 72738685, 85116755, 7685448, 74724075, 7646095, 51464002, 83747892, 67281495, 99965001, 4508700, 59329511, 27041967, 52204879, 43376279, 31126490, 40781449, 72732205, 4069912, 72777973, 43152977, 37435892, 40781854, 66319530, 73021291, 99226875, 77187825, 53632373, 99515901, 39553046, 69136837, 4806458, 33431960, 21001913, 17894977, 16445503, 88251446, 34698428, 66667729, 68702632, 2208785, 8651647, 55470718, 84406788, 4978543, 65454636, 97011160, 22997281, 1197320, 65715134, 78884452, 20486294, 98648327, 6111563, 79880247, 91957544, 49597667, 17146629, 77898274, 20645197, 11757872, 54058788, 45306070, 321665, 77300457, 95251277, 32274392, 72725103, 26493119, 26776922, 78549759, 85023028, 59981773, 75229462, 13862149, 38008118, 98130363, 89499542, 17016156, 22113133, 5970607, 16684829, 5822202, 83269727, 43506672, 30395570, 31733363, 82886381, 94595668, 94516935, 8055981, 7919588, 569864, 29516592, 13819358, 12664567 +3509435, 45381876, 48675329, 11161731, 9575954, 52204879, 31727379, 84127901, 92353856, 19376156, 91240048, 66611759, 33895336, 44915235, 24826575, 63506281, 9188443, 6808825, 10264691, 89699445, 36396314, 168541, 72278539, 96420429, 68816413, 97011160, 81853704, 61928316, 69255765, 66116458, 1022677, 48088883, 66271566, 66322959, 41380093, 99690194, 73109997, 99965001, 4508700, 54232247, 69355476, 63015256, 50188404, 67474219, 70420215, 45996863, 57803235, 83651211, 97940276, 17539625, 35092039, 65271999, 58224549, 53842979, 95010552, 15902805, 54263880, 88130087, 64157906, 52613508, 62051033, 92867155, 47887470, 77301523, 53666583, 55189057, 42307449, 83948335, 92692283, 92541302, 34295794, 15535065, 70372191, 29029316, 18663507, 7182642, 16054533, 9257405, 56484900, 4668450, 14731700, 67030811, 41092102, 86022504, 24168411, 56955985, 22129328, 50806615, 90090964, 61141874, 45075471, 84349107, 35512853, 90272749, 86001008, 17957593, 61623915, 66250369, 3880712, 4515343, 6038457, 23569917, 55470718, 75229462, 36468541, 1788101, 67084351, 30366150, 749283, 67451935, 6157724, 65454636, 10309525, 18466635, 84166196, 65715134, 35456853, 62232211, 38009615, 176257, 76170907, 8129978, 7423788, 95395112, 33123618, 569864, 94360702, 72793444, 99729357, 84904436, 55615885, 30653863, 26507214, 415901, 73124510, 7685448, 7646095, 18783702, 71333116, 59910624, 41481685, 79806380, 7687278, 29994197, 83302115, 43152977, 86242799, 66319530, 57240218, 76330843, 62693428, 94911072, 65017137, 44627776, 99917599, 26664538, 76825057, 13173644, 69641513, 51715482, 49882705, 28550822, 54762643, 3088684, 66045587, 96735716, 42782652, 26114953, 9259676, 89419466, 59981773, 42237907, 57393458, 22200378, 9829782, 15075176, 34667286, 27325428, 17365188, 68128438, 6525948, 20681921, 12416768, 70004753, 64602895, 30090481, 77377183, 30569392, 43045786, 68875490, 65275241, 38341669, 82886381, 61263068, 93933709, 42967683, 3233569, 58180653, 36685023, 52293995, 34698463, 874791, 62740044, 44847298, 7955293, 48892201, 38365584, 45428665, 29188588, 29635537, 98371444, 42692881, 2204165, 6505939, 79136082, 7066775, 56473732, 89811711, 71316369, 4091162, 30811010, 65047700, 19272365, 44479073, 56153345, 72777973, 51466049, 60393039, 1431742, 60955663, 20002147, 56424103, 79322415, 54606384, 87710366, 89078848, 17386996, 29819940, 10453030, 32151165, 34432810, 669105, 4985896, 11365791, 45306070, 62452034, 9886593, 526217, 65038678, 43506672, 71083565, 54517921, 39553046, 82327024, 27411561, 66137019, 22435353, 33797252, 78218845, 57961282, 75820087, 84684495, 26863229, 33304202, 17894977, 22450468, 28928175, 11923835, 63372756, 6221471, 36312813, 33628349, 38494874, 98943869, 26392416, 13862149, 51472275, 32250045, 90061527, 22997281, 34493392, 62430984, 94539824, 64098930, 53547802, 65338021, 22879907, 8807940, 24619760, 44177011, 89046466, 12571310, 75135448, 55753905, 67031644, 13348726, 68000591, 61728685, 55850790, 77312810, 82979980, 16097038, 26275734, 40197395, 76671482, 51149804, 76703609, 51507425, 73617245, 80555751, 62936963, 36505482, 37739481, 35192533, 61815223, 61741594, 32590267, 49597667, 37560164, 23110625, 85116755, 16099750, 71300104, 89214616, 112651, 63967300, 77694700, 12348118, 81172706, 71522968, 37501808, 55960386, 50007421, 99297164, 68644627, 29466406, 97379790, 90654836, 4119747, 71920426, 92033260, 50668599, 72238278, 18806856, 47298834, 20645197, 64055960, 6819644, 77284619, 73021291, 99226875, 66663942, 96193415, 77072625, 36930650, 38256849, 33201905, 83269727, 62428472, 40686254, 17081350, 96709982, 52261574, 46870723, 47738185, 46540998, 40534591, 54058788, 6871053, 94076128, 10597197, 99971982, 65851721, 2717150, 48893685, 69697787, 8696647, 68694897, 2917920, 56531125, 72358170, 99125126, 45049308, 43501211, 83368048, 79191827, 44481640, 24733232, 32161669, 9599614, 10366309, 8115266, 51588015, 76540481, 35996293, 75543508, 49598724, 33336148, 97561745, 30139692, 44473167, 21533347, 13231279, 44348328, 3233084, 25842078, 38510840, 21001913, 93359396, 84840549, 26998766, 78785507, 68102437, 7104732, 73235980, 66667729, 28787861, 30998561, 49328608, 2208785, 60430369, 79922758, 91802888, 81805959, 78549759, 81274566, 83150534, 91990218, 63628376, 93515664, 37957788, 20885148, 15163258, 38008118, 69848388, 32699744, 78884452, 21993752, 38658347, 80246713, 40984766, 99604946, 38061439, 70727211, 99861373, 93790285, 40027975, 59197747, 5482538, 78589145, 98648327, 3235882, 47213183, 86301513, 10649306, 98739783, 92554120, 20765474, 31161687, 73168565, 89217461, 7300047, 42199455, 8913721, 16019925, 84002370, 1239555, 77620120, 75986488, 39847321, 6793819, 96906410, 54868730, 69786756, 59177718, 32426752, 36808486, 63152504, 51464002, 83747892, 51426311, 8791066, 17146629, 39373729, 34044787, 18131876, 43376279, 91664334, 96726697, 31126490, 55143724, 32058615, 68316156, 40781449, 37659250, 74110882, 33935899, 16684829, 68204242, 23194618, 72373496, 40356877, 95726235, 31491938, 99524975, 37435892, 83083131, 40781854, 30218878, 77187825, 11581181, 33565483, 17037369, 84187166, 39986008, 45848907, 17385531, 1250437, 48774913, 44060493, 59318837, 45995325, 40152546, 16380211, 73031054, 14220886, 82897371, 321665, 29065964, 19457589, 36753250, 16387575, 18833224, 74614639, 32274392, 68824981, 93057697, 5267545, 90158785, 23432750, 72725103, 92787493, 22405120, 16405341, 9058407, 48673079, 26102057, 59371804, 93053405, 97281847, 8373289, 8634541, 57241521, 94090109, 81855445, 92215320, 19939935, 83133790, 17068582, 8099994, 87160386, 3487592, 48395186, 62357986, 8055981, 44664587, 93270084, 57248122, 7229550, 62115552, 51315460, 20568363, 8651647, 85023028, 42947632, 14093520, 47792865, 21919959, 95290172, 61271144, 70036438, 36189527, 17976208, 15948937, 30764367, 54663246, 1197320, 7919588, 20836893, 67227442, 3633375, 45794415, 89499542, 73222868, 22942635, 44844121, 30787683, 1569515, 83789391, 8729146, 78561158, 92071990, 75407347, 63059024, 5640302, 21289531, 37675718, 60102965, 17058722, 82052050, 30543215, 13468268, 54427233, 79880247, 65081429, 2331773, 46851987, 4798568, 36845587, 78909561, 19101477, 18411915, 42073124, 33249630, 12303248, 22766820, 66885828, 5073754, 27187213, 74724075, 47708850, 86118021, 47090124, 25636669, 33237508, 16971929, 78766358, 81774825, 7517032, 52060076, 91938191, 49271185, 1204161, 86798033, 14363867, 87720882, 39201414, 45919976, 16567550, 12024238, 1375023, 59501487, 5822202, 61982238, 57658654, 9603598, 11757872, 62762857, 26744917, 50567636, 8128637, 6986898, 82726008, 99224392, 17727650, 7502255, 97057187, 61897582, 66832478, 53888755, 91255408, 44550764, 9398733, 89637706, 88904910, 82339363, 15148031, 36139942, 92803766, 83210802, 40677414, 59109400, 13470059, 17764950, 41092172, 70800879, 85242963, 83533741, 79942022, 14947650, 91727510, 2891150, 45790169, 68041839, 45617087, 80251430, 29510992, 69579137, 71965942, 72019362, 88251446, 40865610, 53084256, 68702632, 79657802, 508198, 78602717, 30694952, 1936762, 60106217, 36933038, 54014062, 84406788, 64848072, 4199704, 4978543, 72495719, 14626618, 32159704, 34236719, 14723410, 15015906, 16595436, 70596786, 19898053, 73392814, 69605283, 21070110, 21673260, 33061250, 3773993, 30395570, 61373987, 20867149, 15064655, 19486173, 48360198, 65880522, 44784505, 13819358, 42644903, 75777973, 60581278, 33553959, 37183543, 39171895, 11543098, 48260151, 9175338, 48658605, 92998591, 70367851, 71469330, 93562257, 70541760, 56461322, 33699435, 22113133, 59329511, 38022834, 77787724, 2607799, 85711894, 16424199, 49641577, 61859143, 24915585, 27625735, 82532312, 76960303, 73786944, 94711842, 55247455, 78300864, 3294781, 824872, 34497327, 74452589, 55669657, 26292919, 53632373, 83155442, 99515901, 4064751, 79821897, 6521313, 67124282, 77300457, 61960400, 92530431, 93566986, 23848565, 38645117, 45407418, 20642888, 45667668, 19891772, 58208470, 96852321, 27185644, 90013093, 72274002, 16445503, 75153252, 91141395, 88653118, 45237957, 95581843, 26776922, 55602660, 3183975, 74137926, 91831487, 26397786, 36580610, 10358899, 9860195, 59405277, 92398073, 53802686, 24058273, 20486294, 82178706, 30891921, 62490109, 68110330, 86460488, 64087743, 79738755, 76434144, 62803858, 29959549, 43933006, 98653983, 85117092, 72614359, 91957544, 57163802, 92604458, 43322743, 41245325, 23740167, 4099191, 7011964, 29834512, 27041967, 74357852, 77898274, 57359924, 90004325, 20122224, 72732205, 65074535, 4069912, 99021067, 29401781, 85571389, 72357096, 20140249, 68939068, 67513640, 37192445, 12664567, 77413012, 14349098, 5832946, 30771409, 71657078, 96318094, 97641116, 54987042, 86821229, 44983451, 22721500, 74743862, 56515456, 5111370, 61859581, 44842615, 90310261, 60176618, 4787945, 99549373, 4095116, 94516935, 26139110, 4434662, 29516592, 47361209, 98462867, 88698958, 43357947, 90457870, 99658235, 58749, 10961421, 34698428, 82427263, 75052463, 66903004, 9860968, 88444207, 28734791, 89996536, 59614746, 98130363, 24591705, 61712234, 53393358, 55770687, 53648154, 22108665, 62552524, 45990383, 6111563, 4204661, 26063929, 80014588, 84293052, 30463802, 44245960, 37620363, 18504197, 67281495, 5970607, 92692978, 66428795, 8733068, 74441448, 37166608, 99333375, 15536795, 82779622, 37891451, 67793644, 31904591, 69136837, 4806458, 33431960, 9623492, 54199160, 14326617, 70782102, 49236559, 37280276, 51590803, 81677380, 85695762, 81898046, 37224844, 39801351, 29932657, 51047803, 44889423, 58751351, 14045383, 36135, 28851716, 27665211, 24314885, 24953498, 98948034, 62496012, 36780454, 34946859, 30163921, 44846932, 36812683, 91281584, 26493119, 18001617, 76315420, 28796059, 47893286, 6497830, 77272628, 26734892, 17857111, 87598888, 31733363, 23134715, 89804152, 95957797, 88047921, 18699206, 247198, 94595668, 75128745, 6153406, 86543538, 82651278, 50702367, 95251277, 80316608, 41442762, 97783876, 17016156, 21397057, 72738685, 57020481 +69605283, 4064751, 62452034, 72019362, 3880712, 76170907, 60581278, 16097038, 48892201, 14731700, 6819644, 31727379, 14220886, 88904910, 51588015, 58749, 38494874, 98943869, 4199704, 40984766, 7423788, 84293052, 65081429, 112651, 93562257, 79136082, 70541760, 56473732, 95957797, 77787724, 74357852, 7182642, 18806856, 79322415, 39553046, 4095116, 18001617, 17957593, 54762643, 26114953, 59981773, 10358899, 44915235, 17365188, 62490109, 44177011, 89046466, 37675718, 77312810, 6111563, 76671482, 8913721, 84904436, 78909561, 61741594, 63967300, 37501808, 63152504, 31126490, 54232247, 69355476, 20645197, 74441448, 66319530, 73021291, 72357096, 59501487, 96193415, 11581181, 96709982, 82779622, 29819940, 40534591, 94076128, 97057187, 168541, 26664538, 9599614, 8115266, 90158785, 45617087, 58208470, 43357947, 6038457, 78549759, 9860968, 95010552, 9829782, 97011160, 67227442, 21993752, 35456853, 33061250, 70004753, 30090481, 9575954, 75407347, 98739783, 55850790, 29932657, 47887470, 76703609, 73617245, 4798568, 36845587, 62936963, 57163802, 16099750, 98371444, 70372191, 33249630, 18663507, 4099191, 27041967, 96726697, 90004325, 4091162, 27625735, 65047700, 99021067, 72238278, 55247455, 83083131, 30218878, 24168411, 83269727, 57240218, 57803235, 30771409, 66832478, 54987042, 90090964, 44550764, 56515456, 526217, 95251277, 71083565, 84349107, 69136837, 13470059, 14947650, 21533347, 44348328, 69641513, 76315420, 53084256, 11923835, 91141395, 6221471, 93270084, 28787861, 33895336, 96735716, 96420429, 79922758, 78602717, 23569917, 61271144, 36933038, 26392416, 42237907, 749283, 70036438, 6497830, 15075176, 9860195, 6157724, 77272628, 32159704, 34236719, 81853704, 70596786, 21070110, 21673260, 176257, 93790285, 40027975, 8729146, 12571310, 76434144, 22108665, 10649306, 42644903, 29959549, 569864, 82052050, 52293995, 30653863, 2331773, 1239555, 91957544, 39847321, 7685448, 51047803, 69786756, 22766820, 66885828, 7646095, 71469330, 58751351, 34044787, 68644627, 38022834, 41481685, 32058615, 52060076, 24314885, 18699206, 50188404, 9257405, 40356877, 51466049, 12024238, 99524975, 99333375, 37192445, 84127901, 40686254, 82726008, 14349098, 53888755, 44846932, 43506672, 61960400, 45075471, 92530431, 82339363, 44481640, 15148031, 91240048, 60176618, 23432750, 17764950, 27411561, 79942022, 94516935, 75543508, 4434662, 97561745, 8634541, 57961282, 71965942, 27185644, 90013093, 90457870, 93359396, 87160386, 34698428, 9623492, 66250369, 28796059, 33628349, 51315460, 75052463, 60106217, 1788101, 48675329, 61712234, 38658347, 53648154, 12416768, 99604946, 54263880, 64602895, 64157906, 61728685, 33123618, 21289531, 77301523, 33553959, 48088883, 55189057, 11543098, 26063929, 63506281, 26507214, 72614359, 30463802, 49597667, 15535065, 72738685, 6793819, 42692881, 44245960, 81172706, 36808486, 6505939, 51464002, 51426311, 55960386, 7011964, 29834512, 55143724, 85711894, 14045383, 49641577, 68316156, 57359924, 90654836, 33935899, 19272365, 66428795, 16684829, 68204242, 72373496, 29994197, 8733068, 60393039, 78300864, 20002147, 98948034, 77072625, 11757872, 97783876, 87710366, 33565483, 17037369, 8128637, 17385531, 46870723, 71657078, 10453030, 6871053, 37891451, 65851721, 73031054, 72278539, 22721500, 92353856, 62693428, 89637706, 36812683, 36753250, 16387575, 32274392, 79191827, 23848565, 5267545, 36139942, 90310261, 70800879, 97940276, 59371804, 80316608, 22435353, 90272749, 33431960, 69579137, 47361209, 17539625, 84684495, 88698958, 19939935, 25842078, 22450468, 26998766, 68102437, 63372756, 65271999, 73235980, 66667729, 95581843, 82427263, 62115552, 81805959, 42947632, 47792865, 75229462, 21919959, 36468541, 91990218, 51472275, 36189527, 17976208, 18466635, 16595436, 32699744, 24058273, 65338021, 73392814, 55770687, 81898046, 68110330, 30395570, 70727211, 31733363, 37224844, 75135448, 19486173, 65880522, 5482538, 77377183, 3235882, 8129978, 5640302, 68875490, 13819358, 92554120, 92867155, 62803858, 82886381, 1022677, 89217461, 42967683, 72793444, 98653983, 13468268, 66322959, 51507425, 874791, 44847298, 37739481, 92541302, 34295794, 29188588, 99690194, 67281495, 7066775, 50007421, 56461322, 17146629, 33237508, 52204879, 2607799, 91664334, 82651278, 71920426, 7687278, 87720882, 23194618, 29401781, 10264691, 83302115, 1375023, 67474219, 3294781, 61982238, 9603598, 41092102, 86022504, 37166608, 56955985, 26292919, 17081350, 1250437, 22129328, 45995325, 247198, 10597197, 34432810, 50806615, 669105, 11365791, 82897371, 69697787, 2917920, 9886593, 45049308, 65038678, 18833224, 83368048, 32161669, 92803766, 59109400, 99549373, 41092172, 16405341, 76540481, 83533741, 93053405, 33797252, 49598724, 68041839, 97281847, 44473167, 57241521, 13173644, 29510992, 75820087, 98462867, 38510840, 33304202, 17894977, 78785507, 40865610, 28550822, 3487592, 8055981, 3088684, 36312813, 68702632, 60430369, 91802888, 30694952, 81274566, 89419466, 14093520, 14326617, 70782102, 36580610, 63628376, 37957788, 65454636, 92398073, 20885148, 90061527, 22997281, 68128438, 62430984, 15163258, 6525948, 26734892, 7919588, 20836893, 64087743, 3773993, 61373987, 20867149, 99861373, 98648327, 68000591, 44784505, 86301513, 43045786, 38341669, 73168565, 86543538, 94360702, 43933006, 3233569, 53666583, 17058722, 40197395, 36685023, 99729357, 16019925, 46851987, 19101477, 83948335, 75986488, 29635537, 48658605, 71300104, 92604458, 59177718, 42073124, 12303248, 77694700, 27187213, 12348118, 74724075, 37620363, 83747892, 6808825, 18504197, 47090124, 33699435, 25636669, 8791066, 41442762, 22113133, 71316369, 43376279, 88047921, 59910624, 77898274, 30811010, 37659250, 79806380, 86798033, 76960303, 14363867, 39201414, 73786944, 16567550, 56484900, 64055960, 86242799, 4668450, 56424103, 70420215, 36930650, 74452589, 55669657, 54606384, 36780454, 33201905, 89699445, 62428472, 67513640, 39986008, 45381876, 45996863, 12664567, 89078848, 50702367, 17386996, 99224392, 21397057, 7502255, 46540998, 34946859, 54058788, 86821229, 4985896, 74743862, 8696647, 9398733, 57020481, 43501211, 74614639, 24733232, 61859581, 44842615, 93057697, 4787945, 35512853, 45407418, 35996293, 91727510, 2891150, 26493119, 29516592, 8373289, 78218845, 92215320, 26863229, 3233084, 21001913, 17068582, 16445503, 84840549, 88251446, 99658235, 61623915, 88653118, 7104732, 58224549, 53842979, 26776922, 66045587, 30998561, 2208785, 57248122, 3183975, 8651647, 1936762, 68816413, 55470718, 67084351, 57393458, 30366150, 54014062, 13862149, 84406788, 37280276, 47893286, 93515664, 28734791, 67451935, 4978543, 10309525, 32250045, 27325428, 69848388, 17857111, 15015906, 20681921, 45794415, 15902805, 8807940, 38009615, 24619760, 87598888, 55753905, 24826575, 45990383, 52613508, 47213183, 92071990, 30569392, 95395112, 20765474, 31161687, 82979980, 37183543, 7300047, 39171895, 26275734, 89804152, 42199455, 34698463, 42307449, 55615885, 80555751, 36505482, 35192533, 7955293, 415901, 73124510, 23110625, 9175338, 9188443, 43322743, 71522968, 23740167, 99965001, 18783702, 4508700, 59329511, 5970607, 29466406, 18131876, 97379790, 16424199, 27665211, 49271185, 1204161, 85571389, 45919976, 95726235, 94711842, 31491938, 43152977, 60955663, 40781854, 77284619, 20140249, 66663942, 62762857, 77187825, 84187166, 68939068, 26744917, 45848907, 53632373, 6986898, 76330843, 48774913, 5832946, 32151165, 30163921, 44983451, 48893685, 56531125, 321665, 72358170, 44627776, 91281584, 77300457, 54517921, 93566986, 68824981, 40677414, 72725103, 4806458, 22405120, 85242963, 66137019, 45790169, 33336148, 80251430, 81855445, 13231279, 3509435, 86001008, 83133790, 35092039, 72274002, 51715482, 49882705, 28928175, 75153252, 75128745, 42782652, 508198, 4515343, 7229550, 85023028, 83150534, 95290172, 88444207, 26397786, 49236559, 53802686, 34667286, 34493392, 94539824, 89996536, 84166196, 64098930, 14723410, 65715134, 53547802, 53393358, 89499542, 20486294, 44844121, 80246713, 38061439, 30787683, 83789391, 15064655, 61928316, 62552524, 78561158, 69255765, 62051033, 61263068, 60102965, 58180653, 30543215, 51149804, 85117092, 62740044, 32590267, 37560164, 92692283, 18411915, 89214616, 54868730, 92998591, 29029316, 47708850, 70367851, 73109997, 39373729, 89811711, 78766358, 7517032, 61859143, 28851716, 92692978, 74110882, 65074535, 63015256, 4069912, 44479073, 56153345, 24953498, 1431742, 824872, 34497327, 50567636, 17727650, 59318837, 16380211, 94595668, 67793644, 97641116, 2717150, 6521313, 68694897, 67124282, 5111370, 94911072, 19457589, 10366309, 19376156, 83210802, 83651211, 92787493, 20642888, 45667668, 19891772, 8099994, 79657802, 9259676, 20568363, 91831487, 11161731, 51590803, 59614746, 54663246, 1197320, 24591705, 3633375, 6153406, 78884452, 85695762, 82178706, 22879907, 22942635, 86460488, 78589145, 75777973, 17016156, 93933709, 66271566, 80014588, 84002370, 38365584, 41380093, 45428665, 2204165, 41245325, 86118021, 99297164, 16971929, 16054533, 24915585, 40781449, 4119747, 82532312, 91938191, 37435892, 38256849, 77413012, 52261574, 47738185, 44060493, 40152546, 91255408, 99125126, 61141874, 29065964, 38645117, 48673079, 30139692, 94090109, 48395186, 74137926, 64848072, 15948937, 30764367, 30891921, 1569515, 79738755, 48360198, 67031644, 13348726, 39801351, 4204661, 79880247, 77620120, 85116755, 32426752, 36135, 81774825, 20122224, 72732205, 92033260, 50668599, 72777973, 47298834, 15536795, 36396314, 99515901, 96318094, 65017137, 99917599, 31904591, 82327024, 26139110, 44664587, 54199160, 66903004, 22200378, 72495719, 81677380, 14626618, 73222868, 59197747, 88130087, 65275241, 48260151, 54427233, 5073754, 44889423, 71333116, 67030811, 5822202, 57658654, 79821897, 45306070, 45237957, 55602660, 59405277, 38008118, 23134715, 61815223, 96906410, 62496012, 99226875, 83155442, 99971982, 9058407, 96852321, 62357986, 49328608, 98130363, 19898053, 63059024, 61897582, 76825057, 26102057, 10961421, 66611759, 62232211, 66116458 +84840549, 19376156, 72274002, 60430369, 96906410, 4508700, 24915585, 65047700, 26744917, 67793644, 16387575, 49882705, 62357986, 20765474, 89214616, 97379790, 33935899, 91938191, 12664567, 50702367, 99971982, 33431960, 6221471, 9259676, 36189527, 17976208, 53802686, 38008118, 24058273, 37224844, 59197747, 69255765, 86543538, 5073754, 72732205, 7687278, 16567550, 70420215, 9603598, 86022504, 36396314, 54058788, 92353856, 29065964, 45049308, 71083565, 31904591, 69136837, 72725103, 9058407, 21533347, 19939935, 3233084, 90013093, 58224549, 66250369, 3183975, 20568363, 91831487, 14626618, 54663246, 6525948, 65338021, 8129978, 55850790, 60581278, 3233569, 26275734, 415901, 45428665, 54868730, 37620363, 79806380, 24314885, 12024238, 1431742, 67474219, 824872, 74452589, 15536795, 47738185, 6871053, 97057187, 82897371, 45306070, 67124282, 61960400, 36139942, 92803766, 40677414, 99549373, 92787493, 33797252, 30139692, 78218845, 44348328, 69641513, 76315420, 93359396, 75128745, 10961421, 44664587, 54199160, 30998561, 75052463, 75229462, 83150534, 95290172, 57393458, 63628376, 59405277, 27325428, 90061527, 30764367, 69848388, 15015906, 3633375, 55770687, 81898046, 30891921, 44844121, 8807940, 54263880, 176257, 8729146, 62552524, 45990383, 86301513, 7423788, 5640302, 61728685, 11543098, 66322959, 62740044, 32590267, 37560164, 18411915, 98371444, 32426752, 42692881, 29029316, 73109997, 6808825, 34044787, 5970607, 29466406, 40781449, 4119747, 92033260, 85571389, 73786944, 8733068, 43152977, 40781854, 99226875, 55669657, 37166608, 33201905, 99333375, 50567636, 44060493, 34946859, 96318094, 16380211, 61897582, 91255408, 74743862, 56531125, 36812683, 91281584, 36753250, 82339363, 68824981, 44481640, 82327024, 44842615, 20642888, 94516935, 29516592, 29510992, 84684495, 17068582, 78785507, 53084256, 66611759, 3088684, 95581843, 74137926, 85023028, 47792865, 26397786, 13862149, 93515664, 28734791, 65454636, 51590803, 81677380, 32159704, 84166196, 73222868, 62232211, 68110330, 12416768, 86460488, 30395570, 61373987, 1569515, 79738755, 55753905, 64157906, 98648327, 92071990, 75407347, 30569392, 66116458, 39801351, 95395112, 42644903, 65275241, 62803858, 75777973, 73168565, 33123618, 82979980, 42967683, 7300047, 42199455, 66271566, 34698463, 42307449, 48260151, 63506281, 54427233, 84002370, 35192533, 30463802, 73124510, 77620120, 29635537, 92604458, 42073124, 47708850, 70367851, 71522968, 18504197, 41245325, 67281495, 56461322, 33699435, 17146629, 39373729, 33237508, 71316369, 59329511, 95957797, 71333116, 32058615, 7517032, 49271185, 68204242, 23194618, 72777973, 47298834, 20645197, 83083131, 4668450, 77284619, 62496012, 59501487, 57658654, 54606384, 17037369, 39986008, 45848907, 76330843, 48774913, 21397057, 5832946, 97641116, 72278539, 14220886, 9886593, 57020481, 65038678, 18833224, 54517921, 39553046, 15148031, 38645117, 51588015, 41092172, 26102057, 83533741, 27411561, 22435353, 93053405, 49598724, 19891772, 58208470, 35092039, 8099994, 16445503, 22450468, 26998766, 68102437, 88251446, 40865610, 8055981, 49328608, 2208785, 96420429, 26114953, 68816413, 60106217, 55470718, 14326617, 36468541, 84406788, 18466635, 34493392, 59614746, 34236719, 17857111, 7919588, 81853704, 20681921, 16595436, 65715134, 53547802, 6153406, 45794415, 53393358, 85695762, 21993752, 38658347, 20486294, 22879907, 22942635, 40984766, 31733363, 64602895, 88130087, 67031644, 22108665, 3235882, 47213183, 13819358, 31161687, 62051033, 93933709, 48088883, 55189057, 40197395, 36685023, 80014588, 46851987, 48892201, 83948335, 38365584, 41380093, 92541302, 29188588, 57163802, 85116755, 99690194, 59177718, 12303248, 66885828, 77694700, 44889423, 2204165, 23740167, 50007421, 25636669, 58751351, 27041967, 74357852, 19272365, 14363867, 44479073, 16684829, 50668599, 72373496, 29994197, 39201414, 18806856, 37435892, 24953498, 60955663, 14731700, 67030811, 5822202, 66663942, 96193415, 11757872, 41092102, 68939068, 17081350, 14349098, 29819940, 46540998, 32151165, 37891451, 34432810, 94595668, 54987042, 86821229, 6521313, 90090964, 11365791, 65017137, 88904910, 99917599, 10366309, 23848565, 83210802, 90310261, 70800879, 14947650, 91727510, 2891150, 26139110, 4434662, 68041839, 13173644, 69579137, 92215320, 57961282, 25842078, 83133790, 38510840, 33304202, 51715482, 99658235, 3487592, 88653118, 73235980, 45237957, 66667729, 36312813, 96735716, 4515343, 57248122, 7229550, 81805959, 23569917, 30694952, 8651647, 1936762, 91990218, 54014062, 10358899, 47893286, 51472275, 48675329, 67451935, 20885148, 15948937, 34667286, 72495719, 77272628, 17365188, 97011160, 1197320, 26734892, 15902805, 64087743, 99604946, 70727211, 87598888, 19486173, 24826575, 5482538, 78589145, 77377183, 9575954, 78561158, 98739783, 38341669, 92867155, 94360702, 43933006, 4204661, 58180653, 82052050, 13468268, 51507425, 79880247, 73617245, 2331773, 80555751, 61815223, 7955293, 19101477, 61741594, 91957544, 15535065, 72738685, 39847321, 6793819, 9175338, 7685448, 71300104, 70372191, 112651, 33249630, 22766820, 74724075, 44245960, 83747892, 70541760, 51426311, 18783702, 86118021, 47090124, 7011964, 22113133, 77787724, 16971929, 2607799, 91664334, 78766358, 16054533, 36135, 54232247, 92692978, 74110882, 1204161, 86798033, 87720882, 9257405, 45919976, 10264691, 51466049, 94711842, 31491938, 1375023, 56484900, 55247455, 78300864, 20002147, 30218878, 20140249, 34497327, 77187825, 38256849, 24168411, 83269727, 62428472, 56955985, 77413012, 53632373, 40686254, 99515901, 22129328, 99224392, 79821897, 66832478, 2717150, 69697787, 68694897, 2917920, 9398733, 44627776, 95251277, 61859581, 84349107, 4787945, 4806458, 45407418, 22405120, 16405341, 35996293, 66137019, 75543508, 59371804, 45790169, 33336148, 26493119, 45667668, 45617087, 97281847, 80251430, 47361209, 75820087, 3509435, 17894977, 72019362, 63372756, 48395186, 68702632, 28796059, 3880712, 61271144, 1788101, 67084351, 30366150, 64848072, 4978543, 15075176, 11161731, 10309525, 15163258, 14723410, 98130363, 24591705, 61712234, 89499542, 62490109, 44177011, 30787683, 20867149, 76170907, 65880522, 52613508, 63059024, 43045786, 29932657, 29959549, 569864, 37183543, 23134715, 60102965, 98653983, 52293995, 16019925, 84904436, 30653863, 65081429, 78909561, 36505482, 1239555, 23110625, 69786756, 27187213, 12348118, 18663507, 71469330, 6505939, 7066775, 8791066, 99297164, 89811711, 38022834, 29834512, 52204879, 96726697, 31126490, 16424199, 49641577, 68316156, 52060076, 27625735, 69355476, 27665211, 18699206, 65074535, 56153345, 60393039, 73021291, 3294781, 61982238, 31727379, 33565483, 89078848, 84127901, 17385531, 52261574, 4064751, 82779622, 71657078, 10453030, 45995325, 40152546, 10597197, 65851721, 73031054, 50806615, 48893685, 8696647, 56515456, 89637706, 94911072, 72358170, 99125126, 44846932, 61141874, 19457589, 526217, 77300457, 83368048, 79191827, 93566986, 9599614, 93057697, 91240048, 59109400, 60176618, 90158785, 4095116, 81855445, 88698958, 21001913, 61623915, 65271999, 9623492, 93270084, 26776922, 66045587, 42782652, 508198, 79922758, 91802888, 33628349, 38494874, 98943869, 81274566, 42947632, 14093520, 21919959, 37280276, 4199704, 6497830, 9860195, 6157724, 92398073, 22997281, 62430984, 94539824, 64098930, 20836893, 70596786, 19898053, 53648154, 3773993, 93790285, 12571310, 48360198, 76434144, 68000591, 44784505, 10649306, 61263068, 77301523, 77312810, 16097038, 39171895, 89804152, 72793444, 30543215, 85117092, 8913721, 99729357, 44847298, 49597667, 92692283, 34295794, 75986488, 9188443, 92998591, 7646095, 37501808, 51464002, 99965001, 41442762, 68644627, 18131876, 43376279, 88047921, 7182642, 55143724, 59910624, 77898274, 37659250, 71920426, 82532312, 4069912, 29401781, 72238278, 95726235, 83302115, 74441448, 64055960, 86242799, 66319530, 62762857, 36780454, 11581181, 84187166, 45381876, 6986898, 30771409, 40534591, 62693428, 321665, 32274392, 45075471, 5267545, 83651211, 35512853, 17764950, 76540481, 85242963, 18001617, 90272749, 44473167, 17539625, 13231279, 96852321, 17957593, 43357947, 28928175, 11923835, 91141395, 53842979, 6038457, 78602717, 89419466, 59981773, 36580610, 49236559, 22200378, 749283, 37957788, 32250045, 89996536, 32699744, 73392814, 78884452, 69605283, 38061439, 33061250, 24619760, 89046466, 83789391, 75135448, 13348726, 30090481, 21289531, 89217461, 53666583, 26063929, 874791, 4798568, 37739481, 16099750, 48658605, 51047803, 63967300, 43322743, 81172706, 36808486, 63152504, 55960386, 4099191, 61859143, 57359924, 20122224, 28851716, 76960303, 40356877, 6819644, 56424103, 87710366, 67513640, 57803235, 17386996, 96709982, 82726008, 17727650, 59318837, 30163921, 44983451, 22721500, 44550764, 5111370, 62452034, 43501211, 74614639, 24733232, 26664538, 32161669, 76825057, 8115266, 48673079, 79942022, 80316608, 97561745, 8373289, 57241521, 98462867, 26863229, 71965942, 87160386, 55602660, 36933038, 26392416, 42237907, 44915235, 68128438, 21070110, 82178706, 21673260, 80246713, 99861373, 47887470, 17016156, 17058722, 51149804, 84293052, 26507214, 93562257, 14045383, 90004325, 66428795, 63015256, 50188404, 99524975, 98948034, 72357096, 77072625, 97783876, 89699445, 37192445, 83155442, 1250437, 46870723, 7502255, 168541, 669105, 97940276, 8634541, 94090109, 90457870, 28550822, 75153252, 34698428, 7104732, 78549759, 66903004, 70036438, 38009615, 61928316, 92554120, 1022677, 33553959, 6111563, 55615885, 72614359, 79136082, 81774825, 82651278, 4091162, 30811010, 79322415, 8128637, 26292919, 247198, 13470059, 86001008, 27185644, 58749, 54762643, 28787861, 79657802, 33895336, 9860968, 95010552, 88444207, 9829782, 67227442, 70004753, 40027975, 15064655, 76671482, 76703609, 36845587, 62936963, 85711894, 41481685, 90654836, 94076128, 4985896, 62115552, 51315460, 35456853, 68875490, 82886381, 99021067, 53888755, 43506672, 92530431, 23432750, 70782102, 37675718, 36930650, 45996863, 57240218, 82427263, 56473732 +99524975, 9860195, 75135448, 96735716, 36580610, 99861373, 36845587, 8791066, 17146629, 94595668, 92787493, 40984766, 12416768, 52293995, 62740044, 18783702, 25636669, 5970607, 85711894, 57359924, 68204242, 68939068, 37192445, 12664567, 17081350, 48774913, 96318094, 56515456, 526217, 26664538, 19376156, 35996293, 80251430, 84840549, 49882705, 95581843, 53842979, 66045587, 65715134, 53648154, 93790285, 30569392, 8129978, 92867155, 82886381, 37675718, 6111563, 7955293, 19101477, 77620120, 34295794, 51426311, 37659250, 65074535, 44479073, 56424103, 36780454, 84127901, 50702367, 99515901, 46870723, 86821229, 22721500, 2917920, 61141874, 5267545, 94516935, 98462867, 68102437, 91990218, 44915235, 6157724, 24058273, 86460488, 61373987, 64602895, 68000591, 7423788, 60581278, 33123618, 26275734, 40197395, 79880247, 92541302, 57163802, 98371444, 71300104, 2204165, 58751351, 33237508, 41481685, 52060076, 4091162, 74110882, 1204161, 74441448, 64055960, 73021291, 76330843, 21397057, 34946859, 97641116, 62693428, 71083565, 45075471, 79191827, 44481640, 10366309, 36139942, 84349107, 91240048, 85242963, 45617087, 44473167, 29510992, 27185644, 87160386, 53084256, 75128745, 508198, 96420429, 91802888, 9259676, 1788101, 26392416, 22200378, 4978543, 97011160, 22879907, 21673260, 33061250, 30395570, 21289531, 7300047, 42199455, 76671482, 11543098, 42307449, 63506281, 30653863, 30463802, 39847321, 96906410, 112651, 43322743, 6808825, 89811711, 38022834, 16971929, 30811010, 33935899, 66428795, 50668599, 9257405, 51466049, 1431742, 20002147, 62496012, 83269727, 99224392, 44060493, 50806615, 53888755, 91255408, 6521313, 11365791, 9398733, 62452034, 72358170, 65038678, 68824981, 15148031, 32161669, 69136837, 83210802, 40677414, 13470059, 35512853, 27411561, 91727510, 26998766, 48395186, 54762643, 3088684, 66250369, 36312813, 42782652, 4515343, 55602660, 23569917, 30694952, 38494874, 98943869, 91831487, 60106217, 42947632, 55470718, 21919959, 67084351, 13862149, 84406788, 10358899, 4199704, 36189527, 51590803, 68128438, 69848388, 32699744, 65338021, 82178706, 44844121, 70004753, 20867149, 87598888, 37224844, 24826575, 65880522, 67031644, 13348726, 3235882, 75407347, 20765474, 62803858, 4204661, 30543215, 26063929, 13468268, 80014588, 84904436, 65081429, 84002370, 35192533, 61741594, 415901, 1239555, 91957544, 38365584, 37560164, 41380093, 15535065, 54868730, 33249630, 12303248, 5073754, 74724075, 70367851, 79136082, 18504197, 23740167, 99965001, 86118021, 50007421, 22113133, 29834512, 7182642, 82651278, 4119747, 65047700, 19272365, 50188404, 95726235, 47298834, 78300864, 60955663, 4668450, 67474219, 3294781, 59501487, 38256849, 54606384, 89699445, 33565483, 26744917, 94076128, 669105, 90090964, 74743862, 48893685, 321665, 29065964, 36753250, 45049308, 54517921, 39553046, 83368048, 99917599, 82339363, 93057697, 60176618, 99549373, 22405120, 16405341, 48673079, 26102057, 14947650, 97940276, 18001617, 13173644, 69641513, 84684495, 21001913, 76315420, 40865610, 28550822, 63372756, 65271999, 58224549, 8055981, 93270084, 3880712, 68816413, 59981773, 34493392, 38008118, 54663246, 24591705, 15015906, 53393358, 78884452, 21993752, 38658347, 62490109, 64087743, 99604946, 24619760, 44177011, 1569515, 15064655, 76170907, 78589145, 45990383, 98739783, 39801351, 95395112, 75777973, 1022677, 93933709, 23134715, 60102965, 53666583, 72793444, 58180653, 36685023, 34698463, 76703609, 55615885, 26507214, 80555751, 9188443, 18411915, 70372191, 69786756, 42073124, 81172706, 71522968, 18663507, 37620363, 37501808, 36808486, 47090124, 39373729, 59329511, 95957797, 77787724, 29466406, 52204879, 18131876, 2607799, 96726697, 16054533, 16424199, 49641577, 72732205, 69355476, 49271185, 63015256, 14363867, 56153345, 23194618, 85571389, 29994197, 39201414, 73786944, 18806856, 60393039, 77284619, 20140249, 5822202, 66663942, 34497327, 70420215, 11757872, 62762857, 77187825, 55669657, 24168411, 33201905, 99333375, 31727379, 8128637, 77413012, 57803235, 17385531, 82726008, 4064751, 17727650, 46540998, 6871053, 37891451, 16380211, 10597197, 34432810, 72278539, 4985896, 89637706, 65017137, 16387575, 32274392, 9599614, 23848565, 90310261, 59109400, 4787945, 41092172, 76540481, 66137019, 75543508, 80316608, 45790169, 29516592, 78218845, 47361209, 21533347, 92215320, 75820087, 3509435, 44348328, 38510840, 90013093, 17068582, 22450468, 88251446, 91141395, 34698428, 66611759, 62357986, 44664587, 66667729, 28787861, 54199160, 79657802, 26776922, 30998561, 62115552, 8651647, 14326617, 42237907, 30366150, 63628376, 49236559, 37280276, 64848072, 93515664, 37957788, 27325428, 77272628, 62430984, 30764367, 64098930, 14723410, 98130363, 26734892, 17857111, 20836893, 20681921, 19898053, 55770687, 62232211, 15902805, 38061439, 3773993, 31733363, 55753905, 19486173, 5482538, 44784505, 52613508, 69255765, 92554120, 29932657, 77312810, 82979980, 43933006, 89804152, 55189057, 54427233, 2331773, 36505482, 61815223, 83948335, 49597667, 92692283, 75986488, 45428665, 7685448, 51047803, 92604458, 63967300, 22766820, 29029316, 47708850, 7646095, 71469330, 73109997, 6505939, 51464002, 83747892, 67281495, 55960386, 4099191, 56473732, 56461322, 33699435, 4508700, 99297164, 27041967, 91664334, 88047921, 32058615, 7517032, 77898274, 24915585, 28851716, 86798033, 99021067, 29401781, 72238278, 10264691, 56484900, 37435892, 83083131, 40781854, 67030811, 96193415, 57658654, 9603598, 79322415, 97783876, 84187166, 67513640, 39986008, 56955985, 45848907, 45381876, 89078848, 36396314, 53632373, 22129328, 30771409, 99971982, 65851721, 61897582, 66832478, 14220886, 2717150, 82897371, 69697787, 8696647, 45306070, 18833224, 43506672, 43501211, 61960400, 24733232, 92803766, 8115266, 72725103, 17764950, 20642888, 9058407, 70800879, 79942022, 22435353, 4434662, 68041839, 97281847, 8373289, 8634541, 69579137, 17539625, 57961282, 26863229, 86001008, 3233084, 33304202, 35092039, 99658235, 61623915, 75153252, 10961421, 11923835, 6221471, 7104732, 60430369, 57248122, 79922758, 6038457, 74137926, 66903004, 85023028, 36468541, 36933038, 88444207, 67451935, 59405277, 15948937, 22997281, 14626618, 94539824, 7919588, 45794415, 69605283, 85695762, 73222868, 30891921, 89046466, 79738755, 40027975, 48360198, 76434144, 64157906, 98648327, 61928316, 62552524, 9575954, 47213183, 78561158, 13819358, 31161687, 62051033, 61728685, 55850790, 33553959, 86543538, 39171895, 3233569, 98653983, 66322959, 51507425, 73617245, 46851987, 44847298, 48892201, 32590267, 73124510, 29188588, 16099750, 48658605, 92998591, 77694700, 27187213, 93562257, 41245325, 7011964, 41442762, 55143724, 97379790, 36135, 61859143, 90004325, 54232247, 71920426, 18699206, 72777973, 55247455, 20645197, 86242799, 14731700, 30218878, 77072625, 36930650, 37166608, 11581181, 17037369, 15536795, 57240218, 6986898, 40686254, 52261574, 14349098, 29819940, 7502255, 54058788, 59318837, 247198, 168541, 44550764, 92353856, 68694897, 44846932, 88904910, 91281584, 77300457, 74614639, 44842615, 38645117, 76825057, 90158785, 83533741, 2891150, 26139110, 33797252, 33336148, 97561745, 30139692, 33431960, 96852321, 25842078, 83133790, 17894977, 72274002, 93359396, 82427263, 3183975, 26114953, 78549759, 89419466, 47792865, 75229462, 95290172, 57393458, 47893286, 48675329, 65454636, 92398073, 53802686, 18466635, 32250045, 17365188, 32159704, 34236719, 6525948, 3633375, 73392814, 35456853, 81898046, 80246713, 68110330, 176257, 12571310, 22108665, 63059024, 10649306, 47887470, 17016156, 29959549, 569864, 37183543, 42967683, 82052050, 85117092, 99729357, 48260151, 23110625, 9175338, 89214616, 42692881, 66885828, 44245960, 63152504, 70541760, 7066775, 34044787, 68644627, 71333116, 74357852, 59910624, 20122224, 90654836, 27665211, 24314885, 7687278, 4069912, 72373496, 31491938, 24953498, 66319530, 98948034, 72357096, 824872, 74452589, 41092102, 96709982, 1250437, 71657078, 40534591, 10453030, 32151165, 79821897, 40152546, 97057187, 30163921, 67124282, 44627776, 36812683, 95251277, 31904591, 93566986, 83651211, 45407418, 93053405, 90272749, 94090109, 19891772, 71965942, 8099994, 51715482, 28928175, 58749, 88653118, 68702632, 28796059, 49328608, 51315460, 20568363, 1936762, 9860968, 81274566, 95010552, 61271144, 70036438, 28734791, 20885148, 81677380, 59614746, 61712234, 16595436, 53547802, 70596786, 20486294, 30787683, 66116458, 86301513, 5640302, 42644903, 65275241, 38341669, 77301523, 89217461, 16097038, 48088883, 17058722, 51149804, 16019925, 84293052, 72614359, 78909561, 37739481, 72738685, 29635537, 99690194, 12348118, 71316369, 43376279, 40781449, 92692978, 92033260, 87720882, 40356877, 45919976, 16567550, 83302115, 94711842, 1375023, 43152977, 6819644, 61982238, 62428472, 50567636, 45996863, 17386996, 5832946, 73031054, 67793644, 54987042, 5111370, 94911072, 9886593, 92530431, 61859581, 4806458, 59371804, 49598724, 26493119, 57241521, 19939935, 90457870, 3487592, 73235980, 78602717, 33628349, 75052463, 14093520, 83150534, 70782102, 749283, 11161731, 72495719, 89996536, 81853704, 6153406, 21070110, 83789391, 70727211, 59197747, 88130087, 30090481, 77377183, 92071990, 61263068, 94360702, 4798568, 62936963, 14045383, 81774825, 68316156, 82532312, 99226875, 83155442, 47738185, 44983451, 99125126, 19457589, 45667668, 58208470, 13231279, 88698958, 17957593, 16445503, 78785507, 9623492, 2208785, 81805959, 26397786, 9829782, 6497830, 17976208, 15075176, 90061527, 15163258, 84166196, 1197320, 67227442, 89499542, 54263880, 8729146, 43045786, 68875490, 73168565, 874791, 32426752, 78766358, 31126490, 27625735, 91938191, 76960303, 16684829, 12024238, 86022504, 26292919, 45995325, 82327024, 23432750, 51588015, 4095116, 72019362, 45237957, 51472275, 10309525, 34667286, 8807940, 66271566, 8913721, 85116755, 59177718, 44889423, 79806380, 87710366, 82779622, 56531125, 57020481, 81855445, 33895336, 7229550, 22942635, 38009615, 8733068, 54014062, 6793819, 43357947 +56424103, 91957544, 33797252, 62803858, 6111563, 53666583, 34044787, 84349107, 18001617, 72274002, 66250369, 6153406, 29959549, 90004325, 18699206, 20140249, 47738185, 46540998, 71083565, 39553046, 9599614, 68041839, 25842078, 6497830, 65454636, 32699744, 38061439, 40197395, 48260151, 84904436, 18411915, 68644627, 7182642, 97379790, 55669657, 83269727, 45848907, 14349098, 53888755, 44842615, 45667668, 58208470, 19939935, 8055981, 95581843, 749283, 54663246, 35456853, 98648327, 39801351, 23134715, 16097038, 94360702, 60102965, 89804152, 51507425, 26507214, 77620120, 59177718, 32426752, 47708850, 71469330, 51426311, 41442762, 5970607, 27041967, 88047921, 59910624, 9257405, 67474219, 99226875, 74452589, 34946859, 94076128, 22405120, 94516935, 59371804, 29516592, 99658235, 26776922, 2208785, 57248122, 7229550, 14093520, 49236559, 70036438, 92398073, 20681921, 30891921, 8807940, 3773993, 176257, 99861373, 12571310, 3235882, 69255765, 10649306, 95395112, 75777973, 43933006, 36845587, 7955293, 49597667, 41380093, 92604458, 54868730, 99690194, 81172706, 2204165, 36808486, 41245325, 4099191, 8791066, 52204879, 36135, 30811010, 19272365, 39201414, 73786944, 8733068, 24953498, 57658654, 26744917, 15536795, 83155442, 7502255, 79821897, 59318837, 45995325, 5111370, 62452034, 44846932, 45049308, 74614639, 61960400, 32274392, 31904591, 24733232, 51588015, 76540481, 45790169, 26493119, 97281847, 97561745, 21533347, 69641513, 90457870, 58749, 6221471, 79657802, 66045587, 3183975, 78602717, 1936762, 75229462, 83150534, 26392416, 13862149, 48675329, 4199704, 6525948, 61712234, 70596786, 82178706, 33061250, 30395570, 19486173, 30090481, 44784505, 47213183, 78561158, 86301513, 61728685, 82886381, 47887470, 89217461, 33553959, 4204661, 99729357, 65081429, 35192533, 34295794, 75986488, 12348118, 71522968, 6505939, 18504197, 7011964, 59329511, 95957797, 16971929, 55143724, 7517032, 85571389, 31491938, 74441448, 83083131, 60955663, 72357096, 59501487, 70420215, 89699445, 62428472, 31727379, 56955985, 12664567, 89078848, 36396314, 17385531, 76330843, 30771409, 86821229, 72278539, 22721500, 82897371, 8696647, 89637706, 65017137, 72358170, 57020481, 43506672, 43501211, 79191827, 32161669, 5267545, 40677414, 83651211, 20642888, 9058407, 48673079, 85242963, 79942022, 35996293, 14947650, 33431960, 13173644, 17894977, 72019362, 10961421, 28796059, 3880712, 42782652, 20568363, 8651647, 85023028, 47792865, 30366150, 37280276, 28734791, 37957788, 4978543, 15075176, 10309525, 81677380, 30764367, 26734892, 17857111, 67227442, 55770687, 53648154, 22879907, 62490109, 38009615, 12416768, 24619760, 54263880, 40027975, 76170907, 24826575, 68000591, 9575954, 43045786, 20765474, 55850790, 86543538, 48088883, 98653983, 58180653, 30543215, 11543098, 66322959, 4798568, 80555751, 36505482, 61741594, 32590267, 23110625, 29635537, 16099750, 7685448, 92998591, 5073754, 44245960, 70367851, 70541760, 55960386, 99965001, 86118021, 33699435, 4508700, 99297164, 33237508, 31126490, 49641577, 57359924, 4091162, 27625735, 79806380, 49271185, 65074535, 16684829, 87720882, 23194618, 29994197, 16567550, 1431742, 78300864, 86242799, 66319530, 98948034, 824872, 11757872, 38256849, 54606384, 33201905, 17037369, 84127901, 6986898, 82726008, 29819940, 5832946, 16380211, 66832478, 54987042, 14220886, 6521313, 90090964, 48893685, 45306070, 321665, 99125126, 88904910, 44627776, 91281584, 16387575, 95251277, 65038678, 54517921, 92530431, 44481640, 90310261, 45407418, 92787493, 16405341, 4095116, 27411561, 26139110, 80316608, 93053405, 45617087, 19891772, 17539625, 92215320, 26863229, 86001008, 83133790, 90013093, 76315420, 51715482, 49882705, 68102437, 88251446, 63372756, 73235980, 44664587, 28787861, 30998561, 49328608, 508198, 82427263, 55602660, 38494874, 98943869, 89419466, 95290172, 84406788, 64848072, 93515664, 18466635, 90061527, 97011160, 14723410, 20836893, 24058273, 78884452, 20486294, 22942635, 40984766, 68110330, 64087743, 44177011, 89046466, 70727211, 87598888, 37224844, 8729146, 66116458, 42644903, 38341669, 73168565, 77312810, 93933709, 42967683, 3233569, 17058722, 55189057, 76671482, 8913721, 34698463, 55615885, 30653863, 2331773, 37739481, 48892201, 73124510, 92541302, 85116755, 98371444, 48658605, 51047803, 71300104, 42073124, 42692881, 37501808, 93562257, 51464002, 67281495, 39373729, 89811711, 29466406, 2607799, 96726697, 78766358, 77898274, 24915585, 40781449, 90654836, 28851716, 92692978, 74110882, 33935899, 66428795, 44479073, 68204242, 72238278, 45919976, 10264691, 60393039, 1375023, 99524975, 43152977, 20645197, 40781854, 97783876, 87710366, 36780454, 11581181, 99333375, 84187166, 57240218, 17386996, 99224392, 48774913, 54058788, 37891451, 10597197, 94595668, 99971982, 73031054, 168541, 44983451, 4985896, 2717150, 74743862, 62693428, 9886593, 19457589, 77300457, 68824981, 15148031, 38645117, 13470059, 70800879, 2891150, 75543508, 30139692, 78218845, 57961282, 75820087, 44348328, 84684495, 88698958, 3233084, 17957593, 17068582, 8099994, 16445503, 87160386, 28550822, 54762643, 68702632, 91802888, 74137926, 26114953, 23569917, 33628349, 51315460, 9259676, 91831487, 14326617, 21919959, 36468541, 67084351, 42237907, 10358899, 22200378, 9829782, 51472275, 44915235, 67451935, 9860195, 6157724, 34493392, 15163258, 89996536, 34236719, 7919588, 81853704, 53547802, 45794415, 89499542, 85695762, 21070110, 81898046, 44844121, 99604946, 70004753, 20867149, 83789391, 31733363, 48360198, 22108665, 75407347, 60581278, 21289531, 26275734, 42199455, 82052050, 36685023, 52293995, 85117092, 63506281, 54427233, 80014588, 19101477, 415901, 83948335, 45428665, 9175338, 9188443, 70372191, 69786756, 112651, 33249630, 43322743, 74724075, 83747892, 25636669, 77787724, 29834512, 85711894, 16424199, 54232247, 91938191, 1204161, 86798033, 63015256, 99021067, 72777973, 40356877, 18806856, 94711842, 55247455, 30218878, 79322415, 67513640, 45381876, 50702367, 40686254, 17081350, 99515901, 21397057, 10453030, 97057187, 67793644, 50806615, 30163921, 61897582, 669105, 11365791, 67124282, 29065964, 36753250, 18833224, 93566986, 61859581, 36139942, 92803766, 60176618, 72725103, 35512853, 26102057, 66137019, 33336148, 44473167, 57241521, 94090109, 29510992, 69579137, 38510840, 21001913, 27185644, 43357947, 84840549, 26998766, 48395186, 88653118, 66667729, 60106217, 36933038, 88444207, 26397786, 54014062, 63628376, 59405277, 27325428, 51590803, 77272628, 68128438, 59614746, 64098930, 15015906, 73222868, 80246713, 1569515, 15064655, 55753905, 59197747, 64602895, 78589145, 67031644, 62552524, 30569392, 13819358, 65275241, 33123618, 77301523, 51149804, 79880247, 62740044, 72614359, 37560164, 6793819, 89214616, 22766820, 29029316, 7646095, 63152504, 18783702, 71333116, 74357852, 14045383, 81774825, 61859143, 4119747, 71920426, 82532312, 50668599, 29401781, 72373496, 51466049, 12024238, 47298834, 37435892, 64055960, 4668450, 6819644, 77284619, 36930650, 68939068, 8128637, 46870723, 71657078, 32151165, 6871053, 34432810, 65851721, 44550764, 92353856, 69697787, 9398733, 526217, 45075471, 26664538, 59109400, 76825057, 90158785, 23432750, 4806458, 22435353, 90272749, 80251430, 47361209, 81855445, 98462867, 33304202, 93359396, 40865610, 75153252, 91141395, 34698428, 9623492, 60430369, 78549759, 66903004, 9860968, 68816413, 59981773, 55470718, 61271144, 91990218, 57393458, 47893286, 17976208, 11161731, 53802686, 34667286, 17365188, 14626618, 84166196, 38008118, 69848388, 21673260, 15902805, 30787683, 5482538, 88130087, 61928316, 45990383, 63059024, 7423788, 98739783, 31161687, 62051033, 1022677, 82979980, 66271566, 42307449, 84293052, 84002370, 44847298, 62936963, 78909561, 96906410, 12303248, 66885828, 44889423, 79136082, 6808825, 7066775, 56473732, 22113133, 71316369, 38022834, 91664334, 16054533, 32058615, 82651278, 52060076, 20122224, 72732205, 92033260, 14363867, 50188404, 95726235, 20002147, 3294781, 62496012, 5822202, 66663942, 77072625, 9603598, 41092102, 24168411, 37166608, 39986008, 37192445, 53632373, 96709982, 17727650, 82779622, 40534591, 247198, 97641116, 91255408, 68694897, 56531125, 36812683, 83368048, 99917599, 82327024, 93057697, 23848565, 17764950, 41092172, 97940276, 4434662, 3509435, 22450468, 28928175, 75128745, 11923835, 45237957, 3088684, 36312813, 54199160, 33895336, 4515343, 96420429, 79922758, 6038457, 62115552, 81805959, 75052463, 20885148, 72495719, 32159704, 24591705, 79738755, 93790285, 75135448, 65880522, 76434144, 64157906, 77377183, 52613508, 92867155, 29932657, 37675718, 37183543, 76703609, 46851987, 30463802, 61815223, 1239555, 92692283, 29188588, 39847321, 57163802, 63967300, 77694700, 18663507, 73109997, 50007421, 17146629, 58751351, 18131876, 37659250, 69355476, 27665211, 24314885, 65047700, 67030811, 73021291, 96193415, 62762857, 50567636, 45996863, 26292919, 77413012, 57803235, 52261574, 22129328, 4064751, 44060493, 96318094, 40152546, 2917920, 56515456, 82339363, 10366309, 69136837, 8115266, 99549373, 8373289, 8634541, 13231279, 35092039, 61623915, 62357986, 7104732, 58224549, 93270084, 96735716, 30694952, 42947632, 36189527, 15948937, 62430984, 1197320, 16595436, 65715134, 3633375, 65338021, 13348726, 8129978, 5640302, 92554120, 17016156, 72793444, 16019925, 13468268, 73617245, 72738685, 27187213, 37620363, 23740167, 56461322, 47090124, 43376279, 41481685, 7687278, 4069912, 56484900, 14731700, 61982238, 34497327, 77187825, 1250437, 94911072, 61141874, 91240048, 83210802, 83533741, 49598724, 96852321, 65271999, 53842979, 81274566, 70782102, 36580610, 94539824, 19898053, 73392814, 69605283, 38658347, 86460488, 61373987, 61263068, 874791, 38365584, 15535065, 68316156, 56153345, 33565483, 91727510, 78785507, 3487592, 95010552, 1788101, 32250045, 98130363, 21993752, 92071990, 569864, 7300047, 26063929, 76960303, 83302115, 86022504, 4787945, 66611759, 22997281, 53393358, 62232211, 68875490, 39171895, 71965942, 19376156, 53084256 +42199455, 59981773, 26063929, 53084256, 17146629, 55143724, 56515456, 5267545, 3235882, 31161687, 37675718, 30543215, 99965001, 54232247, 38256849, 45381876, 4985896, 89637706, 526217, 24733232, 26664538, 84349107, 33336148, 3509435, 3233084, 21001913, 6221471, 60430369, 48675329, 81853704, 78884452, 38061439, 75407347, 66116458, 61741594, 29188588, 112651, 7011964, 16424199, 90654836, 37435892, 50567636, 59109400, 8115266, 72725103, 22435353, 97561745, 17894977, 3088684, 75229462, 36580610, 13862149, 10358899, 15163258, 62232211, 76170907, 75135448, 7423788, 60581278, 21289531, 42967683, 76671482, 73617245, 55615885, 35192533, 41380093, 92692283, 99690194, 12303248, 4119747, 92692978, 65047700, 39201414, 12024238, 24953498, 14731700, 824872, 70420215, 83269727, 84187166, 57803235, 96709982, 44060493, 59318837, 321665, 43501211, 40677414, 38645117, 4095116, 26102057, 83533741, 27411561, 2891150, 98462867, 76315420, 28550822, 63372756, 62357986, 93270084, 26776922, 51472275, 44915235, 59405277, 65454636, 97011160, 32159704, 15015906, 3633375, 80246713, 30395570, 1022677, 44847298, 36845587, 32590267, 34295794, 39847321, 48658605, 51047803, 33249630, 77694700, 7646095, 37501808, 77787724, 32058615, 50188404, 23194618, 85571389, 51466049, 6819644, 73021291, 86022504, 33565483, 57240218, 22129328, 10597197, 669105, 69697787, 44627776, 45075471, 45407418, 94516935, 66137019, 68041839, 69579137, 17539625, 21533347, 44348328, 33304202, 11923835, 95581843, 33895336, 66045587, 9860968, 61271144, 63628376, 6497830, 90061527, 98130363, 16595436, 65715134, 73222868, 82178706, 22879907, 86460488, 15064655, 12571310, 19486173, 5482538, 67031644, 30569392, 86301513, 8129978, 10649306, 92554120, 6111563, 17058722, 82052050, 85117092, 84904436, 30653863, 62740044, 72614359, 30463802, 15535065, 32426752, 92998591, 29029316, 73109997, 4508700, 99297164, 33237508, 74357852, 81774825, 82651278, 52060076, 4091162, 20122224, 86798033, 63015256, 14363867, 4069912, 44479073, 29994197, 66319530, 68939068, 56955985, 37192445, 12664567, 26292919, 6986898, 17727650, 48774913, 54058788, 168541, 54987042, 44983451, 44550764, 62452034, 9886593, 44846932, 36753250, 32274392, 79191827, 13470059, 92787493, 48673079, 33797252, 26493119, 30139692, 29516592, 8634541, 80251430, 13173644, 78218845, 57961282, 69641513, 84684495, 78785507, 88251446, 99658235, 61623915, 48395186, 2208785, 7229550, 78549759, 91831487, 36468541, 30366150, 54014062, 4199704, 15075176, 6157724, 55770687, 81898046, 15902805, 20867149, 83789391, 31733363, 55753905, 59197747, 64157906, 13348726, 52613508, 68875490, 13819358, 38341669, 33123618, 569864, 7300047, 39171895, 36685023, 42307449, 13468268, 46851987, 26507214, 19101477, 415901, 83948335, 38365584, 77620120, 92541302, 57163802, 9175338, 85116755, 70372191, 63967300, 63152504, 7066775, 4099191, 56461322, 34044787, 29834512, 85711894, 14045383, 36135, 30811010, 37659250, 69355476, 1204161, 19272365, 73786944, 18806856, 74441448, 1431742, 4668450, 62496012, 99226875, 74452589, 87710366, 62428472, 39986008, 45848907, 36396314, 84127901, 83155442, 1250437, 14349098, 46540998, 45995325, 94076128, 34432810, 99971982, 50806615, 30163921, 72278539, 6521313, 90090964, 74743862, 9398733, 45306070, 29065964, 15148031, 93057697, 92803766, 76825057, 23432750, 51588015, 99549373, 41092172, 20642888, 76540481, 97940276, 91727510, 45790169, 45617087, 57241521, 19891772, 29510992, 13231279, 75820087, 27185644, 90457870, 22450468, 93359396, 40865610, 65271999, 66611759, 54762643, 8055981, 66667729, 42782652, 82427263, 57248122, 91802888, 6038457, 3183975, 55470718, 21919959, 36933038, 42237907, 37280276, 22200378, 64848072, 9829782, 70036438, 36189527, 34667286, 27325428, 17365188, 22997281, 14626618, 20836893, 67227442, 22942635, 99604946, 79738755, 65880522, 61928316, 44784505, 92071990, 63059024, 20765474, 62051033, 77301523, 77312810, 89217461, 86543538, 16097038, 72793444, 58180653, 55189057, 40197395, 52293995, 63506281, 874791, 65081429, 78909561, 49597667, 23110625, 72738685, 45428665, 6793819, 9188443, 92604458, 69786756, 43322743, 12348118, 44889423, 70367851, 71469330, 36808486, 70541760, 67281495, 55960386, 47090124, 33699435, 25636669, 89811711, 5970607, 71333116, 29466406, 88047921, 59910624, 61859143, 57359924, 28851716, 72732205, 33935899, 49271185, 66428795, 50668599, 72373496, 78300864, 67474219, 66663942, 61982238, 96193415, 34497327, 36930650, 37166608, 36780454, 8128637, 15536795, 50702367, 82726008, 99224392, 4064751, 47738185, 21397057, 29819940, 5832946, 71657078, 34946859, 96318094, 40152546, 37891451, 73031054, 67793644, 53888755, 91255408, 8696647, 68694897, 62693428, 56531125, 72358170, 61141874, 77300457, 43506672, 99917599, 82339363, 61859581, 23848565, 19376156, 83210802, 90158785, 4787945, 35512853, 9058407, 80316608, 93053405, 49598724, 97281847, 92215320, 25842078, 83133790, 72274002, 8099994, 26998766, 49882705, 68102437, 75153252, 3487592, 7104732, 58224549, 73235980, 66250369, 68702632, 28796059, 3880712, 53842979, 96735716, 79922758, 23569917, 20568363, 66903004, 89419466, 47792865, 83150534, 95290172, 95010552, 70782102, 88444207, 1788101, 67084351, 91990218, 57393458, 49236559, 749283, 67451935, 81677380, 68128438, 34493392, 59614746, 34236719, 38008118, 54663246, 64098930, 14723410, 69605283, 21993752, 20486294, 21673260, 8807940, 68110330, 38009615, 12416768, 3773993, 24619760, 1569515, 176257, 70727211, 99861373, 24826575, 48360198, 64602895, 78589145, 22108665, 43045786, 65275241, 92867155, 73168565, 61728685, 17016156, 61263068, 82979980, 93933709, 43933006, 60102965, 48088883, 4204661, 98653983, 51149804, 66322959, 54427233, 84002370, 36505482, 37739481, 73124510, 37560164, 75986488, 18411915, 98371444, 71300104, 54868730, 42692881, 74724075, 44245960, 50007421, 8791066, 22113133, 71316369, 68644627, 38022834, 18131876, 31126490, 41481685, 68316156, 90004325, 24915585, 40781449, 71920426, 74110882, 7687278, 92033260, 16684829, 56153345, 29401781, 83302115, 47298834, 55247455, 20645197, 83083131, 20002147, 30218878, 3294781, 56424103, 77072625, 9603598, 11757872, 79322415, 41092102, 54606384, 24168411, 33201905, 11581181, 99333375, 45996863, 53632373, 17386996, 40686254, 17385531, 52261574, 30771409, 10453030, 32151165, 6871053, 247198, 16380211, 94595668, 65851721, 66832478, 86821229, 2717150, 65017137, 16387575, 18833224, 71083565, 82327024, 10366309, 36139942, 69136837, 16405341, 70800879, 35996293, 59371804, 45667668, 90272749, 8373289, 33431960, 19939935, 17957593, 90013093, 10961421, 88653118, 45237957, 44664587, 36312813, 28787861, 54199160, 79657802, 508198, 96420429, 78602717, 81805959, 9259676, 38494874, 68816413, 42947632, 26397786, 28734791, 4978543, 9860195, 11161731, 10309525, 15948937, 32250045, 72495719, 94539824, 1197320, 7919588, 24591705, 53547802, 65338021, 70596786, 85695762, 38658347, 35456853, 53648154, 62490109, 54263880, 44177011, 61373987, 70004753, 93790285, 45990383, 69255765, 5640302, 95395112, 82886381, 33553959, 37183543, 3233569, 26275734, 76703609, 16019925, 51507425, 4798568, 16099750, 59177718, 66885828, 47708850, 18663507, 2204165, 37620363, 6808825, 51426311, 56473732, 59329511, 16971929, 27041967, 52204879, 96726697, 78766358, 7182642, 49641577, 27625735, 82532312, 24314885, 65074535, 99021067, 68204242, 72238278, 40356877, 45919976, 10264691, 16567550, 60393039, 31491938, 99524975, 86242799, 60955663, 40781854, 5822202, 57658654, 97783876, 55669657, 31727379, 7502255, 97057187, 97641116, 11365791, 94911072, 99125126, 36812683, 45049308, 95251277, 74614639, 92530431, 68824981, 32161669, 9599614, 91240048, 83651211, 85242963, 14947650, 26139110, 18001617, 47361209, 17068582, 43357947, 16445503, 51715482, 84840549, 28928175, 58749, 34698428, 9623492, 30998561, 4515343, 55602660, 74137926, 8651647, 81274566, 14093520, 26392416, 47893286, 93515664, 20885148, 62430984, 30764367, 20681921, 24058273, 6153406, 45794415, 19898053, 30891921, 33061250, 89046466, 87598888, 98648327, 68000591, 78561158, 98739783, 62803858, 75777973, 29932657, 89804152, 66271566, 80014588, 62936963, 61815223, 7955293, 1239555, 91957544, 29635537, 5073754, 51464002, 23740167, 86118021, 95957797, 2607799, 43376279, 91664334, 16054533, 18699206, 43152977, 56484900, 62762857, 89699445, 17037369, 77413012, 89078848, 99515901, 46870723, 82779622, 40534591, 61897582, 22721500, 14220886, 82897371, 2917920, 88904910, 19457589, 91281584, 65038678, 93566986, 44842615, 90310261, 60176618, 4806458, 17764950, 94090109, 81855445, 88698958, 96852321, 71965942, 35092039, 62115552, 26114953, 33628349, 51315460, 1936762, 85023028, 60106217, 84406788, 18466635, 77272628, 6525948, 69848388, 17857111, 32699744, 89499542, 40984766, 64087743, 40027975, 37224844, 88130087, 76434144, 62552524, 9575954, 29959549, 11543098, 79880247, 80555751, 7685448, 89214616, 81172706, 71522968, 83747892, 18504197, 18783702, 58751351, 39373729, 97379790, 7517032, 77898274, 91938191, 76960303, 87720882, 9257405, 95726235, 1375023, 67030811, 77284619, 59501487, 77187825, 26744917, 17081350, 79821897, 54517921, 39553046, 83368048, 44481640, 22405120, 79942022, 4434662, 58208470, 26863229, 86001008, 38510840, 87160386, 72019362, 37957788, 53802686, 26734892, 73392814, 21070110, 30787683, 77377183, 47213183, 55850790, 47887470, 94360702, 34698463, 22766820, 27187213, 93562257, 41442762, 27665211, 72777973, 94711842, 98948034, 72357096, 20140249, 67513640, 48893685, 67124282, 5111370, 31904591, 75128745, 49328608, 14326617, 17976208, 92398073, 51590803, 84166196, 53393358, 39801351, 42644903, 8913721, 99729357, 84293052, 48892201, 96906410, 42073124, 6505939, 79136082, 76330843, 92353856, 44473167, 91141395, 75052463, 98943869, 89996536, 61712234, 44844121, 8729146, 23134715, 53666583, 41245325, 79806380, 64055960, 61960400, 30694952, 30090481, 48260151, 2331773, 57020481, 75543508, 8733068 +38658347, 45428665, 58751351, 1431742, 4064751, 99971982, 90090964, 88698958, 66667729, 79922758, 57393458, 73222868, 30395570, 92867155, 55850790, 44479073, 36780454, 12664567, 38645117, 20642888, 75543508, 45617087, 53084256, 28796059, 97011160, 45794415, 78884452, 21993752, 40984766, 75135448, 63059024, 42644903, 60581278, 13468268, 29188588, 71300104, 33249630, 74724075, 18663507, 37501808, 99965001, 33699435, 77787724, 54232247, 27665211, 9257405, 51466049, 83083131, 73021291, 824872, 37166608, 6986898, 40534591, 54987042, 44550764, 2917920, 39553046, 31904591, 32161669, 59371804, 57961282, 88251446, 28550822, 3880712, 78602717, 55470718, 36933038, 64848072, 77272628, 34236719, 12416768, 86460488, 79738755, 45990383, 569864, 7300047, 26275734, 42199455, 78909561, 30463802, 92541302, 51047803, 37620363, 5970607, 27041967, 96726697, 88047921, 32058615, 24314885, 86798033, 14363867, 23194618, 72238278, 10264691, 16567550, 37435892, 20645197, 14731700, 20002147, 61982238, 83269727, 33565483, 30771409, 54058788, 96318094, 14220886, 74743862, 36812683, 16387575, 71083565, 61859581, 72725103, 17764950, 91727510, 80316608, 49598724, 90272749, 33431960, 13173644, 84684495, 3233084, 72274002, 76315420, 84840549, 28928175, 40865610, 75153252, 45237957, 44664587, 93270084, 57248122, 91802888, 26114953, 23569917, 1936762, 91831487, 36580610, 22200378, 6157724, 18466635, 22997281, 14626618, 15163258, 32159704, 30764367, 59614746, 53393358, 21673260, 64087743, 99604946, 99861373, 19486173, 78589145, 62552524, 8129978, 21289531, 72793444, 36685023, 79880247, 62936963, 35192533, 92604458, 96906410, 66885828, 12348118, 7646095, 86118021, 47090124, 25636669, 71316369, 38022834, 95957797, 74357852, 14045383, 49641577, 24915585, 37659250, 1204161, 56153345, 50668599, 73786944, 96193415, 77072625, 31727379, 84127901, 21397057, 94595668, 97057187, 30163921, 8696647, 61141874, 65038678, 82327024, 93057697, 83210802, 51588015, 99549373, 83533741, 4434662, 33336148, 57241521, 94090109, 44348328, 98462867, 71965942, 38510840, 21001913, 17068582, 22450468, 26998766, 3487592, 65271999, 73235980, 96420429, 60430369, 66903004, 42947632, 54014062, 84406788, 10358899, 47893286, 36189527, 27325428, 34493392, 15015906, 3633375, 69605283, 85695762, 20486294, 22108665, 44784505, 78561158, 82979980, 37183543, 16097038, 39171895, 40197395, 55615885, 30653863, 65081429, 2331773, 84002370, 26507214, 32590267, 15535065, 6793819, 85116755, 16099750, 42073124, 112651, 63967300, 22766820, 42692881, 27187213, 67281495, 56461322, 43376279, 7182642, 16054533, 16424199, 41481685, 52060076, 90654836, 74110882, 92033260, 4069912, 87720882, 40356877, 18806856, 99524975, 74441448, 6819644, 11757872, 74452589, 24168411, 39986008, 45381876, 57803235, 50702367, 52261574, 82726008, 7502255, 37891451, 61897582, 91255408, 22721500, 6521313, 69697787, 68694897, 9398733, 72358170, 88904910, 74614639, 99917599, 15148031, 26664538, 8115266, 23432750, 48673079, 76540481, 26102057, 2891150, 33797252, 68041839, 8373289, 80251430, 78218845, 47361209, 69641513, 96852321, 33304202, 90013093, 72019362, 61623915, 63372756, 58224549, 9623492, 68702632, 96735716, 82427263, 7229550, 30694952, 8651647, 9860968, 95290172, 61271144, 63628376, 93515664, 17976208, 9860195, 10309525, 51590803, 17365188, 81677380, 69848388, 53547802, 24058273, 65338021, 81898046, 44844121, 15902805, 8807940, 68110330, 44177011, 1569515, 70004753, 176257, 12571310, 55753905, 64602895, 5482538, 13348726, 77377183, 43045786, 68875490, 13819358, 92554120, 31161687, 47887470, 3233569, 30543215, 11543098, 85117092, 42307449, 26063929, 84904436, 46851987, 80555751, 83948335, 39847321, 57163802, 9188443, 59177718, 77694700, 99297164, 39373729, 85711894, 36135, 40781449, 18699206, 33935899, 66428795, 68204242, 60393039, 43152977, 40781854, 30218878, 20140249, 5822202, 97783876, 41092102, 62428472, 99333375, 26744917, 8128637, 77413012, 53632373, 96709982, 46870723, 34946859, 45995325, 247198, 94076128, 73031054, 66832478, 72278539, 82897371, 56515456, 65017137, 36753250, 54517921, 45075471, 36139942, 76825057, 90158785, 83651211, 35512853, 45407418, 16405341, 85242963, 27411561, 14947650, 22435353, 18001617, 69579137, 21533347, 58208470, 13231279, 19939935, 27185644, 8099994, 16445503, 90457870, 93359396, 49882705, 34698428, 6221471, 3088684, 79657802, 53842979, 26776922, 66045587, 49328608, 62115552, 20568363, 81274566, 60106217, 59981773, 75229462, 14326617, 83150534, 1788101, 42237907, 91990218, 49236559, 51472275, 70036438, 4199704, 67451935, 11161731, 32250045, 62430984, 14723410, 26734892, 7919588, 6153406, 53648154, 30787683, 20867149, 87598888, 31733363, 40027975, 15064655, 68000591, 75407347, 61728685, 29932657, 82886381, 33123618, 29959549, 61263068, 77312810, 89217461, 42967683, 86543538, 89804152, 82052050, 76671482, 16019925, 66322959, 73617245, 36845587, 61815223, 61741594, 415901, 1239555, 37560164, 92692283, 77620120, 23110625, 9175338, 7685448, 69786756, 92998591, 12303248, 43322743, 5073754, 81172706, 71469330, 93562257, 63152504, 6808825, 18504197, 41245325, 7066775, 50007421, 4508700, 29834512, 18131876, 31126490, 7517032, 82651278, 61859143, 90004325, 20122224, 30811010, 72732205, 92692978, 49271185, 65047700, 72373496, 12024238, 86242799, 67474219, 77284619, 72357096, 34497327, 70420215, 9603598, 54606384, 86022504, 68939068, 67513640, 56955985, 37192445, 45848907, 40686254, 17081350, 47738185, 48774913, 82779622, 71657078, 32151165, 40152546, 16380211, 34432810, 67793644, 50806615, 53888755, 11365791, 92353856, 48893685, 62693428, 94911072, 19457589, 526217, 43506672, 83368048, 82339363, 10366309, 19376156, 91240048, 69136837, 4806458, 41092172, 22405120, 94516935, 97940276, 26139110, 17539625, 92215320, 3509435, 83133790, 17894977, 78785507, 75128745, 11923835, 54762643, 88653118, 2208785, 4515343, 6038457, 74137926, 81805959, 78549759, 75052463, 38494874, 85023028, 70782102, 26397786, 26392416, 6497830, 15075176, 53802686, 54663246, 6525948, 98130363, 16595436, 22879907, 80246713, 83789391, 93790285, 8729146, 88130087, 76434144, 67031644, 52613508, 92071990, 30569392, 66116458, 7423788, 5640302, 10649306, 65275241, 37675718, 23134715, 48088883, 4204661, 17058722, 55189057, 66271566, 76703609, 99729357, 48260151, 874791, 80014588, 62740044, 36505482, 19101477, 38365584, 41380093, 32426752, 29029316, 44245960, 70367851, 36808486, 70541760, 51426311, 4099191, 56473732, 8791066, 41442762, 33237508, 89811711, 22113133, 71333116, 2607799, 91664334, 55143724, 97379790, 77898274, 91938191, 16684829, 50188404, 99021067, 83302115, 8733068, 47298834, 56484900, 55247455, 64055960, 78300864, 3294781, 62496012, 66663942, 56424103, 36930650, 62762857, 79322415, 33201905, 89699445, 45996863, 26292919, 36396314, 17385531, 99515901, 22129328, 14349098, 99224392, 17727650, 44060493, 46540998, 79821897, 6871053, 10597197, 65851721, 97641116, 44983451, 4985896, 45306070, 321665, 89637706, 62452034, 44846932, 44627776, 91281584, 18833224, 93566986, 44481640, 5267545, 92803766, 59109400, 60176618, 92787493, 9058407, 4095116, 93053405, 26493119, 97281847, 30139692, 29516592, 75820087, 26863229, 17957593, 35092039, 43357947, 87160386, 58749, 91141395, 62357986, 8055981, 66250369, 36312813, 95581843, 30998561, 508198, 9259676, 68816413, 89419466, 95010552, 28734791, 37957788, 65454636, 90061527, 94539824, 89996536, 64098930, 1197320, 17857111, 24591705, 20681921, 65715134, 35456853, 82178706, 38061439, 33061250, 3773993, 61373987, 65880522, 98648327, 61928316, 9575954, 3235882, 98739783, 17016156, 1022677, 93933709, 94360702, 98653983, 58180653, 8913721, 34698463, 54427233, 84293052, 37739481, 48892201, 49597667, 72738685, 98371444, 44889423, 71522968, 2204165, 73109997, 79136082, 55960386, 17146629, 68644627, 16971929, 4091162, 4119747, 71920426, 79806380, 63015256, 76960303, 29401781, 85571389, 29994197, 45919976, 95726235, 31491938, 1375023, 59501487, 57658654, 38256849, 87710366, 11581181, 50567636, 15536795, 89078848, 57240218, 1250437, 5832946, 59318837, 669105, 67124282, 95251277, 32274392, 79191827, 92530431, 24733232, 9599614, 84349107, 90310261, 79942022, 66137019, 45790169, 44473167, 19891772, 86001008, 25842078, 68102437, 7104732, 28787861, 54199160, 33895336, 42782652, 55602660, 3183975, 98943869, 47792865, 88444207, 13862149, 37280276, 92398073, 68128438, 38008118, 20836893, 61712234, 81853704, 70596786, 55770687, 62232211, 62490109, 24619760, 54263880, 70727211, 37224844, 24826575, 64157906, 69255765, 86301513, 95395112, 20765474, 38341669, 62803858, 73168565, 33553959, 43933006, 60102965, 51149804, 52293995, 51507425, 7955293, 75986488, 89214616, 47708850, 7011964, 29466406, 78766358, 27625735, 69355476, 82532312, 19272365, 94711842, 67030811, 17037369, 84187166, 29819940, 86821229, 2717150, 5111370, 56531125, 29065964, 57020481, 45049308, 43501211, 61960400, 44842615, 4787945, 70800879, 29510992, 10961421, 66611759, 14093520, 67084351, 9829782, 749283, 48675329, 15948937, 84166196, 67227442, 89499542, 22942635, 30891921, 38009615, 59197747, 30090481, 62051033, 75777973, 72614359, 44847298, 99690194, 23740167, 18783702, 34044787, 52204879, 59910624, 68316156, 57359924, 72777973, 24953498, 60955663, 4668450, 77187825, 76330843, 99125126, 9886593, 77300457, 40677414, 13470059, 35996293, 97561745, 8634541, 81855445, 36468541, 44915235, 4978543, 20885148, 72495719, 32699744, 73392814, 21070110, 89046466, 76170907, 47213183, 77301523, 6111563, 73124510, 34295794, 48658605, 70372191, 6505939, 51464002, 83747892, 59329511, 7687278, 65074535, 39201414, 66319530, 99226875, 55669657, 17386996, 10453030, 68824981, 21919959, 59405277, 34667286, 19898053, 53666583, 63506281, 4798568, 91957544, 29635537, 81774825, 28851716, 98948034, 168541, 23848565, 45667668, 51715482, 99658235, 48395186, 51315460, 30366150, 48360198, 39801351, 18411915, 83155442, 54868730, 33628349 +68939068, 99515901, 67124282, 47361209, 75128745, 15015906, 1569515, 62552524, 66832478, 26139110, 63372756, 81274566, 83150534, 32159704, 59197747, 1204161, 4064751, 99917599, 44842615, 66137019, 40865610, 54199160, 55470718, 42644903, 54427233, 78909561, 30463802, 51047803, 27187213, 18504197, 9257405, 95726235, 55247455, 40781854, 34497327, 36930650, 36780454, 67793644, 95251277, 31904591, 10366309, 90013093, 17894977, 91802888, 74137926, 1197320, 78884452, 21993752, 12571310, 68000591, 92071990, 569864, 3233569, 42307449, 69786756, 37501808, 51426311, 33935899, 92033260, 76960303, 14731700, 45381876, 12664567, 77413012, 6986898, 50702367, 46870723, 34946859, 32151165, 14220886, 48893685, 94911072, 65017137, 9886593, 44627776, 65038678, 54517921, 5267545, 83651211, 16405341, 45617087, 13173644, 81855445, 76315420, 84840549, 28550822, 28796059, 1936762, 21919959, 57393458, 54014062, 54663246, 14723410, 69848388, 53393358, 19898053, 20486294, 15902805, 99604946, 24619760, 8729146, 78589145, 76434144, 66116458, 63059024, 55850790, 89217461, 86543538, 7300047, 48260151, 79880247, 65081429, 61815223, 83948335, 85116755, 92998591, 33249630, 42692881, 2204165, 37620363, 36808486, 83747892, 67281495, 47090124, 7011964, 43376279, 91664334, 74357852, 78766358, 24915585, 30811010, 27665211, 66428795, 44479073, 56153345, 85571389, 73786944, 16567550, 51466049, 6819644, 99226875, 9603598, 11757872, 62428472, 76330843, 79821897, 247198, 669105, 82897371, 2917920, 9398733, 99125126, 16387575, 93566986, 82327024, 60176618, 92787493, 22405120, 75543508, 49598724, 44473167, 8373289, 57241521, 69579137, 21533347, 53084256, 88653118, 45237957, 66667729, 66045587, 49328608, 81805959, 9259676, 68816413, 60106217, 42947632, 26397786, 42237907, 70036438, 77272628, 22997281, 6153406, 85695762, 44844121, 21673260, 86460488, 44177011, 176257, 87598888, 31733363, 99861373, 37224844, 55753905, 64157906, 21289531, 77312810, 93933709, 42967683, 39171895, 60102965, 53666583, 26275734, 51149804, 13468268, 55615885, 2331773, 44847298, 37560164, 45428665, 9175338, 71300104, 63967300, 22766820, 79136082, 51464002, 88047921, 16054533, 68316156, 57359924, 37659250, 54232247, 92692978, 86798033, 72238278, 18806856, 43152977, 24953498, 20002147, 3294781, 33565483, 50567636, 45848907, 26292919, 82726008, 44060493, 40534591, 40152546, 99971982, 69697787, 56531125, 88904910, 29065964, 57020481, 36753250, 74614639, 45075471, 83368048, 44481640, 15148031, 61859581, 8115266, 35512853, 45407418, 20642888, 14947650, 2891150, 93053405, 4434662, 30139692, 29516592, 94090109, 57961282, 84684495, 26863229, 35092039, 17068582, 26998766, 3487592, 58749, 6221471, 62357986, 44664587, 66250369, 36312813, 96735716, 57248122, 79922758, 75052463, 36933038, 88444207, 64848072, 48675329, 17976208, 10309525, 18466635, 81677380, 14626618, 34493392, 34236719, 89499542, 82178706, 8807940, 30395570, 61373987, 76170907, 64602895, 13348726, 45990383, 3235882, 78561158, 69255765, 61728685, 47887470, 37675718, 82979980, 98653983, 42199455, 82052050, 66271566, 85117092, 80014588, 30653863, 72614359, 62936963, 36505482, 19101477, 23110625, 92541302, 98371444, 92604458, 96906410, 42073124, 66885828, 74724075, 71469330, 41245325, 70541760, 55960386, 50007421, 4099191, 56461322, 22113133, 38022834, 29834512, 96726697, 7182642, 55143724, 32058615, 40781449, 4119747, 24314885, 14363867, 4069912, 99021067, 60393039, 47298834, 20645197, 77284619, 73021291, 72357096, 62496012, 96193415, 77072625, 74452589, 11581181, 99333375, 53632373, 17081350, 17385531, 1250437, 14349098, 99224392, 7502255, 97057187, 50806615, 44550764, 8696647, 45306070, 72358170, 44846932, 61141874, 45049308, 18833224, 23848565, 36139942, 19376156, 90310261, 99549373, 17764950, 41092172, 48673079, 68041839, 8634541, 78218845, 69641513, 83133790, 72274002, 22450468, 75153252, 91141395, 65271999, 66611759, 48395186, 58224549, 42782652, 96420429, 7229550, 6038457, 78602717, 66903004, 75229462, 1788101, 22200378, 47893286, 53802686, 15948937, 68128438, 15163258, 94539824, 26734892, 17857111, 20836893, 3633375, 53547802, 24058273, 73392814, 69605283, 21070110, 73222868, 22942635, 30891921, 80246713, 68110330, 38009615, 64087743, 33061250, 30787683, 79738755, 40027975, 75135448, 61928316, 44784505, 52613508, 75407347, 86301513, 98739783, 65275241, 38341669, 29932657, 60581278, 17016156, 61263068, 89804152, 40197395, 30543215, 26063929, 73617245, 26507214, 36845587, 32590267, 73124510, 75986488, 59177718, 12303248, 5073754, 44889423, 81172706, 7646095, 99965001, 33699435, 17146629, 41442762, 58751351, 71316369, 16971929, 85711894, 14045383, 59910624, 97379790, 49641577, 4091162, 20122224, 90654836, 28851716, 71920426, 74110882, 23194618, 29994197, 40356877, 8733068, 94711842, 12024238, 31491938, 1431742, 83083131, 86242799, 20140249, 824872, 5822202, 61982238, 56424103, 70420215, 62762857, 97783876, 38256849, 54606384, 37166608, 31727379, 67513640, 84127901, 57240218, 83155442, 52261574, 48774913, 21397057, 96318094, 94595668, 65851721, 30163921, 53888755, 90090964, 74743862, 11365791, 68694897, 43501211, 71083565, 39553046, 92530431, 68824981, 32161669, 9599614, 93057697, 83210802, 40677414, 76825057, 23432750, 76540481, 91727510, 33336148, 97281847, 90272749, 58208470, 88698958, 86001008, 78785507, 49882705, 28928175, 68702632, 95581843, 30998561, 82427263, 30694952, 8651647, 9860968, 91831487, 14093520, 95290172, 26392416, 65454636, 11161731, 97011160, 30764367, 84166196, 61712234, 16595436, 32699744, 70596786, 53648154, 62232211, 12416768, 89046466, 70004753, 20867149, 83789391, 70727211, 24826575, 48360198, 65880522, 30090481, 30569392, 68875490, 10649306, 62051033, 62803858, 29959549, 37183543, 23134715, 6111563, 4204661, 72793444, 17058722, 55189057, 76671482, 11543098, 52293995, 8913721, 16019925, 4798568, 80555751, 35192533, 38365584, 92692283, 72738685, 39847321, 48658605, 99690194, 112651, 71522968, 63152504, 6808825, 25636669, 4508700, 34044787, 89811711, 31126490, 7517032, 82651278, 90004325, 72732205, 91938191, 49271185, 19272365, 39201414, 99524975, 66319530, 57658654, 24168411, 39986008, 40686254, 22129328, 17727650, 47738185, 5832946, 30771409, 46540998, 59318837, 6871053, 45995325, 37891451, 73031054, 92353856, 56515456, 5111370, 321665, 89637706, 36812683, 19457589, 77300457, 82339363, 91240048, 38645117, 4787945, 70800879, 26102057, 85242963, 83533741, 94516935, 80316608, 22435353, 45667668, 18001617, 33431960, 92215320, 96852321, 19939935, 17957593, 71965942, 33304202, 43357947, 16445503, 90457870, 93359396, 87160386, 68102437, 61623915, 10961421, 3880712, 79657802, 26776922, 4515343, 60430369, 62115552, 3183975, 20568363, 85023028, 59981773, 36468541, 70782102, 49236559, 4199704, 6497830, 93515664, 37957788, 6157724, 27325428, 17365188, 62430984, 89996536, 59614746, 64098930, 81853704, 67227442, 65338021, 45794415, 38658347, 35456853, 38061439, 88130087, 67031644, 98648327, 77377183, 9575954, 95395112, 13819358, 31161687, 75777973, 33123618, 94360702, 58180653, 34698463, 76703609, 66322959, 874791, 84293052, 84002370, 46851987, 48892201, 415901, 49597667, 77620120, 18411915, 89214616, 6505939, 7066775, 18783702, 86118021, 99297164, 39373729, 68644627, 5970607, 77787724, 71333116, 29466406, 27041967, 18131876, 2607799, 81774825, 27625735, 82532312, 63015256, 16684829, 50188404, 29401781, 72373496, 83302115, 74441448, 78300864, 59501487, 77187825, 86022504, 33201905, 83269727, 17037369, 26744917, 15536795, 57803235, 17386996, 71657078, 44983451, 6521313, 61960400, 84349107, 4806458, 35996293, 97940276, 59371804, 29510992, 3233084, 21001913, 27185644, 99658235, 7104732, 73235980, 28787861, 33895336, 508198, 2208785, 55602660, 26114953, 38494874, 98943869, 36580610, 91990218, 30366150, 84406788, 9829782, 36189527, 4978543, 9860195, 59405277, 32250045, 90061527, 51590803, 6525948, 98130363, 24591705, 22879907, 81898046, 40984766, 54263880, 15064655, 47213183, 7423788, 43045786, 20765474, 92867155, 73168565, 82886381, 1022677, 16097038, 48088883, 84904436, 34295794, 15535065, 57163802, 7685448, 32426752, 77694700, 12348118, 18663507, 56473732, 59329511, 36135, 69355476, 18699206, 7687278, 72777973, 45919976, 10264691, 56484900, 37435892, 4668450, 67474219, 98948034, 55669657, 89699445, 84187166, 56955985, 36396314, 94076128, 61897582, 54987042, 72278539, 91255408, 4985896, 22721500, 62693428, 62452034, 526217, 32274392, 24733232, 69136837, 13470059, 9058407, 4095116, 27411561, 79942022, 33797252, 26493119, 8099994, 51715482, 72019362, 34698428, 93270084, 53842979, 23569917, 33628349, 51315460, 47792865, 95010552, 61271144, 67084351, 63628376, 28734791, 67451935, 92398073, 20885148, 72495719, 7919588, 65715134, 5640302, 39801351, 99729357, 1239555, 29188588, 6793819, 29635537, 9188443, 16099750, 47708850, 23740167, 33237508, 52204879, 16424199, 41481685, 65047700, 65074535, 87720882, 68204242, 67030811, 30218878, 79322415, 45996863, 8128637, 89078848, 96709982, 82779622, 29819940, 16380211, 10597197, 168541, 97641116, 86821229, 91281584, 43506672, 26664538, 92803766, 97561745, 80251430, 19891772, 98462867, 25842078, 38510840, 88251446, 11923835, 8055981, 9623492, 3088684, 78549759, 89419466, 13862149, 10358899, 37280276, 749283, 15075176, 34667286, 38008118, 20681921, 55770687, 93790285, 22108665, 8129978, 77301523, 43933006, 36685023, 63506281, 51507425, 62740044, 37739481, 7955293, 41380093, 70372191, 29029316, 44245960, 70367851, 8791066, 61859143, 79806380, 50668599, 64055960, 60955663, 37192445, 79191827, 59109400, 3509435, 54762643, 51472275, 44915235, 3773993, 19486173, 5482538, 61741594, 91957544, 54868730, 43322743, 93562257, 77898274, 52060076, 66663942, 87710366, 34432810, 2717150, 90158785, 72725103, 51588015, 17539625, 13231279, 44348328, 14326617, 62490109, 92554120, 73109997, 1375023, 41092102, 54058788, 75820087, 33553959, 10453030, 45790169, 95957797 +32151165, 67084351, 59910624, 5822202, 36930650, 35512853, 63372756, 53842979, 68816413, 26397786, 99965001, 85711894, 9599614, 85242963, 96852321, 75128745, 95581843, 72495719, 61712234, 17016156, 3233569, 51149804, 29635537, 81172706, 41442762, 28851716, 47298834, 56955985, 99515901, 50806615, 67124282, 65017137, 36753250, 54517921, 99917599, 92787493, 69579137, 59405277, 11161731, 64098930, 20681921, 78884452, 22879907, 88130087, 75407347, 98739783, 61728685, 47887470, 89804152, 48260151, 63506281, 13468268, 30463802, 96906410, 81774825, 71920426, 44479073, 67030811, 62496012, 33201905, 89078848, 17081350, 73031054, 669105, 92353856, 48893685, 91281584, 526217, 74614639, 92530431, 44842615, 20642888, 16405341, 33797252, 78218845, 35092039, 66045587, 55602660, 91831487, 30366150, 92398073, 18466635, 34493392, 99604946, 70004753, 24826575, 68000591, 75777973, 33123618, 82979980, 23134715, 6111563, 94360702, 60102965, 55615885, 61815223, 23110625, 9188443, 33249630, 5073754, 12348118, 6505939, 67281495, 86118021, 50007421, 88047921, 32058615, 20122224, 30811010, 74110882, 50188404, 85571389, 45919976, 99524975, 9603598, 54606384, 36780454, 83269727, 45381876, 50702367, 22129328, 6871053, 10597197, 9398733, 43501211, 32274392, 5267545, 90310261, 59109400, 99549373, 4806458, 68041839, 45667668, 13173644, 92215320, 83133790, 84840549, 26998766, 54199160, 30998561, 1936762, 21919959, 95010552, 749283, 37957788, 97011160, 89499542, 85695762, 21070110, 64602895, 13348726, 98648327, 3235882, 92071990, 38341669, 55850790, 39171895, 48088883, 26275734, 42199455, 40197395, 8913721, 66322959, 51507425, 2331773, 36505482, 1239555, 18411915, 51047803, 54868730, 44245960, 71522968, 2204165, 83747892, 51426311, 39373729, 33237508, 38022834, 78766358, 7182642, 97379790, 36135, 4091162, 72732205, 27665211, 18699206, 91938191, 92033260, 29401781, 66319530, 20140249, 77187825, 31727379, 45996863, 76330843, 14349098, 47738185, 7502255, 40534591, 79821897, 45995325, 40152546, 16380211, 6521313, 82897371, 45306070, 62693428, 88904910, 29065964, 19457589, 57020481, 77300457, 43506672, 93566986, 68824981, 15148031, 10366309, 23848565, 84349107, 83210802, 76825057, 23432750, 41092172, 48673079, 27411561, 14947650, 97940276, 26139110, 59371804, 80316608, 21533347, 75820087, 8099994, 51715482, 49882705, 65271999, 62357986, 36312813, 28787861, 49328608, 57248122, 3183975, 81274566, 89419466, 14093520, 91990218, 48675329, 17976208, 6157724, 22997281, 69848388, 15015906, 65715134, 70596786, 53648154, 62232211, 38009615, 64087743, 89046466, 176257, 83789391, 70727211, 99861373, 79738755, 93790285, 12571310, 55753905, 59197747, 78589145, 69255765, 30569392, 7423788, 68875490, 13819358, 42644903, 62051033, 86543538, 43933006, 76671482, 52293995, 99729357, 84293052, 84904436, 36845587, 62936963, 35192533, 415901, 83948335, 91957544, 73124510, 75986488, 48658605, 69786756, 59177718, 70367851, 37620363, 18783702, 56473732, 99297164, 16971929, 27041967, 18131876, 31126490, 14045383, 68316156, 4119747, 63015256, 76960303, 14363867, 4069912, 31491938, 24953498, 1431742, 40781854, 67474219, 59501487, 99226875, 61982238, 38256849, 33565483, 26292919, 53632373, 6986898, 83155442, 46870723, 48774913, 44060493, 30771409, 37891451, 247198, 99971982, 67793644, 168541, 30163921, 66832478, 86821229, 56531125, 321665, 89637706, 62452034, 99125126, 44846932, 61141874, 45049308, 95251277, 61960400, 39553046, 31904591, 82327024, 36139942, 40677414, 83651211, 72725103, 9058407, 66137019, 45790169, 93053405, 45617087, 30139692, 57241521, 80251430, 13231279, 88698958, 21001913, 27185644, 90457870, 22450468, 28550822, 54762643, 88653118, 73235980, 96735716, 42782652, 2208785, 62115552, 78549759, 23569917, 98943869, 14326617, 36468541, 42237907, 37280276, 22200378, 44915235, 93515664, 15075176, 65454636, 90061527, 81677380, 38008118, 14723410, 1197320, 17857111, 24591705, 53393358, 86460488, 30395570, 76170907, 48360198, 65880522, 77377183, 61928316, 78561158, 66116458, 95395112, 65275241, 29932657, 82886381, 89217461, 17058722, 30543215, 85117092, 79880247, 73617245, 84002370, 26507214, 44847298, 37560164, 34295794, 15535065, 85116755, 71300104, 89214616, 42073124, 92998591, 77694700, 18663507, 93562257, 73109997, 51464002, 18504197, 55960386, 56461322, 22113133, 71316369, 68644627, 71333116, 16054533, 49641577, 41481685, 90004325, 24915585, 27625735, 37659250, 82532312, 56153345, 23194618, 72777973, 40356877, 10264691, 95726235, 18806856, 94711842, 43152977, 20645197, 74441448, 64055960, 6819644, 70420215, 77072625, 97783876, 74452589, 89699445, 99333375, 26744917, 50567636, 67513640, 17727650, 29819940, 61897582, 53888755, 72278539, 14220886, 90090964, 69697787, 8696647, 68694897, 5111370, 83368048, 26664538, 19376156, 91240048, 90158785, 45407418, 17764950, 4095116, 26102057, 49598724, 97561745, 44473167, 8373289, 8634541, 19891772, 58208470, 86001008, 3233084, 90013093, 72274002, 76315420, 16445503, 68102437, 28928175, 53084256, 6221471, 58224549, 45237957, 44664587, 3880712, 96420429, 26114953, 30694952, 33628349, 75052463, 38494874, 85023028, 60106217, 59981773, 55470718, 75229462, 83150534, 95290172, 70782102, 88444207, 49236559, 47893286, 6497830, 10309525, 15163258, 89996536, 84166196, 20836893, 67227442, 3633375, 65338021, 19898053, 69605283, 15902805, 24619760, 54263880, 44177011, 31733363, 15064655, 64157906, 67031644, 30090481, 22108665, 9575954, 47213183, 5640302, 37183543, 42967683, 16097038, 98653983, 55189057, 66271566, 34698463, 54427233, 30653863, 65081429, 80555751, 37739481, 38365584, 92692283, 77620120, 92541302, 45428665, 9175338, 92604458, 63967300, 22766820, 43322743, 66885828, 74724075, 7646095, 41245325, 23740167, 47090124, 7011964, 8791066, 52204879, 2607799, 74357852, 61859143, 52060076, 49271185, 1204161, 65047700, 66428795, 65074535, 72238278, 29994197, 39201414, 12024238, 1375023, 55247455, 86242799, 60955663, 73021291, 3294781, 824872, 56424103, 11757872, 62762857, 79322415, 55669657, 17037369, 84187166, 39986008, 37192445, 12664567, 84127901, 57803235, 40686254, 99224392, 34946859, 59318837, 94076128, 34432810, 94595668, 22721500, 44550764, 74743862, 56515456, 72358170, 44627776, 16387575, 65038678, 71083565, 24733232, 32161669, 93057697, 69136837, 13470059, 51588015, 76540481, 70800879, 75543508, 22435353, 18001617, 90272749, 17539625, 44348328, 17957593, 33304202, 87160386, 61623915, 10961421, 7104732, 9623492, 3088684, 79657802, 33895336, 508198, 7229550, 74137926, 78602717, 81805959, 9259676, 8651647, 1788101, 57393458, 54014062, 84406788, 34667286, 32250045, 77272628, 14626618, 6525948, 98130363, 26734892, 32699744, 24058273, 6153406, 73392814, 21993752, 73222868, 22942635, 30891921, 21673260, 38061439, 40027975, 37224844, 8729146, 76434144, 45990383, 52613508, 86301513, 39801351, 20765474, 92867155, 60581278, 21289531, 29959549, 1022677, 569864, 33553959, 7300047, 4204661, 42307449, 80014588, 62740044, 29188588, 7685448, 99690194, 32426752, 42692881, 27187213, 29029316, 47708850, 79136082, 6808825, 7066775, 34044787, 5970607, 29466406, 29834512, 55143724, 7517032, 77898274, 82651278, 90654836, 54232247, 79806380, 33935899, 7687278, 19272365, 86798033, 16684829, 51466049, 60393039, 83083131, 14731700, 20002147, 72357096, 57658654, 24168411, 87710366, 37166608, 77413012, 57240218, 17386996, 17385531, 82726008, 21397057, 71657078, 46540998, 96318094, 97057187, 97641116, 91255408, 11365791, 94911072, 9886593, 36812683, 79191827, 92803766, 33336148, 26493119, 97281847, 29516592, 94090109, 3509435, 69641513, 26863229, 38510840, 17894977, 93359396, 66611759, 48395186, 66250369, 28796059, 26776922, 60430369, 91802888, 20568363, 66903004, 47792865, 26392416, 64848072, 9829782, 51472275, 4199704, 4978543, 53802686, 15948937, 62430984, 94539824, 30764367, 34236719, 54663246, 7919588, 20486294, 82178706, 44844121, 40984766, 12416768, 3773993, 61373987, 20867149, 62552524, 63059024, 43045786, 10649306, 31161687, 62803858, 73168565, 37675718, 77312810, 53666583, 72793444, 58180653, 82052050, 36685023, 76703609, 16019925, 874791, 46851987, 78909561, 49597667, 41380093, 39847321, 112651, 12303248, 71469330, 37501808, 36808486, 70541760, 33699435, 4508700, 59329511, 95957797, 43376279, 91664334, 16424199, 40781449, 92692978, 69355476, 99021067, 68204242, 8733068, 56484900, 98948034, 66663942, 41092102, 11581181, 68939068, 45848907, 15536795, 54058788, 44983451, 4985896, 44481640, 38645117, 94516935, 4434662, 25842078, 71965942, 17068582, 43357947, 72019362, 58749, 11923835, 91141395, 34698428, 66667729, 82427263, 4515343, 6038457, 51315460, 61271144, 36933038, 13862149, 10358899, 63628376, 9860195, 16595436, 53547802, 35456853, 81898046, 68110330, 87598888, 75135448, 44784505, 7955293, 19101477, 61741594, 44889423, 17146629, 96726697, 57359924, 9257405, 72373496, 73786944, 16567550, 83302115, 78300864, 4668450, 96193415, 34497327, 86022504, 36396314, 5832946, 65851721, 54987042, 2917920, 18833224, 45075471, 61859581, 60176618, 4787945, 83533741, 79942022, 35996293, 91727510, 2891150, 81855445, 84684495, 98462867, 78785507, 88251446, 99658235, 40865610, 8055981, 68702632, 79922758, 42947632, 70036438, 51590803, 68128438, 59614746, 45794415, 80246713, 33061250, 30787683, 1569515, 19486173, 5482538, 8129978, 92554120, 61263068, 93933709, 11543098, 26063929, 72614359, 32590267, 72738685, 16099750, 98371444, 4099191, 58751351, 87720882, 50668599, 37435892, 62428472, 8128637, 1250437, 52261574, 10453030, 2717150, 82339363, 22405120, 19939935, 75153252, 3487592, 36189527, 28734791, 67451935, 20885148, 27325428, 17365188, 32159704, 62490109, 8807940, 77301523, 4798568, 48892201, 57163802, 70372191, 63152504, 25636669, 89811711, 24314885, 8115266, 33431960, 29510992, 47361209, 57961282, 93270084, 9860968, 36580610, 6793819, 77787724, 77284619, 30218878, 96709982, 82779622, 55770687, 4064751, 38658347, 81853704 +94711842, 3235882, 7687278, 75229462, 83789391, 48360198, 37659250, 24953498, 74441448, 34432810, 69697787, 60581278, 29466406, 6871053, 168541, 44627776, 9599614, 48673079, 69579137, 99658235, 32159704, 21070110, 17058722, 73617245, 72614359, 17146629, 85711894, 66428795, 29994197, 45848907, 84127901, 46870723, 16387575, 95251277, 68824981, 45407418, 16405341, 76540481, 83533741, 33336148, 53084256, 44664587, 85023028, 42947632, 14626618, 15163258, 24591705, 15015906, 21993752, 38061439, 67031644, 68000591, 30569392, 55850790, 29932657, 77301523, 39171895, 66271566, 76703609, 874791, 84904436, 72738685, 39847321, 12303248, 66885828, 96726697, 36135, 57359924, 33935899, 91938191, 39201414, 95726235, 18806856, 86022504, 82726008, 4064751, 94595668, 97057187, 91255408, 43501211, 82339363, 92803766, 79942022, 26493119, 98462867, 21001913, 27185644, 76315420, 40865610, 75153252, 3487592, 93270084, 81274566, 21919959, 70782102, 54014062, 10358899, 63628376, 20885148, 72495719, 81853704, 24058273, 38658347, 53648154, 76434144, 8129978, 62051033, 1022677, 26063929, 30653863, 15535065, 70372191, 63967300, 92998591, 5073754, 71469330, 73109997, 4099191, 56461322, 77787724, 16971929, 68316156, 90654836, 4119747, 56153345, 50668599, 31491938, 99524975, 86242799, 20002147, 6819644, 67030811, 66663942, 34497327, 9603598, 37166608, 11581181, 33565483, 26292919, 36396314, 53632373, 48774913, 40534591, 32151165, 65851721, 67793644, 86821229, 11365791, 321665, 61141874, 36753250, 45075471, 10366309, 19376156, 40677414, 13470059, 4787945, 35512853, 92787493, 41092172, 97940276, 22435353, 45667668, 8634541, 17539625, 93359396, 28550822, 66611759, 88653118, 66045587, 49328608, 82427263, 81805959, 33628349, 59981773, 88444207, 36580610, 70036438, 44915235, 59405277, 77272628, 54663246, 20836893, 55770687, 80246713, 3773993, 24619760, 76170907, 59197747, 64602895, 61928316, 33123618, 61263068, 569864, 33553959, 3233569, 76671482, 51149804, 75986488, 9175338, 85116755, 71300104, 99690194, 27187213, 36808486, 86118021, 25636669, 71316369, 2607799, 78766358, 81774825, 4091162, 54232247, 65074535, 72777973, 45919976, 67474219, 70420215, 62762857, 38256849, 31727379, 50567636, 82779622, 97641116, 53888755, 44983451, 48893685, 2917920, 99125126, 71083565, 15148031, 69136837, 60176618, 23432750, 22405120, 70800879, 94516935, 91727510, 80316608, 29510992, 83133790, 8099994, 78785507, 58749, 34698428, 62357986, 7104732, 58224549, 36312813, 68702632, 95581843, 60430369, 51315460, 9259676, 75052463, 14093520, 47792865, 83150534, 4199704, 6497830, 15075176, 92398073, 15948937, 27325428, 90061527, 17365188, 94539824, 26734892, 17857111, 61712234, 3633375, 70596786, 19898053, 40984766, 33061250, 44177011, 89046466, 61373987, 70727211, 37224844, 15064655, 8729146, 55753905, 78561158, 68875490, 92554120, 29959549, 37675718, 86543538, 6111563, 36685023, 84293052, 46851987, 4798568, 44847298, 61815223, 1239555, 38365584, 32590267, 37560164, 29635537, 48658605, 69786756, 44889423, 2204165, 7646095, 70541760, 51426311, 5970607, 7182642, 90004325, 72732205, 92692978, 82532312, 65047700, 19272365, 76960303, 50188404, 99021067, 83302115, 98948034, 61982238, 96193415, 33201905, 89699445, 8128637, 6986898, 96709982, 99515901, 52261574, 99224392, 44060493, 50806615, 82897371, 67124282, 56515456, 5111370, 94911072, 36812683, 19457589, 32274392, 82327024, 44842615, 93057697, 23848565, 5267545, 83210802, 72725103, 66137019, 26139110, 75543508, 59371804, 93053405, 4434662, 18001617, 97561745, 8373289, 80251430, 13173644, 78218845, 47361209, 81855445, 21533347, 58208470, 88698958, 17957593, 90457870, 87160386, 72019362, 61623915, 3088684, 54199160, 33895336, 508198, 79922758, 91802888, 91831487, 60106217, 9829782, 51472275, 6157724, 18466635, 32250045, 34493392, 34236719, 98130363, 20681921, 6153406, 89499542, 20486294, 73222868, 86460488, 30787683, 176257, 99861373, 93790285, 65880522, 22108665, 45990383, 9575954, 52613508, 69255765, 39801351, 20765474, 65275241, 38341669, 62803858, 21289531, 47887470, 17016156, 77312810, 89217461, 42967683, 43933006, 4204661, 89804152, 82052050, 52293995, 34698463, 16019925, 54427233, 80014588, 65081429, 30463802, 61741594, 415901, 73124510, 92692283, 77620120, 34295794, 57163802, 9188443, 98371444, 51047803, 89214616, 32426752, 112651, 33249630, 44245960, 37620363, 63152504, 50007421, 47090124, 68644627, 95957797, 74357852, 88047921, 16054533, 41481685, 61859143, 52060076, 30811010, 27625735, 69355476, 71920426, 49271185, 86798033, 44479073, 16684829, 85571389, 47298834, 55247455, 20645197, 1431742, 78300864, 30218878, 56424103, 11757872, 79322415, 55669657, 24168411, 37192445, 57803235, 17081350, 17385531, 1250437, 76330843, 29819940, 5832946, 7502255, 34946859, 10453030, 96318094, 247198, 4985896, 22721500, 74743862, 92353856, 9398733, 89637706, 62452034, 88904910, 91281584, 45049308, 77300457, 18833224, 39553046, 83368048, 79191827, 93566986, 61859581, 84349107, 90310261, 20642888, 9058407, 2891150, 30139692, 92215320, 75820087, 44348328, 17894977, 43357947, 9623492, 66250369, 28787861, 28796059, 96735716, 42782652, 7229550, 95010552, 36468541, 36933038, 67084351, 22200378, 36189527, 28734791, 53802686, 51590803, 97011160, 68128438, 65715134, 32699744, 53393358, 78884452, 69605283, 54263880, 1569515, 70004753, 20867149, 19486173, 5482538, 64157906, 75407347, 63059024, 43045786, 5640302, 10649306, 98739783, 31161687, 75777973, 73168565, 93933709, 94360702, 48088883, 53666583, 8913721, 42307449, 63506281, 13468268, 51507425, 62740044, 36845587, 62936963, 19101477, 48892201, 49597667, 23110625, 92604458, 59177718, 42073124, 43322743, 12348118, 29029316, 70367851, 79136082, 18504197, 23740167, 7066775, 55960386, 18783702, 33699435, 58751351, 33237508, 89811711, 29834512, 43376279, 16424199, 7517032, 82651278, 28851716, 74110882, 1204161, 92033260, 4069912, 23194618, 29401781, 9257405, 72373496, 12024238, 1375023, 43152977, 56484900, 4668450, 66319530, 824872, 62496012, 99226875, 74452589, 54606384, 83269727, 99333375, 17037369, 45996863, 15536795, 77413012, 89078848, 83155442, 21397057, 71657078, 79821897, 59318837, 40152546, 37891451, 73031054, 30163921, 72278539, 45306070, 44846932, 29065964, 57020481, 99917599, 31904591, 26664538, 36139942, 59109400, 85242963, 35996293, 45790169, 49598724, 68041839, 29516592, 33431960, 69641513, 84684495, 26863229, 3233084, 25842078, 71965942, 90013093, 72274002, 84840549, 26998766, 75128745, 10961421, 11923835, 63372756, 65271999, 6221471, 48395186, 66667729, 26776922, 2208785, 4515343, 96420429, 6038457, 3183975, 30694952, 38494874, 8651647, 9860968, 68816413, 89419466, 55470718, 1788101, 26392416, 30366150, 13862149, 49236559, 47893286, 67451935, 37957788, 4978543, 34667286, 89996536, 59614746, 14723410, 1197320, 67227442, 73392814, 85695762, 82178706, 62232211, 22942635, 21673260, 62490109, 68110330, 99604946, 31733363, 40027975, 12571310, 75135448, 24826575, 98648327, 44784505, 47213183, 92071990, 86301513, 7423788, 92867155, 16097038, 26275734, 42199455, 55189057, 40197395, 85117092, 66322959, 80555751, 78909561, 36505482, 37739481, 7955293, 91957544, 41380093, 29188588, 6793819, 7685448, 42692881, 77694700, 37501808, 51464002, 6808825, 41245325, 8791066, 99297164, 41442762, 59329511, 38022834, 71333116, 27041967, 18131876, 91664334, 55143724, 14045383, 49641577, 32058615, 79806380, 24314885, 14363867, 87720882, 68204242, 16567550, 8733068, 14731700, 73021291, 3294781, 72357096, 20140249, 59501487, 77072625, 87710366, 62428472, 84187166, 68939068, 50702367, 45995325, 16380211, 66832478, 2717150, 44550764, 68694897, 56531125, 65017137, 72358170, 9886593, 526217, 43506672, 61960400, 54517921, 92530431, 44481640, 24733232, 38645117, 90158785, 83651211, 4095116, 97281847, 90272749, 94090109, 19891772, 57961282, 86001008, 19939935, 33304202, 17068582, 68102437, 28928175, 53842979, 55602660, 74137926, 23569917, 66903004, 14326617, 26397786, 65454636, 10309525, 81677380, 22997281, 84166196, 16595436, 53547802, 65338021, 35456853, 22879907, 81898046, 30891921, 15902805, 87598888, 78589145, 30090481, 95395112, 13819358, 82979980, 60102965, 98653983, 30543215, 79880247, 55615885, 84002370, 26507214, 35192533, 92541302, 96906410, 54868730, 56473732, 7011964, 22113133, 31126490, 59910624, 97379790, 63015256, 40356877, 51466049, 60393039, 37435892, 83083131, 40781854, 5822202, 36930650, 97783876, 77187825, 26744917, 39986008, 56955985, 12664567, 40686254, 22129328, 14349098, 17727650, 30771409, 46540998, 61897582, 54987042, 669105, 14220886, 90090964, 8696647, 74614639, 8115266, 17764950, 26102057, 27411561, 33797252, 13231279, 38510840, 35092039, 49882705, 91141395, 8055981, 62115552, 78602717, 78549759, 61271144, 749283, 48675329, 93515664, 17976208, 9860195, 11161731, 69848388, 7919588, 44844121, 8807940, 12416768, 30395570, 13348726, 77377183, 62552524, 66116458, 42644903, 61728685, 82886381, 23134715, 7300047, 58180653, 11543098, 2331773, 83948335, 45428665, 81172706, 71522968, 6505939, 83747892, 67281495, 4508700, 34044787, 52204879, 20122224, 27665211, 77284619, 57658654, 67513640, 45381876, 54058788, 10597197, 65038678, 32161669, 91240048, 99549373, 14947650, 44473167, 57241521, 16445503, 51715482, 22450468, 73235980, 45237957, 79657802, 30998561, 57393458, 37280276, 64848072, 62430984, 30764367, 45794415, 38009615, 79738755, 37183543, 72793444, 48260151, 22766820, 47708850, 93562257, 99965001, 77898274, 24915585, 40781449, 73786944, 64055960, 60955663, 36780454, 57240218, 6521313, 62693428, 76825057, 51588015, 45617087, 3509435, 96852321, 88251446, 3880712, 57248122, 26114953, 20568363, 98943869, 42237907, 91990218, 38008118, 64098930, 6525948, 88130087, 16099750, 18411915, 74724075, 18663507, 39373729, 18699206, 10264691, 47738185, 4806458, 41092102, 17386996, 99971982, 54762643, 84406788, 99729357, 94076128, 1936762, 95290172, 64087743, 72238278 +68110330, 45990383, 3233084, 45237957, 89804152, 61815223, 45428665, 99917599, 75153252, 63059024, 86543538, 61859581, 2891150, 3487592, 91141395, 66611759, 18466635, 17857111, 59177718, 91664334, 88047921, 92033260, 45919976, 62496012, 89699445, 30771409, 7502255, 74614639, 26139110, 69579137, 26863229, 73235980, 64098930, 86460488, 70004753, 22108665, 53666583, 30543215, 13468268, 51507425, 37560164, 16099750, 97379790, 72357096, 62428472, 17081350, 47738185, 90090964, 91240048, 66137019, 45667668, 18001617, 19891772, 96852321, 86001008, 65271999, 96420429, 3183975, 95010552, 61271144, 54014062, 84406788, 94539824, 32159704, 30764367, 53393358, 15902805, 70727211, 61928316, 44784505, 62051033, 61263068, 77301523, 89217461, 23134715, 79880247, 26507214, 7955293, 63967300, 12348118, 7182642, 90004325, 27665211, 33935899, 14363867, 95726235, 37435892, 86242799, 60955663, 36930650, 37166608, 99333375, 39986008, 77413012, 79821897, 99971982, 73031054, 4985896, 2917920, 99125126, 36812683, 19457589, 82339363, 82327024, 45790169, 4434662, 90272749, 47361209, 17539625, 13231279, 99658235, 3088684, 49328608, 42782652, 55602660, 66903004, 57393458, 64848072, 47893286, 27325428, 51590803, 14626618, 69848388, 19898053, 85695762, 21993752, 73222868, 38009615, 87598888, 99861373, 79738755, 5482538, 69255765, 92867155, 4204661, 11543098, 16019925, 54427233, 29188588, 39847321, 42073124, 92998591, 22766820, 70367851, 79136082, 18504197, 50007421, 56461322, 89811711, 14045383, 82651278, 20122224, 4119747, 74110882, 79806380, 24314885, 91938191, 78300864, 6819644, 30218878, 86022504, 84127901, 57240218, 17385531, 52261574, 17727650, 48774913, 82779622, 40152546, 97641116, 30163921, 91255408, 8696647, 67124282, 57020481, 43501211, 83368048, 10366309, 69136837, 83651211, 85242963, 83533741, 97940276, 59371804, 33797252, 68041839, 19939935, 76315420, 93359396, 11923835, 88653118, 36312813, 9860968, 95290172, 36933038, 26392416, 36189527, 37957788, 92398073, 32250045, 22997281, 15163258, 34236719, 98130363, 53547802, 6153406, 82178706, 62490109, 12416768, 20867149, 31733363, 93790285, 55753905, 24826575, 64602895, 78589145, 62552524, 9575954, 66116458, 8129978, 10649306, 95395112, 61728685, 60581278, 17016156, 94360702, 48088883, 3233569, 58180653, 36685023, 85117092, 99729357, 55615885, 4798568, 80555751, 38365584, 77620120, 85116755, 32426752, 27187213, 44889423, 6505939, 33237508, 71316369, 38022834, 74357852, 78766358, 16054533, 16424199, 81774825, 24915585, 90654836, 72732205, 71920426, 4069912, 73786944, 12024238, 40781854, 73021291, 3294781, 99226875, 66663942, 34497327, 77072625, 57658654, 41092102, 36780454, 50567636, 67513640, 6986898, 99515901, 76330843, 99224392, 29819940, 96318094, 59318837, 247198, 97057187, 14220886, 68694897, 45306070, 56531125, 94911072, 88904910, 44627776, 29065964, 36753250, 31904591, 26664538, 36139942, 38645117, 76825057, 17764950, 14947650, 93053405, 57241521, 29510992, 81855445, 88698958, 17957593, 17068582, 68102437, 58749, 30998561, 62115552, 30694952, 1936762, 91990218, 17976208, 15075176, 34667286, 59614746, 22942635, 33061250, 44177011, 83789391, 40027975, 8729146, 59197747, 77377183, 92071990, 86301513, 43045786, 92554120, 62803858, 75777973, 29932657, 82886381, 21289531, 47887470, 569864, 55189057, 8913721, 34698463, 42307449, 48260151, 80014588, 84904436, 1239555, 91957544, 34295794, 98371444, 7685448, 99690194, 112651, 5073754, 71469330, 93562257, 6808825, 55960386, 7011964, 99297164, 41442762, 58751351, 2607799, 40781449, 7687278, 65074535, 63015256, 50188404, 56153345, 99021067, 23194618, 47298834, 55247455, 67474219, 77284619, 20140249, 824872, 5822202, 61982238, 9603598, 79322415, 97783876, 77187825, 55669657, 26744917, 45848907, 45381876, 12664567, 26292919, 40686254, 5832946, 71657078, 34946859, 45995325, 2717150, 44550764, 48893685, 89637706, 72358170, 95251277, 18833224, 54517921, 45075471, 39553046, 92530431, 68824981, 32161669, 44842615, 93057697, 90158785, 4787945, 45407418, 41092172, 20642888, 22405120, 76540481, 26102057, 27411561, 80316608, 97281847, 78218845, 57961282, 75820087, 71965942, 72274002, 16445503, 90457870, 72019362, 40865610, 10961421, 63372756, 7104732, 44664587, 66667729, 28787861, 28796059, 79657802, 33895336, 57248122, 7229550, 78602717, 81805959, 33628349, 9259676, 20568363, 85023028, 68816413, 89419466, 55470718, 88444207, 26397786, 37280276, 22200378, 749283, 48675329, 11161731, 34493392, 54663246, 6525948, 14723410, 15015906, 67227442, 32699744, 89499542, 53648154, 62232211, 8807940, 99604946, 37224844, 75135448, 48360198, 65880522, 88130087, 3235882, 78561158, 30569392, 7423788, 68875490, 77312810, 82979980, 93933709, 98653983, 40197395, 51149804, 26063929, 84002370, 62936963, 78909561, 32590267, 23110625, 6793819, 57163802, 92604458, 96906410, 54868730, 69786756, 42692881, 74724075, 47708850, 51464002, 23740167, 99965001, 18783702, 86118021, 56473732, 33699435, 25636669, 4508700, 22113133, 5970607, 71333116, 29466406, 29834512, 96726697, 55143724, 36135, 68316156, 4091162, 27625735, 37659250, 92692978, 82532312, 18699206, 86798033, 44479073, 87720882, 68204242, 72238278, 85571389, 29994197, 40356877, 18806856, 60393039, 31491938, 1375023, 99524975, 43152977, 24953498, 11757872, 74452589, 11581181, 17037369, 37192445, 45996863, 15536795, 50702367, 83155442, 96709982, 14349098, 54058788, 94076128, 65851721, 66832478, 86821229, 72278539, 74743862, 11365791, 69697787, 9398733, 321665, 9886593, 44846932, 61141874, 43506672, 32274392, 93566986, 24733232, 19376156, 59109400, 35512853, 51588015, 92787493, 16405341, 29516592, 44473167, 8634541, 33431960, 94090109, 13173644, 58208470, 84684495, 83133790, 27185644, 90013093, 43357947, 51715482, 26998766, 78785507, 28928175, 66250369, 54199160, 26776922, 96735716, 508198, 82427263, 2208785, 91802888, 6038457, 14093520, 47792865, 70782102, 1788101, 49236559, 51472275, 6497830, 44915235, 28734791, 67451935, 9860195, 59405277, 6157724, 65454636, 20885148, 81677380, 84166196, 38008118, 1197320, 26734892, 20681921, 3633375, 70596786, 78884452, 55770687, 64087743, 54263880, 30787683, 76434144, 67031644, 39801351, 20765474, 55850790, 42967683, 16097038, 39171895, 72793444, 17058722, 874791, 84293052, 65081429, 2331773, 72614359, 44847298, 30463802, 415901, 92692283, 92541302, 51047803, 43322743, 29029316, 81172706, 18663507, 37620363, 37501808, 36808486, 83747892, 41245325, 70541760, 7066775, 17146629, 68644627, 95957797, 16971929, 27041967, 31126490, 49641577, 32058615, 52060076, 30811010, 65047700, 29401781, 9257405, 10264691, 16567550, 51466049, 8733068, 94711842, 56484900, 74441448, 14731700, 67030811, 98948034, 96193415, 54606384, 33201905, 83269727, 84187166, 68939068, 8128637, 53632373, 17386996, 1250437, 4064751, 21397057, 44060493, 40534591, 6871053, 16380211, 10597197, 67793644, 50806615, 61897582, 82897371, 45049308, 65038678, 71083565, 44481640, 83210802, 40677414, 8115266, 23432750, 72725103, 9058407, 4095116, 33336148, 26493119, 8373289, 80251430, 92215320, 3509435, 25842078, 35092039, 22450468, 87160386, 28550822, 75128745, 34698428, 54762643, 58224549, 9623492, 3880712, 53842979, 4515343, 74137926, 23569917, 42947632, 36468541, 10358899, 63628376, 9829782, 90061527, 17365188, 20836893, 24591705, 61712234, 81853704, 24058273, 45794415, 20486294, 35456853, 81898046, 44844121, 21673260, 40984766, 89046466, 1569515, 176257, 19486173, 64157906, 68000591, 47213183, 5640302, 98739783, 13819358, 42644903, 65275241, 37675718, 37183543, 60102965, 82052050, 66271566, 66322959, 73617245, 36845587, 36505482, 37739481, 35192533, 61741594, 48892201, 83948335, 49597667, 72738685, 9175338, 9188443, 70372191, 71522968, 34044787, 41481685, 28851716, 54232247, 49271185, 1204161, 76960303, 50668599, 72777973, 83302115, 1431742, 83083131, 4668450, 20002147, 38256849, 24168411, 31727379, 33565483, 89078848, 36396314, 22129328, 37891451, 94595668, 53888755, 669105, 44983451, 22721500, 62452034, 65017137, 91281584, 526217, 77300457, 61960400, 79191827, 9599614, 92803766, 13470059, 99549373, 4806458, 48673079, 70800879, 79942022, 35996293, 22435353, 45617087, 97561745, 21533347, 69641513, 38510840, 8099994, 84840549, 61623915, 6221471, 62357986, 8055981, 93270084, 68702632, 66045587, 60430369, 78549759, 51315460, 91831487, 81274566, 59981773, 75229462, 21919959, 67084351, 42237907, 13862149, 70036438, 53802686, 68128438, 89996536, 38658347, 22879907, 30891921, 38061439, 3773993, 24619760, 61373987, 12571310, 30090481, 75407347, 38341669, 31161687, 73168565, 29959549, 1022677, 33553959, 6111563, 43933006, 26275734, 42199455, 63506281, 46851987, 73124510, 48658605, 89214616, 12303248, 77694700, 2204165, 67281495, 8791066, 39373729, 52204879, 59910624, 7517032, 77898274, 61859143, 57359924, 19272365, 39201414, 56424103, 70420215, 82726008, 46870723, 46540998, 32151165, 34432810, 54987042, 5111370, 16387575, 15148031, 5267545, 60176618, 75543508, 49598724, 44348328, 98462867, 21001913, 33304202, 53084256, 48395186, 95581843, 75052463, 14326617, 36580610, 93515664, 15948937, 72495719, 73392814, 69605283, 80246713, 30395570, 76170907, 33123618, 7300047, 76671482, 76703609, 41380093, 15535065, 75986488, 33249630, 66885828, 63152504, 51426311, 59329511, 77787724, 43376279, 85711894, 16684829, 72373496, 62762857, 57803235, 10453030, 168541, 6521313, 92353856, 56515456, 62693428, 23848565, 90310261, 94516935, 91727510, 17894977, 49882705, 88251446, 26114953, 38494874, 83150534, 4199704, 4978543, 62430984, 16595436, 65338021, 15064655, 13348726, 30653863, 18411915, 71300104, 44245960, 47090124, 18131876, 20645197, 64055960, 66319530, 59501487, 30366150, 10309525, 77272628, 7919588, 21070110, 52613508, 62740044, 29635537, 7646095, 73109997, 4099191, 69355476, 66428795, 87710366, 56955985, 30139692, 8651647, 98943869, 60106217, 97011160, 65715134, 52293995, 19101477, 84349107, 98648327, 79922758 +90310261, 22405120, 71300104, 4099191, 2607799, 17764950, 69641513, 84002370, 96906410, 99226875, 99971982, 6521313, 61859581, 8373289, 3509435, 49236559, 51472275, 88130087, 72793444, 2331773, 18806856, 11757872, 36780454, 79821897, 14220886, 67124282, 44627776, 91240048, 38645117, 45617087, 90013093, 26998766, 28928175, 95581843, 96735716, 79922758, 89996536, 73392814, 38061439, 68000591, 20765474, 62803858, 40197395, 12303248, 99965001, 22113133, 52204879, 1204161, 83083131, 34497327, 38256849, 37192445, 53632373, 46540998, 56531125, 99917599, 60176618, 16405341, 68041839, 26493119, 18001617, 78218845, 3487592, 58749, 65271999, 93270084, 66667729, 2208785, 1936762, 9860968, 14093520, 36933038, 18466635, 51590803, 34236719, 98130363, 53547802, 22879907, 44844121, 61373987, 75135448, 61928316, 44784505, 43045786, 38341669, 16097038, 26275734, 58180653, 42199455, 76671482, 85117092, 63506281, 66322959, 874791, 73617245, 30463802, 19101477, 37560164, 45428665, 42073124, 77694700, 25636669, 58751351, 29834512, 82651278, 28851716, 71920426, 86798033, 65074535, 68204242, 10264691, 8733068, 20002147, 20140249, 89699445, 26744917, 50702367, 10597197, 54987042, 2717150, 62693428, 526217, 32274392, 82339363, 36139942, 92803766, 59109400, 90158785, 4806458, 92787493, 20642888, 76540481, 26139110, 80316608, 47361209, 92215320, 71965942, 17894977, 72274002, 43357947, 11923835, 58224549, 3880712, 66045587, 42782652, 508198, 4515343, 96420429, 66903004, 30366150, 13862149, 47893286, 59405277, 34493392, 54663246, 3633375, 45794415, 80246713, 89046466, 30787683, 76170907, 78589145, 55850790, 60581278, 21289531, 29959549, 30543215, 26063929, 80014588, 84904436, 4798568, 7955293, 73124510, 85116755, 16099750, 70372191, 74724075, 93562257, 18783702, 86118021, 8791066, 18131876, 68316156, 30811010, 27625735, 69355476, 49271185, 7687278, 87720882, 39201414, 83302115, 64055960, 60955663, 72357096, 824872, 59501487, 77072625, 77187825, 37166608, 36396314, 57803235, 17081350, 46870723, 34946859, 37891451, 168541, 50806615, 4985896, 9398733, 29065964, 43501211, 45075471, 83368048, 79191827, 9599614, 83651211, 35512853, 9058407, 35996293, 75543508, 59371804, 97281847, 97561745, 94090109, 84684495, 96852321, 83133790, 90457870, 93359396, 88251446, 61623915, 66250369, 28796059, 7229550, 74137926, 23569917, 30694952, 36468541, 26392416, 36580610, 6497830, 44915235, 4978543, 14626618, 68128438, 94539824, 30764367, 84166196, 15015906, 21070110, 38658347, 82178706, 62232211, 62490109, 86460488, 87598888, 93790285, 77377183, 8129978, 75777973, 17016156, 37675718, 569864, 89217461, 82979980, 39171895, 60102965, 53666583, 55189057, 76703609, 65081429, 26507214, 62740044, 44847298, 78909561, 37739481, 35192533, 38365584, 29635537, 9175338, 89214616, 92604458, 59177718, 32426752, 18663507, 7646095, 33699435, 99297164, 41442762, 39373729, 34044787, 5970607, 71333116, 43376279, 91664334, 97379790, 77898274, 24915585, 37659250, 54232247, 44479073, 50188404, 51466049, 12024238, 73021291, 62496012, 62762857, 11581181, 84187166, 56955985, 45381876, 77413012, 89078848, 82726008, 82779622, 32151165, 96318094, 34432810, 94595668, 97641116, 44983451, 22721500, 74743862, 69697787, 5111370, 91281584, 36753250, 26664538, 32161669, 84349107, 48673079, 70800879, 26102057, 79942022, 2891150, 4434662, 49598724, 33336148, 44473167, 80251430, 86001008, 16445503, 84840549, 91141395, 9623492, 3088684, 33895336, 30998561, 62115552, 51315460, 75052463, 81274566, 42237907, 22200378, 70036438, 4199704, 67451935, 27325428, 61712234, 65338021, 55770687, 53648154, 73222868, 22942635, 21673260, 40984766, 33061250, 44177011, 1569515, 70004753, 99861373, 12571310, 19486173, 64602895, 30090481, 62552524, 78561158, 86301513, 7423788, 95395112, 65275241, 31161687, 73168565, 29932657, 33123618, 1022677, 61263068, 93933709, 23134715, 34698463, 99729357, 16019925, 13468268, 54427233, 46851987, 72614359, 62936963, 61741594, 41380093, 92692283, 77620120, 23110625, 15535065, 72738685, 39847321, 57163802, 43322743, 42692881, 66885828, 27187213, 29029316, 71522968, 36808486, 18504197, 41245325, 7066775, 56461322, 59329511, 16971929, 31126490, 59910624, 49641577, 36135, 40781449, 72732205, 65047700, 19272365, 66428795, 63015256, 16684829, 85571389, 95726235, 99524975, 43152977, 56484900, 86242799, 40781854, 3294781, 61982238, 56424103, 97783876, 74452589, 86022504, 33201905, 50567636, 45848907, 45996863, 15536795, 12664567, 96709982, 99515901, 14349098, 44060493, 7502255, 59318837, 6871053, 40152546, 97057187, 30163921, 53888755, 72278539, 90090964, 82897371, 68694897, 2917920, 321665, 72358170, 88904910, 19457589, 54517921, 44481640, 15148031, 10366309, 5267545, 8115266, 72725103, 99549373, 85242963, 27411561, 94516935, 91727510, 33797252, 13173644, 17539625, 81855445, 58208470, 26863229, 19939935, 38510840, 76315420, 22450468, 68102437, 99658235, 28550822, 44664587, 60430369, 91802888, 3183975, 26114953, 20568363, 98943869, 85023028, 59981773, 47792865, 55470718, 21919959, 95290172, 95010552, 61271144, 67084351, 15075176, 11161731, 34667286, 90061527, 77272628, 15163258, 38008118, 64098930, 14723410, 7919588, 16595436, 69605283, 20486294, 81898046, 68110330, 38009615, 3773993, 24619760, 48360198, 5482538, 76434144, 13348726, 98648327, 9575954, 92071990, 69255765, 63059024, 98739783, 13819358, 92554120, 6111563, 94360702, 89804152, 11543098, 48260151, 30653863, 61815223, 415901, 32590267, 49597667, 98371444, 51047803, 69786756, 33249630, 22766820, 37501808, 51464002, 51426311, 55960386, 17146629, 71316369, 38022834, 29466406, 74357852, 16054533, 57359924, 4091162, 92692978, 74110882, 18699206, 92033260, 50668599, 29401781, 9257405, 29994197, 31491938, 20645197, 6819644, 67474219, 77284619, 98948034, 70420215, 57658654, 55669657, 41092102, 24168411, 87710366, 17037369, 6986898, 17386996, 83155442, 52261574, 99224392, 4064751, 17727650, 47738185, 48774913, 71657078, 40534591, 54058788, 94076128, 86821229, 91255408, 92353856, 48893685, 62452034, 65017137, 16387575, 18833224, 43506672, 74614639, 61960400, 71083565, 39553046, 93566986, 24733232, 44842615, 19376156, 76825057, 13470059, 41092172, 93053405, 45667668, 29510992, 17957593, 17068582, 87160386, 6221471, 73235980, 45237957, 36312813, 79657802, 26776922, 49328608, 82427263, 55602660, 78602717, 33628349, 8651647, 75229462, 14326617, 26397786, 91990218, 57393458, 54014062, 10358899, 64848072, 48675329, 65454636, 92398073, 10309525, 15948937, 97011160, 32159704, 59614746, 26734892, 81853704, 67227442, 20681921, 32699744, 24058273, 6153406, 78884452, 89499542, 30891921, 8807940, 12416768, 70727211, 31733363, 40027975, 15064655, 55753905, 3235882, 75407347, 30569392, 66116458, 5640302, 68875490, 42644903, 82886381, 33553959, 86543538, 48088883, 36685023, 52293995, 42307449, 36845587, 80555751, 1239555, 75986488, 29188588, 18411915, 54868730, 5073754, 44245960, 81172706, 71469330, 6505939, 79136082, 83747892, 70541760, 23740167, 50007421, 56473732, 47090124, 68644627, 77787724, 27041967, 78766358, 88047921, 7182642, 55143724, 41481685, 7517032, 61859143, 52060076, 90004325, 20122224, 4119747, 24314885, 14363867, 4069912, 56153345, 99021067, 72238278, 73786944, 40356877, 60393039, 37435892, 74441448, 4668450, 79322415, 62428472, 68939068, 39986008, 84127901, 76330843, 5832946, 30771409, 10453030, 45995325, 73031054, 67793644, 61897582, 45306070, 56515456, 36812683, 57020481, 45049308, 92530431, 93057697, 4787945, 45407418, 97940276, 66137019, 30139692, 29516592, 57241521, 75820087, 88698958, 25842078, 35092039, 51715482, 49882705, 40865610, 75128745, 10961421, 88653118, 62357986, 68702632, 57248122, 91831487, 83150534, 1788101, 84406788, 63628376, 36189527, 93515664, 6157724, 53802686, 32250045, 81677380, 69848388, 1197320, 17857111, 20836893, 65715134, 70596786, 19898053, 30395570, 20867149, 83789391, 8729146, 59197747, 65880522, 67031644, 22108665, 52613508, 10649306, 39801351, 77312810, 7300047, 4204661, 3233569, 82052050, 66271566, 8913721, 51507425, 84293052, 36505482, 91957544, 92541302, 9188443, 7685448, 112651, 12348118, 44889423, 70367851, 2204165, 67281495, 7011964, 95957797, 96726697, 81774825, 90654836, 82532312, 76960303, 72777973, 72373496, 16567550, 94711842, 47298834, 1375023, 55247455, 1431742, 78300864, 66319530, 67030811, 5822202, 9603598, 54606384, 67513640, 57240218, 1250437, 21397057, 247198, 16380211, 669105, 44550764, 99125126, 44846932, 31904591, 83210802, 14947650, 45790169, 69579137, 21533347, 13231279, 57961282, 44348328, 98462867, 21001913, 8099994, 75153252, 53084256, 34698428, 63372756, 48395186, 8055981, 28787861, 54199160, 53842979, 6038457, 68816413, 60106217, 42947632, 37280276, 9829782, 28734791, 9860195, 22997281, 21993752, 99604946, 54263880, 176257, 37224844, 47213183, 62051033, 61728685, 37183543, 42967683, 79880247, 83948335, 6793819, 48658605, 63152504, 33237508, 89811711, 14045383, 16424199, 23194618, 83269727, 99333375, 31727379, 33565483, 26292919, 66832478, 8696647, 89637706, 94911072, 77300457, 95251277, 68824981, 69136837, 40677414, 4095116, 83533741, 33431960, 19891772, 27185644, 78785507, 54762643, 7104732, 78549759, 38494874, 37957788, 17976208, 20885148, 62430984, 6525948, 35456853, 15902805, 79738755, 64157906, 45990383, 92867155, 47887470, 55615885, 34295794, 92998591, 47708850, 73109997, 4508700, 85711894, 27665211, 33935899, 91938191, 24953498, 14731700, 66663942, 96193415, 40686254, 17385531, 22129328, 29819940, 23848565, 8634541, 72019362, 81805959, 89419466, 70782102, 749283, 72495719, 17365188, 85695762, 24826575, 51149804, 48892201, 63967300, 6808825, 32058615, 79806380, 45919976, 36930650, 8128637, 65851721, 11365791, 9886593, 61141874, 82327024, 23432750, 51588015, 90272749, 3233084, 33304202, 66611759, 9259676, 88444207, 24591705, 53393358, 77301523, 98653983, 17058722, 99690194, 37620363, 30218878, 65038678, 22435353, 64087743, 43933006 +10649306, 82052050, 2917920, 81855445, 24314885, 83302115, 11581181, 11365791, 508198, 569864, 85117092, 4099191, 17385531, 45075471, 44481640, 26863229, 96420429, 14626618, 9575954, 33553959, 8913721, 73124510, 7687278, 65074535, 94711842, 50567636, 26292919, 89078848, 5832946, 43501211, 39553046, 68824981, 45407418, 93359396, 9623492, 42782652, 26392416, 22879907, 70004753, 31733363, 30090481, 47213183, 42644903, 61263068, 63506281, 84904436, 2331773, 48658605, 89214616, 74724075, 7646095, 58751351, 30811010, 73021291, 3294781, 70420215, 33201905, 39986008, 45381876, 4064751, 59318837, 168541, 9886593, 29065964, 45049308, 10366309, 13470059, 49598724, 13173644, 69641513, 84684495, 19939935, 90457870, 78785507, 11923835, 8055981, 53842979, 95290172, 95010552, 10358899, 17976208, 18466635, 32250045, 98130363, 67227442, 32699744, 55770687, 30891921, 62490109, 3773993, 15064655, 19486173, 64602895, 98739783, 13819358, 33123618, 47887470, 42967683, 51149804, 76703609, 4798568, 72614359, 35192533, 91957544, 34295794, 51047803, 54868730, 70372191, 92998591, 70367851, 47090124, 95957797, 77787724, 43376279, 90004325, 20122224, 69355476, 18806856, 4668450, 6819644, 97783876, 37166608, 67513640, 6986898, 48774913, 40534591, 96318094, 74743862, 56531125, 91281584, 526217, 18833224, 32274392, 72725103, 70800879, 93053405, 8373289, 8634541, 29510992, 25842078, 35092039, 72274002, 99658235, 34698428, 3088684, 79922758, 81805959, 26397786, 63628376, 90061527, 73392814, 53648154, 38061439, 176257, 79738755, 8729146, 98648327, 77377183, 44784505, 38341669, 62803858, 61728685, 60581278, 17016156, 89217461, 39171895, 60102965, 26507214, 15535065, 6793819, 57163802, 71300104, 77694700, 29029316, 6808825, 23740167, 55960386, 7011964, 68644627, 2607799, 78766358, 7517032, 54232247, 27665211, 92033260, 56153345, 72373496, 45919976, 16567550, 95726235, 8733068, 60393039, 24953498, 34497327, 47738185, 6871053, 45995325, 247198, 34432810, 73031054, 4985896, 82897371, 68694897, 99125126, 65038678, 83368048, 79191827, 24733232, 32161669, 69136837, 92803766, 97940276, 66137019, 75543508, 59371804, 69579137, 47361209, 27185644, 63372756, 66611759, 68702632, 28787861, 28796059, 66045587, 49328608, 30694952, 91831487, 85023028, 42947632, 47792865, 75229462, 67084351, 70036438, 4978543, 6157724, 15948937, 72495719, 17365188, 15163258, 30764367, 84166196, 6525948, 26734892, 24058273, 78884452, 21993752, 35456853, 21673260, 40984766, 86460488, 87598888, 48360198, 22108665, 61928316, 52613508, 78561158, 30569392, 31161687, 17058722, 36685023, 26063929, 874791, 84293052, 84002370, 44847298, 19101477, 48892201, 38365584, 72738685, 98371444, 12303248, 66885828, 27187213, 44245960, 18663507, 67281495, 99965001, 56473732, 34044787, 16971929, 27041967, 16054533, 97379790, 90654836, 37659250, 82532312, 14363867, 44479073, 16684829, 23194618, 29994197, 43152977, 824872, 61982238, 57658654, 36930650, 33565483, 84187166, 84127901, 37891451, 94076128, 94595668, 97641116, 19457589, 61960400, 9599614, 36139942, 59109400, 60176618, 23432750, 4787945, 4806458, 41092172, 94516935, 80316608, 33336148, 45667668, 45617087, 97281847, 90272749, 30139692, 57961282, 3509435, 83133790, 90013093, 16445503, 75153252, 10961421, 66250369, 36312813, 33895336, 2208785, 7229550, 62115552, 74137926, 51315460, 98943869, 89419466, 70782102, 61271144, 30366150, 9829782, 47893286, 51472275, 28734791, 11161731, 89996536, 54663246, 1197320, 24591705, 16595436, 45794415, 69605283, 21070110, 22942635, 99604946, 54263880, 61373987, 93790285, 5482538, 68000591, 92071990, 8129978, 7423788, 95395112, 21289531, 29959549, 77301523, 16097038, 48088883, 53666583, 76671482, 66271566, 11543098, 34698463, 48260151, 16019925, 62740044, 62936963, 49597667, 92692283, 75986488, 32426752, 43322743, 73109997, 6505939, 7066775, 51426311, 18783702, 50007421, 71316369, 59329511, 29466406, 96726697, 31126490, 59910624, 16424199, 32058615, 81774825, 57359924, 4091162, 92692978, 91938191, 19272365, 63015256, 50188404, 99021067, 87720882, 50668599, 72238278, 9257405, 51466049, 47298834, 56484900, 86242799, 60955663, 67474219, 98948034, 5822202, 66663942, 11757872, 86022504, 24168411, 36780454, 99333375, 26744917, 37192445, 57803235, 17386996, 82726008, 22129328, 21397057, 29819940, 7502255, 34946859, 40152546, 54987042, 86821229, 2717150, 48893685, 5111370, 62693428, 62452034, 72358170, 88904910, 44627776, 16387575, 74614639, 99917599, 31904591, 82339363, 44842615, 5267545, 90158785, 35512853, 17764950, 22405120, 76540481, 85242963, 79942022, 26139110, 57241521, 80251430, 98462867, 86001008, 17894977, 43357947, 28550822, 91141395, 88653118, 44664587, 93270084, 66667729, 54199160, 26776922, 30998561, 4515343, 60430369, 20568363, 75052463, 1936762, 59981773, 14093520, 83150534, 36933038, 36580610, 54014062, 4199704, 6497830, 15075176, 92398073, 20885148, 10309525, 53802686, 34667286, 27325428, 22997281, 62430984, 64098930, 7919588, 20836893, 61712234, 20681921, 3633375, 70596786, 15902805, 33061250, 24619760, 30787683, 70727211, 99861373, 75135448, 3235882, 69255765, 75407347, 86301513, 43045786, 92554120, 62051033, 75777973, 29932657, 82886381, 1022677, 37675718, 77312810, 93933709, 94360702, 7300047, 4204661, 98653983, 58180653, 55189057, 30543215, 30653863, 7955293, 83948335, 39847321, 29635537, 96906410, 59177718, 42073124, 33249630, 22766820, 47708850, 37501808, 36808486, 51464002, 17146629, 99297164, 41442762, 22113133, 38022834, 91664334, 74357852, 88047921, 49641577, 41481685, 77898274, 61859143, 28851716, 27625735, 74110882, 18699206, 33935899, 65047700, 76960303, 4069912, 85571389, 39201414, 10264691, 1375023, 20645197, 74441448, 83083131, 40781854, 30218878, 59501487, 77072625, 54606384, 83269727, 31727379, 68939068, 56955985, 8128637, 15536795, 12664567, 50702367, 46870723, 99224392, 71657078, 54058788, 32151165, 79821897, 99971982, 669105, 6521313, 90090964, 69697787, 8696647, 45306070, 56515456, 321665, 65017137, 36812683, 57020481, 36753250, 77300457, 54517921, 92530431, 93566986, 15148031, 26664538, 61859581, 93057697, 84349107, 91240048, 8115266, 51588015, 92787493, 16405341, 4095116, 14947650, 2891150, 22435353, 4434662, 33797252, 26493119, 97561745, 29516592, 17539625, 58208470, 17957593, 71965942, 38510840, 33304202, 17068582, 72019362, 84840549, 3487592, 58749, 65271999, 6221471, 48395186, 62357986, 58224549, 96735716, 82427263, 78549759, 33628349, 9259676, 8651647, 21919959, 88444207, 1788101, 13862149, 44915235, 37957788, 59405277, 97011160, 68128438, 34493392, 94539824, 32159704, 34236719, 38008118, 81853704, 65715134, 6153406, 81898046, 8807940, 38009615, 30395570, 44177011, 1569515, 83789391, 37224844, 12571310, 76434144, 13348726, 62552524, 63059024, 39801351, 20765474, 92867155, 73168565, 86543538, 26275734, 72793444, 42199455, 40197395, 52293995, 42307449, 99729357, 66322959, 54427233, 55615885, 65081429, 36845587, 80555751, 78909561, 36505482, 30463802, 415901, 1239555, 41380093, 23110625, 18411915, 7685448, 99690194, 112651, 63967300, 42692881, 71522968, 37620363, 93562257, 63152504, 79136082, 70541760, 86118021, 33699435, 25636669, 71333116, 52204879, 18131876, 7182642, 68316156, 40781449, 72732205, 71920426, 49271185, 12024238, 31491938, 99524975, 37435892, 1431742, 14731700, 20002147, 66319530, 72357096, 20140249, 99226875, 56424103, 38256849, 41092102, 62428472, 77413012, 36396314, 53632373, 57240218, 96709982, 14349098, 17727650, 30771409, 10453030, 97057187, 67793644, 30163921, 61897582, 44983451, 22721500, 92353856, 94911072, 19376156, 40677414, 90310261, 76825057, 20642888, 35996293, 91727510, 18001617, 33431960, 78218845, 19891772, 92215320, 44348328, 88698958, 3233084, 76315420, 8099994, 68102437, 88251446, 40865610, 53084256, 55602660, 3183975, 78602717, 23569917, 9860968, 81274566, 60106217, 55470718, 36468541, 42237907, 91990218, 84406788, 48675329, 69848388, 53547802, 65338021, 19898053, 89499542, 38658347, 73222868, 82178706, 44844121, 80246713, 12416768, 40027975, 76170907, 59197747, 64157906, 67031644, 65275241, 82979980, 37183543, 23134715, 6111563, 43933006, 89804152, 73617245, 46851987, 61815223, 37560164, 29188588, 85116755, 18504197, 56461322, 29834512, 55143724, 85711894, 14045383, 82651278, 52060076, 79806380, 86798033, 66428795, 68204242, 73786944, 55247455, 78300864, 96193415, 62762857, 79322415, 87710366, 99515901, 76330843, 10597197, 72278539, 91255408, 9398733, 67124282, 89637706, 44846932, 95251277, 71083565, 82327024, 83210802, 38645117, 83651211, 99549373, 48673079, 26102057, 68041839, 94090109, 75820087, 96852321, 28928175, 75128745, 54762643, 45237957, 95581843, 3880712, 79657802, 91802888, 66903004, 68816413, 14326617, 57393458, 37280276, 22200378, 36189527, 67451935, 9860195, 65454636, 51590803, 77272628, 81677380, 15015906, 20486294, 89046466, 20867149, 55753905, 24826575, 65880522, 78589145, 45990383, 66116458, 5640302, 55850790, 3233569, 51507425, 80014588, 79880247, 16099750, 92604458, 12348118, 44889423, 71469330, 83747892, 41245325, 39373729, 33237508, 1204161, 72777973, 40356877, 67030811, 74452589, 77187825, 89699445, 45996863, 40686254, 1250437, 82779622, 16380211, 65851721, 66832478, 14220886, 44550764, 61141874, 9058407, 83533741, 27411561, 22450468, 87160386, 49882705, 7104732, 73235980, 57248122, 6038457, 26114953, 38494874, 49236559, 749283, 93515664, 14723410, 68110330, 88130087, 68875490, 13468268, 37739481, 32590267, 9175338, 69786756, 5073754, 81172706, 8791066, 4508700, 89811711, 36135, 24915585, 29401781, 64055960, 77284619, 9603598, 83155442, 17081350, 44060493, 50806615, 53888755, 43506672, 44473167, 21533347, 13231279, 21001913, 51715482, 61623915, 64848072, 62232211, 77620120, 92541302, 45428665, 5970607, 4119747, 62496012, 55669657, 17037369, 23848565, 45790169, 26998766, 9188443, 45848907, 52261574, 59614746, 17857111, 53393358, 61741594, 2204165, 85695762, 64087743, 46540998 +11543098, 54427233, 53084256, 96735716, 4091162, 6871053, 91255408, 79191827, 92787493, 29510992, 17068582, 28928175, 78561158, 8129978, 4204661, 89214616, 70372191, 12303248, 7646095, 74357852, 16054533, 57359924, 8733068, 32151165, 94595668, 14220886, 19457589, 78218845, 78785507, 16595436, 22108665, 9575954, 66116458, 98739783, 86543538, 13468268, 30653863, 71300104, 36808486, 55960386, 33935899, 29994197, 74452589, 37192445, 17081350, 17727650, 37891451, 66832478, 94911072, 82327024, 90310261, 60176618, 41092172, 18001617, 57241521, 75229462, 59405277, 32159704, 24058273, 55770687, 21993752, 12416768, 44784505, 7423788, 93933709, 80014588, 9175338, 92604458, 66885828, 5073754, 44889423, 86118021, 95957797, 29834512, 32058615, 20122224, 69355476, 71920426, 24314885, 99524975, 55247455, 86242799, 67030811, 98948034, 96193415, 34497327, 97783876, 56955985, 26292919, 77413012, 10453030, 50806615, 11365791, 5111370, 36753250, 54517921, 82339363, 19376156, 69136837, 90158785, 4787945, 17764950, 48673079, 26139110, 17539625, 17957593, 90013093, 72274002, 72019362, 40865610, 93270084, 66250369, 96420429, 6038457, 20568363, 36468541, 42237907, 15948937, 77272628, 14626618, 89996536, 26734892, 15015906, 73222868, 62232211, 22942635, 99604946, 64602895, 30090481, 68000591, 47213183, 31161687, 55850790, 29932657, 58180653, 40197395, 16019925, 874791, 73617245, 19101477, 32590267, 73124510, 72738685, 45428665, 29635537, 74724075, 29029316, 37620363, 51426311, 16971929, 78766358, 97379790, 16424199, 27625735, 49271185, 7687278, 39201414, 94711842, 37435892, 86022504, 24168411, 83269727, 11581181, 33565483, 68939068, 26744917, 45381876, 6986898, 4064751, 29819940, 34946859, 86821229, 74743862, 69697787, 67124282, 526217, 43501211, 61960400, 93566986, 44481640, 61859581, 9599614, 23848565, 38645117, 22405120, 83533741, 79942022, 91727510, 8373289, 21533347, 38510840, 27185644, 35092039, 22450468, 62357986, 3088684, 95581843, 28787861, 66045587, 74137926, 51315460, 9860968, 60106217, 13862149, 84406788, 49236559, 37280276, 47893286, 48675329, 28734791, 15075176, 9860195, 34667286, 32250045, 62430984, 94539824, 34236719, 54663246, 6525948, 1197320, 81853704, 78884452, 38658347, 80246713, 3773993, 87598888, 76170907, 75135448, 48360198, 76434144, 61928316, 62552524, 42644903, 60581278, 21289531, 77312810, 39171895, 48088883, 89804152, 42199455, 30543215, 76671482, 62740044, 44847298, 96906410, 32426752, 77694700, 27187213, 71522968, 51464002, 41245325, 67281495, 56461322, 33699435, 29466406, 59910624, 82651278, 90004325, 72732205, 74110882, 91938191, 66428795, 45919976, 43152977, 56484900, 24953498, 40781854, 20002147, 6819644, 3294781, 824872, 99226875, 66663942, 11757872, 55669657, 37166608, 17037369, 45848907, 50702367, 83155442, 76330843, 44060493, 5832946, 46540998, 44983451, 2917920, 62693428, 321665, 99125126, 88904910, 16387575, 18833224, 26664538, 44842615, 36139942, 91240048, 35512853, 9058407, 80316608, 33336148, 45667668, 30139692, 33431960, 92215320, 3509435, 69641513, 96852321, 19939935, 28550822, 63372756, 65271999, 45237957, 30998561, 26114953, 23569917, 33628349, 9259676, 98943869, 70782102, 1788101, 4199704, 93515664, 67451935, 10309525, 53802686, 17365188, 68128438, 30764367, 20836893, 65338021, 45794415, 85695762, 53648154, 82178706, 81898046, 30891921, 8807940, 24619760, 44177011, 61373987, 8729146, 55753905, 65880522, 3235882, 92071990, 39801351, 75777973, 37675718, 66271566, 34698463, 66322959, 65081429, 46851987, 78909561, 30463802, 415901, 91957544, 37560164, 6793819, 16099750, 18411915, 51047803, 54868730, 99690194, 22766820, 6808825, 70541760, 50007421, 4099191, 7011964, 17146629, 41442762, 58751351, 89811711, 22113133, 2607799, 41481685, 36135, 30811010, 92692978, 56153345, 68204242, 23194618, 29401781, 85571389, 95726235, 83302115, 12024238, 60393039, 1431742, 66319530, 59501487, 61982238, 62762857, 38256849, 33201905, 50567636, 36396314, 84127901, 57803235, 40686254, 47738185, 48774913, 40534591, 67793644, 61897582, 72278539, 22721500, 90090964, 48893685, 68694897, 62452034, 44627776, 29065964, 91281584, 45075471, 83368048, 68824981, 15148031, 32161669, 83651211, 51588015, 99549373, 76540481, 85242963, 94516935, 97940276, 66137019, 75543508, 22435353, 45790169, 45617087, 97281847, 44473167, 19891772, 47361209, 58208470, 75820087, 44348328, 84684495, 25842078, 17894977, 43357947, 8099994, 90457870, 84840549, 68102437, 75128745, 3487592, 44664587, 508198, 2208785, 60430369, 79922758, 55602660, 91831487, 85023028, 47792865, 55470718, 61271144, 88444207, 91990218, 30366150, 54014062, 10358899, 749283, 37957788, 20885148, 90061527, 72495719, 34493392, 84166196, 14723410, 98130363, 61712234, 20681921, 53547802, 6153406, 73392814, 89499542, 21673260, 86460488, 176257, 20867149, 31733363, 99861373, 78589145, 67031644, 45990383, 68875490, 10649306, 13819358, 92554120, 65275241, 38341669, 92867155, 47887470, 1022677, 61263068, 33553959, 42967683, 94360702, 60102965, 3233569, 36685023, 84904436, 4798568, 72614359, 35192533, 92692283, 42073124, 92998591, 42692881, 81172706, 2204165, 6505939, 18504197, 39373729, 59329511, 27041967, 91664334, 96726697, 88047921, 7182642, 68316156, 28851716, 37659250, 54232247, 1204161, 92033260, 63015256, 76960303, 14363867, 44479073, 72777973, 9257405, 72373496, 10264691, 31491938, 20645197, 4668450, 14731700, 67474219, 57658654, 8128637, 12664567, 53632373, 57240218, 17386996, 52261574, 99224392, 82779622, 7502255, 96318094, 34432810, 97057187, 168541, 2717150, 92353856, 9398733, 89637706, 57020481, 45049308, 95251277, 65038678, 71083565, 92530431, 5267545, 84349107, 92803766, 83210802, 59109400, 13470059, 4806458, 45407418, 16405341, 2891150, 4434662, 33797252, 94090109, 69579137, 26863229, 86001008, 93359396, 26998766, 88251446, 75153252, 58749, 11923835, 91141395, 34698428, 6221471, 66611759, 48395186, 58224549, 66667729, 36312813, 3880712, 53842979, 26776922, 57248122, 91802888, 7229550, 3183975, 78549759, 68816413, 14326617, 83150534, 21919959, 36933038, 26392416, 57393458, 11161731, 92398073, 27325428, 3633375, 70596786, 19898053, 21070110, 44844121, 40984766, 38061439, 33061250, 54263880, 89046466, 1569515, 37224844, 13348726, 69255765, 75407347, 63059024, 62051033, 17016156, 29959549, 89217461, 7300047, 82052050, 8913721, 42307449, 76703609, 26063929, 51507425, 84293052, 84002370, 36845587, 80555751, 1239555, 83948335, 23110625, 92541302, 29188588, 57163802, 48658605, 69786756, 47708850, 37501808, 73109997, 23740167, 18783702, 8791066, 34044787, 52204879, 43376279, 85711894, 7517032, 52060076, 4119747, 18699206, 65074535, 4069912, 72238278, 51466049, 74441448, 60955663, 77284619, 73021291, 72357096, 20140249, 9603598, 36930650, 89699445, 84187166, 67513640, 99515901, 82726008, 22129328, 14349098, 79821897, 59318837, 94076128, 10597197, 99971982, 65851721, 54987042, 669105, 82897371, 56531125, 65017137, 44846932, 61141874, 77300457, 99917599, 31904591, 93057697, 40677414, 20642888, 4095116, 26102057, 14947650, 26493119, 90272749, 3233084, 71965942, 33304202, 76315420, 16445503, 61623915, 73235980, 54199160, 49328608, 78602717, 81805959, 38494874, 1936762, 42947632, 14093520, 64848072, 36189527, 44915235, 51590803, 22997281, 15163258, 17857111, 7919588, 65715134, 15902805, 30787683, 70004753, 83789391, 40027975, 59197747, 19486173, 5482538, 88130087, 64157906, 5640302, 20765474, 569864, 82979980, 43933006, 53666583, 26275734, 98653983, 52293995, 63506281, 79880247, 26507214, 48892201, 15535065, 75986488, 12348118, 44245960, 70367851, 71469330, 63152504, 79136082, 99965001, 47090124, 25636669, 4508700, 5970607, 31126490, 55143724, 81774825, 61859143, 27665211, 86798033, 16684829, 87720882, 50668599, 47298834, 64055960, 78300864, 83083131, 77187825, 87710366, 62428472, 15536795, 89078848, 17385531, 46870723, 30771409, 247198, 45306070, 9886593, 36812683, 74614639, 32274392, 76825057, 8115266, 23432750, 72725103, 70800879, 59371804, 49598724, 29516592, 81855445, 83133790, 49882705, 99658235, 79657802, 62115552, 30694952, 66903004, 95290172, 67084351, 36580610, 22200378, 4978543, 17976208, 6157724, 97011160, 69605283, 20486294, 35456853, 22879907, 62490109, 68110330, 93790285, 12571310, 98648327, 77377183, 52613508, 86301513, 33123618, 77301523, 16097038, 72793444, 85117092, 99729357, 48260151, 55615885, 36505482, 61815223, 7955293, 38365584, 49597667, 41380093, 39847321, 85116755, 9188443, 63967300, 43322743, 83747892, 33237508, 71316369, 18131876, 24915585, 79806380, 65047700, 19272365, 1375023, 77072625, 99333375, 21397057, 54058788, 40152546, 16380211, 30163921, 53888755, 4985896, 44550764, 72358170, 24733232, 10366309, 27411561, 35996293, 13173644, 88698958, 87160386, 10961421, 54762643, 33895336, 4515343, 81274566, 89419466, 63628376, 51472275, 70036438, 81677380, 32699744, 38009615, 64087743, 15064655, 24826575, 30569392, 43045786, 62803858, 73168565, 82886381, 37183543, 23134715, 6111563, 17058722, 55189057, 51149804, 2331773, 98371444, 33249630, 56473732, 68644627, 38022834, 14045383, 40356877, 16567550, 41092102, 54606384, 39986008, 96709982, 1250437, 45995325, 73031054, 6521313, 8696647, 56515456, 93053405, 68041839, 97561745, 98462867, 51715482, 7104732, 8055981, 9623492, 68702632, 82427263, 75052463, 59981773, 26397786, 65454636, 18466635, 69848388, 24591705, 67227442, 53393358, 30395570, 70727211, 79738755, 95395112, 61728685, 62936963, 37739481, 77620120, 34295794, 7685448, 59177718, 112651, 99297164, 49641577, 77898274, 82532312, 50188404, 99021067, 56424103, 70420215, 79322415, 36780454, 31727379, 97641116, 8634541, 80251430, 13231279, 57961282, 88653118, 28796059, 42782652, 8651647, 95010552, 9829782, 6497830, 59614746, 61741594, 18663507, 93562257, 7066775, 77787724, 71333116, 30218878, 45996863, 71657078, 43506672, 39553046, 21001913, 38008118, 40781449, 90654836, 73786944, 62496012, 18806856, 5822202, 64098930 +17016156, 76671482, 85117092, 59177718, 57803235, 92787493, 18001617, 92215320, 28550822, 53842979, 66903004, 73617245, 69786756, 56473732, 14045383, 10264691, 12024238, 74441448, 36780454, 6521313, 36139942, 27185644, 51715482, 28787861, 34493392, 24591705, 92554120, 38365584, 71300104, 99965001, 61859143, 65047700, 60955663, 29819940, 91255408, 2917920, 84349107, 20642888, 33336148, 78218845, 62357986, 36468541, 67084351, 91990218, 30366150, 10358899, 72495719, 16595436, 48360198, 64602895, 5482538, 67031644, 68000591, 68875490, 73168565, 26063929, 37560164, 34295794, 43322743, 7066775, 47090124, 99297164, 16971929, 28851716, 4069912, 67030811, 33201905, 84187166, 14349098, 46870723, 62693428, 16387575, 83368048, 82339363, 90158785, 4806458, 80251430, 98462867, 26998766, 49882705, 53084256, 91141395, 1936762, 14093520, 95010552, 26397786, 44915235, 6157724, 90061527, 59614746, 67227442, 24058273, 38658347, 62490109, 65880522, 3235882, 75407347, 42644903, 61728685, 60581278, 1022677, 37183543, 89804152, 72793444, 17058722, 7955293, 1239555, 92692283, 92541302, 39847321, 63967300, 33249630, 33699435, 36135, 4091162, 37659250, 92033260, 51466049, 31491938, 99524975, 54606384, 77413012, 6986898, 96709982, 52261574, 22129328, 82779622, 6871053, 40152546, 247198, 73031054, 86821229, 90090964, 92353856, 69697787, 36753250, 526217, 43501211, 92530431, 26664538, 61859581, 91240048, 90310261, 16405341, 9058407, 48673079, 27411561, 59371804, 97561745, 88698958, 8099994, 54762643, 3088684, 66667729, 28796059, 3880712, 79922758, 7229550, 62115552, 3183975, 21919959, 49236559, 4199704, 22997281, 68128438, 94539824, 20681921, 73392814, 73222868, 38061439, 89046466, 15064655, 12571310, 19486173, 13348726, 98648327, 66116458, 8129978, 38341669, 62051033, 33123618, 21289531, 37675718, 39171895, 36685023, 8913721, 13468268, 66322959, 874791, 80014588, 62740044, 15535065, 16099750, 96906410, 5073754, 77694700, 12348118, 29029316, 73109997, 23740167, 18131876, 27625735, 4119747, 79806380, 63015256, 16684829, 50188404, 50668599, 47298834, 1375023, 67474219, 73021291, 62496012, 56424103, 77072625, 33565483, 68939068, 45996863, 89078848, 53632373, 47738185, 79821897, 45995325, 94595668, 99971982, 97641116, 30163921, 61897582, 72278539, 44550764, 11365791, 9398733, 89637706, 99125126, 88904910, 32274392, 45075471, 32161669, 9599614, 92803766, 83210802, 23432750, 4787945, 41092172, 4095116, 26102057, 45667668, 45617087, 30139692, 57241521, 47361209, 21533347, 58208470, 3509435, 25842078, 87160386, 84840549, 58749, 34698428, 65271999, 7104732, 2208785, 96420429, 23569917, 30694952, 75229462, 36580610, 13862149, 47893286, 65454636, 15948937, 77272628, 38008118, 98130363, 65338021, 55770687, 53648154, 80246713, 15902805, 12416768, 54263880, 75135448, 24826575, 78589145, 64157906, 61928316, 52613508, 39801351, 55850790, 29932657, 29959549, 26275734, 66271566, 51149804, 54427233, 55615885, 84002370, 62936963, 35192533, 61815223, 23110625, 6793819, 29635537, 85116755, 42073124, 7646095, 70541760, 4099191, 7011964, 17146629, 38022834, 27041967, 91664334, 85711894, 59910624, 49641577, 90654836, 69355476, 91938191, 7687278, 1204161, 56153345, 18806856, 37435892, 83083131, 86242799, 34497327, 79322415, 74452589, 38256849, 86022504, 87710366, 83269727, 62428472, 99333375, 50567636, 37192445, 48774913, 71657078, 54058788, 10597197, 34432810, 168541, 66832478, 62452034, 19457589, 39553046, 99917599, 38645117, 35996293, 94516935, 33797252, 8634541, 94090109, 83133790, 21001913, 76315420, 90457870, 61623915, 75153252, 3487592, 11923835, 93270084, 36312813, 66045587, 42782652, 60430369, 91802888, 81805959, 9860968, 89419466, 63628376, 37280276, 22200378, 51472275, 6497830, 59405277, 15163258, 64098930, 26734892, 6153406, 70596786, 19898053, 81898046, 44844121, 68110330, 86460488, 64087743, 176257, 79738755, 93790285, 88130087, 22108665, 45990383, 78561158, 47887470, 82979980, 86543538, 7300047, 52293995, 79880247, 84293052, 84904436, 30653863, 65081429, 2331773, 72614359, 36845587, 80555751, 19101477, 91957544, 41380093, 51047803, 112651, 12303248, 66885828, 27187213, 44245960, 47708850, 70367851, 6808825, 18783702, 86118021, 50007421, 25636669, 39373729, 34044787, 33237508, 71333116, 7182642, 55143724, 68316156, 7517032, 20122224, 27665211, 18699206, 86798033, 76960303, 99021067, 68204242, 29401781, 40356877, 94711842, 43152977, 20645197, 64055960, 1431742, 66319530, 824872, 59501487, 99226875, 70420215, 11757872, 77187825, 55669657, 37166608, 31727379, 12664567, 50702367, 17386996, 17385531, 17727650, 40534591, 37891451, 2717150, 74743862, 56515456, 65017137, 36812683, 77300457, 95251277, 71083565, 93566986, 44481640, 10366309, 23848565, 5267545, 19376156, 40677414, 76825057, 72725103, 35512853, 22405120, 85242963, 66137019, 45790169, 49598724, 90272749, 19891772, 17539625, 57961282, 75820087, 96852321, 86001008, 19939935, 3233084, 22450468, 99658235, 28928175, 40865610, 63372756, 45237957, 44664587, 68702632, 95581843, 33895336, 96735716, 57248122, 26114953, 51315460, 38494874, 81274566, 60106217, 59981773, 42947632, 42237907, 93515664, 37957788, 4978543, 18466635, 32250045, 51590803, 97011160, 81677380, 89996536, 34236719, 3633375, 21993752, 22879907, 33061250, 30395570, 20867149, 99861373, 40027975, 37224844, 55753905, 76434144, 9575954, 69255765, 30569392, 63059024, 95395112, 20765474, 61263068, 82052050, 30543215, 76703609, 63506281, 46851987, 44847298, 37739481, 30463802, 32590267, 29188588, 98371444, 48658605, 54868730, 92998591, 81172706, 71522968, 2204165, 71469330, 37501808, 79136082, 56461322, 58751351, 89811711, 68644627, 95957797, 43376279, 31126490, 16424199, 41481685, 81774825, 82651278, 52060076, 40781449, 30811010, 49271185, 19272365, 66428795, 44479073, 23194618, 56484900, 14731700, 66663942, 61982238, 57658654, 89699445, 11581181, 26744917, 67513640, 36396314, 84127901, 57240218, 99515901, 82726008, 76330843, 7502255, 32151165, 96318094, 59318837, 97057187, 67793644, 50806615, 53888755, 54987042, 44983451, 4985896, 14220886, 67124282, 94911072, 44846932, 44627776, 74614639, 68824981, 44842615, 45407418, 76540481, 14947650, 97940276, 22435353, 93053405, 26493119, 97281847, 13173644, 69579137, 13231279, 69641513, 26863229, 38510840, 33304202, 72274002, 43357947, 10961421, 6221471, 48395186, 8055981, 9623492, 54199160, 26776922, 30998561, 55602660, 33628349, 75052463, 8651647, 68816413, 70782102, 57393458, 9829782, 67451935, 15075176, 34667286, 14626618, 14723410, 69848388, 1197320, 7919588, 32699744, 53393358, 78884452, 69605283, 99604946, 24619760, 44177011, 70004753, 70727211, 87598888, 76170907, 30090481, 92071990, 86301513, 31161687, 92867155, 82886381, 77301523, 77312810, 93933709, 16097038, 43933006, 60102965, 3233569, 53666583, 58180653, 42199455, 55189057, 40197395, 42307449, 51507425, 36505482, 61741594, 49597667, 77620120, 75986488, 45428665, 9188443, 92604458, 99690194, 70372191, 22766820, 44889423, 93562257, 36808486, 63152504, 6505939, 51464002, 41442762, 22113133, 71316369, 5970607, 52204879, 74357852, 78766358, 88047921, 57359924, 90004325, 92692978, 71920426, 72238278, 9257405, 24953498, 3294781, 72357096, 9603598, 24168411, 45848907, 83155442, 17081350, 99224392, 21397057, 44060493, 30771409, 34946859, 10453030, 16380211, 82897371, 321665, 72358170, 61141874, 29065964, 91281584, 57020481, 43506672, 54517921, 79191827, 59109400, 13470059, 83651211, 51588015, 99549373, 70800879, 75543508, 80316608, 4434662, 44473167, 8373289, 29510992, 44348328, 17068582, 16445503, 78785507, 66250369, 82427263, 78549759, 20568363, 55470718, 14326617, 36933038, 36189527, 9860195, 11161731, 10309525, 54663246, 20836893, 15015906, 53547802, 45794415, 62232211, 22942635, 30891921, 21673260, 40984766, 3773993, 83789391, 59197747, 77377183, 44784505, 43045786, 10649306, 65275241, 75777973, 23134715, 6111563, 48088883, 4204661, 11543098, 34698463, 26507214, 48892201, 9175338, 18411915, 7685448, 67281495, 59329511, 77787724, 16054533, 97379790, 32058615, 72732205, 54232247, 82532312, 33935899, 29994197, 45919976, 16567550, 83302115, 20002147, 6819644, 97783876, 41092102, 17037369, 8128637, 1250437, 4064751, 5832946, 46540998, 94076128, 669105, 22721500, 48893685, 8696647, 18833224, 61960400, 31904591, 93057697, 60176618, 8115266, 83533741, 79942022, 91727510, 2891150, 26139110, 68041839, 29516592, 81855445, 84684495, 71965942, 90013093, 17894977, 93359396, 88251446, 75128745, 66611759, 88653118, 58224549, 73235980, 79657802, 74137926, 9259676, 98943869, 91831487, 85023028, 83150534, 61271144, 88444207, 26392416, 54014062, 64848072, 48675329, 70036438, 92398073, 53802686, 62430984, 32159704, 30764367, 17857111, 61712234, 65715134, 21070110, 20486294, 8807940, 30787683, 61373987, 5640302, 13819358, 62803858, 569864, 89217461, 94360702, 16019925, 4798568, 415901, 72738685, 57163802, 42692881, 51426311, 55960386, 4508700, 29834512, 96726697, 77898274, 24915585, 65074535, 72373496, 73786944, 95726235, 55247455, 98948034, 20140249, 5822202, 36930650, 62762857, 45381876, 15536795, 65851721, 68694897, 5111370, 56531125, 24733232, 82327024, 69136837, 72019362, 68102437, 508198, 4515343, 1788101, 84406788, 17976208, 27325428, 84166196, 82178706, 1569515, 62552524, 7423788, 83948335, 89214616, 32426752, 74724075, 37620363, 83747892, 41245325, 8791066, 29466406, 2607799, 74110882, 14363867, 72777973, 85571389, 39201414, 40781854, 4668450, 77284619, 39986008, 56955985, 26292919, 40686254, 45306070, 15148031, 17764950, 17957593, 35092039, 47792865, 749283, 20885148, 17365188, 89499542, 85695762, 35456853, 38009615, 31733363, 8729146, 33553959, 42967683, 99729357, 78909561, 18663507, 18504197, 87720882, 8733068, 60393039, 78300864, 30218878, 45049308, 65038678, 49328608, 6038457, 78602717, 28734791, 47213183, 98739783, 98653983, 48260151, 73124510, 24314885, 9886593, 33431960, 95290172, 6525948, 81853704, 96193415 +76540481, 92604458, 61859143, 32161669, 37957788, 83948335, 64055960, 17037369, 17385531, 99224392, 62693428, 62357986, 47792865, 16595436, 30787683, 12348118, 44245960, 97783876, 91281584, 33336148, 38510840, 43357947, 54762643, 88653118, 4515343, 75407347, 65275241, 42199455, 52293995, 34698463, 62740044, 92998591, 47708850, 43376279, 1431742, 44060493, 30163921, 44550764, 93566986, 40677414, 90310261, 26493119, 18001617, 97561745, 19891772, 21533347, 93359396, 72019362, 10961421, 55602660, 26114953, 42237907, 9860195, 62232211, 3773993, 89046466, 83789391, 88130087, 13348726, 33553959, 86543538, 40197395, 8913721, 42307449, 36845587, 62936963, 38365584, 9188443, 41442762, 95957797, 52060076, 57359924, 4119747, 92692978, 27665211, 33935899, 34497327, 74452589, 68939068, 21397057, 30771409, 7502255, 34946859, 10453030, 97057187, 54987042, 44983451, 91255408, 9398733, 56515456, 43506672, 9599614, 92803766, 8115266, 70800879, 35996293, 14947650, 91727510, 33797252, 57241521, 86001008, 3233084, 25842078, 72274002, 22450468, 26998766, 28928175, 9623492, 53842979, 96735716, 82427263, 36580610, 64848072, 15075176, 97011160, 81677380, 62430984, 85695762, 38658347, 30395570, 15064655, 12571310, 67031644, 38341669, 92867155, 29932657, 16097038, 4204661, 3233569, 26275734, 72793444, 76671482, 76703609, 26063929, 46851987, 37739481, 61741594, 77620120, 57163802, 16099750, 44889423, 71469330, 41245325, 23740167, 7066775, 51426311, 4508700, 68644627, 36135, 7517032, 24915585, 28851716, 27625735, 82532312, 18699206, 4069912, 50668599, 39201414, 1375023, 37435892, 24953498, 55247455, 78300864, 83083131, 3294781, 20140249, 87710366, 89699445, 15536795, 89078848, 96709982, 79821897, 66832478, 669105, 6521313, 19457589, 36753250, 45049308, 95251277, 71083565, 92530431, 61859581, 5267545, 83210802, 59109400, 83651211, 41092172, 4095116, 26102057, 2891150, 47361209, 90013093, 84840549, 58749, 28796059, 79657802, 508198, 57248122, 9860968, 60106217, 88444207, 749283, 36189527, 20885148, 53802686, 32250045, 27325428, 34493392, 94539824, 54663246, 22942635, 70004753, 19486173, 5640302, 39801351, 23134715, 6111563, 43933006, 7300047, 39171895, 874791, 79880247, 84293052, 73617245, 4798568, 92541302, 29188588, 89214616, 59177718, 32426752, 63967300, 66885828, 5073754, 71522968, 37620363, 63152504, 6505939, 8791066, 89811711, 71316369, 38022834, 71333116, 74357852, 49641577, 24314885, 65047700, 19272365, 87720882, 68204242, 83302115, 94711842, 12024238, 99524975, 40781854, 98948034, 824872, 62496012, 59501487, 66663942, 96193415, 37166608, 33565483, 45848907, 50702367, 1250437, 22129328, 14349098, 17727650, 46540998, 96318094, 16380211, 61897582, 86821229, 8696647, 94911072, 36812683, 57020481, 24733232, 23848565, 19376156, 38645117, 23432750, 51588015, 17764950, 9058407, 97940276, 59371804, 97281847, 69641513, 88698958, 96852321, 17957593, 71965942, 49882705, 63372756, 6221471, 48395186, 58224549, 73235980, 93270084, 3880712, 3183975, 23569917, 20568363, 38494874, 1936762, 59981773, 36468541, 61271144, 84406788, 10358899, 49236559, 51472275, 70036438, 93515664, 38008118, 14723410, 7919588, 15015906, 21993752, 22879907, 38009615, 54263880, 1569515, 31733363, 76434144, 30090481, 62552524, 52613508, 30569392, 66116458, 63059024, 7423788, 43045786, 98739783, 62803858, 75777973, 33123618, 21289531, 48088883, 66271566, 11543098, 13468268, 54427233, 84904436, 75986488, 39847321, 29635537, 33249630, 77694700, 74724075, 81172706, 70367851, 93562257, 79136082, 6808825, 70541760, 25636669, 17146629, 58751351, 39373729, 34044787, 33237508, 5970607, 55143724, 85711894, 20122224, 90654836, 79806380, 49271185, 7687278, 1204161, 86798033, 72777973, 72373496, 29994197, 74441448, 60955663, 67030811, 99226875, 9603598, 24168411, 83269727, 62428472, 84187166, 37192445, 45996863, 6986898, 40686254, 99515901, 82779622, 32151165, 99971982, 65851721, 73031054, 50806615, 53888755, 2717150, 11365791, 68694897, 67124282, 65017137, 44627776, 16387575, 18833224, 74614639, 32274392, 45075471, 79191827, 31904591, 68824981, 15148031, 36139942, 35512853, 99549373, 4806458, 45407418, 92787493, 48673079, 83533741, 27411561, 66137019, 80316608, 45790169, 45667668, 90272749, 44473167, 8634541, 29510992, 69579137, 58208470, 13231279, 44348328, 83133790, 76315420, 16445503, 40865610, 28550822, 75153252, 34698428, 7104732, 66250369, 26776922, 42782652, 91802888, 95290172, 36933038, 26397786, 30366150, 9829782, 6497830, 44915235, 67451935, 59405277, 6157724, 65454636, 90061527, 51590803, 89996536, 59614746, 98130363, 20681921, 65338021, 45794415, 73392814, 69605283, 20486294, 35456853, 64087743, 33061250, 37224844, 5482538, 45990383, 44784505, 78561158, 92554120, 20765474, 77301523, 37675718, 89804152, 17058722, 30543215, 48260151, 16019925, 80014588, 55615885, 84002370, 72614359, 78909561, 7955293, 19101477, 1239555, 91957544, 6793819, 98371444, 71300104, 69786756, 42073124, 112651, 22766820, 73109997, 36808486, 83747892, 55960386, 4099191, 56473732, 7011964, 33699435, 99297164, 27041967, 2607799, 91664334, 78766358, 7182642, 14045383, 68316156, 71920426, 76960303, 14363867, 29401781, 85571389, 40356877, 16567550, 47298834, 20645197, 67474219, 77284619, 62762857, 54606384, 36780454, 8128637, 77413012, 84127901, 57803235, 83155442, 48774913, 5832946, 71657078, 54058788, 37891451, 247198, 34432810, 72278539, 69697787, 72358170, 44846932, 93057697, 10366309, 60176618, 72725103, 4787945, 20642888, 85242963, 79942022, 4434662, 45617087, 33431960, 94090109, 3509435, 21001913, 27185644, 51715482, 68102437, 88251446, 75128745, 3487592, 11923835, 65271999, 45237957, 36312813, 28787861, 33895336, 96420429, 74137926, 78549759, 75052463, 8651647, 98943869, 66903004, 91831487, 68816413, 89419466, 75229462, 70782102, 26392416, 54014062, 37280276, 28734791, 4978543, 11161731, 15948937, 72495719, 17365188, 34236719, 17857111, 20836893, 81853704, 3633375, 32699744, 53547802, 24058273, 6153406, 89499542, 80246713, 15902805, 12416768, 86460488, 44177011, 61373987, 93790285, 75135448, 24826575, 77377183, 9575954, 69255765, 86301513, 95395112, 61728685, 82886381, 29959549, 1022677, 37183543, 42967683, 94360702, 60102965, 55189057, 82052050, 85117092, 99729357, 51507425, 30653863, 26507214, 80555751, 36505482, 30463802, 48892201, 41380093, 72738685, 48658605, 51047803, 54868730, 99690194, 67281495, 99965001, 18783702, 86118021, 50007421, 59329511, 77787724, 88047921, 81774825, 30811010, 54232247, 69355476, 74110882, 66428795, 92033260, 65074535, 44479073, 56153345, 51466049, 18806856, 8733068, 73021291, 61982238, 56424103, 70420215, 77072625, 57658654, 11757872, 77187825, 38256849, 33201905, 99333375, 39986008, 56955985, 57240218, 17386996, 52261574, 76330843, 47738185, 59318837, 22721500, 74743862, 45306070, 62452034, 99125126, 88904910, 77300457, 54517921, 99917599, 82339363, 13470059, 26139110, 75543508, 93053405, 49598724, 8373289, 78218845, 57961282, 98462867, 33304202, 35092039, 90457870, 78785507, 8055981, 66667729, 54199160, 7229550, 30694952, 33628349, 51315460, 42947632, 14093520, 95010552, 1788101, 67084351, 47893286, 4199704, 14626618, 15163258, 32159704, 26734892, 65715134, 70596786, 55770687, 21070110, 73222868, 21673260, 8807940, 40984766, 68110330, 99604946, 24619760, 79738755, 48360198, 65880522, 68000591, 61928316, 3235882, 92071990, 8129978, 10649306, 62051033, 60581278, 77312810, 82979980, 36685023, 63506281, 61815223, 415901, 32590267, 92692283, 15535065, 45428665, 85116755, 18411915, 12303248, 18663507, 51464002, 29466406, 29834512, 41481685, 77898274, 82651278, 63015256, 23194618, 72238278, 45919976, 60393039, 56484900, 14731700, 6819644, 66319530, 5822202, 36930650, 55669657, 41092102, 26744917, 36396314, 82726008, 46870723, 29819940, 45995325, 40152546, 94595668, 67793644, 168541, 4985896, 82897371, 48893685, 2917920, 89637706, 9886593, 61141874, 39553046, 83368048, 30139692, 13173644, 81855445, 92215320, 19939935, 87160386, 99658235, 61623915, 91141395, 44664587, 68702632, 66045587, 2208785, 79922758, 81805959, 85023028, 55470718, 21919959, 91990218, 57393458, 22200378, 77272628, 68128438, 64098930, 69848388, 24591705, 53393358, 19898053, 82178706, 30891921, 38061439, 70727211, 99861373, 40027975, 76170907, 78589145, 98648327, 68875490, 42644903, 73168565, 17016156, 61263068, 569864, 53666583, 98653983, 58180653, 51149804, 66322959, 35192533, 49597667, 73124510, 37560164, 23110625, 34295794, 9175338, 70372191, 43322743, 27187213, 29029316, 7646095, 56461322, 16971929, 18131876, 16054533, 97379790, 16424199, 72732205, 37659250, 16684829, 50188404, 99021067, 73786944, 4668450, 20002147, 72357096, 79322415, 31727379, 67513640, 17081350, 4064751, 56531125, 29065964, 526217, 43501211, 91240048, 76825057, 22405120, 22435353, 68041839, 29516592, 80251430, 17539625, 26863229, 53084256, 30998561, 60430369, 62115552, 9259676, 14326617, 83150534, 63628376, 30764367, 6525948, 20867149, 59197747, 22108665, 47213183, 13819358, 31161687, 47887470, 89217461, 93933709, 2331773, 2204165, 37501808, 18504197, 47090124, 96726697, 59910624, 32058615, 90004325, 40781449, 9257405, 95726235, 31491938, 43152977, 30218878, 50567636, 12664567, 26292919, 53632373, 40534591, 94076128, 97641116, 90090964, 321665, 65038678, 44842615, 84349107, 90158785, 16405341, 84684495, 17068582, 8099994, 66611759, 6038457, 78602717, 13862149, 48675329, 17976208, 92398073, 10309525, 84166196, 61712234, 67227442, 78884452, 53648154, 81898046, 44844121, 62490109, 176257, 64602895, 55850790, 44847298, 7685448, 42692881, 52204879, 91938191, 86242799, 45381876, 6871053, 10597197, 92353856, 5111370, 26664538, 82327024, 69136837, 94516935, 49328608, 18466635, 22997281, 1197320, 87598888, 55753905, 64157906, 65081429, 96906410, 22113133, 31126490, 10264691, 11581181, 14220886, 44481640, 75820087, 17894977, 95581843, 81274566, 8729146, 4091162, 86022504, 61960400, 3088684, 34667286 +30998561, 27665211, 49271185, 2717150, 78785507, 84166196, 99971982, 82339363, 10309525, 62552524, 20122224, 73786944, 22721500, 68694897, 49598724, 75820087, 66250369, 54199160, 26392416, 30764367, 54663246, 81898046, 77377183, 66116458, 17016156, 89804152, 84002370, 1239555, 49597667, 18504197, 52204879, 97379790, 99125126, 65038678, 36139942, 76825057, 4787945, 33431960, 62357986, 66045587, 13862149, 17976208, 9860195, 77272628, 64098930, 6525948, 64602895, 5482538, 78589145, 20765474, 75777973, 37183543, 16097038, 94360702, 4204661, 55615885, 39847321, 43322743, 2204165, 56461322, 22113133, 91664334, 16054533, 30811010, 72732205, 18699206, 77284619, 5822202, 62762857, 33565483, 84187166, 56955985, 12664567, 30771409, 96318094, 16380211, 94595668, 2917920, 45306070, 29065964, 59109400, 83651211, 4434662, 13173644, 16445503, 88251446, 58224549, 45237957, 66667729, 36312813, 79657802, 62115552, 66903004, 68816413, 55470718, 14326617, 84406788, 27325428, 22997281, 14626618, 89996536, 69848388, 7919588, 65338021, 45794415, 20486294, 1569515, 55753905, 19486173, 68000591, 52613508, 47213183, 62051033, 61728685, 47887470, 61263068, 43933006, 7300047, 26275734, 42199455, 66271566, 11543098, 80014588, 45428665, 9175338, 29029316, 18783702, 71333116, 36135, 24915585, 40781449, 24314885, 92033260, 76960303, 14363867, 8733068, 83083131, 86242799, 50702367, 17081350, 82726008, 5832946, 54058788, 32151165, 66832478, 72278539, 74743862, 69697787, 60176618, 99549373, 70800879, 94516935, 45617087, 19891772, 47361209, 81855445, 35092039, 43357947, 84840549, 28928175, 66611759, 49328608, 4515343, 60430369, 74137926, 23569917, 75052463, 85023028, 60106217, 47792865, 75229462, 26397786, 42237907, 28734791, 90061527, 61712234, 24058273, 55770687, 44844121, 86460488, 22108665, 45990383, 9575954, 63059024, 68875490, 42644903, 31161687, 569864, 48088883, 58180653, 85117092, 99729357, 48260151, 30653863, 62740044, 7685448, 89214616, 92604458, 69786756, 42073124, 12348118, 71522968, 37501808, 47090124, 71316369, 59329511, 16971929, 27041967, 18131876, 78766358, 55143724, 49641577, 77898274, 33935899, 50668599, 72238278, 9257405, 10264691, 16567550, 1431742, 67474219, 20140249, 70420215, 38256849, 24168411, 50567636, 89078848, 99515901, 40152546, 97641116, 30163921, 67124282, 88904910, 45049308, 18833224, 43501211, 54517921, 31904591, 93566986, 93057697, 40677414, 51588015, 4806458, 92787493, 27411561, 29516592, 3509435, 26863229, 90013093, 10961421, 48395186, 44664587, 3088684, 55602660, 83150534, 36580610, 22200378, 36189527, 67451935, 38008118, 1197320, 53393358, 8807940, 40984766, 20867149, 87598888, 31733363, 79738755, 93790285, 76170907, 12571310, 75135448, 88130087, 76434144, 67031644, 98648327, 92071990, 69255765, 5640302, 39801351, 33553959, 60102965, 53666583, 34698463, 63506281, 79880247, 26507214, 37739481, 35192533, 32590267, 37560164, 15535065, 75986488, 85116755, 96906410, 70372191, 59177718, 70367851, 18663507, 7646095, 79136082, 25636669, 41442762, 43376279, 14045383, 16424199, 32058615, 27625735, 91938191, 99021067, 40356877, 18806856, 47298834, 31491938, 60955663, 40781854, 67030811, 73021291, 72357096, 824872, 34497327, 77187825, 41092102, 86022504, 36780454, 33201905, 89699445, 45381876, 15536795, 6986898, 17386996, 46870723, 4064751, 47738185, 10453030, 247198, 67793644, 168541, 669105, 44983451, 4985896, 56515456, 5111370, 56531125, 89637706, 94911072, 526217, 74614639, 39553046, 99917599, 9599614, 10366309, 5267545, 19376156, 69136837, 17764950, 22405120, 16405341, 4095116, 91727510, 22435353, 8373289, 44348328, 84684495, 96852321, 72274002, 76315420, 51715482, 49882705, 61623915, 75153252, 73235980, 96735716, 81805959, 9259676, 20568363, 91831487, 89419466, 95290172, 30366150, 63628376, 37280276, 6157724, 20885148, 53802686, 15948937, 20836893, 65715134, 53547802, 62490109, 61373987, 70727211, 13348726, 30090481, 8129978, 7423788, 65275241, 73168565, 82886381, 33123618, 89217461, 93933709, 3233569, 17058722, 30543215, 51149804, 8913721, 51507425, 84904436, 2331773, 44847298, 36845587, 61815223, 61741594, 16099750, 51047803, 54868730, 12303248, 42692881, 77694700, 37620363, 93562257, 73109997, 83747892, 67281495, 99965001, 86118021, 50007421, 56473732, 58751351, 39373729, 38022834, 29466406, 2607799, 7182642, 59910624, 81774825, 7517032, 69355476, 74110882, 82532312, 79806380, 65074535, 63015256, 68204242, 95726235, 83302115, 60393039, 43152977, 14731700, 20002147, 99226875, 66663942, 61982238, 54606384, 37166608, 99333375, 17037369, 26744917, 67513640, 37192445, 83155442, 17385531, 52261574, 48774913, 21397057, 71657078, 10597197, 86821229, 6521313, 90090964, 9886593, 16387575, 61960400, 44481640, 15148031, 82327024, 83210802, 38645117, 8115266, 35512853, 9058407, 26102057, 97940276, 80316608, 33336148, 80251430, 17539625, 21533347, 13231279, 17068582, 8099994, 22450468, 87160386, 99658235, 75128745, 3487592, 28787861, 42782652, 79922758, 6038457, 59981773, 21919959, 88444207, 54014062, 47893286, 48675329, 59614746, 34236719, 98130363, 26734892, 67227442, 20681921, 16595436, 3633375, 6153406, 78884452, 85695762, 22879907, 62232211, 30891921, 15902805, 68110330, 12416768, 64087743, 30395570, 54263880, 70004753, 176257, 99861373, 59197747, 64157906, 78561158, 43045786, 10649306, 92867155, 55850790, 82979980, 23134715, 86543538, 6111563, 39171895, 72793444, 98653983, 82052050, 40197395, 36685023, 54427233, 65081429, 46851987, 80555751, 78909561, 19101477, 415901, 91957544, 92692283, 57163802, 18411915, 63967300, 22766820, 5073754, 81172706, 51464002, 41245325, 23740167, 7066775, 55960386, 4099191, 4508700, 29834512, 96726697, 31126490, 88047921, 68316156, 4091162, 90654836, 7687278, 1204161, 19272365, 66428795, 23194618, 29401781, 72777973, 51466049, 56484900, 37435892, 55247455, 77072625, 57658654, 9603598, 11757872, 36930650, 79322415, 87710366, 68939068, 36396314, 53632373, 57240218, 57803235, 1250437, 76330843, 14349098, 34946859, 59318837, 6871053, 94076128, 97057187, 65851721, 44627776, 36753250, 77300457, 95251277, 71083565, 45075471, 79191827, 61859581, 23848565, 84349107, 94090109, 78218845, 29510992, 69641513, 88698958, 3233084, 27185644, 17894977, 90457870, 40865610, 54762643, 88653118, 3880712, 26776922, 508198, 57248122, 7229550, 78602717, 78549759, 30694952, 8651647, 1936762, 81274566, 61271144, 36933038, 91990218, 57393458, 44915235, 59405277, 92398073, 32250045, 51590803, 72495719, 15163258, 32159704, 17857111, 15015906, 19898053, 73392814, 38658347, 82178706, 21673260, 38009615, 38061439, 33061250, 37224844, 8729146, 24826575, 61928316, 44784505, 86301513, 13819358, 29932657, 66322959, 73617245, 83948335, 73124510, 23110625, 92541302, 34295794, 29635537, 48658605, 71300104, 92998591, 33249630, 66885828, 47708850, 6505939, 70541760, 33699435, 17146629, 99297164, 34044787, 89811711, 68644627, 85711894, 82651278, 61859143, 90004325, 28851716, 54232247, 71920426, 86798033, 44479073, 16684829, 72373496, 85571389, 29994197, 12024238, 4668450, 97783876, 74452589, 55669657, 11581181, 39986008, 45996863, 26292919, 77413012, 40686254, 44060493, 46540998, 79821897, 73031054, 50806615, 14220886, 9398733, 62693428, 72358170, 61141874, 36812683, 43506672, 32274392, 24733232, 26664538, 92803766, 90158785, 23432750, 85242963, 35996293, 14947650, 2891150, 75543508, 33797252, 26493119, 90272749, 30139692, 44473167, 57241521, 69579137, 58208470, 92215320, 86001008, 19939935, 25842078, 93359396, 34698428, 6221471, 68702632, 33895336, 2208785, 3183975, 36468541, 67084351, 10358899, 49236559, 64848072, 749283, 37957788, 4978543, 65454636, 97011160, 68128438, 14723410, 81853704, 32699744, 89499542, 53648154, 99604946, 3773993, 30787683, 40027975, 15064655, 65880522, 92554120, 62803858, 60581278, 21289531, 55189057, 52293995, 42307449, 16019925, 13468268, 72614359, 62936963, 30463802, 48892201, 29188588, 6793819, 74724075, 71469330, 36808486, 51426311, 7011964, 33237508, 5970607, 77787724, 74357852, 41481685, 37659250, 65047700, 4069912, 50188404, 1375023, 99524975, 74441448, 78300864, 66319530, 62496012, 8128637, 84127901, 99224392, 45995325, 37891451, 34432810, 61897582, 54987042, 44550764, 92353856, 48893685, 44846932, 19457589, 57020481, 83368048, 92530431, 32161669, 91240048, 45407418, 20642888, 48673079, 76540481, 83533741, 26139110, 68041839, 45667668, 18001617, 71965942, 33304202, 11923835, 91141395, 7104732, 93270084, 95581843, 82427263, 33628349, 98943869, 42947632, 95010552, 1788101, 4199704, 6497830, 93515664, 34667286, 24591705, 70596786, 35456853, 22942635, 89046466, 83789391, 95395112, 38341669, 77301523, 77312810, 42967683, 76671482, 76703609, 874791, 36505482, 7955293, 38365584, 41380093, 77620120, 9188443, 98371444, 32426752, 44889423, 63152504, 8791066, 57359924, 4119747, 92692978, 56153345, 87720882, 45919976, 24953498, 20645197, 3294781, 59501487, 56424103, 83269727, 31727379, 45848907, 40534591, 53888755, 11365791, 82897371, 321665, 65017137, 91281584, 44842615, 90310261, 72725103, 41092172, 79942022, 66137019, 59371804, 97281847, 38510840, 72019362, 26998766, 68102437, 28550822, 53084256, 53842979, 96420429, 91802888, 70782102, 15075176, 18466635, 17365188, 81677380, 34493392, 62430984, 21070110, 24619760, 98739783, 4798568, 112651, 27187213, 44245960, 52060076, 64055960, 6819644, 98948034, 30218878, 17727650, 82779622, 29819940, 91255408, 8696647, 62452034, 13470059, 97561745, 98462867, 17957593, 21001913, 63372756, 65271999, 8055981, 28796059, 26114953, 51315460, 38494874, 9860968, 9829782, 94539824, 69605283, 80246713, 44177011, 48360198, 75407347, 30569392, 37675718, 26063929, 84293052, 99690194, 6808825, 95957797, 39201414, 62428472, 22129328, 68824981, 93053405, 83133790, 9623492, 70036438, 11161731, 29959549, 72738685, 96193415, 96709982, 8634541, 57961282, 58749, 3235882, 1022677, 94711842, 7502255, 45790169, 14093520, 21993752, 73222868, 51472275 +168541, 83789391, 86242799, 1197320, 24058273, 55189057, 29466406, 66319530, 92353856, 43501211, 90310261, 19891772, 88444207, 84166196, 55770687, 48360198, 38341669, 60581278, 47887470, 84293052, 29635537, 91938191, 6819644, 96709982, 36753250, 84349107, 48673079, 22435353, 71965942, 90457870, 7229550, 81805959, 14093520, 84406788, 20486294, 62232211, 73168565, 60102965, 89804152, 76703609, 1239555, 91957544, 51047803, 29029316, 52204879, 43376279, 52060076, 4119747, 69355476, 63015256, 33201905, 17037369, 67793644, 30163921, 669105, 99125126, 32274392, 19376156, 4806458, 92787493, 79942022, 49598724, 30139692, 78218845, 93359396, 91802888, 30694952, 98943869, 75229462, 57393458, 9829782, 22942635, 12416768, 38061439, 54263880, 44177011, 93790285, 8729146, 24826575, 6111563, 13468268, 44847298, 62936963, 61741594, 48892201, 18411915, 71300104, 34044787, 4091162, 72732205, 7687278, 50188404, 1375023, 99524975, 40781854, 73021291, 37192445, 8128637, 46870723, 97057187, 45049308, 18833224, 79191827, 8115266, 72725103, 16405341, 94090109, 75820087, 26863229, 83133790, 76315420, 8099994, 66611759, 58224549, 66667729, 30998561, 96735716, 20568363, 66903004, 95010552, 67451935, 65454636, 11161731, 15948937, 51590803, 20681921, 176257, 40027975, 64157906, 52613508, 98739783, 62051033, 75777973, 23134715, 4204661, 53666583, 17058722, 66271566, 55615885, 2331773, 72614359, 38365584, 70372191, 47090124, 71316369, 68644627, 31126490, 61859143, 37659250, 4069912, 39201414, 94711842, 66663942, 79322415, 50567636, 45381876, 26292919, 53632373, 17386996, 76330843, 17727650, 48774913, 5832946, 40534591, 6871053, 73031054, 72278539, 2717150, 56531125, 321665, 94911072, 61960400, 91240048, 90158785, 35512853, 99549373, 9058407, 29516592, 57241521, 92215320, 33304202, 17068582, 7104732, 36312813, 3880712, 66045587, 2208785, 4515343, 38494874, 1936762, 91831487, 81274566, 68816413, 26397786, 4199704, 28734791, 10309525, 18466635, 72495719, 81677380, 59614746, 61712234, 78884452, 30891921, 99604946, 89046466, 19486173, 64602895, 5482538, 76434144, 13348726, 22108665, 47213183, 30569392, 13819358, 92554120, 92867155, 55850790, 82886381, 77301523, 82052050, 48260151, 51507425, 80014588, 80555751, 35192533, 30463802, 37560164, 34295794, 29188588, 96906410, 22766820, 5073754, 74724075, 44245960, 2204165, 23740167, 50007421, 56473732, 59329511, 38022834, 16971929, 27041967, 91664334, 74357852, 78766358, 85711894, 41481685, 68316156, 30811010, 27625735, 33935899, 19272365, 65074535, 50668599, 60393039, 64055960, 4668450, 20002147, 30218878, 56424103, 11581181, 67513640, 57803235, 17081350, 29819940, 79821897, 34432810, 66832478, 74743862, 11365791, 69697787, 65017137, 44846932, 44627776, 36812683, 526217, 43506672, 39553046, 92530431, 31904591, 36139942, 59109400, 60176618, 22405120, 26102057, 85242963, 80316608, 80251430, 17539625, 21533347, 34698428, 66250369, 508198, 6038457, 23569917, 60106217, 59981773, 70782102, 37280276, 44915235, 17976208, 92398073, 17365188, 62430984, 32159704, 89996536, 7919588, 24591705, 67227442, 3633375, 70596786, 53648154, 81898046, 62490109, 40984766, 38009615, 33061250, 70727211, 65880522, 67031644, 98648327, 68000591, 77377183, 61928316, 78561158, 92071990, 39801351, 20765474, 29959549, 94360702, 48088883, 72793444, 98653983, 85117092, 66322959, 26507214, 36505482, 73124510, 92541302, 15535065, 98371444, 48658605, 54868730, 43322743, 42692881, 77694700, 12348118, 51464002, 83747892, 6808825, 55960386, 99965001, 7011964, 33699435, 29834512, 18131876, 90004325, 90654836, 82532312, 76960303, 16684829, 68204242, 45919976, 51466049, 83302115, 12024238, 31491938, 24953498, 20645197, 67474219, 3294781, 72357096, 824872, 41092102, 45848907, 36396314, 57240218, 83155442, 14349098, 96318094, 59318837, 247198, 97641116, 14220886, 90090964, 82897371, 68694897, 9886593, 82339363, 44481640, 32161669, 82327024, 44842615, 23848565, 5267545, 83210802, 45407418, 94516935, 93053405, 68041839, 18001617, 58208470, 57961282, 84684495, 25842078, 27185644, 72274002, 28550822, 53084256, 3487592, 11923835, 6221471, 73235980, 9623492, 28796059, 54199160, 49328608, 96420429, 60430369, 62115552, 89419466, 47792865, 36933038, 37957788, 34667286, 14626618, 15163258, 14723410, 98130363, 65715134, 32699744, 65338021, 53393358, 89499542, 85695762, 38658347, 44844121, 80246713, 3773993, 30395570, 24619760, 30787683, 37224844, 55753905, 88130087, 78589145, 9575954, 3235882, 86301513, 63059024, 7423788, 43045786, 68875490, 31161687, 1022677, 569864, 33553959, 16097038, 39171895, 76671482, 51149804, 52293995, 874791, 84002370, 61815223, 23110625, 75986488, 39847321, 6793819, 9175338, 89214616, 99690194, 42073124, 63967300, 66885828, 27187213, 81172706, 7646095, 73109997, 63152504, 79136082, 67281495, 18783702, 41442762, 58751351, 22113133, 5970607, 59910624, 49641577, 7517032, 77898274, 57359924, 92692978, 56153345, 87720882, 29401781, 72777973, 8733068, 47298834, 43152977, 83083131, 14731700, 67030811, 59501487, 5822202, 61982238, 57658654, 9603598, 83269727, 31727379, 56955985, 1250437, 82726008, 47738185, 44060493, 7502255, 46540998, 32151165, 94595668, 65851721, 50806615, 44983451, 2917920, 72358170, 19457589, 71083565, 15148031, 9599614, 10366309, 38645117, 76825057, 13470059, 23432750, 83651211, 35996293, 14947650, 91727510, 75543508, 4434662, 33797252, 97281847, 47361209, 96852321, 19939935, 21001913, 90013093, 17894977, 35092039, 43357947, 87160386, 72019362, 99658235, 75153252, 91141395, 63372756, 65271999, 8055981, 93270084, 95581843, 53842979, 57248122, 26114953, 33628349, 51315460, 9860968, 42947632, 21919959, 95290172, 1788101, 67084351, 51472275, 59405277, 20885148, 22997281, 94539824, 6525948, 69848388, 17857111, 16595436, 45794415, 73392814, 21993752, 21070110, 70004753, 87598888, 31733363, 8129978, 10649306, 95395112, 42644903, 61728685, 21289531, 17016156, 61263068, 82979980, 37183543, 93933709, 43933006, 58180653, 16019925, 73617245, 84904436, 46851987, 78909561, 37739481, 32426752, 92998591, 33249630, 70367851, 93562257, 36808486, 70541760, 7066775, 25636669, 17146629, 71333116, 2607799, 24915585, 74110882, 79806380, 49271185, 65047700, 86798033, 92033260, 29994197, 18806856, 37435892, 20140249, 70420215, 36930650, 74452589, 77187825, 87710366, 99333375, 68939068, 39986008, 89078848, 84127901, 6986898, 17385531, 22129328, 21397057, 37891451, 94076128, 10597197, 91255408, 6521313, 8696647, 62693428, 88904910, 16387575, 77300457, 45075471, 93566986, 24733232, 26664538, 69136837, 92803766, 4787945, 51588015, 26139110, 59371804, 33336148, 45617087, 44473167, 33431960, 69579137, 81855445, 16445503, 84840549, 26998766, 78785507, 68102437, 75128745, 48395186, 62357986, 45237957, 44664587, 68702632, 28787861, 79657802, 42782652, 82427263, 9259676, 75052463, 85023028, 83150534, 36468541, 26392416, 42237907, 91990218, 63628376, 6497830, 15075176, 9860195, 32250045, 27325428, 90061527, 97011160, 34493392, 34236719, 38008118, 64098930, 20836893, 15015906, 69605283, 82178706, 21673260, 68110330, 61373987, 20867149, 79738755, 12571310, 62552524, 45990383, 44784505, 75407347, 62803858, 42967683, 86543538, 7300047, 3233569, 26275734, 54427233, 79880247, 65081429, 19101477, 83948335, 92692283, 72738685, 9188443, 16099750, 7685448, 59177718, 112651, 44889423, 18504197, 51426311, 33237508, 88047921, 16424199, 20122224, 40781449, 71920426, 66428795, 23194618, 85571389, 73786944, 40356877, 10264691, 95726235, 74441448, 77284619, 98948034, 62496012, 99226875, 34497327, 77072625, 11757872, 55669657, 86022504, 33565483, 45996863, 12664567, 40686254, 99515901, 52261574, 82779622, 10453030, 40152546, 61897582, 54987042, 86821229, 4985896, 44550764, 45306070, 67124282, 56515456, 89637706, 61141874, 29065964, 91281584, 57020481, 65038678, 54517921, 99917599, 68824981, 61859581, 40677414, 20642888, 4095116, 70800879, 83533741, 97940276, 26493119, 90272749, 8373289, 8634541, 86001008, 3233084, 38510840, 58749, 10961421, 3088684, 26776922, 79922758, 55602660, 78602717, 55470718, 61271144, 13862149, 10358899, 47893286, 749283, 36189527, 4978543, 77272628, 54663246, 26734892, 81853704, 53547802, 6153406, 19898053, 22879907, 15902805, 86460488, 15064655, 76170907, 75135448, 65275241, 29932657, 77312810, 89217461, 42199455, 11543098, 8913721, 34698463, 26063929, 63506281, 30653863, 62740044, 4798568, 49597667, 92604458, 12303248, 71522968, 71469330, 37620363, 41245325, 8791066, 89811711, 77787724, 96726697, 14045383, 97379790, 32058615, 81774825, 82651278, 54232247, 27665211, 24314885, 18699206, 14363867, 99021067, 72238278, 55247455, 1431742, 96193415, 54606384, 24168411, 62428472, 84187166, 99224392, 4064751, 71657078, 34946859, 54058788, 45995325, 16380211, 22721500, 48893685, 9398733, 5111370, 83368048, 93057697, 41092172, 2891150, 45790169, 45667668, 3509435, 44348328, 98462867, 17957593, 22450468, 88251446, 33895336, 74137926, 14326617, 30366150, 64848072, 53802686, 68128438, 30764367, 73222868, 64087743, 30090481, 69255765, 66116458, 5640302, 33123618, 37675718, 36685023, 42307449, 36845587, 7955293, 41380093, 77620120, 85116755, 47708850, 18663507, 6505939, 99297164, 39373729, 55143724, 1204161, 9257405, 16567550, 78300864, 89699445, 15536795, 50702367, 95251277, 17764950, 27411561, 66137019, 69641513, 51715482, 28928175, 40865610, 3183975, 8651647, 93515664, 6157724, 8807940, 99861373, 59197747, 40197395, 99729357, 415901, 32590267, 57163802, 37501808, 86118021, 4099191, 95957797, 16054533, 28851716, 44479073, 72373496, 56484900, 60955663, 62762857, 97783876, 38256849, 36780454, 77413012, 53888755, 74614639, 97561745, 13173644, 29510992, 49882705, 61623915, 54762643, 78549759, 36580610, 54014062, 22200378, 48675329, 70036438, 35456853, 1569515, 30543215, 69786756, 56461322, 36135, 30771409, 76540481, 88698958, 49236559, 45428665, 4508700, 26744917, 99971982, 13231279, 88653118, 37166608, 7182642, 62452034 +17764950, 4515343, 11923835, 77787724, 4064751, 75543508, 76315420, 96420429, 24058273, 61373987, 37224844, 65275241, 75777973, 60581278, 75986488, 27187213, 20140249, 321665, 69641513, 43357947, 93270084, 47792865, 6497830, 20885148, 53547802, 76434144, 47213183, 98739783, 77312810, 4204661, 76703609, 32590267, 72738685, 4099191, 25636669, 8791066, 38022834, 74357852, 96726697, 14363867, 20002147, 57658654, 17727650, 168541, 67124282, 29065964, 18833224, 61960400, 39553046, 79191827, 10366309, 92803766, 72725103, 80316608, 71965942, 16445503, 84840549, 99658235, 3088684, 20568363, 8651647, 1788101, 67084351, 44915235, 67451935, 59405277, 10309525, 81853704, 65715134, 89499542, 53648154, 68110330, 89046466, 78561158, 43045786, 20765474, 21289531, 6111563, 40197395, 34698463, 874791, 84904436, 19101477, 96906410, 99690194, 70372191, 42073124, 66885828, 77898274, 74110882, 99021067, 39201414, 24953498, 74441448, 4668450, 26744917, 45996863, 8128637, 34946859, 79821897, 97057187, 82897371, 92353856, 99917599, 15148031, 26664538, 93057697, 90158785, 51588015, 70800879, 4434662, 29516592, 78218845, 58208470, 84684495, 38510840, 58224549, 9623492, 66667729, 28796059, 85023028, 89419466, 14093520, 75229462, 36468541, 30366150, 84406788, 37280276, 27325428, 14626618, 15163258, 34236719, 54663246, 1197320, 15015906, 16595436, 70596786, 44844121, 1569515, 99861373, 8729146, 44784505, 37675718, 58180653, 82052050, 30543215, 36685023, 35192533, 29635537, 57163802, 98371444, 92604458, 18663507, 7646095, 67281495, 58751351, 34044787, 22113133, 95957797, 2607799, 31126490, 14045383, 7517032, 49271185, 1204161, 65047700, 66428795, 65074535, 16684829, 72777973, 40356877, 45919976, 1375023, 40781854, 66319530, 67474219, 30218878, 824872, 61982238, 96193415, 62762857, 86022504, 11581181, 15536795, 46870723, 47738185, 7502255, 40534591, 96318094, 50806615, 30163921, 53888755, 54987042, 44983451, 22721500, 48893685, 56531125, 9886593, 45049308, 83368048, 31904591, 82327024, 19376156, 90310261, 59109400, 4787945, 45407418, 41092172, 83533741, 35996293, 59371804, 30139692, 8634541, 33431960, 3509435, 26863229, 8099994, 49882705, 40865610, 91141395, 34698428, 6221471, 88653118, 8055981, 68702632, 91802888, 23569917, 9829782, 4199704, 4978543, 15075176, 92398073, 62430984, 14723410, 55770687, 21070110, 30891921, 80246713, 33061250, 70004753, 30090481, 92071990, 82886381, 42967683, 98653983, 78909561, 30463802, 61741594, 415901, 77620120, 85116755, 9188443, 89214616, 54868730, 32426752, 71522968, 2204165, 93562257, 18504197, 5970607, 29466406, 18131876, 78766358, 16424199, 82651278, 29994197, 83302115, 77284619, 3294781, 59501487, 34497327, 9603598, 11757872, 36930650, 79322415, 24168411, 12664567, 26292919, 36396314, 84127901, 57240218, 40686254, 17081350, 17385531, 76330843, 21397057, 5832946, 30771409, 16380211, 10597197, 34432810, 97641116, 66832478, 91255408, 11365791, 44846932, 36812683, 19457589, 65038678, 32274392, 44481640, 24733232, 38645117, 26493119, 97281847, 44473167, 29510992, 69579137, 44348328, 72274002, 17068582, 90457870, 26998766, 68102437, 28550822, 75128745, 65271999, 44664587, 26776922, 82427263, 2208785, 79922758, 3183975, 74137926, 26114953, 1936762, 91831487, 59981773, 95010552, 42237907, 91990218, 57393458, 70036438, 94539824, 45794415, 69605283, 62490109, 8807940, 12416768, 176257, 20867149, 70727211, 31733363, 76170907, 19486173, 24826575, 65880522, 88130087, 98648327, 68000591, 3235882, 52613508, 7423788, 31161687, 73168565, 569864, 33553959, 93933709, 86543538, 60102965, 53666583, 66271566, 26063929, 2331773, 84002370, 4798568, 72614359, 62936963, 48892201, 1239555, 91957544, 73124510, 45428665, 18411915, 71300104, 69786756, 33249630, 29029316, 63152504, 70541760, 86118021, 33699435, 41442762, 68644627, 29834512, 81774825, 52060076, 24915585, 30811010, 27625735, 91938191, 76960303, 56153345, 87720882, 68204242, 8733068, 94711842, 56484900, 55247455, 78300864, 83083131, 66663942, 56424103, 83269727, 33565483, 50567636, 39986008, 50702367, 96709982, 52261574, 82726008, 46540998, 37891451, 67793644, 14220886, 2717150, 2917920, 89637706, 61141874, 526217, 43506672, 71083565, 68824981, 84349107, 83210802, 60176618, 99549373, 22405120, 4095116, 27411561, 94516935, 2891150, 90272749, 8373289, 80251430, 13173644, 47361209, 92215320, 13231279, 17957593, 83133790, 88251446, 61623915, 66611759, 36312813, 95581843, 54199160, 96735716, 60430369, 6038457, 78602717, 30694952, 51315460, 83150534, 63628376, 22200378, 749283, 93515664, 28734791, 6157724, 18466635, 34667286, 32250045, 51590803, 17365188, 34493392, 84166196, 6525948, 26734892, 61712234, 20681921, 19898053, 21993752, 21673260, 15902805, 93790285, 55753905, 9575954, 8129978, 63059024, 10649306, 42644903, 38341669, 55850790, 33123618, 82979980, 94360702, 11543098, 85117092, 66322959, 80014588, 84293052, 55615885, 41380093, 92541302, 29188588, 6793819, 48658605, 22766820, 74724075, 44889423, 37620363, 73109997, 18783702, 56461322, 39373729, 52204879, 97379790, 49641577, 41481685, 57359924, 90654836, 69355476, 27665211, 79806380, 7687278, 19272365, 50668599, 72238278, 72373496, 85571389, 47298834, 99524975, 37435892, 64055960, 14731700, 77187825, 38256849, 54606384, 99333375, 17037369, 56955985, 37192445, 44060493, 71657078, 54058788, 59318837, 65851721, 669105, 6521313, 90090964, 74743862, 69697787, 62693428, 94911072, 62452034, 99125126, 44627776, 57020481, 16405341, 79942022, 66137019, 91727510, 93053405, 68041839, 45667668, 97561745, 94090109, 17539625, 98462867, 96852321, 90013093, 17894977, 78785507, 58749, 63372756, 54762643, 62357986, 66250369, 33895336, 508198, 75052463, 38494874, 98943869, 66903004, 68816413, 55470718, 21919959, 95290172, 10358899, 51472275, 90061527, 77272628, 89996536, 38008118, 69848388, 98130363, 32699744, 6153406, 73392814, 73222868, 62232211, 22942635, 40984766, 86460488, 99604946, 24619760, 83789391, 79738755, 40027975, 12571310, 48360198, 64157906, 13348726, 62552524, 75407347, 86301513, 13819358, 92554120, 61263068, 39171895, 3233569, 26275734, 42199455, 55189057, 76671482, 52293995, 8913721, 48260151, 13468268, 54427233, 79880247, 30653863, 65081429, 26507214, 36505482, 37739481, 7955293, 83948335, 92692283, 92998591, 77694700, 44245960, 6505939, 6808825, 41245325, 23740167, 99965001, 99297164, 33237508, 89811711, 59329511, 43376279, 16054533, 68316156, 61859143, 20122224, 71920426, 24314885, 18699206, 33935899, 73786944, 10264691, 16567550, 18806856, 97783876, 74452589, 41092102, 37166608, 33201905, 89699445, 31727379, 89078848, 99515901, 22129328, 99224392, 82779622, 29819940, 32151165, 99971982, 73031054, 61897582, 86821229, 5111370, 65017137, 16387575, 45075471, 9599614, 23848565, 5267545, 91240048, 23432750, 9058407, 85242963, 26139110, 45790169, 33797252, 45617087, 57241521, 57961282, 75820087, 86001008, 3233084, 27185644, 93359396, 72019362, 10961421, 3880712, 49328608, 42782652, 7229550, 81805959, 78549759, 33628349, 60106217, 61271144, 36933038, 88444207, 26397786, 49236559, 36189527, 37957788, 53802686, 97011160, 81677380, 68128438, 30764367, 59614746, 65338021, 82178706, 38009615, 64087743, 38061439, 3773993, 44177011, 87598888, 59197747, 5482538, 78589145, 39801351, 62803858, 29932657, 17016156, 77301523, 89217461, 43933006, 48088883, 49597667, 37560164, 15535065, 51047803, 59177718, 112651, 43322743, 5073754, 71469330, 36808486, 51464002, 55960386, 50007421, 56473732, 7011964, 4508700, 71333116, 16971929, 27041967, 91664334, 7182642, 59910624, 4091162, 72732205, 37659250, 63015256, 44479073, 50188404, 29401781, 9257405, 43152977, 20645197, 1431742, 67030811, 98948034, 73021291, 72357096, 68939068, 67513640, 45848907, 1250437, 10453030, 6871053, 40152546, 44550764, 8696647, 9398733, 72358170, 91281584, 36753250, 77300457, 92530431, 82339363, 44842615, 36139942, 69136837, 13470059, 4806458, 92787493, 48673079, 76540481, 14947650, 81855445, 88698958, 25842078, 35092039, 22450468, 48395186, 30998561, 42947632, 14326617, 70782102, 26392416, 54014062, 17976208, 9860195, 65454636, 20836893, 53393358, 85695762, 38658347, 20486294, 22879907, 81898046, 30395570, 54263880, 30787683, 64602895, 77377183, 45990383, 5640302, 95395112, 47887470, 29959549, 1022677, 37183543, 16097038, 72793444, 17058722, 62740044, 44847298, 80555751, 23110625, 34295794, 39847321, 16099750, 7685448, 12348118, 81172706, 70367851, 7066775, 47090124, 17146629, 32058615, 40781449, 54232247, 82532312, 92033260, 4069912, 23194618, 95726235, 51466049, 60393039, 86242799, 60955663, 6819644, 5822202, 87710366, 77413012, 57803235, 17386996, 83155442, 48774913, 94076128, 94595668, 4985896, 45306070, 95251277, 54517921, 76825057, 8115266, 83651211, 49598724, 33336148, 19939935, 21001913, 28928175, 75153252, 79657802, 66045587, 57248122, 55602660, 9860968, 81274566, 36580610, 13862149, 64848072, 48675329, 15948937, 22997281, 32159704, 17857111, 67227442, 3633375, 78884452, 35456853, 68875490, 92867155, 23134715, 51149804, 16019925, 51507425, 73617245, 46851987, 36845587, 9175338, 63967300, 12303248, 42692881, 37501808, 79136082, 83747892, 51426311, 88047921, 90004325, 4119747, 86798033, 99226875, 55669657, 36780454, 84187166, 74614639, 32161669, 97940276, 22435353, 21533347, 53084256, 3487592, 7104732, 28787861, 53842979, 62115552, 47893286, 7919588, 75135448, 67031644, 22108665, 69255765, 30569392, 66116458, 62051033, 7300047, 89804152, 42307449, 63506281, 61815223, 38365584, 71316369, 55143724, 85711894, 36135, 31491938, 70420215, 62428472, 45995325, 247198, 68694897, 56515456, 43501211, 93566986, 61859581, 35512853, 26102057, 18001617, 33304202, 51715482, 73235980, 45237957, 9259676, 72495719, 64098930, 15064655, 61728685, 47708850, 92692978, 62496012, 77072625, 45381876, 88904910, 40677414, 20642888, 19891772, 11161731, 99729357, 28851716, 6986898, 72278539, 87160386, 24591705, 61928316, 12024238, 14349098, 53632373 +99690194, 5267545, 8099994, 72019362, 42237907, 18466635, 42644903, 37183543, 92033260, 97057187, 9886593, 92215320, 34493392, 34236719, 569864, 84904436, 30653863, 89214616, 7517032, 54232247, 16567550, 12024238, 50702367, 83155442, 90090964, 40677414, 35512853, 51588015, 4434662, 97281847, 57961282, 35092039, 3487592, 60430369, 78602717, 36468541, 65454636, 17365188, 15163258, 26734892, 81853704, 45794415, 8807940, 24619760, 76434144, 30569392, 874791, 7685448, 2204165, 36808486, 83747892, 55960386, 91664334, 36135, 72732205, 4119747, 23194618, 72777973, 62428472, 99224392, 47738185, 45995325, 99125126, 36753250, 74614639, 93057697, 8115266, 4806458, 83533741, 80316608, 33336148, 33431960, 69641513, 72274002, 22450468, 78785507, 58224549, 45237957, 55602660, 6038457, 20568363, 59981773, 83150534, 67451935, 54663246, 69848388, 80246713, 44177011, 15064655, 55753905, 3235882, 47213183, 20765474, 55850790, 61263068, 43933006, 7300047, 82052050, 76671482, 55615885, 80555751, 78909561, 19101477, 48892201, 29635537, 9175338, 32426752, 44889423, 56461322, 96726697, 78766358, 90004325, 90654836, 71920426, 49271185, 87720882, 85571389, 39201414, 40356877, 20645197, 97783876, 77187825, 56955985, 168541, 669105, 44983451, 11365791, 45306070, 5111370, 44627776, 19457589, 95251277, 19376156, 90310261, 99549373, 17764950, 48673079, 26102057, 17539625, 21001913, 63372756, 88653118, 30998561, 82427263, 91802888, 14326617, 1788101, 6497830, 7919588, 16595436, 32699744, 24058273, 73392814, 78884452, 21993752, 22942635, 38061439, 24826575, 65880522, 78589145, 77377183, 9575954, 52613508, 92071990, 43045786, 98739783, 31161687, 75777973, 61728685, 33123618, 1022677, 82979980, 4204661, 3233569, 89804152, 98653983, 55189057, 66271566, 11543098, 8913721, 76703609, 99729357, 2331773, 62936963, 36505482, 83948335, 23110625, 72738685, 45428665, 39847321, 9188443, 51047803, 70372191, 42073124, 22766820, 74724075, 29029316, 7646095, 50007421, 34044787, 97379790, 16424199, 40781449, 92692978, 27665211, 24314885, 16684829, 68204242, 29401781, 72373496, 29994197, 45919976, 8733068, 1375023, 99524975, 43152977, 14731700, 6819644, 59501487, 70420215, 54606384, 37166608, 17037369, 45848907, 26292919, 57803235, 1250437, 5832946, 91255408, 44550764, 48893685, 8696647, 36812683, 16387575, 43506672, 39553046, 31904591, 36139942, 13470059, 76540481, 14947650, 22435353, 93053405, 45617087, 90272749, 47361209, 38510840, 27185644, 43357947, 58749, 10961421, 66611759, 8055981, 36312813, 74137926, 9860968, 85023028, 95290172, 61271144, 26397786, 13862149, 10358899, 9829782, 51472275, 70036438, 17976208, 59405277, 6157724, 11161731, 97011160, 81677380, 59614746, 61712234, 65715134, 38658347, 20486294, 38009615, 30787683, 176257, 20867149, 59197747, 64157906, 45990383, 33553959, 93933709, 42967683, 17058722, 58180653, 40197395, 30463802, 61815223, 32590267, 29188588, 6793819, 33249630, 77694700, 27187213, 12348118, 41245325, 23740167, 18783702, 17146629, 4508700, 68644627, 38022834, 29834512, 74357852, 88047921, 55143724, 41481685, 79806380, 91938191, 19272365, 63015256, 9257405, 83302115, 1431742, 20002147, 66319530, 5822202, 56424103, 9603598, 11757872, 45381876, 45996863, 89078848, 4064751, 82779622, 44060493, 30771409, 7502255, 10453030, 247198, 30163921, 66832478, 14220886, 82897371, 92353856, 68694897, 56531125, 65017137, 57020481, 526217, 18833224, 61960400, 83368048, 99917599, 44481640, 82327024, 41092172, 16405341, 70800879, 66137019, 26493119, 45667668, 30139692, 8373289, 58208470, 98462867, 96852321, 17894977, 6221471, 48395186, 66250369, 28796059, 96735716, 508198, 2208785, 4515343, 57248122, 7229550, 62115552, 78549759, 30694952, 98943869, 1936762, 63628376, 22200378, 4199704, 93515664, 37957788, 15075176, 20885148, 34667286, 51590803, 30764367, 1197320, 19898053, 85695762, 22879907, 62232211, 81898046, 30891921, 21673260, 40984766, 1569515, 83789391, 8729146, 76170907, 19486173, 69255765, 10649306, 13819358, 62051033, 62803858, 73168565, 29932657, 21289531, 77301523, 23134715, 16097038, 72793444, 36685023, 48260151, 26063929, 16019925, 79880247, 84293052, 46851987, 85116755, 48658605, 96906410, 69786756, 59177718, 112651, 63967300, 12303248, 66885828, 5073754, 47708850, 79136082, 6808825, 67281495, 51426311, 99965001, 47090124, 25636669, 89811711, 71316369, 27041967, 85711894, 81774825, 57359924, 27625735, 33935899, 65047700, 94711842, 47298834, 31491938, 56484900, 24953498, 40781854, 4668450, 98948034, 30218878, 73021291, 72357096, 34497327, 77072625, 41092102, 83269727, 89699445, 11581181, 99333375, 50567636, 36396314, 84127901, 46540998, 40534591, 32151165, 94076128, 16380211, 73031054, 4985896, 2917920, 56515456, 94911072, 72358170, 91281584, 43501211, 92530431, 93566986, 82339363, 68824981, 60176618, 72725103, 45407418, 4095116, 85242963, 97940276, 91727510, 33797252, 80251430, 81855445, 88698958, 19939935, 3233084, 83133790, 90013093, 16445503, 51715482, 87160386, 68102437, 53084256, 11923835, 54762643, 7104732, 9623492, 26776922, 66045587, 3183975, 33628349, 68816413, 47792865, 55470718, 36580610, 57393458, 53802686, 90061527, 14723410, 3633375, 53547802, 65338021, 53393358, 21070110, 3773993, 89046466, 70004753, 87598888, 31733363, 12571310, 64602895, 88130087, 68000591, 61928316, 86301513, 63059024, 5640302, 68875490, 65275241, 38341669, 60581278, 29959549, 89217461, 42199455, 30543215, 63506281, 13468268, 66322959, 65081429, 72614359, 415901, 38365584, 73124510, 41380093, 92541302, 98371444, 92998591, 71469330, 37620363, 93562257, 6505939, 70541760, 7066775, 4099191, 56473732, 7011964, 58751351, 95957797, 43376279, 14045383, 68316156, 4091162, 20122224, 30811010, 74110882, 18699206, 86798033, 66428795, 76960303, 4069912, 50188404, 56153345, 60393039, 37435892, 55247455, 78300864, 67474219, 3294781, 96193415, 36930650, 84187166, 26744917, 12664567, 96709982, 14349098, 48774913, 29819940, 34946859, 6871053, 40152546, 37891451, 99971982, 67793644, 61897582, 54987042, 72278539, 74743862, 69697787, 89637706, 77300457, 71083565, 32274392, 45075471, 79191827, 24733232, 32161669, 9599614, 91240048, 92803766, 9058407, 2891150, 75543508, 59371804, 49598724, 57241521, 94090109, 13173644, 78218845, 29510992, 13231279, 17957593, 71965942, 17068582, 90457870, 84840549, 88251446, 28928175, 40865610, 65271999, 93270084, 3880712, 49328608, 96420429, 79922758, 66903004, 81274566, 54014062, 84406788, 49236559, 37280276, 36189527, 28734791, 14626618, 62430984, 17857111, 15015906, 67227442, 70596786, 89499542, 44844121, 62490109, 15902805, 86460488, 99604946, 40027975, 37224844, 13348726, 98648327, 7423788, 95395112, 92867155, 47887470, 77312810, 86543538, 6111563, 85117092, 42307449, 54427233, 80014588, 73617245, 62740044, 4798568, 44847298, 61741594, 49597667, 15535065, 57163802, 16099750, 18411915, 92604458, 81172706, 73109997, 33699435, 39373729, 16971929, 18131876, 7182642, 16054533, 59910624, 77898274, 61859143, 52060076, 24915585, 69355476, 82532312, 7687278, 99021067, 72238278, 73786944, 95726235, 51466049, 18806856, 74441448, 64055960, 77284619, 62496012, 66663942, 38256849, 86022504, 87710366, 36780454, 31727379, 39986008, 37192445, 15536795, 53632373, 99515901, 76330843, 22129328, 34432810, 97641116, 2717150, 9398733, 67124282, 26664538, 44842615, 23848565, 84349107, 83210802, 38645117, 83651211, 4787945, 92787493, 20642888, 26139110, 29516592, 8634541, 19891772, 3509435, 44348328, 86001008, 26998766, 28550822, 75128745, 91141395, 34698428, 62357986, 73235980, 44664587, 3088684, 95581843, 53842979, 81805959, 26114953, 9259676, 75052463, 38494874, 8651647, 89419466, 36933038, 88444207, 26392416, 67084351, 64848072, 749283, 48675329, 92398073, 32250045, 27325428, 72495719, 89996536, 84166196, 38008118, 98130363, 20681921, 55770687, 69605283, 35456853, 53648154, 73222868, 82178706, 68110330, 64087743, 61373987, 70727211, 99861373, 67031644, 62552524, 44784505, 78561158, 66116458, 92554120, 17016156, 37675718, 39171895, 53666583, 26275734, 51507425, 1239555, 91957544, 77787724, 71333116, 29466406, 52204879, 2607799, 31126490, 32058615, 1204161, 50668599, 10264691, 83083131, 824872, 99226875, 61982238, 57658654, 62762857, 74452589, 55669657, 33565483, 67513640, 8128637, 77413012, 57240218, 6986898, 40686254, 17385531, 52261574, 82726008, 46870723, 17727650, 21397057, 54058788, 79821897, 96318094, 59318837, 65851721, 50806615, 62693428, 44846932, 45049308, 65038678, 61859581, 69136837, 59109400, 76825057, 90158785, 22405120, 94516935, 45790169, 18001617, 69579137, 21533347, 75820087, 26863229, 99658235, 75153252, 66667729, 79657802, 23569917, 42947632, 75229462, 70782102, 30366150, 4978543, 9860195, 10309525, 94539824, 6525948, 20836893, 24591705, 6153406, 12416768, 30395570, 93790285, 75407347, 39801351, 51149804, 36845587, 37739481, 77620120, 71300104, 42692881, 44245960, 18663507, 37501808, 63152504, 18504197, 86118021, 8791066, 33237508, 5970607, 49641577, 82651278, 37659250, 65074535, 86242799, 67030811, 24168411, 94595668, 22721500, 62452034, 88904910, 29065964, 15148031, 10366309, 79942022, 44473167, 25842078, 76315420, 93359396, 68702632, 54199160, 33895336, 91831487, 21919959, 91990218, 15948937, 68128438, 64098930, 75135448, 48360198, 5482538, 94360702, 52293995, 34698463, 26507214, 35192533, 92692283, 75986488, 54868730, 59329511, 14363867, 44479073, 60955663, 17386996, 10597197, 23432750, 27411561, 35996293, 68041839, 97561745, 84684495, 49882705, 51315460, 14093520, 77272628, 22997281, 79738755, 30090481, 8129978, 82886381, 48088883, 84002370, 37560164, 43322743, 20140249, 79322415, 33201905, 71657078, 53888755, 321665, 54517921, 61623915, 28787861, 42782652, 60106217, 47893286, 22108665, 7955293, 34295794, 71522968, 68939068, 17081350, 61141874, 33304202, 95010552, 44915235, 32159704, 41442762, 28851716, 86821229, 6521313, 33061250, 54263880, 60102965, 51464002, 99297164, 22113133, 70367851 +53084256, 70541760, 83083131, 90310261, 99549373, 12416768, 3294781, 6521313, 20642888, 11923835, 32159704, 75777973, 67281495, 74357852, 82779622, 8115266, 51588015, 18001617, 6221471, 75229462, 94539824, 70596786, 19486173, 68000591, 86301513, 61741594, 99297164, 68644627, 79806380, 19272365, 16567550, 29819940, 54058788, 97057187, 32274392, 44481640, 83651211, 75543508, 33336148, 62357986, 60430369, 9259676, 20836893, 81898046, 30395570, 7423788, 5640302, 95395112, 65275241, 7300047, 40197395, 54427233, 46851987, 96906410, 59177718, 63967300, 71522968, 61859143, 92692978, 33935899, 14363867, 40356877, 64055960, 14731700, 824872, 68939068, 37192445, 17727650, 7502255, 67793644, 54987042, 44550764, 67124282, 45049308, 44842615, 4787945, 76540481, 70800879, 83533741, 97940276, 45667668, 78218845, 38510840, 72019362, 58749, 63372756, 57248122, 3183975, 38494874, 91831487, 95290172, 36933038, 10358899, 72495719, 22997281, 34493392, 14723410, 15015906, 81853704, 85695762, 21673260, 76170907, 88130087, 78589145, 3235882, 23134715, 26063929, 13468268, 26507214, 72614359, 36845587, 36505482, 38365584, 32590267, 41380093, 92541302, 66885828, 27187213, 71469330, 79136082, 43376279, 91664334, 57359924, 7687278, 39201414, 12024238, 74441448, 72357096, 62496012, 17037369, 77413012, 52261574, 72278539, 91255408, 89637706, 65017137, 36753250, 68824981, 32161669, 26139110, 80316608, 93053405, 8373289, 21533347, 83133790, 71965942, 21001913, 33304202, 90013093, 40865610, 75128745, 96420429, 68816413, 22200378, 93515664, 28734791, 37957788, 59405277, 6157724, 15948937, 18466635, 30764367, 65338021, 45794415, 38658347, 82178706, 80246713, 86460488, 54263880, 70004753, 37224844, 44784505, 30569392, 92867155, 62803858, 55850790, 21289531, 29959549, 77312810, 4204661, 42307449, 84293052, 15535065, 45428665, 9188443, 51047803, 33249630, 44245960, 37620363, 18783702, 50007421, 7011964, 4508700, 41442762, 71316369, 82532312, 65047700, 29994197, 99524975, 55247455, 20002147, 6819644, 59501487, 74452589, 54606384, 24168411, 62428472, 96709982, 22129328, 14349098, 46870723, 44060493, 40534591, 66832478, 92353856, 56515456, 94911072, 9886593, 36812683, 16387575, 65038678, 15148031, 23848565, 84349107, 92803766, 40677414, 38645117, 13470059, 16405341, 27411561, 4434662, 97561745, 30139692, 57241521, 19891772, 29510992, 17539625, 81855445, 69641513, 19939935, 25842078, 72274002, 51715482, 87160386, 84840549, 28550822, 34698428, 48395186, 88653118, 9623492, 45237957, 66250369, 79657802, 7229550, 23569917, 9860968, 13862149, 64848072, 36189527, 65454636, 97011160, 14626618, 62430984, 15163258, 59614746, 34236719, 1197320, 17857111, 16595436, 69605283, 20486294, 62232211, 30787683, 20867149, 83789391, 12571310, 5482538, 64157906, 47213183, 66116458, 42644903, 73168565, 33553959, 86543538, 26275734, 89804152, 79880247, 55615885, 65081429, 80555751, 62936963, 30463802, 83948335, 37560164, 72738685, 92604458, 54868730, 99690194, 69786756, 112651, 22766820, 5073754, 74724075, 44889423, 7646095, 63152504, 8791066, 17146629, 5970607, 18131876, 68316156, 82651278, 27625735, 69355476, 91938191, 1204161, 86798033, 87720882, 10264691, 43152977, 24953498, 67030811, 9603598, 11757872, 38256849, 41092102, 87710366, 89699445, 15536795, 26292919, 36396314, 6986898, 50702367, 40686254, 10453030, 96318094, 6871053, 40152546, 37891451, 73031054, 53888755, 669105, 86821229, 44983451, 4985896, 14220886, 82897371, 68694897, 62452034, 72358170, 44846932, 57020481, 93566986, 61859581, 83210802, 59109400, 22405120, 26102057, 85242963, 66137019, 59371804, 44473167, 69579137, 58208470, 26863229, 3233084, 8099994, 93270084, 28796059, 96735716, 4515343, 81805959, 98943869, 14326617, 26397786, 36580610, 749283, 4199704, 9860195, 53802686, 81677380, 68128438, 89996536, 98130363, 7919588, 61712234, 53547802, 24058273, 6153406, 22879907, 30891921, 40984766, 24619760, 99861373, 79738755, 15064655, 67031644, 45990383, 69255765, 75407347, 43045786, 3233569, 17058722, 42199455, 11543098, 52293995, 85117092, 874791, 73617245, 62740044, 4798568, 92692283, 77620120, 6793819, 71300104, 42073124, 92998591, 43322743, 77694700, 12348118, 29029316, 47708850, 2204165, 73109997, 6808825, 18504197, 41245325, 99965001, 58751351, 33237508, 95957797, 77787724, 2607799, 96726697, 55143724, 14045383, 49641577, 32058615, 4091162, 20122224, 40781449, 30811010, 90654836, 28851716, 72732205, 4119747, 54232247, 18699206, 49271185, 4069912, 99021067, 50668599, 68204242, 73786944, 1375023, 99226875, 56424103, 34497327, 77072625, 11581181, 39986008, 45848907, 83155442, 99515901, 76330843, 4064751, 46540998, 32151165, 16380211, 10597197, 34432810, 168541, 50806615, 61897582, 90090964, 48893685, 88904910, 44627776, 526217, 74614639, 99917599, 26664538, 5267545, 76825057, 23432750, 45407418, 92787493, 9058407, 48673079, 79942022, 94516935, 14947650, 33797252, 8634541, 57961282, 44348328, 98462867, 88698958, 17957593, 35092039, 76315420, 26998766, 68102437, 88251446, 28928175, 75153252, 44664587, 54199160, 53842979, 33895336, 82427263, 26114953, 33628349, 75052463, 60106217, 42947632, 14093520, 21919959, 95010552, 26392416, 42237907, 30366150, 9829782, 47893286, 51472275, 6497830, 67451935, 27325428, 17365188, 65715134, 21993752, 21070110, 73222868, 8807940, 68110330, 64087743, 89046466, 31733363, 93790285, 65880522, 62552524, 92071990, 63059024, 39801351, 13819358, 20765474, 38341669, 60581278, 33123618, 47887470, 17016156, 1022677, 77301523, 6111563, 16097038, 94360702, 39171895, 60102965, 72793444, 98653983, 36685023, 76671482, 66271566, 30653863, 78909561, 37739481, 61815223, 19101477, 1239555, 73124510, 23110625, 29188588, 85116755, 12303248, 81172706, 37501808, 36808486, 86118021, 56461322, 33699435, 78766358, 31126490, 88047921, 85711894, 16054533, 97379790, 77898274, 37659250, 44479073, 23194618, 29401781, 95726235, 31491938, 20645197, 78300864, 40781854, 4668450, 36930650, 62762857, 97783876, 37166608, 31727379, 84187166, 8128637, 12664567, 89078848, 57803235, 21397057, 5832946, 34946859, 59318837, 94076128, 2717150, 69697787, 9398733, 45306070, 56531125, 61141874, 91281584, 43501211, 61960400, 92530431, 82339363, 82327024, 19376156, 91240048, 60176618, 90158785, 17764950, 91727510, 29516592, 94090109, 80251430, 84684495, 17894977, 22450468, 61623915, 10961421, 91141395, 7104732, 58224549, 66667729, 68702632, 66045587, 42782652, 2208785, 79922758, 91802888, 6038457, 8651647, 47792865, 55470718, 1788101, 91990218, 54014062, 49236559, 37280276, 10309525, 90061527, 77272628, 54663246, 78884452, 62490109, 15902805, 1569515, 70727211, 87598888, 55753905, 13348726, 98648327, 77377183, 10649306, 98739783, 92554120, 31161687, 61728685, 89217461, 43933006, 55189057, 30543215, 48260151, 16019925, 66322959, 51507425, 84904436, 415901, 39847321, 29635537, 9175338, 98371444, 7685448, 93562257, 83747892, 29466406, 16971929, 41481685, 81774825, 63015256, 72238278, 9257405, 51466049, 60393039, 66319530, 77284619, 66663942, 96193415, 57658654, 36780454, 99333375, 33565483, 50567636, 67513640, 56955985, 45996863, 57240218, 17385531, 1250437, 82726008, 99224392, 30771409, 30163921, 11365791, 8696647, 62693428, 99125126, 43506672, 54517921, 39553046, 31904591, 93057697, 10366309, 35996293, 22435353, 45617087, 13231279, 96852321, 17068582, 90457870, 49882705, 65271999, 36312813, 28787861, 3880712, 30998561, 49328608, 55602660, 78602717, 78549759, 59981773, 83150534, 36468541, 61271144, 67084351, 63628376, 44915235, 4978543, 11161731, 20885148, 51590803, 69848388, 3633375, 53393358, 73392814, 89499542, 53648154, 33061250, 3773993, 44177011, 64602895, 52613508, 8129978, 62051033, 29932657, 37675718, 569864, 82979980, 37183543, 42967683, 48088883, 58180653, 8913721, 84002370, 57163802, 48658605, 89214616, 32426752, 70367851, 18663507, 6505939, 7066775, 51426311, 55960386, 56473732, 47090124, 22113133, 38022834, 29834512, 27041967, 7182642, 36135, 7517032, 52060076, 90004325, 74110882, 66428795, 76960303, 45919976, 18806856, 83302115, 8733068, 56484900, 37435892, 1431742, 67474219, 73021291, 55669657, 86022504, 83269727, 45381876, 53632373, 84127901, 48774913, 79821897, 45995325, 94595668, 99971982, 74743862, 321665, 95251277, 71083565, 79191827, 24733232, 9599614, 69136837, 72725103, 35512853, 2891150, 45790169, 49598724, 26493119, 47361209, 92215320, 86001008, 43357947, 16445503, 95581843, 26776922, 30694952, 51315460, 20568363, 66903004, 85023028, 81274566, 70782102, 57393458, 84406788, 92398073, 84166196, 6525948, 26734892, 20681921, 32699744, 22942635, 44844121, 99604946, 176257, 40027975, 8729146, 59197747, 76434144, 68875490, 82886381, 51149804, 34698463, 2331773, 91957544, 49597667, 34295794, 16099750, 18411915, 51464002, 23740167, 4099191, 34044787, 89811711, 59910624, 16424199, 71920426, 24314885, 92033260, 85571389, 86242799, 98948034, 30218878, 61982238, 77187825, 33201905, 26744917, 17081350, 71657078, 247198, 65851721, 29065964, 18833224, 36139942, 4095116, 68041839, 90272749, 3509435, 27185644, 93359396, 3487592, 8055981, 73235980, 3088684, 74137926, 1936762, 88444207, 70036438, 15075176, 32250045, 64098930, 24591705, 19898053, 55770687, 35456853, 38061439, 75135448, 48360198, 30090481, 22108665, 61928316, 78561158, 93933709, 53666583, 76703609, 80014588, 44847298, 7955293, 48892201, 39373729, 52204879, 65074535, 56153345, 60955663, 70420215, 79322415, 19457589, 83368048, 41092172, 97281847, 78785507, 99658235, 66611759, 54762643, 62115552, 89419466, 17976208, 34667286, 61373987, 24826575, 9575954, 61263068, 82052050, 99729357, 35192533, 75986488, 70372191, 42692881, 24915585, 27665211, 16684829, 50188404, 94711842, 47298834, 20140249, 47738185, 2917920, 5111370, 77300457, 45075471, 4806458, 33431960, 75820087, 508198, 63506281, 59329511, 71333116, 72777973, 5822202, 17386996, 22721500, 72373496, 48675329, 38008118, 67227442, 38009615, 97641116, 13173644, 25636669 +11161731, 93053405, 42947632, 61928316, 62496012, 62762857, 69697787, 66137019, 92215320, 33628349, 48675329, 81677380, 44177011, 72793444, 16019925, 5073754, 89811711, 29466406, 41481685, 40781449, 71920426, 86798033, 9257405, 29994197, 24953498, 94595668, 44842615, 9599614, 22405120, 14947650, 91727510, 63372756, 26776922, 23569917, 51315460, 9259676, 55470718, 64848072, 28734791, 97011160, 65880522, 44784505, 30569392, 63059024, 65275241, 55850790, 73617245, 77620120, 85116755, 83747892, 70541760, 25636669, 85711894, 37659250, 99524975, 98948034, 66663942, 83269727, 39986008, 45848907, 1250437, 59318837, 66832478, 72278539, 61859581, 19376156, 40677414, 35512853, 68041839, 18001617, 94090109, 98462867, 17957593, 83133790, 28928175, 3487592, 65271999, 44664587, 96735716, 7229550, 98943869, 81274566, 36933038, 88444207, 54014062, 68128438, 34493392, 94539824, 15015906, 3633375, 22942635, 80246713, 64087743, 30787683, 55753905, 43045786, 13819358, 92867155, 569864, 58180653, 40197395, 42307449, 99729357, 66322959, 65081429, 36505482, 75986488, 9175338, 92604458, 12303248, 22766820, 79136082, 47090124, 8791066, 5970607, 38022834, 91664334, 92692978, 19272365, 50668599, 85571389, 95726235, 56484900, 66319530, 86022504, 62428472, 17037369, 45996863, 77413012, 36396314, 84127901, 10453030, 53888755, 22721500, 14220886, 8696647, 29065964, 93057697, 8115266, 13470059, 92787493, 48673079, 83533741, 22435353, 13173644, 69579137, 47361209, 96852321, 28550822, 61623915, 58749, 3880712, 49328608, 42782652, 91802888, 59981773, 749283, 15163258, 53393358, 78884452, 20486294, 82178706, 62232211, 21673260, 40984766, 24619760, 20867149, 24826575, 48360198, 78589145, 67031644, 78561158, 92071990, 69255765, 75407347, 42644903, 33123618, 6111563, 48088883, 17058722, 51149804, 13468268, 36845587, 415901, 34295794, 15535065, 39847321, 29635537, 57163802, 9188443, 99690194, 59177718, 42073124, 112651, 66885828, 77694700, 44889423, 71469330, 50007421, 71316369, 2607799, 78766358, 36135, 30811010, 28851716, 72732205, 4119747, 54232247, 82532312, 66428795, 39201414, 51466049, 31491938, 1431742, 86242799, 4668450, 79322415, 50567636, 40686254, 96709982, 30163921, 9398733, 45306070, 89637706, 65017137, 43501211, 74614639, 54517921, 83368048, 82339363, 23848565, 76825057, 26139110, 4434662, 49598724, 97281847, 30139692, 29516592, 44473167, 78218845, 29510992, 35092039, 78785507, 11923835, 66611759, 45237957, 95581843, 79657802, 508198, 26114953, 38494874, 9860968, 68816413, 83150534, 95290172, 1788101, 63628376, 70036438, 36189527, 27325428, 32159704, 84166196, 38008118, 69848388, 24591705, 81853704, 20681921, 85695762, 21070110, 68110330, 38061439, 83789391, 76170907, 9575954, 3235882, 47213183, 66116458, 8129978, 92554120, 38341669, 1022677, 42967683, 7300047, 4204661, 55189057, 66271566, 52293995, 48260151, 51507425, 79880247, 30653863, 62740044, 72614359, 44847298, 78909561, 61815223, 16099750, 63967300, 81172706, 70367851, 63152504, 41245325, 7066775, 18783702, 86118021, 29834512, 74357852, 55143724, 4091162, 49271185, 4069912, 44479073, 23194618, 45919976, 10264691, 18806856, 60393039, 37435892, 64055960, 78300864, 72357096, 61982238, 56424103, 77072625, 11757872, 77187825, 38256849, 36780454, 50702367, 82726008, 76330843, 4064751, 44060493, 29819940, 30771409, 6871053, 45995325, 16380211, 168541, 82897371, 92353856, 56515456, 5111370, 99125126, 36753250, 77300457, 95251277, 92530431, 36139942, 69136837, 92803766, 90310261, 23432750, 99549373, 4806458, 9058407, 79942022, 26493119, 8634541, 33431960, 17539625, 13231279, 84684495, 88698958, 21001913, 33304202, 90457870, 72019362, 68102437, 6221471, 58224549, 93270084, 33895336, 82427263, 96420429, 60430369, 57248122, 79922758, 55602660, 62115552, 81805959, 14093520, 21919959, 30366150, 13862149, 49236559, 22200378, 4199704, 44915235, 67451935, 15075176, 32250045, 51590803, 17365188, 30764367, 17857111, 67227442, 53547802, 70596786, 19898053, 73392814, 21993752, 73222868, 15902805, 86460488, 61373987, 99861373, 15064655, 75135448, 59197747, 64157906, 13348726, 98648327, 68000591, 45990383, 5640302, 95395112, 60581278, 29959549, 61263068, 82979980, 34698463, 63506281, 46851987, 19101477, 48892201, 1239555, 41380093, 92541302, 29188588, 18411915, 42692881, 12348118, 74724075, 29029316, 2204165, 37620363, 37501808, 73109997, 6505939, 51426311, 55960386, 17146629, 41442762, 39373729, 33237508, 71333116, 31126490, 7182642, 68316156, 82651278, 52060076, 57359924, 24915585, 27625735, 24314885, 7687278, 65047700, 63015256, 68204242, 83302115, 60955663, 14731700, 96193415, 34497327, 87710366, 37166608, 89699445, 31727379, 68939068, 56955985, 8128637, 26292919, 53632373, 57803235, 99515901, 52261574, 99224392, 48774913, 37891451, 10597197, 97057187, 65851721, 669105, 91255408, 67124282, 9886593, 88904910, 61141874, 44627776, 93566986, 44481640, 24733232, 26664538, 5267545, 38645117, 20642888, 4095116, 76540481, 26102057, 35996293, 59371804, 33797252, 8373289, 57241521, 80251430, 19891772, 58208470, 69641513, 17894977, 26998766, 88251446, 40865610, 54762643, 88653118, 7104732, 54199160, 6038457, 74137926, 78602717, 75052463, 66903004, 91831487, 89419466, 47792865, 14326617, 95010552, 26392416, 67084351, 42237907, 51472275, 20885148, 18466635, 34667286, 90061527, 77272628, 89996536, 65715134, 69605283, 81898046, 44844121, 30395570, 8729146, 88130087, 76434144, 52613508, 20765474, 31161687, 62051033, 62803858, 29932657, 47887470, 37675718, 86543538, 43933006, 39171895, 60102965, 3233569, 89804152, 98653983, 42199455, 82052050, 11543098, 8913721, 55615885, 83948335, 32590267, 23110625, 7685448, 71300104, 96906410, 43322743, 18663507, 18504197, 4099191, 7011964, 4508700, 58751351, 34044787, 59329511, 77787724, 43376279, 96726697, 14045383, 90004325, 74110882, 33935899, 76960303, 87720882, 29401781, 72777973, 73786944, 20645197, 67474219, 77284619, 30218878, 3294781, 5822202, 9603598, 97783876, 55669657, 33565483, 26744917, 37192445, 45381876, 12664567, 57240218, 6986898, 47738185, 21397057, 34946859, 32151165, 79821897, 247198, 50806615, 44983451, 4985896, 6521313, 44550764, 68694897, 62693428, 321665, 72358170, 526217, 71083565, 32274392, 68824981, 15148031, 32161669, 82327024, 10366309, 91240048, 45407418, 41092172, 16405341, 27411561, 94516935, 33336148, 21533347, 86001008, 71965942, 38510840, 90013093, 49882705, 99658235, 34698428, 8055981, 66250369, 36312813, 68702632, 4515343, 30694952, 75229462, 70782102, 36580610, 57393458, 84406788, 37280276, 6497830, 93515664, 6157724, 59614746, 64098930, 98130363, 32699744, 24058273, 89499542, 38658347, 8807940, 38009615, 12416768, 3773993, 54263880, 70727211, 93790285, 19486173, 30090481, 77377183, 62552524, 68875490, 39801351, 33553959, 93933709, 16097038, 53666583, 36685023, 76671482, 85117092, 874791, 80014588, 84904436, 26507214, 30463802, 91957544, 38365584, 92692283, 72738685, 98371444, 48658605, 44245960, 47708850, 36808486, 51464002, 56461322, 99297164, 68644627, 16971929, 52204879, 18131876, 88047921, 16054533, 27665211, 14363867, 50188404, 72373496, 16567550, 47298834, 74441448, 83083131, 20002147, 20140249, 70420215, 57658654, 74452589, 41092102, 24168411, 33201905, 99333375, 15536795, 17081350, 14349098, 46870723, 17727650, 5832946, 73031054, 86821229, 2717150, 48893685, 56531125, 94911072, 19457589, 65038678, 18833224, 45075471, 99917599, 83210802, 60176618, 83651211, 4787945, 45667668, 45617087, 90272749, 57961282, 75820087, 3509435, 44348328, 26863229, 3233084, 17068582, 93359396, 84840549, 53084256, 48395186, 9623492, 66667729, 2208785, 3183975, 8651647, 10358899, 47893286, 37957788, 17976208, 65454636, 10309525, 72495719, 62430984, 34236719, 6525948, 1197320, 26734892, 20836893, 16595436, 6153406, 45794415, 53648154, 33061250, 1569515, 176257, 40027975, 7423788, 61728685, 37183543, 23134715, 76703609, 26063929, 84293052, 2331773, 84002370, 4798568, 62936963, 7955293, 37560164, 45428665, 89214616, 32426752, 33249630, 27187213, 71522968, 6808825, 67281495, 99965001, 56473732, 22113133, 81774825, 7517032, 77898274, 20122224, 90654836, 69355476, 91938191, 1204161, 92033260, 16684829, 72238278, 43152977, 55247455, 67030811, 59501487, 99226875, 83155442, 17385531, 82779622, 7502255, 46540998, 40534591, 40152546, 94076128, 34432810, 99971982, 61897582, 90090964, 16387575, 43506672, 61960400, 39553046, 90158785, 72725103, 17764950, 70800879, 85242963, 97940276, 2891150, 75543508, 45790169, 27185644, 72274002, 8099994, 16445503, 51715482, 75128745, 10961421, 91141395, 73235980, 53842979, 78549759, 20568363, 36468541, 9829782, 4978543, 9860195, 59405277, 22997281, 14723410, 22879907, 30891921, 89046466, 87598888, 37224844, 12571310, 98739783, 75777973, 21289531, 77301523, 89217461, 26275734, 30543215, 54427233, 49597667, 73124510, 6793819, 51047803, 70372191, 7646095, 93562257, 23740167, 27041967, 16424199, 32058615, 65074535, 94711842, 40781854, 36930650, 22129328, 97641116, 54987042, 74743862, 44846932, 45049308, 31904591, 80316608, 97561745, 43357947, 76315420, 87160386, 75153252, 62357986, 3088684, 28787861, 66045587, 30998561, 1936762, 92398073, 53802686, 14626618, 54663246, 7919588, 61712234, 65338021, 35456853, 62490109, 99604946, 70004753, 64602895, 5482538, 17016156, 80555751, 61741594, 92998591, 33699435, 61859143, 56153345, 40356877, 1375023, 6819644, 84187166, 67513640, 96318094, 67793644, 11365791, 62452034, 36812683, 91281584, 84349107, 59109400, 51588015, 22450468, 28796059, 55770687, 31733363, 86301513, 10649306, 73168565, 77312810, 37739481, 54868730, 69786756, 49641577, 18699206, 8733068, 12024238, 73021291, 824872, 54606384, 89078848, 17386996, 71657078, 2917920, 57020481, 79191827, 19939935, 25842078, 60106217, 61271144, 26397786, 91990218, 15948937, 79738755, 82886381, 94360702, 35192533, 95957797, 59910624, 97379790, 79806380, 11581181, 54058788, 85023028, 22108665, 81855445, 99021067 +5073754, 749283, 36685023, 25636669, 32058615, 47738185, 34946859, 10366309, 66137019, 96852321, 93359396, 58224549, 95010552, 67084351, 3633375, 93790285, 48360198, 64602895, 5482538, 32590267, 75986488, 42692881, 33237508, 38022834, 90654836, 87720882, 17386996, 72358170, 99125126, 91281584, 92787493, 4095116, 59371804, 68041839, 30366150, 69848388, 30787683, 9575954, 44784505, 13819358, 66271566, 15535065, 57163802, 89214616, 63152504, 77787724, 7517032, 20122224, 27625735, 71920426, 65074535, 56153345, 60393039, 74441448, 20140249, 32151165, 97057187, 89637706, 15148031, 22435353, 30139692, 3509435, 75153252, 54762643, 49328608, 61271144, 54014062, 47893286, 59405277, 62430984, 81853704, 54263880, 70004753, 83789391, 99861373, 68875490, 48260151, 72614359, 92998591, 37620363, 33699435, 88047921, 16424199, 27665211, 24314885, 86798033, 72373496, 56484900, 40781854, 824872, 11757872, 79322415, 54606384, 87710366, 82726008, 95251277, 71083565, 36139942, 45617087, 90272749, 57241521, 13173644, 21533347, 58208470, 17957593, 43357947, 88251446, 99658235, 28928175, 10961421, 6221471, 9623492, 3088684, 74137926, 23569917, 59981773, 47792865, 83150534, 64848072, 10309525, 18466635, 90061527, 54663246, 67227442, 16595436, 24058273, 22879907, 99604946, 43045786, 93933709, 42967683, 7300047, 72793444, 42307449, 99729357, 63506281, 13468268, 84002370, 48892201, 1239555, 34295794, 29188588, 70372191, 59177718, 77694700, 27187213, 18663507, 37501808, 6808825, 8791066, 68644627, 74357852, 78766358, 28851716, 4069912, 16684829, 50668599, 29994197, 73021291, 66663942, 97783876, 33201905, 33565483, 17037369, 8128637, 5832946, 10453030, 6871053, 168541, 11365791, 67124282, 321665, 29065964, 45049308, 526217, 31904591, 24733232, 40677414, 72725103, 51588015, 17764950, 20642888, 16405341, 94516935, 97940276, 33797252, 33431960, 13231279, 69641513, 72274002, 78785507, 44664587, 66250369, 36312813, 68702632, 54199160, 68816413, 10358899, 22200378, 15075176, 20885148, 77272628, 6525948, 65715134, 53547802, 69605283, 68110330, 87598888, 88130087, 30090481, 92071990, 63059024, 10649306, 31161687, 82886381, 77301523, 94360702, 30543215, 85117092, 8913721, 34698463, 51507425, 874791, 37739481, 19101477, 45428665, 48658605, 92604458, 63967300, 74724075, 44889423, 47708850, 6505939, 7066775, 86118021, 4099191, 89811711, 95957797, 27041967, 41481685, 72732205, 49271185, 7687278, 63015256, 76960303, 50188404, 72777973, 1375023, 37435892, 55247455, 67474219, 77284619, 74452589, 11581181, 56955985, 12664567, 26292919, 77413012, 89078848, 36396314, 96709982, 17385531, 21397057, 7502255, 59318837, 65851721, 73031054, 54987042, 86821229, 8696647, 16387575, 77300457, 18833224, 43501211, 61960400, 45075471, 39553046, 93057697, 92803766, 59109400, 83651211, 27411561, 79942022, 2891150, 75543508, 45790169, 18001617, 29510992, 69579137, 44348328, 88698958, 19939935, 27185644, 35092039, 76315420, 72019362, 49882705, 40865610, 11923835, 66611759, 88653118, 66667729, 95581843, 53842979, 42782652, 2208785, 96420429, 7229550, 6038457, 78602717, 78549759, 75052463, 66903004, 89419466, 42237907, 13862149, 48675329, 92398073, 27325428, 68128438, 34493392, 30764367, 14723410, 20836893, 53648154, 73222868, 22942635, 44844121, 38009615, 86460488, 30395570, 76170907, 55753905, 78589145, 61928316, 75407347, 8129978, 39801351, 75777973, 73168565, 61728685, 47887470, 33553959, 37183543, 86543538, 60102965, 48088883, 26275734, 82052050, 40197395, 73617245, 84904436, 44847298, 78909561, 36505482, 30463802, 37560164, 6793819, 9175338, 98371444, 32426752, 44245960, 71522968, 50007421, 71316369, 29466406, 52204879, 18131876, 96726697, 31126490, 55143724, 16054533, 59910624, 82651278, 24915585, 37659250, 74110882, 29401781, 85571389, 39201414, 16567550, 18806856, 47298834, 24953498, 86242799, 14731700, 30218878, 34497327, 9603598, 36930650, 86022504, 37166608, 89699445, 50567636, 39986008, 17081350, 17727650, 54058788, 61897582, 53888755, 44983451, 91255408, 6521313, 90090964, 2917920, 9398733, 5111370, 88904910, 57020481, 36753250, 54517921, 99917599, 44481640, 32161669, 82327024, 61859581, 5267545, 90310261, 60176618, 23432750, 45407418, 48673079, 26102057, 80316608, 49598724, 26493119, 97281847, 44473167, 80251430, 19891772, 57961282, 75820087, 25842078, 38510840, 17068582, 8099994, 16445503, 75128745, 28796059, 508198, 60430369, 62115552, 26114953, 9860968, 91831487, 14093520, 55470718, 36468541, 70782102, 88444207, 1788101, 26397786, 37280276, 4199704, 44915235, 93515664, 28734791, 37957788, 65454636, 17365188, 81677380, 89996536, 38008118, 26734892, 17857111, 24591705, 61712234, 65338021, 19898053, 38658347, 20486294, 35456853, 64087743, 38061439, 44177011, 61373987, 79738755, 40027975, 37224844, 12571310, 65880522, 64157906, 67031644, 22108665, 62552524, 52613508, 66116458, 65275241, 38341669, 92867155, 29932657, 17016156, 6111563, 42199455, 54427233, 80014588, 79880247, 30653863, 62936963, 61815223, 7955293, 9188443, 16099750, 71300104, 66885828, 2204165, 7646095, 79136082, 70541760, 4508700, 41442762, 39373729, 49641577, 36135, 52060076, 30811010, 54232247, 79806380, 44479073, 99021067, 40356877, 45919976, 20645197, 4668450, 67030811, 70420215, 57658654, 24168411, 36780454, 83269727, 84187166, 26744917, 84127901, 83155442, 30771409, 46540998, 79821897, 16380211, 10597197, 50806615, 30163921, 669105, 4985896, 2717150, 74743862, 48893685, 69697787, 68694897, 94911072, 9886593, 19457589, 65038678, 74614639, 83368048, 79191827, 9599614, 19376156, 91240048, 69136837, 83210802, 4787945, 41092172, 70800879, 35996293, 8373289, 47361209, 84684495, 3233084, 83133790, 33304202, 26998766, 68102437, 3487592, 91141395, 34698428, 65271999, 62357986, 7104732, 8055981, 73235980, 79657802, 26776922, 66045587, 96735716, 57248122, 79922758, 51315460, 8651647, 14326617, 36933038, 57393458, 84406788, 63628376, 49236559, 11161731, 34667286, 72495719, 97011160, 14626618, 15163258, 94539824, 32159704, 84166196, 64098930, 20681921, 6153406, 78884452, 55770687, 21993752, 82178706, 12416768, 33061250, 1569515, 31733363, 75135448, 59197747, 3235882, 47213183, 78561158, 98739783, 95395112, 42644903, 62803858, 55850790, 60581278, 98653983, 58180653, 55189057, 16019925, 66322959, 84293052, 4798568, 85116755, 99690194, 70367851, 73109997, 51464002, 83747892, 67281495, 99965001, 56461322, 99297164, 43376279, 14045383, 97379790, 57359924, 92692978, 69355476, 18699206, 33935899, 14363867, 72238278, 95726235, 83302115, 8733068, 1431742, 60955663, 66319530, 5822202, 99226875, 61982238, 96193415, 62762857, 38256849, 68939068, 52261574, 22129328, 14349098, 4064751, 82779622, 44060493, 96318094, 40152546, 22721500, 62452034, 44627776, 36812683, 43506672, 92530431, 68824981, 38645117, 76825057, 83533741, 91727510, 33336148, 45667668, 97561745, 8634541, 94090109, 78218845, 92215320, 90013093, 17894977, 51715482, 87160386, 93270084, 28787861, 3880712, 82427263, 81805959, 33628349, 20568363, 1936762, 85023028, 95290172, 9829782, 6497830, 53802686, 51590803, 22997281, 34236719, 70596786, 85695762, 81898046, 80246713, 15902805, 8807940, 89046466, 19486173, 76434144, 98648327, 69255765, 7423788, 20765474, 62051033, 33123618, 21289531, 77312810, 89217461, 16097038, 39171895, 4204661, 89804152, 17058722, 51149804, 76703609, 55615885, 46851987, 80555751, 35192533, 415901, 83948335, 91957544, 41380093, 77620120, 29635537, 96906410, 54868730, 69786756, 22766820, 12348118, 71469330, 41245325, 56473732, 58751351, 22113133, 71333116, 29834512, 65047700, 66428795, 92033260, 68204242, 23194618, 9257405, 73786944, 94711842, 99524975, 6819644, 98948034, 3294781, 62496012, 77187825, 99333375, 31727379, 37192445, 45381876, 15536795, 53632373, 57803235, 6986898, 50702367, 1250437, 99224392, 71657078, 40534591, 37891451, 247198, 94076128, 34432810, 97641116, 66832478, 14220886, 45306070, 56515456, 56531125, 93566986, 26664538, 84349107, 8115266, 90158785, 35512853, 22405120, 14947650, 26139110, 81855445, 26863229, 86001008, 21001913, 4515343, 91802888, 9259676, 81274566, 21919959, 91990218, 51472275, 36189527, 4978543, 15948937, 32250045, 1197320, 7919588, 32699744, 53393358, 21070110, 62232211, 3773993, 15064655, 24826575, 68000591, 77377183, 45990383, 86301513, 37675718, 82979980, 23134715, 11543098, 52293995, 26507214, 62740044, 73124510, 92692283, 7685448, 51047803, 42073124, 93562257, 36808486, 18783702, 7011964, 17146629, 59329511, 16971929, 2607799, 7182642, 85711894, 81774825, 90004325, 4091162, 40781449, 4119747, 91938191, 43152977, 78300864, 20002147, 72357096, 41092102, 62428472, 45848907, 45996863, 40686254, 76330843, 46870723, 29819940, 45995325, 92353856, 62693428, 61141874, 9058407, 76540481, 17539625, 71965942, 22450468, 28550822, 61623915, 63372756, 45237957, 33895336, 30998561, 55602660, 30694952, 38494874, 67451935, 59614746, 98130363, 45794415, 73392814, 89499542, 30891921, 21673260, 40984766, 20867149, 70727211, 30569392, 92554120, 43933006, 3233569, 53666583, 76671482, 2331773, 36845587, 61741594, 92541302, 72738685, 12303248, 43322743, 29029316, 51426311, 68316156, 77898274, 19272365, 12024238, 31491938, 56424103, 77072625, 67513640, 48774913, 72278539, 82897371, 65017137, 32274392, 82339363, 99549373, 93053405, 29516592, 84840549, 53084256, 48395186, 17976208, 9860195, 13348726, 5640302, 1022677, 61263068, 569864, 26063929, 38365584, 49597667, 23110625, 18411915, 33249630, 81172706, 18504197, 47090124, 34044787, 91664334, 61859143, 82532312, 59501487, 55669657, 99971982, 67793644, 44550764, 23848565, 13470059, 85242963, 98462867, 90457870, 42947632, 26392416, 36580610, 70036438, 15015906, 176257, 8729146, 29959549, 39847321, 23740167, 1204161, 10264691, 83083131, 57240218, 94595668, 4806458, 3183975, 60106217, 75229462, 6157724, 62490109, 55960386, 5970607, 64055960, 99515901, 44842615, 4434662, 58749, 24619760, 65081429, 51466049, 44846932, 98943869, 112651 +31161687, 37891451, 38341669, 77620120, 80316608, 56473732, 50188404, 40356877, 96193415, 168541, 79191827, 24733232, 48673079, 35996293, 58749, 83150534, 10358899, 22879907, 62803858, 33553959, 48260151, 98371444, 71300104, 33249630, 44245960, 6505939, 86022504, 4787945, 13173644, 84684495, 83133790, 43357947, 49328608, 4515343, 81274566, 67084351, 57393458, 70036438, 65715134, 64087743, 55850790, 89217461, 40197395, 29188588, 42692881, 7066775, 55960386, 47090124, 29834512, 74357852, 85711894, 59910624, 61859143, 54232247, 68204242, 60393039, 30218878, 50702367, 40152546, 97641116, 11365791, 45306070, 43501211, 82327024, 36139942, 45407418, 41092172, 91727510, 22435353, 80251430, 81855445, 21533347, 21001913, 61623915, 66611759, 33628349, 38494874, 9860968, 55470718, 70782102, 1788101, 84166196, 61712234, 67227442, 32699744, 19898053, 44177011, 70727211, 19486173, 22108665, 47213183, 75407347, 98739783, 39801351, 20765474, 77301523, 6111563, 53666583, 42199455, 8913721, 63506281, 874791, 26507214, 30463802, 16099750, 18411915, 43322743, 66885828, 12348118, 51464002, 51426311, 86118021, 5970607, 95957797, 96726697, 88047921, 4119747, 69355476, 18699206, 23194618, 20002147, 6819644, 824872, 56424103, 41092102, 7502255, 44550764, 74743862, 94911072, 9886593, 61141874, 91281584, 57020481, 43506672, 93566986, 92803766, 23432750, 83651211, 35512853, 70800879, 26102057, 66137019, 93053405, 4434662, 26493119, 18001617, 17539625, 26998766, 28550822, 68702632, 66045587, 78549759, 8651647, 59981773, 9829782, 67451935, 9860195, 15948937, 27325428, 89996536, 98130363, 20836893, 3633375, 69605283, 38009615, 99604946, 3773993, 89046466, 1569515, 37224844, 76434144, 13348726, 92071990, 62051033, 42967683, 89804152, 46851987, 36845587, 61741594, 57163802, 70372191, 63152504, 41245325, 25636669, 58751351, 29466406, 43376279, 55143724, 91938191, 99021067, 50668599, 39201414, 16567550, 64055960, 62762857, 37166608, 62428472, 99333375, 77413012, 82726008, 46870723, 48774913, 82779622, 44060493, 79821897, 6871053, 94595668, 73031054, 30163921, 61897582, 53888755, 86821229, 2717150, 2917920, 62693428, 44846932, 61960400, 92530431, 26664538, 8115266, 72725103, 76540481, 8373289, 33431960, 19891772, 47361209, 71965942, 76315420, 90457870, 84840549, 68102437, 75153252, 10961421, 3880712, 79657802, 53842979, 60430369, 81805959, 23569917, 42947632, 14093520, 75229462, 95290172, 36580610, 54014062, 48675329, 28734791, 37957788, 15075176, 6157724, 65454636, 11161731, 14626618, 59614746, 6153406, 73392814, 89499542, 21070110, 81898046, 38061439, 93790285, 55753905, 78589145, 8129978, 73168565, 29932657, 21289531, 17016156, 86543538, 43933006, 39171895, 98653983, 66271566, 52293995, 26063929, 65081429, 19101477, 48892201, 91957544, 75986488, 51047803, 92604458, 54868730, 112651, 63967300, 12303248, 44889423, 7646095, 67281495, 50007421, 4099191, 56461322, 33699435, 8791066, 89811711, 41481685, 68316156, 4091162, 27665211, 1204161, 65047700, 76960303, 10264691, 83302115, 94711842, 12024238, 47298834, 1375023, 99524975, 56484900, 1431742, 78300864, 55669657, 38256849, 87710366, 11581181, 33565483, 17037369, 84187166, 37192445, 15536795, 53632373, 57240218, 83155442, 52261574, 17727650, 247198, 67124282, 56515456, 56531125, 29065964, 36753250, 16387575, 77300457, 32274392, 31904591, 9599614, 5267545, 84349107, 91240048, 69136837, 59109400, 76825057, 51588015, 4806458, 79942022, 68041839, 97561745, 29516592, 44473167, 8634541, 94090109, 92215320, 13231279, 57961282, 75820087, 69641513, 33304202, 72274002, 51715482, 22450468, 93359396, 99658235, 65271999, 79922758, 55602660, 30694952, 51315460, 75052463, 98943869, 47792865, 36468541, 22200378, 64848072, 51590803, 72495719, 81677380, 22997281, 68128438, 54663246, 64098930, 14723410, 26734892, 15015906, 24058273, 70596786, 53648154, 30891921, 80246713, 24619760, 176257, 79738755, 48360198, 77377183, 45990383, 66116458, 86301513, 5640302, 95395112, 92867155, 60581278, 82886381, 61263068, 37675718, 23134715, 94360702, 72793444, 58180653, 34698463, 76703609, 51507425, 79880247, 84293052, 73617245, 84002370, 37739481, 35192533, 49597667, 37560164, 34295794, 15535065, 39847321, 9188443, 48658605, 89214616, 99690194, 27187213, 47708850, 70541760, 7011964, 41442762, 77787724, 71333116, 27041967, 18131876, 2607799, 16054533, 49641577, 77898274, 52060076, 57359924, 37659250, 71920426, 74110882, 86798033, 44479073, 72238278, 45919976, 31491938, 24953498, 74441448, 40781854, 4668450, 66319530, 62496012, 34497327, 77072625, 9603598, 67513640, 56955985, 45848907, 8128637, 40686254, 96709982, 32151165, 54987042, 91255408, 5111370, 36812683, 65038678, 83368048, 32161669, 90310261, 90158785, 16405341, 26139110, 45667668, 90272749, 30139692, 29510992, 69579137, 44348328, 88698958, 86001008, 27185644, 35092039, 8099994, 28928175, 3487592, 34698428, 93270084, 36312813, 33895336, 26776922, 66903004, 91831487, 61271144, 26392416, 63628376, 37280276, 749283, 6497830, 92398073, 20885148, 10309525, 32250045, 77272628, 53393358, 35456853, 22942635, 21673260, 61373987, 70004753, 20867149, 15064655, 12571310, 75135448, 98648327, 52613508, 69255765, 30569392, 13819358, 92554120, 65275241, 61728685, 60102965, 4204661, 26275734, 17058722, 55189057, 82052050, 76671482, 51149804, 16019925, 84904436, 55615885, 2331773, 72614359, 80555751, 62936963, 1239555, 83948335, 38365584, 73124510, 23110625, 45428665, 6793819, 9175338, 96906410, 59177718, 5073754, 74724075, 81172706, 71522968, 36808486, 79136082, 99965001, 17146629, 34044787, 33237508, 14045383, 90654836, 82532312, 33935899, 49271185, 19272365, 85571389, 95726235, 83083131, 67030811, 99226875, 57658654, 36930650, 97783876, 83269727, 68939068, 45996863, 36396314, 6986898, 76330843, 4064751, 46540998, 10597197, 66832478, 4985896, 22721500, 82897371, 68694897, 99125126, 19457589, 74614639, 82339363, 44481640, 15148031, 93057697, 83210802, 38645117, 99549373, 22405120, 4095116, 97940276, 45617087, 98462867, 96852321, 88251446, 53084256, 63372756, 62357986, 73235980, 9623492, 45237957, 66250369, 28796059, 6038457, 74137926, 85023028, 60106217, 21919959, 13862149, 49236559, 51472275, 36189527, 93515664, 59405277, 15163258, 30764367, 6525948, 69848388, 17857111, 7919588, 20681921, 55770687, 20486294, 82178706, 44844121, 62490109, 15902805, 68110330, 86460488, 83789391, 99861373, 59197747, 24826575, 64602895, 68000591, 78561158, 43045786, 42644903, 33123618, 1022677, 48088883, 30543215, 36685023, 85117092, 42307449, 99729357, 66322959, 36505482, 415901, 72738685, 29635537, 7685448, 42073124, 77694700, 18663507, 2204165, 37620363, 18783702, 99297164, 39373729, 22113133, 38022834, 52204879, 31126490, 36135, 81774825, 82651278, 90004325, 20122224, 24915585, 40781449, 28851716, 92692978, 66428795, 92033260, 14363867, 29401781, 9257405, 72373496, 55247455, 60955663, 98948034, 3294781, 59501487, 5822202, 77187825, 54606384, 36780454, 89699445, 31727379, 89078848, 84127901, 14349098, 29819940, 5832946, 71657078, 40534591, 54058788, 10453030, 59318837, 94076128, 97057187, 67793644, 50806615, 669105, 14220886, 6521313, 90090964, 92353856, 8696647, 9398733, 89637706, 62452034, 44627776, 95251277, 44842615, 60176618, 92787493, 20642888, 83533741, 2891150, 75543508, 59371804, 45790169, 49598724, 33336148, 97281847, 57241521, 58208470, 17957593, 17894977, 87160386, 78785507, 91141395, 6221471, 54762643, 88653118, 44664587, 95581843, 28787861, 30998561, 82427263, 91802888, 26114953, 9259676, 20568363, 68816413, 95010552, 88444207, 4199704, 18466635, 90061527, 32159704, 24591705, 81853704, 45794415, 78884452, 8807940, 30395570, 31733363, 40027975, 76170907, 5482538, 67031644, 30090481, 61928316, 3235882, 75777973, 47887470, 29959549, 77312810, 82979980, 37183543, 93933709, 7300047, 13468268, 30653863, 62740044, 4798568, 61815223, 32590267, 92692283, 92998591, 22766820, 29029316, 70367851, 71469330, 93562257, 73109997, 23740167, 4508700, 71316369, 16971929, 32058615, 27625735, 72732205, 24314885, 7687278, 65074535, 4069912, 56153345, 29994197, 73786944, 51466049, 14731700, 73021291, 72357096, 20140249, 61982238, 70420215, 79322415, 33201905, 50567636, 12664567, 57803235, 17386996, 17081350, 99515901, 22129328, 47738185, 44983451, 72358170, 526217, 71083565, 54517921, 68824981, 61859581, 40677414, 13470059, 94516935, 14947650, 3509435, 19939935, 3233084, 90013093, 40865610, 75128745, 48395186, 7104732, 3088684, 54199160, 42782652, 508198, 2208785, 96420429, 57248122, 7229550, 89419466, 42237907, 30366150, 84406788, 53802686, 34667286, 1197320, 21993752, 62232211, 40984766, 12416768, 64157906, 9575954, 10649306, 11543098, 41380093, 69786756, 6808825, 68644627, 59329511, 91664334, 78766358, 7182642, 16424199, 30811010, 79806380, 72777973, 18806856, 8733068, 43152977, 20645197, 86242799, 77284619, 11757872, 26744917, 26292919, 17385531, 30771409, 96318094, 45995325, 16380211, 34432810, 69697787, 65017137, 45049308, 23848565, 19376156, 17764950, 85242963, 78218845, 25842078, 38510840, 72019362, 49882705, 8055981, 66667729, 62115552, 3183975, 78602717, 47893286, 34493392, 34236719, 16595436, 53547802, 65338021, 33061250, 30787683, 8729146, 65880522, 88130087, 62552524, 569864, 16097038, 3233569, 54427233, 80014588, 78909561, 85116755, 7517032, 87720882, 37435892, 67474219, 66663942, 24168411, 39986008, 1250437, 99224392, 21397057, 34946859, 99971982, 72278539, 321665, 45075471, 39553046, 99917599, 27411561, 33797252, 26863229, 26397786, 17976208, 62430984, 94539824, 44784505, 63059024, 92541302, 32426752, 18504197, 97379790, 74452589, 10366309, 58224549, 96735716, 91990218, 17365188, 38008118, 85695762, 73222868, 54263880, 87598888, 7423788, 44847298, 37501808, 16684829, 45381876, 65851721, 48893685, 18833224, 9058407, 17068582, 16445503, 11923835, 1936762, 97011160, 68875490, 7955293, 83747892, 63015256, 88904910, 14326617, 36933038, 44915235, 4978543, 38658347 +48360198, 62496012, 17081350, 19376156, 92803766, 92787493, 54014062, 10358899, 30463802, 41245325, 90310261, 33797252, 28928175, 62490109, 15535065, 44245960, 33699435, 95957797, 37659250, 99021067, 24953498, 34946859, 14220886, 90090964, 9599614, 69136837, 94516935, 84840549, 11923835, 19898053, 75407347, 8129978, 60581278, 17016156, 91957544, 77694700, 50007421, 90004325, 38256849, 17037369, 26744917, 76330843, 94595668, 97057187, 45306070, 16387575, 68824981, 5267545, 69579137, 75128745, 3487592, 36312813, 66045587, 51590803, 17365188, 14626618, 61712234, 65715134, 38658347, 62232211, 55850790, 77301523, 40197395, 63506281, 54427233, 874791, 61741594, 48892201, 63967300, 5073754, 44889423, 70367851, 4508700, 58751351, 38022834, 92692978, 7687278, 8733068, 74441448, 20002147, 86022504, 33201905, 12664567, 26292919, 89078848, 40534591, 37891451, 66832478, 2917920, 9398733, 43501211, 93566986, 10366309, 91727510, 22435353, 18001617, 33431960, 80251430, 47361209, 88698958, 83133790, 72274002, 58749, 88653118, 58224549, 3088684, 95581843, 30694952, 33628349, 47792865, 97011160, 32159704, 24058273, 6153406, 55770687, 44844121, 68110330, 78589145, 68000591, 47213183, 30569392, 98739783, 62051033, 47887470, 37675718, 93933709, 89804152, 13468268, 80014588, 73617245, 46851987, 44847298, 80555751, 61815223, 38365584, 98371444, 69786756, 42073124, 12303248, 22766820, 42692881, 66885828, 63152504, 70541760, 56461322, 17146629, 34044787, 33237508, 5970607, 24915585, 28851716, 4119747, 49271185, 87720882, 23194618, 72373496, 18806856, 56484900, 78300864, 67474219, 67030811, 59501487, 34497327, 33565483, 56955985, 8128637, 36396314, 4064751, 21397057, 44060493, 10453030, 32151165, 40152546, 247198, 67793644, 61897582, 91255408, 2717150, 92353856, 57020481, 45049308, 74614639, 82339363, 38645117, 72725103, 48673079, 26102057, 27411561, 29510992, 17539625, 44348328, 38510840, 76315420, 8099994, 26998766, 99658235, 48395186, 93270084, 66250369, 49328608, 23569917, 68816413, 55470718, 21919959, 51472275, 36189527, 93515664, 20885148, 27325428, 81677380, 22997281, 89996536, 84166196, 6525948, 30395570, 61373987, 40027975, 8729146, 12571310, 75135448, 65880522, 22108665, 69255765, 39801351, 29932657, 86543538, 84293052, 84002370, 62740044, 72614359, 78909561, 1239555, 32590267, 72738685, 7685448, 71300104, 89214616, 32426752, 81172706, 37620363, 73109997, 6808825, 67281495, 77787724, 2607799, 96726697, 16424199, 68316156, 52060076, 30811010, 72732205, 91938191, 86798033, 50668599, 29401781, 94711842, 31491938, 14731700, 77284619, 72357096, 62762857, 55669657, 83269727, 11581181, 50702367, 96709982, 22129328, 17727650, 48774913, 30771409, 7502255, 73031054, 168541, 48893685, 68694897, 56515456, 56531125, 62452034, 36812683, 19457589, 36753250, 61960400, 71083565, 31904591, 93057697, 23848565, 36139942, 8115266, 23432750, 41092172, 22405120, 66137019, 97561745, 21001913, 33304202, 72019362, 78785507, 68102437, 65271999, 66611759, 54199160, 30998561, 96735716, 42782652, 6038457, 98943869, 66903004, 9860968, 59981773, 83150534, 36468541, 1788101, 36580610, 49236559, 6497830, 59405277, 65454636, 90061527, 20836893, 67227442, 89499542, 69605283, 21993752, 21070110, 53648154, 81898046, 22942635, 38061439, 54263880, 15064655, 55753905, 19486173, 64602895, 67031644, 3235882, 63059024, 95395112, 20765474, 38341669, 569864, 89217461, 37183543, 42967683, 58180653, 36685023, 11543098, 84904436, 2331773, 4798568, 7955293, 19101477, 415901, 57163802, 92604458, 54868730, 27187213, 7646095, 79136082, 83747892, 18783702, 86118021, 7011964, 25636669, 8791066, 89811711, 29466406, 27041967, 59910624, 7517032, 61859143, 54232247, 33935899, 65047700, 50188404, 56153345, 72777973, 29994197, 6819644, 73021291, 11757872, 74452589, 68939068, 14349098, 5832946, 79821897, 96318094, 6871053, 34432810, 99971982, 97641116, 86821229, 72278539, 74743862, 11365791, 69697787, 321665, 89637706, 94911072, 99125126, 91281584, 526217, 95251277, 45075471, 26664538, 82327024, 84349107, 59109400, 83651211, 99549373, 20642888, 16405341, 9058407, 4095116, 79942022, 4434662, 49598724, 45667668, 97281847, 44473167, 8634541, 21533347, 58208470, 92215320, 84684495, 98462867, 96852321, 19939935, 17957593, 71965942, 35092039, 17068582, 43357947, 44664587, 68702632, 3880712, 91802888, 78602717, 20568363, 70782102, 61271144, 88444207, 26397786, 91990218, 13862149, 84406788, 63628376, 22200378, 4978543, 9860195, 15948937, 18466635, 34493392, 62430984, 94539824, 64098930, 26734892, 24591705, 3633375, 32699744, 65338021, 45794415, 78884452, 73222868, 40984766, 33061250, 3773993, 30787683, 1569515, 70004753, 83789391, 37224844, 24826575, 5482538, 62552524, 92071990, 66116458, 86301513, 10649306, 13819358, 77312810, 6111563, 43933006, 39171895, 26275734, 42199455, 82052050, 8913721, 99729357, 30653863, 65081429, 36845587, 36505482, 75986488, 45428665, 29188588, 16099750, 18411915, 70372191, 59177718, 43322743, 47708850, 7066775, 41442762, 71316369, 43376279, 74357852, 88047921, 85711894, 41481685, 36135, 32058615, 57359924, 74110882, 27665211, 24314885, 66428795, 4069912, 72238278, 40356877, 12024238, 37435892, 20645197, 1431742, 86242799, 40781854, 4668450, 3294781, 824872, 66663942, 79322415, 37166608, 84127901, 17386996, 83155442, 82726008, 46540998, 65851721, 50806615, 4985896, 44550764, 8696647, 5111370, 44627776, 39553046, 79191827, 99917599, 40677414, 90158785, 51588015, 76540481, 70800879, 35996293, 75543508, 80316608, 45790169, 93053405, 26493119, 13173644, 81855445, 57961282, 75820087, 69641513, 17894977, 93359396, 88251446, 28550822, 53084256, 91141395, 6221471, 73235980, 45237957, 66667729, 28796059, 26776922, 96420429, 60430369, 55602660, 7229550, 91831487, 85023028, 14093520, 14326617, 36933038, 67084351, 42237907, 57393458, 92398073, 77272628, 59614746, 54663246, 14723410, 69848388, 81853704, 73392814, 20486294, 35456853, 82178706, 80246713, 15902805, 24619760, 44177011, 87598888, 30090481, 98648327, 45990383, 9575954, 44784505, 52613508, 92554120, 31161687, 92867155, 62803858, 23134715, 16097038, 60102965, 85117092, 76703609, 26063929, 51507425, 79880247, 55615885, 92541302, 9175338, 48658605, 96906410, 112651, 2204165, 71469330, 36808486, 51464002, 23740167, 56473732, 47090124, 68644627, 59329511, 52204879, 91664334, 14045383, 49641577, 82651278, 4091162, 20122224, 90654836, 69355476, 79806380, 1204161, 19272365, 92033260, 44479073, 68204242, 9257405, 45919976, 16567550, 1375023, 99524975, 43152977, 83083131, 98948034, 30218878, 20140249, 96193415, 77072625, 9603598, 97783876, 54606384, 99333375, 37192445, 15536795, 77413012, 40686254, 1250437, 99224392, 82779622, 29819940, 53888755, 67124282, 62693428, 65017137, 9886593, 61141874, 29065964, 65038678, 43506672, 44481640, 24733232, 91240048, 83210802, 17764950, 83533741, 14947650, 97940276, 2891150, 26139110, 59371804, 68041839, 33336148, 30139692, 94090109, 78218845, 19891772, 3233084, 87160386, 61623915, 10961421, 54762643, 62357986, 508198, 2208785, 74137926, 81805959, 78549759, 51315460, 75052463, 60106217, 89419466, 42947632, 95010552, 37280276, 64848072, 9829782, 70036438, 4199704, 28734791, 10309525, 34667286, 72495719, 15163258, 30764367, 7919588, 20681921, 53393358, 70596786, 85695762, 30891921, 8807940, 38009615, 86460488, 99604946, 176257, 31733363, 99861373, 93790285, 59197747, 64157906, 61928316, 43045786, 68875490, 42644903, 61728685, 33123618, 21289531, 29959549, 61263068, 33553959, 82979980, 94360702, 4204661, 3233569, 53666583, 17058722, 42307449, 48260151, 66322959, 62936963, 35192533, 77620120, 85116755, 51047803, 12348118, 29834512, 18131876, 78766358, 16054533, 81774825, 71920426, 82532312, 65074535, 16684829, 73786944, 83302115, 60393039, 47298834, 64055960, 60955663, 5822202, 99226875, 56424103, 62428472, 31727379, 45848907, 57240218, 6986898, 99515901, 59318837, 54987042, 44983451, 72358170, 88904910, 77300457, 32274392, 92530431, 15148031, 61859581, 60176618, 13470059, 4787945, 35512853, 4806458, 45407418, 85242963, 45617087, 29516592, 57241521, 3509435, 86001008, 90013093, 16445503, 49882705, 75153252, 34698428, 63372756, 28787861, 79657802, 53842979, 79922758, 3183975, 1936762, 81274566, 75229462, 26392416, 30366150, 47893286, 17976208, 6157724, 11161731, 53802686, 32250045, 34236719, 1197320, 15015906, 16595436, 12416768, 76434144, 13348726, 7423788, 75777973, 73168565, 1022677, 7300047, 48088883, 98653983, 30543215, 76671482, 66271566, 34698463, 37739481, 49597667, 73124510, 39847321, 29635537, 9188443, 99690194, 92998591, 33249630, 71522968, 18663507, 6505939, 18504197, 55960386, 99965001, 4099191, 39373729, 22113133, 16971929, 7182642, 97379790, 77898274, 27625735, 63015256, 76960303, 85571389, 95726235, 51466049, 55247455, 66319530, 61982238, 70420215, 57658654, 36930650, 41092102, 24168411, 89699445, 50567636, 53632373, 57803235, 54058788, 45995325, 10597197, 6521313, 18833224, 54517921, 83368048, 90272749, 26863229, 40865610, 7104732, 4515343, 95290172, 749283, 48675329, 44915235, 67451935, 37957788, 38008118, 98130363, 53547802, 21673260, 89046466, 70727211, 79738755, 77377183, 5640302, 82886381, 72793444, 55189057, 51149804, 16019925, 26507214, 41380093, 92692283, 34295794, 37501808, 93562257, 31126490, 18699206, 39201414, 77187825, 87710366, 67513640, 17385531, 52261574, 30163921, 669105, 22721500, 82897371, 44846932, 76825057, 8373289, 13231279, 51715482, 22450468, 8055981, 82427263, 26114953, 38494874, 8651647, 17857111, 22879907, 64087743, 88130087, 78561158, 83948335, 23110625, 6793819, 51426311, 99297164, 55143724, 40781449, 10264691, 36780454, 84187166, 45381876, 45996863, 46870723, 47738185, 71657078, 16380211, 44842615, 27185644, 33895336, 62115552, 9259676, 68128438, 76170907, 52293995, 37560164, 29029316, 14363867, 39986008, 94076128, 32161669, 90457870, 9623492, 65275241, 74724075, 25842078, 15075176, 20867149, 71333116, 57248122 +95726235, 99524975, 44481640, 83651211, 64848072, 37192445, 69641513, 40865610, 36933038, 70036438, 32159704, 61928316, 77072625, 52261574, 2917920, 5267545, 78785507, 66667729, 53393358, 40027975, 64602895, 29932657, 48088883, 85117092, 54427233, 15535065, 16099750, 58751351, 22113133, 38022834, 78766358, 36780454, 38645117, 45407418, 17764950, 45667668, 90457870, 54762643, 68702632, 42947632, 95010552, 14723410, 53547802, 12416768, 5482538, 63059024, 39801351, 61263068, 16097038, 7300047, 55189057, 51149804, 76703609, 13468268, 92692283, 96906410, 12303248, 18504197, 41442762, 29401781, 73786944, 87710366, 21397057, 30771409, 62693428, 62452034, 19457589, 54517921, 23848565, 83210802, 79942022, 75153252, 53084256, 44664587, 36312813, 96735716, 51315460, 55470718, 6157724, 30891921, 68110330, 68000591, 62552524, 92867155, 72793444, 2331773, 415901, 32590267, 27187213, 37620363, 36808486, 51426311, 18131876, 49641577, 81774825, 66428795, 14363867, 4069912, 72238278, 9257405, 16567550, 56484900, 96193415, 36930650, 56955985, 8128637, 6986898, 32151165, 94076128, 53888755, 22721500, 5111370, 88904910, 526217, 26664538, 44842615, 93057697, 59109400, 91727510, 33797252, 94090109, 13231279, 44348328, 84684495, 16445503, 28928175, 66250369, 95581843, 1936762, 37280276, 94539824, 17857111, 24058273, 70596786, 89499542, 53648154, 15902805, 54263880, 76434144, 77377183, 22108665, 8129978, 42644903, 37675718, 77312810, 93933709, 94360702, 26275734, 66271566, 51507425, 79880247, 26507214, 36845587, 7955293, 1239555, 38365584, 85116755, 71300104, 63967300, 33249630, 70367851, 18783702, 86118021, 34044787, 2607799, 96726697, 16054533, 74110882, 18699206, 63015256, 76960303, 51466049, 60393039, 31491938, 60955663, 4668450, 6819644, 57658654, 37166608, 33201905, 11581181, 31727379, 84127901, 17081350, 17385531, 99515901, 22129328, 34946859, 40534591, 247198, 99971982, 73031054, 50806615, 72278539, 91255408, 74743862, 82897371, 8696647, 56531125, 94911072, 61141874, 65038678, 61960400, 71083565, 45075471, 92530431, 82327024, 61859581, 36139942, 91240048, 76825057, 35512853, 99549373, 48673079, 33336148, 8634541, 13173644, 19891772, 57961282, 88698958, 76315420, 26998766, 75128745, 91141395, 88653118, 45237957, 33895336, 81805959, 9259676, 68816413, 60106217, 21919959, 10358899, 49236559, 47893286, 48675329, 22997281, 84166196, 45794415, 85695762, 40984766, 86460488, 64087743, 99604946, 61373987, 70004753, 176257, 83789391, 37224844, 75135448, 64157906, 45990383, 69255765, 92554120, 62051033, 47887470, 39171895, 4204661, 98653983, 30543215, 76671482, 11543098, 84293052, 84904436, 92541302, 75986488, 9188443, 59177718, 32426752, 22766820, 42692881, 29029316, 71522968, 2204165, 71469330, 79136082, 56461322, 25636669, 39373729, 29834512, 31126490, 7182642, 14045383, 68316156, 90004325, 82532312, 19272365, 83302115, 74441448, 1431742, 3294781, 9603598, 62762857, 84187166, 45381876, 26292919, 89078848, 53632373, 82726008, 4064751, 47738185, 40152546, 66832478, 669105, 14220886, 68694897, 321665, 44846932, 91281584, 36753250, 16387575, 95251277, 43506672, 74614639, 79191827, 93566986, 9599614, 90310261, 8115266, 4806458, 22405120, 76540481, 83533741, 94516935, 97940276, 66137019, 59371804, 80316608, 8373289, 17539625, 3509435, 86001008, 19939935, 71965942, 38510840, 27185644, 90013093, 17894977, 84840549, 28550822, 61623915, 65271999, 66611759, 7104732, 93270084, 3088684, 79657802, 49328608, 57248122, 78602717, 38494874, 8651647, 98943869, 81274566, 75229462, 83150534, 95290172, 70782102, 61271144, 91990218, 57393458, 54014062, 4199704, 93515664, 67451935, 17976208, 77272628, 34493392, 69848388, 19898053, 55770687, 21070110, 35456853, 8807940, 38009615, 33061250, 99861373, 30090481, 9575954, 78561158, 92071990, 75407347, 5640302, 68875490, 98739783, 17016156, 29959549, 58180653, 82052050, 99729357, 26063929, 66322959, 73617245, 55615885, 4798568, 72614359, 78909561, 36505482, 19101477, 48892201, 73124510, 41380093, 72738685, 9175338, 18411915, 73109997, 6505939, 41245325, 70541760, 55960386, 50007421, 7011964, 33237508, 5970607, 95957797, 77787724, 43376279, 91664334, 85711894, 41481685, 32058615, 52060076, 24915585, 27665211, 24314885, 49271185, 86798033, 92033260, 24953498, 55247455, 20645197, 20002147, 30218878, 73021291, 99226875, 11757872, 74452589, 86022504, 68939068, 39986008, 57240218, 1250437, 29819940, 5832946, 10597197, 65851721, 168541, 97641116, 48893685, 67124282, 56515456, 9886593, 45049308, 82339363, 15148031, 84349107, 23432750, 27411561, 14947650, 75543508, 22435353, 4434662, 49598724, 18001617, 29516592, 57241521, 80251430, 21533347, 96852321, 17957593, 8099994, 49882705, 58749, 6221471, 66045587, 91802888, 26114953, 23569917, 33628349, 66903004, 89419466, 88444207, 26397786, 26392416, 30366150, 22200378, 36189527, 44915235, 37957788, 4978543, 15075176, 10309525, 81677380, 1197320, 7919588, 67227442, 3633375, 69605283, 81898046, 22942635, 21673260, 38061439, 65880522, 88130087, 13348726, 47213183, 7423788, 65275241, 55850790, 60581278, 89217461, 3233569, 40197395, 42307449, 48260151, 65081429, 80555751, 62936963, 91957544, 77620120, 34295794, 45428665, 39847321, 48658605, 69786756, 112651, 92998591, 12348118, 18663507, 37501808, 51464002, 67281495, 56473732, 47090124, 68644627, 59329511, 71333116, 16971929, 27041967, 52204879, 97379790, 16424199, 7517032, 82651278, 61859143, 40781449, 30811010, 28851716, 27625735, 54232247, 85571389, 40356877, 45919976, 94711842, 47298834, 64055960, 67474219, 98948034, 62496012, 59501487, 77187825, 55669657, 62428472, 57803235, 50702367, 40686254, 96709982, 76330843, 14349098, 46870723, 17727650, 48774913, 71657078, 54058788, 79821897, 59318837, 94595668, 30163921, 54987042, 92353856, 69697787, 89637706, 99125126, 29065964, 18833224, 83368048, 99917599, 19376156, 69136837, 4787945, 92787493, 41092172, 16405341, 4095116, 70800879, 93053405, 68041839, 26493119, 45617087, 97561745, 90272749, 44473167, 78218845, 69579137, 92215320, 98462867, 26863229, 33304202, 17068582, 22450468, 88251446, 28787861, 28796059, 54199160, 3880712, 2208785, 96420429, 55602660, 7229550, 3183975, 78549759, 13862149, 749283, 51472275, 28734791, 20885148, 34667286, 32250045, 27325428, 72495719, 17365188, 97011160, 14626618, 62430984, 89996536, 30764367, 34236719, 15015906, 73392814, 78884452, 82178706, 44844121, 80246713, 44177011, 70727211, 87598888, 31733363, 8729146, 19486173, 48360198, 44784505, 66116458, 43045786, 10649306, 20765474, 31161687, 62803858, 75777973, 77301523, 82979980, 86543538, 16019925, 874791, 80014588, 30653863, 46851987, 62740044, 44847298, 35192533, 61815223, 61741594, 49597667, 29635537, 51047803, 92604458, 99690194, 43322743, 44889423, 81172706, 6808825, 7066775, 99965001, 4099191, 33699435, 99297164, 29466406, 74357852, 77898274, 57359924, 4091162, 90654836, 69355476, 1204161, 50668599, 72373496, 29994197, 39201414, 37435892, 78300864, 86242799, 14731700, 66319530, 77284619, 20140249, 5822202, 66663942, 97783876, 38256849, 41092102, 54606384, 24168411, 83269727, 99333375, 17037369, 26744917, 67513640, 45848907, 45996863, 12664567, 77413012, 17386996, 44060493, 46540998, 44983451, 4985896, 90090964, 65017137, 77300457, 31904591, 32161669, 10366309, 51588015, 26139110, 30139692, 83133790, 35092039, 72274002, 72019362, 68102437, 99658235, 63372756, 58224549, 73235980, 60430369, 79922758, 30694952, 20568363, 75052463, 91831487, 14093520, 14326617, 63628376, 6497830, 11161731, 18466635, 51590803, 54663246, 64098930, 20836893, 24591705, 81853704, 65715134, 21993752, 62232211, 79738755, 93790285, 12571310, 55753905, 59197747, 24826575, 78589145, 98648327, 30569392, 95395112, 569864, 23134715, 42967683, 6111563, 60102965, 53666583, 89804152, 17058722, 36685023, 52293995, 84002370, 37739481, 23110625, 29188588, 44245960, 93562257, 63152504, 8791066, 89811711, 36135, 72732205, 37659250, 71920426, 99021067, 68204242, 10264691, 18806856, 67030811, 72357096, 824872, 61982238, 56424103, 34497327, 79322415, 99224392, 45995325, 37891451, 16380211, 34432810, 67793644, 86821229, 2717150, 9398733, 36812683, 92803766, 60176618, 90158785, 85242963, 33431960, 25842078, 21001913, 51715482, 93359396, 10961421, 34698428, 53842979, 26776922, 42782652, 82427263, 4515343, 6038457, 62115552, 47792865, 36468541, 1788101, 42237907, 36580610, 9829782, 9860195, 59405277, 53802686, 15948937, 68128438, 15163258, 59614746, 38008118, 6525948, 98130363, 61712234, 65338021, 73222868, 89046466, 30787683, 1569515, 20867149, 67031644, 52613508, 86301513, 13819358, 61728685, 82886381, 33123618, 21289531, 37183543, 43933006, 42199455, 34698463, 83948335, 6793819, 98371444, 7685448, 42073124, 66885828, 5073754, 77694700, 7646095, 83747892, 23740167, 17146629, 59910624, 4119747, 33935899, 44479073, 16684829, 72777973, 8733068, 1375023, 40781854, 33565483, 50567636, 82779622, 7502255, 10453030, 96318094, 6871053, 61897582, 6521313, 44550764, 57020481, 43501211, 68824981, 13470059, 72725103, 26102057, 35996293, 2891150, 45790169, 97281847, 29510992, 47361209, 81855445, 58208470, 75820087, 3233084, 87160386, 11923835, 48395186, 9623492, 30998561, 74137926, 90061527, 16595436, 6153406, 38658347, 20486294, 22879907, 62490109, 3773993, 3235882, 38341669, 33553959, 8913721, 63506281, 30463802, 37560164, 57163802, 70372191, 74724075, 47708850, 88047921, 20122224, 65074535, 50188404, 56153345, 87720882, 12024238, 43152977, 83083131, 97057187, 45306070, 72358170, 39553046, 20642888, 9058407, 3487592, 62357986, 85023028, 67084351, 26734892, 32699744, 30395570, 15064655, 1022677, 89214616, 4508700, 71316369, 79806380, 91938191, 7687278, 89699445, 83155442, 11365791, 32274392, 24733232, 43357947, 8055981, 84406788, 92398073, 76170907, 73168565, 54868730, 55143724, 92692978, 65047700, 70420215, 15536795, 36396314, 44627776, 9860968, 20681921, 508198, 59981773, 65454636, 24619760, 23194618, 40677414 +75986488, 74137926, 43322743, 4099191, 7517032, 92787493, 19101477, 25636669, 50668599, 7919588, 52613508, 38341669, 72614359, 29029316, 16971929, 16424199, 1431742, 8128637, 17081350, 5832946, 5111370, 77300457, 78218845, 40865610, 88653118, 54199160, 59614746, 176257, 75135448, 82052050, 91957544, 27187213, 63152504, 67281495, 29466406, 76960303, 37166608, 17727650, 37891451, 69697787, 321665, 48673079, 26493119, 30139692, 19891772, 17894977, 90457870, 84840549, 88251446, 66667729, 36580610, 15948937, 65715134, 24058273, 64602895, 8129978, 63059024, 39171895, 98653983, 62740044, 77620120, 89214616, 34044787, 59329511, 83302115, 94711842, 62762857, 82726008, 21397057, 34432810, 30163921, 66832478, 89637706, 94911072, 29065964, 57020481, 45075471, 5267545, 84349107, 38645117, 83651211, 4787945, 35512853, 66137019, 22435353, 92215320, 63372756, 79922758, 78549759, 20568363, 59405277, 73222868, 33061250, 5482538, 92554120, 65275241, 62803858, 1022677, 48260151, 44847298, 41380093, 72738685, 71300104, 44889423, 41442762, 58751351, 71333116, 29834512, 18131876, 88047921, 27625735, 66428795, 16684829, 29994197, 1375023, 37435892, 83083131, 38256849, 87710366, 37192445, 6986898, 52261574, 59318837, 94595668, 67793644, 97641116, 53888755, 669105, 11365791, 36753250, 26664538, 93057697, 60176618, 76825057, 13470059, 94516935, 91727510, 80316608, 97281847, 75820087, 93359396, 49882705, 99658235, 58224549, 44664587, 93270084, 79657802, 66045587, 60430369, 57248122, 38494874, 66903004, 1788101, 13862149, 51472275, 6497830, 93515664, 28734791, 20885148, 54663246, 78884452, 55770687, 20486294, 8807940, 3773993, 87598888, 76170907, 19486173, 65880522, 39801351, 20765474, 73168565, 4204661, 53666583, 85117092, 34698463, 76703609, 874791, 73617245, 30463802, 48892201, 32590267, 92692283, 29188588, 48658605, 70372191, 32426752, 112651, 2204165, 7646095, 36808486, 33237508, 85711894, 57359924, 4091162, 30811010, 4119747, 7687278, 14363867, 72777973, 18806856, 24953498, 20645197, 74441448, 3294781, 77187825, 33565483, 67513640, 12664567, 77413012, 84127901, 50702367, 40686254, 17385531, 1250437, 99224392, 71657078, 40534591, 45995325, 61897582, 54987042, 91255408, 82897371, 68694897, 2917920, 62693428, 19457589, 16387575, 61960400, 54517921, 79191827, 93566986, 44481640, 23848565, 36139942, 90310261, 23432750, 4806458, 22405120, 45790169, 18001617, 97561745, 8634541, 47361209, 21533347, 3509435, 27185644, 76315420, 28928175, 66611759, 3088684, 95581843, 59981773, 42947632, 55470718, 30366150, 4199704, 34667286, 77272628, 84166196, 34236719, 6525948, 26734892, 81853704, 16595436, 53547802, 38658347, 44844121, 80246713, 38061439, 30395570, 24619760, 61373987, 1569515, 93790285, 40027975, 12571310, 76434144, 68000591, 69255765, 92867155, 37675718, 23134715, 86543538, 60102965, 48088883, 40197395, 30543215, 36685023, 76671482, 30653863, 36505482, 23110625, 15535065, 85116755, 16099750, 92998591, 33249630, 77694700, 47708850, 71469330, 37620363, 79136082, 7066775, 56461322, 47090124, 17146629, 39373729, 38022834, 43376279, 74357852, 59910624, 61859143, 37659250, 54232247, 49271185, 1204161, 92033260, 4069912, 29401781, 72238278, 72373496, 85571389, 39201414, 86242799, 20002147, 66663942, 56424103, 97783876, 86022504, 84187166, 56955985, 53632373, 17386996, 96709982, 46870723, 47738185, 34946859, 86821229, 74743862, 62452034, 65017137, 44846932, 61141874, 44627776, 36812683, 526217, 43501211, 83368048, 31904591, 32161669, 9599614, 9058407, 76540481, 79942022, 35996293, 26139110, 33431960, 94090109, 58208470, 57961282, 44348328, 72274002, 16445503, 28550822, 58749, 10961421, 62357986, 508198, 82427263, 91802888, 55602660, 8651647, 9860968, 75229462, 95290172, 36933038, 10358899, 37280276, 22200378, 749283, 67451935, 15075176, 92398073, 72495719, 68128438, 62430984, 89996536, 38008118, 67227442, 6153406, 19898053, 30891921, 64087743, 48360198, 30090481, 92071990, 31161687, 62051033, 55850790, 29932657, 33553959, 42967683, 72793444, 42199455, 80014588, 84293052, 84904436, 84002370, 73124510, 29635537, 57163802, 7685448, 51047803, 92604458, 96906410, 54868730, 99690194, 69786756, 22766820, 41245325, 86118021, 7011964, 8791066, 2607799, 78766358, 81774825, 82532312, 27665211, 18699206, 44479073, 16567550, 56484900, 60955663, 40781854, 67030811, 77284619, 20140249, 34497327, 11757872, 17037369, 50567636, 45848907, 57240218, 96318094, 247198, 94076128, 16380211, 50806615, 44983451, 22721500, 2717150, 9398733, 99125126, 9886593, 88904910, 95251277, 32274392, 15148031, 10366309, 40677414, 59109400, 72725103, 45407418, 93053405, 45617087, 44473167, 17539625, 69641513, 84684495, 96852321, 86001008, 19939935, 3233084, 38510840, 21001913, 8099994, 22450468, 87160386, 78785507, 75153252, 3487592, 65271999, 6221471, 54762643, 9623492, 66250369, 36312813, 28787861, 53842979, 26776922, 96735716, 2208785, 62115552, 81805959, 26114953, 33628349, 95010552, 36468541, 70782102, 26397786, 26392416, 67084351, 54014062, 84406788, 63628376, 49236559, 36189527, 44915235, 4978543, 6157724, 65454636, 53802686, 81677380, 34493392, 14723410, 1197320, 24591705, 70596786, 73392814, 69605283, 21070110, 62232211, 81898046, 21673260, 38009615, 54263880, 89046466, 30787683, 70004753, 83789391, 99861373, 37224844, 15064655, 67031644, 98648327, 22108665, 61928316, 9575954, 47213183, 75407347, 10649306, 13819358, 33123618, 17016156, 43933006, 7300047, 89804152, 58180653, 66271566, 52293995, 99729357, 26063929, 54427233, 55615885, 26507214, 80555751, 37739481, 415901, 83948335, 34295794, 45428665, 39847321, 6793819, 9175338, 12303248, 42692881, 5073754, 44245960, 70367851, 51464002, 23740167, 99965001, 18783702, 33699435, 89811711, 22113133, 68644627, 95957797, 27041967, 52204879, 41481685, 36135, 82651278, 20122224, 28851716, 91938191, 9257405, 73786944, 14731700, 66319530, 9603598, 74452589, 55669657, 54606384, 24168411, 36780454, 33201905, 89699445, 83155442, 76330843, 48774913, 44060493, 29819940, 46540998, 6871053, 10597197, 99971982, 168541, 72278539, 56531125, 72358170, 91281584, 45049308, 43506672, 24733232, 61859581, 19376156, 91240048, 92803766, 41092172, 4095116, 97940276, 33797252, 49598724, 90272749, 57241521, 80251430, 33304202, 43357947, 8055981, 45237957, 28796059, 3880712, 4515343, 7229550, 75052463, 98943869, 91831487, 85023028, 68816413, 89419466, 83150534, 21919959, 88444207, 57393458, 64848072, 48675329, 70036438, 9860195, 27325428, 14626618, 64098930, 69848388, 98130363, 3633375, 65338021, 89499542, 53648154, 22879907, 15902805, 40984766, 12416768, 99604946, 20867149, 55753905, 13348726, 45990383, 3235882, 78561158, 86301513, 7423788, 43045786, 68875490, 60581278, 89217461, 37183543, 6111563, 26275734, 51149804, 42307449, 63506281, 13468268, 51507425, 79880247, 46851987, 36845587, 78909561, 61741594, 1239555, 92541302, 18411915, 59177718, 66885828, 74724075, 81172706, 37501808, 73109997, 83747892, 50007421, 99297164, 77787724, 55143724, 49641577, 69355476, 71920426, 65074535, 63015256, 50188404, 56153345, 99021067, 95726235, 51466049, 8733068, 99524975, 4668450, 824872, 62496012, 5822202, 99226875, 70420215, 36930650, 83269727, 11581181, 68939068, 39986008, 89078848, 4064751, 7502255, 79821897, 97057187, 65851721, 4985896, 90090964, 45306070, 65038678, 39553046, 99917599, 68824981, 82327024, 69136837, 83210802, 20642888, 83133790, 51715482, 26998766, 68102437, 53084256, 34698428, 42782652, 96420429, 78602717, 51315460, 60106217, 47792865, 14326617, 91990218, 47893286, 37957788, 32250045, 90061527, 51590803, 15163258, 20836893, 20681921, 21993752, 22942635, 62490109, 86460488, 8729146, 59197747, 88130087, 77377183, 44784505, 30569392, 66116458, 98739783, 95395112, 42644903, 61728685, 82886381, 21289531, 569864, 82979980, 93933709, 17058722, 55189057, 2331773, 4798568, 35192533, 49597667, 18663507, 70541760, 51426311, 55960386, 4508700, 31126490, 14045383, 16054533, 68316156, 52060076, 90654836, 24314885, 33935899, 19272365, 68204242, 40356877, 43152977, 64055960, 30218878, 72357096, 59501487, 77072625, 57658654, 41092102, 31727379, 45996863, 15536795, 36396314, 57803235, 82779622, 30771409, 32151165, 71083565, 92530431, 44842615, 8115266, 51588015, 17764950, 16405341, 70800879, 85242963, 14947650, 59371804, 68041839, 13173644, 81855445, 13231279, 71965942, 17068582, 7104732, 33895336, 30998561, 6038457, 14093520, 61271144, 42237907, 10309525, 18466635, 97011160, 22997281, 94539824, 32159704, 17857111, 61712234, 32699744, 35456853, 82178706, 68110330, 24826575, 78589145, 62552524, 5640302, 75777973, 16097038, 3233569, 11543098, 8913721, 16019925, 65081429, 7955293, 9188443, 98371444, 42073124, 12348118, 93562257, 6505939, 56473732, 71316369, 5970607, 96726697, 97379790, 32058615, 77898274, 24915585, 72732205, 92692978, 74110882, 79806380, 86798033, 45919976, 12024238, 60393039, 55247455, 67474219, 96193415, 99333375, 26744917, 45381876, 10453030, 14220886, 6521313, 48893685, 8696647, 67124282, 18833224, 82339363, 90158785, 26102057, 27411561, 75543508, 33336148, 45667668, 29510992, 88698958, 26863229, 25842078, 72019362, 11923835, 48395186, 68702632, 49328608, 23569917, 81274566, 9829782, 17365188, 30764367, 15015906, 45794415, 85695762, 79738755, 64157906, 47887470, 61263068, 77312810, 94360702, 38365584, 71522968, 6808825, 18504197, 90004325, 40781449, 65047700, 87720882, 10264691, 47298834, 78300864, 6819644, 98948034, 73021291, 61982238, 99515901, 54058788, 40152546, 73031054, 44550764, 92353856, 56515456, 74614639, 99549373, 2891150, 29516592, 8373289, 98462867, 17957593, 35092039, 75128745, 73235980, 3183975, 9259676, 1936762, 11161731, 53393358, 31733363, 29959549, 66322959, 61815223, 91664334, 23194618, 79322415, 62428472, 26292919, 22129328, 14349098, 4434662, 69579137, 90013093, 61623915, 91141395, 44177011, 70727211, 37560164, 63967300, 7182642, 30694952, 77301523, 62936963, 31491938, 17976208, 83533741 +3294781, 66611759, 43501211, 47361209, 49328608, 6038457, 19898053, 89214616, 50188404, 72373496, 94911072, 31904591, 61859581, 91727510, 8099994, 40865610, 48395186, 8055981, 96735716, 14626618, 1197320, 21993752, 31733363, 67031644, 47213183, 61263068, 77301523, 42967683, 5970607, 27041967, 91938191, 24953498, 61982238, 57658654, 22129328, 21397057, 96318094, 2917920, 99917599, 5267545, 36139942, 19376156, 41092172, 57241521, 71965942, 38510840, 78785507, 53084256, 75128745, 91802888, 55470718, 26734892, 86460488, 48360198, 78589145, 9575954, 63059024, 43045786, 43933006, 89804152, 76703609, 13468268, 54427233, 9175338, 22766820, 37620363, 20122224, 30811010, 92033260, 51466049, 37435892, 20645197, 46870723, 94076128, 97057187, 44983451, 11365791, 19457589, 36753250, 77300457, 82327024, 85242963, 94516935, 29516592, 8634541, 44348328, 51315460, 67084351, 42237907, 13862149, 28734791, 11161731, 51590803, 24591705, 24058273, 78884452, 85695762, 20486294, 44177011, 13348726, 569864, 23134715, 34698463, 99729357, 874791, 80555751, 83948335, 91957544, 70372191, 42692881, 5073754, 70541760, 43376279, 16424199, 41481685, 82532312, 56153345, 14731700, 36930650, 33201905, 8128637, 26292919, 32151165, 247198, 16380211, 48893685, 99125126, 71083565, 68824981, 26102057, 4434662, 45667668, 75820087, 3509435, 84684495, 21001913, 43357947, 72019362, 62357986, 9623492, 36312813, 54199160, 3183975, 26114953, 9259676, 66903004, 95290172, 26392416, 36580610, 57393458, 37280276, 20885148, 27325428, 62430984, 16595436, 38658347, 61373987, 75135448, 24826575, 76434144, 92071990, 31161687, 29932657, 47887470, 77312810, 7300047, 3233569, 98653983, 79880247, 73617245, 62740044, 36845587, 38365584, 15535065, 9188443, 48658605, 79136082, 18783702, 50007421, 56461322, 47090124, 7011964, 89811711, 22113133, 38022834, 32058615, 92692978, 27665211, 63015256, 76960303, 68204242, 16567550, 95726235, 8733068, 99524975, 43152977, 1431742, 40781854, 20002147, 34497327, 11581181, 99333375, 33565483, 17037369, 50567636, 67513640, 57240218, 99515901, 34946859, 40534591, 79821897, 94595668, 2717150, 56515456, 18833224, 74614639, 90310261, 23432750, 4787945, 16405341, 76540481, 26139110, 90272749, 33431960, 80251430, 69579137, 96852321, 27185644, 33304202, 35092039, 28550822, 10961421, 45237957, 68702632, 79657802, 26776922, 30998561, 96420429, 55602660, 78602717, 68816413, 47792865, 36468541, 36933038, 88444207, 17976208, 9860195, 10309525, 18466635, 68128438, 34493392, 30764367, 81853704, 65715134, 6153406, 53393358, 62232211, 30891921, 80246713, 8807940, 30395570, 176257, 87598888, 62552524, 52613508, 78561158, 13819358, 17016156, 86543538, 4204661, 17058722, 82052050, 36685023, 76671482, 66271566, 55615885, 72614359, 62936963, 415901, 92541302, 39847321, 7685448, 54868730, 92998591, 12303248, 44889423, 73109997, 6505939, 67281495, 23740167, 99965001, 25636669, 17146629, 59329511, 29466406, 14045383, 72732205, 4119747, 71920426, 33935899, 49271185, 7687278, 1204161, 19272365, 99021067, 23194618, 72777973, 29994197, 83302115, 60393039, 31491938, 4668450, 6819644, 67030811, 73021291, 59501487, 77072625, 9603598, 77187825, 68939068, 15536795, 12664567, 6986898, 96709982, 76330843, 47738185, 40152546, 34432810, 168541, 50806615, 90090964, 8696647, 68694897, 45306070, 56531125, 29065964, 57020481, 45049308, 526217, 43506672, 93566986, 26664538, 59109400, 60176618, 8115266, 13470059, 35512853, 4806458, 48673079, 59371804, 80316608, 18001617, 21533347, 17068582, 76315420, 16445503, 84840549, 28928175, 75153252, 34698428, 66045587, 82427263, 23569917, 8651647, 98943869, 83150534, 54014062, 64848072, 9829782, 48675329, 4199704, 53802686, 32250045, 90061527, 17365188, 97011160, 15163258, 54663246, 98130363, 20681921, 3633375, 73392814, 55770687, 21673260, 15902805, 38009615, 70004753, 99861373, 15064655, 8729146, 64157906, 68000591, 66116458, 8129978, 7423788, 10649306, 98739783, 92867155, 61728685, 1022677, 37183543, 93933709, 6111563, 39171895, 26275734, 72793444, 30543215, 11543098, 26063929, 2331773, 36505482, 77620120, 72738685, 45428665, 29635537, 57163802, 85116755, 51047803, 96906410, 99690194, 32426752, 66885828, 12348118, 74724075, 70367851, 83747892, 55960386, 56473732, 4508700, 33237508, 77787724, 91664334, 78766358, 88047921, 16054533, 82651278, 4091162, 27625735, 79806380, 66428795, 4069912, 16684829, 50668599, 72238278, 40356877, 45919976, 96193415, 97783876, 38256849, 87710366, 83269727, 89699445, 26744917, 39986008, 84127901, 17386996, 83155442, 1250437, 14349098, 30771409, 10453030, 99971982, 61897582, 22721500, 9398733, 67124282, 5111370, 62693428, 9886593, 44627776, 32274392, 39553046, 15148031, 44842615, 83210802, 17764950, 97940276, 66137019, 33336148, 44473167, 29510992, 17539625, 58208470, 92215320, 57961282, 88698958, 17894977, 90457870, 68102437, 3487592, 58749, 88653118, 66667729, 81805959, 30694952, 60106217, 59981773, 75229462, 14326617, 30366150, 84406788, 63628376, 51472275, 37957788, 92398073, 34667286, 77272628, 81677380, 22997281, 89996536, 59614746, 34236719, 38008118, 6525948, 17857111, 7919588, 20836893, 32699744, 65338021, 45794415, 89499542, 82178706, 44844121, 12416768, 99604946, 33061250, 24619760, 30787683, 93790285, 40027975, 76170907, 88130087, 22108665, 61928316, 68875490, 92554120, 20765474, 29959549, 37675718, 94360702, 48088883, 58180653, 66322959, 84293052, 84904436, 4798568, 44847298, 78909561, 35192533, 30463802, 48892201, 73124510, 92692283, 34295794, 75986488, 71300104, 69786756, 42073124, 63967300, 7646095, 37501808, 6808825, 86118021, 58751351, 68644627, 18131876, 74357852, 81774825, 61859143, 37659250, 86798033, 14363867, 85571389, 39201414, 94711842, 55247455, 74441448, 78300864, 86242799, 66319530, 98948034, 66663942, 62762857, 45996863, 52261574, 48774913, 82779622, 5832946, 6871053, 67793644, 97641116, 669105, 91255408, 4985896, 74743862, 92353856, 72358170, 88904910, 95251277, 65038678, 44481640, 24733232, 32161669, 93057697, 10366309, 84349107, 69136837, 76825057, 51588015, 99549373, 45407418, 92787493, 20642888, 83533741, 2891150, 22435353, 49598724, 68041839, 97281847, 8373289, 94090109, 19891772, 98462867, 3233084, 83133790, 93359396, 49882705, 88251446, 99658235, 61623915, 95581843, 508198, 2208785, 4515343, 60430369, 79922758, 7229550, 74137926, 20568363, 1936762, 91831487, 21919959, 1788101, 70036438, 36189527, 6497830, 6157724, 65454636, 94539824, 61712234, 67227442, 53547802, 73222868, 68110330, 38061439, 3773993, 89046466, 83789391, 70727211, 39801351, 42644903, 65275241, 62051033, 62803858, 75777973, 73168565, 82886381, 89217461, 85117092, 42307449, 51507425, 80014588, 30653863, 65081429, 19101477, 1239555, 32590267, 29188588, 6793819, 92604458, 59177718, 112651, 77694700, 71522968, 51464002, 33699435, 39373729, 95957797, 71333116, 52204879, 96726697, 49641577, 77898274, 74110882, 24314885, 18699206, 47298834, 1375023, 30218878, 20140249, 62496012, 99226875, 70420215, 54606384, 56955985, 50702367, 40686254, 17385531, 4064751, 17727650, 44060493, 46540998, 54058788, 73031054, 30163921, 66832478, 72278539, 14220886, 44550764, 82897371, 16387575, 82339363, 40677414, 90158785, 83651211, 22405120, 35996293, 75543508, 30139692, 13173644, 13231279, 86001008, 63372756, 93270084, 66250369, 28787861, 28796059, 42782652, 62115552, 75052463, 38494874, 9860968, 85023028, 42947632, 14093520, 70782102, 61271144, 26397786, 10358899, 44915235, 15075176, 59405277, 32159704, 84166196, 64098930, 14723410, 69848388, 22942635, 40984766, 54263880, 55753905, 59197747, 30090481, 98648327, 77377183, 45990383, 75407347, 95395112, 60581278, 82979980, 42199455, 55189057, 40197395, 51149804, 48260151, 16019925, 84002370, 46851987, 26507214, 49597667, 23110625, 18411915, 98371444, 43322743, 27187213, 47708850, 81172706, 2204165, 36808486, 34044787, 2607799, 31126490, 7182642, 55143724, 85711894, 68316156, 7517032, 90004325, 69355476, 10264691, 67474219, 77284619, 5822202, 56424103, 79322415, 31727379, 84187166, 45848907, 53632373, 59318837, 37891451, 69697787, 89637706, 65017137, 54517921, 45075471, 83368048, 79191827, 92530431, 9599614, 23848565, 91240048, 92803766, 72725103, 9058407, 4095116, 79942022, 14947650, 78218845, 81855445, 19939935, 17957593, 90013093, 72274002, 87160386, 11923835, 65271999, 6221471, 7104732, 58224549, 73235980, 44664587, 3088684, 53842979, 57248122, 78549759, 89419466, 91990218, 49236559, 67451935, 72495719, 15015906, 70596786, 69605283, 21070110, 22879907, 62490109, 1569515, 79738755, 37224844, 65880522, 38341669, 55850790, 33123618, 33553959, 60102965, 8913721, 63506281, 7955293, 33249630, 18663507, 63152504, 51426311, 41442762, 71316369, 36135, 52060076, 57359924, 40781449, 90654836, 28851716, 65074535, 29401781, 73786944, 824872, 55669657, 41092102, 86022504, 77413012, 57803235, 17081350, 71657078, 45995325, 65851721, 53888755, 54987042, 86821229, 321665, 44846932, 61141874, 36812683, 61960400, 38645117, 93053405, 33797252, 26863229, 25842078, 51715482, 22450468, 91141395, 54762643, 81274566, 95010552, 22200378, 47893286, 15948937, 35456853, 53648154, 81898046, 64087743, 20867149, 12571310, 19486173, 5482538, 44784505, 30569392, 86301513, 21289531, 53666583, 52293995, 37739481, 37560164, 44245960, 41245325, 4099191, 8791066, 29834512, 16971929, 97379790, 24915585, 65047700, 87720882, 18806856, 12024238, 56484900, 72357096, 37192445, 45381876, 10597197, 6521313, 91281584, 70800879, 45790169, 45617087, 3880712, 33895336, 4978543, 3235882, 5640302, 16097038, 61815223, 16099750, 29029316, 71469330, 93562257, 7066775, 54232247, 83083131, 24168411, 89078848, 99224392, 29819940, 7502255, 27411561, 26493119, 26998766, 33628349, 749283, 64602895, 69255765, 41380093, 18504197, 9257405, 64055960, 11757872, 74452589, 37166608, 36780454, 36396314, 82726008, 69641513, 93515664, 61741594, 59910624, 44479073, 60955663, 62452034, 97561745, 62428472, 99297164 +77413012, 73124510, 74724075, 59109400, 72495719, 13819358, 7300047, 84002370, 56531125, 29065964, 41092172, 23569917, 36580610, 73392814, 20765474, 23134715, 63506281, 35192533, 415901, 15535065, 98371444, 89214616, 54868730, 77694700, 79136082, 70420215, 57658654, 11581181, 50567636, 22129328, 6871053, 62693428, 16387575, 92530431, 23848565, 5267545, 36139942, 92803766, 60176618, 3509435, 98462867, 68702632, 26392416, 91990218, 18466635, 14626618, 89996536, 22879907, 93790285, 76170907, 13348726, 47213183, 7423788, 39801351, 31161687, 34698463, 29029316, 81172706, 17146629, 92692978, 79806380, 91938191, 65074535, 50668599, 47298834, 56484900, 4668450, 66663942, 54606384, 87710366, 37891451, 9886593, 91281584, 43501211, 61960400, 83368048, 93566986, 68824981, 69136837, 13470059, 90158785, 35996293, 94516935, 35092039, 53084256, 58749, 48395186, 42782652, 96420429, 33628349, 1936762, 91831487, 59981773, 55470718, 14326617, 10358899, 59405277, 77272628, 26734892, 88130087, 77377183, 9575954, 95395112, 73168565, 77301523, 51149804, 26063929, 72738685, 70372191, 12303248, 51464002, 55960386, 7011964, 99297164, 16054533, 7687278, 86798033, 92033260, 72373496, 29994197, 9603598, 38256849, 33201905, 99333375, 84187166, 15536795, 94076128, 97641116, 6521313, 56515456, 40677414, 51588015, 97940276, 91727510, 80316608, 93053405, 33336148, 57241521, 19891772, 84684495, 83133790, 72274002, 16445503, 72019362, 61623915, 11923835, 8055981, 95581843, 28787861, 26776922, 60430369, 62115552, 20568363, 68816413, 47792865, 13862149, 37957788, 90061527, 20836893, 3633375, 38658347, 21673260, 40984766, 86460488, 30395570, 78589145, 98648327, 42644903, 55850790, 89217461, 60102965, 26275734, 17058722, 85117092, 84293052, 62740044, 30463802, 91957544, 6793819, 57163802, 59177718, 33249630, 44245960, 47708850, 70367851, 71522968, 7646095, 33237508, 27041967, 52204879, 18131876, 37659250, 69355476, 82532312, 63015256, 76960303, 99021067, 8733068, 74441448, 6819644, 67030811, 67513640, 47738185, 21397057, 34432810, 86821229, 91255408, 4985896, 14220886, 44550764, 2917920, 65017137, 36753250, 45049308, 32274392, 44481640, 15148031, 24733232, 26664538, 82327024, 61859581, 44842615, 76825057, 92787493, 20642888, 4095116, 26139110, 68041839, 30139692, 8634541, 80251430, 13173644, 29510992, 69579137, 81855445, 21533347, 3487592, 34698428, 9623492, 66250369, 79657802, 53842979, 30998561, 3183975, 38494874, 98943869, 75229462, 83150534, 26397786, 67084351, 30366150, 63628376, 11161731, 32250045, 27325428, 51590803, 22997281, 30764367, 54663246, 24591705, 81853704, 24058273, 89046466, 87598888, 55753905, 64157906, 68000591, 45990383, 52613508, 69255765, 75407347, 30569392, 33123618, 17016156, 93933709, 72793444, 13468268, 84904436, 7955293, 48892201, 92692283, 39847321, 9175338, 9188443, 92604458, 99690194, 41245325, 70541760, 23740167, 18783702, 56473732, 8791066, 4508700, 39373729, 22113133, 59329511, 29466406, 43376279, 16424199, 32058615, 82651278, 72732205, 19272365, 50188404, 23194618, 39201414, 12024238, 31491938, 64055960, 78300864, 55669657, 24168411, 31727379, 99515901, 48774913, 5832946, 54058788, 32151165, 96318094, 16380211, 94595668, 168541, 61897582, 2717150, 48893685, 61141874, 45075471, 99917599, 10366309, 19376156, 4787945, 17764950, 22405120, 9058407, 48673079, 83533741, 66137019, 90272749, 8373289, 44348328, 69641513, 71965942, 17894977, 51715482, 84840549, 10961421, 91141395, 54762643, 44664587, 93270084, 508198, 78549759, 9259676, 9860968, 42947632, 57393458, 47893286, 51472275, 4199704, 93515664, 28734791, 15075176, 20885148, 84166196, 98130363, 65715134, 6153406, 80246713, 64087743, 30787683, 31733363, 59197747, 48360198, 65880522, 5482538, 22108665, 86301513, 43045786, 10649306, 75777973, 47887470, 1022677, 37675718, 16097038, 94360702, 48088883, 89804152, 8913721, 42307449, 99729357, 80014588, 73617245, 72614359, 36505482, 61815223, 83948335, 38365584, 45428665, 7685448, 22766820, 5073754, 93562257, 67281495, 99965001, 86118021, 56461322, 47090124, 33699435, 34044787, 71316369, 68644627, 95957797, 71333116, 2607799, 96726697, 14045383, 97379790, 49641577, 77898274, 61859143, 4091162, 20122224, 18699206, 49271185, 65047700, 14363867, 68204242, 16567550, 83302115, 94711842, 60393039, 99524975, 43152977, 37435892, 40781854, 67474219, 3294781, 824872, 59501487, 99226875, 34497327, 77187825, 89699445, 33565483, 39986008, 45848907, 26292919, 57803235, 6986898, 14349098, 79821897, 10597197, 73031054, 669105, 11365791, 68694897, 45306070, 5111370, 72358170, 57020481, 82339363, 32161669, 91240048, 83651211, 99549373, 4806458, 85242963, 27411561, 75543508, 45790169, 4434662, 49598724, 97281847, 92215320, 75820087, 96852321, 3233084, 25842078, 33304202, 68102437, 45237957, 3088684, 36312813, 28796059, 3880712, 49328608, 55602660, 74137926, 81805959, 26114953, 30694952, 51315460, 95010552, 36468541, 36933038, 1788101, 37280276, 36189527, 44915235, 67451935, 92398073, 53802686, 17365188, 68128438, 94539824, 6525948, 69848388, 7919588, 61712234, 15015906, 21993752, 21070110, 35456853, 73222868, 81898046, 62490109, 38061439, 24619760, 54263880, 44177011, 70727211, 79738755, 12571310, 75135448, 76434144, 67031644, 92071990, 66116458, 5640302, 65275241, 29932657, 569864, 86543538, 6111563, 43933006, 39171895, 4204661, 98653983, 42199455, 36685023, 76671482, 66271566, 11543098, 48260151, 54427233, 51507425, 2331773, 46851987, 80555751, 37560164, 41380093, 29635537, 51047803, 71300104, 112651, 42692881, 66885828, 37620363, 83747892, 7066775, 50007421, 4099191, 41442762, 5970607, 88047921, 55143724, 36135, 57359924, 44479073, 9257405, 73786944, 10264691, 95726235, 1375023, 86242799, 14731700, 5822202, 11757872, 62762857, 37166608, 68939068, 89078848, 53632373, 96709982, 17385531, 52261574, 82726008, 46870723, 29819940, 71657078, 40534591, 59318837, 40152546, 99971982, 97057187, 67793644, 54987042, 90090964, 67124282, 321665, 89637706, 62452034, 526217, 65038678, 18833224, 43506672, 54517921, 9599614, 93057697, 90310261, 8115266, 35512853, 16405341, 26102057, 79942022, 59371804, 45667668, 29516592, 78218845, 17539625, 58208470, 19939935, 17957593, 21001913, 27185644, 8099994, 78785507, 49882705, 88251446, 28928175, 40865610, 28550822, 75153252, 75128745, 63372756, 62357986, 75052463, 85023028, 60106217, 21919959, 54014062, 4978543, 17976208, 6157724, 65454636, 10309525, 15948937, 62430984, 15163258, 34236719, 38008118, 64098930, 67227442, 55770687, 69605283, 53648154, 22942635, 44844121, 38009615, 33061250, 1569515, 70004753, 20867149, 37224844, 30090481, 62552524, 3235882, 8129978, 98739783, 38341669, 61728685, 60581278, 21289531, 61263068, 42967683, 3233569, 58180653, 82052050, 65081429, 26507214, 4798568, 44847298, 36845587, 37739481, 19101477, 61741594, 32590267, 49597667, 23110625, 29188588, 48658605, 96906410, 69786756, 63967300, 27187213, 12348118, 18663507, 37501808, 73109997, 6505939, 6808825, 18504197, 58751351, 91664334, 59910624, 41481685, 68316156, 7517032, 52060076, 90654836, 71920426, 27665211, 24314885, 4069912, 56153345, 72777973, 45919976, 24953498, 73021291, 62496012, 56424103, 77072625, 41092102, 86022504, 26744917, 56955985, 12664567, 50702367, 17386996, 7502255, 247198, 66832478, 92353856, 8696647, 94911072, 88904910, 19457589, 39553046, 31904591, 84349107, 83210802, 23432750, 72725103, 76540481, 70800879, 33797252, 18001617, 97561745, 33431960, 47361209, 13231279, 88698958, 26863229, 86001008, 90013093, 17068582, 93359396, 6221471, 73235980, 54199160, 82427263, 4515343, 79922758, 78602717, 95290172, 61271144, 9829782, 17857111, 20681921, 65338021, 45794415, 53393358, 78884452, 30891921, 15902805, 8807940, 12416768, 3773993, 99861373, 40027975, 19486173, 64602895, 61928316, 92554120, 62803858, 82886381, 82979980, 53666583, 55189057, 52293995, 16019925, 66322959, 874791, 79880247, 62936963, 78909561, 77620120, 34295794, 75986488, 32426752, 42073124, 43322743, 44889423, 71469330, 63152504, 51426311, 89811711, 38022834, 77787724, 16971929, 74357852, 7182642, 30811010, 54232247, 33935899, 1204161, 16684829, 87720882, 40356877, 51466049, 55247455, 20645197, 83083131, 60955663, 20002147, 66319530, 77284619, 98948034, 61982238, 96193415, 36780454, 45381876, 36396314, 57240218, 40686254, 76330843, 30771409, 34946859, 10453030, 44983451, 72278539, 74743862, 44846932, 36812683, 95251277, 74614639, 26998766, 65271999, 66611759, 58224549, 33895336, 66045587, 7229550, 8651647, 66903004, 81274566, 89419466, 14093520, 88444207, 42237907, 84406788, 22200378, 749283, 9860195, 81677380, 34493392, 14723410, 70596786, 89499542, 85695762, 20486294, 61373987, 15064655, 8729146, 44784505, 78561158, 62051033, 92867155, 29959549, 77312810, 33553959, 37183543, 40197395, 1239555, 92541302, 18411915, 92998591, 24915585, 4119747, 1431742, 72357096, 36930650, 79322415, 97783876, 83269727, 17037369, 37192445, 45996863, 1250437, 4064751, 17727650, 45995325, 65851721, 9398733, 99125126, 77300457, 79191827, 45407418, 14947650, 2891150, 22435353, 94090109, 57961282, 38510840, 76315420, 90457870, 22450468, 99658235, 88653118, 7104732, 66667729, 96735716, 70782102, 49236559, 48675329, 6497830, 34667286, 97011160, 32159704, 1197320, 16595436, 19898053, 82178706, 68110330, 99604946, 63059024, 30543215, 76703609, 85116755, 25636669, 90004325, 40781449, 28851716, 74110882, 66428795, 29401781, 72238278, 85571389, 30218878, 74452589, 62428472, 8128637, 83155442, 82779622, 44060493, 46540998, 50806615, 22721500, 82897371, 44627776, 71083565, 44473167, 87160386, 2208785, 57248122, 64848072, 70036438, 32699744, 62232211, 68875490, 30653863, 2204165, 36808486, 31126490, 81774825, 27625735, 17081350, 53888755, 69697787, 26493119, 91802888, 6038457, 176257, 24826575, 55615885, 29834512, 78766358, 85711894, 18806856, 20140249, 99224392, 45617087, 43357947, 53547802, 83789391, 30163921, 59614746, 16099750, 84127901, 38645117 +65047700, 66322959, 65851721, 4985896, 44348328, 8055981, 27325428, 30395570, 31733363, 73109997, 33201905, 83269727, 73031054, 4095116, 26776922, 82886381, 43322743, 32058615, 2917920, 62693428, 72725103, 27411561, 85023028, 12416768, 86460488, 24826575, 17058722, 30543215, 52293995, 84002370, 36505482, 34295794, 7646095, 93562257, 4508700, 71920426, 68204242, 51466049, 8733068, 67474219, 22721500, 6521313, 77300457, 68824981, 8115266, 99549373, 97940276, 97561745, 80251430, 27185644, 62357986, 68702632, 95581843, 14326617, 36580610, 4199704, 11161731, 22997281, 5482538, 9575954, 78561158, 63059024, 13819358, 42644903, 6111563, 43933006, 55189057, 37739481, 61741594, 69786756, 77694700, 2204165, 37501808, 25636669, 17146629, 61859143, 91938191, 77284619, 99333375, 67513640, 54058788, 10453030, 91255408, 90090964, 45049308, 526217, 26664538, 9599614, 83210802, 4787945, 68041839, 33431960, 13231279, 3509435, 98462867, 3233084, 33304202, 84840549, 53084256, 75128745, 11923835, 3880712, 33628349, 9259676, 98943869, 67084351, 91990218, 10358899, 10309525, 68128438, 65715134, 3633375, 24058273, 53393358, 62232211, 81898046, 30891921, 40984766, 30787683, 61373987, 20867149, 12571310, 48360198, 66116458, 7423788, 98739783, 77301523, 89217461, 82979980, 60102965, 58180653, 11543098, 63506281, 84904436, 44847298, 91957544, 39847321, 42073124, 112651, 92998591, 33699435, 58751351, 71316369, 59329511, 27041967, 14045383, 16424199, 74110882, 7687278, 14363867, 50188404, 50668599, 73786944, 99524975, 37435892, 66319530, 30218878, 73021291, 824872, 62496012, 39986008, 45996863, 15536795, 26292919, 48774913, 21397057, 37891451, 34432810, 65038678, 18833224, 82339363, 10366309, 23848565, 84349107, 69136837, 4806458, 35996293, 94516935, 59371804, 90272749, 30139692, 13173644, 29510992, 69579137, 47361209, 51715482, 87160386, 78785507, 63372756, 3183975, 23569917, 30694952, 38494874, 66903004, 68816413, 59981773, 30366150, 64848072, 65454636, 92398073, 32159704, 38008118, 6525948, 69848388, 20836893, 20681921, 16595436, 99604946, 59197747, 19486173, 22108665, 44784505, 8129978, 10649306, 95395112, 20765474, 60581278, 33123618, 37675718, 77312810, 42967683, 72793444, 82052050, 79880247, 30653863, 65081429, 62740044, 35192533, 415901, 38365584, 45428665, 18411915, 32426752, 5073754, 44245960, 51464002, 83747892, 47090124, 41442762, 85711894, 68316156, 24915585, 69355476, 24314885, 33935899, 65074535, 23194618, 72777973, 72373496, 40356877, 10264691, 20645197, 78300864, 4668450, 59501487, 5822202, 66663942, 96193415, 57658654, 79322415, 77187825, 41092102, 24168411, 84187166, 12664567, 32151165, 96318094, 92353856, 45306070, 89637706, 72358170, 43501211, 61960400, 71083565, 39553046, 82327024, 61859581, 40677414, 59109400, 90158785, 17764950, 41092172, 70800879, 83533741, 2891150, 22435353, 8373289, 21533347, 92215320, 75820087, 25842078, 38510840, 17894977, 49882705, 75153252, 34698428, 65271999, 54762643, 73235980, 9623492, 36312813, 79657802, 33895336, 6038457, 62115552, 75052463, 91831487, 60106217, 89419466, 42947632, 14093520, 47792865, 55470718, 95290172, 1788101, 84406788, 47893286, 6497830, 67451935, 37957788, 4978543, 90061527, 17365188, 97011160, 54663246, 98130363, 24591705, 81853704, 67227442, 45794415, 19898053, 78884452, 85695762, 38061439, 64157906, 67031644, 13348726, 98648327, 61928316, 45990383, 75407347, 68875490, 31161687, 21289531, 29959549, 61263068, 86543538, 48088883, 26275734, 76671482, 51149804, 99729357, 54427233, 46851987, 36845587, 61815223, 19101477, 49597667, 92692283, 77620120, 72738685, 29188588, 29635537, 9188443, 48658605, 89214616, 92604458, 54868730, 12303248, 47708850, 71522968, 70541760, 7066775, 18783702, 86118021, 56473732, 7011964, 99297164, 34044787, 33237508, 95957797, 77787724, 16971929, 96726697, 97379790, 49641577, 77898274, 52060076, 90004325, 20122224, 30811010, 27625735, 72732205, 4119747, 79806380, 92033260, 4069912, 74441448, 1431742, 36930650, 55669657, 68939068, 26744917, 50567636, 45848907, 89078848, 96709982, 1250437, 29819940, 34946859, 168541, 86821229, 11365791, 48893685, 68694897, 56531125, 9886593, 61141874, 29065964, 32274392, 15148031, 92803766, 60176618, 76825057, 13470059, 23432750, 92787493, 85242963, 45790169, 93053405, 29516592, 78218845, 26863229, 17068582, 43357947, 88251446, 61623915, 10961421, 7104732, 45237957, 93270084, 3088684, 66250369, 96735716, 42782652, 96420429, 79922758, 91802888, 55602660, 26114953, 78549759, 9860968, 70782102, 88444207, 13862149, 9829782, 48675329, 36189527, 62430984, 84166196, 34236719, 64098930, 7919588, 89499542, 55770687, 69605283, 21993752, 20486294, 53648154, 62490109, 15902805, 33061250, 1569515, 79738755, 93790285, 76170907, 75135448, 55753905, 65880522, 30090481, 52613508, 92071990, 86301513, 43045786, 92867155, 62803858, 73168565, 61728685, 55850790, 47887470, 7300047, 89804152, 36685023, 8913721, 874791, 1239555, 15535065, 6793819, 57163802, 16099750, 74724075, 29029316, 18663507, 79136082, 51426311, 99965001, 8791066, 39373729, 89811711, 22113133, 38022834, 71333116, 52204879, 7182642, 55143724, 41481685, 36135, 81774825, 82651278, 92692978, 49271185, 66428795, 16684829, 56153345, 29401781, 39201414, 18806856, 86242799, 60955663, 6819644, 67030811, 3294781, 72357096, 61982238, 77072625, 62762857, 31727379, 33565483, 84127901, 57240218, 40686254, 99224392, 17727650, 30771409, 59318837, 16380211, 10597197, 66832478, 54987042, 44983451, 72278539, 44550764, 74743862, 67124282, 5111370, 321665, 88904910, 91281584, 92530431, 44481640, 24733232, 44842615, 93057697, 19376156, 51588015, 9058407, 66137019, 26139110, 80316608, 33797252, 49598724, 33336148, 26493119, 44473167, 57241521, 17539625, 58208470, 84684495, 96852321, 17957593, 35092039, 72274002, 76315420, 93359396, 68102437, 28928175, 6221471, 66045587, 30998561, 508198, 81805959, 81274566, 75229462, 42237907, 57393458, 63628376, 51472275, 44915235, 28734791, 15075176, 59405277, 34667286, 14626618, 89996536, 26734892, 17857111, 15015906, 32699744, 44844121, 54263880, 44177011, 176257, 70727211, 99861373, 8729146, 78589145, 68000591, 62552524, 3235882, 30569392, 92554120, 17016156, 1022677, 37183543, 16097038, 94360702, 40197395, 66271566, 16019925, 73617245, 62936963, 48892201, 83948335, 37560164, 92541302, 51047803, 71300104, 70372191, 63967300, 42692881, 66885828, 12348118, 81172706, 71469330, 63152504, 41245325, 56461322, 29466406, 18131876, 91664334, 74357852, 78766358, 31126490, 7517032, 40781449, 28851716, 54232247, 19272365, 63015256, 44479073, 99021067, 87720882, 72238278, 9257405, 85571389, 83302115, 47298834, 56484900, 24953498, 64055960, 83083131, 40781854, 14731700, 9603598, 74452589, 38256849, 86022504, 87710366, 36780454, 37192445, 45381876, 6986898, 17081350, 82726008, 22129328, 6871053, 45995325, 247198, 94076128, 97057187, 97641116, 53888755, 669105, 2717150, 82897371, 8696647, 56515456, 94911072, 44846932, 57020481, 36753250, 16387575, 95251277, 79191827, 32161669, 48673079, 76540481, 26102057, 45667668, 8634541, 81855445, 57961282, 88698958, 83133790, 71965942, 21001913, 22450468, 26998766, 28550822, 91141395, 66611759, 48395186, 28787861, 49328608, 82427263, 2208785, 78602717, 36468541, 22200378, 70036438, 6157724, 53802686, 15948937, 32250045, 77272628, 94539824, 30764367, 14723410, 1197320, 53547802, 65338021, 73392814, 38658347, 80246713, 8807940, 68110330, 64087743, 70004753, 87598888, 37224844, 15064655, 64602895, 88130087, 47213183, 69255765, 5640302, 65275241, 38341669, 569864, 23134715, 3233569, 42199455, 51507425, 80014588, 2331773, 80555751, 78909561, 30463802, 41380093, 9175338, 85116755, 98371444, 7685448, 96906410, 70367851, 18504197, 67281495, 55960386, 4099191, 68644627, 5970607, 2607799, 16054533, 59910624, 4091162, 82532312, 27665211, 18699206, 86798033, 45919976, 16567550, 60393039, 55247455, 98948034, 20140249, 56424103, 54606384, 89699445, 17037369, 36396314, 17386996, 52261574, 76330843, 14349098, 47738185, 44060493, 71657078, 46540998, 40152546, 67793644, 50806615, 61897582, 9398733, 44627776, 36812683, 83368048, 93566986, 90310261, 38645117, 45407418, 20642888, 14947650, 75543508, 4434662, 45617087, 19891772, 69641513, 19939935, 90013093, 99658235, 88653118, 66667729, 53842979, 60430369, 57248122, 1936762, 83150534, 95010552, 54014062, 17976208, 9860195, 81677380, 34493392, 61712234, 6153406, 73222868, 82178706, 38009615, 3773993, 76434144, 77377183, 62051033, 75777973, 29932657, 93933709, 85117092, 26063929, 84293052, 4798568, 72614359, 32590267, 59177718, 22766820, 44889423, 23740167, 50007421, 29834512, 29994197, 95726235, 12024238, 31491938, 1375023, 8128637, 77413012, 46870723, 4064751, 5832946, 79821897, 94595668, 14220886, 62452034, 43506672, 74614639, 54517921, 45075471, 5267545, 16405341, 97281847, 18001617, 94090109, 8099994, 72019362, 54199160, 4515343, 7229550, 20568363, 61271144, 26397786, 26392416, 49236559, 37280276, 93515664, 51590803, 72495719, 15163258, 21070110, 24619760, 33553959, 39171895, 4204661, 53666583, 55615885, 23110625, 99690194, 33249630, 6808825, 88047921, 90654836, 1204161, 76960303, 94711842, 20002147, 34497327, 70420215, 97783876, 53632373, 57803235, 83155442, 99515901, 82779622, 7502255, 40534591, 99971982, 99917599, 31904591, 91240048, 83651211, 22405120, 91727510, 86001008, 16445503, 40865610, 3487592, 58224549, 44664587, 51315460, 8651647, 21919959, 36933038, 18466635, 35456853, 22942635, 89046466, 40027975, 39801351, 98653983, 34698463, 76703609, 13468268, 26507214, 7955293, 75986488, 27187213, 37620363, 57359924, 37659250, 43152977, 99226875, 37166608, 11581181, 56955985, 17385531, 65017137, 99125126, 36139942, 35512853, 58749, 28796059, 20885148, 59614746, 70596786, 22879907, 21673260, 83789391, 42307449, 30163921, 69697787, 79942022, 749283, 48260151, 73124510, 36808486, 6505939, 11757872, 62428472, 50702367, 19457589, 90457870, 43376279, 74137926 +74614639, 59614746, 54232247, 69136837, 22450468, 53084256, 7423788, 6793819, 99971982, 54987042, 9398733, 77300457, 76825057, 40865610, 53842979, 74137926, 32699744, 15064655, 92692283, 7646095, 32058615, 52060076, 99515901, 96318094, 11365791, 56515456, 8115266, 45617087, 81855445, 19939935, 21001913, 72274002, 93359396, 58224549, 66045587, 51472275, 93515664, 92398073, 65338021, 86460488, 92071990, 8129978, 65275241, 26275734, 30543215, 83948335, 29188588, 85116755, 7685448, 89214616, 112651, 42692881, 47090124, 17146629, 28851716, 27625735, 12024238, 34497327, 11757872, 97783876, 11581181, 84187166, 77413012, 53632373, 45995325, 73031054, 48893685, 89637706, 62452034, 36812683, 83210802, 38645117, 23432750, 16405341, 85242963, 47361209, 88698958, 79657802, 15163258, 54663246, 98130363, 55770687, 8807940, 99604946, 3773993, 176257, 79738755, 30090481, 45990383, 3235882, 37675718, 52293995, 48260151, 26063929, 63506281, 46851987, 44847298, 36845587, 35192533, 9175338, 70372191, 42073124, 74724075, 47708850, 18663507, 36808486, 33237508, 29834512, 43376279, 88047921, 24915585, 63015256, 99021067, 5822202, 99226875, 86022504, 33565483, 56955985, 45381876, 8128637, 50702367, 17386996, 14349098, 65851721, 50806615, 61897582, 88904910, 61960400, 54517921, 45075471, 82339363, 15148031, 24733232, 26664538, 10366309, 72725103, 45407418, 9058407, 26102057, 35996293, 2891150, 18001617, 21533347, 38510840, 33304202, 75153252, 91141395, 6221471, 62357986, 73235980, 57248122, 78602717, 68816413, 75229462, 48675329, 34667286, 65715134, 3633375, 73392814, 78884452, 38061439, 70727211, 64602895, 62552524, 75407347, 68875490, 13819358, 1022677, 55189057, 16019925, 13468268, 55615885, 62740044, 61741594, 38365584, 23110625, 72738685, 63967300, 92998591, 33249630, 77694700, 44889423, 71469330, 23740167, 50007421, 4099191, 7011964, 4508700, 99297164, 58751351, 18131876, 55143724, 97379790, 57359924, 37659250, 4069912, 23194618, 47298834, 1375023, 43152977, 60955663, 40781854, 4668450, 14731700, 59501487, 96193415, 70420215, 74452589, 41092102, 37166608, 68939068, 57803235, 96709982, 4064751, 67793644, 66832478, 44983451, 14220886, 6521313, 44550764, 69697787, 2917920, 321665, 72358170, 45049308, 43501211, 32274392, 31904591, 68824981, 61859581, 19376156, 13470059, 14947650, 91727510, 45790169, 49598724, 26493119, 57961282, 84684495, 26863229, 83133790, 8099994, 90457870, 87160386, 78785507, 88251446, 99658235, 48395186, 30998561, 82427263, 9259676, 75052463, 85023028, 36580610, 54014062, 9829782, 17976208, 9860195, 6157724, 53802686, 27325428, 32159704, 89996536, 34236719, 24591705, 21993752, 38658347, 35456853, 70004753, 87598888, 31733363, 75135448, 13348726, 52613508, 78561158, 5640302, 29932657, 82886381, 33553959, 42967683, 39171895, 48088883, 42199455, 79880247, 30653863, 2331773, 26507214, 72614359, 37739481, 49597667, 41380093, 57163802, 98371444, 48658605, 22766820, 37620363, 37501808, 67281495, 56473732, 56461322, 41442762, 39373729, 95957797, 77787724, 16971929, 31126490, 7182642, 49641577, 20122224, 1204161, 95726235, 18806856, 94711842, 55247455, 20645197, 6819644, 67030811, 77284619, 73021291, 824872, 56424103, 36930650, 62428472, 26292919, 57240218, 83155442, 71657078, 6871053, 247198, 94595668, 91255408, 4985896, 2717150, 90090964, 45306070, 94911072, 44627776, 91281584, 16387575, 44481640, 82327024, 23848565, 91240048, 40677414, 4787945, 17764950, 20642888, 22405120, 70800879, 97281847, 8373289, 94090109, 90013093, 84840549, 28550822, 3487592, 10961421, 34698428, 9623492, 28787861, 28796059, 33895336, 49328608, 2208785, 62115552, 38494874, 36468541, 70782102, 61271144, 26397786, 49236559, 70036438, 4978543, 59405277, 15948937, 90061527, 22997281, 14626618, 68128438, 64098930, 6525948, 1197320, 81853704, 16595436, 45794415, 62232211, 22942635, 12416768, 93790285, 8729146, 48360198, 64157906, 77377183, 22108665, 30569392, 92554120, 61728685, 29959549, 569864, 37183543, 42307449, 76703609, 54427233, 874791, 65081429, 84002370, 4798568, 80555751, 19101477, 48892201, 15535065, 63152504, 83747892, 18504197, 71333116, 78766358, 16054533, 59910624, 36135, 68316156, 77898274, 40781449, 30811010, 90654836, 82532312, 65047700, 66428795, 73786944, 16567550, 56484900, 37435892, 83083131, 86242799, 61982238, 77072625, 38256849, 31727379, 17037369, 26744917, 12664567, 36396314, 82726008, 46870723, 48774913, 29819940, 30771409, 54058788, 40152546, 97641116, 30163921, 86821229, 72278539, 5111370, 19457589, 57020481, 79191827, 99917599, 44842615, 92803766, 27411561, 79942022, 97940276, 66137019, 80316608, 22435353, 33797252, 68041839, 90272749, 44473167, 8634541, 29510992, 69579137, 44348328, 69641513, 3233084, 17894977, 43357947, 16445503, 49882705, 61623915, 75128745, 66611759, 45237957, 44664587, 3088684, 66250369, 95581843, 54199160, 508198, 4515343, 96420429, 79922758, 78549759, 20568363, 9860968, 81274566, 89419466, 42947632, 47792865, 14326617, 21919959, 1788101, 13862149, 84406788, 47893286, 749283, 4199704, 36189527, 67451935, 18466635, 32250045, 72495719, 17365188, 97011160, 38008118, 17857111, 61712234, 15015906, 89499542, 44844121, 62490109, 80246713, 40984766, 64087743, 30395570, 89046466, 61373987, 76170907, 65880522, 5482538, 78589145, 76434144, 61928316, 69255765, 98739783, 39801351, 20765474, 62803858, 75777973, 73168565, 60581278, 21289531, 17016156, 61263068, 94360702, 72793444, 98653983, 58180653, 82052050, 40197395, 66271566, 51149804, 85117092, 34698463, 51507425, 61815223, 7955293, 415901, 1239555, 73124510, 37560164, 77620120, 92541302, 45428665, 51047803, 66885828, 27187213, 29029316, 44245960, 71522968, 73109997, 6808825, 7066775, 51426311, 33699435, 8791066, 22113133, 71316369, 59329511, 2607799, 91664334, 85711894, 81774825, 7517032, 4119747, 92692978, 74110882, 24314885, 18699206, 7687278, 86798033, 92033260, 76960303, 14363867, 44479073, 72777973, 72238278, 85571389, 45919976, 51466049, 83302115, 20002147, 30218878, 3294781, 54606384, 24168411, 89699445, 50567636, 45848907, 84127901, 6986898, 40686254, 52261574, 22129328, 44060493, 32151165, 79821897, 59318837, 53888755, 82897371, 8696647, 68694897, 67124282, 65017137, 44846932, 95251277, 65038678, 71083565, 32161669, 5267545, 90158785, 83651211, 35512853, 26139110, 4434662, 45667668, 57241521, 33431960, 78218845, 13231279, 96852321, 86001008, 25842078, 17957593, 35092039, 17068582, 76315420, 26998766, 68102437, 28928175, 11923835, 63372756, 54762643, 7104732, 93270084, 68702632, 3880712, 60430369, 81805959, 30694952, 14093520, 83150534, 26392416, 42237907, 57393458, 10358899, 22200378, 64848072, 28734791, 37957788, 11161731, 10309525, 77272628, 34493392, 30764367, 14723410, 26734892, 20836893, 69605283, 20486294, 73222868, 82178706, 22879907, 81898046, 38009615, 33061250, 99861373, 12571310, 55753905, 59197747, 86301513, 95395112, 42644903, 31161687, 62051033, 33123618, 77301523, 77312810, 82979980, 93933709, 86543538, 16097038, 60102965, 36685023, 11543098, 99729357, 80014588, 84293052, 84904436, 32590267, 16099750, 54868730, 32426752, 12303248, 70367851, 2204165, 6505939, 51464002, 41245325, 55960386, 99965001, 18783702, 5970607, 29466406, 14045383, 82651278, 90004325, 4091162, 33935899, 91938191, 49271185, 19272365, 16684829, 56153345, 29401781, 39201414, 24953498, 98948034, 62496012, 66663942, 62762857, 79322415, 83269727, 67513640, 89078848, 17385531, 1250437, 17727650, 5832946, 40534591, 37891451, 94076128, 10597197, 92353856, 61141874, 29065964, 526217, 43506672, 39553046, 93566986, 93057697, 36139942, 90310261, 51588015, 4806458, 4095116, 76540481, 97561745, 98462867, 71965942, 27185644, 51715482, 65271999, 88653118, 96735716, 42782652, 51315460, 66903004, 60106217, 95290172, 95010552, 88444207, 91990218, 37280276, 44915235, 15075176, 62430984, 94539824, 7919588, 53393358, 19898053, 21070110, 30891921, 68110330, 1569515, 20867149, 10649306, 38341669, 92867155, 55850790, 47887470, 89217461, 7300047, 4204661, 3233569, 89804152, 76671482, 8913721, 66322959, 73617245, 30463802, 34295794, 75986488, 71300104, 43322743, 86118021, 25636669, 34044787, 68644627, 52204879, 74357852, 96726697, 16424199, 69355476, 71920426, 27665211, 87720882, 68204242, 72373496, 10264691, 8733068, 99524975, 64055960, 1431742, 78300864, 66319530, 67474219, 72357096, 57658654, 99333375, 37192445, 17081350, 47738185, 82779622, 7502255, 46540998, 16380211, 34432810, 56531125, 9886593, 36753250, 18833224, 83368048, 59109400, 60176618, 99549373, 92787493, 94516935, 75543508, 59371804, 93053405, 33336148, 80251430, 13173644, 19891772, 58208470, 75820087, 3509435, 72019362, 66667729, 91802888, 55602660, 6038457, 26114953, 23569917, 33628349, 98943869, 1936762, 91831487, 36933038, 67084351, 30366150, 6497830, 20885148, 51590803, 84166196, 69848388, 53547802, 24058273, 53648154, 21673260, 44177011, 40027975, 24826575, 67031644, 68000591, 44784505, 47213183, 63059024, 23134715, 6111563, 91957544, 29635537, 96906410, 89811711, 72732205, 50668599, 40356877, 60393039, 31491938, 9603598, 87710366, 33201905, 39986008, 45996863, 21397057, 34946859, 10453030, 97057187, 168541, 669105, 74743862, 62693428, 9599614, 48673079, 83533741, 30139692, 8055981, 36312813, 26776922, 3183975, 8651647, 63628376, 65454636, 81677380, 20681921, 6153406, 70596786, 30787683, 83789391, 9575954, 66116458, 36505482, 18411915, 99690194, 41481685, 79806380, 50188404, 29994197, 20140249, 77187825, 55669657, 36780454, 15536795, 99224392, 99125126, 92530431, 84349107, 92215320, 67227442, 37224844, 19486173, 98648327, 43933006, 62936963, 39847321, 9188443, 92604458, 69786756, 81172706, 93562257, 79136082, 70541760, 27041967, 65074535, 9257405, 22721500, 29516592, 58749, 59981773, 55470718, 85695762, 24619760, 54263880, 88130087, 53666583, 17058722, 78909561, 59177718, 5073754, 12348118, 61859143, 74441448, 41092172, 17539625, 7229550, 43045786, 76330843, 38022834, 15902805 +36933038, 21673260, 39847321, 45667668, 90013093, 53547802, 62552524, 44784505, 59177718, 34497327, 36396314, 10453030, 67124282, 61859581, 36139942, 75543508, 69641513, 40865610, 45237957, 51315460, 47893286, 6153406, 21070110, 81898046, 19486173, 88130087, 66116458, 78909561, 85116755, 41245325, 62762857, 26744917, 47738185, 79821897, 99125126, 91727510, 30139692, 17068582, 33895336, 23569917, 51590803, 15163258, 24058273, 45794415, 68000591, 55850790, 29959549, 86543538, 94360702, 16019925, 84002370, 45428665, 5073754, 79136082, 86118021, 33935899, 9257405, 73786944, 8733068, 31491938, 74452589, 15536795, 77413012, 40686254, 76330843, 44060493, 34946859, 94595668, 30163921, 14220886, 2717150, 48893685, 56531125, 82339363, 44481640, 92803766, 38645117, 83651211, 16405341, 94090109, 3509435, 28550822, 96735716, 21919959, 54014062, 36189527, 34236719, 15015906, 67227442, 22942635, 44177011, 37224844, 76434144, 77377183, 92071990, 86301513, 8129978, 63059024, 95395112, 75777973, 33553959, 4204661, 53666583, 66322959, 2331773, 37739481, 73124510, 29635537, 9175338, 16099750, 92604458, 66885828, 29029316, 71522968, 37501808, 99297164, 22113133, 29466406, 31126490, 32058615, 40781449, 79806380, 18699206, 7687278, 72373496, 10264691, 62496012, 99226875, 62428472, 4064751, 82779622, 71657078, 6871053, 99971982, 22721500, 5111370, 29065964, 36753250, 54517921, 83368048, 99917599, 4787945, 83533741, 33336148, 29516592, 44473167, 8634541, 13173644, 86001008, 72274002, 78785507, 68102437, 58749, 3088684, 28787861, 49328608, 1936762, 14093520, 55470718, 88444207, 49236559, 48675329, 70036438, 10309525, 27325428, 77272628, 7919588, 61712234, 38658347, 80246713, 3773993, 52613508, 69255765, 43045786, 33123618, 48088883, 36685023, 99729357, 26063929, 62936963, 36505482, 61815223, 41380093, 92541302, 15535065, 71300104, 96906410, 77694700, 27187213, 36808486, 18504197, 55960386, 4099191, 71333116, 16971929, 52204879, 7182642, 16054533, 97379790, 4091162, 92692978, 24314885, 86798033, 14363867, 68204242, 29994197, 55247455, 74441448, 61982238, 96193415, 9603598, 33201905, 11581181, 57803235, 96709982, 5832946, 16380211, 34432810, 91255408, 19457589, 83210802, 35512853, 92787493, 41092172, 4434662, 18001617, 80251430, 19891772, 58208470, 13231279, 96852321, 71965942, 72019362, 88251446, 99658235, 28928175, 65271999, 93270084, 66250369, 66667729, 26776922, 42782652, 2208785, 57248122, 81805959, 26114953, 75052463, 47792865, 61271144, 26397786, 64848072, 6157724, 18466635, 14626618, 34493392, 17857111, 73392814, 55770687, 62490109, 40984766, 87598888, 76170907, 55753905, 48360198, 30090481, 61928316, 30569392, 7423788, 65275241, 60581278, 23134715, 3233569, 66271566, 11543098, 48260151, 80014588, 73617245, 55615885, 80555751, 38365584, 77620120, 75986488, 57163802, 98371444, 89214616, 54868730, 12303248, 22766820, 70367851, 51464002, 70541760, 4508700, 29834512, 91664334, 74357852, 88047921, 16424199, 28851716, 4119747, 71920426, 49271185, 19272365, 50668599, 72777973, 45919976, 83302115, 1431742, 60955663, 77284619, 66663942, 55669657, 36780454, 89699445, 56955985, 45381876, 17081350, 99515901, 14349098, 17727650, 46540998, 40534591, 40152546, 168541, 50806615, 66832478, 90090964, 74743862, 92353856, 69697787, 68694897, 2917920, 9886593, 61141874, 93566986, 26664538, 44842615, 91240048, 59109400, 90158785, 17764950, 76540481, 85242963, 2891150, 26139110, 59371804, 33797252, 90272749, 98462867, 17957593, 83133790, 35092039, 16445503, 61623915, 75153252, 75128745, 11923835, 6221471, 66611759, 7104732, 58224549, 9623492, 44664587, 36312813, 7229550, 33628349, 26392416, 30366150, 10358899, 9829782, 44915235, 9860195, 20885148, 53802686, 32159704, 30764367, 84166196, 54663246, 81853704, 20681921, 16595436, 3633375, 65338021, 53393358, 73222868, 62232211, 86460488, 54263880, 15064655, 59197747, 64157906, 67031644, 98648327, 22108665, 78561158, 39801351, 92554120, 92867155, 17016156, 61263068, 16097038, 43933006, 89804152, 72793444, 42199455, 85117092, 13468268, 54427233, 84293052, 30653863, 65081429, 26507214, 62740044, 4798568, 44847298, 415901, 1239555, 49597667, 51047803, 99690194, 32426752, 44889423, 81172706, 2204165, 71469330, 6505939, 67281495, 51426311, 18783702, 50007421, 33699435, 41442762, 43376279, 78766358, 36135, 20122224, 54232247, 82532312, 1204161, 92033260, 50188404, 95726235, 56484900, 83083131, 20002147, 98948034, 3294781, 824872, 70420215, 57658654, 38256849, 86022504, 33565483, 17037369, 84187166, 12664567, 57240218, 50702367, 17385531, 52261574, 10597197, 73031054, 61897582, 53888755, 72278539, 6521313, 82897371, 57020481, 65038678, 18833224, 43501211, 24733232, 90310261, 60176618, 13470059, 4806458, 22405120, 4095116, 27411561, 35996293, 94516935, 66137019, 49598724, 45617087, 97561745, 69579137, 47361209, 21533347, 84684495, 3233084, 43357947, 76315420, 93359396, 49882705, 48395186, 88653118, 95581843, 30998561, 508198, 60430369, 79922758, 20568363, 8651647, 9860968, 68816413, 59981773, 14326617, 67084351, 36580610, 57393458, 63628376, 37280276, 17976208, 59405277, 15948937, 32250045, 97011160, 22997281, 94539824, 69848388, 98130363, 1197320, 70596786, 69605283, 85695762, 22879907, 12416768, 30395570, 24619760, 30787683, 176257, 70727211, 93790285, 40027975, 75135448, 5482538, 47213183, 38341669, 73168565, 61728685, 29932657, 37675718, 569864, 89217461, 93933709, 7300047, 26275734, 40197395, 52293995, 8913721, 42307449, 63506281, 84904436, 30463802, 61741594, 37560164, 9188443, 42073124, 74724075, 47708850, 37620363, 93562257, 23740167, 56461322, 7011964, 8791066, 89811711, 71316369, 68644627, 5970607, 38022834, 18131876, 2607799, 85711894, 68316156, 61859143, 52060076, 90004325, 90654836, 72732205, 27665211, 66428795, 65074535, 40356877, 51466049, 60393039, 99524975, 43152977, 24953498, 64055960, 78300864, 67030811, 72357096, 20140249, 11757872, 41092102, 24168411, 37166608, 50567636, 67513640, 39986008, 37192445, 45848907, 17386996, 46870723, 30771409, 32151165, 96318094, 59318837, 45995325, 97057187, 67793644, 97641116, 45306070, 62693428, 65017137, 71083565, 45075471, 82327024, 5267545, 99549373, 9058407, 70800879, 79942022, 97940276, 80316608, 45790169, 97281847, 57241521, 33431960, 17539625, 81855445, 88698958, 19939935, 17894977, 51715482, 87160386, 3487592, 91141395, 62357986, 3880712, 79657802, 3183975, 74137926, 30694952, 66903004, 91831487, 60106217, 1788101, 91990218, 13862149, 22200378, 28734791, 15075176, 17365188, 59614746, 64098930, 6525948, 20836893, 24591705, 65715134, 32699744, 19898053, 53648154, 82178706, 30891921, 15902805, 68110330, 64087743, 61373987, 83789391, 24826575, 65880522, 64602895, 78589145, 13348726, 3235882, 5640302, 98739783, 20765474, 31161687, 47887470, 42967683, 6111563, 60102965, 58180653, 46851987, 36845587, 35192533, 83948335, 32590267, 18411915, 7685448, 69786756, 33249630, 7646095, 63152504, 6808825, 47090124, 58751351, 39373729, 34044787, 59329511, 95957797, 27041967, 55143724, 14045383, 49641577, 41481685, 81774825, 82651278, 57359924, 30811010, 91938191, 65047700, 44479073, 16684829, 56153345, 99021067, 72238278, 39201414, 94711842, 12024238, 1375023, 86242799, 14731700, 30218878, 59501487, 79322415, 97783876, 83269727, 99333375, 8128637, 26292919, 89078848, 6986898, 1250437, 29819940, 65851721, 44983451, 11365791, 8696647, 321665, 88904910, 44627776, 91281584, 526217, 95251277, 74614639, 39553046, 84349107, 76825057, 51588015, 45407418, 20642888, 22435353, 93053405, 8373289, 29510992, 92215320, 75820087, 25842078, 90457870, 53084256, 8055981, 28796059, 54199160, 96420429, 6038457, 38494874, 98943869, 42947632, 51472275, 6497830, 37957788, 72495719, 68128438, 62430984, 38008118, 20486294, 89046466, 1569515, 70004753, 8729146, 12571310, 45990383, 75407347, 42644903, 21289531, 82979980, 37183543, 55189057, 76671482, 51149804, 34698463, 7955293, 19101477, 72738685, 29188588, 48658605, 70372191, 112651, 63967300, 42692881, 12348118, 18663507, 7066775, 99965001, 25636669, 77787724, 7517032, 77898274, 37659250, 69355476, 16567550, 47298834, 37435892, 40781854, 73021291, 31727379, 68939068, 53632373, 82726008, 48774913, 247198, 54987042, 669105, 4985896, 62452034, 44846932, 16387575, 61960400, 79191827, 92530431, 31904591, 15148031, 32161669, 9599614, 93057697, 10366309, 19376156, 69136837, 48673079, 78218845, 44348328, 26863229, 38510840, 8099994, 26998766, 73235980, 68702632, 53842979, 4515343, 9259676, 81274566, 75229462, 83150534, 95290172, 95010552, 70782102, 749283, 92398073, 44844121, 8807940, 20867149, 79738755, 9575954, 68875490, 13819358, 77301523, 77312810, 39171895, 98653983, 874791, 79880247, 34295794, 44245960, 73109997, 96726697, 59910624, 27625735, 76960303, 4069912, 87720882, 85571389, 66319530, 54606384, 87710366, 45996863, 83155442, 54058788, 37891451, 9398733, 45049308, 77300457, 68824981, 23848565, 40677414, 8115266, 23432750, 72725103, 26102057, 14947650, 68041839, 26493119, 57961282, 21001913, 27185644, 34698428, 66045587, 82427263, 62115552, 78602717, 89419466, 36468541, 42237907, 34667286, 90061527, 14723410, 26734892, 78884452, 21993752, 38061439, 33061250, 62803858, 30543215, 51507425, 72614359, 92692283, 23110625, 6793819, 92998591, 56473732, 17146629, 24915585, 74110882, 29401781, 18806856, 4668450, 67474219, 36930650, 21397057, 94076128, 94911072, 72358170, 32274392, 22450468, 84840549, 10961421, 54762643, 78549759, 84406788, 4199704, 67451935, 81677380, 35456853, 99604946, 99861373, 62051033, 1022677, 82052050, 76703609, 48892201, 91957544, 43322743, 33237508, 63015256, 56424103, 77187825, 84127901, 22129328, 99224392, 7502255, 89637706, 36812683, 33304202, 63372756, 91802888, 93515664, 4978543, 65454636, 11161731, 89996536, 38009615, 10649306, 17058722, 83747892, 23194618, 6819644, 5822202, 77072625, 86821229, 56515456, 43506672, 55602660, 85023028, 31733363, 82886381, 20645197, 44550764, 89499542 +79922758, 43322743, 85711894, 25842078, 78549759, 38494874, 33565483, 61141874, 53842979, 37280276, 38008118, 65338021, 64087743, 70004753, 39801351, 92554120, 36685023, 35192533, 1239555, 16099750, 67281495, 56473732, 41442762, 32058615, 56484900, 34497327, 54987042, 86821229, 80316608, 23569917, 91831487, 61271144, 67084351, 22200378, 6157724, 62430984, 24591705, 77377183, 42644903, 26063929, 80014588, 37560164, 47090124, 88047921, 20122224, 90654836, 92033260, 99021067, 73786944, 824872, 50806615, 11365791, 321665, 89637706, 94911072, 83651211, 79942022, 75543508, 22435353, 45617087, 33304202, 79657802, 55602660, 64848072, 16595436, 98648327, 78561158, 8129978, 33123618, 17016156, 33553959, 6111563, 60102965, 76671482, 52293995, 48260151, 75986488, 74724075, 4099191, 82651278, 65074535, 76960303, 14363867, 1375023, 60955663, 62762857, 84187166, 77413012, 10453030, 10597197, 97641116, 30163921, 74743862, 2917920, 29065964, 65038678, 79191827, 23432750, 72725103, 13173644, 47361209, 35092039, 88251446, 28550822, 65271999, 68702632, 28787861, 2208785, 57248122, 60106217, 95010552, 70782102, 84406788, 10358899, 37957788, 32250045, 77272628, 22997281, 54663246, 67227442, 86460488, 1569515, 93790285, 65880522, 64602895, 88130087, 76434144, 13348726, 30090481, 73168565, 23134715, 58180653, 82052050, 79880247, 83948335, 92692283, 33249630, 47708850, 81172706, 7066775, 51426311, 99297164, 29834512, 52204879, 43376279, 52060076, 27665211, 4069912, 64055960, 14731700, 30218878, 96193415, 97783876, 37166608, 62428472, 50567636, 4064751, 17727650, 48774913, 37891451, 247198, 94595668, 65851721, 66832478, 53888755, 9886593, 44846932, 45075471, 90158785, 4787945, 51588015, 70800879, 97940276, 90272749, 44473167, 57241521, 92215320, 13231279, 44348328, 86001008, 27185644, 93359396, 91141395, 63372756, 54762643, 62357986, 7104732, 54199160, 33895336, 82427263, 75052463, 8651647, 9860968, 89419466, 42947632, 55470718, 1788101, 47893286, 44915235, 4978543, 68128438, 6525948, 3633375, 55770687, 35456853, 53648154, 21673260, 40984766, 12416768, 15064655, 44784505, 30569392, 29932657, 86543538, 4204661, 26275734, 16019925, 84002370, 46851987, 26507214, 62740044, 30463802, 61741594, 41380093, 6793819, 98371444, 71300104, 70372191, 112651, 27187213, 41245325, 33699435, 39373729, 71333116, 16971929, 31126490, 55143724, 14045383, 16054533, 7517032, 57359924, 40781449, 28851716, 54232247, 69355476, 18699206, 44479073, 9257405, 10264691, 47298834, 1431742, 4668450, 62496012, 61982238, 36930650, 36780454, 83269727, 8128637, 15536795, 57803235, 50702367, 17385531, 34946859, 79821897, 4985896, 6521313, 9398733, 45306070, 99125126, 91281584, 45049308, 77300457, 95251277, 43506672, 74614639, 83368048, 93057697, 36139942, 92803766, 76825057, 48673079, 4095116, 94516935, 66137019, 26493119, 58208470, 84684495, 88698958, 83133790, 26998766, 49882705, 68102437, 99658235, 40865610, 75153252, 34698428, 9623492, 28796059, 42782652, 4515343, 62115552, 30694952, 33628349, 51315460, 66903004, 81274566, 59981773, 88444207, 36580610, 57393458, 63628376, 15075176, 92398073, 89996536, 69848388, 98130363, 20836893, 15015906, 20681921, 32699744, 24058273, 73392814, 3773993, 30395570, 61373987, 79738755, 62552524, 3235882, 52613508, 95395112, 62051033, 75777973, 37183543, 89804152, 30543215, 66271566, 51149804, 34698463, 42307449, 36845587, 80555751, 34295794, 29188588, 18411915, 96906410, 99690194, 59177718, 29029316, 71522968, 93562257, 36808486, 55960386, 50007421, 8791066, 22113133, 68644627, 38022834, 95957797, 18131876, 7182642, 16424199, 41481685, 77898274, 4091162, 24915585, 74110882, 24314885, 1204161, 66428795, 68204242, 99524975, 24953498, 98948034, 11757872, 74452589, 87710366, 68939068, 67513640, 45381876, 12664567, 17386996, 17081350, 46870723, 7502255, 71657078, 40534591, 32151165, 45995325, 16380211, 73031054, 168541, 44983451, 2717150, 8696647, 68694897, 62452034, 72358170, 31904591, 93566986, 44481640, 32161669, 10366309, 60176618, 13470059, 99549373, 4806458, 92787493, 41092172, 26102057, 26139110, 4434662, 49598724, 68041839, 33431960, 19891772, 81855445, 21533347, 3509435, 69641513, 98462867, 38510840, 72274002, 43357947, 78785507, 10961421, 6221471, 66611759, 48395186, 58224549, 44664587, 66667729, 26776922, 26114953, 20568363, 68816413, 47792865, 36933038, 91990218, 749283, 48675329, 28734791, 17976208, 9860195, 20885148, 10309525, 27325428, 51590803, 72495719, 94539824, 32159704, 61712234, 65715134, 70596786, 69605283, 22879907, 81898046, 30891921, 62490109, 38009615, 99861373, 37224844, 75135448, 55753905, 19486173, 48360198, 5482538, 78589145, 64157906, 13819358, 20765474, 65275241, 38341669, 21289531, 29959549, 61263068, 37675718, 89217461, 42967683, 72793444, 42199455, 85117092, 8913721, 84293052, 55615885, 72614359, 62936963, 78909561, 36505482, 61815223, 19101477, 48892201, 77620120, 23110625, 15535065, 45428665, 54868730, 63967300, 92998591, 77694700, 70367851, 18663507, 63152504, 79136082, 86118021, 89811711, 77787724, 74357852, 97379790, 36135, 61859143, 4119747, 91938191, 63015256, 56153345, 87720882, 50668599, 29401781, 72777973, 94711842, 78300864, 20002147, 66319530, 67030811, 5822202, 66663942, 57658654, 9603598, 79322415, 77187825, 54606384, 24168411, 99333375, 26744917, 37192445, 26292919, 57240218, 6986898, 82726008, 76330843, 82779622, 21397057, 30771409, 34432810, 99971982, 82897371, 48893685, 62693428, 56531125, 65017137, 526217, 43501211, 71083565, 39553046, 92530431, 68824981, 15148031, 24733232, 26664538, 61859581, 9599614, 91240048, 69136837, 83210802, 59109400, 20642888, 22405120, 85242963, 14947650, 91727510, 59371804, 45790169, 33797252, 18001617, 30139692, 29516592, 75820087, 96852321, 19939935, 17957593, 21001913, 17068582, 51715482, 87160386, 72019362, 75128745, 8055981, 93270084, 36312813, 3880712, 14093520, 75229462, 26392416, 30366150, 9829782, 93515664, 67451935, 59405277, 15948937, 34667286, 81677380, 14626618, 15163258, 84166196, 64098930, 53393358, 19898053, 73222868, 80246713, 68110330, 99604946, 24619760, 44177011, 176257, 83789391, 70727211, 87598888, 31733363, 12571310, 59197747, 24826575, 45990383, 75407347, 66116458, 98739783, 31161687, 92867155, 55850790, 60581278, 77301523, 82979980, 94360702, 43933006, 7300047, 48088883, 40197395, 76703609, 99729357, 54427233, 874791, 73617245, 2331773, 37739481, 415901, 49597667, 9175338, 9188443, 48658605, 51047803, 89214616, 69786756, 22766820, 44245960, 7646095, 37620363, 37501808, 51464002, 18504197, 23740167, 4508700, 58751351, 29466406, 27041967, 59910624, 30811010, 72373496, 39201414, 16567550, 95726235, 18806856, 83302115, 86242799, 77284619, 73021291, 99226875, 55669657, 41092102, 89699445, 45848907, 45996863, 89078848, 53632373, 40686254, 83155442, 99515901, 52261574, 47738185, 44060493, 96318094, 59318837, 40152546, 61897582, 669105, 22721500, 56515456, 88904910, 19457589, 57020481, 16387575, 32274392, 23848565, 5267545, 40677414, 45407418, 17764950, 2891150, 93053405, 97561745, 80251430, 29510992, 26863229, 71965942, 28928175, 61623915, 53084256, 58749, 11923835, 45237957, 3088684, 66045587, 49328608, 508198, 6038457, 81805959, 1936762, 21919959, 95290172, 26397786, 51472275, 70036438, 36189527, 65454636, 97011160, 34493392, 30764367, 34236719, 14723410, 26734892, 17857111, 6153406, 89499542, 21070110, 38658347, 22942635, 15902805, 38061439, 33061250, 30787683, 40027975, 76170907, 47213183, 63059024, 7423788, 10649306, 62803858, 47887470, 77312810, 16097038, 53666583, 55189057, 13468268, 51507425, 30653863, 4798568, 38365584, 32590267, 29635537, 57163802, 92604458, 42073124, 12303248, 42692881, 66885828, 5073754, 44889423, 6808825, 70541760, 99965001, 25636669, 33237508, 71316369, 96726697, 78766358, 81774825, 68316156, 90004325, 92692978, 82532312, 79806380, 33935899, 7687278, 65047700, 85571389, 45919976, 51466049, 60393039, 55247455, 74441448, 83083131, 70420215, 38256849, 36396314, 96709982, 5832946, 6871053, 91255408, 69697787, 67124282, 5111370, 36812683, 18833224, 61960400, 54517921, 82327024, 38645117, 35512853, 9058407, 83533741, 35996293, 97281847, 8373289, 8634541, 94090109, 78218845, 57961282, 22450468, 88653118, 96420429, 91802888, 74137926, 9259676, 14326617, 36468541, 42237907, 54014062, 13862149, 49236559, 53802686, 17365188, 59614746, 1197320, 7919588, 81853704, 21993752, 62232211, 44844121, 89046466, 8729146, 9575954, 92071990, 43045786, 68875490, 61728685, 82886381, 1022677, 569864, 93933709, 98653983, 17058722, 11543098, 63506281, 44847298, 91957544, 73124510, 72738685, 39847321, 7685448, 32426752, 12348118, 73109997, 56461322, 59329511, 2607799, 49641577, 37659250, 16684829, 50188404, 23194618, 29994197, 8733068, 20645197, 40781854, 77072625, 31727379, 84127901, 14349098, 54058788, 94076128, 90090964, 44550764, 92353856, 44627776, 99917599, 82339363, 84349107, 90310261, 8115266, 16405341, 33336148, 3233084, 90013093, 8099994, 90457870, 84840549, 73235980, 95581843, 7229550, 85023028, 4199704, 11161731, 53547802, 45794415, 78884452, 54263880, 67031644, 5640302, 39171895, 3233569, 66322959, 65081429, 92541302, 85116755, 6505939, 7011964, 17146629, 91664334, 19272365, 86798033, 40356877, 31491938, 43152977, 37435892, 67474219, 3294781, 20140249, 59501487, 56424103, 33201905, 11581181, 17037369, 56955985, 1250437, 97057187, 67793644, 36753250, 44842615, 76540481, 45667668, 69579137, 17539625, 76315420, 16445503, 66250369, 96735716, 60430369, 98943869, 83150534, 18466635, 90061527, 85695762, 82178706, 68000591, 22108665, 61928316, 69255765, 86301513, 7955293, 2204165, 71469330, 83747892, 18783702, 5970607, 72732205, 49271185, 72238278, 72357096, 86022504, 39986008, 14220886, 19376156, 3487592, 30998561, 3183975, 78602717, 6497830, 8807940, 20867149, 27625735, 12024238, 6819644, 99224392, 29819940, 46540998, 27411561, 34044787, 71920426, 22129328, 72278539, 17894977, 84904436, 20486294 +17058722, 69786756, 73168565, 27625735, 79806380, 99524975, 68816413, 20765474, 42644903, 37891451, 4985896, 4434662, 49598724, 53084256, 1197320, 92071990, 58180653, 8913721, 12303248, 19272365, 68204242, 64055960, 16405341, 75543508, 28928175, 40865610, 54199160, 3183975, 78549759, 51315460, 72495719, 30891921, 7646095, 86118021, 17146629, 4508700, 18131876, 4069912, 24168411, 45996863, 32274392, 82327024, 17764950, 26102057, 59371804, 44473167, 29510992, 81855445, 84684495, 21001913, 43357947, 28787861, 33628349, 98943869, 66903004, 26392416, 10358899, 4978543, 92398073, 15948937, 69605283, 81898046, 31733363, 30090481, 68000591, 8129978, 16097038, 48088883, 66271566, 13468268, 19101477, 92604458, 96906410, 63967300, 83747892, 33699435, 5970607, 27041967, 30811010, 39201414, 20645197, 59501487, 77072625, 11757872, 89078848, 53632373, 84127901, 57240218, 71657078, 94076128, 67793644, 97641116, 99917599, 82339363, 84349107, 8115266, 83533741, 35996293, 93053405, 21533347, 3509435, 49328608, 67451935, 59405277, 81677380, 15015906, 67227442, 24058273, 45794415, 38061439, 15064655, 8729146, 13348726, 62552524, 77312810, 76671482, 54427233, 65081429, 84002370, 415901, 37560164, 16099750, 42692881, 2204165, 79136082, 56473732, 38022834, 16054533, 72732205, 49271185, 76960303, 87720882, 72373496, 47298834, 74441448, 14731700, 77187825, 87710366, 8128637, 6986898, 17081350, 14349098, 94595668, 22721500, 5111370, 62693428, 89637706, 9886593, 44846932, 61141874, 19457589, 45049308, 526217, 77300457, 65038678, 43501211, 39553046, 92530431, 31904591, 68824981, 44842615, 93057697, 69136837, 23432750, 35512853, 45407418, 97561745, 17539625, 51715482, 49882705, 88251446, 44664587, 68702632, 4515343, 47792865, 75229462, 70782102, 91990218, 84406788, 4199704, 6157724, 22997281, 89996536, 70596786, 22942635, 8807940, 12416768, 89046466, 78589145, 67031644, 98648327, 9575954, 63059024, 43045786, 39171895, 4204661, 79880247, 84904436, 30653863, 62936963, 41380093, 23110625, 6793819, 89214616, 42073124, 66885828, 37620363, 6505939, 56461322, 25636669, 99297164, 68644627, 29466406, 43376279, 36135, 7517032, 69355476, 56153345, 23194618, 10264691, 51466049, 83302115, 1431742, 40781854, 4668450, 38256849, 33201905, 11581181, 99333375, 31727379, 68939068, 56955985, 45381876, 26292919, 82726008, 46870723, 47738185, 21397057, 72278539, 2717150, 48893685, 69697787, 68694897, 2917920, 56531125, 94911072, 57020481, 95251277, 43506672, 79191827, 26664538, 61859581, 9599614, 10366309, 91240048, 76825057, 4787945, 92787493, 20642888, 9058407, 85242963, 94516935, 78218845, 88698958, 26863229, 90457870, 22450468, 75128745, 48395186, 45237957, 3088684, 66045587, 42782652, 57248122, 6038457, 62115552, 81805959, 81274566, 60106217, 14326617, 21919959, 36468541, 13862149, 48675329, 6497830, 93515664, 18466635, 77272628, 34493392, 98130363, 65338021, 53393358, 73392814, 20486294, 15902805, 68110330, 64087743, 24619760, 176257, 99861373, 19486173, 64602895, 64157906, 52613508, 7423788, 10649306, 95395112, 61728685, 21289531, 17016156, 61263068, 77301523, 37675718, 89217461, 33553959, 94360702, 60102965, 82052050, 11543098, 51149804, 52293995, 26063929, 51507425, 84293052, 73617245, 46851987, 36845587, 80555751, 35192533, 30463802, 38365584, 92692283, 9188443, 18411915, 71300104, 59177718, 33249630, 55960386, 34044787, 22113133, 77787724, 2607799, 91664334, 78766358, 16424199, 41481685, 81774825, 82651278, 52060076, 40781449, 37659250, 92692978, 71920426, 82532312, 91938191, 7687278, 1204161, 86798033, 16684829, 99021067, 72238278, 16567550, 8733068, 55247455, 83083131, 60955663, 66319530, 67030811, 3294781, 66663942, 56424103, 57658654, 62762857, 84187166, 37192445, 12664567, 99515901, 48774913, 79821897, 10597197, 34432810, 65851721, 669105, 44983451, 6521313, 74743862, 8696647, 9398733, 16387575, 24733232, 23848565, 19376156, 40677414, 59109400, 79942022, 91727510, 57241521, 94090109, 13173644, 47361209, 58208470, 96852321, 33304202, 17894977, 84840549, 68102437, 75153252, 91141395, 34698428, 63372756, 66250369, 95581843, 79657802, 26776922, 30998561, 79922758, 91802888, 30694952, 8651647, 1936762, 91831487, 59981773, 14093520, 55470718, 42237907, 30366150, 22200378, 749283, 51472275, 70036438, 28734791, 37957788, 10309525, 53802686, 90061527, 14626618, 32159704, 54663246, 7919588, 16595436, 85695762, 38658347, 21673260, 80246713, 33061250, 1569515, 87598888, 79738755, 48360198, 65880522, 76434144, 77377183, 22108665, 61928316, 45990383, 78561158, 30569392, 66116458, 98739783, 62051033, 62803858, 75777973, 55850790, 33123618, 86543538, 98653983, 42199455, 55189057, 40197395, 30543215, 85117092, 34698463, 42307449, 99729357, 26507214, 78909561, 7955293, 91957544, 32590267, 15535065, 85116755, 99690194, 70372191, 22766820, 74724075, 71522968, 18663507, 70541760, 67281495, 51426311, 41442762, 58751351, 39373729, 59329511, 95957797, 29834512, 85711894, 59910624, 49641577, 20122224, 28851716, 66428795, 92033260, 63015256, 44479073, 31491938, 1375023, 24953498, 86242799, 96193415, 79322415, 97783876, 74452589, 54606384, 17037369, 26744917, 52261574, 22129328, 29819940, 32151165, 96318094, 45995325, 40152546, 54987042, 90090964, 44550764, 92353856, 67124282, 56515456, 321665, 62452034, 65017137, 44627776, 91281584, 61960400, 90158785, 99549373, 41092172, 22405120, 4095116, 76540481, 14947650, 97940276, 26139110, 22435353, 68041839, 45667668, 8373289, 80251430, 92215320, 13231279, 98462867, 3233084, 17068582, 61623915, 58749, 54762643, 62357986, 36312813, 2208785, 96420429, 55602660, 7229550, 74137926, 23569917, 75052463, 38494874, 89419466, 88444207, 67084351, 57393458, 64848072, 44915235, 65454636, 11161731, 32250045, 27325428, 17365188, 97011160, 68128438, 38008118, 24591705, 65715134, 53547802, 6153406, 21070110, 53648154, 82178706, 62232211, 86460488, 44177011, 61373987, 70004753, 83789391, 40027975, 59197747, 44784505, 47213183, 86301513, 68875490, 31161687, 82886381, 37183543, 23134715, 43933006, 26275734, 89804152, 72793444, 16019925, 66322959, 80014588, 44847298, 61815223, 1239555, 92541302, 75986488, 29188588, 57163802, 9175338, 112651, 5073754, 77694700, 44889423, 81172706, 18504197, 23740167, 18783702, 8791066, 33237508, 16971929, 52204879, 96726697, 88047921, 7182642, 77898274, 90004325, 4091162, 24915585, 90654836, 74110882, 27665211, 24314885, 65047700, 50188404, 50668599, 72777973, 95726235, 18806856, 94711842, 56484900, 20002147, 77284619, 30218878, 73021291, 72357096, 20140249, 34497327, 9603598, 36930650, 37166608, 83269727, 50567636, 45848907, 77413012, 36396314, 57803235, 40686254, 17385531, 76330843, 54058788, 59318837, 168541, 30163921, 66832478, 45306070, 72358170, 36812683, 83368048, 93566986, 44481640, 32161669, 5267545, 92803766, 83210802, 90310261, 60176618, 51588015, 48673079, 70800879, 27411561, 80316608, 45790169, 30139692, 8634541, 69579137, 44348328, 25842078, 83133790, 27185644, 16445503, 93359396, 87160386, 26998766, 78785507, 99658235, 10961421, 66667729, 53842979, 33895336, 96735716, 82427263, 78602717, 9860968, 85023028, 83150534, 95290172, 36933038, 36580610, 63628376, 49236559, 37280276, 20885148, 34667286, 51590803, 30764367, 59614746, 26734892, 81853704, 20681921, 3633375, 32699744, 78884452, 89499542, 22879907, 44844121, 62490109, 99604946, 30395570, 30787683, 20867149, 37224844, 76170907, 75135448, 88130087, 75407347, 39801351, 65275241, 38341669, 29959549, 93933709, 42967683, 6111563, 3233569, 76703609, 63506281, 874791, 2331773, 62740044, 4798568, 49597667, 48658605, 7685448, 43322743, 47708850, 71469330, 37501808, 73109997, 41245325, 7066775, 50007421, 7011964, 89811711, 71316369, 71333116, 14045383, 68316156, 4119747, 29401781, 9257405, 29994197, 73786944, 45919976, 78300864, 6819644, 98948034, 824872, 62496012, 5822202, 61982238, 70420215, 86022504, 15536795, 17386996, 4064751, 82779622, 44060493, 34946859, 10453030, 73031054, 50806615, 91255408, 14220886, 82897371, 29065964, 18833224, 45075471, 38645117, 83651211, 66137019, 33797252, 33336148, 97281847, 18001617, 90272749, 29516592, 69641513, 19939935, 35092039, 8099994, 28550822, 3487592, 6221471, 66611759, 88653118, 7104732, 58224549, 9623492, 60430369, 9259676, 1788101, 36189527, 9860195, 62430984, 64098930, 14723410, 69848388, 35456853, 12571310, 5482538, 69255765, 92867155, 60581278, 47887470, 569864, 7300047, 53666583, 72614359, 48892201, 29635537, 51047803, 27187213, 29029316, 44245960, 70367851, 93562257, 51464002, 99965001, 4099191, 47090124, 74357852, 31126490, 55143724, 97379790, 32058615, 57359924, 33935899, 14363867, 40356877, 60393039, 37435892, 99226875, 89699445, 62428472, 33565483, 67513640, 39986008, 83155442, 96709982, 17727650, 16380211, 53888755, 86821229, 99125126, 88904910, 36753250, 71083565, 15148031, 72725103, 45617087, 33431960, 57961282, 38510840, 72019362, 11923835, 8055981, 42947632, 61271144, 26397786, 47893286, 17976208, 15163258, 94539824, 84166196, 17857111, 20836893, 19898053, 21993752, 3773993, 70727211, 93790285, 24826575, 5640302, 13819358, 29932657, 36685023, 48260151, 55615885, 37739481, 61741594, 83948335, 34295794, 72738685, 39847321, 92998591, 36808486, 65074535, 43152977, 67474219, 55669657, 41092102, 1250437, 5832946, 46540998, 40534591, 6871053, 247198, 99971982, 11365791, 74614639, 36139942, 2891150, 19891772, 75820087, 17957593, 71965942, 76315420, 73235980, 3880712, 508198, 26114953, 20568363, 95010552, 54014062, 9829782, 15075176, 34236719, 6525948, 61712234, 73222868, 54263880, 55753905, 3235882, 92554120, 36505482, 73124510, 98371444, 54868730, 6808825, 61859143, 85571389, 12024238, 50702367, 99224392, 30771409, 97057187, 61897582, 54517921, 13470059, 4806458, 90013093, 65271999, 55770687, 1022677, 77620120, 32426752, 12348118, 54232247, 18699206, 36780454, 7502255, 26493119, 86001008, 72274002, 93270084, 38009615, 82979980, 45428665, 63152504, 28796059, 40984766 +112651, 13819358, 92554120, 93566986, 4806458, 3487592, 15075176, 55850790, 16097038, 18504197, 16684829, 86242799, 61859581, 14947650, 45617087, 88698958, 22450468, 58749, 59981773, 47792865, 95290172, 51472275, 43045786, 3233569, 2331773, 16099750, 47708850, 95957797, 31126490, 9257405, 77072625, 37166608, 36780454, 74614639, 51588015, 90272749, 47361209, 73235980, 44664587, 91802888, 20568363, 34667286, 73392814, 89046466, 70004753, 569864, 89217461, 99729357, 78909561, 1239555, 23110625, 9188443, 74724075, 7011964, 58751351, 97379790, 4091162, 14363867, 39201414, 1375023, 43152977, 55247455, 5822202, 11757872, 83269727, 89078848, 50702367, 4064751, 10453030, 94595668, 44550764, 91281584, 83368048, 35512853, 99549373, 17764950, 86001008, 16445503, 90457870, 66667729, 78549759, 14326617, 84406788, 47893286, 70036438, 30891921, 8807940, 98739783, 47887470, 72793444, 8913721, 63506281, 73617245, 62936963, 61815223, 83948335, 91957544, 92692283, 57163802, 33249630, 12303248, 22766820, 83747892, 41245325, 4508700, 29834512, 52204879, 52060076, 59501487, 96193415, 99333375, 45381876, 14349098, 82779622, 37891451, 97057187, 67793644, 90090964, 48893685, 62693428, 36812683, 18833224, 44842615, 16405341, 85242963, 80316608, 45790169, 3233084, 38510840, 21001913, 68102437, 28550822, 63372756, 54762643, 9623492, 96420429, 57248122, 6038457, 74137926, 51315460, 89419466, 61271144, 36933038, 63628376, 67451935, 59405277, 18466635, 90061527, 51590803, 15163258, 89996536, 61712234, 85695762, 59197747, 5482538, 88130087, 30090481, 77377183, 52613508, 30569392, 5640302, 21289531, 61263068, 77312810, 23134715, 42199455, 76671482, 16019925, 84293052, 84002370, 45428665, 92604458, 66885828, 12348118, 18663507, 51464002, 51426311, 56473732, 41442762, 68644627, 38022834, 77787724, 27041967, 43376279, 96726697, 24314885, 72238278, 85571389, 29994197, 12024238, 60955663, 6819644, 57658654, 97783876, 33565483, 26744917, 67513640, 57240218, 96709982, 7502255, 71657078, 54058788, 59318837, 10597197, 73031054, 44983451, 22721500, 14220886, 2717150, 82897371, 62452034, 88904910, 526217, 43506672, 71083565, 54517921, 9599614, 36139942, 92803766, 60176618, 8115266, 90158785, 23432750, 20642888, 22405120, 9058407, 97940276, 26139110, 4434662, 33797252, 26493119, 18001617, 97561745, 81855445, 92215320, 57961282, 84684495, 72274002, 17068582, 88251446, 99658235, 65271999, 88653118, 508198, 82427263, 55602660, 3183975, 55470718, 91990218, 10358899, 22200378, 749283, 36189527, 6497830, 92398073, 94539824, 59614746, 64098930, 67227442, 32699744, 24058273, 89499542, 35456853, 8729146, 8129978, 7423788, 42644903, 38341669, 60581278, 37183543, 86543538, 7300047, 89804152, 55189057, 82052050, 13468268, 30653863, 46851987, 4798568, 80555751, 30463802, 7955293, 415901, 72738685, 29188588, 6793819, 89214616, 42073124, 7646095, 93562257, 6505939, 23740167, 55960386, 99965001, 47090124, 25636669, 33237508, 22113133, 2607799, 91664334, 7182642, 14045383, 77898274, 20122224, 82532312, 91938191, 19272365, 50188404, 68204242, 45919976, 10264691, 51466049, 83302115, 8733068, 78300864, 72357096, 70420215, 62762857, 55669657, 41092102, 31727379, 26292919, 99224392, 21397057, 30771409, 40534591, 74743862, 8696647, 45306070, 321665, 72358170, 44627776, 57020481, 45075471, 39553046, 92530431, 82339363, 26664538, 82327024, 93057697, 91240048, 83651211, 72725103, 41092172, 76540481, 35996293, 2891150, 93053405, 68041839, 44473167, 57241521, 58208470, 3509435, 44348328, 25842078, 17957593, 71965942, 90013093, 17894977, 43357947, 26998766, 91141395, 58224549, 28787861, 53842979, 26776922, 49328608, 96735716, 26114953, 33628349, 75052463, 98943869, 9860968, 83150534, 95010552, 26392416, 42237907, 36580610, 30366150, 64848072, 17976208, 6157724, 14626618, 34493392, 54663246, 6525948, 98130363, 15015906, 20681921, 65338021, 78884452, 22879907, 62232211, 22942635, 44844121, 68110330, 38009615, 64087743, 3773993, 22108665, 45990383, 78561158, 69255765, 62803858, 73168565, 29932657, 17016156, 42967683, 94360702, 98653983, 40197395, 85117092, 76703609, 48260151, 54427233, 79880247, 84904436, 62740044, 72614359, 44847298, 61741594, 39847321, 29635537, 9175338, 85116755, 7685448, 99690194, 59177718, 43322743, 42692881, 77694700, 44245960, 81172706, 71522968, 36808486, 86118021, 34044787, 71333116, 29466406, 85711894, 41481685, 61859143, 27625735, 92692978, 27665211, 18699206, 49271185, 1204161, 65074535, 56153345, 23194618, 29401781, 72777973, 72373496, 64055960, 1431742, 66319530, 98948034, 99226875, 66663942, 61982238, 34497327, 77187825, 62428472, 11581181, 39986008, 45996863, 6986898, 17386996, 22129328, 47738185, 45995325, 40152546, 247198, 94076128, 4985896, 11365791, 92353856, 69697787, 68694897, 5111370, 56531125, 29065964, 19457589, 95251277, 43501211, 61960400, 79191827, 99917599, 32161669, 69136837, 13470059, 27411561, 79942022, 91727510, 29510992, 17539625, 13231279, 75820087, 96852321, 8099994, 51715482, 84840549, 61623915, 11923835, 34698428, 45237957, 3088684, 28796059, 3880712, 62115552, 81805959, 8651647, 91831487, 75229462, 57393458, 13862149, 37280276, 48675329, 9860195, 65454636, 11161731, 27325428, 77272628, 30764367, 14723410, 1197320, 6153406, 70596786, 19898053, 38658347, 73222868, 21673260, 62490109, 80246713, 54263880, 87598888, 31733363, 40027975, 63059024, 10649306, 95395112, 20765474, 31161687, 62051033, 75777973, 61728685, 82886381, 33123618, 29959549, 1022677, 77301523, 33553959, 6111563, 39171895, 48088883, 53666583, 58180653, 26063929, 874791, 65081429, 26507214, 36845587, 32590267, 37560164, 18411915, 98371444, 51047803, 32426752, 63967300, 92998591, 27187213, 29029316, 44889423, 33699435, 89811711, 18131876, 88047921, 16054533, 36135, 81774825, 7517032, 24915585, 30811010, 90654836, 54232247, 65047700, 86798033, 92033260, 76960303, 4069912, 99021067, 73786944, 94711842, 20645197, 4668450, 14731700, 20002147, 20140249, 9603598, 54606384, 87710366, 17037369, 84187166, 68939068, 15536795, 40686254, 17385531, 1250437, 79821897, 50806615, 97641116, 30163921, 66832478, 91255408, 6521313, 2917920, 67124282, 44846932, 45049308, 32274392, 31904591, 68824981, 15148031, 24733232, 59109400, 76825057, 4787945, 45407418, 59371804, 49598724, 45667668, 29516592, 98462867, 83133790, 33304202, 72019362, 66611759, 93270084, 66250369, 33895336, 2208785, 79922758, 23569917, 1936762, 68816413, 60106217, 14093520, 67084351, 54014062, 44915235, 37957788, 53802686, 15948937, 97011160, 81677380, 62430984, 38008118, 16595436, 53393358, 55770687, 21070110, 82178706, 81898046, 40984766, 12416768, 38061439, 24619760, 44177011, 176257, 15064655, 76170907, 48360198, 65880522, 64602895, 67031644, 13348726, 98648327, 47213183, 37675718, 82979980, 26275734, 36685023, 34698463, 51507425, 73124510, 77620120, 75986488, 48658605, 96906410, 54868730, 70372191, 37620363, 79136082, 7066775, 18783702, 4099191, 56461322, 8791066, 17146629, 39373729, 71316369, 5970607, 74357852, 49641577, 82651278, 90004325, 40781449, 72732205, 4119747, 63015256, 44479073, 16567550, 18806856, 99524975, 56484900, 30218878, 73021291, 56424103, 74452589, 89699445, 12664567, 36396314, 53632373, 84127901, 83155442, 17727650, 46540998, 16380211, 99971982, 168541, 54987042, 669105, 86821229, 72278539, 89637706, 94911072, 99125126, 9886593, 36753250, 44481640, 10366309, 5267545, 84349107, 38645117, 70800879, 83533741, 66137019, 75543508, 97281847, 8373289, 8634541, 78218845, 19891772, 69579137, 69641513, 26863229, 19939935, 35092039, 76315420, 93359396, 87160386, 28928175, 40865610, 10961421, 36312813, 30998561, 60430369, 85023028, 42947632, 36468541, 70782102, 88444207, 1788101, 49236559, 9829782, 93515664, 10309525, 32250045, 72495719, 17365188, 68128438, 32159704, 34236719, 26734892, 45794415, 69605283, 20486294, 53648154, 1569515, 79738755, 93790285, 75135448, 55753905, 19486173, 78589145, 76434144, 68000591, 61928316, 62552524, 3235882, 60102965, 4204661, 51149804, 42307449, 80014588, 37739481, 35192533, 48892201, 38365584, 49597667, 41380093, 92541302, 15535065, 69786756, 71469330, 59329511, 78766358, 55143724, 68316156, 57359924, 37659250, 79806380, 87720882, 95726235, 60393039, 47298834, 40781854, 67474219, 3294781, 62496012, 79322415, 38256849, 24168411, 8128637, 99515901, 82726008, 76330843, 34946859, 61897582, 77300457, 83210802, 40677414, 90310261, 4095116, 33336148, 94090109, 13173644, 27185644, 78785507, 6221471, 62357986, 8055981, 79657802, 4515343, 7229550, 78602717, 30694952, 9259676, 38494874, 4199704, 4978543, 22997281, 84166196, 7919588, 24591705, 3633375, 53547802, 21993752, 15902805, 86460488, 30395570, 30787683, 20867149, 83789391, 70727211, 37224844, 24826575, 9575954, 86301513, 39801351, 65275241, 92867155, 43933006, 17058722, 30543215, 66271566, 11543098, 70367851, 63152504, 67281495, 59910624, 69355476, 71920426, 74110882, 66428795, 37435892, 24953498, 67030811, 36930650, 86022504, 50567636, 56955985, 45848907, 77413012, 57803235, 17081350, 52261574, 48774913, 29819940, 32151165, 65851721, 9398733, 56515456, 19376156, 48673079, 26102057, 22435353, 80251430, 48395186, 7104732, 68702632, 66903004, 69848388, 65715134, 99604946, 61373987, 99861373, 12571310, 44784505, 92071990, 68875490, 93933709, 36505482, 19101477, 34295794, 71300104, 5073754, 2204165, 37501808, 70541760, 99297164, 16424199, 28851716, 33935899, 40356877, 31491938, 824872, 46870723, 5832946, 34432810, 65017137, 23848565, 92787493, 94516935, 33431960, 21533347, 49882705, 53084256, 95581843, 66045587, 42782652, 81274566, 21919959, 28734791, 20885148, 17857111, 33061250, 64157906, 75407347, 55615885, 6808825, 50007421, 16971929, 32058615, 50668599, 83083131, 77284619, 44060493, 96318094, 6871053, 61141874, 75153252, 26397786, 20836893, 52293995, 66322959, 73109997, 7687278, 74441448, 33201905, 37192445, 16387575, 65038678, 75128745, 54199160, 66116458, 53888755, 81853704, 30139692 +59614746, 56461322, 6871053, 4787945, 28734791, 33237508, 14349098, 71657078, 57020481, 95251277, 43506672, 80316608, 81855445, 87160386, 78602717, 70782102, 26734892, 86301513, 20765474, 13468268, 72738685, 42692881, 47090124, 14045383, 37659250, 68204242, 8128637, 83155442, 67793644, 82339363, 82327024, 84349107, 16405341, 45790169, 83150534, 6497830, 31161687, 17016156, 94360702, 36845587, 80555751, 12348118, 71469330, 72732205, 18699206, 86242799, 97783876, 77187825, 37166608, 11581181, 96709982, 86821229, 68694897, 79191827, 61859581, 69136837, 38645117, 79942022, 45617087, 90272749, 8634541, 68102437, 36312813, 14326617, 65338021, 44844121, 64087743, 89046466, 76170907, 12571310, 21289531, 40197395, 85117092, 51507425, 46851987, 44847298, 30463802, 57163802, 48658605, 7685448, 6505939, 23740167, 7011964, 33699435, 29834512, 16424199, 20122224, 4119747, 99021067, 60955663, 6819644, 67030811, 56424103, 77072625, 87710366, 84187166, 68939068, 52261574, 82779622, 40152546, 73031054, 97641116, 74743862, 94911072, 91281584, 65038678, 43501211, 74614639, 66137019, 2891150, 47361209, 21533347, 51715482, 75153252, 45237957, 44664587, 68702632, 54199160, 81805959, 78549759, 66903004, 42947632, 95290172, 36468541, 13862149, 10358899, 37280276, 47893286, 89996536, 14723410, 7919588, 65715134, 32699744, 15902805, 22108665, 69255765, 5640302, 39801351, 95395112, 60581278, 29959549, 37675718, 77312810, 42967683, 86543538, 39171895, 76703609, 72614359, 23110625, 67281495, 56473732, 5970607, 18131876, 96726697, 88047921, 55143724, 49641577, 41481685, 77898274, 40781449, 82532312, 65074535, 14363867, 87720882, 72373496, 16567550, 95726235, 12024238, 9603598, 55669657, 77413012, 17386996, 247198, 10597197, 97057187, 168541, 53888755, 54987042, 9398733, 44846932, 36812683, 61960400, 92530431, 44481640, 5267545, 23432750, 51588015, 4806458, 26102057, 85242963, 18001617, 33431960, 19891772, 21001913, 43357947, 76315420, 22450468, 93359396, 88251446, 99658235, 53084256, 75128745, 58749, 28796059, 79657802, 49328608, 4515343, 62115552, 38494874, 9860968, 36189527, 30764367, 54663246, 17857111, 67227442, 53393358, 70596786, 73392814, 81898046, 30891921, 21673260, 70727211, 87598888, 75135448, 59197747, 5482538, 98648327, 8129978, 43045786, 13819358, 62051033, 89217461, 33553959, 26275734, 89804152, 48260151, 26063929, 80014588, 32590267, 49597667, 73124510, 29188588, 9175338, 71300104, 12303248, 43322743, 47708850, 37501808, 93562257, 25636669, 59329511, 27041967, 52204879, 28851716, 92692978, 71920426, 49271185, 51466049, 47298834, 31491938, 24953498, 20645197, 1431742, 30218878, 20140249, 59501487, 99226875, 96193415, 54606384, 37192445, 84127901, 6986898, 46870723, 34946859, 32151165, 16380211, 67124282, 56515456, 99125126, 9886593, 77300457, 15148031, 26664538, 90310261, 83651211, 72725103, 20642888, 4095116, 27411561, 35996293, 80251430, 17539625, 13231279, 96852321, 86001008, 83133790, 71965942, 38510840, 27185644, 16445503, 3487592, 9623492, 95581843, 3880712, 3183975, 74137926, 8651647, 81274566, 59981773, 47792865, 36580610, 49236559, 749283, 48675329, 37957788, 9860195, 59405277, 6525948, 69848388, 20681921, 24058273, 21070110, 53648154, 82178706, 22942635, 68110330, 12416768, 3773993, 37224844, 19486173, 76434144, 92071990, 73168565, 61728685, 55850790, 82886381, 23134715, 48088883, 3233569, 30543215, 36685023, 16019925, 874791, 84293052, 78909561, 37739481, 35192533, 61741594, 37560164, 77620120, 75986488, 6793819, 16099750, 98371444, 92998591, 27187213, 74724075, 63152504, 79136082, 83747892, 18504197, 39373729, 89811711, 29466406, 43376279, 81774825, 24915585, 27625735, 54232247, 27665211, 33935899, 91938191, 39201414, 73786944, 40356877, 83083131, 40781854, 20002147, 77284619, 98948034, 824872, 57658654, 11757872, 36780454, 31727379, 67513640, 53632373, 82726008, 40534591, 96318094, 61897582, 44983451, 91255408, 22721500, 14220886, 11365791, 8696647, 5111370, 321665, 72358170, 88904910, 61141874, 29065964, 36753250, 24733232, 36139942, 60176618, 76825057, 76540481, 94516935, 91727510, 22435353, 68041839, 45667668, 97561745, 94090109, 78218845, 57961282, 75820087, 84684495, 19939935, 17957593, 72019362, 26998766, 40865610, 65271999, 48395186, 93270084, 66250369, 53842979, 26776922, 508198, 91802888, 51315460, 21919959, 61271144, 1788101, 22200378, 67451935, 65454636, 92398073, 10309525, 32250045, 22997281, 68128438, 38008118, 98130363, 1197320, 61712234, 53547802, 19898053, 55770687, 69605283, 20486294, 62490109, 99604946, 33061250, 30395570, 79738755, 93790285, 55753905, 48360198, 13348726, 45990383, 63059024, 68875490, 98739783, 75777973, 1022677, 43933006, 7300047, 17058722, 51149804, 34698463, 63506281, 73617245, 30653863, 65081429, 84002370, 26507214, 4798568, 61815223, 19101477, 1239555, 92692283, 51047803, 42073124, 63967300, 44889423, 71522968, 18663507, 37620363, 73109997, 51464002, 70541760, 7066775, 55960386, 50007421, 8791066, 41442762, 58751351, 22113133, 68644627, 77787724, 74357852, 85711894, 32058615, 7517032, 82651278, 4091162, 74110882, 79806380, 65047700, 63015256, 76960303, 50188404, 56153345, 29401781, 85571389, 60393039, 56484900, 55247455, 73021291, 79322415, 74452589, 86022504, 24168411, 26744917, 57240218, 50702367, 40686254, 47738185, 48774913, 21397057, 44060493, 5832946, 54058788, 30163921, 2717150, 69697787, 2917920, 45306070, 62452034, 71083565, 32274392, 83368048, 31904591, 93566986, 44842615, 9599614, 93057697, 59109400, 8115266, 90158785, 35512853, 45407418, 92787493, 41092172, 26139110, 26493119, 29516592, 92215320, 3509435, 3233084, 35092039, 78785507, 61623915, 6221471, 54762643, 62357986, 7104732, 58224549, 73235980, 28787861, 96735716, 57248122, 79922758, 33628349, 9259676, 75052463, 68816413, 36933038, 91990218, 63628376, 64848072, 9829782, 18466635, 17365188, 14626618, 34493392, 64098930, 20836893, 89499542, 85695762, 21993752, 22879907, 38009615, 86460488, 61373987, 1569515, 20867149, 31733363, 40027975, 15064655, 30090481, 68000591, 9575954, 3235882, 47213183, 66116458, 92554120, 77301523, 37183543, 93933709, 6111563, 98653983, 55189057, 66271566, 52293995, 99729357, 79880247, 48892201, 91957544, 15535065, 39847321, 92604458, 96906410, 70372191, 59177718, 112651, 66885828, 44245960, 2204165, 51426311, 18783702, 4099191, 4508700, 34044787, 95957797, 78766358, 16054533, 59910624, 36135, 68316156, 57359924, 69355476, 1204161, 19272365, 66428795, 4069912, 50668599, 9257405, 10264691, 18806856, 64055960, 72357096, 62496012, 34497327, 89699445, 99333375, 26292919, 1250437, 99224392, 4064751, 94076128, 94595668, 72278539, 4985896, 6521313, 90090964, 89637706, 92803766, 83210802, 40677414, 70800879, 75543508, 59371804, 4434662, 44473167, 8373289, 57241521, 29510992, 88698958, 33304202, 17068582, 8099994, 84840549, 28928175, 10961421, 34698428, 66611759, 88653118, 66667729, 60430369, 7229550, 6038457, 26114953, 60106217, 75229462, 88444207, 26397786, 26392416, 67084351, 57393458, 54014062, 70036438, 20885148, 15948937, 90061527, 15163258, 32159704, 15015906, 3633375, 78884452, 35456853, 8807940, 38061439, 44177011, 70004753, 83789391, 67031644, 44784505, 52613508, 30569392, 38341669, 62803858, 33123618, 60102965, 4204661, 53666583, 42199455, 82052050, 76671482, 84904436, 415901, 38365584, 45428665, 85116755, 9188443, 18411915, 89214616, 99690194, 36808486, 86118021, 17146629, 99297164, 71333116, 61859143, 52060076, 90654836, 23194618, 72238278, 29994197, 1375023, 99524975, 4668450, 14731700, 66319530, 5822202, 66663942, 70420215, 41092102, 33201905, 33565483, 15536795, 12664567, 36396314, 99515901, 76330843, 22129328, 79821897, 59318837, 37891451, 34432810, 65851721, 44550764, 62693428, 65017137, 45049308, 54517921, 45075471, 32161669, 23848565, 22405120, 83533741, 14947650, 97940276, 97281847, 13173644, 44348328, 69641513, 98462867, 90013093, 17894977, 91141395, 66045587, 82427263, 85023028, 14093520, 42237907, 30366150, 84406788, 4199704, 44915235, 51590803, 72495719, 77272628, 81677380, 62430984, 34236719, 24591705, 6153406, 73222868, 62232211, 30787683, 65880522, 64602895, 77377183, 61928316, 65275241, 92867155, 16097038, 72793444, 58180653, 11543098, 66322959, 7955293, 83948335, 41380093, 54868730, 69786756, 33249630, 22766820, 77694700, 7646095, 41245325, 71316369, 38022834, 31126490, 7182642, 97379790, 44479073, 8733068, 94711842, 74441448, 3294781, 36930650, 62762857, 38256849, 45996863, 30771409, 46540998, 99971982, 66832478, 669105, 92353856, 56531125, 19457589, 68824981, 19376156, 99549373, 17764950, 48673079, 33797252, 49598724, 69579137, 72274002, 90457870, 11923835, 3088684, 33895336, 2208785, 20568363, 55470718, 95010552, 51472275, 15075176, 11161731, 53802686, 34667286, 84166196, 16595436, 45794415, 38658347, 24619760, 54263880, 99861373, 8729146, 88130087, 78589145, 62552524, 75407347, 10649306, 42644903, 29932657, 47887470, 61263068, 569864, 82979980, 8913721, 42307449, 62936963, 34295794, 5073754, 81172706, 2607799, 91664334, 7687278, 86798033, 16684829, 72777973, 83302115, 43152977, 61982238, 50567636, 39986008, 45848907, 89078848, 17081350, 7502255, 10453030, 45995325, 82897371, 48893685, 44627776, 16387575, 18833224, 10366309, 91240048, 13470059, 9058407, 93053405, 49882705, 28550822, 8055981, 30998561, 96420429, 55602660, 23569917, 30694952, 98943869, 91831487, 89419466, 93515664, 4978543, 17976208, 6157724, 27325428, 94539824, 80246713, 176257, 64157906, 78561158, 54427233, 55615885, 36505482, 29635537, 32426752, 29029316, 99965001, 16971929, 90004325, 24314885, 92033260, 78300864, 62428472, 57803235, 50806615, 526217, 39553046, 99917599, 30139692, 58208470, 25842078, 63372756, 1936762, 81853704, 40984766, 7423788, 2331773, 62740044, 92541302, 70367851, 6808825, 30811010, 37435892, 83269727, 56955985, 17727650, 97011160, 45381876, 17385531, 29819940, 33336148, 45919976, 26863229, 42782652, 24826575, 67474219, 17037369 +17539625, 14723410, 96726697, 77898274, 44844121, 43933006, 44889423, 74110882, 72777973, 98948034, 74743862, 5111370, 31904591, 68041839, 57961282, 72019362, 22108665, 92604458, 87720882, 86242799, 22721500, 44550764, 93566986, 94090109, 78602717, 14326617, 1788101, 57393458, 67451935, 81677380, 20486294, 8729146, 63059024, 89217461, 30543215, 84293052, 80555751, 93562257, 67281495, 58751351, 5970607, 90004325, 76960303, 94711842, 39986008, 52261574, 7502255, 2917920, 57020481, 54517921, 32274392, 45407418, 48673079, 33431960, 33304202, 10961421, 44664587, 49328608, 85023028, 81274566, 70782102, 22200378, 28734791, 4978543, 14626618, 34236719, 1197320, 85695762, 35456853, 30787683, 47213183, 86301513, 43045786, 13819358, 62803858, 98653983, 13468268, 84904436, 65081429, 62740044, 32590267, 92692283, 42073124, 12303248, 43322743, 7646095, 68644627, 41481685, 73786944, 40781854, 6819644, 30218878, 56424103, 77072625, 79821897, 45306070, 72358170, 82327024, 93057697, 99549373, 59371804, 80316608, 86001008, 38510840, 17894977, 66611759, 9259676, 55470718, 54014062, 49236559, 70036438, 92398073, 10309525, 90061527, 94539824, 45794415, 21070110, 53648154, 22879907, 38009615, 19486173, 62552524, 65275241, 82886381, 93933709, 94360702, 17058722, 99729357, 77620120, 6793819, 32426752, 22766820, 37620363, 18504197, 56473732, 25636669, 41442762, 77787724, 27041967, 88047921, 49641577, 66428795, 23194618, 60393039, 67030811, 72357096, 59501487, 99333375, 31727379, 17037369, 8128637, 15536795, 17727650, 34946859, 59318837, 97057187, 73031054, 61897582, 86821229, 4985896, 68694897, 29065964, 16387575, 61960400, 92530431, 44842615, 9599614, 60176618, 76825057, 8115266, 72725103, 22405120, 76540481, 97940276, 91727510, 90272749, 57241521, 80251430, 88698958, 8099994, 90457870, 78785507, 88251446, 75153252, 88653118, 66667729, 28796059, 60430369, 57248122, 6038457, 26114953, 33628349, 95290172, 13862149, 64848072, 51472275, 37957788, 11161731, 18466635, 62430984, 89996536, 59614746, 54663246, 17857111, 81853704, 32699744, 81898046, 80246713, 3773993, 44177011, 20867149, 37224844, 5482538, 78589145, 67031644, 68000591, 9575954, 30569392, 39801351, 55850790, 47887470, 17016156, 29959549, 77301523, 23134715, 58180653, 11543098, 54427233, 26507214, 23110625, 85116755, 9188443, 18411915, 48658605, 96906410, 99690194, 69786756, 59177718, 36808486, 99965001, 18783702, 4099191, 47090124, 33699435, 34044787, 95957797, 85711894, 59910624, 16424199, 36135, 61859143, 4091162, 4119747, 54232247, 91938191, 49271185, 86798033, 14363867, 4069912, 68204242, 10264691, 43152977, 14731700, 3294781, 20140249, 99226875, 77187825, 41092102, 87710366, 45848907, 12664567, 26292919, 89078848, 17081350, 82726008, 99224392, 46540998, 247198, 94076128, 168541, 53888755, 54987042, 669105, 72278539, 91255408, 8696647, 321665, 44846932, 19457589, 65038678, 43506672, 43501211, 91240048, 69136837, 83651211, 4787945, 26102057, 83533741, 26139110, 93053405, 8373289, 19891772, 81855445, 44348328, 43357947, 76315420, 28928175, 61623915, 75128745, 63372756, 58224549, 73235980, 9623492, 45237957, 68702632, 79657802, 30998561, 4515343, 62115552, 74137926, 30694952, 51315460, 20568363, 83150534, 36580610, 47893286, 93515664, 59405277, 6157724, 65454636, 15948937, 32159704, 84166196, 6525948, 3633375, 6153406, 19898053, 69605283, 73222868, 30891921, 38061439, 12571310, 24826575, 48360198, 30090481, 44784505, 92071990, 5640302, 10649306, 31161687, 75777973, 33553959, 82979980, 37183543, 42967683, 6111563, 16097038, 53666583, 85117092, 48260151, 55615885, 2331773, 62936963, 36505482, 35192533, 61815223, 61741594, 38365584, 73124510, 92541302, 75986488, 51047803, 71300104, 27187213, 12348118, 74724075, 71522968, 63152504, 41245325, 86118021, 89811711, 59329511, 52204879, 2607799, 16054533, 81774825, 69355476, 99021067, 9257405, 83302115, 8733068, 37435892, 24953498, 4668450, 67474219, 77284619, 66663942, 57658654, 36930650, 97783876, 55669657, 24168411, 89699445, 67513640, 36396314, 53632373, 6986898, 40686254, 1250437, 21397057, 71657078, 54058788, 34432810, 94595668, 67793644, 97641116, 11365791, 94911072, 91281584, 36753250, 95251277, 61859581, 84349107, 38645117, 13470059, 35512853, 17764950, 20642888, 79942022, 75543508, 4434662, 97561745, 29516592, 29510992, 84684495, 96852321, 83133790, 21001913, 27185644, 22450468, 58749, 34698428, 65271999, 48395186, 7104732, 8055981, 91802888, 3183975, 8651647, 9860968, 91831487, 42947632, 75229462, 21919959, 36933038, 63628376, 749283, 4199704, 6497830, 51590803, 72495719, 20836893, 61712234, 78884452, 21993752, 38658347, 40984766, 24619760, 61373987, 70004753, 83789391, 31733363, 75135448, 78561158, 66116458, 98739783, 21289531, 77312810, 39171895, 48088883, 89804152, 55189057, 36685023, 51149804, 52293995, 34698463, 16019925, 51507425, 874791, 79880247, 4798568, 72614359, 1239555, 91957544, 41380093, 72738685, 45428665, 70372191, 92998591, 66885828, 47708850, 18663507, 2204165, 71469330, 73109997, 6808825, 51426311, 50007421, 56461322, 55143724, 82651278, 40781449, 27625735, 37659250, 7687278, 19272365, 72238278, 72373496, 29994197, 39201414, 51466049, 1431742, 20002147, 66319530, 824872, 5822202, 79322415, 86022504, 26744917, 84127901, 50702367, 83155442, 76330843, 48774913, 82779622, 40534591, 96318094, 16380211, 50806615, 30163921, 44983451, 90090964, 82897371, 92353856, 88904910, 44627776, 36812683, 45049308, 77300457, 45075471, 79191827, 15148031, 24733232, 32161669, 10366309, 51588015, 4806458, 92787493, 9058407, 27411561, 22435353, 45790169, 33797252, 49598724, 26493119, 47361209, 58208470, 92215320, 98462867, 26863229, 3233084, 90013093, 72274002, 99658235, 3487592, 91141395, 36312813, 54199160, 33895336, 26776922, 66045587, 508198, 82427263, 2208785, 55602660, 98943869, 1936762, 14093520, 84406788, 9829782, 34667286, 77272628, 22997281, 69848388, 98130363, 65715134, 53393358, 89499542, 82178706, 62232211, 22942635, 15902805, 68110330, 12416768, 99604946, 33061250, 1569515, 176257, 79738755, 93790285, 64602895, 88130087, 76434144, 45990383, 3235882, 69255765, 68875490, 92554120, 92867155, 1022677, 61263068, 7300047, 3233569, 42199455, 82052050, 66271566, 42307449, 73617245, 46851987, 37739481, 7955293, 48892201, 49597667, 29188588, 9175338, 63967300, 5073754, 44245960, 6505939, 51464002, 7066775, 8791066, 4508700, 33237508, 22113133, 16971929, 31126490, 7182642, 97379790, 7517032, 71920426, 79806380, 24314885, 63015256, 16684829, 50188404, 56153345, 50668599, 40356877, 16567550, 95726235, 47298834, 56484900, 74441448, 60955663, 73021291, 61982238, 34497327, 11757872, 54606384, 57240218, 46870723, 4064751, 29819940, 40152546, 37891451, 2717150, 65017137, 71083565, 39553046, 83368048, 68824981, 44481640, 26664538, 5267545, 83210802, 40677414, 16405341, 85242963, 66137019, 2891150, 30139692, 44473167, 13173644, 13231279, 75820087, 3509435, 69641513, 19939935, 71965942, 17068582, 87160386, 26998766, 68102437, 40865610, 28550822, 11923835, 54762643, 66250369, 3880712, 96420429, 7229550, 78549759, 23569917, 38494874, 66903004, 47792865, 95010552, 61271144, 88444207, 91990218, 36189527, 9860195, 34493392, 15163258, 30764367, 26734892, 67227442, 53547802, 70596786, 55770687, 21673260, 8807940, 70727211, 99861373, 65880522, 77377183, 61928316, 52613508, 8129978, 95395112, 20765474, 42644903, 38341669, 62051033, 73168565, 29932657, 60581278, 37675718, 86543538, 72793444, 40197395, 8913721, 76703609, 63506281, 66322959, 30463802, 16099750, 7685448, 33249630, 29029316, 79136082, 83747892, 17146629, 99297164, 71333116, 29834512, 78766358, 14045383, 32058615, 24915585, 30811010, 90654836, 28851716, 82532312, 27665211, 65047700, 92033260, 85571389, 45919976, 12024238, 31491938, 55247455, 20645197, 64055960, 62496012, 9603598, 37166608, 33201905, 83269727, 33565483, 56955985, 45381876, 45996863, 22129328, 14349098, 5832946, 10453030, 45995325, 99971982, 65851721, 6521313, 67124282, 56515456, 62693428, 89637706, 62452034, 61141874, 18833224, 82339363, 23848565, 36139942, 4095116, 35996293, 33336148, 97281847, 17957593, 51715482, 93270084, 95581843, 81805959, 26392416, 20885148, 97011160, 68128438, 7919588, 20681921, 62490109, 15064655, 55753905, 59197747, 64157906, 98648327, 61728685, 60102965, 30653863, 44847298, 36845587, 78909561, 19101477, 415901, 37560164, 34295794, 29635537, 57163802, 42692881, 77694700, 81172706, 37501808, 70541760, 7011964, 39373729, 91664334, 68316156, 52060076, 57359924, 20122224, 72732205, 65074535, 44479073, 96193415, 62762857, 36780454, 62428472, 11581181, 50567636, 37192445, 77413012, 96709982, 17385531, 47738185, 6871053, 48893685, 99917599, 19376156, 92803766, 90310261, 23432750, 41092172, 94516935, 45617087, 78218845, 69579137, 21533347, 25842078, 35092039, 93359396, 84840549, 3088684, 96735716, 68816413, 26397786, 10358899, 15075176, 32250045, 27325428, 17365188, 64098930, 24591705, 24058273, 86460488, 30395570, 89046466, 87598888, 40027975, 569864, 26275734, 26063929, 83948335, 98371444, 54868730, 23740167, 55960386, 71316369, 38022834, 43376279, 92692978, 33935899, 29401781, 1375023, 78300864, 83083131, 17386996, 99515901, 69697787, 9398733, 56531125, 99125126, 9886593, 74614639, 90158785, 45667668, 18001617, 8634541, 16445503, 49882705, 53084256, 79922758, 89419466, 59981773, 67084351, 37280276, 53802686, 65338021, 64087743, 4204661, 15535065, 112651, 18131876, 18699206, 1204161, 74452589, 38256849, 68939068, 30771409, 32151165, 66832478, 526217, 59109400, 14947650, 28787861, 42782652, 75052463, 60106217, 36468541, 42237907, 15015906, 73392814, 54263880, 7423788, 80014588, 39847321, 89214616, 70367851, 74357852, 18806856, 99524975, 57803235, 44060493, 10597197, 14220886, 70800879, 30366150, 48675329, 44915235, 76170907, 13348726, 75407347, 33123618, 76671482, 84002370, 29466406, 70420215, 6221471, 62357986, 53842979, 17976208, 38008118, 16595436, 84187166 +7685448, 94911072, 24733232, 6505939, 80316608, 57241521, 54762643, 1788101, 53802686, 78589145, 37739481, 83948335, 57163802, 51426311, 37659250, 45848907, 4985896, 44846932, 32161669, 5267545, 29516592, 8099994, 66250369, 26397786, 48675329, 44915235, 176257, 82886381, 77301523, 42199455, 16019925, 84293052, 4798568, 77620120, 36808486, 39373729, 14045383, 19272365, 44479073, 57658654, 15536795, 54058788, 32151165, 90090964, 36753250, 43501211, 23432750, 97940276, 3509435, 19939935, 99658235, 48395186, 36312813, 95581843, 28796059, 3880712, 79657802, 62115552, 85023028, 30764367, 64098930, 62490109, 30395570, 75407347, 60581278, 39171895, 76671482, 13468268, 874791, 79880247, 55615885, 35192533, 39847321, 98371444, 12303248, 42692881, 29029316, 51464002, 8791066, 59329511, 81774825, 77898274, 52060076, 50188404, 99021067, 40356877, 94711842, 20002147, 72357096, 99333375, 29819940, 79821897, 59318837, 73031054, 2717150, 48893685, 45306070, 56531125, 89637706, 72358170, 74614639, 59109400, 60176618, 90158785, 48673079, 66137019, 4434662, 69641513, 35092039, 88653118, 68702632, 49328608, 55602660, 78602717, 20568363, 9860968, 60106217, 36580610, 10358899, 20885148, 17857111, 81853704, 44844121, 13348726, 9575954, 20765474, 17016156, 1022677, 37675718, 23134715, 86543538, 43933006, 80014588, 84904436, 30463802, 9175338, 16099750, 51047803, 99690194, 59177718, 81172706, 23740167, 29466406, 74357852, 96726697, 88047921, 32058615, 24915585, 30811010, 72732205, 7687278, 1204161, 76960303, 29994197, 12024238, 60393039, 47298834, 43152977, 78300864, 4668450, 67474219, 77284619, 30218878, 89699445, 11581181, 31727379, 26744917, 26292919, 53632373, 46870723, 4064751, 48774913, 82779622, 46540998, 6871053, 10597197, 86821229, 6521313, 43506672, 83368048, 9599614, 69136837, 35512853, 85242963, 91727510, 22435353, 90272749, 69579137, 96852321, 86001008, 43357947, 26998766, 62357986, 7104732, 60430369, 23569917, 59981773, 70782102, 26392416, 13862149, 749283, 4199704, 15075176, 34667286, 51590803, 97011160, 14626618, 62430984, 69848388, 61712234, 32699744, 6153406, 54263880, 89046466, 70727211, 40027975, 75135448, 59197747, 3235882, 52613508, 78561158, 63059024, 39801351, 38341669, 31161687, 55850790, 77312810, 89217461, 94360702, 48088883, 51149804, 85117092, 48260151, 62740044, 36845587, 45428665, 9188443, 54868730, 5073754, 37501808, 18504197, 41245325, 4099191, 56461322, 4508700, 5970607, 77787724, 27041967, 55143724, 49641577, 65047700, 23194618, 72777973, 9257405, 1431742, 6819644, 36930650, 38256849, 33201905, 84187166, 50567636, 57803235, 83155442, 168541, 30163921, 54987042, 14220886, 11365791, 56515456, 62693428, 88904910, 16387575, 45075471, 15148031, 61859581, 36139942, 83651211, 22405120, 4095116, 79942022, 49598724, 18001617, 97561745, 30139692, 13173644, 21533347, 88698958, 17894977, 76315420, 87160386, 68102437, 3487592, 34698428, 58224549, 73235980, 44664587, 82427263, 96420429, 8651647, 89419466, 75229462, 83150534, 54014062, 37280276, 36189527, 17976208, 90061527, 59614746, 89499542, 80246713, 38009615, 12416768, 99604946, 33061250, 76170907, 65880522, 64157906, 98648327, 47213183, 7423788, 10649306, 92554120, 75777973, 29932657, 21289531, 61263068, 89804152, 17058722, 42307449, 99729357, 26507214, 72614359, 44847298, 73124510, 15535065, 70372191, 112651, 63967300, 93562257, 70541760, 18783702, 7011964, 58751351, 34044787, 71333116, 29834512, 54232247, 71920426, 65074535, 87720882, 73786944, 45919976, 8733068, 99524975, 56484900, 40781854, 73021291, 59501487, 5822202, 56424103, 96193415, 79322415, 74452589, 54606384, 36780454, 83269727, 68939068, 12664567, 77413012, 84127901, 17081350, 17385531, 99224392, 5832946, 45995325, 94076128, 16380211, 67793644, 50806615, 669105, 44550764, 8696647, 5111370, 62452034, 57020481, 77300457, 95251277, 44481640, 83210802, 76825057, 8115266, 35996293, 33797252, 68041839, 33336148, 26493119, 8373289, 33431960, 80251430, 19891772, 75820087, 3233084, 83133790, 71965942, 38510840, 72274002, 16445503, 22450468, 84840549, 61623915, 53084256, 91141395, 96735716, 508198, 91802888, 78549759, 91831487, 88444207, 22200378, 6497830, 37957788, 9860195, 59405277, 32250045, 27325428, 17365188, 15163258, 89996536, 84166196, 53393358, 70596786, 85695762, 21070110, 20486294, 73222868, 22942635, 21673260, 8807940, 40984766, 44177011, 70004753, 83789391, 99861373, 79738755, 12571310, 55753905, 24826575, 48360198, 88130087, 67031644, 77377183, 22108665, 44784505, 86301513, 8129978, 43045786, 65275241, 62803858, 33123618, 47887470, 93933709, 7300047, 58180653, 82052050, 30543215, 66271566, 52293995, 34698463, 51507425, 73617245, 78909561, 36505482, 61815223, 7955293, 61741594, 48892201, 38365584, 37560164, 41380093, 92541302, 29188588, 6793819, 92998591, 22766820, 71522968, 18663507, 83747892, 99965001, 17146629, 99297164, 33237508, 71316369, 2607799, 91664334, 31126490, 7182642, 16054533, 97379790, 40781449, 4119747, 92692978, 33935899, 86798033, 92033260, 63015256, 56153345, 37435892, 20645197, 74441448, 86242799, 14731700, 98948034, 824872, 99226875, 70420215, 77072625, 9603598, 97783876, 33565483, 56955985, 37192445, 8128637, 57240218, 82726008, 22129328, 47738185, 30771409, 99971982, 61897582, 74743862, 9398733, 44627776, 36812683, 19457589, 65038678, 71083565, 99917599, 68824981, 44842615, 19376156, 72725103, 41092172, 16405341, 27411561, 75543508, 45790169, 97281847, 29510992, 13231279, 84684495, 17957593, 49882705, 28550822, 75153252, 93270084, 66045587, 4515343, 7229550, 6038457, 3183975, 26114953, 33628349, 1936762, 66903004, 47792865, 61271144, 67084351, 57393458, 49236559, 9829782, 51472275, 70036438, 28734791, 6157724, 65454636, 15948937, 18466635, 32159704, 34236719, 7919588, 20836893, 15015906, 65715134, 53547802, 73392814, 78884452, 15902805, 64087743, 3773993, 1569515, 31733363, 64602895, 98739783, 95395112, 13819358, 62051033, 92867155, 61728685, 82979980, 16097038, 60102965, 4204661, 3233569, 53666583, 72793444, 55189057, 76703609, 63506281, 62936963, 32590267, 23110625, 75986488, 48658605, 89214616, 42073124, 27187213, 12348118, 70367851, 71469330, 73109997, 63152504, 86118021, 25636669, 89811711, 22113133, 95957797, 52204879, 78766358, 85711894, 82651278, 90654836, 27625735, 69355476, 74110882, 82532312, 18699206, 91938191, 14363867, 4069912, 24953498, 55247455, 60955663, 66319530, 66663942, 77187825, 37166608, 67513640, 39986008, 45381876, 45996863, 89078848, 50702367, 40686254, 96709982, 99515901, 71657078, 34946859, 40534591, 10453030, 40152546, 34432810, 97057187, 97641116, 53888755, 44983451, 72278539, 29065964, 91281584, 45049308, 526217, 18833224, 61960400, 32274392, 92530431, 31904591, 93566986, 82339363, 82327024, 10366309, 91240048, 92803766, 40677414, 13470059, 4806458, 45407418, 92787493, 76540481, 93053405, 8634541, 17539625, 44348328, 98462867, 26863229, 21001913, 27185644, 17068582, 72019362, 75128745, 66667729, 28787861, 53842979, 33895336, 26776922, 42782652, 57248122, 51315460, 75052463, 98943869, 55470718, 21919959, 95290172, 36468541, 42237907, 30366150, 84406788, 47893286, 93515664, 72495719, 77272628, 94539824, 6525948, 98130363, 26734892, 20681921, 45794415, 38658347, 35456853, 53648154, 82178706, 22879907, 24619760, 93790285, 8729146, 5482538, 76434144, 30569392, 66116458, 5640302, 68875490, 42644903, 73168565, 29959549, 42967683, 6111563, 98653983, 40197395, 8913721, 80555751, 415901, 1239555, 49597667, 34295794, 29635537, 69786756, 32426752, 33249630, 74724075, 44245960, 7646095, 79136082, 67281495, 7066775, 55960386, 50007421, 56473732, 41442762, 38022834, 43376279, 59910624, 79806380, 16684829, 39201414, 31491938, 83083131, 67030811, 3294781, 62496012, 62762857, 86022504, 17037369, 6986898, 37891451, 91255408, 68694897, 2917920, 65017137, 54517921, 39553046, 93057697, 84349107, 51588015, 17764950, 20642888, 26102057, 45667668, 92215320, 25842078, 33304202, 90013093, 93359396, 78785507, 88251446, 58749, 65271999, 6221471, 66611759, 3088684, 79922758, 81805959, 30694952, 81274566, 42947632, 64848072, 67451935, 4978543, 11161731, 92398073, 81677380, 68128438, 67227442, 19898053, 81898046, 68110330, 86460488, 30787683, 20867149, 19486173, 61928316, 45990383, 92071990, 69255765, 36685023, 11543098, 26063929, 54427233, 30653863, 2331773, 91957544, 18411915, 92604458, 66885828, 77694700, 47708850, 2204165, 6808825, 33699435, 68644627, 18131876, 36135, 68316156, 7517032, 57359924, 90004325, 4091162, 20122224, 24314885, 49271185, 50668599, 72238278, 85571389, 10264691, 51466049, 20140249, 61982238, 24168411, 36396314, 17386996, 1250437, 7502255, 247198, 65851721, 66832478, 22721500, 92353856, 69697787, 99125126, 26664538, 70800879, 83533741, 94516935, 14947650, 2891150, 26139110, 45617087, 47361209, 57961282, 8055981, 54199160, 30998561, 2208785, 74137926, 95010552, 91990218, 10309525, 22997281, 54663246, 14723410, 1197320, 3633375, 69605283, 38061439, 61373987, 62552524, 569864, 37183543, 66322959, 84002370, 19101477, 43322743, 16424199, 28851716, 66428795, 29401781, 18806856, 11757872, 55669657, 62428472, 52261574, 82897371, 321665, 9886593, 79191827, 4787945, 44473167, 78218845, 81855445, 90457870, 40865610, 11923835, 63372756, 9623492, 9259676, 36933038, 34493392, 38008118, 16595436, 55770687, 30891921, 37224844, 15064655, 26275734, 65081429, 46851987, 92692283, 72738685, 44889423, 37620363, 61859143, 68204242, 72373496, 95726235, 83302115, 1375023, 41092102, 14349098, 21397057, 96318094, 67124282, 23848565, 90310261, 38645117, 99549373, 28928175, 10961421, 38494874, 14093520, 14326617, 63628376, 24591705, 24058273, 21993752, 87598888, 68000591, 33553959, 71300104, 96906410, 47090124, 16971929, 27665211, 16567550, 34497327, 76330843, 17727650, 44060493, 94595668, 59371804, 94090109, 58208470, 45237957, 68816413, 65338021, 41481685, 64055960, 87710366, 61141874, 9058407, 51715482, 62232211, 30090481, 85116755 +83651211, 72019362, 75128745, 6038457, 75229462, 64602895, 76434144, 26507214, 29029316, 91664334, 77187825, 62428472, 50806615, 30163921, 75153252, 49236559, 92071990, 61728685, 77301523, 11543098, 54427233, 29188588, 67281495, 78766358, 81774825, 76960303, 73786944, 34497327, 1250437, 52261574, 32151165, 669105, 11365791, 19457589, 92530431, 4787945, 40865610, 6221471, 66611759, 66667729, 33895336, 4515343, 36933038, 749283, 94539824, 34236719, 65715134, 45794415, 68000591, 62552524, 44784505, 75407347, 62051033, 29959549, 33553959, 37183543, 30543215, 42073124, 6808825, 41442762, 68644627, 16424199, 4069912, 87720882, 1375023, 24953498, 3294781, 79322415, 37192445, 45848907, 74743862, 94911072, 65038678, 31904591, 36139942, 45407418, 79942022, 94090109, 69641513, 26863229, 38510840, 78785507, 82427263, 2208785, 96420429, 62115552, 38494874, 57393458, 48675329, 44915235, 10309525, 81853704, 70596786, 89499542, 81898046, 176257, 37224844, 29932657, 82886381, 89804152, 52293995, 55615885, 36845587, 70372191, 32426752, 22766820, 7646095, 37620363, 96726697, 31126490, 4091162, 90654836, 54232247, 7687278, 66428795, 68204242, 99524975, 83083131, 14731700, 61982238, 36930650, 89699445, 8128637, 57803235, 6986898, 29819940, 59318837, 66832478, 53888755, 92353856, 48893685, 9886593, 526217, 95251277, 32274392, 32161669, 23848565, 99549373, 4095116, 85242963, 4434662, 33797252, 49598724, 97281847, 33431960, 92215320, 93359396, 11923835, 8055981, 66250369, 66045587, 78549759, 30694952, 1936762, 68816413, 55470718, 70782102, 37280276, 92398073, 81677380, 22997281, 32159704, 17857111, 16595436, 24058273, 22942635, 44177011, 70004753, 15064655, 8729146, 5482538, 67031644, 47213183, 69255765, 30569392, 63059024, 92554120, 65275241, 17016156, 48088883, 4204661, 40197395, 66271566, 80014588, 79880247, 36505482, 35192533, 48892201, 32590267, 72738685, 16099750, 59177718, 63967300, 92998591, 27187213, 93562257, 79136082, 4099191, 22113133, 5970607, 74357852, 85711894, 40781449, 27665211, 92033260, 29401781, 72777973, 824872, 11757872, 86022504, 24168411, 57240218, 17386996, 99515901, 17727650, 47738185, 34946859, 96318094, 16380211, 34432810, 73031054, 168541, 54987042, 44983451, 72278539, 14220886, 8696647, 2917920, 9398733, 56515456, 321665, 44846932, 9599614, 38645117, 4806458, 48673079, 26102057, 66137019, 59371804, 93053405, 33336148, 26493119, 30139692, 8373289, 57241521, 17539625, 13231279, 44348328, 90013093, 76315420, 99658235, 28928175, 58749, 63372756, 48395186, 58224549, 3088684, 28787861, 26776922, 96735716, 91802888, 33628349, 21919959, 26397786, 6157724, 65454636, 15163258, 14723410, 69848388, 98130363, 1197320, 61712234, 15015906, 32699744, 53547802, 19898053, 35456853, 53648154, 73222868, 30891921, 80246713, 15902805, 68110330, 99604946, 61373987, 70727211, 75135448, 24826575, 9575954, 52613508, 66116458, 68875490, 21289531, 37675718, 77312810, 93933709, 6111563, 60102965, 17058722, 42199455, 36685023, 51149804, 26063929, 72614359, 44847298, 80555751, 1239555, 34295794, 15535065, 6793819, 29635537, 9175338, 85116755, 7685448, 96906410, 99690194, 42692881, 37501808, 50007421, 56461322, 99297164, 39373729, 88047921, 14045383, 41481685, 32058615, 7517032, 90004325, 4119747, 69355476, 74110882, 18699206, 91938191, 99021067, 40356877, 10264691, 95726235, 74441448, 40781854, 77284619, 98948034, 30218878, 72357096, 9603598, 87710366, 50567636, 12664567, 40686254, 17081350, 46870723, 5832946, 79821897, 94076128, 99971982, 22721500, 90090964, 44550764, 45306070, 5111370, 89637706, 71083565, 51588015, 41092172, 91727510, 2891150, 26139110, 80316608, 22435353, 90272749, 44473167, 78218845, 81855445, 84684495, 19939935, 27185644, 68102437, 28550822, 53084256, 91141395, 65271999, 45237957, 36312813, 68702632, 54199160, 79657802, 49328608, 60430369, 79922758, 74137926, 51315460, 9259676, 47792865, 88444207, 1788101, 67084351, 91990218, 22200378, 28734791, 59405277, 20885148, 15948937, 32250045, 77272628, 97011160, 14626618, 34493392, 62430984, 6525948, 20836893, 65338021, 21993752, 38658347, 20486294, 40984766, 86460488, 30395570, 79738755, 55753905, 59197747, 48360198, 88130087, 3235882, 43045786, 39801351, 42644903, 75777973, 60581278, 1022677, 42967683, 86543538, 43933006, 58180653, 55189057, 48260151, 46851987, 4798568, 62936963, 78909561, 415901, 49597667, 41380093, 75986488, 45428665, 9188443, 98371444, 33249630, 44889423, 81172706, 71469330, 36808486, 23740167, 99965001, 25636669, 58751351, 34044787, 77787724, 97379790, 49271185, 65047700, 19272365, 50668599, 72373496, 51466049, 94711842, 60393039, 47298834, 55247455, 64055960, 78300864, 60955663, 67474219, 99226875, 97783876, 83269727, 17037369, 39986008, 45381876, 50702367, 83155442, 96709982, 17385531, 82726008, 4064751, 48774913, 44060493, 30771409, 6871053, 40152546, 65851721, 86821229, 68694897, 65017137, 88904910, 36812683, 43506672, 74614639, 54517921, 45075471, 39553046, 79191827, 82339363, 44481640, 15148031, 26664538, 61859581, 44842615, 93057697, 84349107, 69136837, 83210802, 40677414, 8115266, 13470059, 92787493, 27411561, 94516935, 14947650, 45667668, 8634541, 80251430, 29510992, 57961282, 98462867, 72274002, 90457870, 51715482, 88251446, 61623915, 34698428, 7104732, 30998561, 78602717, 20568363, 8651647, 9860968, 14326617, 83150534, 36468541, 64848072, 9829782, 70036438, 36189527, 6497830, 67451935, 18466635, 68128438, 54663246, 7919588, 24591705, 20681921, 3633375, 6153406, 55770687, 85695762, 21070110, 62490109, 12416768, 64087743, 1569515, 20867149, 83789391, 31733363, 12571310, 78589145, 13348726, 30090481, 86301513, 10649306, 92867155, 61263068, 89217461, 23134715, 16097038, 94360702, 53666583, 26275734, 82052050, 8913721, 76703609, 51507425, 84293052, 73617245, 84904436, 30653863, 2331773, 84002370, 19101477, 38365584, 37560164, 92692283, 77620120, 51047803, 89214616, 63152504, 51426311, 86118021, 56473732, 7011964, 33237508, 89811711, 29834512, 16971929, 52204879, 43376279, 68316156, 82651278, 27625735, 71920426, 82532312, 79806380, 33935899, 1204161, 14363867, 83302115, 12024238, 86242799, 4668450, 66319530, 5822202, 66663942, 41092102, 11581181, 99333375, 56955985, 26292919, 84127901, 76330843, 82779622, 7502255, 46540998, 10453030, 45995325, 97057187, 97641116, 91255408, 62452034, 72358170, 61141874, 44627776, 43501211, 83368048, 99917599, 93566986, 68824981, 90310261, 60176618, 72725103, 9058407, 83533741, 97940276, 68041839, 29516592, 19891772, 75820087, 25842078, 83133790, 35092039, 17068582, 43357947, 87160386, 49882705, 10961421, 88653118, 9623492, 95581843, 55602660, 81805959, 91831487, 61271144, 26392416, 13862149, 84406788, 93515664, 4978543, 17976208, 9860195, 17365188, 84166196, 26734892, 73392814, 62232211, 38009615, 38061439, 54263880, 99861373, 40027975, 76170907, 19486173, 64157906, 98648327, 61928316, 78561158, 5640302, 20765474, 38341669, 31161687, 47887470, 39171895, 98653983, 34698463, 42307449, 16019925, 66322959, 61741594, 91957544, 73124510, 23110625, 92541302, 18411915, 48658605, 69786756, 43322743, 66885828, 77694700, 74724075, 70367851, 73109997, 83747892, 18504197, 70541760, 55960386, 18783702, 47090124, 8791066, 59329511, 2607799, 55143724, 16054533, 59910624, 52060076, 57359924, 30811010, 92692978, 65074535, 44479073, 23194618, 72238278, 9257405, 85571389, 43152977, 37435892, 20645197, 6819644, 67030811, 73021291, 20140249, 62496012, 37166608, 77413012, 99224392, 21397057, 247198, 94595668, 2717150, 82897371, 69697787, 56531125, 99125126, 29065964, 57020481, 36753250, 77300457, 18833224, 61960400, 10366309, 19376156, 91240048, 23432750, 35512853, 22405120, 16405341, 70800879, 75543508, 45617087, 97561745, 69579137, 88698958, 96852321, 86001008, 33304202, 17894977, 8099994, 26998766, 73235980, 3880712, 53842979, 57248122, 3183975, 26114953, 23569917, 75052463, 85023028, 14093520, 95290172, 95010552, 54014062, 37957788, 59614746, 38008118, 21673260, 8807940, 33061250, 93790285, 77377183, 45990383, 8129978, 7423788, 98739783, 95395112, 569864, 7300047, 76671482, 13468268, 62740044, 37739481, 30463802, 92604458, 12303248, 5073754, 12348118, 47708850, 2204165, 7066775, 17146629, 71316369, 29466406, 7182642, 49641577, 36135, 61859143, 20122224, 86798033, 63015256, 18806856, 1431742, 20002147, 59501487, 56424103, 96193415, 77072625, 57658654, 74452589, 55669657, 38256849, 54606384, 36780454, 31727379, 67513640, 45996863, 71657078, 67793644, 4985896, 24733232, 82327024, 5267545, 90158785, 76540481, 13173644, 21533347, 58208470, 3509435, 3233084, 17957593, 71965942, 21001913, 54762643, 44664587, 93270084, 28796059, 42782652, 98943869, 66903004, 81274566, 89419466, 59981773, 42947632, 42237907, 36580610, 51472275, 53802686, 51590803, 30764367, 64098930, 67227442, 78884452, 69605283, 24619760, 65880522, 13819358, 73168565, 55850790, 33123618, 82979980, 85117092, 874791, 54868730, 112651, 6505939, 41245325, 4508700, 38022834, 71333116, 77898274, 72732205, 24314885, 16684829, 29994197, 39201414, 16567550, 8733068, 62762857, 33565483, 84187166, 26744917, 36396314, 22129328, 40534591, 54058788, 10597197, 61897582, 62693428, 91281584, 16387575, 17764950, 45790169, 18001617, 47361209, 16445503, 22450468, 84840549, 7229550, 47893286, 4199704, 34667286, 90061527, 53393358, 89046466, 62803858, 72793444, 61815223, 39847321, 57163802, 71300104, 44245960, 71522968, 51464002, 33699435, 95957797, 50188404, 56153345, 45919976, 56484900, 33201905, 68939068, 67124282, 45049308, 59109400, 76825057, 35996293, 3487592, 62357986, 60106217, 30366150, 10358899, 63628376, 15075176, 11161731, 72495719, 82178706, 22879907, 3773993, 30787683, 87598888, 3233569, 65081429, 83948335, 24915585, 28851716, 31491938, 15536795, 89078848, 53632373, 6521313, 92803766, 508198, 22108665, 63506281, 7955293, 27041967, 18131876, 37891451, 20642888, 27325428, 89996536, 99729357, 37659250, 14349098, 70420215, 44844121, 18663507 +71300104, 56461322, 44846932, 4787945, 31161687, 21289531, 47090124, 65715134, 98739783, 39171895, 76703609, 51426311, 55960386, 16424199, 96193415, 34432810, 74743862, 17539625, 98462867, 85023028, 37675718, 72738685, 51047803, 12348118, 37620363, 58751351, 95957797, 96726697, 61859143, 90004325, 12024238, 1431742, 11581181, 5832946, 94595668, 86821229, 94911072, 91281584, 75543508, 33431960, 80251430, 92215320, 61623915, 508198, 96420429, 78602717, 70782102, 36580610, 63628376, 67451935, 37957788, 98130363, 7919588, 21993752, 21070110, 19486173, 48360198, 47213183, 61263068, 43933006, 26063929, 19101477, 41380093, 57163802, 47708850, 67281495, 56473732, 5970607, 85711894, 7517032, 37659250, 31491938, 37435892, 20645197, 40781854, 6819644, 30218878, 67513640, 26292919, 99224392, 17727650, 48774913, 54058788, 94076128, 10597197, 82897371, 8696647, 61141874, 16387575, 95251277, 61960400, 79191827, 93057697, 69136837, 72725103, 4806458, 17764950, 4095116, 83533741, 79942022, 80316608, 22435353, 29510992, 57961282, 3509435, 35092039, 16445503, 68702632, 78549759, 9860968, 75229462, 28734791, 59405277, 32250045, 26734892, 53547802, 6153406, 21673260, 62490109, 15902805, 8807940, 83789391, 79738755, 22108665, 8129978, 60581278, 29959549, 77312810, 89217461, 48088883, 17058722, 84293052, 4798568, 30463802, 92692283, 6793819, 112651, 43322743, 6505939, 6808825, 89811711, 18131876, 74357852, 59910624, 82651278, 4119747, 54232247, 1204161, 76960303, 44479073, 99021067, 23194618, 72373496, 39201414, 95726235, 24953498, 74441448, 98948034, 97783876, 33201905, 83269727, 46870723, 40534591, 96318094, 168541, 54987042, 4985896, 11365791, 56515456, 43501211, 83368048, 82327024, 91240048, 26102057, 66137019, 2891150, 26139110, 4434662, 8634541, 58208470, 75820087, 69641513, 8099994, 22450468, 72019362, 68102437, 40865610, 34698428, 65271999, 66611759, 54762643, 88653118, 7104732, 9623492, 26776922, 81805959, 81274566, 42947632, 21919959, 4199704, 6497830, 4978543, 20836893, 19898053, 53648154, 22879907, 37224844, 13348726, 98648327, 78561158, 92071990, 7423788, 5640302, 20765474, 17016156, 77301523, 569864, 33553959, 93933709, 82052050, 66271566, 874791, 73617245, 65081429, 36505482, 38365584, 32590267, 49597667, 73124510, 77620120, 39847321, 70372191, 66885828, 74724075, 44245960, 41245325, 34044787, 49641577, 41481685, 81774825, 4091162, 71920426, 82532312, 27665211, 18699206, 65047700, 4069912, 50188404, 50668599, 83302115, 8733068, 60393039, 47298834, 56484900, 20002147, 824872, 9603598, 84187166, 26744917, 37192445, 84127901, 57240218, 40686254, 82779622, 29819940, 7502255, 6871053, 97641116, 61897582, 91255408, 2917920, 321665, 89637706, 9886593, 36753250, 65038678, 43506672, 93566986, 68824981, 26664538, 32161669, 84349107, 83210802, 8115266, 90158785, 23432750, 35512853, 51588015, 45407418, 59371804, 13173644, 19939935, 25842078, 38510840, 76315420, 28550822, 53084256, 58749, 11923835, 48395186, 45237957, 93270084, 3088684, 66250369, 28787861, 55602660, 62115552, 30694952, 8651647, 98943869, 66903004, 59981773, 47792865, 83150534, 36468541, 61271144, 1788101, 26392416, 70036438, 15075176, 15948937, 62430984, 30764367, 1197320, 32699744, 45794415, 55770687, 69605283, 44844121, 12416768, 24619760, 99861373, 8729146, 78589145, 86301513, 43045786, 95395112, 65275241, 29932657, 82886381, 4204661, 58180653, 76671482, 16019925, 79880247, 46851987, 44847298, 78909561, 35192533, 75986488, 29188588, 48658605, 99690194, 92998591, 33249630, 44889423, 71522968, 73109997, 36808486, 70541760, 4099191, 27041967, 43376279, 78766358, 14045383, 20122224, 92692978, 74110882, 86798033, 68204242, 72238278, 85571389, 40356877, 94711842, 99524975, 60955663, 3294781, 36930650, 79322415, 41092102, 86022504, 99333375, 31727379, 68939068, 45996863, 8128637, 89078848, 53632373, 6986898, 52261574, 67793644, 669105, 5111370, 62693428, 62452034, 29065964, 19457589, 57020481, 32274392, 39553046, 31904591, 44842615, 5267545, 36139942, 40677414, 59109400, 48673079, 76540481, 70800879, 85242963, 14947650, 97940276, 33797252, 33336148, 97281847, 29516592, 8373289, 69579137, 81855445, 21533347, 13231279, 71965942, 21001913, 33304202, 17894977, 87160386, 84840549, 78785507, 63372756, 53842979, 33895336, 96735716, 79922758, 91802888, 3183975, 26114953, 23569917, 38494874, 95290172, 88444207, 54014062, 10358899, 49236559, 37280276, 36189527, 6157724, 34667286, 97011160, 22997281, 14626618, 14723410, 81853704, 67227442, 65338021, 78884452, 73222868, 30891921, 86460488, 99604946, 3773993, 44177011, 89046466, 70004753, 20867149, 40027975, 15064655, 55753905, 88130087, 67031644, 10649306, 92867155, 62803858, 42967683, 86543538, 3233569, 40197395, 36685023, 85117092, 13468268, 84904436, 30653863, 62740044, 415901, 9188443, 16099750, 18411915, 54868730, 63967300, 12303248, 27187213, 29029316, 2204165, 93562257, 23740167, 33699435, 8791066, 17146629, 29834512, 88047921, 16054533, 52060076, 57359924, 72732205, 69355476, 33935899, 19272365, 65074535, 14363867, 56153345, 29401781, 29994197, 45919976, 16567550, 51466049, 55247455, 64055960, 66319530, 99226875, 56424103, 77187825, 87710366, 37166608, 36780454, 33565483, 39986008, 12664567, 57803235, 17385531, 76330843, 4064751, 47738185, 71657078, 79821897, 59318837, 45995325, 40152546, 37891451, 247198, 73031054, 45306070, 67124282, 526217, 18833224, 9599614, 92803766, 99549373, 41092172, 27411561, 93053405, 26493119, 45667668, 45617087, 97561745, 57241521, 44348328, 84684495, 86001008, 27185644, 17068582, 90457870, 26998766, 10961421, 8055981, 44664587, 28796059, 54199160, 3880712, 79657802, 66045587, 2208785, 9259676, 20568363, 60106217, 55470718, 57393458, 13862149, 9829782, 17976208, 9860195, 65454636, 11161731, 92398073, 53802686, 72495719, 17365188, 81677380, 34493392, 15163258, 38008118, 15015906, 20681921, 53393358, 70596786, 85695762, 20486294, 62232211, 81898046, 22942635, 68110330, 38009615, 38061439, 54263880, 61373987, 176257, 87598888, 31733363, 12571310, 75135448, 59197747, 24826575, 5482538, 30090481, 3235882, 75407347, 63059024, 1022677, 16097038, 7300047, 89804152, 72793444, 11543098, 51149804, 52293995, 54427233, 51507425, 80014588, 55615885, 72614359, 36845587, 80555751, 62936963, 37739481, 61815223, 48892201, 1239555, 91957544, 92541302, 15535065, 29635537, 7685448, 69786756, 42073124, 81172706, 70367851, 18663507, 7646095, 37501808, 79136082, 7011964, 4508700, 39373729, 68644627, 38022834, 29466406, 31126490, 55143724, 77898274, 28851716, 79806380, 24314885, 49271185, 92033260, 87720882, 9257405, 18806856, 78300864, 83083131, 4668450, 14731700, 73021291, 72357096, 59501487, 34497327, 77072625, 57658654, 54606384, 45848907, 77413012, 17081350, 30771409, 10453030, 53888755, 2717150, 90090964, 44550764, 92353856, 65017137, 36812683, 77300457, 71083565, 54517921, 45075471, 44481640, 15148031, 24733232, 90310261, 20642888, 22405120, 35996293, 94516935, 91727510, 45790169, 49598724, 18001617, 30139692, 94090109, 78218845, 19891772, 96852321, 17957593, 90013093, 43357947, 93359396, 75153252, 3487592, 6221471, 73235980, 36312813, 30998561, 82427263, 60430369, 57248122, 7229550, 74137926, 33628349, 51315460, 91831487, 14093520, 14326617, 42237907, 749283, 20885148, 32159704, 89996536, 64098930, 61712234, 16595436, 3633375, 24058273, 89499542, 35456853, 82178706, 64087743, 33061250, 30787683, 70727211, 76170907, 65880522, 64157906, 30569392, 68875490, 92554120, 38341669, 75777973, 61728685, 33123618, 37183543, 26275734, 98653983, 8913721, 34698463, 26507214, 7955293, 83948335, 45428665, 92604458, 32426752, 42692881, 71469330, 63152504, 51464002, 7066775, 18783702, 99297164, 22113133, 77787724, 16971929, 36135, 90654836, 7687278, 16684829, 72777973, 73786944, 86242799, 11757872, 38256849, 24168411, 50567636, 56955985, 50702367, 1250437, 22129328, 14349098, 21397057, 46540998, 32151165, 97057187, 30163921, 44983451, 72278539, 69697787, 68694897, 9398733, 92530431, 82339363, 19376156, 60176618, 13470059, 44473167, 47361209, 3233084, 83133790, 51715482, 49882705, 88251446, 99658235, 75128745, 95581843, 75052463, 36933038, 67084351, 84406788, 22200378, 51472275, 93515664, 18466635, 27325428, 90061527, 51590803, 77272628, 68128438, 59614746, 34236719, 54663246, 6525948, 69848388, 24591705, 40984766, 30395570, 93790285, 76434144, 77377183, 61928316, 9575954, 52613508, 39801351, 13819358, 47887470, 82979980, 23134715, 6111563, 60102965, 53666583, 55189057, 30543215, 42307449, 48260151, 84002370, 37560164, 23110625, 96906410, 59177718, 86118021, 25636669, 59329511, 91664334, 7182642, 32058615, 68316156, 91938191, 66428795, 67030811, 20140249, 5822202, 66663942, 61982238, 55669657, 62428472, 17037369, 45381876, 15536795, 96709982, 16380211, 99971982, 22721500, 6521313, 99125126, 88904910, 44627776, 61859581, 83651211, 62357986, 66667729, 4515343, 68816413, 95010552, 48675329, 94539824, 84166196, 73392814, 80246713, 68000591, 62552524, 44784505, 66116458, 73168565, 94360702, 42199455, 63506281, 66322959, 61741594, 98371444, 22766820, 5073754, 99965001, 33237508, 71316369, 71333116, 52204879, 97379790, 40781449, 30811010, 27625735, 10264691, 43152977, 67474219, 70420215, 74452589, 36396314, 17386996, 83155442, 34946859, 65851721, 50806615, 66832478, 56531125, 45049308, 10366309, 38645117, 90272749, 72274002, 91141395, 58224549, 49328608, 1936762, 91990218, 64848072, 47893286, 44915235, 17857111, 1569515, 64602895, 42644903, 55850790, 99729357, 2331773, 9175338, 77694700, 83747892, 18504197, 50007421, 41442762, 2607799, 63015256, 62762857, 89699445, 82726008, 44060493, 14220886, 48893685, 72358170, 23848565, 16405341, 88698958, 26863229, 28928175, 6038457, 89419466, 10309525, 38658347, 62051033, 34295794, 85116755, 89214616, 24915585, 99515901, 74614639, 99917599, 76825057, 68041839, 42782652, 92787493, 9058407, 26397786, 30366150, 45990383, 69255765, 1375023, 77284619, 62496012 +59329511, 22879907, 52204879, 2917920, 61960400, 99917599, 36933038, 73124510, 18411915, 18504197, 4668450, 17386996, 40152546, 68824981, 98943869, 4978543, 55770687, 44844121, 8129978, 58180653, 30543215, 99729357, 2331773, 92692283, 41442762, 72373496, 73786944, 11757872, 97783876, 41092102, 45381876, 12664567, 94076128, 48893685, 26664538, 69136837, 99549373, 4806458, 78785507, 8055981, 66250369, 95581843, 66045587, 1936762, 73392814, 78884452, 30891921, 87598888, 40027975, 52613508, 42644903, 36505482, 32426752, 12303248, 22766820, 34044787, 97379790, 54232247, 63015256, 8733068, 62496012, 59501487, 26744917, 76330843, 5832946, 56531125, 65017137, 29065964, 54517921, 36139942, 19376156, 90310261, 17764950, 93053405, 80251430, 69641513, 84684495, 33304202, 75128745, 66611759, 55470718, 1788101, 62430984, 1197320, 45794415, 35456853, 3773993, 64602895, 30569392, 75777973, 77312810, 89217461, 23134715, 55189057, 84293052, 80555751, 35192533, 61741594, 48892201, 51047803, 96906410, 42692881, 6808825, 41245325, 55960386, 33699435, 17146629, 90004325, 24915585, 69355476, 49271185, 77072625, 67513640, 56955985, 40534591, 37891451, 90090964, 321665, 91281584, 65038678, 45075471, 79191827, 93566986, 44481640, 91240048, 76825057, 13470059, 83651211, 22405120, 22435353, 81855445, 88698958, 90013093, 17894977, 8099994, 10961421, 48395186, 58224549, 66667729, 36312813, 30998561, 49328608, 79922758, 20568363, 68816413, 14326617, 21919959, 22200378, 47893286, 51472275, 17976208, 34667286, 90061527, 84166196, 54663246, 14723410, 32699744, 53648154, 82178706, 40984766, 99604946, 61373987, 8729146, 64157906, 62552524, 92071990, 63059024, 7423788, 10649306, 95395112, 62803858, 55850790, 7300047, 26275734, 36685023, 85117092, 54427233, 44847298, 30463802, 61815223, 38365584, 72738685, 54868730, 29029316, 47708850, 18663507, 7646095, 37620363, 37501808, 83747892, 67281495, 50007421, 7011964, 58751351, 68644627, 95957797, 91664334, 59910624, 81774825, 77898274, 40781449, 19272365, 68204242, 72777973, 16567550, 14731700, 98948034, 30218878, 73021291, 3294781, 5822202, 70420215, 55669657, 36780454, 39986008, 26292919, 89078848, 36396314, 53632373, 50702367, 17081350, 99515901, 22129328, 17727650, 48774913, 34946859, 96318094, 59318837, 247198, 73031054, 97641116, 72278539, 82897371, 44627776, 45049308, 526217, 39553046, 44842615, 10366309, 5267545, 84349107, 83210802, 59109400, 16405341, 9058407, 85242963, 27411561, 14947650, 33797252, 49598724, 68041839, 29516592, 44473167, 94090109, 3509435, 44348328, 83133790, 38510840, 68102437, 91141395, 34698428, 3880712, 26776922, 57248122, 6038457, 62115552, 74137926, 78602717, 9259676, 60106217, 47792865, 95290172, 95010552, 91990218, 36189527, 97011160, 22997281, 38008118, 24591705, 61712234, 67227442, 62232211, 81898046, 21673260, 68110330, 38061439, 33061250, 54263880, 48360198, 30090481, 68000591, 77377183, 98739783, 39801351, 13819358, 62051033, 47887470, 17016156, 77301523, 569864, 93933709, 42967683, 72793444, 48260151, 63506281, 874791, 84002370, 62740044, 4798568, 78909561, 7955293, 23110625, 9175338, 69786756, 42073124, 112651, 77694700, 27187213, 74724075, 70367851, 2204165, 51464002, 4099191, 56473732, 8791066, 33237508, 22113133, 29466406, 18131876, 43376279, 78766358, 31126490, 68316156, 4091162, 20122224, 30811010, 90654836, 18699206, 91938191, 4069912, 16684829, 23194618, 83302115, 1375023, 60955663, 72357096, 57658654, 54606384, 83269727, 17037369, 50567636, 15536795, 40686254, 21397057, 79821897, 45995325, 10597197, 99971982, 65851721, 168541, 68694897, 9886593, 57020481, 43501211, 9599614, 92803766, 38645117, 60176618, 8115266, 72725103, 35512853, 91727510, 45617087, 8634541, 19891772, 47361209, 57961282, 26863229, 86001008, 35092039, 72274002, 16445503, 88251446, 63372756, 44664587, 96420429, 55602660, 23569917, 51315460, 81274566, 26397786, 57393458, 13862149, 48675329, 70036438, 93515664, 67451935, 92398073, 18466635, 89996536, 6525948, 69848388, 65715134, 19898053, 21070110, 62490109, 86460488, 44177011, 176257, 15064655, 75135448, 78589145, 98648327, 3235882, 47213183, 43933006, 60102965, 3233569, 53666583, 82052050, 11543098, 13468268, 72614359, 19101477, 415901, 83948335, 92541302, 48658605, 71300104, 70372191, 44245960, 71522968, 18783702, 4508700, 71316369, 38022834, 77787724, 88047921, 49641577, 36135, 7517032, 92692978, 92033260, 65074535, 99021067, 72238278, 9257405, 85571389, 60393039, 43152977, 56484900, 1431742, 78300864, 86242799, 40781854, 6819644, 66319530, 67474219, 77284619, 99226875, 38256849, 30771409, 94595668, 67793644, 669105, 86821229, 44983451, 4985896, 2717150, 6521313, 74743862, 11365791, 45306070, 56515456, 5111370, 88904910, 83368048, 31904591, 82339363, 24733232, 61859581, 93057697, 23848565, 51588015, 92787493, 26102057, 94516935, 97940276, 66137019, 26139110, 45790169, 30139692, 57241521, 29510992, 49882705, 99658235, 9623492, 3088684, 28787861, 54199160, 79657802, 2208785, 3183975, 78549759, 66903004, 91831487, 61271144, 84406788, 64848072, 11161731, 15948937, 51590803, 72495719, 17365188, 81677380, 34493392, 59614746, 24058273, 53393358, 85695762, 38658347, 22942635, 64087743, 30395570, 24619760, 1569515, 70004753, 31733363, 37224844, 55753905, 24826575, 76434144, 22108665, 61928316, 9575954, 69255765, 75407347, 86301513, 33123618, 29959549, 61263068, 82979980, 6111563, 39171895, 98653983, 17058722, 8913721, 66322959, 46851987, 62936963, 91957544, 49597667, 77620120, 15535065, 98371444, 89214616, 92604458, 33249630, 43322743, 81172706, 73109997, 36808486, 51426311, 56461322, 47090124, 25636669, 99297164, 39373729, 29834512, 27041967, 85711894, 16424199, 32058615, 61859143, 72732205, 37659250, 86798033, 45919976, 10264691, 12024238, 99524975, 24953498, 36930650, 99333375, 54058788, 97057187, 66832478, 53888755, 54987042, 92353856, 69697787, 62452034, 32274392, 90158785, 20642888, 70800879, 59371804, 80316608, 45667668, 17539625, 75820087, 98462867, 19939935, 3233084, 25842078, 71965942, 21001913, 90457870, 22450468, 11923835, 65271999, 45237957, 93270084, 33895336, 96735716, 42782652, 60430369, 30694952, 33628349, 8651647, 89419466, 14093520, 83150534, 70782102, 26392416, 67084351, 30366150, 54014062, 10358899, 63628376, 37280276, 749283, 28734791, 59405277, 53802686, 14626618, 94539824, 34236719, 98130363, 7919588, 81853704, 20486294, 15902805, 8807940, 12416768, 93790285, 65880522, 5482538, 78561158, 5640302, 65275241, 38341669, 61728685, 1022677, 37675718, 33553959, 37183543, 94360702, 4204661, 89804152, 52293995, 42307449, 26063929, 51507425, 79880247, 36845587, 37739481, 41380093, 57163802, 85116755, 7685448, 99690194, 44889423, 93562257, 79136082, 99965001, 71333116, 16971929, 2607799, 41481685, 82651278, 52060076, 74110882, 24314885, 7687278, 1204161, 66428795, 76960303, 29401781, 51466049, 18806856, 94711842, 31491938, 37435892, 55247455, 64055960, 20002147, 824872, 34497327, 62762857, 86022504, 33201905, 96709982, 1250437, 14349098, 46870723, 4064751, 29819940, 46540998, 50806615, 30163921, 61897582, 22721500, 14220886, 61141874, 36753250, 16387575, 43506672, 74614639, 40677414, 4787945, 45407418, 48673079, 76540481, 35996293, 33431960, 13173644, 93359396, 28928175, 28550822, 53084256, 58749, 6221471, 54762643, 28796059, 82427263, 75052463, 85023028, 42947632, 42237907, 36580610, 49236559, 9829782, 44915235, 6157724, 65454636, 32250045, 27325428, 30764367, 64098930, 26734892, 17857111, 20836893, 20681921, 3633375, 65338021, 70596786, 80246713, 38009615, 76170907, 12571310, 19486173, 88130087, 44784505, 66116458, 20765474, 31161687, 73168565, 21289531, 16097038, 42199455, 51149804, 76703609, 84904436, 55615885, 34295794, 16099750, 5073754, 23740167, 86118021, 74357852, 55143724, 16054533, 28851716, 82532312, 44479073, 50188404, 87720882, 50668599, 29994197, 39201414, 20645197, 67030811, 66663942, 96193415, 9603598, 79322415, 87710366, 62428472, 11581181, 84187166, 37192445, 45996863, 8128637, 57240218, 6986898, 83155442, 17385531, 47738185, 6871053, 91255408, 44550764, 8696647, 72358170, 99125126, 77300457, 18833224, 92530431, 15148031, 82327024, 4095116, 2891150, 18001617, 90272749, 8373289, 78218845, 69579137, 58208470, 13231279, 17957593, 27185644, 17068582, 76315420, 87160386, 72019362, 84840549, 26998766, 40865610, 88653118, 73235980, 68702632, 508198, 4515343, 81805959, 59981773, 75229462, 88444207, 4199704, 9860195, 10309525, 32159704, 6153406, 73222868, 70727211, 79738755, 59197747, 45990383, 43045786, 92554120, 92867155, 29932657, 60581278, 16019925, 80014588, 73617245, 26507214, 32590267, 75986488, 45428665, 29188588, 6793819, 29635537, 66885828, 71469330, 63152504, 70541760, 7066775, 96726697, 7182642, 57359924, 79806380, 56153345, 95726235, 47298834, 83083131, 20140249, 61982238, 77187825, 31727379, 33565483, 68939068, 57803235, 52261574, 99224392, 7502255, 71657078, 10453030, 16380211, 34432810, 62693428, 89637706, 36812683, 19457589, 71083565, 32161669, 41092172, 79942022, 4434662, 33336148, 26493119, 97281847, 97561745, 92215320, 96852321, 43357947, 53842979, 91802888, 26114953, 38494874, 9860968, 36468541, 6497830, 68128438, 15015906, 16595436, 53547802, 21993752, 20867149, 67031644, 13348726, 82886381, 48088883, 34698463, 30653863, 65081429, 1239555, 39847321, 59177718, 12348118, 6505939, 89811711, 5970607, 27625735, 33935899, 14363867, 74452589, 24168411, 37166608, 45848907, 82726008, 82779622, 44060493, 32151165, 44846932, 95251277, 83533741, 75543508, 51715482, 61623915, 75153252, 7104732, 7229550, 37957788, 15075176, 20885148, 77272628, 15163258, 89499542, 69605283, 89046466, 99861373, 68875490, 86543538, 40197395, 37560164, 9188443, 63967300, 27665211, 65047700, 74441448, 56424103, 21533347, 3487592, 92998591, 14045383, 4119747, 77413012, 9398733, 67124282, 94911072, 23432750, 62357986, 30787683, 76671482, 66271566, 71920426, 89699445, 84127901, 83789391, 40356877 +33553959, 73031054, 35996293, 9829782, 92398073, 98739783, 52293995, 56473732, 13470059, 9058407, 9623492, 24591705, 18663507, 93562257, 8733068, 98948034, 61897582, 8696647, 24733232, 86001008, 33895336, 67084351, 70036438, 6497830, 37957788, 30891921, 99604946, 76434144, 62552524, 36685023, 63506281, 2331773, 37739481, 6793819, 27625735, 1375023, 67513640, 89078848, 7502255, 83210802, 14947650, 45667668, 57961282, 72274002, 75153252, 8055981, 42782652, 2208785, 4515343, 1788101, 4978543, 15948937, 55770687, 12416768, 3773993, 70004753, 75135448, 30090481, 7423788, 65275241, 62803858, 61728685, 77301523, 93933709, 16097038, 60102965, 72793444, 58180653, 34698463, 45428665, 98371444, 48658605, 2204165, 41245325, 67281495, 23740167, 18783702, 41442762, 78766358, 88047921, 97379790, 54232247, 49271185, 96193415, 36930650, 41092102, 33201905, 89699445, 99333375, 40686254, 47738185, 34432810, 53888755, 2717150, 11365791, 82897371, 91281584, 36753250, 39553046, 83368048, 22435353, 93053405, 97281847, 94090109, 80251430, 19891772, 81855445, 90013093, 87160386, 91141395, 65271999, 58224549, 3088684, 66045587, 62115552, 30694952, 85023028, 26392416, 51472275, 51590803, 64098930, 38009615, 44177011, 176257, 70727211, 79738755, 19486173, 24826575, 48360198, 5482538, 78561158, 69255765, 10649306, 95395112, 92867155, 73168565, 29932657, 89217461, 43933006, 53666583, 26275734, 98653983, 30543215, 99729357, 48260151, 1239555, 70372191, 69786756, 32426752, 71522968, 71469330, 55960386, 59329511, 59910624, 68316156, 61859143, 71920426, 50188404, 87720882, 68204242, 72777973, 73786944, 18806856, 55247455, 30218878, 62428472, 17037369, 50702367, 76330843, 17727650, 45995325, 94076128, 16380211, 94595668, 65851721, 168541, 30163921, 45306070, 5111370, 62452034, 72358170, 36812683, 57020481, 54517921, 32274392, 31904591, 93566986, 44842615, 93057697, 69136837, 59109400, 90158785, 45407418, 22405120, 2891150, 45790169, 33797252, 49598724, 97561745, 17539625, 34698428, 48395186, 88653118, 28796059, 79657802, 508198, 57248122, 33628349, 51315460, 9259676, 95290172, 26397786, 30366150, 18466635, 32250045, 72495719, 68128438, 61712234, 81853704, 65715134, 19898053, 69605283, 53648154, 62490109, 38061439, 15064655, 8729146, 92071990, 29959549, 6111563, 94360702, 55189057, 66271566, 76703609, 66322959, 54427233, 73617245, 84904436, 26507214, 4798568, 61815223, 48892201, 38365584, 92998591, 42692881, 77694700, 70367851, 37620363, 6808825, 51426311, 25636669, 38022834, 16971929, 16054533, 16424199, 49641577, 7517032, 90004325, 44479073, 99021067, 72238278, 40356877, 45919976, 60393039, 47298834, 74441448, 86242799, 60955663, 6819644, 67474219, 20140249, 97783876, 24168411, 37166608, 56955985, 15536795, 29819940, 30771409, 79821897, 59318837, 50806615, 54987042, 91255408, 22721500, 48893685, 2917920, 67124282, 56515456, 61141874, 43501211, 10366309, 19376156, 91240048, 40677414, 8115266, 51588015, 4095116, 76540481, 66137019, 80316608, 68041839, 45617087, 8634541, 21533347, 58208470, 84684495, 26863229, 25842078, 17957593, 38510840, 84840549, 78785507, 10961421, 53842979, 30998561, 23569917, 68816413, 36468541, 4199704, 67451935, 15075176, 9860195, 65454636, 11161731, 77272628, 34493392, 94539824, 54663246, 32699744, 85695762, 21070110, 20486294, 22879907, 22942635, 8807940, 64087743, 24619760, 1569515, 61928316, 30569392, 5640302, 33123618, 17016156, 39171895, 3233569, 89804152, 26063929, 84293052, 62740044, 80555751, 62936963, 36505482, 91957544, 49597667, 73124510, 77620120, 34295794, 72738685, 29635537, 57163802, 85116755, 71300104, 112651, 63152504, 18504197, 99965001, 47090124, 8791066, 58751351, 22113133, 68644627, 71333116, 2607799, 96726697, 31126490, 85711894, 41481685, 77898274, 74110882, 79806380, 24314885, 85571389, 39201414, 95726235, 51466049, 83302115, 83083131, 77284619, 824872, 59501487, 66663942, 61982238, 62762857, 79322415, 74452589, 82726008, 46540998, 10453030, 96318094, 44983451, 6521313, 92353856, 9398733, 62693428, 9886593, 44846932, 18833224, 71083565, 99917599, 44481640, 15148031, 26664538, 83651211, 4787945, 35512853, 17764950, 70800879, 91727510, 75543508, 59371804, 33336148, 90272749, 8373289, 13173644, 47361209, 75820087, 3233084, 21001913, 27185644, 17894977, 43357947, 8099994, 16445503, 90457870, 51715482, 22450468, 40865610, 11923835, 63372756, 54762643, 95581843, 3880712, 26776922, 91802888, 55602660, 78602717, 8651647, 14093520, 47792865, 57393458, 36189527, 93515664, 28734791, 17976208, 10309525, 53802686, 89996536, 98130363, 6153406, 73392814, 38658347, 82178706, 33061250, 83789391, 99861373, 40027975, 59197747, 98648327, 3235882, 47213183, 75777973, 82886381, 47887470, 37675718, 77312810, 82979980, 23134715, 17058722, 51507425, 30653863, 84002370, 78909561, 35192533, 7955293, 75986488, 29188588, 54868730, 42073124, 33249630, 12348118, 74724075, 44889423, 47708850, 37501808, 36808486, 6505939, 79136082, 5970607, 77787724, 91664334, 81774825, 52060076, 40781449, 30811010, 90654836, 27665211, 92033260, 72373496, 64055960, 78300864, 4668450, 66319530, 67030811, 3294781, 72357096, 62496012, 5822202, 38256849, 86022504, 87710366, 83269727, 45848907, 45381876, 45996863, 17386996, 83155442, 17081350, 17385531, 1250437, 99515901, 14349098, 82779622, 5832946, 71657078, 34946859, 10597197, 99971982, 97641116, 669105, 86821229, 90090964, 44627776, 29065964, 45049308, 16387575, 65038678, 61960400, 45075471, 79191827, 68824981, 61859581, 9599614, 5267545, 36139942, 90310261, 60176618, 72725103, 99549373, 4806458, 26102057, 79942022, 26493119, 18001617, 33431960, 69579137, 13231279, 98462867, 71965942, 17068582, 72019362, 49882705, 68102437, 99658235, 3487592, 58749, 66611759, 66250369, 36312813, 82427263, 96420429, 6038457, 20568363, 75052463, 1936762, 9860968, 91831487, 89419466, 83150534, 95010552, 70782102, 61271144, 88444207, 42237907, 36580610, 49236559, 37280276, 48675329, 59405277, 20885148, 27325428, 97011160, 15163258, 30764367, 84166196, 14723410, 69848388, 1197320, 26734892, 17857111, 24058273, 65338021, 35456853, 73222868, 62232211, 80246713, 40984766, 87598888, 77377183, 52613508, 8129978, 42967683, 86543538, 51149804, 8913721, 42307449, 874791, 55615885, 72614359, 83948335, 32590267, 92692283, 92541302, 15535065, 92604458, 59177718, 63967300, 66885828, 27187213, 4099191, 17146629, 4508700, 99297164, 34044787, 71316369, 95957797, 29466406, 18131876, 7182642, 55143724, 82532312, 76960303, 56153345, 50668599, 94711842, 37435892, 40781854, 99226875, 34497327, 70420215, 57658654, 9603598, 54606384, 50567636, 39986008, 8128637, 46870723, 48774913, 21397057, 66832478, 56531125, 94911072, 99125126, 88904910, 19457589, 526217, 43506672, 92803766, 92787493, 83533741, 27411561, 97940276, 44473167, 69641513, 96852321, 19939935, 83133790, 33304202, 93359396, 28550822, 53084256, 7104732, 73235980, 28787861, 49328608, 74137926, 81805959, 55470718, 36933038, 54014062, 10358899, 22200378, 47893286, 22997281, 14626618, 59614746, 38008118, 6525948, 15015906, 67227442, 20681921, 53547802, 78884452, 21673260, 30395570, 54263880, 61373987, 12571310, 88130087, 67031644, 44784505, 75407347, 86301513, 39801351, 20765474, 38341669, 31161687, 61263068, 569864, 4204661, 40197395, 76671482, 85117092, 16019925, 65081429, 46851987, 36845587, 61741594, 415901, 37560164, 41380093, 16099750, 7685448, 89214616, 96906410, 99690194, 81172706, 7646095, 50007421, 56461322, 89811711, 27041967, 74357852, 14045383, 36135, 57359924, 4119747, 18699206, 33935899, 1204161, 65047700, 19272365, 86798033, 65074535, 14363867, 4069912, 23194618, 9257405, 10264691, 16567550, 31491938, 99524975, 56484900, 20645197, 1431742, 77187825, 11581181, 84187166, 26744917, 37192445, 36396314, 96709982, 52261574, 54058788, 37891451, 247198, 72278539, 44550764, 89637706, 65017137, 32161669, 23848565, 76825057, 23432750, 41092172, 16405341, 48673079, 44348328, 88698958, 35092039, 26998766, 61623915, 62357986, 45237957, 44664587, 68702632, 26114953, 38494874, 98943869, 81274566, 75229462, 14326617, 84406788, 44915235, 62430984, 34236719, 20836893, 16595436, 3633375, 45794415, 53393358, 89499542, 86460488, 89046466, 30787683, 20867149, 93790285, 65880522, 64602895, 22108665, 45990383, 9575954, 63059024, 43045786, 13819358, 55850790, 1022677, 37183543, 48088883, 82052050, 11543098, 44847298, 30463802, 19101477, 23110625, 39847321, 18411915, 51047803, 12303248, 22766820, 43322743, 29029316, 44245960, 73109997, 70541760, 7066775, 33699435, 29834512, 52204879, 32058615, 82651278, 28851716, 69355476, 63015256, 16684829, 29401781, 24953498, 20002147, 56424103, 55669657, 36780454, 33565483, 57240218, 57803235, 44060493, 40534591, 74743862, 95251277, 92530431, 82327024, 38645117, 20642888, 94516935, 4434662, 30139692, 57241521, 78218845, 29510992, 92215320, 76315420, 28928175, 75128745, 93270084, 54199160, 60430369, 3183975, 66903004, 60106217, 59981773, 63628376, 64848072, 749283, 32159704, 21993752, 37224844, 64157906, 13348726, 68875490, 42644903, 21289531, 42199455, 13468268, 80014588, 79880247, 51464002, 43376279, 4091162, 24915585, 37659250, 92692978, 7687278, 66428795, 29994197, 43152977, 14731700, 68939068, 84127901, 40152546, 97057187, 4985896, 14220886, 68694897, 77300457, 26139110, 29516592, 88251446, 6221471, 78549759, 42947632, 21919959, 91990218, 6157724, 34667286, 90061527, 81677380, 70596786, 81898046, 15902805, 68110330, 31733363, 76170907, 78589145, 66116458, 60581278, 9175338, 5073754, 83747892, 7011964, 20122224, 72732205, 91938191, 12024238, 77072625, 11757872, 31727379, 26292919, 77413012, 53632373, 22129328, 99224392, 32151165, 6871053, 69697787, 321665, 74614639, 84349107, 85242963, 3509435, 66667729, 96735716, 44844121, 55753905, 68000591, 92554120, 62051033, 86118021, 39373729, 73021291, 12664567, 4064751, 67793644, 82339363, 79922758, 7229550, 17365188, 7919588, 9188443, 6986898, 13862149, 7300047, 33237508 +60430369, 56461322, 59371804, 78602717, 7685448, 42692881, 18783702, 76960303, 34946859, 26102057, 54199160, 30998561, 30891921, 92071990, 20765474, 75777973, 73168565, 4798568, 38365584, 32590267, 41245325, 68644627, 5970607, 27625735, 59501487, 56955985, 36139942, 8099994, 21919959, 17976208, 59614746, 48360198, 48260151, 54427233, 79880247, 37620363, 20122224, 72732205, 16567550, 4668450, 11581181, 50567636, 17386996, 17081350, 94076128, 67793644, 2917920, 67124282, 91281584, 45049308, 39553046, 19376156, 8115266, 20642888, 80316608, 84684495, 75128745, 66611759, 48395186, 14326617, 13862149, 18466635, 84166196, 53393358, 44844121, 86460488, 65880522, 47213183, 68875490, 3233569, 84293052, 30463802, 91957544, 6793819, 12303248, 22766820, 44889423, 83747892, 70541760, 50007421, 33699435, 33237508, 71316369, 16424199, 63015256, 72373496, 74441448, 67030811, 77072625, 26744917, 67513640, 79821897, 74743862, 88904910, 44481640, 5267545, 84349107, 69136837, 99549373, 27411561, 91727510, 22435353, 49598724, 68041839, 87160386, 78785507, 44664587, 66250369, 3183975, 74137926, 23569917, 66903004, 68816413, 95010552, 88444207, 92398073, 30764367, 17857111, 24591705, 67227442, 3633375, 32699744, 24058273, 54263880, 30787683, 87598888, 93790285, 22108665, 75407347, 43045786, 55850790, 17016156, 37183543, 23134715, 6111563, 48088883, 89804152, 66271566, 13468268, 51507425, 44847298, 48892201, 1239555, 71300104, 89214616, 92604458, 59177718, 77694700, 52204879, 18131876, 78766358, 88047921, 37659250, 71920426, 19272365, 4069912, 99021067, 95726235, 51466049, 8733068, 94711842, 12024238, 6819644, 70420215, 54606384, 99333375, 53632373, 84127901, 83155442, 46870723, 168541, 44983451, 6521313, 89637706, 72358170, 36753250, 526217, 95251277, 31904591, 82327024, 44842615, 59109400, 13470059, 92787493, 16405341, 94090109, 47361209, 17894977, 93359396, 10961421, 58224549, 36312813, 28796059, 66045587, 57248122, 6038457, 81274566, 36933038, 47893286, 51472275, 93515664, 72495719, 77272628, 97011160, 26734892, 61712234, 65338021, 78884452, 20486294, 21673260, 68110330, 38061439, 33061250, 83789391, 31733363, 76434144, 60581278, 29959549, 37675718, 93933709, 4204661, 30543215, 99729357, 63506281, 80555751, 92692283, 15535065, 96906410, 5073754, 12348118, 47708850, 37501808, 79136082, 23740167, 56473732, 17146629, 32058615, 24915585, 82532312, 18806856, 20645197, 40781854, 14731700, 20002147, 9603598, 97783876, 74452589, 77187825, 87710366, 31727379, 17037369, 1250437, 30771409, 45995325, 16380211, 97641116, 54987042, 90090964, 82897371, 5111370, 57020481, 61960400, 99917599, 93566986, 26664538, 91240048, 83210802, 90310261, 22405120, 9058407, 93053405, 33797252, 97281847, 18001617, 29516592, 8373289, 80251430, 17539625, 21001913, 27185644, 33304202, 72274002, 43357947, 72019362, 99658235, 40865610, 61623915, 75153252, 66667729, 68702632, 55602660, 62115552, 78549759, 9259676, 83150534, 26397786, 91990218, 57393458, 84406788, 6497830, 28734791, 4978543, 53802686, 90061527, 51590803, 81677380, 1197320, 70596786, 19898053, 85695762, 44177011, 1569515, 37224844, 55753905, 64157906, 67031644, 61928316, 9575954, 95395112, 31161687, 62051033, 47887470, 77312810, 16097038, 7300047, 60102965, 98653983, 17058722, 55189057, 82052050, 51149804, 42307449, 874791, 80014588, 73617245, 37560164, 23110625, 9175338, 70372191, 43322743, 71469330, 22113133, 38022834, 95957797, 29466406, 27041967, 91664334, 14045383, 41481685, 4091162, 28851716, 4119747, 92692978, 27665211, 79806380, 49271185, 86798033, 87720882, 68204242, 72777973, 73786944, 47298834, 1431742, 86242799, 98948034, 30218878, 73021291, 62496012, 99226875, 66663942, 89699445, 8128637, 12664567, 50702367, 99515901, 47738185, 82779622, 21397057, 71657078, 54058788, 96318094, 6871053, 10597197, 97057187, 30163921, 669105, 72278539, 4985896, 56515456, 62452034, 65017137, 99125126, 19457589, 16387575, 77300457, 43501211, 54517921, 15148031, 10366309, 60176618, 4787945, 4806458, 17764950, 70800879, 85242963, 26139110, 4434662, 92215320, 13231279, 3509435, 44348328, 17068582, 76315420, 16445503, 51715482, 53084256, 54762643, 73235980, 3088684, 95581843, 49328608, 508198, 2208785, 7229550, 85023028, 59981773, 47792865, 36580610, 30366150, 54014062, 63628376, 37280276, 22200378, 4199704, 36189527, 9860195, 65454636, 10309525, 15948937, 14626618, 62430984, 89996536, 54663246, 7919588, 81853704, 65715134, 53547802, 6153406, 73392814, 38658347, 73222868, 82178706, 62490109, 12416768, 89046466, 70004753, 20867149, 40027975, 76170907, 78589145, 13348726, 98648327, 68000591, 8129978, 63059024, 13819358, 62803858, 61728685, 1022677, 61263068, 77301523, 569864, 94360702, 43933006, 36685023, 76671482, 76703609, 65081429, 2331773, 46851987, 62740044, 72614359, 78909561, 36505482, 35192533, 7955293, 83948335, 73124510, 72738685, 75986488, 29188588, 85116755, 70367851, 71522968, 18663507, 6505939, 18504197, 4099191, 47090124, 58751351, 59329511, 74357852, 31126490, 97379790, 49641577, 40781449, 90654836, 18699206, 91938191, 7687278, 1204161, 65047700, 65074535, 23194618, 85571389, 39201414, 40356877, 56484900, 78300864, 72357096, 824872, 96193415, 34497327, 36930650, 62762857, 24168411, 68939068, 39986008, 77413012, 6986898, 96709982, 46540998, 37891451, 247198, 73031054, 91255408, 14220886, 92353856, 48893685, 69697787, 68694897, 94911072, 61141874, 36812683, 18833224, 32274392, 45075471, 82339363, 40677414, 35512853, 94516935, 14947650, 26493119, 45617087, 44473167, 57241521, 21533347, 96852321, 19939935, 71965942, 38510840, 90013093, 35092039, 22450468, 68102437, 28928175, 63372756, 6221471, 9623492, 79657802, 33895336, 96735716, 4515343, 91802888, 33628349, 9860968, 95290172, 61271144, 26392416, 49236559, 749283, 37957788, 59405277, 20885148, 15163258, 34236719, 64098930, 6525948, 20681921, 55770687, 69605283, 81898046, 8807940, 64087743, 3773993, 99861373, 79738755, 12571310, 24826575, 64602895, 77377183, 3235882, 52613508, 39801351, 92867155, 89217461, 86543538, 26275734, 85117092, 66322959, 26507214, 49597667, 41380093, 77620120, 34295794, 29635537, 57163802, 48658605, 99690194, 63967300, 44245960, 2204165, 73109997, 6808825, 67281495, 86118021, 25636669, 8791066, 77787724, 29834512, 96726697, 59910624, 68316156, 82651278, 57359924, 74110882, 24314885, 66428795, 16684829, 50188404, 50668599, 29994197, 99524975, 24953498, 83083131, 3294781, 20140249, 5822202, 56424103, 57658654, 41092102, 33201905, 37192445, 45381876, 15536795, 40686254, 52261574, 82726008, 76330843, 99224392, 40534591, 10453030, 32151165, 59318837, 40152546, 34432810, 94595668, 53888755, 44550764, 11365791, 8696647, 9398733, 9886593, 44846932, 29065964, 65038678, 43506672, 83368048, 68824981, 61859581, 93057697, 92803766, 76825057, 83651211, 72725103, 51588015, 48673079, 4095116, 79942022, 97940276, 75543508, 45790169, 30139692, 33431960, 19891772, 58208470, 57961282, 75820087, 98462867, 26863229, 83133790, 90457870, 49882705, 3487592, 88653118, 28787861, 26114953, 20568363, 38494874, 98943869, 42947632, 14093520, 75229462, 36468541, 70782102, 1788101, 64848072, 44915235, 15075176, 11161731, 68128438, 34493392, 32159704, 38008118, 69848388, 15015906, 45794415, 22879907, 62232211, 80246713, 24619760, 61373987, 70727211, 15064655, 8729146, 59197747, 19486173, 88130087, 44784505, 69255765, 30569392, 86301513, 98739783, 92554120, 65275241, 38341669, 21289531, 39171895, 72793444, 40197395, 11543098, 30653863, 36845587, 62936963, 37739481, 61815223, 19101477, 415901, 16099750, 18411915, 98371444, 51047803, 112651, 92998591, 33249630, 74724075, 36808486, 63152504, 51426311, 55960386, 7011964, 99297164, 39373729, 43376279, 85711894, 16054533, 36135, 81774825, 90004325, 69355476, 92033260, 14363867, 56153345, 29401781, 72238278, 9257405, 31491938, 37435892, 55247455, 37166608, 36780454, 84187166, 45996863, 26292919, 36396314, 57240218, 14349098, 4064751, 5832946, 99971982, 65851721, 50806615, 61897582, 66832478, 86821229, 56531125, 321665, 44627776, 74614639, 71083565, 92530431, 90158785, 2891150, 33336148, 97561745, 8634541, 69579137, 69641513, 88698958, 26998766, 88251446, 11923835, 34698428, 7104732, 45237957, 3880712, 96420429, 81805959, 51315460, 75052463, 8651647, 60106217, 67084351, 42237907, 34667286, 38009615, 99604946, 176257, 75135448, 5482538, 30090481, 62552524, 78561158, 7423788, 5640302, 42644903, 29932657, 82886381, 42967683, 58180653, 42199455, 8913721, 26063929, 55615885, 54868730, 69786756, 32426752, 42073124, 27187213, 81172706, 93562257, 99965001, 4508700, 34044787, 89811711, 77898274, 30811010, 54232247, 33935899, 45919976, 66319530, 67474219, 77284619, 61982238, 11757872, 33565483, 45848907, 89078848, 22721500, 2717150, 45306070, 32161669, 9599614, 23432750, 45407418, 66137019, 45667668, 13173644, 78218845, 29510992, 86001008, 25842078, 84840549, 91141395, 65271999, 82427263, 79922758, 1936762, 55470718, 9829782, 48675329, 6157724, 17365188, 98130363, 20836893, 21993752, 21070110, 35456853, 53648154, 15902805, 40984766, 33123618, 33553959, 52293995, 84002370, 61741594, 92541302, 39847321, 9188443, 29029316, 51464002, 41442762, 16971929, 55143724, 7517032, 52060076, 44479073, 1375023, 64055960, 79322415, 83269727, 17385531, 22129328, 17727650, 62693428, 79191827, 41092172, 83533741, 81855445, 58749, 62357986, 53842979, 26776922, 30694952, 10358899, 70036438, 67451935, 32250045, 14723410, 22942635, 30395570, 45990383, 34698463, 84904436, 66885828, 2607799, 7182642, 61859143, 10264691, 83302115, 60393039, 43152977, 38256849, 86022504, 62428472, 44060493, 23848565, 38645117, 76540481, 35996293, 90272749, 28550822, 8055981, 93270084, 91831487, 27325428, 22997281, 89499542, 66116458, 10649306, 16019925, 45428665, 7646095, 7066775, 71333116, 55669657, 48774913, 29819940, 17957593, 89419466, 94539824, 16595436, 82979980, 53666583, 60955663, 7502255, 3233084, 42782652, 57803235, 24733232 +76960303, 43357947, 12571310, 47213183, 54663246, 31161687, 5832946, 3183975, 27665211, 50702367, 40152546, 70800879, 47361209, 9860968, 13862149, 90061527, 84166196, 64098930, 65275241, 37183543, 19101477, 91957544, 91664334, 86821229, 2717150, 77300457, 72725103, 4787945, 91727510, 92215320, 78785507, 96420429, 95290172, 48675329, 94539824, 20836893, 61373987, 20867149, 66116458, 92554120, 20765474, 33553959, 82052050, 49597667, 77620120, 59177718, 43322743, 54232247, 83083131, 60955663, 87710366, 33565483, 99224392, 96318094, 11365791, 69697787, 9398733, 99125126, 29065964, 45049308, 82339363, 90310261, 4806458, 26102057, 4434662, 26493119, 8373289, 33431960, 35092039, 76315420, 51715482, 10961421, 63372756, 66611759, 62357986, 2208785, 60430369, 74137926, 75229462, 42237907, 28734791, 65454636, 81677380, 14626618, 89499542, 44844121, 1569515, 15064655, 55753905, 19486173, 78589145, 98648327, 38341669, 75777973, 40197395, 30543215, 34698463, 48260151, 55615885, 39847321, 70372191, 42073124, 92998591, 47708850, 89811711, 49641577, 81774825, 20122224, 28851716, 7687278, 92033260, 85571389, 39201414, 1375023, 37435892, 72357096, 20140249, 61982238, 89699445, 15536795, 17727650, 46540998, 54058788, 10453030, 97057187, 44983451, 6521313, 74743862, 526217, 93566986, 36139942, 22435353, 57241521, 29510992, 17539625, 13231279, 75820087, 72019362, 49882705, 88251446, 28928175, 91141395, 45237957, 79657802, 26776922, 6038457, 51315460, 85023028, 47792865, 36468541, 84406788, 22200378, 6497830, 9860195, 3633375, 68000591, 44784505, 92071990, 92867155, 61263068, 82979980, 76703609, 79880247, 62740044, 4798568, 72614359, 61741594, 34295794, 85116755, 54868730, 66885828, 12348118, 29029316, 73109997, 39373729, 34044787, 74357852, 31126490, 14045383, 82651278, 90004325, 71920426, 82532312, 24314885, 65047700, 14363867, 29994197, 73786944, 40356877, 24953498, 40781854, 824872, 59501487, 99226875, 97783876, 77187825, 38256849, 84187166, 39986008, 37192445, 77413012, 17386996, 82779622, 59318837, 30163921, 22721500, 45306070, 9886593, 38645117, 8115266, 76540481, 2891150, 59371804, 80316608, 58208470, 44348328, 69641513, 22450468, 26998766, 99658235, 3487592, 88653118, 8055981, 73235980, 3088684, 30998561, 79922758, 26114953, 78549759, 20568363, 91831487, 67084351, 49236559, 67451935, 6157724, 27325428, 38008118, 61712234, 20681921, 16595436, 32699744, 53547802, 6153406, 45794415, 19898053, 35456853, 81898046, 86460488, 70727211, 76170907, 65880522, 5482538, 52613508, 75407347, 43045786, 5640302, 39801351, 42644903, 62051033, 21289531, 17016156, 1022677, 60102965, 4204661, 89804152, 58180653, 11543098, 874791, 80014588, 84904436, 65081429, 36845587, 80555751, 37739481, 30463802, 1239555, 83948335, 75986488, 29635537, 57163802, 89214616, 22766820, 44889423, 37620363, 79136082, 70541760, 67281495, 7066775, 99965001, 56461322, 47090124, 58751351, 5970607, 16971929, 96726697, 55143724, 36135, 61859143, 57359924, 30811010, 1204161, 83302115, 94711842, 56484900, 1431742, 14731700, 34497327, 57658654, 9603598, 11757872, 41092102, 86022504, 17037369, 50567636, 56955985, 45848907, 84127901, 57240218, 57803235, 6986898, 83155442, 17081350, 1250437, 46870723, 99971982, 168541, 44550764, 82897371, 2917920, 67124282, 44846932, 36753250, 74614639, 32274392, 31904591, 24733232, 26664538, 32161669, 93057697, 83210802, 40677414, 90158785, 51588015, 22405120, 4095116, 35996293, 26139110, 33336148, 18001617, 29516592, 78218845, 96852321, 83133790, 38510840, 90013093, 8099994, 75153252, 58749, 11923835, 9623492, 66667729, 54199160, 508198, 62115552, 33628349, 61271144, 36580610, 63628376, 36189527, 4978543, 17976208, 59405277, 20885148, 34667286, 68128438, 15163258, 30764367, 34236719, 6525948, 98130363, 15015906, 65715134, 70596786, 22879907, 62490109, 68110330, 30787683, 64602895, 88130087, 30090481, 9575954, 68875490, 13819358, 62803858, 29932657, 77301523, 569864, 89217461, 86543538, 6111563, 43933006, 48088883, 53666583, 42199455, 76671482, 52293995, 8913721, 26507214, 44847298, 415901, 37560164, 92692283, 23110625, 9188443, 7685448, 92604458, 96906410, 32426752, 42692881, 44245960, 71522968, 7646095, 4099191, 56473732, 8791066, 41442762, 68644627, 71333116, 85711894, 97379790, 68316156, 7517032, 77898274, 24915585, 40781449, 27625735, 69355476, 79806380, 49271185, 66428795, 4069912, 99021067, 50668599, 10264691, 16567550, 18806856, 8733068, 12024238, 4668450, 20002147, 67474219, 67030811, 98948034, 3294781, 77072625, 36780454, 62428472, 45996863, 12664567, 36396314, 17385531, 47738185, 30771409, 247198, 16380211, 67793644, 50806615, 66832478, 53888755, 68694897, 5111370, 62693428, 89637706, 36812683, 19457589, 43501211, 54517921, 83368048, 92530431, 61859581, 9599614, 92803766, 76825057, 35512853, 45407418, 16405341, 45617087, 97281847, 30139692, 44473167, 3509435, 19939935, 25842078, 72274002, 16445503, 84840549, 34698428, 66250369, 53842979, 33895336, 82427263, 7229550, 38494874, 66903004, 14326617, 83150534, 21919959, 95010552, 26392416, 30366150, 10358899, 37280276, 749283, 44915235, 15075176, 92398073, 15948937, 97011160, 22997281, 89996536, 59614746, 26734892, 65338021, 20486294, 53648154, 21673260, 80246713, 8807940, 24619760, 89046466, 87598888, 31733363, 99861373, 93790285, 37224844, 59197747, 24826575, 67031644, 62552524, 3235882, 86301513, 63059024, 73168565, 61728685, 55850790, 47887470, 23134715, 7300047, 72793444, 36685023, 66271566, 85117092, 26063929, 73617245, 30653863, 38365584, 32590267, 73124510, 6793819, 51047803, 63967300, 27187213, 70367851, 36808486, 63152504, 6808825, 23740167, 18783702, 86118021, 33237508, 22113133, 52204879, 18131876, 78766358, 7182642, 52060076, 4091162, 37659250, 18699206, 33935899, 91938191, 50188404, 23194618, 29401781, 47298834, 74441448, 78300864, 77284619, 62496012, 66663942, 55669657, 54606384, 83269727, 26744917, 89078848, 40686254, 71657078, 45995325, 34432810, 54987042, 72278539, 14220886, 321665, 65017137, 72358170, 88904910, 95251277, 61960400, 71083565, 45075471, 44481640, 23848565, 19376156, 69136837, 92787493, 41092172, 20642888, 66137019, 75543508, 33797252, 97561745, 90272749, 80251430, 84684495, 71965942, 93359396, 28550822, 61623915, 65271999, 6221471, 58224549, 28796059, 66045587, 57248122, 91802888, 55602660, 23569917, 30694952, 8651647, 59981773, 55470718, 88444207, 47893286, 4199704, 93515664, 10309525, 53802686, 32250045, 72495719, 62430984, 14723410, 1197320, 17857111, 55770687, 21070110, 15902805, 40984766, 38009615, 176257, 48360198, 64157906, 13348726, 45990383, 69255765, 30569392, 7423788, 10649306, 60581278, 42967683, 94360702, 3233569, 17058722, 66322959, 84002370, 46851987, 48892201, 45428665, 9175338, 18411915, 71300104, 99690194, 112651, 5073754, 77694700, 81172706, 71469330, 93562257, 51464002, 83747892, 55960386, 50007421, 7011964, 4508700, 71316369, 29466406, 2607799, 88047921, 16424199, 41481685, 90654836, 72732205, 4119747, 92692978, 74110882, 63015256, 45919976, 60393039, 31491938, 64055960, 56424103, 62762857, 79322415, 37166608, 33201905, 68939068, 26292919, 53632373, 99515901, 52261574, 82726008, 14349098, 44060493, 29819940, 34946859, 79821897, 6871053, 10597197, 48893685, 56515456, 94911072, 44627776, 91281584, 57020481, 18833224, 43506672, 39553046, 15148031, 44842615, 5267545, 84349107, 60176618, 83651211, 99549373, 17764950, 9058407, 85242963, 83533741, 27411561, 14947650, 97940276, 45667668, 94090109, 13173644, 81855445, 21533347, 57961282, 26863229, 86001008, 21001913, 27185644, 33304202, 90457870, 68102437, 40865610, 48395186, 7104732, 44664587, 68702632, 28787861, 3880712, 96735716, 4515343, 75052463, 60106217, 89419466, 70782102, 36933038, 26397786, 64848072, 51472275, 11161731, 51590803, 24591705, 73222868, 82178706, 62232211, 38061439, 3773993, 44177011, 79738755, 78561158, 98739783, 82886381, 93933709, 26275734, 42307449, 16019925, 54427233, 84293052, 62936963, 36505482, 35192533, 61815223, 7955293, 41380093, 92541302, 15535065, 48658605, 18663507, 2204165, 6505939, 59329511, 38022834, 95957797, 77787724, 59910624, 32058615, 19272365, 86798033, 65074535, 16684829, 72777973, 72238278, 9257405, 72373496, 20645197, 73021291, 5822202, 36930650, 24168411, 11581181, 99333375, 8128637, 96709982, 48774913, 7502255, 32151165, 94076128, 94595668, 97641116, 669105, 4985896, 92353856, 56531125, 65038678, 68824981, 49598724, 68041839, 19891772, 69579137, 88698958, 3233084, 17957593, 17894977, 87160386, 75128745, 54762643, 93270084, 36312813, 78602717, 9259676, 81274566, 14093520, 1788101, 91990218, 57393458, 54014062, 77272628, 69848388, 7919588, 81853704, 67227442, 73392814, 21993752, 22942635, 30891921, 64087743, 33061250, 30395570, 70004753, 40027975, 22108665, 61928316, 8129978, 95395112, 33123618, 29959549, 37675718, 16097038, 55189057, 51149804, 99729357, 63506281, 51507425, 72738685, 29188588, 16099750, 98371444, 33249630, 41245325, 25636669, 17146629, 99297164, 29834512, 43376279, 44479073, 56153345, 87720882, 95726235, 43152977, 55247455, 96193415, 70420215, 74452589, 76330843, 4064751, 21397057, 37891451, 73031054, 91255408, 8696647, 79191827, 82327024, 59109400, 13470059, 79942022, 94516935, 45790169, 8634541, 17068582, 49328608, 42782652, 81805959, 98943869, 37957788, 34493392, 32159704, 24058273, 78884452, 85695762, 99604946, 54263880, 75135448, 39171895, 98653983, 13468268, 2331773, 78909561, 69786756, 12303248, 18504197, 27041967, 68204242, 51466049, 30218878, 67513640, 22129328, 65851721, 61897582, 90090964, 62452034, 61141874, 16387575, 99917599, 10366309, 91240048, 23432750, 48673079, 93053405, 98462867, 1936762, 9829782, 70036438, 18466635, 17365188, 38658347, 76434144, 77377183, 77312810, 86242799, 31727379, 68816413, 12416768, 83789391, 8729146, 74724075, 37501808, 51426311, 33699435, 16054533, 6819644, 66319530, 45381876, 40534591, 42947632, 53393358, 69605283, 99524975, 95581843, 53084256 +64602895, 62936963, 37560164, 86118021, 24168411, 60176618, 65338021, 68000591, 62051033, 55850790, 27187213, 16054533, 82339363, 75229462, 89046466, 61373987, 8729146, 84002370, 52204879, 65074535, 73021291, 62762857, 84127901, 2917920, 16387575, 93566986, 84349107, 47361209, 70782102, 6157724, 15948937, 69848388, 73392814, 1569515, 20867149, 37224844, 60581278, 61263068, 89217461, 39847321, 92604458, 33249630, 12303248, 74724075, 37620363, 4099191, 71316369, 78766358, 57359924, 33935899, 34497327, 26744917, 57803235, 17081350, 61141874, 82327024, 36139942, 4434662, 81855445, 3509435, 35092039, 76315420, 93359396, 81274566, 60106217, 55470718, 53802686, 59614746, 21070110, 20486294, 22942635, 87598888, 98648327, 45990383, 38341669, 92867155, 75777973, 73168565, 21289531, 3233569, 12348118, 22113133, 29834512, 43376279, 97379790, 24915585, 7687278, 56153345, 72373496, 47298834, 64055960, 98948034, 9603598, 37166608, 89699445, 31727379, 82726008, 71657078, 94595668, 97057187, 65851721, 97641116, 669105, 44550764, 74743862, 11365791, 29065964, 57020481, 18833224, 83368048, 79191827, 92530431, 31904591, 10366309, 91240048, 75543508, 21533347, 3487592, 73235980, 93270084, 66250369, 66667729, 36312813, 28787861, 49328608, 57248122, 81805959, 33628349, 20568363, 75052463, 98943869, 47792865, 26392416, 92398073, 10309525, 38658347, 22879907, 76170907, 59197747, 76434144, 69255765, 68875490, 20765474, 31161687, 52293995, 8913721, 73617245, 30653863, 30463802, 1239555, 49597667, 57163802, 96906410, 69786756, 66885828, 2204165, 51464002, 33699435, 58751351, 39373729, 96726697, 37659250, 4119747, 91938191, 49271185, 1204161, 68204242, 9257405, 1431742, 86242799, 20002147, 6819644, 77187825, 45996863, 99515901, 46540998, 40534591, 34432810, 22721500, 9886593, 36812683, 61960400, 71083565, 32274392, 45075471, 39553046, 99917599, 19376156, 13470059, 4787945, 16405341, 49598724, 97561745, 71965942, 43357947, 16445503, 51715482, 72019362, 99658235, 40865610, 75128745, 65271999, 58224549, 45237957, 44664587, 3880712, 66045587, 4515343, 79922758, 30694952, 85023028, 68816413, 67451935, 20885148, 18466635, 61712234, 67227442, 24058273, 53393358, 89499542, 69605283, 81898046, 24619760, 93790285, 88130087, 9575954, 8129978, 33553959, 16097038, 7300047, 48088883, 98653983, 66271566, 26063929, 73124510, 72738685, 29188588, 85116755, 16099750, 22766820, 44889423, 7646095, 6505939, 56473732, 99297164, 41442762, 34044787, 18131876, 59910624, 49641577, 41481685, 68316156, 7517032, 77898274, 69355476, 27665211, 79806380, 18699206, 66428795, 92033260, 87720882, 50668599, 23194618, 29994197, 94711842, 56484900, 4668450, 96193415, 36930650, 55669657, 41092102, 45381876, 52261574, 4064751, 48774913, 82779622, 7502255, 247198, 67793644, 69697787, 62693428, 65017137, 72358170, 99125126, 44846932, 88904910, 44627776, 19457589, 91281584, 95251277, 83651211, 35512853, 4806458, 92787493, 22405120, 48673079, 83533741, 2891150, 59371804, 93053405, 26493119, 90272749, 44473167, 58208470, 92215320, 84684495, 3233084, 21001913, 90457870, 87160386, 68102437, 61623915, 66611759, 54199160, 30998561, 508198, 9259676, 38494874, 66903004, 14326617, 1788101, 91990218, 57393458, 30366150, 749283, 15075176, 14626618, 34493392, 14723410, 7919588, 15015906, 73222868, 44844121, 40984766, 64087743, 44177011, 30090481, 77377183, 98739783, 39801351, 13819358, 42644903, 61728685, 29932657, 82886381, 47887470, 17016156, 37675718, 23134715, 4204661, 17058722, 51149804, 34698463, 13468268, 80014588, 79880247, 84293052, 65081429, 35192533, 23110625, 71300104, 89214616, 70372191, 63967300, 5073754, 77694700, 18663507, 79136082, 18504197, 41245325, 67281495, 23740167, 55960386, 18783702, 56461322, 47090124, 8791066, 88047921, 85711894, 61859143, 4091162, 27625735, 71920426, 74110882, 76960303, 99021067, 72777973, 16567550, 95726235, 18806856, 83302115, 60393039, 55247455, 83083131, 66319530, 67030811, 59501487, 99226875, 11757872, 11581181, 33565483, 45848907, 8128637, 77413012, 57240218, 46870723, 17727650, 44060493, 10453030, 6871053, 10597197, 168541, 30163921, 90090964, 82897371, 68694897, 67124282, 5111370, 9599614, 76825057, 51588015, 97940276, 91727510, 26139110, 22435353, 78218845, 96852321, 26863229, 86001008, 33304202, 90013093, 22450468, 26998766, 78785507, 53084256, 68702632, 53842979, 33895336, 96420429, 91802888, 6038457, 3183975, 74137926, 78549759, 8651647, 59981773, 88444207, 84406788, 10358899, 37280276, 9829782, 6497830, 37957788, 9860195, 65454636, 34667286, 27325428, 77272628, 68128438, 94539824, 34236719, 54663246, 6525948, 26734892, 3633375, 78884452, 30891921, 12416768, 86460488, 38061439, 33061250, 3773993, 19486173, 5482538, 78589145, 62552524, 47213183, 86301513, 5640302, 92554120, 77301523, 569864, 42967683, 6111563, 94360702, 43933006, 39171895, 53666583, 26275734, 89804152, 874791, 2331773, 44847298, 37739481, 61741594, 38365584, 6793819, 9188443, 42073124, 43322743, 44245960, 47708850, 93562257, 83747892, 51426311, 25636669, 95957797, 71333116, 2607799, 91664334, 55143724, 16424199, 90654836, 54232247, 65047700, 16684829, 51466049, 12024238, 31491938, 1375023, 99524975, 24953498, 20645197, 40781854, 3294781, 72357096, 824872, 56424103, 79322415, 36780454, 99333375, 67513640, 37192445, 36396314, 53632373, 6986898, 50702367, 83155442, 99224392, 47738185, 79821897, 45995325, 37891451, 99971982, 50806615, 66832478, 86821229, 72278539, 91255408, 14220886, 2717150, 9398733, 45306070, 56531125, 321665, 94911072, 65038678, 54517921, 44481640, 44842615, 93057697, 83210802, 23432750, 17764950, 76540481, 35996293, 66137019, 33336148, 8373289, 94090109, 80251430, 69641513, 98462867, 84840549, 49882705, 28928175, 75153252, 58749, 10961421, 91141395, 34698428, 63372756, 7104732, 28796059, 96735716, 55602660, 62115552, 23569917, 51315460, 9860968, 42947632, 14093520, 21919959, 95010552, 36468541, 61271144, 67084351, 36580610, 13862149, 48675329, 4199704, 36189527, 17976208, 17365188, 22997281, 84166196, 64098930, 17857111, 20681921, 53547802, 6153406, 70596786, 19898053, 21993752, 21673260, 15902805, 83789391, 79738755, 55753905, 22108665, 61928316, 44784505, 78561158, 66116458, 63059024, 43045786, 95395112, 65275241, 62803858, 33123618, 29959549, 72793444, 55189057, 36685023, 76703609, 16019925, 46851987, 26507214, 4798568, 72614359, 80555751, 78909561, 41380093, 34295794, 15535065, 29635537, 59177718, 112651, 92998591, 29029316, 71469330, 7066775, 33237508, 59329511, 16971929, 31126490, 7182642, 36135, 82651278, 82532312, 24314885, 29401781, 85571389, 73786944, 10264691, 8733068, 78300864, 77284619, 30218878, 97783876, 74452589, 86022504, 17037369, 40686254, 1250437, 76330843, 21397057, 29819940, 5832946, 30771409, 32151165, 53888755, 54987042, 8696647, 36753250, 526217, 43501211, 26664538, 61859581, 23848565, 69136837, 40677414, 45407418, 9058407, 85242963, 27411561, 94516935, 68041839, 45667668, 30139692, 57241521, 19891772, 29510992, 25842078, 17957593, 83133790, 72274002, 88251446, 54762643, 62357986, 3088684, 26776922, 91831487, 89419466, 36933038, 26397786, 54014062, 63628376, 22200378, 44915235, 93515664, 59405277, 11161731, 32250045, 51590803, 72495719, 81677380, 32159704, 89996536, 38008118, 1197320, 16595436, 65715134, 82178706, 62490109, 30395570, 54263880, 30787683, 176257, 70727211, 65880522, 13348726, 3235882, 52613508, 75407347, 30569392, 1022677, 77312810, 37183543, 60102965, 82052050, 40197395, 11543098, 85117092, 99729357, 48260151, 55615885, 36505482, 61815223, 19101477, 92692283, 92541302, 98371444, 51047803, 50007421, 17146629, 38022834, 77787724, 29466406, 27041967, 74357852, 81774825, 20122224, 92692978, 63015256, 44479073, 50188404, 72238278, 39201414, 74441448, 14731700, 5822202, 70420215, 77072625, 54606384, 87710366, 62428472, 26292919, 89078848, 17386996, 17385531, 54058788, 59318837, 40152546, 94076128, 44983451, 6521313, 92353856, 45049308, 43506672, 74614639, 15148031, 32161669, 90310261, 8115266, 90158785, 41092172, 79942022, 80316608, 97281847, 13173644, 69579137, 17539625, 13231279, 88698958, 38510840, 28550822, 88653118, 8055981, 60430369, 26114953, 83150534, 95290172, 47893286, 28734791, 4978543, 90061527, 62430984, 15163258, 24591705, 81853704, 45794415, 55770687, 85695762, 53648154, 80246713, 8807940, 38009615, 31733363, 99861373, 40027975, 12571310, 75135448, 48360198, 64157906, 92071990, 7423788, 42307449, 63506281, 66322959, 54427233, 7955293, 83948335, 91957544, 75986488, 45428665, 18411915, 48658605, 7685448, 99690194, 42692881, 37501808, 63152504, 6808825, 70541760, 99965001, 5970607, 14045383, 32058615, 52060076, 30811010, 72732205, 14363867, 4069912, 43152977, 60955663, 67474219, 20140249, 66663942, 61982238, 84187166, 15536795, 12664567, 14349098, 68824981, 24733232, 38645117, 59109400, 99549373, 70800879, 33797252, 45617087, 18001617, 57961282, 75820087, 44348328, 27185644, 48395186, 42782652, 82427263, 78602717, 1936762, 42237907, 49236559, 64848072, 51472275, 30764367, 98130363, 35456853, 62232211, 68110330, 99604946, 70004753, 24826575, 10649306, 82979980, 58180653, 42199455, 51507425, 84904436, 415901, 9175338, 54868730, 81172706, 71522968, 89811711, 68644627, 40781449, 86798033, 40356877, 37435892, 62496012, 57658654, 33201905, 68939068, 50567636, 39986008, 56955985, 96709982, 34946859, 96318094, 16380211, 61897582, 4985896, 48893685, 62452034, 77300457, 5267545, 92803766, 72725103, 4095116, 14947650, 33431960, 8099994, 6221471, 9623492, 7229550, 20836893, 67031644, 86543538, 76671482, 62740044, 36845587, 32590267, 77620120, 32426752, 70367851, 73109997, 36808486, 4508700, 38256849, 73031054, 20642888, 26102057, 19939935, 17894977, 17068582, 11923835, 79657802, 2208785, 70036438, 32699744, 15064655, 93933709, 30543215, 48892201, 28851716, 19272365, 83269727, 56515456, 89637706, 45790169, 29516592, 8634541, 95581843, 97011160, 7011964, 45919976, 22129328, 90004325 +9398733, 70800879, 62357986, 42237907, 62740044, 1239555, 45428665, 30163921, 60176618, 30694952, 75052463, 29932657, 37739481, 63967300, 4119747, 50188404, 61982238, 2891150, 8373289, 80251430, 7104732, 28796059, 85023028, 65454636, 89046466, 52613508, 80014588, 78909561, 18504197, 23740167, 99965001, 29834512, 14045383, 68316156, 82651278, 44479073, 29401781, 12024238, 36930650, 33565483, 21397057, 43506672, 74614639, 91240048, 80316608, 97281847, 98462867, 68102437, 48395186, 28787861, 82427263, 60106217, 59981773, 6497830, 18466635, 51590803, 97011160, 89996536, 64098930, 32699744, 65338021, 19898053, 73222868, 22942635, 19486173, 65880522, 78589145, 13348726, 33123618, 21289531, 29959549, 8913721, 42307449, 84293052, 73617245, 62936963, 61741594, 38365584, 29635537, 7685448, 69786756, 12303248, 37501808, 93562257, 33699435, 32058615, 1204161, 9257405, 40356877, 16567550, 8733068, 47298834, 55247455, 98948034, 54606384, 89699445, 99333375, 77413012, 99515901, 30771409, 10453030, 32151165, 11365791, 8696647, 67124282, 56515456, 95251277, 43501211, 79191827, 24733232, 5267545, 8115266, 35512853, 20642888, 79942022, 33336148, 90013093, 43357947, 90457870, 11923835, 54762643, 4515343, 57248122, 66903004, 42947632, 36468541, 1788101, 26397786, 36580610, 9829782, 9860195, 77272628, 68128438, 30764367, 20681921, 16595436, 69605283, 38009615, 70727211, 76170907, 13819358, 17016156, 37183543, 86543538, 43933006, 30543215, 34698463, 30653863, 4798568, 49597667, 92541302, 98371444, 70372191, 33249630, 66885828, 81172706, 7066775, 18783702, 56461322, 38022834, 7182642, 41481685, 90654836, 16684829, 56153345, 99021067, 6819644, 73021291, 20140249, 824872, 34497327, 74452589, 38256849, 84187166, 26744917, 89078848, 57803235, 83155442, 46540998, 34946859, 45995325, 50806615, 54987042, 90090964, 44550764, 45306070, 94911072, 88904910, 19457589, 45075471, 99917599, 15148031, 90158785, 4806458, 27411561, 14947650, 68041839, 45617087, 18001617, 97561745, 13173644, 92215320, 86001008, 72274002, 8099994, 26998766, 93270084, 36312813, 66045587, 96735716, 55602660, 62115552, 78549759, 37280276, 47893286, 51472275, 15075176, 6157724, 53802686, 32250045, 27325428, 59614746, 61712234, 15015906, 62490109, 44177011, 79738755, 37224844, 48360198, 88130087, 76434144, 67031644, 98648327, 92554120, 75777973, 1022677, 3233569, 58180653, 42199455, 36685023, 76671482, 11543098, 85117092, 874791, 65081429, 30463802, 9175338, 16099750, 12348118, 18663507, 6505939, 51464002, 83747892, 51426311, 4099191, 95957797, 52204879, 2607799, 77898274, 20122224, 28851716, 27625735, 91938191, 87720882, 72777973, 85571389, 51466049, 43152977, 20645197, 30218878, 96193415, 62762857, 97783876, 41092102, 24168411, 11581181, 67513640, 45848907, 53632373, 50702367, 17386996, 40686254, 82726008, 48774913, 82779622, 16380211, 10597197, 74743862, 92353856, 44627776, 36753250, 77300457, 18833224, 71083565, 13470059, 83651211, 22405120, 9058407, 48673079, 91727510, 26139110, 22435353, 90272749, 57241521, 69579137, 21533347, 75820087, 25842078, 38510840, 61623915, 3487592, 6221471, 88653118, 73235980, 3088684, 66667729, 508198, 2208785, 78602717, 23569917, 67084351, 30366150, 84406788, 49236559, 48675329, 17976208, 20885148, 15948937, 81677380, 62430984, 26734892, 53393358, 20486294, 44844121, 86460488, 99604946, 3773993, 30787683, 1569515, 176257, 83789391, 87598888, 31733363, 64602895, 77377183, 62552524, 44784505, 68875490, 20765474, 38341669, 62803858, 73168565, 60581278, 47887470, 77301523, 33553959, 82979980, 23134715, 26275734, 72793444, 99729357, 63506281, 79880247, 2331773, 36845587, 48892201, 83948335, 23110625, 29188588, 71300104, 96906410, 74724075, 44245960, 71522968, 36808486, 79136082, 41245325, 67281495, 47090124, 89811711, 68644627, 78766358, 54232247, 18699206, 65047700, 65074535, 99524975, 99226875, 9603598, 36780454, 31727379, 68939068, 14349098, 4064751, 54058788, 79821897, 96318094, 40152546, 247198, 34432810, 99971982, 73031054, 67793644, 168541, 61897582, 66832478, 91255408, 4985896, 2717150, 62452034, 16387575, 526217, 54517921, 93566986, 32161669, 10366309, 69136837, 83210802, 40677414, 17764950, 16405341, 4095116, 97940276, 45790169, 4434662, 33797252, 49598724, 8634541, 3509435, 88698958, 96852321, 83133790, 71965942, 21001913, 17068582, 76315420, 22450468, 93359396, 84840549, 49882705, 28550822, 53084256, 58749, 10961421, 34698428, 65271999, 8055981, 68702632, 54199160, 3880712, 53842979, 79922758, 33628349, 9860968, 14093520, 47792865, 21919959, 57393458, 54014062, 13862149, 4199704, 36189527, 17365188, 34493392, 38008118, 14723410, 69848388, 17857111, 7919588, 73392814, 38658347, 81898046, 24619760, 70004753, 99861373, 93790285, 8729146, 59197747, 3235882, 78561158, 30569392, 86301513, 5640302, 10649306, 39801351, 62051033, 92867155, 61728685, 82886381, 37675718, 77312810, 6111563, 94360702, 39171895, 48088883, 98653983, 55189057, 40197395, 76703609, 26063929, 66322959, 55615885, 36505482, 61815223, 37560164, 34295794, 85116755, 112651, 92998591, 43322743, 70367851, 2204165, 71469330, 63152504, 6808825, 56473732, 99297164, 71316369, 71333116, 27041967, 43376279, 55143724, 16054533, 97379790, 52060076, 24915585, 72732205, 37659250, 92692978, 74110882, 24314885, 49271185, 19272365, 14363867, 1431742, 78300864, 67474219, 62496012, 59501487, 5822202, 77187825, 62428472, 50567636, 45381876, 15536795, 84127901, 17081350, 96709982, 99224392, 44060493, 59318837, 65851721, 97641116, 22721500, 6521313, 62693428, 72358170, 61141874, 29065964, 91281584, 57020481, 39553046, 44481640, 44842615, 36139942, 19376156, 84349107, 59109400, 85242963, 66137019, 26493119, 45667668, 78218845, 19891772, 29510992, 13231279, 44348328, 3233084, 33304202, 51715482, 88251446, 99658235, 75128745, 91141395, 58224549, 44664587, 95581843, 79657802, 33895336, 60430369, 26114953, 1936762, 91831487, 68816413, 55470718, 83150534, 70782102, 63628376, 22200378, 44915235, 37957788, 11161731, 34667286, 90061527, 22997281, 94539824, 32159704, 34236719, 6525948, 24058273, 89499542, 21070110, 15902805, 8807940, 68110330, 64087743, 20867149, 15064655, 24826575, 92071990, 69255765, 7423788, 43045786, 98739783, 65275241, 93933709, 82052050, 51149804, 48260151, 16019925, 54427233, 84904436, 84002370, 7955293, 415901, 73124510, 92692283, 57163802, 48658605, 51047803, 54868730, 42073124, 22766820, 7646095, 37620363, 70541760, 55960386, 8791066, 17146629, 58751351, 34044787, 77787724, 74357852, 88047921, 59910624, 16424199, 49641577, 36135, 7517032, 57359924, 30811010, 71920426, 79806380, 7687278, 86798033, 66428795, 92033260, 68204242, 45919976, 94711842, 60393039, 31491938, 64055960, 83083131, 86242799, 40781854, 20002147, 77284619, 72357096, 37166608, 33201905, 17037369, 45996863, 26292919, 1250437, 47738185, 29819940, 7502255, 71657078, 40534591, 37891451, 97057187, 53888755, 14220886, 56531125, 32274392, 92530431, 68824981, 61859581, 9599614, 72725103, 99549373, 45407418, 92787493, 76540481, 26102057, 35996293, 94516935, 75543508, 30139692, 29516592, 94090109, 47361209, 27185644, 35092039, 87160386, 9623492, 66250369, 42782652, 91802888, 6038457, 20568363, 98943869, 61271144, 70036438, 92398073, 10309525, 14626618, 15163258, 84166196, 6153406, 85695762, 21993752, 80246713, 12416768, 33061250, 54263880, 40027975, 75135448, 9575954, 55850790, 52293995, 13468268, 51507425, 46851987, 72614359, 44847298, 41380093, 77620120, 15535065, 75986488, 39847321, 6793819, 9188443, 99690194, 5073754, 47708850, 7011964, 25636669, 4508700, 39373729, 33237508, 5970607, 96726697, 85711894, 81774825, 61859143, 4091162, 82532312, 27665211, 33935899, 76960303, 72373496, 29994197, 39201414, 18806856, 37435892, 24953498, 74441448, 67030811, 66663942, 56424103, 70420215, 57658654, 11757872, 55669657, 22129328, 46870723, 44983451, 69697787, 2917920, 89637706, 65017137, 99125126, 9886593, 44846932, 45049308, 65038678, 61960400, 83368048, 31904591, 93057697, 38645117, 23432750, 4787945, 41092172, 83533741, 33431960, 17539625, 58208470, 57961282, 69641513, 17957593, 16445503, 78785507, 63372756, 66611759, 30998561, 7229550, 3183975, 81805959, 51315460, 38494874, 8651647, 81274566, 14326617, 36933038, 26392416, 91990218, 10358899, 64848072, 93515664, 4978543, 59405277, 54663246, 98130363, 24591705, 45794415, 55770687, 35456853, 38061439, 30395570, 61373987, 5482538, 30090481, 68000591, 61928316, 45990383, 75407347, 8129978, 95395112, 61263068, 569864, 7300047, 4204661, 89804152, 17058722, 66271566, 26507214, 32590267, 18411915, 89214616, 59177718, 77694700, 86118021, 50007421, 16971929, 18131876, 31126490, 40781449, 72238278, 73786944, 10264691, 95726235, 83302115, 1375023, 56484900, 60955663, 14731700, 66319530, 3294781, 79322415, 83269727, 57240218, 52261574, 94595668, 669105, 321665, 36812683, 51588015, 44473167, 81855445, 84684495, 19939935, 28928175, 75153252, 45237957, 49328608, 89419466, 95010552, 28734791, 72495719, 20836893, 67227442, 3633375, 70596786, 78884452, 53648154, 82178706, 62232211, 21673260, 40984766, 64157906, 66116458, 63059024, 89217461, 60102965, 35192533, 91957544, 72738685, 92604458, 32426752, 42692881, 27187213, 44889423, 41442762, 22113133, 91664334, 90004325, 4069912, 23194618, 4668450, 86022504, 56955985, 8128637, 36396314, 6986898, 17385531, 76330843, 17727650, 6871053, 86821229, 82897371, 48893685, 68694897, 5111370, 92803766, 93053405, 17894977, 40865610, 96420429, 74137926, 9259676, 749283, 67451935, 1197320, 81853704, 65715134, 30891921, 12571310, 55753905, 22108665, 47213183, 42644903, 31161687, 19101477, 29029316, 73109997, 59329511, 69355476, 63015256, 50668599, 87710366, 12664567, 94076128, 72278539, 26664538, 82327024, 90310261, 76825057, 26863229, 72019362, 26776922, 95290172, 88444207, 53547802, 22879907, 53666583, 77072625, 39986008, 37192445, 5832946, 82339363, 16097038, 80555751, 29466406, 23848565, 59371804, 75229462, 42967683 +65074535, 65038678, 99917599, 31904591, 84349107, 26139110, 1569515, 18783702, 32058615, 12664567, 50702367, 37891451, 48893685, 68824981, 82327024, 75128745, 92398073, 10309525, 26734892, 99861373, 76170907, 17058722, 54427233, 4099191, 33699435, 59910624, 91938191, 34946859, 79821897, 321665, 60176618, 81855445, 66250369, 95581843, 55470718, 14626618, 34236719, 6153406, 52613508, 98739783, 65275241, 77312810, 80014588, 70372191, 27187213, 56461322, 17146629, 16054533, 99021067, 72777973, 72373496, 34497327, 86821229, 74743862, 94911072, 95251277, 83651211, 4787945, 45617087, 40865610, 54199160, 49328608, 74137926, 85023028, 21919959, 20885148, 77272628, 76434144, 68000591, 78561158, 75407347, 66116458, 26063929, 77620120, 48658605, 96906410, 32426752, 70367851, 37501808, 67281495, 50007421, 68644627, 71333116, 57359924, 74110882, 27665211, 18699206, 1204161, 824872, 15536795, 40686254, 83155442, 99515901, 52261574, 50806615, 97641116, 30163921, 66832478, 44983451, 45306070, 67124282, 56531125, 93566986, 59109400, 99549373, 41092172, 85242963, 94516935, 29516592, 8373289, 49882705, 28550822, 6221471, 88653118, 28787861, 6038457, 1936762, 68816413, 36580610, 54014062, 47893286, 59405277, 54663246, 17857111, 20836893, 81853704, 24058273, 53393358, 73222868, 80246713, 24619760, 55753905, 45990383, 44784505, 8129978, 43045786, 75777973, 82886381, 37675718, 569864, 23134715, 39171895, 60102965, 53666583, 8913721, 26507214, 32590267, 41380093, 23110625, 89214616, 92604458, 69786756, 18504197, 7011964, 33237508, 16971929, 78766358, 16424199, 81774825, 7517032, 16684829, 23194618, 95726235, 94711842, 74441448, 20002147, 62762857, 33565483, 50567636, 17081350, 76330843, 46870723, 5832946, 32151165, 10597197, 34432810, 99971982, 67793644, 91255408, 22721500, 90090964, 56515456, 5111370, 57020481, 526217, 44481640, 10366309, 48673079, 4095116, 97281847, 94090109, 58208470, 3509435, 44348328, 98462867, 17068582, 75153252, 66611759, 93270084, 28796059, 33895336, 60430369, 57248122, 55602660, 3183975, 81805959, 60106217, 61271144, 88444207, 26397786, 49236559, 28734791, 15075176, 15948937, 32159704, 6525948, 7919588, 19898053, 21993752, 30891921, 8807940, 64087743, 99604946, 64602895, 62552524, 9575954, 3235882, 63059024, 62051033, 21289531, 89804152, 98653983, 42199455, 36685023, 48260151, 13468268, 84293052, 4798568, 61815223, 19101477, 73124510, 75986488, 85116755, 7685448, 51047803, 54868730, 12303248, 22766820, 81172706, 7646095, 6505939, 55960386, 86118021, 99297164, 22113133, 5970607, 52204879, 74357852, 96726697, 31126490, 88047921, 4091162, 20122224, 19272365, 68204242, 45919976, 16567550, 78300864, 5822202, 54606384, 37166608, 68939068, 45848907, 17727650, 82779622, 21397057, 96318094, 6871053, 45995325, 94595668, 669105, 69697787, 62452034, 65017137, 9886593, 61141874, 29065964, 16387575, 83368048, 79191827, 92530431, 17764950, 75543508, 80316608, 45790169, 45667668, 44473167, 29510992, 84684495, 17894977, 76315420, 8099994, 88251446, 28928175, 48395186, 66667729, 30998561, 2208785, 20568363, 47792865, 83150534, 36933038, 57393458, 30366150, 13862149, 37280276, 22200378, 93515664, 37957788, 32250045, 90061527, 68128438, 15163258, 67227442, 65715134, 65338021, 70596786, 78884452, 89499542, 85695762, 68110330, 30395570, 54263880, 61373987, 31733363, 79738755, 37224844, 48360198, 88130087, 98648327, 77377183, 47213183, 68875490, 95395112, 42644903, 33123618, 29959549, 93933709, 58180653, 30543215, 16019925, 874791, 84904436, 91957544, 37560164, 92692283, 72738685, 45428665, 9175338, 71300104, 42073124, 33249630, 43322743, 77694700, 74724075, 44889423, 71522968, 51464002, 41245325, 25636669, 59329511, 95957797, 29834512, 27041967, 14045383, 97379790, 61859143, 30811010, 72732205, 79806380, 49271185, 92033260, 14363867, 56153345, 50668599, 72238278, 29994197, 40781854, 14731700, 67474219, 67030811, 77284619, 62496012, 70420215, 57658654, 74452589, 11581181, 26744917, 57803235, 6986898, 40534591, 16380211, 168541, 53888755, 54987042, 4985896, 11365791, 61960400, 15148031, 26664538, 9599614, 93057697, 36139942, 92803766, 40677414, 13470059, 51588015, 92787493, 9058407, 70800879, 66137019, 59371804, 93053405, 68041839, 97561745, 30139692, 80251430, 26863229, 86001008, 3233084, 83133790, 72274002, 87160386, 58749, 11923835, 62357986, 7104732, 3088684, 26776922, 96735716, 42782652, 82427263, 79922758, 91802888, 78602717, 78549759, 33628349, 8651647, 66903004, 59981773, 42947632, 91990218, 70036438, 6497830, 6157724, 53802686, 62430984, 84166196, 98130363, 1197320, 24591705, 61712234, 15015906, 16595436, 32699744, 82178706, 62232211, 33061250, 3773993, 20867149, 87598888, 93790285, 59197747, 67031644, 22108665, 69255765, 7423788, 39801351, 38341669, 31161687, 73168565, 61728685, 29932657, 60581278, 77301523, 33553959, 37183543, 42967683, 86543538, 94360702, 34698463, 63506281, 51507425, 44847298, 36845587, 62936963, 37739481, 35192533, 30463802, 48892201, 83948335, 49597667, 39847321, 92998591, 5073754, 29029316, 44245960, 37620363, 63152504, 99965001, 4508700, 41442762, 89811711, 77787724, 41481685, 68316156, 71920426, 24314885, 66428795, 76960303, 44479073, 87720882, 29401781, 9257405, 85571389, 83302115, 99524975, 43152977, 55247455, 20645197, 86242799, 4668450, 6819644, 56424103, 96193415, 9603598, 36930650, 77187825, 24168411, 8128637, 36396314, 82726008, 22129328, 4064751, 7502255, 40152546, 61897582, 14220886, 2917920, 62693428, 44627776, 36812683, 91281584, 18833224, 43506672, 74614639, 71083565, 39553046, 82339363, 24733232, 44842615, 23848565, 69136837, 83210802, 90310261, 76825057, 16405341, 14947650, 8634541, 13173644, 69579137, 47361209, 21533347, 19939935, 25842078, 71965942, 90013093, 43357947, 84840549, 61623915, 53084256, 10961421, 58224549, 36312813, 66045587, 4515343, 96420429, 7229550, 23569917, 9259676, 26392416, 749283, 44915235, 18466635, 34667286, 97011160, 94539824, 89996536, 69848388, 20681921, 53547802, 73392814, 69605283, 20486294, 35456853, 53648154, 22879907, 22942635, 70004753, 83789391, 40027975, 8729146, 64157906, 10649306, 13819358, 92554120, 62803858, 55850790, 17016156, 82979980, 43933006, 7300047, 4204661, 40197395, 11543098, 52293995, 85117092, 79880247, 30653863, 72614359, 78909561, 1239555, 38365584, 15535065, 29188588, 6793819, 16099750, 18411915, 99690194, 112651, 42692881, 12348118, 36808486, 79136082, 8791066, 71316369, 29466406, 55143724, 85711894, 82651278, 37659250, 33935899, 50188404, 8733068, 1375023, 24953498, 3294781, 20140249, 59501487, 99226875, 66663942, 77072625, 38256849, 17037369, 67513640, 39986008, 37192445, 45381876, 26292919, 77413012, 84127901, 44060493, 29819940, 46540998, 65851721, 92353856, 8696647, 68694897, 89637706, 44846932, 77300457, 43501211, 54517921, 45075471, 91240048, 35512853, 45407418, 20642888, 26102057, 79942022, 35996293, 91727510, 22435353, 33431960, 17539625, 21001913, 35092039, 90457870, 68102437, 99658235, 63372756, 65271999, 8055981, 9623492, 45237957, 44664587, 98943869, 75229462, 95010552, 70782102, 1788101, 9829782, 36189527, 17976208, 34493392, 30764367, 38008118, 55770687, 38658347, 44844121, 21673260, 12416768, 44177011, 89046466, 176257, 75135448, 19486173, 5482538, 78589145, 30090481, 92071990, 61263068, 89217461, 6111563, 3233569, 26275734, 66322959, 73617245, 55615885, 65081429, 2331773, 80555751, 36505482, 92541302, 34295794, 9188443, 59177718, 63967300, 66885828, 47708850, 93562257, 73109997, 6808825, 70541760, 51426311, 58751351, 39373729, 18131876, 24915585, 40781449, 4119747, 54232247, 69355476, 86798033, 63015256, 39201414, 73786944, 47298834, 64055960, 1431742, 98948034, 30218878, 97783876, 55669657, 86022504, 87710366, 36780454, 83269727, 89699445, 62428472, 56955985, 89078848, 17385531, 99224392, 47738185, 71657078, 94076128, 97057187, 72358170, 99125126, 19457589, 32161669, 61859581, 19376156, 23432750, 72725103, 76540481, 97940276, 2891150, 26493119, 78218845, 13231279, 88698958, 27185644, 16445503, 51715482, 93359396, 72019362, 26998766, 78785507, 54762643, 68702632, 79657802, 53842979, 508198, 30694952, 51315460, 75052463, 81274566, 89419466, 14093520, 14326617, 67084351, 64848072, 67451935, 9860195, 27325428, 59614746, 14723410, 45794415, 21070110, 40984766, 86460488, 15064655, 24826575, 13348726, 86301513, 5640302, 20765474, 1022677, 82052050, 66271566, 51149804, 42307449, 99729357, 84002370, 46851987, 62740044, 7955293, 29635537, 98371444, 18663507, 83747892, 56473732, 34044787, 43376279, 91664334, 77898274, 90004325, 28851716, 7687278, 65047700, 4069912, 40356877, 12024238, 56484900, 66319530, 61982238, 11757872, 33201905, 31727379, 45996863, 96709982, 1250437, 14349098, 48774913, 59318837, 73031054, 72278539, 44550764, 82897371, 45049308, 22405120, 83533741, 27411561, 4434662, 33797252, 49598724, 33336148, 57241521, 19891772, 75820087, 69641513, 96852321, 38510840, 22450468, 73235980, 38494874, 95290172, 36468541, 42237907, 84406788, 63628376, 51472275, 4978543, 11161731, 72495719, 81677380, 22997281, 15902805, 61928316, 47887470, 48088883, 57163802, 71469330, 23740167, 7066775, 47090124, 2607799, 7182642, 36135, 18806856, 73021291, 72357096, 41092102, 99333375, 53632373, 57240218, 17386996, 30771409, 54058788, 10453030, 9398733, 88904910, 36753250, 32274392, 5267545, 38645117, 90158785, 4806458, 18001617, 90272749, 57961282, 33304202, 3487592, 91141395, 34698428, 62115552, 26114953, 91831487, 48675329, 4199704, 65454636, 51590803, 3633375, 81898046, 38009615, 38061439, 30787683, 70727211, 12571310, 30569392, 38022834, 52060076, 90654836, 92692978, 82532312, 10264691, 51466049, 37435892, 247198, 6521313, 8115266, 17957593, 3880712, 9860968, 10358899, 17365188, 92867155, 2204165, 49641577, 27625735, 60393039, 31491938, 83083131, 79322415, 2717150, 92215320, 65880522, 16097038, 55189057, 76671482, 76703609, 61741594, 415901, 84187166, 64098930, 62490109, 72793444, 60955663 +36312813, 95251277, 66250369, 80014588, 4099191, 22113133, 4119747, 95726235, 32151165, 10366309, 76315420, 89499542, 33061250, 61373987, 37675718, 51149804, 67281495, 43376279, 76960303, 824872, 68939068, 26292919, 57803235, 6871053, 56515456, 29065964, 65038678, 91727510, 49882705, 75128745, 68702632, 15948937, 6153406, 89046466, 55753905, 64602895, 6111563, 26063929, 2331773, 26507214, 30463802, 73124510, 74110882, 65074535, 20002147, 99515901, 71657078, 247198, 168541, 30163921, 669105, 45075471, 44481640, 60176618, 83651211, 22405120, 79942022, 75543508, 68102437, 54199160, 42947632, 75229462, 83150534, 26397786, 28734791, 14626618, 54663246, 70596786, 99604946, 30395570, 31733363, 98648327, 39801351, 55850790, 23134715, 16097038, 37739481, 7685448, 27187213, 29029316, 37620363, 51464002, 18504197, 68644627, 59329511, 27041967, 49641577, 55247455, 40781854, 59501487, 34497327, 12664567, 53632373, 52261574, 94595668, 67793644, 11365791, 44846932, 88904910, 45049308, 43506672, 61960400, 92530431, 9599614, 59109400, 80316608, 8373289, 69579137, 19939935, 35092039, 93359396, 87160386, 6221471, 58224549, 73235980, 93270084, 66667729, 95581843, 75052463, 8651647, 47792865, 55470718, 21919959, 88444207, 17365188, 32159704, 65715134, 3633375, 65338021, 80246713, 1569515, 48360198, 78589145, 62552524, 75407347, 98739783, 38341669, 29932657, 60581278, 39171895, 42199455, 16019925, 13468268, 79880247, 84293052, 72614359, 36505482, 77620120, 15535065, 33249630, 12303248, 36808486, 55960386, 33237508, 5970607, 78766358, 85711894, 32058615, 1204161, 29994197, 94711842, 73021291, 99226875, 77187825, 11581181, 31727379, 45848907, 46870723, 4064751, 48774913, 5832946, 66832478, 86821229, 74743862, 56531125, 321665, 89637706, 94911072, 9886593, 16387575, 54517921, 15148031, 32161669, 5267545, 94516935, 66137019, 26139110, 90013093, 8099994, 16445503, 40865610, 48395186, 7104732, 44664587, 28796059, 30998561, 96735716, 82427263, 85023028, 61271144, 26392416, 37280276, 36189527, 44915235, 4978543, 51590803, 77272628, 15163258, 7919588, 61712234, 53393358, 73392814, 78884452, 69605283, 21070110, 40984766, 176257, 93790285, 37224844, 65880522, 76434144, 47213183, 30569392, 75777973, 61728685, 82886381, 29959549, 93933709, 94360702, 7300047, 48088883, 26275734, 89804152, 85117092, 84002370, 62740044, 36845587, 35192533, 19101477, 48892201, 1239555, 72738685, 9175338, 51047803, 71300104, 89214616, 70372191, 42692881, 81172706, 18783702, 7011964, 41442762, 16971929, 52204879, 31126490, 7517032, 52060076, 37659250, 82532312, 18699206, 7687278, 72777973, 83302115, 12024238, 24953498, 20645197, 4668450, 6819644, 98948034, 5822202, 36930650, 33201905, 17037369, 37192445, 15536795, 50702367, 17081350, 96709982, 17385531, 1250437, 40534591, 96318094, 99971982, 65851721, 73031054, 22721500, 2717150, 82897371, 65017137, 526217, 43501211, 31904591, 84349107, 69136837, 90158785, 23432750, 4787945, 35512853, 48673079, 85242963, 30139692, 80251430, 47361209, 21533347, 57961282, 3509435, 69641513, 96852321, 25842078, 83133790, 17894977, 72019362, 88251446, 99658235, 3487592, 74137926, 30694952, 51315460, 91831487, 60106217, 36933038, 1788101, 36580610, 91990218, 93515664, 59405277, 53802686, 62430984, 84166196, 34236719, 6525948, 17857111, 20836893, 32699744, 62232211, 44844121, 12416768, 24619760, 54263880, 70004753, 87598888, 8729146, 76170907, 12571310, 52613508, 69255765, 95395112, 92554120, 20765474, 62051033, 62803858, 17016156, 77312810, 86543538, 43933006, 98653983, 40197395, 34698463, 48260151, 54427233, 73617245, 44847298, 61741594, 91957544, 38365584, 37560164, 75986488, 57163802, 98371444, 63967300, 22766820, 77694700, 74724075, 71469330, 41245325, 7066775, 99965001, 56461322, 47090124, 77787724, 91664334, 55143724, 16054533, 57359924, 24915585, 54232247, 91938191, 19272365, 44479073, 99021067, 39201414, 73786944, 47298834, 99524975, 83083131, 14731700, 77072625, 9603598, 74452589, 54606384, 99333375, 50567636, 67513640, 8128637, 77413012, 40686254, 82726008, 79821897, 40152546, 34432810, 50806615, 53888755, 54987042, 4985896, 90090964, 69697787, 68694897, 62693428, 36812683, 19457589, 18833224, 71083565, 93566986, 26664538, 82327024, 23848565, 83210802, 76825057, 8115266, 99549373, 45407418, 97940276, 2891150, 49598724, 45617087, 97281847, 90272749, 44473167, 33431960, 81855445, 84684495, 98462867, 71965942, 21001913, 72274002, 84840549, 78785507, 75153252, 53084256, 88653118, 62357986, 28787861, 66045587, 49328608, 4515343, 57248122, 79922758, 3183975, 81805959, 33628349, 66903004, 70782102, 57393458, 84406788, 10358899, 9829782, 4199704, 6497830, 67451935, 17976208, 6157724, 92398073, 72495719, 22997281, 59614746, 85695762, 20486294, 81898046, 83789391, 40027975, 75135448, 5482538, 44784505, 92071990, 66116458, 8129978, 68875490, 10649306, 13819358, 31161687, 92867155, 73168565, 1022677, 569864, 60102965, 82052050, 52293995, 76703609, 51507425, 46851987, 4798568, 62936963, 83948335, 41380093, 92604458, 96906410, 54868730, 42073124, 66885828, 5073754, 44889423, 44245960, 47708850, 73109997, 79136082, 23740167, 86118021, 33699435, 17146629, 58751351, 39373729, 34044787, 71316369, 29834512, 18131876, 2607799, 88047921, 59910624, 81774825, 77898274, 4091162, 40781449, 90654836, 28851716, 72732205, 69355476, 86798033, 66428795, 63015256, 4069912, 74441448, 1431742, 66319530, 77284619, 62496012, 56424103, 11757872, 62762857, 38256849, 41092102, 37166608, 33565483, 56955985, 57240218, 17386996, 34946859, 45995325, 44983451, 14220886, 48893685, 9398733, 45306070, 67124282, 99125126, 36753250, 32274392, 24733232, 44842615, 93057697, 36139942, 90310261, 13470059, 4806458, 92787493, 17764950, 16405341, 9058407, 76540481, 70800879, 93053405, 4434662, 68041839, 26493119, 29516592, 94090109, 13173644, 26863229, 33304202, 90457870, 28928175, 28550822, 58749, 91141395, 34698428, 66611759, 54762643, 53842979, 33895336, 60430369, 55602660, 6038457, 78602717, 78549759, 38494874, 1936762, 81274566, 68816413, 59981773, 14326617, 67084351, 30366150, 13862149, 63628376, 49236559, 749283, 70036438, 15075176, 20885148, 32250045, 14723410, 69848388, 26734892, 67227442, 45794415, 19898053, 55770687, 38658347, 35456853, 21673260, 44177011, 59197747, 88130087, 13348726, 68000591, 22108665, 3235882, 86301513, 7423788, 65275241, 77301523, 89217461, 82979980, 76671482, 8913721, 84904436, 30653863, 78909561, 49597667, 23110625, 92541302, 39847321, 6793819, 85116755, 9188443, 18411915, 99690194, 59177718, 32426752, 112651, 7646095, 6808825, 51426311, 4508700, 38022834, 29466406, 74357852, 96726697, 20122224, 30811010, 71920426, 27665211, 92033260, 14363867, 9257405, 16567550, 8733068, 1375023, 64055960, 78300864, 67474219, 20140249, 57658654, 55669657, 45381876, 89078848, 6986898, 14349098, 21397057, 7502255, 10453030, 59318837, 10597197, 97057187, 44550764, 92353856, 2917920, 72358170, 61141874, 57020481, 79191827, 99917599, 68824981, 19376156, 40677414, 38645117, 41092172, 4095116, 26102057, 14947650, 59371804, 22435353, 8634541, 58208470, 92215320, 75820087, 86001008, 17068582, 26998766, 45237957, 42782652, 91802888, 62115552, 23569917, 20568363, 9860968, 95290172, 54014062, 22200378, 9860195, 18466635, 27325428, 97011160, 81677380, 30764367, 98130363, 1197320, 24591705, 15015906, 20681921, 24058273, 21993752, 22942635, 30891921, 62490109, 86460488, 38061439, 3773993, 30787683, 99861373, 79738755, 19486173, 64157906, 77377183, 45990383, 63059024, 42644903, 47887470, 61263068, 42967683, 4204661, 53666583, 17058722, 58180653, 30543215, 63506281, 55615885, 65081429, 80555751, 61815223, 92692283, 34295794, 45428665, 29188588, 29635537, 16099750, 69786756, 92998591, 37501808, 93562257, 6505939, 83747892, 99297164, 71333116, 14045383, 97379790, 16424199, 68316156, 90004325, 33935899, 65047700, 50188404, 50668599, 68204242, 29401781, 40356877, 51466049, 18806856, 60393039, 43152977, 56484900, 3294781, 72357096, 66663942, 61982238, 79322415, 89699445, 84187166, 26744917, 83155442, 22129328, 82779622, 46540998, 16380211, 97641116, 8696647, 5111370, 62452034, 77300457, 74614639, 39553046, 82339363, 61859581, 91240048, 72725103, 35996293, 45790169, 33797252, 18001617, 97561745, 57241521, 78218845, 44348328, 88698958, 27185644, 43357947, 63372756, 65271999, 9623492, 3088684, 79657802, 508198, 2208785, 26114953, 14093520, 47893286, 37957788, 10309525, 34667286, 34493392, 89996536, 38008118, 16595436, 73222868, 15902805, 70727211, 15064655, 30090481, 61928316, 78561158, 55189057, 66271566, 42307449, 874791, 7955293, 415901, 43322743, 12348118, 18663507, 63152504, 50007421, 25636669, 8791066, 36135, 82651278, 61859143, 49271185, 16684829, 56153345, 23194618, 72373496, 37435892, 86242799, 67030811, 30218878, 97783876, 24168411, 87710366, 36780454, 62428472, 76330843, 99224392, 44060493, 30771409, 37891451, 94076128, 61897582, 91281584, 83368048, 29510992, 13231279, 3233084, 38510840, 22450468, 61623915, 10961421, 11923835, 8055981, 26776922, 9259676, 98943869, 64848072, 51472275, 48675329, 68128438, 81853704, 53648154, 22879907, 38009615, 64087743, 67031644, 9575954, 32590267, 48658605, 70367851, 70541760, 41481685, 92692978, 24314885, 72238278, 85571389, 45919976, 10264691, 96193415, 86022504, 84127901, 17727650, 54058788, 91255408, 6521313, 27411561, 33336148, 45667668, 19891772, 96420429, 7229550, 95010552, 42237907, 65454636, 11161731, 94539824, 53547802, 68110330, 20867149, 24826575, 43045786, 21289531, 33553959, 3233569, 72793444, 11543098, 66322959, 56473732, 7182642, 27625735, 79806380, 87720882, 60955663, 70420215, 83269727, 39986008, 29819940, 72278539, 44627776, 92803766, 17539625, 3880712, 89419466, 36468541, 90061527, 8807940, 5640302, 33123618, 37183543, 71522968, 2204165, 89811711, 31491938, 45996863, 47738185, 51588015, 20642888, 17957593, 64098930, 82178706, 36685023, 95957797, 99729357, 36396314, 51715482, 83533741 +27411561, 22113133, 97379790, 44915235, 4978543, 53547802, 55753905, 92554120, 16097038, 89804152, 41245325, 55143724, 99971982, 65851721, 26664538, 92787493, 17068582, 35456853, 7955293, 36808486, 66663942, 6986898, 17385531, 82897371, 71083565, 38645117, 33797252, 38510840, 73235980, 21919959, 82886381, 21289531, 37675718, 55615885, 41380093, 72738685, 32426752, 71469330, 39373729, 89811711, 7182642, 52060076, 40781449, 66428795, 50668599, 72777973, 77072625, 86022504, 89699445, 26744917, 30771409, 71657078, 69697787, 29065964, 54517921, 2891150, 45617087, 94090109, 47361209, 3509435, 96852321, 26863229, 19939935, 3487592, 45237957, 93270084, 33895336, 91990218, 30366150, 54014062, 51472275, 15075176, 8807940, 40984766, 30395570, 30787683, 75135448, 30090481, 78561158, 68875490, 61263068, 99729357, 91957544, 92692283, 6808825, 58751351, 34044787, 59329511, 96726697, 4091162, 4069912, 18806856, 91255408, 22721500, 6521313, 89637706, 45075471, 83368048, 93566986, 22405120, 4095116, 26493119, 18001617, 8634541, 44348328, 51715482, 99658235, 63372756, 58224549, 96420429, 78602717, 66903004, 9860968, 93515664, 6157724, 38008118, 81853704, 33061250, 64157906, 22108665, 61928316, 62552524, 66116458, 7423788, 82979980, 93933709, 7300047, 60102965, 63506281, 79880247, 32590267, 37560164, 34295794, 85116755, 77694700, 51464002, 7066775, 4099191, 4508700, 49641577, 32058615, 77898274, 90004325, 71920426, 24314885, 19272365, 63015256, 76960303, 72238278, 9257405, 73786944, 24953498, 74441448, 14731700, 98948034, 62496012, 56424103, 79322415, 41092102, 17037369, 56955985, 84127901, 57240218, 17727650, 59318837, 94076128, 50806615, 30163921, 66832478, 14220886, 62452034, 61960400, 92530431, 9599614, 83210802, 17764950, 35996293, 14947650, 97561745, 90272749, 29510992, 58208470, 25842078, 90457870, 93359396, 87160386, 72019362, 84840549, 40865610, 11923835, 66667729, 3183975, 51315460, 98943869, 95290172, 36933038, 88444207, 37280276, 22200378, 749283, 48675329, 67451935, 59405277, 10309525, 53802686, 62430984, 67227442, 24058273, 78884452, 38658347, 82178706, 176257, 93790285, 64602895, 5482538, 68000591, 69255765, 63059024, 60581278, 55189057, 40197395, 11543098, 51149804, 73617245, 30653863, 65081429, 62740044, 72614359, 44847298, 36505482, 19101477, 39847321, 16099750, 7685448, 99690194, 70372191, 12348118, 29029316, 44889423, 81172706, 37501808, 6505939, 83747892, 18783702, 50007421, 56461322, 25636669, 41442762, 5970607, 29466406, 18131876, 91664334, 14045383, 28851716, 27625735, 49271185, 92033260, 14363867, 10264691, 51466049, 60393039, 56484900, 60955663, 77284619, 59501487, 70420215, 97783876, 55669657, 24168411, 87710366, 83269727, 84187166, 39986008, 45381876, 77413012, 99515901, 4064751, 54058788, 10453030, 53888755, 44983451, 90090964, 2917920, 5111370, 56531125, 65017137, 88904910, 84349107, 76825057, 4787945, 4806458, 59371804, 68041839, 45667668, 97281847, 8373289, 13231279, 69641513, 84684495, 72274002, 76315420, 58749, 54762643, 7104732, 3088684, 66250369, 36312813, 95581843, 26776922, 96735716, 42782652, 55602660, 23569917, 81274566, 89419466, 75229462, 37957788, 90061527, 94539824, 32159704, 30764367, 20836893, 24591705, 65715134, 3633375, 45794415, 55770687, 73222868, 62232211, 21673260, 24826575, 65880522, 44784505, 92071990, 75407347, 8129978, 13819358, 42967683, 6111563, 94360702, 48088883, 53666583, 98653983, 36685023, 13468268, 54427233, 80014588, 415901, 83948335, 38365584, 92541302, 15535065, 45428665, 57163802, 71300104, 89214616, 92604458, 42073124, 12303248, 27187213, 73109997, 55960386, 99965001, 8791066, 33237508, 78766358, 31126490, 61859143, 57359924, 74110882, 99021067, 72373496, 83302115, 37435892, 64055960, 40781854, 67474219, 72357096, 96193415, 11581181, 31727379, 37192445, 12664567, 82726008, 47738185, 29819940, 34946859, 45995325, 247198, 94595668, 61897582, 669105, 72278539, 44550764, 48893685, 62693428, 72358170, 61141874, 36753250, 526217, 77300457, 18833224, 82339363, 82327024, 10366309, 36139942, 40677414, 8115266, 23432750, 41092172, 48673079, 83533741, 79942022, 91727510, 33336148, 44473167, 92215320, 57961282, 86001008, 27185644, 33304202, 17894977, 49882705, 28928175, 28550822, 10961421, 91141395, 28787861, 2208785, 91802888, 62115552, 74137926, 9259676, 26392416, 67084351, 36580610, 49236559, 64848072, 47893286, 4199704, 17976208, 9860195, 15948937, 17365188, 15163258, 34236719, 6525948, 14723410, 26734892, 32699744, 65338021, 6153406, 53393358, 19898053, 73392814, 22942635, 38009615, 24619760, 20867149, 70727211, 99861373, 12571310, 88130087, 76434144, 45990383, 3235882, 86301513, 42644903, 65275241, 92867155, 61728685, 55850790, 17016156, 569864, 4204661, 58180653, 30543215, 76671482, 42307449, 48260151, 26063929, 16019925, 66322959, 26507214, 36845587, 23110625, 75986488, 29188588, 29635537, 69786756, 63967300, 43322743, 42692881, 5073754, 74724075, 70367851, 2204165, 63152504, 70541760, 47090124, 99297164, 77787724, 16054533, 16424199, 7517032, 82651278, 20122224, 72732205, 37659250, 92692978, 18699206, 68204242, 29994197, 39201414, 31491938, 1431742, 78300864, 20002147, 73021291, 3294781, 11757872, 62762857, 74452589, 36780454, 33201905, 67513640, 45848907, 36396314, 1250437, 99224392, 44060493, 79821897, 6871053, 40152546, 97057187, 97641116, 4985896, 74743862, 68694897, 321665, 94911072, 99125126, 45049308, 16387575, 65038678, 79191827, 31904591, 61859581, 44842615, 69136837, 83651211, 45407418, 9058407, 22435353, 45790169, 93053405, 29516592, 13173644, 88698958, 17957593, 43357947, 22450468, 65271999, 66611759, 88653118, 44664587, 508198, 79922758, 6038457, 30694952, 20568363, 1936762, 68816413, 60106217, 42947632, 47792865, 14326617, 26397786, 57393458, 84406788, 70036438, 6497830, 11161731, 92398073, 34667286, 77272628, 34493392, 64098930, 98130363, 17857111, 20681921, 69605283, 81898046, 62490109, 68110330, 86460488, 3773993, 54263880, 1569515, 70004753, 15064655, 8729146, 76170907, 59197747, 48360198, 78589145, 9575954, 98739783, 31161687, 62803858, 29932657, 89217461, 37183543, 52293995, 8913721, 84293052, 4798568, 80555751, 35192533, 49597667, 77620120, 9188443, 18411915, 98371444, 96906410, 54868730, 59177718, 22766820, 47708850, 7646095, 79136082, 86118021, 71316369, 68644627, 71333116, 16971929, 52204879, 24915585, 30811010, 65047700, 87720882, 23194618, 40356877, 8733068, 67030811, 30218878, 20140249, 824872, 5822202, 34497327, 36930650, 38256849, 37166608, 68939068, 50567636, 45996863, 8128637, 17386996, 83155442, 76330843, 22129328, 14349098, 21397057, 7502255, 96318094, 37891451, 10597197, 54987042, 67124282, 56515456, 44627776, 36812683, 68824981, 44481640, 32161669, 23848565, 92803766, 13470059, 51588015, 20642888, 76540481, 4434662, 57241521, 33431960, 17539625, 75820087, 71965942, 90013093, 16445503, 78785507, 68102437, 75128745, 34698428, 6221471, 54199160, 3880712, 79657802, 66045587, 38494874, 8651647, 14093520, 95010552, 36468541, 42237907, 13862149, 63628376, 22997281, 68128438, 89996536, 59614746, 84166196, 54663246, 69848388, 1197320, 16595436, 21993752, 21070110, 53648154, 22879907, 80246713, 15902805, 12416768, 44177011, 61373987, 87598888, 40027975, 13348726, 30569392, 5640302, 10649306, 39801351, 95395112, 20765474, 38341669, 62051033, 75777973, 29959549, 1022677, 77312810, 33553959, 23134715, 43933006, 3233569, 17058722, 66271566, 85117092, 76703609, 51507425, 874791, 84904436, 84002370, 78909561, 37739481, 61815223, 73124510, 6793819, 92998591, 71522968, 18663507, 37620363, 56473732, 33699435, 17146629, 38022834, 74357852, 81774825, 68316156, 27665211, 33935899, 1204161, 86798033, 44479073, 50188404, 45919976, 95726235, 94711842, 12024238, 55247455, 83083131, 86242799, 4668450, 66319530, 99226875, 57658654, 77187825, 33565483, 15536795, 17081350, 46870723, 46540998, 40534591, 67793644, 168541, 86821229, 2717150, 8696647, 9886593, 91281584, 95251277, 43506672, 43501211, 74614639, 39553046, 15148031, 59109400, 60176618, 90158785, 99549373, 97940276, 26139110, 98462867, 21001913, 26998766, 61623915, 75153252, 53084256, 48395186, 8055981, 9623492, 68702632, 53842979, 30998561, 7229550, 91831487, 83150534, 61271144, 1788101, 10358899, 9829782, 36189527, 28734791, 20885148, 72495719, 97011160, 7919588, 89499542, 85695762, 30891921, 44844121, 79738755, 37224844, 98648327, 47887470, 39171895, 72793444, 42199455, 2331773, 48892201, 48658605, 112651, 23740167, 51426311, 95957797, 29834512, 27041967, 43376279, 88047921, 59910624, 90654836, 54232247, 69355476, 79806380, 65074535, 16684829, 29401781, 47298834, 99524975, 6819644, 61982238, 99333375, 26292919, 50702367, 40686254, 96709982, 52261574, 48774913, 82779622, 5832946, 16380211, 34432810, 73031054, 92353856, 19457589, 32274392, 93057697, 5267545, 19376156, 91240048, 90310261, 72725103, 16405341, 26102057, 85242963, 94516935, 30139692, 80251430, 78218845, 19891772, 69579137, 21533347, 35092039, 88251446, 62357986, 28796059, 4515343, 60430369, 57248122, 81805959, 26114953, 33628349, 85023028, 70782102, 18466635, 32250045, 27325428, 14626618, 61712234, 15015906, 99604946, 19486173, 52613508, 47213183, 43045786, 33123618, 86543538, 82052050, 34698463, 46851987, 61741594, 9175338, 51047803, 33249630, 66885828, 44245960, 18504197, 67281495, 2607799, 36135, 7687278, 56153345, 85571389, 1375023, 43152977, 20645197, 9603598, 62428472, 53632373, 57803235, 11365791, 9398733, 45306070, 44846932, 99917599, 24733232, 35512853, 66137019, 75543508, 3233084, 49328608, 82427263, 75052463, 55470718, 20486294, 64087743, 38061439, 89046466, 83789391, 31733363, 67031644, 77377183, 73168565, 77301523, 26275734, 1239555, 93562257, 85711894, 4119747, 91938191, 16567550, 54606384, 89078848, 32151165, 57020481, 81855445, 83133790, 59981773, 65454636, 70596786, 62936963, 7011964, 82532312, 8099994, 78549759, 41481685, 70800879, 80316608, 51590803, 81677380, 30463802, 49598724 +65338021, 25842078, 26392416, 17976208, 16097038, 84187166, 65851721, 68694897, 27185644, 93933709, 45381876, 89078848, 33304202, 9259676, 42237907, 63628376, 33061250, 87598888, 61263068, 51149804, 18504197, 99965001, 79806380, 24314885, 18806856, 70420215, 77072625, 46540998, 4985896, 82897371, 45049308, 93566986, 3509435, 19939935, 7104732, 73235980, 1936762, 91831487, 20885148, 34667286, 38008118, 78884452, 44844121, 75135448, 30090481, 26275734, 66271566, 54427233, 38365584, 85116755, 59177718, 18663507, 7011964, 33699435, 43376279, 27625735, 14363867, 51466049, 99226875, 15536795, 10366309, 19376156, 99549373, 75543508, 49598724, 88698958, 3233084, 49882705, 53084256, 91141395, 48395186, 54762643, 68702632, 54199160, 30998561, 82427263, 2208785, 79922758, 7229550, 81805959, 26397786, 36580610, 64848072, 70036438, 14626618, 24591705, 15902805, 12416768, 8729146, 12571310, 5482538, 95395112, 86543538, 82052050, 42307449, 44847298, 32590267, 92541302, 70367851, 37620363, 73109997, 51426311, 4508700, 99297164, 40781449, 30811010, 28851716, 69355476, 27665211, 92033260, 63015256, 56153345, 60955663, 4668450, 96193415, 62762857, 55669657, 36780454, 12664567, 26292919, 6986898, 17385531, 32151165, 247198, 99971982, 321665, 89637706, 88904910, 36812683, 43506672, 44842615, 90310261, 76825057, 45617087, 97561745, 63372756, 93270084, 8651647, 4978543, 90061527, 72495719, 22997281, 89996536, 64098930, 73392814, 69605283, 21993752, 40984766, 68110330, 1569515, 31733363, 65880522, 62552524, 44784505, 30569392, 66116458, 5640302, 92867155, 60581278, 17016156, 569864, 82979980, 3233569, 76671482, 85117092, 76703609, 51507425, 79880247, 62740044, 61815223, 83948335, 34295794, 69786756, 63967300, 74724075, 2204165, 37501808, 83747892, 71316369, 68644627, 38022834, 71333116, 27041967, 20122224, 4119747, 49271185, 1204161, 10264691, 16567550, 95726235, 12024238, 31491938, 43152977, 37435892, 14731700, 98948034, 9603598, 54606384, 31727379, 68939068, 50567636, 57803235, 17386996, 82726008, 76330843, 14349098, 40534591, 10453030, 67793644, 669105, 65017137, 45075471, 82339363, 32161669, 93057697, 83210802, 13470059, 35512853, 51588015, 9058407, 94516935, 97940276, 2891150, 33797252, 29516592, 57241521, 47361209, 81855445, 44348328, 98462867, 26863229, 21001913, 51715482, 22450468, 26998766, 78785507, 10961421, 62357986, 8055981, 9623492, 28787861, 66045587, 508198, 62115552, 78602717, 78549759, 20568363, 75052463, 66903004, 59981773, 42947632, 14326617, 83150534, 36933038, 91990218, 47893286, 6157724, 65454636, 53802686, 97011160, 94539824, 15015906, 20681921, 80246713, 86460488, 64087743, 54263880, 59197747, 64602895, 67031644, 98648327, 22108665, 9575954, 78561158, 7423788, 10649306, 42644903, 47887470, 29959549, 77312810, 94360702, 60102965, 72793444, 58180653, 30543215, 36685023, 11543098, 99729357, 80014588, 73617245, 2331773, 4798568, 72614359, 36845587, 62936963, 7955293, 19101477, 92692283, 9175338, 92998591, 43322743, 93562257, 51464002, 6808825, 56473732, 47090124, 17146629, 22113133, 59329511, 95957797, 52204879, 18131876, 78766358, 31126490, 16054533, 41481685, 36135, 7517032, 57359924, 90004325, 24915585, 72732205, 37659250, 92692978, 86798033, 65074535, 16684829, 9257405, 85571389, 73786944, 8733068, 60393039, 73021291, 61982238, 74452589, 24168411, 99333375, 56955985, 53632373, 22129328, 48774913, 21397057, 54058788, 59318837, 45995325, 40152546, 94076128, 34432810, 97641116, 44983451, 22721500, 2717150, 48893685, 69697787, 9398733, 56515456, 62693428, 91281584, 95251277, 18833224, 39553046, 92530431, 44481640, 5267545, 60176618, 23432750, 72725103, 17764950, 20642888, 4095116, 14947650, 33336148, 8373289, 33431960, 29510992, 21533347, 86001008, 17957593, 71965942, 35092039, 17068582, 76315420, 87160386, 28928175, 3487592, 45237957, 96735716, 57248122, 3183975, 81274566, 60106217, 89419466, 95290172, 13862149, 10358899, 9829782, 4199704, 36189527, 93515664, 51590803, 17365188, 68128438, 62430984, 32159704, 30764367, 16595436, 53393358, 85695762, 30891921, 21673260, 38061439, 79738755, 15064655, 19486173, 48360198, 61928316, 39801351, 75777973, 43933006, 7300047, 42199455, 34698463, 26063929, 874791, 65081429, 84002370, 78909561, 36505482, 30463802, 73124510, 16099750, 18411915, 92604458, 32426752, 12303248, 44245960, 71522968, 63152504, 67281495, 7066775, 4099191, 25636669, 77787724, 16971929, 55143724, 16424199, 82532312, 18699206, 91938191, 87720882, 56484900, 20645197, 64055960, 83083131, 6819644, 59501487, 5822202, 36930650, 79322415, 38256849, 87710366, 39986008, 45996863, 17081350, 99224392, 30771409, 71657078, 37891451, 94595668, 72278539, 6521313, 44550764, 74743862, 92353856, 56531125, 99125126, 9886593, 44627776, 526217, 43501211, 74614639, 54517921, 79191827, 99917599, 15148031, 23848565, 84349107, 59109400, 90158785, 4806458, 22405120, 48673079, 26102057, 91727510, 26139110, 59371804, 22435353, 4434662, 26493119, 13173644, 69579137, 58208470, 57961282, 83133790, 90013093, 8099994, 16445503, 93359396, 84840549, 34698428, 88653118, 58224549, 66250369, 28796059, 79657802, 42782652, 60430369, 55602660, 74137926, 98943869, 21919959, 70782102, 84406788, 51472275, 6497830, 11161731, 77272628, 81677380, 34493392, 70596786, 38658347, 73222868, 8807940, 3773993, 89046466, 83789391, 40027975, 24826575, 64157906, 77377183, 45990383, 86301513, 8129978, 68875490, 62051033, 62803858, 61728685, 33123618, 89217461, 6111563, 39171895, 55189057, 16019925, 46851987, 80555751, 61741594, 1239555, 37560164, 41380093, 23110625, 75986488, 39847321, 48658605, 51047803, 89214616, 54868730, 99690194, 42692881, 12348118, 29029316, 81172706, 71469330, 70541760, 23740167, 39373729, 29466406, 2607799, 88047921, 7182642, 49641577, 68316156, 74110882, 44479073, 23194618, 72373496, 29994197, 39201414, 99524975, 1431742, 67474219, 30218878, 824872, 34497327, 37166608, 33201905, 83269727, 62428472, 11581181, 33565483, 67513640, 84127901, 1250437, 99515901, 4064751, 16380211, 10597197, 97057187, 53888755, 8696647, 2917920, 62452034, 29065964, 16387575, 77300457, 71083565, 32274392, 68824981, 61859581, 69136837, 16405341, 76540481, 70800879, 85242963, 80251430, 19891772, 92215320, 75820087, 17894977, 43357947, 72019362, 88251446, 66667729, 36312813, 53842979, 30694952, 33628349, 68816413, 61271144, 57393458, 30366150, 22200378, 48675329, 37957788, 9860195, 59405277, 92398073, 15948937, 32250045, 6525948, 14723410, 98130363, 1197320, 67227442, 3633375, 45794415, 55770687, 35456853, 82178706, 81898046, 55753905, 13348726, 47213183, 92071990, 69255765, 63059024, 55850790, 29932657, 48088883, 4204661, 8913721, 13468268, 66322959, 30653863, 48892201, 415901, 6793819, 57163802, 98371444, 96906410, 70372191, 22766820, 66885828, 5073754, 77694700, 27187213, 44889423, 56461322, 41442762, 58751351, 74357852, 14045383, 97379790, 32058615, 81774825, 82651278, 90654836, 76960303, 29401781, 72238278, 86242799, 40781854, 66319530, 62496012, 66663942, 11757872, 97783876, 77187825, 26744917, 37192445, 96709982, 52261574, 47738185, 44060493, 34946859, 79821897, 61897582, 54987042, 91255408, 14220886, 94911072, 44846932, 19457589, 83368048, 26664538, 36139942, 91240048, 92803766, 92787493, 41092172, 83533741, 66137019, 93053405, 68041839, 30139692, 69641513, 84684495, 72274002, 68102437, 99658235, 61623915, 75153252, 44664587, 3880712, 26776922, 91802888, 23569917, 55470718, 75229462, 95010552, 1788101, 37280276, 749283, 28734791, 10309525, 27325428, 59614746, 84166196, 69848388, 7919588, 61712234, 32699744, 6153406, 20486294, 53648154, 22879907, 62232211, 62490109, 30395570, 30787683, 20867149, 99861373, 88130087, 68000591, 3235882, 43045786, 65275241, 38341669, 31161687, 73168565, 82886381, 1022677, 42967683, 17058722, 37739481, 35192533, 91957544, 49597667, 15535065, 29635537, 9188443, 7685448, 112651, 47708850, 36808486, 86118021, 50007421, 33237508, 29834512, 91664334, 59910624, 77898274, 61859143, 52060076, 33935899, 66428795, 4069912, 99021067, 67030811, 77284619, 3294781, 20140249, 45848907, 57240218, 40686254, 83155442, 46870723, 17727650, 5832946, 7502255, 73031054, 66832478, 86821229, 90090964, 67124282, 72358170, 65038678, 61960400, 82327024, 9599614, 40677414, 38645117, 83651211, 4787945, 45407418, 45790169, 97281847, 18001617, 44473167, 78218845, 17539625, 90457870, 40865610, 75128745, 49328608, 6038457, 26114953, 36468541, 54014062, 44915235, 67451935, 15075176, 18466635, 54663246, 17857111, 20836893, 81853704, 89499542, 99604946, 24619760, 61373987, 70727211, 76170907, 78589145, 52613508, 75407347, 13819358, 92554120, 20765474, 37675718, 37183543, 89804152, 98653983, 55615885, 26507214, 77620120, 72738685, 45428665, 71300104, 33249630, 7646095, 79136082, 18783702, 8791066, 85711894, 7687278, 65047700, 50188404, 50668599, 72777973, 45919976, 83302115, 94711842, 55247455, 74441448, 78300864, 20002147, 57658654, 41092102, 8128637, 77413012, 50702367, 82779622, 96318094, 6871053, 30163921, 11365791, 5111370, 57020481, 36753250, 8115266, 79942022, 35996293, 45667668, 13231279, 11923835, 3088684, 33895336, 51315460, 38494874, 9860968, 14093520, 47792865, 88444207, 67084351, 26734892, 53547802, 76434144, 98739783, 21289531, 40197395, 52293995, 34044787, 4091162, 19272365, 47298834, 1375023, 24953498, 72357096, 86022504, 17037369, 168541, 50806615, 61141874, 27411561, 80316608, 90272749, 96852321, 38510840, 28550822, 58749, 66611759, 95581843, 4515343, 96420429, 85023028, 49236559, 34236719, 24058273, 21070110, 22942635, 44177011, 70004753, 176257, 37224844, 77301523, 33553959, 53666583, 48260151, 63506281, 84293052, 84904436, 29188588, 42073124, 41245325, 55960386, 89811711, 5970607, 54232247, 71920426, 89699445, 36396314, 45306070, 31904591, 24733232, 8634541, 94090109, 65271999, 6221471, 15163258, 65715134, 19898053, 38009615, 93790285, 56424103, 29819940, 23134715, 6505939, 40356877, 96726697, 68204242 +21993752, 82779622, 41092172, 75153252, 39801351, 80014588, 27041967, 74110882, 91281584, 28796059, 17976208, 20885148, 45990383, 75407347, 92867155, 17058722, 37560164, 37192445, 57803235, 14349098, 97057187, 45049308, 71083565, 26139110, 97561745, 25842078, 93270084, 53842979, 47893286, 77272628, 24591705, 54263880, 37224844, 29932657, 89804152, 52293995, 84293052, 59177718, 12348118, 44245960, 47708850, 70367851, 50007421, 16971929, 96726697, 61859143, 79806380, 24314885, 87720882, 23194618, 85571389, 33201905, 15536795, 36396314, 40534591, 54058788, 50806615, 61897582, 86821229, 8696647, 9398733, 32161669, 20642888, 22435353, 96852321, 3233084, 35092039, 68102437, 7104732, 3880712, 2208785, 78549759, 75052463, 60106217, 47792865, 61271144, 97011160, 94539824, 32159704, 6153406, 38658347, 73222868, 3773993, 93790285, 64602895, 22108665, 38341669, 86543538, 94360702, 8913721, 48260151, 84002370, 61815223, 38365584, 98371444, 42692881, 56473732, 71316369, 16054533, 32058615, 7517032, 65047700, 94711842, 67474219, 30218878, 6986898, 50702367, 99224392, 21397057, 34432810, 67793644, 97641116, 11365791, 62693428, 16387575, 74614639, 72725103, 4095116, 94516935, 91727510, 19891772, 19939935, 83133790, 51715482, 84840549, 10961421, 28787861, 26114953, 30694952, 36189527, 9860195, 10309525, 15948937, 81677380, 38008118, 26734892, 21070110, 61373987, 19486173, 44784505, 69255765, 62051033, 60581278, 16019925, 46851987, 62936963, 48892201, 7685448, 73109997, 63152504, 79136082, 6808825, 86118021, 56461322, 39373729, 68644627, 24915585, 28851716, 82532312, 33935899, 50188404, 99021067, 68204242, 72373496, 45919976, 20645197, 1431742, 67030811, 62496012, 96193415, 24168411, 16380211, 53888755, 22721500, 45075471, 31904591, 68824981, 36139942, 69136837, 83210802, 90310261, 60176618, 83533741, 75543508, 45790169, 4434662, 29516592, 57241521, 13173644, 17539625, 21533347, 98462867, 71965942, 28928175, 62357986, 79922758, 23569917, 33628349, 42947632, 6497830, 7919588, 20836893, 45794415, 85695762, 40984766, 64087743, 30395570, 44177011, 30787683, 64157906, 67031644, 68875490, 42644903, 73168565, 82979980, 43933006, 36685023, 11543098, 76703609, 13468268, 66322959, 874791, 4798568, 78909561, 91957544, 77620120, 34295794, 15535065, 72738685, 45428665, 39847321, 69786756, 27187213, 71522968, 51464002, 23740167, 7066775, 58751351, 33237508, 88047921, 30811010, 27665211, 7687278, 65074535, 29401781, 40356877, 74441448, 77284619, 73021291, 37166608, 62428472, 68939068, 26744917, 77413012, 17081350, 29819940, 5832946, 34946859, 6871053, 72278539, 14220886, 90090964, 321665, 72358170, 61141874, 36812683, 36753250, 65038678, 82339363, 15148031, 82327024, 84349107, 92803766, 40677414, 97940276, 97281847, 8373289, 80251430, 44348328, 84684495, 17068582, 93359396, 72019362, 26998766, 61623915, 11923835, 68702632, 54199160, 30998561, 42782652, 60430369, 7229550, 78602717, 81805959, 9860968, 14093520, 70782102, 36580610, 37280276, 65454636, 27325428, 90061527, 51590803, 72495719, 84166196, 64098930, 69848388, 61712234, 20681921, 22942635, 99604946, 33061250, 176257, 83789391, 99861373, 76170907, 88130087, 13348726, 98648327, 77377183, 62552524, 52613508, 78561158, 30569392, 8129978, 13819358, 92554120, 55850790, 33123618, 21289531, 77312810, 39171895, 3233569, 66271566, 34698463, 73617245, 2331773, 26507214, 62740044, 80555751, 37739481, 30463802, 61741594, 92541302, 54868730, 42073124, 92998591, 33249630, 22766820, 77694700, 18663507, 41245325, 70541760, 89811711, 95957797, 52204879, 59910624, 77898274, 82651278, 40781449, 69355476, 66428795, 63015256, 56153345, 31491938, 56484900, 55247455, 6819644, 20140249, 66663942, 61982238, 11757872, 79322415, 74452589, 55669657, 54606384, 45848907, 26292919, 96709982, 76330843, 22129328, 44060493, 71657078, 79821897, 37891451, 99971982, 73031054, 30163921, 54987042, 6521313, 92353856, 48893685, 2917920, 45306070, 29065964, 95251277, 18833224, 43506672, 93566986, 61859581, 44842615, 23848565, 35512853, 92787493, 76540481, 26102057, 79942022, 68041839, 33431960, 47361209, 17957593, 27185644, 43357947, 76315420, 78785507, 49882705, 75128745, 65271999, 95581843, 26776922, 49328608, 508198, 96420429, 81274566, 89419466, 88444207, 26397786, 91990218, 54014062, 13862149, 10358899, 9829782, 15075176, 32250045, 14626618, 15163258, 89996536, 30764367, 59614746, 34236719, 16595436, 24058273, 89499542, 20486294, 53648154, 12416768, 86460488, 89046466, 1569515, 70727211, 40027975, 15064655, 75135448, 55753905, 76434144, 98739783, 20765474, 65275241, 31161687, 77301523, 23134715, 60102965, 53666583, 26275734, 98653983, 42307449, 99729357, 79880247, 36845587, 7955293, 1239555, 49597667, 23110625, 75986488, 6793819, 16099750, 89214616, 43322743, 81172706, 7646095, 93562257, 6505939, 67281495, 33699435, 99297164, 71333116, 97379790, 16424199, 36135, 18699206, 49271185, 1204161, 4069912, 44479073, 50668599, 29994197, 16567550, 8733068, 1375023, 37435892, 64055960, 83083131, 86242799, 98948034, 72357096, 57658654, 62762857, 11581181, 67513640, 8128637, 53632373, 17386996, 99515901, 17727650, 48774913, 10453030, 45995325, 94595668, 65851721, 168541, 66832478, 91255408, 44550764, 74743862, 69697787, 68694897, 5111370, 62452034, 19457589, 57020481, 526217, 61960400, 24733232, 93057697, 19376156, 91240048, 90158785, 4787945, 16405341, 48673079, 14947650, 66137019, 80316608, 49598724, 90272749, 8634541, 29510992, 81855445, 75820087, 26863229, 33304202, 90013093, 8099994, 16445503, 87160386, 3487592, 58749, 6221471, 88653118, 73235980, 9623492, 45237957, 3088684, 66250369, 91802888, 3183975, 51315460, 59981773, 14326617, 1788101, 49236559, 64848072, 28734791, 6157724, 11161731, 53802686, 34667286, 17365188, 22997281, 68128438, 54663246, 3633375, 53393358, 22879907, 44844121, 87598888, 8729146, 9575954, 3235882, 66116458, 7423788, 5640302, 95395112, 75777973, 61728685, 1022677, 37183543, 7300047, 4204661, 42199455, 40197395, 63506281, 51507425, 84904436, 35192533, 415901, 85116755, 48658605, 92604458, 96906410, 63967300, 66885828, 51426311, 99965001, 18783702, 47090124, 25636669, 8791066, 59329511, 41481685, 52060076, 57359924, 72732205, 37659250, 92033260, 14363867, 18806856, 12024238, 47298834, 99524975, 43152977, 24953498, 20002147, 59501487, 70420215, 36930650, 87710366, 36780454, 31727379, 39986008, 12664567, 83155442, 52261574, 46870723, 4064751, 7502255, 46540998, 32151165, 10597197, 44983451, 67124282, 56531125, 94911072, 65017137, 9886593, 39553046, 83368048, 44481640, 26664538, 59109400, 23432750, 83651211, 99549373, 4806458, 85242963, 27411561, 35996293, 59371804, 33797252, 33336148, 45667668, 45617087, 18001617, 30139692, 44473167, 69579137, 58208470, 88698958, 86001008, 21001913, 72274002, 40865610, 53084256, 91141395, 54762643, 8055981, 66667729, 79657802, 57248122, 62115552, 9259676, 20568363, 38494874, 8651647, 98943869, 1936762, 91831487, 95010552, 36468541, 22200378, 37957788, 92398073, 34493392, 6525948, 17857111, 81853704, 65338021, 81898046, 30891921, 15902805, 68110330, 31733363, 79738755, 12571310, 59197747, 48360198, 61928316, 47213183, 43045786, 10649306, 62803858, 29959549, 61263068, 569864, 42967683, 6111563, 48088883, 55189057, 85117092, 26063929, 65081429, 44847298, 19101477, 32590267, 73124510, 41380093, 92692283, 71300104, 5073754, 44889423, 71469330, 37620363, 36808486, 7011964, 5970607, 38022834, 29466406, 43376279, 91664334, 85711894, 14045383, 68316156, 90004325, 20122224, 90654836, 4119747, 91938191, 86798033, 72238278, 9257405, 10264691, 95726235, 51466049, 60393039, 60955663, 824872, 56424103, 34497327, 9603598, 99333375, 17037369, 50567636, 40686254, 17385531, 1250437, 82726008, 2717150, 82897371, 56515456, 44846932, 88904910, 44627776, 54517921, 79191827, 92530431, 10366309, 38645117, 76825057, 13470059, 22405120, 9058407, 70800879, 2891150, 94090109, 3509435, 38510840, 17894977, 88251446, 99658235, 28550822, 66611759, 36312813, 33895336, 55602660, 74137926, 66903004, 85023028, 68816413, 75229462, 83150534, 21919959, 26392416, 42237907, 84406788, 63628376, 749283, 51472275, 48675329, 4199704, 44915235, 59405277, 18466635, 62430984, 1197320, 67227442, 32699744, 53547802, 70596786, 73392814, 69605283, 35456853, 62232211, 62490109, 38061439, 24619760, 5482538, 78589145, 68000591, 82886381, 47887470, 17016156, 37675718, 30543215, 54427233, 55615885, 30653863, 36505482, 83948335, 29188588, 57163802, 18411915, 99690194, 70372191, 12303248, 74724075, 29029316, 18504197, 4099191, 17146629, 77787724, 29834512, 2607799, 74357852, 7182642, 55143724, 54232247, 92692978, 71920426, 76960303, 16684829, 72777973, 73786944, 40781854, 14731700, 66319530, 3294781, 97783876, 77187825, 86022504, 89699445, 33565483, 45381876, 47738185, 59318837, 40152546, 247198, 669105, 4985896, 32274392, 9599614, 51588015, 26493119, 78218845, 13231279, 57961282, 34698428, 82427263, 55470718, 95290172, 67084351, 57393458, 30366150, 65715134, 19898053, 21673260, 80246713, 8807940, 20867149, 86301513, 63059024, 33553959, 72793444, 82052050, 29635537, 9175338, 9188443, 37501808, 4508700, 34044787, 22113133, 18131876, 31126490, 4091162, 19272365, 39201414, 78300864, 41092102, 83269727, 84127901, 96318094, 94076128, 99125126, 43501211, 99917599, 69641513, 90457870, 63372756, 48395186, 44664587, 66045587, 6038457, 36933038, 70036438, 93515664, 4978543, 14723410, 82178706, 38009615, 24826575, 65880522, 30090481, 89217461, 93933709, 32426752, 112651, 55960386, 41442762, 49641577, 81774825, 83302115, 77072625, 38256849, 84187166, 89078848, 30771409, 77300457, 17764950, 92215320, 96735716, 4515343, 67451935, 98130363, 15015906, 55770687, 16097038, 58180653, 76671482, 51149804, 51047803, 4668450, 5822202, 99226875, 93053405, 58224549, 70004753, 92071990, 27625735, 57240218, 89637706, 5267545, 8115266, 78884452, 72614359, 83747892, 78766358, 56955985, 45996863, 45407418, 22450468, 2204165 +88130087, 65851721, 61741594, 1239555, 4119747, 94516935, 7104732, 9860195, 93790285, 43045786, 29635537, 168541, 321665, 43506672, 24733232, 97940276, 80251430, 82427263, 33628349, 65454636, 7919588, 89046466, 6111563, 43322743, 93562257, 67281495, 85711894, 52060076, 85571389, 36930650, 89078848, 17081350, 34946859, 50806615, 91255408, 77300457, 79191827, 93566986, 48673079, 70800879, 30139692, 58208470, 21001913, 63372756, 78549759, 6157724, 20885148, 20681921, 38061439, 48360198, 44784505, 43933006, 55189057, 37739481, 54868730, 99690194, 47708850, 56473732, 30811010, 29401781, 62762857, 56955985, 17727650, 7502255, 46540998, 40534591, 94595668, 11365791, 45306070, 56531125, 89637706, 9886593, 91281584, 45049308, 26664538, 97561745, 86001008, 93359396, 88251446, 11923835, 45237957, 53842979, 42782652, 57248122, 68816413, 26392416, 28734791, 11161731, 92398073, 17365188, 84166196, 19898053, 69605283, 35456853, 53648154, 62490109, 38009615, 24826575, 65880522, 64157906, 67031644, 3235882, 98739783, 39801351, 55850790, 1022677, 34698463, 48260151, 80014588, 84904436, 91957544, 41380093, 89214616, 96906410, 44245960, 18663507, 89811711, 68644627, 38022834, 92033260, 63015256, 14363867, 16684829, 45919976, 1375023, 37435892, 45996863, 57240218, 10453030, 53888755, 74743862, 8696647, 67124282, 5267545, 84349107, 91240048, 90310261, 76825057, 72725103, 16405341, 26139110, 97281847, 35092039, 68102437, 95581843, 79657802, 75052463, 14326617, 1788101, 64848072, 70036438, 4199704, 97011160, 81677380, 22997281, 61712234, 65715134, 89499542, 21673260, 3773993, 70004753, 15064655, 30090481, 98648327, 30569392, 62051033, 75777973, 60581278, 82886381, 47887470, 33553959, 86543538, 89804152, 30543215, 51507425, 874791, 2331773, 80555751, 19101477, 38365584, 75986488, 69786756, 112651, 66885828, 51464002, 99965001, 4099191, 56461322, 8791066, 27041967, 41481685, 36135, 68316156, 82651278, 90004325, 28851716, 74110882, 24314885, 18699206, 72777973, 29994197, 16567550, 94711842, 1431742, 30218878, 59501487, 99226875, 96193415, 33201905, 26292919, 57803235, 59318837, 66832478, 669105, 4985896, 9398733, 99125126, 61960400, 54517921, 92530431, 93057697, 83210802, 40677414, 13470059, 80316608, 68041839, 26493119, 33431960, 3509435, 3233084, 78785507, 65271999, 54762643, 73235980, 3088684, 66667729, 66045587, 55602660, 26114953, 23569917, 38494874, 14093520, 21919959, 88444207, 37280276, 9829782, 749283, 44915235, 37957788, 34667286, 27325428, 54663246, 24591705, 15015906, 70596786, 44844121, 33061250, 24619760, 83789391, 99861373, 40027975, 8729146, 55753905, 22108665, 45990383, 52613508, 92554120, 65275241, 92867155, 77301523, 569864, 89217461, 98653983, 42199455, 82052050, 36685023, 54427233, 55615885, 65081429, 44847298, 61815223, 48892201, 73124510, 77620120, 34295794, 9175338, 18411915, 7685448, 81172706, 2204165, 36808486, 18783702, 33699435, 18131876, 59910624, 16424199, 81774825, 61859143, 57359924, 20122224, 90654836, 86798033, 65074535, 60955663, 14731700, 73021291, 62428472, 33565483, 17037369, 15536795, 36396314, 53632373, 40686254, 83155442, 96709982, 48774913, 97641116, 61897582, 82897371, 48893685, 62693428, 44846932, 88904910, 61141874, 44627776, 43501211, 74614639, 15148031, 9599614, 23848565, 59109400, 8115266, 90158785, 75543508, 44473167, 13231279, 57961282, 75820087, 44348328, 83133790, 72274002, 72019362, 84840549, 28928175, 40865610, 10961421, 66611759, 48395186, 62357986, 8055981, 66250369, 68702632, 96735716, 508198, 2208785, 98943869, 9860968, 55470718, 95010552, 61271144, 36933038, 67084351, 42237907, 57393458, 54014062, 13862149, 84406788, 49236559, 93515664, 15075176, 38008118, 64098930, 69848388, 16595436, 55770687, 21070110, 20486294, 80246713, 15902805, 40984766, 30787683, 1569515, 20867149, 76170907, 64602895, 76434144, 77377183, 8129978, 7423788, 13819358, 38341669, 29959549, 77312810, 82979980, 93933709, 16097038, 94360702, 48088883, 4204661, 58180653, 76671482, 66271566, 42307449, 76703609, 99729357, 26063929, 73617245, 30653863, 62936963, 39847321, 98371444, 70372191, 5073754, 29029316, 70367851, 7646095, 37620363, 41245325, 50007421, 34044787, 22113133, 59329511, 5970607, 29466406, 52204879, 43376279, 78766358, 7182642, 16054533, 77898274, 37659250, 79806380, 1204161, 65047700, 66428795, 50188404, 39201414, 73786944, 83302115, 60393039, 99524975, 55247455, 64055960, 66319530, 67474219, 98948034, 824872, 56424103, 79322415, 41092102, 54606384, 89699445, 31727379, 67513640, 37192445, 8128637, 84127901, 17385531, 14349098, 82779622, 21397057, 29819940, 30771409, 32151165, 45995325, 94076128, 34432810, 22721500, 2717150, 2917920, 5111370, 19457589, 95251277, 65038678, 71083565, 32161669, 92803766, 23432750, 4787945, 35512853, 4806458, 22405120, 9058407, 26102057, 59371804, 45790169, 33797252, 45667668, 18001617, 90272749, 19891772, 17539625, 81855445, 21533347, 98462867, 25842078, 17957593, 27185644, 33304202, 17894977, 43357947, 22450468, 99658235, 91141395, 6221471, 9623492, 36312813, 3880712, 3183975, 30694952, 8651647, 60106217, 47792865, 30366150, 63628376, 48675329, 53802686, 51590803, 15163258, 1197320, 20836893, 62232211, 81898046, 44177011, 61373987, 37224844, 19486173, 78561158, 86301513, 63059024, 73168565, 29932657, 33123618, 17016156, 60102965, 3233569, 53666583, 8913721, 63506281, 66322959, 79880247, 84293052, 26507214, 62740044, 78909561, 92692283, 23110625, 15535065, 85116755, 9188443, 16099750, 48658605, 51047803, 74724075, 63152504, 79136082, 18504197, 55960386, 86118021, 4508700, 99297164, 41442762, 58751351, 95957797, 2607799, 96726697, 14045383, 7517032, 4091162, 24915585, 69355476, 12024238, 56484900, 24953498, 78300864, 86242799, 4668450, 67030811, 77284619, 20140249, 66663942, 61982238, 9603598, 11757872, 97783876, 55669657, 24168411, 36780454, 83269727, 45848907, 12664567, 77413012, 76330843, 47738185, 5832946, 96318094, 40152546, 247198, 73031054, 86821229, 44983451, 90090964, 44550764, 56515456, 62452034, 65017137, 18833224, 32274392, 45075471, 44481640, 44842615, 10366309, 69136837, 83651211, 92787493, 83533741, 27411561, 66137019, 93053405, 49598724, 33336148, 29516592, 8373289, 8634541, 94090109, 13173644, 92215320, 69641513, 71965942, 76315420, 51715482, 87160386, 28550822, 61623915, 58224549, 44664587, 54199160, 4515343, 7229550, 51315460, 1936762, 66903004, 91831487, 85023028, 81274566, 75229462, 36468541, 22200378, 51472275, 36189527, 67451935, 4978543, 17976208, 59405277, 15948937, 34493392, 94539824, 34236719, 6525948, 98130363, 26734892, 67227442, 32699744, 24058273, 65338021, 6153406, 45794415, 73392814, 78884452, 21993752, 22879907, 22942635, 12416768, 86460488, 64087743, 176257, 31733363, 13348726, 61928316, 95395112, 62803858, 61263068, 37183543, 72793444, 51149804, 84002370, 72614359, 36505482, 30463802, 415901, 49597667, 92541302, 29188588, 71300104, 59177718, 42073124, 63967300, 22766820, 27187213, 12348118, 51426311, 39373729, 33237508, 29834512, 16971929, 88047921, 55143724, 97379790, 27665211, 4069912, 44479073, 95726235, 51466049, 8733068, 47298834, 62496012, 5822202, 34497327, 38256849, 26744917, 45381876, 52261574, 82726008, 4064751, 44060493, 71657078, 79821897, 30163921, 72278539, 92353856, 68694897, 72358170, 526217, 83368048, 31904591, 82339363, 68824981, 36139942, 38645117, 60176618, 51588015, 17764950, 20642888, 4095116, 76540481, 85242963, 79942022, 35996293, 2891150, 22435353, 57241521, 29510992, 47361209, 84684495, 88698958, 96852321, 17068582, 26998766, 49882705, 34698428, 88653118, 28787861, 28796059, 30998561, 60430369, 91802888, 6038457, 78602717, 81805959, 89419466, 95290172, 70782102, 26397786, 36580610, 91990218, 10309525, 18466635, 72495719, 77272628, 68128438, 32159704, 30764367, 59614746, 81853704, 3633375, 53393358, 85695762, 38658347, 73222868, 99604946, 54263880, 78589145, 62552524, 66116458, 68875490, 37675718, 23134715, 39171895, 26275734, 17058722, 11543098, 13468268, 35192533, 83948335, 72738685, 45428665, 92604458, 92998591, 33249630, 42692881, 71522968, 6808825, 7066775, 17146629, 71333116, 31126490, 32058615, 27625735, 71920426, 82532312, 91938191, 76960303, 50668599, 9257405, 72373496, 10264691, 18806856, 31491938, 20645197, 74441448, 6819644, 3294781, 72357096, 57658654, 74452589, 86022504, 11581181, 99333375, 84187166, 17386996, 1250437, 46870723, 99224392, 37891451, 97057187, 6521313, 69697787, 16387575, 99917599, 82327024, 61859581, 41092172, 14947650, 91727510, 4434662, 45617087, 26863229, 19939935, 8099994, 90457870, 3487592, 58749, 33895336, 26776922, 49328608, 79922758, 20568363, 10358899, 14626618, 62430984, 17857111, 8807940, 30395570, 79738755, 75135448, 59197747, 5482538, 47213183, 92071990, 69255765, 5640302, 20765474, 42967683, 40197395, 52293995, 16019925, 46851987, 4798568, 37560164, 32426752, 44889423, 6505939, 23740167, 7011964, 71316369, 74357852, 49641577, 40781449, 54232247, 49271185, 99021067, 87720882, 68204242, 23194618, 72238278, 40356877, 77072625, 77187825, 87710366, 39986008, 6986898, 50702367, 54058788, 16380211, 10597197, 99971982, 14220886, 94911072, 36753250, 39553046, 19376156, 99549373, 38510840, 90013093, 75153252, 93270084, 96420429, 9259676, 59981773, 42947632, 47893286, 6497830, 53547802, 82178706, 30891921, 70727211, 12571310, 9575954, 10649306, 42644903, 31161687, 21289531, 7300047, 85117092, 36845587, 32590267, 37501808, 83747892, 70541760, 47090124, 25636669, 92692978, 33935899, 7687278, 19272365, 56153345, 43152977, 20002147, 68939068, 22129328, 54987042, 29065964, 57020481, 78218845, 62115552, 32250045, 90061527, 89996536, 14723410, 68110330, 87598888, 68000591, 75407347, 57163802, 77694700, 71469330, 73109997, 83083131, 40781854, 70420215, 99515901, 6871053, 36812683, 69579137, 16445503, 53084256, 75128745, 74137926, 61728685, 7955293, 12303248, 77787724, 72732205, 37166608, 50567636, 67793644, 45407418, 83150534, 6793819, 91664334 +19939935, 67513640, 97561745, 73235980, 43933006, 39171895, 97783876, 45075471, 78549759, 15948937, 39801351, 38341669, 92541302, 56461322, 29819940, 7502255, 669105, 11365791, 19457589, 43506672, 91240048, 13470059, 4806458, 17539625, 25842078, 8099994, 30998561, 78602717, 1788101, 36189527, 69605283, 85695762, 33061250, 3773993, 99861373, 8729146, 69255765, 65275241, 82886381, 77312810, 40197395, 79880247, 84293052, 26507214, 91957544, 67281495, 18783702, 8791066, 16971929, 97379790, 7517032, 64055960, 40781854, 37192445, 40686254, 17081350, 99224392, 45995325, 69697787, 91727510, 19891772, 51715482, 99658235, 55602660, 13862149, 4978543, 7919588, 64602895, 45990383, 29932657, 4798568, 36845587, 7955293, 44889423, 51464002, 50007421, 25636669, 71316369, 59910624, 16424199, 49641577, 77898274, 30811010, 28851716, 76960303, 87720882, 85571389, 77284619, 98948034, 3294781, 17037369, 57240218, 17386996, 99515901, 46540998, 94076128, 30163921, 61897582, 82897371, 61960400, 44842615, 23848565, 83210802, 22405120, 33336148, 66250369, 28787861, 74137926, 23569917, 98943869, 85023028, 68816413, 26392416, 49236559, 37280276, 72495719, 89996536, 26734892, 20486294, 8807940, 59197747, 65880522, 78589145, 95395112, 21289531, 58180653, 13468268, 36505482, 35192533, 92692283, 77620120, 72738685, 43322743, 81172706, 71469330, 93562257, 6505939, 17146629, 99297164, 38022834, 2607799, 37659250, 82532312, 18699206, 1204161, 65074535, 4069912, 8733068, 37435892, 60955663, 4668450, 72357096, 62428472, 56955985, 26292919, 57803235, 76330843, 59318837, 54987042, 8696647, 68694897, 62693428, 9886593, 16387575, 71083565, 44481640, 15148031, 23432750, 99549373, 49598724, 57241521, 94090109, 98462867, 96852321, 38510840, 17894977, 16445503, 68102437, 28928175, 11923835, 91141395, 42782652, 60430369, 1936762, 66903004, 21919959, 91990218, 84406788, 70036438, 93515664, 9860195, 92398073, 97011160, 62430984, 30764367, 6153406, 45794415, 19898053, 22942635, 64087743, 99604946, 54263880, 61373987, 20867149, 87598888, 5482538, 88130087, 13348726, 22108665, 52613508, 30569392, 98739783, 62051033, 75777973, 61728685, 94360702, 48088883, 26275734, 51149804, 85117092, 76703609, 874791, 55615885, 62740044, 48892201, 38365584, 32590267, 73124510, 41380093, 85116755, 18411915, 70372191, 69786756, 27187213, 29029316, 47708850, 71522968, 55960386, 99965001, 4099191, 39373729, 68644627, 43376279, 96726697, 31126490, 52060076, 65047700, 44479073, 72777973, 43152977, 6819644, 99226875, 34497327, 36930650, 5832946, 40152546, 34432810, 66832478, 53888755, 2717150, 321665, 65017137, 29065964, 36753250, 54517921, 32274392, 10366309, 69136837, 40677414, 4787945, 35512853, 17764950, 48673079, 76540481, 27411561, 66137019, 26139110, 22435353, 97281847, 30139692, 8373289, 33431960, 80251430, 81855445, 21533347, 92215320, 17068582, 76315420, 72019362, 78785507, 88251446, 75128745, 63372756, 8055981, 45237957, 93270084, 81805959, 30694952, 51315460, 20568363, 81274566, 75229462, 95290172, 36468541, 70782102, 26397786, 42237907, 36580610, 4199704, 67451935, 65454636, 32250045, 90061527, 77272628, 81677380, 59614746, 64098930, 16595436, 65338021, 73392814, 53648154, 81898046, 1569515, 83789391, 76434144, 67031644, 77377183, 62552524, 47213183, 78561158, 92071990, 42644903, 62803858, 17016156, 29959549, 33553959, 82979980, 23134715, 6111563, 17058722, 30543215, 36685023, 11543098, 99729357, 54427233, 84002370, 78909561, 61815223, 29188588, 6793819, 16099750, 59177718, 42073124, 112651, 12303248, 66885828, 12348118, 2204165, 37620363, 37501808, 36808486, 79136082, 56473732, 41442762, 33237508, 29834512, 16054533, 68316156, 20122224, 71920426, 79806380, 49271185, 56153345, 99021067, 72238278, 29994197, 95726235, 51466049, 12024238, 60393039, 1375023, 99524975, 1431742, 83083131, 86242799, 20002147, 66319530, 73021291, 55669657, 99333375, 45848907, 8128637, 50702367, 83155442, 22129328, 46870723, 17727650, 21397057, 54058788, 79821897, 99971982, 73031054, 97641116, 86821229, 72278539, 90090964, 44550764, 74743862, 94911072, 62452034, 44846932, 44627776, 57020481, 45049308, 526217, 65038678, 74614639, 92530431, 93566986, 68824981, 26664538, 32161669, 19376156, 84349107, 8115266, 90158785, 9058407, 26102057, 85242963, 79942022, 14947650, 4434662, 26493119, 75820087, 69641513, 84684495, 26863229, 71965942, 72274002, 93359396, 87160386, 61623915, 75153252, 34698428, 54762643, 28796059, 66045587, 82427263, 2208785, 89419466, 36933038, 63628376, 64848072, 9829782, 47893286, 28734791, 37957788, 59405277, 22997281, 34493392, 94539824, 67227442, 53547802, 70596786, 78884452, 22879907, 44844121, 80246713, 38061439, 31733363, 93790285, 37224844, 15064655, 12571310, 19486173, 24826575, 30090481, 61928316, 75407347, 7423788, 5640302, 68875490, 10649306, 92554120, 20765474, 31161687, 92867155, 77301523, 37675718, 569864, 60102965, 3233569, 98653983, 42199455, 55189057, 8913721, 34698463, 48260151, 16019925, 73617245, 65081429, 72614359, 44847298, 80555751, 37739481, 19101477, 415901, 34295794, 98371444, 32426752, 22766820, 42692881, 7646095, 73109997, 18504197, 41245325, 7066775, 71333116, 27041967, 52204879, 91664334, 85711894, 14045383, 41481685, 81774825, 40781449, 27625735, 4119747, 54232247, 27665211, 86798033, 23194618, 29401781, 39201414, 73786944, 40356877, 10264691, 47298834, 24953498, 55247455, 67474219, 62496012, 59501487, 79322415, 38256849, 41092102, 54606384, 24168411, 36780454, 31727379, 26744917, 45996863, 89078848, 36396314, 82726008, 14349098, 97057187, 168541, 44983451, 4985896, 6521313, 9398733, 56515456, 89637706, 39553046, 99917599, 24733232, 9599614, 36139942, 90310261, 60176618, 51588015, 4095116, 35996293, 45790169, 45667668, 44473167, 13231279, 3509435, 27185644, 33304202, 35092039, 84840549, 49882705, 40865610, 66611759, 48395186, 58224549, 36312813, 95581843, 79657802, 26776922, 49328608, 57248122, 91802888, 33628349, 8651647, 9860968, 60106217, 57393458, 54014062, 10358899, 22200378, 6497830, 44915235, 10309525, 53802686, 18466635, 34667286, 84166196, 34236719, 14723410, 98130363, 1197320, 24591705, 55770687, 73222868, 21673260, 62490109, 15902805, 12416768, 30395570, 70004753, 176257, 70727211, 79738755, 76170907, 64157906, 3235882, 66116458, 73168565, 16097038, 4204661, 53666583, 89804152, 26063929, 66322959, 80014588, 84904436, 46851987, 1239555, 83948335, 49597667, 75986488, 39847321, 48658605, 7685448, 51047803, 71300104, 92604458, 96906410, 54868730, 99690194, 63967300, 92998591, 77694700, 70367851, 63152504, 23740167, 22113133, 29466406, 74357852, 88047921, 55143724, 32058615, 82651278, 61859143, 4091162, 92692978, 74110882, 24314885, 19272365, 66428795, 68204242, 18806856, 20140249, 9603598, 74452589, 87710366, 11581181, 53632373, 17385531, 96318094, 247198, 10597197, 94595668, 50806615, 22721500, 92353856, 5111370, 72358170, 88904910, 95251277, 43501211, 31904591, 93057697, 5267545, 38645117, 59109400, 76825057, 83651211, 70800879, 83533741, 97940276, 80316608, 33797252, 90272749, 8634541, 69579137, 47361209, 57961282, 44348328, 86001008, 3233084, 83133790, 21001913, 90013093, 26998766, 28550822, 53084256, 58749, 10961421, 88653118, 68702632, 54199160, 3880712, 508198, 7229550, 6038457, 62115552, 38494874, 91831487, 55470718, 14326617, 88444207, 30366150, 51472275, 48675329, 15075176, 6157724, 20885148, 51590803, 6525948, 17857111, 61712234, 81853704, 53393358, 21070110, 86460488, 24619760, 89046466, 75135448, 98648327, 44784505, 60581278, 47887470, 1022677, 89217461, 37183543, 66271566, 52293995, 51507425, 30653863, 30463802, 23110625, 15535065, 45428665, 29635537, 33249630, 6808825, 51426311, 47090124, 58751351, 34044787, 89811711, 59329511, 95957797, 18131876, 78766358, 57359924, 90004325, 72732205, 69355476, 63015256, 50188404, 9257405, 83302115, 20645197, 14731700, 30218878, 5822202, 62762857, 37166608, 33201905, 68939068, 45381876, 15536795, 84127901, 52261574, 4064751, 47738185, 44060493, 30771409, 34946859, 40534591, 6871053, 48893685, 45306070, 61141874, 91281584, 77300457, 18833224, 61859581, 72725103, 92787493, 20642888, 68041839, 18001617, 88698958, 17957593, 90457870, 7104732, 9623492, 53842979, 33895336, 96735716, 96420429, 3183975, 75052463, 14093520, 47792865, 17976208, 11161731, 54663246, 69848388, 3633375, 32699744, 21993752, 35456853, 62232211, 38009615, 48360198, 68000591, 9575954, 86301513, 63059024, 43045786, 61263068, 86543538, 82052050, 61741594, 37560164, 57163802, 89214616, 5073754, 74724075, 18663507, 83747892, 70541760, 86118021, 4508700, 5970607, 24915585, 33935899, 7687278, 14363867, 16684829, 45919976, 16567550, 94711842, 56484900, 56424103, 96193415, 77072625, 11757872, 77187825, 39986008, 6986898, 1250437, 82779622, 71657078, 65851721, 67793644, 91255408, 14220886, 2917920, 56531125, 36812683, 83368048, 79191827, 92803766, 16405341, 94516935, 93053405, 29516592, 13173644, 78218845, 29510992, 22450468, 44664587, 3088684, 66667729, 79922758, 9259676, 42947632, 61271144, 14626618, 68128438, 32159704, 38008118, 65715134, 38658347, 13819358, 33123618, 93933709, 42967683, 72793444, 76671482, 2331773, 62936963, 44245960, 7011964, 33699435, 36135, 92033260, 72373496, 31491938, 74441448, 824872, 66663942, 86022504, 84187166, 12664567, 96709982, 99125126, 82339363, 45407418, 41092172, 2891150, 59371804, 43357947, 3487592, 65271999, 62357986, 4515343, 26114953, 83150534, 95010552, 67084351, 749283, 17365188, 24058273, 89499542, 30891921, 68110330, 55753905, 8129978, 55850790, 7300047, 42307449, 63506281, 9175338, 9188443, 90654836, 91938191, 50668599, 67030811, 57658654, 83269727, 89699445, 50567636, 77413012, 48774913, 32151165, 37891451, 67124282, 82327024, 75543508, 45617087, 58208470, 6221471, 59981773, 27325428, 15163258, 20836893, 20681921, 82178706, 40984766, 44177011, 30787683, 40027975, 77787724, 7182642, 78300864, 70420215, 33565483, 10453030, 16380211, 15015906, 61982238 +65715134, 82886381, 29188588, 42782652, 88444207, 824872, 34946859, 90090964, 8373289, 69605283, 569864, 63506281, 98371444, 63152504, 72777973, 73021291, 62496012, 7502255, 65851721, 14220886, 74743862, 44473167, 84684495, 28928175, 6038457, 38494874, 95290172, 93515664, 89499542, 35456853, 44177011, 16097038, 42199455, 44245960, 18663507, 4099191, 52060076, 4119747, 74110882, 24314885, 65074535, 83302115, 30218878, 56424103, 17037369, 89078848, 94595668, 50806615, 56531125, 24733232, 93057697, 29510992, 81855445, 88251446, 58749, 7104732, 4515343, 61271144, 70036438, 9860195, 11161731, 3633375, 99861373, 15064655, 66116458, 86301513, 98739783, 34698463, 2331773, 48892201, 1239555, 16099750, 89214616, 81172706, 70367851, 41245325, 7066775, 18783702, 86118021, 25636669, 58751351, 16054533, 77284619, 5822202, 61982238, 79322415, 86022504, 62428472, 17081350, 10453030, 44550764, 56515456, 91281584, 99917599, 10366309, 83651211, 41092172, 48673079, 26102057, 79942022, 35996293, 59371804, 33797252, 13231279, 72274002, 17068582, 93359396, 87160386, 72019362, 11923835, 58224549, 73235980, 66250369, 36312813, 53842979, 30694952, 57393458, 10358899, 51590803, 6525948, 69848388, 32699744, 21993752, 38009615, 86460488, 24619760, 83789391, 93790285, 67031644, 44784505, 75407347, 63059024, 43045786, 92867155, 33553959, 93933709, 11543098, 84904436, 46851987, 37739481, 61741594, 48658605, 77694700, 5970607, 38022834, 55143724, 41481685, 57359924, 27665211, 18806856, 60393039, 78300864, 66319530, 66663942, 74452589, 24168411, 56955985, 45381876, 57803235, 17727650, 30771409, 46540998, 32151165, 68694897, 9886593, 61141874, 32274392, 92530431, 82339363, 9599614, 69136837, 90310261, 97940276, 45790169, 93053405, 45617087, 97281847, 94090109, 58208470, 26863229, 83133790, 75128745, 3487592, 91141395, 63372756, 6221471, 55602660, 7229550, 83150534, 42237907, 92398073, 17365188, 62430984, 15163258, 34236719, 16595436, 19898053, 73392814, 38658347, 62232211, 40984766, 20867149, 8729146, 55753905, 65880522, 64602895, 68000591, 9575954, 95395112, 13819358, 92554120, 60581278, 82979980, 66271566, 8913721, 44847298, 34295794, 29635537, 9188443, 92604458, 36808486, 6808825, 8791066, 17146629, 41442762, 33237508, 77787724, 43376279, 36135, 68316156, 90004325, 54232247, 71920426, 65047700, 44479073, 50668599, 29994197, 94711842, 56484900, 77072625, 11757872, 62762857, 41092102, 83269727, 8128637, 50702367, 96709982, 99224392, 44060493, 79821897, 99971982, 66832478, 91255408, 4985896, 8696647, 45306070, 67124282, 65017137, 77300457, 65038678, 61960400, 54517921, 45075471, 79191827, 44481640, 82327024, 91240048, 83210802, 40677414, 76825057, 8115266, 72725103, 45407418, 92787493, 66137019, 75543508, 68041839, 33431960, 80251430, 90013093, 35092039, 26998766, 78785507, 49882705, 10961421, 65271999, 8055981, 45237957, 95581843, 82427263, 96420429, 26114953, 23569917, 33628349, 75052463, 9860968, 68816413, 14093520, 55470718, 1788101, 67084351, 22200378, 28734791, 20885148, 18466635, 97011160, 81677380, 22997281, 94539824, 84166196, 98130363, 7919588, 20836893, 61712234, 15015906, 81853704, 45794415, 82178706, 62490109, 33061250, 3773993, 30395570, 54263880, 79738755, 75135448, 24826575, 5482538, 64157906, 98648327, 22108665, 61928316, 92071990, 30569392, 7423788, 42644903, 38341669, 62051033, 55850790, 42967683, 6111563, 60102965, 53666583, 89804152, 55189057, 40197395, 52293995, 48260151, 16019925, 13468268, 66322959, 80555751, 36505482, 61815223, 91957544, 37560164, 15535065, 57163802, 18411915, 51047803, 96906410, 54868730, 70372191, 42073124, 63967300, 66885828, 29029316, 47708850, 7646095, 37620363, 55960386, 50007421, 56473732, 34044787, 89811711, 59329511, 31126490, 32058615, 7517032, 24915585, 92692978, 91938191, 50188404, 87720882, 39201414, 45919976, 1375023, 55247455, 20645197, 64055960, 67474219, 98948034, 3294781, 20140249, 33201905, 33565483, 45996863, 17385531, 82726008, 22129328, 29819940, 96318094, 6871053, 37891451, 247198, 54987042, 669105, 86821229, 22721500, 82897371, 92353856, 48893685, 9398733, 62693428, 89637706, 44846932, 44627776, 71083565, 31904591, 68824981, 13470059, 35512853, 51588015, 76540481, 70800879, 94516935, 14947650, 91727510, 2891150, 4434662, 49598724, 26493119, 97561745, 30139692, 69579137, 21001913, 22450468, 40865610, 28550822, 34698428, 88653118, 93270084, 3088684, 54199160, 30998561, 79922758, 1936762, 59981773, 47792865, 14326617, 95010552, 26397786, 36580610, 91990218, 84406788, 49236559, 64848072, 749283, 51472275, 4978543, 53802686, 34493392, 32159704, 26734892, 67227442, 53393358, 85695762, 20486294, 53648154, 73222868, 99604946, 38061439, 70004753, 31733363, 37224844, 19486173, 30090481, 68875490, 61728685, 29932657, 47887470, 29959549, 37183543, 86543538, 7300047, 82052050, 99729357, 54427233, 84002370, 26507214, 62936963, 78909561, 83948335, 49597667, 41380093, 77620120, 39847321, 7685448, 32426752, 5073754, 12348118, 44889423, 93562257, 67281495, 51426311, 99965001, 7011964, 33699435, 99297164, 22113133, 68644627, 29834512, 16971929, 52204879, 96726697, 78766358, 88047921, 85711894, 16424199, 49641577, 82651278, 4091162, 40781449, 90654836, 18699206, 7687278, 86798033, 66428795, 92033260, 63015256, 56153345, 68204242, 29401781, 72238278, 16567550, 95726235, 47298834, 14731700, 34497327, 54606384, 89699445, 31727379, 37192445, 26292919, 36396314, 40686254, 83155442, 99515901, 76330843, 4064751, 48774913, 59318837, 40152546, 168541, 44983451, 321665, 57020481, 526217, 95251277, 43506672, 93566986, 15148031, 32161669, 23848565, 38645117, 23432750, 17764950, 27411561, 22435353, 90272749, 57961282, 69641513, 98462867, 96852321, 86001008, 3233084, 71965942, 33304202, 76315420, 84840549, 68102437, 61623915, 66611759, 54762643, 9623492, 68702632, 79657802, 49328608, 96735716, 91802888, 3183975, 78602717, 78549759, 8651647, 91831487, 81274566, 42947632, 36933038, 30366150, 47893286, 59405277, 6157724, 15948937, 27325428, 14626618, 38008118, 20681921, 65338021, 55770687, 22879907, 81898046, 80246713, 12416768, 64087743, 30787683, 1569515, 48360198, 88130087, 76434144, 62552524, 3235882, 5640302, 10649306, 39801351, 65275241, 62803858, 33123618, 77301523, 77312810, 94360702, 43933006, 48088883, 98653983, 36685023, 26063929, 874791, 80014588, 79880247, 73617245, 55615885, 30653863, 65081429, 35192533, 30463802, 59177718, 74724075, 71469330, 37501808, 73109997, 51464002, 83747892, 39373729, 71316369, 95957797, 29466406, 2607799, 74357852, 61859143, 30811010, 28851716, 19272365, 14363867, 23194618, 72373496, 40356877, 37435892, 83083131, 86242799, 6819644, 59501487, 96193415, 36930650, 55669657, 38256849, 68939068, 12664567, 77413012, 53632373, 57240218, 1250437, 14349098, 5832946, 40534591, 45995325, 72278539, 11365791, 62452034, 72358170, 16387575, 43501211, 26664538, 5267545, 19376156, 84349107, 92803766, 59109400, 16405341, 85242963, 83533741, 26139110, 45667668, 18001617, 29516592, 8634541, 13173644, 19891772, 17539625, 21533347, 17957593, 38510840, 75153252, 44664587, 60430369, 62115552, 51315460, 85023028, 89419466, 21919959, 70782102, 26392416, 54014062, 4199704, 15075176, 65454636, 10309525, 14723410, 21070110, 22942635, 30891921, 21673260, 89046466, 59197747, 13348726, 77377183, 45990383, 47213183, 69255765, 21289531, 61263068, 37675718, 89217461, 23134715, 39171895, 26275734, 72793444, 58180653, 30543215, 51149804, 42307449, 62740044, 38365584, 32590267, 73124510, 92541302, 85116755, 99690194, 33249630, 12303248, 43322743, 42692881, 27187213, 79136082, 70541760, 56461322, 4508700, 71333116, 59910624, 77898274, 20122224, 37659250, 79806380, 33935899, 1204161, 4069912, 16684829, 85571389, 10264691, 99524975, 1431742, 40781854, 72357096, 99226875, 70420215, 57658654, 9603598, 37166608, 36780454, 11581181, 50567636, 39986008, 45848907, 84127901, 6986898, 17386996, 82779622, 16380211, 97057187, 73031054, 97641116, 61897582, 53888755, 69697787, 2917920, 99125126, 88904910, 36753250, 45049308, 74614639, 39553046, 44842615, 36139942, 90158785, 4787945, 99549373, 20642888, 22405120, 4095116, 80316608, 47361209, 75820087, 19939935, 27185644, 17894977, 43357947, 48395186, 28787861, 28796059, 3880712, 26776922, 2208785, 81805959, 20568363, 98943869, 36468541, 9829782, 48675329, 44915235, 72495719, 77272628, 68128438, 70596786, 44844121, 68110330, 70727211, 52613508, 8129978, 20765474, 31161687, 73168565, 4204661, 51507425, 84293052, 36845587, 7955293, 23110625, 72738685, 9175338, 69786756, 22766820, 71522968, 2204165, 6505939, 27041967, 91664334, 7182642, 14045383, 27625735, 72732205, 49271185, 76960303, 99021067, 73786944, 51466049, 8733068, 12024238, 24953498, 74441448, 60955663, 4668450, 20002147, 67030811, 87710366, 15536795, 52261574, 21397057, 71657078, 10597197, 30163921, 19457589, 83368048, 60176618, 78218845, 3509435, 44348328, 88698958, 25842078, 51715482, 66667729, 33895336, 66045587, 508198, 60106217, 63628376, 37280276, 36189527, 67451935, 17976208, 34667286, 89996536, 30764367, 59614746, 54663246, 17857111, 24591705, 53547802, 24058273, 6153406, 78884452, 61373987, 176257, 40027975, 78589145, 78561158, 75777973, 17016156, 1022677, 3233569, 17058722, 76671482, 76703609, 19101477, 92692283, 75986488, 45428665, 6793819, 71300104, 112651, 92998591, 23740167, 47090124, 18131876, 97379790, 81774825, 69355476, 77187825, 26744917, 34432810, 5111370, 94911072, 36812683, 18833224, 33336148, 16445503, 90457870, 53084256, 74137926, 9259676, 75229462, 13862149, 37957788, 32250045, 8807940, 87598888, 76170907, 12571310, 72614359, 82532312, 9257405, 43152977, 97783876, 67513640, 2717150, 6521313, 29065964, 61859581, 4806458, 9058407, 99658235, 62357986, 57248122, 66903004, 6497830, 64098930, 1197320, 4798568, 99333375, 46870723, 47738185, 54058788, 94076128, 67793644, 92215320, 90061527, 415901, 8099994, 85117092, 18504197, 57241521, 15902805, 31491938, 84187166 +45919976, 91255408, 44481640, 57961282, 84293052, 8634541, 29932657, 26063929, 41380093, 57163802, 95957797, 53888755, 83210802, 35996293, 88698958, 2331773, 52060076, 57359924, 56424103, 50702367, 56515456, 61960400, 97281847, 13173644, 88653118, 26114953, 17365188, 17857111, 69605283, 64157906, 62803858, 37620363, 36808486, 50007421, 39373729, 72238278, 9257405, 59501487, 70420215, 61897582, 8696647, 5111370, 62452034, 16387575, 83368048, 24733232, 44842615, 90310261, 45407418, 41092172, 22405120, 45617087, 69641513, 40865610, 3088684, 8651647, 36189527, 21070110, 82178706, 8807940, 40027975, 17016156, 93933709, 11543098, 52293995, 16019925, 84904436, 65081429, 7955293, 91957544, 77694700, 70367851, 70541760, 23740167, 5970607, 18699206, 19272365, 94711842, 14731700, 5822202, 55669657, 84187166, 77413012, 21397057, 48893685, 95251277, 31904591, 36139942, 69136837, 26139110, 58208470, 86001008, 90013093, 22450468, 49882705, 95581843, 60106217, 14093520, 61271144, 26397786, 9829782, 4199704, 18466635, 51590803, 53547802, 89499542, 35456853, 22942635, 62490109, 80246713, 76434144, 3235882, 75407347, 39801351, 73168565, 37183543, 42967683, 85117092, 34698463, 42307449, 76703609, 48260151, 4798568, 72614359, 72738685, 98371444, 7685448, 71300104, 112651, 33249630, 22766820, 44889423, 18663507, 6808825, 7011964, 17146629, 4508700, 34044787, 90004325, 16567550, 64055960, 4668450, 20002147, 57658654, 77187825, 36780454, 26744917, 39986008, 37192445, 15536795, 12664567, 14349098, 82779622, 94076128, 16380211, 14220886, 67124282, 65017137, 99125126, 61141874, 57020481, 36753250, 15148031, 23848565, 84349107, 16405341, 94516935, 14947650, 78218845, 19939935, 25842078, 71965942, 72019362, 58749, 54762643, 7104732, 54199160, 3880712, 60430369, 74137926, 1936762, 54014062, 51472275, 70036438, 93515664, 37957788, 53393358, 19898053, 68110330, 64087743, 99604946, 44177011, 61373987, 176257, 79738755, 93790285, 37224844, 68000591, 5640302, 98739783, 13819358, 92867155, 61263068, 569864, 82979980, 98653983, 55189057, 99729357, 80014588, 48892201, 415901, 38365584, 49597667, 75986488, 45428665, 99690194, 32426752, 27187213, 63152504, 6505939, 41245325, 51426311, 18783702, 86118021, 8791066, 58751351, 59329511, 77787724, 96726697, 7182642, 14045383, 77898274, 61859143, 90654836, 24314885, 86798033, 66428795, 68204242, 72373496, 85571389, 29994197, 73786944, 83302115, 60393039, 47298834, 96193415, 36930650, 37166608, 29819940, 59318837, 45995325, 37891451, 30163921, 90090964, 2917920, 56531125, 321665, 9886593, 36812683, 91281584, 71083565, 93566986, 26664538, 82327024, 5267545, 19376156, 92803766, 40677414, 59109400, 13470059, 90158785, 45790169, 93053405, 45667668, 97561745, 29510992, 44348328, 98462867, 3233084, 83133790, 91141395, 6221471, 8055981, 66667729, 36312813, 28787861, 28796059, 57248122, 55602660, 30694952, 51315460, 21919959, 36933038, 88444207, 1788101, 26392416, 10358899, 48675329, 17976208, 9860195, 6157724, 53802686, 34667286, 77272628, 15163258, 94539824, 7919588, 61712234, 6153406, 73222868, 1569515, 15064655, 76170907, 19486173, 65880522, 47213183, 86301513, 63059024, 92554120, 42644903, 37675718, 23134715, 36685023, 8913721, 63506281, 61815223, 19101477, 83948335, 73124510, 9188443, 18411915, 48658605, 92604458, 96906410, 54868730, 42073124, 44245960, 7646095, 18504197, 55960386, 18131876, 74357852, 31126490, 7517032, 37659250, 54232247, 79806380, 1204161, 99021067, 29401781, 31491938, 1375023, 74441448, 78300864, 6819644, 77284619, 11757872, 62762857, 97783876, 86022504, 62428472, 11581181, 33565483, 45848907, 8128637, 17385531, 1250437, 99515901, 22129328, 47738185, 44060493, 30771409, 34946859, 99971982, 67793644, 97641116, 74743862, 92353856, 89637706, 72358170, 44846932, 526217, 65038678, 74614639, 92530431, 68824981, 61859581, 35512853, 9058407, 4095116, 70800879, 85242963, 83533741, 66137019, 91727510, 33797252, 68041839, 18001617, 30139692, 29516592, 44473167, 33431960, 80251430, 21533347, 13231279, 17957593, 21001913, 33304202, 26998766, 28928175, 10961421, 48395186, 93270084, 79657802, 53842979, 66045587, 49328608, 42782652, 81805959, 9259676, 20568363, 59981773, 55470718, 95010552, 36580610, 91990218, 30366150, 49236559, 64848072, 15075176, 59405277, 20885148, 27325428, 97011160, 68128438, 84166196, 54663246, 32699744, 55770687, 22879907, 30891921, 12416768, 89046466, 20867149, 87598888, 75135448, 24826575, 64602895, 5482538, 67031644, 13348726, 98648327, 61928316, 69255765, 7423788, 31161687, 62051033, 55850790, 33123618, 26275734, 82052050, 51507425, 874791, 26507214, 36505482, 37739481, 32590267, 92541302, 34295794, 39847321, 6793819, 29635537, 85116755, 16099750, 69786756, 59177718, 92998591, 66885828, 71522968, 71469330, 37501808, 83747892, 67281495, 33699435, 89811711, 68644627, 29834512, 27041967, 78766358, 59910624, 16424199, 81774825, 68316156, 82651278, 24915585, 30811010, 28851716, 92033260, 50188404, 72777973, 40356877, 95726235, 43152977, 56484900, 24953498, 30218878, 824872, 62496012, 66663942, 61982238, 34497327, 9603598, 74452589, 38256849, 41092102, 99333375, 68939068, 96709982, 82726008, 5832946, 71657078, 46540998, 96318094, 247198, 10597197, 94595668, 50806615, 66832478, 86821229, 44550764, 11365791, 82897371, 62693428, 29065964, 18833224, 43506672, 79191827, 93057697, 60176618, 8115266, 99549373, 92787493, 17764950, 76540481, 80316608, 4434662, 33336148, 90272749, 8373289, 57241521, 94090109, 81855445, 84684495, 35092039, 72274002, 17068582, 43357947, 16445503, 90457870, 87160386, 84840549, 28550822, 53084256, 11923835, 65271999, 44664587, 33895336, 2208785, 3183975, 78602717, 75052463, 98943869, 9860968, 85023028, 89419466, 83150534, 95290172, 70782102, 57393458, 47893286, 749283, 28734791, 11161731, 15948937, 90061527, 32159704, 30764367, 34236719, 38008118, 64098930, 69848388, 15015906, 81853704, 24058273, 45794415, 53648154, 21673260, 86460488, 3773993, 70004753, 31733363, 99861373, 59197747, 77377183, 62552524, 45990383, 8129978, 95395112, 65275241, 38341669, 75777973, 29959549, 77301523, 33553959, 16097038, 43933006, 39171895, 53666583, 72793444, 17058722, 54427233, 73617245, 78909561, 15535065, 70372191, 43322743, 42692881, 81172706, 93562257, 4099191, 25636669, 33237508, 71333116, 29466406, 16971929, 55143724, 97379790, 36135, 32058615, 69355476, 71920426, 74110882, 7687278, 65074535, 14363867, 87720882, 10264691, 12024238, 98948034, 3294781, 72357096, 54606384, 33201905, 31727379, 56955985, 45996863, 89078848, 36396314, 84127901, 40686254, 76330843, 46870723, 17727650, 48774913, 34432810, 73031054, 72278539, 6521313, 19457589, 77300457, 45075471, 39553046, 82339363, 32161669, 9599614, 10366309, 91240048, 38645117, 76825057, 4787945, 51588015, 27411561, 79942022, 2891150, 75543508, 22435353, 26493119, 19891772, 93359396, 68102437, 61623915, 3487592, 34698428, 63372756, 26776922, 508198, 91802888, 7229550, 33628349, 81274566, 14326617, 36468541, 84406788, 37280276, 22200378, 6497830, 44915235, 4978543, 24591705, 67227442, 3633375, 73392814, 21993752, 38658347, 44844121, 40984766, 38009615, 30395570, 24619760, 54263880, 30787683, 70727211, 8729146, 48360198, 78589145, 30090481, 9575954, 44784505, 52613508, 92071990, 30569392, 10649306, 20765474, 61728685, 1022677, 94360702, 60102965, 89804152, 76671482, 66271566, 55615885, 80555751, 62936963, 61741594, 1239555, 23110625, 9175338, 89214616, 63967300, 12303248, 5073754, 74724075, 47708850, 51464002, 56473732, 56461322, 47090124, 41442762, 38022834, 2607799, 4091162, 40781449, 27625735, 27665211, 65047700, 44479073, 16684829, 23194618, 18806856, 8733068, 99524975, 20645197, 67030811, 99226875, 83269727, 67513640, 53632373, 4064751, 7502255, 79821897, 40152546, 97057187, 54987042, 9398733, 94911072, 88904910, 44627776, 45049308, 43501211, 32274392, 99917599, 72725103, 4806458, 48673079, 26102057, 97940276, 59371804, 47361209, 92215320, 3509435, 96852321, 26863229, 8099994, 78785507, 88251446, 75128745, 66611759, 58224549, 73235980, 9623492, 96735716, 4515343, 96420429, 79922758, 23569917, 38494874, 42947632, 13862149, 67451935, 81677380, 22997281, 14626618, 34493392, 59614746, 14723410, 26734892, 20681921, 70596786, 81898046, 38061439, 33061250, 88130087, 22108665, 78561158, 60581278, 21289531, 89217461, 86543538, 6111563, 7300047, 4204661, 51149804, 13468268, 66322959, 79880247, 84002370, 30463802, 37560164, 92692283, 77620120, 29029316, 73109997, 79136082, 7066775, 99965001, 99297164, 22113133, 43376279, 91664334, 88047921, 85711894, 16054533, 49641577, 92692978, 82532312, 33935899, 50668599, 39201414, 55247455, 1431742, 83083131, 60955663, 66319530, 20140249, 77072625, 87710366, 17037369, 57803235, 6986898, 52261574, 40534591, 10453030, 22721500, 45306070, 54517921, 83651211, 20642888, 38510840, 76315420, 51715482, 99658235, 62357986, 66250369, 68702632, 82427263, 6038457, 66903004, 68816413, 47792865, 65454636, 92398073, 32250045, 89996536, 98130363, 1197320, 20836893, 65715134, 65338021, 62232211, 15902805, 83789391, 43045786, 68875490, 82886381, 47887470, 58180653, 42199455, 40197395, 30653863, 62740044, 29188588, 51047803, 72732205, 91938191, 49271185, 76960303, 4069912, 51466049, 40781854, 67474219, 73021291, 89699445, 50567636, 17386996, 83155442, 99224392, 6871053, 65851721, 168541, 669105, 44983451, 68694897, 23432750, 49598724, 27185644, 17894977, 75153252, 62115552, 78549759, 91831487, 67084351, 42237907, 10309525, 62430984, 85695762, 20486294, 12571310, 55753905, 66116458, 77312810, 3233569, 30543215, 44847298, 36845587, 2204165, 71316369, 52204879, 41481685, 4119747, 63015256, 56153345, 37435892, 86242799, 45381876, 26292919, 17081350, 54058788, 2717150, 69697787, 69579137, 45237957, 30998561, 63628376, 72495719, 16595436, 48088883, 79322415, 24168411, 17539625, 75820087, 75229462, 78884452, 35192533, 12348118, 20122224, 57240218, 4985896, 6525948, 46851987, 32151165 +75135448, 56484900, 68939068, 37891451, 29065964, 79191827, 91831487, 10649306, 62803858, 60102965, 77787724, 16971929, 50702367, 21397057, 54987042, 45407418, 26776922, 89996536, 62552524, 58180653, 82052050, 46851987, 75986488, 57163802, 98371444, 51464002, 25636669, 24314885, 39201414, 66663942, 14220886, 89637706, 45049308, 69136837, 72725103, 91727510, 13173644, 44348328, 84684495, 49882705, 28796059, 508198, 91802888, 9259676, 89419466, 84406788, 32250045, 14626618, 15015906, 31733363, 44784505, 95395112, 7955293, 415901, 44889423, 29834512, 72732205, 98948034, 96193415, 62762857, 87710366, 61897582, 669105, 321665, 45075471, 60176618, 99549373, 83533741, 26139110, 68041839, 29510992, 17894977, 17068582, 90457870, 93270084, 68702632, 49328608, 42947632, 83150534, 67451935, 32159704, 6525948, 81853704, 3633375, 21993752, 8729146, 59197747, 45990383, 47213183, 7423788, 13819358, 33553959, 98653983, 62740044, 73124510, 92604458, 42692881, 74724075, 70367851, 18663507, 41245325, 5970607, 29466406, 2607799, 37659250, 27665211, 23194618, 85571389, 18806856, 78300864, 4668450, 20002147, 20140249, 70420215, 39986008, 94595668, 99971982, 65851721, 97641116, 44983451, 62452034, 18833224, 61960400, 15148031, 23848565, 13470059, 79942022, 14947650, 33431960, 21533347, 25842078, 27185644, 33304202, 22450468, 3487592, 54199160, 42782652, 36580610, 37280276, 93515664, 28734791, 38008118, 80246713, 40984766, 87598888, 12571310, 65880522, 30569392, 8129978, 20765474, 82979980, 26275734, 11543098, 34698463, 48260151, 26063929, 63506281, 51507425, 4798568, 36845587, 35192533, 30463802, 92692283, 72738685, 18411915, 96906410, 27187213, 44245960, 37620363, 63152504, 51426311, 86118021, 47090124, 8791066, 41442762, 58751351, 22113133, 74357852, 88047921, 16054533, 57359924, 74110882, 65047700, 65074535, 14363867, 44479073, 72373496, 45919976, 24168411, 37166608, 99333375, 17037369, 84187166, 45848907, 82726008, 34432810, 2717150, 69697787, 56515456, 9886593, 61141874, 65038678, 32274392, 83368048, 92530431, 31904591, 9599614, 19376156, 92803766, 38645117, 17764950, 41092172, 20642888, 76540481, 45790169, 93053405, 29516592, 8373289, 78218845, 98462867, 3233084, 16445503, 93359396, 84840549, 88251446, 61623915, 75153252, 53084256, 10961421, 88653118, 4515343, 33628349, 51315460, 20568363, 38494874, 98943869, 95290172, 4978543, 11161731, 27325428, 90061527, 77272628, 26734892, 82178706, 44844121, 33061250, 30395570, 30787683, 20867149, 99861373, 55753905, 64157906, 98648327, 22108665, 78561158, 75407347, 42644903, 29932657, 61263068, 93933709, 48088883, 17058722, 30543215, 51149804, 52293995, 99729357, 80014588, 72614359, 91957544, 32426752, 92998591, 22766820, 56461322, 39373729, 33237508, 71316369, 95957797, 97379790, 41481685, 32058615, 24915585, 69355476, 49271185, 87720882, 37435892, 74441448, 64055960, 1431742, 83083131, 40781854, 61982238, 57658654, 9603598, 83269727, 31727379, 12664567, 44060493, 46540998, 79821897, 96318094, 67793644, 66832478, 82897371, 5111370, 94911072, 36812683, 16387575, 54517921, 99917599, 68824981, 24733232, 61859581, 44842615, 93057697, 10366309, 40677414, 76825057, 92787493, 9058407, 48673079, 94516935, 66137019, 2891150, 22435353, 4434662, 97561745, 71965942, 72274002, 76315420, 26998766, 99658235, 91141395, 63372756, 54762643, 44664587, 95581843, 79657802, 33895336, 79922758, 7229550, 36468541, 63628376, 64848072, 70036438, 44915235, 9860195, 20885148, 10309525, 53802686, 17365188, 81677380, 30764367, 54663246, 24591705, 6153406, 55770687, 85695762, 35456853, 73222868, 30891921, 12416768, 64087743, 3773993, 78589145, 76434144, 30090481, 9575954, 69255765, 65275241, 31161687, 60581278, 33123618, 37675718, 569864, 89217461, 23134715, 42199455, 76671482, 13468268, 73617245, 65081429, 84002370, 48892201, 83948335, 32590267, 37560164, 41380093, 77620120, 6793819, 9188443, 33249630, 43322743, 5073754, 77694700, 71522968, 7646095, 71469330, 6505939, 83747892, 18504197, 70541760, 67281495, 55960386, 33699435, 99297164, 71333116, 96726697, 85711894, 16424199, 82651278, 90004325, 28851716, 54232247, 82532312, 19272365, 63015256, 76960303, 4069912, 72238278, 16567550, 95726235, 77284619, 3294781, 62496012, 99226875, 34497327, 11757872, 74452589, 55669657, 38256849, 56955985, 45381876, 8128637, 15536795, 77413012, 22129328, 46870723, 30771409, 59318837, 45995325, 97057187, 53888755, 4985896, 8696647, 65017137, 44846932, 71083565, 44481640, 82327024, 83210802, 90158785, 23432750, 22405120, 35996293, 80316608, 33797252, 49598724, 26493119, 45617087, 97281847, 18001617, 44473167, 19891772, 69579137, 47361209, 17539625, 57961282, 96852321, 26863229, 19939935, 17957593, 83133790, 51715482, 72019362, 28928175, 75128745, 58749, 11923835, 65271999, 6221471, 62357986, 9623492, 2208785, 6038457, 74137926, 78602717, 81274566, 14093520, 75229462, 21919959, 88444207, 26392416, 42237907, 54014062, 10358899, 47893286, 749283, 17976208, 15075176, 59405277, 18466635, 34667286, 68128438, 34493392, 94539824, 84166196, 14723410, 98130363, 20836893, 53547802, 65338021, 21070110, 53648154, 81898046, 24619760, 1569515, 83789391, 79738755, 15064655, 19486173, 48360198, 64602895, 13348726, 52613508, 92071990, 39801351, 92867155, 82886381, 21289531, 77301523, 42967683, 86543538, 6111563, 94360702, 43933006, 7300047, 72793444, 85117092, 42307449, 66322959, 78909561, 61815223, 19101477, 61741594, 45428665, 29188588, 9175338, 71300104, 89214616, 54868730, 70372191, 59177718, 63967300, 12303248, 66885828, 36808486, 6808825, 23740167, 7066775, 50007421, 7011964, 17146629, 34044787, 59329511, 49641577, 36135, 81774825, 77898274, 61859143, 40781449, 30811010, 99021067, 50668599, 10264691, 51466049, 83302115, 94711842, 99524975, 43152977, 6819644, 66319530, 67474219, 73021291, 59501487, 97783876, 41092102, 26744917, 37192445, 53632373, 57803235, 6986898, 40686254, 76330843, 14349098, 4064751, 5832946, 34946859, 247198, 94076128, 16380211, 10597197, 30163921, 91255408, 22721500, 6521313, 44550764, 92353856, 62693428, 56531125, 72358170, 88904910, 19457589, 91281584, 57020481, 93566986, 82339363, 90310261, 59109400, 70800879, 75543508, 57241521, 75820087, 88698958, 38510840, 35092039, 8099994, 34698428, 3088684, 66045587, 82427263, 96420429, 55602660, 3183975, 81805959, 23569917, 66903004, 9860968, 60106217, 59981773, 47792865, 55470718, 14326617, 61271144, 36933038, 1788101, 91990218, 30366150, 4199704, 36189527, 6497830, 6157724, 72495719, 62430984, 17857111, 16595436, 24058273, 89499542, 38658347, 22942635, 21673260, 62490109, 15902805, 8807940, 68110330, 38009615, 38061439, 44177011, 61373987, 40027975, 37224844, 5482538, 66116458, 86301513, 5640302, 98739783, 92554120, 38341669, 73168565, 16097038, 4204661, 3233569, 55189057, 40197395, 36685023, 66271566, 2331773, 44847298, 36505482, 51047803, 99690194, 112651, 47708850, 81172706, 4099191, 89811711, 27041967, 52204879, 18131876, 31126490, 59910624, 4091162, 20122224, 92692978, 71920426, 33935899, 1204161, 66428795, 92033260, 68204242, 29401781, 9257405, 29994197, 73786944, 1375023, 24953498, 60955663, 14731700, 67030811, 30218878, 72357096, 5822202, 56424103, 77072625, 77187825, 62428472, 50567636, 89078848, 84127901, 17386996, 83155442, 1250437, 99515901, 52261574, 7502255, 71657078, 40534591, 10453030, 86821229, 74743862, 68694897, 45306070, 67124282, 99125126, 39553046, 32161669, 5267545, 8115266, 51588015, 4095116, 85242963, 59371804, 30139692, 3509435, 78785507, 40865610, 28550822, 48395186, 7104732, 8055981, 73235980, 3880712, 60430369, 26114953, 78549759, 85023028, 95010552, 70782102, 13862149, 51472275, 51590803, 59614746, 64098930, 69848388, 67227442, 32699744, 53393358, 69605283, 86460488, 99604946, 70727211, 93790285, 68000591, 77377183, 61928316, 43045786, 75777973, 61728685, 29959549, 77312810, 16019925, 79880247, 55615885, 26507214, 23110625, 15535065, 85116755, 16099750, 48658605, 7685448, 2204165, 73109997, 99965001, 18783702, 4508700, 43376279, 91664334, 7182642, 55143724, 14045383, 68316156, 7517032, 52060076, 90654836, 27625735, 91938191, 16684829, 50188404, 56153345, 40356877, 8733068, 60393039, 55247455, 20645197, 86242799, 824872, 54606384, 86022504, 33201905, 33565483, 26292919, 36396314, 57240218, 17385531, 54058788, 32151165, 50806615, 72278539, 11365791, 526217, 77300457, 74614639, 36139942, 84349107, 91240048, 83651211, 35512853, 27411561, 90272749, 8634541, 94090109, 80251430, 81855445, 90013093, 68102437, 58224549, 28787861, 53842979, 30998561, 57248122, 30694952, 75052463, 1936762, 68816413, 67084351, 22200378, 48675329, 92398073, 15163258, 7919588, 65715134, 45794415, 78884452, 89046466, 76170907, 17016156, 1022677, 39171895, 53666583, 89804152, 8913721, 76703609, 84293052, 80555751, 49597667, 92541302, 34295794, 39847321, 42073124, 29029316, 37501808, 38022834, 78766358, 79806380, 18699206, 7687278, 86798033, 72777973, 47298834, 79322415, 36780454, 11581181, 17081350, 99224392, 17727650, 47738185, 48774913, 29819940, 48893685, 2917920, 9398733, 95251277, 43506672, 4806458, 26102057, 92215320, 13231279, 69641513, 66611759, 45237957, 66250369, 66667729, 96735716, 62115552, 8651647, 49236559, 9829782, 65454636, 15948937, 97011160, 34236719, 1197320, 61712234, 70596786, 20486294, 22879907, 62232211, 24826575, 67031644, 63059024, 55850790, 54427233, 30653863, 37739481, 1239555, 69786756, 12348118, 93562257, 56473732, 31491938, 36930650, 67513640, 40152546, 90090964, 43501211, 16405341, 33336148, 45667668, 58208470, 86001008, 43357947, 26397786, 57393458, 37957788, 22997281, 20681921, 19898053, 54263880, 176257, 88130087, 47887470, 37183543, 874791, 62936963, 29635537, 79136082, 68644627, 4119747, 89699445, 82779622, 6871053, 73031054, 36753250, 26664538, 4787945, 97940276, 21001913, 87160386, 70004753, 3235882, 62051033, 84904436, 12024238, 44627776, 36312813, 38365584, 45996863, 96709982, 73392814, 68875490, 168541 +69641513, 68702632, 32159704, 4668450, 94911072, 29510992, 9860968, 68128438, 45794415, 8807940, 66271566, 79880247, 49597667, 6793819, 29834512, 26292919, 70800879, 83533741, 66137019, 44473167, 17957593, 66250369, 81805959, 75229462, 61271144, 77272628, 26734892, 20836893, 37224844, 8729146, 78589145, 77301523, 77312810, 85571389, 96193415, 99515901, 89637706, 74614639, 8115266, 22405120, 79942022, 4434662, 33431960, 47361209, 92215320, 98462867, 8055981, 9259676, 98943869, 53802686, 94539824, 34236719, 81853704, 89499542, 59197747, 44784505, 47213183, 16019925, 46851987, 78909561, 72738685, 16054533, 82651278, 54232247, 76960303, 16567550, 60393039, 77187825, 87710366, 68939068, 48774913, 32151165, 79821897, 8696647, 45306070, 62693428, 9886593, 61859581, 44348328, 26863229, 25842078, 76315420, 72019362, 78785507, 61623915, 75153252, 91141395, 82427263, 30694952, 55470718, 10358899, 28734791, 27325428, 81677380, 84166196, 14723410, 61373987, 62552524, 78561158, 65275241, 31161687, 62051033, 29932657, 89217461, 48088883, 82052050, 52293995, 80014588, 4798568, 39847321, 63967300, 92998591, 2204165, 79136082, 70541760, 8791066, 43376279, 96726697, 57359924, 92692978, 29401781, 40356877, 56484900, 24953498, 66319530, 72357096, 57658654, 36930650, 37192445, 5832946, 54058788, 65851721, 86821229, 56531125, 44846932, 77300457, 61960400, 54517921, 68824981, 32161669, 51588015, 45407418, 17764950, 94516935, 2891150, 75543508, 80316608, 33304202, 88653118, 6038457, 91831487, 47792865, 36468541, 26392416, 48675329, 4199704, 11161731, 22997281, 15015906, 85695762, 81898046, 80246713, 12416768, 89046466, 65880522, 8129978, 39801351, 20765474, 61263068, 72793444, 58180653, 66322959, 30653863, 72614359, 36845587, 83948335, 37560164, 29635537, 7685448, 51047803, 71300104, 22766820, 36808486, 6808825, 18504197, 67281495, 51426311, 58751351, 34044787, 71316369, 74357852, 97379790, 49271185, 1204161, 50668599, 68204242, 95726235, 8733068, 99524975, 64055960, 78300864, 83083131, 6819644, 98948034, 30218878, 17037369, 84187166, 12664567, 53632373, 57240218, 40686254, 1250437, 10453030, 96318094, 54987042, 4985896, 16387575, 95251277, 44481640, 93057697, 5267545, 40677414, 4787945, 14947650, 68041839, 29516592, 8373289, 57241521, 94090109, 69579137, 58208470, 13231279, 84684495, 35092039, 16445503, 99658235, 40865610, 66611759, 48395186, 9623492, 79657802, 33628349, 51315460, 85023028, 64848072, 37957788, 4978543, 6525948, 16595436, 65715134, 3633375, 24058273, 20486294, 30891921, 86460488, 38061439, 31733363, 99861373, 88130087, 9575954, 66116458, 92554120, 42644903, 29959549, 33553959, 93933709, 7300047, 17058722, 40197395, 36685023, 54427233, 65081429, 61741594, 415901, 32590267, 92692283, 89214616, 33249630, 43322743, 44889423, 7646095, 93562257, 51464002, 7066775, 4099191, 56461322, 39373729, 77787724, 16971929, 52204879, 2607799, 78766358, 85711894, 14045383, 41481685, 24915585, 72732205, 69355476, 79806380, 24314885, 33935899, 65047700, 14363867, 4069912, 99021067, 87720882, 23194618, 39201414, 12024238, 55247455, 60955663, 67474219, 20140249, 99226875, 74452589, 62428472, 67513640, 45848907, 76330843, 46870723, 99224392, 29819940, 97057187, 66832478, 2717150, 74743862, 11365791, 9398733, 67124282, 56515456, 61141874, 45049308, 32274392, 82339363, 10366309, 69136837, 90310261, 59109400, 23432750, 83651211, 20642888, 4095116, 97940276, 26139110, 49598724, 33336148, 26493119, 97281847, 30139692, 80251430, 21533347, 72274002, 84840549, 26998766, 28928175, 75128745, 58749, 11923835, 34698428, 63372756, 54762643, 45237957, 3088684, 28796059, 26776922, 66045587, 55602660, 75052463, 81274566, 95290172, 42237907, 13862149, 9860195, 65454636, 92398073, 20885148, 17365188, 34493392, 30764367, 54663246, 69848388, 32699744, 53547802, 65338021, 70596786, 19898053, 35456853, 53648154, 73222868, 22942635, 38009615, 64087743, 54263880, 1569515, 20867149, 5482538, 64157906, 13348726, 45990383, 43045786, 95395112, 13819358, 62803858, 61728685, 82886381, 33123618, 4204661, 3233569, 26275734, 98653983, 55189057, 11543098, 874791, 84293052, 62936963, 35192533, 9188443, 16099750, 98371444, 42692881, 74724075, 47708850, 71522968, 37501808, 55960386, 86118021, 47090124, 41442762, 22113133, 29466406, 91664334, 7182642, 77898274, 90004325, 27625735, 19272365, 65074535, 16684829, 31491938, 1375023, 20002147, 77284619, 3294781, 34497327, 9603598, 38256849, 54606384, 11581181, 33565483, 39986008, 84127901, 57803235, 52261574, 17727650, 82779622, 30771409, 7502255, 46540998, 40152546, 37891451, 10597197, 73031054, 669105, 14220886, 6521313, 90090964, 44550764, 68694897, 2917920, 62452034, 88904910, 44627776, 19457589, 65038678, 18833224, 43501211, 15148031, 24733232, 19376156, 91240048, 92803766, 60176618, 90158785, 99549373, 93053405, 45667668, 18001617, 13173644, 81855445, 57961282, 75820087, 19939935, 71965942, 90013093, 43357947, 51715482, 87160386, 68102437, 65271999, 6221471, 73235980, 3880712, 42782652, 78549759, 23569917, 60106217, 42947632, 14326617, 83150534, 95010552, 70782102, 88444207, 36580610, 749283, 6497830, 93515664, 17976208, 38008118, 53393358, 73392814, 55770687, 21070110, 82178706, 44844121, 33061250, 44177011, 30787683, 70004753, 83789391, 70727211, 40027975, 76170907, 55753905, 48360198, 67031644, 30090481, 98648327, 3235882, 52613508, 69255765, 30569392, 98739783, 75777973, 60581278, 21289531, 47887470, 37675718, 23134715, 86543538, 43933006, 30543215, 76671482, 51149804, 85117092, 34698463, 76703609, 84904436, 44847298, 36505482, 7955293, 19101477, 1239555, 91957544, 38365584, 77620120, 92541302, 34295794, 15535065, 85116755, 18411915, 92604458, 96906410, 70372191, 42073124, 112651, 12303248, 27187213, 12348118, 81172706, 63152504, 41245325, 7011964, 4508700, 33237508, 95957797, 18131876, 16424199, 81774825, 40781449, 30811010, 28851716, 4119747, 27665211, 18699206, 91938191, 66428795, 72373496, 73786944, 51466049, 18806856, 37435892, 20645197, 40781854, 14731700, 59501487, 11757872, 41092102, 37166608, 36780454, 33201905, 31727379, 26744917, 45381876, 45996863, 8128637, 89078848, 17386996, 17081350, 14349098, 47738185, 21397057, 59318837, 99971982, 168541, 50806615, 53888755, 72278539, 48893685, 69697787, 5111370, 72358170, 29065964, 91281584, 71083565, 92530431, 44842615, 36139942, 72725103, 4806458, 76540481, 35996293, 97561745, 17539625, 96852321, 83133790, 38510840, 21001913, 17068582, 90457870, 49882705, 88251446, 28550822, 53084256, 10961421, 62357986, 44664587, 508198, 4515343, 79922758, 91802888, 7229550, 3183975, 26114953, 38494874, 8651647, 21919959, 36933038, 84406788, 49236559, 37280276, 9829782, 47893286, 70036438, 67451935, 6157724, 10309525, 18466635, 34667286, 32250045, 51590803, 62430984, 89996536, 59614746, 7919588, 6153406, 21993752, 24619760, 87598888, 12571310, 64602895, 92071990, 86301513, 5640302, 17016156, 1022677, 569864, 6111563, 60102965, 73617245, 37739481, 30463802, 48892201, 41380093, 45428665, 9175338, 54868730, 59177718, 32426752, 70367851, 71469330, 18783702, 56473732, 33699435, 17146629, 99297164, 89811711, 59329511, 5970607, 71333116, 31126490, 88047921, 68316156, 7517032, 61859143, 37659250, 7687278, 92033260, 72238278, 9257405, 29994197, 45919976, 83302115, 43152977, 61982238, 77072625, 79322415, 55669657, 83269727, 50567636, 77413012, 96709982, 4064751, 34946859, 247198, 94076128, 94595668, 44983451, 82897371, 92353856, 321665, 526217, 45075471, 39553046, 83368048, 79191827, 9599614, 23848565, 83210802, 13470059, 26102057, 91727510, 90272749, 19891772, 27185644, 17894977, 22450468, 7104732, 58224549, 36312813, 28787861, 49328608, 96735716, 2208785, 96420429, 74137926, 20568363, 68816413, 14093520, 1788101, 91990218, 57393458, 54014062, 51472275, 90061527, 17857111, 61712234, 78884452, 62232211, 21673260, 62490109, 15902805, 30395570, 79738755, 93790285, 15064655, 75135448, 19486173, 24826575, 76434144, 68000591, 77377183, 61928316, 63059024, 10649306, 16097038, 42199455, 42307449, 51507425, 55615885, 84002370, 66885828, 44245960, 37620363, 6505939, 83747892, 23740167, 50007421, 38022834, 36135, 4091162, 20122224, 82532312, 44479073, 94711842, 47298834, 74441448, 824872, 66663942, 97783876, 15536795, 6986898, 22129328, 34432810, 61897582, 31904591, 93566986, 82327024, 76825057, 35512853, 48673079, 85242963, 59371804, 45790169, 33797252, 45617087, 78218845, 3509435, 86001008, 3233084, 8099994, 93359396, 3487592, 93270084, 66667729, 30998561, 60430369, 57248122, 78602717, 89419466, 30366150, 59405277, 72495719, 97011160, 98130363, 1197320, 68110330, 99604946, 3773993, 176257, 75407347, 7423788, 68875490, 94360702, 48260151, 13468268, 62740044, 80555751, 61815223, 23110625, 48658605, 69786756, 5073754, 29029316, 18663507, 73109997, 99965001, 55143724, 90654836, 74110882, 63015256, 1431742, 86242799, 73021291, 56424103, 62762857, 24168411, 89699445, 99333375, 83155442, 17385531, 67793644, 97641116, 30163921, 22721500, 65017137, 57020481, 36753250, 43506672, 38645117, 92787493, 9058407, 22435353, 88698958, 54199160, 53842979, 33895336, 66903004, 67084351, 36189527, 44915235, 15075176, 15948937, 64098930, 67227442, 20681921, 69605283, 40984766, 38341669, 92867155, 55850790, 37183543, 39171895, 89804152, 99729357, 26507214, 73124510, 75986488, 29188588, 57163802, 99690194, 25636669, 68644627, 49641577, 32058615, 52060076, 71920426, 86798033, 56153345, 62496012, 36396314, 50702367, 82726008, 44060493, 16380211, 99125126, 36812683, 99917599, 84349107, 41092172, 27411561, 8634541, 62115552, 1936762, 26397786, 63628376, 15163258, 24591705, 22879907, 82979980, 42967683, 53666583, 8913721, 26063929, 2331773, 59910624, 72777973, 10264691, 67030811, 86022504, 56955985, 40534591, 6871053, 26664538, 16405341, 95581843, 59981773, 22200378, 73168565, 77694700, 50188404, 5822202, 70420215, 71657078, 91255408, 14626618, 38658347, 22108665, 27041967, 45995325, 63506281 +7517032, 37166608, 22450468, 39171895, 17058722, 94911072, 34493392, 98739783, 76703609, 74357852, 99971982, 48673079, 13231279, 28928175, 28550822, 9623492, 15075176, 99604946, 78561158, 8129978, 26275734, 48892201, 25636669, 59329511, 824872, 31727379, 37192445, 96709982, 30163921, 9599614, 38645117, 4787945, 35512853, 38510840, 65271999, 6221471, 78602717, 83150534, 95010552, 26397786, 51472275, 53802686, 7919588, 19898053, 59197747, 52613508, 38341669, 16019925, 32590267, 15535065, 72738685, 92998591, 66885828, 81172706, 71333116, 29466406, 61859143, 40356877, 94711842, 86022504, 62428472, 45996863, 34946859, 32151165, 97057187, 54987042, 90090964, 5111370, 56531125, 99917599, 97561745, 19939935, 8099994, 49882705, 28796059, 33895336, 4199704, 44915235, 37957788, 73392814, 38658347, 73222868, 8807940, 54263880, 89046466, 43045786, 68875490, 42644903, 73168565, 1022677, 58180653, 42199455, 36685023, 85117092, 34698463, 78909561, 29188588, 16099750, 63967300, 44245960, 7646095, 37501808, 68316156, 72777973, 18806856, 8733068, 20645197, 1431742, 20002147, 77072625, 97783876, 74452589, 41092102, 17385531, 5832946, 74743862, 39553046, 61859581, 83210802, 51588015, 41092172, 80316608, 93053405, 33336148, 29510992, 17539625, 84684495, 17068582, 72019362, 40865610, 93270084, 95581843, 82427263, 91802888, 26114953, 66903004, 1788101, 36580610, 84406788, 47893286, 36189527, 9860195, 59405277, 72495719, 15163258, 26734892, 20486294, 80246713, 55753905, 5482538, 77377183, 61928316, 92554120, 92867155, 77301523, 16097038, 42307449, 26507214, 37739481, 35192533, 1239555, 92692283, 6793819, 57163802, 89214616, 32426752, 6808825, 41245325, 23740167, 17146629, 39373729, 43376279, 97379790, 16424199, 81774825, 37659250, 54232247, 1204161, 66428795, 92033260, 44479073, 50668599, 23194618, 12024238, 1375023, 67474219, 5822202, 17037369, 89078848, 52261574, 59318837, 10597197, 34432810, 50806615, 61897582, 44983451, 72278539, 8696647, 2917920, 99125126, 91281584, 92530431, 26664538, 36139942, 83651211, 72725103, 17764950, 4095116, 70800879, 26102057, 85242963, 97940276, 26493119, 13173644, 81855445, 92215320, 44348328, 72274002, 43357947, 76315420, 84840549, 88251446, 10961421, 88653118, 73235980, 66667729, 36312813, 68702632, 79922758, 55602660, 7229550, 62115552, 74137926, 20568363, 8651647, 61271144, 54014062, 6497830, 4978543, 11161731, 20885148, 90061527, 77272628, 94539824, 89996536, 81853704, 16595436, 53547802, 30891921, 3773993, 30395570, 88130087, 76434144, 30090481, 65275241, 29932657, 82052050, 30543215, 76671482, 66271566, 99729357, 13468268, 874791, 79880247, 84293052, 84904436, 30463802, 23110625, 45428665, 7685448, 71300104, 70372191, 12303248, 42692881, 27187213, 29029316, 37620363, 7066775, 4099191, 56473732, 56461322, 33699435, 34044787, 71316369, 95957797, 78766358, 49641577, 52060076, 27665211, 65047700, 72238278, 72373496, 29994197, 37435892, 64055960, 83083131, 98948034, 73021291, 9603598, 54606384, 89699445, 33565483, 56955985, 45848907, 1250437, 99224392, 47738185, 7502255, 71657078, 46540998, 40152546, 37891451, 73031054, 67793644, 86821229, 91255408, 82897371, 69697787, 65017137, 72358170, 9886593, 88904910, 36812683, 19457589, 36753250, 71083565, 45075471, 79191827, 24733232, 93057697, 19376156, 84349107, 90310261, 23432750, 4806458, 20642888, 66137019, 91727510, 49598724, 45617087, 94090109, 21533347, 3509435, 88698958, 17894977, 93359396, 87160386, 26998766, 78785507, 53084256, 58749, 63372756, 48395186, 54762643, 7104732, 53842979, 66045587, 38494874, 81274566, 60106217, 36933038, 57393458, 10358899, 49236559, 93515664, 67451935, 15948937, 32250045, 97011160, 32159704, 38008118, 14723410, 98130363, 65715134, 45794415, 70596786, 85695762, 21993752, 81898046, 22942635, 40984766, 64087743, 38061439, 33061250, 30787683, 1569515, 176257, 79738755, 93790285, 15064655, 75135448, 64602895, 67031644, 22108665, 9575954, 3235882, 92071990, 69255765, 75407347, 66116458, 86301513, 95395112, 61728685, 55850790, 82886381, 33123618, 37675718, 569864, 40197395, 63506281, 54427233, 51507425, 73617245, 30653863, 46851987, 61815223, 7955293, 73124510, 41380093, 75986488, 51047803, 92604458, 54868730, 42073124, 12348118, 93562257, 55960386, 86118021, 38022834, 16971929, 85711894, 30811010, 72732205, 92692978, 82532312, 18699206, 76960303, 56153345, 85571389, 45919976, 83302115, 78300864, 60955663, 40781854, 6819644, 77284619, 61982238, 57658654, 87710366, 36780454, 45381876, 8128637, 36396314, 57803235, 6986898, 17386996, 29819940, 40534591, 6871053, 669105, 2717150, 48893685, 9398733, 89637706, 44627776, 57020481, 95251277, 43506672, 61960400, 83368048, 82339363, 68824981, 10366309, 99549373, 9058407, 76540481, 27411561, 14947650, 22435353, 4434662, 33797252, 97281847, 90272749, 30139692, 8373289, 57241521, 19891772, 57961282, 69641513, 17957593, 71965942, 90013093, 75153252, 34698428, 66611759, 62357986, 58224549, 3088684, 66250369, 79657802, 508198, 2208785, 4515343, 96420429, 3183975, 1936762, 9860968, 91831487, 68816413, 89419466, 88444207, 42237907, 22200378, 64848072, 70036438, 14626618, 62430984, 59614746, 34236719, 6525948, 3633375, 55770687, 21070110, 22879907, 44844121, 15902805, 86460488, 24619760, 44177011, 61373987, 20867149, 40027975, 8729146, 76170907, 19486173, 78589145, 64157906, 98648327, 68000591, 10649306, 62051033, 61263068, 37183543, 93933709, 42967683, 48088883, 53666583, 51149804, 52293995, 8913721, 48260151, 80014588, 2331773, 84002370, 4798568, 44847298, 36505482, 92541302, 9188443, 98371444, 96906410, 112651, 5073754, 70367851, 73109997, 6505939, 70541760, 99965001, 18783702, 47090124, 58751351, 68644627, 18131876, 2607799, 96726697, 31126490, 59910624, 32058615, 27625735, 69355476, 71920426, 79806380, 24314885, 91938191, 7687278, 19272365, 65074535, 4069912, 16684829, 87720882, 68204242, 9257405, 39201414, 16567550, 51466049, 56484900, 24953498, 4668450, 14731700, 67030811, 59501487, 11757872, 79322415, 77187825, 33201905, 11581181, 53632373, 50702367, 46870723, 17727650, 82779622, 21397057, 44060493, 54058788, 79821897, 45995325, 168541, 66832478, 14220886, 92353856, 65038678, 31904591, 93566986, 82327024, 91240048, 60176618, 68041839, 18001617, 78218845, 75820087, 25842078, 83133790, 33304202, 3487592, 44664587, 28787861, 54199160, 96735716, 60430369, 57248122, 23569917, 98943869, 85023028, 59981773, 47792865, 75229462, 36468541, 67084351, 30366150, 9829782, 48675329, 17976208, 10309525, 18466635, 17365188, 22997281, 84166196, 64098930, 15015906, 20681921, 24058273, 65338021, 78884452, 35456853, 62232211, 21673260, 12416768, 48360198, 13348726, 45990383, 44784505, 30569392, 63059024, 7423788, 47887470, 17016156, 77312810, 82979980, 23134715, 3233569, 89804152, 72793444, 55189057, 66322959, 83948335, 77620120, 85116755, 69786756, 44889423, 36808486, 63152504, 79136082, 51426311, 50007421, 99297164, 33237508, 22113133, 91664334, 55143724, 14045383, 16054533, 90654836, 49271185, 63015256, 50188404, 99021067, 29401781, 95726235, 47298834, 74441448, 20140249, 62496012, 36930650, 55669657, 84187166, 26292919, 77413012, 84127901, 83155442, 22129328, 30771409, 247198, 94076128, 65851721, 53888755, 11365791, 68694897, 62452034, 44846932, 61141874, 29065964, 16387575, 18833224, 74614639, 44842615, 5267545, 76825057, 8115266, 79942022, 35996293, 2891150, 75543508, 59371804, 8634541, 69579137, 98462867, 96852321, 26863229, 27185644, 16445503, 90457870, 51715482, 99658235, 75128745, 91141395, 49328608, 6038457, 81805959, 78549759, 51315460, 14326617, 95290172, 26392416, 63628376, 37280276, 749283, 28734791, 92398073, 34667286, 30764367, 54663246, 1197320, 61712234, 32699744, 6153406, 53393358, 83789391, 70727211, 87598888, 99861373, 37224844, 12571310, 62552524, 20765474, 31161687, 75777973, 60581278, 33553959, 60102965, 98653983, 55615885, 80555751, 19101477, 38365584, 37560164, 18411915, 22766820, 43322743, 74724075, 47708850, 18663507, 41442762, 89811711, 77787724, 27041967, 88047921, 77898274, 24915585, 14363867, 99524975, 55247455, 86242799, 30218878, 99226875, 66663942, 56424103, 96193415, 62762857, 38256849, 50567636, 15536795, 40686254, 17081350, 82726008, 48774913, 96318094, 97641116, 6521313, 67124282, 56515456, 526217, 43501211, 54517921, 15148031, 32161669, 23848565, 22405120, 16405341, 83533741, 26139110, 45790169, 33431960, 47361209, 86001008, 61623915, 8055981, 45237957, 26776922, 30694952, 42947632, 55470718, 21919959, 27325428, 81677380, 68128438, 17857111, 24591705, 69605283, 53648154, 24826575, 5640302, 89217461, 86543538, 4204661, 11543098, 26063929, 415901, 49597667, 34295794, 39847321, 9175338, 48658605, 99690194, 77694700, 83747892, 67281495, 4508700, 29834512, 52204879, 57359924, 90004325, 4119747, 73786944, 60393039, 31491938, 66319530, 72357096, 34497327, 70420215, 26744917, 39986008, 12664567, 99515901, 76330843, 4064751, 45306070, 62693428, 77300457, 32274392, 44481640, 59109400, 13470059, 92787493, 45667668, 80251430, 3233084, 21001913, 35092039, 11923835, 42782652, 33628349, 9259676, 75052463, 70782102, 91990218, 6157724, 65454636, 51590803, 20836893, 67227442, 89499542, 62490109, 70004753, 65880522, 39801351, 62803858, 21289531, 29959549, 62740044, 72614359, 36845587, 62936963, 91957544, 29635537, 59177718, 33249630, 2204165, 71469330, 51464002, 7011964, 8791066, 5970607, 4091162, 20122224, 40781449, 74110882, 10264691, 83269727, 57240218, 10453030, 16380211, 44550764, 69136837, 92803766, 40677414, 90158785, 45407418, 58208470, 3880712, 30998561, 68110330, 38009615, 31733363, 47213183, 7300047, 65081429, 61741594, 7182642, 41481685, 36135, 82651278, 28851716, 33935899, 86798033, 43152977, 3294781, 24168411, 68939068, 67513640, 94595668, 22721500, 94516935, 29516592, 68102437, 14093520, 13819358, 94360702, 99333375, 14349098, 4985896, 321665, 45049308, 69848388, 6111563, 71522968, 18504197, 43933006, 82178706, 44473167, 13862149 +33237508, 3880712, 93566986, 66611759, 11161731, 90061527, 37501808, 1431742, 31727379, 33895336, 9259676, 17365188, 15902805, 92692283, 12348118, 55143724, 41481685, 67030811, 53888755, 56515456, 16387575, 31904591, 45407418, 21533347, 57961282, 3233084, 33304202, 65271999, 68816413, 54014062, 6157724, 81853704, 20681921, 73222868, 61728685, 37675718, 37183543, 7300047, 55189057, 52293995, 51507425, 32590267, 41380093, 37620363, 6808825, 56461322, 47090124, 59910624, 90654836, 37659250, 16684829, 23194618, 29994197, 66663942, 34497327, 86022504, 84127901, 45995325, 91255408, 69697787, 8696647, 88904910, 92530431, 84349107, 23432750, 83533741, 2891150, 18001617, 57241521, 13173644, 72019362, 61623915, 53084256, 63372756, 66045587, 49328608, 60430369, 81805959, 81274566, 14093520, 749283, 37957788, 34667286, 67227442, 65338021, 78884452, 69605283, 76170907, 67031644, 78561158, 92071990, 66116458, 31161687, 60581278, 39171895, 72793444, 36685023, 99729357, 61815223, 34295794, 39847321, 112651, 22766820, 43322743, 6505939, 7066775, 55960386, 4099191, 25636669, 32058615, 61859143, 19272365, 45919976, 10264691, 16567550, 60393039, 86242799, 14731700, 99226875, 96193415, 37166608, 39986008, 36396314, 40686254, 16380211, 34432810, 65851721, 61897582, 669105, 72278539, 11365791, 2917920, 321665, 89637706, 94911072, 61141874, 71083565, 15148031, 61859581, 9599614, 51588015, 93053405, 97281847, 33431960, 80251430, 17539625, 92215320, 83133790, 22450468, 93359396, 87160386, 78785507, 54762643, 58224549, 75229462, 14326617, 70782102, 70036438, 93515664, 15075176, 65454636, 32250045, 59614746, 3633375, 53547802, 45794415, 99604946, 64602895, 5482538, 61928316, 45990383, 52613508, 68875490, 38341669, 33123618, 1022677, 77301523, 42967683, 43933006, 17058722, 51149804, 13468268, 73617245, 30653863, 72614359, 19101477, 23110625, 15535065, 29188588, 29635537, 57163802, 92604458, 92998591, 42692881, 36808486, 83747892, 50007421, 56473732, 7011964, 39373729, 7182642, 20122224, 54232247, 71920426, 1204161, 65047700, 86798033, 63015256, 50668599, 29401781, 73786944, 31491938, 98948034, 30218878, 77072625, 9603598, 36930650, 29819940, 247198, 94076128, 97641116, 66832478, 6521313, 44550764, 74743862, 62452034, 99125126, 43501211, 74614639, 61960400, 83368048, 82339363, 44842615, 93057697, 36139942, 40677414, 13470059, 4095116, 70800879, 97940276, 66137019, 22435353, 45617087, 58208470, 3509435, 44348328, 88698958, 3487592, 7104732, 73235980, 44664587, 79657802, 53842979, 74137926, 26114953, 21919959, 95010552, 42237907, 57393458, 9829782, 67451935, 10309525, 62430984, 26734892, 24591705, 21070110, 53648154, 12416768, 64087743, 38061439, 44177011, 20867149, 83789391, 93790285, 8729146, 12571310, 75135448, 3235882, 69255765, 8129978, 63059024, 92554120, 42644903, 65275241, 62051033, 62803858, 29932657, 61263068, 77312810, 569864, 33553959, 93933709, 48088883, 66271566, 42307449, 63506281, 80555751, 92541302, 89214616, 99690194, 69786756, 63967300, 33249630, 66885828, 5073754, 27187213, 44889423, 47708850, 7646095, 93562257, 73109997, 63152504, 70541760, 8791066, 41442762, 89811711, 71316369, 52204879, 88047921, 85711894, 14045383, 97379790, 81774825, 90004325, 69355476, 68204242, 72238278, 43152977, 4668450, 6819644, 77284619, 59501487, 41092102, 24168411, 36780454, 99333375, 56955985, 26292919, 57803235, 7502255, 96318094, 59318837, 10597197, 50806615, 54987042, 4985896, 29065964, 77300457, 45075471, 26664538, 82327024, 69136837, 83210802, 76825057, 90158785, 72725103, 99549373, 9058407, 48673079, 27411561, 33797252, 97561745, 78218845, 19891772, 29510992, 69641513, 98462867, 86001008, 17957593, 21001913, 27185644, 17894977, 8099994, 90457870, 49882705, 88251446, 45237957, 57248122, 91802888, 6038457, 62115552, 23569917, 33628349, 38494874, 98943869, 91831487, 89419466, 88444207, 1788101, 26397786, 36580610, 47893286, 48675329, 4199704, 28734791, 59405277, 18466635, 81677380, 30764367, 84166196, 14723410, 69848388, 7919588, 32699744, 55770687, 38658347, 35456853, 22879907, 62232211, 22942635, 38009615, 54263880, 89046466, 70727211, 79738755, 15064655, 55753905, 59197747, 24826575, 48360198, 88130087, 64157906, 22108665, 44784505, 30569392, 95395112, 13819358, 55850790, 82886381, 17016156, 82979980, 94360702, 11543098, 85117092, 8913721, 76703609, 66322959, 874791, 84904436, 46851987, 4798568, 44847298, 36845587, 61741594, 37560164, 6793819, 9188443, 18411915, 98371444, 71300104, 42073124, 74724075, 29029316, 44245960, 71522968, 2204165, 51426311, 18783702, 86118021, 68644627, 95957797, 29466406, 27041967, 96726697, 16424199, 68316156, 77898274, 57359924, 4091162, 24915585, 27625735, 74110882, 91938191, 49271185, 14363867, 4069912, 87720882, 72777973, 72373496, 85571389, 95726235, 51466049, 83302115, 47298834, 1375023, 56484900, 40781854, 20002147, 66319530, 824872, 70420215, 62762857, 79322415, 97783876, 54606384, 87710366, 83269727, 89699445, 84187166, 26744917, 45996863, 8128637, 57240218, 96709982, 17385531, 1250437, 47738185, 30771409, 10453030, 32151165, 73031054, 82897371, 92353856, 72358170, 44627776, 95251277, 54517921, 79191827, 91240048, 92803766, 8115266, 14947650, 59371804, 68041839, 33336148, 30139692, 94090109, 47361209, 13231279, 43357947, 51715482, 84840549, 99658235, 28550822, 34698428, 48395186, 88653118, 62357986, 66250369, 36312813, 28787861, 42782652, 508198, 79922758, 55602660, 78602717, 20568363, 66903004, 42947632, 95290172, 36933038, 13862149, 22200378, 9860195, 53802686, 72495719, 34493392, 15163258, 94539824, 6525948, 17857111, 65715134, 24058273, 21993752, 81898046, 3773993, 19486173, 78589145, 76434144, 75407347, 73168565, 21289531, 42199455, 84293052, 55615885, 65081429, 48892201, 415901, 77620120, 9175338, 16099750, 48658605, 7685448, 96906410, 12303248, 77694700, 71469330, 67281495, 99965001, 33699435, 4508700, 34044787, 77787724, 71333116, 78766358, 16054533, 49641577, 82651278, 52060076, 40781449, 24314885, 18699206, 33935899, 66428795, 92033260, 50188404, 9257405, 18806856, 12024238, 37435892, 24953498, 74441448, 61982238, 56424103, 55669657, 50567636, 45381876, 12664567, 77413012, 52261574, 22129328, 4064751, 44060493, 54058788, 6871053, 97057187, 168541, 30163921, 44983451, 90090964, 45306070, 62693428, 65017137, 9886593, 91281584, 526217, 65038678, 32274392, 44481640, 24733232, 32161669, 10366309, 5267545, 19376156, 90310261, 38645117, 60176618, 4787945, 4806458, 92787493, 17764950, 41092172, 76540481, 85242963, 35996293, 45790169, 26493119, 90272749, 44473167, 81855445, 75820087, 84684495, 25842078, 38510840, 72274002, 76315420, 10961421, 8055981, 9623492, 93270084, 28796059, 26776922, 85023028, 59981773, 55470718, 83150534, 36468541, 26392416, 91990218, 30366150, 63628376, 37280276, 64848072, 51472275, 6497830, 44915235, 20885148, 97011160, 38008118, 54663246, 16595436, 19898053, 44844121, 40984766, 68110330, 33061250, 24619760, 30787683, 70004753, 176257, 40027975, 65880522, 13348726, 30090481, 98648327, 9575954, 7423788, 5640302, 92867155, 29959549, 89217461, 16097038, 3233569, 26275734, 89804152, 82052050, 30543215, 76671482, 34698463, 26063929, 16019925, 26507214, 78909561, 36505482, 7955293, 1239555, 83948335, 38365584, 73124510, 85116755, 54868730, 70372191, 59177718, 18663507, 79136082, 51464002, 23740167, 99297164, 22113133, 59329511, 29834512, 16971929, 18131876, 43376279, 74357852, 36135, 28851716, 72732205, 4119747, 79806380, 99021067, 94711842, 99524975, 55247455, 67474219, 20140249, 5822202, 77187825, 38256849, 33201905, 6986898, 82726008, 14349098, 46870723, 17727650, 34946859, 79821897, 40152546, 99971982, 14220886, 67124282, 57020481, 18833224, 23848565, 83651211, 35512853, 20642888, 22405120, 16405341, 26102057, 79942022, 26139110, 8373289, 8634541, 69579137, 96852321, 19939935, 90013093, 35092039, 16445503, 26998766, 68102437, 28928175, 40865610, 75153252, 11923835, 6221471, 3088684, 95581843, 54199160, 96735716, 96420429, 7229550, 8651647, 1936762, 9860968, 47792865, 61271144, 67084351, 84406788, 10358899, 49236559, 27325428, 77272628, 34236719, 53393358, 73392814, 8807940, 61373987, 1569515, 99861373, 68000591, 62552524, 47213183, 20765474, 23134715, 6111563, 4204661, 54427233, 80014588, 2331773, 62740044, 62936963, 49597667, 72738685, 75986488, 45428665, 32426752, 81172706, 58751351, 30811010, 92692978, 27665211, 7687278, 76960303, 40356877, 64055960, 83083131, 60955663, 73021291, 57658654, 62428472, 17037369, 68939068, 45848907, 50702367, 76330843, 99224392, 21397057, 71657078, 40534591, 37891451, 94595668, 22721500, 44846932, 19457589, 36753250, 39553046, 71965942, 17068582, 91141395, 4515343, 78549759, 36189527, 15948937, 14626618, 68128438, 32159704, 1197320, 6153406, 20486294, 82178706, 80246713, 31733363, 37224844, 86301513, 43045786, 75777973, 47887470, 86543538, 53666583, 98653983, 40197395, 79880247, 37739481, 30463802, 51047803, 18504197, 41245325, 38022834, 31126490, 65074535, 39201414, 20645197, 11757872, 74452589, 33565483, 37192445, 17081350, 82779622, 46540998, 86821229, 5111370, 36812683, 99917599, 91727510, 49598724, 45667668, 26863229, 66667729, 68702632, 82427263, 51315460, 60106217, 92398073, 22997281, 89996536, 64098930, 98130363, 15015906, 89499542, 21673260, 86460488, 30395570, 10649306, 58180653, 48260151, 84002370, 35192533, 70367851, 5970607, 91664334, 44479073, 56153345, 78300864, 72357096, 62496012, 67513640, 15536795, 89078848, 53632373, 17386996, 83155442, 5832946, 67793644, 48893685, 68694897, 43506672, 68824981, 59109400, 30998561, 3183975, 20836893, 61712234, 70596786, 62490109, 77377183, 60102965, 17146629, 2607799, 7517032, 82532312, 3294781, 11581181, 99515901, 48774913, 94516935, 80316608, 29516592, 75128745, 2208785, 75052463, 4978543, 17976208, 85695762, 98739783, 91957544, 8733068, 9398733, 56531125, 75543508, 4434662, 58749, 51590803, 39801351, 30694952, 30891921, 87598888, 2717150, 45049308 +60393039, 45381876, 26493119, 11161731, 72238278, 508198, 36505482, 31126490, 36780454, 67513640, 68694897, 88444207, 75135448, 39171895, 60102965, 63152504, 57359924, 92692978, 71920426, 22721500, 57020481, 60176618, 23432750, 26863229, 68702632, 49328608, 81805959, 67227442, 89217461, 82979980, 98653983, 74724075, 37501808, 41442762, 59329511, 2607799, 43376279, 55143724, 41481685, 74110882, 44479073, 56153345, 95726235, 86242799, 56955985, 53888755, 56515456, 62693428, 72358170, 24733232, 76825057, 98462867, 33304202, 28928175, 8055981, 93270084, 42782652, 78602717, 749283, 4199704, 98130363, 61712234, 3633375, 78884452, 99604946, 13819358, 31161687, 82886381, 47887470, 37675718, 569864, 16097038, 72793444, 30653863, 84002370, 34295794, 72738685, 75986488, 39847321, 16099750, 37620363, 51464002, 99965001, 22113133, 16971929, 27625735, 37659250, 54232247, 66428795, 73786944, 6819644, 77284619, 98948034, 5822202, 79322415, 55669657, 8128637, 57240218, 82726008, 7502255, 32151165, 6871053, 669105, 321665, 61141874, 77300457, 18833224, 54517921, 93566986, 32161669, 10366309, 3233084, 17957593, 35092039, 16445503, 87160386, 75128745, 65271999, 79657802, 96735716, 51315460, 70782102, 67084351, 54014062, 70036438, 15075176, 53802686, 27325428, 97011160, 68128438, 69605283, 82178706, 30395570, 61373987, 83789391, 8729146, 55753905, 63059024, 98739783, 92554120, 33553959, 42967683, 17058722, 58180653, 51149804, 44847298, 48658605, 43322743, 77694700, 29029316, 44889423, 18663507, 93562257, 51426311, 4099191, 25636669, 17146629, 89811711, 77787724, 88047921, 85711894, 28851716, 86798033, 65074535, 14363867, 50668599, 72373496, 83302115, 36930650, 83269727, 33565483, 17081350, 30771409, 94595668, 65851721, 73031054, 72278539, 82897371, 8696647, 65038678, 83368048, 9599614, 83210802, 4806458, 22405120, 35996293, 66137019, 2891150, 75543508, 22435353, 45617087, 44473167, 81855445, 57961282, 75820087, 71965942, 21001913, 28550822, 3487592, 45237957, 30998561, 62115552, 74137926, 98943869, 9860968, 91831487, 55470718, 83150534, 26392416, 49236559, 64848072, 9829782, 93515664, 28734791, 92398073, 15948937, 34667286, 22997281, 62430984, 32159704, 24591705, 65715134, 6153406, 19898053, 73392814, 89499542, 21673260, 40984766, 64087743, 44177011, 79738755, 59197747, 24826575, 48360198, 88130087, 30090481, 61928316, 44784505, 47213183, 78561158, 92071990, 68875490, 92867155, 93933709, 48088883, 52293995, 16019925, 13468268, 54427233, 73617245, 37739481, 77620120, 29188588, 57163802, 9175338, 7685448, 51047803, 32426752, 42692881, 44245960, 36808486, 58751351, 38022834, 71333116, 59910624, 32058615, 77898274, 52060076, 4091162, 20122224, 40781449, 27665211, 24314885, 49271185, 92033260, 39201414, 51466049, 20645197, 66319530, 3294781, 72357096, 66663942, 77187825, 68939068, 39986008, 37192445, 26292919, 53632373, 17386996, 40686254, 4064751, 45995325, 247198, 94076128, 61897582, 66832478, 2717150, 92353856, 5111370, 94911072, 99125126, 91281584, 43501211, 71083565, 68824981, 44842615, 69136837, 59109400, 8115266, 13470059, 45407418, 4095116, 59371804, 33797252, 49598724, 90272749, 8373289, 94090109, 58208470, 92215320, 13231279, 25842078, 83133790, 90457870, 75153252, 63372756, 54762643, 73235980, 9623492, 95581843, 66045587, 2208785, 96420429, 57248122, 79922758, 55602660, 7229550, 6038457, 9259676, 75052463, 8651647, 59981773, 95290172, 1788101, 48675329, 77272628, 34236719, 6525948, 69848388, 20486294, 22942635, 80246713, 12416768, 3773993, 99861373, 93790285, 5482538, 9575954, 3235882, 66116458, 8129978, 95395112, 42644903, 38341669, 1022677, 61263068, 23134715, 43933006, 7300047, 89804152, 34698463, 76703609, 84904436, 62740044, 72614359, 30463802, 91957544, 41380093, 92692283, 85116755, 12303248, 5073754, 47708850, 70367851, 71469330, 67281495, 7011964, 8791066, 29466406, 27041967, 91664334, 96726697, 78766358, 7182642, 16054533, 30811010, 33935899, 63015256, 72777973, 29994197, 45919976, 10264691, 31491938, 37435892, 24953498, 62496012, 74452589, 86022504, 89699445, 14349098, 17727650, 48774913, 5832946, 59318837, 168541, 30163921, 86821229, 6521313, 9398733, 65017137, 44846932, 44627776, 36812683, 29065964, 526217, 95251277, 32274392, 39553046, 31904591, 90158785, 83651211, 4787945, 17764950, 9058407, 97940276, 91727510, 80316608, 68041839, 97281847, 8634541, 3509435, 84684495, 19939935, 90013093, 76315420, 49882705, 40865610, 61623915, 34698428, 44664587, 33895336, 26776922, 38494874, 1936762, 75229462, 14326617, 61271144, 36580610, 84406788, 10358899, 37957788, 17976208, 20885148, 14626618, 89996536, 84166196, 64098930, 14723410, 26734892, 32699744, 53393358, 85695762, 21070110, 73222868, 30891921, 86460488, 38061439, 54263880, 30787683, 20867149, 87598888, 31733363, 40027975, 65880522, 76434144, 67031644, 22108665, 69255765, 30569392, 7423788, 65275241, 29932657, 29959549, 86543538, 6111563, 40197395, 66271566, 48260151, 65081429, 2331773, 61815223, 73124510, 29635537, 71300104, 92604458, 112651, 33249630, 2204165, 7646095, 6505939, 7066775, 55960386, 86118021, 47090124, 39373729, 34044787, 33237508, 5970607, 29834512, 74357852, 16424199, 82532312, 18699206, 68204242, 8733068, 99524975, 4668450, 67474219, 824872, 99226875, 96193415, 70420215, 77072625, 38256849, 41092102, 54606384, 24168411, 87710366, 33201905, 99333375, 15536795, 12664567, 57803235, 6986898, 46870723, 47738185, 44060493, 10453030, 37891451, 16380211, 10597197, 54987042, 91255408, 90090964, 48893685, 69697787, 45306070, 56531125, 89637706, 9886593, 88904910, 36753250, 92530431, 44481640, 26664538, 82327024, 61859581, 23848565, 91240048, 40677414, 90310261, 51588015, 16405341, 48673079, 76540481, 79942022, 94516935, 26139110, 93053405, 18001617, 30139692, 33431960, 13173644, 29510992, 69579137, 47361209, 96852321, 27185644, 17894977, 17068582, 72019362, 88251446, 11923835, 66611759, 36312813, 3880712, 82427263, 91802888, 26114953, 23569917, 33628349, 20568363, 85023028, 47792865, 42237907, 57393458, 51472275, 9860195, 59405277, 18466635, 32250045, 15163258, 94539824, 30764367, 38008118, 20836893, 81853704, 20681921, 16595436, 21993752, 22879907, 62232211, 38009615, 24619760, 70004753, 70727211, 68000591, 45990383, 75407347, 5640302, 61728685, 60581278, 17016156, 3233569, 53666583, 42199455, 55189057, 99729357, 26063929, 51507425, 79880247, 55615885, 26507214, 78909561, 35192533, 7955293, 19101477, 83948335, 49597667, 9188443, 59177718, 63967300, 92998591, 66885828, 27187213, 81172706, 73109997, 18504197, 23740167, 18783702, 99297164, 52204879, 18131876, 36135, 7517032, 61859143, 90004325, 4119747, 16684829, 87720882, 23194618, 29401781, 85571389, 43152977, 56484900, 55247455, 64055960, 83083131, 40781854, 61982238, 56424103, 34497327, 62762857, 62428472, 17037369, 1250437, 76330843, 99971982, 50806615, 44983451, 4985896, 44550764, 74743862, 11365791, 43506672, 74614639, 61960400, 45075471, 99917599, 82339363, 15148031, 93057697, 19376156, 38645117, 35512853, 99549373, 92787493, 85242963, 45790169, 45667668, 57241521, 86001008, 38510840, 8099994, 22450468, 93359396, 53084256, 10961421, 7104732, 58224549, 54199160, 30694952, 66903004, 81274566, 42947632, 14093520, 21919959, 36189527, 44915235, 67451935, 4978543, 6157724, 10309525, 51590803, 81677380, 34493392, 7919588, 15015906, 53547802, 45794415, 35456853, 53648154, 176257, 64602895, 62552524, 43045786, 62051033, 62803858, 55850790, 33123618, 77312810, 94360702, 26275734, 82052050, 30543215, 11543098, 63506281, 80014588, 36845587, 62936963, 61741594, 38365584, 92541302, 15535065, 45428665, 6793819, 18411915, 54868730, 99690194, 12348118, 71522968, 79136082, 6808825, 41245325, 68644627, 49641577, 68316156, 79806380, 76960303, 4069912, 50188404, 18806856, 78300864, 60955663, 20140249, 11757872, 37166608, 31727379, 84187166, 26744917, 45996863, 36396314, 50702367, 52261574, 22129328, 99224392, 46540998, 54058788, 14220886, 2917920, 19457589, 79191827, 92803766, 41092172, 33336148, 97561745, 29516592, 78218845, 21533347, 69641513, 43357947, 26998766, 68102437, 48395186, 88653118, 66250369, 28787861, 28796059, 53842979, 78549759, 68816413, 89419466, 36933038, 37280276, 47893286, 6497830, 54663246, 17857111, 65338021, 38658347, 12571310, 78589145, 13348726, 98648327, 10649306, 4204661, 36685023, 76671482, 8913721, 874791, 84293052, 46851987, 1239555, 37560164, 23110625, 89214616, 22766820, 70541760, 56461322, 4508700, 71316369, 24915585, 7687278, 99021067, 9257405, 94711842, 74441448, 14731700, 20002147, 67030811, 30218878, 73021291, 59501487, 11581181, 50567636, 45848907, 84127901, 82779622, 21397057, 71657078, 40534591, 40152546, 34432810, 67793644, 67124282, 62452034, 84349107, 72725103, 26102057, 14947650, 80251430, 19891772, 44348328, 88698958, 51715482, 78785507, 6221471, 3088684, 95010552, 26397786, 13862149, 63628376, 22200378, 90061527, 17365188, 59614746, 1197320, 24058273, 70596786, 8807940, 33061250, 1569515, 76170907, 19486173, 64157906, 77377183, 75777973, 73168565, 21289531, 37183543, 98371444, 70372191, 69786756, 42073124, 83747892, 56473732, 97379790, 81774825, 82651278, 90654836, 69355476, 91938191, 1204161, 40356877, 12024238, 47298834, 1375023, 57658654, 9603598, 97783876, 77413012, 89078848, 29819940, 96318094, 97057187, 16387575, 5267545, 36139942, 4434662, 17539625, 72274002, 58749, 91141395, 60106217, 30366150, 65454636, 72495719, 81898046, 44844121, 68110330, 52613508, 86301513, 39801351, 85117092, 42307449, 66322959, 415901, 32590267, 96906410, 50007421, 33699435, 14045383, 72732205, 65047700, 83155442, 96709982, 99515901, 34946859, 83533741, 66667729, 60430369, 3183975, 36468541, 91990218, 89046466, 15064655, 77301523, 4798568, 48892201, 19272365, 1431742, 17385531, 79821897, 97641116, 45049308, 27411561, 84840549, 99658235, 62357986, 4515343, 62490109, 37224844, 80555751, 95957797, 20642888, 70800879, 55770687, 16567550, 15902805, 20765474 +41092102, 83210802, 83533741, 51715482, 36312813, 53547802, 85695762, 55753905, 65880522, 64157906, 66116458, 55143724, 66319530, 99549373, 57961282, 81274566, 67451935, 62232211, 12571310, 68875490, 20765474, 11543098, 34295794, 66885828, 63152504, 55247455, 30218878, 66663942, 44060493, 46540998, 54058788, 9886593, 82339363, 4806458, 30139692, 17068582, 76315420, 6038457, 9259676, 83150534, 91990218, 14723410, 20836893, 20486294, 20867149, 59197747, 48360198, 86301513, 73168565, 61263068, 78909561, 415901, 32590267, 92692283, 71469330, 36808486, 51464002, 49641577, 71920426, 33935899, 39201414, 10264691, 60393039, 74452589, 86022504, 99224392, 669105, 6521313, 90090964, 44550764, 92353856, 56531125, 29065964, 18833224, 45075471, 38645117, 51588015, 27411561, 35996293, 69579137, 17539625, 98462867, 96852321, 22450468, 91141395, 93270084, 66250369, 30998561, 3183975, 47893286, 93515664, 4978543, 10309525, 22997281, 89996536, 83789391, 70727211, 79738755, 5482538, 22108665, 69255765, 5640302, 92867155, 82979980, 93933709, 86543538, 16097038, 48088883, 55189057, 66322959, 30653863, 37560164, 9188443, 44889423, 47708850, 71522968, 2204165, 73109997, 91664334, 36135, 82651278, 90654836, 72732205, 4119747, 65047700, 65074535, 56153345, 50668599, 29994197, 40356877, 77284619, 20140249, 57658654, 68939068, 39986008, 45381876, 40686254, 82726008, 71657078, 69697787, 68824981, 93057697, 60176618, 41092172, 22435353, 94090109, 19939935, 87160386, 78785507, 49882705, 75153252, 53084256, 6221471, 48395186, 54762643, 62357986, 73235980, 7229550, 20568363, 66903004, 21919959, 61271144, 42237907, 84406788, 10358899, 15075176, 65454636, 62430984, 84166196, 16595436, 65715134, 81898046, 68110330, 54263880, 30787683, 78589145, 61928316, 95395112, 31161687, 61728685, 21289531, 60102965, 4204661, 89804152, 4798568, 61741594, 29188588, 39847321, 18411915, 77694700, 29029316, 23740167, 59329511, 74357852, 16424199, 41481685, 61859143, 40781449, 92692978, 24314885, 7687278, 50188404, 31491938, 64055960, 83083131, 40781854, 5822202, 9603598, 55669657, 24168411, 87710366, 62428472, 99333375, 84187166, 67513640, 45996863, 17386996, 1250437, 22129328, 17727650, 82779622, 29819940, 5832946, 247198, 10597197, 44983451, 72278539, 22721500, 2717150, 74743862, 82897371, 68694897, 62693428, 99125126, 57020481, 61960400, 26664538, 23848565, 59109400, 76825057, 92787493, 9058407, 66137019, 2891150, 33336148, 45667668, 57241521, 33431960, 19891772, 29510992, 3233084, 25842078, 75128745, 7104732, 9623492, 55602660, 74137926, 51315460, 59981773, 36468541, 36933038, 30366150, 63628376, 64848072, 36189527, 44915235, 9860195, 53802686, 32250045, 27325428, 81677380, 34236719, 1197320, 26734892, 67227442, 3633375, 45794415, 53393358, 19898053, 73392814, 44844121, 15902805, 12416768, 99604946, 38061439, 33061250, 19486173, 67031644, 68000591, 77312810, 37183543, 42967683, 43933006, 7300047, 40197395, 76703609, 51507425, 36505482, 37739481, 35192533, 49597667, 92541302, 85116755, 98371444, 92604458, 54868730, 32426752, 92998591, 70541760, 7066775, 55960386, 56461322, 25636669, 4508700, 71316369, 68316156, 4091162, 20122224, 28851716, 69355476, 19272365, 66428795, 92033260, 72238278, 45919976, 83302115, 37435892, 20645197, 6819644, 67474219, 73021291, 3294781, 824872, 8128637, 83155442, 96709982, 52261574, 4064751, 32151165, 6871053, 97057187, 66832478, 86821229, 89637706, 61141874, 36753250, 77300457, 71083565, 15148031, 5267545, 91240048, 83651211, 45407418, 17764950, 20642888, 48673079, 4095116, 26102057, 91727510, 8373289, 3509435, 27185644, 33304202, 17894977, 43357947, 90457870, 10961421, 11923835, 34698428, 63372756, 65271999, 66611759, 8055981, 54199160, 26776922, 96735716, 2208785, 60430369, 62115552, 1936762, 42947632, 14093520, 9829782, 48675329, 70036438, 4199704, 11161731, 15948937, 34667286, 72495719, 30764367, 59614746, 38008118, 54663246, 6525948, 61712234, 69605283, 80246713, 8807940, 86460488, 3773993, 61373987, 87598888, 93790285, 37224844, 76170907, 75135448, 64602895, 30569392, 10649306, 62051033, 55850790, 47887470, 29959549, 89217461, 6111563, 3233569, 53666583, 26275734, 72793444, 98653983, 17058722, 30543215, 85117092, 73617245, 65081429, 72614359, 62936963, 7955293, 23110625, 72738685, 6793819, 57163802, 99690194, 70372191, 69786756, 42073124, 33249630, 12303248, 5073754, 12348118, 18663507, 37501808, 18783702, 86118021, 50007421, 33699435, 99297164, 39373729, 34044787, 16971929, 78766358, 7182642, 97379790, 32058615, 52060076, 49271185, 29401781, 72777973, 73786944, 8733068, 74441448, 78300864, 60955663, 4668450, 62496012, 99226875, 61982238, 56424103, 77072625, 79322415, 77187825, 33201905, 17037369, 50567636, 56955985, 37192445, 26292919, 77413012, 36396314, 17081350, 14349098, 48774913, 7502255, 10453030, 79821897, 16380211, 73031054, 91255408, 2917920, 44627776, 16387575, 65038678, 82327024, 61859581, 10366309, 19376156, 84349107, 4787945, 97940276, 59371804, 45790169, 93053405, 26493119, 45617087, 47361209, 58208470, 13231279, 75820087, 84684495, 26863229, 71965942, 35092039, 8099994, 28550822, 61623915, 45237957, 44664587, 3880712, 4515343, 57248122, 75052463, 38494874, 8651647, 98943869, 9860968, 91831487, 85023028, 89419466, 95290172, 36580610, 57393458, 13862149, 51472275, 6497830, 37957788, 90061527, 77272628, 34493392, 32159704, 69848388, 7919588, 24591705, 78884452, 89499542, 53648154, 73222868, 22879907, 38009615, 30395570, 44177011, 31733363, 15064655, 88130087, 13348726, 62552524, 45990383, 3235882, 47213183, 92071990, 8129978, 98739783, 42644903, 29932657, 82886381, 58180653, 42199455, 82052050, 76671482, 66271566, 26063929, 63506281, 13468268, 54427233, 79880247, 84002370, 46851987, 26507214, 44847298, 80555751, 61815223, 19101477, 83948335, 77620120, 51047803, 71300104, 43322743, 42692881, 27187213, 37620363, 99965001, 4099191, 56473732, 17146629, 58751351, 89811711, 22113133, 5970607, 71333116, 29834512, 27041967, 18131876, 96726697, 31126490, 59910624, 81774825, 7517032, 57359924, 24915585, 79806380, 18699206, 63015256, 76960303, 23194618, 18806856, 12024238, 43152977, 86242799, 20002147, 96193415, 54606384, 36780454, 45848907, 76330843, 46870723, 47738185, 30771409, 34946859, 59318837, 99971982, 65851721, 67793644, 168541, 4985896, 14220886, 56515456, 62452034, 65017137, 72358170, 44846932, 36812683, 45049308, 526217, 92530431, 69136837, 8115266, 90158785, 70800879, 4434662, 33797252, 49598724, 68041839, 97561745, 29516592, 8634541, 81855445, 92215320, 44348328, 17957593, 90013093, 68102437, 99658235, 28928175, 58749, 88653118, 66667729, 79657802, 53842979, 91802888, 78549759, 33628349, 68816413, 75229462, 26397786, 22200378, 749283, 28734791, 59405277, 20885148, 51590803, 17365188, 64098930, 98130363, 17857111, 15015906, 81853704, 20681921, 65338021, 55770687, 21993752, 24619760, 99861373, 24826575, 77377183, 9575954, 75407347, 39801351, 13819358, 65275241, 62803858, 60581278, 1022677, 77301523, 33553959, 52293995, 8913721, 84293052, 84904436, 30463802, 48892201, 38365584, 73124510, 45428665, 29635537, 59177718, 7646095, 6505939, 47090124, 7011964, 8791066, 33237508, 68644627, 38022834, 2607799, 88047921, 14045383, 16054533, 77898274, 90004325, 27625735, 74110882, 27665211, 91938191, 14363867, 4069912, 44479073, 68204242, 95726235, 47298834, 99524975, 14731700, 67030811, 98948034, 72357096, 70420215, 83269727, 31727379, 26744917, 53632373, 57240218, 50702367, 17385531, 21397057, 96318094, 45995325, 94076128, 97641116, 30163921, 5111370, 94911072, 88904910, 54517921, 99917599, 44481640, 40677414, 90310261, 13470059, 23432750, 22405120, 94516935, 14947650, 90272749, 80251430, 21533347, 69641513, 21001913, 72274002, 16445503, 72019362, 88251446, 3487592, 58224549, 3088684, 33895336, 66045587, 49328608, 508198, 82427263, 78602717, 81805959, 30694952, 60106217, 47792865, 14326617, 95010552, 70782102, 88444207, 67084351, 54014062, 49236559, 6157724, 14626618, 68128438, 15163258, 6153406, 35456853, 82178706, 22942635, 30891921, 76434144, 98648327, 52613508, 63059024, 569864, 23134715, 39171895, 51149804, 42307449, 16019925, 55615885, 62740044, 36845587, 91957544, 41380093, 96906410, 74724075, 44245960, 70367851, 79136082, 83747892, 18504197, 51426311, 41442762, 77787724, 52204879, 43376279, 54232247, 1204161, 9257405, 72373496, 51466049, 56484900, 1431742, 11757872, 38256849, 15536795, 57803235, 6986898, 40152546, 37891451, 50806615, 61897582, 53888755, 54987042, 11365791, 8696647, 45306070, 32274392, 39553046, 79191827, 32161669, 72725103, 35512853, 85242963, 79942022, 97281847, 44473167, 13173644, 88698958, 95581843, 28796059, 55470718, 26392416, 92398073, 94539824, 32699744, 24058273, 70596786, 40984766, 1569515, 44784505, 78561158, 7423788, 43045786, 17016156, 37675718, 94360702, 874791, 1239555, 15535065, 75986488, 16099750, 48658605, 7685448, 89214616, 63967300, 22766820, 6808825, 41245325, 67281495, 29466406, 37659250, 86798033, 87720882, 85571389, 16567550, 1375023, 59501487, 34497327, 36930650, 11581181, 33565483, 12664567, 84127901, 99515901, 48893685, 9398733, 67124282, 321665, 91281584, 43506672, 43501211, 31904591, 24733232, 76540481, 26139110, 75543508, 38510840, 68702632, 28787861, 42782652, 26114953, 23569917, 1788101, 17976208, 21070110, 62490109, 64087743, 89046466, 40027975, 8729146, 92554120, 75777973, 36685023, 48260151, 9175338, 81172706, 95957797, 85711894, 30811010, 82532312, 16684829, 99021067, 94711842, 62762857, 97783876, 37166608, 89078848, 94595668, 19457589, 95251277, 83368048, 93566986, 44842615, 36139942, 18001617, 78218845, 84840549, 26998766, 40865610, 96420429, 18466635, 97011160, 38658347, 21673260, 176257, 30090481, 33123618, 99729357, 2331773, 112651, 93562257, 24953498, 89699445, 40534591, 74614639, 9599614, 92803766, 86001008, 93359396, 37280276, 34698463, 80014588, 34432810, 16405341, 80316608, 79922758, 38341669, 83133790, 70004753 +168541, 71522968, 57393458, 60581278, 62428472, 4064751, 47738185, 40534591, 79191827, 44481640, 90013093, 6221471, 17365188, 24058273, 20765474, 66322959, 84002370, 71300104, 90654836, 51466049, 3294781, 46540998, 669105, 99125126, 45407418, 78218845, 69641513, 66250369, 21919959, 42237907, 10358899, 48675329, 55770687, 44844121, 19486173, 77377183, 33123618, 7300047, 72793444, 2331773, 62936963, 32590267, 6793819, 95957797, 16567550, 20002147, 66319530, 11581181, 44550764, 92353856, 526217, 19376156, 33336148, 57961282, 2208785, 4515343, 10309525, 22997281, 22879907, 62490109, 80246713, 13348726, 7423788, 31161687, 569864, 6111563, 16097038, 4204661, 3233569, 61741594, 38365584, 73124510, 98371444, 70372191, 29029316, 78766358, 16684829, 39201414, 73786944, 4668450, 30218878, 61982238, 37166608, 84187166, 45996863, 57240218, 5832946, 11365791, 19457589, 45075471, 26664538, 36139942, 92803766, 8115266, 90158785, 20642888, 83533741, 4434662, 33797252, 90272749, 94090109, 13173644, 92215320, 21001913, 11923835, 34698428, 66611759, 58224549, 3088684, 66667729, 53842979, 66045587, 96420429, 60430369, 7229550, 23569917, 1936762, 14093520, 95010552, 36468541, 1788101, 91990218, 30366150, 51590803, 61712234, 3633375, 73392814, 21673260, 99604946, 54263880, 44177011, 70727211, 24826575, 8129978, 73168565, 21289531, 86543538, 42199455, 11543098, 55615885, 80555751, 78909561, 48892201, 49597667, 15535065, 85116755, 16099750, 92604458, 96906410, 77694700, 74724075, 2204165, 38022834, 72373496, 40356877, 45919976, 18806856, 31491938, 78300864, 83083131, 824872, 96193415, 97783876, 41092102, 56955985, 96709982, 82726008, 96318094, 10597197, 90090964, 2917920, 9398733, 9886593, 39553046, 99917599, 24733232, 5267545, 84349107, 91240048, 83210802, 90310261, 27411561, 59371804, 22435353, 49598724, 30139692, 80251430, 84684495, 84840549, 28550822, 93270084, 3880712, 91831487, 85023028, 59981773, 84406788, 51472275, 70036438, 4199704, 36189527, 6157724, 18466635, 97011160, 15015906, 21070110, 38658347, 86460488, 3773993, 70004753, 12571310, 5482538, 67031644, 68000591, 61928316, 66116458, 86301513, 92554120, 47887470, 1022677, 60102965, 55189057, 82052050, 85117092, 26063929, 30653863, 65081429, 26507214, 19101477, 415901, 29188588, 29635537, 18411915, 112651, 12303248, 70367851, 7646095, 79136082, 6808825, 70541760, 7011964, 33699435, 71316369, 68644627, 52204879, 31126490, 16054533, 68316156, 54232247, 79806380, 68204242, 10264691, 47298834, 1375023, 99524975, 43152977, 1431742, 86242799, 56424103, 86022504, 24168411, 83269727, 31727379, 39986008, 77413012, 84127901, 76330843, 17727650, 30771409, 34946859, 16380211, 4985896, 22721500, 14220886, 72358170, 18833224, 15148031, 35512853, 99549373, 22405120, 70800879, 26102057, 35996293, 94516935, 75543508, 45617087, 18001617, 47361209, 17539625, 98462867, 19939935, 72274002, 90457870, 78785507, 58749, 91141395, 508198, 79922758, 62115552, 38494874, 98943869, 55470718, 75229462, 83150534, 36933038, 93515664, 15075176, 27325428, 30764367, 34236719, 38008118, 69605283, 35456853, 53648154, 22942635, 30891921, 40984766, 68110330, 38061439, 176257, 20867149, 31733363, 75135448, 65880522, 64157906, 44784505, 3235882, 95395112, 42644903, 38341669, 62051033, 92867155, 75777973, 61728685, 33553959, 94360702, 48088883, 26275734, 98653983, 40197395, 76671482, 52293995, 874791, 84904436, 46851987, 4798568, 35192533, 30463802, 66885828, 5073754, 27187213, 44245960, 47708850, 18663507, 51426311, 99965001, 4099191, 56473732, 47090124, 34044787, 89811711, 77787724, 71333116, 91664334, 88047921, 55143724, 41481685, 57359924, 72732205, 69355476, 27665211, 18699206, 91938191, 7687278, 86798033, 50188404, 9257405, 8733068, 12024238, 56484900, 60955663, 6819644, 99226875, 34497327, 9603598, 11757872, 62762857, 54606384, 36780454, 33201905, 45381876, 89078848, 36396314, 57803235, 40686254, 17385531, 44060493, 29819940, 54058788, 79821897, 37891451, 247198, 97641116, 53888755, 72278539, 69697787, 67124282, 56531125, 65017137, 29065964, 91281584, 36753250, 16387575, 71083565, 83368048, 82339363, 32161669, 51588015, 92787493, 17764950, 14947650, 45667668, 8634541, 57241521, 81855445, 13231279, 83133790, 17894977, 35092039, 17068582, 22450468, 26998766, 28928175, 75153252, 63372756, 65271999, 8055981, 9623492, 36312813, 28796059, 30998561, 96735716, 82427263, 26114953, 33628349, 26392416, 36580610, 13862149, 63628376, 37280276, 9829782, 4978543, 65454636, 11161731, 14626618, 68128438, 34493392, 15163258, 59614746, 84166196, 6525948, 14723410, 81853704, 67227442, 16595436, 32699744, 19898053, 15902805, 12416768, 30787683, 93790285, 40027975, 37224844, 75407347, 63059024, 10649306, 98739783, 13819358, 55850790, 29932657, 61263068, 37183543, 42967683, 66271566, 8913721, 99729357, 73617245, 72614359, 41380093, 92541302, 39847321, 57163802, 9188443, 99690194, 63967300, 33249630, 42692881, 37620363, 37501808, 93562257, 73109997, 18504197, 41245325, 67281495, 23740167, 55960386, 18783702, 50007421, 8791066, 99297164, 22113133, 59329511, 5970607, 2607799, 74357852, 36135, 7517032, 61859143, 27625735, 33935899, 65047700, 19272365, 14363867, 87720882, 94711842, 60393039, 55247455, 64055960, 67474219, 73021291, 72357096, 70420215, 57658654, 26744917, 50567636, 37192445, 8128637, 83155442, 7502255, 10453030, 32151165, 6871053, 94076128, 97057187, 73031054, 91255408, 2717150, 82897371, 48893685, 8696647, 62452034, 36812683, 65038678, 32274392, 10366309, 59109400, 85242963, 91727510, 93053405, 33431960, 3509435, 88698958, 26863229, 71965942, 38510840, 16445503, 93359396, 72019362, 61623915, 10961421, 28787861, 57248122, 3183975, 81805959, 78549759, 9259676, 8651647, 60106217, 95290172, 61271144, 44915235, 77272628, 81677380, 62430984, 69848388, 20681921, 65715134, 6153406, 45794415, 70596786, 89499542, 20486294, 82178706, 81898046, 24619760, 87598888, 15064655, 48360198, 76434144, 30090481, 98648327, 45990383, 47213183, 92071990, 69255765, 30569392, 43045786, 39801351, 17016156, 29959549, 89804152, 34698463, 76703609, 63506281, 62740044, 7955293, 1239555, 45428665, 9175338, 51047803, 32426752, 42073124, 92998591, 22766820, 63152504, 25636669, 17146629, 58751351, 43376279, 96726697, 7182642, 97379790, 16424199, 49641577, 82651278, 52060076, 20122224, 30811010, 92692978, 49271185, 44479073, 50668599, 83302115, 37435892, 24953498, 20645197, 74441448, 40781854, 14731700, 59501487, 5822202, 66663942, 36930650, 99333375, 17037369, 67513640, 45848907, 12664567, 53632373, 50702367, 17386996, 52261574, 99224392, 71657078, 34432810, 94595668, 50806615, 30163921, 6521313, 74743862, 68694897, 321665, 61141874, 61960400, 44842615, 38645117, 83651211, 72725103, 4787945, 4806458, 16405341, 9058407, 76540481, 2891150, 80316608, 29516592, 8373289, 3233084, 33304202, 43357947, 8099994, 51715482, 88251446, 99658235, 40865610, 53084256, 3487592, 7104732, 44664587, 95581843, 79657802, 33895336, 49328608, 42782652, 91802888, 6038457, 74137926, 51315460, 20568363, 75052463, 66903004, 81274566, 89419466, 88444207, 67084351, 47893286, 17976208, 9860195, 34667286, 54663246, 64098930, 98130363, 17857111, 7919588, 53547802, 53393358, 78884452, 73222868, 62232211, 8807940, 38009615, 30395570, 61373987, 83789391, 79738755, 8729146, 76170907, 55753905, 64602895, 78589145, 22108665, 9575954, 5640302, 82886381, 77301523, 93933709, 58180653, 30543215, 36685023, 84293052, 36505482, 37560164, 92692283, 7685448, 89214616, 69786756, 6505939, 83747892, 7066775, 86118021, 39373729, 29466406, 29834512, 27041967, 32058615, 77898274, 90004325, 4091162, 24915585, 82532312, 24314885, 1204161, 56153345, 85571389, 20140249, 74452589, 38256849, 87710366, 89699445, 33565483, 15536795, 1250437, 46870723, 82779622, 45995325, 40152546, 99971982, 67793644, 61897582, 56515456, 5111370, 44627776, 45049308, 43501211, 54517921, 92530431, 82327024, 93057697, 69136837, 60176618, 41092172, 48673079, 44473167, 69579137, 17957593, 27185644, 87160386, 48395186, 88653118, 68702632, 30694952, 42947632, 26397786, 749283, 28734791, 67451935, 37957788, 59405277, 20885148, 15948937, 32159704, 64087743, 89046466, 1569515, 88130087, 62552524, 52613508, 78561158, 37675718, 89217461, 82979980, 23134715, 43933006, 51149804, 42307449, 16019925, 54427233, 51507425, 80014588, 79880247, 36845587, 37739481, 83948335, 91957544, 23110625, 34295794, 75986488, 48658605, 59177718, 12348118, 36808486, 56461322, 33237508, 85711894, 14045383, 59910624, 81774825, 40781449, 28851716, 4119747, 71920426, 74110882, 66428795, 65074535, 63015256, 23194618, 29994197, 77284619, 62496012, 77072625, 79322415, 26292919, 22129328, 59318837, 65851721, 89637706, 43506672, 61859581, 9599614, 76825057, 13470059, 79942022, 97940276, 68041839, 97281847, 97561745, 19891772, 58208470, 75820087, 44348328, 96852321, 86001008, 25842078, 76315420, 68102437, 62357986, 45237957, 54199160, 26776922, 9860968, 47792865, 22200378, 32250045, 90061527, 65338021, 21993752, 68875490, 65275241, 62803858, 39171895, 53666583, 44847298, 61815223, 72738685, 54868730, 81172706, 71469330, 51464002, 4508700, 37659250, 76960303, 72238278, 17081350, 21397057, 66832478, 45306070, 62693428, 44846932, 88904910, 77300457, 95251277, 74614639, 31904591, 68824981, 40677414, 66137019, 45790169, 26493119, 29510992, 21533347, 54762643, 73235980, 55602660, 78602717, 68816413, 14326617, 6497830, 72495719, 89996536, 1197320, 20836893, 99861373, 59197747, 17058722, 77620120, 44889423, 18131876, 72777973, 95726235, 67030811, 77187825, 55669657, 99515901, 86821229, 44983451, 93566986, 23848565, 23432750, 49882705, 75128745, 54014062, 49236559, 64848072, 53802686, 94539824, 85695762, 33061250, 77312810, 48260151, 13468268, 43322743, 16971929, 99021067, 98948034, 68939068, 48774913, 54987042, 94911072, 57020481, 4095116, 26139110, 70782102, 26734892, 92033260, 4069912, 29401781, 6986898, 92398073, 24591705, 41442762, 14349098 +37560164, 65047700, 35512853, 79922758, 61928316, 23110625, 24733232, 4787945, 3509435, 40865610, 75052463, 95010552, 17976208, 8807940, 13348726, 78561158, 55189057, 73617245, 48892201, 92692283, 4099191, 38022834, 85711894, 69355476, 68204242, 51466049, 83302115, 12024238, 64055960, 33201905, 37192445, 54987042, 17764950, 49598724, 78785507, 26114953, 53802686, 34236719, 69605283, 44844121, 33061250, 37224844, 92554120, 77312810, 42199455, 37739481, 29188588, 98371444, 71333116, 63015256, 44479073, 94711842, 20645197, 73021291, 74452589, 86022504, 11581181, 45996863, 57803235, 96318094, 90090964, 69697787, 56531125, 44846932, 44627776, 59109400, 4806458, 35996293, 80316608, 90272749, 13173644, 29510992, 22450468, 54762643, 66250369, 82427263, 4515343, 81274566, 47792865, 63628376, 9860195, 92398073, 54663246, 73392814, 22879907, 21673260, 24619760, 64602895, 76434144, 67031644, 52613508, 92071990, 65275241, 39171895, 30543215, 52293995, 34698463, 16019925, 65081429, 78909561, 57163802, 12348118, 29029316, 37620363, 93562257, 56461322, 47090124, 17146629, 59329511, 29466406, 24915585, 54232247, 56153345, 72238278, 99524975, 24953498, 824872, 37166608, 62428472, 53632373, 17081350, 6871053, 10597197, 30163921, 82897371, 56515456, 72358170, 61141874, 39553046, 93057697, 91240048, 45407418, 70800879, 75543508, 45617087, 80251430, 69641513, 84684495, 49882705, 10961421, 3183975, 78602717, 23569917, 30694952, 61271144, 84406788, 48675329, 4199704, 36189527, 44915235, 67451935, 4978543, 15075176, 51590803, 38008118, 15015906, 67227442, 38658347, 73222868, 22942635, 54263880, 20867149, 55753905, 9575954, 44784505, 8129978, 5640302, 98739783, 20765474, 73168565, 21289531, 1022677, 37675718, 72793444, 98653983, 58180653, 76703609, 84904436, 84002370, 35192533, 19101477, 6793819, 9188443, 96906410, 12303248, 18663507, 36808486, 41245325, 51426311, 56473732, 25636669, 58751351, 39373729, 71316369, 52204879, 91664334, 31126490, 52060076, 30811010, 27625735, 71920426, 92033260, 76960303, 23194618, 40356877, 60955663, 67474219, 61982238, 34497327, 41092102, 50567636, 67513640, 45848907, 17386996, 48774913, 59318837, 37891451, 168541, 669105, 44983451, 6521313, 2917920, 94911072, 36812683, 43506672, 45075471, 82339363, 19376156, 92803766, 23432750, 92787493, 41092172, 79942022, 97940276, 66137019, 91727510, 93053405, 4434662, 68041839, 13231279, 75820087, 25842078, 90013093, 8099994, 16445503, 93359396, 72019362, 84840549, 26998766, 88251446, 28550822, 75153252, 65271999, 3088684, 95581843, 3880712, 508198, 96420429, 55602660, 62115552, 78549759, 20568363, 66903004, 60106217, 42947632, 1788101, 36580610, 57393458, 93515664, 28734791, 32250045, 72495719, 30764367, 98130363, 26734892, 16595436, 65715134, 3633375, 53547802, 70596786, 21993752, 35456853, 30891921, 40984766, 99604946, 89046466, 61373987, 87598888, 24826575, 5482538, 77377183, 75407347, 63059024, 31161687, 92867155, 93933709, 16097038, 17058722, 82052050, 76671482, 42307449, 63506281, 66322959, 874791, 79880247, 30653863, 26507214, 7955293, 83948335, 29635537, 9175338, 16099750, 51047803, 42073124, 44245960, 47708850, 81172706, 73109997, 63152504, 55960386, 33699435, 8791066, 29834512, 43376279, 74357852, 18699206, 66428795, 4069912, 39201414, 47298834, 55247455, 78300864, 83083131, 66319530, 77284619, 30218878, 77072625, 79322415, 77187825, 54606384, 24168411, 31727379, 56955985, 84127901, 96709982, 17385531, 99515901, 82726008, 99224392, 4064751, 5832946, 7502255, 46540998, 79821897, 97057187, 50806615, 66832478, 2717150, 74743862, 92353856, 67124282, 62693428, 65017137, 36753250, 18833224, 32274392, 79191827, 99917599, 26664538, 32161669, 10366309, 5267545, 84349107, 83210802, 60176618, 26102057, 27411561, 59371804, 22435353, 97281847, 44473167, 94090109, 47361209, 98462867, 26863229, 3233084, 76315420, 87160386, 61623915, 48395186, 73235980, 44664587, 93270084, 66667729, 36312813, 28796059, 53842979, 60430369, 81805959, 1936762, 89419466, 83150534, 88444207, 64848072, 15948937, 22997281, 34493392, 62430984, 89996536, 84166196, 20486294, 53648154, 81898046, 38061439, 30395570, 1569515, 176257, 76170907, 65880522, 64157906, 30090481, 68875490, 10649306, 42644903, 38341669, 62051033, 75777973, 48088883, 3233569, 26275734, 8913721, 80014588, 84293052, 62936963, 30463802, 92541302, 39847321, 71300104, 89214616, 99690194, 70372191, 43322743, 42692881, 2204165, 37501808, 99965001, 86118021, 33237508, 68644627, 77787724, 18131876, 55143724, 49641577, 81774825, 7517032, 82651278, 57359924, 4119747, 82532312, 27665211, 1204161, 65074535, 50668599, 16567550, 1431742, 4668450, 11757872, 55669657, 17037369, 36396314, 57240218, 76330843, 46870723, 29819940, 71657078, 99971982, 4985896, 11365791, 48893685, 9398733, 5111370, 91281584, 54517921, 83368048, 82327024, 69136837, 16405341, 14947650, 33431960, 21533347, 17957593, 71965942, 38510840, 21001913, 33304202, 35092039, 43357947, 53084256, 34698428, 7104732, 33895336, 42782652, 91802888, 74137926, 38494874, 98943869, 9860968, 85023028, 95290172, 36468541, 26397786, 26392416, 67084351, 91990218, 13862149, 22200378, 10309525, 18466635, 34667286, 27325428, 77272628, 81677380, 15163258, 32699744, 65338021, 45794415, 62232211, 62490109, 80246713, 30787683, 31733363, 99861373, 93790285, 12571310, 59197747, 19486173, 78589145, 60581278, 33123618, 77301523, 37183543, 42967683, 7300047, 53666583, 36685023, 66271566, 26063929, 13468268, 55615885, 46851987, 44847298, 38365584, 41380093, 77620120, 34295794, 66885828, 71469330, 23740167, 18783702, 99297164, 22113133, 27041967, 59910624, 36135, 37659250, 92692978, 24314885, 86798033, 16684829, 50188404, 72777973, 9257405, 45919976, 95726235, 60393039, 37435892, 86242799, 14731700, 67030811, 98948034, 56424103, 87710366, 36780454, 83269727, 89699445, 33565483, 12664567, 1250437, 52261574, 17727650, 82779622, 45995325, 94076128, 94595668, 65851721, 67793644, 72278539, 22721500, 14220886, 88904910, 45049308, 16387575, 23848565, 36139942, 8115266, 90158785, 83651211, 51588015, 76540481, 33336148, 26493119, 97561745, 29516592, 8373289, 57241521, 19891772, 81855445, 92215320, 96852321, 27185644, 17894977, 90457870, 68102437, 28928175, 91141395, 62357986, 58224549, 8055981, 45237957, 68702632, 79657802, 66045587, 6038457, 51315460, 59981773, 14326617, 70782102, 30366150, 10358899, 49236559, 20885148, 17365188, 97011160, 94539824, 6525948, 1197320, 7919588, 20836893, 24591705, 55770687, 85695762, 21070110, 38009615, 12416768, 86460488, 64087743, 70727211, 8729146, 22108665, 62552524, 45990383, 3235882, 43045786, 55850790, 61263068, 569864, 23134715, 86543538, 43933006, 60102965, 4204661, 40197395, 11543098, 48260151, 61815223, 61741594, 415901, 91957544, 32590267, 73124510, 72738685, 75986488, 7685448, 112651, 63967300, 74724075, 70367851, 7646095, 51464002, 6808825, 70541760, 89811711, 95957797, 16971929, 7182642, 16054533, 68316156, 90004325, 40781449, 74110882, 91938191, 19272365, 87720882, 18806856, 1375023, 74441448, 72357096, 20140249, 96193415, 70420215, 97783876, 99333375, 84187166, 68939068, 26744917, 45381876, 15536795, 89078848, 40686254, 22129328, 54058788, 32151165, 40152546, 34432810, 97641116, 86821229, 91255408, 89637706, 99125126, 19457589, 526217, 77300457, 95251277, 71083565, 92530431, 31904591, 15148031, 44842615, 9599614, 90310261, 38645117, 20642888, 48673079, 85242963, 2891150, 26139110, 69579137, 58208470, 44348328, 88698958, 86001008, 72274002, 99658235, 63372756, 6221471, 9623492, 28787861, 54199160, 30998561, 33628349, 8651647, 91831487, 14093520, 42237907, 47893286, 749283, 51472275, 59405277, 6157724, 11161731, 90061527, 68128438, 32159704, 14723410, 17857111, 61712234, 81853704, 78884452, 89499542, 44177011, 70004753, 79738755, 75135448, 48360198, 88130087, 47213183, 66116458, 39801351, 62803858, 29932657, 82886381, 54427233, 2331773, 72614359, 49597667, 15535065, 45428665, 92604458, 69786756, 92998591, 33249630, 22766820, 77694700, 6505939, 79136082, 18504197, 7066775, 7011964, 34044787, 96726697, 78766358, 88047921, 16424199, 4091162, 90654836, 79806380, 33935899, 7687278, 99021067, 72373496, 85571389, 56484900, 40781854, 62496012, 59501487, 66663942, 9603598, 38256849, 14349098, 34946859, 40534591, 247198, 16380211, 44550764, 321665, 29065964, 57020481, 65038678, 44481640, 61859581, 72725103, 22405120, 4095116, 94516935, 45790169, 8634541, 17539625, 17068582, 75128745, 3487592, 66611759, 88653118, 96735716, 7229550, 68816413, 55470718, 75229462, 21919959, 36933038, 70036438, 14626618, 69848388, 24058273, 19898053, 15902805, 3773993, 83789391, 40027975, 98648327, 68000591, 69255765, 30569392, 61728685, 89217461, 94360702, 51149804, 85117092, 51507425, 1239555, 18411915, 54868730, 32426752, 27187213, 83747892, 67281495, 50007421, 4508700, 5970607, 77898274, 49271185, 14363867, 29401781, 29994197, 73786944, 10264691, 8733068, 20002147, 6819644, 5822202, 57658654, 62762857, 77413012, 50702367, 47738185, 21397057, 44060493, 62452034, 9886593, 43501211, 61960400, 40677414, 9058407, 18001617, 30139692, 19939935, 83133790, 51715482, 57248122, 54014062, 9829782, 37957788, 59614746, 64098930, 53393358, 86301513, 7423788, 82979980, 6111563, 89804152, 99729357, 4798568, 36505482, 85116755, 48658605, 59177718, 44889423, 71522968, 14045383, 41481685, 61859143, 20122224, 72732205, 3294781, 99226875, 8128637, 26292919, 6986898, 30771409, 10453030, 53888755, 8696647, 74614639, 68824981, 83533741, 78218845, 58749, 9259676, 37280276, 6497830, 20681921, 6153406, 82178706, 95395112, 47887470, 33553959, 62740044, 36845587, 5073754, 41442762, 97379790, 32058615, 28851716, 36930650, 39986008, 73031054, 68694897, 45306070, 13470059, 33797252, 57961282, 26776922, 49328608, 15064655, 29959549, 2607799, 31491938, 43152977, 83155442, 61897582, 93566986, 76825057, 99549373, 11923835, 2208785, 65454636, 68110330, 13819358, 80555751, 17016156, 45667668 +93359396, 15075176, 65081429, 91255408, 67124282, 59371804, 18001617, 57248122, 38494874, 37957788, 31733363, 18131876, 30163921, 13470059, 27185644, 95010552, 27325428, 89046466, 92554120, 84904436, 37739481, 6505939, 22113133, 57359924, 71920426, 56955985, 6521313, 23432750, 17764950, 90457870, 40865610, 3633375, 93790285, 45990383, 33553959, 23134715, 98653983, 92604458, 71333116, 27041967, 74357852, 63015256, 39201414, 7502255, 99125126, 61141874, 61859581, 83210802, 72725103, 16405341, 26493119, 13231279, 25842078, 71965942, 72274002, 28928175, 53084256, 65271999, 45237957, 26114953, 6157724, 92398073, 15163258, 17857111, 80246713, 3773993, 30787683, 99861373, 3235882, 65275241, 86543538, 6111563, 4204661, 55189057, 13468268, 51507425, 1239555, 37560164, 41380093, 92692283, 6793819, 96906410, 5073754, 12348118, 74724075, 44245960, 81172706, 38022834, 7182642, 49641577, 27625735, 37659250, 50668599, 72238278, 1375023, 64055960, 67030811, 66663942, 61982238, 97783876, 24168411, 89699445, 33565483, 17037369, 37192445, 45996863, 84127901, 17081350, 46870723, 17727650, 30771409, 54987042, 44983451, 82897371, 44846932, 36812683, 77300457, 99917599, 15148031, 59109400, 76825057, 70800879, 57241521, 21533347, 43357947, 76315420, 28550822, 6221471, 28796059, 79657802, 42782652, 508198, 7229550, 1936762, 66903004, 14093520, 75229462, 36933038, 26397786, 64848072, 749283, 93515664, 34493392, 54663246, 7919588, 81853704, 53547802, 21673260, 44177011, 83789391, 5482538, 30090481, 78561158, 43045786, 98739783, 55850790, 82886381, 48088883, 89804152, 40197395, 874791, 30653863, 83948335, 71300104, 89214616, 112651, 92998591, 22766820, 77694700, 37620363, 18783702, 77787724, 29466406, 31126490, 77898274, 52060076, 92692978, 82532312, 27665211, 18699206, 14363867, 4069912, 99524975, 74441448, 40781854, 20002147, 41092102, 67513640, 57240218, 14349098, 47738185, 71657078, 37891451, 65851721, 73031054, 53888755, 72278539, 65017137, 45049308, 71083565, 92530431, 93566986, 92803766, 90310261, 38645117, 48673079, 68041839, 44473167, 8373289, 94090109, 47361209, 17957593, 38510840, 22450468, 26998766, 75153252, 73235980, 44664587, 23569917, 51315460, 68816413, 95290172, 70782102, 88444207, 67084351, 42237907, 84406788, 22200378, 4199704, 44915235, 67451935, 11161731, 72495719, 89996536, 59614746, 98130363, 26734892, 24591705, 16595436, 70596786, 69605283, 62232211, 61373987, 70004753, 70727211, 88130087, 76434144, 9575954, 92071990, 7423788, 13819358, 38341669, 1022677, 82979980, 16097038, 39171895, 60102965, 72793444, 52293995, 76703609, 99729357, 26063929, 55615885, 26507214, 44847298, 80555751, 91957544, 77620120, 23110625, 45428665, 29635537, 70372191, 18663507, 7646095, 79136082, 99965001, 50007421, 5970607, 91664334, 88047921, 55143724, 85711894, 14045383, 59910624, 32058615, 68316156, 82651278, 61859143, 4091162, 54232247, 33935899, 56153345, 29994197, 40356877, 45919976, 94711842, 47298834, 31491938, 1431742, 6819644, 3294781, 59501487, 96193415, 36930650, 62762857, 74452589, 54606384, 87710366, 31727379, 6986898, 83155442, 21397057, 6871053, 2717150, 2917920, 62693428, 94911072, 19457589, 526217, 95251277, 43506672, 32274392, 83368048, 44481640, 26664538, 32161669, 82327024, 44842615, 91240048, 26102057, 85242963, 27411561, 93053405, 33797252, 97561745, 90272749, 78218845, 19891772, 58208470, 3509435, 44348328, 69641513, 21001913, 88653118, 62357986, 8055981, 9623492, 28787861, 4515343, 60430369, 6038457, 3183975, 20568363, 47792865, 26392416, 30366150, 49236559, 9829782, 47893286, 28734791, 17976208, 32250045, 90061527, 84166196, 1197320, 24058273, 53393358, 38658347, 22879907, 99604946, 13348726, 68000591, 22108665, 86301513, 39801351, 20765474, 75777973, 60581278, 37675718, 42967683, 94360702, 3233569, 26275734, 17058722, 82052050, 76671482, 34698463, 16019925, 79880247, 2331773, 62936963, 36505482, 35192533, 7955293, 34295794, 85116755, 48658605, 54868730, 71469330, 70541760, 86118021, 56473732, 33699435, 17146629, 99297164, 39373729, 33237508, 95957797, 29834512, 16971929, 2607799, 96726697, 16054533, 49271185, 1204161, 19272365, 86798033, 66428795, 83302115, 83083131, 4668450, 98948034, 73021291, 70420215, 77072625, 36780454, 83269727, 11581181, 84187166, 45381876, 8128637, 77413012, 89078848, 40686254, 99515901, 52261574, 22129328, 82779622, 44060493, 79821897, 247198, 97057187, 67793644, 66832478, 669105, 11365791, 69697787, 5111370, 321665, 89637706, 62452034, 44627776, 18833224, 43501211, 54517921, 82339363, 24733232, 9599614, 84349107, 90158785, 83651211, 35512853, 51588015, 20642888, 75543508, 33336148, 45667668, 45617087, 30139692, 69579137, 17539625, 26863229, 86001008, 33304202, 35092039, 8099994, 87160386, 72019362, 75128745, 66667729, 3880712, 33895336, 66045587, 96735716, 96420429, 74137926, 75052463, 98943869, 81274566, 42947632, 61271144, 57393458, 54014062, 10358899, 63628376, 37280276, 70036438, 4978543, 9860195, 20885148, 10309525, 53802686, 18466635, 97011160, 68128438, 14723410, 20836893, 15015906, 89499542, 73222868, 8807940, 38009615, 86460488, 30395570, 54263880, 176257, 20867149, 64602895, 61928316, 62552524, 75407347, 66116458, 63059024, 68875490, 31161687, 92867155, 29932657, 21289531, 47887470, 569864, 7300047, 85117092, 48260151, 84293052, 84002370, 72614359, 36845587, 19101477, 38365584, 32590267, 29188588, 9175338, 9188443, 99690194, 63967300, 33249630, 43322743, 42692881, 27187213, 29029316, 44889423, 47708850, 70367851, 36808486, 23740167, 47090124, 4508700, 41442762, 71316369, 43376279, 41481685, 36135, 20122224, 24915585, 69355476, 76960303, 68204242, 29401781, 10264691, 95726235, 18806856, 56484900, 37435892, 20645197, 78300864, 14731700, 67474219, 20140249, 34497327, 9603598, 77187825, 33201905, 26744917, 36396314, 53632373, 96709982, 82726008, 4064751, 29819940, 46540998, 54058788, 32151165, 45995325, 168541, 86821229, 14220886, 44550764, 48893685, 8696647, 68694897, 56515456, 72358170, 88904910, 91281584, 57020481, 36753250, 61960400, 79191827, 68824981, 93057697, 23848565, 5267545, 36139942, 4787945, 4806458, 41092172, 76540481, 79942022, 35996293, 97940276, 66137019, 91727510, 26139110, 22435353, 49598724, 97281847, 57961282, 84684495, 88698958, 96852321, 3233084, 90013093, 17894977, 17068582, 16445503, 51715482, 49882705, 88251446, 11923835, 63372756, 66611759, 54762643, 58224549, 3088684, 54199160, 79922758, 91802888, 78602717, 78549759, 8651647, 9860968, 85023028, 60106217, 89419466, 36580610, 13862149, 6497830, 59405277, 81677380, 14626618, 32159704, 6525948, 61712234, 67227442, 45794415, 78884452, 21993752, 30891921, 40984766, 64087743, 24619760, 76170907, 55753905, 19486173, 24826575, 48360198, 64157906, 77377183, 52613508, 47213183, 69255765, 42644903, 73168565, 29959549, 37183543, 42199455, 51149804, 73617245, 46851987, 61741594, 415901, 49597667, 92541302, 15535065, 75986488, 39847321, 57163802, 18411915, 69786756, 42073124, 12303248, 66885828, 2204165, 37501808, 73109997, 63152504, 51464002, 7066775, 55960386, 8791066, 58751351, 90004325, 30811010, 28851716, 72732205, 74110882, 92033260, 23194618, 85571389, 16567550, 12024238, 55247455, 77284619, 5822202, 55669657, 38256849, 86022504, 68939068, 50567636, 39986008, 45848907, 50702367, 17385531, 48774913, 40534591, 96318094, 34432810, 50806615, 61897582, 74743862, 92353856, 9398733, 74614639, 39553046, 10366309, 60176618, 9058407, 94516935, 80316608, 4434662, 33431960, 61623915, 53842979, 49328608, 2208785, 81805959, 33628349, 9259676, 91831487, 36468541, 51472275, 77272628, 62430984, 94539824, 34236719, 38008118, 69848388, 65715134, 55770687, 85695762, 20486294, 38061439, 87598888, 79738755, 78589145, 44784505, 8129978, 62803858, 61728685, 61263068, 77301523, 93933709, 43933006, 30543215, 36685023, 66322959, 80014588, 62740044, 4798568, 30463802, 48892201, 73124510, 72738685, 16099750, 51047803, 93562257, 67281495, 51426311, 4099191, 25636669, 89811711, 68644627, 97379790, 16424199, 81774825, 90654836, 91938191, 65047700, 16684829, 50188404, 99021067, 72777973, 72373496, 60393039, 24953498, 86242799, 99226875, 56424103, 62428472, 99333375, 12664567, 57803235, 76330843, 99224392, 10453030, 94076128, 94595668, 99971982, 22721500, 56531125, 29065964, 16387575, 65038678, 31904591, 69136837, 40677414, 8115266, 45407418, 4095116, 14947650, 2891150, 80251430, 13173644, 92215320, 75820087, 98462867, 19939935, 83133790, 99658235, 3487592, 10961421, 91141395, 93270084, 66250369, 36312813, 95581843, 30694952, 55470718, 14326617, 21919959, 15948937, 22997281, 30764367, 20681921, 32699744, 82178706, 81898046, 22942635, 44844121, 68110330, 33061250, 1569515, 75135448, 65880522, 5640302, 62051033, 33123618, 53666583, 58180653, 66271566, 8913721, 42307449, 63506281, 61815223, 98371444, 6808825, 56461322, 59329511, 52204879, 78766358, 7517032, 4119747, 24314885, 65074535, 44479073, 73786944, 51466049, 30218878, 824872, 62496012, 11757872, 37166608, 15536795, 17386996, 1250437, 34946859, 59318837, 40152546, 16380211, 10597197, 97641116, 4985896, 9886593, 19376156, 99549373, 92787493, 83533741, 45790169, 29516592, 8634541, 78785507, 58749, 7104732, 68702632, 30998561, 55602660, 59981773, 83150534, 91990218, 48675329, 36189527, 65454636, 65338021, 6153406, 19898053, 53648154, 15902805, 12416768, 40027975, 37224844, 8729146, 12571310, 59197747, 67031644, 98648327, 30569392, 95395112, 89217461, 54427233, 7685448, 59177718, 32426752, 71522968, 41245325, 34044787, 79806380, 7687278, 9257405, 8733068, 43152977, 66319530, 57658654, 79322415, 5832946, 45306070, 45075471, 22405120, 68102437, 26776922, 82427263, 34667286, 21070110, 62490109, 15064655, 17016156, 11543098, 18504197, 40781449, 87720882, 60955663, 29510992, 84840549, 34698428, 48395186, 1788101, 51590803, 64098930, 73392814, 10649306, 77312810, 26292919, 81855445, 62115552, 17365188, 35456853, 78909561, 83747892, 7011964, 72357096, 90090964 +78785507, 54606384, 65275241, 3233569, 70372191, 78766358, 5832946, 54058788, 50806615, 9886593, 30139692, 40865610, 15163258, 85695762, 80246713, 52613508, 569864, 85116755, 48658605, 112651, 91664334, 16054533, 27665211, 11365791, 49598724, 8099994, 84840549, 63372756, 22997281, 34493392, 20486294, 176257, 78561158, 33123618, 54427233, 874791, 415901, 15535065, 71522968, 56461322, 31126490, 68316156, 7517032, 76960303, 45919976, 1375023, 20002147, 61982238, 36930650, 97783876, 26292919, 83155442, 17764950, 45667668, 17894977, 34698428, 28787861, 85023028, 27325428, 16595436, 19898053, 30891921, 99861373, 55753905, 77377183, 20765474, 77301523, 86543538, 39171895, 17058722, 42199455, 82052050, 48892201, 39847321, 89214616, 92998591, 29029316, 71333116, 82532312, 96193415, 62428472, 50702367, 48893685, 45306070, 89637706, 36812683, 19457589, 24733232, 23848565, 36139942, 19376156, 83210802, 83533741, 14947650, 91727510, 26139110, 29516592, 8373289, 78218845, 21001913, 62357986, 58224549, 3088684, 2208785, 7229550, 74137926, 55470718, 75229462, 83150534, 26397786, 17976208, 32250045, 72495719, 94539824, 34236719, 26734892, 65715134, 6153406, 55770687, 1569515, 5482538, 76434144, 64157906, 67031644, 13348726, 62552524, 8129978, 7423788, 68875490, 42644903, 29932657, 61263068, 77312810, 30543215, 66322959, 79880247, 84904436, 80555751, 1239555, 32590267, 45428665, 18663507, 79136082, 74357852, 32058615, 81774825, 79806380, 24314885, 85571389, 39201414, 95726235, 94711842, 47298834, 37435892, 55247455, 5822202, 62762857, 37166608, 99515901, 82726008, 46870723, 4064751, 47738185, 29819940, 40152546, 94595668, 73031054, 66832478, 44983451, 69697787, 8696647, 67124282, 44846932, 65038678, 45075471, 83368048, 99917599, 82339363, 68824981, 44481640, 15148031, 93057697, 4787945, 48673079, 33336148, 17539625, 19939935, 28928175, 28550822, 53084256, 91141395, 88653118, 45237957, 95581843, 30998561, 9259676, 60106217, 1788101, 30366150, 70036438, 15075176, 14626618, 81853704, 3633375, 12416768, 86460488, 64087743, 70004753, 15064655, 59197747, 64602895, 3235882, 92867155, 73168565, 33553959, 7300047, 4204661, 58180653, 85117092, 26063929, 80014588, 55615885, 46851987, 78909561, 61815223, 72738685, 75986488, 51047803, 99690194, 59177718, 32426752, 33249630, 42692881, 66885828, 77694700, 27187213, 71469330, 7066775, 18783702, 29466406, 16971929, 54232247, 71920426, 91938191, 49271185, 1204161, 86798033, 66428795, 14363867, 4069912, 16684829, 29994197, 16567550, 12024238, 14731700, 30218878, 66663942, 34497327, 57658654, 83269727, 33565483, 50567636, 67513640, 40686254, 76330843, 71657078, 45995325, 10597197, 82897371, 5111370, 56531125, 321665, 526217, 18833224, 61960400, 26664538, 9599614, 5267545, 91240048, 92803766, 40677414, 13470059, 99549373, 4806458, 22405120, 66137019, 33797252, 8634541, 33431960, 13173644, 19891772, 69579137, 13231279, 57961282, 3233084, 76315420, 16445503, 61623915, 75153252, 10961421, 66611759, 44664587, 68702632, 28796059, 53842979, 49328608, 82427263, 6038457, 81805959, 78549759, 8651647, 68816413, 42947632, 61271144, 88444207, 26392416, 42237907, 57393458, 37280276, 9829782, 4199704, 36189527, 44915235, 67451935, 9860195, 6157724, 77272628, 81677380, 59614746, 64098930, 24058273, 53393358, 73392814, 8807940, 68110330, 99604946, 38061439, 24619760, 54263880, 61373987, 20867149, 83789391, 79738755, 76170907, 75135448, 48360198, 65880522, 78589145, 22108665, 45990383, 92071990, 69255765, 43045786, 38341669, 1022677, 89217461, 51149804, 65081429, 37739481, 35192533, 38365584, 73124510, 29188588, 6793819, 9175338, 69786756, 63967300, 12303248, 12348118, 47708850, 2204165, 18504197, 67281495, 33699435, 99297164, 71316369, 18131876, 97379790, 16424199, 36135, 77898274, 82651278, 24915585, 37659250, 92692978, 40356877, 51466049, 18806856, 83302115, 8733068, 99524975, 1431742, 86242799, 4668450, 67474219, 38256849, 86022504, 87710366, 17037369, 37192445, 45848907, 45381876, 8128637, 57240218, 6986898, 1250437, 52261574, 99224392, 17727650, 82779622, 34946859, 79821897, 16380211, 67793644, 168541, 97641116, 61897582, 669105, 86821229, 68694897, 2917920, 9398733, 56515456, 94911072, 29065964, 91281584, 16387575, 77300457, 95251277, 43501211, 54517921, 39553046, 31904591, 82327024, 61859581, 44842615, 69136837, 90310261, 76825057, 35512853, 45407418, 41092172, 59371804, 80316608, 22435353, 4434662, 26493119, 97561745, 90272749, 57241521, 69641513, 88698958, 26863229, 86001008, 90013093, 72274002, 90457870, 99658235, 54199160, 96420429, 60430369, 57248122, 62115552, 26114953, 33628349, 1936762, 59981773, 22200378, 51472275, 48675329, 28734791, 65454636, 11161731, 92398073, 10309525, 15948937, 97011160, 68128438, 30764367, 54663246, 1197320, 53547802, 38658347, 62232211, 81898046, 44844121, 15902805, 30395570, 63059024, 13819358, 17016156, 29959549, 42967683, 94360702, 98653983, 76671482, 66271566, 8913721, 76703609, 13468268, 30653863, 2331773, 62740044, 72614359, 36845587, 62936963, 36505482, 30463802, 41380093, 77620120, 92541302, 16099750, 71300104, 92604458, 96906410, 22766820, 99965001, 17146629, 4508700, 58751351, 22113133, 77787724, 29834512, 96726697, 88047921, 57359924, 20122224, 30811010, 90654836, 27625735, 65047700, 72777973, 73786944, 10264691, 60393039, 43152977, 40781854, 66319530, 62496012, 77072625, 9603598, 11757872, 36780454, 68939068, 39986008, 45996863, 84127901, 57803235, 22129328, 44060493, 7502255, 32151165, 59318837, 6871053, 34432810, 97057187, 30163921, 2717150, 88904910, 44627776, 36753250, 93566986, 90158785, 23432750, 72725103, 20642888, 9058407, 70800879, 85242963, 94516935, 2891150, 75543508, 97281847, 47361209, 81855445, 44348328, 25842078, 71965942, 35092039, 93359396, 6221471, 7104732, 8055981, 9623492, 66250369, 33895336, 26776922, 66045587, 4515343, 79922758, 51315460, 75052463, 81274566, 21919959, 95290172, 95010552, 36933038, 36580610, 54014062, 10358899, 20885148, 51590803, 32159704, 38008118, 6525948, 69848388, 24591705, 61712234, 45794415, 78884452, 21993752, 73222868, 22879907, 38009615, 30787683, 70727211, 88130087, 61928316, 75407347, 30569392, 66116458, 10649306, 98739783, 39801351, 95395112, 31161687, 62051033, 55850790, 60581278, 21289531, 47887470, 6111563, 53666583, 36685023, 99729357, 84002370, 26507214, 83948335, 92692283, 7685448, 5073754, 7646095, 50007421, 47090124, 7011964, 2607799, 85711894, 14045383, 41481685, 90004325, 40781449, 72732205, 4119747, 65074535, 72238278, 56484900, 20645197, 67030811, 3294781, 72357096, 59501487, 99226875, 70420215, 41092102, 24168411, 33201905, 99333375, 84187166, 15536795, 77413012, 46540998, 40534591, 96318094, 37891451, 4985896, 90090964, 44550764, 74743862, 65017137, 99125126, 61141874, 74614639, 32274392, 92530431, 38645117, 51588015, 4095116, 76540481, 35996293, 97940276, 45790169, 93053405, 94090109, 84684495, 33304202, 51715482, 22450468, 87160386, 26998766, 3487592, 11923835, 65271999, 48395186, 93270084, 66667729, 36312813, 508198, 91802888, 3183975, 23569917, 89419466, 14326617, 36468541, 13862149, 6497830, 34667286, 84166196, 98130363, 20836893, 15015906, 89499542, 21070110, 82178706, 62490109, 89046466, 31733363, 93790285, 40027975, 19486173, 98648327, 86301513, 82886381, 43933006, 60102965, 26275734, 89804152, 72793444, 55189057, 40197395, 52293995, 34698463, 63506281, 16019925, 51507425, 73617245, 4798568, 44847298, 19101477, 49597667, 29635537, 98371444, 42073124, 44889423, 44245960, 37620363, 36808486, 51464002, 6808825, 41245325, 70541760, 23740167, 55960386, 86118021, 4099191, 56473732, 25636669, 34044787, 33237508, 59329511, 7182642, 55143724, 59910624, 49641577, 61859143, 52060076, 4091162, 28851716, 69355476, 18699206, 7687278, 92033260, 63015256, 50188404, 56153345, 50668599, 68204242, 72373496, 31491938, 24953498, 60955663, 73021291, 56955985, 89078848, 36396314, 53632373, 96709982, 17385531, 30771409, 94076128, 99971982, 65851721, 53888755, 91255408, 14220886, 62693428, 62452034, 45049308, 43506672, 32161669, 10366309, 8115266, 16405341, 26102057, 29510992, 27185644, 49882705, 58749, 73235980, 9860968, 91831487, 14093520, 47792865, 91990218, 63628376, 49236559, 47893286, 93515664, 4978543, 59405277, 53802686, 18466635, 89996536, 17857111, 7919588, 65338021, 70596786, 53648154, 44177011, 87598888, 37224844, 8729146, 12571310, 24826575, 9575954, 44784505, 5640302, 92554120, 61728685, 37675718, 82979980, 37183543, 16097038, 48088883, 48260151, 7955293, 91957544, 23110625, 34295794, 18411915, 70367851, 73109997, 63152504, 39373729, 89811711, 68644627, 5970607, 95957797, 43376279, 74110882, 33935899, 23194618, 29401781, 64055960, 78300864, 83083131, 6819644, 20140249, 74452589, 11581181, 31727379, 17386996, 21397057, 72278539, 59109400, 83651211, 92787493, 3509435, 98462867, 96852321, 83133790, 38510840, 72019362, 68102437, 75128745, 79657802, 55602660, 70782102, 84406788, 64848072, 37957788, 90061527, 17365188, 14723410, 69605283, 22942635, 21673260, 40984766, 33061250, 3773993, 47213183, 62803858, 75777973, 93933709, 11543098, 42307449, 61741594, 54868730, 81172706, 37501808, 93562257, 6505939, 51426311, 41442762, 38022834, 52204879, 19272365, 99021067, 9257405, 74441448, 824872, 77187825, 26744917, 12664567, 17081350, 48774913, 10453030, 247198, 54987042, 92353856, 57020481, 71083565, 79191827, 80251430, 21533347, 92215320, 17957593, 17068582, 96735716, 42782652, 78602717, 38494874, 98943869, 66903004, 67084351, 749283, 62430984, 67227442, 20681921, 32699744, 35456853, 30090481, 84293052, 37560164, 43322743, 74724075, 27041967, 44479073, 77284619, 56424103, 79322415, 6521313, 72358170, 79942022, 45617087, 18001617, 44473167, 58208470, 43357947, 3880712, 30694952, 20568363, 68000591, 23134715, 57163802, 9188443, 83747892, 98948034, 89699445, 14349098, 22721500, 84349107, 27411561, 75820087, 88251446, 54762643, 8791066, 55669657, 60176618, 68041839, 87720882 +43933006, 874791, 87720882, 8373289, 86301513, 67793644, 91255408, 43357947, 11923835, 66903004, 84406788, 59405277, 84904436, 52060076, 33201905, 83155442, 11365791, 83210802, 60176618, 90158785, 16405341, 9058407, 97561745, 9860968, 70782102, 7423788, 82979980, 80014588, 38365584, 23110625, 7646095, 17146629, 96726697, 14363867, 6819644, 77187825, 55669657, 15536795, 53632373, 65851721, 95251277, 43506672, 40677414, 59109400, 20642888, 68041839, 33431960, 38510840, 3487592, 62115552, 30694952, 75052463, 95290172, 65454636, 18466635, 27325428, 68128438, 17857111, 98648327, 63059024, 5640302, 13819358, 20765474, 31161687, 17016156, 51507425, 72614359, 44847298, 62936963, 45428665, 48658605, 44245960, 93562257, 99965001, 68316156, 61859143, 72732205, 4069912, 12024238, 47298834, 97783876, 40686254, 44550764, 92353856, 45306070, 67124282, 5111370, 65017137, 72358170, 43501211, 61960400, 32274392, 44481640, 24733232, 82327024, 13470059, 26139110, 80316608, 45617087, 80251430, 78218845, 3233084, 25842078, 72274002, 84840549, 26776922, 7229550, 26114953, 33628349, 85023028, 6497830, 28734791, 11161731, 15948937, 51590803, 14626618, 89996536, 30764367, 3633375, 89499542, 20486294, 81898046, 44844121, 80246713, 31733363, 75135448, 5482538, 62803858, 94360702, 39171895, 60102965, 17058722, 84293052, 73617245, 62740044, 37739481, 83948335, 49597667, 37560164, 51047803, 96906410, 70372191, 74724075, 73109997, 83747892, 67281495, 56473732, 58751351, 68644627, 27041967, 77898274, 57359924, 50188404, 72777973, 39201414, 20002147, 3294781, 5822202, 38256849, 77413012, 17727650, 82779622, 21397057, 54058788, 40152546, 2717150, 74743862, 36812683, 93566986, 15148031, 32161669, 93057697, 69136837, 72725103, 99549373, 22405120, 26102057, 27411561, 22435353, 49598724, 26493119, 47361209, 44348328, 27185644, 22450468, 93359396, 62357986, 2208785, 78549759, 20568363, 14093520, 14326617, 1788101, 26397786, 22200378, 34667286, 90061527, 72495719, 97011160, 22997281, 64098930, 98130363, 7919588, 69605283, 73222868, 20867149, 70727211, 15064655, 12571310, 24826575, 78589145, 44784505, 78561158, 98739783, 39801351, 21289531, 23134715, 6111563, 58180653, 30543215, 2331773, 61815223, 61741594, 91957544, 32590267, 72738685, 85116755, 71300104, 43322743, 5073754, 18663507, 51464002, 18504197, 23740167, 47090124, 4508700, 99297164, 89811711, 59329511, 77787724, 2607799, 55143724, 27625735, 37659250, 69355476, 74110882, 49271185, 65047700, 86798033, 65074535, 85571389, 10264691, 8733068, 43152977, 37435892, 20645197, 86242799, 59501487, 33565483, 84187166, 50702367, 14349098, 6871053, 168541, 30163921, 61897582, 86821229, 22721500, 68694897, 9398733, 56531125, 44627776, 91281584, 45049308, 61859581, 44842615, 84349107, 35512853, 4806458, 92787493, 85242963, 91727510, 29516592, 57241521, 21533347, 58208470, 75820087, 96852321, 17894977, 35092039, 26998766, 88251446, 58749, 45237957, 66667729, 95581843, 66045587, 60430369, 57248122, 91802888, 78602717, 81805959, 9259676, 47893286, 4199704, 36189527, 37957788, 92398073, 62430984, 84166196, 54663246, 69848388, 1197320, 15015906, 21673260, 86460488, 3773993, 30395570, 30787683, 87598888, 79738755, 88130087, 64157906, 22108665, 3235882, 52613508, 10649306, 55850790, 60581278, 82886381, 29959549, 1022677, 33553959, 3233569, 42199455, 36685023, 76671482, 11543098, 51149804, 34698463, 99729357, 65081429, 36505482, 19101477, 92541302, 29635537, 9175338, 69786756, 27187213, 12348118, 29029316, 44889423, 63152504, 33699435, 25636669, 41442762, 5970607, 29466406, 31126490, 49641577, 4091162, 20122224, 24915585, 30811010, 4119747, 92692978, 71920426, 24314885, 33935899, 91938191, 7687278, 63015256, 23194618, 9257405, 40356877, 45919976, 83083131, 77284619, 98948034, 72357096, 41092102, 24168411, 89699445, 99333375, 26744917, 50567636, 67513640, 39986008, 8128637, 89078848, 99224392, 46540998, 40534591, 96318094, 37891451, 94595668, 14220886, 6521313, 48893685, 94911072, 44846932, 526217, 79191827, 92530431, 19376156, 90310261, 76825057, 8115266, 23432750, 76540481, 70800879, 94516935, 97940276, 45790169, 93053405, 33336148, 30139692, 19891772, 81855445, 92215320, 57961282, 86001008, 19939935, 71965942, 21001913, 17068582, 87160386, 68102437, 28928175, 53084256, 63372756, 48395186, 73235980, 36312813, 28796059, 3880712, 82427263, 4515343, 55602660, 98943869, 91831487, 60106217, 59981773, 47792865, 55470718, 36468541, 67084351, 42237907, 91990218, 10358899, 51472275, 4978543, 15075176, 81677380, 59614746, 65338021, 70596786, 85695762, 62232211, 22942635, 62490109, 38009615, 61373987, 176257, 93790285, 13348726, 61928316, 30569392, 43045786, 92554120, 38341669, 75777973, 47887470, 89217461, 42967683, 86543538, 85117092, 48260151, 16019925, 13468268, 66322959, 54427233, 55615885, 30653863, 30463802, 77620120, 39847321, 6793819, 9188443, 92604458, 54868730, 59177718, 42073124, 12303248, 42692881, 71469330, 50007421, 71316369, 95957797, 91664334, 16054533, 59910624, 32058615, 82651278, 28851716, 79806380, 19272365, 51466049, 60393039, 1375023, 99524975, 1431742, 40781854, 4668450, 67030811, 73021291, 824872, 66663942, 61982238, 96193415, 77072625, 57658654, 62762857, 45848907, 45996863, 84127901, 6986898, 17081350, 82726008, 48774913, 34946859, 59318837, 16380211, 97057187, 97641116, 44983451, 72278539, 90090964, 69697787, 62693428, 89637706, 19457589, 36753250, 18833224, 39553046, 83368048, 68824981, 10366309, 92803766, 38645117, 4095116, 35996293, 75543508, 33797252, 90272749, 17539625, 3509435, 83133790, 8099994, 75153252, 10961421, 88653118, 7104732, 58224549, 44664587, 93270084, 28787861, 508198, 3183975, 81274566, 42947632, 75229462, 88444207, 26392416, 57393458, 13862149, 63628376, 49236559, 64848072, 17976208, 20885148, 15163258, 34236719, 38008118, 6525948, 14723410, 26734892, 20836893, 61712234, 20681921, 19898053, 55770687, 21993752, 21070110, 38658347, 53648154, 82178706, 30891921, 40984766, 24619760, 89046466, 1569515, 99861373, 8729146, 76170907, 59197747, 19486173, 48360198, 65880522, 64602895, 67031644, 77377183, 45990383, 47213183, 65275241, 62051033, 92867155, 73168565, 569864, 37183543, 93933709, 7300047, 48088883, 89804152, 52293995, 42307449, 26063929, 63506281, 46851987, 4798568, 80555751, 7955293, 1239555, 41380093, 34295794, 57163802, 7685448, 89214616, 32426752, 112651, 22766820, 66885828, 81172706, 37620363, 79136082, 18783702, 86118021, 8791066, 38022834, 16971929, 18131876, 7182642, 85711894, 14045383, 97379790, 36135, 81774825, 27665211, 76960303, 44479073, 68204242, 29401781, 73786944, 16567550, 94711842, 24953498, 60955663, 14731700, 30218878, 20140249, 62496012, 99226875, 9603598, 11757872, 36930650, 86022504, 11581181, 26292919, 57240218, 96709982, 1250437, 22129328, 44060493, 10453030, 247198, 34432810, 54987042, 669105, 4985896, 56515456, 321665, 57020481, 74614639, 71083565, 54517921, 9599614, 23848565, 4787945, 48673079, 2891150, 13231279, 69641513, 84684495, 88698958, 17957593, 33304202, 90013093, 16445503, 90457870, 51715482, 72019362, 75128745, 8055981, 3088684, 54199160, 49328608, 96735716, 96420429, 6038457, 8651647, 83150534, 95010552, 61271144, 36580610, 30366150, 9829782, 44915235, 93515664, 67451935, 10309525, 32250045, 94539824, 32159704, 24591705, 16595436, 24058273, 53393358, 73392814, 35456853, 68110330, 33061250, 40027975, 68000591, 9575954, 69255765, 42644903, 29932657, 77301523, 16097038, 72793444, 55189057, 82052050, 76703609, 79880247, 84002370, 48892201, 92692283, 75986488, 29188588, 18411915, 92998591, 77694700, 47708850, 2204165, 41245325, 70541760, 7066775, 51426311, 56461322, 7011964, 39373729, 29834512, 74357852, 78766358, 16424199, 41481685, 40781449, 54232247, 82532312, 18699206, 56153345, 99021067, 18806856, 83302115, 31491938, 56484900, 64055960, 67474219, 56424103, 79322415, 87710366, 31727379, 68939068, 56955985, 37192445, 45381876, 12664567, 36396314, 57803235, 47738185, 29819940, 30771409, 7502255, 71657078, 45995325, 94076128, 73031054, 50806615, 82897371, 2917920, 99125126, 88904910, 61141874, 77300457, 26664538, 5267545, 36139942, 83651211, 41092172, 66137019, 59371804, 4434662, 45667668, 97281847, 18001617, 29510992, 69579137, 98462867, 26863229, 76315420, 49882705, 91141395, 6221471, 9623492, 79657802, 79922758, 51315460, 38494874, 1936762, 37280276, 749283, 17365188, 81853704, 53547802, 78884452, 22879907, 15902805, 8807940, 99604946, 70004753, 83789391, 37224844, 62552524, 92071990, 66116458, 33123618, 61263068, 77312810, 53666583, 26275734, 66271566, 8913721, 36845587, 415901, 73124510, 99690194, 70367851, 37501808, 55960386, 34044787, 71333116, 88047921, 7517032, 1204161, 92033260, 16684829, 50668599, 72238278, 72373496, 29994197, 78300864, 70420215, 74452589, 54606384, 37166608, 36780454, 83269727, 62428472, 99515901, 76330843, 46870723, 32151165, 99971982, 53888755, 8696647, 62452034, 9886593, 29065964, 16387575, 65038678, 99917599, 31904591, 91240048, 51588015, 45407418, 17764950, 83533741, 79942022, 14947650, 8634541, 94090109, 99658235, 61623915, 66611759, 68702632, 53842979, 30998561, 74137926, 21919959, 54014062, 48675329, 53802686, 34493392, 65715134, 32699744, 6153406, 12416768, 38061439, 44177011, 55753905, 76434144, 95395112, 61728685, 15535065, 98371444, 6505939, 6808825, 33237508, 90004325, 55247455, 74441448, 66319530, 34497327, 17037369, 17386996, 4064751, 5832946, 79821897, 82339363, 44473167, 13173644, 40865610, 34698428, 66250369, 33895336, 42782652, 23569917, 68816413, 89419466, 36933038, 70036438, 9860195, 77272628, 64087743, 54263880, 75407347, 68875490, 37675718, 40197395, 26507214, 35192533, 63967300, 33249630, 36808486, 22113133, 90654836, 95726235, 52261574, 10597197, 78785507, 65271999, 54762643, 6157724, 45794415, 30090481, 4204661, 98653983, 16099750, 71522968, 4099191, 43376279, 66428795, 66832478, 8129978, 52204879, 17385531, 78909561, 45075471, 28550822, 67227442 +61373987, 7423788, 62762857, 69848388, 77301523, 83155442, 1250437, 53888755, 669105, 44983451, 9599614, 90013093, 75128745, 30366150, 12416768, 62552524, 92867155, 1239555, 48658605, 59329511, 78766358, 77284619, 34497327, 72278539, 44846932, 88904910, 45667668, 78785507, 68702632, 85023028, 42947632, 49236559, 37280276, 47893286, 4978543, 94539824, 82178706, 76170907, 98648327, 43933006, 52293995, 72738685, 73109997, 6505939, 18504197, 70541760, 22113133, 2607799, 32058615, 74110882, 82532312, 74441448, 98948034, 73021291, 62496012, 34432810, 50806615, 22721500, 44481640, 27411561, 91727510, 29510992, 17539625, 34698428, 33895336, 91802888, 51315460, 67084351, 42237907, 28734791, 14723410, 17857111, 21070110, 38658347, 30891921, 75135448, 64602895, 5482538, 76434144, 68875490, 61728685, 29932657, 37675718, 33553959, 39171895, 85117092, 73617245, 37739481, 73124510, 15535065, 112651, 12303248, 44889423, 18783702, 56461322, 41442762, 91664334, 31126490, 24953498, 1431742, 9603598, 36930650, 26744917, 45848907, 50702367, 52261574, 5832946, 34946859, 10453030, 97057187, 65851721, 54987042, 86821229, 4985896, 6521313, 67124282, 92530431, 5267545, 76540481, 70800879, 97940276, 80251430, 71965942, 38510840, 8099994, 26998766, 49882705, 10961421, 7104732, 66250369, 26776922, 4515343, 3183975, 8651647, 60106217, 70782102, 1788101, 4199704, 6497830, 20885148, 32250045, 81677380, 34493392, 20836893, 81853704, 53547802, 78884452, 62232211, 1569515, 68000591, 77377183, 44784505, 86301513, 39801351, 95395112, 65275241, 33123618, 7300047, 53666583, 26275734, 26063929, 16019925, 66322959, 54427233, 30653863, 62740044, 44847298, 92692283, 77620120, 57163802, 69786756, 5073754, 71469330, 67281495, 33237508, 29466406, 16424199, 81774825, 66428795, 65074535, 99021067, 29401781, 85571389, 73786944, 95726235, 60955663, 3294781, 20140249, 59501487, 97783876, 54606384, 89699445, 99515901, 71657078, 5111370, 93566986, 93057697, 10366309, 23848565, 84349107, 13470059, 83651211, 99549373, 9058407, 94516935, 4434662, 26493119, 33431960, 17894977, 72274002, 43357947, 91141395, 6221471, 88653118, 95581843, 79657802, 96735716, 6038457, 33628349, 68816413, 89419466, 55470718, 75229462, 749283, 44915235, 9860195, 65454636, 15948937, 77272628, 32159704, 64098930, 7919588, 15015906, 70596786, 89499542, 55770687, 85695762, 15902805, 40984766, 99604946, 30787683, 87598888, 37224844, 12571310, 47213183, 69255765, 75407347, 43045786, 5640302, 98739783, 13819358, 20765474, 38341669, 62803858, 17016156, 3233569, 72793444, 51149804, 99729357, 63506281, 4798568, 72614359, 36845587, 36505482, 61741594, 48892201, 415901, 96906410, 32426752, 63967300, 12348118, 18663507, 83747892, 51426311, 99965001, 86118021, 56473732, 8791066, 29834512, 16054533, 28851716, 72732205, 33935899, 49271185, 7687278, 50668599, 16567550, 51466049, 99524975, 37435892, 66319530, 67474219, 77072625, 57658654, 77187825, 38256849, 41092102, 37166608, 33201905, 17037369, 68939068, 39986008, 36396314, 57803235, 6986898, 99224392, 29819940, 6871053, 40152546, 99971982, 67793644, 66832478, 14220886, 74743862, 45306070, 56515456, 91281584, 57020481, 16387575, 95251277, 65038678, 18833224, 82339363, 61859581, 44842615, 19376156, 90310261, 22435353, 33797252, 97561745, 30139692, 44348328, 84684495, 96852321, 83133790, 35092039, 84840549, 68102437, 99658235, 40865610, 53084256, 11923835, 48395186, 45237957, 93270084, 36312813, 30998561, 42782652, 57248122, 75052463, 47792865, 21919959, 26397786, 91990218, 63628376, 70036438, 36189527, 67451935, 17365188, 97011160, 14626618, 59614746, 34236719, 38008118, 6525948, 1197320, 65715134, 24058273, 19898053, 53648154, 62490109, 3773993, 54263880, 44177011, 89046466, 99861373, 19486173, 88130087, 64157906, 45990383, 10649306, 29959549, 61263068, 77312810, 89217461, 82979980, 6111563, 48088883, 89804152, 98653983, 17058722, 42199455, 30543215, 8913721, 80014588, 2331773, 61815223, 19101477, 45428665, 39847321, 9175338, 54868730, 92998591, 42692881, 27187213, 81172706, 2204165, 41245325, 23740167, 16971929, 52204879, 18131876, 96726697, 49641577, 68316156, 7517032, 82651278, 52060076, 24915585, 24314885, 18699206, 1204161, 29994197, 39201414, 40356877, 18806856, 94711842, 43152977, 55247455, 83083131, 20002147, 6819644, 99226875, 74452589, 24168411, 37192445, 15536795, 84127901, 57240218, 96709982, 82726008, 76330843, 22129328, 21397057, 44060493, 30771409, 32151165, 59318837, 94076128, 10597197, 94595668, 29065964, 36753250, 45049308, 61960400, 54517921, 32274392, 68824981, 32161669, 40677414, 38645117, 90158785, 4787945, 17764950, 22405120, 48673079, 2891150, 93053405, 49598724, 29516592, 78218845, 69579137, 81855445, 92215320, 13231279, 86001008, 76315420, 16445503, 51715482, 87160386, 61623915, 65271999, 62357986, 58224549, 8055981, 44664587, 66667729, 28796059, 54199160, 96420429, 79922758, 55602660, 26114953, 38494874, 14326617, 95290172, 54014062, 51472275, 48675329, 93515664, 59405277, 30764367, 35456853, 73222868, 81898046, 30395570, 24619760, 176257, 20867149, 83789391, 15064655, 3235882, 52613508, 78561158, 66116458, 75777973, 60581278, 82886381, 16097038, 94360702, 40197395, 11543098, 76703609, 51507425, 84293052, 55615885, 80555751, 78909561, 7955293, 91957544, 38365584, 49597667, 9188443, 18411915, 71300104, 99690194, 22766820, 43322743, 77694700, 37620363, 79136082, 51464002, 6808825, 7011964, 25636669, 17146629, 58751351, 34044787, 5970607, 38022834, 27041967, 36135, 61859143, 54232247, 69355476, 71920426, 65047700, 19272365, 9257405, 83302115, 56484900, 40781854, 4668450, 72357096, 824872, 66663942, 96193415, 83269727, 62428472, 50567636, 8128637, 12664567, 89078848, 17385531, 14349098, 48774913, 82779622, 46540998, 45995325, 37891451, 73031054, 168541, 91255408, 2717150, 82897371, 9398733, 62693428, 62452034, 99125126, 9886593, 36812683, 77300457, 43501211, 39553046, 31904591, 24733232, 69136837, 60176618, 8115266, 23432750, 4806458, 41092172, 83533741, 79942022, 75543508, 80316608, 33336148, 97281847, 8634541, 19891772, 21533347, 75820087, 17957593, 27185644, 33304202, 88251446, 75153252, 58749, 73235980, 3088684, 28787861, 49328608, 508198, 62115552, 74137926, 78602717, 78549759, 30694952, 9259676, 20568363, 98943869, 9860968, 91831487, 83150534, 36468541, 61271144, 26392416, 13862149, 10358899, 64848072, 37957788, 17976208, 53802686, 34667286, 62430984, 84166196, 98130363, 24591705, 16595436, 6153406, 53393358, 21993752, 44844121, 33061250, 31733363, 40027975, 59197747, 9575954, 30569392, 63059024, 92554120, 31161687, 21289531, 569864, 23134715, 42967683, 86543538, 4204661, 76671482, 42307449, 48260151, 874791, 79880247, 46851987, 85116755, 16099750, 89214616, 70372191, 44245960, 71522968, 7066775, 99297164, 71316369, 77787724, 88047921, 7182642, 55143724, 14045383, 59910624, 97379790, 57359924, 30811010, 27665211, 79806380, 63015256, 4069912, 16684829, 87720882, 8733068, 47298834, 31491938, 86242799, 67030811, 5822202, 61982238, 79322415, 55669657, 11581181, 99333375, 31727379, 40686254, 17081350, 17727650, 7502255, 40534591, 54058788, 79821897, 16380211, 61897582, 69697787, 8696647, 526217, 43506672, 71083565, 45075471, 79191827, 99917599, 36139942, 59109400, 16405341, 4095116, 26102057, 66137019, 45790169, 8373289, 3509435, 69641513, 26863229, 19939935, 93359396, 72019362, 28928175, 66611759, 2208785, 60430369, 23569917, 66903004, 59981773, 36580610, 84406788, 22200378, 9829782, 15075176, 6157724, 92398073, 27325428, 51590803, 22997281, 89996536, 61712234, 67227442, 65338021, 73392814, 69605283, 80246713, 68110330, 70727211, 79738755, 93790285, 8729146, 48360198, 65880522, 78589145, 67031644, 30090481, 22108665, 42644903, 47887470, 93933709, 60102965, 82052050, 36685023, 34698463, 13468268, 84904436, 84002370, 26507214, 62936963, 35192533, 32590267, 37560164, 41380093, 92541302, 6793819, 98371444, 7685448, 51047803, 59177718, 42073124, 66885828, 7646095, 93562257, 4508700, 95957797, 71333116, 74357852, 85711894, 41481685, 77898274, 90004325, 4091162, 20122224, 40781449, 27625735, 92692978, 50188404, 72238278, 72373496, 20645197, 56424103, 45996863, 26292919, 77413012, 96318094, 44550764, 11365791, 92353856, 68694897, 56531125, 321665, 89637706, 94911072, 65017137, 61141874, 44627776, 19457589, 74614639, 26664538, 82327024, 91240048, 92803766, 72725103, 35512853, 85242963, 14947650, 26139110, 59371804, 18001617, 47361209, 57961282, 3233084, 17068582, 22450468, 28550822, 54762643, 9623492, 3880712, 82427263, 81274566, 14093520, 36933038, 68128438, 54663246, 26734892, 3633375, 20486294, 21673260, 38009615, 64087743, 38061439, 24826575, 61928316, 92071990, 8129978, 73168565, 55850790, 1022677, 58180653, 55189057, 66271566, 30463802, 83948335, 23110625, 34295794, 29188588, 92604458, 37501808, 63152504, 55960386, 4099191, 47090124, 89811711, 37659250, 4119747, 91938191, 86798033, 76960303, 56153345, 68204242, 12024238, 1375023, 78300864, 11757872, 84187166, 53632373, 17386996, 47738185, 97641116, 2917920, 72358170, 15148031, 76825057, 51588015, 92787493, 68041839, 45617087, 44473167, 57241521, 94090109, 13173644, 63372756, 53842979, 81805959, 95010552, 72495719, 45794415, 22879907, 22942635, 8807940, 70004753, 13348726, 37183543, 29635537, 33249630, 74724075, 47708850, 70367851, 50007421, 68644627, 90654836, 14363867, 23194618, 72777973, 60393039, 14731700, 87710366, 33565483, 56955985, 45381876, 46870723, 247198, 30163921, 90090964, 48893685, 83210802, 45407418, 90272749, 58208470, 98462867, 88698958, 25842078, 3487592, 66045587, 7229550, 1936762, 88444207, 10309525, 18466635, 86460488, 55753905, 62051033, 65081429, 75986488, 29029316, 33699435, 39373729, 92033260, 44479073, 45919976, 86022504, 36780454, 4064751, 83368048, 35996293, 21001913, 90457870, 11161731, 90061527, 43376279, 10264691, 70420215, 57393458, 32699744, 36808486, 30218878, 67513640, 20642888, 15163258, 64055960, 20681921 +77300457, 7104732, 44784505, 68875490, 13819358, 82886381, 96906410, 82651278, 63015256, 4985896, 54199160, 78766358, 7687278, 37891451, 94076128, 9398733, 90310261, 16405341, 48673079, 45790169, 79657802, 53842979, 53802686, 17857111, 68000591, 75407347, 7423788, 80555751, 88047921, 32058615, 11757872, 39986008, 34432810, 69697787, 45049308, 54517921, 99917599, 8115266, 72725103, 92787493, 75820087, 28928175, 66250369, 66667729, 54014062, 749283, 28734791, 32159704, 81853704, 48360198, 8129978, 65275241, 75777973, 55850790, 1022677, 72793444, 34698463, 54427233, 1239555, 41380093, 89214616, 22766820, 18663507, 83747892, 67281495, 86118021, 56461322, 33237508, 81774825, 37659250, 71920426, 19272365, 72373496, 30218878, 824872, 66663942, 11581181, 56955985, 53632373, 52261574, 46870723, 29819940, 59318837, 65851721, 54987042, 91255408, 6521313, 8696647, 321665, 89637706, 88904910, 36812683, 16387575, 526217, 79191827, 31904591, 15148031, 26664538, 10366309, 19376156, 91240048, 69136837, 76825057, 79942022, 94516935, 59371804, 33797252, 18001617, 29510992, 21001913, 6221471, 54762643, 58224549, 95581843, 42782652, 93515664, 20885148, 15948937, 38008118, 24591705, 38658347, 65880522, 3235882, 43045786, 77301523, 89804152, 55189057, 42307449, 48260151, 26063929, 63506281, 80014588, 84293052, 62740044, 72614359, 38365584, 34295794, 57163802, 63967300, 77694700, 70367851, 6808825, 18783702, 50007421, 58751351, 22113133, 71316369, 5970607, 18131876, 16424199, 24915585, 40781449, 90654836, 27625735, 72732205, 74110882, 79806380, 92033260, 8733068, 20645197, 40781854, 14731700, 73021291, 59501487, 36930650, 24168411, 17037369, 68939068, 45848907, 36396314, 17081350, 71657078, 10453030, 32151165, 96318094, 99971982, 50806615, 53888755, 72278539, 22721500, 56515456, 62452034, 29065964, 65038678, 32274392, 92530431, 68824981, 85242963, 75543508, 94090109, 3509435, 90457870, 93359396, 88251446, 10961421, 9623492, 3088684, 60430369, 57248122, 78549759, 85023028, 36468541, 88444207, 13862149, 51472275, 4199704, 65454636, 10309525, 34493392, 1197320, 65715134, 65338021, 89499542, 21993752, 12416768, 38061439, 30787683, 99861373, 93790285, 40027975, 67031644, 62552524, 9575954, 63059024, 20765474, 62051033, 33123618, 60102965, 66271566, 85117092, 66322959, 4798568, 45428665, 7685448, 92604458, 70372191, 42073124, 42692881, 5073754, 47708850, 71522968, 4508700, 39373729, 27041967, 74357852, 41481685, 36135, 30811010, 28851716, 82532312, 24314885, 50668599, 29401781, 18806856, 94711842, 56484900, 37435892, 78300864, 86242799, 3294781, 20140249, 62762857, 86022504, 87710366, 26744917, 37192445, 12664567, 77413012, 89078848, 6986898, 83155442, 96709982, 5832946, 30771409, 45995325, 97057187, 66832478, 669105, 86821229, 44983451, 44550764, 48893685, 67124282, 72358170, 91281584, 36753250, 93566986, 61859581, 84349107, 40677414, 13470059, 90158785, 4787945, 9058407, 83533741, 27411561, 26139110, 93053405, 68041839, 97281847, 80251430, 78218845, 69579137, 21533347, 58208470, 27185644, 43357947, 22450468, 40865610, 3487592, 34698428, 62357986, 93270084, 36312813, 3880712, 66045587, 49328608, 96420429, 79922758, 3183975, 33628349, 68816413, 14093520, 70782102, 37280276, 48675329, 44915235, 4978543, 15075176, 11161731, 92398073, 34667286, 17365188, 81677380, 14626618, 94539824, 59614746, 54663246, 32699744, 78884452, 20486294, 62232211, 30891921, 40984766, 3773993, 89046466, 61373987, 31733363, 8729146, 76170907, 64602895, 88130087, 64157906, 30090481, 98648327, 61928316, 47213183, 78561158, 66116458, 95395112, 92867155, 62803858, 61728685, 29959549, 37675718, 33553959, 23134715, 86543538, 48088883, 98653983, 82052050, 8913721, 76703609, 13468268, 51507425, 30653863, 65081429, 36845587, 37739481, 7955293, 19101477, 73124510, 92692283, 29635537, 9175338, 92998591, 44889423, 37620363, 23740167, 8791066, 17146629, 29466406, 16971929, 52204879, 14045383, 68316156, 49271185, 76960303, 4069912, 99021067, 87720882, 29994197, 77284619, 99226875, 61982238, 74452589, 89699445, 8128637, 1250437, 34946859, 6871053, 73031054, 97641116, 82339363, 44842615, 23848565, 41092172, 20642888, 45617087, 17539625, 33304202, 17068582, 72019362, 84840549, 26998766, 49882705, 28550822, 53084256, 75128745, 65271999, 66611759, 33895336, 96735716, 82427263, 2208785, 4515343, 6038457, 62115552, 23569917, 38494874, 81274566, 42947632, 47792865, 36933038, 67084351, 91990218, 30366150, 84406788, 49236559, 47893286, 36189527, 6497830, 97011160, 89996536, 69848388, 98130363, 26734892, 61712234, 15015906, 20681921, 45794415, 21070110, 73222868, 82178706, 8807940, 86460488, 33061250, 30395570, 24619760, 44177011, 83789391, 21289531, 82979980, 37183543, 26275734, 58180653, 42199455, 30543215, 16019925, 79880247, 55615885, 2331773, 44847298, 91957544, 23110625, 15535065, 29188588, 6793819, 18411915, 54868730, 69786756, 112651, 66885828, 74724075, 37501808, 79136082, 41245325, 70541760, 4099191, 33699435, 34044787, 59329511, 95957797, 43376279, 97379790, 90004325, 54232247, 1204161, 65047700, 66428795, 16684829, 68204242, 85571389, 95726235, 83302115, 47298834, 24953498, 55247455, 74441448, 67030811, 98948034, 62496012, 5822202, 56424103, 96193415, 57658654, 9603598, 38256849, 54606384, 83269727, 50567636, 45381876, 26292919, 57803235, 17385531, 82726008, 82779622, 21397057, 40534591, 79821897, 10597197, 94595668, 67793644, 168541, 14220886, 74743862, 11365791, 45306070, 5111370, 94911072, 61141874, 57020481, 18833224, 61960400, 71083565, 5267545, 83210802, 59109400, 60176618, 83651211, 35512853, 26102057, 14947650, 97940276, 22435353, 45667668, 30139692, 44473167, 8373289, 8634541, 57241521, 84684495, 96852321, 26863229, 3233084, 83133790, 71965942, 17894977, 51715482, 87160386, 68102437, 75153252, 508198, 74137926, 78602717, 30694952, 66903004, 9860968, 89419466, 75229462, 14326617, 83150534, 21919959, 26397786, 42237907, 22200378, 64848072, 67451935, 18466635, 72495719, 68128438, 84166196, 34236719, 20836893, 16595436, 3633375, 53547802, 55770687, 35456853, 22942635, 44844121, 21673260, 62490109, 68110330, 99604946, 54263880, 176257, 15064655, 75135448, 55753905, 19486173, 77377183, 92071990, 92554120, 38341669, 73168565, 47887470, 61263068, 89217461, 94360702, 4204661, 17058722, 99729357, 73617245, 84002370, 35192533, 61741594, 37560164, 72738685, 9188443, 33249630, 29029316, 73109997, 99965001, 56473732, 7011964, 25636669, 41442762, 68644627, 38022834, 71333116, 85711894, 7517032, 52060076, 69355476, 91938191, 44479073, 39201414, 1375023, 99524975, 64055960, 67474219, 70420215, 79322415, 77187825, 31727379, 33565483, 84187166, 84127901, 50702367, 99515901, 22129328, 14349098, 17727650, 48774913, 16380211, 68694897, 65017137, 9886593, 74614639, 32161669, 82327024, 9599614, 92803766, 38645117, 23432750, 4095116, 35996293, 66137019, 91727510, 2891150, 80316608, 90272749, 19891772, 81855445, 57961282, 88698958, 86001008, 35092039, 11923835, 91141395, 63372756, 48395186, 8055981, 28787861, 28796059, 26776922, 91802888, 55602660, 51315460, 20568363, 60106217, 26392416, 36580610, 9829782, 32250045, 27325428, 90061527, 51590803, 15163258, 30764367, 6525948, 14723410, 7919588, 6153406, 53393358, 53648154, 81898046, 15902805, 64087743, 1569515, 87598888, 79738755, 5482538, 78589145, 22108665, 45990383, 52613508, 69255765, 30569392, 5640302, 39801351, 29932657, 17016156, 569864, 6111563, 53666583, 36685023, 51149804, 52293995, 874791, 26507214, 62936963, 36505482, 30463802, 48892201, 415901, 77620120, 16099750, 12303248, 27187213, 12348118, 44245960, 81172706, 2204165, 7646095, 89811711, 77787724, 49641577, 77898274, 61859143, 20122224, 27665211, 65074535, 50188404, 56153345, 72777973, 73786944, 40356877, 45919976, 16567550, 12024238, 1431742, 4668450, 77072625, 33201905, 45996863, 15536795, 17386996, 47738185, 46540998, 54058788, 247198, 30163921, 61897582, 92353856, 62693428, 56531125, 99125126, 44627776, 95251277, 43501211, 83368048, 24733232, 4806458, 17764950, 76540481, 26493119, 97561745, 92215320, 13231279, 44348328, 90013093, 72274002, 76315420, 8099994, 78785507, 99658235, 61623915, 88653118, 73235980, 44664587, 30998561, 7229550, 26114953, 9259676, 98943869, 1936762, 91831487, 95290172, 95010552, 61271144, 1788101, 63628376, 17976208, 9860195, 59405277, 77272628, 67227442, 73392814, 69605283, 85695762, 22879907, 80246713, 38009615, 20867149, 10649306, 98739783, 42644903, 60581278, 42967683, 7300047, 40197395, 11543098, 84904436, 75986488, 85116755, 98371444, 71300104, 59177718, 32426752, 43322743, 36808486, 6505939, 51464002, 18504197, 99297164, 96726697, 31126490, 16054533, 59910624, 4091162, 4119747, 92692978, 86798033, 10264691, 51466049, 43152977, 83083131, 60955663, 6819644, 72357096, 34497327, 97783876, 37166608, 99333375, 99224392, 44060493, 7502255, 90090964, 44846932, 43506672, 45075471, 39553046, 93057697, 51588015, 45407418, 22405120, 70800879, 4434662, 49598724, 33336148, 33431960, 47361209, 98462867, 19939935, 17957593, 38510840, 16445503, 45237957, 68702632, 81805959, 59981773, 10358899, 6157724, 62430984, 24058273, 70596786, 12571310, 76434144, 13348726, 86301513, 77312810, 43933006, 76671482, 78909561, 61815223, 32590267, 49597667, 92541302, 99690194, 71469330, 93562257, 55960386, 29834512, 2607799, 91664334, 55143724, 18699206, 33935899, 14363867, 60393039, 31491938, 20002147, 66319530, 41092102, 40686254, 4064751, 82897371, 2917920, 44481640, 36139942, 69641513, 58749, 70036438, 37957788, 64098930, 19898053, 70004753, 70727211, 31161687, 93933709, 16097038, 39171895, 3233569, 46851987, 83948335, 39847321, 48658605, 7066775, 57359924, 57240218, 40152546, 2717150, 19457589, 99549373, 13173644, 75052463, 8651647, 55470718, 57393458, 59197747, 24826575, 63152504, 51426311, 47090124, 7182642, 23194618, 72238278, 62428472, 67513640, 29516592, 22997281, 37224844, 9257405, 36780454, 76330843, 55669657, 51047803, 25842078 +52613508, 45407418, 70036438, 44481640, 89078848, 84166196, 24058273, 84904436, 47090124, 5111370, 93566986, 4806458, 70800879, 27411561, 35996293, 61623915, 47792865, 30090481, 20765474, 94360702, 75986488, 18783702, 95957797, 2607799, 83302115, 41092102, 24168411, 99333375, 10453030, 16380211, 168541, 84349107, 83651211, 45617087, 16445503, 75229462, 57393458, 30366150, 9829782, 54663246, 7919588, 21070110, 65275241, 62803858, 75777973, 61263068, 33553959, 48088883, 63506281, 73617245, 73124510, 85116755, 32426752, 43322743, 71522968, 4099191, 56461322, 25636669, 68644627, 54232247, 18699206, 49271185, 9257405, 16567550, 1431742, 77187825, 67513640, 94595668, 73031054, 97641116, 45306070, 67124282, 91281584, 83368048, 92803766, 4095116, 49598724, 3509435, 69641513, 90457870, 48395186, 79657802, 4515343, 74137926, 67084351, 13862149, 6497830, 20885148, 6525948, 89499542, 30891921, 5482538, 88130087, 13348726, 68000591, 63059024, 10649306, 98739783, 4204661, 82052050, 72614359, 9175338, 18411915, 70372191, 92998591, 2204165, 83747892, 18504197, 55960386, 7011964, 4508700, 33237508, 43376279, 14045383, 49641577, 69355476, 1375023, 99524975, 55247455, 67474219, 96193415, 62762857, 76330843, 54058788, 61897582, 669105, 44983451, 22721500, 74743862, 2917920, 62693428, 62452034, 29065964, 71083565, 79191827, 8373289, 92215320, 43357947, 84840549, 88653118, 62357986, 60430369, 91802888, 10358899, 51472275, 6157724, 62430984, 89996536, 67227442, 53547802, 55770687, 44844121, 8807940, 176257, 99861373, 12571310, 75135448, 59197747, 19486173, 92071990, 86301513, 7423788, 31161687, 42967683, 39171895, 76671482, 66271566, 84293052, 2331773, 46851987, 4798568, 78909561, 19101477, 415901, 6793819, 51047803, 71300104, 89214616, 59177718, 93562257, 36808486, 58751351, 34044787, 89811711, 22113133, 29834512, 78766358, 81774825, 19272365, 14363867, 16684829, 50188404, 72373496, 73786944, 94711842, 60955663, 4668450, 20002147, 20140249, 77072625, 36780454, 11581181, 84187166, 57240218, 52261574, 46870723, 32151165, 94076128, 2717150, 92353856, 68694897, 99125126, 45075471, 15148031, 90310261, 51588015, 92787493, 17764950, 76540481, 97940276, 80316608, 33797252, 45667668, 97561745, 90272749, 80251430, 78218845, 81855445, 71965942, 38510840, 78785507, 34698428, 58224549, 8055981, 66250369, 96735716, 95290172, 36468541, 61271144, 26397786, 36580610, 48675329, 10309525, 15948937, 18466635, 90061527, 59614746, 34236719, 81853704, 65338021, 45794415, 69605283, 38658347, 99604946, 3773993, 70004753, 20867149, 76434144, 67031644, 69255765, 43045786, 13819358, 42644903, 82886381, 17016156, 37675718, 37183543, 60102965, 3233569, 85117092, 30653863, 44847298, 35192533, 30463802, 1239555, 83948335, 91957544, 32590267, 72738685, 29635537, 98371444, 92604458, 69786756, 29029316, 6505939, 51464002, 51426311, 99965001, 86118021, 16054533, 59910624, 97379790, 16424199, 27625735, 99021067, 85571389, 29994197, 43152977, 20645197, 74441448, 86242799, 6819644, 30218878, 86022504, 83269727, 50702367, 79821897, 40152546, 67793644, 82897371, 56531125, 321665, 526217, 77300457, 43506672, 43501211, 61960400, 60176618, 4787945, 35512853, 79942022, 66137019, 91727510, 33336148, 26493119, 44473167, 13173644, 57961282, 86001008, 27185644, 51715482, 49882705, 88251446, 99658235, 40865610, 10961421, 66611759, 54762643, 44664587, 3088684, 66667729, 36312813, 28787861, 26776922, 49328608, 508198, 96420429, 57248122, 62115552, 66903004, 85023028, 14093520, 55470718, 14326617, 83150534, 91990218, 28734791, 15075176, 59405277, 92398073, 53802686, 68128438, 15163258, 15015906, 65715134, 6153406, 73392814, 35456853, 53648154, 22879907, 21673260, 40984766, 38009615, 38061439, 33061250, 89046466, 70727211, 76170907, 98648327, 77377183, 9575954, 30569392, 5640302, 39801351, 38341669, 61728685, 55850790, 47887470, 89217461, 93933709, 7300047, 53666583, 17058722, 42199455, 76703609, 51507425, 874791, 26507214, 62740044, 48892201, 41380093, 92692283, 57163802, 7685448, 42692881, 47708850, 7646095, 79136082, 67281495, 56473732, 8791066, 38022834, 52204879, 96726697, 31126490, 7182642, 41481685, 36135, 32058615, 68316156, 30811010, 4119747, 71920426, 27665211, 1204161, 65047700, 92033260, 76960303, 56153345, 72777973, 39201414, 95726235, 18806856, 8733068, 12024238, 47298834, 40781854, 77284619, 59501487, 99226875, 57658654, 36930650, 37166608, 33201905, 33565483, 39986008, 56955985, 12664567, 53632373, 17386996, 4064751, 17727650, 47738185, 21397057, 5832946, 71657078, 40534591, 96318094, 59318837, 34432810, 97057187, 65851721, 66832478, 53888755, 54987042, 86821229, 14220886, 11365791, 48893685, 16387575, 65038678, 18833224, 39553046, 92530431, 9599614, 5267545, 36139942, 69136837, 76825057, 90158785, 99549373, 16405341, 9058407, 48673079, 14947650, 75543508, 22435353, 97281847, 47361209, 17539625, 75820087, 84684495, 26863229, 83133790, 17894977, 8099994, 58749, 11923835, 91141395, 63372756, 68702632, 54199160, 33895336, 66045587, 2208785, 79922758, 3183975, 8651647, 91831487, 68816413, 59981773, 42947632, 21919959, 26392416, 63628376, 49236559, 44915235, 93515664, 77272628, 97011160, 14626618, 64098930, 20681921, 16595436, 53393358, 15902805, 61373987, 1569515, 87598888, 40027975, 15064655, 8729146, 55753905, 24826575, 95395112, 60581278, 33123618, 77312810, 86543538, 6111563, 16097038, 43933006, 72793444, 55189057, 11543098, 52293995, 8913721, 54427233, 80014588, 84002370, 36845587, 62936963, 37739481, 61815223, 61741594, 49597667, 39847321, 9188443, 16099750, 96906410, 42073124, 112651, 33249630, 12303248, 66885828, 27187213, 18663507, 7066775, 33699435, 59329511, 5970607, 77787724, 71333116, 27041967, 18131876, 7517032, 57359924, 4091162, 20122224, 37659250, 74110882, 82532312, 65074535, 72238278, 83083131, 824872, 5822202, 61982238, 97783876, 38256849, 87710366, 89699445, 26744917, 37192445, 40686254, 17081350, 96709982, 17385531, 6871053, 45995325, 247198, 10597197, 50806615, 91255408, 4985896, 6521313, 56515456, 88904910, 36812683, 19457589, 57020481, 74614639, 54517921, 31904591, 82339363, 68824981, 26664538, 44842615, 91240048, 59109400, 83533741, 2891150, 93053405, 30139692, 8634541, 57241521, 21533347, 44348328, 25842078, 21001913, 90013093, 72274002, 22450468, 93359396, 26998766, 75128745, 6221471, 73235980, 45237957, 3880712, 82427263, 6038457, 81805959, 51315460, 20568363, 38494874, 1936762, 89419466, 95010552, 36933038, 42237907, 37280276, 34667286, 30764367, 38008118, 17857111, 70596786, 19898053, 78884452, 62232211, 22942635, 24619760, 31733363, 93790285, 37224844, 64602895, 22108665, 62552524, 8129978, 92554120, 77301523, 82979980, 26275734, 58180653, 40197395, 30543215, 36685023, 34698463, 26063929, 66322959, 36505482, 7955293, 23110625, 29188588, 48658605, 99690194, 22766820, 77694700, 12348118, 44245960, 71469330, 37620363, 63152504, 41245325, 23740167, 50007421, 41442762, 39373729, 55143724, 77898274, 61859143, 52060076, 40781449, 90654836, 79806380, 33935899, 91938191, 7687278, 66428795, 44479073, 45919976, 10264691, 51466049, 56484900, 67030811, 3294781, 66663942, 9603598, 55669657, 54606384, 17037369, 68939068, 50567636, 45996863, 8128637, 15536795, 6986898, 83155442, 1250437, 99515901, 82726008, 30771409, 34946859, 37891451, 90090964, 89637706, 61141874, 44627776, 36753250, 45049308, 24733232, 19376156, 13470059, 72725103, 41092172, 94516935, 94090109, 19891772, 13231279, 19939935, 17957593, 35092039, 76315420, 87160386, 68102437, 28928175, 65271999, 7104732, 93270084, 95581843, 30998561, 78602717, 26114953, 78549759, 33628349, 75052463, 98943869, 70782102, 1788101, 22200378, 749283, 36189527, 37957788, 65454636, 32250045, 27325428, 51590803, 34493392, 98130363, 1197320, 61712234, 21993752, 82178706, 80246713, 86460488, 30395570, 44177011, 3235882, 47213183, 78561158, 73168565, 21289531, 29959549, 1022677, 48260151, 65081429, 80555751, 77620120, 92541302, 15535065, 45428665, 5073754, 74724075, 44889423, 37501808, 73109997, 99297164, 29466406, 91664334, 85711894, 28851716, 72732205, 86798033, 4069912, 50668599, 68204242, 29401781, 64055960, 78300864, 56424103, 34497327, 70420215, 31727379, 45381876, 77413012, 36396314, 84127901, 57803235, 14349098, 82779622, 44060493, 29819940, 7502255, 72278539, 69697787, 94911072, 72358170, 9886593, 44846932, 95251277, 32161669, 82327024, 61859581, 10366309, 38645117, 8115266, 59371804, 45790169, 29516592, 33431960, 58208470, 98462867, 88698958, 17068582, 28550822, 3487592, 9623492, 53842979, 55602660, 23569917, 88444207, 84406788, 64848072, 67451935, 4978543, 17976208, 14723410, 69848388, 24591705, 62490109, 12416768, 54263880, 48360198, 65880522, 64157906, 45990383, 75407347, 66116458, 29932657, 569864, 23134715, 51149804, 99729357, 38365584, 54868730, 70367851, 6808825, 70541760, 16971929, 82651278, 90004325, 24915585, 23194618, 31491938, 24953498, 66319530, 98948034, 72357096, 74452589, 62428472, 45848907, 26292919, 22129328, 48774913, 46540998, 44550764, 32274392, 99917599, 93057697, 23848565, 83210802, 40677414, 23432750, 22405120, 26102057, 85242963, 96852321, 3233084, 72019362, 75153252, 53084256, 28796059, 42782652, 30694952, 81274566, 60106217, 54014062, 47893286, 11161731, 72495719, 17365188, 32699744, 20486294, 81898046, 30787683, 83789391, 79738755, 44784505, 68875490, 62051033, 92867155, 89804152, 98653983, 55615885, 34295794, 81172706, 17146629, 74357852, 24314885, 40356877, 60393039, 14731700, 11757872, 99971982, 30163921, 65017137, 26139110, 9259676, 9860195, 22997281, 94539824, 26734892, 20836893, 85695762, 68110330, 78589145, 61928316, 42307449, 16019925, 13468268, 88047921, 87720882, 37435892, 73021291, 79322415, 20642888, 4434662, 18001617, 69579137, 4199704, 81677380, 3633375, 73222868, 64087743, 63967300, 71316369, 99224392, 9398733, 68041839, 29510992, 33304202, 32159704, 79880247, 62496012, 63015256, 9860968, 37560164, 7229550, 92692978, 8696647 +31126490, 13173644, 4095116, 78766358, 5822202, 10597197, 24733232, 78602717, 67227442, 60102965, 36505482, 7011964, 25636669, 61960400, 84349107, 17764950, 2891150, 49882705, 83150534, 1788101, 6153406, 30395570, 12571310, 98648327, 9575954, 92071990, 7423788, 57163802, 9188443, 59177718, 32426752, 12348118, 18663507, 33237508, 71920426, 72373496, 56424103, 321665, 82327024, 14947650, 33797252, 29516592, 13231279, 3509435, 73235980, 95581843, 33895336, 59981773, 18466635, 30764367, 3633375, 21070110, 20867149, 55753905, 44784505, 63059024, 68875490, 82886381, 82979980, 51149804, 29188588, 48658605, 63967300, 2204165, 71469330, 63152504, 70541760, 99965001, 34044787, 59329511, 7517032, 65047700, 92033260, 76960303, 44479073, 40356877, 18806856, 40781854, 66319530, 20140249, 824872, 61982238, 59318837, 45995325, 53888755, 54987042, 669105, 14220886, 82897371, 56531125, 92530431, 26664538, 9599614, 8115266, 45407418, 48673079, 27411561, 35996293, 30139692, 88698958, 83133790, 11923835, 54762643, 93270084, 68702632, 26776922, 3183975, 81805959, 89419466, 14093520, 42237907, 48675329, 70036438, 93515664, 67451935, 22997281, 68128438, 15015906, 32699744, 82178706, 64087743, 87598888, 78589145, 5640302, 10649306, 65275241, 31161687, 569864, 42967683, 48088883, 26275734, 17058722, 63506281, 84002370, 44847298, 61815223, 73124510, 77620120, 75986488, 45428665, 85116755, 98371444, 42692881, 83747892, 8791066, 71333116, 7182642, 32058615, 81774825, 20122224, 37659250, 18699206, 73786944, 16567550, 83302115, 24953498, 86242799, 20002147, 67474219, 66663942, 79322415, 83269727, 45996863, 96709982, 5832946, 79821897, 40152546, 99971982, 97057187, 66832478, 44983451, 89637706, 88904910, 29065964, 31904591, 10366309, 76825057, 92787493, 16405341, 68041839, 97281847, 90272749, 8634541, 94090109, 80251430, 84684495, 76315420, 84840549, 88251446, 61623915, 63372756, 7104732, 66667729, 28787861, 28796059, 54199160, 79657802, 79922758, 7229550, 62115552, 74137926, 26114953, 21919959, 6497830, 28734791, 11161731, 27325428, 34236719, 38008118, 7919588, 65715134, 53547802, 44844121, 38009615, 86460488, 99604946, 99861373, 40027975, 75135448, 98739783, 95395112, 13819358, 42644903, 62803858, 21289531, 58180653, 66271566, 85117092, 99729357, 48260151, 84904436, 51047803, 54868730, 42073124, 51464002, 18783702, 4099191, 56461322, 33699435, 41442762, 29834512, 16971929, 55143724, 16424199, 4091162, 30811010, 27625735, 54232247, 27665211, 79806380, 49271185, 63015256, 16684829, 50188404, 72238278, 8733068, 37435892, 60955663, 14731700, 72357096, 55669657, 54606384, 86022504, 87710366, 36780454, 56955985, 45848907, 45381876, 57803235, 83155442, 30771409, 32151165, 50806615, 90090964, 92353856, 5111370, 62693428, 65017137, 9886593, 79191827, 99917599, 92803766, 40677414, 38645117, 59371804, 80316608, 22435353, 45667668, 45617087, 69579137, 96852321, 3233084, 17894977, 8099994, 28550822, 10961421, 58224549, 30998561, 82427263, 8651647, 66903004, 47792865, 55470718, 26397786, 36580610, 51472275, 44915235, 15075176, 10309525, 19898053, 78884452, 69605283, 20486294, 35456853, 22942635, 15902805, 54263880, 76170907, 19486173, 30090481, 66116458, 43045786, 38341669, 92867155, 61728685, 47887470, 29959549, 61263068, 89217461, 16097038, 43933006, 3233569, 55189057, 30543215, 8913721, 13468268, 874791, 73617245, 72614359, 37739481, 415901, 83948335, 92692283, 23110625, 9175338, 7685448, 71300104, 27187213, 47708850, 71522968, 7646095, 37620363, 73109997, 7066775, 29466406, 18131876, 88047921, 90004325, 40781449, 82532312, 66428795, 14363867, 85571389, 29994197, 39201414, 45919976, 47298834, 1431742, 73021291, 62496012, 36930650, 38256849, 41092102, 17037369, 84187166, 17386996, 71657078, 46540998, 40534591, 54058788, 247198, 94076128, 168541, 4985896, 22721500, 6521313, 44550764, 69697787, 68694897, 56515456, 18833224, 43501211, 54517921, 32274392, 5267545, 83210802, 90310261, 99549373, 22405120, 76540481, 94516935, 91727510, 26139110, 33431960, 47361209, 21533347, 58208470, 75820087, 90013093, 35092039, 16445503, 72019362, 78785507, 28928175, 3487592, 58749, 91141395, 6221471, 9623492, 45237957, 3880712, 66045587, 49328608, 23569917, 1936762, 9860968, 68816413, 42947632, 95290172, 36468541, 67084351, 30366150, 54014062, 9829782, 749283, 4199704, 9860195, 59405277, 65454636, 90061527, 17365188, 97011160, 14626618, 84166196, 54663246, 64098930, 30891921, 8807940, 12416768, 3773993, 24619760, 30787683, 1569515, 70727211, 79738755, 15064655, 59197747, 24826575, 65880522, 5482538, 13348726, 61928316, 52613508, 33553959, 93933709, 94360702, 7300047, 72793444, 98653983, 82052050, 34698463, 51507425, 80014588, 78909561, 35192533, 7955293, 61741594, 91957544, 72738685, 39847321, 70372191, 43322743, 66885828, 77694700, 81172706, 37501808, 93562257, 18504197, 41245325, 67281495, 55960386, 56473732, 47090124, 89811711, 71316369, 52204879, 2607799, 74357852, 14045383, 16054533, 49641577, 68316156, 57359924, 1204161, 4069912, 99021067, 95726235, 12024238, 60393039, 56484900, 78300864, 77284619, 3294781, 11757872, 74452589, 11581181, 31727379, 50567636, 39986008, 15536795, 26292919, 89078848, 57240218, 6986898, 50702367, 52261574, 46870723, 29819940, 96318094, 6871053, 34432810, 65851721, 73031054, 94911072, 44846932, 77300457, 43506672, 74614639, 45075471, 39553046, 83368048, 32161669, 36139942, 90158785, 23432750, 72725103, 51588015, 9058407, 85242963, 97940276, 66137019, 45790169, 49598724, 26493119, 81855445, 92215320, 57961282, 98462867, 17957593, 72274002, 90457870, 22450468, 40865610, 75128745, 34698428, 88653118, 44664587, 66250369, 508198, 2208785, 96420429, 91802888, 75052463, 91831487, 85023028, 36933038, 49236559, 37280276, 47893286, 36189527, 92398073, 34667286, 32250045, 77272628, 15163258, 89996536, 59614746, 69848388, 98130363, 73392814, 85695762, 21993752, 73222868, 22879907, 38061439, 44177011, 176257, 93790285, 8729146, 88130087, 64157906, 68000591, 75407347, 86301513, 92554120, 20765474, 55850790, 37675718, 37183543, 53666583, 89804152, 36685023, 11543098, 16019925, 66322959, 55615885, 26507214, 36845587, 19101477, 48892201, 34295794, 29635537, 16099750, 92604458, 99690194, 74724075, 44889423, 4508700, 99297164, 38022834, 27041967, 96726697, 77898274, 52060076, 90654836, 28851716, 92692978, 91938191, 19272365, 86798033, 50668599, 68204242, 9257405, 51466049, 83083131, 99226875, 96193415, 70420215, 77072625, 57658654, 62762857, 33201905, 89699445, 68939068, 26744917, 8128637, 53632373, 84127901, 40686254, 1250437, 99515901, 22129328, 14349098, 17727650, 7502255, 37891451, 16380211, 2717150, 67124282, 44627776, 91281584, 57020481, 36753250, 65038678, 71083565, 93566986, 44481640, 61859581, 44842615, 93057697, 19376156, 69136837, 59109400, 83651211, 4787945, 35512853, 4806458, 18001617, 97561745, 8373289, 57241521, 78218845, 19891772, 44348328, 69641513, 19939935, 71965942, 38510840, 21001913, 51715482, 93359396, 65271999, 42782652, 55602660, 6038457, 30694952, 20568363, 81274566, 75229462, 14326617, 95010552, 88444207, 26392416, 91990218, 57393458, 84406788, 10358899, 4978543, 34493392, 62430984, 32159704, 14723410, 61712234, 16595436, 24058273, 53393358, 38658347, 21673260, 80246713, 40984766, 33061250, 64602895, 67031644, 77377183, 62552524, 47213183, 78561158, 6111563, 4204661, 42307449, 26063929, 84293052, 65081429, 2331773, 80555751, 62936963, 30463802, 32590267, 41380093, 92541302, 18411915, 89214616, 69786756, 92998591, 22766820, 5073754, 29029316, 79136082, 6808825, 22113133, 5970607, 41481685, 82651278, 61859143, 24915585, 24314885, 87720882, 23194618, 72777973, 31491938, 6819644, 59501487, 34497327, 37166608, 62428472, 67513640, 77413012, 36396314, 76330843, 44060493, 94595668, 30163921, 61897582, 72278539, 48893685, 9398733, 72358170, 99125126, 61141874, 95251277, 82339363, 68824981, 15148031, 23848565, 60176618, 20642888, 26102057, 79942022, 29510992, 17539625, 26863229, 86001008, 33304202, 68102437, 62357986, 8055981, 36312813, 96735716, 4515343, 57248122, 98943869, 60106217, 61271144, 37957788, 17976208, 81677380, 94539824, 6525948, 1197320, 20836893, 81853704, 65338021, 81898046, 62490109, 68110330, 89046466, 48360198, 76434144, 22108665, 8129978, 75777973, 73168565, 60581278, 33123618, 17016156, 1022677, 77301523, 39171895, 42199455, 76671482, 52293995, 54427233, 46851987, 62740044, 49597667, 37560164, 15535065, 6793819, 112651, 44245960, 70367851, 51426311, 86118021, 58751351, 39373729, 77787724, 43376279, 85711894, 59910624, 97379790, 72732205, 69355476, 74110882, 33935899, 65074535, 56153345, 99524975, 55247455, 4668450, 98948034, 30218878, 9603598, 97783876, 24168411, 33565483, 17081350, 17385531, 82726008, 47738185, 48774913, 34946859, 67793644, 97641116, 86821229, 74743862, 8696647, 45306070, 19457589, 45049308, 16387575, 41092172, 83533741, 93053405, 4434662, 33336148, 44473167, 25842078, 27185644, 87160386, 75153252, 3088684, 53842979, 60430369, 78549759, 38494874, 70782102, 22200378, 20885148, 26734892, 45794415, 53648154, 83789391, 37224844, 3235882, 77312810, 23134715, 76703609, 79880247, 30653863, 38365584, 33249630, 50007421, 17146629, 68644627, 95957797, 91664334, 36135, 4119747, 7687278, 10264691, 43152977, 20645197, 64055960, 99333375, 37192445, 12664567, 10453030, 91255408, 526217, 91240048, 13470059, 75543508, 17068582, 43357947, 26998766, 66611759, 48395186, 13862149, 63628376, 6157724, 53802686, 72495719, 24591705, 20681921, 70596786, 89499542, 55770687, 62232211, 31733363, 45990383, 69255765, 86543538, 40197395, 12303248, 6505939, 23740167, 74441448, 77187825, 99224392, 21397057, 62452034, 36812683, 70800879, 99658235, 33628349, 51315460, 9259676, 51590803, 17857111, 61373987, 30569392, 39801351, 4798568, 1239555, 96906410, 36808486, 29401781, 94711842, 1375023, 4064751, 11365791, 2917920, 64848072, 70004753, 67030811, 82779622, 15948937, 62051033, 29932657, 53084256 +47792865, 72373496, 76330843, 68824981, 55960386, 58751351, 95957797, 98948034, 94595668, 35996293, 69579137, 87598888, 8129978, 57163802, 37620363, 6808825, 37659250, 19272365, 5832946, 90310261, 2891150, 96735716, 96420429, 82178706, 53666583, 79136082, 83747892, 5970607, 49641577, 7517032, 49271185, 18806856, 31491938, 99524975, 33201905, 26744917, 16380211, 66832478, 14220886, 68694897, 39553046, 19376156, 69136837, 14947650, 47361209, 68102437, 8055981, 9623492, 93270084, 26776922, 508198, 98943869, 59981773, 26392416, 6497830, 92398073, 78884452, 38658347, 44844121, 8729146, 98648327, 68000591, 10649306, 92867155, 30463802, 415901, 49597667, 39847321, 92604458, 70541760, 22113133, 78766358, 79806380, 72238278, 64055960, 60955663, 20140249, 96193415, 11757872, 45996863, 84127901, 17385531, 10453030, 96318094, 2717150, 69697787, 56515456, 65017137, 43501211, 93057697, 85242963, 75543508, 49598724, 21001913, 90013093, 72019362, 28928175, 40865610, 66045587, 30998561, 30694952, 36933038, 10358899, 17365188, 38008118, 45794415, 73392814, 89499542, 21993752, 8807940, 40984766, 68110330, 15064655, 61928316, 62552524, 95395112, 55850790, 61263068, 37675718, 4204661, 40197395, 54427233, 51507425, 73617245, 36845587, 62936963, 71300104, 22766820, 77694700, 12348118, 18663507, 86118021, 4508700, 34044787, 33237508, 52204879, 91664334, 92692978, 9257405, 95726235, 20645197, 78300864, 83083131, 40781854, 83269727, 12664567, 53632373, 99515901, 34432810, 97641116, 44983451, 22721500, 44846932, 57020481, 74614639, 32274392, 4787945, 76540481, 59371804, 44473167, 29510992, 58208470, 38510840, 16445503, 90457870, 93359396, 26998766, 58749, 34698428, 73235980, 28787861, 54199160, 3183975, 68816413, 13862149, 6157724, 15163258, 67227442, 20681921, 65715134, 24058273, 65338021, 21070110, 62232211, 86460488, 33061250, 30395570, 31733363, 76170907, 47213183, 69255765, 63059024, 7423788, 98739783, 13819358, 20765474, 65275241, 61728685, 60581278, 29959549, 569864, 33553959, 42967683, 17058722, 8913721, 30653863, 61815223, 19101477, 91957544, 38365584, 77620120, 85116755, 89214616, 33249630, 66885828, 44245960, 71522968, 56461322, 7011964, 25636669, 29834512, 43376279, 16424199, 77898274, 90004325, 69355476, 27665211, 18699206, 33935899, 4069912, 50668599, 23194618, 72777973, 85571389, 8733068, 24953498, 74441448, 4668450, 99226875, 61982238, 77072625, 55669657, 38256849, 37166608, 37192445, 89078848, 96709982, 22129328, 48774913, 40534591, 6871053, 67793644, 50806615, 90090964, 48893685, 45306070, 36812683, 77300457, 18833224, 54517921, 99917599, 31904591, 93566986, 32161669, 92803766, 35512853, 17764950, 22405120, 4095116, 91727510, 93053405, 29516592, 8634541, 33431960, 81855445, 69641513, 96852321, 71965942, 84840549, 88251446, 48395186, 44664587, 66250369, 74137926, 26114953, 91831487, 36468541, 88444207, 67084351, 91990218, 57393458, 84406788, 9829782, 48675329, 36189527, 28734791, 53802686, 90061527, 51590803, 97011160, 14626618, 34493392, 32699744, 53648154, 15902805, 38061439, 24619760, 61373987, 40027975, 30090481, 22108665, 9575954, 78561158, 5640302, 31161687, 62803858, 29932657, 89217461, 48088883, 26275734, 58180653, 82052050, 11543098, 76703609, 66322959, 80014588, 84904436, 84002370, 4798568, 44847298, 80555751, 78909561, 7955293, 83948335, 73124510, 72738685, 70372191, 59177718, 32426752, 42073124, 63967300, 42692881, 5073754, 70367851, 37501808, 67281495, 51426311, 47090124, 17146629, 41442762, 89811711, 59329511, 71333116, 29466406, 2607799, 85711894, 41481685, 82651278, 4091162, 24915585, 72732205, 54232247, 71920426, 7687278, 86798033, 68204242, 29994197, 39201414, 16567550, 1431742, 20002147, 67474219, 30218878, 3294781, 62496012, 59501487, 41092102, 89699445, 84187166, 50567636, 45848907, 17386996, 40686254, 52261574, 82726008, 99224392, 17727650, 47738185, 54058788, 37891451, 72278539, 92353856, 9398733, 5111370, 62693428, 91281584, 36753250, 16387575, 61960400, 71083565, 83368048, 82339363, 9599614, 84349107, 8115266, 4806458, 45407418, 26102057, 83533741, 66137019, 26139110, 45790169, 4434662, 33797252, 8373289, 94090109, 78218845, 3509435, 98462867, 25842078, 83133790, 33304202, 75128745, 88653118, 45237957, 95581843, 79657802, 53842979, 62115552, 78602717, 9259676, 8651647, 66903004, 85023028, 42947632, 14093520, 55470718, 75229462, 95290172, 70782102, 61271144, 54014062, 17976208, 9860195, 59405277, 11161731, 18466635, 22997281, 89996536, 84166196, 64098930, 6525948, 26734892, 6153406, 69605283, 30891921, 12416768, 99861373, 75135448, 48360198, 92071990, 66116458, 82886381, 21289531, 17016156, 77301523, 16097038, 43933006, 7300047, 99729357, 13468268, 65081429, 2331773, 46851987, 37739481, 1239555, 92541302, 18411915, 74724075, 44889423, 81172706, 2204165, 4099191, 33699435, 8791066, 71316369, 18131876, 7182642, 30811010, 82532312, 1204161, 65074535, 50188404, 87720882, 29401781, 73786944, 51466049, 83302115, 1375023, 73021291, 824872, 34497327, 57658654, 74452589, 86022504, 24168411, 11581181, 31727379, 39986008, 45381876, 8128637, 15536795, 6986898, 50702367, 17081350, 1250437, 32151165, 45995325, 40152546, 94076128, 10597197, 99971982, 97057187, 65851721, 321665, 62452034, 99125126, 29065964, 19457589, 45049308, 45075471, 26664538, 82327024, 90158785, 70800879, 22435353, 97281847, 80251430, 13173644, 21533347, 92215320, 75820087, 44348328, 26863229, 17957593, 17894977, 35092039, 22450468, 49882705, 99658235, 61623915, 53084256, 3487592, 11923835, 6221471, 36312813, 49328608, 79922758, 91802888, 6038457, 81805959, 23569917, 38494874, 14326617, 36580610, 70036438, 4978543, 15075176, 20885148, 34667286, 27325428, 68128438, 62430984, 32159704, 53547802, 53393358, 55770687, 85695762, 21673260, 3773993, 44177011, 89046466, 1569515, 70004753, 176257, 20867149, 70727211, 64602895, 75407347, 30569392, 68875490, 75777973, 6111563, 60102965, 76671482, 52293995, 34698463, 63506281, 26507214, 62740044, 72614359, 48892201, 15535065, 51047803, 112651, 12303248, 27187213, 7646095, 73109997, 36808486, 18783702, 56473732, 39373729, 68644627, 38022834, 77787724, 27041967, 74357852, 31126490, 81774825, 68316156, 57359924, 27625735, 24314885, 91938191, 16684829, 56153345, 60393039, 47298834, 37435892, 55247455, 67030811, 72357096, 70420215, 36930650, 79322415, 54606384, 33565483, 67513640, 57240218, 4064751, 29819940, 34946859, 73031054, 53888755, 4985896, 82897371, 2917920, 67124282, 56531125, 9886593, 61141874, 44627776, 95251277, 92530431, 44481640, 61859581, 44842615, 23848565, 72725103, 99549373, 92787493, 41092172, 16405341, 79942022, 45667668, 97561745, 57241521, 19891772, 57961282, 84684495, 88698958, 19939935, 17068582, 43357947, 8099994, 91141395, 66611759, 54762643, 3088684, 66667729, 68702632, 3880712, 7229550, 33628349, 51315460, 1936762, 81274566, 60106217, 21919959, 42237907, 63628376, 64848072, 4199704, 67451935, 81677380, 94539824, 54663246, 98130363, 17857111, 7919588, 61712234, 15015906, 81853704, 35456853, 22879907, 22942635, 62490109, 54263880, 83789391, 12571310, 19486173, 24826575, 5482538, 78589145, 44784505, 3235882, 52613508, 86301513, 39801351, 62051033, 33123618, 47887470, 77312810, 82979980, 93933709, 23134715, 86543538, 89804152, 72793444, 16019925, 79880247, 36505482, 61741594, 29635537, 9188443, 7685448, 69786756, 92998591, 43322743, 47708850, 51464002, 41245325, 16971929, 97379790, 36135, 52060076, 40781449, 28851716, 63015256, 14363867, 44479073, 99021067, 56484900, 86242799, 14731700, 6819644, 5822202, 56424103, 9603598, 62762857, 97783876, 77187825, 36780454, 17037369, 68939068, 14349098, 46870723, 21397057, 71657078, 79821897, 59318837, 247198, 86821229, 6521313, 44550764, 74743862, 8696647, 88904910, 526217, 65038678, 43506672, 79191827, 24733232, 10366309, 36139942, 91240048, 83210802, 40677414, 38645117, 59109400, 27411561, 94516935, 33336148, 26493119, 18001617, 86001008, 76315420, 78785507, 28550822, 65271999, 58224549, 4515343, 57248122, 55602660, 9860968, 89419466, 83150534, 95010552, 49236559, 749283, 51472275, 93515664, 37957788, 34236719, 1197320, 20836893, 24591705, 16595436, 3633375, 70596786, 19898053, 20486294, 81898046, 38009615, 64087743, 99604946, 30787683, 37224844, 67031644, 43045786, 42644903, 73168565, 94360702, 3233569, 55189057, 66271566, 42307449, 48260151, 26063929, 874791, 84293052, 35192533, 32590267, 41380093, 75986488, 6793819, 16099750, 98371444, 48658605, 54868730, 71469330, 18504197, 88047921, 55143724, 14045383, 16054533, 32058615, 74110882, 76960303, 45919976, 10264691, 66663942, 87710366, 56955985, 26292919, 57803235, 82779622, 168541, 669105, 91255408, 94911072, 15148031, 5267545, 51588015, 9058407, 97940276, 45617087, 17539625, 13231279, 3233084, 51715482, 87160386, 63372756, 33895336, 42782652, 2208785, 60430369, 78549759, 20568363, 75052463, 26397786, 30366150, 37280276, 47893286, 44915235, 65454636, 32250045, 59614746, 73222868, 55753905, 64157906, 45990383, 1022677, 36685023, 37560164, 29188588, 9175338, 96906410, 99690194, 93562257, 6505939, 7066775, 99965001, 99297164, 96726697, 59910624, 90654836, 4119747, 92033260, 94711842, 43152977, 77284619, 36396314, 30771409, 7502255, 54987042, 60176618, 13470059, 83651211, 20642888, 48673079, 75153252, 10961421, 7104732, 1788101, 10309525, 72495719, 77272628, 30764367, 69848388, 79738755, 59197747, 13348726, 98653983, 42199455, 51149804, 85117092, 55615885, 23110625, 34295794, 45428665, 50007421, 61859143, 20122224, 65047700, 66428795, 40356877, 66319530, 62428472, 77413012, 44060493, 46540998, 89637706, 23432750, 90272749, 30139692, 62357986, 82427263, 22200378, 15948937, 14723410, 93790285, 65880522, 88130087, 38341669, 37183543, 29029316, 63152504, 23740167, 12024238, 83155442, 72358170, 76825057, 80316608, 68041839, 80246713, 77377183, 92554120, 39171895, 30543215, 92692283, 99333375, 61897582, 11365791, 27185644, 30163921, 72274002, 28796059, 76434144 +6819644, 7502255, 86821229, 15148031, 13231279, 3633375, 79880247, 99965001, 74743862, 83368048, 4806458, 13819358, 61815223, 92692283, 36780454, 39986008, 97057187, 93566986, 57961282, 28928175, 28796059, 79657802, 11161731, 90061527, 84166196, 69605283, 35456853, 30787683, 5482538, 569864, 82979980, 39171895, 66322959, 18663507, 68316156, 30218878, 33565483, 67513640, 73031054, 67124282, 72358170, 60176618, 4787945, 27411561, 17894977, 17068582, 26998766, 57393458, 36189527, 10309525, 94539824, 30395570, 61373987, 77377183, 66116458, 43045786, 98739783, 92554120, 1022677, 86543538, 3233569, 40197395, 98371444, 12348118, 18783702, 56473732, 58751351, 71333116, 96726697, 59910624, 32058615, 77898274, 61859143, 24915585, 27665211, 65074535, 56153345, 73786944, 83302115, 60393039, 86242799, 20002147, 36930650, 97783876, 62428472, 99224392, 17727650, 34946859, 669105, 6521313, 44550764, 11365791, 8696647, 5111370, 56531125, 99125126, 32161669, 61859581, 91240048, 83210802, 35512853, 41092172, 68041839, 26493119, 97561745, 90272749, 8373289, 47361209, 87160386, 88653118, 66250369, 508198, 2208785, 96420429, 9860968, 81274566, 83150534, 95290172, 54014062, 47893286, 48675329, 15075176, 15948937, 77272628, 81677380, 54663246, 19898053, 78884452, 62232211, 99604946, 1569515, 99861373, 76434144, 64157906, 30090481, 3235882, 38341669, 29932657, 82886381, 21289531, 72793444, 42199455, 63506281, 874791, 84904436, 37739481, 37560164, 45428665, 77694700, 29029316, 37501808, 93562257, 47090124, 74357852, 49641577, 41481685, 28851716, 24314885, 1204161, 85571389, 29994197, 51466049, 94711842, 5822202, 66663942, 62762857, 41092102, 24168411, 33201905, 40686254, 82726008, 47738185, 82779622, 32151165, 6871053, 247198, 61897582, 91255408, 22721500, 82897371, 68694897, 44627776, 36812683, 91281584, 93057697, 90310261, 13470059, 83651211, 97940276, 33336148, 13173644, 98462867, 27185644, 28550822, 91141395, 63372756, 66611759, 9623492, 36312813, 3880712, 42782652, 6038457, 62115552, 81805959, 75052463, 38494874, 95010552, 70782102, 67084351, 42237907, 30366150, 4978543, 22997281, 6525948, 98130363, 26734892, 67227442, 53393358, 73392814, 89499542, 20486294, 73222868, 22879907, 44844121, 80246713, 64087743, 75135448, 59197747, 88130087, 22108665, 61928316, 62552524, 78561158, 5640302, 68875490, 95395112, 42644903, 65275241, 55850790, 47887470, 17016156, 16097038, 7300047, 53666583, 89804152, 55189057, 82052050, 99729357, 84293052, 26507214, 44847298, 80555751, 62936963, 78909561, 1239555, 83948335, 32590267, 72738685, 6793819, 85116755, 18411915, 51047803, 70372191, 92998591, 66885828, 74724075, 81172706, 70541760, 4099191, 8791066, 22113133, 71316369, 59329511, 16971929, 88047921, 55143724, 72732205, 69355476, 14363867, 50188404, 10264691, 8733068, 56484900, 55247455, 73021291, 61982238, 9603598, 68939068, 83155442, 46540998, 65851721, 88904910, 71083565, 54517921, 24733232, 51588015, 99549373, 35996293, 26139110, 22435353, 45617087, 57241521, 19891772, 81855445, 84684495, 96852321, 25842078, 90013093, 72274002, 43357947, 93359396, 75153252, 11923835, 34698428, 8055981, 66667729, 79922758, 91802888, 3183975, 23569917, 8651647, 1936762, 91831487, 42947632, 55470718, 21919959, 36933038, 1788101, 26397786, 36580610, 13862149, 49236559, 22200378, 64848072, 44915235, 72495719, 34236719, 14723410, 7919588, 70596786, 85695762, 38658347, 21673260, 38009615, 20867149, 70727211, 31733363, 93790285, 76170907, 12571310, 9575954, 69255765, 75407347, 39801351, 37675718, 33553959, 51149804, 85117092, 48260151, 30653863, 65081429, 2331773, 84002370, 48892201, 38365584, 39847321, 92604458, 112651, 43322743, 5073754, 27187213, 44889423, 44245960, 71522968, 7646095, 79136082, 51464002, 25636669, 39373729, 38022834, 29466406, 27041967, 2607799, 14045383, 16054533, 97379790, 82651278, 71920426, 65047700, 86798033, 87720882, 72238278, 72373496, 16567550, 18806856, 12024238, 83083131, 66319530, 99226875, 77072625, 89699445, 31727379, 17037369, 45996863, 77413012, 36396314, 96709982, 76330843, 4064751, 29819940, 79821897, 16380211, 10597197, 97641116, 53888755, 72278539, 62693428, 94911072, 65017137, 9886593, 45049308, 65038678, 61960400, 92530431, 68824981, 10366309, 36139942, 92803766, 59109400, 23432750, 48673079, 66137019, 2891150, 18001617, 8634541, 33431960, 19939935, 3233084, 38510840, 21001913, 16445503, 51715482, 22450468, 84840549, 75128745, 58749, 10961421, 65271999, 62357986, 73235980, 68702632, 53842979, 57248122, 74137926, 26114953, 98943869, 89419466, 75229462, 84406788, 70036438, 4199704, 37957788, 18466635, 32250045, 97011160, 68128438, 15163258, 38008118, 81853704, 53547802, 15902805, 8807940, 12416768, 70004753, 48360198, 67031644, 13348726, 98648327, 45990383, 92071990, 86301513, 63059024, 31161687, 60581278, 77312810, 89217461, 23134715, 94360702, 60102965, 26275734, 17058722, 30543215, 52293995, 76703609, 26063929, 16019925, 80014588, 73617245, 36505482, 30463802, 19101477, 91957544, 49597667, 77620120, 23110625, 16099750, 48658605, 42692881, 6808825, 41245325, 43376279, 31126490, 52060076, 54232247, 18699206, 19272365, 76960303, 16684829, 50668599, 9257405, 39201414, 1431742, 40781854, 67030811, 3294781, 20140249, 56424103, 74452589, 86022504, 87710366, 99333375, 45848907, 12664567, 6986898, 50702367, 17081350, 14349098, 46870723, 21397057, 44060493, 40534591, 40152546, 94076128, 50806615, 66832478, 4985896, 48893685, 9398733, 45306070, 56515456, 95251277, 18833224, 45075471, 82339363, 44481640, 69136837, 40677414, 90158785, 17764950, 20642888, 22405120, 4095116, 83533741, 94516935, 59371804, 93053405, 33797252, 49598724, 94090109, 80251430, 69579137, 58208470, 88698958, 71965942, 33304202, 35092039, 72019362, 40865610, 6221471, 45237957, 30998561, 4515343, 78549759, 30694952, 33628349, 51315460, 85023028, 60106217, 14326617, 61271144, 88444207, 91990218, 63628376, 93515664, 67451935, 17976208, 9860195, 65454636, 34493392, 32159704, 89996536, 64098930, 69848388, 24591705, 61712234, 15015906, 16595436, 6153406, 21070110, 82178706, 40984766, 86460488, 33061250, 3773993, 54263880, 79738755, 37224844, 55753905, 24826575, 65880522, 7423788, 20765474, 62803858, 61728685, 61263068, 37183543, 93933709, 42967683, 43933006, 98653983, 8913721, 13468268, 54427233, 36845587, 73124510, 41380093, 34295794, 75986488, 29188588, 9175338, 71300104, 69786756, 63967300, 47708850, 37620363, 63152504, 33699435, 17146629, 4508700, 89811711, 68644627, 95957797, 77787724, 29834512, 91664334, 85711894, 81774825, 7517032, 30811010, 37659250, 74110882, 33935899, 91938191, 66428795, 63015256, 72777973, 40356877, 95726235, 47298834, 20645197, 77284619, 72357096, 96193415, 34497327, 70420215, 57658654, 77187825, 84187166, 37192445, 45381876, 15536795, 1250437, 5832946, 71657078, 45995325, 37891451, 168541, 30163921, 2717150, 90090964, 44846932, 29065964, 19457589, 36753250, 77300457, 43501211, 32274392, 26664538, 23848565, 38645117, 92787493, 70800879, 91727510, 75820087, 3509435, 69641513, 26863229, 86001008, 83133790, 76315420, 49882705, 68102437, 61623915, 53084256, 3487592, 48395186, 54762643, 44664587, 93270084, 28787861, 33895336, 26776922, 49328608, 82427263, 60430369, 7229550, 14093520, 47792865, 10358899, 9829782, 59405277, 53802686, 14626618, 17857111, 24058273, 53648154, 81898046, 22942635, 62490109, 89046466, 176257, 8729146, 19486173, 44784505, 73168565, 33123618, 29959549, 77301523, 34698463, 4798568, 35192533, 89214616, 96906410, 32426752, 42073124, 70367851, 36808486, 6505939, 23740167, 51426311, 50007421, 56461322, 41442762, 33237508, 18131876, 7182642, 16424199, 4091162, 20122224, 40781449, 4119747, 49271185, 23194618, 29401781, 45919976, 31491938, 1375023, 43152977, 37435892, 64055960, 78300864, 60955663, 14731700, 67474219, 824872, 62496012, 11757872, 55669657, 54606384, 37166608, 11581181, 50567636, 52261574, 48774913, 30771409, 10453030, 94595668, 99971982, 54987042, 44983451, 14220886, 2917920, 57020481, 16387575, 526217, 99917599, 82327024, 9599614, 8115266, 16405341, 9058407, 45667668, 30139692, 92215320, 44348328, 17957593, 8099994, 7104732, 78602717, 9259676, 20568363, 66903004, 59981773, 28734791, 6157724, 92398073, 27325428, 30764367, 59614746, 1197320, 20836893, 20681921, 65338021, 21993752, 30891921, 68000591, 30569392, 10649306, 6111563, 48088883, 66271566, 11543098, 51507425, 72614359, 7955293, 92541302, 15535065, 29635537, 57163802, 54868730, 33249630, 22766820, 71469330, 67281495, 7066775, 86118021, 7011964, 52204879, 92692978, 82532312, 92033260, 4069912, 44479073, 68204242, 98948034, 83269727, 26744917, 56955985, 84127901, 57240218, 17386996, 17385531, 99515901, 22129328, 54058788, 59318837, 34432810, 321665, 62452034, 61141874, 43506672, 74614639, 39553046, 79191827, 44842615, 84349107, 76825057, 72725103, 76540481, 26102057, 85242963, 79942022, 45790169, 29516592, 78218845, 17539625, 90457870, 78785507, 99658235, 58224549, 3088684, 95581843, 54199160, 55602660, 68816413, 51472275, 6497830, 24619760, 44177011, 83789391, 64602895, 52613508, 47213183, 62051033, 92867155, 55615885, 415901, 7685448, 99690194, 59177718, 12303248, 2204165, 73109997, 83747892, 55960386, 5970607, 57359924, 90004325, 99021067, 74441448, 4668450, 59501487, 79322415, 8128637, 26292919, 89078848, 67793644, 92353856, 69697787, 89637706, 5267545, 19376156, 14947650, 75543508, 97281847, 29510992, 21533347, 36468541, 26392416, 37280276, 749283, 51590803, 17365188, 62430984, 65715134, 45794415, 68110330, 87598888, 78589145, 8129978, 75777973, 36685023, 76671482, 42307449, 62740044, 9188443, 18504197, 99297164, 34044787, 78766358, 7687278, 99524975, 24953498, 53632373, 57803235, 80316608, 4434662, 44473167, 88251446, 96735716, 20885148, 15064655, 4204661, 61741594, 36135, 90654836, 79806380, 96318094, 31904591, 32699744, 55770687, 38061439, 58180653, 46851987, 27625735, 38256849, 45407418, 66045587, 40027975, 34667286 +18001617, 92692283, 77620120, 54058788, 82897371, 36580610, 67451935, 40984766, 112651, 43376279, 20645197, 64055960, 9603598, 11581181, 14349098, 23432750, 17957593, 35092039, 93270084, 64848072, 30764367, 24591705, 70596786, 59197747, 92554120, 77312810, 82979980, 40197395, 36845587, 9175338, 98371444, 7685448, 63152504, 92692978, 82532312, 66428795, 55669657, 10453030, 61897582, 72278539, 8696647, 44481640, 38645117, 13470059, 26493119, 92215320, 88698958, 19939935, 83133790, 68102437, 7104732, 68702632, 81805959, 51315460, 75052463, 38494874, 34667286, 22942635, 64602895, 5482538, 45990383, 39801351, 62051033, 874791, 72614359, 72738685, 74724075, 41245325, 96726697, 50668599, 56484900, 6819644, 84187166, 26292919, 77413012, 84127901, 96709982, 48774913, 54987042, 69697787, 88904910, 19457589, 91281584, 74614639, 79942022, 97940276, 45790169, 3509435, 44348328, 99658235, 3487592, 58749, 53842979, 91802888, 78549759, 33628349, 9259676, 14326617, 10358899, 63628376, 47893286, 53802686, 15163258, 89996536, 19898053, 21993752, 89046466, 30090481, 47213183, 69255765, 5640302, 95395112, 42644903, 29932657, 29959549, 23134715, 94360702, 39171895, 76703609, 48260151, 84002370, 73124510, 37560164, 44245960, 47708850, 70367851, 73109997, 47090124, 8791066, 33237508, 89811711, 71316369, 95957797, 61859143, 52060076, 28851716, 91938191, 65074535, 99021067, 29994197, 95726235, 60393039, 24953498, 77072625, 74452589, 87710366, 37166608, 33565483, 57803235, 82779622, 21397057, 46540998, 97057187, 2717150, 11365791, 321665, 94911072, 36812683, 45049308, 71083565, 45075471, 83651211, 35996293, 75543508, 4434662, 33336148, 29510992, 13231279, 75820087, 98462867, 71965942, 21001913, 10961421, 65271999, 6221471, 54762643, 62357986, 508198, 9860968, 42947632, 47792865, 75229462, 83150534, 13862149, 17976208, 6157724, 72495719, 14626618, 6525948, 61712234, 32699744, 65338021, 89499542, 8807940, 3773993, 54263880, 76170907, 12571310, 55753905, 65880522, 68000591, 92867155, 60581278, 569864, 7300047, 26275734, 72793444, 82052050, 66271566, 51149804, 26063929, 16019925, 62740044, 78909561, 7955293, 19101477, 83948335, 91957544, 29188588, 39847321, 57163802, 99690194, 92998591, 22766820, 43322743, 27187213, 36808486, 99297164, 41442762, 39373729, 34044787, 29466406, 52204879, 31126490, 14045383, 49641577, 57359924, 4119747, 54232247, 92033260, 29401781, 9257405, 39201414, 8733068, 94711842, 12024238, 37435892, 83083131, 96193415, 31727379, 37192445, 53632373, 40686254, 30771409, 67793644, 168541, 91255408, 14220886, 9398733, 5111370, 62693428, 62452034, 72358170, 99125126, 43506672, 32161669, 82327024, 23848565, 90310261, 60176618, 72725103, 4806458, 76540481, 85242963, 14947650, 80316608, 45667668, 97561745, 8634541, 3233084, 25842078, 38510840, 27185644, 33304202, 72274002, 16445503, 87160386, 72019362, 26998766, 28550822, 66611759, 88653118, 73235980, 79657802, 96735716, 2208785, 79922758, 23569917, 91831487, 70782102, 67084351, 749283, 27325428, 90061527, 17365188, 62430984, 94539824, 73392814, 21070110, 21673260, 68110330, 33061250, 30787683, 93790285, 40027975, 37224844, 88130087, 13348726, 77377183, 30569392, 13819358, 62803858, 55850790, 1022677, 61263068, 13468268, 46851987, 4798568, 44847298, 30463802, 49597667, 41380093, 6793819, 16099750, 89214616, 59177718, 12303248, 66885828, 12348118, 29029316, 71469330, 6505939, 51464002, 51426311, 18783702, 86118021, 71333116, 16971929, 2607799, 74357852, 37659250, 24314885, 18699206, 33935899, 7687278, 1204161, 65047700, 14363867, 44479073, 85571389, 31491938, 1375023, 43152977, 78300864, 66663942, 70420215, 11757872, 62762857, 79322415, 26744917, 36396314, 82726008, 46870723, 5832946, 40534591, 32151165, 6871053, 37891451, 34432810, 86821229, 4985896, 92353856, 68694897, 89637706, 9886593, 29065964, 57020481, 16387575, 83368048, 82339363, 15148031, 9599614, 76825057, 90158785, 4787945, 51588015, 20642888, 83533741, 66137019, 26139110, 22435353, 8373289, 47361209, 21533347, 69641513, 96852321, 86001008, 17068582, 8099994, 51715482, 75153252, 11923835, 63372756, 45237957, 28796059, 33895336, 30998561, 49328608, 42782652, 82427263, 96420429, 78602717, 26114953, 20568363, 98943869, 85023028, 37280276, 36189527, 44915235, 93515664, 28734791, 15075176, 59405277, 92398073, 15948937, 51590803, 77272628, 97011160, 32159704, 84166196, 64098930, 26734892, 20681921, 44844121, 62490109, 64087743, 30395570, 83789391, 87598888, 31733363, 8729146, 19486173, 48360198, 98648327, 61928316, 44784505, 3235882, 92071990, 86301513, 8129978, 7423788, 43045786, 21289531, 47887470, 77301523, 89217461, 93933709, 98653983, 55189057, 36685023, 52293995, 99729357, 51507425, 79880247, 84293052, 73617245, 84904436, 36505482, 61741594, 415901, 1239555, 92541302, 34295794, 15535065, 75986488, 85116755, 32426752, 71522968, 6808825, 23740167, 7011964, 17146629, 4508700, 58751351, 77787724, 97379790, 32058615, 68316156, 7517032, 30811010, 27625735, 76960303, 16684829, 56153345, 73786944, 16567550, 74441448, 66319530, 67474219, 67030811, 72357096, 99226875, 56424103, 86022504, 36780454, 62428472, 68939068, 67513640, 45848907, 89078848, 6986898, 50702367, 99515901, 52261574, 17727650, 247198, 10597197, 73031054, 44983451, 6521313, 44550764, 65017137, 61141874, 36753250, 18833224, 61960400, 54517921, 79191827, 61859581, 44842615, 36139942, 84349107, 91240048, 92803766, 83210802, 8115266, 41092172, 16405341, 48673079, 4095116, 2891150, 45617087, 90272749, 44473167, 57241521, 33431960, 69579137, 58208470, 57961282, 84684495, 93359396, 88251446, 40865610, 53084256, 9623492, 66250369, 57248122, 30694952, 8651647, 68816413, 60106217, 89419466, 59981773, 21919959, 95290172, 36933038, 54014062, 51472275, 4199704, 37957788, 65454636, 11161731, 18466635, 32250045, 81677380, 22997281, 68128438, 38008118, 98130363, 7919588, 20836893, 15015906, 24058273, 53393358, 35456853, 53648154, 73222868, 12416768, 86460488, 24619760, 61373987, 70004753, 75135448, 9575954, 75407347, 20765474, 65275241, 38341669, 73168565, 86543538, 16097038, 60102965, 48088883, 4204661, 3233569, 53666583, 58180653, 76671482, 85117092, 80555751, 23110625, 29635537, 9188443, 96906410, 54868730, 42073124, 63967300, 2204165, 37620363, 93562257, 79136082, 18504197, 70541760, 7066775, 50007421, 4099191, 68644627, 29834512, 85711894, 36135, 4091162, 40781449, 63015256, 4069912, 50188404, 87720882, 23194618, 51466049, 83302115, 99524975, 60955663, 4668450, 98948034, 3294781, 62496012, 59501487, 34497327, 97783876, 38256849, 41092102, 89699445, 17037369, 8128637, 15536795, 12664567, 17386996, 76330843, 99224392, 4064751, 47738185, 29819940, 7502255, 59318837, 65851721, 67124282, 56531125, 77300457, 39553046, 93566986, 26664538, 69136837, 59109400, 45407418, 22405120, 9058407, 70800879, 94516935, 80251430, 19891772, 17539625, 43357947, 49882705, 8055981, 66667729, 28787861, 3880712, 26776922, 55602660, 62115552, 74137926, 81274566, 95010552, 61271144, 26392416, 42237907, 22200378, 48675329, 70036438, 6497830, 4978543, 20885148, 59614746, 54663246, 17857111, 67227442, 16595436, 53547802, 45794415, 20486294, 62232211, 81898046, 80246713, 38061439, 1569515, 70727211, 76434144, 64157906, 22108665, 31161687, 75777973, 33123618, 37675718, 33553959, 6111563, 89804152, 11543098, 34698463, 63506281, 80014588, 37739481, 35192533, 48892201, 45428665, 5073754, 83747892, 67281495, 27041967, 18131876, 7182642, 55143724, 16054533, 59910624, 41481685, 77898274, 82651278, 20122224, 90654836, 74110882, 19272365, 86798033, 40356877, 10264691, 18806856, 1431742, 30218878, 77187825, 24168411, 99333375, 57240218, 83155442, 22129328, 71657078, 96318094, 45995325, 94595668, 30163921, 53888755, 90090964, 48893685, 2917920, 56515456, 44627776, 526217, 65038678, 43501211, 68824981, 93057697, 19376156, 35512853, 91727510, 68041839, 78218845, 81855445, 26863229, 76315420, 90457870, 84840549, 61623915, 75128745, 91141395, 3088684, 95581843, 66045587, 7229550, 66903004, 34493392, 14723410, 81853704, 65715134, 3633375, 6153406, 78884452, 69605283, 38658347, 176257, 99861373, 79738755, 24826575, 78589145, 52613508, 78561158, 68875490, 10649306, 82886381, 37183543, 42967683, 17058722, 30543215, 8913721, 42307449, 66322959, 55615885, 62936963, 61815223, 32590267, 18411915, 48658605, 51047803, 71300104, 42692881, 44889423, 81172706, 25636669, 38022834, 91664334, 88047921, 81774825, 90004325, 72732205, 27665211, 79806380, 68204242, 47298834, 86242799, 40781854, 73021291, 5822202, 36930650, 54606384, 83269727, 50567636, 39986008, 56955985, 45381876, 17385531, 1250437, 99971982, 66832478, 74743862, 45306070, 95251277, 32274392, 92530431, 5267545, 40677414, 99549373, 17764950, 59371804, 30139692, 29516592, 22450468, 78785507, 48395186, 58224549, 44664587, 60430369, 14093520, 36468541, 88444207, 1788101, 30366150, 9860195, 10309525, 1197320, 55770687, 82178706, 15902805, 44177011, 20867149, 67031644, 62552524, 66116458, 98739783, 17016156, 43933006, 65081429, 2331773, 38365584, 70372191, 69786756, 33249630, 77694700, 18663507, 7646095, 37501808, 99965001, 56461322, 5970607, 24915585, 71920426, 49271185, 72777973, 72373496, 45919976, 55247455, 14731700, 20002147, 77284619, 57658654, 33201905, 45996863, 17081350, 44060493, 97641116, 22721500, 24733232, 10366309, 26102057, 27411561, 33797252, 97281847, 13173644, 17894977, 28928175, 34698428, 55470718, 49236559, 9829782, 69848388, 85695762, 22879907, 30891921, 38009615, 15064655, 42199455, 54427233, 30653863, 26507214, 92604458, 55960386, 22113133, 59329511, 78766358, 16424199, 69355476, 824872, 40152546, 94076128, 16380211, 50806615, 92787493, 93053405, 49598724, 94090109, 36312813, 6038457, 3183975, 91990218, 34236719, 61728685, 56473732, 33699435, 20140249, 61982238, 79821897, 44846932, 31904591, 84406788, 72238278, 34946859, 669105, 90013093, 54199160, 4515343, 1936762, 57393458, 99604946, 63059024, 99917599, 26397786 +15064655, 73124510, 95726235, 45407418, 89078848, 44842615, 1936762, 21993752, 52613508, 78909561, 97379790, 90004325, 79806380, 92033260, 17764950, 76540481, 19939935, 83133790, 78785507, 47893286, 36505482, 42073124, 36808486, 56461322, 82532312, 24314885, 55247455, 45381876, 53632373, 14349098, 40152546, 54517921, 44481640, 2891150, 13173644, 90013093, 35092039, 49328608, 17976208, 8729146, 64602895, 10649306, 95395112, 89217461, 3233569, 72793444, 51149804, 9175338, 89214616, 59177718, 42692881, 40781449, 76960303, 56153345, 12024238, 43152977, 98948034, 72357096, 99226875, 84187166, 99971982, 18833224, 31904591, 93566986, 61859581, 33431960, 81855445, 86001008, 17894977, 40865610, 73235980, 30998561, 7229550, 85023028, 70036438, 27325428, 14626618, 32159704, 54663246, 1197320, 17857111, 32699744, 21070110, 31733363, 12571310, 59197747, 62552524, 30569392, 63059024, 5640302, 42644903, 61728685, 29959549, 77301523, 82052050, 54427233, 2331773, 46851987, 37739481, 415901, 32426752, 18504197, 7011964, 29466406, 16971929, 2607799, 31126490, 32058615, 54232247, 66428795, 4069912, 18806856, 56484900, 59501487, 96193415, 26292919, 6986898, 17386996, 99515901, 96318094, 45995325, 50806615, 44983451, 321665, 9886593, 61960400, 45075471, 92530431, 9599614, 69136837, 40677414, 16405341, 66137019, 26139110, 75543508, 45790169, 45667668, 97281847, 57241521, 29510992, 57961282, 69641513, 87160386, 88653118, 7104732, 9623492, 74137926, 26114953, 9259676, 14093520, 26397786, 36580610, 91990218, 18466635, 34236719, 30891921, 8807940, 38061439, 87598888, 79738755, 40027975, 55753905, 5482538, 64157906, 78561158, 86301513, 31161687, 77312810, 569864, 93933709, 94360702, 66271566, 85117092, 84002370, 44847298, 61815223, 41380093, 39847321, 7685448, 96906410, 99690194, 47708850, 71522968, 18663507, 37620363, 51426311, 99297164, 34044787, 71316369, 29834512, 91664334, 20122224, 90654836, 92692978, 86798033, 65074535, 85571389, 73786944, 45919976, 8733068, 40781854, 14731700, 77284619, 34497327, 74452589, 54606384, 11581181, 68939068, 52261574, 54058788, 59318837, 97057187, 65851721, 67793644, 54987042, 11365791, 82897371, 8696647, 62452034, 29065964, 95251277, 43501211, 24733232, 82327024, 5267545, 76825057, 8115266, 90158785, 4787945, 4806458, 22405120, 4095116, 26102057, 85242963, 97940276, 33797252, 49598724, 33336148, 30139692, 44473167, 72274002, 93359396, 72019362, 88251446, 75153252, 10961421, 63372756, 58224549, 44664587, 28787861, 33895336, 66045587, 508198, 62115552, 55470718, 26392416, 9829782, 51472275, 4199704, 36189527, 22997281, 94539824, 98130363, 24591705, 61712234, 53547802, 6153406, 45794415, 38658347, 53648154, 62490109, 38009615, 99604946, 24619760, 83789391, 70727211, 65880522, 88130087, 68000591, 61928316, 45990383, 44784505, 92071990, 7423788, 68875490, 13819358, 65275241, 92867155, 33123618, 61263068, 37675718, 82979980, 37183543, 7300047, 48088883, 26275734, 58180653, 26063929, 16019925, 13468268, 51507425, 80014588, 83948335, 77620120, 112651, 71469330, 83747892, 7066775, 55960386, 33699435, 78766358, 49641577, 36135, 37659250, 71920426, 27665211, 44479073, 94711842, 60393039, 47298834, 60955663, 824872, 5822202, 57658654, 11757872, 36780454, 83269727, 67513640, 17385531, 5832946, 40534591, 32151165, 73031054, 92353856, 2917920, 67124282, 56531125, 44846932, 88904910, 19457589, 83368048, 99917599, 32161669, 83533741, 14947650, 80316608, 29516592, 8373289, 94090109, 25842078, 17957593, 21001913, 8099994, 51715482, 22450468, 28550822, 53084256, 75128745, 48395186, 45237957, 68702632, 95581843, 96735716, 6038457, 8651647, 9860968, 81274566, 68816413, 14326617, 83150534, 95290172, 70782102, 36933038, 88444207, 54014062, 51590803, 17365188, 62430984, 15015906, 3633375, 65338021, 55770687, 35456853, 44844121, 15902805, 86460488, 54263880, 44177011, 48360198, 78589145, 76434144, 67031644, 30090481, 22108665, 98739783, 20765474, 55850790, 47887470, 23134715, 86543538, 16097038, 30543215, 99729357, 84293052, 73617245, 4798568, 72614359, 80555751, 7955293, 48892201, 92692283, 15535065, 72738685, 98371444, 70372191, 92998591, 12303248, 12348118, 37501808, 51464002, 6808825, 41245325, 67281495, 23740167, 47090124, 41442762, 95957797, 43376279, 81774825, 28851716, 27625735, 74110882, 7687278, 63015256, 16684829, 72238278, 83302115, 20645197, 1431742, 36930650, 37166608, 62428472, 39986008, 50702367, 1250437, 44060493, 6871053, 94076128, 94595668, 61897582, 4985896, 90090964, 48893685, 68694897, 56515456, 5111370, 99125126, 61141874, 36753250, 16387575, 74614639, 82339363, 15148031, 10366309, 23848565, 92803766, 35996293, 91727510, 45617087, 90272749, 47361209, 17539625, 21533347, 26863229, 71965942, 16445503, 49882705, 28928175, 3487592, 91141395, 54762643, 3088684, 42782652, 2208785, 60430369, 91802888, 3183975, 81805959, 78549759, 51315460, 60106217, 21919959, 61271144, 42237907, 57393458, 10358899, 63628376, 49236559, 37280276, 48675329, 28734791, 9860195, 59405277, 65454636, 11161731, 53802686, 32250045, 72495719, 15163258, 89996536, 30764367, 64098930, 6525948, 26734892, 20836893, 81853704, 73222868, 22942635, 3773993, 30787683, 1569515, 99861373, 75135448, 19486173, 13348726, 69255765, 66116458, 8129978, 29932657, 1022677, 33553959, 42967683, 39171895, 60102965, 53666583, 36685023, 76671482, 11543098, 48260151, 63506281, 66322959, 36845587, 62936963, 35192533, 30463802, 38365584, 23110625, 45428665, 29635537, 57163802, 18411915, 51047803, 71300104, 92604458, 63967300, 22766820, 77694700, 27187213, 44889423, 2204165, 6505939, 86118021, 50007421, 58751351, 89811711, 59329511, 96726697, 14045383, 16424199, 68316156, 7517032, 82651278, 57359924, 24915585, 72732205, 4119747, 49271185, 68204242, 29994197, 40356877, 16567550, 1375023, 99524975, 24953498, 4668450, 6819644, 30218878, 3294781, 20140249, 62496012, 66663942, 61982238, 70420215, 77072625, 97783876, 38256849, 33201905, 99333375, 12664567, 36396314, 57803235, 83155442, 22129328, 99224392, 30771409, 71657078, 10453030, 37891451, 10597197, 34432810, 66832478, 14220886, 74743862, 69697787, 45306070, 94911072, 36812683, 57020481, 32274392, 39553046, 90310261, 72725103, 99549373, 92787493, 9058407, 48673079, 79942022, 59371804, 8634541, 78218845, 69579137, 58208470, 92215320, 96852321, 27185644, 90457870, 68102437, 99658235, 61623915, 58749, 34698428, 6221471, 66611759, 53842979, 26776922, 55602660, 78602717, 75052463, 66903004, 91831487, 42947632, 36468541, 1788101, 64848072, 34667286, 97011160, 34493392, 7919588, 16595436, 24058273, 53393358, 89499542, 69605283, 85695762, 22879907, 62232211, 81898046, 21673260, 12416768, 24826575, 98648327, 77377183, 9575954, 3235882, 75407347, 38341669, 62803858, 82886381, 43933006, 4204661, 8913721, 42307449, 76703609, 65081429, 26507214, 62740044, 91957544, 34295794, 6793819, 9188443, 16099750, 54868730, 69786756, 74724075, 81172706, 99965001, 18783702, 4099191, 25636669, 4508700, 33237508, 22113133, 27041967, 18131876, 7182642, 16054533, 52060076, 69355476, 91938191, 19272365, 29401781, 9257405, 72373496, 39201414, 51466049, 31491938, 64055960, 67474219, 73021291, 55669657, 41092102, 86022504, 87710366, 17037369, 56955985, 37192445, 15536795, 96709982, 76330843, 46870723, 21397057, 53888755, 669105, 72278539, 91255408, 6521313, 9398733, 89637706, 72358170, 44627776, 91281584, 77300457, 71083565, 79191827, 68824981, 19376156, 35512853, 20642888, 70800879, 93053405, 44348328, 84684495, 17068582, 76315420, 26998766, 65271999, 62357986, 8055981, 93270084, 79657802, 82427263, 79922758, 67451935, 37957788, 92398073, 15948937, 77272628, 81677380, 68128438, 67227442, 78884452, 20486294, 80246713, 40984766, 33061250, 89046466, 61373987, 70004753, 20867149, 93790285, 43045786, 62051033, 75777973, 17058722, 61741594, 1239555, 37560164, 92541302, 75986488, 85116755, 5073754, 73109997, 56473732, 38022834, 77787724, 71333116, 52204879, 88047921, 55143724, 77898274, 4091162, 33935899, 65047700, 50188404, 99021067, 87720882, 72777973, 10264691, 37435892, 86242799, 67030811, 56424103, 24168411, 31727379, 26744917, 50567636, 77413012, 57240218, 17081350, 82779622, 7502255, 34946859, 16380211, 2717150, 44550764, 65017137, 526217, 65038678, 26664538, 84349107, 91240048, 83210802, 38645117, 60176618, 83651211, 4434662, 3509435, 98462867, 33304202, 66250369, 28796059, 54199160, 57248122, 30694952, 33628349, 38494874, 47792865, 75229462, 22200378, 93515664, 6157724, 90061527, 59614746, 84166196, 38008118, 69848388, 73392814, 82178706, 68110330, 30395570, 176257, 47213183, 73168565, 21289531, 17016156, 6111563, 42199455, 52293995, 34698463, 874791, 84904436, 55615885, 66885828, 79136082, 39373729, 5970607, 85711894, 41481685, 30811010, 18699206, 1204161, 14363867, 50668599, 23194618, 89699445, 40686254, 82726008, 4064751, 48774913, 79821897, 247198, 168541, 22721500, 62693428, 45049308, 13470059, 23432750, 51588015, 41092172, 22435353, 26493119, 19891772, 88698958, 3233084, 36312813, 4515343, 96420429, 59981773, 84406788, 6497830, 44915235, 4978543, 15075176, 10309525, 14723410, 65715134, 19898053, 39801351, 60581278, 89804152, 98653983, 55189057, 19101477, 29188588, 48658605, 33249630, 43322743, 7646095, 93562257, 17146629, 68644627, 61859143, 83083131, 66319530, 9603598, 79322415, 77187825, 33565483, 45848907, 47738185, 29819940, 46540998, 86821229, 43506672, 36139942, 27411561, 68041839, 18001617, 97561745, 75820087, 11923835, 66667729, 3880712, 23569917, 20568363, 95010552, 13862149, 20681921, 70596786, 64087743, 37224844, 76170907, 92554120, 30653863, 49597667, 29029316, 74357852, 59910624, 20002147, 62762857, 45996863, 84127901, 59109400, 94516935, 80251430, 13231279, 38510840, 84840549, 67084351, 30366150, 749283, 20885148, 44245960, 70367851, 63152504, 70541760, 8791066, 74441448, 8128637, 97641116, 43357947, 98943869, 89419466, 30163921, 40197395, 79880247, 32590267, 78300864, 17727650, 93057697 +79880247, 26392416, 66116458, 61728685, 22766820, 79136082, 91664334, 4434662, 59981773, 70727211, 36845587, 49597667, 18411915, 42692881, 10264691, 78300864, 61982238, 4064751, 97057187, 67793644, 45306070, 69579137, 47361209, 63628376, 36189527, 27325428, 81677380, 38341669, 53666583, 2331773, 7066775, 71333116, 27041967, 43376279, 55143724, 92692978, 23194618, 72238278, 39201414, 55669657, 99333375, 67513640, 12664567, 54058788, 2717150, 67124282, 99125126, 20642888, 22435353, 93053405, 68041839, 26493119, 13231279, 33304202, 99658235, 6038457, 98943869, 47792865, 83150534, 84406788, 22200378, 9829782, 65454636, 59614746, 6525948, 67227442, 81898046, 68110330, 61373987, 31733363, 8129978, 13819358, 31161687, 62051033, 6111563, 7300047, 4204661, 58180653, 30463802, 61741594, 37560164, 34295794, 54868730, 43322743, 12348118, 44245960, 51464002, 22113133, 38022834, 91938191, 65047700, 50188404, 50668599, 72373496, 40356877, 12024238, 31491938, 17727650, 68694897, 44627776, 39553046, 70800879, 97561745, 29516592, 13173644, 57961282, 88698958, 3233084, 25842078, 90013093, 76315420, 66611759, 20568363, 91831487, 55470718, 95290172, 57393458, 10358899, 64848072, 17976208, 9860195, 84166196, 20681921, 3633375, 70596786, 22879907, 15902805, 64087743, 76170907, 5482538, 52613508, 47213183, 86301513, 77301523, 65081429, 84002370, 36505482, 48892201, 77620120, 66885828, 37501808, 71316369, 29466406, 29834512, 18131876, 31126490, 33935899, 83083131, 30218878, 73021291, 20140249, 70420215, 77187825, 41092102, 84187166, 15536795, 21397057, 40152546, 247198, 73031054, 6521313, 72358170, 36812683, 57020481, 61859581, 44842615, 93057697, 36139942, 38645117, 2891150, 59371804, 49598724, 57241521, 3509435, 98462867, 96852321, 26863229, 71965942, 21001913, 43357947, 3487592, 91141395, 8055981, 73235980, 49328608, 79922758, 7229550, 30694952, 9259676, 95010552, 88444207, 34667286, 51590803, 72495719, 22997281, 68128438, 89996536, 98130363, 16595436, 32699744, 45794415, 62232211, 40984766, 99604946, 30395570, 89046466, 30787683, 59197747, 24826575, 62552524, 92071990, 92554120, 62803858, 17016156, 23134715, 86543538, 89804152, 98653983, 42199455, 40197395, 11543098, 85117092, 99729357, 62936963, 78909561, 37739481, 415901, 98371444, 92604458, 42073124, 47708850, 7646095, 18504197, 47090124, 7011964, 41442762, 58751351, 77787724, 74357852, 54232247, 18699206, 49271185, 4069912, 16684829, 85571389, 16567550, 60393039, 4668450, 20002147, 67474219, 72357096, 11757872, 36780454, 83269727, 62428472, 26744917, 39986008, 57240218, 17386996, 96709982, 99515901, 48774913, 82779622, 5832946, 30163921, 66832478, 22721500, 45049308, 526217, 43501211, 74614639, 61960400, 54517921, 82339363, 15148031, 24733232, 26664538, 69136837, 76825057, 13470059, 72725103, 99549373, 4806458, 17764950, 41092172, 4095116, 76540481, 35996293, 66137019, 18001617, 29510992, 92215320, 75820087, 72274002, 8099994, 90457870, 93359396, 68102437, 28550822, 75153252, 58749, 34698428, 66667729, 36312813, 30998561, 508198, 78549759, 33628349, 68816413, 75229462, 14326617, 61271144, 36933038, 67084351, 13862149, 47893286, 4978543, 90061527, 14626618, 30764367, 38008118, 64098930, 20836893, 61712234, 53393358, 30891921, 44844121, 33061250, 24619760, 79738755, 93790285, 37224844, 55753905, 19486173, 48360198, 64602895, 78589145, 67031644, 3235882, 20765474, 92867155, 55850790, 60581278, 33123618, 1022677, 89217461, 37183543, 42967683, 43933006, 72793444, 55189057, 82052050, 52293995, 8913721, 874791, 46851987, 72614359, 7955293, 1239555, 38365584, 92541302, 75986488, 39847321, 7685448, 70372191, 59177718, 63967300, 33249630, 81172706, 70367851, 93562257, 73109997, 83747892, 23740167, 17146629, 4508700, 59329511, 95957797, 52204879, 88047921, 49641577, 77898274, 24915585, 90654836, 1204161, 19272365, 68204242, 73786944, 55247455, 96193415, 77072625, 57658654, 86022504, 89699445, 33565483, 17037369, 68939068, 45381876, 45996863, 84127901, 76330843, 99224392, 47738185, 10453030, 37891451, 44983451, 92353856, 69697787, 2917920, 56515456, 62693428, 9886593, 29065964, 95251277, 18833224, 32274392, 10366309, 92803766, 4787945, 35512853, 51588015, 22405120, 26102057, 97940276, 90272749, 8634541, 33431960, 19891772, 17539625, 17894977, 35092039, 16445503, 49882705, 88251446, 61623915, 48395186, 88653118, 62357986, 66250369, 54199160, 3880712, 42782652, 96420429, 60430369, 74137926, 81805959, 51315460, 75052463, 8651647, 66903004, 9860968, 70782102, 36580610, 749283, 48675329, 70036438, 6497830, 44915235, 67451935, 15075176, 10309525, 53802686, 54663246, 14723410, 1197320, 17857111, 7919588, 81853704, 19898053, 78884452, 73222868, 86460488, 54263880, 44177011, 20867149, 13348726, 10649306, 39801351, 95395112, 73168565, 47887470, 16097038, 48088883, 3233569, 36685023, 51149804, 51507425, 55615885, 30653863, 26507214, 83948335, 32590267, 15535065, 45428665, 29188588, 6793819, 29635537, 57163802, 16099750, 112651, 92998591, 12303248, 29029316, 70541760, 25636669, 2607799, 7182642, 85711894, 16424199, 36135, 81774825, 90004325, 4091162, 40781449, 72732205, 69355476, 82532312, 86798033, 65074535, 14363867, 29401781, 45919976, 18806856, 83302115, 1431742, 86242799, 60955663, 77284619, 62496012, 99226875, 36930650, 97783876, 38256849, 24168411, 33201905, 50567636, 26292919, 77413012, 36396314, 40686254, 17081350, 44060493, 7502255, 46540998, 59318837, 94076128, 16380211, 65851721, 669105, 11365791, 82897371, 56531125, 77300457, 71083565, 68824981, 32161669, 5267545, 8115266, 23432750, 45407418, 27411561, 91727510, 75543508, 33797252, 30139692, 44473167, 44348328, 69641513, 19939935, 38510840, 51715482, 72019362, 84840549, 28928175, 75128745, 10961421, 6221471, 9623492, 68702632, 82427263, 4515343, 62115552, 42947632, 42237907, 92398073, 77272628, 97011160, 62430984, 15163258, 32159704, 6153406, 20486294, 82178706, 22942635, 21673260, 62490109, 80246713, 3773993, 87598888, 40027975, 8729146, 22108665, 61928316, 69255765, 7423788, 5640302, 29932657, 82886381, 61263068, 77312810, 569864, 26275734, 66271566, 76703609, 48260151, 16019925, 80014588, 84904436, 44847298, 19101477, 91957544, 9175338, 89214616, 69786756, 5073754, 37620363, 6505939, 41245325, 51426311, 33699435, 16054533, 82651278, 61859143, 57359924, 20122224, 30811010, 27665211, 56153345, 99021067, 51466049, 8733068, 1375023, 43152977, 56484900, 37435892, 20645197, 64055960, 40781854, 98948034, 3294781, 824872, 59501487, 5822202, 34497327, 9603598, 54606384, 37166608, 31727379, 53632373, 17385531, 1250437, 30771409, 6871053, 94595668, 97641116, 61897582, 86821229, 72278539, 4985896, 14220886, 5111370, 91281584, 36753250, 65038678, 99917599, 93566986, 59109400, 90158785, 83651211, 48673079, 85242963, 83533741, 14947650, 26139110, 45617087, 78218845, 84684495, 86001008, 17957593, 27185644, 26998766, 11923835, 65271999, 58224549, 3088684, 28796059, 79657802, 33895336, 66045587, 91802888, 26114953, 1936762, 81274566, 26397786, 91990218, 6157724, 15948937, 18466635, 17365188, 94539824, 34236719, 65715134, 24058273, 89499542, 85695762, 53648154, 38009615, 38061439, 12571310, 75135448, 65880522, 64157906, 30090481, 98648327, 77377183, 45990383, 44784505, 30569392, 43045786, 33553959, 84293052, 62740044, 4798568, 80555751, 85116755, 9188443, 48658605, 51047803, 99690194, 77694700, 71522968, 18663507, 36808486, 55960386, 86118021, 4099191, 56473732, 8791066, 99297164, 34044787, 33237508, 89811711, 68644627, 5970607, 59910624, 97379790, 41481685, 32058615, 27625735, 71920426, 79806380, 7687278, 76960303, 44479073, 9257405, 95726235, 47298834, 24953498, 14731700, 6819644, 66663942, 79322415, 87710366, 37192445, 6986898, 50702367, 83155442, 52261574, 82726008, 14349098, 46870723, 40534591, 32151165, 79821897, 45995325, 99971982, 168541, 53888755, 54987042, 91255408, 90090964, 44550764, 74743862, 8696647, 9398733, 89637706, 62452034, 44846932, 88904910, 61141874, 45075471, 79191827, 44481640, 84349107, 40677414, 60176618, 9058407, 94516935, 80316608, 80251430, 58208470, 83133790, 17068582, 78785507, 53084256, 45237957, 26776922, 2208785, 57248122, 3183975, 23569917, 4199704, 28734791, 20885148, 69848388, 24591705, 65338021, 73392814, 38658347, 1569515, 70004753, 15064655, 9575954, 78561158, 75407347, 42644903, 82979980, 93933709, 94360702, 60102965, 17058722, 30543215, 76671482, 63506281, 13468268, 66322959, 73617245, 72738685, 74724075, 2204165, 71469330, 63152504, 6808825, 67281495, 16971929, 78766358, 92033260, 63015256, 87720882, 29994197, 99524975, 74441448, 66319530, 67030811, 8128637, 89078848, 22129328, 29819940, 71657078, 10597197, 50806615, 94911072, 65017137, 19457589, 92530431, 31904591, 82327024, 83210802, 90310261, 16405341, 33336148, 45667668, 8373289, 94090109, 81855445, 21533347, 22450468, 63372756, 93270084, 28787861, 53842979, 55602660, 78602717, 38494874, 60106217, 89419466, 14093520, 1788101, 30366150, 54014062, 37280276, 26734892, 55770687, 21070110, 176257, 99861373, 76434144, 63059024, 68875490, 98739783, 65275241, 37675718, 42307449, 26063929, 35192533, 61815223, 73124510, 92692283, 96906410, 44889423, 99965001, 50007421, 14045383, 68316156, 28851716, 37659250, 4119747, 74110882, 72777973, 94711842, 62762857, 11581181, 56955985, 34946859, 96318094, 48893685, 321665, 43506672, 83368048, 91240048, 79942022, 97281847, 87160386, 40865610, 44664587, 95581843, 96735716, 21919959, 49236559, 59405277, 32250045, 34493392, 15015906, 53547802, 69605283, 21993752, 35456853, 8807940, 83789391, 88130087, 68000591, 75777973, 21289531, 29959549, 39171895, 34698463, 23110625, 32426752, 27187213, 39373729, 7517032, 24314885, 56424103, 74452589, 16387575, 23848565, 92787493, 54762643, 7104732, 85023028, 36468541, 51472275, 11161731, 12416768, 71300104, 18783702, 96726697, 52060076, 66428795, 45848907, 57803235, 34432810, 9599614, 45790169, 37957788, 54427233, 41380093, 56461322, 19376156, 93515664 +22879907, 39801351, 11365791, 35996293, 21673260, 33553959, 65081429, 56473732, 77898274, 56424103, 36396314, 97641116, 83368048, 15148031, 87160386, 72495719, 63506281, 13468268, 55143724, 86242799, 83155442, 7502255, 168541, 86821229, 92803766, 40677414, 18001617, 19891772, 58224549, 3880712, 3633375, 38658347, 81898046, 94360702, 7300047, 42199455, 52293995, 80555751, 9175338, 7646095, 79136082, 16054533, 59910624, 68204242, 64055960, 66663942, 11757872, 97783876, 61960400, 24733232, 23848565, 36139942, 90158785, 48673079, 97561745, 21001913, 95581843, 53842979, 75052463, 10358899, 61712234, 40984766, 64087743, 176257, 45990383, 3235882, 30569392, 7423788, 31161687, 55850790, 60581278, 47887470, 6111563, 84002370, 46851987, 48658605, 70372191, 63967300, 25636669, 71316369, 52060076, 37659250, 69355476, 31491938, 30218878, 72357096, 37192445, 57803235, 4064751, 44060493, 71657078, 46540998, 40534591, 50806615, 61897582, 44550764, 5111370, 74614639, 82339363, 16405341, 27411561, 94516935, 14947650, 91727510, 96852321, 8099994, 28796059, 47792865, 1788101, 92398073, 10309525, 73392814, 62232211, 30395570, 20867149, 55753905, 19486173, 76434144, 75407347, 82886381, 82979980, 43933006, 3233569, 89804152, 99729357, 84293052, 84904436, 26507214, 36845587, 62936963, 19101477, 23110625, 85116755, 51047803, 77694700, 27187213, 81172706, 18663507, 93562257, 63152504, 2607799, 96726697, 88047921, 97379790, 90004325, 33935899, 86798033, 66428795, 87720882, 10264691, 1375023, 20002147, 5822202, 62428472, 77413012, 50702367, 14349098, 82779622, 6871053, 94595668, 30163921, 2717150, 72358170, 36812683, 526217, 43501211, 32274392, 79191827, 91240048, 26102057, 83533741, 79942022, 97940276, 80316608, 68041839, 90272749, 94090109, 47361209, 17539625, 17957593, 76315420, 28928175, 75153252, 3487592, 45237957, 49328608, 60430369, 79922758, 74137926, 9860968, 14326617, 57393458, 22200378, 36189527, 37957788, 9860195, 6157724, 27325428, 15163258, 59614746, 24058273, 53648154, 89046466, 99861373, 5482538, 13348726, 22108665, 78561158, 98739783, 92554120, 20765474, 38341669, 21289531, 16097038, 53666583, 55189057, 11543098, 48260151, 66322959, 874791, 2331773, 61741594, 6793819, 98371444, 66885828, 5073754, 44245960, 2204165, 99965001, 50007421, 8791066, 22113133, 77787724, 74357852, 41481685, 61859143, 57359924, 24915585, 72732205, 71920426, 49271185, 63015256, 56153345, 23194618, 18806856, 47298834, 67030811, 73021291, 24168411, 36780454, 89699445, 33565483, 17037369, 67513640, 46870723, 99224392, 17727650, 32151165, 97057187, 73031054, 54987042, 22721500, 6521313, 45306070, 321665, 65017137, 44627776, 44481640, 26664538, 32161669, 83210802, 76825057, 35512853, 92787493, 20642888, 9058407, 59371804, 4434662, 45617087, 80251430, 13173644, 21533347, 98462867, 88698958, 3233084, 38510840, 17894977, 72274002, 90457870, 51715482, 72019362, 61623915, 53084256, 91141395, 73235980, 79657802, 26776922, 30998561, 91802888, 62115552, 23569917, 38494874, 8651647, 91831487, 89419466, 26397786, 67084351, 13862149, 67451935, 18466635, 51590803, 77272628, 68128438, 94539824, 89996536, 34236719, 20836893, 69605283, 21070110, 35456853, 22942635, 62490109, 86460488, 30787683, 61373987, 70004753, 93790285, 37224844, 15064655, 78589145, 77377183, 75777973, 73168565, 17016156, 60102965, 4204661, 82052050, 76671482, 42307449, 26063929, 55615885, 62740044, 36505482, 61815223, 1239555, 37560164, 92541302, 34295794, 9188443, 18411915, 89214616, 42073124, 22766820, 12348118, 74724075, 71469330, 47090124, 33237508, 16971929, 18131876, 43376279, 91664334, 7182642, 36135, 68316156, 7517032, 30811010, 74110882, 82532312, 24314885, 65074535, 29994197, 73786944, 12024238, 55247455, 6819644, 67474219, 70420215, 62762857, 79322415, 37166608, 11581181, 56955985, 45996863, 57240218, 17081350, 96709982, 82726008, 29819940, 10453030, 40152546, 247198, 94076128, 669105, 91255408, 82897371, 48893685, 68694897, 62693428, 89637706, 62452034, 91281584, 45049308, 71083565, 92530431, 82327024, 10366309, 5267545, 19376156, 69136837, 90310261, 23432750, 72725103, 51588015, 4095116, 76540481, 75543508, 22435353, 26493119, 57241521, 78218845, 57961282, 75820087, 69641513, 84684495, 27185644, 90013093, 22450468, 84840549, 26998766, 78785507, 68102437, 99658235, 28550822, 58749, 34698428, 48395186, 62357986, 42782652, 508198, 4515343, 26114953, 30694952, 1936762, 81274566, 60106217, 83150534, 21919959, 84406788, 37280276, 6497830, 28734791, 59405277, 65454636, 11161731, 15948937, 97011160, 81677380, 34493392, 54663246, 98130363, 26734892, 65338021, 6153406, 45794415, 53393358, 70596786, 20486294, 30891921, 44844121, 80246713, 99604946, 3773993, 1569515, 70727211, 76170907, 12571310, 98648327, 62552524, 86301513, 68875490, 13819358, 42644903, 65275241, 62051033, 92867155, 1022677, 37675718, 26275734, 98653983, 76703609, 80014588, 79880247, 37739481, 30463802, 48892201, 91957544, 32590267, 77620120, 39847321, 57163802, 92604458, 33249630, 43322743, 42692881, 29029316, 71522968, 73109997, 55960386, 18783702, 99297164, 41442762, 58751351, 39373729, 68644627, 95957797, 29834512, 31126490, 85711894, 32058615, 82651278, 4091162, 27625735, 4119747, 54232247, 27665211, 65047700, 4069912, 44479073, 50188404, 40356877, 83302115, 8733068, 94711842, 60393039, 99524975, 24953498, 40781854, 98948034, 59501487, 9603598, 55669657, 86022504, 87710366, 31727379, 8128637, 6986898, 99515901, 47738185, 54058788, 16380211, 65851721, 53888755, 44983451, 14220886, 2917920, 56515456, 99125126, 9886593, 44846932, 43506672, 93566986, 38645117, 8115266, 13470059, 4787945, 99549373, 41092172, 85242963, 66137019, 49598724, 33336148, 30139692, 44473167, 8373289, 33431960, 69579137, 13231279, 83133790, 11923835, 44664587, 3088684, 66667729, 36312813, 28787861, 66045587, 82427263, 2208785, 6038457, 78602717, 81805959, 33628349, 98943869, 68816413, 14093520, 75229462, 36933038, 42237907, 36580610, 49236559, 64848072, 47893286, 48675329, 70036438, 93515664, 4978543, 15075176, 32250045, 22997281, 32159704, 14723410, 17857111, 16595436, 65715134, 44177011, 83789391, 64602895, 88130087, 9575954, 44784505, 52613508, 92071990, 8129978, 5640302, 93933709, 42967683, 86543538, 39171895, 72793444, 16019925, 30653863, 44847298, 35192533, 41380093, 15535065, 45428665, 29188588, 16099750, 71300104, 99690194, 112651, 92998591, 6808825, 70541760, 7066775, 33699435, 17146629, 38022834, 71333116, 29466406, 27041967, 52204879, 49641577, 81774825, 20122224, 19272365, 14363867, 50668599, 29401781, 72777973, 85571389, 39201414, 45919976, 16567550, 51466049, 43152977, 56484900, 78300864, 14731700, 66319530, 77284619, 96193415, 34497327, 77072625, 26292919, 89078848, 53632373, 76330843, 22129328, 48774913, 10597197, 74743862, 69697787, 9398733, 67124282, 56531125, 36753250, 16387575, 95251277, 65038678, 18833224, 61859581, 44842615, 9599614, 93057697, 59109400, 60176618, 4806458, 2891150, 93053405, 81855445, 26863229, 86001008, 35092039, 17068582, 93359396, 75128745, 10961421, 6221471, 54762643, 7104732, 9623492, 93270084, 96735716, 57248122, 78549759, 51315460, 20568363, 59981773, 42947632, 55470718, 70782102, 88444207, 30366150, 63628376, 9829782, 749283, 44915235, 90061527, 14626618, 6525948, 7919588, 24591705, 81853704, 67227442, 20681921, 32699744, 78884452, 85695762, 12416768, 24619760, 8729146, 75135448, 64157906, 30090481, 63059024, 10649306, 62803858, 29959549, 569864, 23134715, 17058722, 40197395, 30543215, 66271566, 51149804, 8913721, 51507425, 78909561, 83948335, 49597667, 73124510, 72738685, 75986488, 54868730, 69786756, 59177718, 12303248, 47708850, 36808486, 6505939, 41245325, 23740167, 51426311, 4099191, 4508700, 40781449, 90654836, 92692978, 18699206, 91938191, 1204161, 76960303, 72238278, 37435892, 1431742, 83083131, 4668450, 62496012, 61982238, 36930650, 41092102, 54606384, 33201905, 99333375, 45848907, 15536795, 52261574, 45995325, 4985896, 90090964, 8696647, 88904910, 84349107, 22405120, 45667668, 92215320, 25842078, 71965942, 16445503, 40865610, 63372756, 68702632, 95290172, 95010552, 36468541, 4199704, 20885148, 53802686, 38008118, 69848388, 55770687, 8807940, 24826575, 67031644, 61928316, 69255765, 66116458, 95395112, 33123618, 37183543, 58180653, 36685023, 85117092, 72614359, 7955293, 38365584, 37620363, 37501808, 51464002, 67281495, 56461322, 5970607, 16424199, 7687278, 16684829, 9257405, 74441448, 60955663, 20140249, 77187825, 68939068, 50567636, 39986008, 45381876, 40686254, 30771409, 79821897, 59318837, 67793644, 66832478, 72278539, 29065964, 19457589, 57020481, 39553046, 68824981, 45407418, 17764950, 33797252, 8634541, 29510992, 3509435, 44348328, 19939935, 33304202, 43357947, 49882705, 65271999, 88653118, 66250369, 96420429, 7229550, 9259676, 66903004, 85023028, 61271144, 26392416, 51472275, 17976208, 30764367, 84166196, 89499542, 82178706, 38009615, 38061439, 33061250, 54263880, 31733363, 79738755, 68000591, 43045786, 29932657, 77301523, 415901, 29635537, 7685448, 44889423, 83747892, 18504197, 86118021, 7011964, 34044787, 89811711, 14045383, 28851716, 99021067, 95726235, 74452589, 38256849, 83269727, 84187166, 17386996, 21397057, 37891451, 34432810, 99971982, 92353856, 54517921, 45075471, 83651211, 70800879, 26139110, 97281847, 29516592, 88251446, 66611759, 8055981, 91990218, 54014062, 17365188, 62430984, 64098930, 1197320, 15015906, 19898053, 68110330, 40027975, 59197747, 65880522, 47213183, 61728685, 89217461, 34698463, 73617245, 92692283, 96906410, 59329511, 79806380, 92033260, 72373496, 20645197, 3294781, 99226875, 26744917, 12664567, 17385531, 1250437, 5832946, 34946859, 96318094, 94911072, 61141874, 99917599, 31904591, 45790169, 58208470, 54199160, 3183975, 53547802, 21993752, 15902805, 87598888, 4798568, 32426752, 84127901, 55602660, 48360198, 77312810, 78766358, 77300457, 33895336, 34667286, 73222868, 48088883, 70367851, 824872, 57658654, 61263068, 54427233 +16684829, 86242799, 19891772, 10309525, 77272628, 68128438, 59614746, 20486294, 47887470, 48658605, 1250437, 16380211, 5111370, 30139692, 22450468, 79657802, 38658347, 38009615, 43045786, 13819358, 1022677, 52293995, 30653863, 15535065, 27187213, 2204165, 43152977, 66319530, 62762857, 76825057, 13470059, 23432750, 22405120, 49598724, 21001913, 17894977, 9259676, 75229462, 72495719, 64098930, 19898053, 64087743, 62740044, 48892201, 33249630, 7646095, 63152504, 79136082, 51426311, 55143724, 68316156, 40781449, 37659250, 63015256, 56424103, 11757872, 62428472, 50806615, 66832478, 77300457, 32274392, 82339363, 10366309, 97940276, 97281847, 92215320, 51715482, 40865610, 9623492, 26776922, 508198, 62115552, 26114953, 88444207, 15075176, 15163258, 69848388, 65715134, 24058273, 93790285, 88130087, 67031644, 78561158, 68875490, 65275241, 82886381, 33123618, 29959549, 37183543, 7300047, 60102965, 53666583, 98653983, 55615885, 46851987, 36505482, 1239555, 34295794, 89214616, 5073754, 44245960, 47708850, 59329511, 29466406, 88047921, 16054533, 4091162, 72777973, 73786944, 31491938, 1375023, 67030811, 70420215, 57658654, 36930650, 82726008, 71657078, 67793644, 53888755, 72278539, 8696647, 89637706, 18833224, 74614639, 61960400, 54517921, 83368048, 31904591, 93566986, 68824981, 44481640, 40677414, 48673079, 93053405, 4434662, 26493119, 78218845, 33304202, 90013093, 90457870, 99658235, 61623915, 34698428, 63372756, 66611759, 8055981, 44664587, 49328608, 91802888, 81805959, 75052463, 38494874, 89419466, 55470718, 42237907, 54014062, 37280276, 749283, 20885148, 97011160, 22997281, 32159704, 14723410, 26734892, 61712234, 81853704, 3633375, 78884452, 22879907, 30395570, 54263880, 30787683, 70004753, 20867149, 15064655, 45990383, 52613508, 30569392, 73168565, 61728685, 17016156, 17058722, 99729357, 44847298, 61815223, 83948335, 41380093, 23110625, 92541302, 72738685, 75986488, 29188588, 63967300, 71469330, 7066775, 4099191, 89811711, 77787724, 29834512, 16971929, 18131876, 43376279, 31126490, 7517032, 61859143, 27625735, 33935899, 19272365, 66428795, 29401781, 37435892, 54606384, 39986008, 45381876, 8128637, 36396314, 99224392, 21397057, 10453030, 30163921, 61897582, 48893685, 45306070, 9886593, 88904910, 44627776, 57020481, 16387575, 45075471, 39553046, 15148031, 44842615, 99549373, 22435353, 8634541, 81855445, 27185644, 28550822, 91141395, 3088684, 66667729, 53842979, 30998561, 2208785, 60430369, 6038457, 74137926, 60106217, 95010552, 26397786, 57393458, 30366150, 9829782, 6497830, 34667286, 90061527, 62430984, 7919588, 20836893, 16595436, 73392814, 55770687, 80246713, 8807940, 38061439, 176257, 83789391, 79738755, 59197747, 24826575, 78589145, 64157906, 30090481, 98648327, 39801351, 42644903, 92867155, 42967683, 89804152, 72793444, 66271566, 54427233, 84002370, 77620120, 45428665, 57163802, 85116755, 92998591, 77694700, 74724075, 29029316, 44889423, 71522968, 18663507, 83747892, 70541760, 67281495, 56473732, 25636669, 39373729, 22113133, 71333116, 52204879, 90654836, 49271185, 76960303, 14363867, 45919976, 16567550, 95726235, 60393039, 47298834, 30218878, 5822202, 86022504, 36780454, 45996863, 77413012, 96709982, 29819940, 7502255, 54058788, 45995325, 10597197, 97057187, 73031054, 168541, 97641116, 86821229, 6521313, 11365791, 92353856, 69697787, 68694897, 94911072, 72358170, 44846932, 36812683, 19457589, 95251277, 43501211, 99917599, 61859581, 93057697, 36139942, 60176618, 8115266, 51588015, 4806458, 20642888, 83533741, 79942022, 94516935, 68041839, 18001617, 29516592, 21533347, 58208470, 38510840, 87160386, 75153252, 11923835, 6221471, 88653118, 28787861, 7229550, 9860968, 81274566, 59981773, 42947632, 14326617, 83150534, 1788101, 22200378, 37957788, 9860195, 6157724, 65454636, 81677380, 98130363, 32699744, 45794415, 53393358, 44844121, 21673260, 40984766, 12416768, 76170907, 55753905, 5482538, 76434144, 3235882, 92071990, 92554120, 569864, 33553959, 86543538, 6111563, 3233569, 51149804, 34698463, 42307449, 76703609, 48260151, 16019925, 13468268, 72614359, 37739481, 61741594, 32590267, 70372191, 112651, 22766820, 43322743, 66885828, 81172706, 37620363, 6808825, 18504197, 17146629, 99297164, 41442762, 96726697, 78766358, 7182642, 85711894, 59910624, 97379790, 41481685, 36135, 32058615, 77898274, 24915585, 54232247, 92692978, 74110882, 91938191, 86798033, 87720882, 50668599, 23194618, 72373496, 40356877, 10264691, 18806856, 20645197, 4668450, 20002147, 77284619, 98948034, 72357096, 66663942, 61982238, 96193415, 34497327, 37166608, 83269727, 89699445, 31727379, 33565483, 50567636, 53632373, 57240218, 83155442, 17385531, 52261574, 4064751, 47738185, 44060493, 34946859, 40534591, 32151165, 6871053, 65851721, 321665, 65017137, 99125126, 24733232, 32161669, 5267545, 91240048, 35512853, 16405341, 9058407, 14947650, 26139110, 33797252, 90272749, 13231279, 44348328, 84684495, 98462867, 25842078, 35092039, 76315420, 16445503, 84840549, 49882705, 3487592, 28796059, 54199160, 66045587, 79922758, 78602717, 66903004, 21919959, 70782102, 13862149, 10358899, 49236559, 64848072, 47893286, 44915235, 93515664, 11161731, 92398073, 14626618, 34236719, 6525948, 1197320, 17857111, 24591705, 20681921, 85695762, 81898046, 30891921, 86460488, 44177011, 8729146, 12571310, 75135448, 48360198, 77377183, 44784505, 69255765, 8129978, 5640302, 20765474, 38341669, 75777973, 55850790, 29932657, 60581278, 23134715, 39171895, 4204661, 26275734, 55189057, 36685023, 85117092, 80555751, 62936963, 35192533, 19101477, 49597667, 39847321, 29635537, 9175338, 7685448, 92604458, 54868730, 69786756, 32426752, 42692881, 37501808, 99965001, 68644627, 57359924, 20122224, 82532312, 24314885, 7687278, 1204161, 72238278, 85571389, 29994197, 55247455, 14731700, 67474219, 59501487, 79322415, 97783876, 77187825, 24168411, 17037369, 37192445, 15536795, 6986898, 50702367, 22129328, 46870723, 17727650, 46540998, 59318837, 94595668, 669105, 14220886, 2717150, 90090964, 9398733, 92530431, 23848565, 19376156, 69136837, 92787493, 85242963, 2891150, 75543508, 80316608, 33336148, 8373289, 57241521, 69579137, 69641513, 26863229, 19939935, 83133790, 71965942, 8099994, 88251446, 28928175, 53084256, 48395186, 62357986, 7104732, 58224549, 93270084, 68702632, 3880712, 33895336, 42782652, 57248122, 78549759, 33628349, 8651647, 85023028, 68816413, 14093520, 36933038, 84406788, 63628376, 48675329, 4199704, 67451935, 53802686, 32250045, 94539824, 89996536, 84166196, 38008118, 67227442, 65338021, 6153406, 53648154, 82178706, 99604946, 61373987, 99861373, 40027975, 37224844, 64602895, 66116458, 10649306, 31161687, 62051033, 61263068, 43933006, 48088883, 42199455, 51507425, 65081429, 2331773, 26507214, 78909561, 73124510, 37560164, 6793819, 16099750, 51047803, 59177718, 42073124, 36808486, 6505939, 23740167, 47090124, 7011964, 33699435, 58751351, 34044787, 33237508, 71316369, 2607799, 16424199, 81774825, 82651278, 4119747, 27665211, 79806380, 99021067, 9257405, 51466049, 94711842, 56484900, 24953498, 60955663, 20140249, 99226875, 38256849, 99333375, 84187166, 67513640, 56955985, 17386996, 48774913, 82779622, 99971982, 91255408, 4985896, 44550764, 2917920, 67124282, 61141874, 91281584, 36753250, 65038678, 82327024, 9599614, 92803766, 83210802, 4787945, 4095116, 66137019, 91727510, 45790169, 97561745, 33431960, 94090109, 13173644, 47361209, 57961282, 86001008, 17957593, 72274002, 17068582, 43357947, 10961421, 73235980, 96420429, 55602660, 23569917, 20568363, 1936762, 91831487, 36468541, 67084351, 36580610, 51472275, 17976208, 59405277, 27325428, 15015906, 21070110, 73222868, 22942635, 62490109, 68110330, 24619760, 1569515, 70727211, 31733363, 65880522, 22108665, 62552524, 47213183, 86301513, 63059024, 62803858, 89217461, 76671482, 874791, 80014588, 79880247, 4798568, 30463802, 7955293, 38365584, 92692283, 18411915, 71300104, 96906410, 99690194, 12303248, 70367851, 93562257, 73109997, 51464002, 86118021, 50007421, 8791066, 4508700, 38022834, 27041967, 91664334, 14045383, 49641577, 28851716, 69355476, 18699206, 92033260, 65074535, 4069912, 44479073, 56153345, 39201414, 99524975, 1431742, 78300864, 83083131, 40781854, 3294781, 62496012, 41092102, 68939068, 45848907, 26292919, 84127901, 57803235, 40686254, 99515901, 76330843, 14349098, 5832946, 30771409, 79821897, 40152546, 247198, 54987042, 62452034, 45049308, 71083565, 84349107, 90158785, 83651211, 72725103, 41092172, 70800879, 26102057, 45617087, 29510992, 88698958, 96852321, 3233084, 78785507, 65271999, 54762643, 45237957, 95581843, 96735716, 82427263, 4515343, 30694952, 98943869, 47792865, 91990218, 70036438, 4978543, 34493392, 30764367, 70596786, 21993752, 35456853, 15902805, 13348726, 68000591, 9575954, 7423788, 21289531, 77301523, 77312810, 82979980, 93933709, 16097038, 94360702, 82052050, 8913721, 63506281, 36845587, 98371444, 12348118, 41245325, 55960386, 18783702, 56461322, 74357852, 30811010, 72732205, 65047700, 8733068, 74441448, 64055960, 9603598, 87710366, 34432810, 44983451, 22721500, 82897371, 56515456, 56531125, 43506672, 79191827, 90310261, 35996293, 45667668, 44473167, 80251430, 17539625, 75820087, 93359396, 72019362, 26998766, 75128745, 66250369, 36312813, 51315460, 28734791, 51590803, 17365188, 89499542, 69605283, 19486173, 61928316, 75407347, 95395112, 58180653, 40197395, 66322959, 73617245, 415901, 9188443, 95957797, 90004325, 50188404, 68204242, 83302115, 12024238, 6819644, 73021291, 74452589, 55669657, 33201905, 26744917, 12664567, 89078848, 17081350, 37891451, 94076128, 29065964, 526217, 38645117, 45407418, 27411561, 68102437, 3183975, 95290172, 61271144, 36189527, 15948937, 18466635, 54663246, 53547802, 62232211, 33061250, 3773993, 89046466, 87598888, 30543215, 11543098, 84293052, 91957544, 52060076, 824872, 77072625, 11581181, 96318094, 62693428, 26664538, 59109400, 17764950, 58749, 26392416, 98739783, 37675718, 26063929, 84904436, 71920426, 74743862, 59371804, 76540481, 3509435, 5970607 +55470718, 99729357, 80555751, 43152977, 58749, 39801351, 26063929, 415901, 69786756, 50007421, 76330843, 76540481, 69641513, 86001008, 20836893, 53648154, 38009615, 30569392, 72614359, 78909561, 32426752, 6505939, 90004325, 34497327, 36780454, 12664567, 97641116, 61897582, 93566986, 44481640, 36139942, 44473167, 57961282, 17957593, 26998766, 99658235, 26776922, 23569917, 8651647, 36933038, 36580610, 37280276, 35456853, 70004753, 63059024, 93933709, 94360702, 39171895, 45428665, 39847321, 9188443, 74724075, 58751351, 77787724, 7182642, 92033260, 18806856, 1375023, 56484900, 64055960, 11757872, 99515901, 79821897, 96318094, 48893685, 16387575, 45075471, 83368048, 5267545, 79942022, 14947650, 66137019, 94090109, 47361209, 63372756, 88653118, 58224549, 49328608, 9860968, 68816413, 69848388, 53547802, 24058273, 21673260, 75135448, 61728685, 29932657, 33123618, 53666583, 36685023, 63506281, 36845587, 51047803, 96906410, 81172706, 55960386, 34044787, 38022834, 52204879, 16054533, 59910624, 41481685, 50668599, 68204242, 9257405, 16567550, 51466049, 31491938, 99524975, 72357096, 20140249, 70420215, 62762857, 89078848, 14349098, 86821229, 8696647, 2917920, 54517921, 61859581, 92803766, 45407418, 41092172, 85242963, 97281847, 80251430, 84684495, 72019362, 78785507, 66611759, 54762643, 8055981, 9623492, 44664587, 3088684, 96735716, 82427263, 91802888, 81805959, 51315460, 1936762, 95010552, 57393458, 37957788, 6157724, 77272628, 94539824, 26734892, 55770687, 69605283, 82178706, 22879907, 62490109, 15902805, 68110330, 99861373, 15064655, 30090481, 68000591, 77377183, 17016156, 569864, 33553959, 82979980, 26275734, 72793444, 73617245, 2331773, 4798568, 15535065, 75986488, 48658605, 112651, 33249630, 5073754, 6808825, 51426311, 25636669, 8791066, 29466406, 97379790, 36135, 72732205, 24314885, 19272365, 66428795, 65074535, 76960303, 10264691, 83302115, 47298834, 1431742, 67030811, 56424103, 96193415, 9603598, 97783876, 24168411, 39986008, 77413012, 6986898, 7502255, 71657078, 10453030, 94595668, 91255408, 22721500, 82897371, 62693428, 56531125, 321665, 88904910, 29065964, 95251277, 61960400, 92530431, 44842615, 9599614, 23432750, 83651211, 99549373, 27411561, 45790169, 45667668, 18001617, 33431960, 17539625, 92215320, 13231279, 88698958, 38510840, 90013093, 17068582, 61623915, 75153252, 3487592, 91141395, 65271999, 28787861, 3880712, 4515343, 79922758, 6038457, 38494874, 42947632, 21919959, 70782102, 30366150, 47893286, 70036438, 81677380, 89996536, 34236719, 45794415, 53393358, 70596786, 73392814, 3773993, 1569515, 37224844, 76434144, 67031644, 9575954, 95395112, 62051033, 62803858, 29959549, 77312810, 89217461, 23134715, 42967683, 16097038, 7300047, 8913721, 48892201, 83948335, 91957544, 49597667, 92541302, 98371444, 7685448, 71300104, 89214616, 99690194, 70372191, 22766820, 27187213, 93562257, 36808486, 70541760, 4508700, 41442762, 22113133, 95957797, 14045383, 16424199, 90654836, 33935899, 49271185, 86798033, 16684829, 99021067, 87720882, 60393039, 14731700, 20002147, 66319530, 30218878, 3294781, 77072625, 77187825, 33565483, 67513640, 8128637, 84127901, 46870723, 99224392, 4064751, 21397057, 46540998, 34946859, 40534591, 16380211, 34432810, 14220886, 74743862, 45306070, 44627776, 91281584, 57020481, 43506672, 71083565, 31904591, 32161669, 10366309, 23848565, 91240048, 40677414, 90158785, 92787493, 17764950, 20642888, 22405120, 16405341, 4095116, 83533741, 35996293, 94516935, 97940276, 75543508, 59371804, 4434662, 30139692, 13173644, 78218845, 19891772, 58208470, 44348328, 96852321, 19939935, 27185644, 35092039, 72274002, 93359396, 87160386, 84840549, 48395186, 36312813, 95581843, 96420429, 55602660, 74137926, 75052463, 91831487, 60106217, 95290172, 36468541, 26392416, 67084351, 49236559, 18466635, 84166196, 14723410, 98130363, 24591705, 81853704, 65715134, 21993752, 21070110, 24619760, 44177011, 83789391, 76170907, 12571310, 59197747, 24826575, 88130087, 69255765, 86301513, 43045786, 37183543, 86543538, 89804152, 42199455, 82052050, 40197395, 66271566, 11543098, 52293995, 42307449, 874791, 30463802, 19101477, 1239555, 41380093, 23110625, 6793819, 85116755, 18411915, 12303248, 43322743, 42692881, 70367851, 71522968, 37501808, 83747892, 67281495, 56461322, 71333116, 16971929, 31126490, 7517032, 61859143, 57359924, 4119747, 91938191, 7687278, 72238278, 73786944, 55247455, 74441448, 77284619, 5822202, 66663942, 87710366, 11581181, 50567636, 45996863, 57240218, 83155442, 17385531, 52261574, 30771409, 40152546, 94076128, 10597197, 97057187, 50806615, 53888755, 4985896, 11365791, 5111370, 89637706, 62452034, 9886593, 61141874, 19457589, 65038678, 43501211, 82339363, 68824981, 93057697, 19376156, 69136837, 90310261, 13470059, 35512853, 51588015, 26102057, 33336148, 97561745, 90272749, 29516592, 8634541, 83133790, 68102437, 45237957, 66667729, 79657802, 53842979, 42782652, 26114953, 78549759, 33628349, 9259676, 98943869, 81274566, 1788101, 26397786, 42237907, 10358899, 63628376, 22200378, 64848072, 51472275, 67451935, 59405277, 65454636, 32250045, 51590803, 22997281, 62430984, 15163258, 38008118, 1197320, 17857111, 67227442, 38658347, 62232211, 80246713, 8807940, 12416768, 86460488, 64087743, 33061250, 93790285, 55753905, 19486173, 65880522, 5482538, 78589145, 62552524, 45990383, 52613508, 8129978, 7423788, 98739783, 20765474, 42644903, 38341669, 21289531, 47887470, 61263068, 37675718, 3233569, 85117092, 34698463, 16019925, 66322959, 84293052, 36505482, 35192533, 73124510, 77620120, 72738685, 29635537, 9175338, 92604458, 59177718, 77694700, 44889423, 71469330, 73109997, 63152504, 79136082, 18504197, 41245325, 86118021, 4099191, 56473732, 47090124, 7011964, 33699435, 99297164, 33237508, 5970607, 96726697, 52060076, 4091162, 54232247, 72373496, 85571389, 39201414, 8733068, 94711842, 86242799, 4668450, 62496012, 61982238, 36930650, 79322415, 55669657, 38256849, 54606384, 83269727, 84187166, 68939068, 26292919, 40686254, 1250437, 82726008, 47738185, 59318837, 99971982, 44550764, 69697787, 68694897, 94911072, 99125126, 526217, 77300457, 74614639, 15148031, 59109400, 76825057, 72725103, 48673079, 26139110, 33797252, 68041839, 45617087, 57241521, 69579137, 21533347, 3509435, 26863229, 71965942, 90457870, 28928175, 40865610, 28550822, 11923835, 62115552, 20568363, 89419466, 59981773, 14326617, 91990218, 54014062, 13862149, 48675329, 36189527, 11161731, 72495719, 97011160, 7919588, 65338021, 19898053, 89499542, 44844121, 40984766, 54263880, 89046466, 61373987, 176257, 40027975, 8729146, 98648327, 3235882, 78561158, 75407347, 5640302, 13819358, 92554120, 65275241, 60581278, 82886381, 77301523, 43933006, 60102965, 48088883, 55189057, 76671482, 48260151, 84904436, 84002370, 46851987, 26507214, 62936963, 61815223, 7955293, 32590267, 57163802, 63967300, 29029316, 2204165, 51464002, 23740167, 7066775, 18783702, 39373729, 89811711, 68644627, 59329511, 29834512, 27041967, 43376279, 78766358, 88047921, 55143724, 81774825, 68316156, 77898274, 82651278, 20122224, 40781449, 37659250, 82532312, 27665211, 79806380, 18699206, 1204161, 29401781, 72777973, 29994197, 95726235, 24953498, 40781854, 99226875, 57658654, 37166608, 31727379, 17037369, 26744917, 57803235, 17081350, 96709982, 22129328, 17727650, 48774913, 44060493, 32151165, 6871053, 37891451, 669105, 67124282, 65017137, 82327024, 8115266, 4787945, 4806458, 80316608, 93053405, 26493119, 29510992, 81855445, 75820087, 98462867, 33304202, 17894977, 22450468, 53084256, 7104732, 68702632, 28796059, 508198, 2208785, 57248122, 30694952, 85023028, 14093520, 61271144, 44915235, 93515664, 28734791, 17976208, 92398073, 20885148, 15948937, 27325428, 17365188, 34493392, 32159704, 30764367, 64098930, 61712234, 32699744, 6153406, 78884452, 85695762, 22942635, 30891921, 30787683, 79738755, 64157906, 13348726, 44784505, 47213183, 68875490, 92867155, 75777973, 6111563, 58180653, 51149804, 76703609, 13468268, 54427233, 30653863, 62740044, 44847298, 61741594, 37560164, 16099750, 42073124, 92998591, 47708850, 18663507, 37620363, 85711894, 49641577, 32058615, 28851716, 69355476, 4069912, 44479073, 50188404, 23194618, 45919976, 60955663, 98948034, 33201905, 62428472, 99333375, 45381876, 15536795, 36396314, 50702367, 29819940, 45995325, 65851721, 73031054, 168541, 44983451, 72278539, 90090964, 92353856, 44846932, 36753250, 18833224, 79191827, 99917599, 91727510, 21001913, 76315420, 51715482, 49882705, 75128745, 10961421, 34698428, 66250369, 54199160, 3183975, 47792865, 75229462, 83150534, 749283, 6497830, 4978543, 15075176, 9860195, 10309525, 53802686, 90061527, 68128438, 15015906, 20681921, 16595436, 30395570, 70727211, 48360198, 64602895, 22108665, 92071990, 66116458, 10649306, 73168565, 55850790, 4204661, 51507425, 80014588, 92692283, 29188588, 12348118, 7646095, 99965001, 17146629, 18131876, 2607799, 91664334, 74357852, 24915585, 30811010, 27625735, 74110882, 14363867, 12024238, 78300864, 59501487, 74452589, 86022504, 89699445, 37192445, 82779622, 54058788, 247198, 66832478, 56515456, 24733232, 26664538, 84349107, 83210802, 9058407, 70800879, 8373289, 3233084, 25842078, 43357947, 8099994, 88251446, 62357986, 93270084, 66045587, 7229550, 66903004, 9829782, 14626618, 59614746, 54663246, 3633375, 73222868, 81898046, 38061439, 31733363, 31161687, 17058722, 79880247, 55615885, 65081429, 71316369, 92692978, 71920426, 56153345, 37435892, 83083131, 6819644, 56955985, 45848907, 53632373, 30163921, 54987042, 2717150, 6521313, 72358170, 36812683, 45049308, 39553046, 38645117, 60176618, 2891150, 49598724, 16445503, 73235980, 33895336, 30998561, 60430369, 78602717, 34667286, 6525948, 20486294, 99604946, 20867149, 87598888, 61928316, 98653983, 38365584, 34295794, 65047700, 63015256, 40356877, 20645197, 17386996, 5832946, 67793644, 32274392, 6221471, 88444207, 84406788, 1022677, 30543215, 54868730, 67474219, 73021291, 9398733, 4199704, 37739481, 44245960, 824872, 41092102, 22435353, 66885828 +61623915, 29516592, 85695762, 82886381, 57393458, 30787683, 65047700, 168541, 44842615, 26114953, 98943869, 98130363, 13348726, 43045786, 39201414, 57240218, 59318837, 67793644, 669105, 22721500, 18833224, 35996293, 17894977, 91802888, 85023028, 14093520, 51472275, 10309525, 5640302, 63506281, 65081429, 2331773, 73124510, 9188443, 18411915, 42073124, 66885828, 83747892, 22113133, 5970607, 16684829, 40781854, 79322415, 45996863, 29819940, 54058788, 72358170, 4806458, 17764950, 48673079, 3509435, 86001008, 71965942, 90457870, 10961421, 34698428, 4515343, 96420429, 51315460, 36580610, 21993752, 20486294, 48360198, 52613508, 78561158, 65275241, 77312810, 51149804, 26507214, 71522968, 25636669, 91664334, 49641577, 86798033, 50188404, 83302115, 1250437, 52261574, 96318094, 44983451, 32274392, 82339363, 8115266, 99549373, 80316608, 94090109, 38510840, 22450468, 7104732, 33895336, 62115552, 3183975, 9259676, 30366150, 9829782, 70036438, 9860195, 14626618, 64098930, 35456853, 44844121, 20867149, 64157906, 67031644, 44784505, 10649306, 61263068, 89217461, 16097038, 60102965, 53666583, 17058722, 62936963, 48892201, 91957544, 29635537, 57163802, 92998591, 22766820, 42692881, 93562257, 33699435, 58751351, 27625735, 4069912, 72238278, 51466049, 86242799, 59501487, 77072625, 86022504, 39986008, 53632373, 83155442, 96709982, 17385531, 94076128, 2717150, 44550764, 45306070, 92530431, 31904591, 4787945, 45407418, 68041839, 97281847, 8373289, 8634541, 13173644, 81855445, 43357947, 99658235, 28550822, 58749, 66250369, 66667729, 68702632, 33628349, 66903004, 81274566, 68816413, 22200378, 44915235, 28734791, 67451935, 15075176, 11161731, 30764367, 1197320, 19898053, 55770687, 15902805, 38009615, 12416768, 44177011, 76170907, 55753905, 24826575, 5482538, 98739783, 31161687, 29959549, 77301523, 37183543, 7300047, 8913721, 76703609, 99729357, 73617245, 72614359, 7955293, 415901, 72738685, 70372191, 112651, 18663507, 23740167, 56473732, 7011964, 96726697, 52060076, 4091162, 28851716, 71920426, 82532312, 76960303, 14363867, 95726235, 37435892, 55247455, 74441448, 20002147, 67474219, 30218878, 3294781, 61982238, 96193415, 57658654, 77187825, 89699445, 31727379, 17037369, 15536795, 84127901, 76330843, 46540998, 40534591, 94595668, 50806615, 72278539, 56515456, 29065964, 91281584, 65038678, 43501211, 39553046, 83368048, 79191827, 26664538, 82327024, 19376156, 84349107, 69136837, 20642888, 9058407, 27411561, 93053405, 4434662, 49598724, 29510992, 26863229, 93359396, 84840549, 78785507, 88251446, 11923835, 63372756, 6221471, 48395186, 62357986, 58224549, 73235980, 9623492, 79657802, 96735716, 508198, 82427263, 60430369, 74137926, 23569917, 59981773, 55470718, 83150534, 88444207, 13862149, 63628376, 4199704, 36189527, 59405277, 51590803, 81677380, 62430984, 89996536, 84166196, 20836893, 15015906, 20681921, 3633375, 53547802, 45794415, 53393358, 78884452, 69605283, 21070110, 81898046, 30891921, 62490109, 80246713, 33061250, 79738755, 93790285, 12571310, 30090481, 22108665, 92071990, 8129978, 13819358, 38341669, 73168565, 33123618, 21289531, 47887470, 37675718, 569864, 42967683, 72793444, 58180653, 30543215, 54427233, 874791, 79880247, 84002370, 35192533, 19101477, 32590267, 75986488, 6793819, 85116755, 89214616, 43322743, 12348118, 44889423, 79136082, 51464002, 18504197, 4508700, 89811711, 71333116, 74357852, 78766358, 81774825, 30811010, 69355476, 74110882, 56153345, 40356877, 94711842, 12024238, 60393039, 99524975, 43152977, 78300864, 60955663, 67030811, 98948034, 20140249, 62762857, 41092102, 83269727, 26292919, 36396314, 6986898, 22129328, 99224392, 4064751, 17727650, 21397057, 79821897, 37891451, 10597197, 97057187, 61897582, 4985896, 90090964, 48893685, 67124282, 5111370, 88904910, 36753250, 45049308, 526217, 77300457, 45075471, 68824981, 32161669, 9599614, 92803766, 59109400, 76540481, 14947650, 91727510, 2891150, 26139110, 33336148, 80251430, 78218845, 17539625, 13231279, 57961282, 84684495, 96852321, 76315420, 75153252, 3487592, 95581843, 78602717, 81805959, 30694952, 38494874, 1936762, 91831487, 89419466, 42947632, 75229462, 14326617, 21919959, 95290172, 36933038, 1788101, 84406788, 17976208, 92398073, 34667286, 32250045, 27325428, 90061527, 17857111, 7919588, 24591705, 65715134, 6153406, 53648154, 62232211, 21673260, 68110330, 99604946, 38061439, 30395570, 24619760, 70727211, 31733363, 99861373, 75135448, 59197747, 76434144, 77377183, 75407347, 86301513, 7423788, 20765474, 92867155, 62803858, 1022677, 39171895, 98653983, 42199455, 82052050, 11543098, 13468268, 66322959, 51507425, 84904436, 46851987, 4798568, 44847298, 36845587, 37739481, 38365584, 49597667, 15535065, 9175338, 51047803, 71300104, 92604458, 96906410, 29029316, 7646095, 73109997, 7066775, 18783702, 86118021, 56461322, 47090124, 34044787, 71316369, 68644627, 38022834, 27041967, 52204879, 2607799, 31126490, 77898274, 57359924, 90004325, 72732205, 65074535, 68204242, 72777973, 72373496, 45919976, 10264691, 24953498, 64055960, 1431742, 66319530, 97783876, 24168411, 87710366, 37166608, 68939068, 50567636, 89078848, 40686254, 46870723, 71657078, 34432810, 65851721, 97641116, 14220886, 74743862, 82897371, 2917920, 56531125, 89637706, 94911072, 44846932, 36812683, 54517921, 93566986, 24733232, 61859581, 10366309, 40677414, 38645117, 13470059, 23432750, 35512853, 26102057, 85242963, 83533741, 75543508, 59371804, 18001617, 90272749, 30139692, 44473167, 92215320, 44348328, 27185644, 90013093, 17068582, 68102437, 40865610, 54762643, 88653118, 3088684, 28796059, 30998561, 49328608, 2208785, 7229550, 95010552, 37280276, 48675329, 37957788, 72495719, 77272628, 68128438, 34236719, 6525948, 14723410, 81853704, 32699744, 38658347, 73222868, 82178706, 22879907, 54263880, 61373987, 1569515, 40027975, 37224844, 8729146, 19486173, 78589145, 61928316, 45990383, 9575954, 68875490, 82979980, 6111563, 94360702, 48088883, 4204661, 3233569, 89804152, 55189057, 66271566, 52293995, 16019925, 55615885, 78909561, 37560164, 77620120, 34295794, 29188588, 48658605, 99690194, 32426752, 5073754, 74724075, 47708850, 6505939, 99965001, 8791066, 17146629, 59329511, 77787724, 29466406, 7182642, 85711894, 37659250, 4119747, 27665211, 7687278, 19272365, 44479073, 50668599, 23194618, 9257405, 85571389, 29994197, 73021291, 72357096, 824872, 66663942, 56424103, 54606384, 33201905, 11581181, 99333375, 67513640, 17386996, 82726008, 5832946, 10453030, 40152546, 16380211, 73031054, 86821229, 6521313, 92353856, 69697787, 62693428, 321665, 62452034, 9886593, 19457589, 43506672, 61960400, 15148031, 23848565, 60176618, 76825057, 90158785, 92787493, 22405120, 16405341, 70800879, 66137019, 33797252, 26493119, 45667668, 97561745, 33431960, 47361209, 75820087, 33304202, 51715482, 72019362, 28928175, 53084256, 65271999, 66611759, 54199160, 3880712, 26776922, 6038457, 36468541, 26392416, 67084351, 42237907, 54014062, 4978543, 65454636, 53802686, 15948937, 18466635, 32159704, 26734892, 67227442, 65338021, 70596786, 22942635, 8807940, 40984766, 3773993, 89046466, 83789391, 15064655, 98648327, 62552524, 47213183, 63059024, 95395112, 55850790, 60581278, 86543538, 26275734, 76671482, 85117092, 34698463, 42307449, 30653863, 80555751, 61741594, 1239555, 92692283, 23110625, 7685448, 54868730, 63967300, 12303248, 77694700, 44245960, 2204165, 36808486, 41245325, 51426311, 4099191, 99297164, 95957797, 29834512, 18131876, 16054533, 16424199, 41481685, 36135, 92692978, 24314885, 33935899, 91938191, 1204161, 66428795, 99021067, 18806856, 8733068, 47298834, 31491938, 56484900, 77284619, 5822202, 99226875, 34497327, 70420215, 11757872, 62428472, 26744917, 8128637, 12664567, 77413012, 44060493, 34946859, 6871053, 11365791, 9398733, 44627776, 16387575, 99917599, 44481640, 36139942, 72725103, 41092172, 4095116, 97940276, 45790169, 45617087, 57241521, 19891772, 88698958, 19939935, 72274002, 87160386, 26998766, 91141395, 8055981, 44664587, 93270084, 66045587, 75052463, 47792865, 70782102, 61271144, 64848072, 749283, 20885148, 97011160, 22997281, 34493392, 59614746, 38008118, 54663246, 69848388, 16595436, 24058273, 73392814, 64087743, 92554120, 75777973, 61728685, 17016156, 33553959, 48260151, 84293052, 62740044, 36505482, 30463802, 41380093, 45428665, 71469330, 37620363, 37501808, 63152504, 6808825, 55960386, 41442762, 43376279, 55143724, 14045383, 97379790, 32058615, 68316156, 82651278, 61859143, 20122224, 79806380, 92033260, 63015256, 16567550, 20645197, 4668450, 14731700, 6819644, 38256849, 36780454, 56955985, 37192445, 45848907, 45381876, 99515901, 47738185, 82779622, 54987042, 91255408, 8696647, 65017137, 99125126, 61141874, 57020481, 95251277, 5267545, 83210802, 90310261, 79942022, 69641513, 17957593, 83133790, 35092039, 8099994, 16445503, 49882705, 75128745, 45237957, 36312813, 79922758, 78549759, 20568363, 9860968, 26397786, 91990218, 47893286, 17365188, 61712234, 89499542, 70004753, 87598888, 65880522, 88130087, 42644903, 29932657, 40197395, 80014588, 92541302, 39847321, 16099750, 98371444, 70541760, 33237508, 40781449, 29401781, 73786944, 62496012, 9603598, 36930650, 74452589, 55669657, 33565483, 14349098, 7502255, 45995325, 99971982, 91240048, 94516935, 22435353, 69579137, 58208470, 3233084, 25842078, 21001913, 28787861, 55602660, 8651647, 60106217, 6497830, 6157724, 15163258, 94539824, 86460488, 3235882, 69255765, 30569392, 66116458, 39801351, 23134715, 26063929, 61815223, 83948335, 69786756, 33249630, 81172706, 67281495, 16971929, 7517032, 24915585, 54232247, 49271185, 87720882, 1375023, 84187166, 57803235, 48774913, 247198, 30163921, 66832478, 53888755, 68694897, 74614639, 71083565, 93057697, 21533347, 98462867, 42782652, 49236559, 93515664, 176257, 68000591, 93933709, 43933006, 36685023, 59177718, 27187213, 70367851, 50007421, 39373729, 88047921, 18699206, 83083131, 51588015, 53842979, 57248122, 64602895, 90654836, 50702367, 17081350, 83651211, 62051033, 59910624, 30771409, 10358899, 32151165 +86022504, 11923835, 70372191, 7646095, 4069912, 25842078, 54199160, 16595436, 3633375, 7423788, 52060076, 83302115, 94595668, 61960400, 68824981, 6221471, 68816413, 27325428, 98130363, 99861373, 64157906, 84904436, 77694700, 58751351, 18131876, 65047700, 72238278, 5822202, 24168411, 89637706, 92803766, 83210802, 40677414, 92787493, 17764950, 70800879, 88251446, 28928175, 53084256, 30694952, 47792865, 75229462, 59405277, 10309525, 54663246, 81853704, 24058273, 99604946, 52613508, 42644903, 29635537, 42073124, 66885828, 29834512, 16971929, 2607799, 32058615, 45919976, 14731700, 67474219, 67513640, 44060493, 29819940, 99971982, 66832478, 92353856, 56531125, 32274392, 69136837, 4787945, 78218845, 21533347, 98462867, 40865610, 62357986, 14093520, 84406788, 28734791, 92398073, 34236719, 89499542, 62232211, 81898046, 30395570, 15064655, 8729146, 55753905, 76434144, 98739783, 65275241, 82886381, 60102965, 55189057, 11543098, 8913721, 26063929, 63506281, 36505482, 415901, 91957544, 18411915, 48658605, 54868730, 29029316, 99965001, 25636669, 17146629, 91664334, 74357852, 16424199, 68316156, 30811010, 69355476, 71920426, 49271185, 92033260, 1375023, 66663942, 34497327, 41092102, 50567636, 56955985, 50702367, 83155442, 32151165, 59318837, 91255408, 90090964, 67124282, 526217, 18833224, 44481640, 61859581, 26102057, 66137019, 33797252, 97561745, 13173644, 58208470, 90457870, 75128745, 95581843, 96420429, 60430369, 3183975, 75052463, 38494874, 67084351, 10358899, 9829782, 48675329, 44915235, 18466635, 68128438, 15163258, 84166196, 38008118, 7919588, 61712234, 30891921, 62490109, 8807940, 86460488, 20867149, 70727211, 79738755, 78589145, 92071990, 75407347, 33553959, 43933006, 58180653, 48260151, 80014588, 30653863, 65081429, 84002370, 44847298, 37739481, 41380093, 45428665, 85116755, 9188443, 96906410, 33249630, 44889423, 73109997, 18783702, 56461322, 4508700, 41442762, 55143724, 74110882, 7687278, 19272365, 68204242, 29401781, 72777973, 85571389, 39201414, 99524975, 55247455, 86242799, 20002147, 77284619, 38256849, 17037369, 45381876, 99515901, 96318094, 45995325, 40152546, 34432810, 54987042, 86821229, 2717150, 48893685, 56515456, 321665, 9886593, 45049308, 43501211, 71083565, 15148031, 26664538, 10366309, 23848565, 19376156, 13470059, 99549373, 16405341, 9058407, 4095116, 27411561, 79942022, 35996293, 94516935, 68041839, 26493119, 45617087, 8373289, 94090109, 69579137, 47361209, 44348328, 96852321, 26863229, 3233084, 72274002, 93359396, 78785507, 49882705, 28550822, 3088684, 79657802, 42782652, 2208785, 4515343, 78602717, 66903004, 55470718, 95290172, 36468541, 13862149, 63628376, 67451935, 32250045, 90061527, 22997281, 6525948, 1197320, 17857111, 53547802, 65338021, 21993752, 61373987, 31733363, 59197747, 24826575, 5482538, 13348726, 68000591, 45990383, 9575954, 3235882, 78561158, 86301513, 63059024, 39801351, 95395112, 20765474, 62803858, 82979980, 42967683, 86543538, 4204661, 26275734, 82052050, 66271566, 51507425, 874791, 73617245, 55615885, 2331773, 26507214, 37560164, 23110625, 34295794, 72738685, 75986488, 57163802, 89214616, 99690194, 5073754, 71469330, 37620363, 86118021, 47090124, 99297164, 68644627, 5970607, 38022834, 77787724, 96726697, 78766358, 31126490, 97379790, 36135, 7517032, 57359924, 4091162, 27665211, 56153345, 50668599, 23194618, 10264691, 20645197, 40781854, 3294781, 824872, 99226875, 61982238, 11757872, 74452589, 77187825, 99333375, 68939068, 37192445, 45996863, 8128637, 12664567, 36396314, 17386996, 17081350, 52261574, 46870723, 4064751, 17727650, 7502255, 16380211, 67793644, 50806615, 30163921, 61897582, 669105, 44983451, 6521313, 74743862, 45306070, 5111370, 44846932, 44627776, 74614639, 39553046, 83368048, 79191827, 99917599, 24733232, 38645117, 90158785, 72725103, 4806458, 45407418, 48673079, 26139110, 22435353, 93053405, 49598724, 33336148, 30139692, 44473167, 33431960, 13231279, 57961282, 75820087, 3509435, 69641513, 84684495, 17894977, 17068582, 76315420, 8099994, 51715482, 61623915, 88653118, 44664587, 68702632, 26776922, 66045587, 30998561, 49328608, 79922758, 6038457, 81805959, 78549759, 20568363, 98943869, 91831487, 42947632, 30366150, 93515664, 15075176, 6157724, 15948937, 34667286, 81677380, 14626618, 32159704, 55770687, 69605283, 176257, 83789391, 87598888, 37224844, 64602895, 62552524, 69255765, 30569392, 66116458, 13819358, 92554120, 38341669, 31161687, 89804152, 76671482, 85117092, 13468268, 62740044, 72614359, 19101477, 32590267, 92692283, 29188588, 98371444, 43322743, 42692881, 36808486, 70541760, 55960386, 50007421, 33699435, 39373729, 22113133, 29466406, 43376279, 7182642, 16054533, 59910624, 81774825, 40781449, 28851716, 27625735, 72732205, 54232247, 82532312, 33935899, 44479073, 18806856, 24953498, 74441448, 60955663, 4668450, 73021291, 36930650, 55669657, 87710366, 33201905, 62428472, 84187166, 26292919, 57803235, 82726008, 76330843, 14349098, 48774913, 21397057, 10597197, 65851721, 97641116, 14220886, 11365791, 82897371, 62693428, 99125126, 61141874, 44842615, 9599614, 91240048, 59109400, 60176618, 23432750, 83651211, 51588015, 85242963, 97940276, 91727510, 2891150, 59371804, 45667668, 8634541, 19891772, 29510992, 81855445, 38510840, 35092039, 16445503, 22450468, 99658235, 10961421, 48395186, 8055981, 93270084, 66250369, 66667729, 36312813, 91802888, 55602660, 74137926, 33628349, 51315460, 1936762, 83150534, 21919959, 88444207, 26397786, 57393458, 49236559, 37280276, 4978543, 17976208, 53802686, 34493392, 89996536, 24591705, 15015906, 65715134, 6153406, 19898053, 38658347, 22942635, 80246713, 40984766, 30787683, 76170907, 19486173, 65880522, 30090481, 98648327, 5640302, 10649306, 75777973, 61728685, 55850790, 29932657, 21289531, 61263068, 48088883, 42199455, 52293995, 66322959, 54427233, 79880247, 46851987, 48892201, 15535065, 6793819, 47708850, 81172706, 18663507, 2204165, 7066775, 8791066, 95957797, 27041967, 85711894, 41481685, 77898274, 82651278, 20122224, 37659250, 65074535, 63015256, 72373496, 29994197, 16567550, 95726235, 47298834, 37435892, 83083131, 66319530, 67030811, 30218878, 62496012, 56424103, 70420215, 83269727, 11581181, 33565483, 6986898, 47738185, 5832946, 34946859, 6871053, 37891451, 247198, 73031054, 72278539, 68694897, 94911072, 62452034, 36812683, 19457589, 65038678, 82327024, 84349107, 90310261, 76825057, 41092172, 83533741, 4434662, 57241521, 80251430, 17539625, 19939935, 27185644, 72019362, 84840549, 75153252, 91141395, 58224549, 73235980, 96735716, 23569917, 89419466, 14326617, 95010552, 26392416, 22200378, 4199704, 36189527, 9860195, 11161731, 72495719, 97011160, 94539824, 59614746, 26734892, 20836893, 45794415, 78884452, 21070110, 22879907, 21673260, 12416768, 70004753, 93790285, 75135448, 88130087, 67031644, 44784505, 47213183, 8129978, 47887470, 569864, 37183543, 39171895, 40197395, 36685023, 34698463, 36845587, 61815223, 1239555, 92541302, 51047803, 71300104, 32426752, 63967300, 44245960, 63152504, 79136082, 51464002, 67281495, 33237508, 71316369, 90004325, 90654836, 4119747, 24314885, 86798033, 50188404, 40356877, 8733068, 12024238, 43152977, 64055960, 1431742, 78300864, 6819644, 20140249, 59501487, 57658654, 62762857, 79322415, 54606384, 89699445, 26744917, 89078848, 57240218, 22129328, 30771409, 10453030, 97057187, 53888755, 2917920, 29065964, 36753250, 77300457, 95251277, 45075471, 92530431, 31904591, 93566986, 82339363, 93057697, 5267545, 8115266, 76540481, 14947650, 75543508, 80316608, 97281847, 18001617, 83133790, 90013093, 43357947, 63372756, 54762643, 45237957, 53842979, 7229550, 62115552, 81274566, 61271144, 1788101, 54014062, 6497830, 37957788, 51590803, 77272628, 30764367, 67227442, 53393358, 53648154, 82178706, 15902805, 38061439, 48360198, 61928316, 62051033, 73168565, 60581278, 17016156, 77301523, 37675718, 93933709, 16097038, 53666583, 17058722, 30543215, 76703609, 7955293, 77620120, 9175338, 7685448, 92604458, 92998591, 74724075, 70367851, 93562257, 83747892, 6808825, 41245325, 23740167, 56473732, 71333116, 14045383, 49641577, 61859143, 24915585, 18699206, 66428795, 14363867, 87720882, 60393039, 98948034, 31727379, 77413012, 53632373, 82779622, 79821897, 94076128, 168541, 22721500, 69697787, 88904910, 16387575, 54517921, 35512853, 20642888, 22405120, 92215320, 33304202, 34698428, 65271999, 66611759, 9623492, 28787861, 3880712, 33895336, 508198, 82427263, 57248122, 26114953, 9259676, 85023028, 59981773, 70782102, 36933038, 64848072, 51472275, 65454636, 20885148, 17365188, 14723410, 20486294, 35456853, 68110330, 38009615, 1569515, 12571310, 77377183, 22108665, 43045786, 68875490, 92867155, 33123618, 6111563, 3233569, 98653983, 16019925, 84293052, 78909561, 61741594, 83948335, 38365584, 49597667, 16099750, 69786756, 59177718, 112651, 22766820, 71522968, 37501808, 4099191, 7011964, 89811711, 59329511, 92692978, 79806380, 91938191, 76960303, 16684829, 99021067, 73786944, 31491938, 56484900, 96193415, 9603598, 97783876, 36780454, 39986008, 45848907, 84127901, 17385531, 54058788, 4985896, 9398733, 72358170, 57020481, 36139942, 45790169, 29516592, 17957593, 87160386, 26998766, 68102437, 3487592, 58749, 8651647, 9860968, 42237907, 36580610, 91990218, 749283, 70036438, 62430984, 64098930, 20681921, 85695762, 73222868, 33061250, 3773993, 54263880, 44177011, 89046466, 29959549, 77312810, 89217461, 94360702, 7300047, 72793444, 51149804, 99729357, 80555751, 62936963, 35192533, 30463802, 73124510, 39847321, 12303248, 27187213, 6505939, 18504197, 1204161, 9257405, 51466049, 72357096, 37166608, 15536795, 40686254, 1250437, 99224392, 46540998, 40534591, 44550764, 65017137, 91281584, 32161669, 90272749, 88698958, 71965942, 21001913, 47893286, 69848388, 32699744, 64087743, 40027975, 1022677, 42307449, 4798568, 12348118, 51426311, 88047921, 94711842, 77072625, 96709982, 43506672, 7104732, 28796059, 44844121, 24619760, 23134715, 34044787, 52204879, 8696647, 70596786, 73392814, 71657078, 86001008, 60106217 +16019925, 44627776, 65851721, 89046466, 92554120, 6111563, 72357096, 97783876, 72358170, 95251277, 22405120, 8055981, 70782102, 70596786, 3235882, 23134715, 37560164, 51047803, 66319530, 7502255, 247198, 43506672, 10366309, 90272749, 91141395, 28787861, 28796059, 82427263, 4199704, 4978543, 17365188, 14723410, 70727211, 38341669, 3233569, 98653983, 79880247, 61815223, 7955293, 72738685, 16099750, 112651, 27187213, 81172706, 37501808, 18504197, 92692978, 12024238, 3294781, 55669657, 67513640, 57240218, 99224392, 4064751, 11365791, 67124282, 16405341, 4434662, 97281847, 26863229, 17957593, 21001913, 90013093, 87160386, 58749, 3880712, 508198, 30694952, 75052463, 9860968, 1788101, 91990218, 9829782, 69848388, 73222868, 44177011, 8729146, 60581278, 1022677, 84293052, 26507214, 62936963, 38365584, 69786756, 74724075, 2204165, 93562257, 79136082, 51464002, 38022834, 77898274, 82651278, 57359924, 1204161, 87720882, 86242799, 6819644, 79322415, 36780454, 89699445, 45848907, 45996863, 96709982, 82779622, 97057187, 30163921, 53888755, 44550764, 44846932, 39553046, 90158785, 94516935, 57241521, 47361209, 92215320, 98462867, 25842078, 83133790, 71965942, 72019362, 26998766, 68102437, 99658235, 34698428, 65271999, 66667729, 79922758, 7229550, 8651647, 1936762, 89419466, 26392416, 42237907, 84406788, 36189527, 37957788, 18466635, 51590803, 22997281, 94539824, 73392814, 21070110, 22942635, 80246713, 70004753, 93790285, 24826575, 65880522, 76434144, 45990383, 75777973, 17016156, 569864, 86543538, 16097038, 39171895, 4204661, 51149804, 44847298, 61741594, 45428665, 42073124, 63967300, 51426311, 89811711, 22113133, 59329511, 29466406, 91664334, 31126490, 90654836, 79806380, 33935899, 16684829, 56153345, 9257405, 51466049, 31491938, 56424103, 34497327, 36930650, 26744917, 84127901, 17081350, 48774913, 30771409, 71657078, 79821897, 168541, 4985896, 22721500, 82897371, 321665, 36812683, 24733232, 32161669, 36139942, 60176618, 27411561, 97940276, 2891150, 26493119, 18001617, 29516592, 44473167, 94090109, 58208470, 13231279, 57961282, 16445503, 3487592, 58224549, 73235980, 36312813, 42782652, 98943869, 91831487, 60106217, 14093520, 21919959, 95010552, 36933038, 88444207, 57393458, 37280276, 92398073, 68128438, 34236719, 38008118, 15015906, 67227442, 16595436, 32699744, 85695762, 82178706, 21673260, 99604946, 33061250, 3773993, 24619760, 87598888, 37224844, 59197747, 48360198, 77377183, 86301513, 63059024, 43045786, 61728685, 21289531, 37675718, 55189057, 80014588, 65081429, 84002370, 80555751, 49597667, 23110625, 92541302, 39847321, 6793819, 54868730, 12303248, 12348118, 83747892, 23740167, 7066775, 56473732, 52204879, 88047921, 7182642, 55143724, 85711894, 20122224, 40781449, 37659250, 4119747, 18699206, 91938191, 73786944, 40356877, 18806856, 60393039, 24953498, 78300864, 40781854, 98948034, 9603598, 77187825, 83269727, 62428472, 11581181, 99333375, 31727379, 45381876, 8128637, 15536795, 36396314, 40686254, 76330843, 46870723, 17727650, 40534591, 6871053, 73031054, 86821229, 91255408, 6521313, 8696647, 2917920, 9398733, 99125126, 71083565, 83368048, 99917599, 61859581, 44842615, 93057697, 91240048, 40677414, 90310261, 13470059, 4787945, 35512853, 20642888, 26139110, 49598724, 68041839, 78218845, 69579137, 17539625, 81855445, 88698958, 3233084, 27185644, 17068582, 76315420, 93359396, 66611759, 57248122, 55602660, 74137926, 81805959, 42947632, 55470718, 75229462, 14326617, 26397786, 70036438, 44915235, 77272628, 97011160, 34493392, 54663246, 1197320, 7919588, 53393358, 19898053, 78884452, 69605283, 38658347, 22879907, 62232211, 44844121, 62490109, 15902805, 38009615, 64087743, 54263880, 40027975, 76170907, 88130087, 78589145, 9575954, 30569392, 98739783, 62051033, 29932657, 82886381, 77301523, 89217461, 43933006, 89804152, 17058722, 40197395, 52293995, 85117092, 42307449, 874791, 55615885, 72614359, 36505482, 37739481, 83948335, 32590267, 85116755, 18411915, 98371444, 7685448, 71300104, 92998591, 22766820, 47708850, 71469330, 63152504, 6505939, 70541760, 8791066, 17146629, 34044787, 71316369, 71333116, 2607799, 43376279, 14045383, 16054533, 90004325, 4091162, 86798033, 65074535, 99021067, 50668599, 68204242, 23194618, 85571389, 39201414, 45919976, 8733068, 47298834, 43152977, 55247455, 4668450, 20002147, 30218878, 73021291, 62496012, 5822202, 61982238, 33201905, 33565483, 39986008, 53632373, 17385531, 99515901, 40152546, 94595668, 97641116, 669105, 72278539, 2717150, 65017137, 88904910, 61141874, 43501211, 54517921, 15148031, 26664538, 92803766, 72725103, 51588015, 26102057, 85242963, 75543508, 93053405, 45667668, 30139692, 8634541, 19891772, 86001008, 40865610, 28550822, 75128745, 11923835, 45237957, 44664587, 4515343, 96420429, 91802888, 62115552, 23569917, 20568363, 66903004, 81274566, 59981773, 61271144, 49236559, 22200378, 749283, 48675329, 9860195, 65454636, 11161731, 15948937, 59614746, 17857111, 20681921, 53547802, 6153406, 20486294, 35456853, 53648154, 38061439, 30787683, 20867149, 67031644, 13348726, 30090481, 61928316, 44784505, 47213183, 68875490, 39801351, 95395112, 47887470, 37183543, 93933709, 94360702, 53666583, 99729357, 13468268, 73617245, 84904436, 30653863, 2331773, 4798568, 78909561, 30463802, 1239555, 41380093, 92692283, 29188588, 9175338, 99690194, 43322743, 29029316, 7646095, 6808825, 55960386, 4099191, 47090124, 39373729, 68644627, 16971929, 18131876, 96726697, 36135, 52060076, 24915585, 30811010, 69355476, 66428795, 76960303, 29994197, 94711842, 99524975, 56484900, 37435892, 64055960, 83083131, 60955663, 77284619, 20140249, 59501487, 11757872, 74452589, 17037369, 12664567, 26292919, 77413012, 57803235, 14349098, 46540998, 45995325, 10597197, 61897582, 48893685, 68694897, 19457589, 91281584, 57020481, 18833224, 32274392, 92530431, 82339363, 83210802, 38645117, 23432750, 17764950, 83533741, 79942022, 45617087, 33431960, 80251430, 13173644, 69641513, 38510840, 33304202, 35092039, 72274002, 43357947, 63372756, 7104732, 66045587, 30998561, 6038457, 26114953, 85023028, 95290172, 47893286, 51472275, 93515664, 17976208, 15075176, 27325428, 62430984, 15163258, 32159704, 26734892, 45794415, 89499542, 55770687, 40984766, 68110330, 30395570, 61373987, 1569515, 176257, 83789391, 31733363, 99861373, 19486173, 64157906, 78561158, 7423788, 5640302, 29959549, 77312810, 82979980, 42967683, 7300047, 72793444, 36685023, 76671482, 34698463, 76703609, 26063929, 35192533, 73124510, 34295794, 15535065, 29635537, 57163802, 5073754, 44889423, 44245960, 67281495, 7011964, 33699435, 25636669, 95957797, 77787724, 29834512, 74357852, 59910624, 61859143, 28851716, 27625735, 72732205, 63015256, 44479073, 50188404, 72777973, 95726235, 14731700, 57658654, 38256849, 41092102, 37166608, 68939068, 50567636, 83155442, 1250437, 22129328, 47738185, 44060493, 29819940, 94076128, 67793644, 54987042, 62693428, 94911072, 29065964, 36753250, 16387575, 526217, 65038678, 74614639, 61960400, 68824981, 44481640, 82327024, 9599614, 23848565, 8115266, 83651211, 99549373, 9058407, 76540481, 35996293, 14947650, 33336148, 84684495, 19939935, 84840549, 48395186, 88653118, 9623492, 68702632, 26776922, 96735716, 2208785, 3183975, 78602717, 51315460, 9259676, 47792865, 54014062, 63628376, 6497830, 59405277, 20885148, 34667286, 89996536, 30764367, 6525948, 61712234, 81853704, 3633375, 86460488, 75135448, 68000591, 22108665, 62552524, 66116458, 8129978, 13819358, 31161687, 33123618, 61263068, 33553959, 48088883, 58180653, 42199455, 82052050, 11543098, 8913721, 51507425, 46851987, 48892201, 75986488, 9188443, 48658605, 92604458, 70367851, 18663507, 36808486, 50007421, 4508700, 99297164, 33237508, 5970607, 27041967, 16424199, 49641577, 68316156, 7517032, 54232247, 82532312, 49271185, 7687278, 14363867, 72373496, 10264691, 16567550, 1375023, 1431742, 99226875, 66663942, 96193415, 70420215, 62762857, 17386996, 52261574, 21397057, 34946859, 10453030, 34432810, 99971982, 66832478, 14220886, 90090964, 74743862, 92353856, 69697787, 45306070, 56515456, 89637706, 45049308, 45075471, 93566986, 84349107, 59109400, 92787493, 70800879, 59371804, 80316608, 22435353, 97561745, 29510992, 75820087, 96852321, 51715482, 61623915, 75153252, 53084256, 3088684, 54199160, 78549759, 33628349, 38494874, 68816413, 36580610, 30366150, 6157724, 90061527, 81677380, 64098930, 21993752, 81898046, 12416768, 79738755, 12571310, 55753905, 5482538, 92867155, 55850790, 26275734, 66271566, 66322959, 54427233, 91957544, 77620120, 70372191, 33249630, 71522968, 41245325, 99965001, 18783702, 78766358, 97379790, 41481685, 71920426, 24314885, 65047700, 19272365, 4069912, 67474219, 77072625, 54606384, 24168411, 84187166, 37192445, 89078848, 6986898, 5832946, 59318837, 37891451, 16380211, 50806615, 5111370, 62452034, 77300457, 79191827, 31904591, 5267545, 19376156, 76825057, 4806458, 41092172, 48673079, 4095116, 66137019, 45790169, 33797252, 8373289, 17894977, 8099994, 90457870, 49882705, 93270084, 66250369, 95581843, 33895336, 49328608, 83150534, 36468541, 10358899, 67451935, 53802686, 32250045, 72495719, 84166196, 98130363, 20836893, 24058273, 64602895, 98648327, 92071990, 69255765, 10649306, 20765474, 62803858, 73168565, 60102965, 30543215, 36845587, 415901, 59177718, 42692881, 66885828, 77694700, 37620363, 41442762, 81774825, 27665211, 92033260, 29401781, 72238278, 83302115, 74441448, 824872, 86022504, 82726008, 32151165, 96318094, 44983451, 56531125, 9886593, 69136837, 45407418, 21533347, 3509435, 10961421, 6221471, 54762643, 79657802, 60430369, 67084351, 13862149, 28734791, 24591705, 65715134, 65338021, 30891921, 8807940, 52613508, 42644903, 62740044, 19101477, 96906410, 32426752, 73109997, 86118021, 56461322, 58751351, 32058615, 74110882, 20645197, 67030811, 56955985, 50702367, 54058788, 91727510, 53842979, 64848072, 10309525, 15064655, 75407347, 65275241, 48260151, 89214616, 78785507, 88251446, 28928175, 62357986, 87710366, 44348328, 22450468, 14626618, 63506281 +3509435, 85117092, 38365584, 12303248, 59329511, 27041967, 8733068, 29516592, 69579137, 93270084, 66667729, 59981773, 26392416, 42237907, 10358899, 38008118, 62936963, 15535065, 57163802, 54868730, 33699435, 4985896, 36139942, 44915235, 20885148, 54263880, 48360198, 66116458, 6111563, 89804152, 34295794, 70367851, 49271185, 31491938, 79322415, 11581181, 26292919, 96709982, 71657078, 56531125, 26664538, 5267545, 84349107, 76825057, 90158785, 94516935, 98462867, 71965942, 43357947, 3880712, 37280276, 10309525, 17365188, 20486294, 80246713, 44177011, 89046466, 55850790, 60581278, 8913721, 16019925, 874791, 92541302, 51464002, 18504197, 17146629, 22113133, 29466406, 72732205, 7687278, 73021291, 3294781, 57658654, 38256849, 33201905, 26744917, 21397057, 168541, 22721500, 61859581, 23432750, 16405341, 26102057, 30139692, 80251430, 17957593, 35092039, 3487592, 36312813, 95581843, 96735716, 20568363, 8651647, 6497830, 15075176, 14626618, 73392814, 15902805, 3773993, 61373987, 55753905, 78589145, 13819358, 86543538, 26275734, 99729357, 13468268, 84002370, 62740044, 36505482, 37739481, 69786756, 44245960, 73109997, 70541760, 23740167, 52204879, 31126490, 92692978, 63015256, 16684829, 50188404, 99021067, 72373496, 39201414, 78300864, 6819644, 56955985, 4064751, 10453030, 32151165, 79821897, 37891451, 30163921, 14220886, 99125126, 29065964, 18833224, 99917599, 82339363, 24733232, 59109400, 60176618, 41092172, 22435353, 4434662, 68041839, 57241521, 29510992, 47361209, 88698958, 76315420, 65271999, 66250369, 28796059, 49328608, 7229550, 78602717, 81805959, 33628349, 66903004, 30366150, 63628376, 9829782, 4199704, 4978543, 17976208, 51590803, 32159704, 67227442, 55770687, 22879907, 44844121, 40984766, 93790285, 67031644, 77377183, 9575954, 44784505, 8129978, 43045786, 20765474, 38341669, 31161687, 17016156, 61263068, 77301523, 16097038, 48088883, 55189057, 82052050, 36685023, 4798568, 98371444, 42692881, 6505939, 89811711, 38022834, 14045383, 36135, 20122224, 79806380, 85571389, 12024238, 824872, 59501487, 61982238, 83269727, 45381876, 15536795, 17385531, 48774913, 46540998, 59318837, 247198, 97057187, 72278539, 2717150, 74743862, 2917920, 9886593, 43506672, 44842615, 10366309, 13470059, 72725103, 48673079, 4095116, 27411561, 97940276, 8634541, 94090109, 86001008, 17894977, 78785507, 88251446, 28550822, 54762643, 62357986, 7104732, 58224549, 33895336, 3183975, 23569917, 9860968, 36933038, 88444207, 91990218, 54014062, 34667286, 27325428, 90061527, 97011160, 1197320, 61712234, 81853704, 32699744, 78884452, 81898046, 22942635, 38061439, 33061250, 70727211, 31733363, 40027975, 5482538, 88130087, 22108665, 98739783, 61728685, 47887470, 569864, 3233569, 72793444, 98653983, 58180653, 80014588, 72614359, 44847298, 36845587, 78909561, 35192533, 30463802, 7955293, 1239555, 16099750, 18411915, 92604458, 42073124, 22766820, 66885828, 12348118, 74724075, 47708850, 79136082, 51426311, 56473732, 34044787, 95957797, 18131876, 43376279, 78766358, 16424199, 32058615, 61859143, 30811010, 90654836, 18699206, 91938191, 86798033, 50668599, 40356877, 37435892, 24953498, 20645197, 74441448, 14731700, 66319530, 70420215, 62762857, 99333375, 50567636, 39986008, 12664567, 53632373, 22129328, 47738185, 34946859, 6871053, 10597197, 99971982, 65851721, 73031054, 669105, 91255408, 6521313, 69697787, 62693428, 36753250, 45075471, 39553046, 15148031, 82327024, 83210802, 8115266, 51588015, 20642888, 22405120, 59371804, 18001617, 97561745, 17539625, 21533347, 75820087, 44348328, 21001913, 33304202, 90013093, 99658235, 8055981, 9623492, 3088684, 68702632, 26776922, 508198, 82427263, 60430369, 98943869, 47792865, 55470718, 75229462, 83150534, 95010552, 61271144, 1788101, 36580610, 47893286, 51472275, 48675329, 70036438, 36189527, 67451935, 59614746, 84166196, 14723410, 69848388, 98130363, 17857111, 7919588, 20681921, 16595436, 65715134, 24058273, 6153406, 53393358, 82178706, 86460488, 30395570, 30787683, 79738755, 65880522, 98648327, 62552524, 47213183, 78561158, 75407347, 86301513, 63059024, 95395112, 62051033, 43933006, 39171895, 60102965, 4204661, 17058722, 42199455, 66271566, 42307449, 76703609, 26063929, 51507425, 84293052, 73617245, 55615885, 2331773, 32590267, 49597667, 37560164, 45428665, 9175338, 48658605, 71300104, 89214616, 70372191, 63967300, 29029316, 7646095, 63152504, 7066775, 99965001, 86118021, 4099191, 7011964, 4508700, 71316369, 68644627, 74357852, 55143724, 68316156, 52060076, 40781449, 28851716, 69355476, 82532312, 1204161, 65047700, 19272365, 65074535, 72777973, 45919976, 18806856, 86242799, 4668450, 67474219, 5822202, 77072625, 9603598, 11757872, 86022504, 17037369, 84187166, 84127901, 57240218, 83155442, 14349098, 46870723, 30771409, 40534591, 54058788, 90090964, 8696647, 67124282, 88904910, 91281584, 526217, 74614639, 9599614, 99549373, 4806458, 85242963, 2891150, 93053405, 49598724, 33336148, 45667668, 45617087, 58208470, 57961282, 69641513, 27185644, 72274002, 16445503, 93359396, 84840549, 49882705, 11923835, 6221471, 48395186, 28787861, 54199160, 79657802, 30998561, 42782652, 79922758, 55602660, 51315460, 75052463, 68816413, 60106217, 89419466, 42947632, 21919959, 36468541, 749283, 65454636, 77272628, 22997281, 34493392, 62430984, 15163258, 89996536, 30764367, 64098930, 6525948, 26734892, 20836893, 70596786, 21070110, 62490109, 8807940, 37224844, 8729146, 76170907, 19486173, 64157906, 61928316, 3235882, 7423788, 68875490, 92554120, 42644903, 33123618, 21289531, 77312810, 23134715, 7300047, 51149804, 34698463, 79880247, 84904436, 61741594, 73124510, 75986488, 39847321, 29635537, 7685448, 59177718, 33249630, 5073754, 77694700, 81172706, 71522968, 71469330, 37501808, 41245325, 67281495, 47090124, 58751351, 77787724, 71333116, 16054533, 7517032, 77898274, 82651278, 37659250, 24314885, 33935899, 87720882, 95726235, 51466049, 60393039, 47298834, 43152977, 98948034, 62496012, 96193415, 97783876, 36780454, 31727379, 45996863, 8128637, 1250437, 52261574, 82779622, 44060493, 5832946, 16380211, 34432810, 67793644, 66832478, 53888755, 321665, 65017137, 44627776, 36812683, 45049308, 43501211, 61960400, 71083565, 54517921, 79191827, 93566986, 68824981, 91240048, 92803766, 40677414, 90310261, 83651211, 35512853, 92787493, 83533741, 14947650, 91727510, 75543508, 33797252, 26493119, 90272749, 33431960, 13173644, 92215320, 13231279, 19939935, 83133790, 17068582, 75128745, 10961421, 91141395, 66611759, 4515343, 62115552, 30694952, 1936762, 95290172, 70782102, 26397786, 67084351, 28734791, 9860195, 53547802, 19898053, 89499542, 69605283, 73222868, 62232211, 30891921, 21673260, 87598888, 75135448, 64602895, 76434144, 68000591, 30569392, 10649306, 75777973, 29932657, 1022677, 89217461, 82979980, 42967683, 76671482, 11543098, 66322959, 30653863, 65081429, 80555751, 19101477, 48892201, 415901, 83948335, 92692283, 29188588, 6793819, 96906410, 99690194, 92998591, 18663507, 2204165, 83747892, 50007421, 25636669, 99297164, 41442762, 39373729, 29834512, 7182642, 59910624, 57359924, 27625735, 4119747, 74110882, 27665211, 66428795, 76960303, 56153345, 68204242, 23194618, 29994197, 20002147, 66663942, 36930650, 74452589, 41092102, 33565483, 37192445, 77413012, 89078848, 36396314, 57803235, 17386996, 17081350, 99515901, 17727650, 29819940, 40152546, 94076128, 86821229, 44550764, 92353856, 48893685, 45306070, 56515456, 89637706, 72358170, 16387575, 77300457, 95251277, 92530431, 19376156, 38645117, 4787945, 17764950, 19891772, 84684495, 3233084, 25842078, 8099994, 51715482, 72019362, 61623915, 58749, 34698428, 63372756, 88653118, 73235980, 45237957, 66045587, 57248122, 6038457, 78549759, 91831487, 81274566, 14326617, 37957788, 59405277, 6157724, 11161731, 53802686, 72495719, 94539824, 15015906, 3633375, 21993752, 38658347, 53648154, 68110330, 38009615, 1569515, 83789391, 59197747, 13348726, 30090481, 45990383, 52613508, 69255765, 92867155, 62803858, 73168565, 82886381, 29959549, 40197395, 46851987, 77620120, 23110625, 72738685, 85116755, 37620363, 93562257, 18783702, 56461322, 8791066, 2607799, 91664334, 88047921, 49641577, 41481685, 81774825, 24915585, 92033260, 44479073, 29401781, 9257405, 73786944, 16567550, 1375023, 99524975, 1431742, 83083131, 60955663, 40781854, 77284619, 20140249, 87710366, 68939068, 67513640, 50702367, 99224392, 94595668, 82897371, 68694897, 9398733, 5111370, 57020481, 65038678, 32274392, 83368048, 44481640, 32161669, 93057697, 69136837, 79942022, 66137019, 80316608, 97281847, 38510840, 90457870, 26998766, 68102437, 28928175, 75153252, 53084256, 44664587, 2208785, 96420429, 9259676, 14093520, 57393458, 13862149, 84406788, 49236559, 15948937, 81677380, 45794415, 64087743, 24619760, 15064655, 24826575, 93933709, 94360702, 48260151, 63506281, 54427233, 91957544, 112651, 27187213, 36808486, 33237508, 96726697, 85711894, 71920426, 14363867, 4069912, 10264691, 30218878, 72357096, 99226875, 77187825, 55669657, 54606384, 24168411, 37166608, 6986898, 40686254, 82726008, 45995325, 97641116, 62452034, 19457589, 45407418, 9058407, 76540481, 45790169, 44473167, 8373289, 81855445, 22450468, 74137926, 85023028, 22200378, 32250045, 68128438, 34236719, 54663246, 24591705, 65338021, 85695762, 99604946, 92071990, 5640302, 39801351, 37675718, 33553959, 37183543, 52293995, 26507214, 61815223, 41380093, 9188443, 51047803, 32426752, 44889423, 55960386, 4091162, 83302115, 56484900, 56424103, 34497327, 89699445, 50806615, 44983451, 94911072, 44846932, 31904591, 70800879, 26139110, 96852321, 26863229, 87160386, 40865610, 53842979, 91802888, 26114953, 64848072, 93515664, 18466635, 70004753, 176257, 99861373, 12571310, 30543215, 6808825, 5970607, 55247455, 45848907, 61897582, 11365791, 35996293, 38494874, 35456853, 12416768, 20867149, 53666583, 97379790, 90004325, 72238278, 64055960, 67030811, 76330843, 7502255, 96318094, 54987042, 61141874, 23848565, 78218845, 65275241, 43322743, 16971929, 54232247, 94711842, 62428472, 92398073 +14349098, 23432750, 9860968, 54987042, 26734892, 20836893, 70596786, 21289531, 13468268, 89811711, 29466406, 56424103, 97783876, 33565483, 86821229, 33431960, 19891772, 17068582, 45237957, 14326617, 4199704, 36845587, 61741594, 43322743, 63152504, 8791066, 88047921, 85711894, 77898274, 72357096, 8128637, 48774913, 94911072, 72358170, 44627776, 43506672, 38645117, 4095116, 94516935, 18001617, 27185644, 87160386, 75153252, 3880712, 79657802, 57248122, 47792865, 37957788, 89996536, 69848388, 20486294, 63059024, 6111563, 89804152, 17058722, 69786756, 27041967, 74357852, 96726697, 82651278, 61859143, 24314885, 68204242, 40356877, 6986898, 99224392, 21397057, 65851721, 22721500, 11365791, 65017137, 95251277, 65038678, 32161669, 40677414, 20642888, 68041839, 45617087, 57241521, 47361209, 88698958, 25842078, 17957593, 83133790, 38510840, 76315420, 26998766, 62357986, 26776922, 78549759, 8651647, 81274566, 95290172, 88444207, 42237907, 84406788, 47893286, 67451935, 24058273, 82178706, 64087743, 93790285, 76170907, 76434144, 3235882, 37675718, 77312810, 82979980, 23134715, 86543538, 30543215, 42307449, 84293052, 78909561, 23110625, 51047803, 63967300, 81172706, 37501808, 41442762, 55143724, 14045383, 59910624, 41481685, 20122224, 28851716, 92692978, 33935899, 14363867, 87720882, 40781854, 20140249, 66663942, 11757872, 79322415, 74452589, 68939068, 67513640, 46870723, 79821897, 247198, 6521313, 44550764, 2917920, 99125126, 88904910, 36753250, 74614639, 61960400, 71083565, 15148031, 24733232, 82327024, 93057697, 23848565, 92803766, 90310261, 59109400, 4787945, 92787493, 16405341, 35996293, 81855445, 75820087, 96852321, 33304202, 68102437, 53084256, 6221471, 28796059, 49328608, 7229550, 75052463, 38494874, 91831487, 68816413, 60106217, 42947632, 70782102, 49236559, 22200378, 10309525, 72495719, 54663246, 14723410, 32699744, 85695762, 53648154, 73222868, 37224844, 12571310, 55753905, 19486173, 17016156, 33553959, 11543098, 48260151, 55615885, 19101477, 37560164, 92692283, 48658605, 12348118, 7646095, 71469330, 70541760, 67281495, 56473732, 68644627, 16971929, 7182642, 57359924, 90004325, 30811010, 72732205, 66428795, 65074535, 63015256, 99021067, 23194618, 72238278, 73786944, 51466049, 31491938, 24953498, 67030811, 30218878, 37166608, 36780454, 84187166, 26292919, 77413012, 40686254, 30771409, 71657078, 40534591, 6871053, 168541, 97641116, 30163921, 91255408, 45306070, 89637706, 44846932, 61141874, 77300457, 72725103, 41092172, 70800879, 85242963, 27411561, 80316608, 45667668, 44473167, 94090109, 13231279, 57961282, 8099994, 54762643, 44664587, 28787861, 79922758, 81805959, 26114953, 89419466, 67084351, 93515664, 28734791, 27325428, 90061527, 77272628, 34493392, 15163258, 32159704, 38008118, 1197320, 35456853, 62232211, 81898046, 89046466, 83789391, 22108665, 39801351, 75777973, 60581278, 94360702, 43933006, 39171895, 60102965, 4204661, 40197395, 51149804, 874791, 80014588, 84904436, 46851987, 30463802, 32590267, 34295794, 75986488, 6793819, 9175338, 7685448, 92604458, 92998591, 22766820, 27187213, 74724075, 44245960, 71522968, 2204165, 36808486, 7066775, 50007421, 56461322, 33699435, 38022834, 77787724, 29834512, 32058615, 4091162, 90654836, 37659250, 4119747, 4069912, 29401781, 60393039, 99524975, 55247455, 74441448, 1431742, 6819644, 96193415, 9603598, 62762857, 38256849, 24168411, 87710366, 26744917, 84127901, 57240218, 96709982, 22129328, 4064751, 44060493, 7502255, 34946859, 10597197, 50806615, 72278539, 74743862, 48893685, 8696647, 9398733, 67124282, 62452034, 91281584, 57020481, 45049308, 43501211, 79191827, 61859581, 9599614, 83210802, 60176618, 35512853, 48673079, 79942022, 97940276, 29516592, 8634541, 58208470, 92215320, 98462867, 3233084, 43357947, 51715482, 11923835, 65271999, 9623492, 95581843, 82427263, 62115552, 23569917, 66903004, 64848072, 6497830, 9860195, 6157724, 92398073, 15948937, 97011160, 81677380, 22997281, 68128438, 59614746, 24591705, 20681921, 53547802, 6153406, 15902805, 12416768, 3773993, 70727211, 31733363, 40027975, 48360198, 13348726, 45990383, 78561158, 98739783, 92554120, 31161687, 73168565, 29959549, 93933709, 7300047, 3233569, 26275734, 66322959, 79880247, 30653863, 415901, 1239555, 49597667, 92541302, 45428665, 29188588, 9188443, 71300104, 112651, 42692881, 5073754, 93562257, 6505939, 79136082, 51464002, 18504197, 4099191, 47090124, 7011964, 58751351, 71316369, 18131876, 16424199, 36135, 7517032, 52060076, 40781449, 69355476, 74110882, 82532312, 91938191, 1204161, 9257405, 16567550, 8733068, 37435892, 64055960, 78300864, 66319530, 77284619, 3294781, 77072625, 57658654, 36930650, 54606384, 33201905, 89699445, 12664567, 36396314, 17386996, 17081350, 52261574, 29819940, 54058788, 37891451, 97057187, 61897582, 66832478, 53888755, 4985896, 69697787, 68694897, 56515456, 36812683, 32274392, 83368048, 5267545, 76825057, 83651211, 76540481, 14947650, 66137019, 26493119, 97561745, 90272749, 80251430, 13173644, 78218845, 84684495, 17894977, 22450468, 84840549, 78785507, 99658235, 63372756, 88653118, 73235980, 36312813, 68702632, 96735716, 508198, 2208785, 55602660, 9259676, 59981773, 21919959, 36468541, 91990218, 30366150, 10358899, 37280276, 9829782, 32250045, 62430984, 65715134, 45794415, 78884452, 22879907, 22942635, 21673260, 80246713, 68110330, 86460488, 33061250, 30395570, 30787683, 70004753, 79738755, 15064655, 65880522, 9575954, 44784505, 86301513, 8129978, 43045786, 68875490, 13819358, 62051033, 82886381, 47887470, 1022677, 98653983, 55189057, 76671482, 52293995, 8913721, 76703609, 99729357, 51507425, 65081429, 80555751, 37739481, 61815223, 83948335, 91957544, 29635537, 16099750, 89214616, 42073124, 12303248, 18663507, 23740167, 51426311, 86118021, 25636669, 17146629, 4508700, 34044787, 33237508, 97379790, 24915585, 18699206, 49271185, 76960303, 44479073, 50668599, 72777973, 85571389, 29994197, 45919976, 47298834, 86242799, 14731700, 98948034, 73021291, 62496012, 5822202, 83269727, 15536795, 83155442, 1250437, 82726008, 76330843, 17727650, 82779622, 46540998, 10453030, 96318094, 73031054, 669105, 2717150, 82897371, 321665, 29065964, 31904591, 82339363, 84349107, 90158785, 51588015, 99549373, 75543508, 59371804, 22435353, 45790169, 4434662, 97281847, 30139692, 29510992, 17539625, 21533347, 44348328, 26863229, 93359396, 72019362, 28550822, 75128745, 48395186, 7104732, 8055981, 66667729, 53842979, 66045587, 4515343, 78602717, 85023028, 14093520, 36933038, 1788101, 26397786, 36580610, 749283, 59405277, 65454636, 11161731, 34236719, 61712234, 15015906, 16595436, 73392814, 38658347, 30891921, 44844121, 62490109, 38009615, 99604946, 87598888, 99861373, 78589145, 30090481, 98648327, 68000591, 77377183, 75407347, 30569392, 20765474, 38341669, 55850790, 33123618, 569864, 37183543, 42967683, 16097038, 72793444, 36685023, 66271566, 34698463, 72614359, 7955293, 48892201, 85116755, 18411915, 59177718, 29029316, 47708850, 70367851, 73109997, 55960386, 99297164, 59329511, 5970607, 43376279, 49641577, 81774825, 27625735, 71920426, 27665211, 86798033, 50188404, 56153345, 39201414, 83302115, 94711842, 43152977, 61982238, 70420215, 11581181, 31727379, 17037369, 39986008, 56955985, 53632373, 32151165, 94076128, 16380211, 67793644, 526217, 54517921, 93566986, 68824981, 44842615, 10366309, 36139942, 19376156, 69136837, 13470059, 45407418, 22405120, 9058407, 26102057, 83533741, 91727510, 26139110, 93053405, 33336148, 8373289, 86001008, 35092039, 72274002, 90457870, 49882705, 66611759, 3088684, 66250369, 54199160, 42782652, 60430369, 6038457, 74137926, 51315460, 98943869, 1936762, 83150534, 61271144, 57393458, 54014062, 70036438, 36189527, 17365188, 94539824, 6525948, 98130363, 17857111, 7919588, 67227442, 53393358, 19898053, 21070110, 38061439, 24619760, 54263880, 44177011, 176257, 64602895, 88130087, 67031644, 62552524, 66116458, 7423788, 5640302, 58180653, 42199455, 85117092, 63506281, 16019925, 84002370, 26507214, 44847298, 62936963, 36505482, 35192533, 38365584, 41380093, 72738685, 39847321, 98371444, 54868730, 99690194, 70372191, 33249630, 66885828, 83747892, 99965001, 39373729, 22113133, 95957797, 71333116, 16054533, 79806380, 7687278, 19272365, 92033260, 12024238, 1375023, 56484900, 20645197, 83083131, 60955663, 67474219, 59501487, 34497327, 77187825, 55669657, 89078848, 50702367, 59318837, 40152546, 44983451, 14220886, 90090964, 92353856, 39553046, 92530431, 44481640, 8115266, 4806458, 2891150, 69641513, 19939935, 16445503, 28928175, 40865610, 61623915, 58749, 10961421, 34698428, 58224549, 93270084, 96420429, 91802888, 30694952, 75229462, 95010552, 13862149, 63628376, 48675329, 4978543, 17976208, 15075176, 20885148, 53802686, 51590803, 30764367, 84166196, 3633375, 65338021, 89499542, 69605283, 21993752, 8807940, 40984766, 61373987, 1569515, 20867149, 75135448, 59197747, 24826575, 64157906, 52613508, 47213183, 92071990, 42644903, 61728685, 29932657, 48088883, 82052050, 26063929, 77620120, 15535065, 57163802, 77694700, 37620363, 41245325, 68316156, 65047700, 16684829, 72373496, 10264691, 20002147, 824872, 45848907, 45381876, 45996863, 17385531, 99515901, 47738185, 5832946, 94595668, 5111370, 62693428, 56531125, 9886593, 26664538, 17764950, 69579137, 21001913, 88251446, 3487592, 3183975, 33628349, 20568363, 51472275, 44915235, 81853704, 55770687, 8729146, 5482538, 10649306, 92867155, 62803858, 53666583, 4798568, 96906410, 32426752, 44889423, 6808825, 2607799, 91664334, 78766358, 95726235, 18806856, 86022504, 99333375, 37192445, 57803235, 45995325, 34432810, 16387575, 18833224, 33797252, 49598724, 3509435, 71965942, 90013093, 91141395, 55470718, 64098930, 61928316, 69255765, 95395112, 89217461, 54427233, 73124510, 18783702, 52204879, 54232247, 4668450, 99226875, 62428472, 99971982, 19457589, 91240048, 33895336, 30998561, 26392416, 18466635, 14626618, 65275241, 73617245, 2331773, 31126490, 41092102, 45075471, 34667286, 77301523, 62740044, 50567636, 99917599, 61263068 +66250369, 54427233, 66885828, 41481685, 62452034, 44627776, 94090109, 81855445, 22450468, 28928175, 11923835, 6221471, 14093520, 36468541, 93515664, 37675718, 33553959, 4798568, 80555751, 18663507, 5970607, 97379790, 99226875, 11757872, 99224392, 321665, 68824981, 99549373, 44473167, 26863229, 57248122, 74137926, 78549759, 13862149, 49236559, 70036438, 35456853, 48360198, 88130087, 65275241, 47887470, 70372191, 90654836, 72777973, 29994197, 40781854, 56955985, 26292919, 7502255, 44550764, 48893685, 9886593, 526217, 32274392, 15148031, 69136837, 40677414, 8115266, 70800879, 59371804, 17894977, 42782652, 82427263, 1936762, 68816413, 21919959, 84406788, 59405277, 11161731, 32699744, 85695762, 68110330, 99604946, 70004753, 31733363, 29959549, 37183543, 16097038, 4204661, 2331773, 54868730, 71522968, 4099191, 56461322, 68644627, 2607799, 91664334, 69355476, 44479073, 64055960, 86242799, 14731700, 72357096, 62428472, 45381876, 17386996, 83155442, 17081350, 52261574, 4064751, 17727650, 29819940, 94076128, 66832478, 669105, 8696647, 45306070, 44842615, 76825057, 9058407, 26139110, 93053405, 58208470, 69641513, 88251446, 28550822, 58224549, 9623492, 66045587, 6038457, 57393458, 51472275, 65454636, 53802686, 62430984, 15015906, 86460488, 83789391, 15064655, 8729146, 30090481, 7423788, 68875490, 95395112, 62803858, 82886381, 1022677, 6111563, 66271566, 11543098, 8913721, 66322959, 46851987, 19101477, 61741594, 29635537, 32426752, 42073124, 33249630, 22766820, 42692881, 7646095, 37501808, 55960386, 56473732, 41442762, 16971929, 90004325, 20122224, 1204161, 56153345, 39201414, 10264691, 55247455, 30218878, 3294781, 824872, 5822202, 66663942, 34497327, 87710366, 17385531, 53888755, 4985896, 90090964, 19457589, 65038678, 31904591, 32161669, 90310261, 17764950, 85242963, 83533741, 35996293, 33797252, 45617087, 8373289, 29510992, 90013093, 34698428, 79657802, 2208785, 4515343, 55602660, 33628349, 36933038, 26392416, 42237907, 64848072, 749283, 4199704, 27325428, 17365188, 81677380, 84166196, 14723410, 1197320, 20681921, 16595436, 20486294, 12416768, 76434144, 64157906, 67031644, 47213183, 75407347, 30569392, 98739783, 13819358, 20765474, 55850790, 569864, 86543538, 26275734, 72793444, 55189057, 30543215, 42307449, 84293052, 30653863, 415901, 91957544, 32590267, 9188443, 99690194, 112651, 29029316, 47708850, 37620363, 36808486, 83747892, 18504197, 18783702, 50007421, 17146629, 4508700, 43376279, 74357852, 88047921, 36135, 81774825, 77898274, 82651278, 27625735, 54232247, 24314885, 86798033, 14363867, 95726235, 8733068, 99524975, 43152977, 78300864, 62762857, 41092102, 89078848, 1250437, 30771409, 32151165, 45995325, 73031054, 54987042, 6521313, 82897371, 9398733, 89637706, 65017137, 72358170, 91281584, 45049308, 77300457, 74614639, 99917599, 24733232, 10366309, 23848565, 4806458, 4095116, 97940276, 80316608, 33431960, 78218845, 98462867, 19939935, 21001913, 17068582, 43357947, 72019362, 49882705, 40865610, 53084256, 58749, 63372756, 48395186, 53842979, 60430369, 7229550, 51315460, 38494874, 85023028, 47792865, 48675329, 28734791, 67451935, 15075176, 6157724, 92398073, 20885148, 10309525, 59614746, 65715134, 45794415, 89499542, 69605283, 82178706, 15902805, 8807940, 38061439, 24619760, 30787683, 40027975, 76170907, 12571310, 19486173, 98648327, 77377183, 61928316, 62552524, 42644903, 62051033, 60581278, 61263068, 77301523, 82979980, 60102965, 3233569, 53666583, 82052050, 76671482, 48260151, 26063929, 51507425, 79880247, 84002370, 26507214, 36505482, 23110625, 6793819, 9175338, 85116755, 89214616, 74724075, 44889423, 67281495, 99965001, 7011964, 89811711, 71316369, 77787724, 29834512, 78766358, 57359924, 4091162, 63015256, 4069912, 87720882, 72373496, 18806856, 83302115, 1375023, 20645197, 60955663, 66319530, 67474219, 77284619, 59501487, 61982238, 56424103, 97783876, 54606384, 24168411, 83269727, 31727379, 84187166, 68939068, 39986008, 45996863, 8128637, 36396314, 57240218, 50702367, 40686254, 22129328, 47738185, 44060493, 34946859, 50806615, 97641116, 44983451, 72278539, 2717150, 11365791, 68694897, 56515456, 62693428, 56531125, 44846932, 16387575, 95251277, 18833224, 79191827, 93566986, 82339363, 19376156, 92803766, 60176618, 13470059, 83651211, 4787945, 20642888, 22405120, 48673079, 26102057, 27411561, 79942022, 2891150, 22435353, 49598724, 68041839, 97281847, 30139692, 13173644, 57961282, 3509435, 17957593, 83133790, 33304202, 51715482, 68102437, 75128745, 10961421, 73235980, 54199160, 30998561, 96420429, 3183975, 78602717, 9259676, 8651647, 81274566, 75229462, 1788101, 30366150, 9829782, 4978543, 9860195, 18466635, 32250045, 34236719, 98130363, 7919588, 3633375, 65338021, 19898053, 78884452, 55770687, 21993752, 62232211, 38009615, 3773993, 20867149, 70727211, 93790285, 24826575, 13348726, 3235882, 52613508, 92071990, 8129978, 43045786, 5640302, 75777973, 73168565, 33123618, 21289531, 23134715, 94360702, 48088883, 89804152, 98653983, 42199455, 40197395, 63506281, 65081429, 44847298, 35192533, 61815223, 48892201, 38365584, 92692283, 18411915, 7685448, 71300104, 92604458, 43322743, 2204165, 79136082, 86118021, 58751351, 95957797, 16054533, 59910624, 16424199, 32058615, 24915585, 72732205, 71920426, 74110882, 19272365, 92033260, 16684829, 68204242, 29401781, 85571389, 73786944, 60393039, 74441448, 83083131, 6819644, 98948034, 96193415, 57658654, 36930650, 74452589, 77187825, 38256849, 89699445, 17037369, 12664567, 82779622, 46540998, 79821897, 37891451, 65851721, 168541, 86821229, 91255408, 22721500, 92353856, 57020481, 36753250, 71083565, 39553046, 26664538, 61859581, 9599614, 93057697, 91240048, 83210802, 38645117, 51588015, 94516935, 66137019, 91727510, 33336148, 45667668, 18001617, 97561745, 69579137, 75820087, 86001008, 90457870, 87160386, 78785507, 61623915, 91141395, 66611759, 54762643, 44664587, 96735716, 98943869, 91831487, 89419466, 55470718, 14326617, 88444207, 67084351, 54014062, 47893286, 15948937, 97011160, 89996536, 30764367, 54663246, 24591705, 61712234, 81853704, 67227442, 24058273, 70596786, 73392814, 38658347, 44844121, 21673260, 80246713, 33061250, 54263880, 1569515, 79738755, 65880522, 68000591, 22108665, 45990383, 44784505, 69255765, 10649306, 29932657, 42967683, 58180653, 51149804, 76703609, 80014588, 84904436, 36845587, 62936963, 78909561, 37739481, 83948335, 73124510, 41380093, 16099750, 59177718, 63967300, 5073754, 77694700, 81172706, 93562257, 6808825, 33699435, 8791066, 39373729, 33237508, 59329511, 38022834, 71333116, 18131876, 96726697, 7182642, 85711894, 68316156, 52060076, 40781449, 30811010, 28851716, 79806380, 33935899, 50188404, 72238278, 40356877, 45919976, 16567550, 94711842, 73021291, 20140249, 77072625, 37166608, 45848907, 53632373, 96709982, 82726008, 46870723, 40534591, 96318094, 40152546, 16380211, 94595668, 14220886, 69697787, 94911072, 99125126, 29065964, 61960400, 83368048, 44481640, 59109400, 35512853, 14947650, 45790169, 4434662, 8634541, 92215320, 96852321, 25842078, 38510840, 35092039, 93359396, 75153252, 62357986, 7104732, 8055981, 3088684, 66667729, 36312813, 95581843, 28787861, 508198, 91802888, 26114953, 95290172, 91990218, 36189527, 6497830, 37957788, 90061527, 51590803, 72495719, 77272628, 14626618, 68128438, 34493392, 6525948, 17857111, 53547802, 6153406, 21070110, 53648154, 30891921, 62490109, 40984766, 64087743, 44177011, 89046466, 61373987, 99861373, 9575954, 78561158, 66116458, 63059024, 17016156, 93933709, 34698463, 99729357, 62740044, 7955293, 92541302, 34295794, 72738685, 70367851, 63152504, 70541760, 34044787, 27041967, 55143724, 82532312, 27665211, 7687278, 65047700, 65074535, 37435892, 4668450, 20002147, 67030811, 70420215, 9603598, 86022504, 11581181, 50567636, 67513640, 37192445, 77413012, 57803235, 6986898, 99515901, 76330843, 5832946, 71657078, 54058788, 61897582, 74743862, 43501211, 54517921, 45075471, 92530431, 82327024, 5267545, 84349107, 45407418, 92787493, 41092172, 16405341, 75543508, 26493119, 17539625, 44348328, 3233084, 71965942, 27185644, 72274002, 8099994, 45237957, 3880712, 33895336, 26776922, 49328608, 30694952, 9860968, 59981773, 83150534, 61271144, 26397786, 22200378, 44915235, 17976208, 22997281, 94539824, 26734892, 20836893, 176257, 55753905, 64602895, 78589145, 77312810, 89217461, 43933006, 52293995, 85117092, 13468268, 874791, 55615885, 72614359, 1239555, 49597667, 77620120, 75986488, 29188588, 98371444, 48658605, 96906410, 69786756, 27187213, 44245960, 73109997, 41245325, 23740167, 25636669, 99297164, 14045383, 49641577, 7517032, 61859143, 4119747, 92692978, 91938191, 49271185, 66428795, 76960303, 99021067, 50668599, 23194618, 9257405, 51466049, 12024238, 31491938, 56484900, 36780454, 99333375, 59318837, 6871053, 99971982, 67124282, 88904910, 61141874, 36139942, 90158785, 29516592, 57241521, 88698958, 16445503, 65271999, 88653118, 93270084, 79922758, 62115552, 75052463, 66903004, 36580610, 10358899, 34667286, 15163258, 32159704, 38008118, 53393358, 73222868, 22879907, 81898046, 87598888, 37224844, 75135448, 38341669, 92867155, 61728685, 7300047, 17058722, 15535065, 92998591, 71469330, 6505939, 7066775, 51426311, 18699206, 84127901, 14349098, 21397057, 10453030, 34432810, 67793644, 30163921, 5111370, 76540481, 80251430, 19891772, 47361209, 84684495, 76315420, 26998766, 99658235, 68702632, 28796059, 23569917, 20568363, 60106217, 70782102, 63628376, 69848388, 22942635, 59197747, 5482538, 39801351, 31161687, 36685023, 30463802, 51047803, 12303248, 51464002, 47090124, 22113133, 52204879, 37659250, 79322415, 26744917, 48774913, 247198, 10597197, 2917920, 43506672, 72725103, 21533347, 13231279, 84840549, 81805959, 95010552, 37280276, 30395570, 92554120, 39171895, 16019925, 39847321, 12348118, 29466406, 31126490, 47298834, 24953498, 1431742, 55669657, 33565483, 97057187, 36812683, 23432750, 90272749, 3487592, 42947632, 86301513, 73617245, 37560164, 57163802, 62496012, 15536795, 64098930, 45428665, 33201905 +82052050, 72725103, 85242963, 7104732, 30694952, 24314885, 65047700, 4806458, 70782102, 1569515, 31733363, 44784505, 29959549, 49597667, 26744917, 73031054, 90158785, 97281847, 32699744, 61263068, 62936963, 23740167, 71316369, 95957797, 16424199, 65074535, 44479073, 71657078, 10453030, 247198, 34432810, 74743862, 45075471, 59109400, 4434662, 26493119, 98462867, 88698958, 71965942, 78785507, 68102437, 54762643, 89419466, 88444207, 38061439, 48360198, 77377183, 95395112, 17016156, 33553959, 51149804, 26063929, 16019925, 4798568, 35192533, 1239555, 6793819, 57163802, 7685448, 92998591, 43322743, 66885828, 93562257, 67281495, 86118021, 4119747, 69355476, 92033260, 45919976, 4668450, 98948034, 99333375, 50567636, 17386996, 48774913, 32151165, 96318094, 59318837, 8696647, 68694897, 65038678, 18833224, 83210802, 49598724, 90272749, 26863229, 25842078, 93359396, 49882705, 8055981, 53842979, 508198, 85023028, 26397786, 84406788, 17365188, 89996536, 38008118, 70596786, 12416768, 54263880, 87598888, 29932657, 60581278, 33123618, 86543538, 52293995, 2331773, 84002370, 36505482, 72738685, 16099750, 33249630, 77694700, 44889423, 18663507, 34044787, 16971929, 78766358, 20122224, 54232247, 27665211, 18699206, 91938191, 7687278, 1204161, 72373496, 8733068, 73021291, 59501487, 5822202, 61982238, 62762857, 74452589, 37166608, 68939068, 15536795, 89078848, 5832946, 79821897, 44983451, 4985896, 11365791, 9886593, 29065964, 91281584, 57020481, 82339363, 99549373, 14947650, 2891150, 75543508, 80316608, 45790169, 27185644, 90013093, 26998766, 34698428, 9623492, 93270084, 66045587, 79922758, 91831487, 59981773, 61271144, 37280276, 9829782, 53802686, 27325428, 61712234, 67227442, 55770687, 69605283, 30891921, 62490109, 15064655, 59197747, 19486173, 98648327, 62552524, 45990383, 47213183, 30569392, 1022677, 569864, 89217461, 42199455, 36685023, 11543098, 30653863, 80555751, 30463802, 38365584, 29188588, 44245960, 51426311, 4099191, 4508700, 29834512, 74357852, 36135, 77898274, 61859143, 92692978, 33935899, 40356877, 94711842, 83083131, 66319530, 67030811, 20140249, 97783876, 77187825, 83269727, 33565483, 37192445, 45381876, 8128637, 45995325, 37891451, 99971982, 30163921, 54987042, 44550764, 2917920, 9398733, 56515456, 62693428, 88904910, 19457589, 74614639, 39553046, 79191827, 44481640, 19376156, 51588015, 30139692, 33431960, 80251430, 13173644, 96852321, 22450468, 65271999, 73235980, 3880712, 4515343, 91802888, 81805959, 78549759, 14326617, 1788101, 10358899, 64848072, 36189527, 17976208, 15948937, 32250045, 51590803, 64098930, 69848388, 15015906, 20681921, 65715134, 81898046, 40984766, 3773993, 44177011, 79738755, 65880522, 5482538, 13348726, 22108665, 69255765, 98739783, 92554120, 38341669, 92867155, 61728685, 77301523, 82979980, 23134715, 94360702, 43933006, 4204661, 98653983, 76703609, 61741594, 83948335, 73124510, 29635537, 18411915, 51047803, 71522968, 37620363, 63152504, 18504197, 41245325, 18783702, 56473732, 99297164, 52204879, 31126490, 88047921, 14045383, 32058615, 52060076, 24915585, 40781449, 27625735, 82532312, 66428795, 56153345, 68204242, 10264691, 95726235, 47298834, 37435892, 70420215, 57658654, 36930650, 84187166, 67513640, 39986008, 26292919, 6986898, 96709982, 82726008, 76330843, 30771409, 10597197, 94595668, 61897582, 2717150, 6521313, 90090964, 45306070, 44846932, 36812683, 16387575, 526217, 77300457, 54517921, 9599614, 93057697, 10366309, 8115266, 13470059, 45407418, 9058407, 70800879, 83533741, 33336148, 8373289, 69579137, 3233084, 76315420, 16445503, 90457870, 72019362, 99658235, 61623915, 10961421, 48395186, 3088684, 66667729, 68702632, 54199160, 26776922, 30998561, 23569917, 51315460, 75052463, 38494874, 42947632, 47792865, 37957788, 15075176, 9860195, 65454636, 92398073, 20885148, 10309525, 18466635, 14626618, 15163258, 30764367, 84166196, 98130363, 24591705, 16595436, 45794415, 53393358, 73392814, 78884452, 89499542, 22942635, 80246713, 89046466, 30787683, 20867149, 70727211, 93790285, 37224844, 8729146, 75135448, 76434144, 61928316, 92071990, 8129978, 68875490, 10649306, 39801351, 62051033, 62803858, 82886381, 37675718, 37183543, 39171895, 72793444, 17058722, 8913721, 63506281, 66322959, 84293052, 37739481, 39847321, 9175338, 48658605, 71300104, 89214616, 96906410, 12303248, 74724075, 73109997, 41442762, 58751351, 71333116, 91664334, 85711894, 59910624, 82651278, 90004325, 72732205, 76960303, 14363867, 99021067, 23194618, 83302115, 12024238, 99524975, 43152977, 78300864, 30218878, 824872, 62496012, 66663942, 96193415, 34497327, 79322415, 54606384, 83155442, 1250437, 34946859, 6871053, 97641116, 82897371, 92353856, 5111370, 94911072, 65017137, 99125126, 61141874, 83368048, 92530431, 68824981, 36139942, 69136837, 60176618, 76825057, 23432750, 4787945, 41092172, 4095116, 79942022, 97940276, 44473167, 57241521, 19891772, 29510992, 13231279, 3509435, 84684495, 17957593, 35092039, 43357947, 8099994, 28550822, 75153252, 53084256, 66611759, 36312813, 28787861, 33895336, 49328608, 96735716, 82427263, 55602660, 6038457, 26114953, 81274566, 60106217, 36580610, 47893286, 51472275, 6497830, 67451935, 72495719, 77272628, 32159704, 65338021, 6153406, 19898053, 62232211, 68110330, 99604946, 24619760, 61373987, 83789391, 12571310, 64602895, 88130087, 67031644, 9575954, 78561158, 43045786, 13819358, 77312810, 26275734, 58180653, 40197395, 30543215, 80014588, 55615885, 46851987, 26507214, 62740044, 41380093, 23110625, 75986488, 45428665, 9188443, 98371444, 92604458, 59177718, 32426752, 112651, 12348118, 29029316, 70367851, 7646095, 6808825, 55960386, 47090124, 8791066, 27041967, 43376279, 7182642, 16054533, 81774825, 7517032, 57359924, 4091162, 90654836, 28851716, 79806380, 4069912, 16684829, 73786944, 51466049, 60393039, 55247455, 64055960, 9603598, 55669657, 33201905, 62428472, 45848907, 12664567, 57803235, 50702367, 47738185, 82779622, 21397057, 7502255, 54058788, 16380211, 65851721, 67793644, 50806615, 66832478, 669105, 86821229, 91255408, 22721500, 48893685, 67124282, 72358170, 36753250, 61960400, 32274392, 31904591, 24733232, 32161669, 61859581, 23848565, 91240048, 91727510, 59371804, 45667668, 8634541, 94090109, 81855445, 57961282, 44348328, 69641513, 86001008, 19939935, 83133790, 21001913, 33304202, 72274002, 17068582, 87160386, 40865610, 62357986, 58224549, 44664587, 28796059, 42782652, 60430369, 74137926, 33628349, 9259676, 9860968, 14093520, 83150534, 95010552, 36933038, 91990218, 30366150, 49236559, 28734791, 34667286, 81677380, 22997281, 59614746, 6525948, 26734892, 7919588, 3633375, 21070110, 44844121, 86460488, 33061250, 70004753, 176257, 75407347, 42644903, 73168565, 47887470, 93933709, 16097038, 55189057, 76671482, 48260151, 54427233, 79880247, 44847298, 36845587, 78909561, 415901, 91957544, 92692283, 92541302, 34295794, 54868730, 42073124, 63967300, 22766820, 47708850, 6505939, 33699435, 17146629, 39373729, 5970607, 41481685, 71920426, 19272365, 63015256, 87720882, 50668599, 85571389, 29994197, 16567550, 18806856, 1375023, 56484900, 74441448, 1431742, 60955663, 40781854, 77284619, 3294781, 72357096, 99226875, 36780454, 11581181, 56955985, 77413012, 36396314, 84127901, 99515901, 52261574, 22129328, 46870723, 99224392, 4064751, 168541, 321665, 89637706, 62452034, 45049308, 95251277, 43506672, 99917599, 82327024, 38645117, 35512853, 16405341, 26102057, 94516935, 26139110, 33797252, 45617087, 47361209, 17539625, 75128745, 88653118, 45237957, 66250369, 2208785, 57248122, 62115552, 3183975, 20568363, 8651647, 66903004, 26392416, 67084351, 63628376, 48675329, 44915235, 93515664, 6157724, 11161731, 90061527, 97011160, 34236719, 14723410, 85695762, 21993752, 35456853, 73222868, 21673260, 15902805, 30395570, 99861373, 76170907, 78589145, 30090481, 52613508, 86301513, 75777973, 42967683, 6111563, 7300047, 60102965, 48088883, 66271566, 85117092, 99729357, 874791, 73617245, 61815223, 7955293, 48892201, 85116755, 81172706, 36808486, 51464002, 50007421, 7011964, 68644627, 59329511, 18131876, 37659250, 74110882, 29401781, 72777973, 9257405, 14731700, 41092102, 87710366, 31727379, 53632373, 57240218, 17385531, 44060493, 46540998, 94076128, 97057187, 53888755, 69697787, 44627776, 5267545, 40677414, 48673079, 22435353, 93053405, 97561745, 58208470, 92215320, 75820087, 38510840, 3487592, 91141395, 79657802, 78602717, 21919959, 42237907, 749283, 68128438, 62430984, 94539824, 54663246, 1197320, 38658347, 53648154, 82178706, 8807940, 55753905, 24826575, 68000591, 63059024, 7423788, 20765474, 65275241, 55850790, 21289531, 53666583, 42307449, 51507425, 84904436, 65081429, 19101477, 37560164, 77620120, 15535065, 99690194, 69786756, 5073754, 70541760, 7066775, 99965001, 25636669, 89811711, 22113133, 77787724, 96726697, 97379790, 49641577, 68316156, 30811010, 49271185, 50188404, 24953498, 20645197, 56424103, 24168411, 17037369, 45996863, 40686254, 14349098, 29819940, 72278539, 56531125, 71083565, 15148031, 84349107, 92803766, 90310261, 92787493, 17764950, 76540481, 66137019, 68041839, 29516592, 84840549, 63372756, 98943869, 1936762, 55470718, 36468541, 57393458, 70036438, 4978543, 17857111, 20836893, 53547802, 20486294, 22879907, 38009615, 40027975, 3235882, 66116458, 31161687, 3233569, 89804152, 34698463, 71469330, 37501808, 38022834, 2607799, 72238278, 39201414, 31491938, 20002147, 6819644, 77072625, 86022504, 89699445, 17727650, 40152546, 14220886, 43501211, 93566986, 44842615, 22405120, 35996293, 18001617, 78218845, 51715482, 88251446, 28928175, 11923835, 6221471, 95581843, 7229550, 68816413, 75229462, 95290172, 54014062, 13862149, 4199704, 59405277, 34493392, 81853704, 24058273, 5640302, 13468268, 72614359, 42692881, 27187213, 2204165, 83747892, 86798033, 86242799, 67474219, 17081350, 40534591, 26664538, 83651211, 20642888, 27411561, 22200378, 64087743, 64157906, 79136082, 56461322, 11757872, 38256849, 21533347, 58749, 96420429, 32590267, 70372191, 33237508, 29466406, 55143724, 17894977 +24591705, 45794415, 37183543, 52060076, 17976208, 53666583, 22766820, 66611759, 62115552, 15064655, 8913721, 84904436, 2331773, 29635537, 42073124, 23740167, 83302115, 47298834, 96318094, 94595668, 2917920, 26102057, 26863229, 86001008, 17068582, 78785507, 48395186, 30694952, 70782102, 15948937, 62232211, 52613508, 55189057, 52293995, 49597667, 70367851, 94711842, 67030811, 61982238, 79322415, 67793644, 168541, 72278539, 48893685, 56515456, 82339363, 85242963, 35996293, 28787861, 33628349, 51315460, 68816413, 47792865, 14326617, 36580610, 92398073, 15163258, 1197320, 7919588, 22942635, 92071990, 5640302, 61728685, 82886381, 16097038, 76703609, 80555751, 19101477, 18411915, 7685448, 29029316, 18783702, 14045383, 90004325, 30811010, 54232247, 4069912, 10264691, 45996863, 89078848, 17385531, 14349098, 48774913, 10453030, 61897582, 91255408, 11365791, 92353856, 8696647, 45306070, 44842615, 23848565, 4787945, 16405341, 49598724, 45617087, 33431960, 80251430, 81855445, 87160386, 72019362, 61623915, 3487592, 65271999, 66045587, 26397786, 22200378, 72495719, 14626618, 84166196, 61712234, 20681921, 55770687, 62490109, 40027975, 48360198, 67031644, 13348726, 30090481, 68875490, 569864, 82979980, 93933709, 92541302, 83747892, 55960386, 47090124, 4508700, 68644627, 36135, 77898274, 90654836, 82532312, 92033260, 99021067, 23194618, 31491938, 64055960, 70420215, 77072625, 77187825, 33201905, 96709982, 1250437, 99515901, 45995325, 50806615, 97641116, 82897371, 43501211, 38645117, 13470059, 23432750, 4806458, 93053405, 13231279, 11923835, 7104732, 3088684, 33895336, 60430369, 91802888, 81805959, 26114953, 85023028, 81274566, 95290172, 88444207, 57393458, 63628376, 70036438, 34667286, 90061527, 38008118, 98130363, 20836893, 65715134, 89499542, 24619760, 76170907, 59197747, 5482538, 73168565, 29932657, 17016156, 77312810, 23134715, 94360702, 39171895, 11543098, 63506281, 874791, 44847298, 48892201, 1239555, 23110625, 29188588, 69786756, 112651, 66885828, 74724075, 7646095, 93562257, 73109997, 36808486, 79136082, 50007421, 56473732, 58751351, 89811711, 22113133, 52204879, 91664334, 85711894, 16424199, 49641577, 61859143, 57359924, 19272365, 16684829, 50668599, 51466049, 1375023, 43152977, 20645197, 5822202, 34497327, 62762857, 97783876, 86022504, 24168411, 11581181, 31727379, 33565483, 36396314, 53632373, 40686254, 17727650, 247198, 34432810, 73031054, 22721500, 2717150, 5111370, 61141874, 19457589, 71083565, 93566986, 93057697, 83210802, 41092172, 48673079, 14947650, 33336148, 44473167, 8373289, 29510992, 47361209, 17539625, 58208470, 75820087, 3509435, 88698958, 27185644, 43357947, 22450468, 93359396, 68102437, 73235980, 45237957, 66250369, 30998561, 508198, 96420429, 74137926, 9259676, 66903004, 75229462, 9829782, 47893286, 67451935, 37957788, 51590803, 17365188, 97011160, 81677380, 54663246, 64098930, 32699744, 65338021, 19898053, 73392814, 38658347, 81898046, 21673260, 40984766, 38009615, 99604946, 33061250, 44177011, 89046466, 1569515, 176257, 79738755, 12571310, 19486173, 24826575, 64602895, 76434144, 64157906, 98648327, 68000591, 77377183, 61928316, 47213183, 78561158, 86301513, 7423788, 92554120, 38341669, 62051033, 92867155, 1022677, 61263068, 86543538, 4204661, 72793444, 51149804, 26063929, 26507214, 36845587, 30463802, 91957544, 73124510, 37560164, 34295794, 15535065, 45428665, 6793819, 48658605, 89214616, 96906410, 54868730, 70372191, 92998591, 43322743, 77694700, 12348118, 44245960, 47708850, 6808825, 7011964, 99297164, 71333116, 16971929, 27041967, 78766358, 59910624, 7517032, 4119747, 74110882, 91938191, 1204161, 76960303, 50188404, 68204242, 72238278, 73786944, 1431742, 60955663, 14731700, 98948034, 30218878, 9603598, 62428472, 26744917, 50567636, 56955985, 37192445, 26292919, 77413012, 84127901, 83155442, 52261574, 46870723, 71657078, 94076128, 30163921, 74743862, 68694897, 9398733, 67124282, 99125126, 88904910, 91281584, 526217, 43506672, 39553046, 92530431, 15148031, 26664538, 69136837, 59109400, 83651211, 35512853, 9058407, 76540481, 70800879, 27411561, 79942022, 97940276, 26139110, 75543508, 4434662, 45667668, 97281847, 29516592, 19891772, 92215320, 69641513, 98462867, 96852321, 71965942, 38510840, 90013093, 17894977, 35092039, 90457870, 88251446, 99658235, 10961421, 54762643, 62357986, 8055981, 3880712, 53842979, 57248122, 55602660, 78602717, 20568363, 75052463, 38494874, 1936762, 14093520, 83150534, 61271144, 36933038, 26392416, 48675329, 28734791, 6157724, 10309525, 32250045, 30764367, 26734892, 17857111, 70596786, 78884452, 21070110, 20486294, 35456853, 68110330, 38061439, 30395570, 70004753, 93790285, 75135448, 55753905, 30569392, 39801351, 65275241, 29959549, 37675718, 33553959, 42967683, 43933006, 89804152, 58180653, 40197395, 66271566, 34698463, 16019925, 13468268, 55615885, 65081429, 84002370, 415901, 38365584, 77620120, 57163802, 85116755, 98371444, 42692881, 81172706, 37620363, 37501808, 63152504, 67281495, 17146629, 39373729, 59329511, 5970607, 38022834, 55143724, 97379790, 41481685, 68316156, 33935899, 49271185, 7687278, 86798033, 66428795, 63015256, 9257405, 45919976, 8733068, 12024238, 37435892, 86242799, 59501487, 11757872, 74452589, 54606384, 89699445, 84187166, 12664567, 57240218, 57803235, 6986898, 82779622, 21397057, 29819940, 30771409, 54058788, 99971982, 65851721, 86821229, 69697787, 89637706, 72358170, 44627776, 36812683, 36753250, 45049308, 77300457, 95251277, 65038678, 32274392, 45075471, 31904591, 32161669, 82327024, 36139942, 92803766, 40677414, 72725103, 45407418, 4095116, 33797252, 26493119, 90272749, 8634541, 57241521, 94090109, 84684495, 21001913, 40865610, 28550822, 75153252, 53084256, 44664587, 36312813, 95581843, 54199160, 79657802, 96735716, 42782652, 2208785, 4515343, 78549759, 59981773, 21919959, 67084351, 91990218, 13862149, 36189527, 6497830, 44915235, 4978543, 9860195, 65454636, 53802686, 22997281, 62430984, 89996536, 81853704, 16595436, 6153406, 53393358, 85695762, 21993752, 53648154, 22879907, 30891921, 44844121, 15902805, 86460488, 3773993, 54263880, 70727211, 31733363, 37224844, 9575954, 69255765, 75407347, 62803858, 33123618, 47887470, 6111563, 7300047, 82052050, 99729357, 48260151, 54427233, 80014588, 84293052, 73617245, 46851987, 62936963, 61741594, 92692283, 39847321, 71300104, 32426752, 41245325, 86118021, 4099191, 41442762, 34044787, 29466406, 43376279, 96726697, 31126490, 16054533, 81774825, 4091162, 20122224, 40781449, 28851716, 71920426, 24314885, 44479073, 29401781, 72777973, 29994197, 60393039, 24953498, 55247455, 67474219, 77284619, 73021291, 824872, 99226875, 56424103, 57658654, 36930650, 41092102, 37166608, 17037369, 67513640, 45381876, 8128637, 17386996, 17081350, 82726008, 99224392, 4064751, 5832946, 79821897, 59318837, 40152546, 16380211, 10597197, 4985896, 14220886, 6521313, 94911072, 57020481, 16387575, 18833224, 74614639, 61960400, 54517921, 44481640, 61859581, 9599614, 76825057, 90158785, 59371804, 80316608, 45790169, 18001617, 30139692, 78218845, 57961282, 19939935, 25842078, 17957593, 33304202, 84840549, 49882705, 28928175, 63372756, 6221471, 66667729, 26776922, 79922758, 23569917, 98943869, 60106217, 89419466, 95010552, 42237907, 49236559, 64848072, 59405277, 11161731, 77272628, 14723410, 69848388, 53547802, 24058273, 73222868, 82178706, 8807940, 30787683, 61373987, 99861373, 88130087, 3235882, 66116458, 20765474, 42644903, 75777973, 21289531, 77301523, 60102965, 36685023, 42307449, 66322959, 51507425, 79880247, 62740044, 72614359, 78909561, 7955293, 41380093, 72738685, 9188443, 12303248, 5073754, 71469330, 6505939, 7066775, 99965001, 33699435, 71316369, 95957797, 77787724, 32058615, 72732205, 37659250, 92692978, 27665211, 18699206, 65047700, 72373496, 78300864, 40781854, 66319530, 72357096, 55669657, 87710366, 68939068, 39986008, 15536795, 76330843, 22129328, 46540998, 65017137, 9886593, 79191827, 68824981, 5267545, 19376156, 84349107, 91240048, 90310261, 8115266, 17764950, 20642888, 22405120, 94516935, 97561745, 13173644, 69579137, 72274002, 8099994, 51715482, 26998766, 58749, 91141395, 34698428, 88653118, 58224549, 49328608, 82427263, 3183975, 8651647, 42947632, 55470718, 54014062, 10358899, 749283, 20885148, 18466635, 27325428, 68128438, 94539824, 59614746, 67227442, 3633375, 69605283, 8729146, 22108665, 62552524, 45990383, 43045786, 95395112, 55850790, 89217461, 48088883, 26275734, 17058722, 30653863, 37739481, 61815223, 9175338, 51047803, 99690194, 18663507, 2204165, 25636669, 18131876, 82651278, 69355476, 65074535, 56153345, 87720882, 39201414, 16567550, 95726235, 99524975, 56484900, 83083131, 20002147, 20140249, 62496012, 66663942, 38256849, 50702367, 44060493, 34946859, 97057187, 66832478, 53888755, 44550764, 56531125, 321665, 44846932, 83368048, 10366309, 99549373, 91727510, 2891150, 22435353, 21533347, 3233084, 83133790, 75128745, 9623492, 6038457, 91831487, 36468541, 30366150, 84406788, 51472275, 93515664, 34493392, 34236719, 6525948, 12416768, 78589145, 63059024, 10649306, 98739783, 13819358, 3233569, 30543215, 4798568, 35192533, 83948335, 32590267, 75986488, 92604458, 71522968, 51464002, 18504197, 51426311, 56461322, 7182642, 24915585, 79806380, 85571389, 18806856, 6819644, 3294781, 96193415, 83269727, 99333375, 47738185, 40534591, 32151165, 6871053, 54987042, 44983451, 90090964, 62693428, 62452034, 29065964, 51588015, 83533741, 68041839, 44348328, 68702632, 7229550, 37280276, 4199704, 15075176, 15015906, 80246713, 64087743, 83789391, 44784505, 8129978, 31161687, 42199455, 85117092, 36505482, 63967300, 70541760, 33237508, 29834512, 2607799, 27625735, 40356877, 74441448, 36780454, 45848907, 7502255, 37891451, 669105, 99917599, 60176618, 92787493, 16445503, 93270084, 28796059, 9860968, 32159704, 20867149, 87598888, 60581278, 98653983, 76671482, 33249630, 8791066, 88047921, 14363867, 24733232, 66137019, 65880522, 16099750, 59177718, 44889423, 74357852, 76315420, 1788101, 4668450, 27187213 +23432750, 84187166, 68939068, 67513640, 30139692, 21919959, 48360198, 61928316, 48260151, 75986488, 71316369, 59910624, 54987042, 82897371, 94911072, 8099994, 75153252, 81274566, 89996536, 77377183, 82532312, 63015256, 11581181, 77413012, 17386996, 4787945, 63372756, 53842979, 68816413, 95010552, 67451935, 37957788, 15948937, 84166196, 53547802, 65880522, 68875490, 92554120, 8913721, 84293052, 55615885, 92692283, 85116755, 41245325, 70541760, 68644627, 16971929, 28851716, 24314885, 91938191, 7687278, 92033260, 85571389, 8733068, 62762857, 54606384, 82726008, 66832478, 92353856, 99125126, 88904910, 91240048, 22405120, 70800879, 69579137, 51715482, 93359396, 66611759, 62357986, 26776922, 23569917, 75052463, 93515664, 10309525, 24591705, 67227442, 32699744, 176257, 55753905, 76434144, 64157906, 68000591, 63059024, 73168565, 569864, 37183543, 93933709, 94360702, 39171895, 63506281, 72614359, 415901, 23110625, 34295794, 69786756, 59177718, 43322743, 29029316, 44889423, 47708850, 71522968, 71469330, 50007421, 4099191, 58751351, 29466406, 18131876, 97379790, 61859143, 49271185, 65047700, 76960303, 33565483, 26744917, 39986008, 84127901, 46870723, 4064751, 79821897, 37891451, 97641116, 669105, 69697787, 2917920, 67124282, 72358170, 61141874, 45049308, 74614639, 71083565, 54517921, 45075471, 44481640, 15148031, 92803766, 83210802, 76825057, 4806458, 45407418, 92787493, 16405341, 33336148, 45667668, 8634541, 57241521, 80251430, 98462867, 88698958, 26863229, 54762643, 54199160, 57248122, 79922758, 81805959, 42947632, 9829782, 90061527, 54663246, 1197320, 19898053, 8807940, 1569515, 99861373, 12571310, 64602895, 22108665, 52613508, 62051033, 77312810, 58180653, 66271566, 874791, 4798568, 36845587, 19101477, 15535065, 29188588, 29635537, 70367851, 36808486, 63152504, 51464002, 56461322, 85711894, 32058615, 68316156, 30811010, 71920426, 79806380, 1204161, 50668599, 16567550, 43152977, 1431742, 66319530, 72357096, 66663942, 70420215, 9603598, 36930650, 97783876, 38256849, 56955985, 8128637, 26292919, 30771409, 7502255, 32151165, 34432810, 61897582, 91255408, 6521313, 44550764, 89637706, 62452034, 19457589, 16387575, 65038678, 23848565, 5267545, 14947650, 26139110, 59371804, 97561745, 44473167, 94090109, 58208470, 87160386, 26998766, 78785507, 3880712, 49328608, 82427263, 33628349, 9860968, 89419466, 70782102, 67084351, 30366150, 54014062, 64848072, 47893286, 17976208, 15075176, 18466635, 27325428, 72495719, 68128438, 15163258, 69848388, 26734892, 20836893, 61712234, 24058273, 70596786, 69605283, 38061439, 93790285, 15064655, 76170907, 88130087, 13348726, 47213183, 78561158, 42644903, 38341669, 75777973, 1022677, 33553959, 82979980, 53666583, 55189057, 42307449, 66322959, 84002370, 36505482, 91957544, 77620120, 92541302, 45428665, 7685448, 71300104, 33249630, 66885828, 99965001, 56473732, 47090124, 34044787, 27041967, 31126490, 14045383, 16054533, 16424199, 41481685, 81774825, 4119747, 69355476, 18699206, 86798033, 66428795, 72238278, 40356877, 45919976, 83302115, 47298834, 1375023, 99524975, 56484900, 74441448, 4668450, 14731700, 67030811, 99226875, 34497327, 86022504, 87710366, 33201905, 99333375, 17037369, 5832946, 40534591, 247198, 73031054, 168541, 30163921, 4985896, 8696647, 62693428, 9886593, 61960400, 90310261, 60176618, 8115266, 13470059, 83651211, 27411561, 26493119, 45617087, 47361209, 13231279, 19939935, 17957593, 17894977, 76315420, 16445503, 68102437, 40865610, 28550822, 53084256, 88653118, 45237957, 44664587, 36312813, 28796059, 66045587, 78549759, 9259676, 36580610, 37280276, 65454636, 11161731, 22997281, 30764367, 59614746, 14723410, 81853704, 20486294, 35456853, 53648154, 82178706, 15902805, 38009615, 3773993, 89046466, 30787683, 20867149, 5482538, 75407347, 30569392, 47887470, 37675718, 43933006, 89804152, 82052050, 40197395, 30543215, 51149804, 26063929, 79880247, 61815223, 48658605, 112651, 92998591, 12303248, 42692881, 12348118, 2204165, 37620363, 37501808, 73109997, 18504197, 23740167, 18783702, 25636669, 17146629, 99297164, 33237508, 89811711, 96726697, 88047921, 7517032, 82651278, 57359924, 4091162, 20122224, 40781449, 90654836, 27625735, 19272365, 14363867, 50188404, 99021067, 23194618, 73786944, 18806856, 55247455, 83083131, 6819644, 30218878, 3294781, 20140249, 62496012, 56424103, 74452589, 24168411, 36780454, 62428472, 50567636, 45381876, 12664567, 36396314, 57803235, 6986898, 76330843, 14349098, 29819940, 59318837, 94076128, 53888755, 86821229, 11365791, 68694897, 321665, 65017137, 44627776, 29065964, 91281584, 526217, 77300457, 18833224, 83368048, 79191827, 93566986, 24733232, 32161669, 82327024, 9599614, 93057697, 36139942, 40677414, 59109400, 99549373, 20642888, 85242963, 83533741, 94516935, 93053405, 18001617, 90272749, 13173644, 19891772, 17539625, 57961282, 96852321, 35092039, 84840549, 99658235, 3487592, 11923835, 34698428, 65271999, 7104732, 93270084, 30998561, 96735716, 55602660, 62115552, 8651647, 85023028, 75229462, 36468541, 63628376, 49236559, 44915235, 6157724, 34667286, 77272628, 94539824, 32159704, 38008118, 16595436, 3633375, 73392814, 89499542, 21993752, 21070110, 62232211, 62490109, 80246713, 40984766, 68110330, 12416768, 54263880, 70004753, 83789391, 59197747, 67031644, 30090481, 45990383, 44784505, 3235882, 92071990, 69255765, 66116458, 7423788, 39801351, 29932657, 60581278, 33123618, 21289531, 17016156, 61263068, 23134715, 42967683, 7300047, 60102965, 48088883, 4204661, 72793444, 98653983, 76671482, 11543098, 85117092, 76703609, 99729357, 54427233, 80014588, 73617245, 84904436, 61741594, 48892201, 83948335, 38365584, 41380093, 39847321, 6793819, 9175338, 89214616, 96906410, 99690194, 32426752, 63967300, 27187213, 44245960, 7646095, 6505939, 79136082, 83747892, 67281495, 86118021, 71333116, 43376279, 78766358, 77898274, 90004325, 37659250, 54232247, 27665211, 33935899, 44479073, 68204242, 29401781, 72373496, 95726235, 51466049, 94711842, 24953498, 64055960, 73021291, 5822202, 96193415, 11757872, 41092102, 83269727, 53632373, 57240218, 83155442, 17081350, 17727650, 47738185, 82779622, 96318094, 6871053, 45995325, 10597197, 97057187, 65851721, 44983451, 14220886, 2717150, 48893685, 56515456, 5111370, 56531125, 36812683, 57020481, 43506672, 68824981, 61859581, 44842615, 10366309, 84349107, 9058407, 48673079, 4095116, 79942022, 66137019, 80316608, 45790169, 97281847, 8373289, 33431960, 75820087, 84684495, 3233084, 25842078, 83133790, 27185644, 33304202, 72274002, 17068582, 90457870, 10961421, 3088684, 66250369, 66667729, 28787861, 33895336, 42782652, 508198, 2208785, 96420429, 7229550, 78602717, 98943869, 60106217, 47792865, 14326617, 83150534, 26397786, 26392416, 91990218, 4199704, 6497830, 9860195, 92398073, 81677380, 15015906, 65715134, 65338021, 6153406, 45794415, 30891921, 44844121, 86460488, 33061250, 87598888, 79738755, 40027975, 24826575, 78589145, 9575954, 98739783, 95395112, 20765474, 65275241, 92867155, 86543538, 6111563, 26275734, 17058722, 51507425, 30653863, 62936963, 78909561, 37739481, 30463802, 7955293, 1239555, 73124510, 9188443, 51047803, 22766820, 81172706, 18663507, 7066775, 51426311, 55960386, 7011964, 8791066, 39373729, 38022834, 95957797, 29834512, 91664334, 74357852, 7182642, 49641577, 72732205, 74110882, 65074535, 4069912, 16684829, 29994197, 39201414, 12024238, 31491938, 59501487, 77072625, 79322415, 31727379, 45848907, 89078848, 50702367, 40686254, 96709982, 99515901, 99224392, 72278539, 90090964, 9398733, 43501211, 38645117, 72725103, 35512853, 51588015, 35996293, 97940276, 22435353, 49598724, 68041839, 21533347, 44348328, 69641513, 86001008, 71965942, 43357947, 28928175, 61623915, 75128745, 91141395, 58224549, 95581843, 79657802, 4515343, 60430369, 6038457, 74137926, 30694952, 38494874, 1936762, 14093520, 1788101, 13862149, 84406788, 10358899, 22200378, 70036438, 36189527, 20885148, 32250045, 17365188, 64098930, 6525948, 20681921, 78884452, 64087743, 70727211, 31733363, 19486173, 8129978, 5640302, 62803858, 61728685, 55850790, 36685023, 52293995, 62740044, 44847298, 80555751, 32590267, 37560164, 98371444, 5073754, 77694700, 93562257, 4508700, 77787724, 55143724, 52060076, 24915585, 56153345, 9257405, 10264691, 37435892, 40781854, 67474219, 77284619, 57658654, 37166608, 37192445, 15536795, 17385531, 1250437, 52261574, 22129328, 48774913, 34946859, 54058788, 10453030, 16380211, 99971982, 50806615, 22721500, 74743862, 44846932, 36753250, 95251277, 31904591, 26664538, 19376156, 69136837, 17764950, 76540481, 26102057, 91727510, 75543508, 4434662, 33797252, 78218845, 81855445, 92215320, 3509435, 38510840, 21001913, 72019362, 8055981, 51315460, 66903004, 91831487, 59981773, 95290172, 88444207, 42237907, 749283, 48675329, 4978543, 53802686, 51590803, 97011160, 14626618, 34236719, 98130363, 7919588, 55770687, 81898046, 22942635, 21673260, 24619760, 37224844, 8729146, 10649306, 13819358, 31161687, 29959549, 3233569, 42199455, 16019925, 13468268, 65081429, 2331773, 26507214, 49597667, 57163802, 16099750, 92604458, 70372191, 42073124, 74724075, 6808825, 33699435, 41442762, 22113133, 52204879, 87720882, 72777973, 78300864, 86242799, 55669657, 44060493, 71657078, 46540998, 40152546, 45306070, 39553046, 92530431, 6221471, 48395186, 73235980, 9623492, 68702632, 26114953, 20568363, 55470718, 36933038, 51472275, 28734791, 34493392, 17857111, 53393358, 22879907, 99604946, 30395570, 61373987, 75135448, 98648327, 62552524, 86301513, 77301523, 35192533, 18411915, 54868730, 36135, 60393039, 20645197, 824872, 61982238, 89699445, 45996863, 21397057, 67793644, 32274392, 99917599, 82339363, 90158785, 41092172, 2891150, 90013093, 22450468, 49882705, 88251446, 61271144, 59405277, 62430984, 73222868, 44177011, 43045786, 82886381, 89217461, 16097038, 72738685, 59329511, 60955663, 77187825, 94595668, 29516592, 58749, 91802888, 85695762, 38658347, 34698463, 2607799, 92692978, 20002147, 29510992, 3183975, 46851987, 5970607, 98948034, 57393458 +48658605, 47090124, 68644627, 91938191, 96318094, 14093520, 67031644, 42967683, 55189057, 71469330, 7011964, 95957797, 4119747, 68204242, 64055960, 34497327, 11581181, 7502255, 6521313, 48893685, 71083565, 44842615, 85242963, 44664587, 63628376, 78561158, 43933006, 92541302, 29635537, 85116755, 71300104, 66885828, 45919976, 51466049, 20140249, 55669657, 77413012, 96709982, 99515901, 91255408, 32274392, 23432750, 79942022, 80251430, 47361209, 28550822, 9623492, 53842979, 26776922, 42947632, 49236559, 27325428, 20836893, 78884452, 80246713, 48360198, 13819358, 65275241, 86543538, 26063929, 2331773, 9188443, 12348118, 74724075, 27041967, 41481685, 69355476, 86242799, 67030811, 62496012, 76330843, 14349098, 40534591, 79821897, 61859581, 33797252, 69579137, 98462867, 17068582, 61623915, 54199160, 81805959, 21919959, 95010552, 64848072, 9829782, 69605283, 38658347, 73222868, 21673260, 8807940, 87598888, 37224844, 15064655, 68000591, 92554120, 60581278, 65081429, 61815223, 43322743, 18663507, 70541760, 56473732, 33699435, 99297164, 58751351, 16054533, 61859143, 52060076, 4091162, 18699206, 65047700, 14363867, 56153345, 72373496, 16567550, 1375023, 99524975, 61982238, 56424103, 70420215, 54606384, 33201905, 6871053, 94595668, 97641116, 14220886, 65017137, 61141874, 44627776, 36753250, 18833224, 31904591, 84349107, 92803766, 27411561, 35996293, 14947650, 68041839, 45617087, 13173644, 58208470, 26863229, 17957593, 72274002, 93359396, 88251446, 40865610, 53084256, 58749, 11923835, 34698428, 45237957, 3880712, 82427263, 30694952, 75052463, 89419466, 36933038, 30366150, 51472275, 10309525, 18466635, 34493392, 89996536, 67227442, 24058273, 6153406, 21993752, 82178706, 22879907, 30891921, 70004753, 8729146, 77377183, 22108665, 61928316, 44784505, 63059024, 98739783, 569864, 33553959, 94360702, 26275734, 36685023, 26507214, 44847298, 80555751, 415901, 34295794, 45428665, 54868730, 81172706, 37620363, 93562257, 99965001, 86118021, 38022834, 77787724, 18131876, 68316156, 57359924, 82532312, 33935899, 72238278, 39201414, 43152977, 30218878, 73021291, 66663942, 41092102, 68939068, 67513640, 44060493, 59318837, 247198, 10597197, 97057187, 669105, 44550764, 11365791, 92353856, 8696647, 67124282, 321665, 74614639, 61960400, 44481640, 15148031, 24733232, 82327024, 93057697, 69136837, 83210802, 76825057, 13470059, 4806458, 45407418, 92787493, 41092172, 91727510, 26139110, 59371804, 45667668, 90272749, 44473167, 57241521, 94090109, 78218845, 19891772, 81855445, 86001008, 25842078, 71965942, 38510840, 21001913, 84840549, 3487592, 10961421, 93270084, 42782652, 79922758, 91802888, 26114953, 23569917, 33628349, 51315460, 91831487, 47792865, 95290172, 36468541, 70782102, 26397786, 67084351, 57393458, 10358899, 47893286, 67451935, 37957788, 51590803, 68128438, 15163258, 30764367, 38008118, 14723410, 3633375, 53547802, 65338021, 55770687, 21070110, 62232211, 40984766, 12416768, 86460488, 30395570, 83789391, 19486173, 30090481, 52613508, 5640302, 95395112, 62803858, 75777973, 21289531, 23134715, 3233569, 99729357, 16019925, 73617245, 78909561, 83948335, 23110625, 72738685, 6793819, 57163802, 32426752, 77694700, 70367851, 2204165, 7646095, 73109997, 23740167, 50007421, 33237508, 91664334, 77898274, 30811010, 86798033, 44479073, 50188404, 10264691, 83302115, 24953498, 96193415, 62762857, 24168411, 87710366, 84187166, 39986008, 8128637, 36396314, 34946859, 67793644, 90090964, 74743862, 82897371, 68694897, 62693428, 56531125, 44846932, 43501211, 93566986, 82339363, 68824981, 26664538, 4787945, 35512853, 20642888, 16405341, 83533741, 66137019, 75543508, 80316608, 8634541, 33431960, 29510992, 21533347, 84684495, 3233084, 83133790, 90013093, 78785507, 7104732, 8055981, 95581843, 28787861, 4515343, 57248122, 3183975, 74137926, 78602717, 9259676, 38494874, 98943869, 1936762, 1788101, 11161731, 72495719, 1197320, 61712234, 15015906, 20681921, 65715134, 19898053, 73392814, 35456853, 81898046, 44844121, 62490109, 24619760, 70727211, 5482538, 88130087, 76434144, 13348726, 30569392, 86301513, 7423788, 10649306, 38341669, 33123618, 47887470, 7300047, 60102965, 4204661, 53666583, 11543098, 34698463, 63506281, 13468268, 54427233, 874791, 84904436, 84002370, 46851987, 62936963, 38365584, 37560164, 92692283, 98371444, 92998591, 33249630, 47708850, 51464002, 18504197, 7066775, 55960386, 41442762, 71333116, 29466406, 55143724, 85711894, 14045383, 97379790, 90004325, 92692978, 79806380, 65074535, 23194618, 1431742, 20002147, 5822202, 57658654, 97783876, 83269727, 31727379, 50567636, 45996863, 57240218, 17385531, 17727650, 48774913, 71657078, 61897582, 54987042, 22721500, 72358170, 29065964, 43506672, 54517921, 45075471, 79191827, 5267545, 90310261, 59109400, 60176618, 17764950, 48673079, 76540481, 94516935, 45790169, 93053405, 33336148, 30139692, 8373289, 75820087, 33304202, 17894977, 90457870, 51715482, 87160386, 99658235, 63372756, 62357986, 73235980, 49328608, 96420429, 6038457, 66903004, 9860968, 68816413, 42237907, 91990218, 84406788, 6497830, 28734791, 9860195, 59405277, 32250045, 17365188, 22997281, 14626618, 59614746, 34236719, 69848388, 98130363, 32699744, 70596786, 85695762, 20486294, 15902805, 38009615, 44177011, 30787683, 99861373, 79738755, 12571310, 75135448, 78589145, 45990383, 92071990, 66116458, 39801351, 82886381, 82979980, 37183543, 89804152, 72793444, 85117092, 8913721, 66322959, 84293052, 36845587, 19101477, 91957544, 32590267, 29188588, 39847321, 51047803, 92604458, 99690194, 63967300, 5073754, 44245960, 71522968, 37501808, 36808486, 63152504, 83747892, 25636669, 39373729, 89811711, 71316369, 59329511, 29834512, 43376279, 96726697, 88047921, 16424199, 49641577, 7517032, 82651278, 20122224, 40781449, 37659250, 71920426, 27665211, 7687278, 92033260, 16684829, 87720882, 50668599, 9257405, 85571389, 29994197, 95726235, 8733068, 94711842, 60393039, 31491938, 4668450, 14731700, 66319530, 3294781, 9603598, 86022504, 62428472, 84127901, 6986898, 83155442, 4064751, 30771409, 37891451, 34432810, 65851721, 73031054, 53888755, 86821229, 4985896, 2917920, 9398733, 45306070, 19457589, 91281584, 45049308, 39553046, 10366309, 19376156, 38645117, 8115266, 90158785, 72725103, 51588015, 99549373, 22405120, 4095116, 70800879, 26102057, 2891150, 4434662, 18001617, 29516592, 69641513, 88698958, 96852321, 27185644, 35092039, 76315420, 72019362, 28928175, 65271999, 58224549, 3088684, 36312813, 96735716, 508198, 7229550, 8651647, 81274566, 14326617, 13862149, 36189527, 17976208, 94539824, 84166196, 54663246, 6525948, 24591705, 16595436, 89499542, 99604946, 54263880, 20867149, 31733363, 76170907, 59197747, 62552524, 9575954, 20765474, 31161687, 55850790, 29932657, 29959549, 1022677, 61263068, 37675718, 77312810, 93933709, 6111563, 48088883, 40197395, 51149804, 48260151, 80014588, 79880247, 72614359, 36505482, 61741594, 73124510, 41380093, 77620120, 15535065, 96906410, 70372191, 12303248, 44889423, 41245325, 18783702, 4099191, 17146629, 4508700, 5970607, 52204879, 31126490, 7182642, 72732205, 24314885, 49271185, 1204161, 19272365, 66428795, 63015256, 99021067, 74441448, 72357096, 824872, 59501487, 99226875, 99333375, 33565483, 26292919, 17386996, 40686254, 22129328, 99224392, 47738185, 29819940, 99971982, 50806615, 2717150, 89637706, 88904910, 57020481, 65038678, 92530431, 99917599, 23848565, 91240048, 40677414, 9058407, 97561745, 22450468, 75153252, 75128745, 91141395, 66250369, 66667729, 28796059, 79657802, 60106217, 55470718, 75229462, 83150534, 88444207, 26392416, 48675329, 70036438, 4199704, 92398073, 15948937, 90061527, 77272628, 97011160, 81677380, 62430984, 32159704, 64098930, 17857111, 7919588, 64087743, 38061439, 33061250, 176257, 93790285, 98648327, 47213183, 8129978, 68875490, 42644903, 17016156, 39171895, 98653983, 58180653, 76671482, 66271566, 52293995, 76703609, 55615885, 30653863, 35192533, 48892201, 7685448, 89214616, 69786756, 42073124, 112651, 29029316, 6505939, 79136082, 51426311, 59910624, 28851716, 27625735, 54232247, 4069912, 18806856, 47298834, 40781854, 79322415, 38256849, 37166608, 36780454, 45381876, 12664567, 21397057, 5832946, 46540998, 10453030, 94076128, 16380211, 168541, 30163921, 66832478, 69697787, 56515456, 94911072, 99125126, 526217, 95251277, 83368048, 32161669, 9599614, 36139942, 22435353, 97281847, 3509435, 19939935, 8099994, 16445503, 49882705, 66611759, 54762643, 88653118, 33895336, 66045587, 2208785, 55602660, 59981773, 36580610, 54014062, 37280276, 22200378, 15075176, 6157724, 20885148, 34667286, 26734892, 53393358, 22942635, 68110330, 3773993, 55753905, 24826575, 65880522, 64157906, 69255765, 75407347, 92867155, 73168565, 61728685, 89217461, 16097038, 17058722, 30543215, 42307449, 51507425, 49597667, 75986488, 59177718, 42692881, 27187213, 56461322, 8791066, 22113133, 16971929, 24915585, 29401781, 72777973, 73786944, 40356877, 56484900, 55247455, 20645197, 83083131, 6819644, 98948034, 74452589, 17037369, 26744917, 37192445, 15536795, 89078848, 52261574, 82726008, 32151165, 40152546, 5111370, 9886593, 16387575, 77300457, 97940276, 17539625, 92215320, 57961282, 26998766, 68102437, 48395186, 60430369, 62115552, 78549759, 20568363, 93515664, 4978543, 65454636, 89046466, 61373987, 3235882, 82052050, 62740044, 4798568, 37739481, 7955293, 9175338, 18411915, 67281495, 74357852, 78766358, 32058615, 81774825, 90654836, 60955663, 77284619, 11757872, 36930650, 89699445, 56955985, 45848907, 57803235, 1250437, 46870723, 44983451, 62452034, 49598724, 13231279, 6221471, 68702632, 85023028, 61271144, 749283, 81853704, 45794415, 1569515, 40027975, 64602895, 43045786, 62051033, 42199455, 30463802, 22766820, 6808825, 34044787, 2607799, 74110882, 12024238, 78300864, 77072625, 77187825, 50702367, 45995325, 83651211, 44348328, 43357947, 44915235, 53802686, 53648154, 1239555, 16099750, 36135, 37435892, 17081350, 82779622, 54058788, 72278539, 36812683, 26493119, 30998561, 77301523, 76960303, 67474219, 53632373 +36505482, 75052463, 42967683, 72738685, 70596786, 37224844, 39171895, 74357852, 86821229, 53084256, 15015906, 76434144, 44784505, 75407347, 77301523, 16019925, 23110625, 23740167, 95726235, 12024238, 62762857, 37192445, 67793644, 5111370, 9886593, 36139942, 4434662, 26493119, 76315420, 9259676, 55470718, 9860195, 68128438, 55770687, 43045786, 92554120, 38341669, 1022677, 52293995, 48892201, 9175338, 51426311, 4091162, 69355476, 1204161, 65047700, 20002147, 77187825, 62428472, 15536795, 59318837, 97057187, 95251277, 83368048, 16405341, 27185644, 78785507, 28550822, 79922758, 1788101, 36189527, 17857111, 3633375, 21070110, 20486294, 80246713, 54263880, 1569515, 13348726, 29959549, 61263068, 37183543, 4204661, 26275734, 17058722, 76703609, 54427233, 84002370, 4798568, 57163802, 4099191, 92692978, 7687278, 66428795, 24953498, 66319530, 96709982, 1250437, 44060493, 6871053, 168541, 72278539, 11365791, 88904910, 61960400, 82339363, 15148031, 61859581, 13470059, 26102057, 49598724, 30139692, 71965942, 38510840, 90013093, 75153252, 10961421, 34698428, 66667729, 66045587, 82427263, 81805959, 42947632, 77272628, 94539824, 59614746, 98130363, 7919588, 20836893, 73392814, 81898046, 61373987, 93790285, 76170907, 67031644, 78561158, 92071990, 5640302, 98653983, 42307449, 80014588, 44847298, 35192533, 37560164, 15535065, 29188588, 54868730, 70372191, 66885828, 7646095, 33699435, 18131876, 91664334, 31126490, 81774825, 57359924, 76960303, 29994197, 98948034, 83269727, 50567636, 82779622, 32151165, 34432810, 94911072, 44627776, 32274392, 31904591, 68824981, 32161669, 44842615, 94516935, 14947650, 97940276, 97281847, 29516592, 29510992, 17539625, 21533347, 88698958, 26863229, 25842078, 93359396, 87160386, 49882705, 99658235, 40865610, 61623915, 7104732, 4515343, 60430369, 91802888, 26114953, 60106217, 14093520, 75229462, 83150534, 70782102, 15075176, 17365188, 14723410, 69848388, 62490109, 55753905, 48360198, 5482538, 78589145, 77377183, 61928316, 45990383, 9575954, 98739783, 42644903, 77312810, 93933709, 51149804, 874791, 79880247, 73617245, 30653863, 46851987, 26507214, 51047803, 112651, 92998591, 12303248, 27187213, 29029316, 71469330, 37620363, 70541760, 8791066, 71316369, 29466406, 43376279, 68316156, 40781449, 27625735, 4119747, 18699206, 33935899, 65074535, 87720882, 85571389, 45919976, 18806856, 83302115, 99524975, 40781854, 72357096, 34497327, 86022504, 36780454, 39986008, 45848907, 57803235, 83155442, 4064751, 21397057, 46540998, 40152546, 16380211, 94595668, 30163921, 91255408, 2917920, 72358170, 16387575, 43506672, 44481640, 9599614, 10366309, 91240048, 40677414, 4787945, 99549373, 4806458, 41092172, 22405120, 48673079, 79942022, 26139110, 80316608, 68041839, 78218845, 13231279, 98462867, 17957593, 72274002, 48395186, 58224549, 8055981, 44664587, 93270084, 66250369, 36312813, 28787861, 3880712, 96735716, 51315460, 38494874, 8651647, 66903004, 59981773, 47792865, 26397786, 30366150, 49236559, 37280276, 749283, 67451935, 37957788, 10309525, 32250045, 14626618, 32159704, 30764367, 26734892, 81853704, 16595436, 65715134, 32699744, 38658347, 22879907, 30891921, 12416768, 20867149, 64602895, 88130087, 30569392, 39801351, 92867155, 62803858, 33123618, 37675718, 23134715, 7300047, 3233569, 72793444, 42199455, 55189057, 76671482, 8913721, 34698463, 48260151, 2331773, 62740044, 72614359, 34295794, 75986488, 45428665, 29635537, 48658605, 89214616, 33249630, 5073754, 81172706, 73109997, 86118021, 50007421, 56473732, 47090124, 25636669, 59329511, 2607799, 85711894, 97379790, 72732205, 82532312, 91938191, 92033260, 63015256, 56153345, 43152977, 14731700, 59501487, 57658654, 79322415, 97783876, 89699445, 99333375, 17037369, 68939068, 67513640, 77413012, 36396314, 6986898, 50702367, 52261574, 22129328, 99224392, 34946859, 37891451, 97641116, 66832478, 54987042, 44983451, 14220886, 82897371, 67124282, 89637706, 44846932, 36812683, 45049308, 526217, 65038678, 54517921, 92530431, 24733232, 26664538, 23848565, 84349107, 23432750, 70800879, 2891150, 93053405, 8634541, 57241521, 33431960, 80251430, 81855445, 92215320, 86001008, 35092039, 26998766, 68102437, 75128745, 58749, 26776922, 30998561, 74137926, 95010552, 9829782, 44915235, 72495719, 89996536, 54663246, 24591705, 61712234, 20681921, 53547802, 24058273, 65338021, 45794415, 53393358, 85695762, 21993752, 21673260, 8807940, 40984766, 99604946, 89046466, 70004753, 176257, 70727211, 79738755, 52613508, 8129978, 65275241, 31161687, 62051033, 29932657, 60581278, 89217461, 86543538, 43933006, 26063929, 65081429, 62936963, 37739481, 61815223, 19101477, 1239555, 38365584, 73124510, 41380093, 39847321, 16099750, 96906410, 43322743, 44245960, 63152504, 79136082, 18504197, 7066775, 55960386, 18783702, 7011964, 17146629, 68644627, 77787724, 29834512, 16971929, 96726697, 78766358, 7182642, 16424199, 27665211, 49271185, 86798033, 14363867, 16684829, 50668599, 68204242, 94711842, 60393039, 47298834, 37435892, 55247455, 86242799, 4668450, 30218878, 56424103, 96193415, 77072625, 11757872, 55669657, 26292919, 84127901, 76330843, 46870723, 48774913, 29819940, 79821897, 10597197, 99971982, 50806615, 22721500, 6521313, 48893685, 56515456, 62693428, 74614639, 79191827, 5267545, 19376156, 59109400, 60176618, 51588015, 92787493, 85242963, 66137019, 91727510, 33336148, 45667668, 13173644, 19891772, 21001913, 16445503, 22450468, 84840549, 62357986, 9623492, 28796059, 54199160, 33895336, 508198, 55602660, 6038457, 78602717, 30694952, 98943869, 68816413, 89419466, 36933038, 26392416, 84406788, 22200378, 4199704, 4978543, 17976208, 59405277, 65454636, 34667286, 27325428, 90061527, 34236719, 1197320, 19898053, 69605283, 15902805, 44177011, 87598888, 99861373, 12571310, 75135448, 59197747, 62552524, 47213183, 69255765, 68875490, 73168565, 55850790, 47887470, 569864, 6111563, 82052050, 36685023, 85117092, 99729357, 51507425, 84293052, 415901, 32590267, 92692283, 77620120, 6793819, 98371444, 71300104, 99690194, 69786756, 42073124, 63967300, 77694700, 44889423, 47708850, 2204165, 6808825, 58751351, 39373729, 33237508, 22113133, 52204879, 16054533, 20122224, 4069912, 23194618, 9257405, 16567550, 1375023, 74441448, 64055960, 1431742, 78300864, 67474219, 3294781, 61982238, 9603598, 37166608, 11581181, 45996863, 8128637, 17386996, 99515901, 71657078, 45995325, 247198, 73031054, 53888755, 2717150, 74743862, 68694897, 99125126, 91281584, 36753250, 43501211, 71083565, 83210802, 8115266, 35512853, 83533741, 27411561, 18001617, 90272749, 44473167, 8373289, 44348328, 69641513, 84684495, 19939935, 83133790, 33304202, 17894977, 43357947, 8099994, 90457870, 3487592, 63372756, 65271999, 6221471, 66611759, 95581843, 49328608, 62115552, 3183975, 78549759, 23569917, 33628349, 9860968, 91831487, 81274566, 14326617, 61271144, 88444207, 63628376, 64848072, 47893286, 51472275, 48675329, 28734791, 20885148, 53802686, 15948937, 51590803, 22997281, 62430984, 64098930, 6525948, 78884452, 53648154, 62232211, 86460488, 64087743, 15064655, 19486173, 24826575, 65880522, 3235882, 86301513, 63059024, 10649306, 75777973, 21289531, 33553959, 16097038, 48088883, 89804152, 40197395, 30543215, 13468268, 55615885, 36845587, 80555751, 61741594, 18411915, 32426752, 12348118, 93562257, 83747892, 41245325, 56461322, 99297164, 41442762, 34044787, 89811711, 71333116, 88047921, 55143724, 36135, 32058615, 7517032, 61859143, 52060076, 54232247, 24314885, 19272365, 44479073, 72238278, 31491938, 6819644, 67030811, 73021291, 62496012, 5822202, 66663942, 70420215, 74452589, 41092102, 33565483, 26744917, 12664567, 10453030, 96318094, 65851721, 61897582, 90090964, 44550764, 8696647, 9398733, 56531125, 321665, 29065964, 19457589, 77300457, 18833224, 45075471, 99917599, 82327024, 90310261, 38645117, 83651211, 72725103, 9058407, 35996293, 75543508, 22435353, 97561745, 94090109, 69579137, 47361209, 3509435, 3233084, 51715482, 72019362, 88251446, 91141395, 88653118, 3088684, 79657802, 42782652, 1936762, 21919959, 42237907, 36580610, 57393458, 54014062, 6497830, 92398073, 97011160, 38008118, 6153406, 89499542, 73222868, 68110330, 38009615, 30787683, 83789391, 31733363, 40027975, 68000591, 66116458, 95395112, 20765474, 61728685, 82886381, 60102965, 53666583, 66271566, 11543098, 66322959, 84904436, 30463802, 9188443, 7685448, 92604458, 42692881, 74724075, 37501808, 6505939, 51464002, 67281495, 99965001, 5970607, 38022834, 95957797, 27041967, 59910624, 41481685, 82651278, 24915585, 37659250, 99021067, 72777973, 39201414, 40356877, 10264691, 51466049, 8733068, 83083131, 77284619, 824872, 99226875, 38256849, 33201905, 84187166, 45381876, 57240218, 17385531, 82726008, 17727650, 5832946, 40534591, 94076128, 669105, 4985896, 61141874, 39553046, 93566986, 92803766, 90158785, 45407418, 17764950, 45617087, 58208470, 57961282, 11923835, 54762643, 68702632, 53842979, 20568363, 85023028, 13862149, 70036438, 11161731, 18466635, 81677380, 35456853, 22942635, 38061439, 33061250, 3773993, 30395570, 8729146, 64157906, 7423788, 94360702, 58180653, 63506281, 7955293, 83948335, 91957544, 49597667, 92541302, 85116755, 22766820, 71522968, 14045383, 90004325, 71920426, 79806380, 50188404, 29401781, 73786944, 56484900, 60955663, 36930650, 24168411, 87710366, 31727379, 17081350, 47738185, 30771409, 54058788, 69697787, 45306070, 62452034, 65017137, 57020481, 76825057, 20642888, 4095116, 76540481, 28928175, 45237957, 2208785, 96420429, 7229550, 67084351, 91990218, 93515664, 34493392, 84166196, 82178706, 24619760, 98648327, 22108665, 17016156, 82979980, 59177718, 70367851, 18663507, 72373496, 20645197, 54606384, 89078848, 53632373, 14349098, 7502255, 92353856, 93057697, 69136837, 59371804, 33797252, 75820087, 96852321, 73235980, 95290172, 36468541, 6157724, 44844121, 78909561, 4508700, 30811010, 90654836, 28851716, 56955985, 40686254, 45790169, 10358899, 15163258, 67227442, 30090481, 36808486, 49641577, 77898274, 74110882, 57248122, 13819358, 20140249, 17068582 +6038457, 91727510, 77272628, 91664334, 13819358, 37183543, 41481685, 91938191, 83651211, 28550822, 52613508, 47213183, 61263068, 7685448, 7646095, 8115266, 66611759, 9623492, 4515343, 74137926, 91990218, 49236559, 47893286, 44177011, 70004753, 64602895, 68000591, 42644903, 92867155, 1022677, 92692283, 29188588, 29029316, 37501808, 50668599, 1375023, 4668450, 57658654, 39986008, 96709982, 92353856, 36139942, 22435353, 8373289, 44348328, 33304202, 34698428, 96420429, 57248122, 8651647, 36933038, 67084351, 9860195, 6157724, 59614746, 6525948, 65715134, 80246713, 99604946, 176257, 93790285, 76434144, 8129978, 92554120, 55850790, 29932657, 72793444, 35192533, 16099750, 12303248, 4091162, 54232247, 10264691, 83083131, 66319530, 824872, 41092102, 33565483, 50702367, 37891451, 6521313, 90090964, 94911072, 77300457, 32161669, 93057697, 69136837, 93053405, 30139692, 71965942, 38510840, 8055981, 53842979, 60430369, 91802888, 51315460, 9259676, 59981773, 42237907, 36580610, 64848072, 22997281, 62430984, 84166196, 1197320, 20836893, 61712234, 32699744, 24058273, 53393358, 73222868, 44844121, 12416768, 86460488, 77377183, 43045786, 68875490, 65275241, 73168565, 77301523, 569864, 33553959, 48088883, 40197395, 54427233, 79880247, 80555751, 48892201, 1239555, 83948335, 77620120, 15535065, 70372191, 43322743, 66885828, 74724075, 18663507, 37620363, 7066775, 4099191, 17146629, 5970607, 77787724, 14363867, 50188404, 9257405, 83302115, 20645197, 5822202, 34497327, 77072625, 77413012, 1250437, 6871053, 97057187, 73031054, 30163921, 2717150, 44550764, 11365791, 9398733, 36753250, 65038678, 32274392, 31904591, 82339363, 38645117, 59371804, 80316608, 49598724, 45617087, 84684495, 19939935, 83133790, 21001913, 22450468, 78785507, 88251446, 40865610, 6221471, 44664587, 66250369, 79657802, 96735716, 79922758, 7229550, 62115552, 78549759, 60106217, 55470718, 83150534, 61271144, 13862149, 97011160, 14626618, 68128438, 45794415, 38658347, 64087743, 3773993, 92071990, 31161687, 33123618, 47887470, 17016156, 93933709, 23134715, 7300047, 30543215, 76671482, 52293995, 34698463, 16019925, 62740044, 6793819, 42692881, 77694700, 79136082, 18783702, 86118021, 7011964, 33237508, 59329511, 71333116, 52204879, 18131876, 96726697, 31126490, 88047921, 90654836, 76960303, 60393039, 43152977, 56484900, 64055960, 73021291, 3294781, 66663942, 61982238, 56424103, 77187825, 54606384, 62428472, 45848907, 45381876, 12664567, 82726008, 22129328, 4064751, 82779622, 34946859, 16380211, 72278539, 8696647, 68694897, 61141874, 71083565, 99917599, 44481640, 26664538, 60176618, 4787945, 35512853, 51588015, 4806458, 41092172, 70800879, 97940276, 4434662, 90272749, 29510992, 27185644, 90013093, 35092039, 72274002, 17068582, 51715482, 49882705, 75128745, 58749, 91141395, 66667729, 36312813, 30998561, 49328608, 2208785, 55602660, 23569917, 38494874, 9860968, 95290172, 95010552, 36468541, 1788101, 57393458, 84406788, 10358899, 51472275, 4199704, 67451935, 92398073, 20885148, 53802686, 89996536, 14723410, 69848388, 24591705, 15015906, 20681921, 70596786, 19898053, 76170907, 75135448, 48360198, 62552524, 9575954, 30569392, 62051033, 61728685, 42967683, 6111563, 89804152, 98653983, 42199455, 36685023, 8913721, 48260151, 30653863, 2331773, 46851987, 26507214, 78909561, 37739481, 38365584, 32590267, 39847321, 9188443, 71300104, 96906410, 112651, 63967300, 33249630, 22766820, 27187213, 70367851, 36808486, 63152504, 51464002, 99965001, 47090124, 58751351, 39373729, 89811711, 2607799, 43376279, 55143724, 85711894, 20122224, 40781449, 69355476, 27665211, 19272365, 29994197, 51466049, 94711842, 14731700, 77284619, 62496012, 96193415, 86022504, 24168411, 87710366, 83269727, 89699445, 17037369, 84187166, 50567636, 8128637, 17386996, 40686254, 17727650, 47738185, 29819940, 5832946, 30771409, 54058788, 168541, 66832478, 54987042, 22721500, 2917920, 19457589, 57020481, 45049308, 43501211, 45075471, 84349107, 91240048, 76825057, 13470059, 99549373, 20642888, 22405120, 29516592, 80251430, 78218845, 81855445, 92215320, 98462867, 17894977, 43357947, 87160386, 53084256, 10961421, 11923835, 63372756, 3088684, 28796059, 33895336, 26114953, 98943869, 75229462, 21919959, 22200378, 749283, 48675329, 44915235, 93515664, 4978543, 65454636, 11161731, 18466635, 34667286, 32250045, 34493392, 15163258, 54663246, 64098930, 7919588, 16595436, 3633375, 73392814, 89499542, 20486294, 30891921, 68110330, 54263880, 15064655, 19486173, 65880522, 67031644, 61928316, 75407347, 66116458, 98739783, 95395112, 20765474, 60581278, 82886381, 37675718, 3233569, 26275734, 58180653, 55189057, 11543098, 85117092, 76703609, 61741594, 415901, 91957544, 37560164, 29635537, 98371444, 89214616, 54868730, 32426752, 42073124, 47708850, 81172706, 71522968, 6505939, 67281495, 23740167, 56473732, 25636669, 22113133, 29834512, 14045383, 16054533, 16424199, 32058615, 77898274, 66428795, 92033260, 16684829, 72777973, 72238278, 72373496, 39201414, 73786944, 40356877, 16567550, 74441448, 1431742, 20002147, 6819644, 30218878, 72357096, 9603598, 36930650, 62762857, 99333375, 31727379, 45996863, 57803235, 46870723, 50806615, 86821229, 44983451, 91255408, 4985896, 14220886, 74743862, 5111370, 62693428, 62452034, 72358170, 9886593, 526217, 43506672, 39553046, 82327024, 61859581, 83210802, 90310261, 72725103, 48673079, 94516935, 44473167, 8634541, 57241521, 13231279, 57961282, 75820087, 88698958, 26863229, 8099994, 28928175, 65271999, 88653118, 68702632, 28787861, 82427263, 30694952, 75052463, 91831487, 85023028, 89419466, 42947632, 26392416, 30366150, 54014062, 70036438, 59405277, 10309525, 15948937, 51590803, 94539824, 32159704, 98130363, 26734892, 81853704, 67227442, 55770687, 8807940, 40984766, 30787683, 70727211, 31733363, 40027975, 5482538, 88130087, 64157906, 30090481, 98648327, 69255765, 29959549, 86543538, 16097038, 60102965, 17058722, 82052050, 66271566, 84293052, 4798568, 44847298, 62936963, 61815223, 73124510, 34295794, 99690194, 59177718, 44245960, 2204165, 73109997, 41245325, 50007421, 68644627, 38022834, 95957797, 29466406, 27041967, 74357852, 68316156, 82651278, 4119747, 79806380, 24314885, 7687278, 65047700, 65074535, 56153345, 99021067, 68204242, 23194618, 45919976, 95726235, 99524975, 78300864, 60955663, 40781854, 67474219, 59501487, 99226875, 70420215, 36780454, 37192445, 36396314, 84127901, 6986898, 17081350, 99515901, 48774913, 21397057, 40534591, 32151165, 79821897, 40152546, 99971982, 56515456, 56531125, 44846932, 88904910, 54517921, 93566986, 68824981, 9599614, 23432750, 16405341, 26102057, 79942022, 35996293, 26139110, 75543508, 33797252, 18001617, 33431960, 94090109, 47361209, 17539625, 96852321, 86001008, 76315420, 68102437, 75153252, 3487592, 54762643, 58224549, 73235980, 93270084, 54199160, 3183975, 33628349, 68816413, 14326617, 63628376, 36189527, 15075176, 30764367, 34236719, 53547802, 35456853, 82178706, 38061439, 30395570, 24619760, 61373987, 20867149, 79738755, 55753905, 13348726, 22108665, 78561158, 63059024, 38341669, 75777973, 21289531, 89217461, 53666583, 99729357, 63506281, 66322959, 55615885, 84002370, 36845587, 30463802, 19101477, 49597667, 9175338, 48658605, 92604458, 44889423, 71469330, 6808825, 70541760, 8791066, 41442762, 16971929, 78766358, 49641577, 81774825, 7517032, 61859143, 52060076, 57359924, 30811010, 27625735, 72732205, 37659250, 92692978, 18699206, 85571389, 37435892, 86242799, 67030811, 11757872, 97783876, 74452589, 37166608, 33201905, 11581181, 68939068, 56955985, 53632373, 52261574, 99224392, 71657078, 10453030, 96318094, 247198, 94076128, 94595668, 67793644, 53888755, 669105, 48893685, 69697787, 45306070, 99125126, 91281584, 92530431, 24733232, 44842615, 5267545, 17764950, 9058407, 83533741, 68041839, 45667668, 97281847, 3509435, 25842078, 17957593, 26998766, 45237957, 26776922, 66045587, 42782652, 20568363, 1936762, 66903004, 14093520, 26397786, 37957788, 27325428, 17365188, 65338021, 78884452, 85695762, 22879907, 81898046, 21673260, 62490109, 89046466, 1569515, 83789391, 87598888, 37224844, 12571310, 24826575, 78589145, 45990383, 86301513, 7423788, 5640302, 39801351, 77312810, 82979980, 43933006, 39171895, 4204661, 42307449, 26063929, 13468268, 73617245, 84904436, 36505482, 41380093, 23110625, 85116755, 69786756, 92998591, 18504197, 51426311, 56461322, 71316369, 97379790, 90004325, 71920426, 82532312, 1204161, 63015256, 29401781, 18806856, 8733068, 98948034, 38256849, 26744917, 67513640, 17385531, 44060493, 46540998, 67124282, 321665, 89637706, 29065964, 16387575, 18833224, 61960400, 83368048, 79191827, 15148031, 10366309, 19376156, 85242963, 66137019, 45790169, 26493119, 58208470, 69641513, 93359396, 72019362, 84840549, 61623915, 48395186, 508198, 70782102, 9829782, 6497830, 17976208, 38008118, 17857111, 62232211, 38009615, 8729146, 10649306, 51149804, 51507425, 874791, 65081429, 72614359, 72738685, 75986488, 45428665, 57163802, 5073754, 93562257, 83747892, 55960386, 4508700, 99297164, 24915585, 28851716, 74110882, 4069912, 44479073, 87720882, 47298834, 55247455, 15536795, 89078848, 57240218, 83155442, 14349098, 7502255, 59318837, 45995325, 10597197, 97641116, 82897371, 65017137, 44627776, 95251277, 23848565, 45407418, 27411561, 14947650, 21533347, 3233084, 16445503, 90457870, 99658235, 62357986, 7104732, 95581843, 3880712, 81805959, 88444207, 37280276, 28734791, 90061527, 81677380, 69605283, 21993752, 21070110, 53648154, 22942635, 15902805, 99861373, 3235882, 92541302, 51047803, 33699435, 34044787, 7182642, 33935899, 49271185, 12024238, 20140249, 55669657, 34432810, 65851721, 36812683, 74614639, 59109400, 90158785, 92787493, 4095116, 2891150, 33336148, 13173644, 69579137, 78602717, 81274566, 47792865, 6153406, 94360702, 7955293, 18411915, 59910624, 86798033, 79322415, 26292919, 92803766, 76540481, 33061250, 59197747, 44784505, 62803858, 12348118, 24953498, 76330843, 61897582, 40677414, 97561745, 72495719, 80014588, 36135, 31491938, 19891772 +17857111, 86022504, 91255408, 30139692, 1788101, 3633375, 99604946, 42967683, 74357852, 71920426, 5822202, 29065964, 9599614, 33431960, 28550822, 34698428, 74137926, 38494874, 15075176, 80246713, 22108665, 75407347, 77301523, 63506281, 37739481, 71316369, 45919976, 40781854, 89699445, 52261574, 67124282, 57961282, 27185644, 85023028, 83150534, 61928316, 98739783, 1022677, 37675718, 36505482, 29188588, 32426752, 92998591, 18663507, 25636669, 17146629, 55143724, 69355476, 50188404, 40356877, 95726235, 18806856, 1431742, 6871053, 37891451, 73031054, 44550764, 89637706, 36812683, 36753250, 95251277, 71083565, 83368048, 72725103, 17539625, 58224549, 54199160, 14093520, 57393458, 9829782, 89996536, 32699744, 53393358, 44177011, 20867149, 31733363, 76434144, 95395112, 82979980, 42199455, 51507425, 46851987, 32590267, 57163802, 70372191, 5073754, 63152504, 88047921, 27625735, 4119747, 94711842, 74441448, 56424103, 97783876, 99333375, 50702367, 7502255, 61897582, 669105, 65038678, 31904591, 24733232, 82327024, 19376156, 13470059, 41092172, 35996293, 2891150, 22435353, 68041839, 26493119, 8634541, 71965942, 95581843, 3880712, 26776922, 4515343, 66903004, 84406788, 49236559, 4199704, 69848388, 81853704, 1569515, 93790285, 76170907, 48360198, 45990383, 92071990, 55850790, 60581278, 33553959, 4204661, 53666583, 98653983, 73124510, 72738685, 6793819, 7685448, 92604458, 5970607, 27041967, 16424199, 54232247, 56153345, 29994197, 6819644, 62762857, 83269727, 39986008, 56955985, 44060493, 5832946, 247198, 54987042, 86821229, 5111370, 99125126, 57020481, 61960400, 15148031, 26664538, 44842615, 76825057, 48673079, 4434662, 97281847, 90272749, 29510992, 13231279, 93359396, 26998766, 65271999, 6221471, 7104732, 93270084, 33895336, 91802888, 55602660, 6038457, 54014062, 70036438, 93515664, 67451935, 59405277, 17365188, 59614746, 6525948, 20681921, 19898053, 21993752, 40984766, 24619760, 89046466, 30787683, 61373987, 75135448, 55753905, 69255765, 66116458, 86301513, 65275241, 38341669, 92867155, 29932657, 82886381, 89217461, 6111563, 39171895, 17058722, 30543215, 52293995, 99729357, 65081429, 62740044, 4798568, 44847298, 80555751, 75986488, 99690194, 44245960, 67281495, 4099191, 77787724, 71333116, 85711894, 27665211, 7687278, 65047700, 86798033, 39201414, 60393039, 43152977, 37435892, 14731700, 66319530, 67030811, 98948034, 30218878, 66663942, 96193415, 37166608, 45848907, 83155442, 99224392, 65851721, 168541, 53888755, 22721500, 14220886, 11365791, 62452034, 72358170, 9886593, 16387575, 54517921, 32274392, 79191827, 23848565, 83210802, 76540481, 26102057, 91727510, 93053405, 78218845, 88698958, 96852321, 86001008, 22450468, 49882705, 53084256, 88653118, 44664587, 49328608, 7229550, 78602717, 42947632, 55470718, 88444207, 30366150, 63628376, 749283, 36189527, 37957788, 14626618, 34236719, 20836893, 61712234, 65715134, 6153406, 70596786, 55770687, 21070110, 20486294, 35456853, 73222868, 82178706, 62232211, 15902805, 12416768, 70004753, 65880522, 77377183, 62552524, 78561158, 68875490, 10649306, 13819358, 31161687, 569864, 93933709, 23134715, 43933006, 3233569, 82052050, 76671482, 51149804, 48260151, 26063929, 16019925, 66322959, 30653863, 26507214, 62936963, 61815223, 48892201, 415901, 1239555, 77620120, 15535065, 45428665, 9175338, 85116755, 16099750, 48658605, 71300104, 54868730, 112651, 63967300, 66885828, 71469330, 6505939, 83747892, 7066775, 47090124, 89811711, 95957797, 18131876, 96726697, 14045383, 59910624, 41481685, 32058615, 52060076, 40781449, 90654836, 72732205, 74110882, 33935899, 44479073, 50668599, 72777973, 8733068, 12024238, 56484900, 78300864, 20002147, 77284619, 73021291, 72357096, 20140249, 62496012, 9603598, 55669657, 38256849, 33565483, 17037369, 8128637, 84127901, 96709982, 82726008, 29819940, 30771409, 71657078, 46540998, 34946859, 59318837, 97057187, 50806615, 97641116, 4985896, 61141874, 44627776, 526217, 43506672, 92530431, 40677414, 90158785, 4787945, 35512853, 92787493, 4095116, 66137019, 80316608, 13173644, 26863229, 19939935, 17957593, 21001913, 17894977, 35092039, 43357947, 87160386, 78785507, 99658235, 75128745, 45237957, 36312813, 57248122, 26114953, 91831487, 81274566, 68816413, 59981773, 36468541, 67084351, 4978543, 92398073, 10309525, 27325428, 68128438, 34493392, 15163258, 84166196, 38008118, 14723410, 78884452, 89499542, 69605283, 68110330, 70727211, 99861373, 40027975, 8729146, 12571310, 19486173, 24826575, 78589145, 9575954, 44784505, 7423788, 92554120, 62803858, 17016156, 61263068, 86543538, 94360702, 60102965, 26275734, 66271566, 34698463, 42307449, 76703609, 874791, 79880247, 84293052, 84904436, 2331773, 61741594, 38365584, 49597667, 42692881, 77694700, 27187213, 12348118, 44889423, 81172706, 71522968, 2204165, 7646095, 73109997, 18504197, 41245325, 51426311, 56473732, 4508700, 59329511, 16971929, 91664334, 49641577, 77898274, 57359924, 90004325, 37659250, 79806380, 18699206, 91938191, 72373496, 86242799, 4668450, 67474219, 824872, 59501487, 99226875, 36930650, 41092102, 50567636, 37192445, 45996863, 12664567, 36396314, 57240218, 17386996, 22129328, 17727650, 10453030, 67793644, 2717150, 45306070, 65017137, 39553046, 99917599, 44481640, 93057697, 45407418, 83533741, 27411561, 94516935, 59371804, 18001617, 69579137, 81855445, 38510840, 90013093, 72274002, 8099994, 72019362, 40865610, 61623915, 10961421, 66611759, 73235980, 28796059, 42782652, 60430369, 79922758, 33628349, 8651647, 21919959, 22200378, 64848072, 51472275, 48675329, 6497830, 65454636, 34667286, 97011160, 32159704, 98130363, 81898046, 44844121, 21673260, 54263880, 59197747, 5482538, 67031644, 30090481, 3235882, 47213183, 63059024, 5640302, 75777973, 47887470, 16097038, 48088883, 89804152, 58180653, 55189057, 40197395, 36685023, 85117092, 8913721, 55615885, 35192533, 83948335, 41380093, 9188443, 18411915, 98371444, 69786756, 33249630, 43322743, 29029316, 37620363, 93562257, 18783702, 86118021, 50007421, 56461322, 33699435, 8791066, 41442762, 33237508, 68644627, 29466406, 29834512, 2607799, 78766358, 31126490, 16054533, 81774825, 68316156, 7517032, 61859143, 82532312, 65074535, 76960303, 14363867, 4069912, 68204242, 83302115, 61982238, 77072625, 79322415, 74452589, 77187825, 87710366, 26744917, 77413012, 17081350, 4064751, 47738185, 48774913, 82779622, 40534591, 79821897, 45995325, 94076128, 16380211, 34432810, 99971982, 30163921, 44983451, 90090964, 92353856, 8696647, 44846932, 88904910, 77300457, 43501211, 93566986, 36139942, 91240048, 69136837, 90310261, 38645117, 51588015, 99549373, 4806458, 17764950, 85242963, 97940276, 75543508, 97561745, 44473167, 94090109, 19891772, 75820087, 44348328, 3233084, 83133790, 33304202, 76315420, 90457870, 51715482, 84840549, 28928175, 75153252, 58749, 91141395, 8055981, 3088684, 66667729, 68702632, 66045587, 96735716, 508198, 2208785, 96420429, 78549759, 9259676, 75052463, 9860968, 14326617, 26397786, 42237907, 47893286, 44915235, 28734791, 20885148, 18466635, 32250045, 51590803, 72495719, 77272628, 62430984, 54663246, 1197320, 7919588, 15015906, 67227442, 24058273, 85695762, 38658347, 22879907, 62490109, 38061439, 3773993, 83789391, 87598888, 37224844, 88130087, 43045786, 73168565, 33123618, 7300047, 72793444, 13468268, 54427233, 80014588, 36845587, 91957544, 89214616, 59177718, 74724075, 37501808, 51464002, 55960386, 38022834, 97379790, 36135, 82651278, 4091162, 24915585, 28851716, 92692978, 66428795, 63015256, 99021067, 87720882, 72238278, 85571389, 73786944, 31491938, 1375023, 99524975, 55247455, 20645197, 83083131, 3294781, 34497327, 70420215, 62428472, 11581181, 31727379, 68939068, 89078848, 53632373, 1250437, 46870723, 54058788, 32151165, 94595668, 66832478, 6521313, 82897371, 48893685, 2917920, 56515456, 56531125, 321665, 94911072, 45049308, 18833224, 10366309, 84349107, 59109400, 60176618, 8115266, 23432750, 83651211, 20642888, 14947650, 45790169, 33797252, 45667668, 47361209, 21533347, 3509435, 98462867, 3487592, 63372756, 54762643, 9623492, 53842979, 82427263, 23569917, 30694952, 1936762, 89419466, 70782102, 26392416, 36580610, 91990218, 37280276, 11161731, 15948937, 94539824, 64098930, 24591705, 16595436, 30891921, 8807940, 86460488, 64087743, 33061250, 30395570, 176257, 64602895, 8129978, 20765474, 29959549, 37183543, 73617245, 84002370, 72614359, 78909561, 30463802, 19101477, 37560164, 92541302, 34295794, 29635537, 51047803, 42073124, 12303248, 22766820, 70367851, 36808486, 79136082, 6808825, 70541760, 23740167, 39373729, 34044787, 22113133, 52204879, 20122224, 1204161, 92033260, 16567550, 47298834, 60955663, 57658654, 54606384, 24168411, 36780454, 67513640, 6986898, 40686254, 17385531, 76330843, 74743862, 68694897, 9398733, 62693428, 91281584, 82339363, 68824981, 32161669, 92803766, 16405341, 9058407, 70800879, 79942022, 49598724, 45617087, 58208470, 92215320, 84684495, 17068582, 11923835, 48395186, 66250369, 30998561, 62115552, 47792865, 95290172, 95010552, 61271144, 13862149, 53802686, 30764367, 26734892, 53547802, 53648154, 38009615, 79738755, 15064655, 64157906, 13348726, 68000591, 30569392, 39801351, 62051033, 61728685, 21289531, 7955293, 92692283, 39847321, 96906410, 47708850, 99965001, 99297164, 58751351, 7182642, 30811010, 24314885, 49271185, 23194618, 29401781, 9257405, 10264691, 51466049, 24953498, 11757872, 21397057, 96318094, 72278539, 69697787, 22405120, 26139110, 29516592, 8373289, 57241521, 80251430, 69641513, 16445503, 68102437, 88251446, 79657802, 81805959, 51315460, 98943869, 75229462, 10358899, 90061527, 73392814, 22942635, 52613508, 42644903, 11543098, 19272365, 33201905, 15536795, 26292919, 57803235, 19457589, 74614639, 45075471, 61859581, 5267545, 25842078, 28787861, 20568363, 60106217, 36933038, 9860195, 6157724, 22997281, 77312810, 23110625, 7011964, 43376279, 64055960, 84187166, 45381876, 99515901, 14349098, 40152546, 33336148, 62357986, 3183975, 81677380, 65338021, 98648327, 16684829, 17976208, 45794415, 10597197 +44784505, 7104732, 81898046, 5482538, 67031644, 86798033, 14220886, 54517921, 45407418, 45237957, 28734791, 20867149, 99861373, 86301513, 77301523, 4508700, 31126490, 7182642, 97783876, 17727650, 10453030, 44983451, 72278539, 14947650, 3509435, 88698958, 72019362, 21919959, 13862149, 30764367, 65338021, 69605283, 62552524, 92071990, 73617245, 65081429, 69786756, 59177718, 18504197, 8791066, 95957797, 16424199, 12024238, 40781854, 33565483, 46540998, 77300457, 13470059, 51588015, 92787493, 97561745, 57241521, 29510992, 49882705, 63372756, 48395186, 91831487, 64848072, 65454636, 34236719, 21673260, 88130087, 66116458, 7423788, 95395112, 62803858, 89217461, 33553959, 43933006, 48088883, 76671482, 32590267, 6793819, 85116755, 92604458, 12303248, 43322743, 82651278, 28851716, 27625735, 79806380, 65047700, 74441448, 4668450, 77072625, 55669657, 12664567, 57240218, 76330843, 79821897, 59318837, 54987042, 45306070, 89637706, 92530431, 93566986, 82339363, 75543508, 44473167, 17539625, 58208470, 13231279, 78785507, 61623915, 34698428, 79657802, 96735716, 57248122, 6038457, 26392416, 22200378, 47893286, 92398073, 15015906, 67227442, 6153406, 8807940, 78589145, 10649306, 37183543, 7300047, 42199455, 36685023, 26063929, 44847298, 36845587, 9175338, 51047803, 5073754, 71522968, 99297164, 33237508, 74357852, 88047921, 36135, 32058615, 81774825, 52060076, 57359924, 90004325, 4091162, 4119747, 27665211, 18699206, 33935899, 92033260, 76960303, 14731700, 62496012, 9603598, 99333375, 89078848, 40686254, 34946859, 247198, 94595668, 86821229, 44550764, 74743862, 69697787, 99125126, 88904910, 93057697, 5267545, 84349107, 91240048, 99549373, 20642888, 83533741, 27411561, 35996293, 33336148, 26493119, 80251430, 21533347, 21001913, 26998766, 88251446, 75128745, 91141395, 65271999, 58224549, 28787861, 3183975, 51315460, 38494874, 55470718, 36933038, 57393458, 49236559, 70036438, 93515664, 34667286, 77272628, 62430984, 94539824, 17857111, 20836893, 85695762, 35456853, 80246713, 24619760, 87598888, 79738755, 76170907, 19486173, 65880522, 22108665, 9575954, 30569392, 63059024, 92554120, 65275241, 29959549, 77312810, 94360702, 82052050, 42307449, 72614359, 78909561, 35192533, 61815223, 19101477, 1239555, 92692283, 92541302, 29188588, 48658605, 96906410, 92998591, 66885828, 70367851, 93562257, 79136082, 67281495, 51426311, 18783702, 41442762, 68644627, 91664334, 14045383, 54232247, 92692978, 69355476, 99021067, 29994197, 73786944, 18806856, 78300864, 98948034, 34497327, 62762857, 74452589, 38256849, 87710366, 83269727, 11581181, 26744917, 45848907, 14349098, 54058788, 96318094, 94076128, 34432810, 50806615, 669105, 2717150, 6521313, 67124282, 56515456, 5111370, 62452034, 16387575, 65038678, 71083565, 39553046, 31904591, 44481640, 61859581, 9599614, 36139942, 90158785, 17764950, 4095116, 76540481, 26102057, 79942022, 2891150, 59371804, 4434662, 33797252, 49598724, 45667668, 18001617, 86001008, 3233084, 38510840, 33304202, 53084256, 10961421, 54762643, 88653118, 8055981, 3880712, 33895336, 42782652, 91802888, 81805959, 23569917, 1936762, 68816413, 95010552, 61271144, 30366150, 10358899, 63628376, 37957788, 4978543, 9860195, 6157724, 53802686, 72495719, 22997281, 6525948, 7919588, 24591705, 81853704, 16595436, 3633375, 70596786, 19898053, 53648154, 22879907, 15902805, 30395570, 31733363, 12571310, 48360198, 64602895, 30090481, 68000591, 77377183, 45990383, 5640302, 86543538, 16097038, 39171895, 4204661, 3233569, 17058722, 58180653, 55189057, 99729357, 54427233, 30653863, 62740044, 62936963, 61741594, 83948335, 38365584, 49597667, 75986488, 99690194, 32426752, 63967300, 22766820, 44889423, 36808486, 63152504, 6505939, 83747892, 86118021, 4099191, 56461322, 47090124, 5970607, 85711894, 16054533, 49641577, 61859143, 91938191, 7687278, 1204161, 14363867, 4069912, 72373496, 16567550, 8733068, 94711842, 60393039, 37435892, 64055960, 1431742, 67030811, 99226875, 66663942, 41092102, 89699445, 39986008, 15536795, 84127901, 83155442, 17081350, 52261574, 47738185, 29819940, 5832946, 7502255, 37891451, 61897582, 53888755, 4985896, 48893685, 68694897, 2917920, 62693428, 56531125, 72358170, 91281584, 57020481, 526217, 95251277, 74614639, 32274392, 24733232, 26664538, 40677414, 76825057, 66137019, 30139692, 19891772, 17957593, 8099994, 40865610, 28550822, 75153252, 58749, 62357986, 73235980, 3088684, 54199160, 26776922, 66045587, 55602660, 78549759, 33628349, 85023028, 14326617, 95290172, 88444207, 67084351, 42237907, 36580610, 749283, 36189527, 67451935, 90061527, 81677380, 68128438, 59614746, 38008118, 64098930, 1197320, 21993752, 21070110, 20486294, 73222868, 62490109, 44177011, 1569515, 37224844, 15064655, 8729146, 75135448, 24826575, 61928316, 98739783, 42644903, 60581278, 17016156, 93933709, 23134715, 42967683, 60102965, 89804152, 52293995, 874791, 80014588, 79880247, 84293052, 84904436, 46851987, 80555751, 36505482, 415901, 37560164, 41380093, 29635537, 9188443, 7685448, 71300104, 112651, 12348118, 81172706, 2204165, 37620363, 70541760, 23740167, 99965001, 56473732, 33699435, 34044787, 89811711, 71316369, 71333116, 27041967, 43376279, 55143724, 59910624, 20122224, 37659250, 71920426, 19272365, 66428795, 16684829, 50668599, 72238278, 9257405, 51466049, 31491938, 99524975, 83083131, 60955663, 59501487, 61982238, 96193415, 86022504, 36780454, 17037369, 50567636, 56955985, 77413012, 57803235, 96709982, 17385531, 21397057, 44060493, 45995325, 65851721, 91255408, 9886593, 61141874, 44627776, 18833224, 43501211, 44842615, 69136837, 23432750, 83651211, 4787945, 9058407, 70800879, 80316608, 22435353, 45617087, 33431960, 69579137, 47361209, 57961282, 69641513, 26863229, 25842078, 71965942, 43357947, 51715482, 87160386, 99658235, 28928175, 11923835, 66250369, 66667729, 82427263, 96420429, 26114953, 98943869, 9860968, 47792865, 75229462, 26397786, 84406788, 51472275, 10309525, 15948937, 27325428, 51590803, 17365188, 14626618, 32159704, 89996536, 84166196, 54663246, 14723410, 69848388, 98130363, 78884452, 55770687, 82178706, 62232211, 44844121, 40984766, 12416768, 33061250, 3773993, 30787683, 93790285, 59197747, 3235882, 52613508, 75407347, 8129978, 68875490, 39801351, 13819358, 92867155, 75777973, 33123618, 61263068, 37675718, 53666583, 72793444, 98653983, 30543215, 11543098, 85117092, 63506281, 55615885, 26507214, 4798568, 30463802, 48892201, 91957544, 39847321, 57163802, 16099750, 18411915, 70372191, 33249630, 42692881, 47708850, 71469330, 41245325, 50007421, 7011964, 22113133, 38022834, 52204879, 78766358, 97379790, 41481685, 68316156, 24314885, 44479073, 50188404, 87720882, 85571389, 39201414, 83302115, 24953498, 55247455, 6819644, 3294781, 72357096, 20140249, 77187825, 54606384, 24168411, 62428472, 68939068, 37192445, 53632373, 48774913, 30771409, 97641116, 82897371, 94911072, 65017137, 44846932, 36812683, 29065964, 43506672, 68824981, 23848565, 92803766, 38645117, 60176618, 8115266, 4806458, 85242963, 94516935, 97940276, 45790169, 90272749, 94090109, 92215320, 44348328, 98462867, 96852321, 19939935, 27185644, 90013093, 35092039, 72274002, 22450468, 93359396, 44664587, 28796059, 53842979, 30998561, 79922758, 74137926, 30694952, 9259676, 75052463, 60106217, 59981773, 83150534, 91990218, 54014062, 9829782, 4199704, 18466635, 32250045, 97011160, 34493392, 53547802, 24058273, 38658347, 64087743, 54263880, 70004753, 83789391, 76434144, 13348726, 43045786, 20765474, 31161687, 55850790, 29932657, 21289531, 47887470, 40197395, 66271566, 16019925, 13468268, 66322959, 51507425, 37739481, 73124510, 77620120, 72738685, 89214616, 74724075, 44245960, 37501808, 51464002, 17146629, 29834512, 30811010, 49271185, 65074535, 56153345, 23194618, 29401781, 45919976, 95726235, 20645197, 77284619, 30218878, 824872, 79322415, 67513640, 45996863, 8128637, 36396314, 6986898, 50702367, 17386996, 99515901, 46870723, 71657078, 32151165, 10597197, 97057187, 67793644, 66832478, 90090964, 11365791, 92353856, 321665, 45049308, 61960400, 45075471, 79191827, 99917599, 15148031, 59109400, 72725103, 41092172, 91727510, 68041839, 97281847, 83133790, 6221471, 66611759, 93270084, 508198, 78602717, 66903004, 42947632, 14093520, 70782102, 44915235, 20885148, 15163258, 65715134, 45794415, 22942635, 30891921, 68110330, 61373987, 55753905, 64157906, 78561158, 69255765, 38341669, 73168565, 26275734, 8913721, 7955293, 23110625, 34295794, 15535065, 98371444, 54868730, 29029316, 18663507, 7646095, 55960386, 58751351, 59329511, 77787724, 16971929, 18131876, 2607799, 40781449, 90654836, 72777973, 10264691, 43152977, 56484900, 20002147, 66319530, 67474219, 56424103, 70420215, 33201905, 82726008, 4064751, 82779622, 40152546, 16380211, 73031054, 22721500, 19457589, 83368048, 32161669, 82327024, 19376156, 83210802, 90310261, 35512853, 16405341, 48673079, 26139110, 29516592, 75820087, 84684495, 76315420, 16445503, 90457870, 9623492, 36312813, 68702632, 95581843, 2208785, 60430369, 62115552, 20568363, 81274566, 1788101, 37280276, 48675329, 6497830, 59405277, 20681921, 73392814, 89499542, 38061439, 89046466, 98648327, 62051033, 61728685, 82886381, 1022677, 569864, 82979980, 51149804, 34698463, 48260151, 77694700, 73109997, 6808825, 25636669, 96726697, 7517032, 24915585, 68204242, 40356877, 1375023, 86242799, 73021291, 5822202, 37166608, 31727379, 26292919, 6871053, 99971982, 168541, 8696647, 9398733, 36753250, 10366309, 93053405, 81855445, 17894977, 84840549, 68102437, 3487592, 49328608, 17976208, 15075176, 11161731, 61712234, 32699744, 38009615, 86460488, 70727211, 6111563, 2331773, 84002370, 45428665, 42073124, 27187213, 7066775, 29466406, 72732205, 82532312, 63015256, 47298834, 57658654, 36930650, 84187166, 1250437, 22129328, 99224392, 40534591, 30163921, 22405120, 8634541, 13173644, 78218845, 4515343, 26734892, 53393358, 99604946, 39373729, 11757872, 45381876, 8373289, 17068582, 7229550, 8651647, 89419466, 36468541, 40027975, 47213183, 77898274, 74110882, 176257, 76703609 diff --git a/src/test/resources/data/test_ground_truth_l2_100.csv b/src/test/resources/data/test_ground_truth_l2_100.csv new file mode 100644 index 000000000..c1986261f --- /dev/null +++ b/src/test/resources/data/test_ground_truth_l2_100.csv @@ -0,0 +1,100 @@ +76960303,76703609,62740044,29188588,66428795,37435892,60106217,71469330,9575954,6986898,2607799,66045587,39847321,48774913,16097038,61263068,75229462,39801351,247198,44627776,7182642,40677414,29834512,95957797,30771409,84293052,16099750,38658347,86001008,76434144,9860968,94076128,77187825,67793644,30998561,50806615,96193415,42967683,4095116,16595436,4099191,54987042,56424103,61271144,74137926,2208785,24915585,43376279,92033260,21993752,70596786,24058273,71083565,9623492,6808825,7687278,51472275,35192533,97011160,38645117,34236719,86460488,56153345,61859143,43933006,99224392,17081350,7646095,37675718,66667729,8733068,32250045,35456853,44889423,14947650,96726697,62452034,83210802,86022504,79136082,73235980,1569515,62496012,22108665,92803766,7919588,92787493,40865610,72738685,75407347,63967300,23848565,33237508,51047803,6153406,69786756,55247455,24826575,6793819,5111370,63059024,64157906,42307449,3233084,75986488,38061439,80014588,72373496,14349098,72357096,20885148,53842979,23740167,17764950,32161669,98943869,33935899,65851721,43506672,54199160,7423788,16971929,60176618,9599614,36753250,9860195,97783876,83948335,53632373,79922758,94711842,176257,86301513,4064751,26863229,5640302,79821897,17385531,68694897,94539824,57241521,40781854,29994197,8634541,60430369,19376156,30218878,7685448,58749,57163802,60955663,44847298,30543215,19939935,75777973,26493119,62693428,39171895,20002147,92692283,57248122,70372191,40781449,78549759,22879907,62232211,73786944,14731700,3233569,72274002,41092172,76315420,72725103,85116755,9603598,93933709,22942635,1431742,46870723,57359924,15015906,20568363,77620120,44177011,17386996,749283,40984766,76671482,21001913,49236559,36396314,33061250,76540481,54427233,44550764,54762643,88653118,95581843,31904591,48893685,78589145,27625735,36780454,17037369,63152504,47792865,2917920,84840549,39553046,21070110,4668450,36812683,32426752,45848907,81853704,29029316,73392814,37620363,50702367,34698463,26664538,77413012,8129978,59371804,84904436,61373987,27665211,16019925,54663246,68939068,48675329,80316608,70004753,53802686,1788101,87720882,37891451,64602895,74110882,53666583,18131876,71316369,13231279,16054533,66271566,29466406,55189057,12024238,96735716,19101477,42199455,64848072,45794415,15064655,68000591,78602717,3487592,68875490,67030811,85242963,26998766,77272628,18699206,99515901,18783702,55960386,89811711,13468268,19486173,66903004,41481685,93270084,90272749,1197320,91802888,73617245,17146629,15536795,18806856,17957593,89214616,28796059,23569917,29819940,71300104,72777973,50668599,36685023,48260151,93515664,3509435,2204165,37192445,24168411,11365791,99690194,3773993,36312813,8807940,77284619,82779622,91240048,15075176,49882705,74452589,61960400,5267545,12571310,90457870,16387575,99333375,86821229,36505482,36933038,508198,569864,33797252,70036438,65880522,62803858,32159704,69136837,21289531,7066775,73124510,16380211,49598724,8913721,9175338,55602660,63372756,14326617,91664334,54058788,16405341,62430984,52060076,38365584,54517921,8651647,7011964,30569392,35092039,71657078,59981773,99658235,68644627,19457589,90004325,78218845,29510992,92692978,24591705,83302115,7955293,51464002,9058407,91990218,874791,56484900,62051033,39986008,88047921,57803235,45996863,53547802,36845587,83533741,82427263,9188443,23194618,45995325,27185644,85571389,91957544,51149804,4508700,31126490,27325428,42073124,6505939,30764367,415901,34497327,12348118,67451935,29959549,77694700,26744917,40027975,29932657,48892201,41245325,69355476,22129328,38341669,68824981,55669657,9398733,3880712,26392416,34698428,52261574,41092102,43357947,526217,20140249,36189527,45075471,3294781,66250369,112651,11543098,44915235,89217461,38009615,79880247,40534591,45237957,26507214,44060493,87710366,80555751,33565483,92071990,83150534,59910624,15948937,57961282,61859581,34946859,2891150,25636669,49271185,84187166,19272365,17365188,81805959,96420429,97057187,92604458,87598888,62936963,20122224,40152546,2717150,97281847,66832478,15535065,83269727,28787861,17539625,95251277,10358899,99604946,82178706,13470059,68204242,84127901,36930650,77377183,6497830,3235882,36580610,92554120,30090481,4204661,87160386,34044787,30787683,36808486,168541,669105,38494874,90013093,34295794,73222868,91141395,99971982,90061527,1022677,55753905,68102437,4434662,10366309,69641513,43322743,33304202,45790169,17068582,55143724,98948034,18833224,64087743,20642888,9257405,79806380,8128637,97940276,4787945,11757872,30811010,75135448,74743862,32274392,99549373,78909561,16684829,30891921,82897371,37224844,77300457,45919976,98739783,59177718,99917599,92867155,62490109,72019362,45049308,99021067,10961421,61741594,24953498,94516935,18001617,58208470,4119747,61982238,25842078,47708850,59501487,45667668,41442762,53648154,83368048,96852321,57240218,37501808,44348328,68316156,67281495,5970607,65338021,56955985,58180653,20836893,3633375,11923835,64055960,40686254,79322415,68110330,66663942,44983451,19891772,77072625,99861373,89046466,90654836,66319530,79191827,86242799,7517032,83133790,11581181,8791066,59614746,13348726,92541302,47361209,26139110,13173644,39373729,44846932,65081429,45407418,84684495,37183543,89419466,60102965,97641116,49641577,56515456,90310261,33249630,71333116,46851987,78766358,61141874,54014062,21919959,99297164,9886593,31161687,50007421,82532312,43501211,23110625,99965001,30694952,78884452,98371444,93057697,65454636,26292919,6871053,95010552,67513640,76170907,30139692,53084256,92215320,44784505,22450468,28928175,33628349,91727510,58751351,5832946,44479073,18411915,77312810,12664567,71920426,94911072,82726008,94360702,30463802,16445503,82651278,48673079,81898046,51426311,24314885,37659250,6038457,58224549,91281584,79942022,1204161,93566986,20486294,27041967,39201414,73031054,32590267,94090109,30163921,50188404,57020481,35996293,33123618,15148031,74357852,4806458,76330843,30395570,37739481,31491938,70420215,70367851,1250437,26114953,26776922,80246713,88698958,56531125,67227442,48088883,52613508,75153252,4515343,42644903,73168565,77301523,26102057,32699744,7104732,7300047,6521313,9829782,40197395,6111563,75128745,2331773,57658654,56461322,78785507,26063929,83083131,45381876,82886381,8373289,22721500,92398073,91255408,321665,42947632,51715482,68128438,83747892,17727650,10649306,83155442,72278539,83789391,82339363,49597667,73021291,36135,63015256,43045786,16424199,27187213,60581278,5073754,21533347,82979980,33895336,55470718,93359396,68041839,72495719,52204879,47887470,72614359,86118021,74724075,74614639,14093520,22997281,54868730,88904910,79657802,67474219,95290172,22766820,6819644,72238278,51507425,38022834,51588015,94595668,3183975,75543508,98462867,61728685,99524975,27411561,65275241,66611759,45617087,97379790,83651211,63506281,66885828,47213183,53393358,22435353,96709982,18663507,53888755,81774825,28851716,20765474,29065964,4091162,36139942,8099994,35512853,52293995,4069912,61897582,81677380,44473167,47893286,59109400,89699445,7502255,42782652,84406788,65715134,97561745,59197747,12416768,6157724,18504197,69579137,33699435,26734892,4798568,51466049,1936762,24619760,22200378,61815223,82327024,86543538,43152977,68702632,85711894,33201905,37560164,26397786,86798033,65271999,48658605,55770687,65047700,19898053,28734791,36468541,92998591,72793444,72358170,15902805,61623915,93790285,98130363,81274566,32058615,12303248,14220886,10597197,84002370,21673260,88444207,89499542,11161731,33431960,34493392,42237907,75052463,82052050,93562257,98653983,37957788,70727211,6221471,47298834,74441448,93053405,15163258,34432810,49328608,81172706,65074535,88251446,48395186,46540998,70800879,40356877,54232247,14626618,29635537,5822202,4985896,59329511,66322959,21397057,8115266,66116458,33553959,4199704,59318837,54263880,76825057,90090964,38510840,33336148,95726235,20645197,62762857,63628376,89637706,9259676,31727379,98648327,78300864,95395112,44844121,44664587,44245960,73109997,89996536,13862149,45990383,77787724,44481640,32151165,78561158,62357986,34667286,92530431,85023028,18466635,57393458,99226875,55615885,70782102,29401781,89078848,75820087,45428665,30653863,68816413,51590803,77898274,81855445,38008118,23134715,28550822,64098930,5482538,6525948,8696647,71522968,10309525,30366150,50567636,31733363,84349107,61928316,69848388,4978543,41380093,17016156,8729146,56473732,38256849,62552524,47090124,26275734,44842615,17058722,47738185,69605283,17894977,91938191,67031644,3088684,65017137,71965942,62428472,14723410,20867149,17976208,65038678,89804152,10264691,10453030,85117092,99729357,60393039,42692881,62115552,51315460,48360198,45306070,91831487,92353856,8055981,23432750,70541760,29516592,69697787,7229550,69255765,16567550,55850790,72732205,61712234,17857111,14363867,96318094,88130087,37166608,13819358,66137019,85695762,14045383,67084351,54606384,1375023,824872,67124282,84166196,90158785,20681921,22405120,22113133,99125126,24733232,79738755,37280276,80251430,1239555,59405277,96906410 +84840549,73124510,14326617,99658235,9623492,89214616,68702632,30787683,97940276,5267545,76434144,35092039,40534591,19376156,23569917,29819940,5111370,80316608,98943869,83368048,1788101,41245325,15536795,6986898,37620363,80251430,22942635,64087743,42644903,36930650,13173644,18663507,63967300,72495719,247198,21070110,16684829,30694952,79136082,45996863,17016156,74614639,93053405,18466635,17764950,36685023,80555751,27665211,48774913,76703609,36505482,79922758,18131876,94076128,18783702,40027975,22879907,72373496,64098930,38061439,30543215,1022677,1431742,43501211,9886593,32250045,62936963,60955663,99917599,20642888,44889423,72738685,44348328,40865610,16424199,73235980,65047700,13468268,66885828,38365584,45995325,16097038,97783876,73392814,63372756,33304202,39553046,168541,83302115,6808825,47213183,13231279,15535065,31904591,62115552,65074535,3509435,10961421,16445503,81677380,81898046,7502255,26744917,33123618,40152546,62496012,77312810,96726697,35456853,93933709,11365791,36780454,37659250,39986008,92033260,43933006,78766358,34432810,61263068,82052050,9603598,68204242,56515456,34295794,7919588,48675329,29029316,66045587,38658347,82897371,61373987,78884452,48260151,10366309,53842979,54762643,32161669,51715482,73031054,2917920,40677414,92787493,55770687,90272749,71469330,17727650,71657078,63628376,57803235,55753905,37166608,96193415,34497327,79322415,56484900,43376279,84684495,75986488,669105,30139692,15902805,43045786,90090964,45075471,23194618,83150534,62490109,50668599,23848565,47738185,71333116,72777973,4668450,8115266,19891772,69136837,54199160,16054533,3294781,3233084,24591705,7646095,62803858,65338021,33628349,68041839,78602717,51047803,91664334,91802888,19898053,36139942,6157724,92803766,2717150,10358899,62693428,7517032,90457870,24915585,76960303,98371444,7300047,69355476,40984766,22766820,70004753,77377183,112651,98130363,29959549,29994197,38645117,51149804,63506281,33249630,52293995,61897582,79880247,14731700,4099191,7687278,41481685,6153406,59197747,16380211,20486294,55615885,75229462,86022504,34698428,8634541,17894977,26292919,14626618,88047921,4434662,60430369,62357986,39373729,99515901,30163921,24953498,874791,29188588,78909561,43322743,92215320,30366150,83083131,95581843,29466406,3233569,19457589,87710366,30463802,73222868,95726235,74137926,83210802,19486173,10597197,87720882,12348118,39801351,62452034,22129328,60102965,92398073,18833224,66663942,97057187,569864,50806615,17068582,24058273,85242963,12416768,29065964,36753250,5822202,8696647,48893685,85571389,78300864,26392416,33895336,26114953,17957593,66322959,59981773,88698958,85116755,21533347,88653118,89419466,48892201,36933038,93562257,30764367,22108665,98653983,43152977,77272628,66271566,50188404,6521313,10309525,45667668,4064751,42073124,73168565,95957797,25636669,11543098,78589145,61271144,72274002,36580610,47298834,52613508,79657802,42307449,82726008,68816413,77187825,75777973,44844121,66428795,55143724,37435892,56955985,30090481,50702367,67030811,70596786,9398733,67227442,96852321,33061250,16099750,31727379,49328608,6871053,26507214,47361209,62552524,99297164,45919976,69255765,69641513,61728685,53802686,60581278,44550764,2891150,37739481,76315420,95395112,99226875,30998561,46851987,99971982,9188443,53393358,14093520,98462867,46870723,65851721,70727211,29834512,4985896,36812683,28796059,56473732,75153252,32590267,8913721,53084256,89699445,16387575,86001008,42967683,26139110,35192533,16971929,20122224,13348726,66667729,68316156,99021067,74110882,74743862,38008118,28851716,58751351,82532312,79191827,11161731,30891921,39171895,64157906,65017137,81805959,21993752,72732205,70541760,91727510,99333375,45237957,61141874,16567550,36845587,71316369,66611759,61982238,68644627,99965001,37183543,49598724,57658654,81172706,44245960,4806458,97641116,8791066,67281495,98648327,22997281,7955293,20568363,45306070,44983451,526217,59501487,40781449,57163802,18411915,61623915,56531125,53547802,41442762,3487592,70036438,92692978,37192445,83533741,84293052,48673079,54058788,15064655,98948034,1569515,94516935,78549759,321665,73021291,9860968,59371804,92692283,69848388,54014062,37224844,508198,99729357,30811010,14947650,82339363,7066775,43506672,22435353,64055960,47090124,2607799,30395570,97561745,9860195,36312813,3633375,28550822,96420429,89217461,92071990,3183975,74357852,39847321,30218878,81274566,22450468,95290172,8651647,99549373,37675718,17976208,33797252,3880712,8733068,4095116,64602895,67793644,12571310,89637706,27187213,60106217,49641577,92604458,34493392,64848072,36808486,99690194,1197320,51466049,53666583,9599614,47708850,19939935,19272365,68694897,44915235,66903004,62740044,24619760,65454636,86242799,42947632,20002147,42782652,66319530,65715134,3773993,40356877,34946859,9257405,51588015,93566986,17386996,13470059,7423788,55247455,21001913,83155442,20765474,415901,36396314,61928316,50007421,16595436,72725103,50567636,71300104,83133790,66250369,59177718,26397786,72019362,18001617,61859143,44481640,30569392,30771409,62428472,2208785,12024238,20885148,33553959,2204165,99524975,87598888,33201905,60393039,79821897,63015256,14723410,86460488,76170907,38009615,81853704,6793819,15015906,91831487,47792865,67084351,44177011,85117092,76540481,40197395,79942022,44060493,70800879,749283,29635537,8729146,91141395,77413012,45848907,78785507,75128745,77898274,72358170,8373289,91957544,4787945,52060076,75543508,49271185,3088684,55470718,66832478,77284619,5640302,48658605,62762857,90310261,56424103,67451935,44842615,27185644,7685448,19101477,24314885,55189057,9058407,77072625,4119747,42199455,12303248,38341669,57248122,75135448,67124282,62051033,70420215,82886381,57020481,77620120,99604946,23740167,83789391,68102437,24826575,45049308,18504197,73786944,93270084,59614746,26734892,5832946,40686254,31161687,68875490,8129978,89046466,55602660,91281584,97011160,54987042,94711842,94911072,70372191,29932657,44664587,9175338,95010552,69786756,45990383,84349107,33237508,38494874,94595668,21673260,60176618,92554120,92541302,76330843,54427233,89811711,31126490,63059024,59910624,61960400,98739783,55669657,24168411,77301523,93057697,71920426,88904910,51472275,17037369,63152504,90654836,18699206,76671482,91990218,22200378,44479073,32159704,54868730,7182642,39201414,10649306,61859581,80014588,96906410,24733232,59109400,49236559,9259676,81855445,69605283,57961282,88444207,76825057,17365188,53648154,90061527,56153345,78218845,97379790,8055981,62232211,86821229,35512853,10264691,65038678,18806856,7229550,84187166,83948335,14349098,62430984,53632373,57359924,58224549,6525948,85711894,32274392,83651211,65081429,26063929,72357096,94090109,6497830,41092172,54606384,75407347,49882705,84904436,34236719,6505939,37280276,52204879,69579137,33336148,51507425,30653863,9575954,86301513,79806380,5073754,72278539,66137019,57241521,70782102,20645197,59318837,32426752,23432750,84166196,13862149,4069912,51464002,37501808,56461322,57240218,52261574,93359396,99861373,87160386,16019925,15948937,8807940,93790285,72614359,7011964,94539824,23110625,26493119,84002370,25842078,6038457,21919959,58749,34044787,37560164,16405341,55850790,34698463,48088883,8099994,47887470,78561158,11923835,38510840,8128637,77694700,65271999,57393458,48395186,27325428,74724075,74441448,45428665,22721500,96709982,49597667,35996293,15075176,17385531,1204161,36468541,45794415,51590803,82779622,26275734,73617245,84127901,13819358,71522968,824872,34667286,22113133,28928175,44473167,90013093,36189527,17146629,29516592,4508700,51426311,91240048,26998766,22405120,91938191,54517921,44784505,4515343,54663246,92867155,83747892,68000591,67513640,86118021,61712234,9829782,44627776,43357947,33935899,75052463,85023028,7104732,17058722,94360702,41092102,48360198,46540998,92998591,95251277,1375023,38256849,82979980,44847298,2331773,67474219,31491938,86798033,74452589,17539625,90004325,32058615,26664538,59329511,6819644,33565483,37891451,65275241,68824981,72793444,14045383,15148031,11581181,75820087,21397057,97281847,54232247,29510992,1250437,82178706,27411561,71083565,5482538,54263880,96318094,93515664,61741594,99125126,4798568,38022834,82427263,27625735,44846932,33699435,58180653,45381876,99224392,32151165,36135,33431960,28734791,67031644,6221471,14363867,20867149,61815223,96735716,89078848,45617087,45790169,4199704,71965942,29401781,17081350,3235882,45407418,17857111,176257,88251446,82651278,6111563,4978543,68128438,20681921,77300457,69697787,79738755,58208470,26102057,26863229,14220886,53888755,65880522,82327024,32699744,83269727,88130087,70367851,66116458,15163258,10453030,77787724,73109997,21289531,55960386,90158785,80246713,41380093,42692881,27041967,4091162,86543538,68110330,26776922,20140249,81774825,72238278,89804152,4204661,5970607,31733363,85695762,37957788,11757872,92353856,47893286,20836893,42237907,23134715,40781854,1936762,91255408,51315460,59405277,89499542,1239555,68939068,92530431,28787861,12664567,89996536,84406788 +56461322,63015256,74614639,48260151,91664334,54987042,92803766,4119747,6153406,12348118,33237508,7011964,75153252,88047921,33553959,9398733,72777973,4069912,67793644,89214616,62452034,15064655,19891772,21070110,73168565,97641116,34493392,30218878,87710366,3294781,80316608,62496012,68644627,669105,26114953,68204242,45306070,62936963,44846932,42967683,51507425,35092039,415901,62428472,62115552,54199160,73392814,57240218,45237957,22450468,55143724,54427233,65715134,1431742,65454636,48675329,13468268,92692978,68816413,81855445,59177718,99965001,70541760,29188588,48893685,34236719,29466406,9175338,78549759,26776922,97783876,93933709,45995325,99690194,83368048,99297164,90061527,57163802,79806380,6986898,93566986,5832946,10961421,95290172,874791,29819940,60430369,76671482,31904591,47738185,77377183,79136082,88653118,99226875,70004753,78785507,49598724,32161669,36468541,19272365,19376156,33628349,56473732,33797252,77284619,40781449,38658347,26139110,44983451,37183543,14093520,36812683,4787945,76540481,50806615,4099191,68316156,34698428,14326617,35192533,45919976,14731700,55753905,60102965,66137019,20765474,92071990,96193415,40152546,40677414,47708850,53084256,48673079,72238278,64087743,16445503,6793819,90654836,9886593,8807940,66322959,44889423,27665211,70372191,59318837,47792865,30891921,15015906,8913721,16097038,61263068,90090964,71469330,74357852,11161731,38061439,91281584,22129328,93053405,24733232,94911072,36808486,31161687,3233084,7687278,60955663,96726697,21289531,20486294,43045786,749283,17058722,77072625,12571310,68824981,62430984,92787493,46851987,32426752,70596786,62232211,85116755,54606384,67030811,63967300,24619760,9623492,81172706,79821897,32699744,9188443,45407418,81853704,30139692,94076128,51047803,36753250,43501211,91727510,22108665,17386996,4806458,63059024,66832478,50702367,40356877,48395186,83210802,68694897,22879907,3633375,8099994,508198,80246713,81274566,96420429,33249630,5267545,30395570,28550822,21673260,99917599,17957593,72738685,54663246,72278539,75135448,13231279,73617245,76330843,7517032,81898046,38009615,12024238,95395112,72373496,23569917,18131876,44550764,26102057,32590267,78909561,77620120,73031054,71657078,42644903,78602717,76960303,7919588,44245960,80555751,55615885,18783702,82327024,53842979,569864,83302115,56424103,57803235,7229550,9860968,83083131,48088883,37739481,168541,88444207,92033260,49271185,28928175,69641513,34497327,30787683,42307449,36396314,16595436,86022504,59197747,35996293,22200378,33123618,92604458,70800879,42199455,49641577,14626618,63506281,16424199,78218845,92554120,90272749,112651,59614746,36580610,56531125,74137926,77312810,16971929,61271144,71316369,53632373,82532312,7423788,81677380,28851716,34295794,247198,29029316,98371444,16380211,12416768,59501487,67281495,33304202,7502255,78589145,66663942,5970607,36505482,43506672,37620363,43933006,23110625,32274392,29994197,50007421,5111370,40027975,20140249,36189527,74441448,8373289,20122224,89811711,30543215,66611759,63628376,18699206,5640302,3235882,51590803,43322743,86821229,18466635,70420215,37166608,2891150,72357096,16019925,95957797,2917920,41481685,45996863,83150534,27625735,69355476,51472275,94360702,89637706,66271566,65047700,30569392,25842078,10649306,93515664,1204161,30764367,38645117,6871053,49328608,57020481,55602660,78561158,79657802,20885148,86242799,54058788,3233569,38008118,53547802,48360198,89078848,22942635,64157906,92215320,50188404,38494874,39847321,83789391,3509435,75986488,78766358,29510992,56515456,7104732,82052050,18663507,1375023,85242963,37501808,76434144,67474219,76703609,36930650,4985896,99333375,7955293,33895336,59910624,99658235,73124510,7300047,39171895,17037369,54762643,57241521,72019362,33431960,4668450,40781854,92353856,98739783,43152977,61623915,65275241,61741594,96318094,9860195,40686254,7685448,24953498,55770687,80014588,51715482,15163258,50668599,71333116,40865610,97940276,48892201,61897582,99861373,1022677,65017137,88130087,73235980,76170907,66903004,59371804,75543508,83948335,82779622,51588015,61859581,24168411,92998591,42692881,37659250,77413012,79191827,35456853,6808825,63372756,33201905,26063929,43376279,92692283,72732205,84187166,81774825,13173644,24591705,70727211,3487592,75052463,99549373,22997281,89419466,71300104,64602895,37957788,8696647,37675718,1250437,45667668,74743862,30653863,72495719,67031644,50567636,64098930,24915585,17764950,3183975,20836893,99524975,14947650,35512853,17385531,31491938,86301513,71522968,70036438,2208785,82726008,36685023,95726235,9603598,16099750,6221471,32159704,65338021,57658654,79922758,52613508,17539625,15148031,10597197,10358899,34432810,59109400,48658605,15948937,72274002,11365791,92398073,68102437,27325428,42237907,15535065,36845587,29959549,93359396,23432750,321665,25636669,45428665,75777973,61859143,94539824,24058273,23740167,66116458,77694700,69136837,55247455,36780454,96852321,98130363,19101477,21993752,73222868,41092102,74110882,97561745,91957544,8634541,77300457,26998766,8651647,82339363,8733068,62740044,23194618,99021067,52060076,98462867,54014062,8129978,86543538,49236559,3880712,87160386,39986008,66319530,20002147,33699435,67227442,13348726,86001008,85023028,62803858,68939068,44842615,1197320,73786944,4798568,20568363,2204165,42073124,79738755,176257,19486173,55189057,23848565,59981773,79880247,7066775,83155442,94711842,15536795,90013093,17146629,42947632,45794415,88698958,84293052,45617087,37891451,11581181,47298834,30998561,45848907,67084351,73021291,19898053,13862149,66250369,47361209,45790169,83651211,39553046,64848072,93562257,56955985,41245325,65271999,82178706,17068582,65880522,68875490,29834512,10366309,99125126,40534591,26493119,53666583,61728685,16567550,83533741,14045383,29635537,91240048,87720882,97011160,97057187,26292919,60581278,85711894,84904436,34698463,4434662,24314885,17976208,20642888,85571389,47090124,4091162,65074535,44664587,17016156,16684829,11543098,33935899,86798033,9575954,61141874,54868730,83747892,62357986,4064751,89046466,11923835,69697787,2717150,32250045,68128438,69786756,63152504,32151165,91802888,21533347,61928316,38365584,71920426,4095116,26734892,99224392,84349107,40197395,18411915,526217,26275734,20681921,6525948,27411561,95581843,61815223,96735716,80251430,84127901,62693428,15075176,89699445,61982238,47213183,77301523,52261574,8791066,9829782,61373987,48774913,17894977,23134715,17727650,75407347,21001913,58751351,61960400,24826575,84002370,38256849,90310261,10309525,58180653,44060493,46870723,82651278,96709982,22435353,99971982,84406788,45381876,58749,19457589,77187825,52293995,21919959,37435892,67124282,41092172,1936762,4199704,6819644,4515343,19939935,31733363,61712234,41380093,68041839,81805959,30771409,77787724,31126490,14220886,97379790,62490109,27041967,30163921,30463802,51466049,34946859,51464002,44847298,59405277,29065964,85117092,7646095,44784505,72358170,26392416,89804152,28734791,91938191,37192445,68702632,74452589,75128745,26863229,56153345,82427263,47887470,42782652,44348328,66045587,65038678,66885828,43357947,26397786,70367851,14349098,92541302,30366150,83133790,26507214,15902805,83269727,30811010,98648327,8115266,99604946,95251277,86460488,29932657,84684495,44177011,22721500,39373729,85695762,18504197,69605283,14723410,72793444,75229462,4204661,29516592,39201414,69579137,53648154,57248122,72725103,52204879,67451935,4508700,5822202,54517921,6497830,22766820,18806856,58208470,90004325,55669657,4978543,44481640,59329511,69848388,28796059,44627776,57393458,84840549,44915235,91990218,17857111,18001617,82897371,17081350,39801351,99729357,6505939,88904910,9058407,46540998,6038457,79942022,91831487,30694952,76315420,87598888,54232247,82886381,27187213,9599614,69255765,79322415,37224844,6521313,94516935,75820087,38022834,9257405,88251446,40984766,98948034,62051033,16054533,6157724,62552524,73109997,66667729,93057697,5482538,89217461,92867155,36139942,57359924,74724075,56484900,51426311,90158785,86118021,14363867,55850790,90457870,98653983,78884452,5073754,60176618,20645197,76825057,44844121,66428795,51315460,91141395,55470718,53802686,28787861,92530431,18833224,38510840,8128637,33336148,10264691,33565483,41442762,93270084,49597667,20867149,97281847,71083565,65851721,3088684,7182642,12303248,94090109,34667286,11757872,72614359,93790285,68000591,16405341,82979980,68110330,26664538,10453030,45075471,94595668,44479073,16387575,1788101,57961282,26744917,2607799,98943869,51149804,31727379,36135,22405120,55960386,45990383,34044787,78300864,22113133,36312813,27185644,77272628,3773993,60106217,17365188,2331773,30090481,89996536,824872,9259676,32058615,91255408,77898274,58224549,65081429,29401781,37560164,44473167,70782102,64055960,49882705,6111563,13470059,89499542,71965942,12664567,99515901,54263880,47893286,96906410,62762857,1569515,13819358,95010552,8055981,8729146,45049308,53888755,67513640,36933038,84166196,38341669,60393039,1239555,33061250,37280276,53393358,21397057 +9623492,75153252,19101477,39201414,14326617,3183975,63967300,38658347,33304202,22108665,6497830,70596786,68702632,35456853,67030811,6793819,24591705,30463802,69136837,12348118,74441448,73168565,10366309,44245960,89699445,34698428,68102437,76540481,20836893,39986008,17957593,10961421,71333116,2717150,59501487,67793644,78884452,31904591,49882705,71657078,9058407,10358899,99125126,8115266,97057187,3487592,62430984,5832946,60430369,30543215,20002147,6871053,51047803,569864,48892201,77072625,91664334,13231279,8099994,98371444,76170907,61373987,55143724,40152546,51472275,83302115,86821229,47361209,77312810,69355476,91802888,112651,3233084,22450468,59109400,70420215,91141395,93933709,96318094,38494874,85116755,98739783,70800879,3509435,99549373,8129978,43322743,49597667,37675718,11543098,20122224,47090124,27411561,70727211,5970607,38510840,96420429,37620363,93562257,35092039,62232211,83083131,415901,96193415,9603598,31733363,77620120,9175338,10649306,12024238,92692283,36396314,38645117,7300047,20885148,99658235,72738685,36812683,6819644,90272749,53084256,52293995,23194618,61982238,26275734,29994197,44348328,96709982,40984766,36933038,63628376,23569917,67451935,9860968,37183543,12571310,33628349,12416768,35996293,99604946,27665211,98130363,85242963,2891150,61859581,59910624,72732205,37166608,19939935,30569392,65851721,53393358,66885828,70541760,508198,73786944,33699435,26493119,44983451,73031054,21673260,61263068,60581278,86022504,87598888,29819940,73222868,66250369,55753905,526217,7502255,95957797,44550764,73235980,18833224,48360198,72725103,22129328,80316608,48774913,90061527,9188443,82339363,33061250,40197395,7646095,29510992,80555751,42237907,78300864,25842078,59197747,15535065,79657802,45990383,28796059,49641577,4515343,45049308,61741594,4668450,99297164,45790169,62452034,91281584,3088684,65047700,6521313,32426752,54663246,89419466,36580610,39847321,81677380,47213183,55247455,6153406,6986898,61859143,9259676,57803235,89214616,90004325,33553959,89811711,7011964,23848565,66663942,74137926,97940276,32159704,71469330,60955663,51315460,36685023,16380211,68644627,20642888,36753250,43357947,41092102,37224844,99690194,5111370,79880247,26292919,14626618,54232247,92398073,26063929,98948034,86543538,57020481,81898046,8373289,83210802,81172706,54199160,64087743,30787683,91727510,51464002,49236559,55960386,29959549,87710366,17727650,75543508,72019362,99333375,52261574,65038678,50188404,61623915,91831487,15536795,4787945,77284619,99021067,16019925,93053405,45237957,39171895,31491938,77301523,44844121,1250437,62803858,9886593,44479073,18001617,62428472,66611759,57248122,4099191,68110330,45996863,64098930,33935899,17058722,13468268,1431742,54868730,14349098,36808486,43045786,40781854,73124510,35192533,33123618,56153345,30163921,93057697,7955293,55189057,55770687,59371804,64848072,43376279,32250045,92998591,92692978,98462867,45794415,82178706,1569515,30090481,69579137,42782652,72357096,33797252,33895336,84840549,53648154,33565483,73109997,88698958,68939068,54058788,94090109,77413012,33201905,5267545,62936963,57359924,42967683,37560164,9860195,56531125,66322959,2917920,17764950,1204161,16567550,29466406,31161687,65017137,53666583,98648327,29029316,56473732,65074535,72373496,84684495,99515901,33237508,89046466,20867149,75229462,72495719,75135448,27041967,67031644,86301513,80251430,321665,20140249,78602717,29834512,3235882,12303248,88047921,70004753,95395112,44842615,20681921,22942635,22997281,24058273,6038457,57163802,61728685,66116458,51588015,8696647,92541302,94516935,24619760,824872,4978543,72274002,17068582,22721500,33249630,16684829,40865610,77787724,26664538,62552524,56515456,78909561,84187166,16445503,52204879,1022677,16097038,76671482,92071990,54606384,22435353,7423788,85023028,44915235,57658654,87720882,78785507,30694952,36189527,74357852,95010552,14723410,83133790,82897371,89078848,17365188,72777973,30764367,45075471,8634541,14220886,28851716,86001008,82052050,81853704,37501808,66137019,57240218,65081429,48893685,7182642,75777973,37435892,2208785,2204165,29932657,97783876,57393458,95581843,34667286,59177718,19272365,8055981,3633375,46851987,4119747,57961282,53842979,4095116,73021291,81855445,96735716,68204242,82726008,34236719,24915585,68875490,61271144,91957544,84166196,43152977,874791,89804152,99965001,62496012,33336148,77187825,27325428,59614746,42307449,99226875,168541,96852321,84127901,67124282,5482538,24826575,49328608,44473167,7919588,34698463,15948937,45919976,82779622,6808825,36468541,82427263,55669657,15064655,44481640,22879907,39801351,28734791,44177011,59329511,42644903,24168411,82327024,56424103,38365584,72278539,83533741,61815223,32161669,30218878,55850790,15015906,92554120,52060076,65338021,79922758,16387575,21993752,30395570,66832478,38008118,40027975,11161731,4798568,69255765,69786756,56484900,22766820,20486294,11923835,91938191,61960400,18806856,30998561,67084351,15075176,30366150,4064751,247198,3880712,38061439,66903004,40686254,13862149,14093520,26507214,65275241,44060493,68041839,10309525,84349107,37739481,59981773,30653863,26102057,53888755,4985896,90013093,99917599,90457870,3773993,47792865,10453030,9575954,94539824,4508700,67227442,84406788,36505482,34044787,99524975,65715134,34295794,80014588,18783702,23110625,9398733,17976208,28550822,76434144,32590267,42692881,4806458,18504197,68824981,13173644,76330843,22200378,9257405,68816413,68316156,26734892,26863229,77300457,90090964,37659250,41481685,58749,99861373,44889423,12664567,72614359,69641513,71083565,51715482,71300104,72358170,21533347,44846932,30139692,93790285,26114953,62490109,669105,88653118,79136082,46870723,95290172,54427233,19376156,40677414,20765474,17037369,79738755,73392814,97011160,92604458,87160386,16971929,34946859,31126490,34432810,45848907,14363867,10597197,18466635,36312813,45617087,92033260,50567636,75128745,16595436,45407418,98943869,23134715,34497327,42199455,97561745,94076128,7685448,76315420,42073124,8729146,1197320,83368048,95726235,86460488,17146629,66428795,6221471,47887470,51426311,8791066,17386996,24733232,64055960,83150534,92803766,63506281,44784505,62357986,74743862,50702367,82979980,93566986,1375023,53632373,26392416,3233569,36780454,65880522,43506672,6505939,31727379,22405120,26397786,74110882,37192445,84293052,98653983,85117092,13470059,33431960,13819358,44627776,18663507,78766358,17385531,5822202,21001913,46540998,15902805,4204661,81274566,94711842,89996536,88251446,11581181,29516592,62115552,22113133,83155442,89637706,38341669,19486173,14045383,75986488,4199704,78549759,80246713,26744917,2607799,77272628,14731700,70036438,89499542,99224392,1788101,7104732,27625735,48658605,17857111,61897582,5073754,88444207,18411915,97641116,45306070,63059024,65454636,15148031,50668599,749283,70367851,73617245,69848388,75820087,8733068,48675329,75407347,78589145,51507425,21397057,18131876,57241521,38022834,67281495,72793444,54014062,52613508,43933006,96726697,90158785,35512853,4069912,30891921,7229550,91990218,26998766,54762643,81805959,64157906,68694897,7517032,51149804,77898274,4434662,15163258,67474219,48395186,29188588,63152504,77694700,72238278,77377183,37891451,8807940,79806380,47893286,45428665,4091162,47738185,47708850,32699744,83948335,37957788,10264691,78218845,50007421,41092172,40781449,60106217,92215320,58751351,61712234,48673079,55470718,55602660,84904436,34493392,88904910,91255408,58208470,49271185,36845587,42947632,79322415,45667668,39553046,79942022,7687278,17016156,53802686,17081350,84002370,71965942,68128438,54987042,20568363,62740044,8651647,40534591,43501211,30771409,26139110,16405341,16054533,53547802,94360702,11757872,51466049,83269727,76703609,70782102,58180653,19891772,27185644,1239555,6111563,63015256,8913721,23740167,89217461,41442762,83651211,29401781,92353856,74724075,71920426,51590803,93359396,66667729,44847298,70372191,11365791,65271999,36135,79821897,9829782,48088883,90654836,19457589,5640302,54263880,69605283,6525948,18699206,92787493,93270084,83789391,66319530,21919959,92867155,64602895,32274392,17539625,40356877,48260151,85571389,76825057,16099750,21070110,71316369,17894977,8128637,81774825,93515664,62693428,95251277,50806615,6157724,61928316,97281847,90310261,97379790,86798033,3294781,45381876,14947650,62051033,85711894,63372756,56955985,23432750,59318837,82886381,60102965,25636669,66271566,99971982,58224549,86118021,26776922,74614639,69697787,7066775,79191827,45995325,16424199,28928175,29065964,68000591,47298834,82532312,86242799,54517921,59405277,60176618,92530431,24953498,66045587,78561158,74452589,44664587,2331773,55615885,41245325,71522968,29635537,76960303,60393039,85695762,99729357,32151165,20645197,13348726,36930650,94911072,82651278,21289531,83747892,61141874,24314885,56461322,37280276,28787861,38009615,19898053,94595668,62762857,41380093,49598724,96906410,27187213,9599614,30811010,75052463,176257,38256849,32058615,36139942,39373729,91240048,67513640,88130087,1936762 +73031054,81853704,39171895,65851721,26998766,26863229,92692283,10366309,98739783,23110625,54427233,54663246,38061439,6793819,4985896,22108665,29932657,16971929,61271144,40677414,38008118,77284619,15535065,56473732,33628349,87710366,12348118,44889423,68702632,17957593,2717150,30543215,55602660,24591705,58180653,75153252,60176618,70727211,83210802,36505482,23569917,3633375,98462867,52293995,17146629,56484900,84187166,63967300,3509435,77377183,37560164,15948937,96193415,38341669,32250045,25636669,54014062,8807940,28851716,53084256,29819940,8129978,44348328,59109400,77312810,77272628,52060076,17068582,20122224,94911072,61263068,66611759,74110882,63015256,20765474,29510992,77187825,37739481,25842078,81274566,88047921,11161731,93270084,29834512,45990383,16019925,48088883,26493119,29188588,83651211,55770687,91664334,16445503,22450468,49882705,72777973,81898046,15075176,66832478,9398733,57803235,13231279,54987042,89811711,72495719,97940276,18833224,61373987,29029316,30653863,9623492,72732205,35456853,78561158,2891150,16099750,7423788,48360198,35192533,68041839,55143724,89046466,43933006,32159704,79821897,43322743,34432810,29466406,92541302,3773993,4069912,74452589,36685023,14723410,17037369,74724075,99965001,66319530,98130363,4099191,62232211,92215320,55247455,95290172,55669657,75820087,17764950,37675718,4668450,61897582,72793444,86821229,44060493,72738685,64848072,42947632,8115266,91990218,91240048,36468541,68816413,38365584,64602895,79136082,22435353,79191827,43376279,69136837,17386996,19376156,68824981,10358899,24915585,16424199,78549759,24058273,26102057,66663942,99333375,1431742,31161687,57248122,69355476,59197747,14093520,85023028,19891772,29065964,81805959,7955293,64087743,22942635,61928316,9860968,51472275,38494874,67793644,12024238,35092039,23194618,69848388,49236559,47298834,85571389,87598888,37891451,73222868,98948034,508198,68875490,99917599,64157906,89419466,66250369,1022677,89214616,62430984,27665211,72274002,112651,33304202,3880712,68204242,14626618,62936963,87720882,61859143,62496012,55753905,10597197,168541,85242963,54762643,66903004,91281584,23848565,68644627,1788101,2204165,51588015,86022504,99658235,47090124,3088684,31727379,50567636,84406788,40152546,67084351,30694952,12303248,6505939,79880247,60955663,4787945,61815223,74137926,7646095,4119747,92554120,26776922,71300104,15015906,28787861,8696647,48892201,7300047,57359924,38658347,51715482,75229462,61141874,67030811,82979980,16387575,83789391,96906410,66137019,99524975,51149804,24733232,69697787,35512853,84002370,19457589,16097038,32590267,71316369,8733068,96726697,96852321,30366150,92398073,9603598,81172706,70596786,14349098,91938191,39373729,526217,63059024,33201905,10961421,45995325,83155442,8128637,17539625,17058722,38256849,44473167,95395112,62115552,33237508,13468268,44842615,40865610,86001008,86460488,24826575,16595436,19939935,74743862,50702367,40781449,95581843,6153406,47708850,17894977,99297164,13173644,50188404,47361209,96709982,49271185,91802888,95726235,40356877,48774913,3294781,3183975,2917920,63628376,415901,3233084,46870723,93790285,16054533,76540481,71333116,88444207,32161669,70004753,6521313,46851987,93933709,62452034,33431960,6525948,11581181,33336148,37183543,76703609,7687278,75407347,47792865,41245325,33797252,77898274,65038678,30569392,92071990,85116755,30771409,10309525,45075471,48673079,16380211,42692881,30163921,824872,38645117,3487592,54058788,6819644,65074535,27625735,22879907,6871053,74357852,61859581,90061527,99549373,9058407,99971982,96318094,44550764,36580610,62740044,73235980,41481685,72373496,36396314,81855445,4806458,94539824,30395570,11365791,669105,77301523,66116458,42782652,99729357,49641577,6808825,72278539,75986488,23432750,37501808,67451935,69786756,42307449,1204161,99224392,23740167,39201414,92033260,30090481,68102437,90158785,34667286,66322959,30787683,45428665,6986898,33123618,70800879,82886381,21289531,77413012,247198,99604946,50668599,40686254,61741594,31733363,21533347,12416768,48675329,20645197,65715134,33553959,16684829,3233569,87160386,51426311,44479073,78602717,73021291,72725103,28796059,93515664,8099994,62552524,34698428,59910624,4095116,80014588,49597667,9259676,31904591,37280276,40197395,60430369,37435892,84293052,75777973,18001617,88130087,97379790,23134715,61728685,5640302,92803766,62357986,59371804,20885148,45790169,27325428,874791,68939068,15536795,99226875,38510840,97641116,17976208,27041967,43506672,44847298,60102965,39986008,79942022,56531125,76825057,53632373,78589145,66045587,9886593,93053405,14947650,73392814,33895336,45848907,45381876,84840549,5111370,67227442,97561745,70372191,7517032,14326617,44177011,37957788,17016156,42644903,92998591,82052050,26275734,36312813,80246713,20867149,66428795,30218878,8913721,91255408,93057697,78300864,37166608,83368048,27185644,95010552,8791066,62693428,42237907,26664538,63506281,73168565,32426752,76960303,55189057,56461322,9188443,59405277,57240218,93359396,7685448,65047700,34236719,22129328,77072625,10649306,99515901,36753250,95957797,33249630,79922758,70036438,67124282,37620363,84684495,47213183,90457870,34493392,321665,79806380,90090964,40027975,83133790,55470718,89699445,55960386,2208785,69641513,51464002,29401781,90013093,9257405,76330843,83150534,80251430,18131876,57241521,21993752,94090109,17727650,18411915,26744917,56515456,98371444,76434144,74614639,51047803,40984766,9175338,36780454,51466049,39553046,18663507,53842979,15148031,38022834,66271566,65454636,78785507,36845587,19101477,80316608,99861373,47738185,34497327,83302115,44844121,4199704,20836893,11543098,42199455,81677380,43045786,27411561,75052463,17365188,82339363,75135448,66667729,30764367,86798033,36812683,77300457,6221471,22721500,34295794,65275241,44245960,45617087,73617245,60106217,33935899,83533741,71657078,45919976,61960400,99021067,36189527,5832946,62803858,65017137,26063929,7011964,30998561,83083131,82897371,45667668,73124510,65338021,4798568,33061250,61623915,92604458,56153345,97783876,60393039,65880522,89499542,18699206,32058615,34698463,1569515,20642888,44627776,68316156,13348726,68128438,9575954,44915235,41442762,45237957,57658654,9599614,92353856,60581278,89996536,44784505,94076128,20568363,64098930,36930650,59501487,79657802,37224844,76671482,45306070,53393358,44983451,1197320,75128745,43501211,83948335,34946859,77620120,91831487,70367851,1375023,18466635,68000591,8373289,16405341,92692978,53648154,37192445,78884452,85711894,29994197,45996863,5267545,30891921,22997281,4091162,89804152,73109997,75543508,5482538,32151165,30463802,19486173,7502255,82327024,10453030,96420429,12571310,42073124,96735716,15064655,95251277,49598724,70541760,68110330,73786944,89637706,93562257,74441448,41380093,36808486,52613508,24168411,5073754,88653118,39801351,80555751,12664567,1250437,56424103,51507425,21001913,49328608,91727510,18783702,43357947,59614746,86242799,65271999,4508700,70782102,55615885,26507214,52204879,31126490,6038457,17081350,82726008,28734791,42967683,92787493,85117092,30811010,6497830,18504197,82651278,53547802,47893286,48260151,41092172,52261574,86543538,7066775,58208470,76170907,59318837,76315420,48658605,67281495,7182642,26734892,97057187,9829782,54199160,54868730,4515343,50806615,71469330,63372756,90272749,16567550,749283,39847321,61982238,72357096,86118021,2607799,24953498,62051033,14731700,82532312,26139110,36139942,1239555,569864,82427263,43152977,54232247,91957544,3235882,54517921,11757872,4434662,35996293,22405120,9860195,33699435,90654836,99125126,26392416,33565483,58224549,28928175,56955985,79738755,24314885,93566986,22766820,20140249,40781854,63152504,17857111,13470059,20486294,70420215,17385531,45407418,51315460,94360702,22113133,78218845,7919588,69579137,84349107,4064751,48893685,14045383,26114953,27187213,31491938,7104732,36933038,90310261,8651647,30139692,71522968,88698958,57393458,71083565,54263880,28550822,4978543,53802686,78766358,45794415,22200378,67474219,18806856,69255765,57163802,34044787,26397786,26292919,58751351,13819358,50007421,77694700,21070110,59329511,82178706,6111563,83269727,44846932,91141395,72358170,94516935,19272365,84127901,62762857,89078848,57961282,79322415,88904910,88251446,58749,13862149,54606384,99690194,32699744,71920426,98943869,97281847,57020481,38009615,44664587,68694897,86301513,59177718,89217461,97011160,62490109,8055981,7229550,14220886,84904436,15902805,29959549,85695762,78909561,41092102,21397057,53666583,62428472,11923835,92530431,8729146,84166196,36135,29516592,5822202,21919959,37659250,92867155,72019362,46540998,51590803,59981773,44481640,15163258,69605283,176257,94595668,45049308,98653983,53888755,66885828,24619760,67513640,40534591,47887470,6157724,1936762,90004325,19898053,20002147,64055960,61712234,98648327,55850790,82779622,77787724,94711842,32274392,8634541,29635537,48395186,20681921,5970607,10264691,14363867,4204661,67031644,71965942,72614359,72238278,65081429,21673260,81774825,83747892,2331773 +33565483,33304202,92692978,44983451,62740044,874791,26292919,68939068,7300047,57803235,51047803,67793644,40677414,55247455,33336148,7229550,66045587,72614359,12348118,12024238,36189527,8115266,30998561,89214616,40356877,66832478,73021291,65338021,40152546,60430369,7423788,62430984,22200378,9886593,64602895,59197747,60106217,3487592,80014588,54663246,46870723,569864,96735716,76540481,94539824,65851721,63967300,112651,37183543,78589145,62452034,526217,13468268,54058788,68102437,37435892,54517921,57241521,65454636,62357986,3233569,77284619,1197320,99021067,9175338,37560164,26392416,92692283,37620363,99297164,16445503,68875490,63015256,22766820,79880247,34493392,3235882,70372191,55615885,56484900,31126490,36685023,18411915,7955293,47361209,27187213,73617245,64157906,66611759,96193415,44784505,9860195,83210802,26063929,3773993,66667729,97011160,18833224,1788101,44889423,81898046,87720882,15535065,70004753,71920426,29834512,38645117,72274002,45428665,98948034,20122224,71316369,83533741,5640302,45381876,92554120,89217461,54427233,22450468,87598888,35092039,72357096,34667286,11365791,14349098,11923835,1239555,94911072,65275241,41380093,3509435,98943869,4199704,29994197,73124510,42307449,8696647,39847321,38365584,85117092,60955663,31491938,14045383,89699445,43501211,55602660,62232211,2208785,83789391,83155442,13470059,75543508,81677380,99515901,23740167,92033260,47213183,20568363,58224549,69579137,99224392,79738755,45237957,2917920,89046466,6986898,83083131,53842979,1022677,97940276,22405120,8128637,50702367,70800879,23848565,29188588,81172706,9058407,30653863,81853704,57240218,39201414,21993752,34295794,13819358,78549759,38510840,38658347,76960303,31161687,65880522,16595436,61373987,7919588,72793444,17068582,36312813,72278539,84127901,62496012,57359924,9259676,40027975,22435353,90013093,98130363,6808825,27665211,88251446,99226875,66271566,44627776,86001008,9398733,21001913,53802686,43933006,16405341,67281495,46540998,65271999,15015906,168541,69697787,76671482,49597667,2717150,51464002,82779622,30787683,67451935,59501487,19939935,75153252,30771409,20002147,33895336,93933709,21289531,99549373,89078848,75128745,9860968,29029316,61928316,20642888,36780454,30764367,13348726,15148031,1204161,81855445,86460488,4069912,77301523,4064751,82532312,43376279,8807940,95010552,36505482,29401781,21533347,93270084,61741594,62051033,9188443,88904910,49641577,68316156,47298834,20867149,55143724,95290172,72725103,88653118,56153345,30463802,86301513,26664538,18783702,16097038,17764950,19376156,9575954,43045786,72777973,61263068,83948335,3233084,22997281,73392814,26114953,25842078,34698463,14326617,34044787,68816413,29959549,66322959,36753250,32590267,9603598,36808486,17365188,36580610,321665,26493119,2204165,29819940,38022834,62490109,92541302,52261574,57961282,55753905,74357852,27325428,54232247,78766358,20486294,24168411,77620120,6505939,44550764,31904591,48658605,44060493,61859581,65074535,60176618,99861373,98371444,669105,4668450,33628349,8129978,53632373,52613508,37739481,91240048,61271144,18663507,14220886,5832946,20140249,99125126,55470718,59177718,98739783,11581181,38009615,9257405,99965001,99917599,45667668,85023028,30543215,97783876,58749,50806615,63059024,40534591,70596786,5970607,82726008,98648327,29510992,56461322,8099994,82979980,10366309,53393358,56515456,55960386,68702632,64848072,72238278,77300457,17037369,14626618,37659250,1569515,45995325,85242963,8373289,68128438,66885828,77312810,53648154,94516935,749283,68824981,22108665,5267545,10453030,37501808,78602717,45075471,91802888,19898053,74137926,75777973,52204879,47887470,51149804,96726697,45049308,54762643,58208470,54014062,87160386,45617087,4798568,55189057,94595668,15948937,76170907,56955985,20765474,40686254,44847298,24915585,56531125,1431742,96906410,72738685,75986488,48088883,27625735,16019925,74724075,77413012,97379790,62936963,82651278,53084256,70420215,75229462,83302115,57248122,28796059,61982238,53666583,89637706,26734892,15064655,16099750,24826575,6521313,50188404,75820087,64087743,49882705,6793819,81274566,93790285,64098930,65081429,65017137,17976208,38256849,92867155,66116458,82427263,3880712,34497327,99690194,28851716,65715134,17894977,51426311,65047700,74452589,2891150,28928175,67124282,61815223,34236719,71333116,69255765,24591705,47090124,1250437,66663942,6038457,26275734,14731700,4204661,38341669,31733363,66903004,44846932,44915235,3088684,85571389,56473732,5111370,77187825,95726235,19101477,13862149,94076128,35192533,90457870,52060076,38061439,24058273,90158785,86798033,21673260,39171895,63372756,38494874,19486173,83651211,27411561,33431960,54868730,17081350,43506672,88130087,247198,44842615,35512853,23569917,48360198,88444207,40781854,18806856,18131876,78218845,74743862,85116755,28787861,68694897,37675718,6221471,44348328,87710366,40197395,32161669,77694700,97561745,84840549,99971982,69848388,8729146,44844121,508198,4978543,10358899,41481685,53547802,51466049,63628376,91727510,42782652,28550822,36933038,10961421,30366150,77272628,4787945,51315460,33201905,15536795,91831487,91664334,59318837,36135,16684829,93057697,96420429,55770687,55669657,66250369,22129328,73222868,17727650,96852321,21919959,78300864,29466406,54199160,73786944,11543098,42237907,41442762,50668599,92353856,7646095,7104732,45790169,26507214,29516592,84904436,57163802,84187166,70541760,6497830,12571310,92530431,76703609,8634541,54987042,76315420,6819644,51715482,75135448,19457589,95395112,17957593,68000591,3294781,77072625,22942635,91255408,40865610,49271185,44481640,91141395,55850790,33237508,24619760,11161731,90004325,45848907,33797252,99658235,78909561,18699206,70782102,65038678,30891921,17539625,80555751,16971929,29635537,18504197,43322743,73235980,85695762,72495719,9623492,96318094,67030811,39801351,16387575,98462867,6153406,42073124,59614746,14947650,60102965,90272749,23134715,79136082,4091162,37166608,32699744,32159704,23194618,14093520,5822202,7011964,84406788,86821229,92398073,78561158,34698428,7687278,33123618,20645197,4099191,93562257,45794415,66137019,10264691,84684495,29932657,92215320,48395186,69355476,47708850,20885148,13231279,18466635,71300104,45996863,61728685,48675329,6525948,51472275,69605283,62803858,66428795,89499542,38008118,82052050,71657078,44177011,17146629,4508700,53888755,57020481,33699435,91990218,30139692,26744917,10597197,30569392,68110330,16054533,46851987,86242799,73168565,93515664,61897582,88047921,61623915,98653983,97057187,89996536,71469330,12303248,15163258,92998591,30694952,60581278,15075176,77787724,82339363,3183975,52293995,84349107,40781449,61141874,93566986,69641513,30163921,39553046,89811711,48673079,44473167,69786756,92787493,5482538,29065964,41245325,45990383,15902805,36812683,4119747,35996293,58180653,44479073,97281847,8651647,17385531,47738185,50567636,81805959,76330843,24953498,16567550,7502255,76434144,78785507,2607799,7066775,40984766,88698958,62552524,68041839,94090109,42199455,30395570,90310261,82897371,19272365,4434662,80316608,42967683,67474219,24314885,85711894,83747892,48774913,35456853,61859143,79922758,47893286,20836893,59405277,68644627,95581843,73031054,26102057,63152504,95251277,39986008,31727379,11757872,80246713,19891772,10309525,89419466,26139110,82886381,7685448,63506281,84293052,45407418,27185644,57658654,415901,7182642,78884452,21070110,79806380,50007421,93359396,36930650,26998766,56424103,59981773,22879907,61712234,33935899,66319530,824872,4985896,2331773,86022504,51507425,70727211,45306070,81774825,5073754,58751351,61960400,34946859,59910624,48260151,54606384,21397057,26863229,71965942,90090964,91281584,43357947,99729357,4515343,74110882,59371804,62115552,90061527,54263880,83368048,64055960,17386996,92604458,14363867,75052463,59109400,93053405,26397786,74614639,69136837,10649306,36396314,51590803,41092102,41092172,3633375,49598724,48892201,83133790,8791066,62693428,70367851,82327024,33249630,4095116,99524975,22721500,36468541,76825057,44245960,86543538,30811010,32274392,71083565,24733232,73109997,96709982,72358170,72019362,7517032,23432750,28734791,4806458,95957797,36139942,77377183,90654836,14723410,47792865,23110625,18001617,79191827,17857111,37280276,12416768,57393458,79657802,89804152,42644903,60393039,42947632,70036438,6871053,32058615,32151165,37957788,9829782,72732205,8055981,51588015,48893685,33061250,26776922,67084351,77898274,17016156,62762857,30218878,27041967,34432810,79821897,92071990,94711842,1936762,22113133,44664587,74441448,1375023,92803766,37891451,67031644,37192445,30090481,49328608,91957544,32250045,72373496,80251430,6157724,13173644,49236559,62428472,45919976,83150534,39373729,8733068,16380211,37224844,6111563,9599614,67513640,99604946,68204242,17058722,20681921,82178706,42692881,94360702,16424199,33553959,71522968,43152977,84166196,36845587,99333375,84002370,83269727,59329511,91938191,79942022,12664567,8913721,25636669,79322415,67227442,32426752,97641116,75407347,86118021,176257 +54663246,63967300,62452034,34295794,75543508,6525948,19457589,11581181,99965001,60430369,66045587,44784505,99524975,98948034,40781449,89078848,30764367,62740044,6808825,26507214,87720882,9398733,43376279,17764950,33797252,168541,48675329,16019925,14947650,73392814,5111370,75777973,9829782,26392416,14093520,35092039,78549759,90158785,90457870,97011160,2717150,98130363,81805959,67281495,67793644,57803235,38645117,88047921,98462867,95726235,31904591,32250045,34667286,6793819,8807940,28550822,70596786,22721500,80316608,77413012,14220886,81855445,18466635,44481640,526217,93270084,48260151,16405341,99226875,69355476,31161687,18663507,4064751,19486173,96735716,44842615,50702367,16445503,10366309,89214616,64087743,71333116,48893685,73031054,74110882,39373729,2891150,45428665,22129328,10358899,37659250,33201905,36808486,30998561,22942635,7011964,81853704,18504197,26664538,17365188,44889423,38365584,72793444,9257405,79880247,70727211,35456853,40152546,37620363,60102965,27665211,66667729,8696647,60955663,32161669,93515664,76540481,31491938,26863229,14626618,92215320,30811010,5640302,28787861,18833224,51426311,20002147,37891451,6153406,80014588,98371444,65454636,13468268,37183543,30891921,5832946,61263068,57241521,96193415,26292919,43501211,63628376,66903004,85117092,73235980,34698463,53648154,112651,15148031,93933709,54427233,66885828,61373987,45995325,67474219,40197395,62428472,39801351,45990383,22200378,91831487,84684495,68128438,874791,66250369,73124510,58749,61712234,72274002,92554120,49328608,78884452,91240048,72373496,508198,70372191,28734791,30694952,247198,47361209,94076128,65038678,89499542,36780454,99658235,78561158,48360198,9886593,16971929,82979980,93562257,64848072,2208785,84840549,4099191,72278539,56515456,57240218,749283,4204661,29819940,85116755,30163921,33249630,69255765,40686254,51047803,47738185,44060493,99297164,76703609,2607799,33237508,21001913,40677414,98653983,86242799,11923835,83150534,75229462,81172706,11365791,79942022,70004753,65074535,62936963,47213183,92692283,41481685,47090124,55143724,62490109,44245960,78589145,81774825,3880712,19939935,92530431,53842979,84904436,2331773,77694700,18699206,68824981,2917920,85571389,78766358,83533741,51507425,15536795,24733232,99515901,6221471,68102437,16054533,70036438,12571310,32699744,38061439,35192533,7646095,81677380,92033260,48774913,45919976,92398073,36845587,87598888,79191827,62051033,42307449,89419466,83133790,47887470,53666583,94911072,19101477,69605283,3773993,9058407,47298834,36812683,94360702,29834512,83302115,7300047,74614639,93053405,63015256,54014062,15902805,42947632,88904910,91957544,8129978,77300457,68702632,34236719,74724075,88698958,31126490,55247455,98648327,55615885,7955293,83155442,51590803,13173644,75153252,10309525,72614359,59177718,33123618,91990218,9175338,37192445,70782102,49598724,22879907,49641577,38022834,56153345,7423788,77377183,3088684,1022677,21673260,16380211,54517921,80555751,9860195,64055960,36505482,55189057,17068582,16567550,14349098,37501808,94516935,91664334,69579137,43933006,40356877,99021067,38658347,81898046,72019362,30771409,569864,6521313,75820087,44479073,47792865,71965942,8099994,85242963,63059024,44983451,29510992,66832478,18783702,84187166,45667668,64098930,53632373,27041967,39986008,92803766,7182642,13862149,21993752,24619760,3294781,41380093,65081429,11161731,99971982,45790169,9623492,77284619,24058273,59614746,55960386,20645197,52293995,45049308,10649306,54762643,62357986,36685023,65338021,92692978,669105,40781854,24168411,17957593,33565483,36753250,45794415,22108665,6497830,86821229,20568363,70420215,43152977,30395570,40027975,91727510,26275734,59318837,61623915,13231279,55753905,61741594,29029316,89637706,28796059,55770687,68939068,28851716,29466406,58751351,4668450,824872,62430984,91141395,95957797,8634541,23740167,23432750,22766820,12416768,19376156,3509435,89811711,5267545,73786944,66611759,18131876,59109400,24591705,51149804,56473732,18411915,35996293,4985896,70367851,53888755,9188443,69697787,56531125,54058788,51464002,36312813,96906410,77272628,76825057,84166196,26998766,10264691,77620120,29188588,17857111,29959549,24915585,20140249,22450468,96726697,58180653,26776922,54987042,19272365,36396314,58208470,54199160,86798033,55470718,33628349,65275241,21289531,96709982,88251446,97379790,86022504,15075176,17976208,73168565,23569917,77187825,23194618,79806380,33304202,20642888,68644627,61928316,56955985,95290172,98739783,54868730,50188404,77312810,16099750,20486294,27187213,14045383,30090481,94595668,8913721,99333375,49271185,79657802,10453030,66322959,26734892,84293052,86001008,8791066,68000591,56484900,92604458,99604946,45075471,12024238,79738755,3233084,72357096,44844121,39847321,89699445,76330843,5073754,62552524,61815223,14731700,74137926,82726008,43357947,82532312,1569515,20765474,83368048,24826575,13470059,29635537,45617087,61982238,23110625,75135448,87710366,2204165,79136082,8115266,71300104,92353856,12348118,31727379,3233569,80246713,73222868,4069912,72495719,95395112,20122224,68316156,56424103,47708850,29994197,30218878,99690194,90272749,70800879,72725103,22405120,82427263,20836893,15064655,66137019,55669657,77301523,78602717,30366150,57359924,72738685,17727650,76671482,73021291,48892201,1788101,72238278,65880522,44550764,71522968,17894977,57658654,9575954,5970607,26063929,32159704,43506672,92787493,97641116,84127901,33895336,51466049,42644903,4515343,98943869,30569392,36930650,33935899,51588015,30653863,37435892,4787945,39553046,67451935,21533347,86543538,89046466,45407418,67513640,71469330,92998591,20885148,52613508,34497327,53393358,85023028,42782652,1204161,83789391,71083565,15948937,64602895,49236559,321665,29516592,61960400,79922758,31733363,45381876,1375023,34698428,38008118,55602660,14363867,40984766,30543215,8373289,81274566,61271144,11543098,54232247,32151165,23848565,42967683,37560164,90310261,59501487,45306070,38009615,62803858,27185644,27325428,415901,16595436,83651211,17539625,6986898,53802686,69641513,66271566,50806615,91802888,93566986,4508700,16684829,83210802,57163802,72777973,66428795,75407347,25842078,1250437,24953498,13819358,32590267,52204879,74743862,99917599,29932657,42692881,70541760,32274392,68694897,49882705,61728685,66319530,82886381,90090964,99125126,62693428,38341669,1197320,74357852,7919588,26102057,95010552,52261574,68875490,6505939,41245325,80251430,42237907,32058615,26114953,37166608,28928175,84406788,17081350,46870723,15015906,76170907,51715482,48673079,33336148,67084351,45996863,6157724,10597197,96852321,34493392,34946859,21070110,17385531,69136837,33553959,4798568,73617245,97057187,94090109,3487592,40534591,7502255,13348726,7685448,18001617,57961282,7229550,63506281,16097038,85711894,77787724,61859581,59981773,65851721,83269727,71657078,69848388,48395186,33699435,32426752,59329511,77898274,51315460,54606384,65271999,53547802,87160386,8729146,55850790,1936762,7104732,36468541,50567636,6871053,8055981,83948335,75052463,78300864,62496012,71316369,67030811,66116458,68816413,68041839,37739481,43322743,48088883,82327024,21919959,26139110,43045786,60393039,19891772,67031644,9259676,99549373,76434144,1431742,68204242,97940276,65047700,97281847,17037369,66663942,61897582,60106217,39201414,85695762,75128745,99861373,3235882,90061527,50007421,46851987,16424199,42073124,15535065,14723410,4095116,45848907,94539824,76960303,50668599,51472275,35512853,36135,62762857,91938191,52060076,37675718,83083131,22997281,77072625,40865610,60176618,60581278,59197747,97783876,46540998,45237957,36189527,4806458,88444207,56461322,3633375,82897371,53084256,90013093,74452589,30463802,27625735,4091162,44627776,82779622,38510840,29401781,10961421,89217461,82178706,29065964,59371804,59910624,75986488,27411561,8651647,34432810,176257,5482538,90654836,69786756,22435353,39171895,30787683,67227442,38494874,37957788,15163258,96420429,44473167,96318094,4199704,26744917,76315420,41092172,65017137,14326617,33431960,78218845,20681921,99729357,86460488,93057697,74441448,36580610,71920426,78785507,44846932,26493119,4119747,44177011,57393458,25636669,48658605,61141874,9603598,4434662,8128637,6819644,58224549,33061250,86301513,44664587,16387575,5822202,62115552,44348328,92867155,6038457,88653118,62232211,41092102,86118021,57248122,9860968,42199455,4978543,34044787,12664567,79821897,1239555,73109997,94711842,82339363,90004325,68110330,23134715,7517032,63372756,82651278,21397057,54263880,7687278,93359396,20867149,91281584,82052050,72358170,3183975,7066775,37280276,61859143,36139942,8733068,91255408,72732205,95581843,41442762,63152504,12303248,49597667,89804152,84349107,78909561,64157906,99224392,59405277,44915235,9599614,18806856,97561745,22113133,65715134,37224844,83747892,6111563,26397786,95251277,47893286,17058722,84002370,88130087,79322415,44847298,67124282,17386996,30139692,57020481,89996536,24314885,92541302,11757872,93790285,36933038,17016156,17146629,38256849,19898053,92071990 +62496012,74357852,19891772,29819940,95010552,83210802,88047921,49641577,34493392,50702367,92787493,21533347,66832478,45848907,36753250,70541760,17764950,40356877,16099750,37620363,62803858,62693428,44481640,12348118,93515664,13468268,3233084,45990383,63967300,66137019,77284619,87160386,95957797,61928316,41245325,65271999,96193415,2917920,18783702,16019925,68000591,55247455,68204242,71920426,63152504,64848072,58749,6808825,64055960,71522968,18699206,88653118,18001617,8129978,45407418,86022504,37739481,35996293,9623492,71333116,17727650,89078848,84840549,8115266,62428472,86543538,9257405,75153252,1788101,37166608,97940276,41092102,57241521,26114953,83789391,17539625,21993752,44784505,66045587,61859581,83368048,44060493,36780454,28550822,92803766,77413012,9603598,55143724,51047803,96726697,94090109,88904910,3880712,42782652,6221471,74452589,99524975,30787683,53547802,39801351,99224392,92554120,50668599,91255408,91240048,8807940,4204661,27625735,32590267,44889423,53842979,45996863,79821897,37957788,57163802,47893286,22108665,73124510,93933709,26998766,71469330,7687278,36685023,73031054,85117092,83651211,66885828,98371444,54987042,52261574,16387575,45428665,17857111,34236719,64602895,5111370,749283,67451935,7502255,64087743,6505939,53802686,6793819,55189057,93270084,39847321,78300864,69641513,40686254,24733232,6986898,27325428,61815223,79136082,36580610,26863229,32159704,65275241,33237508,81898046,68816413,30764367,50188404,45667668,70800879,98943869,33628349,42307449,34295794,4119747,99515901,62357986,14947650,69579137,59197747,50007421,51472275,61859143,37192445,38494874,93053405,78218845,68128438,67124282,67793644,3294781,79657802,65047700,71316369,51426311,8128637,81853704,75135448,29834512,28796059,60955663,47738185,40027975,62936963,20867149,43501211,38061439,23569917,72793444,82726008,7182642,8696647,54762643,44177011,90090964,97783876,83533741,84127901,97561745,67030811,53084256,4787945,34044787,63506281,43376279,78602717,62452034,61141874,55753905,52293995,54263880,95581843,60102965,79942022,59177718,68316156,9575954,15015906,76671482,32161669,76434144,28928175,10358899,78766358,14220886,29065964,61271144,10961421,48673079,19898053,94539824,99861373,4099191,70727211,61897582,33336148,22450468,9860195,66611759,36808486,20642888,91664334,70004753,17037369,73392814,84904436,9259676,44245960,96852321,23194618,36312813,19486173,247198,76960303,58751351,38022834,89699445,90272749,95395112,45995325,65880522,45919976,74614639,45075471,29188588,68824981,57803235,63372756,11365791,68041839,82779622,34432810,669105,27665211,99604946,58224549,83948335,4668450,39986008,30366150,99297164,22942635,4064751,43933006,3487592,76315420,41442762,83155442,48675329,46540998,75777973,26392416,95290172,33935899,99917599,3233569,92692978,6521313,31126490,47361209,15075176,20568363,93057697,86001008,40197395,72495719,26664538,59614746,18466635,20486294,9829782,73617245,33201905,70036438,14045383,16097038,76540481,26507214,63059024,47090124,11543098,19272365,55770687,98948034,23110625,5267545,66322959,4434662,9188443,11581181,72274002,21070110,20002147,37891451,66271566,23848565,90457870,36505482,19457589,69697787,874791,80555751,38365584,30543215,73168565,89046466,27187213,7423788,33304202,75543508,48658605,508198,91727510,66663942,62740044,55615885,10366309,84684495,48088883,48260151,40865610,70596786,89996536,92867155,42947632,47708850,18663507,54427233,65715134,33895336,3773993,78589145,83083131,15535065,24619760,7955293,1431742,66667729,68644627,38645117,85695762,74110882,87598888,81172706,5073754,13348726,56461322,78561158,88698958,51315460,62051033,22879907,98462867,23740167,68939068,70372191,2717150,96709982,82979980,66116458,49236559,80246713,65338021,82897371,44844121,73235980,39171895,27411561,48892201,53393358,56531125,92692283,69355476,84293052,44842615,92998591,72358170,56473732,90013093,5832946,45237957,80316608,79738755,2204165,10597197,4798568,83133790,89419466,95726235,11161731,15163258,56484900,9599614,55669657,92071990,72614359,40152546,22435353,26063929,54014062,5640302,73222868,66428795,42073124,94711842,53648154,74441448,44479073,526217,36189527,28734791,99965001,25636669,42644903,61741594,74743862,47792865,82651278,59910624,9886593,38658347,59109400,56424103,12024238,92604458,79191827,79806380,43045786,53888755,90310261,168541,14731700,40984766,92033260,76330843,83150534,18806856,39201414,93562257,30139692,62430984,86301513,99729357,98653983,81677380,16971929,57359924,81274566,7104732,94911072,29994197,58180653,24915585,82427263,66250369,60581278,18504197,98739783,91831487,4199704,24058273,69605283,3183975,29029316,71657078,24168411,33797252,7300047,8913721,46851987,14093520,30771409,57020481,16567550,34946859,59318837,86798033,93359396,16405341,36139942,62490109,77377183,99125126,72357096,44664587,30163921,81805959,45790169,85571389,77072625,46870723,36930650,55470718,36812683,77312810,65081429,92541302,72019362,44550764,37560164,20140249,51466049,20885148,5482538,34698463,40677414,61728685,69848388,64157906,94360702,87720882,33249630,30694952,77694700,69786756,99658235,77272628,75407347,30891921,77620120,43322743,55960386,1375023,57248122,14723410,64098930,61982238,86821229,9398733,43357947,32250045,35512853,17068582,91141395,35192533,81855445,99549373,59501487,6525948,60106217,1569515,54606384,55602660,1204161,31491938,20765474,112651,47213183,35092039,56153345,72238278,67227442,16595436,8634541,53666583,16380211,3235882,52060076,2891150,415901,67474219,29932657,21289531,45617087,89214616,2607799,96906410,97057187,31161687,93790285,85116755,62762857,7011964,19376156,32058615,33061250,50567636,16054533,21919959,40534591,36468541,34497327,61263068,82339363,70367851,30218878,92398073,13231279,30653863,3633375,31727379,3088684,37501808,92353856,17957593,24314885,37224844,176257,18833224,62232211,53632373,44983451,26275734,72777973,26493119,77301523,32151165,17081350,4515343,54199160,12571310,74724075,82532312,23134715,96318094,65454636,73786944,18131876,15536795,17385531,60430369,8099994,72373496,82052050,54663246,71965942,37435892,57658654,49271185,16445503,61623915,69255765,39553046,50806615,4069912,57240218,18411915,36396314,21673260,31904591,11923835,76170907,30998561,28851716,824872,7229550,66903004,13470059,37675718,78785507,26139110,29401781,89811711,25842078,79922758,38341669,36933038,86242799,13173644,84349107,17058722,51464002,38009615,37183543,22200378,89637706,59371804,87710366,29959549,63015256,30569392,79880247,14363867,47298834,82886381,57961282,65038678,88444207,52613508,3509435,81774825,24953498,54517921,24826575,6153406,56955985,91802888,66319530,36135,54232247,44915235,76703609,35456853,85711894,85023028,83302115,41092172,91281584,1022677,89804152,49598724,96420429,51507425,62552524,22405120,90004325,34698428,96735716,60176618,72725103,99690194,76825057,78909561,7517032,88130087,8373289,15148031,70420215,57393458,80251430,10309525,39373729,29510992,77187825,93566986,56515456,75052463,19939935,15064655,34667286,12416768,7066775,28787861,9058407,67513640,31733363,14349098,37659250,71300104,14626618,85242963,40781854,80014588,51715482,67281495,91938191,33123618,65017137,32699744,68875490,26734892,26292919,19101477,98130363,51588015,38008118,26776922,30090481,23432750,22129328,29466406,10649306,42199455,90061527,8791066,4091162,30811010,9860968,24591705,48360198,82178706,51149804,4508700,51590803,62115552,8733068,30395570,48893685,89217461,42967683,1936762,68102437,22766820,52204879,58208470,15948937,82327024,60393039,4095116,49882705,65851721,84187166,47887470,90158785,97281847,48774913,44473167,92215320,92530431,75229462,44846932,7646095,6497830,6157724,99971982,71083565,17365188,26397786,30463802,55850790,8651647,97379790,91990218,45049308,45306070,15902805,68110330,54868730,12303248,99333375,74137926,2331773,32426752,43506672,88251446,7685448,86460488,33553959,6038457,61960400,72278539,75986488,94076128,6871053,40781449,98648327,72732205,29635537,45794415,89499542,5970607,59329511,49328608,63628376,14326617,33565483,99226875,33699435,97011160,91957544,4806458,6819644,10453030,20645197,7919588,67031644,20122224,41481685,26744917,27185644,99021067,54058788,42692881,65074535,38256849,1197320,59405277,68702632,77787724,73109997,9175338,94595668,61373987,42237907,77898274,22997281,26102057,17976208,86118021,83269727,90654836,20836893,16424199,59981773,77300457,10264691,68694897,2208785,20681921,4985896,43152977,72738685,49597667,70782102,37280276,21001913,17016156,1250437,569864,44627776,41380093,75128745,321665,44348328,73021291,13862149,17146629,44847298,78549759,36845587,97641116,95251277,67084351,17894977,38510840,84166196,75820087,78884452,33431960,79322415,45381876,8729146,22113133,84002370,17386996,69136837,22721500,83747892,48395186,11757872,16684829,1239555,21397057,61712234,27041967,94516935,6111563,5822202,4978543,84406788,8055981,13819358,32274392,29516592,12664567 +45381876,96735716,66322959,3509435,81853704,57803235,64157906,33895336,35192533,29188588,23569917,69355476,63506281,56531125,65271999,6808825,44915235,26392416,52204879,62693428,415901,65275241,29029316,86001008,168541,48088883,66250369,70004753,95010552,35092039,99965001,67030811,48675329,8807940,9188443,70372191,10309525,1022677,66319530,7423788,66271566,63372756,22997281,80246713,17957593,53842979,73617245,62936963,37739481,28928175,65715134,24826575,84904436,9257405,66045587,22129328,62430984,41092102,54232247,569864,34295794,11161731,76170907,99690194,29994197,30366150,63967300,31126490,91240048,44983451,20002147,4668450,26863229,7646095,15535065,66667729,33304202,61928316,51047803,62232211,4515343,97011160,16097038,8129978,96709982,83651211,18663507,10264691,69641513,38494874,59981773,72274002,1431742,13231279,27411561,60102965,53666583,50806615,71333116,9886593,40197395,50188404,52613508,66832478,68816413,75135448,84127901,61263068,61815223,72278539,55189057,65338021,56424103,67474219,92554120,93933709,75986488,25842078,54517921,9575954,66611759,86301513,30787683,55247455,84166196,71920426,62740044,85116755,9860195,73124510,42307449,176257,19272365,47738185,61982238,83210802,62051033,39847321,97561745,62452034,43501211,63015256,71316369,24168411,36780454,33565483,99549373,88130087,89699445,92692283,43045786,7182642,112651,76330843,16054533,19376156,29819940,7955293,26114953,34698463,44479073,31727379,41380093,36505482,60430369,34493392,32426752,24619760,44348328,54263880,97940276,19939935,56473732,77377183,96193415,68875490,14731700,95395112,13348726,12348118,48892201,3773993,70800879,65454636,8696647,20642888,99226875,72793444,51466049,6221471,51464002,30764367,44177011,17365188,83948335,73031054,69255765,98648327,93790285,3233569,20765474,36930650,16099750,2204165,65017137,29466406,47887470,66137019,66116458,86242799,64087743,41481685,68102437,22879907,60106217,6038457,36685023,7685448,78589145,67084351,1788101,64602895,1569515,9623492,98739783,45075471,8115266,749283,44889423,33201905,30569392,52060076,77694700,30998561,67031644,38645117,2208785,17727650,15075176,49597667,5111370,38365584,76703609,67451935,59910624,16595436,874791,86022504,35512853,61373987,26507214,76434144,73021291,89214616,26275734,61623915,99515901,15902805,36396314,59177718,50007421,12416768,75229462,99729357,99125126,17037369,86460488,33123618,37183543,47090124,18783702,58224549,43376279,3294781,87710366,36753250,72373496,39553046,17081350,17386996,76540481,3633375,81172706,68128438,46540998,32161669,21533347,61141874,89046466,526217,45848907,53802686,51472275,59371804,78766358,3088684,55470718,11923835,99604946,82178706,90272749,57240218,22435353,40781449,77284619,98130363,6505939,33249630,33797252,30090481,77312810,37435892,90013093,26664538,15163258,18833224,4508700,17068582,98371444,39171895,92033260,54762643,3880712,669105,23194618,91802888,71657078,26998766,82979980,6793819,57248122,18466635,42967683,17539625,51507425,59197747,78785507,52293995,37620363,56955985,91664334,77301523,36189527,20645197,32151165,83150534,9259676,79136082,72725103,94911072,13862149,45617087,83302115,6986898,82726008,13173644,89637706,22200378,45306070,61741594,60955663,30139692,37957788,18806856,96726697,8651647,98943869,48360198,68694897,508198,68939068,77072625,824872,42237907,49641577,40677414,34497327,7300047,2717150,16019925,94360702,24733232,35456853,84187166,92541302,68644627,54663246,83083131,21993752,10366309,37501808,54014062,44842615,33237508,54427233,38658347,4119747,83155442,63152504,72358170,19101477,70036438,55850790,5970607,78602717,49598724,53547802,45996863,56153345,55753905,72777973,7919588,42692881,78909561,19457589,10453030,83789391,14947650,90654836,92787493,99861373,68041839,30653863,33628349,48893685,92867155,69697787,84349107,76671482,7229550,16380211,92353856,94539824,39801351,54058788,44473167,19486173,62496012,44847298,38061439,22942635,51715482,89419466,57241521,20681921,24591705,99224392,1250437,68204242,29834512,28550822,27325428,77187825,17894977,55143724,61271144,6157724,64098930,9829782,3233084,42199455,66663942,1239555,14220886,7687278,33061250,17146629,41245325,52261574,79880247,90090964,38341669,14093520,51588015,57961282,85571389,321665,75153252,40356877,77413012,51426311,40865610,10358899,70727211,76960303,98948034,50668599,36933038,37675718,4099191,13470059,44627776,48260151,99971982,73168565,75543508,29635537,80555751,22405120,95726235,34044787,83269727,38022834,32699744,35996293,1197320,8373289,59318837,81677380,64055960,47298834,38009615,9599614,22108665,43506672,9398733,57393458,44784505,58749,16971929,36808486,89811711,10597197,65851721,40534591,73222868,26102057,69786756,86798033,27665211,69605283,78561158,92692978,5482538,97379790,68000591,30771409,70420215,49328608,36580610,58180653,77272628,4806458,61859581,64848072,21070110,94076128,22450468,99297164,47361209,16445503,62428472,83533741,43322743,49271185,39201414,71083565,54199160,29959549,88904910,1204161,77620120,57359924,4091162,15148031,15015906,36312813,51149804,95290172,2607799,84002370,71469330,75820087,55615885,61859143,92398073,26139110,44844121,8913721,90061527,30395570,65038678,16567550,84684495,6819644,81274566,44060493,60393039,73235980,11581181,45995325,42782652,47708850,79821897,30218878,94090109,88251446,74724075,63628376,96420429,20122224,54606384,49882705,16424199,9058407,89078848,30543215,99917599,20486294,76825057,56515456,48673079,14349098,9860968,39986008,59329511,89499542,28787861,5073754,9175338,20885148,74137926,7011964,93270084,53888755,79806380,81855445,5267545,78549759,82532312,23848565,82886381,4798568,79191827,85242963,14626618,88047921,98462867,8634541,32590267,34667286,10649306,44481640,30891921,46870723,47792865,2891150,67793644,54987042,73392814,84406788,74614639,91727510,71522968,77300457,91831487,17764950,34946859,7104732,44550764,67227442,33336148,4787945,83368048,53632373,48395186,18411915,68824981,96318094,69579137,12571310,93053405,45237957,71300104,59501487,62357986,61728685,21001913,72495719,2917920,36139942,83133790,38256849,96852321,70596786,74357852,20140249,18131876,72614359,17976208,50567636,86821229,31491938,31904591,78218845,24058273,13468268,87160386,2331773,65074535,82052050,73109997,68316156,82339363,38510840,47213183,34432810,79657802,40152546,72357096,90457870,26292919,65047700,24915585,40984766,79322415,29065964,29516592,91957544,55669657,65880522,31161687,66885828,36468541,32250045,11757872,29510992,7517032,53648154,60581278,67281495,91990218,37560164,27625735,74452589,8128637,51315460,66903004,62762857,9603598,67124282,30463802,1375023,20836893,12024238,14363867,72238278,3487592,74743862,55960386,60176618,38008118,4064751,33553959,36845587,82897371,93359396,21919959,29932657,48658605,6525948,21673260,70367851,92604458,5640302,8729146,18001617,247198,19891772,70541760,88653118,54868730,82327024,77787724,15064655,91141395,66428795,92071990,11543098,45790169,7502255,40686254,89217461,81774825,53084256,42073124,94516935,97057187,55770687,45407418,93566986,39373729,4095116,4069912,30694952,62552524,45428665,56484900,11365791,93057697,45049308,90004325,26776922,32058615,61897582,78884452,93515664,57163802,37659250,85711894,69136837,43152977,59109400,22766820,48774913,79942022,87720882,15948937,23110625,34236719,23432750,14326617,28796059,34698428,18699206,95581843,92215320,69848388,80014588,8099994,55602660,99524975,42947632,4985896,8791066,3235882,62803858,91255408,90310261,46851987,23740167,85023028,45919976,19898053,3183975,26744917,26063929,79922758,73786944,28734791,16387575,88444207,30163921,27185644,45794415,99333375,15536795,12664567,92998591,74110882,82427263,6871053,28851716,43357947,37891451,32274392,8733068,4978543,20867149,31733363,1936762,43933006,33699435,62490109,99658235,17058722,7066775,32159704,20568363,8055981,96906410,84840549,97281847,30811010,84293052,61712234,22721500,14723410,68702632,49236559,94711842,77898274,95957797,91281584,92803766,10961421,75407347,12303248,26493119,6521313,98653983,17385531,72019362,25636669,97783876,75777973,99021067,21289531,6153406,50702367,59405277,87598888,5832946,18504197,85695762,71965942,80251430,13819358,45667668,36812683,81898046,82779622,93562257,37166608,36135,57658654,85117092,72732205,4199704,5822202,40027975,14045383,56461322,62115552,4204661,16684829,6111563,26397786,27041967,33935899,79738755,65081429,67513640,81805959,63059024,6497830,44664587,16405341,41092172,88698958,94595668,86118021,59614746,42644903,37192445,40781854,58208470,53393358,86543538,91938191,92530431,58751351,27187213,51590803,72738685,33431960,75052463,26734892,90158785,22113133,24314885,61960400,80316608,37224844,44846932,44245960,89804152,82651278,78300864,95251277,29401781,68110330,83747892,89996536,47893286,4434662,37280276,17016156,23134715,70782102,74441448,75128745,45990383,24953498,57020481,76315420,41442762,17857111,97641116,21397057 +77312810,96726697,61741594,4064751,51047803,112651,17365188,63967300,53084256,66319530,44177011,17957593,31727379,71083565,7423788,55247455,57803235,69605283,81853704,62452034,23569917,40984766,7182642,22108665,44348328,168541,4099191,51464002,77272628,99333375,62936963,98371444,10358899,69355476,94076128,16387575,44915235,48892201,96193415,35192533,88904910,49641577,91990218,23432750,21533347,44481640,66250369,10366309,30395570,55770687,7646095,89419466,34698428,9829782,76671482,61271144,60581278,53648154,55143724,95581843,39553046,77787724,90272749,93270084,14220886,16097038,56473732,39847321,63152504,15535065,9860195,37675718,84293052,27411561,3880712,54762643,29959549,50188404,79136082,4095116,84904436,22129328,26392416,9886593,19939935,66832478,9599614,27041967,73617245,24733232,76170907,95957797,91802888,8115266,14093520,39171895,83083131,95010552,82726008,67281495,18663507,74357852,59177718,84684495,81898046,10309525,26664538,70004753,30771409,57240218,34946859,34432810,73031054,73786944,6819644,45617087,53666583,29819940,70036438,75229462,40865610,14731700,33201905,44842615,94516935,9603598,51588015,27665211,31126490,18001617,89046466,26998766,99965001,53632373,31733363,77284619,77301523,176257,73168565,62490109,33304202,69641513,33895336,56515456,7685448,12571310,42692881,69786756,11581181,89637706,40686254,59371804,26114953,93790285,27625735,79922758,64087743,14349098,3294781,51315460,96420429,29994197,50702367,56424103,6793819,60106217,91240048,247198,3088684,18699206,72373496,8651647,57163802,29932657,99658235,8913721,79322415,64157906,38494874,24591705,33565483,8733068,8634541,93562257,12348118,15075176,83533741,17037369,51472275,62740044,20645197,36780454,41092102,82779622,10649306,58751351,33061250,75543508,73021291,18806856,43933006,68702632,65038678,16684829,89217461,76540481,97011160,75986488,21070110,37501808,33431960,38022834,34044787,96735716,34236719,59501487,29188588,7919588,55753905,6157724,72238278,508198,83368048,61263068,29834512,78549759,44784505,20885148,4199704,44550764,54232247,58208470,35456853,73124510,65851721,3487592,26292919,59109400,48675329,415901,97561745,29510992,90090964,749283,20867149,72357096,824872,78602717,37739481,12416768,669105,8128637,51426311,27185644,33797252,8807940,42967683,34698463,98943869,56484900,70727211,37891451,9257405,59329511,6497830,51466049,16595436,5267545,33123618,52060076,59910624,13470059,28796059,88047921,74441448,58749,11923835,89078848,60430369,65271999,99021067,54014062,22997281,37957788,70596786,86022504,54987042,17894977,93053405,65081429,75153252,4434662,67227442,66885828,43357947,32159704,75135448,58180653,65715134,32161669,33237508,6221471,49882705,8129978,48088883,21993752,98739783,68128438,83269727,12664567,67451935,68644627,16019925,98648327,17146629,9259676,63372756,9398733,5970607,60102965,64848072,39201414,88653118,71657078,97783876,38658347,86301513,45995325,4798568,45794415,79806380,19101477,23194618,93933709,24826575,99861373,88698958,40027975,42237907,42199455,30653863,81805959,24168411,73392814,42307449,68102437,37435892,36845587,37280276,45790169,98462867,43506672,82979980,4668450,3233084,59981773,31904591,86821229,30463802,66667729,86798033,65047700,94360702,78218845,38061439,65880522,52293995,54517921,44245960,36580610,1431742,81172706,9623492,99604946,62693428,9860968,87160386,53842979,30543215,25842078,26776922,30218878,3773993,5111370,26863229,95395112,30764367,90654836,76434144,14045383,51507425,65338021,75820087,3509435,66428795,55470718,2717150,92530431,66045587,43376279,1204161,56531125,24058273,1569515,49271185,55189057,63506281,6153406,77072625,39986008,60393039,25636669,17058722,73235980,86001008,40534591,42644903,47090124,90457870,36753250,84349107,84127901,68939068,5482538,76703609,6986898,54606384,71965942,6505939,67474219,29466406,97057187,84187166,22450468,50007421,68816413,54058788,67793644,17385531,15015906,55669657,16054533,34295794,1239555,44844121,30787683,37620363,65454636,1022677,77620120,90013093,61712234,28787861,70372191,44473167,30366150,81274566,7011964,11543098,77694700,5832946,92541302,23848565,75052463,36312813,21673260,70800879,20681921,53802686,64602895,6525948,16099750,24619760,63628376,43045786,37659250,61982238,72793444,47708850,6808825,2204165,77300457,82339363,40781854,84406788,76330843,13231279,46540998,38341669,99524975,55960386,13468268,90004325,51149804,45848907,26734892,47887470,87710366,72019362,96709982,42782652,43152977,77377183,43322743,526217,36808486,82052050,52204879,44983451,40197395,26063929,86543538,76315420,57961282,80316608,321665,75407347,49598724,67084351,68875490,92692283,62496012,85571389,36505482,78561158,4508700,62430984,14723410,16405341,91957544,72614359,4091162,22435353,20836893,61859143,13173644,7104732,70367851,95251277,14947650,9575954,19272365,12024238,77187825,74452589,37183543,89214616,98653983,40356877,74110882,53393358,91664334,61960400,874791,1250437,57359924,9188443,92604458,13862149,26493119,63015256,78589145,7229550,78909561,32058615,20140249,61623915,53888755,11757872,48360198,62762857,18131876,20122224,8696647,69697787,26744917,92554120,45075471,16445503,90310261,72738685,44846932,79191827,98948034,70541760,77413012,15148031,99549373,7300047,62803858,69136837,14326617,46851987,7955293,61373987,55850790,78785507,83302115,92803766,33249630,10597197,2331773,82886381,88444207,49236559,87598888,68041839,66322959,71469330,92353856,60955663,72278539,72274002,69848388,47361209,28851716,54427233,29029316,89499542,19486173,68204242,16380211,5073754,97379790,61728685,45919976,33628349,36930650,30090481,79942022,74724075,4806458,89811711,2917920,6871053,39801351,62051033,32274392,99690194,569864,30569392,18833224,32250045,49597667,87720882,82427263,20486294,20002147,1375023,2891150,6111563,21001913,62428472,22113133,37192445,68000591,56153345,17764950,17016156,6038457,56461322,54663246,61859581,42947632,36812683,35092039,8729146,30163921,27325428,4985896,18466635,3233569,64098930,4515343,51715482,50806615,71300104,14363867,30998561,99917599,71333116,37224844,38256849,65074535,97940276,44847298,91831487,32699744,30694952,41481685,46870723,73222868,17081350,8373289,84840549,38645117,57241521,60176618,20765474,91141395,35996293,91281584,8099994,51590803,7687278,40152546,89699445,47738185,85116755,2607799,92787493,85023028,36189527,22721500,99297164,80251430,1936762,82178706,90158785,40677414,21289531,15536795,28550822,99515901,1788101,93359396,17539625,35512853,66663942,82897371,22942635,61815223,93515664,95726235,15902805,34493392,45407418,78766358,33699435,71522968,44479073,92867155,11365791,33935899,71316369,83150534,74743862,17386996,16567550,52261574,55615885,48673079,71920426,59318837,99125126,54263880,99224392,57248122,15064655,2208785,4978543,80555751,26507214,8055981,18783702,33553959,42073124,52613508,48893685,22766820,36933038,90061527,85242963,9175338,70420215,17068582,92071990,48658605,12303248,29065964,65275241,80014588,93566986,45996863,24314885,94539824,80246713,41380093,30139692,39373729,48774913,1197320,40781449,54868730,10264691,31161687,88251446,29516592,26275734,83210802,4787945,92033260,84002370,4069912,10453030,21397057,38510840,44060493,18504197,78300864,32426752,17976208,91255408,47792865,14626618,17727650,43501211,7502255,36468541,16424199,89996536,95290172,56955985,22405120,83133790,37166608,8791066,45667668,34497327,24915585,92692978,88130087,7517032,37560164,91727510,19891772,3183975,3235882,34667286,19457589,41442762,24953498,81855445,11161731,68824981,50567636,97281847,82327024,72725103,69255765,20642888,57658654,75777973,94711842,99226875,13819358,5640302,47213183,59197747,59614746,9058407,44889423,54199160,3633375,83789391,57393458,94911072,68694897,92398073,83651211,79880247,94595668,89804152,86460488,79657802,45428665,92215320,28734791,94090109,69579137,85711894,17857111,97641116,29635537,30811010,47298834,62115552,74614639,45049308,45237957,21919959,45381876,57020481,61141874,66137019,72358170,67030811,82651278,75128745,98130363,77898274,33336148,50668599,74137926,19376156,72777973,13348726,53547802,36139942,66903004,48395186,15163258,45306070,76960303,93057697,38009615,83948335,67513640,7066775,86118021,67124282,18411915,68316156,85117092,30891921,45990383,48260151,68110330,72495719,26102057,62552524,96852321,22879907,64055960,15948937,79821897,47893286,4119747,41245325,36685023,99729357,79738755,20568363,66271566,26397786,16971929,63059024,38365584,83747892,6521313,32590267,61897582,91938191,85695762,58224549,99971982,29401781,67031644,44627776,41092172,28928175,31491938,92998591,49328608,55602660,65017137,66611759,96318094,81677380,23740167,38008118,23110625,27187213,23134715,36396314,86242799,5822202,26139110,78884452,4204661,83155442,96906410,22200378,62232211,32151165,72732205,36135,44664587,70782102,62357986,61928316,59405277,66116458,19898053,84166196,82532312,81774825,10961421,73109997,76825057 +72274002,89214616,59197747,67793644,112651,77284619,45428665,95290172,60430369,44348328,20765474,70420215,81898046,99658235,50668599,55753905,16387575,84840549,96906410,99549373,19939935,29029316,83150534,12024238,30395570,29834512,66250369,54058788,93933709,74137926,99971982,53547802,3633375,3233084,55470718,69255765,20568363,66322959,55247455,14626618,13862149,71333116,17068582,50702367,47708850,24058273,95957797,6793819,6808825,71316369,72725103,9259676,81853704,15536795,6525948,8129978,82339363,54663246,53802686,30998561,5640302,4508700,33797252,98371444,78589145,63506281,40781449,43152977,77620120,62452034,10961421,74724075,44481640,15535065,86543538,37739481,97379790,37166608,39847321,64087743,2717150,8807940,38645117,65047700,73124510,49882705,54014062,29188588,20486294,33935899,62496012,29819940,415901,29065964,9603598,63628376,7687278,36189527,27411561,40356877,54868730,51047803,3088684,54427233,37620363,26744917,15015906,96193415,86301513,66832478,7919588,65338021,84002370,18699206,35192533,16019925,18411915,92033260,73222868,3183975,75407347,3233569,62490109,19891772,65275241,69136837,72278539,31904591,69848388,34236719,88653118,33201905,84684495,6221471,9257405,96318094,20642888,42967683,14326617,47090124,35092039,98648327,83083131,77272628,68102437,40865610,1431742,48892201,6871053,79806380,37560164,61728685,91664334,26275734,98130363,4119747,32590267,24915585,21533347,7685448,40677414,61263068,96726697,54263880,92803766,9623492,9886593,90013093,86460488,6153406,34044787,66137019,62740044,26493119,62430984,62357986,824872,10309525,25842078,72777973,46851987,38008118,19376156,30366150,22108665,19486173,36812683,26114953,74452589,32159704,42073124,14093520,42199455,92692978,51590803,34493392,61960400,83533741,5111370,61897582,99226875,44889423,53084256,91938191,14947650,66611759,29510992,76170907,67124282,94516935,5832946,68816413,65715134,24826575,33431960,16445503,7300047,12348118,73617245,8651647,73235980,31161687,44060493,92353856,57163802,29466406,31126490,64602895,58224549,36753250,33237508,22766820,65851721,72738685,55189057,67030811,66663942,23848565,99515901,9188443,34698463,30764367,508198,73109997,39373729,16567550,65454636,82052050,50567636,16380211,49328608,59177718,43376279,73168565,12416768,85116755,91281584,61982238,37891451,89046466,14220886,13231279,61271144,72732205,7011964,64098930,95395112,45306070,37183543,79738755,3509435,5970607,38658347,17976208,22450468,3294781,30787683,39171895,65038678,75229462,85023028,55770687,50007421,72358170,40686254,75052463,69641513,16971929,95581843,26734892,84127901,77694700,56484900,22942635,51472275,68204242,94595668,17385531,95726235,23569917,40197395,61373987,26139110,51588015,48260151,27325428,41245325,30891921,27665211,40984766,56531125,91831487,83948335,526217,34497327,40781854,55669657,9575954,44842615,45848907,45049308,12664567,43933006,30139692,86022504,11543098,66271566,54199160,78218845,36580610,99917599,99297164,80014588,23194618,66116458,93053405,16595436,15148031,56461322,7955293,60106217,14349098,41092102,91957544,82327024,35996293,48675329,57803235,13173644,22879907,72495719,22129328,4668450,18806856,5073754,17037369,65017137,49641577,15163258,47792865,83210802,57658654,70372191,39553046,97057187,88904910,11365791,42692881,48774913,16099750,29516592,11161731,16684829,69355476,59501487,10366309,33628349,61141874,10453030,69579137,32426752,44245960,81677380,30694952,2891150,67281495,29994197,42307449,24591705,41092172,20836893,85242963,45407418,34295794,78785507,58749,38061439,7423788,83651211,59371804,33123618,39801351,34667286,84187166,60581278,11581181,62803858,45237957,38494874,99604946,64157906,96420429,33249630,90061527,86821229,55850790,74743862,83269727,90090964,36139942,72373496,3880712,52261574,62552524,4099191,73031054,71657078,57248122,44983451,8733068,71083565,92692283,57020481,75777973,80555751,43045786,68824981,49598724,76330843,43501211,4787945,84406788,79821897,51507425,51715482,26063929,30771409,51464002,17365188,17857111,12303248,39201414,85571389,54987042,62232211,26102057,45990383,62051033,1204161,70596786,4064751,6986898,46540998,92998591,79880247,36396314,36685023,73786944,97561745,3487592,57961282,4515343,7229550,60955663,26507214,70541760,33304202,18504197,38365584,68041839,92787493,84904436,89637706,20002147,99965001,98462867,77301523,86001008,15075176,44479073,82897371,54517921,54606384,77787724,9398733,17146629,10649306,50806615,60102965,36468541,99224392,98739783,20122224,77187825,75543508,76315420,44177011,47738185,68000591,59614746,21993752,6505939,50188404,22200378,28734791,37224844,20140249,37501808,26664538,7517032,20645197,18833224,99690194,10358899,48088883,92604458,30569392,16405341,26392416,47361209,19272365,66428795,77312810,44473167,63967300,1250437,87598888,88047921,47298834,48395186,82979980,97940276,16054533,7182642,44844121,39986008,78602717,66667729,2208785,70367851,33699435,61859581,83368048,63152504,17764950,67474219,11757872,45794415,63015256,77413012,1197320,38256849,29959549,17957593,53632373,23134715,19101477,81855445,1022677,18783702,98653983,51426311,87720882,67031644,75153252,45790169,72614359,321665,76540481,44550764,96709982,70727211,36505482,92215320,34432810,62428472,5822202,53842979,3235882,8729146,4204661,8696647,35456853,10597197,55143724,75128745,32058615,32161669,75986488,53666583,82726008,66045587,74357852,77377183,88251446,45617087,26776922,23740167,61741594,34946859,86118021,669105,42782652,40152546,78561158,90272749,99861373,92071990,45995325,31491938,15902805,176257,91990218,1569515,70004753,7646095,41380093,28796059,61623915,89811711,65074535,96735716,28928175,17386996,34698428,83133790,5482538,93790285,56153345,91255408,36780454,62693428,53648154,31727379,2917920,94911072,36312813,71469330,72019362,57359924,78909561,87710366,1239555,53393358,52613508,67084351,59109400,18663507,2607799,26998766,22997281,89499542,89078848,21001913,26397786,52293995,88130087,68644627,569864,44784505,15948937,59329511,92554120,30653863,47213183,16424199,99021067,78766358,79136082,89804152,22405120,17081350,48893685,4798568,84166196,9058407,37435892,8055981,14731700,58751351,8913721,99333375,16097038,51466049,2204165,44847298,33895336,18131876,94076128,60176618,97641116,32151165,30543215,83155442,21070110,42947632,61712234,78300864,91727510,58208470,10264691,77300457,30218878,14045383,59405277,78549759,22113133,87160386,44664587,9860195,76960303,93515664,18466635,54232247,1375023,70800879,17727650,12571310,22435353,55602660,45075471,27187213,86798033,75135448,74110882,68316156,168541,63372756,72357096,7066775,8115266,45667668,24168411,36808486,68110330,24953498,83302115,29932657,68939068,36135,8099994,13348726,42644903,62936963,51149804,91802888,71920426,20867149,80251430,56515456,28550822,4434662,13819358,4978543,79191827,72793444,43322743,18001617,11923835,84349107,93057697,20885148,95251277,4095116,97011160,17894977,81172706,65271999,89419466,32699744,63059024,47887470,45381876,69697787,97783876,55615885,80316608,52060076,79657802,56955985,33565483,59910624,71522968,6038457,49271185,78884452,57393458,61859143,65880522,17539625,49236559,73392814,74614639,13468268,38510840,83789391,47893286,68702632,97281847,35512853,8373289,58180653,749283,4069912,89699445,88698958,64848072,59318837,90310261,90457870,93359396,98943869,32250045,24314885,64055960,1788101,80246713,38009615,48360198,96852321,52204879,92541302,37659250,9175338,29401781,75820087,79922758,31733363,88444207,76434144,92867155,85117092,44627776,60393039,49597667,92398073,33336148,44915235,27625735,62762857,94711842,4806458,66319530,99524975,8791066,30463802,9599614,6521313,61815223,69786756,73021291,40027975,54762643,38341669,20681921,48673079,76825057,8634541,45919976,29635537,21919959,74441448,37957788,82532312,90004325,28787861,82178706,37280276,26292919,22721500,21397057,93566986,56424103,93270084,6111563,66903004,89217461,95010552,69605283,7104732,77072625,36930650,68694897,55960386,874791,5267545,57241521,24619760,6497830,19898053,247198,28851716,7502255,82779622,14723410,3773993,81805959,21289531,61928316,71300104,4199704,23110625,76671482,68128438,99729357,26863229,89996536,17058722,84293052,66885828,94539824,59981773,81774825,48658605,45996863,46870723,30090481,57240218,9860968,70036438,40534591,65081429,33061250,44846932,85695762,81274566,43506672,83747892,99125126,24733232,98948034,14363867,13470059,67451935,27041967,36933038,30163921,6157724,82886381,91240048,27185644,72238278,91141395,6819644,67513640,8128637,94090109,42237907,90654836,30811010,86242799,9829782,94360702,21673260,2331773,4985896,67227442,43357947,32274392,23432750,90158785,1936762,25636669,4091162,19457589,33553959,37675718,15064655,37192445,38022834,85711894,41442762,68875490,82651278,79322415,17016156,62115552,76703609,93562257,41481685,77898274,79942022,53888755,92530431,82427263,56473732,71965942,36845587,51315460,70782102 +99861373,62740044,36580610,38494874,96318094,77620120,40984766,99524975,53648154,67281495,54517921,1022677,65880522,5970607,61373987,40197395,37659250,50806615,91990218,3088684,12416768,112651,62452034,93057697,8791066,19939935,84406788,97783876,61141874,29834512,7955293,526217,74441448,26664538,48360198,55470718,91802888,85242963,34698428,70004753,19101477,39986008,82651278,44481640,50188404,4978543,84840549,19376156,415901,46870723,27665211,37435892,15535065,6808825,49641577,78589145,89637706,49882705,99515901,30543215,30771409,39553046,9860195,35092039,7182642,43933006,5267545,68102437,59501487,1431742,79880247,1204161,29932657,66045587,69355476,18783702,37675718,64848072,14220886,94516935,56515456,38061439,63967300,29994197,24058273,87598888,95957797,2204165,1788101,96726697,26392416,29029316,55602660,39847321,69136837,64098930,57359924,29959549,4668450,12571310,91727510,10366309,52060076,20002147,25636669,54762643,42782652,75135448,92554120,89811711,99965001,28550822,91831487,53842979,99658235,22200378,34698463,6111563,68939068,33565483,45075471,36845587,35456853,29510992,66611759,60955663,38008118,66663942,57163802,80251430,83210802,44844121,95581843,36780454,99604946,71657078,99917599,45919976,64087743,24591705,96735716,5822202,569864,2917920,9188443,60106217,20140249,52293995,8696647,20765474,95726235,62496012,44915235,92541302,12664567,42199455,88047921,27411561,508198,28851716,44479073,18504197,42237907,50702367,65715134,77312810,91957544,93933709,88653118,77272628,64602895,13470059,33249630,83269727,37183543,72278539,15015906,92692283,321665,98462867,66428795,7685448,77413012,53632373,26998766,31733363,78884452,3233084,8055981,71083565,26744917,84127901,34295794,4199704,4204661,45790169,39801351,27185644,51426311,33237508,17146629,5111370,99549373,14947650,49597667,7423788,48892201,60430369,44177011,10358899,19272365,56424103,17081350,3235882,53802686,47213183,79657802,63015256,48774913,32274392,80014588,22405120,85711894,30998561,66667729,81172706,57240218,26776922,39201414,94595668,32250045,27041967,44348328,65074535,83083131,76703609,67451935,44784505,30569392,26275734,33431960,98371444,71300104,30764367,35192533,68204242,8128637,77284619,76330843,71522968,37192445,70372191,9259676,2717150,70367851,11365791,66832478,41481685,54058788,91240048,57803235,85023028,84904436,20681921,31904591,21397057,36312813,16971929,97011160,68000591,76315420,37620363,65038678,70596786,54868730,6038457,66250369,84349107,45428665,34946859,24826575,82726008,30653863,35996293,8129978,99021067,84187166,44842615,23134715,9257405,17068582,21001913,96906410,5482538,7300047,57961282,87160386,80555751,1239555,55753905,69579137,4434662,13231279,63628376,23740167,61741594,53666583,669105,96193415,80316608,49236559,36808486,33123618,61859143,96420429,74110882,68824981,20836893,65851721,86821229,35512853,60581278,99333375,75543508,76540481,61960400,53393358,62490109,33797252,61263068,60102965,45996863,10649306,92787493,51590803,18806856,21993752,30139692,37891451,72725103,82339363,40781854,2891150,82886381,86001008,30395570,76671482,77898274,57658654,30463802,36505482,44473167,21919959,90004325,15148031,12348118,74357852,93562257,93790285,65081429,90090964,48893685,17365188,92033260,95395112,81677380,67084351,73222868,78300864,40027975,86460488,77300457,4099191,7919588,47090124,74743862,50668599,9398733,33201905,247198,69786756,70800879,6525948,26063929,73021291,47708850,40686254,3880712,6153406,9058407,84684495,16380211,65338021,55770687,22129328,18833224,23569917,9599614,20867149,29065964,4091162,81898046,9603598,13173644,18411915,54199160,14326617,82979980,73786944,38365584,22997281,32590267,92215320,65047700,62803858,45848907,34493392,90457870,97940276,6505939,79191827,81853704,62693428,70420215,51149804,23848565,52204879,9575954,93515664,48675329,6157724,47298834,18663507,82052050,73124510,4985896,3294781,8807940,29819940,77187825,75820087,4508700,4064751,62552524,11923835,61982238,54606384,59109400,42947632,66137019,53084256,20885148,21289531,9886593,24915585,32161669,68128438,66903004,43501211,97641116,98943869,14731700,19898053,67793644,54427233,23194618,72793444,89214616,40677414,21673260,61623915,89419466,98653983,40356877,77301523,67474219,98130363,30218878,13468268,16054533,75986488,56484900,51047803,45407418,17857111,7104732,36930650,4515343,18131876,45049308,91664334,33061250,30163921,34497327,36685023,22721500,95010552,65017137,45990383,94076128,89078848,3633375,22450468,52613508,72357096,29188588,39171895,40152546,78549759,15064655,62430984,73617245,22766820,24733232,43322743,19457589,78561158,8115266,17727650,20486294,168541,39373729,72373496,79942022,49328608,53888755,77787724,32699744,33304202,26734892,5073754,3509435,83368048,65271999,41380093,75407347,72732205,63059024,27325428,89804152,77377183,37957788,24619760,88444207,20642888,28787861,13862149,78218845,5832946,59910624,82427263,57393458,26493119,66885828,89046466,11581181,44060493,85117092,55189057,17037369,8634541,67031644,11543098,52261574,58224549,31727379,48395186,33935899,69641513,48673079,20645197,31161687,79922758,56461322,3183975,96852321,46851987,93270084,92604458,75229462,30811010,13348726,7517032,6986898,48088883,20122224,36753250,92867155,22108665,36139942,91141395,69255765,2607799,99224392,50007421,26507214,83150534,91255408,44245960,40865610,21533347,4119747,72777973,62936963,74724075,76170907,22879907,14363867,90158785,75777973,57248122,98739783,73031054,74137926,68041839,1197320,47887470,99971982,33628349,38022834,7687278,89699445,1250437,31126490,58751351,9175338,88251446,79136082,36189527,61728685,68702632,30891921,4787945,43506672,58180653,55247455,45794415,6871053,63372756,69848388,16387575,51466049,6521313,80246713,44550764,16424199,99297164,88904910,38510840,19486173,76434144,72358170,44846932,63506281,45306070,4798568,42692881,65454636,47361209,72738685,30694952,36933038,38256849,68644627,56153345,16445503,68875490,68816413,97561745,17016156,34432810,61271144,15948937,29466406,30366150,56473732,69697787,61815223,32151165,34236719,83533741,46540998,73168565,86118021,41245325,10597197,44983451,59329511,25842078,98948034,14093520,54663246,17385531,75153252,18001617,94711842,17764950,32159704,90654836,36468541,6221471,8099994,73109997,53547802,14349098,44664587,92998591,2331773,51507425,72274002,3233569,59318837,36135,97281847,42073124,66319530,59371804,92398073,26114953,62762857,98648327,17957593,99125126,61928316,6793819,17539625,86301513,79821897,71469330,56955985,55960386,8651647,71920426,82178706,70727211,749283,42967683,90272749,73235980,83651211,9623492,14045383,75128745,1569515,67030811,78766358,51715482,34044787,79738755,85116755,90013093,22435353,86543538,26102057,86022504,17894977,93053405,84002370,58749,63152504,62051033,68110330,71333116,16099750,824872,14626618,51464002,31491938,26292919,30787683,77072625,37280276,33336148,32426752,69605283,10309525,38645117,64157906,55143724,42307449,94360702,83155442,72238278,5640302,92530431,9860968,22113133,44889423,47738185,28796059,62115552,36396314,15902805,8733068,26139110,95251277,82897371,62357986,21070110,78602717,41442762,26863229,55669657,34667286,60176618,28734791,32058615,58208470,24953498,72019362,16405341,23432750,15536795,18699206,33553959,29401781,36812683,41092102,41092172,97057187,70036438,54232247,14723410,45237957,11757872,55615885,85695762,38658347,59197747,3487592,16097038,51588015,67227442,90310261,92071990,92353856,85571389,84293052,3773993,7502255,51315460,48260151,94911072,17976208,60393039,99729357,72495719,2208785,40534591,37739481,43045786,94090109,16019925,77694700,45667668,37224844,17058722,18466635,76960303,64055960,94539824,89499542,10961421,12024238,86242799,33699435,93359396,92803766,50567636,83789391,93566986,57241521,28928175,15075176,71316369,79322415,65275241,13819358,51472275,47893286,45617087,874791,42644903,89996536,24168411,8373289,37560164,78909561,59614746,176257,6497830,96709982,43152977,88698958,83948335,56531125,86798033,90061527,17386996,43376279,62232211,83302115,73392814,66322959,59981773,62428472,29516592,92692978,91938191,4069912,61712234,66116458,71965942,45995325,55850790,70541760,61859581,74452589,10264691,89217461,44847298,87720882,87710366,83747892,7011964,95290172,76825057,59177718,4095116,54014062,7066775,49271185,12303248,72614359,37501808,81774825,6819644,20568363,16595436,79806380,8729146,45381876,23110625,82532312,38009615,4806458,37166608,61897582,68694897,74614639,81805959,9829782,1375023,81274566,27187213,38341669,29635537,7229550,83133790,15163258,57020481,11161731,10453030,78785507,54263880,7646095,8913721,44627776,19891772,40781449,88130087,97379790,67124282,16684829,30090481,22942635,91281584,33895336,67513640,99690194,75052463,27625735,99226875,84166196,48658605,26397786,54987042,1936762,49598724,82327024,59405277,16567550,68316156,24314885,47792865,66271566,70782102,81855445,82779622,43357947 +91957544,60102965,44842615,56424103,66250369,39801351,62803858,29959549,73168565,53666583,97379790,85023028,35192533,88047921,14093520,24619760,95957797,38061439,89637706,98648327,59177718,25842078,81172706,415901,16971929,74452589,99524975,84349107,4204661,34044787,62496012,14349098,45848907,39553046,89078848,47708850,23134715,40197395,19939935,6153406,90004325,12416768,9886593,74110882,8807940,71083565,42644903,72357096,77620120,35996293,70596786,4099191,59910624,67474219,77284619,9575954,82979980,30395570,84904436,33237508,93933709,20140249,33201905,10649306,29819940,46540998,7685448,55143724,6221471,24733232,42782652,20765474,66832478,68000591,65454636,48260151,55753905,14220886,79657802,89804152,51588015,15075176,20681921,78218845,22942635,77300457,16097038,26776922,2717150,19891772,96726697,58180653,18001617,19272365,95581843,49641577,94516935,90457870,8651647,55669657,74357852,89419466,73031054,6525948,75986488,19486173,58208470,53888755,93515664,60955663,3235882,92215320,34295794,40781854,57248122,14947650,77312810,32426752,16380211,99965001,81853704,76330843,13862149,43322743,9599614,18699206,6111563,75777973,42073124,64087743,34698463,24591705,34698428,71522968,70036438,90310261,18411915,89046466,76170907,44983451,99658235,59318837,34946859,12664567,61741594,8913721,176257,8733068,79821897,72274002,8055981,32699744,2208785,3773993,99861373,90090964,66428795,29994197,749283,8099994,91831487,40027975,29466406,7517032,67227442,43045786,85571389,53648154,21533347,8791066,88653118,73222868,83150534,10309525,78602717,69697787,82897371,4798568,36780454,75543508,58749,31904591,67281495,36505482,112651,7955293,70727211,85242963,36808486,62452034,37891451,4119747,39201414,26507214,44481640,7011964,41442762,30998561,28734791,74614639,36753250,7229550,12571310,56531125,73235980,76540481,54663246,50188404,5832946,35456853,32274392,11543098,76671482,69641513,22405120,83269727,18466635,30811010,38494874,45995325,45667668,97561745,30090481,6497830,3233084,508198,49236559,68644627,92604458,29065964,61271144,34236719,6986898,73786944,51315460,9257405,5111370,5970607,93053405,7182642,64848072,54517921,44348328,77272628,33797252,45790169,3183975,59371804,17386996,94360702,80316608,54868730,95290172,17058722,26114953,50668599,94711842,29834512,37957788,98653983,87710366,97783876,45919976,17037369,247198,50702367,61960400,66319530,30764367,24826575,89699445,51047803,95395112,27625735,82178706,44177011,2204165,98948034,28550822,89214616,98943869,70367851,84187166,40356877,81677380,82052050,92787493,14326617,36845587,26392416,4806458,20486294,30891921,63015256,8696647,29510992,27411561,47738185,83747892,32161669,13470059,81898046,48893685,9058407,73124510,49597667,46870723,23848565,68102437,20836893,526217,83651211,69255765,4091162,16387575,20568363,72278539,51507425,45049308,17539625,824872,31733363,28851716,49882705,15536795,68702632,62490109,45306070,89996536,67793644,18783702,30139692,55602660,33249630,23569917,40686254,68204242,74441448,44245960,80555751,43933006,37183543,18504197,47298834,40781449,45407418,66322959,70420215,17857111,669105,33304202,75229462,37739481,20642888,50007421,72725103,93566986,99224392,92803766,42692881,30771409,10961421,94076128,57020481,59329511,92398073,55850790,26493119,78561158,26744917,30543215,13231279,87598888,61859143,86543538,90272749,31126490,55247455,54762643,34493392,48774913,36580610,10358899,92530431,71657078,83789391,68041839,50806615,6505939,75820087,38008118,92692283,57359924,70004753,62693428,6871053,68824981,72777973,75153252,84406788,99549373,51426311,91664334,98739783,17016156,6157724,24058273,27185644,7502255,74724075,36812683,2607799,88904910,86001008,1204161,71333116,91802888,55470718,30163921,6808825,9860195,70541760,15064655,1375023,98371444,33553959,99125126,21397057,23110625,12348118,27325428,84684495,55615885,44844121,72495719,49271185,27041967,59501487,27665211,48360198,33431960,40984766,9603598,43501211,86118021,63372756,79191827,62740044,16595436,92692978,76315420,78589145,74743862,40677414,68816413,57658654,94090109,38645117,17764950,32590267,9398733,29029316,52204879,5267545,48892201,20885148,4434662,66663942,69786756,22721500,63506281,60176618,321665,53632373,168541,83155442,3233569,9623492,1250437,62051033,86242799,41380093,91938191,70800879,90061527,16405341,61712234,72019362,11757872,17957593,26734892,1936762,23194618,74137926,1431742,44915235,83083131,17894977,22766820,79806380,44889423,54987042,85117092,85116755,53802686,51472275,4095116,82886381,99333375,78300864,86821229,3633375,97281847,47213183,31727379,3880712,66045587,13819358,54014062,48673079,11161731,4199704,66137019,26102057,99604946,3294781,67084351,55189057,52293995,68694897,61373987,30653863,88698958,37435892,21070110,92033260,45428665,99971982,43357947,4668450,77787724,6793819,66667729,49328608,99729357,38009615,39986008,54606384,57240218,87720882,37280276,18833224,15163258,39847321,80014588,1022677,26292919,17068582,63967300,93562257,16684829,88251446,33123618,62430984,44784505,51715482,9175338,60430369,48675329,28796059,47361209,48088883,42967683,36312813,54199160,45996863,35092039,77072625,62936963,43152977,67451935,79942022,44473167,4978543,72358170,8729146,44479073,86301513,82726008,71920426,20122224,33628349,42307449,9829782,70372191,53547802,91141395,78766358,65880522,37675718,54263880,63152504,54427233,72238278,41245325,84293052,26863229,4064751,26998766,97057187,4985896,65017137,99021067,10366309,66903004,13173644,45617087,83133790,77377183,43376279,59109400,4508700,48395186,99917599,16445503,90013093,85711894,37192445,24915585,38256849,30569392,65074535,68110330,33061250,43506672,64098930,96193415,80246713,77301523,47887470,22200378,15148031,45237957,61982238,35512853,92998591,71300104,85695762,44627776,89499542,71469330,14626618,17727650,36933038,31161687,7919588,24953498,28787861,30218878,46851987,82327024,17081350,42237907,16019925,99690194,20645197,45381876,59197747,9188443,26139110,94539824,30366150,45075471,61263068,89811711,99226875,26664538,12024238,73392814,83948335,37659250,32159704,64055960,84166196,8128637,45794415,20867149,95726235,29635537,82779622,60106217,2331773,63059024,38341669,10453030,51590803,81855445,55770687,79136082,29188588,47792865,76434144,61859581,18663507,11923835,84127901,36135,92353856,66116458,2917920,22435353,16424199,36685023,61141874,88444207,91990218,62552524,67031644,76960303,72373496,72614359,77898274,31491938,22108665,19101477,50567636,52261574,60581278,61728685,18131876,36396314,82339363,5822202,75407347,63628376,90654836,92554120,5073754,34667286,54058788,4069912,44846932,38022834,19457589,33699435,1197320,97011160,87160386,17365188,39373729,98462867,48658605,56515456,37501808,40152546,22450468,68939068,22129328,39171895,36930650,5482538,65081429,41481685,18806856,86460488,69848388,51464002,44550764,29401781,65275241,99297164,82532312,65715134,96318094,14723410,17385531,67124282,17976208,6038457,64602895,77187825,21673260,9259676,7687278,33935899,14045383,84840549,96906410,78909561,4515343,21993752,10597197,59614746,75052463,42199455,1239555,82427263,56955985,96709982,34432810,51466049,1569515,83210802,14363867,93270084,56473732,15015906,29516592,73617245,62115552,21289531,22879907,22113133,66611759,84002370,72738685,81805959,16567550,99515901,30463802,59405277,76703609,15535065,96852321,7646095,15948937,58751351,83368048,21001913,72732205,77413012,67513640,86022504,83533741,47090124,7104732,8115266,49598724,36468541,11581181,41092102,65038678,97940276,57803235,69355476,92541302,68128438,56461322,44060493,89217461,76825057,3088684,77694700,61815223,78884452,66885828,33895336,69605283,75135448,79738755,95251277,52613508,26397786,91281584,71316369,67030811,93057697,91240048,32058615,72793444,56484900,57961282,79922758,56153345,20002147,25636669,96420429,41092172,2891150,5640302,14731700,37560164,88130087,16099750,82651278,55960386,33565483,30787683,15902805,1788101,7423788,6521313,60393039,3509435,93790285,3487592,29932657,86798033,47893286,17146629,96735716,6819644,57241521,91255408,8634541,874791,52060076,57393458,62762857,80251430,9860968,94911072,92071990,61897582,97641116,65271999,69136837,36139942,19898053,44664587,79322415,53084256,65047700,54232247,28928175,53842979,32250045,79880247,94595668,8129978,95010552,90158785,37166608,30694952,98130363,13348726,62428472,91727510,21919959,32151165,37620363,26275734,19376156,81274566,81774825,24168411,37224844,23432750,59981773,64157906,26063929,40865610,36189527,68875490,7300047,569864,51149804,65851721,69579137,13468268,75128745,66271566,11365791,23740167,57163802,38658347,22997281,44847298,65338021,27187213,58224549,78549759,10264691,16054533,62232211,83302115,42947632,61623915,62357986,92867155,45990383,34497327,93359396,7066775,8373289,68316156,40534591,24314885,38510840,33336148,73021291,4787945,53393358,12303248,78785507,71965942,73109997,38365584,61928316,70782102 +3233084,112651,42199455,53084256,81853704,33304202,97561745,24733232,99965001,3235882,56515456,38061439,26063929,29188588,75135448,31161687,73222868,45407418,5267545,38645117,6221471,1022677,55143724,76671482,30653863,62496012,59109400,92554120,38256849,321665,51047803,669105,17146629,63628376,74357852,15535065,92692978,86022504,44060493,45790169,33895336,33237508,89637706,96193415,65047700,60430369,67793644,42967683,42782652,60581278,8129978,30366150,62232211,30543215,37675718,29466406,26664538,7011964,10358899,59981773,83789391,92692283,37183543,81677380,15015906,55753905,33249630,12024238,61741594,8115266,45381876,21001913,75229462,44983451,33123618,526217,57961282,73617245,88653118,4199704,168541,7423788,72725103,54232247,85117092,50188404,1431742,29029316,6808825,13173644,84349107,13470059,34295794,63967300,10597197,824872,99658235,569864,68939068,67281495,36580610,16595436,29834512,3633375,80246713,81898046,83210802,66319530,7646095,57803235,33565483,66832478,7229550,30139692,55770687,35192533,3088684,69697787,68204242,62452034,98462867,66116458,76315420,51472275,72614359,65271999,44889423,44550764,31126490,78218845,83302115,41380093,83533741,99690194,93270084,48658605,24953498,84904436,39847321,98130363,17365188,72274002,72738685,28550822,43933006,15163258,37659250,36685023,36812683,55615885,56531125,14731700,415901,66611759,77787724,23194618,53842979,63015256,85571389,18806856,78884452,89214616,26998766,59318837,36505482,38658347,749283,77312810,92071990,39201414,4099191,7300047,92604458,54058788,43045786,95581843,1788101,92787493,86301513,65454636,32590267,22435353,77694700,61141874,43501211,88444207,29994197,16097038,34698428,76170907,70004753,10961421,4064751,30395570,55247455,54199160,66667729,82726008,27185644,16971929,72278539,96709982,16424199,37435892,63152504,36930650,39171895,18131876,3509435,17894977,52613508,45617087,61263068,70420215,1204161,70036438,66045587,99549373,61623915,82339363,94516935,1250437,70800879,82178706,46540998,9860968,62357986,93933709,69641513,14093520,22942635,57240218,16445503,80316608,61271144,88047921,65851721,62740044,93057697,68041839,77300457,80251430,10366309,72373496,50567636,9398733,50668599,98371444,17727650,78549759,4069912,5482538,59197747,9058407,90457870,16684829,23432750,19101477,47090124,32159704,24058273,69355476,54517921,4508700,65715134,9886593,36189527,77284619,20002147,68644627,90013093,33797252,17957593,98653983,12303248,2891150,30569392,83269727,19939935,9175338,96726697,29065964,30218878,57241521,61728685,91831487,44847298,12348118,54014062,72495719,7066775,75407347,64602895,78589145,44784505,45306070,55189057,26102057,71333116,25842078,70596786,49236559,75153252,4985896,55470718,91802888,22450468,40152546,84684495,78909561,24619760,8634541,18699206,66428795,93053405,66137019,90654836,20642888,40534591,22108665,34044787,84187166,11365791,33336148,247198,38008118,36780454,34493392,21070110,45237957,75986488,17539625,49271185,15148031,4668450,69786756,39986008,85116755,48673079,35092039,8696647,13231279,77072625,32274392,4119747,36753250,40356877,44627776,874791,8807940,82979980,46851987,38365584,33628349,90272749,37739481,52060076,27411561,43322743,87710366,9623492,14363867,64848072,57359924,51466049,29819940,56424103,61373987,21533347,9259676,74110882,53648154,83150534,14626618,21993752,27665211,92998591,54868730,13468268,15064655,14723410,68694897,66885828,30998561,23848565,22129328,48088883,63059024,35996293,37891451,15536795,22997281,40677414,63372756,45667668,58224549,56461322,60102965,90061527,24915585,508198,99917599,48774913,68875490,50806615,13819358,3880712,22113133,17058722,93790285,77620120,42237907,70372191,8128637,50702367,61859143,79657802,70541760,68000591,3773993,37166608,30463802,62936963,15075176,62430984,68702632,91664334,34236719,7955293,37620363,34946859,65038678,49641577,27325428,22879907,67031644,24591705,22200378,40865610,99226875,49597667,99125126,79191827,74743862,86001008,79922758,27625735,97011160,61982238,62693428,21289531,97940276,5111370,67227442,88251446,10309525,95290172,99297164,28851716,82427263,18504197,82651278,31733363,70727211,31904591,62552524,20122224,66250369,76434144,55602660,78300864,19891772,71469330,6793819,40197395,20867149,92541302,47738185,2204165,12416768,79738755,1239555,69136837,7502255,26114953,44481640,14349098,61859581,79821897,5970607,96735716,38341669,72777973,34497327,13862149,29510992,76540481,59614746,12664567,65338021,33201905,59405277,83133790,19376156,57163802,95010552,74724075,83083131,94595668,17016156,36135,50007421,18663507,86460488,73124510,6986898,1197320,92803766,62428472,95395112,41481685,86821229,44245960,92215320,26776922,4095116,48360198,67451935,73168565,73021291,26292919,36845587,16099750,99861373,64157906,64087743,59177718,9188443,20568363,44842615,86798033,41092172,66663942,30163921,8099994,26392416,45075471,61897582,23569917,63506281,73031054,78785507,51426311,24168411,98739783,91727510,56955985,26493119,9860195,72732205,4091162,84127901,85711894,65275241,94090109,54987042,48675329,87598888,38494874,20645197,59910624,46870723,19272365,54663246,36468541,41092102,6111563,52261574,65081429,22721500,37501808,57658654,59329511,54427233,44348328,45919976,33431960,26275734,8055981,20486294,36312813,40686254,53547802,82052050,99604946,40781449,69848388,8791066,67084351,66271566,45848907,10649306,51507425,77301523,99971982,11923835,16019925,29401781,34432810,30811010,73786944,12571310,65880522,39553046,14045383,43152977,69579137,52293995,24826575,57248122,45996863,59501487,6038457,9603598,36808486,74452589,44915235,90090964,14326617,3487592,82532312,99515901,94076128,71657078,29932657,6153406,59371804,95726235,92033260,60393039,40027975,61815223,53888755,88698958,17037369,78602717,54762643,78766358,16380211,71300104,13348726,95957797,47213183,43376279,76330843,28787861,18783702,18411915,60955663,34667286,25636669,73392814,75820087,7919588,47708850,14220886,93566986,97057187,44846932,32161669,97783876,93515664,85023028,35512853,91990218,20836893,3294781,62051033,55669657,68816413,18833224,72358170,15948937,28734791,44479073,96318094,28928175,19486173,53632373,86543538,47361209,71316369,88904910,28796059,52204879,98943869,55960386,6157724,31491938,51588015,89217461,81805959,99224392,51464002,58749,83368048,36396314,81855445,77187825,17386996,37192445,83651211,74614639,61928316,37560164,56484900,7182642,80014588,2917920,95251277,176257,87720882,7104732,66322959,30764367,77272628,89811711,86242799,1375023,70367851,74137926,72793444,37280276,84406788,26744917,45995325,6521313,32426752,69255765,6871053,33553959,62803858,81274566,84840549,5822202,43506672,99021067,42947632,94539824,89046466,86118021,17764950,89078848,7687278,68824981,7685448,76703609,33699435,69605283,89419466,62490109,5832946,74441448,6497830,68128438,18001617,91281584,4787945,44177011,91255408,34698463,17385531,81774825,4515343,3233569,65017137,49598724,40984766,32699744,96906410,9599614,58180653,2208785,17068582,93359396,20681921,1569515,71083565,99524975,43357947,5073754,20765474,16387575,30787683,8373289,76825057,72238278,97281847,16405341,73235980,8651647,6819644,30771409,4806458,47298834,4434662,89699445,42307449,53802686,48892201,4978543,45428665,67474219,49882705,81172706,87160386,96420429,94911072,23110625,68102437,39801351,83155442,7517032,48893685,38022834,48395186,73109997,15902805,23740167,19898053,11543098,79322415,2717150,26139110,64098930,91240048,44473167,66903004,3183975,8729146,60176618,36933038,10264691,26863229,83747892,99333375,27187213,38009615,77377183,85242963,22405120,90310261,77413012,9829782,45990383,98648327,62762857,75777973,35456853,96852321,67030811,11161731,20885148,78561158,47792865,16567550,37224844,72357096,22766820,51149804,91938191,31727379,94360702,58751351,4798568,29516592,29635537,45794415,92867155,27041967,49328608,54263880,40781854,92530431,39373729,93562257,94711842,71965942,14947650,70782102,85695762,41442762,82886381,82327024,32250045,51315460,29959549,30891921,5640302,56473732,57020481,92353856,54606384,6505939,6525948,26397786,42073124,10453030,9257405,20140249,76960303,90004325,92398073,33061250,56153345,84002370,26734892,4204661,36139942,71522968,44664587,21919959,75543508,65074535,26507214,62115552,84293052,17976208,18466635,42644903,23134715,11581181,51590803,2607799,79880247,33935899,19457589,98948034,57393458,16054533,71920426,21673260,2331773,42692881,60106217,24314885,67124282,51715482,68316156,97641116,97379790,89804152,21397057,83948335,88130087,91957544,79136082,55850790,41245325,99729357,79942022,17857111,37957788,90158785,91141395,64055960,17081350,80555751,47887470,53393358,9575954,68110330,1936762,32058615,38510840,75052463,89499542,8913721,82779622,44844121,72019362,58208470,30694952,8733068,47893286,61960400,32151165,79806380,53666583,11757872,77898274,82897371,61712234,75128745,30090481,89996536,48260151,84166196,45049308,67513640 +20002147,99965001,51047803,37183543,60581278,96193415,59177718,36780454,45617087,38645117,61982238,98462867,4064751,60430369,9623492,53084256,57961282,74357852,72274002,7300047,36753250,27665211,85116755,29834512,34236719,72725103,21993752,1431742,82979980,43376279,10358899,45407418,6793819,90090964,17764950,98371444,99549373,88047921,14626618,88698958,33249630,77072625,44983451,58751351,31126490,22450468,17068582,39847321,98130363,97011160,65338021,79922758,83210802,2917920,29029316,67793644,40356877,14220886,45237957,22129328,77787724,95290172,44550764,92554120,55189057,86022504,5970607,72238278,63059024,64848072,94090109,3233084,40984766,57803235,14045383,38022834,47090124,75543508,36505482,38658347,45428665,91802888,83269727,73786944,22997281,12664567,15015906,36580610,4069912,14731700,40152546,59371804,65271999,73222868,81898046,53842979,56153345,67281495,96852321,31904591,40781449,7229550,84684495,45794415,20642888,28796059,30395570,99524975,93270084,54058788,8807940,37659250,55753905,50668599,83789391,64087743,13173644,82726008,44479073,15148031,93057697,36312813,14947650,34698428,72732205,569864,66667729,49236559,99515901,55850790,57393458,24058273,62496012,7646095,78602717,14363867,96906410,10309525,27411561,72357096,16567550,88653118,50702367,8099994,37501808,63967300,77272628,67451935,25636669,9603598,92353856,12348118,24168411,30764367,62740044,22435353,68875490,30218878,20568363,15535065,76671482,32590267,6986898,23194618,52060076,92803766,59614746,59109400,72373496,30463802,18699206,87720882,13468268,49641577,21289531,415901,96735716,68204242,53648154,4806458,82339363,62936963,39171895,70004753,18131876,91831487,19939935,18806856,44889423,10264691,56461322,61728685,96318094,37435892,96726697,62430984,81274566,30771409,29188588,89214616,69697787,30569392,65047700,54014062,8129978,41092102,95957797,37620363,65715134,76315420,30653863,99971982,824872,16445503,38365584,93562257,27041967,32161669,7919588,34493392,71333116,53802686,16595436,73617245,61623915,77694700,68041839,11923835,40534591,73031054,61263068,51472275,69641513,84840549,22200378,3088684,48260151,15163258,81677380,69355476,9058407,71965942,91990218,43152977,89078848,66271566,46870723,38061439,1197320,30163921,33797252,63152504,44844121,33431960,56473732,9257405,32250045,42644903,10366309,37166608,80316608,90457870,20122224,75153252,77284619,44348328,33123618,51590803,3487592,36812683,34497327,93053405,176257,55143724,63628376,27185644,35192533,669105,70367851,62452034,65275241,61859581,43322743,51588015,66885828,96420429,45995325,6808825,91664334,66832478,35092039,13819358,45381876,66903004,29959549,3233569,42967683,1936762,79738755,70727211,84904436,57241521,56515456,4099191,20645197,49598724,12416768,63015256,74110882,94516935,7011964,89804152,5832946,55615885,78561158,93933709,70036438,85117092,99917599,54232247,112651,9886593,7182642,72793444,92215320,99297164,26998766,39553046,55470718,83150534,8373289,14093520,7423788,3880712,4095116,91727510,20140249,33304202,24314885,19101477,20486294,46851987,29994197,1204161,77312810,72278539,61815223,18466635,64602895,41481685,23432750,40197395,8115266,74743862,74614639,87710366,65038678,80251430,75135448,23569917,9188443,73021291,66250369,54987042,70420215,54606384,21001913,37891451,43933006,90310261,96709982,91255408,78218845,11161731,39373729,19486173,28550822,7502255,90272749,33336148,97783876,32159704,33201905,77413012,15536795,2204165,42199455,68644627,74724075,4434662,93515664,44481640,26292919,47738185,92033260,44784505,6521313,99604946,78589145,526217,54663246,4668450,42782652,15064655,6157724,90013093,43045786,66663942,33553959,78909561,26063929,74441448,39986008,92604458,85242963,59318837,48893685,14326617,36685023,42237907,71083565,59981773,321665,11365791,42073124,66322959,71316369,31733363,66428795,65081429,92398073,10597197,44664587,26139110,67474219,80014588,40781854,54517921,57248122,16684829,6221471,8128637,5073754,53632373,86460488,16405341,78884452,6819644,29932657,83533741,2717150,68000591,17365188,78785507,11543098,70800879,16099750,31161687,59501487,20765474,89419466,16971929,33237508,16019925,84406788,8791066,21673260,33565483,40686254,45306070,61859143,51464002,83083131,79821897,52613508,22108665,92692978,84349107,17539625,70596786,16380211,40027975,24619760,84293052,6497830,56531125,51466049,38510840,45990383,33699435,17016156,76960303,90061527,87598888,49271185,9860968,26392416,28928175,508198,44245960,27187213,42947632,83302115,26275734,34946859,85711894,53547802,84127901,82427263,44060493,45996863,47361209,36808486,48892201,50567636,67084351,50188404,77620120,69605283,1250437,69848388,29819940,49328608,72019362,12024238,79657802,77377183,75229462,3294781,16097038,13231279,92787493,16424199,5267545,44846932,99333375,53393358,82052050,73235980,68694897,6153406,65017137,48673079,45790169,26397786,76434144,71657078,15902805,30694952,55770687,26493119,36135,76703609,30543215,82178706,44842615,18663507,66137019,85571389,53888755,7955293,20867149,43501211,71300104,56424103,89217461,66045587,16387575,58224549,60393039,54762643,22721500,81853704,24591705,50007421,62693428,61897582,26664538,31491938,97561745,62552524,37560164,73168565,92541302,80555751,76540481,68702632,42307449,30139692,38008118,61741594,5111370,65851721,95395112,89637706,17727650,35996293,78549759,22766820,4515343,48088883,12571310,82886381,47893286,99861373,8696647,1788101,43357947,88251446,95010552,24733232,32058615,23110625,90654836,26776922,247198,30787683,69579137,4787945,72777973,86242799,61271144,14723410,97057187,4204661,97641116,40865610,32274392,37192445,83368048,82779622,68816413,75407347,98648327,91141395,60102965,69136837,8651647,22113133,17385531,874791,16054533,86543538,24826575,36930650,59197747,48774913,72738685,9259676,62762857,3509435,8634541,98739783,43506672,19376156,41092172,55247455,44627776,9829782,42692881,20885148,62428472,75777973,68128438,26114953,88444207,71469330,168541,58208470,31727379,62115552,85695762,34432810,61373987,70541760,3773993,94076128,36933038,83651211,61141874,69786756,99125126,94595668,45848907,66116458,1022677,58180653,68939068,18504197,22879907,57240218,29466406,4119747,24915585,70372191,81805959,56484900,2891150,88904910,36189527,82651278,57163802,92071990,81172706,5482538,44177011,54199160,3633375,18833224,5640302,17976208,77300457,7066775,1239555,84187166,20681921,55960386,64055960,79136082,92692283,45919976,26102057,92530431,40677414,39201414,13348726,17957593,92867155,57658654,62051033,75128745,67030811,71920426,36468541,9398733,86798033,82897371,12303248,8733068,25842078,64098930,17894977,21070110,8729146,86001008,50806615,41245325,38494874,71522968,79880247,48658605,92998591,26507214,74137926,90004325,29516592,1375023,52261574,99690194,27625735,4091162,19891772,62490109,36396314,5822202,89996536,36139942,79806380,94911072,9175338,81855445,79942022,2607799,78766358,2208785,28734791,6505939,63372756,76825057,21397057,26734892,99226875,89046466,93566986,35512853,99021067,41380093,94539824,46540998,32699744,8055981,58749,21533347,77898274,68110330,86118021,64157906,54427233,73124510,7685448,55602660,30891921,14349098,89811711,37739481,78300864,32426752,17857111,6038457,17386996,91281584,10649306,32151165,9575954,83948335,3183975,29065964,66611759,4199704,1569515,65880522,91240048,68824981,60955663,73392814,61960400,44473167,94711842,7517032,66319530,88130087,35456853,22405120,98943869,47887470,13470059,99224392,47298834,56955985,17146629,68102437,65454636,79191827,77187825,9860195,83155442,17081350,76170907,11581181,23848565,72358170,89699445,91938191,97940276,26863229,30366150,80246713,45075471,30811010,38341669,75820087,27325428,6525948,30998561,72614359,75986488,6871053,62803858,34698463,81774825,93359396,83133790,20836893,57020481,13862149,82327024,18783702,99658235,17058722,86301513,49597667,15075176,51426311,62232211,22942635,3235882,34667286,60106217,95581843,79322415,90158785,73109997,93790285,48675329,11757872,63506281,7104732,29635537,74452589,2331773,59910624,749283,19457589,86821229,4508700,29401781,61928316,52293995,33895336,34295794,62357986,29510992,23134715,49882705,37957788,84002370,45667668,69255765,38009615,97379790,19272365,98653983,26744917,47792865,18001617,18411915,94360702,65074535,59329511,53666583,85023028,67227442,9599614,61712234,41442762,15948937,95251277,76330843,68316156,10453030,34044787,28787861,75052463,77301523,38256849,4985896,84166196,52204879,70782102,48360198,72495719,33628349,67513640,37224844,89499542,47213183,59405277,36845587,39801351,7687278,87160386,67124282,37675718,47708850,97281847,99729357,24953498,10961421,23740167,95726235,51507425,28851716,51715482,67031644,33935899,55669657,44847298,17037369,8913721,37280276,4798568,57359924,54868730,60176618,48395186,51149804,19898053,98948034,82532312,54263880,4978543,33061250,44915235,91957544,6111563,30090481,45049308,21919959,83747892,51315460 +21993752,99515901,14731700,47361209,37620363,77312810,14220886,6986898,55247455,32159704,66832478,55753905,10309525,59197747,52261574,86543538,76960303,66250369,15015906,1569515,96193415,26139110,62552524,53666583,67793644,45237957,9886593,14626618,29834512,83150534,77284619,15902805,17764950,78602717,1204161,4064751,48892201,67281495,77413012,14947650,51047803,24619760,62496012,50702367,30463802,55470718,54606384,59177718,92033260,9257405,93933709,27665211,68939068,67124282,48260151,40865610,91664334,83651211,75543508,669105,18504197,70420215,54014062,82178706,48675329,99524975,20486294,35192533,77072625,41245325,42692881,80014588,6808825,96726697,33935899,12571310,23848565,95726235,84684495,73786944,33249630,44842615,31904591,92787493,79821897,45848907,66322959,45381876,54199160,99604946,1197320,62936963,98130363,72373496,82726008,38494874,3487592,96420429,66045587,17068582,69355476,16405341,56153345,42644903,55850790,26292919,85242963,48893685,66137019,49641577,12664567,36930650,10366309,47090124,78909561,81677380,70036438,65038678,7011964,50007421,53632373,53802686,73124510,69848388,89214616,74137926,77272628,88653118,78766358,40781854,3233569,90272749,34497327,36753250,7300047,569864,38009615,36780454,30764367,16595436,88904910,75128745,32151165,81855445,20002147,32161669,88047921,24915585,89217461,14349098,68816413,71316369,63372756,11757872,26998766,44177011,17386996,45428665,60430369,74614639,22997281,20122224,96735716,29959549,81274566,59910624,85116755,16445503,51464002,45617087,68875490,24058273,99917599,74357852,4119747,16387575,35092039,44889423,76330843,40356877,64848072,37501808,21533347,57248122,33061250,42307449,74110882,98371444,66116458,37224844,92692978,50567636,99965001,51426311,75153252,22766820,53393358,95251277,79136082,72357096,6819644,415901,53084256,7182642,66667729,44983451,62428472,43376279,1250437,44847298,247198,94090109,46870723,21673260,18699206,71333116,69786756,8634541,78589145,64087743,78884452,36808486,6221471,86460488,19101477,99125126,60102965,42199455,28550822,12024238,93053405,44784505,37675718,68644627,176257,3235882,61728685,65715134,4508700,38658347,5832946,74724075,20140249,8807940,69255765,60106217,37183543,29932657,58751351,65017137,87598888,44479073,14093520,90310261,90013093,44473167,3294781,83948335,62452034,26863229,44481640,22942635,53648154,82339363,84127901,38061439,526217,17385531,64602895,99021067,20645197,24591705,56515456,22450468,17365188,82052050,68694897,77300457,34044787,72614359,84840549,6793819,10358899,88444207,13348726,37891451,62803858,40534591,61859581,36312813,79922758,50806615,30163921,92541302,3233084,28796059,17081350,86001008,42967683,39171895,94911072,93566986,27187213,29065964,51466049,54663246,31733363,66428795,39553046,73168565,62051033,49328608,68102437,21289531,61373987,57803235,18466635,9398733,63015256,85571389,30395570,824872,34236719,55143724,34493392,38645117,76315420,40781449,63059024,34946859,73235980,26734892,508198,73031054,65851721,78218845,2204165,57241521,89046466,69641513,19939935,22129328,66319530,8729146,68000591,54427233,32058615,99226875,99224392,92353856,33201905,7104732,4099191,11543098,4515343,54517921,65271999,43152977,16971929,40677414,26102057,55189057,13173644,82897371,97783876,39801351,42237907,48774913,74452589,5111370,17894977,5073754,92071990,321665,43501211,50188404,35512853,26275734,70596786,59371804,61982238,96318094,49236559,40197395,99861373,8651647,6871053,32699744,63967300,86301513,9259676,80316608,30366150,99549373,61263068,42073124,26392416,9603598,76170907,66903004,94711842,95290172,31491938,36505482,1431742,62430984,15535065,75777973,30139692,1936762,75229462,8696647,2891150,82979980,91802888,74743862,66611759,6505939,78561158,67451935,68702632,97011160,29029316,112651,55960386,33304202,6153406,69579137,23194618,92215320,76434144,81898046,76703609,76540481,73222868,22405120,56531125,64157906,39847321,16567550,81805959,24826575,45407418,22113133,73392814,42782652,62693428,30891921,77620120,18411915,26776922,30653863,4069912,57961282,57393458,2917920,68316156,57240218,56461322,53547802,49271185,79657802,40027975,75986488,44060493,29819940,40152546,85711894,77694700,30694952,45075471,33628349,24953498,14045383,5970607,47792865,26397786,32426752,30787683,57020481,48360198,83789391,80555751,69697787,16684829,46851987,94516935,44627776,83747892,8733068,85023028,44550764,68824981,59981773,29994197,12348118,47738185,31126490,19898053,82651278,16054533,45306070,20836893,58749,63506281,4091162,72274002,64098930,7517032,7687278,3880712,91831487,70541760,18806856,70800879,14723410,24168411,98739783,62232211,9623492,48395186,91240048,14363867,70727211,17957593,29188588,8373289,98648327,58208470,75135448,27041967,72732205,89637706,12416768,72495719,38022834,66271566,3183975,33699435,57658654,36812683,49598724,41092172,40686254,97057187,37659250,82327024,9829782,20568363,90654836,18833224,28928175,96906410,71657078,7919588,15163258,90090964,69136837,61141874,10649306,60581278,20765474,76671482,46540998,36933038,15948937,6038457,71920426,99297164,34295794,8115266,73021291,32250045,4204661,35456853,44846932,91990218,30998561,16019925,24314885,36139942,2331773,58180653,27411561,35996293,72793444,4095116,9175338,17727650,4434662,92998591,7229550,72019362,68128438,34698463,92554120,83210802,4798568,57359924,22200378,90457870,71469330,81853704,18131876,86118021,61712234,1788101,92692283,94539824,56424103,26507214,19376156,1022677,79880247,48088883,37560164,54232247,49882705,83368048,60176618,47298834,18783702,95395112,79806380,39373729,44844121,93057697,22435353,89499542,92604458,15148031,7423788,56484900,68041839,50668599,8099994,51149804,13819358,72238278,44664587,58224549,42947632,8129978,38365584,80246713,45990383,25842078,99690194,16097038,72738685,61271144,67030811,99971982,21070110,37166608,70004753,17857111,69605283,2208785,30811010,84349107,30543215,2717150,4787945,61623915,83155442,31161687,32590267,84293052,41442762,29516592,83533741,55770687,77787724,36135,52204879,68110330,3088684,7685448,19486173,19272365,79322415,26114953,70372191,59501487,7502255,79738755,71522968,4668450,91141395,47893286,34698428,94360702,16380211,37739481,73617245,16099750,17037369,18663507,47708850,13468268,90004325,87720882,3633375,89804152,71965942,87160386,52293995,15075176,65275241,48658605,8913721,67474219,72358170,3509435,5482538,20867149,84166196,97281847,168541,7646095,10961421,45794415,61741594,37435892,33431960,98462867,37957788,98653983,9575954,11161731,6497830,65338021,84904436,92398073,75407347,19891772,89419466,84187166,49597667,17976208,89811711,61928316,92530431,38256849,72777973,97561745,26063929,97940276,29510992,27625735,36580610,68204242,65081429,89996536,36845587,99658235,93515664,23134715,85117092,89078848,51715482,55615885,45049308,98948034,11365791,71300104,45996863,31727379,18001617,65454636,87710366,45995325,27185644,61815223,30090481,77301523,43045786,8128637,4806458,59614746,63152504,95581843,30569392,33123618,59109400,56473732,86022504,33565483,97641116,96852321,39986008,66885828,96709982,13862149,38341669,17016156,84002370,86242799,78549759,86798033,7955293,90061527,29401781,95957797,62740044,36468541,47887470,17058722,5267545,23432750,17146629,51472275,21919959,52613508,53842979,14326617,40984766,88698958,93790285,15064655,45667668,86821229,30771409,84406788,27325428,77377183,43357947,65880522,98943869,24733232,67084351,26493119,22721500,52060076,60955663,63628376,11581181,79942022,59405277,91727510,65047700,20642888,39201414,83083131,59318837,26744917,99333375,23110625,93270084,6157724,54987042,89699445,81172706,9860968,33336148,95010552,67227442,91938191,82886381,72725103,13231279,51590803,79191827,91957544,33797252,66663942,82779622,62357986,54263880,62762857,54058788,57163802,94595668,38008118,33895336,55669657,22879907,51507425,10597197,45790169,67513640,51588015,23569917,53888755,85695762,41481685,78300864,92803766,29466406,65074535,93359396,19457589,77187825,61960400,36685023,82427263,15536795,54868730,47213183,9058407,56955985,78785507,61859143,83302115,7066775,59329511,71083565,20885148,48673079,70367851,10453030,72278539,28734791,874791,6525948,43506672,9599614,41092102,43933006,97379790,5640302,94076128,75052463,28851716,6111563,33237508,74441448,30218878,55602660,22108665,37192445,80251430,62115552,9188443,44348328,43322743,16424199,749283,8055981,28787861,83269727,11923835,23740167,20681921,26664538,81774825,88251446,76825057,91255408,36396314,13470059,5822202,34667286,82532312,60393039,44245960,29635537,44915235,4985896,37280276,12303248,34432810,9860195,36189527,21397057,33553959,2607799,75820087,83133790,25636669,4199704,10264691,3773993,1239555,45919976,54762643,88130087,73109997,21001913,93562257,17539625,91281584,4978543,90158785,70782102,64055960,32274392,8791066,61897582,67031644,41380093,92867155,77898274,99729357,1375023,38510840,6521313,51315460,62490109 +6505939,68816413,40197395,89214616,78766358,34493392,35092039,61728685,71920426,67084351,99515901,22879907,36930650,99917599,26397786,48260151,67124282,85242963,17016156,83789391,65017137,70004753,68000591,27665211,62496012,20140249,28851716,13173644,74110882,12348118,99965001,53842979,36753250,50806615,83210802,9603598,75128745,57020481,59371804,20122224,57248122,73124510,8651647,14326617,97783876,44177011,69579137,18783702,98943869,11161731,10366309,47298834,86118021,72495719,73031054,59910624,35512853,45667668,72725103,96852321,92998591,6986898,96906410,26275734,29994197,23848565,66322959,51047803,53547802,62936963,669105,19457589,67030811,63506281,9623492,46870723,54517921,50007421,99524975,98739783,84840549,90457870,67281495,5073754,33123618,8696647,39171895,32151165,74441448,44842615,74614639,42073124,62430984,60102965,91281584,33201905,55247455,65715134,415901,7687278,71657078,95957797,6808825,75777973,33249630,50188404,85711894,51464002,62693428,247198,5111370,9188443,95395112,61859143,41092172,99604946,89419466,79880247,64098930,82897371,17058722,22766820,34295794,92787493,2331773,526217,14045383,82979980,10597197,53802686,9398733,6153406,48893685,56955985,88904910,13468268,86460488,76330843,61141874,88653118,63372756,95010552,1197320,55753905,30366150,3294781,13231279,44473167,66428795,14947650,69848388,59501487,30771409,88047921,3233084,45919976,89078848,90310261,75543508,54014062,24591705,36780454,99549373,29834512,70036438,49328608,45996863,32274392,45995325,54606384,89804152,76434144,64087743,37739481,8913721,16380211,19272365,6521313,9575954,82886381,3183975,74137926,30090481,26744917,43933006,64602895,99861373,92692978,7502255,49597667,33895336,508198,33553959,99971982,75986488,5822202,95581843,63967300,27185644,54199160,22129328,77312810,88444207,40356877,26114953,19376156,62232211,47738185,68875490,56515456,33797252,59197747,36505482,30395570,61982238,67227442,8634541,18411915,66903004,84293052,83155442,6221471,44479073,79821897,99125126,33237508,18466635,55615885,44889423,68644627,12571310,39553046,31126490,38008118,17764950,36812683,66045587,50702367,80555751,18131876,24826575,51149804,31727379,18699206,48673079,66611759,76540481,35192533,83747892,69786756,83651211,3233569,45407418,43357947,9599614,99658235,79738755,70367851,54762643,75820087,96735716,44060493,78218845,90272749,4069912,10961421,6111563,67793644,569864,21919959,70596786,16405341,62452034,92398073,39801351,70420215,26392416,14349098,92215320,76671482,6871053,53666583,62803858,22942635,61897582,40534591,18663507,23194618,37620363,37957788,23432750,22997281,11757872,91664334,81677380,1431742,97940276,72777973,17068582,7423788,71316369,1022677,37183543,84349107,20642888,29819940,24058273,92071990,49271185,16971929,55189057,61960400,749283,45790169,80251430,90090964,40865610,71300104,45428665,26507214,55602660,44983451,30998561,59177718,26998766,51715482,85571389,68204242,48774913,92530431,70727211,43376279,81172706,94090109,30811010,62357986,95290172,83368048,50668599,14093520,68694897,4119747,83269727,70782102,48088883,86001008,72732205,76960303,32058615,73617245,38022834,176257,66137019,56153345,49641577,52293995,78884452,91831487,84904436,42307449,41442762,59109400,21070110,77377183,22108665,72373496,9175338,69255765,73235980,37166608,95726235,17976208,21533347,22113133,3880712,79322415,46851987,70800879,10309525,71522968,54263880,61623915,82532312,7517032,874791,62051033,61373987,99297164,38494874,30569392,64055960,36135,91957544,74357852,42967683,43506672,98653983,19939935,68102437,81774825,2607799,19486173,7300047,85116755,49236559,66271566,39373729,13348726,20486294,49882705,77620120,9058407,38341669,16445503,72274002,44245960,43045786,98948034,63059024,92604458,65271999,68824981,43322743,5267545,53084256,3509435,98130363,75407347,57241521,74724075,53632373,77187825,55850790,32161669,53648154,34698428,29959549,7955293,23569917,52261574,29466406,31904591,90004325,77413012,28928175,2204165,56424103,73021291,23134715,97057187,61815223,53888755,42782652,5970607,81274566,60955663,56473732,40686254,85023028,16019925,65338021,89046466,79191827,47213183,68041839,21993752,36685023,36845587,94711842,51507425,37675718,30891921,45848907,65038678,30463802,45306070,39847321,29188588,92353856,78602717,26493119,54868730,4091162,168541,38009615,17727650,29401781,43501211,99224392,97011160,61712234,88698958,19898053,91990218,20002147,34698463,1204161,79136082,4787945,26292919,82178706,16097038,35456853,92541302,47792865,73168565,91240048,73222868,27041967,40781854,23740167,60106217,47887470,48360198,1788101,33565483,99021067,57803235,8791066,76170907,26102057,18833224,30543215,93566986,17957593,66832478,86301513,77694700,48675329,66250369,27187213,20681921,56484900,89811711,84187166,39986008,94076128,6157724,69605283,33628349,3235882,33304202,69641513,2917920,88130087,99729357,40152546,79657802,8099994,30139692,14220886,15064655,4515343,54663246,36312813,24953498,42237907,9257405,17081350,7182642,98648327,321665,17857111,26664538,29635537,41481685,67031644,15148031,44550764,80316608,72238278,84127901,2208785,91802888,45237957,87598888,37280276,90013093,28550822,90654836,82339363,22200378,66667729,38365584,19891772,20765474,17037369,29065964,30787683,40781449,82726008,4095116,38061439,20885148,96193415,22450468,44348328,77898274,37224844,42199455,69697787,99226875,92033260,65275241,96726697,15163258,30694952,36139942,93790285,87160386,78589145,65851721,31491938,97561745,66319530,62740044,94595668,52613508,13470059,92554120,26063929,85695762,85117092,84684495,9860195,7685448,26139110,31161687,6819644,89217461,30764367,15015906,23110625,71083565,93515664,32250045,78300864,93053405,68939068,96318094,77072625,78909561,18806856,70541760,84002370,61271144,24168411,29932657,18001617,72357096,92692283,65047700,1375023,24733232,77272628,24619760,13819358,62552524,55470718,36396314,73392814,22721500,95251277,96420429,27411561,67513640,9259676,37659250,82052050,68316156,40984766,56531125,74743862,11581181,93933709,65880522,76703609,55143724,36933038,7919588,44784505,19101477,44915235,14626618,41092102,51426311,1250437,15535065,30163921,15948937,71333116,34497327,7229550,89637706,93562257,47708850,38645117,61928316,67474219,77300457,6497830,83150534,45381876,57393458,69355476,8373289,29029316,40677414,33061250,72019362,21001913,72793444,37501808,17385531,28796059,43152977,94360702,69136837,11543098,7104732,67451935,18504197,82327024,79806380,65074535,37192445,60430369,28787861,1239555,92867155,5832946,89499542,44846932,16054533,66885828,56461322,65081429,74452589,99333375,45075471,32699744,58751351,63015256,41245325,50567636,45990383,82651278,4204661,78561158,6793819,824872,45617087,10453030,47090124,11365791,21289531,86821229,80014588,35996293,45049308,97379790,83533741,44481640,84166196,17386996,65454636,8733068,59405277,83948335,7011964,81898046,31733363,92803766,76825057,52060076,36468541,64848072,75229462,75135448,21673260,26734892,47893286,3773993,33935899,32426752,8729146,57240218,90061527,1936762,57163802,15902805,21397057,98371444,94516935,36808486,8807940,10649306,86543538,42644903,112651,15075176,4806458,36189527,41380093,76315420,83133790,34432810,14363867,72614359,26863229,37435892,16387575,5640302,83083131,4508700,91938191,48658605,4798568,49598724,72278539,10358899,6525948,48892201,42692881,51588015,16567550,77787724,34044787,86798033,7646095,34946859,47361209,81805959,87710366,58224549,91255408,51472275,55669657,42947632,14731700,44844121,93359396,79942022,8129978,62115552,53393358,63628376,62762857,97641116,44847298,73109997,99690194,17539625,12416768,61859581,37891451,68702632,72738685,54427233,20836893,37560164,30653863,16099750,3633375,40027975,33336148,5482538,59981773,4668450,57961282,72358170,66116458,55960386,48395186,64157906,17365188,81855445,54058788,2717150,4434662,77284619,71469330,16595436,38256849,32590267,25842078,71965942,3487592,51466049,51590803,58749,34236719,8128637,86242799,12664567,27325428,54987042,89699445,84406788,86022504,39201414,44627776,3088684,8055981,75153252,13862149,61263068,66663942,63152504,10264691,14723410,60581278,91727510,32159704,27625735,20645197,78549759,51315460,97281847,52204879,75052463,83302115,4064751,15536795,89996536,4099191,2891150,20568363,6038457,29516592,44664587,36580610,58208470,60393039,24314885,9886593,45794415,12024238,78785507,59614746,28734791,11923835,68110330,81853704,33699435,94539824,54232247,88251446,16684829,94911072,4985896,9829782,8115266,20867149,17146629,93057697,25636669,87720882,68128438,93270084,46540998,1569515,4199704,17894977,79922758,16424199,24915585,90158785,7066775,73786944,22405120,59318837,57658654,59329511,22435353,29510992,57359924,96709982,61741594,12303248,70372191,82779622,38658347,77301523,38510840,55770687,30218878,80246713,91141395,33431960,58180653,82427263,26776922,98462867,62490109,4978543,60176618,9860968,34667286,62428472 +83789391,7687278,3235882,32159704,76540481,29466406,24058273,10366309,20885148,46870723,38061439,70596786,33336148,99658235,68000591,38658347,65074535,36812683,3773993,33628349,6871053,72495719,61373987,4119747,66271566,67793644,63059024,81853704,42782652,85116755,39847321,45848907,76434144,63967300,75229462,39201414,60430369,79942022,48893685,36396314,53084256,36312813,61263068,81805959,68644627,84904436,66045587,37659250,48360198,3487592,9058407,95957797,94711842,96193415,44627776,40984766,3233084,83533741,17058722,72732205,74357852,34432810,52261574,59197747,168541,24591705,45667668,2917920,16971929,33201905,9175338,24619760,60106217,80014588,70420215,33061250,96726697,29932657,29029316,74441448,93057697,99524975,4798568,62051033,60581278,66667729,78766358,15015906,75153252,62452034,86118021,39801351,88047921,31491938,62936963,12348118,76315420,69355476,35092039,93933709,70800879,23848565,9603598,95581843,59109400,89699445,71316369,55247455,69697787,8729146,70727211,17539625,44842615,15535065,14626618,27185644,72019362,72614359,69579137,76671482,72738685,75777973,77272628,85023028,55770687,44177011,66428795,8129978,19272365,10358899,36753250,99125126,65851721,36808486,36685023,26392416,44060493,62496012,66250369,6819644,88653118,13470059,71657078,15163258,26292919,63628376,247198,7229550,81274566,17068582,25842078,62803858,40677414,73222868,6793819,7955293,71469330,19939935,33565483,20002147,68316156,82427263,92787493,19891772,45407418,86821229,91255408,46851987,32151165,99515901,71083565,33935899,49328608,54663246,874791,48774913,9623492,50188404,2717150,77284619,83210802,55753905,53802686,75543508,19376156,88444207,80246713,72274002,70367851,30463802,68816413,82339363,93359396,415901,36135,68824981,2204165,89419466,53547802,89214616,91802888,34493392,72725103,16405341,5073754,70541760,97940276,72777973,62357986,4064751,59177718,45919976,77620120,15148031,82726008,4668450,9259676,45790169,31727379,81677380,28796059,15948937,44889423,40865610,22108665,27325428,29834512,85711894,5970607,23569917,89046466,6808825,29994197,37560164,569864,47792865,49236559,112651,40197395,37620363,30569392,98371444,7502255,89078848,58749,90457870,8128637,66663942,45990383,98739783,76703609,17365188,34295794,67031644,77312810,21993752,51047803,11543098,43376279,92692978,65338021,16387575,99021067,23194618,39553046,21533347,82897371,73124510,32590267,16019925,92998591,99297164,57359924,22721500,17957593,74743862,95010552,4099191,25636669,53666583,71333116,73617245,39171895,70782102,45075471,87720882,36468541,38365584,20765474,44847298,57803235,16097038,54199160,42307449,77413012,47213183,11365791,27665211,91831487,4199704,78909561,8733068,66611759,61859581,78300864,86022504,53632373,18806856,94090109,6497830,61859143,40152546,92803766,1204161,26998766,17081350,21070110,92554120,66137019,37224844,18504197,2607799,1431742,84840549,84293052,40534591,35192533,37183543,35512853,49641577,86460488,71920426,50702367,28851716,56531125,4515343,1022677,15075176,59318837,84127901,8099994,5111370,17764950,47361209,72278539,92604458,17857111,36933038,5267545,36580610,61982238,6986898,321665,93515664,81855445,41092172,68204242,30366150,4204661,48673079,43322743,73109997,21001913,14326617,30218878,91727510,53842979,50567636,14093520,40027975,18783702,17894977,11581181,37192445,83133790,55850790,90061527,34698463,37675718,90310261,17146629,82532312,92541302,6153406,87160386,30787683,64848072,98130363,83155442,96318094,7423788,50668599,30811010,61712234,30163921,31733363,84187166,67124282,97641116,44664587,34698428,56153345,9575954,87598888,15064655,43501211,79922758,51472275,59981773,67030811,70372191,66885828,80316608,4069912,49271185,94360702,54606384,20486294,18466635,54014062,26275734,99224392,38008118,96709982,5832946,61928316,18833224,22435353,98948034,19457589,7919588,36189527,37166608,30653863,44983451,29959549,20836893,75128745,77377183,16445503,80555751,64602895,73168565,29065964,51315460,26493119,73235980,42947632,34946859,38494874,75820087,79821897,26734892,34497327,73786944,43152977,95726235,69786756,26863229,63015256,3088684,67474219,77301523,26063929,93053405,73392814,99690194,59329511,95251277,71300104,526217,66903004,79136082,92215320,3183975,10961421,51590803,14045383,33249630,78884452,24168411,92692283,70036438,9188443,51464002,55669657,33123618,50007421,68875490,99549373,72357096,65880522,66832478,56424103,45049308,79191827,3880712,30139692,84349107,98462867,97379790,33237508,23432750,8651647,669105,29819940,85117092,31904591,94516935,97561745,46540998,13173644,52060076,70004753,43933006,83302115,61741594,1375023,67451935,37501808,86301513,29635537,78602717,9599614,35996293,60102965,32250045,55189057,83651211,82779622,3633375,75986488,52293995,83083131,508198,39986008,13468268,76960303,90654836,61815223,6221471,17037369,82979980,9257405,14349098,45237957,35456853,33431960,92033260,2891150,82651278,87710366,85242963,77072625,16380211,4091162,48088883,51149804,76330843,93270084,53888755,97057187,1569515,16595436,40356877,65271999,41481685,17385531,68041839,93790285,99333375,49597667,4787945,86543538,58224549,19101477,78561158,93562257,40686254,37739481,22129328,61271144,54427233,15536795,69848388,8373289,56484900,23110625,3294781,9886593,57241521,69641513,77300457,1788101,1197320,27187213,69255765,28928175,68110330,21919959,36505482,22942635,96735716,64157906,68702632,7685448,42692881,65715134,10453030,30771409,61960400,44479073,20568363,62740044,99861373,7104732,17976208,40781449,96906410,86242799,77787724,43357947,91938191,37891451,78785507,61141874,14220886,83150534,60176618,2208785,44550764,44245960,92398073,18131876,9398733,26776922,62762857,53393358,81898046,55602660,44915235,59501487,8696647,99604946,26139110,7011964,43506672,78218845,22405120,9860968,91664334,19486173,11161731,57020481,44481640,824872,21673260,45996863,68694897,4434662,62693428,28550822,88698958,54263880,33797252,51426311,79806380,20140249,22450468,51507425,44784505,33895336,59371804,24953498,11757872,10649306,74137926,7646095,53648154,30764367,34236719,47887470,84684495,48892201,44846932,9829782,12303248,38256849,65017137,72373496,79880247,68102437,94911072,23134715,29188588,29401781,74110882,89811711,61728685,69136837,89217461,94539824,67227442,75052463,49598724,20642888,62232211,47893286,91990218,52613508,6111563,27625735,9860195,11923835,33553959,32426752,42199455,37957788,45428665,85571389,58208470,31126490,5482538,55960386,89804152,59614746,55143724,65047700,77187825,38645117,28734791,90004325,14731700,24915585,16054533,56955985,18699206,22766820,43045786,57658654,7517032,34044787,64055960,90013093,41245325,91281584,26744917,20867149,3509435,8807940,68939068,30543215,1250437,80251430,6038457,48260151,14363867,84406788,24733232,85695762,36780454,42967683,38022834,31161687,23740167,44844121,33304202,97011160,22997281,24826575,55470718,3233569,19898053,50806615,99226875,47298834,62430984,93566986,89637706,10597197,82052050,176257,66319530,68128438,26397786,56461322,30891921,97783876,12664567,5822202,16684829,21289531,8634541,57240218,81172706,88904910,89996536,76170907,74614639,58751351,59910624,91957544,97281847,57163802,17016156,14723410,20645197,48658605,59405277,45306070,63152504,18001617,49882705,27411561,62490109,84166196,5640302,82178706,29516592,67084351,33699435,749283,30395570,78589145,32161669,38341669,94595668,41442762,26114953,22200378,47090124,4978543,36930650,4985896,6525948,6521313,30694952,26507214,77694700,7182642,88130087,92353856,95395112,21397057,89499542,66322959,54987042,52204879,74724075,75407347,12024238,1239555,67281495,41092102,36139942,54868730,88251446,86001008,57961282,99917599,42073124,32699744,96420429,54232247,83368048,30090481,79657802,8791066,26102057,73031054,20681921,54762643,57248122,74452589,82327024,26664538,72358170,29510992,24314885,54517921,48395186,56515456,61623915,37435892,6505939,65275241,16567550,98943869,44348328,63506281,64087743,99965001,86798033,61897582,16099750,40781854,18411915,96852321,60955663,36845587,10309525,28787861,75135448,71965942,17727650,90272749,22879907,42644903,14947650,16424199,62552524,95290172,90158785,79322415,44473167,83948335,81774825,4095116,41380093,39373729,7066775,12416768,13862149,62428472,56473732,79738755,47738185,45995325,48675329,45794415,51715482,22113133,20122224,15902805,54058788,8055981,77898274,27041967,65081429,12571310,2331773,63372756,55615885,78549759,58180653,8913721,51588015,57393458,45381876,13231279,98648327,62115552,67513640,4508700,65038678,65454636,32058615,32274392,7300047,34667286,6157724,38009615,66116458,82886381,72793444,30998561,90090964,42237907,76825057,91141395,84002370,92867155,69605283,8115266,92071990,51466049,47708850,60393039,83269727,73021291,91240048,4806458,99971982,45617087,64098930,71522968,94076128,18663507,83747892,1936762,13348726,98653983,92530431,38510840,37280276,10264691,17386996,13819358,99729357,72238278 +45990383,21993752,16019925,30543215,90272749,59371804,63059024,91664334,45237957,27665211,77413012,16099750,73031054,68110330,3233084,45428665,3487592,44889423,66137019,51472275,75153252,62496012,39986008,45919976,62936963,68204242,53393358,26863229,92398073,85116755,70004753,87598888,59177718,86001008,90090964,61928316,6793819,68041839,53547802,79821897,22108665,30764367,69848388,88653118,77301523,12348118,2917920,55247455,11543098,99333375,47738185,14947650,36812683,7687278,7502255,63967300,2891150,6153406,86022504,50567636,29188588,54427233,16054533,74110882,89699445,13468268,62490109,66611759,89804152,42782652,98130363,29819940,19891772,4119747,40865610,17857111,7955293,62452034,44784505,3088684,4787945,73235980,26139110,61815223,33237508,77284619,99658235,99917599,9575954,50188404,86543538,69579137,13231279,18466635,4204661,31733363,36685023,60955663,31126490,45407418,45667668,30771409,95290172,61859581,64848072,74614639,17764950,97783876,54014062,55753905,57240218,99861373,15015906,53648154,82327024,72357096,10366309,34236719,93933709,36780454,96726697,3183975,20140249,78300864,53666583,71469330,47361209,61271144,33935899,62428472,23194618,88047921,7182642,65047700,25636669,78589145,6986898,37435892,62051033,18504197,28550822,89811711,91990218,43501211,8733068,55143724,65271999,16445503,91255408,35092039,749283,23848565,93053405,415901,8696647,2717150,3294781,60581278,45848907,41092102,40197395,17068582,61728685,20867149,6505939,82979980,9886593,71316369,12416768,20486294,42967683,56153345,8128637,27411561,8807940,80555751,10649306,32590267,72274002,22942635,91141395,40356877,84904436,45996863,99224392,81898046,18833224,39847321,95726235,96193415,67793644,66903004,47893286,93562257,6525948,48675329,77187825,18783702,15902805,3880712,34493392,32250045,68644627,32159704,4798568,40152546,89046466,29834512,71333116,17058722,77072625,31904591,35456853,57393458,17539625,6808825,99524975,66832478,49641577,3233569,508198,92554120,38022834,65454636,89217461,70596786,49328608,81774825,34295794,31491938,86242799,92998591,78602717,83651211,82726008,49236559,59318837,24058273,85242963,14045383,22997281,32161669,79136082,71920426,70541760,48088883,72358170,91957544,26744917,89214616,32426752,39171895,82532312,19898053,57163802,9603598,88698958,72019362,12024238,57961282,48774913,14626618,99515901,82339363,80014588,14220886,91240048,55470718,83789391,62803858,73168565,97379790,51507425,26292919,75777973,9829782,92033260,57241521,23110625,37957788,37501808,20122224,5970607,98948034,42073124,82427263,74357852,26507214,44550764,65074535,9398733,19457589,73786944,84406788,67451935,63628376,1788101,37166608,22766820,61373987,36312813,91938191,11161731,11581181,247198,4985896,37620363,96318094,73222868,669105,20765474,96852321,14093520,68102437,57803235,59197747,66663942,29065964,38365584,44481640,92692978,51047803,18699206,569864,92867155,86460488,37183543,33797252,83269727,44177011,54663246,52060076,36933038,36930650,93057697,29510992,26063929,82178706,36753250,30694952,26776922,96420429,79880247,30163921,83533741,4434662,65851721,80316608,33628349,8634541,77620120,38494874,18001617,75135448,76540481,30787683,89637706,66667729,62232211,86821229,97641116,27325428,5482538,83210802,44479073,34698463,55189057,38008118,17727650,30998561,90004325,61263068,51590803,15075176,19939935,89419466,24591705,83155442,82651278,40686254,27625735,2607799,90457870,78766358,40677414,54762643,93515664,68875490,69786756,51588015,112651,77312810,84684495,5832946,43045786,99297164,64098930,26664538,8115266,47298834,44842615,82897371,63372756,55669657,55602660,9257405,84293052,93359396,84127901,74441448,98462867,47792865,40781449,85023028,54199160,67030811,43152977,95957797,99125126,38061439,8129978,37739481,48260151,9188443,79738755,26392416,71657078,51715482,68816413,36189527,87160386,65081429,82779622,57658654,16424199,6221471,4064751,40781854,48360198,5822202,47708850,70367851,10961421,73021291,26998766,7229550,66322959,24826575,70727211,89078848,7011964,45075471,1569515,68824981,79322415,81855445,20642888,22129328,36505482,39553046,9623492,82886381,58749,72238278,33201905,98371444,4806458,321665,37560164,9058407,64157906,28928175,79942022,81172706,17957593,34698428,34667286,35192533,16387575,5073754,45306070,67124282,40027975,94516935,66116458,24915585,56473732,4508700,90654836,78549759,4668450,63015256,40984766,67227442,59614746,69255765,58180653,47887470,61982238,85571389,99604946,52261574,65017137,17081350,63506281,3633375,76825057,68000591,58751351,92787493,68702632,84840549,77377183,81677380,57020481,72777973,94539824,19486173,95581843,4515343,95010552,99549373,76330843,5111370,23569917,73617245,76315420,51464002,6497830,54987042,76960303,79806380,29466406,65880522,85695762,85117092,17365188,43376279,526217,4099191,36468541,27041967,44473167,24733232,72725103,99965001,30218878,18806856,11923835,70420215,16097038,43506672,97940276,89996536,9860968,23134715,29932657,30891921,34497327,16595436,20568363,6819644,14723410,38645117,62552524,1197320,36808486,29994197,83368048,874791,10358899,54058788,45790169,17386996,50668599,81805959,57359924,50007421,84002370,21289531,53632373,26493119,59501487,37891451,59109400,42692881,33061250,14349098,76434144,69136837,67513640,88904910,62740044,92803766,6111563,36135,1250437,87720882,88444207,7423788,50702367,30139692,97281847,43322743,35996293,44844121,43933006,42644903,74137926,45995325,64602895,94090109,99971982,97057187,86798033,72373496,66250369,28796059,51466049,49597667,80251430,51149804,92541302,44915235,44664587,1936762,63152504,57248122,58224549,83150534,29029316,65038678,95251277,99729357,38658347,6038457,99021067,17037369,68694897,94911072,78218845,14363867,19376156,13173644,78561158,96709982,84187166,8913721,72732205,30569392,78909561,15163258,824872,38009615,49271185,91802888,24314885,9259676,68316156,66885828,82052050,17146629,4069912,7104732,44060493,74452589,54868730,54606384,71522968,39801351,13470059,68128438,1431742,37675718,16380211,66428795,14326617,42307449,11365791,28851716,30090481,28734791,17385531,73124510,33699435,56424103,14731700,34044787,48892201,21673260,28787861,62430984,43357947,24619760,17976208,18663507,26114953,64087743,75407347,75128745,70372191,69605283,69697787,73392814,44627776,77787724,33431960,98653983,24168411,9860195,67474219,9599614,78785507,7685448,98739783,2204165,86118021,93270084,70036438,22200378,48893685,93790285,97561745,83302115,36580610,37224844,37192445,7300047,94076128,6871053,30366150,1022677,65275241,53842979,81274566,41245325,88130087,76671482,20885148,91727510,97011160,176257,33895336,90158785,29959549,95395112,66271566,36139942,69355476,99690194,86301513,20836893,1375023,26734892,22450468,76170907,3773993,61960400,30395570,47213183,55770687,56461322,94360702,45617087,10309525,15536795,61623915,68939068,51315460,90013093,22721500,74743862,67281495,8729146,20681921,45794415,44348328,32699744,30811010,92604458,15535065,33249630,92692283,79657802,47090124,4091162,56531125,9175338,3235882,72278539,96735716,17016156,11757872,61859143,56484900,89499542,16405341,4095116,60176618,92215320,81853704,54517921,42199455,99226875,59910624,12571310,20002147,77272628,26102057,61141874,83083131,83948335,60102965,75543508,55960386,42947632,71083565,32151165,54263880,24953498,3509435,65715134,94711842,55615885,77694700,22405120,23740167,59981773,31161687,44846932,41092172,32058615,53802686,77898274,60430369,44983451,15948937,16567550,44245960,61712234,49598724,4978543,62115552,12664567,8099994,16971929,58208470,33565483,70800879,37659250,48658605,168541,72793444,30653863,32274392,27187213,2208785,25842078,74724075,53888755,54232247,50806615,33336148,18411915,34432810,8651647,55850790,36396314,64055960,38256849,92530431,13348726,61741594,48673079,29516592,62693428,70782102,66319530,92353856,66045587,96906410,22879907,79191827,5640302,10597197,23432750,46851987,30463802,7066775,10453030,53084256,69641513,72614359,71300104,21533347,41481685,67084351,34946859,75986488,21919959,61897582,87710366,77300457,59405277,80246713,48395186,75820087,71965942,75229462,39373729,44847298,83133790,45049308,56955985,21001913,67031644,45381876,91281584,85711894,7517032,51426311,10264691,15064655,15148031,2331773,35512853,12303248,41380093,84349107,98943869,26275734,40534591,62762857,78884452,1204161,26397786,8373289,90061527,22435353,7646095,8791066,38510840,13862149,19272365,73109997,52293995,72738685,22113133,41442762,91831487,65338021,76703609,83747892,27185644,42237907,39201414,33553959,8055981,31727379,33123618,56515456,6521313,38341669,21070110,79922758,92071990,20645197,94595668,62357986,46870723,6157724,46540998,37280276,33304202,1239555,36845587,17894977,29635537,84166196,60393039,59329511,7919588,75052463,49882705,93566986,72495719,21397057,5267545,19101477,29401781,60106217,52204879,13819358,98648327,88251446,90310261,4199704,18131876,52613508,16684829 +17764950,4099191,90013093,55189057,60430369,38061439,34497327,66667729,26998766,16405341,49236559,22405120,68041839,14093520,18806856,29834512,11923835,79821897,76540481,3509435,20765474,72274002,61859581,68204242,15535065,91240048,51472275,50702367,97561745,95581843,28928175,52204879,2331773,20002147,40197395,2208785,2607799,96735716,72357096,8099994,16097038,99965001,51047803,45428665,66045587,71300104,44627776,9058407,59501487,22721500,35512853,73392814,80316608,80246713,90310261,87598888,39553046,49271185,71920426,34236719,98130363,415901,69355476,99226875,57803235,89046466,99515901,7646095,38658347,87720882,6497830,61271144,30366150,55247455,73617245,8733068,20140249,68000591,58749,18466635,33336148,46540998,94090109,33895336,65271999,99971982,4515343,70800879,38365584,47361209,54663246,62936963,36753250,36780454,73235980,44784505,54987042,82178706,76170907,99524975,29029316,73168565,62803858,40356877,53632373,9623492,37192445,91664334,72793444,36312813,2717150,96906410,34432810,89214616,76703609,50188404,98943869,33061250,78589145,99917599,99224392,30764367,10358899,65074535,3183975,20642888,37739481,14349098,94539824,76671482,29188588,44842615,82651278,18783702,45794415,62740044,99549373,4668450,9599614,44844121,96193415,60176618,38645117,26493119,26664538,68644627,89996536,93562257,17539625,36933038,34493392,10597197,73124510,32274392,49641577,84002370,16445503,47893286,30463802,85242963,95957797,86460488,62496012,90158785,53547802,28796059,25636669,78602717,76434144,78909561,6521313,95726235,46870723,61373987,96852321,35996293,51466049,526217,43152977,66322959,73021291,59371804,77272628,7955293,14220886,85116755,81898046,21993752,67793644,40781449,36505482,27325428,3487592,39201414,16099750,79922758,63506281,70372191,42073124,84904436,89078848,85117092,18131876,1204161,168541,99297164,18833224,64602895,38022834,33201905,7687278,77620120,77413012,78218845,17081350,73786944,508198,93933709,44177011,34946859,92803766,92215320,80014588,95395112,1431742,35192533,26744917,59109400,81853704,33249630,44889423,83651211,85571389,824872,38341669,10366309,77694700,68102437,66250369,29994197,61263068,73222868,44481640,86118021,82979980,18001617,84187166,37501808,43376279,20486294,31161687,34044787,45848907,26507214,32161669,31126490,84684495,42692881,70596786,9603598,4798568,67451935,22113133,53666583,62232211,66885828,8373289,22450468,61623915,26397786,26275734,74357852,83083131,16567550,55470718,29065964,22942635,7502255,98371444,59981773,69641513,26863229,45996863,4119747,5267545,63967300,5482538,77072625,50806615,67474219,26139110,54014062,71333116,65047700,83133790,56531125,48088883,3633375,23569917,19486173,72725103,88130087,19272365,24058273,79136082,26102057,44479073,71965942,90457870,17727650,29959549,67124282,36468541,19376156,24619760,30787683,12024238,26392416,42199455,5970607,44983451,33699435,75543508,29819940,61741594,91281584,14626618,1569515,52261574,75777973,91831487,16054533,8651647,79322415,70004753,61728685,27411561,28851716,11581181,21289531,98948034,93057697,32151165,95010552,59197747,21070110,61982238,97281847,37620363,30163921,10309525,9575954,29635537,3880712,18411915,56955985,55770687,81855445,18504197,76315420,62490109,4199704,63059024,82897371,7229550,51590803,38256849,62452034,33797252,8791066,11757872,57248122,5111370,53802686,77312810,96318094,43045786,97783876,58180653,9829782,89699445,37675718,14947650,74743862,3088684,22879907,9398733,17146629,36580610,92604458,68816413,9860968,68316156,42782652,72732205,86242799,45075471,16595436,45617087,37891451,6505939,42237907,77284619,61815223,9257405,72738685,14326617,45407418,96726697,91802888,39171895,67030811,98648327,83789391,37183543,86798033,57240218,40152546,6153406,13231279,8807940,70727211,77377183,19457589,37560164,51464002,67227442,17016156,85023028,88251446,30543215,16971929,17957593,1197320,45667668,89217461,62693428,70036438,44915235,61960400,874791,36396314,99861373,92692978,56461322,15536795,49328608,40027975,93270084,88904910,10264691,13468268,41442762,99658235,11543098,56424103,19939935,61928316,32590267,82339363,55850790,6221471,4787945,19101477,32426752,58751351,98462867,86022504,49598724,17068582,41481685,7011964,6819644,24168411,12348118,91141395,83302115,6871053,1936762,92554120,17386996,6793819,33237508,57658654,12303248,15148031,81805959,18663507,8129978,60581278,54427233,4806458,53648154,83155442,23134715,40984766,64055960,26063929,27041967,73031054,4091162,69605283,89804152,75128745,96420429,66428795,9175338,14723410,22997281,74452589,7919588,48360198,96709982,56515456,2891150,43501211,3235882,92787493,4064751,87710366,43357947,20122224,19891772,41245325,31904591,78884452,1022677,2917920,92398073,50567636,55753905,8913721,4434662,63015256,65081429,72373496,40865610,92033260,4985896,61712234,35092039,36189527,18699206,30811010,31491938,67084351,112651,57241521,23110625,84840549,39801351,81677380,91990218,69786756,54606384,69255765,75052463,33628349,52060076,82427263,38008118,71083565,99729357,74724075,62051033,97940276,9188443,13862149,13470059,92071990,55960386,1788101,20885148,30139692,31733363,36135,74137926,95290172,55602660,59177718,79191827,29466406,48675329,34698463,1250437,36808486,30771409,59405277,55669657,33304202,569864,90090964,8055981,88653118,19898053,2204165,83210802,59910624,92353856,30653863,92998591,47792865,60955663,49597667,70420215,48260151,62552524,30998561,77187825,86001008,60102965,66137019,39847321,17037369,60106217,20568363,9886593,13173644,1375023,27665211,3773993,81172706,30569392,94516935,98739783,65715134,58224549,16684829,38009615,29932657,7423788,23194618,27625735,15163258,26114953,32250045,68875490,66116458,50668599,53842979,45381876,70541760,32159704,93359396,15015906,63628376,93790285,75407347,247198,72278539,669105,10453030,38494874,93053405,57393458,30891921,44847298,36139942,78785507,54232247,64848072,68702632,48892201,82886381,68939068,65017137,38510840,84293052,21673260,22766820,22108665,76960303,37435892,99604946,41092172,71316369,16387575,93515664,36930650,71469330,84349107,80555751,12416768,72358170,20645197,33123618,54058788,81274566,5832946,3294781,57163802,57961282,66319530,64098930,74614639,66903004,321665,72019362,65338021,36812683,54263880,41092102,62762857,23740167,99333375,27187213,5640302,54199160,86301513,71657078,93566986,82532312,15075176,94076128,77898274,44060493,67513640,54868730,11161731,15064655,37166608,44846932,59318837,39986008,92692283,6986898,62430984,72614359,92541302,66832478,45995325,29510992,3233084,30090481,44664587,69136837,51507425,6038457,16019925,74110882,66271566,47090124,72495719,47738185,78766358,37659250,40686254,53084256,12664567,82726008,55143724,21533347,3233569,83150534,89419466,34698428,69579137,54762643,48774913,48673079,24314885,72777973,97379790,99690194,63372756,94595668,45237957,13348726,90061527,31727379,4069912,75229462,88047921,30694952,6111563,88698958,97057187,56153345,24915585,90654836,176257,56473732,49882705,26292919,17365188,52613508,36685023,16380211,83368048,45919976,62115552,64157906,40781854,91957544,32699744,40534591,58208470,8696647,51315460,5822202,17894977,8115266,97011160,46851987,89499542,54517921,91727510,71522968,34667286,86543538,6525948,75135448,23848565,7182642,75820087,43322743,15902805,80251430,28734791,69848388,8634541,34295794,7517032,20836893,22129328,44550764,79806380,10649306,77787724,33553959,86821229,26734892,42307449,69697787,83747892,43506672,87160386,48893685,79657802,4978543,41380093,53888755,84127901,6808825,72238278,4204661,11365791,44245960,68128438,65275241,81774825,7066775,25842078,78561158,97641116,45790169,8729146,4095116,42644903,30395570,76330843,24733232,40677414,51426311,39373729,61859143,8128637,47213183,57359924,42967683,57020481,15948937,99021067,75986488,99125126,44473167,7685448,65454636,79880247,56484900,47298834,82052050,90004325,35456853,91255408,20867149,94360702,48658605,9860195,68824981,83533741,47708850,29401781,36845587,51715482,75153252,4508700,24953498,17385531,84166196,17857111,37280276,21919959,84406788,22200378,67281495,45990383,37224844,66611759,10961421,59614746,94911072,43933006,27185644,66663942,82779622,17976208,65880522,59329511,70367851,14045383,16424199,32058615,28550822,48395186,26776922,12571310,44348328,5073754,33431960,24591705,78300864,42947632,89637706,14731700,749283,67031644,29516592,14363867,62428472,45306070,89811711,79942022,74441448,60393039,13819358,65038678,68694897,63152504,68110330,33935899,51588015,76825057,78549759,7300047,94711842,33565483,22435353,7104732,52293995,77300457,90272749,21397057,50007421,28787861,21001913,45049308,20681921,30218878,91938191,73109997,88444207,61897582,62357986,83948335,77301523,85711894,61141874,24826575,53393358,1239555,92530431,98653983,95251277,51149804,17058722,47887470,79738755,92867155,85695762,55615885,64087743,6157724,65851721,37957788,70782102,83269727,9259676,82327024,23432750 +4099191,14626618,72373496,89214616,27665211,75543508,65074535,50567636,81855445,7687278,9623492,526217,73124510,60430369,508198,66250369,83302115,35092039,29994197,54199160,61263068,38061439,30090481,44481640,51047803,8913721,15535065,29029316,63628376,86821229,68644627,17764950,168541,5832946,95726235,61373987,70800879,19939935,32250045,82052050,47090124,6871053,89078848,10366309,9886593,59371804,98739783,66137019,4064751,42967683,67793644,43501211,55247455,61271144,99658235,63506281,13173644,72614359,56515456,30891921,10649306,93933709,47792865,70036438,15536795,68824981,43376279,16424199,7011964,97783876,96318094,34432810,91664334,80316608,31904591,74137926,6986898,55770687,26744917,26292919,79922758,33123618,16445503,68204242,26392416,34698428,70372191,247198,2917920,24591705,54663246,50188404,68694897,24058273,71657078,13470059,78766358,16971929,54606384,20122224,69355476,95290172,33797252,10358899,93053405,6153406,45075471,99965001,74743862,62452034,22108665,11365791,93562257,10309525,81805959,65038678,4668450,47708850,34295794,32161669,5111370,34946859,75229462,74724075,39986008,29065964,90457870,7517032,17385531,21993752,3633375,34497327,4515343,43322743,53666583,14093520,1569515,87598888,63059024,76703609,59318837,33201905,92554120,18833224,22879907,98371444,1022677,26139110,95395112,54868730,52613508,6793819,35192533,63967300,26734892,8634541,7229550,15064655,59109400,36812683,75777973,30395570,18466635,99861373,31161687,37659250,58751351,47213183,18783702,26063929,98130363,42947632,33304202,8651647,68702632,66667729,16567550,36930650,85117092,37675718,61728685,78785507,95957797,44784505,30139692,8733068,32159704,79738755,11581181,824872,46870723,60102965,25842078,90013093,11923835,57240218,79821897,39171895,42782652,91957544,1431742,32699744,9575954,88047921,70367851,3509435,81898046,26863229,40865610,45995325,67281495,56484900,26507214,94711842,98462867,321665,72019362,1204161,47738185,84904436,66611759,96193415,8696647,40781449,72274002,96420429,41380093,76540481,67227442,90272749,75986488,19101477,8373289,20002147,20885148,43152977,44889423,30764367,61960400,51149804,70420215,73031054,37183543,55753905,70004753,59177718,62740044,99297164,85116755,38341669,75153252,112651,63372756,4787945,81853704,18411915,93359396,7646095,48774913,85023028,33565483,19272365,42644903,38494874,69848388,569864,77377183,55143724,62803858,48260151,84840549,7423788,4119747,18131876,36780454,82339363,54987042,44847298,99226875,45407418,48892201,44842615,94516935,77312810,17016156,14363867,54014062,79191827,92033260,669105,83368048,56531125,415901,18699206,2331773,30366150,29834512,33553959,36580610,85242963,8055981,98948034,62936963,76170907,49328608,72725103,93566986,8729146,80014588,57658654,75407347,49236559,94076128,81677380,9398733,66885828,59197747,44245960,17068582,28796059,91240048,6808825,45794415,28851716,74614639,56153345,77300457,39201414,34236719,95010552,86301513,93790285,30998561,31733363,45049308,55470718,28928175,27185644,88653118,57163802,18806856,28734791,99604946,29819940,27041967,67513640,3773993,30787683,99333375,69579137,6497830,34698463,33249630,73168565,76671482,50702367,6525948,97940276,14326617,49641577,23569917,86460488,57803235,21673260,61982238,68939068,30218878,92692978,37192445,82886381,90004325,53802686,3294781,98648327,96906410,43045786,71300104,98943869,17957593,38658347,47361209,53648154,77787724,35456853,8807940,24314885,55960386,74110882,29959549,7919588,92215320,45667668,23848565,45919976,36753250,74357852,65271999,66903004,99125126,33061250,62693428,46851987,19891772,87710366,36312813,88444207,69605283,65338021,36808486,67031644,69136837,77694700,53547802,66045587,9829782,29635537,51472275,79136082,86001008,91831487,65715134,33895336,72495719,86543538,37891451,29510992,84684495,32426752,6819644,4798568,56955985,15948937,62490109,51588015,22450468,37739481,17365188,37957788,54232247,39553046,16380211,62357986,15075176,12571310,16019925,97281847,24733232,17146629,73222868,72738685,19486173,99524975,52204879,83150534,36505482,66663942,57248122,32058615,67124282,22997281,62115552,24619760,82897371,11161731,8129978,72238278,17727650,57359924,45617087,77284619,18663507,40027975,91281584,3235882,92398073,44348328,44479073,16595436,48893685,92998591,73392814,2607799,30694952,42199455,16097038,16054533,80251430,60955663,49597667,54058788,53632373,17539625,68000591,17976208,31126490,71333116,37166608,30569392,12416768,4806458,40152546,7685448,66319530,54427233,84166196,48360198,26114953,3088684,88904910,20836893,77620120,7955293,34044787,78884452,37620363,5822202,91938191,77072625,4985896,48088883,59501487,42307449,74441448,99224392,34493392,85571389,22721500,70596786,53842979,50668599,15902805,61859581,91990218,47887470,40534591,69255765,40197395,12664567,61859143,18001617,22129328,92803766,49598724,16387575,14349098,62496012,20681921,82726008,36685023,72732205,56424103,41092102,17386996,78561158,30463802,12348118,33336148,32274392,97379790,30543215,15148031,41481685,67084351,38008118,35996293,58224549,80246713,5970607,76330843,52293995,99515901,29466406,61623915,51466049,99021067,62430984,80555751,3487592,40984766,20765474,37501808,6221471,13862149,92071990,83651211,97561745,99917599,84187166,11543098,68816413,79942022,63015256,38365584,10961421,92604458,2208785,92530431,86118021,36396314,21533347,1936762,92692283,84349107,61141874,81274566,9599614,45237957,44177011,26397786,62051033,89499542,26664538,48658605,19376156,69641513,17894977,84127901,23194618,26998766,40781854,44550764,4434662,3183975,67030811,10597197,92787493,78909561,73617245,40677414,8128637,93515664,29516592,22405120,54762643,33628349,33699435,31727379,86022504,4204661,71920426,57241521,51464002,55189057,49271185,96726697,21070110,78549759,1788101,3880712,93270084,5267545,12024238,65081429,73021291,13231279,79880247,96735716,60106217,19457589,7066775,22200378,43357947,20486294,68128438,23740167,22766820,41092172,87720882,45381876,30653863,23432750,65047700,9603598,29932657,66428795,79657802,82979980,20568363,76960303,84293052,36139942,2717150,95581843,82532312,18504197,27325428,60176618,9188443,39801351,22942635,77413012,91727510,64098930,30771409,90310261,28550822,88251446,20642888,73235980,44473167,54263880,42073124,78218845,14731700,39847321,85711894,55602660,3233084,89699445,50806615,98653983,20140249,53084256,7300047,59910624,64157906,60581278,71316369,77272628,24168411,99549373,4978543,51315460,37435892,70727211,83083131,94090109,44915235,66271566,65017137,7502255,51590803,41442762,37224844,65275241,9257405,66832478,7104732,4199704,66322959,79806380,72777973,59329511,68102437,17058722,75128745,97057187,94360702,45996863,95251277,24953498,76434144,97641116,78589145,82178706,9058407,51426311,71083565,94539824,82427263,58180653,67474219,45428665,77187825,33431960,30811010,26102057,89637706,99690194,13348726,25636669,26493119,14723410,22113133,89419466,83269727,44664587,86242799,94595668,44983451,58749,84002370,51507425,72358170,1197320,14045383,89217461,16684829,83789391,36933038,44846932,15163258,47893286,2891150,8099994,90061527,71965942,71522968,30163921,4091162,78602717,99971982,28787861,48395186,27625735,23134715,89996536,6505939,94911072,50007421,68875490,56461322,9259676,10264691,1375023,73109997,17081350,58208470,93057697,84406788,33237508,81774825,4069912,64087743,73786944,9175338,45848907,77898274,89046466,32151165,5482538,68316156,40686254,57961282,26776922,66116458,55615885,52261574,26275734,14947650,83533741,62232211,27187213,42692881,35512853,56473732,45306070,61712234,75820087,20867149,16405341,14220886,21919959,62552524,176257,13819358,8115266,29188588,32590267,83210802,9860968,71469330,89811711,90654836,19898053,76315420,9860195,874791,57020481,4095116,23110625,48675329,38645117,38009615,4508700,74452589,64602895,36845587,61897582,62428472,70541760,68041839,3233569,6038457,11757872,43933006,10453030,36468541,38256849,5640302,20645197,96709982,86798033,44627776,17037369,39373729,31491938,65454636,1250437,54517921,16099750,22435353,91141395,82779622,60393039,53888755,90090964,78300864,72357096,82651278,57393458,43506672,59981773,6157724,63152504,44060493,53393358,59405277,52060076,48673079,37560164,40356877,6521313,34667286,65851721,97011160,5073754,24915585,59614746,72793444,21397057,49882705,87160386,24826575,92353856,55669657,88130087,15015906,29401781,62762857,70782102,83948335,92867155,42237907,88698958,83155442,36189527,45790169,749283,45990383,83747892,44844121,8791066,77301523,41245325,51715482,61928316,72278539,67451935,64055960,69697787,91802888,12303248,75052463,75135448,81172706,47298834,64848072,7182642,92541302,90158785,65880522,79322415,17857111,38022834,33935899,37280276,83133790,6111563,1239555,82327024,89804152,21289531,13468268,27411561,69786756,36135,61815223,76825057,68110330,99729357,21001913,46540998,96852321,2204165,55850790,61741594,38510840,91255408,85695762 +14220886,24058273,17068582,82327024,67793644,49236559,66250369,96193415,44889423,21993752,29834512,66832478,96735716,78785507,95957797,93933709,1197320,22108665,16595436,4204661,81172706,44784505,66116458,6986898,74743862,29819940,60430369,37620363,29510992,74724075,44348328,6793819,50806615,96726697,67030811,26292919,78561158,20568363,68000591,14626618,58180653,55770687,69355476,78218845,57241521,5073754,19457589,15015906,53084256,89214616,9599614,54427233,10358899,29029316,66045587,7646095,74110882,16971929,92787493,33628349,14947650,71300104,73124510,45237957,16054533,16019925,72274002,72777973,35092039,6871053,99604946,55247455,86543538,81853704,99524975,55470718,26114953,69136837,99549373,20122224,93515664,75229462,38658347,47213183,30764367,76434144,32250045,74357852,17764950,21289531,23194618,32151165,61982238,44481640,17539625,55753905,6505939,37891451,73235980,36808486,36753250,35192533,65715134,55850790,72019362,88047921,6525948,39171895,60581278,66428795,91255408,94090109,77284619,98130363,14326617,4064751,29994197,77312810,93270084,1431742,2204165,13468268,8807940,4515343,73222868,98739783,62452034,52261574,5111370,41092172,415901,9175338,9575954,2717150,45790169,26139110,62430984,20140249,50702367,8733068,92554120,5832946,29466406,72738685,28928175,70372191,39201414,67451935,6038457,40197395,12416768,11543098,11581181,8129978,12303248,53648154,29959549,90310261,73617245,53802686,57359924,6153406,92604458,16099750,91938191,48360198,29188588,43501211,70596786,6808825,80555751,33201905,12348118,29635537,33431960,24168411,45407418,4787945,9886593,19101477,56153345,7502255,84406788,4099191,59910624,83155442,83269727,42237907,34236719,66663942,62232211,36468541,79821897,72725103,34497327,79191827,33336148,40152546,58749,17365188,64602895,55960386,67281495,32159704,78909561,16097038,31126490,4806458,27665211,68824981,85242963,79922758,39801351,59501487,25842078,68816413,89804152,20885148,33935899,97783876,44842615,58208470,44479073,27625735,26998766,51426311,34946859,80014588,77272628,76960303,61960400,91990218,33797252,32058615,55602660,99515901,9860195,23848565,23569917,59371804,91727510,41245325,3294781,92033260,22450468,59405277,31491938,72373496,93566986,30090481,87598888,18833224,17037369,30543215,15902805,44177011,55669657,15535065,74452589,8099994,92803766,94595668,34044787,77413012,4668450,40781854,65271999,10309525,58224549,84127901,82651278,26863229,8729146,3088684,9623492,69641513,57658654,4091162,36685023,66667729,78884452,28734791,508198,71920426,78589145,18504197,17957593,54868730,44983451,4508700,64848072,37435892,79942022,90158785,43376279,48088883,98948034,1204161,86118021,51047803,50567636,38494874,94911072,569864,66885828,31161687,38645117,48673079,36312813,83533741,7423788,72357096,99125126,49271185,96420429,90457870,89996536,32161669,70800879,56955985,77694700,61263068,90013093,80246713,60176618,85023028,28787861,40984766,45919976,63059024,66903004,40677414,45990383,62740044,7300047,32590267,44847298,82339363,86242799,65338021,99965001,13862149,1569515,83789391,22942635,32426752,68644627,64055960,61712234,76540481,2917920,18699206,71522968,45428665,42644903,78602717,57240218,83210802,19939935,18411915,54263880,22405120,34667286,9259676,31904591,50668599,15075176,72495719,168541,15948937,72278539,51588015,57803235,33699435,5970607,99226875,78766358,48774913,3509435,86001008,22129328,12664567,26734892,66611759,99021067,31733363,51464002,82979980,96906410,9058407,41442762,36505482,92398073,4069912,40356877,61859581,37192445,4434662,63967300,52204879,74137926,70004753,94711842,80316608,65275241,4798568,321665,45848907,19376156,60106217,77898274,3880712,39373729,44060493,81898046,44844121,74441448,90061527,36396314,92215320,34432810,16405341,20002147,95581843,1788101,91664334,6819644,61373987,47893286,82897371,40686254,34698428,69697787,14363867,77377183,73031054,41481685,17081350,824872,30653863,11161731,46540998,8128637,66271566,1936762,30787683,19891772,68875490,26392416,24619760,91802888,82427263,68102437,56461322,20867149,22113133,61271144,84349107,89046466,46851987,26102057,96318094,56531125,96852321,47738185,62051033,72732205,54517921,16380211,68204242,1250437,87710366,44473167,98943869,54663246,75543508,669105,66319530,52060076,26063929,8791066,86022504,93359396,92692978,43152977,94539824,8373289,14093520,24953498,3183975,71316369,38061439,3233569,2208785,29932657,49328608,7182642,27041967,68939068,14723410,37957788,37659250,48893685,33061250,16684829,70367851,57393458,86821229,33304202,47361209,16445503,11757872,75777973,10453030,89811711,7687278,53547802,15148031,75153252,14731700,4119747,14349098,36580610,17386996,30998561,77301523,9257405,62490109,83150534,91831487,84684495,92692283,34698463,85695762,90004325,61859143,90090964,57163802,44846932,65017137,28550822,71965942,37183543,749283,526217,17976208,36933038,28796059,54987042,7011964,2891150,53842979,48675329,8651647,37224844,82178706,42199455,95290172,65038678,69786756,40865610,50007421,82726008,54762643,59197747,36139942,43322743,84166196,3233084,51472275,92353856,98371444,63015256,94360702,62693428,45794415,81677380,11923835,49598724,62936963,86301513,9188443,47887470,11365791,22435353,26776922,45381876,47792865,95726235,89419466,30366150,91957544,97379790,58751351,77072625,45667668,66322959,85571389,20765474,74614639,59109400,44627776,39553046,38341669,26744917,66137019,98462867,65880522,79880247,22997281,22766820,14045383,30139692,76703609,57020481,19486173,69579137,17727650,22721500,48260151,3487592,30463802,84904436,23134715,61728685,94516935,21673260,68128438,82779622,39847321,86460488,68110330,10597197,26275734,67124282,62762857,51315460,54058788,78549759,99971982,60102965,1239555,70782102,43357947,84293052,70541760,13819358,99690194,20836893,82052050,9603598,84840549,75820087,88251446,76315420,53632373,18001617,15163258,32274392,39986008,54014062,49641577,73109997,47708850,9860968,97561745,75135448,33237508,59318837,65851721,4199704,76671482,59329511,2607799,874791,76330843,12571310,79806380,71083565,99658235,83651211,89499542,72238278,37675718,36812683,89078848,65081429,35512853,88130087,81274566,68316156,13348726,61928316,73392814,62552524,87160386,82886381,13173644,64087743,13470059,10366309,19272365,83368048,92530431,64157906,87720882,40781449,75986488,30891921,3633375,16387575,68041839,89217461,27185644,37280276,53666583,3235882,38256849,37166608,46870723,7229550,35456853,53888755,29516592,48892201,93057697,10264691,26493119,68694897,54606384,62357986,91240048,42782652,72793444,1022677,15536795,86798033,97011160,61141874,42967683,99861373,7919588,56484900,4095116,5267545,6221471,7685448,24733232,90272749,67474219,88444207,27325428,60955663,63628376,26664538,38365584,18783702,20681921,57248122,51466049,88904910,33553959,99333375,89637706,18131876,23110625,5482538,67084351,36135,61623915,71469330,43933006,76170907,15064655,21919959,81805959,51507425,30395570,17016156,63372756,16424199,10649306,45306070,24591705,42073124,88653118,43045786,81855445,44245960,30811010,30163921,63152504,247198,50188404,29401781,84002370,45075471,51590803,95395112,75128745,79657802,92998591,70036438,77620120,98653983,83133790,3773993,59614746,69605283,34493392,55189057,65074535,55143724,48658605,49882705,30771409,32699744,27187213,17385531,77187825,41092102,36845587,6497830,1375023,89699445,17894977,49597667,36930650,85116755,21533347,38022834,77787724,98648327,67031644,21397057,33565483,28851716,12024238,52613508,37560164,42947632,18663507,7955293,33249630,62496012,97057187,44550764,47298834,83083131,44664587,8696647,79136082,7517032,24314885,65454636,61897582,29065964,59177718,99297164,92541302,42692881,23740167,62803858,92867155,24915585,76825057,70420215,92071990,91141395,65047700,44915235,30694952,16567550,18466635,36780454,20642888,99224392,37739481,17146629,9829782,19898053,8913721,94076128,72614359,54232247,56515456,40027975,47090124,24826575,60393039,13231279,99917599,97940276,176257,35996293,84187166,91281584,38510840,20486294,51149804,93562257,61815223,37501808,45617087,22200378,69848388,97281847,77300457,27411561,40534591,30569392,71333116,52293995,81774825,69255765,6521313,34295794,56473732,112651,95251277,9398733,4978543,23432750,83948335,54199160,96709982,8115266,17857111,5640302,73786944,73168565,55615885,83302115,42307449,2331773,30218878,48395186,38009615,21070110,38008118,45995325,36189527,20645197,82532312,63506281,21001913,25636669,7104732,41380093,71657078,61741594,75407347,33123618,8055981,26507214,45049308,88698958,4985896,22879907,93053405,95010552,99729357,45996863,56424103,51715482,79322415,43506672,68702632,93790285,10961421,67513640,73021291,85711894,8634541,97641116,59981773,53393358,31727379,80251430,17058722,62428472,70727211,85117092,79738755,62115552,78300864,75052463,6157724,5822202,57961282,72358170,33895336,67227442,6111563,18806856,64098930,7066775,26397786,90654836,83747892 +37183543,24591705,69786756,53842979,73168565,30366150,99965001,29819940,92787493,27185644,20642888,9398733,73617245,59177718,60430369,97561745,14045383,65047700,19101477,61859143,39801351,92554120,26664538,76671482,50188404,34493392,1022677,64602895,28787861,78218845,16595436,66322959,57803235,8913721,62740044,39847321,32161669,874791,68128438,67084351,3880712,526217,34295794,1431742,40152546,99297164,77413012,55770687,49641577,112651,38061439,40197395,14093520,84187166,51715482,65338021,77272628,62936963,73222868,1204161,92215320,28550822,79821897,176257,42782652,60955663,29188588,5482538,48360198,17016156,36780454,36468541,63015256,62693428,89214616,84840549,50702367,18131876,21289531,1375023,15535065,18001617,28851716,80014588,27041967,98130363,53648154,21533347,15064655,10264691,45667668,66903004,51047803,47298834,75407347,86022504,41092172,96906410,89046466,39553046,79922758,65454636,43933006,92803766,62496012,4091162,18783702,68000591,70596786,45790169,23569917,29029316,44983451,11365791,36685023,70367851,89419466,66832478,62357986,22997281,77620120,36580610,43501211,99861373,69355476,68939068,67031644,168541,83083131,74441448,86821229,31126490,38365584,53084256,85242963,37560164,6521313,6986898,56473732,71083565,92033260,6153406,92541302,6793819,9058407,7955293,27411561,77300457,61728685,24168411,45995325,33123618,46870723,37435892,98462867,33249630,66116458,96852321,66045587,96709982,72793444,42073124,26998766,19891772,10358899,75229462,90061527,26397786,41481685,68204242,16971929,45996863,33237508,82979980,30694952,38645117,81172706,14349098,3509435,67474219,36753250,23110625,4099191,54762643,67793644,88653118,55189057,89078848,44550764,62051033,71300104,36135,43322743,30163921,3235882,36139942,77284619,67281495,72278539,4806458,4787945,71333116,40781449,88904910,95957797,19939935,63967300,33201905,69697787,83368048,11923835,90090964,66271566,38341669,8099994,37192445,96735716,33565483,83651211,77694700,91957544,13862149,34698428,42199455,90457870,36505482,91990218,20765474,24058273,3233084,29932657,38022834,3183975,2208785,9623492,99658235,17539625,18806856,78589145,68816413,2917920,99515901,88698958,247198,82339363,94911072,76960303,51472275,78561158,84904436,51464002,29635537,28796059,89811711,50007421,33336148,24619760,52204879,62490109,20681921,84349107,72357096,82886381,7919588,35192533,53666583,23432750,27665211,25842078,26114953,14947650,82726008,13348726,93933709,72725103,4069912,60106217,55615885,23194618,80251430,38658347,90272749,62452034,26063929,19486173,43152977,49597667,16684829,20002147,16099750,19272365,42644903,89804152,63506281,99524975,34432810,16387575,73021291,39373729,91938191,30395570,38008118,52261574,86118021,7011964,64098930,44479073,90310261,50668599,72495719,85711894,66319530,40534591,70420215,60102965,92998591,95010552,29834512,66250369,22200378,8373289,12348118,91802888,14220886,27625735,93790285,47090124,8807940,74357852,97011160,99333375,91664334,66667729,76434144,20122224,43506672,61815223,99917599,78549759,73392814,12024238,6808825,21001913,65271999,33304202,70004753,3088684,76540481,3487592,67030811,52613508,98648327,74614639,24826575,7646095,88130087,49236559,1197320,70800879,45617087,82779622,32274392,32250045,77072625,29959549,85117092,12571310,57241521,33628349,54014062,77312810,36845587,81274566,9603598,18699206,14326617,96420429,5970607,59910624,54058788,61741594,13468268,70727211,61982238,8055981,10309525,44177011,75153252,59614746,2717150,31904591,38256849,40677414,70372191,35996293,55143724,61263068,73031054,82651278,38494874,99971982,86001008,26863229,20486294,9599614,55753905,37739481,59109400,45075471,15015906,3294781,15948937,7229550,88047921,14731700,2204165,99549373,91141395,43045786,48673079,68875490,64087743,4064751,42947632,67227442,415901,16380211,54606384,53888755,79191827,96726697,30653863,40356877,30139692,43376279,47708850,61623915,98943869,1239555,81898046,95395112,52060076,44842615,22405120,72274002,17146629,20867149,61897582,52293995,56424103,15536795,94539824,17081350,68316156,22129328,16405341,53632373,92692283,20140249,97641116,54427233,68102437,8634541,7687278,92398073,46540998,34946859,22879907,4119747,21070110,63628376,44889423,77377183,6111563,29401781,51466049,83789391,22721500,28928175,97783876,20836893,83210802,70541760,10366309,94516935,94595668,42237907,22108665,29994197,36312813,37659250,58751351,64848072,92353856,76330843,99224392,93515664,23848565,20645197,23740167,63059024,81677380,26776922,86242799,48088883,79136082,91240048,9257405,10649306,30811010,4508700,59371804,75135448,5073754,79806380,89996536,60581278,47738185,25636669,57961282,32151165,72777973,22942635,44915235,9860968,63372756,55247455,36812683,51149804,26102057,47893286,84293052,45428665,54199160,44844121,12416768,48893685,71469330,45237957,74452589,26744917,37891451,91255408,47361209,58749,92530431,7066775,45306070,8696647,65880522,39171895,37675718,73786944,50806615,13231279,34698463,94090109,57240218,15163258,17037369,80316608,83302115,79322415,93270084,31733363,87710366,10453030,44245960,87160386,6505939,71920426,19898053,62803858,50567636,55850790,51590803,68824981,96193415,16054533,19376156,88251446,92604458,9575954,99226875,5111370,41380093,68644627,56461322,55602660,67451935,97281847,37501808,95581843,59981773,59501487,62115552,93053405,57248122,5267545,81774825,32159704,17386996,824872,85023028,49271185,84002370,4095116,7423788,17068582,69641513,58208470,19457589,57658654,11543098,61859581,3633375,89637706,49598724,77187825,72732205,49882705,64157906,79880247,99021067,69136837,17058722,77787724,79657802,6871053,26734892,34236719,6157724,34667286,62430984,91831487,81805959,34497327,51588015,76170907,93562257,4434662,87598888,45848907,85116755,40686254,64055960,26493119,90004325,75820087,40984766,65017137,7104732,76703609,42307449,54868730,22435353,93566986,45990383,86301513,4798568,35456853,35512853,66611759,54663246,80246713,33797252,38510840,34044787,22766820,40027975,9860195,68041839,44481640,92692978,71316369,21919959,86460488,10597197,61960400,99604946,33895336,90158785,40865610,17727650,7502255,55669657,22450468,89699445,7300047,87720882,85571389,79738755,8129978,73235980,57020481,86543538,44784505,4199704,31161687,33699435,99125126,15148031,98371444,66885828,76315420,6497830,30764367,1569515,61141874,32699744,569864,84684495,56515456,8791066,75777973,43357947,16445503,74743862,4668450,75543508,94360702,20885148,46851987,48774913,74110882,4204661,16424199,88444207,36396314,65851721,48395186,65275241,48675329,6221471,53393358,16097038,41442762,8128637,78785507,65715134,82178706,31491938,62232211,48892201,54263880,13470059,59318837,26392416,70036438,7685448,80555751,21993752,48658605,24953498,44060493,44348328,17385531,56531125,69848388,18411915,1936762,73109997,45919976,61373987,22113133,83269727,59197747,4985896,12303248,44627776,11161731,81853704,54987042,37957788,76825057,75052463,61928316,37620363,97379790,95251277,321665,90654836,45794415,99690194,21673260,30771409,29466406,71657078,26139110,24915585,94076128,7517032,56153345,26275734,78766358,92071990,44846932,83133790,9188443,55470718,60176618,37280276,12664567,15902805,58180653,45407418,42692881,96318094,36808486,86798033,98948034,9886593,36933038,30463802,32590267,1788101,71522968,30998561,66137019,30543215,35092039,23134715,69255765,83155442,51507425,30218878,77898274,83533741,669105,30569392,37166608,9259676,17365188,13173644,66663942,55960386,94711842,39986008,53802686,65038678,33061250,78300864,57393458,5640302,36930650,10961421,93057697,81855445,24733232,29510992,62762857,53547802,9175338,4515343,18466635,91281584,61271144,65081429,95290172,18833224,3233569,69579137,68110330,31727379,16567550,65074535,17957593,83747892,3773993,8733068,91727510,97940276,66428795,48260151,8651647,69605283,1250437,14723410,28734791,508198,72614359,79942022,54517921,32426752,11581181,62428472,7182642,77301523,5822202,49328608,47887470,83150534,14626618,2891150,18663507,56484900,15075176,72373496,29516592,78884452,82427263,30891921,67124282,89499542,82052050,16019925,41245325,90013093,73124510,68702632,26507214,67513640,84127901,44473167,68694897,17857111,75128745,32058615,13819358,63152504,72358170,8115266,27325428,36189527,33935899,6038457,44847298,2607799,17976208,2331773,99729357,30787683,6819644,54232247,40781854,82532312,59329511,38009615,74137926,37224844,92867155,57359924,30090481,11757872,82897371,72738685,47213183,9829782,51315460,26292919,61712234,33431960,72019362,5832946,56955985,62552524,33553959,42967683,97057187,75986488,27187213,17894977,60393039,72238278,20568363,39201414,59405277,93359396,21397057,17764950,70782102,74724075,4978543,47792865,749283,29065964,78909561,84406788,71965942,78602717,83948335,14363867,57163802,44664587,45049308,24314885,51426311,84166196,89217461,98739783,6525948,58224549,95726235,45381876,41092102,85695762,82327024,8729146,18504197,98653983 +61859143,17037369,43376279,66832478,8913721,64055960,12348118,3233084,4119747,89214616,44889423,16097038,16595436,19891772,95957797,15015906,76540481,99224392,34497327,62693428,30787683,37957788,32161669,98948034,9623492,83789391,88653118,92803766,18699206,62496012,86001008,65454636,62740044,92998591,7502255,63967300,65275241,83948335,55247455,92692978,70596786,19457589,34698428,40197395,68939068,57803235,28851716,62936963,669105,36580610,14349098,96735716,75153252,19486173,81677380,22942635,60955663,43506672,45848907,29959549,61741594,42782652,38022834,38658347,54427233,89046466,99226875,33553959,39801351,20140249,76434144,73235980,93359396,39847321,44550764,68644627,45790169,41481685,27665211,62357986,21993752,5970607,6808825,59614746,17385531,89078848,749283,508198,29188588,70800879,12571310,43357947,2891150,9188443,49641577,34493392,53842979,70004753,9860195,18001617,96709982,47792865,46851987,26063929,36753250,49236559,52261574,34698463,27325428,33797252,62803858,15064655,9398733,20642888,68816413,56515456,37739481,57248122,70367851,64848072,26114953,30694952,42307449,14947650,61897582,22766820,92604458,40677414,20885148,26998766,9603598,23848565,76960303,97940276,77620120,33336148,74614639,50188404,88047921,18783702,2717150,7687278,36845587,26392416,77301523,62430984,51715482,43045786,65851721,37435892,4515343,72019362,47708850,24591705,65338021,59177718,1431742,37620363,90272749,91255408,50702367,1204161,86543538,9860968,21533347,6986898,3294781,82779622,99515901,30569392,92554120,51047803,75407347,33201905,95395112,33935899,29994197,62232211,33237508,79821897,30163921,96193415,35996293,36812683,74452589,66250369,1375023,84187166,50668599,70036438,44060493,54663246,66903004,4099191,71920426,23569917,92541302,3183975,99524975,23740167,6505939,68824981,72725103,28734791,65074535,42199455,54987042,31733363,99965001,13231279,26275734,45049308,77284619,48088883,67031644,83083131,54762643,96318094,76671482,79136082,20486294,1022677,38365584,94539824,26493119,39171895,15536795,58749,93566986,71083565,33628349,62452034,72777973,61982238,56424103,44245960,25842078,52060076,38341669,75777973,8696647,30395570,11543098,11161731,50806615,5267545,98371444,22879907,69786756,97561745,52293995,97281847,79191827,54199160,168541,33249630,10366309,51472275,83302115,6153406,91664334,72793444,97783876,36468541,55189057,64087743,67281495,29834512,44177011,61859581,66116458,86798033,43933006,46540998,7104732,48260151,30543215,81853704,16387575,90013093,63059024,73617245,66885828,63372756,78589145,78300864,11923835,89699445,45237957,36135,47361209,65880522,71333116,85571389,45995325,7955293,74441448,112651,39201414,17068582,3633375,57241521,50007421,874791,1250437,23194618,71316369,30771409,78549759,86821229,24915585,22450468,51588015,91281584,68102437,9575954,4508700,3509435,8055981,79657802,77413012,28796059,32250045,82427263,68694897,66271566,24733232,4798568,48360198,23134715,8115266,24826575,57359924,73031054,38061439,29819940,57393458,16019925,17365188,56153345,5640302,94090109,57020481,20122224,85711894,13470059,22108665,59371804,98130363,48673079,247198,28550822,70541760,32426752,17058722,18806856,38494874,72274002,83533741,51464002,3773993,14093520,10961421,53802686,88444207,96852321,40686254,66663942,83155442,73124510,61271144,38008118,66137019,33565483,83210802,6793819,36933038,22129328,7919588,4204661,6111563,13348726,17764950,26102057,98648327,36505482,74724075,45996863,98462867,36780454,93933709,55143724,45667668,69641513,39373729,68316156,51590803,34236719,84904436,91727510,44983451,9175338,35092039,36189527,38510840,87160386,92692283,90310261,8791066,74357852,98739783,45990383,36930650,61815223,44473167,33304202,3487592,78766358,39553046,77377183,67474219,60581278,51426311,99658235,71657078,87720882,35456853,87710366,88130087,85242963,81172706,91831487,38645117,58208470,29932657,67084351,37183543,40152546,21397057,80014588,93270084,29466406,69579137,92033260,54517921,44915235,19376156,34295794,42237907,6221471,74137926,86301513,77312810,53888755,68128438,42073124,76703609,30090481,55602660,82979980,82532312,7517032,72373496,73392814,66045587,12024238,83368048,48774913,29029316,68204242,92787493,9829782,7066775,16099750,78602717,57961282,45306070,45075471,99549373,21001913,19101477,8733068,18833224,8634541,38009615,66611759,73786944,97057187,90654836,8807940,79880247,44348328,68875490,2607799,97011160,415901,57240218,84293052,80555751,8128637,67793644,40984766,33895336,34946859,9599614,91957544,41442762,59197747,51507425,27625735,56531125,39986008,84840549,89811711,83651211,53547802,61623915,40781854,40356877,59109400,61373987,69255765,4095116,55669657,26776922,1569515,45407418,59501487,16380211,85116755,6521313,54014062,33123618,37192445,4668450,31904591,20568363,569864,7300047,7423788,99729357,75052463,65047700,14326617,45428665,45919976,55470718,44784505,3880712,92215320,64098930,4069912,99917599,63506281,72614359,37675718,71469330,14723410,67030811,79942022,91802888,62051033,70420215,56473732,79806380,76315420,77187825,24058273,77272628,44627776,94711842,94911072,29635537,9058407,69697787,61728685,85695762,27185644,78218845,17957593,84127901,92867155,57163802,59981773,98943869,30139692,97641116,19272365,63152504,80316608,17146629,41092172,15163258,33431960,53632373,4434662,53084256,24168411,99861373,4806458,26863229,46870723,73222868,21070110,82651278,99971982,74110882,59910624,49271185,89804152,6497830,14045383,29510992,93515664,20681921,90457870,99297164,75543508,10358899,29401781,2208785,96726697,62552524,9886593,76330843,66322959,30463802,30366150,71522968,68702632,17727650,83133790,83269727,33061250,58751351,32699744,37166608,24314885,55615885,55960386,95251277,16971929,63015256,82886381,26139110,48395186,27411561,77694700,28928175,60176618,15948937,75986488,47298834,67124282,20765474,19939935,42967683,36685023,43322743,54868730,13468268,34432810,5482538,77300457,42947632,57658654,65715134,70727211,24619760,15075176,28787861,43501211,36139942,5073754,40781449,69355476,75229462,99125126,84406788,88904910,26507214,72495719,71300104,44842615,1788101,48892201,26734892,3233569,35192533,95290172,68000591,80246713,67451935,7182642,41092102,15535065,12416768,41380093,4985896,53666583,93053405,91240048,51466049,51315460,47213183,85023028,99333375,96420429,82339363,11365791,4787945,17386996,82052050,24953498,89996536,68041839,10453030,49597667,4064751,60106217,94076128,93562257,30218878,58224549,99604946,21289531,53648154,62490109,95726235,81855445,78884452,19898053,84002370,72357096,72732205,9259676,32274392,1239555,94516935,47090124,77787724,65271999,8651647,15148031,48675329,52613508,13862149,47738185,74743862,65017137,93790285,75135448,44479073,40865610,8129978,48893685,73021291,6157724,11757872,91141395,14731700,32159704,7011964,35512853,58180653,61263068,49882705,31126490,82726008,87598888,62428472,59318837,526217,21673260,99690194,60430369,16054533,29065964,26744917,67227442,83150534,62762857,54606384,88251446,26397786,36808486,94360702,72738685,7229550,14363867,69605283,78909561,18466635,49598724,76170907,70372191,37280276,10649306,60102965,55753905,30764367,22721500,824872,37224844,97379790,89419466,18663507,36396314,8099994,92398073,38256849,30998561,18131876,20002147,67513640,1936762,20867149,93057697,92530431,17081350,16684829,45794415,17976208,80251430,44846932,72278539,78561158,54232247,82178706,20645197,91938191,92071990,90061527,90090964,37501808,31491938,75128745,22435353,55770687,66319530,36312813,69848388,61928316,56461322,40027975,6525948,47887470,51149804,99021067,54263880,66428795,63628376,95010552,54058788,5832946,88698958,7685448,53393358,23110625,77072625,86460488,17016156,89637706,8373289,44664587,56484900,10309525,30653863,81805959,3235882,23432750,16445503,69136837,4199704,79922758,9257405,72358170,52204879,37659250,41245325,2917920,2204165,16567550,84349107,44847298,33699435,6038457,25636669,89499542,31161687,50567636,12664567,62115552,30811010,34044787,20836893,17539625,73168565,7646095,48658605,79738755,26292919,5111370,16424199,42692881,27041967,32151165,1197320,72238278,6871053,94595668,21919959,66667729,81898046,44844121,90004325,79322415,64602895,30891921,17857111,61141874,37891451,78785507,86022504,40534591,70782102,17894977,75820087,14626618,47893286,22200378,32590267,83747892,71965942,44481640,12303248,84684495,64157906,68110330,92353856,18411915,26664538,82897371,15902805,76825057,86242799,321665,91990218,65081429,59329511,6819644,45617087,13173644,59405277,14220886,77898274,16405341,37560164,22997281,27187213,86118021,96906410,85117092,81774825,55850790,29516592,49328608,81274566,176257,10597197,22405120,5822202,42644903,98653983,60393039,84166196,11581181,8729146,89217461,73109997,45381876,56955985,43152977,32058615,82327024,18504197,4978543,90158785,22113133,4091162,95581843,13819358,3088684,34667286,31727379,2331773,61960400,61712234,10264691,65038678 +66045587,70420215,2717150,77284619,54199160,30998561,82339363,27665211,10309525,62740044,88653118,55470718,33201905,89214616,93933709,62552524,35092039,37620363,37183543,16445503,47090124,54663246,26392416,20765474,13862149,77272628,71333116,81898046,15535065,79657802,17764950,29834512,13173644,67793644,49641577,89637706,43501211,64087743,9257405,10366309,27325428,61263068,20122224,63059024,21533347,73786944,89804152,80014588,76960303,1197320,62357986,39847321,50668599,84840549,49328608,89078848,31904591,14626618,84187166,74137926,40356877,8807940,5111370,1431742,92787493,44983451,27411561,92033260,55753905,10453030,91957544,93566986,99515901,53802686,5822202,30764367,54762643,18504197,83651211,71657078,45428665,6525948,64098930,24915585,39373729,63967300,27041967,75777973,36580610,18833224,94516935,37739481,49271185,3509435,72274002,60430369,64602895,91664334,67474219,65081429,20836893,67451935,34044787,40677414,9860195,67030811,39801351,49882705,20486294,65851721,33431960,94595668,52613508,75986488,12348118,24058273,17081350,7919588,84904436,82726008,54606384,85116755,59329511,22997281,75820087,321665,32590267,96318094,59177718,69136837,31161687,92604458,9259676,35192533,54014062,2917920,26507214,66250369,34698463,36780454,29994197,55247455,66903004,88047921,16054533,22108665,50702367,4204661,53666583,3880712,78785507,44844121,99021067,30163921,37501808,57803235,66667729,40152546,66116458,508198,247198,62051033,3487592,51464002,53547802,3233084,4099191,526217,87160386,66271566,8129978,45049308,47792865,47887470,43376279,68816413,19376156,19891772,77787724,99549373,92398073,70372191,44481640,52204879,14349098,71316369,14326617,12571310,42199455,5482538,96193415,76825057,10649306,3088684,62490109,85117092,60106217,38061439,49598724,73235980,33935899,65038678,36808486,29516592,61960400,65338021,65275241,2204165,45794415,42782652,7229550,66428795,77620120,56461322,78589145,19101477,10961421,47361209,88251446,16097038,43322743,56515456,68000591,48260151,96735716,80316608,69848388,68644627,18783702,74110882,99917599,58224549,60176618,33237508,4064751,46851987,6986898,45306070,30366150,50567636,73124510,89046466,99971982,66611759,62452034,4787945,73168565,19486173,4119747,20140249,14045383,30811010,94360702,72373496,65715134,18663507,90457870,16387575,69355476,4095116,8651647,28734791,18131876,1569515,95290172,43933006,60102965,51149804,1239555,46870723,84406788,72357096,9886593,45407418,55615885,73617245,59910624,61271144,66832478,70800879,99965001,30891921,99658235,91727510,30395570,40027975,43357947,30090481,669105,20002147,78602717,29065964,33699435,44479073,95726235,14220886,44842615,5832946,76315420,9058407,75135448,99125126,90004325,51047803,18806856,84684495,35996293,35456853,22766820,55143724,84002370,26397786,16380211,45075471,68939068,30139692,7300047,28928175,29029316,61373987,82897371,9575954,72278539,98648327,22200378,12664567,4515343,89699445,14947650,53393358,38022834,6793819,80251430,51507425,9175338,22879907,3633375,7685448,22405120,92998591,62693428,45237957,56484900,57240218,62496012,23848565,81853704,17037369,15536795,14731700,59109400,67031644,83150534,8696647,32151165,75543508,80555751,20867149,79806380,18411915,91938191,63628376,22450468,93057697,7687278,61141874,8733068,15148031,4668450,22113133,16567550,98130363,40865610,32161669,61712234,77694700,86022504,34497327,53648154,61741594,70727211,5073754,50188404,22721500,75153252,3233569,8115266,36139942,40781854,46540998,52261574,30787683,97940276,69786756,37891451,37659250,77300457,32159704,72238278,33565483,79136082,27625735,22435353,2891150,68204242,68875490,98653983,44784505,31126490,79738755,89996536,824872,90013093,19939935,86242799,71920426,40984766,99729357,34295794,26139110,88444207,86001008,33797252,82886381,96726697,83210802,89419466,83155442,54058788,77898274,38256849,16099750,6808825,51426311,87598888,20568363,99226875,16405341,42692881,25636669,6505939,76170907,56955985,39986008,62803858,6153406,17068582,65880522,4508700,79821897,47213183,89217461,16019925,6521313,569864,36753250,42073124,59371804,2208785,41245325,45617087,40781449,36312813,82052050,62936963,9599614,30771409,94911072,15902805,63506281,81172706,51588015,48892201,98371444,7182642,74441448,62430984,17386996,96906410,5640302,6157724,29466406,23569917,45996863,24826575,39553046,61982238,32699744,5970607,81677380,74743862,81805959,95957797,92554120,5267545,58208470,38008118,33249630,84166196,84127901,168541,75128745,58180653,99604946,99524975,17385531,86118021,48395186,91831487,41380093,68128438,76330843,31491938,70367851,88904910,67281495,17016156,13348726,44889423,52293995,36135,72358170,21070110,78300864,66137019,48088883,15015906,72725103,42644903,61623915,26102057,4985896,94090109,37560164,47708850,63015256,61859581,71469330,42237907,90061527,17976208,83789391,48774913,72614359,30543215,26275734,1788101,112651,12416768,3183975,55189057,54427233,42307449,67227442,19272365,18699206,92692283,85571389,1204161,71083565,79880247,47738185,44664587,45848907,76434144,57163802,44473167,62762857,55770687,76540481,32058615,86460488,33061250,28851716,26493119,26292919,7066775,95010552,95251277,99297164,92803766,97057187,86821229,53632373,86798033,47298834,45990383,29932657,1250437,81855445,92692978,78218845,54517921,34698428,70004753,31733363,38658347,79922758,23194618,7646095,55602660,9188443,68702632,93053405,24619760,77187825,92071990,30653863,59197747,16595436,32250045,93790285,99861373,11543098,7955293,83747892,50007421,18001617,41092102,16971929,69579137,36930650,33304202,57248122,73109997,66663942,84349107,20885148,33628349,51590803,8099994,97011160,59318837,56473732,8913721,60955663,67124282,30463802,93562257,78561158,63372756,49597667,6111563,57241521,45381876,75229462,61859143,82979980,83083131,71522968,64848072,44348328,77312810,16684829,91990218,44846932,44847298,98943869,49236559,81774825,78909561,43506672,69697787,88130087,64157906,72777973,55669657,77413012,14363867,95581843,25842078,36396314,43045786,62115552,24314885,68694897,85023028,176257,65047700,29635537,85242963,56424103,69641513,21397057,8729146,4434662,7011964,48673079,74614639,77072625,66322959,36685023,94711842,57020481,68041839,69605283,40197395,78884452,24168411,7517032,40686254,99333375,92541302,38009615,37957788,37435892,90272749,71300104,59501487,75052463,10597197,22942635,37192445,21919959,60581278,70036438,34432810,36812683,61728685,37675718,95395112,97641116,34493392,58751351,99224392,65017137,17539625,69255765,50806615,26998766,73031054,93359396,24733232,89811711,86301513,2607799,36505482,43152977,53888755,874791,54868730,83368048,54263880,86543538,35512853,23740167,57393458,61928316,2331773,59405277,12303248,38365584,26744917,7423788,24591705,44245960,11757872,17857111,29188588,44177011,65074535,83302115,72495719,97379790,97561745,11581181,62232211,88698958,92215320,56531125,53842979,78766358,36468541,17058722,44060493,70541760,6221471,29819940,26114953,90090964,19898053,87710366,87720882,42947632,29401781,82327024,4091162,17727650,48675329,15948937,67513640,28796059,57359924,82532312,21993752,68824981,30694952,51715482,48360198,749283,82651278,41442762,67084351,4069912,82178706,74724075,21673260,38494874,19457589,39171895,6038457,85695762,9603598,57658654,26863229,47893286,10358899,59614746,8373289,15163258,76671482,28787861,28550822,23432750,75407347,54987042,3773993,36189527,17894977,22129328,90310261,82427263,4199704,63152504,36933038,74357852,26776922,65271999,38645117,13819358,79191827,97281847,13468268,26734892,91802888,20642888,61815223,93515664,6871053,72732205,15064655,92867155,37224844,68102437,37166608,24953498,23134715,91281584,98462867,23110625,32274392,55960386,44550764,55850790,9398733,44915235,16424199,56153345,1936762,59981773,68110330,96852321,70596786,42967683,83133790,37280276,73021291,33336148,8128637,10264691,415901,15075176,20681921,77301523,68316156,89499542,13231279,4806458,38341669,85711894,34236719,77377183,98948034,9623492,60393039,33895336,79322415,45995325,18466635,14093520,34667286,97783876,92530431,40534591,82779622,48658605,79942022,21001913,29510992,12024238,92353856,34946859,52060076,33553959,39201414,30218878,54232247,17957593,83948335,36845587,4978543,66319530,72019362,20645197,78549759,73392814,91141395,81274566,83533741,99690194,94076128,11365791,91255408,29959549,61897582,1375023,11161731,41481685,72793444,26063929,17365188,98739783,31727379,48893685,57961282,17146629,96420429,64055960,4798568,26664538,7104732,83269727,3235882,65454636,30569392,74452589,3294781,7502255,27185644,44627776,1022677,51466049,8055981,45667668,94539824,41092172,8791066,33123618,90654836,53084256,21289531,32426752,93270084,84293052,80246713,58749,96709982,45919976,51472275,66885828,91240048,9829782,71965942,9860968,11923835,90158785,76703609,13470059,45790169,73222868,27187213,6497830,72738685,8634541,14723410,6819644,70782102,62428472,51315460,38510840 +168541,61741594,83789391,90457870,50188404,44177011,55189057,84349107,14093520,88444207,73168565,20486294,4069912,46870723,4119747,24591705,91240048,38061439,669105,36753250,39171895,29466406,22435353,48360198,57803235,2208785,30694952,94090109,53632373,75543508,31904591,40027975,42692881,29188588,1197320,81677380,69355476,71333116,67793644,8099994,62936963,53842979,51047803,7104732,5832946,33201905,24058273,12348118,66667729,96735716,66250369,56424103,34295794,68644627,57240218,8128637,48673079,67451935,51590803,33304202,5970607,48088883,70727211,30163921,19376156,76703609,11161731,48892201,15535065,61859143,9058407,41481685,43376279,81805959,26863229,43501211,96709982,89214616,63967300,62232211,28734791,57393458,4806458,56473732,7229550,34044787,93053405,66045587,66832478,94711842,46851987,29029316,67474219,22108665,99524975,52060076,66611759,19457589,65454636,18833224,61712234,60106217,30463802,21533347,44983451,63015256,14349098,30998561,84293052,86242799,78218845,22766820,55247455,13468268,74357852,49598724,21993752,30139692,79191827,90310261,66319530,66903004,38494874,37183543,77312810,15064655,38341669,85711894,60581278,73031054,62430984,29819940,36812683,20765474,10366309,47887470,6819644,99604946,37192445,40534591,10309525,17081350,6808825,39553046,32274392,80316608,75153252,52204879,37739481,72725103,84406788,1239555,247198,85117092,66885828,92398073,71920426,26392416,16097038,27665211,44627776,18411915,91727510,96318094,66271566,2717150,53666583,76330843,59318837,7423788,36312813,35192533,3880712,89804152,50668599,37659250,40781854,68939068,16971929,60430369,81898046,31126490,68875490,71965942,38022834,55753905,52293995,38658347,92692978,4204661,47361209,48774913,20645197,24826575,92033260,98371444,6153406,37620363,56515456,75820087,73235980,70372191,79821897,47090124,60102965,70004753,53084256,96193415,59501487,112651,44847298,55850790,19486173,4668450,33249630,81855445,92554120,56153345,32161669,99965001,69697787,8115266,51472275,91802888,17539625,62496012,71300104,17037369,92353856,77694700,11365791,87710366,83302115,65074535,54199160,4515343,92787493,7011964,50567636,45919976,44915235,98130363,6221471,59614746,508198,67513640,76315420,64602895,19891772,40984766,7955293,15148031,82726008,51464002,44842615,35996293,61373987,73021291,72495719,2204165,61960400,76434144,56531125,77300457,23110625,44245960,43357947,29959549,97641116,55770687,74110882,77620120,76671482,84002370,526217,5482538,40152546,80014588,99333375,38365584,20140249,13348726,99549373,22942635,46540998,57658654,91957544,71316369,62740044,89046466,98648327,321665,89078848,27625735,84840549,43322743,44473167,68000591,90004325,87598888,95010552,90090964,91831487,84166196,35512853,84684495,24733232,72614359,4091162,2917920,52261574,29994197,34698428,26664538,21001913,81274566,30787683,8696647,34493392,29834512,68816413,47213183,93790285,45848907,9829782,30891921,26114953,40356877,14220886,43045786,83651211,45049308,79942022,80555751,98943869,23134715,27325428,3233084,98739783,83210802,18783702,30218878,42782652,65271999,33237508,43506672,31733363,82886381,7687278,72274002,72357096,45794415,45407418,44784505,75229462,91664334,8729146,53648154,3487592,6986898,3294781,29516592,71083565,77284619,24619760,1022677,32250045,63059024,62693428,23569917,17727650,17058722,73124510,30395570,81172706,68204242,85023028,68316156,87720882,15075176,36505482,9175338,61928316,83133790,52613508,72732205,82532312,54058788,26292919,76960303,5267545,874791,12416768,14731700,67281495,39847321,57241521,55602660,39801351,17365188,34667286,17957593,9603598,51507425,7919588,44846932,12024238,37675718,54263880,75777973,70420215,64087743,61815223,3773993,44889423,32159704,86460488,44060493,77187825,19272365,66322959,5822202,70800879,33431960,65715134,27041967,9886593,59981773,3633375,21673260,6871053,85242963,569864,6525948,89419466,415901,18131876,89499542,57359924,82339363,84904436,99515901,9398733,18466635,85116755,36396314,70596786,30811010,37560164,1936762,13470059,34432810,22997281,4199704,14947650,77272628,79136082,34698463,54606384,30569392,1431742,38645117,9599614,3088684,54232247,9575954,34236719,62051033,14626618,82052050,26102057,45790169,88047921,79806380,86301513,40686254,70367851,5073754,27185644,78909561,81853704,44844121,77301523,72793444,35092039,87160386,22405120,42644903,78589145,20642888,93359396,17386996,6497830,59177718,57248122,68694897,16405341,37435892,53802686,17894977,99125126,9860195,10649306,18663507,75128745,20885148,59329511,34497327,18504197,94076128,80251430,1375023,92541302,72278539,63628376,26998766,20867149,96852321,26275734,89811711,23848565,29635537,67031644,94516935,20002147,8634541,26397786,31161687,16595436,65047700,7646095,83083131,49641577,4099191,91990218,17764950,1788101,15948937,48260151,74724075,75986488,824872,77377183,42073124,96906410,45237957,92215320,54427233,16684829,64157906,16445503,60176618,90158785,92867155,58208470,55615885,94360702,9623492,6038457,83155442,24915585,47792865,23740167,99224392,97281847,83150534,28796059,39201414,2331773,6111563,90654836,73392814,86001008,62490109,30764367,11923835,66663942,61263068,19101477,99297164,50007421,13819358,94539824,33336148,18806856,28928175,94911072,47708850,95395112,54987042,78766358,12571310,72777973,89699445,62803858,83533741,24953498,28550822,60393039,17068582,21289531,64055960,7300047,64848072,65081429,30653863,176257,47298834,26139110,38009615,93933709,58224549,71657078,65338021,37891451,22129328,3235882,41442762,99226875,8733068,78561158,74137926,8807940,69848388,1204161,68041839,8129978,45075471,96726697,78300864,33895336,7502255,40781449,20568363,36808486,32699744,83747892,57961282,95957797,19939935,68128438,70036438,65851721,39986008,48893685,17146629,10961421,91141395,55143724,45667668,93562257,98948034,8791066,21070110,33797252,86022504,61728685,9188443,17976208,73786944,65017137,3509435,37957788,59197747,59371804,64098930,53393358,99658235,74441448,58180653,68702632,97379790,40197395,7517032,79922758,92692283,68102437,19898053,77072625,89637706,23194618,97057187,82779622,72373496,8373289,54663246,15015906,90013093,78785507,74743862,55669657,75135448,82651278,76540481,92071990,49597667,79738755,65275241,65880522,45306070,62115552,31727379,49271185,50806615,36933038,32426752,33628349,45995325,92803766,5111370,49328608,45996863,6505939,85695762,86821229,86543538,44550764,63152504,36139942,4434662,1250437,55960386,51315460,62552524,25636669,25842078,69136837,4798568,2891150,10358899,95726235,38008118,88130087,92998591,73222868,20836893,47738185,36468541,59109400,86798033,72238278,23432750,88251446,36780454,60955663,82897371,33061250,6793819,93566986,78602717,33565483,14723410,77898274,14045383,57020481,49236559,749283,26507214,56955985,63506281,93515664,51466049,29401781,92530431,34946859,20681921,42967683,54517921,91938191,68824981,93270084,67030811,79657802,96420429,82178706,28851716,18001617,7066775,58749,98462867,17857111,40865610,61271144,12664567,29932657,40677414,73617245,71522968,61982238,79880247,68110330,4787945,37224844,67084351,76170907,3183975,84127901,17385531,49882705,61623915,61141874,90272749,83269727,80246713,45617087,39373729,61897582,44481640,30543215,11581181,99917599,16099750,59910624,78884452,63372756,51149804,41380093,9860968,2607799,16387575,36580610,85571389,45381876,26063929,9257405,77787724,43152977,13862149,10597197,36135,22450468,75407347,33935899,66137019,33699435,42307449,8651647,58751351,69605283,95581843,88653118,8913721,55470718,31491938,98653983,30366150,26493119,43933006,66428795,83368048,3233569,82427263,26734892,82327024,38256849,89217461,22879907,7685448,41245325,69641513,99021067,82979980,17016156,20122224,84187166,29065964,91281584,16019925,28787861,32151165,8055981,42947632,54868730,88904910,67124282,95290172,61859581,62357986,72358170,94595668,22113133,51426311,41092102,92604458,77413012,72738685,45990383,97011160,4095116,72019362,6521313,30090481,70782102,36930650,42199455,99861373,74452589,93057697,10453030,16424199,47893286,65038678,67227442,48658605,70541760,71469330,13173644,35456853,56461322,4978543,62452034,33553959,15163258,54762643,69579137,51715482,99690194,95251277,4985896,59405277,27187213,89996536,81774825,36845587,86118021,75052463,78549759,37280276,44479073,24168411,97561745,27411561,14326617,41092172,26744917,9259676,22200378,15902805,97940276,53888755,14363867,42237907,18699206,91255408,33123618,30771409,88698958,50702367,26776922,66116458,79322415,76825057,16380211,36189527,21919959,51588015,62428472,54014062,15536795,13231279,24314885,16054533,32590267,99971982,53547802,4064751,29510992,97783876,83948335,56484900,11543098,44664587,21397057,69255765,45428665,1569515,36685023,44348328,11757872,22721500,4508700,48395186,74614639,90061527,99729357,73109997,16567550,48675329,5640302,38510840,37501808,6157724,57163802,12303248,69786756,32058615,37166608,10264691,62762857,7182642 +4515343,61373987,17764950,29065964,9623492,74357852,93933709,55247455,39201414,24058273,11923835,66885828,68644627,97783876,60430369,75777973,96193415,29994197,77312810,20885148,44983451,75543508,20002147,14626618,40197395,69355476,415901,55753905,44915235,67451935,43357947,75986488,6497830,29029316,9603598,77787724,10309525,16445503,95957797,321665,168541,34698463,70596786,98371444,20140249,17727650,4099191,70004753,4668450,62430984,88653118,46870723,10366309,93057697,83533741,81853704,70800879,48892201,39553046,34698428,9886593,71657078,7517032,79821897,53648154,96726697,65275241,38494874,5832946,67793644,44784505,86022504,44844121,2717150,31904591,99658235,7955293,76434144,65047700,66667729,29834512,89046466,57240218,73222868,92604458,20765474,9188443,37224844,53547802,7502255,50806615,90457870,72725103,71965942,66428795,74110882,15015906,34044787,99861373,72373496,77620120,3088684,15148031,35192533,1431742,77272628,82897371,45996863,16595436,76960303,37620363,99690194,4787945,72738685,97057187,54663246,32590267,57163802,27665211,69786756,65715134,43045786,56531125,61982238,44473167,6221471,2204165,85116755,96318094,31126490,40027975,47792865,52261574,86001008,30463802,39986008,40865610,1569515,36685023,76703609,93515664,30090481,74441448,14363867,77284619,82052050,62452034,60581278,19101477,32426752,89499542,18833224,30163921,86543538,98648327,78909561,4064751,49882705,91141395,98739783,42073124,67084351,34946859,59371804,95010552,5970607,80316608,8651647,3509435,56424103,71522968,24168411,17539625,54199160,31733363,89214616,749283,94090109,85023028,50567636,80246713,8128637,23848565,18411915,59501487,4434662,99549373,45794415,38658347,55470718,99917599,4204661,8634541,28796059,66319530,94711842,1197320,32250045,66137019,27187213,49271185,18131876,33061250,82651278,54987042,79191827,37183543,26493119,30543215,6505939,5111370,14093520,6793819,35092039,54058788,73124510,63059024,17957593,67124282,8129978,93270084,874791,2607799,67474219,93562257,22405120,29510992,99021067,30771409,8807940,21993752,34236719,45790169,37739481,569864,96420429,39171895,36468541,68000591,8791066,72777973,57658654,2208785,83150534,66832478,58208470,38022834,26664538,45049308,84406788,98653983,47361209,91802888,96906410,26863229,69641513,82979980,3294781,54868730,30787683,68102437,84684495,22721500,18504197,50702367,63628376,38645117,67281495,75229462,26063929,26998766,36505482,87710366,66663942,35996293,29466406,40781854,73168565,92398073,14326617,44348328,3183975,45407418,11581181,27325428,669105,51047803,36812683,78766358,83368048,84349107,44177011,70036438,36312813,66903004,9575954,56153345,7687278,63152504,78602717,42237907,28734791,68702632,22113133,91831487,59197747,30139692,76170907,83210802,508198,50668599,33797252,66611759,35456853,82427263,61263068,31161687,8729146,68204242,8055981,47213183,62490109,63372756,58180653,4798568,4119747,16380211,44842615,20867149,30366150,1788101,89637706,89078848,50188404,12348118,30891921,42782652,69136837,15163258,76315420,15075176,43376279,16097038,92353856,65074535,89419466,7646095,72357096,49236559,29516592,25636669,84840549,99125126,90272749,40686254,6038457,29819940,92803766,26507214,8099994,10597197,87598888,824872,82726008,43506672,1204161,57803235,24619760,96735716,48893685,78218845,41092102,51315460,44481640,57241521,9058407,42967683,22450468,94539824,20122224,27411561,91938191,72274002,13231279,52204879,80014588,48774913,44479073,59318837,98130363,33431960,45919976,86301513,84127901,61960400,23569917,96709982,62762857,25842078,88047921,38008118,51472275,44846932,36396314,84293052,83302115,53666583,74137926,24733232,1250437,37192445,91727510,53888755,33628349,83083131,86821229,21397057,21289531,70372191,15535065,44889423,56484900,33237508,84904436,18806856,8733068,24953498,11543098,19486173,95581843,29188588,64157906,82886381,38061439,15536795,6153406,78589145,61623915,89699445,59405277,10961421,55770687,68824981,61712234,36780454,69579137,43322743,58749,66250369,26139110,16684829,73786944,14731700,52613508,92071990,45428665,1375023,90158785,92692283,18663507,76671482,6525948,40781449,17037369,81677380,79922758,60102965,11757872,41092172,58224549,44550764,39373729,14723410,59109400,10358899,36930650,33553959,48360198,72614359,61859143,36753250,55189057,79738755,37675718,26114953,19376156,68041839,72238278,12571310,94516935,36933038,72495719,61141874,20681921,62936963,33895336,51590803,15902805,40534591,65880522,99297164,71333116,92998591,47738185,4508700,99524975,526217,80555751,58751351,9829782,14045383,57248122,87720882,48088883,74743862,40984766,59177718,98948034,33249630,65271999,7229550,21070110,98462867,20645197,61741594,34497327,16424199,63967300,96852321,89811711,14947650,13173644,92554120,5267545,30764367,3233084,43933006,99604946,61271144,90310261,68110330,91957544,90654836,12664567,22200378,16971929,91990218,16567550,42644903,62232211,6871053,53802686,30218878,9257405,62552524,67513640,34432810,83651211,90061527,59614746,33699435,2917920,95726235,37166608,77187825,37891451,89996536,17385531,33201905,16405341,17068582,20486294,91664334,95290172,17081350,9599614,19939935,72019362,62740044,67030811,78300864,46851987,71300104,24826575,29959549,26292919,47090124,46540998,56955985,43152977,14220886,40152546,85242963,97641116,95395112,79806380,92215320,76540481,4978543,44060493,99515901,91255408,77898274,6986898,91240048,54517921,19272365,18783702,32274392,51588015,22942635,88251446,26392416,45848907,28928175,6111563,61897582,44245960,54606384,66116458,76330843,66271566,83269727,64087743,19457589,59910624,30811010,54762643,5482538,72732205,78561158,17976208,33565483,65338021,65038678,73392814,21673260,37435892,75052463,64848072,26744917,93053405,62693428,3235882,83789391,68816413,27185644,81172706,73031054,70727211,22879907,39801351,8373289,18699206,49328608,59329511,82339363,93790285,77072625,85571389,26776922,38510840,56461322,99965001,80251430,10649306,45075471,77413012,38256849,74724075,83133790,20836893,70541760,38341669,99971982,22766820,6808825,81774825,86460488,30569392,4199704,68316156,34667286,20568363,40356877,27625735,79657802,79942022,7919588,18466635,34493392,12416768,73617245,37957788,28550822,81898046,32699744,88130087,112651,1022677,85117092,37280276,99333375,97379790,49641577,62051033,98943869,79322415,41442762,32159704,66322959,16054533,51464002,64098930,2331773,54263880,75407347,41481685,84187166,90013093,33304202,26734892,79880247,71920426,42947632,62496012,45237957,37560164,36189527,41245325,5822202,62803858,75128745,21919959,56473732,48260151,65017137,7423788,60955663,41380093,17386996,26275734,7104732,57961282,38009615,34295794,78549759,37659250,45667668,8696647,82327024,79136082,16019925,75153252,44664587,90090964,28851716,29635537,1239555,82178706,70367851,24915585,69605283,57393458,4069912,5640302,64602895,84002370,97281847,36808486,55850790,99224392,13862149,57359924,82779622,78884452,31727379,77694700,4095116,97561745,23194618,11365791,55960386,81805959,3880712,97011160,7011964,49598724,61728685,52060076,36580610,30998561,4806458,23110625,30395570,78785507,56515456,92787493,9175338,45617087,60106217,16099750,71083565,54427233,17016156,73235980,16387575,61859581,94911072,17058722,57020481,21533347,92692978,3233569,26397786,81855445,4985896,42307449,64055960,55602660,17365188,68939068,48395186,45990383,47708850,30653863,32151165,13468268,13470059,53393358,5073754,71469330,65851721,3487592,15064655,54014062,32161669,97940276,22435353,7685448,44847298,23740167,26102057,7182642,52293995,1936762,2891150,55143724,94076128,72793444,6521313,84166196,33123618,67227442,19898053,86118021,77377183,10453030,13348726,69848388,23134715,24591705,33935899,63015256,76825057,69697787,51426311,88444207,55615885,74452589,8115266,94360702,20642888,44627776,75820087,65454636,4091162,60176618,92541302,68875490,27041967,65081429,69255765,45995325,53084256,72358170,29401781,19891772,88698958,38365584,42199455,247198,22129328,30694952,33336148,42692881,6819644,48673079,83155442,73109997,176257,85695762,7300047,66045587,49597667,22997281,15948937,48658605,9259676,77301523,9860195,47298834,39847321,90004325,17894977,81274566,70420215,22108665,53632373,93566986,40677414,9398733,50007421,99226875,62357986,35512853,67031644,94595668,59981773,17857111,47887470,12024238,68694897,92033260,51715482,93359396,3773993,8913721,61815223,31491938,82532312,45306070,29932657,83948335,51507425,77300457,6157724,73021291,43501211,9860968,62115552,51149804,23432750,63506281,47893286,91281584,92867155,89217461,17146629,21001913,62428472,87160386,71316369,53842979,86798033,92530431,83747892,7066775,51466049,86242799,54232247,75135448,48675329,24314885,13819358,55669657,3633375,95251277,89804152,36139942,74614639,68128438,10264691,11161731,12303248,18001617,36135,45381876,88904910,60393039,37501808,36845587,32058615,28787861,14349098,61928316,70782102,99729357,85711894,72278539 +37183543,24619760,29994197,89214616,92033260,5267545,45306070,88653118,99690194,17764950,9188443,91664334,90090964,9886593,80316608,68644627,30569392,57803235,89078848,90310261,81853704,72777973,35092039,76671482,78602717,69641513,8807940,36808486,96726697,3487592,59177718,29029316,18466635,47738185,58224549,21993752,34493392,38061439,96735716,60430369,98371444,44983451,51047803,83789391,36753250,54663246,82979980,22942635,57961282,33249630,68204242,4119747,526217,45848907,62936963,83150534,84904436,70036438,24058273,27665211,66250369,36312813,36468541,91802888,29819940,55753905,70420215,55615885,30653863,16097038,45428665,43501211,14947650,8099994,40677414,62740044,86001008,10358899,20568363,15163258,99549373,33797252,75543508,4064751,72274002,6221471,72373496,83155442,7300047,55602660,58749,61263068,99965001,97281847,65715134,96193415,168541,23194618,17539625,39847321,78589145,34236719,8733068,44889423,93933709,874791,4668450,36812683,39553046,20002147,72732205,28796059,6808825,16595436,5832946,22450468,8115266,3233084,50702367,59981773,45919976,62051033,56531125,99917599,68000591,30218878,67451935,56424103,55850790,30998561,11543098,80555751,38009615,32590267,99524975,18699206,90272749,95957797,96852321,9257405,67474219,44627776,17037369,61271144,75777973,29188588,20885148,22129328,92215320,22435353,415901,73392814,89804152,7517032,63015256,63967300,14326617,13173644,35512853,55189057,42644903,59371804,76434144,669105,16445503,15064655,98462867,12024238,90654836,36135,20765474,99515901,51588015,62496012,72725103,94711842,67793644,45617087,56515456,38022834,36780454,74614639,32161669,34044787,2208785,1197320,48893685,61728685,66271566,61815223,21001913,98948034,45407418,16387575,18783702,31161687,49271185,11161731,569864,72019362,85571389,66045587,44481640,98739783,16019925,33123618,19376156,45996863,2891150,77312810,54762643,508198,73124510,40356877,53802686,1788101,32699744,26392416,92692978,72614359,78909561,20486294,71333116,41245325,44550764,53842979,48260151,4204661,31904591,38658347,65454636,73031054,26139110,41092102,97379790,70372191,19939935,65338021,7104732,43376279,87720882,7502255,44177011,33336148,30764367,70004753,4806458,26292919,16567550,31491938,6819644,89046466,36505482,10366309,23848565,2204165,69848388,45794415,11365791,74357852,4099191,8651647,52261574,3235882,2717150,76540481,45075471,29635537,77694700,42237907,73617245,48673079,6986898,70800879,29834512,9575954,57393458,97057187,88047921,83533741,14220886,85023028,41092172,78766358,72738685,56473732,66667729,14626618,40781449,7646095,17068582,11923835,10649306,93057697,47361209,66832478,48892201,99224392,30787683,37620363,99729357,66663942,76703609,82052050,6497830,42967683,68816413,96318094,12664567,14349098,77377183,1375023,55470718,9623492,50567636,74724075,7229550,55143724,17976208,78884452,69255765,32426752,77072625,24826575,36580610,12348118,43506672,57248122,85242963,5970607,59197747,86460488,99021067,48395186,9175338,20645197,17365188,92803766,15535065,69786756,176257,80246713,36930650,6505939,5111370,62430984,84293052,13862149,80014588,62115552,74110882,22766820,33553959,38365584,99125126,59501487,68694897,90004325,27185644,93515664,47213183,30811010,83948335,19457589,48774913,55960386,19486173,50188404,5073754,8055981,53547802,59405277,29401781,89699445,40027975,52204879,76170907,8696647,43357947,51590803,77301523,42199455,18833224,7011964,1250437,43045786,23134715,97940276,18663507,51472275,56461322,45237957,14093520,79191827,23569917,65275241,84349107,79880247,97011160,19272365,45790169,89637706,1431742,70541760,4515343,42073124,82427263,50668599,37739481,35456853,98943869,30395570,67281495,40534591,5822202,40197395,9603598,34698428,14731700,3509435,34497327,73222868,87598888,85116755,95395112,4069912,2917920,99971982,13470059,64602895,71920426,62452034,30463802,68875490,40781854,89217461,47298834,83368048,39201414,54606384,14045383,87710366,38256849,4508700,36685023,81855445,49641577,45381876,40152546,84840549,58180653,81898046,54427233,79136082,29959549,26102057,9829782,1936762,66137019,6157724,96420429,21070110,3183975,30771409,90457870,7685448,33628349,74441448,58208470,47090124,61982238,78218845,66611759,77187825,45995325,24733232,66428795,73168565,46851987,37957788,15075176,36139942,90061527,83210802,52613508,26734892,83651211,29932657,96906410,70596786,15015906,84127901,92998591,6153406,30139692,3088684,10453030,30163921,16405341,82779622,3880712,9398733,36396314,64848072,94911072,37166608,27041967,53648154,13819358,78785507,69697787,92530431,62232211,7182642,95251277,43933006,19101477,11757872,65271999,20681921,73021291,83269727,56153345,98648327,58751351,26063929,39801351,6793819,13348726,85117092,53084256,41380093,79922758,49236559,62803858,42782652,9058407,28550822,20642888,22879907,247198,33201905,82327024,1022677,21289531,91240048,77284619,16971929,86798033,99861373,63059024,82726008,60176618,78300864,82339363,35996293,77898274,55247455,15536795,21673260,10961421,68316156,45667668,62552524,49597667,24915585,66903004,6038457,18806856,10309525,89811711,63628376,62490109,76330843,50007421,77413012,24314885,2331773,93053405,27625735,64157906,71083565,8913721,17957593,37891451,92604458,71965942,63372756,3233569,7423788,62693428,72238278,81677380,30891921,12303248,55770687,92554120,97783876,95290172,16099750,72357096,54232247,74137926,22200378,93270084,71316369,98653983,35192533,61928316,23740167,65081429,91727510,83747892,38645117,94076128,92353856,76960303,77272628,17146629,44784505,48675329,67084351,40865610,34946859,91255408,34667286,63506281,25636669,61859143,7687278,57020481,15148031,59318837,79806380,33431960,37192445,8634541,53666583,59109400,18411915,19898053,61712234,72495719,64087743,68824981,18504197,22997281,51507425,95726235,12416768,16054533,4798568,98130363,92787493,17058722,1569515,99604946,3773993,30694952,97561745,86301513,61897582,36933038,80251430,61373987,65017137,7955293,46870723,16684829,65880522,75153252,92398073,824872,4434662,29466406,23110625,24168411,7919588,29065964,47792865,37435892,16380211,34295794,52060076,86022504,46540998,66322959,69355476,82178706,31733363,48658605,54014062,40984766,57658654,47887470,3294781,8729146,54058788,65074535,13468268,72278539,29510992,29516592,83302115,94090109,48360198,1204161,43152977,85711894,33237508,30366150,4985896,40686254,84684495,9599614,44060493,78549759,11581181,4095116,20122224,81774825,37501808,44348328,33895336,66116458,65047700,28928175,9860968,47708850,36189527,68702632,84187166,77787724,60581278,24953498,39171895,82532312,4199704,83133790,69579137,93562257,51426311,96709982,50806615,87160386,91957544,64098930,99658235,18131876,27187213,49328608,15902805,66885828,44844121,68102437,71300104,44842615,9860195,22721500,41481685,30543215,83083131,19891772,93566986,26776922,85695762,2607799,73235980,8791066,75407347,54199160,79738755,99226875,62357986,81172706,57241521,88130087,20867149,17894977,39986008,16424199,64055960,97641116,32151165,749283,34432810,8373289,49598724,72358170,112651,53632373,26744917,75229462,79657802,53393358,68041839,92692283,59910624,79821897,39373729,76315420,31126490,13231279,61859581,56484900,63152504,52293995,26664538,99333375,26275734,61741594,4091162,88698958,45990383,3633375,8128637,17727650,57359924,76825057,59614746,95581843,77620120,44846932,94516935,74743862,42307449,17385531,33935899,66319530,33699435,75052463,91938191,26493119,22108665,82897371,37280276,57240218,71469330,42692881,90013093,26397786,91831487,75128745,82886381,37659250,92541302,68110330,68128438,4787945,27325428,26114953,82651278,7066775,5482538,89419466,86543538,75135448,38008118,43322743,32250045,26998766,62428472,81274566,51715482,72793444,17016156,67030811,14723410,88251446,32274392,24591705,91281584,86242799,67124282,37675718,69136837,54987042,1239555,70367851,60393039,26507214,57163802,73109997,44479073,12571310,6111563,17857111,65038678,91141395,33304202,54517921,73786944,26863229,8129978,55669657,28734791,5640302,33565483,89996536,38510840,21533347,84406788,321665,71657078,75820087,92071990,14363867,92867155,44245960,94595668,88444207,37224844,86118021,56955985,79942022,38494874,44473167,34698463,25842078,70727211,74452589,91990218,75986488,77300457,10264691,17386996,78561158,48088883,61960400,86821229,94539824,6871053,38341669,65851721,21397057,6525948,61623915,9259676,79322415,60106217,51464002,27411561,60955663,22405120,30090481,36845587,31727379,32159704,20836893,15948937,44847298,20140249,81805959,61141874,42947632,44664587,18001617,68939068,44915235,10597197,4978543,32058615,93790285,95010552,60102965,45049308,54868730,99297164,69605283,51315460,28851716,93359396,67031644,51466049,90158785,89499542,59329511,84166196,67227442,71522968,84002370,53888755,21919959,49882705,22113133,47893286,94360702,67513640,70782102,28787861,37560164,23432750,33061250,62762857,88904910,17081350,6521313,51149804,41442762,54263880 +99297164,70541760,3294781,20642888,70596786,53084256,20486294,51047803,19486173,78589145,75543508,40152546,90310261,34698428,33304202,62496012,64848072,34493392,68816413,6521313,79806380,59177718,60430369,32161669,83083131,8115266,68644627,45667668,15535065,46870723,61741594,67793644,92541302,15536795,75777973,43376279,79136082,72357096,74357852,39201414,13468268,37620363,68000591,73124510,669105,14731700,30787683,874791,72278539,7687278,11923835,81853704,64055960,59614746,42307449,33336148,54427233,65275241,18131876,36753250,14045383,62430984,77620120,30395570,5640302,21533347,83651211,61859143,49641577,40356877,32159704,70800879,44842615,749283,65017137,19891772,7423788,22766820,12348118,3233084,92692978,9623492,2717150,26392416,66271566,75229462,7502255,87160386,94090109,62803858,3183975,89046466,66250369,72495719,96318094,54058788,63967300,10358899,8373289,55247455,26507214,23569917,56424103,99524975,9188443,53842979,15015906,28734791,20867149,38645117,23134715,55189057,40197395,40984766,67281495,44550764,77072625,73168565,7011964,62936963,88047921,46851987,68939068,57803235,168541,22997281,81855445,42782652,83789391,78602717,38658347,12416768,96420429,57359924,64087743,1375023,70004753,66832478,89637706,96906410,98943869,55143724,4119747,4204661,9886593,824872,99965001,16097038,59501487,45237957,17037369,4434662,98462867,52261574,68316156,99515901,79657802,22879907,29819940,14626618,44060493,21673260,55753905,94539824,6153406,45996863,16405341,8099994,37957788,14349098,26102057,71316369,80246713,85116755,86798033,53547802,21001913,7300047,58749,75128745,23848565,97057187,71469330,50007421,40027975,74614639,39847321,44481640,91831487,54987042,71522968,30366150,22405120,62740044,74441448,14093520,99861373,19939935,75153252,18001617,30764367,24953498,82897371,44844121,34236719,45790169,84684495,36580610,93053405,8128637,55615885,19101477,96726697,99226875,29029316,54606384,82178706,92353856,27411561,26139110,36505482,81898046,99549373,46540998,89214616,66885828,40677414,18699206,83533741,77413012,57241521,81677380,59109400,19272365,26063929,65338021,91664334,45049308,20836893,60106217,65715134,1197320,62357986,80316608,63372756,20122224,1204161,40781449,30891921,30463802,96193415,51715482,88653118,82339363,82532312,95957797,16567550,15064655,26114953,17727650,68824981,77312810,33628349,87720882,31727379,40686254,20765474,8913721,93515664,48260151,33895336,92803766,57248122,95290172,50702367,89078848,44245960,44784505,4515343,79738755,53802686,6793819,98648327,35092039,72732205,48893685,93057697,25842078,32274392,96709982,69641513,20002147,91255408,68128438,72019362,29510992,59371804,36812683,86460488,39171895,95395112,4064751,55850790,13470059,6986898,26664538,44889423,30139692,91938191,53632373,78218845,34698463,30694952,54263880,54199160,5970607,84904436,50806615,44983451,29834512,37224844,26734892,9860968,99917599,14220886,98130363,14947650,31126490,40865610,89804152,67030811,7229550,9603598,10264691,72274002,64157906,78884452,42073124,80555751,38494874,61897582,73617245,17539625,26292919,76825057,77284619,16387575,5482538,47361209,76540481,14326617,82726008,39801351,82779622,6808825,37192445,5267545,1431742,45428665,69355476,77377183,65047700,21070110,18466635,73031054,10309525,78766358,41245325,45794415,94911072,86543538,84840549,9860195,97940276,63059024,7646095,27625735,58751351,45407418,95581843,4985896,33237508,24168411,68694897,15163258,86301513,94516935,65038678,37659250,33797252,23432750,9259676,73786944,60581278,61712234,13862149,83210802,21289531,29466406,36396314,6505939,26863229,83133790,14723410,40534591,4798568,6871053,36845587,18783702,60102965,53888755,92787493,66116458,68204242,88130087,26275734,79821897,74110882,96735716,87710366,29188588,61859581,90013093,97561745,76170907,69786756,81172706,8791066,9398733,6221471,61623915,37560164,7517032,43501211,3233569,63015256,28550822,66322959,95726235,44177011,76330843,88251446,2208785,112651,24058273,17764950,62232211,8807940,99690194,54014062,74724075,65454636,84293052,32590267,57020481,56473732,33553959,4668450,48673079,44348328,94076128,97011160,29959549,75407347,84349107,49271185,39986008,38510840,43933006,415901,30163921,27665211,53666583,9058407,85242963,8634541,28851716,35996293,55470718,4787945,22129328,70420215,26397786,41442762,95010552,41092102,34432810,92604458,17058722,43045786,3235882,62428472,10961421,50668599,59318837,9829782,43322743,14363867,62693428,24619760,63152504,64098930,92554120,66428795,33201905,29994197,81805959,49236559,94360702,28928175,32151165,19376156,2607799,67451935,16054533,85711894,69579137,22200378,30653863,90090964,34497327,43152977,9175338,85695762,62452034,48774913,35192533,54663246,34946859,99021067,4099191,68102437,38256849,72358170,88904910,16595436,38061439,42644903,6819644,22450468,83150534,73392814,51590803,44846932,7104732,8696647,96852321,36685023,30811010,58208470,61960400,93566986,92867155,47708850,36933038,5832946,93933709,44473167,57240218,51588015,38365584,2204165,88698958,77272628,33123618,7919588,13231279,51464002,56515456,72793444,92692283,43506672,42199455,16971929,21993752,526217,50567636,20140249,75052463,66045587,51472275,76315420,37435892,33935899,29401781,4069912,11161731,48658605,47213183,36930650,47090124,81274566,33249630,56461322,31904591,6157724,98371444,78561158,8651647,67124282,76960303,83948335,65074535,33699435,31733363,17365188,72614359,13348726,3880712,20568363,39553046,17146629,10453030,12571310,15148031,11543098,87598888,18663507,11581181,4508700,93270084,56955985,91240048,76671482,17068582,44664587,98739783,37891451,69605283,76434144,31491938,61982238,41481685,91281584,176257,86022504,80251430,71920426,36780454,54868730,47887470,27325428,70367851,97783876,44479073,79880247,19457589,30998561,18504197,70727211,89699445,12024238,56531125,61815223,89996536,1250437,10597197,79922758,569864,69697787,24591705,52293995,36189527,77787724,45381876,65081429,20885148,38341669,6497830,58180653,67474219,41380093,49598724,26493119,98653983,79942022,16019925,7955293,36312813,9257405,53393358,77301523,99658235,22435353,61728685,84002370,5073754,92998591,62490109,61373987,59405277,4199704,27187213,23194618,68875490,69255765,38022834,30543215,17857111,67031644,69848388,7066775,52060076,82979980,51149804,50188404,45995325,74452589,92071990,17976208,68041839,62051033,74137926,36135,49328608,2891150,247198,91802888,79191827,32250045,72238278,29065964,82427263,84187166,18411915,93790285,64602895,22113133,86242799,59197747,62762857,71333116,89217461,59981773,8129978,33061250,42947632,86821229,61141874,66137019,86001008,76703609,66663942,71300104,68702632,17385531,65851721,80014588,34295794,62552524,57961282,16380211,53648154,33565483,67513640,3773993,17894977,66903004,20645197,77694700,97379790,78785507,89419466,47792865,78549759,1788101,19898053,61263068,12664567,42967683,22108665,73222868,1022677,44915235,92398073,7685448,57658654,91141395,27185644,71965942,11365791,40781854,93562257,17016156,62115552,72738685,30569392,37183543,90061527,30771409,58224549,82327024,91727510,83155442,5111370,3509435,61928316,48088883,86118021,90457870,99224392,45990383,89499542,47298834,9575954,88444207,48395186,85023028,90272749,85571389,45919976,82886381,36468541,16684829,29635537,66611759,48892201,45848907,72777973,56153345,13819358,24733232,75135448,32699744,65880522,3633375,37166608,26998766,4091162,15948937,73109997,73235980,12303248,92215320,6038457,74743862,47893286,39373729,321665,60955663,57163802,11757872,4978543,24826575,44847298,18833224,78909561,65271999,55960386,9599614,17957593,28796059,37739481,84127901,83302115,63628376,42692881,44627776,84166196,10649306,52613508,52204879,60393039,54517921,79322415,99729357,78300864,85117092,30090481,55770687,1569515,70036438,51315460,31161687,23110625,94595668,90654836,24314885,21919959,71657078,70372191,72725103,3487592,18806856,99971982,99125126,37501808,36808486,90004325,32058615,51426311,92033260,17081350,55602660,37280276,45617087,20681921,92530431,63506281,34044787,66667729,77300457,29516592,59910624,15902805,67227442,10366309,42237907,67084351,49597667,91957544,59329511,8733068,48360198,35512853,2917920,508198,71083565,1239555,54762643,7182642,57393458,77898274,69136837,8055981,35456853,82651278,16099750,22942635,8729146,54232247,43357947,3088684,33431960,6525948,66319530,29932657,16424199,51507425,16445503,77187825,37675718,91990218,49882705,6111563,93359396,99604946,61271144,1936762,94711842,83747892,45075471,28787861,51466049,75820087,98948034,81774825,26776922,47738185,48675329,26744917,99333375,4095116,27041967,17386996,82052050,34667286,5822202,30218878,23740167,83368048,60176618,21397057,24915585,83269727,73021291,36139942,89811711,55669657,70782102,38009615,38008118,75986488,84406788,45306070,56484900,41092172,32426752,22721500,68110330,90158785,72373496,15075176,2331773,97641116,4806458,95251277,13173644,25636669,97281847 +71920426,33628349,77413012,62740044,62496012,81677380,34493392,1250437,30787683,39986008,89811711,96735716,5970607,36505482,55470718,59177718,99729357,98462867,35092039,79136082,36933038,14947650,49236559,30569392,44842615,44889423,4119747,7955293,42782652,78602717,85116755,2607799,17037369,79657802,64848072,75986488,94090109,3487592,99524975,60955663,9575954,37435892,23194618,89699445,11161731,62762857,97011160,18783702,65017137,5073754,29819940,13468268,23848565,42947632,66832478,45848907,8791066,8696647,95581843,53547802,61859581,22942635,94595668,44177011,15015906,78589145,40197395,73617245,13231279,93053405,99861373,22721500,55753905,48673079,50668599,63967300,66137019,65271999,13173644,45306070,68041839,92692978,13470059,7229550,16019925,415901,48675329,34295794,92554120,15535065,45237957,54517921,54762643,61928316,91664334,44479073,65715134,34698463,81853704,65081429,9175338,71469330,8634541,52261574,41481685,9188443,88653118,47090124,18806856,50188404,68128438,12303248,45919976,40677414,17146629,14349098,63059024,47361209,77300457,9257405,1431742,77187825,91727510,36396314,17058722,74137926,26776922,26139110,4668450,37501808,11543098,43501211,58749,48260151,28928175,17068582,61373987,67124282,86022504,49328608,54014062,51315460,94516935,62115552,21993752,70036438,29834512,33123618,83155442,78300864,64087743,508198,68816413,80246713,23569917,50702367,3294781,86798033,30764367,5111370,4204661,72357096,77620120,95726235,53888755,45996863,24591705,29959549,66663942,33797252,49271185,42307449,30218878,669105,6871053,99549373,61982238,92787493,26392416,72777973,36580610,96726697,78218845,53393358,7182642,29466406,48360198,57163802,20885148,3183975,65880522,58180653,28851716,65275241,77694700,51466049,14093520,20002147,78766358,9886593,92803766,77284619,73222868,70541760,25636669,89214616,66428795,98943869,4806458,21673260,79322415,10366309,63506281,83789391,56424103,3633375,31491938,38008118,92215320,68000591,24826575,68204242,84127901,76170907,93057697,38022834,63372756,37183543,45407418,38061439,33237508,63152504,4064751,38645117,30163921,72793444,36753250,72278539,39847321,27665211,89637706,57803235,19272365,9398733,68824981,32161669,27325428,56515456,98948034,99515901,32250045,20765474,21070110,1788101,82178706,44473167,38494874,99690194,78909561,13819358,57359924,36780454,38256849,75135448,91255408,20486294,62430984,16380211,82532312,62803858,44983451,29994197,749283,13348726,37675718,53842979,11923835,55602660,46851987,70367851,77072625,59197747,9259676,17957593,7687278,72732205,19376156,24619760,59981773,6505939,54199160,76540481,3235882,6221471,48892201,86001008,93515664,16099750,95290172,29635537,94539824,92604458,37659250,55143724,29065964,22766820,91802888,14326617,42692881,41380093,6986898,39801351,112651,69579137,72373496,61623915,30139692,84904436,49598724,7011964,51590803,7300047,67031644,59614746,26493119,40686254,55850790,65047700,57393458,8913721,4434662,61263068,569864,82979980,33553959,15064655,36468541,82327024,92867155,99125126,3233569,90310261,48088883,26998766,15163258,19101477,22435353,66322959,82339363,69697787,28550822,40865610,29188588,83210802,98371444,68939068,55669657,43045786,60102965,8129978,68644627,26102057,34497327,6808825,40781449,83269727,17894977,35996293,4099191,8099994,33336148,20867149,55189057,6111563,33201905,68702632,58224549,5267545,96709982,4095116,19898053,27411561,30653863,68102437,68875490,8115266,4515343,18001617,76825057,39201414,63628376,62232211,42073124,57658654,88251446,76671482,65851721,95395112,39171895,67227442,28796059,8651647,74110882,83133790,86118021,20642888,45990383,44664587,75407347,90457870,66250369,15075176,43933006,85571389,47738185,79821897,35512853,96193415,51507425,82726008,51472275,9860968,44847298,1022677,77377183,30771409,7685448,29029316,92692283,82651278,68110330,69136837,80251430,89996536,9599614,57248122,44550764,83302115,92071990,38365584,69255765,53632373,78884452,70596786,11757872,36845587,83368048,55960386,67451935,9623492,96852321,52613508,92541302,4069912,66116458,83533741,99917599,80014588,51464002,66885828,92353856,62936963,49641577,3233084,29510992,42237907,47887470,70727211,6793819,93933709,70800879,45995325,88047921,50567636,60430369,92033260,88698958,17081350,83150534,42644903,9058407,40152546,14220886,36312813,35456853,10453030,70372191,87710366,2204165,55247455,72274002,32590267,47213183,48893685,53666583,59371804,38341669,65454636,70004753,21533347,34698428,64157906,52293995,17539625,44245960,64055960,18504197,17857111,84684495,9603598,56484900,76960303,247198,37620363,71083565,51047803,99965001,74357852,97281847,83747892,70420215,93562257,36930650,7104732,6153406,59318837,30395570,2717150,86242799,24058273,15902805,38510840,74724075,22405120,44784505,66611759,22200378,86301513,15148031,26114953,76330843,95957797,97940276,27625735,78561158,44060493,42967683,78785507,33249630,38658347,89419466,63015256,81274566,87720882,82779622,97057187,81172706,45381876,67084351,76434144,52060076,54427233,50007421,54987042,71333116,6157724,321665,79922758,74441448,71300104,99658235,17764950,53802686,62693428,4978543,18411915,47792865,16054533,8807940,72358170,44481640,61815223,90004325,57240218,62452034,67281495,26063929,12348118,66271566,7423788,32159704,53084256,98130363,85711894,12416768,22129328,2208785,43376279,97561745,62428472,72725103,40984766,30090481,43322743,23110625,10264691,4508700,73786944,19457589,88444207,82427263,88130087,84349107,3880712,7919588,72614359,168541,99333375,73031054,99226875,99224392,97783876,1197320,19939935,45075471,4091162,73392814,30998561,44348328,31126490,44844121,56153345,74614639,36135,83651211,4798568,77787724,41245325,75543508,8373289,30891921,26292919,23432750,33431960,12664567,33565483,4199704,67030811,6525948,60581278,69605283,9829782,79880247,50806615,13862149,26507214,67793644,31727379,72738685,49597667,1204161,8128637,66903004,94076128,18466635,91957544,14731700,67474219,62552524,10961421,17727650,10358899,95010552,10309525,86460488,61859143,87160386,90090964,39553046,30463802,6497830,42199455,72019362,41092102,61141874,3509435,56955985,82052050,69641513,93270084,6521313,59501487,8055981,37957788,24953498,20140249,77272628,58751351,71657078,29401781,18833224,36189527,5822202,90013093,62490109,81898046,33304202,24915585,65038678,33699435,30694952,64098930,37280276,526217,75128745,5640302,35192533,18131876,84166196,30543215,28734791,86543538,33895336,31904591,24168411,20681921,88904910,55615885,47708850,32274392,72495719,16445503,15536795,83083131,45428665,92530431,98648327,91831487,34946859,39373729,24733232,16971929,96906410,22108665,90061527,16595436,73124510,74743862,73109997,92398073,62051033,90272749,60176618,29932657,7502255,57961282,85695762,32699744,14045383,37739481,19891772,21001913,22997281,59329511,44627776,91240048,75052463,53648154,75229462,38009615,51426311,79942022,68316156,89217461,27041967,40781854,51149804,95251277,37192445,96420429,19486173,33935899,2891150,26744917,26734892,7646095,31161687,37166608,66667729,20568363,36808486,59910624,98739783,56531125,97641116,73168565,98653983,32151165,21397057,85242963,89804152,51715482,47298834,89046466,17976208,72238278,1239555,80555751,37891451,69786756,73235980,21919959,18663507,56473732,82897371,89078848,56461322,9860195,99971982,4985896,83948335,17386996,48774913,58208470,57241521,54232247,46540998,85117092,44915235,70782102,34044787,2331773,2917920,87598888,48395186,71522968,93359396,84002370,40027975,45667668,41092172,30366150,6038457,7066775,99604946,80316608,66319530,61960400,99297164,93566986,78549759,57020481,29516592,60106217,7517032,84840549,8733068,75820087,176257,874791,26664538,14626618,96318094,33061250,26275734,82886381,61897582,12024238,45617087,32426752,86821229,36812683,71316369,20122224,45790169,69355476,61728685,30811010,75153252,16097038,3088684,49882705,48658605,40534591,65338021,17385531,54606384,69848388,20645197,8729146,43506672,10649306,59109400,54868730,1569515,3773993,34432810,14363867,36139942,44846932,1936762,91281584,51588015,55770687,61271144,34667286,77312810,90654836,5832946,93790285,23134715,84406788,34236719,91141395,17365188,16387575,22450468,77301523,15948937,77898274,18699206,21289531,40356877,26863229,76315420,36685023,43152977,45794415,75777973,81805959,24314885,10597197,84293052,46870723,81774825,65074535,11365791,32058615,31733363,84187166,20836893,16567550,66045587,76703609,43357947,68694897,12571310,17016156,5482538,4787945,64602895,22113133,16405341,45049308,60393039,92998591,14723410,73021291,54663246,94711842,79806380,23740167,94911072,28787861,99021067,54263880,91990218,16684829,67513640,47893286,824872,27185644,22879907,89499542,91938191,61741594,52204879,41442762,11581181,81855445,59405277,79191827,90158785,71965942,16424199,26397786,74452589,85023028,1375023,54058788,37224844,97379790,6819644,61712234,25842078,62357986,79738755,27187213,94360702,37560164 +93933709,70004753,10366309,5073754,68041839,99658235,64602895,88047921,92787493,19891772,9623492,71083565,63059024,72373496,38022834,42692881,61271144,36685023,55247455,69848388,89811711,20140249,16424199,75986488,36312813,67227442,47738185,57163802,15148031,5111370,62430984,10309525,89214616,48360198,30787683,78766358,54014062,72274002,32590267,29188588,42967683,33237508,749283,20122224,83789391,56153345,34946859,31904591,33201905,87720882,75153252,37739481,96193415,72777973,168541,59371804,37501808,37620363,81677380,29466406,90272749,68644627,74110882,4668450,40197395,97783876,34493392,89419466,34295794,63152504,77187825,23569917,95010552,48260151,85571389,77787724,94516935,27665211,36780454,81853704,15535065,38061439,87598888,17068582,44889423,50188404,64848072,44784505,70800879,30764367,45428665,36753250,77272628,54762643,6793819,4099191,53648154,73222868,43045786,63506281,30395570,66250369,73031054,77312810,88904910,60102965,2204165,95395112,71657078,16099750,55753905,30139692,79657802,28851716,65715134,7517032,54427233,22942635,92398073,92998591,22108665,526217,77284619,17386996,24058273,78602717,3088684,42782652,22879907,14326617,65074535,35092039,66137019,98371444,25842078,91281584,30366150,15075176,18466635,95290172,99861373,32151165,83651211,33797252,59109400,9575954,93359396,34698463,62452034,55470718,70036438,95957797,48892201,71920426,96726697,99125126,2917920,82886381,43501211,49328608,321665,20642888,72725103,19101477,27625735,63015256,40865610,53547802,79821897,16445503,99524975,20486294,61859581,44473167,7955293,75543508,55770687,13862149,49641577,80251430,63967300,43357947,4095116,83269727,62496012,88653118,48675329,51588015,98739783,83210802,13468268,47792865,19898053,24733232,16567550,93790285,93515664,71522968,17764950,54663246,49236559,83150534,74357852,25636669,33304202,48658605,7646095,66667729,67084351,10358899,27187213,6525948,66832478,37183543,5482538,44245960,57658654,66611759,86001008,45617087,26493119,74441448,8129978,24591705,9603598,36505482,89637706,60581278,14093520,4798568,44177011,76170907,29834512,824872,44348328,57241521,12664567,82427263,72732205,72358170,93562257,61728685,73235980,32161669,42307449,48893685,79136082,24314885,97940276,4508700,54199160,6808825,80014588,45075471,17081350,17016156,508198,5832946,77072625,40677414,43933006,60176618,32426752,8696647,47298834,13231279,8791066,3509435,6505939,32058615,86022504,669105,19376156,72614359,16405341,20885148,95581843,40027975,10453030,59177718,59197747,98130363,44479073,10597197,38008118,13173644,47361209,92803766,14626618,74743862,30090481,28796059,96852321,66903004,38658347,17957593,50668599,4119747,3233084,21533347,53888755,39373729,27041967,43322743,96735716,44983451,16595436,3487592,61960400,8128637,3294781,6871053,90654836,97057187,43376279,69605283,86821229,63628376,30543215,19939935,9188443,61373987,6038457,45407418,8807940,29819940,10961421,61897582,28928175,62936963,70367851,56484900,41481685,16684829,18833224,79942022,8733068,24915585,84684495,91990218,68875490,82726008,61815223,56955985,32159704,9398733,67124282,7687278,44481640,77300457,18663507,54263880,45667668,68824981,6153406,47887470,31126490,1250437,29994197,18001617,37659250,58208470,68702632,11923835,56515456,37435892,53802686,64087743,26998766,79322415,23194618,49271185,74724075,88698958,26114953,76315420,415901,6521313,3633375,21993752,29932657,68816413,66319530,70372191,4064751,16387575,75777973,66663942,35456853,16380211,92604458,22721500,50702367,54606384,31161687,874791,7300047,11161731,27185644,90310261,45237957,74452589,68204242,68110330,12416768,30463802,16019925,80555751,33565483,60955663,96709982,83368048,61982238,59910624,3880712,65454636,78561158,84904436,99917599,75135448,86543538,69355476,57020481,69136837,82651278,51590803,27411561,41092172,13819358,30771409,1431742,49598724,6986898,78300864,54058788,72019362,62051033,92554120,69579137,62693428,7502255,58224549,39201414,87710366,50806615,73021291,8099994,17727650,73168565,14349098,56531125,55143724,3235882,70596786,91831487,30811010,38009615,36812683,78589145,2208785,78909561,81898046,39801351,89078848,33699435,30218878,67281495,11757872,65271999,69641513,1569515,99965001,26776922,71333116,61928316,33431960,83533741,67474219,35192533,26292919,18699206,1375023,74137926,39986008,23432750,67513640,51472275,94911072,64157906,14723410,91255408,88251446,9599614,66322959,88444207,45790169,14947650,30653863,67031644,71316369,7919588,77620120,52613508,99729357,18131876,6221471,54987042,51047803,24168411,66271566,72357096,65038678,7229550,39171895,6111563,7685448,76960303,92692978,33553959,86118021,30569392,2717150,9829782,93057697,39553046,247198,60393039,90061527,12348118,73617245,65338021,9259676,22200378,40781854,37560164,16097038,20002147,37166608,1239555,8651647,87160386,66428795,90090964,22435353,67030811,91957544,92353856,51464002,78884452,1197320,29401781,68939068,97011160,20765474,91802888,43506672,52261574,38365584,72793444,20568363,57248122,82178706,30163921,22129328,50007421,18806856,46540998,36396314,94090109,9058407,65851721,65047700,11581181,73786944,31491938,77301523,34497327,99021067,89996536,74614639,36139942,48088883,31733363,9257405,84002370,99297164,4069912,17365188,84187166,72238278,9886593,61741594,94076128,82897371,84293052,40534591,84840549,95251277,44915235,77694700,51315460,24826575,26744917,21397057,66885828,86242799,44664587,88130087,99333375,92033260,70782102,57393458,76434144,41092102,45848907,569864,68102437,55189057,11365791,57803235,84127901,59318837,44060493,62232211,47708850,34236719,30998561,85116755,80316608,77413012,97641116,22113133,17037369,17857111,9175338,99549373,59614746,83302115,70541760,51507425,61263068,34698428,26275734,79880247,11543098,62740044,60430369,5267545,67793644,35996293,34432810,7011964,53842979,36933038,82979980,78785507,20681921,86798033,40152546,16054533,33061250,33628349,18504197,66045587,9860968,89217461,27325428,44844121,52060076,50567636,8913721,45306070,40781449,94539824,20836893,98948034,14220886,38494874,82052050,19486173,41442762,91141395,91240048,34667286,79738755,99604946,73124510,78549759,85242963,40356877,14045383,81855445,85117092,45995325,42237907,26734892,45919976,36468541,56461322,29065964,48673079,79191827,22450468,72738685,56473732,91938191,95726235,15163258,10649306,86460488,29516592,4985896,38645117,82779622,22997281,43152977,47893286,20645197,59501487,62357986,2607799,90004325,37192445,47213183,36930650,29635537,84406788,62803858,26102057,28734791,97281847,81805959,12571310,52204879,92215320,38510840,30891921,29510992,8055981,57961282,59981773,45794415,82327024,81774825,1022677,93270084,76703609,92071990,99226875,83155442,46870723,75820087,96420429,44627776,51715482,5970607,37891451,38341669,37957788,90457870,61712234,98462867,93053405,76330843,13348726,3183975,7423788,75128745,23848565,53084256,15536795,67451935,61141874,82339363,59329511,16971929,81274566,29029316,22405120,97561745,98653983,76671482,24953498,30694952,3233569,37675718,36135,65275241,33935899,55850790,48774913,77377183,70420215,65880522,68128438,71300104,26863229,55602660,6497830,97379790,2891150,73392814,69786756,89499542,68000591,22766820,84349107,19272365,37224844,59405277,61859143,41245325,71469330,58751351,62762857,89804152,23110625,17058722,14731700,54868730,54232247,29959549,26139110,75052463,6819644,32250045,18783702,1788101,8634541,38256849,42644903,1204161,4091162,17146629,89699445,44847298,35512853,4787945,85695762,36808486,94711842,3773993,34044787,15015906,8373289,54517921,69697787,40686254,78218845,56424103,32699744,86301513,83133790,5822202,96906410,55669657,31727379,58180653,14363867,26392416,98648327,57359924,85023028,70727211,65017137,7182642,21289531,1936762,64098930,33123618,96318094,53632373,66116458,92692283,68694897,26507214,19457589,51426311,71965942,23134715,92867155,72495719,99224392,84166196,90013093,9860195,8115266,42947632,53393358,13470059,53666583,49882705,15902805,93566986,33336148,26397786,81172706,94360702,7104732,45381876,21001913,90158785,99690194,7066775,47090124,18411915,21673260,79806380,5640302,62552524,26063929,33249630,15948937,63372756,20867149,17385531,46851987,21919959,83948335,91727510,92541302,69255765,45049308,21070110,17894977,45996863,79922758,2331773,62115552,91664334,75407347,15064655,4199704,39847321,40984766,76540481,45990383,36580610,99515901,36189527,62490109,37280276,28550822,60106217,4806458,98943869,44842615,89046466,28787861,83747892,42199455,42073124,49597667,80246713,62428472,12303248,41380093,76825057,51149804,52293995,4515343,55960386,33895336,8729146,36845587,82532312,48395186,17976208,61623915,44550764,23740167,17539625,92530431,72278539,73109997,24619760,112651,55615885,4204661,58749,68316156,94595668,57240218,83083131,99971982,4978543,85711894,26664538,64055960,51466049,176257,75229462,10264691,65081429,77898274,12024238,4434662,44846932,32274392,6157724 +96193415,58749,37891451,98371444,18699206,64087743,95957797,33249630,50188404,51047803,30218878,5970607,10358899,97641116,81274566,35996293,49328608,47090124,44889423,83651211,68204242,80316608,20002147,31904591,44245960,26998766,62496012,50702367,40197395,29834512,15064655,74743862,66137019,31161687,88047921,46851987,66611759,77413012,24733232,4515343,7502255,61623915,44550764,1431742,74357852,14093520,247198,45919976,48673079,78589145,67281495,61859143,82327024,42199455,28734791,70036438,86022504,48260151,99524975,27325428,86301513,35092039,70727211,57020481,6505939,33553959,30463802,59177718,43376279,82726008,8696647,62452034,40356877,45996863,77620120,19101477,38494874,60430369,15535065,56424103,73392814,84840549,71333116,79191827,83210802,39847321,29188588,76540481,89214616,4787945,86543538,65454636,55143724,84684495,53648154,8634541,15015906,26392416,38061439,50668599,55753905,44473167,61141874,18783702,79657802,73168565,4434662,57240218,83150534,81677380,57359924,56531125,55669657,27665211,34698428,1788101,30569392,79821897,53842979,55470718,77312810,12348118,22108665,17727650,4668450,22435353,78549759,69355476,81898046,28928175,40152546,62803858,72777973,45407418,43501211,4099191,40686254,42782652,92803766,94595668,33628349,26507214,44177011,56473732,41092172,76330843,87160386,8913721,13231279,71300104,96726697,32699744,32250045,22879907,76960303,83302115,9188443,89078848,76434144,65715134,92398073,29994197,19486173,3880712,35192533,46870723,98739783,61897582,51464002,75153252,45990383,75543508,54987042,16099750,70800879,39801351,34295794,24619760,62740044,8651647,13173644,65047700,51426311,53666583,70596786,61960400,99333375,5111370,20486294,20885148,76671482,98948034,47361209,28550822,874791,36753250,77377183,93566986,50806615,81805959,99965001,39171895,59614746,43357947,63506281,82779622,26063929,42967683,19272365,75407347,9623492,4119747,77072625,48360198,92604458,66116458,36580610,8373289,99604946,3633375,65271999,89811711,55247455,85711894,37957788,42692881,57393458,88653118,86118021,37659250,62051033,94090109,96852321,45306070,77284619,6808825,50007421,25636669,65038678,24915585,62490109,18001617,29819940,79136082,51590803,57241521,70004753,36312813,43506672,86821229,59910624,92530431,90457870,66319530,21993752,71920426,81855445,56515456,62936963,63967300,97783876,36139942,20836893,112651,91727510,60581278,26664538,26139110,6793819,99861373,89046466,73617245,23194618,7104732,53888755,10961421,37183543,58751351,68644627,21533347,99125126,57961282,98648327,8791066,51588015,40677414,33304202,53084256,52261574,6157724,36930650,9603598,43933006,83789391,52293995,99549373,68939068,36780454,26102057,48892201,74110882,30395570,2917920,93515664,10264691,7011964,22200378,53632373,3294781,96906410,83083131,64848072,14349098,2717150,15948937,29959549,44983451,69605283,47708850,57163802,54663246,18411915,55189057,20642888,12024238,89996536,69579137,32159704,67227442,21070110,80251430,91802888,669105,92215320,49641577,32161669,415901,18806856,55602660,66903004,97561745,82886381,3233084,84293052,56484900,68816413,48774913,30764367,44481640,24591705,62693428,51472275,91831487,33237508,9860195,17016156,92692283,24058273,1569515,99515901,61263068,6153406,77694700,43322743,42644903,39373729,69255765,17764950,4064751,47792865,12571310,41481685,89804152,17058722,37620363,38341669,44842615,61373987,6221471,55960386,41245325,33201905,21001913,63152504,73031054,93057697,45848907,58180653,22942635,84349107,85116755,91957544,11365791,11161731,17539625,95395112,1204161,83269727,39201414,44060493,68102437,91664334,72274002,13348726,75229462,30163921,9886593,77300457,57658654,8099994,31491938,47213183,89499542,94711842,51507425,54014062,36808486,28851716,65081429,84166196,6871053,68824981,68316156,27041967,26776922,14220886,83368048,99658235,68000591,51315460,16380211,44846932,80555751,95290172,35456853,78218845,7685448,9599614,80246713,33123618,16019925,45428665,74137926,20568363,14045383,33565483,79922758,89637706,32274392,83133790,69641513,81172706,21289531,66045587,64098930,91240048,89419466,70372191,99690194,36812683,9860968,60102965,67793644,70541760,99021067,508198,64055960,34497327,68041839,17146629,8129978,45617087,69136837,50567636,15536795,84187166,72614359,9829782,15148031,74724075,83533741,8115266,81853704,7687278,90158785,38008118,26734892,70782102,83155442,59981773,63372756,98462867,4806458,56461322,42947632,57803235,19939935,47738185,30694952,49597667,67084351,60393039,54868730,88698958,16971929,79880247,29029316,66428795,53547802,37166608,62428472,36468541,24826575,4069912,14731700,29466406,92071990,824872,66271566,7919588,37192445,3773993,98130363,41092102,82178706,19898053,66885828,82052050,69848388,26493119,17081350,73124510,88444207,96709982,31126490,16054533,13468268,40781449,74614639,9257405,36685023,92692978,1022677,4204661,11581181,22450468,55850790,52613508,29510992,38645117,75986488,75777973,68128438,93790285,48675329,91281584,77787724,61741594,45790169,27187213,18131876,71083565,23569917,90272749,72725103,16387575,74441448,56153345,23848565,94911072,79806380,36845587,66667729,13470059,8807940,7300047,69786756,77272628,72495719,19891772,59318837,4091162,14947650,99297164,7646095,28796059,95581843,34493392,93270084,29401781,23110625,61982238,5832946,36505482,23134715,93053405,34236719,99224392,40781854,96735716,71316369,45995325,71657078,29065964,92554120,94516935,59197747,66250369,60106217,85571389,16405341,37675718,70420215,39986008,22405120,59501487,6986898,11757872,82339363,94360702,54517921,176257,93359396,85242963,22766820,92033260,91255408,9175338,38365584,14626618,61712234,41442762,33895336,66322959,90654836,67031644,31733363,79738755,21673260,72019362,46540998,168541,49271185,20140249,36135,63059024,59109400,33699435,22997281,92787493,79942022,93933709,77898274,53802686,569864,41380093,37224844,67030811,33797252,87720882,60955663,72373496,45237957,59371804,7066775,526217,55770687,15075176,62232211,26275734,20681921,749283,91938191,70367851,44627776,90004325,32590267,76703609,37280276,36189527,30139692,72793444,17857111,89217461,93562257,67451935,10366309,98653983,63628376,53393358,62552524,20765474,73222868,7423788,55615885,19457589,38009615,49598724,10597197,44844121,75135448,37560164,80014588,94076128,98943869,23432750,35512853,16567550,20122224,6525948,40534591,62430984,86798033,96318094,49236559,6819644,30771409,15902805,52204879,17037369,16595436,88251446,66832478,12303248,61728685,67124282,54606384,18663507,5822202,77301523,82979980,84406788,37739481,29932657,12416768,82651278,65275241,17068582,81774825,84904436,90310261,99917599,26114953,90090964,34698463,30891921,85117092,30998561,51466049,72357096,44479073,68702632,18504197,9398733,44348328,3183975,36396314,78300864,6038457,76170907,97281847,2204165,16097038,77187825,4985896,8128637,75052463,95726235,9575954,62357986,48395186,43045786,11543098,321665,52060076,54232247,61271144,61815223,18466635,22129328,65017137,72278539,71965942,91141395,7229550,42307449,6497830,65851721,22721500,2331773,78766358,37435892,5267545,72238278,30366150,75820087,99729357,90061527,72732205,38256849,10309525,2891150,71522968,14723410,54199160,54762643,44784505,61859581,57248122,30543215,20867149,6111563,4508700,5073754,92867155,34667286,2607799,34432810,86001008,92998591,68694897,73786944,59405277,48893685,4095116,31727379,21919959,3235882,83948335,15163258,78909561,42073124,17385531,99226875,61928316,56955985,38022834,37501808,26863229,76315420,27625735,10649306,84002370,12664567,48658605,85023028,82427263,19376156,65074535,48088883,64157906,87710366,45667668,6521313,17957593,47298834,2208785,84127901,90013093,73235980,30653863,1197320,4798568,76825057,26292919,9259676,32151165,78561158,43152977,33431960,8733068,92353856,95010552,58208470,71469330,27185644,99971982,62762857,45794415,3509435,13862149,8055981,40027975,40865610,54058788,96420429,3233569,72358170,24168411,34044787,13819358,42237907,86460488,16445503,51715482,17894977,38658347,1239555,3487592,7955293,23740167,97940276,65880522,67513640,28787861,33935899,39553046,3088684,65338021,30787683,5482538,30811010,78602717,60176618,94539824,11923835,24953498,40984766,66663942,82532312,68110330,69697787,10453030,26744917,44847298,75128745,14363867,33336148,7182642,47893286,44664587,33061250,14326617,95251277,4199704,29516592,1250437,59329511,72738685,83747892,25842078,78785507,86242799,64602895,67474219,5640302,30090481,27411561,51149804,1375023,73021291,82897371,54427233,45075471,32058615,92541302,79322415,74452589,78884452,97011160,1936762,17365188,62115552,34946859,9058407,7517032,68875490,87598888,73109997,36933038,29635537,17386996,91990218,49882705,16424199,47887470,32426752,17976208,58224549,22113133,24314885,16684829,89699445,38510840,88130087,63015256,8729146,20645197,97379790,26397786,88904910,21397057,85695762,54263880,45049308,97057187,4978543,45381876,44915235,18833224 +10358899,99021067,33797252,15535065,54014062,62490109,92803766,30463802,26292919,24058273,19376156,95957797,62452034,84840549,38658347,31904591,89078848,37659250,40197395,55770687,33304202,96193415,93933709,44245960,18806856,48360198,62740044,72777973,34946859,63967300,92787493,62496012,5267545,9599614,27665211,41245325,66045587,43501211,61741594,2717150,78602717,66832478,89637706,46851987,14349098,48892201,26744917,14220886,36685023,28928175,11923835,23569917,96726697,45306070,3088684,17081350,49271185,94516935,50007421,14731700,17068582,22129328,27411561,38365584,70004753,99549373,29834512,67793644,40677414,33201905,72373496,4508700,44889423,3487592,60581278,8733068,72274002,77284619,55470718,9398733,44847298,51472275,569864,6153406,4064751,65271999,5111370,24953498,16097038,67281495,247198,64098930,20568363,78589145,88653118,5073754,80251430,32590267,65715134,92604458,22942635,12664567,20885148,99658235,69355476,36753250,80014588,72738685,8099994,58751351,68824981,36468541,35092039,36580610,31491938,29466406,47361209,61815223,28796059,77787724,1431742,59981773,38061439,92692978,77312810,61897582,80316608,17037369,33431960,51590803,52060076,42782652,33628349,78909561,73617245,20486294,82339363,34698463,22108665,1788101,71083565,40534591,38256849,19898053,8128637,59614746,61373987,28851716,20642888,17365188,40152546,77694700,58749,57020481,81853704,77272628,19939935,61712234,37620363,7687278,42967683,80555751,81898046,72732205,17016156,14045383,20002147,83150534,60430369,66250369,56473732,77301523,91957544,98739783,48893685,56515456,22435353,84293052,61960400,23194618,36312813,61982238,47792865,50702367,30366150,29994197,24591705,69786756,68816413,5970607,98462867,44473167,38022834,82178706,76330843,10366309,71300104,21001913,16387575,55850790,83789391,63059024,2917920,36808486,13862149,53084256,168541,34044787,89214616,73235980,70367851,16595436,88698958,22997281,321665,68000591,40027975,68694897,54199160,60106217,98371444,33237508,91802888,72357096,79657802,39801351,49598724,35456853,21070110,38645117,87720882,37183543,9886593,68644627,4099191,35192533,19891772,44844121,16424199,4434662,83651211,68102437,84904436,33699435,1250437,91727510,36812683,90310261,6819644,53648154,67030811,79136082,4668450,3294781,30569392,74110882,63152504,54427233,75128745,43322743,34497327,70420215,81677380,59501487,4095116,90090964,78785507,72725103,7919588,57240218,47090124,70596786,3880712,8129978,47213183,27325428,61859143,69136837,65338021,43506672,18783702,6808825,44983451,66611759,14626618,21397057,8651647,36505482,22766820,16099750,45790169,45237957,26392416,86301513,43376279,90272749,7955293,83083131,13468268,29932657,82327024,37891451,74614639,30395570,88047921,97011160,81172706,12303248,68316156,17539625,29819940,17386996,63628376,37675718,8807940,16445503,51047803,62232211,13470059,20765474,41481685,66903004,57359924,66428795,93566986,92398073,874791,98948034,51588015,34432810,58224549,82726008,55753905,81805959,99861373,19486173,55247455,76703609,70541760,56461322,32159704,55189057,99515901,76434144,61263068,92554120,11161731,26275734,30787683,56153345,24915585,43152977,63506281,77413012,85571389,30090481,14326617,88444207,85711894,48395186,9603598,41092172,14093520,49236559,59910624,45428665,30653863,26493119,44060493,6871053,8634541,86001008,17146629,98943869,44664587,40781449,53393358,112651,62430984,59109400,8729146,8115266,61271144,3233084,65454636,33565483,93053405,26998766,6038457,15015906,98648327,415901,26102057,43357947,91664334,29401781,65851721,23432750,42237907,5482538,79191827,1239555,8696647,39847321,17957593,75407347,36396314,83533741,6986898,30998561,7502255,62936963,84349107,34698428,71469330,78884452,67084351,94911072,1197320,50668599,36139942,72278539,97641116,7646095,76315420,4985896,7517032,99965001,96709982,50806615,56955985,86543538,79942022,72019362,59197747,73222868,8791066,9058407,99125126,15064655,84684495,49328608,59371804,11581181,14947650,28550822,98130363,79821897,23848565,62762857,68702632,16405341,93790285,87160386,8913721,97561745,20122224,86821229,73786944,55669657,83133790,73392814,10309525,62051033,78218845,77072625,70372191,15536795,75135448,32250045,7011964,91281584,17764950,48673079,59177718,43933006,47887470,45667668,42199455,526217,84166196,91255408,45919976,73031054,61623915,83210802,89804152,10453030,824872,24733232,86118021,94711842,96735716,90061527,34493392,79922758,46540998,44177011,39171895,54868730,93359396,70036438,50188404,86022504,39553046,21993752,25636669,72614359,39201414,2607799,6793819,56531125,57658654,70800879,65081429,48774913,7685448,75229462,36930650,19101477,11543098,58208470,44481640,76540481,57393458,4978543,20140249,9860195,44627776,7423788,44348328,4119747,40356877,97783876,60102965,24826575,10961421,16684829,30771409,16054533,38008118,67474219,89217461,73124510,74357852,50567636,30163921,85242963,29188588,26734892,53842979,68110330,33061250,26664538,68939068,3235882,92033260,89811711,29510992,69848388,508198,9829782,12348118,94595668,7229550,36135,68041839,71965942,33336148,45075471,93057697,69641513,55602660,35996293,15148031,13173644,749283,3233569,32426752,18001617,95581843,55615885,82532312,1375023,82779622,75153252,89419466,45990383,2204165,24168411,6111563,74743862,1569515,3509435,27625735,94090109,64087743,54762643,21919959,86460488,70727211,7066775,93515664,34295794,9860968,12416768,45407418,82052050,26139110,69579137,33249630,4798568,12571310,30694952,66137019,96852321,26063929,42692881,77300457,66885828,44842615,51464002,99524975,42644903,62803858,2208785,6157724,59318837,71657078,91990218,64157906,44479073,6525948,56484900,9575954,86242799,52261574,30218878,89996536,61859581,96420429,48088883,44784505,36189527,59329511,20836893,54606384,83155442,45794415,91938191,74441448,5832946,40984766,9623492,59405277,15902805,75543508,49641577,75986488,68875490,99971982,29959549,6497830,76671482,32151165,84406788,7182642,16380211,61728685,60955663,17727650,77377183,68204242,78300864,66667729,67227442,65074535,4515343,79806380,37739481,37192445,21533347,83302115,4204661,74452589,64848072,48658605,81855445,40686254,82886381,52204879,3773993,42073124,38510840,32161669,77898274,57803235,58180653,1204161,669105,93270084,44550764,95395112,97057187,57163802,19272365,86798033,96318094,62693428,19457589,69605283,51507425,30764367,92215320,43045786,2891150,53547802,42947632,18504197,48260151,83269727,17894977,99729357,41092102,66271566,14723410,92998591,47738185,79322415,30811010,29065964,66663942,67031644,54263880,4806458,40781854,75820087,34667286,79880247,10649306,92541302,31161687,51715482,18131876,69255765,62357986,99690194,57961282,23740167,85116755,22405120,26776922,84127901,4787945,41442762,47708850,91831487,6505939,21289531,3183975,94539824,37224844,84002370,89046466,65047700,32058615,24314885,48675329,75777973,63015256,90004325,53802686,36845587,40865610,92353856,31727379,6221471,22200378,33935899,84187166,24619760,73021291,78549759,91240048,36933038,64602895,34236719,62552524,97281847,22113133,9175338,87598888,72495719,99224392,36780454,38009615,82651278,74137926,28734791,47298834,37435892,82979980,83747892,95010552,37957788,27041967,77620120,18663507,99333375,53888755,97940276,71333116,73168565,91141395,38341669,30139692,45995325,45996863,45049308,55143724,13819358,10597197,65880522,85117092,83368048,37166608,71316369,65017137,4069912,18411915,30543215,89499542,76960303,16019925,39373729,54232247,31126490,32699744,20681921,81274566,18699206,78766358,65038678,15948937,13348726,67451935,99917599,49597667,9188443,9257405,29516592,26114953,46870723,22721500,99226875,12024238,31733363,56424103,11757872,82427263,39986008,72238278,2331773,72358170,20645197,4091162,63372756,85023028,37501808,15163258,88251446,92692283,61141874,32274392,7300047,45617087,29635537,37280276,81774825,3633375,38494874,64055960,15075176,96906410,54663246,52613508,16567550,33895336,29029316,26863229,1022677,9259676,94360702,66116458,57241521,90158785,17058722,28787861,16971929,93562257,95290172,44846932,26397786,18466635,90654836,87710366,7104732,33123618,11365791,60176618,53666583,30891921,69697787,99604946,95251277,90013093,27187213,54058788,66322959,66319530,55960386,67124282,41380093,51466049,98653983,51315460,89699445,8055981,70782102,80246713,27185644,76825057,176257,5822202,18833224,73109997,25842078,99297164,4199704,78561158,51149804,42307449,77187825,13231279,45848907,53632373,33553959,22450468,51426311,6521313,79738755,94076128,82897371,17976208,26507214,23134715,35512853,71522968,88904910,71920426,14363867,44915235,54987042,85695762,88130087,95726235,75052463,1936762,17385531,62428472,22879907,5640302,21673260,92071990,97379790,68128438,52293995,90457870,49882705,47893286,61928316,83948335,54517921,65275241,62115552,76170907,60393039,92867155,72793444,74724075,67513640,8373289,17857111,10264691,45381876,23110625,37560164,92530431,20867149,57248122 +44481640,36780454,16097038,83210802,88653118,77072625,66667729,16099750,61263068,53393358,69641513,36933038,247198,96193415,77312810,16445503,68000591,35456853,53547802,90457870,55189057,39801351,83651211,40027975,2917920,38645117,95726235,29834512,36312813,15535065,61141874,6986898,9257405,60106217,96735716,79942022,99549373,78766358,7919588,99515901,26114953,64087743,49641577,68875490,32590267,66250369,43376279,6808825,73124510,6505939,92554120,64602895,40865610,63967300,33237508,64848072,95957797,62051033,19376156,96726697,18783702,88444207,63059024,81805959,79136082,99524975,56484900,49236559,2204165,70596786,85116755,56531125,70004753,9575954,39373729,36505482,92692283,17764950,7955293,37192445,68110330,75543508,29932657,31126490,4668450,8129978,12348118,31904591,85117092,55470718,59501487,86022504,5111370,58751351,10309525,36808486,37620363,61271144,59109400,54762643,32159704,38365584,5267545,73786944,9259676,18504197,22997281,63015256,45237957,48892201,16054533,90013093,33895336,51047803,18699206,54263880,82339363,40152546,44664587,76434144,61373987,13231279,76960303,67451935,87710366,33699435,52261574,79657802,54427233,5482538,9603598,26507214,66428795,59177718,7300047,55669657,8696647,39847321,59371804,98943869,22942635,61859143,53632373,55753905,17857111,94539824,99729357,86001008,70036438,82897371,62430984,5640302,30366150,22766820,74743862,47738185,61960400,65038678,81898046,16567550,45794415,34044787,45407418,29959549,94360702,48088883,14947650,84127901,78785507,89499542,72357096,50668599,96906410,9886593,43322743,30771409,54014062,67793644,88904910,32250045,98648327,51464002,92033260,61859581,8807940,72238278,68316156,68644627,8651647,1431742,16387575,49328608,55770687,81274566,66271566,93933709,4806458,95290172,90310261,76703609,44842615,60393039,3509435,81677380,27665211,88698958,57803235,44784505,10358899,33249630,62496012,73392814,53888755,30764367,31727379,8128637,84187166,70367851,83150534,78602717,35192533,94911072,62452034,4798568,4099191,37501808,18131876,74110882,62552524,44983451,21533347,73031054,51149804,24058273,27411561,38061439,65338021,40984766,22113133,41092102,93270084,77300457,28928175,34493392,26493119,68702632,77187825,61928316,20002147,94090109,84349107,47887470,97940276,30891921,41442762,10597197,77377183,19939935,88130087,92541302,7011964,53084256,32161669,93057697,62693428,17068582,3880712,95010552,42947632,18663507,26392416,71333116,19457589,39986008,21070110,74357852,6157724,72495719,65275241,66045587,874791,24915585,4064751,72738685,22108665,91664334,66832478,66903004,40686254,21919959,22435353,51426311,52060076,61623915,72278539,36753250,22405120,71657078,72373496,53666583,78300864,84684495,29466406,51507425,45919976,68204242,43045786,26292919,65081429,94076128,44889423,415901,20486294,29994197,39171895,14220886,33797252,19891772,50702367,52204879,98130363,97011160,30543215,33201905,91802888,14723410,168541,62936963,321665,38008118,65715134,78218845,83789391,64055960,78589145,17957593,65851721,95581843,44348328,6221471,71469330,60430369,54517921,62803858,48893685,37166608,92867155,76315420,31491938,2331773,11161731,38022834,23848565,41245325,22450468,69355476,19101477,80014588,62740044,69697787,65017137,97561745,526217,16019925,13173644,749283,26139110,36189527,75153252,74452589,22129328,9188443,42644903,82726008,18833224,17727650,68816413,14093520,37659250,35092039,26998766,20765474,25636669,79821897,2607799,72777973,29065964,19272365,81774825,72274002,66611759,37224844,97379790,17539625,34295794,17365188,45990383,75986488,69848388,69255765,37280276,99297164,72793444,90004325,47090124,38494874,71522968,44245960,84293052,17016156,30787683,83533741,99658235,80316608,82052050,40356877,62490109,90090964,41481685,97783876,64157906,93053405,71300104,18001617,64098930,99971982,12416768,23432750,85695762,17386996,77284619,29819940,57658654,89217461,83302115,91255408,69136837,89214616,15075176,86118021,87160386,60955663,10366309,73235980,11543098,32426752,19898053,60102965,40197395,3633375,50806615,5832946,11581181,49598724,37739481,99226875,56424103,112651,29029316,669105,86460488,49882705,95395112,30218878,3088684,43506672,89078848,14045383,19486173,16380211,9860195,47213183,1197320,29401781,82327024,23569917,17976208,1250437,48260151,26397786,41380093,33628349,51315460,75135448,89419466,85242963,98462867,54199160,68694897,79191827,20885148,40781449,20568363,57359924,54058788,37675718,79880247,70541760,37957788,9599614,71965942,77301523,13348726,10453030,67227442,49597667,1239555,92787493,63372756,56515456,34698428,52293995,84904436,45996863,36139942,37183543,59329511,94516935,13468268,14626618,77787724,77272628,4069912,45667668,7182642,65271999,67030811,81853704,14326617,44844121,80555751,63152504,30163921,4787945,44473167,28796059,68041839,81172706,99917599,76671482,91957544,21397057,75407347,27625735,43933006,91990218,56461322,42307449,78561158,30998561,77898274,44060493,18411915,54987042,57961282,55960386,94711842,98948034,44177011,70800879,36930650,76330843,20645197,61741594,89637706,99604946,2891150,53802686,87598888,51472275,29188588,49271185,7517032,66322959,71920426,57248122,47893286,34236719,5970607,73222868,92998591,66137019,76825057,46851987,7104732,75128745,33431960,99965001,88047921,91240048,92071990,82886381,8373289,3233084,9175338,45075471,72019362,7685448,20836893,91281584,63628376,8099994,5073754,13862149,85023028,99861373,26863229,34667286,26275734,48673079,38510840,43501211,16971929,65880522,5822202,91141395,2208785,68824981,20140249,90272749,569864,21673260,48675329,40534591,54663246,93566986,6793819,59910624,77620120,4095116,55247455,14349098,58208470,54606384,8115266,56955985,32151165,58749,55615885,84840549,91727510,82178706,75229462,67281495,50007421,26664538,17081350,42692881,61815223,31733363,79922758,37435892,88251446,76540481,57393458,46540998,28787861,45995325,15015906,98739783,80246713,59197747,66663942,52613508,48774913,54232247,86821229,8729146,7502255,72725103,10961421,12303248,14731700,62762857,26776922,58180653,44915235,86242799,98371444,61982238,2717150,87720882,57241521,21993752,11757872,92604458,15902805,96420429,51715482,99125126,1788101,7229550,35512853,83368048,60581278,96709982,62428472,48360198,30090481,62232211,59614746,31161687,24314885,61897582,176257,48658605,1936762,24591705,6525948,42967683,46870723,26734892,99224392,77413012,65074535,36812683,71083565,50188404,53648154,33935899,44847298,80251430,3294781,53842979,75777973,73168565,6153406,33336148,20867149,79806380,30139692,14363867,28851716,28550822,16405341,25842078,33304202,44479073,9623492,29635537,95251277,34497327,28734791,8634541,71316369,55602660,18806856,7687278,74441448,29510992,38009615,27187213,94595668,17058722,16595436,56153345,93790285,34946859,78909561,60176618,56473732,18466635,70727211,15536795,3773993,1375023,51466049,1204161,89046466,73109997,34432810,82979980,17385531,39553046,3233569,20122224,43152977,4091162,72614359,24826575,4204661,38341669,4434662,22721500,7646095,37560164,47298834,33061250,33123618,17894977,36845587,26063929,45848907,74614639,44550764,4119747,84166196,45381876,36685023,92215320,50567636,67474219,97641116,66319530,12664567,22200378,67031644,16424199,69579137,27041967,6521313,61712234,35996293,10649306,92530431,92803766,93562257,32058615,45790169,75052463,27325428,4515343,55850790,8733068,45617087,34698463,92398073,15148031,82779622,83269727,70372191,30694952,37891451,10264691,72732205,82651278,83133790,57240218,17037369,20642888,74137926,78549759,24168411,6497830,6819644,89811711,30653863,41092172,3183975,99333375,84002370,47361209,96852321,70420215,47792865,93359396,92692978,508198,6038457,77694700,23740167,12571310,68939068,68102437,51590803,99690194,1569515,7066775,59318837,42782652,45428665,15163258,13470059,38256849,24953498,44846932,73617245,42073124,16684829,36580610,69605283,85571389,3487592,90061527,81855445,93515664,36135,66885828,38658347,82427263,27185644,23110625,11365791,83948335,62115552,824872,51588015,82532312,86301513,26744917,47708850,9829782,59405277,39201414,67124282,67513640,92353856,20681921,23134715,79738755,6871053,86543538,83155442,30463802,9058407,30811010,57020481,99021067,21001913,98653983,89996536,83083131,76170907,24733232,30395570,55143724,30569392,74724075,63506281,48395186,9398733,33565483,45049308,69786756,8791066,65047700,86798033,12024238,66116458,91938191,4508700,36468541,89804152,57163802,8913721,40781854,96318094,15064655,43357947,97281847,26102057,75820087,13819358,90654836,22879907,15948937,33553959,61728685,70782102,4985896,11923835,7423788,29516592,97057187,83747892,23194618,40677414,32699744,36396314,4978543,54868730,91831487,24619760,58224549,85711894,73021291,68128438,90158785,45306070,72358170,32274392,62357986,1022677,84406788,42199455,65454636,89699445,59981773,79322415,78884452,9860968,44627776,21289531,17146629,3235882,42237907,6111563,8055981,67084351,4199704 +88653118,66832478,1431742,24058273,669105,19101477,7919588,54199160,50668599,36753250,76960303,4099191,88047921,30139692,75986488,52613508,84349107,77620120,74137926,29029316,77284619,36580610,92787493,29834512,66667729,27665211,73222868,6986898,19939935,31904591,7517032,72614359,112651,51047803,80316608,67793644,43322743,20568363,40865610,5111370,60430369,65275241,60102965,95581843,29994197,74357852,8129978,67281495,16971929,37620363,48260151,17081350,48673079,39171895,5832946,38061439,19891772,62496012,17727650,27625735,15948937,24619760,91957544,16099750,99658235,40534591,36808486,40027975,72777973,29466406,30163921,34493392,11365791,90457870,37192445,27187213,84904436,53666583,53547802,5482538,81677380,38341669,71333116,59318837,44889423,74441448,76540481,64087743,90310261,40197395,66045587,321665,62803858,72238278,44983451,86001008,7011964,8128637,59329511,25636669,26392416,3294781,93566986,78218845,38645117,3509435,46870723,1788101,99224392,16595436,58751351,17894977,78884452,94711842,73168565,80246713,57020481,36812683,9623492,176257,66250369,20002147,77272628,52261574,71657078,38658347,16684829,21533347,75135448,97561745,34236719,37891451,66903004,55753905,50702367,63059024,39553046,63152504,5267545,45995325,9599614,37166608,57248122,95957797,84840549,8807940,12664567,26493119,57359924,99604946,71469330,72373496,65715134,22435353,38494874,15535065,44842615,84127901,21070110,45790169,26139110,98653983,4787945,33628349,63372756,83651211,43501211,8099994,29065964,16424199,23848565,29188588,18783702,89214616,39801351,22108665,3233084,34432810,48892201,20486294,92554120,4119747,66137019,68204242,34295794,93359396,95290172,97783876,41442762,34044787,18131876,76315420,40984766,34946859,59614746,87598888,77300457,61859143,62693428,73786944,55247455,37659250,54517921,32590267,76434144,32161669,13862149,59910624,98943869,40686254,70372191,51472275,64602895,82427263,18001617,83302115,86242799,45075471,37675718,8634541,77413012,58224549,9058407,78549759,89637706,70596786,43376279,247198,26114953,14220886,65017137,46540998,34497327,53842979,33304202,79657802,66611759,62430984,81853704,83789391,6153406,49328608,2208785,69136837,68644627,67227442,91727510,94090109,93933709,87160386,81898046,36505482,28550822,33237508,54987042,65880522,30395570,83210802,93270084,13468268,7687278,39201414,72274002,8696647,55770687,45237957,44481640,97281847,15148031,20885148,82897371,74743862,4069912,93515664,42692881,55470718,28734791,10961421,99861373,92692283,62452034,3633375,30787683,37739481,57240218,85571389,64848072,57961282,30543215,73124510,96709982,18699206,93053405,73617245,24953498,80014588,29819940,33201905,30811010,83083131,16097038,53648154,61141874,4199704,96726697,12348118,13470059,43506672,82726008,79191827,4204661,90090964,29932657,37435892,72738685,98130363,17385531,69255765,47708850,76170907,22879907,79922758,749283,92033260,70420215,94516935,22766820,2204165,55143724,68816413,3773993,45848907,45428665,22405120,26734892,508198,77694700,7502255,22942635,36780454,12571310,23134715,42199455,51464002,99021067,88251446,50806615,7685448,67451935,89419466,99549373,1204161,27185644,74110882,53888755,33431960,12416768,3487592,34698463,71300104,7229550,82052050,65338021,42782652,30463802,10366309,33797252,78602717,43933006,60581278,24733232,14731700,2917920,38008118,39847321,62740044,76703609,26664538,4064751,28928175,50188404,85116755,75153252,58749,32159704,33061250,16445503,6505939,55189057,72358170,33249630,98371444,30366150,48360198,86543538,54014062,30891921,17037369,85242963,31161687,3880712,61373987,44846932,47090124,54762643,61960400,65851721,86821229,70004753,79136082,72725103,40677414,26507214,22200378,45919976,17068582,60176618,37183543,97641116,49641577,41245325,77312810,66428795,82979980,62051033,4091162,19486173,92604458,91802888,72495719,874791,95395112,78766358,99524975,93057697,6525948,36933038,28796059,10597197,96735716,40356877,20140249,61897582,54868730,7646095,56424103,22721500,26063929,36312813,72357096,89811711,526217,35512853,91281584,20645197,61271144,59405277,59177718,69355476,47738185,57163802,6871053,36189527,35192533,9886593,28851716,1022677,56531125,1375023,68939068,93790285,83155442,52204879,71920426,2717150,34698428,7955293,7066775,24591705,18466635,96193415,88904910,3235882,10358899,6038457,60955663,62232211,77072625,86022504,99965001,18411915,22450468,44348328,16405341,36139942,4668450,11543098,4798568,17386996,37501808,18806856,6793819,42237907,49236559,27325428,9257405,92803766,47361209,54427233,26776922,73235980,4806458,30998561,46851987,55602660,30218878,38022834,62552524,99690194,415901,84684495,84166196,32426752,66271566,34667286,63506281,70541760,82779622,75820087,69697787,78909561,41481685,61741594,21993752,92541302,87710366,15075176,39986008,92692978,75543508,44915235,19457589,53084256,68824981,75229462,20642888,53632373,1197320,6221471,65271999,57393458,79806380,45306070,15536795,56461322,81855445,95010552,19898053,63628376,59981773,62762857,99917599,65038678,24915585,9860968,63967300,54663246,42967683,88444207,85117092,51507425,74724075,69786756,50007421,85711894,91938191,55669657,36685023,64098930,77187825,14363867,94911072,83150534,9175338,84293052,6808825,96906410,53802686,33699435,67513640,44847298,29401781,94076128,56955985,44479073,84187166,31126490,9860195,70036438,44473167,68000591,65074535,92215320,7423788,45407418,35092039,56515456,92398073,3088684,569864,17146629,19376156,9603598,80555751,35996293,8733068,8791066,51590803,89804152,8729146,75407347,66663942,14626618,21673260,49882705,83368048,16019925,57241521,66319530,96852321,76671482,20765474,61712234,4515343,8651647,55850790,86118021,44844121,23432750,25842078,54606384,60106217,87720882,8115266,44784505,96318094,21289531,23110625,55615885,79942022,68875490,2607799,70800879,6497830,81805959,14947650,14326617,69848388,75777973,68041839,36930650,39373729,85023028,4095116,29959549,59197747,84406788,57803235,44245960,61982238,91664334,71083565,92071990,43045786,7300047,81172706,1569515,42947632,92998591,16380211,42644903,44627776,94595668,168541,33565483,86301513,38009615,76825057,72278539,29635537,90272749,33553959,99297164,52060076,17016156,52293995,91831487,21919959,16387575,17764950,77787724,69641513,61928316,35456853,45617087,54058788,98739783,99515901,68102437,18663507,68694897,96420429,5970607,42307449,40781449,9398733,11757872,76330843,97940276,21397057,1250437,36468541,91240048,17539625,40781854,36135,45996863,78561158,45990383,74452589,20122224,56153345,23569917,67031644,99125126,5073754,70727211,9188443,59109400,33895336,48658605,98648327,71316369,95726235,824872,48088883,65454636,61859581,41380093,26744917,62115552,92867155,27041967,62490109,13819358,51466049,44177011,67084351,89078848,81274566,24168411,62357986,20867149,18504197,47213183,30569392,63015256,69605283,49271185,89699445,43357947,10309525,48774913,82327024,70367851,30653863,48675329,9575954,64157906,37957788,90013093,71522968,48893685,82532312,78589145,44664587,14093520,15064655,57658654,59501487,26292919,20836893,73392814,89499542,67474219,36396314,19272365,41092172,82178706,17857111,10649306,4985896,26998766,8913721,44060493,93562257,38365584,40152546,7104732,6111563,91990218,51715482,17957593,32274392,3233569,13231279,32250045,22997281,18833224,11923835,26275734,89996536,45381876,67030811,43152977,44550764,83948335,82886381,58208470,79821897,72019362,21001913,33935899,14349098,68702632,6157724,56484900,45667668,26863229,55960386,88698958,51588015,54263880,75052463,78785507,85695762,28787861,30764367,90654836,49598724,91255408,90004325,86798033,13173644,62936963,89046466,14723410,14045383,61728685,23194618,73109997,64055960,95251277,4508700,59371804,60393039,89217461,82651278,11161731,83133790,20681921,66885828,42073124,54232247,47792865,97379790,27411561,16054533,8055981,15015906,26397786,61263068,36845587,37280276,53393358,65047700,24826575,66116458,51426311,23740167,83269727,37224844,32699744,97057187,78300864,15902805,79738755,5822202,74614639,15163258,56473732,99226875,72793444,68128438,70782102,67124282,73031054,45049308,80251430,99729357,83747892,97011160,31727379,84002370,22113133,94539824,13348726,17058722,16567550,32058615,24314885,94360702,77377183,99333375,22129328,38510840,99971982,41092102,12024238,2331773,72732205,11581181,33123618,62428472,91141395,82339363,9259676,29510992,38256849,6521313,90061527,61815223,3183975,58180653,4434662,68110330,30090481,37560164,50567636,51315460,98948034,12303248,86460488,88130087,61623915,51149804,6819644,4978543,79880247,98462867,31733363,1239555,47893286,45794415,8373289,65081429,66322959,30771409,17365188,5640302,69579137,92353856,29516592,2891150,81774825,49597667,71965942,68316156,32151165,47887470,30694952,9829782,33336148,73021291,47298834,17976208,48395186,7182642,1936762,92530431,77898274,79322415,26102057,75128745,83533741,31491938,77301523,10453030,10264691,90158785 +89214616,77301523,92033260,43501211,21993752,66611759,72373496,19376156,55470718,8099994,5970607,3294781,61263068,50188404,1197320,17037369,9175338,96735716,91727510,669105,22129328,45237957,50567636,22766820,6808825,75128745,35092039,37183543,72777973,66667729,99515901,40152546,44177011,62430984,11161731,19898053,31904591,56515456,42692881,90272749,57240218,65715134,415901,70372191,52261574,62936963,56461322,24591705,63059024,61373987,44983451,88653118,36468541,72614359,54199160,48360198,67084351,63015256,49328608,51590803,6153406,24953498,76540481,24619760,6038457,36139942,24058273,78602717,72019362,56153345,73124510,8729146,9886593,10649306,96193415,34698428,76960303,247198,36753250,20486294,41481685,23134715,35192533,99917599,66903004,34698463,96318094,13862149,34497327,8055981,94516935,48395186,68816413,78589145,91664334,81853704,7011964,77300457,4668450,62740044,48892201,44889423,78766358,9575954,99965001,55753905,54606384,36930650,57241521,47361209,57393458,59371804,94911072,16445503,63967300,48675329,68875490,59177718,4119747,98130363,20645197,7300047,22997281,34493392,42967683,8634541,61859581,37435892,20885148,5267545,8696647,62552524,53084256,4069912,45306070,77787724,91957544,8733068,80555751,68644627,92554120,26292919,46870723,29932657,31161687,10366309,67281495,32250045,24826575,26139110,16595436,52613508,76434144,34295794,91802888,99524975,43045786,40865610,17068582,99125126,90310261,30366150,77312810,87598888,26114953,48673079,1250437,569864,61982238,68939068,15535065,62496012,6793819,17764950,40677414,83269727,92215320,76671482,67031644,51047803,78785507,45407418,11365791,55770687,40534591,26392416,68204242,82532312,874791,92692978,43506672,1431742,70367851,57658654,2917920,99604946,6505939,39847321,37620363,33431960,27665211,77072625,77620120,47213183,168541,20002147,49271185,5073754,68702632,33237508,66250369,93933709,1204161,14731700,79136082,38009615,57803235,9259676,59197747,82651278,76330843,95290172,6986898,1788101,81677380,1022677,53632373,83789391,28734791,47738185,17058722,26734892,7423788,4787945,38008118,27041967,3509435,66045587,92541302,58749,73617245,8128637,97641116,45428665,8651647,30395570,40027975,17386996,29959549,46851987,85242963,4508700,53802686,73392814,97783876,80251430,21001913,68824981,95726235,38658347,9188443,14093520,38022834,14045383,59614746,82327024,96420429,36312813,50806615,97057187,44473167,55602660,19939935,33201905,66137019,30891921,83651211,4515343,7229550,38256849,83155442,87710366,14626618,47887470,18833224,23848565,33249630,17365188,7955293,15902805,29188588,17539625,80316608,28550822,8807940,29516592,19457589,71083565,94076128,78561158,79806380,54427233,51507425,44348328,90457870,5832946,39553046,3487592,16380211,62803858,82427263,3233569,17146629,99021067,75820087,75543508,75986488,79922758,14220886,9623492,38061439,89811711,3633375,75153252,51315460,33565483,92787493,45667668,55615885,27325428,76703609,526217,47708850,2891150,53393358,53648154,65338021,51464002,28928175,30764367,74614639,7687278,13348726,7919588,31491938,62357986,28796059,31733363,55247455,71965942,60176618,36580610,54058788,79821897,79657802,66319530,63372756,1936762,49236559,72732205,14723410,18783702,84349107,95395112,7685448,77187825,47792865,85695762,96906410,82779622,22435353,50668599,32161669,79942022,10453030,64157906,81855445,15015906,29466406,76315420,89804152,31126490,67793644,32151165,62693428,44847298,52204879,22942635,84840549,10961421,36505482,27185644,30543215,21070110,26776922,55189057,65275241,82886381,94360702,32426752,98653983,72738685,91938191,88444207,29065964,36808486,80014588,71333116,94595668,71920426,60430369,78300864,15148031,37675718,22113133,2204165,99690194,42237907,12348118,98739783,17976208,24733232,19891772,68694897,69848388,73109997,20765474,74743862,98371444,99333375,40781854,29635537,16684829,86001008,79191827,57020481,62051033,55960386,98943869,85116755,30694952,57961282,71657078,72357096,54014062,37659250,65851721,68128438,39171895,58180653,26275734,67474219,13468268,83210802,66116458,3233084,54868730,41092172,83302115,508198,99729357,26102057,99861373,14349098,65454636,8129978,93359396,62232211,92071990,33304202,65271999,29994197,83150534,37957788,48893685,60106217,4434662,112651,3088684,66832478,43376279,48088883,73222868,61859143,16054533,43933006,70036438,50007421,32058615,70420215,6525948,78218845,20122224,30998561,66271566,77284619,2717150,83533741,39986008,59329511,70541760,93566986,82052050,94711842,63628376,9398733,1375023,18699206,39201414,321665,33336148,10358899,96726697,77694700,69579137,78884452,72278539,21673260,44842615,70004753,18411915,56531125,90004325,21397057,69355476,7104732,72725103,77272628,81774825,59501487,78909561,39801351,74441448,75229462,4064751,17957593,13470059,38365584,30787683,84684495,9829782,30139692,34236719,2208785,45919976,90654836,19101477,45848907,49641577,88251446,43357947,53547802,35456853,4806458,15075176,88047921,15064655,83747892,72793444,54663246,37280276,62452034,74724075,18806856,5822202,33935899,84904436,3183975,51466049,51472275,34432810,18131876,99658235,61141874,16567550,69136837,61623915,88130087,98948034,6819644,30771409,42782652,61728685,66663942,44481640,30163921,94090109,74357852,6871053,23194618,66322959,26744917,34946859,74137926,92398073,43152977,18466635,83948335,92530431,48774913,86543538,70596786,40197395,30811010,12664567,71316369,86460488,14363867,99226875,26063929,90013093,77898274,36685023,79880247,88904910,90090964,47090124,37501808,29819940,97281847,75135448,13819358,44784505,70800879,69697787,13173644,65074535,16424199,25636669,32159704,40356877,25842078,60102965,17894977,84187166,95957797,58224549,176257,42307449,72274002,73021291,60581278,21533347,34044787,89419466,36933038,16097038,68000591,85117092,99549373,22108665,89637706,5111370,9603598,36812683,59109400,38510840,92998591,84406788,82339363,40984766,68316156,23569917,91831487,81172706,76170907,34667286,67513640,30463802,8115266,89217461,4798568,29029316,35512853,29834512,49598724,11581181,36845587,67227442,78549759,83368048,93270084,54263880,57163802,16971929,20681921,61271144,40686254,18001617,20140249,88698958,51588015,18663507,67451935,46540998,59981773,14947650,26397786,18504197,22200378,55669657,56424103,50702367,53666583,54232247,91990218,92803766,73786944,6221471,55143724,7502255,95251277,89046466,36780454,85023028,30218878,45996863,92604458,62115552,36135,86301513,77413012,42644903,33123618,89699445,19272365,2331773,90061527,17385531,59318837,80246713,17016156,23740167,64848072,53888755,15948937,38494874,10309525,14326617,54517921,72238278,48260151,13231279,64087743,33699435,84127901,66885828,85711894,44550764,61928316,33797252,45794415,21919959,44627776,82178706,9860195,4204661,81805959,92692283,61897582,4199704,97011160,98462867,89078848,45790169,1239555,39373729,9599614,22450468,71522968,65017137,4099191,16405341,56484900,37739481,11923835,20568363,45995325,45075471,27625735,56473732,8373289,93562257,98648327,92353856,49597667,6157724,57248122,86242799,74110882,84293052,15536795,59910624,19486173,91240048,40781449,32590267,51715482,42947632,20642888,20836893,749283,30569392,42073124,53842979,69641513,70727211,12571310,30090481,94539824,93053405,16099750,58208470,69255765,38645117,3235882,26998766,43322743,96709982,33061250,72358170,77377183,33628349,75052463,12416768,86798033,75777973,62490109,72495719,52060076,68110330,1569515,85571389,4091162,41092102,66428795,32699744,3880712,12303248,73168565,97940276,824872,92867155,64098930,15163258,65081429,82979980,61712234,58751351,73031054,55850790,73235980,82897371,68102437,6111563,59405277,97379790,67124282,86118021,12024238,45990383,24168411,32274392,35996293,81898046,67030811,11543098,68041839,23432750,60393039,9058407,29401781,37192445,42199455,22405120,26507214,47298834,65038678,71300104,4978543,7517032,22721500,69786756,21289531,5482538,29510992,44060493,97561745,33895336,60955663,38341669,49882705,51149804,71469330,86821229,61741594,84166196,3773993,8913721,54987042,17857111,93790285,30653863,4985896,44915235,99224392,96852321,54762643,2607799,84002370,24915585,99971982,51426311,79322415,6497830,79738755,17081350,26664538,28787861,89996536,95010552,93057697,48658605,82726008,61960400,17727650,27187213,52293995,63506281,16387575,87720882,20867149,4095116,37166608,37224844,69605283,57359924,44664587,65047700,44846932,7646095,89499542,91141395,16019925,36189527,41245325,83133790,62762857,56955985,7066775,33553959,65880522,22879907,87160386,5640302,95581843,93515664,81274566,75407347,64602895,44844121,76825057,24314885,41380093,28851716,83083131,86022504,91255408,27411561,7182642,8791066,37891451,11757872,44245960,10597197,45381876,70782102,36396314,90158785,9860968,41442762,47893286,26863229,45617087,9257405,45049308,63152504,74452589,23110625,10264691,31727379,64055960,62428472,99297164,44479073,26493119,61815223,6521313,91281584,37560164 +98371444,55960386,99333375,91990218,53842979,89214616,59109400,41092172,33201905,7011964,35192533,98462867,72495719,30395570,415901,6871053,94516935,10358899,73124510,6793819,24058273,50188404,15535065,72738685,42782652,77413012,51047803,77694700,10366309,66663942,60430369,74724075,57658654,94076128,23848565,22129328,15536795,15015906,47792865,78589145,44550764,23569917,29065964,6153406,29029316,35092039,68824981,77187825,30694952,84684495,56531125,73168565,48893685,95581843,61960400,62740044,32250045,68644627,55753905,26392416,63015256,47298834,92033260,17146629,43152977,92803766,51472275,18131876,508198,84002370,14326617,79136082,17068582,65074535,16097038,35996293,9886593,29994197,9188443,54868730,43376279,39847321,9603598,50567636,11581181,73392814,70367851,21993752,53648154,99297164,84293052,96193415,84840549,40152546,87710366,14220886,40984766,62693428,39801351,53084256,93933709,34698428,92692978,4668450,26063929,58749,98943869,45790169,17365188,67793644,13348726,16054533,97641116,35456853,38658347,63628376,20765474,26734892,86460488,96420429,48892201,5111370,38061439,66250369,23134715,5267545,36580610,16387575,67281495,33628349,70420215,112651,43501211,7646095,44245960,8634541,3509435,85242963,80316608,3183975,9623492,33249630,6521313,89046466,84187166,69136837,91831487,80251430,21673260,31727379,62490109,99524975,51464002,83150534,77272628,42967683,99658235,20568363,93270084,75229462,34698463,76671482,26139110,80014588,87598888,33336148,7423788,63506281,51149804,46851987,59177718,65017137,6808825,57803235,99021067,51590803,42307449,11923835,31161687,4064751,83368048,22997281,59981773,24591705,33237508,2717150,4508700,75543508,39986008,74441448,168541,92787493,72019362,37891451,69355476,72725103,17957593,81853704,60955663,17385531,37659250,72793444,77284619,96726697,95010552,27041967,93053405,44177011,7687278,29819940,49641577,57241521,91957544,83533741,98130363,22108665,54762643,90272749,73235980,40865610,18466635,64055960,1022677,14093520,44348328,14626618,36505482,15075176,79821897,4099191,13470059,40677414,99515901,86301513,30764367,62452034,23194618,97379790,24168411,30366150,60102965,96852321,89078848,76170907,47213183,4985896,68041839,18699206,55189057,77301523,89996536,34432810,8913721,44481640,71333116,75777973,26998766,8055981,61623915,21533347,39171895,3088684,669105,33797252,3487592,75407347,9860968,16445503,71657078,29466406,14045383,69579137,79657802,48360198,3233084,30569392,30787683,63967300,65047700,20642888,72777973,44889423,56484900,18783702,13819358,86798033,92530431,45848907,8733068,55850790,17016156,26664538,32161669,20885148,56515456,55470718,70541760,59371804,78549759,48774913,95395112,82532312,17539625,68816413,36753250,4434662,50668599,6986898,17976208,44842615,81172706,26776922,68204242,61859143,67513640,69255765,72614359,12664567,95957797,14349098,5073754,3880712,18663507,45237957,13173644,47090124,65715134,53666583,72373496,247198,88444207,45049308,69848388,38365584,52060076,39373729,30218878,73031054,37435892,43933006,7300047,47738185,92215320,54058788,8791066,22450468,57163802,76825057,569864,51507425,99965001,76330843,8807940,68702632,99917599,16380211,23740167,21001913,94595668,22766820,9575954,1197320,29188588,26397786,53802686,10649306,38256849,88130087,27665211,65454636,20867149,92398073,81274566,90158785,97561745,31491938,72274002,12024238,95726235,77377183,526217,79806380,4091162,7955293,32590267,70372191,99549373,40534591,54427233,34295794,75153252,55669657,36468541,80555751,29510992,7685448,91255408,27411561,15064655,72732205,64087743,4798568,44847298,7502255,72278539,60581278,5970607,62430984,53632373,78218845,22113133,61859581,16424199,31904591,19376156,19101477,47893286,54606384,82726008,55770687,81898046,76960303,19939935,36312813,83302115,49598724,19891772,56424103,49328608,45919976,6111563,83651211,66137019,16405341,68102437,31126490,13862149,48395186,65081429,42692881,30463802,91938191,33565483,25842078,39201414,17894977,12348118,47708850,36812683,74614639,92604458,93057697,61263068,67084351,73617245,56153345,54663246,37166608,70036438,93515664,96906410,44915235,50007421,26863229,22942635,81855445,47887470,55247455,2917920,10961421,17058722,51588015,90457870,4095116,67474219,36933038,60106217,20836893,50702367,34044787,89811711,21397057,13231279,81805959,9257405,14947650,68000591,5832946,5822202,26275734,86821229,91141395,33431960,33123618,74137926,51715482,16019925,39553046,45990383,4978543,24733232,78300864,98739783,28787861,4806458,64157906,82886381,38008118,36780454,61728685,86118021,59197747,93790285,18833224,45428665,61815223,38022834,66667729,26114953,66885828,56461322,40781854,22879907,47361209,42947632,32274392,44784505,8651647,84904436,64098930,4204661,67227442,37957788,75820087,18504197,99226875,68939068,19272365,4069912,96318094,30998561,13468268,62115552,76703609,43506672,3294781,5482538,321665,36930650,61741594,2891150,4787945,28734791,40027975,53393358,70596786,38494874,9175338,34236719,26292919,61982238,92692283,78909561,31733363,15148031,40686254,95290172,54014062,79322415,69605283,10309525,33304202,6525948,94539824,91240048,48088883,79880247,30139692,41380093,77312810,749283,71469330,65851721,65038678,26744917,12303248,89419466,91281584,83269727,92554120,45996863,3633375,90090964,62496012,71300104,99729357,3235882,63059024,70004753,44627776,49236559,54232247,54263880,42199455,97940276,49597667,85116755,45075471,59329511,11161731,89699445,21070110,33895336,55143724,9398733,1250437,36685023,60176618,90061527,34497327,40781449,73222868,61141874,61373987,67124282,52261574,59405277,91664334,94911072,82327024,49271185,36845587,18411915,87720882,56473732,62051033,51315460,10264691,37675718,1431742,58751351,1936762,5640302,89804152,11365791,72358170,99971982,64602895,94360702,91727510,62232211,2607799,52613508,82651278,89217461,44983451,85117092,65880522,46870723,76540481,61897582,34493392,8373289,54199160,90654836,18806856,64848072,14731700,21289531,9829782,44479073,11543098,61271144,19486173,92541302,1204161,62936963,59501487,69641513,84406788,30771409,57359924,33061250,88698958,1788101,28928175,77620120,65338021,8129978,57248122,45794415,79942022,75128745,79738755,7104732,52204879,43045786,77072625,9058407,66045587,66322959,73786944,37280276,42644903,8696647,81677380,20486294,57393458,37192445,6819644,70727211,97783876,35512853,86543538,54987042,23110625,824872,93562257,68128438,23432750,20122224,17386996,54517921,78602717,66271566,82979980,59318837,77787724,22405120,44664587,92998591,29635537,82339363,57961282,2331773,53547802,96709982,79191827,82052050,14363867,37560164,4515343,63372756,80246713,74357852,94711842,83133790,92353856,66832478,32159704,37183543,45667668,8099994,99690194,37620363,19457589,62803858,86242799,45407418,55602660,8115266,77898274,28550822,61928316,9599614,73109997,16684829,83210802,97281847,24953498,57240218,33935899,12571310,68316156,29959549,65271999,75135448,69786756,70800879,99125126,15902805,20002147,20645197,99604946,68694897,10597197,86001008,24826575,29834512,71316369,48260151,72357096,40197395,98648327,16971929,36139942,30090481,58208470,97057187,14723410,11757872,98948034,88047921,86022504,88904910,27185644,56955985,44846932,96735716,36135,34946859,1569515,874791,28796059,44473167,88653118,83083131,45995325,26507214,45306070,82897371,57020481,58180653,41481685,78884452,7517032,37739481,41245325,83948335,28851716,88251446,24619760,93566986,98653983,22721500,48673079,71920426,26102057,67030811,94090109,68110330,7919588,52293995,38009615,34667286,99861373,71965942,20681921,89637706,90004325,60393039,84166196,33699435,30891921,44844121,38341669,74110882,91802888,1239555,17037369,43322743,7066775,6505939,45617087,17764950,75986488,17727650,30811010,79922758,66116458,75052463,71083565,37501808,93359396,42073124,76315420,82427263,49882705,8128637,90310261,6497830,67031644,84349107,24314885,41092102,59614746,66319530,30653863,22435353,9860195,1375023,51426311,27325428,29932657,27187213,36189527,78785507,9259676,66903004,50806615,32151165,62552524,29516592,51466049,22200378,3773993,7182642,12416768,20140249,83747892,89499542,85711894,66428795,74743862,78561158,63152504,15163258,76434144,87160386,69697787,78766358,62762857,32058615,82178706,4119747,10453030,65275241,32699744,17081350,36396314,67451935,41442762,48675329,32426752,45381876,7229550,73021291,15948937,83789391,90013093,92071990,58224549,99224392,19898053,62428472,40356877,66611759,30543215,85023028,18001617,37224844,74452589,61712234,2208785,77300457,59910624,6221471,3233569,16567550,26493119,17857111,55615885,92867155,62357986,6038457,25636669,21919959,29401781,4199704,30163921,85571389,24915585,16595436,71522968,95251277,8729146,48658605,38510840,70782102,82779622,44060493,2204165,83155442,46540998,38645117,53888755,42237907,36808486,97011160,16099750,85695762,72238278,68875490,81774825,6157724,176257,27625735,43357947,33553959,84127901 +61373987,44348328,65851721,112651,3509435,31733363,29959549,25842078,8733068,98462867,8055981,98739783,62693428,15536795,22997281,10358899,30395570,97561745,66322959,53084256,49641577,27325428,3233084,33304202,12416768,81853704,96193415,526217,48774913,54058788,59109400,85023028,99549373,63059024,54427233,3183975,19939935,99965001,4508700,415901,91938191,4985896,79922758,95581843,27411561,83210802,43933006,2917920,8807940,6793819,35092039,72274002,35192533,36505482,23569917,12348118,12664567,72725103,13231279,56531125,23848565,22108665,17146629,19101477,69355476,27041967,55189057,4095116,92033260,60581278,33201905,36580610,55753905,62936963,59371804,18833224,73031054,80251430,40984766,66250369,45996863,9188443,4668450,9575954,30218878,3294781,5822202,65074535,91957544,321665,29029316,20765474,51472275,39553046,61263068,68204242,63628376,4806458,45237957,62740044,79806380,4199704,84187166,61859143,65038678,62496012,10366309,22721500,7011964,34698428,92398073,90090964,89637706,51466049,9886593,71920426,71333116,84002370,569864,99333375,94516935,44846932,82886381,24591705,13173644,14093520,73109997,38494874,66663942,26776922,17365188,24168411,93562257,50668599,83269727,2204165,77312810,93270084,3880712,10309525,38061439,75543508,78884452,13862149,30366150,26063929,82979980,19486173,17957593,26292919,59197747,6153406,26744917,8115266,17068582,50567636,30139692,1204161,91802888,10453030,9623492,45790169,508198,62232211,51047803,54762643,33431960,43322743,42073124,30543215,55602660,27665211,92604458,78561158,65047700,60430369,4978543,79880247,11161731,61982238,87598888,95290172,6111563,50188404,99524975,63967300,98943869,72738685,65715134,57240218,85116755,85242963,44550764,669105,27185644,68041839,26664538,14045383,60106217,70800879,16595436,37435892,14326617,39986008,6521313,38008118,36312813,81172706,68102437,34295794,74743862,72278539,17058722,43501211,41380093,65454636,77300457,14626618,15015906,62452034,1431742,51715482,18411915,69136837,71657078,55143724,33628349,7300047,68128438,34236719,20122224,68702632,67793644,73786944,89214616,12303248,2891150,7687278,68816413,60102965,40152546,24826575,4119747,68644627,37891451,80014588,69579137,48360198,48893685,7423788,72777973,37675718,69786756,33797252,21001913,91255408,3088684,66116458,42967683,3235882,44983451,8129978,77284619,44842615,22129328,61859581,33237508,39373729,54199160,6986898,76330843,70541760,77620120,67031644,30787683,84840549,99604946,64848072,39847321,59501487,86460488,4064751,11543098,24058273,96726697,45306070,14731700,32590267,6497830,72373496,30653863,24915585,90272749,70004753,37739481,34432810,75229462,7646095,45794415,61741594,74357852,91990218,55470718,1569515,65017137,96318094,20486294,58751351,65271999,73168565,54014062,68824981,10961421,16424199,20681921,72358170,92803766,68694897,68000591,18783702,44844121,33699435,30694952,95957797,78589145,99658235,84349107,21993752,83302115,5111370,9058407,75153252,96735716,78218845,92554120,15535065,45428665,37183543,34044787,44784505,47792865,13470059,80555751,36780454,92541302,37501808,2717150,874791,93933709,84904436,40781449,76170907,29834512,67474219,66832478,24733232,97011160,57658654,87710366,5970607,1022677,66428795,33123618,81677380,37957788,6525948,86543538,77694700,40197395,17976208,51588015,66137019,168541,52060076,92215320,76540481,30891921,36845587,36930650,16380211,89046466,49597667,20645197,95395112,6038457,56153345,47090124,4099191,23194618,13348726,52293995,67084351,4787945,43045786,89419466,39201414,99224392,28928175,35996293,55669657,824872,42947632,62803858,86821229,77187825,60176618,69605283,76671482,26392416,57393458,17081350,61960400,9860968,19376156,79821897,75128745,31126490,33336148,92692283,73235980,82726008,62490109,29819940,68316156,40677414,32159704,26139110,77072625,8651647,93053405,78785507,71083565,64087743,10649306,10597197,51464002,41481685,64157906,62051033,77787724,66319530,67513640,66667729,81898046,64055960,29466406,26998766,67281495,57961282,36753250,5482538,8128637,43506672,82327024,15075176,53842979,92998591,92692978,1250437,56424103,24314885,60955663,91664334,11923835,247198,62357986,54663246,42782652,99861373,40356877,18806856,68875490,78602717,84127901,93057697,47298834,97940276,84406788,9398733,10264691,90061527,85695762,90004325,8373289,23432750,48673079,69848388,38645117,78549759,68939068,44481640,36468541,7919588,89699445,53648154,47213183,8696647,38022834,45848907,95726235,86301513,93566986,94360702,65338021,59981773,1788101,44479073,82052050,34497327,89078848,82339363,71522968,57241521,78300864,16099750,20867149,18663507,98648327,18131876,55850790,67227442,53802686,44245960,8791066,14363867,15902805,20836893,13819358,65081429,69641513,20002147,31727379,6871053,54868730,49236559,32058615,32426752,45381876,9599614,16684829,86242799,67451935,42644903,4798568,46870723,29510992,1197320,29635537,98130363,59318837,78909561,9259676,57803235,16097038,79136082,35456853,75777973,83533741,32161669,84684495,89804152,30771409,22435353,32250045,73222868,71300104,42237907,72793444,29932657,33249630,58208470,47361209,31904591,53547802,97641116,71469330,61141874,55770687,62430984,47887470,66903004,29188588,38365584,28851716,85711894,15948937,72732205,79657802,9257405,88444207,29516592,25636669,99297164,89811711,26114953,41092172,29994197,9603598,99515901,16971929,1239555,26863229,99729357,46540998,96709982,1936762,99125126,33565483,6819644,74441448,70367851,22113133,70372191,47708850,83083131,57248122,40865610,73617245,80246713,59910624,44177011,33061250,61815223,82897371,71316369,51149804,70727211,88251446,73021291,99021067,67030811,38658347,90013093,41092102,55960386,68110330,38256849,74110882,59329511,28550822,16019925,30764367,58180653,51590803,86118021,53393358,45617087,63015256,77377183,9829782,90310261,34667286,33895336,66611759,30090481,24619760,82651278,15148031,90158785,8729146,44473167,83651211,89499542,36812683,53666583,70596786,27625735,48892201,17857111,51507425,26275734,20642888,72614359,53888755,7955293,9175338,17386996,7685448,7517032,91831487,81274566,87720882,26102057,64098930,6157724,56461322,1375023,39171895,28734791,17727650,7229550,88698958,89217461,32151165,92787493,75407347,82532312,34493392,3633375,72357096,63372756,72238278,91281584,44847298,52261574,37192445,86001008,30998561,96420429,61623915,94539824,84166196,29401781,77301523,40686254,82178706,39801351,81855445,19891772,24953498,66045587,99690194,7066775,54987042,77898274,83150534,49328608,82427263,21533347,72019362,46851987,28787861,47738185,75052463,76315420,83789391,36135,7502255,11581181,52204879,38009615,96906410,45995325,61928316,17764950,45667668,50806615,14349098,4515343,69255765,76825057,20140249,45407418,64602895,26734892,86798033,31161687,77413012,30811010,94076128,48675329,17539625,8913721,20885148,4434662,65275241,37620363,14220886,48088883,6808825,34946859,21289531,22200378,16387575,8634541,12571310,50007421,66885828,65880522,14723410,55247455,19272365,43152977,56515456,59177718,83948335,80316608,23110625,44627776,41245325,14947650,36189527,44889423,40781854,4204661,74452589,51426311,62552524,75820087,75135448,70036438,43376279,84293052,86022504,22766820,88130087,31491938,79191827,19898053,98371444,26493119,97379790,56473732,43357947,66271566,5640302,37560164,88653118,3773993,6221471,49882705,7104732,11365791,49271185,77272628,22942635,32274392,26507214,95251277,95010552,5073754,36685023,97783876,44060493,30569392,15064655,29065964,79738755,61271144,17385531,21673260,16567550,44915235,45990383,85117092,57020481,36933038,54232247,74724075,16054533,93359396,42199455,54606384,32699744,57359924,4069912,9860195,30463802,40534591,67124282,74614639,92353856,34698463,93515664,90457870,45919976,42307449,52613508,57163802,94911072,83368048,45075471,18699206,89996536,35512853,83155442,33935899,70782102,3233569,81805959,74137926,49598724,96852321,38510840,63506281,45049308,7182642,12024238,23134715,91240048,58749,42692881,26397786,75986488,176257,40027975,94090109,48395186,62762857,56955985,20568363,91141395,21397057,21070110,79322415,38341669,99971982,62115552,87160386,69697787,70420215,37280276,72495719,61728685,22405120,16445503,58224549,28796059,17894977,17037369,78766358,16405341,54263880,83747892,37659250,3487592,92071990,8099994,88047921,73392814,73124510,76434144,2607799,37166608,99917599,22450468,93790285,85571389,18504197,2208785,81774825,5267545,99226875,56484900,97057187,48658605,91727510,22879907,33553959,92867155,5832946,76960303,98948034,36396314,61712234,41442762,94595668,76703609,97281847,18001617,92530431,83133790,55615885,48260151,88904910,19457589,21919959,90654836,60393039,98653983,54517921,17016156,4091162,37224844,36808486,47893286,749283,15163258,2331773,59405277,23740167,61897582,53632373,30163921,27187213,50702367,18466635,79942022,13468268,62428472,51315460,71965942,82779622,63152504,94711842,59614746,36139942,6505939,44664587,11757872 +33304202,62452034,89214616,47361209,61960400,35192533,83210802,15535065,47090124,4668450,92398073,93933709,6793819,38494874,4099191,77413012,86460488,78884452,54987042,55143724,53842979,22450468,76825057,29834512,29188588,7423788,34497327,7646095,56515456,9398733,30543215,80316608,96193415,45407418,99297164,43376279,93515664,11365791,59614746,73235980,37620363,14326617,74137926,74614639,97783876,16567550,51472275,66045587,53084256,112651,54663246,95957797,4064751,63506281,68816413,69136837,8807940,78602717,8099994,17764950,15064655,26664538,37675718,36808486,57240218,34236719,59910624,78785507,62232211,66832478,8129978,99917599,38658347,85242963,81855445,75153252,70004753,61897582,55247455,38061439,63152504,63967300,59501487,35996293,67030811,99971982,247198,60430369,70372191,55770687,99515901,72274002,73392814,96318094,19376156,48892201,26275734,45237957,73031054,1375023,9175338,14731700,67793644,71469330,52060076,98371444,56955985,26863229,61623915,4119747,12024238,31904591,62430984,81898046,32699744,73168565,19939935,99658235,14349098,99604946,51464002,24915585,82427263,23432750,68939068,56424103,45990383,66428795,85116755,48360198,415901,20122224,70727211,87160386,38645117,51047803,23848565,36580610,14947650,17386996,63372756,80246713,86022504,84840549,26063929,84187166,10366309,19891772,44481640,62496012,57803235,18663507,23194618,99021067,68824981,66137019,61859581,7011964,78589145,47708850,20836893,3235882,9623492,79922758,30463802,16099750,57241521,19101477,39801351,36812683,15148031,45075471,50007421,55189057,7182642,43501211,54232247,5111370,98462867,40865610,26493119,16445503,65017137,99965001,57359924,48675329,26292919,36685023,22129328,26744917,9058407,50702367,44245960,95726235,45617087,72738685,42199455,23110625,83133790,74724075,62357986,78766358,57248122,3294781,33699435,17957593,34698463,58224549,43152977,40686254,526217,49328608,72725103,94090109,89046466,88047921,26102057,98130363,89637706,51507425,84684495,874791,52613508,40677414,48893685,91664334,30771409,13468268,36930650,36505482,13348726,67281495,569864,17068582,1431742,26507214,22108665,91727510,59371804,65851721,92541302,33249630,77284619,46851987,11581181,61741594,72614359,34295794,92071990,75135448,77072625,65081429,3773993,27325428,16054533,54199160,3088684,44889423,2917920,77694700,42967683,4515343,94076128,18001617,49236559,65038678,24591705,3880712,82327024,57163802,99226875,40984766,38365584,62740044,77300457,37659250,13470059,8128637,20568363,14220886,42073124,7685448,88698958,65715134,8696647,95581843,4199704,20642888,98653983,86301513,55960386,39373729,35092039,14626618,70596786,99524975,30998561,33237508,79657802,70800879,78218845,54868730,22942635,83948335,53632373,1204161,82726008,8115266,44473167,29819940,77787724,40781854,2208785,71333116,5267545,1022677,18131876,10358899,15163258,30218878,61263068,91240048,4508700,9259676,90457870,4787945,78549759,86001008,73124510,64157906,321665,70420215,32161669,29029316,87598888,34432810,79738755,3233084,27665211,82532312,73786944,68702632,61271144,82897371,59109400,42692881,18783702,83155442,38341669,16405341,73617245,98739783,92033260,95395112,18699206,1569515,45428665,54427233,6505939,37183543,27625735,42307449,90272749,64848072,99690194,54014062,65271999,71657078,2717150,45790169,55753905,13231279,48088883,37501808,97940276,48260151,82339363,15948937,72777973,44842615,63628376,16971929,6525948,75543508,75407347,7517032,6221471,82178706,23569917,89419466,88904910,50806615,32151165,39171895,33061250,21533347,70036438,72373496,69579137,8634541,176257,62490109,16019925,56461322,53648154,45667668,64087743,26392416,92998591,30366150,93790285,77620120,92692978,89996536,20765474,45995325,32159704,54762643,35456853,34698428,92554120,76170907,76703609,66885828,93359396,56473732,76434144,45919976,48658605,76540481,9860968,88653118,44847298,51466049,68644627,28796059,77312810,31161687,30139692,824872,81853704,37166608,29994197,18466635,98943869,18806856,44479073,65047700,27411561,49641577,44060493,31126490,31733363,92692283,5822202,24733232,32250045,74452589,20486294,96735716,79821897,61982238,44983451,17365188,30764367,11161731,3633375,99861373,77377183,90090964,63059024,6986898,44550764,66250369,2331773,85571389,12571310,50567636,93053405,57658654,24058273,89499542,6153406,53802686,66611759,33565483,38510840,95290172,28550822,84127901,22879907,34493392,10309525,65338021,72358170,30090481,91141395,43322743,56531125,44664587,17146629,52261574,52204879,16097038,45794415,41380093,6819644,58180653,84406788,71316369,51588015,44627776,12416768,5640302,59318837,38008118,21993752,94911072,34667286,11543098,16595436,65275241,72793444,68875490,71083565,46540998,9829782,68041839,8651647,37891451,7502255,20002147,3487592,26397786,26734892,33797252,30395570,22997281,7955293,60581278,57020481,42947632,67451935,33553959,37435892,81805959,22766820,73021291,99549373,80251430,47298834,5970607,62428472,32426752,37739481,92353856,75229462,54606384,6808825,36753250,92803766,97281847,1788101,79880247,47792865,33336148,31727379,72495719,9603598,68102437,75777973,44348328,79191827,86242799,32590267,28928175,1197320,3509435,7919588,99729357,42782652,54058788,6871053,41442762,61928316,40534591,73222868,28851716,44846932,95010552,64602895,4204661,94516935,14093520,26114953,69355476,80555751,34946859,72357096,91938191,21001913,49598724,37957788,89078848,60102965,40197395,19272365,16387575,33201905,43933006,66903004,69641513,28787861,6521313,33628349,61712234,50188404,30891921,7687278,40027975,26998766,52293995,45996863,20885148,98948034,40781449,26139110,56153345,4069912,53393358,36845587,10961421,68316156,28734791,10597197,86543538,61373987,83150534,2891150,61141874,22113133,7104732,8373289,29932657,94711842,15015906,84293052,60106217,96709982,61728685,57961282,68128438,4985896,508198,58749,29959549,68204242,87720882,37280276,51149804,59197747,36396314,63015256,66322959,81774825,7300047,82979980,46870723,92787493,55615885,24826575,96726697,79136082,48673079,77272628,84002370,77898274,64098930,90013093,2204165,48774913,9188443,6157724,82886381,91281584,36933038,11757872,17385531,36189527,4434662,97011160,38009615,83368048,81274566,65454636,30653863,16380211,83083131,36780454,27187213,45381876,72019362,71300104,1250437,40152546,94360702,61859143,47213183,669105,29635537,88444207,15075176,14045383,81677380,18504197,78561158,62552524,30811010,30163921,36135,30569392,13819358,94539824,66271566,66667729,44784505,48395186,99333375,32058615,51426311,55602660,73109997,12303248,41092102,86821229,49271185,62115552,53547802,5482538,23740167,29510992,24953498,43506672,86118021,68110330,96420429,13862149,22200378,97641116,62936963,44177011,25636669,65880522,39201414,85023028,4798568,78300864,1239555,66319530,83302115,58751351,91802888,30694952,49597667,19486173,9860195,20140249,17016156,82651278,22405120,82052050,17037369,49882705,98648327,91255408,72238278,17727650,20867149,18833224,97561745,57393458,72278539,71965942,9886593,11923835,24168411,39986008,91990218,36468541,36312813,83651211,76315420,88251446,33123618,32274392,93566986,31491938,69255765,168541,34044787,90654836,45848907,74110882,29065964,3233569,99125126,42237907,17081350,90310261,13173644,65074535,20645197,80014588,38256849,84904436,69848388,77301523,12348118,56484900,75052463,25842078,76671482,76960303,29466406,16684829,23134715,33935899,2607799,50668599,42644903,55669657,83269727,749283,17857111,93562257,90061527,37192445,68000591,90158785,55470718,4095116,81172706,71522968,39553046,12664567,21070110,72732205,19457589,4806458,75128745,91957544,14723410,59177718,47738185,10264691,62803858,54517921,45049308,59405277,75986488,40356877,6038457,96852321,69605283,96906410,85117092,17976208,44844121,8913721,33431960,67084351,41481685,43357947,62051033,97379790,77187825,68694897,10649306,51715482,74357852,45306070,93057697,76330843,8729146,60955663,41245325,79806380,15536795,18411915,94595668,91831487,89699445,6111563,16424199,21673260,62762857,58208470,92604458,51590803,35512853,47887470,92215320,7066775,47893286,64055960,67124282,66663942,37224844,9599614,89217461,5832946,30787683,8733068,33895336,75820087,43045786,39847321,4978543,3183975,83747892,53888755,87710366,71920426,83789391,9257405,70367851,26776922,92867155,60176618,61815223,59329511,9575954,70782102,4091162,10453030,21289531,55850790,95251277,70541760,78909561,99224392,17894977,86798033,67513640,92530431,19898053,67474219,8055981,74743862,7229550,93270084,90004325,1936762,14363867,29401781,85711894,89804152,89811711,54263880,6497830,79322415,79942022,20681921,66116458,21919959,53666583,62693428,37560164,22435353,38022834,69697787,44915235,27041967,24619760,17539625,82779622,69786756,97057187,5073754,83533741,51315460,24314885,27185644,36139942,67031644,8791066,41092172,59981773,15902805,84166196,74441448,17058722,67227442,21397057,60393039,29516592,84349107,22721500,88130087,85695762 +47738185,75543508,67124282,78909561,17068582,77413012,59177718,21673260,22942635,44784505,80014588,6153406,53666583,45667668,85116755,57803235,83533741,39847321,71657078,99297164,7919588,18699206,35092039,29029316,99125126,92692978,72373496,9886593,37739481,9603598,75777973,77284619,30090481,81898046,15536795,44889423,98371444,62496012,88047921,40781449,81853704,21070110,55247455,95395112,4064751,72274002,29834512,27665211,9257405,30787683,10453030,36505482,9623492,94090109,88130087,2717150,73124510,16099750,36933038,5073754,37501808,29635537,79136082,86001008,70036438,75986488,67084351,61373987,62936963,66116458,55753905,29065964,96735716,4119747,33797252,33123618,44177011,51590803,64848072,24058273,23569917,20885148,30163921,69641513,16971929,92554120,168541,58749,40686254,3233084,67227442,5970607,16019925,15163258,22721500,65074535,38645117,82779622,26063929,34236719,18411915,99226875,72777973,97561745,33628349,98462867,99524975,66322959,51047803,26392416,89214616,15535065,5832946,44060493,48260151,7687278,61271144,90013093,30139692,28796059,66667729,669105,7229550,28851716,5111370,33336148,30764367,79806380,41380093,99658235,48893685,55850790,96193415,86798033,73031054,36753250,71920426,77377183,45428665,33201905,94595668,29819940,74357852,44481640,8913721,65275241,34493392,73392814,45996863,77272628,51464002,56531125,14093520,53547802,87598888,41245325,14045383,45995325,99690194,55470718,80246713,8651647,38365584,36780454,68816413,36396314,16445503,34497327,31126490,86301513,36312813,50702367,56955985,82339363,83651211,96709982,96852321,74724075,11923835,16405341,91831487,3487592,6793819,14326617,29959549,59197747,18783702,60102965,6221471,18833224,8733068,30395570,71333116,61982238,76434144,66271566,74441448,21919959,93270084,11543098,34946859,64087743,86543538,45237957,70367851,89217461,20002147,76540481,73222868,17037369,47893286,4099191,45794415,62740044,33249630,79821897,17857111,83789391,68204242,44842615,38658347,28928175,38008118,62452034,34432810,40984766,1936762,17727650,64157906,63628376,45848907,6808825,17539625,36808486,83210802,76330843,62490109,60955663,9829782,68000591,6505939,51315460,72019362,63967300,6871053,62762857,93933709,40027975,22766820,43376279,14349098,3509435,92803766,48088883,91664334,90272749,61859581,74452589,30694952,44479073,48675329,40534591,65038678,73786944,78766358,33895336,45617087,2208785,98948034,53842979,9175338,26139110,57248122,65338021,63506281,36139942,11757872,61623915,17957593,24591705,49236559,29466406,92353856,35996293,66250369,63059024,77312810,73617245,70596786,3233569,62430984,99515901,19486173,17016156,62552524,46870723,2204165,8634541,40197395,33553959,30463802,16097038,65880522,66611759,43045786,4204661,9058407,68644627,66137019,53802686,16380211,57240218,88653118,75128745,17764950,67281495,8129978,84187166,10309525,22113133,4787945,30569392,13231279,35512853,98130363,40677414,55143724,70800879,50806615,52261574,86022504,62232211,42947632,88444207,67793644,112651,44983451,29994197,89078848,54517921,60430369,59109400,23848565,96906410,62428472,16595436,68102437,69255765,24619760,50188404,93562257,83368048,71469330,91727510,84002370,58208470,90457870,19891772,14220886,14626618,66832478,69579137,4508700,59371804,50567636,22108665,89699445,55615885,70004753,21993752,36685023,91240048,10597197,65715134,82979980,10264691,93566986,65271999,28550822,69786756,65851721,7104732,20681921,34295794,54014062,53632373,749283,98943869,29516592,3088684,22200378,99917599,97379790,9860195,99965001,2331773,53393358,55770687,60581278,57393458,12416768,18001617,59501487,43933006,1197320,39801351,61712234,49271185,14363867,41092172,92604458,8099994,57163802,90158785,98648327,508198,84904436,55189057,99971982,81677380,99333375,31491938,95726235,30998561,88251446,44915235,83083131,321665,51715482,6157724,81805959,31727379,10358899,52613508,52204879,80555751,30366150,57020481,26507214,64055960,78561158,7300047,3633375,47090124,30653863,23134715,68316156,35192533,97011160,26292919,50668599,7955293,34698428,81172706,77694700,36580610,45919976,4095116,90090964,6986898,20568363,10366309,15948937,24826575,15075176,65017137,44473167,52293995,91141395,91281584,75820087,1431742,15015906,26998766,88698958,26664538,39986008,54606384,9188443,66903004,76671482,11581181,79322415,79880247,99549373,55669657,79922758,77620120,20122224,37620363,26744917,49597667,7423788,40152546,36189527,3294781,16054533,71300104,24168411,37224844,89811711,39201414,89804152,30218878,55960386,40865610,7182642,40356877,18504197,19101477,14947650,37166608,59981773,38022834,19376156,26397786,87160386,42782652,74110882,13468268,45790169,68702632,46540998,34698463,16424199,89046466,78602717,92071990,66428795,61263068,59614746,20765474,32590267,79657802,71316369,61859143,62693428,95957797,90654836,13173644,73235980,86460488,49641577,91255408,85571389,92033260,1250437,20140249,526217,25842078,99021067,94516935,28734791,12664567,92215320,61141874,3773993,8807940,15148031,12024238,6525948,32699744,33565483,26114953,66663942,97940276,64602895,56484900,65047700,76170907,2917920,70420215,94360702,30891921,20642888,47213183,49328608,1788101,87710366,47792865,57961282,27325428,33935899,53648154,93790285,43501211,47361209,4434662,18466635,66885828,2607799,7646095,67513640,58751351,7011964,19898053,78300864,20486294,61897582,415901,83155442,46851987,24733232,98739783,84293052,54058788,72357096,38494874,93359396,68128438,75153252,70541760,17976208,32161669,62051033,52060076,72278539,73168565,61728685,49598724,65081429,69848388,54663246,78218845,13862149,74743862,58224549,3183975,76960303,95581843,93053405,41092102,90310261,38256849,569864,67030811,60176618,32058615,11365791,69697787,58180653,42967683,9860968,4806458,7685448,47298834,36135,37183543,17385531,51588015,48360198,30771409,72732205,92541302,59910624,82427263,15064655,85242963,61741594,50007421,21533347,38341669,17894977,81855445,18806856,60106217,29188588,72793444,74614639,96726697,76825057,19272365,54199160,88904910,247198,27041967,44844121,54987042,11161731,3235882,36468541,7517032,26776922,22405120,44847298,77187825,53888755,37675718,176257,43357947,22879907,56461322,80251430,1204161,42307449,54263880,72725103,45306070,47708850,99861373,86242799,19457589,39553046,84684495,8791066,83150534,38061439,85117092,68939068,4985896,4091162,62357986,8729146,6497830,3880712,81274566,32159704,97783876,18131876,32250045,78785507,32426752,2891150,99729357,42073124,54427233,89637706,13348726,44245960,72238278,5640302,39373729,87720882,85711894,29401781,824872,18663507,95290172,12348118,82651278,874791,61815223,17365188,83302115,84349107,95010552,59318837,75135448,56153345,64098930,74137926,99604946,12303248,27187213,95251277,53084256,42237907,26275734,66319530,48673079,82726008,57359924,13470059,43152977,71083565,79191827,37659250,35456853,86118021,37435892,42692881,23740167,68824981,62803858,8115266,7502255,92398073,19939935,54762643,1375023,61928316,82532312,26493119,71522968,31904591,13819358,78589145,57658654,47887470,90004325,45075471,9575954,42199455,68875490,17386996,51466049,57241521,63152504,8055981,4515343,1022677,75229462,16684829,44664587,80316608,39171895,77787724,48395186,37280276,5482538,96420429,68694897,89419466,67031644,8373289,16387575,68041839,51426311,37192445,44348328,34044787,85695762,94076128,27411561,8696647,6521313,69605283,79738755,5267545,33431960,97641116,22450468,67451935,1239555,84840549,55602660,69136837,83269727,92530431,41481685,72614359,85023028,15902805,17081350,36845587,91990218,70372191,51472275,24314885,30811010,72495719,97281847,84166196,17146629,10961421,76315420,48774913,22997281,4668450,94711842,65454636,54232247,82327024,54868730,34667286,37560164,48892201,29510992,83133790,66045587,93515664,31161687,42644903,61960400,9398733,71965942,75052463,82897371,72738685,4798568,8128637,78549759,32151165,70727211,69355476,20867149,45990383,96318094,92787493,12571310,45381876,99224392,48658605,28787861,27185644,56515456,6111563,63015256,51149804,45407418,73021291,37891451,43506672,41442762,77300457,16567550,36930650,22129328,33304202,79942022,75407347,82886381,4978543,21001913,44627776,24915585,62115552,49882705,33699435,23194618,45049308,23432750,97057187,4069912,24953498,84127901,26734892,56473732,92998591,68110330,37957788,17058722,43322743,56424103,29932657,25636669,92867155,82052050,36812683,22435353,92692283,98653983,93057697,60393039,91938191,44550764,51507425,10649306,7066775,94911072,44846932,33237508,82178706,59405277,90061527,76703609,91957544,72358170,77301523,33061250,20836893,77898274,81774825,9259676,38510840,26863229,9599614,63372756,91802888,30543215,21289531,5822202,40781854,77072625,78884452,1569515,14731700,67474219,6038457,59329511,89996536,84406788,73109997,4199704,26102057,27625735,83948335,32274392,38009615,94539824,89499542,86821229,21397057,31733363,14723410,6819644,20645197,70782102,23110625,83747892 +47090124,38494874,79922758,64087743,85711894,15064655,29834512,27665211,65338021,67281495,92554120,35192533,98648327,38008118,55602660,61271144,20122224,91831487,88251446,32699744,61141874,33565483,96193415,62452034,16099750,36930650,9623492,36685023,33304202,70004753,1204161,65851721,75543508,69641513,88047921,43322743,86001008,98371444,80316608,95957797,80014588,39801351,16595436,54199160,1569515,78549759,84187166,58208470,31904591,24591705,45995325,7104732,74743862,62430984,4064751,60955663,34698428,50806615,4099191,14731700,77272628,52293995,57803235,1239555,25842078,90457870,30463802,64848072,76960303,61982238,16971929,69355476,17016156,45617087,83651211,8696647,33553959,57248122,2208785,32161669,99861373,60102965,29959549,26063929,53842979,42644903,62357986,57240218,97783876,95395112,50702367,46851987,33628349,9398733,47708850,55143724,2917920,247198,59177718,26392416,99524975,43376279,15535065,46870723,1431742,6793819,18699206,71657078,34493392,78561158,36780454,42947632,68702632,66903004,29029316,89419466,82427263,60106217,23569917,70800879,36580610,33249630,67084351,41481685,30163921,8913721,30090481,26998766,40027975,66611759,526217,4515343,34698463,72777973,40984766,63372756,40677414,66832478,41442762,89637706,37891451,82052050,34497327,79738755,55470718,22200378,37620363,52613508,31126490,43501211,55247455,28550822,48892201,45996863,63967300,93790285,54987042,76671482,56473732,99965001,82651278,17068582,4985896,71333116,19891772,30139692,77300457,4069912,33123618,51149804,52204879,44479073,99297164,77787724,37192445,95010552,30891921,62496012,14363867,35092039,77620120,94911072,58180653,64602895,10597197,28851716,28787861,92033260,37280276,112651,57241521,68644627,65880522,92215320,10358899,18806856,24058273,97641116,89078848,4119747,90272749,52060076,79191827,30543215,66319530,51047803,24915585,73617245,65038678,18411915,30218878,321665,72725103,37183543,34295794,83210802,93562257,74724075,68204242,92398073,84406788,4787945,61623915,99549373,1788101,48360198,26114953,86821229,29188588,74357852,51464002,51426311,28796059,41245325,90654836,89214616,54606384,56484900,176257,79657802,44473167,44889423,8128637,40686254,76434144,38658347,11365791,86242799,70596786,24826575,55770687,12571310,65074535,168541,54663246,22435353,7517032,21533347,4668450,10309525,3773993,66271566,62428472,51588015,10453030,99021067,72019362,81898046,68939068,76330843,38061439,77413012,2717150,81805959,53666583,98462867,42199455,44784505,87160386,19939935,30694952,75777973,3880712,8651647,23134715,14093520,40534591,75153252,55753905,93053405,44550764,6157724,64098930,43933006,50567636,47213183,28928175,9860968,6111563,43506672,96709982,37224844,74614639,6153406,16380211,62740044,76825057,69786756,75986488,50007421,56531125,59318837,80251430,66667729,8129978,66137019,73031054,68128438,14045383,19486173,47361209,61859143,94090109,79880247,17727650,62051033,77377183,669105,61373987,79821897,1375023,77284619,51590803,75229462,44983451,12416768,30569392,70036438,73392814,29819940,42782652,29932657,32159704,10366309,14947650,53648154,26776922,68694897,62693428,60581278,81677380,45428665,49882705,67793644,96906410,87598888,37957788,53888755,57393458,32250045,67227442,24733232,26507214,79806380,23432750,91938191,30771409,99658235,84840549,78589145,55669657,70367851,73168565,94360702,98948034,70372191,98130363,91802888,83083131,17976208,94595668,73786944,61263068,57359924,39373729,99690194,22766820,38365584,45790169,3509435,13348726,78218845,62936963,69136837,3233084,66045587,50188404,15948937,63152504,82726008,7066775,55960386,82979980,75820087,7919588,99125126,20681921,96726697,75135448,74110882,33797252,47298834,35456853,91727510,83302115,61741594,55615885,27185644,92692978,20885148,5970607,6525948,66322959,19101477,29065964,51466049,89499542,86460488,1197320,16019925,41380093,21673260,9257405,26139110,20002147,9886593,27325428,26664538,22997281,88653118,36808486,71083565,11923835,6221471,48260151,34236719,18504197,6808825,81855445,83948335,21070110,72614359,17081350,9599614,45237957,72373496,81274566,42307449,3235882,54427233,62762857,68816413,99515901,26863229,22942635,82779622,68102437,20645197,65017137,7687278,7229550,38341669,48774913,16097038,26292919,12348118,36505482,97561745,65715134,40781449,15075176,40152546,824872,569864,15163258,93566986,98739783,33895336,16424199,12664567,89996536,37659250,49328608,73222868,71300104,51472275,77312810,13231279,38645117,40197395,62115552,7182642,63628376,79942022,9259676,91990218,28734791,99604946,17764950,89804152,81172706,3294781,78766358,25636669,88698958,63059024,22405120,54762643,85116755,66428795,68824981,91141395,48658605,44177011,49597667,61712234,91664334,65271999,14326617,66116458,62490109,61897582,7502255,6038457,48088883,93359396,23848565,30395570,48673079,83789391,64055960,37560164,93933709,47893286,92692283,83368048,29994197,91957544,93515664,44348328,40356877,49641577,17365188,34946859,83155442,8055981,5482538,94711842,6986898,88444207,13173644,19376156,20642888,80246713,44060493,64157906,57658654,4091162,97940276,89046466,4806458,7955293,60430369,39553046,59109400,44846932,29516592,79136082,24314885,3233569,53632373,14349098,16054533,44915235,84349107,36135,15015906,65275241,52261574,20140249,13470059,62803858,59910624,20765474,8115266,74452589,34667286,15536795,69848388,53802686,84293052,43357947,96318094,56515456,9860195,22450468,34432810,77898274,33699435,53084256,16445503,37675718,93270084,42073124,87710366,87720882,29635537,18663507,45075471,44844121,55189057,85571389,66885828,10649306,19457589,67513640,37166608,99226875,54868730,70541760,51315460,29510992,67030811,86543538,58751351,85242963,32058615,99917599,80555751,47738185,10961421,4095116,54517921,94516935,7685448,30366150,4204661,37435892,749283,18833224,68875490,92530431,23740167,53393358,21919959,82886381,5111370,61815223,29401781,83533741,8373289,33431960,95290172,38022834,78785507,17385531,97281847,99971982,9188443,22879907,4434662,45049308,31733363,8729146,59371804,36312813,39201414,2331773,92803766,3633375,15902805,27187213,84002370,44842615,9058407,24953498,90013093,30653863,45848907,18001617,99224392,24619760,78300864,65047700,68000591,62552524,45919976,54014062,21993752,7646095,26275734,58224549,45306070,72793444,45407418,83269727,1022677,44245960,61960400,73124510,67474219,91281584,93057697,45990383,89811711,17386996,92353856,84684495,71316369,33336148,26493119,90090964,415901,56461322,11757872,70727211,70782102,58749,59614746,37739481,5822202,8807940,86022504,91240048,18131876,26744917,12024238,97011160,43045786,48893685,7011964,32426752,30764367,95251277,45381876,39847321,47792865,6505939,96735716,90158785,30811010,83133790,85117092,14626618,88904910,92071990,54232247,77187825,35996293,20486294,7423788,38009615,44664587,5640302,4508700,90310261,76540481,22113133,32274392,48675329,3487592,1250437,38256849,59501487,8634541,9175338,16567550,31161687,63506281,49236559,72274002,20836893,4199704,26734892,82339363,37501808,45667668,36139942,2607799,75052463,36753250,18783702,17037369,48395186,36812683,44481640,98943869,95726235,15148031,61728685,874791,29466406,68110330,32590267,49598724,21289531,92541302,51715482,60176618,74441448,56153345,42237907,92787493,81774825,92867155,11543098,61859581,69579137,33237508,10264691,70420215,42967683,78909561,27625735,50668599,9603598,57163802,67451935,92604458,66663942,23194618,82532312,71522968,19272365,56955985,16684829,59197747,9829782,17957593,72495719,13819358,1936762,6819644,26397786,26102057,19898053,36933038,30998561,69605283,63015256,9575954,18466635,17857111,2204165,20568363,6521313,31727379,89217461,57020481,94539824,68316156,77301523,7300047,66250369,39171895,74137926,49271185,17058722,69255765,97379790,86301513,59981773,5267545,41092172,41092102,47887470,76703609,27041967,90004325,73235980,81853704,92998591,6871053,62232211,99333375,4798568,89699445,16405341,24168411,33061250,8099994,96852321,57961282,21001913,39986008,94076128,65454636,83747892,23110625,3088684,75407347,53547802,22108665,14723410,98653983,67031644,2891150,54058788,71965942,59329511,72357096,69697787,17146629,51507425,5073754,3183975,36845587,45794415,13468268,88130087,33935899,20867149,42692881,40865610,73021291,22721500,84127901,77694700,99729357,17539625,77072625,13862149,27411561,22129328,38510840,55850790,34044787,72238278,32151165,79322415,82897371,35512853,96420429,82178706,44627776,36468541,21397057,46540998,85023028,30787683,60393039,4978543,16387575,56424103,14220886,75128745,65081429,85695762,68041839,72358170,8733068,71469330,40781854,86118021,11161731,78884452,78602717,72278539,76170907,72732205,44847298,43152977,91255408,6497830,8791066,95581843,71920426,84166196,59405277,12303248,5832946,11581181,33201905,83150534,67124282,508198,76315420,82327024,84904436,36396314,72738685,54263880,73109997,36189527,97057187,86798033,90061527,61928316,31491938,17894977 +73168565,68816413,10358899,67793644,63967300,16097038,3183975,40865610,47792865,53084256,49598724,5111370,4434662,17058722,26102057,70596786,20765474,35192533,31904591,82327024,72495719,4508700,76671482,70541760,15015906,79806380,54199160,75543508,37620363,77312810,71657078,18131876,8129978,6986898,415901,24168411,96726697,79136082,33628349,14731700,89637706,30395570,26392416,50188404,96193415,30891921,37957788,92398073,28928175,19101477,1197320,56461322,39201414,9886593,19272365,6793819,60106217,59371804,38658347,61141874,8913721,65715134,64055960,59981773,69786756,68824981,24058273,89214616,8128637,112651,27625735,8115266,2717150,98739783,19376156,99524975,93057697,43357947,72777973,77072625,8807940,21533347,8696647,26063929,81677380,44842615,14349098,78602717,56531125,55960386,82339363,62936963,55247455,50567636,38061439,9398733,17081350,29834512,68644627,39553046,53802686,23848565,25842078,45996863,98943869,7517032,16405341,37659250,3509435,40152546,42644903,98462867,44846932,77300457,20486294,24619760,72357096,35996293,10366309,51047803,91802888,97641116,17764950,77787724,84127901,69605283,33304202,93515664,22435353,10597197,15064655,19486173,4119747,76170907,15535065,7687278,39171895,7646095,61263068,92692978,84349107,13468268,21001913,42782652,30463802,8651647,99917599,97561745,84904436,40197395,52060076,6505939,9623492,68694897,51472275,43506672,95957797,53632373,94076128,64087743,68204242,83789391,84684495,89078848,92071990,36468541,5970607,16099750,53547802,26507214,46870723,9575954,82052050,66045587,43501211,22450468,49641577,50668599,95581843,77272628,20002147,49271185,4069912,72373496,92604458,37675718,66903004,83533741,4985896,51315460,49882705,61373987,72732205,4515343,81898046,22997281,14626618,11581181,57241521,31733363,37891451,14326617,35512853,55470718,508198,55143724,78766358,42967683,69355476,65851721,67451935,93053405,74441448,17365188,11757872,58180653,81855445,70800879,80555751,34493392,1569515,68000591,46851987,92787493,47298834,31727379,78218845,68875490,62232211,74452589,71469330,81274566,88653118,57020481,48893685,77284619,48774913,38022834,45407418,57240218,89419466,55189057,6497830,66250369,31491938,78589145,66885828,4668450,24591705,59109400,44473167,36930650,65081429,65017137,63372756,19457589,1204161,62051033,60430369,10309525,526217,26292919,43376279,2204165,62693428,45237957,91664334,95395112,62496012,13862149,91990218,56424103,82726008,75153252,30998561,32590267,81853704,33201905,76960303,89046466,32159704,77187825,49236559,29510992,99658235,44983451,19939935,27041967,45381876,62430984,31126490,99021067,61271144,48892201,44847298,4798568,86118021,17068582,99515901,69697787,22942635,23194618,56955985,57248122,70004753,14093520,26998766,29819940,35092039,52261574,87160386,55753905,41481685,44889423,59177718,85695762,29466406,21673260,4204661,16019925,81805959,30653863,42692881,73617245,30090481,17146629,94595668,3294781,37183543,53393358,73124510,669105,45995325,66832478,60102965,3233084,67281495,34432810,30811010,72238278,24733232,50007421,7423788,23432750,45794415,67030811,30366150,54427233,56515456,78549759,7919588,79657802,97940276,53666583,55602660,36753250,62452034,51466049,93933709,23110625,57658654,40781854,34044787,749283,17037369,33249630,66611759,16595436,37166608,73786944,67227442,83302115,39801351,12416768,94090109,91938191,38494874,58749,77413012,70782102,66271566,82178706,5640302,68110330,89996536,4978543,38008118,6221471,36812683,71333116,99690194,17016156,2917920,33237508,15948937,99297164,16380211,24953498,88444207,83747892,22108665,10649306,68102437,52293995,13231279,79821897,26863229,13173644,87710366,36933038,61982238,32250045,66428795,61859581,43045786,79942022,85116755,62803858,27325428,51507425,82897371,1250437,2208785,89217461,39373729,84840549,3088684,44479073,54987042,62552524,44664587,14947650,9058407,40677414,30163921,12348118,99333375,59501487,59614746,168541,33797252,45990383,7685448,17539625,56473732,72278539,65454636,29959549,6157724,77694700,73392814,57803235,3487592,16445503,6153406,49597667,7011964,83210802,87598888,56153345,48088883,38645117,7182642,12664567,27411561,94516935,34698463,65047700,30764367,33061250,34698428,30787683,40781449,54058788,42307449,62490109,85711894,74357852,68316156,86460488,22129328,94711842,66322959,18466635,94911072,40027975,90004325,71920426,64848072,43322743,65038678,68939068,8634541,73031054,79738755,62357986,86543538,38256849,92692283,84406788,95726235,1431742,47361209,51715482,49328608,874791,75128745,45790169,54014062,41380093,4064751,53842979,92033260,21289531,57393458,9257405,7229550,60955663,98648327,59197747,45049308,74110882,54762643,16387575,58751351,26734892,70036438,87720882,82532312,89699445,63059024,78909561,37192445,99965001,35456853,71300104,99861373,18663507,64157906,33935899,5482538,98130363,27665211,90310261,91957544,18806856,82779622,4095116,64098930,61728685,76434144,98371444,84187166,45848907,8373289,97011160,86301513,83651211,9188443,70372191,1375023,80251430,18783702,88047921,92803766,74743862,75229462,54663246,20645197,58208470,77377183,62740044,12303248,63628376,8729146,77301523,84002370,26139110,43933006,97783876,50806615,29188588,95290172,8733068,66663942,24915585,15902805,3233569,30694952,569864,23569917,44177011,2607799,67031644,33895336,36135,76540481,9860195,9603598,54606384,28734791,85023028,48395186,42073124,71316369,51149804,86242799,25636669,21993752,16684829,14045383,59910624,69579137,16567550,57163802,20122224,72738685,17727650,78300864,33123618,20885148,80316608,80014588,34295794,45667668,81172706,18504197,13348726,69136837,7955293,62115552,47090124,73235980,44784505,79922758,47738185,44348328,92541302,37560164,94360702,22766820,74137926,86022504,70420215,37224844,28851716,92215320,70367851,85242963,20642888,61815223,5267545,68041839,61960400,61623915,36808486,33553959,78884452,30569392,44550764,92867155,88698958,63015256,321665,86798033,29029316,59405277,48675329,83083131,11161731,10961421,247198,40686254,16424199,19891772,68702632,26114953,47213183,88251446,8099994,36396314,4787945,22405120,76703609,5822202,89499542,30218878,51426311,6808825,90654836,6521313,32161669,26275734,65338021,71083565,98653983,79880247,72019362,88904910,4199704,99729357,21070110,44844121,90457870,67084351,22113133,1239555,40356877,99224392,86001008,17976208,79322415,16971929,16054533,12571310,91141395,70727211,9175338,33565483,32274392,89804152,18411915,15075176,6525948,59318837,92353856,55770687,30771409,91255408,42237907,53648154,45919976,30543215,96318094,91727510,14220886,26664538,23134715,76330843,37501808,11923835,83948335,48360198,78785507,83368048,26397786,66137019,39986008,32058615,31161687,6038457,41092102,83150534,82886381,37739481,91831487,72725103,176257,4099191,75986488,44627776,90272749,36312813,96420429,75052463,44245960,55615885,3773993,82979980,24314885,99549373,99125126,96906410,17857111,6819644,824872,82427263,36189527,17894977,44481640,66667729,74614639,44060493,60581278,29065964,77898274,93270084,21919959,41245325,4806458,91281584,68128438,63506281,90061527,59329511,6111563,5073754,40534591,2891150,41092172,15536795,45306070,90090964,84293052,6871053,69848388,32426752,3235882,64602895,47708850,36685023,29994197,18833224,77620120,86821229,53888755,48260151,10264691,34497327,69255765,61741594,63152504,61928316,81774825,74724075,22879907,39847321,52613508,95010552,15163258,34667286,37280276,36580610,9259676,92998591,75777973,7104732,1788101,27185644,30139692,92554120,72793444,48658605,21397057,91240048,51588015,44915235,26776922,45075471,13470059,28787861,48673079,55850790,66319530,65275241,37435892,33699435,95251277,92530431,76825057,85117092,20140249,93359396,36845587,61897582,41442762,96709982,42199455,2331773,93566986,50702367,67474219,62762857,54868730,62428472,28550822,26744917,73222868,99604946,56484900,61859143,9599614,54263880,72614359,1022677,20836893,24826575,67513640,38365584,93562257,79191827,54517921,66116458,38341669,38510840,33431960,45428665,7300047,65271999,20681921,3633375,36780454,17385531,96735716,23740167,11543098,89811711,83155442,1936762,78561158,67124282,98948034,57359924,14723410,17386996,9829782,60176618,20568363,51590803,75407347,32699744,88130087,18001617,65880522,10453030,51464002,18699206,72358170,97281847,97379790,80246713,94539824,75820087,8791066,75135448,22200378,7502255,29932657,83133790,36505482,34236719,69641513,4091162,3880712,97057187,93790285,55669657,20867149,29401781,34946859,17957593,43152977,73021291,40984766,47893286,85571389,57961282,58224549,8055981,27187213,38009615,32151165,22721500,19898053,72274002,12024238,5832946,71522968,83269727,42947632,61712234,46540998,15148031,11365791,96852321,99226875,7066775,47887470,26493119,28796059,9860968,90158785,73109997,33336148,82651278,52204879,29635537,76315420,71965942,99971982,45617087,65074535,29516592,90013093,54232247,14363867,60393039,84166196,36139942,13819358 +16097038,73392814,14947650,7011964,7300047,99549373,43152977,90457870,32699744,95957797,45617087,89214616,91802888,95290172,68644627,22450468,44550764,88698958,65271999,96726697,96193415,44889423,89996536,74614639,4806458,45407418,67793644,63372756,37183543,21289531,86242799,66667729,48893685,4069912,14363867,55247455,89078848,51472275,44664587,61815223,62430984,57248122,73617245,89419466,29834512,70004753,51464002,17957593,31126490,51047803,569864,99226875,14093520,57241521,16019925,13819358,1250437,59177718,44983451,51588015,62452034,2331773,36780454,8807940,92554120,37166608,85242963,74724075,69255765,82178706,45790169,98130363,15015906,99965001,14349098,24058273,50702367,33237508,22129328,14326617,98371444,91281584,72725103,749283,54517921,70036438,76671482,33797252,97783876,21001913,45381876,22766820,33249630,112651,73235980,83302115,34667286,29994197,10358899,73222868,97379790,77072625,415901,83747892,18504197,3487592,93566986,72793444,83789391,4064751,91990218,37891451,22942635,15064655,18833224,38658347,36812683,72777973,26139110,9257405,16684829,80555751,74137926,47361209,58749,5111370,52204879,20140249,51426311,29959549,3183975,31904591,4798568,9623492,59318837,44842615,20568363,68102437,90272749,62936963,64098930,86001008,61982238,17068582,41481685,61623915,43322743,33628349,60102965,92692283,51466049,85571389,32426752,43045786,7955293,6793819,59405277,48260151,8099994,34493392,99224392,1431742,65454636,36580610,60430369,27665211,43376279,45990383,47708850,23134715,59981773,16445503,66885828,12348118,22721500,83269727,16099750,63967300,40197395,54427233,62496012,60955663,82886381,8913721,83368048,90061527,50188404,59614746,29029316,247198,49641577,26776922,85116755,32161669,26998766,38510840,21070110,97641116,65338021,95395112,36933038,68939068,53842979,7182642,17016156,5640302,22113133,4099191,88047921,58208470,17365188,10597197,2717150,78218845,508198,13468268,17764950,15163258,91831487,7646095,20642888,48360198,54762643,6153406,14220886,46851987,60581278,30998561,23110625,80316608,70596786,321665,72738685,99515901,76960303,55850790,82532312,88904910,76540481,56473732,73031054,68694897,99729357,29188588,39553046,39986008,6986898,1204161,3233084,49328608,57240218,78909561,35192533,84684495,78766358,10309525,44627776,26292919,67451935,51315460,62740044,94595668,59501487,77272628,54663246,20122224,15075176,78589145,35996293,67281495,6497830,53632373,98948034,33699435,96735716,51590803,34698428,29819940,55960386,72278539,71657078,29466406,69697787,84187166,64087743,93515664,58751351,2891150,83651211,90158785,31161687,92692978,16405341,3233569,66045587,47893286,93270084,14626618,18466635,55602660,77694700,42073124,65047700,18806856,4668450,98739783,9175338,1197320,72357096,52060076,13231279,93053405,24314885,55189057,59197747,37620363,27041967,84293052,17386996,92353856,45995325,526217,25842078,44479073,33304202,45919976,82779622,9058407,61859581,35456853,90090964,88653118,26275734,44844121,71083565,91727510,40781449,63015256,43357947,57961282,40027975,59371804,54987042,89046466,92033260,82427263,72274002,98462867,9188443,87710366,27625735,16380211,6505939,5482538,19939935,89804152,39171895,41442762,9599614,55669657,82979980,94539824,5822202,4985896,5970607,79821897,40534591,77312810,57020481,42237907,12303248,44245960,67227442,39847321,92530431,89217461,18663507,66250369,23432750,98943869,56531125,82339363,54014062,38645117,26114953,18699206,45237957,99524975,92604458,35512853,86118021,38009615,83150534,82052050,10366309,7502255,84002370,61263068,14045383,29065964,57163802,4508700,92803766,63059024,14723410,47792865,92398073,22200378,36930650,39201414,62232211,81172706,62762857,4787945,40984766,66611759,49597667,72373496,90013093,33895336,38061439,30764367,99917599,78549759,10453030,41092102,77413012,14731700,41380093,36135,94090109,26392416,30569392,73786944,61712234,18131876,22108665,11161731,99690194,30653863,30463802,40781854,76330843,42307449,3509435,91664334,45306070,28796059,80014588,29635537,99658235,66319530,31733363,51507425,73168565,45428665,10649306,20486294,97561745,72019362,36753250,68204242,40677414,51715482,9259676,6111563,71333116,7423788,56153345,61960400,68816413,64848072,16595436,84406788,6808825,52261574,26063929,72614359,84349107,61271144,27187213,64055960,12416768,30891921,47090124,36505482,13862149,55470718,93562257,63506281,96709982,99021067,81677380,20002147,874791,87720882,669105,92787493,44348328,15535065,69136837,53084256,98648327,39801351,68824981,71469330,12024238,29510992,29516592,15148031,11757872,77187825,1375023,68041839,92998591,50806615,30771409,4091162,68000591,53666583,45794415,66832478,74452589,61728685,36189527,23194618,74357852,77787724,63628376,2208785,48675329,56515456,19891772,11581181,4515343,78561158,43501211,8651647,61741594,34044787,85711894,44847298,4434662,91938191,55753905,3880712,62803858,81805959,16971929,40686254,66137019,83210802,24168411,40152546,17894977,20645197,78785507,65275241,86543538,61859143,67030811,2917920,83948335,28787861,72732205,99297164,73124510,75543508,81855445,81898046,78884452,28550822,71300104,82897371,75777973,30218878,25636669,32590267,24619760,37435892,91141395,18001617,75820087,26493119,64602895,32274392,48088883,3773993,5267545,96420429,89811711,8373289,46540998,42692881,36808486,12571310,95726235,79136082,70420215,84840549,75229462,94076128,42644903,55143724,97940276,92215320,8129978,27411561,88251446,34295794,16054533,42967683,7919588,62115552,38008118,24591705,7517032,45667668,44177011,81774825,77620120,82726008,79942022,42199455,68110330,30366150,97281847,49271185,1239555,47887470,45848907,57658654,4095116,8115266,49598724,52613508,77300457,65081429,30395570,84127901,66663942,40356877,82651278,28851716,30543215,30090481,53648154,62051033,8791066,8733068,17385531,38022834,62490109,94911072,70727211,10264691,83155442,9603598,18783702,81274566,85117092,43506672,19457589,48658605,75153252,12664567,85695762,37560164,13470059,17539625,69605283,30787683,83533741,55770687,36685023,41245325,54606384,93933709,66903004,57803235,20765474,76825057,58180653,77898274,78602717,34236719,44473167,7685448,22879907,26744917,8696647,6525948,38365584,26507214,3294781,31491938,77377183,72495719,90654836,95010552,33553959,37280276,19486173,74743862,59109400,98653983,22997281,76170907,79322415,38341669,86798033,26863229,45075471,7229550,63152504,28928175,65880522,54058788,47213183,21993752,32159704,9575954,60176618,76703609,83133790,70367851,86022504,54868730,10961421,13173644,11923835,96852321,22405120,56484900,17727650,69641513,88444207,41092172,91957544,50567636,72238278,37224844,11543098,19376156,99333375,56461322,59910624,74110882,9398733,33336148,65715134,70541760,68875490,53802686,27325428,94360702,79880247,48892201,2607799,47298834,79191827,34497327,29932657,51149804,26734892,168541,35092039,93359396,79922758,99971982,42947632,8634541,62693428,68128438,69786756,18411915,23740167,50007421,1569515,32250045,33565483,6157724,48395186,176257,6038457,42782652,56424103,44784505,99861373,82327024,1022677,79657802,13348726,72358170,31727379,69355476,30694952,6819644,67474219,44481640,24953498,87160386,34698463,23569917,94516935,15902805,36312813,97057187,75135448,75986488,70800879,80246713,96906410,75052463,36468541,53547802,86460488,69579137,66428795,66322959,58224549,40865610,39373729,71522968,99125126,21397057,84904436,15536795,38494874,76434144,26664538,44846932,87598888,2204165,21673260,5073754,54232247,90310261,26397786,57393458,7104732,54263880,6221471,89499542,45996863,91240048,9860968,62552524,8729146,77284619,43933006,66271566,79806380,17976208,19272365,65074535,24826575,78300864,36845587,9886593,17146629,33935899,37659250,46870723,29401781,19101477,68316156,65851721,67084351,90004325,11365791,50668599,49236559,3088684,89637706,85023028,37675718,70372191,4119747,65017137,36139942,47738185,33123618,4204661,20681921,24733232,88130087,33201905,57359924,17058722,3633375,23848565,37957788,89699445,68702632,30811010,37501808,20885148,38256849,93057697,20867149,94711842,62428472,86821229,1788101,66116458,44915235,71920426,30163921,48774913,62357986,71316369,61141874,34946859,70782102,33431960,67124282,1936762,15948937,77301523,86301513,36396314,24915585,92867155,71965942,99604946,91255408,61897582,95251277,76315420,8128637,97011160,53393358,55615885,17037369,96318094,56955985,27185644,59329511,54199160,19898053,6871053,9860195,93790285,3235882,60393039,7066775,49882705,61928316,32058615,9829782,64157906,81853704,17081350,67513640,80251430,21919959,92071990,824872,22435353,6521313,79738755,44060493,83083131,16424199,48673079,8055981,65038678,74441448,73021291,5832946,67031644,52293995,16567550,60106217,92541302,20836893,32151165,33061250,37739481,34432810,45049308,4199704,21533347,84166196,26102057,17857111,69848388,61373987,16387575,4978543,95581843,75407347,28734791,7687278,37192445,75128745,53888755,30139692,73109997 +64087743,33237508,73031054,77312810,26734892,20122224,40197395,27665211,6871053,65074535,87160386,57020481,31904591,29834512,80555751,6505939,71657078,51588015,80316608,67793644,35092039,73168565,18699206,17957593,16445503,94360702,4787945,89046466,55669657,37659250,69136837,78549759,98130363,14326617,14045383,98371444,14349098,36812683,47090124,7011964,99658235,22108665,40152546,16405341,96193415,70596786,97641116,12348118,51047803,96709982,84349107,91802888,31161687,69848388,569864,87598888,97783876,42692881,41481685,44844121,74110882,11581181,49328608,53648154,51507425,45790169,99021067,47361209,17016156,72732205,85242963,39171895,43376279,55143724,65454636,14220886,16019925,75153252,81805959,77072625,66611759,247198,32699744,29994197,43506672,55753905,83150534,73124510,112651,60581278,30463802,17058722,70782102,8696647,68816413,29959549,12571310,92692978,47298834,66903004,67281495,86301513,12416768,28734791,77284619,89996536,21533347,37183543,83651211,21289531,49641577,83155442,66137019,24058273,69255765,74743862,874791,87720882,26275734,95251277,46851987,8128637,79657802,61741594,68102437,96726697,36780454,82327024,99333375,9175338,17068582,93933709,93790285,56461322,321665,59614746,54199160,14723410,13231279,82339363,59501487,22129328,62051033,94911072,81855445,53393358,52204879,68702632,73786944,92998591,10358899,40356877,63967300,40781449,38061439,61859581,80014588,35192533,76170907,43357947,43501211,81274566,73392814,56473732,88047921,89214616,9603598,54987042,43045786,38494874,60430369,6793819,65038678,76540481,40686254,79806380,16380211,70800879,4095116,22997281,77187825,3773993,44481640,79136082,19939935,56515456,26998766,20140249,508198,57241521,44847298,42967683,54606384,9623492,33699435,1431742,51464002,95290172,13468268,68204242,33553959,17386996,36685023,37166608,14093520,34698428,22450468,93359396,86821229,68644627,44983451,77620120,10597197,44177011,29188588,84002370,76703609,90272749,74614639,37280276,51715482,36468541,67227442,39373729,91957544,16099750,61960400,19101477,18131876,63059024,7919588,4099191,79191827,53547802,33304202,93562257,749283,30764367,39801351,28851716,36312813,49597667,72738685,61728685,13862149,70727211,6986898,75777973,7300047,91281584,15148031,73235980,50188404,99524975,95395112,70036438,62452034,86242799,65851721,83083131,47893286,65338021,78218845,7685448,10309525,9398733,5970607,22766820,30543215,8651647,54663246,37501808,35996293,1197320,95726235,6525948,44889423,27041967,2891150,32590267,62430984,71469330,66250369,74724075,89811711,75128745,53084256,65715134,81898046,98739783,32159704,87710366,44846932,5832946,18783702,45237957,9886593,44842615,4119747,90310261,31491938,3233084,18001617,19891772,19486173,84684495,32250045,15535065,24591705,72725103,92398073,95957797,50007421,79942022,29466406,71300104,77413012,55770687,84187166,23848565,36845587,27187213,44348328,57248122,60955663,61623915,56424103,37891451,20486294,47792865,45617087,59197747,84293052,83210802,23432750,68316156,71083565,8099994,6497830,74137926,57359924,61271144,17764950,54058788,76960303,89804152,46870723,20765474,45667668,26392416,42947632,25636669,18663507,88653118,3880712,82886381,52261574,91938191,55247455,6819644,2204165,50567636,99125126,16097038,30787683,94076128,37675718,30218878,78602717,44550764,17365188,75543508,92692283,24733232,66319530,99515901,73617245,40781854,64848072,73222868,5482538,79322415,15015906,48395186,23134715,44784505,22942635,48774913,83789391,75135448,93566986,60102965,81677380,43152977,26102057,57163802,98648327,26063929,48360198,8129978,13348726,23740167,68041839,77787724,68694897,1239555,20002147,85571389,92530431,78766358,94090109,59329511,77301523,30366150,40534591,52293995,8634541,99729357,9860195,80251430,77300457,9259676,9188443,70541760,58749,37620363,26292919,50702367,15902805,7423788,10366309,59177718,72357096,37957788,79738755,38008118,96735716,16567550,34432810,23569917,59109400,3487592,4668450,59371804,88251446,44245960,69355476,45428665,29510992,62115552,36580610,33201905,48892201,78785507,38645117,77898274,33628349,76330843,11161731,62936963,62496012,56153345,36753250,99549373,68000591,20681921,4798568,75986488,16424199,63152504,8791066,48675329,83133790,42073124,82726008,18504197,44473167,55960386,2917920,26744917,26397786,63628376,824872,2717150,14626618,168541,82178706,61141874,82779622,53888755,66832478,8807940,57803235,21993752,73109997,31126490,37192445,4434662,99604946,43322743,55850790,96852321,61859143,70004753,26664538,61712234,44664587,40027975,66271566,61815223,69605283,36139942,34497327,91141395,48260151,72777973,5111370,48658605,23110625,76315420,45990383,72019362,68875490,81172706,91990218,65017137,31733363,62490109,37560164,79821897,42199455,59910624,53632373,90457870,77272628,19376156,67030811,41092172,76434144,3233569,7502255,22721500,66116458,99297164,24619760,6808825,55189057,82532312,40677414,54232247,26139110,63015256,67451935,15064655,33797252,20568363,88904910,72614359,16595436,7104732,98948034,85023028,57240218,40865610,61897582,12303248,26114953,71333116,54014062,79922758,1250437,4064751,96318094,7517032,7066775,1204161,62232211,93270084,33123618,94516935,63506281,51590803,57658654,86022504,47708850,96906410,4091162,72373496,99226875,30395570,56531125,13819358,69786756,22879907,77377183,12024238,90654836,49236559,89419466,89217461,89637706,14731700,17539625,97057187,33431960,14363867,4806458,49271185,29401781,6153406,3509435,20867149,65271999,74452589,32161669,3088684,415901,6038457,28787861,33249630,4985896,84127901,65047700,88444207,67124282,88698958,20885148,91255408,66667729,33895336,99965001,68939068,76825057,26507214,11543098,78909561,48088883,86001008,32274392,86543538,65081429,26776922,42307449,50806615,47213183,32151165,1022677,50668599,84904436,70420215,91664334,51426311,45306070,20642888,62693428,3183975,70372191,71920426,60393039,3294781,21001913,34493392,8115266,24915585,59405277,24953498,22113133,19457589,72278539,39847321,24168411,13173644,62552524,7229550,13470059,9860968,14947650,9599614,5640302,34295794,64602895,68110330,43933006,10649306,64055960,42237907,71965942,53842979,95581843,54868730,47738185,22435353,30891921,66428795,4515343,2208785,72793444,6521313,69641513,1375023,18466635,92071990,27185644,61982238,3235882,55602660,74357852,53666583,45996863,91831487,90090964,75229462,49598724,61373987,91727510,55470718,22405120,56484900,52613508,34698463,36930650,30163921,42644903,89078848,93057697,4069912,11757872,92803766,84166196,36505482,51466049,28796059,89499542,96420429,51472275,63372756,72495719,9058407,97561745,11365791,67513640,72274002,27411561,7955293,30653863,59981773,57961282,48893685,5267545,16054533,29516592,38658347,90158785,98943869,85117092,36933038,92787493,16971929,94711842,60106217,29029316,20836893,36189527,86460488,33935899,26493119,99917599,6221471,64098930,1569515,38341669,27625735,99224392,30090481,30771409,84406788,25842078,38022834,97281847,92604458,51149804,8373289,54427233,79880247,75820087,51315460,78561158,42782652,34946859,21919959,32058615,669105,84840549,68128438,30694952,6111563,90061527,19272365,45407418,54263880,5822202,29065964,97379790,66663942,526217,92554120,65880522,31727379,21070110,85116755,71522968,39201414,69579137,4508700,36396314,45794415,73021291,90004325,39986008,76671482,18411915,81853704,54762643,52060076,35456853,18806856,34236719,45848907,58180653,45995325,16684829,10961421,92215320,93053405,34044787,28928175,8733068,38510840,36808486,93515664,9829782,98653983,81774825,59318837,9257405,19898053,75407347,45075471,58224549,35512853,65275241,37739481,83302115,99690194,28550822,21673260,17081350,66885828,17857111,72358170,78884452,62357986,98462867,29932657,95010552,36135,94539824,83368048,15536795,74441448,62803858,10453030,48673079,44060493,82979980,61263068,8055981,37224844,58751351,7687278,9575954,20645197,16387575,67084351,30998561,41442762,69697787,33061250,72238278,77694700,5073754,82651278,29819940,83747892,8913721,82427263,99861373,6157724,78589145,23194618,11923835,12664567,44915235,38365584,97940276,57393458,41092102,30569392,18833224,67031644,45381876,38009615,88130087,66045587,10264691,68824981,3633375,86118021,66322959,89699445,82052050,86798033,30811010,33565483,24826575,40984766,92541302,53802686,71316369,82897371,62740044,26863229,30139692,85695762,85711894,4204661,60176618,70367851,46540998,75052463,62428472,7646095,58208470,1788101,41245325,22200378,90013093,44627776,92353856,15075176,80246713,17727650,47887470,83533741,7182642,54517921,2607799,45919976,99971982,78300864,27325428,92033260,29635537,17146629,83948335,17894977,4199704,92867155,38256849,39553046,176257,41380093,94595668,91240048,56955985,15948937,97011160,1936762,44479073,34667286,32426752,62762857,49882705,15163258,61928316,45049308,67474219,83269727,8729146,17976208,64157906,37435892,21397057,24314885,4978543,2331773,33336148,17385531,55615885,17037369 +44889423,98948034,44550764,96726697,44844121,99549373,51047803,20486294,78602717,1788101,68644627,74110882,92398073,76960303,14723410,43322743,81677380,68041839,77072625,17894977,39986008,43357947,72777973,1197320,60430369,14731700,30787683,30543215,48673079,79821897,67451935,56473732,93053405,4099191,82886381,85116755,65081429,37183543,10961421,247198,31904591,77898274,29994197,21993752,52261574,19457589,44842615,93057697,94711842,42073124,63059024,49641577,66667729,11161731,2917920,14626618,25636669,37620363,80316608,56424103,40781854,88698958,93933709,35192533,67281495,8099994,4668450,53648154,72725103,87720882,17764950,63967300,74743862,5822202,94090109,29959549,70036438,72019362,41481685,35456853,36505482,72357096,84684495,92604458,17727650,88653118,7104732,82979980,62430984,62552524,45407418,32590267,14326617,63372756,97783876,77620120,38658347,20867149,57241521,28796059,57393458,72278539,57961282,79191827,45237957,65271999,1431742,59614746,17016156,66428795,24619760,86242799,67793644,33431960,59371804,66611759,8729146,91255408,43376279,59177718,81274566,168541,36580610,45306070,97641116,26998766,69848388,47090124,80555751,93562257,9575954,34698463,32274392,13231279,22108665,96193415,59501487,10309525,91802888,43933006,98130363,18504197,15148031,84293052,62740044,49236559,6819644,21070110,61623915,53632373,93515664,36812683,63152504,55189057,46851987,68702632,68694897,26392416,65275241,8791066,39801351,13470059,4119747,22766820,38645117,85023028,54014062,92692283,89078848,8115266,23194618,34236719,13468268,84166196,83155442,36930650,62115552,51590803,61373987,54517921,90457870,99965001,22721500,83150534,69786756,5970607,19486173,874791,8373289,69605283,93566986,30998561,48260151,75777973,29188588,5111370,7300047,7502255,5640302,34698428,86022504,55143724,57248122,99226875,51472275,44846932,3233084,40984766,12348118,54427233,9603598,40677414,669105,39171895,36753250,26139110,16387575,62803858,6157724,33304202,73786944,4985896,19898053,46540998,26114953,59329511,30569392,43045786,94539824,13819358,95290172,43501211,46870723,95957797,72358170,89217461,54762643,62496012,43152977,40686254,99224392,48774913,23848565,49271185,5482538,17539625,91957544,64848072,36780454,49328608,16097038,59318837,2204165,47887470,9829782,12664567,82897371,43506672,2717150,40781449,10366309,41092102,83789391,90061527,27041967,91727510,91831487,19891772,44348328,18783702,28928175,80251430,22129328,99658235,14093520,69355476,22200378,60581278,22879907,73235980,7646095,47213183,9188443,6808825,9259676,7229550,53393358,15536795,11543098,38365584,82427263,66885828,83210802,61859581,84904436,18466635,68204242,61859143,8696647,50188404,30764367,44983451,56515456,4069912,58224549,38061439,29819940,23110625,59405277,45848907,4515343,58751351,27665211,44245960,29834512,90272749,31161687,86001008,45794415,70782102,28734791,2208785,73031054,58180653,20642888,13862149,90004325,61271144,81853704,73168565,37957788,99333375,86301513,19939935,6986898,64087743,22435353,3773993,38009615,74137926,7955293,33237508,49598724,81898046,65017137,15902805,88047921,16380211,39553046,81855445,61741594,89996536,89811711,54199160,12416768,47361209,30218878,65454636,85695762,26507214,70004753,42782652,33895336,37435892,76315420,32161669,54663246,73124510,96318094,71920426,50702367,9599614,75153252,66271566,6497830,49597667,18411915,82726008,97011160,17857111,33565483,54987042,62936963,77187825,79657802,40534591,65038678,67030811,17058722,89699445,4978543,67474219,36685023,99021067,55615885,86798033,88251446,44664587,98371444,17068582,2607799,76434144,62051033,66045587,98462867,42644903,97940276,77272628,9623492,68000591,69136837,41245325,51464002,8651647,85711894,78589145,52293995,34493392,70727211,75986488,99604946,1569515,36808486,38494874,54058788,321665,22450468,79880247,29029316,749283,34497327,20140249,71333116,20568363,9886593,4787945,34667286,72738685,93790285,78884452,9175338,82327024,14363867,16054533,45996863,68875490,33699435,98739783,4434662,73617245,35092039,48892201,508198,72373496,92998591,55850790,33628349,76703609,82178706,16971929,75229462,76540481,17146629,11365791,31126490,61815223,58749,95726235,26863229,7517032,77284619,569864,28550822,62452034,40356877,98653983,2891150,66250369,72495719,40152546,35996293,73222868,45049308,6153406,24826575,26292919,42307449,4091162,9860195,61982238,55669657,1239555,6038457,60955663,71965942,37739481,27625735,4199704,91990218,24591705,26102057,7687278,72274002,48395186,1250437,30694952,62490109,6525948,33553959,176257,86821229,97057187,92033260,96852321,29065964,12571310,75543508,3633375,92541302,21397057,6793819,18663507,31733363,54232247,44177011,45428665,67124282,51315460,50806615,89046466,77312810,56153345,90013093,22997281,94076128,14045383,65338021,63628376,94360702,24168411,27411561,3509435,70596786,57020481,20645197,89214616,45075471,92071990,59910624,8634541,61263068,30891921,75135448,67031644,77300457,82651278,36312813,92530431,72614359,66903004,17037369,1936762,44627776,19376156,72238278,64157906,55247455,16019925,86460488,19101477,58208470,8055981,55470718,28851716,23569917,24953498,30139692,15064655,50007421,79922758,24733232,37224844,26776922,8807940,71522968,55753905,33797252,66663942,50668599,32699744,37659250,77413012,90090964,12303248,77787724,32250045,5832946,78785507,4064751,26493119,61897582,42947632,14947650,3233569,824872,70420215,47298834,89637706,84406788,15535065,51466049,77301523,99729357,27185644,68316156,47708850,9058407,44473167,92803766,55602660,53547802,41442762,87160386,24058273,8733068,63015256,22113133,99861373,34044787,53084256,2331773,99515901,17081350,34946859,65715134,24915585,91664334,83651211,83747892,84840549,71300104,90310261,7011964,18833224,74724075,96735716,66116458,51507425,10649306,78218845,78561158,11923835,68816413,62232211,85571389,36396314,22942635,95010552,83302115,96906410,30463802,3235882,29932657,16424199,45919976,42199455,3183975,48658605,91240048,13173644,29635537,38510840,12024238,67227442,8913721,37891451,94516935,66319530,76825057,14349098,20885148,40197395,23134715,64098930,67513640,89419466,30090481,51588015,22405120,54606384,91938191,61712234,45790169,30811010,92215320,91141395,61960400,16445503,70372191,68128438,75128745,6111563,37501808,92554120,52204879,36135,93270084,82779622,3294781,83133790,47792865,15015906,69641513,84349107,38341669,79806380,41380093,98648327,32058615,53888755,76330843,53666583,40865610,88904910,65851721,33201905,79136082,57803235,82339363,30771409,44479073,26734892,8128637,20002147,57359924,74357852,89499542,27187213,89804152,99917599,71657078,44784505,1022677,51149804,4806458,92692978,94911072,31491938,20122224,45990383,75820087,80246713,66137019,33249630,47893286,31727379,7066775,86118021,48360198,36933038,71083565,87710366,24314885,3487592,32159704,37675718,68110330,10264691,50567636,78300864,53802686,59197747,65047700,79942022,94595668,33061250,78766358,99125126,92867155,99690194,60176618,112651,63506281,9860968,83533741,62762857,53842979,34295794,21289531,6505939,30653863,21001913,51715482,35512853,18806856,29516592,20681921,4508700,16567550,32426752,415901,81172706,21673260,80014588,56461322,10358899,99971982,17365188,34432810,85117092,68102437,78549759,5267545,26275734,36139942,77694700,11757872,74441448,6871053,74614639,97561745,42692881,61141874,95395112,29401781,42967683,51426311,40027975,73021291,38022834,73392814,29510992,64055960,91281584,62693428,18131876,1204161,85242963,77377183,70800879,4095116,71316369,78909561,10453030,30163921,14220886,84127901,81805959,39201414,69579137,57240218,82532312,9257405,68824981,65880522,65074535,79322415,97281847,26744917,48893685,69255765,20765474,71469330,92787493,16405341,72793444,62428472,59981773,82052050,33336148,96420429,7919588,36468541,16684829,87598888,3880712,81774825,4798568,45617087,70541760,5073754,7685448,52613508,18699206,97379790,90654836,39373729,26063929,6221471,96709982,56484900,86543538,17957593,76170907,21533347,37166608,45381876,37560164,7182642,61728685,92353856,15948937,526217,36189527,99524975,7423788,76671482,88444207,19272365,60102965,60393039,93359396,52060076,44847298,64602895,98943869,57163802,44481640,42237907,79738755,47738185,75407347,55960386,66322959,72732205,45667668,55770687,25842078,37192445,75052463,83368048,16099750,4204661,95251277,67084351,83083131,66832478,70367851,21919959,61928316,17385531,13348726,88130087,57658654,83948335,9398733,45995325,83269727,56531125,20836893,95581843,29466406,15163258,23432750,73109997,3088684,36845587,16595436,69697787,99297164,48088883,74452589,48675329,18001617,38008118,62357986,15075176,23740167,68939068,39847321,49882705,26397786,8129978,6521313,30395570,17386996,27325428,10597197,56955985,41092172,33935899,17976208,38256849,33123618,54868730,30366150,11581181,1375023,90158785,59109400,60106217,84002370,37280276,28787861,32151165,26664538,44060493,54263880,84187166,44915235 +24733232,35192533,76671482,57658654,99658235,6505939,98371444,77620120,7685448,99021067,99333375,35092039,4064751,15536795,45848907,26998766,37659250,4985896,79821897,29466406,44846932,54762643,30764367,94911072,14045383,44915235,29029316,79880247,17365188,37739481,44844121,73031054,49641577,80316608,82886381,62740044,1431742,90272749,57241521,73124510,62430984,39847321,14220886,4099191,57803235,29188588,19939935,16445503,96726697,8651647,59329511,55753905,78589145,15015906,62452034,65074535,508198,4668450,89046466,112651,1204161,70596786,60102965,51047803,83150534,36780454,2717150,29834512,99690194,91831487,51464002,32250045,64098930,80014588,36930650,81172706,16019925,68702632,42199455,59501487,65880522,89419466,64087743,75229462,90090964,71657078,80246713,37675718,59197747,72358170,66250369,88653118,15075176,53802686,51590803,10358899,60581278,50567636,34698428,96852321,74357852,81774825,30395570,36312813,72732205,89214616,8791066,60430369,81853704,69641513,39373729,85242963,43501211,48360198,20002147,28796059,30463802,86301513,11581181,5267545,40356877,57163802,36812683,9575954,62936963,18504197,5970607,13470059,59177718,52060076,7955293,39171895,32151165,54058788,77413012,29994197,34698463,73392814,45996863,21289531,98648327,63967300,29819940,31727379,89699445,78602717,30163921,20885148,83155442,19272365,48675329,83651211,48774913,26392416,33237508,30218878,59318837,15535065,85023028,40781449,32161669,79657802,70727211,57240218,7687278,22450468,6153406,14626618,48893685,56424103,73617245,31126490,74614639,61982238,9257405,22129328,20568363,92803766,78909561,67793644,72777973,30139692,89078848,51507425,67474219,95581843,4199704,3233084,77284619,44784505,61859581,66663942,70036438,61741594,8807940,17016156,3183975,46870723,99965001,8129978,68939068,92554120,4434662,38256849,44550764,15148031,96193415,24619760,415901,65715134,6871053,71920426,31161687,84187166,36753250,76960303,26292919,48088883,40152546,9188443,51149804,1788101,86821229,98948034,99549373,96735716,56531125,669105,8099994,62490109,50806615,83948335,55850790,66137019,66667729,1022677,56461322,6521313,71469330,54868730,4508700,10366309,3509435,5822202,53084256,18663507,53547802,61263068,46540998,12303248,78549759,7423788,61815223,2204165,8733068,49328608,35456853,27041967,4798568,5111370,70004753,72357096,75407347,43045786,90158785,39986008,89637706,45075471,96420429,79942022,84904436,16380211,6808825,97783876,7011964,23134715,21070110,45306070,73222868,98130363,18833224,18783702,99604946,749283,72278539,88047921,32274392,82779622,93562257,14731700,65454636,92215320,63015256,45790169,13231279,93933709,44842615,30366150,17068582,50188404,92998591,23432750,62693428,36580610,62115552,61712234,51426311,33628349,77898274,89217461,60176618,44177011,12348118,20486294,88444207,44664587,33895336,51588015,98739783,55143724,19457589,91727510,76703609,55470718,39801351,60955663,48395186,61623915,9860195,5073754,77312810,86543538,54014062,77301523,52204879,1197320,97561745,76170907,95395112,47738185,59981773,55602660,23740167,48260151,23569917,99524975,67281495,17957593,7104732,68102437,31904591,247198,79136082,73168565,69136837,53648154,77187825,40686254,42692881,70372191,94360702,77300457,10649306,83789391,36505482,29510992,84293052,63059024,53632373,77377183,18131876,44847298,12416768,87710366,59109400,40984766,90654836,59614746,4095116,77272628,69355476,26397786,66045587,9623492,9886593,52613508,40197395,43506672,29516592,60106217,34236719,44481640,44348328,10597197,37183543,40027975,20765474,72274002,55189057,38022834,5832946,91664334,168541,78300864,42307449,176257,51472275,9175338,43357947,44479073,77787724,79322415,56153345,62496012,19376156,78561158,47213183,526217,43152977,54606384,54987042,73786944,37435892,45995325,72725103,72019362,17764950,75135448,29959549,321665,72495719,9599614,75543508,11365791,13862149,12571310,3880712,33797252,26744917,89804152,99917599,24915585,39553046,37957788,91141395,45407418,65271999,66319530,4119747,22405120,6793819,58180653,91802888,35512853,90061527,47298834,52261574,17385531,37501808,49236559,83368048,94711842,87720882,44983451,38494874,13468268,55247455,4091162,32058615,20122224,75777973,69579137,78766358,32699744,22942635,88130087,73235980,22108665,34295794,92787493,98462867,86242799,16405341,3088684,99226875,49598724,23194618,55615885,65275241,28550822,67030811,26114953,9603598,53666583,7182642,4515343,30653863,10309525,48673079,47887470,24168411,7229550,92692283,91938191,44889423,68000591,33061250,17146629,33201905,15902805,75820087,55960386,86001008,2607799,13348726,67451935,80555751,43933006,20867149,84406788,65038678,99224392,36845587,76434144,25842078,69848388,76315420,16097038,30771409,93057697,82726008,71333116,19891772,20645197,17857111,36808486,30787683,61960400,88251446,84840549,3235882,78884452,37891451,21673260,83210802,4069912,92692978,7919588,45428665,93515664,11161731,54517921,26863229,68644627,41380093,27665211,24826575,3294781,28734791,95290172,6525948,33699435,87160386,50668599,70420215,1250437,99971982,34044787,97940276,66885828,91990218,82178706,77072625,92604458,22721500,6497830,27411561,14093520,93790285,8128637,15163258,72373496,74110882,8696647,28787861,569864,68824981,4204661,69255765,3233569,93053405,874791,32590267,17058722,71965942,83269727,77694700,95957797,34432810,66832478,17539625,43376279,26664538,26507214,97011160,79191827,6038457,13173644,75153252,42073124,10453030,6819644,82427263,26776922,27185644,97641116,18001617,69786756,14947650,81898046,17976208,37280276,21533347,66611759,50702367,68041839,80251430,67227442,29635537,81677380,31733363,9829782,22879907,67084351,75986488,3487592,74452589,76540481,42967683,18411915,66271566,45381876,26734892,42947632,90457870,39201414,90013093,41245325,67031644,23110625,70800879,85695762,78218845,33123618,83133790,30543215,22997281,18699206,92530431,92541302,68128438,58749,49882705,17081350,16099750,22113133,16595436,38341669,59910624,75128745,38365584,8913721,97057187,92398073,28851716,66116458,10264691,26139110,61271144,34946859,79738755,17894977,8634541,33304202,56515456,6111563,93270084,92071990,89499542,52293995,71083565,89811711,12024238,14326617,68875490,4806458,91957544,95251277,58224549,14723410,34493392,21993752,40677414,29065964,30569392,24058273,29932657,9860968,19101477,30811010,48892201,72738685,51315460,1239555,45919976,66903004,64602895,81274566,64157906,62357986,63628376,86118021,74724075,44245960,16054533,99297164,47090124,7300047,34667286,99729357,62803858,824872,70541760,94516935,85117092,41092172,4787945,62762857,57020481,26063929,75052463,47893286,98943869,98653983,79806380,25636669,7646095,54199160,44627776,84684495,84349107,27625735,68316156,72614359,79922758,35996293,22435353,1569515,99125126,36189527,41092102,88904910,89996536,31491938,63152504,87598888,70367851,93566986,21001913,33431960,56473732,81805959,97379790,94539824,38008118,53393358,2891150,84127901,69697787,16387575,38061439,82339363,33565483,61859143,38645117,74441448,20681921,65851721,86022504,82532312,45617087,33249630,42237907,76825057,26493119,40781854,38009615,11923835,20642888,56484900,8373289,94076128,36135,36685023,22200378,82979980,88698958,99515901,47792865,71522968,36468541,62051033,6157724,38510840,66428795,54663246,65017137,83747892,37560164,71316369,45667668,99861373,95726235,58751351,30694952,12664567,13819358,16424199,61373987,83083131,20836893,17037369,92353856,5482538,67513640,74743862,33935899,65047700,5640302,23848565,36396314,34497327,85711894,6986898,55770687,84002370,82327024,59405277,2331773,60393039,82651278,48658605,76330843,16971929,91255408,54427233,2917920,27187213,21919959,55669657,20140249,96906410,19486173,72793444,86460488,47361209,19898053,42644903,45794415,46851987,8729146,14349098,1936762,49271185,91240048,61728685,85116755,32159704,53888755,47708850,38658347,27325428,32426752,51715482,70782102,9398733,92033260,2208785,16684829,24953498,11543098,14363867,43322743,26102057,63372756,57393458,51466049,96318094,82052050,95010552,37192445,58208470,30891921,53842979,7066775,66322959,64848072,22766820,17386996,63506281,42782652,56955985,94090109,49597667,96709982,3633375,40865610,8055981,45049308,57248122,73021291,54263880,59371804,4978543,6221471,91281584,8115266,54232247,68204242,41442762,7502255,86798033,1375023,28928175,45237957,61928316,57359924,41481685,61141874,83533741,62428472,71300104,74137926,9259676,37166608,36139942,30090481,18466635,18806856,15948937,68816413,78785507,69605283,82897371,37620363,29401781,26275734,30998561,44473167,81855445,92867155,90004325,9058407,15064655,83302115,33336148,67124282,68694897,50007421,45990383,3773993,40534591,65081429,97281847,65338021,17727650,61897582,24591705,73109997,85571389,68110330,72238278,64055960,7517032,84166196,44060493,94595668,62552524,36933038,37224844,90310261,62232211,93359396,57961282,33553959,16567550,11757872,24314885,21397057,10961421 +669105,67281495,75153252,49236559,30139692,57803235,31904591,26507214,44784505,29188588,9886593,6986898,30543215,4099191,76434144,29029316,65715134,76960303,70596786,38494874,63967300,50806615,66832478,68644627,15535065,10309525,26863229,80014588,72274002,29932657,11365791,45407418,89214616,45794415,96726697,77312810,62430984,6038457,33553959,34044787,29834512,92554120,24058273,66667729,68204242,62452034,52261574,48360198,37620363,34236719,23848565,48673079,93933709,91664334,52293995,4515343,81853704,35192533,61263068,97281847,65038678,7104732,55470718,55753905,45237957,5970607,90272749,7687278,34497327,87720882,168541,4119747,14220886,62552524,44177011,47213183,29959549,62936963,50188404,415901,50007421,40197395,4787945,64602895,74743862,4091162,53648154,86001008,94539824,27665211,3294781,51047803,12664567,31126490,85711894,38658347,66250369,8128637,35092039,58224549,38645117,83651211,43506672,44889423,15015906,6221471,61373987,44983451,61728685,2917920,24953498,64848072,55615885,14045383,88047921,66045587,92071990,96318094,66611759,68000591,19939935,54427233,33895336,37183543,59318837,96193415,526217,34698428,1250437,44473167,91990218,33431960,749283,1569515,6153406,62051033,16099750,60430369,34295794,59177718,1204161,53084256,84349107,40356877,30891921,34946859,92033260,59197747,72357096,61982238,36312813,86022504,78766358,98130363,90090964,55247455,37675718,8807940,16595436,15064655,32426752,28550822,32151165,20486294,18699206,66428795,77301523,81898046,57240218,99524975,30163921,57241521,63059024,92353856,6808825,61741594,57393458,39373729,89699445,74357852,29819940,90457870,16424199,55189057,75229462,79821897,14731700,4798568,85116755,99549373,83083131,72019362,74441448,69848388,79922758,65338021,98371444,94516935,91802888,57359924,95726235,94911072,61712234,94090109,76315420,55770687,8696647,80316608,8055981,40984766,44550764,1022677,81677380,47090124,65017137,9829782,93359396,11923835,26776922,77187825,40781449,3088684,32161669,4064751,65851721,69641513,44915235,69255765,6525948,70004753,48675329,99515901,73222868,62428472,72738685,30694952,96906410,75543508,32159704,54199160,22113133,65271999,34493392,84127901,37739481,58751351,53547802,7646095,24591705,61623915,92530431,49328608,321665,82726008,82427263,35456853,21993752,37192445,99658235,20140249,99297164,48892201,8634541,62496012,17857111,99965001,54232247,93053405,60102965,70367851,60581278,84684495,91938191,77272628,17068582,68702632,82339363,9575954,19457589,84904436,75777973,72777973,44348328,30395570,33336148,40865610,42692881,34432810,30463802,54014062,5111370,48260151,81855445,33249630,92398073,88444207,26102057,56515456,99021067,42782652,17037369,8729146,71333116,54517921,69355476,26392416,26063929,78602717,5832946,4668450,61141874,78549759,73786944,22942635,38365584,98948034,17365188,54987042,48774913,88653118,93562257,10649306,4806458,22766820,83789391,4204661,19486173,80246713,89637706,78909561,96709982,73031054,82886381,44479073,98462867,14947650,1431742,67793644,44481640,36930650,92998591,96735716,89811711,45848907,42073124,1197320,247198,83533741,79942022,99125126,77620120,27185644,89804152,48893685,26292919,26493119,9599614,15148031,49598724,14626618,66116458,27411561,569864,53666583,75407347,12416768,46870723,30998561,91727510,9175338,66137019,5482538,38061439,26998766,43322743,32590267,42199455,79191827,1239555,49271185,176257,89499542,32699744,18504197,30569392,65454636,50668599,83302115,29466406,99861373,46851987,12571310,56424103,45075471,78589145,76330843,8651647,43045786,83210802,79136082,13231279,24619760,92787493,36505482,6111563,70372191,33628349,68816413,22997281,36812683,9623492,24826575,38341669,36933038,95957797,26139110,90654836,17058722,77413012,1936762,71083565,97641116,30653863,63015256,68875490,15948937,77787724,45428665,68694897,42967683,9398733,36685023,12024238,70800879,22200378,97561745,81172706,40027975,16019925,16971929,17386996,62740044,74614639,65275241,79806380,70036438,37435892,74110882,85242963,99226875,824872,56461322,94711842,90004325,36753250,74724075,7685448,39801351,10453030,8373289,2891150,43376279,99604946,11543098,77072625,15902805,75986488,92692978,77284619,16380211,17539625,47887470,30764367,90310261,86301513,80251430,31733363,16097038,1375023,65047700,3235882,20002147,29510992,62693428,4434662,75135448,39171895,45919976,87710366,72725103,48088883,14093520,6157724,44842615,53888755,71920426,36808486,22405120,20836893,45306070,89217461,51466049,36139942,98739783,9603598,61859581,38008118,26734892,9860195,6871053,3233084,18466635,79738755,69786756,71469330,79880247,9259676,40686254,1788101,55960386,17727650,40534591,22435353,74137926,65074535,6793819,47738185,72614359,97379790,83269727,77300457,72238278,19101477,23194618,68102437,7919588,99917599,20885148,9188443,73617245,37166608,99690194,51472275,29065964,66271566,73124510,67030811,73235980,33304202,89046466,81774825,7300047,55143724,20867149,45667668,7955293,3880712,55602660,43501211,72373496,91957544,71657078,24733232,58749,66322959,94360702,63372756,13468268,41442762,18806856,59371804,52060076,14326617,27187213,70541760,4069912,61859143,76170907,6505939,21673260,92215320,59614746,508198,41092172,38009615,59501487,31161687,7502255,21070110,23569917,36580610,37224844,18783702,39553046,33797252,72278539,82651278,97011160,67031644,32250045,24168411,2717150,84293052,62490109,12348118,62232211,28734791,97783876,45381876,91141395,11581181,17385531,17016156,19891772,52204879,67474219,2208785,16445503,86543538,17976208,68316156,82052050,112651,8115266,29635537,21533347,63152504,67084351,26114953,45790169,69136837,51590803,15536795,30366150,2204165,85023028,40152546,64087743,84166196,21919959,25842078,80555751,51715482,19376156,8099994,45990383,92604458,67451935,37501808,20765474,40677414,76540481,93057697,39847321,47792865,23432750,82327024,95290172,86821229,41481685,43152977,59910624,27625735,78785507,29401781,28928175,37957788,88130087,44060493,50702367,44847298,73021291,15163258,66319530,51588015,30218878,43933006,98648327,73168565,78218845,99333375,39986008,90013093,87598888,50567636,22721500,22129328,61960400,70782102,41380093,37659250,23134715,51426311,71965942,46540998,7517032,44846932,52613508,96420429,81274566,75128745,13348726,3633375,14723410,87160386,7229550,29994197,68041839,19898053,95251277,82979980,40781854,61897582,17957593,30771409,13470059,34698463,51464002,75052463,23110625,3487592,91831487,3509435,53842979,54663246,51507425,38510840,76671482,33935899,20122224,71316369,94076128,92867155,45617087,9257405,95581843,84187166,7011964,88251446,36845587,84002370,42307449,56955985,2607799,4508700,57961282,92541302,16054533,84406788,78884452,57658654,69697787,86460488,95010552,6497830,88698958,41245325,56531125,9058407,17894977,33565483,78561158,36189527,67124282,82897371,98653983,47708850,42947632,26397786,44627776,10366309,68939068,68128438,43357947,58180653,97940276,60106217,93515664,8129978,66663942,6819644,48395186,33123618,20681921,78300864,25636669,66903004,45995325,97057187,28787861,54058788,10358899,59109400,79657802,53632373,83150534,77694700,47361209,28796059,89419466,21289531,26744917,66885828,49597667,93270084,26664538,86798033,73392814,14363867,95395112,874791,54606384,36468541,20645197,74452589,51149804,33061250,82532312,36780454,53802686,99971982,20568363,18411915,47298834,37560164,36135,4985896,57020481,13862149,49641577,98943869,8913721,33237508,83155442,9860968,68824981,22450468,63506281,93566986,60955663,26275734,72358170,62115552,17081350,69579137,32274392,5822202,13173644,45996863,68110330,85571389,42644903,63628376,94595668,72495719,64157906,70727211,85695762,39201414,31727379,4978543,55850790,56473732,37891451,5267545,56153345,77377183,83133790,55669657,91255408,91281584,92692283,61928316,8791066,64098930,93790285,88904910,49882705,70420215,48658605,30787683,76703609,4199704,61271144,21001913,41092102,57163802,91240048,35512853,53393358,86118021,38256849,3233569,83747892,33699435,54263880,84840549,51315460,83368048,33201905,29516592,90061527,37280276,12303248,89078848,10961421,5640302,23740167,62762857,82779622,57248122,16387575,36396314,2331773,30811010,72732205,15075176,11161731,79322415,18131876,3183975,5073754,60393039,22108665,19272365,8733068,75820087,67227442,32058615,16567550,17764950,81805959,59329511,24314885,71300104,11757872,16684829,73109997,30090481,35996293,64055960,24915585,44245960,65880522,96852321,54868730,7423788,69605283,14349098,76825057,60176618,59405277,62357986,18833224,10264691,28851716,82178706,72793444,18001617,16405341,85117092,27041967,17146629,42237907,3773993,71522968,77898274,92803766,7182642,59981773,67513640,38022834,4095116,99224392,62803858,31491938,83948335,86242799,21397057,20642888,56484900,89996536,58208470,7066775,27325428,6521313,13819358,65081429,22879907,61815223,34667286,47893286,10597197,44664587,54762643,18663507,45049308,44844121,90158785,99729357 +47090124,71300104,39171895,12348118,8807940,27665211,97783876,4099191,98739783,44846932,51047803,65715134,26063929,67281495,29959549,16445503,26392416,508198,81274566,1431742,98462867,6819644,85023028,96193415,4787945,96726697,6153406,80251430,61263068,569864,34698428,56473732,78549759,43933006,1204161,45995325,56461322,35092039,37957788,86821229,17016156,21289531,68644627,8696647,96420429,66137019,3509435,77312810,5832946,92692283,77620120,51426311,47708850,61859143,168541,55753905,70004753,33304202,7919588,43501211,6793819,18699206,57163802,69641513,32250045,78589145,57240218,19939935,98130363,89078848,31491938,80316608,95290172,46870723,88653118,54058788,69355476,36580610,85242963,112651,22997281,17764950,35192533,75543508,89046466,76671482,83302115,4806458,20486294,38494874,21993752,74743862,67793644,44348328,9623492,40865610,44844121,43376279,7502255,4798568,12024238,15535065,29834512,43322743,57803235,3233569,34432810,20122224,20765474,42947632,55770687,5970607,62430984,74724075,14093520,46851987,76703609,29819940,62936963,7517032,13348726,99297164,24058273,36930650,321665,29994197,16387575,3294781,45996863,65074535,29510992,33553959,60955663,24619760,31161687,26734892,50702367,83210802,9398733,36780454,26664538,6505939,95395112,91240048,88047921,96318094,60581278,92215320,9860968,26292919,33431960,90004325,68702632,90013093,32161669,83083131,28851716,12416768,59371804,63628376,99333375,49641577,62452034,69605283,99549373,37659250,33797252,68102437,56424103,55143724,48088883,53084256,4119747,19486173,83789391,31733363,4069912,74357852,7423788,37620363,44245960,97641116,40534591,94911072,71657078,40152546,62740044,93562257,9603598,61623915,14045383,70541760,8099994,49597667,30395570,69136837,10358899,29029316,76540481,29065964,99604946,99861373,58208470,22450468,30569392,92398073,36505482,51466049,20645197,40356877,79922758,38061439,75229462,72738685,54762643,36753250,85711894,24733232,81805959,11923835,87710366,34493392,60430369,77377183,70800879,16097038,84349107,19101477,65038678,48360198,14349098,61728685,99965001,71920426,11161731,54199160,15015906,4095116,11581181,51588015,65271999,25842078,6986898,73124510,92554120,45237957,9886593,79821897,74110882,22108665,29932657,70372191,94595668,78561158,74441448,66903004,84293052,18783702,23194618,73617245,86301513,33895336,84187166,92803766,4199704,8651647,10597197,59910624,26493119,83150534,4985896,16595436,55960386,82886381,14326617,64087743,69579137,68204242,45407418,39201414,62490109,22435353,40686254,91802888,34497327,2917920,82052050,33123618,30764367,41380093,37435892,66250369,59501487,65047700,47213183,16019925,57961282,42967683,93933709,67451935,36468541,72777973,31126490,65338021,85117092,15948937,98648327,81898046,20867149,874791,32590267,76330843,38365584,72238278,99021067,70036438,37675718,9599614,8634541,91281584,53648154,52261574,33249630,89214616,26102057,45667668,30771409,51464002,50188404,30463802,87160386,21673260,17539625,15075176,61982238,20002147,66611759,6808825,40197395,83651211,89811711,65017137,54987042,53632373,55602660,98371444,19376156,83533741,72373496,81853704,61815223,29188588,4434662,89637706,28796059,60106217,24953498,45617087,95726235,73392814,72495719,16424199,10366309,59109400,43045786,53842979,78766358,669105,63967300,50567636,31904591,99226875,79880247,41481685,66319530,18833224,34044787,81855445,93270084,18131876,19272365,95957797,26776922,8373289,44889423,6871053,1022677,1197320,52060076,79191827,30218878,68824981,68939068,21070110,54014062,29466406,45790169,40781854,80555751,73168565,48774913,5267545,18806856,6497830,16971929,77898274,59197747,61960400,415901,49328608,44842615,32159704,62693428,64157906,51715482,79136082,28550822,2717150,41092102,4978543,15902805,32274392,17146629,99224392,7011964,14731700,23432750,47738185,30090481,36312813,88130087,14626618,30787683,84002370,49236559,64098930,47298834,97561745,59177718,61141874,35512853,66885828,40984766,56515456,92692978,3880712,82897371,51472275,23569917,86460488,88444207,86001008,84684495,76170907,82327024,44481640,55470718,37739481,1239555,38008118,73031054,71316369,33628349,90457870,54427233,79942022,97940276,24591705,8115266,78909561,68875490,82726008,61897582,50668599,42644903,72019362,78785507,96735716,58180653,61271144,7300047,48673079,15064655,13862149,53547802,5111370,62803858,75820087,17976208,99658235,78218845,93053405,3773993,98948034,65275241,93566986,37224844,8791066,40781449,37183543,57248122,43506672,27411561,92353856,95251277,47361209,526217,30139692,26114953,12571310,67084351,28787861,65081429,39986008,18663507,63015256,33201905,58749,83368048,44983451,39847321,16054533,96906410,44479073,76960303,13231279,67227442,20885148,39373729,247198,8129978,91727510,7104732,67124282,81172706,38658347,8913721,55615885,17727650,66428795,77187825,90654836,79738755,75986488,57020481,37192445,7955293,62232211,39553046,59318837,69848388,26744917,83269727,4064751,11365791,28928175,10961421,60102965,30653863,4091162,66271566,77072625,21533347,36685023,67513640,52293995,40677414,16380211,47792865,32699744,45075471,89804152,9188443,66322959,72725103,79657802,13173644,14723410,3487592,36189527,13468268,17058722,41245325,97011160,65851721,55247455,5482538,82779622,53802686,62115552,9575954,2891150,26139110,19457589,44550764,13470059,33699435,92033260,2204165,66832478,85116755,33237508,71333116,59329511,45919976,81677380,56531125,63059024,77301523,20681921,72732205,75407347,8733068,19891772,90272749,58751351,79806380,9259676,57393458,36812683,35996293,21001913,15163258,4515343,94516935,17957593,84840549,75052463,92071990,70596786,78602717,93790285,37166608,57359924,54663246,48395186,17068582,3183975,9257405,73235980,26998766,14947650,7646095,19898053,5640302,77272628,68000591,92541302,28734791,55189057,22942635,3235882,42237907,22879907,33565483,96852321,30163921,17365188,67031644,38510840,82178706,21919959,65454636,77284619,20642888,40027975,27625735,31727379,54606384,77300457,80246713,56484900,42199455,59405277,14363867,82532312,27041967,72274002,35456853,70367851,23740167,45428665,7229550,84406788,3088684,18411915,68816413,63152504,93057697,48658605,61373987,72793444,23848565,73786944,20836893,63372756,30891921,41092172,57241521,52204879,54232247,30998561,43152977,68128438,27185644,70727211,77787724,94711842,18001617,20140249,94076128,79322415,36808486,65880522,49598724,34236719,10309525,80014588,42073124,89419466,824872,16099750,4668450,7687278,23134715,38022834,94539824,88251446,5822202,48260151,82339363,7182642,54517921,30366150,38256849,91957544,33336148,44177011,64602895,48892201,3633375,99125126,74137926,66045587,47887470,69697787,89217461,99515901,75153252,99690194,36139942,26275734,3233084,6157724,50806615,87598888,43357947,86798033,82651278,83948335,90310261,39801351,89499542,24314885,34698463,42307449,27325428,44473167,30694952,53888755,8128637,30543215,73222868,67474219,24168411,90090964,60176618,57658654,44847298,53666583,86242799,10649306,86543538,92998591,72357096,37891451,59614746,9175338,72614359,4204661,27187213,85695762,90158785,38645117,62428472,75777973,62496012,82979980,2208785,45794415,67030811,91938191,16684829,56153345,1250437,36135,36845587,71522968,15536795,69786756,64848072,45306070,15148031,29401781,66663942,93515664,37560164,70420215,98653983,95581843,53393358,81774825,22129328,8055981,24826575,51149804,94090109,77413012,68694897,72278539,49271185,29516592,75135448,91664334,84127901,7685448,61859581,51590803,42782652,97281847,54868730,38009615,94360702,86022504,23110625,82427263,92604458,9860195,95010552,59981773,18466635,20568363,99524975,6111563,12303248,22405120,56955985,73021291,12664567,90061527,71469330,91990218,51507425,33061250,76434144,25636669,8729146,84904436,91255408,92867155,18504197,1788101,44915235,749283,83155442,75128745,37280276,17894977,17385531,6221471,45848907,49882705,11543098,60393039,74614639,11757872,88904910,33935899,61712234,2607799,62051033,45381876,22766820,16567550,71083565,70782102,99917599,68316156,44664587,63506281,29635537,52613508,10264691,6521313,17386996,17081350,66667729,55669657,78884452,14220886,16405341,73109997,38341669,48893685,44627776,1569515,44784505,78300864,32426752,87720882,46540998,85571389,44060493,64055960,34667286,9829782,26863229,98943869,92530431,96709982,32151165,32058615,77694700,91831487,34295794,72358170,22200378,13819358,26507214,89996536,68041839,97057187,86118021,62357986,74452589,76315420,93359396,7066775,99971982,92787493,30811010,51315460,37501808,99729357,55850790,22721500,9058407,88698958,71965942,48675329,4508700,6525948,66116458,1936762,176257,50007421,5073754,36933038,24915585,61741594,61928316,54263880,10453030,41442762,62762857,17037369,45990383,1375023,83133790,17857111,22113133,62552524,58224549,6038457,91141395,68110330,84166196,97379790,45049308,34946859,69255765,42692881,89699445,83747892,21397057,36396314,76825057,47893286,2331773,26397786 +66667729,72373496,2917920,69136837,14731700,4668450,22879907,35192533,52204879,61373987,93933709,1197320,93053405,18411915,66250369,59329511,14723410,36505482,12664567,62496012,66611759,97783876,33304202,34946859,83210802,76330843,99549373,45794415,526217,30543215,41092102,69355476,64157906,247198,1788101,48893685,72777973,37620363,81853704,40152546,24591705,95395112,58180653,112651,73786944,61859581,91240048,55770687,30891921,18783702,85242963,54232247,55189057,22129328,79821897,36780454,26292919,17386996,29029316,68824981,51047803,19272365,57248122,58751351,99917599,69641513,29834512,63372756,68102437,95010552,49271185,54606384,40984766,84684495,90061527,63059024,44177011,8055981,29188588,61960400,77312810,35456853,36933038,60430369,98943869,40027975,52613508,82897371,6808825,20486294,44842615,40781449,26776922,42692881,81898046,17894977,49641577,5267545,30463802,19891772,26139110,78785507,72278539,54427233,70004753,30998561,73031054,50702367,26744917,22200378,4508700,1204161,7011964,76315420,25842078,44983451,66137019,59501487,17146629,92033260,51472275,67281495,48774913,34698428,3294781,15535065,48360198,48892201,39553046,50007421,18504197,54199160,39986008,53648154,8099994,59197747,13470059,55753905,88698958,26664538,53632373,34044787,77072625,63015256,8129978,92692283,87598888,38061439,96193415,99658235,95290172,66319530,19376156,99515901,78884452,43376279,75777973,46870723,22997281,44481640,49598724,7300047,73124510,33237508,61271144,7955293,10358899,14326617,4099191,65074535,59910624,66045587,62430984,55247455,176257,75128745,68694897,51464002,61263068,62803858,68316156,34295794,874791,15902805,95957797,65038678,6038457,68000591,62490109,65851721,92398073,2204165,98130363,34493392,4787945,56461322,36312813,65338021,43501211,61741594,84293052,9886593,44348328,35512853,90013093,3509435,40534591,415901,62452034,57241521,96726697,35092039,99965001,45790169,93566986,45381876,60106217,89078848,47708850,60102965,29994197,23569917,83651211,23194618,55470718,55850790,1375023,37501808,91664334,3233084,32426752,61712234,98948034,36685023,56515456,168541,21070110,68644627,1431742,33699435,7646095,30218878,3088684,5111370,21919959,10366309,64602895,55669657,669105,5832946,65271999,321665,59318837,17976208,59614746,72738685,19898053,61815223,4978543,43152977,24915585,65715134,54517921,79922758,72274002,97379790,17957593,91990218,29819940,26392416,38645117,7423788,14626618,92787493,41442762,78766358,47792865,4064751,62115552,86460488,44889423,95581843,50567636,30569392,43933006,67451935,30771409,13862149,77787724,83269727,86301513,7919588,77620120,50188404,82427263,83150534,18663507,68204242,11365791,54663246,99604946,94076128,58224549,67793644,8729146,36139942,53666583,9599614,47213183,64087743,30366150,98371444,54868730,74357852,86821229,8634541,33797252,31126490,47361209,29065964,48675329,60176618,41245325,38658347,72357096,44844121,9829782,79191827,24058273,12303248,22405120,80251430,3235882,26863229,53084256,32250045,88047921,96735716,98462867,70800879,82178706,91727510,15064655,40686254,89217461,22113133,10597197,98739783,31904591,16445503,99690194,86001008,82651278,42644903,53802686,22450468,5822202,17081350,96318094,84840549,77694700,14947650,30139692,19101477,44473167,25636669,73392814,58749,29959549,93790285,99524975,64098930,57240218,24826575,8651647,52261574,73021291,45667668,9259676,47893286,96420429,51149804,70372191,72725103,17764950,97641116,69786756,98648327,81677380,70036438,75229462,62051033,92692978,62740044,70596786,90272749,42073124,27665211,72614359,85117092,8733068,74743862,60955663,10961421,83302115,16405341,33628349,99297164,61141874,82339363,6153406,16424199,15148031,91141395,42947632,39801351,80246713,51588015,40677414,85116755,77898274,67227442,63628376,24168411,66832478,78602717,42307449,44784505,33249630,2331773,75986488,10649306,62552524,61982238,97561745,99226875,99333375,11757872,65275241,12024238,45617087,29466406,23848565,57020481,72793444,33061250,36930650,19939935,5482538,45237957,62232211,85711894,51590803,54058788,49328608,81774825,90090964,42967683,9175338,26397786,17016156,44664587,55143724,37183543,24733232,33431960,64848072,94539824,86022504,44245960,45075471,94090109,89419466,17385531,20122224,34667286,43506672,15536795,80316608,56955985,67513640,3773993,22766820,94360702,93057697,74724075,81855445,36808486,749283,59371804,78909561,18131876,62428472,43322743,9623492,69848388,66428795,26275734,76825057,68939068,17365188,26998766,91938191,17727650,88904910,92071990,80014588,67084351,59109400,8807940,91831487,90310261,13231279,69255765,93270084,76703609,9603598,3880712,16684829,39171895,86242799,22435353,37891451,77284619,86543538,28796059,47090124,76434144,99971982,55960386,44479073,92604458,18699206,88653118,73222868,48673079,65017137,30764367,68041839,68110330,37224844,82886381,5640302,70727211,9575954,47887470,73168565,71657078,22108665,43045786,37659250,4806458,57658654,31727379,85571389,34698463,84187166,44627776,56531125,91281584,16099750,7687278,78549759,77187825,63967300,56473732,85695762,2717150,55615885,70420215,89214616,44847298,33565483,92541302,84349107,20568363,86118021,8373289,68816413,99861373,14363867,45306070,80555751,48260151,48395186,88251446,2208785,49882705,40197395,71469330,77300457,39847321,71083565,99021067,54263880,95726235,7517032,84166196,77377183,16595436,36189527,45428665,33895336,84904436,32274392,36845587,15948937,29510992,75135448,78589145,27411561,26063929,17539625,81274566,90004325,59981773,4515343,34236719,6986898,20885148,62693428,23110625,71522968,71333116,76540481,32699744,75153252,93515664,1936762,54762643,90654836,12416768,44915235,72019362,68702632,61859143,93359396,82979980,38494874,29516592,24953498,9257405,97940276,17068582,41481685,4069912,60581278,1250437,83368048,48658605,6871053,3183975,28928175,83747892,30694952,54014062,78300864,36753250,92554120,12348118,569864,35996293,13173644,53842979,77301523,5073754,62936963,28550822,21993752,63506281,30090481,20140249,74137926,12571310,46851987,62762857,14220886,42782652,33336148,13468268,79657802,96906410,93562257,21673260,56484900,11161731,70367851,16971929,22721500,71316369,16054533,66885828,72732205,94516935,37675718,38008118,72238278,67474219,17037369,42199455,27041967,16567550,31161687,67031644,99729357,50806615,66322959,33201905,92803766,4798568,3233569,90457870,40781854,4091162,66903004,37435892,57163802,4095116,67030811,32058615,5970607,89811711,89637706,6157724,36396314,38365584,37739481,76960303,76170907,53393358,36812683,8791066,20002147,82726008,27185644,45049308,75820087,10309525,78218845,6497830,22942635,97057187,38022834,72495719,36135,73235980,1022677,46540998,10453030,4985896,20642888,57961282,56153345,53547802,11923835,88444207,49236559,45996863,14093520,97011160,20681921,26734892,16380211,20645197,24619760,6793819,53888755,16097038,30395570,18833224,36580610,7182642,6525948,91802888,39373729,38510840,52293995,73617245,28787861,23134715,61728685,6111563,26493119,34497327,87710366,82779622,51315460,91957544,39201414,9860195,84002370,47738185,55602660,29635537,82532312,8128637,99224392,58208470,15015906,8913721,37192445,87160386,8115266,79136082,42237907,99125126,75543508,94911072,32161669,59177718,1569515,6819644,45407418,34432810,71965942,83948335,28734791,72358170,11543098,30653863,14045383,45919976,68128438,18466635,83083131,66116458,27325428,6505939,38341669,4204661,32590267,79880247,38009615,40865610,13819358,84406788,57359924,23432750,7066775,74614639,18806856,20867149,26507214,98653983,96709982,38256849,33123618,4119747,74441448,45995325,57803235,9058407,28851716,78561158,74110882,81805959,51507425,23740167,61623915,81172706,88130087,50668599,52060076,6221471,7502255,19486173,49597667,65454636,824872,16019925,69579137,21001913,65880522,44550764,60393039,17058722,92530431,27187213,26102057,66663942,9188443,48088883,89046466,17857111,51715482,79738755,65047700,8696647,59405277,77413012,24314885,31733363,26114953,44060493,86798033,75052463,20836893,20765474,83155442,62357986,14349098,83533741,75407347,71920426,51426311,10264691,83133790,87720882,9860968,30163921,21397057,41380093,54987042,77272628,19457589,2607799,27625735,73109997,89499542,41092172,15163258,30811010,71300104,2891150,94595668,15075176,4199704,37280276,3633375,7104732,7685448,4434662,95251277,3487592,21533347,89996536,44846932,47298834,13348726,40356877,32151165,45848907,65081429,57393458,32159704,21289531,29401781,16387575,508198,69697787,79942022,33553959,92353856,11581181,79806380,51466049,56424103,84127901,91255408,9398733,94711842,31491938,6521313,74452589,29932657,45990383,96852321,82052050,61928316,70541760,92215320,68875490,37166608,79322415,37957788,18001617,43357947,83789391,7229550,1239555,85023028,69605283,92998591,66271566,70782102,67124282,61897582,63152504,37560164,90158785,64055960,76671482,89804152,82327024,33935899,36468541,30787683,92867155,97281847,89699445 +81855445,24591705,98739783,36753250,45428665,97281847,8733068,9058407,37739481,68204242,30569392,10366309,72274002,93562257,6793819,42073124,8696647,98948034,92398073,30771409,42782652,50702367,11161731,61728685,16971929,49271185,18663507,93933709,66322959,19939935,66045587,89214616,56473732,34295794,33895336,86001008,27625735,93053405,85116755,62496012,20486294,34698463,89699445,19486173,95290172,38061439,18806856,88653118,16445503,9623492,54427233,96193415,73168565,35996293,68816413,33553959,52293995,91664334,83210802,65715134,99604946,45995325,36312813,57248122,72777973,3880712,15535065,4119747,1788101,16380211,61859143,73031054,2208785,17539625,8807940,63967300,6497830,2917920,40027975,72357096,63506281,77620120,51472275,39553046,4099191,67281495,15064655,62552524,71657078,22942635,40781449,11365791,31904591,54014062,29188588,40686254,37620363,92033260,70727211,54663246,87720882,61897582,70004753,71920426,14349098,61263068,59501487,37183543,9398733,62452034,30543215,36685023,20765474,75135448,98371444,16097038,62740044,26392416,53666583,3088684,20140249,18833224,70372191,62430984,45617087,68041839,18783702,70036438,38009615,19891772,30366150,50188404,8115266,7955293,60102965,24619760,9599614,59910624,98648327,77272628,88047921,43501211,36812683,12348118,9829782,47298834,67030811,90013093,23134715,33201905,55247455,95395112,22721500,51047803,44842615,14947650,78766358,61271144,30998561,92867155,54987042,55960386,17764950,77284619,39171895,73786944,30891921,2331773,40677414,97379790,89804152,74452589,44889423,28796059,95010552,79738755,24826575,34493392,80316608,62803858,76434144,9603598,17037369,43933006,88904910,65851721,47090124,3233569,2717150,526217,29959549,53648154,41442762,26776922,76540481,81853704,73392814,87160386,55602660,5111370,47708850,36505482,26493119,3487592,29819940,11923835,82979980,86460488,99861373,35192533,89078848,67513640,44177011,26863229,48260151,96318094,72358170,49882705,53547802,59197747,75153252,69786756,2607799,14093520,29932657,41092102,27665211,22405120,99917599,99658235,47887470,67124282,33431960,15948937,18466635,97783876,62693428,68644627,75543508,32590267,3773993,49641577,24058273,57961282,47361209,59109400,112651,73222868,48360198,64098930,84840549,6505939,42692881,2204165,4515343,56424103,89046466,6111563,91802888,29994197,99515901,62936963,33304202,58180653,3235882,92353856,7502255,54762643,32426752,69848388,94595668,78561158,9886593,96726697,99549373,55770687,69255765,38645117,71083565,79136082,38365584,85023028,99333375,72793444,63152504,82897371,247198,99125126,37166608,83155442,16099750,36930650,57803235,27185644,76330843,508198,46870723,65338021,27411561,13470059,49236559,55189057,55753905,22450468,51466049,26664538,38658347,42307449,4668450,34044787,37501808,9259676,83789391,1239555,17058722,45306070,61960400,5822202,56955985,1204161,26397786,80555751,4199704,8651647,37675718,39986008,73617245,61859581,37957788,71522968,90004325,14326617,90457870,36468541,1375023,26507214,61815223,51507425,40356877,41245325,95726235,37435892,51590803,55143724,17016156,96709982,33628349,43322743,30653863,83368048,53842979,56461322,4787945,176257,27325428,43045786,17068582,8099994,22435353,99729357,90090964,29834512,92215320,57393458,9860968,67451935,98130363,10649306,5482538,78218845,94090109,71469330,13468268,6153406,65271999,66903004,32699744,71333116,70367851,84187166,38022834,17727650,72725103,49597667,24314885,66137019,30764367,74110882,66663942,45667668,47792865,18001617,77187825,40984766,34698428,23569917,21533347,82178706,77312810,16054533,24733232,36780454,26102057,92998591,8055981,77787724,67474219,26734892,62490109,94539824,68128438,82886381,15163258,6819644,6521313,8634541,16405341,96852321,77694700,10961421,62115552,75052463,40781854,7423788,68824981,64087743,72738685,29065964,45919976,58749,79657802,21001913,69136837,99965001,63372756,824872,66832478,51464002,92071990,3233084,25636669,12416768,30218878,19898053,23110625,21070110,6808825,99021067,32161669,49328608,67793644,91957544,54232247,68102437,84684495,59177718,68875490,81805959,59329511,9575954,98462867,13231279,36580610,5970607,67227442,8128637,16019925,60955663,83651211,79806380,52261574,17857111,96906410,91727510,58208470,72495719,78602717,45407418,45790169,9257405,76960303,3633375,415901,19376156,62232211,93515664,34432810,669105,84293052,80246713,68316156,168541,4978543,56515456,60430369,70800879,78300864,28928175,53802686,15148031,34497327,95581843,6525948,63059024,41481685,44481640,1569515,65017137,22766820,34236719,30163921,79821897,569864,33797252,26275734,89811711,57241521,30090481,53888755,15075176,33249630,87598888,1431742,55470718,85711894,20122224,33123618,35456853,81898046,59371804,97940276,84904436,66250369,45996863,44983451,44784505,59318837,71300104,91831487,90158785,18699206,51426311,38494874,30395570,16387575,93057697,80251430,23740167,92787493,51149804,28734791,61712234,42237907,85117092,86242799,32159704,64157906,19101477,66611759,91141395,73124510,89637706,97011160,82726008,32250045,70420215,30694952,74724075,2891150,4806458,40197395,83133790,22997281,8373289,7104732,99224392,76671482,45237957,4095116,68702632,19457589,3294781,35512853,22200378,17081350,75986488,89996536,91281584,31126490,13173644,50668599,89419466,7229550,10358899,88444207,874791,99971982,45075471,67084351,83533741,39847321,98653983,23194618,48395186,72278539,12571310,47213183,92803766,72238278,48892201,30139692,90310261,75777973,19272365,44479073,29029316,52060076,10453030,65880522,86301513,48658605,61141874,78785507,32274392,40865610,16567550,77413012,4798568,10309525,63015256,1197320,77301523,51315460,91990218,73235980,37891451,3509435,36189527,44550764,77072625,54058788,7517032,95957797,37560164,56484900,83150534,9860195,43357947,33061250,20002147,92692283,89217461,99297164,31491938,90272749,52204879,74137926,61982238,5832946,84406788,45848907,61741594,84166196,54517921,97561745,58224549,6038457,38008118,45990383,49598724,8129978,10597197,94711842,7011964,23848565,44473167,66667729,70596786,77898274,34946859,96735716,75407347,56531125,29635537,89499542,17957593,64602895,47738185,75820087,22879907,6986898,7182642,16595436,71316369,69605283,84002370,72614359,50567636,48088883,86022504,7685448,78589145,51588015,94076128,36933038,53393358,64848072,46851987,22113133,45381876,7687278,56153345,76703609,62762857,68939068,57163802,36808486,83302115,18504197,48675329,57240218,14220886,20642888,31733363,20836893,92692978,57020481,66319530,30463802,99524975,4069912,38510840,41092172,85242963,33699435,39801351,27187213,50806615,39201414,41380093,61623915,42967683,51715482,65074535,43376279,16424199,69641513,44627776,30787683,4204661,1250437,45794415,11543098,31161687,83083131,54263880,91240048,65454636,87710366,30811010,21397057,76170907,97641116,20885148,90654836,82532312,21993752,74441448,80014588,26114953,77300457,91255408,61928316,48673079,54199160,83948335,53084256,94911072,26998766,85571389,66271566,86821229,54606384,7646095,33935899,60393039,65275241,85695762,17386996,48893685,40534591,44846932,14045383,24168411,58751351,55615885,50007421,20645197,17146629,98943869,26744917,14626618,59614746,33237508,93790285,6221471,69697787,73021291,26063929,81172706,82779622,42644903,92541302,76825057,37192445,22108665,72019362,74743862,5267545,38341669,65038678,70541760,42199455,94360702,8913721,36139942,7066775,66885828,46540998,37280276,93566986,20867149,79191827,60176618,69579137,5640302,18131876,92604458,99226875,13819358,78884452,43152977,20681921,65047700,48774913,57359924,76315420,53632373,28851716,61373987,35092039,8729146,57658654,20568363,3183975,82427263,25842078,82339363,60106217,94516935,43506672,28550822,55669657,13348726,29401781,15536795,36135,33336148,66116458,63628376,17894977,14723410,44664587,81774825,88251446,99690194,62428472,44245960,65081429,75229462,6871053,88698958,18411915,33565483,21673260,9188443,4091162,40152546,92554120,66428795,88130087,44348328,68000591,92530431,36396314,83269727,24915585,47893286,64055960,44847298,4508700,54868730,22129328,75128745,91938191,45049308,1022677,86543538,59405277,74357852,73109997,28787861,72732205,78909561,69355476,10264691,12024238,9175338,17976208,11581181,86798033,52613508,72373496,83747892,95251277,34667286,62357986,42947632,93359396,27041967,79880247,60581278,26139110,17385531,29466406,81677380,32058615,1936762,14731700,84349107,14363867,62051033,24953498,321665,15015906,44060493,82651278,31727379,55850790,38256849,79322415,84127901,86118021,17365188,68694897,7919588,8791066,96420429,82052050,44844121,77377183,749283,29510992,82327024,79922758,78549759,4434662,97057187,5073754,4985896,32151165,12303248,71965942,79942022,68110330,37224844,26292919,90061527,37659250,93270084,67031644,70782102,74614639,16684829,81274566,4064751,39373729,21919959,12664567,21289531,23432750,15902805,29516592,44915235,59981773,11757872,7300047,6157724,13862149,36845587 +4668450,60430369,54199160,14326617,19376156,70420215,89214616,67793644,56955985,59501487,50567636,96193415,32590267,84684495,77072625,68644627,78602717,91957544,13470059,54606384,36812683,22108665,99549373,33699435,66611759,7685448,88444207,92692978,44889423,17386996,4119747,59177718,38365584,73168565,73617245,27625735,5970607,6819644,19939935,31904591,79821897,93933709,60581278,67227442,92398073,48892201,20002147,91727510,72373496,94076128,70372191,59371804,48260151,42692881,569864,93515664,15535065,54427233,75153252,33237508,20765474,20122224,37659250,94711842,95957797,81677380,68110330,874791,526217,16380211,48360198,72274002,13862149,26139110,53393358,60102965,6793819,99021067,86460488,39553046,59614746,63015256,89637706,71316369,19272365,43045786,34295794,99658235,88047921,20642888,14626618,75986488,47361209,78766358,62452034,18783702,29188588,6038457,35192533,83210802,61623915,8099994,74743862,16445503,63152504,93790285,20568363,30998561,68041839,99297164,24058273,30543215,44983451,57020481,24591705,56461322,18833224,49598724,73124510,18663507,30764367,67030811,17976208,27665211,70541760,36685023,26102057,69136837,2208785,36505482,63967300,18466635,51507425,44245960,62430984,75128745,51047803,94516935,11161731,54762643,53802686,33304202,17081350,1431742,99333375,66271566,16424199,19891772,71657078,31126490,77787724,28928175,68316156,30787683,49236559,31733363,62496012,2917920,48088883,37620363,44847298,55753905,321665,73222868,76960303,6153406,14093520,38061439,38494874,73235980,37675718,37183543,20885148,9575954,16567550,33061250,3233569,82178706,36930650,6871053,92787493,59910624,61960400,50188404,17764950,85116755,72777973,44348328,33895336,54058788,7229550,39986008,92604458,79806380,72614359,3633375,74110882,53084256,5111370,72732205,82327024,67513640,6808825,15075176,33628349,30891921,26744917,89804152,31161687,26063929,85242963,95010552,28796059,68875490,72278539,43322743,88653118,45049308,30463802,4798568,72738685,30218878,95395112,13231279,68702632,12416768,22450468,99861373,12348118,59109400,71920426,11581181,92071990,47213183,87598888,88904910,26493119,44481640,78549759,75777973,8733068,86242799,82897371,97783876,79880247,74441448,15536795,63628376,51464002,80316608,45790169,77620120,7300047,78785507,77272628,44550764,47887470,20140249,20867149,84127901,61712234,55850790,14220886,99515901,3235882,43501211,26392416,59197747,20486294,7687278,1022677,33797252,64087743,36753250,82532312,97641116,84293052,72357096,29834512,65074535,93053405,4806458,60176618,14731700,3183975,65017137,52204879,49328608,66667729,65454636,6986898,77694700,37891451,54263880,56484900,95290172,70004753,40781449,53632373,40197395,21919959,29994197,68816413,57248122,4978543,22405120,95726235,21673260,68824981,36780454,35092039,98130363,72019362,67124282,80014588,66250369,56153345,98371444,51472275,30139692,64098930,72725103,66903004,3880712,87720882,18806856,61373987,42307449,81853704,54014062,34946859,87160386,22435353,77312810,55247455,66663942,55189057,41481685,61263068,75135448,47738185,89699445,99226875,51590803,22766820,10309525,17068582,36139942,66045587,3088684,44844121,84406788,51466049,46870723,75543508,99917599,5073754,93566986,1239555,1250437,6505939,34667286,69848388,70367851,34236719,63506281,4515343,48673079,9603598,78884452,17146629,91255408,34698428,65880522,18504197,74357852,9175338,9259676,89046466,29466406,4204661,30366150,1788101,54987042,34698463,23569917,2891150,62936963,61859581,91990218,97011160,76434144,41245325,9623492,83155442,77300457,19898053,7423788,1197320,508198,90090964,38658347,11365791,41380093,93359396,65338021,78300864,39171895,61897582,17857111,30771409,8128637,26114953,17539625,43506672,70800879,20681921,8634541,15148031,49597667,56531125,46851987,7011964,62051033,44473167,44784505,71083565,80555751,67281495,68204242,91281584,71300104,10597197,74137926,40677414,14947650,66322959,7955293,168541,32699744,59318837,63059024,57393458,52261574,21993752,91664334,53888755,86798033,76671482,4099191,13468268,64157906,73031054,71469330,90457870,10366309,79136082,48893685,92554120,51149804,36312813,17058722,18131876,83150534,45996863,21001913,84166196,76170907,67031644,27411561,8373289,4069912,84349107,99965001,6221471,9398733,77301523,82779622,73786944,50007421,29959549,59981773,92353856,56515456,51715482,7919588,82886381,40865610,93562257,247198,65271999,10358899,40781854,90061527,62115552,61271144,98739783,96906410,3294781,82726008,81274566,73021291,56473732,10961421,48658605,32161669,1569515,86118021,65038678,55143724,55602660,99729357,65851721,61141874,66137019,749283,69605283,53648154,15015906,6525948,62803858,49271185,70596786,83789391,1204161,44842615,59329511,24619760,5822202,72358170,79657802,54517921,36580610,34432810,20645197,78589145,41092102,24826575,65047700,47792865,45407418,9058407,29635537,824872,85023028,73392814,53547802,48395186,87710366,45919976,17894977,38008118,96726697,6111563,10453030,92998591,28851716,72495719,14349098,44177011,23134715,669105,16099750,71965942,6497830,98462867,91938191,4091162,29516592,24168411,36933038,40356877,34493392,23194618,77284619,50702367,79738755,9886593,8651647,23740167,3487592,27041967,112651,22113133,37560164,86821229,43357947,82427263,62740044,45075471,8129978,8115266,29401781,75407347,92803766,68939068,40686254,94090109,86543538,96735716,69579137,98943869,37501808,17016156,4064751,83368048,28734791,26734892,49641577,99125126,68128438,77187825,415901,32274392,39801351,99524975,95581843,44664587,43376279,51588015,18699206,13348726,89217461,83747892,80246713,58224549,3233084,77413012,4199704,32058615,45381876,57163802,12664567,8696647,57241521,16684829,65081429,22200378,66428795,17037369,91141395,98948034,92215320,32426752,83651211,44846932,40027975,27185644,45848907,33565483,3509435,24915585,4434662,16405341,40152546,99690194,15163258,78218845,77377183,82052050,60106217,15064655,44627776,42967683,50806615,13819358,96318094,89419466,65715134,12024238,14045383,64602895,61982238,32159704,9860968,16971929,47893286,76330843,83948335,70727211,57240218,71522968,14363867,19457589,81805959,76540481,81898046,42782652,90013093,47090124,34497327,81774825,2717150,97281847,70036438,55770687,23110625,32250045,63372756,85117092,8807940,26863229,39201414,86001008,82339363,68694897,91240048,72238278,31727379,47708850,92692283,16097038,75229462,2331773,57359924,45995325,45237957,94360702,7502255,90654836,33249630,69255765,54868730,12303248,53842979,52060076,99604946,36189527,32151165,29029316,78909561,58751351,88698958,86301513,96420429,39847321,43152977,66832478,36468541,33201905,57803235,97561745,97940276,37280276,25842078,25636669,4095116,35456853,29932657,37435892,58208470,61928316,30163921,7104732,76825057,22721500,98653983,17727650,51315460,57658654,64848072,60955663,84840549,92033260,48675329,62232211,21533347,93057697,16054533,26292919,26998766,89996536,92867155,38645117,56424103,74452589,79191827,47298834,8729146,40984766,24314885,54663246,22129328,26507214,69355476,88130087,62490109,15948937,5267545,26275734,55470718,36396314,4508700,4985896,37166608,5832946,90272749,27187213,91802888,43933006,97379790,37739481,96852321,44479073,7517032,40534591,21289531,98648327,42947632,82651278,11923835,17365188,38022834,55669657,1375023,29510992,30090481,2204165,37192445,57961282,76703609,22997281,19486173,35512853,16387575,36135,73109997,37957788,95251277,3773993,51426311,89078848,85571389,46540998,22879907,33123618,80251430,74724075,69697787,5640302,59405277,99224392,18001617,62762857,6521313,90004325,70782102,90310261,42199455,94595668,61815223,55960386,45428665,68000591,45667668,94911072,45617087,50668599,36808486,83133790,2607799,97057187,30653863,78561158,30569392,33336148,85695762,81855445,61728685,26776922,86022504,67084351,9829782,18411915,15902805,45990383,31491938,89499542,83533741,28787861,83269727,29819940,21397057,85711894,66319530,4787945,83083131,5482538,8913721,41092172,88251446,33431960,76315420,84002370,13173644,58749,61741594,9860195,53666583,58180653,26397786,96709982,11543098,8791066,92530431,68102437,38009615,23432750,92541302,84187166,23848565,9599614,84904436,65275241,10649306,19101477,71333116,39373729,27325428,37224844,9188443,45306070,89811711,8055981,35996293,62693428,52613508,38510840,26664538,45794415,33935899,52293995,62428472,22942635,49882705,90158785,17957593,94539824,75820087,10264691,79922758,16019925,62552524,34044787,24953498,81172706,69786756,30694952,74614639,38256849,54232247,75052463,99971982,91831487,66885828,20836893,67451935,30811010,79322415,44915235,79942022,42237907,77898274,38341669,30395570,9257405,62357986,7182642,93270084,6157724,1936762,11757872,42644903,48774913,69641513,29065964,67474219,72793444,21070110,55615885,66116458,83302115,28550822,61859143,12571310,42073124,7066775,7646095,60393039,33553959,16595436,176257,41442762,44060493,82979980,64055960,17385531,24733232,14723410,36845587 +31161687,27665211,2717150,40152546,60430369,82979980,55753905,91664334,12571310,86821229,37183543,83083131,3183975,47361209,92554120,51715482,38494874,70800879,22721500,13862149,35092039,43322743,5970607,49641577,86022504,40356877,77620120,4099191,12348118,29029316,75820087,26493119,75229462,98948034,85571389,75777973,48675329,43357947,96420429,45996863,47090124,66116458,79657802,61373987,90061527,14626618,61982238,33565483,31126490,81677380,11543098,67793644,36580610,17727650,569864,526217,85116755,4434662,91727510,68644627,92215320,88251446,45617087,50702367,99965001,89499542,34698428,22450468,9188443,1022677,76960303,14045383,13231279,77300457,4806458,75153252,66611759,90013093,62496012,4787945,26998766,80014588,7687278,40197395,20867149,87710366,8373289,9623492,15064655,44889423,78549759,49597667,73786944,77284619,19101477,30163921,168541,15536795,54663246,10453030,99224392,39847321,17386996,51047803,70727211,28851716,59177718,66250369,49236559,16380211,37435892,824872,38008118,74357852,22766820,59501487,60102965,78589145,61859143,66832478,98130363,29401781,55615885,32161669,76315420,1250437,45237957,1569515,63059024,65275241,34698463,73617245,82886381,79922758,89811711,96193415,6038457,6986898,75407347,20836893,2208785,44842615,36780454,73235980,73124510,54606384,60955663,8055981,89214616,41481685,68316156,47708850,27325428,15015906,67474219,25842078,39986008,35456853,79136082,72725103,19486173,65454636,77072625,50567636,38341669,20122224,30090481,19939935,26507214,89699445,37620363,6808825,62452034,44844121,54232247,76540481,55247455,91957544,74743862,62740044,57240218,29510992,99549373,99226875,36808486,95290172,74110882,81855445,45306070,38256849,58751351,85023028,43045786,92398073,53888755,94539824,96735716,64098930,83155442,84166196,33628349,11923835,79821897,62357986,71920426,96726697,23848565,36753250,65047700,55143724,92998591,39201414,29819940,33336148,28928175,5832946,88653118,8913721,97561745,81172706,68000591,71333116,88047921,50806615,1239555,30543215,83651211,67124282,67281495,72274002,64848072,69255765,74137926,3487592,22435353,26776922,89078848,61741594,54517921,20140249,78602717,45407418,57803235,21993752,22997281,10309525,67084351,92692283,74614639,70596786,9886593,44983451,53802686,63372756,46870723,16971929,79880247,61712234,24058273,1431742,38645117,54199160,52261574,4668450,9860968,29932657,53547802,92071990,61263068,28550822,84187166,47213183,39801351,99604946,33304202,73031054,92033260,14220886,62051033,61271144,65081429,34236719,15535065,52293995,82339363,54427233,26063929,59614746,66322959,83210802,99861373,14326617,14947650,76170907,42644903,18806856,30653863,54014062,73222868,54058788,56473732,98648327,8807940,96318094,24915585,8099994,96906410,2891150,44348328,17539625,40781449,70541760,20002147,78218845,8791066,32699744,14093520,46540998,82726008,17016156,92353856,26114953,33201905,38365584,29834512,49882705,84840549,32590267,69641513,7300047,77272628,20642888,247198,14363867,72357096,39373729,29994197,9398733,82779622,66885828,51464002,30764367,68128438,6497830,36505482,80251430,50188404,90272749,7066775,48360198,18783702,53648154,63967300,78785507,61623915,33553959,9575954,34295794,12024238,45794415,77187825,89804152,6157724,6793819,74441448,37192445,81774825,66428795,95957797,91831487,14731700,4064751,36468541,5111370,48892201,33237508,99971982,32159704,40677414,93057697,93562257,36812683,36139942,19898053,93566986,17976208,75543508,26102057,40984766,37739481,28734791,71657078,68875490,53666583,20885148,91802888,33431960,17764950,15948937,93515664,23110625,67031644,56461322,86301513,65880522,99658235,90457870,49271185,4119747,70420215,9603598,31733363,65271999,14349098,44550764,89046466,51588015,82897371,37560164,45428665,62115552,77413012,16445503,9860195,70367851,20765474,26392416,33797252,6871053,21919959,89419466,10264691,60581278,36685023,55470718,29516592,87160386,874791,99125126,19376156,34667286,22879907,4798568,59318837,66045587,76671482,76703609,3233569,8115266,23569917,73109997,82052050,97281847,31904591,6521313,96852321,75128745,48658605,59197747,44245960,30998561,80555751,90004325,62430984,22108665,10649306,7011964,81898046,508198,44473167,7229550,48260151,65715134,15163258,56955985,24591705,34044787,59371804,9175338,23194618,69136837,36135,26292919,36845587,64602895,75986488,65338021,20568363,38061439,85242963,13819358,2917920,99297164,7423788,17068582,321665,44784505,34497327,61728685,84406788,55602660,5640302,99021067,4095116,76330843,71300104,29635537,83302115,17037369,84684495,68702632,32151165,54868730,72614359,24826575,6153406,3233084,94090109,67030811,91938191,72278539,83133790,24619760,73168565,36312813,62936963,1197320,56515456,70372191,57020481,1375023,29959549,55669657,2607799,92604458,52613508,18699206,47792865,72373496,90310261,9599614,4069912,43933006,12416768,57248122,5482538,80316608,1204161,66667729,3633375,30771409,42199455,79322415,57658654,32250045,45848907,21673260,45075471,63015256,30694952,7685448,88444207,76825057,20486294,43376279,32426752,30787683,92803766,95010552,27411561,26744917,18833224,44481640,62803858,11365791,58224549,35512853,21289531,44479073,16019925,97011160,3088684,20681921,42073124,84904436,82532312,86543538,66271566,53842979,3880712,29466406,65017137,415901,44177011,85117092,92541302,52204879,53632373,67451935,68694897,10961421,57163802,57961282,71083565,62490109,56424103,64157906,93270084,85711894,4091162,9259676,47738185,54762643,69579137,97940276,86460488,27625735,29188588,3294781,94711842,44846932,81274566,83368048,81853704,72019362,36396314,46851987,68824981,7955293,61859581,6221471,7646095,3509435,98739783,97057187,94516935,1788101,65851721,17146629,45990383,10358899,26734892,3235882,17081350,22200378,71316369,36930650,11757872,58180653,33895336,54987042,36189527,6525948,95726235,60106217,30569392,15075176,9058407,62552524,72732205,30463802,35192533,30891921,4515343,39553046,58749,41092102,51472275,29065964,69355476,83150534,60176618,73021291,19457589,69697787,35996293,22113133,77301523,51466049,97783876,37675718,26863229,669105,64087743,57359924,32274392,86001008,52060076,78766358,65074535,90158785,58208470,92692978,56531125,99515901,84293052,68939068,83747892,91255408,84127901,59329511,66903004,48774913,48088883,63506281,93790285,78561158,82651278,6505939,51426311,68102437,37224844,94911072,18663507,17894977,57241521,4204661,43501211,18411915,7502255,33935899,45790169,55189057,63628376,37166608,16595436,26139110,44060493,84349107,17857111,19891772,16684829,11161731,24953498,112651,79806380,33061250,67227442,89996536,22405120,88904910,83533741,8733068,45667668,16424199,76434144,17385531,25636669,57393458,99917599,68204242,8651647,45995325,49328608,4508700,34432810,48673079,44627776,47298834,34493392,47887470,90090964,89637706,48893685,83269727,42237907,93933709,44915235,66137019,6819644,38658347,59910624,8129978,7182642,41442762,78300864,80246713,30811010,71469330,18001617,33249630,3773993,5822202,30366150,61141874,91281584,7919588,23740167,83948335,38510840,81805959,62428472,72793444,71522968,75135448,16567550,15902805,98462867,8634541,40865610,87598888,27187213,56484900,68816413,69786756,79191827,83789391,50668599,17058722,77312810,94595668,99524975,61815223,24168411,40781854,72777973,55770687,88130087,38009615,38022834,40686254,92530431,82178706,92787493,9257405,98371444,31491938,93359396,70004753,43506672,68110330,86798033,55960386,62762857,95251277,97379790,91990218,75052463,82427263,77787724,86118021,93053405,21070110,94076128,70782102,61928316,51590803,45919976,77898274,73392814,5267545,42967683,18131876,2204165,30218878,95395112,63152504,19272365,7104732,21533347,33123618,59109400,15148031,92867155,69848388,45049308,51315460,22942635,4199704,749283,55850790,53393358,13348726,32058615,6111563,84002370,16405341,72358170,34946859,77694700,28796059,98943869,42307449,42782652,91141395,26664538,4985896,16099750,9829782,18504197,70036438,49598724,61960400,16097038,72238278,50007421,27185644,62693428,8696647,37891451,41245325,37659250,13468268,37501808,21001913,13173644,59405277,24314885,72738685,65038678,68041839,39171895,74452589,59981773,24733232,11581181,51507425,40027975,27041967,20645197,42692881,56153345,16054533,71965942,77377183,33699435,8128637,79942022,97641116,51149804,16387575,61897582,30139692,23134715,89217461,12664567,28787861,87720882,42947632,78884452,99690194,86242799,40534591,18466635,48395186,31727379,44847298,10597197,41092172,26397786,10366309,47893286,13470059,14723410,17365188,41380093,62232211,37280276,96709982,94360702,88698958,8729146,54263880,30395570,66663942,176257,17957593,5073754,72495719,2331773,22129328,43152977,78909561,64055960,99333375,7517032,36933038,66319530,99729357,26275734,95581843,12303248,37957788,79738755,60393039,85695762,1936762,98653983,90654836,4978543,91240048,67513640,74724075,21397057,45381876,23432750,44664587,69605283,82327024,53084256 +62051033,53802686,65851721,37620363,89046466,52204879,92398073,31904591,43376279,33628349,29834512,61373987,20486294,62936963,99658235,3509435,26392416,59197747,65338021,57248122,96193415,82339363,86118021,89214616,73617245,24915585,74724075,52261574,91802888,99515901,99917599,69355476,37560164,34493392,78766358,66667729,66611759,38658347,69786756,90457870,84127901,51464002,71316369,67793644,69255765,85116755,39373729,64602895,874791,84349107,73222868,4099191,10366309,49328608,98948034,16054533,35092039,2204165,57803235,22766820,2917920,73235980,57020481,39847321,16099750,11365791,66271566,33935899,55247455,4119747,7300047,3233084,45990383,66137019,59614746,66903004,57240218,7687278,4668450,71333116,64087743,10309525,84684495,23194618,92033260,29188588,17081350,20867149,87598888,29994197,27187213,65271999,66250369,1022677,44889423,75229462,44550764,35192533,36930650,6505939,73168565,81274566,69848388,12348118,61263068,26744917,24168411,1431742,53393358,97641116,68000591,51047803,78549759,66045587,40865610,34295794,49641577,60430369,37739481,71657078,89217461,14326617,3880712,71920426,12303248,39553046,77187825,79806380,37659250,1204161,45848907,63967300,21533347,22879907,45237957,96726697,6793819,247198,24058273,87160386,33304202,37675718,44844121,8129978,93053405,44842615,96906410,68875490,67030811,56153345,61141874,72373496,78884452,28796059,8729146,65074535,36139942,30463802,55143724,74137926,38494874,40534591,22113133,99297164,68816413,77272628,55470718,36812683,1569515,55850790,47298834,26734892,76434144,60106217,78909561,112651,73021291,40677414,83651211,52293995,49597667,66319530,39171895,24619760,68316156,90272749,1239555,62693428,29819940,72777973,20568363,37183543,94711842,29029316,69136837,26998766,79821897,19486173,75777973,53842979,38061439,43357947,18699206,16387575,61623915,95957797,6808825,60581278,26863229,92867155,44627776,38341669,57241521,2891150,88653118,5073754,91938191,48673079,21001913,71083565,53666583,97379790,15948937,18783702,53632373,59371804,34497327,55669657,70782102,3487592,47361209,63372756,93790285,73392814,46851987,65275241,93515664,37891451,91240048,60176618,33699435,73124510,98943869,45996863,21289531,40686254,6525948,17058722,30998561,16019925,14626618,61960400,92530431,86301513,29510992,669105,57359924,37224844,22108665,34044787,82726008,43501211,22942635,508198,50188404,74110882,19376156,15015906,76960303,18663507,51715482,64055960,75543508,44784505,81805959,67451935,6153406,86460488,91990218,47090124,84002370,6986898,19457589,27665211,59910624,168541,16445503,36468541,98130363,89699445,30218878,4515343,26063929,34667286,79922758,7104732,35512853,83155442,68204242,14093520,65454636,17764950,30891921,33249630,81677380,92604458,54199160,61271144,44847298,18833224,83789391,20885148,75128745,34236719,10358899,36312813,31161687,81898046,9860195,41481685,15535065,66832478,56531125,58224549,44177011,92215320,83368048,51149804,91281584,65017137,81855445,17365188,93566986,92787493,6038457,86001008,83210802,3183975,93270084,45075471,92541302,72019362,30366150,20645197,88047921,10453030,99021067,46540998,33895336,64098930,66116458,13862149,61728685,12024238,82327024,66885828,96735716,82886381,6497830,44473167,73031054,20002147,95726235,31727379,62452034,76540481,48774913,77694700,43506672,1788101,98653983,62430984,78218845,90090964,17727650,32274392,68702632,88444207,52613508,7502255,53648154,22450468,49882705,67124282,8651647,57393458,24591705,77284619,77413012,49598724,79738755,47887470,16971929,99224392,21993752,65047700,83083131,94090109,30694952,42199455,7919588,85023028,9259676,42782652,44983451,89419466,77301523,7646095,97281847,72614359,33237508,99125126,46870723,22997281,18466635,61712234,30787683,37166608,13468268,80014588,72725103,32159704,95251277,33565483,30771409,34698428,72278539,95010552,36780454,26493119,39801351,41245325,20122224,14045383,45381876,97561745,69697787,65038678,4806458,87720882,50668599,13470059,47708850,19891772,8696647,77312810,99226875,76315420,23432750,10597197,5111370,72738685,43045786,3233569,92998591,48088883,55189057,54663246,91664334,82779622,62762857,68128438,75052463,45995325,51466049,51588015,55753905,9188443,44060493,88130087,78785507,23110625,47213183,8373289,2717150,526217,20765474,97940276,99604946,30653863,16405341,92692978,98371444,54987042,36580610,45306070,40781854,16097038,13231279,79880247,56484900,3633375,76330843,98648327,68041839,5822202,23134715,76170907,34432810,60955663,38365584,95395112,86022504,29959549,44846932,92554120,1375023,54606384,86821229,55602660,68694897,77072625,75153252,89499542,26776922,40356877,77620120,91727510,6819644,21070110,84406788,28787861,83533741,59501487,63628376,8128637,36135,16595436,70727211,33061250,94076128,23569917,63059024,54517921,54014062,65880522,62496012,79191827,45794415,41380093,85711894,98739783,67227442,1250437,99524975,54762643,58751351,47792865,97783876,74743862,86242799,824872,27325428,67281495,5482538,44481640,43322743,84840549,72357096,7423788,16567550,9829782,13348726,30139692,78589145,9575954,81853704,93933709,14220886,66322959,47738185,5970607,30163921,50702367,22129328,14723410,11161731,40152546,24826575,83302115,36685023,1197320,30543215,6157724,22721500,82651278,26139110,16684829,36505482,44245960,62232211,4434662,32250045,54058788,51472275,26507214,96852321,54263880,95290172,19939935,3088684,92692283,48360198,77377183,40197395,68644627,56461322,88904910,61859581,60102965,78300864,62803858,2208785,4069912,10961421,47893286,9886593,79136082,14349098,4204661,26275734,13173644,3235882,28928175,21673260,40781449,17037369,31126490,7685448,15902805,72495719,70420215,9257405,41092102,28550822,8791066,70372191,76825057,82427263,81774825,61859143,69579137,35996293,9603598,99971982,62490109,23848565,26664538,27625735,70596786,54427233,99333375,62357986,61741594,48675329,72274002,4064751,44664587,23740167,42307449,32161669,58749,37501808,17068582,18504197,70800879,4798568,3773993,94911072,78561158,17976208,63015256,94595668,65715134,67474219,69605283,18806856,22435353,25636669,6871053,55960386,61928316,17386996,38645117,74614639,24953498,26397786,56424103,17146629,13819358,63506281,66428795,14731700,8099994,8913721,9398733,749283,29065964,37957788,59329511,4199704,26292919,6221471,53084256,38008118,9599614,33553959,71300104,99861373,42947632,45428665,93359396,9058407,62552524,3294781,29635537,82532312,7955293,21919959,57163802,176257,57658654,59318837,91141395,82897371,55770687,17957593,38009615,30764367,84166196,50806615,19101477,74357852,85242963,44348328,94539824,34698463,89804152,16424199,29932657,56473732,41092172,84293052,321665,2607799,43933006,7066775,18411915,8807940,33797252,37435892,73786944,29401781,80555751,4508700,92353856,61982238,9623492,4091162,36753250,93057697,89637706,53888755,65081429,96709982,24733232,59981773,80246713,75820087,64848072,42692881,42073124,80316608,70004753,19898053,7229550,14947650,8055981,68102437,12664567,9860968,89811711,43152977,91831487,59177718,30811010,83133790,17857111,72358170,94516935,36189527,96318094,45407418,62740044,37192445,51426311,40027975,37280276,71965942,15075176,30090481,86543538,70036438,82052050,569864,32426752,75135448,36396314,30395570,20140249,4787945,89996536,83150534,12416768,82178706,67084351,82979980,40984766,51590803,7182642,99690194,18131876,22405120,77300457,5640302,71469330,45667668,68824981,38022834,9175338,98462867,99965001,42967683,89078848,83948335,88251446,50567636,61897582,56515456,48260151,63152504,48395186,74452589,90654836,15148031,17539625,7517032,88698958,93562257,11581181,48892201,85571389,33201905,67513640,77898274,90158785,26114953,31491938,54232247,78602717,5832946,27411561,38256849,77787724,90013093,22200378,15064655,28734791,91255408,64157906,44479073,99549373,97011160,58208470,27041967,83747892,74441448,12571310,61815223,39986008,70367851,48893685,29466406,75407347,53547802,45049308,20642888,62115552,11757872,48658605,8634541,33123618,39201414,50007421,41442762,87710366,62428472,49271185,8115266,33336148,7011964,31733363,69641513,91957544,8733068,76703609,17894977,19272365,6521313,36933038,10649306,6111563,72238278,35456853,99729357,44915235,17385531,56955985,58180653,42237907,86798033,60393039,10264691,15163258,26102057,68939068,59405277,95581843,4095116,4985896,5267545,72793444,25842078,67031644,97057187,36808486,66663942,96420429,32151165,57961282,27185644,85117092,20681921,17016156,51507425,49236559,32590267,55615885,54868730,2331773,32699744,84187166,45919976,68110330,45790169,11543098,76671482,16380211,51315460,70541760,28851716,42644903,90310261,73109997,75986488,79657802,81172706,18001617,415901,72732205,32058615,38510840,52060076,85695762,1936762,80251430,92071990,21397057,94360702,15536795,79322415,84904436,92803766,29516592,4978543,30569392,36845587,24314885,45617087,90061527,59109400,20836893,79942022,14363867,34946859,33431960,11923835,90004325,83269727,71522968 +9398733,63967300,62740044,85023028,42237907,62936963,56515456,77272628,37183543,66045587,19486173,10366309,62430984,78602717,97011160,29834512,73617245,4099191,45428665,28796059,40356877,33797252,38365584,99965001,95957797,38022834,87598888,54606384,48395186,17764950,90457870,99515901,64098930,36930650,55602660,79191827,55247455,7182642,8696647,26744917,52613508,36580610,7229550,77413012,9257405,99917599,89046466,30764367,37501808,62357986,29959549,14093520,45075471,27041967,526217,11543098,61741594,70800879,29932657,34698428,112651,99861373,83155442,47298834,8099994,95251277,4119747,62452034,68875490,61271144,1204161,98948034,69786756,89078848,20765474,43357947,80251430,92215320,60430369,34497327,51590803,98371444,80316608,37739481,54762643,16387575,61982238,33336148,8651647,28851716,13819358,99524975,14349098,1239555,44479073,18504197,43152977,33304202,70372191,89996536,32058615,96193415,66903004,61623915,73222868,92554120,44550764,36685023,50188404,16595436,7955293,5640302,4064751,43933006,74614639,66832478,26397786,55470718,20486294,20645197,7104732,89214616,18663507,77284619,32699744,15148031,1431742,87720882,43506672,97783876,68128438,69605283,65880522,6505939,68644627,74110882,57803235,68702632,83789391,9058407,9829782,45306070,93270084,99224392,65074535,68102437,99021067,82979980,33699435,77898274,168541,73392814,22942635,35512853,86001008,93933709,69579137,76671482,18833224,84187166,33123618,38009615,90272749,65454636,81172706,8733068,35092039,90090964,80014588,98653983,20885148,16567550,4199704,72357096,24733232,97281847,43501211,73168565,48360198,70367851,51464002,90013093,24619760,65081429,824872,36753250,47090124,10597197,16019925,10309525,37620363,31904591,73124510,12024238,60176618,17365188,16445503,10453030,20642888,8807940,508198,73235980,5267545,30771409,68939068,19376156,50806615,60106217,42947632,53802686,2891150,70727211,29065964,77787724,8913721,53084256,78909561,3233084,53632373,54199160,56153345,98739783,26392416,49597667,19939935,55143724,99549373,30163921,92353856,85571389,38061439,45995325,93562257,53393358,874791,85242963,30694952,36808486,1022677,13231279,1788101,40677414,14947650,71083565,45848907,50702367,4985896,2208785,7685448,40197395,30463802,92604458,26998766,77377183,36505482,39553046,44842615,29401781,92541302,54014062,65338021,77300457,27325428,72373496,27665211,6157724,44784505,17037369,36312813,21397057,91727510,76540481,11581181,60581278,27411561,13173644,18466635,24058273,44844121,38256849,69848388,67084351,45407418,78589145,75777973,67793644,46851987,36780454,56461322,78549759,44245960,35996293,20122224,37280276,10649306,62803858,15948937,30787683,12348118,91281584,79806380,88653118,39801351,99729357,57393458,20140249,89637706,48892201,17068582,73786944,19457589,2717150,44889423,62496012,51426311,91831487,73021291,2204165,33249630,38341669,20002147,96318094,59197747,89699445,6793819,93566986,57248122,61897582,86460488,75052463,75543508,67474219,68041839,54058788,21289531,51047803,3509435,14045383,38008118,247198,3773993,61141874,79880247,84127901,22129328,7517032,49271185,71333116,2607799,62051033,9860195,23134715,30543215,18783702,91240048,67281495,69136837,78766358,72777973,29819940,6808825,66250369,84293052,34432810,54987042,32151165,84904436,3233569,6221471,34698463,13862149,44177011,68816413,22721500,56473732,58208470,35456853,98462867,68824981,83150534,44481640,38658347,94090109,82651278,40781854,66667729,75128745,63059024,33553959,84406788,52204879,32161669,41092102,11923835,23740167,11365791,48675329,16684829,28787861,99658235,33935899,99604946,66611759,44627776,62762857,4798568,29635537,71657078,24826575,30998561,70420215,21070110,96735716,81898046,79922758,77312810,71300104,77301523,74724075,64087743,43376279,33565483,7919588,74452589,92692978,71316369,12303248,57240218,54427233,26292919,29994197,49882705,34946859,65038678,59981773,91957544,26275734,35192533,31733363,8115266,4668450,20681921,6111563,72274002,30653863,98648327,42199455,41245325,59177718,97057187,19898053,34236719,37675718,81677380,13348726,669105,97561745,97940276,88047921,28928175,17857111,64602895,49236559,48088883,9603598,51466049,57241521,45667668,9623492,89419466,71965942,44060493,3088684,68316156,91990218,48673079,4515343,16099750,79738755,1250437,5970607,34493392,6525948,30218878,89811711,7423788,98943869,65017137,24168411,46540998,78561158,84840549,83133790,25636669,6497830,62490109,90061527,79136082,39171895,3880712,24915585,42307449,94911072,66137019,79942022,68000591,71522968,6986898,26493119,2917920,69641513,10358899,4095116,18806856,32159704,19272365,18699206,64848072,40027975,6153406,52293995,82726008,3235882,95726235,15075176,51588015,8729146,83651211,76315420,55615885,83948335,21001913,14220886,51472275,6819644,42782652,41481685,41380093,62693428,70036438,37659250,44348328,94516935,31491938,85116755,88904910,93790285,8373289,15536795,84349107,33628349,61263068,81853704,22108665,92530431,88251446,17386996,415901,98130363,23569917,58224549,2331773,9175338,7066775,1569515,9188443,97641116,96906410,76170907,30139692,33237508,63152504,11161731,37891451,96852321,72732205,55189057,61373987,43322743,31161687,56424103,19891772,55669657,30366150,16405341,83747892,18001617,54663246,49328608,75820087,50567636,36468541,51715482,7687278,3294781,47887470,22879907,29188588,53648154,48260151,1936762,91802888,9259676,40686254,45790169,36189527,76330843,3487592,67227442,91938191,88698958,44846932,61728685,43045786,31727379,30569392,54517921,28550822,73031054,27625735,29516592,48774913,22405120,65047700,15535065,62552524,18411915,65851721,15015906,38494874,72725103,88130087,17016156,47792865,59501487,57020481,33201905,16380211,45381876,32250045,57359924,44983451,17146629,34295794,47361209,29510992,85117092,36812683,45996863,67031644,82427263,60955663,45617087,76434144,49598724,26102057,99125126,83210802,63506281,99226875,72614359,96726697,12416768,72238278,92998591,82886381,99690194,58749,58180653,16054533,34044787,6521313,39373729,40152546,24953498,52261574,71920426,79657802,61960400,32426752,95395112,97379790,66885828,90654836,82178706,30090481,92033260,93515664,49641577,69255765,45919976,81274566,37957788,67513640,20867149,99297164,47893286,22435353,16971929,8055981,68694897,569864,8791066,59910624,78218845,83083131,47738185,13470059,23194618,40534591,42073124,48658605,82532312,67124282,94711842,15902805,60102965,4508700,14731700,20568363,5832946,14723410,17058722,66428795,33431960,45990383,78300864,26734892,59109400,14626618,77187825,67030811,61815223,53842979,89217461,23432750,39847321,68110330,65275241,31126490,5111370,45237957,59318837,6038457,17976208,22200378,77620120,38510840,94539824,61712234,78884452,39201414,62115552,7300047,92803766,56484900,22450468,63628376,81855445,21993752,91255408,40781449,21919959,32590267,99333375,4806458,8634541,59614746,92398073,52060076,36845587,66116458,56531125,53888755,84002370,72793444,83533741,66322959,14326617,90158785,60393039,74137926,16424199,72495719,82779622,47213183,10961421,7011964,69355476,9575954,9599614,25842078,86798033,55960386,75407347,30395570,4978543,57961282,26507214,75986488,47708850,4091162,5822202,7646095,66663942,66319530,82052050,95010552,4434662,53666583,92692283,24591705,99971982,70004753,69697787,81805959,90310261,17539625,70596786,79821897,59371804,39986008,84684495,14363867,72358170,321665,9886593,22997281,26139110,33061250,30891921,4204661,45794415,6871053,92867155,86242799,74743862,26114953,23110625,26063929,17957593,55770687,12571310,40984766,37224844,74357852,76703609,30811010,15163258,4787945,86543538,1197320,24314885,75135448,91141395,76960303,94360702,3633375,53547802,18131876,66271566,54232247,51507425,68204242,80246713,15064655,95581843,96709982,89499542,87160386,17727650,48893685,44473167,7502255,5073754,38645117,46870723,89804152,44664587,83302115,40865610,57658654,5482538,21533347,42692881,91664334,16097038,11757872,23848565,67451935,65271999,74441448,8129978,72019362,50668599,12664567,93359396,13468268,58751351,90004325,61859581,17081350,92071990,29029316,36135,83269727,33895336,92787493,64157906,27185644,86301513,3183975,75153252,176257,37166608,70541760,749283,51315460,61859143,57163802,22766820,85695762,87710366,50007421,63372756,64055960,32274392,37435892,63015256,22113133,54868730,59405277,51149804,54263880,76825057,77694700,9860968,26863229,65715134,93053405,29466406,37560164,70782102,71469330,34667286,4069912,95290172,80555751,72738685,41092172,82339363,83368048,85711894,59329511,82897371,36139942,8128637,44847298,77072625,94076128,62232211,36933038,45049308,84166196,56955985,72278539,26664538,1375023,19101477,62428472,55753905,36396314,82327024,79322415,61928316,42644903,26776922,86118021,88444207,20836893,42967683,86821229,27187213,86022504,96420429,17385531,21673260,55850790,37192445,44915235,78785507,73109997,75229462,93057697,17894977,94595668,41442762,81774825,28734791,10264691 +84349107,68644627,26139110,66250369,77312810,79821897,70367851,14626618,73222868,4099191,91938191,24058273,1204161,31904591,98739783,321665,74137926,60430369,55753905,10597197,4787945,6986898,1569515,26734892,32426752,74110882,49328608,85242963,92398073,23194618,99515901,72777973,42782652,824872,33699435,66832478,66611759,83651211,41092172,17146629,85116755,55602660,96193415,72373496,99861373,60176618,55470718,10309525,63059024,44348328,89214616,76170907,52613508,54014062,48893685,37501808,27665211,86821229,29188588,77272628,79191827,50702367,99917599,99549373,37675718,37891451,65074535,65038678,49236559,68000591,78561158,39171895,9623492,29510992,7517032,20122224,65017137,5111370,10366309,62430984,32058615,95581843,26063929,65275241,30395570,70800879,78766358,34497327,93270084,68816413,20885148,99524975,20486294,75777973,32250045,18783702,85023028,16019925,22879907,74724075,68824981,97561745,23110625,99297164,99021067,7011964,96726697,82886381,51047803,32159704,29994197,61859143,80014588,6153406,44889423,77620120,40865610,71333116,82327024,50567636,57359924,66116458,60102965,7646095,43501211,8807940,59371804,36580610,52204879,6221471,95957797,15535065,526217,12664567,57248122,34946859,69136837,18699206,88047921,50188404,37739481,4798568,92692283,5822202,55247455,54199160,29029316,5832946,15148031,669105,33237508,97641116,29466406,17058722,68204242,57803235,75407347,28787861,99604946,35092039,81855445,90457870,19939935,16054533,48260151,94090109,20867149,62051033,74614639,37166608,56515456,68875490,89811711,6525948,67281495,44177011,1431742,48673079,30366150,569864,37183543,26507214,62496012,21919959,66667729,91990218,88653118,42199455,54263880,61373987,43506672,29819940,39553046,8696647,36933038,15948937,19898053,92554120,44983451,9599614,74743862,70372191,16099750,38061439,34432810,52261574,79922758,34236719,83789391,30998561,16684829,37620363,8129978,12303248,39373729,88444207,3509435,28796059,86001008,16971929,86118021,28550822,4985896,46870723,56153345,45617087,26776922,36312813,78300864,75128745,48360198,67793644,50806615,35456853,90090964,54868730,70004753,68316156,50668599,70036438,6505939,86543538,99965001,93933709,17068582,32590267,33565483,43045786,99224392,98130363,90061527,44473167,54427233,24619760,83155442,55143724,77072625,9886593,62936963,7687278,3233084,34698428,51472275,15075176,34295794,80246713,91957544,96318094,59197747,36930650,7229550,63967300,61141874,20002147,38008118,8651647,53666583,24733232,81853704,60106217,12348118,7685448,74441448,17037369,92604458,29834512,84684495,47792865,66903004,55960386,61271144,69786756,66428795,8128637,44784505,23848565,48658605,4668450,40686254,65851721,24953498,34493392,32151165,45990383,97783876,16424199,15015906,76434144,16595436,45790169,64087743,8913721,17727650,59910624,93515664,90310261,28928175,30543215,4119747,2208785,49271185,11365791,56424103,38341669,63372756,99658235,40197395,78549759,21993752,7502255,6111563,43357947,48892201,29065964,48774913,98462867,508198,95395112,45919976,73168565,23134715,25842078,84127901,29516592,31126490,77694700,9058407,81677380,40781854,54987042,53084256,56461322,26292919,94516935,14326617,19101477,3294781,47893286,43376279,38658347,17764950,22942635,95726235,4515343,63506281,77787724,43322743,82427263,42237907,45848907,81172706,83302115,78602717,82726008,81274566,42644903,33553959,64602895,45306070,3235882,9257405,35192533,60581278,45995325,94711842,61263068,11543098,20140249,9575954,76315420,92998591,16567550,14947650,19891772,71316369,78884452,45075471,24168411,75229462,67124282,61859581,51464002,44842615,57658654,7423788,59109400,1936762,33249630,38494874,92033260,73124510,18504197,64848072,84904436,8373289,3183975,87598888,36685023,24591705,40534591,62357986,21289531,45667668,30163921,83150534,33797252,27187213,37957788,84293052,66137019,79738755,82339363,55770687,87160386,42073124,33123618,44847298,56531125,168541,45237957,91727510,75153252,6038457,66663942,2917920,53547802,20836893,80316608,95290172,54663246,26744917,30139692,112651,42947632,11923835,4064751,19272365,39986008,21397057,66045587,71083565,69697787,49598724,30764367,51588015,57020481,76960303,82979980,4204661,68041839,9603598,8733068,874791,45381876,92530431,68128438,93053405,73392814,23569917,9188443,71657078,75543508,32699744,89046466,13173644,8634541,20642888,42692881,96735716,94911072,8099994,44245960,16405341,22766820,86460488,83133790,62452034,69579137,65715134,26863229,68110330,17857111,70420215,77413012,11581181,36505482,77284619,11757872,44550764,40027975,1197320,97940276,6871053,85571389,44481640,30891921,76540481,30463802,45996863,13231279,88130087,54606384,70596786,89419466,38256849,13470059,65271999,68102437,72274002,92787493,33628349,6497830,97011160,5970607,80251430,20645197,18411915,8055981,98371444,4091162,22129328,6808825,35512853,22108665,62803858,14220886,7919588,58749,96906410,26392416,92692978,67227442,10358899,75986488,33895336,40152546,82651278,31161687,50007421,89804152,89996536,14045383,91802888,74452589,47213183,30811010,83210802,10961421,68939068,30090481,57240218,22721500,95010552,7955293,26664538,16445503,15163258,82897371,65338021,39847321,15064655,41245325,10649306,28851716,42967683,96420429,97281847,68702632,32161669,33304202,82779622,72738685,63152504,54058788,39801351,74357852,20568363,15536795,73235980,7300047,31733363,67031644,9175338,21533347,97379790,93566986,11161731,58208470,27041967,7104732,5073754,67084351,78909561,29932657,16387575,25636669,67474219,49597667,9860195,33431960,47708850,57241521,17539625,15902805,72495719,17894977,82178706,87710366,12416768,40677414,93790285,51466049,64157906,17385531,99971982,59981773,29959549,41380093,86242799,1022677,62232211,72357096,89499542,58224549,83533741,23432750,47361209,8729146,14731700,61982238,86022504,55615885,6819644,61712234,90013093,62552524,53888755,1375023,33336148,69848388,46851987,87720882,91281584,31727379,69355476,96709982,22450468,91664334,44479073,98648327,81805959,66319530,62762857,99226875,79136082,16380211,98943869,18833224,37659250,71920426,94539824,70541760,68694897,13348726,72725103,17081350,6793819,749283,53842979,61815223,43152977,33935899,61960400,36812683,45407418,72732205,4508700,41442762,38365584,14093520,84166196,44915235,18663507,72614359,95251277,29401781,36845587,73031054,17976208,26102057,12024238,59177718,35996293,26114953,98948034,62693428,1788101,57393458,3880712,53632373,52293995,41092102,20681921,26397786,88251446,3088684,71522968,20765474,78589145,36396314,22435353,93562257,3773993,77377183,24915585,72278539,28734791,85711894,48675329,33201905,37560164,83368048,53648154,79657802,73617245,71300104,13468268,90272749,58751351,24314885,39201414,83948335,19486173,77301523,66322959,53393358,46540998,72358170,22405120,24826575,41481685,17016156,26998766,22200378,93057697,53802686,22113133,2717150,66885828,84840549,59614746,45428665,36139942,59329511,44060493,247198,59405277,1250437,13862149,14723410,61728685,67030811,59318837,415901,44627776,36780454,18466635,61623915,48395186,6157724,19376156,61897582,14349098,63628376,9398733,83269727,27325428,88698958,86798033,54762643,51507425,58180653,78218845,44844121,84406788,22997281,47738185,79880247,3633375,37280276,2891150,33061250,4095116,91240048,73786944,43933006,79806380,89699445,81898046,47298834,36808486,40356877,13819358,92541302,89217461,30218878,14363867,38009615,76330843,79942022,18806856,81774825,67451935,42307449,77300457,49882705,17957593,72019362,8791066,77898274,21673260,26493119,94595668,3233569,9259676,80555751,89078848,40984766,82532312,34698463,65047700,55669657,2204165,84002370,34044787,85695762,73109997,57163802,84187166,5482538,78785507,92803766,30787683,72238278,47090124,40781449,16097038,36753250,71965942,38645117,98653983,4806458,21001913,45794415,94076128,62740044,37224844,97057187,26275734,77187825,19457589,52060076,56473732,56484900,59501487,91831487,30694952,27185644,21070110,64055960,17365188,66271566,4199704,9829782,37435892,30653863,30569392,76825057,91255408,75820087,63015256,65454636,4434662,83747892,51315460,4069912,90654836,92353856,99690194,18001617,73021291,99125126,5640302,36189527,69255765,51426311,176257,18131876,96852321,69641513,89637706,75052463,47887470,37192445,3487592,34667286,55850790,51149804,90004325,17386996,94360702,54517921,69605283,32274392,92215320,1239555,76671482,2607799,82052050,12571310,85117092,76703609,88904910,8115266,51715482,7066775,86301513,44846932,70727211,27411561,92071990,48088883,62115552,71469330,51590803,93359396,49641577,38510840,10453030,5267545,30771409,60393039,99333375,56955985,70782102,29635537,36135,55189057,62428472,36468541,64098930,67513640,9860968,57961282,61741594,6521313,99729357,61928316,92867155,38022834,4978543,65081429,62490109,23740167,91141395,2331773,27625735,45049308,83083131,60955663,7182642,44664587,54232247,75135448,79322415,72793444,65880522,31491938,90158785,10264691 +77620120,89499542,57803235,67281495,29029316,10366309,669105,4099191,66250369,99604946,36312813,321665,35192533,43376279,16097038,19939935,55753905,65074535,14626618,55470718,6153406,39801351,68644627,34497327,247198,39171895,5970607,24058273,68702632,37620363,75229462,51464002,26292919,67793644,73235980,76315420,38658347,90013093,48892201,59109400,40984766,99549373,824872,35092039,95726235,26392416,61373987,12664567,5267545,29834512,30163921,62430984,69355476,29932657,63059024,99515901,20836893,19101477,66667729,5822202,74110882,17365188,98739783,55247455,34698428,28734791,52261574,49882705,15535065,71657078,27665211,32159704,70596786,56515456,83302115,75543508,54427233,77272628,54199160,65038678,79880247,83651211,86821229,96735716,8651647,16445503,85242963,36505482,65851721,26063929,33061250,12348118,70004753,44481640,569864,77312810,98371444,22405120,4985896,40865610,30463802,61728685,30395570,75777973,66045587,74137926,80014588,98130363,48360198,95251277,23848565,44889423,31904591,49236559,91727510,47090124,168541,45075471,22113133,69579137,89046466,54663246,84406788,80316608,4119747,1204161,91990218,87160386,95581843,65715134,66832478,99965001,39373729,56424103,52060076,62496012,65017137,14220886,37675718,61712234,85116755,34698463,92554120,93933709,78884452,99226875,60106217,5482538,4064751,53632373,55143724,40781854,31126490,32151165,68939068,22766820,26507214,18504197,7919588,62232211,20122224,69136837,96193415,30998561,37739481,68875490,15948937,94516935,1569515,55960386,33249630,43506672,6111563,72274002,20486294,43933006,6871053,55602660,86460488,16387575,73786944,96726697,51047803,77187825,31727379,65338021,36930650,6819644,10358899,10309525,93790285,16054533,78549759,4515343,7685448,90090964,67451935,82897371,40197395,83150534,52293995,45790169,29188588,62740044,26275734,7687278,69605283,25842078,22108665,38494874,59910624,51149804,49641577,16971929,37659250,79942022,71333116,61741594,59329511,26139110,47361209,54014062,88444207,4668450,1239555,30139692,8099994,16019925,72777973,46851987,20002147,94360702,15075176,12571310,57240218,75128745,29466406,84840549,32161669,42199455,63967300,37192445,76960303,2717150,66428795,93053405,73392814,74743862,60430369,87598888,82427263,94911072,99524975,96318094,44842615,92033260,31733363,33431960,176257,62552524,77694700,29994197,26863229,36580610,98462867,64602895,61960400,22997281,9058407,83210802,78589145,48260151,37224844,75407347,55770687,50806615,75153252,46870723,75820087,3509435,72738685,21993752,30366150,33237508,9175338,9188443,84349107,61263068,48673079,53393358,3487592,72614359,526217,32250045,99658235,62936963,88047921,86001008,57359924,53842979,62051033,33797252,74724075,1250437,75986488,24953498,17385531,76434144,58751351,14731700,32426752,33565483,21533347,38341669,73124510,85023028,18699206,48774913,44846932,20765474,51472275,50702367,99021067,9886593,61982238,15148031,8128637,56531125,3233084,89214616,82726008,28796059,98648327,71316369,83789391,17894977,60176618,38061439,26493119,91957544,19376156,29959549,44473167,78602717,1431742,26114953,94711842,40152546,73031054,12416768,18833224,55850790,44915235,93057697,92541302,82979980,81805959,47792865,48893685,53666583,29065964,44784505,69641513,96709982,66611759,35456853,23134715,33201905,27041967,64848072,68694897,88251446,82886381,4787945,77284619,81677380,58224549,88653118,39201414,98948034,62693428,63628376,84187166,7502255,45617087,7646095,72019362,23432750,62452034,15536795,84684495,77072625,4798568,91664334,80246713,6986898,62803858,38365584,94090109,14093520,44844121,44177011,33628349,36812683,112651,7517032,66137019,54517921,5111370,3633375,44348328,19457589,33304202,81855445,89078848,91802888,18783702,30694952,52613508,5832946,9599614,2331773,24733232,26998766,40686254,4508700,90272749,8729146,7104732,33699435,18131876,68102437,73109997,71300104,80251430,47213183,17068582,45237957,16405341,70800879,61859143,17857111,79821897,89804152,34295794,11581181,92692283,60581278,26734892,73617245,45848907,9860195,8807940,34236719,40534591,61271144,20645197,89419466,44550764,45049308,57658654,77787724,88904910,14947650,89637706,26664538,90457870,6808825,67474219,42782652,415901,60102965,17081350,97561745,52204879,99861373,99333375,66319530,51590803,51426311,97940276,42947632,59501487,84904436,7300047,72373496,42692881,4069912,68816413,6793819,9860968,53084256,17764950,14326617,7423788,3088684,70541760,32699744,92398073,36808486,45407418,99297164,76825057,6525948,78766358,37957788,68824981,11365791,1788101,64087743,70372191,61815223,47708850,59371804,77301523,3294781,749283,71083565,15163258,17957593,49328608,95957797,63015256,42967683,22129328,44479073,17037369,50668599,93515664,69255765,31161687,26776922,93270084,15902805,50567636,72495719,24591705,20140249,30543215,21001913,77413012,55189057,90310261,82339363,18663507,81172706,2891150,1197320,45428665,17016156,76703609,28787861,17146629,91831487,16380211,46540998,84166196,72725103,36780454,76330843,7011964,91938191,7955293,8373289,39847321,14045383,30764367,86543538,76540481,66903004,18411915,55669657,65880522,54868730,6505939,10649306,40027975,84293052,22721500,16424199,85711894,35512853,21919959,78561158,70036438,94595668,5073754,84002370,59177718,34432810,6038457,2208785,95290172,23569917,37183543,41092102,50188404,9829782,8115266,24915585,63506281,36189527,83133790,68000591,59405277,71469330,61623915,67084351,57393458,20568363,59614746,83948335,17976208,77377183,33895336,59981773,14363867,44245960,508198,81274566,92787493,16595436,41245325,13470059,13862149,6221471,93566986,54263880,37280276,44983451,74441448,8129978,97011160,36753250,43152977,1375023,20885148,13173644,78785507,40781449,54606384,29516592,3773993,95395112,38022834,87710366,7066775,34946859,96906410,26397786,14349098,30569392,53888755,57961282,24619760,85571389,44847298,3183975,53648154,38008118,70420215,33935899,82327024,83155442,86118021,69848388,57248122,15015906,24826575,70367851,9623492,53802686,92604458,78300864,16099750,28550822,81853704,27411561,30891921,32274392,48088883,79922758,22450468,30787683,6497830,22942635,27185644,45794415,57241521,73222868,79657802,76170907,73168565,22879907,4095116,62357986,39553046,37891451,41442762,99224392,44060493,3235882,77300457,85695762,92530431,12024238,14723410,34493392,50007421,51466049,4978543,45381876,4199704,36933038,75052463,40356877,65275241,34044787,44664587,36845587,66885828,36135,45667668,21070110,38009615,97281847,45919976,91281584,47887470,9257405,99971982,13348726,54762643,1022677,4091162,16684829,86301513,43501211,67227442,43322743,18466635,69786756,2917920,67513640,83083131,9259676,79191827,2607799,56153345,54058788,85117092,28851716,47298834,78218845,61141874,66271566,45995325,64055960,11757872,74357852,97783876,87720882,17727650,26102057,23194618,75135448,6157724,80555751,22435353,93359396,8696647,68204242,33336148,19272365,37166608,19486173,63372756,8733068,8634541,96852321,97057187,70727211,74452589,18806856,82532312,54987042,88130087,23110625,92215320,95010552,72732205,10597197,53547802,97641116,89217461,29635537,79738755,78909561,41092172,30653863,17386996,73021291,38645117,86798033,65047700,11543098,58208470,40677414,81898046,90654836,27325428,10453030,41380093,45306070,59318837,76671482,61928316,45990383,13231279,68316156,21673260,41481685,86022504,27187213,19898053,37435892,57163802,67031644,71920426,42307449,38256849,42073124,30811010,36139942,79136082,10961421,29819940,21397057,56473732,92692978,62762857,77898274,37560164,91240048,99125126,28928175,82052050,3880712,90158785,66322959,15064655,71965942,11923835,19891772,68041839,99917599,12303248,30771409,35996293,92998591,72357096,51507425,874791,39986008,30090481,89811711,33553959,64157906,74614639,65271999,4806458,8913721,93562257,5640302,8055981,61859581,32590267,42237907,70782102,97379790,17058722,55615885,29510992,66116458,20681921,4204661,99690194,7182642,69697787,26744917,83533741,20867149,56484900,94076128,61897582,9398733,13819358,56461322,62490109,2204165,72238278,13468268,66663942,8791066,29401781,38510840,59197747,9603598,92353856,43357947,64098930,25636669,72278539,72358170,23740167,86242799,62115552,48395186,56955985,96420429,91141395,24168411,49598724,84127901,18001617,1936762,83747892,49271185,16567550,92071990,89699445,32058615,67030811,92867155,57020481,54232247,44627776,79806380,65081429,30218878,94539824,11161731,37501808,58749,62428472,82651278,51315460,67124282,9575954,98653983,45996863,6521313,51715482,88698958,48675329,42644903,34667286,48658605,91255408,22200378,58180653,47738185,90004325,98943869,49597667,79322415,92803766,7229550,63152504,83269727,60393039,20642888,68128438,89996536,51588015,21289531,3233569,43045786,60955663,31491938,36468541,47893286,4434662,36685023,72793444,71522968,83368048,17539625,82178706,81774825,65454636,82779622,27625735,24314885,33123618,68110330,36396314,90061527,10264691,99729357 +53547802,73617245,41245325,17385531,40781449,86022504,6808825,44889423,64157906,61263068,41380093,29065964,7955293,55753905,85116755,27411561,99971982,35456853,92554120,8807940,66428795,53802686,15535065,92787493,79821897,4064751,60102965,96193415,4978543,17727650,45237957,40984766,32590267,36505482,37620363,72777973,34044787,67793644,14947650,52060076,4099191,16097038,45617087,38008118,66250369,51464002,82897371,97561745,72738685,41092102,92033260,54517921,19939935,51472275,93933709,33797252,65851721,54762643,89811711,8634541,17068582,55247455,31126490,9599614,91255408,29819940,59329511,96852321,88653118,97379790,93515664,56955985,10309525,6986898,89419466,93270084,73235980,5111370,58751351,44842615,91664334,21919959,15075176,94595668,22113133,62740044,63372756,38645117,50668599,40865610,59318837,82979980,2891150,62496012,26744917,29029316,89214616,36753250,6793819,31904591,96726697,9886593,26493119,57240218,68204242,53842979,1431742,83210802,21993752,77413012,7182642,90457870,21070110,78602717,71657078,6505939,15536795,16445503,80014588,36808486,44348328,44915235,62430984,72373496,97783876,84187166,79322415,24058273,49641577,4787945,73786944,75229462,78589145,44784505,65275241,30764367,55669657,30771409,3509435,5970607,98948034,84684495,65271999,38494874,18806856,16099750,77284619,54014062,95290172,60955663,62452034,50806615,36580610,99515901,8129978,99658235,71920426,29994197,72278539,247198,91990218,98943869,51047803,65017137,17764950,51149804,19101477,7646095,61859143,67030811,67451935,33061250,9188443,415901,98371444,25842078,58224549,77072625,35996293,94090109,1197320,65715134,78561158,66903004,99690194,69697787,39373729,84127901,24953498,37435892,18699206,39986008,68644627,82726008,3633375,44481640,28550822,22942635,48260151,56531125,10366309,71469330,66832478,54987042,83269727,27665211,63059024,34493392,23569917,9257405,45996863,36780454,21289531,74137926,98130363,82178706,18833224,4091162,66667729,81853704,95581843,18131876,9575954,83651211,26664538,32161669,81274566,92692283,42782652,72274002,30163921,55189057,30366150,22997281,13231279,36312813,669105,56424103,26998766,38658347,7502255,16019925,1250437,66885828,72019362,10264691,73222868,78766358,37675718,37891451,28928175,44983451,35192533,45407418,62762857,8913721,39801351,14731700,1788101,74110882,47090124,15015906,89699445,45075471,77694700,17365188,72238278,62552524,73124510,37957788,28851716,76960303,29510992,6221471,29834512,7066775,96735716,22405120,3487592,82886381,62490109,57961282,55615885,83533741,7104732,4798568,99297164,26392416,4508700,44550764,40197395,89637706,71300104,79657802,90090964,63015256,14093520,86001008,4515343,30090481,41442762,99549373,99965001,86118021,70036438,42073124,50702367,45428665,68875490,14220886,34946859,45990383,48675329,98653983,68000591,50188404,83368048,37560164,66663942,63152504,30694952,16054533,27625735,43376279,65338021,74441448,4204661,76434144,4434662,24591705,61982238,66116458,76315420,89804152,90013093,56515456,94911072,30787683,54058788,20867149,569864,50567636,39847321,11543098,4806458,57658654,94076128,23194618,8651647,68824981,71083565,42967683,4095116,98462867,7300047,24168411,26776922,17037369,10597197,9259676,26863229,52261574,92215320,176257,58749,99021067,67281495,91957544,83302115,8128637,75986488,76330843,64848072,62115552,12664567,70727211,36933038,749283,62693428,45794415,77272628,70372191,57803235,67084351,59405277,9829782,96906410,61960400,55143724,73392814,29635537,63967300,87160386,16380211,24314885,22721500,7423788,72358170,60106217,20568363,69641513,45381876,32699744,13173644,11757872,67227442,32058615,33628349,26114953,34295794,68102437,55602660,11581181,34236719,54427233,14326617,18783702,57241521,98739783,33431960,39171895,74743862,57359924,83150534,12348118,20885148,21673260,69255765,71333116,22108665,81898046,96420429,56484900,63506281,4069912,68041839,8373289,55960386,69355476,33895336,17976208,39553046,38365584,53888755,53648154,26063929,92604458,59197747,7687278,92692978,84840549,90272749,59371804,10961421,39201414,65047700,11923835,4668450,34497327,55770687,78909561,6525948,40781854,30569392,9623492,3183975,44479073,82339363,12416768,32159704,2331773,47738185,72357096,25636669,34667286,45848907,48360198,78218845,93053405,14045383,77620120,99861373,45919976,59177718,33237508,19272365,30395570,37501808,95957797,75407347,18663507,15064655,30218878,95395112,82327024,74357852,77898274,86301513,65081429,29188588,95726235,37280276,3294781,22129328,65880522,73168565,69605283,81172706,29466406,58180653,30998561,97281847,3088684,13348726,20002147,64055960,57163802,66045587,84349107,5482538,60430369,45790169,7919588,17146629,68816413,74452589,79880247,88444207,53632373,80246713,45667668,18411915,92867155,66322959,14363867,66611759,58208470,321665,40534591,38510840,92398073,10358899,99524975,51715482,32426752,43501211,53393358,93790285,34698463,47708850,61928316,65038678,74724075,22450468,40686254,75135448,59614746,26139110,66319530,22200378,99729357,508198,70420215,69136837,33336148,87598888,3773993,37739481,59501487,61271144,16424199,64602895,24619760,53666583,2208785,33201905,2204165,48893685,47792865,90004325,31491938,1204161,37183543,77787724,34432810,84002370,18001617,49236559,28787861,45995325,2717150,41092172,13468268,93359396,76825057,44177011,20140249,49271185,21397057,61373987,44847298,62232211,43357947,17539625,51315460,40152546,90061527,69786756,84904436,50007421,82651278,9860195,16971929,26507214,8791066,168541,91802888,37166608,11161731,99604946,56473732,72725103,1936762,35092039,10453030,54868730,47361209,34698428,5073754,31727379,84406788,52204879,54199160,99226875,7517032,43933006,29516592,1569515,15902805,40677414,67474219,7685448,79942022,17016156,76703609,112651,32274392,79191827,824872,61859581,6871053,2607799,92541302,77312810,44844121,85242963,8729146,72614359,89217461,26292919,91727510,99224392,71316369,29959549,77187825,51426311,79922758,6111563,15163258,95010552,94539824,5822202,70367851,99125126,19486173,55470718,89996536,14349098,91831487,17386996,32250045,37192445,51466049,60581278,69579137,49328608,23432750,36930650,89499542,2917920,29932657,12303248,42947632,9603598,9058407,7011964,23848565,71522968,89046466,38009615,66137019,19898053,30543215,12571310,40356877,36396314,42307449,38256849,43322743,83747892,9860968,75543508,92353856,88698958,33699435,80555751,26734892,6157724,48395186,30891921,38061439,92530431,526217,42199455,37224844,61141874,7229550,6521313,4119747,37659250,70004753,62051033,43506672,44664587,98648327,36812683,6153406,70541760,36685023,76540481,61623915,53084256,64098930,8115266,93566986,15948937,6497830,17894977,44627776,92803766,17957593,3233569,30653863,68939068,63628376,20642888,20122224,19457589,13470059,47893286,49597667,20486294,48088883,62803858,16387575,22435353,93057697,51588015,48673079,92071990,60393039,96318094,20765474,8696647,8733068,44245960,97940276,75128745,87710366,44473167,85117092,88904910,3880712,67124282,56153345,9398733,75777973,91281584,24733232,88047921,36139942,33249630,33565483,14626618,55850790,52293995,42237907,54663246,38341669,44846932,83789391,99917599,77300457,43045786,90310261,26102057,72495719,31161687,68110330,94516935,61728685,40027975,22766820,24826575,83083131,48892201,78785507,97011160,20681921,36468541,66271566,4985896,20836893,86460488,28734791,51590803,19891772,42644903,75153252,14723410,49882705,56461322,79136082,57393458,6038457,72732205,3233084,70596786,43152977,69848388,36135,86242799,96709982,42692881,16595436,27187213,23110625,83948335,68702632,89078848,88130087,33304202,16684829,22879907,874791,68694897,47298834,17857111,71965942,57248122,91141395,5640302,62936963,87720882,82427263,64087743,12024238,27325428,99333375,76170907,38022834,26397786,97641116,44060493,94711842,83155442,4199704,78300864,46851987,80316608,62428472,33553959,82052050,59910624,80251430,18466635,51507425,59109400,60176618,78884452,74614639,61897582,83133790,5832946,81774825,52613508,86543538,94360702,61815223,54232247,13862149,13819358,46870723,97057187,92998591,30139692,81855445,45049308,10649306,21533347,76671482,15148031,20645197,5267545,81805959,84293052,75052463,88251446,85571389,48774913,82779622,79806380,1022677,84166196,47213183,27185644,33123618,18504197,81677380,11365791,21001913,73109997,17058722,28796059,73031054,82532312,3235882,79738755,75820087,17081350,54263880,23740167,31733363,24915585,93562257,54606384,30811010,35512853,33935899,36189527,8055981,85711894,1239555,86821229,61741594,77377183,19376156,59981773,85695762,72793444,68316156,86798033,36845587,29401781,57020481,91240048,26275734,67031644,65074535,6819644,85023028,16405341,91938191,46540998,9175338,61712234,70800879,68128438,27041967,45306070,47887470,30463802,67513640,78549759,23134715,95251277,41481685,48658605,73021291,1375023,49598724,90654836,65454636,70782102,32151165,8099994,90158785,16567550,77301523,62357986 +63967300,59177718,93933709,87598888,54427233,78602717,33304202,42307449,89078848,79806380,30090481,99965001,14626618,63628376,54762643,19376156,27625735,27665211,16097038,4985896,7011964,99226875,18504197,22108665,49882705,95395112,75543508,44844121,54199160,10366309,26392416,25842078,2208785,65338021,1204161,247198,93053405,37620363,61263068,19486173,18833224,51464002,41481685,73235980,43376279,68694897,51149804,75135448,38022834,46540998,67793644,68702632,19939935,14349098,66271566,34698463,37183543,9398733,49598724,99549373,26275734,14731700,22997281,3509435,20885148,22129328,7104732,1250437,89637706,40152546,39801351,7229550,64602895,98462867,24314885,84187166,95726235,33061250,15536795,26863229,32590267,4668450,85116755,38061439,30998561,44842615,22450468,51590803,69355476,16019925,55669657,42782652,66045587,38658347,82427263,60955663,59318837,64848072,81898046,73031054,73786944,92541302,94516935,36780454,64087743,93270084,26998766,3773993,4119747,38365584,8651647,86001008,32159704,44784505,17068582,2204165,10309525,60102965,93562257,3233084,99297164,70420215,5640302,18663507,96193415,32161669,42237907,60430369,4434662,6793819,80246713,72614359,16445503,62740044,78785507,34493392,21993752,54606384,56531125,99917599,63372756,63059024,57803235,27041967,18806856,88047921,1936762,58180653,91831487,20568363,62762857,35192533,12664567,9259676,60581278,7955293,55602660,82339363,33699435,41380093,71657078,15015906,99333375,33797252,82897371,83150534,17386996,49641577,31733363,82726008,63152504,47887470,16054533,44479073,65851721,83210802,51588015,40984766,76825057,44847298,34667286,50567636,47090124,9175338,37891451,39553046,93057697,81855445,10358899,79922758,68644627,12416768,89419466,99224392,53802686,321665,34432810,70036438,63015256,66250369,5482538,62357986,8729146,86242799,88698958,92692978,79880247,72725103,73124510,94539824,824872,29510992,70800879,95290172,65454636,61815223,17764950,89996536,81805959,45381876,57241521,17365188,56153345,59371804,64098930,80014588,77284619,87720882,58208470,23848565,48892201,95957797,34295794,92033260,77272628,62936963,14093520,27185644,61982238,85571389,68128438,8099994,36685023,91990218,70596786,93566986,40781449,35512853,38008118,9058407,36580610,88904910,67474219,62452034,30764367,24826575,73392814,40027975,3487592,70367851,9623492,77072625,1197320,65074535,112651,24591705,6153406,31727379,2717150,81274566,15902805,61271144,9886593,51466049,91727510,89046466,20765474,45049308,2607799,30891921,25636669,47361209,84406788,55189057,37501808,53084256,7919588,62115552,1569515,61741594,56473732,44983451,62430984,45075471,36812683,44550764,42947632,4099191,569864,79821897,30787683,2891150,508198,20122224,7423788,12571310,73617245,37435892,55143724,89214616,50188404,13173644,43506672,2917920,66322959,17081350,86798033,94076128,21533347,91240048,81677380,30771409,57248122,56955985,17976208,54987042,83368048,36505482,68939068,10649306,17957593,75777973,74110882,526217,62051033,19891772,78549759,6808825,34698428,82052050,68824981,43152977,16099750,65880522,54663246,669105,88653118,39171895,45617087,75820087,78589145,78766358,20642888,4064751,72732205,66667729,85242963,55470718,48088883,69786756,749283,79657802,37659250,16684829,68110330,99515901,20486294,76671482,91664334,93515664,77787724,30694952,14363867,83155442,92998591,33201905,71316369,91141395,36930650,97561745,62552524,94090109,3183975,4798568,29932657,60176618,4978543,78884452,14947650,48360198,28928175,10961421,78561158,23432750,26102057,87710366,91281584,8807940,45237957,36135,18411915,72793444,74357852,11543098,76315420,87160386,26292919,69605283,76703609,75153252,5267545,96735716,85117092,40677414,53393358,17385531,72278539,31126490,74137926,51507425,29959549,98130363,67227442,28851716,49271185,10453030,45848907,42692881,73168565,48658605,35092039,7517032,62803858,79738755,53842979,66903004,74724075,28796059,9603598,82178706,59197747,4508700,39986008,88251446,68816413,82979980,46851987,16567550,90013093,77413012,51715482,64055960,74743862,42644903,15535065,37560164,14326617,66116458,31904591,22721500,92867155,13862149,76540481,75986488,22766820,53632373,59981773,94911072,70372191,91938191,6986898,44889423,62496012,92554120,32426752,29834512,11161731,26397786,71333116,68041839,56461322,53648154,43501211,30163921,51472275,65081429,38256849,7687278,97940276,24915585,43322743,59109400,18131876,23740167,8733068,84002370,49328608,26114953,33431960,12348118,12024238,66663942,65017137,66137019,83789391,12303248,20867149,30543215,61373987,98371444,19457589,73222868,4095116,874791,21673260,66319530,16971929,77620120,1431742,97783876,8696647,29029316,54058788,92692283,44245960,9860195,38645117,5111370,86543538,64157906,55247455,82532312,77377183,86460488,79322415,68000591,77300457,79191827,59329511,67281495,77187825,168541,68875490,9257405,90310261,3880712,70727211,82779622,40534591,10264691,21001913,6497830,45407418,33237508,31491938,76330843,99524975,23569917,86118021,20645197,16380211,5073754,6525948,17016156,11581181,35996293,30395570,92353856,8129978,18466635,59501487,22942635,52261574,43933006,51047803,9575954,92604458,5822202,72373496,15064655,69255765,84840549,15948937,51426311,33628349,72357096,45996863,47893286,76960303,29994197,67031644,84684495,33336148,75052463,40781854,33123618,4515343,99658235,72274002,53888755,26139110,3233569,54014062,61859581,44664587,29466406,42199455,48395186,44348328,46870723,4806458,92398073,54868730,62428472,88444207,29516592,45990383,32151165,22200378,3294781,95251277,75128745,43045786,7685448,43357947,1788101,98948034,59614746,7300047,6819644,31161687,18699206,10597197,16424199,71083565,66611759,8913721,81853704,57359924,74614639,83083131,40356877,80555751,60106217,13470059,70541760,94360702,96709982,81172706,98648327,22435353,20140249,8055981,16387575,45794415,61623915,78909561,32250045,17146629,19101477,92787493,13348726,22405120,86821229,74441448,33565483,83533741,40197395,69579137,34497327,4199704,39373729,98943869,38341669,65271999,92215320,37739481,99125126,57240218,40686254,44627776,36933038,8373289,61712234,18783702,67451935,37192445,29065964,1239555,65275241,99729357,97011160,37166608,71965942,97641116,91957544,96726697,71522968,86301513,9829782,77312810,53547802,14045383,35456853,99971982,6111563,47213183,66428795,9188443,23194618,91802888,36189527,99690194,26507214,78218845,57961282,99861373,26063929,30811010,26744917,80316608,29188588,52204879,92803766,5970607,44481640,50702367,79136082,44060493,77301523,70004753,14723410,55753905,82886381,66885828,98739783,62232211,79942022,8128637,36753250,56484900,16405341,54263880,71469330,7066775,1022677,30463802,28734791,92530431,48893685,7182642,2331773,34946859,28787861,90158785,4204661,56515456,32699744,44473167,84127901,90457870,59405277,72777973,49597667,38494874,37957788,89217461,17539625,59910624,17037369,69848388,30366150,89804152,39201414,83747892,69697787,95010552,90272749,67030811,86022504,30653863,36139942,91255408,68316156,14220886,72019362,96906410,65038678,45790169,90004325,3088684,42967683,24058273,89499542,47708850,27411561,61960400,42073124,21070110,51315460,69136837,15148031,30218878,22113133,36845587,20002147,37675718,99021067,26493119,17894977,47738185,61859143,6038457,90090964,62693428,61928316,83133790,30569392,13231279,52293995,4069912,48774913,67513640,45995325,49236559,57658654,6157724,57393458,82651278,65715134,29635537,37224844,66832478,55770687,6871053,76170907,80251430,22879907,73109997,83651211,16595436,41442762,84293052,54517921,52060076,37280276,33935899,39847321,20836893,17857111,40865610,7646095,53666583,34236719,85023028,88130087,71300104,36468541,36312813,27325428,3235882,29819940,83948335,45667668,48260151,13468268,72495719,48675329,21919959,58224549,8115266,68102437,85711894,6505939,71920426,44846932,69641513,17058722,3633375,56424103,30139692,72238278,48673079,8791066,75229462,47298834,72738685,74452589,52613508,36808486,81774825,26734892,50007421,90654836,15075176,18001617,84904436,23110625,24168411,26776922,26664538,78300864,62490109,44915235,5832946,9599614,11757872,98653983,77694700,57163802,83269727,73021291,6521313,92071990,55960386,60393039,33249630,41245325,1375023,77898274,97379790,20681921,93359396,19272365,21397057,415901,54232247,90061527,176257,97057187,93790285,96318094,58751351,67124282,99604946,58749,83302115,85695762,44177011,45428665,61728685,61897582,17727650,24733232,9860968,65047700,32274392,27187213,28550822,75407347,84349107,21289531,55850790,63506281,11365791,41092172,89699445,47792865,4787945,82327024,34044787,29401781,70782102,6221471,33895336,24619760,72358170,61141874,68204242,50668599,41092102,55615885,23134715,84166196,76434144,36396314,11923835,24953498,7502255,94595668,45306070,57020481,19898053,89811711,94711842,67084351,45919976,15163258,32058615,33553959,95581843,4091162,97281847,96420429,50806615,38510840,38009615,13819358,96852321,8634541 +12348118,96193415,75153252,44245960,70367851,33201905,74110882,20885148,23194618,35092039,22108665,61859143,75407347,53842979,57803235,81677380,97561745,86821229,27665211,6986898,21993752,99524975,70541760,9623492,52293995,75543508,9860195,46851987,62496012,17068582,83533741,32159704,17058722,77312810,10309525,55753905,15535065,39801351,93270084,15536795,85571389,37192445,41092172,99297164,77284619,14349098,62693428,6808825,80014588,61271144,55247455,50702367,26114953,84293052,33628349,74614639,38008118,9398733,20642888,68644627,24591705,28851716,17976208,61373987,62936963,91281584,67793644,8913721,22129328,16971929,5111370,64087743,84349107,94539824,69355476,63967300,21533347,30694952,6793819,33797252,18699206,79821897,97011160,19891772,16019925,64848072,77272628,95957797,66322959,47792865,96726697,59177718,3233084,26139110,89214616,99515901,68204242,79136082,34432810,44784505,31904591,39847321,89804152,47893286,3880712,23848565,75777973,29819940,50806615,6153406,96852321,88653118,70727211,87720882,80251430,20140249,60581278,26998766,33237508,30787683,91727510,31126490,93790285,19486173,50007421,92541302,91957544,79806380,99224392,60106217,7104732,38658347,44842615,40984766,82779622,18833224,56473732,45919976,24314885,8696647,89419466,44060493,17957593,13173644,44348328,50188404,76540481,2208785,79657802,98739783,36580610,40356877,68824981,16380211,17539625,68875490,3509435,3183975,25842078,61982238,88047921,97641116,71083565,36753250,74441448,65851721,30569392,71657078,72725103,51715482,67474219,98371444,7919588,81898046,13862149,40152546,9603598,57241521,45790169,32058615,61741594,58751351,45990383,99965001,56531125,48892201,60430369,7687278,83210802,34295794,20836893,11161731,53888755,6505939,98462867,78549759,61897582,16424199,81274566,24733232,81172706,4095116,34236719,11543098,83789391,32161669,10366309,73168565,16054533,75128745,37560164,73124510,94516935,7423788,22435353,36139942,38365584,92554120,38341669,53084256,22879907,37280276,1431742,22766820,62357986,27325428,7685448,54263880,62740044,71333116,44889423,88444207,40677414,73222868,64055960,27041967,19101477,92998591,43376279,65271999,18783702,78602717,14220886,62051033,38061439,37891451,34946859,67030811,91831487,32250045,44177011,68316156,72373496,321665,91990218,90310261,17727650,92803766,65047700,94360702,13468268,19939935,17037369,26776922,97940276,37620363,39986008,45237957,168541,28928175,37739481,69848388,47708850,91664334,29516592,43357947,89499542,31727379,82979980,70596786,526217,65454636,77620120,2917920,36930650,4798568,11923835,81805959,415901,22997281,66663942,77377183,54014062,54762643,23569917,60102965,70800879,82339363,61623915,14093520,9886593,29401781,1204161,91802888,61815223,10358899,97057187,73031054,30395570,44983451,42644903,40197395,2717150,51047803,85116755,7502255,33249630,8128637,9188443,89811711,26734892,64602895,59614746,26292919,28796059,6525948,29510992,10597197,42947632,49236559,21070110,36396314,44550764,83133790,36685023,55669657,63152504,53632373,49598724,6157724,87160386,15064655,72019362,96709982,84840549,71316369,54199160,37224844,17365188,5832946,68816413,37183543,42692881,40865610,54868730,6871053,18663507,72278539,4434662,62552524,95395112,59910624,49271185,45428665,10961421,44844121,16387575,61141874,39373729,28734791,82532312,20486294,40781449,96906410,93562257,23110625,21673260,73235980,508198,72738685,29466406,65275241,48260151,8651647,3088684,68102437,62452034,57658654,84684495,43322743,72732205,27411561,42199455,99549373,52613508,17081350,77413012,7066775,99658235,36780454,24058273,88130087,30163921,54606384,56153345,63015256,43506672,99604946,78909561,42307449,92867155,55189057,4119747,71965942,47090124,45995325,43933006,83651211,8129978,54058788,48893685,14947650,54987042,93057697,4668450,6521313,20645197,42782652,78785507,79942022,87710366,29994197,72777973,73109997,36505482,24915585,7229550,67281495,11365791,66271566,65074535,36845587,14626618,99021067,59197747,98948034,91255408,96420429,24168411,76315420,86543538,8373289,6497830,69697787,61859581,69786756,34493392,30218878,55143724,4099191,33935899,21397057,69255765,3233569,66832478,77301523,51464002,66250369,90090964,29932657,75135448,669105,14045383,61263068,66116458,59109400,53547802,20002147,56484900,60955663,94711842,45794415,26397786,89046466,68702632,34497327,58749,89078848,26102057,56461322,81853704,84002370,8055981,48658605,35192533,62803858,94090109,80555751,87598888,44473167,65017137,26392416,46540998,74743862,9860968,18806856,45049308,33431960,95010552,75986488,67031644,38645117,66611759,33123618,2891150,53802686,90061527,77787724,67227442,70420215,39201414,22942635,92398073,63506281,99861373,33304202,41481685,1788101,39171895,98943869,79922758,38494874,12571310,59501487,75229462,30543215,26744917,45075471,22721500,44847298,7517032,6221471,37501808,13231279,64098930,75052463,92033260,8733068,93566986,98648327,28787861,76170907,45848907,37166608,16445503,90272749,92604458,65715134,3773993,29959549,50567636,30366150,51590803,78589145,40534591,30891921,73021291,1022677,83155442,54427233,78561158,82726008,44481640,62428472,19376156,31161687,51472275,42967683,49882705,3294781,8791066,569864,50668599,874791,3633375,59371804,83083131,4069912,85242963,20122224,26063929,45306070,12416768,86118021,52060076,40686254,55770687,98130363,49597667,36312813,1569515,69641513,47738185,30998561,67124282,92692978,36812683,11581181,15163258,36808486,64157906,16684829,53393358,66137019,82052050,16595436,48774913,55615885,68041839,6038457,84187166,21919959,76330843,10453030,15948937,32590267,17764950,72274002,90013093,86798033,68939068,53666583,51426311,33699435,16405341,57961282,749283,99729357,24826575,34667286,4064751,9575954,66428795,26507214,9259676,15902805,92787493,28550822,30764367,68128438,65081429,78218845,55470718,57359924,41245325,7955293,10649306,20568363,9829782,16567550,4787945,57240218,65038678,74357852,5073754,70782102,20867149,67084351,85711894,92692283,95290172,79880247,1197320,62430984,93359396,21289531,5970607,41092102,69579137,37675718,85695762,40027975,76434144,20765474,66885828,23740167,17857111,81855445,77694700,57248122,73617245,92353856,34698428,7011964,52261574,29834512,75820087,12024238,43045786,63628376,1250437,47213183,30811010,79191827,3487592,63372756,37659250,26493119,42073124,26863229,72357096,14731700,65338021,84166196,54663246,73392814,33565483,824872,15015906,45667668,39553046,93053405,76960303,86022504,80316608,8099994,72495719,33336148,91141395,7300047,247198,48675329,11757872,33061250,32426752,86460488,33895336,26275734,90004325,29065964,1375023,17894977,47361209,19457589,24953498,15148031,54232247,99971982,84127901,94911072,35456853,69605283,12303248,77072625,51588015,49641577,51315460,48088883,56424103,88251446,52204879,91240048,69136837,76703609,4985896,63059024,73786944,35512853,92530431,36189527,17386996,53648154,56515456,97281847,16097038,66045587,94076128,19272365,74137926,112651,38022834,9257405,95581843,32151165,48673079,14326617,24619760,61960400,12664567,22200378,42237907,1239555,34698463,8634541,58208470,4204661,176257,54517921,51466049,14723410,71522968,51507425,60176618,88904910,93933709,90457870,41380093,31491938,59318837,68000591,84904436,55602660,89996536,83368048,70036438,82327024,26664538,36468541,66319530,99917599,86001008,13470059,18466635,32274392,62490109,8115266,17016156,6111563,92215320,31733363,99333375,30463802,30139692,89699445,78300864,45617087,33553959,18001617,83150534,17146629,27185644,68694897,88698958,68110330,74452589,7646095,77898274,96735716,59329511,13819358,44479073,74724075,37957788,22450468,82886381,96318094,45996863,76671482,29188588,37435892,71469330,97379790,62762857,46870723,85117092,5267545,9599614,15075176,9058407,44846932,48360198,77300457,61728685,43501211,30771409,62232211,79322415,29029316,66903004,72358170,95726235,21001913,20681921,77187825,83302115,18504197,83269727,71300104,8807940,94595668,8729146,44915235,97783876,16099750,98653983,85023028,76825057,9175338,7182642,57020481,72238278,35996293,86301513,61712234,4508700,47298834,30653863,4806458,66667729,14363867,4091162,49328608,59981773,36135,30090481,19898053,23432750,43152977,80246713,90158785,38256849,18411915,6819644,99226875,72793444,47887470,25636669,93515664,57393458,17385531,67513640,40781854,82427263,3235882,1936762,22113133,5640302,32699744,71920426,84406788,27187213,82651278,82178706,34044787,91938191,27625735,2331773,90654836,70372191,89637706,81774825,48395186,57163802,45407418,78766358,99125126,95251277,83948335,70004753,23134715,2607799,44627776,13348726,82897371,4515343,18131876,86242799,79738755,60393039,10264691,55850790,62115552,41442762,5482538,38510840,89217461,92071990,72614359,61928316,51149804,4199704,38009615,22405120,67451935,2204165,45381876,78884452,65880522,56955985,5822202,44664587,99690194,36933038,4978543,58180653,55960386,59405277,58224549,83747892,29635537 +65851721,94516935,65454636,58208470,4119747,67281495,33628349,20885148,55247455,89637706,41380093,56531125,91255408,80251430,70800879,28851716,82427263,54427233,97281847,54762643,24733232,99861373,17727650,26392416,44784505,29994197,61741594,168541,30653863,45790169,66045587,97940276,96193415,3233084,70036438,86001008,38061439,77300457,92398073,89078848,29959549,43045786,87160386,69786756,89214616,30218878,37739481,9860195,4508700,81677380,19101477,53842979,15064655,321665,27325428,88047921,33201905,45996863,77312810,99658235,112651,35456853,22721500,59501487,18783702,7919588,62693428,46540998,5970607,99524975,96726697,17081350,44473167,30543215,21673260,70004753,66832478,1022677,57240218,30139692,20122224,89699445,61373987,63372756,70596786,57248122,43933006,18663507,97561745,75543508,60955663,1239555,50806615,52060076,5111370,71657078,43357947,80555751,43322743,64848072,54517921,98948034,33565483,44245960,33797252,81853704,40677414,36930650,93790285,1788101,17037369,65715134,93515664,85571389,68644627,57803235,48260151,12348118,94076128,12664567,56515456,29635537,53888755,45306070,98943869,40865610,65880522,37957788,669105,62452034,91957544,78549759,77413012,34698463,14093520,48360198,92033260,51047803,49236559,14326617,50188404,29401781,55753905,97011160,30811010,526217,20140249,38494874,26493119,62496012,88130087,7502255,93566986,38022834,53666583,4668450,30569392,84904436,9886593,88444207,9398733,48673079,24591705,15015906,99965001,13231279,47708850,749283,10366309,35092039,95581843,80014588,15536795,34946859,75986488,57359924,75153252,62490109,93933709,37183543,98739783,47298834,8696647,27665211,42782652,34698428,99125126,34295794,66250369,9860968,74110882,36780454,874791,96735716,6111563,2204165,79657802,44889423,4099191,39801351,21533347,65038678,3235882,90457870,16405341,33553959,7104732,82886381,6153406,16097038,95957797,61271144,83210802,27041967,94711842,64087743,53648154,40686254,6221471,74743862,75777973,80316608,18466635,69641513,68816413,24619760,90004325,69848388,91802888,36505482,9188443,89811711,1569515,99604946,11923835,90090964,53632373,77272628,58749,86543538,23110625,56473732,32161669,13470059,65047700,67031644,28734791,45428665,6986898,21001913,63967300,29819940,79191827,30787683,62936963,29834512,68102437,20642888,8791066,89046466,16684829,5267545,88653118,79136082,93359396,59910624,53802686,59109400,54606384,56955985,9058407,15535065,22108665,84349107,24826575,44479073,64098930,20867149,88251446,51315460,57393458,17058722,26292919,86798033,51464002,3088684,48893685,45919976,1431742,60581278,37192445,60430369,37435892,40534591,93562257,39847321,19486173,59318837,73617245,67124282,75820087,13862149,61141874,63506281,59371804,39201414,43506672,7685448,36685023,76671482,11365791,44842615,82651278,14349098,61897582,40152546,20486294,62051033,90272749,41481685,96709982,37280276,8115266,88904910,55850790,16424199,24915585,82779622,37675718,20681921,28928175,38008118,9175338,83651211,66611759,77377183,55470718,78300864,47887470,73021291,58751351,78602717,38365584,44983451,68041839,12416768,69605283,26114953,38009615,10453030,14731700,48675329,30771409,70782102,72777973,85711894,72274002,91831487,15948937,17068582,8733068,26998766,26063929,49597667,52293995,49641577,50007421,82979980,1204161,96906410,3880712,21993752,3773993,54868730,17894977,12571310,8913721,8128637,31733363,18699206,3509435,84187166,2717150,26102057,73222868,45049308,1250437,84127901,16971929,45667668,66903004,91281584,89499542,66885828,46870723,4515343,18833224,4985896,72725103,98130363,72357096,81898046,45848907,94595668,57961282,37620363,71920426,23569917,62740044,42199455,61859143,84840549,71300104,16595436,63059024,73031054,7182642,77284619,32699744,4199704,78589145,9599614,85116755,92530431,31904591,55602660,3183975,76825057,9623492,36468541,19457589,48774913,40027975,98648327,30090481,98653983,15075176,54199160,2331773,81172706,8651647,19939935,86821229,22942635,36933038,94090109,97641116,52261574,33123618,84406788,68939068,53547802,29188588,35512853,78884452,33431960,46851987,10309525,51507425,22450468,95726235,83155442,29029316,97783876,95010552,92554120,45237957,76170907,13173644,72373496,52613508,20836893,47213183,64157906,73168565,68702632,39171895,65271999,74357852,56484900,99297164,72738685,44481640,65081429,62762857,44627776,83789391,44915235,95251277,91240048,76540481,48892201,51426311,47738185,6157724,43501211,23848565,30463802,21070110,60106217,66319530,76315420,62428472,5073754,4798568,73235980,78766358,8055981,60102965,44060493,39986008,82052050,77620120,79880247,29466406,38341669,17539625,44846932,6808825,73124510,66137019,85695762,16567550,54663246,95395112,21397057,6505939,35996293,67227442,18806856,33061250,86301513,67793644,85023028,51472275,77301523,2208785,84293052,19891772,93057697,63015256,65074535,17365188,31727379,44550764,69355476,43376279,92604458,37659250,63152504,99224392,61623915,30764367,9829782,7229550,72614359,40356877,68128438,76330843,92215320,27185644,7687278,65017137,55189057,11757872,39553046,62357986,26776922,66271566,35192533,57658654,36808486,79806380,39373729,68204242,83533741,53084256,28550822,4064751,63628376,4091162,10358899,44348328,90310261,86460488,74441448,75052463,99690194,44177011,56461322,22405120,26139110,98462867,45794415,38658347,18001617,55669657,247198,65275241,19898053,569864,19272365,20765474,1197320,93053405,40197395,23194618,59329511,66667729,56424103,84684495,48088883,33336148,47361209,36580610,66322959,33304202,24058273,44844121,824872,67513640,83269727,89217461,51715482,49271185,4806458,10597197,84166196,33249630,2917920,36312813,44847298,20002147,27625735,68316156,99021067,42307449,73786944,25842078,26744917,37891451,62803858,34667286,10961421,99226875,9603598,87598888,34044787,68875490,68824981,55770687,92803766,78218845,22200378,27411561,69136837,67084351,66428795,34432810,71083565,86118021,3487592,11161731,8373289,32159704,99549373,91990218,89419466,17857111,18131876,38645117,42237907,82726008,96318094,65338021,59177718,508198,26664538,68694897,30366150,96852321,92692978,17957593,92787493,61263068,18411915,61859581,45995325,14363867,69255765,33237508,4095116,17386996,14220886,17016156,50668599,30694952,21919959,58180653,80246713,92867155,22766820,36189527,15148031,6525948,72278539,79821897,1375023,42692881,40781449,8129978,28796059,17976208,45617087,61928316,15902805,53393358,36812683,90654836,5822202,23432750,47090124,8634541,83302115,7517032,55615885,37501808,86242799,89804152,4069912,98371444,16099750,16387575,6038457,415901,66116458,51466049,24168411,71965942,45990383,20568363,81274566,22113133,76960303,34236719,74452589,7646095,41092102,61982238,75135448,54058788,81855445,94360702,9257405,36135,42073124,14947650,6521313,9259676,92692283,30395570,75229462,11543098,62232211,92998591,36396314,99729357,51588015,7423788,70541760,97379790,67451935,24314885,77187825,33895336,71333116,176257,54014062,58224549,91141395,74724075,59197747,41245325,47792865,50702367,76434144,31126490,51149804,70727211,64602895,3294781,13348726,61960400,82339363,81774825,77072625,91727510,86022504,34493392,83747892,48395186,55960386,99917599,36753250,5832946,66663942,84002370,70367851,6497830,36845587,31161687,55143724,68110330,33699435,72732205,49882705,74614639,72793444,70420215,92541302,26507214,12024238,14045383,77787724,57241521,83368048,40984766,2891150,16380211,15163258,26734892,72019362,59614746,8807940,72238278,90158785,20645197,72495719,64055960,17764950,4204661,22997281,10264691,62430984,91664334,32590267,49598724,78785507,81805959,2607799,45075471,92071990,48658605,83083131,28787861,82327024,78909561,22879907,71316369,54987042,25636669,69697787,29932657,29516592,71522968,3633375,36139942,5482538,61712234,60176618,13819358,73392814,7955293,69579137,30163921,30998561,74137926,78561158,67474219,6871053,61815223,38256849,85242963,32058615,7066775,16054533,52204879,18504197,57020481,34497327,8729146,83150534,87720882,59405277,99333375,29510992,14626618,19376156,32274392,22435353,70372191,16019925,4978543,26863229,61728685,42967683,68000591,79942022,23740167,6793819,45381876,4787945,90013093,67030811,33935899,51590803,41442762,26275734,79922758,1936762,83133790,54232247,91938191,54263880,4434662,23134715,3233569,75407347,37224844,79322415,94539824,94911072,79738755,72358170,95290172,71469330,17146629,13468268,76703609,5640302,30891921,62115552,26397786,82532312,97057187,32426752,40781854,99971982,83948335,92353856,7011964,50567636,62552524,77898274,11581181,88698958,99515901,9575954,82897371,17385531,32151165,24953498,57163802,16445503,82178706,45407418,42947632,59981773,87710366,37166608,49328608,6819644,7300047,56153345,90061527,44664587,93270084,32250045,27187213,60393039,10649306,77694700,14723410,8099994,75128745,37560164,21289531,31491938,89996536,22129328,42644903,43152977,85117092,96420429,47893286,38510840,41092172,12303248,29065964,73109997 +19939935,19891772,29819940,78602717,8807940,45075471,68816413,33061250,39171895,92541302,26392416,97783876,63967300,73235980,3773993,99658235,40197395,12348118,55602660,28851716,669105,66832478,96726697,25842078,39801351,2717150,81172706,97561745,18783702,49641577,66250369,38022834,17068582,60102965,1431742,8696647,99224392,56461322,91957544,23848565,62496012,51715482,38061439,35192533,7517032,42782652,77312810,19457589,44889423,4119747,60955663,62051033,6505939,95395112,29959549,22108665,69697787,8099994,16971929,30998561,2204165,30764367,83155442,43322743,4099191,66045587,79880247,65074535,91727510,44842615,72357096,12416768,6153406,20486294,67474219,43506672,65275241,23569917,9860195,69786756,35512853,93562257,73392814,79821897,27665211,85023028,99515901,37620363,99861373,57240218,34493392,62430984,14349098,4985896,26507214,61815223,17016156,59910624,75777973,76540481,67281495,22942635,7955293,15948937,97379790,43933006,73124510,40686254,44479073,93270084,98948034,65017137,1204161,48260151,73031054,84293052,14947650,85695762,77284619,36189527,4806458,5832946,93515664,77898274,8733068,82886381,58180653,37501808,68824981,85242963,72495719,66903004,29834512,29029316,40356877,45667668,90090964,36753250,69605283,73168565,45237957,55247455,11923835,55189057,62428472,92398073,87598888,4978543,51464002,56473732,59197747,32590267,29994197,19376156,13468268,4798568,48673079,21919959,25636669,11543098,2607799,7502255,69255765,36930650,68102437,81855445,73786944,71920426,45990383,81898046,99524975,78909561,23194618,97011160,90272749,86242799,98739783,61271144,63372756,569864,3233084,32161669,81274566,45996863,18411915,70004753,79806380,13470059,74614639,53547802,18699206,17037369,4204661,33553959,26734892,65454636,70036438,7229550,42073124,76960303,54987042,48088883,75820087,168541,36808486,33304202,78549759,7919588,26292919,4668450,28928175,64055960,56955985,60430369,16445503,67227442,82979980,62803858,51047803,112651,29188588,47708850,30811010,84406788,55960386,83210802,508198,45794415,29065964,70800879,53802686,89078848,64602895,79136082,42237907,99549373,59318837,55669657,68644627,70372191,64157906,20568363,77377183,58749,81677380,74743862,82897371,65880522,9886593,43376279,33201905,89419466,72777973,89637706,10366309,68694897,84684495,95290172,89811711,67513640,16019925,16595436,38365584,19486173,67451935,98943869,66116458,20140249,91990218,40677414,70727211,80014588,8115266,4434662,45995325,55615885,4069912,80555751,99965001,53648154,30139692,8729146,77272628,64087743,70596786,16387575,21289531,52613508,27411561,96852321,26863229,5111370,53632373,33336148,22129328,42644903,91240048,51466049,87720882,71316369,36505482,16424199,62740044,54762643,50007421,31904591,66137019,9599614,73617245,88653118,9398733,7423788,17386996,36312813,92692283,6986898,36580610,30543215,83789391,54014062,17081350,10358899,99226875,247198,37192445,45790169,99604946,49597667,49598724,30395570,54058788,19898053,66322959,99021067,13862149,8791066,40781449,27041967,57803235,91141395,24733232,22997281,37435892,55143724,83083131,8651647,52204879,62452034,76330843,86821229,24953498,75153252,44844121,23134715,54199160,22435353,97641116,65038678,33431960,50188404,31491938,92803766,3233569,82532312,61859143,77620120,3294781,89214616,92554120,19272365,73021291,88047921,51149804,17146629,89996536,37739481,57241521,96193415,83533741,62693428,69136837,34236719,38645117,70367851,71083565,20765474,80251430,28550822,91664334,71657078,13231279,526217,51315460,20867149,17539625,16097038,41481685,84349107,90004325,5640302,35092039,45306070,76703609,61728685,74137926,49271185,46540998,39553046,16099750,79191827,44983451,17058722,16054533,37280276,74357852,54606384,29466406,8913721,67793644,4787945,50702367,1788101,40152546,46851987,54427233,18833224,26275734,45848907,12571310,71333116,64098930,29932657,20885148,68316156,10649306,34698428,38341669,44846932,62936963,89804152,94911072,56515456,75052463,2208785,93053405,11365791,92215320,72278539,94360702,99971982,415901,94539824,85117092,87160386,71522968,34295794,76315420,36845587,39847321,94076128,24619760,78766358,85571389,84840549,33628349,34497327,17727650,53666583,48675329,36685023,3509435,98648327,1936762,46870723,66611759,22879907,65047700,68875490,68204242,91281584,52060076,7687278,35456853,99729357,4064751,64848072,874791,86001008,95726235,66319530,53393358,62490109,44550764,99297164,66885828,74452589,8634541,32058615,33237508,54517921,84904436,32274392,30163921,41092102,57393458,44481640,85711894,41245325,23110625,31126490,65338021,61960400,96318094,30090481,9058407,47361209,56531125,7104732,35996293,38008118,88904910,30694952,27625735,92787493,32159704,55753905,9257405,79657802,50567636,85116755,32250045,21673260,94090109,90061527,18504197,49236559,78561158,56153345,74110882,30569392,44627776,77072625,51507425,88251446,99333375,44784505,90310261,14093520,26744917,72019362,20642888,76170907,20122224,6111563,45919976,70541760,67030811,26397786,24168411,59177718,40781854,749283,63015256,97281847,43501211,23432750,72274002,98462867,13348726,6793819,56424103,22721500,36933038,59614746,55770687,91802888,7011964,33797252,86460488,14045383,63628376,61897582,34946859,1197320,26493119,17976208,53842979,3880712,3183975,61373987,88130087,8055981,11581181,58224549,22405120,42199455,84166196,32426752,44915235,92071990,22450468,77413012,7685448,60581278,54263880,47213183,97940276,76825057,92604458,39201414,21070110,37659250,92353856,37183543,93933709,38494874,47792865,72732205,63152504,36468541,41442762,6525948,37957788,5822202,83948335,67031644,47090124,29401781,17894977,10309525,72614359,41380093,79942022,17764950,78589145,78785507,31727379,9575954,78884452,72738685,51472275,10961421,14731700,43045786,45407418,59501487,75229462,53084256,26998766,21993752,15536795,65851721,39986008,321665,34698463,14220886,48360198,10597197,71300104,68041839,48658605,43152977,3235882,36396314,21533347,31161687,59329511,81853704,15163258,92692978,80316608,52261574,59405277,57020481,84127901,15075176,78218845,54868730,76434144,5482538,51590803,75543508,92998591,98130363,93057697,68000591,40984766,60106217,68939068,92033260,93359396,30787683,11161731,36780454,2917920,48892201,62232211,48395186,8373289,6497830,62357986,47298834,37166608,69848388,51588015,15148031,6808825,7066775,94516935,4199704,95957797,24591705,75128745,28787861,65715134,14626618,15535065,91831487,17857111,88444207,72238278,68702632,34432810,61263068,9623492,30653863,43357947,18466635,19101477,45428665,26776922,34044787,26139110,33123618,39373729,57248122,82427263,95581843,61928316,82726008,93566986,24915585,49328608,69641513,38510840,84002370,26664538,4091162,6521313,14326617,20002147,45381876,26102057,29510992,84187166,36812683,81805959,61741594,86301513,89046466,82178706,65271999,38658347,71469330,23740167,44847298,99917599,40534591,42692881,87710366,4095116,40865610,16684829,1375023,72373496,63059024,83651211,61623915,3487592,52293995,90158785,4508700,31733363,60176618,30366150,30771409,44348328,69579137,74441448,30463802,18001617,65081429,90457870,30891921,47887470,98371444,75986488,33935899,61141874,59109400,24826575,58751351,66663942,93790285,9829782,8128637,57359924,62115552,99125126,59371804,44664587,37675718,90013093,44473167,6819644,15064655,62552524,41092172,83150534,18806856,15902805,91938191,18663507,66667729,11757872,6871053,27185644,42307449,74724075,9603598,37560164,83302115,24058273,17385531,28796059,9175338,99690194,76671482,12024238,5267545,63506281,61859581,37224844,61712234,38256849,82651278,2331773,29516592,82779622,83747892,71965942,4515343,96735716,176257,16380211,38009615,77694700,7182642,15015906,66428795,86022504,22200378,9860968,6157724,69355476,47738185,98653983,44245960,27187213,824872,73222868,16567550,90654836,88698958,1250437,50806615,28734791,36139942,57961282,7646095,48774913,1569515,20681921,62762857,89699445,89217461,75407347,29635537,42947632,72358170,92530431,79322415,54663246,49882705,22766820,54232247,21397057,16405341,24314885,55470718,40027975,57163802,82339363,8129978,45049308,9188443,66271566,2891150,33895336,33699435,26063929,80246713,32699744,70782102,5970607,50668599,14723410,30218878,53888755,18131876,92867155,70420215,61982238,97057187,14363867,6038457,79738755,42967683,96420429,44177011,86798033,91255408,82052050,83368048,45617087,17957593,1239555,75135448,77300457,77301523,95010552,20645197,47893286,6221471,27325428,67084351,58208470,67124282,36135,26114953,86543538,68128438,13173644,48893685,3633375,94595668,12303248,81774825,73109997,79922758,12664567,96709982,10264691,1022677,72725103,33249630,68110330,10453030,72793444,7300047,95251277,57658654,56484900,17365188,33565483,21001913,32151165,82327024,37891451,55850790,83133790,77187825,96906410,94711842,3088684,22113133,78300864,44060493,9259676,51426311,86118021,77787724,60393039,34667286,83269727,59981773,5073754,89499542,13819358,20836893 +42782652,44473167,62496012,66250369,7502255,29188588,10358899,17081350,69605283,84684495,10366309,38494874,569864,70004753,77694700,98371444,29994197,65851721,16097038,824872,24733232,49641577,63152504,34295794,13231279,4099191,23569917,8791066,6808825,77284619,73021291,93057697,29510992,56424103,44245960,56531125,71920426,82178706,81677380,51047803,72777973,72274002,9886593,55850790,18783702,77312810,33201905,34698463,73222868,36505482,59371804,20486294,88444207,14220886,98648327,45237957,90090964,95290172,3633375,14326617,88047921,23848565,26292919,61982238,68041839,73235980,96193415,4119747,93515664,13470059,75543508,38061439,41245325,33061250,9188443,45790169,24915585,63506281,44842615,40197395,17037369,93933709,27411561,79821897,86460488,74110882,32699744,26863229,50806615,29959549,55470718,66832478,34946859,81172706,18663507,35996293,65880522,18806856,20140249,7229550,37620363,79657802,48892201,92398073,44550764,56153345,9860195,73392814,12348118,41481685,83150534,94516935,86022504,9575954,24168411,30771409,44177011,24619760,72725103,98130363,44889423,68816413,87160386,6111563,81855445,98739783,82886381,54517921,14093520,98948034,55753905,64087743,61741594,57803235,77787724,55247455,99515901,83789391,63967300,44627776,6525948,66116458,30395570,46540998,84904436,17068582,54014062,53842979,70800879,78602717,49236559,81853704,33304202,70036438,68644627,89078848,52060076,83651211,31904591,82726008,63015256,58208470,93053405,96726697,36753250,28928175,34493392,68875490,33237508,15535065,43045786,66322959,59981773,40686254,72495719,63372756,30218878,33797252,89637706,91957544,24826575,43506672,38658347,20836893,77072625,47213183,68204242,55143724,8055981,57359924,50668599,32161669,31126490,74743862,35456853,21533347,60393039,321665,82979980,70596786,6038457,85116755,87720882,16019925,89214616,35192533,19486173,60430369,95395112,67281495,56515456,56955985,29029316,17727650,94539824,91831487,42199455,66428795,37739481,50702367,22942635,92787493,33431960,99917599,29466406,79322415,69848388,99125126,8696647,5970607,37675718,1239555,99224392,89419466,3233084,38009615,22721500,51472275,50188404,29819940,92692978,74452589,71083565,72019362,30543215,26998766,41380093,80251430,247198,27325428,65715134,76540481,48893685,82339363,1569515,45306070,1788101,33565483,68102437,11543098,16099750,7011964,1204161,36312813,92604458,75153252,65038678,77300457,83133790,44784505,4508700,53393358,13468268,78766358,26734892,86118021,42237907,45381876,88251446,70367851,25636669,36780454,57393458,27665211,41092172,62051033,61271144,61141874,14947650,89499542,7066775,20867149,54762643,29635537,83302115,48673079,30787683,88653118,749283,55189057,83533741,68694897,82327024,669105,14349098,10453030,62693428,19101477,68000591,73031054,29834512,99021067,62740044,9623492,90310261,60102965,32590267,1250437,49328608,18466635,40152546,4434662,12664567,39801351,15015906,10309525,39201414,26102057,526217,51588015,42967683,77620120,58749,15064655,32274392,44847298,7646095,60581278,6221471,69355476,97561745,96852321,26392416,67474219,80555751,61815223,62430984,77413012,37891451,99971982,9603598,4515343,16054533,43501211,30366150,38341669,34698428,64055960,41442762,22200378,92071990,36580610,62762857,63059024,42644903,33628349,69579137,86001008,59109400,65271999,30653863,20122224,35092039,84166196,44479073,9860968,99524975,86798033,8128637,48260151,82651278,45407418,95581843,37501808,15148031,45428665,30463802,99861373,3294781,72793444,2204165,65017137,77272628,21993752,7517032,43376279,508198,96709982,55770687,24314885,74357852,44983451,47298834,45848907,91990218,8115266,49271185,59177718,30998561,33699435,82427263,16971929,8729146,40781854,67124282,2331773,62452034,90272749,36930650,4668450,51464002,17365188,49597667,39553046,9398733,4985896,99549373,48360198,81274566,30811010,59318837,72357096,70541760,97940276,72358170,62936963,61623915,91938191,16405341,5073754,36135,78300864,59501487,92530431,71469330,62232211,36685023,64602895,5482538,50007421,76434144,53648154,54987042,65081429,66663942,99297164,7104732,26139110,9599614,71316369,1375023,59910624,83269727,82897371,3487592,16595436,39373729,38022834,45617087,40677414,73168565,20885148,90013093,53666583,80246713,8913721,2917920,92033260,86543538,61859143,3183975,30569392,63628376,47361209,31727379,84127901,40984766,20642888,22405120,26114953,78884452,47792865,22113133,91255408,7687278,44060493,64848072,84349107,57658654,45996863,97011160,8373289,48774913,40356877,39847321,99965001,19272365,3088684,61373987,4064751,65074535,26507214,28851716,3880712,23194618,66885828,78589145,54427233,95010552,85023028,89699445,85711894,6871053,94711842,45794415,92554120,2891150,95957797,22129328,79136082,62490109,91802888,11923835,94595668,46851987,22108665,11161731,37183543,94090109,91141395,6986898,87598888,41092102,93790285,58224549,14626618,89811711,874791,60106217,20681921,19898053,99658235,24058273,54199160,8651647,10597197,19891772,51590803,16380211,76170907,81898046,26776922,67031644,76315420,24591705,75407347,69255765,12416768,69641513,3235882,76960303,38365584,67030811,99729357,53802686,62552524,79922758,7423788,47708850,38256849,66137019,30139692,66611759,86301513,80014588,98943869,57240218,38645117,96318094,7955293,95726235,83210802,61712234,69697787,16424199,64157906,8733068,54868730,49598724,61960400,15163258,2208785,75777973,4978543,32426752,14045383,28550822,39171895,34236719,78785507,5267545,16567550,97281847,73617245,55602660,16445503,66319530,33553959,17957593,53888755,91281584,13173644,68702632,20568363,62428472,56484900,53547802,79880247,42073124,91240048,96906410,71657078,88698958,96735716,17764950,5822202,40781449,28787861,32151165,67227442,5111370,52261574,89046466,98462867,68939068,1431742,68316156,91664334,18699206,168541,22879907,37560164,67793644,34044787,65047700,92803766,48658605,33249630,57163802,30694952,34497327,21289531,76703609,83747892,45919976,10961421,62803858,415901,96420429,72373496,56473732,47887470,54232247,26493119,51426311,93359396,93562257,30764367,43933006,58751351,71522968,84840549,67084351,71333116,79191827,42307449,69136837,69786756,42692881,65454636,92998591,11581181,28734791,17385531,26275734,55669657,75820087,75128745,17894977,20645197,74614639,14731700,70727211,37192445,37659250,6153406,61928316,19376156,65338021,70372191,74724075,11757872,22435353,19939935,93270084,53084256,57020481,21070110,49882705,59197747,50567636,89804152,91727510,78549759,27185644,81805959,78909561,79738755,35512853,55960386,29065964,23432750,1936762,90004325,13348726,84002370,14723410,43322743,31733363,79806380,85242963,27187213,6793819,16684829,76825057,20002147,97641116,32159704,37224844,21673260,44481640,15075176,43357947,75135448,23134715,7919588,45667668,10649306,66271566,37435892,7685448,54263880,4204661,40027975,32058615,68110330,17146629,9175338,82052050,75052463,15536795,26063929,98653983,3509435,45990383,89217461,45075471,51507425,79942022,51315460,44664587,93566986,54663246,78218845,18411915,92867155,70782102,1197320,2717150,36812683,22997281,44844121,13819358,68128438,29932657,72732205,61728685,6157724,33895336,45049308,21397057,59329511,53632373,66667729,61859581,48675329,99604946,88904910,85571389,52204879,76330843,52293995,68824981,45995325,84293052,8634541,36808486,9829782,7300047,4095116,99226875,70420215,29401781,83948335,85695762,87710366,77377183,77301523,33123618,72278539,73109997,60955663,97783876,12571310,40865610,90654836,84406788,32250045,29516592,4798568,89996536,6497830,30090481,92353856,59405277,46870723,19457589,38008118,58180653,9058407,27625735,6505939,112651,10264691,36933038,74137926,17386996,80316608,54606384,6521313,2607799,94360702,5832946,24953498,92541302,22450468,97057187,36396314,57961282,51149804,83083131,90457870,44348328,82532312,8807940,40534591,1022677,72738685,21001913,25842078,36189527,77898274,72238278,86821229,20765474,78561158,92692283,23110625,65275241,31161687,4091162,74441448,51715482,17539625,4199704,22766820,39986008,18131876,3773993,26397786,6819644,66045587,95251277,55615885,61263068,17976208,44846932,51466049,4069912,37957788,90158785,47090124,27041967,42947632,26664538,52613508,57248122,17857111,67451935,17016156,86242799,88130087,7182642,66903004,72614359,71965942,34432810,26744917,71300104,47893286,75986488,30163921,61897582,18504197,17058722,18001617,92215320,56461322,83368048,83155442,33935899,99690194,37166608,9257405,16387575,15948937,30891921,18833224,36468541,94076128,11365791,13862149,34667286,44915235,12024238,99333375,33336148,36845587,9259676,73786944,176257,5640302,64098930,14363867,8129978,28796059,77187825,97379790,84187166,15902805,62115552,47738185,59614746,73124510,90061527,4806458,3233569,43152977,31491938,57241521,54058788,76671482,82779622,4787945,94911072,60176618,48395186,12303248,48088883,8099994,62357986,23740167,37280276,36139942,75229462,38510840,81774825,85117092,21919959,67513640 +45919976,22942635,56515456,86001008,44481640,88653118,35456853,95957797,8651647,8634541,526217,61373987,50188404,57961282,90310261,97783876,29932657,97281847,112651,11543098,8696647,56424103,61859143,68000591,84293052,83210802,26392416,59109400,73124510,98371444,62452034,73222868,36780454,36933038,19272365,36808486,70367851,36189527,2917920,5970607,31733363,73168565,39801351,4668450,62803858,31904591,8807940,36312813,7011964,33249630,35996293,6808825,85117092,68204242,92554120,16380211,64157906,53547802,70420215,62496012,93933709,79657802,60106217,61263068,36812683,18783702,26114953,54199160,57163802,24733232,48892201,56531125,69641513,76703609,53888755,5640302,99125126,26063929,20002147,37183543,53393358,14093520,77898274,63059024,39847321,99861373,65081429,55189057,9886593,14947650,44177011,82178706,50702367,39986008,66611759,36685023,34698463,68644627,48893685,61141874,49328608,82979980,95726235,66667729,19939935,73786944,88698958,42782652,91255408,24591705,29994197,38008118,415901,96318094,71522968,81853704,21397057,36580610,68110330,5482538,93270084,70372191,43376279,36505482,17857111,62490109,16019925,64098930,81855445,17016156,7104732,52293995,45790169,75543508,6221471,53802686,29065964,76170907,32159704,16445503,83533741,37620363,34295794,44842615,6153406,40865610,94076128,14349098,247198,7182642,59501487,49597667,17764950,19486173,69136837,84904436,41380093,91957544,5111370,85242963,32426752,15536795,9829782,77284619,60102965,4798568,44550764,45407418,18806856,72614359,40027975,84349107,569864,61897582,51047803,94711842,43501211,9603598,72738685,4099191,27665211,99604946,71657078,14220886,17976208,98462867,83302115,3773993,43322743,74357852,80555751,33895336,16387575,34698428,71965942,36753250,72373496,44060493,96193415,80014588,24058273,26664538,77312810,85023028,18699206,75407347,53084256,23134715,51472275,5832946,38061439,61982238,67281495,55770687,71333116,17365188,3880712,76434144,37739481,77187825,76330843,78766358,10358899,44889423,27187213,80246713,65038678,14045383,1788101,84187166,21070110,3088684,94539824,65017137,39171895,7517032,30771409,47090124,37957788,62552524,71300104,12024238,13348726,28787861,31491938,99690194,72777973,98739783,58749,90013093,55470718,95395112,28928175,28550822,13173644,59329511,92803766,7955293,25842078,76540481,77694700,38365584,11161731,49641577,34493392,29819940,61960400,66832478,77413012,70036438,18131876,13470059,39553046,92033260,72793444,15064655,33123618,45848907,99515901,93359396,58208470,34432810,82052050,41092172,99021067,89078848,11923835,16567550,13819358,30218878,23740167,3294781,45428665,18663507,41092102,9575954,83368048,45667668,49882705,99549373,21993752,15535065,1375023,93566986,48774913,67793644,90457870,32590267,34044787,23569917,88444207,22129328,60430369,63967300,14731700,26744917,64055960,3235882,96906410,2331773,61271144,13231279,74441448,77787724,874791,12416768,37166608,89499542,92604458,16405341,55669657,68824981,6793819,45996863,6986898,38494874,84840549,78589145,30694952,74137926,23848565,37192445,64602895,47213183,93057697,16097038,13862149,9188443,57359924,33336148,99917599,43933006,53648154,70541760,36930650,12664567,69848388,59318837,37891451,15148031,70004753,66137019,6505939,19376156,94516935,79738755,77300457,64087743,10649306,17727650,86821229,26139110,66250369,29401781,96726697,16971929,58751351,96735716,30764367,48360198,90004325,90061527,40686254,54663246,6819644,30366150,66428795,89214616,9257405,11581181,45990383,54014062,65338021,40152546,22405120,33797252,6038457,98653983,77301523,52060076,88047921,53666583,68816413,8055981,65880522,87598888,64848072,3633375,18833224,33237508,72238278,57248122,73031054,2607799,91990218,81898046,51315460,97561745,95290172,62430984,18504197,93790285,95581843,67451935,10366309,10961421,24619760,81805959,59177718,38645117,41442762,32161669,79806380,97379790,54762643,3233084,97011160,33304202,35092039,33565483,21919959,17385531,53842979,44245960,73235980,20885148,30395570,65074535,78300864,40677414,85116755,42967683,8129978,86022504,15948937,29834512,32250045,79191827,82779622,31126490,27325428,95010552,28734791,18411915,49236559,18466635,69355476,72357096,50668599,7687278,1569515,76671482,48675329,66885828,4434662,50007421,80316608,67124282,33201905,72019362,37501808,44983451,83133790,71083565,10597197,43045786,60393039,47738185,96420429,51426311,19101477,78218845,55602660,89637706,7919588,15163258,1022677,61815223,27411561,9623492,98648327,92530431,20867149,69255765,57803235,86460488,42307449,82651278,72495719,92787493,19898053,97940276,33628349,79942022,66045587,98943869,4806458,6157724,90272749,4204661,9398733,7646095,74452589,1204161,22879907,98130363,46870723,87710366,34236719,73392814,62051033,4787945,12348118,82339363,89046466,70596786,34667286,48658605,17058722,7502255,29959549,94090109,83789391,51590803,54232247,1250437,89419466,72725103,37659250,75777973,45995325,28796059,50806615,2891150,1936762,29466406,63152504,43152977,24953498,17037369,66663942,22113133,22450468,4508700,99524975,83083131,23194618,61859581,26493119,824872,37675718,97057187,98948034,55247455,19891772,36139942,99658235,39373729,30139692,87160386,77272628,26507214,20681921,15015906,83150534,37280276,75986488,70727211,44348328,30569392,55960386,61623915,92692283,97641116,508198,26998766,54606384,62936963,321665,9259676,87720882,44473167,52261574,70800879,45075471,60581278,21533347,8128637,29510992,30998561,17146629,91141395,7300047,7423788,91831487,5267545,63506281,79922758,65275241,48395186,40534591,11757872,94911072,16424199,29635537,7685448,1431742,9599614,57020481,45617087,80251430,91727510,91802888,37224844,83948335,81677380,51464002,81274566,28851716,2717150,62693428,71920426,33431960,96709982,11365791,20486294,43506672,26292919,78602717,24915585,30163921,94595668,90090964,62762857,4064751,1197320,69786756,56484900,42073124,68939068,84127901,63372756,47298834,35512853,59197747,39201414,8733068,31161687,51507425,68128438,8729146,5822202,26397786,17068582,20765474,72274002,4119747,22997281,51588015,84684495,21001913,57658654,1239555,61928316,73617245,669105,35192533,85571389,6497830,66903004,14626618,5073754,55753905,68102437,72732205,55143724,2208785,54427233,72278539,8913721,42692881,82327024,77072625,4199704,93562257,92692978,83269727,25636669,59910624,59371804,57393458,4515343,93053405,95251277,63015256,59981773,54263880,88904910,53632373,57241521,9058407,24826575,68316156,17957593,168541,75135448,68702632,52204879,45237957,62232211,74743862,93515664,22766820,79136082,43357947,45794415,68875490,88251446,22435353,54868730,8791066,40781449,42644903,82726008,29516592,67227442,56473732,76960303,30787683,75229462,79821897,91664334,40356877,78909561,84166196,99729357,20836893,44627776,17539625,81172706,26776922,40781854,34497327,16099750,38009615,14326617,8115266,20568363,32699744,44846932,3509435,33553959,92215320,15075176,71316369,67031644,51715482,99297164,44784505,74110882,26734892,41245325,8099994,76825057,77620120,62740044,74724075,89811711,58224549,52613508,60955663,91938191,68041839,6521313,71469330,38341669,4095116,61728685,42947632,33699435,27625735,18001617,22108665,69605283,65715134,82897371,99965001,38022834,3487592,749283,51149804,9860195,45306070,82427263,55615885,72358170,89699445,4978543,36396314,86301513,47361209,20645197,29029316,8373289,16595436,30090481,19457589,92998591,96852321,45049308,40197395,20642888,67030811,99333375,6871053,14363867,83155442,38658347,90654836,37560164,60176618,33061250,3183975,92071990,44844121,30463802,56461322,48673079,89804152,30811010,56153345,59405277,10309525,176257,92398073,86798033,36135,69579137,21673260,50567636,48260151,40984766,89996536,36468541,4091162,86543538,22200378,92541302,65047700,77377183,67084351,44915235,59614746,65851721,13468268,23432750,89217461,83651211,82886381,7229550,32058615,10453030,47708850,99226875,79880247,34946859,83747892,74614639,86118021,44479073,33935899,65271999,22721500,67513640,62357986,38256849,56955985,47887470,84002370,3233569,91240048,16054533,42237907,27041967,62428472,15902805,30891921,63628376,4069912,48088883,82532312,78561158,38510840,27185644,9860968,66322959,26863229,46540998,54058788,92867155,10264691,54987042,47893286,92353856,66319530,17386996,47792865,65454636,30543215,67474219,12571310,66116458,46851987,85695762,75153252,84406788,91281584,54517921,78785507,66271566,26102057,75128745,94360702,61741594,29188588,24314885,68694897,49598724,78884452,99224392,31727379,20140249,44664587,37435892,20122224,88130087,85711894,69697787,78549759,2204165,99971982,76315420,49271185,21289531,81774825,6525948,70782102,23110625,86242799,9175338,90158785,6111563,55850790,75052463,58180653,26275734,17894977,32274392,62115552,41481685,73109997,79322415,14723410,4985896,30653863,44847298,57240218,36845587,61712234,16684829,7066775,51466049,12303248,42199455,17081350,73021291,24168411,75820087,45381876,32151165 +72725103,91727510,98371444,27665211,84406788,96193415,44889423,95957797,50702367,22450468,29065964,95395112,44348328,88653118,37891451,77787724,29834512,89419466,82052050,51464002,15015906,44983451,62452034,4668450,90457870,10649306,9259676,28796059,33431960,49882705,6793819,47213183,46851987,70420215,7955293,23848565,22129328,16445503,83155442,23194618,60102965,83210802,6505939,77284619,56484900,68824981,84684495,8129978,53802686,99515901,66663942,40984766,62740044,16054533,2717150,35192533,82726008,45407418,1431742,42692881,95290172,73124510,19939935,99549373,669105,7423788,6525948,321665,75135448,42199455,9575954,61263068,99971982,18833224,14220886,81853704,30543215,20568363,32250045,25636669,96318094,69136837,99917599,415901,47090124,4515343,34698463,61960400,39801351,65851721,63152504,45790169,33699435,74724075,31904591,62552524,17068582,35456853,93053405,37435892,9623492,79191827,89214616,37620363,88047921,81898046,69355476,49328608,18806856,13470059,44479073,32590267,45990383,57163802,61982238,91802888,18663507,74137926,62762857,48892201,11543098,6808825,83150534,10366309,20140249,16971929,20002147,44481640,59614746,4798568,61271144,33304202,76434144,77272628,15535065,78602717,93270084,94516935,42782652,72274002,54987042,45919976,21993752,65275241,18411915,68939068,22108665,39986008,59318837,61623915,39201414,508198,45049308,98648327,11161731,75986488,9886593,72738685,55247455,87710366,64087743,62803858,3633375,18504197,59371804,99524975,68204242,33797252,99658235,5970607,77620120,54663246,40197395,10309525,44245960,75407347,14947650,68702632,91990218,52613508,40677414,76540481,3773993,96726697,55753905,39553046,15148031,87720882,38645117,82178706,30891921,55470718,74110882,33237508,83368048,89996536,72732205,38008118,65017137,749283,51588015,34698428,89637706,67793644,17764950,54762643,8729146,37659250,43376279,55770687,44784505,40686254,44842615,55669657,33628349,72373496,65038678,247198,56515456,74441448,66137019,31161687,44473167,12571310,70367851,59197747,84187166,26139110,35092039,5111370,26776922,66250369,42947632,30771409,26863229,65715134,58180653,12416768,45848907,80246713,82979980,4069912,33249630,53547802,99333375,29959549,14626618,91664334,79821897,93057697,13173644,29932657,54199160,84840549,83533741,7685448,78589145,4099191,53393358,76960303,8807940,43322743,51426311,20885148,6038457,9188443,43933006,20867149,44177011,59910624,63967300,34493392,7646095,62496012,18699206,97783876,50188404,38494874,21397057,93933709,53084256,71083565,26392416,36580610,98462867,16019925,29510992,1197320,33061250,46540998,1788101,65338021,91957544,51149804,32274392,81677380,14349098,17957593,92787493,85116755,42967683,82339363,57803235,19376156,68816413,5073754,3294781,44550764,51507425,30395570,75153252,67030811,48088883,28734791,27411561,29188588,7011964,60430369,1569515,45075471,78218845,7919588,51047803,3880712,52293995,77187825,91831487,94090109,19272365,96906410,30569392,54517921,79942022,87598888,62430984,89046466,85117092,29466406,78561158,3233569,60176618,36812683,56153345,36468541,94911072,3487592,92803766,77301523,50668599,22942635,94076128,40027975,67451935,12348118,97379790,3183975,8128637,61897582,71657078,31733363,92692283,78785507,52261574,61859143,37675718,9058407,69255765,15064655,63506281,98653983,55960386,53648154,59177718,97641116,36685023,16380211,83789391,72019362,51590803,77694700,32159704,92554120,6153406,47361209,61373987,57241521,82897371,27185644,72614359,68000591,75229462,99021067,79922758,3233084,21533347,44844121,24314885,99861373,70372191,37280276,40781854,66045587,4064751,8099994,24591705,26063929,88251446,25842078,64055960,57359924,39373729,83269727,73222868,61141874,4204661,64157906,31727379,39847321,86118021,8791066,55189057,66885828,94595668,66832478,14363867,99729357,8696647,97561745,29819940,17365188,54868730,75543508,48360198,28928175,63015256,37183543,20765474,26998766,50567636,82886381,98948034,92998591,73617245,33201905,65074535,92530431,26744917,73235980,63628376,1204161,41245325,84349107,72495719,51315460,99226875,18466635,37560164,12664567,22766820,4434662,85242963,65081429,29994197,78300864,9599614,59329511,98739783,9860195,65880522,76671482,78766358,90013093,65271999,70727211,6497830,3509435,7182642,22435353,92692978,71920426,49598724,26734892,71469330,49597667,89078848,38061439,82327024,30218878,65047700,66428795,36505482,48260151,7229550,56461322,59501487,15075176,34667286,9603598,44847298,61815223,17894977,64602895,49236559,30463802,34044787,85571389,24058273,77312810,41092172,1250437,14045383,92604458,59405277,18783702,41380093,92071990,30163921,16387575,70541760,30787683,93359396,81805959,48658605,79880247,54014062,9257405,73168565,38341669,17976208,92398073,34432810,4978543,60581278,97281847,88444207,29029316,22113133,71333116,23134715,77898274,70800879,33553959,54232247,74357852,51472275,84127901,2607799,61859581,34295794,43152977,89804152,2891150,62490109,49271185,16595436,13468268,49641577,67474219,10961421,56424103,68041839,42644903,72777973,37166608,67227442,17037369,57658654,67281495,80316608,64848072,47708850,59109400,41442762,14093520,50007421,98943869,66611759,44060493,93562257,38658347,82427263,69697787,20642888,24915585,84293052,80014588,46870723,10358899,30998561,29516592,71316369,40356877,19891772,53842979,51715482,6986898,5832946,21289531,51466049,45995325,53888755,66319530,39171895,56955985,27625735,93515664,7502255,79657802,57248122,99604946,94711842,7300047,17146629,16099750,42307449,70036438,83948335,90272749,62936963,55143724,26507214,24168411,5640302,26275734,99297164,6221471,2208785,95726235,66322959,88698958,24733232,86821229,89499542,44915235,76703609,26114953,14326617,80251430,16097038,17058722,80555751,33935899,69579137,91141395,15536795,55602660,89217461,40781449,2917920,95581843,43501211,31126490,74743862,24826575,63372756,58749,28851716,20645197,68110330,9175338,76315420,11365791,8651647,72357096,52204879,86001008,58224549,37192445,45428665,56531125,36753250,41481685,15948937,48673079,12024238,53632373,7104732,60106217,4806458,2204165,30139692,83083131,79806380,34946859,72278539,36312813,35996293,97057187,14731700,57961282,68644627,50806615,8373289,91255408,36930650,17016156,60955663,27187213,3088684,94360702,33895336,45617087,36808486,4095116,34497327,76170907,23740167,99965001,72238278,83133790,95010552,43357947,1022677,18131876,44846932,68128438,30764367,5822202,569864,85023028,99224392,68102437,20486294,90090964,27325428,112651,38365584,75128745,94539824,26664538,41092102,82779622,3235882,63059024,81855445,66903004,56473732,70004753,91281584,26493119,14723410,86301513,19486173,92033260,45237957,36135,32161669,17385531,24619760,15163258,70596786,86022504,76330843,17539625,96852321,98130363,8634541,61728685,16567550,8115266,20122224,526217,57020481,81774825,23432750,37739481,37501808,4119747,71300104,78909561,6111563,17857111,4199704,81274566,54606384,7687278,73031054,8055981,61712234,81172706,16684829,30090481,8733068,69641513,13862149,57240218,32426752,5267545,24953498,64098930,77072625,73109997,74614639,84904436,21070110,9860968,37224844,87160386,92353856,45381876,82651278,52060076,82532312,75820087,34236719,33565483,47893286,6871053,75777973,96420429,90004325,10264691,66271566,44664587,36933038,45794415,71522968,13819358,30366150,36189527,2331773,43045786,6157724,78549759,61741594,42237907,4508700,58751351,17386996,90310261,85695762,42073124,75052463,72793444,66667729,20836893,79738755,68694897,11581181,33123618,92541302,76825057,1375023,176257,86460488,4985896,15902805,54058788,62357986,84166196,37957788,62693428,88904910,54427233,38256849,32699744,6819644,23569917,32058615,19457589,21919959,13348726,26102057,92867155,12303248,67084351,29401781,55615885,66116458,26292919,84002370,23110625,22721500,9398733,72358170,48893685,73392814,19101477,77413012,28787861,78884452,40865610,7517032,99690194,30694952,73786944,6521313,85711894,1239555,22200378,16405341,93566986,40152546,36780454,18001617,53666583,77377183,11757872,45667668,47792865,77300457,28550822,48395186,69605283,32151165,89699445,31491938,91240048,71965942,91938191,10597197,60393039,47738185,69848388,83747892,67124282,36845587,36396314,35512853,86543538,27041967,62232211,30653863,83302115,97011160,21001913,62051033,4787945,90061527,874791,45306070,9829782,90654836,68875490,16424199,22405120,43506672,96735716,11923835,93790285,90158785,10453030,824872,99125126,74452589,8913721,48675329,48774913,45996863,65454636,30811010,38022834,38009615,47887470,73021291,1936762,62428472,5482538,86242799,83651211,57393458,67031644,58208470,33336148,29635537,38510840,44627776,26397786,40534591,70782102,21673260,13231279,22997281,4091162,17727650,62115552,22879907,168541,20681921,7066775,89811711,61928316,17081350,68316156,36139942,59981773,67513640,79136082,47298834,69786756,86798033,95251277,79322415,92215320,55850790,97940276,96709982,88130087,54263880,19898053 +4668450,68702632,96193415,66250369,8807940,77312810,85571389,9886593,17764950,55470718,29834512,57241521,89046466,43376279,64087743,94539824,70800879,32159704,61263068,81805959,26734892,15015906,44784505,29959549,13231279,79821897,44348328,87710366,51047803,26292919,81677380,99658235,78589145,14349098,47361209,89637706,63967300,75543508,9860968,38061439,25842078,40152546,96726697,89214616,62051033,29510992,39373729,16019925,6793819,39801351,4099191,69641513,2717150,76960303,65851721,4119747,54427233,45794415,45237957,4434662,35092039,77272628,30764367,61859581,65275241,81853704,43501211,12348118,16595436,72274002,14947650,65074535,20140249,48774913,77300457,88653118,34698428,26392416,99515901,76315420,77301523,77620120,54058788,92692283,53802686,8696647,78561158,72019362,79136082,58208470,21533347,48675329,40356877,6525948,12416768,20486294,44473167,33431960,12024238,16445503,1204161,68128438,18504197,82886381,84187166,19486173,19939935,31126490,80014588,6808825,53547802,64848072,112651,22766820,73031054,53632373,56531125,20765474,34497327,57240218,17957593,93933709,80316608,89499542,75777973,62936963,70596786,4095116,92554120,54517921,95957797,85116755,74743862,2891150,14723410,44889423,47090124,51588015,83948335,49236559,82427263,8129978,67793644,61373987,95726235,30891921,80251430,68939068,92215320,1569515,49641577,68041839,4798568,74724075,9623492,17365188,89217461,5832946,51426311,57803235,91664334,17058722,61271144,72357096,82339363,91957544,75229462,31161687,94911072,16567550,93562257,73235980,98943869,98462867,36780454,78909561,16097038,18699206,13862149,57658654,51464002,34236719,569864,85023028,86821229,79942022,9575954,33304202,40686254,4064751,75153252,79880247,40027975,88251446,55753905,47792865,10309525,43045786,46851987,97940276,247198,63059024,59197747,8115266,30787683,62693428,47708850,97561745,63372756,7685448,9259676,27665211,28734791,168541,26102057,24915585,10358899,71333116,27325428,28851716,92604458,32161669,91802888,14045383,54987042,30694952,39201414,53393358,90310261,44481640,8729146,98130363,83533741,92692978,8055981,6986898,44844121,36930650,68644627,70541760,9860195,98648327,83789391,98371444,92398073,96318094,90013093,24058273,26998766,21993752,50668599,33201905,55669657,66116458,8791066,44842615,58180653,40781449,33237508,52293995,22405120,80246713,60430369,39847321,72725103,66271566,4985896,68204242,22942635,41481685,749283,55247455,16054533,45996863,74452589,45848907,16099750,60955663,60106217,74357852,62740044,72614359,29635537,20836893,78549759,82651278,35192533,65454636,92998591,14731700,56424103,37957788,49597667,66319530,54232247,94090109,46540998,66137019,14093520,17068582,5640302,99524975,55189057,33699435,9175338,9829782,20867149,37620363,73786944,65880522,46870723,42644903,17037369,32590267,91938191,669105,57359924,69697787,62552524,16684829,30543215,99861373,73392814,66322959,59501487,42782652,39986008,74614639,61741594,7011964,51315460,99549373,13470059,1239555,6153406,45407418,10453030,36580610,59177718,10366309,99226875,87720882,90457870,31733363,26507214,1022677,30771409,1250437,55770687,58751351,49598724,59318837,78785507,29819940,59109400,81898046,37659250,16387575,78300864,76170907,95290172,7300047,79657802,22997281,29466406,51715482,26863229,78766358,79922758,19376156,65081429,40865610,30090481,67474219,32250045,69579137,9188443,37739481,64055960,26139110,44177011,15536795,29401781,6521313,508198,18783702,54663246,87598888,51466049,7502255,7955293,84684495,40197395,60581278,88444207,68694897,88130087,45428665,24733232,8128637,94516935,2917920,23569917,61859143,36808486,77787724,33797252,62452034,36685023,4069912,66428795,12571310,49271185,90272749,35456853,59614746,92033260,53888755,72738685,50567636,2204165,26776922,98948034,70004753,30139692,36505482,61623915,62428472,43322743,5111370,81172706,18833224,71316369,58749,93057697,30218878,87160386,20122224,8651647,31904591,874791,78602717,84293052,60102965,52261574,37192445,4204661,40677414,37891451,26493119,71083565,62490109,22113133,6505939,7066775,30811010,56515456,38256849,66832478,38341669,83150534,34295794,84904436,3294781,38022834,15535065,17976208,37501808,42692881,67281495,54762643,13348726,55602660,96735716,47213183,67031644,79806380,6497830,30395570,1197320,97379790,77284619,23194618,18001617,7646095,80555751,7229550,30366150,321665,5970607,75052463,78218845,8373289,84127901,33628349,36312813,31491938,38494874,37280276,30653863,3183975,65047700,76671482,28550822,61960400,65338021,22450468,65715134,1375023,99224392,29029316,19101477,52204879,69355476,66611759,8733068,82052050,3233569,49328608,20002147,68824981,74137926,73124510,24619760,50188404,56461322,90004325,19891772,92803766,47887470,3235882,70036438,53648154,54014062,11161731,73222868,33565483,29516592,28796059,38008118,81855445,33336148,62430984,14326617,85695762,29932657,91831487,77187825,69136837,99297164,91727510,97783876,2208785,69255765,45075471,85242963,43506672,82897371,4508700,20885148,14363867,96709982,71469330,89078848,34493392,83651211,16971929,69848388,56484900,37224844,48892201,54606384,44846932,40781854,48088883,61141874,68875490,51472275,45381876,43933006,76540481,34698463,68102437,86001008,23848565,4806458,7919588,61982238,54199160,71522968,36845587,98739783,50806615,29188588,82779622,30163921,83083131,60393039,78884452,95251277,37435892,29065964,3633375,38645117,15148031,83133790,63628376,21673260,11365791,99021067,70367851,66667729,97057187,88047921,72793444,29994197,82532312,19272365,20642888,52613508,83210802,42199455,9603598,84166196,97011160,86301513,37560164,33935899,11923835,28928175,55143724,45990383,6038457,92530431,4515343,72278539,66885828,93053405,77072625,62803858,17386996,40984766,41092102,34044787,27625735,26114953,41380093,68316156,14220886,73617245,63015256,10597197,19457589,76330843,50702367,88904910,3487592,42307449,17727650,24953498,4199704,68816413,93515664,38365584,75986488,3088684,43357947,68000591,44983451,3233084,38009615,86460488,90158785,91141395,93270084,59329511,89996536,30463802,94711842,9398733,11581181,99965001,71300104,36468541,67513640,48360198,53666583,70372191,66045587,3773993,45790169,76434144,85117092,62232211,37675718,99604946,66903004,74110882,97281847,71920426,45667668,64157906,7104732,20645197,35996293,4787945,86543538,415901,65271999,30998561,57961282,9257405,12664567,50007421,82178706,48893685,57393458,81274566,54868730,49882705,66663942,5267545,44550764,13173644,36753250,61712234,18131876,45306070,98653983,91990218,824872,55960386,526217,21070110,97641116,89699445,77413012,39171895,44847298,92541302,33061250,60176618,90090964,14626618,32151165,84002370,4978543,59371804,17539625,33553959,32274392,15902805,63152504,21289531,94360702,82327024,71965942,37183543,95010552,84406788,38658347,26744917,71657078,21001913,9599614,82726008,62357986,44627776,62496012,83302115,48658605,48673079,47893286,7182642,45049308,5482538,1431742,95395112,15064655,76703609,39553046,70782102,26063929,92071990,85711894,45919976,23432750,51149804,83269727,72358170,72373496,99333375,94076128,72732205,52060076,2607799,77377183,36933038,27411561,6871053,53084256,11543098,91240048,42073124,35512853,89419466,13819358,61728685,44245960,24314885,6221471,93359396,47298834,45617087,70727211,17894977,51507425,99729357,7687278,32699744,31727379,47738185,10649306,18411915,28787861,57248122,22129328,93566986,19898053,57163802,44479073,67124282,72495719,20568363,42947632,13468268,33249630,61815223,65038678,6111563,48260151,22721500,86242799,10961421,30569392,73168565,75820087,58224549,3880712,94595668,44060493,23134715,7517032,83155442,96906410,18663507,92787493,77898274,6819644,24826575,79191827,67227442,72238278,27041967,18466635,3509435,53842979,65017137,54263880,91281584,48395186,76825057,59910624,41245325,82979980,7423788,17857111,18806856,36135,68110330,9058407,23110625,89804152,34667286,17385531,5822202,16424199,32426752,86022504,51590803,8913721,15075176,90061527,17016156,33123618,89811711,75407347,43152977,67451935,86118021,69605283,36812683,16405341,77694700,1788101,72777973,8099994,16380211,34432810,25636669,44915235,96420429,33895336,75128745,15163258,79738755,42237907,27185644,22435353,91255408,23740167,70420215,84349107,90654836,1936762,20681921,44664587,64602895,24591705,73109997,17146629,99125126,92353856,92867155,21919959,83368048,42967683,21397057,55615885,27187213,75135448,99971982,11757872,62115552,26275734,93790285,84840549,99917599,86798033,22108665,36139942,81774825,61897582,12303248,56473732,6157724,41442762,69786756,56153345,176257,15948937,95581843,59981773,88698958,32058615,74441448,26664538,96852321,45995325,57020481,79322415,99690194,61928316,64098930,4091162,59405277,38510840,36189527,22879907,36396314,37166608,34946859,62762857,24168411,10264691,73021291,17081350,67030811,83747892,26397786,5073754,63506281,8634541,67084351,55850790,56955985,22200378,41092172,40534591,2331773 +9623492,34493392,22450468,28928175,99917599,5111370,2917920,39171895,74357852,16097038,20885148,94911072,70596786,59197747,19939935,6986898,12348118,95010552,57803235,97561745,17764950,99965001,92554120,83150534,52261574,4787945,29188588,51047803,68875490,63967300,93933709,85242963,7919588,37501808,1204161,78602717,37166608,56531125,37739481,48673079,25636669,74614639,86022504,54987042,71333116,61859581,73392814,13231279,33895336,66885828,27665211,81898046,8807940,89046466,66903004,65271999,68702632,38658347,26114953,95581843,76703609,66045587,3509435,8129978,7517032,49236559,96709982,87710366,84904436,94090109,97783876,73222868,10366309,24619760,42307449,1022677,36580610,34236719,86821229,99604946,98130363,29029316,81274566,20642888,32590267,98739783,35192533,1569515,37620363,48892201,89214616,80246713,73168565,37192445,88047921,61897582,16445503,28796059,29466406,43376279,78561158,77272628,29819940,40356877,17058722,89078848,24733232,36685023,90090964,64087743,15535065,26275734,4199704,64848072,51472275,247198,4099191,43045786,16099750,34432810,92998591,6221471,6793819,8696647,61982238,66271566,16595436,36312813,16019925,20645197,49598724,20002147,7646095,79922758,65338021,7955293,44479073,30891921,16424199,53084256,78766358,46870723,93270084,5832946,62115552,9603598,1431742,70800879,36780454,14326617,6808825,77377183,54014062,41092102,88653118,17068582,81855445,69355476,71657078,18833224,71316369,83210802,6505939,34946859,52613508,70372191,79191827,47738185,36189527,72777973,67030811,33061250,56473732,53802686,77620120,61859143,8099994,65715134,74743862,62430984,54199160,3773993,59329511,68041839,77413012,6819644,77072625,23194618,36812683,38494874,71920426,76540481,10597197,88251446,72238278,79942022,19898053,67474219,22108665,14626618,32426752,76434144,92692978,44889423,38022834,9257405,8729146,32159704,50668599,26397786,68204242,30163921,40865610,55602660,54762643,98371444,40027975,67793644,29994197,824872,26734892,82427263,58749,38008118,38341669,19457589,21993752,73031054,99549373,2717150,5267545,18699206,82886381,63059024,61271144,60430369,64602895,85116755,66322959,93515664,66611759,52060076,40197395,39553046,38645117,97940276,93562257,89419466,43357947,38510840,66667729,9599614,8651647,76960303,18131876,31904591,77312810,61928316,80316608,35512853,17727650,7229550,90272749,99971982,64055960,72495719,33336148,56461322,4119747,24058273,37675718,38061439,26507214,8733068,44784505,9575954,14220886,112651,72358170,94711842,23740167,91802888,31727379,5482538,83302115,19891772,81172706,55247455,33304202,55960386,43506672,26998766,26493119,33237508,92398073,79806380,20486294,86001008,99125126,73235980,22879907,59910624,45075471,34698463,10358899,18466635,68816413,88698958,32250045,62428472,73124510,29510992,48260151,26863229,88130087,36753250,40152546,53632373,30463802,95957797,72373496,32151165,81853704,7502255,41245325,96193415,66137019,44842615,50188404,65017137,62693428,50806615,55143724,9886593,87720882,99861373,28550822,31733363,21533347,72732205,15148031,9398733,90457870,62452034,70367851,49641577,88904910,44983451,47090124,78884452,9058407,95395112,89699445,49882705,93053405,68316156,82779622,42199455,53842979,73617245,78300864,70541760,31126490,99224392,77187825,55753905,19376156,87160386,68128438,11161731,28734791,15163258,57241521,72738685,62357986,29932657,30395570,18783702,83155442,65074535,74110882,9188443,36505482,63506281,66250369,75153252,45237957,94539824,16971929,92692283,33565483,37957788,36930650,45996863,38365584,30787683,26392416,34698428,22129328,61815223,60955663,62936963,22997281,76671482,83083131,6153406,77301523,20867149,95290172,32161669,1788101,9860195,35996293,66832478,1375023,17976208,17037369,93057697,99226875,23569917,33553959,51507425,79136082,4668450,54427233,6497830,37435892,83789391,61263068,669105,99515901,11581181,69786756,61373987,4095116,74452589,30139692,91281584,27411561,53666583,68644627,13468268,41092172,65275241,15075176,72019362,56153345,874791,83651211,16567550,27625735,76170907,49328608,70036438,40534591,30764367,16054533,18001617,45428665,54263880,98943869,12024238,34044787,92215320,54663246,37183543,54606384,17539625,94076128,61728685,29959549,42782652,15948937,72357096,62803858,68000591,77284619,84127901,80014588,30366150,51466049,84293052,3487592,44915235,18806856,72274002,75135448,44245960,30090481,53547802,85695762,83948335,1936762,3633375,44348328,50702367,40984766,42073124,90310261,20765474,45990383,65038678,45995325,88444207,36808486,81677380,69579137,72725103,62496012,7687278,26102057,44844121,42692881,64157906,60106217,48675329,34497327,15902805,14947650,87598888,42237907,82897371,67451935,9175338,96735716,48360198,30543215,98948034,63152504,72278539,92604458,46851987,37659250,95726235,73786944,75820087,11365791,92541302,44550764,99297164,55770687,70727211,22942635,46540998,1197320,21070110,71300104,84187166,67031644,19272365,29834512,26292919,45848907,39373729,79738755,55189057,35456853,43501211,53888755,47893286,10649306,98462867,79880247,43322743,96726697,51464002,90013093,3088684,86242799,23110625,8128637,4064751,69641513,66116458,57248122,49271185,62051033,58180653,74441448,16387575,37891451,78909561,5073754,98648327,84406788,30653863,2607799,17016156,92071990,36933038,91727510,56515456,44177011,176257,6038457,91664334,33123618,45790169,56424103,16684829,82726008,7685448,569864,20568363,89811711,34295794,82052050,19101477,89637706,749283,84840549,19486173,65851721,92033260,61623915,17365188,85023028,91831487,17385531,79657802,78549759,29065964,4515343,99524975,42947632,60102965,14723410,33699435,14731700,97641116,67084351,4806458,79821897,3183975,71083565,36139942,1250437,82979980,59371804,66428795,23848565,75543508,62232211,42644903,40686254,8373289,84684495,85571389,508198,13173644,13348726,54232247,74724075,75128745,92353856,25842078,76315420,7104732,3235882,68824981,22200378,10961421,18663507,61141874,41481685,12303248,2204165,35092039,90061527,23432750,415901,91240048,91938191,26139110,20140249,84349107,59318837,17386996,60176618,68694897,54058788,73021291,82532312,69697787,59109400,52293995,11543098,39801351,33201905,41442762,3294781,63628376,17894977,3233084,22405120,20122224,78785507,45667668,48658605,93359396,59177718,45306070,23134715,51315460,18411915,80555751,24591705,4069912,33797252,15064655,70004753,86301513,7423788,63372756,27185644,89804152,97057187,93790285,14045383,526217,71469330,2208785,77898274,45919976,28851716,48088883,60581278,75986488,62740044,84166196,85117092,99658235,57020481,51588015,97011160,85711894,26063929,21919959,4798568,6521313,96318094,56484900,33628349,91957544,54868730,52204879,86460488,55470718,96906410,7011964,17146629,75407347,48395186,64098930,91255408,95251277,37560164,99021067,5822202,27325428,92867155,67227442,53648154,78218845,6525948,30771409,42967683,89996536,78589145,17957593,26744917,51149804,30569392,47792865,15536795,31491938,44664587,57393458,11923835,8115266,30218878,40677414,45407418,30694952,77787724,6871053,86798033,83368048,92803766,58751351,71965942,48774913,57163802,91990218,17081350,57658654,63015256,86118021,96852321,55669657,3233569,9829782,97379790,5640302,43152977,82339363,74137926,81805959,27041967,54517921,13470059,44627776,15015906,24915585,7182642,57240218,45381876,76825057,10309525,30811010,21397057,27187213,91141395,89499542,33249630,59501487,168541,51590803,99690194,75229462,98653983,99729357,68939068,36468541,69255765,62762857,90004325,75052463,44060493,33431960,26664538,1239555,39986008,47298834,24826575,44847298,66663942,2331773,4091162,31161687,59981773,4978543,51426311,69136837,93566986,5970607,8913721,89217461,47708850,17857111,57961282,83133790,22435353,65047700,9860968,4204661,21673260,40781449,9259676,2891150,58224549,72793444,18504197,48893685,45617087,21289531,28787861,77300457,67124282,39201414,73109997,61960400,47887470,44481640,92787493,53393358,22113133,79322415,61741594,50007421,62552524,94360702,3880712,4434662,82327024,84002370,80251430,49597667,11757872,82651278,59614746,40781854,16380211,29401781,8634541,41380093,66319530,12416768,32058615,75777973,51715482,90654836,81774825,65880522,44846932,67281495,24953498,36845587,7300047,47213183,14093520,39847321,36396314,97281847,45794415,62490109,96420429,56955985,72614359,55615885,22721500,94516935,4985896,69848388,36135,38256849,32699744,38009615,8055981,24314885,43933006,92530431,68102437,69605283,34667286,29635537,47361209,8791066,7066775,57359924,26776922,22766820,99333375,14363867,12664567,24168411,14349098,55850790,70420215,59405277,37224844,12571310,10453030,44473167,4508700,86543538,65454636,21001913,60393039,50567636,13862149,16405341,321665,10264691,58208470,77694700,20681921,20836893,33935899,70782102,67513640,65081429,83533741,82178706,32274392,68110330,83269727,13819358,76330843,94595668,30998561,61712234,71522968,37280276,6111563,45049308,6157724,83747892,29516592,90158785 +37183543,86022504,33237508,65271999,1431742,31904591,56515456,81853704,53888755,67030811,66611759,23194618,63967300,29994197,13468268,8696647,53547802,73222868,4099191,83210802,13173644,15535065,67451935,68816413,37501808,23110625,6808825,247198,36780454,40686254,3233084,17365188,37166608,34493392,61263068,70036438,68204242,9599614,98462867,65851721,72777973,92554120,33553959,57241521,94911072,92692283,93053405,112651,69355476,77072625,30218878,69697787,6505939,66663942,54014062,41380093,98371444,96193415,65275241,7300047,26998766,32590267,49328608,12348118,81274566,42782652,72278539,55189057,30764367,12571310,65338021,77284619,95395112,11161731,82979980,78909561,62452034,45407418,99658235,93562257,19101477,83533741,93933709,81898046,96726697,99125126,29029316,89214616,90457870,91802888,89811711,57803235,66885828,2891150,51588015,70800879,31491938,38061439,39847321,22129328,7955293,57961282,3880712,83789391,33895336,61859581,39171895,87160386,55143724,73617245,63372756,44889423,66137019,56461322,9829782,33123618,47090124,38645117,95290172,66116458,66045587,24591705,43933006,25636669,61859143,93515664,37659250,68644627,84349107,70727211,88653118,29819940,30569392,23432750,45919976,39986008,83133790,23569917,43322743,39373729,15902805,9058407,53802686,1022677,59197747,61815223,73031054,16445503,64087743,36396314,72725103,43376279,19939935,33431960,99690194,61141874,29834512,36685023,68875490,43501211,89699445,54762643,10597197,10366309,19891772,54987042,6793819,69641513,44842615,62430984,41481685,65715134,14093520,44550764,9603598,59614746,49641577,14731700,84127901,45617087,44983451,9259676,4064751,45237957,37620363,5073754,50668599,84904436,61623915,83651211,20002147,17068582,55753905,97281847,14326617,8651647,92803766,71083565,66322959,99729357,65038678,34432810,77898274,62693428,45995325,92530431,61897582,58224549,45794415,53842979,93359396,53084256,61728685,91831487,15148031,17058722,29466406,45990383,76170907,9623492,72373496,42073124,8807940,93057697,97940276,56473732,16380211,9188443,69786756,66832478,78218845,75135448,73786944,63015256,95726235,84840549,7104732,81677380,51472275,2917920,36930650,90061527,40677414,74614639,2204165,76434144,64602895,85116755,60393039,51507425,31727379,42307449,53648154,79657802,17764950,74743862,99226875,13231279,44664587,32161669,16019925,60581278,90654836,12416768,37739481,7502255,89046466,68041839,36580610,77413012,78561158,669105,96709982,72793444,1204161,8129978,63628376,7229550,38658347,31161687,6521313,34295794,8791066,64157906,71333116,36753250,22108665,34497327,7011964,29635537,86001008,29188588,59318837,63152504,21070110,18001617,16567550,21533347,35456853,72738685,17957593,18783702,69255765,65017137,98948034,15064655,41092102,45306070,55960386,88047921,96735716,82339363,16099750,415901,95581843,45075471,36933038,92787493,4668450,55602660,874791,79821897,96318094,52261574,99549373,52613508,37675718,49271185,40781449,50188404,97561745,99604946,87720882,36468541,59910624,37957788,10264691,91255408,36808486,17539625,33304202,62803858,54058788,74137926,67793644,4069912,65047700,61928316,38494874,22942635,3509435,1788101,77312810,97641116,7066775,66903004,99965001,26114953,4095116,42692881,59177718,65454636,40197395,749283,52293995,72274002,94076128,89637706,35092039,7182642,83368048,92033260,80555751,24733232,81805959,6221471,45428665,55770687,20486294,90090964,79942022,30653863,63059024,60430369,33628349,47738185,73235980,56424103,71469330,44784505,48673079,40152546,57240218,11365791,16387575,92998591,71920426,3633375,569864,93270084,88904910,30366150,90004325,45790169,26664538,3487592,83302115,30543215,66271566,74110882,93566986,13348726,18806856,63506281,21919959,70782102,94090109,75229462,48088883,77300457,38008118,62762857,33797252,7919588,14045383,49236559,61271144,74724075,20681921,168541,92604458,73124510,34667286,29932657,52060076,99917599,84187166,16097038,92215320,71657078,62051033,33336148,78884452,19457589,27665211,88698958,38341669,78602717,6497830,24619760,8128637,5267545,26493119,28796059,9886593,88130087,95010552,75153252,82178706,26292919,98130363,67031644,89419466,19272365,66250369,99971982,51464002,50702367,81855445,55247455,91990218,32250045,21993752,80014588,62936963,43152977,94539824,36505482,76540481,16684829,80251430,50007421,6153406,7423788,36312813,14723410,99515901,62115552,28928175,79922758,61982238,8099994,20867149,52204879,44481640,5482538,51047803,321665,61741594,18699206,7646095,26392416,59501487,16971929,72357096,29065964,20885148,44348328,17016156,28550822,18663507,64848072,4787945,30771409,62232211,6525948,38365584,35996293,26734892,33249630,3235882,26863229,82886381,55669657,62496012,77620120,20642888,22450468,32159704,60955663,57658654,34044787,3773993,37560164,22879907,22766820,97783876,10309525,6157724,67227442,40356877,73168565,71522968,78589145,54232247,72019362,62357986,10358899,48260151,86118021,26776922,79136082,4199704,25842078,44245960,67281495,1250437,40027975,44060493,15163258,78766358,96852321,72732205,22405120,3088684,92353856,79806380,4985896,8729146,97379790,81774825,88444207,57393458,82726008,87710366,40534591,69848388,48360198,5822202,57163802,67474219,99524975,81172706,54199160,72495719,50567636,14947650,9257405,68316156,66428795,26063929,37435892,86798033,50806615,43357947,77187825,93790285,24168411,98648327,68000591,96906410,7685448,14349098,27625735,54606384,70004753,72238278,42199455,76315420,18466635,20568363,54663246,6986898,55850790,83083131,72614359,55615885,34236719,51149804,78785507,90310261,46870723,83150534,95957797,28851716,28734791,508198,74441448,26275734,59329511,824872,8373289,40984766,18411915,97011160,70541760,1375023,91664334,36812683,43506672,45667668,12024238,9860195,44844121,85117092,51466049,18131876,34698463,61960400,74357852,19486173,69136837,4508700,48395186,45996863,32699744,49597667,64055960,85711894,27411561,56531125,85023028,62740044,15536795,99333375,15075176,1197320,24058273,56955985,92541302,86242799,15015906,83155442,59109400,27187213,26397786,45848907,57248122,54517921,70420215,30163921,40781854,12303248,16424199,79191827,97057187,37280276,98943869,69605283,48675329,42947632,20122224,17727650,22721500,27041967,59405277,89217461,71300104,13470059,98739783,45381876,43045786,58208470,92398073,22435353,68702632,34946859,92692978,5970607,76671482,17894977,76825057,41442762,88251446,44473167,17857111,18833224,57359924,71316369,34698428,10453030,32058615,5111370,33201905,85242963,26744917,44627776,36135,99297164,99861373,91281584,29401781,4119747,86301513,77787724,77694700,39801351,22200378,42237907,21001913,46851987,41245325,20140249,42644903,4806458,29510992,91938191,30787683,54427233,75407347,8634541,77272628,9575954,68102437,8115266,16595436,66667729,30395570,36845587,22997281,86821229,73392814,59371804,17385531,85571389,16054533,79322415,526217,70596786,79738755,38009615,62552524,24826575,62490109,10961421,3233569,76960303,65880522,38022834,48658605,70372191,32426752,53632373,23848565,12664567,19376156,35192533,14220886,1936762,83269727,30139692,77301523,90272749,44846932,92071990,4798568,82427263,47361209,65074535,23740167,80246713,18504197,8913721,11923835,46540998,176257,95251277,30463802,91727510,51715482,41092172,72358170,66319530,1569515,75820087,21289531,64098930,94360702,70367851,11543098,76703609,60102965,78549759,99021067,42967683,24314885,58180653,38510840,54868730,82651278,26139110,58749,28787861,24915585,38256849,84166196,47298834,69579137,5832946,75777973,7517032,44915235,89078848,75986488,33699435,77377183,17037369,99224392,20765474,48893685,80316608,9175338,53393358,55470718,40865610,44177011,74452589,29959549,47708850,54263880,75543508,37192445,14626618,76330843,31126490,9398733,4091162,7687278,84406788,24953498,49882705,89499542,27325428,44479073,26507214,62428472,90158785,15948937,89804152,31733363,4204661,68939068,56484900,94711842,39553046,37891451,22113133,48774913,82327024,53666583,32274392,84684495,51590803,98653983,47792865,36139942,51426311,57020481,68110330,30998561,1239555,79880247,2607799,68824981,86543538,6038457,10649306,26102057,33061250,13862149,78300864,47213183,39201414,6819644,2208785,32151165,14363867,33565483,58751351,92867155,67124282,13819358,73109997,67084351,84293052,96420429,8055981,65081429,47893286,68128438,20645197,16405341,35512853,2717150,89996536,94595668,3294781,61373987,59981773,17081350,5640302,9860968,87598888,48892201,82779622,82052050,90013093,83747892,30090481,4515343,60176618,82897371,68694897,6111563,36189527,85695762,19898053,11581181,30811010,23134715,33935899,51315460,27185644,21673260,94516935,56153345,37224844,17146629,49598724,20836893,8733068,2331773,47887470,6871053,21397057,91141395,44847298,71965942,60106217,4434662,73021291,30694952,67513640,82532312,91240048,3183975,29516592,61712234,83948335,84002370,75052463,17976208,11757872,17386996,4978543,75128745,30891921,91957544,86460488,45049308 +26392416,45381876,36780454,71920426,16971929,36505482,60393039,508198,98130363,55753905,39171895,60102965,73031054,22721500,55143724,24733232,19939935,98739783,5822202,669105,77620120,6871053,41481685,81805959,18833224,49328608,16097038,75543508,49641577,33553959,90013093,10358899,30395570,58180653,24591705,99604946,37620363,37501808,33201905,69605283,82979980,13231279,32161669,57359924,56515456,51464002,27665211,31126490,77284619,2717150,4099191,26493119,15075176,65038678,67281495,65715134,74110882,18663507,29029316,9175338,51047803,74724075,96735716,73235980,93562257,44889423,34295794,96726697,95290172,569864,55470718,37659250,53666583,27325428,71333116,39847321,83210802,68875490,62693428,44842615,73392814,5832946,67227442,81855445,30764367,82726008,54014062,96193415,43376279,3773993,35092039,75820087,75135448,88444207,98462867,59329511,33304202,64848072,83789391,42782652,68939068,4095116,95726235,26734892,3233084,35996293,94090109,73786944,79657802,61859143,30653863,54427233,89217461,31904591,93270084,12416768,68694897,8696647,29834512,95581843,77312810,28851716,11161731,23848565,55669657,20486294,48774913,49271185,39201414,83302115,51466049,3633375,92398073,5111370,41442762,17068582,16019925,16445503,56955985,84166196,77694700,29466406,26863229,28928175,54517921,57248122,81677380,30787683,93053405,13470059,70782102,98948034,14349098,68702632,29959549,74137926,65851721,63967300,10366309,87598888,44479073,321665,2208785,31161687,9623492,247198,72238278,22997281,36312813,62496012,99965001,36580610,4064751,54762643,50668599,52060076,61373987,99524975,14626618,61271144,55602660,59981773,60176618,38061439,32159704,7502255,91664334,65074535,62936963,4806458,37192445,16099750,33565483,66045587,91831487,59910624,72777973,9886593,44473167,22942635,61712234,2607799,19101477,23432750,33628349,37739481,75986488,63059024,74743862,53802686,21673260,59197747,53632373,30463802,59109400,824872,36812683,95395112,92033260,9860968,17146629,84684495,30163921,6153406,82886381,87160386,8651647,12348118,40781449,40984766,68824981,64087743,46870723,22108665,70004753,72358170,68041839,54058788,70036438,69136837,30569392,38022834,66832478,93566986,98943869,37675718,6986898,22435353,36930650,99125126,89499542,22405120,6808825,42692881,90457870,35192533,62740044,47361209,45237957,15948937,112651,97011160,43933006,92692978,67474219,45617087,85116755,75153252,54232247,7300047,62430984,9398733,3487592,92692283,13819358,38494874,45996863,52293995,56531125,26998766,67084351,15535065,30694952,66250369,57241521,1022677,50188404,8115266,67513640,43501211,57240218,8807940,70727211,24314885,91281584,17016156,55960386,29188588,9058407,67793644,84002370,62115552,17957593,91802888,73124510,77187825,53888755,53393358,22879907,68644627,29994197,8055981,16380211,56153345,82178706,39553046,69848388,89811711,45407418,78561158,68204242,2331773,20122224,40197395,78602717,79322415,28550822,5482538,526217,55247455,61982238,6521313,40686254,77898274,66428795,8733068,49236559,28734791,76540481,66663942,43322743,36753250,65017137,86460488,72793444,17081350,86242799,24058273,8128637,45995325,29932657,34698428,47887470,63152504,17058722,30771409,22113133,2204165,83150534,39986008,85711894,37435892,14363867,31733363,42644903,88047921,44348328,71657078,89214616,76825057,65880522,61623915,48675329,749283,85242963,72373496,42199455,48360198,24826575,51507425,10309525,90158785,76703609,61728685,25842078,8129978,30139692,27041967,77300457,82897371,6038457,30998561,33431960,27625735,92353856,74357852,26063929,7011964,34698463,4119747,6525948,32274392,58751351,30218878,93933709,13468268,61141874,48260151,90272749,59371804,9599614,47090124,21993752,23134715,16054533,1569515,84904436,30090481,80246713,75229462,70367851,3294781,33797252,44177011,76960303,57803235,21001913,99297164,78766358,29510992,51426311,9829782,78884452,65275241,17764950,5970607,7229550,56424103,67451935,62803858,92215320,44245960,69697787,81172706,41380093,22450468,69255765,40677414,25636669,64055960,87720882,99226875,92554120,92867155,26776922,44784505,48673079,9860195,53648154,42967683,49597667,83651211,51149804,168541,16405341,82427263,45306070,51315460,61263068,33061250,99333375,89699445,32699744,4787945,97561745,45848907,37891451,73617245,2917920,17976208,31491938,98371444,33249630,7646095,54199160,87710366,91957544,66319530,34236719,18504197,20568363,1788101,20836893,62232211,1197320,72738685,17727650,77787724,8373289,14093520,47738185,71083565,93790285,4199704,64098930,44846932,75052463,89419466,65081429,89637706,50702367,3235882,14947650,7066775,86022504,88904910,3880712,23569917,33895336,7685448,47213183,47792865,10597197,77272628,58208470,78549759,76330843,69579137,59318837,47708850,24619760,20140249,66116458,66903004,86821229,72357096,12571310,14220886,51588015,45428665,8791066,8729146,72278539,26292919,44060493,92541302,19376156,74452589,6819644,83368048,62051033,85023028,18699206,60955663,36845587,34497327,94516935,7423788,6505939,94595668,57393458,61928316,75128745,19898053,72019362,86798033,86001008,71316369,51472275,84406788,99861373,89078848,80316608,66667729,30811010,67030811,99224392,45990383,83083131,63015256,9575954,29819940,40865610,62552524,65454636,50806615,63372756,30543215,44481640,39373729,77072625,99549373,99658235,40027975,63628376,72614359,33237508,38341669,53842979,54606384,17386996,9188443,89804152,61815223,41092102,18783702,98653983,39801351,18466635,46540998,3509435,61741594,64602895,54663246,62452034,13173644,26102057,50567636,79136082,20645197,40152546,57020481,91727510,63506281,38008118,16424199,32426752,86118021,66322959,4668450,85695762,70596786,88653118,91240048,78909561,4069912,21070110,18411915,36189527,86543538,67031644,29065964,19457589,47298834,44627776,65271999,57163802,80014588,18806856,29635537,76315420,66885828,69355476,42237907,38658347,38365584,10264691,14326617,88251446,15536795,18131876,70420215,61859581,23194618,90090964,52613508,32058615,57961282,83155442,98648327,66137019,2891150,9259676,48893685,97940276,71300104,17894977,1936762,26507214,84187166,81853704,94911072,38009615,51590803,7919588,93515664,68128438,36135,40356877,415901,53547802,17857111,68816413,78785507,93057697,60430369,66611759,70372191,43357947,1204161,94360702,88130087,26139110,60581278,71522968,6793819,57658654,34493392,48088883,24168411,44847298,53084256,83269727,32250045,59177718,5640302,44844121,15015906,5073754,44983451,20867149,30891921,82779622,55189057,72274002,70800879,36808486,20885148,96420429,27185644,56484900,72495719,73168565,93359396,34432810,97281847,79191827,26275734,16595436,54987042,32151165,73222868,40534591,45919976,90310261,8913721,7687278,37957788,21533347,3233569,34044787,14723410,33935899,83948335,49882705,49598724,89996536,76434144,81898046,45667668,82532312,68000591,79922758,78589145,79738755,1250437,91938191,94539824,38645117,16684829,33123618,4985896,12664567,37560164,45794415,94076128,55850790,66271566,37183543,32590267,92604458,71469330,68102437,7955293,26114953,96852321,79821897,61960400,41092172,61897582,6497830,45075471,9257405,41245325,52204879,15163258,75407347,44550764,65338021,92071990,58749,24915585,77413012,7182642,77377183,81274566,59501487,10961421,1239555,92803766,79942022,80251430,85571389,44664587,6111563,40781854,45790169,15148031,35456853,27187213,82339363,43152977,17365188,92787493,19891772,4204661,7104732,43506672,20681921,17037369,11923835,71965942,75777973,34667286,95251277,79880247,99917599,91990218,26664538,176257,69641513,76170907,43045786,18001617,46851987,24953498,7517032,64157906,58224549,91255408,92998591,1375023,38256849,35512853,3088684,76671482,30366150,70541760,62762857,22129328,22200378,1431742,11365791,80555751,14731700,48658605,28796059,51715482,42073124,79806380,78300864,4508700,82052050,44915235,33699435,97783876,88698958,83133790,99971982,19486173,54263880,72725103,8099994,82327024,90004325,99729357,38510840,4091162,37280276,62428472,4515343,56473732,97379790,13862149,74614639,78218845,13348726,11757872,83533741,99515901,12303248,54868730,874791,42307449,73109997,84127901,29401781,48892201,10453030,99021067,59614746,28787861,60106217,6157724,22766820,84293052,99690194,8634541,50007421,86301513,92530431,59405277,52261574,29516592,3183975,4978543,11543098,96906410,23740167,89046466,23110625,4434662,20002147,96318094,55770687,37166608,82651278,17385531,33336148,15064655,62357986,14045383,74441448,96709982,16387575,84349107,26744917,11581181,84840549,72732205,97057187,73021291,10649306,97641116,94711842,90654836,31727379,9603598,68316156,69786756,17539625,5267545,19272365,67124282,36685023,36468541,36139942,65047700,21397057,36933038,20765474,42947632,95957797,62490109,56461322,21919959,47893286,20642888,6221471,12024238,26397786,4798568,95010552,91141395,48395186,27411561,90061527,77301523,21289531,85117092,16567550,55615885,15902805,36396314,81774825,37224844,34946859,68110330,45049308,83747892 +73168565,99549373,64157906,76170907,55753905,53084256,49641577,247198,34295794,9623492,38494874,51472275,25842078,29029316,99125126,54058788,59197747,2204165,31161687,74357852,92692283,83210802,81677380,87710366,37183543,41092102,92033260,14093520,63628376,415901,86543538,12416768,29959549,3183975,62232211,168541,57961282,95395112,10309525,41481685,31733363,83789391,669105,52261574,85116755,18833224,51715482,47708850,99333375,86022504,66319530,20486294,66250369,33237508,96193415,4119747,35996293,99224392,50567636,38061439,76315420,66322959,4806458,22108665,69355476,12571310,83533741,65454636,9259676,71920426,74743862,55602660,86301513,67451935,99604946,61263068,61741594,29188588,66885828,35192533,68939068,20642888,78909561,34236719,85695762,69579137,22450468,82726008,68694897,10358899,40781854,51464002,50188404,77284619,1197320,44060493,18411915,45407418,36753250,5970607,92353856,30218878,16595436,23848565,60430369,59501487,71333116,45996863,19898053,9603598,7300047,81898046,9058407,15075176,83150534,1204161,81853704,48360198,68110330,87598888,82651278,59371804,62430984,21993752,3233084,66428795,57658654,67793644,60176618,7104732,96735716,77187825,16097038,4204661,55143724,3235882,61859581,39986008,61373987,55247455,6221471,37435892,67031644,65074535,31491938,66116458,7229550,33304202,6808825,89811711,62936963,24591705,19939935,39201414,9886593,36312813,29994197,73235980,91664334,87160386,36933038,61271144,95290172,76703609,45667668,22997281,70596786,30998561,54199160,20140249,28851716,70800879,6038457,62496012,5832946,44983451,9188443,65715134,53547802,36808486,7011964,77620120,99965001,82339363,18783702,32590267,78766358,68875490,112651,53632373,14723410,53648154,73031054,85023028,70036438,38645117,35092039,4668450,59109400,34698428,73786944,36135,78218845,93515664,61859143,15015906,55669657,26063929,68824981,20867149,54427233,55189057,824872,40686254,29834512,75153252,65271999,66611759,5822202,3633375,77300457,26102057,30764367,53802686,10597197,98462867,16380211,44889423,91802888,62452034,44550764,42967683,20765474,26275734,93270084,51047803,37620363,20122224,71316369,65880522,81274566,76671482,60955663,66667729,90654836,73392814,54762643,98739783,19272365,95726235,12348118,10453030,78589145,33201905,526217,68316156,54868730,80555751,10961421,71657078,92998591,56153345,80246713,56531125,74137926,84002370,58180653,64848072,68816413,40197395,84349107,92692978,93053405,51588015,6793819,62357986,23110625,56424103,30139692,55850790,61728685,88251446,54663246,83651211,96726697,57241521,71083565,36780454,84684495,10366309,45794415,6111563,33935899,68644627,13470059,92554120,48774913,90004325,65047700,20681921,32151165,49597667,89214616,44177011,72373496,54606384,34667286,60102965,79821897,78785507,82897371,19891772,10264691,83083131,6497830,71522968,72777973,62428472,38008118,7687278,2208785,30395570,59981773,36812683,7955293,8129978,43357947,64055960,51590803,8733068,93057697,62051033,17068582,4508700,22129328,45237957,99658235,24619760,48673079,68102437,84406788,99226875,74452589,91990218,14363867,15902805,47090124,44479073,76960303,8696647,43322743,63015256,90457870,45617087,72614359,40984766,45790169,92398073,569864,6819644,92604458,94090109,98648327,2917920,73617245,96709982,11543098,45075471,73109997,89499542,51507425,62803858,31904591,2891150,17727650,75128745,17857111,17081350,82979980,72278539,92215320,9829782,30569392,3294781,84187166,57803235,65038678,6153406,1250437,26664538,77072625,63967300,1936762,54987042,72732205,41092172,27325428,73222868,46870723,97561745,4787945,19486173,32426752,14349098,91831487,24826575,44481640,58208470,4064751,30787683,17365188,89996536,57393458,43045786,9175338,40781449,29819940,20836893,17385531,90013093,11161731,6525948,47213183,31126490,68000591,27041967,72738685,74724075,89804152,13231279,15148031,8651647,92541302,36468541,85242963,7502255,50702367,66903004,3487592,14626618,56461322,39847321,57020481,79880247,57248122,54263880,70541760,33249630,74110882,47361209,30366150,46540998,26776922,1022677,75543508,70727211,45848907,13862149,56955985,5482538,66137019,9860195,20002147,33431960,57359924,94076128,93933709,19101477,90090964,33628349,53842979,508198,47893286,82779622,58749,50806615,67474219,15064655,23740167,14326617,61712234,24915585,99690194,874791,65338021,32159704,26734892,79806380,5640302,26392416,42073124,33061250,17386996,77377183,23194618,33797252,95010552,63152504,37560164,10649306,70004753,67227442,66832478,5073754,37739481,29065964,3233569,50007421,61982238,72019362,4798568,74441448,6521313,64602895,7919588,69697787,65851721,44844121,55960386,17764950,69786756,99297164,3509435,44784505,17058722,99524975,42644903,98130363,17539625,36505482,17957593,59177718,49882705,61928316,22435353,77312810,90272749,34044787,48893685,48088883,33336148,4069912,50668599,1431742,85117092,84166196,75229462,9398733,78884452,39553046,70372191,2717150,79738755,97641116,33699435,79136082,37957788,24058273,29510992,88904910,53888755,96318094,45381876,9575954,72725103,4099191,21919959,37501808,49271185,8373289,26493119,90061527,27411561,40152546,30771409,57240218,88130087,48395186,59329511,71469330,13468268,62693428,61141874,4095116,12024238,38022834,76330843,40356877,5267545,86118021,22721500,83155442,99861373,88653118,26292919,16387575,30653863,91141395,8055981,4978543,94516935,27185644,33565483,14731700,29635537,67030811,44627776,88444207,29401781,45306070,26744917,6986898,8099994,97783876,60581278,75135448,73124510,27665211,67281495,68204242,26998766,31727379,8807940,26507214,30543215,46851987,68041839,8128637,19376156,83302115,16971929,39801351,44915235,78300864,3773993,69641513,79191827,86821229,72793444,43376279,11923835,72495719,63372756,20885148,43152977,24168411,53666583,82327024,78602717,18663507,66271566,20568363,72238278,98948034,59614746,78549759,7066775,77413012,96852321,28734791,38365584,42237907,16424199,36396314,56515456,44842615,48892201,95957797,55470718,20645197,28550822,38658347,47887470,83269727,21289531,89637706,67513640,36845587,77694700,72274002,66045587,44473167,77272628,72357096,86242799,76540481,44348328,69255765,4515343,2607799,44245960,86460488,63059024,77301523,34493392,35456853,70420215,41380093,55770687,39373729,81855445,76434144,66663942,44847298,51315460,53393358,22942635,84127901,4434662,22405120,16445503,92787493,12303248,1239555,97379790,80014588,61815223,94911072,91938191,36580610,15163258,47738185,14947650,14220886,22879907,45428665,42782652,49328608,43933006,37192445,97940276,21001913,17976208,40677414,72358170,64087743,81805959,75407347,44846932,89217461,76825057,75820087,16099750,62740044,7423788,18699206,24314885,321665,17894977,82178706,59318837,32250045,45919976,95581843,68128438,34497327,98371444,94360702,18504197,28928175,61623915,4199704,91240048,7646095,45990383,15535065,98653983,23134715,54232247,52060076,34946859,38009615,65081429,69605283,89419466,65017137,47792865,57163802,73021291,9257405,7182642,85571389,33895336,92071990,51466049,79942022,27625735,16019925,91255408,4091162,23569917,69848388,79657802,26863229,8634541,43501211,33123618,15536795,88047921,94539824,18806856,29466406,99971982,82886381,29932657,67124282,8729146,70367851,19457589,8913721,40027975,22113133,17146629,71300104,75777973,49236559,93790285,62115552,12664567,81774825,37891451,42692881,30463802,82532312,48675329,21673260,16684829,79922758,59910624,41442762,17037369,16567550,93562257,38256849,6505939,99917599,92803766,9860968,29516592,3088684,49598724,90310261,32274392,91957544,32161669,6871053,81172706,22766820,22200378,47298834,85711894,84904436,8115266,37659250,40865610,60106217,89078848,36189527,40534591,18131876,84293052,64098930,15948937,88698958,13348726,30891921,26139110,1375023,87720882,8791066,97057187,14045383,43506672,26114953,37675718,83948335,54517921,67084351,21533347,75986488,89046466,749283,3880712,78561158,33553959,74614639,52204879,61960400,42307449,65275241,75052463,51149804,69136837,25636669,82427263,93566986,91727510,39171895,92867155,32699744,99515901,1569515,4985896,68702632,90158785,91281584,60393039,32058615,6157724,36685023,23432750,37224844,13173644,17016156,52293995,97281847,18001617,54014062,48658605,82052050,42199455,13819358,5111370,24733232,44664587,96906410,36930650,7685448,79322415,97011160,80251430,28787861,51426311,94711842,35512853,63506281,42947632,98943869,26397786,30811010,30090481,7517032,56473732,95251277,30694952,83368048,89699445,37166608,56484900,41245325,52613508,16054533,11757872,86001008,30163921,21070110,92530431,86798033,45049308,18466635,28796059,58224549,77898274,59405277,83747892,62490109,48260151,62552524,45995325,62762857,58751351,1788101,84840549,80316608,21397057,99021067,77787724,34698463,11365791,55615885,83133790,11581181,71965942,38341669,96420429,94595668,24953498,34432810,176257,93359396,2331773,27187213,37280276,16405341,70782102,9599614,61897582,99729357,36139942,38510840 +168541,6221471,69641513,20002147,10309525,44481640,96193415,30366150,66319530,33797252,18806856,92554120,62936963,14093520,6793819,66667729,10358899,19486173,48892201,48675329,45407418,57393458,84002370,70372191,53648154,37620363,95957797,40534591,1431742,60430369,48360198,24058273,17365188,51047803,15015906,85116755,1022677,45996863,44784505,15535065,68644627,22997281,72373496,78589145,97783876,79191827,96906410,90654836,44844121,86301513,31161687,2204165,77377183,57240218,66250369,99125126,4668450,874791,38061439,66045587,5267545,3880712,53842979,73124510,82726008,60102965,21070110,31733363,3294781,75543508,3088684,91990218,30218878,34236719,78218845,6808825,99549373,53084256,44060493,84127901,16380211,46540998,34497327,9398733,20642888,415901,7300047,17539625,77272628,5970607,96735716,35192533,38494874,20765474,60581278,39553046,94090109,73617245,75135448,61741594,57803235,47887470,35092039,62452034,56531125,16099750,72019362,78766358,37183543,19376156,526217,34946859,11923835,4064751,55770687,824872,57961282,29029316,62740044,16097038,508198,55470718,61712234,99604946,55189057,38365584,26664538,53632373,43152977,51590803,76671482,3633375,62051033,79136082,11581181,26102057,86022504,73222868,38645117,73392814,92353856,27665211,71083565,4434662,7229550,72793444,38008118,36312813,31126490,2208785,69355476,99658235,68204242,59501487,56955985,51466049,80246713,33336148,88047921,92604458,65338021,62490109,4515343,40984766,93933709,20486294,93270084,87720882,94516935,47361209,11365791,52204879,84684495,72278539,71300104,81853704,48893685,569864,21001913,71333116,83083131,26392416,70004753,90013093,34698428,98653983,62428472,83150534,50806615,95010552,67281495,12571310,33699435,41481685,61263068,19101477,24826575,7955293,26998766,92215320,40027975,9623492,68000591,27411561,39201414,84406788,19457589,112651,77413012,73168565,58749,29188588,2917920,33123618,89214616,17727650,1250437,42237907,48088883,72274002,82651278,36580610,96318094,66322959,4099191,44550764,6111563,72358170,3509435,47738185,14220886,91831487,39847321,90272749,79922758,76330843,3233569,51426311,47090124,62430984,32151165,63967300,67793644,17764950,19939935,10366309,42967683,98371444,28796059,36468541,59371804,5111370,60106217,99297164,32590267,8129978,73786944,30764367,78602717,36189527,45617087,33628349,9188443,9603598,59109400,16684829,90457870,65081429,99524975,26114953,86543538,79806380,38658347,50188404,16567550,78909561,66116458,71522968,80316608,83210802,84349107,56424103,44245960,22113133,30771409,55753905,63152504,26275734,34432810,33249630,83368048,83533741,85117092,34044787,96709982,13348726,86460488,27325428,12303248,13231279,61982238,32159704,22721500,20645197,82339363,17957593,44177011,92787493,8128637,22942635,10597197,54606384,15064655,37435892,91664334,37891451,7687278,74724075,9886593,88653118,57241521,7919588,91802888,96726697,5073754,1936762,14731700,83269727,22450468,11161731,96420429,68041839,63059024,70727211,55247455,34698463,84840549,51472275,83651211,5832946,34295794,49598724,10453030,54263880,98462867,33237508,97561745,21919959,44479073,3233084,65271999,32426752,37192445,71316369,76540481,75229462,30653863,36135,80555751,37166608,57658654,52261574,15148031,29065964,13173644,2607799,77787724,39373729,4199704,63015256,39801351,59329511,6153406,72732205,30139692,3235882,10264691,45990383,39986008,24915585,70367851,33935899,669105,31491938,72357096,36753250,8696647,7104732,22435353,34667286,33304202,8733068,30998561,45075471,30787683,35996293,98648327,99917599,15075176,54199160,26507214,97011160,29834512,80014588,29959549,14723410,68316156,89811711,94711842,46851987,98130363,36505482,87598888,99690194,61373987,15536795,4798568,41092102,61141874,29635537,21673260,77694700,91727510,78549759,75153252,79821897,7646095,20681921,57359924,68875490,18833224,62496012,29932657,76434144,50567636,47708850,95395112,99965001,65880522,82427263,70800879,22108665,78785507,36812683,36780454,84187166,52060076,92033260,71657078,93562257,99861373,26744917,40356877,32161669,7011964,53888755,66903004,68128438,65038678,85023028,23569917,74110882,91141395,38009615,24168411,1788101,35456853,64157906,50668599,49328608,45794415,38022834,43501211,90158785,66611759,59177718,9829782,14626618,16054533,87710366,17386996,24619760,28928175,82178706,45919976,93515664,9259676,13862149,92803766,31904591,51715482,40197395,56461322,65454636,31727379,6525948,88444207,2331773,70036438,42199455,50007421,5482538,76703609,29819940,17068582,61897582,40686254,24733232,18663507,21993752,61928316,56484900,81898046,58224549,37957788,99515901,18504197,70596786,77312810,66885828,96852321,10649306,37659250,18783702,57248122,99226875,11543098,7502255,65715134,8099994,38256849,2891150,17857111,8807940,99971982,83155442,54058788,43933006,9257405,16595436,90090964,2717150,41245325,37280276,85242963,26063929,30163921,70420215,42692881,48673079,16445503,89078848,6819644,61623915,82052050,79657802,20122224,63372756,73109997,20568363,92867155,93790285,29994197,93053405,247198,7423788,37739481,36139942,7182642,49271185,98943869,78300864,84293052,30891921,67474219,95290172,94076128,321665,45790169,37675718,23134715,63628376,77187825,75986488,20140249,98739783,59981773,72725103,88251446,20885148,23740167,44889423,64087743,9058407,30569392,88904910,30395570,3773993,92541302,81677380,81774825,3183975,18411915,72614359,61859143,64055960,49597667,43045786,50702367,33565483,89499542,61960400,75407347,83133790,91240048,60955663,55615885,69579137,90310261,749283,49641577,16387575,35512853,70541760,9860195,16424199,68694897,53802686,17037369,94360702,66428795,77284619,21289531,18466635,42644903,76960303,62115552,14947650,90061527,33201905,36685023,44983451,36930650,77072625,69786756,30811010,66663942,28787861,30543215,58208470,4204661,1204161,28550822,4806458,91938191,8791066,46870723,69848388,32250045,83789391,19898053,55960386,13819358,59614746,8913721,12416768,44473167,30463802,74357852,64098930,14045383,80251430,8651647,26776922,44842615,38341669,65074535,89637706,22879907,4119747,32699744,65047700,60393039,4978543,99729357,55602660,66271566,21533347,22405120,88698958,68102437,74452589,76170907,19891772,18699206,18001617,43506672,54232247,79942022,9175338,99333375,22200378,77620120,48395186,20867149,1239555,67084351,5640302,91255408,19272365,36808486,69255765,65851721,40781449,14363867,47298834,41380093,27625735,23194618,51464002,87160386,7685448,42782652,97641116,38510840,24953498,55850790,44847298,8634541,86242799,1197320,8055981,33895336,66832478,72495719,29466406,36396314,34493392,16971929,55143724,59197747,51588015,95251277,72777973,92692978,43357947,33431960,86798033,99224392,61271144,22766820,74743862,42947632,9575954,12024238,17385531,57163802,69697787,22129328,47893286,1375023,64602895,7517032,6038457,59318837,26139110,8115266,75777973,15902805,79738755,75052463,9599614,82327024,8373289,92692283,95581843,29510992,67451935,27185644,12664567,54663246,40865610,37560164,86001008,65017137,81855445,47213183,62762857,6505939,28851716,84904436,91281584,58751351,64848072,44348328,58180653,95726235,32058615,5822202,27187213,63506281,67030811,176257,45428665,27041967,69136837,40152546,45848907,25636669,30090481,78561158,39171895,16019925,81172706,43376279,51507425,76315420,15163258,61728685,67031644,89804152,82897371,73031054,94595668,56515456,45667668,52293995,45381876,82532312,56153345,36933038,6157724,68110330,4787945,79880247,42307449,33061250,86118021,78884452,62357986,91957544,73235980,56473732,52613508,89046466,44915235,67227442,37224844,79322415,92998591,6871053,81805959,14326617,94911072,1569515,45237957,72738685,20836893,82886381,3487592,26397786,67124282,13468268,53547802,81274566,12348118,24314885,55669657,18131876,82979980,26493119,71965942,97379790,24591705,40781854,62232211,83302115,92071990,4985896,68702632,8729146,23110625,40677414,26292919,16405341,17081350,11757872,66137019,54868730,74137926,61859581,49236559,6986898,29516592,89419466,77301523,10961421,69605283,54762643,67513640,9860968,98948034,17976208,4508700,51315460,26863229,45995325,93057697,42073124,17894977,76825057,59910624,60176618,53393358,37501808,97940276,48658605,85571389,75128745,54987042,54427233,68939068,54517921,77300457,26734892,68816413,83747892,25842078,4091162,13470059,45306070,88130087,84166196,7066775,53666583,23432750,97281847,68824981,73021291,82779622,89699445,92530431,71469330,62693428,30694952,61815223,14349098,44627776,43322743,74614639,62552524,75820087,17146629,93566986,41092172,4069912,32274392,28734791,44664587,17016156,6497830,57020481,21397057,23848565,54014062,51149804,86821229,89996536,71920426,6521313,90004325,48260151,99021067,72238278,47792865,33553959,97057187,44846932,92398073,49882705,85711894,93359396,36845587,15948937,62803858,65275241,77898274,48774913,94539824,83948335,45049308,89217461,74441448,59405277,29401781,4095116,41442762,17058722,85695762,70782102 +4099191,37560164,17764950,57803235,38022834,29029316,37620363,48892201,31126490,47090124,65047700,12348118,78785507,95010552,44479073,30694952,60430369,37739481,38658347,92554120,3183975,73617245,62430984,49598724,86022504,70800879,93933709,61982238,98130363,29834512,87710366,83302115,8129978,77272628,77312810,67474219,68204242,75052463,45996863,81274566,51047803,23569917,69355476,56531125,92071990,24733232,55189057,92398073,39171895,22879907,22435353,16099750,35512853,10597197,3509435,29510992,17976208,35996293,40865610,3088684,26998766,4119747,80316608,4515343,8791066,63628376,62428472,73168565,64602895,45407418,80251430,10366309,36753250,17081350,36580610,49882705,73392814,55247455,98371444,2917920,15535065,10309525,95957797,52060076,44915235,94090109,79821897,69136837,5111370,37183543,168541,59109400,83210802,92033260,36808486,67793644,16445503,55753905,76434144,55602660,4064751,42199455,72274002,5267545,4787945,89078848,45075471,22942635,21533347,35192533,16019925,30163921,66137019,62496012,79922758,45237957,65715134,50188404,63152504,43506672,37192445,32250045,7955293,88444207,22997281,38008118,11581181,61141874,52613508,64087743,96709982,91664334,26392416,30395570,16097038,13231279,53632373,2717150,9257405,54987042,5640302,1204161,17727650,35092039,60581278,74357852,8807940,16595436,76960303,92803766,4668450,18131876,1431742,71333116,38061439,96193415,73222868,49641577,22450468,2204165,59318837,36312813,65338021,36780454,60106217,67030811,33201905,44784505,85711894,44177011,70596786,20867149,65271999,82339363,91240048,38494874,17146629,61271144,51472275,27411561,824872,66832478,86821229,41245325,30891921,98739783,53842979,98462867,93057697,99515901,47708850,26507214,17037369,9398733,76671482,20645197,86460488,84002370,44627776,79942022,9575954,12024238,75543508,20765474,70367851,88251446,13173644,78602717,30764367,66250369,62693428,27665211,9860195,29819940,99297164,15015906,14326617,33336148,51590803,40356877,83083131,40152546,93562257,7687278,33304202,14626618,63059024,69605283,29188588,84684495,99965001,34497327,59501487,87720882,83651211,61373987,29466406,247198,21993752,65081429,44844121,48675329,34493392,4806458,34236719,53802686,78909561,73235980,26102057,10649306,30543215,6793819,61741594,92353856,64848072,9175338,62051033,71657078,81172706,99524975,67451935,77187825,9188443,62490109,87598888,80014588,18699206,24915585,67031644,40984766,94711842,66116458,23432750,96735716,38256849,37224844,26863229,81898046,30653863,23194618,53648154,3233084,57163802,44473167,78549759,19486173,31161687,41092102,91281584,44245960,72373496,26664538,91141395,19101477,77284619,57240218,54663246,4434662,47361209,62936963,70036438,66667729,76315420,55770687,66319530,58180653,76170907,56461322,63967300,89811711,45617087,59329511,66428795,5970607,57359924,60955663,14947650,38645117,34295794,92787493,24058273,68041839,6038457,7919588,77620120,21070110,48774913,69697787,88653118,7646095,95581843,17385531,30771409,53547802,21289531,73021291,61263068,24826575,70004753,38365584,61623915,44983451,20486294,55470718,44842615,54762643,56473732,8099994,79880247,40027975,23110625,54199160,74743862,25842078,28928175,30463802,30787683,90272749,33061250,6871053,55669657,96318094,60102965,13348726,90090964,32161669,19376156,53084256,68939068,4069912,27325428,97011160,93515664,42782652,34698463,56515456,79136082,75820087,81677380,84904436,24168411,10358899,75153252,42947632,17386996,30366150,59197747,47298834,97940276,42073124,86001008,88047921,17957593,76703609,16567550,59981773,56153345,89214616,35456853,74441448,26114953,90013093,28851716,569864,22200378,50806615,31904591,62115552,93053405,7517032,50567636,25636669,36505482,52293995,78561158,77377183,52204879,39373729,87160386,63015256,82427263,13862149,69641513,84127901,22721500,27041967,18806856,3633375,1569515,24619760,61928316,508198,20642888,75777973,13468268,79322415,45848907,63506281,85116755,36685023,44846932,68000591,22108665,9623492,66903004,59371804,66271566,92541302,26063929,669105,62740044,99917599,3233569,48893685,42307449,3880712,50668599,33895336,69786756,91990218,874791,28734791,82726008,97561745,68875490,91802888,76330843,1022677,55143724,39847321,5482538,6808825,81855445,49236559,6505939,7011964,72238278,48360198,26493119,96906410,46851987,43322743,92692283,33699435,526217,62452034,33249630,5832946,14220886,51466049,43376279,78300864,29994197,4095116,71920426,71316369,34698428,68816413,84349107,91957544,65880522,75986488,70372191,55960386,20885148,64055960,67124282,3487592,72495719,8651647,72358170,86798033,32590267,91831487,48673079,11543098,1197320,70420215,94516935,82897371,83150534,77787724,47792865,31733363,89046466,38341669,24953498,18504197,33237508,32274392,4199704,6153406,30811010,96726697,20836893,74452589,33628349,26275734,60176618,68644627,83789391,37891451,17068582,89419466,99604946,62232211,68128438,19457589,41092172,8913721,54263880,99658235,30090481,36135,71300104,30569392,14349098,20140249,98653983,56955985,14093520,40781449,66045587,321665,31727379,9058407,54232247,30218878,73124510,65851721,10961421,82651278,36189527,15948937,77072625,48260151,37166608,36812683,1250437,39553046,40686254,18833224,52261574,62357986,51426311,39201414,70727211,37435892,88698958,69848388,95290172,42237907,46540998,41481685,94911072,44664587,67227442,32159704,22405120,44060493,71083565,34432810,83948335,65017137,20002147,23848565,94539824,40197395,72357096,39801351,36139942,71469330,81805959,95726235,79738755,59910624,29635537,99224392,68702632,14731700,80246713,77413012,65275241,18783702,96852321,1936762,78589145,63372756,34946859,90004325,54058788,68102437,42692881,12303248,71965942,112651,98648327,72777973,42644903,94595668,20568363,91727510,67084351,82979980,93270084,79191827,68316156,20122224,58751351,32426752,93359396,4978543,72019362,42967683,89499542,23740167,29932657,65074535,99549373,9886593,21673260,27625735,79657802,78766358,8696647,70541760,73031054,57241521,66322959,36933038,15148031,51464002,61815223,37675718,90457870,40534591,99690194,51588015,37659250,15163258,45995325,64157906,22113133,9860968,84293052,32699744,74110882,58749,26397786,4798568,4985896,92998591,415901,1788101,96420429,11923835,84840549,44481640,6986898,44550764,69579137,57658654,8373289,26139110,16684829,75407347,27185644,14045383,19891772,76540481,97057187,45428665,18411915,89637706,89699445,73786944,43357947,7502255,37501808,43501211,99333375,92604458,97783876,9603598,28550822,6521313,56424103,48088883,92215320,19898053,22129328,61859581,45794415,30139692,4204661,67513640,46870723,15902805,57393458,29516592,66611759,72738685,66885828,28796059,34044787,7104732,17539625,38510840,81853704,83269727,44889423,16971929,6525948,44847298,24591705,59177718,82532312,12664567,82779622,61859143,47213183,72725103,48658605,88904910,18466635,74614639,54517921,749283,77300457,55615885,17365188,30998561,53393358,8729146,98948034,64098930,92692978,29401781,54014062,50702367,33797252,84187166,61728685,60393039,75128745,29065964,23134715,18663507,97281847,15064655,77694700,58208470,62552524,82886381,72793444,36396314,99021067,43045786,16380211,41380093,81774825,3294781,84406788,99125126,58224549,54606384,33431960,99861373,85242963,45990383,86301513,92867155,90158785,1375023,74724075,85571389,7685448,57248122,50007421,43933006,19272365,44348328,15536795,95251277,72278539,32151165,97641116,26734892,98943869,33123618,176257,91938191,61712234,38009615,54427233,1239555,34667286,67281495,75135448,15075176,7300047,49271185,82052050,83133790,74137926,7229550,89804152,93790285,90061527,8115266,6221471,7182642,12571310,85117092,55850790,56484900,72732205,36468541,53666583,2331773,78884452,51507425,89996536,6111563,86242799,11365791,45919976,53888755,80555751,99971982,72614359,88130087,29959549,10453030,75229462,4508700,33565483,73109997,78218845,33935899,84166196,28787861,95395112,18001617,45306070,16424199,8634541,26776922,82327024,4091162,83533741,90654836,86118021,83368048,13470059,41442762,33553959,86543538,9599614,26292919,37957788,51149804,66663942,90310261,49597667,49328608,10264691,45049308,54868730,7066775,51315460,19939935,31491938,51715482,62762857,61960400,82178706,65038678,26744917,47738185,99226875,69255765,47893286,68824981,16054533,91255408,2891150,45381876,12416768,2208785,40677414,79806380,3235882,16387575,77301523,14363867,21919959,3773993,45790169,59614746,8055981,47887470,48395186,22766820,62803858,21001913,36845587,14723410,17058722,61897582,8733068,7423788,17857111,8128637,16405341,83155442,9259676,37280276,94076128,85695762,70782102,11757872,76825057,68694897,6819644,21397057,17016156,9829782,94360702,77898274,85023028,93566986,5822202,20681921,59405277,39986008,99729357,36930650,2607799,6157724,17894977,89217461,40781854,24314885,27187213,65454636,83747892,57961282,45667668,92530431,5073754,57020481,6497830,71522968,43152977,68110330,11161731,32058615,97379790,13819358 +31733363,38494874,31126490,4204661,36505482,57248122,82979980,15075176,49236559,89214616,36812683,74357852,26493119,13470059,92692978,92554120,16595436,38022834,7502255,66903004,27185644,30139692,71333116,70596786,33237508,27325428,86543538,91664334,83210802,65081429,18131876,40865610,6221471,28851716,99965001,63015256,12348118,57359924,40781854,91255408,45237957,18783702,6521313,97783876,90457870,68939068,68816413,6505939,67793644,92692283,93515664,70004753,66832478,65017137,56955985,55753905,508198,874791,40197395,53084256,27665211,23432750,18699206,1022677,59371804,72238278,26063929,29994197,60102965,26863229,57241521,23110625,99524975,94090109,89046466,80246713,85242963,13231279,4064751,17957593,48088883,96193415,19101477,55189057,47361209,76671482,16971929,37560164,54663246,26664538,14093520,415901,79821897,1431742,3633375,99861373,81172706,47893286,26114953,34295794,24591705,82897371,72725103,3773993,88444207,44846932,1375023,61373987,98739783,37957788,99125126,60430369,20002147,50668599,38008118,18833224,95010552,54987042,44060493,22766820,16097038,112651,20765474,26397786,84127901,36780454,46870723,78218845,18001617,37183543,67451935,83155442,84406788,22450468,4069912,37192445,43376279,76703609,247198,68000591,99917599,29029316,43045786,33304202,27625735,37675718,35192533,44983451,3509435,17058722,4806458,62496012,21533347,98943869,79657802,62232211,71300104,5111370,32161669,59109400,44915235,17727650,76434144,45990383,18663507,73235980,6986898,83789391,2717150,41380093,37620363,77284619,20836893,8128637,93359396,98653983,27041967,30163921,26102057,3880712,28734791,95957797,7955293,61859143,89637706,71920426,76825057,14220886,14626618,6793819,84187166,35092039,33201905,42782652,93933709,67124282,39201414,26275734,93566986,56424103,44847298,20122224,22113133,23134715,54199160,569864,66428795,70036438,57240218,59501487,3235882,73031054,68824981,51464002,74724075,7229550,30771409,44842615,74137926,42967683,64055960,28787861,9058407,53842979,99297164,66250369,73222868,15163258,8696647,34236719,17764950,88047921,7423788,7919588,73168565,89419466,92398073,87598888,54868730,29834512,77694700,84904436,95290172,98130363,37739481,92541302,21993752,19457589,55143724,61141874,26998766,3294781,22108665,97281847,9575954,61271144,43357947,33565483,68644627,44245960,1197320,49641577,77620120,39171895,25842078,26392416,749283,59614746,89996536,91957544,88653118,82886381,62452034,72274002,81677380,47090124,10309525,30366150,1204161,33553959,92998591,44473167,50567636,77377183,6525948,20885148,17081350,4515343,8807940,70727211,19272365,69355476,83083131,32250045,64848072,28796059,96735716,28550822,30787683,36580610,14363867,35996293,82339363,34497327,75153252,62740044,60106217,5482538,75229462,22997281,45919976,15015906,28928175,24168411,17976208,80014588,36933038,23848565,47738185,70800879,44177011,80555751,75407347,99729357,90272749,53632373,24058273,93790285,66885828,77300457,48673079,47298834,61859581,66667729,44481640,63628376,74452589,44889423,71657078,38645117,45381876,3233084,52060076,14947650,27411561,71316369,9623492,39801351,77072625,78589145,9603598,40686254,11581181,66045587,3183975,23569917,2208785,65271999,72793444,38341669,78300864,669105,65454636,65851721,61960400,26744917,14349098,6871053,23740167,95726235,33935899,50702367,49597667,7300047,54263880,67084351,16405341,30653863,13348726,45428665,81805959,63059024,66663942,97011160,67030811,32274392,99549373,91240048,29188588,77787724,64602895,51047803,37435892,78602717,34493392,74441448,4091162,4119747,85116755,57803235,2204165,88904910,526217,84349107,56153345,50188404,99604946,55770687,70541760,9398733,9175338,1239555,54058788,45049308,90013093,40984766,92803766,14045383,68875490,29959549,38658347,50806615,78766358,82532312,75543508,33249630,65047700,7646095,8099994,85023028,51507425,75128745,61982238,59910624,99515901,68041839,69697787,14326617,73021291,20642888,86301513,1936762,11161731,79136082,69786756,48360198,51472275,17857111,95251277,70372191,53666583,18806856,45996863,15535065,53888755,9860195,82726008,76960303,34698463,54762643,81853704,48893685,72278539,42237907,29466406,72495719,15148031,79880247,77413012,60581278,45667668,321665,17068582,73786944,94516935,79922758,33797252,90310261,8129978,24733232,90061527,30764367,53802686,30090481,16019925,69641513,68694897,42073124,87160386,16054533,55247455,85571389,14731700,4668450,31904591,68204242,76315420,44784505,48892201,39553046,4434662,92604458,4508700,20140249,41481685,91802888,82651278,37891451,6038457,89699445,69579137,71965942,66319530,19891772,42947632,62936963,72614359,4199704,47708850,32590267,19939935,6808825,93053405,37659250,20867149,33699435,36753250,36845587,94911072,36930650,99224392,83269727,86821229,81274566,78561158,29819940,17539625,86242799,56515456,65880522,63967300,77187825,69605283,66322959,50007421,68128438,42644903,40356877,88130087,42307449,4798568,56531125,29635537,34432810,21673260,30395570,62430984,40152546,96420429,7182642,12024238,53393358,33061250,19486173,83651211,46851987,40027975,53547802,24619760,10961421,55850790,5970607,77312810,45617087,77272628,17385531,2331773,17386996,84002370,82327024,81898046,176257,3088684,83948335,65715134,61263068,8055981,86118021,95581843,44550764,65275241,30543215,96709982,73124510,47792865,49882705,89078848,55669657,32159704,32058615,1569515,36312813,32151165,51715482,2917920,86022504,66611759,75986488,98462867,67281495,49271185,89804152,30218878,73617245,8373289,10366309,98948034,96906410,86460488,10358899,36808486,45848907,78884452,89499542,82052050,70420215,61897582,76170907,62693428,85117092,92071990,69848388,13862149,39373729,11923835,15064655,63152504,44627776,17146629,83368048,41092102,97057187,67474219,61741594,57020481,52261574,71083565,4099191,41092172,88251446,68316156,62762857,16445503,44844121,97561745,68102437,18466635,49598724,5073754,84684495,12416768,80316608,21001913,26507214,85711894,39847321,67513640,63372756,13468268,38061439,65338021,7104732,54427233,168541,26734892,44348328,74110882,72732205,90158785,40534591,30891921,70367851,10597197,99971982,36189527,22435353,17037369,54606384,43933006,6153406,54517921,12664567,75777973,64087743,13819358,89811711,92215320,62357986,54232247,75052463,77898274,74614639,71469330,58208470,88698958,65038678,9860968,1250437,79738755,9188443,4787945,33628349,19376156,83302115,38365584,72738685,61815223,53648154,94711842,43322743,52204879,40781449,46540998,42199455,2607799,37501808,45995325,82779622,45790169,97379790,14723410,56473732,64157906,48260151,8651647,18411915,16387575,6497830,93270084,33431960,36468541,11365791,61928316,8634541,31491938,54014062,94360702,76540481,98371444,91990218,32699744,62803858,23194618,38510840,78549759,99690194,56484900,87710366,51315460,90004325,86798033,39986008,76330843,52613508,29401781,83150534,52293995,21289531,96726697,97940276,51466049,3487592,6111563,7517032,62490109,8913721,43506672,69136837,57393458,9886593,98648327,99658235,59329511,33895336,3233569,27187213,16380211,99333375,82178706,16099750,5267545,37280276,45407418,24915585,74743862,30463802,79942022,84293052,8115266,79191827,92787493,84166196,9259676,93562257,94076128,16567550,58180653,57961282,43501211,44479073,92033260,25636669,63506281,43152977,31727379,48774913,31161687,38256849,90654836,42692881,5822202,72019362,21397057,61728685,26139110,30811010,22129328,8733068,86001008,45794415,91727510,67227442,60955663,55960386,30998561,78785507,11757872,4978543,7011964,96852321,59177718,55602660,72373496,59318837,40677414,66271566,15902805,55470718,48658605,9829782,44664587,91938191,91141395,82427263,51149804,15948937,51588015,2891150,67031644,10264691,22200378,22405120,93057697,73392814,36396314,68110330,7685448,17016156,84840549,70782102,51426311,94595668,35512853,16684829,73109997,92530431,95395112,6819644,57658654,58751351,6157724,11543098,59197747,20681921,69255765,47213183,99226875,96318094,66137019,60176618,22879907,99021067,1788101,75135448,55615885,71522968,16424199,57163802,66116458,20568363,12571310,72358170,33336148,41442762,33123618,20486294,824872,61623915,7066775,24826575,34946859,9257405,9599614,17894977,21070110,8791066,20645197,36685023,34044787,5640302,7687278,72777973,37166608,92867155,62428472,47887470,58224549,34698428,36135,38009615,72357096,58749,49328608,17365188,91831487,75820087,77301523,59405277,21919959,29932657,13173644,15536795,87720882,22942635,83133790,61712234,62051033,29510992,81855445,59981773,94539824,79806380,24314885,62552524,92353856,29516592,65074535,26292919,81774825,85695762,30694952,5832946,80251430,35456853,24953498,79322415,62115552,45306070,34667286,10453030,29065964,32426752,64098930,48675329,22721500,26776922,10649306,48395186,91281584,37224844,68702632,36139942,19898053,45075471,56461322,4095116,12303248,83533741,18504197,8729146,60393039,90090964,83747892,97641116,4985896,78909561,41245325,30569392,89217461,51590803 +55470718,55753905,9886593,83210802,89214616,62496012,84840549,64087743,91664334,76960303,36930650,30139692,23848565,65715134,30366150,27665211,83150534,6153406,72495719,16971929,80246713,29029316,91727510,48893685,3233569,50567636,569864,15535065,70004753,34698428,112651,20002147,415901,61982238,66322959,85116755,71333116,33628349,19376156,11365791,99861373,15075176,78218845,34493392,83155442,78785507,71657078,65275241,44983451,78766358,5111370,31904591,70372191,86001008,33249630,96193415,35192533,54058788,66832478,63372756,26292919,79806380,29819940,88653118,72777973,65038678,14947650,50806615,39847321,72274002,78589145,9623492,26139110,40152546,91938191,22997281,31126490,97783876,18663507,16595436,99549373,49328608,43501211,30998561,19939935,92692978,36780454,4806458,90457870,54606384,45919976,88444207,35092039,32426752,71522968,33123618,87710366,51047803,53842979,29188588,79136082,82726008,28928175,32590267,15163258,5822202,56531125,17539625,83269727,68824981,67793644,78549759,29466406,526217,68875490,73392814,28787861,30891921,66428795,36812683,16054533,4668450,62936963,39171895,92398073,78561158,64098930,13348726,68316156,80555751,8651647,45237957,55770687,61373987,73222868,30653863,28796059,26114953,85023028,76825057,26392416,90013093,34295794,86460488,60106217,88047921,61263068,96726697,14731700,74357852,38645117,45848907,6808825,50668599,10961421,7502255,16099750,89637706,176257,61623915,11161731,45996863,15148031,321665,16445503,72725103,90272749,53393358,37435892,13173644,4508700,7300047,67281495,20486294,8696647,18833224,32250045,15015906,57240218,82052050,27325428,49598724,95581843,46851987,62552524,12348118,40677414,99604946,28550822,42199455,63967300,38365584,69136837,71469330,45306070,32159704,81677380,61141874,30787683,62232211,36505482,40865610,62693428,55247455,1204161,50702367,24058273,73031054,40356877,24619760,77272628,34236719,94595668,71920426,18504197,54427233,48892201,7423788,39201414,47298834,8807940,83302115,45990383,61928316,17764950,4787945,76671482,68702632,42644903,34698463,26734892,65017137,75229462,4064751,62357986,85695762,38494874,74137926,45428665,1375023,24915585,51472275,93057697,77377183,68128438,29834512,86543538,30218878,59371804,874791,83533741,3235882,6793819,16380211,26063929,29065964,77620120,7955293,99515901,1569515,98648327,73124510,44889423,59197747,99917599,26493119,83789391,9599614,38061439,94539824,69641513,29994197,56461322,83368048,60102965,18131876,62452034,3088684,37166608,80316608,66885828,81774825,86821229,45667668,8129978,77694700,20765474,81853704,51149804,69579137,97561745,84904436,21001913,33304202,66271566,57803235,33565483,52613508,94711842,62762857,65454636,44550764,95726235,77300457,17037369,7182642,58224549,8373289,91141395,30395570,1788101,53547802,61859143,19898053,55602660,75543508,99690194,4515343,1431742,59501487,669105,78909561,53666583,58749,68816413,45995325,44481640,9398733,15948937,24733232,8733068,18806856,36753250,66137019,4099191,95395112,92803766,66667729,43045786,51464002,61960400,52261574,6505939,79922758,67031644,27187213,77413012,95010552,39801351,59981773,5832946,57658654,56424103,47090124,73168565,18783702,99224392,3233084,46870723,60430369,62430984,57393458,10358899,16684829,82651278,72373496,37659250,77072625,82532312,54014062,26998766,30764367,92554120,45407418,38008118,52293995,76170907,8634541,37739481,99297164,47887470,47738185,36189527,2917920,92033260,19101477,62051033,69255765,56515456,20642888,84127901,7104732,2208785,79880247,63059024,48658605,13470059,42967683,82427263,54517921,70036438,96906410,36685023,95290172,20122224,26664538,20885148,49641577,5073754,4069912,59177718,6986898,38256849,65880522,19457589,70596786,58751351,78884452,63628376,83948335,72614359,54199160,7011964,54663246,2331773,71316369,86022504,47708850,7229550,20568363,70541760,42947632,1239555,17058722,55615885,3294781,22766820,79738755,97011160,61815223,56484900,39553046,33797252,38341669,66663942,6871053,30463802,4119747,22405120,22108665,8099994,92867155,97281847,17727650,81805959,13819358,77787724,23569917,34497327,26863229,22942635,37891451,18411915,1022677,2607799,72738685,9175338,41380093,61859581,5970607,82979980,69697787,65338021,48360198,33895336,64602895,57241521,57359924,32161669,77312810,99125126,6038457,70800879,5267545,27625735,13468268,17146629,85571389,14326617,7517032,45790169,49882705,62740044,78300864,68204242,99658235,30163921,82886381,76330843,41092172,94516935,49597667,93053405,76434144,1250437,4434662,29959549,12416768,86798033,44177011,90310261,92998591,19891772,43376279,59109400,41245325,17068582,14626618,85242963,81855445,66319530,15902805,59910624,55143724,50188404,65851721,61897582,98130363,7687278,77301523,48088883,68644627,75986488,3487592,10264691,6221471,99971982,33201905,29932657,12024238,33699435,75135448,92787493,16097038,40686254,89419466,35512853,37183543,76703609,39986008,7066775,7646095,44842615,90090964,8115266,22129328,45075471,82339363,17976208,82327024,67124282,36135,16387575,31161687,4095116,71300104,77284619,73617245,72357096,9860195,26397786,81274566,25842078,39373729,46540998,66116458,58180653,77898274,67451935,66611759,88698958,66250369,89217461,10366309,95957797,92215320,94911072,52060076,51466049,36139942,86242799,35456853,68694897,13231279,32058615,7685448,57248122,47361209,28734791,24591705,99965001,79821897,70727211,1197320,75407347,38658347,60955663,63506281,45381876,85117092,51588015,9259676,29635537,76315420,91240048,92604458,14363867,44348328,85711894,17386996,69786756,33336148,70367851,30543215,30771409,3633375,64055960,14093520,64157906,86301513,73786944,67030811,54263880,3509435,247198,65074535,17957593,37192445,9603598,92071990,29516592,37280276,36580610,22450468,30569392,22200378,5482538,91831487,59318837,89499542,30811010,48673079,9860968,61728685,34432810,17894977,73235980,43933006,37620363,96709982,2204165,9188443,74743862,36312813,98653983,65047700,55850790,10597197,72732205,54232247,26507214,66903004,69848388,99524975,54868730,15536795,65271999,21993752,92692283,56955985,22435353,93515664,55960386,18699206,94076128,20836893,20140249,99729357,53084256,44846932,22879907,59329511,40781449,76540481,51715482,90158785,48675329,66045587,14220886,31491938,10309525,68939068,98462867,168541,16019925,17016156,98371444,94090109,26776922,47213183,91802888,34044787,62803858,93790285,12303248,40027975,44627776,2717150,19486173,43152977,62115552,40197395,14045383,44844121,81898046,34946859,80014588,59614746,21070110,64848072,53802686,7919588,60393039,31727379,21919959,33237508,65081429,89046466,49271185,40534591,67513640,51315460,53648154,97379790,70420215,91281584,42073124,97641116,51507425,98948034,20867149,62428472,24826575,9257405,98943869,56153345,44060493,93270084,24168411,42307449,57961282,6157724,77187825,82779622,4978543,48260151,83651211,62490109,3183975,16567550,72019362,22113133,49236559,16424199,68110330,8055981,52204879,32151165,63015256,43322743,95251277,15064655,51590803,2891150,84166196,93566986,17385531,74614639,44479073,60581278,30090481,82178706,55669657,55189057,38009615,9575954,87160386,10649306,45794415,8128637,99333375,72793444,29510992,73109997,98739783,47792865,35996293,69355476,87598888,82897371,75153252,3880712,96735716,74724075,508198,68000591,22721500,91990218,23110625,68102437,44473167,36468541,41481685,33553959,53888755,37957788,44784505,84187166,16405341,8913721,84002370,97940276,42237907,57020481,86118021,92530431,40984766,4204661,72278539,75052463,61712234,88904910,4798568,75777973,81172706,71965942,91957544,61271144,33935899,23432750,19272365,9058407,42692881,33431960,54762643,12571310,6525948,6497830,58208470,37675718,6111563,25636669,83083131,44664587,24314885,42782652,17365188,32699744,20645197,74110882,90654836,93359396,96318094,70782102,4199704,84406788,34667286,32274392,89078848,21673260,71083565,79657802,80251430,48774913,67474219,8729146,31733363,59405277,21289531,88251446,89811711,99021067,96420429,37501808,44245960,10453030,44915235,83133790,44847298,6521313,26102057,93933709,18466635,36396314,11923835,84684495,41092102,99226875,92541302,5640302,88130087,50007421,41442762,36808486,75128745,79942022,89996536,54987042,40781854,17857111,36933038,94360702,11757872,13862149,749283,3773993,9829782,72358170,1936762,53632373,28851716,43506672,56473732,36845587,63152504,51426311,824872,17081350,97057187,45049308,21533347,78602717,11543098,11581181,74452589,93562257,57163802,23134715,84293052,27041967,38022834,72238278,67227442,26744917,90004325,30694952,20681921,23740167,89699445,33061250,83747892,89804152,24953498,68041839,96852321,29401781,84349107,48395186,8791066,79191827,27185644,23194618,45617087,26275734,12664567,47893286,92353856,4985896,18001617,14723410,14349098,74441448,4091162,87720882,61741594,37224844,69605283,75820087,73021291,67084351,91255408,38510840,79322415,90061527,27411561,60176618,43357947,21397057,6819644,37560164 +874791,67793644,14093520,62936963,55669657,51464002,73617245,54058788,49641577,20765474,30764367,43933006,33628349,99965001,13470059,77787724,63059024,65851721,67281495,72274002,51047803,9860968,40686254,40677414,36812683,18663507,36312813,38365584,96193415,39171895,93566986,12348118,59109400,40152546,7423788,86301513,91255408,82979980,31733363,3233084,24733232,2717150,68644627,47361209,64098930,65454636,84840549,93562257,99549373,45428665,27325428,75135448,36505482,33565483,83210802,61741594,56531125,14220886,68128438,66903004,38022834,83155442,96726697,81898046,95290172,1788101,54663246,75052463,97011160,55143724,89078848,30218878,76540481,56473732,43357947,61859143,66045587,77620120,62740044,20486294,59501487,39201414,78589145,33797252,44245960,96906410,4069912,58208470,68041839,60430369,72738685,60102965,53084256,44481640,8373289,45306070,40356877,83083131,52060076,77413012,44844121,10358899,11161731,3880712,84406788,71920426,62430984,22721500,36580610,33201905,84904436,9188443,70727211,9058407,39553046,57359924,26114953,69641513,30395570,54427233,80014588,87720882,44842615,73222868,1431742,78549759,92398073,80251430,20642888,60581278,45790169,83150534,92541302,19939935,71300104,24619760,3294781,97561745,30787683,85116755,70372191,61373987,62496012,75543508,6793819,96318094,27665211,37560164,17146629,36685023,44348328,77284619,95251277,15148031,22997281,43501211,27625735,20867149,62803858,66250369,89637706,4806458,526217,99333375,78561158,7229550,57240218,35092039,6871053,29466406,88653118,32590267,72614359,33431960,44550764,72732205,14731700,37183543,12024238,85023028,37891451,14349098,86242799,3509435,40984766,65017137,65074535,61960400,44784505,47090124,26392416,21001913,59329511,88251446,11365791,81172706,5640302,51590803,79806380,80316608,76330843,16054533,64157906,84293052,6497830,49597667,19891772,26493119,10366309,45237957,76825057,81274566,77187825,45617087,65338021,99297164,32161669,69786756,4119747,34698463,3487592,38645117,33699435,80246713,27411561,69605283,69136837,29994197,23110625,48893685,58749,64848072,94360702,29959549,25842078,98739783,5970607,26776922,6038457,20568363,30771409,10264691,6525948,86798033,4668450,13231279,88047921,80555751,14045383,247198,27185644,53632373,19486173,2208785,43152977,30139692,97783876,85242963,37620363,84166196,17857111,75229462,33304202,14626618,52293995,22450468,34667286,18833224,58751351,66667729,90158785,16405341,44846932,23432750,70782102,78602717,15948937,83789391,89214616,78218845,61982238,46851987,62115552,69355476,35996293,98462867,81855445,59405277,91727510,68316156,65081429,99021067,81853704,72777973,5111370,10453030,93053405,22108665,82886381,74110882,14326617,68939068,55470718,17058722,36753250,4508700,24826575,11923835,27041967,82726008,68102437,15535065,25636669,28928175,20645197,37957788,90013093,7502255,5073754,93515664,91831487,112651,89046466,30694952,31161687,4199704,72278539,55753905,33336148,24168411,91802888,57803235,94516935,60106217,20140249,21993752,51472275,68000591,20002147,42199455,92215320,99524975,9398733,5822202,68816413,60176618,17727650,12416768,45996863,48088883,50188404,17764950,99861373,38494874,86821229,47298834,34236719,69848388,44889423,95957797,13819358,31126490,37739481,43322743,5482538,72495719,3183975,63628376,39847321,29188588,36930650,62762857,50567636,57393458,77377183,74743862,40781449,83948335,48774913,40027975,44627776,32274392,71657078,93057697,70004753,60955663,84187166,91664334,29834512,30163921,53393358,81805959,3773993,18783702,51507425,61712234,44983451,55850790,87710366,63152504,42967683,20122224,98130363,93933709,3633375,6153406,96735716,15015906,2204165,14947650,72725103,93270084,17068582,63015256,33061250,86543538,46540998,76671482,15075176,90061527,98648327,29029316,38658347,3233569,26397786,7955293,77312810,73786944,3235882,18466635,69579137,8733068,49271185,41092102,168541,59318837,15064655,55189057,7646095,41380093,98948034,12571310,48658605,73235980,96852321,14363867,70596786,95010552,28851716,1197320,7687278,48360198,59177718,66137019,53648154,68204242,61271144,19272365,88444207,44847298,7104732,28734791,89811711,26664538,67124282,74357852,77272628,35456853,57241521,38008118,70800879,24591705,72019362,82779622,89996536,6521313,32250045,61897582,92692978,87160386,6819644,68824981,95581843,89804152,42947632,22435353,74724075,99224392,41481685,91938191,26292919,43506672,81677380,93790285,7685448,44060493,28787861,22200378,55602660,17539625,92530431,37435892,43376279,46870723,30463802,92803766,99604946,30569392,61859581,19457589,4099191,77898274,55247455,26063929,2917920,79191827,63967300,34497327,11581181,90090964,36780454,87598888,9603598,28796059,85117092,86460488,9175338,92353856,54762643,72358170,79136082,64602895,19101477,75820087,8128637,23194618,17081350,824872,51588015,53842979,6111563,22942635,27187213,7011964,54606384,77300457,67031644,54987042,65880522,59197747,4978543,4095116,13862149,51149804,9829782,37659250,7919588,20885148,75128745,91957544,12303248,21289531,31727379,52613508,86022504,48673079,15536795,29401781,86001008,84684495,2891150,19376156,32159704,9259676,40534591,16019925,26744917,18504197,33237508,64087743,85695762,669105,92787493,415901,73168565,73021291,20836893,70541760,93359396,97940276,13468268,61623915,98943869,7300047,56955985,92692283,38256849,21673260,51715482,18131876,61815223,90457870,98371444,36468541,34295794,16097038,42073124,62452034,1204161,71083565,92033260,67030811,26102057,83651211,42237907,79738755,61928316,92554120,79880247,82327024,76315420,26139110,6808825,29516592,73392814,84349107,38009615,569864,77072625,30543215,34698428,83302115,54199160,18806856,75153252,45407418,73109997,72357096,97641116,89699445,3088684,51466049,45919976,22766820,12664567,53547802,36189527,321665,29932657,85711894,92604458,65038678,35512853,39801351,59614746,1375023,74137926,21070110,71316369,16380211,66271566,23134715,89217461,89499542,24058273,2331773,53802686,6505939,30653863,29635537,17976208,4064751,33935899,26998766,6221471,73124510,49236559,10961421,10649306,24915585,44479073,34493392,18699206,70367851,99226875,67513640,39986008,69697787,94595668,66663942,31491938,21397057,9257405,49598724,47887470,83747892,37501808,57248122,76170907,99658235,11543098,99917599,37192445,94911072,36808486,62357986,17016156,91990218,33249630,65275241,58224549,4515343,4787945,54014062,8791066,40781854,57961282,49882705,15902805,9886593,86118021,22129328,62490109,38341669,90272749,88904910,82651278,16567550,82427263,17957593,47708850,55770687,1022677,23848565,88698958,47792865,50702367,75986488,16595436,23569917,47893286,60393039,71333116,45667668,7066775,52261574,29819940,1569515,67084351,76960303,33553959,94076128,18411915,99515901,8055981,13348726,97057187,75777973,79657802,53666583,66322959,508198,17365188,65047700,99729357,91281584,96709982,4434662,47213183,92867155,59371804,4798568,14723410,54232247,66319530,56424103,72793444,79922758,19898053,42692881,21533347,66611759,72373496,73031054,56461322,68694897,64055960,95395112,66832478,44177011,66116458,45990383,15163258,85571389,22879907,54517921,59910624,50668599,9599614,16971929,69255765,26863229,48395186,57658654,76703609,5267545,42307449,31904591,8099994,8115266,16445503,56515456,78300864,65715134,32699744,70420215,34946859,71469330,94090109,71965942,8807940,95726235,22405120,61141874,17894977,8651647,32058615,58180653,65271999,78766358,91141395,59981773,66885828,53888755,35192533,43045786,92998591,749283,61263068,48675329,7182642,24314885,67451935,34432810,92071990,63372756,4985896,76434144,40865610,45848907,30366150,99690194,88130087,42644903,18001617,97281847,55960386,89419466,16684829,55615885,30090481,62051033,33123618,84127901,48892201,90310261,45794415,10309525,78884452,57020481,26275734,9575954,24953498,97379790,41245325,1936762,70036438,16387575,32151165,1239555,99125126,72238278,50806615,30811010,23740167,8634541,9623492,4091162,39373729,48260151,57163802,68702632,10597197,82178706,56153345,2607799,38510840,8129978,45049308,21919959,5832946,38061439,77301523,32426752,33895336,82339363,26734892,49328608,51426311,63506281,16424199,56484900,62693428,41092172,54868730,83133790,13173644,83533741,83368048,74441448,36135,96420429,37224844,6986898,82897371,42782652,8696647,51315460,40197395,176257,1250437,41442762,37280276,29510992,44915235,6157724,29065964,68110330,36845587,66428795,62552524,44473167,22113133,17037369,84002370,36139942,8729146,74614639,20681921,4204661,61728685,50007421,75407347,94539824,47738185,67474219,94711842,45381876,30891921,9860195,90654836,79821897,68875490,77694700,91240048,78909561,17386996,62428472,62232211,78785507,11757872,99971982,67227442,82052050,81774825,37675718,44664587,37166608,28550822,54263880,79322415,34044787,79942022,71522968,82532312,8913721,83269727,90004325,45075471,30998561,45995325,98653983,36933038,74452589,52204879,16099750,7517032,26507214,36396314,17385531 +70800879,60106217,83155442,7423788,44983451,90013093,55770687,81853704,6505939,61373987,34698428,34497327,97783876,6497830,8651647,12348118,62762857,77620120,8099994,26998766,15535065,1250437,27411561,55247455,44177011,76170907,26744917,9599614,49236559,10366309,77301523,85023028,1431742,76540481,72274002,69848388,44842615,76434144,26493119,75128745,22129328,53666583,40152546,98948034,89214616,72738685,12303248,4668450,78766358,70596786,50806615,62552524,52261574,48892201,29466406,14723410,44889423,73786944,62496012,415901,37280276,33431960,99729357,55470718,26114953,93057697,18663507,44846932,99965001,26776922,94539824,112651,62452034,99549373,98648327,30891921,31126490,81677380,50702367,89419466,29959549,68816413,65454636,6038457,39171895,38658347,35092039,73021291,9603598,44481640,98739783,77284619,33201905,40356877,29834512,71657078,30366150,59329511,96193415,62232211,48658605,67124282,42947632,62803858,39801351,61741594,99515901,5267545,65851721,37675718,66832478,50188404,1197320,91802888,63967300,29188588,70372191,11543098,34432810,33237508,68875490,69786756,1239555,86301513,81172706,43933006,52293995,22766820,57240218,22108665,1788101,24915585,91664334,84349107,39201414,63059024,669105,30787683,18783702,96726697,65074535,67281495,47792865,73124510,42692881,19272365,3233569,19486173,68102437,82178706,88653118,38341669,70541760,99021067,86821229,33895336,77072625,30543215,79136082,76330843,29932657,53888755,13862149,49882705,78549759,17365188,73617245,97057187,20836893,75052463,48088883,38494874,66319530,95581843,20885148,60581278,59501487,99224392,31904591,53648154,78884452,37891451,45306070,1569515,4515343,22721500,57658654,1204161,42237907,55602660,51426311,71300104,82532312,20122224,18504197,60955663,14220886,39986008,10649306,22435353,54427233,36930650,86001008,12416768,53547802,17957593,91727510,32250045,4978543,24619760,18699206,78589145,51590803,61960400,45848907,95395112,8791066,44847298,9188443,4798568,38008118,72278539,66611759,2891150,56461322,77413012,17894977,34946859,33797252,68702632,43045786,19101477,46851987,67793644,32161669,33628349,34493392,70036438,89699445,3183975,5111370,2607799,45237957,44348328,9886593,78785507,27325428,95957797,95251277,56515456,15902805,42967683,37739481,30771409,99524975,9259676,74110882,51047803,89046466,13470059,10358899,51464002,4099191,28734791,63628376,42307449,5073754,64087743,87598888,71920426,54014062,3509435,35192533,824872,62740044,28796059,27625735,73222868,38256849,62490109,19939935,85117092,99658235,60430369,6871053,41442762,34044787,85571389,8115266,23848565,33304202,97561745,9860195,8634541,2204165,72725103,54987042,79922758,55143724,69355476,45667668,16445503,80316608,36505482,16054533,9575954,8733068,20002147,62936963,22405120,36753250,17037369,19898053,59614746,29510992,65715134,6111563,7955293,87160386,21993752,10453030,77787724,168541,40865610,59910624,99604946,44784505,94516935,47213183,27665211,17764950,8055981,37620363,17539625,54517921,45428665,49641577,49597667,68316156,72777973,77694700,95726235,749283,40677414,10961421,6521313,77377183,75135448,11365791,54263880,83747892,66428795,56473732,20765474,48360198,4199704,61728685,4434662,21289531,64602895,64848072,62430984,18131876,77300457,874791,76671482,51472275,44245960,72357096,93566986,33123618,508198,73235980,88904910,14326617,6793819,92692283,78218845,93053405,80251430,87720882,72614359,78300864,91957544,7687278,37435892,71522968,66250369,7104732,24953498,18833224,69579137,91281584,33935899,29401781,99226875,57020481,83789391,526217,42199455,92692978,80555751,83651211,55189057,30139692,16019925,33699435,39553046,16595436,52204879,70727211,30764367,55669657,92867155,9860968,37183543,88047921,14093520,7011964,66322959,20486294,68939068,76315420,71965942,99917599,54606384,67084351,75986488,16405341,6153406,40781854,97940276,43501211,40984766,84406788,67227442,64098930,17146629,17016156,30998561,14349098,43357947,4806458,569864,36312813,93562257,93933709,51715482,84840549,99861373,4985896,24733232,35456853,92998591,65271999,55753905,24058273,84127901,74743862,92604458,26397786,20140249,49598724,40197395,57803235,89996536,23569917,45790169,90310261,44627776,21070110,15536795,99333375,6986898,84187166,77312810,36812683,96735716,38510840,68824981,97281847,34295794,98130363,91990218,81274566,5482538,9257405,59197747,34236719,36189527,27041967,61263068,24591705,83150534,75407347,74441448,61815223,75153252,6525948,3773993,15948937,61271144,17857111,5970607,66667729,54868730,95010552,47893286,71083565,78909561,85242963,19457589,54058788,16380211,70004753,50668599,36396314,68644627,38022834,61859581,93270084,86798033,9829782,9058407,93790285,16097038,51149804,33553959,92541302,97011160,63015256,19376156,8696647,7919588,92530431,15064655,4119747,30163921,32426752,43322743,61859143,51466049,95290172,18806856,89804152,17058722,56424103,19891772,92398073,68000591,77272628,82726008,48675329,89499542,53802686,4508700,71469330,15015906,12571310,3294781,38645117,15148031,32159704,65017137,37957788,23194618,6221471,85695762,7182642,2331773,42782652,69641513,4095116,73168565,28851716,37166608,9175338,17068582,30463802,3233084,36139942,9398733,29994197,17386996,29819940,43152977,33249630,77187825,99125126,50567636,53084256,61623915,6808825,33061250,83533741,46540998,66885828,69255765,96906410,69136837,34698463,72793444,96852321,92215320,48260151,69605283,83083131,79657802,14947650,59371804,90272749,13819358,36845587,41092102,22997281,44550764,79806380,30653863,76703609,32590267,32058615,7229550,83269727,60176618,51507425,80014588,26102057,88130087,16387575,78602717,89811711,28787861,92803766,52613508,67451935,49328608,7502255,20867149,27187213,51315460,10597197,18001617,32274392,44844121,27185644,93359396,3088684,94076128,98943869,31161687,55615885,8128637,81855445,26392416,16567550,7300047,61982238,73109997,92554120,94360702,88251446,12664567,43506672,72373496,74137926,82979980,38009615,36685023,13231279,13173644,9623492,38365584,91255408,26292919,60102965,82327024,1022677,8913721,41245325,98653983,26063929,79191827,7685448,49271185,68041839,75229462,83133790,24314885,45990383,48673079,17727650,54762643,21919959,90158785,35512853,68204242,15075176,54199160,22113133,58751351,54663246,3880712,73031054,82779622,91831487,58224549,67474219,7646095,26734892,11161731,3235882,2208785,11923835,96420429,79821897,78561158,62693428,75543508,89078848,31733363,98371444,4091162,82897371,74357852,10309525,14626618,44915235,22942635,72732205,8807940,40686254,247198,71316369,5832946,64157906,91141395,89637706,84684495,17385531,59177718,30569392,16099750,63506281,57248122,45075471,82886381,80246713,89217461,47887470,82651278,45995325,2717150,81898046,83302115,66137019,62051033,16424199,81774825,91938191,31491938,31727379,21397057,56484900,94595668,96709982,5640302,37501808,84904436,93515664,44473167,47361209,72495719,13348726,99297164,74724075,38061439,57241521,11581181,85116755,18411915,4069912,45919976,68128438,34667286,321665,25636669,70782102,12024238,26139110,30395570,17976208,58749,17081350,40534591,39847321,75820087,56531125,1375023,79942022,59109400,44664587,25842078,79880247,36580610,65275241,57163802,67030811,59318837,33336148,16684829,94711842,65038678,23134715,53393358,69697787,71333116,23432750,54232247,90004325,53842979,26275734,73392814,40781449,3487592,55960386,63152504,6819644,47090124,20645197,43376279,21533347,14045383,16971929,99690194,92787493,8129978,90090964,5822202,86118021,21001913,57359924,20568363,37224844,36468541,81805959,22879907,97379790,58208470,84293052,92071990,28928175,75777973,79322415,30090481,67031644,3633375,97641116,51588015,48893685,33565483,32151165,24168411,45049308,47298834,61141874,99971982,14731700,66663942,82427263,44060493,86543538,41092172,52060076,40027975,59981773,59405277,7517032,41481685,23740167,36135,61712234,4064751,66045587,91240048,20642888,4787945,176257,26863229,72238278,62428472,29065964,8373289,22450468,83210802,55850790,56153345,83948335,62115552,53632373,41380093,70367851,61928316,88444207,86242799,68110330,74614639,45381876,90654836,94911072,29516592,11757872,29635537,45794415,6157724,72019362,82052050,87710366,42073124,26507214,70420215,47708850,48774913,76960303,7066775,13468268,23110625,74452589,2917920,28550822,56955985,36933038,92033260,30694952,72358170,82339363,65880522,77898274,66271566,47738185,66903004,90457870,65047700,36780454,48395186,96318094,88698958,24826575,94090109,84002370,37560164,45407418,22200378,45996863,18466635,4204661,57393458,63372756,21673260,35996293,58180653,29029316,42644903,39373729,37659250,86460488,57961282,98462867,79738755,30218878,65338021,66116458,92353856,8729146,68694897,32699744,62357986,83368048,37192445,44479073,36808486,86022504,61897582,85711894,26664538,65081429,45617087,14363867,67513640,1936762,76825057,84166196,46870723,50007421,20681921,15163258,64055960,90061527,30811010,10264691,60393039 +80555751,53842979,53802686,90310261,54199160,94076128,52261574,81853704,66250369,78766358,32159704,65715134,3294781,89214616,63967300,82886381,37891451,34493392,96906410,46870723,54427233,7687278,72732205,28928175,68875490,79191827,36812683,37192445,83155442,45790169,68000591,61373987,669105,1197320,34432810,79657802,48360198,93515664,9623492,91255408,66667729,95581843,7423788,44177011,10366309,96193415,77413012,59371804,71657078,86821229,87160386,88904910,99604946,24591705,63059024,29819940,98130363,36753250,51472275,92033260,67793644,18783702,43357947,60106217,6986898,99917599,96318094,94516935,54014062,53547802,66045587,526217,57658654,99861373,56515456,67281495,6793819,70372191,44784505,9398733,99524975,4787945,20885148,66832478,49236559,92787493,24619760,5970607,55753905,1250437,63506281,7104732,14731700,68824981,36312813,40865610,15064655,80014588,11161731,19272365,74110882,11581181,59329511,45428665,6221471,44889423,49641577,3509435,19939935,57803235,15948937,8115266,68204242,75128745,85242963,85117092,34295794,4515343,8733068,62740044,15535065,81855445,69355476,56424103,17068582,54987042,78589145,64848072,39986008,3183975,4798568,9599614,6808825,73168565,34044787,48675329,26063929,58751351,99965001,37739481,84840549,48673079,61859581,38008118,38061439,38658347,6521313,42692881,59501487,57248122,8807940,13819358,20642888,84349107,28734791,71920426,3235882,68316156,44983451,17857111,62452034,12664567,3773993,63015256,36930650,90457870,34698428,77300457,36468541,30891921,72278539,88047921,12348118,22450468,14947650,43045786,33565483,56461322,52293995,40781449,21993752,74137926,79942022,16019925,77694700,20486294,4668450,45667668,89804152,72793444,94090109,92398073,34698463,55247455,99226875,8373289,29959549,68816413,73031054,9188443,76170907,28851716,58224549,62496012,76703609,16405341,40677414,66611759,55770687,54606384,19376156,17764950,1239555,53632373,82339363,69579137,29510992,61263068,33237508,8129978,33797252,14220886,81677380,74743862,72738685,48774913,60102965,99021067,11757872,4978543,43501211,50668599,92554120,5482538,64157906,415901,50702367,82651278,8696647,5111370,14349098,95726235,91990218,72725103,50188404,19101477,168541,86118021,89419466,50007421,72614359,26998766,42782652,83651211,75543508,17058722,79922758,15148031,79821897,18131876,21919959,33123618,31904591,42307449,89046466,32161669,20140249,27625735,72495719,93270084,7011964,66885828,88251446,84187166,29029316,17727650,86242799,93359396,20645197,78561158,98739783,19486173,39201414,30787683,54663246,53084256,26734892,47708850,8128637,52204879,11365791,8634541,38365584,44844121,35456853,26744917,87710366,37183543,42073124,20765474,80251430,77301523,65338021,17037369,7646095,38645117,13862149,40197395,824872,24058273,32058615,45848907,6871053,38494874,36845587,8913721,32151165,67124282,2717150,39373729,16971929,95957797,10961421,86022504,16387575,71333116,54762643,6038457,94539824,35192533,10453030,82726008,85023028,65271999,92692283,17365188,95395112,96726697,44550764,4985896,43376279,23110625,42947632,61859143,27665211,78218845,1375023,84684495,94911072,48893685,18806856,18466635,96709982,89078848,26392416,73392814,66428795,50806615,31733363,83789391,37675718,18833224,65851721,79806380,70727211,23569917,7955293,97783876,59614746,60430369,81274566,26275734,44842615,29994197,30366150,72373496,17539625,97940276,91802888,22108665,96735716,99515901,3088684,67451935,1569515,75407347,6497830,5073754,22766820,83210802,66663942,23134715,81898046,47213183,26664538,65038678,36933038,10309525,99658235,33553959,40356877,68128438,112651,1022677,83302115,73124510,21533347,98943869,14723410,12416768,75153252,28550822,65454636,4508700,37620363,33628349,77187825,62430984,45990383,68939068,62803858,62936963,83150534,89499542,18663507,4119747,48892201,91664334,247198,29932657,35092039,37659250,62357986,97057187,749283,23848565,49328608,36580610,32699744,69136837,97281847,9860968,2331773,77284619,5822202,51464002,36396314,4099191,26292919,83747892,41380093,70596786,30218878,74357852,2208785,62693428,30395570,92530431,50567636,30694952,89699445,40534591,874791,75777973,24915585,35512853,71316369,34236719,9829782,22113133,47298834,72357096,321665,53666583,64602895,41245325,69848388,98948034,47792865,58749,14045383,14093520,29188588,61928316,26507214,91240048,33304202,80246713,4069912,82052050,57163802,59318837,56473732,5267545,40027975,77377183,45075471,64055960,61960400,73021291,62552524,49271185,65275241,26114953,5832946,33249630,7685448,55850790,34497327,65017137,97561745,84293052,56531125,14326617,61741594,6525948,9175338,89637706,33201905,1204161,16424199,17976208,4434662,93933709,29466406,1788101,77072625,90090964,22997281,17016156,3487592,78300864,73617245,70782102,77620120,93053405,9603598,26776922,69697787,61728685,56484900,40781854,62115552,30771409,96420429,73222868,79136082,35996293,89996536,70541760,33061250,9058407,76825057,29834512,45996863,16097038,21673260,39801351,74441448,62232211,33895336,21001913,9575954,95010552,18001617,44481640,62051033,31727379,41481685,45306070,26102057,75229462,40984766,10358899,83533741,44915235,17146629,84166196,21289531,508198,68644627,67084351,78884452,72274002,67031644,99549373,66322959,30543215,74724075,18411915,54517921,39171895,86460488,26863229,11923835,82779622,79880247,17081350,8099994,15015906,33336148,30653863,78602717,6505939,57241521,82979980,81774825,20836893,13470059,98648327,16380211,27041967,60176618,67030811,92692978,82178706,31126490,32250045,75820087,30764367,53888755,93790285,82427263,90013093,99971982,71965942,10649306,52060076,89811711,97641116,48260151,22942635,27411561,37280276,93566986,30090481,76540481,6153406,25636669,71083565,91938191,44348328,37501808,29065964,66903004,12303248,94360702,59109400,20002147,55143724,3880712,4806458,44847298,61897582,26139110,73235980,61982238,90004325,55189057,53648154,88444207,17894977,7502255,65074535,66137019,12024238,45919976,71522968,80316608,51047803,84904436,92998591,76315420,9886593,82532312,14626618,73786944,2891150,30139692,38341669,10597197,30811010,8651647,27325428,72019362,36135,84002370,70800879,71300104,45617087,77272628,84406788,29401781,91957544,88653118,87720882,51507425,7517032,44245960,90272749,7919588,45794415,57961282,8055981,6819644,63372756,49597667,54232247,57020481,13468268,47090124,27185644,47893286,13231279,61271144,97379790,28796059,86001008,36808486,24953498,15536795,54058788,82897371,16054533,91831487,91727510,45049308,46851987,19457589,16099750,47361209,68102437,87598888,77898274,86543538,23432750,9259676,34946859,40152546,23194618,66271566,88130087,19898053,62490109,3633375,23740167,44473167,24314885,32426752,83368048,2204165,76960303,20122224,27187213,46540998,74452589,22129328,51315460,99125126,43933006,47887470,55602660,17385531,99224392,2607799,85116755,22721500,33431960,64087743,15075176,569864,176257,96852321,37435892,68694897,24733232,67227442,62762857,22200378,56153345,2917920,81172706,56955985,94595668,58180653,51466049,65081429,39553046,84127901,67474219,1431742,93057697,90654836,41442762,33699435,60955663,94711842,95290172,70036438,54868730,81805959,19891772,76434144,4199704,20867149,4091162,43152977,68041839,21070110,92215320,85695762,90158785,65047700,29635537,72777973,86301513,36685023,68110330,36139942,85571389,54263880,22435353,34667286,69641513,83083131,30569392,44846932,16445503,42199455,59405277,37166608,15163258,78549759,77312810,65880522,51715482,57359924,37560164,37957788,92604458,83269727,78785507,74614639,53393358,72358170,58208470,69255765,98653983,21397057,30163921,13348726,32274392,76671482,15902805,28787861,68702632,61712234,30998561,98462867,66319530,45237957,7300047,70004753,70420215,16567550,3233084,82327024,41092172,69786756,38022834,24168411,92071990,18504197,7229550,43322743,16595436,22405120,70367851,98371444,92541302,44479073,22879907,51149804,99297164,8729146,45381876,4064751,36780454,4204661,32590267,26493119,48088883,92353856,11543098,99729357,8791066,18699206,60581278,69605283,38510840,42644903,4095116,45407418,59197747,12571310,5640302,26397786,17957593,99333375,33935899,63628376,93562257,20568363,85711894,42237907,88698958,36505482,45995325,42967683,44060493,75986488,38009615,1936762,48658605,57240218,48395186,75135448,52613508,30463802,51590803,61141874,89217461,66116458,73109997,16684829,40686254,90061527,92803766,59910624,77787724,61815223,55470718,41092102,51426311,59177718,9860195,86798033,72238278,39847321,49598724,55960386,92867155,6111563,31491938,83948335,44627776,95251277,13173644,55615885,83133790,61623915,57393458,91281584,71469330,7182642,78909561,49882705,31161687,36189527,9257405,38256849,47738185,17386996,79322415,43506672,51588015,76330843,91141395,64098930,97011160,10264691,3233569,24826575,99690194,25842078,7066775,29516592,44664587,20681921,6157724,79738755,59981773,14363867,63152504,55669657,75052463,62428472,67513640,37224844,60393039 +5111370,52613508,45407418,43376279,89078848,70036438,24058273,51047803,47090124,44481640,47792865,24168411,95957797,54663246,63059024,30090481,84904436,44983451,60102965,10453030,20765474,1431742,4806458,55247455,35996293,29065964,90457870,98371444,67474219,33201905,93566986,96735716,20002147,29994197,19101477,168541,68644627,71333116,2717150,669105,5482538,88653118,68000591,75543508,29834512,2607799,94595668,76330843,92604458,99524975,99549373,9058407,51464002,9257405,89214616,73786944,83302115,3509435,18411915,66667729,45617087,49598724,16445503,62430984,85116755,60430369,49641577,55753905,72373496,66045587,75777973,71657078,18783702,68816413,35192533,30463802,36808486,53666583,38061439,96193415,17727650,70800879,18699206,43322743,61623915,22721500,56531125,8807940,65338021,39801351,48088883,83651211,6793819,34236719,415901,34044787,45794415,52261574,65275241,55960386,66250369,43357947,62693428,76540481,14093520,4204661,4099191,29029316,7646095,33553959,78766358,41092102,13348726,73124510,27411561,50702367,97561745,66832478,26392416,7919588,82979980,92998591,66903004,16097038,21070110,92803766,36780454,86001008,92787493,61859143,45306070,14326617,38658347,34698428,13862149,75986488,84349107,78909561,72614359,79191827,76434144,36312813,15148031,67084351,61263068,4668450,84840549,10309525,93515664,508198,62452034,32426752,99604946,44842615,6525948,22942635,27665211,74137926,47361209,91281584,4095116,83368048,17764950,95395112,92692283,24619760,30366150,69136837,42644903,66137019,9599614,99965001,99226875,14626618,49271185,81853704,13173644,33797252,83747892,61373987,30891921,94711842,39373729,84166196,19272365,25842078,73617245,67793644,63506281,18131876,90310261,98648327,91802888,29819940,60955663,58751351,45075471,86301513,79657802,37620363,46851987,35092039,33237508,53547802,247198,84187166,69355476,17016156,26507214,14220886,2331773,5640302,71522968,44479073,52204879,92215320,57240218,2204165,67513640,16054533,98739783,67281495,35456853,89499542,39553046,57248122,54199160,9175338,90061527,75229462,99125126,93562257,54058788,95010552,48260151,95290172,321665,32590267,49328608,77272628,37183543,77787724,76671482,77284619,44889423,22113133,92353856,36580610,72777973,55470718,18504197,4119747,7011964,4508700,99861373,28928175,59197747,34493392,10358899,38008118,4064751,30395570,94360702,62357986,61982238,40984766,47738185,33249630,85571389,2917920,94090109,71920426,53802686,93933709,31161687,92398073,26397786,77187825,76170907,46870723,8129978,37739481,1204161,7423788,72738685,14947650,62936963,19486173,8651647,15064655,26493119,36505482,74357852,9188443,16380211,4787945,67031644,84684495,73031054,71083565,14363867,57020481,61897582,61271144,5832946,1375023,82897371,15163258,22450468,57393458,10597197,30764367,16567550,40197395,62740044,20140249,38022834,48893685,15535065,7517032,47708850,41442762,82886381,8099994,77694700,50188404,81898046,40781449,66271566,61141874,42967683,54762643,68939068,88251446,65047700,59910624,30139692,16405341,11543098,74743862,26776922,12664567,7955293,96726697,87710366,6505939,85117092,9829782,22879907,69255765,19376156,43152977,61815223,32250045,4515343,67124282,97940276,74441448,80014588,74614639,39171895,99333375,56473732,31126490,90272749,176257,10649306,86118021,51590803,40865610,62803858,65271999,19939935,71300104,83210802,72725103,74110882,3773993,18663507,59177718,48892201,32151165,86022504,89811711,86242799,61960400,98943869,69786756,5970607,53632373,97641116,10366309,96906410,54987042,11923835,81855445,76825057,53648154,92554120,6221471,29188588,79136082,18833224,70367851,80316608,59329511,15948937,99021067,8733068,85023028,69641513,37659250,19891772,48395186,526217,70727211,824872,20568363,54232247,16099750,51588015,4798568,6157724,40686254,68694897,10961421,12571310,66428795,99515901,67227442,56424103,91831487,44473167,99658235,68824981,33061250,15015906,28851716,3183975,874791,34497327,40534591,62232211,65715134,73168565,37675718,51507425,56515456,73235980,54517921,30811010,29635537,8791066,64602895,82052050,30218878,60176618,2208785,51426311,45237957,42073124,17386996,57163802,14045383,89419466,93270084,8055981,84293052,48673079,33895336,36812683,32161669,38494874,51472275,70372191,20885148,42947632,13819358,61712234,62051033,41380093,44177011,1197320,38365584,24733232,55770687,9603598,73392814,16595436,28734791,83150534,12416768,45995325,37192445,77072625,83533741,85242963,25636669,77620120,82339363,38341669,40027975,50806615,36933038,76960303,75820087,58749,29466406,6497830,78218845,83083131,44915235,9623492,66322959,27325428,33336148,20486294,3233084,77312810,99971982,112651,66885828,29959549,88047921,62762857,43501211,91957544,5822202,59109400,79821897,3880712,30998561,35512853,61728685,86821229,45996863,56153345,49882705,72238278,6153406,34946859,33628349,89996536,92071990,30569392,38645117,63372756,52293995,43045786,80251430,33699435,91141395,91727510,91990218,65074535,55143724,59614746,6521313,62115552,81677380,18466635,20645197,80246713,98462867,42237907,55189057,45667668,97011160,23569917,27187213,75135448,97783876,62552524,99297164,87598888,95726235,16684829,44844121,65038678,17539625,67451935,66611759,57803235,49236559,21993752,97379790,72274002,15075176,65017137,7182642,67030811,64098930,91664334,17081350,57241521,44348328,26063929,64848072,23848565,42199455,92033260,26275734,34295794,37166608,36135,6986898,8729146,68204242,79922758,89046466,54606384,82427263,70004753,63628376,45919976,8634541,78602717,36396314,31904591,50668599,27041967,9575954,30163921,90013093,88130087,1239555,77377183,77413012,50007421,39847321,58224549,36468541,42692881,75052463,56461322,89637706,8373289,52060076,17385531,41245325,74724075,48360198,17857111,33935899,90090964,4069912,47298834,98130363,39201414,22435353,38009615,99690194,54014062,64157906,96318094,7687278,82726008,83133790,7300047,40152546,21673260,3294781,61741594,22108665,33565483,36930650,43933006,81805959,37891451,26998766,94076128,1569515,65880522,36753250,95581843,88444207,64087743,20867149,22200378,70596786,5267545,21533347,68316156,81774825,40781854,88904910,31733363,37957788,77300457,76703609,17068582,20681921,90158785,83948335,65851721,26114953,21001913,31491938,93053405,72495719,37501808,48675329,36845587,44846932,45790169,96709982,34432810,30787683,57961282,56955985,37224844,51715482,63967300,17957593,12348118,17037369,44784505,83155442,78884452,53888755,8913721,26139110,44847298,28796059,39986008,70420215,23194618,45428665,49597667,89217461,72278539,41092172,26863229,44245960,62496012,22766820,30653863,1250437,16971929,71965942,36189527,21919959,82532312,83269727,92541302,84002370,9886593,73222868,55850790,16019925,26664538,83789391,47887470,79806380,92530431,97281847,59981773,60581278,11757872,55669657,97057187,78589145,41481685,88698958,68702632,54868730,69605283,16387575,63152504,46540998,21397057,82779622,60106217,34698463,4091162,6111563,53842979,64055960,24591705,66319530,17058722,7685448,75407347,14731700,68102437,29401781,33431960,65081429,72793444,99917599,3235882,22997281,16424199,2891150,78561158,6871053,66116458,62490109,5073754,18806856,61859581,14349098,77898274,33304202,87160386,40356877,59501487,33123618,78785507,75128745,89699445,69697787,79738755,18001617,85695762,78300864,91938191,44664587,81274566,749283,99729357,80555751,31727379,68128438,98948034,96420429,27185644,82178706,6819644,50567636,59405277,7502255,1936762,32058615,51466049,54427233,76315420,6808825,3088684,19457589,11365791,26744917,3233569,40677414,82651278,13231279,15902805,96852321,22405120,89804152,44060493,20122224,59318837,15536795,12024238,70541760,84406788,84127901,94516935,37435892,38256849,24915585,78549759,57658654,58180653,36685023,81172706,17894977,569864,59371804,3487592,11581181,72019362,28550822,26102057,7066775,86543538,55602660,51315460,23432750,43506672,53084256,24953498,42307449,66663942,91240048,58208470,17976208,45381876,20642888,90004325,8115266,48774913,65454636,68875490,71469330,79942022,94911072,4985896,32699744,27625735,72357096,45848907,38510840,73109997,56484900,48658605,17146629,51149804,86460488,23110625,54263880,7104732,23740167,44550764,24826575,19898053,8128637,74452589,36139942,55615885,75153252,45990383,53393358,90654836,10264691,26292919,93359396,34667286,87720882,85711894,98653983,86798033,93057697,45049308,23134715,29932657,6038457,29516592,9860195,1788101,17365188,32159704,92692978,69579137,28787861,93790285,30543215,30694952,21289531,62428472,42782652,20836893,9398733,1022677,69848388,72358170,79322415,94539824,44627776,7229550,13470059,3633375,91255408,72732205,95251277,79880247,22129328,82327024,9259676,47893286,24314885,57359924,73021291,37280276,99224392,13468268,92867155,12303248,30771409,68041839,60393039,71316369,4434662,11161731,8696647,32274392,61928316,14723410,4199704,47213183,26734892,29510992,77301523,4978543,70782102,63015256,37560164,68110330,9860968 +67451935,63967300,4095116,13173644,55753905,36505482,56424103,17764950,9886593,29466406,62496012,12348118,44983451,88698958,76960303,64087743,40152546,48673079,85116755,33237508,24733232,60102965,31126490,54987042,9575954,75986488,10597197,68875490,17037369,569864,40356877,57163802,13231279,83210802,89637706,82327024,29188588,22997281,72373496,45996863,78602717,71469330,73235980,87598888,78589145,51047803,87710366,59177718,13468268,29819940,57803235,92398073,71333116,14947650,30764367,90090964,7423788,8791066,20002147,86022504,9623492,14220886,2204165,94516935,99965001,6153406,36780454,89419466,92787493,27665211,5832946,30395570,65715134,80316608,9860195,8099994,61982238,7182642,35092039,14731700,79322415,78909561,68041839,3487592,42644903,16567550,66319530,1788101,29834512,40027975,8807940,66832478,65851721,20486294,63372756,4099191,68702632,20140249,83155442,79657802,18663507,38645117,18806856,15015906,5111370,48260151,73031054,44915235,54762643,68128438,9188443,35456853,82427263,97057187,55247455,23569917,10366309,74357852,67227442,99549373,247198,71657078,96726697,98371444,63152504,79821897,45848907,61960400,18783702,92033260,70372191,88047921,49882705,25636669,45237957,35192533,37739481,81172706,321665,44889423,99971982,78766358,66428795,7646095,20122224,72357096,55143724,48892201,96852321,26114953,45995325,88653118,43501211,21070110,66885828,98653983,22942635,45407418,30139692,41092102,7955293,93933709,85117092,33699435,66667729,57240218,59329511,7011964,56515456,7685448,51464002,65275241,32426752,26292919,94090109,8115266,77694700,79922758,9599614,82178706,41245325,30771409,21993752,3509435,67474219,95581843,32699744,27325428,59981773,168541,83302115,91802888,73124510,92803766,60955663,92071990,29994197,71920426,81898046,34698428,38008118,669105,29959549,56531125,1197320,96193415,61928316,749283,45617087,95290172,16019925,72274002,88251446,98648327,76170907,80251430,10309525,19898053,33895336,82886381,29516592,43045786,58180653,40677414,34295794,20867149,90272749,59197747,68816413,55602660,42237907,77620120,34236719,28550822,44784505,84684495,53547802,30998561,45428665,61263068,32058615,6221471,4091162,81774825,78785507,48658605,82897371,7919588,40781854,99917599,96709982,71300104,15535065,28928175,53632373,18504197,55850790,83150534,1431742,15064655,73786944,66663942,415901,62803858,70541760,30787683,4119747,37659250,89078848,40865610,17386996,44479073,93515664,44844121,37620363,34698463,70036438,55669657,70004753,56473732,9860968,70727211,96735716,5267545,50188404,54199160,508198,16445503,7517032,42967683,91141395,22129328,42073124,89811711,83133790,78300864,30543215,17957593,73222868,874791,68204242,28734791,36753250,48675329,4668450,98462867,54517921,77312810,6793819,10649306,47361209,27411561,63015256,26507214,5822202,92692283,9175338,15075176,73617245,61815223,84904436,55189057,86242799,31904591,2891150,84349107,38022834,35996293,81805959,7502255,26139110,26664538,34493392,49641577,63506281,2208785,81274566,99604946,51472275,67793644,91831487,69605283,5640302,83368048,59318837,37183543,38009615,8733068,59910624,90013093,5970607,16684829,71316369,77284619,32590267,14093520,64602895,92554120,8651647,30163921,79136082,69579137,6808825,44348328,66903004,30090481,99729357,89214616,83269727,65880522,99515901,24619760,64157906,97783876,22405120,45667668,99690194,32159704,65271999,16099750,62936963,37435892,21533347,65047700,11923835,15902805,76315420,55470718,40781449,75820087,69697787,67281495,12416768,43933006,8634541,18131876,83747892,32250045,33797252,47708850,19272365,66116458,50806615,3233569,39847321,77072625,70800879,99861373,76434144,6986898,19891772,47298834,30366150,3633375,54868730,86001008,93562257,49271185,61859581,58208470,66271566,73021291,8129978,3088684,82979980,47887470,43506672,81853704,52060076,26493119,33553959,91727510,33431960,98130363,73168565,18833224,16097038,23194618,23848565,99125126,32274392,16054533,85571389,55615885,45075471,77272628,42692881,38256849,59501487,84127901,72725103,43322743,66322959,51149804,45919976,14326617,32161669,16405341,2717150,24826575,11161731,3233084,51507425,3183975,31161687,53842979,61623915,51588015,23110625,93053405,1204161,2607799,65017137,42307449,98739783,34044787,82726008,89699445,63059024,17539625,75135448,62430984,26776922,10358899,62693428,86460488,99658235,71522968,57658654,63628376,99524975,1569515,5482538,94911072,47090124,84840549,7300047,52613508,39171895,44550764,40984766,64098930,99021067,1250437,30463802,3294781,95395112,22450468,10453030,21289531,18699206,17058722,93790285,6038457,28796059,14349098,83651211,95726235,89046466,99297164,22435353,36930650,69355476,33123618,22879907,91664334,76540481,824872,24058273,56153345,92867155,4434662,9829782,39986008,38061439,56461322,37166608,72238278,53888755,6521313,52204879,4985896,60106217,61728685,73392814,36580610,9058407,44481640,76825057,5073754,92692978,16971929,76330843,16424199,40686254,68102437,92530431,61859143,83948335,66250369,50668599,78218845,50702367,52261574,13348726,18411915,30811010,90004325,22108665,66611759,69641513,44177011,95010552,44060493,87720882,79738755,49328608,19376156,93270084,85695762,30694952,48088883,4787945,98943869,79191827,61741594,44473167,27625735,62552524,77787724,45790169,18001617,80555751,77300457,46540998,72777973,36685023,65038678,18466635,43376279,39201414,19101477,88904910,45306070,7229550,10961421,94539824,29510992,15148031,44847298,13819358,11543098,72614359,56484900,56955985,7066775,92215320,37501808,8128637,34497327,90457870,44846932,4069912,28787861,54663246,91990218,84293052,72358170,89804152,20765474,34432810,20836893,84187166,65338021,94595668,80246713,38658347,78561158,94076128,50567636,69136837,15536795,46870723,55770687,68000591,54606384,72278539,57241521,68694897,72793444,59371804,83533741,62740044,9398733,54232247,97940276,55960386,43357947,74614639,82532312,75229462,19486173,87160386,82052050,6525948,68939068,17727650,11581181,8696647,49236559,86301513,82339363,4806458,85242963,16595436,71965942,9257405,12571310,54014062,92353856,61271144,70420215,62232211,74137926,6505939,12024238,59109400,80014588,79806380,44627776,75777973,62115552,93057697,75543508,45381876,24953498,30653863,112651,26392416,57393458,97561745,9603598,44842615,48774913,4204661,75153252,33304202,62762857,71083565,15163258,33201905,6871053,17068582,7104732,68644627,14045383,36808486,77377183,30891921,75128745,3773993,58749,39553046,92541302,91957544,33565483,4515343,8913721,62490109,27187213,17146629,37891451,77898274,14626618,53666583,76671482,59614746,30218878,86821229,81855445,97011160,38494874,25842078,46851987,72738685,54058788,21673260,24915585,26998766,39801351,62428472,42782652,91240048,31727379,49598724,67124282,47792865,60581278,36468541,36312813,89217461,53802686,58224549,8729146,65454636,47213183,79880247,48893685,66137019,4199704,42947632,77187825,60430369,37192445,17081350,24168411,53648154,69848388,20568363,37280276,88444207,72019362,94360702,17894977,72732205,27041967,6497830,17365188,29065964,8373289,82651278,26397786,40534591,67084351,23432750,74110882,77413012,32151165,16380211,53393358,31491938,98948034,47738185,95957797,34667286,20885148,41481685,38341669,21919959,34946859,47893286,65074535,36189527,79942022,19939935,78884452,84002370,57961282,35512853,92604458,30569392,176257,70596786,22721500,37957788,84406788,41442762,37675718,97641116,36396314,90158785,28851716,19457589,61373987,3880712,57248122,66045587,33249630,36845587,72495719,88130087,4978543,65081429,67031644,57359924,90310261,74743862,96318094,68316156,99333375,33061250,51315460,83083131,20642888,83789391,78549759,26275734,33628349,17976208,93359396,81677380,44245960,48360198,2917920,90061527,10264691,76703609,1936762,64055960,9259676,4508700,26744917,14723410,51466049,6819644,29635537,13470059,54263880,45794415,4064751,61141874,61897582,26734892,51426311,62452034,85711894,6157724,36933038,29029316,69786756,45990383,91938191,99226875,36812683,60393039,74441448,22766820,84166196,26063929,61712234,41380093,2331773,11757872,22200378,64848072,12664567,26102057,50007421,89996536,1022677,58751351,75052463,36135,97379790,67030811,60176618,85023028,96420429,53084256,42199455,91281584,67513640,68824981,68110330,54427233,17016156,51715482,90654836,86798033,7687278,62051033,26863229,20645197,74724075,1239555,17385531,39373729,24591705,77301523,24314885,82779622,49597667,40197395,73109997,69255765,74452589,97281847,14363867,51590803,93566986,21001913,44664587,70367851,27185644,99224392,37560164,52293995,43152977,13862149,4798568,37224844,33336148,23134715,62357986,36139942,29932657,94711842,86118021,31733363,38365584,8055981,526217,89499542,92998591,22113133,57020481,86543538,23740167,70782102,20681921,91255408,33935899,45049308,38510840,15948937,1375023,17857111,75407347,95251277,12303248,48395186,41092172,96906410,16387575,3235882,59405277,6111563,11365791,29401781,21397057 +10358899,2891150,8634541,47792865,72373496,53666583,19272365,415901,77620120,29834512,7011964,98943869,76330843,11923835,31491938,37620363,14220886,99524975,55960386,22129328,24058273,6808825,68824981,29959549,62936963,68102437,56515456,26776922,6497830,69255765,37659250,99515901,76170907,59981773,28928175,73124510,27665211,26744917,96420429,68816413,96735716,45790169,95957797,96193415,66885828,93270084,21001913,93057697,52261574,79136082,49271185,12348118,49641577,17365188,22766820,18663507,9829782,91802888,35996293,26734892,39553046,14093520,75543508,95395112,2717150,40865610,51047803,19376156,85242963,76540481,40197395,34698428,97641116,21993752,569864,69355476,74614639,33249630,33201905,47361209,95726235,38658347,67793644,49597667,43501211,77694700,29994197,52204879,50188404,54199160,55247455,4668450,32161669,79806380,16380211,9623492,44847298,8729146,40781854,83083131,6986898,5970607,78884452,65074535,62452034,71300104,22997281,16445503,33237508,508198,85116755,15064655,48893685,90310261,45237957,50806615,89214616,14947650,34493392,13862149,40152546,60430369,53632373,72738685,80014588,65715134,8807940,57163802,60955663,17957593,53842979,20765474,18806856,69579137,35092039,96726697,44245960,98948034,84002370,92604458,98462867,44473167,41092102,98648327,60581278,42967683,30463802,46851987,40984766,67030811,54517921,68644627,44844121,48892201,38061439,36753250,45794415,55470718,26392416,82178706,65017137,38494874,14326617,3294781,53802686,81805959,9188443,63967300,76703609,92692978,83533741,1250437,66045587,44889423,7423788,81855445,6871053,59501487,69136837,8055981,98739783,65851721,5111370,73235980,44481640,91727510,9886593,4508700,67084351,40534591,58749,23848565,33431960,33797252,62496012,39201414,96709982,30395570,88653118,51426311,36468541,40686254,73786944,78602717,92353856,78218845,99021067,93053405,61741594,14349098,3487592,61960400,44348328,44842615,62740044,51590803,67281495,61623915,112651,78766358,61373987,70541760,93359396,68204242,84904436,8129978,68000591,20002147,15535065,99917599,33304202,39801351,75128745,99658235,51507425,20140249,83150534,27325428,66832478,64098930,9575954,19101477,31904591,80251430,64055960,71920426,37501808,50702367,48675329,39847321,63059024,55850790,72238278,17058722,1375023,86301513,44177011,25636669,31161687,54232247,68694897,90090964,40027975,44846932,7687278,62232211,23569917,89046466,61982238,65271999,57240218,7300047,55189057,5832946,38256849,80555751,20645197,6525948,89499542,72777973,48360198,18783702,11543098,20681921,93933709,81172706,74357852,72019362,87598888,30998561,4204661,36933038,15902805,96318094,98371444,34432810,67227442,7919588,77312810,73392814,47090124,36812683,11757872,91664334,45996863,68316156,9398733,36930650,83133790,58751351,23194618,99226875,41481685,89699445,7517032,30218878,34044787,8651647,81853704,86001008,39986008,9259676,64848072,18131876,69641513,94090109,50567636,19939935,51715482,26114953,95581843,62430984,59109400,4434662,54014062,68110330,65338021,43376279,82726008,54606384,14045383,83747892,90013093,2607799,30694952,57020481,8733068,24619760,3509435,3088684,45306070,6038457,29401781,4095116,99125126,34497327,57961282,53084256,66250369,42692881,59197747,3880712,4119747,59371804,74441448,84127901,70372191,49598724,73031054,40781449,36312813,85023028,5482538,1431742,18466635,48774913,89419466,4787945,61728685,24591705,57803235,72725103,98130363,61815223,84187166,53648154,26139110,53393358,71333116,57393458,6793819,36580610,3233084,99861373,73617245,3183975,70800879,97561745,77787724,26998766,58208470,9058407,59614746,52613508,59177718,8128637,54058788,9257405,10366309,32274392,92398073,36845587,43933006,20885148,62803858,54427233,72278539,36808486,247198,61859143,82532312,13231279,1204161,21070110,93562257,48395186,12416768,44983451,27411561,60106217,26275734,56424103,94595668,66322959,79657802,5073754,99604946,74137926,62693428,8791066,10453030,34698463,10649306,16595436,63015256,1197320,18833224,57658654,70596786,17764950,13819358,15163258,5640302,14363867,71469330,42073124,49328608,84349107,77301523,45995325,66903004,89217461,16684829,32159704,72732205,85711894,97783876,26102057,56531125,31733363,30787683,78909561,33895336,29466406,81677380,83948335,47738185,49236559,33061250,9599614,71965942,33628349,18699206,526217,77284619,66137019,42237907,64157906,84684495,16405341,42947632,40677414,37166608,44550764,11161731,91990218,51464002,33553959,29819940,79922758,89078848,32590267,61271144,61263068,81898046,30653863,38008118,6153406,8696647,36505482,76671482,22405120,90061527,75229462,2204165,45428665,4806458,66611759,14626618,7646095,4515343,39373729,8115266,45919976,21289531,86798033,77300457,99965001,56473732,66667729,8099994,88251446,38510840,4064751,17386996,874791,55770687,92803766,35512853,20486294,35456853,17068582,90457870,38022834,6505939,56153345,70004753,91831487,38365584,22721500,71522968,99297164,29932657,53888755,1022677,6819644,52293995,29029316,77898274,1569515,33565483,4069912,22450468,66663942,12664567,45075471,54263880,70367851,44664587,99729357,8373289,19898053,62552524,96852321,56461322,29516592,19486173,37435892,88444207,68128438,89811711,71657078,84406788,16424199,16054533,97011160,83210802,24168411,64087743,29510992,35192533,72793444,86118021,13468268,82327024,43152977,32699744,28851716,82651278,37891451,70420215,44784505,7955293,9603598,70727211,749283,95010552,17146629,5822202,72357096,55669657,60102965,61141874,21673260,4091162,4099191,15015906,29188588,37675718,79322415,84840549,99224392,82339363,71083565,22942635,3773993,19891772,94516935,30569392,29065964,27187213,669105,45667668,82052050,25842078,17016156,67124282,37192445,99549373,77072625,50668599,97057187,45990383,86242799,69697787,47213183,46870723,67474219,4798568,48658605,57359924,54762643,59405277,29635537,7182642,61859581,69605283,36780454,5267545,30891921,34946859,32426752,16097038,65081429,13470059,61712234,42782652,31727379,86022504,83789391,45407418,78589145,18504197,38009615,16019925,48088883,73168565,32250045,22113133,3235882,93566986,73222868,65454636,95290172,32151165,70036438,43045786,24826575,83368048,26397786,6221471,84166196,52060076,33699435,57241521,97281847,168541,68875490,55753905,92541302,18001617,45381876,63628376,77187825,94539824,24915585,36189527,62490109,7229550,65047700,72495719,45848907,33935899,87720882,75135448,17081350,65275241,54663246,48260151,8913721,34667286,79821897,41442762,26292919,51588015,79191827,7685448,91281584,2331773,49882705,88047921,99971982,82886381,62762857,26493119,38645117,78300864,30090481,89637706,18411915,24953498,21533347,22108665,20867149,87710366,75777973,91957544,33123618,34295794,15536795,84293052,77413012,90004325,99690194,14731700,92554120,62115552,53547802,16971929,37739481,68939068,71316369,15148031,74110882,74724075,41245325,78561158,20122224,62051033,19457589,17727650,30764367,85571389,12571310,39171895,36135,92215320,17894977,321665,94911072,86460488,68702632,80316608,90272749,3233569,91255408,75407347,9860195,3633375,41092172,59329511,58180653,72614359,23134715,22879907,77272628,92787493,94360702,54868730,9175338,30139692,28734791,1936762,55143724,2208785,31126490,57248122,47298834,17385531,91141395,37224844,14723410,26063929,17976208,92998591,10597197,58224549,43357947,92867155,82979980,6111563,40356877,92530431,78549759,75052463,7066775,1788101,75820087,33336148,21919959,4985896,16099750,83269727,26507214,83651211,34236719,73109997,24733232,66116458,66271566,30366150,75986488,89996536,37957788,9860968,63506281,87160386,74743862,80246713,30771409,27625735,43506672,56955985,59910624,74452589,56484900,92071990,72274002,88904910,16567550,23432750,67513640,22435353,78785507,64602895,79942022,61928316,51472275,7502255,83302115,67451935,20568363,51315460,69786756,88698958,1239555,76434144,20642888,73021291,70782102,176257,26664538,94076128,89804152,28550822,6157724,86543538,16387575,90158785,22200378,51466049,91240048,43322743,76315420,10309525,44479073,63372756,51149804,65038678,55602660,91938191,92033260,12303248,2917920,11581181,66319530,36396314,42307449,4199704,6521313,17857111,44060493,37183543,54987042,75153252,21397057,48673079,15075176,97940276,86821229,68041839,41380093,93515664,85695762,824872,17539625,63152504,17037369,60176618,62428472,32058615,69848388,99333375,47887470,47708850,85117092,95251277,36139942,27041967,83155442,13173644,98653983,24314885,67031644,82897371,92692283,65880522,7104732,30163921,81274566,4978543,82779622,90654836,79880247,45617087,12024238,44627776,47893286,81774825,13348726,36685023,97379790,66428795,20836893,42644903,42199455,30543215,37280276,94711842,96906410,59318837,50007421,30811010,76825057,44915235,61897582,72358170,26863229,79738755,45049308,27185644,93790285,23740167,76960303,10264691,38341669,60393039,88130087,28787861,82427263,46540998,28796059,10961421,77377183,55615885,37560164,11365791,23110625,15948937,62357986 +13231279,99965001,98739783,99549373,61859143,59197747,54014062,73786944,27665211,28796059,247198,95290172,112651,44889423,51047803,6819644,12348118,86821229,36812683,4806458,28851716,66322959,5111370,47090124,82979980,19939935,37183543,51464002,36505482,9623492,30395570,18783702,569864,49641577,36930650,30787683,35456853,37739481,508198,7502255,90272749,36580610,83368048,36780454,62936963,43376279,98462867,45428665,99861373,96726697,61373987,669105,40197395,59910624,92554120,15148031,68939068,29994197,92692283,83210802,8696647,71657078,90061527,65017137,79657802,99125126,77284619,33565483,85116755,6793819,11161731,88653118,7300047,1431742,8807940,39986008,18833224,90013093,26998766,93566986,57961282,44550764,10309525,26493119,24591705,74743862,16445503,99224392,43045786,68316156,39171895,73031054,98371444,1022677,19101477,97783876,55247455,9886593,26063929,68041839,60102965,13173644,98130363,63506281,62496012,29959549,1569515,83302115,61897582,9603598,40781449,89214616,82726008,50702367,17068582,15536795,74137926,70727211,20002147,61859581,66319530,22450468,42199455,64087743,60430369,3294781,64157906,31733363,37675718,14093520,67227442,45617087,65038678,63059024,40677414,39847321,33553959,92803766,45996863,55753905,8733068,81853704,71333116,3633375,1204161,85242963,91802888,59329511,69255765,54762643,99604946,82779622,10597197,36189527,45990383,80014588,48260151,72738685,48675329,4099191,14349098,68702632,29932657,51588015,27411561,35092039,73392814,30218878,6521313,81274566,76434144,83155442,18663507,37501808,65851721,22721500,65275241,13819358,84166196,61815223,29819940,34236719,56515456,22879907,66611759,62452034,67124282,72358170,13470059,30366150,51466049,16380211,97057187,7423788,28928175,75407347,68875490,84187166,56424103,81677380,20486294,57248122,3233569,24733232,4787945,69355476,76960303,22942635,44348328,68644627,32161669,10366309,65074535,18806856,93562257,53547802,72793444,87710366,7955293,85571389,874791,12571310,12416768,83651211,15948937,44245960,77312810,12024238,14045383,73222868,87720882,50188404,3235882,32159704,91664334,18411915,28550822,36312813,32058615,16971929,22997281,70036438,89811711,77413012,38494874,84293052,8791066,2717150,67084351,53666583,77377183,95395112,23848565,97561745,75128745,63967300,6808825,51715482,6986898,42073124,33201905,91957544,29834512,69605283,56473732,58751351,22129328,83150534,15163258,55143724,89419466,43357947,62428472,38008118,96193415,17016156,89499542,8651647,14326617,74357852,5822202,77620120,73168565,54663246,94539824,99333375,30463802,88904910,5640302,66250369,44844121,4095116,33304202,15015906,24915585,61263068,24314885,22108665,77694700,26397786,8634541,44983451,6153406,61982238,42947632,76540481,34295794,41092102,26776922,69786756,38061439,42644903,47708850,59109400,92998591,97940276,16595436,43152977,67281495,70596786,62693428,16097038,17727650,82178706,62803858,42237907,95726235,8055981,79880247,39801351,77272628,48892201,23194618,65338021,68816413,76170907,33797252,50668599,57393458,62762857,9175338,94076128,90310261,94911072,49597667,40356877,90090964,82886381,67793644,7919588,99515901,56531125,83083131,19891772,93790285,70800879,11365791,40686254,9398733,36753250,68102437,26507214,84684495,76703609,44481640,99658235,53084256,81855445,75153252,34497327,40152546,45306070,17957593,16567550,3183975,92604458,71316369,20140249,54427233,66116458,33699435,15075176,88444207,57241521,33237508,66903004,23569917,96709982,62115552,18699206,89078848,14731700,75986488,63372756,17146629,41092172,72357096,73235980,30771409,70541760,24826575,19272365,8115266,26734892,7011964,53888755,74614639,80555751,55470718,71920426,79821897,3233084,72274002,52293995,80246713,33895336,27041967,87160386,88698958,95957797,66832478,4119747,25842078,3880712,32250045,37560164,29029316,23110625,43933006,57240218,8373289,66667729,78909561,76330843,72732205,77072625,74110882,77898274,9860968,78549759,56484900,73617245,34698428,44842615,5832946,70420215,14723410,68128438,55669657,99226875,85023028,97281847,33336148,38658347,79922758,2917920,27185644,17764950,78218845,1250437,94516935,88130087,37435892,47887470,4668450,72278539,45995325,62357986,97011160,53648154,19898053,6871053,10358899,64098930,21070110,38009615,10961421,91831487,52060076,70004753,40865610,38365584,24058273,50007421,56153345,3773993,66137019,65454636,93270084,89046466,9058407,63628376,82052050,93933709,4064751,62740044,17058722,43322743,66428795,68694897,64848072,66663942,82897371,13862149,19486173,44060493,52261574,3509435,38645117,88047921,55189057,37620363,67030811,41245325,30543215,30090481,48893685,17894977,20122224,92541302,83948335,6505939,92033260,74724075,54987042,17037369,27325428,13348726,6221471,35996293,99971982,48395186,2607799,70372191,84840549,46870723,63015256,14947650,31126490,59177718,67474219,47738185,51472275,45237957,16099750,62490109,34493392,75229462,62232211,47361209,37957788,53393358,9575954,35192533,33431960,21001913,96852321,78884452,15535065,84904436,97641116,89699445,38022834,53802686,57803235,34698463,58749,61728685,7104732,65880522,415901,21993752,10453030,78602717,49328608,99917599,11543098,90457870,81898046,68204242,24619760,81172706,91255408,81805959,71300104,48673079,73021291,29466406,33628349,2891150,32590267,51590803,20642888,43501211,18131876,54199160,26139110,95010552,41380093,92215320,68000591,51149804,75777973,5482538,62552524,78561158,20765474,46540998,52204879,85117092,20867149,44846932,32151165,91281584,41481685,72777973,45848907,72019362,79191827,27187213,86022504,60176618,30653863,94360702,39373729,61623915,98648327,30694952,42782652,53842979,92692978,21919959,72725103,26114953,60955663,23740167,50806615,1788101,76671482,67513640,168541,69136837,6497830,4199704,73124510,54058788,38341669,9257405,89804152,94090109,7646095,17386996,61271144,47792865,67451935,96735716,31161687,33249630,39201414,18001617,65271999,98653983,2208785,36845587,59501487,65715134,79136082,86301513,30163921,17976208,60106217,23432750,526217,82651278,74452589,93053405,29188588,91240048,32426752,83789391,75543508,71083565,19376156,29401781,72495719,37166608,47893286,4798568,14626618,8099994,4515343,77300457,25636669,30569392,77787724,72373496,42692881,60581278,6038457,54606384,70367851,91727510,58208470,5970607,16387575,99729357,42967683,51507425,8913721,26744917,40534591,98948034,61960400,7182642,26392416,55602660,49271185,69579137,89637706,80251430,44847298,45667668,42307449,52613508,21289531,26292919,99021067,11923835,26275734,99297164,97379790,20645197,59614746,14220886,10264691,16019925,96906410,75052463,9860195,33935899,75135448,48360198,85711894,96420429,72238278,39553046,16405341,57163802,90158785,92398073,45790169,78589145,72614359,321665,77187825,91990218,98943869,31904591,44915235,23134715,30891921,30764367,75820087,37192445,96318094,31491938,40984766,61928316,35512853,86798033,55770687,33061250,7229550,48774913,44177011,36139942,82339363,22405120,86543538,94711842,91141395,15902805,45919976,20885148,86242799,68824981,34432810,1936762,37891451,29065964,67031644,21673260,2204165,95251277,1197320,18466635,50567636,31727379,17857111,66045587,64055960,84002370,33123618,20568363,11581181,4985896,176257,49598724,93515664,79806380,71522968,49236559,61141874,99690194,16054533,57020481,44479073,30139692,6525948,32274392,59371804,69641513,69848388,54517921,22113133,37659250,43506672,7517032,61712234,76315420,65047700,79738755,95581843,38510840,44784505,36396314,54232247,37280276,93359396,16684829,84406788,21533347,65081429,36685023,86001008,26664538,45075471,40027975,92787493,34044787,92530431,66885828,44627776,9829782,78300864,45381876,62430984,30998561,824872,1375023,62051033,90654836,28787861,5073754,2331773,17081350,82532312,13468268,15064655,9188443,10649306,49882705,63152504,85695762,9599614,16424199,24168411,83533741,53632373,89996536,55850790,70782102,92353856,24953498,84349107,84127901,4508700,99524975,56461322,34946859,36468541,26102057,47213183,51315460,4978543,28734791,36808486,86118021,92071990,78785507,22766820,93057697,57658654,87598888,47298834,1239555,6157724,82327024,88251446,17385531,79942022,22200378,21397057,22435353,45794415,86460488,7687278,48658605,44473167,3088684,14363867,12303248,73109997,45407418,8129978,46851987,66271566,83133790,17365188,44664587,60393039,20836893,64602895,57359924,9259676,19457589,8729146,4434662,89217461,55960386,78766358,7066775,40781854,91938191,4069912,51426311,12664567,36135,59981773,3487592,26863229,36933038,11757872,41442762,55615885,61741594,71469330,5267545,7685448,79322415,54868730,80316608,6111563,76825057,69697787,32699744,82427263,45049308,8128637,29516592,56955985,90004325,83269727,17539625,59318837,29510992,74441448,94595668,83747892,27625735,81774825,77301523,20681921,48088883,71965942,18504197,4091162,37224844,68110330,749283,58224549,54263880,4204661,38256849,29635537,30811010,58180653,92867155,34667286,59405277 +39801351,17957593,35092039,43376279,64848072,30764367,79657802,92692283,8807940,96726697,38061439,70596786,97940276,99658235,85571389,14349098,75153252,74724075,91802888,73124510,13231279,40197395,5111370,64602895,13470059,2717150,112651,77312810,65074535,98371444,58749,93933709,16019925,36580610,96193415,77620120,49641577,51315460,26292919,26734892,66428795,95726235,35192533,15535065,10358899,89996536,69255765,55669657,25842078,89419466,47090124,83083131,13862149,65454636,72738685,69641513,95957797,23569917,89214616,6793819,47708850,19939935,75229462,44348328,17068582,62452034,44481640,91831487,64055960,53842979,90272749,29994197,83133790,46540998,9259676,19891772,9886593,4064751,78218845,17365188,74614639,84187166,38494874,60102965,59197747,66250369,36845587,32161669,99021067,54232247,48774913,62051033,96709982,62430984,73031054,168541,73392814,415901,64087743,68102437,11581181,55753905,29029316,61741594,61859143,51464002,77413012,75820087,14045383,14947650,29065964,20645197,89046466,79922758,22942635,569864,18699206,29401781,53802686,51715482,4668450,48360198,3233084,42644903,23194618,77272628,50668599,45075471,10309525,62740044,72278539,10961421,9175338,76825057,15148031,61859581,12416768,30090481,89811711,7517032,22766820,34432810,43506672,37891451,874791,73235980,97379790,9603598,47792865,4119747,29834512,99861373,6986898,749283,7104732,24591705,86821229,67793644,10453030,77301523,33565483,93270084,33628349,87598888,29466406,40686254,36505482,84002370,28851716,54762643,21993752,57163802,47361209,19101477,26998766,78549759,61897582,16445503,29959549,61271144,39171895,48675329,1204161,18001617,54663246,48892201,87710366,3773993,67084351,29510992,14326617,61373987,4798568,7011964,76170907,72358170,75052463,44844121,74357852,69579137,26275734,38658347,91957544,75543508,68644627,77377183,26392416,15536795,83651211,89637706,57359924,9257405,4434662,34698428,72274002,92692978,84684495,76540481,3487592,53666583,45428665,40984766,19898053,5832946,39201414,45667668,92033260,76703609,9623492,38008118,45790169,70541760,50188404,54987042,62693428,26776922,8696647,82532312,93790285,16387575,33304202,83150534,23848565,6871053,9398733,4099191,30787683,49597667,51588015,89078848,30694952,28550822,6819644,92215320,80251430,79880247,57803235,66885828,70367851,63967300,12348118,4985896,61263068,68000591,7955293,51426311,44842615,81853704,45848907,92541302,44784505,43152977,66137019,6525948,247198,62357986,13348726,70800879,37192445,33201905,29188588,15015906,94360702,78602717,35996293,63152504,53632373,69355476,36753250,17976208,98462867,54427233,44245960,27325428,27185644,8733068,8913721,19486173,43322743,65338021,5640302,8791066,20765474,30395570,9860968,57658654,14093520,34295794,63372756,23134715,83533741,72793444,65271999,78589145,7687278,40027975,12303248,29819940,68702632,1250437,95395112,74452589,60430369,99729357,70004753,14626618,92554120,91281584,15163258,72495719,33336148,54058788,6808825,36808486,67451935,29932657,40677414,44847298,78909561,59177718,38645117,24058273,37620363,52204879,3509435,42967683,58180653,68816413,37280276,80014588,89217461,50007421,16054533,36685023,91938191,22129328,26744917,8055981,26493119,82897371,18131876,83948335,79942022,88251446,99297164,27665211,56153345,80316608,8115266,73617245,2204165,96735716,36189527,85023028,94911072,1022677,82052050,10366309,77284619,42307449,71657078,80555751,99524975,3880712,7300047,51590803,14220886,81172706,86001008,79191827,72725103,21001913,45996863,9188443,82779622,55470718,99515901,62803858,508198,23432750,57241521,86022504,99965001,56424103,87160386,88653118,50702367,85242963,97561745,44889423,72614359,8129978,1431742,321665,7685448,17539625,16971929,17386996,71965942,34497327,83789391,36780454,58208470,48260151,32699744,30998561,2208785,13819358,22108665,3294781,5482538,79136082,21533347,17727650,30463802,42782652,94539824,97783876,92398073,92998591,46851987,88698958,82979980,62496012,41092102,33237508,66319530,76330843,52293995,20885148,8099994,81898046,34698463,55189057,45990383,37560164,52613508,90310261,26114953,16424199,26063929,36930650,22997281,32159704,8373289,11757872,73168565,40865610,99549373,70727211,44177011,45049308,94516935,37435892,41245325,60106217,91255408,54517921,66271566,42073124,68128438,71083565,6505939,95290172,47213183,90090964,21673260,24953498,11365791,37183543,45237957,71333116,81677380,16097038,52261574,9860195,87720882,66832478,90061527,51149804,39373729,65275241,74110882,81805959,41481685,84127901,51047803,54263880,56531125,47893286,30569392,75777973,12664567,56484900,24733232,89499542,11543098,61982238,2917920,98943869,37224844,98739783,70420215,42692881,36812683,44479073,53084256,78561158,96420429,37166608,16380211,44983451,93515664,9575954,31126490,51472275,8651647,99226875,31727379,76960303,2607799,45306070,55247455,74137926,70036438,82339363,61712234,53547802,71316369,55602660,46870723,42947632,40534591,68694897,5267545,49328608,26139110,75986488,92867155,86118021,73222868,19376156,35456853,93057697,64098930,11923835,34044787,82327024,66611759,77187825,73786944,99125126,1197320,75135448,20568363,4787945,63628376,22450468,43045786,57248122,59318837,7919588,40356877,9829782,43501211,1375023,65851721,45617087,65017137,78884452,15075176,30218878,21070110,60955663,84349107,20122224,36312813,37659250,526217,18504197,7502255,34493392,18466635,81274566,17146629,86301513,67281495,67474219,30771409,68824981,96852321,32590267,41092172,34667286,39847321,11161731,77300457,85711894,65715134,88047921,30811010,21397057,66663942,76671482,12571310,62490109,2891150,14363867,33431960,7423788,44627776,59109400,55143724,36135,51466049,69697787,98948034,38256849,16405341,62762857,57020481,38341669,20486294,30543215,16684829,74743862,55960386,98130363,74441448,3235882,4204661,52060076,63506281,54014062,31733363,62428472,61141874,14723410,60176618,44473167,40152546,22200378,18783702,24619760,16595436,20836893,72777973,73109997,10597197,63015256,3233569,27187213,22435353,18663507,88130087,20681921,86460488,59614746,47887470,38022834,99333375,70782102,33553959,23740167,39986008,94076128,4806458,60581278,68939068,66667729,28787861,98653983,86242799,84293052,77072625,20642888,68316156,66322959,61623915,53648154,22405120,84166196,22879907,82726008,90654836,32426752,83155442,93359396,59910624,1569515,97057187,66045587,45794415,6157724,57240218,18833224,51507425,32250045,86543538,88904910,7066775,55770687,92803766,44550764,65047700,64157906,83210802,48893685,45407418,36396314,33123618,30366150,40781449,85116755,79821897,68041839,3088684,69786756,83368048,47738185,24826575,80246713,13468268,98648327,72357096,89699445,99604946,67227442,6038457,54606384,8634541,66903004,44915235,32151165,65880522,38510840,33935899,93053405,61960400,16099750,7646095,4091162,12024238,93562257,19457589,31491938,10649306,82427263,95010552,42237907,72732205,17058722,84840549,40781854,83302115,14731700,4069912,6153406,1239555,26507214,62232211,60393039,67124282,824872,55850790,6221471,27411561,17764950,76315420,39553046,71469330,36933038,81855445,37739481,6497830,95581843,33797252,8729146,76434144,89804152,92353856,79322415,13173644,45381876,35512853,91240048,69848388,78785507,6521313,31161687,77787724,62115552,43933006,37501808,56473732,26863229,44060493,91727510,47298834,62936963,59501487,25636669,50806615,18411915,71920426,57961282,59981773,4095116,61928316,72019362,99224392,69136837,17857111,48088883,17016156,45995325,48673079,54868730,4508700,99690194,28734791,90158785,27041967,84406788,68110330,37957788,96906410,9058407,44664587,70372191,37675718,24314885,26664538,56515456,9599614,16567550,24915585,32058615,20867149,61815223,90457870,17894977,15948937,91664334,50567636,53888755,56461322,97011160,36139942,49236559,82178706,97641116,71300104,10264691,30653863,96318094,43357947,67030811,58751351,53393358,36468541,17037369,92071990,78766358,33061250,82886381,18806856,45919976,85117092,33895336,65081429,71522968,93566986,38009615,176257,65038678,8128637,28796059,48395186,91141395,15064655,4199704,30139692,79738755,48658605,86798033,27625735,49598724,4978543,72373496,41380093,31904591,24168411,5073754,79806380,75407347,23110625,83269727,84904436,34946859,82651278,33249630,20140249,67513640,94711842,94595668,15902805,26102057,90004325,88444207,7182642,29635537,21289531,59371804,75128745,91990218,69605283,5970607,669105,66116458,61728685,17385531,3633375,3183975,94090109,49271185,34236719,6111563,41442762,62552524,32274392,97281847,22721500,22113133,30891921,54199160,67031644,19272365,56955985,33699435,59329511,1788101,63059024,78300864,42199455,77694700,38365584,58224549,55615885,29516592,59405277,99971982,92604458,92787493,4515343,28928175,5822202,92530431,7229550,90013093,26397786,49882705,17081350,21919959,73021291,99917599,44846932,83747892,68204242,30163921,81774825,68875490,2331773,57393458,85695762,20002147,77898274,1936762,95251277,72238278 +73124510,67793644,45407418,17764950,36505482,13173644,9886593,21993752,18833224,95726235,51149804,76540481,96193415,92033260,40152546,44842615,19939935,16971929,89214616,38061439,32426752,54663246,56461322,55247455,20765474,26139110,37620363,1936762,61859581,42073124,44481640,49641577,77284619,415901,14093520,40781449,15064655,97379790,86001008,82532312,53632373,31733363,61263068,72357096,59501487,76960303,63059024,54987042,52613508,73235980,75543508,8651647,78909561,30764367,44784505,91664334,55960386,99524975,72793444,59177718,35092039,73031054,8807940,45381876,14947650,90004325,34044787,24619760,84187166,45995325,88653118,91990218,10649306,14626618,62452034,81855445,89078848,15015906,55753905,99690194,73786944,99226875,90013093,16097038,93933709,4099191,84002370,35192533,29466406,32699744,71316369,1197320,36780454,44889423,55470718,47893286,24591705,18806856,7229550,85023028,56153345,17386996,18663507,2891150,66045587,66137019,99297164,92692978,63372756,9599614,88444207,31126490,95395112,29065964,26392416,87598888,71522968,18504197,22942635,57241521,68875490,26292919,41380093,79806380,44479073,51047803,64157906,20122224,93566986,62496012,53802686,508198,14349098,18466635,36808486,82979980,2917920,40027975,83133790,66903004,21070110,92215320,44664587,54232247,27665211,99917599,95290172,25842078,88130087,59197747,3233569,17976208,17068582,62430984,9175338,71920426,7011964,89217461,37675718,17539625,17365188,92867155,2204165,33123618,8733068,49328608,45667668,32159704,54427233,80014588,30569392,70036438,77620120,26863229,86460488,68816413,81274566,53547802,16445503,94360702,9623492,47708850,36135,6986898,66428795,10366309,7955293,66271566,36580610,87710366,26114953,37183543,78785507,62803858,22450468,66322959,55602660,45075471,2331773,39553046,42307449,56484900,24058273,92803766,39847321,27325428,50806615,45794415,16405341,80316608,76825057,57803235,74614639,30998561,99604946,58180653,19486173,28928175,44983451,43376279,61728685,68939068,51464002,82052050,18411915,22997281,40677414,98739783,12348118,69697787,6153406,66832478,37891451,1431742,82897371,64602895,97940276,72278539,48088883,36753250,99515901,82339363,92530431,321665,15535065,44177011,29959549,32058615,4095116,5640302,13348726,94539824,60430369,83533741,59318837,99965001,83269727,70420215,45919976,569864,64848072,54517921,65074535,65851721,46851987,91831487,78561158,75777973,29834512,85116755,90090964,30771409,51426311,91802888,31904591,60106217,17385531,33431960,16019925,40984766,65271999,18131876,78589145,16099750,87160386,74110882,40356877,53666583,71300104,32161669,41442762,69641513,34236719,31491938,72738685,38009615,71469330,56531125,49882705,37501808,41245325,1250437,70596786,26397786,4064751,26998766,11161731,42692881,9829782,36933038,8729146,43152977,75153252,54606384,36930650,22405120,62936963,54058788,77301523,4199704,10358899,85242963,72274002,83789391,77312810,58749,9259676,78218845,30891921,29994197,95957797,23134715,74137926,32250045,6808825,60102965,35456853,63015256,42644903,8373289,24314885,34295794,45790169,51507425,72373496,24826575,68000591,96906410,47792865,33699435,98948034,98130363,61928316,21673260,72732205,17957593,669105,112651,77072625,43322743,7687278,66663942,4798568,27625735,33201905,61141874,65275241,99971982,54868730,92787493,27187213,53648154,88251446,30543215,79922758,81898046,20867149,44846932,3633375,9575954,94076128,71657078,11365791,49236559,15536795,51472275,26063929,62552524,80246713,31161687,33336148,90310261,38645117,14731700,92554120,4806458,2208785,76330843,97281847,48260151,62740044,80555751,17857111,54762643,42782652,61982238,98462867,90654836,57240218,39171895,7517032,40781854,86118021,59371804,4119747,15902805,6505939,83210802,8634541,63967300,68644627,68316156,54263880,10309525,61623915,96726697,96318094,61960400,78602717,98648327,40534591,21001913,7104732,6793819,94711842,37739481,72614359,2607799,73392814,69255765,17058722,37659250,83150534,61859143,39986008,22108665,13468268,67513640,40865610,76703609,36396314,60955663,36812683,72019362,84293052,68694897,38658347,9058407,94090109,85117092,50188404,33061250,41092102,92353856,22129328,66667729,69136837,81853704,10961421,23194618,92604458,26734892,9257405,67474219,60176618,12571310,62693428,19891772,93515664,96735716,33237508,18699206,30090481,86242799,81805959,16380211,61271144,79136082,67451935,90457870,56424103,99333375,68204242,85571389,89996536,24953498,33797252,33895336,53842979,68128438,72725103,69355476,61373987,3088684,66611759,47361209,77413012,47887470,9188443,45990383,93053405,1022677,824872,99224392,8115266,70372191,30139692,99861373,14220886,99549373,34698428,10597197,99658235,38008118,55189057,64098930,70727211,44847298,73617245,36312813,50007421,39801351,93270084,526217,97561745,16054533,34497327,8696647,17894977,89046466,3773993,72777973,48360198,37166608,33628349,83948335,31727379,52261574,30787683,66116458,76434144,12416768,74724075,83368048,4204661,75128745,34493392,53888755,78766358,5482538,81677380,35996293,92998591,15163258,36189527,98371444,90272749,67030811,67281495,5111370,4668450,29188588,20140249,168541,44844121,43501211,10264691,45617087,87720882,73222868,5832946,54014062,4787945,26776922,28550822,72495719,50567636,20645197,50702367,49598724,3294781,65880522,66885828,65715134,76315420,58208470,9398733,79821897,51590803,61815223,247198,6038457,48892201,72358170,45848907,57658654,50668599,28787861,8129978,22200378,78549759,3183975,7423788,64055960,65017137,99125126,99729357,91281584,47090124,11543098,52204879,57248122,30218878,44627776,7300047,45237957,48675329,44550764,55669657,66250369,79738755,7182642,7685448,23848565,82327024,97011160,14326617,57961282,82178706,8099994,61741594,36468541,94911072,26275734,12664567,75135448,37435892,3509435,45428665,12024238,18783702,76671482,82427263,30395570,62232211,34946859,29635537,29510992,23110625,36685023,51466049,82651278,85695762,30463802,90158785,44060493,89811711,97783876,15148031,5970607,48893685,27411561,93359396,68702632,38022834,97057187,51715482,68102437,37192445,47298834,30366150,40686254,43045786,75986488,6871053,67227442,70004753,77272628,88047921,3233084,70800879,77187825,94595668,16595436,74452589,30653863,26102057,19457589,52293995,5267545,91141395,71333116,20568363,22879907,27185644,78884452,68824981,3487592,1788101,9603598,4069912,20486294,91957544,92398073,77694700,58224549,61712234,89699445,95581843,54199160,73168565,56473732,10453030,7919588,65338021,77787724,55850790,63628376,7066775,48395186,49271185,89637706,22766820,71965942,81172706,33553959,42967683,55143724,1204161,46870723,75407347,89804152,83651211,62115552,38341669,99021067,89419466,42947632,38494874,56955985,62051033,92071990,6525948,16567550,6221471,14045383,19272365,81774825,82726008,13819358,52060076,28734791,33935899,34667286,57393458,72238278,32151165,47213183,24733232,11757872,24915585,86798033,84349107,34432810,93562257,73109997,83155442,21919959,3880712,92692283,16387575,67124282,41092172,55770687,4508700,30811010,84406788,82886381,11581181,58751351,14363867,15948937,51315460,68041839,69786756,5822202,62428472,29932657,65454636,53084256,51588015,57359924,92541302,56515456,19101477,30694952,65038678,17081350,17037369,35512853,22721500,86301513,70367851,64087743,60581278,26507214,83302115,85711894,7502255,65081429,2717150,59405277,95251277,27041967,6497830,37957788,44348328,3235882,90061527,20642888,75229462,39373729,44473167,38365584,74357852,96709982,16424199,29029316,6111563,9860195,20836893,5073754,89499542,91727510,84684495,68110330,8913721,1569515,7646095,57020481,83747892,4091162,62490109,57163802,33304202,6521313,69605283,26493119,75052463,12303248,23740167,61897582,84904436,4985896,96852321,29819940,26744917,66319530,69579137,59109400,17016156,96420429,77300457,19898053,63506281,19376156,82779622,88904910,79191827,22113133,91938191,9860968,29516592,86543538,84127901,28851716,42237907,48774913,34698463,59981773,13231279,98943869,47738185,98653983,37280276,22435353,26664538,84840549,77377183,59910624,13470059,21397057,45996863,46540998,24168411,25636669,15075176,20002147,70541760,48673079,75820087,43933006,23569917,79657802,74743862,65047700,69848388,71083565,16684829,79942022,62762857,176257,93790285,13862149,33249630,28796059,62357986,59329511,21289531,95010552,45049308,42199455,86022504,32590267,53393358,21533347,11923835,4978543,8791066,91255408,44915235,32274392,55615885,39201414,49597667,37560164,17146629,20885148,74441448,77898274,6819644,874791,91240048,4434662,73021291,44245960,67031644,36845587,6157724,1375023,94516935,76170907,14723410,40197395,63152504,70782102,60393039,45306070,749283,38256849,8128637,59614746,4515343,23432750,93057697,17727650,79880247,97641116,86821229,37224844,1239555,29401781,30163921,8055981,84166196,79322415,48658605,88698958,41481685,83083131,18001617,20681921,36139942,67084351,43357947,43506672,80251430,33565483,78300864,38510840 +50188404,26392416,63628376,92692978,12024238,55189057,81677380,81898046,3233084,93053405,71333116,76170907,91664334,69579137,42692881,77620120,4434662,23194618,34295794,7011964,31126490,35092039,12348118,13231279,7300047,72274002,22879907,79880247,65454636,73235980,33201905,62496012,60581278,37560164,20642888,33304202,30395570,53666583,70596786,98943869,66611759,66116458,12664567,44245960,61982238,37501808,62803858,85571389,64098930,95290172,33249630,83150534,34667286,55247455,526217,67793644,93057697,31161687,18699206,95010552,99658235,29834512,47792865,79136082,55143724,22766820,79821897,39847321,27041967,86301513,99515901,66137019,247198,61373987,30764367,86543538,6525948,508198,41092102,74357852,70727211,17727650,749283,5832946,4064751,62232211,49597667,64848072,112651,57241521,73031054,16019925,4204661,78300864,38008118,48892201,78884452,74614639,68041839,96193415,39201414,2717150,45996863,51464002,43501211,81172706,26493119,39986008,50668599,72725103,67124282,89214616,40152546,98653983,25842078,59614746,88047921,65047700,17976208,6153406,67227442,66667729,98371444,22108665,6808825,4668450,68816413,83269727,669105,1022677,59371804,76315420,27325428,22942635,64087743,47361209,14626618,98462867,85116755,22200378,60955663,47090124,45794415,9623492,37435892,97057187,59197747,72373496,93933709,53802686,62452034,97561745,88444207,38022834,50567636,18833224,62552524,88698958,14947650,32590267,55470718,44627776,90013093,84406788,5111370,20836893,40356877,30998561,7919588,70036438,27665211,6505939,63967300,61960400,78909561,86798033,26102057,43933006,47738185,45237957,3880712,62051033,70420215,24733232,66885828,91831487,45306070,8099994,86022504,8696647,18411915,42967683,99333375,48774913,86001008,51590803,44983451,43322743,1197320,46540998,20140249,22113133,92398073,90457870,98130363,95395112,52613508,54517921,92554120,16595436,8129978,61271144,43376279,415901,15902805,97783876,31733363,20568363,84840549,62428472,26744917,54868730,95726235,99224392,65081429,23134715,55753905,59981773,38645117,10961421,18504197,30163921,70367851,73222868,67031644,73168565,20002147,17764950,13862149,9860195,31491938,44842615,62936963,40984766,88653118,62430984,35996293,51047803,30463802,65271999,4069912,17385531,3487592,7229550,66250369,10358899,53632373,95957797,29516592,80014588,76540481,37620363,6497830,30139692,68204242,2891150,40027975,57240218,36189527,37224844,68939068,2917920,77284619,5970607,61741594,66903004,34236719,74137926,54199160,57961282,42199455,36780454,99524975,29466406,60102965,33237508,30891921,77312810,24915585,98648327,22435353,92071990,56531125,87598888,22129328,16445503,7066775,37739481,48088883,49641577,20681921,81853704,55669657,8651647,99125126,79657802,19272365,15148031,70004753,45919976,7685448,82779622,89078848,27411561,36505482,29029316,77072625,37891451,70800879,64602895,30218878,77187825,61623915,91957544,54263880,15535065,54232247,90004325,29188588,61859581,75153252,42073124,96709982,83210802,14349098,83083131,26664538,89637706,45428665,8807940,3509435,72238278,17068582,13470059,20122224,36753250,30694952,21993752,59910624,99917599,42782652,82886381,17081350,29510992,2331773,89217461,84127901,33797252,49328608,58180653,84684495,33061250,75543508,61728685,40197395,68102437,92998591,85117092,69255765,26275734,9829782,56515456,59109400,89804152,78589145,15536795,29401781,31904591,36808486,91802888,82339363,14093520,19486173,77413012,39801351,96420429,4508700,19891772,49598724,39171895,60430369,76330843,57020481,69355476,67030811,17957593,91938191,80251430,51588015,54014062,99604946,19939935,77301523,36845587,57163802,72358170,6111563,6793819,78602717,59318837,22721500,97011160,53393358,6038457,82427263,30787683,40686254,48260151,44473167,94516935,18131876,10366309,9257405,46851987,90272749,11543098,72614359,16380211,3633375,8055981,48395186,32159704,84187166,16684829,1204161,53648154,58749,93359396,3233569,48675329,36933038,84166196,98948034,79738755,83789391,99297164,9886593,77787724,94090109,8729146,57393458,75135448,94595668,65074535,54058788,89699445,54663246,13173644,57248122,10453030,75820087,19101477,4099191,61263068,67451935,59177718,99965001,51715482,14363867,17146629,14045383,17016156,73786944,38365584,22997281,43152977,44784505,17365188,4095116,10309525,15075176,6521313,9259676,36139942,74441448,96735716,74743862,57803235,41092172,47213183,26507214,10264691,10649306,78766358,66322959,44348328,24591705,9603598,65715134,24826575,7955293,38341669,51507425,18001617,16099750,55960386,569864,18806856,55602660,8115266,48360198,29994197,82052050,78549759,90654836,16567550,43357947,80555751,14326617,67474219,91727510,20486294,4985896,81774825,14220886,87720882,37183543,6986898,68644627,44060493,44844121,26998766,36312813,36812683,77300457,77694700,66832478,28550822,68702632,9188443,21673260,65851721,92803766,59501487,44889423,32161669,62490109,71316369,96852321,92541302,4806458,75986488,17894977,88251446,3183975,83533741,57658654,29819940,16097038,44177011,89046466,7423788,76825057,28734791,73124510,30771409,91141395,99971982,89996536,5073754,82178706,9058407,76671482,45049308,18783702,69641513,39553046,7502255,69136837,92604458,72019362,1431742,92692283,26863229,7646095,74724075,93562257,99549373,49236559,43045786,38061439,33895336,47708850,21001913,45617087,58751351,64055960,57359924,72357096,33628349,33935899,48658605,24619760,36685023,85242963,17386996,21533347,47893286,29065964,79922758,17857111,99226875,49271185,99729357,4119747,62693428,68824981,45407418,66045587,30543215,8913721,33431960,23740167,69697787,89419466,85711894,94076128,84904436,53842979,73021291,97641116,80316608,71657078,43506672,34698463,26292919,8634541,45995325,49882705,53084256,90061527,48893685,66271566,75052463,36580610,44847298,65275241,62740044,34698428,40534591,9575954,92353856,89499542,30653863,69786756,34497327,55850790,38658347,72732205,36930650,71083565,95251277,91990218,16971929,16424199,67513640,68128438,4978543,14723410,5822202,22450468,44915235,62357986,42307449,83302115,67281495,93270084,75777973,35192533,75229462,15163258,23848565,33565483,35512853,50806615,1250437,42237907,23432750,36135,84349107,40781854,56153345,4515343,44481640,83155442,33123618,41481685,26397786,71965942,72278539,29635537,13819358,20765474,15064655,86242799,83651211,63506281,73392814,45667668,17037369,61859143,24168411,71920426,50702367,79806380,34493392,52204879,75128745,21397057,40781449,76703609,13468268,23569917,44479073,86460488,6871053,11923835,99861373,66319530,18663507,90158785,91240048,26139110,69848388,32699744,1936762,28928175,46870723,6221471,55770687,98739783,82532312,5482538,63059024,45381876,38256849,44550764,70541760,29959549,45848907,92033260,29932657,8791066,30569392,81855445,5640302,12303248,88904910,32274392,3088684,40677414,72793444,30811010,28851716,93566986,56424103,3294781,78561158,824872,84293052,59329511,54606384,19898053,9398733,67084351,80246713,45990383,82651278,3773993,92215320,2204165,86821229,61815223,95581843,51472275,61928316,40865610,17058722,75407347,87710366,68694897,51426311,21070110,97940276,7687278,4787945,97379790,1788101,16054533,65038678,58208470,51149804,168541,52261574,68000591,77272628,73617245,22405120,20867149,26063929,37659250,50007421,26734892,56955985,83747892,47887470,3235882,72495719,53888755,89811711,81805959,55615885,45075471,86118021,874791,44664587,70372191,12571310,52293995,79322415,1569515,91255408,9175338,24058273,26114953,63015256,74110882,20885148,4798568,61712234,87160386,71469330,83948335,84002370,9860968,94539824,93790285,78218845,63152504,44846932,41442762,2208785,48673079,176257,76434144,90090964,68110330,8733068,65017137,82897371,64157906,10597197,60106217,11581181,96726697,35456853,28796059,58224549,82979980,76960303,54427233,17539625,53547802,11757872,38009615,65880522,13348726,37192445,96318094,91281584,51315460,99021067,78785507,42947632,7182642,97281847,14731700,12416768,7104732,62115552,38494874,1239555,321665,30090481,69605283,63372756,39373729,19457589,19376156,92867155,27625735,52060076,51466049,60176618,5267545,30366150,41380093,61141874,33699435,15948937,41245325,32250045,45790169,11365791,70782102,36396314,54987042,33553959,99690194,37166608,56484900,82726008,25636669,54762643,60393039,18466635,27185644,85695762,81274566,73109997,6157724,31727379,61897582,72777973,79942022,32426752,56461322,77377183,71522968,66663942,85023028,15015906,20645197,34946859,26776922,8373289,1375023,16387575,82327024,56473732,72738685,34044787,21919959,4091162,47298834,79191827,83368048,62762857,7517032,37675718,66428795,16405341,11161731,74452589,6819644,4199704,59405277,2607799,28787861,71300104,92530431,42644903,8128637,65338021,94711842,33336148,96906410,21289531,68875490,38510840,36468541,77898274,37280276,90310261,32151165,24953498,23110625,94360702,94911072,32058615,68316156,88130087,83133790,92787493,93515664,37957788,34432810,24314885,27187213,9599614 +55143724,39801351,40677414,63967300,81898046,42199455,96726697,51047803,87160386,22879907,46851987,56424103,56473732,53648154,43933006,29994197,7423788,4099191,29819940,46540998,39847321,2717150,16097038,80316608,5822202,47090124,16054533,35996293,80555751,64087743,7502255,84293052,36580610,33553959,8099994,40984766,43501211,19891772,59910624,9175338,97641116,71083565,7300047,68204242,65081429,21533347,77413012,83155442,66663942,29834512,14349098,89804152,53842979,91727510,81172706,87720882,37659250,34236719,27411561,38658347,45237957,66428795,79136082,94360702,9860195,63506281,11365791,61960400,77898274,26507214,62740044,75820087,14326617,23110625,57803235,83368048,31161687,94090109,99965001,15064655,7182642,60581278,88047921,74614639,89046466,99224392,13468268,13862149,22108665,50188404,30139692,65454636,9860968,86821229,6157724,16971929,99515901,21070110,18466635,85116755,14947650,3235882,69355476,50007421,95957797,44550764,83210802,3633375,72357096,72495719,97783876,99524975,62452034,39986008,23194618,47887470,71657078,1431742,77284619,17539625,44844121,70727211,9603598,31491938,30163921,22129328,23848565,45990383,98462867,12348118,83789391,176257,61859143,57241521,66045587,59177718,3773993,77272628,18806856,62936963,33249630,44060493,82339363,70372191,67793644,60430369,75777973,5111370,89214616,91831487,79657802,96193415,44842615,61741594,65017137,37192445,15535065,32274392,3088684,526217,58224549,92398073,86301513,6986898,86460488,29959549,73392814,26998766,30787683,58749,48673079,30569392,54987042,30218878,76434144,93562257,6793819,43322743,75153252,52293995,62490109,24058273,37183543,94076128,33237508,91664334,7646095,96852321,20642888,41481685,50806615,65338021,20765474,29466406,90158785,62051033,20836893,40534591,68816413,85571389,73168565,55753905,32161669,92803766,24733232,17016156,17081350,99658235,36505482,89078848,19376156,91802888,36845587,67474219,72019362,38494874,19486173,12024238,49641577,31126490,90457870,1788101,30463802,53666583,59614746,82979980,25636669,57359924,74137926,508198,50702367,69136837,42782652,52204879,95581843,36780454,21673260,44983451,70036438,71316369,61623915,69605283,20486294,6808825,70596786,37739481,9623492,32159704,20002147,8651647,62232211,874791,40152546,94711842,21001913,88904910,35192533,77377183,16405341,23569917,6153406,55247455,89699445,73235980,3880712,2208785,98371444,5482538,68041839,28796059,168541,4095116,247198,19101477,53084256,3233569,83533741,39201414,27665211,62552524,76671482,37675718,74743862,78589145,63628376,88653118,34295794,22997281,1022677,73786944,57240218,43376279,33304202,8115266,80251430,45919976,76540481,5267545,91957544,67451935,93566986,17068582,4064751,37620363,3233084,20867149,51588015,9058407,47361209,17764950,30998561,4668450,49271185,35456853,86022504,74110882,91240048,94516935,26664538,89419466,36312813,33565483,78218845,18001617,15148031,15163258,45306070,68875490,57393458,99549373,4119747,51590803,93933709,26776922,2917920,61815223,63372756,69786756,68644627,8791066,13173644,98948034,40356877,17957593,34497327,36930650,42644903,67030811,76315420,36753250,82651278,669105,2891150,36139942,16387575,1239555,26275734,91141395,90004325,71333116,64848072,36396314,97561745,47792865,12571310,66322959,24591705,45407418,57248122,10366309,84904436,6521313,77787724,86242799,75407347,62115552,85242963,49328608,97379790,98130363,81853704,62496012,14220886,53632373,32590267,75052463,16380211,18699206,18663507,75543508,84684495,44627776,72274002,99729357,52060076,26102057,22721500,6871053,26493119,44481640,98739783,4434662,64055960,18833224,43152977,82427263,36808486,34698428,26392416,71300104,44177011,72278539,37957788,7104732,72777973,112651,68000591,6525948,4806458,10358899,92604458,82052050,57961282,49236559,45794415,58751351,46870723,79806380,72725103,7919588,24915585,1569515,93790285,78785507,94539824,8807940,77620120,33935899,18783702,61712234,99861373,53802686,90272749,5073754,70004753,61897582,84840549,76170907,82726008,89637706,2607799,66271566,28734791,97940276,14731700,79922758,72732205,22435353,99604946,92998591,30653863,81274566,35092039,55770687,82897371,29029316,55615885,48658605,66667729,97011160,4199704,45848907,78561158,10309525,19272365,70541760,79942022,13231279,63152504,75229462,4204661,39171895,6111563,94595668,75135448,44889423,9188443,33201905,45996863,97057187,27325428,53393358,5970607,65851721,78602717,86798033,569864,66319530,49597667,36685023,8696647,92541302,42307449,96735716,77694700,47738185,44245960,92787493,63059024,55470718,64602895,2331773,81805959,50668599,43357947,65275241,98648327,13819358,92554120,74724075,11757872,10961421,48675329,52613508,13470059,22200378,19457589,7685448,28851716,45617087,37166608,31904591,55669657,51715482,77187825,84406788,76330843,415901,22942635,12416768,14093520,99125126,55189057,56515456,42237907,62357986,48395186,74357852,82886381,76960303,44664587,73031054,83083131,95290172,47893286,11543098,7687278,8055981,8128637,68102437,40197395,15075176,38341669,48260151,59329511,17037369,21919959,66611759,22450468,66137019,2204165,60106217,30366150,30395570,99297164,38365584,51472275,54058788,20885148,81677380,44846932,85117092,76825057,32151165,96709982,88698958,77072625,99917599,33336148,61373987,22766820,9398733,47298834,18411915,9886593,11581181,93053405,1375023,63015256,21289531,28928175,78909561,59371804,70420215,5640302,68702632,80014588,30694952,82532312,60102965,34493392,19939935,89996536,72358170,92215320,67281495,14723410,22113133,48893685,99333375,26397786,36812683,18131876,24826575,51426311,68694897,56531125,16445503,54762643,52261574,26114953,38061439,32250045,38645117,91281584,30891921,92033260,28550822,26734892,20140249,59501487,4069912,3509435,24314885,4091162,68128438,90013093,73021291,3487592,321665,49598724,41380093,69848388,30543215,56153345,26292919,79821897,27625735,90090964,82178706,34432810,16019925,82779622,37435892,17894977,37224844,17386996,3294781,23134715,73124510,78884452,59318837,1197320,98653983,54663246,51466049,18504197,66116458,61859581,37280276,26063929,67084351,65074535,36189527,29188588,7229550,32699744,33431960,42073124,79191827,35512853,6497830,59981773,61263068,60955663,40781854,38256849,16099750,67227442,7955293,33797252,23432750,69641513,8373289,36135,71469330,55850790,10453030,83302115,4515343,10649306,65715134,65880522,54014062,6819644,55960386,83150534,75986488,26863229,51464002,56461322,80246713,44479073,17058722,24619760,15948937,3183975,6221471,59405277,11161731,84002370,44847298,95010552,98943869,20568363,34044787,1204161,59109400,93057697,33628349,57163802,40781449,42692881,64098930,45428665,70782102,95251277,50567636,73617245,62693428,78549759,44348328,92692978,82327024,45667668,62762857,65271999,54199160,95395112,9257405,4787945,17727650,8733068,93359396,31727379,21993752,72738685,86001008,88251446,12303248,99971982,87710366,62803858,85711894,20122224,66250369,61271144,66885828,9599614,16567550,69255765,9575954,99690194,81855445,69697787,42947632,79880247,95726235,38022834,90654836,27041967,68939068,84349107,33699435,45049308,83651211,48774913,7517032,93270084,71920426,33061250,6038457,48892201,43045786,4985896,93515664,14363867,29510992,14045383,23740167,90310261,44915235,91990218,45075471,74441448,10264691,78300864,81774825,41092102,11923835,38510840,68316156,70800879,37501808,48088883,54517921,16595436,85023028,51507425,83133790,15536795,37891451,17857111,44784505,47213183,29065964,67513640,40027975,86118021,65047700,30771409,92530431,39553046,61982238,83747892,62428472,76703609,62430984,72614359,53888755,13348726,99226875,73109997,34698463,749283,96906410,53547802,77312810,40686254,28787861,66903004,16424199,96318094,4508700,88444207,6505939,30811010,84166196,4978543,77300457,61728685,84187166,42967683,65038678,9259676,39373729,87598888,37560164,67031644,8634541,47708850,33123618,30090481,55602660,29932657,43506672,54232247,90061527,72793444,61928316,64157906,15015906,27185644,20681921,69579137,7011964,41442762,73222868,29401781,89811711,8913721,99021067,61141874,83948335,41092172,40865610,45995325,45381876,51149804,14626618,56955985,97281847,31733363,24953498,75128745,32058615,67124282,45790169,36468541,8729146,4798568,85695762,58180653,77301523,91255408,74452589,66832478,26139110,34946859,25842078,71522968,92692283,17146629,38008118,44473167,72373496,79322415,57020481,51315460,96420429,16684829,92867155,36933038,15902805,5832946,89499542,24168411,7066775,8129978,88130087,54606384,29635537,92071990,56484900,30764367,60176618,10597197,48360198,54868730,58208470,71965942,22405120,27187213,94911072,32426752,41245325,92353856,54427233,49882705,1250437,91938191,60393039,29516592,68824981,84127901,38009615,79738755,72238278,17385531,26744917,59197747,17365188,33895336,824872,19898053,34667286,86543538,54263880,83269727,1936762,78766358,9829782,70367851,17976208,89217461,21397057,12664567,57658654,20645197,68110330 +15535065,1022677,17894977,64087743,59614746,92554120,70004753,68128438,77300457,10309525,37659250,65715134,77284619,34295794,51715482,71657078,5111370,8696647,26493119,37183543,79136082,55770687,71333116,38494874,89214616,73168565,48893685,78218845,9623492,73031054,36505482,81853704,83789391,68644627,88653118,99658235,29834512,54606384,19898053,96726697,80246713,36780454,99917599,64098930,30653863,16380211,30787683,93790285,34698428,86242799,86022504,4434662,33628349,30139692,20486294,68875490,38658347,53666583,88047921,75229462,52293995,40865610,31904591,66832478,24058273,48658605,4091162,67793644,55143724,39801351,68824981,61960400,47887470,7300047,43045786,16684829,22879907,73786944,56424103,1250437,22405120,87160386,48892201,83368048,37739481,4064751,7687278,20867149,61623915,6986898,62740044,16595436,36396314,82886381,81677380,19891772,7229550,62762857,43322743,7646095,39553046,4099191,19272365,26114953,33304202,38009615,55470718,99125126,46851987,63015256,91938191,35092039,35192533,60430369,98653983,27665211,63628376,90013093,15075176,18131876,43152977,72373496,66667729,71469330,3633375,79738755,44889423,77694700,42237907,2208785,89419466,39373729,90457870,27625735,33797252,7919588,49236559,92692978,61373987,39986008,15148031,36930650,97641116,40677414,508198,53888755,30998561,96193415,52261574,42782652,59197747,44842615,24915585,54199160,30366150,66428795,13470059,99549373,66611759,65275241,82726008,91240048,21070110,70372191,43376279,63967300,50806615,59501487,4668450,75986488,2204165,99297164,47708850,97281847,27187213,76825057,6808825,1431742,168541,51426311,36812683,52204879,33249630,45428665,14626618,93057697,38008118,81855445,76703609,77620120,48673079,31733363,85116755,37620363,95010552,23432750,77787724,44245960,20885148,34698463,9259676,72278539,76434144,9886593,1239555,5482538,82339363,33237508,8634541,41380093,30218878,10366309,112651,29188588,49271185,68316156,43501211,34497327,8807940,21533347,874791,49328608,70596786,69848388,78589145,14326617,78549759,45075471,44983451,5640302,8128637,44060493,93053405,70420215,24826575,29819940,29959549,92215320,56515456,29994197,79657802,97940276,54014062,66271566,99224392,18663507,4119747,60102965,49598724,33123618,1375023,9860195,88130087,23110625,95726235,72495719,86001008,48360198,79922758,1204161,86821229,86543538,2607799,54263880,68000591,69786756,44847298,66319530,18783702,44177011,67031644,72777973,57240218,77413012,70036438,18806856,78766358,54427233,75543508,40027975,62430984,25636669,54868730,67451935,20836893,3487592,20122224,42307449,22450468,83150534,34493392,26776922,93933709,30395570,9829782,6038457,6505939,74614639,94516935,19101477,7517032,28928175,59329511,63152504,40781449,81805959,47298834,32159704,50702367,32161669,45990383,74110882,29466406,16971929,84002370,16054533,7066775,99861373,7955293,32590267,72614359,53842979,77187825,40152546,74137926,55189057,7104732,65454636,89699445,5073754,55247455,62936963,15064655,22200378,22997281,20765474,10358899,58208470,30163921,12348118,92604458,13231279,13819358,98130363,20002147,68041839,14093520,44550764,88904910,92353856,61897582,61859581,45995325,67281495,30090481,14723410,6221471,49641577,14731700,61741594,23740167,3088684,57803235,99965001,61859143,42967683,47090124,4204661,40534591,87720882,8651647,36135,62452034,26998766,83155442,72358170,73392814,91664334,76170907,569864,73235980,83210802,84127901,91802888,85023028,11757872,77272628,39171895,55615885,17016156,78884452,3235882,15163258,57163802,43933006,6521313,40356877,51047803,14947650,46870723,79191827,60106217,89637706,65851721,64602895,23569917,26734892,98462867,97783876,44481640,44664587,17068582,176257,57658654,5822202,50188404,19939935,62496012,29029316,526217,27325428,16445503,71522968,99690194,8913721,3294781,32274392,38256849,22129328,55753905,749283,75135448,52613508,2717150,67474219,69641513,61982238,80014588,6157724,75777973,60176618,26507214,70727211,26397786,17857111,65880522,9398733,16567550,64157906,17764950,20140249,6871053,66885828,4806458,59177718,85571389,247198,19457589,37280276,13862149,63372756,4515343,98948034,50668599,45381876,31161687,60955663,28796059,78300864,59910624,66903004,94539824,75153252,45306070,1197320,74724075,65038678,93566986,94090109,97379790,47738185,57359924,38061439,44846932,76960303,38341669,76671482,95957797,28550822,44627776,1788101,53802686,34236719,321665,33553959,62490109,32699744,26863229,68939068,65017137,99021067,62115552,66045587,34432810,70800879,86301513,45794415,79942022,45996863,26139110,43357947,42947632,81274566,53632373,32426752,42644903,61263068,18833224,78602717,89811711,34667286,16099750,20645197,98371444,73124510,6525948,12416768,30891921,92398073,27185644,2917920,7502255,72738685,13468268,67124282,92692283,81898046,90061527,7182642,90272749,47213183,89499542,95251277,69355476,96709982,5267545,61728685,15536795,28851716,5832946,80316608,82178706,30463802,37957788,6153406,68694897,41481685,79806380,23848565,66663942,77377183,61928316,82427263,55669657,10453030,18699206,12571310,26292919,21993752,31727379,16424199,24591705,96735716,56473732,64848072,88444207,30764367,29932657,62357986,63506281,76330843,33565483,97011160,17385531,9575954,3880712,59981773,93515664,45919976,94911072,57020481,48088883,17386996,2891150,80555751,99729357,37501808,51588015,14220886,55602660,94711842,18504197,51507425,17037369,9603598,54987042,10597197,31126490,61141874,51464002,36580610,415901,60581278,15015906,97057187,91831487,4985896,62803858,85711894,92998591,2331773,58751351,67227442,16097038,71083565,41092102,21001913,34044787,79821897,67030811,11923835,74357852,89217461,82052050,669105,68816413,46540998,24733232,18001617,56484900,72238278,73222868,19376156,43506672,62051033,98648327,3233569,58224549,22435353,23194618,17058722,7011964,70782102,44473167,33336148,83533741,75128745,16405341,8129978,42692881,11365791,32151165,69579137,29510992,77312810,82779622,8115266,99524975,85242963,22113133,33699435,82651278,47792865,68204242,33935899,84904436,61712234,40781854,75052463,98943869,84187166,36189527,16019925,91141395,10961421,86798033,17727650,44915235,90654836,17146629,62428472,45848907,28787861,92071990,56461322,54517921,53648154,71316369,92787493,55850790,69255765,17539625,39847321,51149804,44844121,48675329,22108665,72274002,13348726,16387575,68110330,68702632,72357096,30771409,34946859,92541302,66250369,41442762,26102057,49882705,23134715,82532312,10649306,51472275,6111563,30569392,88251446,57248122,50567636,8055981,15902805,77072625,85117092,65074535,21673260,53084256,68102437,21289531,3773993,97561745,8373289,39201414,36139942,88698958,92033260,90310261,9860968,93359396,45790169,95581843,8099994,29401781,45237957,26063929,6497830,3233084,33895336,47893286,92803766,66137019,45407418,65338021,87710366,63059024,9175338,19486173,35456853,20681921,78561158,66116458,7423788,61271144,47361209,17081350,59318837,62232211,26744917,33431960,83651211,38022834,37166608,81172706,26392416,77898274,1569515,69136837,44348328,61815223,99971982,84349107,90090964,56153345,11543098,95395112,4199704,89046466,48260151,17976208,91990218,57393458,99333375,96420429,22766820,25842078,53393358,83083131,54058788,54762643,9257405,86460488,7685448,82897371,57241521,11161731,40984766,13173644,89078848,54232247,91255408,31491938,96906410,20642888,42073124,56531125,65271999,37675718,3183975,73617245,9599614,84684495,27041967,37224844,78909561,36685023,36808486,17957593,1936762,54663246,45667668,71300104,49597667,83948335,41092172,37192445,93562257,14349098,99604946,71920426,44784505,27411561,70367851,50007421,21919959,36468541,4787945,40197395,48774913,4798568,84840549,14363867,82979980,72725103,44479073,58749,72793444,51590803,4095116,84406788,12303248,4069912,53547802,86118021,36753250,89804152,14045383,59371804,89996536,32058615,6793819,94360702,71965942,24168411,91281584,29635537,38510840,42199455,37435892,37891451,35996293,17365188,67513640,70541760,3509435,99226875,36845587,9188443,58180653,37560164,8791066,93270084,65081429,55960386,76315420,83269727,22721500,69697787,10264691,73021291,51315460,51466049,65047700,80251430,94076128,824872,83302115,29516592,62693428,72732205,92530431,38365584,83747892,38645117,45617087,32250045,83133790,56955985,36312813,76540481,81774825,75407347,33201905,24619760,79880247,4508700,30543215,74441448,21397057,66322959,5970607,77301523,82327024,40686254,79322415,9058407,78785507,85695762,91727510,95290172,74743862,36933038,64055960,99515901,8733068,69605283,67084351,12024238,33061250,75820087,92867155,4978543,18411915,8729146,22942635,12664567,59405277,26275734,29065964,94595668,35512853,59109400,28734791,15948937,62552524,57961282,98739783,20568363,24953498,30694952,24314885,73109997,45049308,41245325,96318094,87598888,18466635,91957544,6819644,72019362,11581181,52060076,84166196,26664538,90158785,30811010,90004325,74452589,60393039,48395186,84293052,96852321 +55470718,76540481,51047803,58749,30569392,26776922,39801351,17068582,50007421,18806856,77413012,36580610,76330843,49328608,43152977,8696647,34497327,68204242,66137019,6986898,63372756,99515901,88653118,68816413,72614359,45428665,85242963,26998766,44481640,15902805,78909561,93933709,44177011,14349098,21993752,53666583,8651647,20140249,65715134,96193415,26734892,89996536,63059024,35092039,38009615,58751351,74724075,69786756,63506281,7687278,86001008,61728685,9860968,95290172,9188443,72274002,31491938,53547802,77787724,27665211,96735716,4798568,57359924,34044787,82979980,62452034,29959549,36933038,3487592,30366150,93359396,97783876,92033260,16054533,62430984,14947650,3294781,95957797,43501211,49597667,69848388,77272628,20002147,80555751,29834512,6808825,9599614,89078848,77312810,99224392,96906410,99524975,57393458,61859143,65271999,50188404,2917920,9623492,35456853,75543508,75135448,38494874,50702367,36685023,83789391,4119747,89214616,415901,81172706,68000591,70367851,97281847,39847321,40677414,61859581,99861373,74743862,20836893,70596786,94516935,72357096,33336148,47361209,47090124,44348328,44889423,24058273,9603598,67281495,29466406,23194618,57163802,1431742,91802888,53648154,26063929,29932657,36780454,569864,50668599,59910624,77284619,72777973,86301513,46540998,72793444,874791,19486173,10366309,37957788,51426311,72019362,29065964,60106217,95395112,59177718,23569917,83651211,87720882,59197747,56424103,17957593,94539824,4515343,95010552,19939935,61263068,99729357,63967300,12571310,91957544,23848565,62490109,62936963,81853704,57961282,24591705,9886593,46851987,76960303,42782652,22721500,70036438,71657078,79136082,55960386,82178706,92787493,51466049,22129328,40197395,61897582,70420215,93566986,48893685,56484900,54014062,84127901,36505482,38008118,84684495,96318094,62496012,38022834,79942022,98943869,40027975,48675329,39986008,15064655,8055981,36312813,61623915,17539625,11757872,83948335,60102965,26392416,10358899,99658235,22997281,1569515,26275734,42307449,96726697,97641116,90310261,92803766,86821229,6038457,69641513,66428795,75986488,73392814,61982238,7502255,16380211,97561745,18411915,45407418,18466635,4099191,55247455,45075471,82427263,56531125,78589145,92604458,4668450,12664567,16567550,66271566,7011964,6505939,35996293,91938191,36930650,51472275,65017137,41092172,68041839,32161669,98371444,70372191,65074535,70541760,64087743,64055960,55602660,55753905,21533347,46870723,90004325,44983451,40152546,66667729,50806615,82052050,40534591,62740044,62051033,20765474,30463802,44473167,57241521,3880712,83155442,72725103,4508700,13173644,37183543,69579137,4095116,59371804,11543098,64848072,33249630,37891451,43376279,8807940,33061250,44842615,21673260,80246713,27411561,43322743,13231279,14220886,62762857,34432810,3509435,5111370,95581843,94090109,3773993,61960400,42692881,89217461,83368048,52204879,15015906,98948034,81677380,6793819,99549373,80316608,80251430,16099750,70782102,38658347,65338021,16097038,9575954,34493392,15535065,22942635,33431960,19891772,45990383,71469330,4064751,39171895,75777973,65454636,55669657,66611759,74137926,67030811,89046466,70004753,61373987,53842979,28851716,3088684,55770687,18699206,84187166,49271185,33935899,68644627,72495719,23432750,33628349,526217,79806380,20486294,77187825,16595436,1250437,14731700,37435892,17764950,62803858,5073754,91664334,37501808,88904910,32426752,48892201,42947632,5970607,33797252,1204161,33201905,27187213,34698428,48673079,49641577,73617245,92530431,3235882,14723410,9259676,92692978,43933006,5267545,73124510,38061439,33123618,88047921,13470059,21070110,32590267,669105,61141874,66319530,91240048,69605283,24826575,41481685,65851721,44479073,8791066,88698958,72732205,48774913,99125126,112651,25636669,36845587,40781449,42237907,10453030,4787945,51464002,47887470,99226875,7182642,16019925,33553959,29819940,26102057,85116755,57248122,37620363,53393358,78766358,75153252,35512853,90457870,30139692,84293052,51588015,33565483,71333116,63628376,43506672,26863229,16405341,54762643,22879907,30218878,64098930,18001617,66663942,17385531,97011160,54427233,45790169,68939068,82532312,45237957,18504197,27185644,72278539,48260151,32151165,30787683,59109400,87160386,79821897,48360198,51590803,1197320,7685448,11161731,91141395,44550764,53888755,41245325,98739783,79922758,93562257,19272365,6525948,13468268,58224549,68110330,38341669,508198,68824981,92998591,9257405,6157724,36468541,69136837,33699435,5640302,48395186,99297164,77072625,42199455,57803235,57020481,90013093,62693428,17016156,36139942,98462867,76434144,45667668,35192533,65880522,24314885,11923835,22766820,7300047,78561158,67227442,69255765,45381876,26493119,19376156,94595668,23134715,71300104,96420429,17037369,97379790,91990218,18783702,82651278,14045383,95251277,57240218,52293995,45996863,51315460,42967683,78785507,81805959,7955293,4434662,92554120,67793644,21919959,84166196,39201414,83533741,8733068,93270084,95726235,47298834,48658605,81274566,34295794,99021067,90272749,78218845,30771409,94360702,98648327,45306070,247198,58208470,40686254,44060493,9175338,28796059,49236559,97940276,20642888,1022677,21397057,47213183,76671482,30090481,14093520,91255408,8634541,84840549,2717150,92215320,22200378,26139110,31904591,99965001,1788101,51507425,10649306,321665,37280276,77620120,29029316,22405120,20885148,80014588,2891150,68102437,76825057,86798033,83269727,33304202,63152504,3233569,32159704,60955663,54517921,71083565,19101477,47893286,85711894,41442762,37659250,54606384,40356877,89804152,168541,96709982,6111563,90654836,8913721,55143724,6153406,99690194,15948937,77300457,67084351,8115266,82897371,62232211,45848907,29994197,59981773,30395570,12303248,31727379,97057187,8128637,87598888,94911072,66832478,22113133,9398733,6497830,98130363,4091162,24168411,2208785,73222868,7517032,59318837,45794415,30998561,36135,66250369,78549759,29188588,44627776,36753250,31126490,8129978,34236719,72738685,82726008,15148031,71920426,65038678,36808486,12416768,16971929,93057697,27325428,32250045,77694700,51715482,10961421,55189057,22450468,74614639,83150534,96852321,74441448,20568363,75052463,26292919,74110882,40865610,76170907,3233084,54232247,83302115,19457589,66045587,91727510,39553046,53802686,12348118,54199160,41380093,61271144,68694897,85571389,52261574,66322959,72373496,24619760,73109997,89419466,5822202,62552524,1375023,93053405,32699744,44784505,73168565,16387575,37166608,10309525,37675718,52613508,56461322,99333375,74357852,75229462,89811711,72238278,9829782,33237508,47792865,70800879,83210802,78300864,26397786,12024238,82339363,91831487,81855445,99917599,20122224,26507214,53084256,68128438,10597197,17857111,87710366,13348726,45995325,4985896,50567636,45617087,56515456,11581181,75407347,60581278,67451935,23740167,19898053,94076128,69355476,60430369,78884452,89637706,1936762,28550822,36189527,28787861,44664587,54987042,31161687,66116458,83747892,34946859,83133790,54663246,82886381,34698463,86460488,81898046,88130087,77377183,17386996,30764367,14626618,11365791,23110625,16445503,92541302,2204165,52060076,29635537,73235980,42073124,31733363,61741594,29516592,7229550,2607799,40984766,68316156,7423788,79738755,5832946,6221471,749283,92398073,79191827,58180653,37739481,84904436,79322415,7919588,43045786,92867155,64157906,76315420,86543538,61815223,2331773,82327024,64602895,68702632,38510840,77301523,28928175,17976208,26664538,8099994,91281584,65275241,68875490,24953498,9058407,1239555,73786944,67124282,71316369,51149804,33895336,99971982,44847298,7104732,74452589,22108665,30543215,44846932,85023028,30694952,7646095,83083131,89699445,89499542,94711842,84002370,26114953,79657802,29510992,5482538,14326617,67513640,6871053,44844121,90061527,176257,18131876,92692283,92353856,57658654,38365584,18833224,42644903,17081350,56473732,45919976,17727650,90158785,82779622,76703609,15536795,38256849,16424199,55850790,39373729,59329511,62115552,59501487,48088883,16684829,3633375,20867149,92071990,44245960,47738185,4806458,30891921,90090964,55615885,61712234,3183975,59405277,17365188,65081429,59614746,47708850,17894977,27041967,54263880,40781854,32058615,18663507,29401781,60176618,86118021,26744917,85117092,30653863,10264691,24733232,67474219,37560164,99604946,49882705,17058722,53632373,4978543,84349107,71522968,38645117,77898274,25842078,21289531,44915235,56153345,63015256,17146629,9860195,13862149,4069912,75128745,41092102,78602717,36812683,60393039,21001913,20645197,36396314,71965942,86022504,15163258,67031644,66903004,15075176,30163921,86242799,20681921,27625735,6819644,6521313,4204661,75820087,8729146,14363867,93790285,69697787,93515664,81774825,85695762,79880247,7066775,98653983,37224844,54868730,62428472,43357947,62357986,13819358,73031054,30811010,28734791,34667286,88444207,61928316,37192445,8373289,4199704,56955985,65047700,84406788,88251446,70727211,73021291,22435353,72358170,32274392,49598724,54058788,24915585,824872,45049308,66885828 +14093520,44842615,29819940,61623915,55247455,12416768,31904591,21993752,77312810,81677380,20486294,52261574,30787683,18833224,90457870,98130363,66885828,33699435,17727650,66250369,65047700,85023028,29516592,48892201,4515343,99549373,17764950,84127901,91957544,18411915,26114953,73124510,94090109,22108665,67031644,82886381,27665211,44550764,51047803,57240218,44983451,59318837,1197320,31733363,7955293,60430369,96420429,9188443,62936963,67793644,98943869,44889423,56531125,91802888,2331773,1250437,19939935,38494874,28851716,14626618,69355476,85695762,34698428,3509435,80316608,5970607,44784505,68816413,6793819,8913721,48360198,17058722,52060076,44844121,55753905,3183975,65038678,22721500,32161669,53547802,96318094,68644627,12348118,18663507,73168565,10309525,72738685,415901,73222868,69136837,9575954,33061250,68204242,59501487,97783876,669105,88444207,22450468,17976208,55770687,99524975,35092039,30366150,43357947,45919976,13348726,91664334,56515456,78766358,20885148,20002147,75543508,20867149,58751351,53632373,53393358,26998766,53666583,43045786,7104732,72373496,14220886,49597667,5111370,92803766,39986008,81805959,66903004,96193415,6505939,50567636,16684829,97641116,22766820,9829782,15015906,29466406,26863229,18504197,57393458,39553046,7011964,44473167,4806458,74357852,33628349,99604946,33895336,77072625,15535065,70036438,28734791,81898046,27041967,89419466,99658235,34236719,72019362,81274566,67030811,32250045,86001008,19101477,99861373,15902805,75153252,97057187,36580610,35456853,64087743,37183543,8791066,78602717,86798033,13862149,65275241,82178706,74137926,17385531,81853704,9623492,53084256,73617245,76434144,23569917,22113133,85242963,6153406,29065964,66137019,76330843,79322415,1431742,4798568,8634541,62430984,97281847,72614359,33304202,55470718,54058788,56473732,28928175,84349107,8129978,74441448,10366309,16097038,7685448,37675718,45990383,4787945,10649306,2717150,1022677,59329511,43152977,20642888,78589145,65271999,4064751,45996863,71965942,83150534,63506281,78561158,71333116,168541,83302115,83747892,44245960,6808825,39171895,55189057,61373987,77284619,74110882,38341669,10961421,98371444,68939068,95395112,72357096,30218878,51472275,59197747,50188404,64098930,25636669,29510992,19376156,79821897,4668450,20140249,48893685,29959549,49641577,27411561,60955663,60102965,42692881,36812683,78909561,93562257,50806615,3633375,4508700,80251430,21070110,81855445,54199160,7300047,7919588,90272749,65081429,44177011,24733232,94711842,70800879,92604458,80246713,91727510,13470059,76170907,15075176,37957788,64157906,89214616,80555751,32590267,61263068,40865610,82427263,93270084,4099191,22942635,93566986,72732205,85116755,63628376,3294781,83155442,7229550,95957797,17146629,18783702,37620363,45407418,91990218,39201414,65715134,76540481,51466049,70420215,44348328,1788101,68041839,4434662,59371804,35996293,38365584,72278539,72358170,13231279,4069912,51315460,88653118,98739783,26392416,33935899,99021067,40356877,89637706,12024238,71522968,79880247,34493392,67451935,82339363,44915235,24619760,37435892,3880712,70596786,5073754,28796059,91255408,77272628,92787493,86022504,35192533,77787724,26102057,42967683,72495719,40027975,47090124,29635537,17037369,26292919,11923835,93359396,96726697,33797252,8115266,51464002,93053405,21289531,749283,69605283,874791,67474219,92530431,84684495,91831487,48673079,15536795,48260151,112651,6221471,51590803,57359924,98948034,12664567,1936762,33431960,27625735,40781854,84166196,54762643,20568363,66667729,54014062,14947650,79191827,13468268,84904436,84840549,61982238,27325428,5832946,37224844,12571310,95726235,72238278,30764367,73235980,40152546,82979980,40534591,44481640,68702632,53648154,247198,31126490,96735716,97561745,17957593,98653983,18131876,70727211,28550822,42073124,72777973,99690194,16971929,43322743,98648327,59405277,82052050,5482538,33237508,26139110,90013093,36468541,60581278,569864,84406788,21673260,77377183,69848388,36780454,23194618,57658654,89811711,77300457,91281584,9599614,2891150,37891451,78785507,89078848,92998591,38008118,66322959,99917599,31491938,508198,34667286,99224392,17857111,99729357,29029316,57241521,96906410,62803858,24058273,34946859,63372756,13173644,90090964,11161731,38510840,31161687,92554120,33201905,4119747,84293052,88904910,63059024,33249630,75986488,68875490,68824981,52293995,82726008,90004325,14731700,23848565,95010552,6871053,74743862,17068582,61897582,77413012,9257405,7646095,23432750,66611759,36753250,23740167,4095116,66116458,1569515,45794415,19486173,8729146,89699445,47361209,83368048,71300104,78884452,79136082,62490109,32426752,6986898,5640302,8733068,50668599,79806380,24591705,38022834,9259676,76703609,73031054,22129328,30694952,88251446,7502255,57163802,66271566,59614746,38658347,36189527,17365188,20645197,45790169,72793444,60176618,18001617,53802686,49598724,71657078,37560164,40686254,92033260,30139692,94076128,79942022,37501808,46540998,8696647,83210802,30543215,41245325,9603598,36808486,29834512,30090481,54868730,57803235,61859581,49328608,95581843,99125126,57248122,26493119,63967300,66832478,75407347,16445503,65880522,62452034,51149804,526217,88698958,87598888,48395186,1204161,54606384,46851987,77898274,86821229,32159704,9886593,31727379,72725103,51507425,34497327,92071990,71920426,91141395,73392814,56484900,79657802,17894977,82532312,18806856,77620120,61741594,26507214,86242799,22405120,24826575,321665,83789391,45049308,67124282,46870723,88047921,52613508,62762857,38645117,36505482,70372191,34698463,76960303,63152504,26734892,95290172,19272365,49882705,30163921,16019925,29994197,80014588,16424199,32274392,65074535,44627776,65017137,14326617,42947632,44479073,78300864,38009615,16380211,10597197,36312813,77694700,7423788,45306070,43501211,56424103,96852321,30395570,16387575,54517921,66319530,75777973,68316156,99965001,34295794,26063929,40984766,21397057,47213183,4978543,93515664,33123618,9860195,21919959,86301513,70541760,33336148,19457589,59109400,69579137,39801351,17016156,65851721,30891921,99515901,17386996,30653863,11581181,37166608,62552524,17539625,86543538,24168411,54427233,2917920,824872,77187825,37739481,62115552,96709982,93790285,11543098,2204165,24953498,99297164,34044787,30463802,55602660,62496012,92353856,6497830,40197395,14723410,45237957,42199455,54987042,29932657,22200378,64055960,36933038,66663942,9058407,60106217,85117092,92215320,8373289,41481685,18699206,26744917,87710366,56153345,41092102,48774913,94595668,81172706,94911072,26664538,57961282,5267545,58749,98462867,16054533,55143724,67227442,62740044,29188588,13819358,92398073,18466635,89804152,32699744,37659250,42237907,36396314,99333375,59981773,93933709,87720882,72274002,55669657,30811010,64848072,90310261,8807940,35512853,14349098,89996536,4199704,42644903,19898053,48088883,79738755,75135448,10358899,82327024,8099994,4985896,15064655,58224549,61859143,40677414,20836893,61960400,7687278,99226875,25842078,60393039,14363867,42307449,90061527,51426311,49236559,62428472,44847298,3487592,62232211,74614639,9175338,42782652,20681921,94516935,10264691,89217461,74724075,75229462,83269727,63015256,48658605,54263880,36930650,50702367,41380093,26397786,89499542,43376279,52204879,83533741,41442762,15148031,69641513,6038457,3773993,68110330,37280276,39847321,45428665,26275734,44846932,3233084,1239555,38061439,89046466,4091162,57020481,43506672,8651647,51588015,34432810,47298834,69786756,92867155,47887470,16405341,36685023,37192445,86118021,40781449,62357986,91938191,75052463,6521313,77301523,6819644,65454636,51715482,8128637,22997281,97379790,90158785,45617087,84002370,75128745,85571389,45848907,48675329,44060493,66045587,82897371,61815223,47708850,62693428,67281495,20765474,76671482,36845587,2208785,69697787,59177718,97011160,69255765,9398733,61141874,78218845,94539824,23134715,71469330,8055981,68102437,50007421,81774825,82779622,68000591,78549759,92692978,47893286,20122224,92692283,5822202,92541302,68128438,29401781,15948937,3088684,4204661,79922758,44664587,47738185,30569392,27185644,99971982,3233569,66428795,14045383,73786944,76315420,21533347,87160386,36135,56955985,58180653,19891772,7182642,67513640,16595436,54232247,45075471,56461322,65338021,11365791,53888755,38256849,93057697,30998561,61271144,26776922,6525948,70004753,84187166,55615885,7517032,74452589,55960386,73109997,59910624,45667668,71316369,76825057,53842979,10453030,58208470,73021291,7066775,23110625,47792865,2607799,3235882,61728685,83651211,55850790,95251277,27187213,45995325,70367851,16567550,32058615,41092172,61928316,39373729,49271185,11757872,82651278,85711894,22879907,33553959,75820087,24915585,94360702,83133790,24314885,30771409,83083131,91240048,22435353,68694897,83948335,36139942,6111563,71083565,54663246,9860968,43933006,67084351,97940276,33565483,70782102,16099750,88130087,90654836,61712234,45381876,62051033,28787861,12303248,1375023,64602895,86460488,6157724,21001913,17081350,15163258,176257,32151165 +415901,98462867,55753905,86022504,29834512,17764950,68816413,68204242,67793644,68000591,91664334,54199160,28928175,3183975,33249630,89637706,72274002,4069912,69355476,11923835,25842078,64157906,4787945,30395570,99965001,81898046,52613508,3633375,43501211,13173644,24058273,39201414,24168411,81853704,5111370,14093520,77284619,70372191,2717150,67084351,18131876,95290172,9058407,29635537,84904436,8807940,20765474,67474219,92398073,55189057,33797252,36505482,96193415,52060076,83210802,59318837,97561745,99524975,9188443,48774913,38658347,20002147,82979980,87710366,85117092,38645117,63628376,70800879,45919976,49271185,21993752,29819940,6221471,7423788,40781449,83302115,49236559,92803766,1197320,10309525,99515901,47792865,55770687,35092039,43322743,98739783,5822202,65047700,55143724,26102057,3088684,21533347,10358899,62740044,10366309,98130363,40865610,94595668,72238278,36468541,66832478,37739481,19376156,27411561,92033260,85116755,55602660,6525948,4119747,14731700,50567636,75543508,78602717,68644627,90090964,15535065,33237508,32250045,40677414,23848565,15064655,61960400,60102965,66885828,66663942,98371444,60430369,508198,16971929,75820087,95581843,78218845,35456853,44842615,92787493,526217,37620363,61263068,74357852,59197747,54868730,54663246,72373496,57803235,63967300,85242963,27325428,96318094,62490109,30694952,30569392,50702367,8729146,91957544,26139110,9623492,44479073,72777973,53084256,6793819,33553959,65338021,41092102,17037369,75229462,7646095,3509435,92554120,36753250,42967683,66428795,76434144,18833224,80014588,45237957,92692283,40152546,25636669,30139692,47361209,95395112,29466406,16595436,44889423,23110625,45428665,9575954,63506281,70004753,58751351,22997281,29029316,55615885,68824981,15148031,30811010,80251430,99690194,9886593,38008118,55470718,99658235,99549373,29994197,42782652,669105,38494874,45306070,42644903,61373987,70727211,99604946,44846932,70541760,26063929,3294781,82886381,89214616,88251446,54987042,95726235,44473167,66116458,85571389,33431960,18411915,67451935,6986898,14326617,1788101,96726697,34497327,4204661,39553046,50188404,35996293,45617087,44983451,50668599,45996863,56424103,78589145,44060493,47090124,76170907,55669657,42073124,4508700,34295794,5970607,7955293,62496012,84187166,83789391,60955663,31126490,26664538,22942635,62452034,77694700,15163258,86460488,28550822,91831487,2891150,81172706,19272365,86301513,92071990,66903004,321665,43933006,98943869,28734791,31904591,38061439,72738685,68702632,46851987,14220886,62803858,30366150,30653863,51507425,17727650,16424199,46870723,51047803,37183543,96735716,81677380,16054533,9829782,81855445,7502255,83155442,82052050,73021291,48673079,97940276,39373729,2208785,65275241,66137019,30891921,77300457,82339363,12416768,56531125,18783702,72725103,51472275,69579137,86242799,80316608,56515456,16405341,61728685,92353856,48260151,2204165,13862149,56153345,61982238,40781854,71333116,1375023,44915235,92215320,76671482,23569917,69136837,18806856,68041839,36808486,6038457,53547802,37192445,73109997,17081350,4064751,13468268,14626618,66250369,16380211,61859581,6153406,27665211,9860195,93053405,78561158,24733232,71469330,34493392,8129978,22450468,26863229,61712234,64602895,24826575,26292919,71920426,72495719,34236719,8913721,7517032,79880247,68316156,7919588,73235980,87598888,57240218,4668450,24591705,44348328,32590267,61623915,93933709,15075176,75052463,3233569,16445503,8128637,62232211,62357986,26392416,824872,77787724,14947650,32058615,18466635,82726008,47708850,43376279,20486294,58180653,7687278,11581181,91802888,40984766,17068582,66045587,3233084,52293995,13231279,4806458,73617245,80555751,89078848,75777973,30163921,44481640,66322959,7685448,32274392,29401781,93057697,84406788,88653118,73168565,90457870,76330843,99971982,74441448,59614746,99297164,79738755,33628349,37224844,56461322,30787683,5640302,54058788,38256849,112651,19101477,89046466,34946859,41481685,73031054,6808825,54014062,83533741,4199704,42947632,79657802,20122224,58208470,874791,79922758,67513640,10597197,67031644,45995325,72019362,17146629,95957797,9257405,29065964,168541,34698463,31161687,2607799,84349107,78785507,26275734,82897371,38341669,37659250,17539625,74110882,7229550,40197395,30218878,62693428,83651211,9398733,94090109,67030811,41245325,56955985,99333375,90061527,57163802,71083565,12348118,62552524,34667286,89699445,51715482,28851716,33304202,82178706,77898274,65715134,77187825,76825057,48892201,62430984,63015256,31491938,77620120,22129328,36930650,45407418,8115266,14349098,67124282,48088883,52261574,72732205,75128745,8055981,74743862,12664567,8696647,79821897,1431742,75986488,59405277,93562257,61897582,74137926,23194618,20140249,3235882,17386996,7182642,44847298,78909561,70420215,89811711,93515664,36396314,4095116,54427233,55960386,65038678,19939935,5482538,54606384,48658605,19898053,50806615,79191827,34432810,37891451,1204161,84002370,98648327,74614639,77272628,92604458,59371804,17857111,65851721,53802686,90310261,40356877,21070110,49328608,8099994,569864,33336148,56473732,97057187,30998561,68939068,15948937,55247455,10961421,96906410,66667729,19891772,45794415,4515343,76315420,96852321,83368048,68128438,45381876,90013093,33201905,17957593,22879907,84840549,89804152,66271566,43357947,7011964,59109400,16567550,89499542,30090481,9599614,17976208,48893685,39801351,32151165,38022834,82532312,65081429,11543098,4099191,77072625,30764367,31727379,41092172,99917599,6521313,59177718,86821229,41380093,15015906,57961282,33699435,91240048,78549759,20836893,57359924,42692881,99861373,18504197,81805959,69255765,5073754,99021067,27625735,27041967,23432750,47298834,26493119,18663507,19486173,61141874,29510992,29932657,8651647,4091162,64055960,20645197,84166196,8791066,37675718,32159704,16099750,35192533,97379790,99125126,3487592,94516935,11161731,20867149,59501487,92692978,7066775,33123618,36780454,57393458,69605283,13470059,72278539,27185644,88130087,32426752,94911072,78884452,98948034,67281495,24619760,8634541,36685023,5267545,6871053,61815223,59910624,69697787,48360198,76540481,93270084,16019925,30771409,79136082,61271144,47738185,48675329,85695762,26507214,49882705,38365584,4978543,42199455,49641577,82651278,55850790,85711894,22200378,47213183,26776922,29188588,37957788,86001008,95010552,43152977,22435353,74724075,16097038,34698428,2917920,51590803,63059024,54762643,39171895,71300104,84684495,64098930,45790169,20642888,79806380,40027975,75407347,6157724,50007421,53842979,83150534,18001617,26397786,36312813,99226875,57658654,43506672,19457589,10264691,68102437,29516592,44784505,57248122,97641116,1936762,91727510,86543538,80246713,86118021,5832946,88047921,69641513,93359396,72614359,2331773,43045786,53632373,8373289,22405120,22108665,10453030,29959549,88444207,36135,13348726,176257,71316369,71522968,62428472,87720882,37560164,18699206,73222868,26114953,93566986,53888755,36580610,65074535,60176618,15902805,62051033,26734892,77312810,65017137,78766358,53666583,45667668,44627776,6505939,58224549,247198,71657078,91255408,90004325,94539824,96420429,70036438,92530431,74452589,91938191,70367851,65454636,41442762,61859143,21289531,24953498,53648154,78300864,6111563,97011160,37501808,12024238,17365188,61741594,76960303,62115552,14363867,33565483,66611759,8733068,44245960,99729357,90654836,63372756,42237907,4434662,81274566,76703609,62936963,21919959,20681921,65271999,61928316,83133790,16684829,79942022,39847321,17058722,30543215,17385531,33895336,72357096,97783876,54517921,47887470,77413012,77377183,82327024,83269727,31733363,13819358,45075471,45990383,81774825,57241521,45848907,51464002,37280276,12571310,85023028,21673260,67227442,48395186,9860968,65880522,6497830,15536795,94076128,68875490,36812683,91141395,69786756,53393358,83747892,64087743,66319530,22113133,60393039,9259676,14045383,75135448,10649306,64848072,39986008,9603598,32161669,92998591,44177011,89419466,28787861,1239555,22721500,73124510,51466049,75153252,82427263,38009615,51315460,17894977,44550764,54232247,37435892,3880712,26998766,98653983,58749,749283,22766820,20568363,63152504,91990218,51588015,59981773,84127901,4985896,91281584,9175338,14723410,84293052,17016156,32699744,96709982,36845587,6819644,11757872,83083131,60106217,35512853,94360702,34044787,62762857,92541302,49598724,1022677,1569515,88904910,77301523,79322415,68110330,21001913,72358170,90158785,20885148,72793444,26744917,93790285,7104732,45049308,97281847,83948335,90272749,24915585,36933038,40686254,99224392,70596786,37166608,3773993,30463802,69848388,1250437,33061250,73786944,59329511,51149804,68694897,86798033,11365791,95251277,71965942,89996536,7300047,44844121,44664587,16387575,60581278,49597667,42307449,36189527,33935899,12303248,94711842,51426311,52204879,73392814,24314885,4798568,87160386,92867155,23740167,38510840,23134715,88698958,21397057,40534591,46540998,56484900,89217461,70782102,28796059,54263880,57020481,36139942,47893286,27187213,82779622 +97783876,65851721,66319530,63059024,51047803,87598888,36505482,57240218,90272749,57241521,44627776,24733232,112651,4119747,82427263,93933709,87720882,34497327,4199704,1788101,8128637,63967300,13231279,60581278,55189057,68041839,16019925,55669657,93562257,89699445,95251277,28796059,66667729,75777973,10366309,80014588,12024238,92554120,247198,89046466,39171895,65880522,43506672,84293052,6793819,8055981,45848907,69848388,508198,30139692,80246713,17068582,45990383,4434662,43376279,93790285,3773993,97281847,16405341,35092039,40677414,30787683,72019362,68702632,73222868,3233084,18783702,44889423,98943869,8651647,30163921,33304202,59197747,77413012,22942635,34236719,30771409,21070110,37501808,55470718,3880712,85242963,83210802,13470059,23134715,99125126,98653983,68204242,81172706,4668450,62496012,1022677,11365791,79821897,9829782,91255408,98462867,36189527,3233569,57359924,42782652,92692978,14045383,38022834,70782102,40686254,26493119,68644627,72357096,55602660,91831487,62936963,45996863,27325428,14349098,26392416,44842615,40356877,7955293,94090109,48360198,19486173,17957593,94516935,74724075,67281495,70004753,16595436,38365584,60955663,44784505,69786756,7919588,14093520,15015906,79880247,90013093,37957788,88047921,50188404,3294781,70596786,39986008,59329511,85711894,6871053,17365188,2917920,8696647,73786944,72738685,11161731,98948034,52060076,43501211,83789391,57393458,71657078,21993752,38008118,18699206,28787861,40152546,85116755,569864,26744917,39553046,43933006,47361209,4204661,7104732,72614359,62490109,36812683,9860968,98739783,34698463,7502255,86301513,14723410,75052463,23740167,47090124,40027975,38061439,76434144,9575954,6153406,71333116,51472275,71300104,7685448,49597667,7229550,75986488,29188588,18833224,82779622,2717150,51464002,26998766,6808825,55247455,24826575,73617245,14731700,72358170,37560164,70727211,1204161,32590267,73031054,15536795,18806856,84127901,17058722,39847321,78300864,56424103,53648154,96709982,60106217,44473167,65074535,9175338,45428665,98371444,16097038,5970607,36312813,99297164,66611759,31126490,874791,89811711,70036438,28851716,88698958,58208470,64087743,80316608,669105,41380093,19891772,67124282,2204165,30694952,53084256,55143724,61815223,37739481,16099750,96726697,16971929,99224392,73235980,77787724,40781449,62740044,72274002,91990218,37183543,38658347,58224549,78589145,4064751,6111563,26863229,57961282,97011160,81805959,68102437,99021067,99658235,78602717,91664334,37675718,58749,92541302,38341669,92215320,36780454,44844121,76315420,23194618,22405120,62232211,17037369,61859581,99549373,71316369,22997281,97057187,78884452,34295794,9886593,34698428,44550764,67793644,63152504,12348118,83155442,27665211,749283,33431960,23848565,45237957,79806380,29994197,17146629,99604946,79738755,77284619,39801351,89637706,6505939,99515901,97940276,35512853,44177011,24058273,3235882,68939068,46870723,18466635,20122224,7300047,16445503,20486294,99333375,71965942,51588015,6521313,67513640,52293995,57248122,48774913,61859143,29466406,21001913,97641116,1431742,89419466,51590803,66903004,28550822,17764950,65017137,26397786,87160386,99524975,88653118,84406788,26507214,82651278,29932657,84187166,93359396,94539824,32159704,45381876,77272628,59614746,95957797,4985896,72777973,95290172,91802888,27187213,50668599,74357852,29834512,53632373,76960303,47893286,32161669,18504197,9398733,36930650,30569392,76540481,91957544,9603598,20002147,4515343,36685023,13173644,17016156,90654836,91141395,90158785,26102057,30395570,89078848,20885148,45995325,26292919,19939935,30218878,14326617,29959549,86242799,22721500,55770687,22108665,33336148,57803235,20140249,34493392,23569917,78909561,92803766,80555751,66137019,4091162,30463802,33935899,76330843,61741594,35192533,88444207,16684829,1569515,77377183,53393358,81898046,31733363,44664587,36139942,4099191,11581181,40781854,85695762,76671482,65454636,40534591,321665,81853704,71083565,73392814,3487592,99690194,21673260,4978543,86798033,35996293,168541,56484900,75543508,21533347,34432810,54517921,30366150,85117092,51466049,31491938,66663942,61373987,2891150,89214616,27411561,8129978,7423788,6819644,33201905,44847298,1197320,33565483,36580610,74137926,59177718,63628376,92398073,31904591,45919976,96852321,68000591,40865610,91240048,5111370,40984766,8733068,11923835,25842078,47213183,48088883,37620363,42307449,37280276,9623492,6525948,75135448,77898274,53547802,81855445,2607799,82532312,9257405,77187825,15535065,42237907,26664538,42967683,98130363,93053405,78218845,62452034,95010552,82178706,47887470,61897582,17385531,69605283,59910624,10597197,61263068,33061250,1936762,35456853,50007421,30764367,29635537,22129328,8099994,72725103,3633375,45617087,48893685,79322415,66045587,526217,4798568,92787493,8729146,83083131,86543538,36135,37659250,63015256,83269727,70541760,44846932,30543215,5073754,60102965,89804152,37224844,44060493,86821229,42073124,68128438,52204879,61960400,53888755,62051033,43045786,85023028,93057697,54987042,46851987,67227442,84904436,13468268,88904910,17386996,32699744,27185644,99729357,31727379,54199160,7687278,5822202,86001008,99861373,96735716,17539625,51149804,76825057,17727650,22766820,92033260,37435892,68875490,9188443,21289531,10358899,43152977,75128745,39201414,33237508,49236559,7066775,22200378,29510992,92692283,17857111,65271999,99965001,7182642,33249630,9860195,4069912,71469330,83747892,40197395,80251430,64157906,30653863,9058407,8791066,77312810,83302115,90310261,30090481,78549759,91938191,59501487,68816413,8115266,62803858,82979980,66832478,49598724,85571389,93515664,8807940,62428472,61728685,1250437,56531125,52261574,18001617,89217461,38009615,59109400,24953498,45790169,15148031,38256849,38494874,69255765,41481685,27625735,55753905,4787945,70800879,44915235,66322959,73168565,2208785,61982238,30998561,95395112,77620120,48675329,61271144,97561745,70420215,99917599,20642888,33628349,75153252,38645117,42947632,70372191,48892201,11543098,43357947,15075176,12664567,79922758,59318837,69579137,25636669,48673079,63372756,78766358,33699435,38510840,18663507,69641513,8634541,54868730,74452589,33797252,65081429,47298834,16424199,16567550,12416768,83651211,83948335,62693428,24915585,24619760,95726235,51426311,49328608,54014062,54427233,50702367,21919959,89996536,20867149,82897371,15948937,19457589,56153345,90090964,43322743,54762643,76170907,26063929,23110625,81274566,29516592,67030811,18131876,72278539,83133790,13348726,62115552,15902805,29819940,41092102,1239555,48395186,59981773,32274392,17976208,28734791,45794415,36396314,6221471,33123618,34044787,57163802,49882705,58751351,4806458,94711842,6986898,54663246,93270084,5640302,66116458,34946859,92604458,23432750,47792865,92998591,65715134,62430984,68824981,42692881,36933038,26776922,96193415,94911072,69355476,97379790,45049308,19898053,4095116,79942022,29029316,56515456,33553959,18411915,73021291,6038457,44245960,7011964,61928316,72793444,60393039,56473732,65038678,82886381,20836893,55960386,84684495,24591705,4508700,46540998,64055960,53842979,36753250,42199455,12303248,53666583,17081350,66271566,70367851,90457870,88130087,51315460,82327024,3088684,77694700,62762857,84002370,50567636,71920426,65338021,27041967,73124510,94595668,65047700,5832946,45407418,33895336,90004325,54232247,77300457,13819358,61141874,60176618,9599614,83368048,51715482,7646095,44983451,96318094,84840549,22113133,62552524,92530431,176257,14947650,49641577,26139110,44481640,15064655,48260151,96906410,67451935,16380211,64098930,12571310,61623915,66250369,64848072,99971982,45667668,68694897,79657802,92353856,16054533,60430369,93566986,5482538,31161687,58180653,56955985,415901,69136837,44348328,75229462,67474219,79136082,90061527,65275241,48658605,59371804,82339363,44479073,14220886,10453030,2331773,3509435,20568363,74614639,47738185,37166608,86460488,72373496,47708850,49271185,74110882,11757872,91281584,72732205,19272365,39373729,45306070,26114953,57658654,26734892,55615885,22879907,16387575,37891451,91727510,77301523,19376156,92867155,41092172,64602895,20765474,79191827,82726008,19101477,9259676,10649306,45075471,82052050,81677380,74743862,75820087,32250045,13862149,99226875,30811010,53802686,66428795,52613508,76703609,22435353,95581843,51507425,54263880,1375023,94360702,36468541,824872,5267545,57020481,36808486,8913721,83150534,24314885,30891921,37192445,72238278,41245325,26275734,67031644,83533741,22450468,10309525,84349107,6497830,10961421,20681921,29065964,32058615,66885828,15163258,69697787,21397057,72495719,3183975,89499542,14363867,54058788,67084351,68110330,41442762,78561158,73109997,28928175,77072625,86118021,14626618,29401781,74441448,55850790,75407347,68316156,88251446,63506281,50806615,7517032,56461322,10264691,17894977,71522968,87710366,42644903,6157724,24168411,84166196,94076128,34667286,96420429,86022504,20645197,92071990,81774825,61712234,98648327,59405277,62357986,32426752,54606384,32151165,36845587,78785507,8373289 +16019925,66667729,59329511,94516935,62936963,27041967,89046466,48360198,8733068,3880712,79821897,4985896,8651647,78602717,20486294,55753905,17365188,49271185,68041839,38008118,3509435,71657078,247198,93270084,66250369,20885148,35192533,36312813,42237907,22108665,10453030,35092039,33699435,10358899,34295794,85117092,26392416,3487592,168541,17146629,81853704,29466406,15535065,8913721,11581181,61373987,62740044,5267545,37739481,12303248,10309525,14349098,62496012,17037369,56531125,55143724,36139942,1250437,84349107,55770687,14220886,73392814,33304202,29516592,44245960,57658654,50188404,44844121,69355476,31491938,61263068,85242963,51464002,38256849,9829782,30366150,63628376,93790285,6793819,54263880,89419466,4798568,4434662,43357947,42307449,16405341,78589145,69848388,95581843,47361209,96193415,99021067,30163921,41092172,79322415,75986488,63967300,44842615,98371444,82339363,55189057,57163802,79657802,54014062,3773993,7687278,24733232,68644627,84904436,59109400,17058722,65715134,26102057,37891451,69579137,10366309,12348118,29065964,26292919,62430984,44177011,43933006,72373496,18504197,27665211,15075176,34698463,26664538,12664567,17957593,77377183,60430369,63015256,14045383,80014588,4668450,60176618,874791,59981773,5111370,26744917,65271999,54199160,9599614,14626618,89078848,96735716,89804152,54868730,14093520,38061439,43506672,44348328,3294781,36780454,93933709,92541302,27325428,55850790,60102965,24058273,97561745,60581278,98462867,67227442,22997281,22942635,7955293,99125126,40027975,749283,99549373,50567636,68000591,67451935,8129978,92604458,72777973,36812683,14731700,86301513,70727211,33237508,81898046,19939935,508198,14947650,23569917,31126490,62051033,36505482,57241521,56473732,20568363,2717150,38022834,61982238,78766358,32159704,29510992,76825057,73617245,70367851,63059024,6497830,52204879,5482538,96709982,41092102,99658235,44784505,86821229,81805959,70372191,59614746,17539625,70541760,68204242,16445503,71333116,72614359,48774913,98739783,36753250,97783876,71083565,69786756,57240218,92787493,66319530,86001008,22879907,89214616,1788101,5970607,86543538,23432750,42967683,47708850,72278539,91255408,7300047,60106217,26275734,30764367,52261574,1197320,4095116,112651,6505939,34493392,51472275,6153406,27411561,88698958,39201414,37183543,77787724,66116458,33797252,72732205,8807940,28796059,37280276,94090109,90272749,73031054,76170907,22129328,46870723,4978543,67793644,61741594,15536795,50668599,87598888,46540998,3088684,48673079,55602660,67281495,88653118,19272365,77694700,67474219,13819358,98943869,90061527,92692283,9575954,7919588,42782652,88904910,93053405,5073754,3233084,70420215,22113133,73222868,4508700,83210802,75229462,17976208,83651211,22766820,45075471,8128637,83155442,88251446,93562257,92692978,31733363,17857111,96726697,15902805,54762643,67031644,65081429,36580610,19376156,669105,72274002,80246713,30395570,22721500,40781449,50702367,6871053,99965001,33201905,34044787,82532312,38365584,6111563,43045786,18131876,39553046,80251430,13173644,59318837,19891772,4204661,73168565,91938191,40677414,33061250,69641513,99333375,86118021,89811711,84187166,29819940,45617087,59910624,77620120,18806856,78909561,92554120,13862149,40197395,66903004,22435353,77413012,91990218,44550764,80555751,71300104,39986008,61859581,83150534,47792865,16097038,85571389,36468541,20836893,76315420,95957797,7011964,18783702,43501211,74357852,79191827,18699206,4099191,45790169,19101477,61271144,6819644,9886593,11161731,23194618,4064751,62490109,7229550,35512853,32699744,42692881,57803235,47738185,30090481,89637706,74110882,5822202,7685448,1204161,45428665,42073124,65074535,2917920,49641577,95010552,58224549,49328608,27625735,74743862,90013093,24591705,84840549,824872,62803858,16380211,48658605,29834512,81172706,99729357,86242799,61960400,75820087,59371804,48088883,27185644,20681921,36933038,72725103,88130087,47887470,13470059,36135,89217461,66045587,4515343,26776922,79136082,77300457,61712234,29188588,15148031,5832946,70004753,86022504,30139692,53888755,16424199,7646095,68702632,20642888,526217,59197747,78300864,34432810,415901,33628349,71965942,66322959,1239555,21533347,53547802,52060076,87710366,59501487,569864,77187825,29029316,77312810,28851716,97011160,53632373,73124510,59177718,99917599,70596786,40984766,89996536,14723410,18001617,18833224,71469330,9188443,64602895,91957544,45996863,82327024,36189527,97940276,20002147,68824981,68816413,37659250,38658347,33431960,95395112,47090124,21993752,62693428,74441448,3183975,61859143,71316369,44847298,37501808,56424103,83302115,36685023,44983451,55470718,10961421,66611759,20140249,12024238,62452034,45919976,50007421,17764950,98653983,65047700,51047803,95726235,88444207,93359396,77898274,82178706,8099994,33895336,6986898,21070110,14326617,21397057,21919959,69255765,31904591,64098930,6221471,43376279,97057187,77272628,83083131,43152977,82886381,39171895,29994197,3235882,74614639,51588015,77072625,8055981,40356877,8115266,84293052,321665,16684829,36930650,37224844,62357986,88047921,47298834,78785507,34698428,7066775,8696647,65851721,33123618,20122224,66663942,99690194,55669657,62762857,44915235,58180653,25842078,77284619,73786944,69697787,16387575,81274566,36845587,85116755,28734791,92033260,68875490,51590803,65017137,7517032,30998561,48892201,70036438,51507425,2204165,61815223,82427263,45237957,48260151,8634541,57248122,74724075,83747892,9175338,44481640,37192445,66428795,84127901,4119747,99515901,23134715,1022677,17894977,33565483,76703609,9623492,18411915,77301523,26493119,55615885,68694897,53393358,23740167,39373729,7423788,26998766,1375023,32151165,83269727,66832478,84166196,26063929,90158785,48893685,29959549,39801351,53842979,13231279,33249630,78561158,87160386,16054533,87720882,48675329,4806458,44060493,30787683,58751351,63506281,10597197,6521313,31161687,1431742,20765474,92215320,6525948,26114953,93515664,33336148,9603598,56955985,30811010,95290172,83789391,17386996,45667668,53084256,45306070,83533741,75543508,46851987,53802686,17016156,92998591,28550822,23848565,53648154,68939068,13468268,10649306,7104732,91240048,30463802,67084351,76434144,72357096,73235980,44473167,56153345,40152546,34497327,22405120,51715482,40865610,98948034,24619760,99604946,58749,84406788,98648327,44889423,92803766,79806380,75777973,63152504,19486173,69136837,79922758,31727379,49236559,41380093,86798033,57961282,47213183,90457870,82726008,11757872,76330843,60955663,75128745,37560164,41481685,25636669,32161669,89499542,98130363,65454636,82779622,56515456,73021291,49597667,78884452,17068582,30771409,84684495,80316608,9860195,93566986,55247455,67124282,20645197,4069912,17081350,90310261,17385531,66885828,84002370,75135448,47893286,42644903,34667286,64157906,38341669,76671482,51315460,32426752,9257405,54606384,78218845,9058407,99971982,24953498,39847321,57020481,86460488,21001913,7182642,34946859,36396314,57359924,82897371,81855445,15064655,42947632,82979980,73109997,18663507,11923835,54663246,28787861,33935899,3233569,61728685,1569515,45381876,30569392,22450468,16567550,15015906,32590267,30891921,32274392,37620363,74137926,76540481,37957788,45407418,72019362,71920426,24168411,72358170,53666583,94539824,72495719,9860968,99524975,54058788,24826575,66137019,9398733,37435892,16595436,30694952,68102437,11543098,28928175,2891150,94076128,66271566,83368048,99226875,6808825,17727650,52613508,38645117,62552524,64087743,2208785,34236719,41245325,62232211,82052050,29635537,81677380,65880522,36808486,40781854,70800879,38494874,90090964,4199704,99297164,45990383,26734892,97379790,35996293,96906410,63372756,64848072,75153252,94595668,72793444,79942022,24915585,91802888,44627776,58208470,48395186,68316156,1936762,3633375,91664334,4787945,94911072,67030811,90654836,6157724,95251277,92530431,32250045,99861373,75052463,15163258,75407347,38009615,61928316,42199455,79880247,2331773,93057697,65338021,40534591,23110625,97641116,26139110,8791066,92867155,65038678,91831487,78549759,7502255,2607799,6038457,38510840,45848907,29932657,37166608,26507214,51426311,68128438,69605283,19898053,8729146,21289531,64055960,13348726,51149804,89699445,9259676,54517921,51466049,52293995,26397786,91727510,22200378,59405277,81774825,29401781,54427233,4091162,56484900,79738755,74452589,35456853,45794415,12571310,15948937,62115552,40686254,55960386,30218878,45995325,96318094,99224392,21673260,49882705,30653863,56461322,85023028,94360702,83133790,41442762,91141395,19457589,43322743,72738685,92353856,83948335,76960303,96420429,60393039,14363867,91281584,176257,49598724,96852321,92398073,33553959,44664587,82651278,44846932,71522968,24314885,20867149,68110330,44479073,32058615,67513640,54232247,45049308,72238278,12416768,85711894,16099750,92071990,65275241,90004325,61897582,61623915,30543215,37675718,54987042,26863229,16971929,8373289,70782102,27187213,57393458,11365791,85695762,50806615,5640302,97281847,10264691,94711842,61141874,18466635,62428472 +55753905,72357096,20486294,69786756,96726697,45237957,3233084,70596786,46870723,89811711,68644627,19891772,53084256,77312810,88047921,48774913,64087743,247198,65017137,55143724,83789391,23432750,97783876,17068582,97641116,48260151,4119747,61859143,6793819,40677414,56424103,12571310,54987042,6986898,48893685,44627776,20140249,22129328,27665211,40356877,29466406,89214616,25842078,63059024,33304202,23848565,74724075,42307449,75986488,63152504,67281495,4199704,75153252,14731700,14349098,50007421,78218845,74357852,8651647,1204161,59910624,73168565,37435892,88698958,95290172,69848388,4099191,74110882,26734892,14326617,67451935,26292919,94516935,39986008,569864,79657802,79821897,75777973,9860968,17058722,38008118,4095116,81274566,8128637,89804152,57241521,10366309,43322743,86821229,8696647,57248122,38494874,68816413,29959549,28851716,85242963,24733232,32161669,85711894,60102965,94090109,60581278,92604458,26776922,88904910,77300457,32590267,23194618,112651,6153406,10309525,33431960,34236719,15148031,26998766,21289531,40781854,92787493,77413012,43376279,9623492,94911072,65038678,99524975,89996536,89419466,43506672,45428665,65851721,72777973,30543215,46851987,37957788,59109400,71657078,14045383,20122224,13231279,93790285,64848072,82651278,32274392,3235882,22721500,55602660,3880712,27185644,68204242,41481685,43357947,36845587,7502255,1431742,89046466,41092172,80014588,93515664,93933709,84406788,76434144,3294781,62452034,73222868,23110625,20836893,51047803,14723410,82979980,76960303,53547802,19101477,87160386,40027975,92998591,874791,7517032,44550764,16971929,9175338,99658235,8791066,78300864,89637706,4064751,77787724,33553959,30218878,13468268,50806615,82178706,31904591,4069912,82327024,87720882,33935899,32159704,68041839,66832478,68128438,7685448,65715134,85695762,69355476,40534591,6111563,36753250,14947650,38645117,40686254,32699744,11543098,55247455,22766820,47298834,669105,1197320,16595436,62496012,84293052,24058273,54762643,33565483,99965001,78909561,49328608,61741594,45617087,66319530,73786944,92541302,84127901,7182642,87598888,13470059,33249630,43933006,81172706,78561158,63967300,52261574,98653983,37183543,47361209,55615885,59177718,83210802,39171895,72358170,29994197,74441448,91255408,77898274,67513640,20642888,98739783,77272628,92692978,36812683,74614639,79806380,35996293,48673079,15163258,70004753,93562257,58208470,96852321,7300047,34493392,70800879,17957593,1250437,54663246,60430369,29834512,37224844,66428795,49236559,90272749,65880522,88653118,92803766,90310261,72495719,18699206,90457870,22108665,92554120,85116755,61960400,19486173,66250369,2917920,47090124,73235980,99021067,27041967,36580610,62232211,28796059,37166608,76315420,99224392,56515456,4515343,42782652,30163921,2204165,22450468,70367851,45306070,42237907,38009615,30366150,61859581,84904436,23134715,62357986,29819940,9259676,80316608,72738685,35092039,10597197,76170907,49597667,72238278,79322415,6521313,45790169,3233569,14363867,99549373,81855445,91831487,6221471,70541760,82427263,31733363,75052463,77284619,27325428,44846932,66137019,59329511,15015906,7229550,80555751,6871053,77072625,78884452,86118021,45667668,62430984,70036438,96735716,37675718,8913721,30811010,71469330,68939068,26114953,6808825,66903004,83651211,77377183,47792865,57961282,63015256,33201905,91957544,94360702,29516592,59197747,48658605,75543508,44245960,56461322,24953498,7646095,168541,78602717,71300104,78549759,9398733,40197395,13173644,54199160,96193415,68102437,4787945,5073754,57240218,23569917,4091162,30463802,11161731,15064655,28787861,2717150,12416768,50702367,51464002,12348118,79922758,59614746,7104732,99125126,17016156,91938191,75135448,8807940,92692283,4204661,18131876,74452589,90061527,88444207,6505939,74137926,71333116,45990383,30787683,97011160,16097038,37501808,43045786,95957797,34698428,98462867,34497327,12664567,8099994,81898046,36930650,34698463,61141874,33237508,90654836,44784505,81853704,1022677,24591705,53648154,28550822,44060493,26397786,72278539,7919588,71083565,7011964,48360198,57020481,37560164,67793644,21397057,93057697,35192533,80251430,9603598,40781449,10961421,34295794,44348328,8733068,7066775,24314885,62803858,76671482,17386996,38658347,30395570,22879907,23740167,83155442,95251277,85571389,77187825,18504197,36468541,30653863,50188404,80246713,79136082,6157724,63372756,45407418,97379790,21673260,73392814,71316369,11365791,72725103,21070110,42947632,86543538,4798568,66611759,1569515,51715482,39847321,26493119,14093520,75820087,29029316,75229462,67474219,84187166,67030811,74743862,81805959,38061439,19898053,57359924,99297164,68000591,29401781,11757872,15902805,16019925,29188588,66322959,42967683,26744917,27187213,56473732,44177011,53842979,51507425,62762857,9257405,67124282,21533347,51588015,26863229,77620120,91802888,36505482,76825057,16405341,96318094,11923835,7687278,81677380,17365188,66116458,42692881,86022504,31161687,62051033,87710366,83150534,30771409,70420215,6525948,38022834,20867149,20885148,15948937,39553046,41442762,93359396,35456853,97281847,37891451,36808486,44473167,22997281,91990218,92215320,89078848,68875490,54058788,13819358,33336148,84840549,61271144,4985896,53888755,9575954,44842615,66271566,83302115,32058615,5970607,43501211,9188443,21993752,89699445,78589145,71522968,5822202,36780454,749283,51466049,16567550,1239555,415901,16445503,30139692,78766358,42199455,62936963,61373987,36312813,38256849,93270084,86242799,31491938,18001617,39801351,70782102,92033260,97057187,72373496,96709982,79942022,28734791,44664587,66045587,16684829,46540998,48395186,24168411,14220886,53802686,50668599,64602895,8129978,18833224,13348726,57658654,40984766,83368048,8055981,51472275,45919976,92398073,15075176,69641513,6819644,15536795,92353856,39373729,82532312,84349107,98371444,34946859,44983451,33699435,69136837,45794415,99515901,54427233,76330843,42644903,19939935,54014062,508198,15535065,79738755,27411561,58751351,83269727,82886381,4434662,79191827,70727211,20681921,30764367,65047700,26275734,95726235,5111370,65454636,57163802,66663942,17037369,42073124,99971982,8634541,62115552,84684495,21919959,54606384,26507214,9599614,72019362,95581843,82897371,16380211,62693428,49641577,9860195,98943869,38341669,65271999,55189057,526217,82779622,17146629,9058407,17385531,53632373,83948335,72614359,68110330,68702632,18663507,24826575,20765474,99861373,54517921,76540481,45381876,45996863,3773993,72732205,86301513,12024238,94076128,82726008,68694897,94711842,65074535,29635537,17894977,66885828,37280276,90004325,61897582,66667729,10358899,59318837,40865610,47893286,30891921,19376156,70372191,26102057,99604946,99729357,5832946,86460488,85023028,67084351,24619760,55669657,40152546,37620363,43152977,7955293,37659250,56531125,51590803,29065964,7423788,16424199,83133790,44889423,22200378,321665,6038457,62552524,32250045,83083131,49271185,176257,44844121,61982238,24915585,44479073,69697787,54868730,98130363,38510840,4668450,33628349,29510992,59501487,47708850,36685023,97940276,45848907,60176618,9886593,55470718,17764950,62490109,52060076,60106217,19457589,47887470,93566986,33123618,27625735,55770687,30090481,67227442,73617245,64055960,60393039,56955985,20002147,57393458,58749,6497830,1788101,18783702,65275241,68824981,25636669,75128745,22942635,73124510,75407347,52293995,10649306,4508700,65338021,30569392,18411915,48892201,55960386,26063929,45995325,51149804,72274002,59371804,96420429,96906410,36396314,83533741,72793444,3487592,93053405,94539824,16099750,52613508,78785507,61623915,20645197,95010552,34044787,60955663,13862149,71920426,31727379,97561745,79880247,91281584,63506281,49598724,17857111,17081350,3509435,95395112,91664334,26139110,73031054,53666583,14626618,2208785,35512853,44481640,36933038,10453030,47213183,22405120,36135,41380093,30694952,824872,9829782,57803235,69605283,44915235,12303248,53393358,86798033,26392416,55850790,30998561,26664538,98948034,50567636,91727510,45049308,32151165,17727650,29932657,90090964,99917599,56153345,8373289,58224549,16387575,58180653,51426311,82052050,88251446,47738185,85117092,63628376,22113133,99333375,61815223,36139942,69579137,51315460,34432810,33061250,82339363,19272365,61728685,61263068,1375023,5640302,11581181,39201414,99690194,65081429,1936762,99226875,5267545,84166196,83747892,17539625,88130087,49882705,3183975,3088684,62740044,92071990,76703609,91240048,48675329,2891150,8115266,86001008,31126490,89217461,61712234,84002370,48088883,59405277,59981773,20568363,3633375,67031644,17976208,37739481,94595668,81774825,56484900,98648327,61928316,4806458,64098930,32426752,90158785,37192445,36189527,73109997,41092102,38365584,77694700,8729146,54263880,41245325,16054533,28928175,18806856,45075471,18466635,4978543,64157906,71965942,33797252,89499542,68316156,73021291,33895336,52204879,92530431,21001913,44847298,62428472,69255765,2607799,10264691,22435353,92867155,5482538,90013093,2331773,91141395,54232247,34667286,77301523 +62452034,99604946,66250369,44842615,74137926,14093520,48360198,99549373,54427233,28928175,16097038,5970607,37675718,94090109,62430984,9188443,44473167,98739783,52261574,80555751,31733363,31904591,68816413,3294781,65275241,81855445,82886381,96193415,15064655,42782652,51472275,22450468,72777973,68644627,26863229,69355476,669105,60430369,36468541,54868730,16971929,84406788,17081350,9623492,20486294,48893685,35456853,32426752,4099191,37620363,46851987,33553959,19101477,17727650,26392416,43357947,99524975,38494874,19939935,57248122,66045587,47887470,44479073,85023028,41442762,43322743,70004753,66832478,37183543,69136837,49236559,55602660,68824981,72357096,91664334,415901,41481685,14731700,44550764,526217,77072625,70800879,98130363,29959549,40677414,2331773,99224392,71333116,70596786,98648327,85116755,61741594,59371804,10309525,10366309,4515343,29819940,321665,98948034,65715134,26139110,67474219,33628349,93515664,24591705,8913721,34698428,1250437,70036438,50806615,77284619,48892201,67031644,15015906,99226875,88251446,68875490,35192533,17385531,40781854,30653863,18663507,33304202,27325428,9058407,55247455,84166196,26507214,50007421,19486173,26275734,33797252,53802686,1197320,29029316,44627776,2204165,78549759,77300457,27665211,7919588,71522968,29994197,81677380,7502255,91802888,20122224,72373496,62803858,95957797,86001008,59405277,56531125,92215320,83155442,82427263,6111563,89419466,22721500,66885828,89499542,91957544,824872,57393458,18833224,33249630,4798568,84002370,54199160,21673260,32250045,24826575,29635537,4064751,56473732,53632373,58224549,50188404,73031054,63967300,32699744,30366150,18806856,99690194,67793644,86242799,93566986,40197395,97561745,78589145,53666583,6038457,17386996,88444207,48260151,20642888,73235980,3509435,65038678,32161669,80316608,79136082,24619760,11923835,56424103,59501487,57240218,4787945,20885148,54014062,64087743,53084256,569864,18504197,66322959,30218878,42692881,45306070,60102965,87160386,45919976,1936762,87710366,4199704,80014588,54987042,93053405,7011964,26292919,95395112,8651647,66903004,112651,2607799,28550822,61373987,84293052,17764950,95726235,38658347,39553046,12416768,82178706,39986008,64157906,77272628,79880247,27625735,7646095,3183975,6221471,73124510,15075176,81172706,86460488,34946859,45617087,20140249,90310261,90654836,24058273,58208470,36396314,37891451,91281584,22108665,17068582,28734791,72358170,56153345,75407347,60106217,47792865,2717150,16595436,9599614,4806458,52060076,7104732,168541,55960386,85695762,38645117,33061250,69641513,1022677,66663942,20681921,22129328,92604458,34236719,92398073,1569515,5822202,96318094,70372191,50702367,15535065,67281495,19376156,36505482,34497327,11757872,76434144,30090481,21919959,39201414,18783702,66271566,12664567,82052050,65017137,47708850,83789391,65454636,85242963,29834512,57359924,92033260,99917599,66137019,78766358,13862149,34295794,48088883,1239555,62051033,8733068,64055960,57803235,90457870,77377183,33431960,15948937,75820087,79191827,8115266,24915585,40984766,64848072,89214616,82979980,91255408,53888755,83651211,97641116,35996293,61623915,10358899,56955985,72278539,4508700,61263068,45790169,29510992,55753905,58751351,20002147,97281847,27411561,70727211,28851716,66611759,97379790,17146629,77187825,1204161,23848565,4069912,49641577,92554120,79657802,72738685,8373289,39373729,86543538,72732205,9398733,52613508,11161731,8634541,63372756,24733232,40534591,59318837,37501808,66116458,90013093,58749,63059024,43376279,60176618,77301523,79806380,34698463,71920426,7955293,96735716,33123618,14363867,12571310,47213183,25842078,99965001,9886593,76825057,77413012,78602717,73392814,30764367,7229550,61982238,16099750,17365188,22766820,20765474,8729146,68316156,38341669,67030811,65851721,55143724,36808486,21993752,80246713,68000591,43045786,48675329,51149804,7423788,19457589,23110625,45794415,11543098,2891150,44889423,61859143,73168565,51715482,6986898,83210802,3633375,9175338,4668450,37957788,6793819,14626618,74614639,74441448,38061439,42307449,30569392,78884452,21001913,97940276,38365584,45996863,16387575,84187166,86022504,15148031,63506281,75153252,9860195,18411915,26102057,32159704,56515456,75543508,23134715,33237508,40781449,37739481,6521313,749283,91831487,71657078,78218845,30139692,90090964,30787683,53842979,2208785,8129978,7685448,94076128,36930650,89078848,62740044,74357852,22435353,73222868,99125126,83302115,82897371,4204661,48395186,72019362,49882705,20867149,45995325,36135,8807940,82339363,55770687,62936963,81853704,32590267,29188588,89811711,49328608,33699435,46870723,1431742,59614746,66667729,90004325,88047921,76170907,30771409,44983451,9257405,55189057,51590803,36933038,17857111,73786944,72495719,5640302,97783876,96726697,88130087,44846932,9829782,91990218,44245960,89637706,42199455,29401781,31727379,71316369,76671482,51507425,26114953,18131876,68102437,26493119,30543215,75777973,14947650,53648154,55470718,6153406,68041839,26998766,99658235,17957593,77620120,44844121,74110882,20836893,42073124,99861373,62232211,45990383,56461322,61897582,72793444,92787493,33935899,77312810,10961421,27041967,99515901,247198,17894977,36312813,85571389,44847298,41092102,44060493,1788101,26063929,37435892,92803766,84904436,42237907,68204242,82651278,51464002,84349107,69255765,12348118,21070110,17016156,36753250,40027975,79821897,38022834,13348726,29065964,67451935,36780454,4095116,98943869,16380211,68110330,49597667,76960303,44784505,37166608,8696647,89046466,62693428,44348328,62762857,89804152,61712234,83747892,93562257,76703609,89699445,50567636,62357986,6808825,30811010,81274566,29932657,508198,16054533,36845587,35092039,48673079,53547802,59329511,45848907,83269727,14723410,71300104,94711842,92353856,86118021,40152546,58180653,43152977,22200378,5073754,95581843,57658654,91727510,81774825,38009615,51047803,93933709,50668599,75128745,62428472,13231279,4978543,45049308,45407418,22942635,78561158,14326617,65338021,17037369,99021067,40865610,83533741,49598724,94360702,98462867,38008118,8791066,60581278,77787724,69697787,54606384,3235882,30891921,17058722,8128637,29466406,38256849,18466635,6505939,39171895,15902805,54517921,71083565,82726008,68128438,47090124,54058788,3880712,49271185,51466049,55850790,84840549,60955663,36812683,14220886,22405120,52293995,41380093,22997281,65081429,95251277,96709982,19898053,32151165,9575954,4985896,59109400,34493392,6497830,31161687,34667286,95290172,38510840,61859581,16684829,54663246,5111370,76540481,40356877,5482538,69605283,64098930,72238278,40686254,13470059,68939068,78300864,65047700,11365791,93790285,42644903,92692283,86821229,66428795,37224844,25636669,26664538,92998591,67084351,4119747,83368048,65271999,62552524,94911072,79922758,51426311,65880522,39847321,44481640,23569917,61141874,10649306,16019925,3773993,45237957,24314885,68694897,30163921,17976208,32058615,83150534,99297164,75052463,52204879,69786756,75229462,98371444,87720882,45381876,85711894,29516592,3233084,83133790,22879907,4091162,47361209,10264691,1375023,13819358,77694700,61815223,46540998,36189527,76330843,7517032,99971982,13173644,9259676,7182642,83083131,79942022,72614359,51588015,6525948,98653983,94516935,61928316,7300047,66319530,61271144,61960400,59910624,64602895,44177011,15536795,93270084,31126490,72274002,81898046,74452589,37192445,6819644,74724075,30395570,47738185,24168411,96420429,42967683,3088684,84127901,36580610,54232247,89996536,54263880,43501211,92541302,54762643,63015256,874791,70420215,67513640,35512853,65074535,7687278,67227442,94539824,6871053,20568363,7066775,6157724,33336148,78909561,57961282,97011160,82327024,73617245,77898274,30694952,99729357,19272365,90158785,78785507,30998561,22113133,33895336,44915235,51315460,32274392,39801351,8055981,41245325,16424199,48774913,3233569,8099994,26734892,5267545,9603598,93057697,176257,72725103,43506672,18001617,59177718,75986488,42947632,67124282,18699206,33565483,61728685,36685023,44664587,73109997,57020481,90272749,30463802,83948335,91938191,31491938,33201905,26397786,82779622,93359396,74743862,11581181,69579137,5832946,62496012,59981773,94595668,90061527,86301513,70541760,82532312,28796059,16405341,4434662,62490109,88653118,20645197,87598888,57241521,91141395,92692978,19891772,86798033,23740167,47298834,14349098,92530431,80251430,41092172,96852321,91240048,43933006,23432750,55669657,26776922,81805959,63152504,53393358,59197747,34044787,76315420,99333375,55615885,45667668,63628376,28787861,9860968,14045383,79738755,62115552,95010552,10453030,34432810,37659250,92867155,12303248,96906410,89217461,68702632,70367851,16567550,92071990,47893286,60393039,75135448,21289531,27185644,73021291,23194618,71965942,16445503,48658605,17539625,21533347,12024238,27187213,2917920,85117092,88904910,24953498,45075471,13468268,56484900,15163258,26744917,69848388,21397057,37280276,10597197,71469330,84684495,37560164,88698958,70782102,36139942,45428665,97057187,57163802,79322415,3487592 +72725103,71657078,7104732,73031054,30694952,9623492,87598888,4668450,29959549,35192533,44479073,44784505,49597667,59109400,98462867,26744917,95957797,26998766,31126490,52293995,9398733,53842979,16971929,10309525,32699744,44550764,69355476,27665211,85242963,6793819,65074535,91831487,40984766,62452034,62496012,75543508,526217,90272749,20122224,54762643,65047700,1204161,15536795,75153252,83210802,38494874,33123618,22108665,47213183,47090124,7687278,93562257,34432810,89214616,10358899,44889423,62936963,1022677,16019925,61982238,96193415,33249630,45790169,29834512,89419466,59318837,38008118,34698428,17365188,88047921,99549373,34295794,1569515,67281495,29029316,8733068,30569392,99297164,7685448,66885828,70596786,45919976,18833224,36812683,16380211,73235980,13173644,30139692,38061439,77377183,99524975,36505482,60581278,16424199,50567636,20140249,69579137,79922758,57803235,17068582,99658235,35092039,82052050,80251430,40356877,14947650,62693428,66428795,24314885,96318094,415901,65038678,99333375,65715134,3233084,92398073,45075471,70782102,4806458,37620363,48360198,73124510,64098930,12571310,61373987,3880712,90457870,17957593,8055981,26063929,6153406,67793644,62552524,72373496,26863229,63967300,72274002,55753905,88444207,66322959,96906410,23848565,71965942,19939935,68694897,4099191,45990383,84406788,90090964,247198,44481640,49882705,61859143,64848072,6871053,15015906,37192445,74357852,91990218,98739783,54517921,79821897,17976208,31733363,36685023,39986008,39553046,98948034,44983451,12348118,41380093,37891451,81805959,61623915,59501487,19486173,14326617,82979980,51149804,27325428,55247455,1788101,25842078,6521313,57658654,92033260,83083131,93790285,36780454,15064655,51588015,51472275,38365584,47298834,71333116,80316608,12416768,52060076,4119747,57241521,53802686,30764367,59177718,16445503,10597197,61271144,6986898,12664567,40781449,48893685,48088883,83269727,68702632,44348328,31904591,51047803,69255765,65851721,7182642,68204242,32159704,30543215,62740044,94090109,91664334,33628349,17386996,4064751,71316369,19457589,1239555,15535065,26493119,39801351,33553959,66045587,37280276,97783876,98371444,36930650,14363867,9886593,90158785,61728685,70800879,85711894,37183543,98648327,66832478,20885148,5111370,65017137,45407418,33304202,9860968,90013093,30787683,321665,50806615,30163921,36753250,8128637,91938191,18504197,97281847,80555751,95726235,99125126,63628376,81898046,3088684,69641513,61263068,93933709,55770687,80014588,23569917,8115266,30218878,44245960,30653863,70727211,43322743,34044787,84187166,54427233,38658347,70367851,7955293,78766358,59197747,33201905,11543098,28734791,50188404,77272628,92692978,62232211,64602895,53084256,49236559,4515343,95395112,82726008,48774913,83533741,92554120,44177011,22450468,85023028,38341669,59371804,65338021,60102965,99515901,72738685,78884452,14626618,93270084,26292919,74743862,91802888,37166608,168541,10366309,5970607,18663507,78785507,6497830,74441448,32426752,71300104,99604946,4095116,91281584,66611759,26392416,66663942,69605283,75777973,24591705,44844121,9829782,56955985,68102437,33797252,89499542,28851716,82427263,60430369,54232247,85116755,45848907,92998591,6808825,99971982,61741594,33431960,87720882,60955663,86118021,58180653,508198,43376279,89078848,17016156,15948937,76315420,17058722,10264691,93359396,77187825,9188443,56153345,70004753,18466635,22997281,37659250,67030811,72732205,68644627,20681921,56515456,23194618,76330843,79738755,4508700,61141874,69848388,21673260,78589145,45428665,32250045,70541760,55602660,59910624,77284619,44846932,73786944,18699206,89996536,68875490,9058407,82339363,53393358,45381876,69136837,5822202,78549759,84684495,76960303,4985896,86543538,75986488,23740167,30463802,29635537,64055960,30998561,3773993,2717150,24915585,57248122,29932657,96852321,86001008,9259676,569864,72777973,77312810,36135,4798568,21993752,669105,81677380,8373289,2331773,8651647,29188588,24733232,57359924,71522968,86460488,45995325,56531125,32058615,96726697,29065964,36580610,79880247,44473167,2917920,91727510,42947632,28787861,74724075,79942022,40197395,92541302,99965001,4434662,11365791,81274566,30771409,6038457,39847321,42199455,22129328,7300047,96709982,45617087,14045383,47792865,5832946,2891150,83368048,1250437,63152504,33565483,99021067,79657802,34236719,99861373,56484900,46851987,10453030,93515664,19272365,92353856,51426311,84002370,76825057,83789391,75407347,73222868,88698958,14093520,13231279,33336148,54058788,112651,79191827,38645117,51466049,27185644,75135448,63059024,39171895,15163258,84840549,37435892,92867155,58751351,4069912,93053405,26114953,70420215,45306070,22942635,40677414,54987042,92215320,84293052,86821229,54663246,37739481,26776922,94516935,76671482,66667729,65880522,14731700,16099750,40865610,43933006,67451935,34497327,37224844,74110882,54606384,78909561,11161731,5482538,44060493,15075176,62430984,64087743,17539625,87710366,61960400,65271999,89699445,57961282,28550822,93057697,3509435,24058273,29510992,57163802,55189057,68824981,77898274,10649306,8913721,39201414,22721500,18783702,55470718,97561745,62051033,89637706,29819940,89046466,4204661,63015256,26102057,51590803,77694700,8129978,65454636,42307449,53666583,19376156,18806856,66319530,77413012,1197320,45794415,72495719,94711842,41245325,45237957,42782652,7502255,45996863,9860195,24619760,27625735,54263880,96735716,77787724,3487592,10961421,81172706,9257405,32161669,21001913,73109997,50702367,77620120,8729146,84127901,30090481,33895336,4787945,53648154,73168565,84349107,19101477,45667668,94360702,13470059,62490109,74452589,9175338,67084351,95010552,67227442,30891921,76434144,98130363,7646095,74614639,8807940,28796059,41442762,69697787,78561158,22435353,9603598,29994197,51464002,26397786,9575954,47738185,39373729,83948335,824872,62357986,26275734,50668599,48658605,57020481,72019362,82532312,75229462,8696647,54199160,49641577,46870723,83150534,79322415,73392814,18411915,68128438,50007421,83155442,27041967,47893286,6505939,32151165,97011160,53547802,3183975,42073124,52204879,92803766,30395570,59981773,55669657,23110625,61897582,60106217,92692283,68939068,2208785,44842615,73617245,63506281,56424103,11581181,68000591,35456853,17385531,78300864,88653118,55615885,6525948,62762857,26507214,1431742,62803858,26139110,58224549,89811711,47708850,73021291,42967683,88904910,72793444,89217461,68316156,61815223,82897371,6111563,24826575,34946859,98653983,16097038,58208470,20642888,16595436,87160386,48892201,8791066,71920426,91240048,47887470,6221471,80246713,12303248,36312813,16387575,57240218,84166196,36845587,52613508,79806380,55143724,85571389,81853704,52261574,44627776,22879907,8099994,35512853,69786756,749283,22766820,66271566,86301513,92071990,77300457,66250369,43152977,53888755,61859581,54868730,3633375,7517032,92604458,20568363,37675718,36189527,83302115,20002147,26664538,98943869,16405341,23134715,63372756,92530431,67513640,32274392,66137019,18131876,35996293,7423788,94911072,20765474,1375023,31727379,56473732,176257,51315460,74137926,2204165,89804152,53632373,78218845,82651278,51715482,14220886,19891772,81774825,83133790,86022504,65081429,3235882,20645197,99917599,97940276,57393458,43357947,42644903,12024238,76703609,6157724,31161687,90004325,16567550,23432750,17857111,27411561,68110330,85117092,26734892,4091162,33061250,20836893,36933038,30366150,43506672,14723410,17764950,41092172,40152546,48260151,22113133,70036438,46540998,97641116,20867149,34698463,49328608,34667286,76170907,72357096,7011964,9599614,15902805,32590267,55960386,71083565,7919588,92787493,41092102,5267545,62428472,13348726,44915235,97057187,5073754,82886381,59329511,47361209,94076128,40781854,95581843,75820087,67124282,3233569,16054533,66903004,14349098,17037369,33935899,29401781,24168411,21397057,40027975,62115552,61712234,41481685,45049308,60393039,13819358,3294781,91255408,21070110,88251446,874791,16684829,4978543,71469330,36139942,17146629,91141395,78602717,72278539,67031644,97379790,94539824,37560164,91957544,49598724,75128745,60176618,40686254,48673079,43045786,59614746,7066775,36808486,94595668,77301523,11923835,84904436,30811010,85695762,76540481,95290172,20486294,54014062,36396314,99226875,8634541,95251277,44847298,61928316,2607799,66116458,99729357,81855445,37957788,37501808,67474219,75052463,15148031,55850790,29516592,42237907,33699435,79136082,48675329,99224392,82178706,72358170,17727650,68816413,29466406,21919959,28928175,7229550,82327024,40534591,88130087,68041839,4199704,1936762,82779622,22200378,51507425,38022834,31491938,13468268,72614359,90654836,21533347,27187213,77072625,22405120,33237508,21289531,6819644,83651211,25636669,13862149,99690194,59405277,49271185,19898053,38510840,72238278,43501211,17081350,34493392,42692881,93566986,86798033,44664587,70372191,38256849,38009615,48395186,58749,65275241,24953498,18001617,86242799,36468541,90061527,90310261,96420429,56461322,17894977,5640302,64157906,11757872,83747892 +42073124,4787945,24591705,15535065,52060076,17068582,53666583,99604946,29029316,45794415,80555751,67030811,17016156,5111370,14326617,8913721,92398073,82886381,37183543,94595668,18783702,52293995,93933709,1197320,49597667,112651,35996293,19101477,2917920,5822202,36580610,68816413,22766820,40027975,526217,74137926,61859143,48774913,5970607,46870723,48360198,47090124,8696647,65271999,73168565,15064655,23194618,77898274,32161669,11923835,66045587,13231279,50007421,30395570,23848565,89214616,80251430,61897582,62232211,72278539,16097038,99021067,33201905,86022504,35092039,50806615,98130363,67793644,97783876,16971929,14045383,92353856,12571310,17976208,98462867,44842615,45996863,26392416,55770687,415901,14349098,62115552,168541,53632373,569864,50188404,36808486,97641116,17727650,53842979,85711894,68644627,89078848,77620120,47298834,66611759,70367851,19891772,91957544,85242963,22997281,1022677,61982238,47708850,39171895,2717150,65715134,48893685,64157906,40984766,29994197,77413012,17539625,96318094,4064751,45306070,76703609,43152977,12416768,70596786,4434662,26102057,7919588,56515456,247198,5640302,92033260,91802888,26292919,56424103,77312810,81898046,91664334,96726697,96420429,33431960,53084256,86001008,67281495,97940276,6808825,22942635,72495719,89811711,84840549,30090481,3509435,96735716,38008118,60430369,63506281,2331773,62430984,61623915,55753905,8651647,81855445,26863229,92554120,6153406,62740044,18833224,19376156,82979980,38494874,59501487,51047803,75543508,9603598,16380211,29819940,64098930,82339363,16424199,34497327,76170907,34432810,90004325,15148031,28851716,43322743,9398733,45617087,26744917,58208470,19939935,14626618,70036438,74724075,43501211,30998561,39801351,50567636,71657078,37957788,57359924,10366309,55189057,54232247,23740167,93053405,20486294,33565483,29635537,40686254,68204242,34295794,61263068,57803235,78766358,87160386,30764367,72793444,99658235,3487592,51507425,73031054,66667729,38061439,26114953,11161731,14731700,86242799,6986898,62357986,77300457,34698463,19486173,62490109,41481685,30694952,26734892,38645117,18411915,31126490,83210802,48395186,67451935,32699744,54427233,57248122,45428665,79136082,97281847,27665211,14947650,70372191,26397786,57240218,7011964,80014588,7104732,75229462,72738685,68102437,54058788,71333116,11757872,95290172,47792865,50668599,68875490,59197747,37192445,63152504,90654836,44473167,98943869,91990218,77284619,81677380,19457589,68939068,85023028,26493119,48892201,21070110,5482538,49598724,38341669,83155442,92071990,75135448,62452034,52261574,1204161,45995325,22450468,70004753,94711842,9623492,78785507,30463802,30569392,65081429,13348726,31727379,12348118,93790285,82779622,83302115,874791,92803766,6793819,51466049,11365791,73617245,29188588,36780454,7517032,20836893,88698958,60106217,4069912,73235980,24058273,56473732,7685448,17365188,61728685,508198,1239555,10264691,91831487,2208785,22200378,67031644,66885828,76540481,44844121,99965001,55247455,66903004,7955293,72358170,99861373,54762643,47213183,84904436,95010552,36812683,79922758,4119747,61141874,69641513,84187166,54199160,3088684,54606384,9259676,27411561,78218845,70420215,4099191,38658347,57393458,29932657,19898053,89996536,73124510,21673260,16019925,59910624,68000591,93359396,66428795,51149804,48675329,75820087,9575954,32426752,8807940,92998591,44664587,5832946,55602660,88444207,86798033,17385531,23569917,72019362,88904910,13470059,24826575,30771409,98948034,55960386,37435892,99549373,37675718,50702367,71083565,69579137,44245960,9058407,17957593,92541302,12664567,70727211,43357947,22108665,79880247,86821229,26664538,35192533,38365584,15163258,96709982,39986008,14093520,77694700,81805959,90013093,55143724,37560164,49271185,90457870,26063929,84002370,83083131,95726235,94076128,3233084,79322415,4508700,33304202,31904591,35456853,45667668,84127901,90272749,42967683,9188443,66250369,90061527,31491938,45919976,99297164,4806458,51590803,95395112,176257,1569515,33336148,67124282,64602895,86301513,23110625,63628376,13468268,96852321,15948937,30218878,61960400,69136837,75986488,63372756,98371444,6505939,1375023,7300047,60955663,45407418,33628349,63967300,36312813,16054533,77787724,36396314,77272628,32590267,22879907,40677414,20642888,86460488,72777973,12024238,10453030,59371804,36139942,97011160,75153252,74614639,42644903,59109400,82897371,99125126,57163802,21993752,47361209,89637706,82726008,17146629,31733363,26507214,78589145,96906410,13862149,11581181,53547802,57658654,53648154,44915235,7423788,69355476,65338021,40865610,15902805,49641577,51464002,91255408,28928175,77072625,67084351,99224392,30787683,6157724,22129328,28550822,9829782,53802686,43506672,65454636,36135,8055981,37166608,91938191,93562257,61271144,92692283,22405120,16445503,66322959,99226875,7687278,73392814,93057697,64848072,26998766,64087743,20122224,36753250,85116755,7646095,10309525,17081350,93566986,90310261,4515343,89419466,8128637,37501808,25636669,8634541,84166196,19272365,48673079,66271566,74452589,72725103,36930650,54263880,84293052,46851987,58751351,46540998,94911072,75777973,28787861,33895336,5073754,16595436,1936762,83651211,41380093,80316608,34493392,61859581,70800879,8099994,30139692,79738755,61373987,29466406,1431742,71300104,20645197,69848388,83150534,57961282,99971982,20681921,89046466,22113133,43376279,3880712,61741594,94090109,77187825,56955985,38009615,37620363,54014062,37739481,76825057,78561158,82052050,33553959,89499542,82532312,23432750,44177011,34044787,92215320,39553046,76960303,34667286,34698428,29510992,36933038,3773993,30811010,83789391,71920426,3235882,95581843,88251446,94360702,87710366,39847321,89699445,60102965,39373729,76330843,81853704,17857111,94539824,40534591,17764950,45990383,98648327,55615885,57241521,26139110,47893286,81274566,33123618,18806856,40152546,79821897,30366150,47887470,18663507,18001617,30653863,72732205,76434144,97561745,54663246,69255765,72373496,61712234,52204879,29959549,55470718,72238278,21533347,88653118,81172706,44348328,20765474,74110882,40197395,45237957,44847298,4204661,73222868,29834512,62496012,20140249,34236719,44479073,32250045,70782102,24619760,36189527,49236559,42692881,27185644,62051033,54868730,40781449,29516592,23134715,45049308,44627776,7229550,9860195,94516935,73109997,69786756,9886593,73786944,17037369,78884452,89804152,24733232,98739783,72274002,33249630,27041967,82178706,99524975,62936963,321665,79657802,43045786,3294781,16405341,8373289,64055960,6111563,48658605,9599614,37659250,65851721,68694897,68110330,18131876,92867155,99515901,43933006,10597197,3633375,1250437,72614359,61928316,45075471,41092172,16387575,59329511,62803858,77377183,68316156,67474219,6871053,86543538,79942022,44889423,48260151,4985896,51588015,22721500,27325428,65038678,72357096,79191827,49882705,33061250,87720882,51715482,11543098,2607799,9175338,52613508,17894977,78602717,51472275,99333375,44481640,21001913,79806380,84684495,83533741,41442762,9257405,95957797,8115266,6819644,30891921,21919959,59614746,68824981,3233569,66319530,66832478,65017137,56461322,6525948,49328608,53393358,75052463,4199704,4095116,20885148,15536795,36845587,83747892,14220886,33237508,33797252,29401781,93515664,41092102,37224844,68128438,63015256,15075176,62693428,85695762,28734791,42307449,25842078,8129978,10358899,34946859,44550764,55669657,96193415,4091162,88047921,36685023,29065964,26275734,91727510,8733068,44983451,44846932,75128745,13173644,57020481,10649306,75407347,92692978,45790169,39201414,59981773,89217461,10961421,38510840,85571389,27625735,73021291,80246713,74441448,63059024,97379790,6521313,36505482,26776922,33935899,24168411,40781854,35512853,99917599,71965942,60581278,6497830,69697787,7502255,4668450,824872,41245325,30163921,84406788,91281584,95251277,82427263,30543215,58749,40356877,99729357,45848907,84349107,38022834,6221471,62552524,3183975,16684829,76671482,42782652,91141395,2204165,74357852,58180653,31161687,59318837,78300864,32151165,32058615,99690194,42947632,78909561,98653983,68702632,67513640,65275241,59177718,24953498,58224549,32274392,20002147,62428472,66663942,20568363,71316369,6038457,20867149,17058722,78549759,97057187,48088883,83269727,18466635,13819358,51315460,54517921,61815223,92530431,74743862,4978543,8729146,1788101,81774825,62762857,5267545,4798568,65880522,65047700,82327024,83368048,71469330,71522968,56531125,24314885,83133790,14723410,42199455,12303248,66116458,22435353,2891150,17386996,21397057,24915585,60393039,65074535,91240048,70541760,56484900,69605283,42237907,38256849,90090964,18699206,37891451,45381876,44060493,87598888,86118021,47738185,82651278,749283,7182642,32159704,7066775,55850790,90158785,77301523,67227442,51426311,53888755,59405277,85117092,88130087,16567550,92787493,669105,36468541,16099750,37280276,8791066,68041839,18504197,92604458,44784505,56153345,66137019,28796059,14363867,33699435,54987042,21289531,15015906,9860968,83948335,93270084,76315420,27187213,60176618 +82897371,84187166,55615885,43322743,66832478,53547802,76330843,8807940,92554120,68939068,48360198,168541,8099994,24591705,67513640,75153252,45667668,28851716,4064751,29029316,84293052,37183543,59177718,67451935,92692283,51715482,64157906,62452034,3773993,38061439,68816413,47090124,55247455,48260151,99965001,62496012,12348118,112651,50188404,415901,33201905,82979980,85571389,26744917,4099191,1431742,70420215,98462867,68644627,33336148,27665211,69786756,8913721,45075471,53666583,46870723,92803766,14349098,54606384,70800879,23432750,77377183,15535065,43933006,39801351,3233084,15948937,63372756,99524975,16971929,82532312,34044787,58751351,669105,7955293,53842979,62693428,2717150,26392416,93515664,44889423,34432810,39847321,16445503,94911072,82726008,29466406,176257,6986898,61859143,62051033,77413012,96726697,17386996,40197395,50007421,21919959,23848565,4787945,61928316,8651647,76434144,71316369,30366150,70367851,83651211,24058273,29819940,9257405,66322959,69697787,32699744,18783702,73786944,75986488,874791,44983451,3294781,70541760,14947650,89046466,69848388,85116755,51047803,40984766,23110625,39986008,22108665,91664334,34295794,67084351,64848072,59910624,29834512,52613508,81274566,84840549,35092039,92033260,97379790,68000591,66611759,1197320,92353856,33553959,26863229,7687278,93933709,73168565,72614359,62357986,9599614,67227442,40781449,34698428,19939935,23194618,4119747,66045587,36505482,75777973,10309525,54014062,55753905,569864,91957544,4806458,77312810,63967300,88653118,36580610,99861373,79821897,39171895,62936963,21070110,33249630,38008118,92398073,84904436,89996536,47708850,13348726,23569917,7517032,12571310,9398733,95957797,526217,49271185,53648154,44479073,34497327,16405341,30569392,81805959,70596786,57248122,8696647,26998766,61141874,45407418,76960303,77898274,94360702,36780454,75229462,92541302,89419466,89078848,19898053,73617245,87160386,70036438,91938191,85711894,73124510,80014588,63015256,96193415,19457589,10366309,99658235,66428795,17539625,1569515,16595436,247198,57961282,66319530,31126490,5832946,1022677,44842615,79136082,99515901,65454636,83210802,44481640,68316156,61263068,80251430,93790285,93359396,29635537,73031054,10961421,22405120,26063929,11161731,71333116,27325428,95010552,63059024,35456853,41245325,64602895,89214616,96906410,61815223,32161669,51464002,55770687,43152977,15163258,50668599,82178706,36808486,78766358,54427233,71522968,27411561,60106217,15064655,63506281,56473732,54762643,77072625,54987042,85242963,59371804,99549373,47361209,13862149,69579137,85117092,37620363,83155442,91240048,8115266,49641577,54663246,30764367,76671482,65880522,35192533,84127901,9259676,19272365,68875490,6157724,16387575,19101477,33628349,57240218,50702367,37280276,30771409,17976208,36845587,7423788,75820087,94090109,20765474,57241521,11543098,40534591,68204242,65017137,45617087,14731700,99125126,29065964,57803235,45428665,37957788,61741594,8733068,62232211,58180653,75052463,71469330,77272628,42692881,22997281,68102437,12416768,47792865,18699206,47298834,26776922,36753250,62740044,74441448,26292919,8128637,55189057,38494874,18131876,40677414,20486294,54199160,65038678,65271999,83302115,72373496,22721500,38365584,40356877,50567636,13470059,1375023,56515456,98943869,4668450,47213183,18833224,44915235,36312813,18466635,97561745,56424103,70727211,81853704,1204161,67474219,90310261,14045383,41092102,72274002,77284619,16567550,72358170,60102965,30139692,9058407,17068582,33123618,8634541,99297164,51149804,51466049,7011964,48658605,26139110,30543215,62490109,59318837,86821229,88904910,37501808,88047921,19891772,59614746,43506672,53802686,32058615,35996293,43376279,90654836,17037369,15536795,38022834,83789391,38341669,59109400,62803858,89811711,42947632,84002370,15015906,96735716,33565483,93566986,56461322,33304202,37891451,7300047,36930650,45995325,85023028,28796059,98653983,40865610,15075176,90061527,36396314,45990383,72495719,78785507,33237508,37675718,92787493,97783876,3487592,6153406,61373987,93053405,4798568,42782652,90272749,79191827,87598888,7646095,12664567,33797252,74614639,9829782,28550822,80555751,95726235,36812683,65338021,96318094,83083131,59197747,3880712,67281495,86001008,40027975,42307449,20140249,55602660,5482538,81677380,79922758,65047700,76540481,27041967,24168411,4204661,66667729,91255408,30163921,68694897,82427263,63628376,2917920,48893685,16380211,28787861,11581181,9603598,19486173,48673079,69355476,47738185,321665,83533741,68824981,26493119,96852321,44473167,71300104,88130087,91990218,61897582,6521313,508198,42644903,2204165,3235882,61960400,66663942,25842078,49328608,27625735,9860968,7182642,29994197,31904591,66250369,49597667,68128438,72357096,86022504,30694952,24733232,98739783,99224392,74110882,84349107,22129328,99690194,6819644,94711842,71083565,90004325,4069912,30090481,7502255,30891921,48892201,86460488,90457870,45919976,33431960,76825057,44846932,79806380,9886593,61712234,42073124,65715134,32426752,18504197,84406788,34493392,78561158,56484900,23740167,10649306,77620120,7685448,33895336,99021067,31733363,99729357,45790169,44177011,64098930,84166196,88698958,21993752,99333375,48774913,70004753,4091162,4515343,20642888,17764950,81855445,22766820,20122224,73021291,15148031,66885828,38645117,29188588,37739481,75543508,65074535,78589145,77300457,65851721,60581278,41481685,66271566,13231279,53084256,26734892,17894977,69255765,75407347,24826575,99226875,31491938,5111370,38256849,51507425,29932657,3233569,17058722,44844121,71657078,60430369,54517921,2208785,30811010,36685023,61728685,9623492,24619760,80316608,53632373,54058788,14363867,38009615,76315420,52204879,45237957,63152504,94539824,95581843,17957593,17365188,45794415,48088883,26664538,97641116,41380093,28928175,9188443,11923835,13173644,24314885,55143724,6793819,4095116,29959549,94516935,95290172,62762857,14723410,72278539,86798033,45996863,89637706,23134715,22942635,93057697,30998561,25636669,53888755,89804152,18806856,97281847,26397786,15902805,98130363,7229550,44627776,60176618,65275241,93562257,73392814,46540998,20867149,16019925,58224549,69605283,9175338,29401781,55960386,79942022,6505939,71920426,6808825,77787724,67793644,84684495,39201414,11757872,17727650,20836893,6111563,27185644,7919588,44550764,44784505,22879907,96420429,11365791,72793444,49236559,80246713,17385531,67030811,55669657,9575954,76170907,90090964,78549759,64087743,70372191,16054533,78602717,69641513,67124282,83368048,22435353,36189527,17146629,14326617,20885148,96709982,30787683,24953498,51588015,74724075,3509435,30463802,89699445,45848907,58208470,4199704,78218845,31727379,86118021,1936762,17016156,74137926,45049308,67031644,81774825,18663507,26507214,36468541,5640302,61859581,30653863,4985896,52293995,81172706,14093520,30218878,37192445,43501211,94076128,51472275,10358899,32590267,93270084,62430984,44664587,91802888,14220886,1788101,69136837,56955985,59329511,70782102,72238278,76703609,83269727,92998591,73109997,57359924,40152546,45381876,9860195,5822202,59501487,18411915,10597197,97011160,16424199,5267545,92604458,39553046,19376156,43357947,3088684,31161687,44245960,5970607,2331773,749283,92692978,91727510,88444207,66903004,37560164,51590803,99971982,83133790,58749,78909561,68041839,14626618,40686254,95395112,50806615,74357852,78884452,36135,32151165,61271144,92215320,37435892,7104732,47893286,79880247,89499542,44060493,57020481,97057187,99604946,41442762,72777973,86242799,73235980,87710366,52261574,62428472,74743862,83150534,36139942,32159704,26275734,81898046,91281584,21533347,42199455,48675329,74452589,37166608,17081350,36933038,75128745,16097038,6871053,21289531,6497830,12303248,77694700,79657802,38658347,51426311,32250045,97940276,16099750,66137019,34236719,82651278,87720882,61623915,2607799,52060076,35512853,91831487,6221471,98948034,8373289,71965942,98371444,82779622,8129978,72725103,33935899,82052050,62552524,98648327,54263880,99917599,55850790,56531125,20681921,45306070,13819358,83948335,75135448,16684829,39373729,53393358,44348328,30395570,47887470,10453030,59405277,8055981,33061250,24915585,18001617,66116458,49598724,37224844,20568363,86301513,51315460,17857111,37659250,27187213,85695762,79322415,78300864,61982238,21673260,1239555,34946859,92867155,59981773,82886381,13468268,12024238,20002147,46851987,68110330,42237907,54232247,68702632,34667286,44847298,4978543,22450468,88251446,83747892,29516592,62115552,824872,90013093,4434662,6038457,3633375,72738685,28734791,77301523,6525948,43045786,32274392,26102057,10264691,73222868,20645197,1250437,77187825,72732205,57658654,92071990,56153345,34698463,65081429,8791066,42967683,60955663,5073754,82327024,57163802,7066775,55470718,3183975,64055960,38510840,22200378,26114953,8729146,89217461,4508700,21001913,48395186,54868730,94595668,40781854,90158785,86543538,79738755,2891150,72019362,82339363,21397057,49882705,33699435,95251277,22113133,91141395,41092172,29510992,60393039,92530431,57393458 +99515901,96318094,48893685,98462867,68204242,4119747,68644627,47090124,12348118,18699206,85116755,4787945,62496012,65338021,55189057,53842979,34497327,8807940,33797252,55770687,15535065,61623915,68816413,44842615,96709982,63628376,96193415,48360198,83210802,14093520,99524975,9623492,26664538,30366150,33249630,20486294,35996293,95957797,43933006,98371444,7011964,24058273,72274002,6153406,80555751,99965001,81855445,29834512,43501211,45617087,71469330,69355476,80251430,84349107,66137019,5970607,415901,80246713,4515343,6871053,92554120,11581181,16054533,65275241,15064655,20122224,45237957,82339363,35092039,42967683,99297164,27411561,82178706,37183543,49236559,34698428,20836893,65017137,86001008,34946859,56424103,7502255,26139110,55143724,68000591,42947632,4099191,59318837,51466049,74724075,50188404,77413012,42782652,32699744,34295794,26063929,48658605,98130363,55669657,76330843,69136837,47361209,10366309,78561158,73222868,27665211,77787724,14220886,31904591,76434144,67031644,56473732,71657078,73392814,14947650,61982238,94090109,72373496,96906410,64055960,59371804,86543538,62936963,40984766,14349098,70004753,78785507,44481640,19101477,29959549,63506281,7300047,10649306,22997281,22108665,18663507,16405341,38658347,65715134,56153345,17068582,21070110,91938191,10961421,28550822,38365584,88047921,18783702,4806458,96726697,40781449,51472275,66250369,36753250,18131876,33123618,6793819,24591705,69605283,21993752,42237907,14045383,68128438,92033260,30395570,91664334,75543508,43152977,22129328,71083565,61815223,61263068,77377183,58751351,10309525,34493392,65081429,569864,33699435,7423788,53084256,97783876,47708850,67793644,78589145,44847298,68939068,9188443,40534591,40865610,3294781,54606384,7517032,43322743,10597197,62762857,14363867,29635537,44664587,85242963,93566986,80316608,44060493,30694952,29994197,84840549,84187166,78884452,70420215,94595668,33336148,61859143,16019925,95726235,52613508,59109400,10358899,83789391,33553959,93933709,50567636,19898053,37620363,37224844,63372756,35192533,45790169,51047803,20140249,7104732,24733232,97641116,77898274,30764367,2204165,23569917,87598888,53547802,66322959,86460488,68102437,9257405,60581278,16567550,61960400,44844121,96735716,20765474,60430369,3233569,91727510,3880712,69579137,65047700,45667668,9398733,1197320,508198,8634541,36505482,44479073,41481685,26397786,73617245,56531125,71333116,90090964,67030811,13862149,91990218,52060076,83302115,12416768,24619760,9259676,66663942,45428665,50007421,77312810,18833224,75777973,64848072,57240218,17764950,40197395,45919976,27325428,18504197,73031054,44177011,62740044,63967300,84904436,41092102,58749,84684495,21289531,92398073,54868730,1431742,57359924,6521313,54427233,5482538,92787493,33628349,99690194,32274392,14326617,38494874,99549373,66611759,14626618,26392416,67281495,33304202,43506672,71300104,90457870,51464002,33237508,21919959,14723410,66428795,22879907,70541760,95395112,81898046,42073124,47298834,32426752,64098930,5640302,66667729,76825057,321665,66116458,44889423,55850790,38008118,51715482,1204161,99125126,8099994,29065964,67124282,99861373,48774913,36685023,44784505,20002147,86242799,29819940,44983451,60102965,91831487,88130087,77620120,55247455,39171895,75052463,88251446,49328608,57803235,66885828,72738685,6986898,93359396,89419466,3633375,49641577,1788101,62232211,36580610,15148031,3183975,24168411,13468268,98739783,669105,83155442,29029316,59177718,68316156,74137926,17957593,82979980,7685448,62803858,73124510,21001913,22721500,7919588,168541,26776922,47792865,94516935,69848388,30139692,74614639,40152546,94360702,15015906,65074535,39847321,55753905,79821897,30891921,79738755,30787683,45407418,36468541,76540481,4091162,67451935,15163258,5111370,26863229,19891772,30543215,3233084,41092172,92353856,82427263,53666583,14731700,70800879,39801351,33201905,87160386,26275734,62490109,52204879,54014062,19486173,89811711,91802888,33431960,83150534,90272749,26507214,63059024,44550764,86022504,13348726,18806856,8696647,88698958,38061439,13173644,30569392,66832478,89996536,20681921,23432750,91240048,25842078,13470059,78766358,50702367,8115266,20642888,62693428,70372191,82726008,58208470,46851987,79942022,37957788,31126490,64087743,32250045,77694700,3509435,11365791,824872,99021067,95290172,90013093,97561745,6505939,61928316,88653118,11923835,79191827,76170907,23134715,98943869,16380211,95581843,4064751,11543098,83533741,89214616,874791,93057697,5267545,97379790,52261574,45306070,64157906,69786756,18466635,17146629,28787861,4204661,84002370,89804152,24953498,57248122,16971929,62552524,27041967,67227442,80014588,59614746,69641513,65038678,29466406,92541302,72495719,56515456,3088684,78602717,21673260,63015256,68824981,16595436,7646095,8651647,66319530,30771409,24826575,61373987,93053405,93562257,39553046,44627776,1375023,55960386,70367851,62357986,5822202,67084351,2331773,86798033,247198,81274566,95010552,50668599,44473167,15902805,21533347,43376279,79806380,78218845,82897371,70727211,70036438,16097038,59405277,48260151,29510992,9829782,74441448,65851721,29188588,96852321,48088883,23848565,39373729,77072625,29516592,99333375,40356877,40677414,28928175,59197747,48892201,26114953,72777973,37166608,22405120,8055981,72793444,93790285,176257,47893286,92692283,83651211,50806615,2891150,28734791,35456853,41380093,79922758,6808825,36780454,17386996,77272628,6111563,55470718,81805959,36933038,54987042,55615885,66903004,61897582,85571389,39201414,11161731,59329511,54199160,37659250,45075471,89078848,85711894,8373289,57163802,1022677,19939935,45990383,61141874,66271566,81172706,68041839,40686254,30090481,46540998,32161669,55602660,28796059,29932657,32590267,10264691,7955293,48673079,45996863,37501808,60106217,99658235,47213183,90310261,1936762,4668450,62430984,36312813,72732205,2917920,2717150,30218878,8128637,25636669,9860968,15948937,71522968,68875490,3487592,84293052,75135448,12571310,9058407,83269727,22942635,77284619,60393039,98653983,526217,5073754,92692978,17539625,85695762,81853704,63152504,34698463,85117092,45995325,31733363,39986008,91255408,70596786,93270084,32159704,73235980,99917599,65271999,112651,87720882,8129978,59501487,36930650,40781854,74743862,3773993,38645117,26292919,54263880,17385531,53393358,36396314,69255765,75153252,8729146,66045587,86118021,57393458,89637706,82886381,73021291,73109997,58224549,34236719,17976208,77300457,20867149,8733068,87710366,4069912,13819358,71920426,74110882,51588015,34432810,73168565,92803766,23194618,2208785,62452034,37675718,78909561,72614359,60176618,57961282,72358170,86301513,56461322,51590803,64602895,36808486,94711842,82651278,51426311,9175338,31727379,73786944,36139942,98648327,92071990,49271185,17365188,7229550,61859581,22200378,83948335,19272365,30811010,8913721,9575954,17727650,62428472,6157724,44245960,57241521,27185644,16424199,67513640,9860195,30998561,12664567,83368048,57658654,36812683,42692881,82327024,89046466,84127901,54663246,7687278,9886593,72725103,97011160,82532312,23110625,35512853,72019362,38510840,99604946,9599614,91957544,54517921,83133790,99729357,97057187,37739481,86821229,91281584,68694897,4095116,88444207,75128745,26744917,61712234,31161687,51149804,41245325,40027975,49597667,79657802,4434662,38009615,37192445,79880247,99226875,36189527,42307449,52293995,96420429,37560164,6038457,33895336,45848907,75820087,9603598,7182642,30653863,53888755,17894977,6525948,19376156,48395186,54058788,20885148,97281847,37891451,44846932,22766820,79136082,47738185,10453030,88904910,90061527,78549759,16445503,89499542,46870723,26102057,84166196,53648154,69697787,4199704,99971982,23740167,85023028,33935899,1569515,43045786,16684829,17037369,42199455,17857111,72357096,47887470,31491938,38022834,4978543,58180653,81677380,28851716,13231279,7066775,84406788,75229462,4798568,6819644,99224392,71965942,30163921,44348328,749283,89699445,90004325,93515664,22450468,92998591,4508700,24314885,65880522,94911072,59981773,76703609,12303248,15536795,51315460,6497830,5832946,27625735,17016156,37280276,92530431,18001617,19457589,59910624,33565483,61728685,38256849,32058615,82052050,41442762,20645197,76315420,22113133,61741594,76671482,60955663,62051033,48675329,38341669,94539824,72278539,91141395,32151165,26998766,72238278,54762643,26734892,15075176,4985896,6221471,1250437,98948034,34667286,1239555,70782102,3235882,65454636,92215320,76960303,53632373,97940276,92604458,16099750,94076128,12024238,42644903,75986488,62115552,68702632,30463802,89217461,17058722,36135,75407347,56955985,45794415,29401781,79322415,44915235,49598724,49882705,77301523,22435353,53802686,71316369,95251277,61271144,74357852,24915585,92867155,45049308,81774825,51507425,90654836,36845587,45381876,17081350,83083131,43357947,8791066,83747892,27187213,20568363,54232247,33061250,67474219,57020481,2607799,11757872,90158785,16387575,74452589,26493119,34044787,21397057,18411915,56484900,82779622,77187825,37435892,68110330,78300864 +70596786,9860195,37620363,76434144,67793644,1431742,78218845,55770687,14093520,80014588,26114953,36505482,21070110,1204161,15148031,70004753,57803235,37183543,29959549,99549373,52293995,15015906,5111370,53084256,49236559,112651,75407347,9259676,47213183,76315420,29188588,79922758,1250437,68644627,31904591,75052463,54427233,44983451,1788101,92554120,66832478,87720882,61859581,74357852,72738685,52261574,70800879,27665211,34497327,99658235,89214616,65047700,75153252,3633375,26102057,12416768,91664334,38494874,55470718,91802888,18131876,92692978,59614746,95726235,60430369,11365791,28550822,20486294,66885828,39171895,30139692,42967683,4798568,97783876,1022677,10309525,30543215,62428472,62740044,4091162,43045786,87160386,77620120,15535065,20002147,32161669,38658347,74137926,45919976,26493119,68128438,16595436,35192533,77187825,86821229,42073124,31126490,8696647,54868730,50806615,749283,35092039,36189527,15536795,76330843,1569515,27185644,22405120,9886593,9623492,44784505,83155442,77312810,13470059,7104732,39801351,83368048,18806856,39986008,16380211,3880712,43933006,13231279,43152977,44889423,72278539,96726697,80246713,66319530,50702367,88653118,33237508,76170907,82779622,61373987,13348726,80316608,62936963,168541,62496012,874791,44550764,62490109,4099191,40152546,4204661,68204242,415901,93933709,62452034,23740167,61263068,21533347,68041839,44842615,12348118,29819940,8128637,79880247,14731700,10366309,67281495,79821897,4064751,88047921,90013093,66045587,50188404,18783702,12571310,40865610,34946859,38341669,96193415,9398733,68102437,88904910,72358170,55247455,48260151,63015256,33431960,68939068,7687278,55189057,83083131,19101477,78549759,70372191,57359924,27625735,30653863,42782652,7955293,36396314,526217,23110625,69786756,14947650,96709982,4515343,37739481,62762857,48360198,59501487,72357096,24953498,17058722,76703609,85711894,51590803,29994197,73168565,34432810,82427263,98739783,44915235,39553046,33628349,81853704,93562257,94516935,67451935,94539824,20122224,9058407,17957593,26275734,43506672,6871053,247198,68316156,30366150,97057187,71333116,38510840,30764367,24591705,34698428,37435892,33304202,84349107,32274392,8634541,33249630,85571389,36812683,69355476,76960303,26998766,19939935,23569917,37560164,75229462,99965001,17857111,3233084,29029316,61960400,9599614,72274002,71316369,60106217,37675718,29932657,78884452,8913721,40534591,79136082,40197395,97281847,65715134,95957797,85242963,7646095,18699206,55602660,4119747,65074535,64087743,78589145,32250045,14723410,61741594,19457589,76671482,6221471,78561158,81677380,55753905,3294781,58224549,3233569,86242799,75986488,26139110,16019925,98130363,54199160,16405341,17365188,62430984,66903004,5482538,54987042,48673079,66667729,65454636,81172706,37192445,20836893,66428795,57241521,4787945,99224392,53547802,30998561,5640302,2204165,77301523,2208785,90310261,99861373,70541760,54263880,48892201,45428665,36580610,92998591,84187166,15064655,51047803,98462867,44481640,42307449,1197320,73031054,45996863,98648327,40027975,9603598,34698463,93053405,19376156,40677414,77284619,58749,40984766,92692283,89699445,31733363,55669657,60955663,23134715,44847298,41442762,42199455,77272628,89419466,34295794,38008118,37224844,6153406,65338021,44177011,66250369,73786944,59197747,77413012,23194618,64848072,33123618,95010552,9575954,84002370,33797252,57658654,57163802,98943869,85116755,43322743,68816413,4434662,46851987,36312813,81898046,17037369,11923835,37501808,9829782,77072625,86001008,29834512,26392416,22129328,16971929,99524975,22450468,44627776,56424103,78602717,97940276,8791066,6521313,53666583,40781449,87598888,46870723,10597197,19272365,43376279,92071990,48658605,59318837,44060493,6986898,62803858,98948034,86543538,78785507,25636669,36930650,98653983,73124510,65038678,48893685,83302115,7919588,73109997,89046466,99690194,2891150,21993752,38645117,68824981,9175338,47090124,40781854,669105,78766358,4199704,73617245,47708850,6525948,4668450,10961421,30163921,26776922,42644903,20867149,83210802,23432750,56515456,14220886,52613508,30787683,99297164,73222868,49598724,94595668,92604458,65275241,72777973,3235882,36139942,22435353,15075176,64602895,26507214,14326617,95251277,81774825,91831487,21673260,39847321,51315460,9257405,51507425,99333375,92215320,82897371,92867155,33895336,321665,46540998,569864,50668599,74441448,86798033,89811711,96735716,76540481,44348328,63967300,8099994,40356877,95581843,88698958,44846932,24058273,71522968,62232211,8055981,56473732,27325428,75543508,82532312,33699435,27041967,25842078,2331773,1375023,81805959,19486173,19898053,26664538,93790285,5970607,89637706,22721500,6793819,35456853,24826575,22879907,32159704,83789391,68000591,37891451,72732205,44473167,57240218,99604946,30218878,26734892,17727650,84406788,98371444,77694700,93057697,55615885,508198,14626618,97641116,45848907,62693428,74614639,51426311,97379790,92803766,53888755,41092172,77787724,28787861,80555751,176257,20140249,23848565,7300047,51149804,34236719,26863229,16099750,56484900,84904436,39373729,62552524,18001617,59109400,3183975,84840549,75135448,71469330,84127901,62115552,78300864,5267545,6111563,42947632,30891921,13468268,26744917,93270084,79657802,8651647,16684829,93359396,67513640,4806458,20885148,38061439,28796059,82726008,24915585,96318094,36933038,55143724,8115266,20765474,36135,26063929,17539625,12024238,33553959,32590267,16097038,71083565,90090964,26397786,63059024,69255765,61982238,73392814,61728685,33935899,45790169,73021291,26292919,4069912,36780454,72614359,8129978,99917599,91938191,7229550,24733232,6808825,90272749,59177718,22200378,30395570,42692881,77377183,29466406,2917920,22766820,28734791,34493392,45995325,67031644,70727211,53648154,83150534,60581278,74743862,9188443,72793444,32699744,85695762,69848388,72495719,51472275,18411915,49882705,85023028,18663507,52204879,36753250,43501211,91957544,45990383,19891772,27411561,40686254,47792865,99515901,51715482,73235980,61141874,16424199,96906410,4978543,6038457,50007421,16387575,61859143,60102965,70420215,7502255,91727510,88444207,1936762,86022504,61928316,75777973,47887470,61623915,37957788,54663246,94911072,45049308,8807940,99125126,81274566,58208470,27187213,55960386,99729357,75128745,94360702,65851721,7423788,77300457,69641513,67124282,92398073,66116458,99021067,94076128,63152504,91240048,28851716,63372756,36685023,97561745,79322415,82339363,88251446,56153345,49641577,15902805,11543098,37280276,67474219,66663942,66611759,12664567,65017137,48675329,80251430,68875490,90457870,20681921,16445503,17386996,54517921,83133790,2607799,79191827,92787493,7011964,2717150,90061527,59910624,14363867,49271185,12303248,99226875,79738755,33565483,3487592,53842979,45667668,17385531,45794415,86118021,18504197,57020481,35996293,58751351,5832946,30463802,71920426,92033260,83269727,17146629,95395112,32151165,29635537,71300104,24168411,22108665,64098930,86301513,54014062,43357947,31161687,72725103,63628376,20642888,47893286,66322959,37166608,7517032,1239555,94090109,30694952,79942022,48774913,37659250,9860968,10649306,61712234,8733068,89996536,21289531,99971982,66271566,66137019,60176618,8373289,70782102,50567636,49328608,82651278,64157906,89499542,34044787,24619760,91990218,48088883,3509435,36845587,83747892,44245960,81855445,13173644,17764950,57961282,44664587,6505939,5822202,53632373,47361209,62357986,83533741,53802686,65271999,61815223,51466049,71657078,41245325,30569392,85117092,62051033,61897582,6497830,82886381,70367851,58180653,32058615,71965942,39201414,91255408,41092102,4985896,61271144,41481685,64055960,21001913,17081350,88130087,83651211,30771409,59329511,17976208,97011160,38009615,33336148,65880522,82979980,13862149,55850790,56461322,69136837,72373496,44479073,29510992,21397057,69605283,93515664,70036438,5073754,16054533,89804152,17894977,15948937,87710366,89217461,72019362,84293052,4508700,82327024,41380093,38022834,68694897,93566986,54762643,18833224,57248122,32426752,67030811,54058788,38365584,52060076,51588015,53393358,92541302,68702632,22997281,59981773,63506281,17016156,45237957,48395186,90004325,59405277,74452589,67084351,11757872,91281584,34667286,68110330,28928175,45075471,20645197,29516592,14349098,96852321,45617087,91141395,69579137,10358899,7685448,49597667,47738185,3773993,74724075,59371804,21919959,33201905,42237907,69697787,47298834,10453030,10264691,57393458,92530431,11161731,7066775,96420429,35512853,84684495,95290172,33061250,45306070,51464002,82052050,54606384,824872,65081429,72238278,22942635,14045383,16567550,22113133,3088684,29065964,45407418,11581181,76825057,31491938,44844121,86460488,31727379,7182642,94711842,79806380,75820087,56531125,29401781,82178706,20568363,15163258,6819644,78909561,90654836,36468541,90158785,74110882,18466635,83948335,89078848,45381876,30811010,17068582,54232247,92353856,38256849,67227442,24314885,8729146,56955985,36808486,84166196,60393039,77898274,6157724,4095116,30090481,13819358 +4668450,49236559,9623492,31904591,29029316,77272628,6038457,33304202,91664334,91802888,29188588,47213183,41481685,64848072,44177011,30139692,66832478,59329511,91727510,34698428,66667729,73031054,48892201,37620363,69136837,19939935,37183543,36312813,66611759,77620120,5970607,36580610,96726697,93053405,23569917,22435353,96709982,4099191,15535065,66250369,75407347,35192533,61271144,2717150,76434144,68000591,60430369,83083131,7646095,61263068,29994197,39986008,69355476,38658347,10358899,79922758,48774913,30787683,7066775,28550822,91990218,8651647,51464002,43322743,92692283,37891451,62430984,93057697,8129978,60102965,84684495,60581278,47893286,31161687,4515343,8055981,40197395,80246713,70004753,44844121,6871053,38645117,7685448,1197320,53393358,50188404,57803235,52204879,24058273,26998766,95957797,96193415,22450468,47090124,36753250,67474219,16971929,37501808,29819940,8115266,62452034,57658654,6521313,16019925,22129328,32161669,66885828,34295794,9257405,59614746,63152504,30653863,17068582,73168565,33249630,61741594,81172706,52613508,55602660,42644903,12303248,51047803,44550764,26392416,8373289,76315420,35092039,526217,4787945,34497327,64087743,70596786,1431742,76703609,71333116,17146629,51472275,9860195,1239555,29834512,61373987,11923835,45794415,16099750,65880522,31126490,61623915,99965001,11365791,74137926,88251446,77187825,53842979,7229550,77300457,12664567,29932657,86022504,40781449,55850790,42692881,6808825,53802686,68204242,63967300,71657078,44348328,85711894,78549759,55753905,9575954,9860968,64602895,62496012,73222868,53084256,99549373,24591705,86460488,9188443,20642888,14220886,43376279,8807940,76960303,60106217,59371804,6525948,74441448,48673079,66319530,56153345,36780454,96735716,824872,22405120,54427233,74357852,77284619,98948034,90272749,73124510,48360198,65715134,93790285,84406788,39847321,72274002,14626618,44889423,92554120,71083565,55470718,61141874,13819358,98739783,39553046,62740044,77072625,4119747,52293995,45617087,37659250,43045786,56424103,38494874,4064751,569864,46870723,80555751,30163921,59501487,20645197,71965942,83150534,17727650,19486173,75153252,4434662,77694700,67281495,54058788,7687278,91957544,61712234,98130363,49328608,61982238,76671482,91938191,19457589,54987042,19101477,37675718,40356877,43357947,84349107,8128637,112651,92604458,62693428,2208785,99604946,30764367,33565483,22997281,33237508,89811711,1022677,20885148,75229462,66663942,12416768,67793644,92353856,21070110,50567636,98648327,74724075,82339363,40984766,55143724,14723410,3773993,16424199,30771409,8099994,16097038,7011964,91831487,51715482,78785507,49641577,59981773,32250045,93270084,22766820,68128438,13470059,40534591,54232247,83133790,72777973,36468541,75128745,8696647,71920426,30694952,10309525,32274392,13862149,98943869,3294781,77413012,19376156,54014062,50702367,32699744,49598724,89996536,47708850,72373496,70367851,7300047,26734892,50668599,7919588,40781854,38061439,77787724,2607799,58180653,67451935,9259676,73392814,39373729,17081350,98462867,25636669,72278539,30998561,56461322,79657802,1788101,33895336,247198,4806458,6986898,20836893,45996863,415901,80316608,29510992,83210802,71469330,39201414,30569392,90013093,90090964,78218845,176257,60393039,38510840,65038678,68702632,57241521,27665211,67084351,82886381,23848565,2204165,22108665,79821897,65275241,87710366,49882705,48088883,82651278,42782652,96420429,66271566,89214616,44664587,26863229,93933709,64055960,28928175,92033260,37739481,69641513,73235980,24168411,76170907,79136082,59177718,72725103,23432750,54663246,66903004,44842615,45848907,36139942,49597667,29466406,61859143,62115552,19898053,61859581,51315460,81677380,33431960,62936963,55189057,84166196,29959549,57248122,56484900,33123618,26744917,14363867,40027975,72793444,88047921,16595436,1569515,32590267,86821229,30218878,88653118,2917920,37560164,13231279,81853704,70800879,54263880,74743862,11161731,55770687,57020481,53632373,99524975,54199160,36505482,58751351,27185644,68875490,60176618,90310261,92867155,30463802,95010552,30366150,64157906,15148031,94911072,84293052,26507214,24915585,99515901,98371444,92071990,5832946,68316156,89499542,33201905,57163802,15948937,92692978,10649306,32159704,669105,28851716,9398733,99971982,51590803,18504197,18131876,75543508,58749,4091162,25842078,28796059,168541,54606384,8733068,5111370,79880247,41245325,75820087,15064655,20765474,23134715,17386996,47361209,32426752,82779622,14045383,84187166,44915235,75052463,44481640,20486294,48675329,83302115,53547802,92398073,36930650,6221471,97281847,88698958,87598888,34236719,8791066,17016156,6505939,35996293,41092102,40152546,83789391,20122224,39171895,97011160,83269727,97057187,22113133,89699445,67227442,59318837,39801351,16380211,10264691,82726008,70372191,54517921,70727211,42947632,17385531,27411561,14731700,77312810,23740167,4204661,51466049,37192445,75777973,38341669,99333375,9599614,4069912,6793819,99224392,63628376,72357096,67030811,7955293,18663507,41092172,3633375,1375023,34698463,63372756,20002147,1250437,64098930,508198,8634541,69848388,99917599,23194618,51588015,43501211,87160386,90654836,37435892,74452589,68939068,8729146,36812683,43506672,17058722,45381876,15163258,30543215,36808486,65271999,82979980,36396314,68644627,45075471,89419466,6157724,3088684,94090109,42199455,63059024,69255765,97379790,26114953,95726235,44473167,11543098,47887470,56515456,83651211,33553959,68102437,97783876,36933038,15536795,42073124,3233084,91240048,9886593,53666583,65047700,96906410,52060076,34493392,84002370,18806856,21919959,4199704,30891921,79806380,50806615,76825057,66116458,85117092,46851987,93359396,61728685,34432810,44479073,44846932,22721500,69786756,34667286,85116755,63015256,55960386,38365584,14093520,40865610,24733232,78561158,17037369,31733363,57393458,20867149,5822202,14349098,10961421,62428472,73021291,26776922,75135448,92215320,94711842,66428795,81855445,27625735,62552524,99658235,65074535,749283,72732205,62051033,13468268,16684829,65454636,43152977,65338021,54762643,77898274,35512853,26275734,31727379,22879907,77301523,27041967,70541760,78766358,1204161,78300864,33628349,9603598,47738185,98653983,33797252,38008118,72614359,81898046,59109400,6497830,42967683,95395112,21673260,21001913,17957593,44245960,96852321,30395570,28787861,74110882,15015906,19272365,30090481,69697787,90457870,70036438,4978543,72738685,17764950,18833224,15075176,65017137,47298834,71522968,32058615,83948335,71300104,17894977,12571310,40686254,38022834,78884452,44983451,72238278,8913721,88130087,57359924,61815223,66322959,3509435,60955663,89804152,94516935,20140249,5482538,86118021,73617245,16445503,3487592,34946859,79942022,3880712,4508700,26664538,78909561,44627776,86242799,6111563,29516592,92803766,84127901,89078848,16054533,86543538,68824981,20681921,21993752,27325428,11581181,44784505,3183975,24826575,14326617,48260151,78589145,42307449,73786944,26139110,45990383,57961282,86301513,27187213,10366309,62762857,94076128,44847298,83533741,55669657,49271185,56531125,82178706,41442762,96318094,53648154,95290172,21533347,66045587,18783702,4095116,55247455,62490109,82427263,28734791,72358170,70420215,45919976,10597197,36685023,45428665,99861373,85023028,65081429,36189527,99226875,48893685,68694897,80251430,2891150,45237957,90061527,84840549,29065964,17365188,93515664,7423788,18001617,22200378,72019362,32151165,42237907,82052050,82532312,93562257,59197747,87720882,46540998,99021067,56955985,45790169,88444207,85571389,3233569,48395186,22942635,85695762,4798568,26063929,83368048,89637706,93566986,9829782,24619760,70782102,14947650,45306070,47792865,7104732,15902805,29401781,63506281,9058407,9175338,34044787,95581843,67031644,26102057,66137019,12348118,50007421,37224844,16405341,321665,99690194,99125126,54868730,56473732,77377183,5640302,61960400,89046466,89217461,29635537,85242963,68816413,90004325,58224549,6153406,17857111,26397786,7517032,11757872,35456853,91141395,65851721,79191827,45049308,81805959,91255408,18699206,83155442,44060493,16387575,69605283,68041839,86001008,81274566,18466635,94539824,23110625,6819644,17539625,43933006,71316369,21289531,5073754,45407418,88904910,33699435,78602717,73109997,51149804,94360702,38256849,92787493,84904436,21397057,55615885,5267545,36845587,51507425,80014588,4985896,59405277,16567550,24314885,62232211,1936762,57240218,17976208,52261574,99297164,48658605,30811010,12024238,97561745,37166608,7182642,75986488,45995325,13348726,97641116,40677414,58208470,79738755,62803858,33061250,92998591,97940276,18411915,37957788,92541302,51426311,37280276,2331773,26493119,82897371,24953498,62357986,67124282,94595668,10453030,69579137,7502255,53888755,90158785,91281584,67513640,874791,61897582,45667668,33336148,41380093,72495719,68110330,76330843,26292919,82327024,20568363,99729357,81774825,19891772,61928316,86798033,74614639,76540481,95251277,36135,33935899,83747892,13173644,3235882,59910624,92530431,79322415,38009615,31491938 +33431960,1788101,27665211,17857111,86022504,99604946,74357852,55143724,95726235,247198,38494874,42967683,56424103,50188404,14093520,97783876,30139692,55753905,74137926,34698428,93270084,36812683,70372191,73222868,5822202,23848565,98739783,45919976,18806856,67281495,85023028,5111370,84406788,15535065,49236559,61373987,62496012,35092039,18663507,22108665,39986008,71920426,19939935,26998766,95581843,52261574,68644627,48360198,89637706,5970607,1569515,1022677,29188588,45990383,37739481,34236719,25636669,91255408,1431742,28550822,81853704,44889423,68041839,32590267,17146629,96193415,16019925,77620120,5073754,5832946,74441448,4119747,43933006,58224549,2891150,83155442,40781854,39171895,83150534,9599614,55470718,71333116,67124282,63967300,19376156,88653118,50702367,6871053,86821229,80555751,72725103,74110882,37891451,26063929,91727510,3633375,88047921,57961282,15148031,75135448,14731700,29994197,36312813,83368048,82979980,71316369,55602660,82726008,18504197,43357947,37675718,65715134,415901,8733068,65038678,66663942,17058722,16424199,68204242,31904591,99125126,77284619,85116755,83210802,29065964,60581278,63152504,29932657,75407347,93057697,27625735,91802888,93933709,73031054,69355476,44177011,47090124,69848388,9188443,22435353,99658235,76434144,36505482,62552524,33553959,8634541,3880712,4668450,96726697,35456853,30543215,11365791,10366309,14220886,7104732,80246713,13470059,30653863,72738685,24591705,49328608,89699445,26493119,7685448,46851987,95395112,68000591,6038457,53393358,4434662,61897582,42782652,62452034,4787945,40984766,9603598,44348328,30787683,48892201,2204165,569864,63059024,24733232,17365188,9829782,6793819,4199704,88698958,67451935,12416768,32426752,36580610,21993752,9886593,37435892,75543508,33237508,44481640,20486294,86301513,77301523,66903004,62936963,57393458,54014062,44842615,84904436,78218845,22450468,81898046,66611759,62762857,83269727,4099191,87598888,77072625,26776922,89811711,17539625,62740044,63506281,508198,78589145,44550764,112651,68816413,42199455,82327024,55770687,8651647,48673079,29510992,51507425,78549759,16971929,54232247,75986488,669105,98371444,27185644,89996536,30218878,23134715,67793644,7502255,44784505,57359924,45848907,72777973,36685023,33304202,78602717,6221471,48774913,4798568,52293995,92998591,87160386,53084256,99333375,66428795,90272749,24058273,3235882,20885148,1197320,88444207,79806380,29819940,7423788,50806615,51047803,56153345,4204661,40027975,6153406,81172706,13231279,86798033,50668599,59614746,84840549,77187825,61928316,40356877,76170907,50567636,3233084,66885828,95251277,68875490,43152977,40197395,65081429,14045383,15075176,45306070,26392416,45996863,38008118,99861373,92071990,66319530,46540998,65851721,98948034,63628376,77413012,44060493,61141874,70800879,57163802,31733363,32159704,48260151,36930650,73124510,18131876,16387575,35996293,29834512,51472275,72274002,20002147,60430369,44473167,3088684,59329511,45237957,62430984,92604458,46870723,9623492,53888755,37183543,7687278,62803858,64848072,8055981,33895336,60393039,34497327,70596786,97281847,52060076,54517921,13173644,65017137,59318837,7229550,89046466,874791,19898053,36189527,34432810,59910624,77787724,48658605,79922758,38341669,89217461,24619760,2717150,90457870,99549373,47361209,92530431,97641116,12348118,11161731,96318094,28851716,41092172,8115266,42947632,62232211,36753250,96852321,93053405,78561158,38256849,26114953,29466406,43501211,54199160,20867149,53666583,6819644,56955985,65271999,31161687,61263068,75052463,80316608,56473732,87720882,69255765,70004753,6525948,61623915,8807940,92398073,9175338,44847298,34493392,81855445,33565483,95957797,99021067,91664334,168541,94711842,71657078,93359396,12024238,61960400,94090109,93790285,83789391,67030811,20681921,6808825,92554120,54987042,82886381,61859143,78884452,44245960,28928175,76825057,91990218,65275241,71083565,72614359,71300104,93562257,12571310,66116458,47213183,49597667,99224392,19486173,37166608,74724075,6505939,14326617,20836893,61741594,19101477,82052050,18699206,18783702,83533741,14723410,1239555,43322743,4095116,55669657,72732205,8791066,57020481,6986898,30366150,49641577,59197747,89214616,45995325,87710366,61815223,38061439,99965001,56484900,84127901,3233569,95290172,72357096,59501487,91831487,28734791,73235980,68702632,17764950,4069912,10649306,15064655,41092102,2917920,17727650,17957593,41481685,4978543,61271144,12664567,21001913,3773993,53547802,34698463,14626618,27187213,27325428,90013093,67227442,38365584,10961421,82178706,65880522,17037369,45407418,30569392,80014588,13819358,65454636,26102057,9575954,22129328,69136837,21919959,29029316,17016156,89419466,26863229,77300457,93515664,40677414,7300047,8128637,29959549,77272628,66271566,27041967,4806458,81274566,79880247,42692881,76671482,71965942,33699435,99690194,90310261,84166196,76315420,37192445,51590803,36468541,40686254,38645117,98653983,33249630,53648154,22721500,90090964,79191827,35192533,76703609,78785507,5482538,59177718,8696647,70036438,55247455,90004325,75777973,96709982,76540481,73617245,824872,72373496,37659250,30998561,56515456,72495719,72019362,66322959,79821897,84293052,82532312,47887470,38658347,31491938,86001008,3294781,53842979,32699744,40152546,69786756,64602895,30764367,54762643,17894977,78300864,85117092,45617087,8099994,57248122,23569917,47893286,40781449,94516935,4985896,69579137,99917599,83083131,43045786,44844121,3509435,16380211,54058788,4515343,56531125,99971982,92867155,84349107,44627776,24826575,1250437,83302115,78766358,66045587,77377183,83747892,59405277,16097038,89078848,98462867,23194618,9259676,68316156,15015906,99729357,71522968,32250045,31126490,62490109,37620363,43506672,8129978,66667729,321665,83133790,749283,16099750,64087743,21070110,45428665,10309525,51588015,78909561,9257405,33628349,68128438,18411915,72358170,95010552,2607799,526217,51464002,16054533,92692978,30463802,44846932,92692283,74743862,4064751,26744917,68824981,37957788,91957544,41442762,48675329,82651278,11543098,49882705,6497830,9398733,97940276,70727211,75820087,22766820,18833224,66832478,71469330,70420215,97057187,76330843,16445503,66137019,94076128,20122224,57803235,55960386,73168565,57241521,70367851,76960303,75153252,79657802,39553046,54868730,40865610,80251430,92215320,6521313,94595668,32161669,26664538,60102965,45790169,54427233,93566986,72793444,13862149,9058407,33935899,32274392,33797252,44983451,50007421,33123618,89499542,36396314,22879907,47738185,82339363,30771409,13468268,79136082,34946859,27411561,90158785,65047700,28796059,92787493,35512853,97379790,55189057,89804152,48088883,77898274,42644903,99515901,33201905,90654836,37501808,34295794,61982238,48893685,34667286,81677380,56461322,14947650,66250369,73786944,91240048,99226875,77694700,75128745,11581181,47298834,42307449,99524975,20140249,36780454,10358899,7066775,53802686,55850790,26507214,85242963,47792865,7646095,98130363,86242799,10453030,18466635,70782102,73109997,86543538,15902805,4508700,26275734,77312810,62115552,23432750,51315460,9860968,51149804,92803766,21533347,54663246,92033260,39847321,24915585,36808486,14349098,7517032,20642888,68110330,72278539,59109400,52613508,34044787,61712234,85711894,51426311,62051033,94539824,96735716,19891772,97561745,39201414,31727379,72238278,97011160,17068582,88251446,69641513,17386996,15948937,44479073,84684495,92541302,70541760,30395570,45667668,30090481,7919588,39373729,52204879,88904910,65338021,73392814,5640302,57240218,30163921,30694952,36139942,85695762,22405120,51715482,29401781,83651211,67474219,11923835,17081350,65074535,91938191,41245325,60955663,44664587,42073124,20765474,22997281,11757872,68694897,64157906,96906410,54263880,15163258,86460488,61859581,58749,45075471,63015256,59371804,67031644,16405341,21673260,86118021,85571389,8729146,58180653,64098930,16595436,82779622,79942022,20568363,40534591,18001617,22942635,88130087,84187166,57658654,61728685,1936762,26292919,19457589,82897371,39801351,44915235,98943869,54606384,20645197,74452589,43376279,24953498,8913721,32058615,49271185,2208785,36135,38510840,67084351,7011964,26734892,81805959,30891921,94360702,4091162,37560164,7955293,37224844,6111563,22200378,74614639,69605283,3487592,99297164,82427263,12303248,53632373,2331773,41380093,58751351,75229462,92353856,96420429,68939068,38022834,58208470,83948335,32151165,25842078,49598724,22113133,60106217,55615885,16567550,176257,30811010,26397786,9860195,94911072,59981773,19272365,51466049,5267545,14363867,26139110,73021291,23110625,48395186,1204161,38009615,69697787,13348726,42237907,67513640,33061250,81774825,29635537,17385531,28787861,79738755,24314885,1375023,62693428,15536795,91141395,62428472,63372756,79322415,6157724,33336148,60176618,8373289,37280276,64055960,24168411,21289531,21397057,47708850,7182642,90061527,45794415,62357986,36933038,68102437,91281584,3183975,17976208,45049308,98648327,10597197,36845587,84002370,23740167,29516592,10264691,16684829,45381876 +30764367,44784505,74357852,31126490,81898046,54517921,62430984,51047803,9603598,45237957,62496012,49641577,88653118,7104732,96193415,63967300,89214616,99861373,54058788,4508700,28851716,14947650,13862149,77300457,86798033,72278539,5482538,59177718,14731700,77301523,99965001,68816413,65271999,62452034,76960303,95395112,92692283,97561745,89217461,24591705,92554120,82651278,247198,34497327,40781854,34698428,34236719,4668450,61859143,44983451,57241521,95726235,15015906,3509435,64848072,91831487,13231279,65081429,78589145,44889423,73617245,18504197,5640302,65454636,33237508,1197320,55189057,33565483,89637706,42307449,99549373,65338021,92692978,3183975,48088883,61859581,66428795,93566986,10453030,97783876,88444207,64087743,99226875,67227442,55247455,54762643,14349098,63059024,21993752,74441448,70367851,95957797,86001008,18001617,7300047,61271144,51315460,92398073,10961421,38022834,68939068,77272628,2717150,31733363,74452589,51588015,27625735,55470718,37435892,71920426,90004325,8129978,81853704,44177011,79136082,32058615,45407418,20642888,9886593,29834512,76671482,50806615,54987042,36685023,82339363,30787683,26998766,35192533,92998591,3233084,9575954,29188588,53084256,69579137,30395570,61741594,17764950,40686254,45848907,35456853,44473167,51464002,67281495,96735716,92071990,26063929,20867149,20122224,54427233,87710366,85116755,37183543,17727650,6793819,63372756,34946859,29994197,67031644,88698958,95010552,38494874,65275241,67793644,15535065,54014062,42692881,43322743,8807940,27665211,36580610,62740044,32590267,6986898,62693428,50188404,53648154,6153406,112651,53802686,5970607,52060076,79806380,75543508,61373987,12416768,4119747,64602895,69786756,17037369,92033260,29959549,37957788,874791,92787493,42199455,94595668,12348118,66663942,68128438,86543538,18833224,70036438,97940276,14093520,52261574,49236559,13470059,56515456,47893286,90272749,99021067,4064751,18806856,36808486,26493119,60955663,79821897,17539625,50007421,77284619,22108665,51472275,62552524,28734791,22766820,80555751,16387575,63506281,9175338,99729357,38645117,64098930,12664567,19101477,71657078,95290172,36780454,37675718,47090124,77072625,17385531,93270084,53632373,17058722,36505482,87598888,18699206,27041967,79880247,61623915,14045383,36933038,7423788,99515901,32250045,19939935,38365584,65715134,77312810,83789391,1204161,78561158,67124282,65047700,92604458,33304202,99125126,53547802,60102965,19891772,77413012,415901,2917920,38256849,10309525,669105,84904436,81677380,56461322,26114953,68875490,88047921,74743862,92541302,68644627,78766358,33935899,44550764,76170907,73786944,62936963,9860195,36845587,58208470,39986008,44481640,77620120,45790169,3633375,56531125,85695762,2204165,6505939,8791066,9188443,91141395,69848388,59318837,30139692,2891150,22200378,64157906,92530431,57393458,8733068,57359924,21533347,14220886,75986488,72725103,39171895,30569392,49597667,59371804,44348328,66611759,60581278,41442762,82052050,25842078,79657802,72738685,23569917,61141874,38008118,80014588,65880522,62803858,35996293,57803235,34295794,46540998,99524975,4095116,10649306,26392416,33895336,59501487,54232247,51590803,20765474,8913721,72777973,99917599,70800879,91664334,34667286,22942635,749283,38341669,78785507,99658235,35092039,20486294,29510992,57163802,21919959,5073754,21070110,85571389,91957544,57248122,50668599,33201905,168541,44842615,31904591,41380093,89996536,27325428,40677414,12024238,39847321,15536795,9257405,18131876,6038457,526217,74614639,80316608,16595436,70596786,43357947,86301513,66045587,55753905,72238278,39801351,7011964,73124510,14363867,40152546,46870723,27411561,88251446,72373496,24953498,99333375,70004753,68000591,6808825,48260151,69605283,3233569,5111370,78300864,24619760,93933709,4434662,88904910,13819358,94090109,7687278,44479073,1239555,77377183,71522968,22997281,47738185,34493392,4787945,72274002,23432750,94539824,91802888,18783702,41092102,24314885,16097038,86821229,26102057,40197395,58751351,3294781,91281584,6525948,69697787,94076128,16054533,47361209,21001913,19376156,63152504,30366150,48675329,29635537,21673260,66137019,45306070,33249630,42967683,54663246,46851987,7685448,72614359,33123618,59197747,68824981,36812683,58749,8099994,72357096,23134715,83651211,57020481,89804152,9599614,94516935,19898053,55602660,55669657,10366309,30694952,40356877,98371444,28787861,24058273,40984766,4091162,72019362,51715482,30543215,58180653,71333116,29065964,72793444,12303248,43045786,81805959,51149804,17365188,66832478,4069912,23848565,508198,76330843,56473732,68694897,10358899,50702367,65851721,1431742,62762857,91240048,50567636,37620363,76540481,93562257,16019925,17857111,37739481,66250369,29932657,89078848,98739783,8115266,19486173,9259676,66667729,4806458,79738755,60430369,42947632,93359396,37891451,15148031,69136837,24733232,44847298,48360198,62357986,33628349,22113133,7955293,83210802,85242963,54199160,61263068,17957593,81172706,43933006,94360702,85023028,75153252,96709982,16099750,20002147,83269727,78909561,61982238,53842979,26139110,53666583,96906410,7182642,43376279,77787724,30218878,13348726,99297164,11581181,90061527,6157724,29819940,3088684,34044787,96318094,98130363,18411915,31491938,57961282,61815223,32161669,48658605,92803766,45667668,42644903,40781449,40865610,84127901,99690194,91255408,44060493,49882705,11161731,30998561,26664538,4978543,83533741,86022504,47708850,68041839,80251430,73222868,74137926,51426311,71083565,16424199,17016156,81274566,22450468,87160386,65038678,83150534,43506672,54606384,55143724,36189527,7919588,22879907,48395186,28550822,42782652,75407347,69641513,37659250,84349107,20885148,1936762,21397057,76434144,4099191,49598724,60106217,79942022,93057697,32159704,27185644,66903004,63628376,36753250,33797252,36135,45996863,26507214,321665,26397786,82897371,45075471,90457870,81774825,1569515,73235980,29401781,26275734,82178706,37501808,4985896,59109400,83948335,13468268,89046466,5267545,78549759,38061439,77187825,20140249,96726697,85711894,78602717,59614746,56153345,29466406,6521313,57240218,8696647,3235882,33699435,14326617,67451935,76315420,92215320,70420215,62051033,12571310,55960386,60393039,28796059,67084351,86118021,64055960,66322959,66271566,97011160,17146629,4204661,6221471,33336148,9623492,48892201,17386996,75128745,34432810,26776922,33431960,89811711,93790285,88130087,39553046,66116458,98653983,14626618,45990383,84187166,83747892,28928175,51507425,43501211,84684495,45049308,68316156,73031054,33553959,52204879,41481685,56484900,68110330,5832946,98943869,99224392,48893685,94711842,72495719,36312813,93515664,91938191,85117092,55615885,17068582,84293052,83155442,45428665,20568363,4798568,74724075,9398733,67030811,48774913,30090481,82886381,16380211,9058407,30463802,82779622,70541760,90013093,75052463,29029316,54263880,1250437,68204242,53888755,70372191,20836893,91990218,47792865,92867155,8055981,58224549,37192445,74110882,10597197,15902805,97641116,78884452,94911072,59910624,3880712,15064655,15948937,45919976,84406788,75777973,16567550,66319530,35512853,56424103,95581843,61897582,21289531,62232211,90090964,73392814,98648327,8128637,68702632,30891921,11365791,73168565,7517032,98462867,38510840,79922758,96420429,72358170,3773993,89419466,22435353,8634541,44844121,54868730,52613508,22129328,82979980,78218845,30653863,19272365,16445503,90158785,42237907,1022677,69355476,36139942,56955985,8729146,65074535,3487592,68102437,71300104,98948034,67513640,26292919,86242799,26744917,34698463,19457589,71965942,76825057,24826575,824872,42073124,99604946,39201414,52293995,75135448,36396314,23740167,18466635,66885828,14723410,29516592,80246713,97379790,8651647,11543098,18663507,89699445,53393358,9829782,44664587,45995325,49271185,6497830,1788101,82726008,47887470,30771409,16971929,84166196,4199704,55770687,60176618,37560164,17081350,61928316,69255765,49328608,17976208,24915585,84840549,45617087,93053405,71469330,79191827,38658347,16405341,62490109,51466049,63015256,16684829,44846932,82427263,82327024,25636669,45794415,96852321,84002370,71316369,24168411,61960400,23194618,37224844,6111563,2208785,26863229,36930650,87720882,33061250,55850790,7502255,17894977,31161687,72732205,95251277,44245960,5822202,59981773,91727510,75229462,7229550,75820087,48673079,97057187,41092172,82532312,86460488,22721500,569864,6819644,7646095,30811010,44915235,59329511,32151165,31727379,11757872,9860968,83133790,83083131,37280276,62428472,176257,40027975,92353856,39373729,83302115,83368048,6871053,90654836,57658654,2607799,43152977,32274392,37166608,8373289,32426752,2331773,81855445,1375023,47298834,65017137,89499542,13173644,47213183,41245325,10264691,30163921,97281847,15163258,90310261,11923835,26734892,77694700,15075176,73021291,23110625,38009615,36468541,27187213,61728685,20645197,70782102,79322415,70727211,22405120,76703609,99971982,32699744,45381876,62115552,7066775,20681921,61712234,59405277,40534591,67474219,44627776,4515343,77898274,73109997 \ No newline at end of file diff --git a/src/testFixtures/java/org/opensearch/knn/KNNJsonIndexMappingsBuilder.java b/src/testFixtures/java/org/opensearch/knn/KNNJsonIndexMappingsBuilder.java index 165dfaf64..15ac0c211 100644 --- a/src/testFixtures/java/org/opensearch/knn/KNNJsonIndexMappingsBuilder.java +++ b/src/testFixtures/java/org/opensearch/knn/KNNJsonIndexMappingsBuilder.java @@ -102,9 +102,17 @@ private void addParameters(final XContentBuilder xContentBuilder) throws IOExcep @Builder public static class Parameters { private Encoder encoder; + private Integer efConstruction; + private Integer efSearch; private void addTo(final XContentBuilder xContentBuilder) throws IOException { xContentBuilder.startObject("parameters"); + if (efConstruction != null) { + xContentBuilder.field("ef_construction", efConstruction); + } + if (efSearch != null) { + xContentBuilder.field("ef_search", efSearch); + } addEncoder(xContentBuilder); xContentBuilder.endObject(); } From c4445d0bec9702b746810fa68c10ec3fe4c2da31 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Wed, 11 Sep 2024 15:58:01 -0700 Subject: [PATCH 367/416] Improving Defaults Hyper Parameter for Binary Quantization Indexes (#2087) (#2094) Signed-off-by: VIKASH TIWARI (cherry picked from commit e55e49b249feced18f3b39535969ea1cc5ad1798) Co-authored-by: Vikasht34 --- .../knn/index/engine/MethodComponent.java | 36 ++++++++++++++--- .../engine/faiss/FaissMethodResolver.java | 5 ++- .../knn/index/mapper/CompressionLevel.java | 6 +-- .../index/query/rescore/RescoreContext.java | 5 ++- .../index/util/IndexHyperParametersUtil.java | 20 ++++++++++ .../index/engine/MethodComponentTests.java | 40 +++++++++++++++++++ .../faiss/FaissMethodResolverTests.java | 27 +++++++++++++ .../index/mapper/CompressionLevelTests.java | 27 +++++++++++++ .../query/rescore/RescoreContextTests.java | 38 +++++++++++++++++- .../util/IndexHyperParametersUtilTests.java | 8 ++++ 10 files changed, 199 insertions(+), 13 deletions(-) diff --git a/src/main/java/org/opensearch/knn/index/engine/MethodComponent.java b/src/main/java/org/opensearch/knn/index/engine/MethodComponent.java index 75e18a243..d456ea89f 100644 --- a/src/main/java/org/opensearch/knn/index/engine/MethodComponent.java +++ b/src/main/java/org/opensearch/knn/index/engine/MethodComponent.java @@ -11,6 +11,8 @@ import org.opensearch.common.ValidationException; import org.opensearch.knn.common.KNNConstants; import org.opensearch.knn.index.VectorDataType; +import org.opensearch.knn.index.mapper.CompressionLevel; +import org.opensearch.knn.index.mapper.Mode; import org.opensearch.knn.index.util.IndexHyperParametersUtil; import java.util.HashMap; @@ -328,6 +330,15 @@ public static Map getParameterMapWithDefaultsAdded( Map parametersWithDefaultsMap = new HashMap<>(); Map userProvidedParametersMap = methodComponentContext.getParameters(); Version indexCreationVersion = knnMethodConfigContext.getVersionCreated(); + Mode mode = knnMethodConfigContext.getMode(); + CompressionLevel compressionLevel = knnMethodConfigContext.getCompressionLevel(); + + // Check if the mode is ON_DISK and the compression level is one of the binary quantization levels (x32, x16, or x8). + // This determines whether to use binary quantization-specific values for parameters like ef_search and ef_construction. + boolean isOnDiskWithBinaryQuantization = (compressionLevel == CompressionLevel.x32 + || compressionLevel == CompressionLevel.x16 + || compressionLevel == CompressionLevel.x8); + for (Parameter parameter : methodComponent.getParameters().values()) { if (methodComponentContext.getParameters().containsKey(parameter.getName())) { parametersWithDefaultsMap.put(parameter.getName(), userProvidedParametersMap.get(parameter.getName())); @@ -335,12 +346,27 @@ public static Map getParameterMapWithDefaultsAdded( // Picking the right values for the parameters whose values are different based on different index // created version. if (parameter.getName().equals(KNNConstants.METHOD_PARAMETER_EF_SEARCH)) { - parametersWithDefaultsMap.put(parameter.getName(), IndexHyperParametersUtil.getHNSWEFSearchValue(indexCreationVersion)); + if (isOnDiskWithBinaryQuantization) { + parametersWithDefaultsMap.put(parameter.getName(), IndexHyperParametersUtil.getBinaryQuantizationEFSearchValue()); + } else { + parametersWithDefaultsMap.put( + parameter.getName(), + IndexHyperParametersUtil.getHNSWEFSearchValue(indexCreationVersion) + ); + } } else if (parameter.getName().equals(KNNConstants.METHOD_PARAMETER_EF_CONSTRUCTION)) { - parametersWithDefaultsMap.put( - parameter.getName(), - IndexHyperParametersUtil.getHNSWEFConstructionValue(indexCreationVersion) - ); + if (isOnDiskWithBinaryQuantization) { + parametersWithDefaultsMap.put( + parameter.getName(), + IndexHyperParametersUtil.getBinaryQuantizationEFConstructionValue() + ); + } else { + parametersWithDefaultsMap.put( + parameter.getName(), + IndexHyperParametersUtil.getHNSWEFConstructionValue(indexCreationVersion) + ); + } + } else { Object value = parameter.getDefaultValue(); if (value != null) { diff --git a/src/main/java/org/opensearch/knn/index/engine/faiss/FaissMethodResolver.java b/src/main/java/org/opensearch/knn/index/engine/faiss/FaissMethodResolver.java index 90e938eb3..c976a0959 100644 --- a/src/main/java/org/opensearch/knn/index/engine/faiss/FaissMethodResolver.java +++ b/src/main/java/org/opensearch/knn/index/engine/faiss/FaissMethodResolver.java @@ -65,8 +65,6 @@ public ResolvedMethodContext resolveMethod( // Fill in parameters for the encoder and then the method. resolveEncoder(resolvedKNNMethodContext, knnMethodConfigContext, encoderMap); - resolveMethodParams(resolvedKNNMethodContext.getMethodComponentContext(), knnMethodConfigContext, method); - // From the resolved method context, get the compression level and validate it against the passed in // configuration CompressionLevel resolvedCompressionLevel = resolveCompressionLevelFromMethodContext( @@ -77,6 +75,9 @@ public ResolvedMethodContext resolveMethod( // Validate that resolved compression doesnt have any conflicts validateCompressionConflicts(knnMethodConfigContext.getCompressionLevel(), resolvedCompressionLevel); + knnMethodConfigContext.setCompressionLevel(resolvedCompressionLevel); + resolveMethodParams(resolvedKNNMethodContext.getMethodComponentContext(), knnMethodConfigContext, method); + return ResolvedMethodContext.builder() .knnMethodContext(resolvedKNNMethodContext) .compressionLevel(resolvedCompressionLevel) diff --git a/src/main/java/org/opensearch/knn/index/mapper/CompressionLevel.java b/src/main/java/org/opensearch/knn/index/mapper/CompressionLevel.java index cc80bb1ed..222e042b6 100644 --- a/src/main/java/org/opensearch/knn/index/mapper/CompressionLevel.java +++ b/src/main/java/org/opensearch/knn/index/mapper/CompressionLevel.java @@ -25,9 +25,9 @@ public enum CompressionLevel { x1(1, "1x", null, Collections.emptySet()), x2(2, "2x", null, Collections.emptySet()), x4(4, "4x", null, Collections.emptySet()), - x8(8, "8x", new RescoreContext(1.5f), Set.of(Mode.ON_DISK)), - x16(16, "16x", new RescoreContext(2.0f), Set.of(Mode.ON_DISK)), - x32(32, "32x", new RescoreContext(2.0f), Set.of(Mode.ON_DISK)); + x8(8, "8x", new RescoreContext(2.0f), Set.of(Mode.ON_DISK)), + x16(16, "16x", new RescoreContext(3.0f), Set.of(Mode.ON_DISK)), + x32(32, "32x", new RescoreContext(3.0f), Set.of(Mode.ON_DISK)); // Internally, an empty string is easier to deal with them null. However, from the mapping, // we do not want users to pass in the empty string and instead want null. So we make the conversion herex diff --git a/src/main/java/org/opensearch/knn/index/query/rescore/RescoreContext.java b/src/main/java/org/opensearch/knn/index/query/rescore/RescoreContext.java index 9fe2ddbc5..5e57b4aba 100644 --- a/src/main/java/org/opensearch/knn/index/query/rescore/RescoreContext.java +++ b/src/main/java/org/opensearch/knn/index/query/rescore/RescoreContext.java @@ -22,6 +22,9 @@ public final class RescoreContext { public static final int MAX_FIRST_PASS_RESULTS = 10000; + // Todo:- We will improve this in upcoming releases + public static final int MIN_FIRST_PASS_RESULTS = 100; + @Builder.Default private float oversampleFactor = DEFAULT_OVERSAMPLE_FACTOR; @@ -40,6 +43,6 @@ public static RescoreContext getDefault() { * @return The number of results to return for the first pass of rescoring */ public int getFirstPassK(int finalK) { - return Math.min(MAX_FIRST_PASS_RESULTS, (int) Math.ceil(finalK * oversampleFactor)); + return Math.min(MAX_FIRST_PASS_RESULTS, Math.max(MIN_FIRST_PASS_RESULTS, (int) Math.ceil(finalK * oversampleFactor))); } } diff --git a/src/main/java/org/opensearch/knn/index/util/IndexHyperParametersUtil.java b/src/main/java/org/opensearch/knn/index/util/IndexHyperParametersUtil.java index af842788a..c88a28e5c 100644 --- a/src/main/java/org/opensearch/knn/index/util/IndexHyperParametersUtil.java +++ b/src/main/java/org/opensearch/knn/index/util/IndexHyperParametersUtil.java @@ -28,6 +28,8 @@ public class IndexHyperParametersUtil { private static final int INDEX_KNN_DEFAULT_ALGO_PARAM_EF_CONSTRUCTION_OLD_VALUE = 512; private static final int INDEX_KNN_DEFAULT_ALGO_PARAM_EF_SEARCH_OLD_VALUE = 512; + private static final int INDEX_BINARY_QUANTIZATION_KNN_DEFAULT_ALGO_PARAM_EF_CONSTRUCTION = 256; + private static final int INDEX_BINARY_QUANTIZATION_KNN_DEFAULT_ALGO_PARAM_EF_SEARCH = 256; /** * Returns the default value of EF Construction that should be used for the input index version. After version 2.12.0 @@ -76,4 +78,22 @@ public static int getHNSWEFSearchValue(@NonNull final Version indexVersion) { ); return KNNSettings.INDEX_KNN_DEFAULT_ALGO_PARAM_EF_SEARCH; } + + /* + * Returns the default value of EF Construction that should be used with Binary Quantization. + * + * @return default value of EF Construction + */ + public static int getBinaryQuantizationEFConstructionValue() { + return INDEX_BINARY_QUANTIZATION_KNN_DEFAULT_ALGO_PARAM_EF_CONSTRUCTION; + } + + /* + * Returns the default value of EF Search that should be used with Binary Quantization. + * + * @return default value of EF Search + */ + public static int getBinaryQuantizationEFSearchValue() { + return INDEX_BINARY_QUANTIZATION_KNN_DEFAULT_ALGO_PARAM_EF_SEARCH; + } } diff --git a/src/test/java/org/opensearch/knn/index/engine/MethodComponentTests.java b/src/test/java/org/opensearch/knn/index/engine/MethodComponentTests.java index 7730095c7..e247fb70a 100644 --- a/src/test/java/org/opensearch/knn/index/engine/MethodComponentTests.java +++ b/src/test/java/org/opensearch/knn/index/engine/MethodComponentTests.java @@ -11,6 +11,9 @@ import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.knn.index.VectorDataType; +import org.opensearch.knn.index.mapper.CompressionLevel; +import org.opensearch.knn.index.mapper.Mode; +import org.opensearch.knn.index.util.IndexHyperParametersUtil; import java.io.IOException; import java.util.Map; @@ -214,4 +217,41 @@ public void testBuilder() { .getLibraryParameters() ); } + + /** + * Test the new flow where EF_SEARCH and EF_CONSTRUCTION are set for ON_DISK mode + * with binary quantization compression levels. + */ + public void testGetParameterMapWithDefaultsAdded_forOnDiskWithBinaryQuantization() { + // Set up MethodComponent and context + String methodName = "test-method"; + String parameterEFSearch = "ef_search"; + String parameterEFConstruction = "ef_construction"; + + MethodComponent methodComponent = MethodComponent.Builder.builder(methodName) + .addParameter(parameterEFSearch, new Parameter.IntegerParameter(parameterEFSearch, 512, (v, context) -> v > 0)) + .addParameter(parameterEFConstruction, new Parameter.IntegerParameter(parameterEFConstruction, 512, (v, context) -> v > 0)) + .build(); + + // Simulate ON_DISK mode and binary quantization compression levels + KNNMethodConfigContext knnMethodConfigContext = KNNMethodConfigContext.builder() + .versionCreated(Version.CURRENT) + .mode(Mode.ON_DISK) // ON_DISK mode + .compressionLevel(CompressionLevel.x32) // Binary quantization compression level + .build(); + + MethodComponentContext methodComponentContext = new MethodComponentContext(methodName, Map.of()); + + // Retrieve parameter map with defaults added + Map resultMap = MethodComponent.getParameterMapWithDefaultsAdded( + methodComponentContext, + methodComponent, + knnMethodConfigContext + ); + + // Check that binary quantization values are used + assertEquals(IndexHyperParametersUtil.getBinaryQuantizationEFSearchValue(), resultMap.get(parameterEFSearch)); + assertEquals(IndexHyperParametersUtil.getBinaryQuantizationEFConstructionValue(), resultMap.get(parameterEFConstruction)); + } + } diff --git a/src/test/java/org/opensearch/knn/index/engine/faiss/FaissMethodResolverTests.java b/src/test/java/org/opensearch/knn/index/engine/faiss/FaissMethodResolverTests.java index ad466d4bb..3a33736fa 100644 --- a/src/test/java/org/opensearch/knn/index/engine/faiss/FaissMethodResolverTests.java +++ b/src/test/java/org/opensearch/knn/index/engine/faiss/FaissMethodResolverTests.java @@ -136,6 +136,33 @@ public void testResolveMethod_whenValid_thenResolve() { SpaceType.L2 ); validateResolveMethodContext(resolvedMethodContext, CompressionLevel.x1, SpaceType.L2, ENCODER_FLAT, false); + + KNNMethodConfigContext knnMethodConfigContext = KNNMethodConfigContext.builder() + .vectorDataType(VectorDataType.FLOAT) + .versionCreated(Version.CURRENT) + .build(); + + resolvedMethodContext = TEST_RESOLVER.resolveMethod( + new KNNMethodContext( + KNNEngine.FAISS, + SpaceType.L2, + new MethodComponentContext( + METHOD_HNSW, + Map.of( + METHOD_ENCODER_PARAMETER, + new MethodComponentContext( + QFrameBitEncoder.NAME, + Map.of(QFrameBitEncoder.BITCOUNT_PARAM, CompressionLevel.x8.numBitsForFloat32()) + ) + ) + ) + ), + knnMethodConfigContext, + false, + SpaceType.L2 + ); + assertEquals(knnMethodConfigContext.getCompressionLevel(), CompressionLevel.x8); + validateResolveMethodContext(resolvedMethodContext, CompressionLevel.x8, SpaceType.L2, QFrameBitEncoder.NAME, true); } private void validateResolveMethodContext( diff --git a/src/test/java/org/opensearch/knn/index/mapper/CompressionLevelTests.java b/src/test/java/org/opensearch/knn/index/mapper/CompressionLevelTests.java index 07475109a..9eb302b13 100644 --- a/src/test/java/org/opensearch/knn/index/mapper/CompressionLevelTests.java +++ b/src/test/java/org/opensearch/knn/index/mapper/CompressionLevelTests.java @@ -7,6 +7,7 @@ import org.opensearch.core.common.Strings; import org.opensearch.knn.KNNTestCase; +import org.opensearch.knn.index.query.rescore.RescoreContext; public class CompressionLevelTests extends KNNTestCase { @@ -39,4 +40,30 @@ public void testIsConfigured() { assertFalse(CompressionLevel.isConfigured(null)); assertTrue(CompressionLevel.isConfigured(CompressionLevel.x1)); } + + public void testGetDefaultRescoreContext() { + // Test rescore context for ON_DISK mode + Mode mode = Mode.ON_DISK; + + // x32 should have RescoreContext with an oversample factor of 3.0f + RescoreContext rescoreContext = CompressionLevel.x32.getDefaultRescoreContext(mode); + assertNotNull(rescoreContext); + assertEquals(3.0f, rescoreContext.getOversampleFactor(), 0.0f); + + // x16 should have RescoreContext with an oversample factor of 3.0f + rescoreContext = CompressionLevel.x16.getDefaultRescoreContext(mode); + assertNotNull(rescoreContext); + assertEquals(3.0f, rescoreContext.getOversampleFactor(), 0.0f); + + // x8 should have RescoreContext with an oversample factor of 2.0f + rescoreContext = CompressionLevel.x8.getDefaultRescoreContext(mode); + assertNotNull(rescoreContext); + assertEquals(2.0f, rescoreContext.getOversampleFactor(), 0.0f); + + // Other compression levels should not have a RescoreContext for ON_DISK mode + assertNull(CompressionLevel.x4.getDefaultRescoreContext(mode)); + assertNull(CompressionLevel.x2.getDefaultRescoreContext(mode)); + assertNull(CompressionLevel.x1.getDefaultRescoreContext(mode)); + assertNull(CompressionLevel.NOT_CONFIGURED.getDefaultRescoreContext(mode)); + } } diff --git a/src/test/java/org/opensearch/knn/index/query/rescore/RescoreContextTests.java b/src/test/java/org/opensearch/knn/index/query/rescore/RescoreContextTests.java index 6d872ddb8..fd94667db 100644 --- a/src/test/java/org/opensearch/knn/index/query/rescore/RescoreContextTests.java +++ b/src/test/java/org/opensearch/knn/index/query/rescore/RescoreContextTests.java @@ -8,6 +8,7 @@ import org.opensearch.knn.KNNTestCase; import static org.opensearch.knn.index.query.rescore.RescoreContext.MAX_FIRST_PASS_RESULTS; +import static org.opensearch.knn.index.query.rescore.RescoreContext.MIN_FIRST_PASS_RESULTS; public class RescoreContextTests extends KNNTestCase { @@ -17,10 +18,43 @@ public void testGetFirstPassK() { int finalK = 100; assertEquals(260, rescoreContext.getFirstPassK(finalK)); finalK = 1; - assertEquals(3, rescoreContext.getFirstPassK(finalK)); + assertEquals(MIN_FIRST_PASS_RESULTS, rescoreContext.getFirstPassK(finalK)); finalK = 0; - assertEquals(0, rescoreContext.getFirstPassK(finalK)); + assertEquals(MIN_FIRST_PASS_RESULTS, rescoreContext.getFirstPassK(finalK)); finalK = MAX_FIRST_PASS_RESULTS; assertEquals(MAX_FIRST_PASS_RESULTS, rescoreContext.getFirstPassK(finalK)); } + + public void testGetFirstPassKWithMinPassK() { + float oversample = 2.6f; + RescoreContext rescoreContext = RescoreContext.builder().oversampleFactor(oversample).build(); + + // Case 1: Test with a finalK that results in a value greater than MIN_FIRST_PASS_RESULTS + int finalK = 100; + assertEquals(260, rescoreContext.getFirstPassK(finalK)); + + // Case 2: Test with a very small finalK that should result in a value less than MIN_FIRST_PASS_RESULTS + finalK = 1; + assertEquals(MIN_FIRST_PASS_RESULTS, rescoreContext.getFirstPassK(finalK)); + + // Case 3: Test with finalK = 0, should return 0 + finalK = 0; + assertEquals(MIN_FIRST_PASS_RESULTS, rescoreContext.getFirstPassK(finalK)); + + // Case 4: Test with finalK = MAX_FIRST_PASS_RESULTS, should cap at MAX_FIRST_PASS_RESULTS + finalK = MAX_FIRST_PASS_RESULTS; + assertEquals(MAX_FIRST_PASS_RESULTS, rescoreContext.getFirstPassK(finalK)); + + // Case 5: Test where finalK * oversample is smaller than MIN_FIRST_PASS_RESULTS + finalK = 10; + oversample = 0.5f; // This will result in 5, which is less than MIN_FIRST_PASS_RESULTS + rescoreContext = RescoreContext.builder().oversampleFactor(oversample).build(); + assertEquals(MIN_FIRST_PASS_RESULTS, rescoreContext.getFirstPassK(finalK)); + + // Case 6: Test where finalK * oversample results in exactly MIN_FIRST_PASS_RESULTS + finalK = 100; + oversample = 1.0f; // This will result in exactly 100 (MIN_FIRST_PASS_RESULTS) + rescoreContext = RescoreContext.builder().oversampleFactor(oversample).build(); + assertEquals(MIN_FIRST_PASS_RESULTS, rescoreContext.getFirstPassK(finalK)); + } } diff --git a/src/test/java/org/opensearch/knn/index/util/IndexHyperParametersUtilTests.java b/src/test/java/org/opensearch/knn/index/util/IndexHyperParametersUtilTests.java index 508b8765c..96eb41476 100644 --- a/src/test/java/org/opensearch/knn/index/util/IndexHyperParametersUtilTests.java +++ b/src/test/java/org/opensearch/knn/index/util/IndexHyperParametersUtilTests.java @@ -41,4 +41,12 @@ public void testGetHNSWEFSearchValue_withDifferentValues_thenSuccess() { IndexHyperParametersUtil.getHNSWEFConstructionValue(Version.CURRENT) ); } + + public void testGetBinaryQuantizationEFValues_thenSuccess() { + // Test for Binary Quantization EF Construction value + Assert.assertEquals(256, IndexHyperParametersUtil.getBinaryQuantizationEFConstructionValue()); + + // Test for Binary Quantization EF Search value + Assert.assertEquals(256, IndexHyperParametersUtil.getBinaryQuantizationEFSearchValue()); + } } From 7972f08b74d4d7f28e49869220ca7d8c67b5cb35 Mon Sep 17 00:00:00 2001 From: Tejas Shah Date: Thu, 12 Sep 2024 13:43:53 -0700 Subject: [PATCH 368/416] Adds index and query BWC tests for missing engines (#2035) (#2096) This is done to make sure that KNN80DocsValueConsumer code path is hit Signed-off-by: Tejas Shah (cherry picked from commit e3200260e7360f6ff90600535ac759d11cb1fefb) --- qa/restart-upgrade/build.gradle | 30 +++++++++++ .../org/opensearch/knn/bwc/IndexingIT.java | 50 ++++++++++++++++--- .../org/opensearch/knn/bwc/QueryANNIT.java | 40 ++++++++++++--- 3 files changed, 104 insertions(+), 16 deletions(-) diff --git a/qa/restart-upgrade/build.gradle b/qa/restart-upgrade/build.gradle index a629e87ff..a308699aa 100644 --- a/qa/restart-upgrade/build.gradle +++ b/qa/restart-upgrade/build.gradle @@ -27,6 +27,8 @@ testClusters { } } + def versionsBelow1_3 = ["1.1.", "1.2."] + def versionsBelow2_3 = ["1.", "2.0.", "2.1.", "2.2."] // Task to run BWC tests against the old cluster task testAgainstOldCluster(type: StandaloneRestIntegTestTask) { dependsOn "zipBwcPlugin" @@ -81,6 +83,20 @@ testClusters { } } + if (versionsBelow2_3.any { knn_bwc_version.startsWith(it) }) { + filter { + excludeTestsMatching "org.opensearch.knn.bwc.QueryANNIT.testQueryOnLuceneIndex" + excludeTestsMatching "org.opensearch.knn.bwc.IndexingIT.testKNNIndexLuceneForceMerge" + } + } + + if (versionsBelow1_3.any { knn_bwc_version.startsWith(it) }) { + filter { + excludeTestsMatching "org.opensearch.knn.bwc.QueryANNIT.testQueryOnFaissIndex" + excludeTestsMatching "org.opensearch.knn.bwc.IndexingIT.testKNNIndexFaissForceMerge" + } + } + nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}".allHttpSocketURI.join(",")}") nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}".getName()}") systemProperty 'tests.security.manager', 'false' @@ -147,6 +163,20 @@ testClusters { } } + if (versionsBelow2_3.any {knn_bwc_version.startsWith(it) }) { + filter { + excludeTestsMatching "org.opensearch.knn.bwc.QueryANNIT.testQueryOnLuceneIndex" + excludeTestsMatching "org.opensearch.knn.bwc.IndexingIT.testKNNIndexLuceneForceMerge" + } + } + + if (versionsBelow1_3.any { knn_bwc_version.startsWith(it) }) { + filter { + excludeTestsMatching "org.opensearch.knn.bwc.QueryANNIT.testQueryOnFaissIndex" + excludeTestsMatching "org.opensearch.knn.bwc.IndexingIT.testKNNIndexFaissForceMerge" + } + } + nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}".allHttpSocketURI.join(",")}") nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}".getName()}") systemProperty 'tests.security.manager', 'false' diff --git a/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/IndexingIT.java b/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/IndexingIT.java index 9c1dfb018..6b0bab9ee 100644 --- a/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/IndexingIT.java +++ b/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/IndexingIT.java @@ -23,6 +23,7 @@ import static org.opensearch.knn.common.KNNConstants.FAISS_NAME; import static org.opensearch.knn.common.KNNConstants.KNN_ENGINE; import static org.opensearch.knn.common.KNNConstants.KNN_METHOD; +import static org.opensearch.knn.common.KNNConstants.LUCENE_NAME; import static org.opensearch.knn.common.KNNConstants.METHOD_HNSW; import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_EF_CONSTRUCTION; import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_EF_SEARCH; @@ -51,7 +52,7 @@ public void testKNNIndexDefaultLegacyFieldMapping() throws Exception { createKnnIndex(testIndex, getKNNDefaultIndexSettings(), createKnnIndexMapping(TEST_FIELD, DIMENSIONS)); addKNNDocs(testIndex, TEST_FIELD, DIMENSIONS, DOC_ID, NUM_DOCS); } else { - validateKNNIndexingOnUpgrade(); + validateKNNIndexingOnUpgrade(NUM_DOCS); } } @@ -65,8 +66,41 @@ public void testKNNIndexDefaultLegacyFieldMappingForceMerge() throws Exception { addKNNDocs(testIndex, TEST_FIELD, DIMENSIONS, DOC_ID, 100); // Flush to ensure that index is not re-indexed when node comes back up flush(testIndex, true); + validateKNNSearch(testIndex, TEST_FIELD, DIMENSIONS, 100, K); } else { - forceMergeKnnIndex(testIndex); + validateKNNIndexingOnUpgrade(100); + } + } + + public void testKNNIndexFaissForceMerge() throws Exception { + waitForClusterHealthGreen(NODES_BWC_CLUSTER); + + if (isRunningAgainstOldCluster()) { + createKnnIndex(testIndex, getKNNDefaultIndexSettings(), createKnnIndexMapping(TEST_FIELD, DIMENSIONS, METHOD_HNSW, FAISS_NAME)); + addKNNDocs(testIndex, TEST_FIELD, DIMENSIONS, DOC_ID, 100); + // Flush to ensure that index is not re-indexed when node comes back up + flush(testIndex, true); + validateKNNSearch(testIndex, TEST_FIELD, DIMENSIONS, 100, K); + } else { + validateKNNIndexingOnUpgrade(100); + } + } + + public void testKNNIndexLuceneForceMerge() throws Exception { + waitForClusterHealthGreen(NODES_BWC_CLUSTER); + + if (isRunningAgainstOldCluster()) { + createKnnIndex( + testIndex, + getKNNDefaultIndexSettings(), + createKnnIndexMapping(TEST_FIELD, DIMENSIONS, METHOD_HNSW, LUCENE_NAME) + ); + addKNNDocs(testIndex, TEST_FIELD, DIMENSIONS, DOC_ID, 100); + // Flush to ensure that index is not re-indexed when node comes back up + flush(testIndex, true); + validateKNNSearch(testIndex, TEST_FIELD, DIMENSIONS, 100, K); + } else { + validateKNNIndexingOnUpgrade(100); } } @@ -115,7 +149,7 @@ public void testKNNIndexCustomLegacyFieldMapping() throws Exception { ); addKNNDocs(testIndex, TEST_FIELD, DIMENSIONS, DOC_ID, NUM_DOCS); } else { - validateKNNIndexingOnUpgrade(); + validateKNNIndexingOnUpgrade(NUM_DOCS); } } @@ -126,7 +160,7 @@ public void testKNNIndexDefaultMethodFieldMapping() throws Exception { createKnnIndex(testIndex, getKNNDefaultIndexSettings(), createKNNIndexMethodFieldMapping(TEST_FIELD, DIMENSIONS)); addKNNDocs(testIndex, TEST_FIELD, DIMENSIONS, DOC_ID, NUM_DOCS); } else { - validateKNNIndexingOnUpgrade(); + validateKNNIndexingOnUpgrade(NUM_DOCS); } } @@ -150,7 +184,7 @@ public void testKNNIndexCustomMethodFieldMapping() throws Exception { addKNNDocs(testIndex, TEST_FIELD, DIMENSIONS, DOC_ID, NUM_DOCS); } else { validateCustomMethodFieldMappingAfterUpgrade(); - validateKNNIndexingOnUpgrade(); + validateKNNIndexingOnUpgrade(NUM_DOCS); } } @@ -240,11 +274,11 @@ public void testNoParametersOnUpgrade() throws Exception { } // KNN indexing tests when the cluster is upgraded to latest version - public void validateKNNIndexingOnUpgrade() throws Exception { - QUERY_COUNT = NUM_DOCS; + public void validateKNNIndexingOnUpgrade(int numOfDocs) throws Exception { + QUERY_COUNT = numOfDocs; validateKNNSearch(testIndex, TEST_FIELD, DIMENSIONS, QUERY_COUNT, K); cleanUpCache(); - DOC_ID = NUM_DOCS; + DOC_ID = numOfDocs; addKNNDocs(testIndex, TEST_FIELD, DIMENSIONS, DOC_ID, NUM_DOCS); QUERY_COUNT = QUERY_COUNT + NUM_DOCS; validateKNNSearch(testIndex, TEST_FIELD, DIMENSIONS, QUERY_COUNT, K); diff --git a/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/QueryANNIT.java b/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/QueryANNIT.java index 566f40383..2cf4f335b 100644 --- a/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/QueryANNIT.java +++ b/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/QueryANNIT.java @@ -1,20 +1,19 @@ /* + * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - * - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. */ package org.opensearch.knn.bwc; import java.util.Map; +import static org.opensearch.knn.common.KNNConstants.LUCENE_NAME; import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_EF_SEARCH; +import static org.opensearch.knn.common.KNNConstants.NMSLIB_NAME; +/** + * Use case: Test queries on indexes created on older versions + */ public class QueryANNIT extends AbstractRestartUpgradeTestCase { private static final String TEST_FIELD = "test-field"; @@ -22,8 +21,9 @@ public class QueryANNIT extends AbstractRestartUpgradeTestCase { private static final int K = 5; private static final Integer EF_SEARCH = 10; private static final int NUM_DOCS = 10; + private static final String ALGORITHM = "hnsw"; - public void testQueryANN() throws Exception { + public void testQueryOnFaissIndex() throws Exception { if (isRunningAgainstOldCluster()) { createKnnIndex(testIndex, getKNNDefaultIndexSettings(), createKnnIndexMapping(TEST_FIELD, DIMENSIONS)); addKNNDocs(testIndex, TEST_FIELD, DIMENSIONS, 0, NUM_DOCS); @@ -34,4 +34,28 @@ public void testQueryANN() throws Exception { deleteKNNIndex(testIndex); } } + + public void testQueryOnNmslibIndex() throws Exception { + if (isRunningAgainstOldCluster()) { + createKnnIndex(testIndex, getKNNDefaultIndexSettings(), createKnnIndexMapping(TEST_FIELD, DIMENSIONS, ALGORITHM, NMSLIB_NAME)); + addKNNDocs(testIndex, TEST_FIELD, DIMENSIONS, 0, NUM_DOCS); + validateKNNSearch(testIndex, TEST_FIELD, DIMENSIONS, NUM_DOCS, K); + } else { + validateKNNSearch(testIndex, TEST_FIELD, DIMENSIONS, NUM_DOCS, K); + validateKNNSearch(testIndex, TEST_FIELD, DIMENSIONS, NUM_DOCS, K, Map.of(METHOD_PARAMETER_EF_SEARCH, EF_SEARCH)); + deleteKNNIndex(testIndex); + } + } + + public void testQueryOnLuceneIndex() throws Exception { + if (isRunningAgainstOldCluster()) { + createKnnIndex(testIndex, getKNNDefaultIndexSettings(), createKnnIndexMapping(TEST_FIELD, DIMENSIONS, ALGORITHM, LUCENE_NAME)); + addKNNDocs(testIndex, TEST_FIELD, DIMENSIONS, 0, NUM_DOCS); + validateKNNSearch(testIndex, TEST_FIELD, DIMENSIONS, NUM_DOCS, K); + } else { + validateKNNSearch(testIndex, TEST_FIELD, DIMENSIONS, NUM_DOCS, K); + validateKNNSearch(testIndex, TEST_FIELD, DIMENSIONS, NUM_DOCS, K, Map.of(METHOD_PARAMETER_EF_SEARCH, EF_SEARCH)); + deleteKNNIndex(testIndex); + } + } } From cddbdedf5fcd9fbaea214b2b347b3d703310780e Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Fri, 13 Sep 2024 14:29:51 -0700 Subject: [PATCH 369/416] Add short circuit if no live docs are in segments (#2059) (#2104) * Add short circuit if no live docs are in segments Signed-off-by: Vijayan Balasubramanian --------- Signed-off-by: Vijayan Balasubramanian (cherry picked from commit 7aedefd9f0229ce4ae3832b5dd7ae26e6e864983) Co-authored-by: Vijayan Balasubramanian --- CHANGELOG.md | 1 + .../NativeEngines990KnnVectorsWriter.java | 27 +++++++++---------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2954d898c..fcdbe1974 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ## [Unreleased 2.x](https://github.com/opensearch-project/k-NN/compare/2.17...2.x) ### Features ### Enhancements +* Add short circuit if no live docs are in segments [#2059](https://github.com/opensearch-project/k-NN/pull/2059) ### Bug Fixes ### Infrastructure ### Documentation diff --git a/src/main/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsWriter.java b/src/main/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsWriter.java index dba0926ff..0d016a60b 100644 --- a/src/main/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsWriter.java +++ b/src/main/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsWriter.java @@ -244,26 +244,25 @@ private void trainAndIndex( final String operationName ) throws IOException { final VectorDataType vectorDataType = extractVectorDataType(fieldInfo); - KNNVectorValues knnVectorValues = vectorValuesRetriever.apply(vectorDataType, fieldInfo, VectorProcessingContext); - QuantizationParams quantizationParams = quantizationService.getQuantizationParams(fieldInfo); - QuantizationState quantizationState = null; // Count the docIds int totalLiveDocs = getLiveDocs(vectorValuesRetriever.apply(vectorDataType, fieldInfo, VectorProcessingContext)); - if (quantizationParams != null && totalLiveDocs > 0) { + if (totalLiveDocs == 0) { + log.debug("No live docs for field " + fieldInfo.name); + return; + } + QuantizationState quantizationState = null; + QuantizationParams quantizationParams = quantizationService.getQuantizationParams(fieldInfo); + if (quantizationParams != null) { initQuantizationStateWriterIfNecessary(); + KNNVectorValues knnVectorValues = vectorValuesRetriever.apply(vectorDataType, fieldInfo, VectorProcessingContext); quantizationState = quantizationService.train(quantizationParams, knnVectorValues, totalLiveDocs); quantizationStateWriter.writeState(fieldInfo.getFieldNumber(), quantizationState); } - NativeIndexWriter writer = (quantizationParams != null) - ? NativeIndexWriter.getWriter(fieldInfo, segmentWriteState, quantizationState) - : NativeIndexWriter.getWriter(fieldInfo, segmentWriteState); - - knnVectorValues = vectorValuesRetriever.apply(vectorDataType, fieldInfo, VectorProcessingContext); - - StopWatch stopWatch = new StopWatch(); - stopWatch.start(); - indexOperation.buildAndWrite(writer, knnVectorValues, totalLiveDocs); - long time_in_millis = stopWatch.totalTime().millis(); + NativeIndexWriter writer = NativeIndexWriter.getWriter(fieldInfo, segmentWriteState, quantizationState); + KNNVectorValues knnVectors = vectorValuesRetriever.apply(vectorDataType, fieldInfo, VectorProcessingContext); + StopWatch stopWatch = new StopWatch().start(); + indexOperation.buildAndWrite(writer, knnVectors, totalLiveDocs); + long time_in_millis = stopWatch.stop().totalTime().millis(); graphBuildTime.incrementBy(time_in_millis); log.warn("Graph build took " + time_in_millis + " ms for " + operationName); } From 5529a2691f507ec2d0a30a89ad99e9277195c86d Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Fri, 13 Sep 2024 20:24:58 -0700 Subject: [PATCH 370/416] Adds Unit tests for NativeEngines990KnnVectorsWriter (#2097) (#2108) Had to separate out the common code to make it easy to write tests Mocking was difficult to do with the functional interfaces and it was throwing NPE in the test especially with the mock of NativeIndexWriter. Signed-off-by: Tejas Shah (cherry picked from commit 0df06131fa51aa5f5c714472d9bc7153e4d1818f) Co-authored-by: Tejas Shah --- .../NativeEngines990KnnVectorsWriter.java | 152 ++++----- ...eEngines990KnnVectorsWriterFlushTests.java | 288 ++++++++++++++++++ ...eEngines990KnnVectorsWriterMergeTests.java | 240 +++++++++++++++ .../index/vectorvalues/TestVectorValues.java | 6 +- 4 files changed, 585 insertions(+), 101 deletions(-) create mode 100644 src/test/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsWriterFlushTests.java create mode 100644 src/test/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsWriterMergeTests.java diff --git a/src/main/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsWriter.java b/src/main/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsWriter.java index 0d016a60b..3f32003ac 100644 --- a/src/main/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsWriter.java +++ b/src/main/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsWriter.java @@ -25,11 +25,10 @@ import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.RamUsageEstimator; import org.opensearch.common.StopWatch; -import org.opensearch.knn.index.quantizationservice.QuantizationService; import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.codec.nativeindex.NativeIndexWriter; +import org.opensearch.knn.index.quantizationservice.QuantizationService; import org.opensearch.knn.index.vectorvalues.KNNVectorValues; -import org.opensearch.knn.index.vectorvalues.KNNVectorValuesFactory; import org.opensearch.knn.plugin.stats.KNNGraphValue; import org.opensearch.knn.quantization.models.quantizationParams.QuantizationParams; import org.opensearch.knn.quantization.models.quantizationState.QuantizationState; @@ -39,6 +38,7 @@ import java.util.List; import static org.opensearch.knn.common.FieldInfoExtractor.extractVectorDataType; +import static org.opensearch.knn.index.vectorvalues.KNNVectorValuesFactory.getVectorValues; /** * A KNNVectorsWriter class for writing the vector data strcutures and flat vectors for Native Engines. @@ -47,15 +47,11 @@ public class NativeEngines990KnnVectorsWriter extends KnnVectorsWriter { private static final long SHALLOW_SIZE = RamUsageEstimator.shallowSizeOfInstance(NativeEngines990KnnVectorsWriter.class); - private static final String FLUSH_OPERATION = "flush"; - private static final String MERGE_OPERATION = "merge"; - private final SegmentWriteState segmentWriteState; private final FlatVectorsWriter flatVectorsWriter; private KNN990QuantizationStateWriter quantizationStateWriter; private final List> fields = new ArrayList<>(); private boolean finished; - private final QuantizationService quantizationService = QuantizationService.getInstance(); public NativeEngines990KnnVectorsWriter(SegmentWriteState segmentWriteState, FlatVectorsWriter flatVectorsWriter) { this.segmentWriteState = segmentWriteState; @@ -84,14 +80,27 @@ public void flush(int maxDoc, final Sorter.DocMap sortMap) throws IOException { flatVectorsWriter.flush(maxDoc, sortMap); for (final NativeEngineFieldVectorsWriter field : fields) { - trainAndIndex( - field.getFieldInfo(), - (vectorDataType, fieldInfo, fieldVectorsWriter) -> getKNNVectorValues(vectorDataType, fieldVectorsWriter), - NativeIndexWriter::flushIndex, - field, - KNNGraphValue.REFRESH_TOTAL_TIME_IN_MILLIS, - FLUSH_OPERATION - ); + final FieldInfo fieldInfo = field.getFieldInfo(); + final VectorDataType vectorDataType = extractVectorDataType(fieldInfo); + int totalLiveDocs = getLiveDocs(getVectorValues(vectorDataType, field.getDocsWithField(), field.getVectors())); + if (totalLiveDocs > 0) { + KNNVectorValues knnVectorValues = getVectorValues(vectorDataType, field.getDocsWithField(), field.getVectors()); + + final QuantizationState quantizationState = train(field.getFieldInfo(), knnVectorValues, totalLiveDocs); + final NativeIndexWriter writer = NativeIndexWriter.getWriter(fieldInfo, segmentWriteState, quantizationState); + + knnVectorValues = getVectorValues(vectorDataType, field.getDocsWithField(), field.getVectors()); + + StopWatch stopWatch = new StopWatch().start(); + + writer.flushIndex(knnVectorValues, totalLiveDocs); + + long time_in_millis = stopWatch.stop().totalTime().millis(); + KNNGraphValue.REFRESH_TOTAL_TIME_IN_MILLIS.incrementBy(time_in_millis); + log.debug("Flush took {} ms for vector field [{}]", time_in_millis, fieldInfo.getName()); + } else { + log.debug("[Flush] No live docs for field {}", fieldInfo.getName()); + } } } @@ -100,15 +109,26 @@ public void mergeOneField(final FieldInfo fieldInfo, final MergeState mergeState // This will ensure that we are merging the FlatIndex during force merge. flatVectorsWriter.mergeOneField(fieldInfo, mergeState); - // For merge, pick values from flat vector and reindex again. This will use the flush operation to create graphs - trainAndIndex( - fieldInfo, - this::getKNNVectorValuesForMerge, - NativeIndexWriter::mergeIndex, - mergeState, - KNNGraphValue.MERGE_TOTAL_TIME_IN_MILLIS, - MERGE_OPERATION - ); + final VectorDataType vectorDataType = extractVectorDataType(fieldInfo); + int totalLiveDocs = getLiveDocs(getKNNVectorValuesForMerge(vectorDataType, fieldInfo, mergeState)); + if (totalLiveDocs == 0) { + log.debug("[Merge] No live docs for field {}", fieldInfo.getName()); + return; + } + + KNNVectorValues knnVectorValues = getKNNVectorValuesForMerge(vectorDataType, fieldInfo, mergeState); + final QuantizationState quantizationState = train(fieldInfo, knnVectorValues, totalLiveDocs); + final NativeIndexWriter writer = NativeIndexWriter.getWriter(fieldInfo, segmentWriteState, quantizationState); + + knnVectorValues = getKNNVectorValuesForMerge(vectorDataType, fieldInfo, mergeState); + + StopWatch stopWatch = new StopWatch().start(); + + writer.mergeIndex(knnVectorValues, totalLiveDocs); + + long time_in_millis = stopWatch.stop().totalTime().millis(); + KNNGraphValue.MERGE_TOTAL_TIME_IN_MILLIS.incrementBy(time_in_millis); + log.debug("Merge took {} ms for vector field [{}]", time_in_millis, fieldInfo.getName()); } /** @@ -157,18 +177,6 @@ public long ramBytesUsed() { .sum(); } - /** - * Retrieves the {@link KNNVectorValues} for a specific field based on the vector data type and field writer. - * - * @param vectorDataType The {@link VectorDataType} representing the type of vectors stored. - * @param field The {@link NativeEngineFieldVectorsWriter} representing the field from which to retrieve vectors. - * @param The type of vectors being processed. - * @return The {@link KNNVectorValues} associated with the field. - */ - private KNNVectorValues getKNNVectorValues(final VectorDataType vectorDataType, final NativeEngineFieldVectorsWriter field) { - return (KNNVectorValues) KNNVectorValuesFactory.getVectorValues(vectorDataType, field.getDocsWithField(), field.getVectors()); - } - /** * Retrieves the {@link KNNVectorValues} for a specific field during a merge operation, based on the vector data type. * @@ -187,84 +195,28 @@ private KNNVectorValues getKNNVectorValuesForMerge( switch (fieldInfo.getVectorEncoding()) { case FLOAT32: FloatVectorValues mergedFloats = MergedVectorValues.mergeFloatVectorValues(fieldInfo, mergeState); - return (KNNVectorValues) KNNVectorValuesFactory.getVectorValues(vectorDataType, mergedFloats); + return getVectorValues(vectorDataType, mergedFloats); case BYTE: ByteVectorValues mergedBytes = MergedVectorValues.mergeByteVectorValues(fieldInfo, mergeState); - return (KNNVectorValues) KNNVectorValuesFactory.getVectorValues(vectorDataType, mergedBytes); + return getVectorValues(vectorDataType, mergedBytes); default: throw new IllegalStateException("Unsupported vector encoding [" + fieldInfo.getVectorEncoding() + "]"); } } - /** - * Functional interface representing an operation that indexes the provided {@link KNNVectorValues}. - * - * @param The type of vectors being processed. - */ - @FunctionalInterface - private interface IndexOperation { - void buildAndWrite(NativeIndexWriter writer, KNNVectorValues knnVectorValues, int totalLiveDocs) throws IOException; - } - - /** - * Functional interface representing a method that retrieves {@link KNNVectorValues} based on - * the vector data type, field information, and the merge state. - * - * @param The type of the data representing the vector (e.g., {@link VectorDataType}). - * @param The metadata about the field. - * @param The state of the merge operation. - * @param The result of the retrieval, typically {@link KNNVectorValues}. - */ - @FunctionalInterface - private interface VectorValuesRetriever { - Result apply(DataType vectorDataType, FieldInfo fieldInfo, MergeState mergeState) throws IOException; - } + private QuantizationState train(final FieldInfo fieldInfo, final KNNVectorValues knnVectorValues, final int totalLiveDocs) + throws IOException { - /** - * Unified method for processing a field during either the indexing or merge operation. This method retrieves vector values - * based on the provided vector data type and applies the specified index operation, potentially including quantization if needed. - * - * @param fieldInfo The {@link FieldInfo} object containing metadata about the field. - * @param vectorValuesRetriever A functional interface that retrieves {@link KNNVectorValues} based on the vector data type, - * field information, and additional context (e.g., merge state or field writer). - * @param indexOperation A functional interface that performs the indexing operation using the retrieved - * {@link KNNVectorValues}. - * @param VectorProcessingContext The additional context required for retrieving the vector values (e.g., {@link MergeState} or {@link NativeEngineFieldVectorsWriter}). - * From Flush we need NativeFieldWriter which contains total number of vectors while from Merge we need merge state which contains vector information - * @param The type of vectors being processed. - * @param The type of the context needed for retrieving the vector values. - * @throws IOException If an I/O error occurs during the processing. - */ - private void trainAndIndex( - final FieldInfo fieldInfo, - final VectorValuesRetriever> vectorValuesRetriever, - final IndexOperation indexOperation, - final C VectorProcessingContext, - final KNNGraphValue graphBuildTime, - final String operationName - ) throws IOException { - final VectorDataType vectorDataType = extractVectorDataType(fieldInfo); - // Count the docIds - int totalLiveDocs = getLiveDocs(vectorValuesRetriever.apply(vectorDataType, fieldInfo, VectorProcessingContext)); - if (totalLiveDocs == 0) { - log.debug("No live docs for field " + fieldInfo.name); - return; - } + final QuantizationService quantizationService = QuantizationService.getInstance(); + final QuantizationParams quantizationParams = quantizationService.getQuantizationParams(fieldInfo); QuantizationState quantizationState = null; - QuantizationParams quantizationParams = quantizationService.getQuantizationParams(fieldInfo); - if (quantizationParams != null) { + if (quantizationParams != null && totalLiveDocs > 0) { initQuantizationStateWriterIfNecessary(); - KNNVectorValues knnVectorValues = vectorValuesRetriever.apply(vectorDataType, fieldInfo, VectorProcessingContext); quantizationState = quantizationService.train(quantizationParams, knnVectorValues, totalLiveDocs); quantizationStateWriter.writeState(fieldInfo.getFieldNumber(), quantizationState); } - NativeIndexWriter writer = NativeIndexWriter.getWriter(fieldInfo, segmentWriteState, quantizationState); - KNNVectorValues knnVectors = vectorValuesRetriever.apply(vectorDataType, fieldInfo, VectorProcessingContext); - StopWatch stopWatch = new StopWatch().start(); - indexOperation.buildAndWrite(writer, knnVectors, totalLiveDocs); - long time_in_millis = stopWatch.stop().totalTime().millis(); - graphBuildTime.incrementBy(time_in_millis); - log.warn("Graph build took " + time_in_millis + " ms for " + operationName); + + return quantizationState; } /** diff --git a/src/test/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsWriterFlushTests.java b/src/test/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsWriterFlushTests.java new file mode 100644 index 000000000..ad72f5b24 --- /dev/null +++ b/src/test/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsWriterFlushTests.java @@ -0,0 +1,288 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.codec.KNN990Codec; + +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import lombok.RequiredArgsConstructor; +import lombok.SneakyThrows; +import org.apache.lucene.codecs.hnsw.FlatVectorsWriter; +import org.apache.lucene.index.DocsWithFieldSet; +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.SegmentWriteState; +import org.apache.lucene.index.VectorEncoding; +import org.mockito.Mock; +import org.mockito.MockedConstruction; +import org.mockito.MockedStatic; +import org.mockito.MockitoAnnotations; +import org.opensearch.knn.common.KNNConstants; +import org.opensearch.knn.index.VectorDataType; +import org.opensearch.knn.index.codec.nativeindex.NativeIndexWriter; +import org.opensearch.knn.index.quantizationservice.QuantizationService; +import org.opensearch.knn.index.vectorvalues.KNNVectorValues; +import org.opensearch.knn.index.vectorvalues.KNNVectorValuesFactory; +import org.opensearch.knn.index.vectorvalues.TestVectorValues; +import org.opensearch.knn.plugin.stats.KNNGraphValue; +import org.opensearch.knn.quantization.models.quantizationParams.QuantizationParams; +import org.opensearch.knn.quantization.models.quantizationState.QuantizationState; +import org.opensearch.test.OpenSearchTestCase; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.stream.IntStream; + +import static com.carrotsearch.randomizedtesting.RandomizedTest.$; +import static com.carrotsearch.randomizedtesting.RandomizedTest.$$; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.mockConstruction; +import static org.mockito.Mockito.mockStatic; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +@RequiredArgsConstructor +public class NativeEngines990KnnVectorsWriterFlushTests extends OpenSearchTestCase { + + @Mock + private FlatVectorsWriter flatVectorsWriter; + @Mock + private SegmentWriteState segmentWriteState; + @Mock + private QuantizationParams quantizationParams; + @Mock + private QuantizationState quantizationState; + @Mock + private QuantizationService quantizationService; + @Mock + private NativeIndexWriter nativeIndexWriter; + + private NativeEngines990KnnVectorsWriter objectUnderTest; + + private final String description; + private final List> vectorsPerField; + + @Override + public void setUp() throws Exception { + super.setUp(); + MockitoAnnotations.openMocks(this); + objectUnderTest = new NativeEngines990KnnVectorsWriter(segmentWriteState, flatVectorsWriter); + } + + @ParametersFactory + public static Collection data() { + return Arrays.asList( + $$( + $("Single field", List.of(Map.of(0, new float[] { 1, 2, 3 }, 1, new float[] { 2, 3, 4 }, 2, new float[] { 3, 4, 5 }))), + $("Single field, no total live docs", List.of()), + $( + "Multi Field", + List.of( + Map.of(0, new float[] { 1, 2, 3 }, 1, new float[] { 2, 3, 4 }, 2, new float[] { 3, 4, 5 }), + Map.of( + 0, + new float[] { 1, 2, 3, 4 }, + 1, + new float[] { 2, 3, 4, 5 }, + 2, + new float[] { 3, 4, 5, 6 }, + 3, + new float[] { 4, 5, 6, 7 } + ) + ) + ) + ) + ); + } + + @SneakyThrows + public void testFlush() { + // Given + List> expectedVectorValues = new ArrayList<>(); + IntStream.range(0, vectorsPerField.size()).forEach(i -> { + final TestVectorValues.PreDefinedFloatVectorValues randomVectorValues = new TestVectorValues.PreDefinedFloatVectorValues( + new ArrayList<>(vectorsPerField.get(i).values()) + ); + final KNNVectorValues knnVectorValues = KNNVectorValuesFactory.getVectorValues( + VectorDataType.FLOAT, + randomVectorValues + ); + expectedVectorValues.add(knnVectorValues); + + }); + + try ( + MockedStatic fieldWriterMockedStatic = mockStatic(NativeEngineFieldVectorsWriter.class); + MockedStatic knnVectorValuesFactoryMockedStatic = mockStatic(KNNVectorValuesFactory.class); + MockedStatic quantizationServiceMockedStatic = mockStatic(QuantizationService.class); + MockedStatic nativeIndexWriterMockedStatic = mockStatic(NativeIndexWriter.class); + MockedConstruction knn990QuantWriterMockedConstruction = mockConstruction( + KNN990QuantizationStateWriter.class + ); + ) { + quantizationServiceMockedStatic.when(() -> QuantizationService.getInstance()).thenReturn(quantizationService); + IntStream.range(0, vectorsPerField.size()).forEach(i -> { + final FieldInfo fieldInfo = fieldInfo( + i, + VectorEncoding.FLOAT32, + Map.of(KNNConstants.VECTOR_DATA_TYPE_FIELD, "float", KNNConstants.KNN_ENGINE, "faiss") + ); + + NativeEngineFieldVectorsWriter field = nativeEngineFieldVectorsWriter(fieldInfo, vectorsPerField.get(i)); + fieldWriterMockedStatic.when(() -> NativeEngineFieldVectorsWriter.create(fieldInfo, segmentWriteState.infoStream)) + .thenReturn(field); + + try { + objectUnderTest.addField(fieldInfo); + } catch (Exception e) { + throw new RuntimeException(e); + } + + DocsWithFieldSet docsWithFieldSet = field.getDocsWithField(); + knnVectorValuesFactoryMockedStatic.when( + () -> KNNVectorValuesFactory.getVectorValues(VectorDataType.FLOAT, docsWithFieldSet, vectorsPerField.get(i)) + ).thenReturn(expectedVectorValues.get(i)); + + when(quantizationService.getQuantizationParams(fieldInfo)).thenReturn(null); + nativeIndexWriterMockedStatic.when(() -> NativeIndexWriter.getWriter(fieldInfo, segmentWriteState, null)) + .thenReturn(nativeIndexWriter); + }); + + doAnswer(answer -> { + Thread.sleep(2); // Need this for KNNGraph value assertion, removing this will fail the assertion + return null; + }).when(nativeIndexWriter).flushIndex(any(), anyInt()); + + // When + objectUnderTest.flush(5, null); + + // Then + verify(flatVectorsWriter).flush(5, null); + if (vectorsPerField.size() > 0) { + assertEquals(0, knn990QuantWriterMockedConstruction.constructed().size()); + assertNotEquals(0L, (long) KNNGraphValue.REFRESH_TOTAL_TIME_IN_MILLIS.getValue()); + } + + IntStream.range(0, vectorsPerField.size()).forEach(i -> { + try { + verify(nativeIndexWriter).flushIndex(expectedVectorValues.get(i), vectorsPerField.get(i).size()); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + } + } + + @SneakyThrows + public void testFlush_WithQuantization() { + // Given + List> expectedVectorValues = new ArrayList<>(); + IntStream.range(0, vectorsPerField.size()).forEach(i -> { + final TestVectorValues.PreDefinedFloatVectorValues randomVectorValues = new TestVectorValues.PreDefinedFloatVectorValues( + new ArrayList<>(vectorsPerField.get(i).values()) + ); + final KNNVectorValues knnVectorValues = KNNVectorValuesFactory.getVectorValues( + VectorDataType.FLOAT, + randomVectorValues + ); + expectedVectorValues.add(knnVectorValues); + + }); + + try ( + MockedStatic fieldWriterMockedStatic = mockStatic(NativeEngineFieldVectorsWriter.class); + MockedStatic knnVectorValuesFactoryMockedStatic = mockStatic(KNNVectorValuesFactory.class); + MockedStatic quantizationServiceMockedStatic = mockStatic(QuantizationService.class); + MockedStatic nativeIndexWriterMockedStatic = mockStatic(NativeIndexWriter.class); + MockedConstruction knn990QuantWriterMockedConstruction = mockConstruction( + KNN990QuantizationStateWriter.class + ); + ) { + quantizationServiceMockedStatic.when(() -> QuantizationService.getInstance()).thenReturn(quantizationService); + + IntStream.range(0, vectorsPerField.size()).forEach(i -> { + final FieldInfo fieldInfo = fieldInfo( + i, + VectorEncoding.FLOAT32, + Map.of(KNNConstants.VECTOR_DATA_TYPE_FIELD, "float", KNNConstants.KNN_ENGINE, "faiss") + ); + + NativeEngineFieldVectorsWriter field = nativeEngineFieldVectorsWriter(fieldInfo, vectorsPerField.get(i)); + fieldWriterMockedStatic.when(() -> NativeEngineFieldVectorsWriter.create(fieldInfo, segmentWriteState.infoStream)) + .thenReturn(field); + + try { + objectUnderTest.addField(fieldInfo); + } catch (Exception e) { + throw new RuntimeException(e); + } + + DocsWithFieldSet docsWithFieldSet = field.getDocsWithField(); + knnVectorValuesFactoryMockedStatic.when( + () -> KNNVectorValuesFactory.getVectorValues(VectorDataType.FLOAT, docsWithFieldSet, vectorsPerField.get(i)) + ).thenReturn(expectedVectorValues.get(i)); + + when(quantizationService.getQuantizationParams(fieldInfo)).thenReturn(quantizationParams); + try { + when(quantizationService.train(quantizationParams, expectedVectorValues.get(i), vectorsPerField.get(i).size())) + .thenReturn(quantizationState); + } catch (Exception e) { + throw new RuntimeException(e); + } + + nativeIndexWriterMockedStatic.when(() -> NativeIndexWriter.getWriter(fieldInfo, segmentWriteState, quantizationState)) + .thenReturn(nativeIndexWriter); + }); + doAnswer(answer -> { + Thread.sleep(2); // Need this for KNNGraph value assertion, removing this will fail the assertion + return null; + }).when(nativeIndexWriter).flushIndex(any(), anyInt()); + + // When + objectUnderTest.flush(5, null); + + // Then + verify(flatVectorsWriter).flush(5, null); + if (vectorsPerField.size() > 0) { + verify(knn990QuantWriterMockedConstruction.constructed().get(0)).writeHeader(segmentWriteState); + assertTrue(KNNGraphValue.REFRESH_TOTAL_TIME_IN_MILLIS.getValue() > 0L); + } else { + assertEquals(0, knn990QuantWriterMockedConstruction.constructed().size()); + } + + IntStream.range(0, vectorsPerField.size()).forEach(i -> { + try { + verify(knn990QuantWriterMockedConstruction.constructed().get(0)).writeState(i, quantizationState); + verify(nativeIndexWriter).flushIndex(expectedVectorValues.get(i), vectorsPerField.get(i).size()); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + } + } + + private FieldInfo fieldInfo(int fieldNumber, VectorEncoding vectorEncoding, Map attributes) { + FieldInfo fieldInfo = mock(FieldInfo.class); + when(fieldInfo.getFieldNumber()).thenReturn(fieldNumber); + when(fieldInfo.getVectorEncoding()).thenReturn(vectorEncoding); + when(fieldInfo.attributes()).thenReturn(attributes); + attributes.forEach((key, value) -> when(fieldInfo.getAttribute(key)).thenReturn(value)); + return fieldInfo; + } + + private NativeEngineFieldVectorsWriter nativeEngineFieldVectorsWriter(FieldInfo fieldInfo, Map vectors) { + NativeEngineFieldVectorsWriter fieldVectorsWriter = mock(NativeEngineFieldVectorsWriter.class); + DocsWithFieldSet docsWithFieldSet = new DocsWithFieldSet(); + vectors.keySet().stream().sorted().forEach(docsWithFieldSet::add); + when(fieldVectorsWriter.getFieldInfo()).thenReturn(fieldInfo); + when(fieldVectorsWriter.getVectors()).thenReturn(vectors); + when(fieldVectorsWriter.getDocsWithField()).thenReturn(docsWithFieldSet); + return fieldVectorsWriter; + } +} diff --git a/src/test/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsWriterMergeTests.java b/src/test/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsWriterMergeTests.java new file mode 100644 index 000000000..440e8bbc5 --- /dev/null +++ b/src/test/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsWriterMergeTests.java @@ -0,0 +1,240 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.codec.KNN990Codec; + +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import lombok.RequiredArgsConstructor; +import lombok.SneakyThrows; +import org.apache.lucene.codecs.KnnVectorsWriter; +import org.apache.lucene.codecs.hnsw.FlatVectorsWriter; +import org.apache.lucene.index.DocsWithFieldSet; +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.FloatVectorValues; +import org.apache.lucene.index.MergeState; +import org.apache.lucene.index.SegmentWriteState; +import org.apache.lucene.index.VectorEncoding; +import org.mockito.Mock; +import org.mockito.MockedConstruction; +import org.mockito.MockedStatic; +import org.mockito.MockitoAnnotations; +import org.opensearch.knn.common.KNNConstants; +import org.opensearch.knn.index.VectorDataType; +import org.opensearch.knn.index.codec.nativeindex.NativeIndexWriter; +import org.opensearch.knn.index.quantizationservice.QuantizationService; +import org.opensearch.knn.index.vectorvalues.KNNVectorValues; +import org.opensearch.knn.index.vectorvalues.KNNVectorValuesFactory; +import org.opensearch.knn.index.vectorvalues.TestVectorValues; +import org.opensearch.knn.plugin.stats.KNNGraphValue; +import org.opensearch.knn.quantization.models.quantizationParams.QuantizationParams; +import org.opensearch.knn.quantization.models.quantizationState.QuantizationState; +import org.opensearch.test.OpenSearchTestCase; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Map; + +import static com.carrotsearch.randomizedtesting.RandomizedTest.$; +import static com.carrotsearch.randomizedtesting.RandomizedTest.$$; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.mockConstruction; +import static org.mockito.Mockito.mockStatic; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.when; + +@RequiredArgsConstructor +public class NativeEngines990KnnVectorsWriterMergeTests extends OpenSearchTestCase { + + @Mock + private FlatVectorsWriter flatVectorsWriter; + @Mock + private SegmentWriteState segmentWriteState; + @Mock + private QuantizationParams quantizationParams; + @Mock + private QuantizationState quantizationState; + @Mock + private QuantizationService quantizationService; + @Mock + private NativeIndexWriter nativeIndexWriter; + @Mock + private FloatVectorValues floatVectorValues; + @Mock + private MergeState mergeState; + + private NativeEngines990KnnVectorsWriter objectUnderTest; + + private final String description; + private final Map mergedVectors; + + @Override + public void setUp() throws Exception { + super.setUp(); + MockitoAnnotations.openMocks(this); + objectUnderTest = new NativeEngines990KnnVectorsWriter(segmentWriteState, flatVectorsWriter); + } + + @ParametersFactory + public static Collection data() { + return Arrays.asList( + $$( + $("Merge one field", Map.of(0, new float[] { 1, 2, 3 }, 1, new float[] { 2, 3, 4 }, 2, new float[] { 3, 4, 5 })), + $("Merge, no live docs", Map.of()) + ) + ); + } + + @SneakyThrows + public void testMerge() { + // Given + final TestVectorValues.PreDefinedFloatVectorValues randomVectorValues = new TestVectorValues.PreDefinedFloatVectorValues( + new ArrayList<>(mergedVectors.values()) + ); + final KNNVectorValues knnVectorValues = KNNVectorValuesFactory.getVectorValues(VectorDataType.FLOAT, randomVectorValues); + + try ( + MockedStatic fieldWriterMockedStatic = mockStatic(NativeEngineFieldVectorsWriter.class); + MockedStatic knnVectorValuesFactoryMockedStatic = mockStatic(KNNVectorValuesFactory.class); + MockedStatic quantizationServiceMockedStatic = mockStatic(QuantizationService.class); + MockedStatic nativeIndexWriterMockedStatic = mockStatic(NativeIndexWriter.class); + MockedStatic mergedVectorValuesMockedStatic = mockStatic( + KnnVectorsWriter.MergedVectorValues.class + ); + MockedConstruction knn990QuantWriterMockedConstruction = mockConstruction( + KNN990QuantizationStateWriter.class + ); + ) { + quantizationServiceMockedStatic.when(() -> QuantizationService.getInstance()).thenReturn(quantizationService); + final FieldInfo fieldInfo = fieldInfo( + 0, + VectorEncoding.FLOAT32, + Map.of(KNNConstants.VECTOR_DATA_TYPE_FIELD, "float", KNNConstants.KNN_ENGINE, "faiss") + ); + + NativeEngineFieldVectorsWriter field = nativeEngineFieldVectorsWriter(fieldInfo, mergedVectors); + fieldWriterMockedStatic.when(() -> NativeEngineFieldVectorsWriter.create(fieldInfo, segmentWriteState.infoStream)) + .thenReturn(field); + + mergedVectorValuesMockedStatic.when(() -> KnnVectorsWriter.MergedVectorValues.mergeFloatVectorValues(fieldInfo, mergeState)) + .thenReturn(floatVectorValues); + knnVectorValuesFactoryMockedStatic.when(() -> KNNVectorValuesFactory.getVectorValues(VectorDataType.FLOAT, floatVectorValues)) + .thenReturn(knnVectorValues); + + when(quantizationService.getQuantizationParams(fieldInfo)).thenReturn(null); + nativeIndexWriterMockedStatic.when(() -> NativeIndexWriter.getWriter(fieldInfo, segmentWriteState, null)) + .thenReturn(nativeIndexWriter); + doAnswer(answer -> { + Thread.sleep(2); // Need this for KNNGraph value assertion, removing this will fail the assertion + return null; + }).when(nativeIndexWriter).mergeIndex(any(), anyInt()); + + // When + objectUnderTest.mergeOneField(fieldInfo, mergeState); + + // Then + verify(flatVectorsWriter).mergeOneField(fieldInfo, mergeState); + assertEquals(0, knn990QuantWriterMockedConstruction.constructed().size()); + if (!mergedVectors.isEmpty()) { + verify(nativeIndexWriter).mergeIndex(knnVectorValues, mergedVectors.size()); + assertTrue(KNNGraphValue.MERGE_TOTAL_TIME_IN_MILLIS.getValue() > 0L); + } else { + verifyNoInteractions(nativeIndexWriter); + } + } + } + + @SneakyThrows + public void testMerge_WithQuantization() { + // Given + final TestVectorValues.PreDefinedFloatVectorValues randomVectorValues = new TestVectorValues.PreDefinedFloatVectorValues( + new ArrayList<>(mergedVectors.values()) + ); + final KNNVectorValues knnVectorValues = KNNVectorValuesFactory.getVectorValues(VectorDataType.FLOAT, randomVectorValues); + + try ( + MockedStatic fieldWriterMockedStatic = mockStatic(NativeEngineFieldVectorsWriter.class); + MockedStatic knnVectorValuesFactoryMockedStatic = mockStatic(KNNVectorValuesFactory.class); + MockedStatic quantizationServiceMockedStatic = mockStatic(QuantizationService.class); + MockedStatic nativeIndexWriterMockedStatic = mockStatic(NativeIndexWriter.class); + MockedConstruction knn990QuantWriterMockedConstruction = mockConstruction( + KNN990QuantizationStateWriter.class + ); + MockedStatic mergedVectorValuesMockedStatic = mockStatic( + KnnVectorsWriter.MergedVectorValues.class + ); + ) { + quantizationServiceMockedStatic.when(() -> QuantizationService.getInstance()).thenReturn(quantizationService); + + final FieldInfo fieldInfo = fieldInfo( + 0, + VectorEncoding.FLOAT32, + Map.of(KNNConstants.VECTOR_DATA_TYPE_FIELD, "float", KNNConstants.KNN_ENGINE, "faiss") + ); + + NativeEngineFieldVectorsWriter field = nativeEngineFieldVectorsWriter(fieldInfo, mergedVectors); + fieldWriterMockedStatic.when(() -> NativeEngineFieldVectorsWriter.create(fieldInfo, segmentWriteState.infoStream)) + .thenReturn(field); + + mergedVectorValuesMockedStatic.when(() -> KnnVectorsWriter.MergedVectorValues.mergeFloatVectorValues(fieldInfo, mergeState)) + .thenReturn(floatVectorValues); + knnVectorValuesFactoryMockedStatic.when(() -> KNNVectorValuesFactory.getVectorValues(VectorDataType.FLOAT, floatVectorValues)) + .thenReturn(knnVectorValues); + + when(quantizationService.getQuantizationParams(fieldInfo)).thenReturn(quantizationParams); + try { + when(quantizationService.train(quantizationParams, knnVectorValues, mergedVectors.size())).thenReturn(quantizationState); + } catch (Exception e) { + throw new RuntimeException(e); + } + + nativeIndexWriterMockedStatic.when(() -> NativeIndexWriter.getWriter(fieldInfo, segmentWriteState, quantizationState)) + .thenReturn(nativeIndexWriter); + doAnswer(answer -> { + Thread.sleep(2); // Need this for KNNGraph value assertion, removing this will fail the assertion + return null; + }).when(nativeIndexWriter).mergeIndex(any(), anyInt()); + + // When + objectUnderTest.mergeOneField(fieldInfo, mergeState); + + // Then + verify(flatVectorsWriter).mergeOneField(fieldInfo, mergeState); + if (!mergedVectors.isEmpty()) { + verify(knn990QuantWriterMockedConstruction.constructed().get(0)).writeHeader(segmentWriteState); + verify(knn990QuantWriterMockedConstruction.constructed().get(0)).writeState(0, quantizationState); + verify(nativeIndexWriter).mergeIndex(knnVectorValues, mergedVectors.size()); + assertTrue(KNNGraphValue.MERGE_TOTAL_TIME_IN_MILLIS.getValue() > 0L); + } else { + assertEquals(0, knn990QuantWriterMockedConstruction.constructed().size()); + verifyNoInteractions(nativeIndexWriter); + } + + } + } + + private FieldInfo fieldInfo(int fieldNumber, VectorEncoding vectorEncoding, Map attributes) { + FieldInfo fieldInfo = mock(FieldInfo.class); + when(fieldInfo.getFieldNumber()).thenReturn(fieldNumber); + when(fieldInfo.getVectorEncoding()).thenReturn(vectorEncoding); + when(fieldInfo.attributes()).thenReturn(attributes); + attributes.forEach((key, value) -> when(fieldInfo.getAttribute(key)).thenReturn(value)); + return fieldInfo; + } + + private NativeEngineFieldVectorsWriter nativeEngineFieldVectorsWriter(FieldInfo fieldInfo, Map vectors) { + NativeEngineFieldVectorsWriter fieldVectorsWriter = mock(NativeEngineFieldVectorsWriter.class); + DocsWithFieldSet docsWithFieldSet = new DocsWithFieldSet(); + vectors.keySet().stream().sorted().forEach(docsWithFieldSet::add); + when(fieldVectorsWriter.getFieldInfo()).thenReturn(fieldInfo); + when(fieldVectorsWriter.getVectors()).thenReturn(vectors); + when(fieldVectorsWriter.getDocsWithField()).thenReturn(docsWithFieldSet); + return fieldVectorsWriter; + } +} diff --git a/src/test/java/org/opensearch/knn/index/vectorvalues/TestVectorValues.java b/src/test/java/org/opensearch/knn/index/vectorvalues/TestVectorValues.java index 3bf79b004..0f15d5240 100644 --- a/src/test/java/org/opensearch/knn/index/vectorvalues/TestVectorValues.java +++ b/src/test/java/org/opensearch/knn/index/vectorvalues/TestVectorValues.java @@ -184,7 +184,11 @@ public static class PreDefinedFloatVectorValues extends FloatVectorValues { public PreDefinedFloatVectorValues(final List vectors) { super(); this.count = vectors.size(); - this.dimension = vectors.get(0).length; + if (!vectors.isEmpty()) { + this.dimension = vectors.get(0).length; + } else { + this.dimension = 0; + } this.vectors = vectors; this.current = -1; vector = new float[dimension]; From cd4e75dfa67bd475a44d41042e7f9ddc89f1f69b Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Fri, 13 Sep 2024 23:33:16 -0700 Subject: [PATCH 371/416] Add DocValuesProducers for releasing memory when close index (#1946) (#2109) Add DocValuesProducers for releasing memory when close index #1946 (cherry picked from commit 004fcc0aa93476ce20e9940baf24d69f62a8f47b) Co-authored-by: luyuncheng --- CHANGELOG.md | 3 +- .../KNN80Codec/KNN80CompoundDirectory.java | 63 ++++++++ .../codec/KNN80Codec/KNN80CompoundFormat.java | 2 +- .../KNN80Codec/KNN80DocValuesFormat.java | 2 +- .../KNN80Codec/KNN80DocValuesProducer.java | 143 ++++++++++++++++++ .../knn/index/codec/util/KNNCodecUtil.java | 31 ++++ .../index/memory/NativeMemoryAllocation.java | 43 +++++- .../opensearch/knn/index/query/KNNWeight.java | 26 +--- .../opensearch/knn/index/OpenSearchIT.java | 124 +++++++++++++++ .../KNN80Codec/KNN80CompoundFormatTests.java | 4 +- .../KNN80DocValuesProducerTests.java | 130 ++++++++++++++++ .../index/codec/util/KNNCodecUtilTests.java | 18 +++ .../knn/index/query/KNNWeightTests.java | 3 +- .../org/opensearch/knn/KNNRestTestCase.java | 6 + 14 files changed, 569 insertions(+), 29 deletions(-) create mode 100644 src/main/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80CompoundDirectory.java create mode 100644 src/main/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesProducer.java create mode 100644 src/test/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesProducerTests.java diff --git a/CHANGELOG.md b/CHANGELOG.md index fcdbe1974..f70ca5e24 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ## [Unreleased 3.0](https://github.com/opensearch-project/k-NN/compare/2.x...HEAD) ### Features ### Enhancements -### Bug Fixes +### Bug Fixes +* Add DocValuesProducers for releasing memory when close index [#1946](https://github.com/opensearch-project/k-NN/pull/1946) ### Infrastructure ### Documentation ### Maintenance diff --git a/src/main/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80CompoundDirectory.java b/src/main/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80CompoundDirectory.java new file mode 100644 index 000000000..0821b19ef --- /dev/null +++ b/src/main/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80CompoundDirectory.java @@ -0,0 +1,63 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.codec.KNN80Codec; + +import lombok.Getter; +import org.apache.lucene.codecs.CompoundDirectory; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IOContext; +import org.apache.lucene.store.IndexInput; +import org.opensearch.knn.index.engine.KNNEngine; + +import java.io.IOException; +import java.util.Set; + +public class KNN80CompoundDirectory extends CompoundDirectory { + + @Getter + private CompoundDirectory delegate; + @Getter + private Directory dir; + + public KNN80CompoundDirectory(CompoundDirectory delegate, Directory dir) { + this.delegate = delegate; + this.dir = dir; + } + + @Override + public void checkIntegrity() throws IOException { + delegate.checkIntegrity(); + } + + @Override + public String[] listAll() throws IOException { + return delegate.listAll(); + } + + @Override + public long fileLength(String name) throws IOException { + return delegate.fileLength(name); + } + + @Override + public IndexInput openInput(String name, IOContext context) throws IOException { + if (KNNEngine.getEnginesThatCreateCustomSegmentFiles().stream().anyMatch(engine -> name.endsWith(engine.getCompoundExtension()))) { + return dir.openInput(name, context); + } + return delegate.openInput(name, context); + } + + @Override + public void close() throws IOException { + delegate.close(); + } + + @Override + public Set getPendingDeletions() throws IOException { + return delegate.getPendingDeletions(); + } + +} diff --git a/src/main/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80CompoundFormat.java b/src/main/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80CompoundFormat.java index 0f51bdcd5..24dbfb78b 100644 --- a/src/main/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80CompoundFormat.java +++ b/src/main/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80CompoundFormat.java @@ -41,7 +41,7 @@ public KNN80CompoundFormat(CompoundFormat delegate) { @Override public CompoundDirectory getCompoundReader(Directory dir, SegmentInfo si, IOContext context) throws IOException { - return delegate.getCompoundReader(dir, si, context); + return new KNN80CompoundDirectory(delegate.getCompoundReader(dir, si, context), dir); } @Override diff --git a/src/main/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesFormat.java b/src/main/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesFormat.java index fe329eb1c..7e45270b6 100644 --- a/src/main/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesFormat.java +++ b/src/main/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesFormat.java @@ -41,6 +41,6 @@ public DocValuesConsumer fieldsConsumer(SegmentWriteState state) throws IOExcept @Override public DocValuesProducer fieldsProducer(SegmentReadState state) throws IOException { - return delegate.fieldsProducer(state); + return new KNN80DocValuesProducer(delegate.fieldsProducer(state), state); } } diff --git a/src/main/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesProducer.java b/src/main/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesProducer.java new file mode 100644 index 000000000..0cfd9c668 --- /dev/null +++ b/src/main/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesProducer.java @@ -0,0 +1,143 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.knn.index.codec.KNN80Codec; + +import lombok.NonNull; +import lombok.extern.log4j.Log4j2; +import org.apache.lucene.codecs.DocValuesProducer; +import org.apache.lucene.index.BinaryDocValues; +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.NumericDocValues; +import org.apache.lucene.index.SegmentReadState; +import org.apache.lucene.index.SortedDocValues; +import org.apache.lucene.index.SortedNumericDocValues; +import org.apache.lucene.index.SortedSetDocValues; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.FSDirectory; +import org.apache.lucene.store.FilterDirectory; +import org.opensearch.common.io.PathUtils; +import org.opensearch.knn.common.FieldInfoExtractor; +import org.opensearch.knn.index.codec.util.KNNCodecUtil; +import org.opensearch.knn.index.engine.KNNEngine; +import org.opensearch.knn.index.memory.NativeMemoryCacheManager; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.opensearch.knn.common.KNNConstants.MODEL_ID; +import static org.opensearch.knn.index.mapper.KNNVectorFieldMapper.KNN_FIELD; + +@Log4j2 +public class KNN80DocValuesProducer extends DocValuesProducer { + + private final SegmentReadState state; + private final DocValuesProducer delegate; + private final NativeMemoryCacheManager nativeMemoryCacheManager; + private final Map indexPathMap = new HashMap(); + + public KNN80DocValuesProducer(DocValuesProducer delegate, SegmentReadState state) { + this.delegate = delegate; + this.state = state; + this.nativeMemoryCacheManager = NativeMemoryCacheManager.getInstance(); + + Directory directory = state.directory; + // directory would be CompoundDirectory, we need get directory firstly and then unwrap + if (state.directory instanceof KNN80CompoundDirectory) { + directory = ((KNN80CompoundDirectory) state.directory).getDir(); + } + + Directory dir = FilterDirectory.unwrap(directory); + if (!(dir instanceof FSDirectory)) { + log.warn("{} can not casting to FSDirectory", directory); + return; + } + String directoryPath = ((FSDirectory) dir).getDirectory().toString(); + for (FieldInfo field : state.fieldInfos) { + if (!field.attributes().containsKey(KNN_FIELD)) { + continue; + } + // Only Native Engine put into indexPathMap + KNNEngine knnEngine = getNativeKNNEngine(field); + if (knnEngine == null) { + continue; + } + List engineFiles = KNNCodecUtil.getEngineFiles(knnEngine.getExtension(), field.name, state.segmentInfo); + Path indexPath = PathUtils.get(directoryPath, engineFiles.get(0)); + indexPathMap.putIfAbsent(field.getName(), indexPath.toString()); + } + } + + @Override + public BinaryDocValues getBinary(FieldInfo field) throws IOException { + return delegate.getBinary(field); + } + + @Override + public NumericDocValues getNumeric(FieldInfo field) throws IOException { + return delegate.getNumeric(field); + } + + @Override + public SortedDocValues getSorted(FieldInfo field) throws IOException { + return delegate.getSorted(field); + } + + @Override + public SortedNumericDocValues getSortedNumeric(FieldInfo field) throws IOException { + return delegate.getSortedNumeric(field); + } + + @Override + public SortedSetDocValues getSortedSet(FieldInfo field) throws IOException { + return delegate.getSortedSet(field); + } + + @Override + public void checkIntegrity() throws IOException { + delegate.checkIntegrity(); + } + + @Override + public void close() throws IOException { + for (String path : indexPathMap.values()) { + nativeMemoryCacheManager.invalidate(path); + } + delegate.close(); + } + + public final List getOpenedIndexPath() { + return new ArrayList<>(indexPathMap.values()); + } + + /** + * Get KNNEngine From FieldInfo + * + * @param field which field we need produce from engine + * @return if and only if Native Engine we return specific engine, else return null + */ + private KNNEngine getNativeKNNEngine(@NonNull FieldInfo field) { + + final String modelId = field.attributes().get(MODEL_ID); + if (modelId != null) { + return null; + } + KNNEngine engine = FieldInfoExtractor.extractKNNEngine(field); + if (KNNEngine.getEnginesThatCreateCustomSegmentFiles().contains(engine)) { + return engine; + } + return null; + } +} diff --git a/src/main/java/org/opensearch/knn/index/codec/util/KNNCodecUtil.java b/src/main/java/org/opensearch/knn/index/codec/util/KNNCodecUtil.java index 51100a1e0..84c7c4675 100644 --- a/src/main/java/org/opensearch/knn/index/codec/util/KNNCodecUtil.java +++ b/src/main/java/org/opensearch/knn/index/codec/util/KNNCodecUtil.java @@ -6,8 +6,15 @@ package org.opensearch.knn.index.codec.util; import org.apache.lucene.index.BinaryDocValues; +import org.apache.lucene.index.SegmentInfo; +import org.opensearch.knn.common.KNNConstants; import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.codec.KNN80Codec.KNN80BinaryDocValues; +import org.opensearch.knn.index.engine.KNNEngine; + +import java.util.Comparator; +import java.util.List; +import java.util.stream.Collectors; public class KNNCodecUtil { // Floats are 4 bytes in size @@ -53,4 +60,28 @@ public static long getTotalLiveDocsCount(final BinaryDocValues binaryDocValues) } return totalLiveDocs; } + + /** + * Get Engine Files from segment with specific fieldName and engine extension + * + * @param extension Engine extension comes from {@link KNNEngine#getExtension()}} + * @param fieldName Filed for knn field + * @param segmentInfo {@link SegmentInfo} One Segment info to use for compute. + * @return List of engine files + */ + public static List getEngineFiles(String extension, String fieldName, SegmentInfo segmentInfo) { + /* + * In case of compound file, extension would be + c otherwise + */ + String engineExtension = segmentInfo.getUseCompoundFile() ? extension + KNNConstants.COMPOUND_EXTENSION : extension; + String engineSuffix = fieldName + engineExtension; + String underLineEngineSuffix = "_" + engineSuffix; + + List engineFiles = segmentInfo.files() + .stream() + .filter(fileName -> fileName.endsWith(underLineEngineSuffix)) + .sorted(Comparator.comparingInt(String::length)) + .collect(Collectors.toList()); + return engineFiles; + } } diff --git a/src/main/java/org/opensearch/knn/index/memory/NativeMemoryAllocation.java b/src/main/java/org/opensearch/knn/index/memory/NativeMemoryAllocation.java index 02b480ed4..f42b256c8 100644 --- a/src/main/java/org/opensearch/knn/index/memory/NativeMemoryAllocation.java +++ b/src/main/java/org/opensearch/knn/index/memory/NativeMemoryAllocation.java @@ -14,6 +14,7 @@ import lombok.Getter; import org.apache.lucene.index.LeafReaderContext; import org.opensearch.knn.common.featureflags.KNNFeatureFlags; +import org.opensearch.common.concurrent.RefCountedReleasable; import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.query.KNNWeight; import org.opensearch.knn.jni.JNIService; @@ -79,6 +80,26 @@ public interface NativeMemoryAllocation { */ int getSizeInKB(); + /** + * Increments the refCount of this instance. + * + * @see #decRef + * @throws IllegalStateException iff the reference counter can not be incremented. + */ + default void incRef() {} + + /** + * Decreases the refCount of this instance. If the refCount drops to 0, then this + * instance is considered as closed and should not be used anymore. + * + * @see #incRef + * + * @return returns {@code true} if the ref count dropped to 0 as a result of calling this method + */ + default boolean decRef() { + return true; + } + /** * Represents native indices loaded into memory. Because these indices are backed by files, they should be * freed when file is deleted. @@ -100,6 +121,7 @@ class IndexAllocation implements NativeMemoryAllocation { private final SharedIndexState sharedIndexState; @Getter private final boolean isBinaryIndex; + private final RefCountedReleasable refCounted; /** * Constructor @@ -158,10 +180,10 @@ class IndexAllocation implements NativeMemoryAllocation { this.watcherHandle = watcherHandle; this.sharedIndexState = sharedIndexState; this.isBinaryIndex = isBinaryIndex; + this.refCounted = new RefCountedReleasable<>("IndexAllocation-Reference", this, this::closeInternal); } - @Override - public void close() { + protected void closeInternal() { Runnable onClose = () -> { writeLock(); cleanup(); @@ -177,6 +199,13 @@ public void close() { } } + @Override + public void close() { + if (!closed && refCounted.refCount() > 0) { + refCounted.close(); + } + } + private void cleanup() { if (this.closed) { return; @@ -240,6 +269,16 @@ public void writeUnlock() { public int getSizeInKB() { return size; } + + @Override + public void incRef() { + refCounted.incRef(); + } + + @Override + public boolean decRef() { + return refCounted.decRef(); + } } /** diff --git a/src/main/java/org/opensearch/knn/index/query/KNNWeight.java b/src/main/java/org/opensearch/knn/index/query/KNNWeight.java index b1ba9de59..1c31ed725 100644 --- a/src/main/java/org/opensearch/knn/index/query/KNNWeight.java +++ b/src/main/java/org/opensearch/knn/index/query/KNNWeight.java @@ -5,7 +5,6 @@ package org.opensearch.knn.index.query; -import com.google.common.annotations.VisibleForTesting; import lombok.extern.log4j.Log4j2; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.LeafReaderContext; @@ -27,6 +26,7 @@ import org.opensearch.knn.index.KNNSettings; import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.VectorDataType; +import org.opensearch.knn.index.codec.util.KNNCodecUtil; import org.opensearch.knn.index.memory.NativeMemoryAllocation; import org.opensearch.knn.index.memory.NativeMemoryCacheManager; import org.opensearch.knn.index.memory.NativeMemoryEntryContext; @@ -43,7 +43,6 @@ import java.nio.file.Path; import java.util.Arrays; import java.util.Collections; -import java.util.Comparator; import java.util.List; import java.util.Map; import java.util.concurrent.ExecutionException; @@ -272,7 +271,7 @@ private Map doANNSearch( // TODO: Change type of vector once more quantization methods are supported final byte[] quantizedVector = SegmentLevelQuantizationUtil.quantizeVector(knnQuery.getQueryVector(), segmentLevelQuantizationInfo); - List engineFiles = getEngineFiles(reader, knnEngine.getExtension()); + List engineFiles = KNNCodecUtil.getEngineFiles(knnEngine.getExtension(), knnQuery.getField(), reader.getSegmentInfo().info); if (engineFiles.isEmpty()) { log.debug("[KNN] No engine index found for field {} for segment {}", knnQuery.getField(), reader.getSegmentName()); return null; @@ -312,6 +311,7 @@ private Map doANNSearch( FilterIdsSelector.FilterIdsSelectorType filterType = filterIdsSelector.getFilterType(); // Now that we have the allocation, we need to readLock it indexAllocation.readLock(); + indexAllocation.incRef(); try { if (indexAllocation.isClosed()) { throw new RuntimeException("Index has already been closed"); @@ -361,6 +361,7 @@ private Map doANNSearch( throw new RuntimeException(e); } finally { indexAllocation.readUnlock(); + indexAllocation.decRef(); } /* @@ -378,25 +379,6 @@ private Map doANNSearch( .collect(Collectors.toMap(KNNQueryResult::getId, result -> knnEngine.score(result.getScore(), spaceType))); } - @VisibleForTesting - List getEngineFiles(SegmentReader reader, String extension) throws IOException { - /* - * In case of compound file, extension would be + c otherwise - */ - String engineExtension = reader.getSegmentInfo().info.getUseCompoundFile() - ? extension + KNNConstants.COMPOUND_EXTENSION - : extension; - String engineSuffix = knnQuery.getField() + engineExtension; - String underLineEngineSuffix = "_" + engineSuffix; - List engineFiles = reader.getSegmentInfo() - .files() - .stream() - .filter(fileName -> fileName.endsWith(underLineEngineSuffix)) - .sorted(Comparator.comparingInt(String::length)) - .collect(Collectors.toList()); - return engineFiles; - } - /** * Execute exact search for the given matched doc ids and return the results as a map of docId to score. * diff --git a/src/test/java/org/opensearch/knn/index/OpenSearchIT.java b/src/test/java/org/opensearch/knn/index/OpenSearchIT.java index c1d3b47c3..3ef22c758 100644 --- a/src/test/java/org/opensearch/knn/index/OpenSearchIT.java +++ b/src/test/java/org/opensearch/knn/index/OpenSearchIT.java @@ -483,4 +483,128 @@ public void testIndexingVectorValidation_updateVectorWithNull() throws Exception assertArrayEquals(vectorForDocumentOne, vectorRestoreInitialValue); } + public void testCacheClear_whenCloseIndex() throws Exception { + String indexName = "test-index-1"; + KNNEngine knnEngine1 = KNNEngine.NMSLIB; + KNNEngine knnEngine2 = KNNEngine.FAISS; + String fieldName1 = "test-field-1"; + String fieldName2 = "test-field-2"; + SpaceType spaceType1 = SpaceType.COSINESIMIL; + SpaceType spaceType2 = SpaceType.L2; + + List mValues = ImmutableList.of(16, 32, 64, 128); + List efConstructionValues = ImmutableList.of(16, 32, 64, 128); + List efSearchValues = ImmutableList.of(16, 32, 64, 128); + + Integer dimension = testData.indexData.vectors[0].length; + + // Create an index + XContentBuilder builder = XContentFactory.jsonBuilder() + .startObject() + .startObject("properties") + .startObject(fieldName1) + .field("type", "knn_vector") + .field("dimension", dimension) + .startObject(KNNConstants.KNN_METHOD) + .field(KNNConstants.NAME, KNNConstants.METHOD_HNSW) + .field(KNNConstants.METHOD_PARAMETER_SPACE_TYPE, spaceType1.getValue()) + .field(KNNConstants.KNN_ENGINE, knnEngine1.getName()) + .startObject(KNNConstants.PARAMETERS) + .field(KNNConstants.METHOD_PARAMETER_M, mValues.get(random().nextInt(mValues.size()))) + .field(KNNConstants.METHOD_PARAMETER_EF_CONSTRUCTION, efConstructionValues.get(random().nextInt(efConstructionValues.size()))) + .endObject() + .endObject() + .endObject() + .startObject(fieldName2) + .field("type", "knn_vector") + .field("dimension", dimension) + .startObject(KNNConstants.KNN_METHOD) + .field(KNNConstants.NAME, KNNConstants.METHOD_HNSW) + .field(KNNConstants.METHOD_PARAMETER_SPACE_TYPE, spaceType2.getValue()) + .field(KNNConstants.KNN_ENGINE, knnEngine2.getName()) + .startObject(KNNConstants.PARAMETERS) + .field(KNNConstants.METHOD_PARAMETER_M, mValues.get(random().nextInt(mValues.size()))) + .field(KNNConstants.METHOD_PARAMETER_EF_CONSTRUCTION, efConstructionValues.get(random().nextInt(efConstructionValues.size()))) + .field(KNNConstants.METHOD_PARAMETER_EF_SEARCH, efSearchValues.get(random().nextInt(efSearchValues.size()))) + .endObject() + .endObject() + .endObject() + .endObject() + .endObject(); + + Map mappingMap = xContentBuilderToMap(builder); + String mapping = builder.toString(); + createKnnIndex(indexName, mapping); + assertEquals(new TreeMap<>(mappingMap), new TreeMap<>(getIndexMappingAsMap(indexName))); + + // Index the test data + for (int i = 0; i < testData.indexData.docs.length; i++) { + addKnnDoc( + indexName, + Integer.toString(testData.indexData.docs[i]), + ImmutableList.of(fieldName1, fieldName2), + ImmutableList.of( + Floats.asList(testData.indexData.vectors[i]).toArray(), + Floats.asList(testData.indexData.vectors[i]).toArray() + ) + ); + } + + // Assert we have the right number of documents in the index + refreshAllIndices(); + assertEquals(testData.indexData.docs.length, getDocCount(indexName)); + + int k = 10; + for (int i = 0; i < testData.queries.length; i++) { + // Search the first field + Response response = searchKNNIndex(indexName, new KNNQueryBuilder(fieldName1, testData.queries[i], k), k); + String responseBody = EntityUtils.toString(response.getEntity()); + List knnResults = parseSearchResponse(responseBody, fieldName1); + assertEquals(k, knnResults.size()); + + List actualScores = parseSearchResponseScore(responseBody, fieldName1); + for (int j = 0; j < k; j++) { + float[] primitiveArray = knnResults.get(j).getVector(); + assertEquals( + knnEngine1.score(1 - KNNScoringUtil.cosinesimil(testData.queries[i], primitiveArray), spaceType1), + actualScores.get(j), + 0.0001 + ); + } + + // Search the second field + response = searchKNNIndex(indexName, new KNNQueryBuilder(fieldName2, testData.queries[i], k), k); + responseBody = EntityUtils.toString(response.getEntity()); + knnResults = parseSearchResponse(responseBody, fieldName2); + assertEquals(k, knnResults.size()); + + actualScores = parseSearchResponseScore(responseBody, fieldName2); + for (int j = 0; j < k; j++) { + float[] primitiveArray = knnResults.get(j).getVector(); + assertEquals( + knnEngine2.score(KNNScoringUtil.l2Squared(testData.queries[i], primitiveArray), spaceType2), + actualScores.get(j), + 0.0001 + ); + } + } + + // Get Stats + int graphCount = getTotalGraphsInCache(); + assertTrue(graphCount > 0); + // Close index + closeKNNIndex(indexName); + + // Search every 5 seconds 14 times to confirm graph gets evicted + int intervals = 14; + for (int i = 0; i < intervals; i++) { + if (getTotalGraphsInCache() == 0) { + return; + } + + Thread.sleep(5 * 1000); + } + + fail("Graphs are not getting evicted"); + } } diff --git a/src/test/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80CompoundFormatTests.java b/src/test/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80CompoundFormatTests.java index 0ecabcce6..6001a9729 100644 --- a/src/test/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80CompoundFormatTests.java +++ b/src/test/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80CompoundFormatTests.java @@ -49,7 +49,9 @@ public void testGetCompoundReader() throws IOException { CompoundFormat delegate = mock(CompoundFormat.class); when(delegate.getCompoundReader(null, null, null)).thenReturn(dir); KNN80CompoundFormat knn80CompoundFormat = new KNN80CompoundFormat(delegate); - assertEquals(dir, knn80CompoundFormat.getCompoundReader(null, null, null)); + CompoundDirectory knnDir = knn80CompoundFormat.getCompoundReader(null, null, null); + assertTrue(knnDir instanceof KNN80CompoundDirectory); + assertEquals(dir, ((KNN80CompoundDirectory) knnDir).getDelegate()); } public void testWrite() throws IOException { diff --git a/src/test/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesProducerTests.java b/src/test/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesProducerTests.java new file mode 100644 index 000000000..b9a85bbcc --- /dev/null +++ b/src/test/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesProducerTests.java @@ -0,0 +1,130 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.codec.KNN80Codec; + +import com.google.common.collect.ImmutableMap; +import org.apache.lucene.codecs.Codec; +import org.apache.lucene.codecs.DocValuesFormat; +import org.apache.lucene.codecs.DocValuesProducer; +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.FieldInfos; +import org.apache.lucene.index.SegmentInfo; +import org.apache.lucene.index.SegmentReadState; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IOContext; +import org.apache.lucene.store.IndexOutput; +import org.junit.Before; +import org.opensearch.Version; +import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.knn.KNNTestCase; +import org.opensearch.knn.common.KNNConstants; +import org.opensearch.knn.index.SpaceType; +import org.opensearch.knn.index.VectorDataType; +import org.opensearch.knn.index.codec.KNN87Codec.KNN87Codec; +import org.opensearch.knn.index.codec.KNNCodecTestUtil; +import org.opensearch.knn.index.engine.KNNEngine; +import org.opensearch.knn.index.engine.KNNMethodConfigContext; +import org.opensearch.knn.index.engine.KNNMethodContext; +import org.opensearch.knn.index.engine.MethodComponentContext; +import org.opensearch.knn.index.mapper.KNNVectorFieldMapper; + +import java.io.IOException; +import java.util.Arrays; +import java.util.List; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import static org.opensearch.knn.common.KNNConstants.METHOD_HNSW; +import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_EF_CONSTRUCTION; +import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_M; + +public class KNN80DocValuesProducerTests extends KNNTestCase { + + private static Directory directory; + + @Before + public void setUp() throws Exception { + super.setUp(); + directory = newFSDirectory(createTempDir()); + } + + @Override + public void tearDown() throws Exception { + super.tearDown(); + directory.close(); + } + + public void testProduceKNNBinaryField_fromCodec_nmslibCurrent() throws IOException { + // Set information about the segment and the fields + DocValuesFormat mockDocValuesFormat = mock(DocValuesFormat.class); + Codec mockDelegateCodec = mock(Codec.class); + DocValuesProducer mockDocValuesProducer = mock(DocValuesProducer.class); + when(mockDelegateCodec.docValuesFormat()).thenReturn(mockDocValuesFormat); + when(mockDocValuesFormat.fieldsProducer(any())).thenReturn(mockDocValuesProducer); + when(mockDocValuesFormat.getName()).thenReturn("mockDocValuesFormat"); + Codec codec = new KNN87Codec(mockDelegateCodec); + + String segmentName = "_test"; + int docsInSegment = 100; + String fieldName1 = String.format("test_field1%s", randomAlphaOfLength(4)); + String fieldName2 = String.format("test_field2%s", randomAlphaOfLength(4)); + List segmentFiles = Arrays.asList( + String.format("%s_2011_%s%s", segmentName, fieldName1, KNNEngine.NMSLIB.getExtension()), + String.format("%s_165_%s%s", segmentName, fieldName2, KNNEngine.FAISS.getExtension()) + ); + + KNNEngine knnEngine = KNNEngine.NMSLIB; + SpaceType spaceType = SpaceType.COSINESIMIL; + SegmentInfo segmentInfo = KNNCodecTestUtil.segmentInfoBuilder() + .directory(directory) + .segmentName(segmentName) + .docsInSegment(docsInSegment) + .codec(codec) + .build(); + + for (String name : segmentFiles) { + IndexOutput indexOutput = directory.createOutput(name, IOContext.DEFAULT); + indexOutput.close(); + } + segmentInfo.setFiles(segmentFiles); + + KNNMethodContext knnMethodContext = new KNNMethodContext( + knnEngine, + spaceType, + new MethodComponentContext(METHOD_HNSW, ImmutableMap.of(METHOD_PARAMETER_M, 16, METHOD_PARAMETER_EF_CONSTRUCTION, 512)) + ); + KNNMethodConfigContext knnMethodConfigContext = KNNMethodConfigContext.builder() + .vectorDataType(VectorDataType.FLOAT) + .versionCreated(Version.CURRENT) + .build(); + String parameterString = XContentFactory.jsonBuilder() + .map(knnEngine.getKNNLibraryIndexingContext(knnMethodContext, knnMethodConfigContext).getLibraryParameters()) + .toString(); + + FieldInfo[] fieldInfoArray = new FieldInfo[] { + KNNCodecTestUtil.FieldInfoBuilder.builder(fieldName1) + .addAttribute(KNNVectorFieldMapper.KNN_FIELD, "true") + .addAttribute(KNNConstants.KNN_ENGINE, knnEngine.getName()) + .addAttribute(KNNConstants.SPACE_TYPE, spaceType.getValue()) + .addAttribute(KNNConstants.PARAMETERS, parameterString) + .build() }; + + FieldInfos fieldInfos = new FieldInfos(fieldInfoArray); + SegmentReadState state = new SegmentReadState(directory, segmentInfo, fieldInfos, IOContext.DEFAULT); + + DocValuesFormat docValuesFormat = codec.docValuesFormat(); + assertTrue(docValuesFormat instanceof KNN80DocValuesFormat); + DocValuesProducer producer = docValuesFormat.fieldsProducer(state); + assertTrue(producer instanceof KNN80DocValuesProducer); + int pathSize = ((KNN80DocValuesProducer) producer).getOpenedIndexPath().size(); + assertEquals(pathSize, 1); + + String path = ((KNN80DocValuesProducer) producer).getOpenedIndexPath().get(0); + assertTrue(path.contains(segmentFiles.get(0))); + } + +} diff --git a/src/test/java/org/opensearch/knn/index/codec/util/KNNCodecUtilTests.java b/src/test/java/org/opensearch/knn/index/codec/util/KNNCodecUtilTests.java index dbea6375b..86e22cd88 100644 --- a/src/test/java/org/opensearch/knn/index/codec/util/KNNCodecUtilTests.java +++ b/src/test/java/org/opensearch/knn/index/codec/util/KNNCodecUtilTests.java @@ -6,8 +6,15 @@ package org.opensearch.knn.index.codec.util; import junit.framework.TestCase; +import org.apache.lucene.index.SegmentInfo; import org.opensearch.knn.index.VectorDataType; +import org.opensearch.knn.index.engine.KNNEngine; +import java.util.List; +import java.util.Set; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; import static org.opensearch.knn.index.codec.util.KNNCodecUtil.calculateArraySize; public class KNNCodecUtilTests extends TestCase { @@ -28,4 +35,15 @@ public void testCalculateArraySize() { vectorDataType = VectorDataType.BINARY; assertEquals(40, calculateArraySize(numVectors, vectorLength, vectorDataType)); } + + public void testGetKNNEngines() { + SegmentInfo segmentInfo = mock(SegmentInfo.class); + KNNEngine knnEngine = KNNEngine.FAISS; + Set SEGMENT_MULTI_FIELD_FILES_FAISS = Set.of("_0.cfe", "_0_2011_long_target_field.faissc", "_0_2011_target_field.faissc"); + when(segmentInfo.getUseCompoundFile()).thenReturn(true); + when(segmentInfo.files()).thenReturn(SEGMENT_MULTI_FIELD_FILES_FAISS); + List engineFiles = KNNCodecUtil.getEngineFiles(knnEngine.getExtension(), "target_field", segmentInfo); + assertEquals(engineFiles.size(), 2); + assertTrue(engineFiles.get(0).equals("_0_2011_target_field.faissc")); + } } diff --git a/src/test/java/org/opensearch/knn/index/query/KNNWeightTests.java b/src/test/java/org/opensearch/knn/index/query/KNNWeightTests.java index 810f49c15..f92f32406 100644 --- a/src/test/java/org/opensearch/knn/index/query/KNNWeightTests.java +++ b/src/test/java/org/opensearch/knn/index/query/KNNWeightTests.java @@ -41,6 +41,7 @@ import org.opensearch.knn.KNNTestCase; import org.opensearch.knn.index.KNNSettings; import org.opensearch.knn.index.codec.KNN990Codec.QuantizationConfigKNNCollector; +import org.opensearch.knn.index.codec.util.KNNCodecUtil; import org.opensearch.knn.index.engine.MethodComponentContext; import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.VectorDataType; @@ -1318,7 +1319,7 @@ private void testQueryScore( String engineName = fieldInfo.attributes().getOrDefault(KNN_ENGINE, KNNEngine.NMSLIB.getName()); KNNEngine knnEngine = KNNEngine.getEngine(engineName); - List engineFiles = knnWeight.getEngineFiles(reader, knnEngine.getExtension()); + List engineFiles = KNNCodecUtil.getEngineFiles(knnEngine.getExtension(), query.getField(), reader.getSegmentInfo().info); String expectIndexPath = String.format("%s_%s_%s%s%s", SEGMENT_NAME, 2011, FIELD_NAME, knnEngine.getExtension(), "c"); assertEquals(engineFiles.get(0), expectIndexPath); diff --git a/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java b/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java index a4659f691..f434ee928 100644 --- a/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java +++ b/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java @@ -357,6 +357,12 @@ protected void deleteKNNIndex(String index) throws IOException { assertEquals(request.getEndpoint() + ": failed", RestStatus.OK, RestStatus.fromCode(response.getStatusLine().getStatusCode())); } + protected void closeKNNIndex(String index) throws IOException { + Request request = new Request("POST", "/" + index + "/_close"); + Response response = client().performRequest(request); + assertEquals(request.getEndpoint() + ": failed", RestStatus.OK, RestStatus.fromCode(response.getStatusLine().getStatusCode())); + } + /** * For a given index, make a mapping request */ From 7ff8dd0b990aed2f83a185e15d1d424f3f1aba16 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Fri, 13 Sep 2024 23:33:42 -0700 Subject: [PATCH 372/416] Fix native engine vector format test (#2103) (#2107) Previosuly we were not creating hnsw file on segment flush for faiss engine. After successfully integrating hnsw file creation, we forgot to update unit test. Here, we will confirm that required files are being created based on field type. Signed-off-by: Vijayan Balasubramanian (cherry picked from commit 8277bf0f57a045112dfa7098f64b26dac4b63652) Co-authored-by: Vijayan Balasubramanian --- .../NativeEngines990KnnVectorsFormatTests.java | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/test/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsFormatTests.java b/src/test/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsFormatTests.java index f1e48c3a1..90ed18d0d 100644 --- a/src/test/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsFormatTests.java +++ b/src/test/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsFormatTests.java @@ -81,9 +81,9 @@ public class NativeEngines990KnnVectorsFormatTests extends KNNTestCase { private static final Codec TESTING_CODEC = new UnitTestCodec(); private static final String FLAT_VECTOR_FILE_EXT = ".vec"; - private static final String HNSW_FILE_EXT = ".hnsw"; + private static final String FAISS_ENGINE_FILE_EXT = ".faiss"; private static final String FLOAT_VECTOR_FIELD = "float_field"; - private static final String FLOAT_VECTOR_FIELD_BINARY = "float_field_binary"; + private static final String FLOAT_VECTOR_FIELD_BINARY = "float_binary_field"; private static final String BYTE_VECTOR_FIELD = "byte_field"; private Directory dir; private RandomIndexWriter indexWriter; @@ -220,11 +220,11 @@ public void testNativeEngineVectorFormat_whenMultipleVectorFieldIndexed_thenSucc IndexSearcher searcher = new IndexSearcher(indexReader); final LeafReader leafReader = searcher.getLeafContexts().get(0).reader(); SegmentReader segmentReader = Lucene.segmentReader(leafReader); - final List hnswfiles = getFilesFromSegment(dir, HNSW_FILE_EXT); - // 0 hnsw files for now as we have not integrated graph creation here. - assertEquals(0, hnswfiles.size()); - assertEquals(hnswfiles.stream().filter(x -> x.contains(FLOAT_VECTOR_FIELD)).count(), 0); - assertEquals(hnswfiles.stream().filter(x -> x.contains(BYTE_VECTOR_FIELD)).count(), 0); + final List hnswfiles = getFilesFromSegment(dir, FAISS_ENGINE_FILE_EXT); + assertEquals(3, hnswfiles.size()); + assertEquals(hnswfiles.stream().filter(x -> x.contains(FLOAT_VECTOR_FIELD)).count(), 1); + assertEquals(hnswfiles.stream().filter(x -> x.contains(BYTE_VECTOR_FIELD)).count(), 1); + assertEquals(hnswfiles.stream().filter(x -> x.contains(FLOAT_VECTOR_FIELD_BINARY)).count(), 1); // Even setting IWC to not use compound file it still uses compound file, hence ensuring we don't check .vec // file in case segment uses compound format. use this seed once we fix this to validate everything is From d96f227a9b0ddf51c8c962a0e46e80e4f924b0cb Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Fri, 13 Sep 2024 23:34:03 -0700 Subject: [PATCH 373/416] Fix bug where quantization framework does not work with training (#2100) (#2101) * Initial implementation Signed-off-by: Ryan Bogan * Modify integration test and fix bugs in jni Signed-off-by: Ryan Bogan * Fix unit test Signed-off-by: Ryan Bogan * Fix integration test after merge Signed-off-by: Ryan Bogan * Add changelog (release notes) Signed-off-by: Ryan Bogan * Add unit test Signed-off-by: Ryan Bogan * Remove entry for release notes Signed-off-by: Ryan Bogan * Add null checks Signed-off-by: Ryan Bogan --------- Signed-off-by: Ryan Bogan (cherry picked from commit 5d10d64c80f71f383d3e8690d8502d133859ffbf) Co-authored-by: Ryan Bogan --- jni/src/faiss_wrapper.cpp | 25 +++++--- .../knn/index/mapper/CompressionLevel.java | 2 +- .../index/memory/NativeMemoryAllocation.java | 5 ++ .../memory/NativeMemoryEntryContext.java | 8 ++- .../memory/NativeMemoryLoadStrategy.java | 4 ++ .../TrainingModelTransportAction.java | 15 ++++- .../training/FloatTrainingDataConsumer.java | 60 +++++++++++++++++-- .../memory/NativeMemoryEntryContextTests.java | 19 ++++-- .../memory/NativeMemoryLoadStrategyTests.java | 4 +- .../knn/integ/ModeAndCompressionIT.java | 43 ++++--------- .../FloatTrainingDataConsumerTests.java | 44 ++++++++++++-- 11 files changed, 168 insertions(+), 61 deletions(-) diff --git a/jni/src/faiss_wrapper.cpp b/jni/src/faiss_wrapper.cpp index 227fcb477..45548e0f7 100644 --- a/jni/src/faiss_wrapper.cpp +++ b/jni/src/faiss_wrapper.cpp @@ -684,20 +684,29 @@ jobjectArray knn_jni::faiss_wrapper::QueryBinaryIndex_WithFilter(knn_jni::JNIUti } else { faiss::SearchParameters *searchParameters = nullptr; faiss::SearchParametersHNSW hnswParams; + faiss::SearchParametersIVF ivfParams; std::unique_ptr idGrouper; std::vector idGrouperBitmap; - auto hnswReader = dynamic_cast(indexReader->index); + auto ivfReader = dynamic_cast(indexReader->index); // TODO currently, search parameter is not supported in binary index // To avoid test failure, we skip setting ef search when methodPramsJ is null temporary - if(hnswReader!= nullptr && (methodParamsJ != nullptr || parentIdsJ != nullptr)) { - // Query param efsearch supersedes ef_search provided during index setting. - hnswParams.efSearch = knn_jni::commons::getIntegerMethodParameter(env, jniUtil, methodParams, EF_SEARCH, hnswReader->hnsw.efSearch); - if (parentIdsJ != nullptr) { - idGrouper = buildIDGrouperBitmap(jniUtil, env, parentIdsJ, &idGrouperBitmap); - hnswParams.grp = idGrouper.get(); + if (ivfReader) { + int indexNprobe = ivfReader->nprobe; + ivfParams.nprobe = commons::getIntegerMethodParameter(env, jniUtil, methodParams, NPROBES, indexNprobe); + searchParameters = &ivfParams; + } else { + auto hnswReader = dynamic_cast(indexReader->index); + if(hnswReader != nullptr && (methodParamsJ != nullptr || parentIdsJ != nullptr)) { + // Query param efsearch supersedes ef_search provided during index setting. + hnswParams.efSearch = knn_jni::commons::getIntegerMethodParameter(env, jniUtil, methodParams, EF_SEARCH, hnswReader->hnsw.efSearch); + if (parentIdsJ != nullptr) { + idGrouper = buildIDGrouperBitmap(jniUtil, env, parentIdsJ, &idGrouperBitmap); + hnswParams.grp = idGrouper.get(); + } + searchParameters = &hnswParams; } - searchParameters = &hnswParams; } + try { indexReader->search(1, reinterpret_cast(rawQueryvector), kJ, dis.data(), ids.data(), searchParameters); } catch (...) { diff --git a/src/main/java/org/opensearch/knn/index/mapper/CompressionLevel.java b/src/main/java/org/opensearch/knn/index/mapper/CompressionLevel.java index 222e042b6..0709239cf 100644 --- a/src/main/java/org/opensearch/knn/index/mapper/CompressionLevel.java +++ b/src/main/java/org/opensearch/knn/index/mapper/CompressionLevel.java @@ -30,7 +30,7 @@ public enum CompressionLevel { x32(32, "32x", new RescoreContext(3.0f), Set.of(Mode.ON_DISK)); // Internally, an empty string is easier to deal with them null. However, from the mapping, - // we do not want users to pass in the empty string and instead want null. So we make the conversion herex + // we do not want users to pass in the empty string and instead want null. So we make the conversion here public static final String[] NAMES_ARRAY = new String[] { NOT_CONFIGURED.getName(), x1.getName(), diff --git a/src/main/java/org/opensearch/knn/index/memory/NativeMemoryAllocation.java b/src/main/java/org/opensearch/knn/index/memory/NativeMemoryAllocation.java index f42b256c8..635bc3883 100644 --- a/src/main/java/org/opensearch/knn/index/memory/NativeMemoryAllocation.java +++ b/src/main/java/org/opensearch/knn/index/memory/NativeMemoryAllocation.java @@ -12,10 +12,12 @@ package org.opensearch.knn.index.memory; import lombok.Getter; +import lombok.Setter; import org.apache.lucene.index.LeafReaderContext; import org.opensearch.knn.common.featureflags.KNNFeatureFlags; import org.opensearch.common.concurrent.RefCountedReleasable; import org.opensearch.knn.index.VectorDataType; +import org.opensearch.knn.index.engine.qframe.QuantizationConfig; import org.opensearch.knn.index.query.KNNWeight; import org.opensearch.knn.jni.JNIService; import org.opensearch.knn.index.engine.KNNEngine; @@ -291,6 +293,9 @@ class TrainingDataAllocation implements NativeMemoryAllocation { private volatile boolean closed; private long memoryAddress; private final int size; + @Getter + @Setter + private QuantizationConfig quantizationConfig = QuantizationConfig.EMPTY; // Implement reader/writer with semaphores to deal with passing lock conditions between threads private int readCount; diff --git a/src/main/java/org/opensearch/knn/index/memory/NativeMemoryEntryContext.java b/src/main/java/org/opensearch/knn/index/memory/NativeMemoryEntryContext.java index 2dfc5fafb..dd219593d 100644 --- a/src/main/java/org/opensearch/knn/index/memory/NativeMemoryEntryContext.java +++ b/src/main/java/org/opensearch/knn/index/memory/NativeMemoryEntryContext.java @@ -11,8 +11,10 @@ package org.opensearch.knn.index.memory; +import lombok.Getter; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.Nullable; +import org.opensearch.knn.index.engine.qframe.QuantizationConfig; import org.opensearch.knn.index.util.IndexUtil; import org.opensearch.knn.index.VectorDataType; @@ -171,6 +173,8 @@ public static class TrainingDataEntryContext extends NativeMemoryEntryContext listener) { + KNNMethodContext knnMethodContext = request.getKnnMethodContext(); + KNNMethodConfigContext knnMethodConfigContext = request.getKnnMethodConfigContext(); + QuantizationConfig quantizationConfig = QuantizationConfig.EMPTY; + + if (knnMethodContext != null && request.getKnnMethodConfigContext() != null) { + KNNLibraryIndexingContext knnLibraryIndexingContext = knnMethodContext.getKnnEngine() + .getKNNLibraryIndexingContext(knnMethodContext, knnMethodConfigContext); + quantizationConfig = knnLibraryIndexingContext.getQuantizationConfig(); + } NativeMemoryEntryContext.TrainingDataEntryContext trainingDataEntryContext = new NativeMemoryEntryContext.TrainingDataEntryContext( request.getTrainingDataSizeInKB(), @@ -54,7 +66,8 @@ protected void doExecute(Task task, TrainingModelRequest request, ActionListener clusterService, request.getMaximumVectorCount(), request.getSearchSize(), - request.getVectorDataType() + request.getVectorDataType(), + quantizationConfig ); // Allocation representing size model will occupy in memory during training diff --git a/src/main/java/org/opensearch/knn/training/FloatTrainingDataConsumer.java b/src/main/java/org/opensearch/knn/training/FloatTrainingDataConsumer.java index d742a9184..292752945 100644 --- a/src/main/java/org/opensearch/knn/training/FloatTrainingDataConsumer.java +++ b/src/main/java/org/opensearch/knn/training/FloatTrainingDataConsumer.java @@ -13,10 +13,19 @@ import org.apache.commons.lang.ArrayUtils; import org.opensearch.action.search.SearchResponse; +import org.opensearch.knn.index.engine.qframe.QuantizationConfig; +import org.opensearch.knn.jni.JNICommons; import org.opensearch.knn.jni.JNIService; import org.opensearch.knn.index.memory.NativeMemoryAllocation; +import org.opensearch.knn.quantization.factory.QuantizerFactory; +import org.opensearch.knn.quantization.models.quantizationOutput.BinaryQuantizationOutput; +import org.opensearch.knn.quantization.models.quantizationParams.ScalarQuantizationParams; +import org.opensearch.knn.quantization.models.quantizationState.QuantizationState; +import org.opensearch.knn.quantization.models.requests.TrainingRequest; +import org.opensearch.knn.quantization.quantizer.Quantizer; import org.opensearch.search.SearchHit; +import java.io.IOException; import java.util.ArrayList; import java.util.List; @@ -25,6 +34,8 @@ */ public class FloatTrainingDataConsumer extends TrainingDataConsumer { + private final QuantizationConfig quantizationConfig; + /** * Constructor * @@ -32,16 +43,28 @@ public class FloatTrainingDataConsumer extends TrainingDataConsumer { */ public FloatTrainingDataConsumer(NativeMemoryAllocation.TrainingDataAllocation trainingDataAllocation) { super(trainingDataAllocation); + this.quantizationConfig = trainingDataAllocation.getQuantizationConfig(); } @Override public void accept(List floats) { - trainingDataAllocation.setMemoryAddress( - JNIService.transferVectors( - trainingDataAllocation.getMemoryAddress(), - floats.stream().map(v -> ArrayUtils.toPrimitive((Float[]) v)).toArray(float[][]::new) - ) - ); + if (isValidFloatsAndQuantizationConfig(floats)) { + try { + List byteVectors = quantizeVectors(floats); + long memoryAddress = trainingDataAllocation.getMemoryAddress(); + memoryAddress = JNICommons.storeBinaryVectorData(memoryAddress, byteVectors.toArray(new byte[0][0]), byteVectors.size()); + trainingDataAllocation.setMemoryAddress(memoryAddress); + } catch (IOException e) { + throw new RuntimeException(e); + } + } else { + trainingDataAllocation.setMemoryAddress( + JNIService.transferVectors( + trainingDataAllocation.getMemoryAddress(), + floats.stream().map(v -> ArrayUtils.toPrimitive((Float[]) v)).toArray(float[][]::new) + ) + ); + } } @Override @@ -64,4 +87,29 @@ public void processTrainingVectors(SearchResponse searchResponse, int vectorsToA accept(vectors); } + + private List quantizeVectors(List vectors) throws IOException { + List bytes = new ArrayList<>(); + ScalarQuantizationParams quantizationParams = new ScalarQuantizationParams(quantizationConfig.getQuantizationType()); + Quantizer quantizer = QuantizerFactory.getQuantizer(quantizationParams); + // Create training request + TrainingRequest trainingRequest = new TrainingRequest(vectors.size()) { + @Override + public float[] getVectorAtThePosition(int position) { + return ArrayUtils.toPrimitive((Float[]) vectors.get(position)); + } + }; + QuantizationState quantizationState = quantizer.train(trainingRequest); + BinaryQuantizationOutput binaryQuantizationOutput = new BinaryQuantizationOutput(quantizationConfig.getQuantizationType().getId()); + for (int i = 0; i < vectors.size(); i++) { + quantizer.quantize(ArrayUtils.toPrimitive((Float[]) vectors.get(i)), quantizationState, binaryQuantizationOutput); + bytes.add(binaryQuantizationOutput.getQuantizedVectorCopy()); + } + + return bytes; + } + + private boolean isValidFloatsAndQuantizationConfig(List floats) { + return floats != null && floats.isEmpty() == false && quantizationConfig != null && quantizationConfig != QuantizationConfig.EMPTY; + } } diff --git a/src/test/java/org/opensearch/knn/index/memory/NativeMemoryEntryContextTests.java b/src/test/java/org/opensearch/knn/index/memory/NativeMemoryEntryContextTests.java index 385572cb4..1720da1ed 100644 --- a/src/test/java/org/opensearch/knn/index/memory/NativeMemoryEntryContextTests.java +++ b/src/test/java/org/opensearch/knn/index/memory/NativeMemoryEntryContextTests.java @@ -14,6 +14,7 @@ import com.google.common.collect.ImmutableMap; import org.opensearch.cluster.service.ClusterService; import org.opensearch.knn.KNNTestCase; +import org.opensearch.knn.index.engine.qframe.QuantizationConfig; import org.opensearch.knn.index.util.IndexUtil; import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.engine.KNNEngine; @@ -124,7 +125,8 @@ public void testTrainingDataEntryContext_load() { null, 0, 0, - VectorDataType.DEFAULT + VectorDataType.DEFAULT, + QuantizationConfig.EMPTY ); NativeMemoryAllocation.TrainingDataAllocation trainingDataAllocation = new NativeMemoryAllocation.TrainingDataAllocation( @@ -149,7 +151,8 @@ public void testTrainingDataEntryContext_getTrainIndexName() { null, 0, 0, - VectorDataType.DEFAULT + VectorDataType.DEFAULT, + QuantizationConfig.EMPTY ); assertEquals(trainIndexName, trainingDataEntryContext.getTrainIndexName()); @@ -165,7 +168,8 @@ public void testTrainingDataEntryContext_getTrainFieldName() { null, 0, 0, - VectorDataType.DEFAULT + VectorDataType.DEFAULT, + QuantizationConfig.EMPTY ); assertEquals(trainFieldName, trainingDataEntryContext.getTrainFieldName()); @@ -181,7 +185,8 @@ public void testTrainingDataEntryContext_getMaxVectorCount() { null, maxVectorCount, 0, - VectorDataType.DEFAULT + VectorDataType.DEFAULT, + QuantizationConfig.EMPTY ); assertEquals(maxVectorCount, trainingDataEntryContext.getMaxVectorCount()); @@ -197,7 +202,8 @@ public void testTrainingDataEntryContext_getSearchSize() { null, 0, searchSize, - VectorDataType.DEFAULT + VectorDataType.DEFAULT, + QuantizationConfig.EMPTY ); assertEquals(searchSize, trainingDataEntryContext.getSearchSize()); @@ -213,7 +219,8 @@ public void testTrainingDataEntryContext_getIndicesService() { clusterService, 0, 0, - VectorDataType.DEFAULT + VectorDataType.DEFAULT, + QuantizationConfig.EMPTY ); assertEquals(clusterService, trainingDataEntryContext.getClusterService()); diff --git a/src/test/java/org/opensearch/knn/index/memory/NativeMemoryLoadStrategyTests.java b/src/test/java/org/opensearch/knn/index/memory/NativeMemoryLoadStrategyTests.java index 29fbdb978..bdd8d7e45 100644 --- a/src/test/java/org/opensearch/knn/index/memory/NativeMemoryLoadStrategyTests.java +++ b/src/test/java/org/opensearch/knn/index/memory/NativeMemoryLoadStrategyTests.java @@ -18,6 +18,7 @@ import org.opensearch.knn.TestUtils; import org.opensearch.knn.common.KNNConstants; import org.opensearch.knn.index.VectorDataType; +import org.opensearch.knn.index.engine.qframe.QuantizationConfig; import org.opensearch.knn.jni.JNICommons; import org.opensearch.knn.jni.JNIService; import org.opensearch.knn.index.query.KNNQueryResult; @@ -180,7 +181,8 @@ public void testTrainingLoadStrategy_load() { null, 0, 0, - VectorDataType.FLOAT + VectorDataType.FLOAT, + QuantizationConfig.EMPTY ); // Load the allocation. Initially, the memory address should be 0. However, after the readlock is obtained, diff --git a/src/test/java/org/opensearch/knn/integ/ModeAndCompressionIT.java b/src/test/java/org/opensearch/knn/integ/ModeAndCompressionIT.java index ea9203196..59c435e2c 100644 --- a/src/test/java/org/opensearch/knn/integ/ModeAndCompressionIT.java +++ b/src/test/java/org/opensearch/knn/integ/ModeAndCompressionIT.java @@ -8,7 +8,6 @@ import lombok.SneakyThrows; import org.apache.http.util.EntityUtils; import org.junit.Assert; -import org.junit.Ignore; import org.opensearch.client.Request; import org.opensearch.client.Response; import org.opensearch.client.ResponseException; @@ -252,12 +251,14 @@ public void testTraining_whenInvalid_thenFail() { // Training isnt currently supported for mode and compression because quantization framework does not quantize // the training vectors. So, commenting out for now. - @Ignore @SneakyThrows public void testTraining_whenValid_thenSucceed() { setupTrainingIndex(); XContentBuilder builder; for (String compressionLevel : CompressionLevel.NAMES_ARRAY) { + if (compressionLevel.equals("4x")) { + continue; + } String indexName = INDEX_NAME + compressionLevel; String modelId = indexName; builder = XContentFactory.jsonBuilder() @@ -287,38 +288,13 @@ public void testTraining_whenValid_thenSucceed() { compressionLevel, Mode.NOT_CONFIGURED.getName() ); + deleteKNNIndex(indexName); } - - for (String compressionLevel : CompressionLevel.NAMES_ARRAY) { - for (String mode : Mode.NAMES_ARRAY) { - String indexName = INDEX_NAME + compressionLevel + "_" + mode; - String modelId = indexName; - builder = XContentFactory.jsonBuilder() - .startObject() - .field(TRAIN_INDEX_PARAMETER, TRAINING_INDEX_NAME) - .field(TRAIN_FIELD_PARAMETER, TRAINING_FIELD_NAME) - .field(KNNConstants.DIMENSION, DIMENSION) - .field(MODEL_DESCRIPTION, "") - .field(COMPRESSION_LEVEL_PARAMETER, compressionLevel) - .field(MODE_PARAMETER, mode) - .endObject(); - validateTraining(modelId, builder); - builder = XContentFactory.jsonBuilder() - .startObject() - .startObject("properties") - .startObject(FIELD_NAME) - .field("type", "knn_vector") - .field("model_id", modelId) - .endObject() - .endObject() - .endObject(); - String mapping = builder.toString(); - validateIndex(indexName, mapping); - validateSearch(indexName, METHOD_PARAMETER_NPROBES, METHOD_PARAMETER_NLIST_DEFAULT, compressionLevel, mode); - } - } - for (String mode : Mode.NAMES_ARRAY) { + if (mode == null) { + continue; + } + mode = mode.toLowerCase(); String indexName = INDEX_NAME + mode; String modelId = indexName; builder = XContentFactory.jsonBuilder() @@ -348,8 +324,8 @@ public void testTraining_whenValid_thenSucceed() { CompressionLevel.NOT_CONFIGURED.getName(), mode ); + deleteKNNIndex(indexName); } - } @SneakyThrows @@ -459,6 +435,7 @@ private void validateSearch( String exactSearchResponseBody = EntityUtils.toString(exactSearchResponse.getEntity()); List exactSearchKnnResults = parseSearchResponseScore(exactSearchResponseBody, FIELD_NAME); assertEquals(NUM_DOCS, exactSearchKnnResults.size()); + if (CompressionLevel.x4.getName().equals(compressionLevelString) == false && Mode.ON_DISK.getName().equals(mode)) { Assert.assertEquals(exactSearchKnnResults, knnResults); } diff --git a/src/test/java/org/opensearch/knn/training/FloatTrainingDataConsumerTests.java b/src/test/java/org/opensearch/knn/training/FloatTrainingDataConsumerTests.java index 27e02b46b..6e0410853 100644 --- a/src/test/java/org/opensearch/knn/training/FloatTrainingDataConsumerTests.java +++ b/src/test/java/org/opensearch/knn/training/FloatTrainingDataConsumerTests.java @@ -13,7 +13,9 @@ import org.mockito.ArgumentCaptor; import org.opensearch.knn.KNNTestCase; +import org.opensearch.knn.index.engine.qframe.QuantizationConfig; import org.opensearch.knn.index.memory.NativeMemoryAllocation; +import org.opensearch.knn.quantization.enums.ScalarQuantizationType; import java.util.ArrayList; import java.util.Arrays; @@ -29,12 +31,46 @@ public void testAccept() { // Mock the training data allocation int dimension = 128; - NativeMemoryAllocation.TrainingDataAllocation trainingDataAllocation = mock(NativeMemoryAllocation.TrainingDataAllocation.class); // new - // NativeMemoryAllocation.TrainingDataAllocation(0, - // numVectors*dimension* - // Float.BYTES); + NativeMemoryAllocation.TrainingDataAllocation trainingDataAllocation = mock(NativeMemoryAllocation.TrainingDataAllocation.class); + when(trainingDataAllocation.getMemoryAddress()).thenReturn(0L); + when(trainingDataAllocation.getQuantizationConfig()).thenReturn(QuantizationConfig.EMPTY); + + // Capture argument passed to set pointer + ArgumentCaptor valueCapture = ArgumentCaptor.forClass(Long.class); + + FloatTrainingDataConsumer floatTrainingDataConsumer = new FloatTrainingDataConsumer(trainingDataAllocation); + + List vectorSet1 = new ArrayList<>(3); + for (int i = 0; i < 3; i++) { + Float[] vector = new Float[dimension]; + Arrays.fill(vector, (float) i); + vectorSet1.add(vector); + } + + // Transfer vectors + floatTrainingDataConsumer.accept(vectorSet1); + + // Ensure that the pointer captured has been updated + verify(trainingDataAllocation).setMemoryAddress(valueCapture.capture()); + when(trainingDataAllocation.getMemoryAddress()).thenReturn(valueCapture.getValue()); + + assertNotEquals(0, trainingDataAllocation.getMemoryAddress()); + } + + public void testAccept_withQuantizationConfig() { + + // Mock the training data allocation + int dimension = 128; + NativeMemoryAllocation.TrainingDataAllocation trainingDataAllocation = mock(NativeMemoryAllocation.TrainingDataAllocation.class); + + when(trainingDataAllocation.getMemoryAddress()).thenReturn(0L); + + QuantizationConfig quantizationConfig = mock(QuantizationConfig.class); + when(quantizationConfig.getQuantizationType()).thenReturn(ScalarQuantizationType.ONE_BIT); + when(trainingDataAllocation.getQuantizationConfig()).thenReturn(QuantizationConfig.EMPTY); + // Capture argument passed to set pointer ArgumentCaptor valueCapture = ArgumentCaptor.forClass(Long.class); From b4896e3c7c26e7c85b07f589372bf08c172c8bf3 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Wed, 18 Sep 2024 14:07:46 -0700 Subject: [PATCH 374/416] Remove outdated comment in integration tests (#2118) Signed-off-by: Ryan Bogan (cherry picked from commit b6d6ec520f87a2465a50254072cb30b14d52bedc) Co-authored-by: Ryan Bogan --- .../java/org/opensearch/knn/integ/ModeAndCompressionIT.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/test/java/org/opensearch/knn/integ/ModeAndCompressionIT.java b/src/test/java/org/opensearch/knn/integ/ModeAndCompressionIT.java index 59c435e2c..cea3400af 100644 --- a/src/test/java/org/opensearch/knn/integ/ModeAndCompressionIT.java +++ b/src/test/java/org/opensearch/knn/integ/ModeAndCompressionIT.java @@ -249,8 +249,6 @@ public void testTraining_whenInvalid_thenFail() { expectThrows(ResponseException.class, () -> trainModel(modelId, builder2)); } - // Training isnt currently supported for mode and compression because quantization framework does not quantize - // the training vectors. So, commenting out for now. @SneakyThrows public void testTraining_whenValid_thenSucceed() { setupTrainingIndex(); From 097278b3ee56a26a90e1ba57192568be8a8d13f4 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Wed, 18 Sep 2024 14:33:42 -0700 Subject: [PATCH 375/416] Change min oversample to 1 (#2117) (#2119) Signed-off-by: John Mazanec (cherry picked from commit e3cf8140674f91055f1fcbba20d5866fab781031) Co-authored-by: John Mazanec --- .../org/opensearch/knn/index/query/rescore/RescoreContext.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/opensearch/knn/index/query/rescore/RescoreContext.java b/src/main/java/org/opensearch/knn/index/query/rescore/RescoreContext.java index 5e57b4aba..ea4e92215 100644 --- a/src/main/java/org/opensearch/knn/index/query/rescore/RescoreContext.java +++ b/src/main/java/org/opensearch/knn/index/query/rescore/RescoreContext.java @@ -18,7 +18,7 @@ public final class RescoreContext { public static final float DEFAULT_OVERSAMPLE_FACTOR = 1.0f; public static final float MAX_OVERSAMPLE_FACTOR = 100.0f; - public static final float MIN_OVERSAMPLE_FACTOR = 0.0f; + public static final float MIN_OVERSAMPLE_FACTOR = 1.0f; public static final int MAX_FIRST_PASS_RESULTS = 10000; From 662100feefae0941e5afa361b5072f2d8f103060 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Wed, 18 Sep 2024 17:51:04 -0500 Subject: [PATCH 376/416] Increment version to 2.18.0-SNAPSHOT (#2057) Signed-off-by: opensearch-ci-bot Co-authored-by: opensearch-ci-bot --- .../workflows/backwards_compatibility_tests_workflow.yml | 8 ++++---- build.gradle | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/backwards_compatibility_tests_workflow.yml b/.github/workflows/backwards_compatibility_tests_workflow.yml index e8702820e..e670703de 100644 --- a/.github/workflows/backwards_compatibility_tests_workflow.yml +++ b/.github/workflows/backwards_compatibility_tests_workflow.yml @@ -34,8 +34,8 @@ jobs: strategy: matrix: java: [ 11, 17 ] - bwc_version : [ "1.1.0", "1.2.4", "1.3.8", "2.0.1", "2.1.0", "2.2.1", "2.3.0", "2.4.1", "2.5.0", "2.6.0", "2.7.0", "2.8.0", "2.9.0", "2.10.0", "2.11.0", "2.12.0", "2.13.0", "2.14.0", "2.15.0", "2.16.0" ] - opensearch_version : [ "2.17.0-SNAPSHOT" ] + bwc_version : [ "1.1.0", "1.2.4", "1.3.8", "2.0.1", "2.1.0", "2.2.1", "2.3.0", "2.4.1", "2.5.0", "2.6.0", "2.7.0", "2.8.0", "2.9.0", "2.10.0", "2.11.0", "2.12.0", "2.13.0", "2.14.0", "2.15.0", "2.16.0", "2.17.0" ] + opensearch_version : [ "2.18.0-SNAPSHOT" ] name: k-NN Restart-Upgrade BWC Tests runs-on: ubuntu-latest @@ -72,8 +72,8 @@ jobs: strategy: matrix: java: [ 11, 17 ] - bwc_version: [ "1.3.8", "2.0.1", "2.1.0", "2.2.1", "2.3.0", "2.4.1", "2.5.0", "2.6.0", "2.7.0", "2.8.0", "2.9.0", "2.10.0", "2.11.0", "2.12.0", "2.13.0", "2.14.0", "2.15.0", "2.16.0"] - opensearch_version: [ "2.17.0-SNAPSHOT" ] + bwc_version: [ "1.3.8", "2.0.1", "2.1.0", "2.2.1", "2.3.0", "2.4.1", "2.5.0", "2.6.0", "2.7.0", "2.8.0", "2.9.0", "2.10.0", "2.11.0", "2.12.0", "2.13.0", "2.14.0", "2.15.0", "2.16.0", "2.17.0"] + opensearch_version: [ "2.18.0-SNAPSHOT" ] name: k-NN Rolling-Upgrade BWC Tests runs-on: ubuntu-latest diff --git a/build.gradle b/build.gradle index 355d46320..e54df9a17 100644 --- a/build.gradle +++ b/build.gradle @@ -13,7 +13,7 @@ buildscript { ext { // build.version_qualifier parameter applies to knn plugin artifacts only. OpenSearch version must be set // explicitly as 'opensearch.version' property, for instance opensearch.version=2.0.0-rc1-SNAPSHOT - opensearch_version = System.getProperty("opensearch.version", "2.17.0-SNAPSHOT") + opensearch_version = System.getProperty("opensearch.version", "2.18.0-SNAPSHOT") version_qualifier = System.getProperty("build.version_qualifier", "") opensearch_group = "org.opensearch" isSnapshot = "true" == System.getProperty("build.snapshot", "true") From 87ca73a703cd245ee702988a894da78b445fe62c Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Wed, 18 Sep 2024 16:37:32 -0700 Subject: [PATCH 377/416] Add log message for quantization state cache eviction due to size (#2120) (#2122) Signed-off-by: Ryan Bogan (cherry picked from commit ccc59b197125ffbba14428310419b0989f0c9bae) Co-authored-by: Ryan Bogan --- .../models/quantizationState/QuantizationStateCache.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/main/java/org/opensearch/knn/quantization/models/quantizationState/QuantizationStateCache.java b/src/main/java/org/opensearch/knn/quantization/models/quantizationState/QuantizationStateCache.java index cc5a34bcd..f057026b9 100644 --- a/src/main/java/org/opensearch/knn/quantization/models/quantizationState/QuantizationStateCache.java +++ b/src/main/java/org/opensearch/knn/quantization/models/quantizationState/QuantizationStateCache.java @@ -107,6 +107,11 @@ public void evict(String fieldName) { private void onRemoval(RemovalNotification removalNotification) { if (RemovalCause.SIZE == removalNotification.getCause()) { updateEvictedDueToSizeAt(); + log.info( + "[KNN] Quantization state evicted from cache. Key {}, Reason: {}", + removalNotification.getKey(), + removalNotification.getCause() + ); } } From 3967d19383e550294037dd55acc2a2299053f6da Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Thu, 19 Sep 2024 09:36:21 -0700 Subject: [PATCH 378/416] Remove benchmarks folder from k-NN repo (#2129) Signed-off-by: Navneet Verma (cherry picked from commit fe1d3e427349b108f0ad270fb45978dba4b18d29) Co-authored-by: Navneet Verma --- CHANGELOG.md | 1 + benchmarks/README.md | 4 ++++ 2 files changed, 5 insertions(+) create mode 100644 benchmarks/README.md diff --git a/CHANGELOG.md b/CHANGELOG.md index f70ca5e24..b320b05ab 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,4 +22,5 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Infrastructure ### Documentation ### Maintenance +* Remove benchmarks folder from k-NN repo [#2127](https://github.com/opensearch-project/k-NN/pull/2127) ### Refactoring diff --git a/benchmarks/README.md b/benchmarks/README.md new file mode 100644 index 000000000..2e642d41b --- /dev/null +++ b/benchmarks/README.md @@ -0,0 +1,4 @@ +## Benchmark Folder Tools Deprecated +All benchmark workloads have been moved to [OpenSearch Benchmark Workloads](https://github.com/opensearch-project/opensearch-benchmark-workloads/tree/main/vectorsearch). Please use OSB tool to run the benchmarks. + +If you are still interested in using the old tool, the benchmarks are moved to the [branch](https://github.com/opensearch-project/k-NN/tree/old-benchmarks/benchmarks). From 9601677f7816e4d7e867542683eb57363a32f4cb Mon Sep 17 00:00:00 2001 From: Tejas Shah Date: Fri, 20 Sep 2024 10:21:35 -0700 Subject: [PATCH 379/416] Integrates KNN plugin with ConcurrentSearchRequestDecider interface (#2111) (#2126) This allows knn queries to enable concurrency when index.search.concurrent_segment_search.mode or search.concurrent_segment_search.mode in auto mode. Without this the default behavior of auto mode is non-concurrent search Signed-off-by: Tejas Shah (cherry picked from commit 0421cdc907b43e4a930bd5a51454e5efea8413b6) --- CHANGELOG.md | 1 + .../org/opensearch/knn/plugin/KNNPlugin.java | 7 + .../KNNConcurrentSearchRequestDecider.java | 65 ++++++++ .../search/ConcurrentSegmentSearchIT.java | 141 ++++++++++++++++++ ...NNConcurrentSearchRequestDeciderTests.java | 65 ++++++++ .../knn/KNNJsonIndexMappingsBuilder.java | 15 +- 6 files changed, 291 insertions(+), 3 deletions(-) create mode 100644 src/main/java/org/opensearch/knn/plugin/search/KNNConcurrentSearchRequestDecider.java create mode 100644 src/test/java/org/opensearch/knn/integ/search/ConcurrentSegmentSearchIT.java create mode 100644 src/test/java/org/opensearch/knn/plugin/search/KNNConcurrentSearchRequestDeciderTests.java diff --git a/CHANGELOG.md b/CHANGELOG.md index b320b05ab..638448110 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ## [Unreleased 3.0](https://github.com/opensearch-project/k-NN/compare/2.x...HEAD) ### Features ### Enhancements +* Adds concurrent segment search support for mode auto [#2111](https://github.com/opensearch-project/k-NN/pull/2111) ### Bug Fixes * Add DocValuesProducers for releasing memory when close index [#1946](https://github.com/opensearch-project/k-NN/pull/1946) ### Infrastructure diff --git a/src/main/java/org/opensearch/knn/plugin/KNNPlugin.java b/src/main/java/org/opensearch/knn/plugin/KNNPlugin.java index 6bfcb7eee..cae8960bb 100644 --- a/src/main/java/org/opensearch/knn/plugin/KNNPlugin.java +++ b/src/main/java/org/opensearch/knn/plugin/KNNPlugin.java @@ -13,6 +13,7 @@ import org.opensearch.index.engine.EngineFactory; import org.opensearch.indices.SystemIndexDescriptor; import org.opensearch.knn.index.KNNCircuitBreaker; +import org.opensearch.knn.plugin.search.KNNConcurrentSearchRequestDecider; import org.opensearch.knn.index.util.KNNClusterUtil; import org.opensearch.knn.index.query.KNNQueryBuilder; import org.opensearch.knn.index.KNNSettings; @@ -96,6 +97,7 @@ import org.opensearch.script.ScriptContext; import org.opensearch.script.ScriptEngine; import org.opensearch.script.ScriptService; +import org.opensearch.search.deciders.ConcurrentSearchRequestDecider; import org.opensearch.threadpool.ExecutorBuilder; import org.opensearch.threadpool.FixedExecutorBuilder; import org.opensearch.threadpool.ThreadPool; @@ -373,4 +375,9 @@ public Settings additionalSettings() { ).collect(Collectors.toList()); return Settings.builder().putList(IndexModule.INDEX_STORE_HYBRID_MMAP_EXTENSIONS.getKey(), combinedSettings).build(); } + + @Override + public Optional getConcurrentSearchRequestDeciderFactory() { + return Optional.of(new KNNConcurrentSearchRequestDecider.Factory()); + } } diff --git a/src/main/java/org/opensearch/knn/plugin/search/KNNConcurrentSearchRequestDecider.java b/src/main/java/org/opensearch/knn/plugin/search/KNNConcurrentSearchRequestDecider.java new file mode 100644 index 000000000..92b0a3e3c --- /dev/null +++ b/src/main/java/org/opensearch/knn/plugin/search/KNNConcurrentSearchRequestDecider.java @@ -0,0 +1,65 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.plugin.search; + +import lombok.EqualsAndHashCode; +import org.opensearch.index.IndexSettings; +import org.opensearch.index.query.QueryBuilder; +import org.opensearch.knn.index.KNNSettings; +import org.opensearch.knn.index.query.KNNQueryBuilder; +import org.opensearch.search.deciders.ConcurrentSearchDecision; +import org.opensearch.search.deciders.ConcurrentSearchRequestDecider; + +import java.util.Optional; + +/** + * Decides if the knn query uses concurrent segment search + * As of 2.17, this is only used when + * - "index.search.concurrent_segment_search.mode": "auto" or + * - "search.concurrent_segment_search.mode": "auto" + * + * Note: the class is not thread-safe and a new instance needs to be created for each request + */ +@EqualsAndHashCode(callSuper = true) +public class KNNConcurrentSearchRequestDecider extends ConcurrentSearchRequestDecider { + + private static final ConcurrentSearchDecision DEFAULT_KNN_DECISION = new ConcurrentSearchDecision( + ConcurrentSearchDecision.DecisionStatus.NO_OP, + "Default decision" + ); + private static final ConcurrentSearchDecision YES = new ConcurrentSearchDecision( + ConcurrentSearchDecision.DecisionStatus.YES, + "Enable concurrent search for knn as Query has k-NN query in it and index is k-nn index" + ); + + private ConcurrentSearchDecision knnDecision = DEFAULT_KNN_DECISION; + + @Override + public void evaluateForQuery(final QueryBuilder queryBuilder, final IndexSettings indexSettings) { + if (queryBuilder instanceof KNNQueryBuilder && indexSettings.getValue(KNNSettings.IS_KNN_INDEX_SETTING)) { + knnDecision = YES; + } else { + knnDecision = DEFAULT_KNN_DECISION; + } + } + + @Override + public ConcurrentSearchDecision getConcurrentSearchDecision() { + return knnDecision; + } + + /** + * Returns {@link KNNConcurrentSearchRequestDecider} when index.knn is true + */ + public static class Factory implements ConcurrentSearchRequestDecider.Factory { + public Optional create(final IndexSettings indexSettings) { + if (indexSettings.getValue(KNNSettings.IS_KNN_INDEX_SETTING)) { + return Optional.of(new KNNConcurrentSearchRequestDecider()); + } + return Optional.empty(); + } + } +} diff --git a/src/test/java/org/opensearch/knn/integ/search/ConcurrentSegmentSearchIT.java b/src/test/java/org/opensearch/knn/integ/search/ConcurrentSegmentSearchIT.java new file mode 100644 index 000000000..06346d1ca --- /dev/null +++ b/src/test/java/org/opensearch/knn/integ/search/ConcurrentSegmentSearchIT.java @@ -0,0 +1,141 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.integ.search; + +import com.google.common.primitives.Floats; +import lombok.SneakyThrows; +import org.apache.http.util.EntityUtils; +import org.junit.BeforeClass; +import org.opensearch.client.Response; +import org.opensearch.common.settings.Settings; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.knn.KNNJsonIndexMappingsBuilder; +import org.opensearch.knn.KNNRestTestCase; +import org.opensearch.knn.KNNResult; +import org.opensearch.knn.TestUtils; +import org.opensearch.knn.index.SpaceType; +import org.opensearch.knn.index.engine.KNNEngine; +import org.opensearch.knn.index.query.KNNQueryBuilder; +import org.opensearch.knn.plugin.script.KNNScoringUtil; + +import java.io.IOException; +import java.net.URL; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; + +import static org.opensearch.knn.common.KNNConstants.METHOD_HNSW; + +/** + * Note that this is simply a sanity test to make sure that concurrent search code path is hit E2E and scores are intact + * There is no latency verification as it can be better encapsulated in nightly runs. + */ +public class ConcurrentSegmentSearchIT extends KNNRestTestCase { + + static TestUtils.TestData testData; + + @BeforeClass + public static void setUpClass() throws IOException { + if (ConcurrentSegmentSearchIT.class.getClassLoader() == null) { + throw new IllegalStateException("ClassLoader of ConcurrentSegmentSearchIT Class is null"); + } + URL testIndexVectors = ConcurrentSegmentSearchIT.class.getClassLoader().getResource("data/test_vectors_1000x128.json"); + URL testQueries = ConcurrentSegmentSearchIT.class.getClassLoader().getResource("data/test_queries_100x128.csv"); + assert testIndexVectors != null; + assert testQueries != null; + testData = new TestUtils.TestData(testIndexVectors.getPath(), testQueries.getPath()); + } + + @SneakyThrows + public void testConcurrentSegmentSearch_thenSucceed() { + String indexName = "test-concurrent-segment"; + String fieldName = "test-field-1"; + int dimension = testData.indexData.vectors[0].length; + final XContentBuilder indexBuilder = createFaissHnswIndexMapping(fieldName, dimension); + Map mappingMap = xContentBuilderToMap(indexBuilder); + String mapping = indexBuilder.toString(); + createKnnIndex(indexName, mapping); + assertEquals(new TreeMap<>(mappingMap), new TreeMap<>(getIndexMappingAsMap(indexName))); + + // Index the test data + for (int i = 0; i < testData.indexData.docs.length; i++) { + addKnnDoc( + indexName, + Integer.toString(testData.indexData.docs[i]), + fieldName, + Floats.asList(testData.indexData.vectors[i]).toArray() + ); + } + refreshAllNonSystemIndices(); + updateIndexSettings(indexName, Settings.builder().put("index.search.concurrent_segment_search.mode", "auto")); + + // Test search queries + int k = 10; + verifySearch(indexName, fieldName, k); + + updateIndexSettings(indexName, Settings.builder().put("index.search.concurrent_segment_search.mode", "all")); + verifySearch(indexName, fieldName, k); + + deleteKNNIndex(indexName); + } + + /* + { + "properties": { + "": { + "type": "knn_vector", + "dimension": , + "method": { + "name": "hnsw", + "space_type": "l2", + "engine": "faiss", + "parameters": { + "m": 16, + "ef_construction": 128, + "ef_search": 128 + } + } + } + } + */ + @SneakyThrows + private XContentBuilder createFaissHnswIndexMapping(String fieldName, int dimension) { + return KNNJsonIndexMappingsBuilder.builder() + .fieldName(fieldName) + .dimension(dimension) + .method( + KNNJsonIndexMappingsBuilder.Method.builder() + .engine(KNNEngine.FAISS.getName()) + .methodName(METHOD_HNSW) + .spaceType(SpaceType.L2.getValue()) + .parameters(KNNJsonIndexMappingsBuilder.Method.Parameters.builder().efConstruction(128).efSearch(128).m(16).build()) + .build() + ) + .build() + .getIndexMappingBuilder(); + } + + @SneakyThrows + private void verifySearch(String indexName, String fieldName, int k) { + for (int i = 0; i < testData.queries.length; i++) { + final KNNQueryBuilder queryBuilder = KNNQueryBuilder.builder().fieldName(fieldName).vector(testData.queries[i]).k(k).build(); + Response response = searchKNNIndex(indexName, queryBuilder, k); + String responseBody = EntityUtils.toString(response.getEntity()); + List knnResults = parseSearchResponse(responseBody, fieldName); + assertEquals(k, knnResults.size()); + + List actualScores = parseSearchResponseScore(responseBody, fieldName); + for (int j = 0; j < k; j++) { + float[] primitiveArray = knnResults.get(j).getVector(); + assertEquals( + KNNEngine.FAISS.score(KNNScoringUtil.l2Squared(testData.queries[i], primitiveArray), SpaceType.L2), + actualScores.get(j), + 0.0001 + ); + } + } + } +} diff --git a/src/test/java/org/opensearch/knn/plugin/search/KNNConcurrentSearchRequestDeciderTests.java b/src/test/java/org/opensearch/knn/plugin/search/KNNConcurrentSearchRequestDeciderTests.java new file mode 100644 index 000000000..53fd520f7 --- /dev/null +++ b/src/test/java/org/opensearch/knn/plugin/search/KNNConcurrentSearchRequestDeciderTests.java @@ -0,0 +1,65 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.plugin.search; + +import org.opensearch.index.IndexSettings; +import org.opensearch.index.query.MatchAllQueryBuilder; +import org.opensearch.knn.KNNTestCase; +import org.opensearch.knn.index.KNNSettings; +import org.opensearch.knn.index.query.KNNQueryBuilder; +import org.opensearch.search.deciders.ConcurrentSearchDecision; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class KNNConcurrentSearchRequestDeciderTests extends KNNTestCase { + + public void testDecider_thenSucceed() { + ConcurrentSearchDecision noop = new ConcurrentSearchDecision(ConcurrentSearchDecision.DecisionStatus.NO_OP, "Default decision"); + + KNNConcurrentSearchRequestDecider decider = new KNNConcurrentSearchRequestDecider(); + assertDecision(noop, decider.getConcurrentSearchDecision()); + IndexSettings indexSettingsMock = mock(IndexSettings.class); + when(indexSettingsMock.getValue(KNNSettings.IS_KNN_INDEX_SETTING)).thenReturn(Boolean.FALSE); + + // Non KNNQueryBuilder + decider.evaluateForQuery(new MatchAllQueryBuilder(), indexSettingsMock); + assertDecision(noop, decider.getConcurrentSearchDecision()); + decider.evaluateForQuery( + KNNQueryBuilder.builder().vector(new float[] { 1f, 2f, 3f, 4f, 5f, 6f }).fieldName("decider").k(10).build(), + indexSettingsMock + ); + assertDecision(noop, decider.getConcurrentSearchDecision()); + + when(indexSettingsMock.getValue(KNNSettings.IS_KNN_INDEX_SETTING)).thenReturn(Boolean.TRUE); + decider.evaluateForQuery( + KNNQueryBuilder.builder().vector(new float[] { 1f, 2f, 3f, 4f, 5f, 6f }).fieldName("decider").k(10).build(), + indexSettingsMock + ); + ConcurrentSearchDecision yes = new ConcurrentSearchDecision( + ConcurrentSearchDecision.DecisionStatus.YES, + "Enable concurrent search for knn as Query has k-NN query in it and index is k-nn index" + ); + assertDecision(yes, decider.getConcurrentSearchDecision()); + + decider.evaluateForQuery(new MatchAllQueryBuilder(), indexSettingsMock); + assertDecision(noop, decider.getConcurrentSearchDecision()); + } + + public void testDeciderFactory_thenSucceed() { + KNNConcurrentSearchRequestDecider.Factory factory = new KNNConcurrentSearchRequestDecider.Factory(); + IndexSettings indexSettingsMock = mock(IndexSettings.class); + when(indexSettingsMock.getValue(KNNSettings.IS_KNN_INDEX_SETTING)).thenReturn(Boolean.TRUE); + assertNotSame(factory.create(indexSettingsMock).get(), factory.create(indexSettingsMock).get()); + when(indexSettingsMock.getValue(KNNSettings.IS_KNN_INDEX_SETTING)).thenReturn(Boolean.FALSE); + assertTrue(factory.create(indexSettingsMock).isEmpty()); + } + + private void assertDecision(ConcurrentSearchDecision expected, ConcurrentSearchDecision actual) { + assertEquals(expected.getDecisionReason(), actual.getDecisionReason()); + assertEquals(expected.getDecisionStatus(), actual.getDecisionStatus()); + } +} diff --git a/src/testFixtures/java/org/opensearch/knn/KNNJsonIndexMappingsBuilder.java b/src/testFixtures/java/org/opensearch/knn/KNNJsonIndexMappingsBuilder.java index 15ac0c211..817684f89 100644 --- a/src/testFixtures/java/org/opensearch/knn/KNNJsonIndexMappingsBuilder.java +++ b/src/testFixtures/java/org/opensearch/knn/KNNJsonIndexMappingsBuilder.java @@ -9,6 +9,7 @@ import lombok.NonNull; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.knn.common.KNNConstants; import java.io.IOException; @@ -26,7 +27,7 @@ public class KNNJsonIndexMappingsBuilder { private String vectorDataType; private Method method; - public String getIndexMapping() throws IOException { + public XContentBuilder getIndexMappingBuilder() throws IOException { if (nestedFieldName != null) { XContentBuilder xContentBuilder = XContentFactory.jsonBuilder() .startObject() @@ -40,7 +41,7 @@ public String getIndexMapping() throws IOException { addVectorDataType(xContentBuilder); addMethod(xContentBuilder); xContentBuilder.endObject().endObject().endObject().endObject().endObject(); - return xContentBuilder.toString(); + return xContentBuilder; } else { XContentBuilder xContentBuilder = XContentFactory.jsonBuilder() .startObject() @@ -51,10 +52,14 @@ public String getIndexMapping() throws IOException { addVectorDataType(xContentBuilder); addMethod(xContentBuilder); xContentBuilder.endObject().endObject().endObject(); - return xContentBuilder.toString(); + return xContentBuilder; } } + public String getIndexMapping() throws IOException { + return getIndexMappingBuilder().toString(); + } + private void addVectorDataType(final XContentBuilder xContentBuilder) throws IOException { if (vectorDataType == null) { return; @@ -104,6 +109,7 @@ public static class Parameters { private Encoder encoder; private Integer efConstruction; private Integer efSearch; + private Integer m; private void addTo(final XContentBuilder xContentBuilder) throws IOException { xContentBuilder.startObject("parameters"); @@ -113,6 +119,9 @@ private void addTo(final XContentBuilder xContentBuilder) throws IOException { if (efSearch != null) { xContentBuilder.field("ef_search", efSearch); } + if (m != null) { + xContentBuilder.field(KNNConstants.METHOD_PARAMETER_M, m); + } addEncoder(xContentBuilder); xContentBuilder.endObject(); } From 550e8a1bcdd24b2c9112a1c9c5f6264762f846df Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Mon, 23 Sep 2024 11:52:44 -0500 Subject: [PATCH 380/416] Add changes for AVX-512 support in k-NN. (#2110) (#2130) * changes for AVX-512. Signed-off by: Akash Shankaran * add cpu detection logic to security workflow. Signed-off by: Akash Shankaran * add cpu detection logic to backward compat test workflow. Signed-off by: Akash Shankaran * fix bwc workflow. Signed-off by: Akash Shankaran * address PR feedback. Signed-off by: Akash Shankaran * fix a bug in KNNSettings. Signed-off by: Akash Shankaran * fix a bug in KNNSettings. Signed-off by: Akash Shankaran * update KNNSettings. Signed-off by: Akash Shankaran --------- (cherry picked from commit 5423cc15d838ef557b22e88c86a621cd639f04a3) Signed-off-by: Akash Shankaran Signed-off-by: Ryan Bogan Co-authored-by: akashsha1 <113050768+akashsha1@users.noreply.github.com> --- .github/workflows/CI.yml | 16 +++-- ...backwards_compatibility_tests_workflow.yml | 29 ++++++++-- .github/workflows/test_security.yml | 13 ++++- CHANGELOG.md | 1 + DEVELOPER_GUIDE.md | 13 +++-- build.gradle | 6 +- jni/cmake/init-faiss.cmake | 14 ++++- scripts/build.sh | 7 ++- .../opensearch/knn/common/KNNConstants.java | 1 + .../org/opensearch/knn/index/KNNSettings.java | 21 +++++++ .../org/opensearch/knn/jni/FaissService.java | 10 +++- .../org/opensearch/knn/jni/PlatformUtils.java | 40 ++++++++++++- .../plugin-metadata/plugin-security.policy | 1 + .../opensearch/knn/jni/PlatformUtilTests.java | 58 +++++++++++++++++++ 14 files changed, 202 insertions(+), 28 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 932ce8022..fda1e5e0f 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -73,13 +73,17 @@ jobs: # switching the user, as OpenSearch cluster can only be started as root/Administrator on linux-deb/linux-rpm/windows-zip. run: | chown -R 1000:1000 `pwd` - if lscpu | grep -i avx2 + if lscpu | grep -i avx512f | grep -i avx512cd | grep -i avx512vl | grep -i avx512dq | grep -i avx512bw then - echo "avx2 available on system" + echo "avx512 available on system" su `id -un 1000` -c "whoami && java -version && ./gradlew build -Dnproc.count=`nproc`" + elif lscpu | grep -i avx2 + then + echo "avx2 available on system" + su `id -un 1000` -c "whoami && java -version && ./gradlew build -Dnproc.count=`nproc` -Davx512.enabled=false" else - echo "avx2 not available on system" - su `id -un 1000` -c "whoami && java -version && ./gradlew build -Dsimd.enabled=false -Dnproc.count=`nproc`" + echo "avx512 and avx2 not available on system" + su `id -un 1000` -c "whoami && java -version && ./gradlew build -Davx2.enabled=false -Davx512.enabled=false -Dnproc.count=`nproc`" fi @@ -126,7 +130,7 @@ jobs: ./gradlew build -Dnproc.count=3 else echo "avx2 not available on system" - ./gradlew build -Dsimd.enabled=false -Dnproc.count=3 + ./gradlew build -Davx2.enabled=false -Davx512.enabled=false -Dnproc.count=3 fi Build-k-NN-Windows: @@ -187,4 +191,4 @@ jobs: # TODO: Detect processor count and set the value of nproc.count - name: Run build run: | - ./gradlew.bat build -D'simd.enabled=false' -D'nproc.count=4' + ./gradlew.bat build -D'avx2.enabled=false' -D'avx512.enabled=false' -D'nproc.count=4' diff --git a/.github/workflows/backwards_compatibility_tests_workflow.yml b/.github/workflows/backwards_compatibility_tests_workflow.yml index e670703de..91a46e2e8 100644 --- a/.github/workflows/backwards_compatibility_tests_workflow.yml +++ b/.github/workflows/backwards_compatibility_tests_workflow.yml @@ -64,9 +64,19 @@ jobs: - name: Run k-NN Restart-Upgrade BWC Tests from BWCVersion-${{ matrix.bwc_version }} to OpenSearch Version-${{ matrix.opensearch_version }} run: | - echo "Running restart-upgrade backwards compatibility tests ..." - ./gradlew :qa:restart-upgrade:testRestartUpgrade -Dtests.bwc.version=$BWC_VERSION_RESTART_UPGRADE - + echo "Running restart-upgrade backwards compatibility tests ..." + if lscpu | grep -i avx512f | grep -i avx512cd | grep -i avx512vl | grep -i avx512dq | grep -i avx512bw + then + echo "avx512 available on system" + ./gradlew :qa:restart-upgrade:testRestartUpgrade -Dtests.bwc.version=$BWC_VERSION_RESTART_UPGRADE -Dnproc.count=`nproc` + elif lscpu | grep -i avx2 + then + echo "avx2 available on system" + ./gradlew :qa:restart-upgrade:testRestartUpgrade -Dtests.bwc.version=$BWC_VERSION_RESTART_UPGRADE -Dnproc.count=`nproc` -Davx512.enabled=false + else + echo "avx512 and avx2 not available on system" + ./gradlew :qa:restart-upgrade:testRestartUpgrade -Dtests.bwc.version=$BWC_VERSION_RESTART_UPGRADE -Davx2.enabled=false -Davx512.enabled=false -Dsimd.enabled=false -Dnproc.count=`nproc` + fi Rolling-Upgrade-BWCTests-k-NN: strategy: @@ -102,4 +112,15 @@ jobs: - name: Run k-NN Rolling-Upgrade BWC Tests from BWCVersion-${{ matrix.bwc_version }} to OpenSearch Version-${{ matrix.opensearch_version }} run: | echo "Running rolling-upgrade backwards compatibility tests ..." - ./gradlew :qa:rolling-upgrade:testRollingUpgrade -Dtests.bwc.version=$BWC_VERSION_ROLLING_UPGRADE + if lscpu | grep -i avx512f | grep -i avx512cd | grep -i avx512vl | grep -i avx512dq | grep -i avx512bw + then + echo "avx512 available on system" + ./gradlew :qa:rolling-upgrade:testRollingUpgrade -Dtests.bwc.version=$BWC_VERSION_ROLLING_UPGRADE -Dnproc.count=`nproc` + elif lscpu | grep -i avx2 + then + echo "avx2 available on system" + ./gradlew :qa:rolling-upgrade:testRollingUpgrade -Dtests.bwc.version=$BWC_VERSION_ROLLING_UPGRADE -Dnproc.count=`nproc` -Davx512.enabled=false + else + echo "avx512 and avx2 not available on system" + ./gradlew :qa:rolling-upgrade:testRollingUpgrade -Dtests.bwc.version=$BWC_VERSION_ROLLING_UPGRADE -Davx2.enabled=false -Davx512.enabled=false -Dsimd.enabled=false -Dnproc.count=`nproc` + fi diff --git a/.github/workflows/test_security.yml b/.github/workflows/test_security.yml index 2f8df8526..f3deb096e 100644 --- a/.github/workflows/test_security.yml +++ b/.github/workflows/test_security.yml @@ -70,4 +70,15 @@ jobs: # switching the user, as OpenSearch cluster can only be started as root/Administrator on linux-deb/linux-rpm/windows-zip. run: | chown -R 1000:1000 `pwd` - su `id -un 1000` -c "whoami && java -version && ./gradlew integTest -Dsecurity.enabled=true -Dsimd.enabled=true -Dnproc.count=`nproc`" + if lscpu | grep -i avx512f | grep -i avx512cd | grep -i avx512vl | grep -i avx512dq | grep -i avx512bw + then + echo "avx512 available on system" + su `id -un 1000` -c "whoami && java -version && ./gradlew build -Dnproc.count=`nproc`" + elif lscpu | grep -i avx2 + then + echo "avx2 available on system" + su `id -un 1000` -c "whoami && java -version && ./gradlew build -Dnproc.count=`nproc` -Davx512.enabled=false" + else + echo "avx512 and avx2 not available on system" + su `id -un 1000` -c "whoami && java -version && ./gradlew build -Davx2.enabled=false -Davx512.enabled=false -Dnproc.count=`nproc`" + fi diff --git a/CHANGELOG.md b/CHANGELOG.md index 638448110..d4e333da9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ## [Unreleased 2.x](https://github.com/opensearch-project/k-NN/compare/2.17...2.x) ### Features +* Add AVX512 support to k-NN for FAISS library [#2069](https://github.com/opensearch-project/k-NN/pull/2069) ### Enhancements * Add short circuit if no live docs are in segments [#2059](https://github.com/opensearch-project/k-NN/pull/2059) ### Bug Fixes diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index d6eb1d2f6..68e624d5d 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -278,19 +278,22 @@ make -j 4 ### Enable SIMD Optimization SIMD(Single Instruction/Multiple Data) Optimization is enabled by default on Linux and Mac which boosts the performance -by enabling `AVX2` on `x86 architecture` and `NEON` on `ARM64 architecture` while building the Faiss library. But to enable SIMD, the underlying processor -should support this (AVX2 or NEON). It can be disabled by setting the parameter `simd.enabled` to `false`. As of now, it is not supported on Windows OS. +by enabling `AVX2` and `AVX512` on `x86 architecture` and `NEON` on `ARM64 architecture` where applicable while building the Faiss library. But to enable SIMD, +the underlying processor should support these capabilities (AVX512, AVX2 or NEON). It can be disabled by setting the parameter `avx2.enabled` to `false` and +`avx512.enabled` to `false`. If your processor supports `AVX512` or `AVX2`, they can be set by enabling the setting . By default, these values are enabled on +OpenSearch. Some exceptions: As of now, SIMD support is not supported on Windows OS, and AVX512 is not present on MAC systems due to hardware not supporting the +feature. ``` # While building OpenSearch k-NN -./gradlew build -Dsimd.enabled=true +./gradlew build -Davx2.enabled=true -Davx512.enabled=true # While running OpenSearch k-NN -./gradlew run -Dsimd.enabled=true +./gradlew run -Davx2.enabled=true -Davx512.enabled=true # While building the JNI libraries cd jni -cmake . -DSIMD_ENABLED=true +cmake . -DAVX2_ENABLED=true -DAVX512_ENABLED=true ``` ## Run OpenSearch k-NN diff --git a/build.gradle b/build.gradle index e54df9a17..fe9b7a076 100644 --- a/build.gradle +++ b/build.gradle @@ -17,8 +17,9 @@ buildscript { version_qualifier = System.getProperty("build.version_qualifier", "") opensearch_group = "org.opensearch" isSnapshot = "true" == System.getProperty("build.snapshot", "true") - simd_enabled = System.getProperty("simd.enabled", "true") + avx2_enabled = System.getProperty("avx2.enabled", "true") nproc_count = System.getProperty("nproc.count", "1") + avx512_enabled = System.getProperty("avx512.enabled", "true") // This flag determines whether the CMake build system should apply a custom patch. It prevents build failures // when the cmakeJniLib task is run multiple times. If the build.lib.commit_patches is true, the CMake build // system skips applying the patch if the patches have been applied already. If build.lib.commit_patches is @@ -316,7 +317,8 @@ task cmakeJniLib(type:Exec) { args.add("cmake") args.add(".") args.add("-DKNN_PLUGIN_VERSION=${opensearch_version}") - args.add("-DSIMD_ENABLED=${simd_enabled}") + args.add("-DAVX2_ENABLED=${avx2_enabled}") + args.add("-DAVX512_ENABLED=${avx512_enabled}") args.add("-DCOMMIT_LIB_PATCHES=${commit_lib_patches}") args.add("-DAPPLY_LIB_PATCHES=${apply_lib_patches}") if (Os.isFamily(Os.FAMILY_WINDOWS)) { diff --git a/jni/cmake/init-faiss.cmake b/jni/cmake/init-faiss.cmake index 08f3ccc40..4492d9f45 100644 --- a/jni/cmake/init-faiss.cmake +++ b/jni/cmake/init-faiss.cmake @@ -81,13 +81,21 @@ set(BUILD_TESTING OFF) # Avoid building faiss tests set(FAISS_ENABLE_GPU OFF) set(FAISS_ENABLE_PYTHON OFF) -if(NOT DEFINED SIMD_ENABLED) - set(SIMD_ENABLED true) # set default value as true if the argument is not set +if(NOT DEFINED AVX2_ENABLED) + set(AVX2_ENABLED true) # set default value as true if the argument is not set endif() -if(${CMAKE_SYSTEM_NAME} STREQUAL Windows OR ${CMAKE_SYSTEM_PROCESSOR} MATCHES "aarch64" OR ${CMAKE_SYSTEM_PROCESSOR} MATCHES "arm64" OR NOT ${SIMD_ENABLED}) +if(NOT DEFINED AVX512_ENABLED) + set(AVX512_ENABLED true) # set default value as true if the argument is not set +endif() + +if(${CMAKE_SYSTEM_NAME} STREQUAL Windows OR ${CMAKE_SYSTEM_PROCESSOR} MATCHES "aarch64" OR ${CMAKE_SYSTEM_PROCESSOR} MATCHES "arm64" OR ( NOT AVX2_ENABLED AND NOT AVX512_ENABLED)) set(FAISS_OPT_LEVEL generic) # Keep optimization level as generic on Windows OS as it is not supported due to MINGW64 compiler issue. Also, on aarch64 avx2 is not supported. set(TARGET_LINK_FAISS_LIB faiss) +elseif(${CMAKE_SYSTEM_NAME} STREQUAL Linux AND AVX512_ENABLED) + set(FAISS_OPT_LEVEL avx512) # Keep optimization level as avx512 to improve performance on Linux. This is not present on mac systems, and presently not supported on Windows OS. + set(TARGET_LINK_FAISS_LIB faiss_avx512) + string(PREPEND LIB_EXT "_avx512") # Prepend "_avx512" to lib extension to create the library as "libopensearchknn_faiss_avx512.so" on linux else() set(FAISS_OPT_LEVEL avx2) # Keep optimization level as avx2 to improve performance on Linux and Mac. set(TARGET_LINK_FAISS_LIB faiss_avx2) diff --git a/scripts/build.sh b/scripts/build.sh index 12798633b..be7304ee5 100644 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -118,13 +118,16 @@ fi # Build k-NN lib and plugin through gradle tasks cd $work_dir ./gradlew build --no-daemon --refresh-dependencies -x integTest -x test -Dopensearch.version=$VERSION -Dbuild.snapshot=$SNAPSHOT -Dbuild.version_qualifier=$QUALIFIER -Dbuild.lib.commit_patches=false -./gradlew :buildJniLib -Dsimd.enabled=false -Dbuild.lib.commit_patches=false +./gradlew :buildJniLib -Davx512.enabled=false -Davx2.enabled=false -Dbuild.lib.commit_patches=false if [ "$PLATFORM" != "windows" ] && [ "$ARCHITECTURE" = "x64" ]; then echo "Building k-NN library after enabling AVX2" # Skip applying patches as patches were applied already from previous :buildJniLib task # If we apply patches again, it fails with conflict - ./gradlew :buildJniLib -Dsimd.enabled=true -Dbuild.lib.commit_patches=false -Dbuild.lib.apply_patches=false + ./gradlew :buildJniLib -Davx2.enabled=true -Davx512.enabled=false -Dbuild.lib.commit_patches=false -Dbuild.lib.apply_patches=false + + echo "Building k-NN library after enabling AVX512" + ./gradlew :buildJniLib -Davx512.enabled=true -Dbuild.lib.commit_patches=false -Dbuild.lib.apply_patches=false fi ./gradlew publishPluginZipPublicationToZipStagingRepository -Dopensearch.version=$VERSION -Dbuild.snapshot=$SNAPSHOT -Dbuild.version_qualifier=$QUALIFIER diff --git a/src/main/java/org/opensearch/knn/common/KNNConstants.java b/src/main/java/org/opensearch/knn/common/KNNConstants.java index ed21d3005..4869e9896 100644 --- a/src/main/java/org/opensearch/knn/common/KNNConstants.java +++ b/src/main/java/org/opensearch/knn/common/KNNConstants.java @@ -140,6 +140,7 @@ public class KNNConstants { private static final String JNI_LIBRARY_PREFIX = "opensearchknn_"; public static final String FAISS_JNI_LIBRARY_NAME = JNI_LIBRARY_PREFIX + FAISS_NAME; public static final String FAISS_AVX2_JNI_LIBRARY_NAME = JNI_LIBRARY_PREFIX + FAISS_NAME + "_avx2"; + public static final String FAISS_AVX512_JNI_LIBRARY_NAME = JNI_LIBRARY_PREFIX + FAISS_NAME + "_avx512"; public static final String NMSLIB_JNI_LIBRARY_NAME = JNI_LIBRARY_PREFIX + NMSLIB_NAME; public static final String COMMON_JNI_LIBRARY_NAME = JNI_LIBRARY_PREFIX + COMMONS_NAME; diff --git a/src/main/java/org/opensearch/knn/index/KNNSettings.java b/src/main/java/org/opensearch/knn/index/KNNSettings.java index 4da11a2ad..5fcc51bb5 100644 --- a/src/main/java/org/opensearch/knn/index/KNNSettings.java +++ b/src/main/java/org/opensearch/knn/index/KNNSettings.java @@ -14,6 +14,7 @@ import org.opensearch.client.Client; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.Booleans; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; @@ -86,11 +87,13 @@ public class KNNSettings { public static final String KNN_FAISS_AVX2_DISABLED = "knn.faiss.avx2.disabled"; public static final String QUANTIZATION_STATE_CACHE_SIZE_LIMIT = "knn.quantization.cache.size.limit"; public static final String QUANTIZATION_STATE_CACHE_EXPIRY_TIME_MINUTES = "knn.quantization.cache.expiry.minutes"; + public static final String KNN_FAISS_AVX512_DISABLED = "knn.faiss.avx512.disabled"; /** * Default setting values */ public static final boolean KNN_DEFAULT_FAISS_AVX2_DISABLED_VALUE = false; + public static final boolean KNN_DEFAULT_FAISS_AVX512_DISABLED_VALUE = false; public static final String INDEX_KNN_DEFAULT_SPACE_TYPE = "l2"; public static final String INDEX_KNN_DEFAULT_SPACE_TYPE_FOR_BINARY = "hamming"; public static final Integer INDEX_KNN_DEFAULT_ALGO_PARAM_M = 16; @@ -302,6 +305,12 @@ public class KNNSettings { Dynamic ); + public static final Setting KNN_FAISS_AVX512_DISABLED_SETTING = Setting.boolSetting( + KNN_FAISS_AVX512_DISABLED, + KNN_DEFAULT_FAISS_AVX512_DISABLED_VALUE, + NodeScope + ); + /** * Dynamic settings */ @@ -429,6 +438,10 @@ private Setting getSetting(String key) { return KNN_FAISS_AVX2_DISABLED_SETTING; } + if (KNN_FAISS_AVX512_DISABLED.equals(key)) { + return KNN_FAISS_AVX512_DISABLED_SETTING; + } + if (KNN_VECTOR_STREAMING_MEMORY_LIMIT_IN_MB.equals(key)) { return KNN_VECTOR_STREAMING_MEMORY_LIMIT_PCT_SETTING; } @@ -460,6 +473,7 @@ public List> getSettings() { ADVANCED_FILTERED_EXACT_SEARCH_THRESHOLD_SETTING, KNN_FAISS_AVX2_DISABLED_SETTING, KNN_VECTOR_STREAMING_MEMORY_LIMIT_PCT_SETTING, + KNN_FAISS_AVX512_DISABLED_SETTING, QUANTIZATION_STATE_CACHE_SIZE_LIMIT_SETTING, QUANTIZATION_STATE_CACHE_EXPIRY_TIME_MINUTES_SETTING ); @@ -499,6 +513,13 @@ public static boolean isFaissAVX2Disabled() { } } + public static boolean isFaissAVX512Disabled() { + return Booleans.parseBoolean( + KNNSettings.state().getSettingValue(KNNSettings.KNN_FAISS_AVX512_DISABLED).toString(), + KNN_DEFAULT_FAISS_AVX512_DISABLED_VALUE + ); + } + public static Integer getFilteredExactSearchThreshold(final String indexName) { return KNNSettings.state().clusterService.state() .getMetadata() diff --git a/src/main/java/org/opensearch/knn/jni/FaissService.java b/src/main/java/org/opensearch/knn/jni/FaissService.java index 037171b98..4bceed015 100644 --- a/src/main/java/org/opensearch/knn/jni/FaissService.java +++ b/src/main/java/org/opensearch/knn/jni/FaissService.java @@ -20,7 +20,9 @@ import java.util.Map; import static org.opensearch.knn.index.KNNSettings.isFaissAVX2Disabled; +import static org.opensearch.knn.index.KNNSettings.isFaissAVX512Disabled; import static org.opensearch.knn.jni.PlatformUtils.isAVX2SupportedBySystem; +import static org.opensearch.knn.jni.PlatformUtils.isAVX512SupportedBySystem;; /** * Service to interact with faiss jni layer. Class dependencies should be minimal @@ -35,9 +37,11 @@ class FaissService { static { AccessController.doPrivileged((PrivilegedAction) () -> { - // Even if the underlying system supports AVX2, users can override and disable it by using the - // 'knn.faiss.avx2.disabled' setting by setting it to true in the opensearch.yml configuration - if (!isFaissAVX2Disabled() && isAVX2SupportedBySystem()) { + // Even if the underlying system supports AVX512 and AVX2, users can override and disable it by setting + // 'knn.faiss.avx2.disabled' or 'knn.faiss.avx512.disabled' to true in the opensearch.yml configuration + if (!isFaissAVX512Disabled() && isAVX512SupportedBySystem()) { + System.loadLibrary(KNNConstants.FAISS_AVX512_JNI_LIBRARY_NAME); + } else if (!isFaissAVX2Disabled() && isAVX2SupportedBySystem()) { System.loadLibrary(KNNConstants.FAISS_AVX2_JNI_LIBRARY_NAME); } else { System.loadLibrary(KNNConstants.FAISS_JNI_LIBRARY_NAME); diff --git a/src/main/java/org/opensearch/knn/jni/PlatformUtils.java b/src/main/java/org/opensearch/knn/jni/PlatformUtils.java index 8a5549dec..445862f24 100644 --- a/src/main/java/org/opensearch/knn/jni/PlatformUtils.java +++ b/src/main/java/org/opensearch/knn/jni/PlatformUtils.java @@ -20,8 +20,11 @@ import java.nio.file.Files; import java.nio.file.Paths; import java.security.AccessController; +import java.security.PrivilegedActionException; import java.security.PrivilegedExceptionAction; +import java.util.Arrays; import java.util.Locale; +import java.util.stream.Stream; public class PlatformUtils { @@ -37,7 +40,7 @@ public class PlatformUtils { * the flags contains 'avx2' and return true if it exists else false. */ public static boolean isAVX2SupportedBySystem() { - if (!Platform.isIntel()) { + if (!Platform.isIntel() || Platform.isWindows()) { return false; } @@ -58,7 +61,6 @@ public static boolean isAVX2SupportedBySystem() { } } else if (Platform.isLinux()) { - // The "/proc/cpuinfo" is a virtual file which identifies and provides the processor details used // by system. This info contains "flags" for each processor which determines the qualities of that processor // and it's ability to process different instruction sets like mmx, avx, avx2 and so on. @@ -80,4 +82,38 @@ public static boolean isAVX2SupportedBySystem() { } return false; } + + public static boolean isAVX512SupportedBySystem() { + + if (!Platform.isIntel() || Platform.isMac() || Platform.isWindows()) { + return false; + } + + if (Platform.isLinux()) { + // The "/proc/cpuinfo" is a virtual file which identifies and provides the processor details used + // by system. This info contains "flags" for each processor which determines the qualities of that processor + // and it's ability to process different instruction sets like mmx, avx, avx2, avx512 and so on. + // https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/6/html/deployment_guide/s2-proc-cpuinfo + // Here, we are trying to read the details of all processors used by system and find if any of the processor + // supports AVX512 instructions supported by faiss. + String fileName = "/proc/cpuinfo"; + + // AVX512 has multiple flags, which control various features. k-nn requires the same set of flags as faiss to compile + // using avx512. Please update these if faiss updates their compilation instructions in the future. + // https://github.com/facebookresearch/faiss/blob/main/faiss/CMakeLists.txt + String[] avx512 = { "avx512f", "avx512cd", "avx512vl", "avx512dq", "avx512bw" }; + + try { + return AccessController.doPrivileged((PrivilegedExceptionAction) () -> { + Stream linestream = Files.lines(Paths.get(fileName)); + String flags = linestream.filter(line -> line.startsWith("flags")).findFirst().orElse(""); + return Arrays.stream(avx512).allMatch(flags::contains); + }); + + } catch (PrivilegedActionException e) { + logger.error("[KNN] Error reading file [{}]. [{}]", fileName, e.getMessage(), e); + } + } + return false; + } } diff --git a/src/main/plugin-metadata/plugin-security.policy b/src/main/plugin-metadata/plugin-security.policy index d5ab0be21..ed329740f 100644 --- a/src/main/plugin-metadata/plugin-security.policy +++ b/src/main/plugin-metadata/plugin-security.policy @@ -3,6 +3,7 @@ grant { permission java.lang.RuntimePermission "loadLibrary.opensearchknn_faiss"; permission java.lang.RuntimePermission "loadLibrary.opensearchknn_common"; permission java.lang.RuntimePermission "loadLibrary.opensearchknn_faiss_avx2"; + permission java.lang.RuntimePermission "loadLibrary.opensearchknn_faiss_avx512"; permission java.net.SocketPermission "*", "connect,resolve"; permission java.lang.RuntimePermission "accessDeclaredMembers"; permission java.io.FilePermission "/proc/cpuinfo", "read"; diff --git a/src/test/java/org/opensearch/knn/jni/PlatformUtilTests.java b/src/test/java/org/opensearch/knn/jni/PlatformUtilTests.java index 7816505de..19c0abb07 100644 --- a/src/test/java/org/opensearch/knn/jni/PlatformUtilTests.java +++ b/src/test/java/org/opensearch/knn/jni/PlatformUtilTests.java @@ -22,6 +22,7 @@ import static org.mockito.Mockito.mockStatic; import static org.opensearch.knn.jni.PlatformUtils.isAVX2SupportedBySystem; +import static org.opensearch.knn.jni.PlatformUtils.isAVX512SupportedBySystem; public class PlatformUtilTests extends KNNTestCase { public static final String MAC_CPU_FEATURES = "machdep.cpu.leaf7_features"; @@ -124,4 +125,61 @@ public void testIsAVX2SupportedBySystem_platformIsLinux_throwsExceptionReturnsFa } + // AVX512 tests + + public void testIsAVX512SupportedBySystem_platformIsNotIntel_returnsFalse() { + try (MockedStatic mockedPlatform = mockStatic(Platform.class)) { + mockedPlatform.when(Platform::isIntel).thenReturn(false); + assertFalse(isAVX512SupportedBySystem()); + } + } + + public void testIsAVX512SupportedBySystem_platformIsMac_returnsFalse() { + try (MockedStatic mockedPlatform = mockStatic(Platform.class)) { + mockedPlatform.when(Platform::isMac).thenReturn(false); + assertFalse(isAVX512SupportedBySystem()); + } + } + + public void testIsAVX512SupportedBySystem_platformIsIntelMac_returnsFalse() { + try (MockedStatic mockedPlatform = mockStatic(Platform.class)) { + mockedPlatform.when(Platform::isIntel).thenReturn(true); + mockedPlatform.when(Platform::isMac).thenReturn(true); + assertFalse(isAVX512SupportedBySystem()); + } + } + + public void testIsAVX512SupportedBySystem_platformIsIntelWithOSAsWindows_returnsFalse() { + try (MockedStatic mockedPlatform = mockStatic(Platform.class)) { + mockedPlatform.when(Platform::isIntel).thenReturn(true); + mockedPlatform.when(Platform::isWindows).thenReturn(true); + assertFalse(isAVX512SupportedBySystem()); + } + } + + public void testIsAVX512SupportedBySystem_platformIsLinuxAllAVX512FlagsPresent_returnsTrue() { + try (MockedStatic mockedPlatform = mockStatic(Platform.class)) { + mockedPlatform.when(Platform::isIntel).thenReturn(true); + mockedPlatform.when(Platform::isLinux).thenReturn(true); + + try (MockedStatic mockedFiles = mockStatic(Files.class)) { + mockedFiles.when(() -> Files.lines(Paths.get(LINUX_PROC_CPU_INFO))) + .thenReturn(Stream.of("flags: AVX2 avx512f avx512cd avx512vl avx512dq avx512bw", "dummy string")); + assertTrue(isAVX512SupportedBySystem()); + } + } + } + + public void testIsAVX512SupportedBySystem_platformIsLinuxSomeAVX512FlagsPresent_returnsFalse() { + try (MockedStatic mockedPlatform = mockStatic(Platform.class)) { + mockedPlatform.when(Platform::isIntel).thenReturn(true); + mockedPlatform.when(Platform::isLinux).thenReturn(true); + + try (MockedStatic mockedFiles = mockStatic(Files.class)) { + mockedFiles.when(() -> Files.lines(Paths.get(LINUX_PROC_CPU_INFO))) + .thenReturn(Stream.of("flags: AVX2 avx512vl avx512dq avx512bw avx512vbmi umip pku ospke avx512_vbmi2", "dummy string")); + assertFalse(isAVX512SupportedBySystem()); + } + } + } } From 33c05ead48e636585c7b743c7cb26c129c861876 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Mon, 23 Sep 2024 16:55:20 -0700 Subject: [PATCH 381/416] Makes sure KNNVectorValues aren't recreated unnecessarily when quantization isn't needed (#2137) Signed-off-by: Tejas Shah (cherry picked from commit e33afa5de5f8658ad7fbe71125707436e81cc5b8) Co-authored-by: Tejas Shah --- CHANGELOG.md | 1 + .../NativeEngines990KnnVectorsWriter.java | 63 +++++++++++-------- ...eEngines990KnnVectorsWriterFlushTests.java | 11 ++++ ...eEngines990KnnVectorsWriterMergeTests.java | 9 +++ 4 files changed, 59 insertions(+), 25 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d4e333da9..a730a201b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Documentation ### Maintenance ### Refactoring +* Does not create additional KNNVectorValues in NativeEngines990KNNVectorWriter when quantization is not needed [#2133](https://github.com/opensearch-project/k-NN/pull/2133) ## [Unreleased 2.x](https://github.com/opensearch-project/k-NN/compare/2.17...2.x) ### Features diff --git a/src/main/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsWriter.java b/src/main/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsWriter.java index 3f32003ac..23cd2a4de 100644 --- a/src/main/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsWriter.java +++ b/src/main/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsWriter.java @@ -36,6 +36,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.List; +import java.util.function.Supplier; import static org.opensearch.knn.common.FieldInfoExtractor.extractVectorDataType; import static org.opensearch.knn.index.vectorvalues.KNNVectorValuesFactory.getVectorValues; @@ -82,19 +83,19 @@ public void flush(int maxDoc, final Sorter.DocMap sortMap) throws IOException { for (final NativeEngineFieldVectorsWriter field : fields) { final FieldInfo fieldInfo = field.getFieldInfo(); final VectorDataType vectorDataType = extractVectorDataType(fieldInfo); - int totalLiveDocs = getLiveDocs(getVectorValues(vectorDataType, field.getDocsWithField(), field.getVectors())); + int totalLiveDocs = field.getVectors().size(); if (totalLiveDocs > 0) { - KNNVectorValues knnVectorValues = getVectorValues(vectorDataType, field.getDocsWithField(), field.getVectors()); - - final QuantizationState quantizationState = train(field.getFieldInfo(), knnVectorValues, totalLiveDocs); + final Supplier> knnVectorValuesSupplier = () -> getVectorValues( + vectorDataType, + field.getDocsWithField(), + field.getVectors() + ); + final QuantizationState quantizationState = train(field.getFieldInfo(), knnVectorValuesSupplier, totalLiveDocs); final NativeIndexWriter writer = NativeIndexWriter.getWriter(fieldInfo, segmentWriteState, quantizationState); - - knnVectorValues = getVectorValues(vectorDataType, field.getDocsWithField(), field.getVectors()); + final KNNVectorValues knnVectorValues = knnVectorValuesSupplier.get(); StopWatch stopWatch = new StopWatch().start(); - writer.flushIndex(knnVectorValues, totalLiveDocs); - long time_in_millis = stopWatch.stop().totalTime().millis(); KNNGraphValue.REFRESH_TOTAL_TIME_IN_MILLIS.incrementBy(time_in_millis); log.debug("Flush took {} ms for vector field [{}]", time_in_millis, fieldInfo.getName()); @@ -110,17 +111,20 @@ public void mergeOneField(final FieldInfo fieldInfo, final MergeState mergeState flatVectorsWriter.mergeOneField(fieldInfo, mergeState); final VectorDataType vectorDataType = extractVectorDataType(fieldInfo); - int totalLiveDocs = getLiveDocs(getKNNVectorValuesForMerge(vectorDataType, fieldInfo, mergeState)); + final Supplier> knnVectorValuesSupplier = () -> getKNNVectorValuesForMerge( + vectorDataType, + fieldInfo, + mergeState + ); + int totalLiveDocs = getLiveDocs(knnVectorValuesSupplier.get()); if (totalLiveDocs == 0) { log.debug("[Merge] No live docs for field {}", fieldInfo.getName()); return; } - KNNVectorValues knnVectorValues = getKNNVectorValuesForMerge(vectorDataType, fieldInfo, mergeState); - final QuantizationState quantizationState = train(fieldInfo, knnVectorValues, totalLiveDocs); + final QuantizationState quantizationState = train(fieldInfo, knnVectorValuesSupplier, totalLiveDocs); final NativeIndexWriter writer = NativeIndexWriter.getWriter(fieldInfo, segmentWriteState, quantizationState); - - knnVectorValues = getKNNVectorValuesForMerge(vectorDataType, fieldInfo, mergeState); + final KNNVectorValues knnVectorValues = knnVectorValuesSupplier.get(); StopWatch stopWatch = new StopWatch().start(); @@ -191,27 +195,36 @@ private KNNVectorValues getKNNVectorValuesForMerge( final VectorDataType vectorDataType, final FieldInfo fieldInfo, final MergeState mergeState - ) throws IOException { - switch (fieldInfo.getVectorEncoding()) { - case FLOAT32: - FloatVectorValues mergedFloats = MergedVectorValues.mergeFloatVectorValues(fieldInfo, mergeState); - return getVectorValues(vectorDataType, mergedFloats); - case BYTE: - ByteVectorValues mergedBytes = MergedVectorValues.mergeByteVectorValues(fieldInfo, mergeState); - return getVectorValues(vectorDataType, mergedBytes); - default: - throw new IllegalStateException("Unsupported vector encoding [" + fieldInfo.getVectorEncoding() + "]"); + ) { + try { + switch (fieldInfo.getVectorEncoding()) { + case FLOAT32: + FloatVectorValues mergedFloats = MergedVectorValues.mergeFloatVectorValues(fieldInfo, mergeState); + return getVectorValues(vectorDataType, mergedFloats); + case BYTE: + ByteVectorValues mergedBytes = MergedVectorValues.mergeByteVectorValues(fieldInfo, mergeState); + return getVectorValues(vectorDataType, mergedBytes); + default: + throw new IllegalStateException("Unsupported vector encoding [" + fieldInfo.getVectorEncoding() + "]"); + } + } catch (final IOException e) { + log.error("Unable to merge vectors for field [{}]", fieldInfo.getName(), e); + throw new IllegalStateException("Unable to merge vectors for field [" + fieldInfo.getName() + "]", e); } } - private QuantizationState train(final FieldInfo fieldInfo, final KNNVectorValues knnVectorValues, final int totalLiveDocs) - throws IOException { + private QuantizationState train( + final FieldInfo fieldInfo, + final Supplier> knnVectorValuesSupplier, + final int totalLiveDocs + ) throws IOException { final QuantizationService quantizationService = QuantizationService.getInstance(); final QuantizationParams quantizationParams = quantizationService.getQuantizationParams(fieldInfo); QuantizationState quantizationState = null; if (quantizationParams != null && totalLiveDocs > 0) { initQuantizationStateWriterIfNecessary(); + KNNVectorValues knnVectorValues = knnVectorValuesSupplier.get(); quantizationState = quantizationService.train(quantizationParams, knnVectorValues, totalLiveDocs); quantizationStateWriter.writeState(fieldInfo.getFieldNumber(), quantizationState); } diff --git a/src/test/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsWriterFlushTests.java b/src/test/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsWriterFlushTests.java index ad72f5b24..dbb564908 100644 --- a/src/test/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsWriterFlushTests.java +++ b/src/test/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsWriterFlushTests.java @@ -44,6 +44,7 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.mockConstruction; import static org.mockito.Mockito.mockStatic; +import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -176,6 +177,11 @@ public void testFlush() { throw new RuntimeException(e); } }); + + knnVectorValuesFactoryMockedStatic.verify( + () -> KNNVectorValuesFactory.getVectorValues(any(VectorDataType.class), any(DocsWithFieldSet.class), any()), + times(expectedVectorValues.size()) + ); } } @@ -264,6 +270,11 @@ public void testFlush_WithQuantization() { throw new RuntimeException(e); } }); + + knnVectorValuesFactoryMockedStatic.verify( + () -> KNNVectorValuesFactory.getVectorValues(any(VectorDataType.class), any(DocsWithFieldSet.class), any()), + times(expectedVectorValues.size() * 2) + ); } } diff --git a/src/test/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsWriterMergeTests.java b/src/test/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsWriterMergeTests.java index 440e8bbc5..41940c4d4 100644 --- a/src/test/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsWriterMergeTests.java +++ b/src/test/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsWriterMergeTests.java @@ -45,6 +45,7 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.mockConstruction; import static org.mockito.Mockito.mockStatic; +import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoInteractions; import static org.mockito.Mockito.when; @@ -144,6 +145,10 @@ public void testMerge() { if (!mergedVectors.isEmpty()) { verify(nativeIndexWriter).mergeIndex(knnVectorValues, mergedVectors.size()); assertTrue(KNNGraphValue.MERGE_TOTAL_TIME_IN_MILLIS.getValue() > 0L); + knnVectorValuesFactoryMockedStatic.verify( + () -> KNNVectorValuesFactory.getVectorValues(VectorDataType.FLOAT, floatVectorValues), + times(2) + ); } else { verifyNoInteractions(nativeIndexWriter); } @@ -211,6 +216,10 @@ public void testMerge_WithQuantization() { verify(knn990QuantWriterMockedConstruction.constructed().get(0)).writeState(0, quantizationState); verify(nativeIndexWriter).mergeIndex(knnVectorValues, mergedVectors.size()); assertTrue(KNNGraphValue.MERGE_TOTAL_TIME_IN_MILLIS.getValue() > 0L); + knnVectorValuesFactoryMockedStatic.verify( + () -> KNNVectorValuesFactory.getVectorValues(VectorDataType.FLOAT, floatVectorValues), + times(3) + ); } else { assertEquals(0, knn990QuantWriterMockedConstruction.constructed().size()); verifyNoInteractions(nativeIndexWriter); From 5bad05c653240e3c23c09926b2cb91d607fa5b66 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Wed, 25 Sep 2024 10:07:53 -0700 Subject: [PATCH 382/416] Add space_type as acceptable field in train API (#2141) (#2143) Signed-off-by: Ryan Bogan (cherry picked from commit 189562e89e5205381a8641d123627bec701cadbf) Co-authored-by: Ryan Bogan --- .../knn/plugin/rest/RestTrainModelHandler.java | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/opensearch/knn/plugin/rest/RestTrainModelHandler.java b/src/main/java/org/opensearch/knn/plugin/rest/RestTrainModelHandler.java index 71f7201de..4380310c3 100644 --- a/src/main/java/org/opensearch/knn/plugin/rest/RestTrainModelHandler.java +++ b/src/main/java/org/opensearch/knn/plugin/rest/RestTrainModelHandler.java @@ -126,11 +126,12 @@ private TrainingModelRequest createTransportRequest(RestRequest restRequest) thr mode = parser.text(); } else if (KNNConstants.COMPRESSION_LEVEL_PARAMETER.equals(fieldName) && ensureNotSet(fieldName, compressionLevel)) { compressionLevel = parser.text(); - } else if (KNNConstants.SPACE_TYPE.equals(fieldName) && ensureSpaceTypeNotSet(topLevelSpaceType)) { - topLevelSpaceType = SpaceType.getSpace(parser.text()); - } else { - throw new IllegalArgumentException("Unable to parse token. \"" + fieldName + "\" is not a valid " + "parameter."); - } + } else if ((KNNConstants.SPACE_TYPE.equals(fieldName) || KNNConstants.TOP_LEVEL_PARAMETER_SPACE_TYPE.equals(fieldName)) + && ensureSpaceTypeNotSet(topLevelSpaceType)) { + topLevelSpaceType = SpaceType.getSpace(parser.text()); + } else { + throw new IllegalArgumentException("Unable to parse token. \"" + fieldName + "\" is not a valid " + "parameter."); + } } ensureAtleastOneSet(KNN_METHOD, knnMethodContext, MODE_PARAMETER, mode, COMPRESSION_LEVEL_PARAMETER, compressionLevel); From 4447a439713144c06bea18a00de3b6ac0ca5fe40 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Thu, 26 Sep 2024 14:30:23 -0700 Subject: [PATCH 383/416] KNN80DocValues should only be considered for BinaryDocValues fields (#2147) (#2151) Consider adding files from fields that has BinaryDocValues and doesn't have filter values in producer. Signed-off-by: Vijayan Balasubramanian (cherry picked from commit 329fc5754c4cfb2458b065489cfce0c732e9ce4d) Co-authored-by: Vijayan Balasubramanian --- CHANGELOG.md | 1 + .../KNN80Codec/KNN80DocValuesProducer.java | 9 ++++ .../opensearch/knn/index/OpenSearchIT.java | 4 ++ .../KNN80DocValuesProducerTests.java | 54 +++++++++++++++++++ 4 files changed, 68 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a730a201b..eaad513b6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Enhancements * Add short circuit if no live docs are in segments [#2059](https://github.com/opensearch-project/k-NN/pull/2059) ### Bug Fixes +* KNN80DocValues should only be considered for BinaryDocValues fields [#2147](https://github.com/opensearch-project/k-NN/pull/2147) ### Infrastructure ### Documentation ### Maintenance diff --git a/src/main/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesProducer.java b/src/main/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesProducer.java index 0cfd9c668..b78566f2e 100644 --- a/src/main/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesProducer.java +++ b/src/main/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesProducer.java @@ -15,6 +15,7 @@ import lombok.extern.log4j.Log4j2; import org.apache.lucene.codecs.DocValuesProducer; import org.apache.lucene.index.BinaryDocValues; +import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.NumericDocValues; import org.apache.lucene.index.SegmentReadState; @@ -69,6 +70,13 @@ public KNN80DocValuesProducer(DocValuesProducer delegate, SegmentReadState state if (!field.attributes().containsKey(KNN_FIELD)) { continue; } + // Only segments that contains BinaryDocValues and doesn't have vector values should be considered. + // By default, we don't create BinaryDocValues for knn field anymore. However, users can set doc_values = true + // to create binary doc values explicitly like any other field. Hence, we only want to include fields + // where approximate search is possible only by BinaryDocValues. + if (field.getDocValuesType() != DocValuesType.BINARY || field.hasVectorValues() == true) { + continue; + } // Only Native Engine put into indexPathMap KNNEngine knnEngine = getNativeKNNEngine(field); if (knnEngine == null) { @@ -77,6 +85,7 @@ public KNN80DocValuesProducer(DocValuesProducer delegate, SegmentReadState state List engineFiles = KNNCodecUtil.getEngineFiles(knnEngine.getExtension(), field.name, state.segmentInfo); Path indexPath = PathUtils.get(directoryPath, engineFiles.get(0)); indexPathMap.putIfAbsent(field.getName(), indexPath.toString()); + } } diff --git a/src/test/java/org/opensearch/knn/index/OpenSearchIT.java b/src/test/java/org/opensearch/knn/index/OpenSearchIT.java index 3ef22c758..3333feba7 100644 --- a/src/test/java/org/opensearch/knn/index/OpenSearchIT.java +++ b/src/test/java/org/opensearch/knn/index/OpenSearchIT.java @@ -16,6 +16,7 @@ import java.util.Locale; import lombok.SneakyThrows; import org.junit.BeforeClass; +import org.junit.Ignore; import org.opensearch.knn.KNNRestTestCase; import org.opensearch.knn.KNNResult; import org.apache.http.util.EntityUtils; @@ -483,6 +484,9 @@ public void testIndexingVectorValidation_updateVectorWithNull() throws Exception assertArrayEquals(vectorForDocumentOne, vectorRestoreInitialValue); } + // This doesn't work since indices that are created post 2.17 don't evict by default when indices are closed or deleted. + // Enable this PR once https://github.com/opensearch-project/k-NN/issues/2148 is resolved. + @Ignore public void testCacheClear_whenCloseIndex() throws Exception { String indexName = "test-index-1"; KNNEngine knnEngine1 = KNNEngine.NMSLIB; diff --git a/src/test/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesProducerTests.java b/src/test/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesProducerTests.java index b9a85bbcc..ccceee62b 100644 --- a/src/test/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesProducerTests.java +++ b/src/test/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesProducerTests.java @@ -9,6 +9,7 @@ import org.apache.lucene.codecs.Codec; import org.apache.lucene.codecs.DocValuesFormat; import org.apache.lucene.codecs.DocValuesProducer; +import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.FieldInfos; import org.apache.lucene.index.SegmentInfo; @@ -127,4 +128,57 @@ public void testProduceKNNBinaryField_fromCodec_nmslibCurrent() throws IOExcepti assertTrue(path.contains(segmentFiles.get(0))); } + public void testProduceKNNBinaryField_whenFieldHasNonBinaryDocValues_thenSkipThoseField() throws IOException { + // Set information about the segment and the fields + DocValuesFormat mockDocValuesFormat = mock(DocValuesFormat.class); + Codec mockDelegateCodec = mock(Codec.class); + DocValuesProducer mockDocValuesProducer = mock(DocValuesProducer.class); + when(mockDelegateCodec.docValuesFormat()).thenReturn(mockDocValuesFormat); + when(mockDocValuesFormat.fieldsProducer(any())).thenReturn(mockDocValuesProducer); + when(mockDocValuesFormat.getName()).thenReturn("mockDocValuesFormat"); + Codec codec = new KNN87Codec(mockDelegateCodec); + + String segmentName = "_test"; + int docsInSegment = 100; + String fieldName1 = String.format("test_field1%s", randomAlphaOfLength(4)); + String fieldName2 = String.format("test_field2%s", randomAlphaOfLength(4)); + List segmentFiles = Arrays.asList( + String.format("%s_2011_%s%s", segmentName, fieldName1, KNNEngine.NMSLIB.getExtension()), + String.format("%s_165_%s%s", segmentName, fieldName2, KNNEngine.FAISS.getExtension()) + ); + + KNNEngine knnEngine = KNNEngine.NMSLIB; + SpaceType spaceType = SpaceType.COSINESIMIL; + SegmentInfo segmentInfo = KNNCodecTestUtil.segmentInfoBuilder() + .directory(directory) + .segmentName(segmentName) + .docsInSegment(docsInSegment) + .codec(codec) + .build(); + + for (String name : segmentFiles) { + IndexOutput indexOutput = directory.createOutput(name, IOContext.DEFAULT); + indexOutput.close(); + } + segmentInfo.setFiles(segmentFiles); + + FieldInfo[] fieldInfoArray = new FieldInfo[] { + KNNCodecTestUtil.FieldInfoBuilder.builder(fieldName1) + .addAttribute(KNNVectorFieldMapper.KNN_FIELD, "true") + .addAttribute(KNNConstants.KNN_ENGINE, knnEngine.getName()) + .addAttribute(KNNConstants.SPACE_TYPE, spaceType.getValue()) + .docValuesType(DocValuesType.NONE) + .dvGen(-1) + .build() }; + + FieldInfos fieldInfos = new FieldInfos(fieldInfoArray); + SegmentReadState state = new SegmentReadState(directory, segmentInfo, fieldInfos, IOContext.DEFAULT); + + DocValuesFormat docValuesFormat = codec.docValuesFormat(); + assertTrue(docValuesFormat instanceof KNN80DocValuesFormat); + DocValuesProducer producer = docValuesFormat.fieldsProducer(state); + assertTrue(producer instanceof KNN80DocValuesProducer); + assertEquals(0, ((KNN80DocValuesProducer) producer).getOpenedIndexPath().size()); + } + } From db37c937930abe0e861d5851a29e25663a7db5fb Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Thu, 26 Sep 2024 16:54:08 -0500 Subject: [PATCH 384/416] Update Default Rescore Context based on Dimesion (#2149) (#2150) (cherry picked from commit a1227eafe1f539cfb230468c5213e02a015cb8dc) Signed-off-by: Naveen Tatikonda Co-authored-by: Vikasht34 --- CHANGELOG.md | 1 + .../knn/index/mapper/CompressionLevel.java | 29 ++++++++- .../knn/index/mapper/KNNVectorFieldType.java | 6 +- .../index/query/rescore/RescoreContext.java | 2 + .../index/mapper/CompressionLevelTests.java | 61 +++++++++++++++---- 5 files changed, 85 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index eaad513b6..61d20155f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), * Add AVX512 support to k-NN for FAISS library [#2069](https://github.com/opensearch-project/k-NN/pull/2069) ### Enhancements * Add short circuit if no live docs are in segments [#2059](https://github.com/opensearch-project/k-NN/pull/2059) +* Update Default Rescore Context based on Dimension [#2149](https://github.com/opensearch-project/k-NN/pull/2149) ### Bug Fixes * KNN80DocValues should only be considered for BinaryDocValues fields [#2147](https://github.com/opensearch-project/k-NN/pull/2147) ### Infrastructure diff --git a/src/main/java/org/opensearch/knn/index/mapper/CompressionLevel.java b/src/main/java/org/opensearch/knn/index/mapper/CompressionLevel.java index 0709239cf..3e1b47db7 100644 --- a/src/main/java/org/opensearch/knn/index/mapper/CompressionLevel.java +++ b/src/main/java/org/opensearch/knn/index/mapper/CompressionLevel.java @@ -94,9 +94,34 @@ public static boolean isConfigured(CompressionLevel compressionLevel) { return compressionLevel != null && compressionLevel != NOT_CONFIGURED; } - public RescoreContext getDefaultRescoreContext(Mode mode) { + /** + * Returns the appropriate {@link RescoreContext} based on the given {@code mode} and {@code dimension}. + * + *

    If the {@code mode} is present in the valid {@code modesForRescore} set, the method checks the value of + * {@code dimension}: + *

      + *
    • If {@code dimension} is less than or equal to 1000, it returns a {@link RescoreContext} with an + * oversample factor of 5.0f.
      • + *
    • If {@code dimension} is greater than 1000, it returns the default {@link RescoreContext} associated with + * the {@link CompressionLevel}. If no default is set, it falls back to {@link RescoreContext#getDefault()}.
      • + *
    + * If the {@code mode} is not valid, the method returns {@code null}. + * + * @param mode The {@link Mode} for which to retrieve the {@link RescoreContext}. + * @param dimension The dimensional value that determines the {@link RescoreContext} behavior. + * @return A {@link RescoreContext} with an oversample factor of 5.0f if {@code dimension} is less than + * or equal to 1000, the default {@link RescoreContext} if greater, or {@code null} if the mode + * is invalid. + */ + public RescoreContext getDefaultRescoreContext(Mode mode, int dimension) { if (modesForRescore.contains(mode)) { - return defaultRescoreContext; + // Adjust RescoreContext based on dimension + if (dimension <= RescoreContext.DIMENSION_THRESHOLD) { + // For dimensions <= 1000, return a RescoreContext with 5.0f oversample factor + return RescoreContext.builder().oversampleFactor(RescoreContext.OVERSAMPLE_FACTOR_BELOW_DIMENSION_THRESHOLD).build(); + } else { + return defaultRescoreContext; + } } return null; } diff --git a/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldType.java b/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldType.java index e684ba4f1..a0832c1d0 100644 --- a/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldType.java +++ b/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldType.java @@ -93,6 +93,10 @@ public RescoreContext resolveRescoreContext(RescoreContext userProvidedContext) if (userProvidedContext != null) { return userProvidedContext; } - return getKnnMappingConfig().getCompressionLevel().getDefaultRescoreContext(getKnnMappingConfig().getMode()); + KNNMappingConfig knnMappingConfig = getKnnMappingConfig(); + int dimension = knnMappingConfig.getDimension(); + CompressionLevel compressionLevel = knnMappingConfig.getCompressionLevel(); + Mode mode = knnMappingConfig.getMode(); + return compressionLevel.getDefaultRescoreContext(mode, dimension); } } diff --git a/src/main/java/org/opensearch/knn/index/query/rescore/RescoreContext.java b/src/main/java/org/opensearch/knn/index/query/rescore/RescoreContext.java index ea4e92215..51d4e491c 100644 --- a/src/main/java/org/opensearch/knn/index/query/rescore/RescoreContext.java +++ b/src/main/java/org/opensearch/knn/index/query/rescore/RescoreContext.java @@ -21,6 +21,8 @@ public final class RescoreContext { public static final float MIN_OVERSAMPLE_FACTOR = 1.0f; public static final int MAX_FIRST_PASS_RESULTS = 10000; + public static final int DIMENSION_THRESHOLD = 1000; + public static final float OVERSAMPLE_FACTOR_BELOW_DIMENSION_THRESHOLD = 5.0f; // Todo:- We will improve this in upcoming releases public static final int MIN_FIRST_PASS_RESULTS = 100; diff --git a/src/test/java/org/opensearch/knn/index/mapper/CompressionLevelTests.java b/src/test/java/org/opensearch/knn/index/mapper/CompressionLevelTests.java index 9eb302b13..cc70d4c2d 100644 --- a/src/test/java/org/opensearch/knn/index/mapper/CompressionLevelTests.java +++ b/src/test/java/org/opensearch/knn/index/mapper/CompressionLevelTests.java @@ -44,26 +44,65 @@ public void testIsConfigured() { public void testGetDefaultRescoreContext() { // Test rescore context for ON_DISK mode Mode mode = Mode.ON_DISK; + int belowThresholdDimension = 500; // A dimension below the threshold + int aboveThresholdDimension = 1500; // A dimension above the threshold - // x32 should have RescoreContext with an oversample factor of 3.0f - RescoreContext rescoreContext = CompressionLevel.x32.getDefaultRescoreContext(mode); + // x32 with dimension <= 1000 should have an oversample factor of 5.0f + RescoreContext rescoreContext = CompressionLevel.x32.getDefaultRescoreContext(mode, belowThresholdDimension); + assertNotNull(rescoreContext); + assertEquals(5.0f, rescoreContext.getOversampleFactor(), 0.0f); + + // x32 with dimension > 1000 should have an oversample factor of 3.0f + rescoreContext = CompressionLevel.x32.getDefaultRescoreContext(mode, aboveThresholdDimension); assertNotNull(rescoreContext); assertEquals(3.0f, rescoreContext.getOversampleFactor(), 0.0f); - // x16 should have RescoreContext with an oversample factor of 3.0f - rescoreContext = CompressionLevel.x16.getDefaultRescoreContext(mode); + // x16 with dimension <= 1000 should have an oversample factor of 5.0f + rescoreContext = CompressionLevel.x16.getDefaultRescoreContext(mode, belowThresholdDimension); + assertNotNull(rescoreContext); + assertEquals(5.0f, rescoreContext.getOversampleFactor(), 0.0f); + + // x16 with dimension > 1000 should have an oversample factor of 3.0f + rescoreContext = CompressionLevel.x16.getDefaultRescoreContext(mode, aboveThresholdDimension); assertNotNull(rescoreContext); assertEquals(3.0f, rescoreContext.getOversampleFactor(), 0.0f); - // x8 should have RescoreContext with an oversample factor of 2.0f - rescoreContext = CompressionLevel.x8.getDefaultRescoreContext(mode); + // x8 with dimension <= 1000 should have an oversample factor of 5.0f + rescoreContext = CompressionLevel.x8.getDefaultRescoreContext(mode, belowThresholdDimension); + assertNotNull(rescoreContext); + assertEquals(5.0f, rescoreContext.getOversampleFactor(), 0.0f); + + // x8 with dimension > 1000 should have an oversample factor of 2.0f + rescoreContext = CompressionLevel.x8.getDefaultRescoreContext(mode, aboveThresholdDimension); assertNotNull(rescoreContext); assertEquals(2.0f, rescoreContext.getOversampleFactor(), 0.0f); - // Other compression levels should not have a RescoreContext for ON_DISK mode - assertNull(CompressionLevel.x4.getDefaultRescoreContext(mode)); - assertNull(CompressionLevel.x2.getDefaultRescoreContext(mode)); - assertNull(CompressionLevel.x1.getDefaultRescoreContext(mode)); - assertNull(CompressionLevel.NOT_CONFIGURED.getDefaultRescoreContext(mode)); + // x4 with dimension <= 1000 should have an oversample factor of 5.0f (though it doesn't have its own RescoreContext) + rescoreContext = CompressionLevel.x4.getDefaultRescoreContext(mode, belowThresholdDimension); + assertNull(rescoreContext); + // x4 with dimension > 1000 should return null (no RescoreContext is configured for x4) + rescoreContext = CompressionLevel.x4.getDefaultRescoreContext(mode, aboveThresholdDimension); + assertNull(rescoreContext); + + // Other compression levels should behave similarly with respect to dimension + + rescoreContext = CompressionLevel.x2.getDefaultRescoreContext(mode, belowThresholdDimension); + assertNull(rescoreContext); + + // x2 with dimension > 1000 should return null + rescoreContext = CompressionLevel.x2.getDefaultRescoreContext(mode, aboveThresholdDimension); + assertNull(rescoreContext); + + rescoreContext = CompressionLevel.x1.getDefaultRescoreContext(mode, belowThresholdDimension); + assertNull(rescoreContext); + + // x1 with dimension > 1000 should return null + rescoreContext = CompressionLevel.x1.getDefaultRescoreContext(mode, aboveThresholdDimension); + assertNull(rescoreContext); + + // NOT_CONFIGURED with dimension <= 1000 should return a RescoreContext with an oversample factor of 5.0f + rescoreContext = CompressionLevel.NOT_CONFIGURED.getDefaultRescoreContext(mode, belowThresholdDimension); + assertNull(rescoreContext); + } } From af02da5a46ff41bfa67dab41d4da0060f9790190 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Fri, 27 Sep 2024 11:10:31 -0500 Subject: [PATCH 385/416] Add Release Notes for 2.17.1.0 (#2154) (#2156) (cherry picked from commit 23b95e7571f28e36725fd8944c11c25a50253f14) Signed-off-by: Naveen Tatikonda Co-authored-by: Naveen Tatikonda --- CHANGELOG.md | 1 - release-notes/opensearch-knn.release-notes-2.17.1.0.md | 8 ++++++++ 2 files changed, 8 insertions(+), 1 deletion(-) create mode 100644 release-notes/opensearch-knn.release-notes-2.17.1.0.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 61d20155f..864cd527d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,6 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ## [Unreleased 3.0](https://github.com/opensearch-project/k-NN/compare/2.x...HEAD) ### Features ### Enhancements -* Adds concurrent segment search support for mode auto [#2111](https://github.com/opensearch-project/k-NN/pull/2111) ### Bug Fixes * Add DocValuesProducers for releasing memory when close index [#1946](https://github.com/opensearch-project/k-NN/pull/1946) ### Infrastructure diff --git a/release-notes/opensearch-knn.release-notes-2.17.1.0.md b/release-notes/opensearch-knn.release-notes-2.17.1.0.md new file mode 100644 index 000000000..0af275a5b --- /dev/null +++ b/release-notes/opensearch-knn.release-notes-2.17.1.0.md @@ -0,0 +1,8 @@ +## Version 2.17.1.0 Release Notes + +Compatible with OpenSearch 2.17.1 + +### Enhancements +* Adds concurrent segment search support for mode auto [#2111](https://github.com/opensearch-project/k-NN/pull/2111) +### Bug Fixes +* Change min oversample to 1 [#2117](https://github.com/opensearch-project/k-NN/pull/2117) From 47d47c0b35d0bb8426ef99db21505e7e98ccb712 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Fri, 27 Sep 2024 23:53:34 -0700 Subject: [PATCH 386/416] KNNIterators should support with and without filters (#2155) (#2166) * Rename class names to represent both and filter and non filter use cases * Iterator should support with filters Update VectorIterator and NesterVector Iterator to iterate even if there is no filters provided to iterator. Currently this is used by exact search to score either topk docs or all docs when filter is provided by users. However, in future we will be allowing exact search even if there are no filters. Hence, decouple filter and make it option to support both cases. --------- Signed-off-by: Vijayan Balasubramanian (cherry picked from commit 6f6dd566e4871ddb2d0b5829c08aab9e4af90774) Co-authored-by: Vijayan Balasubramanian --- CHANGELOG.md | 1 + .../knn/index/query/ExactSearcher.java | 47 +++++---- .../ByteVectorIdsKNNIterator.java} | 45 ++++++--- .../{filtered => iterators}/KNNIterator.java | 2 +- .../NestedByteVectorIdsKNNIterator.java} | 32 ++++-- .../NestedVectorIdsKNNIterator.java} | 37 ++++--- .../VectorIdsKNNIterator.java} | 51 ++++++---- .../FilteredIdsKNNByteIteratorTests.java | 50 ---------- .../filtered/FilteredIdsKNNIteratorTests.java | 54 ---------- .../ByteVectorIdsKNNIteratorTests.java | 97 ++++++++++++++++++ .../NestedByteVectorIdsKNNIteratorTests.java} | 36 ++++++- .../NestedVectorIdsKNNIteratorTests.java} | 46 +++++++-- .../iterators/VectorIdsKNNIteratorTests.java | 98 +++++++++++++++++++ 13 files changed, 404 insertions(+), 192 deletions(-) rename src/main/java/org/opensearch/knn/index/query/{filtered/FilteredIdsKNNByteIterator.java => iterators/ByteVectorIdsKNNIterator.java} (57%) rename src/main/java/org/opensearch/knn/index/query/{filtered => iterators}/KNNIterator.java (80%) rename src/main/java/org/opensearch/knn/index/query/{filtered/NestedFilteredIdsKNNByteIterator.java => iterators/NestedByteVectorIdsKNNIterator.java} (54%) rename src/main/java/org/opensearch/knn/index/query/{filtered/NestedFilteredIdsKNNIterator.java => iterators/NestedVectorIdsKNNIterator.java} (59%) rename src/main/java/org/opensearch/knn/index/query/{filtered/FilteredIdsKNNIterator.java => iterators/VectorIdsKNNIterator.java} (65%) delete mode 100644 src/test/java/org/opensearch/knn/index/query/filtered/FilteredIdsKNNByteIteratorTests.java delete mode 100644 src/test/java/org/opensearch/knn/index/query/filtered/FilteredIdsKNNIteratorTests.java create mode 100644 src/test/java/org/opensearch/knn/index/query/iterators/ByteVectorIdsKNNIteratorTests.java rename src/test/java/org/opensearch/knn/index/query/{filtered/NestedFilteredIdsKNNByteIteratorTests.java => iterators/NestedByteVectorIdsKNNIteratorTests.java} (54%) rename src/test/java/org/opensearch/knn/index/query/{filtered/NestedFilteredIdsKNNIteratorTests.java => iterators/NestedVectorIdsKNNIteratorTests.java} (55%) create mode 100644 src/test/java/org/opensearch/knn/index/query/iterators/VectorIdsKNNIteratorTests.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 864cd527d..edf390f80 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Enhancements * Add short circuit if no live docs are in segments [#2059](https://github.com/opensearch-project/k-NN/pull/2059) * Update Default Rescore Context based on Dimension [#2149](https://github.com/opensearch-project/k-NN/pull/2149) +* KNNIterators should support with and without filters [#2155](https://github.com/opensearch-project/k-NN/pull/2155) ### Bug Fixes * KNN80DocValues should only be considered for BinaryDocValues fields [#2147](https://github.com/opensearch-project/k-NN/pull/2147) ### Infrastructure diff --git a/src/main/java/org/opensearch/knn/index/query/ExactSearcher.java b/src/main/java/org/opensearch/knn/index/query/ExactSearcher.java index 5b6029766..193cba8c1 100644 --- a/src/main/java/org/opensearch/knn/index/query/ExactSearcher.java +++ b/src/main/java/org/opensearch/knn/index/query/ExactSearcher.java @@ -20,11 +20,11 @@ import org.opensearch.knn.common.FieldInfoExtractor; import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.VectorDataType; -import org.opensearch.knn.index.query.filtered.FilteredIdsKNNByteIterator; -import org.opensearch.knn.index.query.filtered.FilteredIdsKNNIterator; -import org.opensearch.knn.index.query.filtered.KNNIterator; -import org.opensearch.knn.index.query.filtered.NestedFilteredIdsKNNByteIterator; -import org.opensearch.knn.index.query.filtered.NestedFilteredIdsKNNIterator; +import org.opensearch.knn.index.query.iterators.ByteVectorIdsKNNIterator; +import org.opensearch.knn.index.query.iterators.VectorIdsKNNIterator; +import org.opensearch.knn.index.query.iterators.KNNIterator; +import org.opensearch.knn.index.query.iterators.NestedByteVectorIdsKNNIterator; +import org.opensearch.knn.index.query.iterators.NestedVectorIdsKNNIterator; import org.opensearch.knn.index.vectorvalues.KNNBinaryVectorValues; import org.opensearch.knn.index.vectorvalues.KNNFloatVectorValues; import org.opensearch.knn.index.vectorvalues.KNNVectorValues; @@ -51,8 +51,9 @@ public class ExactSearcher { */ public Map searchLeaf(final LeafReaderContext leafReaderContext, final ExactSearcherContext exactSearcherContext) throws IOException { - KNNIterator iterator = getMatchedKNNIterator(leafReaderContext, exactSearcherContext); - if (exactSearcherContext.getMatchedDocs().cardinality() <= exactSearcherContext.getK()) { + KNNIterator iterator = getKNNIterator(leafReaderContext, exactSearcherContext); + if (exactSearcherContext.getMatchedDocs() != null + && exactSearcherContext.getMatchedDocs().cardinality() <= exactSearcherContext.getK()) { return scoreAllDocs(iterator); } return searchTopK(iterator, exactSearcherContext.getK()); @@ -98,8 +99,7 @@ private Map searchTopK(KNNIterator iterator, int k) throws IOExc return docToScore; } - private KNNIterator getMatchedKNNIterator(LeafReaderContext leafReaderContext, ExactSearcherContext exactSearcherContext) - throws IOException { + private KNNIterator getKNNIterator(LeafReaderContext leafReaderContext, ExactSearcherContext exactSearcherContext) throws IOException { final KNNQuery knnQuery = exactSearcherContext.getKnnQuery(); final BitSet matchedDocs = exactSearcherContext.getMatchedDocs(); final SegmentReader reader = Lucene.segmentReader(leafReaderContext.reader()); @@ -108,20 +108,18 @@ private KNNIterator getMatchedKNNIterator(LeafReaderContext leafReaderContext, E boolean isNestedRequired = exactSearcherContext.isParentHits() && knnQuery.getParentsFilter() != null; - if (VectorDataType.BINARY == knnQuery.getVectorDataType() && isNestedRequired) { - final KNNVectorValues vectorValues = KNNVectorValuesFactory.getVectorValues(fieldInfo, reader); - return new NestedFilteredIdsKNNByteIterator( - matchedDocs, - knnQuery.getByteQueryVector(), - (KNNBinaryVectorValues) vectorValues, - spaceType, - knnQuery.getParentsFilter().getBitSet(leafReaderContext) - ); - } - if (VectorDataType.BINARY == knnQuery.getVectorDataType()) { final KNNVectorValues vectorValues = KNNVectorValuesFactory.getVectorValues(fieldInfo, reader); - return new FilteredIdsKNNByteIterator( + if (isNestedRequired) { + return new NestedByteVectorIdsKNNIterator( + matchedDocs, + knnQuery.getByteQueryVector(), + (KNNBinaryVectorValues) vectorValues, + spaceType, + knnQuery.getParentsFilter().getBitSet(leafReaderContext) + ); + } + return new ByteVectorIdsKNNIterator( matchedDocs, knnQuery.getByteQueryVector(), (KNNBinaryVectorValues) vectorValues, @@ -142,7 +140,7 @@ private KNNIterator getMatchedKNNIterator(LeafReaderContext leafReaderContext, E final KNNVectorValues vectorValues = KNNVectorValuesFactory.getVectorValues(fieldInfo, reader); if (isNestedRequired) { - return new NestedFilteredIdsKNNIterator( + return new NestedVectorIdsKNNIterator( matchedDocs, knnQuery.getQueryVector(), (KNNFloatVectorValues) vectorValues, @@ -152,8 +150,7 @@ private KNNIterator getMatchedKNNIterator(LeafReaderContext leafReaderContext, E segmentLevelQuantizationInfo ); } - - return new FilteredIdsKNNIterator( + return new VectorIdsKNNIterator( matchedDocs, knnQuery.getQueryVector(), (KNNFloatVectorValues) vectorValues, @@ -180,7 +177,7 @@ public static class ExactSearcherContext { KNNQuery knnQuery; /** * whether the matchedDocs contains parent ids or child ids. This is relevant in the case of - * filtered nested search where the matchedDocs contain the parent ids and {@link NestedFilteredIdsKNNIterator} + * filtered nested search where the matchedDocs contain the parent ids and {@link NestedVectorIdsKNNIterator} * needs to be used. */ boolean isParentHits; diff --git a/src/main/java/org/opensearch/knn/index/query/filtered/FilteredIdsKNNByteIterator.java b/src/main/java/org/opensearch/knn/index/query/iterators/ByteVectorIdsKNNIterator.java similarity index 57% rename from src/main/java/org/opensearch/knn/index/query/filtered/FilteredIdsKNNByteIterator.java rename to src/main/java/org/opensearch/knn/index/query/iterators/ByteVectorIdsKNNIterator.java index ccfe626a0..b1aea4284 100644 --- a/src/main/java/org/opensearch/knn/index/query/filtered/FilteredIdsKNNByteIterator.java +++ b/src/main/java/org/opensearch/knn/index/query/iterators/ByteVectorIdsKNNIterator.java @@ -3,11 +3,12 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.knn.index.query.filtered; +package org.opensearch.knn.index.query.iterators; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.util.BitSet; import org.apache.lucene.util.BitSetIterator; +import org.opensearch.common.Nullable; import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.vectorvalues.KNNBinaryVectorValues; @@ -17,11 +18,9 @@ * Inspired by DiversifyingChildrenFloatKnnVectorQuery in lucene * https://github.com/apache/lucene/blob/7b8aece125aabff2823626d5b939abf4747f63a7/lucene/join/src/java/org/apache/lucene/search/join/DiversifyingChildrenFloatKnnVectorQuery.java#L162 * - * The class is used in KNNWeight to score filtered KNN field by iterating filterIdsArray. + * The class is used in KNNWeight to score all docs, but, it iterates over filterIdsArray if filter is provided */ -public class FilteredIdsKNNByteIterator implements KNNIterator { - // Array of doc ids to iterate - protected final BitSet filterIdsBitSet; +public class ByteVectorIdsKNNIterator implements KNNIterator { protected final BitSetIterator bitSetIterator; protected final byte[] queryVector; protected final KNNBinaryVectorValues binaryVectorValues; @@ -29,18 +28,24 @@ public class FilteredIdsKNNByteIterator implements KNNIterator { protected float currentScore = Float.NEGATIVE_INFINITY; protected int docId; - public FilteredIdsKNNByteIterator( - final BitSet filterIdsBitSet, + public ByteVectorIdsKNNIterator( + @Nullable final BitSet filterIdsBitSet, final byte[] queryVector, final KNNBinaryVectorValues binaryVectorValues, final SpaceType spaceType - ) { - this.filterIdsBitSet = filterIdsBitSet; - this.bitSetIterator = new BitSetIterator(filterIdsBitSet, filterIdsBitSet.length()); + ) throws IOException { + this.bitSetIterator = filterIdsBitSet == null ? null : new BitSetIterator(filterIdsBitSet, filterIdsBitSet.length()); this.queryVector = queryVector; this.binaryVectorValues = binaryVectorValues; this.spaceType = spaceType; - this.docId = bitSetIterator.nextDoc(); + // This cannot be moved inside nextDoc() method since it will break when we have nested field, where + // nextDoc should already be referring to next knnVectorValues + this.docId = getNextDocId(); + } + + public ByteVectorIdsKNNIterator(final byte[] queryVector, final KNNBinaryVectorValues binaryVectorValues, final SpaceType spaceType) + throws IOException { + this(null, queryVector, binaryVectorValues, spaceType); } /** @@ -55,10 +60,10 @@ public int nextDoc() throws IOException { if (docId == DocIdSetIterator.NO_MORE_DOCS) { return DocIdSetIterator.NO_MORE_DOCS; } - int doc = binaryVectorValues.advance(docId); currentScore = computeScore(); - docId = bitSetIterator.nextDoc(); - return doc; + int currentDocId = docId; + docId = getNextDocId(); + return currentDocId; } @Override @@ -72,4 +77,16 @@ protected float computeScore() throws IOException { // scores correspond to closer vectors. return spaceType.getKnnVectorSimilarityFunction().compare(queryVector, vector); } + + protected int getNextDocId() throws IOException { + if (bitSetIterator == null) { + return binaryVectorValues.nextDoc(); + } + int nextDocID = this.bitSetIterator.nextDoc(); + // For filter case, advance vector values to corresponding doc id from filter bit set + if (nextDocID != DocIdSetIterator.NO_MORE_DOCS) { + binaryVectorValues.advance(nextDocID); + } + return nextDocID; + } } diff --git a/src/main/java/org/opensearch/knn/index/query/filtered/KNNIterator.java b/src/main/java/org/opensearch/knn/index/query/iterators/KNNIterator.java similarity index 80% rename from src/main/java/org/opensearch/knn/index/query/filtered/KNNIterator.java rename to src/main/java/org/opensearch/knn/index/query/iterators/KNNIterator.java index 4a105975a..00cbb3aa2 100644 --- a/src/main/java/org/opensearch/knn/index/query/filtered/KNNIterator.java +++ b/src/main/java/org/opensearch/knn/index/query/iterators/KNNIterator.java @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.knn.index.query.filtered; +package org.opensearch.knn.index.query.iterators; import java.io.IOException; diff --git a/src/main/java/org/opensearch/knn/index/query/filtered/NestedFilteredIdsKNNByteIterator.java b/src/main/java/org/opensearch/knn/index/query/iterators/NestedByteVectorIdsKNNIterator.java similarity index 54% rename from src/main/java/org/opensearch/knn/index/query/filtered/NestedFilteredIdsKNNByteIterator.java rename to src/main/java/org/opensearch/knn/index/query/iterators/NestedByteVectorIdsKNNIterator.java index b69a90518..3c93ec888 100644 --- a/src/main/java/org/opensearch/knn/index/query/filtered/NestedFilteredIdsKNNByteIterator.java +++ b/src/main/java/org/opensearch/knn/index/query/iterators/NestedByteVectorIdsKNNIterator.java @@ -3,33 +3,45 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.knn.index.query.filtered; +package org.opensearch.knn.index.query.iterators; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.util.BitSet; +import org.opensearch.common.Nullable; import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.vectorvalues.KNNBinaryVectorValues; import java.io.IOException; /** - * This iterator iterates filterIdsArray to score. However, it dedupe docs per each parent doc + * This iterator iterates filterIdsArray to score if filter is provided else it iterates over all docs. + * However, it dedupe docs per each parent doc * of which ID is set in parentBitSet and only return best child doc with the highest score. */ -public class NestedFilteredIdsKNNByteIterator extends FilteredIdsKNNByteIterator { +public class NestedByteVectorIdsKNNIterator extends ByteVectorIdsKNNIterator { private final BitSet parentBitSet; - public NestedFilteredIdsKNNByteIterator( - final BitSet filterIdsArray, + public NestedByteVectorIdsKNNIterator( + @Nullable final BitSet filterIdsArray, final byte[] queryVector, final KNNBinaryVectorValues binaryVectorValues, final SpaceType spaceType, final BitSet parentBitSet - ) { + ) throws IOException { super(filterIdsArray, queryVector, binaryVectorValues, spaceType); this.parentBitSet = parentBitSet; } + public NestedByteVectorIdsKNNIterator( + final byte[] queryVector, + final KNNBinaryVectorValues binaryVectorValues, + final SpaceType spaceType, + final BitSet parentBitSet + ) throws IOException { + super(null, queryVector, binaryVectorValues, spaceType); + this.parentBitSet = parentBitSet; + } + /** * Advance to the next best child doc per parent and update score with the best score among child docs from the parent. * DocIdSetIterator.NO_MORE_DOCS is returned when there is no more docs @@ -46,14 +58,18 @@ public int nextDoc() throws IOException { int currentParent = parentBitSet.nextSetBit(docId); int bestChild = -1; + // In order to traverse all children for given parent, we have to use docId < parentId, because, + // kNNVectorValues will not have parent id since DocId is unique per segment. For ex: let's say for doc id 1, there is one child + // and for doc id 5, there are three children. In that case knnVectorValues iterator will have [0, 2, 3, 4] + // and parentBitSet will have [1,5] + // Hence, we have to iterate till docId from knnVectorValues is less than parentId instead of till equal to parentId while (docId != DocIdSetIterator.NO_MORE_DOCS && docId < currentParent) { - binaryVectorValues.advance(docId); float score = computeScore(); if (score > currentScore) { bestChild = docId; currentScore = score; } - docId = bitSetIterator.nextDoc(); + docId = getNextDocId(); } return bestChild; diff --git a/src/main/java/org/opensearch/knn/index/query/filtered/NestedFilteredIdsKNNIterator.java b/src/main/java/org/opensearch/knn/index/query/iterators/NestedVectorIdsKNNIterator.java similarity index 59% rename from src/main/java/org/opensearch/knn/index/query/filtered/NestedFilteredIdsKNNIterator.java rename to src/main/java/org/opensearch/knn/index/query/iterators/NestedVectorIdsKNNIterator.java index 53ac72882..692793b99 100644 --- a/src/main/java/org/opensearch/knn/index/query/filtered/NestedFilteredIdsKNNIterator.java +++ b/src/main/java/org/opensearch/knn/index/query/iterators/NestedVectorIdsKNNIterator.java @@ -3,10 +3,11 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.knn.index.query.filtered; +package org.opensearch.knn.index.query.iterators; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.util.BitSet; +import org.opensearch.common.Nullable; import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.query.SegmentLevelQuantizationInfo; import org.opensearch.knn.index.vectorvalues.KNNFloatVectorValues; @@ -14,31 +15,41 @@ import java.io.IOException; /** - * This iterator iterates filterIdsArray to score. However, it dedupe docs per each parent doc + * This iterator iterates filterIdsArray to score if filter is provided else it iterates over all docs. + * However, it dedupe docs per each parent doc * of which ID is set in parentBitSet and only return best child doc with the highest score. */ -public class NestedFilteredIdsKNNIterator extends FilteredIdsKNNIterator { +public class NestedVectorIdsKNNIterator extends VectorIdsKNNIterator { private final BitSet parentBitSet; - NestedFilteredIdsKNNIterator( - final BitSet filterIdsArray, + public NestedVectorIdsKNNIterator( + @Nullable final BitSet filterIdsArray, final float[] queryVector, final KNNFloatVectorValues knnFloatVectorValues, final SpaceType spaceType, final BitSet parentBitSet - ) { + ) throws IOException { this(filterIdsArray, queryVector, knnFloatVectorValues, spaceType, parentBitSet, null, null); } - public NestedFilteredIdsKNNIterator( - final BitSet filterIdsArray, + public NestedVectorIdsKNNIterator( + final float[] queryVector, + final KNNFloatVectorValues knnFloatVectorValues, + final SpaceType spaceType, + final BitSet parentBitSet + ) throws IOException { + this(null, queryVector, knnFloatVectorValues, spaceType, parentBitSet, null, null); + } + + public NestedVectorIdsKNNIterator( + @Nullable final BitSet filterIdsArray, final float[] queryVector, final KNNFloatVectorValues knnFloatVectorValues, final SpaceType spaceType, final BitSet parentBitSet, final byte[] quantizedVector, final SegmentLevelQuantizationInfo segmentLevelQuantizationInfo - ) { + ) throws IOException { super(filterIdsArray, queryVector, knnFloatVectorValues, spaceType, quantizedVector, segmentLevelQuantizationInfo); this.parentBitSet = parentBitSet; } @@ -59,14 +70,18 @@ public int nextDoc() throws IOException { int currentParent = parentBitSet.nextSetBit(docId); int bestChild = -1; + // In order to traverse all children for given parent, we have to use docId < parentId, because, + // kNNVectorValues will not have parent id since DocId is unique per segment. For ex: let's say for doc id 1, there is one child + // and for doc id 5, there are three children. In that case knnVectorValues iterator will have [0, 2, 3, 4] + // and parentBitSet will have [1,5] + // Hence, we have to iterate till docId from knnVectorValues is less than parentId instead of till equal to parentId while (docId != DocIdSetIterator.NO_MORE_DOCS && docId < currentParent) { - knnFloatVectorValues.advance(docId); float score = computeScore(); if (score > currentScore) { bestChild = docId; currentScore = score; } - docId = bitSetIterator.nextDoc(); + docId = getNextDocId(); } return bestChild; diff --git a/src/main/java/org/opensearch/knn/index/query/filtered/FilteredIdsKNNIterator.java b/src/main/java/org/opensearch/knn/index/query/iterators/VectorIdsKNNIterator.java similarity index 65% rename from src/main/java/org/opensearch/knn/index/query/filtered/FilteredIdsKNNIterator.java rename to src/main/java/org/opensearch/knn/index/query/iterators/VectorIdsKNNIterator.java index 56d291470..9fb354242 100644 --- a/src/main/java/org/opensearch/knn/index/query/filtered/FilteredIdsKNNIterator.java +++ b/src/main/java/org/opensearch/knn/index/query/iterators/VectorIdsKNNIterator.java @@ -3,11 +3,12 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.knn.index.query.filtered; +package org.opensearch.knn.index.query.iterators; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.util.BitSet; import org.apache.lucene.util.BitSetIterator; +import org.opensearch.common.Nullable; import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.query.SegmentLevelQuantizationInfo; import org.opensearch.knn.index.query.SegmentLevelQuantizationUtil; @@ -19,11 +20,9 @@ * Inspired by DiversifyingChildrenFloatKnnVectorQuery in lucene * https://github.com/apache/lucene/blob/7b8aece125aabff2823626d5b939abf4747f63a7/lucene/join/src/java/org/apache/lucene/search/join/DiversifyingChildrenFloatKnnVectorQuery.java#L162 * - * The class is used in KNNWeight to score filtered KNN field by iterating filterIdsArray. + * The class is used in KNNWeight to score all docs, but, it iterates over filterIdsArray if filter is provided */ -public class FilteredIdsKNNIterator implements KNNIterator { - // Array of doc ids to iterate - protected final BitSet filterIdsBitSet; +public class VectorIdsKNNIterator implements KNNIterator { protected final BitSetIterator bitSetIterator; protected final float[] queryVector; private final byte[] quantizedQueryVector; @@ -33,29 +32,35 @@ public class FilteredIdsKNNIterator implements KNNIterator { protected int docId; private final SegmentLevelQuantizationInfo segmentLevelQuantizationInfo; - FilteredIdsKNNIterator( - final BitSet filterIdsBitSet, + public VectorIdsKNNIterator( + @Nullable final BitSet filterIdsBitSet, final float[] queryVector, final KNNFloatVectorValues knnFloatVectorValues, final SpaceType spaceType - ) { + ) throws IOException { this(filterIdsBitSet, queryVector, knnFloatVectorValues, spaceType, null, null); } - public FilteredIdsKNNIterator( - final BitSet filterIdsBitSet, + public VectorIdsKNNIterator(final float[] queryVector, final KNNFloatVectorValues knnFloatVectorValues, final SpaceType spaceType) + throws IOException { + this(null, queryVector, knnFloatVectorValues, spaceType, null, null); + } + + public VectorIdsKNNIterator( + @Nullable final BitSet filterIdsBitSet, final float[] queryVector, final KNNFloatVectorValues knnFloatVectorValues, final SpaceType spaceType, final byte[] quantizedQueryVector, final SegmentLevelQuantizationInfo segmentLevelQuantizationInfo - ) { - this.filterIdsBitSet = filterIdsBitSet; - this.bitSetIterator = new BitSetIterator(filterIdsBitSet, filterIdsBitSet.length()); + ) throws IOException { + this.bitSetIterator = filterIdsBitSet == null ? null : new BitSetIterator(filterIdsBitSet, filterIdsBitSet.length()); this.queryVector = queryVector; this.knnFloatVectorValues = knnFloatVectorValues; this.spaceType = spaceType; - this.docId = bitSetIterator.nextDoc(); + // This cannot be moved inside nextDoc() method since it will break when we have nested field, where + // nextDoc should already be referring to next knnVectorValues + this.docId = getNextDocId(); this.quantizedQueryVector = quantizedQueryVector; this.segmentLevelQuantizationInfo = segmentLevelQuantizationInfo; } @@ -72,10 +77,10 @@ public int nextDoc() throws IOException { if (docId == DocIdSetIterator.NO_MORE_DOCS) { return DocIdSetIterator.NO_MORE_DOCS; } - int doc = knnFloatVectorValues.advance(docId); currentScore = computeScore(); - docId = bitSetIterator.nextDoc(); - return doc; + int currentDocId = docId; + docId = getNextDocId(); + return currentDocId; } @Override @@ -94,4 +99,16 @@ protected float computeScore() throws IOException { return spaceType.getKnnVectorSimilarityFunction().compare(queryVector, vector); } } + + protected int getNextDocId() throws IOException { + if (bitSetIterator == null) { + return knnFloatVectorValues.nextDoc(); + } + int nextDocID = this.bitSetIterator.nextDoc(); + // For filter case, advance vector values to corresponding doc id from filter bit set + if (nextDocID != DocIdSetIterator.NO_MORE_DOCS) { + knnFloatVectorValues.advance(nextDocID); + } + return nextDocID; + } } diff --git a/src/test/java/org/opensearch/knn/index/query/filtered/FilteredIdsKNNByteIteratorTests.java b/src/test/java/org/opensearch/knn/index/query/filtered/FilteredIdsKNNByteIteratorTests.java deleted file mode 100644 index c52798c05..000000000 --- a/src/test/java/org/opensearch/knn/index/query/filtered/FilteredIdsKNNByteIteratorTests.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.knn.index.query.filtered; - -import junit.framework.TestCase; -import lombok.SneakyThrows; -import org.apache.lucene.search.DocIdSetIterator; -import org.apache.lucene.util.FixedBitSet; -import org.opensearch.knn.index.SpaceType; -import org.opensearch.knn.index.vectorvalues.KNNBinaryVectorValues; - -import java.util.Arrays; -import java.util.List; -import java.util.stream.Collectors; - -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -public class FilteredIdsKNNByteIteratorTests extends TestCase { - @SneakyThrows - public void testNextDoc_whenCalled_IterateAllDocs() { - final SpaceType spaceType = SpaceType.HAMMING; - final byte[] queryVector = { 1, 2, 3 }; - final int[] filterIds = { 1, 2, 3 }; - final List dataVectors = Arrays.asList(new byte[] { 11, 12, 13 }, new byte[] { 14, 15, 16 }, new byte[] { 17, 18, 19 }); - final List expectedScores = dataVectors.stream() - .map(vector -> spaceType.getKnnVectorSimilarityFunction().compare(queryVector, vector)) - .collect(Collectors.toList()); - - KNNBinaryVectorValues values = mock(KNNBinaryVectorValues.class); - when(values.getVector()).thenReturn(dataVectors.get(0), dataVectors.get(1), dataVectors.get(2)); - - FixedBitSet filterBitSet = new FixedBitSet(4); - for (int id : filterIds) { - when(values.advance(id)).thenReturn(id); - filterBitSet.set(id); - } - - // Execute and verify - FilteredIdsKNNByteIterator iterator = new FilteredIdsKNNByteIterator(filterBitSet, queryVector, values, spaceType); - for (int i = 0; i < filterIds.length; i++) { - assertEquals(filterIds[i], iterator.nextDoc()); - assertEquals(expectedScores.get(i), (Float) iterator.score()); - } - assertEquals(DocIdSetIterator.NO_MORE_DOCS, iterator.nextDoc()); - } -} diff --git a/src/test/java/org/opensearch/knn/index/query/filtered/FilteredIdsKNNIteratorTests.java b/src/test/java/org/opensearch/knn/index/query/filtered/FilteredIdsKNNIteratorTests.java deleted file mode 100644 index 731eed2cc..000000000 --- a/src/test/java/org/opensearch/knn/index/query/filtered/FilteredIdsKNNIteratorTests.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.knn.index.query.filtered; - -import lombok.SneakyThrows; -import org.apache.lucene.search.DocIdSetIterator; -import org.apache.lucene.util.FixedBitSet; -import org.opensearch.knn.KNNTestCase; -import org.opensearch.knn.index.SpaceType; -import org.opensearch.knn.index.vectorvalues.KNNFloatVectorValues; - -import java.util.Arrays; -import java.util.List; -import java.util.stream.Collectors; - -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -public class FilteredIdsKNNIteratorTests extends KNNTestCase { - @SneakyThrows - public void testNextDoc_whenCalled_IterateAllDocs() { - final SpaceType spaceType = SpaceType.L2; - final float[] queryVector = { 1.0f, 2.0f, 3.0f }; - final int[] filterIds = { 1, 2, 3 }; - final List dataVectors = Arrays.asList( - new float[] { 11.0f, 12.0f, 13.0f }, - new float[] { 14.0f, 15.0f, 16.0f }, - new float[] { 17.0f, 18.0f, 19.0f } - ); - final List expectedScores = dataVectors.stream() - .map(vector -> spaceType.getKnnVectorSimilarityFunction().compare(queryVector, vector)) - .collect(Collectors.toList()); - - KNNFloatVectorValues values = mock(KNNFloatVectorValues.class); - when(values.getVector()).thenReturn(dataVectors.get(0), dataVectors.get(1), dataVectors.get(2)); - - FixedBitSet filterBitSet = new FixedBitSet(4); - for (int id : filterIds) { - when(values.advance(id)).thenReturn(id); - filterBitSet.set(id); - } - - // Execute and verify - FilteredIdsKNNIterator iterator = new FilteredIdsKNNIterator(filterBitSet, queryVector, values, spaceType); - for (int i = 0; i < filterIds.length; i++) { - assertEquals(filterIds[i], iterator.nextDoc()); - assertEquals(expectedScores.get(i), (Float) iterator.score()); - } - assertEquals(DocIdSetIterator.NO_MORE_DOCS, iterator.nextDoc()); - } -} diff --git a/src/test/java/org/opensearch/knn/index/query/iterators/ByteVectorIdsKNNIteratorTests.java b/src/test/java/org/opensearch/knn/index/query/iterators/ByteVectorIdsKNNIteratorTests.java new file mode 100644 index 000000000..0b1b71286 --- /dev/null +++ b/src/test/java/org/opensearch/knn/index/query/iterators/ByteVectorIdsKNNIteratorTests.java @@ -0,0 +1,97 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.query.iterators; + +import junit.framework.TestCase; +import lombok.SneakyThrows; +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.util.FixedBitSet; +import org.mockito.stubbing.OngoingStubbing; +import org.opensearch.knn.index.SpaceType; +import org.opensearch.knn.index.vectorvalues.KNNBinaryVectorValues; + +import java.io.IOException; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; + +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +public class ByteVectorIdsKNNIteratorTests extends TestCase { + @SneakyThrows + public void testNextDoc_whenCalled_thenIterateAllDocs() { + final SpaceType spaceType = SpaceType.HAMMING; + final byte[] queryVector = { 1, 2, 3 }; + final int[] filterIds = { 1, 2, 3 }; + final List dataVectors = Arrays.asList(new byte[] { 11, 12, 13 }, new byte[] { 14, 15, 16 }, new byte[] { 17, 18, 19 }); + final List expectedScores = dataVectors.stream() + .map(vector -> spaceType.getKnnVectorSimilarityFunction().compare(queryVector, vector)) + .collect(Collectors.toList()); + + KNNBinaryVectorValues values = mock(KNNBinaryVectorValues.class); + when(values.getVector()).thenReturn(dataVectors.get(0), dataVectors.get(1), dataVectors.get(2)); + + FixedBitSet filterBitSet = new FixedBitSet(4); + for (int id : filterIds) { + when(values.advance(id)).thenReturn(id); + filterBitSet.set(id); + } + + // Execute and verify + ByteVectorIdsKNNIterator iterator = new ByteVectorIdsKNNIterator(filterBitSet, queryVector, values, spaceType); + for (int i = 0; i < filterIds.length; i++) { + assertEquals(filterIds[i], iterator.nextDoc()); + assertEquals(expectedScores.get(i), iterator.score()); + } + assertEquals(DocIdSetIterator.NO_MORE_DOCS, iterator.nextDoc()); + } + + @SneakyThrows + public void testNextDoc_whenCalled_thenIterateAllDocsWithoutFilter() throws IOException { + final SpaceType spaceType = SpaceType.HAMMING; + final byte[] queryVector = { 1, 2, 3 }; + final List dataVectors = Arrays.asList( + new byte[] { 11, 12, 13 }, + new byte[] { 14, 15, 16 }, + new byte[] { 17, 18, 19 }, + new byte[] { 20, 21, 22 }, + new byte[] { 23, 24, 25 } + ); + final List expectedScores = dataVectors.stream() + .map(vector -> spaceType.getKnnVectorSimilarityFunction().compare(queryVector, vector)) + .collect(Collectors.toList()); + + KNNBinaryVectorValues values = mock(KNNBinaryVectorValues.class); + when(values.getVector()).thenReturn( + dataVectors.get(0), + dataVectors.get(1), + dataVectors.get(2), + dataVectors.get(3), + dataVectors.get(4) + ); + + // stub return value when nextDoc is called + OngoingStubbing stubbing = when(values.nextDoc()); + for (int i = 0; i < dataVectors.size(); i++) { + stubbing = stubbing.thenReturn(i); + } + // set last return to be Integer.MAX_VALUE to represent no more docs + stubbing.thenReturn(Integer.MAX_VALUE); + + // Execute and verify + ByteVectorIdsKNNIterator iterator = new ByteVectorIdsKNNIterator(queryVector, values, spaceType); + for (int i = 0; i < dataVectors.size(); i++) { + assertEquals(i, iterator.nextDoc()); + assertEquals(expectedScores.get(i), iterator.score()); + } + assertEquals(DocIdSetIterator.NO_MORE_DOCS, iterator.nextDoc()); + verify(values, never()).advance(anyInt()); + } +} diff --git a/src/test/java/org/opensearch/knn/index/query/filtered/NestedFilteredIdsKNNByteIteratorTests.java b/src/test/java/org/opensearch/knn/index/query/iterators/NestedByteVectorIdsKNNIteratorTests.java similarity index 54% rename from src/test/java/org/opensearch/knn/index/query/filtered/NestedFilteredIdsKNNByteIteratorTests.java rename to src/test/java/org/opensearch/knn/index/query/iterators/NestedByteVectorIdsKNNIteratorTests.java index 1940ffe12..eff021234 100644 --- a/src/test/java/org/opensearch/knn/index/query/filtered/NestedFilteredIdsKNNByteIteratorTests.java +++ b/src/test/java/org/opensearch/knn/index/query/iterators/NestedByteVectorIdsKNNIteratorTests.java @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.knn.index.query.filtered; +package org.opensearch.knn.index.query.iterators; import junit.framework.TestCase; import lombok.SneakyThrows; @@ -17,10 +17,13 @@ import java.util.List; import java.util.stream.Collectors; +import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; -public class NestedFilteredIdsKNNByteIteratorTests extends TestCase { +public class NestedByteVectorIdsKNNIteratorTests extends TestCase { @SneakyThrows public void testNextDoc_whenIterate_ReturnBestChildDocsPerParent() { final SpaceType spaceType = SpaceType.HAMMING; @@ -45,7 +48,7 @@ public void testNextDoc_whenIterate_ReturnBestChildDocsPerParent() { } // Execute and verify - NestedFilteredIdsKNNByteIterator iterator = new NestedFilteredIdsKNNByteIterator( + NestedByteVectorIdsKNNIterator iterator = new NestedByteVectorIdsKNNIterator( filterBitSet, queryVector, values, @@ -58,4 +61,31 @@ public void testNextDoc_whenIterate_ReturnBestChildDocsPerParent() { assertEquals(expectedScores.get(2), iterator.score()); assertEquals(DocIdSetIterator.NO_MORE_DOCS, iterator.nextDoc()); } + + @SneakyThrows + public void testNextDoc_whenIterateWithoutFilters_thenReturnBestChildDocsPerParent() { + final SpaceType spaceType = SpaceType.HAMMING; + final byte[] queryVector = { 1, 2, 3 }; + // Parent id for 0 -> 1 + // Parent id for 2, 3 -> 4 + // In bit representation, it is 10010. In long, it is 18. + final BitSet parentBitSet = new FixedBitSet(new long[] { 18 }, 5); + final List dataVectors = Arrays.asList(new byte[] { 11, 12, 13 }, new byte[] { 14, 15, 16 }, new byte[] { 17, 18, 19 }); + final List expectedScores = dataVectors.stream() + .map(vector -> spaceType.getKnnVectorSimilarityFunction().compare(queryVector, vector)) + .collect(Collectors.toList()); + + KNNBinaryVectorValues values = mock(KNNBinaryVectorValues.class); + when(values.getVector()).thenReturn(dataVectors.get(0), dataVectors.get(1), dataVectors.get(2)); + when(values.nextDoc()).thenReturn(0, 2, 3, Integer.MAX_VALUE); + + // Execute and verify + NestedByteVectorIdsKNNIterator iterator = new NestedByteVectorIdsKNNIterator(queryVector, values, spaceType, parentBitSet); + assertEquals(0, iterator.nextDoc()); + assertEquals(expectedScores.get(0), iterator.score()); + assertEquals(3, iterator.nextDoc()); + assertEquals(expectedScores.get(2), iterator.score()); + assertEquals(DocIdSetIterator.NO_MORE_DOCS, iterator.nextDoc()); + verify(values, never()).advance(anyInt()); + } } diff --git a/src/test/java/org/opensearch/knn/index/query/filtered/NestedFilteredIdsKNNIteratorTests.java b/src/test/java/org/opensearch/knn/index/query/iterators/NestedVectorIdsKNNIteratorTests.java similarity index 55% rename from src/test/java/org/opensearch/knn/index/query/filtered/NestedFilteredIdsKNNIteratorTests.java rename to src/test/java/org/opensearch/knn/index/query/iterators/NestedVectorIdsKNNIteratorTests.java index cca789a4d..f94ddb4e1 100644 --- a/src/test/java/org/opensearch/knn/index/query/filtered/NestedFilteredIdsKNNIteratorTests.java +++ b/src/test/java/org/opensearch/knn/index/query/iterators/NestedVectorIdsKNNIteratorTests.java @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.knn.index.query.filtered; +package org.opensearch.knn.index.query.iterators; import junit.framework.TestCase; import lombok.SneakyThrows; @@ -17,10 +17,13 @@ import java.util.List; import java.util.stream.Collectors; +import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; -public class NestedFilteredIdsKNNIteratorTests extends TestCase { +public class NestedVectorIdsKNNIteratorTests extends TestCase { @SneakyThrows public void testNextDoc_whenIterate_ReturnBestChildDocsPerParent() { final SpaceType spaceType = SpaceType.L2; @@ -53,17 +56,42 @@ public void testNextDoc_whenIterate_ReturnBestChildDocsPerParent() { } // Execute and verify - NestedFilteredIdsKNNIterator iterator = new NestedFilteredIdsKNNIterator( - filterBitSet, - queryVector, - values, - spaceType, - parentBitSet - ); + NestedVectorIdsKNNIterator iterator = new NestedVectorIdsKNNIterator(filterBitSet, queryVector, values, spaceType, parentBitSet); assertEquals(filterIds[0], iterator.nextDoc()); assertEquals(expectedScores.get(0), iterator.score()); assertEquals(filterIds[2], iterator.nextDoc()); assertEquals(expectedScores.get(2), iterator.score()); assertEquals(DocIdSetIterator.NO_MORE_DOCS, iterator.nextDoc()); } + + @SneakyThrows + public void testNextDoc_whenIterateWithoutFilters_thenReturnBestChildDocsPerParent() { + final SpaceType spaceType = SpaceType.L2; + final float[] queryVector = { 1.0f, 2.0f, 3.0f }; + // Parent id for 0 -> 1 + // Parent id for 2, 3 -> 4 + // In bit representation, it is 10010. In long, it is 18. + final BitSet parentBitSet = new FixedBitSet(new long[] { 18 }, 5); + final List dataVectors = Arrays.asList( + new float[] { 11.0f, 12.0f, 13.0f }, + new float[] { 17.0f, 18.0f, 19.0f }, + new float[] { 14.0f, 15.0f, 16.0f } + ); + final List expectedScores = dataVectors.stream() + .map(vector -> spaceType.getKnnVectorSimilarityFunction().compare(queryVector, vector)) + .collect(Collectors.toList()); + + KNNFloatVectorValues values = mock(KNNFloatVectorValues.class); + when(values.getVector()).thenReturn(dataVectors.get(0), dataVectors.get(1), dataVectors.get(2)); + when(values.nextDoc()).thenReturn(0, 2, 3, Integer.MAX_VALUE); + + // Execute and verify + NestedVectorIdsKNNIterator iterator = new NestedVectorIdsKNNIterator(queryVector, values, spaceType, parentBitSet); + assertEquals(0, iterator.nextDoc()); + assertEquals(expectedScores.get(0), iterator.score()); + assertEquals(3, iterator.nextDoc()); + assertEquals(expectedScores.get(2), iterator.score()); + assertEquals(DocIdSetIterator.NO_MORE_DOCS, iterator.nextDoc()); + verify(values, never()).advance(anyInt()); + } } diff --git a/src/test/java/org/opensearch/knn/index/query/iterators/VectorIdsKNNIteratorTests.java b/src/test/java/org/opensearch/knn/index/query/iterators/VectorIdsKNNIteratorTests.java new file mode 100644 index 000000000..96932d0f1 --- /dev/null +++ b/src/test/java/org/opensearch/knn/index/query/iterators/VectorIdsKNNIteratorTests.java @@ -0,0 +1,98 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.query.iterators; + +import lombok.SneakyThrows; +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.util.FixedBitSet; +import org.mockito.stubbing.OngoingStubbing; +import org.opensearch.knn.KNNTestCase; +import org.opensearch.knn.index.SpaceType; +import org.opensearch.knn.index.vectorvalues.KNNFloatVectorValues; + +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; + +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +public class VectorIdsKNNIteratorTests extends KNNTestCase { + @SneakyThrows + public void testNextDoc_whenCalledWithFilters_thenIterateAllDocs() { + final SpaceType spaceType = SpaceType.L2; + final float[] queryVector = { 1.0f, 2.0f, 3.0f }; + final int[] filterIds = { 1, 2, 3 }; + final List dataVectors = Arrays.asList( + new float[] { 11.0f, 12.0f, 13.0f }, + new float[] { 14.0f, 15.0f, 16.0f }, + new float[] { 17.0f, 18.0f, 19.0f } + ); + final List expectedScores = dataVectors.stream() + .map(vector -> spaceType.getKnnVectorSimilarityFunction().compare(queryVector, vector)) + .collect(Collectors.toList()); + + KNNFloatVectorValues values = mock(KNNFloatVectorValues.class); + when(values.getVector()).thenReturn(dataVectors.get(0), dataVectors.get(1), dataVectors.get(2)); + + FixedBitSet filterBitSet = new FixedBitSet(4); + for (int id : filterIds) { + when(values.advance(id)).thenReturn(id); + filterBitSet.set(id); + } + + // Execute and verify + VectorIdsKNNIterator iterator = new VectorIdsKNNIterator(filterBitSet, queryVector, values, spaceType); + for (int i = 0; i < filterIds.length; i++) { + assertEquals(filterIds[i], iterator.nextDoc()); + assertEquals(expectedScores.get(i), (Float) iterator.score()); + } + assertEquals(DocIdSetIterator.NO_MORE_DOCS, iterator.nextDoc()); + } + + @SneakyThrows + public void testNextDoc_whenCalledWithoutFilters_thenIterateAllDocs() { + final SpaceType spaceType = SpaceType.L2; + final float[] queryVector = { 1.0f, 2.0f, 3.0f }; + final List dataVectors = Arrays.asList( + new float[] { 11.0f, 12.0f, 13.0f }, + new float[] { 14.0f, 15.0f, 16.0f }, + new float[] { 17.0f, 18.0f, 19.0f }, + new float[] { 20.0f, 21.0f, 22.0f }, + new float[] { 23.0f, 24.0f, 25.0f } + ); + final List expectedScores = dataVectors.stream() + .map(vector -> spaceType.getKnnVectorSimilarityFunction().compare(queryVector, vector)) + .collect(Collectors.toList()); + + KNNFloatVectorValues values = mock(KNNFloatVectorValues.class); + when(values.getVector()).thenReturn( + dataVectors.get(0), + dataVectors.get(1), + dataVectors.get(2), + dataVectors.get(3), + dataVectors.get(4) + ); + // stub return value when nextDoc is called + OngoingStubbing stubbing = when(values.nextDoc()); + for (int i = 0; i < dataVectors.size(); i++) { + stubbing = stubbing.thenReturn(i); + } + // set last return to be Integer.MAX_VALUE to represent no more docs + stubbing.thenReturn(Integer.MAX_VALUE); + // Execute and verify + VectorIdsKNNIterator iterator = new VectorIdsKNNIterator(queryVector, values, spaceType); + for (int i = 0; i < dataVectors.size(); i++) { + assertEquals(i, iterator.nextDoc()); + assertEquals(expectedScores.get(i), (Float) iterator.score()); + } + assertEquals(DocIdSetIterator.NO_MORE_DOCS, iterator.nextDoc()); + verify(values, never()).advance(anyInt()); + } +} From 7f7ea5935fbd9824fb83c059d8816bb69ebc84e3 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Mon, 30 Sep 2024 13:07:15 -0700 Subject: [PATCH 387/416] Optimize reduceToTopK in ResultUtil by removing pre-filling and reducing peek calls (#2146) (#2164) Signed-off-by: Junqiu Lei (cherry picked from commit e0c3afe5fe2f0048b9422364aeeb620e35a44deb) Co-authored-by: Junqiu Lei --- CHANGELOG.md | 1 + .../java/org/opensearch/knn/index/query/ResultUtil.java | 7 +++---- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index edf390f80..e9acc9e05 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), * Add AVX512 support to k-NN for FAISS library [#2069](https://github.com/opensearch-project/k-NN/pull/2069) ### Enhancements * Add short circuit if no live docs are in segments [#2059](https://github.com/opensearch-project/k-NN/pull/2059) +* Optimize reduceToTopK in ResultUtil by removing pre-filling and reducing peek calls [#2146](https://github.com/opensearch-project/k-NN/pull/2146) * Update Default Rescore Context based on Dimension [#2149](https://github.com/opensearch-project/k-NN/pull/2149) * KNNIterators should support with and without filters [#2155](https://github.com/opensearch-project/k-NN/pull/2155) ### Bug Fixes diff --git a/src/main/java/org/opensearch/knn/index/query/ResultUtil.java b/src/main/java/org/opensearch/knn/index/query/ResultUtil.java index 2ed70b8b3..f62c09cb0 100644 --- a/src/main/java/org/opensearch/knn/index/query/ResultUtil.java +++ b/src/main/java/org/opensearch/knn/index/query/ResultUtil.java @@ -33,15 +33,14 @@ public final class ResultUtil { public static void reduceToTopK(List> perLeafResults, int k) { // Iterate over all scores to get min competitive score PriorityQueue topKMinQueue = new PriorityQueue<>(k); - for (int i = 0; i < k; i++) { - topKMinQueue.add(-Float.MAX_VALUE); - } int count = 0; for (Map perLeafResult : perLeafResults) { count += perLeafResult.size(); for (Float score : perLeafResult.values()) { - if (topKMinQueue.peek() != null && score > topKMinQueue.peek()) { + if (topKMinQueue.size() < k) { + topKMinQueue.add(score); + } else if (topKMinQueue.peek() != null && score > topKMinQueue.peek()) { topKMinQueue.poll(); topKMinQueue.add(score); } From 035e5daafd2d780697ed435c1adf0b924d9bc68a Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Mon, 30 Sep 2024 19:17:14 -0500 Subject: [PATCH 388/416] Fix Faiss efficient filter exact search using byte vector datatype (#2165) (#2171) * Fix Faiss efficient filter exact search using byte vector datatype * Address Review Comments --------- (cherry picked from commit 6d098cf5266160033eccc7ee2e90bd4e4c1c071c) Signed-off-by: Naveen Tatikonda Co-authored-by: Naveen Tatikonda --- .../knn/index/query/ExactSearcher.java | 21 ++- .../iterators/BinaryVectorIdsKNNIterator.java | 92 +++++++++++ .../iterators/ByteVectorIdsKNNIterator.java | 34 ++-- .../NestedBinaryVectorIdsKNNIterator.java | 77 +++++++++ .../NestedByteVectorIdsKNNIterator.java | 12 +- .../BinaryVectorIdsKNNIteratorTests.java | 97 +++++++++++ .../ByteVectorIdsKNNIteratorTests.java | 24 +-- ...NestedBinaryVectorIdsKNNIteratorTests.java | 91 ++++++++++ .../NestedByteVectorIdsKNNIteratorTests.java | 24 +-- .../knn/integ/FilteredSearchByteIT.java | 104 ++++++++++++ .../knn/integ/NestedSearchByteIT.java | 156 ++++++++++++++++++ 11 files changed, 690 insertions(+), 42 deletions(-) create mode 100644 src/main/java/org/opensearch/knn/index/query/iterators/BinaryVectorIdsKNNIterator.java create mode 100644 src/main/java/org/opensearch/knn/index/query/iterators/NestedBinaryVectorIdsKNNIterator.java create mode 100644 src/test/java/org/opensearch/knn/index/query/iterators/BinaryVectorIdsKNNIteratorTests.java create mode 100644 src/test/java/org/opensearch/knn/index/query/iterators/NestedBinaryVectorIdsKNNIteratorTests.java create mode 100644 src/test/java/org/opensearch/knn/integ/FilteredSearchByteIT.java create mode 100644 src/test/java/org/opensearch/knn/integ/NestedSearchByteIT.java diff --git a/src/main/java/org/opensearch/knn/index/query/ExactSearcher.java b/src/main/java/org/opensearch/knn/index/query/ExactSearcher.java index 193cba8c1..8e5849abb 100644 --- a/src/main/java/org/opensearch/knn/index/query/ExactSearcher.java +++ b/src/main/java/org/opensearch/knn/index/query/ExactSearcher.java @@ -20,12 +20,15 @@ import org.opensearch.knn.common.FieldInfoExtractor; import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.VectorDataType; +import org.opensearch.knn.index.query.iterators.BinaryVectorIdsKNNIterator; import org.opensearch.knn.index.query.iterators.ByteVectorIdsKNNIterator; +import org.opensearch.knn.index.query.iterators.NestedBinaryVectorIdsKNNIterator; import org.opensearch.knn.index.query.iterators.VectorIdsKNNIterator; import org.opensearch.knn.index.query.iterators.KNNIterator; import org.opensearch.knn.index.query.iterators.NestedByteVectorIdsKNNIterator; import org.opensearch.knn.index.query.iterators.NestedVectorIdsKNNIterator; import org.opensearch.knn.index.vectorvalues.KNNBinaryVectorValues; +import org.opensearch.knn.index.vectorvalues.KNNByteVectorValues; import org.opensearch.knn.index.vectorvalues.KNNFloatVectorValues; import org.opensearch.knn.index.vectorvalues.KNNVectorValues; import org.opensearch.knn.index.vectorvalues.KNNVectorValuesFactory; @@ -111,7 +114,7 @@ private KNNIterator getKNNIterator(LeafReaderContext leafReaderContext, ExactSea if (VectorDataType.BINARY == knnQuery.getVectorDataType()) { final KNNVectorValues vectorValues = KNNVectorValuesFactory.getVectorValues(fieldInfo, reader); if (isNestedRequired) { - return new NestedByteVectorIdsKNNIterator( + return new NestedBinaryVectorIdsKNNIterator( matchedDocs, knnQuery.getByteQueryVector(), (KNNBinaryVectorValues) vectorValues, @@ -119,13 +122,27 @@ private KNNIterator getKNNIterator(LeafReaderContext leafReaderContext, ExactSea knnQuery.getParentsFilter().getBitSet(leafReaderContext) ); } - return new ByteVectorIdsKNNIterator( + return new BinaryVectorIdsKNNIterator( matchedDocs, knnQuery.getByteQueryVector(), (KNNBinaryVectorValues) vectorValues, spaceType ); } + + if (VectorDataType.BYTE == knnQuery.getVectorDataType()) { + final KNNVectorValues vectorValues = KNNVectorValuesFactory.getVectorValues(fieldInfo, reader); + if (isNestedRequired) { + return new NestedByteVectorIdsKNNIterator( + matchedDocs, + knnQuery.getQueryVector(), + (KNNByteVectorValues) vectorValues, + spaceType, + knnQuery.getParentsFilter().getBitSet(leafReaderContext) + ); + } + return new ByteVectorIdsKNNIterator(matchedDocs, knnQuery.getQueryVector(), (KNNByteVectorValues) vectorValues, spaceType); + } final byte[] quantizedQueryVector; final SegmentLevelQuantizationInfo segmentLevelQuantizationInfo; if (exactSearcherContext.isUseQuantizedVectorsForSearch()) { diff --git a/src/main/java/org/opensearch/knn/index/query/iterators/BinaryVectorIdsKNNIterator.java b/src/main/java/org/opensearch/knn/index/query/iterators/BinaryVectorIdsKNNIterator.java new file mode 100644 index 000000000..5bab5b573 --- /dev/null +++ b/src/main/java/org/opensearch/knn/index/query/iterators/BinaryVectorIdsKNNIterator.java @@ -0,0 +1,92 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.query.iterators; + +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.util.BitSet; +import org.apache.lucene.util.BitSetIterator; +import org.opensearch.common.Nullable; +import org.opensearch.knn.index.SpaceType; +import org.opensearch.knn.index.vectorvalues.KNNBinaryVectorValues; + +import java.io.IOException; + +/** + * Inspired by DiversifyingChildrenFloatKnnVectorQuery in lucene + * https://github.com/apache/lucene/blob/7b8aece125aabff2823626d5b939abf4747f63a7/lucene/join/src/java/org/apache/lucene/search/join/DiversifyingChildrenFloatKnnVectorQuery.java#L162 + * + * The class is used in KNNWeight to score all docs, but, it iterates over filterIdsArray if filter is provided + */ +public class BinaryVectorIdsKNNIterator implements KNNIterator { + protected final BitSetIterator bitSetIterator; + protected final byte[] queryVector; + protected final KNNBinaryVectorValues binaryVectorValues; + protected final SpaceType spaceType; + protected float currentScore = Float.NEGATIVE_INFINITY; + protected int docId; + + public BinaryVectorIdsKNNIterator( + @Nullable final BitSet filterIdsBitSet, + final byte[] queryVector, + final KNNBinaryVectorValues binaryVectorValues, + final SpaceType spaceType + ) throws IOException { + this.bitSetIterator = filterIdsBitSet == null ? null : new BitSetIterator(filterIdsBitSet, filterIdsBitSet.length()); + this.queryVector = queryVector; + this.binaryVectorValues = binaryVectorValues; + this.spaceType = spaceType; + // This cannot be moved inside nextDoc() method since it will break when we have nested field, where + // nextDoc should already be referring to next knnVectorValues + this.docId = getNextDocId(); + } + + public BinaryVectorIdsKNNIterator(final byte[] queryVector, final KNNBinaryVectorValues binaryVectorValues, final SpaceType spaceType) + throws IOException { + this(null, queryVector, binaryVectorValues, spaceType); + } + + /** + * Advance to the next doc and update score value with score of the next doc. + * DocIdSetIterator.NO_MORE_DOCS is returned when there is no more docs + * + * @return next doc id + */ + @Override + public int nextDoc() throws IOException { + + if (docId == DocIdSetIterator.NO_MORE_DOCS) { + return DocIdSetIterator.NO_MORE_DOCS; + } + currentScore = computeScore(); + int currentDocId = docId; + docId = getNextDocId(); + return currentDocId; + } + + @Override + public float score() { + return currentScore; + } + + protected float computeScore() throws IOException { + final byte[] vector = binaryVectorValues.getVector(); + // Calculates a similarity score between the two vectors with a specified function. Higher similarity + // scores correspond to closer vectors. + return spaceType.getKnnVectorSimilarityFunction().compare(queryVector, vector); + } + + protected int getNextDocId() throws IOException { + if (bitSetIterator == null) { + return binaryVectorValues.nextDoc(); + } + int nextDocID = this.bitSetIterator.nextDoc(); + // For filter case, advance vector values to corresponding doc id from filter bit set + if (nextDocID != DocIdSetIterator.NO_MORE_DOCS) { + binaryVectorValues.advance(nextDocID); + } + return nextDocID; + } +} diff --git a/src/main/java/org/opensearch/knn/index/query/iterators/ByteVectorIdsKNNIterator.java b/src/main/java/org/opensearch/knn/index/query/iterators/ByteVectorIdsKNNIterator.java index b1aea4284..0e8005163 100644 --- a/src/main/java/org/opensearch/knn/index/query/iterators/ByteVectorIdsKNNIterator.java +++ b/src/main/java/org/opensearch/knn/index/query/iterators/ByteVectorIdsKNNIterator.java @@ -10,7 +10,7 @@ import org.apache.lucene.util.BitSetIterator; import org.opensearch.common.Nullable; import org.opensearch.knn.index.SpaceType; -import org.opensearch.knn.index.vectorvalues.KNNBinaryVectorValues; +import org.opensearch.knn.index.vectorvalues.KNNByteVectorValues; import java.io.IOException; @@ -22,30 +22,30 @@ */ public class ByteVectorIdsKNNIterator implements KNNIterator { protected final BitSetIterator bitSetIterator; - protected final byte[] queryVector; - protected final KNNBinaryVectorValues binaryVectorValues; + protected final float[] queryVector; + protected final KNNByteVectorValues byteVectorValues; protected final SpaceType spaceType; protected float currentScore = Float.NEGATIVE_INFINITY; protected int docId; public ByteVectorIdsKNNIterator( @Nullable final BitSet filterIdsBitSet, - final byte[] queryVector, - final KNNBinaryVectorValues binaryVectorValues, + final float[] queryVector, + final KNNByteVectorValues byteVectorValues, final SpaceType spaceType ) throws IOException { this.bitSetIterator = filterIdsBitSet == null ? null : new BitSetIterator(filterIdsBitSet, filterIdsBitSet.length()); this.queryVector = queryVector; - this.binaryVectorValues = binaryVectorValues; + this.byteVectorValues = byteVectorValues; this.spaceType = spaceType; // This cannot be moved inside nextDoc() method since it will break when we have nested field, where // nextDoc should already be referring to next knnVectorValues this.docId = getNextDocId(); } - public ByteVectorIdsKNNIterator(final byte[] queryVector, final KNNBinaryVectorValues binaryVectorValues, final SpaceType spaceType) + public ByteVectorIdsKNNIterator(final float[] queryVector, final KNNByteVectorValues byteVectorValues, final SpaceType spaceType) throws IOException { - this(null, queryVector, binaryVectorValues, spaceType); + this(null, queryVector, byteVectorValues, spaceType); } /** @@ -72,20 +72,30 @@ public float score() { } protected float computeScore() throws IOException { - final byte[] vector = binaryVectorValues.getVector(); + final byte[] vector = byteVectorValues.getVector(); // Calculates a similarity score between the two vectors with a specified function. Higher similarity // scores correspond to closer vectors. - return spaceType.getKnnVectorSimilarityFunction().compare(queryVector, vector); + + // The query vector of Faiss byte vector is a Float array because ScalarQuantizer accepts it as float array. + // To compute the score between this query vector and each vector in KNNByteVectorValues we are casting this query vector into byte + // array directly. + // This is safe to do so because float query vector already has validated byte values. Do not reuse this direct cast at any other + // place. + final byte[] byteQueryVector = new byte[queryVector.length]; + for (int i = 0; i < queryVector.length; i++) { + byteQueryVector[i] = (byte) queryVector[i]; + } + return spaceType.getKnnVectorSimilarityFunction().compare(byteQueryVector, vector); } protected int getNextDocId() throws IOException { if (bitSetIterator == null) { - return binaryVectorValues.nextDoc(); + return byteVectorValues.nextDoc(); } int nextDocID = this.bitSetIterator.nextDoc(); // For filter case, advance vector values to corresponding doc id from filter bit set if (nextDocID != DocIdSetIterator.NO_MORE_DOCS) { - binaryVectorValues.advance(nextDocID); + byteVectorValues.advance(nextDocID); } return nextDocID; } diff --git a/src/main/java/org/opensearch/knn/index/query/iterators/NestedBinaryVectorIdsKNNIterator.java b/src/main/java/org/opensearch/knn/index/query/iterators/NestedBinaryVectorIdsKNNIterator.java new file mode 100644 index 000000000..97bf3517e --- /dev/null +++ b/src/main/java/org/opensearch/knn/index/query/iterators/NestedBinaryVectorIdsKNNIterator.java @@ -0,0 +1,77 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.query.iterators; + +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.util.BitSet; +import org.opensearch.common.Nullable; +import org.opensearch.knn.index.SpaceType; +import org.opensearch.knn.index.vectorvalues.KNNBinaryVectorValues; + +import java.io.IOException; + +/** + * This iterator iterates filterIdsArray to scoreif filter is provided else it iterates over all docs. + * However, it dedupe docs per each parent doc + * of which ID is set in parentBitSet and only return best child doc with the highest score. + */ +public class NestedBinaryVectorIdsKNNIterator extends BinaryVectorIdsKNNIterator { + private final BitSet parentBitSet; + + public NestedBinaryVectorIdsKNNIterator( + @Nullable final BitSet filterIdsArray, + final byte[] queryVector, + final KNNBinaryVectorValues binaryVectorValues, + final SpaceType spaceType, + final BitSet parentBitSet + ) throws IOException { + super(filterIdsArray, queryVector, binaryVectorValues, spaceType); + this.parentBitSet = parentBitSet; + } + + public NestedBinaryVectorIdsKNNIterator( + final byte[] queryVector, + final KNNBinaryVectorValues binaryVectorValues, + final SpaceType spaceType, + final BitSet parentBitSet + ) throws IOException { + super(null, queryVector, binaryVectorValues, spaceType); + this.parentBitSet = parentBitSet; + } + + /** + * Advance to the next best child doc per parent and update score with the best score among child docs from the parent. + * DocIdSetIterator.NO_MORE_DOCS is returned when there is no more docs + * + * @return next best child doc id + */ + @Override + public int nextDoc() throws IOException { + if (docId == DocIdSetIterator.NO_MORE_DOCS) { + return DocIdSetIterator.NO_MORE_DOCS; + } + + currentScore = Float.NEGATIVE_INFINITY; + int currentParent = parentBitSet.nextSetBit(docId); + int bestChild = -1; + + // In order to traverse all children for given parent, we have to use docId < parentId, because, + // kNNVectorValues will not have parent id since DocId is unique per segment. For ex: let's say for doc id 1, there is one child + // and for doc id 5, there are three children. In that case knnVectorValues iterator will have [0, 2, 3, 4] + // and parentBitSet will have [1,5] + // Hence, we have to iterate till docId from knnVectorValues is less than parentId instead of till equal to parentId + while (docId != DocIdSetIterator.NO_MORE_DOCS && docId < currentParent) { + float score = computeScore(); + if (score > currentScore) { + bestChild = docId; + currentScore = score; + } + docId = getNextDocId(); + } + + return bestChild; + } +} diff --git a/src/main/java/org/opensearch/knn/index/query/iterators/NestedByteVectorIdsKNNIterator.java b/src/main/java/org/opensearch/knn/index/query/iterators/NestedByteVectorIdsKNNIterator.java index 3c93ec888..9644b620f 100644 --- a/src/main/java/org/opensearch/knn/index/query/iterators/NestedByteVectorIdsKNNIterator.java +++ b/src/main/java/org/opensearch/knn/index/query/iterators/NestedByteVectorIdsKNNIterator.java @@ -9,7 +9,7 @@ import org.apache.lucene.util.BitSet; import org.opensearch.common.Nullable; import org.opensearch.knn.index.SpaceType; -import org.opensearch.knn.index.vectorvalues.KNNBinaryVectorValues; +import org.opensearch.knn.index.vectorvalues.KNNByteVectorValues; import java.io.IOException; @@ -23,18 +23,18 @@ public class NestedByteVectorIdsKNNIterator extends ByteVectorIdsKNNIterator { public NestedByteVectorIdsKNNIterator( @Nullable final BitSet filterIdsArray, - final byte[] queryVector, - final KNNBinaryVectorValues binaryVectorValues, + final float[] queryVector, + final KNNByteVectorValues byteVectorValues, final SpaceType spaceType, final BitSet parentBitSet ) throws IOException { - super(filterIdsArray, queryVector, binaryVectorValues, spaceType); + super(filterIdsArray, queryVector, byteVectorValues, spaceType); this.parentBitSet = parentBitSet; } public NestedByteVectorIdsKNNIterator( - final byte[] queryVector, - final KNNBinaryVectorValues binaryVectorValues, + final float[] queryVector, + final KNNByteVectorValues binaryVectorValues, final SpaceType spaceType, final BitSet parentBitSet ) throws IOException { diff --git a/src/test/java/org/opensearch/knn/index/query/iterators/BinaryVectorIdsKNNIteratorTests.java b/src/test/java/org/opensearch/knn/index/query/iterators/BinaryVectorIdsKNNIteratorTests.java new file mode 100644 index 000000000..6d5dffa98 --- /dev/null +++ b/src/test/java/org/opensearch/knn/index/query/iterators/BinaryVectorIdsKNNIteratorTests.java @@ -0,0 +1,97 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.query.iterators; + +import junit.framework.TestCase; +import lombok.SneakyThrows; +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.util.FixedBitSet; +import org.mockito.stubbing.OngoingStubbing; +import org.opensearch.knn.index.SpaceType; +import org.opensearch.knn.index.vectorvalues.KNNBinaryVectorValues; + +import java.io.IOException; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; + +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +public class BinaryVectorIdsKNNIteratorTests extends TestCase { + @SneakyThrows + public void testNextDoc_whenCalled_IterateAllDocs() { + final SpaceType spaceType = SpaceType.HAMMING; + final byte[] queryVector = { 1, 2, 3 }; + final int[] filterIds = { 1, 2, 3 }; + final List dataVectors = Arrays.asList(new byte[] { 11, 12, 13 }, new byte[] { 14, 15, 16 }, new byte[] { 17, 18, 19 }); + final List expectedScores = dataVectors.stream() + .map(vector -> spaceType.getKnnVectorSimilarityFunction().compare(queryVector, vector)) + .collect(Collectors.toList()); + + KNNBinaryVectorValues values = mock(KNNBinaryVectorValues.class); + when(values.getVector()).thenReturn(dataVectors.get(0), dataVectors.get(1), dataVectors.get(2)); + + FixedBitSet filterBitSet = new FixedBitSet(4); + for (int id : filterIds) { + when(values.advance(id)).thenReturn(id); + filterBitSet.set(id); + } + + // Execute and verify + BinaryVectorIdsKNNIterator iterator = new BinaryVectorIdsKNNIterator(filterBitSet, queryVector, values, spaceType); + for (int i = 0; i < filterIds.length; i++) { + assertEquals(filterIds[i], iterator.nextDoc()); + assertEquals(expectedScores.get(i), (Float) iterator.score()); + } + assertEquals(DocIdSetIterator.NO_MORE_DOCS, iterator.nextDoc()); + } + + @SneakyThrows + public void testNextDoc_whenCalled_thenIterateAllDocsWithoutFilter() throws IOException { + final SpaceType spaceType = SpaceType.HAMMING; + final byte[] queryVector = { 1, 2, 3 }; + final List dataVectors = Arrays.asList( + new byte[] { 11, 12, 13 }, + new byte[] { 14, 15, 16 }, + new byte[] { 17, 18, 19 }, + new byte[] { 20, 21, 22 }, + new byte[] { 23, 24, 25 } + ); + final List expectedScores = dataVectors.stream() + .map(vector -> spaceType.getKnnVectorSimilarityFunction().compare(queryVector, vector)) + .collect(Collectors.toList()); + + KNNBinaryVectorValues values = mock(KNNBinaryVectorValues.class); + when(values.getVector()).thenReturn( + dataVectors.get(0), + dataVectors.get(1), + dataVectors.get(2), + dataVectors.get(3), + dataVectors.get(4) + ); + + // stub return value when nextDoc is called + OngoingStubbing stubbing = when(values.nextDoc()); + for (int i = 0; i < dataVectors.size(); i++) { + stubbing = stubbing.thenReturn(i); + } + // set last return to be Integer.MAX_VALUE to represent no more docs + stubbing.thenReturn(Integer.MAX_VALUE); + + // Execute and verify + BinaryVectorIdsKNNIterator iterator = new BinaryVectorIdsKNNIterator(queryVector, values, spaceType); + for (int i = 0; i < dataVectors.size(); i++) { + assertEquals(i, iterator.nextDoc()); + assertEquals(expectedScores.get(i), iterator.score()); + } + assertEquals(DocIdSetIterator.NO_MORE_DOCS, iterator.nextDoc()); + verify(values, never()).advance(anyInt()); + } +} diff --git a/src/test/java/org/opensearch/knn/index/query/iterators/ByteVectorIdsKNNIteratorTests.java b/src/test/java/org/opensearch/knn/index/query/iterators/ByteVectorIdsKNNIteratorTests.java index 0b1b71286..60169b95f 100644 --- a/src/test/java/org/opensearch/knn/index/query/iterators/ByteVectorIdsKNNIteratorTests.java +++ b/src/test/java/org/opensearch/knn/index/query/iterators/ByteVectorIdsKNNIteratorTests.java @@ -11,7 +11,7 @@ import org.apache.lucene.util.FixedBitSet; import org.mockito.stubbing.OngoingStubbing; import org.opensearch.knn.index.SpaceType; -import org.opensearch.knn.index.vectorvalues.KNNBinaryVectorValues; +import org.opensearch.knn.index.vectorvalues.KNNByteVectorValues; import java.io.IOException; import java.util.Arrays; @@ -26,16 +26,17 @@ public class ByteVectorIdsKNNIteratorTests extends TestCase { @SneakyThrows - public void testNextDoc_whenCalled_thenIterateAllDocs() { - final SpaceType spaceType = SpaceType.HAMMING; - final byte[] queryVector = { 1, 2, 3 }; + public void testNextDoc_whenCalled_IterateAllDocs() { + final SpaceType spaceType = SpaceType.L2; + final byte[] byteQueryVector = { 1, 2, 3 }; + final float[] queryVector = { 1f, 2f, 3f }; final int[] filterIds = { 1, 2, 3 }; final List dataVectors = Arrays.asList(new byte[] { 11, 12, 13 }, new byte[] { 14, 15, 16 }, new byte[] { 17, 18, 19 }); final List expectedScores = dataVectors.stream() - .map(vector -> spaceType.getKnnVectorSimilarityFunction().compare(queryVector, vector)) + .map(vector -> spaceType.getKnnVectorSimilarityFunction().compare(byteQueryVector, vector)) .collect(Collectors.toList()); - KNNBinaryVectorValues values = mock(KNNBinaryVectorValues.class); + KNNByteVectorValues values = mock(KNNByteVectorValues.class); when(values.getVector()).thenReturn(dataVectors.get(0), dataVectors.get(1), dataVectors.get(2)); FixedBitSet filterBitSet = new FixedBitSet(4); @@ -48,15 +49,16 @@ public void testNextDoc_whenCalled_thenIterateAllDocs() { ByteVectorIdsKNNIterator iterator = new ByteVectorIdsKNNIterator(filterBitSet, queryVector, values, spaceType); for (int i = 0; i < filterIds.length; i++) { assertEquals(filterIds[i], iterator.nextDoc()); - assertEquals(expectedScores.get(i), iterator.score()); + assertEquals(expectedScores.get(i), (Float) iterator.score()); } assertEquals(DocIdSetIterator.NO_MORE_DOCS, iterator.nextDoc()); } @SneakyThrows public void testNextDoc_whenCalled_thenIterateAllDocsWithoutFilter() throws IOException { - final SpaceType spaceType = SpaceType.HAMMING; - final byte[] queryVector = { 1, 2, 3 }; + final SpaceType spaceType = SpaceType.L2; + final byte[] byteQueryVector = { 1, 2, 3 }; + final float[] queryVector = { 1.0f, 2.0f, 3.0f }; final List dataVectors = Arrays.asList( new byte[] { 11, 12, 13 }, new byte[] { 14, 15, 16 }, @@ -65,10 +67,10 @@ public void testNextDoc_whenCalled_thenIterateAllDocsWithoutFilter() throws IOEx new byte[] { 23, 24, 25 } ); final List expectedScores = dataVectors.stream() - .map(vector -> spaceType.getKnnVectorSimilarityFunction().compare(queryVector, vector)) + .map(vector -> spaceType.getKnnVectorSimilarityFunction().compare(byteQueryVector, vector)) .collect(Collectors.toList()); - KNNBinaryVectorValues values = mock(KNNBinaryVectorValues.class); + KNNByteVectorValues values = mock(KNNByteVectorValues.class); when(values.getVector()).thenReturn( dataVectors.get(0), dataVectors.get(1), diff --git a/src/test/java/org/opensearch/knn/index/query/iterators/NestedBinaryVectorIdsKNNIteratorTests.java b/src/test/java/org/opensearch/knn/index/query/iterators/NestedBinaryVectorIdsKNNIteratorTests.java new file mode 100644 index 000000000..a39a3b2e9 --- /dev/null +++ b/src/test/java/org/opensearch/knn/index/query/iterators/NestedBinaryVectorIdsKNNIteratorTests.java @@ -0,0 +1,91 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.query.iterators; + +import junit.framework.TestCase; +import lombok.SneakyThrows; +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.util.BitSet; +import org.apache.lucene.util.FixedBitSet; +import org.opensearch.knn.index.SpaceType; +import org.opensearch.knn.index.vectorvalues.KNNBinaryVectorValues; + +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; + +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +public class NestedBinaryVectorIdsKNNIteratorTests extends TestCase { + @SneakyThrows + public void testNextDoc_whenIterate_ReturnBestChildDocsPerParent() { + final SpaceType spaceType = SpaceType.HAMMING; + final byte[] queryVector = { 1, 2, 3 }; + final int[] filterIds = { 0, 2, 3 }; + // Parent id for 0 -> 1 + // Parent id for 2, 3 -> 4 + // In bit representation, it is 10010. In long, it is 18. + final BitSet parentBitSet = new FixedBitSet(new long[] { 18 }, 5); + final List dataVectors = Arrays.asList(new byte[] { 11, 12, 13 }, new byte[] { 14, 15, 16 }, new byte[] { 17, 18, 19 }); + final List expectedScores = dataVectors.stream() + .map(vector -> spaceType.getKnnVectorSimilarityFunction().compare(queryVector, vector)) + .collect(Collectors.toList()); + + KNNBinaryVectorValues values = mock(KNNBinaryVectorValues.class); + when(values.getVector()).thenReturn(dataVectors.get(0), dataVectors.get(1), dataVectors.get(2)); + + FixedBitSet filterBitSet = new FixedBitSet(4); + for (int id : filterIds) { + when(values.advance(id)).thenReturn(id); + filterBitSet.set(id); + } + + // Execute and verify + NestedBinaryVectorIdsKNNIterator iterator = new NestedBinaryVectorIdsKNNIterator( + filterBitSet, + queryVector, + values, + spaceType, + parentBitSet + ); + assertEquals(filterIds[0], iterator.nextDoc()); + assertEquals(expectedScores.get(0), iterator.score()); + assertEquals(filterIds[2], iterator.nextDoc()); + assertEquals(expectedScores.get(2), iterator.score()); + assertEquals(DocIdSetIterator.NO_MORE_DOCS, iterator.nextDoc()); + } + + @SneakyThrows + public void testNextDoc_whenIterateWithoutFilters_thenReturnBestChildDocsPerParent() { + final SpaceType spaceType = SpaceType.HAMMING; + final byte[] queryVector = { 1, 2, 3 }; + // Parent id for 0 -> 1 + // Parent id for 2, 3 -> 4 + // In bit representation, it is 10010. In long, it is 18. + final BitSet parentBitSet = new FixedBitSet(new long[] { 18 }, 5); + final List dataVectors = Arrays.asList(new byte[] { 11, 12, 13 }, new byte[] { 14, 15, 16 }, new byte[] { 17, 18, 19 }); + final List expectedScores = dataVectors.stream() + .map(vector -> spaceType.getKnnVectorSimilarityFunction().compare(queryVector, vector)) + .collect(Collectors.toList()); + + KNNBinaryVectorValues values = mock(KNNBinaryVectorValues.class); + when(values.getVector()).thenReturn(dataVectors.get(0), dataVectors.get(1), dataVectors.get(2)); + when(values.nextDoc()).thenReturn(0, 2, 3, Integer.MAX_VALUE); + + // Execute and verify + NestedBinaryVectorIdsKNNIterator iterator = new NestedBinaryVectorIdsKNNIterator(queryVector, values, spaceType, parentBitSet); + assertEquals(0, iterator.nextDoc()); + assertEquals(expectedScores.get(0), iterator.score()); + assertEquals(3, iterator.nextDoc()); + assertEquals(expectedScores.get(2), iterator.score()); + assertEquals(DocIdSetIterator.NO_MORE_DOCS, iterator.nextDoc()); + verify(values, never()).advance(anyInt()); + } +} diff --git a/src/test/java/org/opensearch/knn/index/query/iterators/NestedByteVectorIdsKNNIteratorTests.java b/src/test/java/org/opensearch/knn/index/query/iterators/NestedByteVectorIdsKNNIteratorTests.java index eff021234..08c859779 100644 --- a/src/test/java/org/opensearch/knn/index/query/iterators/NestedByteVectorIdsKNNIteratorTests.java +++ b/src/test/java/org/opensearch/knn/index/query/iterators/NestedByteVectorIdsKNNIteratorTests.java @@ -11,7 +11,7 @@ import org.apache.lucene.util.BitSet; import org.apache.lucene.util.FixedBitSet; import org.opensearch.knn.index.SpaceType; -import org.opensearch.knn.index.vectorvalues.KNNBinaryVectorValues; +import org.opensearch.knn.index.vectorvalues.KNNByteVectorValues; import java.util.Arrays; import java.util.List; @@ -26,19 +26,20 @@ public class NestedByteVectorIdsKNNIteratorTests extends TestCase { @SneakyThrows public void testNextDoc_whenIterate_ReturnBestChildDocsPerParent() { - final SpaceType spaceType = SpaceType.HAMMING; - final byte[] queryVector = { 1, 2, 3 }; + final SpaceType spaceType = SpaceType.L2; + final byte[] byteQueryVector = { 1, 2, 3 }; + final float[] queryVector = { 1.0f, 2.0f, 3.0f }; final int[] filterIds = { 0, 2, 3 }; // Parent id for 0 -> 1 // Parent id for 2, 3 -> 4 // In bit representation, it is 10010. In long, it is 18. final BitSet parentBitSet = new FixedBitSet(new long[] { 18 }, 5); - final List dataVectors = Arrays.asList(new byte[] { 11, 12, 13 }, new byte[] { 14, 15, 16 }, new byte[] { 17, 18, 19 }); + final List dataVectors = Arrays.asList(new byte[] { 11, 12, 13 }, new byte[] { 17, 18, 19 }, new byte[] { 14, 15, 16 }); final List expectedScores = dataVectors.stream() - .map(vector -> spaceType.getKnnVectorSimilarityFunction().compare(queryVector, vector)) + .map(vector -> spaceType.getKnnVectorSimilarityFunction().compare(byteQueryVector, vector)) .collect(Collectors.toList()); - KNNBinaryVectorValues values = mock(KNNBinaryVectorValues.class); + KNNByteVectorValues values = mock(KNNByteVectorValues.class); when(values.getVector()).thenReturn(dataVectors.get(0), dataVectors.get(1), dataVectors.get(2)); FixedBitSet filterBitSet = new FixedBitSet(4); @@ -64,18 +65,19 @@ public void testNextDoc_whenIterate_ReturnBestChildDocsPerParent() { @SneakyThrows public void testNextDoc_whenIterateWithoutFilters_thenReturnBestChildDocsPerParent() { - final SpaceType spaceType = SpaceType.HAMMING; - final byte[] queryVector = { 1, 2, 3 }; + final SpaceType spaceType = SpaceType.L2; + final byte[] byteQueryVector = { 1, 2, 3 }; + final float[] queryVector = { 1.0f, 2.0f, 3.0f }; // Parent id for 0 -> 1 // Parent id for 2, 3 -> 4 // In bit representation, it is 10010. In long, it is 18. final BitSet parentBitSet = new FixedBitSet(new long[] { 18 }, 5); - final List dataVectors = Arrays.asList(new byte[] { 11, 12, 13 }, new byte[] { 14, 15, 16 }, new byte[] { 17, 18, 19 }); + final List dataVectors = Arrays.asList(new byte[] { 11, 12, 13 }, new byte[] { 17, 18, 19 }, new byte[] { 14, 15, 16 }); final List expectedScores = dataVectors.stream() - .map(vector -> spaceType.getKnnVectorSimilarityFunction().compare(queryVector, vector)) + .map(vector -> spaceType.getKnnVectorSimilarityFunction().compare(byteQueryVector, vector)) .collect(Collectors.toList()); - KNNBinaryVectorValues values = mock(KNNBinaryVectorValues.class); + KNNByteVectorValues values = mock(KNNByteVectorValues.class); when(values.getVector()).thenReturn(dataVectors.get(0), dataVectors.get(1), dataVectors.get(2)); when(values.nextDoc()).thenReturn(0, 2, 3, Integer.MAX_VALUE); diff --git a/src/test/java/org/opensearch/knn/integ/FilteredSearchByteIT.java b/src/test/java/org/opensearch/knn/integ/FilteredSearchByteIT.java new file mode 100644 index 000000000..9f91aa6f8 --- /dev/null +++ b/src/test/java/org/opensearch/knn/integ/FilteredSearchByteIT.java @@ -0,0 +1,104 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.integ; + +import com.google.common.collect.ImmutableMap; +import lombok.SneakyThrows; +import lombok.extern.log4j.Log4j2; +import org.apache.http.util.EntityUtils; +import org.junit.After; +import org.opensearch.client.Response; +import org.opensearch.common.settings.Settings; +import org.opensearch.knn.KNNJsonIndexMappingsBuilder; +import org.opensearch.knn.KNNJsonQueryBuilder; +import org.opensearch.knn.KNNRestTestCase; +import org.opensearch.knn.index.KNNSettings; +import org.opensearch.knn.index.VectorDataType; +import org.opensearch.knn.index.engine.KNNEngine; + +import java.util.List; + +import static org.opensearch.knn.common.KNNConstants.METHOD_HNSW; + +@Log4j2 +public class FilteredSearchByteIT extends KNNRestTestCase { + @After + public void cleanUp() { + try { + deleteKNNIndex(INDEX_NAME); + } catch (Exception e) { + log.error(e); + } + } + + @SneakyThrows + public void testFilteredSearchWithFaissHnswByte_whenDoingApproximateSearch_thenReturnCorrectResults() { + validateFilteredSearchWithFaissHnswByte(INDEX_NAME, false); + } + + @SneakyThrows + public void testFilteredSearchWithFaissHnswByte_whenDoingExactSearch_thenReturnCorrectResults() { + validateFilteredSearchWithFaissHnswByte(INDEX_NAME, true); + } + + private void validateFilteredSearchWithFaissHnswByte(final String indexName, final boolean doExactSearch) throws Exception { + String filterFieldName = "parking"; + createKnnByteIndex(indexName, FIELD_NAME, 3, KNNEngine.FAISS); + + for (byte i = 1; i < 4; i++) { + addKnnDocWithAttributes( + indexName, + Integer.toString(i), + FIELD_NAME, + new float[] { i, i, i }, + ImmutableMap.of(filterFieldName, i % 2 == 1 ? "true" : "false") + ); + } + refreshIndex(indexName); + forceMergeKnnIndex(indexName); + + // Set it as 0 for approximate search and 100(larger than number of filtered id) for exact search + updateIndexSettings( + indexName, + Settings.builder().put(KNNSettings.ADVANCED_FILTERED_EXACT_SEARCH_THRESHOLD, doExactSearch ? 100 : 0) + ); + + Float[] queryVector = { 3f, 3f, 3f }; + String query = KNNJsonQueryBuilder.builder() + .fieldName(FIELD_NAME) + .vector(queryVector) + .k(3) + .filterFieldName(filterFieldName) + .filterValue("true") + .build() + .getQueryString(); + Response response = searchKNNIndex(indexName, query, 3); + String entity = EntityUtils.toString(response.getEntity()); + List docIds = parseIds(entity); + assertEquals(2, docIds.size()); + assertEquals("3", docIds.get(0)); + assertEquals("1", docIds.get(1)); + assertEquals(2, parseTotalSearchHits(entity)); + } + + private void createKnnByteIndex(final String indexName, final String fieldName, final int dimension, final KNNEngine knnEngine) + throws Exception { + KNNJsonIndexMappingsBuilder.Method method = KNNJsonIndexMappingsBuilder.Method.builder() + .methodName(METHOD_HNSW) + .engine(knnEngine.getName()) + .build(); + + String knnIndexMapping = KNNJsonIndexMappingsBuilder.builder() + .fieldName(fieldName) + .dimension(dimension) + .vectorDataType(VectorDataType.BYTE.getValue()) + .method(method) + .build() + .getIndexMapping(); + + createKnnIndex(indexName, knnIndexMapping); + } +} diff --git a/src/test/java/org/opensearch/knn/integ/NestedSearchByteIT.java b/src/test/java/org/opensearch/knn/integ/NestedSearchByteIT.java new file mode 100644 index 000000000..f7b51c1aa --- /dev/null +++ b/src/test/java/org/opensearch/knn/integ/NestedSearchByteIT.java @@ -0,0 +1,156 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.integ; + +import lombok.SneakyThrows; +import lombok.extern.log4j.Log4j2; +import org.apache.http.util.EntityUtils; +import org.junit.After; +import org.opensearch.client.Response; +import org.opensearch.common.settings.Settings; +import org.opensearch.knn.KNNJsonIndexMappingsBuilder; +import org.opensearch.knn.KNNJsonQueryBuilder; +import org.opensearch.knn.KNNRestTestCase; +import org.opensearch.knn.NestedKnnDocBuilder; +import org.opensearch.knn.index.KNNSettings; +import org.opensearch.knn.index.VectorDataType; +import org.opensearch.knn.index.engine.KNNEngine; + +import java.util.List; + +import static org.opensearch.knn.common.KNNConstants.METHOD_HNSW; + +@Log4j2 +public class NestedSearchByteIT extends KNNRestTestCase { + @After + public void cleanUp() { + try { + deleteKNNIndex(INDEX_NAME); + } catch (Exception e) { + log.error(e); + } + } + + @SneakyThrows + public void testNestedSearchWithFaissHnswByte_whenKIsTwo_thenReturnTwoResults() { + String nestedFieldName = "nested"; + createKnnByteIndexWithNestedField(INDEX_NAME, nestedFieldName, FIELD_NAME, 2, KNNEngine.FAISS); + + int totalDocCount = 15; + for (byte i = 0; i < totalDocCount; i++) { + String doc = NestedKnnDocBuilder.create(nestedFieldName) + .addVectors(FIELD_NAME, new Byte[] { i, i }, new Byte[] { i, i }) + .build(); + addKnnDoc(INDEX_NAME, String.valueOf(i), doc); + } + + refreshIndex(INDEX_NAME); + forceMergeKnnIndex(INDEX_NAME); + + Byte[] queryVector = { 14, 14 }; + String query = KNNJsonQueryBuilder.builder() + .nestedFieldName(nestedFieldName) + .fieldName(FIELD_NAME) + .vector(queryVector) + .k(2) + .build() + .getQueryString(); + Response response = searchKNNIndex(INDEX_NAME, query, 2); + String entity = EntityUtils.toString(response.getEntity()); + + assertEquals(2, parseHits(entity)); + assertEquals(2, parseTotalSearchHits(entity)); + assertEquals("14", parseIds(entity).get(0)); + assertNotEquals("14", parseIds(entity).get(1)); + } + + /** + * { + * "query": { + * "nested": { + * "path": "test_nested", + * "query": { + * "knn": { + * "test_nested.test_vector": { + * "vector": [ + * 1, 1, 1 + * ], + * "k": 3, + * "filter": { + * "term": { + * "parking": "true" + * } + * } + * } + * } + * } + * } + * } + * } + * + */ + @SneakyThrows + public void testNestedSearchWithFaissHnswByte_whenDoingExactSearch_thenReturnCorrectResults() { + String nestedFieldName = "nested"; + String filterFieldName = "parking"; + createKnnByteIndexWithNestedField(INDEX_NAME, nestedFieldName, FIELD_NAME, 3, KNNEngine.FAISS); + + for (byte i = 1; i < 4; i++) { + String doc = NestedKnnDocBuilder.create(nestedFieldName) + .addVectors(FIELD_NAME, new Byte[] { i, i, i }, new Byte[] { i, i, i }, new Byte[] { i, i, i }) + .addTopLevelField(filterFieldName, i % 2 == 1 ? "true" : "false") + .build(); + addKnnDoc(INDEX_NAME, String.valueOf(i), doc); + } + refreshIndex(INDEX_NAME); + forceMergeKnnIndex(INDEX_NAME); + + // Make it as an exact search by setting the threshold larger than size of filteredIds(6) + updateIndexSettings(INDEX_NAME, Settings.builder().put(KNNSettings.ADVANCED_FILTERED_EXACT_SEARCH_THRESHOLD, 100)); + + Byte[] queryVector = { 3, 3, 3 }; + String query = KNNJsonQueryBuilder.builder() + .nestedFieldName(nestedFieldName) + .fieldName(FIELD_NAME) + .vector(queryVector) + .k(3) + .filterFieldName(filterFieldName) + .filterValue("true") + .build() + .getQueryString(); + Response response = searchKNNIndex(INDEX_NAME, query, 3); + String entity = EntityUtils.toString(response.getEntity()); + List docIds = parseIds(entity); + assertEquals(2, docIds.size()); + assertEquals("3", docIds.get(0)); + assertEquals("1", docIds.get(1)); + assertEquals(2, parseTotalSearchHits(entity)); + } + + private void createKnnByteIndexWithNestedField( + final String indexName, + final String nestedFieldName, + final String fieldName, + final int dimension, + final KNNEngine knnEngine + ) throws Exception { + KNNJsonIndexMappingsBuilder.Method method = KNNJsonIndexMappingsBuilder.Method.builder() + .methodName(METHOD_HNSW) + .engine(knnEngine.getName()) + .build(); + + String knnIndexMapping = KNNJsonIndexMappingsBuilder.builder() + .nestedFieldName(nestedFieldName) + .fieldName(fieldName) + .dimension(dimension) + .vectorDataType(VectorDataType.BYTE.getValue()) + .method(method) + .build() + .getIndexMapping(); + + createKnnIndex(indexName, knnIndexMapping); + } +} From 4b6472da1a1f7a7452643c677e9413ccf7a6a88c Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Mon, 30 Sep 2024 21:43:34 -0700 Subject: [PATCH 389/416] Adding Support to Enable/Disble Share level Rescoring and Update Oversampling Factor (#2172) (#2173) Signed-off-by: VIKASH TIWARI Signed-off-by: John Mazanec Co-authored-by: Vikasht34 --- CHANGELOG.md | 1 + .../org/opensearch/knn/index/KNNSettings.java | 36 ++++++++- .../knn/index/mapper/CompressionLevel.java | 31 ++++---- .../nativelib/NativeEngineKnnVectorQuery.java | 6 +- .../index/query/rescore/RescoreContext.java | 9 +++ .../knn/index/KNNSettingsTests.java | 35 +++++++++ .../index/mapper/CompressionLevelTests.java | 73 ++++++++++++------- .../NativeEngineKNNVectorQueryTests.java | 71 +++++++++++++++++- 8 files changed, 214 insertions(+), 48 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e9acc9e05..2191fc3cf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), * Optimize reduceToTopK in ResultUtil by removing pre-filling and reducing peek calls [#2146](https://github.com/opensearch-project/k-NN/pull/2146) * Update Default Rescore Context based on Dimension [#2149](https://github.com/opensearch-project/k-NN/pull/2149) * KNNIterators should support with and without filters [#2155](https://github.com/opensearch-project/k-NN/pull/2155) +* Adding Support to Enable/Disble Share level Rescoring and Update Oversampling Factor[#2172](https://github.com/opensearch-project/k-NN/pull/2172) ### Bug Fixes * KNN80DocValues should only be considered for BinaryDocValues fields [#2147](https://github.com/opensearch-project/k-NN/pull/2147) ### Infrastructure diff --git a/src/main/java/org/opensearch/knn/index/KNNSettings.java b/src/main/java/org/opensearch/knn/index/KNNSettings.java index 5fcc51bb5..1753140e6 100644 --- a/src/main/java/org/opensearch/knn/index/KNNSettings.java +++ b/src/main/java/org/opensearch/knn/index/KNNSettings.java @@ -88,6 +88,7 @@ public class KNNSettings { public static final String QUANTIZATION_STATE_CACHE_SIZE_LIMIT = "knn.quantization.cache.size.limit"; public static final String QUANTIZATION_STATE_CACHE_EXPIRY_TIME_MINUTES = "knn.quantization.cache.expiry.minutes"; public static final String KNN_FAISS_AVX512_DISABLED = "knn.faiss.avx512.disabled"; + public static final String KNN_DISK_VECTOR_SHARD_LEVEL_RESCORING_DISABLED = "index.knn.disk.vector.shard_level_rescoring_disabled"; /** * Default setting values @@ -112,11 +113,31 @@ public class KNNSettings { public static final Integer KNN_MAX_QUANTIZATION_STATE_CACHE_SIZE_LIMIT_PERCENTAGE = 10; // Quantization state cache limit cannot exceed // 10% of the JVM heap public static final Integer KNN_DEFAULT_QUANTIZATION_STATE_CACHE_EXPIRY_TIME_MINUTES = 60; + public static final boolean KNN_DISK_VECTOR_SHARD_LEVEL_RESCORING_DISABLED_VALUE = true; /** * Settings Definition */ + /** + * This setting controls whether shard-level re-scoring for KNN disk-based vectors is turned off. + * The setting uses: + *
      + *
    • KNN_DISK_VECTOR_SHARD_LEVEL_RESCORING_DISABLED: The name of the setting.
      • + *
    • KNN_DISK_VECTOR_SHARD_LEVEL_RESCORING_DISABLED_VALUE: The default value (true or false).
      • + *
    • IndexScope: The setting works at the index level.
      • + *
    • Dynamic: This setting can be changed without restarting the cluster.
      • + *
    + * + * @see Setting#boolSetting(String, boolean, Setting.Property...) + */ + public static final Setting KNN_DISK_VECTOR_SHARD_LEVEL_RESCORING_DISABLED_SETTING = Setting.boolSetting( + KNN_DISK_VECTOR_SHARD_LEVEL_RESCORING_DISABLED, + KNN_DISK_VECTOR_SHARD_LEVEL_RESCORING_DISABLED_VALUE, + IndexScope, + Dynamic + ); + // This setting controls how much memory should be used to transfer vectors from Java to JNI Layer. The default // 1% of the JVM heap public static final Setting KNN_VECTOR_STREAMING_MEMORY_LIMIT_PCT_SETTING = Setting.memorySizeSetting( @@ -454,6 +475,10 @@ private Setting getSetting(String key) { return QUANTIZATION_STATE_CACHE_EXPIRY_TIME_MINUTES_SETTING; } + if (KNN_DISK_VECTOR_SHARD_LEVEL_RESCORING_DISABLED.equals(key)) { + return KNN_DISK_VECTOR_SHARD_LEVEL_RESCORING_DISABLED_SETTING; + } + throw new IllegalArgumentException("Cannot find setting by key [" + key + "]"); } @@ -475,7 +500,8 @@ public List> getSettings() { KNN_VECTOR_STREAMING_MEMORY_LIMIT_PCT_SETTING, KNN_FAISS_AVX512_DISABLED_SETTING, QUANTIZATION_STATE_CACHE_SIZE_LIMIT_SETTING, - QUANTIZATION_STATE_CACHE_EXPIRY_TIME_MINUTES_SETTING + QUANTIZATION_STATE_CACHE_EXPIRY_TIME_MINUTES_SETTING, + KNN_DISK_VECTOR_SHARD_LEVEL_RESCORING_DISABLED_SETTING ); return Stream.concat(settings.stream(), Stream.concat(getFeatureFlags().stream(), dynamicCacheSettings.values().stream())) .collect(Collectors.toList()); @@ -528,6 +554,14 @@ public static Integer getFilteredExactSearchThreshold(final String indexName) { .getAsInt(ADVANCED_FILTERED_EXACT_SEARCH_THRESHOLD, ADVANCED_FILTERED_EXACT_SEARCH_THRESHOLD_DEFAULT_VALUE); } + public static boolean isShardLevelRescoringDisabledForDiskBasedVector(String indexName) { + return KNNSettings.state().clusterService.state() + .getMetadata() + .index(indexName) + .getSettings() + .getAsBoolean(KNN_DISK_VECTOR_SHARD_LEVEL_RESCORING_DISABLED, true); + } + public void initialize(Client client, ClusterService clusterService) { this.client = client; this.clusterService = clusterService; diff --git a/src/main/java/org/opensearch/knn/index/mapper/CompressionLevel.java b/src/main/java/org/opensearch/knn/index/mapper/CompressionLevel.java index 3e1b47db7..c9a169efc 100644 --- a/src/main/java/org/opensearch/knn/index/mapper/CompressionLevel.java +++ b/src/main/java/org/opensearch/knn/index/mapper/CompressionLevel.java @@ -97,32 +97,35 @@ public static boolean isConfigured(CompressionLevel compressionLevel) { /** * Returns the appropriate {@link RescoreContext} based on the given {@code mode} and {@code dimension}. * - *

    If the {@code mode} is present in the valid {@code modesForRescore} set, the method checks the value of - * {@code dimension}: + *

    If the {@code mode} is present in the valid {@code modesForRescore} set, the method adjusts the oversample factor based on the + * {@code dimension} value: *

      - *
    • If {@code dimension} is less than or equal to 1000, it returns a {@link RescoreContext} with an - * oversample factor of 5.0f.
      • - *
    • If {@code dimension} is greater than 1000, it returns the default {@link RescoreContext} associated with - * the {@link CompressionLevel}. If no default is set, it falls back to {@link RescoreContext#getDefault()}.
      • + *
    • If {@code dimension} is greater than or equal to 1000, no oversampling is applied (oversample factor = 1.0).
      • + *
    • If {@code dimension} is greater than or equal to 768 but less than 1000, a 2x oversample factor is applied (oversample factor = 2.0).
      • + *
    • If {@code dimension} is less than 768, a 3x oversample factor is applied (oversample factor = 3.0).
      • *
    - * If the {@code mode} is not valid, the method returns {@code null}. + * If the {@code mode} is not present in the {@code modesForRescore} set, the method returns {@code null}. * * @param mode The {@link Mode} for which to retrieve the {@link RescoreContext}. * @param dimension The dimensional value that determines the {@link RescoreContext} behavior. - * @return A {@link RescoreContext} with an oversample factor of 5.0f if {@code dimension} is less than - * or equal to 1000, the default {@link RescoreContext} if greater, or {@code null} if the mode - * is invalid. + * @return A {@link RescoreContext} with the appropriate oversample factor based on the dimension, or {@code null} if the mode + * is not valid. */ public RescoreContext getDefaultRescoreContext(Mode mode, int dimension) { if (modesForRescore.contains(mode)) { // Adjust RescoreContext based on dimension - if (dimension <= RescoreContext.DIMENSION_THRESHOLD) { - // For dimensions <= 1000, return a RescoreContext with 5.0f oversample factor - return RescoreContext.builder().oversampleFactor(RescoreContext.OVERSAMPLE_FACTOR_BELOW_DIMENSION_THRESHOLD).build(); + if (dimension >= RescoreContext.DIMENSION_THRESHOLD_1000) { + // No oversampling for dimensions >= 1000 + return RescoreContext.builder().oversampleFactor(RescoreContext.OVERSAMPLE_FACTOR_1000).build(); + } else if (dimension >= RescoreContext.DIMENSION_THRESHOLD_768) { + // 2x oversampling for dimensions >= 768 but < 1000 + return RescoreContext.builder().oversampleFactor(RescoreContext.OVERSAMPLE_FACTOR_768).build(); } else { - return defaultRescoreContext; + // 3x oversampling for dimensions < 768 + return RescoreContext.builder().oversampleFactor(RescoreContext.OVERSAMPLE_FACTOR_BELOW_768).build(); } } return null; } + } diff --git a/src/main/java/org/opensearch/knn/index/query/nativelib/NativeEngineKnnVectorQuery.java b/src/main/java/org/opensearch/knn/index/query/nativelib/NativeEngineKnnVectorQuery.java index 945da850a..adb2875d5 100644 --- a/src/main/java/org/opensearch/knn/index/query/nativelib/NativeEngineKnnVectorQuery.java +++ b/src/main/java/org/opensearch/knn/index/query/nativelib/NativeEngineKnnVectorQuery.java @@ -20,6 +20,7 @@ import org.apache.lucene.util.BitSet; import org.apache.lucene.util.Bits; import org.opensearch.common.StopWatch; +import org.opensearch.knn.index.KNNSettings; import org.opensearch.knn.index.query.ExactSearcher; import org.opensearch.knn.index.query.KNNQuery; import org.opensearch.knn.index.query.KNNWeight; @@ -54,7 +55,6 @@ public Weight createWeight(IndexSearcher indexSearcher, ScoreMode scoreMode, flo final IndexReader reader = indexSearcher.getIndexReader(); final KNNWeight knnWeight = (KNNWeight) knnQuery.createWeight(indexSearcher, ScoreMode.COMPLETE, 1); List leafReaderContexts = reader.leaves(); - List> perLeafResults; RescoreContext rescoreContext = knnQuery.getRescoreContext(); int finalK = knnQuery.getK(); @@ -63,7 +63,9 @@ public Weight createWeight(IndexSearcher indexSearcher, ScoreMode scoreMode, flo } else { int firstPassK = rescoreContext.getFirstPassK(finalK); perLeafResults = doSearch(indexSearcher, leafReaderContexts, knnWeight, firstPassK); - ResultUtil.reduceToTopK(perLeafResults, firstPassK); + if (KNNSettings.isShardLevelRescoringDisabledForDiskBasedVector(knnQuery.getIndexName()) == false) { + ResultUtil.reduceToTopK(perLeafResults, firstPassK); + } StopWatch stopWatch = new StopWatch().start(); perLeafResults = doRescore(indexSearcher, leafReaderContexts, knnWeight, perLeafResults, finalK); diff --git a/src/main/java/org/opensearch/knn/index/query/rescore/RescoreContext.java b/src/main/java/org/opensearch/knn/index/query/rescore/RescoreContext.java index 51d4e491c..a2563b2a6 100644 --- a/src/main/java/org/opensearch/knn/index/query/rescore/RescoreContext.java +++ b/src/main/java/org/opensearch/knn/index/query/rescore/RescoreContext.java @@ -24,6 +24,15 @@ public final class RescoreContext { public static final int DIMENSION_THRESHOLD = 1000; public static final float OVERSAMPLE_FACTOR_BELOW_DIMENSION_THRESHOLD = 5.0f; + // Dimension thresholds for adjusting oversample factor + public static final int DIMENSION_THRESHOLD_1000 = 1000; + public static final int DIMENSION_THRESHOLD_768 = 768; + + // Oversample factors based on dimension thresholds + public static final float OVERSAMPLE_FACTOR_1000 = 1.0f; // No oversampling for dimensions >= 1000 + public static final float OVERSAMPLE_FACTOR_768 = 2.0f; // 2x oversampling for dimensions >= 768 and < 1000 + public static final float OVERSAMPLE_FACTOR_BELOW_768 = 3.0f; // 3x oversampling for dimensions < 768 + // Todo:- We will improve this in upcoming releases public static final int MIN_FIRST_PASS_RESULTS = 100; diff --git a/src/test/java/org/opensearch/knn/index/KNNSettingsTests.java b/src/test/java/org/opensearch/knn/index/KNNSettingsTests.java index 8008c547f..961a82a72 100644 --- a/src/test/java/org/opensearch/knn/index/KNNSettingsTests.java +++ b/src/test/java/org/opensearch/knn/index/KNNSettingsTests.java @@ -162,6 +162,41 @@ public void testGetEfSearch_whenEFSearchValueSetByUser_thenReturnValue() { assertEquals(userProvidedEfSearch, efSearchValue); } + @SneakyThrows + public void testShardLevelRescoringDisabled_whenNoValuesProvidedByUser_thenDefaultSettingsUsed() { + Node mockNode = createMockNode(Collections.emptyMap()); + mockNode.start(); + ClusterService clusterService = mockNode.injector().getInstance(ClusterService.class); + mockNode.client().admin().cluster().state(new ClusterStateRequest()).actionGet(); + mockNode.client().admin().indices().create(new CreateIndexRequest(INDEX_NAME)).actionGet(); + KNNSettings.state().setClusterService(clusterService); + + boolean shardLevelRescoringDisabled = KNNSettings.isShardLevelRescoringDisabledForDiskBasedVector(INDEX_NAME); + mockNode.close(); + assertTrue(shardLevelRescoringDisabled); + } + + @SneakyThrows + public void testShardLevelRescoringDisabled_whenValueProvidedByUser_thenSettingApplied() { + boolean userDefinedRescoringDisabled = false; + Node mockNode = createMockNode(Collections.emptyMap()); + mockNode.start(); + ClusterService clusterService = mockNode.injector().getInstance(ClusterService.class); + mockNode.client().admin().cluster().state(new ClusterStateRequest()).actionGet(); + mockNode.client().admin().indices().create(new CreateIndexRequest(INDEX_NAME)).actionGet(); + KNNSettings.state().setClusterService(clusterService); + + final Settings rescoringDisabledSetting = Settings.builder() + .put(KNNSettings.KNN_DISK_VECTOR_SHARD_LEVEL_RESCORING_DISABLED, userDefinedRescoringDisabled) + .build(); + + mockNode.client().admin().indices().updateSettings(new UpdateSettingsRequest(rescoringDisabledSetting, INDEX_NAME)).actionGet(); + + boolean shardLevelRescoringDisabled = KNNSettings.isShardLevelRescoringDisabledForDiskBasedVector(INDEX_NAME); + mockNode.close(); + assertEquals(userDefinedRescoringDisabled, shardLevelRescoringDisabled); + } + @SneakyThrows public void testGetFaissAVX2DisabledSettingValueFromConfig_enableSetting_thenValidateAndSucceed() { boolean expectedKNNFaissAVX2Disabled = true; diff --git a/src/test/java/org/opensearch/knn/index/mapper/CompressionLevelTests.java b/src/test/java/org/opensearch/knn/index/mapper/CompressionLevelTests.java index cc70d4c2d..57372b11e 100644 --- a/src/test/java/org/opensearch/knn/index/mapper/CompressionLevelTests.java +++ b/src/test/java/org/opensearch/knn/index/mapper/CompressionLevelTests.java @@ -44,65 +44,84 @@ public void testIsConfigured() { public void testGetDefaultRescoreContext() { // Test rescore context for ON_DISK mode Mode mode = Mode.ON_DISK; - int belowThresholdDimension = 500; // A dimension below the threshold - int aboveThresholdDimension = 1500; // A dimension above the threshold - // x32 with dimension <= 1000 should have an oversample factor of 5.0f + // Test various dimensions based on the updated oversampling logic + int belowThresholdDimension = 500; // A dimension below 768 + int between768and1000Dimension = 800; // A dimension between 768 and 1000 + int above1000Dimension = 1500; // A dimension above 1000 + + // Compression level x32 with dimension < 768 should have an oversample factor of 3.0f RescoreContext rescoreContext = CompressionLevel.x32.getDefaultRescoreContext(mode, belowThresholdDimension); assertNotNull(rescoreContext); - assertEquals(5.0f, rescoreContext.getOversampleFactor(), 0.0f); + assertEquals(3.0f, rescoreContext.getOversampleFactor(), 0.0f); - // x32 with dimension > 1000 should have an oversample factor of 3.0f - rescoreContext = CompressionLevel.x32.getDefaultRescoreContext(mode, aboveThresholdDimension); + // Compression level x32 with dimension between 768 and 1000 should have an oversample factor of 2.0f + rescoreContext = CompressionLevel.x32.getDefaultRescoreContext(mode, between768and1000Dimension); assertNotNull(rescoreContext); - assertEquals(3.0f, rescoreContext.getOversampleFactor(), 0.0f); + assertEquals(2.0f, rescoreContext.getOversampleFactor(), 0.0f); - // x16 with dimension <= 1000 should have an oversample factor of 5.0f - rescoreContext = CompressionLevel.x16.getDefaultRescoreContext(mode, belowThresholdDimension); + // Compression level x32 with dimension > 1000 should have no oversampling (1.0f) + rescoreContext = CompressionLevel.x32.getDefaultRescoreContext(mode, above1000Dimension); assertNotNull(rescoreContext); - assertEquals(5.0f, rescoreContext.getOversampleFactor(), 0.0f); + assertEquals(1.0f, rescoreContext.getOversampleFactor(), 0.0f); - // x16 with dimension > 1000 should have an oversample factor of 3.0f - rescoreContext = CompressionLevel.x16.getDefaultRescoreContext(mode, aboveThresholdDimension); + // Compression level x16 with dimension < 768 should have an oversample factor of 3.0f + rescoreContext = CompressionLevel.x16.getDefaultRescoreContext(mode, belowThresholdDimension); assertNotNull(rescoreContext); assertEquals(3.0f, rescoreContext.getOversampleFactor(), 0.0f); - // x8 with dimension <= 1000 should have an oversample factor of 5.0f + // Compression level x16 with dimension between 768 and 1000 should have an oversample factor of 2.0f + rescoreContext = CompressionLevel.x16.getDefaultRescoreContext(mode, between768and1000Dimension); + assertNotNull(rescoreContext); + assertEquals(2.0f, rescoreContext.getOversampleFactor(), 0.0f); + + // Compression level x16 with dimension > 1000 should have no oversampling (1.0f) + rescoreContext = CompressionLevel.x16.getDefaultRescoreContext(mode, above1000Dimension); + assertNotNull(rescoreContext); + assertEquals(1.0f, rescoreContext.getOversampleFactor(), 0.0f); + + // Compression level x8 with dimension < 768 should have an oversample factor of 3.0f rescoreContext = CompressionLevel.x8.getDefaultRescoreContext(mode, belowThresholdDimension); assertNotNull(rescoreContext); - assertEquals(5.0f, rescoreContext.getOversampleFactor(), 0.0f); + assertEquals(3.0f, rescoreContext.getOversampleFactor(), 0.0f); - // x8 with dimension > 1000 should have an oversample factor of 2.0f - rescoreContext = CompressionLevel.x8.getDefaultRescoreContext(mode, aboveThresholdDimension); + // Compression level x8 with dimension between 768 and 1000 should have an oversample factor of 2.0f + rescoreContext = CompressionLevel.x8.getDefaultRescoreContext(mode, between768and1000Dimension); assertNotNull(rescoreContext); assertEquals(2.0f, rescoreContext.getOversampleFactor(), 0.0f); - // x4 with dimension <= 1000 should have an oversample factor of 5.0f (though it doesn't have its own RescoreContext) + // Compression level x8 with dimension > 1000 should have no oversampling (1.0f) + rescoreContext = CompressionLevel.x8.getDefaultRescoreContext(mode, above1000Dimension); + assertNotNull(rescoreContext); + assertEquals(1.0f, rescoreContext.getOversampleFactor(), 0.0f); + + // Compression level x4 with dimension < 768 should return null (no RescoreContext) rescoreContext = CompressionLevel.x4.getDefaultRescoreContext(mode, belowThresholdDimension); assertNull(rescoreContext); - // x4 with dimension > 1000 should return null (no RescoreContext is configured for x4) - rescoreContext = CompressionLevel.x4.getDefaultRescoreContext(mode, aboveThresholdDimension); - assertNull(rescoreContext); - // Other compression levels should behave similarly with respect to dimension + // Compression level x4 with dimension > 1000 should return null (no RescoreContext) + rescoreContext = CompressionLevel.x4.getDefaultRescoreContext(mode, above1000Dimension); + assertNull(rescoreContext); + // Compression level x2 with dimension < 768 should return null rescoreContext = CompressionLevel.x2.getDefaultRescoreContext(mode, belowThresholdDimension); assertNull(rescoreContext); - // x2 with dimension > 1000 should return null - rescoreContext = CompressionLevel.x2.getDefaultRescoreContext(mode, aboveThresholdDimension); + // Compression level x2 with dimension > 1000 should return null + rescoreContext = CompressionLevel.x2.getDefaultRescoreContext(mode, above1000Dimension); assertNull(rescoreContext); + // Compression level x1 with dimension < 768 should return null rescoreContext = CompressionLevel.x1.getDefaultRescoreContext(mode, belowThresholdDimension); assertNull(rescoreContext); - // x1 with dimension > 1000 should return null - rescoreContext = CompressionLevel.x1.getDefaultRescoreContext(mode, aboveThresholdDimension); + // Compression level x1 with dimension > 1000 should return null + rescoreContext = CompressionLevel.x1.getDefaultRescoreContext(mode, above1000Dimension); assertNull(rescoreContext); - // NOT_CONFIGURED with dimension <= 1000 should return a RescoreContext with an oversample factor of 5.0f + // NOT_CONFIGURED mode should return null for any dimension rescoreContext = CompressionLevel.NOT_CONFIGURED.getDefaultRescoreContext(mode, belowThresholdDimension); assertNull(rescoreContext); - } + } diff --git a/src/test/java/org/opensearch/knn/index/query/nativelib/NativeEngineKNNVectorQueryTests.java b/src/test/java/org/opensearch/knn/index/query/nativelib/NativeEngineKNNVectorQueryTests.java index 06350f39c..7fd96c6df 100644 --- a/src/test/java/org/opensearch/knn/index/query/nativelib/NativeEngineKNNVectorQueryTests.java +++ b/src/test/java/org/opensearch/knn/index/query/nativelib/NativeEngineKNNVectorQueryTests.java @@ -17,11 +17,16 @@ import org.apache.lucene.search.TaskExecutor; import org.apache.lucene.search.TopDocs; import org.apache.lucene.search.Weight; +import org.apache.lucene.search.ScoreDoc; +import org.apache.lucene.search.TotalHits; import org.apache.lucene.util.Bits; import org.mockito.InjectMocks; import org.mockito.Mock; import org.mockito.MockedStatic; import org.mockito.invocation.InvocationOnMock; +import org.opensearch.cluster.ClusterState; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.knn.index.KNNSettings; import org.opensearch.knn.index.query.KNNQuery; import org.opensearch.knn.index.query.KNNWeight; import org.opensearch.knn.index.query.ResultUtil; @@ -35,12 +40,11 @@ import java.util.Map; import java.util.concurrent.Callable; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyInt; -import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.ArgumentMatchers.*; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.mockStatic; import static org.mockito.Mockito.when; +import static org.mockito.Mockito.times; import static org.mockito.MockitoAnnotations.openMocks; public class NativeEngineKNNVectorQueryTests extends OpenSearchTestCase { @@ -66,6 +70,9 @@ public class NativeEngineKNNVectorQueryTests extends OpenSearchTestCase { @Mock private LeafReader leafReader2; + @Mock + private ClusterService clusterService; + @InjectMocks private NativeEngineKnnVectorQuery objectUnderTest; @@ -91,6 +98,11 @@ public void setUp() throws Exception { }); when(reader.getContext()).thenReturn(indexReaderContext); + + when(clusterService.state()).thenReturn(mock(ClusterState.class)); // Mock ClusterState + + // Set ClusterService in KNNSettings + KNNSettings.state().setClusterService(clusterService); } @SneakyThrows @@ -127,6 +139,49 @@ public void testMultiLeaf() { assertEquals(expected, actual.getQuery()); } + @SneakyThrows + public void testRescoreWhenShardLevelRescoringEnabled() { + // Given + List leaves = List.of(leaf1, leaf2); + when(reader.leaves()).thenReturn(leaves); + + int k = 2; + int firstPassK = 3; + Map initialLeaf1Results = new HashMap<>(Map.of(0, 21f, 1, 19f, 2, 17f)); + Map initialLeaf2Results = new HashMap<>(Map.of(0, 20f, 1, 18f, 2, 16f)); + Map rescoredLeaf1Results = new HashMap<>(Map.of(0, 18f, 1, 20f)); + Map rescoredLeaf2Results = new HashMap<>(Map.of(0, 21f)); + + when(knnQuery.getRescoreContext()).thenReturn(RescoreContext.builder().oversampleFactor(1.5f).build()); + when(knnQuery.getK()).thenReturn(k); + when(knnWeight.getQuery()).thenReturn(knnQuery); + when(knnWeight.searchLeaf(leaf1, firstPassK)).thenReturn(initialLeaf1Results); + when(knnWeight.searchLeaf(leaf2, firstPassK)).thenReturn(initialLeaf2Results); + when(knnWeight.exactSearch(eq(leaf1), any())).thenReturn(rescoredLeaf1Results); + when(knnWeight.exactSearch(eq(leaf2), any())).thenReturn(rescoredLeaf2Results); + + try ( + MockedStatic mockedKnnSettings = mockStatic(KNNSettings.class); + MockedStatic mockedResultUtil = mockStatic(ResultUtil.class) + ) { + + // When shard-level re-scoring is enabled + mockedKnnSettings.when(() -> KNNSettings.isShardLevelRescoringDisabledForDiskBasedVector(any())).thenReturn(false); + + // Mock ResultUtil to return valid TopDocs + mockedResultUtil.when(() -> ResultUtil.resultMapToTopDocs(any(), anyInt())) + .thenReturn(new TopDocs(new TotalHits(0, TotalHits.Relation.EQUAL_TO), new ScoreDoc[0])); + mockedResultUtil.when(() -> ResultUtil.reduceToTopK(any(), anyInt())).thenCallRealMethod(); + + // When + Weight actual = objectUnderTest.createWeight(searcher, ScoreMode.COMPLETE, 1); + + // Then + mockedResultUtil.verify(() -> ResultUtil.reduceToTopK(any(), anyInt()), times(2)); + assertNotNull(actual); + } + } + @SneakyThrows public void testSingleLeaf() { // Given @@ -188,7 +243,15 @@ public void testRescore() { when(knnWeight.exactSearch(eq(leaf1), any())).thenReturn(rescoredLeaf1Results); when(knnWeight.exactSearch(eq(leaf2), any())).thenReturn(rescoredLeaf2Results); - try (MockedStatic mockedResultUtil = mockStatic(ResultUtil.class)) { + + try ( + MockedStatic mockedKnnSettings = mockStatic(KNNSettings.class); + MockedStatic mockedResultUtil = mockStatic(ResultUtil.class) + ) { + + // When shard-level re-scoring is enabled + mockedKnnSettings.when(() -> KNNSettings.isShardLevelRescoringDisabledForDiskBasedVector(any())).thenReturn(true); + mockedResultUtil.when(() -> ResultUtil.reduceToTopK(any(), anyInt())).thenAnswer(InvocationOnMock::callRealMethod); mockedResultUtil.when(() -> ResultUtil.resultMapToTopDocs(eq(rescoredLeaf1Results), anyInt())).thenAnswer(t -> topDocs1); mockedResultUtil.when(() -> ResultUtil.resultMapToTopDocs(eq(rescoredLeaf2Results), anyInt())).thenAnswer(t -> topDocs2); From ec588371fc4059af9e1525c3d217c2efa9524a27 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Tue, 1 Oct 2024 10:28:07 -0700 Subject: [PATCH 390/416] Refactor and Update unit test to include field with no live docs (#2167) (#2170) Refactored if/else to reduce nesting. Added unit test when one of the field doesn't have live docs. Signed-off-by: Vijayan Balasubramanian (cherry picked from commit f16f2250b082ce8e2c5eda23b38d27a89365fa13) Co-authored-by: Vijayan Balasubramanian --- CHANGELOG.md | 1 + .../NativeEngines990KnnVectorsWriter.java | 32 ++++++++-------- ...eEngines990KnnVectorsWriterFlushTests.java | 38 ++++++++++++------- 3 files changed, 42 insertions(+), 29 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2191fc3cf..c11d258e8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,3 +31,4 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Maintenance * Remove benchmarks folder from k-NN repo [#2127](https://github.com/opensearch-project/k-NN/pull/2127) ### Refactoring +* Minor refactoring and refactored some unit test [#2167](https://github.com/opensearch-project/k-NN/pull/2167) diff --git a/src/main/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsWriter.java b/src/main/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsWriter.java index 23cd2a4de..2f22565c9 100644 --- a/src/main/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsWriter.java +++ b/src/main/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsWriter.java @@ -84,24 +84,24 @@ public void flush(int maxDoc, final Sorter.DocMap sortMap) throws IOException { final FieldInfo fieldInfo = field.getFieldInfo(); final VectorDataType vectorDataType = extractVectorDataType(fieldInfo); int totalLiveDocs = field.getVectors().size(); - if (totalLiveDocs > 0) { - final Supplier> knnVectorValuesSupplier = () -> getVectorValues( - vectorDataType, - field.getDocsWithField(), - field.getVectors() - ); - final QuantizationState quantizationState = train(field.getFieldInfo(), knnVectorValuesSupplier, totalLiveDocs); - final NativeIndexWriter writer = NativeIndexWriter.getWriter(fieldInfo, segmentWriteState, quantizationState); - final KNNVectorValues knnVectorValues = knnVectorValuesSupplier.get(); - - StopWatch stopWatch = new StopWatch().start(); - writer.flushIndex(knnVectorValues, totalLiveDocs); - long time_in_millis = stopWatch.stop().totalTime().millis(); - KNNGraphValue.REFRESH_TOTAL_TIME_IN_MILLIS.incrementBy(time_in_millis); - log.debug("Flush took {} ms for vector field [{}]", time_in_millis, fieldInfo.getName()); - } else { + if (totalLiveDocs == 0) { log.debug("[Flush] No live docs for field {}", fieldInfo.getName()); + continue; } + final Supplier> knnVectorValuesSupplier = () -> getVectorValues( + vectorDataType, + field.getDocsWithField(), + field.getVectors() + ); + final QuantizationState quantizationState = train(field.getFieldInfo(), knnVectorValuesSupplier, totalLiveDocs); + final NativeIndexWriter writer = NativeIndexWriter.getWriter(fieldInfo, segmentWriteState, quantizationState); + final KNNVectorValues knnVectorValues = knnVectorValuesSupplier.get(); + + StopWatch stopWatch = new StopWatch().start(); + writer.flushIndex(knnVectorValues, totalLiveDocs); + long time_in_millis = stopWatch.stop().totalTime().millis(); + KNNGraphValue.REFRESH_TOTAL_TIME_IN_MILLIS.incrementBy(time_in_millis); + log.debug("Flush took {} ms for vector field [{}]", time_in_millis, fieldInfo.getName()); } } diff --git a/src/test/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsWriterFlushTests.java b/src/test/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsWriterFlushTests.java index dbb564908..9f74b2c10 100644 --- a/src/test/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsWriterFlushTests.java +++ b/src/test/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsWriterFlushTests.java @@ -32,8 +32,11 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; +import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.function.Predicate; +import java.util.stream.Collectors; import java.util.stream.IntStream; import static com.carrotsearch.randomizedtesting.RandomizedTest.$; @@ -44,6 +47,7 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.mockConstruction; import static org.mockito.Mockito.mockStatic; +import static org.mockito.Mockito.never; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -86,6 +90,7 @@ public static Collection data() { "Multi Field", List.of( Map.of(0, new float[] { 1, 2, 3 }, 1, new float[] { 2, 3, 4 }, 2, new float[] { 3, 4, 5 }), + Collections.emptyMap(), Map.of( 0, new float[] { 1, 2, 3, 4 }, @@ -105,18 +110,16 @@ public static Collection data() { @SneakyThrows public void testFlush() { // Given - List> expectedVectorValues = new ArrayList<>(); - IntStream.range(0, vectorsPerField.size()).forEach(i -> { + final List> expectedVectorValues = vectorsPerField.stream().map(vectors -> { final TestVectorValues.PreDefinedFloatVectorValues randomVectorValues = new TestVectorValues.PreDefinedFloatVectorValues( - new ArrayList<>(vectorsPerField.get(i).values()) + new ArrayList<>(vectors.values()) ); final KNNVectorValues knnVectorValues = KNNVectorValuesFactory.getVectorValues( VectorDataType.FLOAT, randomVectorValues ); - expectedVectorValues.add(knnVectorValues); - - }); + return knnVectorValues; + }).collect(Collectors.toList()); try ( MockedStatic fieldWriterMockedStatic = mockStatic(NativeEngineFieldVectorsWriter.class); @@ -172,15 +175,19 @@ public void testFlush() { IntStream.range(0, vectorsPerField.size()).forEach(i -> { try { - verify(nativeIndexWriter).flushIndex(expectedVectorValues.get(i), vectorsPerField.get(i).size()); + if (vectorsPerField.get(i).isEmpty()) { + verify(nativeIndexWriter, never()).flushIndex(expectedVectorValues.get(i), vectorsPerField.get(i).size()); + } else { + verify(nativeIndexWriter).flushIndex(expectedVectorValues.get(i), vectorsPerField.get(i).size()); + } } catch (Exception e) { throw new RuntimeException(e); } }); - + final Long expectedTimesGetVectorValuesIsCalled = vectorsPerField.stream().filter(Predicate.not(Map::isEmpty)).count(); knnVectorValuesFactoryMockedStatic.verify( () -> KNNVectorValuesFactory.getVectorValues(any(VectorDataType.class), any(DocsWithFieldSet.class), any()), - times(expectedVectorValues.size()) + times(Math.toIntExact(expectedTimesGetVectorValuesIsCalled)) ); } } @@ -264,16 +271,21 @@ public void testFlush_WithQuantization() { IntStream.range(0, vectorsPerField.size()).forEach(i -> { try { - verify(knn990QuantWriterMockedConstruction.constructed().get(0)).writeState(i, quantizationState); - verify(nativeIndexWriter).flushIndex(expectedVectorValues.get(i), vectorsPerField.get(i).size()); + if (vectorsPerField.get(i).isEmpty()) { + verify(knn990QuantWriterMockedConstruction.constructed().get(0), never()).writeState(i, quantizationState); + verify(nativeIndexWriter, never()).flushIndex(expectedVectorValues.get(i), vectorsPerField.get(i).size()); + } else { + verify(knn990QuantWriterMockedConstruction.constructed().get(0)).writeState(i, quantizationState); + verify(nativeIndexWriter).flushIndex(expectedVectorValues.get(i), vectorsPerField.get(i).size()); + } } catch (Exception e) { throw new RuntimeException(e); } }); - + final Long expectedTimesGetVectorValuesIsCalled = vectorsPerField.stream().filter(Predicate.not(Map::isEmpty)).count(); knnVectorValuesFactoryMockedStatic.verify( () -> KNNVectorValuesFactory.getVectorValues(any(VectorDataType.class), any(DocsWithFieldSet.class), any()), - times(expectedVectorValues.size() * 2) + times(Math.toIntExact(expectedTimesGetVectorValuesIsCalled) * 2) ); } } From c271ef72c0bf89619874597afac23eff8128c35e Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Wed, 2 Oct 2024 11:26:22 -0700 Subject: [PATCH 391/416] Update release notes 2.17.1 (#2159) Signed-off-by: Naveen Tatikonda (cherry picked from commit fbec0aa6c6b8dd58b42ce87fe82e178913486c3f) Co-authored-by: Naveen Tatikonda --- release-notes/opensearch-knn.release-notes-2.17.1.0.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/release-notes/opensearch-knn.release-notes-2.17.1.0.md b/release-notes/opensearch-knn.release-notes-2.17.1.0.md index 0af275a5b..0d2083959 100644 --- a/release-notes/opensearch-knn.release-notes-2.17.1.0.md +++ b/release-notes/opensearch-knn.release-notes-2.17.1.0.md @@ -2,7 +2,6 @@ Compatible with OpenSearch 2.17.1 -### Enhancements -* Adds concurrent segment search support for mode auto [#2111](https://github.com/opensearch-project/k-NN/pull/2111) ### Bug Fixes +* Adds concurrent segment search support for mode auto [#2111](https://github.com/opensearch-project/k-NN/pull/2111) * Change min oversample to 1 [#2117](https://github.com/opensearch-project/k-NN/pull/2117) From eb802222e1c73b051cb4b1faac9874ce2d30b25c Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Wed, 2 Oct 2024 11:55:35 -0700 Subject: [PATCH 392/416] Added NMSLIB patched allowing load/write APIs with a stream object. (#2162) (cherry picked from commit eba9d98f8fbc7c107b289ea9019f42f12045217d) Signed-off-by: Dooyong Kim Signed-off-by: John Mazanec Co-authored-by: Doo Yong Kim <0ctopus13prime@gmail.com> Co-authored-by: Dooyong Kim --- jni/cmake/init-nmslib.cmake | 1 + ...is-using-stream-to-load-save-in-Hnsw.patch | 93 +++++++++++++++++++ 2 files changed, 94 insertions(+) create mode 100644 jni/patches/nmslib/0003-Adding-two-apis-using-stream-to-load-save-in-Hnsw.patch diff --git a/jni/cmake/init-nmslib.cmake b/jni/cmake/init-nmslib.cmake index b2c16f1fe..64df457c1 100644 --- a/jni/cmake/init-nmslib.cmake +++ b/jni/cmake/init-nmslib.cmake @@ -19,6 +19,7 @@ if(NOT DEFINED APPLY_LIB_PATCHES OR "${APPLY_LIB_PATCHES}" STREQUAL true) set(PATCH_FILE_LIST) list(APPEND PATCH_FILE_LIST "${CMAKE_CURRENT_SOURCE_DIR}/patches/nmslib/0001-Initialize-maxlevel-during-add-from-enterpoint-level.patch") list(APPEND PATCH_FILE_LIST "${CMAKE_CURRENT_SOURCE_DIR}/patches/nmslib/0002-Adds-ability-to-pass-ef-parameter-in-the-query-for-h.patch") + list(APPEND PATCH_FILE_LIST "${CMAKE_CURRENT_SOURCE_DIR}/patches/nmslib/0003-Adding-two-apis-using-stream-to-load-save-in-Hnsw.patch") # Get patch id of the last commit execute_process(COMMAND sh -c "git --no-pager show HEAD | git patch-id --stable" OUTPUT_VARIABLE PATCH_ID_OUTPUT_FROM_COMMIT WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/external/nmslib) diff --git a/jni/patches/nmslib/0003-Adding-two-apis-using-stream-to-load-save-in-Hnsw.patch b/jni/patches/nmslib/0003-Adding-two-apis-using-stream-to-load-save-in-Hnsw.patch new file mode 100644 index 000000000..bbba329b4 --- /dev/null +++ b/jni/patches/nmslib/0003-Adding-two-apis-using-stream-to-load-save-in-Hnsw.patch @@ -0,0 +1,93 @@ +From 7e099ec111e5c9db4b243da249c73f0ecc206281 Mon Sep 17 00:00:00 2001 +From: Dooyong Kim +Date: Thu, 26 Sep 2024 15:20:53 -0700 +Subject: [PATCH] Adding two apis using stream to load/save in Hnsw. + +Signed-off-by: Dooyong Kim +--- + similarity_search/include/method/hnsw.h | 4 +++ + similarity_search/src/method/hnsw.cc | 44 +++++++++++++++++++++++++ + 2 files changed, 48 insertions(+) + +diff --git a/similarity_search/include/method/hnsw.h b/similarity_search/include/method/hnsw.h +index 57d99d0..7ff3f3d 100644 +--- a/similarity_search/include/method/hnsw.h ++++ b/similarity_search/include/method/hnsw.h +@@ -455,8 +455,12 @@ namespace similarity { + public: + virtual void SaveIndex(const string &location) override; + ++ void SaveIndexWithStream(std::ostream& output); ++ + virtual void LoadIndex(const string &location) override; + ++ void LoadIndexWithStream(std::istream& in); ++ + Hnsw(bool PrintProgress, const Space &space, const ObjectVector &data); + void CreateIndex(const AnyParams &IndexParams) override; + +diff --git a/similarity_search/src/method/hnsw.cc b/similarity_search/src/method/hnsw.cc +index 35b372c..e7a2c9e 100644 +--- a/similarity_search/src/method/hnsw.cc ++++ b/similarity_search/src/method/hnsw.cc +@@ -771,6 +771,25 @@ namespace similarity { + output.close(); + } + ++ template ++ void Hnsw::SaveIndexWithStream(std::ostream &output) { ++ output.exceptions(ios::badbit | ios::failbit); ++ ++ unsigned int optimIndexFlag = data_level0_memory_ != nullptr; ++ ++ writeBinaryPOD(output, optimIndexFlag); ++ ++ if (!optimIndexFlag) { ++#if USE_TEXT_REGULAR_INDEX ++ SaveRegularIndexText(output); ++#else ++ SaveRegularIndexBin(output); ++#endif ++ } else { ++ SaveOptimizedIndex(output); ++ } ++ } ++ + template + void + Hnsw::SaveOptimizedIndex(std::ostream& output) { +@@ -1021,6 +1040,31 @@ namespace similarity { + + } + ++ template ++ void Hnsw::LoadIndexWithStream(std::istream& input) { ++ LOG(LIB_INFO) << "Loading index from an input stream."; ++ CHECK_MSG(input, "Cannot open file for reading with an input stream"); ++ ++ input.exceptions(ios::badbit | ios::failbit); ++ ++#if USE_TEXT_REGULAR_INDEX ++ LoadRegularIndexText(input); ++#else ++ unsigned int optimIndexFlag= 0; ++ ++ readBinaryPOD(input, optimIndexFlag); ++ ++ if (!optimIndexFlag) { ++ LoadRegularIndexBin(input); ++ } else { ++ LoadOptimizedIndex(input); ++ } ++#endif ++ ++ LOG(LIB_INFO) << "Finished loading index"; ++ visitedlistpool = new VisitedListPool(1, totalElementsStored_); ++ } ++ + + template + void +-- +2.39.5 (Apple Git-154) + From 642fdf49e537d1a8838b94eadf4a0b437917b6ae Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Thu, 3 Oct 2024 13:19:06 -0700 Subject: [PATCH 393/416] Introducing a loading layer in FAISS native engine. (#2139) (#2179) * Introducing a loading layer in FAISS native engine. Signed-off-by: Dooyong Kim * Update change log. Signed-off-by: Dooyong Kim * Added unit tests for Faiss stream support. Signed-off-by: Dooyong Kim * Fix a bug to pass a KB size integer value as a byte size integer parameter. Signed-off-by: Dooyong Kim * Fix a casting bugs when it tries to laod more than 4G sized index file. Signed-off-by: Dooyong Kim * Added unit tests for new methods in JNIService. Signed-off-by: Dooyong Kim * Fix formatting and removed nmslib_stream_support. Signed-off-by: Dooyong Kim * Removing redundant exception message in JNIService.loadIndex. Signed-off-by: Dooyong Kim * Fix a flaky testing - testIndexAllocation_closeBlocking Signed-off-by: Dooyong Kim --------- Signed-off-by: Dooyong Kim Signed-off-by: Doo Yong Kim <0ctopus13prime@gmail.com> Co-authored-by: Dooyong Kim (cherry picked from commit f3b2bd08e32dc12a08fb6917c018c6bbbfa4bd3f) Co-authored-by: Doo Yong Kim <0ctopus13prime@gmail.com> --- CHANGELOG.md | 1 + jni/CMakeLists.txt | 1 + jni/include/faiss_stream_support.h | 136 +++++++++++++ jni/include/faiss_wrapper.h | 10 + jni/include/jni_util.h | 102 ++++++---- .../org_opensearch_knn_jni_FaissService.h | 16 ++ jni/src/faiss_wrapper.cpp | 28 +++ jni/src/jni_util.cpp | 28 +++ .../org_opensearch_knn_jni_FaissService.cpp | 43 ++++ jni/tests/commons_test.cpp | 2 +- jni/tests/faiss_stream_support_test.cpp | 132 ++++++++++++ jni/tests/test_util.h | 8 + .../opensearch/knn/index/KNNIndexShard.java | 3 + .../index/memory/NativeMemoryAllocation.java | 34 ++-- .../memory/NativeMemoryEntryContext.java | 48 ++--- .../memory/NativeMemoryLoadStrategy.java | 62 +++++- .../opensearch/knn/index/query/KNNWeight.java | 1 + .../knn/index/store/IndexInputWithBuffer.java | 46 +++++ .../org/opensearch/knn/jni/FaissService.java | 19 ++ .../org/opensearch/knn/jni/JNIService.java | 23 +++ .../memory/NativeMemoryAllocationTests.java | 24 ++- .../memory/NativeMemoryEntryContextTests.java | 5 + .../memory/NativeMemoryLoadStrategyTests.java | 6 + .../opensearch/knn/jni/JNIServiceTests.java | 189 ++++++++++++++++++ 24 files changed, 864 insertions(+), 103 deletions(-) create mode 100644 jni/include/faiss_stream_support.h create mode 100644 jni/tests/faiss_stream_support_test.cpp create mode 100644 src/main/java/org/opensearch/knn/index/store/IndexInputWithBuffer.java diff --git a/CHANGELOG.md b/CHANGELOG.md index c11d258e8..0a54a242a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ## [Unreleased 3.0](https://github.com/opensearch-project/k-NN/compare/2.x...HEAD) ### Features ### Enhancements +* Introducing a loading layer in FAISS [#2033](https://github.com/opensearch-project/k-NN/issues/2033) ### Bug Fixes * Add DocValuesProducers for releasing memory when close index [#1946](https://github.com/opensearch-project/k-NN/pull/1946) ### Infrastructure diff --git a/jni/CMakeLists.txt b/jni/CMakeLists.txt index 3c071fc1f..4caa907b3 100644 --- a/jni/CMakeLists.txt +++ b/jni/CMakeLists.txt @@ -154,6 +154,7 @@ if ("${WIN32}" STREQUAL "") tests/nmslib_wrapper_unit_test.cpp tests/test_util.cpp tests/commons_test.cpp + tests/faiss_stream_support_test.cpp tests/faiss_index_service_test.cpp ) diff --git a/jni/include/faiss_stream_support.h b/jni/include/faiss_stream_support.h new file mode 100644 index 000000000..65f1631d4 --- /dev/null +++ b/jni/include/faiss_stream_support.h @@ -0,0 +1,136 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +#ifndef OPENSEARCH_KNN_JNI_STREAM_SUPPORT_H +#define OPENSEARCH_KNN_JNI_STREAM_SUPPORT_H + +#include "faiss/impl/io.h" +#include "jni_util.h" + +#include +#include +#include +#include + +namespace knn_jni { +namespace stream { + +/** + * This class contains Java IndexInputWithBuffer reference and calls its API to copy required bytes into a read buffer. + */ + +class NativeEngineIndexInputMediator { + public: + // Expect IndexInputWithBuffer is given as `_indexInput`. + NativeEngineIndexInputMediator(JNIUtilInterface *_jni_interface, + JNIEnv *_env, + jobject _indexInput) + : jni_interface(_jni_interface), + env(_env), + indexInput(_indexInput), + bufferArray((jbyteArray) (_jni_interface->GetObjectField(_env, + _indexInput, + getBufferFieldId(_jni_interface, _env)))), + copyBytesMethod(getCopyBytesMethod(_jni_interface, _env)) { + } + + void copyBytes(int64_t nbytes, uint8_t *destination) { + while (nbytes > 0) { + // Call `copyBytes` to read bytes as many as possible. + const auto readBytes = + jni_interface->CallIntMethodLong(env, indexInput, copyBytesMethod, nbytes); + + // === Critical Section Start === + + // Get primitive array pointer, no copy is happening in OpenJDK. + auto primitiveArray = + (jbyte *) jni_interface->GetPrimitiveArrayCritical(env, bufferArray, nullptr); + + // Copy Java bytes to C++ destination address. + std::memcpy(destination, primitiveArray, readBytes); + + // Release the acquired primitive array pointer. + // JNI_ABORT tells JVM to directly free memory without copying back to Java byte[]. + // Since we're merely copying data, we don't need to copying back. + jni_interface->ReleasePrimitiveArrayCritical(env, bufferArray, primitiveArray, JNI_ABORT); + + // === Critical Section End === + + destination += readBytes; + nbytes -= readBytes; + } // End while + } + + private: + static jclass getIndexInputWithBufferClass(JNIUtilInterface *jni_interface, JNIEnv *env) { + static jclass INDEX_INPUT_WITH_BUFFER_CLASS = + jni_interface->FindClassFromJNIEnv(env, "org/opensearch/knn/index/store/IndexInputWithBuffer"); + return INDEX_INPUT_WITH_BUFFER_CLASS; + } + + static jmethodID getCopyBytesMethod(JNIUtilInterface *jni_interface, JNIEnv *env) { + static jmethodID COPY_METHOD_ID = + jni_interface->GetMethodID(env, getIndexInputWithBufferClass(jni_interface, env), "copyBytes", "(J)I"); + return COPY_METHOD_ID; + } + + static jfieldID getBufferFieldId(JNIUtilInterface *jni_interface, JNIEnv *env) { + static jfieldID BUFFER_FIELD_ID = + jni_interface->GetFieldID(env, getIndexInputWithBufferClass(jni_interface, env), "buffer", "[B"); + return BUFFER_FIELD_ID; + } + + JNIUtilInterface *jni_interface; + JNIEnv *env; + + // `IndexInputWithBuffer` instance having `IndexInput` instance obtained from `Directory` for reading. + jobject indexInput; + jbyteArray bufferArray; + jmethodID copyBytesMethod; +}; // class NativeEngineIndexInputMediator + + + +/** + * A glue component inheriting IOReader to be passed down to Faiss library. + * This will then indirectly call the mediator component and eventually read required bytes from Lucene's IndexInput. + */ +class FaissOpenSearchIOReader final : public faiss::IOReader { + public: + explicit FaissOpenSearchIOReader(NativeEngineIndexInputMediator *_mediator) + : faiss::IOReader(), + mediator(_mediator) { + name = "FaissOpenSearchIOReader"; + } + + size_t operator()(void *ptr, size_t size, size_t nitems) final { + const auto readBytes = size * nitems; + if (readBytes > 0) { + // Mediator calls IndexInput, then copy read bytes to `ptr`. + mediator->copyBytes(readBytes, (uint8_t *) ptr); + } + return nitems; + } + + int filedescriptor() final { + throw std::runtime_error("filedescriptor() is not supported in FaissOpenSearchIOReader."); + } + + private: + NativeEngineIndexInputMediator *mediator; +}; // class FaissOpenSearchIOReader + + + +} +} + +#endif //OPENSEARCH_KNN_JNI_STREAM_SUPPORT_H diff --git a/jni/include/faiss_wrapper.h b/jni/include/faiss_wrapper.h index d6375653d..8ffce4ad1 100644 --- a/jni/include/faiss_wrapper.h +++ b/jni/include/faiss_wrapper.h @@ -47,11 +47,21 @@ namespace knn_jni { // Return a pointer to the loaded index jlong LoadIndex(knn_jni::JNIUtilInterface * jniUtil, JNIEnv * env, jstring indexPathJ); + // Loads an index with a reader implemented IOReader + // + // Returns a pointer of the loaded index + jlong LoadIndexWithStream(faiss::IOReader* ioReader); + // Load a binary index from indexPathJ into memory. // // Return a pointer to the loaded index jlong LoadBinaryIndex(knn_jni::JNIUtilInterface * jniUtil, JNIEnv * env, jstring indexPathJ); + // Loads a binary index with a reader implemented IOReader + // + // Returns a pointer of the loaded index + jlong LoadBinaryIndexWithStream(faiss::IOReader* ioReader); + // Check if a loaded index requires shared state bool IsSharedIndexStateRequired(jlong indexPointerJ); diff --git a/jni/include/jni_util.h b/jni/include/jni_util.h index 825471a3c..6b1b926e7 100644 --- a/jni/include/jni_util.h +++ b/jni/include/jni_util.h @@ -22,8 +22,7 @@ namespace knn_jni { // Interface for making calls to JNI - class JNIUtilInterface { - public: + struct JNIUtilInterface { // -------------------------- EXCEPTION HANDLING ---------------------------- // Takes the name of a Java exception type and a message and throws the corresponding exception // to the JVM @@ -127,56 +126,77 @@ namespace knn_jni { virtual void SetByteArrayRegion(JNIEnv *env, jbyteArray array, jsize start, jsize len, const jbyte * buf) = 0; + virtual jobject GetObjectField(JNIEnv * env, jobject obj, jfieldID fieldID) = 0; + + virtual jclass FindClassFromJNIEnv(JNIEnv * env, const char *name) = 0; + + virtual jmethodID GetMethodID(JNIEnv * env, jclass clazz, const char *name, const char *sig) = 0; + + virtual jfieldID GetFieldID(JNIEnv * env, jclass clazz, const char *name, const char *sig) = 0; + + virtual void * GetPrimitiveArrayCritical(JNIEnv * env, jarray array, jboolean *isCopy) = 0; + + virtual void ReleasePrimitiveArrayCritical(JNIEnv * env, jarray array, void *carray, jint mode) = 0; + + virtual jint CallIntMethodLong(JNIEnv * env, jobject obj, jmethodID methodID, int64_t longArg) = 0; + // -------------------------------------------------------------------------- }; jobject GetJObjectFromMapOrThrow(std::unordered_map map, std::string key); // Class that implements JNIUtilInterface methods - class JNIUtil: public JNIUtilInterface { + class JNIUtil final : public JNIUtilInterface { public: // Initialize and Uninitialize methods are used for caching/cleaning up Java classes and methods void Initialize(JNIEnv* env); void Uninitialize(JNIEnv* env); - void ThrowJavaException(JNIEnv* env, const char* type = "", const char* message = ""); - void HasExceptionInStack(JNIEnv* env); - void HasExceptionInStack(JNIEnv* env, const std::string& message); - void CatchCppExceptionAndThrowJava(JNIEnv* env); - jclass FindClass(JNIEnv * env, const std::string& className); - jmethodID FindMethod(JNIEnv * env, const std::string& className, const std::string& methodName); - std::string ConvertJavaStringToCppString(JNIEnv * env, jstring javaString); - std::unordered_map ConvertJavaMapToCppMap(JNIEnv *env, jobject parametersJ); - std::string ConvertJavaObjectToCppString(JNIEnv *env, jobject objectJ); - int ConvertJavaObjectToCppInteger(JNIEnv *env, jobject objectJ); - std::vector Convert2dJavaObjectArrayToCppFloatVector(JNIEnv *env, jobjectArray array2dJ, int dim); - std::vector ConvertJavaIntArrayToCppIntVector(JNIEnv *env, jintArray arrayJ); - int GetInnerDimensionOf2dJavaFloatArray(JNIEnv *env, jobjectArray array2dJ); - int GetInnerDimensionOf2dJavaByteArray(JNIEnv *env, jobjectArray array2dJ); - int GetJavaObjectArrayLength(JNIEnv *env, jobjectArray arrayJ); - int GetJavaIntArrayLength(JNIEnv *env, jintArray arrayJ); - int GetJavaLongArrayLength(JNIEnv *env, jlongArray arrayJ); - int GetJavaBytesArrayLength(JNIEnv *env, jbyteArray arrayJ); - int GetJavaFloatArrayLength(JNIEnv *env, jfloatArray arrayJ); - - void DeleteLocalRef(JNIEnv *env, jobject obj); - jbyte * GetByteArrayElements(JNIEnv *env, jbyteArray array, jboolean * isCopy); - jfloat * GetFloatArrayElements(JNIEnv *env, jfloatArray array, jboolean * isCopy); - jint * GetIntArrayElements(JNIEnv *env, jintArray array, jboolean * isCopy); - jlong * GetLongArrayElements(JNIEnv *env, jlongArray array, jboolean * isCopy); - jobject GetObjectArrayElement(JNIEnv *env, jobjectArray array, jsize index); - jobject NewObject(JNIEnv *env, jclass clazz, jmethodID methodId, int id, float distance); - jobjectArray NewObjectArray(JNIEnv *env, jsize len, jclass clazz, jobject init); - jbyteArray NewByteArray(JNIEnv *env, jsize len); - void ReleaseByteArrayElements(JNIEnv *env, jbyteArray array, jbyte *elems, int mode); - void ReleaseFloatArrayElements(JNIEnv *env, jfloatArray array, jfloat *elems, int mode); - void ReleaseIntArrayElements(JNIEnv *env, jintArray array, jint *elems, jint mode); - void ReleaseLongArrayElements(JNIEnv *env, jlongArray array, jlong *elems, jint mode); - void SetObjectArrayElement(JNIEnv *env, jobjectArray array, jsize index, jobject val); - void SetByteArrayRegion(JNIEnv *env, jbyteArray array, jsize start, jsize len, const jbyte * buf); - void Convert2dJavaObjectArrayAndStoreToFloatVector(JNIEnv *env, jobjectArray array2dJ, int dim, std::vector *vect); - void Convert2dJavaObjectArrayAndStoreToBinaryVector(JNIEnv *env, jobjectArray array2dJ, int dim, std::vector *vect); - void Convert2dJavaObjectArrayAndStoreToByteVector(JNIEnv *env, jobjectArray array2dJ, int dim, std::vector *vect); + void ThrowJavaException(JNIEnv* env, const char* type = "", const char* message = "") final; + void HasExceptionInStack(JNIEnv* env) final; + void HasExceptionInStack(JNIEnv* env, const std::string& message) final; + void CatchCppExceptionAndThrowJava(JNIEnv* env) final; + jclass FindClass(JNIEnv * env, const std::string& className) final; + jmethodID FindMethod(JNIEnv * env, const std::string& className, const std::string& methodName) final; + std::string ConvertJavaStringToCppString(JNIEnv * env, jstring javaString) final; + std::unordered_map ConvertJavaMapToCppMap(JNIEnv *env, jobject parametersJ) final; + std::string ConvertJavaObjectToCppString(JNIEnv *env, jobject objectJ) final; + int ConvertJavaObjectToCppInteger(JNIEnv *env, jobject objectJ) final; + std::vector Convert2dJavaObjectArrayToCppFloatVector(JNIEnv *env, jobjectArray array2dJ, int dim) final; + std::vector ConvertJavaIntArrayToCppIntVector(JNIEnv *env, jintArray arrayJ) final; + int GetInnerDimensionOf2dJavaFloatArray(JNIEnv *env, jobjectArray array2dJ) final; + int GetInnerDimensionOf2dJavaByteArray(JNIEnv *env, jobjectArray array2dJ) final; + int GetJavaObjectArrayLength(JNIEnv *env, jobjectArray arrayJ) final; + int GetJavaIntArrayLength(JNIEnv *env, jintArray arrayJ) final; + int GetJavaLongArrayLength(JNIEnv *env, jlongArray arrayJ) final; + int GetJavaBytesArrayLength(JNIEnv *env, jbyteArray arrayJ) final; + int GetJavaFloatArrayLength(JNIEnv *env, jfloatArray arrayJ) final; + + void DeleteLocalRef(JNIEnv *env, jobject obj) final; + jbyte * GetByteArrayElements(JNIEnv *env, jbyteArray array, jboolean * isCopy) final; + jfloat * GetFloatArrayElements(JNIEnv *env, jfloatArray array, jboolean * isCopy) final; + jint * GetIntArrayElements(JNIEnv *env, jintArray array, jboolean * isCopy) final; + jlong * GetLongArrayElements(JNIEnv *env, jlongArray array, jboolean * isCopy) final; + jobject GetObjectArrayElement(JNIEnv *env, jobjectArray array, jsize index) final; + jobject NewObject(JNIEnv *env, jclass clazz, jmethodID methodId, int id, float distance) final; + jobjectArray NewObjectArray(JNIEnv *env, jsize len, jclass clazz, jobject init) final; + jbyteArray NewByteArray(JNIEnv *env, jsize len) final; + void ReleaseByteArrayElements(JNIEnv *env, jbyteArray array, jbyte *elems, int mode) final; + void ReleaseFloatArrayElements(JNIEnv *env, jfloatArray array, jfloat *elems, int mode) final; + void ReleaseIntArrayElements(JNIEnv *env, jintArray array, jint *elems, jint mode) final; + void ReleaseLongArrayElements(JNIEnv *env, jlongArray array, jlong *elems, jint mode) final; + void SetObjectArrayElement(JNIEnv *env, jobjectArray array, jsize index, jobject val) final; + void SetByteArrayRegion(JNIEnv *env, jbyteArray array, jsize start, jsize len, const jbyte * buf) final; + void Convert2dJavaObjectArrayAndStoreToFloatVector(JNIEnv *env, jobjectArray array2dJ, int dim, std::vector *vect) final; + void Convert2dJavaObjectArrayAndStoreToBinaryVector(JNIEnv *env, jobjectArray array2dJ, int dim, std::vector *vect) final; + void Convert2dJavaObjectArrayAndStoreToByteVector(JNIEnv *env, jobjectArray array2dJ, int dim, std::vector *vect) final; + jobject GetObjectField(JNIEnv * env, jobject obj, jfieldID fieldID) final; + jclass FindClassFromJNIEnv(JNIEnv * env, const char *name) final; + jmethodID GetMethodID(JNIEnv * env, jclass clazz, const char *name, const char *sig) final; + jfieldID GetFieldID(JNIEnv * env, jclass clazz, const char *name, const char *sig) final; + jint CallIntMethodLong(JNIEnv * env, jobject obj, jmethodID methodID, int64_t longArg) final; + void * GetPrimitiveArrayCritical(JNIEnv * env, jarray array, jboolean *isCopy) final; + void ReleasePrimitiveArrayCritical(JNIEnv * env, jarray array, void *carray, jint mode) final; private: std::unordered_map cachedClasses; diff --git a/jni/include/org_opensearch_knn_jni_FaissService.h b/jni/include/org_opensearch_knn_jni_FaissService.h index d42ce197c..2969df3ae 100644 --- a/jni/include/org_opensearch_knn_jni_FaissService.h +++ b/jni/include/org_opensearch_knn_jni_FaissService.h @@ -128,6 +128,14 @@ JNIEXPORT void JNICALL Java_org_opensearch_knn_jni_FaissService_createIndexFromT JNIEXPORT jlong JNICALL Java_org_opensearch_knn_jni_FaissService_loadIndex (JNIEnv *, jclass, jstring); +/* + * Class: org_opensearch_knn_jni_FaissService + * Method: loadIndexWithStream + * Signature: (Lorg/opensearch/knn/index/util/IndexInputWithBuffer;)J + */ +JNIEXPORT jlong JNICALL Java_org_opensearch_knn_jni_FaissService_loadIndexWithStream + (JNIEnv *, jclass, jobject); + /* * Class: org_opensearch_knn_jni_FaissService * Method: loadBinaryIndex @@ -136,6 +144,14 @@ JNIEXPORT jlong JNICALL Java_org_opensearch_knn_jni_FaissService_loadIndex JNIEXPORT jlong JNICALL Java_org_opensearch_knn_jni_FaissService_loadBinaryIndex (JNIEnv *, jclass, jstring); +/* + * Class: org_opensearch_knn_jni_FaissService + * Method: loadBinaryIndexWithStream + * Signature: (Lorg/opensearch/knn/index/util/IndexInputWithBuffer;)J + */ +JNIEXPORT jlong JNICALL Java_org_opensearch_knn_jni_FaissService_loadBinaryIndexWithStream + (JNIEnv *, jclass, jobject); + /* * Class: org_opensearch_knn_jni_FaissService * Method: isSharedIndexStateRequired diff --git a/jni/src/faiss_wrapper.cpp b/jni/src/faiss_wrapper.cpp index 45548e0f7..d1c7648dc 100644 --- a/jni/src/faiss_wrapper.cpp +++ b/jni/src/faiss_wrapper.cpp @@ -423,6 +423,20 @@ jlong knn_jni::faiss_wrapper::LoadIndex(knn_jni::JNIUtilInterface * jniUtil, JNI return (jlong) indexReader; } +jlong knn_jni::faiss_wrapper::LoadIndexWithStream(faiss::IOReader* ioReader) { + if (ioReader == nullptr) [[unlikely]] { + throw std::runtime_error("IOReader cannot be null"); + } + + faiss::Index* indexReader = + faiss::read_index(ioReader, + faiss::IO_FLAG_READ_ONLY + | faiss::IO_FLAG_PQ_SKIP_SDC_TABLE + | faiss::IO_FLAG_SKIP_PRECOMPUTE_TABLE); + + return (jlong) indexReader; +} + jlong knn_jni::faiss_wrapper::LoadBinaryIndex(knn_jni::JNIUtilInterface * jniUtil, JNIEnv * env, jstring indexPathJ) { if (indexPathJ == nullptr) { throw std::runtime_error("Index path cannot be null"); @@ -436,6 +450,20 @@ jlong knn_jni::faiss_wrapper::LoadBinaryIndex(knn_jni::JNIUtilInterface * jniUti return (jlong) indexReader; } +jlong knn_jni::faiss_wrapper::LoadBinaryIndexWithStream(faiss::IOReader* ioReader) { + if (ioReader == nullptr) [[unlikely]] { + throw std::runtime_error("IOReader cannot be null"); + } + + faiss::IndexBinary* indexReader = + faiss::read_index_binary(ioReader, + faiss::IO_FLAG_READ_ONLY + | faiss::IO_FLAG_PQ_SKIP_SDC_TABLE + | faiss::IO_FLAG_SKIP_PRECOMPUTE_TABLE); + + return (jlong) indexReader; +} + bool knn_jni::faiss_wrapper::IsSharedIndexStateRequired(jlong indexPointerJ) { auto * index = reinterpret_cast(indexPointerJ); return isIndexIVFPQL2(index); diff --git a/jni/src/jni_util.cpp b/jni/src/jni_util.cpp index 82900b5ce..3eaf3b0a1 100644 --- a/jni/src/jni_util.cpp +++ b/jni/src/jni_util.cpp @@ -547,6 +547,34 @@ void knn_jni::JNIUtil::SetByteArrayRegion(JNIEnv *env, jbyteArray array, jsize s this->HasExceptionInStack(env, "Unable to set byte array region"); } +jobject knn_jni::JNIUtil::GetObjectField(JNIEnv * env, jobject obj, jfieldID fieldID) { + return env->GetObjectField(obj, fieldID); +} + +jclass knn_jni::JNIUtil::FindClassFromJNIEnv(JNIEnv * env, const char *name) { + return env->FindClass(name); +} + +jmethodID knn_jni::JNIUtil::GetMethodID(JNIEnv * env, jclass clazz, const char *name, const char *sig) { + return env->GetMethodID(clazz, name, sig); +} + +jfieldID knn_jni::JNIUtil::GetFieldID(JNIEnv * env, jclass clazz, const char *name, const char *sig) { + return env->GetFieldID(clazz, name, sig); +} + +jint knn_jni::JNIUtil::CallIntMethodLong(JNIEnv * env, jobject obj, jmethodID methodID, int64_t longArg) { + return env->CallIntMethod(obj, methodID, longArg); +} + +void * knn_jni::JNIUtil::GetPrimitiveArrayCritical(JNIEnv * env, jarray array, jboolean *isCopy) { + return env->GetPrimitiveArrayCritical(array, isCopy); +} + +void knn_jni::JNIUtil::ReleasePrimitiveArrayCritical(JNIEnv * env, jarray array, void *carray, jint mode) { + return env->ReleasePrimitiveArrayCritical(array, carray, mode); +} + jobject knn_jni::GetJObjectFromMapOrThrow(std::unordered_map map, std::string key) { if(map.find(key) == map.end()) { throw std::runtime_error(key + " not found"); diff --git a/jni/src/org_opensearch_knn_jni_FaissService.cpp b/jni/src/org_opensearch_knn_jni_FaissService.cpp index 70c986b7d..7326c7ba0 100644 --- a/jni/src/org_opensearch_knn_jni_FaissService.cpp +++ b/jni/src/org_opensearch_knn_jni_FaissService.cpp @@ -17,6 +17,7 @@ #include "faiss_wrapper.h" #include "jni_util.h" +#include "faiss_stream_support.h" static knn_jni::JNIUtil jniUtil; static const jint KNN_FAISS_JNI_VERSION = JNI_VERSION_1_1; @@ -217,6 +218,27 @@ JNIEXPORT jlong JNICALL Java_org_opensearch_knn_jni_FaissService_loadIndex(JNIEn return NULL; } +JNIEXPORT jlong JNICALL Java_org_opensearch_knn_jni_FaissService_loadIndexWithStream + (JNIEnv * env, jclass cls, jobject readStream) +{ + try { + // Create a mediator locally. + // Note that `indexInput` is `IndexInputWithBuffer` type. + knn_jni::stream::NativeEngineIndexInputMediator mediator {&jniUtil, env, readStream}; + + // Wrap the mediator with a glue code inheriting IOReader. + knn_jni::stream::FaissOpenSearchIOReader faissOpenSearchIOReader {&mediator}; + + // Pass IOReader to Faiss for loading vector index. + return knn_jni::faiss_wrapper::LoadIndexWithStream( + &faissOpenSearchIOReader); + } catch (...) { + jniUtil.CatchCppExceptionAndThrowJava(env); + } + + return NULL; +} + JNIEXPORT jlong JNICALL Java_org_opensearch_knn_jni_FaissService_loadBinaryIndex(JNIEnv * env, jclass cls, jstring indexPathJ) { try { @@ -227,6 +249,27 @@ JNIEXPORT jlong JNICALL Java_org_opensearch_knn_jni_FaissService_loadBinaryIndex return NULL; } +JNIEXPORT jlong JNICALL Java_org_opensearch_knn_jni_FaissService_loadBinaryIndexWithStream + (JNIEnv * env, jclass cls, jobject readStream) +{ + try { + // Create a mediator locally. + // Note that `indexInput` is `IndexInputWithBuffer` type. + knn_jni::stream::NativeEngineIndexInputMediator mediator {&jniUtil, env, readStream}; + + // Wrap the mediator with a glue code inheriting IOReader. + knn_jni::stream::FaissOpenSearchIOReader faissOpenSearchIOReader {&mediator}; + + // Pass IOReader to Faiss for loading vector index. + return knn_jni::faiss_wrapper::LoadBinaryIndexWithStream( + &faissOpenSearchIOReader); + } catch (...) { + jniUtil.CatchCppExceptionAndThrowJava(env); + } + + return NULL; +} + JNIEXPORT jboolean JNICALL Java_org_opensearch_knn_jni_FaissService_isSharedIndexStateRequired (JNIEnv * env, jclass cls, jlong indexPointerJ) { diff --git a/jni/tests/commons_test.cpp b/jni/tests/commons_test.cpp index d469fe268..39d7f3c99 100644 --- a/jni/tests/commons_test.cpp +++ b/jni/tests/commons_test.cpp @@ -179,7 +179,7 @@ TEST(StoreByteVectorTest, BasicAssertions) { } // Check that freeing vector data works - knn_jni::commons::freeVectorData(memoryAddress); + knn_jni::commons::freeBinaryVectorData(memoryAddress); } TEST(CommonTests, GetIntegerMethodParam) { diff --git a/jni/tests/faiss_stream_support_test.cpp b/jni/tests/faiss_stream_support_test.cpp new file mode 100644 index 000000000..4045985bb --- /dev/null +++ b/jni/tests/faiss_stream_support_test.cpp @@ -0,0 +1,132 @@ +// SPDX-License-Identifier: Apache-2.0 +// +// The OpenSearch Contributors require contributions made to +// this file be licensed under the Apache-2.0 license or a +// compatible open source license. +// +// Modifications Copyright OpenSearch Contributors. See +// GitHub history for details. + +#include "faiss_stream_support.h" +#include +#include "test_util.h" +#include +#include +#include +#include + +using ::testing::Return; +using knn_jni::stream::FaissOpenSearchIOReader; +using knn_jni::stream::NativeEngineIndexInputMediator; +using test_util::MockJNIUtil; + +// Mocking IndexInputWithBuffer. +struct JavaIndexInputMock { + JavaIndexInputMock(std::string _readTargetBytes, int32_t _bufSize) + : readTargetBytes(std::move(_readTargetBytes)), + nextReadIdx(), + buffer(_bufSize) { + } + + // This method is simulating `copyBytes` in IndexInputWithBuffer. + int32_t simulateCopyReads(int64_t readBytes) { + readBytes = std::min(readBytes, (int64_t) buffer.size()); + readBytes = std::min(readBytes, (int64_t) (readTargetBytes.size() - nextReadIdx)); + std::memcpy(buffer.data(), readTargetBytes.data() + nextReadIdx, readBytes); + nextReadIdx += readBytes; + return (int32_t) readBytes; + } + + static std::string makeRandomBytes(int32_t bytesSize) { + // Define the list of possible characters + static const string CHARACTERS + = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuv" + "wxyz0123456789"; + + // Create a random number generator + std::random_device rd; + std::mt19937 generator(rd()); + + // Create a distribution to uniformly select from all characters + std::uniform_int_distribution<> distribution( + 0, CHARACTERS.size() - 1); + + // Pre-allocate the string with the desired length + std::string randomString(bytesSize, '\0'); + + // Use generate_n with a back_inserter iterator + std::generate_n(randomString.begin(), bytesSize, [&]() { + return CHARACTERS[distribution(generator)]; + }); + + return randomString; + } + + std::string readTargetBytes; + int64_t nextReadIdx; + std::vector buffer; +}; // struct JavaIndexInputMock + +void setUpMockJNIUtil(JavaIndexInputMock &javaIndexInputMock, MockJNIUtil &mockJni) { + // Set up mocking values + mocking behavior in a method. + ON_CALL(mockJni, FindClassFromJNIEnv).WillByDefault(Return((jclass) 1)); + ON_CALL(mockJni, GetMethodID).WillByDefault(Return((jmethodID) 1)); + ON_CALL(mockJni, GetFieldID).WillByDefault(Return((jfieldID) 1)); + ON_CALL(mockJni, GetObjectField).WillByDefault(Return((jobject) 1)); + ON_CALL(mockJni, CallIntMethodLong).WillByDefault([&javaIndexInputMock](JNIEnv *env, + jobject obj, + jmethodID methodID, + int64_t longArg) { + return javaIndexInputMock.simulateCopyReads(longArg); + }); + ON_CALL(mockJni, GetPrimitiveArrayCritical).WillByDefault([&javaIndexInputMock](JNIEnv *env, + jarray array, + jboolean *isCopy) { + return (jbyte *) javaIndexInputMock.buffer.data(); + }); + ON_CALL(mockJni, ReleasePrimitiveArrayCritical).WillByDefault(Return()); +} + +TEST(FaissStreamSupportTest, NativeEngineIndexInputMediatorCopyWhenEmpty) { + for (auto contentSize : std::vector{0, 2222, 7777, 1024, 77, 1}) { + // Set up mockings + MockJNIUtil mockJni; + JavaIndexInputMock javaIndexInputMock{ + JavaIndexInputMock::makeRandomBytes(contentSize), 1024}; + setUpMockJNIUtil(javaIndexInputMock, mockJni); + + // Prepare copying + NativeEngineIndexInputMediator mediator{&mockJni, nullptr, nullptr}; + std::string readBuffer(javaIndexInputMock.readTargetBytes.size(), '\0'); + + // Call copyBytes + mediator.copyBytes((int32_t) javaIndexInputMock.readTargetBytes.size(), (uint8_t *) readBuffer.data()); + + // Expected that we acquired the same contents as readTargetBytes + ASSERT_EQ(javaIndexInputMock.readTargetBytes, readBuffer); + } // End for +} + +TEST(FaissStreamSupportTest, FaissOpenSearchIOReaderCopy) { + for (auto contentSize : std::vector{0, 2222, 7777, 1024, 77, 1}) { + // Set up mockings + MockJNIUtil mockJni; + JavaIndexInputMock javaIndexInputMock{ + JavaIndexInputMock::makeRandomBytes(contentSize), 1024}; + setUpMockJNIUtil(javaIndexInputMock, mockJni); + + // Prepare copying + NativeEngineIndexInputMediator mediator{&mockJni, nullptr, nullptr}; + std::string readBuffer; + readBuffer.resize(javaIndexInputMock.readTargetBytes.size()); + FaissOpenSearchIOReader ioReader{&mediator}; + + // Read bytes + const auto readBytes = + ioReader((void *) readBuffer.data(), 1, javaIndexInputMock.readTargetBytes.size()); + + // Expected that we acquired the same contents as readTargetBytes + ASSERT_EQ(javaIndexInputMock.readTargetBytes.size(), readBytes); + ASSERT_EQ(javaIndexInputMock.readTargetBytes, readBuffer); + } // End for +} diff --git a/jni/tests/test_util.h b/jni/tests/test_util.h index ea02da6f2..286000c08 100644 --- a/jni/tests/test_util.h +++ b/jni/tests/test_util.h @@ -106,6 +106,14 @@ namespace test_util { (JNIEnv * env, jobjectArray array, jsize index, jobject val)); MOCK_METHOD(void, ThrowJavaException, (JNIEnv * env, const char* type, const char* message)); + MOCK_METHOD(jobject, GetObjectField, + (JNIEnv * env, jobject obj, jfieldID fieldID)); + MOCK_METHOD(jclass, FindClassFromJNIEnv, (JNIEnv * env, const char *name)); + MOCK_METHOD(jmethodID, GetMethodID, (JNIEnv * env, jclass clazz, const char *name, const char *sig)); + MOCK_METHOD(jfieldID, GetFieldID, (JNIEnv * env, jclass clazz, const char *name, const char *sig)); + MOCK_METHOD(jint, CallIntMethodLong, (JNIEnv * env, jobject obj, jmethodID methodID, int64_t longArg)); + MOCK_METHOD(void *, GetPrimitiveArrayCritical, (JNIEnv * env, jarray array, jboolean *isCopy)); + MOCK_METHOD(void, ReleasePrimitiveArrayCritical, (JNIEnv * env, jarray array, void *carray, jint mode)); }; // For our unit tests, we want to ensure that each test tests one function in diff --git a/src/main/java/org/opensearch/knn/index/KNNIndexShard.java b/src/main/java/org/opensearch/knn/index/KNNIndexShard.java index cc022b310..a4d31c546 100644 --- a/src/main/java/org/opensearch/knn/index/KNNIndexShard.java +++ b/src/main/java/org/opensearch/knn/index/KNNIndexShard.java @@ -13,6 +13,7 @@ import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.SegmentReader; +import org.apache.lucene.store.Directory; import org.apache.lucene.store.FSDirectory; import org.apache.lucene.store.FilterDirectory; import org.opensearch.common.lucene.Lucene; @@ -89,11 +90,13 @@ public String getIndexName() { */ public void warmup() throws IOException { log.info("[KNN] Warming up index: [{}]", getIndexName()); + final Directory directory = indexShard.store().directory(); try (Engine.Searcher searcher = indexShard.acquireSearcher("knn-warmup")) { getAllEngineFileContexts(searcher.getIndexReader()).forEach((engineFileContext) -> { try { nativeMemoryCacheManager.get( new NativeMemoryEntryContext.IndexEntryContext( + directory, engineFileContext.getIndexPath(), NativeMemoryLoadStrategy.IndexLoadStrategy.getInstance(), getParametersAtLoading( diff --git a/src/main/java/org/opensearch/knn/index/memory/NativeMemoryAllocation.java b/src/main/java/org/opensearch/knn/index/memory/NativeMemoryAllocation.java index 635bc3883..8adf35447 100644 --- a/src/main/java/org/opensearch/knn/index/memory/NativeMemoryAllocation.java +++ b/src/main/java/org/opensearch/knn/index/memory/NativeMemoryAllocation.java @@ -110,7 +110,7 @@ class IndexAllocation implements NativeMemoryAllocation { private final ExecutorService executor; private final long memoryAddress; - private final int size; + private final int sizeKb; private volatile boolean closed; @Getter private final KNNEngine knnEngine; @@ -130,7 +130,7 @@ class IndexAllocation implements NativeMemoryAllocation { * * @param executorService Executor service used to close the allocation * @param memoryAddress Pointer in memory to the index - * @param size Size this index consumes in kilobytes + * @param sizeKb Size this index consumes in kilobytes * @param knnEngine KNNEngine associated with the index allocation * @param indexPath File path to index * @param openSearchIndexName Name of OpenSearch index this index is associated with @@ -139,13 +139,13 @@ class IndexAllocation implements NativeMemoryAllocation { IndexAllocation( ExecutorService executorService, long memoryAddress, - int size, + int sizeKb, KNNEngine knnEngine, String indexPath, String openSearchIndexName, WatcherHandle watcherHandle ) { - this(executorService, memoryAddress, size, knnEngine, indexPath, openSearchIndexName, watcherHandle, null, false); + this(executorService, memoryAddress, sizeKb, knnEngine, indexPath, openSearchIndexName, watcherHandle, null, false); } /** @@ -153,7 +153,7 @@ class IndexAllocation implements NativeMemoryAllocation { * * @param executorService Executor service used to close the allocation * @param memoryAddress Pointer in memory to the index - * @param size Size this index consumes in kilobytes + * @param sizeKb Size this index consumes in kilobytes * @param knnEngine KNNEngine associated with the index allocation * @param indexPath File path to index * @param openSearchIndexName Name of OpenSearch index this index is associated with @@ -163,7 +163,7 @@ class IndexAllocation implements NativeMemoryAllocation { IndexAllocation( ExecutorService executorService, long memoryAddress, - int size, + int sizeKb, KNNEngine knnEngine, String indexPath, String openSearchIndexName, @@ -178,7 +178,7 @@ class IndexAllocation implements NativeMemoryAllocation { this.openSearchIndexName = openSearchIndexName; this.memoryAddress = memoryAddress; this.readWriteLock = new ReentrantReadWriteLock(); - this.size = size; + this.sizeKb = sizeKb; this.watcherHandle = watcherHandle; this.sharedIndexState = sharedIndexState; this.isBinaryIndex = isBinaryIndex; @@ -187,9 +187,12 @@ class IndexAllocation implements NativeMemoryAllocation { protected void closeInternal() { Runnable onClose = () -> { - writeLock(); - cleanup(); - writeUnlock(); + try { + writeLock(); + cleanup(); + } finally { + writeUnlock(); + } }; // The close operation needs to be blocking to prevent overflow @@ -269,7 +272,7 @@ public void writeUnlock() { @Override public int getSizeInKB() { - return size; + return sizeKb; } @Override @@ -325,9 +328,12 @@ public TrainingDataAllocation(ExecutorService executor, long memoryAddress, int @Override public void close() { executor.execute(() -> { - writeLock(); - cleanup(); - writeUnlock(); + try { + writeLock(); + cleanup(); + } finally { + writeUnlock(); + } }); } diff --git a/src/main/java/org/opensearch/knn/index/memory/NativeMemoryEntryContext.java b/src/main/java/org/opensearch/knn/index/memory/NativeMemoryEntryContext.java index dd219593d..00bf023f9 100644 --- a/src/main/java/org/opensearch/knn/index/memory/NativeMemoryEntryContext.java +++ b/src/main/java/org/opensearch/knn/index/memory/NativeMemoryEntryContext.java @@ -12,6 +12,7 @@ package org.opensearch.knn.index.memory; import lombok.Getter; +import org.apache.lucene.store.Directory; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.Nullable; import org.opensearch.knn.index.engine.qframe.QuantizationConfig; @@ -64,32 +65,40 @@ public String getKey() { public static class IndexEntryContext extends NativeMemoryEntryContext { + @Getter + private final Directory directory; private final NativeMemoryLoadStrategy.IndexLoadStrategy indexLoadStrategy; + @Getter private final String openSearchIndexName; + @Getter private final Map parameters; @Nullable + @Getter private final String modelId; /** * Constructor * - * @param indexPath path to index file. Also used as key in cache. - * @param indexLoadStrategy strategy to load index into memory - * @param parameters load time parameters - * @param openSearchIndexName opensearch index associated with index + * @param directory Lucene directory to create required IndexInput/IndexOutput to access files. + * @param indexPath Path to index file. Also used as key in cache. + * @param indexLoadStrategy Strategy to load index into memory + * @param parameters Load time parameters + * @param openSearchIndexName Opensearch index associated with index */ public IndexEntryContext( + Directory directory, String indexPath, NativeMemoryLoadStrategy.IndexLoadStrategy indexLoadStrategy, Map parameters, String openSearchIndexName ) { - this(indexPath, indexLoadStrategy, parameters, openSearchIndexName, null); + this(directory, indexPath, indexLoadStrategy, parameters, openSearchIndexName, null); } /** * Constructor * + * @param directory Lucene directory to create required IndexInput/IndexOutput to access files. * @param indexPath path to index file. Also used as key in cache. * @param indexLoadStrategy strategy to load index into memory * @param parameters load time parameters @@ -97,6 +106,7 @@ public IndexEntryContext( * @param modelId model to be loaded. If none available, pass null */ public IndexEntryContext( + Directory directory, String indexPath, NativeMemoryLoadStrategy.IndexLoadStrategy indexLoadStrategy, Map parameters, @@ -104,6 +114,7 @@ public IndexEntryContext( String modelId ) { super(indexPath); + this.directory = directory; this.indexLoadStrategy = indexLoadStrategy; this.openSearchIndexName = openSearchIndexName; this.parameters = parameters; @@ -120,33 +131,6 @@ public NativeMemoryAllocation.IndexAllocation load() throws IOException { return indexLoadStrategy.load(this); } - /** - * Getter for OpenSearch index name. - * - * @return OpenSearch index name - */ - public String getOpenSearchIndexName() { - return openSearchIndexName; - } - - /** - * Getter for parameters. - * - * @return parameters - */ - public Map getParameters() { - return parameters; - } - - /** - * Getter - * - * @return return model ID for the index. null if no model is in use - */ - public String getModelId() { - return modelId; - } - private static class IndexSizeCalculator implements Function { static IndexSizeCalculator INSTANCE = new IndexSizeCalculator(); diff --git a/src/main/java/org/opensearch/knn/index/memory/NativeMemoryLoadStrategy.java b/src/main/java/org/opensearch/knn/index/memory/NativeMemoryLoadStrategy.java index 960c4f5f0..51158d00c 100644 --- a/src/main/java/org/opensearch/knn/index/memory/NativeMemoryLoadStrategy.java +++ b/src/main/java/org/opensearch/knn/index/memory/NativeMemoryLoadStrategy.java @@ -12,8 +12,12 @@ package org.opensearch.knn.index.memory; import lombok.extern.log4j.Log4j2; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IOContext; +import org.apache.lucene.store.IndexInput; import org.opensearch.core.action.ActionListener; import org.opensearch.knn.index.engine.qframe.QuantizationConfig; +import org.opensearch.knn.index.store.IndexInputWithBuffer; import org.opensearch.knn.index.util.IndexUtil; import org.opensearch.knn.jni.JNIService; import org.opensearch.knn.index.engine.KNNEngine; @@ -87,9 +91,9 @@ public void onFileDeleted(Path indexFilePath) { }; } - @Override - public NativeMemoryAllocation.IndexAllocation load(NativeMemoryEntryContext.IndexEntryContext indexEntryContext) - throws IOException { + private NativeMemoryAllocation.IndexAllocation loadWithAbsoluteIndexPath( + NativeMemoryEntryContext.IndexEntryContext indexEntryContext + ) throws IOException { Path indexPath = Paths.get(indexEntryContext.getKey()); FileWatcher fileWatcher = new FileWatcher(indexPath); fileWatcher.addListener(indexFileOnDeleteListener); @@ -97,6 +101,54 @@ public NativeMemoryAllocation.IndexAllocation load(NativeMemoryEntryContext.Inde KNNEngine knnEngine = KNNEngine.getEngineNameFromPath(indexPath.toString()); long indexAddress = JNIService.loadIndex(indexPath.toString(), indexEntryContext.getParameters(), knnEngine); + return createIndexAllocation( + indexEntryContext, + knnEngine, + indexAddress, + fileWatcher, + indexEntryContext.calculateSizeInKB(), + indexPath + ); + } + + @Override + public NativeMemoryAllocation.IndexAllocation load(NativeMemoryEntryContext.IndexEntryContext indexEntryContext) + throws IOException { + final Path absoluteIndexPath = Paths.get(indexEntryContext.getKey()); + final KNNEngine knnEngine = KNNEngine.getEngineNameFromPath(absoluteIndexPath.toString()); + if (knnEngine != KNNEngine.FAISS) { + // We will support other non-FAISS native engines (ex: NMSLIB) soon. + return loadWithAbsoluteIndexPath(indexEntryContext); + } + + final FileWatcher fileWatcher = new FileWatcher(absoluteIndexPath); + fileWatcher.addListener(indexFileOnDeleteListener); + fileWatcher.init(); + + final Directory directory = indexEntryContext.getDirectory(); + + // Ex: Input -> /a/b/c/_0_NativeEngines990KnnVectorsFormat_0.vec + // Output -> _0_NativeEngines990KnnVectorsFormat_0.vec + final String logicalIndexPath = absoluteIndexPath.getFileName().toString(); + + final int indexSizeKb = Math.toIntExact(directory.fileLength(logicalIndexPath) / 1024); + + try (IndexInput readStream = directory.openInput(logicalIndexPath, IOContext.READONCE)) { + IndexInputWithBuffer indexInputWithBuffer = new IndexInputWithBuffer(readStream); + long indexAddress = JNIService.loadIndex(indexInputWithBuffer, indexEntryContext.getParameters(), knnEngine); + + return createIndexAllocation(indexEntryContext, knnEngine, indexAddress, fileWatcher, indexSizeKb, absoluteIndexPath); + } + } + + private NativeMemoryAllocation.IndexAllocation createIndexAllocation( + final NativeMemoryEntryContext.IndexEntryContext indexEntryContext, + final KNNEngine knnEngine, + final long indexAddress, + final FileWatcher fileWatcher, + final int indexSizeKb, + final Path absoluteIndexPath + ) throws IOException { SharedIndexState sharedIndexState = null; String modelId = indexEntryContext.getModelId(); if (IndexUtil.isSharedIndexStateRequired(knnEngine, modelId, indexAddress)) { @@ -109,9 +161,9 @@ public NativeMemoryAllocation.IndexAllocation load(NativeMemoryEntryContext.Inde return new NativeMemoryAllocation.IndexAllocation( executor, indexAddress, - indexEntryContext.calculateSizeInKB(), + indexSizeKb, knnEngine, - indexPath.toString(), + absoluteIndexPath.toString(), indexEntryContext.getOpenSearchIndexName(), watcherHandle, sharedIndexState, diff --git a/src/main/java/org/opensearch/knn/index/query/KNNWeight.java b/src/main/java/org/opensearch/knn/index/query/KNNWeight.java index 1c31ed725..0fd2fddf7 100644 --- a/src/main/java/org/opensearch/knn/index/query/KNNWeight.java +++ b/src/main/java/org/opensearch/knn/index/query/KNNWeight.java @@ -286,6 +286,7 @@ private Map doANNSearch( try { indexAllocation = nativeMemoryCacheManager.get( new NativeMemoryEntryContext.IndexEntryContext( + reader.directory(), indexPath.toString(), NativeMemoryLoadStrategy.IndexLoadStrategy.getInstance(), getParametersAtLoading( diff --git a/src/main/java/org/opensearch/knn/index/store/IndexInputWithBuffer.java b/src/main/java/org/opensearch/knn/index/store/IndexInputWithBuffer.java new file mode 100644 index 000000000..273a4deac --- /dev/null +++ b/src/main/java/org/opensearch/knn/index/store/IndexInputWithBuffer.java @@ -0,0 +1,46 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.store; + +import lombok.NonNull; +import org.apache.lucene.store.IndexInput; + +import java.io.IOException; + +/** + * This class contains a Lucene's IndexInput with a reader buffer. + * A Java reference of this class will be passed to native engines, then 'copyBytes' method will be + * called by native engine via JNI API. + * Therefore, this class servers as a read layer in native engines to read the bytes it wants. + */ +public class IndexInputWithBuffer { + private IndexInput indexInput; + // 4K buffer. + private byte[] buffer = new byte[4 * 1024]; + + public IndexInputWithBuffer(@NonNull IndexInput indexInput) { + this.indexInput = indexInput; + } + + /** + * This method will be invoked in native engines via JNI API. + * Then it will call IndexInput to read required bytes then copy them into a read buffer. + * + * @param nbytes Desired number of bytes to be read. + * @return The number of read bytes in a buffer. + * @throws IOException + */ + private int copyBytes(long nbytes) throws IOException { + final int readBytes = (int) Math.min(nbytes, buffer.length); + indexInput.readBytes(buffer, 0, readBytes); + return readBytes; + } + + @Override + public String toString() { + return "{indexInput=" + indexInput + ", len(buffer)=" + buffer.length + "}"; + } +} diff --git a/src/main/java/org/opensearch/knn/jni/FaissService.java b/src/main/java/org/opensearch/knn/jni/FaissService.java index 4bceed015..c56726c66 100644 --- a/src/main/java/org/opensearch/knn/jni/FaissService.java +++ b/src/main/java/org/opensearch/knn/jni/FaissService.java @@ -14,6 +14,7 @@ import org.opensearch.knn.common.KNNConstants; import org.opensearch.knn.index.query.KNNQueryResult; import org.opensearch.knn.index.engine.KNNEngine; +import org.opensearch.knn.index.store.IndexInputWithBuffer; import java.security.AccessController; import java.security.PrivilegedAction; @@ -217,6 +218,15 @@ public static native void createByteIndexFromTemplate( */ public static native long loadIndex(String indexPath); + /** + * Load an index into memory via a wrapping having Lucene's IndexInput. + * Instead of directly accessing an index path, this will make Faiss delegate IndexInput to load bytes. + * + * @param readStream IndexInput wrapper having a Lucene's IndexInput reference. + * @return pointer to location in memory the index resides in + */ + public static native long loadIndexWithStream(IndexInputWithBuffer readStream); + /** * Load a binary index into memory * @@ -225,6 +235,15 @@ public static native void createByteIndexFromTemplate( */ public static native long loadBinaryIndex(String indexPath); + /** + * Load a binary index into memory with a wrapping having Lucene's IndexInput. + * Instead of directly accessing an index path, this will make Faiss delegate IndexInput to load bytes. + * + * @param readStream IndexInput wrapper having a Lucene's IndexInput reference. + * @return pointer to location in memory the index resides in + */ + public static native long loadBinaryIndexWithStream(IndexInputWithBuffer readStream); + /** * Determine if index contains shared state. * diff --git a/src/main/java/org/opensearch/knn/jni/JNIService.java b/src/main/java/org/opensearch/knn/jni/JNIService.java index 94c1ec48e..448241f9c 100644 --- a/src/main/java/org/opensearch/knn/jni/JNIService.java +++ b/src/main/java/org/opensearch/knn/jni/JNIService.java @@ -16,6 +16,7 @@ import org.opensearch.knn.common.KNNConstants; import org.opensearch.knn.index.engine.KNNEngine; import org.opensearch.knn.index.query.KNNQueryResult; +import org.opensearch.knn.index.store.IndexInputWithBuffer; import org.opensearch.knn.index.util.IndexUtil; import java.util.Locale; @@ -211,6 +212,28 @@ public static long loadIndex(String indexPath, Map parameters, K ); } + /** + * Load an index via Lucene's IndexInput. + * + * @param readStream A wrapper having Lucene's IndexInput to load bytes from a file. + * @param parameters Parameters to be used when loading index + * @param knnEngine Engine to load index + * @return Pointer to location in memory the index resides in + */ + public static long loadIndex(IndexInputWithBuffer readStream, Map parameters, KNNEngine knnEngine) { + if (KNNEngine.FAISS == knnEngine) { + if (IndexUtil.isBinaryIndex(knnEngine, parameters)) { + return FaissService.loadBinaryIndexWithStream(readStream); + } else { + return FaissService.loadIndexWithStream(readStream); + } + } + + throw new IllegalArgumentException( + String.format(Locale.ROOT, "LoadIndex not supported for provided engine : %s", knnEngine.getName()) + ); + } + /** * Determine if index contains shared state. Currently, we cannot do this in the plugin because we do not store the * model definition anywhere. Only faiss supports indices that have shared state. So for all other engines it will diff --git a/src/test/java/org/opensearch/knn/index/memory/NativeMemoryAllocationTests.java b/src/test/java/org/opensearch/knn/index/memory/NativeMemoryAllocationTests.java index cb5fbaeba..906ff4cb7 100644 --- a/src/test/java/org/opensearch/knn/index/memory/NativeMemoryAllocationTests.java +++ b/src/test/java/org/opensearch/knn/index/memory/NativeMemoryAllocationTests.java @@ -37,6 +37,7 @@ import java.util.concurrent.Executors; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicReference; import static org.mockito.Mockito.doNothing; @@ -259,11 +260,12 @@ public void testIndexAllocation_closeDefault() { } public void testIndexAllocation_closeBlocking() throws InterruptedException, ExecutionException { + // Prepare mocking and a thread pool. WatcherHandle watcherHandle = (WatcherHandle) mock(WatcherHandle.class); - ExecutorService executorService = Executors.newFixedThreadPool(2); - AtomicReference expectedException = new AtomicReference<>(); + ExecutorService executorService = Executors.newSingleThreadExecutor(); - // Blocking close + // Enable `KNN_FORCE_EVICT_CACHE_ENABLED_SETTING` to force it to block other threads. + // Having it false will make `IndexAllocation` to run close logic in a different thread. when(clusterSettings.get(KNN_FORCE_EVICT_CACHE_ENABLED_SETTING)).thenReturn(true); NativeMemoryAllocation.IndexAllocation blockingIndexAllocation = new NativeMemoryAllocation.IndexAllocation( mock(ExecutorService.class), @@ -275,19 +277,21 @@ public void testIndexAllocation_closeBlocking() throws InterruptedException, Exe watcherHandle ); - executorService.submit(blockingIndexAllocation::readLock); + // Acquire a read lock + blockingIndexAllocation.readLock(); + + // This should be blocked as a read lock is still being held. Future closingThread = executorService.submit(blockingIndexAllocation::close); // Check if thread is currently blocked try { closingThread.get(5, TimeUnit.SECONDS); - } catch (Exception e) { - expectedException.set(e); - } - - assertNotNull(expectedException.get()); + fail("Closing should be blocked. We are still holding a read lock."); + } catch (TimeoutException ignored) {} - executorService.submit(blockingIndexAllocation::readUnlock); + // Now, we unlock a read lock. + blockingIndexAllocation.readUnlock(); + // As we don't hold any locking, the closing thread can now good to acquire a write lock. closingThread.get(); // Waits until close diff --git a/src/test/java/org/opensearch/knn/index/memory/NativeMemoryEntryContextTests.java b/src/test/java/org/opensearch/knn/index/memory/NativeMemoryEntryContextTests.java index 1720da1ed..72cab9a1b 100644 --- a/src/test/java/org/opensearch/knn/index/memory/NativeMemoryEntryContextTests.java +++ b/src/test/java/org/opensearch/knn/index/memory/NativeMemoryEntryContextTests.java @@ -12,6 +12,7 @@ package org.opensearch.knn.index.memory; import com.google.common.collect.ImmutableMap; +import org.apache.lucene.store.Directory; import org.opensearch.cluster.service.ClusterService; import org.opensearch.knn.KNNTestCase; import org.opensearch.knn.index.engine.qframe.QuantizationConfig; @@ -44,6 +45,7 @@ public void testAbstract_getKey() { public void testIndexEntryContext_load() throws IOException { NativeMemoryLoadStrategy.IndexLoadStrategy indexLoadStrategy = mock(NativeMemoryLoadStrategy.IndexLoadStrategy.class); NativeMemoryEntryContext.IndexEntryContext indexEntryContext = new NativeMemoryEntryContext.IndexEntryContext( + (Directory) null, "test", indexLoadStrategy, null, @@ -82,6 +84,7 @@ public void testIndexEntryContext_calculateSize() throws IOException { // Check that the indexEntryContext will return the same thing NativeMemoryEntryContext.IndexEntryContext indexEntryContext = new NativeMemoryEntryContext.IndexEntryContext( + (Directory) null, tmpFile.toAbsolutePath().toString(), null, null, @@ -94,6 +97,7 @@ public void testIndexEntryContext_calculateSize() throws IOException { public void testIndexEntryContext_getOpenSearchIndexName() { String openSearchIndexName = "test-index"; NativeMemoryEntryContext.IndexEntryContext indexEntryContext = new NativeMemoryEntryContext.IndexEntryContext( + (Directory) null, "test", null, null, @@ -106,6 +110,7 @@ public void testIndexEntryContext_getOpenSearchIndexName() { public void testIndexEntryContext_getParameters() { Map parameters = ImmutableMap.of("test-1", 10); NativeMemoryEntryContext.IndexEntryContext indexEntryContext = new NativeMemoryEntryContext.IndexEntryContext( + (Directory) null, "test", null, parameters, diff --git a/src/test/java/org/opensearch/knn/index/memory/NativeMemoryLoadStrategyTests.java b/src/test/java/org/opensearch/knn/index/memory/NativeMemoryLoadStrategyTests.java index bdd8d7e45..373afddc7 100644 --- a/src/test/java/org/opensearch/knn/index/memory/NativeMemoryLoadStrategyTests.java +++ b/src/test/java/org/opensearch/knn/index/memory/NativeMemoryLoadStrategyTests.java @@ -12,6 +12,8 @@ package org.opensearch.knn.index.memory; import com.google.common.collect.ImmutableMap; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.MMapDirectory; import org.opensearch.core.action.ActionListener; import org.opensearch.action.search.SearchResponse; import org.opensearch.knn.KNNTestCase; @@ -47,6 +49,7 @@ public class NativeMemoryLoadStrategyTests extends KNNTestCase { public void testIndexLoadStrategy_load() throws IOException { // Create basic nmslib HNSW index Path dir = createTempDir(); + Directory luceneDirectory = new MMapDirectory(dir); KNNEngine knnEngine = KNNEngine.NMSLIB; String indexName = "test1" + knnEngine.getExtension(); String path = dir.resolve(indexName).toAbsolutePath().toString(); @@ -68,6 +71,7 @@ public void testIndexLoadStrategy_load() throws IOException { NativeMemoryLoadStrategy.IndexLoadStrategy.initialize(resourceWatcherService); NativeMemoryEntryContext.IndexEntryContext indexEntryContext = new NativeMemoryEntryContext.IndexEntryContext( + luceneDirectory, path, NativeMemoryLoadStrategy.IndexLoadStrategy.getInstance(), parameters, @@ -87,6 +91,7 @@ public void testIndexLoadStrategy_load() throws IOException { public void testLoad_whenFaissBinary_thenSuccess() throws IOException { Path dir = createTempDir(); + Directory luceneDirectory = new MMapDirectory(dir); KNNEngine knnEngine = KNNEngine.FAISS; String indexName = "test1" + knnEngine.getExtension(); String path = dir.resolve(indexName).toAbsolutePath().toString(); @@ -116,6 +121,7 @@ public void testLoad_whenFaissBinary_thenSuccess() throws IOException { NativeMemoryLoadStrategy.IndexLoadStrategy.initialize(resourceWatcherService); NativeMemoryEntryContext.IndexEntryContext indexEntryContext = new NativeMemoryEntryContext.IndexEntryContext( + luceneDirectory, path, NativeMemoryLoadStrategy.IndexLoadStrategy.getInstance(), parameters, diff --git a/src/test/java/org/opensearch/knn/jni/JNIServiceTests.java b/src/test/java/org/opensearch/knn/jni/JNIServiceTests.java index c78478f4d..8566b0223 100644 --- a/src/test/java/org/opensearch/knn/jni/JNIServiceTests.java +++ b/src/test/java/org/opensearch/knn/jni/JNIServiceTests.java @@ -14,6 +14,10 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import lombok.SneakyThrows; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IOContext; +import org.apache.lucene.store.IndexInput; +import org.apache.lucene.store.MMapDirectory; import org.junit.BeforeClass; import org.opensearch.Version; import org.opensearch.common.xcontent.XContentFactory; @@ -29,6 +33,7 @@ import org.opensearch.knn.index.engine.MethodComponentContext; import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.engine.KNNEngine; +import org.opensearch.knn.index.store.IndexInputWithBuffer; import java.io.IOException; import java.net.URL; @@ -871,6 +876,29 @@ public void testQueryIndex_faiss_invalid_nullQueryVector() throws IOException { expectThrows(Exception.class, () -> JNIService.queryIndex(pointer, null, 10, null, KNNEngine.FAISS, null, 0, null)); } + public void testQueryIndex_faiss_streaming_invalid_nullQueryVector() throws IOException { + Path tmpFile = createTempFile(); + + TestUtils.createIndex( + testData.indexData.docs, + testData.loadDataToMemoryAddress(), + testData.indexData.getDimension(), + tmpFile.toAbsolutePath().toString(), + ImmutableMap.of(INDEX_DESCRIPTION_PARAMETER, faissMethod, KNNConstants.SPACE_TYPE, SpaceType.L2.getValue()), + KNNEngine.FAISS + ); + assertTrue(tmpFile.toFile().length() > 0); + + try (final Directory directory = new MMapDirectory(tmpFile.getParent())) { + try (IndexInput indexInput = directory.openInput(tmpFile.getFileName().toString(), IOContext.READONCE)) { + long pointer = JNIService.loadIndex(new IndexInputWithBuffer(indexInput), Collections.emptyMap(), KNNEngine.FAISS); + assertNotEquals(0, pointer); + + expectThrows(Exception.class, () -> JNIService.queryIndex(pointer, null, 10, null, KNNEngine.FAISS, null, 0, null)); + } + } + } + public void testQueryIndex_faiss_valid() throws IOException { int k = 10; @@ -930,6 +958,68 @@ public void testQueryIndex_faiss_valid() throws IOException { } } + public void testQueryIndex_faiss_streaming_valid() throws IOException { + int k = 10; + int efSearch = 100; + + List methods = ImmutableList.of(faissMethod); + List spaces = ImmutableList.of(SpaceType.L2, SpaceType.INNER_PRODUCT); + for (String method : methods) { + for (SpaceType spaceType : spaces) { + Path tmpFile = createTempFile(); + TestUtils.createIndex( + testData.indexData.docs, + testData.loadDataToMemoryAddress(), + testData.indexData.getDimension(), + tmpFile.toAbsolutePath().toString(), + ImmutableMap.of(INDEX_DESCRIPTION_PARAMETER, method, KNNConstants.SPACE_TYPE, spaceType.getValue()), + KNNEngine.FAISS + ); + assertTrue(tmpFile.toFile().length() > 0); + + try (final Directory directory = new MMapDirectory(tmpFile.getParent())) { + try (IndexInput indexInput = directory.openInput(tmpFile.getFileName().toString(), IOContext.READONCE)) { + long pointer = JNIService.loadIndex( + new IndexInputWithBuffer(indexInput), + ImmutableMap.of(KNNConstants.SPACE_TYPE, spaceType.getValue()), + KNNEngine.FAISS + ); + assertNotEquals(0, pointer); + + for (float[] query : testData.queries) { + KNNQueryResult[] results = JNIService.queryIndex( + pointer, + query, + k, + Map.of("ef_search", efSearch), + KNNEngine.FAISS, + null, + 0, + null + ); + assertEquals(k, results.length); + } + + // Filter will result in no ids + for (float[] query : testData.queries) { + KNNQueryResult[] results = JNIService.queryIndex( + pointer, + query, + k, + Map.of("ef_search", efSearch), + KNNEngine.FAISS, + new long[] { 0 }, + 0, + null + ); + assertEquals(0, results.length); + } // End for + } // End try + } // End try + } // End for + } // End for + } + public void testQueryIndex_faiss_parentIds() throws IOException { int k = 100; @@ -978,6 +1068,58 @@ public void testQueryIndex_faiss_parentIds() throws IOException { } } + public void testQueryIndex_faiss_streaming_parentIds() throws IOException { + + int k = 100; + int efSearch = 100; + + List methods = ImmutableList.of(faissMethod); + List spaces = ImmutableList.of(SpaceType.L2, SpaceType.INNER_PRODUCT); + int[] parentIds = toParentIdArray(testDataNested.indexData.docs); + Map idToParentIdMap = toIdToParentIdMap(testDataNested.indexData.docs); + for (String method : methods) { + for (SpaceType spaceType : spaces) { + Path tmpFile = createTempFile(); + TestUtils.createIndex( + testDataNested.indexData.docs, + testData.loadDataToMemoryAddress(), + testDataNested.indexData.getDimension(), + tmpFile.toAbsolutePath().toString(), + ImmutableMap.of(INDEX_DESCRIPTION_PARAMETER, method, KNNConstants.SPACE_TYPE, spaceType.getValue()), + KNNEngine.FAISS + ); + assertTrue(tmpFile.toFile().length() > 0); + + try (final Directory directory = new MMapDirectory(tmpFile.getParent())) { + try (IndexInput indexInput = directory.openInput(tmpFile.getFileName().toString(), IOContext.READONCE)) { + long pointer = JNIService.loadIndex( + new IndexInputWithBuffer(indexInput), + ImmutableMap.of(KNNConstants.SPACE_TYPE, spaceType.getValue()), + KNNEngine.FAISS + ); + assertNotEquals(0, pointer); + + for (float[] query : testDataNested.queries) { + KNNQueryResult[] results = JNIService.queryIndex( + pointer, + query, + k, + Map.of("ef_search", efSearch), + KNNEngine.FAISS, + null, + 0, + parentIds + ); + // Verify there is no more than one result from same parent + Set parentIdSet = toParentIdSet(results, idToParentIdMap); + assertEquals(results.length, parentIdSet.size()); + } // End for + } // End try + } // End try + } // End for + } // End for + } + @SneakyThrows public void testQueryBinaryIndex_faiss_valid() { int k = 10; @@ -1016,6 +1158,53 @@ public void testQueryBinaryIndex_faiss_valid() { } } + @SneakyThrows + public void testQueryBinaryIndex_faiss_streaming_valid() { + int k = 10; + List methods = ImmutableList.of(faissBinaryMethod); + for (String method : methods) { + Path tmpFile = createTempFile(); + long memoryAddr = testData.loadBinaryDataToMemoryAddress(); + TestUtils.createIndex( + testData.indexData.docs, + memoryAddr, + testData.indexData.getDimension(), + tmpFile.toAbsolutePath().toString(), + ImmutableMap.of( + INDEX_DESCRIPTION_PARAMETER, + method, + KNNConstants.SPACE_TYPE, + SpaceType.HAMMING.getValue(), + KNNConstants.VECTOR_DATA_TYPE_FIELD, + VectorDataType.BINARY.getValue() + ), + KNNEngine.FAISS + ); + assertTrue(tmpFile.toFile().length() > 0); + + try (final Directory directory = new MMapDirectory(tmpFile.getParent())) { + try (IndexInput indexInput = directory.openInput(tmpFile.getFileName().toString(), IOContext.READONCE)) { + long pointer = JNIService.loadIndex( + new IndexInputWithBuffer(indexInput), + ImmutableMap.of( + INDEX_DESCRIPTION_PARAMETER, + method, + KNNConstants.VECTOR_DATA_TYPE_FIELD, + VectorDataType.BINARY.getValue() + ), + KNNEngine.FAISS + ); + assertNotEquals(0, pointer); + + for (byte[] query : testData.binaryQueries) { + KNNQueryResult[] results = JNIService.queryBinaryIndex(pointer, query, k, null, KNNEngine.FAISS, null, 0, null); + assertEquals(k, results.length); + } // End for + } // End try + } // End try + } // End for + } + private Set toParentIdSet(KNNQueryResult[] results, Map idToParentIdMap) { return Arrays.stream(results).map(result -> idToParentIdMap.get(result.getId())).collect(Collectors.toSet()); } From 2caae12e98f3af3c04c4c1f228017359da6274c3 Mon Sep 17 00:00:00 2001 From: Doo Yong Kim <0ctopus13prime@gmail.com> Date: Thu, 3 Oct 2024 16:12:30 -0700 Subject: [PATCH 394/416] Fix sed command in DEVELOPER_GUIDE to append a new line character '\n'. (#2181) (#2184) Signed-off-by: Dooyong Kim Co-authored-by: Dooyong Kim (cherry picked from commit 677184230bccec01ecfaf43fa3ed368dacf5a510) --- CHANGELOG.md | 1 + DEVELOPER_GUIDE.md | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0a54a242a..b6f9393ba 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), * Add DocValuesProducers for releasing memory when close index [#1946](https://github.com/opensearch-project/k-NN/pull/1946) ### Infrastructure ### Documentation +* Fix sed command in DEVELOPER_GUIDE.md to append a new line character '\n'. [#2181](https://github.com/opensearch-project/k-NN/pull/2181) ### Maintenance ### Refactoring * Does not create additional KNNVectorValues in NativeEngines990KNNVectorWriter when quantization is not needed [#2133](https://github.com/opensearch-project/k-NN/pull/2133) diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index 68e624d5d..c39a6f1b6 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -116,6 +116,11 @@ source ~/.zshrc export CC=/opt/homebrew/opt/llvm/bin/clang export CXX=/opt/homebrew/opt/llvm/bin/clang++ +// In case of linking issues with the external libraries and clang, you can try setting the CMAKE compiler to gcc/g++ instead through the following commands: +export CC=gcc +export CXX=g++ +sed -i '' '/set(CMAKE_CXX_STANDARD_REQUIRED True)/a\'$'\n''set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Xclang -fopenmp -L/opt/homebrew/opt/libomp/lib -I/opt/homebrew/opt/libomp/include -lomp -arch arm64 -fexceptions")'$'\n''' CMakeLists.txt + // Build cmake . --fresh make From 1c77c9270d33484002a30a155667d20120e7c117 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Mon, 7 Oct 2024 08:16:16 -0700 Subject: [PATCH 395/416] [Backport 2.x] Score Fix for Binary Quantized Vector and Setting Default value in case of shard level rescoring is disabled for oversampling factor (#2187) * Score Fix for Binary Quantized Vector and Setting Default value in case of shard level rescoring is disabled for oversampling factor (#2183) Signed-off-by: VIKASH TIWARI (cherry picked from commit 01d7981c8a9ddab24188b08ae9c7ace2ba1e2ac5) Signed-off-by: John Mazanec * Java Docs Fix for 2.x in Linux Machine (#2191) Signed-off-by: VIKASH TIWARI Signed-off-by: John Mazanec --------- Signed-off-by: VIKASH TIWARI Signed-off-by: John Mazanec Co-authored-by: Vikasht34 --- CHANGELOG.md | 2 + .../org/opensearch/knn/index/KNNSettings.java | 7 +- .../knn/index/mapper/CompressionLevel.java | 39 +++---- .../opensearch/knn/index/query/KNNWeight.java | 4 + .../nativelib/NativeEngineKnnVectorQuery.java | 6 +- .../index/query/rescore/RescoreContext.java | 39 ++++++- .../knn/index/KNNSettingsTests.java | 10 +- .../index/mapper/CompressionLevelTests.java | 76 +++++-------- .../knn/index/query/KNNQueryBuilderTests.java | 3 +- .../knn/index/query/KNNWeightTests.java | 106 ++++++++++++++++++ .../NativeEngineKNNVectorQueryTests.java | 6 +- .../query/rescore/RescoreContextTests.java | 79 ++++++++----- 12 files changed, 262 insertions(+), 115 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b6f9393ba..d854f2f0c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,6 +28,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), * Adding Support to Enable/Disble Share level Rescoring and Update Oversampling Factor[#2172](https://github.com/opensearch-project/k-NN/pull/2172) ### Bug Fixes * KNN80DocValues should only be considered for BinaryDocValues fields [#2147](https://github.com/opensearch-project/k-NN/pull/2147) +* Score Fix for Binary Quantized Vector and Setting Default value in case of shard level rescoring is disabled for oversampling factor[#2183](https://github.com/opensearch-project/k-NN/pull/2183) +* Java Docs Fix For 2.x ### Infrastructure ### Documentation ### Maintenance diff --git a/src/main/java/org/opensearch/knn/index/KNNSettings.java b/src/main/java/org/opensearch/knn/index/KNNSettings.java index 1753140e6..f5980879a 100644 --- a/src/main/java/org/opensearch/knn/index/KNNSettings.java +++ b/src/main/java/org/opensearch/knn/index/KNNSettings.java @@ -92,6 +92,7 @@ public class KNNSettings { /** * Default setting values + * */ public static final boolean KNN_DEFAULT_FAISS_AVX2_DISABLED_VALUE = false; public static final boolean KNN_DEFAULT_FAISS_AVX512_DISABLED_VALUE = false; @@ -113,7 +114,7 @@ public class KNNSettings { public static final Integer KNN_MAX_QUANTIZATION_STATE_CACHE_SIZE_LIMIT_PERCENTAGE = 10; // Quantization state cache limit cannot exceed // 10% of the JVM heap public static final Integer KNN_DEFAULT_QUANTIZATION_STATE_CACHE_EXPIRY_TIME_MINUTES = 60; - public static final boolean KNN_DISK_VECTOR_SHARD_LEVEL_RESCORING_DISABLED_VALUE = true; + public static final boolean KNN_DISK_VECTOR_SHARD_LEVEL_RESCORING_DISABLED_VALUE = false; /** * Settings Definition @@ -554,12 +555,12 @@ public static Integer getFilteredExactSearchThreshold(final String indexName) { .getAsInt(ADVANCED_FILTERED_EXACT_SEARCH_THRESHOLD, ADVANCED_FILTERED_EXACT_SEARCH_THRESHOLD_DEFAULT_VALUE); } - public static boolean isShardLevelRescoringDisabledForDiskBasedVector(String indexName) { + public static boolean isShardLevelRescoringEnabledForDiskBasedVector(String indexName) { return KNNSettings.state().clusterService.state() .getMetadata() .index(indexName) .getSettings() - .getAsBoolean(KNN_DISK_VECTOR_SHARD_LEVEL_RESCORING_DISABLED, true); + .getAsBoolean(KNN_DISK_VECTOR_SHARD_LEVEL_RESCORING_DISABLED, false); } public void initialize(Client client, ClusterService clusterService) { diff --git a/src/main/java/org/opensearch/knn/index/mapper/CompressionLevel.java b/src/main/java/org/opensearch/knn/index/mapper/CompressionLevel.java index c9a169efc..ab583a2e0 100644 --- a/src/main/java/org/opensearch/knn/index/mapper/CompressionLevel.java +++ b/src/main/java/org/opensearch/knn/index/mapper/CompressionLevel.java @@ -25,9 +25,9 @@ public enum CompressionLevel { x1(1, "1x", null, Collections.emptySet()), x2(2, "2x", null, Collections.emptySet()), x4(4, "4x", null, Collections.emptySet()), - x8(8, "8x", new RescoreContext(2.0f), Set.of(Mode.ON_DISK)), - x16(16, "16x", new RescoreContext(3.0f), Set.of(Mode.ON_DISK)), - x32(32, "32x", new RescoreContext(3.0f), Set.of(Mode.ON_DISK)); + x8(8, "8x", new RescoreContext(2.0f, false), Set.of(Mode.ON_DISK)), + x16(16, "16x", new RescoreContext(3.0f, false), Set.of(Mode.ON_DISK)), + x32(32, "32x", new RescoreContext(3.0f, false), Set.of(Mode.ON_DISK)); // Internally, an empty string is easier to deal with them null. However, from the mapping, // we do not want users to pass in the empty string and instead want null. So we make the conversion here @@ -97,32 +97,33 @@ public static boolean isConfigured(CompressionLevel compressionLevel) { /** * Returns the appropriate {@link RescoreContext} based on the given {@code mode} and {@code dimension}. * - *

    If the {@code mode} is present in the valid {@code modesForRescore} set, the method adjusts the oversample factor based on the - * {@code dimension} value: + *

    If the {@code mode} is present in the valid {@code modesForRescore} set, the method checks the value of + * {@code dimension}: *

      - *
    • If {@code dimension} is greater than or equal to 1000, no oversampling is applied (oversample factor = 1.0).
      • - *
    • If {@code dimension} is greater than or equal to 768 but less than 1000, a 2x oversample factor is applied (oversample factor = 2.0).
      • - *
    • If {@code dimension} is less than 768, a 3x oversample factor is applied (oversample factor = 3.0).
      • + *
    • If {@code dimension} is less than or equal to 1000, it returns a {@link RescoreContext} with an + * oversample factor of 5.0f.
      • + *
    • If {@code dimension} is greater than 1000, it returns the default {@link RescoreContext} associated with + * the {@link CompressionLevel}. If no default is set, it falls back to {@link RescoreContext#getDefault()}.
      • *
    - * If the {@code mode} is not present in the {@code modesForRescore} set, the method returns {@code null}. + * If the {@code mode} is not valid, the method returns {@code null}. * * @param mode The {@link Mode} for which to retrieve the {@link RescoreContext}. * @param dimension The dimensional value that determines the {@link RescoreContext} behavior. - * @return A {@link RescoreContext} with the appropriate oversample factor based on the dimension, or {@code null} if the mode - * is not valid. + * @return A {@link RescoreContext} with an oversample factor of 5.0f if {@code dimension} is less than + * or equal to 1000, the default {@link RescoreContext} if greater, or {@code null} if the mode + * is invalid. */ public RescoreContext getDefaultRescoreContext(Mode mode, int dimension) { if (modesForRescore.contains(mode)) { // Adjust RescoreContext based on dimension - if (dimension >= RescoreContext.DIMENSION_THRESHOLD_1000) { - // No oversampling for dimensions >= 1000 - return RescoreContext.builder().oversampleFactor(RescoreContext.OVERSAMPLE_FACTOR_1000).build(); - } else if (dimension >= RescoreContext.DIMENSION_THRESHOLD_768) { - // 2x oversampling for dimensions >= 768 but < 1000 - return RescoreContext.builder().oversampleFactor(RescoreContext.OVERSAMPLE_FACTOR_768).build(); + if (dimension <= RescoreContext.DIMENSION_THRESHOLD) { + // For dimensions <= 1000, return a RescoreContext with 5.0f oversample factor + return RescoreContext.builder() + .oversampleFactor(RescoreContext.OVERSAMPLE_FACTOR_BELOW_DIMENSION_THRESHOLD) + .userProvided(false) + .build(); } else { - // 3x oversampling for dimensions < 768 - return RescoreContext.builder().oversampleFactor(RescoreContext.OVERSAMPLE_FACTOR_BELOW_768).build(); + return defaultRescoreContext; } } return null; diff --git a/src/main/java/org/opensearch/knn/index/query/KNNWeight.java b/src/main/java/org/opensearch/knn/index/query/KNNWeight.java index 0fd2fddf7..37695c208 100644 --- a/src/main/java/org/opensearch/knn/index/query/KNNWeight.java +++ b/src/main/java/org/opensearch/knn/index/query/KNNWeight.java @@ -376,6 +376,10 @@ private Map doANNSearch( return null; } + if (quantizedVector != null) { + return Arrays.stream(results) + .collect(Collectors.toMap(KNNQueryResult::getId, result -> knnEngine.score(result.getScore(), SpaceType.HAMMING))); + } return Arrays.stream(results) .collect(Collectors.toMap(KNNQueryResult::getId, result -> knnEngine.score(result.getScore(), spaceType))); } diff --git a/src/main/java/org/opensearch/knn/index/query/nativelib/NativeEngineKnnVectorQuery.java b/src/main/java/org/opensearch/knn/index/query/nativelib/NativeEngineKnnVectorQuery.java index adb2875d5..c97a0d061 100644 --- a/src/main/java/org/opensearch/knn/index/query/nativelib/NativeEngineKnnVectorQuery.java +++ b/src/main/java/org/opensearch/knn/index/query/nativelib/NativeEngineKnnVectorQuery.java @@ -61,9 +61,11 @@ public Weight createWeight(IndexSearcher indexSearcher, ScoreMode scoreMode, flo if (rescoreContext == null) { perLeafResults = doSearch(indexSearcher, leafReaderContexts, knnWeight, finalK); } else { - int firstPassK = rescoreContext.getFirstPassK(finalK); + boolean isShardLevelRescoringEnabled = KNNSettings.isShardLevelRescoringEnabledForDiskBasedVector(knnQuery.getIndexName()); + int dimension = knnQuery.getQueryVector().length; + int firstPassK = rescoreContext.getFirstPassK(finalK, isShardLevelRescoringEnabled, dimension); perLeafResults = doSearch(indexSearcher, leafReaderContexts, knnWeight, firstPassK); - if (KNNSettings.isShardLevelRescoringDisabledForDiskBasedVector(knnQuery.getIndexName()) == false) { + if (isShardLevelRescoringEnabled == true) { ResultUtil.reduceToTopK(perLeafResults, firstPassK); } diff --git a/src/main/java/org/opensearch/knn/index/query/rescore/RescoreContext.java b/src/main/java/org/opensearch/knn/index/query/rescore/RescoreContext.java index a2563b2a6..09aeb7591 100644 --- a/src/main/java/org/opensearch/knn/index/query/rescore/RescoreContext.java +++ b/src/main/java/org/opensearch/knn/index/query/rescore/RescoreContext.java @@ -39,21 +39,50 @@ public final class RescoreContext { @Builder.Default private float oversampleFactor = DEFAULT_OVERSAMPLE_FACTOR; + /** + * Flag to track whether the oversample factor is user-provided or default. The Reason to introduce + * this is to set default when Shard Level rescoring is false, + * else we end up overriding user provided value in NativeEngineKnnVectorQuery + */ + @Builder.Default + private boolean userProvided = true; + /** * * @return default RescoreContext */ public static RescoreContext getDefault() { - return RescoreContext.builder().build(); + return RescoreContext.builder().oversampleFactor(DEFAULT_OVERSAMPLE_FACTOR).userProvided(false).build(); } /** - * Gets the number of results to return for the first pass of rescoring. + * Calculates the number of results to return for the first pass of rescoring (firstPassK). + * This method considers whether shard-level rescoring is enabled and adjusts the oversample factor + * based on the vector dimension if shard-level rescoring is disabled. * - * @param finalK The final number of results to return for the entire shard - * @return The number of results to return for the first pass of rescoring + * @param finalK The final number of results to return for the entire shard. + * @param isShardLevelRescoringEnabled A boolean flag indicating whether shard-level rescoring is enabled. + * If true, the dimension-based oversampling logic is bypassed. + * @param dimension The dimension of the vector. This is used to determine the oversampling factor when + * shard-level rescoring is disabled. + * @return The number of results to return for the first pass of rescoring, adjusted by the oversample factor. */ - public int getFirstPassK(int finalK) { + public int getFirstPassK(int finalK, boolean isShardLevelRescoringEnabled, int dimension) { + // Only apply default dimension-based oversampling logic when: + // 1. Shard-level rescoring is disabled + // 2. The oversample factor was not provided by the user + if (!isShardLevelRescoringEnabled && !userProvided) { + // Apply new dimension-based oversampling logic when shard-level rescoring is disabled + if (dimension >= DIMENSION_THRESHOLD_1000) { + oversampleFactor = OVERSAMPLE_FACTOR_1000; // No oversampling for dimensions >= 1000 + } else if (dimension >= DIMENSION_THRESHOLD_768) { + oversampleFactor = OVERSAMPLE_FACTOR_768; // 2x oversampling for dimensions >= 768 and < 1000 + } else { + oversampleFactor = OVERSAMPLE_FACTOR_BELOW_768; // 3x oversampling for dimensions < 768 + } + } + // The calculation for firstPassK remains the same, applying the oversample factor return Math.min(MAX_FIRST_PASS_RESULTS, Math.max(MIN_FIRST_PASS_RESULTS, (int) Math.ceil(finalK * oversampleFactor))); } + } diff --git a/src/test/java/org/opensearch/knn/index/KNNSettingsTests.java b/src/test/java/org/opensearch/knn/index/KNNSettingsTests.java index 961a82a72..60be2ee19 100644 --- a/src/test/java/org/opensearch/knn/index/KNNSettingsTests.java +++ b/src/test/java/org/opensearch/knn/index/KNNSettingsTests.java @@ -163,7 +163,7 @@ public void testGetEfSearch_whenEFSearchValueSetByUser_thenReturnValue() { } @SneakyThrows - public void testShardLevelRescoringDisabled_whenNoValuesProvidedByUser_thenDefaultSettingsUsed() { + public void testShardLevelRescoringEnabled_whenNoValuesProvidedByUser_thenDefaultSettingsUsed() { Node mockNode = createMockNode(Collections.emptyMap()); mockNode.start(); ClusterService clusterService = mockNode.injector().getInstance(ClusterService.class); @@ -171,14 +171,14 @@ public void testShardLevelRescoringDisabled_whenNoValuesProvidedByUser_thenDefau mockNode.client().admin().indices().create(new CreateIndexRequest(INDEX_NAME)).actionGet(); KNNSettings.state().setClusterService(clusterService); - boolean shardLevelRescoringDisabled = KNNSettings.isShardLevelRescoringDisabledForDiskBasedVector(INDEX_NAME); + boolean shardLevelRescoringDisabled = KNNSettings.isShardLevelRescoringEnabledForDiskBasedVector(INDEX_NAME); mockNode.close(); - assertTrue(shardLevelRescoringDisabled); + assertFalse(shardLevelRescoringDisabled); } @SneakyThrows public void testShardLevelRescoringDisabled_whenValueProvidedByUser_thenSettingApplied() { - boolean userDefinedRescoringDisabled = false; + boolean userDefinedRescoringDisabled = true; Node mockNode = createMockNode(Collections.emptyMap()); mockNode.start(); ClusterService clusterService = mockNode.injector().getInstance(ClusterService.class); @@ -192,7 +192,7 @@ public void testShardLevelRescoringDisabled_whenValueProvidedByUser_thenSettingA mockNode.client().admin().indices().updateSettings(new UpdateSettingsRequest(rescoringDisabledSetting, INDEX_NAME)).actionGet(); - boolean shardLevelRescoringDisabled = KNNSettings.isShardLevelRescoringDisabledForDiskBasedVector(INDEX_NAME); + boolean shardLevelRescoringDisabled = KNNSettings.isShardLevelRescoringEnabledForDiskBasedVector(INDEX_NAME); mockNode.close(); assertEquals(userDefinedRescoringDisabled, shardLevelRescoringDisabled); } diff --git a/src/test/java/org/opensearch/knn/index/mapper/CompressionLevelTests.java b/src/test/java/org/opensearch/knn/index/mapper/CompressionLevelTests.java index 57372b11e..e882d6697 100644 --- a/src/test/java/org/opensearch/knn/index/mapper/CompressionLevelTests.java +++ b/src/test/java/org/opensearch/knn/index/mapper/CompressionLevelTests.java @@ -45,83 +45,57 @@ public void testGetDefaultRescoreContext() { // Test rescore context for ON_DISK mode Mode mode = Mode.ON_DISK; - // Test various dimensions based on the updated oversampling logic - int belowThresholdDimension = 500; // A dimension below 768 - int between768and1000Dimension = 800; // A dimension between 768 and 1000 - int above1000Dimension = 1500; // A dimension above 1000 + int belowThresholdDimension = 500; // A dimension below the threshold + int aboveThresholdDimension = 1500; // A dimension above the threshold - // Compression level x32 with dimension < 768 should have an oversample factor of 3.0f + // x32 with dimension <= 1000 should have an oversample factor of 5.0f RescoreContext rescoreContext = CompressionLevel.x32.getDefaultRescoreContext(mode, belowThresholdDimension); assertNotNull(rescoreContext); - assertEquals(3.0f, rescoreContext.getOversampleFactor(), 0.0f); - - // Compression level x32 with dimension between 768 and 1000 should have an oversample factor of 2.0f - rescoreContext = CompressionLevel.x32.getDefaultRescoreContext(mode, between768and1000Dimension); - assertNotNull(rescoreContext); - assertEquals(2.0f, rescoreContext.getOversampleFactor(), 0.0f); + assertEquals(5.0f, rescoreContext.getOversampleFactor(), 0.0f); - // Compression level x32 with dimension > 1000 should have no oversampling (1.0f) - rescoreContext = CompressionLevel.x32.getDefaultRescoreContext(mode, above1000Dimension); - assertNotNull(rescoreContext); - assertEquals(1.0f, rescoreContext.getOversampleFactor(), 0.0f); - - // Compression level x16 with dimension < 768 should have an oversample factor of 3.0f - rescoreContext = CompressionLevel.x16.getDefaultRescoreContext(mode, belowThresholdDimension); + // x32 with dimension > 1000 should have an oversample factor of 3.0f + rescoreContext = CompressionLevel.x32.getDefaultRescoreContext(mode, aboveThresholdDimension); assertNotNull(rescoreContext); assertEquals(3.0f, rescoreContext.getOversampleFactor(), 0.0f); - // Compression level x16 with dimension between 768 and 1000 should have an oversample factor of 2.0f - rescoreContext = CompressionLevel.x16.getDefaultRescoreContext(mode, between768and1000Dimension); + // x16 with dimension <= 1000 should have an oversample factor of 5.0f + rescoreContext = CompressionLevel.x16.getDefaultRescoreContext(mode, belowThresholdDimension); assertNotNull(rescoreContext); - assertEquals(2.0f, rescoreContext.getOversampleFactor(), 0.0f); + assertEquals(5.0f, rescoreContext.getOversampleFactor(), 0.0f); - // Compression level x16 with dimension > 1000 should have no oversampling (1.0f) - rescoreContext = CompressionLevel.x16.getDefaultRescoreContext(mode, above1000Dimension); + // x16 with dimension > 1000 should have an oversample factor of 3.0f + rescoreContext = CompressionLevel.x16.getDefaultRescoreContext(mode, aboveThresholdDimension); assertNotNull(rescoreContext); - assertEquals(1.0f, rescoreContext.getOversampleFactor(), 0.0f); + assertEquals(3.0f, rescoreContext.getOversampleFactor(), 0.0f); - // Compression level x8 with dimension < 768 should have an oversample factor of 3.0f + // x8 with dimension <= 1000 should have an oversample factor of 5.0f rescoreContext = CompressionLevel.x8.getDefaultRescoreContext(mode, belowThresholdDimension); assertNotNull(rescoreContext); - assertEquals(3.0f, rescoreContext.getOversampleFactor(), 0.0f); - - // Compression level x8 with dimension between 768 and 1000 should have an oversample factor of 2.0f - rescoreContext = CompressionLevel.x8.getDefaultRescoreContext(mode, between768and1000Dimension); + assertEquals(5.0f, rescoreContext.getOversampleFactor(), 0.0f); + // x8 with dimension > 1000 should have an oversample factor of 2.0f + rescoreContext = CompressionLevel.x8.getDefaultRescoreContext(mode, aboveThresholdDimension); assertNotNull(rescoreContext); assertEquals(2.0f, rescoreContext.getOversampleFactor(), 0.0f); - // Compression level x8 with dimension > 1000 should have no oversampling (1.0f) - rescoreContext = CompressionLevel.x8.getDefaultRescoreContext(mode, above1000Dimension); - assertNotNull(rescoreContext); - assertEquals(1.0f, rescoreContext.getOversampleFactor(), 0.0f); - - // Compression level x4 with dimension < 768 should return null (no RescoreContext) + // x4 with dimension <= 1000 should have an oversample factor of 5.0f (though it doesn't have its own RescoreContext) rescoreContext = CompressionLevel.x4.getDefaultRescoreContext(mode, belowThresholdDimension); assertNull(rescoreContext); - - // Compression level x4 with dimension > 1000 should return null (no RescoreContext) - rescoreContext = CompressionLevel.x4.getDefaultRescoreContext(mode, above1000Dimension); + // x4 with dimension > 1000 should return null (no RescoreContext is configured for x4) + rescoreContext = CompressionLevel.x4.getDefaultRescoreContext(mode, aboveThresholdDimension); assertNull(rescoreContext); - - // Compression level x2 with dimension < 768 should return null + // Other compression levels should behave similarly with respect to dimension rescoreContext = CompressionLevel.x2.getDefaultRescoreContext(mode, belowThresholdDimension); assertNull(rescoreContext); - - // Compression level x2 with dimension > 1000 should return null - rescoreContext = CompressionLevel.x2.getDefaultRescoreContext(mode, above1000Dimension); + // x2 with dimension > 1000 should return null + rescoreContext = CompressionLevel.x2.getDefaultRescoreContext(mode, aboveThresholdDimension); assertNull(rescoreContext); - - // Compression level x1 with dimension < 768 should return null rescoreContext = CompressionLevel.x1.getDefaultRescoreContext(mode, belowThresholdDimension); assertNull(rescoreContext); - - // Compression level x1 with dimension > 1000 should return null - rescoreContext = CompressionLevel.x1.getDefaultRescoreContext(mode, above1000Dimension); + // x1 with dimension > 1000 should return null + rescoreContext = CompressionLevel.x1.getDefaultRescoreContext(mode, aboveThresholdDimension); assertNull(rescoreContext); - - // NOT_CONFIGURED mode should return null for any dimension + // NOT_CONFIGURED with dimension <= 1000 should return a RescoreContext with an oversample factor of 5.0f rescoreContext = CompressionLevel.NOT_CONFIGURED.getDefaultRescoreContext(mode, belowThresholdDimension); assertNull(rescoreContext); } - } diff --git a/src/test/java/org/opensearch/knn/index/query/KNNQueryBuilderTests.java b/src/test/java/org/opensearch/knn/index/query/KNNQueryBuilderTests.java index 3db03085b..b28b790d1 100644 --- a/src/test/java/org/opensearch/knn/index/query/KNNQueryBuilderTests.java +++ b/src/test/java/org/opensearch/knn/index/query/KNNQueryBuilderTests.java @@ -912,7 +912,8 @@ private void assertRescore(Version version, RescoreContext expectedRescoreContex } if (expectedRescoreContext != null) { - assertEquals(expectedRescoreContext, actualRescoreContext); + assertNotNull(actualRescoreContext); + assertEquals(expectedRescoreContext.getOversampleFactor(), actualRescoreContext.getOversampleFactor(), 0.0f); } } diff --git a/src/test/java/org/opensearch/knn/index/query/KNNWeightTests.java b/src/test/java/org/opensearch/knn/index/query/KNNWeightTests.java index f92f32406..2a2c3ed4d 100644 --- a/src/test/java/org/opensearch/knn/index/query/KNNWeightTests.java +++ b/src/test/java/org/opensearch/knn/index/query/KNNWeightTests.java @@ -79,6 +79,7 @@ import static java.util.Collections.emptyMap; import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyFloat; import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.ArgumentMatchers.anyLong; @@ -516,6 +517,111 @@ public void testANNWithFilterQuery_whenDoingANNBinary_thenSuccess() { validateANNWithFilterQuery_whenDoingANN_thenSuccess(true); } + @SneakyThrows + public void testScorerWithQuantizedVector() { + // Given + int k = 3; + byte[] quantizedVector = new byte[] { 1, 2, 3 }; // Mocked quantized vector + float[] queryVector = new float[] { 0.1f, 0.3f }; + + // Mock the JNI service to return KNNQueryResults + KNNQueryResult[] knnQueryResults = new KNNQueryResult[] { + new KNNQueryResult(1, 10.0f), // Mock result with id 1 and score 10 + new KNNQueryResult(2, 20.0f) // Mock result with id 2 and score 20 + }; + jniServiceMockedStatic.when( + () -> JNIService.queryBinaryIndex(anyLong(), eq(quantizedVector), eq(k), any(), any(), any(), anyInt(), any()) + ).thenReturn(knnQueryResults); + + KNNEngine knnEngine = mock(KNNEngine.class); + when(knnEngine.score(anyFloat(), eq(SpaceType.HAMMING))).thenAnswer(invocation -> { + Float score = invocation.getArgument(0); + return 1 / (1 + score); + }); + + // Build the KNNQuery object + final KNNQuery query = KNNQuery.builder() + .field(FIELD_NAME) + .queryVector(queryVector) + .k(k) + .indexName(INDEX_NAME) + .vectorDataType(VectorDataType.BINARY) // Simulate binary vector type for quantization + .build(); + + final float boost = 1.0F; + final KNNWeight knnWeight = new KNNWeight(query, boost); + + final LeafReaderContext leafReaderContext = mock(LeafReaderContext.class); + final SegmentReader reader = mock(SegmentReader.class); + when(leafReaderContext.reader()).thenReturn(reader); + + final FieldInfos fieldInfos = mock(FieldInfos.class); + final FieldInfo fieldInfo = mock(FieldInfo.class); + when(reader.getFieldInfos()).thenReturn(fieldInfos); + when(fieldInfos.fieldInfo(FIELD_NAME)).thenReturn(fieldInfo); + + when(fieldInfo.attributes()).thenReturn(Map.of(KNN_ENGINE, KNNEngine.FAISS.getName(), SPACE_TYPE, SpaceType.HAMMING.getValue())); + + FSDirectory directory = mock(FSDirectory.class); + when(reader.directory()).thenReturn(directory); + Path path = mock(Path.class); + when(directory.getDirectory()).thenReturn(path); + when(path.toString()).thenReturn("/fake/directory"); + + SegmentInfo segmentInfo = new SegmentInfo( + directory, // The directory where the segment is stored + Version.LATEST, // Lucene version + Version.LATEST, // Version of the segment info + "0", // Segment name + 100, // Max document count for this segment + false, // Is this a compound file segment + false, // Is this a merged segment + KNNCodecVersion.current().getDefaultCodecDelegate(), // Codec delegate for KNN + Map.of(), // Diagnostics map + new byte[StringHelper.ID_LENGTH], // Segment ID + Map.of(), // Attributes + Sort.RELEVANCE // Default sort order + ); + + final SegmentCommitInfo segmentCommitInfo = new SegmentCommitInfo(segmentInfo, 0, 0, 0, 0, 0, new byte[StringHelper.ID_LENGTH]); + + when(reader.getSegmentInfo()).thenReturn(segmentCommitInfo); + + try (MockedStatic knnCodecUtilMockedStatic = mockStatic(KNNCodecUtil.class)) { + List engineFiles = List.of("_0_1_target_field.faiss"); + knnCodecUtilMockedStatic.when(() -> KNNCodecUtil.getEngineFiles(anyString(), anyString(), eq(segmentInfo))) + .thenReturn(engineFiles); + + try (MockedStatic quantizationUtilMockedStatic = mockStatic(SegmentLevelQuantizationUtil.class)) { + quantizationUtilMockedStatic.when(() -> SegmentLevelQuantizationUtil.quantizeVector(any(), any())) + .thenReturn(quantizedVector); + + // When: Call the scorer method + final KNNScorer knnScorer = (KNNScorer) knnWeight.scorer(leafReaderContext); + + // Then: Ensure scorer is not null + assertNotNull(knnScorer); + + // Verify that JNIService.queryBinaryIndex is called with the quantized vector + jniServiceMockedStatic.verify( + () -> JNIService.queryBinaryIndex(anyLong(), eq(quantizedVector), eq(k), any(), any(), any(), anyInt(), any()), + times(1) + ); + + // Iterate over the results and ensure they are scored with SpaceType.HAMMING + final DocIdSetIterator docIdSetIterator = knnScorer.iterator(); + assertNotNull(docIdSetIterator); + while (docIdSetIterator.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) { + int docId = docIdSetIterator.docID(); + float expectedScore = knnEngine.score(knnQueryResults[docId - 1].getScore(), SpaceType.HAMMING); + float actualScore = knnScorer.score(); + // Check if the score is calculated using HAMMING + assertEquals(expectedScore, actualScore, 0.01f); // Tolerance for floating-point comparison + } + } + } + } + public void validateANNWithFilterQuery_whenDoingANN_thenSuccess(final boolean isBinary) throws IOException { // Given int k = 3; diff --git a/src/test/java/org/opensearch/knn/index/query/nativelib/NativeEngineKNNVectorQueryTests.java b/src/test/java/org/opensearch/knn/index/query/nativelib/NativeEngineKNNVectorQueryTests.java index 7fd96c6df..53873e15f 100644 --- a/src/test/java/org/opensearch/knn/index/query/nativelib/NativeEngineKNNVectorQueryTests.java +++ b/src/test/java/org/opensearch/knn/index/query/nativelib/NativeEngineKNNVectorQueryTests.java @@ -103,6 +103,8 @@ public void setUp() throws Exception { // Set ClusterService in KNNSettings KNNSettings.state().setClusterService(clusterService); + when(knnQuery.getQueryVector()).thenReturn(new float[] { 1.0f, 2.0f, 3.0f }); // Example vector + } @SneakyThrows @@ -166,7 +168,7 @@ public void testRescoreWhenShardLevelRescoringEnabled() { ) { // When shard-level re-scoring is enabled - mockedKnnSettings.when(() -> KNNSettings.isShardLevelRescoringDisabledForDiskBasedVector(any())).thenReturn(false); + mockedKnnSettings.when(() -> KNNSettings.isShardLevelRescoringEnabledForDiskBasedVector(any())).thenReturn(true); // Mock ResultUtil to return valid TopDocs mockedResultUtil.when(() -> ResultUtil.resultMapToTopDocs(any(), anyInt())) @@ -250,7 +252,7 @@ public void testRescore() { ) { // When shard-level re-scoring is enabled - mockedKnnSettings.when(() -> KNNSettings.isShardLevelRescoringDisabledForDiskBasedVector(any())).thenReturn(true); + mockedKnnSettings.when(() -> KNNSettings.isShardLevelRescoringEnabledForDiskBasedVector(any())).thenReturn(true); mockedResultUtil.when(() -> ResultUtil.reduceToTopK(any(), anyInt())).thenAnswer(InvocationOnMock::callRealMethod); mockedResultUtil.when(() -> ResultUtil.resultMapToTopDocs(eq(rescoredLeaf1Results), anyInt())).thenAnswer(t -> topDocs1); diff --git a/src/test/java/org/opensearch/knn/index/query/rescore/RescoreContextTests.java b/src/test/java/org/opensearch/knn/index/query/rescore/RescoreContextTests.java index fd94667db..2b309e4ab 100644 --- a/src/test/java/org/opensearch/knn/index/query/rescore/RescoreContextTests.java +++ b/src/test/java/org/opensearch/knn/index/query/rescore/RescoreContextTests.java @@ -14,47 +14,72 @@ public class RescoreContextTests extends KNNTestCase { public void testGetFirstPassK() { float oversample = 2.6f; - RescoreContext rescoreContext = RescoreContext.builder().oversampleFactor(oversample).build(); + RescoreContext rescoreContext = RescoreContext.builder().oversampleFactor(oversample).userProvided(true).build(); int finalK = 100; - assertEquals(260, rescoreContext.getFirstPassK(finalK)); - finalK = 1; - assertEquals(MIN_FIRST_PASS_RESULTS, rescoreContext.getFirstPassK(finalK)); - finalK = 0; - assertEquals(MIN_FIRST_PASS_RESULTS, rescoreContext.getFirstPassK(finalK)); - finalK = MAX_FIRST_PASS_RESULTS; - assertEquals(MAX_FIRST_PASS_RESULTS, rescoreContext.getFirstPassK(finalK)); - } - - public void testGetFirstPassKWithMinPassK() { - float oversample = 2.6f; - RescoreContext rescoreContext = RescoreContext.builder().oversampleFactor(oversample).build(); + boolean isShardLevelRescoringEnabled = true; + int dimension = 500; - // Case 1: Test with a finalK that results in a value greater than MIN_FIRST_PASS_RESULTS - int finalK = 100; - assertEquals(260, rescoreContext.getFirstPassK(finalK)); + // Case 1: Test with standard oversample factor when shard-level rescoring is enabled + assertEquals(260, rescoreContext.getFirstPassK(finalK, isShardLevelRescoringEnabled, dimension)); // Case 2: Test with a very small finalK that should result in a value less than MIN_FIRST_PASS_RESULTS finalK = 1; - assertEquals(MIN_FIRST_PASS_RESULTS, rescoreContext.getFirstPassK(finalK)); + assertEquals(MIN_FIRST_PASS_RESULTS, rescoreContext.getFirstPassK(finalK, isShardLevelRescoringEnabled, dimension)); - // Case 3: Test with finalK = 0, should return 0 + // Case 3: Test with finalK = 0, should return MIN_FIRST_PASS_RESULTS finalK = 0; - assertEquals(MIN_FIRST_PASS_RESULTS, rescoreContext.getFirstPassK(finalK)); + assertEquals(MIN_FIRST_PASS_RESULTS, rescoreContext.getFirstPassK(finalK, isShardLevelRescoringEnabled, dimension)); // Case 4: Test with finalK = MAX_FIRST_PASS_RESULTS, should cap at MAX_FIRST_PASS_RESULTS finalK = MAX_FIRST_PASS_RESULTS; - assertEquals(MAX_FIRST_PASS_RESULTS, rescoreContext.getFirstPassK(finalK)); + assertEquals(MAX_FIRST_PASS_RESULTS, rescoreContext.getFirstPassK(finalK, isShardLevelRescoringEnabled, dimension)); + } - // Case 5: Test where finalK * oversample is smaller than MIN_FIRST_PASS_RESULTS + public void testGetFirstPassKWithDimensionBasedOversampling() { + int finalK = 100; + int dimension; + + // Case 1: Test no oversampling for dimensions >= 1000 when shard-level rescoring is disabled + dimension = 1000; + RescoreContext rescoreContext = RescoreContext.builder().userProvided(false).build(); // Ensuring dimension-based logic applies + assertEquals(100, rescoreContext.getFirstPassK(finalK, false, dimension)); // No oversampling + + // Case 2: Test 2x oversampling for dimensions >= 768 but < 1000 when shard-level rescoring is disabled + dimension = 800; + rescoreContext = RescoreContext.builder().userProvided(false).build(); // Ensure previous values don't carry over + assertEquals(200, rescoreContext.getFirstPassK(finalK, false, dimension)); // 2x oversampling + + // Case 3: Test 3x oversampling for dimensions < 768 when shard-level rescoring is disabled + dimension = 700; + rescoreContext = RescoreContext.builder().userProvided(false).build(); // Ensure previous values don't carry over + assertEquals(300, rescoreContext.getFirstPassK(finalK, false, dimension)); // 3x oversampling + + // Case 4: Shard-level rescoring enabled, oversample factor should be used as provided by the user (ignore dimension) + rescoreContext = RescoreContext.builder().oversampleFactor(5.0f).userProvided(true).build(); // Provided by user + dimension = 500; + assertEquals(500, rescoreContext.getFirstPassK(finalK, true, dimension)); // User-defined oversample factor should be used + + // Case 5: Test finalK where oversampling factor results in a value less than MIN_FIRST_PASS_RESULTS finalK = 10; - oversample = 0.5f; // This will result in 5, which is less than MIN_FIRST_PASS_RESULTS - rescoreContext = RescoreContext.builder().oversampleFactor(oversample).build(); - assertEquals(MIN_FIRST_PASS_RESULTS, rescoreContext.getFirstPassK(finalK)); + dimension = 700; + rescoreContext = RescoreContext.builder().userProvided(false).build(); // Ensure dimension-based logic applies + assertEquals(100, rescoreContext.getFirstPassK(finalK, false, dimension)); // 3x oversampling results in 30 + } + + public void testGetFirstPassKWithMinPassK() { + float oversample = 0.5f; + RescoreContext rescoreContext = RescoreContext.builder().oversampleFactor(oversample).userProvided(true).build(); // User provided + boolean isShardLevelRescoringEnabled = false; + + // Case 1: Test where finalK * oversample is smaller than MIN_FIRST_PASS_RESULTS + int finalK = 10; + int dimension = 700; + assertEquals(MIN_FIRST_PASS_RESULTS, rescoreContext.getFirstPassK(finalK, isShardLevelRescoringEnabled, dimension)); - // Case 6: Test where finalK * oversample results in exactly MIN_FIRST_PASS_RESULTS + // Case 2: Test where finalK * oversample results in exactly MIN_FIRST_PASS_RESULTS finalK = 100; oversample = 1.0f; // This will result in exactly 100 (MIN_FIRST_PASS_RESULTS) - rescoreContext = RescoreContext.builder().oversampleFactor(oversample).build(); - assertEquals(MIN_FIRST_PASS_RESULTS, rescoreContext.getFirstPassK(finalK)); + rescoreContext = RescoreContext.builder().oversampleFactor(oversample).userProvided(true).build(); // User provided + assertEquals(MIN_FIRST_PASS_RESULTS, rescoreContext.getFirstPassK(finalK, isShardLevelRescoringEnabled, dimension)); } } From f9af48d59b8204dbec60694cb289908b52010b0b Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Wed, 9 Oct 2024 11:34:16 -0500 Subject: [PATCH 396/416] [Backport 2.x] Fix lucene codec after lucene version bumped to 9.12 (#2196) * Fix lucene codec after lucene version bumped to 9.12 (#2195) Signed-off-by: Navneet Verma (cherry picked from commit 33229126045b545c9f8e215e11b72b2335e287ae) * Update CHANGELOG.md Signed-off-by: Navneet Verma --------- Signed-off-by: Navneet Verma Co-authored-by: Navneet Verma --- CHANGELOG.md | 1 + .../codec/KNN9120Codec/KNN9120Codec.java | 61 +++++++++++++++++++ .../NativeEngineFieldVectorsWriter.java | 32 ++++++++-- .../NativeEngines990KnnVectorsWriter.java | 8 ++- .../knn/index/codec/KNNCodecVersion.java | 21 ++++++- ...KNNScalarQuantizedVectorsFormatParams.java | 14 ++++- .../services/org.apache.lucene.codecs.Codec | 1 + .../NativeEngineFieldVectorsWriterTests.java | 51 ++++++++++++---- ...eEngines990KnnVectorsWriterFlushTests.java | 17 ++++-- ...eEngines990KnnVectorsWriterMergeTests.java | 16 +++-- ...alarQuantizedVectorsFormatParamsTests.java | 53 +++++++++++++++- 11 files changed, 241 insertions(+), 34 deletions(-) create mode 100644 src/main/java/org/opensearch/knn/index/codec/KNN9120Codec/KNN9120Codec.java diff --git a/CHANGELOG.md b/CHANGELOG.md index d854f2f0c..3fac378ef 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,5 +34,6 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Documentation ### Maintenance * Remove benchmarks folder from k-NN repo [#2127](https://github.com/opensearch-project/k-NN/pull/2127) +* Fix lucene codec after lucene version bumped to 9.12. [#2195](https://github.com/opensearch-project/k-NN/pull/2195) ### Refactoring * Minor refactoring and refactored some unit test [#2167](https://github.com/opensearch-project/k-NN/pull/2167) diff --git a/src/main/java/org/opensearch/knn/index/codec/KNN9120Codec/KNN9120Codec.java b/src/main/java/org/opensearch/knn/index/codec/KNN9120Codec/KNN9120Codec.java new file mode 100644 index 000000000..a370197ec --- /dev/null +++ b/src/main/java/org/opensearch/knn/index/codec/KNN9120Codec/KNN9120Codec.java @@ -0,0 +1,61 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.codec.KNN9120Codec; + +import lombok.Builder; +import org.apache.lucene.codecs.Codec; +import org.apache.lucene.codecs.CompoundFormat; +import org.apache.lucene.codecs.DocValuesFormat; +import org.apache.lucene.codecs.FilterCodec; +import org.apache.lucene.codecs.KnnVectorsFormat; +import org.apache.lucene.codecs.perfield.PerFieldKnnVectorsFormat; +import org.opensearch.knn.index.codec.KNNCodecVersion; +import org.opensearch.knn.index.codec.KNNFormatFacade; + +/** + * KNN Codec that wraps the Lucene Codec which is part of Lucene 9.12 + */ +public class KNN9120Codec extends FilterCodec { + private static final KNNCodecVersion VERSION = KNNCodecVersion.V_9_12_0; + private final KNNFormatFacade knnFormatFacade; + private final PerFieldKnnVectorsFormat perFieldKnnVectorsFormat; + + /** + * No arg constructor that uses Lucene99 as the delegate + */ + public KNN9120Codec() { + this(VERSION.getDefaultCodecDelegate(), VERSION.getPerFieldKnnVectorsFormat()); + } + + /** + * Sole constructor. When subclassing this codec, create a no-arg ctor and pass the delegate codec + * and a unique name to this ctor. + * + * @param delegate codec that will perform all operations this codec does not override + * @param knnVectorsFormat per field format for KnnVector + */ + @Builder + protected KNN9120Codec(Codec delegate, PerFieldKnnVectorsFormat knnVectorsFormat) { + super(VERSION.getCodecName(), delegate); + knnFormatFacade = VERSION.getKnnFormatFacadeSupplier().apply(delegate); + perFieldKnnVectorsFormat = knnVectorsFormat; + } + + @Override + public DocValuesFormat docValuesFormat() { + return knnFormatFacade.docValuesFormat(); + } + + @Override + public CompoundFormat compoundFormat() { + return knnFormatFacade.compoundFormat(); + } + + @Override + public KnnVectorsFormat knnVectorsFormat() { + return perFieldKnnVectorsFormat; + } +} diff --git a/src/main/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngineFieldVectorsWriter.java b/src/main/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngineFieldVectorsWriter.java index 1abb84944..389c76e49 100644 --- a/src/main/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngineFieldVectorsWriter.java +++ b/src/main/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngineFieldVectorsWriter.java @@ -13,11 +13,13 @@ import lombok.Getter; import org.apache.lucene.codecs.KnnFieldVectorsWriter; +import org.apache.lucene.codecs.hnsw.FlatFieldVectorsWriter; import org.apache.lucene.index.DocsWithFieldSet; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.util.InfoStream; import org.apache.lucene.util.RamUsageEstimator; +import java.io.IOException; import java.util.HashMap; import java.util.Map; @@ -44,22 +46,37 @@ class NativeEngineFieldVectorsWriter extends KnnFieldVectorsWriter { @Getter private final DocsWithFieldSet docsWithField; private final InfoStream infoStream; + private final FlatFieldVectorsWriter flatFieldVectorsWriter; - static NativeEngineFieldVectorsWriter create(final FieldInfo fieldInfo, final InfoStream infoStream) { + @SuppressWarnings("unchecked") + static NativeEngineFieldVectorsWriter create( + final FieldInfo fieldInfo, + final FlatFieldVectorsWriter flatFieldVectorsWriter, + final InfoStream infoStream + ) { switch (fieldInfo.getVectorEncoding()) { case FLOAT32: - return new NativeEngineFieldVectorsWriter(fieldInfo, infoStream); + return new NativeEngineFieldVectorsWriter<>( + fieldInfo, + (FlatFieldVectorsWriter) flatFieldVectorsWriter, + infoStream + ); case BYTE: - return new NativeEngineFieldVectorsWriter(fieldInfo, infoStream); + return new NativeEngineFieldVectorsWriter<>(fieldInfo, (FlatFieldVectorsWriter) flatFieldVectorsWriter, infoStream); } throw new IllegalStateException("Unsupported Vector encoding : " + fieldInfo.getVectorEncoding()); } - private NativeEngineFieldVectorsWriter(final FieldInfo fieldInfo, final InfoStream infoStream) { + private NativeEngineFieldVectorsWriter( + final FieldInfo fieldInfo, + final FlatFieldVectorsWriter flatFieldVectorsWriter, + final InfoStream infoStream + ) { this.fieldInfo = fieldInfo; this.infoStream = infoStream; vectors = new HashMap<>(); this.docsWithField = new DocsWithFieldSet(); + this.flatFieldVectorsWriter = flatFieldVectorsWriter; } /** @@ -70,7 +87,7 @@ private NativeEngineFieldVectorsWriter(final FieldInfo fieldInfo, final InfoStre * @param vectorValue T */ @Override - public void addValue(int docID, T vectorValue) { + public void addValue(int docID, T vectorValue) throws IOException { if (docID == lastDocID) { throw new IllegalArgumentException( "[NativeEngineKNNVectorWriter]VectorValuesField \"" @@ -81,6 +98,8 @@ public void addValue(int docID, T vectorValue) { // TODO: we can build the graph here too iteratively. but right now I am skipping that as we need iterative // graph build support on the JNI layer. assert docID > lastDocID; + // ensuring that vector is provided to flatFieldWriter. + flatFieldVectorsWriter.addValue(docID, vectorValue); vectors.put(docID, vectorValue); docsWithField.add(docID); lastDocID = docID; @@ -105,6 +124,7 @@ public long ramBytesUsed() { return SHALLOW_SIZE + docsWithField.ramBytesUsed() + (long) this.vectors.size() * (long) (RamUsageEstimator.NUM_BYTES_OBJECT_REF + RamUsageEstimator.NUM_BYTES_ARRAY_HEADER) + (long) this.vectors.size() * RamUsageEstimator.shallowSizeOfInstance( Integer.class - ) + (long) vectors.size() * fieldInfo.getVectorDimension() * fieldInfo.getVectorEncoding().byteSize; + ) + (long) vectors.size() * fieldInfo.getVectorDimension() * fieldInfo.getVectorEncoding().byteSize + flatFieldVectorsWriter + .ramBytesUsed(); } } diff --git a/src/main/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsWriter.java b/src/main/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsWriter.java index 2f22565c9..eccad41c8 100644 --- a/src/main/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsWriter.java +++ b/src/main/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsWriter.java @@ -65,9 +65,13 @@ public NativeEngines990KnnVectorsWriter(SegmentWriteState segmentWriteState, Fla */ @Override public KnnFieldVectorsWriter addField(final FieldInfo fieldInfo) throws IOException { - final NativeEngineFieldVectorsWriter newField = NativeEngineFieldVectorsWriter.create(fieldInfo, segmentWriteState.infoStream); + final NativeEngineFieldVectorsWriter newField = NativeEngineFieldVectorsWriter.create( + fieldInfo, + flatVectorsWriter.addField(fieldInfo), + segmentWriteState.infoStream + ); fields.add(newField); - return flatVectorsWriter.addField(fieldInfo, newField); + return newField; } /** diff --git a/src/main/java/org/opensearch/knn/index/codec/KNNCodecVersion.java b/src/main/java/org/opensearch/knn/index/codec/KNNCodecVersion.java index 505dd50a5..fb9af0109 100644 --- a/src/main/java/org/opensearch/knn/index/codec/KNNCodecVersion.java +++ b/src/main/java/org/opensearch/knn/index/codec/KNNCodecVersion.java @@ -12,12 +12,14 @@ import org.apache.lucene.codecs.Codec; import org.apache.lucene.backward_codecs.lucene94.Lucene94Codec; import org.apache.lucene.backward_codecs.lucene95.Lucene95Codec; -import org.apache.lucene.codecs.lucene99.Lucene99Codec; +import org.apache.lucene.backward_codecs.lucene99.Lucene99Codec; +import org.apache.lucene.codecs.lucene912.Lucene912Codec; import org.apache.lucene.codecs.perfield.PerFieldKnnVectorsFormat; import org.opensearch.index.mapper.MapperService; import org.opensearch.knn.index.codec.KNN80Codec.KNN80CompoundFormat; import org.opensearch.knn.index.codec.KNN80Codec.KNN80DocValuesFormat; import org.opensearch.knn.index.codec.KNN910Codec.KNN910Codec; +import org.opensearch.knn.index.codec.KNN9120Codec.KNN9120Codec; import org.opensearch.knn.index.codec.KNN920Codec.KNN920Codec; import org.opensearch.knn.index.codec.KNN920Codec.KNN920PerFieldKnnVectorsFormat; import org.opensearch.knn.index.codec.KNN940Codec.KNN940Codec; @@ -110,9 +112,24 @@ public enum KNNCodecVersion { .knnVectorsFormat(new KNN990PerFieldKnnVectorsFormat(Optional.ofNullable(mapperService))) .build(), KNN990Codec::new + ), + + V_9_12_0( + "KNN9120Codec", + new Lucene912Codec(), + new KNN990PerFieldKnnVectorsFormat(Optional.empty()), + (delegate) -> new KNNFormatFacade( + new KNN80DocValuesFormat(delegate.docValuesFormat()), + new KNN80CompoundFormat(delegate.compoundFormat()) + ), + (userCodec, mapperService) -> KNN9120Codec.builder() + .delegate(userCodec) + .knnVectorsFormat(new KNN990PerFieldKnnVectorsFormat(Optional.ofNullable(mapperService))) + .build(), + KNN9120Codec::new ); - private static final KNNCodecVersion CURRENT = V_9_9_0; + private static final KNNCodecVersion CURRENT = V_9_12_0; private final String codecName; private final Codec defaultCodecDelegate; diff --git a/src/main/java/org/opensearch/knn/index/codec/params/KNNScalarQuantizedVectorsFormatParams.java b/src/main/java/org/opensearch/knn/index/codec/params/KNNScalarQuantizedVectorsFormatParams.java index e2d31183b..3498119c1 100644 --- a/src/main/java/org/opensearch/knn/index/codec/params/KNNScalarQuantizedVectorsFormatParams.java +++ b/src/main/java/org/opensearch/knn/index/codec/params/KNNScalarQuantizedVectorsFormatParams.java @@ -31,7 +31,8 @@ public KNNScalarQuantizedVectorsFormatParams(Map params, int def Map sqEncoderParams = encoderMethodComponentContext.getParameters(); this.initConfidenceInterval(sqEncoderParams); this.initBits(sqEncoderParams); - this.initCompressFlag(); + // compression flag should be set after bits has been initialised as compressionFlag depends on bits. + this.setCompressionFlag(); } @Override @@ -76,7 +77,14 @@ private void initBits(final Map params) { this.bits = LUCENE_SQ_DEFAULT_BITS; } - private void initCompressFlag() { - this.compressFlag = true; + private void setCompressionFlag() { + if (this.bits <= 0) { + throw new IllegalArgumentException( + "Either bits are set to less than 0 or they have not been initialized." + " Bit value: " + this.bits + ); + } + // This check is coming from Lucene. Code ref: + // https://github.com/apache/lucene/blob/branch_9_12/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99ScalarQuantizedVectorsFormat.java#L113-L116 + this.compressFlag = this.bits <= 4; } } diff --git a/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec b/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec index 401b094b2..7a8916981 100644 --- a/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec +++ b/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec @@ -7,4 +7,5 @@ org.opensearch.knn.index.codec.KNN920Codec.KNN920Codec org.opensearch.knn.index.codec.KNN940Codec.KNN940Codec org.opensearch.knn.index.codec.KNN950Codec.KNN950Codec org.opensearch.knn.index.codec.KNN990Codec.KNN990Codec +org.opensearch.knn.index.codec.KNN9120Codec.KNN9120Codec org.opensearch.knn.index.codec.KNN990Codec.UnitTestCodec diff --git a/src/test/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngineFieldVectorsWriterTests.java b/src/test/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngineFieldVectorsWriterTests.java index 29e3531cf..6e6a51b88 100644 --- a/src/test/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngineFieldVectorsWriterTests.java +++ b/src/test/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngineFieldVectorsWriterTests.java @@ -11,6 +11,8 @@ package org.opensearch.knn.index.codec.KNN990Codec; +import lombok.SneakyThrows; +import org.apache.lucene.codecs.hnsw.FlatFieldVectorsWriter; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.VectorEncoding; import org.apache.lucene.util.InfoStream; @@ -21,82 +23,109 @@ public class NativeEngineFieldVectorsWriterTests extends KNNCodecTestCase { @SuppressWarnings("unchecked") + @SneakyThrows public void testCreate_ForDifferentInputs_thenSuccess() { final FieldInfo fieldInfo = Mockito.mock(FieldInfo.class); Mockito.when(fieldInfo.getVectorEncoding()).thenReturn(VectorEncoding.FLOAT32); + final FlatFieldVectorsWriter mockedFlatFieldVectorsWriter = Mockito.mock(FlatFieldVectorsWriter.class); NativeEngineFieldVectorsWriter floatWriter = (NativeEngineFieldVectorsWriter) NativeEngineFieldVectorsWriter - .create(fieldInfo, InfoStream.getDefault()); - floatWriter.addValue(1, new float[] { 1.0f, 2.0f }); + .create(fieldInfo, mockedFlatFieldVectorsWriter, InfoStream.getDefault()); + final float[] floatVector = new float[] { 1.0f, 2.0f }; + floatWriter.addValue(1, floatVector); + Mockito.doNothing().when(mockedFlatFieldVectorsWriter).addValue(1, floatVector); Mockito.verify(fieldInfo).getVectorEncoding(); + Mockito.verify(mockedFlatFieldVectorsWriter).addValue(1, floatVector); + final byte[] byteVector = new byte[] { 1, 2 }; + final FlatFieldVectorsWriter mockedFlatFieldByteVectorsWriter = Mockito.mock(FlatFieldVectorsWriter.class); + Mockito.doNothing().when(mockedFlatFieldByteVectorsWriter).addValue(1, byteVector); Mockito.when(fieldInfo.getVectorEncoding()).thenReturn(VectorEncoding.BYTE); NativeEngineFieldVectorsWriter byteWriter = (NativeEngineFieldVectorsWriter) NativeEngineFieldVectorsWriter.create( fieldInfo, + mockedFlatFieldByteVectorsWriter, InfoStream.getDefault() ); Assert.assertNotNull(byteWriter); Mockito.verify(fieldInfo, Mockito.times(2)).getVectorEncoding(); - byteWriter.addValue(1, new byte[] { 1, 2 }); + byteWriter.addValue(1, byteVector); + Mockito.verify(mockedFlatFieldByteVectorsWriter).addValue(1, byteVector); } @SuppressWarnings("unchecked") + @SneakyThrows public void testAddValue_ForDifferentInputs_thenSuccess() { final FieldInfo fieldInfo = Mockito.mock(FieldInfo.class); Mockito.when(fieldInfo.getVectorEncoding()).thenReturn(VectorEncoding.FLOAT32); - final NativeEngineFieldVectorsWriter floatWriter = (NativeEngineFieldVectorsWriter) NativeEngineFieldVectorsWriter - .create(fieldInfo, InfoStream.getDefault()); + final FlatFieldVectorsWriter mockedFlatFieldVectorsWriter = Mockito.mock(FlatFieldVectorsWriter.class); final float[] vec1 = new float[] { 1.0f, 2.0f }; final float[] vec2 = new float[] { 2.0f, 2.0f }; + Mockito.doNothing().when(mockedFlatFieldVectorsWriter).addValue(1, vec1); + Mockito.doNothing().when(mockedFlatFieldVectorsWriter).addValue(2, vec2); + final NativeEngineFieldVectorsWriter floatWriter = (NativeEngineFieldVectorsWriter) NativeEngineFieldVectorsWriter + .create(fieldInfo, mockedFlatFieldVectorsWriter, InfoStream.getDefault()); floatWriter.addValue(1, vec1); floatWriter.addValue(2, vec2); + Mockito.verify(mockedFlatFieldVectorsWriter).addValue(1, vec1); + Mockito.verify(mockedFlatFieldVectorsWriter).addValue(2, vec2); Assert.assertEquals(vec1, floatWriter.getVectors().get(1)); Assert.assertEquals(vec2, floatWriter.getVectors().get(2)); Mockito.verify(fieldInfo).getVectorEncoding(); Mockito.when(fieldInfo.getVectorEncoding()).thenReturn(VectorEncoding.BYTE); - final NativeEngineFieldVectorsWriter byteWriter = (NativeEngineFieldVectorsWriter) NativeEngineFieldVectorsWriter - .create(fieldInfo, InfoStream.getDefault()); + final FlatFieldVectorsWriter mockedFlatFieldByteVectorsWriter = Mockito.mock(FlatFieldVectorsWriter.class); final byte[] bvec1 = new byte[] { 1, 2 }; final byte[] bvec2 = new byte[] { 2, 2 }; + Mockito.doNothing().when(mockedFlatFieldByteVectorsWriter).addValue(1, bvec1); + Mockito.doNothing().when(mockedFlatFieldByteVectorsWriter).addValue(2, bvec2); + final NativeEngineFieldVectorsWriter byteWriter = (NativeEngineFieldVectorsWriter) NativeEngineFieldVectorsWriter + .create(fieldInfo, mockedFlatFieldByteVectorsWriter, InfoStream.getDefault()); byteWriter.addValue(1, bvec1); byteWriter.addValue(2, bvec2); Assert.assertEquals(bvec1, byteWriter.getVectors().get(1)); Assert.assertEquals(bvec2, byteWriter.getVectors().get(2)); Mockito.verify(fieldInfo, Mockito.times(2)).getVectorEncoding(); + Mockito.verify(mockedFlatFieldByteVectorsWriter).addValue(1, bvec1); + Mockito.verify(mockedFlatFieldByteVectorsWriter).addValue(2, bvec2); } @SuppressWarnings("unchecked") + @SneakyThrows public void testCopyValue_whenValidInput_thenException() { final FieldInfo fieldInfo = Mockito.mock(FieldInfo.class); + FlatFieldVectorsWriter mockedFlatFieldVectorsWriter = Mockito.mock(FlatFieldVectorsWriter.class); Mockito.when(fieldInfo.getVectorEncoding()).thenReturn(VectorEncoding.FLOAT32); final NativeEngineFieldVectorsWriter floatWriter = (NativeEngineFieldVectorsWriter) NativeEngineFieldVectorsWriter - .create(fieldInfo, InfoStream.getDefault()); + .create(fieldInfo, mockedFlatFieldVectorsWriter, InfoStream.getDefault()); expectThrows(UnsupportedOperationException.class, () -> floatWriter.copyValue(new float[3])); Mockito.when(fieldInfo.getVectorEncoding()).thenReturn(VectorEncoding.BYTE); final NativeEngineFieldVectorsWriter byteWriter = (NativeEngineFieldVectorsWriter) NativeEngineFieldVectorsWriter - .create(fieldInfo, InfoStream.getDefault()); + .create(fieldInfo, mockedFlatFieldVectorsWriter, InfoStream.getDefault()); expectThrows(UnsupportedOperationException.class, () -> byteWriter.copyValue(new byte[3])); } @SuppressWarnings("unchecked") + @SneakyThrows public void testRamByteUsed_whenValidInput_thenSuccess() { final FieldInfo fieldInfo = Mockito.mock(FieldInfo.class); Mockito.when(fieldInfo.getVectorEncoding()).thenReturn(VectorEncoding.FLOAT32); Mockito.when(fieldInfo.getVectorDimension()).thenReturn(2); + FlatFieldVectorsWriter mockedFlatFieldVectorsWriter = Mockito.mock(FlatFieldVectorsWriter.class); + Mockito.when(mockedFlatFieldVectorsWriter.ramBytesUsed()).thenReturn(1L); final NativeEngineFieldVectorsWriter floatWriter = (NativeEngineFieldVectorsWriter) NativeEngineFieldVectorsWriter - .create(fieldInfo, InfoStream.getDefault()); + .create(fieldInfo, mockedFlatFieldVectorsWriter, InfoStream.getDefault()); // testing for value > 0 as we don't have a concrete way to find out expected bytes. This can OS dependent too. Assert.assertTrue(floatWriter.ramBytesUsed() > 0); Mockito.when(fieldInfo.getVectorEncoding()).thenReturn(VectorEncoding.BYTE); final NativeEngineFieldVectorsWriter byteWriter = (NativeEngineFieldVectorsWriter) NativeEngineFieldVectorsWriter - .create(fieldInfo, InfoStream.getDefault()); + .create(fieldInfo, mockedFlatFieldVectorsWriter, InfoStream.getDefault()); // testing for value > 0 as we don't have a concrete way to find out expected bytes. This can OS dependent too. Assert.assertTrue(byteWriter.ramBytesUsed() > 0); + Mockito.verify(mockedFlatFieldVectorsWriter, Mockito.times(2)).ramBytesUsed(); } } diff --git a/src/test/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsWriterFlushTests.java b/src/test/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsWriterFlushTests.java index 9f74b2c10..5bb6d1926 100644 --- a/src/test/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsWriterFlushTests.java +++ b/src/test/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsWriterFlushTests.java @@ -8,6 +8,7 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; import lombok.RequiredArgsConstructor; import lombok.SneakyThrows; +import org.apache.lucene.codecs.hnsw.FlatFieldVectorsWriter; import org.apache.lucene.codecs.hnsw.FlatVectorsWriter; import org.apache.lucene.index.DocsWithFieldSet; import org.apache.lucene.index.FieldInfo; @@ -16,6 +17,7 @@ import org.mockito.Mock; import org.mockito.MockedConstruction; import org.mockito.MockedStatic; +import org.mockito.Mockito; import org.mockito.MockitoAnnotations; import org.opensearch.knn.common.KNNConstants; import org.opensearch.knn.index.VectorDataType; @@ -68,6 +70,8 @@ public class NativeEngines990KnnVectorsWriterFlushTests extends OpenSearchTestCa @Mock private NativeIndexWriter nativeIndexWriter; + private FlatFieldVectorsWriter mockedFlatFieldVectorsWriter; + private NativeEngines990KnnVectorsWriter objectUnderTest; private final String description; @@ -78,6 +82,9 @@ public void setUp() throws Exception { super.setUp(); MockitoAnnotations.openMocks(this); objectUnderTest = new NativeEngines990KnnVectorsWriter(segmentWriteState, flatVectorsWriter); + mockedFlatFieldVectorsWriter = Mockito.mock(FlatFieldVectorsWriter.class); + Mockito.doNothing().when(mockedFlatFieldVectorsWriter).addValue(Mockito.anyInt(), Mockito.any()); + Mockito.when(flatVectorsWriter.addField(Mockito.any())).thenReturn(mockedFlatFieldVectorsWriter); } @ParametersFactory @@ -139,8 +146,9 @@ public void testFlush() { ); NativeEngineFieldVectorsWriter field = nativeEngineFieldVectorsWriter(fieldInfo, vectorsPerField.get(i)); - fieldWriterMockedStatic.when(() -> NativeEngineFieldVectorsWriter.create(fieldInfo, segmentWriteState.infoStream)) - .thenReturn(field); + fieldWriterMockedStatic.when( + () -> NativeEngineFieldVectorsWriter.create(fieldInfo, mockedFlatFieldVectorsWriter, segmentWriteState.infoStream) + ).thenReturn(field); try { objectUnderTest.addField(fieldInfo); @@ -227,8 +235,9 @@ public void testFlush_WithQuantization() { ); NativeEngineFieldVectorsWriter field = nativeEngineFieldVectorsWriter(fieldInfo, vectorsPerField.get(i)); - fieldWriterMockedStatic.when(() -> NativeEngineFieldVectorsWriter.create(fieldInfo, segmentWriteState.infoStream)) - .thenReturn(field); + fieldWriterMockedStatic.when( + () -> NativeEngineFieldVectorsWriter.create(fieldInfo, mockedFlatFieldVectorsWriter, segmentWriteState.infoStream) + ).thenReturn(field); try { objectUnderTest.addField(fieldInfo); diff --git a/src/test/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsWriterMergeTests.java b/src/test/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsWriterMergeTests.java index 41940c4d4..af18cd281 100644 --- a/src/test/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsWriterMergeTests.java +++ b/src/test/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsWriterMergeTests.java @@ -9,6 +9,7 @@ import lombok.RequiredArgsConstructor; import lombok.SneakyThrows; import org.apache.lucene.codecs.KnnVectorsWriter; +import org.apache.lucene.codecs.hnsw.FlatFieldVectorsWriter; import org.apache.lucene.codecs.hnsw.FlatVectorsWriter; import org.apache.lucene.index.DocsWithFieldSet; import org.apache.lucene.index.FieldInfo; @@ -19,6 +20,7 @@ import org.mockito.Mock; import org.mockito.MockedConstruction; import org.mockito.MockedStatic; +import org.mockito.Mockito; import org.mockito.MockitoAnnotations; import org.opensearch.knn.common.KNNConstants; import org.opensearch.knn.index.VectorDataType; @@ -74,12 +76,16 @@ public class NativeEngines990KnnVectorsWriterMergeTests extends OpenSearchTestCa private final String description; private final Map mergedVectors; + private FlatFieldVectorsWriter mockedFlatFieldVectorsWriter; @Override public void setUp() throws Exception { super.setUp(); MockitoAnnotations.openMocks(this); objectUnderTest = new NativeEngines990KnnVectorsWriter(segmentWriteState, flatVectorsWriter); + mockedFlatFieldVectorsWriter = Mockito.mock(FlatFieldVectorsWriter.class); + Mockito.doNothing().when(mockedFlatFieldVectorsWriter).addValue(Mockito.anyInt(), Mockito.any()); + Mockito.when(flatVectorsWriter.addField(Mockito.any())).thenReturn(mockedFlatFieldVectorsWriter); } @ParametersFactory @@ -120,8 +126,9 @@ public void testMerge() { ); NativeEngineFieldVectorsWriter field = nativeEngineFieldVectorsWriter(fieldInfo, mergedVectors); - fieldWriterMockedStatic.when(() -> NativeEngineFieldVectorsWriter.create(fieldInfo, segmentWriteState.infoStream)) - .thenReturn(field); + fieldWriterMockedStatic.when( + () -> NativeEngineFieldVectorsWriter.create(fieldInfo, mockedFlatFieldVectorsWriter, segmentWriteState.infoStream) + ).thenReturn(field); mergedVectorValuesMockedStatic.when(() -> KnnVectorsWriter.MergedVectorValues.mergeFloatVectorValues(fieldInfo, mergeState)) .thenReturn(floatVectorValues); @@ -184,8 +191,9 @@ public void testMerge_WithQuantization() { ); NativeEngineFieldVectorsWriter field = nativeEngineFieldVectorsWriter(fieldInfo, mergedVectors); - fieldWriterMockedStatic.when(() -> NativeEngineFieldVectorsWriter.create(fieldInfo, segmentWriteState.infoStream)) - .thenReturn(field); + fieldWriterMockedStatic.when( + () -> NativeEngineFieldVectorsWriter.create(fieldInfo, mockedFlatFieldVectorsWriter, segmentWriteState.infoStream) + ).thenReturn(field); mergedVectorValuesMockedStatic.when(() -> KnnVectorsWriter.MergedVectorValues.mergeFloatVectorValues(fieldInfo, mergeState)) .thenReturn(floatVectorValues); diff --git a/src/test/java/org/opensearch/knn/index/codec/params/KNNScalarQuantizedVectorsFormatParamsTests.java b/src/test/java/org/opensearch/knn/index/codec/params/KNNScalarQuantizedVectorsFormatParamsTests.java index 573e826e0..b7394b06a 100644 --- a/src/test/java/org/opensearch/knn/index/codec/params/KNNScalarQuantizedVectorsFormatParamsTests.java +++ b/src/test/java/org/opensearch/knn/index/codec/params/KNNScalarQuantizedVectorsFormatParamsTests.java @@ -12,12 +12,14 @@ package org.opensearch.knn.index.codec.params; import junit.framework.TestCase; +import org.junit.Assert; import org.opensearch.knn.index.engine.MethodComponentContext; import java.util.HashMap; import java.util.Map; import static org.opensearch.knn.common.KNNConstants.ENCODER_SQ; +import static org.opensearch.knn.common.KNNConstants.LUCENE_SQ_BITS; import static org.opensearch.knn.common.KNNConstants.LUCENE_SQ_CONFIDENCE_INTERVAL; import static org.opensearch.knn.common.KNNConstants.LUCENE_SQ_DEFAULT_BITS; import static org.opensearch.knn.common.KNNConstants.METHOD_ENCODER_PARAMETER; @@ -39,7 +41,7 @@ public void testInitParams_whenCalled_thenReturnDefaultParams() { assertEquals(DEFAULT_MAX_CONNECTIONS, knnScalarQuantizedVectorsFormatParams.getMaxConnections()); assertEquals(DEFAULT_BEAM_WIDTH, knnScalarQuantizedVectorsFormatParams.getBeamWidth()); assertNull(knnScalarQuantizedVectorsFormatParams.getConfidenceInterval()); - assertTrue(knnScalarQuantizedVectorsFormatParams.isCompressFlag()); + assertFalse(knnScalarQuantizedVectorsFormatParams.isCompressFlag()); assertEquals(LUCENE_SQ_DEFAULT_BITS, knnScalarQuantizedVectorsFormatParams.getBits()); } @@ -65,10 +67,57 @@ public void testInitParams_whenCalled_thenReturnParams() { assertEquals(m, knnScalarQuantizedVectorsFormatParams.getMaxConnections()); assertEquals(efConstruction, knnScalarQuantizedVectorsFormatParams.getBeamWidth()); assertEquals((float) MINIMUM_CONFIDENCE_INTERVAL, knnScalarQuantizedVectorsFormatParams.getConfidenceInterval()); - assertTrue(knnScalarQuantizedVectorsFormatParams.isCompressFlag()); + assertFalse(knnScalarQuantizedVectorsFormatParams.isCompressFlag()); assertEquals(LUCENE_SQ_DEFAULT_BITS, knnScalarQuantizedVectorsFormatParams.getBits()); } + public void testInitParams_whenBitsIs4_thenReturnParams() { + int m = 64; + int efConstruction = 128; + + Map encoderParams = new HashMap<>(); + encoderParams.put(LUCENE_SQ_CONFIDENCE_INTERVAL, MINIMUM_CONFIDENCE_INTERVAL); + encoderParams.put(LUCENE_SQ_BITS, 4); + MethodComponentContext encoderComponentContext = new MethodComponentContext(ENCODER_SQ, encoderParams); + + Map params = new HashMap<>(); + params.put(METHOD_ENCODER_PARAMETER, encoderComponentContext); + params.put(METHOD_PARAMETER_M, m); + params.put(METHOD_PARAMETER_EF_CONSTRUCTION, efConstruction); + + KNNScalarQuantizedVectorsFormatParams knnScalarQuantizedVectorsFormatParams = new KNNScalarQuantizedVectorsFormatParams( + params, + DEFAULT_MAX_CONNECTIONS, + DEFAULT_BEAM_WIDTH + ); + + assertEquals(m, knnScalarQuantizedVectorsFormatParams.getMaxConnections()); + assertEquals(efConstruction, knnScalarQuantizedVectorsFormatParams.getBeamWidth()); + assertEquals((float) MINIMUM_CONFIDENCE_INTERVAL, knnScalarQuantizedVectorsFormatParams.getConfidenceInterval()); + assertTrue(knnScalarQuantizedVectorsFormatParams.isCompressFlag()); + assertEquals(4, knnScalarQuantizedVectorsFormatParams.getBits()); + } + + public void testInitParams_whenBitsIs0_thenThrowException() { + int m = 64; + int efConstruction = 128; + + Map encoderParams = new HashMap<>(); + encoderParams.put(LUCENE_SQ_CONFIDENCE_INTERVAL, MINIMUM_CONFIDENCE_INTERVAL); + encoderParams.put(LUCENE_SQ_BITS, 0); + MethodComponentContext encoderComponentContext = new MethodComponentContext(ENCODER_SQ, encoderParams); + + Map params = new HashMap<>(); + params.put(METHOD_ENCODER_PARAMETER, encoderComponentContext); + params.put(METHOD_PARAMETER_M, m); + params.put(METHOD_PARAMETER_EF_CONSTRUCTION, efConstruction); + + Assert.assertThrows( + IllegalArgumentException.class, + () -> new KNNScalarQuantizedVectorsFormatParams(params, DEFAULT_MAX_CONNECTIONS, DEFAULT_BEAM_WIDTH) + ); + } + public void testValidate_whenCalled_thenReturnTrue() { Map params = getDefaultParamsForConstructor(); KNNScalarQuantizedVectorsFormatParams knnScalarQuantizedVectorsFormatParams = new KNNScalarQuantizedVectorsFormatParams( From 5900aa14f0c94c0bf270c658da49340daf893054 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Wed, 9 Oct 2024 13:40:41 -0500 Subject: [PATCH 397/416] Fix changelog (#2198) (#2199) --- CHANGELOG.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3fac378ef..a46185ef2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,33 +7,33 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ## [Unreleased 3.0](https://github.com/opensearch-project/k-NN/compare/2.x...HEAD) ### Features ### Enhancements -* Introducing a loading layer in FAISS [#2033](https://github.com/opensearch-project/k-NN/issues/2033) -### Bug Fixes -* Add DocValuesProducers for releasing memory when close index [#1946](https://github.com/opensearch-project/k-NN/pull/1946) +### Bug Fixes ### Infrastructure ### Documentation -* Fix sed command in DEVELOPER_GUIDE.md to append a new line character '\n'. [#2181](https://github.com/opensearch-project/k-NN/pull/2181) ### Maintenance ### Refactoring -* Does not create additional KNNVectorValues in NativeEngines990KNNVectorWriter when quantization is not needed [#2133](https://github.com/opensearch-project/k-NN/pull/2133) ## [Unreleased 2.x](https://github.com/opensearch-project/k-NN/compare/2.17...2.x) ### Features * Add AVX512 support to k-NN for FAISS library [#2069](https://github.com/opensearch-project/k-NN/pull/2069) ### Enhancements +* Introducing a loading layer in FAISS [#2033](https://github.com/opensearch-project/k-NN/issues/2033) * Add short circuit if no live docs are in segments [#2059](https://github.com/opensearch-project/k-NN/pull/2059) * Optimize reduceToTopK in ResultUtil by removing pre-filling and reducing peek calls [#2146](https://github.com/opensearch-project/k-NN/pull/2146) * Update Default Rescore Context based on Dimension [#2149](https://github.com/opensearch-project/k-NN/pull/2149) * KNNIterators should support with and without filters [#2155](https://github.com/opensearch-project/k-NN/pull/2155) * Adding Support to Enable/Disble Share level Rescoring and Update Oversampling Factor[#2172](https://github.com/opensearch-project/k-NN/pull/2172) ### Bug Fixes +* Add DocValuesProducers for releasing memory when close index [#1946](https://github.com/opensearch-project/k-NN/pull/1946) * KNN80DocValues should only be considered for BinaryDocValues fields [#2147](https://github.com/opensearch-project/k-NN/pull/2147) * Score Fix for Binary Quantized Vector and Setting Default value in case of shard level rescoring is disabled for oversampling factor[#2183](https://github.com/opensearch-project/k-NN/pull/2183) * Java Docs Fix For 2.x ### Infrastructure ### Documentation +* Fix sed command in DEVELOPER_GUIDE.md to append a new line character '\n'. [#2181](https://github.com/opensearch-project/k-NN/pull/2181) ### Maintenance * Remove benchmarks folder from k-NN repo [#2127](https://github.com/opensearch-project/k-NN/pull/2127) * Fix lucene codec after lucene version bumped to 9.12. [#2195](https://github.com/opensearch-project/k-NN/pull/2195) ### Refactoring +* Does not create additional KNNVectorValues in NativeEngines990KNNVectorWriter when quantization is not needed [#2133](https://github.com/opensearch-project/k-NN/pull/2133) * Minor refactoring and refactored some unit test [#2167](https://github.com/opensearch-project/k-NN/pull/2167) From 425687bbae0c84d00925bf73fa87042d525dfdc7 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Thu, 10 Oct 2024 10:18:47 -0700 Subject: [PATCH 398/416] [Backport 2.x] Add support to build vector data structures greedily and perform exact search when there are no engine files (#2201) * Add support to build vector data structures greedily and perform exact search when there are no engine files (#2188) * Introduce new setting to configure when to build graph during segment creation (#2007) Added new updatable index setting "build_vector_data_structure_threshold", which will be considered when to build braph or not for native engines. This is noop for lucene. This depends on use lucene format as prerequisite. We don't need to add flag since it is only enable if lucene format is already enabled. Signed-off-by: Vijayan Balasubramanian * Add integration test for binary vector values (#2142) Signed-off-by: Vijayan Balasubramanian * Allow build graph greedily for quantization scenarios (#2175) Previosuly we only added support to build greedily for non quantization scenario. In this commit, we can remove that constraint, however, we cannot skip writing quanitization state since it is required irrespective of type of search is executed later. Signed-off-by: Vijayan Balasubramanian * Add exact search if no native engine files are available (#2136) * Add exact search if no engine files are in segments When graph is not available, plugin will return empty results. With this change, exact search will be performed when only no engine file is available in segment. We also don't need version check or feature flag because, option to not build vector data structure will only be available post 2.17. If an index is created using pre 2.17 version, segment will always have engine files and this feature will never be called during search. --------- Signed-off-by: Vijayan Balasubramanian * Add support for radial search in exact search (#2174) * Add support for radial search in exact search When threshold value is set, knn plugin will not be creating graph. Hence, when search request is trigged during that time, exact search will return valid results. However, radial search was never included as part of exact search. This will break radial search when threshold is added and radial search is requested. In this commit, new method is introduced to accept min score and return documents that are greater than min score, similar to how radial search is performed by native engines. This search is independent of engine, but, radial search is supported only for FAISS engine out of all native engines. Signed-off-by: Vijayan Balasubramanian --------- Signed-off-by: Vijayan Balasubramanian (cherry picked from commit 5a56829385806c9c41e51518b8139744b1e0a0fb) * Fix compilation issue due to package error Signed-off-by: Vijayan Balasubramanian --------- Signed-off-by: Vijayan Balasubramanian Co-authored-by: Vijayan Balasubramanian --- CHANGELOG.md | 1 + .../org/opensearch/knn/index/KNNSettings.java | 20 + .../codec/BasePerFieldKnnVectorsFormat.java | 20 +- .../NativeEngines990KnnVectorsFormat.java | 24 +- .../NativeEngines990KnnVectorsWriter.java | 37 +- .../knn/index/query/ExactSearcher.java | 55 +- .../opensearch/knn/index/query/KNNWeight.java | 140 +++-- .../knn/index/query/RNNQueryFactory.java | 1 + .../nativelib/NativeEngineKnnVectorQuery.java | 2 +- .../org/opensearch/knn/index/FaissIT.java | 231 +++++++- .../opensearch/knn/index/OpenSearchIT.java | 296 ++++++++++ .../NativeEngineFieldVectorsWriterTests.java | 1 - ...eEngines990KnnVectorsWriterFlushTests.java | 540 +++++++++++++++++- ...eEngines990KnnVectorsWriterMergeTests.java | 125 +++- .../knn/index/query/ExactSearcherTests.java | 139 +++++ .../knn/index/query/KNNWeightTests.java | 86 ++- .../opensearch/knn/integ/BinaryIndexIT.java | 75 ++- .../org/opensearch/knn/KNNRestTestCase.java | 18 +- 18 files changed, 1731 insertions(+), 80 deletions(-) create mode 100644 src/test/java/org/opensearch/knn/index/query/ExactSearcherTests.java diff --git a/CHANGELOG.md b/CHANGELOG.md index a46185ef2..c3a7e3722 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), * Update Default Rescore Context based on Dimension [#2149](https://github.com/opensearch-project/k-NN/pull/2149) * KNNIterators should support with and without filters [#2155](https://github.com/opensearch-project/k-NN/pull/2155) * Adding Support to Enable/Disble Share level Rescoring and Update Oversampling Factor[#2172](https://github.com/opensearch-project/k-NN/pull/2172) +* Add support to build vector data structures greedily and perform exact search when there are no engine files [#1942](https://github.com/opensearch-project/k-NN/issues/1942) ### Bug Fixes * Add DocValuesProducers for releasing memory when close index [#1946](https://github.com/opensearch-project/k-NN/pull/1946) * KNN80DocValues should only be considered for BinaryDocValues fields [#2147](https://github.com/opensearch-project/k-NN/pull/2147) diff --git a/src/main/java/org/opensearch/knn/index/KNNSettings.java b/src/main/java/org/opensearch/knn/index/KNNSettings.java index f5980879a..fb9c26fef 100644 --- a/src/main/java/org/opensearch/knn/index/KNNSettings.java +++ b/src/main/java/org/opensearch/knn/index/KNNSettings.java @@ -67,6 +67,7 @@ public class KNNSettings { * Settings name */ public static final String KNN_SPACE_TYPE = "index.knn.space_type"; + public static final String INDEX_KNN_ADVANCED_APPROXIMATE_THRESHOLD = "index.knn.advanced.approximate_threshold"; public static final String KNN_ALGO_PARAM_M = "index.knn.algo_param.m"; public static final String KNN_ALGO_PARAM_EF_CONSTRUCTION = "index.knn.algo_param.ef_construction"; public static final String KNN_ALGO_PARAM_EF_SEARCH = "index.knn.algo_param.ef_search"; @@ -97,6 +98,9 @@ public class KNNSettings { public static final boolean KNN_DEFAULT_FAISS_AVX2_DISABLED_VALUE = false; public static final boolean KNN_DEFAULT_FAISS_AVX512_DISABLED_VALUE = false; public static final String INDEX_KNN_DEFAULT_SPACE_TYPE = "l2"; + public static final Integer INDEX_KNN_ADVANCED_APPROXIMATE_THRESHOLD_DEFAULT_VALUE = 0; + public static final Integer INDEX_KNN_BUILD_VECTOR_DATA_STRUCTURE_THRESHOLD_MIN = -1; + public static final Integer INDEX_KNN_BUILD_VECTOR_DATA_STRUCTURE_THRESHOLD_MAX = Integer.MAX_VALUE - 2; public static final String INDEX_KNN_DEFAULT_SPACE_TYPE_FOR_BINARY = "hamming"; public static final Integer INDEX_KNN_DEFAULT_ALGO_PARAM_M = 16; public static final Integer INDEX_KNN_DEFAULT_ALGO_PARAM_EF_SEARCH = 100; @@ -156,6 +160,21 @@ public class KNNSettings { Setting.Property.Deprecated ); + /** + * build_vector_data_structure_threshold - This parameter determines when to build vector data structure for knn fields during indexing + * and merging. Setting -1 (min) will skip building graph, whereas on any other values, the graph will be built if + * number of live docs in segment is greater than this threshold. Since max number of documents in a segment can + * be Integer.MAX_VALUE - 1, this setting will allow threshold to be up to 1 less than max number of documents in a segment + */ + public static final Setting INDEX_KNN_ADVANCED_APPROXIMATE_THRESHOLD_SETTING = Setting.intSetting( + INDEX_KNN_ADVANCED_APPROXIMATE_THRESHOLD, + INDEX_KNN_ADVANCED_APPROXIMATE_THRESHOLD_DEFAULT_VALUE, + INDEX_KNN_BUILD_VECTOR_DATA_STRUCTURE_THRESHOLD_MIN, + INDEX_KNN_BUILD_VECTOR_DATA_STRUCTURE_THRESHOLD_MAX, + IndexScope, + Dynamic + ); + /** * M - the number of bi-directional links created for every new element during construction. * Reasonable range for M is 2-100. Higher M work better on datasets with high intrinsic @@ -486,6 +505,7 @@ private Setting getSetting(String key) { public List> getSettings() { List> settings = Arrays.asList( INDEX_KNN_SPACE_TYPE, + INDEX_KNN_ADVANCED_APPROXIMATE_THRESHOLD_SETTING, INDEX_KNN_ALGO_PARAM_M_SETTING, INDEX_KNN_ALGO_PARAM_EF_CONSTRUCTION_SETTING, INDEX_KNN_ALGO_PARAM_EF_SEARCH_SETTING, diff --git a/src/main/java/org/opensearch/knn/index/codec/BasePerFieldKnnVectorsFormat.java b/src/main/java/org/opensearch/knn/index/codec/BasePerFieldKnnVectorsFormat.java index b06c2a1d8..72187516f 100644 --- a/src/main/java/org/opensearch/knn/index/codec/BasePerFieldKnnVectorsFormat.java +++ b/src/main/java/org/opensearch/knn/index/codec/BasePerFieldKnnVectorsFormat.java @@ -11,7 +11,9 @@ import org.apache.lucene.codecs.hnsw.FlatVectorScorerUtil; import org.apache.lucene.codecs.lucene99.Lucene99FlatVectorsFormat; import org.apache.lucene.codecs.perfield.PerFieldKnnVectorsFormat; +import org.opensearch.index.IndexSettings; import org.opensearch.index.mapper.MapperService; +import org.opensearch.knn.index.KNNSettings; import org.opensearch.knn.index.codec.KNN990Codec.NativeEngines990KnnVectorsFormat; import org.opensearch.knn.index.codec.params.KNNScalarQuantizedVectorsFormatParams; import org.opensearch.knn.index.codec.params.KNNVectorsFormatParams; @@ -129,7 +131,23 @@ public KnnVectorsFormat getKnnVectorsFormatForField(final String field) { } private NativeEngines990KnnVectorsFormat nativeEngineVectorsFormat() { - return new NativeEngines990KnnVectorsFormat(new Lucene99FlatVectorsFormat(FlatVectorScorerUtil.getLucene99FlatVectorsScorer())); + // mapperService is already checked for null or valid instance type at caller, hence we don't need + // addition isPresent check here. + int approximateThreshold = getApproximateThresholdValue(); + return new NativeEngines990KnnVectorsFormat( + new Lucene99FlatVectorsFormat(FlatVectorScorerUtil.getLucene99FlatVectorsScorer()), + approximateThreshold + ); + } + + private int getApproximateThresholdValue() { + // This is private method and mapperService is already checked for null or valid instance type before this call + // at caller, hence we don't need additional isPresent check here. + final IndexSettings indexSettings = mapperService.get().getIndexSettings(); + final Integer approximateThresholdValue = indexSettings.getValue(KNNSettings.INDEX_KNN_ADVANCED_APPROXIMATE_THRESHOLD_SETTING); + return approximateThresholdValue != null + ? approximateThresholdValue + : KNNSettings.INDEX_KNN_ADVANCED_APPROXIMATE_THRESHOLD_DEFAULT_VALUE; } @Override diff --git a/src/main/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsFormat.java b/src/main/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsFormat.java index 626210f25..520a9838d 100644 --- a/src/main/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsFormat.java +++ b/src/main/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsFormat.java @@ -19,6 +19,7 @@ import org.apache.lucene.codecs.lucene99.Lucene99FlatVectorsFormat; import org.apache.lucene.index.SegmentReadState; import org.apache.lucene.index.SegmentWriteState; +import org.opensearch.knn.index.KNNSettings; import java.io.IOException; @@ -30,15 +31,20 @@ public class NativeEngines990KnnVectorsFormat extends KnnVectorsFormat { /** The format for storing, reading, merging vectors on disk */ private static FlatVectorsFormat flatVectorsFormat; private static final String FORMAT_NAME = "NativeEngines990KnnVectorsFormat"; + private static int approximateThreshold; public NativeEngines990KnnVectorsFormat() { - super(FORMAT_NAME); - flatVectorsFormat = new Lucene99FlatVectorsFormat(new DefaultFlatVectorScorer()); + this(new Lucene99FlatVectorsFormat(new DefaultFlatVectorScorer())); + } + + public NativeEngines990KnnVectorsFormat(final FlatVectorsFormat flatVectorsFormat) { + this(flatVectorsFormat, KNNSettings.INDEX_KNN_ADVANCED_APPROXIMATE_THRESHOLD_DEFAULT_VALUE); } - public NativeEngines990KnnVectorsFormat(final FlatVectorsFormat lucene99FlatVectorsFormat) { + public NativeEngines990KnnVectorsFormat(final FlatVectorsFormat flatVectorsFormat, int approximateThreshold) { super(FORMAT_NAME); - flatVectorsFormat = lucene99FlatVectorsFormat; + NativeEngines990KnnVectorsFormat.flatVectorsFormat = flatVectorsFormat; + NativeEngines990KnnVectorsFormat.approximateThreshold = approximateThreshold; } /** @@ -48,7 +54,7 @@ public NativeEngines990KnnVectorsFormat(final FlatVectorsFormat lucene99FlatVect */ @Override public KnnVectorsWriter fieldsWriter(final SegmentWriteState state) throws IOException { - return new NativeEngines990KnnVectorsWriter(state, flatVectorsFormat.fieldsWriter(state)); + return new NativeEngines990KnnVectorsWriter(state, flatVectorsFormat.fieldsWriter(state), approximateThreshold); } /** @@ -63,6 +69,12 @@ public KnnVectorsReader fieldsReader(final SegmentReadState state) throws IOExce @Override public String toString() { - return "NativeEngines99KnnVectorsFormat(name=" + this.getClass().getSimpleName() + ", flatVectorsFormat=" + flatVectorsFormat + ")"; + return "NativeEngines99KnnVectorsFormat(name=" + + this.getClass().getSimpleName() + + ", flatVectorsFormat=" + + flatVectorsFormat + + ", approximateThreshold=" + + approximateThreshold + + ")"; } } diff --git a/src/main/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsWriter.java b/src/main/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsWriter.java index eccad41c8..7c8636577 100644 --- a/src/main/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsWriter.java +++ b/src/main/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsWriter.java @@ -53,10 +53,16 @@ public class NativeEngines990KnnVectorsWriter extends KnnVectorsWriter { private KNN990QuantizationStateWriter quantizationStateWriter; private final List> fields = new ArrayList<>(); private boolean finished; + private final Integer approximateThreshold; - public NativeEngines990KnnVectorsWriter(SegmentWriteState segmentWriteState, FlatVectorsWriter flatVectorsWriter) { + public NativeEngines990KnnVectorsWriter( + SegmentWriteState segmentWriteState, + FlatVectorsWriter flatVectorsWriter, + Integer approximateThreshold + ) { this.segmentWriteState = segmentWriteState; this.flatVectorsWriter = flatVectorsWriter; + this.approximateThreshold = approximateThreshold; } /** @@ -98,6 +104,17 @@ public void flush(int maxDoc, final Sorter.DocMap sortMap) throws IOException { field.getVectors() ); final QuantizationState quantizationState = train(field.getFieldInfo(), knnVectorValuesSupplier, totalLiveDocs); + // Check only after quantization state writer finish writing its state, since it is required + // even if there are no graph files in segment, which will be later used by exact search + if (shouldSkipBuildingVectorDataStructure(totalLiveDocs)) { + log.info( + "Skip building vector data structure for field: {}, as liveDoc: {} is less than the threshold {} during flush", + fieldInfo.name, + totalLiveDocs, + approximateThreshold + ); + continue; + } final NativeIndexWriter writer = NativeIndexWriter.getWriter(fieldInfo, segmentWriteState, quantizationState); final KNNVectorValues knnVectorValues = knnVectorValuesSupplier.get(); @@ -127,6 +144,17 @@ public void mergeOneField(final FieldInfo fieldInfo, final MergeState mergeState } final QuantizationState quantizationState = train(fieldInfo, knnVectorValuesSupplier, totalLiveDocs); + // Check only after quantization state writer finish writing its state, since it is required + // even if there are no graph files in segment, which will be later used by exact search + if (shouldSkipBuildingVectorDataStructure(totalLiveDocs)) { + log.info( + "Skip building vector data structure for field: {}, as liveDoc: {} is less than the threshold {} during merge", + fieldInfo.name, + totalLiveDocs, + approximateThreshold + ); + return; + } final NativeIndexWriter writer = NativeIndexWriter.getWriter(fieldInfo, segmentWriteState, quantizationState); final KNNVectorValues knnVectorValues = knnVectorValuesSupplier.get(); @@ -257,4 +285,11 @@ private void initQuantizationStateWriterIfNecessary() throws IOException { quantizationStateWriter.writeHeader(segmentWriteState); } } + + private boolean shouldSkipBuildingVectorDataStructure(final long docCount) { + if (approximateThreshold < 0) { + return true; + } + return docCount < approximateThreshold; + } } diff --git a/src/main/java/org/opensearch/knn/index/query/ExactSearcher.java b/src/main/java/org/opensearch/knn/index/query/ExactSearcher.java index 8e5849abb..77e993297 100644 --- a/src/main/java/org/opensearch/knn/index/query/ExactSearcher.java +++ b/src/main/java/org/opensearch/knn/index/query/ExactSearcher.java @@ -5,8 +5,10 @@ package org.opensearch.knn.index.query; +import com.google.common.base.Predicates; import lombok.AllArgsConstructor; import lombok.Builder; +import lombok.NonNull; import lombok.Value; import lombok.extern.log4j.Log4j2; import org.apache.lucene.index.FieldInfo; @@ -21,6 +23,7 @@ import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.query.iterators.BinaryVectorIdsKNNIterator; +import org.opensearch.knn.index.engine.KNNEngine; import org.opensearch.knn.index.query.iterators.ByteVectorIdsKNNIterator; import org.opensearch.knn.index.query.iterators.NestedBinaryVectorIdsKNNIterator; import org.opensearch.knn.index.query.iterators.VectorIdsKNNIterator; @@ -36,7 +39,9 @@ import java.io.IOException; import java.util.HashMap; +import java.util.Locale; import java.util.Map; +import java.util.function.Predicate; @Log4j2 @AllArgsConstructor @@ -55,11 +60,41 @@ public class ExactSearcher { public Map searchLeaf(final LeafReaderContext leafReaderContext, final ExactSearcherContext exactSearcherContext) throws IOException { KNNIterator iterator = getKNNIterator(leafReaderContext, exactSearcherContext); + if (exactSearcherContext.getKnnQuery().getRadius() != null) { + return doRadialSearch(leafReaderContext, exactSearcherContext, iterator); + } if (exactSearcherContext.getMatchedDocs() != null && exactSearcherContext.getMatchedDocs().cardinality() <= exactSearcherContext.getK()) { return scoreAllDocs(iterator); } - return searchTopK(iterator, exactSearcherContext.getK()); + return searchTopCandidates(iterator, exactSearcherContext.getK(), Predicates.alwaysTrue()); + } + + /** + * Perform radial search by comparing scores with min score. Currently, FAISS from native engine supports radial search. + * Hence, we assume that Radius from knnQuery is always distance, and we convert it to score since we do exact search uses scores + * to filter out the documents that does not have given min score. + * @param leafReaderContext + * @param exactSearcherContext + * @param iterator {@link KNNIterator} + * @return Map of docId and score + * @throws IOException exception raised by iterator during traversal + */ + private Map doRadialSearch( + LeafReaderContext leafReaderContext, + ExactSearcherContext exactSearcherContext, + KNNIterator iterator + ) throws IOException { + final SegmentReader reader = Lucene.segmentReader(leafReaderContext.reader()); + final KNNQuery knnQuery = exactSearcherContext.getKnnQuery(); + final FieldInfo fieldInfo = reader.getFieldInfos().fieldInfo(knnQuery.getField()); + final KNNEngine engine = FieldInfoExtractor.extractKNNEngine(fieldInfo); + if (KNNEngine.FAISS != engine) { + throw new IllegalArgumentException(String.format(Locale.ROOT, "Engine [%s] does not support radial search", engine)); + } + final SpaceType spaceType = FieldInfoExtractor.getSpaceType(modelDao, fieldInfo); + final float minScore = spaceType.scoreTranslation(knnQuery.getRadius()); + return filterDocsByMinScore(exactSearcherContext, iterator, minScore); } private Map scoreAllDocs(KNNIterator iterator) throws IOException { @@ -71,15 +106,17 @@ private Map scoreAllDocs(KNNIterator iterator) throws IOExceptio return docToScore; } - private Map searchTopK(KNNIterator iterator, int k) throws IOException { + private Map searchTopCandidates(KNNIterator iterator, int limit, @NonNull Predicate filterScore) + throws IOException { // Creating min heap and init with MAX DocID and Score as -INF. - final HitQueue queue = new HitQueue(k, true); + final HitQueue queue = new HitQueue(limit, true); ScoreDoc topDoc = queue.top(); final Map docToScore = new HashMap<>(); int docId; while ((docId = iterator.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) { - if (iterator.score() > topDoc.score) { - topDoc.score = iterator.score(); + final float currentScore = iterator.score(); + if (filterScore.test(currentScore) && currentScore > topDoc.score) { + topDoc.score = currentScore; topDoc.doc = docId; // As the HitQueue is min heap, updating top will bring the doc with -INF score or worst score we // have seen till now on top. @@ -98,10 +135,16 @@ private Map searchTopK(KNNIterator iterator, int k) throws IOExc final ScoreDoc doc = queue.pop(); docToScore.put(doc.doc, doc.score); } - return docToScore; } + private Map filterDocsByMinScore(ExactSearcherContext context, KNNIterator iterator, float minScore) + throws IOException { + int maxResultWindow = context.getKnnQuery().getContext().getMaxResultWindow(); + Predicate scoreGreaterThanOrEqualToMinScore = score -> score >= minScore; + return searchTopCandidates(iterator, maxResultWindow, scoreGreaterThanOrEqualToMinScore); + } + private KNNIterator getKNNIterator(LeafReaderContext leafReaderContext, ExactSearcherContext exactSearcherContext) throws IOException { final KNNQuery knnQuery = exactSearcherContext.getKnnQuery(); final BitSet matchedDocs = exactSearcherContext.getMatchedDocs(); diff --git a/src/main/java/org/opensearch/knn/index/query/KNNWeight.java b/src/main/java/org/opensearch/knn/index/query/KNNWeight.java index 37695c208..87c99b884 100644 --- a/src/main/java/org/opensearch/knn/index/query/KNNWeight.java +++ b/src/main/java/org/opensearch/knn/index/query/KNNWeight.java @@ -5,6 +5,7 @@ package org.opensearch.knn.index.query; +import com.google.common.annotations.VisibleForTesting; import lombok.extern.log4j.Log4j2; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.LeafReaderContext; @@ -22,6 +23,7 @@ import org.apache.lucene.util.FixedBitSet; import org.opensearch.common.io.PathUtils; import org.opensearch.common.lucene.Lucene; +import org.opensearch.knn.common.FieldInfoExtractor; import org.opensearch.knn.common.KNNConstants; import org.opensearch.knn.index.KNNSettings; import org.opensearch.knn.index.SpaceType; @@ -33,6 +35,7 @@ import org.opensearch.knn.index.memory.NativeMemoryLoadStrategy; import org.opensearch.knn.index.engine.KNNEngine; import org.opensearch.knn.index.quantizationservice.QuantizationService; +import org.opensearch.knn.index.query.ExactSearcher.ExactSearcherContext.ExactSearcherContextBuilder; import org.opensearch.knn.indices.ModelDao; import org.opensearch.knn.indices.ModelMetadata; import org.opensearch.knn.indices.ModelUtil; @@ -93,8 +96,13 @@ public KNNWeight(KNNQuery query, float boost, Weight filterWeight) { } public static void initialize(ModelDao modelDao) { + initialize(modelDao, new ExactSearcher(modelDao)); + } + + @VisibleForTesting + static void initialize(ModelDao modelDao, ExactSearcher exactSearcher) { KNNWeight.modelDao = modelDao; - KNNWeight.DEFAULT_EXACT_SEARCHER = new ExactSearcher(modelDao); + KNNWeight.DEFAULT_EXACT_SEARCHER = exactSearcher; } @Override @@ -129,42 +137,21 @@ public Map searchLeaf(LeafReaderContext context, int k) throws I if (filterWeight != null && cardinality == 0) { return Collections.emptyMap(); } - /* - * The idea for this optimization is to get K results, we need to atleast look at K vectors in the HNSW graph + * The idea for this optimization is to get K results, we need to at least look at K vectors in the HNSW graph * . Hence, if filtered results are less than K and filter query is present we should shift to exact search. * This improves the recall. */ - Map docIdsToScoreMap; - final ExactSearcher.ExactSearcherContext exactSearcherContext = ExactSearcher.ExactSearcherContext.builder() - .k(k) - .isParentHits(true) - .matchedDocs(filterBitSet) - // setting to true, so that if quantization details are present we want to do search on the quantized - // vectors as this flow is used in first pass of search. - .useQuantizedVectorsForSearch(true) - .knnQuery(knnQuery) - .build(); - if (filterWeight != null && canDoExactSearch(cardinality)) { - docIdsToScoreMap = exactSearch(context, exactSearcherContext); - } else { - docIdsToScoreMap = doANNSearch(context, filterBitSet, cardinality, k); - if (docIdsToScoreMap == null) { - return Collections.emptyMap(); - } - if (canDoExactSearchAfterANNSearch(cardinality, docIdsToScoreMap.size())) { - log.debug( - "Doing ExactSearch after doing ANNSearch as the number of documents returned are less than " - + "K, even when we have more than K filtered Ids. K: {}, ANNResults: {}, filteredIdCount: {}", - k, - docIdsToScoreMap.size(), - cardinality - ); - docIdsToScoreMap = exactSearch(context, exactSearcherContext); - } + if (isFilteredExactSearchPreferred(cardinality)) { + return doExactSearch(context, filterBitSet, k); } - if (docIdsToScoreMap.isEmpty()) { - return Collections.emptyMap(); + Map docIdsToScoreMap = doANNSearch(context, filterBitSet, cardinality, k); + // See whether we have to perform exact search based on approx search results + // This is required if there are no native engine files or if approximate search returned + // results less than K, though we have more than k filtered docs + if (isExactSearchRequire(context, cardinality, docIdsToScoreMap.size())) { + final BitSet docs = filterWeight != null ? filterBitSet : null; + return doExactSearch(context, docs, k); } return docIdsToScoreMap; } @@ -221,6 +208,20 @@ private int[] bitSetToIntArray(final BitSet bitSet) { return intArray; } + private Map doExactSearch(final LeafReaderContext context, final BitSet acceptedDocs, int k) throws IOException { + final ExactSearcherContextBuilder exactSearcherContextBuilder = ExactSearcher.ExactSearcherContext.builder() + .isParentHits(true) + .k(k) + // setting to true, so that if quantization details are present we want to do search on the quantized + // vectors as this flow is used in first pass of search. + .useQuantizedVectorsForSearch(true) + .knnQuery(knnQuery); + if (acceptedDocs != null) { + exactSearcherContextBuilder.matchedDocs(acceptedDocs); + } + return exactSearch(context, exactSearcherContextBuilder.build()); + } + private Map doANNSearch( final LeafReaderContext context, final BitSet filterIdsBitSet, @@ -234,7 +235,7 @@ private Map doANNSearch( if (fieldInfo == null) { log.debug("[KNN] Field info not found for {}:{}", knnQuery.getField(), reader.getSegmentName()); - return null; + return Collections.emptyMap(); } KNNEngine knnEngine; @@ -273,8 +274,8 @@ private Map doANNSearch( List engineFiles = KNNCodecUtil.getEngineFiles(knnEngine.getExtension(), knnQuery.getField(), reader.getSegmentInfo().info); if (engineFiles.isEmpty()) { - log.debug("[KNN] No engine index found for field {} for segment {}", knnQuery.getField(), reader.getSegmentName()); - return null; + log.debug("[KNN] No native engine files found for field {} for segment {}", knnQuery.getField(), reader.getSegmentName()); + return Collections.emptyMap(); } Path indexPath = PathUtils.get(directory, engineFiles.get(0)); @@ -364,16 +365,9 @@ private Map doANNSearch( indexAllocation.readUnlock(); indexAllocation.decRef(); } - - /* - * Scores represent the distance of the documents with respect to given query vector. - * Lesser the score, the closer the document is to the query vector. - * Since by default results are retrieved in the descending order of scores, to get the nearest - * neighbors we are inverting the scores. - */ if (results.length == 0) { log.debug("[KNN] Query yielded 0 results"); - return null; + return Collections.emptyMap(); } if (quantizedVector != null) { @@ -386,7 +380,6 @@ private Map doANNSearch( /** * Execute exact search for the given matched doc ids and return the results as a map of docId to score. - * * @return Map of docId to score for the exact search results. * @throws IOException If an error occurs during the search. */ @@ -407,18 +400,18 @@ public static float normalizeScore(float score) { return -score + 1; } - private boolean canDoExactSearch(final int filterIdsCount) { + private boolean isFilteredExactSearchPreferred(final int filterIdsCount) { + if (filterWeight == null) { + return false; + } log.debug( "Info for doing exact search filterIdsLength : {}, Threshold value: {}", filterIdsCount, KNNSettings.getFilteredExactSearchThreshold(knnQuery.getIndexName()) ); - if (knnQuery.getRadius() != null) { - return false; - } int filterThresholdValue = KNNSettings.getFilteredExactSearchThreshold(knnQuery.getIndexName()); // Refer this GitHub around more details https://github.com/opensearch-project/k-NN/issues/1049 on the logic - if (filterIdsCount <= knnQuery.getK()) { + if (knnQuery.getRadius() == null && filterIdsCount <= knnQuery.getK()) { return true; } // See user has defined Exact Search filtered threshold. if yes, then use that setting. @@ -446,14 +439,59 @@ private boolean isExactSearchThresholdSettingSet(int filterThresholdValue) { return filterThresholdValue != KNNSettings.ADVANCED_FILTERED_EXACT_SEARCH_THRESHOLD_DEFAULT_VALUE; } + /** + * This condition mainly checks whether exact search should be performed or not + * @param context LeafReaderContext + * @param filterIdsCount count of filtered Doc ids + * @param annResultCount Count of Nearest Neighbours we got after doing filtered ANN Search. + * @return boolean - true if exactSearch needs to be done after ANNSearch. + */ + private boolean isExactSearchRequire(final LeafReaderContext context, final int filterIdsCount, final int annResultCount) { + if (annResultCount == 0 && isMissingNativeEngineFiles(context)) { + log.debug("Perform exact search after approximate search since no native engine files are available"); + return true; + } + if (isFilteredExactSearchRequireAfterANNSearch(filterIdsCount, annResultCount)) { + log.debug( + "Doing ExactSearch after doing ANNSearch as the number of documents returned are less than " + + "K, even when we have more than K filtered Ids. K: {}, ANNResults: {}, filteredIdCount: {}", + this.knnQuery.getK(), + annResultCount, + filterIdsCount + ); + return true; + } + return false; + } + /** * This condition mainly checks during filtered search we have more than K elements in filterIds but the ANN - * doesn't yeild K nearest neighbors. + * doesn't yield K nearest neighbors. * @param filterIdsCount count of filtered Doc ids * @param annResultCount Count of Nearest Neighbours we got after doing filtered ANN Search. * @return boolean - true if exactSearch needs to be done after ANNSearch. */ - private boolean canDoExactSearchAfterANNSearch(final int filterIdsCount, final int annResultCount) { + private boolean isFilteredExactSearchRequireAfterANNSearch(final int filterIdsCount, final int annResultCount) { return filterWeight != null && filterIdsCount >= knnQuery.getK() && knnQuery.getK() > annResultCount; } + + /** + * This condition mainly checks whether segments has native engine files or not + * @return boolean - false if exactSearch needs to be done since no native engine files are in segments. + */ + private boolean isMissingNativeEngineFiles(LeafReaderContext context) { + final SegmentReader reader = Lucene.segmentReader(context.reader()); + final FieldInfo fieldInfo = reader.getFieldInfos().fieldInfo(knnQuery.getField()); + // if segment has no documents with at least 1 vector field, field info will be null + if (fieldInfo == null) { + return false; + } + final KNNEngine knnEngine = FieldInfoExtractor.extractKNNEngine(fieldInfo); + final List engineFiles = KNNCodecUtil.getEngineFiles( + knnEngine.getExtension(), + knnQuery.getField(), + reader.getSegmentInfo().info + ); + return engineFiles.isEmpty(); + } } diff --git a/src/main/java/org/opensearch/knn/index/query/RNNQueryFactory.java b/src/main/java/org/opensearch/knn/index/query/RNNQueryFactory.java index 99152ef6b..b5166866c 100644 --- a/src/main/java/org/opensearch/knn/index/query/RNNQueryFactory.java +++ b/src/main/java/org/opensearch/knn/index/query/RNNQueryFactory.java @@ -88,6 +88,7 @@ public static Query create(RNNQueryFactory.CreateQueryRequest createQueryRequest .indexName(indexName) .parentsFilter(parentFilter) .radius(radius) + .vectorDataType(vectorDataType) .methodParameters(methodParameters) .context(knnQueryContext) .filterQuery(filterQuery) diff --git a/src/main/java/org/opensearch/knn/index/query/nativelib/NativeEngineKnnVectorQuery.java b/src/main/java/org/opensearch/knn/index/query/nativelib/NativeEngineKnnVectorQuery.java index c97a0d061..8b861b430 100644 --- a/src/main/java/org/opensearch/knn/index/query/nativelib/NativeEngineKnnVectorQuery.java +++ b/src/main/java/org/opensearch/knn/index/query/nativelib/NativeEngineKnnVectorQuery.java @@ -57,7 +57,7 @@ public Weight createWeight(IndexSearcher indexSearcher, ScoreMode scoreMode, flo List leafReaderContexts = reader.leaves(); List> perLeafResults; RescoreContext rescoreContext = knnQuery.getRescoreContext(); - int finalK = knnQuery.getK(); + final int finalK = knnQuery.getK(); if (rescoreContext == null) { perLeafResults = doSearch(indexSearcher, leafReaderContexts, knnWeight, finalK); } else { diff --git a/src/test/java/org/opensearch/knn/index/FaissIT.java b/src/test/java/org/opensearch/knn/index/FaissIT.java index 72c0b01f4..4782b0ce6 100644 --- a/src/test/java/org/opensearch/knn/index/FaissIT.java +++ b/src/test/java/org/opensearch/knn/index/FaissIT.java @@ -92,6 +92,7 @@ public class FaissIT extends KNNRestTestCase { private static final String INTEGER_FIELD_NAME = "int_field"; private static final String FILED_TYPE_INTEGER = "integer"; private static final String NON_EXISTENT_INTEGER_FIELD_NAME = "nonexistent_int_field"; + public static final int NEVER_BUILD_VECTOR_DATA_STRUCTURE_THRESHOLD = -1; static TestUtils.TestData testData; @@ -575,6 +576,125 @@ public void testHNSWSQFP16_whenIndexedAndQueried_thenSucceed() { validateGraphEviction(); } + @SneakyThrows + public void testHNSWSQFP16_whenGraphThresholdIsNegative_whenIndexed_thenSkipCreatingGraph() { + final String indexName = "test-index-hnsw-sqfp16"; + final String fieldName = "test-field-hnsw-sqfp16"; + final SpaceType[] spaceTypes = { SpaceType.L2, SpaceType.INNER_PRODUCT }; + final Random random = new Random(); + final SpaceType spaceType = spaceTypes[random.nextInt(spaceTypes.length)]; + + final int dimension = 128; + final int numDocs = 100; + + // Create an index + final XContentBuilder builder = XContentFactory.jsonBuilder() + .startObject() + .startObject("properties") + .startObject(fieldName) + .field("type", "knn_vector") + .field("dimension", dimension) + .startObject(KNN_METHOD) + .field(NAME, METHOD_HNSW) + .field(METHOD_PARAMETER_SPACE_TYPE, spaceType.getValue()) + .field(KNN_ENGINE, KNNEngine.FAISS.getName()) + .startObject(PARAMETERS) + .startObject(METHOD_ENCODER_PARAMETER) + .field(NAME, ENCODER_SQ) + .startObject(PARAMETERS) + .field(FAISS_SQ_TYPE, FAISS_SQ_ENCODER_FP16) + .endObject() + .endObject() + .endObject() + .endObject() + .endObject() + .endObject() + .endObject(); + + final Map mappingMap = xContentBuilderToMap(builder); + final String mapping = builder.toString(); + final Settings knnIndexSettings = buildKNNIndexSettings(-1); + createKnnIndex(indexName, knnIndexSettings, mapping); + assertEquals(new TreeMap<>(mappingMap), new TreeMap<>(getIndexMappingAsMap(indexName))); + indexTestData(indexName, fieldName, dimension, numDocs); + + final float[] queryVector = new float[dimension]; + Arrays.fill(queryVector, (float) numDocs); + + // Assert we have the right number of documents in the index + assertEquals(numDocs, getDocCount(indexName)); + + final Response searchResponse = searchKNNIndex(indexName, buildSearchQuery(fieldName, 1, queryVector, null), 1); + final List results = parseSearchResponse(EntityUtils.toString(searchResponse.getEntity()), fieldName); + // expect result due to exact search + assertEquals(1, results.size()); + + deleteKNNIndex(indexName); + validateGraphEviction(); + } + + @SneakyThrows + public void testHNSWSQFP16_whenGraphThresholdIsMetDuringMerge_thenCreateGraph() { + final String indexName = "test-index-hnsw-sqfp16"; + final String fieldName = "test-field-hnsw-sqfp16"; + final SpaceType[] spaceTypes = { SpaceType.L2, SpaceType.INNER_PRODUCT }; + final Random random = new Random(); + final SpaceType spaceType = spaceTypes[random.nextInt(spaceTypes.length)]; + final int dimension = 128; + final int numDocs = 100; + + // Create an index + final XContentBuilder builder = XContentFactory.jsonBuilder() + .startObject() + .startObject("properties") + .startObject(fieldName) + .field("type", "knn_vector") + .field("dimension", dimension) + .startObject(KNN_METHOD) + .field(NAME, METHOD_HNSW) + .field(METHOD_PARAMETER_SPACE_TYPE, spaceType.getValue()) + .field(KNN_ENGINE, KNNEngine.FAISS.getName()) + .startObject(PARAMETERS) + .startObject(METHOD_ENCODER_PARAMETER) + .field(NAME, ENCODER_SQ) + .startObject(PARAMETERS) + .field(FAISS_SQ_TYPE, FAISS_SQ_ENCODER_FP16) + .endObject() + .endObject() + .endObject() + .endObject() + .endObject() + .endObject() + .endObject(); + + final Map mappingMap = xContentBuilderToMap(builder); + final String mapping = builder.toString(); + final Settings knnIndexSettings = buildKNNIndexSettings(numDocs); + createKnnIndex(indexName, knnIndexSettings, mapping); + assertEquals(new TreeMap<>(mappingMap), new TreeMap<>(getIndexMappingAsMap(indexName))); + indexTestData(indexName, fieldName, dimension, numDocs); + + final float[] queryVector = new float[dimension]; + Arrays.fill(queryVector, (float) numDocs); + + // Assert we have the right number of documents in the index + assertEquals(numDocs, getDocCount(indexName)); + + // KNN Query should return empty result + final Response searchResponse = searchKNNIndex(indexName, buildSearchQuery(fieldName, 1, queryVector, null), 1); + final List results = parseSearchResponse(EntityUtils.toString(searchResponse.getEntity()), fieldName); + assertEquals(1, results.size()); + + // update index setting to build graph and do force merge + // update build vector data structure setting + forceMergeKnnIndex(indexName, 1); + + queryTestData(indexName, fieldName, dimension, numDocs); + + deleteKNNIndex(indexName); + validateGraphEviction(); + } + @SneakyThrows public void testIVFSQFP16_whenIndexedAndQueried_thenSucceed() { @@ -1706,6 +1826,111 @@ public void testIVF_whenBinaryFormat_whenIVF_thenSuccess() { validateGraphEviction(); } + @SneakyThrows + public void testEndToEnd_whenDoRadiusSearch_whenNoGraphFileIsCreated_whenDistanceThreshold_thenSucceed() { + final SpaceType spaceType = SpaceType.L2; + + final List mValues = ImmutableList.of(16, 32, 64, 128); + final List efConstructionValues = ImmutableList.of(16, 32, 64, 128); + final List efSearchValues = ImmutableList.of(16, 32, 64, 128); + + final Integer dimension = testData.indexData.vectors[0].length; + final Settings knnIndexSettings = buildKNNIndexSettings(NEVER_BUILD_VECTOR_DATA_STRUCTURE_THRESHOLD); + + // Create an index + final XContentBuilder builder = XContentFactory.jsonBuilder() + .startObject() + .startObject("properties") + .startObject(FIELD_NAME) + .field("type", "knn_vector") + .field("dimension", dimension) + .startObject(KNN_METHOD) + .field(NAME, METHOD_HNSW) + .field(METHOD_PARAMETER_SPACE_TYPE, spaceType.getValue()) + .field(KNN_ENGINE, KNNEngine.FAISS.getName()) + .startObject(PARAMETERS) + .field(METHOD_PARAMETER_M, mValues.get(random().nextInt(mValues.size()))) + .field(METHOD_PARAMETER_EF_CONSTRUCTION, efConstructionValues.get(random().nextInt(efConstructionValues.size()))) + .field(KNNConstants.METHOD_PARAMETER_EF_SEARCH, efSearchValues.get(random().nextInt(efSearchValues.size()))) + .endObject() + .endObject() + .endObject() + .endObject() + .endObject(); + createKnnIndex(INDEX_NAME, knnIndexSettings, builder.toString()); + + // Index the test data + for (int i = 0; i < testData.indexData.docs.length; i++) { + addKnnDoc( + INDEX_NAME, + Integer.toString(testData.indexData.docs[i]), + FIELD_NAME, + Floats.asList(testData.indexData.vectors[i]).toArray() + ); + } + + // Assert we have the right number of documents + refreshAllNonSystemIndices(); + assertEquals(testData.indexData.docs.length, getDocCount(INDEX_NAME)); + + final float distance = 300000000000f; + final List> resultsFromDistance = validateRadiusSearchResults( + INDEX_NAME, + FIELD_NAME, + testData.queries, + distance, + null, + spaceType, + null, + null + ); + assertFalse(resultsFromDistance.isEmpty()); + resultsFromDistance.forEach(result -> { assertFalse(result.isEmpty()); }); + final float score = spaceType.scoreTranslation(distance); + final List> resultsFromScore = validateRadiusSearchResults( + INDEX_NAME, + FIELD_NAME, + testData.queries, + null, + score, + spaceType, + null, + null + ); + assertFalse(resultsFromScore.isEmpty()); + resultsFromScore.forEach(result -> { assertFalse(result.isEmpty()); }); + + // Delete index + deleteKNNIndex(INDEX_NAME); + } + + @SneakyThrows + public void testRadialQueryWithFilter_whenNoGraphIsCreated_thenSuccess() { + setupKNNIndexForFilterQuery(buildKNNIndexSettings(NEVER_BUILD_VECTOR_DATA_STRUCTURE_THRESHOLD)); + + final float[][] searchVector = new float[][] { { 3.3f, 3.0f, 5.0f } }; + TermQueryBuilder termQueryBuilder = QueryBuilders.termQuery("color", "red"); + List expectedDocIds = Arrays.asList(DOC_ID_3); + + float distance = 15f; + List> queryResult = validateRadiusSearchResults( + INDEX_NAME, + FIELD_NAME, + searchVector, + distance, + null, + SpaceType.L2, + termQueryBuilder, + null + ); + + assertEquals(1, queryResult.get(0).size()); + assertEquals(expectedDocIds.get(0), queryResult.get(0).get(0).getDocId()); + + // Delete index + deleteKNNIndex(INDEX_NAME); + } + @SneakyThrows public void testQueryWithFilter_whenNonExistingFieldUsedInFilter_thenSuccessful() { XContentBuilder builder = XContentFactory.jsonBuilder() @@ -1778,6 +2003,10 @@ public void testQueryWithFilter_whenNonExistingFieldUsedInFilter_thenSuccessful( } protected void setupKNNIndexForFilterQuery() throws Exception { + setupKNNIndexForFilterQuery(getKNNDefaultIndexSettings()); + } + + protected void setupKNNIndexForFilterQuery(Settings settings) throws Exception { // Create Mappings XContentBuilder builder = XContentFactory.jsonBuilder() .startObject() @@ -1795,7 +2024,7 @@ protected void setupKNNIndexForFilterQuery() throws Exception { .endObject(); final String mapping = builder.toString(); - createKnnIndex(INDEX_NAME, mapping); + createKnnIndex(INDEX_NAME, settings, mapping); addKnnDocWithAttributes( DOC_ID_1, diff --git a/src/test/java/org/opensearch/knn/index/OpenSearchIT.java b/src/test/java/org/opensearch/knn/index/OpenSearchIT.java index 3333feba7..99aea03df 100644 --- a/src/test/java/org/opensearch/knn/index/OpenSearchIT.java +++ b/src/test/java/org/opensearch/knn/index/OpenSearchIT.java @@ -15,6 +15,7 @@ import com.google.common.primitives.Floats; import java.util.Locale; import lombok.SneakyThrows; +import org.apache.http.ParseException; import org.junit.BeforeClass; import org.junit.Ignore; import org.opensearch.knn.KNNRestTestCase; @@ -42,6 +43,8 @@ import java.util.TreeMap; import static org.hamcrest.Matchers.containsString; +import static org.opensearch.knn.index.KNNSettings.INDEX_KNN_BUILD_VECTOR_DATA_STRUCTURE_THRESHOLD_MAX; +import static org.opensearch.knn.index.KNNSettings.INDEX_KNN_BUILD_VECTOR_DATA_STRUCTURE_THRESHOLD_MIN; public class OpenSearchIT extends KNNRestTestCase { @@ -611,4 +614,297 @@ public void testCacheClear_whenCloseIndex() throws Exception { fail("Graphs are not getting evicted"); } + + public void testKNNIndex_whenBuildGraphThresholdIsPresent_thenGetThresholdValue() throws Exception { + final Integer buildVectorDataStructureThreshold = randomIntBetween( + INDEX_KNN_BUILD_VECTOR_DATA_STRUCTURE_THRESHOLD_MIN, + INDEX_KNN_BUILD_VECTOR_DATA_STRUCTURE_THRESHOLD_MAX + ); + final Settings settings = Settings.builder().put(buildKNNIndexSettings(buildVectorDataStructureThreshold)).build(); + final String knnIndexMapping = createKnnIndexMapping(FIELD_NAME, KNNEngine.getMaxDimensionByEngine(KNNEngine.DEFAULT)); + final String indexName = "test-index-with-build-graph-settings"; + createKnnIndex(indexName, settings, knnIndexMapping); + final String buildVectorDataStructureThresholdSetting = getIndexSettingByName( + indexName, + KNNSettings.INDEX_KNN_ADVANCED_APPROXIMATE_THRESHOLD + ); + assertNotNull("build_vector_data_structure_threshold index setting is not found", buildVectorDataStructureThresholdSetting); + assertEquals( + "incorrect setting for build_vector_data_structure_threshold", + buildVectorDataStructureThreshold, + Integer.valueOf(buildVectorDataStructureThresholdSetting) + ); + deleteKNNIndex(indexName); + } + + public void testKNNIndex_whenBuildThresholdIsNotProvided_thenShouldNotReturnSetting() throws Exception { + final String knnIndexMapping = createKnnIndexMapping(FIELD_NAME, KNNEngine.getMaxDimensionByEngine(KNNEngine.DEFAULT)); + final String indexName = "test-index-with-build-graph-settings"; + createKnnIndex(indexName, knnIndexMapping); + final String buildVectorDataStructureThresholdSetting = getIndexSettingByName( + indexName, + KNNSettings.INDEX_KNN_ADVANCED_APPROXIMATE_THRESHOLD + ); + assertNull( + "build_vector_data_structure_threshold index setting should not be added in index setting", + buildVectorDataStructureThresholdSetting + ); + deleteKNNIndex(indexName); + } + + public void testKNNIndex_whenGetIndexSettingWithDefaultIsCalled_thenReturnDefaultBuildGraphThresholdValue() throws Exception { + final String knnIndexMapping = createKnnIndexMapping(FIELD_NAME, KNNEngine.getMaxDimensionByEngine(KNNEngine.DEFAULT)); + final String indexName = "test-index-with-build-vector-graph-settings"; + createKnnIndex(indexName, knnIndexMapping); + final String buildVectorDataStructureThresholdSetting = getIndexSettingByName( + indexName, + KNNSettings.INDEX_KNN_ADVANCED_APPROXIMATE_THRESHOLD, + true + ); + assertNotNull("build_vector_data_structure index setting is not found", buildVectorDataStructureThresholdSetting); + assertEquals( + "incorrect default setting for build_vector_data_structure_threshold", + KNNSettings.INDEX_KNN_ADVANCED_APPROXIMATE_THRESHOLD_DEFAULT_VALUE, + Integer.valueOf(buildVectorDataStructureThresholdSetting) + ); + deleteKNNIndex(indexName); + } + + /* + For this testcase, we will create index with setting build_vector_data_structure_threshold as -1, then index few documents, perform knn search, + then, confirm hits because of exact search though there are no graph. In next step, update setting to 0, force merge segment to 1, perform knn search and confirm expected + hits are returned. + */ + public void testKNNIndex_whenBuildVectorGraphThresholdIsProvidedEndToEnd_thenBuildGraphBasedOnSetting() throws Exception { + final String indexName = "test-index-1"; + final String fieldName1 = "test-field-1"; + final String fieldName2 = "test-field-2"; + + final Integer dimension = testData.indexData.vectors[0].length; + final Settings knnIndexSettings = buildKNNIndexSettings(-1); + + // Create an index + final XContentBuilder builder = XContentFactory.jsonBuilder() + .startObject() + .startObject("properties") + .startObject(fieldName1) + .field("type", "knn_vector") + .field("dimension", dimension) + .startObject(KNNConstants.KNN_METHOD) + .field(KNNConstants.NAME, KNNConstants.METHOD_HNSW) + .field(KNNConstants.KNN_ENGINE, KNNEngine.NMSLIB.getName()) + .startObject(KNNConstants.PARAMETERS) + .endObject() + .endObject() + .endObject() + .startObject(fieldName2) + .field("type", "knn_vector") + .field("dimension", dimension) + .startObject(KNNConstants.KNN_METHOD) + .field(KNNConstants.NAME, KNNConstants.METHOD_HNSW) + .field(KNNConstants.KNN_ENGINE, KNNEngine.FAISS.getName()) + .startObject(KNNConstants.PARAMETERS) + .endObject() + .endObject() + .endObject() + .endObject() + .endObject(); + + createKnnIndex(indexName, knnIndexSettings, builder.toString()); + + // Index the test data + for (int i = 0; i < testData.indexData.docs.length; i++) { + addKnnDoc( + indexName, + Integer.toString(testData.indexData.docs[i]), + ImmutableList.of(fieldName1, fieldName2), + ImmutableList.of( + Floats.asList(testData.indexData.vectors[i]).toArray(), + Floats.asList(testData.indexData.vectors[i]).toArray() + ) + ); + } + + refreshAllIndices(); + // Assert we have the right number of documents in the index + assertEquals(testData.indexData.docs.length, getDocCount(indexName)); + + final List nmslibNeighbors = getResults(indexName, fieldName1, testData.queries[0], 1); + assertEquals("unexpected neighbors are returned", nmslibNeighbors.size(), nmslibNeighbors.size()); + + final List faissNeighbors = getResults(indexName, fieldName2, testData.queries[0], 1); + assertEquals("unexpected neighbors are returned", faissNeighbors.size(), faissNeighbors.size()); + + // update build vector data structure setting + updateIndexSettings(indexName, Settings.builder().put(KNNSettings.INDEX_KNN_ADVANCED_APPROXIMATE_THRESHOLD, 0)); + forceMergeKnnIndex(indexName, 1); + + final int k = 10; + for (int i = 0; i < testData.queries.length; i++) { + // Search nmslib field + final Response response = searchKNNIndex(indexName, new KNNQueryBuilder(fieldName1, testData.queries[i], k), k); + final String responseBody = EntityUtils.toString(response.getEntity()); + final List nmslibValidNeighbors = parseSearchResponse(responseBody, fieldName1); + assertEquals(k, nmslibValidNeighbors.size()); + // Search faiss field + final List faissValidNeighbors = getResults(indexName, fieldName2, testData.queries[i], k); + assertEquals(k, faissValidNeighbors.size()); + } + + // Delete index + deleteKNNIndex(indexName); + } + + /* + For this testcase, we will create index with setting build_vector_data_structure_threshold number of documents to ingest, then index x documents, perform knn search, + then, confirm expected hits are returned. Here, we don't need force merge to build graph, since, threshold is less than + actual number of documents in segments + */ + public void testKNNIndex_whenBuildVectorDataStructureIsLessThanDocCount_thenBuildGraphBasedSuccessfully() throws Exception { + final String indexName = "test-index-1"; + final String fieldName = "test-field-1"; + + final Integer dimension = testData.indexData.vectors[0].length; + final Settings knnIndexSettings = buildKNNIndexSettings(testData.indexData.docs.length); + + // Create an index + final XContentBuilder builder = XContentFactory.jsonBuilder() + .startObject() + .startObject("properties") + .startObject(fieldName) + .field("type", "knn_vector") + .field("dimension", dimension) + .startObject(KNNConstants.KNN_METHOD) + .field(KNNConstants.NAME, KNNConstants.METHOD_HNSW) + .field(KNNConstants.KNN_ENGINE, KNNEngine.NMSLIB.getName()) + .startObject(KNNConstants.PARAMETERS) + .endObject() + .endObject() + .endObject() + .endObject() + .endObject(); + + createKnnIndex(indexName, knnIndexSettings, builder.toString()); + // Disable refresh + updateIndexSettings(indexName, Settings.builder().put("index.refresh_interval", -1)); + + // Index the test data without refresh on every document + for (int i = 0; i < testData.indexData.docs.length; i++) { + addKnnDoc( + indexName, + Integer.toString(testData.indexData.docs[i]), + ImmutableList.of(fieldName), + ImmutableList.of(Floats.asList(testData.indexData.vectors[i]).toArray()), + false + ); + } + + refreshAllIndices(); + // Assert we have the right number of documents in the index + assertEquals(testData.indexData.docs.length, getDocCount(indexName)); + + final int k = 10; + for (int i = 0; i < testData.queries.length; i++) { + final Response response = searchKNNIndex(indexName, new KNNQueryBuilder(fieldName, testData.queries[i], k), k); + final String responseBody = EntityUtils.toString(response.getEntity()); + final List nmslibValidNeighbors = parseSearchResponse(responseBody, fieldName); + assertEquals(k, nmslibValidNeighbors.size()); + } + // Delete index + deleteKNNIndex(indexName); + } + + /* + For this testcase, we will create index with setting build_vector_data_structure_threshold as -1, then index few documents, perform knn search, + then, confirm hits because of exact search though there are no graph. In next step, update setting to 0, force merge segment to 1, perform knn search and confirm expected + hits are returned. + */ + public void testKNNIndex_whenBuildVectorGraphThresholdIsProvidedEndToEnd_thenBuildGraphBasedOnSettingUsingRadialSearch() + throws Exception { + final String indexName = "test-index-1"; + final String fieldName1 = "test-field-1"; + final String fieldName2 = "test-field-2"; + + final Integer dimension = testData.indexData.vectors[0].length; + final Settings knnIndexSettings = buildKNNIndexSettings(-1); + + // Create an index + final XContentBuilder builder = XContentFactory.jsonBuilder() + .startObject() + .startObject("properties") + .startObject(fieldName1) + .field("type", "knn_vector") + .field("dimension", dimension) + .startObject(KNNConstants.KNN_METHOD) + .field(KNNConstants.NAME, KNNConstants.METHOD_HNSW) + .field(KNNConstants.KNN_ENGINE, KNNEngine.NMSLIB.getName()) + .startObject(KNNConstants.PARAMETERS) + .endObject() + .endObject() + .endObject() + .startObject(fieldName2) + .field("type", "knn_vector") + .field("dimension", dimension) + .startObject(KNNConstants.KNN_METHOD) + .field(KNNConstants.NAME, KNNConstants.METHOD_HNSW) + .field(KNNConstants.KNN_ENGINE, KNNEngine.FAISS.getName()) + .startObject(KNNConstants.PARAMETERS) + .endObject() + .endObject() + .endObject() + .endObject() + .endObject(); + + createKnnIndex(indexName, knnIndexSettings, builder.toString()); + + // Index the test data + for (int i = 0; i < testData.indexData.docs.length; i++) { + addKnnDoc( + indexName, + Integer.toString(testData.indexData.docs[i]), + ImmutableList.of(fieldName1, fieldName2), + ImmutableList.of( + Floats.asList(testData.indexData.vectors[i]).toArray(), + Floats.asList(testData.indexData.vectors[i]).toArray() + ) + ); + } + + refreshAllIndices(); + // Assert we have the right number of documents in the index + assertEquals(testData.indexData.docs.length, getDocCount(indexName)); + + final List nmslibNeighbors = getResults(indexName, fieldName1, testData.queries[0], 1); + assertEquals("unexpected neighbors are returned", nmslibNeighbors.size(), nmslibNeighbors.size()); + + final List faissNeighbors = getResults(indexName, fieldName2, testData.queries[0], 1); + assertEquals("unexpected neighbors are returned", faissNeighbors.size(), faissNeighbors.size()); + + // update build vector data structure setting + updateIndexSettings(indexName, Settings.builder().put(KNNSettings.INDEX_KNN_ADVANCED_APPROXIMATE_THRESHOLD, 0)); + forceMergeKnnIndex(indexName, 1); + + final int k = 10; + for (int i = 0; i < testData.queries.length; i++) { + // Search nmslib field + final Response response = searchKNNIndex(indexName, new KNNQueryBuilder(fieldName1, testData.queries[i], k), k); + final String responseBody = EntityUtils.toString(response.getEntity()); + final List nmslibValidNeighbors = parseSearchResponse(responseBody, fieldName1); + assertEquals(k, nmslibValidNeighbors.size()); + // Search faiss field + final List faissValidNeighbors = getResults(indexName, fieldName2, testData.queries[i], k); + assertEquals(k, faissValidNeighbors.size()); + } + + // Delete index + deleteKNNIndex(indexName); + } + + private List getResults(final String indexName, final String fieldName, final float[] vector, final int k) + throws IOException, ParseException { + final Response searchResponseField = searchKNNIndex(indexName, new KNNQueryBuilder(fieldName, vector, k), k); + final String searchResponseBody = EntityUtils.toString(searchResponseField.getEntity()); + return parseSearchResponse(searchResponseBody, fieldName); + } + } diff --git a/src/test/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngineFieldVectorsWriterTests.java b/src/test/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngineFieldVectorsWriterTests.java index 6e6a51b88..4f68a360e 100644 --- a/src/test/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngineFieldVectorsWriterTests.java +++ b/src/test/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngineFieldVectorsWriterTests.java @@ -126,6 +126,5 @@ public void testRamByteUsed_whenValidInput_thenSuccess() { // testing for value > 0 as we don't have a concrete way to find out expected bytes. This can OS dependent too. Assert.assertTrue(byteWriter.ramBytesUsed() > 0); Mockito.verify(mockedFlatFieldVectorsWriter, Mockito.times(2)).ramBytesUsed(); - } } diff --git a/src/test/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsWriterFlushTests.java b/src/test/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsWriterFlushTests.java index 5bb6d1926..03d0f6160 100644 --- a/src/test/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsWriterFlushTests.java +++ b/src/test/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsWriterFlushTests.java @@ -31,10 +31,12 @@ import org.opensearch.knn.quantization.models.quantizationState.QuantizationState; import org.opensearch.test.OpenSearchTestCase; +import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.function.Predicate; @@ -52,6 +54,7 @@ import static org.mockito.Mockito.never; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; import static org.mockito.Mockito.when; @RequiredArgsConstructor @@ -76,12 +79,14 @@ public class NativeEngines990KnnVectorsWriterFlushTests extends OpenSearchTestCa private final String description; private final List> vectorsPerField; + private static final Integer BUILD_GRAPH_ALWAYS_THRESHOLD = 0; + private static final Integer BUILD_GRAPH_NEVER_THRESHOLD = -1; @Override public void setUp() throws Exception { super.setUp(); MockitoAnnotations.openMocks(this); - objectUnderTest = new NativeEngines990KnnVectorsWriter(segmentWriteState, flatVectorsWriter); + objectUnderTest = new NativeEngines990KnnVectorsWriter(segmentWriteState, flatVectorsWriter, BUILD_GRAPH_ALWAYS_THRESHOLD); mockedFlatFieldVectorsWriter = Mockito.mock(FlatFieldVectorsWriter.class); Mockito.doNothing().when(mockedFlatFieldVectorsWriter).addValue(Mockito.anyInt(), Mockito.any()); Mockito.when(flatVectorsWriter.addField(Mockito.any())).thenReturn(mockedFlatFieldVectorsWriter); @@ -299,6 +304,539 @@ public void testFlush_WithQuantization() { } } + public void testFlush_whenThresholdIsNegative_thenNativeIndexWriterIsNeverCalled() throws IOException { + // Given + List> expectedVectorValues = new ArrayList<>(); + IntStream.range(0, vectorsPerField.size()).forEach(i -> { + final TestVectorValues.PreDefinedFloatVectorValues randomVectorValues = new TestVectorValues.PreDefinedFloatVectorValues( + new ArrayList<>(vectorsPerField.get(i).values()) + ); + final KNNVectorValues knnVectorValues = KNNVectorValuesFactory.getVectorValues( + VectorDataType.FLOAT, + randomVectorValues + ); + expectedVectorValues.add(knnVectorValues); + + }); + + final NativeEngines990KnnVectorsWriter nativeEngineWriter = new NativeEngines990KnnVectorsWriter( + segmentWriteState, + flatVectorsWriter, + BUILD_GRAPH_NEVER_THRESHOLD + ); + + try ( + MockedStatic fieldWriterMockedStatic = mockStatic(NativeEngineFieldVectorsWriter.class); + MockedStatic knnVectorValuesFactoryMockedStatic = mockStatic(KNNVectorValuesFactory.class); + MockedStatic quantizationServiceMockedStatic = mockStatic(QuantizationService.class); + MockedStatic nativeIndexWriterMockedStatic = mockStatic(NativeIndexWriter.class); + MockedConstruction knn990QuantWriterMockedConstruction = mockConstruction( + KNN990QuantizationStateWriter.class + ); + ) { + quantizationServiceMockedStatic.when(() -> QuantizationService.getInstance()).thenReturn(quantizationService); + IntStream.range(0, vectorsPerField.size()).forEach(i -> { + final FieldInfo fieldInfo = fieldInfo( + i, + VectorEncoding.FLOAT32, + Map.of(KNNConstants.VECTOR_DATA_TYPE_FIELD, "float", KNNConstants.KNN_ENGINE, "faiss") + ); + + NativeEngineFieldVectorsWriter field = nativeEngineFieldVectorsWriter(fieldInfo, vectorsPerField.get(i)); + fieldWriterMockedStatic.when( + () -> NativeEngineFieldVectorsWriter.create(fieldInfo, mockedFlatFieldVectorsWriter, segmentWriteState.infoStream) + ).thenReturn(field); + try { + nativeEngineWriter.addField(fieldInfo); + } catch (Exception e) { + throw new RuntimeException(e); + } + + DocsWithFieldSet docsWithFieldSet = field.getDocsWithField(); + knnVectorValuesFactoryMockedStatic.when( + () -> KNNVectorValuesFactory.getVectorValues(VectorDataType.FLOAT, docsWithFieldSet, vectorsPerField.get(i)) + ).thenReturn(expectedVectorValues.get(i)); + + when(quantizationService.getQuantizationParams(fieldInfo)).thenReturn(null); + nativeIndexWriterMockedStatic.when(() -> NativeIndexWriter.getWriter(fieldInfo, segmentWriteState, null)) + .thenReturn(nativeIndexWriter); + }); + + doAnswer(answer -> { + Thread.sleep(2); // Need this for KNNGraph value assertion, removing this will fail the assertion + return null; + }).when(nativeIndexWriter).flushIndex(any(), anyInt()); + + // When + nativeEngineWriter.flush(5, null); + + // Then + verify(flatVectorsWriter).flush(5, null); + if (vectorsPerField.size() > 0) { + assertEquals(0, knn990QuantWriterMockedConstruction.constructed().size()); + } + verifyNoInteractions(nativeIndexWriter); + } + } + + public void testFlush_whenThresholdIsGreaterThanVectorSize_thenNativeIndexWriterIsNeverCalled() throws IOException { + // Given + List> expectedVectorValues = new ArrayList<>(); + final Map sizeMap = new HashMap<>(); + IntStream.range(0, vectorsPerField.size()).forEach(i -> { + final TestVectorValues.PreDefinedFloatVectorValues randomVectorValues = new TestVectorValues.PreDefinedFloatVectorValues( + new ArrayList<>(vectorsPerField.get(i).values()) + ); + final KNNVectorValues knnVectorValues = KNNVectorValuesFactory.getVectorValues( + VectorDataType.FLOAT, + randomVectorValues + ); + sizeMap.put(i, randomVectorValues.size()); + expectedVectorValues.add(knnVectorValues); + + }); + final int maxThreshold = sizeMap.values().stream().filter(count -> count != 0).max(Integer::compareTo).orElse(0); + final NativeEngines990KnnVectorsWriter nativeEngineWriter = new NativeEngines990KnnVectorsWriter( + segmentWriteState, + flatVectorsWriter, + maxThreshold + 1 + ); + + try ( + MockedStatic fieldWriterMockedStatic = mockStatic(NativeEngineFieldVectorsWriter.class); + MockedStatic knnVectorValuesFactoryMockedStatic = mockStatic(KNNVectorValuesFactory.class); + MockedStatic quantizationServiceMockedStatic = mockStatic(QuantizationService.class); + MockedStatic nativeIndexWriterMockedStatic = mockStatic(NativeIndexWriter.class); + MockedConstruction knn990QuantWriterMockedConstruction = mockConstruction( + KNN990QuantizationStateWriter.class + ); + ) { + quantizationServiceMockedStatic.when(() -> QuantizationService.getInstance()).thenReturn(quantizationService); + IntStream.range(0, vectorsPerField.size()).forEach(i -> { + final FieldInfo fieldInfo = fieldInfo( + i, + VectorEncoding.FLOAT32, + Map.of(KNNConstants.VECTOR_DATA_TYPE_FIELD, "float", KNNConstants.KNN_ENGINE, "faiss") + ); + + NativeEngineFieldVectorsWriter field = nativeEngineFieldVectorsWriter(fieldInfo, vectorsPerField.get(i)); + fieldWriterMockedStatic.when( + () -> NativeEngineFieldVectorsWriter.create(fieldInfo, mockedFlatFieldVectorsWriter, segmentWriteState.infoStream) + ).thenReturn(field); + try { + nativeEngineWriter.addField(fieldInfo); + } catch (Exception e) { + throw new RuntimeException(e); + } + + DocsWithFieldSet docsWithFieldSet = field.getDocsWithField(); + knnVectorValuesFactoryMockedStatic.when( + () -> KNNVectorValuesFactory.getVectorValues(VectorDataType.FLOAT, docsWithFieldSet, vectorsPerField.get(i)) + ).thenReturn(expectedVectorValues.get(i)); + + when(quantizationService.getQuantizationParams(fieldInfo)).thenReturn(null); + nativeIndexWriterMockedStatic.when(() -> NativeIndexWriter.getWriter(fieldInfo, segmentWriteState, null)) + .thenReturn(nativeIndexWriter); + }); + + doAnswer(answer -> { + Thread.sleep(2); // Need this for KNNGraph value assertion, removing this will fail the assertion + return null; + }).when(nativeIndexWriter).flushIndex(any(), anyInt()); + + // When + nativeEngineWriter.flush(5, null); + + // Then + verify(flatVectorsWriter).flush(5, null); + if (vectorsPerField.size() > 0) { + assertEquals(0, knn990QuantWriterMockedConstruction.constructed().size()); + } + verifyNoInteractions(nativeIndexWriter); + } + } + + public void testFlush_whenThresholdIsEqualToMinNumberOfVectors_thenNativeIndexWriterIsCalled() throws IOException { + // Given + List> expectedVectorValues = new ArrayList<>(); + final Map sizeMap = new HashMap<>(); + IntStream.range(0, vectorsPerField.size()).forEach(i -> { + final TestVectorValues.PreDefinedFloatVectorValues randomVectorValues = new TestVectorValues.PreDefinedFloatVectorValues( + new ArrayList<>(vectorsPerField.get(i).values()) + ); + final KNNVectorValues knnVectorValues = KNNVectorValuesFactory.getVectorValues( + VectorDataType.FLOAT, + randomVectorValues + ); + sizeMap.put(i, randomVectorValues.size()); + expectedVectorValues.add(knnVectorValues); + + }); + + final int minThreshold = sizeMap.values().stream().filter(count -> count != 0).min(Integer::compareTo).orElse(0); + final NativeEngines990KnnVectorsWriter nativeEngineWriter = new NativeEngines990KnnVectorsWriter( + segmentWriteState, + flatVectorsWriter, + minThreshold + ); + + try ( + MockedStatic fieldWriterMockedStatic = mockStatic(NativeEngineFieldVectorsWriter.class); + MockedStatic knnVectorValuesFactoryMockedStatic = mockStatic(KNNVectorValuesFactory.class); + MockedStatic quantizationServiceMockedStatic = mockStatic(QuantizationService.class); + MockedStatic nativeIndexWriterMockedStatic = mockStatic(NativeIndexWriter.class); + MockedConstruction knn990QuantWriterMockedConstruction = mockConstruction( + KNN990QuantizationStateWriter.class + ); + ) { + quantizationServiceMockedStatic.when(() -> QuantizationService.getInstance()).thenReturn(quantizationService); + IntStream.range(0, vectorsPerField.size()).forEach(i -> { + final FieldInfo fieldInfo = fieldInfo( + i, + VectorEncoding.FLOAT32, + Map.of(KNNConstants.VECTOR_DATA_TYPE_FIELD, "float", KNNConstants.KNN_ENGINE, "faiss") + ); + + NativeEngineFieldVectorsWriter field = nativeEngineFieldVectorsWriter(fieldInfo, vectorsPerField.get(i)); + fieldWriterMockedStatic.when( + () -> NativeEngineFieldVectorsWriter.create(fieldInfo, mockedFlatFieldVectorsWriter, segmentWriteState.infoStream) + ).thenReturn(field); + try { + nativeEngineWriter.addField(fieldInfo); + } catch (Exception e) { + throw new RuntimeException(e); + } + + DocsWithFieldSet docsWithFieldSet = field.getDocsWithField(); + knnVectorValuesFactoryMockedStatic.when( + () -> KNNVectorValuesFactory.getVectorValues(VectorDataType.FLOAT, docsWithFieldSet, vectorsPerField.get(i)) + ).thenReturn(expectedVectorValues.get(i)); + + when(quantizationService.getQuantizationParams(fieldInfo)).thenReturn(null); + nativeIndexWriterMockedStatic.when(() -> NativeIndexWriter.getWriter(fieldInfo, segmentWriteState, null)) + .thenReturn(nativeIndexWriter); + }); + + doAnswer(answer -> { + Thread.sleep(2); // Need this for KNNGraph value assertion, removing this will fail the assertion + return null; + }).when(nativeIndexWriter).flushIndex(any(), anyInt()); + + // When + nativeEngineWriter.flush(5, null); + + // Then + verify(flatVectorsWriter).flush(5, null); + if (vectorsPerField.size() > 0) { + assertEquals(0, knn990QuantWriterMockedConstruction.constructed().size()); + assertTrue((long) KNNGraphValue.REFRESH_TOTAL_TIME_IN_MILLIS.getValue() > 0); + } + IntStream.range(0, vectorsPerField.size()).forEach(i -> { + try { + if (vectorsPerField.get(i).size() > 0) { + verify(nativeIndexWriter).flushIndex(expectedVectorValues.get(i), vectorsPerField.get(i).size()); + } else { + verify(nativeIndexWriter, never()).flushIndex(expectedVectorValues.get(i), vectorsPerField.get(i).size()); + } + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + } + } + + public void testFlush_whenThresholdIsEqualToFixedValue_thenRelevantNativeIndexWriterIsCalled() throws IOException { + // Given + List> expectedVectorValues = new ArrayList<>(); + IntStream.range(0, vectorsPerField.size()).forEach(i -> { + final TestVectorValues.PreDefinedFloatVectorValues randomVectorValues = new TestVectorValues.PreDefinedFloatVectorValues( + new ArrayList<>(vectorsPerField.get(i).values()) + ); + final KNNVectorValues knnVectorValues = KNNVectorValuesFactory.getVectorValues( + VectorDataType.FLOAT, + randomVectorValues + ); + expectedVectorValues.add(knnVectorValues); + + }); + final int threshold = 4; + final NativeEngines990KnnVectorsWriter nativeEngineWriter = new NativeEngines990KnnVectorsWriter( + segmentWriteState, + flatVectorsWriter, + threshold + ); + + try ( + MockedStatic fieldWriterMockedStatic = mockStatic(NativeEngineFieldVectorsWriter.class); + MockedStatic knnVectorValuesFactoryMockedStatic = mockStatic(KNNVectorValuesFactory.class); + MockedStatic quantizationServiceMockedStatic = mockStatic(QuantizationService.class); + MockedStatic nativeIndexWriterMockedStatic = mockStatic(NativeIndexWriter.class); + MockedConstruction knn990QuantWriterMockedConstruction = mockConstruction( + KNN990QuantizationStateWriter.class + ); + ) { + quantizationServiceMockedStatic.when(() -> QuantizationService.getInstance()).thenReturn(quantizationService); + IntStream.range(0, vectorsPerField.size()).forEach(i -> { + final FieldInfo fieldInfo = fieldInfo( + i, + VectorEncoding.FLOAT32, + Map.of(KNNConstants.VECTOR_DATA_TYPE_FIELD, "float", KNNConstants.KNN_ENGINE, "faiss") + ); + + NativeEngineFieldVectorsWriter field = nativeEngineFieldVectorsWriter(fieldInfo, vectorsPerField.get(i)); + fieldWriterMockedStatic.when( + () -> NativeEngineFieldVectorsWriter.create(fieldInfo, mockedFlatFieldVectorsWriter, segmentWriteState.infoStream) + ).thenReturn(field); + try { + nativeEngineWriter.addField(fieldInfo); + } catch (Exception e) { + throw new RuntimeException(e); + } + + DocsWithFieldSet docsWithFieldSet = field.getDocsWithField(); + knnVectorValuesFactoryMockedStatic.when( + () -> KNNVectorValuesFactory.getVectorValues(VectorDataType.FLOAT, docsWithFieldSet, vectorsPerField.get(i)) + ).thenReturn(expectedVectorValues.get(i)); + + when(quantizationService.getQuantizationParams(fieldInfo)).thenReturn(null); + nativeIndexWriterMockedStatic.when(() -> NativeIndexWriter.getWriter(fieldInfo, segmentWriteState, null)) + .thenReturn(nativeIndexWriter); + }); + + doAnswer(answer -> { + Thread.sleep(2); // Need this for KNNGraph value assertion, removing this will fail the assertion + return null; + }).when(nativeIndexWriter).flushIndex(any(), anyInt()); + + // When + nativeEngineWriter.flush(5, null); + + // Then + verify(flatVectorsWriter).flush(5, null); + if (vectorsPerField.size() > 0) { + assertEquals(0, knn990QuantWriterMockedConstruction.constructed().size()); + } + IntStream.range(0, vectorsPerField.size()).forEach(i -> { + try { + if (vectorsPerField.get(i).size() >= threshold) { + verify(nativeIndexWriter).flushIndex(expectedVectorValues.get(i), vectorsPerField.get(i).size()); + } else { + verify(nativeIndexWriter, never()).flushIndex(expectedVectorValues.get(i), vectorsPerField.get(i).size()); + } + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + } + } + + public void testFlush_whenQuantizationIsProvided_whenBuildGraphDatStructureThresholdIsNotMet_thenSkipBuildingGraph() + throws IOException { + // Given + List> expectedVectorValues = new ArrayList<>(); + final Map sizeMap = new HashMap<>(); + IntStream.range(0, vectorsPerField.size()).forEach(i -> { + final TestVectorValues.PreDefinedFloatVectorValues randomVectorValues = new TestVectorValues.PreDefinedFloatVectorValues( + new ArrayList<>(vectorsPerField.get(i).values()) + ); + final KNNVectorValues knnVectorValues = KNNVectorValuesFactory.getVectorValues( + VectorDataType.FLOAT, + randomVectorValues + ); + sizeMap.put(i, randomVectorValues.size()); + expectedVectorValues.add(knnVectorValues); + + }); + final int maxThreshold = sizeMap.values().stream().filter(count -> count != 0).max(Integer::compareTo).orElse(0); + final NativeEngines990KnnVectorsWriter nativeEngineWriter = new NativeEngines990KnnVectorsWriter( + segmentWriteState, + flatVectorsWriter, + maxThreshold + 1 // to avoid building graph using max doc threshold, the same can be achieved by -1 too + ); + + try ( + MockedStatic fieldWriterMockedStatic = mockStatic(NativeEngineFieldVectorsWriter.class); + MockedStatic knnVectorValuesFactoryMockedStatic = mockStatic(KNNVectorValuesFactory.class); + MockedStatic quantizationServiceMockedStatic = mockStatic(QuantizationService.class); + MockedStatic nativeIndexWriterMockedStatic = mockStatic(NativeIndexWriter.class); + MockedConstruction knn990QuantWriterMockedConstruction = mockConstruction( + KNN990QuantizationStateWriter.class + ); + ) { + quantizationServiceMockedStatic.when(() -> QuantizationService.getInstance()).thenReturn(quantizationService); + + IntStream.range(0, vectorsPerField.size()).forEach(i -> { + final FieldInfo fieldInfo = fieldInfo( + i, + VectorEncoding.FLOAT32, + Map.of(KNNConstants.VECTOR_DATA_TYPE_FIELD, "float", KNNConstants.KNN_ENGINE, "faiss") + ); + + NativeEngineFieldVectorsWriter field = nativeEngineFieldVectorsWriter(fieldInfo, vectorsPerField.get(i)); + fieldWriterMockedStatic.when( + () -> NativeEngineFieldVectorsWriter.create(fieldInfo, mockedFlatFieldVectorsWriter, segmentWriteState.infoStream) + ).thenReturn(field); + + try { + nativeEngineWriter.addField(fieldInfo); + } catch (Exception e) { + throw new RuntimeException(e); + } + + DocsWithFieldSet docsWithFieldSet = field.getDocsWithField(); + knnVectorValuesFactoryMockedStatic.when( + () -> KNNVectorValuesFactory.getVectorValues(VectorDataType.FLOAT, docsWithFieldSet, vectorsPerField.get(i)) + ).thenReturn(expectedVectorValues.get(i)); + + when(quantizationService.getQuantizationParams(fieldInfo)).thenReturn(quantizationParams); + try { + when(quantizationService.train(quantizationParams, expectedVectorValues.get(i), vectorsPerField.get(i).size())) + .thenReturn(quantizationState); + } catch (Exception e) { + throw new RuntimeException(e); + } + + nativeIndexWriterMockedStatic.when(() -> NativeIndexWriter.getWriter(fieldInfo, segmentWriteState, quantizationState)) + .thenReturn(nativeIndexWriter); + }); + doAnswer(answer -> { + Thread.sleep(2); // Need this for KNNGraph value assertion, removing this will fail the assertion + return null; + }).when(nativeIndexWriter).flushIndex(any(), anyInt()); + + // When + nativeEngineWriter.flush(5, null); + + // Then + verify(flatVectorsWriter).flush(5, null); + if (vectorsPerField.size() > 0) { + verify(knn990QuantWriterMockedConstruction.constructed().get(0)).writeHeader(segmentWriteState); + } else { + assertEquals(0, knn990QuantWriterMockedConstruction.constructed().size()); + } + verifyNoInteractions(nativeIndexWriter); + IntStream.range(0, vectorsPerField.size()).forEach(i -> { + try { + if (vectorsPerField.get(i).isEmpty()) { + verify(knn990QuantWriterMockedConstruction.constructed().get(0), never()).writeState(i, quantizationState); + } else { + verify(knn990QuantWriterMockedConstruction.constructed().get(0)).writeState(i, quantizationState); + } + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + final Long expectedTimesGetVectorValuesIsCalled = vectorsPerField.stream().filter(Predicate.not(Map::isEmpty)).count(); + knnVectorValuesFactoryMockedStatic.verify( + () -> KNNVectorValuesFactory.getVectorValues(any(VectorDataType.class), any(DocsWithFieldSet.class), any()), + times(Math.toIntExact(expectedTimesGetVectorValuesIsCalled)) + ); + } + } + + public void testFlush_whenQuantizationIsProvided_whenBuildGraphDatStructureThresholdIsNegative_thenSkipBuildingGraph() + throws IOException { + // Given + List> expectedVectorValues = new ArrayList<>(); + final Map sizeMap = new HashMap<>(); + IntStream.range(0, vectorsPerField.size()).forEach(i -> { + final TestVectorValues.PreDefinedFloatVectorValues randomVectorValues = new TestVectorValues.PreDefinedFloatVectorValues( + new ArrayList<>(vectorsPerField.get(i).values()) + ); + final KNNVectorValues knnVectorValues = KNNVectorValuesFactory.getVectorValues( + VectorDataType.FLOAT, + randomVectorValues + ); + sizeMap.put(i, randomVectorValues.size()); + expectedVectorValues.add(knnVectorValues); + + }); + final NativeEngines990KnnVectorsWriter nativeEngineWriter = new NativeEngines990KnnVectorsWriter( + segmentWriteState, + flatVectorsWriter, + BUILD_GRAPH_NEVER_THRESHOLD + ); + + try ( + MockedStatic fieldWriterMockedStatic = mockStatic(NativeEngineFieldVectorsWriter.class); + MockedStatic knnVectorValuesFactoryMockedStatic = mockStatic(KNNVectorValuesFactory.class); + MockedStatic quantizationServiceMockedStatic = mockStatic(QuantizationService.class); + MockedStatic nativeIndexWriterMockedStatic = mockStatic(NativeIndexWriter.class); + MockedConstruction knn990QuantWriterMockedConstruction = mockConstruction( + KNN990QuantizationStateWriter.class + ); + ) { + quantizationServiceMockedStatic.when(() -> QuantizationService.getInstance()).thenReturn(quantizationService); + + IntStream.range(0, vectorsPerField.size()).forEach(i -> { + final FieldInfo fieldInfo = fieldInfo( + i, + VectorEncoding.FLOAT32, + Map.of(KNNConstants.VECTOR_DATA_TYPE_FIELD, "float", KNNConstants.KNN_ENGINE, "faiss") + ); + + NativeEngineFieldVectorsWriter field = nativeEngineFieldVectorsWriter(fieldInfo, vectorsPerField.get(i)); + fieldWriterMockedStatic.when( + () -> NativeEngineFieldVectorsWriter.create(fieldInfo, mockedFlatFieldVectorsWriter, segmentWriteState.infoStream) + ).thenReturn(field); + + try { + nativeEngineWriter.addField(fieldInfo); + } catch (Exception e) { + throw new RuntimeException(e); + } + + DocsWithFieldSet docsWithFieldSet = field.getDocsWithField(); + knnVectorValuesFactoryMockedStatic.when( + () -> KNNVectorValuesFactory.getVectorValues(VectorDataType.FLOAT, docsWithFieldSet, vectorsPerField.get(i)) + ).thenReturn(expectedVectorValues.get(i)); + + when(quantizationService.getQuantizationParams(fieldInfo)).thenReturn(quantizationParams); + try { + when(quantizationService.train(quantizationParams, expectedVectorValues.get(i), vectorsPerField.get(i).size())) + .thenReturn(quantizationState); + } catch (Exception e) { + throw new RuntimeException(e); + } + + nativeIndexWriterMockedStatic.when(() -> NativeIndexWriter.getWriter(fieldInfo, segmentWriteState, quantizationState)) + .thenReturn(nativeIndexWriter); + }); + doAnswer(answer -> { + Thread.sleep(2); // Need this for KNNGraph value assertion, removing this will fail the assertion + return null; + }).when(nativeIndexWriter).flushIndex(any(), anyInt()); + + // When + nativeEngineWriter.flush(5, null); + + // Then + verify(flatVectorsWriter).flush(5, null); + if (vectorsPerField.size() > 0) { + verify(knn990QuantWriterMockedConstruction.constructed().get(0)).writeHeader(segmentWriteState); + } else { + assertEquals(0, knn990QuantWriterMockedConstruction.constructed().size()); + } + verifyNoInteractions(nativeIndexWriter); + IntStream.range(0, vectorsPerField.size()).forEach(i -> { + try { + if (vectorsPerField.get(i).isEmpty()) { + verify(knn990QuantWriterMockedConstruction.constructed().get(0), never()).writeState(i, quantizationState); + } else { + verify(knn990QuantWriterMockedConstruction.constructed().get(0)).writeState(i, quantizationState); + } + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + final Long expectedTimesGetVectorValuesIsCalled = vectorsPerField.stream().filter(Predicate.not(Map::isEmpty)).count(); + knnVectorValuesFactoryMockedStatic.verify( + () -> KNNVectorValuesFactory.getVectorValues(any(VectorDataType.class), any(DocsWithFieldSet.class), any()), + times(Math.toIntExact(expectedTimesGetVectorValuesIsCalled)) + ); + } + } + private FieldInfo fieldInfo(int fieldNumber, VectorEncoding vectorEncoding, Map attributes) { FieldInfo fieldInfo = mock(FieldInfo.class); when(fieldInfo.getFieldNumber()).thenReturn(fieldNumber); diff --git a/src/test/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsWriterMergeTests.java b/src/test/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsWriterMergeTests.java index af18cd281..77f3fd8ed 100644 --- a/src/test/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsWriterMergeTests.java +++ b/src/test/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsWriterMergeTests.java @@ -34,6 +34,7 @@ import org.opensearch.knn.quantization.models.quantizationState.QuantizationState; import org.opensearch.test.OpenSearchTestCase; +import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -77,12 +78,14 @@ public class NativeEngines990KnnVectorsWriterMergeTests extends OpenSearchTestCa private final String description; private final Map mergedVectors; private FlatFieldVectorsWriter mockedFlatFieldVectorsWriter; + private static final Integer BUILD_GRAPH_ALWAYS_THRESHOLD = 0; + private static final Integer BUILD_GRAPH_NEVER_THRESHOLD = -1; @Override public void setUp() throws Exception { super.setUp(); MockitoAnnotations.openMocks(this); - objectUnderTest = new NativeEngines990KnnVectorsWriter(segmentWriteState, flatVectorsWriter); + objectUnderTest = new NativeEngines990KnnVectorsWriter(segmentWriteState, flatVectorsWriter, BUILD_GRAPH_ALWAYS_THRESHOLD); mockedFlatFieldVectorsWriter = Mockito.mock(FlatFieldVectorsWriter.class); Mockito.doNothing().when(mockedFlatFieldVectorsWriter).addValue(Mockito.anyInt(), Mockito.any()); Mockito.when(flatVectorsWriter.addField(Mockito.any())).thenReturn(mockedFlatFieldVectorsWriter); @@ -162,6 +165,126 @@ public void testMerge() { } } + public void testMerge_whenThresholdIsNegative_thenNativeIndexWriterIsNeverCalled() throws IOException { + // Given + final TestVectorValues.PreDefinedFloatVectorValues randomVectorValues = new TestVectorValues.PreDefinedFloatVectorValues( + new ArrayList<>(mergedVectors.values()) + ); + final KNNVectorValues knnVectorValues = KNNVectorValuesFactory.getVectorValues(VectorDataType.FLOAT, randomVectorValues); + final NativeEngines990KnnVectorsWriter nativeEngineWriter = new NativeEngines990KnnVectorsWriter( + segmentWriteState, + flatVectorsWriter, + BUILD_GRAPH_NEVER_THRESHOLD + ); + try ( + MockedStatic fieldWriterMockedStatic = mockStatic(NativeEngineFieldVectorsWriter.class); + MockedStatic knnVectorValuesFactoryMockedStatic = mockStatic(KNNVectorValuesFactory.class); + MockedStatic quantizationServiceMockedStatic = mockStatic(QuantizationService.class); + MockedStatic nativeIndexWriterMockedStatic = mockStatic(NativeIndexWriter.class); + MockedStatic mergedVectorValuesMockedStatic = mockStatic( + KnnVectorsWriter.MergedVectorValues.class + ); + MockedConstruction knn990QuantWriterMockedConstruction = mockConstruction( + KNN990QuantizationStateWriter.class + ); + ) { + quantizationServiceMockedStatic.when(() -> QuantizationService.getInstance()).thenReturn(quantizationService); + final FieldInfo fieldInfo = fieldInfo( + 0, + VectorEncoding.FLOAT32, + Map.of(KNNConstants.VECTOR_DATA_TYPE_FIELD, "float", KNNConstants.KNN_ENGINE, "faiss") + ); + + NativeEngineFieldVectorsWriter field = nativeEngineFieldVectorsWriter(fieldInfo, mergedVectors); + fieldWriterMockedStatic.when( + () -> NativeEngineFieldVectorsWriter.create(fieldInfo, mockedFlatFieldVectorsWriter, segmentWriteState.infoStream) + ).thenReturn(field); + + mergedVectorValuesMockedStatic.when(() -> KnnVectorsWriter.MergedVectorValues.mergeFloatVectorValues(fieldInfo, mergeState)) + .thenReturn(floatVectorValues); + knnVectorValuesFactoryMockedStatic.when(() -> KNNVectorValuesFactory.getVectorValues(VectorDataType.FLOAT, floatVectorValues)) + .thenReturn(knnVectorValues); + + when(quantizationService.getQuantizationParams(fieldInfo)).thenReturn(null); + nativeIndexWriterMockedStatic.when(() -> NativeIndexWriter.getWriter(fieldInfo, segmentWriteState, null)) + .thenReturn(nativeIndexWriter); + doAnswer(answer -> { + Thread.sleep(2); // Need this for KNNGraph value assertion, removing this will fail the assertion + return null; + }).when(nativeIndexWriter).mergeIndex(any(), anyInt()); + + // When + nativeEngineWriter.mergeOneField(fieldInfo, mergeState); + + // Then + verify(flatVectorsWriter).mergeOneField(fieldInfo, mergeState); + assertEquals(0, knn990QuantWriterMockedConstruction.constructed().size()); + verifyNoInteractions(nativeIndexWriter); + } + } + + public void testMerge_whenThresholdIsEqualToNumberOfVectors_thenNativeIndexWriterIsCalled() throws IOException { + // Given + final TestVectorValues.PreDefinedFloatVectorValues randomVectorValues = new TestVectorValues.PreDefinedFloatVectorValues( + new ArrayList<>(mergedVectors.values()) + ); + final KNNVectorValues knnVectorValues = KNNVectorValuesFactory.getVectorValues(VectorDataType.FLOAT, randomVectorValues); + final NativeEngines990KnnVectorsWriter nativeEngineWriter = new NativeEngines990KnnVectorsWriter( + segmentWriteState, + flatVectorsWriter, + mergedVectors.size() + ); + try ( + MockedStatic fieldWriterMockedStatic = mockStatic(NativeEngineFieldVectorsWriter.class); + MockedStatic knnVectorValuesFactoryMockedStatic = mockStatic(KNNVectorValuesFactory.class); + MockedStatic quantizationServiceMockedStatic = mockStatic(QuantizationService.class); + MockedStatic nativeIndexWriterMockedStatic = mockStatic(NativeIndexWriter.class); + MockedStatic mergedVectorValuesMockedStatic = mockStatic( + KnnVectorsWriter.MergedVectorValues.class + ); + MockedConstruction knn990QuantWriterMockedConstruction = mockConstruction( + KNN990QuantizationStateWriter.class + ); + ) { + quantizationServiceMockedStatic.when(() -> QuantizationService.getInstance()).thenReturn(quantizationService); + final FieldInfo fieldInfo = fieldInfo( + 0, + VectorEncoding.FLOAT32, + Map.of(KNNConstants.VECTOR_DATA_TYPE_FIELD, "float", KNNConstants.KNN_ENGINE, "faiss") + ); + + NativeEngineFieldVectorsWriter field = nativeEngineFieldVectorsWriter(fieldInfo, mergedVectors); + fieldWriterMockedStatic.when( + () -> NativeEngineFieldVectorsWriter.create(fieldInfo, mockedFlatFieldVectorsWriter, segmentWriteState.infoStream) + ).thenReturn(field); + + mergedVectorValuesMockedStatic.when(() -> KnnVectorsWriter.MergedVectorValues.mergeFloatVectorValues(fieldInfo, mergeState)) + .thenReturn(floatVectorValues); + knnVectorValuesFactoryMockedStatic.when(() -> KNNVectorValuesFactory.getVectorValues(VectorDataType.FLOAT, floatVectorValues)) + .thenReturn(knnVectorValues); + + when(quantizationService.getQuantizationParams(fieldInfo)).thenReturn(null); + nativeIndexWriterMockedStatic.when(() -> NativeIndexWriter.getWriter(fieldInfo, segmentWriteState, null)) + .thenReturn(nativeIndexWriter); + doAnswer(answer -> { + Thread.sleep(2); // Need this for KNNGraph value assertion, removing this will fail the assertion + return null; + }).when(nativeIndexWriter).mergeIndex(any(), anyInt()); + + // When + nativeEngineWriter.mergeOneField(fieldInfo, mergeState); + + // Then + verify(flatVectorsWriter).mergeOneField(fieldInfo, mergeState); + assertEquals(0, knn990QuantWriterMockedConstruction.constructed().size()); + if (!mergedVectors.isEmpty()) { + verify(nativeIndexWriter).mergeIndex(knnVectorValues, mergedVectors.size()); + } else { + verifyNoInteractions(nativeIndexWriter); + } + } + } + @SneakyThrows public void testMerge_WithQuantization() { // Given diff --git a/src/test/java/org/opensearch/knn/index/query/ExactSearcherTests.java b/src/test/java/org/opensearch/knn/index/query/ExactSearcherTests.java new file mode 100644 index 000000000..8492ca1f0 --- /dev/null +++ b/src/test/java/org/opensearch/knn/index/query/ExactSearcherTests.java @@ -0,0 +1,139 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.query; + +import lombok.SneakyThrows; +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.FieldInfos; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.SegmentCommitInfo; +import org.apache.lucene.index.SegmentInfo; +import org.apache.lucene.index.SegmentReader; +import org.apache.lucene.search.Sort; +import org.apache.lucene.store.FSDirectory; +import org.apache.lucene.util.StringHelper; +import org.apache.lucene.util.Version; +import org.mockito.MockedStatic; +import org.mockito.Mockito; +import org.opensearch.knn.KNNTestCase; +import org.opensearch.knn.index.SpaceType; +import org.opensearch.knn.index.codec.KNNCodecVersion; +import org.opensearch.knn.index.engine.KNNEngine; +import org.opensearch.knn.index.vectorvalues.KNNFloatVectorValues; +import org.opensearch.knn.index.vectorvalues.KNNVectorValuesFactory; + +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import static org.opensearch.knn.KNNRestTestCase.FIELD_NAME; +import static org.opensearch.knn.KNNRestTestCase.INDEX_NAME; +import static org.opensearch.knn.common.KNNConstants.INDEX_DESCRIPTION_PARAMETER; +import static org.opensearch.knn.common.KNNConstants.KNN_ENGINE; +import static org.opensearch.knn.common.KNNConstants.PARAMETERS; +import static org.opensearch.knn.common.KNNConstants.SPACE_TYPE; + +public class ExactSearcherTests extends KNNTestCase { + + private static final String SEGMENT_NAME = "0"; + + @SneakyThrows + public void testRadialSearch_whenNoEngineFiles_thenSuccess() { + try (MockedStatic valuesFactoryMockedStatic = Mockito.mockStatic(KNNVectorValuesFactory.class)) { + final float[] queryVector = new float[] { 0.1f, 2.0f, 3.0f }; + final SpaceType spaceType = randomFrom(SpaceType.L2, SpaceType.INNER_PRODUCT); + final List dataVectors = Arrays.asList( + new float[] { 11.0f, 12.0f, 13.0f }, + new float[] { 14.0f, 15.0f, 16.0f }, + new float[] { 17.0f, 18.0f, 19.0f } + ); + final List expectedScores = dataVectors.stream() + .map(vector -> spaceType.getKnnVectorSimilarityFunction().compare(queryVector, vector)) + .collect(Collectors.toList()); + final Float score = Collections.min(expectedScores); + final float radius = KNNEngine.FAISS.scoreToRadialThreshold(score, spaceType); + final int maxResults = 1000; + final KNNQuery.Context context = mock(KNNQuery.Context.class); + when(context.getMaxResultWindow()).thenReturn(maxResults); + KNNWeight.initialize(null); + + final KNNQuery query = KNNQuery.builder() + .field(FIELD_NAME) + .queryVector(queryVector) + .radius(radius) + .indexName(INDEX_NAME) + .context(context) + .build(); + + final ExactSearcher.ExactSearcherContext.ExactSearcherContextBuilder exactSearcherContextBuilder = + ExactSearcher.ExactSearcherContext.builder() + // setting to true, so that if quantization details are present we want to do search on the quantized + // vectors as this flow is used in first pass of search. + .useQuantizedVectorsForSearch(false) + .knnQuery(query); + + ExactSearcher exactSearcher = new ExactSearcher(null); + final LeafReaderContext leafReaderContext = mock(LeafReaderContext.class); + final SegmentReader reader = mock(SegmentReader.class); + when(leafReaderContext.reader()).thenReturn(reader); + + final FSDirectory directory = mock(FSDirectory.class); + when(reader.directory()).thenReturn(directory); + final SegmentInfo segmentInfo = new SegmentInfo( + directory, + Version.LATEST, + Version.LATEST, + SEGMENT_NAME, + 100, + false, + false, + KNNCodecVersion.current().getDefaultCodecDelegate(), + Map.of(), + new byte[StringHelper.ID_LENGTH], + Map.of(), + Sort.RELEVANCE + ); + segmentInfo.setFiles(Set.of()); + final SegmentCommitInfo segmentCommitInfo = new SegmentCommitInfo(segmentInfo, 0, 0, 0, 0, 0, new byte[StringHelper.ID_LENGTH]); + when(reader.getSegmentInfo()).thenReturn(segmentCommitInfo); + + final Path path = mock(Path.class); + when(directory.getDirectory()).thenReturn(path); + final FieldInfos fieldInfos = mock(FieldInfos.class); + final FieldInfo fieldInfo = mock(FieldInfo.class); + when(reader.getFieldInfos()).thenReturn(fieldInfos); + when(fieldInfos.fieldInfo(any())).thenReturn(fieldInfo); + when(fieldInfo.attributes()).thenReturn( + Map.of( + SPACE_TYPE, + spaceType.getValue(), + KNN_ENGINE, + KNNEngine.FAISS.getName(), + PARAMETERS, + String.format(Locale.ROOT, "{\"%s\":\"%s\"}", INDEX_DESCRIPTION_PARAMETER, "HNSW32") + ) + ); + when(fieldInfo.getAttribute(SPACE_TYPE)).thenReturn(spaceType.getValue()); + KNNFloatVectorValues floatVectorValues = mock(KNNFloatVectorValues.class); + valuesFactoryMockedStatic.when(() -> KNNVectorValuesFactory.getVectorValues(fieldInfo, reader)).thenReturn(floatVectorValues); + when(floatVectorValues.nextDoc()).thenReturn(0, 1, 2, NO_MORE_DOCS); + when(floatVectorValues.getVector()).thenReturn(dataVectors.get(0), dataVectors.get(1), dataVectors.get(2)); + final Map integerFloatMap = exactSearcher.searchLeaf(leafReaderContext, exactSearcherContextBuilder.build()); + assertEquals(integerFloatMap.size(), dataVectors.size()); + assertEquals(expectedScores, new ArrayList<>(integerFloatMap.values())); + } + } +} diff --git a/src/test/java/org/opensearch/knn/index/query/KNNWeightTests.java b/src/test/java/org/opensearch/knn/index/query/KNNWeightTests.java index 2a2c3ed4d..482e7e0cb 100644 --- a/src/test/java/org/opensearch/knn/index/query/KNNWeightTests.java +++ b/src/test/java/org/opensearch/knn/index/query/KNNWeightTests.java @@ -90,6 +90,7 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.mockStatic; import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import static org.opensearch.knn.KNNRestTestCase.INDEX_NAME; import static org.opensearch.knn.common.KNNConstants.INDEX_DESCRIPTION_PARAMETER; @@ -328,8 +329,9 @@ public void testQueryScoreForFaissWithNonExistingModel() throws IOException { } @SneakyThrows - public void testShardWithoutFiles() { + public void testScorer_whenNoVectorFieldsInDocument_thenEmptyScorerIsReturned() { final KNNQuery query = new KNNQuery(FIELD_NAME, QUERY_VECTOR, K, INDEX_NAME, (BitSetProducer) null); + KNNWeight.initialize(null); final KNNWeight knnWeight = new KNNWeight(query, 0.0f); final LeafReaderContext leafReaderContext = mock(LeafReaderContext.class); @@ -360,10 +362,9 @@ public void testShardWithoutFiles() { final Path path = mock(Path.class); when(directory.getDirectory()).thenReturn(path); final FieldInfos fieldInfos = mock(FieldInfos.class); - final FieldInfo fieldInfo = mock(FieldInfo.class); when(reader.getFieldInfos()).thenReturn(fieldInfos); - when(fieldInfos.fieldInfo(any())).thenReturn(fieldInfo); - + // When no knn fields are available , field info for vector field will be null + when(fieldInfos.fieldInfo(FIELD_NAME)).thenReturn(null); final Scorer knnScorer = knnWeight.scorer(leafReaderContext); assertEquals(KNNScorer.emptyScorer(knnWeight), knnScorer); } @@ -868,6 +869,83 @@ public void validateANNWithFilterQuery_whenExactSearch_thenSuccess(final boolean } } + @SneakyThrows + public void testRadialSearch_whenNoEngineFiles_thenPerformExactSearch() { + ExactSearcher mockedExactSearcher = mock(ExactSearcher.class); + final float[] queryVector = new float[] { 0.1f, 2.0f, 3.0f }; + final SpaceType spaceType = randomFrom(SpaceType.L2, SpaceType.INNER_PRODUCT); + KNNWeight.initialize(null, mockedExactSearcher); + final KNNQuery query = KNNQuery.builder() + .field(FIELD_NAME) + .queryVector(queryVector) + .indexName(INDEX_NAME) + .methodParameters(HNSW_METHOD_PARAMETERS) + .build(); + final KNNWeight knnWeight = new KNNWeight(query, 1.0f); + + final LeafReaderContext leafReaderContext = mock(LeafReaderContext.class); + final SegmentReader reader = mock(SegmentReader.class); + when(leafReaderContext.reader()).thenReturn(reader); + + final FSDirectory directory = mock(FSDirectory.class); + when(reader.directory()).thenReturn(directory); + final SegmentInfo segmentInfo = new SegmentInfo( + directory, + Version.LATEST, + Version.LATEST, + SEGMENT_NAME, + 100, + false, + false, + KNNCodecVersion.current().getDefaultCodecDelegate(), + Map.of(), + new byte[StringHelper.ID_LENGTH], + Map.of(), + Sort.RELEVANCE + ); + segmentInfo.setFiles(Set.of()); + final SegmentCommitInfo segmentCommitInfo = new SegmentCommitInfo(segmentInfo, 0, 0, 0, 0, 0, new byte[StringHelper.ID_LENGTH]); + when(reader.getSegmentInfo()).thenReturn(segmentCommitInfo); + + final Path path = mock(Path.class); + when(directory.getDirectory()).thenReturn(path); + final FieldInfos fieldInfos = mock(FieldInfos.class); + final FieldInfo fieldInfo = mock(FieldInfo.class); + when(reader.getFieldInfos()).thenReturn(fieldInfos); + when(fieldInfos.fieldInfo(FIELD_NAME)).thenReturn(fieldInfo); + when(fieldInfo.attributes()).thenReturn( + Map.of( + SPACE_TYPE, + spaceType.getValue(), + KNN_ENGINE, + KNNEngine.FAISS.getName(), + PARAMETERS, + String.format(Locale.ROOT, "{\"%s\":\"%s\"}", INDEX_DESCRIPTION_PARAMETER, "HNSW32") + ) + ); + final ExactSearcher.ExactSearcherContext exactSearchContext = ExactSearcher.ExactSearcherContext.builder() + .isParentHits(true) + // setting to true, so that if quantization details are present we want to do search on the quantized + // vectors as this flow is used in first pass of search. + .useQuantizedVectorsForSearch(true) + .knnQuery(query) + .build(); + when(mockedExactSearcher.searchLeaf(leafReaderContext, exactSearchContext)).thenReturn(DOC_ID_TO_SCORES); + final KNNScorer knnScorer = (KNNScorer) knnWeight.scorer(leafReaderContext); + assertNotNull(knnScorer); + final DocIdSetIterator docIdSetIterator = knnScorer.iterator(); + final List actualDocIds = new ArrayList<>(); + for (int docId = docIdSetIterator.nextDoc(); docId != NO_MORE_DOCS; docId = docIdSetIterator.nextDoc()) { + actualDocIds.add(docId); + assertEquals(DOC_ID_TO_SCORES.get(docId), knnScorer.score(), 0.00000001f); + } + assertEquals(docIdSetIterator.cost(), actualDocIds.size()); + assertTrue(Comparators.isInOrder(actualDocIds, Comparator.naturalOrder())); + // verify JNI Service is not called + jniServiceMockedStatic.verifyNoInteractions(); + verify(mockedExactSearcher).searchLeaf(leafReaderContext, exactSearchContext); + } + @SneakyThrows public void testANNWithFilterQuery_whenExactSearchAndThresholdComputations_thenSuccess() { ModelDao modelDao = mock(ModelDao.class); diff --git a/src/test/java/org/opensearch/knn/integ/BinaryIndexIT.java b/src/test/java/org/opensearch/knn/integ/BinaryIndexIT.java index 86cee6a28..e98a1d769 100644 --- a/src/test/java/org/opensearch/knn/integ/BinaryIndexIT.java +++ b/src/test/java/org/opensearch/knn/integ/BinaryIndexIT.java @@ -5,6 +5,7 @@ package org.opensearch.knn.integ; +import com.google.common.collect.ImmutableList; import com.google.common.primitives.Floats; import lombok.SneakyThrows; import lombok.extern.log4j.Log4j2; @@ -13,11 +14,13 @@ import org.junit.After; import org.junit.BeforeClass; import org.opensearch.client.Response; +import org.opensearch.common.settings.Settings; import org.opensearch.knn.KNNJsonIndexMappingsBuilder; import org.opensearch.knn.KNNJsonQueryBuilder; import org.opensearch.knn.KNNRestTestCase; import org.opensearch.knn.KNNResult; import org.opensearch.knn.TestUtils; +import org.opensearch.knn.index.KNNSettings; import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.engine.KNNEngine; @@ -36,6 +39,8 @@ @Log4j2 public class BinaryIndexIT extends KNNRestTestCase { private static TestUtils.TestData testData; + private static final int NEVER_BUILD_GRAPH = -1; + private static final int ALWAYS_BUILD_GRAPH = 0; @BeforeClass public static void setUpClass() throws IOException { @@ -104,6 +109,52 @@ public void testFaissHnswBinary_when1000Data_thenRecallIsAboveNinePointZero() { } } + @SneakyThrows + public void testFaissHnswBinary_whenBuildVectorGraphThresholdIsNegativeEndToEnd_thenBuildGraphBasedOnSetting() { + // Create Index + createKnnHnswBinaryIndex(KNNEngine.FAISS, INDEX_NAME, FIELD_NAME, 128, NEVER_BUILD_GRAPH); + ingestTestData(INDEX_NAME, FIELD_NAME); + + assertEquals(1, runKnnQuery(INDEX_NAME, FIELD_NAME, testData.queries[0], 1).size()); + + // update build vector data structure setting + updateIndexSettings(INDEX_NAME, Settings.builder().put(KNNSettings.INDEX_KNN_ADVANCED_APPROXIMATE_THRESHOLD, ALWAYS_BUILD_GRAPH)); + forceMergeKnnIndex(INDEX_NAME, 1); + + int k = 100; + for (int i = 0; i < testData.queries.length; i++) { + List knnResults = runKnnQuery(INDEX_NAME, FIELD_NAME, testData.queries[i], k); + float recall = getRecall( + Set.of(Arrays.copyOf(testData.groundTruthValues[i], k)), + knnResults.stream().map(KNNResult::getDocId).collect(Collectors.toSet()) + ); + assertTrue("Recall: " + recall, recall > 0.1); + } + } + + @SneakyThrows + public void testFaissHnswBinary_whenBuildVectorGraphThresholdIsProvidedEndToEnd_thenBuildGraphBasedOnSetting() { + // Create Index + createKnnHnswBinaryIndex(KNNEngine.FAISS, INDEX_NAME, FIELD_NAME, 128, testData.indexData.docs.length); + ingestTestData(INDEX_NAME, FIELD_NAME, false); + + assertEquals(1, runKnnQuery(INDEX_NAME, FIELD_NAME, testData.queries[0], 1).size()); + + // update build vector data structure setting + updateIndexSettings(INDEX_NAME, Settings.builder().put(KNNSettings.INDEX_KNN_ADVANCED_APPROXIMATE_THRESHOLD, ALWAYS_BUILD_GRAPH)); + forceMergeKnnIndex(INDEX_NAME, 1); + + int k = 100; + for (int i = 0; i < testData.queries.length; i++) { + List knnResults = runKnnQuery(INDEX_NAME, FIELD_NAME, testData.queries[i], k); + float recall = getRecall( + Set.of(Arrays.copyOf(testData.groundTruthValues[i], k)), + knnResults.stream().map(KNNResult::getDocId).collect(Collectors.toSet()) + ); + assertTrue("Recall: " + recall, recall > 0.1); + } + } + @SneakyThrows public void testFaissHnswBinary_whenRadialSearch_thenThrowException() { // Create Index @@ -157,13 +208,18 @@ private List runKnnQuery(final String indexName, final String fieldNa } private void ingestTestData(final String indexName, final String fieldName) throws Exception { + ingestTestData(indexName, fieldName, true); + } + + private void ingestTestData(final String indexName, final String fieldName, boolean refresh) throws Exception { // Index the test data for (int i = 0; i < testData.indexData.docs.length; i++) { addKnnDoc( indexName, Integer.toString(testData.indexData.docs[i]), - fieldName, - Floats.asList(testData.indexData.vectors[i]).toArray() + ImmutableList.of(fieldName), + ImmutableList.of(Floats.asList(testData.indexData.vectors[i]).toArray()), + refresh ); } @@ -172,8 +228,13 @@ private void ingestTestData(final String indexName, final String fieldName) thro assertEquals(testData.indexData.docs.length, getDocCount(indexName)); } - private void createKnnHnswBinaryIndex(final KNNEngine knnEngine, final String indexName, final String fieldName, final int dimension) - throws IOException { + private void createKnnHnswBinaryIndex( + final KNNEngine knnEngine, + final String indexName, + final String fieldName, + final int dimension, + final int threshold + ) throws IOException { KNNJsonIndexMappingsBuilder.Method method = KNNJsonIndexMappingsBuilder.Method.builder() .methodName(METHOD_HNSW) .engine(knnEngine.getName()) @@ -186,7 +247,11 @@ private void createKnnHnswBinaryIndex(final KNNEngine knnEngine, final String in .method(method) .build() .getIndexMapping(); + createKnnIndex(indexName, buildKNNIndexSettings(threshold), knnIndexMapping); + } - createKnnIndex(indexName, knnIndexMapping); + private void createKnnHnswBinaryIndex(final KNNEngine knnEngine, final String indexName, final String fieldName, final int dimension) + throws IOException { + createKnnHnswBinaryIndex(knnEngine, indexName, fieldName, dimension, ALWAYS_BUILD_GRAPH); } } diff --git a/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java b/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java index f434ee928..cce70838f 100644 --- a/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java +++ b/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java @@ -100,6 +100,8 @@ import static org.opensearch.knn.TestUtils.computeGroundTruthValues; import static org.opensearch.knn.common.KNNConstants.VECTOR_DATA_TYPE_FIELD; +import static org.opensearch.knn.index.KNNSettings.INDEX_KNN_ADVANCED_APPROXIMATE_THRESHOLD; +import static org.opensearch.knn.index.KNNSettings.KNN_INDEX; import static org.opensearch.knn.index.SpaceType.L2; import static org.opensearch.knn.index.memory.NativeMemoryCacheManager.GRAPH_COUNT; import static org.opensearch.knn.index.engine.KNNEngine.FAISS; @@ -638,7 +640,12 @@ protected void addKnnDocWithNestedField(String index, String docId, String neste * Add a single KNN Doc to an index with multiple fields */ protected void addKnnDoc(String index, String docId, List fieldNames, List vectors) throws IOException { - Request request = new Request("POST", "/" + index + "/_doc/" + docId + "?refresh=true"); + addKnnDoc(index, docId, fieldNames, vectors, true); + } + + protected void addKnnDoc(String index, String docId, List fieldNames, List vectors, boolean refresh) + throws IOException { + Request request = new Request("POST", "/" + index + "/_doc/" + docId + "?refresh=" + refresh); XContentBuilder builder = XContentFactory.jsonBuilder().startObject(); for (int i = 0; i < fieldNames.size(); i++) { @@ -768,6 +775,15 @@ protected Settings getKNNSegmentReplicatedIndexSettings() { .build(); } + protected Settings buildKNNIndexSettings(int approximateThreshold) { + return Settings.builder() + .put("number_of_shards", 1) + .put("number_of_replicas", 0) + .put(KNN_INDEX, true) + .put(INDEX_KNN_ADVANCED_APPROXIMATE_THRESHOLD, approximateThreshold) + .build(); + } + @SneakyThrows protected int getDataNodeCount() { Request request = new Request("GET", "_nodes/stats?filter_path=nodes.*.roles"); From cbc7dac9feb52b488771837a9f9a1f885d90f08d Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Fri, 11 Oct 2024 17:53:13 -0500 Subject: [PATCH 399/416] Bump byte-buddy to 1.15.4 (#2204) (#2205) Signed-off-by: Naveen Tatikonda (cherry picked from commit 19162c24525a0b761e0e38ea8572b88cf812795a) Co-authored-by: Naveen Tatikonda --- build.gradle | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build.gradle b/build.gradle index fe9b7a076..31ada6681 100644 --- a/build.gradle +++ b/build.gradle @@ -295,9 +295,9 @@ dependencies { api group: 'com.google.guava', name: 'guava', version:'32.1.3-jre' api group: 'commons-lang', name: 'commons-lang', version: '2.6' testFixturesImplementation "org.opensearch.test:framework:${opensearch_version}" - testImplementation group: 'net.bytebuddy', name: 'byte-buddy', version: '1.14.9' + testImplementation group: 'net.bytebuddy', name: 'byte-buddy', version: '1.15.4' testImplementation group: 'org.objenesis', name: 'objenesis', version: '3.2' - testImplementation group: 'net.bytebuddy', name: 'byte-buddy-agent', version: '1.14.9' + testImplementation group: 'net.bytebuddy', name: 'byte-buddy-agent', version: '1.15.4' testFixturesImplementation "org.opensearch:common-utils:${version}" implementation 'com.github.oshi:oshi-core:6.4.13' api "net.java.dev.jna:jna:5.13.0" From 75f9a186abf6d6ab6fda4980299fb970d575193e Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Tue, 15 Oct 2024 12:51:28 -0500 Subject: [PATCH 400/416] Bump Faiss commit from 33c0ba5 to 4eecd91 (#2194) (#2203) * Bump Faiss commit from 33c0ba5 to 4eecd91 Signed-off-by: Naveen Tatikonda * Update Faiss patches after commit bump Signed-off-by: Naveen Tatikonda --------- Signed-off-by: Naveen Tatikonda (cherry picked from commit d9c7ba5e1b25856cd52e33aa54b37f1c3eb6a882) Co-authored-by: Naveen Tatikonda --- jni/external/faiss | 2 +- ...Custom-patch-to-support-multi-vector.patch | 46 +++++++++---------- ...ble-precomp-table-to-be-shared-ivfpq.patch | 22 ++++----- 3 files changed, 34 insertions(+), 36 deletions(-) diff --git a/jni/external/faiss b/jni/external/faiss index 33c0ba5d0..4eecd9165 160000 --- a/jni/external/faiss +++ b/jni/external/faiss @@ -1 +1 @@ -Subproject commit 33c0ba5d002a7cd9761513f06ecc9822079d4a2f +Subproject commit 4eecd9165ae56bc8ff4afbf14cc8b359fdfe4004 diff --git a/jni/patches/faiss/0001-Custom-patch-to-support-multi-vector.patch b/jni/patches/faiss/0001-Custom-patch-to-support-multi-vector.patch index 7afdabc1f..26b143346 100644 --- a/jni/patches/faiss/0001-Custom-patch-to-support-multi-vector.patch +++ b/jni/patches/faiss/0001-Custom-patch-to-support-multi-vector.patch @@ -1,4 +1,4 @@ -From 9e5affabe2caacf38f5585a0b906620dd35deef5 Mon Sep 17 00:00:00 2001 +From e775a8e65da96232822d5aed77f538592fccffda Mon Sep 17 00:00:00 2001 From: Heemin Kim Date: Tue, 30 Jan 2024 14:43:56 -0800 Subject: [PATCH] Add IDGrouper for HNSW @@ -6,7 +6,7 @@ Subject: [PATCH] Add IDGrouper for HNSW Signed-off-by: Heemin Kim --- faiss/CMakeLists.txt | 3 + - faiss/Index.h | 8 +- + faiss/Index.h | 6 +- faiss/IndexHNSW.cpp | 13 +- faiss/IndexIDMap.cpp | 29 +++++ faiss/IndexIDMap.h | 22 ++++ @@ -18,7 +18,7 @@ Signed-off-by: Heemin Kim tests/CMakeLists.txt | 2 + tests/test_group_heap.cpp | 98 +++++++++++++++ tests/test_id_grouper.cpp | 241 +++++++++++++++++++++++++++++++++++++ - 13 files changed, 890 insertions(+), 5 deletions(-) + 13 files changed, 889 insertions(+), 4 deletions(-) create mode 100644 faiss/impl/IDGrouper.cpp create mode 100644 faiss/impl/IDGrouper.h create mode 100644 faiss/utils/GroupHeap.h @@ -26,10 +26,10 @@ Signed-off-by: Heemin Kim create mode 100644 tests/test_id_grouper.cpp diff --git a/faiss/CMakeLists.txt b/faiss/CMakeLists.txt -index 1b0860f3..f3d72df3 100644 +index 2871d974..d0bcec6a 100644 --- a/faiss/CMakeLists.txt +++ b/faiss/CMakeLists.txt -@@ -54,6 +54,7 @@ set(FAISS_SRC +@@ -55,6 +55,7 @@ set(FAISS_SRC impl/AuxIndexStructures.cpp impl/CodePacker.cpp impl/IDSelector.cpp @@ -37,7 +37,7 @@ index 1b0860f3..f3d72df3 100644 impl/FaissException.cpp impl/HNSW.cpp impl/NSG.cpp -@@ -149,6 +150,7 @@ set(FAISS_HEADERS +@@ -151,6 +152,7 @@ set(FAISS_HEADERS impl/AuxIndexStructures.h impl/CodePacker.h impl/IDSelector.h @@ -45,7 +45,7 @@ index 1b0860f3..f3d72df3 100644 impl/DistanceComputer.h impl/FaissAssert.h impl/FaissException.h -@@ -184,6 +186,7 @@ set(FAISS_HEADERS +@@ -186,6 +188,7 @@ set(FAISS_HEADERS invlists/InvertedListsIOHook.h utils/AlignedTable.h utils/bf16.h @@ -54,23 +54,21 @@ index 1b0860f3..f3d72df3 100644 utils/WorkerThread.h utils/distances.h diff --git a/faiss/Index.h b/faiss/Index.h -index 3d1bdb99..a8622858 100644 +index f57140ec..8f511e5d 100644 --- a/faiss/Index.h +++ b/faiss/Index.h -@@ -38,9 +38,10 @@ - +@@ -51,8 +51,9 @@ namespace faiss { --/// Forward declarations see impl/AuxIndexStructures.h, impl/IDSelector.h and --/// impl/DistanceComputer.h -+/// Forward declarations see impl/AuxIndexStructures.h, impl/IDSelector.h + /// Forward declarations see impl/AuxIndexStructures.h, impl/IDSelector.h +-/// and impl/DistanceComputer.h +/// ,impl/IDGrouper.h and impl/DistanceComputer.h struct IDSelector; +struct IDGrouper; struct RangeSearchResult; struct DistanceComputer; -@@ -52,6 +53,9 @@ struct DistanceComputer; +@@ -64,6 +65,9 @@ struct DistanceComputer; struct SearchParameters { /// if non-null, only these IDs will be considered during search. IDSelector* sel = nullptr; @@ -81,10 +79,10 @@ index 3d1bdb99..a8622858 100644 virtual ~SearchParameters() {} }; diff --git a/faiss/IndexHNSW.cpp b/faiss/IndexHNSW.cpp -index 8e5c654f..d473b6ad 100644 +index 6a1186ca..9c8a8255 100644 --- a/faiss/IndexHNSW.cpp +++ b/faiss/IndexHNSW.cpp -@@ -320,10 +320,17 @@ void IndexHNSW::search( +@@ -301,10 +301,17 @@ void IndexHNSW::search( const SearchParameters* params_in) const { FAISS_THROW_IF_NOT(k > 0); @@ -198,10 +196,10 @@ index 2d164123..a68887bd 100644 + } // namespace faiss diff --git a/faiss/impl/HNSW.cpp b/faiss/impl/HNSW.cpp -index 3ba5f72f..c574ce39 100644 +index c3693fd9..7ae28062 100644 --- a/faiss/impl/HNSW.cpp +++ b/faiss/impl/HNSW.cpp -@@ -831,6 +831,12 @@ int extract_k_from_ResultHandler(ResultHandler& res) { +@@ -906,6 +906,12 @@ int extract_k_from_ResultHandler(ResultHandler& res) { if (auto hres = dynamic_cast(&res)) { return hres->k; } @@ -329,19 +327,19 @@ index 00000000..d56113d9 + +} // namespace faiss diff --git a/faiss/impl/ResultHandler.h b/faiss/impl/ResultHandler.h -index 713fe8e4..d307fd70 100644 +index 3116eb24..126ed015 100644 --- a/faiss/impl/ResultHandler.h +++ b/faiss/impl/ResultHandler.h -@@ -13,6 +13,8 @@ - +@@ -14,6 +14,8 @@ #include #include + #include +#include +#include #include #include - #include -@@ -267,6 +269,193 @@ struct HeapBlockResultHandler : BlockResultHandler { + +@@ -286,6 +288,193 @@ struct HeapBlockResultHandler : BlockResultHandler { } }; @@ -725,7 +723,7 @@ index 00000000..3b7078da +} // namespace faiss \ No newline at end of file diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt -index 3980d7dd..c888a5a6 100644 +index c41edf0c..87ab2020 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -27,6 +27,8 @@ set(FAISS_TEST_SRC diff --git a/jni/patches/faiss/0002-Enable-precomp-table-to-be-shared-ivfpq.patch b/jni/patches/faiss/0002-Enable-precomp-table-to-be-shared-ivfpq.patch index 619832f62..88e6a8106 100644 --- a/jni/patches/faiss/0002-Enable-precomp-table-to-be-shared-ivfpq.patch +++ b/jni/patches/faiss/0002-Enable-precomp-table-to-be-shared-ivfpq.patch @@ -1,4 +1,4 @@ -From a33e6ef35385009f24200586294f96235cb95d61 Mon Sep 17 00:00:00 2001 +From 9b33874562c9e62abf4a863657c54f0d349b0f67 Mon Sep 17 00:00:00 2001 From: John Mazanec Date: Wed, 21 Feb 2024 15:34:15 -0800 Subject: [PATCH] Enable precomp table to be shared ivfpq @@ -22,7 +22,7 @@ Signed-off-by: John Mazanec create mode 100644 tests/test_ivfpq_share_table.cpp diff --git a/faiss/IndexIVFPQ.cpp b/faiss/IndexIVFPQ.cpp -index 0b7f4d05..07bc7e83 100644 +index 100f499c..09508890 100644 --- a/faiss/IndexIVFPQ.cpp +++ b/faiss/IndexIVFPQ.cpp @@ -59,6 +59,29 @@ IndexIVFPQ::IndexIVFPQ( @@ -55,7 +55,7 @@ index 0b7f4d05..07bc7e83 100644 } /**************************************************************** -@@ -466,11 +489,23 @@ void IndexIVFPQ::precompute_table() { +@@ -464,11 +487,23 @@ void IndexIVFPQ::precompute_table() { use_precomputed_table, quantizer, pq, @@ -80,7 +80,7 @@ index 0b7f4d05..07bc7e83 100644 namespace { #define TIC t0 = get_cycles() -@@ -650,7 +685,7 @@ struct QueryTables { +@@ -648,7 +683,7 @@ struct QueryTables { fvec_madd( pq.M * pq.ksub, @@ -89,7 +89,7 @@ index 0b7f4d05..07bc7e83 100644 -2.0, sim_table_2, sim_table); -@@ -679,7 +714,7 @@ struct QueryTables { +@@ -677,7 +712,7 @@ struct QueryTables { k >>= cpq.nbits; // get corresponding table @@ -98,7 +98,7 @@ index 0b7f4d05..07bc7e83 100644 (ki * pq.M + cm * Mf) * pq.ksub; if (polysemous_ht == 0) { -@@ -709,7 +744,7 @@ struct QueryTables { +@@ -707,7 +742,7 @@ struct QueryTables { dis0 = coarse_dis; const float* s = @@ -107,7 +107,7 @@ index 0b7f4d05..07bc7e83 100644 for (int m = 0; m < pq.M; m++) { sim_table_ptrs[m] = s; s += pq.ksub; -@@ -729,7 +764,7 @@ struct QueryTables { +@@ -727,7 +762,7 @@ struct QueryTables { int ki = k & ((uint64_t(1) << cpq.nbits) - 1); k >>= cpq.nbits; @@ -116,7 +116,7 @@ index 0b7f4d05..07bc7e83 100644 (ki * pq.M + cm * Mf) * pq.ksub; for (int m = m0; m < m0 + Mf; m++) { -@@ -1346,6 +1381,8 @@ IndexIVFPQ::IndexIVFPQ() { +@@ -1344,6 +1379,8 @@ IndexIVFPQ::IndexIVFPQ() { do_polysemous_training = false; polysemous_ht = 0; polysemous_training = nullptr; @@ -302,13 +302,13 @@ index 00dd2f11..91f35a6e 100644 /// same as the regular IVFPQ encoder. The codes are not reorganized by /// blocks a that point diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt -index c888a5a6..83ecedfd 100644 +index 87ab2020..a859516c 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt -@@ -37,6 +37,7 @@ set(FAISS_TEST_SRC - test_disable_pq_sdc_tables.cpp +@@ -38,6 +38,7 @@ set(FAISS_TEST_SRC test_common_ivf_empty_index.cpp test_callback.cpp + test_utils.cpp + test_ivfpq_share_table.cpp ) From a142d2b543862f99579c2cfcd9e0e8c2081fc52a Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Wed, 16 Oct 2024 12:32:52 -0700 Subject: [PATCH 401/416] Add CompressionLevel Calculation for PQ (#2216) Currently, for product quantization, we set the calculated compression level to NOT_CONFIGURED. The main issue with this is that if a user sets up a disk-based index with PQ, no re-scoring will happen by default. This change adds the calculation so that the proper re-scoring will happen. The formula is fairly straightforward => actual compression = (d * 32) / (m * code_size). Then, we round to the neareste compression level (because we only support discrete compression levels). One small issue with this is that if PQ is configured to have compression > 64x, the value will be 64x. Functionally, the only issue will be that we may not be as aggressive on oversampling for on disk mode. Signed-off-by: John Mazanec (cherry picked from commit 228aead4b016f355be6f91934777bda8ccd895a0) Co-authored-by: John Mazanec --- CHANGELOG.md | 1 + .../engine/faiss/AbstractFaissPQEncoder.java | 92 +++++++++++++++++++ .../engine/faiss/FaissHNSWPQEncoder.java | 15 +-- .../index/engine/faiss/FaissIVFPQEncoder.java | 19 +--- .../knn/index/mapper/CompressionLevel.java | 14 +-- .../index/mapper/KNNVectorFieldMapper.java | 14 ++- .../org/opensearch/knn/index/FaissIT.java | 3 +- .../faiss/AbstractFaissPQEncoderTests.java | 79 ++++++++++++++++ .../engine/faiss/FaissHNSWPQEncoderTests.java | 16 ---- .../engine/faiss/FaissIVFPQEncoderTests.java | 16 ---- .../knn/integ/ModeAndCompressionIT.java | 3 +- 11 files changed, 197 insertions(+), 75 deletions(-) create mode 100644 src/main/java/org/opensearch/knn/index/engine/faiss/AbstractFaissPQEncoder.java create mode 100644 src/test/java/org/opensearch/knn/index/engine/faiss/AbstractFaissPQEncoderTests.java delete mode 100644 src/test/java/org/opensearch/knn/index/engine/faiss/FaissHNSWPQEncoderTests.java delete mode 100644 src/test/java/org/opensearch/knn/index/engine/faiss/FaissIVFPQEncoderTests.java diff --git a/CHANGELOG.md b/CHANGELOG.md index c3a7e3722..11b740055 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), * KNNIterators should support with and without filters [#2155](https://github.com/opensearch-project/k-NN/pull/2155) * Adding Support to Enable/Disble Share level Rescoring and Update Oversampling Factor[#2172](https://github.com/opensearch-project/k-NN/pull/2172) * Add support to build vector data structures greedily and perform exact search when there are no engine files [#1942](https://github.com/opensearch-project/k-NN/issues/1942) +* Add CompressionLevel Calculation for PQ [#2200](https://github.com/opensearch-project/k-NN/pull/2200) ### Bug Fixes * Add DocValuesProducers for releasing memory when close index [#1946](https://github.com/opensearch-project/k-NN/pull/1946) * KNN80DocValues should only be considered for BinaryDocValues fields [#2147](https://github.com/opensearch-project/k-NN/pull/2147) diff --git a/src/main/java/org/opensearch/knn/index/engine/faiss/AbstractFaissPQEncoder.java b/src/main/java/org/opensearch/knn/index/engine/faiss/AbstractFaissPQEncoder.java new file mode 100644 index 000000000..a894d8ed6 --- /dev/null +++ b/src/main/java/org/opensearch/knn/index/engine/faiss/AbstractFaissPQEncoder.java @@ -0,0 +1,92 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.engine.faiss; + +import org.opensearch.common.ValidationException; +import org.opensearch.knn.index.engine.Encoder; +import org.opensearch.knn.index.engine.KNNMethodConfigContext; +import org.opensearch.knn.index.engine.MethodComponentContext; +import org.opensearch.knn.index.mapper.CompressionLevel; + +import static org.opensearch.knn.common.KNNConstants.ENCODER_PARAMETER_PQ_CODE_SIZE; +import static org.opensearch.knn.common.KNNConstants.ENCODER_PARAMETER_PQ_M; + +/** + * Abstract class for Faiss PQ encoders. This class provides the common logic for product quantization based encoders + */ +public abstract class AbstractFaissPQEncoder implements Encoder { + + @Override + public CompressionLevel calculateCompressionLevel( + MethodComponentContext methodComponentContext, + KNNMethodConfigContext knnMethodConfigContext + ) { + // Roughly speaking, PQ can be configured to produce a lot of different compression levels. The "m" parameter + // specifies how many sub-vectors to break the vector up into, and then the "code_size" represents the number + // of bits to encode each subvector. Thus, a d-dimensional vector of float32s goes from + // d*32 -> (m)*code_size bits. So if we want (d*32)/(m*code_size) will be the compression level. + // + // Example: + // d=768, m=384, code_size=8 + // (768*32)/(384*8) = 8x (i.e. 24,576 vs. 3,072). + // + // Because of this variability, we will need to properly round to one of the supported values. + if (methodComponentContext.getParameters().containsKey(ENCODER_PARAMETER_PQ_M) == false + || methodComponentContext.getParameters().containsKey(ENCODER_PARAMETER_PQ_CODE_SIZE) == false) { + return CompressionLevel.NOT_CONFIGURED; + } + + // Map the number of bits passed in, back to the compression level + Object value = methodComponentContext.getParameters().get(ENCODER_PARAMETER_PQ_M); + ValidationException validationException = getMethodComponent().getParameters() + .get(ENCODER_PARAMETER_PQ_M) + .validate(value, knnMethodConfigContext); + if (validationException != null) { + throw validationException; + } + Integer m = (Integer) value; + value = methodComponentContext.getParameters().get(ENCODER_PARAMETER_PQ_CODE_SIZE); + validationException = getMethodComponent().getParameters() + .get(ENCODER_PARAMETER_PQ_CODE_SIZE) + .validate(value, knnMethodConfigContext); + if (validationException != null) { + throw validationException; + } + Integer codeSize = (Integer) value; + int dimension = knnMethodConfigContext.getDimension(); + + float actualCompression = ((float) dimension * 32) / (m * codeSize); + + if (actualCompression < 2.0f) { + return CompressionLevel.x1; + } + + if (actualCompression < 4.0f) { + return CompressionLevel.x2; + } + + if (actualCompression < 8.0f) { + return CompressionLevel.x4; + } + + if (actualCompression < 16.0f) { + return CompressionLevel.x8; + } + + if (actualCompression < 32.0f) { + return CompressionLevel.x16; + } + + if (actualCompression < 64.0f) { + return CompressionLevel.x32; + } + + // TODO: The problem is that the theoretical compression level of PQ can be in the thousands. Thus, Im not sure + // it makes sense to have an enum all the way up to that value. So, for now, we will just return the max + // compression + return CompressionLevel.MAX_COMPRESSION_LEVEL; + } +} diff --git a/src/main/java/org/opensearch/knn/index/engine/faiss/FaissHNSWPQEncoder.java b/src/main/java/org/opensearch/knn/index/engine/faiss/FaissHNSWPQEncoder.java index 6750d84ed..c22a9dec7 100644 --- a/src/main/java/org/opensearch/knn/index/engine/faiss/FaissHNSWPQEncoder.java +++ b/src/main/java/org/opensearch/knn/index/engine/faiss/FaissHNSWPQEncoder.java @@ -8,12 +8,8 @@ import com.google.common.collect.ImmutableSet; import org.opensearch.knn.common.KNNConstants; import org.opensearch.knn.index.VectorDataType; -import org.opensearch.knn.index.engine.Encoder; -import org.opensearch.knn.index.engine.KNNMethodConfigContext; import org.opensearch.knn.index.engine.MethodComponent; -import org.opensearch.knn.index.engine.MethodComponentContext; import org.opensearch.knn.index.engine.Parameter; -import org.opensearch.knn.index.mapper.CompressionLevel; import java.util.Objects; import java.util.Set; @@ -30,7 +26,7 @@ * Faiss HNSW PQ encoder. Right now, the implementations are slightly different during validation between this an * {@link FaissIVFPQEncoder}. Hence, they are separate classes. */ -public class FaissHNSWPQEncoder implements Encoder { +public class FaissHNSWPQEncoder extends AbstractFaissPQEncoder { private static final Set SUPPORTED_DATA_TYPES = ImmutableSet.of(VectorDataType.FLOAT); @@ -72,13 +68,4 @@ public class FaissHNSWPQEncoder implements Encoder { public MethodComponent getMethodComponent() { return METHOD_COMPONENT; } - - @Override - public CompressionLevel calculateCompressionLevel( - MethodComponentContext methodComponentContext, - KNNMethodConfigContext knnMethodConfigContext - ) { - // TODO: For now, not supported out of the box - return CompressionLevel.NOT_CONFIGURED; - } } diff --git a/src/main/java/org/opensearch/knn/index/engine/faiss/FaissIVFPQEncoder.java b/src/main/java/org/opensearch/knn/index/engine/faiss/FaissIVFPQEncoder.java index 8d54548bd..8c10aebdf 100644 --- a/src/main/java/org/opensearch/knn/index/engine/faiss/FaissIVFPQEncoder.java +++ b/src/main/java/org/opensearch/knn/index/engine/faiss/FaissIVFPQEncoder.java @@ -5,15 +5,12 @@ package org.opensearch.knn.index.engine.faiss; +import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableSet; import org.opensearch.knn.common.KNNConstants; import org.opensearch.knn.index.VectorDataType; -import org.opensearch.knn.index.engine.Encoder; -import org.opensearch.knn.index.engine.KNNMethodConfigContext; import org.opensearch.knn.index.engine.MethodComponent; -import org.opensearch.knn.index.engine.MethodComponentContext; import org.opensearch.knn.index.engine.Parameter; -import org.opensearch.knn.index.mapper.CompressionLevel; import java.util.Set; @@ -30,11 +27,12 @@ * Faiss IVF PQ encoder. Right now, the implementations are slightly different during validation between this an * {@link FaissHNSWPQEncoder}. Hence, they are separate classes. */ -public class FaissIVFPQEncoder implements Encoder { +public class FaissIVFPQEncoder extends AbstractFaissPQEncoder { private static final Set SUPPORTED_DATA_TYPES = ImmutableSet.of(VectorDataType.FLOAT); - private final static MethodComponent METHOD_COMPONENT = MethodComponent.Builder.builder(KNNConstants.ENCODER_PQ) + @VisibleForTesting + final static MethodComponent METHOD_COMPONENT = MethodComponent.Builder.builder(KNNConstants.ENCODER_PQ) .addSupportedDataTypes(SUPPORTED_DATA_TYPES) .addParameter( ENCODER_PARAMETER_PQ_M, @@ -93,13 +91,4 @@ public class FaissIVFPQEncoder implements Encoder { public MethodComponent getMethodComponent() { return METHOD_COMPONENT; } - - @Override - public CompressionLevel calculateCompressionLevel( - MethodComponentContext methodComponentContext, - KNNMethodConfigContext knnMethodConfigContext - ) { - // TODO: For now, not supported out of the box - return CompressionLevel.NOT_CONFIGURED; - } } diff --git a/src/main/java/org/opensearch/knn/index/mapper/CompressionLevel.java b/src/main/java/org/opensearch/knn/index/mapper/CompressionLevel.java index ab583a2e0..99f74c246 100644 --- a/src/main/java/org/opensearch/knn/index/mapper/CompressionLevel.java +++ b/src/main/java/org/opensearch/knn/index/mapper/CompressionLevel.java @@ -27,18 +27,10 @@ public enum CompressionLevel { x4(4, "4x", null, Collections.emptySet()), x8(8, "8x", new RescoreContext(2.0f, false), Set.of(Mode.ON_DISK)), x16(16, "16x", new RescoreContext(3.0f, false), Set.of(Mode.ON_DISK)), - x32(32, "32x", new RescoreContext(3.0f, false), Set.of(Mode.ON_DISK)); + x32(32, "32x", new RescoreContext(3.0f, false), Set.of(Mode.ON_DISK)), + x64(64, "64x", new RescoreContext(5.0f, false), Set.of(Mode.ON_DISK)); - // Internally, an empty string is easier to deal with them null. However, from the mapping, - // we do not want users to pass in the empty string and instead want null. So we make the conversion here - public static final String[] NAMES_ARRAY = new String[] { - NOT_CONFIGURED.getName(), - x1.getName(), - x2.getName(), - x4.getName(), - x8.getName(), - x16.getName(), - x32.getName() }; + public static final CompressionLevel MAX_COMPRESSION_LEVEL = CompressionLevel.x64; /** * Default is set to 1x and is a noop diff --git a/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapper.java b/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapper.java index 6e5138a56..18d4f7b64 100644 --- a/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapper.java +++ b/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapper.java @@ -15,6 +15,7 @@ import java.util.function.Supplier; import java.util.stream.Collectors; +import com.google.common.annotations.VisibleForTesting; import lombok.Getter; import lombok.Setter; import lombok.extern.log4j.Log4j2; @@ -75,6 +76,17 @@ private static KNNVectorFieldMapper toType(FieldMapper in) { return (KNNVectorFieldMapper) in; } + // Supported compression levels for knn_vector field type + @VisibleForTesting + public static final String[] MAPPING_COMPRESSION_NAMES_ARRAY = new String[] { + CompressionLevel.NOT_CONFIGURED.getName(), + CompressionLevel.x1.getName(), + CompressionLevel.x2.getName(), + CompressionLevel.x4.getName(), + CompressionLevel.x8.getName(), + CompressionLevel.x16.getName(), + CompressionLevel.x32.getName() }; + /** * Builder for KNNVectorFieldMapper. This class defines the set of parameters that can be applied to the knn_vector * field type @@ -161,7 +173,7 @@ public static class Builder extends ParametrizedFieldMapper.Builder { KNNConstants.COMPRESSION_LEVEL_PARAMETER, false, m -> toType(m).originalMappingParameters.getCompressionLevel(), - CompressionLevel.NAMES_ARRAY + MAPPING_COMPRESSION_NAMES_ARRAY ).acceptsNull(); // A top level space Type field. diff --git a/src/test/java/org/opensearch/knn/index/FaissIT.java b/src/test/java/org/opensearch/knn/index/FaissIT.java index 4782b0ce6..317cff4d4 100644 --- a/src/test/java/org/opensearch/knn/index/FaissIT.java +++ b/src/test/java/org/opensearch/knn/index/FaissIT.java @@ -1712,7 +1712,8 @@ public void testIVF_InvalidPQM_thenFail() { () -> ingestDataAndTrainModel(modelId, trainingIndexName, trainingFieldName, dimension, modelDescription, in, trainingDataCount) ); assertTrue( - re.getMessage().contains("Validation Failed: 1: parameter validation failed for MethodComponentContext parameter [encoder].;") + re.getMessage(), + re.getMessage().contains("Validation Failed: 1: parameter validation failed for Integer parameter [m].;") ); } diff --git a/src/test/java/org/opensearch/knn/index/engine/faiss/AbstractFaissPQEncoderTests.java b/src/test/java/org/opensearch/knn/index/engine/faiss/AbstractFaissPQEncoderTests.java new file mode 100644 index 000000000..704657c11 --- /dev/null +++ b/src/test/java/org/opensearch/knn/index/engine/faiss/AbstractFaissPQEncoderTests.java @@ -0,0 +1,79 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.engine.faiss; + +import lombok.SneakyThrows; +import org.opensearch.knn.KNNTestCase; +import org.opensearch.knn.index.engine.Encoder; +import org.opensearch.knn.index.engine.KNNMethodConfigContext; +import org.opensearch.knn.index.engine.MethodComponent; +import org.opensearch.knn.index.engine.MethodComponentContext; +import org.opensearch.knn.index.mapper.CompressionLevel; + +import java.util.Map; + +import static org.opensearch.knn.common.KNNConstants.ENCODER_PARAMETER_PQ_CODE_SIZE; +import static org.opensearch.knn.common.KNNConstants.ENCODER_PARAMETER_PQ_M; +import static org.opensearch.knn.common.KNNConstants.ENCODER_PQ; + +public class AbstractFaissPQEncoderTests extends KNNTestCase { + + @SneakyThrows + public void testCalculateCompressionLevel() { + AbstractFaissPQEncoder encoder = new AbstractFaissPQEncoder() { + @Override + public MethodComponent getMethodComponent() { + return FaissIVFPQEncoder.METHOD_COMPONENT; + } + }; + + // Compression formula is: + // actual_compression = (d*32)/(m*code_size) and then round down to nearest: 1x, 2x, 4x, 8x, 16x, 32x + + // d=768 + // m=2 + // code_size=8 + // actual_compression = (768*32)/(2*8) = 1,536x + // expected_compression = Max compression level + assertCompressionLevel(2, 8, 768, CompressionLevel.MAX_COMPRESSION_LEVEL, encoder); + + // d=32 + // m=4 + // code_size=16 + // actual_compression = (32*32)/(4*16) = 16x + // expected_compression = Max compression level + assertCompressionLevel(4, 16, 32, CompressionLevel.x16, encoder); + + // d=1536 + // m=768 + // code_size=8 + // actual_compression = (1536*32)/(768*8) = 8x + // expected_compression = Max compression level + assertCompressionLevel(768, 8, 1536, CompressionLevel.x8, encoder); + + // d=128 + // m=128 + // code_size=8 + // actual_compression = (128*32)/(128*8) = 4x + // expected_compression = Max compression level + assertCompressionLevel(128, 8, 128, CompressionLevel.x4, encoder); + } + + private void assertCompressionLevel(int m, int codeSize, int d, CompressionLevel expectedCompression, Encoder encoder) { + assertEquals( + expectedCompression, + encoder.calculateCompressionLevel(generateMethodComponentContext(m, codeSize), generateKNNMethodConfigContext(d)) + ); + } + + private MethodComponentContext generateMethodComponentContext(int m, int codeSize) { + return new MethodComponentContext(ENCODER_PQ, Map.of(ENCODER_PARAMETER_PQ_M, m, ENCODER_PARAMETER_PQ_CODE_SIZE, codeSize)); + } + + private KNNMethodConfigContext generateKNNMethodConfigContext(int dimension) { + return KNNMethodConfigContext.builder().dimension(dimension).build(); + } +} diff --git a/src/test/java/org/opensearch/knn/index/engine/faiss/FaissHNSWPQEncoderTests.java b/src/test/java/org/opensearch/knn/index/engine/faiss/FaissHNSWPQEncoderTests.java deleted file mode 100644 index 3f7dd9dcd..000000000 --- a/src/test/java/org/opensearch/knn/index/engine/faiss/FaissHNSWPQEncoderTests.java +++ /dev/null @@ -1,16 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.knn.index.engine.faiss; - -import org.opensearch.knn.KNNTestCase; -import org.opensearch.knn.index.mapper.CompressionLevel; - -public class FaissHNSWPQEncoderTests extends KNNTestCase { - public void testCalculateCompressionLevel() { - FaissHNSWPQEncoder encoder = new FaissHNSWPQEncoder(); - assertEquals(CompressionLevel.NOT_CONFIGURED, encoder.calculateCompressionLevel(null, null)); - } -} diff --git a/src/test/java/org/opensearch/knn/index/engine/faiss/FaissIVFPQEncoderTests.java b/src/test/java/org/opensearch/knn/index/engine/faiss/FaissIVFPQEncoderTests.java deleted file mode 100644 index 35b7a64ab..000000000 --- a/src/test/java/org/opensearch/knn/index/engine/faiss/FaissIVFPQEncoderTests.java +++ /dev/null @@ -1,16 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.knn.index.engine.faiss; - -import org.opensearch.knn.KNNTestCase; -import org.opensearch.knn.index.mapper.CompressionLevel; - -public class FaissIVFPQEncoderTests extends KNNTestCase { - public void testCalculateCompressionLevel() { - FaissIVFPQEncoder encoder = new FaissIVFPQEncoder(); - assertEquals(CompressionLevel.NOT_CONFIGURED, encoder.calculateCompressionLevel(null, null)); - } -} diff --git a/src/test/java/org/opensearch/knn/integ/ModeAndCompressionIT.java b/src/test/java/org/opensearch/knn/integ/ModeAndCompressionIT.java index cea3400af..0903b3c41 100644 --- a/src/test/java/org/opensearch/knn/integ/ModeAndCompressionIT.java +++ b/src/test/java/org/opensearch/knn/integ/ModeAndCompressionIT.java @@ -39,6 +39,7 @@ import static org.opensearch.knn.common.KNNConstants.TRAIN_FIELD_PARAMETER; import static org.opensearch.knn.common.KNNConstants.TRAIN_INDEX_PARAMETER; import static org.opensearch.knn.common.KNNConstants.VECTOR_DATA_TYPE_FIELD; +import static org.opensearch.knn.index.mapper.KNNVectorFieldMapper.MAPPING_COMPRESSION_NAMES_ARRAY; public class ModeAndCompressionIT extends KNNRestTestCase { @@ -253,7 +254,7 @@ public void testTraining_whenInvalid_thenFail() { public void testTraining_whenValid_thenSucceed() { setupTrainingIndex(); XContentBuilder builder; - for (String compressionLevel : CompressionLevel.NAMES_ARRAY) { + for (String compressionLevel : MAPPING_COMPRESSION_NAMES_ARRAY) { if (compressionLevel.equals("4x")) { continue; } From 4a224e3a056ece5624ebea992bb8ce41ef3dd395 Mon Sep 17 00:00:00 2001 From: Doo Yong Kim <0ctopus13prime@gmail.com> Date: Mon, 21 Oct 2024 12:07:18 -0700 Subject: [PATCH 402/416] [Backport 2.x] Introduce a loading layer in NMSLIB. (#2185) (#2220) * Introduce a loading layer in NMSLIB. (#2185) * Introduce a loading layer in NMSLIB. Signed-off-by: Dooyong Kim * Added NMSLIB istream implementation. Signed-off-by: Dooyong Kim * Fix integer overflow issue when passing read size for loading NMSLIB vector index. Signed-off-by: Dooyong Kim * Added unit test for NMSLIB loading layer. Signed-off-by: Dooyong Kim * Made a patch in NMSLIB to avoid frequently calling JNI for better loading index performance. Signed-off-by: Dooyong Kim * Compliance constexpr function in C++11 having nullstatement. Signed-off-by: Dooyong Kim --------- Signed-off-by: Dooyong Kim Co-authored-by: Dooyong Kim * Fixed that it's failing to resolve a package in import statement. Signed-off-by: Dooyong Kim * Move the element in the changelog from 3.x to 2.x. Signed-off-by: Dooyong Kim --------- Signed-off-by: Dooyong Kim Co-authored-by: Dooyong Kim --- CHANGELOG.md | 1 + jni/CMakeLists.txt | 3 +- jni/cmake/init-nmslib.cmake | 3 +- jni/include/faiss_stream_support.h | 81 +-- jni/include/jni_util.h | 9 +- jni/include/native_engines_stream_support.h | 125 ++++ jni/include/nmslib_stream_support.h | 51 ++ jni/include/nmslib_wrapper.h | 8 + .../org_opensearch_knn_jni_NmslibService.h | 8 + ...pis-for-vector-index-loading-in-Hnsw.patch | 221 ++++++++ ...is-using-stream-to-load-save-in-Hnsw.patch | 93 --- jni/src/jni_util.cpp | 10 +- jni/src/nmslib_wrapper.cpp | 533 ++++++++++-------- .../org_opensearch_knn_jni_NmslibService.cpp | 117 ++-- jni/tests/faiss_stream_support_test.cpp | 98 ++-- jni/tests/native_stream_support_util.h | 102 ++++ jni/tests/nmslib_stream_support_test.cpp | 120 ++++ jni/tests/test_util.h | 3 +- .../org/opensearch/knn/index/KNNSettings.java | 6 +- .../index/memory/NativeMemoryAllocation.java | 4 +- .../memory/NativeMemoryLoadStrategy.java | 27 +- .../knn/index/store/IndexInputWithBuffer.java | 10 +- .../org/opensearch/knn/jni/JNIService.java | 2 + .../org/opensearch/knn/jni/NmslibService.java | 10 + .../knn/index/KNNCircuitBreakerIT.java | 2 +- .../opensearch/knn/jni/JNIServiceTests.java | 25 + 26 files changed, 1108 insertions(+), 564 deletions(-) create mode 100644 jni/include/native_engines_stream_support.h create mode 100644 jni/include/nmslib_stream_support.h create mode 100644 jni/patches/nmslib/0003-Added-streaming-apis-for-vector-index-loading-in-Hnsw.patch delete mode 100644 jni/patches/nmslib/0003-Adding-two-apis-using-stream-to-load-save-in-Hnsw.patch create mode 100644 jni/tests/native_stream_support_util.h create mode 100644 jni/tests/nmslib_stream_support_test.cpp diff --git a/CHANGELOG.md b/CHANGELOG.md index 11b740055..141d01c7d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), * Adding Support to Enable/Disble Share level Rescoring and Update Oversampling Factor[#2172](https://github.com/opensearch-project/k-NN/pull/2172) * Add support to build vector data structures greedily and perform exact search when there are no engine files [#1942](https://github.com/opensearch-project/k-NN/issues/1942) * Add CompressionLevel Calculation for PQ [#2200](https://github.com/opensearch-project/k-NN/pull/2200) +* Introduce a loading layer in native engine [#2185](https://github.com/opensearch-project/k-NN/pull/2185) ### Bug Fixes * Add DocValuesProducers for releasing memory when close index [#1946](https://github.com/opensearch-project/k-NN/pull/1946) * KNN80DocValues should only be considered for BinaryDocValues fields [#2147](https://github.com/opensearch-project/k-NN/pull/2147) diff --git a/jni/CMakeLists.txt b/jni/CMakeLists.txt index 4caa907b3..1920453c7 100644 --- a/jni/CMakeLists.txt +++ b/jni/CMakeLists.txt @@ -156,7 +156,8 @@ if ("${WIN32}" STREQUAL "") tests/commons_test.cpp tests/faiss_stream_support_test.cpp tests/faiss_index_service_test.cpp - ) + tests/nmslib_stream_support_test.cpp + ) target_link_libraries( jni_test diff --git a/jni/cmake/init-nmslib.cmake b/jni/cmake/init-nmslib.cmake index 64df457c1..2554b2bd7 100644 --- a/jni/cmake/init-nmslib.cmake +++ b/jni/cmake/init-nmslib.cmake @@ -12,14 +12,13 @@ if (NOT EXISTS ${NMS_REPO_DIR}) execute_process(COMMAND git submodule update --init -- external/nmslib WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}) endif () - # Apply patches if(NOT DEFINED APPLY_LIB_PATCHES OR "${APPLY_LIB_PATCHES}" STREQUAL true) # Define list of patch files set(PATCH_FILE_LIST) list(APPEND PATCH_FILE_LIST "${CMAKE_CURRENT_SOURCE_DIR}/patches/nmslib/0001-Initialize-maxlevel-during-add-from-enterpoint-level.patch") list(APPEND PATCH_FILE_LIST "${CMAKE_CURRENT_SOURCE_DIR}/patches/nmslib/0002-Adds-ability-to-pass-ef-parameter-in-the-query-for-h.patch") - list(APPEND PATCH_FILE_LIST "${CMAKE_CURRENT_SOURCE_DIR}/patches/nmslib/0003-Adding-two-apis-using-stream-to-load-save-in-Hnsw.patch") + list(APPEND PATCH_FILE_LIST "${CMAKE_CURRENT_SOURCE_DIR}/patches/nmslib/0003-Added-streaming-apis-for-vector-index-loading-in-Hnsw.patch") # Get patch id of the last commit execute_process(COMMAND sh -c "git --no-pager show HEAD | git patch-id --stable" OUTPUT_VARIABLE PATCH_ID_OUTPUT_FROM_COMMIT WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/external/nmslib) diff --git a/jni/include/faiss_stream_support.h b/jni/include/faiss_stream_support.h index 65f1631d4..a12d66ae9 100644 --- a/jni/include/faiss_stream_support.h +++ b/jni/include/faiss_stream_support.h @@ -9,11 +9,12 @@ * GitHub history for details. */ -#ifndef OPENSEARCH_KNN_JNI_STREAM_SUPPORT_H -#define OPENSEARCH_KNN_JNI_STREAM_SUPPORT_H +#ifndef OPENSEARCH_KNN_JNI_FAISS_STREAM_SUPPORT_H +#define OPENSEARCH_KNN_JNI_FAISS_STREAM_SUPPORT_H #include "faiss/impl/io.h" #include "jni_util.h" +#include "native_engines_stream_support.h" #include #include @@ -23,80 +24,6 @@ namespace knn_jni { namespace stream { -/** - * This class contains Java IndexInputWithBuffer reference and calls its API to copy required bytes into a read buffer. - */ - -class NativeEngineIndexInputMediator { - public: - // Expect IndexInputWithBuffer is given as `_indexInput`. - NativeEngineIndexInputMediator(JNIUtilInterface *_jni_interface, - JNIEnv *_env, - jobject _indexInput) - : jni_interface(_jni_interface), - env(_env), - indexInput(_indexInput), - bufferArray((jbyteArray) (_jni_interface->GetObjectField(_env, - _indexInput, - getBufferFieldId(_jni_interface, _env)))), - copyBytesMethod(getCopyBytesMethod(_jni_interface, _env)) { - } - - void copyBytes(int64_t nbytes, uint8_t *destination) { - while (nbytes > 0) { - // Call `copyBytes` to read bytes as many as possible. - const auto readBytes = - jni_interface->CallIntMethodLong(env, indexInput, copyBytesMethod, nbytes); - - // === Critical Section Start === - - // Get primitive array pointer, no copy is happening in OpenJDK. - auto primitiveArray = - (jbyte *) jni_interface->GetPrimitiveArrayCritical(env, bufferArray, nullptr); - - // Copy Java bytes to C++ destination address. - std::memcpy(destination, primitiveArray, readBytes); - - // Release the acquired primitive array pointer. - // JNI_ABORT tells JVM to directly free memory without copying back to Java byte[]. - // Since we're merely copying data, we don't need to copying back. - jni_interface->ReleasePrimitiveArrayCritical(env, bufferArray, primitiveArray, JNI_ABORT); - - // === Critical Section End === - - destination += readBytes; - nbytes -= readBytes; - } // End while - } - - private: - static jclass getIndexInputWithBufferClass(JNIUtilInterface *jni_interface, JNIEnv *env) { - static jclass INDEX_INPUT_WITH_BUFFER_CLASS = - jni_interface->FindClassFromJNIEnv(env, "org/opensearch/knn/index/store/IndexInputWithBuffer"); - return INDEX_INPUT_WITH_BUFFER_CLASS; - } - - static jmethodID getCopyBytesMethod(JNIUtilInterface *jni_interface, JNIEnv *env) { - static jmethodID COPY_METHOD_ID = - jni_interface->GetMethodID(env, getIndexInputWithBufferClass(jni_interface, env), "copyBytes", "(J)I"); - return COPY_METHOD_ID; - } - - static jfieldID getBufferFieldId(JNIUtilInterface *jni_interface, JNIEnv *env) { - static jfieldID BUFFER_FIELD_ID = - jni_interface->GetFieldID(env, getIndexInputWithBufferClass(jni_interface, env), "buffer", "[B"); - return BUFFER_FIELD_ID; - } - - JNIUtilInterface *jni_interface; - JNIEnv *env; - - // `IndexInputWithBuffer` instance having `IndexInput` instance obtained from `Directory` for reading. - jobject indexInput; - jbyteArray bufferArray; - jmethodID copyBytesMethod; -}; // class NativeEngineIndexInputMediator - /** @@ -133,4 +60,4 @@ class FaissOpenSearchIOReader final : public faiss::IOReader { } } -#endif //OPENSEARCH_KNN_JNI_STREAM_SUPPORT_H +#endif //OPENSEARCH_KNN_JNI_FAISS_STREAM_SUPPORT_H diff --git a/jni/include/jni_util.h b/jni/include/jni_util.h index 6b1b926e7..9f4daef7c 100644 --- a/jni/include/jni_util.h +++ b/jni/include/jni_util.h @@ -138,7 +138,11 @@ namespace knn_jni { virtual void ReleasePrimitiveArrayCritical(JNIEnv * env, jarray array, void *carray, jint mode) = 0; - virtual jint CallIntMethodLong(JNIEnv * env, jobject obj, jmethodID methodID, int64_t longArg) = 0; + virtual jint CallNonvirtualIntMethodA(JNIEnv *env, jobject obj, jclass clazz, + jmethodID methodID, jvalue *args) = 0; + + virtual jlong CallNonvirtualLongMethodA(JNIEnv * env, jobject obj, jclass clazz, + jmethodID methodID, jvalue* args) = 0; // -------------------------------------------------------------------------- }; @@ -194,7 +198,8 @@ namespace knn_jni { jclass FindClassFromJNIEnv(JNIEnv * env, const char *name) final; jmethodID GetMethodID(JNIEnv * env, jclass clazz, const char *name, const char *sig) final; jfieldID GetFieldID(JNIEnv * env, jclass clazz, const char *name, const char *sig) final; - jint CallIntMethodLong(JNIEnv * env, jobject obj, jmethodID methodID, int64_t longArg) final; + jint CallNonvirtualIntMethodA(JNIEnv *env, jobject obj, jclass clazz, jmethodID methodID, jvalue *args) final; + jlong CallNonvirtualLongMethodA(JNIEnv * env, jobject obj, jclass clazz, jmethodID methodID, jvalue* args) final; void * GetPrimitiveArrayCritical(JNIEnv * env, jarray array, jboolean *isCopy) final; void ReleasePrimitiveArrayCritical(JNIEnv * env, jarray array, void *carray, jint mode) final; diff --git a/jni/include/native_engines_stream_support.h b/jni/include/native_engines_stream_support.h new file mode 100644 index 000000000..5d4b32d3d --- /dev/null +++ b/jni/include/native_engines_stream_support.h @@ -0,0 +1,125 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +#ifndef OPENSEARCH_KNN_JNI_STREAM_SUPPORT_H +#define OPENSEARCH_KNN_JNI_STREAM_SUPPORT_H + +#include "jni_util.h" + +#include +#include +#include +#include + +namespace knn_jni { +namespace stream { + + + +/** + * This class contains Java IndexInputWithBuffer reference and calls its API to copy required bytes into a read buffer. + */ +class NativeEngineIndexInputMediator { + public: + // Expect IndexInputWithBuffer is given as `_indexInput`. + NativeEngineIndexInputMediator(JNIUtilInterface *_jni_interface, + JNIEnv *_env, + jobject _indexInput) + : jni_interface(_jni_interface), + env(_env), + indexInput(_indexInput), + bufferArray((jbyteArray) (_jni_interface->GetObjectField(_env, + _indexInput, + getBufferFieldId(_jni_interface, _env)))), + copyBytesMethod(getCopyBytesMethod(_jni_interface, _env)), + remainingBytesMethod(getRemainingBytesMethod(_jni_interface, _env)) { + } + + void copyBytes(int64_t nbytes, uint8_t *destination) { + auto jclazz = getIndexInputWithBufferClass(jni_interface, env); + + while (nbytes > 0) { + // Call `copyBytes` to read bytes as many as possible. + jvalue args; + args.j = nbytes; + const auto readBytes = + jni_interface->CallNonvirtualIntMethodA(env, indexInput, jclazz, copyBytesMethod, &args); + + // === Critical Section Start === + + // Get primitive array pointer, no copy is happening in OpenJDK. + auto primitiveArray = + (jbyte *) jni_interface->GetPrimitiveArrayCritical(env, bufferArray, nullptr); + + // Copy Java bytes to C++ destination address. + std::memcpy(destination, primitiveArray, readBytes); + + // Release the acquired primitive array pointer. + // JNI_ABORT tells JVM to directly free memory without copying back to Java byte[]. + // Since we're merely copying data, we don't need to copying back. + jni_interface->ReleasePrimitiveArrayCritical(env, bufferArray, primitiveArray, JNI_ABORT); + + // === Critical Section End === + + destination += readBytes; + nbytes -= readBytes; + } // End while + } + + int64_t remainingBytes() { + return jni_interface->CallNonvirtualLongMethodA(env, + indexInput, + getIndexInputWithBufferClass(jni_interface, env), + remainingBytesMethod, + nullptr); + } + + private: + static jclass getIndexInputWithBufferClass(JNIUtilInterface *jni_interface, JNIEnv *env) { + static jclass INDEX_INPUT_WITH_BUFFER_CLASS = + jni_interface->FindClassFromJNIEnv(env, "org/opensearch/knn/index/store/IndexInputWithBuffer"); + return INDEX_INPUT_WITH_BUFFER_CLASS; + } + + static jmethodID getCopyBytesMethod(JNIUtilInterface *jni_interface, JNIEnv *env) { + static jmethodID COPY_METHOD_ID = + jni_interface->GetMethodID(env, getIndexInputWithBufferClass(jni_interface, env), "copyBytes", "(J)I"); + return COPY_METHOD_ID; + } + + static jmethodID getRemainingBytesMethod(JNIUtilInterface *jni_interface, JNIEnv *env) { + static jmethodID COPY_METHOD_ID = + jni_interface->GetMethodID(env, getIndexInputWithBufferClass(jni_interface, env), "remainingBytes", "()J"); + return COPY_METHOD_ID; + } + + static jfieldID getBufferFieldId(JNIUtilInterface *jni_interface, JNIEnv *env) { + static jfieldID BUFFER_FIELD_ID = + jni_interface->GetFieldID(env, getIndexInputWithBufferClass(jni_interface, env), "buffer", "[B"); + return BUFFER_FIELD_ID; + } + + JNIUtilInterface *jni_interface; + JNIEnv *env; + + // `IndexInputWithBuffer` instance having `IndexInput` instance obtained from `Directory` for reading. + jobject indexInput; + jbyteArray bufferArray; + jmethodID copyBytesMethod; + jmethodID remainingBytesMethod; +}; // class NativeEngineIndexInputMediator + + + +} +} + +#endif //OPENSEARCH_KNN_JNI_STREAM_SUPPORT_H diff --git a/jni/include/nmslib_stream_support.h b/jni/include/nmslib_stream_support.h new file mode 100644 index 000000000..38c06cb95 --- /dev/null +++ b/jni/include/nmslib_stream_support.h @@ -0,0 +1,51 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +#ifndef OPENSEARCH_KNN_JNI_NMSLIB_STREAM_SUPPORT_H +#define OPENSEARCH_KNN_JNI_NMSLIB_STREAM_SUPPORT_H + +#include "native_engines_stream_support.h" + +namespace knn_jni { +namespace stream { + + + +/** + * NmslibIOReader implementation delegating NativeEngineIndexInputMediator to read bytes. + */ +class NmslibOpenSearchIOReader final : public similarity::NmslibIOReader { + public: + explicit NmslibOpenSearchIOReader(NativeEngineIndexInputMediator *_mediator) + : mediator(_mediator) { + } + + void read(char *bytes, size_t len) final { + if (len > 0) { + // Mediator calls IndexInput, then copy read bytes to `ptr`. + mediator->copyBytes(len, (uint8_t *) bytes); + } + } + + size_t remainingBytes() final { + return mediator->remainingBytes(); + } + + private: + NativeEngineIndexInputMediator *mediator; +}; // class NmslibOpenSearchIOReader + + + +} +} + +#endif //OPENSEARCH_KNN_JNI_NMSLIB_STREAM_SUPPORT_H diff --git a/jni/include/nmslib_wrapper.h b/jni/include/nmslib_wrapper.h index 27a013c10..2853cd71f 100644 --- a/jni/include/nmslib_wrapper.h +++ b/jni/include/nmslib_wrapper.h @@ -33,6 +33,14 @@ namespace knn_jni { // Return a pointer to the loaded index jlong LoadIndex(knn_jni::JNIUtilInterface * jniUtil, JNIEnv * env, jstring indexPathJ, jobject parametersJ); + // Load an index via an input stream into memory. Use parametersJ to set any query time parameters + // + // Return a pointer to the loaded index + jlong LoadIndexWithStream(knn_jni::JNIUtilInterface * jniUtil, + JNIEnv * env, + jobject readStream, + jobject parametersJ); + // Execute a query against the index located in memory at indexPointerJ. // // Return an array of KNNQueryResults diff --git a/jni/include/org_opensearch_knn_jni_NmslibService.h b/jni/include/org_opensearch_knn_jni_NmslibService.h index a9d5238b7..8d6633aff 100644 --- a/jni/include/org_opensearch_knn_jni_NmslibService.h +++ b/jni/include/org_opensearch_knn_jni_NmslibService.h @@ -34,6 +34,14 @@ JNIEXPORT void JNICALL Java_org_opensearch_knn_jni_NmslibService_createIndex JNIEXPORT jlong JNICALL Java_org_opensearch_knn_jni_NmslibService_loadIndex (JNIEnv *, jclass, jstring, jobject); +/* + * Class: org_opensearch_knn_jni_NmslibService + * Method: loadIndexWithStream + * Signature: (Lorg/opensearch/knn/index/store/IndexInputWithBuffer;Ljava/util/Map;)J + */ +JNIEXPORT jlong JNICALL Java_org_opensearch_knn_jni_NmslibService_loadIndexWithStream + (JNIEnv *, jclass, jobject, jobject); + /* * Class: org_opensearch_knn_jni_NmslibService * Method: queryIndex diff --git a/jni/patches/nmslib/0003-Added-streaming-apis-for-vector-index-loading-in-Hnsw.patch b/jni/patches/nmslib/0003-Added-streaming-apis-for-vector-index-loading-in-Hnsw.patch new file mode 100644 index 000000000..55e7a8c81 --- /dev/null +++ b/jni/patches/nmslib/0003-Added-streaming-apis-for-vector-index-loading-in-Hnsw.patch @@ -0,0 +1,221 @@ +From 2e9b7f7117842009e081dd79e8ab8b019122a3de Mon Sep 17 00:00:00 2001 +From: Dooyong Kim +Date: Fri, 11 Oct 2024 16:19:45 -0700 +Subject: [PATCH] Added streaming apis for vector index loading in Hnsw. + +Signed-off-by: Dooyong Kim +--- + similarity_search/include/method/hnsw.h | 3 + + similarity_search/include/utils.h | 12 +++ + similarity_search/src/method/hnsw.cc | 138 +++++++++++++++++++++++- + 3 files changed, 152 insertions(+), 1 deletion(-) + +diff --git a/similarity_search/include/method/hnsw.h b/similarity_search/include/method/hnsw.h +index e6dcea7..433f98f 100644 +--- a/similarity_search/include/method/hnsw.h ++++ b/similarity_search/include/method/hnsw.h +@@ -457,6 +457,8 @@ namespace similarity { + + virtual void LoadIndex(const string &location) override; + ++ void LoadIndexWithStream(similarity::NmslibIOReader& in); ++ + Hnsw(bool PrintProgress, const Space &space, const ObjectVector &data); + void CreateIndex(const AnyParams &IndexParams) override; + +@@ -500,6 +502,7 @@ namespace similarity { + + void SaveOptimizedIndex(std::ostream& output); + void LoadOptimizedIndex(std::istream& input); ++ void LoadOptimizedIndex(NmslibIOReader& input); + + void SaveRegularIndexBin(std::ostream& output); + void LoadRegularIndexBin(std::istream& input); +diff --git a/similarity_search/include/utils.h b/similarity_search/include/utils.h +index b521c26..a3931b7 100644 +--- a/similarity_search/include/utils.h ++++ b/similarity_search/include/utils.h +@@ -299,12 +299,24 @@ inline void WriteField(ostream& out, const string& fieldName, const FieldType& f + } + } + ++struct NmslibIOReader { ++ virtual ~NmslibIOReader() = default; ++ ++ virtual void read(char* bytes, size_t len) = 0; ++ ++ virtual size_t remainingBytes() = 0; ++}; + + template + void writeBinaryPOD(ostream& out, const T& podRef) { + out.write((char*)&podRef, sizeof(T)); + } + ++template ++static void readBinaryPOD(NmslibIOReader& in, T& podRef) { ++ in.read((char*)&podRef, sizeof(T)); ++} ++ + template + static void readBinaryPOD(istream& in, T& podRef) { + in.read((char*)&podRef, sizeof(T)); +diff --git a/similarity_search/src/method/hnsw.cc b/similarity_search/src/method/hnsw.cc +index 4080b3b..662f06c 100644 +--- a/similarity_search/src/method/hnsw.cc ++++ b/similarity_search/src/method/hnsw.cc +@@ -950,7 +950,6 @@ namespace similarity { + " read so far doesn't match the number of read lines: " + ConvertToString(lineNum)); + } + +- + template + void + Hnsw::LoadRegularIndexBin(std::istream& input) { +@@ -1034,6 +1033,143 @@ namespace similarity { + + } + ++ constexpr bool _isLittleEndian() { ++ return (((uint32_t) 1) & 0xFFU) == 1; ++ } ++ ++ SIZEMASS_TYPE _readIntBigEndian(uint8_t byte0, uint8_t byte1, uint8_t byte2, uint8_t byte3) noexcept { ++ return (static_cast(byte0) << 24) | ++ (static_cast(byte1) << 16) | ++ (static_cast(byte2) << 8) | ++ static_cast(byte3); ++ } ++ ++ SIZEMASS_TYPE _readIntLittleEndian(uint8_t byte0, uint8_t byte1, uint8_t byte2, uint8_t byte3) noexcept { ++ return (static_cast(byte3) << 24) | ++ (static_cast(byte2) << 16) | ++ (static_cast(byte1) << 8) | ++ static_cast(byte0); ++ } ++ ++ template ++ void Hnsw::LoadIndexWithStream(NmslibIOReader& input) { ++ LOG(LIB_INFO) << "Loading index from an input stream(NmslibIOReader)."; ++ ++ unsigned int optimIndexFlag= 0; ++ readBinaryPOD(input, optimIndexFlag); ++ ++ if (!optimIndexFlag) { ++ throw std::runtime_error("With stream, we only support optimized index type."); ++ } else { ++ LoadOptimizedIndex(input); ++ } ++ ++ LOG(LIB_INFO) << "Finished loading index"; ++ visitedlistpool = new VisitedListPool(1, totalElementsStored_); ++ } ++ ++ template ++ void Hnsw::LoadOptimizedIndex(NmslibIOReader& input) { ++ static_assert(sizeof(SIZEMASS_TYPE) == 4, "Expected sizeof(SIZEMASS_TYPE) == 4."); ++ ++ LOG(LIB_INFO) << "Loading optimized index(NmslibIOReader)."; ++ ++ readBinaryPOD(input, totalElementsStored_); ++ readBinaryPOD(input, memoryPerObject_); ++ readBinaryPOD(input, offsetLevel0_); ++ readBinaryPOD(input, offsetData_); ++ readBinaryPOD(input, maxlevel_); ++ readBinaryPOD(input, enterpointId_); ++ readBinaryPOD(input, maxM_); ++ readBinaryPOD(input, maxM0_); ++ readBinaryPOD(input, dist_func_type_); ++ readBinaryPOD(input, searchMethod_); ++ ++ LOG(LIB_INFO) << "searchMethod: " << searchMethod_; ++ ++ fstdistfunc_ = getDistFunc(dist_func_type_); ++ iscosine_ = (dist_func_type_ == kNormCosine); ++ CHECK_MSG(fstdistfunc_ != nullptr, "Unknown distance function code: " + ConvertToString(dist_func_type_)); ++ ++ LOG(LIB_INFO) << "Total: " << totalElementsStored_ << ", Memory per object: " << memoryPerObject_; ++ size_t data_plus_links0_size = memoryPerObject_ * totalElementsStored_; ++ ++ // we allocate a few extra bytes to prevent prefetch from accessing out of range memory ++ data_level0_memory_ = (char *)malloc(data_plus_links0_size + EXTRA_MEM_PAD_SIZE); ++ CHECK(data_level0_memory_); ++ input.read(data_level0_memory_, data_plus_links0_size); ++ // we allocate a few extra bytes to prevent prefetch from accessing out of range memory ++ linkLists_ = (char **)malloc( (sizeof(void *) * totalElementsStored_) + EXTRA_MEM_PAD_SIZE); ++ CHECK(linkLists_); ++ ++ data_rearranged_.resize(totalElementsStored_); ++ ++ const size_t bufferSize = 64 * 1024; // 64KB ++ std::unique_ptr buffer (new char[bufferSize]); ++ uint32_t end = 0; ++ uint32_t pos = 0; ++ constexpr bool isLittleEndian = _isLittleEndian(); ++ ++ for (size_t i = 0, remainingBytes = input.remainingBytes(); i < totalElementsStored_; i++) { ++ if ((pos + sizeof(SIZEMASS_TYPE)) >= end) { ++ // Underflow during reading an integer size field. ++ // So the idea is to move the first partial bytes (which is < 4 bytes) to the beginning section of ++ // buffer. ++ // Ex: buffer -> [..., b0, b1] where we only have two bytes and still need to read two bytes more ++ // buffer -> [b0, b1, ...] after move the first part. firstPartLen = 2. ++ const auto firstPartLen = end - pos; ++ if (firstPartLen > 0) { ++ std::memcpy(buffer.get(), buffer.get() + pos, firstPartLen); ++ } ++ // Then, bulk load bytes from input stream. Note that the first few bytes are already occupied by ++ // earlier moving logic, hence required bytes are bufferSize - firstPartLen. ++ const auto copyBytes = std::min(remainingBytes, bufferSize - firstPartLen); ++ input.read(buffer.get() + firstPartLen, copyBytes); ++ remainingBytes -= copyBytes; ++ end = copyBytes + firstPartLen; ++ pos = 0; ++ } ++ ++ // Read data size field. ++ // Since NMSLIB directly write 4 bytes integer casting to char*, bytes outline may differ among systems. ++ SIZEMASS_TYPE linkListSize = 0; ++ if (isLittleEndian) { ++ linkListSize = _readIntLittleEndian(buffer[pos], buffer[pos + 1], buffer[pos + 2], buffer[pos + 3]); ++ } else { ++ linkListSize = _readIntBigEndian(buffer[pos], buffer[pos + 1], buffer[pos + 2], buffer[pos + 3]); ++ } ++ pos += sizeof(SIZEMASS_TYPE); ++ ++ if (linkListSize == 0) { ++ linkLists_[i] = nullptr; ++ } else { ++ linkLists_[i] = (char *)malloc(linkListSize); ++ CHECK(linkLists_[i]); ++ ++ SIZEMASS_TYPE leftLinkListData = linkListSize; ++ auto dataPtr = linkLists_[i]; ++ while (leftLinkListData > 0) { ++ if (pos >= end) { ++ // Underflow during read linked list bytes. ++ const auto copyBytes = std::min(remainingBytes, bufferSize); ++ input.read(buffer.get(), copyBytes); ++ remainingBytes -= copyBytes; ++ end = copyBytes; ++ pos = 0; ++ } ++ ++ // Read linked list bytes. ++ const auto copyBytes = std::min(leftLinkListData, end - pos); ++ std::memcpy(dataPtr, buffer.get() + pos, copyBytes); ++ dataPtr += copyBytes; ++ leftLinkListData -= copyBytes; ++ pos += copyBytes; ++ } // End while ++ } // End if ++ ++ data_rearranged_[i] = new Object(data_level0_memory_ + (i)*memoryPerObject_ + offsetData_); ++ } // End for ++ } + + template + void +-- +2.39.5 (Apple Git-154) + diff --git a/jni/patches/nmslib/0003-Adding-two-apis-using-stream-to-load-save-in-Hnsw.patch b/jni/patches/nmslib/0003-Adding-two-apis-using-stream-to-load-save-in-Hnsw.patch deleted file mode 100644 index bbba329b4..000000000 --- a/jni/patches/nmslib/0003-Adding-two-apis-using-stream-to-load-save-in-Hnsw.patch +++ /dev/null @@ -1,93 +0,0 @@ -From 7e099ec111e5c9db4b243da249c73f0ecc206281 Mon Sep 17 00:00:00 2001 -From: Dooyong Kim -Date: Thu, 26 Sep 2024 15:20:53 -0700 -Subject: [PATCH] Adding two apis using stream to load/save in Hnsw. - -Signed-off-by: Dooyong Kim ---- - similarity_search/include/method/hnsw.h | 4 +++ - similarity_search/src/method/hnsw.cc | 44 +++++++++++++++++++++++++ - 2 files changed, 48 insertions(+) - -diff --git a/similarity_search/include/method/hnsw.h b/similarity_search/include/method/hnsw.h -index 57d99d0..7ff3f3d 100644 ---- a/similarity_search/include/method/hnsw.h -+++ b/similarity_search/include/method/hnsw.h -@@ -455,8 +455,12 @@ namespace similarity { - public: - virtual void SaveIndex(const string &location) override; - -+ void SaveIndexWithStream(std::ostream& output); -+ - virtual void LoadIndex(const string &location) override; - -+ void LoadIndexWithStream(std::istream& in); -+ - Hnsw(bool PrintProgress, const Space &space, const ObjectVector &data); - void CreateIndex(const AnyParams &IndexParams) override; - -diff --git a/similarity_search/src/method/hnsw.cc b/similarity_search/src/method/hnsw.cc -index 35b372c..e7a2c9e 100644 ---- a/similarity_search/src/method/hnsw.cc -+++ b/similarity_search/src/method/hnsw.cc -@@ -771,6 +771,25 @@ namespace similarity { - output.close(); - } - -+ template -+ void Hnsw::SaveIndexWithStream(std::ostream &output) { -+ output.exceptions(ios::badbit | ios::failbit); -+ -+ unsigned int optimIndexFlag = data_level0_memory_ != nullptr; -+ -+ writeBinaryPOD(output, optimIndexFlag); -+ -+ if (!optimIndexFlag) { -+#if USE_TEXT_REGULAR_INDEX -+ SaveRegularIndexText(output); -+#else -+ SaveRegularIndexBin(output); -+#endif -+ } else { -+ SaveOptimizedIndex(output); -+ } -+ } -+ - template - void - Hnsw::SaveOptimizedIndex(std::ostream& output) { -@@ -1021,6 +1040,31 @@ namespace similarity { - - } - -+ template -+ void Hnsw::LoadIndexWithStream(std::istream& input) { -+ LOG(LIB_INFO) << "Loading index from an input stream."; -+ CHECK_MSG(input, "Cannot open file for reading with an input stream"); -+ -+ input.exceptions(ios::badbit | ios::failbit); -+ -+#if USE_TEXT_REGULAR_INDEX -+ LoadRegularIndexText(input); -+#else -+ unsigned int optimIndexFlag= 0; -+ -+ readBinaryPOD(input, optimIndexFlag); -+ -+ if (!optimIndexFlag) { -+ LoadRegularIndexBin(input); -+ } else { -+ LoadOptimizedIndex(input); -+ } -+#endif -+ -+ LOG(LIB_INFO) << "Finished loading index"; -+ visitedlistpool = new VisitedListPool(1, totalElementsStored_); -+ } -+ - - template - void --- -2.39.5 (Apple Git-154) - diff --git a/jni/src/jni_util.cpp b/jni/src/jni_util.cpp index 3eaf3b0a1..8dc818c94 100644 --- a/jni/src/jni_util.cpp +++ b/jni/src/jni_util.cpp @@ -563,8 +563,14 @@ jfieldID knn_jni::JNIUtil::GetFieldID(JNIEnv * env, jclass clazz, const char *na return env->GetFieldID(clazz, name, sig); } -jint knn_jni::JNIUtil::CallIntMethodLong(JNIEnv * env, jobject obj, jmethodID methodID, int64_t longArg) { - return env->CallIntMethod(obj, methodID, longArg); +jint knn_jni::JNIUtil::CallNonvirtualIntMethodA(JNIEnv * env, jobject obj, jclass clazz, + jmethodID methodID, jvalue* args) { + return env->CallNonvirtualIntMethodA(obj, clazz, methodID, args); +} + +jlong knn_jni::JNIUtil::CallNonvirtualLongMethodA(JNIEnv * env, jobject obj, jclass clazz, + jmethodID methodID, jvalue* args) { + return env->CallNonvirtualLongMethodA(obj, clazz, methodID, args); } void * knn_jni::JNIUtil::GetPrimitiveArrayCritical(JNIEnv * env, jarray array, jboolean *isCopy) { diff --git a/jni/src/nmslib_wrapper.cpp b/jni/src/nmslib_wrapper.cpp index 21b34eb83..536558caa 100644 --- a/jni/src/nmslib_wrapper.cpp +++ b/jni/src/nmslib_wrapper.cpp @@ -11,6 +11,7 @@ #include "jni_util.h" #include "nmslib_wrapper.h" +#include "nmslib_stream_support.h" #include "commons.h" @@ -25,303 +26,365 @@ #include #include +#include #include "hnswquery.h" +#include "method/hnsw.h" - -std::string TranslateSpaceType(const std::string& spaceType); +std::string TranslateSpaceType(const std::string &spaceType); // We do not use label functionality of nmslib so we pass default label. Setting as a const allows us to avoid a few // allocations const similarity::LabelType DEFAULT_LABEL = -1; -void knn_jni::nmslib_wrapper::CreateIndex(knn_jni::JNIUtilInterface * jniUtil, JNIEnv * env, jintArray idsJ, +void knn_jni::nmslib_wrapper::CreateIndex(knn_jni::JNIUtilInterface *jniUtil, JNIEnv *env, jintArray idsJ, jlong vectorsAddressJ, jint dimJ, jstring indexPathJ, jobject parametersJ) { - if (idsJ == nullptr) { - throw std::runtime_error("IDs cannot be null"); - } + if (idsJ == nullptr) { + throw std::runtime_error("IDs cannot be null"); + } - if (vectorsAddressJ <= 0) { - throw std::runtime_error("VectorsAddress cannot be less than 0"); - } + if (vectorsAddressJ <= 0) { + throw std::runtime_error("VectorsAddress cannot be less than 0"); + } - if(dimJ <= 0) { - throw std::runtime_error("Vectors dimensions cannot be less than or equal to 0"); - } + if (dimJ <= 0) { + throw std::runtime_error("Vectors dimensions cannot be less than or equal to 0"); + } - if (indexPathJ == nullptr) { - throw std::runtime_error("Index path cannot be null"); - } + if (indexPathJ == nullptr) { + throw std::runtime_error("Index path cannot be null"); + } - if (parametersJ == nullptr) { - throw std::runtime_error("Parameters cannot be null"); - } - - // Handle parameters - auto parametersCpp = jniUtil->ConvertJavaMapToCppMap(env, parametersJ); - std::vector indexParameters; + if (parametersJ == nullptr) { + throw std::runtime_error("Parameters cannot be null"); + } - // Algorithm parameters will be in a sub map - if(parametersCpp.find(knn_jni::PARAMETERS) != parametersCpp.end()) { - jobject subParametersJ = parametersCpp[knn_jni::PARAMETERS]; - auto subParametersCpp = jniUtil->ConvertJavaMapToCppMap(env, subParametersJ); + // Handle parameters + auto parametersCpp = jniUtil->ConvertJavaMapToCppMap(env, parametersJ); + std::vector indexParameters; - if(subParametersCpp.find(knn_jni::EF_CONSTRUCTION) != subParametersCpp.end()) { - auto efConstruction = jniUtil->ConvertJavaObjectToCppInteger(env, subParametersCpp[knn_jni::EF_CONSTRUCTION]); - indexParameters.push_back(knn_jni::EF_CONSTRUCTION_NMSLIB + "=" + std::to_string(efConstruction)); - } + // Algorithm parameters will be in a sub map + if (parametersCpp.find(knn_jni::PARAMETERS) != parametersCpp.end()) { + jobject subParametersJ = parametersCpp[knn_jni::PARAMETERS]; + auto subParametersCpp = jniUtil->ConvertJavaMapToCppMap(env, subParametersJ); - if(subParametersCpp.find(knn_jni::M) != subParametersCpp.end()) { - auto m = jniUtil->ConvertJavaObjectToCppInteger(env, subParametersCpp[knn_jni::M]); - indexParameters.push_back(knn_jni::M_NMSLIB + "=" + std::to_string(m)); - } - - jniUtil->DeleteLocalRef(env, subParametersJ); + if (subParametersCpp.find(knn_jni::EF_CONSTRUCTION) != subParametersCpp.end()) { + auto efConstruction = jniUtil->ConvertJavaObjectToCppInteger(env, subParametersCpp[knn_jni::EF_CONSTRUCTION]); + indexParameters.push_back(knn_jni::EF_CONSTRUCTION_NMSLIB + "=" + std::to_string(efConstruction)); } - if(parametersCpp.find(knn_jni::INDEX_THREAD_QUANTITY) != parametersCpp.end()) { - auto indexThreadQty = jniUtil->ConvertJavaObjectToCppInteger(env, parametersCpp[knn_jni::INDEX_THREAD_QUANTITY]); - indexParameters.push_back(knn_jni::INDEX_THREAD_QUANTITY + "=" + std::to_string(indexThreadQty)); + if (subParametersCpp.find(knn_jni::M) != subParametersCpp.end()) { + auto m = jniUtil->ConvertJavaObjectToCppInteger(env, subParametersCpp[knn_jni::M]); + indexParameters.push_back(knn_jni::M_NMSLIB + "=" + std::to_string(m)); } - jniUtil->DeleteLocalRef(env, parametersJ); - - // Get the path to save the index - std::string indexPathCpp(jniUtil->ConvertJavaStringToCppString(env, indexPathJ)); - - // Get space type for this index - jobject spaceTypeJ = knn_jni::GetJObjectFromMapOrThrow(parametersCpp, knn_jni::SPACE_TYPE); - std::string spaceTypeCpp(jniUtil->ConvertJavaObjectToCppString(env, spaceTypeJ)); - spaceTypeCpp = TranslateSpaceType(spaceTypeCpp); - - std::unique_ptr> space; - space.reset(similarity::SpaceFactoryRegistry::Instance().CreateSpace(spaceTypeCpp,similarity::AnyParams())); - - // Get number of ids and vectors and dimension - auto *inputVectors = reinterpret_cast*>(vectorsAddressJ); - int dim = (int)dimJ; - // The number of vectors can be int here because a lucene segment number of total docs never crosses INT_MAX value - int numVectors = (int) ( inputVectors->size() / (uint64_t) dim); - if(numVectors == 0) { - throw std::runtime_error("Number of vectors cannot be 0"); + jniUtil->DeleteLocalRef(env, subParametersJ); + } + + if (parametersCpp.find(knn_jni::INDEX_THREAD_QUANTITY) != parametersCpp.end()) { + auto indexThreadQty = jniUtil->ConvertJavaObjectToCppInteger(env, parametersCpp[knn_jni::INDEX_THREAD_QUANTITY]); + indexParameters.push_back(knn_jni::INDEX_THREAD_QUANTITY + "=" + std::to_string(indexThreadQty)); + } + + jniUtil->DeleteLocalRef(env, parametersJ); + + // Get the path to save the index + std::string indexPathCpp(jniUtil->ConvertJavaStringToCppString(env, indexPathJ)); + + // Get space type for this index + jobject spaceTypeJ = knn_jni::GetJObjectFromMapOrThrow(parametersCpp, knn_jni::SPACE_TYPE); + std::string spaceTypeCpp(jniUtil->ConvertJavaObjectToCppString(env, spaceTypeJ)); + spaceTypeCpp = TranslateSpaceType(spaceTypeCpp); + + std::unique_ptr> space; + space.reset(similarity::SpaceFactoryRegistry::Instance().CreateSpace(spaceTypeCpp, similarity::AnyParams())); + + // Get number of ids and vectors and dimension + auto *inputVectors = reinterpret_cast *>(vectorsAddressJ); + int dim = (int) dimJ; + // The number of vectors can be int here because a lucene segment number of total docs never crosses INT_MAX value + int numVectors = (int) (inputVectors->size() / (uint64_t) dim); + if (numVectors == 0) { + throw std::runtime_error("Number of vectors cannot be 0"); + } + + int numIds = jniUtil->GetJavaIntArrayLength(env, idsJ); + if (numIds != numVectors) { + throw std::runtime_error("Number of IDs does not match number of vectors"); + } + + // Read dataset + similarity::ObjectVector dataset; + dataset.reserve(numVectors); + int *idsCpp; + try { + // Read in data set + idsCpp = jniUtil->GetIntArrayElements(env, idsJ, nullptr); + size_t vectorSizeInBytes = dim * sizeof(float); + // vectorPointer needs to be unsigned long long, this will ensure that out of range doesn't happen for this pointer + // when the values of numVectors * dim becomes very large. + // Example: for 10M vectors of 1536 dim vectorPointer max value will be ~15.3B which is already > range of ints. + // keeping it unsigned long long we will never go above the range. + unsigned long long vectorPointer = 0; + + // Allocate a large buffer that will contain all the vectors. Allocating the objects in one large buffer as + // opposed to individually will prevent heap fragmentation. We have observed that allocating individual + // objects causes RSS to rise throughout the lifetime of a process + // (see https://github.com/opensearch-project/k-NN/issues/772 and + // https://github.com/opensearch-project/k-NN/issues/72). This is because, in typical systems, small + // allocations will reside on some kind of heap managed by an allocator. Once freed, the allocator does not + // always return the memory to the OS. If the heap gets fragmented, this will cause the allocator + // to ask for more memory, causing RSS to grow. On large allocations (> 128 kb), most allocators will + // internally use mmap. Once freed, unmap will be called, which will immediately return memory to the OS + // which in turn prevents RSS from growing out of control. Wrap with a smart pointer so that buffer will be + // freed once variable goes out of scope. For reference, the code that specifies the layout of the buffer can be + // found: https://github.com/nmslib/nmslib/blob/v2.1.1/similarity_search/include/object.h#L61-L75 + std::unique_ptr objectBuffer + (new char[(similarity::ID_SIZE + similarity::LABEL_SIZE + similarity::DATALENGTH_SIZE + vectorSizeInBytes) + * numVectors]); + char *ptr = objectBuffer.get(); + for (int i = 0; i < numVectors; i++) { + dataset.push_back(new similarity::Object(ptr)); + + memcpy(ptr, &idsCpp[i], similarity::ID_SIZE); + ptr += similarity::ID_SIZE; + memcpy(ptr, &DEFAULT_LABEL, similarity::LABEL_SIZE); + ptr += similarity::LABEL_SIZE; + memcpy(ptr, &vectorSizeInBytes, similarity::DATALENGTH_SIZE); + ptr += similarity::DATALENGTH_SIZE; + + memcpy(ptr, &(inputVectors->at(vectorPointer)), vectorSizeInBytes); + ptr += vectorSizeInBytes; + vectorPointer += dim; } - - int numIds = jniUtil->GetJavaIntArrayLength(env, idsJ); - if (numIds != numVectors) { - throw std::runtime_error("Number of IDs does not match number of vectors"); + jniUtil->ReleaseIntArrayElements(env, idsJ, idsCpp, JNI_ABORT); + + // Releasing the vectorsAddressJ memory as that is not required once we have created the index. + // This is not the ideal approach, please refer this gh issue for long term solution: + // https://github.com/opensearch-project/k-NN/issues/1600 + //commons::freeVectorData(vectorsAddressJ); + delete inputVectors; + + std::unique_ptr> index; + index.reset(similarity::MethodFactoryRegistry::Instance().CreateMethod(false, + "hnsw", + spaceTypeCpp, + *(space), + dataset)); + index->CreateIndex(similarity::AnyParams(indexParameters)); + index->SaveIndex(indexPathCpp); + + for (auto &it : dataset) { + delete it; } - - // Read dataset - similarity::ObjectVector dataset; - dataset.reserve(numVectors); - int* idsCpp; - try { - // Read in data set - idsCpp = jniUtil->GetIntArrayElements(env, idsJ, nullptr); - size_t vectorSizeInBytes = dim*sizeof(float); - // vectorPointer needs to be unsigned long long, this will ensure that out of range doesn't happen for this pointer - // when the values of numVectors * dim becomes very large. - // Example: for 10M vectors of 1536 dim vectorPointer max value will be ~15.3B which is already > range of ints. - // keeping it unsigned long long we will never go above the range. - unsigned long long vectorPointer = 0; - - // Allocate a large buffer that will contain all the vectors. Allocating the objects in one large buffer as - // opposed to individually will prevent heap fragmentation. We have observed that allocating individual - // objects causes RSS to rise throughout the lifetime of a process - // (see https://github.com/opensearch-project/k-NN/issues/772 and - // https://github.com/opensearch-project/k-NN/issues/72). This is because, in typical systems, small - // allocations will reside on some kind of heap managed by an allocator. Once freed, the allocator does not - // always return the memory to the OS. If the heap gets fragmented, this will cause the allocator - // to ask for more memory, causing RSS to grow. On large allocations (> 128 kb), most allocators will - // internally use mmap. Once freed, unmap will be called, which will immediately return memory to the OS - // which in turn prevents RSS from growing out of control. Wrap with a smart pointer so that buffer will be - // freed once variable goes out of scope. For reference, the code that specifies the layout of the buffer can be - // found: https://github.com/nmslib/nmslib/blob/v2.1.1/similarity_search/include/object.h#L61-L75 - std::unique_ptr objectBuffer(new char[(similarity::ID_SIZE + similarity::LABEL_SIZE + similarity::DATALENGTH_SIZE + vectorSizeInBytes) * numVectors]); - char* ptr = objectBuffer.get(); - for (int i = 0; i < numVectors; i++) { - dataset.push_back(new similarity::Object(ptr)); - - memcpy(ptr, &idsCpp[i], similarity::ID_SIZE); - ptr += similarity::ID_SIZE; - memcpy(ptr, &DEFAULT_LABEL, similarity::LABEL_SIZE); - ptr += similarity::LABEL_SIZE; - memcpy(ptr, &vectorSizeInBytes, similarity::DATALENGTH_SIZE); - ptr += similarity::DATALENGTH_SIZE; - - memcpy(ptr, &(inputVectors->at(vectorPointer)), vectorSizeInBytes); - ptr += vectorSizeInBytes; - vectorPointer += dim; - } - jniUtil->ReleaseIntArrayElements(env, idsJ, idsCpp, JNI_ABORT); - - // Releasing the vectorsAddressJ memory as that is not required once we have created the index. - // This is not the ideal approach, please refer this gh issue for long term solution: - // https://github.com/opensearch-project/k-NN/issues/1600 - //commons::freeVectorData(vectorsAddressJ); - delete inputVectors; - - std::unique_ptr> index; - index.reset(similarity::MethodFactoryRegistry::Instance().CreateMethod(false, "hnsw", spaceTypeCpp, *(space), dataset)); - index->CreateIndex(similarity::AnyParams(indexParameters)); - index->SaveIndex(indexPathCpp); - - for (auto & it : dataset) { - delete it; - } - } catch (...) { - for (auto & it : dataset) { - delete it; - } - - jniUtil->ReleaseIntArrayElements(env, idsJ, idsCpp, JNI_ABORT); - throw; + } catch (...) { + for (auto &it : dataset) { + delete it; } + + jniUtil->ReleaseIntArrayElements(env, idsJ, idsCpp, JNI_ABORT); + throw; + } } -jlong knn_jni::nmslib_wrapper::LoadIndex(knn_jni::JNIUtilInterface * jniUtil, JNIEnv * env, jstring indexPathJ, +jlong knn_jni::nmslib_wrapper::LoadIndex(knn_jni::JNIUtilInterface *jniUtil, JNIEnv *env, jstring indexPathJ, jobject parametersJ) { - if (indexPathJ == nullptr) { - throw std::runtime_error("Index path cannot be null"); - } + if (indexPathJ == nullptr) { + throw std::runtime_error("Index path cannot be null"); + } - if (parametersJ == nullptr) { - throw std::runtime_error("Parameters cannot be null"); - } + if (parametersJ == nullptr) { + throw std::runtime_error("Parameters cannot be null"); + } - std::string indexPathCpp(jniUtil->ConvertJavaStringToCppString(env, indexPathJ)); + std::string indexPathCpp(jniUtil->ConvertJavaStringToCppString(env, indexPathJ)); - auto parametersCpp = jniUtil->ConvertJavaMapToCppMap(env, parametersJ); + auto parametersCpp = jniUtil->ConvertJavaMapToCppMap(env, parametersJ); - // Get space type for this index - jobject spaceTypeJ = knn_jni::GetJObjectFromMapOrThrow(parametersCpp, knn_jni::SPACE_TYPE); - std::string spaceTypeCpp(jniUtil->ConvertJavaObjectToCppString(env, spaceTypeJ)); - spaceTypeCpp = TranslateSpaceType(spaceTypeCpp); + // Get space type for this index + jobject spaceTypeJ = knn_jni::GetJObjectFromMapOrThrow(parametersCpp, knn_jni::SPACE_TYPE); + std::string spaceTypeCpp(jniUtil->ConvertJavaObjectToCppString(env, spaceTypeJ)); + spaceTypeCpp = TranslateSpaceType(spaceTypeCpp); - // Parse query params - std::vector queryParams; + // Parse query params + std::vector queryParams; - if(parametersCpp.find("efSearch") != parametersCpp.end()) { - auto efSearch = std::to_string(jniUtil->ConvertJavaObjectToCppInteger(env, parametersCpp["efSearch"])); - queryParams.push_back("efSearch=" + efSearch); - } + auto it = parametersCpp.find("efSearch"); + if (it != parametersCpp.end()) { + auto efSearch = std::to_string(jniUtil->ConvertJavaObjectToCppInteger(env, it->second)); + queryParams.push_back("efSearch=" + efSearch); + } - // Load index - knn_jni::nmslib_wrapper::IndexWrapper * indexWrapper; - try { - indexWrapper = new knn_jni::nmslib_wrapper::IndexWrapper(spaceTypeCpp); - indexWrapper->index->LoadIndex(indexPathCpp); - indexWrapper->index->SetQueryTimeParams(similarity::AnyParams(queryParams)); - } catch (...) { - delete indexWrapper; - throw; + // Load index + knn_jni::nmslib_wrapper::IndexWrapper *indexWrapper = nullptr; + try { + indexWrapper = new knn_jni::nmslib_wrapper::IndexWrapper(spaceTypeCpp); + indexWrapper->index->LoadIndex(indexPathCpp); + indexWrapper->index->SetQueryTimeParams(similarity::AnyParams(queryParams)); + } catch (...) { + delete indexWrapper; + throw; + } + + return (jlong) indexWrapper; +} + +jlong knn_jni::nmslib_wrapper::LoadIndexWithStream(knn_jni::JNIUtilInterface *jniUtil, + JNIEnv *env, + jobject readStream, + jobject parametersJ) { + if (readStream == nullptr) { + throw std::runtime_error("Read stream cannot be null"); + } + + if (parametersJ == nullptr) { + throw std::runtime_error("Parameters cannot be null"); + } + + auto parametersCpp = jniUtil->ConvertJavaMapToCppMap(env, parametersJ); + + // Get space type for this index + jobject spaceTypeJ = knn_jni::GetJObjectFromMapOrThrow(parametersCpp, knn_jni::SPACE_TYPE); + std::string spaceTypeCpp(jniUtil->ConvertJavaObjectToCppString(env, spaceTypeJ)); + spaceTypeCpp = TranslateSpaceType(spaceTypeCpp); + + // Parse query params + std::vector queryParams; + + auto it = parametersCpp.find("efSearch"); + if (it != parametersCpp.end()) { + auto efSearch = std::to_string(jniUtil->ConvertJavaObjectToCppInteger(env, it->second)); + queryParams.push_back("efSearch=" + efSearch); + } + + // Create a mediator locally. + // Note that `indexInput` is `IndexInputWithBuffer` type. + knn_jni::stream::NativeEngineIndexInputMediator mediator{jniUtil, env, readStream}; + + knn_jni::stream::NmslibOpenSearchIOReader ioReader {&mediator}; + + // Load index + knn_jni::nmslib_wrapper::IndexWrapper *indexWrapper = nullptr; + try { + indexWrapper = new knn_jni::nmslib_wrapper::IndexWrapper(spaceTypeCpp); + indexWrapper->index->SetQueryTimeParams(similarity::AnyParams(queryParams)); + + if (auto hnswFloatIndex = dynamic_cast *>(indexWrapper->index.get())) { + hnswFloatIndex->LoadIndexWithStream(ioReader); + } else { + throw std::runtime_error("We only support similarity::Hnsw in NMSLIB."); } + } catch (...) { + delete indexWrapper; + throw; + } - return (jlong) indexWrapper; + return (jlong) indexWrapper; } -jobjectArray knn_jni::nmslib_wrapper::QueryIndex(knn_jni::JNIUtilInterface * jniUtil, JNIEnv * env, jlong indexPointerJ, +jobjectArray knn_jni::nmslib_wrapper::QueryIndex(knn_jni::JNIUtilInterface *jniUtil, JNIEnv *env, jlong indexPointerJ, jfloatArray queryVectorJ, jint kJ, jobject methodParamsJ) { - if (queryVectorJ == nullptr) { - throw std::runtime_error("Query Vector cannot be null"); - } - - if (indexPointerJ == 0) { - throw std::runtime_error("Invalid pointer to index"); - } + if (queryVectorJ == nullptr) { + throw std::runtime_error("Query Vector cannot be null"); + } - auto *indexWrapper = reinterpret_cast(indexPointerJ); + if (indexPointerJ == 0) { + throw std::runtime_error("Invalid pointer to index"); + } - int dim = jniUtil->GetJavaFloatArrayLength(env, queryVectorJ); + auto *indexWrapper = reinterpret_cast(indexPointerJ); - float* rawQueryvector = jniUtil->GetFloatArrayElements(env, queryVectorJ, nullptr); // Have to call release on this + int dim = jniUtil->GetJavaFloatArrayLength(env, queryVectorJ); - std::unique_ptr queryObject; - try { - queryObject.reset(new similarity::Object(-1, -1, dim*sizeof(float), rawQueryvector)); - } catch (...) { - jniUtil->ReleaseFloatArrayElements(env, queryVectorJ, rawQueryvector, JNI_ABORT); - throw; - } + float *rawQueryvector = jniUtil->GetFloatArrayElements(env, queryVectorJ, nullptr); // Have to call release on this + std::unique_ptr queryObject; + try { + queryObject.reset(new similarity::Object(-1, -1, dim * sizeof(float), rawQueryvector)); + } catch (...) { jniUtil->ReleaseFloatArrayElements(env, queryVectorJ, rawQueryvector, JNI_ABORT); - std::unordered_map methodParams; - if (methodParamsJ != nullptr) { - methodParams = jniUtil->ConvertJavaMapToCppMap(env, methodParamsJ); + throw; + } + + jniUtil->ReleaseFloatArrayElements(env, queryVectorJ, rawQueryvector, JNI_ABORT); + std::unordered_map methodParams; + if (methodParamsJ != nullptr) { + methodParams = jniUtil->ConvertJavaMapToCppMap(env, methodParamsJ); + } + + int queryEfSearch = knn_jni::commons::getIntegerMethodParameter(env, jniUtil, methodParams, EF_SEARCH, -1); + similarity::KNNQuery + *query; // TODO: Replace with smart pointers https://github.com/opensearch-project/k-NN/issues/1785 + std::unique_ptr> neighbors; + try { + if (queryEfSearch == -1) { + query = new similarity::KNNQuery(*(indexWrapper->space), queryObject.get(), kJ); + } else { + query = new similarity::HNSWQuery(*(indexWrapper->space), queryObject.get(), kJ, queryEfSearch); } - int queryEfSearch = knn_jni::commons::getIntegerMethodParameter(env, jniUtil, methodParams, EF_SEARCH, -1); - similarity::KNNQuery* query; // TODO: Replace with smart pointers https://github.com/opensearch-project/k-NN/issues/1785 - std::unique_ptr> neighbors; - try { - if (queryEfSearch == -1) { - query = new similarity::KNNQuery(*(indexWrapper->space), queryObject.get(), kJ); - } else { - query = new similarity::HNSWQuery(*(indexWrapper->space), queryObject.get(), kJ, queryEfSearch); - } - - indexWrapper->index->Search(query); - neighbors.reset(query->Result()->Clone()); - } catch (...) { - if (query != nullptr) { - delete query; - } - throw; + indexWrapper->index->Search(query); + neighbors.reset(query->Result()->Clone()); + } catch (...) { + if (query != nullptr) { + delete query; } - delete query; - - int resultSize = neighbors->Size(); - jclass resultClass = jniUtil->FindClass(env,"org/opensearch/knn/index/query/KNNQueryResult"); - jmethodID allArgs = jniUtil->FindMethod(env, "org/opensearch/knn/index/query/KNNQueryResult", ""); - - jobjectArray results = jniUtil->NewObjectArray(env, resultSize, resultClass, nullptr); - - jobject result; - float distance; - long id; - for(int i = 0; i < resultSize; ++i) { - distance = neighbors->TopDistance(); - id = neighbors->Pop()->id(); - result = jniUtil->NewObject(env, resultClass, allArgs, id, distance); - jniUtil->SetObjectArrayElement(env, results, i, result); - } - - return results; + throw; + } + delete query; + + int resultSize = neighbors->Size(); + jclass resultClass = jniUtil->FindClass(env, "org/opensearch/knn/index/query/KNNQueryResult"); + jmethodID allArgs = jniUtil->FindMethod(env, "org/opensearch/knn/index/query/KNNQueryResult", ""); + + jobjectArray results = jniUtil->NewObjectArray(env, resultSize, resultClass, nullptr); + + jobject result; + float distance; + long id; + for (int i = 0; i < resultSize; ++i) { + distance = neighbors->TopDistance(); + id = neighbors->Pop()->id(); + result = jniUtil->NewObject(env, resultClass, allArgs, id, distance); + jniUtil->SetObjectArrayElement(env, results, i, result); + } + + return results; } void knn_jni::nmslib_wrapper::Free(jlong indexPointerJ) { - auto *indexWrapper = reinterpret_cast(indexPointerJ); - delete indexWrapper; + auto *indexWrapper = reinterpret_cast(indexPointerJ); + delete indexWrapper; } void knn_jni::nmslib_wrapper::InitLibrary() { - similarity::initLibrary(); + similarity::initLibrary(); } -std::string TranslateSpaceType(const std::string& spaceType) { - if (spaceType == knn_jni::L2) { - return spaceType; - } +std::string TranslateSpaceType(const std::string &spaceType) { + if (spaceType == knn_jni::L2) { + return spaceType; + } - if (spaceType == knn_jni::L1) { - return spaceType; - } + if (spaceType == knn_jni::L1) { + return spaceType; + } - if (spaceType == knn_jni::LINF) { - return spaceType; - } + if (spaceType == knn_jni::LINF) { + return spaceType; + } - if (spaceType == knn_jni::COSINESIMIL) { - return spaceType; - } + if (spaceType == knn_jni::COSINESIMIL) { + return spaceType; + } - if (spaceType == knn_jni::INNER_PRODUCT) { - return knn_jni::NEG_DOT_PRODUCT; - } + if (spaceType == knn_jni::INNER_PRODUCT) { + return knn_jni::NEG_DOT_PRODUCT; + } - throw std::runtime_error("Invalid spaceType"); + throw std::runtime_error("Invalid spaceType"); } diff --git a/jni/src/org_opensearch_knn_jni_NmslibService.cpp b/jni/src/org_opensearch_knn_jni_NmslibService.cpp index e265827cd..8e4df2e9c 100644 --- a/jni/src/org_opensearch_knn_jni_NmslibService.cpp +++ b/jni/src/org_opensearch_knn_jni_NmslibService.cpp @@ -12,7 +12,6 @@ #include "org_opensearch_knn_jni_NmslibService.h" #include -#include #include "jni_util.h" #include "nmslib_wrapper.h" @@ -20,71 +19,85 @@ static knn_jni::JNIUtil jniUtil; static const jint KNN_NMSLIB_JNI_VERSION = JNI_VERSION_1_1; -jint JNI_OnLoad(JavaVM* vm, void* reserved) { - JNIEnv* env; - if (vm->GetEnv((void**)&env, KNN_NMSLIB_JNI_VERSION) != JNI_OK) { - return JNI_ERR; - } +jint JNI_OnLoad(JavaVM *vm, void *reserved) { + JNIEnv *env; + if (vm->GetEnv((void **) &env, KNN_NMSLIB_JNI_VERSION) != JNI_OK) { + return JNI_ERR; + } - jniUtil.Initialize(env); + jniUtil.Initialize(env); - return KNN_NMSLIB_JNI_VERSION; + return KNN_NMSLIB_JNI_VERSION; } void JNI_OnUnload(JavaVM *vm, void *reserved) { - JNIEnv* env; - vm->GetEnv((void**)&env, KNN_NMSLIB_JNI_VERSION); - jniUtil.Uninitialize(env); + JNIEnv *env; + vm->GetEnv((void **) &env, KNN_NMSLIB_JNI_VERSION); + jniUtil.Uninitialize(env); } -JNIEXPORT void JNICALL Java_org_opensearch_knn_jni_NmslibService_createIndex(JNIEnv * env, jclass cls, jintArray idsJ, - jlong vectorsAddressJ, jint dimJ, jstring indexPathJ, - jobject parametersJ) -{ - try { - knn_jni::nmslib_wrapper::CreateIndex(&jniUtil, env, idsJ, vectorsAddressJ, dimJ, indexPathJ, parametersJ); - } catch (...) { - jniUtil.CatchCppExceptionAndThrowJava(env); - } +JNIEXPORT void JNICALL Java_org_opensearch_knn_jni_NmslibService_createIndex(JNIEnv *env, + jclass cls, + jintArray idsJ, + jlong vectorsAddressJ, + jint dimJ, + jstring indexPathJ, + jobject parametersJ) { + try { + knn_jni::nmslib_wrapper::CreateIndex(&jniUtil, env, idsJ, vectorsAddressJ, dimJ, indexPathJ, parametersJ); + } catch (...) { + jniUtil.CatchCppExceptionAndThrowJava(env); + } } -JNIEXPORT jlong JNICALL Java_org_opensearch_knn_jni_NmslibService_loadIndex(JNIEnv * env, jclass cls, - jstring indexPathJ, jobject parametersJ) -{ - try { - return knn_jni::nmslib_wrapper::LoadIndex(&jniUtil, env, indexPathJ, parametersJ); - } catch (...) { - jniUtil.CatchCppExceptionAndThrowJava(env); - } - return NULL; +JNIEXPORT jlong JNICALL Java_org_opensearch_knn_jni_NmslibService_loadIndex(JNIEnv *env, jclass cls, + jstring indexPathJ, jobject parametersJ) { + try { + return knn_jni::nmslib_wrapper::LoadIndex(&jniUtil, env, indexPathJ, parametersJ); + } catch (...) { + jniUtil.CatchCppExceptionAndThrowJava(env); + } + return NULL; } -JNIEXPORT jobjectArray JNICALL Java_org_opensearch_knn_jni_NmslibService_queryIndex(JNIEnv * env, jclass cls, +JNIEXPORT jlong JNICALL Java_org_opensearch_knn_jni_NmslibService_loadIndexWithStream(JNIEnv *env, + jclass cls, + jobject readStream, + jobject parametersJ) { + try { + return knn_jni::nmslib_wrapper::LoadIndexWithStream(&jniUtil, env, readStream, parametersJ); + } catch (...) { + jniUtil.CatchCppExceptionAndThrowJava(env); + } + return NULL; +} + +JNIEXPORT jobjectArray JNICALL Java_org_opensearch_knn_jni_NmslibService_queryIndex(JNIEnv *env, + jclass cls, jlong indexPointerJ, - jfloatArray queryVectorJ, jint kJ, jobject methodParamsJ) -{ - try { - return knn_jni::nmslib_wrapper::QueryIndex(&jniUtil, env, indexPointerJ, queryVectorJ, kJ, methodParamsJ); - } catch (...) { - jniUtil.CatchCppExceptionAndThrowJava(env); - } - return nullptr; + jfloatArray queryVectorJ, + jint kJ, + jobject methodParamsJ) { + try { + return knn_jni::nmslib_wrapper::QueryIndex(&jniUtil, env, indexPointerJ, queryVectorJ, kJ, methodParamsJ); + } catch (...) { + jniUtil.CatchCppExceptionAndThrowJava(env); + } + return nullptr; } -JNIEXPORT void JNICALL Java_org_opensearch_knn_jni_NmslibService_free(JNIEnv * env, jclass cls, jlong indexPointerJ) -{ - try { - return knn_jni::nmslib_wrapper::Free(indexPointerJ); - } catch (...) { - jniUtil.CatchCppExceptionAndThrowJava(env); - } +JNIEXPORT void JNICALL Java_org_opensearch_knn_jni_NmslibService_free(JNIEnv *env, jclass cls, jlong indexPointerJ) { + try { + return knn_jni::nmslib_wrapper::Free(indexPointerJ); + } catch (...) { + jniUtil.CatchCppExceptionAndThrowJava(env); + } } -JNIEXPORT void JNICALL Java_org_opensearch_knn_jni_NmslibService_initLibrary(JNIEnv * env, jclass cls) -{ - try { - knn_jni::nmslib_wrapper::InitLibrary(); - } catch (...) { - jniUtil.CatchCppExceptionAndThrowJava(env); - } +JNIEXPORT void JNICALL Java_org_opensearch_knn_jni_NmslibService_initLibrary(JNIEnv *env, jclass cls) { + try { + knn_jni::nmslib_wrapper::InitLibrary(); + } catch (...) { + jniUtil.CatchCppExceptionAndThrowJava(env); + } } diff --git a/jni/tests/faiss_stream_support_test.cpp b/jni/tests/faiss_stream_support_test.cpp index 4045985bb..94a9b3991 100644 --- a/jni/tests/faiss_stream_support_test.cpp +++ b/jni/tests/faiss_stream_support_test.cpp @@ -8,83 +8,49 @@ // GitHub history for details. #include "faiss_stream_support.h" -#include +#include "native_stream_support_util.h" #include "test_util.h" + +#include #include -#include #include #include +using ::testing::_; using ::testing::Return; using knn_jni::stream::FaissOpenSearchIOReader; using knn_jni::stream::NativeEngineIndexInputMediator; using test_util::MockJNIUtil; - -// Mocking IndexInputWithBuffer. -struct JavaIndexInputMock { - JavaIndexInputMock(std::string _readTargetBytes, int32_t _bufSize) - : readTargetBytes(std::move(_readTargetBytes)), - nextReadIdx(), - buffer(_bufSize) { - } - - // This method is simulating `copyBytes` in IndexInputWithBuffer. - int32_t simulateCopyReads(int64_t readBytes) { - readBytes = std::min(readBytes, (int64_t) buffer.size()); - readBytes = std::min(readBytes, (int64_t) (readTargetBytes.size() - nextReadIdx)); - std::memcpy(buffer.data(), readTargetBytes.data() + nextReadIdx, readBytes); - nextReadIdx += readBytes; - return (int32_t) readBytes; - } - - static std::string makeRandomBytes(int32_t bytesSize) { - // Define the list of possible characters - static const string CHARACTERS - = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuv" - "wxyz0123456789"; - - // Create a random number generator - std::random_device rd; - std::mt19937 generator(rd()); - - // Create a distribution to uniformly select from all characters - std::uniform_int_distribution<> distribution( - 0, CHARACTERS.size() - 1); - - // Pre-allocate the string with the desired length - std::string randomString(bytesSize, '\0'); - - // Use generate_n with a back_inserter iterator - std::generate_n(randomString.begin(), bytesSize, [&]() { - return CHARACTERS[distribution(generator)]; - }); - - return randomString; - } - - std::string readTargetBytes; - int64_t nextReadIdx; - std::vector buffer; -}; // struct JavaIndexInputMock +using test_util::JavaIndexInputMock; +using ::testing::NiceMock; +using ::testing::Return; void setUpMockJNIUtil(JavaIndexInputMock &javaIndexInputMock, MockJNIUtil &mockJni) { // Set up mocking values + mocking behavior in a method. - ON_CALL(mockJni, FindClassFromJNIEnv).WillByDefault(Return((jclass) 1)); - ON_CALL(mockJni, GetMethodID).WillByDefault(Return((jmethodID) 1)); - ON_CALL(mockJni, GetFieldID).WillByDefault(Return((jfieldID) 1)); - ON_CALL(mockJni, GetObjectField).WillByDefault(Return((jobject) 1)); - ON_CALL(mockJni, CallIntMethodLong).WillByDefault([&javaIndexInputMock](JNIEnv *env, - jobject obj, - jmethodID methodID, - int64_t longArg) { - return javaIndexInputMock.simulateCopyReads(longArg); - }); - ON_CALL(mockJni, GetPrimitiveArrayCritical).WillByDefault([&javaIndexInputMock](JNIEnv *env, - jarray array, - jboolean *isCopy) { - return (jbyte *) javaIndexInputMock.buffer.data(); - }); - ON_CALL(mockJni, ReleasePrimitiveArrayCritical).WillByDefault(Return()); + EXPECT_CALL(mockJni, CallNonvirtualIntMethodA(_, _, _, _, _)) + .WillRepeatedly([&javaIndexInputMock](JNIEnv *env, + jobject obj, + jclass clazz, + jmethodID methodID, + jvalue* args) { + return javaIndexInputMock.simulateCopyReads(args[0].j); + }); + EXPECT_CALL(mockJni, CallNonvirtualLongMethodA(_, _, _, _, _)) + .WillRepeatedly([&javaIndexInputMock](JNIEnv *env, + jobject obj, + jclass clazz, + jmethodID methodID, + jvalue* args) { + return javaIndexInputMock.remainingBytes(); + }); + EXPECT_CALL(mockJni, GetPrimitiveArrayCritical(_, _, _)) + .WillRepeatedly([&javaIndexInputMock](JNIEnv *env, + jarray array, + jboolean *isCopy) { + return (jbyte *) javaIndexInputMock.buffer.data(); + }); + EXPECT_CALL(mockJni, ReleasePrimitiveArrayCritical(_, _, _, _)) + .WillRepeatedly(Return()); } TEST(FaissStreamSupportTest, NativeEngineIndexInputMediatorCopyWhenEmpty) { @@ -110,7 +76,7 @@ TEST(FaissStreamSupportTest, NativeEngineIndexInputMediatorCopyWhenEmpty) { TEST(FaissStreamSupportTest, FaissOpenSearchIOReaderCopy) { for (auto contentSize : std::vector{0, 2222, 7777, 1024, 77, 1}) { // Set up mockings - MockJNIUtil mockJni; + NiceMock mockJni; JavaIndexInputMock javaIndexInputMock{ JavaIndexInputMock::makeRandomBytes(contentSize), 1024}; setUpMockJNIUtil(javaIndexInputMock, mockJni); diff --git a/jni/tests/native_stream_support_util.h b/jni/tests/native_stream_support_util.h new file mode 100644 index 000000000..e33f3beb4 --- /dev/null +++ b/jni/tests/native_stream_support_util.h @@ -0,0 +1,102 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +#ifndef KNNPLUGIN_JNI_TESTS_NATIVE_STREAM_SUPPORT_UTIL_H_ +#define KNNPLUGIN_JNI_TESTS_NATIVE_STREAM_SUPPORT_UTIL_H_ + +#include "test_util.h" +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +namespace test_util { + + +// Mocking IndexInputWithBuffer. +struct JavaIndexInputMock { + JavaIndexInputMock(std::string _readTargetBytes, int32_t _bufSize) + : readTargetBytes(std::move(_readTargetBytes)), + nextReadIdx(), + buffer(_bufSize) { + } + + // This method is simulating `copyBytes` in IndexInputWithBuffer. + int32_t simulateCopyReads(int64_t readBytes) { + readBytes = std::min(readBytes, (int64_t) buffer.size()); + readBytes = std::min(readBytes, (int64_t) (readTargetBytes.size() - nextReadIdx)); + std::memcpy(buffer.data(), readTargetBytes.data() + nextReadIdx, readBytes); + nextReadIdx += readBytes; + return (int32_t) readBytes; + } + + int64_t remainingBytes() { + return readTargetBytes.size() - nextReadIdx; + } + + static std::string makeRandomBytes(int32_t bytesSize) { + // Define the list of possible characters + static const string CHARACTERS + = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuv" + "wxyz0123456789"; + + // Create a random number generator + std::random_device rd; + std::mt19937 generator(rd()); + + // Create a distribution to uniformly select from all characters + std::uniform_int_distribution<> distribution( + 0, CHARACTERS.size() - 1); + + // Pre-allocate the string with the desired length + std::string randomString(bytesSize, '\0'); + + // Use generate_n with a back_inserter iterator + std::generate_n(randomString.begin(), bytesSize, [&]() { + return CHARACTERS[distribution(generator)]; + }); + + return randomString; + } + + std::string readTargetBytes; + int64_t nextReadIdx; + std::vector buffer; +}; // struct JavaIndexInputMock + + + +struct JavaFileIndexInputMock { + JavaFileIndexInputMock(std::ifstream &_file_input, int32_t _buf_size) + : file_input(_file_input), + buffer(_buf_size) { + } + + int64_t remainingBytes() { + std::streampos currentPos = file_input.tellg(); + file_input.seekg(0, std::ios::end); + std::streamsize fileSize = file_input.tellg(); + file_input.seekg(currentPos); + return fileSize - currentPos; + } + + int32_t copyBytes(int64_t read_size) { + const auto copy_size = std::min((int64_t) buffer.size(), read_size); + file_input.read(buffer.data(), copy_size); + return (int32_t) copy_size; + } + + std::ifstream &file_input; + std::vector buffer; +}; // class JavaFileIndexInputMock + + +} // namespace test_util + +#endif //KNNPLUGIN_JNI_TESTS_NATIVE_STREAM_SUPPORT_UTIL_H_ diff --git a/jni/tests/nmslib_stream_support_test.cpp b/jni/tests/nmslib_stream_support_test.cpp new file mode 100644 index 000000000..e0e7a2d08 --- /dev/null +++ b/jni/tests/nmslib_stream_support_test.cpp @@ -0,0 +1,120 @@ +// SPDX-License-Identifier: Apache-2.0 +// +// The OpenSearch Contributors require contributions made to +// this file be licensed under the Apache-2.0 license or a +// compatible open source license. +// +// Modifications Copyright OpenSearch Contributors. See +// GitHub history for details. + +#include "nmslib_wrapper.h" + +#include + +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "jni_util.h" +#include "test_util.h" +#include "native_stream_support_util.h" + +using ::testing::_; +using ::testing::NiceMock; +using ::testing::Return; +using ::test_util::MockJNIUtil; +using ::test_util::JavaFileIndexInputMock; + +void setUpJavaFileInputMocking(JavaFileIndexInputMock &java_index_input, MockJNIUtil &mockJni) { + // Set up mocking values + mocking behavior in a method. + EXPECT_CALL(mockJni, CallNonvirtualIntMethodA(_, _, _, _, _)) + .WillRepeatedly([&java_index_input](JNIEnv *env, + jobject obj, + jclass clazz, + jmethodID methodID, + jvalue *args) { + return java_index_input.copyBytes(args[0].j); + }); + EXPECT_CALL(mockJni, CallNonvirtualLongMethodA(_, _, _, _, _)) + .WillRepeatedly([&java_index_input](JNIEnv *env, + jobject obj, + jclass clazz, + jmethodID methodID, + jvalue *args) { + return java_index_input.remainingBytes(); + }); + EXPECT_CALL(mockJni, GetPrimitiveArrayCritical(_, _, _)).WillRepeatedly([&java_index_input](JNIEnv *env, + jarray array, + jboolean *isCopy) { + return (jbyte *) java_index_input.buffer.data(); + }); + EXPECT_CALL(mockJni, ReleasePrimitiveArrayCritical(_, _, _, _)).WillRepeatedly(Return()); +} + +TEST(NmslibStreamLoadingTest, BasicAssertions) { + // Initialize nmslib + similarity::initLibrary(); + + // Define index data + int numIds = 100; + std::vector ids; + auto vectors = new std::vector(); + int dim = 2; + vectors->reserve(dim * numIds); + for (int i = 0; i < numIds; ++i) { + ids.push_back(i); + for (int j = 0; j < dim; ++j) { + vectors->push_back(test_util::RandomFloat(-500.0, 500.0)); + } + } + + std::string spaceType = knn_jni::L2; + std::string indexPath = test_util::RandomString( + 10, "/tmp/", ".nmslib"); + + std::unordered_map parametersMap; + int efConstruction = 512; + int m = 96; + + parametersMap[knn_jni::SPACE_TYPE] = (jobject) &spaceType; + parametersMap[knn_jni::EF_CONSTRUCTION] = (jobject) &efConstruction; + parametersMap[knn_jni::M] = (jobject) &m; + + // Set up jni + JNIEnv *jniEnv = nullptr; + NiceMock mockJNIUtil; + + EXPECT_CALL(mockJNIUtil, + GetJavaObjectArrayLength( + jniEnv, reinterpret_cast(vectors))) + .WillRepeatedly(Return(vectors->size())); + + EXPECT_CALL(mockJNIUtil, + GetJavaIntArrayLength(jniEnv, reinterpret_cast(&ids))) + .WillRepeatedly(Return(ids.size())); + + EXPECT_CALL(mockJNIUtil, + ConvertJavaMapToCppMap(jniEnv, reinterpret_cast(¶metersMap))) + .WillRepeatedly(Return(parametersMap)); + + // Create the index + knn_jni::nmslib_wrapper::CreateIndex( + &mockJNIUtil, jniEnv, reinterpret_cast(&ids), + (jlong) vectors, dim, (jstring) &indexPath, + (jobject) ¶metersMap); + + // Create Java index input mock. + std::ifstream file_input{indexPath, std::ios::binary}; + const int32_t buffer_size = 128; + JavaFileIndexInputMock java_file_index_input_mock{file_input, buffer_size}; + setUpJavaFileInputMocking(java_file_index_input_mock, mockJNIUtil); + + // Make sure index can be loaded + jlong index = knn_jni::nmslib_wrapper::LoadIndexWithStream( + &mockJNIUtil, jniEnv, + (jobject) (&java_file_index_input_mock), + (jobject) (¶metersMap)); + + knn_jni::nmslib_wrapper::Free(index); + + // Clean up + std::remove(indexPath.c_str()); +} diff --git a/jni/tests/test_util.h b/jni/tests/test_util.h index 286000c08..a6b39aa41 100644 --- a/jni/tests/test_util.h +++ b/jni/tests/test_util.h @@ -111,7 +111,8 @@ namespace test_util { MOCK_METHOD(jclass, FindClassFromJNIEnv, (JNIEnv * env, const char *name)); MOCK_METHOD(jmethodID, GetMethodID, (JNIEnv * env, jclass clazz, const char *name, const char *sig)); MOCK_METHOD(jfieldID, GetFieldID, (JNIEnv * env, jclass clazz, const char *name, const char *sig)); - MOCK_METHOD(jint, CallIntMethodLong, (JNIEnv * env, jobject obj, jmethodID methodID, int64_t longArg)); + MOCK_METHOD(jint, CallNonvirtualIntMethodA, (JNIEnv * env, jobject obj, jclass clazz, jmethodID methodID, jvalue* args)); + MOCK_METHOD(jlong, CallNonvirtualLongMethodA, (JNIEnv * env, jobject obj, jclass clazz, jmethodID methodID, jvalue* args)); MOCK_METHOD(void *, GetPrimitiveArrayCritical, (JNIEnv * env, jarray array, jboolean *isCopy)); MOCK_METHOD(void, ReleasePrimitiveArrayCritical, (JNIEnv * env, jarray array, void *carray, jint mode)); }; diff --git a/src/main/java/org/opensearch/knn/index/KNNSettings.java b/src/main/java/org/opensearch/knn/index/KNNSettings.java index fb9c26fef..6eff61d4b 100644 --- a/src/main/java/org/opensearch/knn/index/KNNSettings.java +++ b/src/main/java/org/opensearch/knn/index/KNNSettings.java @@ -562,8 +562,10 @@ public static boolean isFaissAVX2Disabled() { public static boolean isFaissAVX512Disabled() { return Booleans.parseBoolean( - KNNSettings.state().getSettingValue(KNNSettings.KNN_FAISS_AVX512_DISABLED).toString(), - KNN_DEFAULT_FAISS_AVX512_DISABLED_VALUE + Objects.requireNonNullElse( + KNNSettings.state().getSettingValue(KNNSettings.KNN_FAISS_AVX512_DISABLED), + KNN_DEFAULT_FAISS_AVX512_DISABLED_VALUE + ).toString() ); } diff --git a/src/main/java/org/opensearch/knn/index/memory/NativeMemoryAllocation.java b/src/main/java/org/opensearch/knn/index/memory/NativeMemoryAllocation.java index 8adf35447..360c827f9 100644 --- a/src/main/java/org/opensearch/knn/index/memory/NativeMemoryAllocation.java +++ b/src/main/java/org/opensearch/knn/index/memory/NativeMemoryAllocation.java @@ -187,8 +187,8 @@ class IndexAllocation implements NativeMemoryAllocation { protected void closeInternal() { Runnable onClose = () -> { + writeLock(); try { - writeLock(); cleanup(); } finally { writeUnlock(); @@ -328,8 +328,8 @@ public TrainingDataAllocation(ExecutorService executor, long memoryAddress, int @Override public void close() { executor.execute(() -> { + writeLock(); try { - writeLock(); cleanup(); } finally { writeUnlock(); diff --git a/src/main/java/org/opensearch/knn/index/memory/NativeMemoryLoadStrategy.java b/src/main/java/org/opensearch/knn/index/memory/NativeMemoryLoadStrategy.java index 51158d00c..5daa1e047 100644 --- a/src/main/java/org/opensearch/knn/index/memory/NativeMemoryLoadStrategy.java +++ b/src/main/java/org/opensearch/knn/index/memory/NativeMemoryLoadStrategy.java @@ -91,36 +91,11 @@ public void onFileDeleted(Path indexFilePath) { }; } - private NativeMemoryAllocation.IndexAllocation loadWithAbsoluteIndexPath( - NativeMemoryEntryContext.IndexEntryContext indexEntryContext - ) throws IOException { - Path indexPath = Paths.get(indexEntryContext.getKey()); - FileWatcher fileWatcher = new FileWatcher(indexPath); - fileWatcher.addListener(indexFileOnDeleteListener); - fileWatcher.init(); - - KNNEngine knnEngine = KNNEngine.getEngineNameFromPath(indexPath.toString()); - long indexAddress = JNIService.loadIndex(indexPath.toString(), indexEntryContext.getParameters(), knnEngine); - return createIndexAllocation( - indexEntryContext, - knnEngine, - indexAddress, - fileWatcher, - indexEntryContext.calculateSizeInKB(), - indexPath - ); - } - @Override public NativeMemoryAllocation.IndexAllocation load(NativeMemoryEntryContext.IndexEntryContext indexEntryContext) throws IOException { final Path absoluteIndexPath = Paths.get(indexEntryContext.getKey()); final KNNEngine knnEngine = KNNEngine.getEngineNameFromPath(absoluteIndexPath.toString()); - if (knnEngine != KNNEngine.FAISS) { - // We will support other non-FAISS native engines (ex: NMSLIB) soon. - return loadWithAbsoluteIndexPath(indexEntryContext); - } - final FileWatcher fileWatcher = new FileWatcher(absoluteIndexPath); fileWatcher.addListener(indexFileOnDeleteListener); fileWatcher.init(); @@ -182,7 +157,7 @@ class TrainingLoadStrategy NativeMemoryLoadStrategy, Closeable { - private static TrainingLoadStrategy INSTANCE; + private static volatile TrainingLoadStrategy INSTANCE; private final ExecutorService executor; private VectorReader vectorReader; diff --git a/src/main/java/org/opensearch/knn/index/store/IndexInputWithBuffer.java b/src/main/java/org/opensearch/knn/index/store/IndexInputWithBuffer.java index 273a4deac..0b1c934e5 100644 --- a/src/main/java/org/opensearch/knn/index/store/IndexInputWithBuffer.java +++ b/src/main/java/org/opensearch/knn/index/store/IndexInputWithBuffer.java @@ -18,11 +18,13 @@ */ public class IndexInputWithBuffer { private IndexInput indexInput; - // 4K buffer. - private byte[] buffer = new byte[4 * 1024]; + private long contentLength; + // 64K buffer. + private byte[] buffer = new byte[64 * 1024]; public IndexInputWithBuffer(@NonNull IndexInput indexInput) { this.indexInput = indexInput; + this.contentLength = indexInput.length(); } /** @@ -39,6 +41,10 @@ private int copyBytes(long nbytes) throws IOException { return readBytes; } + private long remainingBytes() { + return contentLength - indexInput.getFilePointer(); + } + @Override public String toString() { return "{indexInput=" + indexInput + ", len(buffer)=" + buffer.length + "}"; diff --git a/src/main/java/org/opensearch/knn/jni/JNIService.java b/src/main/java/org/opensearch/knn/jni/JNIService.java index 448241f9c..a0daf65a7 100644 --- a/src/main/java/org/opensearch/knn/jni/JNIService.java +++ b/src/main/java/org/opensearch/knn/jni/JNIService.java @@ -227,6 +227,8 @@ public static long loadIndex(IndexInputWithBuffer readStream, Map parameters); + /** + * Load an index into memory through the provided read stream wrapping Lucene's IndexInput. + * + * @param readStream Read stream wrapping Lucene's IndexInput. + * @param parameters Parameters to be used when loading index + * @return Pointer to location in memory the index resides in + */ + public static native long loadIndexWithStream(IndexInputWithBuffer readStream, Map parameters); + /** * Query an index * diff --git a/src/test/java/org/opensearch/knn/index/KNNCircuitBreakerIT.java b/src/test/java/org/opensearch/knn/index/KNNCircuitBreakerIT.java index 02ebdf2e0..11bde224a 100644 --- a/src/test/java/org/opensearch/knn/index/KNNCircuitBreakerIT.java +++ b/src/test/java/org/opensearch/knn/index/KNNCircuitBreakerIT.java @@ -48,7 +48,7 @@ private void tripCb() throws Exception { createKnnIndex(indexName2, settings, createKnnIndexMapping(FIELD_NAME, 2)); Float[] vector = { 1.3f, 2.2f }; - int docsInIndex = 5; // through testing, 7 is minimum number of docs to trip circuit breaker at 1kb + int docsInIndex = 7; // through testing, 7 is minimum number of docs to trip circuit breaker at 1kb for (int i = 0; i < docsInIndex; i++) { addKnnDoc(indexName1, Integer.toString(i), FIELD_NAME, vector); diff --git a/src/test/java/org/opensearch/knn/jni/JNIServiceTests.java b/src/test/java/org/opensearch/knn/jni/JNIServiceTests.java index 8566b0223..f6d118092 100644 --- a/src/test/java/org/opensearch/knn/jni/JNIServiceTests.java +++ b/src/test/java/org/opensearch/knn/jni/JNIServiceTests.java @@ -749,6 +749,31 @@ public void testLoadIndex_nmslib_valid() throws IOException { assertNotEquals(0, pointer); } + public void testLoadIndex_nmslib_valid_with_stream() throws IOException { + Path tmpFile = createTempFile(); + + TestUtils.createIndex( + testData.indexData.docs, + testData.loadDataToMemoryAddress(), + testData.indexData.getDimension(), + tmpFile.toAbsolutePath().toString(), + ImmutableMap.of(KNNConstants.SPACE_TYPE, SpaceType.L2.getValue()), + KNNEngine.NMSLIB + ); + assertTrue(tmpFile.toFile().length() > 0); + + try (final Directory directory = new MMapDirectory(tmpFile.getParent())) { + try (IndexInput indexInput = directory.openInput(tmpFile.getFileName().toString(), IOContext.READONCE)) { + long pointer = JNIService.loadIndex( + new IndexInputWithBuffer(indexInput), + ImmutableMap.of(KNNConstants.SPACE_TYPE, SpaceType.L2.getValue()), + KNNEngine.NMSLIB + ); + assertNotEquals(0, pointer); + } + } + } + public void testLoadIndex_faiss_invalid_fileDoesNotExist() { expectThrows(Exception.class, () -> JNIService.loadIndex("invalid", Collections.emptyMap(), KNNEngine.FAISS)); } From 8e07928972cf60d2563c591a78b8c01271ad43ba Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Mon, 21 Oct 2024 19:18:36 -0500 Subject: [PATCH 403/416] Bump Faiss commit from 4eecd916 to 1f42e815 (#2224) (#2226) Signed-off-by: Naveen Tatikonda (cherry picked from commit d52ee1448742f1480c4794313cd163ac74488e0d) Co-authored-by: Naveen Tatikonda --- jni/external/faiss | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jni/external/faiss b/jni/external/faiss index 4eecd9165..1f42e815d 160000 --- a/jni/external/faiss +++ b/jni/external/faiss @@ -1 +1 @@ -Subproject commit 4eecd9165ae56bc8ff4afbf14cc8b359fdfe4004 +Subproject commit 1f42e815db7754297e3b4467763352b829b6cde0 From ee00593415bc3e50656aa300c8aa432ce6667c29 Mon Sep 17 00:00:00 2001 From: Doo Yong Kim <0ctopus13prime@gmail.com> Date: Mon, 21 Oct 2024 21:38:48 -0700 Subject: [PATCH 404/416] Remove FileWatcher from KNN (#2182) (#2225) Signed-off-by: Dooyong Kim (cherry picked from commit e5599aa151cf43700d0ea327ad536cc106e5bb61) --- CHANGELOG.md | 1 + .../opensearch/knn/index/KNNIndexShard.java | 51 +++++---- .../KNN80Codec/KNN80DocValuesProducer.java | 105 +++++------------- .../NativeEngines990KnnVectorsReader.java | 40 ++++++- .../knn/index/codec/util/KNNCodecUtil.java | 45 ++++++++ .../util/NativeMemoryCacheKeyHelper.java | 45 ++++++++ .../index/memory/NativeMemoryAllocation.java | 56 ++++------ .../memory/NativeMemoryCacheManager.java | 4 +- .../memory/NativeMemoryEntryContext.java | 35 +++--- .../memory/NativeMemoryLoadStrategy.java | 65 ++++------- .../opensearch/knn/index/query/KNNWeight.java | 12 +- .../org/opensearch/knn/jni/JNIService.java | 1 - .../org/opensearch/knn/plugin/KNNPlugin.java | 1 - .../knn/index/KNNIndexShardTests.java | 46 ++++++-- .../KNN80DocValuesProducerTests.java | 10 +- ...NativeEngines990KnnVectorsFormatTests.java | 2 + .../knn/index/codec/KNNCodecTestCase.java | 9 -- .../memory/NativeMemoryAllocationTests.java | 46 ++------ .../memory/NativeMemoryCacheManagerTests.java | 27 ++--- .../memory/NativeMemoryEntryContextTests.java | 38 +++---- .../memory/NativeMemoryLoadStrategyTests.java | 16 +-- .../java/org/opensearch/knn/TestUtils.java | 5 + 22 files changed, 335 insertions(+), 325 deletions(-) create mode 100644 src/main/java/org/opensearch/knn/index/codec/util/NativeMemoryCacheKeyHelper.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 141d01c7d..03d6d43a9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), * Add support to build vector data structures greedily and perform exact search when there are no engine files [#1942](https://github.com/opensearch-project/k-NN/issues/1942) * Add CompressionLevel Calculation for PQ [#2200](https://github.com/opensearch-project/k-NN/pull/2200) * Introduce a loading layer in native engine [#2185](https://github.com/opensearch-project/k-NN/pull/2185) +* Remove FSDirectory dependency from native engine constructing side and deprecated FileWatcher [#2182](https://github.com/opensearch-project/k-NN/pull/2182) ### Bug Fixes * Add DocValuesProducers for releasing memory when close index [#1946](https://github.com/opensearch-project/k-NN/pull/1946) * KNN80DocValues should only be considered for BinaryDocValues fields [#2147](https://github.com/opensearch-project/k-NN/pull/2147) diff --git a/src/main/java/org/opensearch/knn/index/KNNIndexShard.java b/src/main/java/org/opensearch/knn/index/KNNIndexShard.java index a4d31c546..ac4c055b0 100644 --- a/src/main/java/org/opensearch/knn/index/KNNIndexShard.java +++ b/src/main/java/org/opensearch/knn/index/KNNIndexShard.java @@ -12,14 +12,15 @@ import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.SegmentCommitInfo; +import org.apache.lucene.index.SegmentInfo; import org.apache.lucene.index.SegmentReader; import org.apache.lucene.store.Directory; -import org.apache.lucene.store.FSDirectory; -import org.apache.lucene.store.FilterDirectory; import org.opensearch.common.lucene.Lucene; import org.opensearch.index.engine.Engine; import org.opensearch.index.shard.IndexShard; import org.opensearch.knn.common.FieldInfoExtractor; +import org.opensearch.knn.index.codec.util.NativeMemoryCacheKeyHelper; import org.opensearch.knn.index.engine.qframe.QuantizationConfig; import org.opensearch.knn.index.mapper.KNNVectorFieldMapper; import org.opensearch.knn.index.memory.NativeMemoryAllocation; @@ -29,9 +30,7 @@ import org.opensearch.knn.index.engine.KNNEngine; import java.io.IOException; -import java.nio.file.Path; import java.util.ArrayList; -import java.util.Collection; import java.util.List; import java.util.Optional; import java.util.concurrent.ExecutionException; @@ -94,14 +93,18 @@ public void warmup() throws IOException { try (Engine.Searcher searcher = indexShard.acquireSearcher("knn-warmup")) { getAllEngineFileContexts(searcher.getIndexReader()).forEach((engineFileContext) -> { try { + final String cacheKey = NativeMemoryCacheKeyHelper.constructCacheKey( + engineFileContext.vectorFileName, + engineFileContext.segmentInfo + ); nativeMemoryCacheManager.get( new NativeMemoryEntryContext.IndexEntryContext( directory, - engineFileContext.getIndexPath(), + cacheKey, NativeMemoryLoadStrategy.IndexLoadStrategy.getInstance(), getParametersAtLoading( engineFileContext.getSpaceType(), - KNNEngine.getEngineNameFromPath(engineFileContext.getIndexPath()), + KNNEngine.getEngineNameFromPath(engineFileContext.getVectorFileName()), getIndexName(), engineFileContext.getVectorDataType() ), @@ -133,9 +136,13 @@ public void clearCache() { indexAllocation.writeLock(); log.info("[KNN] Evicting index from cache: [{}]", indexName); try (Engine.Searcher searcher = indexShard.acquireSearcher(INDEX_SHARD_CLEAR_CACHE_SEARCHER)) { - getAllEngineFileContexts(searcher.getIndexReader()).forEach( - (engineFileContext) -> nativeMemoryCacheManager.invalidate(engineFileContext.getIndexPath()) - ); + getAllEngineFileContexts(searcher.getIndexReader()).forEach((engineFileContext) -> { + final String cacheKey = NativeMemoryCacheKeyHelper.constructCacheKey( + engineFileContext.vectorFileName, + engineFileContext.segmentInfo + ); + nativeMemoryCacheManager.invalidate(cacheKey); + }); } catch (IOException ex) { log.error("[KNN] Failed to evict index from cache: [{}]", indexName, ex); throw new RuntimeException(ex); @@ -166,7 +173,6 @@ List getEngineFileContexts(IndexReader indexReader, KNNEngine for (LeafReaderContext leafReaderContext : indexReader.leaves()) { SegmentReader reader = Lucene.segmentReader(leafReaderContext.reader()); - Path shardPath = ((FSDirectory) FilterDirectory.unwrap(reader.directory())).getDirectory(); String fileExtension = reader.getSegmentInfo().info.getUseCompoundFile() ? knnEngine.getCompoundExtension() : knnEngine.getExtension(); @@ -180,11 +186,9 @@ List getEngineFileContexts(IndexReader indexReader, KNNEngine String modelId = fieldInfo.attributes().getOrDefault(MODEL_ID, null); engineFiles.addAll( getEngineFileContexts( - reader.getSegmentInfo().files(), - reader.getSegmentInfo().info.name, + reader.getSegmentInfo(), fieldInfo.name, fileExtension, - shardPath, spaceType, modelId, FieldInfoExtractor.extractQuantizationConfig(fieldInfo) == QuantizationConfig.EMPTY @@ -202,22 +206,22 @@ List getEngineFileContexts(IndexReader indexReader, KNNEngine @VisibleForTesting List getEngineFileContexts( - Collection files, - String segmentName, + SegmentCommitInfo segmentCommitInfo, String fieldName, String fileExtension, - Path shardPath, SpaceType spaceType, String modelId, VectorDataType vectorDataType - ) { - String prefix = buildEngineFilePrefix(segmentName); - String suffix = buildEngineFileSuffix(fieldName, fileExtension); - return files.stream() + ) throws IOException { + // Ex: 0_ + final String prefix = buildEngineFilePrefix(segmentCommitInfo.info.name); + // Ex: _my_field.faiss + final String suffix = buildEngineFileSuffix(fieldName, fileExtension); + return segmentCommitInfo.files() + .stream() .filter(fileName -> fileName.startsWith(prefix)) .filter(fileName -> fileName.endsWith(suffix)) - .map(fileName -> shardPath.resolve(fileName).toString()) - .map(fileName -> new EngineFileContext(spaceType, modelId, fileName, vectorDataType)) + .map(vectorFileName -> new EngineFileContext(spaceType, modelId, vectorFileName, vectorDataType, segmentCommitInfo.info)) .collect(Collectors.toList()); } @@ -227,7 +231,8 @@ List getEngineFileContexts( static class EngineFileContext { private final SpaceType spaceType; private final String modelId; - private final String indexPath; + private final String vectorFileName; private final VectorDataType vectorDataType; + private final SegmentInfo segmentInfo; } } diff --git a/src/main/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesProducer.java b/src/main/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesProducer.java index b78566f2e..23c9f3105 100644 --- a/src/main/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesProducer.java +++ b/src/main/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesProducer.java @@ -11,82 +11,34 @@ package org.opensearch.knn.index.codec.KNN80Codec; -import lombok.NonNull; import lombok.extern.log4j.Log4j2; import org.apache.lucene.codecs.DocValuesProducer; import org.apache.lucene.index.BinaryDocValues; import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.FieldInfo; + +import java.io.IOException; + import org.apache.lucene.index.NumericDocValues; import org.apache.lucene.index.SegmentReadState; import org.apache.lucene.index.SortedDocValues; import org.apache.lucene.index.SortedNumericDocValues; import org.apache.lucene.index.SortedSetDocValues; -import org.apache.lucene.store.Directory; -import org.apache.lucene.store.FSDirectory; -import org.apache.lucene.store.FilterDirectory; -import org.opensearch.common.io.PathUtils; -import org.opensearch.knn.common.FieldInfoExtractor; import org.opensearch.knn.index.codec.util.KNNCodecUtil; -import org.opensearch.knn.index.engine.KNNEngine; +import org.opensearch.knn.index.codec.util.NativeMemoryCacheKeyHelper; import org.opensearch.knn.index.memory.NativeMemoryCacheManager; -import java.io.IOException; -import java.nio.file.Path; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; -import java.util.Map; - -import static org.opensearch.knn.common.KNNConstants.MODEL_ID; -import static org.opensearch.knn.index.mapper.KNNVectorFieldMapper.KNN_FIELD; @Log4j2 public class KNN80DocValuesProducer extends DocValuesProducer { - - private final SegmentReadState state; private final DocValuesProducer delegate; - private final NativeMemoryCacheManager nativeMemoryCacheManager; - private final Map indexPathMap = new HashMap(); + private List cacheKeys; public KNN80DocValuesProducer(DocValuesProducer delegate, SegmentReadState state) { this.delegate = delegate; - this.state = state; - this.nativeMemoryCacheManager = NativeMemoryCacheManager.getInstance(); - - Directory directory = state.directory; - // directory would be CompoundDirectory, we need get directory firstly and then unwrap - if (state.directory instanceof KNN80CompoundDirectory) { - directory = ((KNN80CompoundDirectory) state.directory).getDir(); - } - - Directory dir = FilterDirectory.unwrap(directory); - if (!(dir instanceof FSDirectory)) { - log.warn("{} can not casting to FSDirectory", directory); - return; - } - String directoryPath = ((FSDirectory) dir).getDirectory().toString(); - for (FieldInfo field : state.fieldInfos) { - if (!field.attributes().containsKey(KNN_FIELD)) { - continue; - } - // Only segments that contains BinaryDocValues and doesn't have vector values should be considered. - // By default, we don't create BinaryDocValues for knn field anymore. However, users can set doc_values = true - // to create binary doc values explicitly like any other field. Hence, we only want to include fields - // where approximate search is possible only by BinaryDocValues. - if (field.getDocValuesType() != DocValuesType.BINARY || field.hasVectorValues() == true) { - continue; - } - // Only Native Engine put into indexPathMap - KNNEngine knnEngine = getNativeKNNEngine(field); - if (knnEngine == null) { - continue; - } - List engineFiles = KNNCodecUtil.getEngineFiles(knnEngine.getExtension(), field.name, state.segmentInfo); - Path indexPath = PathUtils.get(directoryPath, engineFiles.get(0)); - indexPathMap.putIfAbsent(field.getName(), indexPath.toString()); - - } + this.cacheKeys = getVectorCacheKeysFromSegmentReaderState(state); } @Override @@ -121,32 +73,35 @@ public void checkIntegrity() throws IOException { @Override public void close() throws IOException { - for (String path : indexPathMap.values()) { - nativeMemoryCacheManager.invalidate(path); - } + final NativeMemoryCacheManager nativeMemoryCacheManager = NativeMemoryCacheManager.getInstance(); + cacheKeys.forEach(nativeMemoryCacheManager::invalidate); delegate.close(); } - public final List getOpenedIndexPath() { - return new ArrayList<>(indexPathMap.values()); + public final List getCacheKeys() { + return new ArrayList<>(cacheKeys); } - /** - * Get KNNEngine From FieldInfo - * - * @param field which field we need produce from engine - * @return if and only if Native Engine we return specific engine, else return null - */ - private KNNEngine getNativeKNNEngine(@NonNull FieldInfo field) { - - final String modelId = field.attributes().get(MODEL_ID); - if (modelId != null) { - return null; - } - KNNEngine engine = FieldInfoExtractor.extractKNNEngine(field); - if (KNNEngine.getEnginesThatCreateCustomSegmentFiles().contains(engine)) { - return engine; + private static List getVectorCacheKeysFromSegmentReaderState(SegmentReadState segmentReadState) { + final List cacheKeys = new ArrayList<>(); + + for (FieldInfo field : segmentReadState.fieldInfos) { + // Only segments that contains BinaryDocValues and doesn't have vector values should be considered. + // By default, we don't create BinaryDocValues for knn field anymore. However, users can set doc_values = true + // to create binary doc values explicitly like any other field. Hence, we only want to include fields + // where approximate search is possible only by BinaryDocValues. + if (field.getDocValuesType() != DocValuesType.BINARY || field.hasVectorValues()) { + continue; + } + + final String vectorIndexFileName = KNNCodecUtil.getNativeEngineFileFromFieldInfo(field, segmentReadState.segmentInfo); + if (vectorIndexFileName == null) { + continue; + } + final String cacheKey = NativeMemoryCacheKeyHelper.constructCacheKey(vectorIndexFileName, segmentReadState.segmentInfo); + cacheKeys.add(cacheKey); } - return null; + + return cacheKeys; } } diff --git a/src/main/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsReader.java b/src/main/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsReader.java index 16631fd97..efabc3a70 100644 --- a/src/main/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsReader.java +++ b/src/main/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsReader.java @@ -24,13 +24,18 @@ import org.apache.lucene.util.Bits; import org.apache.lucene.util.IOUtils; import org.opensearch.common.UUIDs; +import org.opensearch.knn.index.codec.util.KNNCodecUtil; +import org.opensearch.knn.index.codec.util.NativeMemoryCacheKeyHelper; +import org.opensearch.knn.index.memory.NativeMemoryCacheManager; import org.opensearch.knn.index.quantizationservice.QuantizationService; import org.opensearch.knn.quantization.models.quantizationState.QuantizationState; import org.opensearch.knn.quantization.models.quantizationState.QuantizationStateCacheManager; import org.opensearch.knn.quantization.models.quantizationState.QuantizationStateReadConfig; import java.io.IOException; +import java.util.ArrayList; import java.util.HashMap; +import java.util.List; import java.util.Map; /** @@ -40,12 +45,14 @@ public class NativeEngines990KnnVectorsReader extends KnnVectorsReader { private final FlatVectorsReader flatVectorsReader; - private final SegmentReadState segmentReadState; private Map quantizationStateCacheKeyPerField; + private SegmentReadState segmentReadState; + private final List cacheKeys; - public NativeEngines990KnnVectorsReader(final SegmentReadState state, final FlatVectorsReader flatVectorsReader) throws IOException { - this.segmentReadState = state; + public NativeEngines990KnnVectorsReader(final SegmentReadState state, final FlatVectorsReader flatVectorsReader) { this.flatVectorsReader = flatVectorsReader; + this.segmentReadState = state; + this.cacheKeys = getVectorCacheKeysFromSegmentReaderState(state); loadCacheKeyMap(); } @@ -176,10 +183,18 @@ public void search(String field, byte[] target, KnnCollector knnCollector, Bits */ @Override public void close() throws IOException { + // Clean up allocated vector indices resources from cache. + final NativeMemoryCacheManager nativeMemoryCacheManager = NativeMemoryCacheManager.getInstance(); + cacheKeys.forEach(nativeMemoryCacheManager::invalidate); + + // Close a reader. IOUtils.close(flatVectorsReader); + + // Clean up quantized state cache. if (quantizationStateCacheKeyPerField != null) { + final QuantizationStateCacheManager quantizationStateCacheManager = QuantizationStateCacheManager.getInstance(); for (String cacheKey : quantizationStateCacheKeyPerField.values()) { - QuantizationStateCacheManager.getInstance().evict(cacheKey); + quantizationStateCacheManager.evict(cacheKey); } } } @@ -192,11 +207,26 @@ public long ramBytesUsed() { return flatVectorsReader.ramBytesUsed(); } - private void loadCacheKeyMap() throws IOException { + private void loadCacheKeyMap() { quantizationStateCacheKeyPerField = new HashMap<>(); for (FieldInfo fieldInfo : segmentReadState.fieldInfos) { String cacheKey = UUIDs.base64UUID(); quantizationStateCacheKeyPerField.put(fieldInfo.getName(), cacheKey); } } + + private static List getVectorCacheKeysFromSegmentReaderState(SegmentReadState segmentReadState) { + final List cacheKeys = new ArrayList<>(); + + for (FieldInfo field : segmentReadState.fieldInfos) { + final String vectorIndexFileName = KNNCodecUtil.getNativeEngineFileFromFieldInfo(field, segmentReadState.segmentInfo); + if (vectorIndexFileName == null) { + continue; + } + final String cacheKey = NativeMemoryCacheKeyHelper.constructCacheKey(vectorIndexFileName, segmentReadState.segmentInfo); + cacheKeys.add(cacheKey); + } + + return cacheKeys; + } } diff --git a/src/main/java/org/opensearch/knn/index/codec/util/KNNCodecUtil.java b/src/main/java/org/opensearch/knn/index/codec/util/KNNCodecUtil.java index 84c7c4675..3ccfc3c2b 100644 --- a/src/main/java/org/opensearch/knn/index/codec/util/KNNCodecUtil.java +++ b/src/main/java/org/opensearch/knn/index/codec/util/KNNCodecUtil.java @@ -5,8 +5,11 @@ package org.opensearch.knn.index.codec.util; +import lombok.NonNull; import org.apache.lucene.index.BinaryDocValues; +import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.SegmentInfo; +import org.opensearch.knn.common.FieldInfoExtractor; import org.opensearch.knn.common.KNNConstants; import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.codec.KNN80Codec.KNN80BinaryDocValues; @@ -16,6 +19,8 @@ import java.util.List; import java.util.stream.Collectors; +import static org.opensearch.knn.index.mapper.KNNVectorFieldMapper.KNN_FIELD; + public class KNNCodecUtil { // Floats are 4 bytes in size public static final int FLOAT_BYTE_SIZE = 4; @@ -84,4 +89,44 @@ public static List getEngineFiles(String extension, String fieldName, Se .collect(Collectors.toList()); return engineFiles; } + + /** + * Get engine file name from given field and segment info. + * Ex: _0_165_my_field.faiss + * + * @param field : Field info that might have a vector index file. Not always it has it. + * @param segmentInfo : Segment where we are collecting an engine file list. + * @return : Found vector engine names, if not found, returns null. + */ + public static String getNativeEngineFileFromFieldInfo(FieldInfo field, SegmentInfo segmentInfo) { + if (!field.attributes().containsKey(KNN_FIELD)) { + return null; + } + // Only Native Engine put into indexPathMap + final KNNEngine knnEngine = getNativeKNNEngine(field); + if (knnEngine == null) { + return null; + } + final List engineFiles = KNNCodecUtil.getEngineFiles(knnEngine.getExtension(), field.name, segmentInfo); + if (engineFiles.isEmpty()) { + return null; + } else { + final String vectorIndexFileName = engineFiles.get(0); + return vectorIndexFileName; + } + } + + /** + * Get KNNEngine From FieldInfo + * + * @param field which field we need produce from engine + * @return if and only if Native Engine we return specific engine, else return null + */ + private static KNNEngine getNativeKNNEngine(@NonNull FieldInfo field) { + final KNNEngine engine = FieldInfoExtractor.extractKNNEngine(field); + if (KNNEngine.getEnginesThatCreateCustomSegmentFiles().contains(engine)) { + return engine; + } + return null; + } } diff --git a/src/main/java/org/opensearch/knn/index/codec/util/NativeMemoryCacheKeyHelper.java b/src/main/java/org/opensearch/knn/index/codec/util/NativeMemoryCacheKeyHelper.java new file mode 100644 index 000000000..8d50bf029 --- /dev/null +++ b/src/main/java/org/opensearch/knn/index/codec/util/NativeMemoryCacheKeyHelper.java @@ -0,0 +1,45 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.codec.util; + +import org.apache.lucene.index.SegmentInfo; + +import java.util.Base64; + +public final class NativeMemoryCacheKeyHelper { + private NativeMemoryCacheKeyHelper() {} + + /** + * Construct a unique cache key for look-up operation in {@link org.opensearch.knn.index.memory.NativeMemoryCacheManager} + * + * @param vectorIndexFileName Vector index file name. Ex: _0_165_test_field.faiss. + * @param segmentInfo Segment info object representing a logical segment unit containing a vector index. + * @return Unique cache key that can be used for look-up and invalidating in + * {@link org.opensearch.knn.index.memory.NativeMemoryCacheManager} + */ + public static String constructCacheKey(final String vectorIndexFileName, final SegmentInfo segmentInfo) { + final String segmentId = Base64.getEncoder().encodeToString(segmentInfo.getId()); + final String cacheKey = vectorIndexFileName + "@" + segmentId; + return cacheKey; + } + + /** + * From cacheKey, we extract a vector file name. + * Note that expected format of cacheKey consists of two part with '@' as a delimiter. + * First part would be the vector file name, the second one is the segment id. + * + * @param cacheKey : Cache key for {@link org.opensearch.knn.index.memory.NativeMemoryCacheManager} + * @return : Vector file name, if the given cacheKey was invalid format, returns null. + */ + public static String extractVectorIndexFileName(final String cacheKey) { + final int indexOfDelimiter = cacheKey.indexOf('@'); + if (indexOfDelimiter != -1) { + final String vectorFileName = cacheKey.substring(0, indexOfDelimiter); + return vectorFileName; + } + return null; + } +} diff --git a/src/main/java/org/opensearch/knn/index/memory/NativeMemoryAllocation.java b/src/main/java/org/opensearch/knn/index/memory/NativeMemoryAllocation.java index 360c827f9..9ac3caa23 100644 --- a/src/main/java/org/opensearch/knn/index/memory/NativeMemoryAllocation.java +++ b/src/main/java/org/opensearch/knn/index/memory/NativeMemoryAllocation.java @@ -21,8 +21,6 @@ import org.opensearch.knn.index.query.KNNWeight; import org.opensearch.knn.jni.JNIService; import org.opensearch.knn.index.engine.KNNEngine; -import org.opensearch.watcher.FileWatcher; -import org.opensearch.watcher.WatcherHandle; import java.util.concurrent.ExecutorService; import java.util.concurrent.Semaphore; @@ -115,11 +113,10 @@ class IndexAllocation implements NativeMemoryAllocation { @Getter private final KNNEngine knnEngine; @Getter - private final String indexPath; + private final String vectorFileName; @Getter private final String openSearchIndexName; private final ReadWriteLock readWriteLock; - private final WatcherHandle watcherHandle; private final SharedIndexState sharedIndexState; @Getter private final boolean isBinaryIndex; @@ -132,20 +129,18 @@ class IndexAllocation implements NativeMemoryAllocation { * @param memoryAddress Pointer in memory to the index * @param sizeKb Size this index consumes in kilobytes * @param knnEngine KNNEngine associated with the index allocation - * @param indexPath File path to index + * @param vectorFileName Vector file name. Ex: _0_165_my_field.faiss * @param openSearchIndexName Name of OpenSearch index this index is associated with - * @param watcherHandle Handle for watching index file */ IndexAllocation( ExecutorService executorService, long memoryAddress, int sizeKb, KNNEngine knnEngine, - String indexPath, - String openSearchIndexName, - WatcherHandle watcherHandle + String vectorFileName, + String openSearchIndexName ) { - this(executorService, memoryAddress, sizeKb, knnEngine, indexPath, openSearchIndexName, watcherHandle, null, false); + this(executorService, memoryAddress, sizeKb, knnEngine, vectorFileName, openSearchIndexName, null, false); } /** @@ -155,9 +150,8 @@ class IndexAllocation implements NativeMemoryAllocation { * @param memoryAddress Pointer in memory to the index * @param sizeKb Size this index consumes in kilobytes * @param knnEngine KNNEngine associated with the index allocation - * @param indexPath File path to index + * @param vectorFileName Vector file name. Ex: _0_165_my_field.faiss * @param openSearchIndexName Name of OpenSearch index this index is associated with - * @param watcherHandle Handle for watching index file * @param sharedIndexState Shared index state. If not shared state present, pass null. */ IndexAllocation( @@ -165,21 +159,19 @@ class IndexAllocation implements NativeMemoryAllocation { long memoryAddress, int sizeKb, KNNEngine knnEngine, - String indexPath, + String vectorFileName, String openSearchIndexName, - WatcherHandle watcherHandle, SharedIndexState sharedIndexState, boolean isBinaryIndex ) { this.executor = executorService; this.closed = false; this.knnEngine = knnEngine; - this.indexPath = indexPath; + this.vectorFileName = vectorFileName; this.openSearchIndexName = openSearchIndexName; this.memoryAddress = memoryAddress; this.readWriteLock = new ReentrantReadWriteLock(); this.sizeKb = sizeKb; - this.watcherHandle = watcherHandle; this.sharedIndexState = sharedIndexState; this.isBinaryIndex = isBinaryIndex; this.refCounted = new RefCountedReleasable<>("IndexAllocation-Reference", this, this::closeInternal); @@ -218,8 +210,6 @@ private void cleanup() { this.closed = true; - watcherHandle.stop(); - // memoryAddress is sometimes initialized to 0. If this is ever the case, freeing will surely fail. if (memoryAddress != 0) { JNIService.free(memoryAddress, knnEngine, isBinaryIndex); @@ -294,30 +284,31 @@ class TrainingDataAllocation implements NativeMemoryAllocation { private final ExecutorService executor; private volatile boolean closed; + @Setter private long memoryAddress; - private final int size; + private final int sizeKb; @Getter @Setter private QuantizationConfig quantizationConfig = QuantizationConfig.EMPTY; // Implement reader/writer with semaphores to deal with passing lock conditions between threads private int readCount; - private Semaphore readSemaphore; - private Semaphore writeSemaphore; - private VectorDataType vectorDataType; + private final Semaphore readSemaphore; + private final Semaphore writeSemaphore; + private final VectorDataType vectorDataType; /** * Constructor * * @param executor Executor used for allocation close * @param memoryAddress pointer in memory to the training data allocation - * @param size amount memory needed for allocation in kilobytes + * @param sizeKb amount memory needed for allocation in kilobytes */ - public TrainingDataAllocation(ExecutorService executor, long memoryAddress, int size, VectorDataType vectorDataType) { + public TrainingDataAllocation(ExecutorService executor, long memoryAddress, int sizeKb, VectorDataType vectorDataType) { this.executor = executor; this.closed = false; this.memoryAddress = memoryAddress; - this.size = size; + this.sizeKb = sizeKb; this.readCount = 0; this.readSemaphore = new Semaphore(1); @@ -401,7 +392,7 @@ public void readLock() { /** * A write lock will be obtained either on eviction from {@link NativeMemoryCacheManager NativeMemoryManager's} * or when training data is actually being loaded. A semaphore is used because collecting training data - * happens asynchrously, so the thread that obtains the lock will not be the same thread that releases the + * happens asynchronously, so the thread that obtains the lock will not be the same thread that releases the * lock. */ @Override @@ -438,23 +429,14 @@ public void writeUnlock() { @Override public int getSizeInKB() { - return size; - } - - /** - * Setter for memory address to training data - * - * @param memoryAddress Pointer to training data - */ - public void setMemoryAddress(long memoryAddress) { - this.memoryAddress = memoryAddress; + return sizeKb; } } /** * An anonymous allocation is used to reserve space in the native memory cache. It does not have a * memory address. This allocation type should be used when a function allocates a large portion of memory in the - * function, runs for awhile, and then frees it. + * function, runs for a while, and then frees it. */ class AnonymousAllocation implements NativeMemoryAllocation { diff --git a/src/main/java/org/opensearch/knn/index/memory/NativeMemoryCacheManager.java b/src/main/java/org/opensearch/knn/index/memory/NativeMemoryCacheManager.java index 649fb9774..b8aecc5a5 100644 --- a/src/main/java/org/opensearch/knn/index/memory/NativeMemoryCacheManager.java +++ b/src/main/java/org/opensearch/knn/index/memory/NativeMemoryCacheManager.java @@ -291,8 +291,8 @@ public CacheStats getCacheStats() { public NativeMemoryAllocation get(NativeMemoryEntryContext nativeMemoryEntryContext, boolean isAbleToTriggerEviction) throws ExecutionException { if (!isAbleToTriggerEviction - && !cache.asMap().containsKey(nativeMemoryEntryContext.getKey()) - && maxWeight - getCacheSizeInKilobytes() - nativeMemoryEntryContext.calculateSizeInKB() <= 0) { + && (maxWeight - getCacheSizeInKilobytes() - nativeMemoryEntryContext.calculateSizeInKB()) <= 0 + && !cache.asMap().containsKey(nativeMemoryEntryContext.getKey())) { throw new OutOfNativeMemoryException( "Entry cannot be loaded into cache because it would not fit. " + "Entry size: " diff --git a/src/main/java/org/opensearch/knn/index/memory/NativeMemoryEntryContext.java b/src/main/java/org/opensearch/knn/index/memory/NativeMemoryEntryContext.java index 00bf023f9..0af13fb46 100644 --- a/src/main/java/org/opensearch/knn/index/memory/NativeMemoryEntryContext.java +++ b/src/main/java/org/opensearch/knn/index/memory/NativeMemoryEntryContext.java @@ -15,14 +15,13 @@ import org.apache.lucene.store.Directory; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.Nullable; +import org.opensearch.knn.index.codec.util.NativeMemoryCacheKeyHelper; import org.opensearch.knn.index.engine.qframe.QuantizationConfig; -import org.opensearch.knn.index.util.IndexUtil; import org.opensearch.knn.index.VectorDataType; import java.io.IOException; import java.util.Map; import java.util.UUID; -import java.util.function.Function; /** * Encapsulates all information needed to load a component into native memory. @@ -80,26 +79,26 @@ public static class IndexEntryContext extends NativeMemoryEntryContext parameters, String openSearchIndexName ) { - this(directory, indexPath, indexLoadStrategy, parameters, openSearchIndexName, null); + this(directory, vectorIndexCacheKey, indexLoadStrategy, parameters, openSearchIndexName, null); } /** * Constructor * * @param directory Lucene directory to create required IndexInput/IndexOutput to access files. - * @param indexPath path to index file. Also used as key in cache. + * @param vectorIndexCacheKey Cache key for {@link NativeMemoryCacheManager}. It must contain a vector file name. * @param indexLoadStrategy strategy to load index into memory * @param parameters load time parameters * @param openSearchIndexName opensearch index associated with index @@ -107,13 +106,13 @@ public IndexEntryContext( */ public IndexEntryContext( Directory directory, - String indexPath, + String vectorIndexCacheKey, NativeMemoryLoadStrategy.IndexLoadStrategy indexLoadStrategy, Map parameters, String openSearchIndexName, String modelId ) { - super(indexPath); + super(vectorIndexCacheKey); this.directory = directory; this.indexLoadStrategy = indexLoadStrategy; this.openSearchIndexName = openSearchIndexName; @@ -123,25 +122,19 @@ public IndexEntryContext( @Override public Integer calculateSizeInKB() { - return IndexSizeCalculator.INSTANCE.apply(this); + final String indexFileName = NativeMemoryCacheKeyHelper.extractVectorIndexFileName(key); + try { + final long fileLength = directory.fileLength(indexFileName); + return (int) (fileLength / 1024L); + } catch (IOException e) { + throw new RuntimeException(e); + } } @Override public NativeMemoryAllocation.IndexAllocation load() throws IOException { return indexLoadStrategy.load(this); } - - private static class IndexSizeCalculator implements Function { - - static IndexSizeCalculator INSTANCE = new IndexSizeCalculator(); - - IndexSizeCalculator() {} - - @Override - public Integer apply(IndexEntryContext indexEntryContext) { - return IndexUtil.getFileSizeInKB(indexEntryContext.getKey()); - } - } } public static class TrainingDataEntryContext extends NativeMemoryEntryContext { diff --git a/src/main/java/org/opensearch/knn/index/memory/NativeMemoryLoadStrategy.java b/src/main/java/org/opensearch/knn/index/memory/NativeMemoryLoadStrategy.java index 5daa1e047..8cbdb4fd7 100644 --- a/src/main/java/org/opensearch/knn/index/memory/NativeMemoryLoadStrategy.java +++ b/src/main/java/org/opensearch/knn/index/memory/NativeMemoryLoadStrategy.java @@ -16,6 +16,7 @@ import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexInput; import org.opensearch.core.action.ActionListener; +import org.opensearch.knn.index.codec.util.NativeMemoryCacheKeyHelper; import org.opensearch.knn.index.engine.qframe.QuantizationConfig; import org.opensearch.knn.index.store.IndexInputWithBuffer; import org.opensearch.knn.index.util.IndexUtil; @@ -23,15 +24,9 @@ import org.opensearch.knn.index.engine.KNNEngine; import org.opensearch.knn.training.TrainingDataConsumer; import org.opensearch.knn.training.VectorReader; -import org.opensearch.watcher.FileChangesListener; -import org.opensearch.watcher.FileWatcher; -import org.opensearch.watcher.ResourceWatcherService; -import org.opensearch.watcher.WatcherHandle; import java.io.Closeable; import java.io.IOException; -import java.nio.file.Path; -import java.nio.file.Paths; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -57,8 +52,6 @@ class IndexLoadStrategy private static IndexLoadStrategy INSTANCE; private final ExecutorService executor; - private final FileChangesListener indexFileOnDeleteListener; - private ResourceWatcherService resourceWatcherService; /** * Get Singleton of this load strategy. @@ -72,47 +65,34 @@ public static synchronized IndexLoadStrategy getInstance() { return INSTANCE; } - /** - * Initialize singleton. - * - * @param resourceWatcherService service used to monitor index files for deletion - */ - public static void initialize(final ResourceWatcherService resourceWatcherService) { - getInstance().resourceWatcherService = resourceWatcherService; - } - private IndexLoadStrategy() { executor = Executors.newSingleThreadExecutor(); - indexFileOnDeleteListener = new FileChangesListener() { - @Override - public void onFileDeleted(Path indexFilePath) { - NativeMemoryCacheManager.getInstance().invalidate(indexFilePath.toString()); - } - }; } @Override public NativeMemoryAllocation.IndexAllocation load(NativeMemoryEntryContext.IndexEntryContext indexEntryContext) throws IOException { - final Path absoluteIndexPath = Paths.get(indexEntryContext.getKey()); - final KNNEngine knnEngine = KNNEngine.getEngineNameFromPath(absoluteIndexPath.toString()); - final FileWatcher fileWatcher = new FileWatcher(absoluteIndexPath); - fileWatcher.addListener(indexFileOnDeleteListener); - fileWatcher.init(); + // Extract vector file name from the given cache key. + // Ex: _0_165_my_field.faiss@1vaqiupVUwvkXAG4Qc/RPg== + final String cacheKey = indexEntryContext.getKey(); + final String vectorFileName = NativeMemoryCacheKeyHelper.extractVectorIndexFileName(cacheKey); + if (vectorFileName == null) { + throw new IllegalStateException( + "Invalid cache key was given. The key [" + cacheKey + "] does not contain the corresponding vector file name." + ); + } + // Prepare for opening index input from directory. + final KNNEngine knnEngine = KNNEngine.getEngineNameFromPath(vectorFileName); final Directory directory = indexEntryContext.getDirectory(); + final int indexSizeKb = Math.toIntExact(directory.fileLength(vectorFileName) / 1024); - // Ex: Input -> /a/b/c/_0_NativeEngines990KnnVectorsFormat_0.vec - // Output -> _0_NativeEngines990KnnVectorsFormat_0.vec - final String logicalIndexPath = absoluteIndexPath.getFileName().toString(); - - final int indexSizeKb = Math.toIntExact(directory.fileLength(logicalIndexPath) / 1024); + // Try to open an index input then pass it down to native engine for loading an index. + try (IndexInput readStream = directory.openInput(vectorFileName, IOContext.READONCE)) { + final IndexInputWithBuffer indexInputWithBuffer = new IndexInputWithBuffer(readStream); + final long indexAddress = JNIService.loadIndex(indexInputWithBuffer, indexEntryContext.getParameters(), knnEngine); - try (IndexInput readStream = directory.openInput(logicalIndexPath, IOContext.READONCE)) { - IndexInputWithBuffer indexInputWithBuffer = new IndexInputWithBuffer(readStream); - long indexAddress = JNIService.loadIndex(indexInputWithBuffer, indexEntryContext.getParameters(), knnEngine); - - return createIndexAllocation(indexEntryContext, knnEngine, indexAddress, fileWatcher, indexSizeKb, absoluteIndexPath); + return createIndexAllocation(indexEntryContext, knnEngine, indexAddress, indexSizeKb, vectorFileName); } } @@ -120,10 +100,9 @@ private NativeMemoryAllocation.IndexAllocation createIndexAllocation( final NativeMemoryEntryContext.IndexEntryContext indexEntryContext, final KNNEngine knnEngine, final long indexAddress, - final FileWatcher fileWatcher, final int indexSizeKb, - final Path absoluteIndexPath - ) throws IOException { + final String vectorFileName + ) { SharedIndexState sharedIndexState = null; String modelId = indexEntryContext.getModelId(); if (IndexUtil.isSharedIndexStateRequired(knnEngine, modelId, indexAddress)) { @@ -132,15 +111,13 @@ private NativeMemoryAllocation.IndexAllocation createIndexAllocation( JNIService.setSharedIndexState(indexAddress, sharedIndexState.getSharedIndexStateAddress(), knnEngine); } - final WatcherHandle watcherHandle = resourceWatcherService.add(fileWatcher); return new NativeMemoryAllocation.IndexAllocation( executor, indexAddress, indexSizeKb, knnEngine, - absoluteIndexPath.toString(), + vectorFileName, indexEntryContext.getOpenSearchIndexName(), - watcherHandle, sharedIndexState, IndexUtil.isBinaryIndex(knnEngine, indexEntryContext.getParameters()) ); diff --git a/src/main/java/org/opensearch/knn/index/query/KNNWeight.java b/src/main/java/org/opensearch/knn/index/query/KNNWeight.java index 87c99b884..b5b2a5d22 100644 --- a/src/main/java/org/opensearch/knn/index/query/KNNWeight.java +++ b/src/main/java/org/opensearch/knn/index/query/KNNWeight.java @@ -15,13 +15,10 @@ import org.apache.lucene.search.FilteredDocIdSetIterator; import org.apache.lucene.search.Scorer; import org.apache.lucene.search.Weight; -import org.apache.lucene.store.FSDirectory; -import org.apache.lucene.store.FilterDirectory; import org.apache.lucene.util.BitSet; import org.apache.lucene.util.BitSetIterator; import org.apache.lucene.util.Bits; import org.apache.lucene.util.FixedBitSet; -import org.opensearch.common.io.PathUtils; import org.opensearch.common.lucene.Lucene; import org.opensearch.knn.common.FieldInfoExtractor; import org.opensearch.knn.common.KNNConstants; @@ -29,6 +26,7 @@ import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.codec.util.KNNCodecUtil; +import org.opensearch.knn.index.codec.util.NativeMemoryCacheKeyHelper; import org.opensearch.knn.index.memory.NativeMemoryAllocation; import org.opensearch.knn.index.memory.NativeMemoryCacheManager; import org.opensearch.knn.index.memory.NativeMemoryEntryContext; @@ -43,7 +41,6 @@ import org.opensearch.knn.plugin.stats.KNNCounter; import java.io.IOException; -import java.nio.file.Path; import java.util.Arrays; import java.util.Collections; import java.util.List; @@ -229,7 +226,6 @@ private Map doANNSearch( final int k ) throws IOException { final SegmentReader reader = Lucene.segmentReader(context.reader()); - String directory = ((FSDirectory) FilterDirectory.unwrap(reader.directory())).getDirectory().toString(); FieldInfo fieldInfo = reader.getFieldInfos().fieldInfo(knnQuery.getField()); @@ -278,7 +274,9 @@ private Map doANNSearch( return Collections.emptyMap(); } - Path indexPath = PathUtils.get(directory, engineFiles.get(0)); + final String vectorIndexFileName = engineFiles.get(0); + final String cacheKey = NativeMemoryCacheKeyHelper.constructCacheKey(vectorIndexFileName, reader.getSegmentInfo().info); + final KNNQueryResult[] results; KNNCounter.GRAPH_QUERY_REQUESTS.increment(); @@ -288,7 +286,7 @@ private Map doANNSearch( indexAllocation = nativeMemoryCacheManager.get( new NativeMemoryEntryContext.IndexEntryContext( reader.directory(), - indexPath.toString(), + cacheKey, NativeMemoryLoadStrategy.IndexLoadStrategy.getInstance(), getParametersAtLoading( spaceType, diff --git a/src/main/java/org/opensearch/knn/jni/JNIService.java b/src/main/java/org/opensearch/knn/jni/JNIService.java index a0daf65a7..dd4dcef17 100644 --- a/src/main/java/org/opensearch/knn/jni/JNIService.java +++ b/src/main/java/org/opensearch/knn/jni/JNIService.java @@ -135,7 +135,6 @@ public static void createIndex( Map parameters, KNNEngine knnEngine ) { - if (KNNEngine.NMSLIB == knnEngine) { NmslibService.createIndex(ids, vectorsAddress, dim, indexPath, parameters); return; diff --git a/src/main/java/org/opensearch/knn/plugin/KNNPlugin.java b/src/main/java/org/opensearch/knn/plugin/KNNPlugin.java index cae8960bb..d05bbaa05 100644 --- a/src/main/java/org/opensearch/knn/plugin/KNNPlugin.java +++ b/src/main/java/org/opensearch/knn/plugin/KNNPlugin.java @@ -195,7 +195,6 @@ public Collection createComponents( this.clusterService = clusterService; // Initialize Native Memory loading strategies - NativeMemoryLoadStrategy.IndexLoadStrategy.initialize(resourceWatcherService); VectorReader vectorReader = new VectorReader(client); NativeMemoryLoadStrategy.TrainingLoadStrategy.initialize(vectorReader); diff --git a/src/test/java/org/opensearch/knn/index/KNNIndexShardTests.java b/src/test/java/org/opensearch/knn/index/KNNIndexShardTests.java index aaeee31e1..18b9656e5 100644 --- a/src/test/java/org/opensearch/knn/index/KNNIndexShardTests.java +++ b/src/test/java/org/opensearch/knn/index/KNNIndexShardTests.java @@ -8,6 +8,12 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import lombok.SneakyThrows; +import org.apache.lucene.index.SegmentCommitInfo; +import org.apache.lucene.index.SegmentInfo; +import org.apache.lucene.store.Directory; +import org.apache.lucene.util.StringHelper; +import org.apache.lucene.util.Version; +import org.mockito.Mockito; import org.opensearch.knn.KNNSingleNodeTestCase; import org.opensearch.index.IndexService; import org.opensearch.index.engine.Engine; @@ -15,8 +21,9 @@ import org.opensearch.knn.index.memory.NativeMemoryCacheManager; import java.io.IOException; -import java.nio.file.Path; -import java.nio.file.Paths; +import java.lang.reflect.Field; +import java.util.Collections; +import java.util.HashSet; import java.util.List; import java.util.Set; import java.util.concurrent.ExecutionException; @@ -113,11 +120,14 @@ public void testGetAllEngineFileContexts() throws IOException, ExecutionExceptio searcher = indexShard.acquireSearcher("test-hnsw-paths-2"); engineFileContexts = knnIndexShard.getAllEngineFileContexts(searcher.getIndexReader()); assertEquals(1, engineFileContexts.size()); - List paths = engineFileContexts.stream().map(KNNIndexShard.EngineFileContext::getIndexPath).collect(Collectors.toList()); + List paths = engineFileContexts.stream() + .map(KNNIndexShard.EngineFileContext::getVectorFileName) + .collect(Collectors.toList()); assertTrue(paths.get(0).contains("hnsw") || paths.get(0).contains("hnswc")); searcher.close(); } + @SneakyThrows public void testGetEngineFileContexts() { // Check that the correct engine paths are being returned by the KNNIndexShard String segmentName = "_0"; @@ -143,20 +153,40 @@ public void testGetEngineFileContexts() { KNNIndexShard knnIndexShard = new KNNIndexShard(null); - Path path = Paths.get(""); - List included = knnIndexShard.getEngineFileContexts( - files, + final Directory dummyDirectory = Mockito.mock(Directory.class); + final SegmentInfo segmentInfo = new SegmentInfo( + dummyDirectory, + Version.LATEST, + null, segmentName, + 0, + false, + false, + null, + Collections.emptyMap(), + new byte[StringHelper.ID_LENGTH], + Collections.emptyMap(), + null + ); + // Inject 'files' into the segment info instance. + // Since SegmentInfo class does trim out its given file list, for example removing segment name from a file name etc, + // we can't just use 'setFiles' api to assign the file list. Which will lead this unit test to be fail. + final Field setFilesPrivateField = SegmentInfo.class.getDeclaredField("setFiles"); + setFilesPrivateField.setAccessible(true); + setFilesPrivateField.set(segmentInfo, new HashSet<>(files)); + + final SegmentCommitInfo segmentCommitInfo = new SegmentCommitInfo(segmentInfo, 0, 0, -1, 0, 0, null); + List included = knnIndexShard.getEngineFileContexts( + segmentCommitInfo, fieldName, fileExt, - path, spaceType, modelId, vectorDataType ); assertEquals(includedFileNames.size(), included.size()); - included.stream().map(KNNIndexShard.EngineFileContext::getIndexPath).forEach(o -> assertTrue(includedFileNames.contains(o))); + included.stream().map(KNNIndexShard.EngineFileContext::getVectorFileName).forEach(o -> assertTrue(includedFileNames.contains(o))); } @SneakyThrows diff --git a/src/test/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesProducerTests.java b/src/test/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesProducerTests.java index ccceee62b..6b18c51c3 100644 --- a/src/test/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesProducerTests.java +++ b/src/test/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesProducerTests.java @@ -121,11 +121,11 @@ public void testProduceKNNBinaryField_fromCodec_nmslibCurrent() throws IOExcepti assertTrue(docValuesFormat instanceof KNN80DocValuesFormat); DocValuesProducer producer = docValuesFormat.fieldsProducer(state); assertTrue(producer instanceof KNN80DocValuesProducer); - int pathSize = ((KNN80DocValuesProducer) producer).getOpenedIndexPath().size(); - assertEquals(pathSize, 1); + int cacheKeySize = ((KNN80DocValuesProducer) producer).getCacheKeys().size(); + assertEquals(cacheKeySize, 1); - String path = ((KNN80DocValuesProducer) producer).getOpenedIndexPath().get(0); - assertTrue(path.contains(segmentFiles.get(0))); + String cacheKey = ((KNN80DocValuesProducer) producer).getCacheKeys().get(0); + assertTrue(cacheKey.contains(segmentFiles.get(0))); } public void testProduceKNNBinaryField_whenFieldHasNonBinaryDocValues_thenSkipThoseField() throws IOException { @@ -178,7 +178,7 @@ public void testProduceKNNBinaryField_whenFieldHasNonBinaryDocValues_thenSkipTho assertTrue(docValuesFormat instanceof KNN80DocValuesFormat); DocValuesProducer producer = docValuesFormat.fieldsProducer(state); assertTrue(producer instanceof KNN80DocValuesProducer); - assertEquals(0, ((KNN80DocValuesProducer) producer).getOpenedIndexPath().size()); + assertEquals(0, ((KNN80DocValuesProducer) producer).getCacheKeys().size()); } } diff --git a/src/test/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsFormatTests.java b/src/test/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsFormatTests.java index 90ed18d0d..f30484586 100644 --- a/src/test/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsFormatTests.java +++ b/src/test/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsFormatTests.java @@ -338,6 +338,8 @@ private RandomIndexWriter createIndexWriter(final Directory dir) throws IOExcept iwc.setUseCompoundFile(false); // Set merge policy to no merges so that we create a predictable number of segments. iwc.setMergePolicy(NoMergePolicy.INSTANCE); + iwc.setMaxBufferedDocs(Integer.MAX_VALUE); + iwc.setRAMBufferSizeMB(1024); return new RandomIndexWriter(random(), dir, iwc); } diff --git a/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestCase.java b/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestCase.java index 3d9969a1e..e6fcb643d 100644 --- a/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestCase.java +++ b/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestCase.java @@ -185,8 +185,6 @@ public void testMultiFieldsKnnIndex(Codec codec) throws Exception { writer.flush(); IndexReader reader = writer.getReader(); writer.close(); - ResourceWatcherService resourceWatcherService = createDisabledResourceWatcherService(); - NativeMemoryLoadStrategy.IndexLoadStrategy.initialize(resourceWatcherService); List hnswfiles = Arrays.stream(dir.listAll()).filter(x -> x.contains("hnsw")).collect(Collectors.toList()); // there should be 2 hnsw index files created. one for test_vector and one for my_vector @@ -213,7 +211,6 @@ public void testMultiFieldsKnnIndex(Codec codec) throws Exception { reader.close(); dir.close(); - resourceWatcherService.close(); NativeMemoryLoadStrategy.IndexLoadStrategy.getInstance().close(); } @@ -298,8 +295,6 @@ public void testBuildFromModelTemplate(Codec codec) throws IOException, Executio // Make sure that search returns the correct results KNNWeight.initialize(modelDao); - ResourceWatcherService resourceWatcherService = createDisabledResourceWatcherService(); - NativeMemoryLoadStrategy.IndexLoadStrategy.initialize(resourceWatcherService); float[] query = { 10.0f, 10.0f, 10.0f }; IndexSearcher searcher = new IndexSearcher(reader); TopDocs topDocs = searcher.search(new KNNQuery(fieldName, query, 4, "dummy", (BitSetProducer) null), 10); @@ -311,7 +306,6 @@ public void testBuildFromModelTemplate(Codec codec) throws IOException, Executio reader.close(); dir.close(); - resourceWatcherService.close(); NativeMemoryLoadStrategy.IndexLoadStrategy.getInstance().close(); } } @@ -422,8 +416,6 @@ public void testKnnVectorIndex( writer.addDocument(doc1); IndexReader reader1 = writer.getReader(); writer.close(); - ResourceWatcherService resourceWatcherService = createDisabledResourceWatcherService(); - NativeMemoryLoadStrategy.IndexLoadStrategy.initialize(resourceWatcherService); verify(perFieldKnnVectorsFormatSpy, atLeastOnce()).getKnnVectorsFormatForField(eq(FIELD_NAME_TWO)); verify(perFieldKnnVectorsFormatSpy, atLeastOnce()).getMaxDimensions(eq(FIELD_NAME_TWO)); @@ -444,7 +436,6 @@ public void testKnnVectorIndex( reader1.close(); dir.close(); - resourceWatcherService.close(); NativeMemoryLoadStrategy.IndexLoadStrategy.getInstance().close(); } } diff --git a/src/test/java/org/opensearch/knn/index/memory/NativeMemoryAllocationTests.java b/src/test/java/org/opensearch/knn/index/memory/NativeMemoryAllocationTests.java index 906ff4cb7..db6231adf 100644 --- a/src/test/java/org/opensearch/knn/index/memory/NativeMemoryAllocationTests.java +++ b/src/test/java/org/opensearch/knn/index/memory/NativeMemoryAllocationTests.java @@ -26,8 +26,6 @@ import org.opensearch.knn.index.util.IndexUtil; import org.opensearch.knn.jni.JNICommons; import org.opensearch.knn.jni.JNIService; -import org.opensearch.watcher.FileWatcher; -import org.opensearch.watcher.WatcherHandle; import java.nio.file.Path; import java.util.Arrays; @@ -40,7 +38,6 @@ import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicReference; -import static org.mockito.Mockito.doNothing; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; import static org.opensearch.knn.common.featureflags.KNNFeatureFlags.KNN_FORCE_EVICT_CACHE_ENABLED_SETTING; @@ -86,10 +83,6 @@ public void testIndexAllocation_close() throws InterruptedException { // Load index into memory long memoryAddress = JNIService.loadIndex(path, parameters, knnEngine); - @SuppressWarnings("unchecked") - WatcherHandle watcherHandle = (WatcherHandle) mock(WatcherHandle.class); - doNothing().when(watcherHandle).stop(); - ExecutorService executorService = Executors.newSingleThreadExecutor(); NativeMemoryAllocation.IndexAllocation indexAllocation = new NativeMemoryAllocation.IndexAllocation( executorService, @@ -97,8 +90,7 @@ public void testIndexAllocation_close() throws InterruptedException { IndexUtil.getFileSizeInKB(path), knnEngine, path, - "test", - watcherHandle + "test" ); indexAllocation.close(); @@ -147,10 +139,6 @@ public void testClose_whenBinaryFiass_thenSuccess() { // Load index into memory long memoryAddress = JNIService.loadIndex(path, parameters, knnEngine); - @SuppressWarnings("unchecked") - WatcherHandle watcherHandle = (WatcherHandle) mock(WatcherHandle.class); - doNothing().when(watcherHandle).stop(); - ExecutorService executorService = Executors.newSingleThreadExecutor(); NativeMemoryAllocation.IndexAllocation indexAllocation = new NativeMemoryAllocation.IndexAllocation( executorService, @@ -159,7 +147,6 @@ public void testClose_whenBinaryFiass_thenSuccess() { knnEngine, path, "test", - watcherHandle, null, true ); @@ -189,8 +176,7 @@ public void testIndexAllocation_getMemoryAddress() { 0, null, "test", - "test", - null + "test" ); assertEquals(memoryAddress, indexAllocation.getMemoryAddress()); @@ -205,8 +191,7 @@ public void testIndexAllocation_readLock() throws InterruptedException { 0, null, "test", - "test", - null + "test" ); int initialValue = 10; @@ -232,7 +217,6 @@ public void testIndexAllocation_readLock() throws InterruptedException { } public void testIndexAllocation_closeDefault() { - WatcherHandle watcherHandle = (WatcherHandle) mock(WatcherHandle.class); ExecutorService executorService = Executors.newFixedThreadPool(2); AtomicReference expectedException = new AtomicReference<>(); @@ -243,8 +227,7 @@ public void testIndexAllocation_closeDefault() { 0, null, "test", - "test", - watcherHandle + "test" ); executorService.submit(nonBlockingIndexAllocation::readLock); @@ -261,7 +244,6 @@ public void testIndexAllocation_closeDefault() { public void testIndexAllocation_closeBlocking() throws InterruptedException, ExecutionException { // Prepare mocking and a thread pool. - WatcherHandle watcherHandle = (WatcherHandle) mock(WatcherHandle.class); ExecutorService executorService = Executors.newSingleThreadExecutor(); // Enable `KNN_FORCE_EVICT_CACHE_ENABLED_SETTING` to force it to block other threads. @@ -273,8 +255,7 @@ public void testIndexAllocation_closeBlocking() throws InterruptedException, Exe 0, null, "test", - "test", - watcherHandle + "test" ); // Acquire a read lock @@ -309,8 +290,7 @@ public void testIndexAllocation_writeLock() throws InterruptedException { 0, null, "test", - "test", - null + "test" ); int initialValue = 10; @@ -342,8 +322,7 @@ public void testIndexAllocation_getSize() { size, null, "test", - "test", - null + "test" ); assertEquals(size, indexAllocation.getSizeInKB()); @@ -357,8 +336,7 @@ public void testIndexAllocation_getKnnEngine() { 0, knnEngine, "test", - "test", - null + "test" ); assertEquals(knnEngine, indexAllocation.getKnnEngine()); @@ -372,11 +350,10 @@ public void testIndexAllocation_getIndexPath() { 0, null, indexPath, - "test", - null + "test" ); - assertEquals(indexPath, indexAllocation.getIndexPath()); + assertEquals(indexPath, indexAllocation.getVectorFileName()); } public void testIndexAllocation_getOsIndexName() { @@ -387,8 +364,7 @@ public void testIndexAllocation_getOsIndexName() { 0, null, "test", - osIndexName, - null + osIndexName ); assertEquals(osIndexName, indexAllocation.getOpenSearchIndexName()); diff --git a/src/test/java/org/opensearch/knn/index/memory/NativeMemoryCacheManagerTests.java b/src/test/java/org/opensearch/knn/index/memory/NativeMemoryCacheManagerTests.java index 85eaf3322..4baf66cb4 100644 --- a/src/test/java/org/opensearch/knn/index/memory/NativeMemoryCacheManagerTests.java +++ b/src/test/java/org/opensearch/knn/index/memory/NativeMemoryCacheManagerTests.java @@ -118,8 +118,7 @@ public void testGetIndexSizeInKilobytes() throws ExecutionException, IOException indexEntryWeight, null, key, - indexName, - null + indexName ); NativeMemoryEntryContext.IndexEntryContext indexEntryContext = mock(NativeMemoryEntryContext.IndexEntryContext.class); @@ -152,8 +151,7 @@ public void testGetIndexSizeAsPercentage() throws ExecutionException, IOExceptio indexEntryWeight, null, key, - indexName, - null + indexName ); NativeMemoryEntryContext.IndexEntryContext indexEntryContext = mock(NativeMemoryEntryContext.IndexEntryContext.class); @@ -231,8 +229,7 @@ public void testGetIndexGraphCount() throws ExecutionException, IOException { indexEntryWeight, null, key1, - indexName1, - null + indexName1 ); NativeMemoryEntryContext.IndexEntryContext indexEntryContext = mock(NativeMemoryEntryContext.IndexEntryContext.class); @@ -247,8 +244,7 @@ public void testGetIndexGraphCount() throws ExecutionException, IOException { indexEntryWeight, null, key2, - indexName1, - null + indexName1 ); indexEntryContext = mock(NativeMemoryEntryContext.IndexEntryContext.class); @@ -263,8 +259,7 @@ public void testGetIndexGraphCount() throws ExecutionException, IOException { indexEntryWeight, null, key3, - indexName2, - null + indexName2 ); indexEntryContext = mock(NativeMemoryEntryContext.IndexEntryContext.class); @@ -408,8 +403,7 @@ public void testGetIndicesCacheStats() throws IOException, ExecutionException { size1, null, testKey1, - indexName1, - null + indexName1 ); NativeMemoryAllocation.IndexAllocation indexAllocation2 = new NativeMemoryAllocation.IndexAllocation( @@ -418,8 +412,7 @@ public void testGetIndicesCacheStats() throws IOException, ExecutionException { size2, null, testKey2, - indexName1, - null + indexName1 ); NativeMemoryAllocation.IndexAllocation indexAllocation3 = new NativeMemoryAllocation.IndexAllocation( @@ -428,8 +421,7 @@ public void testGetIndicesCacheStats() throws IOException, ExecutionException { size1, null, testKey3, - indexName2, - null + indexName2 ); NativeMemoryAllocation.IndexAllocation indexAllocation4 = new NativeMemoryAllocation.IndexAllocation( @@ -438,8 +430,7 @@ public void testGetIndicesCacheStats() throws IOException, ExecutionException { size2, null, testKey4, - indexName2, - null + indexName2 ); NativeMemoryEntryContext.IndexEntryContext indexEntryContext1 = mock(NativeMemoryEntryContext.IndexEntryContext.class); diff --git a/src/test/java/org/opensearch/knn/index/memory/NativeMemoryEntryContextTests.java b/src/test/java/org/opensearch/knn/index/memory/NativeMemoryEntryContextTests.java index 72cab9a1b..5379abc74 100644 --- a/src/test/java/org/opensearch/knn/index/memory/NativeMemoryEntryContextTests.java +++ b/src/test/java/org/opensearch/knn/index/memory/NativeMemoryEntryContextTests.java @@ -13,23 +13,21 @@ import com.google.common.collect.ImmutableMap; import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IOContext; +import org.apache.lucene.store.IndexOutput; +import org.apache.lucene.store.MMapDirectory; import org.opensearch.cluster.service.ClusterService; import org.opensearch.knn.KNNTestCase; +import org.opensearch.knn.TestUtils; import org.opensearch.knn.index.engine.qframe.QuantizationConfig; -import org.opensearch.knn.index.util.IndexUtil; import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.engine.KNNEngine; -import java.io.BufferedOutputStream; import java.io.IOException; -import java.io.OutputStream; -import java.nio.file.Files; import java.nio.file.Path; import java.util.Arrays; import java.util.Map; -import static java.nio.file.StandardOpenOption.APPEND; -import static java.nio.file.StandardOpenOption.CREATE; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -46,7 +44,7 @@ public void testIndexEntryContext_load() throws IOException { NativeMemoryLoadStrategy.IndexLoadStrategy indexLoadStrategy = mock(NativeMemoryLoadStrategy.IndexLoadStrategy.class); NativeMemoryEntryContext.IndexEntryContext indexEntryContext = new NativeMemoryEntryContext.IndexEntryContext( (Directory) null, - "test", + TestUtils.createFakeNativeMamoryCacheKey("test"), indexLoadStrategy, null, "test" @@ -58,8 +56,7 @@ public void testIndexEntryContext_load() throws IOException { 10, KNNEngine.DEFAULT, "test-path", - "test-name", - null + "test-name" ); when(indexLoadStrategy.load(indexEntryContext)).thenReturn(indexAllocation); @@ -69,36 +66,37 @@ public void testIndexEntryContext_load() throws IOException { public void testIndexEntryContext_calculateSize() throws IOException { // Create a file and write random bytes to it - Path tmpFile = createTempFile(); + final Path tmpDirectory = createTempDir(); + final Directory directory = new MMapDirectory(tmpDirectory); + final String indexFileName = "test.faiss"; byte[] data = new byte[1024 * 3]; Arrays.fill(data, (byte) 'c'); - try (OutputStream out = new BufferedOutputStream(Files.newOutputStream(tmpFile, CREATE, APPEND))) { - out.write(data, 0, data.length); - } catch (IOException x) { - fail("Failed to write to file"); + try (IndexOutput output = directory.createOutput(indexFileName, IOContext.DEFAULT)) { + output.writeBytes(data, data.length); } // Get the expected size of this function - int expectedSize = IndexUtil.getFileSizeInKB(tmpFile.toAbsolutePath().toString()); + final long expectedSizeBytes = directory.fileLength(indexFileName); + final long expectedSizeKb = expectedSizeBytes / 1024L; // Check that the indexEntryContext will return the same thing NativeMemoryEntryContext.IndexEntryContext indexEntryContext = new NativeMemoryEntryContext.IndexEntryContext( - (Directory) null, - tmpFile.toAbsolutePath().toString(), + directory, + TestUtils.createFakeNativeMamoryCacheKey(indexFileName), null, null, "test" ); - assertEquals(expectedSize, indexEntryContext.calculateSizeInKB().longValue()); + assertEquals(expectedSizeKb, indexEntryContext.calculateSizeInKB().longValue()); } public void testIndexEntryContext_getOpenSearchIndexName() { String openSearchIndexName = "test-index"; NativeMemoryEntryContext.IndexEntryContext indexEntryContext = new NativeMemoryEntryContext.IndexEntryContext( (Directory) null, - "test", + TestUtils.createFakeNativeMamoryCacheKey("test"), null, null, openSearchIndexName @@ -111,7 +109,7 @@ public void testIndexEntryContext_getParameters() { Map parameters = ImmutableMap.of("test-1", 10); NativeMemoryEntryContext.IndexEntryContext indexEntryContext = new NativeMemoryEntryContext.IndexEntryContext( (Directory) null, - "test", + TestUtils.createFakeNativeMamoryCacheKey("test"), null, parameters, "test" diff --git a/src/test/java/org/opensearch/knn/index/memory/NativeMemoryLoadStrategyTests.java b/src/test/java/org/opensearch/knn/index/memory/NativeMemoryLoadStrategyTests.java index 373afddc7..8236d0518 100644 --- a/src/test/java/org/opensearch/knn/index/memory/NativeMemoryLoadStrategyTests.java +++ b/src/test/java/org/opensearch/knn/index/memory/NativeMemoryLoadStrategyTests.java @@ -28,7 +28,6 @@ import org.opensearch.knn.index.engine.KNNEngine; import org.opensearch.knn.training.FloatTrainingDataConsumer; import org.opensearch.knn.training.VectorReader; -import org.opensearch.watcher.ResourceWatcherService; import java.io.IOException; import java.nio.file.Path; @@ -38,10 +37,7 @@ import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.any; -import static org.mockito.Mockito.eq; import static org.mockito.Mockito.doAnswer; -import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; public class NativeMemoryLoadStrategyTests extends KNNTestCase { @@ -66,13 +62,9 @@ public void testIndexLoadStrategy_load() throws IOException { TestUtils.createIndex(ids, memoryAddress, dimension, path, parameters, knnEngine); // Setup mock resource manager - ResourceWatcherService resourceWatcherService = mock(ResourceWatcherService.class); - doReturn(null).when(resourceWatcherService).add(any()); - NativeMemoryLoadStrategy.IndexLoadStrategy.initialize(resourceWatcherService); - NativeMemoryEntryContext.IndexEntryContext indexEntryContext = new NativeMemoryEntryContext.IndexEntryContext( luceneDirectory, - path, + TestUtils.createFakeNativeMamoryCacheKey(indexName), NativeMemoryLoadStrategy.IndexLoadStrategy.getInstance(), parameters, "test" @@ -116,13 +108,9 @@ public void testLoad_whenFaissBinary_thenSuccess() throws IOException { TestUtils.createIndex(ids, memoryAddress, dimension, path, parameters, knnEngine); // Setup mock resource manager - ResourceWatcherService resourceWatcherService = mock(ResourceWatcherService.class); - doReturn(null).when(resourceWatcherService).add(any()); - NativeMemoryLoadStrategy.IndexLoadStrategy.initialize(resourceWatcherService); - NativeMemoryEntryContext.IndexEntryContext indexEntryContext = new NativeMemoryEntryContext.IndexEntryContext( luceneDirectory, - path, + TestUtils.createFakeNativeMamoryCacheKey(indexName), NativeMemoryLoadStrategy.IndexLoadStrategy.getInstance(), parameters, "test" diff --git a/src/testFixtures/java/org/opensearch/knn/TestUtils.java b/src/testFixtures/java/org/opensearch/knn/TestUtils.java index 621688b62..9145bcd16 100644 --- a/src/testFixtures/java/org/opensearch/knn/TestUtils.java +++ b/src/testFixtures/java/org/opensearch/knn/TestUtils.java @@ -22,6 +22,7 @@ import org.opensearch.knn.jni.JNIService; import org.opensearch.knn.plugin.script.KNNScoringUtil; +import java.util.Base64; import java.util.Collections; import java.util.Comparator; import java.util.Random; @@ -184,6 +185,10 @@ public static float[][] getIndexVectors(int docCount, int dimensions, boolean is } } + public static String createFakeNativeMamoryCacheKey(final String fileName) { + return fileName + "@" + Base64.getEncoder().encodeToString(fileName.getBytes()); + } + /* * Recall is the number of relevant documents retrieved by a search divided by the total number of existing relevant documents. * We are similarly calculating recall by verifying number of relevant documents obtained in the search results by comparing with From be0837d575cda44f6be771851f48543059807f8b Mon Sep 17 00:00:00 2001 From: Naveen Tatikonda Date: Tue, 22 Oct 2024 10:41:44 -0500 Subject: [PATCH 405/416] Add Release Notes for 2.18.0.0 (#2227) (#2228) * Add Release Notes for 2.18.0.0 Signed-off-by: Vikasht34 * Add Release Notes for 2.18.0.0 Signed-off-by: Vikasht34 --------- Signed-off-by: Vikasht34 (cherry picked from commit 8f2d9114df33d232a9b749b8863922219cf56afd) Co-authored-by: Vikasht34 --- CHANGELOG.md | 22 +------------- .../opensearch-knn.release-notes-2.18.0.0.md | 29 +++++++++++++++++++ 2 files changed, 30 insertions(+), 21 deletions(-) create mode 100644 release-notes/opensearch-knn.release-notes-2.18.0.0.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 03d6d43a9..cb1005678 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,31 +13,11 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Maintenance ### Refactoring -## [Unreleased 2.x](https://github.com/opensearch-project/k-NN/compare/2.17...2.x) +## [Unreleased 2.x](https://github.com/opensearch-project/k-NN/compare/2.18...2.x) ### Features -* Add AVX512 support to k-NN for FAISS library [#2069](https://github.com/opensearch-project/k-NN/pull/2069) ### Enhancements -* Introducing a loading layer in FAISS [#2033](https://github.com/opensearch-project/k-NN/issues/2033) -* Add short circuit if no live docs are in segments [#2059](https://github.com/opensearch-project/k-NN/pull/2059) -* Optimize reduceToTopK in ResultUtil by removing pre-filling and reducing peek calls [#2146](https://github.com/opensearch-project/k-NN/pull/2146) -* Update Default Rescore Context based on Dimension [#2149](https://github.com/opensearch-project/k-NN/pull/2149) -* KNNIterators should support with and without filters [#2155](https://github.com/opensearch-project/k-NN/pull/2155) -* Adding Support to Enable/Disble Share level Rescoring and Update Oversampling Factor[#2172](https://github.com/opensearch-project/k-NN/pull/2172) -* Add support to build vector data structures greedily and perform exact search when there are no engine files [#1942](https://github.com/opensearch-project/k-NN/issues/1942) -* Add CompressionLevel Calculation for PQ [#2200](https://github.com/opensearch-project/k-NN/pull/2200) -* Introduce a loading layer in native engine [#2185](https://github.com/opensearch-project/k-NN/pull/2185) -* Remove FSDirectory dependency from native engine constructing side and deprecated FileWatcher [#2182](https://github.com/opensearch-project/k-NN/pull/2182) ### Bug Fixes -* Add DocValuesProducers for releasing memory when close index [#1946](https://github.com/opensearch-project/k-NN/pull/1946) -* KNN80DocValues should only be considered for BinaryDocValues fields [#2147](https://github.com/opensearch-project/k-NN/pull/2147) -* Score Fix for Binary Quantized Vector and Setting Default value in case of shard level rescoring is disabled for oversampling factor[#2183](https://github.com/opensearch-project/k-NN/pull/2183) -* Java Docs Fix For 2.x ### Infrastructure ### Documentation -* Fix sed command in DEVELOPER_GUIDE.md to append a new line character '\n'. [#2181](https://github.com/opensearch-project/k-NN/pull/2181) ### Maintenance -* Remove benchmarks folder from k-NN repo [#2127](https://github.com/opensearch-project/k-NN/pull/2127) -* Fix lucene codec after lucene version bumped to 9.12. [#2195](https://github.com/opensearch-project/k-NN/pull/2195) ### Refactoring -* Does not create additional KNNVectorValues in NativeEngines990KNNVectorWriter when quantization is not needed [#2133](https://github.com/opensearch-project/k-NN/pull/2133) -* Minor refactoring and refactored some unit test [#2167](https://github.com/opensearch-project/k-NN/pull/2167) diff --git a/release-notes/opensearch-knn.release-notes-2.18.0.0.md b/release-notes/opensearch-knn.release-notes-2.18.0.0.md new file mode 100644 index 000000000..ec9114fde --- /dev/null +++ b/release-notes/opensearch-knn.release-notes-2.18.0.0.md @@ -0,0 +1,29 @@ +## Version 2.18.0.0 Release Notes + +Compatible with OpenSearch 2.18.0 + +### Features +* Add AVX512 support to k-NN for FAISS library [#2069](https://github.com/opensearch-project/k-NN/pull/2069) +### Enhancements +* Introducing a loading layer in FAISS [#2033](https://github.com/opensearch-project/k-NN/issues/2033) +* Add short circuit if no live docs are in segments [#2059](https://github.com/opensearch-project/k-NN/pull/2059) +* Optimize reduceToTopK in ResultUtil by removing pre-filling and reducing peek calls [#2146](https://github.com/opensearch-project/k-NN/pull/2146) +* Update Default Rescore Context based on Dimension [#2149](https://github.com/opensearch-project/k-NN/pull/2149) +* KNNIterators should support with and without filters [#2155](https://github.com/opensearch-project/k-NN/pull/2155) +* Adding Support to Enable/Disble Share level Rescoring and Update Oversampling Factor[#2172](https://github.com/opensearch-project/k-NN/pull/2172) +* Add support to build vector data structures greedily and perform exact search when there are no engine files [#1942](https://github.com/opensearch-project/k-NN/issues/1942) +* Add CompressionLevel Calculation for PQ [#2200](https://github.com/opensearch-project/k-NN/pull/2200) +* Remove FSDirectory dependency from native engine constructing side and deprecated FileWatcher [#2182](https://github.com/opensearch-project/k-NN/pull/2182) +### Bug Fixes +* Add DocValuesProducers for releasing memory when close index [#1946](https://github.com/opensearch-project/k-NN/pull/1946) +* KNN80DocValues should only be considered for BinaryDocValues fields [#2147](https://github.com/opensearch-project/k-NN/pull/2147) +* Score Fix for Binary Quantized Vector and Setting Default value in case of shard level rescoring is disabled for oversampling factor[#2183](https://github.com/opensearch-project/k-NN/pull/2183) +* Java Docs Fix For 2.x[#2190](https://github.com/opensearch-project/k-NN/pull/2190) +### Documentation +* Fix sed command in DEVELOPER_GUIDE.md to append a new line character '\n'. [#2181](https://github.com/opensearch-project/k-NN/pull/2181) +### Maintenance +* Remove benchmarks folder from k-NN repo [#2127](https://github.com/opensearch-project/k-NN/pull/2127) +* Fix lucene codec after lucene version bumped to 9.12. [#2195](https://github.com/opensearch-project/k-NN/pull/2195) +### Refactoring +* Does not create additional KNNVectorValues in NativeEngines990KNNVectorWriter when quantization is not needed [#2133](https://github.com/opensearch-project/k-NN/pull/2133) +* Minor refactoring and refactored some unit test [#2167](https://github.com/opensearch-project/k-NN/pull/2167) From bdbddcf60160a587bad8af505b31785f1f4216f4 Mon Sep 17 00:00:00 2001 From: Vijayan Balasubramanian Date: Wed, 23 Oct 2024 21:03:21 -0700 Subject: [PATCH 406/416] [Backport 2.x] Update approximate_threshold to 15K documents (#2229) (#2231) * Update approximate_threshold to 15K documents (#2229) * Update threshold to 15K documents After comparing indexing and search performance, we are updating default value to be 15000. Signed-off-by: Vijayan Balasubramanian * Fix bwc test Signed-off-by: Vijayan Balasubramanian * Update test method Signed-off-by: Vijayan Balasubramanian * Flush data after index Signed-off-by: Vijayan Balasubramanian --------- Signed-off-by: Vijayan Balasubramanian * Remove udpate cluster setting Signed-off-by: Vijayan Balasubramanian * update warmup rolling upgrade scenario Signed-off-by: Vijayan Balasubramanian --------- Signed-off-by: Vijayan Balasubramanian --- .../org/opensearch/knn/bwc/ClearCacheIT.java | 17 ++----- .../org/opensearch/knn/bwc/IndexingIT.java | 10 +++- .../java/org/opensearch/knn/bwc/WarmupIT.java | 12 ++++- .../org/opensearch/knn/bwc/ClearCacheIT.java | 9 ++-- .../java/org/opensearch/knn/bwc/WarmupIT.java | 46 ++++++------------- .../opensearch-knn.release-notes-2.18.0.0.md | 1 + .../org/opensearch/knn/index/KNNSettings.java | 2 +- .../NativeEngines990KnnVectorsFormat.java | 4 ++ .../codec/KNN990Codec/UnitTestCodec.java | 4 +- .../opensearch/knn/KNNSingleNodeTestCase.java | 22 +++++++++ .../knn/index/KNNCircuitBreakerIT.java | 3 ++ .../knn/index/KNNESSettingsTestIT.java | 9 ++-- .../knn/index/KNNIndexShardTests.java | 5 ++ .../org/opensearch/knn/index/NmslibIT.java | 2 +- .../opensearch/knn/index/OpenSearchIT.java | 2 +- .../action/RestClearCacheHandlerIT.java | 13 +++--- .../plugin/action/RestKNNStatsHandlerIT.java | 4 +- .../plugin/action/RestKNNWarmupHandlerIT.java | 6 +-- .../action/RestLegacyKNNStatsHandlerIT.java | 2 +- .../action/RestLegacyKNNWarmupHandlerIT.java | 9 ++-- .../ClearCacheTransportActionTests.java | 2 +- .../KNNWarmupTransportActionTests.java | 2 +- .../org/opensearch/knn/KNNRestTestCase.java | 9 ++++ 23 files changed, 116 insertions(+), 79 deletions(-) diff --git a/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/ClearCacheIT.java b/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/ClearCacheIT.java index 045821e09..f93f2f883 100644 --- a/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/ClearCacheIT.java +++ b/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/ClearCacheIT.java @@ -22,14 +22,11 @@ public void testClearCache() throws Exception { if (isRunningAgainstOldCluster()) { createKnnIndex(testIndex, getKNNDefaultIndexSettings(), createKnnIndexMapping(TEST_FIELD, DIMENSIONS)); addKNNDocs(testIndex, TEST_FIELD, DIMENSIONS, docId, NUM_DOCS); - } else { queryCnt = NUM_DOCS; - validateClearCacheOnUpgrade(queryCnt); - - docId = NUM_DOCS; - addKNNDocs(testIndex, TEST_FIELD, DIMENSIONS, docId, NUM_DOCS); - - queryCnt = queryCnt + NUM_DOCS; + int graphCount = getTotalGraphsInCache(); + knnWarmup(Collections.singletonList(testIndex)); + assertTrue(getTotalGraphsInCache() > graphCount); + } else { validateClearCacheOnUpgrade(queryCnt); deleteKNNIndex(testIndex); } @@ -37,13 +34,7 @@ public void testClearCache() throws Exception { // validation steps for Clear Cache API after upgrading node to new version private void validateClearCacheOnUpgrade(int queryCount) throws Exception { - int graphCount = getTotalGraphsInCache(); - knnWarmup(Collections.singletonList(testIndex)); - assertTrue(getTotalGraphsInCache() > graphCount); - validateKNNSearch(testIndex, TEST_FIELD, DIMENSIONS, queryCount, K); - clearCache(Collections.singletonList(testIndex)); assertEquals(0, getTotalGraphsInCache()); - validateKNNSearch(testIndex, TEST_FIELD, DIMENSIONS, queryCount, K); } } diff --git a/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/IndexingIT.java b/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/IndexingIT.java index 6b0bab9ee..bf4c8f7ec 100644 --- a/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/IndexingIT.java +++ b/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/IndexingIT.java @@ -6,11 +6,14 @@ package org.opensearch.knn.bwc; import org.junit.Assert; +import org.opensearch.common.settings.Settings; import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.knn.index.KNNSettings; import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.engine.KNNEngine; +import java.util.List; import java.util.Map; import static org.opensearch.knn.TestUtils.KNN_ALGO_PARAM_EF_CONSTRUCTION_MIN_VALUE; @@ -52,6 +55,8 @@ public void testKNNIndexDefaultLegacyFieldMapping() throws Exception { createKnnIndex(testIndex, getKNNDefaultIndexSettings(), createKnnIndexMapping(TEST_FIELD, DIMENSIONS)); addKNNDocs(testIndex, TEST_FIELD, DIMENSIONS, DOC_ID, NUM_DOCS); } else { + // update index setting to allow build graph always since we test graph count that are loaded into memory + updateIndexSettings(testIndex, Settings.builder().put(KNNSettings.INDEX_KNN_ADVANCED_APPROXIMATE_THRESHOLD, 0)); validateKNNIndexingOnUpgrade(NUM_DOCS); } } @@ -275,13 +280,14 @@ public void testNoParametersOnUpgrade() throws Exception { // KNN indexing tests when the cluster is upgraded to latest version public void validateKNNIndexingOnUpgrade(int numOfDocs) throws Exception { + updateIndexSettings(testIndex, Settings.builder().put(KNNSettings.INDEX_KNN_ADVANCED_APPROXIMATE_THRESHOLD, 0)); + forceMergeKnnIndex(testIndex); QUERY_COUNT = numOfDocs; validateKNNSearch(testIndex, TEST_FIELD, DIMENSIONS, QUERY_COUNT, K); - cleanUpCache(); + clearCache(List.of(testIndex)); DOC_ID = numOfDocs; addKNNDocs(testIndex, TEST_FIELD, DIMENSIONS, DOC_ID, NUM_DOCS); QUERY_COUNT = QUERY_COUNT + NUM_DOCS; - validateKNNSearch(testIndex, TEST_FIELD, DIMENSIONS, QUERY_COUNT, K); forceMergeKnnIndex(testIndex); validateKNNSearch(testIndex, TEST_FIELD, DIMENSIONS, QUERY_COUNT, K); deleteKNNIndex(testIndex); diff --git a/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/WarmupIT.java b/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/WarmupIT.java index 8a965d2fa..db3555a8d 100644 --- a/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/WarmupIT.java +++ b/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/WarmupIT.java @@ -6,6 +6,7 @@ package org.opensearch.knn.bwc; import org.opensearch.common.settings.Settings; +import org.opensearch.knn.index.KNNSettings; import org.opensearch.knn.index.SpaceType; import java.util.Collections; @@ -34,6 +35,8 @@ public void testKNNWarmupDefaultLegacyFieldMapping() throws Exception { createKnnIndex(testIndex, getKNNDefaultIndexSettings(), createKnnIndexMapping(TEST_FIELD, DIMENSIONS)); addKNNDocs(testIndex, TEST_FIELD, DIMENSIONS, DOC_ID, NUM_DOCS); } else { + // update index setting to allow build graph always since we test graph count that are loaded into memory + updateIndexSettings(testIndex, Settings.builder().put(KNNSettings.INDEX_KNN_ADVANCED_APPROXIMATE_THRESHOLD, 0)); validateKNNWarmupOnUpgrade(); } } @@ -65,6 +68,8 @@ public void testKNNWarmupDefaultMethodFieldMapping() throws Exception { createKnnIndex(testIndex, getKNNDefaultIndexSettings(), createKNNIndexMethodFieldMapping(TEST_FIELD, DIMENSIONS)); addKNNDocs(testIndex, TEST_FIELD, DIMENSIONS, DOC_ID, NUM_DOCS); } else { + // update index setting to allow build graph always since we test graph count that are loaded into memory + updateIndexSettings(testIndex, Settings.builder().put(KNNSettings.INDEX_KNN_ADVANCED_APPROXIMATE_THRESHOLD, 0)); validateKNNWarmupOnUpgrade(); } } @@ -85,21 +90,26 @@ public void testKNNWarmupCustomMethodFieldMapping() throws Exception { } public void validateKNNWarmupOnUpgrade() throws Exception { + // update index setting to allow build graph always since we test graph count that are loaded into memory + updateIndexSettings(testIndex, Settings.builder().put(KNNSettings.INDEX_KNN_ADVANCED_APPROXIMATE_THRESHOLD, 0)); int graphCount = getTotalGraphsInCache(); knnWarmup(Collections.singletonList(testIndex)); - assertTrue(getTotalGraphsInCache() > graphCount); + int totalGraph = getTotalGraphsInCache(); + assertTrue(totalGraph > graphCount); QUERY_COUNT = NUM_DOCS; validateKNNSearch(testIndex, TEST_FIELD, DIMENSIONS, QUERY_COUNT, K); DOC_ID = NUM_DOCS; addKNNDocs(testIndex, TEST_FIELD, DIMENSIONS, DOC_ID, NUM_DOCS); + forceMergeKnnIndex(testIndex); int updatedGraphCount = getTotalGraphsInCache(); knnWarmup(Collections.singletonList(testIndex)); assertTrue(getTotalGraphsInCache() > updatedGraphCount); QUERY_COUNT = QUERY_COUNT + NUM_DOCS; + validateKNNSearch(testIndex, TEST_FIELD, DIMENSIONS, QUERY_COUNT, K); deleteKNNIndex(testIndex); } diff --git a/qa/rolling-upgrade/src/test/java/org/opensearch/knn/bwc/ClearCacheIT.java b/qa/rolling-upgrade/src/test/java/org/opensearch/knn/bwc/ClearCacheIT.java index 24c674d0d..5ef3b1d97 100644 --- a/qa/rolling-upgrade/src/test/java/org/opensearch/knn/bwc/ClearCacheIT.java +++ b/qa/rolling-upgrade/src/test/java/org/opensearch/knn/bwc/ClearCacheIT.java @@ -25,6 +25,9 @@ public void testClearCache() throws Exception { createKnnIndex(testIndex, getKNNDefaultIndexSettings(), createKnnIndexMapping(TEST_FIELD, DIMENSIONS)); int docIdOld = 0; addKNNDocs(testIndex, TEST_FIELD, DIMENSIONS, docIdOld, NUM_DOCS); + int graphCount = getTotalGraphsInCache(); + knnWarmup(Collections.singletonList(testIndex)); + assertTrue(getTotalGraphsInCache() > graphCount); break; case UPGRADED: queryCnt = NUM_DOCS; @@ -42,14 +45,8 @@ public void testClearCache() throws Exception { // validation steps for Clear Cache API after upgrading all nodes from old version to new version public void validateClearCacheOnUpgrade(int queryCount) throws Exception { - int graphCount = getTotalGraphsInCache(); - knnWarmup(Collections.singletonList(testIndex)); - assertTrue(getTotalGraphsInCache() > graphCount); - validateKNNSearch(testIndex, TEST_FIELD, DIMENSIONS, queryCount, K); - clearCache(Collections.singletonList(testIndex)); assertEquals(0, getTotalGraphsInCache()); - validateKNNSearch(testIndex, TEST_FIELD, DIMENSIONS, queryCount, K); } } diff --git a/qa/rolling-upgrade/src/test/java/org/opensearch/knn/bwc/WarmupIT.java b/qa/rolling-upgrade/src/test/java/org/opensearch/knn/bwc/WarmupIT.java index 9cbb99d87..7d2a796a0 100644 --- a/qa/rolling-upgrade/src/test/java/org/opensearch/knn/bwc/WarmupIT.java +++ b/qa/rolling-upgrade/src/test/java/org/opensearch/knn/bwc/WarmupIT.java @@ -5,7 +5,11 @@ package org.opensearch.knn.bwc; +import org.opensearch.common.settings.Settings; +import org.opensearch.knn.index.KNNSettings; + import java.util.Collections; +import java.util.List; import static org.opensearch.knn.TestUtils.NODES_BWC_CLUSTER; @@ -20,44 +24,22 @@ public void testKNNWarmup() throws Exception { switch (getClusterType()) { case OLD: createKnnIndex(testIndex, getKNNDefaultIndexSettings(), createKnnIndexMapping(TEST_FIELD, DIMENSIONS)); - int docIdOld = 0; - addKNNDocs(testIndex, TEST_FIELD, DIMENSIONS, docIdOld, NUM_DOCS); + addKNNDocs(testIndex, TEST_FIELD, DIMENSIONS, 0, NUM_DOCS); break; case MIXED: - int totalDocsCountMixed; - int docIdMixed; - if (isFirstMixedRound()) { - docIdMixed = NUM_DOCS; - totalDocsCountMixed = 2 * NUM_DOCS; - } else { - docIdMixed = 2 * NUM_DOCS; - totalDocsCountMixed = 3 * NUM_DOCS; - } - validateKNNWarmupOnUpgrade(totalDocsCountMixed, docIdMixed); + int graphCount = getTotalGraphsInCache(); + knnWarmup(Collections.singletonList(testIndex)); + assertTrue(getTotalGraphsInCache() > graphCount); + clearCache(List.of(testIndex)); break; case UPGRADED: - int docIdUpgraded = 3 * NUM_DOCS; - int totalDocsCountUpgraded = 4 * NUM_DOCS; - validateKNNWarmupOnUpgrade(totalDocsCountUpgraded, docIdUpgraded); - + updateIndexSettings(testIndex, Settings.builder().put(KNNSettings.INDEX_KNN_ADVANCED_APPROXIMATE_THRESHOLD, 0)); + addKNNDocs(testIndex, TEST_FIELD, DIMENSIONS, NUM_DOCS, NUM_DOCS); + int updatedGraphCount = getTotalGraphsInCache(); + knnWarmup(Collections.singletonList(testIndex)); + assertTrue(getTotalGraphsInCache() > updatedGraphCount); deleteKNNIndex(testIndex); } - - } - - // validation steps for KNN Warmup after upgrading each node from old version to new version - public void validateKNNWarmupOnUpgrade(int totalDocsCount, int docId) throws Exception { - int graphCount = getTotalGraphsInCache(); - knnWarmup(Collections.singletonList(testIndex)); - assertTrue(getTotalGraphsInCache() > graphCount); - - addKNNDocs(testIndex, TEST_FIELD, DIMENSIONS, docId, NUM_DOCS); - - int updatedGraphCount = getTotalGraphsInCache(); - knnWarmup(Collections.singletonList(testIndex)); - assertTrue(getTotalGraphsInCache() > updatedGraphCount); - - validateKNNSearch(testIndex, TEST_FIELD, DIMENSIONS, totalDocsCount, K); } } diff --git a/release-notes/opensearch-knn.release-notes-2.18.0.0.md b/release-notes/opensearch-knn.release-notes-2.18.0.0.md index ec9114fde..9e85fd9ca 100644 --- a/release-notes/opensearch-knn.release-notes-2.18.0.0.md +++ b/release-notes/opensearch-knn.release-notes-2.18.0.0.md @@ -14,6 +14,7 @@ Compatible with OpenSearch 2.18.0 * Add support to build vector data structures greedily and perform exact search when there are no engine files [#1942](https://github.com/opensearch-project/k-NN/issues/1942) * Add CompressionLevel Calculation for PQ [#2200](https://github.com/opensearch-project/k-NN/pull/2200) * Remove FSDirectory dependency from native engine constructing side and deprecated FileWatcher [#2182](https://github.com/opensearch-project/k-NN/pull/2182) +* Update approximate_threshold to 15K documents [#2229](https://github.com/opensearch-project/k-NN/pull/2229) ### Bug Fixes * Add DocValuesProducers for releasing memory when close index [#1946](https://github.com/opensearch-project/k-NN/pull/1946) * KNN80DocValues should only be considered for BinaryDocValues fields [#2147](https://github.com/opensearch-project/k-NN/pull/2147) diff --git a/src/main/java/org/opensearch/knn/index/KNNSettings.java b/src/main/java/org/opensearch/knn/index/KNNSettings.java index 6eff61d4b..b81a54124 100644 --- a/src/main/java/org/opensearch/knn/index/KNNSettings.java +++ b/src/main/java/org/opensearch/knn/index/KNNSettings.java @@ -98,7 +98,7 @@ public class KNNSettings { public static final boolean KNN_DEFAULT_FAISS_AVX2_DISABLED_VALUE = false; public static final boolean KNN_DEFAULT_FAISS_AVX512_DISABLED_VALUE = false; public static final String INDEX_KNN_DEFAULT_SPACE_TYPE = "l2"; - public static final Integer INDEX_KNN_ADVANCED_APPROXIMATE_THRESHOLD_DEFAULT_VALUE = 0; + public static final Integer INDEX_KNN_ADVANCED_APPROXIMATE_THRESHOLD_DEFAULT_VALUE = 15_000; public static final Integer INDEX_KNN_BUILD_VECTOR_DATA_STRUCTURE_THRESHOLD_MIN = -1; public static final Integer INDEX_KNN_BUILD_VECTOR_DATA_STRUCTURE_THRESHOLD_MAX = Integer.MAX_VALUE - 2; public static final String INDEX_KNN_DEFAULT_SPACE_TYPE_FOR_BINARY = "hamming"; diff --git a/src/main/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsFormat.java b/src/main/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsFormat.java index 520a9838d..dd326123e 100644 --- a/src/main/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsFormat.java +++ b/src/main/java/org/opensearch/knn/index/codec/KNN990Codec/NativeEngines990KnnVectorsFormat.java @@ -37,6 +37,10 @@ public NativeEngines990KnnVectorsFormat() { this(new Lucene99FlatVectorsFormat(new DefaultFlatVectorScorer())); } + public NativeEngines990KnnVectorsFormat(int approximateThreshold) { + this(new Lucene99FlatVectorsFormat(new DefaultFlatVectorScorer()), approximateThreshold); + } + public NativeEngines990KnnVectorsFormat(final FlatVectorsFormat flatVectorsFormat) { this(flatVectorsFormat, KNNSettings.INDEX_KNN_ADVANCED_APPROXIMATE_THRESHOLD_DEFAULT_VALUE); } diff --git a/src/main/java/org/opensearch/knn/index/codec/KNN990Codec/UnitTestCodec.java b/src/main/java/org/opensearch/knn/index/codec/KNN990Codec/UnitTestCodec.java index 6998efbc3..d651f410a 100644 --- a/src/main/java/org/opensearch/knn/index/codec/KNN990Codec/UnitTestCodec.java +++ b/src/main/java/org/opensearch/knn/index/codec/KNN990Codec/UnitTestCodec.java @@ -21,6 +21,8 @@ * able to pick this class if its in test folder. Don't use this codec outside of testing */ public class UnitTestCodec extends FilterCodec { + private static final Integer BUILD_GRAPH_ALWAYS = 0; + public UnitTestCodec() { super("UnitTestCodec", KNNCodecVersion.current().getDefaultKnnCodecSupplier().get()); } @@ -30,7 +32,7 @@ public KnnVectorsFormat knnVectorsFormat() { return new PerFieldKnnVectorsFormat() { @Override public KnnVectorsFormat getKnnVectorsFormatForField(String field) { - return new NativeEngines990KnnVectorsFormat(); + return new NativeEngines990KnnVectorsFormat(BUILD_GRAPH_ALWAYS); } }; } diff --git a/src/test/java/org/opensearch/knn/KNNSingleNodeTestCase.java b/src/test/java/org/opensearch/knn/KNNSingleNodeTestCase.java index 06431bf07..587e80e5d 100644 --- a/src/test/java/org/opensearch/knn/KNNSingleNodeTestCase.java +++ b/src/test/java/org/opensearch/knn/KNNSingleNodeTestCase.java @@ -5,6 +5,7 @@ package org.opensearch.knn; +import org.opensearch.action.admin.indices.settings.put.UpdateSettingsRequest; import org.opensearch.core.action.ActionListener; import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.cluster.ClusterName; @@ -14,6 +15,7 @@ import org.opensearch.cluster.block.ClusterBlocks; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.xcontent.XContentHelper; +import org.opensearch.knn.index.KNNSettings; import org.opensearch.knn.index.query.KNNQueryBuilder; import org.opensearch.knn.index.memory.NativeMemoryCacheManager; import org.opensearch.knn.index.memory.NativeMemoryLoadStrategy; @@ -104,6 +106,14 @@ protected void createKnnIndexMapping(String indexName, String fieldName, Integer OpenSearchAssertions.assertAcked(client().admin().indices().putMapping(request).actionGet()); } + /** + * Create simple k-NN mapping with engine + */ + protected void updateIndexSetting(String indexName, Settings setting) { + UpdateSettingsRequest request = new UpdateSettingsRequest(setting, indexName); + OpenSearchAssertions.assertAcked(client().admin().indices().updateSettings(request).actionGet()); + } + /** * Create simple k-NN mapping which can be nested. * e.g. fieldPath = "a.b.c" will create mapping for "c" as knn_vector @@ -140,6 +150,18 @@ protected Settings getKNNDefaultIndexSettings() { return Settings.builder().put("number_of_shards", 1).put("number_of_replicas", 0).put("index.knn", true).build(); } + /** + * Get default k-NN settings for test cases with build graph always + */ + protected Settings getKNNDefaultIndexSettingsBuildsGraphAlways() { + return Settings.builder() + .put("number_of_shards", 1) + .put("number_of_replicas", 0) + .put("index.knn", true) + .put(KNNSettings.INDEX_KNN_ADVANCED_APPROXIMATE_THRESHOLD, 0) + .build(); + } + /** * Add a k-NN doc to an index */ diff --git a/src/test/java/org/opensearch/knn/index/KNNCircuitBreakerIT.java b/src/test/java/org/opensearch/knn/index/KNNCircuitBreakerIT.java index 11bde224a..fe0c880e4 100644 --- a/src/test/java/org/opensearch/knn/index/KNNCircuitBreakerIT.java +++ b/src/test/java/org/opensearch/knn/index/KNNCircuitBreakerIT.java @@ -20,6 +20,8 @@ * Integration tests to test Circuit Breaker functionality */ public class KNNCircuitBreakerIT extends KNNRestTestCase { + private static final Integer ALWAYS_BUILD_GRAPH = 0; + /** * To trip the circuit breaker, we will create two indices and index documents. Each index will be small enough so * that individually they fit into the cache, but together they do not. To prevent Lucene conditions where @@ -39,6 +41,7 @@ private void tripCb() throws Exception { .put("number_of_shards", 1) .put("number_of_replicas", numNodes - 1) .put("index.knn", true) + .put(KNNSettings.INDEX_KNN_ADVANCED_APPROXIMATE_THRESHOLD, ALWAYS_BUILD_GRAPH) .build(); String indexName1 = INDEX_NAME + "1"; diff --git a/src/test/java/org/opensearch/knn/index/KNNESSettingsTestIT.java b/src/test/java/org/opensearch/knn/index/KNNESSettingsTestIT.java index 09acc2599..19a57f0ac 100644 --- a/src/test/java/org/opensearch/knn/index/KNNESSettingsTestIT.java +++ b/src/test/java/org/opensearch/knn/index/KNNESSettingsTestIT.java @@ -21,6 +21,9 @@ import static org.hamcrest.Matchers.containsString; public class KNNESSettingsTestIT extends KNNRestTestCase { + + public static final int ALWAYS_BUILD_GRAPH = 0; + /** * KNN Index writes should be blocked when the plugin disabled * @throws Exception Exception from test @@ -72,7 +75,7 @@ public void testQueriesPluginDisabled() throws Exception { } public void testItemRemovedFromCache_expiration() throws Exception { - createKnnIndex(INDEX_NAME, createKnnIndexMapping(FIELD_NAME, 2)); + createKnnIndex(INDEX_NAME, buildKNNIndexSettings(ALWAYS_BUILD_GRAPH), createKnnIndexMapping(FIELD_NAME, 2)); updateClusterSettings(KNNSettings.KNN_CACHE_ITEM_EXPIRY_ENABLED, true); updateClusterSettings(KNNSettings.KNN_CACHE_ITEM_EXPIRY_TIME_MINUTES, "1m"); @@ -118,8 +121,8 @@ public void testUpdateIndexSetting() throws IOException { } @SuppressWarnings("unchecked") - public void testCacheRebuiltAfterUpdateIndexSettings() throws IOException { - createKnnIndex(INDEX_NAME, createKnnIndexMapping(FIELD_NAME, 2)); + public void testCacheRebuiltAfterUpdateIndexSettings() throws Exception { + createKnnIndex(INDEX_NAME, buildKNNIndexSettings(ALWAYS_BUILD_GRAPH), createKnnIndexMapping(FIELD_NAME, 2)); Float[] vector = { 6.0f, 6.0f }; addKnnDoc(INDEX_NAME, "1", FIELD_NAME, vector); diff --git a/src/test/java/org/opensearch/knn/index/KNNIndexShardTests.java b/src/test/java/org/opensearch/knn/index/KNNIndexShardTests.java index 18b9656e5..c25d2a390 100644 --- a/src/test/java/org/opensearch/knn/index/KNNIndexShardTests.java +++ b/src/test/java/org/opensearch/knn/index/KNNIndexShardTests.java @@ -14,6 +14,7 @@ import org.apache.lucene.util.StringHelper; import org.apache.lucene.util.Version; import org.mockito.Mockito; +import org.opensearch.common.settings.Settings; import org.opensearch.knn.KNNSingleNodeTestCase; import org.opensearch.index.IndexService; import org.opensearch.index.engine.Engine; @@ -71,6 +72,7 @@ public void testWarmup_emptyIndex() throws IOException { public void testWarmup_shardPresentInCache() throws InterruptedException, ExecutionException, IOException { IndexService indexService = createKNNIndex(testIndexName); createKnnIndexMapping(testIndexName, testFieldName, dimensions); + updateIndexSetting(testIndexName, Settings.builder().put(KNNSettings.INDEX_KNN_ADVANCED_APPROXIMATE_THRESHOLD, 0).build()); addKnnDoc(testIndexName, "1", testFieldName, new Float[] { 2.5F, 3.5F }); searchKNNIndex(testIndexName, testFieldName, new float[] { 1.0f, 2.0f }, 1); @@ -85,6 +87,7 @@ public void testWarmup_shardPresentInCache() throws InterruptedException, Execut public void testWarmup_shardNotPresentInCache() throws InterruptedException, ExecutionException, IOException { IndexService indexService = createKNNIndex(testIndexName); createKnnIndexMapping(testIndexName, testFieldName, dimensions); + updateIndexSetting(testIndexName, Settings.builder().put(KNNSettings.INDEX_KNN_ADVANCED_APPROXIMATE_THRESHOLD, 0).build()); IndexShard indexShard; KNNIndexShard knnIndexShard; @@ -106,6 +109,7 @@ public void testWarmup_shardNotPresentInCache() throws InterruptedException, Exe public void testGetAllEngineFileContexts() throws IOException, ExecutionException, InterruptedException { IndexService indexService = createKNNIndex(testIndexName); createKnnIndexMapping(testIndexName, testFieldName, dimensions); + updateIndexSetting(testIndexName, Settings.builder().put(KNNSettings.INDEX_KNN_ADVANCED_APPROXIMATE_THRESHOLD, 0).build()); IndexShard indexShard = indexService.iterator().next(); KNNIndexShard knnIndexShard = new KNNIndexShard(indexShard); @@ -204,6 +208,7 @@ public void testClearCache_emptyIndex() { public void testClearCache_shardPresentInCache() { IndexService indexService = createKNNIndex(testIndexName); createKnnIndexMapping(testIndexName, testFieldName, dimensions); + updateIndexSetting(testIndexName, Settings.builder().put(KNNSettings.INDEX_KNN_ADVANCED_APPROXIMATE_THRESHOLD, 0).build()); addKnnDoc(testIndexName, String.valueOf(randomInt()), testFieldName, new Float[] { randomFloat(), randomFloat() }); IndexShard indexShard = indexService.iterator().next(); diff --git a/src/test/java/org/opensearch/knn/index/NmslibIT.java b/src/test/java/org/opensearch/knn/index/NmslibIT.java index 113f84b3a..495fc55ba 100644 --- a/src/test/java/org/opensearch/knn/index/NmslibIT.java +++ b/src/test/java/org/opensearch/knn/index/NmslibIT.java @@ -155,7 +155,7 @@ public void testEndToEnd() throws Exception { Map mappingMap = xContentBuilderToMap(builder); String mapping = builder.toString(); - createKnnIndex(indexName, mapping); + createKnnIndex(indexName, buildKNNIndexSettings(0), mapping); assertEquals(new TreeMap<>(mappingMap), new TreeMap<>(getIndexMappingAsMap(indexName))); // Index the test data diff --git a/src/test/java/org/opensearch/knn/index/OpenSearchIT.java b/src/test/java/org/opensearch/knn/index/OpenSearchIT.java index 99aea03df..3d6137db9 100644 --- a/src/test/java/org/opensearch/knn/index/OpenSearchIT.java +++ b/src/test/java/org/opensearch/knn/index/OpenSearchIT.java @@ -113,7 +113,7 @@ public void testEndToEnd() throws Exception { Map mappingMap = xContentBuilderToMap(builder); String mapping = builder.toString(); - createKnnIndex(indexName, mapping); + createKnnIndex(indexName, buildKNNIndexSettings(0), mapping); assertEquals(new TreeMap<>(mappingMap), new TreeMap<>(getIndexMappingAsMap(indexName))); // Index the test data diff --git a/src/test/java/org/opensearch/knn/plugin/action/RestClearCacheHandlerIT.java b/src/test/java/org/opensearch/knn/plugin/action/RestClearCacheHandlerIT.java index 2618519f2..2b9f8a82f 100644 --- a/src/test/java/org/opensearch/knn/plugin/action/RestClearCacheHandlerIT.java +++ b/src/test/java/org/opensearch/knn/plugin/action/RestClearCacheHandlerIT.java @@ -25,6 +25,7 @@ public class RestClearCacheHandlerIT extends KNNRestTestCase { private static final String TEST_FIELD = "test-field"; private static final int DIMENSIONS = 2; + public static final int ALWAYS_BUILD_GRAPH = 0; @SneakyThrows public void testNonExistentIndex() { @@ -53,7 +54,7 @@ public void testNotKnnIndex() { public void testClearCacheSingleIndex() { String testIndex = getTestName().toLowerCase(); int graphCountBefore = getTotalGraphsInCache(); - createKnnIndex(testIndex, getKNNDefaultIndexSettings(), createKnnIndexMapping(TEST_FIELD, DIMENSIONS)); + createKnnIndex(testIndex, buildKNNIndexSettings(ALWAYS_BUILD_GRAPH), createKnnIndexMapping(TEST_FIELD, DIMENSIONS)); addKnnDoc(testIndex, String.valueOf(randomInt()), TEST_FIELD, new Float[] { randomFloat(), randomFloat() }); knnWarmup(Collections.singletonList(testIndex)); @@ -70,10 +71,10 @@ public void testClearCacheMultipleIndices() { String testIndex2 = getTestName().toLowerCase() + 1; int graphCountBefore = getTotalGraphsInCache(); - createKnnIndex(testIndex1, getKNNDefaultIndexSettings(), createKnnIndexMapping(TEST_FIELD, DIMENSIONS)); + createKnnIndex(testIndex1, buildKNNIndexSettings(ALWAYS_BUILD_GRAPH), createKnnIndexMapping(TEST_FIELD, DIMENSIONS)); addKnnDoc(testIndex1, String.valueOf(randomInt()), TEST_FIELD, new Float[] { randomFloat(), randomFloat() }); - createKnnIndex(testIndex2, getKNNDefaultIndexSettings(), createKnnIndexMapping(TEST_FIELD, DIMENSIONS)); + createKnnIndex(testIndex2, buildKNNIndexSettings(0), createKnnIndexMapping(TEST_FIELD, DIMENSIONS)); addKnnDoc(testIndex2, String.valueOf(randomInt()), TEST_FIELD, new Float[] { randomFloat(), randomFloat() }); knnWarmup(Arrays.asList(testIndex1, testIndex2)); @@ -91,13 +92,13 @@ public void testClearCacheMultipleIndicesWithPatterns() { String testIndex3 = "abc" + getTestName().toLowerCase(); int graphCountBefore = getTotalGraphsInCache(); - createKnnIndex(testIndex1, getKNNDefaultIndexSettings(), createKnnIndexMapping(TEST_FIELD, DIMENSIONS)); + createKnnIndex(testIndex1, buildKNNIndexSettings(ALWAYS_BUILD_GRAPH), createKnnIndexMapping(TEST_FIELD, DIMENSIONS)); addKnnDoc(testIndex1, String.valueOf(randomInt()), TEST_FIELD, new Float[] { randomFloat(), randomFloat() }); - createKnnIndex(testIndex2, getKNNDefaultIndexSettings(), createKnnIndexMapping(TEST_FIELD, DIMENSIONS)); + createKnnIndex(testIndex2, buildKNNIndexSettings(ALWAYS_BUILD_GRAPH), createKnnIndexMapping(TEST_FIELD, DIMENSIONS)); addKnnDoc(testIndex2, String.valueOf(randomInt()), TEST_FIELD, new Float[] { randomFloat(), randomFloat() }); - createKnnIndex(testIndex3, getKNNDefaultIndexSettings(), createKnnIndexMapping(TEST_FIELD, DIMENSIONS)); + createKnnIndex(testIndex3, buildKNNIndexSettings(ALWAYS_BUILD_GRAPH), createKnnIndexMapping(TEST_FIELD, DIMENSIONS)); addKnnDoc(testIndex3, String.valueOf(randomInt()), TEST_FIELD, new Float[] { randomFloat(), randomFloat() }); knnWarmup(Arrays.asList(testIndex1, testIndex2, testIndex3)); diff --git a/src/test/java/org/opensearch/knn/plugin/action/RestKNNStatsHandlerIT.java b/src/test/java/org/opensearch/knn/plugin/action/RestKNNStatsHandlerIT.java index 850f24bb7..be71ff5bd 100644 --- a/src/test/java/org/opensearch/knn/plugin/action/RestKNNStatsHandlerIT.java +++ b/src/test/java/org/opensearch/knn/plugin/action/RestKNNStatsHandlerIT.java @@ -109,7 +109,7 @@ public void testStatsValueCheck() throws IOException { Integer knnQueryWithFilterCount0 = (Integer) nodeStats0.get(StatNames.KNN_QUERY_WITH_FILTER_REQUESTS.getName()); // Setup index - createKnnIndex(INDEX_NAME, createKnnIndexMapping(FIELD_NAME, 2)); + createKnnIndex(INDEX_NAME, buildKNNIndexSettings(0), createKnnIndexMapping(FIELD_NAME, 2)); // Index test document Float[] vector = { 6.0f, 6.0f }; @@ -392,7 +392,7 @@ public void testModelIndexingDegradedMetricsStats() throws IOException { * @throws IOException throws IOException */ public void testFieldByEngineStats() throws Exception { - createKnnIndex(INDEX_NAME, createKnnIndexMapping(FIELD_NAME, 2, METHOD_HNSW, NMSLIB_NAME)); + createKnnIndex(INDEX_NAME, buildKNNIndexSettings(0), createKnnIndexMapping(FIELD_NAME, 2, METHOD_HNSW, NMSLIB_NAME)); putMappingRequest(INDEX_NAME, createKnnIndexMapping(FIELD_NAME_2, 3, METHOD_HNSW, LUCENE_NAME)); putMappingRequest(INDEX_NAME, createKnnIndexMapping(FIELD_NAME_3, 3, METHOD_HNSW, FAISS_NAME)); diff --git a/src/test/java/org/opensearch/knn/plugin/action/RestKNNWarmupHandlerIT.java b/src/test/java/org/opensearch/knn/plugin/action/RestKNNWarmupHandlerIT.java index d433cd285..bc1865522 100644 --- a/src/test/java/org/opensearch/knn/plugin/action/RestKNNWarmupHandlerIT.java +++ b/src/test/java/org/opensearch/knn/plugin/action/RestKNNWarmupHandlerIT.java @@ -47,7 +47,7 @@ public void testEmptyIndex() throws Exception { public void testSingleIndex() throws Exception { int graphCountBefore = getTotalGraphsInCache(); - createKnnIndex(testIndexName, getKNNDefaultIndexSettings(), createKnnIndexMapping(testFieldName, dimensions)); + createKnnIndex(testIndexName, buildKNNIndexSettings(0), createKnnIndexMapping(testFieldName, dimensions)); addKnnDoc(testIndexName, "1", testFieldName, new Float[] { 6.0f, 6.0f }); knnWarmup(Collections.singletonList(testIndexName)); @@ -58,10 +58,10 @@ public void testSingleIndex() throws Exception { public void testMultipleIndices() throws Exception { int graphCountBefore = getTotalGraphsInCache(); - createKnnIndex(testIndexName + "1", getKNNDefaultIndexSettings(), createKnnIndexMapping(testFieldName, dimensions)); + createKnnIndex(testIndexName + "1", buildKNNIndexSettings(0), createKnnIndexMapping(testFieldName, dimensions)); addKnnDoc(testIndexName + "1", "1", testFieldName, new Float[] { 6.0f, 6.0f }); - createKnnIndex(testIndexName + "2", getKNNDefaultIndexSettings(), createKnnIndexMapping(testFieldName, dimensions)); + createKnnIndex(testIndexName + "2", buildKNNIndexSettings(0), createKnnIndexMapping(testFieldName, dimensions)); addKnnDoc(testIndexName + "2", "1", testFieldName, new Float[] { 6.0f, 6.0f }); knnWarmup(Arrays.asList(testIndexName + "1", testIndexName + "2")); diff --git a/src/test/java/org/opensearch/knn/plugin/action/RestLegacyKNNStatsHandlerIT.java b/src/test/java/org/opensearch/knn/plugin/action/RestLegacyKNNStatsHandlerIT.java index 90dd12947..631d63874 100644 --- a/src/test/java/org/opensearch/knn/plugin/action/RestLegacyKNNStatsHandlerIT.java +++ b/src/test/java/org/opensearch/knn/plugin/action/RestLegacyKNNStatsHandlerIT.java @@ -82,7 +82,7 @@ public void testStatsValueCheck() throws IOException { Integer missCount0 = (Integer) nodeStats0.get(StatNames.MISS_COUNT.getName()); // Setup index - createKnnIndex(INDEX_NAME, createKnnIndexMapping(FIELD_NAME, 2)); + createKnnIndex(INDEX_NAME, buildKNNIndexSettings(0), createKnnIndexMapping(FIELD_NAME, 2)); // Index test document Float[] vector = { 6.0f, 6.0f }; diff --git a/src/test/java/org/opensearch/knn/plugin/action/RestLegacyKNNWarmupHandlerIT.java b/src/test/java/org/opensearch/knn/plugin/action/RestLegacyKNNWarmupHandlerIT.java index e6345faba..e0fe9ec5c 100644 --- a/src/test/java/org/opensearch/knn/plugin/action/RestLegacyKNNWarmupHandlerIT.java +++ b/src/test/java/org/opensearch/knn/plugin/action/RestLegacyKNNWarmupHandlerIT.java @@ -27,6 +27,7 @@ public class RestLegacyKNNWarmupHandlerIT extends KNNRestTestCase { + public static final int ALWAYS_BUILD_GRAPH = 0; private final String testIndexName = "test-index"; private final String testFieldName = "test-field"; private final int dimensions = 2; @@ -45,7 +46,7 @@ public void testNonKnnIndex() throws IOException { public void testEmptyIndex() throws Exception { int graphCountBefore = getTotalGraphsInCache(); - createKnnIndex(testIndexName, getKNNDefaultIndexSettings(), createKnnIndexMapping(testFieldName, dimensions)); + createKnnIndex(testIndexName, buildKNNIndexSettings(ALWAYS_BUILD_GRAPH), createKnnIndexMapping(testFieldName, dimensions)); executeWarmupRequest(Collections.singletonList(testIndexName), KNNPlugin.LEGACY_KNN_BASE_URI); @@ -54,7 +55,7 @@ public void testEmptyIndex() throws Exception { public void testSingleIndex() throws Exception { int graphCountBefore = getTotalGraphsInCache(); - createKnnIndex(testIndexName, getKNNDefaultIndexSettings(), createKnnIndexMapping(testFieldName, dimensions)); + createKnnIndex(testIndexName, buildKNNIndexSettings(ALWAYS_BUILD_GRAPH), createKnnIndexMapping(testFieldName, dimensions)); addKnnDoc(testIndexName, "1", testFieldName, new Float[] { 6.0f, 6.0f }); executeWarmupRequest(Collections.singletonList(testIndexName), KNNPlugin.LEGACY_KNN_BASE_URI); @@ -65,10 +66,10 @@ public void testSingleIndex() throws Exception { public void testMultipleIndices() throws Exception { int graphCountBefore = getTotalGraphsInCache(); - createKnnIndex(testIndexName + "1", getKNNDefaultIndexSettings(), createKnnIndexMapping(testFieldName, dimensions)); + createKnnIndex(testIndexName + "1", buildKNNIndexSettings(0), createKnnIndexMapping(testFieldName, dimensions)); addKnnDoc(testIndexName + "1", "1", testFieldName, new Float[] { 6.0f, 6.0f }); - createKnnIndex(testIndexName + "2", getKNNDefaultIndexSettings(), createKnnIndexMapping(testFieldName, dimensions)); + createKnnIndex(testIndexName + "2", buildKNNIndexSettings(0), createKnnIndexMapping(testFieldName, dimensions)); addKnnDoc(testIndexName + "2", "1", testFieldName, new Float[] { 6.0f, 6.0f }); executeWarmupRequest(Arrays.asList(testIndexName + "1", testIndexName + "2"), KNNPlugin.LEGACY_KNN_BASE_URI); diff --git a/src/test/java/org/opensearch/knn/plugin/transport/ClearCacheTransportActionTests.java b/src/test/java/org/opensearch/knn/plugin/transport/ClearCacheTransportActionTests.java index 3222a3eb7..d79f2f738 100644 --- a/src/test/java/org/opensearch/knn/plugin/transport/ClearCacheTransportActionTests.java +++ b/src/test/java/org/opensearch/knn/plugin/transport/ClearCacheTransportActionTests.java @@ -33,7 +33,7 @@ public void testShardOperation() { KNNWarmupTransportAction knnWarmupTransportAction = node().injector().getInstance(KNNWarmupTransportAction.class); assertEquals(0, NativeMemoryCacheManager.getInstance().getIndicesCacheStats().size()); - IndexService indexService = createKNNIndex(testIndex); + IndexService indexService = createIndex(testIndex, getKNNDefaultIndexSettingsBuildsGraphAlways()); createKnnIndexMapping(testIndex, TEST_FIELD, DIMENSIONS); addKnnDoc(testIndex, String.valueOf(randomInt()), TEST_FIELD, new Float[] { randomFloat(), randomFloat() }); ShardRouting shardRouting = indexService.iterator().next().routingEntry(); diff --git a/src/test/java/org/opensearch/knn/plugin/transport/KNNWarmupTransportActionTests.java b/src/test/java/org/opensearch/knn/plugin/transport/KNNWarmupTransportActionTests.java index 1f72f78a7..f4b52246d 100644 --- a/src/test/java/org/opensearch/knn/plugin/transport/KNNWarmupTransportActionTests.java +++ b/src/test/java/org/opensearch/knn/plugin/transport/KNNWarmupTransportActionTests.java @@ -37,7 +37,7 @@ public void testShardOperation() throws IOException, ExecutionException, Interru KNNWarmupTransportAction knnWarmupTransportAction = node().injector().getInstance(KNNWarmupTransportAction.class); assertEquals(0, NativeMemoryCacheManager.getInstance().getIndicesCacheStats().size()); - indexService = createKNNIndex(testIndexName); + indexService = createIndex(testIndexName, getKNNDefaultIndexSettingsBuildsGraphAlways()); createKnnIndexMapping(testIndexName, testFieldName, dimensions); shardRouting = indexService.iterator().next().routingEntry(); diff --git a/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java b/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java index cce70838f..8aa8d984f 100644 --- a/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java +++ b/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java @@ -1309,6 +1309,7 @@ public void addKNNDocs(String testIndex, String testField, int dimension, int fi Arrays.fill(indexVector, (float) i); addKnnDoc(testIndex, Integer.toString(i), testField, indexVector); } + flushIndex(testIndex); } public void addKNNByteDocs(String testIndex, String testField, int dimension, int firstDocID, int numDocs) throws IOException { @@ -1317,6 +1318,7 @@ public void addKNNByteDocs(String testIndex, String testField, int dimension, in Arrays.fill(indexVector, (byte) i); addKnnDoc(testIndex, Integer.toString(i), testField, indexVector); } + flushIndex(testIndex); } public void validateKNNSearch(String testIndex, String testField, int dimension, int numDocs, int k) throws Exception { @@ -1792,6 +1794,13 @@ protected void refreshIndex(final String index) throws IOException { assertEquals(request.getEndpoint() + ": failed", RestStatus.OK, RestStatus.fromCode(response.getStatusLine().getStatusCode())); } + protected void flushIndex(final String index) throws IOException { + Request request = new Request("POST", "/" + index + "/_flush"); + + Response response = client().performRequest(request); + assertEquals(request.getEndpoint() + ": failed", RestStatus.OK, RestStatus.fromCode(response.getStatusLine().getStatusCode())); + } + protected void addKnnDocWithAttributes(String docId, float[] vector, Map fieldValues) throws IOException { Request request = new Request("POST", "/" + INDEX_NAME + "/_doc/" + docId + "?refresh=true"); From 194478343e3534828a8bc2123b2c02774043567d Mon Sep 17 00:00:00 2001 From: Vijayan Balasubramanian Date: Fri, 25 Oct 2024 13:56:27 -0700 Subject: [PATCH 407/416] Update bwc test to update index setting (#2236) (#2238) Signed-off-by: Vijayan Balasubramanian --- CHANGELOG.md | 1 + .../org/opensearch/knn/bwc/ClearCacheIT.java | 9 +++++- .../org/opensearch/knn/bwc/IndexingIT.java | 16 +++++------ .../java/org/opensearch/knn/bwc/WarmupIT.java | 22 +++++++++++---- .../org/opensearch/knn/bwc/ClearCacheIT.java | 7 ++++- .../java/org/opensearch/knn/bwc/WarmupIT.java | 5 +++- .../org/opensearch/knn/KNNRestTestCase.java | 28 +++++++++++++++++-- 7 files changed, 69 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cb1005678..be168322b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,4 +20,5 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Infrastructure ### Documentation ### Maintenance +* Select index settings based on cluster version[2236](https://github.com/opensearch-project/k-NN/pull/2236) ### Refactoring diff --git a/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/ClearCacheIT.java b/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/ClearCacheIT.java index f93f2f883..e4ac25000 100644 --- a/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/ClearCacheIT.java +++ b/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/ClearCacheIT.java @@ -5,7 +5,10 @@ package org.opensearch.knn.bwc; +import org.opensearch.common.settings.Settings; + import java.util.Collections; + import static org.opensearch.knn.TestUtils.NODES_BWC_CLUSTER; public class ClearCacheIT extends AbstractRestartUpgradeTestCase { @@ -20,7 +23,11 @@ public class ClearCacheIT extends AbstractRestartUpgradeTestCase { public void testClearCache() throws Exception { waitForClusterHealthGreen(NODES_BWC_CLUSTER); if (isRunningAgainstOldCluster()) { - createKnnIndex(testIndex, getKNNDefaultIndexSettings(), createKnnIndexMapping(TEST_FIELD, DIMENSIONS)); + // if approximate threshold is supported, set value to 0, to build graph always + Settings indexSettings = isApproximateThresholdSupported(getBWCVersion()) + ? buildKNNIndexSettings(0) + : getKNNDefaultIndexSettings(); + createKnnIndex(testIndex, indexSettings, createKnnIndexMapping(TEST_FIELD, DIMENSIONS)); addKNNDocs(testIndex, TEST_FIELD, DIMENSIONS, docId, NUM_DOCS); queryCnt = NUM_DOCS; int graphCount = getTotalGraphsInCache(); diff --git a/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/IndexingIT.java b/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/IndexingIT.java index bf4c8f7ec..7d2b3807a 100644 --- a/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/IndexingIT.java +++ b/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/IndexingIT.java @@ -143,15 +143,15 @@ public void testKNNIndexCustomLegacyFieldMapping() throws Exception { // When the cluster is in old version, create a KNN index with custom legacy field mapping settings // and add documents into that index if (isRunningAgainstOldCluster()) { - createKnnIndex( - testIndex, - createKNNIndexCustomLegacyFieldMappingSettings( - SpaceType.LINF, - KNN_ALGO_PARAM_M_MIN_VALUE, - KNN_ALGO_PARAM_EF_CONSTRUCTION_MIN_VALUE - ), - createKnnIndexMapping(TEST_FIELD, DIMENSIONS) + Settings.Builder indexMappingSettings = createKNNIndexCustomLegacyFieldMappingIndexSettingsBuilder( + SpaceType.LINF, + KNN_ALGO_PARAM_M_MIN_VALUE, + KNN_ALGO_PARAM_EF_CONSTRUCTION_MIN_VALUE ); + if (isApproximateThresholdSupported(getBWCVersion())) { + indexMappingSettings.put(KNNSettings.INDEX_KNN_ADVANCED_APPROXIMATE_THRESHOLD, 0); + } + createKnnIndex(testIndex, indexMappingSettings.build(), createKnnIndexMapping(TEST_FIELD, DIMENSIONS)); addKNNDocs(testIndex, TEST_FIELD, DIMENSIONS, DOC_ID, NUM_DOCS); } else { validateKNNIndexingOnUpgrade(NUM_DOCS); diff --git a/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/WarmupIT.java b/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/WarmupIT.java index db3555a8d..64c55f6fd 100644 --- a/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/WarmupIT.java +++ b/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/WarmupIT.java @@ -32,7 +32,10 @@ public void testKNNWarmupDefaultLegacyFieldMapping() throws Exception { waitForClusterHealthGreen(NODES_BWC_CLUSTER); if (isRunningAgainstOldCluster()) { - createKnnIndex(testIndex, getKNNDefaultIndexSettings(), createKnnIndexMapping(TEST_FIELD, DIMENSIONS)); + Settings indexSettings = isApproximateThresholdSupported(getBWCVersion()) + ? buildKNNIndexSettings(0) + : getKNNDefaultIndexSettings(); + createKnnIndex(testIndex, indexSettings, createKnnIndexMapping(TEST_FIELD, DIMENSIONS)); addKNNDocs(testIndex, TEST_FIELD, DIMENSIONS, DOC_ID, NUM_DOCS); } else { // update index setting to allow build graph always since we test graph count that are loaded into memory @@ -48,13 +51,16 @@ public void testKNNWarmupCustomLegacyFieldMapping() throws Exception { // When the cluster is in old version, create a KNN index with custom legacy field mapping settings // and add documents into that index if (isRunningAgainstOldCluster()) { - Settings indexMappingSettings = createKNNIndexCustomLegacyFieldMappingSettings( + Settings.Builder indexMappingSettings = createKNNIndexCustomLegacyFieldMappingIndexSettingsBuilder( SpaceType.LINF, KNN_ALGO_PARAM_M_MIN_VALUE, KNN_ALGO_PARAM_EF_CONSTRUCTION_MIN_VALUE ); + if (isApproximateThresholdSupported(getBWCVersion())) { + indexMappingSettings.put(KNNSettings.INDEX_KNN_ADVANCED_APPROXIMATE_THRESHOLD, 0); + } String indexMapping = createKnnIndexMapping(TEST_FIELD, DIMENSIONS); - createKnnIndex(testIndex, indexMappingSettings, indexMapping); + createKnnIndex(testIndex, indexMappingSettings.build(), indexMapping); addKNNDocs(testIndex, TEST_FIELD, DIMENSIONS, DOC_ID, NUM_DOCS); } else { validateKNNWarmupOnUpgrade(); @@ -65,7 +71,10 @@ public void testKNNWarmupCustomLegacyFieldMapping() throws Exception { // space_type : "l2", engine : "nmslib", m : 16, ef_construction : 512 public void testKNNWarmupDefaultMethodFieldMapping() throws Exception { if (isRunningAgainstOldCluster()) { - createKnnIndex(testIndex, getKNNDefaultIndexSettings(), createKNNIndexMethodFieldMapping(TEST_FIELD, DIMENSIONS)); + Settings indexSettings = isApproximateThresholdSupported(getBWCVersion()) + ? buildKNNIndexSettings(0) + : getKNNDefaultIndexSettings(); + createKnnIndex(testIndex, indexSettings, createKNNIndexMethodFieldMapping(TEST_FIELD, DIMENSIONS)); addKNNDocs(testIndex, TEST_FIELD, DIMENSIONS, DOC_ID, NUM_DOCS); } else { // update index setting to allow build graph always since we test graph count that are loaded into memory @@ -78,9 +87,12 @@ public void testKNNWarmupDefaultMethodFieldMapping() throws Exception { // space_type : "innerproduct", engine : "faiss", m : 50, ef_construction : 1024 public void testKNNWarmupCustomMethodFieldMapping() throws Exception { if (isRunningAgainstOldCluster()) { + Settings indexSettings = isApproximateThresholdSupported(getBWCVersion()) + ? buildKNNIndexSettings(0) + : getKNNDefaultIndexSettings(); createKnnIndex( testIndex, - getKNNDefaultIndexSettings(), + indexSettings, createKNNIndexCustomMethodFieldMapping(TEST_FIELD, DIMENSIONS, SpaceType.INNER_PRODUCT, FAISS_NAME, M, EF_CONSTRUCTION) ); addKNNDocs(testIndex, TEST_FIELD, DIMENSIONS, DOC_ID, NUM_DOCS); diff --git a/qa/rolling-upgrade/src/test/java/org/opensearch/knn/bwc/ClearCacheIT.java b/qa/rolling-upgrade/src/test/java/org/opensearch/knn/bwc/ClearCacheIT.java index 5ef3b1d97..0e8748eb6 100644 --- a/qa/rolling-upgrade/src/test/java/org/opensearch/knn/bwc/ClearCacheIT.java +++ b/qa/rolling-upgrade/src/test/java/org/opensearch/knn/bwc/ClearCacheIT.java @@ -5,6 +5,8 @@ package org.opensearch.knn.bwc; +import org.opensearch.common.settings.Settings; + import java.util.Collections; import static org.opensearch.knn.TestUtils.NODES_BWC_CLUSTER; @@ -22,7 +24,10 @@ public void testClearCache() throws Exception { waitForClusterHealthGreen(NODES_BWC_CLUSTER); switch (getClusterType()) { case OLD: - createKnnIndex(testIndex, getKNNDefaultIndexSettings(), createKnnIndexMapping(TEST_FIELD, DIMENSIONS)); + Settings indexSettings = isApproximateThresholdSupported(getBWCVersion()) + ? buildKNNIndexSettings(0) + : getKNNDefaultIndexSettings(); + createKnnIndex(testIndex, indexSettings, createKnnIndexMapping(TEST_FIELD, DIMENSIONS)); int docIdOld = 0; addKNNDocs(testIndex, TEST_FIELD, DIMENSIONS, docIdOld, NUM_DOCS); int graphCount = getTotalGraphsInCache(); diff --git a/qa/rolling-upgrade/src/test/java/org/opensearch/knn/bwc/WarmupIT.java b/qa/rolling-upgrade/src/test/java/org/opensearch/knn/bwc/WarmupIT.java index 7d2a796a0..7e7e9c1df 100644 --- a/qa/rolling-upgrade/src/test/java/org/opensearch/knn/bwc/WarmupIT.java +++ b/qa/rolling-upgrade/src/test/java/org/opensearch/knn/bwc/WarmupIT.java @@ -23,7 +23,10 @@ public void testKNNWarmup() throws Exception { waitForClusterHealthGreen(NODES_BWC_CLUSTER); switch (getClusterType()) { case OLD: - createKnnIndex(testIndex, getKNNDefaultIndexSettings(), createKnnIndexMapping(TEST_FIELD, DIMENSIONS)); + Settings indexSettings = isApproximateThresholdSupported(getBWCVersion()) + ? buildKNNIndexSettings(0) + : getKNNDefaultIndexSettings(); + createKnnIndex(testIndex, indexSettings, createKnnIndexMapping(TEST_FIELD, DIMENSIONS)); addKNNDocs(testIndex, TEST_FIELD, DIMENSIONS, 0, NUM_DOCS); break; case MIXED: diff --git a/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java b/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java index 8aa8d984f..3cef7f5b7 100644 --- a/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java +++ b/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java @@ -13,6 +13,7 @@ import org.apache.commons.lang.StringUtils; import org.apache.http.client.utils.URIBuilder; import org.apache.http.util.EntityUtils; +import org.opensearch.Version; import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.common.xcontent.XContentHelper; import org.opensearch.core.xcontent.DeprecationHandler; @@ -64,6 +65,7 @@ import java.util.Locale; import java.util.Map; import java.util.Objects; +import java.util.Optional; import java.util.PriorityQueue; import java.util.Set; import java.util.concurrent.TimeUnit; @@ -1345,15 +1347,22 @@ public void validateKNNSearch(String testIndex, String testField, int dimension, } } - protected Settings createKNNIndexCustomLegacyFieldMappingSettings(SpaceType spaceType, Integer m, Integer ef_construction) { + protected Settings.Builder createKNNIndexCustomLegacyFieldMappingIndexSettingsBuilder( + SpaceType spaceType, + Integer m, + Integer ef_construction + ) { return Settings.builder() .put(NUMBER_OF_SHARDS, 1) .put(NUMBER_OF_REPLICAS, 0) .put(INDEX_KNN, true) .put(KNNSettings.KNN_SPACE_TYPE, spaceType.getValue()) .put(KNNSettings.KNN_ALGO_PARAM_M, m) - .put(KNNSettings.KNN_ALGO_PARAM_EF_CONSTRUCTION, ef_construction) - .build(); + .put(KNNSettings.KNN_ALGO_PARAM_EF_CONSTRUCTION, ef_construction); + } + + protected Settings createKNNIndexCustomLegacyFieldMappingIndexSettings(SpaceType spaceType, Integer m, Integer ef_construction) { + return createKNNIndexCustomLegacyFieldMappingIndexSettingsBuilder(spaceType, m, ef_construction).build(); } public String createKNNIndexMethodFieldMapping(String fieldName, Integer dimensions) throws IOException { @@ -1859,4 +1868,17 @@ protected XContentBuilder buildSearchQuery(String fieldName, int k, float[] vect builder.endObject().endObject().endObject().endObject(); return builder; } + + // approximate threshold parameter is only supported on or after V_2_18_0 + protected boolean isApproximateThresholdSupported(final Optional bwcVersion) { + if (bwcVersion.isEmpty()) { + return false; + } + String versionString = bwcVersion.get(); + if (versionString.endsWith("-SNAPSHOT")) { + versionString = versionString.substring(0, versionString.length() - 9); + } + final Version version = Version.fromString(versionString); + return version.onOrAfter(Version.V_2_18_0); + } } From 7d020409d8474a90ba0f915755f06a7ddc943b49 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Tue, 5 Nov 2024 09:12:41 -0800 Subject: [PATCH 408/416] Added 'j' option to use multiple cpus to build JNI library. (#2244) (#2245) Signed-off-by: Dooyong Kim Co-authored-by: Dooyong Kim (cherry picked from commit a029fa8c9664d6c37b2c78dbb22e0d16b7b8878a) Co-authored-by: Doo Yong Kim <0ctopus13prime@gmail.com> --- scripts/build.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) mode change 100644 => 100755 scripts/build.sh diff --git a/scripts/build.sh b/scripts/build.sh old mode 100644 new mode 100755 index be7304ee5..203b76c99 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -18,10 +18,11 @@ function usage() { echo -e "-p PLATFORM\t[Optional] Platform, ignored." echo -e "-a ARCHITECTURE\t[Optional] Build architecture, ignored." echo -e "-o OUTPUT\t[Optional] Output path, default is 'artifacts'." + echo -e "-j NPROC_COUNT\t[Optional] Number of CPUs to use when building JNI library. Default is 1." echo -e "-h help" } -while getopts ":h:v:q:s:o:p:a:" arg; do +while getopts ":h:v:q:s:o:p:a:j:" arg; do case $arg in h) usage @@ -45,6 +46,9 @@ while getopts ":h:v:q:s:o:p:a:" arg; do a) ARCHITECTURE=$OPTARG ;; + j) + NPROC_COUNT=$OPTARG + ;; :) echo "Error: -${OPTARG} requires an argument" usage @@ -118,7 +122,7 @@ fi # Build k-NN lib and plugin through gradle tasks cd $work_dir ./gradlew build --no-daemon --refresh-dependencies -x integTest -x test -Dopensearch.version=$VERSION -Dbuild.snapshot=$SNAPSHOT -Dbuild.version_qualifier=$QUALIFIER -Dbuild.lib.commit_patches=false -./gradlew :buildJniLib -Davx512.enabled=false -Davx2.enabled=false -Dbuild.lib.commit_patches=false +./gradlew :buildJniLib -Davx512.enabled=false -Davx2.enabled=false -Dbuild.lib.commit_patches=false -Dnproc.count=${NPROC_COUNT:-1} if [ "$PLATFORM" != "windows" ] && [ "$ARCHITECTURE" = "x64" ]; then echo "Building k-NN library after enabling AVX2" From 2851844ff652c8f98800f42019dc755a41dc20a7 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Wed, 6 Nov 2024 16:31:58 -0800 Subject: [PATCH 409/416] Increment version to 2.19.0-SNAPSHOT (#2237) Signed-off-by: opensearch-ci-bot Co-authored-by: opensearch-ci-bot --- .../workflows/backwards_compatibility_tests_workflow.yml | 8 ++++---- build.gradle | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/backwards_compatibility_tests_workflow.yml b/.github/workflows/backwards_compatibility_tests_workflow.yml index 91a46e2e8..3229b5d69 100644 --- a/.github/workflows/backwards_compatibility_tests_workflow.yml +++ b/.github/workflows/backwards_compatibility_tests_workflow.yml @@ -34,8 +34,8 @@ jobs: strategy: matrix: java: [ 11, 17 ] - bwc_version : [ "1.1.0", "1.2.4", "1.3.8", "2.0.1", "2.1.0", "2.2.1", "2.3.0", "2.4.1", "2.5.0", "2.6.0", "2.7.0", "2.8.0", "2.9.0", "2.10.0", "2.11.0", "2.12.0", "2.13.0", "2.14.0", "2.15.0", "2.16.0", "2.17.0" ] - opensearch_version : [ "2.18.0-SNAPSHOT" ] + bwc_version : [ "1.1.0", "1.2.4", "1.3.8", "2.0.1", "2.1.0", "2.2.1", "2.3.0", "2.4.1", "2.5.0", "2.6.0", "2.7.0", "2.8.0", "2.9.0", "2.10.0", "2.11.0", "2.12.0", "2.13.0", "2.14.0", "2.15.0", "2.16.0", "2.17.0", "2.18.0" ] + opensearch_version : [ "2.19.0-SNAPSHOT" ] name: k-NN Restart-Upgrade BWC Tests runs-on: ubuntu-latest @@ -82,8 +82,8 @@ jobs: strategy: matrix: java: [ 11, 17 ] - bwc_version: [ "1.3.8", "2.0.1", "2.1.0", "2.2.1", "2.3.0", "2.4.1", "2.5.0", "2.6.0", "2.7.0", "2.8.0", "2.9.0", "2.10.0", "2.11.0", "2.12.0", "2.13.0", "2.14.0", "2.15.0", "2.16.0", "2.17.0"] - opensearch_version: [ "2.18.0-SNAPSHOT" ] + bwc_version: [ "1.3.8", "2.0.1", "2.1.0", "2.2.1", "2.3.0", "2.4.1", "2.5.0", "2.6.0", "2.7.0", "2.8.0", "2.9.0", "2.10.0", "2.11.0", "2.12.0", "2.13.0", "2.14.0", "2.15.0", "2.16.0", "2.17.0", "2.18.0"] + opensearch_version: [ "2.19.0-SNAPSHOT" ] name: k-NN Rolling-Upgrade BWC Tests runs-on: ubuntu-latest diff --git a/build.gradle b/build.gradle index 31ada6681..a71af5f08 100644 --- a/build.gradle +++ b/build.gradle @@ -13,7 +13,7 @@ buildscript { ext { // build.version_qualifier parameter applies to knn plugin artifacts only. OpenSearch version must be set // explicitly as 'opensearch.version' property, for instance opensearch.version=2.0.0-rc1-SNAPSHOT - opensearch_version = System.getProperty("opensearch.version", "2.18.0-SNAPSHOT") + opensearch_version = System.getProperty("opensearch.version", "2.19.0-SNAPSHOT") version_qualifier = System.getProperty("build.version_qualifier", "") opensearch_group = "org.opensearch" isSnapshot = "true" == System.getProperty("build.snapshot", "true") From 3d1649d3d930e115682a1a4c542e41008fbb3396 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Thu, 7 Nov 2024 12:08:05 -0800 Subject: [PATCH 410/416] Fix backport branch deletion on merge (#2255) Signed-off-by: Kunal Kotwani (cherry picked from commit 3554ebf44d69715a34c94cf65edc4916b6e989b2) Co-authored-by: Kunal Kotwani --- .github/workflows/delete_backport_branch.yml | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/.github/workflows/delete_backport_branch.yml b/.github/workflows/delete_backport_branch.yml index d654df6b4..a4d186ca7 100644 --- a/.github/workflows/delete_backport_branch.yml +++ b/.github/workflows/delete_backport_branch.yml @@ -7,9 +7,16 @@ on: jobs: delete-branch: runs-on: ubuntu-latest - if: startsWith(github.event.pull_request.head.ref,'backport/') + permissions: + contents: write + if: github.repository == 'opensearch-project/k-NN' && startsWith(github.event.pull_request.head.ref,'backport/') steps: - - name: Delete merged branch - uses: SvanBoxel/delete-merged-branch@main - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file + - name: Delete merged branch + uses: actions/github-script@v7 + with: + script: | + github.rest.git.deleteRef({ + owner: context.repo.owner, + repo: context.repo.repo, + ref: `heads/${context.payload.pull_request.head.ref}`, + }) From 2e603a1b47fdda3444625ca1c1ce7c34c62d510a Mon Sep 17 00:00:00 2001 From: Doo Yong Kim <0ctopus13prime@gmail.com> Date: Thu, 7 Nov 2024 13:14:07 -0800 Subject: [PATCH 411/416] Introduced writing layer, getting rid of writing logic that uses an absolute path in the filesystem. (#2248) Signed-off-by: Dooyong Kim Co-authored-by: Dooyong Kim --- CHANGELOG.md | 1 + jni/cmake/init-nmslib.cmake | 1 + jni/include/commons.h | 6 + jni/include/faiss_index_service.h | 63 +- jni/include/faiss_methods.h | 14 +- jni/include/faiss_stream_support.h | 37 +- jni/include/faiss_wrapper.h | 35 +- jni/include/jni_util.h | 35 +- jni/include/memory_util.h | 23 + jni/include/native_engines_stream_support.h | 133 +- jni/include/nmslib_stream_support.h | 28 +- jni/include/nmslib_wrapper.h | 2 +- .../org_opensearch_knn_jni_FaissService.h | 60 +- .../org_opensearch_knn_jni_NmslibService.h | 4 +- jni/include/parameter_utils.h | 39 + ...-apis-in-Hnsw-with-streaming-interfa.patch | 165 ++ jni/src/commons.cpp | 3 - jni/src/faiss_index_service.cpp | 89 +- jni/src/faiss_methods.cpp | 9 +- jni/src/faiss_wrapper.cpp | 131 +- jni/src/jni_util.cpp | 20 +- jni/src/nmslib_wrapper.cpp | 54 +- .../org_opensearch_knn_jni_FaissService.cpp | 202 +- .../org_opensearch_knn_jni_NmslibService.cpp | 4 +- jni/tests/faiss_index_service_test.cpp | 17 +- jni/tests/faiss_stream_support_test.cpp | 10 +- jni/tests/faiss_wrapper_test.cpp | 398 ++-- jni/tests/mocks/faiss_index_service_mock.h | 11 +- jni/tests/mocks/faiss_methods_mock.h | 4 +- jni/tests/native_stream_support_util.h | 70 +- jni/tests/nmslib_stream_support_test.cpp | 174 +- jni/tests/nmslib_wrapper_test.cpp | 194 +- jni/tests/test_util.cpp | 10 +- jni/tests/test_util.h | 6 +- .../DefaultIndexBuildStrategy.java | 4 +- .../MemOptimizedNativeIndexBuildStrategy.java | 3 +- .../codec/nativeindex/NativeIndexWriter.java | 76 +- .../nativeindex/model/BuildIndexParams.java | 3 +- .../index/store/IndexOutputWithBuffer.java | 40 + .../org/opensearch/knn/jni/FaissService.java | 31 +- .../org/opensearch/knn/jni/JNIService.java | 54 +- .../org/opensearch/knn/jni/NmslibService.java | 24 +- .../common/RaisingIOExceptionIndexInput.java | 51 + .../common/RasingIOExceptionIndexOutput.java | 41 + .../knn/index/codec/KNNCodecTestCase.java | 113 +- .../knn/index/codec/KNNCodecTestUtil.java | 73 +- .../DefaultIndexBuildStrategyTests.java | 17 +- ...ptimizedNativeIndexBuildStrategyTests.java | 14 +- .../memory/NativeMemoryAllocationTests.java | 200 +- .../memory/NativeMemoryLoadStrategyTests.java | 175 +- .../opensearch/knn/jni/JNIServiceTests.java | 2039 ++++++++++------- .../knn/training/TrainingJobTests.java | 34 +- .../java/org/opensearch/knn/TestUtils.java | 30 +- 53 files changed, 3144 insertions(+), 1930 deletions(-) create mode 100644 jni/include/memory_util.h create mode 100644 jni/include/parameter_utils.h create mode 100644 jni/patches/nmslib/0004-Added-a-new-save-apis-in-Hnsw-with-streaming-interfa.patch create mode 100644 src/main/java/org/opensearch/knn/index/store/IndexOutputWithBuffer.java create mode 100644 src/test/java/org/opensearch/knn/common/RaisingIOExceptionIndexInput.java create mode 100644 src/test/java/org/opensearch/knn/common/RasingIOExceptionIndexOutput.java diff --git a/CHANGELOG.md b/CHANGELOG.md index be168322b..3abe09f20 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ## [Unreleased 2.x](https://github.com/opensearch-project/k-NN/compare/2.18...2.x) ### Features ### Enhancements +- Introduced a writing layer in native engines where relies on the writing interface to process IO. (#2241)[https://github.com/opensearch-project/k-NN/pull/2241] ### Bug Fixes ### Infrastructure ### Documentation diff --git a/jni/cmake/init-nmslib.cmake b/jni/cmake/init-nmslib.cmake index 2554b2bd7..a7c3f7d93 100644 --- a/jni/cmake/init-nmslib.cmake +++ b/jni/cmake/init-nmslib.cmake @@ -19,6 +19,7 @@ if(NOT DEFINED APPLY_LIB_PATCHES OR "${APPLY_LIB_PATCHES}" STREQUAL true) list(APPEND PATCH_FILE_LIST "${CMAKE_CURRENT_SOURCE_DIR}/patches/nmslib/0001-Initialize-maxlevel-during-add-from-enterpoint-level.patch") list(APPEND PATCH_FILE_LIST "${CMAKE_CURRENT_SOURCE_DIR}/patches/nmslib/0002-Adds-ability-to-pass-ef-parameter-in-the-query-for-h.patch") list(APPEND PATCH_FILE_LIST "${CMAKE_CURRENT_SOURCE_DIR}/patches/nmslib/0003-Added-streaming-apis-for-vector-index-loading-in-Hnsw.patch") + list(APPEND PATCH_FILE_LIST "${CMAKE_CURRENT_SOURCE_DIR}/patches/nmslib/0004-Added-a-new-save-apis-in-Hnsw-with-streaming-interfa.patch") # Get patch id of the last commit execute_process(COMMAND sh -c "git --no-pager show HEAD | git patch-id --stable" OUTPUT_VARIABLE PATCH_ID_OUTPUT_FROM_COMMIT WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/external/nmslib) diff --git a/jni/include/commons.h b/jni/include/commons.h index 3f1ee19a3..38b00cc5d 100644 --- a/jni/include/commons.h +++ b/jni/include/commons.h @@ -8,6 +8,10 @@ * Modifications Copyright OpenSearch Contributors. See * GitHub history for details. */ + +#ifndef OPENSEARCH_KNN_COMMONS_H +#define OPENSEARCH_KNN_COMMONS_H + #include "jni_util.h" #include namespace knn_jni { @@ -99,3 +103,5 @@ namespace knn_jni { int getIntegerMethodParameter(JNIEnv *, knn_jni::JNIUtilInterface *, std::unordered_map, std::string, int); } } + +#endif \ No newline at end of file diff --git a/jni/include/faiss_index_service.h b/jni/include/faiss_index_service.h index 29ec90e80..d96c3e755 100644 --- a/jni/include/faiss_index_service.h +++ b/jni/include/faiss_index_service.h @@ -16,8 +16,10 @@ #include #include "faiss/MetricType.h" +#include "faiss/impl/io.h" #include "jni_util.h" #include "faiss_methods.h" +#include "faiss_stream_support.h" #include namespace knn_jni { @@ -30,7 +32,8 @@ namespace faiss_wrapper { */ class IndexService { public: - IndexService(std::unique_ptr faissMethods); + explicit IndexService(std::unique_ptr faissMethods); + /** * Initialize index * @@ -45,6 +48,7 @@ class IndexService { * @return memory address of the native index object */ virtual jlong initIndex(knn_jni::JNIUtilInterface *jniUtil, JNIEnv *env, faiss::MetricType metric, std::string indexDescription, int dim, int numVectors, int threadCount, std::unordered_map parameters); + /** * Add vectors to index * @@ -55,29 +59,34 @@ class IndexService { * @param idMapAddress memory address of the native index object */ virtual void insertToIndex(int dim, int numIds, int threadCount, int64_t vectorsAddress, std::vector &ids, jlong idMapAddress); + /** * Write index to disk * - * @param threadCount number of thread count to be used while adding data - * @param indexPath path to write index - * @param idMap memory address of the native index object + * @param writer IOWriter implementation doing IO processing. + * In most cases, it is expected to have underlying Lucene's IndexOuptut. + * @param idMapAddress memory address of the native index object */ - virtual void writeIndex(std::string indexPath, jlong idMapAddress); + virtual void writeIndex(faiss::IOWriter* writer, jlong idMapAddress); + virtual ~IndexService() = default; + protected: virtual void allocIndex(faiss::Index * index, size_t dim, size_t numVectors); + std::unique_ptr faissMethods; -}; +}; // class IndexService /** * A class to provide operations on index * This class should evolve to have only cpp object but not jni object */ -class BinaryIndexService : public IndexService { +class BinaryIndexService final : public IndexService { public: //TODO Remove dependency on JNIUtilInterface and JNIEnv //TODO Reduce the number of parameters - BinaryIndexService(std::unique_ptr faissMethods); + explicit BinaryIndexService(std::unique_ptr faissMethods); + /** * Initialize index * @@ -91,7 +100,8 @@ class BinaryIndexService : public IndexService { * @param parameters parameters to be applied to faiss index * @return memory address of the native index object */ - virtual jlong initIndex(knn_jni::JNIUtilInterface *jniUtil, JNIEnv *env, faiss::MetricType metric, std::string indexDescription, int dim, int numVectors, int threadCount, std::unordered_map parameters) override; + jlong initIndex(knn_jni::JNIUtilInterface *jniUtil, JNIEnv *env, faiss::MetricType metric, std::string indexDescription, int dim, int numVectors, int threadCount, std::unordered_map parameters) final; + /** * Add vectors to index * @@ -106,7 +116,8 @@ class BinaryIndexService : public IndexService { * @param idMap a map of document id and vector id * @param parameters parameters to be applied to faiss index */ - virtual void insertToIndex(int dim, int numIds, int threadCount, int64_t vectorsAddress, std::vector &ids, jlong idMapAddress) override; + void insertToIndex(int dim, int numIds, int threadCount, int64_t vectorsAddress, std::vector &ids, jlong idMapAddress) final; + /** * Write index to disk * @@ -119,23 +130,23 @@ class BinaryIndexService : public IndexService { * @param idMap a map of document id and vector id * @param parameters parameters to be applied to faiss index */ - virtual void writeIndex(std::string indexPath, jlong idMapAddress) override; - virtual ~BinaryIndexService() = default; + void writeIndex(faiss::IOWriter* writer, jlong idMapAddress) final; + protected: - virtual void allocIndex(faiss::Index * index, size_t dim, size_t numVectors) override; -}; + void allocIndex(faiss::Index * index, size_t dim, size_t numVectors) final; +}; // class BinaryIndexService /** * A class to provide operations on index * This class should evolve to have only cpp object but not jni object */ -class ByteIndexService : public IndexService { +class ByteIndexService final : public IndexService { public: //TODO Remove dependency on JNIUtilInterface and JNIEnv //TODO Reduce the number of parameters - ByteIndexService(std::unique_ptr faissMethods); + explicit ByteIndexService(std::unique_ptr faissMethods); -/** + /** * Initialize index * * @param jniUtil jni util @@ -148,7 +159,8 @@ class ByteIndexService : public IndexService { * @param parameters parameters to be applied to faiss index * @return memory address of the native index object */ - virtual jlong initIndex(knn_jni::JNIUtilInterface *jniUtil, JNIEnv *env, faiss::MetricType metric, std::string indexDescription, int dim, int numVectors, int threadCount, std::unordered_map parameters) override; + jlong initIndex(knn_jni::JNIUtilInterface *jniUtil, JNIEnv *env, faiss::MetricType metric, std::string indexDescription, int dim, int numVectors, int threadCount, std::unordered_map parameters) final; + /** * Add vectors to index * @@ -163,7 +175,8 @@ class ByteIndexService : public IndexService { * @param idMap a map of document id and vector id * @param parameters parameters to be applied to faiss index */ - virtual void insertToIndex(int dim, int numIds, int threadCount, int64_t vectorsAddress, std::vector &ids, jlong idMapAddress) override; + void insertToIndex(int dim, int numIds, int threadCount, int64_t vectorsAddress, std::vector &ids, jlong idMapAddress) final; + /** * Write index to disk * @@ -176,14 +189,14 @@ class ByteIndexService : public IndexService { * @param idMap a map of document id and vector id * @param parameters parameters to be applied to faiss index */ - virtual void writeIndex(std::string indexPath, jlong idMapAddress) override; - virtual ~ByteIndexService() = default; -protected: - virtual void allocIndex(faiss::Index * index, size_t dim, size_t numVectors) override; -}; + void writeIndex(faiss::IOWriter* writer, jlong idMapAddress) final; + + protected: + void allocIndex(faiss::Index * index, size_t dim, size_t numVectors) final; +}; // class ByteIndexService } } -#endif //OPENSEARCH_KNN_FAISS_INDEX_SERVICE_H \ No newline at end of file +#endif //OPENSEARCH_KNN_FAISS_INDEX_SERVICE_H diff --git a/jni/include/faiss_methods.h b/jni/include/faiss_methods.h index 38d8d756a..d8f14d03f 100644 --- a/jni/include/faiss_methods.h +++ b/jni/include/faiss_methods.h @@ -10,6 +10,7 @@ #ifndef OPENSEARCH_KNN_FAISS_METHODS_H #define OPENSEARCH_KNN_FAISS_METHODS_H +#include "faiss/impl/io.h" #include "faiss/Index.h" #include "faiss/IndexBinary.h" #include "faiss/IndexIDMap.h" @@ -26,14 +27,21 @@ namespace faiss_wrapper { class FaissMethods { public: FaissMethods() = default; + virtual faiss::Index* indexFactory(int d, const char* description, faiss::MetricType metric); + virtual faiss::IndexBinary* indexBinaryFactory(int d, const char* description); + virtual faiss::IndexIDMapTemplate* indexIdMap(faiss::Index* index); + virtual faiss::IndexIDMapTemplate* indexBinaryIdMap(faiss::IndexBinary* index); - virtual void writeIndex(const faiss::Index* idx, const char* fname); - virtual void writeIndexBinary(const faiss::IndexBinary* idx, const char* fname); + + virtual void writeIndex(const faiss::Index* idx, faiss::IOWriter* writer); + + virtual void writeIndexBinary(const faiss::IndexBinary* idx, faiss::IOWriter* writer); + virtual ~FaissMethods() = default; -}; +}; // class FaissMethods } //namespace faiss_wrapper } //namespace knn_jni diff --git a/jni/include/faiss_stream_support.h b/jni/include/faiss_stream_support.h index a12d66ae9..eb1b2a404 100644 --- a/jni/include/faiss_stream_support.h +++ b/jni/include/faiss_stream_support.h @@ -15,6 +15,7 @@ #include "faiss/impl/io.h" #include "jni_util.h" #include "native_engines_stream_support.h" +#include "parameter_utils.h" #include #include @@ -34,7 +35,7 @@ class FaissOpenSearchIOReader final : public faiss::IOReader { public: explicit FaissOpenSearchIOReader(NativeEngineIndexInputMediator *_mediator) : faiss::IOReader(), - mediator(_mediator) { + mediator(knn_jni::util::ParameterCheck::require_non_null(_mediator, "mediator")) { name = "FaissOpenSearchIOReader"; } @@ -56,6 +57,40 @@ class FaissOpenSearchIOReader final : public faiss::IOReader { }; // class FaissOpenSearchIOReader +/** + * A glue component inheriting IOWriter to delegate IO processing down to the given + * mediator. The mediator is expected to do write bytes via the provided Lucene's IndexOutput. + */ +class FaissOpenSearchIOWriter final : public faiss::IOWriter { + public: + explicit FaissOpenSearchIOWriter(NativeEngineIndexOutputMediator *_mediator) + : faiss::IOWriter(), + mediator(knn_jni::util::ParameterCheck::require_non_null(_mediator, "mediator")) { + name = "FaissOpenSearchIOWriter"; + } + + size_t operator()(const void *ptr, size_t size, size_t nitems) final { + const auto writeBytes = size * nitems; + if (writeBytes > 0) { + mediator->writeBytes(reinterpret_cast(ptr), writeBytes); + } + return nitems; + } + + // return a file number that can be memory-mapped + int filedescriptor() final { + throw std::runtime_error("filedescriptor() is not supported in FaissOpenSearchIOWriter."); + } + + void flush() { + mediator->flush(); + } + + private: + NativeEngineIndexOutputMediator *mediator; +}; // class FaissOpenSearchIOWriter + + } } diff --git a/jni/include/faiss_wrapper.h b/jni/include/faiss_wrapper.h index 8ffce4ad1..e48e6faa9 100644 --- a/jni/include/faiss_wrapper.h +++ b/jni/include/faiss_wrapper.h @@ -14,6 +14,7 @@ #include "jni_util.h" #include "faiss_index_service.h" +#include "faiss_stream_support.h" #include namespace knn_jni { @@ -22,25 +23,25 @@ namespace knn_jni { void InsertToIndex(knn_jni::JNIUtilInterface *jniUtil, JNIEnv *env, jintArray idsJ, jlong vectorsAddressJ, jint dimJ, jlong indexAddr, jint threadCount, IndexService *indexService); - void WriteIndex(knn_jni::JNIUtilInterface *jniUtil, JNIEnv *env, jstring indexPathJ, jlong indexAddr, IndexService *indexService); + void WriteIndex(knn_jni::JNIUtilInterface *jniUtil, JNIEnv *env, jobject output, jlong indexAddr, IndexService *indexService); // Create an index with ids and vectors. Instead of creating a new index, this function creates the index // based off of the template index passed in. The index is serialized to indexPathJ. void CreateIndexFromTemplate(knn_jni::JNIUtilInterface * jniUtil, JNIEnv * env, jintArray idsJ, - jlong vectorsAddressJ, jint dimJ, jstring indexPathJ, jbyteArray templateIndexJ, + jlong vectorsAddressJ, jint dimJ, jobject output, jbyteArray templateIndexJ, jobject parametersJ); // Create an index with ids and vectors. Instead of creating a new index, this function creates the index // based off of the template index passed in. The index is serialized to indexPathJ. void CreateBinaryIndexFromTemplate(knn_jni::JNIUtilInterface * jniUtil, JNIEnv * env, jintArray idsJ, - jlong vectorsAddressJ, jint dimJ, jstring indexPathJ, jbyteArray templateIndexJ, - jobject parametersJ); + jlong vectorsAddressJ, jint dimJ, jobject output, jbyteArray templateIndexJ, + jobject parametersJ); // Create a index with ids and byte vectors. Instead of creating a new index, this function creates the index // based off of the template index passed in. The index is serialized to indexPathJ. void CreateByteIndexFromTemplate(knn_jni::JNIUtilInterface * jniUtil, JNIEnv * env, jintArray idsJ, - jlong vectorsAddressJ, jint dimJ, jstring indexPathJ, jbyteArray templateIndexJ, - jobject parametersJ); + jlong vectorsAddressJ, jint dimJ, jobject output, jbyteArray templateIndexJ, + jobject parametersJ); // Load an index from indexPathJ into memory. // @@ -74,28 +75,28 @@ namespace knn_jni { // Sets the sharedIndexState for an index void SetSharedIndexState(jlong indexPointerJ, jlong shareIndexStatePointerJ); - /** + /** * Execute a query against the index located in memory at indexPointerJ - * + * * Parameters: * methodParamsJ: introduces a map to have additional method parameters - * + * * Return an array of KNNQueryResults - */ + */ jobjectArray QueryIndex(knn_jni::JNIUtilInterface * jniUtil, JNIEnv * env, jlong indexPointerJ, jfloatArray queryVectorJ, jint kJ, jobject methodParamsJ, jintArray parentIdsJ); /** * Execute a query against the index located in memory at indexPointerJ along with Filters - * + * * Parameters: * methodParamsJ: introduces a map to have additional method parameters - * + * * Return an array of KNNQueryResults */ jobjectArray QueryIndex_WithFilter(knn_jni::JNIUtilInterface * jniUtil, JNIEnv * env, jlong indexPointerJ, - jfloatArray queryVectorJ, jint kJ, jobject methodParamsJ, jlongArray filterIdsJ, - jint filterIdsTypeJ, jintArray parentIdsJ); + jfloatArray queryVectorJ, jint kJ, jobject methodParamsJ, jlongArray filterIdsJ, + jint filterIdsTypeJ, jintArray parentIdsJ); // Execute a query against the binary index located in memory at indexPointerJ along with Filters // @@ -124,14 +125,14 @@ namespace knn_jni { // // Return the serialized representation jbyteArray TrainBinaryIndex(knn_jni::JNIUtilInterface * jniUtil, JNIEnv * env, jobject parametersJ, jint dimension, - jlong trainVectorsPointerJ); + jlong trainVectorsPointerJ); // Create an empty byte index defined by the values in the Java map, parametersJ. Train the index with // the byte vectors located at trainVectorsPointerJ. // // Return the serialized representation jbyteArray TrainByteIndex(knn_jni::JNIUtilInterface * jniUtil, JNIEnv * env, jobject parametersJ, jint dimension, - jlong trainVectorsPointerJ); + jlong trainVectorsPointerJ); /* * Perform a range search with filter against the index located in memory at indexPointerJ. @@ -163,7 +164,7 @@ namespace knn_jni { * @return an array of RangeQueryResults */ jobjectArray RangeSearch(knn_jni::JNIUtilInterface *jniUtil, JNIEnv *env, jlong indexPointerJ, jfloatArray queryVectorJ, - jfloat radiusJ, jobject methodParamsJ, jint maxResultWindowJ, jintArray parentIdsJ); + jfloat radiusJ, jobject methodParamsJ, jint maxResultWindowJ, jintArray parentIdsJ); } } diff --git a/jni/include/jni_util.h b/jni/include/jni_util.h index 9f4daef7c..45068ae1b 100644 --- a/jni/include/jni_util.h +++ b/jni/include/jni_util.h @@ -18,6 +18,7 @@ #include #include #include +#include namespace knn_jni { @@ -33,7 +34,7 @@ namespace knn_jni { virtual void HasExceptionInStack(JNIEnv* env) = 0; // HasExceptionInStack with ability to specify message - virtual void HasExceptionInStack(JNIEnv* env, const std::string& message) = 0; + virtual void HasExceptionInStack(JNIEnv* env, const char *message) = 0; // Catches a C++ exception and throws the corresponding exception to the JVM virtual void CatchCppExceptionAndThrowJava(JNIEnv* env) = 0; @@ -144,6 +145,9 @@ namespace knn_jni { virtual jlong CallNonvirtualLongMethodA(JNIEnv * env, jobject obj, jclass clazz, jmethodID methodID, jvalue* args) = 0; + virtual void CallNonvirtualVoidMethodA(JNIEnv * env, jobject obj, jclass clazz, + jmethodID methodID, jvalue* args) = 0; + // -------------------------------------------------------------------------- }; @@ -158,7 +162,7 @@ namespace knn_jni { void ThrowJavaException(JNIEnv* env, const char* type = "", const char* message = "") final; void HasExceptionInStack(JNIEnv* env) final; - void HasExceptionInStack(JNIEnv* env, const std::string& message) final; + void HasExceptionInStack(JNIEnv* env, const char* message) final; void CatchCppExceptionAndThrowJava(JNIEnv* env) final; jclass FindClass(JNIEnv * env, const std::string& className) final; jmethodID FindMethod(JNIEnv * env, const std::string& className, const std::string& methodName) final; @@ -200,13 +204,38 @@ namespace knn_jni { jfieldID GetFieldID(JNIEnv * env, jclass clazz, const char *name, const char *sig) final; jint CallNonvirtualIntMethodA(JNIEnv *env, jobject obj, jclass clazz, jmethodID methodID, jvalue *args) final; jlong CallNonvirtualLongMethodA(JNIEnv * env, jobject obj, jclass clazz, jmethodID methodID, jvalue* args) final; + void CallNonvirtualVoidMethodA(JNIEnv * env, jobject obj, jclass clazz, jmethodID methodID, jvalue* args) final; void * GetPrimitiveArrayCritical(JNIEnv * env, jarray array, jboolean *isCopy) final; void ReleasePrimitiveArrayCritical(JNIEnv * env, jarray array, void *carray, jint mode) final; private: std::unordered_map cachedClasses; std::unordered_map cachedMethods; - }; + }; // class JNIUtil + + /** + * It's common cleaner to release a primitive array within its destructor. + * Ex: JNIReleaseElements release_int_array_elements {[=](){ + * jniUtil->ReleaseIntArrayElements(env, idsJ, idsCpp, JNI_ABORT); + * }}; + */ + struct JNIReleaseElements { + explicit JNIReleaseElements(std::function _release_func) + : release_func(std::move(_release_func)) { + } + + ~JNIReleaseElements() { + try { + if (release_func) { + release_func(); + } + } catch (...) { + // Ignore + } + } + + std::function release_func; + }; // struct ReleaseIntArrayElements // ------------------------------- CONSTANTS -------------------------------- extern const std::string FAISS_NAME; diff --git a/jni/include/memory_util.h b/jni/include/memory_util.h new file mode 100644 index 000000000..5e1fc13ae --- /dev/null +++ b/jni/include/memory_util.h @@ -0,0 +1,23 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +#ifndef KNNPLUGIN_JNI_INCLUDE_MEMORY_UTIL_H_ +#define KNNPLUGIN_JNI_INCLUDE_MEMORY_UTIL_H_ + +#if defined(__GNUC__) || defined(__clang__) +#define RESTRICT __restrict__ +#elif defined(_MSC_VER) +#define RESTRICT __declspec(restrict) +#else +#define RESTRICT +#endif + +#endif //KNNPLUGIN_JNI_INCLUDE_MEMORY_UTIL_H_ diff --git a/jni/include/native_engines_stream_support.h b/jni/include/native_engines_stream_support.h index 5d4b32d3d..07f97f3ac 100644 --- a/jni/include/native_engines_stream_support.h +++ b/jni/include/native_engines_stream_support.h @@ -13,6 +13,8 @@ #define OPENSEARCH_KNN_JNI_STREAM_SUPPORT_H #include "jni_util.h" +#include "parameter_utils.h" +#include "memory_util.h" #include #include @@ -22,8 +24,6 @@ namespace knn_jni { namespace stream { - - /** * This class contains Java IndexInputWithBuffer reference and calls its API to copy required bytes into a read buffer. */ @@ -33,9 +33,10 @@ class NativeEngineIndexInputMediator { NativeEngineIndexInputMediator(JNIUtilInterface *_jni_interface, JNIEnv *_env, jobject _indexInput) - : jni_interface(_jni_interface), - env(_env), - indexInput(_indexInput), + : jni_interface(knn_jni::util::ParameterCheck::require_non_null( + _jni_interface, "jni_interface")), + env(knn_jni::util::ParameterCheck::require_non_null(_env, "env")), + indexInput(knn_jni::util::ParameterCheck::require_non_null(_indexInput, "indexInput")), bufferArray((jbyteArray) (_jni_interface->GetObjectField(_env, _indexInput, getBufferFieldId(_jni_interface, _env)))), @@ -43,7 +44,7 @@ class NativeEngineIndexInputMediator { remainingBytesMethod(getRemainingBytesMethod(_jni_interface, _env)) { } - void copyBytes(int64_t nbytes, uint8_t *destination) { + void copyBytes(int64_t nbytes, uint8_t * RESTRICT destination) { auto jclazz = getIndexInputWithBufferClass(jni_interface, env); while (nbytes > 0) { @@ -52,11 +53,12 @@ class NativeEngineIndexInputMediator { args.j = nbytes; const auto readBytes = jni_interface->CallNonvirtualIntMethodA(env, indexInput, jclazz, copyBytesMethod, &args); + jni_interface->HasExceptionInStack(env, "Reading bytes via IndexInput has failed."); // === Critical Section Start === // Get primitive array pointer, no copy is happening in OpenJDK. - auto primitiveArray = + jbyte * RESTRICT primitiveArray = (jbyte *) jni_interface->GetPrimitiveArrayCritical(env, bufferArray, nullptr); // Copy Java bytes to C++ destination address. @@ -65,6 +67,7 @@ class NativeEngineIndexInputMediator { // Release the acquired primitive array pointer. // JNI_ABORT tells JVM to directly free memory without copying back to Java byte[]. // Since we're merely copying data, we don't need to copying back. + // Note than when we received an internal primitive array pointer, then the mode will be ignored. jni_interface->ReleasePrimitiveArrayCritical(env, bufferArray, primitiveArray, JNI_ABORT); // === Critical Section End === @@ -75,11 +78,13 @@ class NativeEngineIndexInputMediator { } int64_t remainingBytes() { - return jni_interface->CallNonvirtualLongMethodA(env, - indexInput, - getIndexInputWithBufferClass(jni_interface, env), - remainingBytesMethod, - nullptr); + auto bytes = jni_interface->CallNonvirtualLongMethodA(env, + indexInput, + getIndexInputWithBufferClass(jni_interface, env), + remainingBytesMethod, + nullptr); + jni_interface->HasExceptionInStack(env, "Checking remaining bytes has failed."); + return bytes; } private: @@ -119,6 +124,110 @@ class NativeEngineIndexInputMediator { +/** + * This class delegates the provided index output to do IO processing. + * In most cases, it is expected that IndexOutputWithBuffer was passed down to this, + * which eventually have Lucene's IndexOutput to write bytes. + */ +class NativeEngineIndexOutputMediator { + public: + NativeEngineIndexOutputMediator(JNIUtilInterface *_jni_interface, + JNIEnv *_env, + jobject _indexOutput) + : jni_interface(knn_jni::util::ParameterCheck::require_non_null(_jni_interface, "jni_interface")), + env(knn_jni::util::ParameterCheck::require_non_null(_env, "env")), + indexOutput(knn_jni::util::ParameterCheck::require_non_null(_indexOutput, "indexOutput")), + bufferArray((jbyteArray) (_jni_interface->GetObjectField(_env, + _indexOutput, + getBufferFieldId(_jni_interface, _env)))), + writeBytesMethod(getWriteBytesMethod(_jni_interface, _env)), + bufferLength(jni_interface->GetJavaBytesArrayLength(env, bufferArray)), + nextWriteIndex() { + } + + void writeBytes(const uint8_t * RESTRICT source, size_t nbytes) { + auto left = nbytes; + while (left > 0) { + const auto writeBytes = std::min(bufferLength - nextWriteIndex, left); + + // === Critical Section Start === + + // Get primitive array pointer, no copy is happening in OpenJDK. + jbyte * RESTRICT primitiveArray = + (jbyte *) jni_interface->GetPrimitiveArrayCritical(env, bufferArray, nullptr); + + // Copy the given bytes to Java byte[] address. + std::memcpy(primitiveArray + nextWriteIndex, source, writeBytes); + + // Release the acquired primitive array pointer. + // 0 tells JVM to copy back the content, and to free the pointer. It will be ignored if we acquired an internal + // primitive array pointer instead of a copied version. + // From JNI docs: + // Mode 0 : copy back the content and free the elems buffer + // The mode argument provides information on how the array buffer should be released. mode has no effect if elems + // is not a copy of the elements in array. + jni_interface->ReleasePrimitiveArrayCritical(env, bufferArray, primitiveArray, 0); + + // === Critical Section End === + + nextWriteIndex += writeBytes; + if (nextWriteIndex >= bufferLength) { + callWriteBytesInIndexOutput(); + } + + source += writeBytes; + left -= writeBytes; + } // End while + } + + void flush() { + if (nextWriteIndex > 0) { + callWriteBytesInIndexOutput(); + } + } + + private: + static jclass getIndexOutputWithBufferClass(JNIUtilInterface *jni_interface, JNIEnv *env) { + static jclass INDEX_OUTPUT_WITH_BUFFER_CLASS = + jni_interface->FindClassFromJNIEnv(env, "org/opensearch/knn/index/store/IndexOutputWithBuffer"); + return INDEX_OUTPUT_WITH_BUFFER_CLASS; + } + + static jmethodID getWriteBytesMethod(JNIUtilInterface *jni_interface, JNIEnv *env) { + static jmethodID WRITE_METHOD_ID = + jni_interface->GetMethodID(env, getIndexOutputWithBufferClass(jni_interface, env), "writeBytes", "(I)V"); + return WRITE_METHOD_ID; + } + + static jfieldID getBufferFieldId(JNIUtilInterface *jni_interface, JNIEnv *env) { + static jfieldID BUFFER_FIELD_ID = + jni_interface->GetFieldID(env, getIndexOutputWithBufferClass(jni_interface, env), "buffer", "[B"); + return BUFFER_FIELD_ID; + } + + void callWriteBytesInIndexOutput() { + auto jclazz = getIndexOutputWithBufferClass(jni_interface, env); + // Initializing the first integer parameter of `writeBytes`. + // `i` represents an integer parameter. + jvalue args {.i = nextWriteIndex}; + jni_interface->CallNonvirtualVoidMethodA(env, indexOutput, jclazz, writeBytesMethod, &args); + jni_interface->HasExceptionInStack(env, "Writing bytes via IndexOutput has failed."); + nextWriteIndex = 0; + } + + JNIUtilInterface *jni_interface; + JNIEnv *env; + + // `IndexOutputWithBuffer` instance having `IndexOutput` instance obtained from `Directory` for reading. + jobject indexOutput; + jbyteArray bufferArray; + jmethodID writeBytesMethod; + size_t bufferLength; + int32_t nextWriteIndex; +}; // NativeEngineIndexOutputMediator + + + } } diff --git a/jni/include/nmslib_stream_support.h b/jni/include/nmslib_stream_support.h index 38c06cb95..2c410dde6 100644 --- a/jni/include/nmslib_stream_support.h +++ b/jni/include/nmslib_stream_support.h @@ -13,19 +13,20 @@ #define OPENSEARCH_KNN_JNI_NMSLIB_STREAM_SUPPORT_H #include "native_engines_stream_support.h" +#include "utils.h" // This is from NMSLIB +#include "parameter_utils.h" namespace knn_jni { namespace stream { - - /** * NmslibIOReader implementation delegating NativeEngineIndexInputMediator to read bytes. */ class NmslibOpenSearchIOReader final : public similarity::NmslibIOReader { public: explicit NmslibOpenSearchIOReader(NativeEngineIndexInputMediator *_mediator) - : mediator(_mediator) { + : similarity::NmslibIOReader(), + mediator(knn_jni::util::ParameterCheck::require_non_null(_mediator, "mediator")) { } void read(char *bytes, size_t len) final { @@ -44,6 +45,27 @@ class NmslibOpenSearchIOReader final : public similarity::NmslibIOReader { }; // class NmslibOpenSearchIOReader +class NmslibOpenSearchIOWriter final : public similarity::NmslibIOWriter { + public: + explicit NmslibOpenSearchIOWriter(NativeEngineIndexOutputMediator *_mediator) + : similarity::NmslibIOWriter(), + mediator(knn_jni::util::ParameterCheck::require_non_null(_mediator, "mediator")) { + } + + void write(char *bytes, size_t len) final { + if (len > 0) { + mediator->writeBytes((uint8_t *) bytes, len); + } + } + + void flush() final { + mediator->flush(); + } + + private: + NativeEngineIndexOutputMediator *mediator; +}; // class NmslibOpenSearchIOWriter + } } diff --git a/jni/include/nmslib_wrapper.h b/jni/include/nmslib_wrapper.h index 2853cd71f..687a96d59 100644 --- a/jni/include/nmslib_wrapper.h +++ b/jni/include/nmslib_wrapper.h @@ -26,7 +26,7 @@ namespace knn_jni { // Create an index with ids and vectors. The configuration is defined by values in the Java map, parametersJ. // The index is serialized to indexPathJ. void CreateIndex(knn_jni::JNIUtilInterface * jniUtil, JNIEnv * env, jintArray idsJ, jlong vectorsAddress, jint dim, - jstring indexPathJ, jobject parametersJ); + jobject output, jobject parametersJ); // Load an index from indexPathJ into memory. Use parametersJ to set any query time parameters // diff --git a/jni/include/org_opensearch_knn_jni_FaissService.h b/jni/include/org_opensearch_knn_jni_FaissService.h index 2969df3ae..dce580138 100644 --- a/jni/include/org_opensearch_knn_jni_FaissService.h +++ b/jni/include/org_opensearch_knn_jni_FaissService.h @@ -24,16 +24,16 @@ extern "C" { * Signature: ([IJILjava/lang/String;Ljava/util/Map;)V */ JNIEXPORT jlong JNICALL Java_org_opensearch_knn_jni_FaissService_initIndex(JNIEnv * env, jclass cls, - jlong numDocs, jint dimJ, - jobject parametersJ); + jlong numDocs, jint dimJ, + jobject parametersJ); /* * Class: org_opensearch_knn_jni_FaissService * Method: initBinaryIndex * Signature: ([IJILjava/lang/String;Ljava/util/Map;)V */ JNIEXPORT jlong JNICALL Java_org_opensearch_knn_jni_FaissService_initBinaryIndex(JNIEnv * env, jclass cls, - jlong numDocs, jint dimJ, - jobject parametersJ); + jlong numDocs, jint dimJ, + jobject parametersJ); /* * Class: org_opensearch_knn_jni_FaissService @@ -41,8 +41,8 @@ JNIEXPORT jlong JNICALL Java_org_opensearch_knn_jni_FaissService_initBinaryIndex * Signature: ([IJILjava/lang/String;Ljava/util/Map;)V */ JNIEXPORT jlong JNICALL Java_org_opensearch_knn_jni_FaissService_initByteIndex(JNIEnv * env, jclass cls, - jlong numDocs, jint dimJ, - jobject parametersJ); + jlong numDocs, jint dimJ, + jobject parametersJ); /* * Class: org_opensearch_knn_jni_FaissService @@ -50,16 +50,16 @@ JNIEXPORT jlong JNICALL Java_org_opensearch_knn_jni_FaissService_initByteIndex(J * Signature: ([IJILjava/lang/String;Ljava/util/Map;)V */ JNIEXPORT void JNICALL Java_org_opensearch_knn_jni_FaissService_insertToIndex(JNIEnv * env, jclass cls, jintArray idsJ, - jlong vectorsAddressJ, jint dimJ, - jlong indexAddress, jint threadCount); + jlong vectorsAddressJ, jint dimJ, + jlong indexAddress, jint threadCount); /* * Class: org_opensearch_knn_jni_FaissService * Method: insertToBinaryIndex * Signature: ([IJILjava/lang/String;Ljava/util/Map;)V */ JNIEXPORT void JNICALL Java_org_opensearch_knn_jni_FaissService_insertToBinaryIndex(JNIEnv * env, jclass cls, jintArray idsJ, - jlong vectorsAddressJ, jint dimJ, - jlong indexAddress, jint threadCount); + jlong vectorsAddressJ, jint dimJ, + jlong indexAddress, jint threadCount); /* * Class: org_opensearch_knn_jni_FaissService @@ -67,58 +67,54 @@ JNIEXPORT void JNICALL Java_org_opensearch_knn_jni_FaissService_insertToBinaryIn * Signature: ([IJILjava/lang/String;Ljava/util/Map;)V */ JNIEXPORT void JNICALL Java_org_opensearch_knn_jni_FaissService_insertToByteIndex(JNIEnv * env, jclass cls, jintArray idsJ, - jlong vectorsAddressJ, jint dimJ, - jlong indexAddress, jint threadCount); + jlong vectorsAddressJ, jint dimJ, + jlong indexAddress, jint threadCount); /* * Class: org_opensearch_knn_jni_FaissService * Method: writeIndex - * Signature: ([IJILjava/lang/String;Ljava/util/Map;)V + * Signature: (JLorg/opensearch/knn/index/store/IndexOutputWithBuffer;)V */ -JNIEXPORT void JNICALL Java_org_opensearch_knn_jni_FaissService_writeIndex(JNIEnv * env, jclass cls, - jlong indexAddress, - jstring indexPathJ); +JNIEXPORT void JNICALL Java_org_opensearch_knn_jni_FaissService_writeIndex(JNIEnv *, jclass, jlong, jobject); + + /* * Class: org_opensearch_knn_jni_FaissService * Method: writeBinaryIndex - * Signature: ([IJILjava/lang/String;Ljava/util/Map;)V + * Signature: (JLorg/opensearch/knn/index/store/IndexOutputWithBuffer;)V */ -JNIEXPORT void JNICALL Java_org_opensearch_knn_jni_FaissService_writeBinaryIndex(JNIEnv * env, jclass cls, - jlong indexAddress, - jstring indexPathJ); +JNIEXPORT void JNICALL Java_org_opensearch_knn_jni_FaissService_writeBinaryIndex(JNIEnv *, jclass, jlong, jobject); /* * Class: org_opensearch_knn_jni_FaissService * Method: writeByteIndex - * Signature: ([IJILjava/lang/String;Ljava/util/Map;)V + * Signature: (JLorg/opensearch/knn/index/store/IndexOutputWithBuffer;)V */ -JNIEXPORT void JNICALL Java_org_opensearch_knn_jni_FaissService_writeByteIndex(JNIEnv * env, jclass cls, - jlong indexAddress, - jstring indexPathJ); +JNIEXPORT void JNICALL Java_org_opensearch_knn_jni_FaissService_writeByteIndex(JNIEnv *, jclass, jlong, jobject); /* * Class: org_opensearch_knn_jni_FaissService * Method: createIndexFromTemplate - * Signature: ([IJILjava/lang/String;[BLjava/util/Map;)V + * Signature: ([IJILorg/opensearch/knn/index/store/IndexOutputWithBuffer;[BLjava/util/Map;)V */ JNIEXPORT void JNICALL Java_org_opensearch_knn_jni_FaissService_createIndexFromTemplate - (JNIEnv *, jclass, jintArray, jlong, jint, jstring, jbyteArray, jobject); + (JNIEnv *, jclass, jintArray, jlong, jint, jobject, jbyteArray, jobject); /* * Class: org_opensearch_knn_jni_FaissService * Method: createBinaryIndexFromTemplate - * Signature: ([IJILjava/lang/String;[BLjava/util/Map;)V + * Signature: ([IJILorg/opensearch/knn/index/store/IndexOutputWithBuffer;[BLjava/util/Map;)V */ - JNIEXPORT void JNICALL Java_org_opensearch_knn_jni_FaissService_createBinaryIndexFromTemplate - (JNIEnv *, jclass, jintArray, jlong, jint, jstring, jbyteArray, jobject); +JNIEXPORT void JNICALL Java_org_opensearch_knn_jni_FaissService_createBinaryIndexFromTemplate + (JNIEnv *, jclass, jintArray, jlong, jint, jobject, jbyteArray, jobject); /* * Class: org_opensearch_knn_jni_FaissService * Method: createByteIndexFromTemplate - * Signature: ([IJILjava/lang/String;[BLjava/util/Map;)V + * Signature: ([IJILorg/opensearch/knn/index/store/IndexOutputWithBuffer;[BLjava/util/Map;)V */ - JNIEXPORT void JNICALL Java_org_opensearch_knn_jni_FaissService_createByteIndexFromTemplate - (JNIEnv *, jclass, jintArray, jlong, jint, jstring, jbyteArray, jobject); +JNIEXPORT void JNICALL Java_org_opensearch_knn_jni_FaissService_createByteIndexFromTemplate + (JNIEnv *, jclass, jintArray, jlong, jint, jobject, jbyteArray, jobject); /* * Class: org_opensearch_knn_jni_FaissService diff --git a/jni/include/org_opensearch_knn_jni_NmslibService.h b/jni/include/org_opensearch_knn_jni_NmslibService.h index 8d6633aff..0e035c3dd 100644 --- a/jni/include/org_opensearch_knn_jni_NmslibService.h +++ b/jni/include/org_opensearch_knn_jni_NmslibService.h @@ -21,10 +21,10 @@ extern "C" { /* * Class: org_opensearch_knn_jni_NmslibService * Method: createIndex - * Signature: ([IJILjava/lang/String;Ljava/util/Map;)V + * Signature: ([IJILorg/opensearch/knn/index/store/IndexOutputWithBuffer;Ljava/util/Map;)V */ JNIEXPORT void JNICALL Java_org_opensearch_knn_jni_NmslibService_createIndex - (JNIEnv *, jclass, jintArray, jlong, jint, jstring, jobject); + (JNIEnv *, jclass, jintArray, jlong, jint, jobject, jobject); /* * Class: org_opensearch_knn_jni_NmslibService diff --git a/jni/include/parameter_utils.h b/jni/include/parameter_utils.h new file mode 100644 index 000000000..aff922324 --- /dev/null +++ b/jni/include/parameter_utils.h @@ -0,0 +1,39 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +#ifndef KNNPLUGIN_JNI_INCLUDE_PARAMETER_UTILS_H_ +#define KNNPLUGIN_JNI_INCLUDE_PARAMETER_UTILS_H_ + +#include +#include + +namespace knn_jni { +namespace util { + +struct ParameterCheck { + template + static PtrType *require_non_null(PtrType *ptr, const char *parameter_name) { + if (ptr == nullptr) { + throw std::invalid_argument(std::string("Parameter [") + parameter_name + "] should not be null."); + } + return ptr; + } + + private: + ParameterCheck() = default; +}; // class ParameterCheck + + + +} +} // namespace knn_jni + +#endif //KNNPLUGIN_JNI_INCLUDE_PARAMETER_UTILS_H_ diff --git a/jni/patches/nmslib/0004-Added-a-new-save-apis-in-Hnsw-with-streaming-interfa.patch b/jni/patches/nmslib/0004-Added-a-new-save-apis-in-Hnsw-with-streaming-interfa.patch new file mode 100644 index 000000000..0d324bc29 --- /dev/null +++ b/jni/patches/nmslib/0004-Added-a-new-save-apis-in-Hnsw-with-streaming-interfa.patch @@ -0,0 +1,165 @@ +From eb9ceb60c695f795895b80ea66b251aea1fbf781 Mon Sep 17 00:00:00 2001 +From: Dooyong Kim +Date: Wed, 30 Oct 2024 15:44:34 -0700 +Subject: [PATCH] Added a new IOWriter interface in Hnsw node to support + streaming writing. + +Signed-off-by: Dooyong Kim +--- + similarity_search/include/method/hnsw.h | 3 ++ + similarity_search/include/utils.h | 20 ++++++++-- + similarity_search/src/method/hnsw.cc | 52 +++++++++++++++++++++++-- + 3 files changed, 68 insertions(+), 7 deletions(-) + +diff --git a/similarity_search/include/method/hnsw.h b/similarity_search/include/method/hnsw.h +index 433f98f..d235c15 100644 +--- a/similarity_search/include/method/hnsw.h ++++ b/similarity_search/include/method/hnsw.h +@@ -459,6 +459,8 @@ namespace similarity { + + void LoadIndexWithStream(similarity::NmslibIOReader& in); + ++ void SaveIndexWithStream(similarity::NmslibIOWriter& out); ++ + Hnsw(bool PrintProgress, const Space &space, const ObjectVector &data); + void CreateIndex(const AnyParams &IndexParams) override; + +@@ -501,6 +503,7 @@ namespace similarity { + + + void SaveOptimizedIndex(std::ostream& output); ++ void SaveOptimizedIndex(NmslibIOWriter& output); + void LoadOptimizedIndex(std::istream& input); + void LoadOptimizedIndex(NmslibIOReader& input); + +diff --git a/similarity_search/include/utils.h b/similarity_search/include/utils.h +index a3931b7..1cc55b2 100644 +--- a/similarity_search/include/utils.h ++++ b/similarity_search/include/utils.h +@@ -305,19 +305,33 @@ struct NmslibIOReader { + virtual void read(char* bytes, size_t len) = 0; + + virtual size_t remainingBytes() = 0; +-}; ++}; // class NmslibIOReader ++ ++struct NmslibIOWriter { ++ virtual ~NmslibIOWriter() = default; ++ ++ virtual void write(char* bytes, size_t len) = 0; ++ ++ virtual void flush() { ++ } ++}; // class NmslibIOWriter + + template + void writeBinaryPOD(ostream& out, const T& podRef) { + out.write((char*)&podRef, sizeof(T)); + } + +-template ++template ++void writeBinaryPOD(NmslibIOWriter& out, const T& podRef) { ++ out.write((char*)&podRef, sizeof(T)); ++} ++ ++template + static void readBinaryPOD(NmslibIOReader& in, T& podRef) { + in.read((char*)&podRef, sizeof(T)); + } + +-template ++template + static void readBinaryPOD(istream& in, T& podRef) { + in.read((char*)&podRef, sizeof(T)); + } +diff --git a/similarity_search/src/method/hnsw.cc b/similarity_search/src/method/hnsw.cc +index 662f06c..bfa3a23 100644 +--- a/similarity_search/src/method/hnsw.cc ++++ b/similarity_search/src/method/hnsw.cc +@@ -784,6 +784,19 @@ namespace similarity { + output.close(); + } + ++ template ++ void Hnsw::SaveIndexWithStream(NmslibIOWriter& output) { ++ unsigned int optimIndexFlag = data_level0_memory_ != nullptr; ++ ++ writeBinaryPOD(output, optimIndexFlag); ++ ++ if (!optimIndexFlag) { ++ throw std::runtime_error("With stream, we only support optimized index type."); ++ } else { ++ SaveOptimizedIndex(output); ++ } ++ } ++ + template + void + Hnsw::SaveOptimizedIndex(std::ostream& output) { +@@ -818,6 +831,37 @@ namespace similarity { + + } + ++ template ++ void ++ Hnsw::SaveOptimizedIndex(NmslibIOWriter& output) { ++ totalElementsStored_ = ElList_.size(); ++ ++ writeBinaryPOD(output, totalElementsStored_); ++ writeBinaryPOD(output, memoryPerObject_); ++ writeBinaryPOD(output, offsetLevel0_); ++ writeBinaryPOD(output, offsetData_); ++ writeBinaryPOD(output, maxlevel_); ++ writeBinaryPOD(output, enterpointId_); ++ writeBinaryPOD(output, maxM_); ++ writeBinaryPOD(output, maxM0_); ++ writeBinaryPOD(output, dist_func_type_); ++ writeBinaryPOD(output, searchMethod_); ++ ++ const size_t data_plus_links0_size = memoryPerObject_ * totalElementsStored_; ++ LOG(LIB_INFO) << "writing " << data_plus_links0_size << " bytes"; ++ output.write(data_level0_memory_, data_plus_links0_size); ++ ++ for (size_t i = 0; i < totalElementsStored_; i++) { ++ // TODO Can this one overflow? I really doubt ++ const SIZEMASS_TYPE sizemass = ((ElList_[i]->level) * (maxM_ + 1)) * sizeof(int); ++ writeBinaryPOD(output, sizemass); ++ if (sizemass) { ++ output.write(linkLists_[i], sizemass); ++ } ++ } ++ output.flush(); ++ } ++ + template + void + Hnsw::SaveRegularIndexBin(std::ostream& output) { +@@ -1036,20 +1080,20 @@ namespace similarity { + constexpr bool _isLittleEndian() { + return (((uint32_t) 1) & 0xFFU) == 1; + } +- ++ + SIZEMASS_TYPE _readIntBigEndian(uint8_t byte0, uint8_t byte1, uint8_t byte2, uint8_t byte3) noexcept { + return (static_cast(byte0) << 24) | + (static_cast(byte1) << 16) | + (static_cast(byte2) << 8) | + static_cast(byte3); +- } +- ++ } ++ + SIZEMASS_TYPE _readIntLittleEndian(uint8_t byte0, uint8_t byte1, uint8_t byte2, uint8_t byte3) noexcept { + return (static_cast(byte3) << 24) | + (static_cast(byte2) << 16) | + (static_cast(byte1) << 8) | + static_cast(byte0); +- } ++ } + + template + void Hnsw::LoadIndexWithStream(NmslibIOReader& input) { +-- +2.39.5 (Apple Git-154) + diff --git a/jni/src/commons.cpp b/jni/src/commons.cpp index 38e3ac8a4..444dc18b0 100644 --- a/jni/src/commons.cpp +++ b/jni/src/commons.cpp @@ -8,8 +8,6 @@ * Modifications Copyright OpenSearch Contributors. See * GitHub history for details. */ -#ifndef OPENSEARCH_KNN_COMMONS_H -#define OPENSEARCH_KNN_COMMONS_H #include #include @@ -109,4 +107,3 @@ int knn_jni::commons::getIntegerMethodParameter(JNIEnv * env, knn_jni::JNIUtilIn return defaultValue; } -#endif //OPENSEARCH_KNN_COMMONS_H diff --git a/jni/src/faiss_index_service.cpp b/jni/src/faiss_index_service.cpp index 16ded4bcb..4999e3172 100644 --- a/jni/src/faiss_index_service.cpp +++ b/jni/src/faiss_index_service.cpp @@ -9,7 +9,6 @@ #include "faiss_index_service.h" #include "faiss_methods.h" -#include "faiss/index_factory.h" #include "faiss/Index.h" #include "faiss/IndexBinary.h" #include "faiss/IndexHNSW.h" @@ -17,8 +16,7 @@ #include "faiss/IndexIVFFlat.h" #include "faiss/IndexBinaryIVF.h" #include "faiss/IndexIDMap.h" -#include "faiss/index_io.h" -#include + #include #include #include @@ -55,20 +53,18 @@ void SetExtraParameters(knn_jni::JNIUtilInterface * jniUtil, JNIEnv *env, } } -IndexService::IndexService(std::unique_ptr faissMethods) : faissMethods(std::move(faissMethods)) {} +IndexService::IndexService(std::unique_ptr _faissMethods) : faissMethods(std::move(_faissMethods)) {} void IndexService::allocIndex(faiss::Index * index, size_t dim, size_t numVectors) { - if(auto * indexHNSWSQ = dynamic_cast(index)) { - if(auto * indexScalarQuantizer = dynamic_cast(indexHNSWSQ->storage)) { + if (auto * indexHNSWSQ = dynamic_cast(index)) { + if (auto * indexScalarQuantizer = dynamic_cast(indexHNSWSQ->storage)) { indexScalarQuantizer->codes.reserve(indexScalarQuantizer->code_size * numVectors); } - return; } - if(auto * indexHNSW = dynamic_cast(index)) { + if (auto * indexHNSW = dynamic_cast(index)) { if(auto * indexFlat = dynamic_cast(indexHNSW->storage)) { indexFlat->codes.reserve(indexFlat->code_size * numVectors); } - return; } } @@ -86,7 +82,7 @@ jlong IndexService::initIndex( std::unique_ptr index(faissMethods->indexFactory(dim, indexDescription.c_str(), metric)); // Set thread count if it is passed in as a parameter. Setting this variable will only impact the current thread - if(threadCount != 0) { + if (threadCount != 0) { omp_set_num_threads(threadCount); } @@ -94,7 +90,7 @@ jlong IndexService::initIndex( SetExtraParameters(jniUtil, env, parameters, index.get()); // Check that the index does not need to be trained - if(!index->is_trained) { + if (!index->is_trained) { throw std::runtime_error("Index is not trained"); } @@ -123,7 +119,7 @@ void IndexService::insertToIndex( // The number of vectors can be int here because a lucene segment number of total docs never crosses INT_MAX value int numVectors = (int) (inputVectors->size() / (uint64_t) dim); - if(numVectors == 0) { + if (numVectors == 0) { throw std::runtime_error("Number of vectors cannot be 0"); } @@ -132,7 +128,7 @@ void IndexService::insertToIndex( } // Set thread count if it is passed in as a parameter. Setting this variable will only impact the current thread - if(threadCount != 0) { + if (threadCount != 0) { omp_set_num_threads(threadCount); } @@ -143,26 +139,30 @@ void IndexService::insertToIndex( } void IndexService::writeIndex( - std::string indexPath, - jlong idMapAddress - ) { + faiss::IOWriter* writer, + jlong idMapAddress +) { std::unique_ptr idMap (reinterpret_cast (idMapAddress)); try { // Write the index to disk - faissMethods->writeIndex(idMap.get(), indexPath.c_str()); + faissMethods->writeIndex(idMap.get(), writer); + if (auto openSearchIOWriter = dynamic_cast(writer)) { + openSearchIOWriter->flush(); + } } catch(std::exception &e) { throw std::runtime_error("Failed to write index to disk"); } } -BinaryIndexService::BinaryIndexService(std::unique_ptr faissMethods) : IndexService(std::move(faissMethods)) {} +BinaryIndexService::BinaryIndexService(std::unique_ptr _faissMethods) + : IndexService(std::move(_faissMethods)) { +} void BinaryIndexService::allocIndex(faiss::Index * index, size_t dim, size_t numVectors) { - if(auto * indexBinaryHNSW = dynamic_cast(index)) { + if (auto * indexBinaryHNSW = dynamic_cast(index)) { auto * indexBinaryFlat = dynamic_cast(indexBinaryHNSW->storage); indexBinaryFlat->xb.reserve(dim * numVectors / 8); - return; } } @@ -179,15 +179,15 @@ jlong BinaryIndexService::initIndex( // Create index using Faiss factory method std::unique_ptr index(faissMethods->indexBinaryFactory(dim, indexDescription.c_str())); // Set thread count if it is passed in as a parameter. Setting this variable will only impact the current thread - if(threadCount != 0) { - omp_set_num_threads(threadCount); + if (threadCount != 0) { + omp_set_num_threads(threadCount); } // Add extra parameters that cant be configured with the index factory SetExtraParameters(jniUtil, env, parameters, index.get()); // Check that the index does not need to be trained - if(!index->is_trained) { + if (!index->is_trained) { throw std::runtime_error("Index is not trained"); } @@ -216,7 +216,7 @@ void BinaryIndexService::insertToIndex( // The number of vectors can be int here because a lucene segment number of total docs never crosses INT_MAX value int numVectors = (int) (inputVectors->size() / (uint64_t) (dim / 8)); - if(numVectors == 0) { + if (numVectors == 0) { throw std::runtime_error("Number of vectors cannot be 0"); } @@ -225,7 +225,7 @@ void BinaryIndexService::insertToIndex( } // Set thread count if it is passed in as a parameter. Setting this variable will only impact the current thread - if(threadCount != 0) { + if (threadCount != 0) { omp_set_num_threads(threadCount); } @@ -236,28 +236,31 @@ void BinaryIndexService::insertToIndex( } void BinaryIndexService::writeIndex( - std::string indexPath, - jlong idMapAddress - ) { - + faiss::IOWriter* writer, + jlong idMapAddress +) { std::unique_ptr idMap (reinterpret_cast (idMapAddress)); try { // Write the index to disk - faissMethods->writeIndexBinary(idMap.get(), indexPath.c_str()); + faissMethods->writeIndexBinary(idMap.get(), writer); + if (auto openSearchIOWriter = dynamic_cast(writer)) { + openSearchIOWriter->flush(); + } } catch(std::exception &e) { throw std::runtime_error("Failed to write index to disk"); } } -ByteIndexService::ByteIndexService(std::unique_ptr faissMethods) : IndexService(std::move(faissMethods)) {} +ByteIndexService::ByteIndexService(std::unique_ptr _faissMethods) + : IndexService(std::move(_faissMethods)) { +} void ByteIndexService::allocIndex(faiss::Index * index, size_t dim, size_t numVectors) { - if(auto * indexHNSWSQ = dynamic_cast(index)) { + if (auto * indexHNSWSQ = dynamic_cast(index)) { if(auto * indexScalarQuantizer = dynamic_cast(indexHNSWSQ->storage)) { indexScalarQuantizer->codes.reserve(indexScalarQuantizer->code_size * numVectors); } - return; } } @@ -275,7 +278,7 @@ jlong ByteIndexService::initIndex( std::unique_ptr index(faissMethods->indexFactory(dim, indexDescription.c_str(), metric)); // Set thread count if it is passed in as a parameter. Setting this variable will only impact the current thread - if(threadCount != 0) { + if (threadCount != 0) { omp_set_num_threads(threadCount); } @@ -312,7 +315,7 @@ void ByteIndexService::insertToIndex( // The number of vectors can be int here because a lucene segment number of total docs never crosses INT_MAX value int numVectors = inputVectors->size() / dim; - if(numVectors == 0) { + if (numVectors == 0) { throw std::runtime_error("Number of vectors cannot be 0"); } @@ -321,7 +324,7 @@ void ByteIndexService::insertToIndex( } // Set thread count if it is passed in as a parameter. Setting this variable will only impact the current thread - if(threadCount != 0) { + if (threadCount != 0) { omp_set_num_threads(threadCount); } @@ -332,7 +335,6 @@ void ByteIndexService::insertToIndex( int batchSize = 1000; std::vector inputFloatVectors(batchSize * dim); std::vector floatVectorsIds(batchSize); - int id = 0; auto iter = inputVectors->begin(); for (int id = 0; id < numVectors; id += batchSize) { @@ -351,17 +353,20 @@ void ByteIndexService::insertToIndex( } void ByteIndexService::writeIndex( - std::string indexPath, - jlong idMapAddress - ) { + faiss::IOWriter* writer, + jlong idMapAddress +) { std::unique_ptr idMap (reinterpret_cast (idMapAddress)); try { // Write the index to disk - faissMethods->writeIndex(idMap.get(), indexPath.c_str()); + faissMethods->writeIndex(idMap.get(), writer); + if (auto openSearchIOWriter = dynamic_cast(writer)) { + openSearchIOWriter->flush(); + } } catch(std::exception &e) { throw std::runtime_error("Failed to write index to disk"); } } } // namespace faiss_wrapper -} // namesapce knn_jni \ No newline at end of file +} // namesapce knn_jni diff --git a/jni/src/faiss_methods.cpp b/jni/src/faiss_methods.cpp index 05c8f459a..dc44c0df9 100644 --- a/jni/src/faiss_methods.cpp +++ b/jni/src/faiss_methods.cpp @@ -29,11 +29,12 @@ faiss::IndexIDMapTemplate* FaissMethods::indexBinaryIdMap(fa return new faiss::IndexBinaryIDMap(index); } -void FaissMethods::writeIndex(const faiss::Index* idx, const char* fname) { - faiss::write_index(idx, fname); +void FaissMethods::writeIndex(const faiss::Index* idx, faiss::IOWriter* writer) { + faiss::write_index(idx, writer); } -void FaissMethods::writeIndexBinary(const faiss::IndexBinary* idx, const char* fname) { - faiss::write_index_binary(idx, fname); + +void FaissMethods::writeIndexBinary(const faiss::IndexBinary* idx, faiss::IOWriter* writer) { + faiss::write_index_binary(idx, writer); } } // namespace faiss_wrapper diff --git a/jni/src/faiss_wrapper.cpp b/jni/src/faiss_wrapper.cpp index d1c7648dc..98c40cf6b 100644 --- a/jni/src/faiss_wrapper.cpp +++ b/jni/src/faiss_wrapper.cpp @@ -13,13 +13,13 @@ #include "faiss_wrapper.h" #include "faiss_util.h" #include "faiss_index_service.h" +#include "faiss_stream_support.h" #include "faiss/impl/io.h" #include "faiss/index_factory.h" #include "faiss/index_io.h" #include "faiss/IndexHNSW.h" #include "faiss/IndexIVFFlat.h" -#include "faiss/MetaIndexes.h" #include "faiss/Index.h" #include "faiss/impl/IDSelector.h" #include "faiss/IndexIVFPQ.h" @@ -48,18 +48,25 @@ struct IDSelectorJlongBitmap : IDSelector { * @param n size of the bitmap array * @param bitmap id like Lucene FixedBitSet bits */ - IDSelectorJlongBitmap(size_t n, const jlong* bitmap) : n(n), bitmap(bitmap) {}; + IDSelectorJlongBitmap(size_t _n, const jlong* _bitmap) + : IDSelector(), + n(_n), + bitmap(_bitmap) { + } + bool is_member(idx_t id) const final { - uint64_t index = id; - uint64_t i = index >> 6; // div 64 - if (i >= n ) { + const uint64_t index = id; + const uint64_t i = index >> 6ULL; // div 64 + if (i >= n) { return false; } - return (bitmap[i] >> ( index & 63)) & 1L; + return (bitmap[i] >> (index & 63ULL)) & 1ULL; } - ~IDSelectorJlongBitmap() override {} -}; -} +}; // class IDSelectorJlongBitmap + +} // namespace faiss + + // Translate space type to faiss metric faiss::MetricType TranslateSpaceToMetric(const std::string& spaceType); @@ -136,7 +143,14 @@ jlong knn_jni::faiss_wrapper::InitIndex(knn_jni::JNIUtilInterface * jniUtil, JNI // end parameters to pass // Create index - return indexService->initIndex(jniUtil, env, metric, indexDescriptionCpp, dim, numDocs, threadCount, subParametersCpp); + return indexService->initIndex(jniUtil, + env, + metric, + std::move(indexDescriptionCpp), + dim, + numDocs, + threadCount, + std::move(subParametersCpp)); } void knn_jni::faiss_wrapper::InsertToIndex(knn_jni::JNIUtilInterface * jniUtil, JNIEnv * env, jintArray idsJ, jlong vectorsAddressJ, jint dimJ, @@ -170,21 +184,22 @@ void knn_jni::faiss_wrapper::InsertToIndex(knn_jni::JNIUtilInterface * jniUtil, } void knn_jni::faiss_wrapper::WriteIndex(knn_jni::JNIUtilInterface * jniUtil, JNIEnv * env, - jstring indexPathJ, jlong index_ptr, IndexService* indexService) { + jobject output, jlong index_ptr, IndexService* indexService) { - if (indexPathJ == nullptr) { - throw std::runtime_error("Index path cannot be null"); + if (output == nullptr) { + throw std::runtime_error("Index output stream cannot be null"); } - // Index path - std::string indexPathCpp(jniUtil->ConvertJavaStringToCppString(env, indexPathJ)); + // IndexOutput wrapper. + knn_jni::stream::NativeEngineIndexOutputMediator mediator {jniUtil, env, output}; + knn_jni::stream::FaissOpenSearchIOWriter writer {&mediator}; - // Create index - indexService->writeIndex(indexPathCpp, index_ptr); + // Create index. + indexService->writeIndex(&writer, index_ptr); } void knn_jni::faiss_wrapper::CreateIndexFromTemplate(knn_jni::JNIUtilInterface * jniUtil, JNIEnv * env, jintArray idsJ, - jlong vectorsAddressJ, jint dimJ, jstring indexPathJ, + jlong vectorsAddressJ, jint dimJ, jobject output, jbyteArray templateIndexJ, jobject parametersJ) { if (idsJ == nullptr) { throw std::runtime_error("IDs cannot be null"); @@ -198,8 +213,8 @@ void knn_jni::faiss_wrapper::CreateIndexFromTemplate(knn_jni::JNIUtilInterface * throw std::runtime_error("Vectors dimensions cannot be less than or equal to 0"); } - if (indexPathJ == nullptr) { - throw std::runtime_error("Index path cannot be null"); + if (output == nullptr) { + throw std::runtime_error("Index output stream cannot be null"); } if (templateIndexJ == nullptr) { @@ -245,14 +260,17 @@ void knn_jni::faiss_wrapper::CreateIndexFromTemplate(knn_jni::JNIUtilInterface * // This is not the ideal approach, please refer this gh issue for long term solution: // https://github.com/opensearch-project/k-NN/issues/1600 delete inputVectors; + // Write the index to disk - std::string indexPathCpp(jniUtil->ConvertJavaStringToCppString(env, indexPathJ)); - faiss::write_index(&idMap, indexPathCpp.c_str()); + knn_jni::stream::NativeEngineIndexOutputMediator mediator {jniUtil, env, output}; + knn_jni::stream::FaissOpenSearchIOWriter writer {&mediator}; + faiss::write_index(&idMap, &writer); + mediator.flush(); } void knn_jni::faiss_wrapper::CreateBinaryIndexFromTemplate(knn_jni::JNIUtilInterface * jniUtil, JNIEnv * env, jintArray idsJ, - jlong vectorsAddressJ, jint dimJ, jstring indexPathJ, - jbyteArray templateIndexJ, jobject parametersJ) { + jlong vectorsAddressJ, jint dimJ, jobject output, + jbyteArray templateIndexJ, jobject parametersJ) { if (idsJ == nullptr) { throw std::runtime_error("IDs cannot be null"); } @@ -261,12 +279,12 @@ void knn_jni::faiss_wrapper::CreateBinaryIndexFromTemplate(knn_jni::JNIUtilInter throw std::runtime_error("VectorsAddress cannot be less than 0"); } - if(dimJ <= 0) { + if (dimJ <= 0) { throw std::runtime_error("Vectors dimensions cannot be less than or equal to 0"); } - if (indexPathJ == nullptr) { - throw std::runtime_error("Index path cannot be null"); + if (output == nullptr) { + throw std::runtime_error("Index output stream cannot be null"); } if (templateIndexJ == nullptr) { @@ -315,14 +333,17 @@ void knn_jni::faiss_wrapper::CreateBinaryIndexFromTemplate(knn_jni::JNIUtilInter // This is not the ideal approach, please refer this gh issue for long term solution: // https://github.com/opensearch-project/k-NN/issues/1600 delete inputVectors; + // Write the index to disk - std::string indexPathCpp(jniUtil->ConvertJavaStringToCppString(env, indexPathJ)); - faiss::write_index_binary(&idMap, indexPathCpp.c_str()); + knn_jni::stream::NativeEngineIndexOutputMediator mediator {jniUtil, env, output}; + knn_jni::stream::FaissOpenSearchIOWriter writer {&mediator}; + faiss::write_index_binary(&idMap, &writer); + mediator.flush(); } void knn_jni::faiss_wrapper::CreateByteIndexFromTemplate(knn_jni::JNIUtilInterface * jniUtil, JNIEnv * env, jintArray idsJ, - jlong vectorsAddressJ, jint dimJ, jstring indexPathJ, - jbyteArray templateIndexJ, jobject parametersJ) { + jlong vectorsAddressJ, jint dimJ, jobject output, + jbyteArray templateIndexJ, jobject parametersJ) { if (idsJ == nullptr) { throw std::runtime_error("IDs cannot be null"); } @@ -331,12 +352,12 @@ void knn_jni::faiss_wrapper::CreateByteIndexFromTemplate(knn_jni::JNIUtilInterfa throw std::runtime_error("VectorsAddress cannot be less than 0"); } - if(dimJ <= 0) { + if (dimJ <= 0) { throw std::runtime_error("Vectors dimensions cannot be less than or equal to 0"); } - if (indexPathJ == nullptr) { - throw std::runtime_error("Index path cannot be null"); + if (output == nullptr) { + throw std::runtime_error("Index output stream cannot be null"); } if (templateIndexJ == nullptr) { @@ -345,8 +366,9 @@ void knn_jni::faiss_wrapper::CreateByteIndexFromTemplate(knn_jni::JNIUtilInterfa // Set thread count if it is passed in as a parameter. Setting this variable will only impact the current thread auto parametersCpp = jniUtil->ConvertJavaMapToCppMap(env, parametersJ); - if(parametersCpp.find(knn_jni::INDEX_THREAD_QUANTITY) != parametersCpp.end()) { - auto threadCount = jniUtil->ConvertJavaObjectToCppInteger(env, parametersCpp[knn_jni::INDEX_THREAD_QUANTITY]); + auto it = parametersCpp.find(knn_jni::INDEX_THREAD_QUANTITY); + if (it != parametersCpp.end()) { + auto threadCount = jniUtil->ConvertJavaObjectToCppInteger(env, it->second); omp_set_num_threads(threadCount); } jniUtil->DeleteLocalRef(env, parametersJ); @@ -354,8 +376,8 @@ void knn_jni::faiss_wrapper::CreateByteIndexFromTemplate(knn_jni::JNIUtilInterfa // Read data set // Read vectors from memory address auto *inputVectors = reinterpret_cast*>(vectorsAddressJ); - int dim = (int)dimJ; - int numVectors = (int) (inputVectors->size() / (uint64_t) dim); + auto dim = (int) dimJ; + auto numVectors = (int) (inputVectors->size() / (uint64_t) dim); int numIds = jniUtil->GetJavaIntArrayLength(env, idsJ); if (numIds != numVectors) { @@ -367,14 +389,14 @@ void knn_jni::faiss_wrapper::CreateByteIndexFromTemplate(knn_jni::JNIUtilInterfa jbyte * indexBytesJ = jniUtil->GetByteArrayElements(env, templateIndexJ, nullptr); faiss::VectorIOReader vectorIoReader; + vectorIoReader.data.reserve(indexBytesCount); for (int i = 0; i < indexBytesCount; i++) { vectorIoReader.data.push_back((uint8_t) indexBytesJ[i]); } jniUtil->ReleaseByteArrayElements(env, templateIndexJ, indexBytesJ, JNI_ABORT); // Create faiss index - std::unique_ptr indexWriter; - indexWriter.reset(faiss::read_index(&vectorIoReader, 0)); + std::unique_ptr indexWriter (faiss::read_index(&vectorIoReader, 0)); auto ids = jniUtil->ConvertJavaIntArrayToCppIntVector(env, idsJ); faiss::IndexIDMap idMap = faiss::IndexIDMap(indexWriter.get()); @@ -405,9 +427,12 @@ void knn_jni::faiss_wrapper::CreateByteIndexFromTemplate(knn_jni::JNIUtilInterfa // This is not the ideal approach, please refer this gh issue for long term solution: // https://github.com/opensearch-project/k-NN/issues/1600 delete inputVectors; + // Write the index to disk - std::string indexPathCpp(jniUtil->ConvertJavaStringToCppString(env, indexPathJ)); - faiss::write_index(&idMap, indexPathCpp.c_str()); + knn_jni::stream::NativeEngineIndexOutputMediator mediator {jniUtil, env, output}; + knn_jni::stream::FaissOpenSearchIOWriter writer {&mediator}; + faiss::write_index(&idMap, &writer); + mediator.flush(); } jlong knn_jni::faiss_wrapper::LoadIndex(knn_jni::JNIUtilInterface * jniUtil, JNIEnv * env, jstring indexPathJ) { @@ -424,7 +449,7 @@ jlong knn_jni::faiss_wrapper::LoadIndex(knn_jni::JNIUtilInterface * jniUtil, JNI } jlong knn_jni::faiss_wrapper::LoadIndexWithStream(faiss::IOReader* ioReader) { - if (ioReader == nullptr) [[unlikely]] { + if (ioReader == nullptr) { throw std::runtime_error("IOReader cannot be null"); } @@ -451,7 +476,7 @@ jlong knn_jni::faiss_wrapper::LoadBinaryIndex(knn_jni::JNIUtilInterface * jniUti } jlong knn_jni::faiss_wrapper::LoadBinaryIndexWithStream(faiss::IOReader* ioReader) { - if (ioReader == nullptr) [[unlikely]] { + if (ioReader == nullptr) { throw std::runtime_error("IOReader cannot be null"); } @@ -820,13 +845,13 @@ jbyteArray knn_jni::faiss_wrapper::TrainIndex(knn_jni::JNIUtilInterface * jniUti } // Set thread count if it is passed in as a parameter. Setting this variable will only impact the current thread - if(parametersCpp.find(knn_jni::INDEX_THREAD_QUANTITY) != parametersCpp.end()) { + if (parametersCpp.find(knn_jni::INDEX_THREAD_QUANTITY) != parametersCpp.end()) { auto threadCount = jniUtil->ConvertJavaObjectToCppInteger(env, parametersCpp[knn_jni::INDEX_THREAD_QUANTITY]); omp_set_num_threads(threadCount); } // Add extra parameters that cant be configured with the index factory - if(parametersCpp.find(knn_jni::PARAMETERS) != parametersCpp.end()) { + if (parametersCpp.find(knn_jni::PARAMETERS) != parametersCpp.end()) { jobject subParametersJ = parametersCpp[knn_jni::PARAMETERS]; auto subParametersCpp = jniUtil->ConvertJavaMapToCppMap(env, subParametersJ); SetExtraParameters(jniUtil, env, subParametersCpp, indexWriter.get()); @@ -934,13 +959,13 @@ jbyteArray knn_jni::faiss_wrapper::TrainByteIndex(knn_jni::JNIUtilInterface * jn indexWriter.reset(faiss::index_factory((int) dimensionJ, indexDescriptionCpp.c_str(), metric)); // Set thread count if it is passed in as a parameter. Setting this variable will only impact the current thread - if(parametersCpp.find(knn_jni::INDEX_THREAD_QUANTITY) != parametersCpp.end()) { + if (parametersCpp.find(knn_jni::INDEX_THREAD_QUANTITY) != parametersCpp.end()) { auto threadCount = jniUtil->ConvertJavaObjectToCppInteger(env, parametersCpp[knn_jni::INDEX_THREAD_QUANTITY]); omp_set_num_threads(threadCount); } // Add extra parameters that cant be configured with the index factory - if(parametersCpp.find(knn_jni::PARAMETERS) != parametersCpp.end()) { + if (parametersCpp.find(knn_jni::PARAMETERS) != parametersCpp.end()) { jobject subParametersJ = parametersCpp[knn_jni::PARAMETERS]; auto subParametersCpp = jniUtil->ConvertJavaMapToCppMap(env, subParametersJ); SetExtraParameters(jniUtil, env, subParametersCpp, indexWriter.get()); @@ -957,7 +982,7 @@ jbyteArray knn_jni::faiss_wrapper::TrainByteIndex(knn_jni::JNIUtilInterface * jn trainingFloatVectors[i] = static_cast(*iter); } - if(!indexWriter->is_trained) { + if (!indexWriter->is_trained) { InternalTrainIndex(indexWriter.get(), numVectors, trainingFloatVectors.data()); } jniUtil->DeleteLocalRef(env, parametersJ); @@ -1115,11 +1140,11 @@ jobjectArray knn_jni::faiss_wrapper::RangeSearchWithFilter(knn_jni::JNIUtilInter // The second parameter is always true, as lims is allocated by FAISS faiss::RangeSearchResult res(1, true); - if(filterIdsJ != nullptr) { + if (filterIdsJ != nullptr) { jlong *filteredIdsArray = jniUtil->GetLongArrayElements(env, filterIdsJ, nullptr); int filterIdsLength = jniUtil->GetJavaLongArrayLength(env, filterIdsJ); std::unique_ptr idSelector; - if(filterIdsTypeJ == BITMAP) { + if (filterIdsTypeJ == BITMAP) { idSelector.reset(new faiss::IDSelectorJlongBitmap(filterIdsLength, filteredIdsArray)); } else { faiss::idx_t* batchIndices = reinterpret_cast(filteredIdsArray); @@ -1131,7 +1156,7 @@ jobjectArray knn_jni::faiss_wrapper::RangeSearchWithFilter(knn_jni::JNIUtilInter std::unique_ptr idGrouper; std::vector idGrouperBitmap; auto hnswReader = dynamic_cast(indexReader->index); - if(hnswReader) { + if (hnswReader) { // Query param ef_search supersedes ef_search provided during index setting. hnswParams.efSearch = knn_jni::commons::getIntegerMethodParameter(env, jniUtil, methodParams, EF_SEARCH, hnswReader->hnsw.efSearch); hnswParams.sel = idSelector.get(); @@ -1195,7 +1220,7 @@ jobjectArray knn_jni::faiss_wrapper::RangeSearchWithFilter(knn_jni::JNIUtilInter jobjectArray results = jniUtil->NewObjectArray(env, resultSize, resultClass, nullptr); jobject result; - for(int i = 0; i < resultSize; ++i) { + for (int i = 0; i < resultSize; ++i) { result = jniUtil->NewObject(env, resultClass, allArgs, res.labels[i], res.distances[i]); jniUtil->SetObjectArrayElement(env, results, i, result); } diff --git a/jni/src/jni_util.cpp b/jni/src/jni_util.cpp index 8dc818c94..3ff79752a 100644 --- a/jni/src/jni_util.cpp +++ b/jni/src/jni_util.cpp @@ -88,7 +88,7 @@ void knn_jni::JNIUtil::HasExceptionInStack(JNIEnv* env) { this->HasExceptionInStack(env, "Exception in jni occurred"); } -void knn_jni::JNIUtil::HasExceptionInStack(JNIEnv* env, const std::string& message) { +void knn_jni::JNIUtil::HasExceptionInStack(JNIEnv* env, const char* message) { if (env->ExceptionCheck() == JNI_TRUE) { throw std::runtime_error(message); } @@ -252,11 +252,11 @@ void knn_jni::JNIUtil::Convert2dJavaObjectArrayAndStoreToFloatVector(JNIEnv *env throw std::runtime_error("Unable to get float array elements"); } - for(int j = 0; j < dim; ++j) { + for (int j = 0; j < dim; ++j) { vect->push_back(vector[j]); } env->ReleaseFloatArrayElements(vectorArray, vector, JNI_ABORT); - } + } // End for this->HasExceptionInStack(env); env->DeleteLocalRef(array2dJ); } @@ -285,7 +285,7 @@ void knn_jni::JNIUtil::Convert2dJavaObjectArrayAndStoreToBinaryVector(JNIEnv *en throw std::runtime_error("Unable to get byte array elements"); } - for(int j = 0; j < dim; ++j) { + for (int j = 0; j < dim; ++j) { vect->push_back(vector[j]); } env->ReleaseByteArrayElements(vectorArray, reinterpret_cast(vector), JNI_ABORT); @@ -573,6 +573,11 @@ jlong knn_jni::JNIUtil::CallNonvirtualLongMethodA(JNIEnv * env, jobject obj, jcl return env->CallNonvirtualLongMethodA(obj, clazz, methodID, args); } +void knn_jni::JNIUtil::CallNonvirtualVoidMethodA(JNIEnv * env, jobject obj, jclass clazz, + jmethodID methodID, jvalue* args) { + return env->CallNonvirtualVoidMethodA(obj, clazz, methodID, args); +} + void * knn_jni::JNIUtil::GetPrimitiveArrayCritical(JNIEnv * env, jarray array, jboolean *isCopy) { return env->GetPrimitiveArrayCritical(array, isCopy); } @@ -582,10 +587,11 @@ void knn_jni::JNIUtil::ReleasePrimitiveArrayCritical(JNIEnv * env, jarray array, } jobject knn_jni::GetJObjectFromMapOrThrow(std::unordered_map map, std::string key) { - if(map.find(key) == map.end()) { - throw std::runtime_error(key + " not found"); + auto it = map.find(key); + if (it != map.end()) { + return it->second; } - return map[key]; + throw std::runtime_error(key + " not found"); } //TODO: This potentially should use const char * diff --git a/jni/src/nmslib_wrapper.cpp b/jni/src/nmslib_wrapper.cpp index 536558caa..e8fc77fdb 100644 --- a/jni/src/nmslib_wrapper.cpp +++ b/jni/src/nmslib_wrapper.cpp @@ -26,7 +26,6 @@ #include #include -#include #include "hnswquery.h" #include "method/hnsw.h" @@ -39,7 +38,7 @@ const similarity::LabelType DEFAULT_LABEL = -1; void knn_jni::nmslib_wrapper::CreateIndex(knn_jni::JNIUtilInterface *jniUtil, JNIEnv *env, jintArray idsJ, jlong vectorsAddressJ, jint dimJ, - jstring indexPathJ, jobject parametersJ) { + jobject output, jobject parametersJ) { if (idsJ == nullptr) { throw std::runtime_error("IDs cannot be null"); @@ -53,8 +52,8 @@ void knn_jni::nmslib_wrapper::CreateIndex(knn_jni::JNIUtilInterface *jniUtil, JN throw std::runtime_error("Vectors dimensions cannot be less than or equal to 0"); } - if (indexPathJ == nullptr) { - throw std::runtime_error("Index path cannot be null"); + if (output == nullptr) { + throw std::runtime_error("Index output stream cannot be null"); } if (parametersJ == nullptr) { @@ -90,9 +89,6 @@ void knn_jni::nmslib_wrapper::CreateIndex(knn_jni::JNIUtilInterface *jniUtil, JN jniUtil->DeleteLocalRef(env, parametersJ); - // Get the path to save the index - std::string indexPathCpp(jniUtil->ConvertJavaStringToCppString(env, indexPathJ)); - // Get space type for this index jobject spaceTypeJ = knn_jni::GetJObjectFromMapOrThrow(parametersCpp, knn_jni::SPACE_TYPE); std::string spaceTypeCpp(jniUtil->ConvertJavaObjectToCppString(env, spaceTypeJ)); @@ -159,7 +155,9 @@ void knn_jni::nmslib_wrapper::CreateIndex(knn_jni::JNIUtilInterface *jniUtil, JN ptr += vectorSizeInBytes; vectorPointer += dim; } - jniUtil->ReleaseIntArrayElements(env, idsJ, idsCpp, JNI_ABORT); + JNIReleaseElements release_int_array_elements {[=](){ + jniUtil->ReleaseIntArrayElements(env, idsJ, idsCpp, JNI_ABORT); + }}; // Releasing the vectorsAddressJ memory as that is not required once we have created the index. // This is not the ideal approach, please refer this gh issue for long term solution: @@ -174,17 +172,24 @@ void knn_jni::nmslib_wrapper::CreateIndex(knn_jni::JNIUtilInterface *jniUtil, JN *(space), dataset)); index->CreateIndex(similarity::AnyParams(indexParameters)); - index->SaveIndex(indexPathCpp); - for (auto &it : dataset) { + knn_jni::stream::NativeEngineIndexOutputMediator mediator {jniUtil, env, output}; + knn_jni::stream::NmslibOpenSearchIOWriter writer {&mediator}; + + if (auto hnswFloatIndex = dynamic_cast *>(index.get())) { + hnswFloatIndex->SaveIndexWithStream(writer); + } else { + throw std::runtime_error("We only support similarity::Hnsw in NMSLIB."); + } + + for (auto it : dataset) { delete it; } } catch (...) { - for (auto &it : dataset) { + for (auto it : dataset) { delete it; } - jniUtil->ReleaseIntArrayElements(env, idsJ, idsCpp, JNI_ABORT); throw; } } @@ -317,25 +322,16 @@ jobjectArray knn_jni::nmslib_wrapper::QueryIndex(knn_jni::JNIUtilInterface *jniU } int queryEfSearch = knn_jni::commons::getIntegerMethodParameter(env, jniUtil, methodParams, EF_SEARCH, -1); - similarity::KNNQuery - *query; // TODO: Replace with smart pointers https://github.com/opensearch-project/k-NN/issues/1785 + std::unique_ptr> query; std::unique_ptr> neighbors; - try { - if (queryEfSearch == -1) { - query = new similarity::KNNQuery(*(indexWrapper->space), queryObject.get(), kJ); - } else { - query = new similarity::HNSWQuery(*(indexWrapper->space), queryObject.get(), kJ, queryEfSearch); - } - - indexWrapper->index->Search(query); - neighbors.reset(query->Result()->Clone()); - } catch (...) { - if (query != nullptr) { - delete query; - } - throw; + if (queryEfSearch == -1) { + query.reset(new similarity::KNNQuery(*(indexWrapper->space), queryObject.get(), kJ)); + } else { + query.reset(new similarity::HNSWQuery(*(indexWrapper->space), queryObject.get(), kJ, queryEfSearch)); } - delete query; + + indexWrapper->index->Search(query.get()); + neighbors.reset(query->Result()->Clone()); int resultSize = neighbors->Size(); jclass resultClass = jniUtil->FindClass(env, "org/opensearch/knn/index/query/KNNQueryResult"); diff --git a/jni/src/org_opensearch_knn_jni_FaissService.cpp b/jni/src/org_opensearch_knn_jni_FaissService.cpp index 7326c7ba0..836774402 100644 --- a/jni/src/org_opensearch_knn_jni_FaissService.cpp +++ b/jni/src/org_opensearch_knn_jni_FaissService.cpp @@ -41,8 +41,8 @@ void JNI_OnUnload(JavaVM *vm, void *reserved) { } JNIEXPORT jlong JNICALL Java_org_opensearch_knn_jni_FaissService_initIndex(JNIEnv * env, jclass cls, - jlong numDocs, jint dimJ, - jobject parametersJ) + jlong numDocs, jint dimJ, + jobject parametersJ) { try { std::unique_ptr faissMethods(new knn_jni::faiss_wrapper::FaissMethods()); @@ -55,8 +55,8 @@ JNIEXPORT jlong JNICALL Java_org_opensearch_knn_jni_FaissService_initIndex(JNIEn } JNIEXPORT jlong JNICALL Java_org_opensearch_knn_jni_FaissService_initBinaryIndex(JNIEnv * env, jclass cls, - jlong numDocs, jint dimJ, - jobject parametersJ) + jlong numDocs, jint dimJ, + jobject parametersJ) { try { std::unique_ptr faissMethods(new knn_jni::faiss_wrapper::FaissMethods()); @@ -69,8 +69,8 @@ JNIEXPORT jlong JNICALL Java_org_opensearch_knn_jni_FaissService_initBinaryIndex } JNIEXPORT jlong JNICALL Java_org_opensearch_knn_jni_FaissService_initByteIndex(JNIEnv * env, jclass cls, - jlong numDocs, jint dimJ, - jobject parametersJ) + jlong numDocs, jint dimJ, + jobject parametersJ) { try { std::unique_ptr faissMethods(new knn_jni::faiss_wrapper::FaissMethods()); @@ -83,8 +83,8 @@ JNIEXPORT jlong JNICALL Java_org_opensearch_knn_jni_FaissService_initByteIndex(J } JNIEXPORT void JNICALL Java_org_opensearch_knn_jni_FaissService_insertToIndex(JNIEnv * env, jclass cls, jintArray idsJ, - jlong vectorsAddressJ, jint dimJ, - jlong indexAddress, jint threadCount) + jlong vectorsAddressJ, jint dimJ, + jlong indexAddress, jint threadCount) { try { std::unique_ptr faissMethods(new knn_jni::faiss_wrapper::FaissMethods()); @@ -97,8 +97,8 @@ JNIEXPORT void JNICALL Java_org_opensearch_knn_jni_FaissService_insertToIndex(JN } JNIEXPORT void JNICALL Java_org_opensearch_knn_jni_FaissService_insertToBinaryIndex(JNIEnv * env, jclass cls, jintArray idsJ, - jlong vectorsAddressJ, jint dimJ, - jlong indexAddress, jint threadCount) + jlong vectorsAddressJ, jint dimJ, + jlong indexAddress, jint threadCount) { try { std::unique_ptr faissMethods(new knn_jni::faiss_wrapper::FaissMethods()); @@ -111,8 +111,8 @@ JNIEXPORT void JNICALL Java_org_opensearch_knn_jni_FaissService_insertToBinaryIn } JNIEXPORT void JNICALL Java_org_opensearch_knn_jni_FaissService_insertToByteIndex(JNIEnv * env, jclass cls, jintArray idsJ, - jlong vectorsAddressJ, jint dimJ, - jlong indexAddress, jint threadCount) + jlong vectorsAddressJ, jint dimJ, + jlong indexAddress, jint threadCount) { try { std::unique_ptr faissMethods(new knn_jni::faiss_wrapper::FaissMethods()); @@ -124,85 +124,112 @@ JNIEXPORT void JNICALL Java_org_opensearch_knn_jni_FaissService_insertToByteInde } } -JNIEXPORT void JNICALL Java_org_opensearch_knn_jni_FaissService_writeIndex(JNIEnv * env, jclass cls, - jlong indexAddress, - jstring indexPathJ) +JNIEXPORT void JNICALL Java_org_opensearch_knn_jni_FaissService_writeIndex(JNIEnv * env, + jclass cls, + jlong indexAddress, + jobject output) { - try { - std::unique_ptr faissMethods(new knn_jni::faiss_wrapper::FaissMethods()); - knn_jni::faiss_wrapper::IndexService indexService(std::move(faissMethods)); - knn_jni::faiss_wrapper::WriteIndex(&jniUtil, env, indexPathJ, indexAddress, &indexService); - } catch (...) { - jniUtil.CatchCppExceptionAndThrowJava(env); - } -} - -JNIEXPORT void JNICALL Java_org_opensearch_knn_jni_FaissService_writeBinaryIndex(JNIEnv * env, jclass cls, - jlong indexAddress, - jstring indexPathJ) + try { + std::unique_ptr faissMethods(new knn_jni::faiss_wrapper::FaissMethods()); + knn_jni::faiss_wrapper::IndexService indexService(std::move(faissMethods)); + knn_jni::faiss_wrapper::WriteIndex(&jniUtil, env, output, indexAddress, &indexService); + } catch (...) { + jniUtil.CatchCppExceptionAndThrowJava(env); + } +} + +JNIEXPORT void JNICALL Java_org_opensearch_knn_jni_FaissService_writeBinaryIndex(JNIEnv * env, + jclass cls, + jlong indexAddress, + jobject output) { - try { - std::unique_ptr faissMethods(new knn_jni::faiss_wrapper::FaissMethods()); - knn_jni::faiss_wrapper::BinaryIndexService binaryIndexService(std::move(faissMethods)); - knn_jni::faiss_wrapper::WriteIndex(&jniUtil, env, indexPathJ, indexAddress, &binaryIndexService); - } catch (...) { - jniUtil.CatchCppExceptionAndThrowJava(env); - } -} - -JNIEXPORT void JNICALL Java_org_opensearch_knn_jni_FaissService_writeByteIndex(JNIEnv * env, jclass cls, - jlong indexAddress, - jstring indexPathJ) + try { + std::unique_ptr faissMethods(new knn_jni::faiss_wrapper::FaissMethods()); + knn_jni::faiss_wrapper::BinaryIndexService binaryIndexService(std::move(faissMethods)); + knn_jni::faiss_wrapper::WriteIndex(&jniUtil, env, output, indexAddress, &binaryIndexService); + } catch (...) { + jniUtil.CatchCppExceptionAndThrowJava(env); + } +} + +JNIEXPORT void JNICALL Java_org_opensearch_knn_jni_FaissService_writeByteIndex(JNIEnv * env, + jclass cls, + jlong indexAddress, + jobject output) { - try { - std::unique_ptr faissMethods(new knn_jni::faiss_wrapper::FaissMethods()); - knn_jni::faiss_wrapper::ByteIndexService byteIndexService(std::move(faissMethods)); - knn_jni::faiss_wrapper::WriteIndex(&jniUtil, env, indexPathJ, indexAddress, &byteIndexService); - } catch (...) { - jniUtil.CatchCppExceptionAndThrowJava(env); - } + try { + std::unique_ptr faissMethods(new knn_jni::faiss_wrapper::FaissMethods()); + knn_jni::faiss_wrapper::ByteIndexService byteIndexService(std::move(faissMethods)); + knn_jni::faiss_wrapper::WriteIndex(&jniUtil, env, output, indexAddress, &byteIndexService); + } catch (...) { + jniUtil.CatchCppExceptionAndThrowJava(env); + } } -JNIEXPORT void JNICALL Java_org_opensearch_knn_jni_FaissService_createIndexFromTemplate(JNIEnv * env, jclass cls, +JNIEXPORT void JNICALL Java_org_opensearch_knn_jni_FaissService_createIndexFromTemplate(JNIEnv * env, + jclass cls, jintArray idsJ, jlong vectorsAddressJ, jint dimJ, - jstring indexPathJ, + jobject output, jbyteArray templateIndexJ, jobject parametersJ) { try { - knn_jni::faiss_wrapper::CreateIndexFromTemplate(&jniUtil, env, idsJ, vectorsAddressJ, dimJ, indexPathJ, templateIndexJ, parametersJ); + knn_jni::faiss_wrapper::CreateIndexFromTemplate(&jniUtil, + env, + idsJ, + vectorsAddressJ, + dimJ, + output, + templateIndexJ, + parametersJ); } catch (...) { jniUtil.CatchCppExceptionAndThrowJava(env); } } -JNIEXPORT void JNICALL Java_org_opensearch_knn_jni_FaissService_createBinaryIndexFromTemplate(JNIEnv * env, jclass cls, - jintArray idsJ, - jlong vectorsAddressJ, - jint dimJ, - jstring indexPathJ, - jbyteArray templateIndexJ, - jobject parametersJ) +JNIEXPORT void JNICALL Java_org_opensearch_knn_jni_FaissService_createBinaryIndexFromTemplate(JNIEnv * env, + jclass cls, + jintArray idsJ, + jlong vectorsAddressJ, + jint dimJ, + jobject output, + jbyteArray templateIndexJ, + jobject parametersJ) { try { - knn_jni::faiss_wrapper::CreateBinaryIndexFromTemplate(&jniUtil, env, idsJ, vectorsAddressJ, dimJ, indexPathJ, templateIndexJ, parametersJ); + knn_jni::faiss_wrapper::CreateBinaryIndexFromTemplate(&jniUtil, + env, + idsJ, + vectorsAddressJ, + dimJ, + output, + templateIndexJ, + parametersJ); } catch (...) { jniUtil.CatchCppExceptionAndThrowJava(env); } } -JNIEXPORT void JNICALL Java_org_opensearch_knn_jni_FaissService_createByteIndexFromTemplate(JNIEnv * env, jclass cls, - jintArray idsJ, - jlong vectorsAddressJ, - jint dimJ, - jstring indexPathJ, - jbyteArray templateIndexJ, - jobject parametersJ) +JNIEXPORT void JNICALL Java_org_opensearch_knn_jni_FaissService_createByteIndexFromTemplate(JNIEnv * env, + jclass cls, + jintArray idsJ, + jlong vectorsAddressJ, + jint dimJ, + jobject output, + jbyteArray templateIndexJ, + jobject parametersJ) { try { - knn_jni::faiss_wrapper::CreateByteIndexFromTemplate(&jniUtil, env, idsJ, vectorsAddressJ, dimJ, indexPathJ, templateIndexJ, parametersJ); + knn_jni::faiss_wrapper::CreateByteIndexFromTemplate(&jniUtil, + env, + idsJ, + vectorsAddressJ, + dimJ, + output, + templateIndexJ, + parametersJ); } catch (...) { jniUtil.CatchCppExceptionAndThrowJava(env); } @@ -210,16 +237,17 @@ JNIEXPORT void JNICALL Java_org_opensearch_knn_jni_FaissService_createByteIndexF JNIEXPORT jlong JNICALL Java_org_opensearch_knn_jni_FaissService_loadIndex(JNIEnv * env, jclass cls, jstring indexPathJ) { - try { - return knn_jni::faiss_wrapper::LoadIndex(&jniUtil, env, indexPathJ); - } catch (...) { - jniUtil.CatchCppExceptionAndThrowJava(env); - } - return NULL; + try { + return knn_jni::faiss_wrapper::LoadIndex(&jniUtil, env, indexPathJ); + } catch (...) { + jniUtil.CatchCppExceptionAndThrowJava(env); + } + return NULL; } -JNIEXPORT jlong JNICALL Java_org_opensearch_knn_jni_FaissService_loadIndexWithStream - (JNIEnv * env, jclass cls, jobject readStream) +JNIEXPORT jlong JNICALL Java_org_opensearch_knn_jni_FaissService_loadIndexWithStream(JNIEnv * env, + jclass cls, + jobject readStream) { try { // Create a mediator locally. @@ -249,8 +277,9 @@ JNIEXPORT jlong JNICALL Java_org_opensearch_knn_jni_FaissService_loadBinaryIndex return NULL; } -JNIEXPORT jlong JNICALL Java_org_opensearch_knn_jni_FaissService_loadBinaryIndexWithStream - (JNIEnv * env, jclass cls, jobject readStream) +JNIEXPORT jlong JNICALL Java_org_opensearch_knn_jni_FaissService_loadBinaryIndexWithStream(JNIEnv * env, + jclass cls, + jobject readStream) { try { // Create a mediator locally. @@ -262,7 +291,7 @@ JNIEXPORT jlong JNICALL Java_org_opensearch_knn_jni_FaissService_loadBinaryIndex // Pass IOReader to Faiss for loading vector index. return knn_jni::faiss_wrapper::LoadBinaryIndexWithStream( - &faissOpenSearchIOReader); + &faissOpenSearchIOReader); } catch (...) { jniUtil.CatchCppExceptionAndThrowJava(env); } @@ -270,8 +299,9 @@ JNIEXPORT jlong JNICALL Java_org_opensearch_knn_jni_FaissService_loadBinaryIndex return NULL; } -JNIEXPORT jboolean JNICALL Java_org_opensearch_knn_jni_FaissService_isSharedIndexStateRequired - (JNIEnv * env, jclass cls, jlong indexPointerJ) +JNIEXPORT jboolean JNICALL Java_org_opensearch_knn_jni_FaissService_isSharedIndexStateRequired(JNIEnv * env, + jclass cls, + jlong indexPointerJ) { try { return knn_jni::faiss_wrapper::IsSharedIndexStateRequired(indexPointerJ); @@ -425,10 +455,10 @@ JNIEXPORT jlong JNICALL Java_org_opensearch_knn_jni_FaissService_transferVectors } JNIEXPORT jobjectArray JNICALL Java_org_opensearch_knn_jni_FaissService_rangeSearchIndex(JNIEnv * env, jclass cls, - jlong indexPointerJ, - jfloatArray queryVectorJ, - jfloat radiusJ, jobject methodParamsJ, - jint maxResultWindowJ, jintArray parentIdsJ) + jlong indexPointerJ, + jfloatArray queryVectorJ, + jfloat radiusJ, jobject methodParamsJ, + jint maxResultWindowJ, jintArray parentIdsJ) { try { return knn_jni::faiss_wrapper::RangeSearch(&jniUtil, env, indexPointerJ, queryVectorJ, radiusJ, methodParamsJ, maxResultWindowJ, parentIdsJ); @@ -439,10 +469,10 @@ JNIEXPORT jobjectArray JNICALL Java_org_opensearch_knn_jni_FaissService_rangeSea } JNIEXPORT jobjectArray JNICALL Java_org_opensearch_knn_jni_FaissService_rangeSearchIndexWithFilter(JNIEnv * env, jclass cls, - jlong indexPointerJ, - jfloatArray queryVectorJ, - jfloat radiusJ, jobject methodParamsJ, jint maxResultWindowJ, - jlongArray filterIdsJ, jint filterIdsTypeJ, jintArray parentIdsJ) + jlong indexPointerJ, + jfloatArray queryVectorJ, + jfloat radiusJ, jobject methodParamsJ, jint maxResultWindowJ, + jlongArray filterIdsJ, jint filterIdsTypeJ, jintArray parentIdsJ) { try { return knn_jni::faiss_wrapper::RangeSearchWithFilter(&jniUtil, env, indexPointerJ, queryVectorJ, radiusJ, methodParamsJ, maxResultWindowJ, filterIdsJ, filterIdsTypeJ, parentIdsJ); diff --git a/jni/src/org_opensearch_knn_jni_NmslibService.cpp b/jni/src/org_opensearch_knn_jni_NmslibService.cpp index 8e4df2e9c..15bc8420e 100644 --- a/jni/src/org_opensearch_knn_jni_NmslibService.cpp +++ b/jni/src/org_opensearch_knn_jni_NmslibService.cpp @@ -41,10 +41,10 @@ JNIEXPORT void JNICALL Java_org_opensearch_knn_jni_NmslibService_createIndex(JNI jintArray idsJ, jlong vectorsAddressJ, jint dimJ, - jstring indexPathJ, + jobject output, jobject parametersJ) { try { - knn_jni::nmslib_wrapper::CreateIndex(&jniUtil, env, idsJ, vectorsAddressJ, dimJ, indexPathJ, parametersJ); + knn_jni::nmslib_wrapper::CreateIndex(&jniUtil, env, idsJ, vectorsAddressJ, dimJ, output, parametersJ); } catch (...) { jniUtil.CatchCppExceptionAndThrowJava(env); } diff --git a/jni/tests/faiss_index_service_test.cpp b/jni/tests/faiss_index_service_test.cpp index 8d9e4bb43..127ca07b8 100644 --- a/jni/tests/faiss_index_service_test.cpp +++ b/jni/tests/faiss_index_service_test.cpp @@ -19,9 +19,9 @@ #include "gtest/gtest.h" #include "commons.h" -using ::testing::_; using ::testing::NiceMock; using ::testing::Return; +using ::testing::_; TEST(CreateIndexTest, BasicAssertions) { // Define the data @@ -38,6 +38,7 @@ TEST(CreateIndexTest, BasicAssertions) { } std::string indexPath = test_util::RandomString(10, "tmp/", ".faiss"); + faiss::FileIOWriter fileIOWriter {indexPath.c_str()}; faiss::MetricType metricType = faiss::METRIC_L2; std::string indexDescription = "HNSW32,Flat"; int threadCount = 1; @@ -59,14 +60,14 @@ TEST(CreateIndexTest, BasicAssertions) { .WillOnce(Return(index)); EXPECT_CALL(*mockFaissMethods, indexIdMap(index)) .WillOnce(Return(indexIdMap)); - EXPECT_CALL(*mockFaissMethods, writeIndex(indexIdMap, ::testing::StrEq(indexPath.c_str()))) + EXPECT_CALL(*mockFaissMethods, writeIndex(indexIdMap, ::testing::Eq(&fileIOWriter))) .Times(1); // Create the index knn_jni::faiss_wrapper::IndexService indexService(std::move(mockFaissMethods)); long indexAddress = indexService.initIndex(&mockJNIUtil, jniEnv, metricType, indexDescription, dim, numIds, threadCount, parametersMap); indexService.insertToIndex(dim, numIds, threadCount, (int64_t) &vectors, ids, indexAddress); - indexService.writeIndex(indexPath, indexAddress); + indexService.writeIndex(&fileIOWriter, indexAddress); } TEST(CreateBinaryIndexTest, BasicAssertions) { @@ -84,6 +85,7 @@ TEST(CreateBinaryIndexTest, BasicAssertions) { } std::string indexPath = test_util::RandomString(10, "tmp/", ".faiss"); + faiss::FileIOWriter fileIOWriter {indexPath.c_str()}; faiss::MetricType metricType = faiss::METRIC_L2; std::string indexDescription = "BHNSW32"; int threadCount = 1; @@ -105,14 +107,14 @@ TEST(CreateBinaryIndexTest, BasicAssertions) { .WillOnce(Return(index)); EXPECT_CALL(*mockFaissMethods, indexBinaryIdMap(index)) .WillOnce(Return(indexIdMap)); - EXPECT_CALL(*mockFaissMethods, writeIndexBinary(indexIdMap, ::testing::StrEq(indexPath.c_str()))) + EXPECT_CALL(*mockFaissMethods, writeIndexBinary(indexIdMap, ::testing::Eq(&fileIOWriter))) .Times(1); // Create the index knn_jni::faiss_wrapper::BinaryIndexService indexService(std::move(mockFaissMethods)); long indexAddress = indexService.initIndex(&mockJNIUtil, jniEnv, metricType, indexDescription, dim, numIds, threadCount, parametersMap); indexService.insertToIndex(dim, numIds, threadCount, (int64_t) &vectors, ids, indexAddress); - indexService.writeIndex(indexPath, indexAddress); + indexService.writeIndex(&fileIOWriter, indexAddress); } TEST(CreateByteIndexTest, BasicAssertions) { @@ -130,6 +132,7 @@ TEST(CreateByteIndexTest, BasicAssertions) { } std::string indexPath = test_util::RandomString(10, "tmp/", ".faiss"); + faiss::FileIOWriter fileIOWriter {indexPath.c_str()}; faiss::MetricType metricType = faiss::METRIC_L2; std::string indexDescription = "HNSW16,SQ8_direct_signed"; int threadCount = 1; @@ -149,12 +152,12 @@ TEST(CreateByteIndexTest, BasicAssertions) { .WillOnce(Return(index)); EXPECT_CALL(*mockFaissMethods, indexIdMap(index)) .WillOnce(Return(indexIdMap)); - EXPECT_CALL(*mockFaissMethods, writeIndex(indexIdMap, ::testing::StrEq(indexPath.c_str()))) + EXPECT_CALL(*mockFaissMethods, writeIndex(indexIdMap, ::testing::Eq(&fileIOWriter))) .Times(1); // Create the index knn_jni::faiss_wrapper::ByteIndexService indexService(std::move(mockFaissMethods)); long indexAddress = indexService.initIndex(&mockJNIUtil, jniEnv, metricType, indexDescription, dim, numIds, threadCount, parametersMap); indexService.insertToIndex(dim, numIds, threadCount, (int64_t) &vectors, ids, indexAddress); - indexService.writeIndex(indexPath, indexAddress); + indexService.writeIndex(&fileIOWriter, indexAddress); } \ No newline at end of file diff --git a/jni/tests/faiss_stream_support_test.cpp b/jni/tests/faiss_stream_support_test.cpp index 94a9b3991..79beea7cb 100644 --- a/jni/tests/faiss_stream_support_test.cpp +++ b/jni/tests/faiss_stream_support_test.cpp @@ -62,7 +62,10 @@ TEST(FaissStreamSupportTest, NativeEngineIndexInputMediatorCopyWhenEmpty) { setUpMockJNIUtil(javaIndexInputMock, mockJni); // Prepare copying - NativeEngineIndexInputMediator mediator{&mockJni, nullptr, nullptr}; + NiceMock jniEnv; + // It's a dummy value, which will not be used. If we pass a null, then NPE will be raised. + jobject jobjectDummy = reinterpret_cast(1); + NativeEngineIndexInputMediator mediator{&mockJni, &jniEnv, jobjectDummy}; std::string readBuffer(javaIndexInputMock.readTargetBytes.size(), '\0'); // Call copyBytes @@ -82,7 +85,10 @@ TEST(FaissStreamSupportTest, FaissOpenSearchIOReaderCopy) { setUpMockJNIUtil(javaIndexInputMock, mockJni); // Prepare copying - NativeEngineIndexInputMediator mediator{&mockJni, nullptr, nullptr}; + NiceMock jniEnv; + // It's a dummy value, which will not be used. If we pass a null, then NPE will be raised. + jobject jobjectDummy = reinterpret_cast(1); + NativeEngineIndexInputMediator mediator{&mockJni, &jniEnv, jobjectDummy}; std::string readBuffer; readBuffer.resize(javaIndexInputMock.readTargetBytes.size()); FaissOpenSearchIOReader ioReader{&mediator}; diff --git a/jni/tests/faiss_wrapper_test.cpp b/jni/tests/faiss_wrapper_test.cpp index 5f6f83c46..651201964 100644 --- a/jni/tests/faiss_wrapper_test.cpp +++ b/jni/tests/faiss_wrapper_test.cpp @@ -20,17 +20,22 @@ #include "faiss/IndexHNSW.h" #include "faiss/IndexIVFPQ.h" #include "mocks/faiss_index_service_mock.h" +#include "native_stream_support_util.h" -using ::testing::_; +using ::test_util::JavaFileIndexOutputMock; +using ::test_util::MockJNIUtil; +using ::test_util::StreamIOError; +using ::test_util::setUpJavaFileOutputMocking; +using ::testing::Mock; using ::testing::NiceMock; using ::testing::Return; -using ::testing::Mock; +using ::testing::_; -float randomDataMin = -500.0; -float randomDataMax = 500.0; -float rangeSearchRandomDataMin = -50; -float rangeSearchRandomDataMax = 50; -float rangeSearchRadius = 20000; +const float randomDataMin = -500.0; +const float randomDataMax = 500.0; +const float rangeSearchRandomDataMin = -50; +const float rangeSearchRandomDataMax = 50; +const float rangeSearchRadius = 20000; void createIndexIteratively( knn_jni::JNIUtilInterface * JNIUtil, @@ -38,23 +43,25 @@ void createIndexIteratively( std::vector & ids, std::vector & vectors, int dim, - std::string & indexPath, - std::unordered_map parametersMap, + jobject javaFileOutputMock, + std::unordered_map parametersMap, IndexService * indexService, int insertions = 10 ) { long numDocs = ids.size(); - if(numDocs % insertions != 0) { + if (numDocs % insertions != 0) { throw std::invalid_argument("Number of documents should be divisible by number of insertions"); } long docsPerInsertion = numDocs / insertions; long index_ptr = knn_jni::faiss_wrapper::InitIndex(JNIUtil, jniEnv, numDocs, dim, (jobject)¶metersMap, indexService); - for(int i = 0; i < insertions; i++) { + std::vector insertIds; + std::vector insertVecs; + for (int i = 0; i < insertions; i++) { + insertIds.clear(); + insertVecs.clear(); int start_idx = i * docsPerInsertion; int end_idx = start_idx + docsPerInsertion; - std::vector insertIds; - std::vector insertVecs; - for(int j = start_idx; j < end_idx; j++) { + for (int j = start_idx; j < end_idx; j++) { insertIds.push_back(j); for(int k = 0; k < dim; k++) { insertVecs.push_back(vectors[j * dim + k]); @@ -62,7 +69,7 @@ void createIndexIteratively( } knn_jni::faiss_wrapper::InsertToIndex(JNIUtil, jniEnv, reinterpret_cast(&insertIds), (jlong)&insertVecs, dim, index_ptr, 0, indexService); } - knn_jni::faiss_wrapper::WriteIndex(JNIUtil, jniEnv, (jstring)&indexPath, index_ptr, indexService); + knn_jni::faiss_wrapper::WriteIndex(JNIUtil, jniEnv, javaFileOutputMock, index_ptr, indexService); } void createBinaryIndexIteratively( @@ -71,21 +78,25 @@ void createBinaryIndexIteratively( std::vector & ids, std::vector & vectors, int dim, - std::string & indexPath, + jobject javaFileOutputMock, std::unordered_map parametersMap, IndexService * indexService, int insertions = 10 ) { - long numDocs = ids.size();; + long numDocs = ids.size(); long index_ptr = knn_jni::faiss_wrapper::InitIndex(JNIUtil, jniEnv, numDocs, dim, (jobject)¶metersMap, indexService); - for(int i = 0; i < insertions; i++) { + std::vector insertIds; + std::vector insertVecs; + for (int i = 0; i < insertions; i++) { int start_idx = numDocs * i / insertions; int end_idx = numDocs * (i + 1) / insertions; int docs_to_insert = end_idx - start_idx; - if(docs_to_insert == 0) continue; - std::vector insertIds; - std::vector insertVecs; - for(int j = start_idx; j < end_idx; j++) { + if (docs_to_insert == 0) { + continue; + } + insertIds.clear(); + insertVecs.clear(); + for (int j = start_idx; j < end_idx; j++) { insertIds.push_back(j); for(int k = 0; k < dim / 8; k++) { insertVecs.push_back(vectors[j * (dim / 8) + k]); @@ -93,7 +104,8 @@ void createBinaryIndexIteratively( } knn_jni::faiss_wrapper::InsertToIndex(JNIUtil, jniEnv, reinterpret_cast(&insertIds), (jlong)&insertVecs, dim, index_ptr, 0, indexService); } - knn_jni::faiss_wrapper::WriteIndex(JNIUtil, jniEnv, (jstring)&indexPath, index_ptr, indexService); + + knn_jni::faiss_wrapper::WriteIndex(JNIUtil, jniEnv, javaFileOutputMock, index_ptr, indexService); } TEST(FaissCreateIndexTest, BasicAssertions) { @@ -104,10 +116,10 @@ TEST(FaissCreateIndexTest, BasicAssertions) { int dim = 2; vectors.reserve(dim * numIds); for (int64_t i = 0; i < numIds; ++i) { - ids.push_back(i); - for (int j = 0; j < dim; ++j) { - vectors.push_back(test_util::RandomFloat(-500.0, 500.0)); - } + ids.push_back(i); + for (int j = 0; j < dim; ++j) { + vectors.push_back(test_util::RandomFloat(-500.0, 500.0)); + } } std::string indexPath = test_util::RandomString(10, "tmp/", ".faiss"); @@ -121,8 +133,10 @@ TEST(FaissCreateIndexTest, BasicAssertions) { parametersMap[knn_jni::PARAMETERS] = (jobject)&subParametersMap; // Set up jni - JNIEnv *jniEnv = nullptr; + NiceMock jniEnv; NiceMock mockJNIUtil; + JavaFileIndexOutputMock javaFileIndexOutputMock {indexPath}; + setUpJavaFileOutputMocking(javaFileIndexOutputMock, mockJNIUtil, false); // Create the index std::unique_ptr faissMethods(new FaissMethods()); @@ -132,10 +146,18 @@ TEST(FaissCreateIndexTest, BasicAssertions) { .Times(1); EXPECT_CALL(mockIndexService, insertToIndex(dim, numIds / insertions, 0, _, _, _)) .Times(insertions); - EXPECT_CALL(mockIndexService, writeIndex(indexPath, _)) + EXPECT_CALL(mockIndexService, writeIndex(_, _)) .Times(1); - createIndexIteratively(&mockJNIUtil, jniEnv, ids, vectors, dim, indexPath, parametersMap, &mockIndexService, insertions); + createIndexIteratively(&mockJNIUtil, + &jniEnv, + ids, + vectors, + dim, + (jobject) (&javaFileIndexOutputMock), + parametersMap, + &mockIndexService, + insertions); } TEST(FaissCreateBinaryIndexTest, BasicAssertions) { @@ -146,10 +168,10 @@ TEST(FaissCreateBinaryIndexTest, BasicAssertions) { int dim = 128; vectors.reserve(numIds); for (int64_t i = 0; i < numIds; ++i) { - ids.push_back(i); - for (int j = 0; j < dim / 8; ++j) { - vectors.push_back(test_util::RandomInt(0, 255)); - } + ids.push_back(i); + for (int j = 0; j < dim / 8; ++j) { + vectors.push_back(test_util::RandomInt(0, 255)); + } } std::string indexPath = test_util::RandomString(10, "tmp/", ".faiss"); @@ -163,8 +185,10 @@ TEST(FaissCreateBinaryIndexTest, BasicAssertions) { parametersMap[knn_jni::PARAMETERS] = (jobject)&subParametersMap; // Set up jni - JNIEnv *jniEnv = nullptr; + NiceMock jniEnv; NiceMock mockJNIUtil; + JavaFileIndexOutputMock javaFileIndexOutputMock {indexPath}; + setUpJavaFileOutputMocking(javaFileIndexOutputMock, mockJNIUtil, false); // Create the index std::unique_ptr faissMethods(new FaissMethods()); @@ -174,109 +198,132 @@ TEST(FaissCreateBinaryIndexTest, BasicAssertions) { .Times(1); EXPECT_CALL(mockIndexService, insertToIndex(dim, numIds / insertions, 0, _, _, _)) .Times(insertions); - EXPECT_CALL(mockIndexService, writeIndex(indexPath, _)) + EXPECT_CALL(mockIndexService, writeIndex(_, _)) .Times(1); // This method calls delete vectors at the end - createBinaryIndexIteratively(&mockJNIUtil, jniEnv, ids, vectors, dim, indexPath, parametersMap, &mockIndexService, insertions); + createBinaryIndexIteratively(&mockJNIUtil, + &jniEnv, + ids, + vectors, + dim, + (jobject) (&javaFileIndexOutputMock), + parametersMap, + &mockIndexService, + insertions); } TEST(FaissCreateIndexFromTemplateTest, BasicAssertions) { - // Define the data - faiss::idx_t numIds = 100; - std::vector ids; - auto *vectors = new std::vector(); - int dim = 2; - vectors->reserve(dim * numIds); - for (int64_t i = 0; i < numIds; ++i) { - ids.push_back(i); - for (int j = 0; j < dim; ++j) { + for (auto throwIOException : std::array {false, true}) { + // Define the data + faiss::idx_t numIds = 100; + std::vector ids; + auto *vectors = new std::vector(); + int dim = 2; + vectors->reserve(dim * numIds); + for (int64_t i = 0; i < numIds; ++i) { + ids.push_back(i); + for (int j = 0; j < dim; ++j) { vectors->push_back(test_util::RandomFloat(-500.0, 500.0)); + } } - } - std::string indexPath = test_util::RandomString(10, "tmp/", ".faiss"); - faiss::MetricType metricType = faiss::METRIC_L2; - std::string method = "HNSW32,Flat"; + std::string indexPath = test_util::RandomString(10, "tmp/", ".faiss"); + faiss::MetricType metricType = faiss::METRIC_L2; + std::string method = "HNSW32,Flat"; - std::unique_ptr createdIndex( + std::unique_ptr createdIndex( test_util::FaissCreateIndex(dim, method, metricType)); - auto vectorIoWriter = test_util::FaissGetSerializedIndex(createdIndex.get()); - - // Setup jni - JNIEnv *jniEnv = nullptr; - NiceMock mockJNIUtil; - - EXPECT_CALL(mockJNIUtil, - GetJavaObjectArrayLength( - jniEnv, reinterpret_cast(&vectors))) - .WillRepeatedly(Return(vectors->size())); - - std::string spaceType = knn_jni::L2; - std::unordered_map parametersMap; - parametersMap[knn_jni::SPACE_TYPE] = (jobject) &spaceType; + auto vectorIoWriter = test_util::FaissGetSerializedIndex(createdIndex.get()); + + // Setup jni + NiceMock jniEnv; + NiceMock mockJNIUtil; + JavaFileIndexOutputMock javaFileIndexOutputMock {indexPath}; + setUpJavaFileOutputMocking(javaFileIndexOutputMock, mockJNIUtil, throwIOException); + + std::string spaceType = knn_jni::L2; + std::unordered_map parametersMap; + parametersMap[knn_jni::SPACE_TYPE] = (jobject) &spaceType; + + try { + knn_jni::faiss_wrapper::CreateIndexFromTemplate( + &mockJNIUtil, &jniEnv, reinterpret_cast(&ids), + (jlong)vectors, dim, (jobject)(&javaFileIndexOutputMock), + reinterpret_cast(&(vectorIoWriter.data)), + (jobject) ¶metersMap); + javaFileIndexOutputMock.file_writer.close(); + } catch (const StreamIOError& e) { + ASSERT_TRUE(throwIOException); + continue; + } - knn_jni::faiss_wrapper::CreateIndexFromTemplate( - &mockJNIUtil, jniEnv, reinterpret_cast(&ids), - (jlong)vectors, dim, (jstring)&indexPath, - reinterpret_cast(&(vectorIoWriter.data)), - (jobject) ¶metersMap - ); + ASSERT_FALSE(throwIOException); - // Make sure index can be loaded - std::unique_ptr index(test_util::FaissLoadIndex(indexPath)); + // Make sure index can be loaded + std::unique_ptr index(test_util::FaissLoadIndex(indexPath)); - // Clean up - std::remove(indexPath.c_str()); + // Clean up + std::remove(indexPath.c_str()); + } // End for } TEST(FaissCreateByteIndexFromTemplateTest, BasicAssertions) { - // Define the data - faiss::idx_t numIds = 100; - std::vector ids; - auto *vectors = new std::vector(); - int dim = 8; - vectors->reserve(dim * numIds); - for (int64_t i = 0; i < numIds; ++i) { - ids.push_back(i); - for (int j = 0; j < dim; ++j) { + for (auto throwIOException : std::array {false, true}) { + // Define the data + faiss::idx_t numIds = 100; + std::vector ids; + auto *vectors = new std::vector(); + int dim = 8; + vectors->reserve(dim * numIds); + for (int64_t i = 0; i < numIds; ++i) { + ids.push_back(i); + for (int j = 0; j < dim; ++j) { vectors->push_back(test_util::RandomInt(-128, 127)); + } } - } - std::string indexPath = test_util::RandomString(10, "tmp/", ".faiss"); - faiss::MetricType metricType = faiss::METRIC_L2; - std::string method = "HNSW32,SQ8_direct_signed"; + std::string indexPath = test_util::RandomString(10, "tmp/", ".faiss"); + faiss::MetricType metricType = faiss::METRIC_L2; + std::string method = "HNSW32,SQ8_direct_signed"; - std::unique_ptr createdIndex( + std::unique_ptr createdIndex( test_util::FaissCreateIndex(dim, method, metricType)); - auto vectorIoWriter = test_util::FaissGetSerializedIndex(createdIndex.get()); - - // Setup jni - JNIEnv *jniEnv = nullptr; - NiceMock mockJNIUtil; - - EXPECT_CALL(mockJNIUtil, - GetJavaObjectArrayLength( - jniEnv, reinterpret_cast(&vectors))) - .WillRepeatedly(Return(vectors->size())); + auto vectorIoWriter = test_util::FaissGetSerializedIndex(createdIndex.get()); + + // Setup jni + NiceMock jniEnv; + NiceMock mockJNIUtil; + JavaFileIndexOutputMock javaFileIndexOutputMock {indexPath}; + setUpJavaFileOutputMocking(javaFileIndexOutputMock, mockJNIUtil, throwIOException); + + std::string spaceType = knn_jni::L2; + std::unordered_map parametersMap; + parametersMap[knn_jni::SPACE_TYPE] = (jobject) &spaceType; + + try { + knn_jni::faiss_wrapper::CreateByteIndexFromTemplate( + &mockJNIUtil, &jniEnv, reinterpret_cast(&ids), + (jlong) vectors, dim, (jstring) (&javaFileIndexOutputMock), + reinterpret_cast(&(vectorIoWriter.data)), + (jobject) ¶metersMap + ); - std::string spaceType = knn_jni::L2; - std::unordered_map parametersMap; - parametersMap[knn_jni::SPACE_TYPE] = (jobject) &spaceType; + // Make sure we close a file stream before reopening the created file. + javaFileIndexOutputMock.file_writer.close(); + } catch (const StreamIOError& e) { + ASSERT_TRUE(throwIOException); + continue; + } - knn_jni::faiss_wrapper::CreateByteIndexFromTemplate( - &mockJNIUtil, jniEnv, reinterpret_cast(&ids), - (jlong)vectors, dim, (jstring)&indexPath, - reinterpret_cast(&(vectorIoWriter.data)), - (jobject) ¶metersMap - ); + ASSERT_FALSE(throwIOException); - // Make sure index can be loaded - std::unique_ptr index(test_util::FaissLoadIndex(indexPath)); + // Make sure index can be loaded + std::unique_ptr index(test_util::FaissLoadIndex(indexPath)); - // Clean up - std::remove(indexPath.c_str()); + // Clean up + std::remove(indexPath.c_str()); + } // End for } TEST(FaissLoadIndexTest, BasicAssertions) { @@ -299,12 +346,12 @@ TEST(FaissLoadIndexTest, BasicAssertions) { test_util::FaissWriteIndex(&createdIndexWithData, indexPath); // Setup jni - JNIEnv *jniEnv = nullptr; + NiceMock jniEnv; NiceMock mockJNIUtil; std::unique_ptr loadedIndexPointer( reinterpret_cast(knn_jni::faiss_wrapper::LoadIndex( - &mockJNIUtil, jniEnv, (jstring)&indexPath))); + &mockJNIUtil, &jniEnv, (jstring)&indexPath))); // Compare serialized versions auto createIndexSerialization = @@ -339,7 +386,6 @@ TEST(FaissLoadBinaryIndexTest, BasicAssertions) { } std::string indexPath = test_util::RandomString(10, "tmp/", ".faiss"); - std::string spaceType = knn_jni::HAMMING; std::string method = "BHNSW32"; // Create the index @@ -351,12 +397,12 @@ TEST(FaissLoadBinaryIndexTest, BasicAssertions) { test_util::FaissWriteBinaryIndex(&createdIndexWithData, indexPath); // Setup jni - JNIEnv *jniEnv = nullptr; + NiceMock jniEnv; NiceMock mockJNIUtil; std::unique_ptr loadedIndexPointer( reinterpret_cast(knn_jni::faiss_wrapper::LoadBinaryIndex( - &mockJNIUtil, jniEnv, (jstring)&indexPath))); + &mockJNIUtil, &jniEnv, (jstring)&indexPath))); // Compare serialized versions auto createIndexSerialization = @@ -394,12 +440,12 @@ TEST(FaissLoadIndexTest, HNSWPQDisableSdcTable) { test_util::FaissWriteIndex(&faissIndexWithIDMap, indexPath); // Setup jni - JNIEnv *jniEnv = nullptr; + NiceMock jniEnv; NiceMock mockJNIUtil; std::unique_ptr loadedIndexPointer( reinterpret_cast(knn_jni::faiss_wrapper::LoadIndex( - &mockJNIUtil, jniEnv, (jstring)&indexPath))); + &mockJNIUtil, &jniEnv, (jstring)&indexPath))); // Cast down until we get to the pq backed storage index and checke the size of the table auto idMapIndex = dynamic_cast(loadedIndexPointer.get()); @@ -427,12 +473,12 @@ TEST(FaissLoadIndexTest, IVFPQDisablePrecomputeTable) { test_util::FaissWriteIndex(&faissIndexWithIDMap, indexPath); // Setup jni - JNIEnv *jniEnv = nullptr; + NiceMock jniEnv; NiceMock mockJNIUtil; std::unique_ptr loadedIndexPointer( reinterpret_cast(knn_jni::faiss_wrapper::LoadIndex( - &mockJNIUtil, jniEnv, (jstring)&indexPath))); + &mockJNIUtil, &jniEnv, (jstring)&indexPath))); // Cast down until we get to the ivfpq-l2 state auto idMapIndex = dynamic_cast(loadedIndexPointer.get()); @@ -477,7 +523,7 @@ TEST(FaissQueryIndexTest, BasicAssertions) { test_util::FaissAddData(createdIndex.get(), ids, vectors); // Setup jni - JNIEnv *jniEnv = nullptr; + NiceMock jniEnv; NiceMock mockJNIUtil; auto methodParamsJ = reinterpret_cast(&methodParams); @@ -485,7 +531,7 @@ TEST(FaissQueryIndexTest, BasicAssertions) { std::unique_ptr *>> results( reinterpret_cast *> *>( knn_jni::faiss_wrapper::QueryIndex( - &mockJNIUtil, jniEnv, + &mockJNIUtil, &jniEnv, reinterpret_cast(&createdIndexWithData), reinterpret_cast(&query), k, methodParamsJ, nullptr))); @@ -533,14 +579,14 @@ TEST(FaissQueryBinaryIndexTest, BasicAssertions) { test_util::FaissAddBinaryData(createdIndex.get(), ids, vectors); // Setup jni - JNIEnv *jniEnv = nullptr; + NiceMock jniEnv; NiceMock mockJNIUtil; for (auto query : queries) { std::unique_ptr *>> results( reinterpret_cast *> *>( knn_jni::faiss_wrapper::QueryBinaryIndex_WithFilter( - &mockJNIUtil, jniEnv, + &mockJNIUtil, &jniEnv, reinterpret_cast(&createdIndexWithData), reinterpret_cast(&query), k, nullptr, nullptr, 0, nullptr))); @@ -594,11 +640,11 @@ TEST(FaissQueryIndexWithFilterTest1435, BasicAssertions) { test_util::FaissAddData(createdIndex.get(), ids, vectors); // Setup jni - JNIEnv *jniEnv = nullptr; + NiceMock jniEnv; NiceMock mockJNIUtil; EXPECT_CALL(mockJNIUtil, GetJavaLongArrayLength( - jniEnv, reinterpret_cast(&bitmap))) + &jniEnv, reinterpret_cast(&bitmap))) .WillRepeatedly(Return(bitmap.size())); int k = 20; @@ -606,7 +652,7 @@ TEST(FaissQueryIndexWithFilterTest1435, BasicAssertions) { std::unique_ptr *>> results( reinterpret_cast *> *>( knn_jni::faiss_wrapper::QueryIndex_WithFilter( - &mockJNIUtil, jniEnv, + &mockJNIUtil, &jniEnv, reinterpret_cast(&createdIndexWithData), reinterpret_cast(&query), k, nullptr, reinterpret_cast(&bitmap), 0, nullptr))); @@ -671,17 +717,17 @@ TEST(FaissQueryIndexWithParentFilterTest, BasicAssertions) { methodParams[knn_jni::EF_SEARCH] = reinterpret_cast(&efSearch); // Setup jni - JNIEnv *jniEnv = nullptr; + NiceMock jniEnv; NiceMock mockJNIUtil; EXPECT_CALL(mockJNIUtil, GetJavaIntArrayLength( - jniEnv, reinterpret_cast(&parentIds))) + &jniEnv, reinterpret_cast(&parentIds))) .WillRepeatedly(Return(parentIds.size())); for (auto query : queries) { std::unique_ptr *>> results( reinterpret_cast *> *>( knn_jni::faiss_wrapper::QueryIndex( - &mockJNIUtil, jniEnv, + &mockJNIUtil, &jniEnv, reinterpret_cast(&createdIndexWithData), reinterpret_cast(&query), k, reinterpret_cast(&methodParams), reinterpret_cast(&parentIds)))); @@ -749,14 +795,14 @@ TEST(FaissTrainIndexTest, BasicAssertions) { std::vector trainingVectors = test_util::RandomVectors(dim, numTrainingVectors, randomDataMin, randomDataMax); // Setup jni - JNIEnv *jniEnv = nullptr; + NiceMock jniEnv; NiceMock mockJNIUtil; // Perform training std::unique_ptr> trainedIndexSerialization( reinterpret_cast *>( knn_jni::faiss_wrapper::TrainIndex( - &mockJNIUtil, jniEnv, (jobject) ¶metersMap, dim, + &mockJNIUtil, &jniEnv, (jobject) ¶metersMap, dim, reinterpret_cast(&trainingVectors)))); std::unique_ptr trainedIndex( @@ -781,14 +827,14 @@ TEST(FaissTrainByteIndexTest, BasicAssertions) { std::vector trainingVectors = test_util::RandomByteVectors(dim, numTrainingVectors, -128, 127); // Setup jni - JNIEnv *jniEnv = nullptr; + NiceMock jniEnv; NiceMock mockJNIUtil; // Perform training std::unique_ptr> trainedIndexSerialization( reinterpret_cast *>( knn_jni::faiss_wrapper::TrainByteIndex( - &mockJNIUtil, jniEnv, (jobject) ¶metersMap, dim, + &mockJNIUtil, &jniEnv, (jobject) ¶metersMap, dim, reinterpret_cast(&trainingVectors)))); std::unique_ptr trainedIndex( @@ -812,7 +858,6 @@ TEST(FaissCreateHnswSQfp16IndexTest, BasicAssertions) { } } - std::string indexPath = test_util::RandomString(10, "tmp/", ".faiss"); std::string spaceType = knn_jni::L2; std::string index_description = "HNSW32,SQfp16"; @@ -820,31 +865,46 @@ TEST(FaissCreateHnswSQfp16IndexTest, BasicAssertions) { parametersMap[knn_jni::SPACE_TYPE] = (jobject)&spaceType; parametersMap[knn_jni::INDEX_DESCRIPTION] = (jobject)&index_description; - // Set up jni - JNIEnv *jniEnv = nullptr; - NiceMock mockJNIUtil; + for (auto throwIOException : std::array {false, true}) { + const std::string indexPath = test_util::RandomString(10, "tmp/", ".faiss"); - EXPECT_CALL(mockJNIUtil, - GetJavaObjectArrayLength( - jniEnv, reinterpret_cast(&vectors))) + // Set up jni + NiceMock jniEnv; + NiceMock mockJNIUtil; + JavaFileIndexOutputMock javaFileIndexOutputMock {indexPath}; + setUpJavaFileOutputMocking(javaFileIndexOutputMock, mockJNIUtil, throwIOException); + + EXPECT_CALL(mockJNIUtil, + GetJavaObjectArrayLength( + &jniEnv, reinterpret_cast(&vectors))) .WillRepeatedly(Return(vectors.size())); - // Create the index - std::unique_ptr faissMethods(new FaissMethods()); - knn_jni::faiss_wrapper::IndexService IndexService(std::move(faissMethods)); - - createIndexIteratively(&mockJNIUtil, jniEnv, ids, vectors, dim, indexPath, parametersMap, &IndexService); - - // Make sure index can be loaded - std::unique_ptr index(test_util::FaissLoadIndex(indexPath)); - auto indexIDMap = dynamic_cast(index.get()); - - // Assert that Index is of type IndexHNSWSQ - ASSERT_NE(indexIDMap, nullptr); - ASSERT_NE(dynamic_cast(indexIDMap->index), nullptr); - - // Clean up - std::remove(indexPath.c_str()); + // Create the index + std::unique_ptr faissMethods(new FaissMethods()); + knn_jni::faiss_wrapper::IndexService IndexService(std::move(faissMethods)); + + try { + createIndexIteratively(&mockJNIUtil, &jniEnv, ids, vectors, dim, (jobject) (&javaFileIndexOutputMock), parametersMap, &IndexService); + // Make sure we close a file stream before reopening the created file. + javaFileIndexOutputMock.file_writer.close(); + } catch (const std::exception& e) { + ASSERT_STREQ("Failed to write index to disk", e.what()); + ASSERT_TRUE(throwIOException); + continue; + } + ASSERT_FALSE(throwIOException); + + // Make sure index can be loaded + std::unique_ptr index(test_util::FaissLoadIndex(indexPath)); + auto indexIDMap = dynamic_cast(index.get()); + + // Assert that Index is of type IndexHNSWSQ + ASSERT_NE(indexIDMap, nullptr); + ASSERT_NE(dynamic_cast(indexIDMap->index), nullptr); + + // Clean up + std::remove(indexPath.c_str()); + } // End for } TEST(FaissIsSharedIndexStateRequired, BasicAssertions) { @@ -917,12 +977,12 @@ TEST(FaissInitAndSetSharedIndexState, BasicAssertions) { test_util::FaissWriteIndex(&faissIndexWithIDMap, indexPath); // Setup jni - JNIEnv *jniEnv = nullptr; + NiceMock jniEnv; NiceMock mockJNIUtil; std::unique_ptr loadedIndexPointer( reinterpret_cast(knn_jni::faiss_wrapper::LoadIndex( - &mockJNIUtil, jniEnv, (jstring)&indexPath))); + &mockJNIUtil, &jniEnv, (jstring)&indexPath))); auto idMapIndex = dynamic_cast(loadedIndexPointer.get()); ASSERT_NE(idMapIndex, nullptr); @@ -973,7 +1033,7 @@ TEST(FaissRangeSearchQueryIndexTest, BasicAssertions) { test_util::FaissAddData(createdIndex.get(), ids, vectors); // Setup jni - JNIEnv *jniEnv = nullptr; + NiceMock jniEnv; NiceMock mockJNIUtil; int maxResultWindow = 20000; @@ -983,7 +1043,7 @@ TEST(FaissRangeSearchQueryIndexTest, BasicAssertions) { reinterpret_cast *> *>( knn_jni::faiss_wrapper::RangeSearch( - &mockJNIUtil, jniEnv, + &mockJNIUtil, &jniEnv, reinterpret_cast(&createdIndexWithData), reinterpret_cast(&query), rangeSearchRadius, methodParamsJ, maxResultWindow, nullptr))); @@ -1028,7 +1088,7 @@ TEST(FaissRangeSearchQueryIndexTest_WhenHitMaxWindowResult, BasicAssertions){ test_util::FaissAddData(createdIndex.get(), ids, vectors); // Setup jni - JNIEnv *jniEnv = nullptr; + NiceMock jniEnv; NiceMock mockJNIUtil; int maxResultWindow = 10; @@ -1038,7 +1098,7 @@ TEST(FaissRangeSearchQueryIndexTest_WhenHitMaxWindowResult, BasicAssertions){ reinterpret_cast *> *>( knn_jni::faiss_wrapper::RangeSearch( - &mockJNIUtil, jniEnv, + &mockJNIUtil, &jniEnv, reinterpret_cast(&createdIndexWithData), reinterpret_cast(&query), rangeSearchRadius, nullptr, maxResultWindow, nullptr))); @@ -1084,7 +1144,7 @@ TEST(FaissRangeSearchQueryIndexTestWithFilterTest, BasicAssertions) { test_util::FaissAddData(createdIndex.get(), ids, vectors); // Setup jni - JNIEnv *jniEnv = nullptr; + NiceMock jniEnv; NiceMock mockJNIUtil; int num_bits = test_util::bits2words(164); @@ -1104,7 +1164,7 @@ TEST(FaissRangeSearchQueryIndexTestWithFilterTest, BasicAssertions) { reinterpret_cast *> *>( knn_jni::faiss_wrapper::RangeSearchWithFilter( - &mockJNIUtil, jniEnv, + &mockJNIUtil, &jniEnv, reinterpret_cast(&createdIndexWithData), reinterpret_cast(&query), rangeSearchRadius, nullptr, maxResultWindow, reinterpret_cast(&bitmap), 0, nullptr))); @@ -1165,11 +1225,11 @@ TEST(FaissRangeSearchQueryIndexTestWithParentFilterTest, BasicAssertions) { test_util::FaissAddData(createdIndex.get(), ids, vectors); // Setup jni - JNIEnv *jniEnv = nullptr; + NiceMock jniEnv; NiceMock mockJNIUtil; EXPECT_CALL(mockJNIUtil, GetJavaIntArrayLength( - jniEnv, reinterpret_cast(&parentIds))) + &jniEnv, reinterpret_cast(&parentIds))) .WillRepeatedly(Return(parentIds.size())); int maxResultWindow = 10000; @@ -1179,7 +1239,7 @@ TEST(FaissRangeSearchQueryIndexTestWithParentFilterTest, BasicAssertions) { reinterpret_cast *> *>( knn_jni::faiss_wrapper::RangeSearchWithFilter( - &mockJNIUtil, jniEnv, + &mockJNIUtil, &jniEnv, reinterpret_cast(&createdIndexWithData), reinterpret_cast(&query), rangeSearchRadius, nullptr, maxResultWindow, nullptr, 0, reinterpret_cast(&parentIds)))); diff --git a/jni/tests/mocks/faiss_index_service_mock.h b/jni/tests/mocks/faiss_index_service_mock.h index 285e34053..6065b5bbe 100644 --- a/jni/tests/mocks/faiss_index_service_mock.h +++ b/jni/tests/mocks/faiss_index_service_mock.h @@ -21,7 +21,10 @@ typedef std::unordered_map StringToJObjectMap; class MockIndexService : public IndexService { public: - MockIndexService(std::unique_ptr faissMethods) : IndexService(std::move(faissMethods)) {}; + explicit MockIndexService(std::unique_ptr _faissMethods) + : IndexService(std::move(_faissMethods)) { + } + MOCK_METHOD( long, initIndex, @@ -36,6 +39,7 @@ class MockIndexService : public IndexService { StringToJObjectMap parameters ), (override)); + MOCK_METHOD( void, insertToIndex, @@ -48,14 +52,15 @@ class MockIndexService : public IndexService { long indexPtr ), (override)); + MOCK_METHOD( void, writeIndex, ( - std::string indexPath, + faiss::IOWriter* writer, long indexPtr ), (override)); }; -#endif // OPENSEARCH_KNN_FAISS_INDEX_SERVICE_MOCK_H \ No newline at end of file +#endif // OPENSEARCH_KNN_FAISS_INDEX_SERVICE_MOCK_H diff --git a/jni/tests/mocks/faiss_methods_mock.h b/jni/tests/mocks/faiss_methods_mock.h index 64a23b895..304269501 100644 --- a/jni/tests/mocks/faiss_methods_mock.h +++ b/jni/tests/mocks/faiss_methods_mock.h @@ -21,8 +21,8 @@ class MockFaissMethods : public knn_jni::faiss_wrapper::FaissMethods { MOCK_METHOD(faiss::IndexBinary*, indexBinaryFactory, (int d, const char* description), (override)); MOCK_METHOD(faiss::IndexIDMapTemplate*, indexIdMap, (faiss::Index* index), (override)); MOCK_METHOD(faiss::IndexIDMapTemplate*, indexBinaryIdMap, (faiss::IndexBinary* index), (override)); - MOCK_METHOD(void, writeIndex, (const faiss::Index* idx, const char* fname), (override)); - MOCK_METHOD(void, writeIndexBinary, (const faiss::IndexBinary* idx, const char* fname), (override)); + MOCK_METHOD(void, writeIndex, (const faiss::Index* idx, faiss::IOWriter* writer), (override)); + MOCK_METHOD(void, writeIndexBinary, (const faiss::IndexBinary* idx, faiss::IOWriter* writer), (override)); }; #endif // OPENSEARCH_KNN_FAISS_METHODS_MOCK_H \ No newline at end of file diff --git a/jni/tests/native_stream_support_util.h b/jni/tests/native_stream_support_util.h index e33f3beb4..ba926d690 100644 --- a/jni/tests/native_stream_support_util.h +++ b/jni/tests/native_stream_support_util.h @@ -12,13 +12,15 @@ #ifndef KNNPLUGIN_JNI_TESTS_NATIVE_STREAM_SUPPORT_UTIL_H_ #define KNNPLUGIN_JNI_TESTS_NATIVE_STREAM_SUPPORT_UTIL_H_ +#include +#include + #include "test_util.h" #include "gmock/gmock.h" #include "gtest/gtest.h" namespace test_util { - // Mocking IndexInputWithBuffer. struct JavaIndexInputMock { JavaIndexInputMock(std::string _readTargetBytes, int32_t _bufSize) @@ -37,7 +39,7 @@ struct JavaIndexInputMock { } int64_t remainingBytes() { - return readTargetBytes.size() - nextReadIdx; + return readTargetBytes.size() - nextReadIdx; } static std::string makeRandomBytes(int32_t bytesSize) { @@ -94,8 +96,70 @@ struct JavaFileIndexInputMock { std::ifstream &file_input; std::vector buffer; -}; // class JavaFileIndexInputMock +}; // struct JavaFileIndexInputMock + + + +struct JavaFileIndexOutputMock { + explicit JavaFileIndexOutputMock(const std::string &_file_path) + : file_writer(_file_path, std::ios::out | std::ios::binary), + buffer(64 * 1024) { + file_writer.exceptions(std::ios::failbit | std::ios::badbit); + } + + void writeBytes(int length) { + file_writer.write(buffer.data(), length); + } + + std::ofstream file_writer; + std::vector buffer; +}; // struct JavaFileIndexOutputMock +struct StreamIOError : public std::runtime_error { + StreamIOError() + : std::runtime_error(what()) { + } + + const char* what() const noexcept final { + return "Mocking IOError in Java side."; + } +}; // struct StreamIOError + +inline void setUpJavaFileOutputMocking(JavaFileIndexOutputMock &java_index_output, + MockJNIUtil &mockJni, + bool throwIOException) { + EXPECT_CALL(mockJni, GetPrimitiveArrayCritical(::testing::_, ::testing::_, ::testing::_)) + .WillRepeatedly([&java_index_output](JNIEnv *env, + jarray array, + jboolean *isCopy) { + return (jbyte *) java_index_output.buffer.data(); + }); + + EXPECT_CALL(mockJni, CallNonvirtualVoidMethodA(::testing::_, ::testing::_, ::testing::_, ::testing::_, ::testing::_)) + .WillRepeatedly([&java_index_output](JNIEnv *env, + jobject obj, + jclass clazz, + jmethodID methodID, + jvalue *args) { + const auto bytes_to_write = args[0].i; + java_index_output.writeBytes(bytes_to_write); + }); + + EXPECT_CALL(mockJni, GetJavaBytesArrayLength(::testing::_, ::testing::_)) + .WillRepeatedly([&java_index_output](JNIEnv *env, jbyteArray arrayJ) { + return java_index_output.buffer.size(); + }); + + if (throwIOException) { + EXPECT_CALL(mockJni, HasExceptionInStack(::testing::_, ::testing::_)) + .WillRepeatedly([](JNIEnv *env, const char* errorMsg){ + throw StreamIOError{}; + }); + } else { + EXPECT_CALL(mockJni, HasExceptionInStack(::testing::_, ::testing::_)) + .WillRepeatedly(::testing::Return()); + } +} } // namespace test_util diff --git a/jni/tests/nmslib_stream_support_test.cpp b/jni/tests/nmslib_stream_support_test.cpp index e0e7a2d08..7db94d5e3 100644 --- a/jni/tests/nmslib_stream_support_test.cpp +++ b/jni/tests/nmslib_stream_support_test.cpp @@ -17,11 +17,13 @@ #include "test_util.h" #include "native_stream_support_util.h" -using ::testing::_; +using ::test_util::JavaFileIndexInputMock; +using ::test_util::JavaFileIndexOutputMock; +using ::test_util::MockJNIUtil; +using ::test_util::StreamIOError; using ::testing::NiceMock; using ::testing::Return; -using ::test_util::MockJNIUtil; -using ::test_util::JavaFileIndexInputMock; +using ::testing::_; void setUpJavaFileInputMocking(JavaFileIndexInputMock &java_index_input, MockJNIUtil &mockJni) { // Set up mocking values + mocking behavior in a method. @@ -35,86 +37,100 @@ void setUpJavaFileInputMocking(JavaFileIndexInputMock &java_index_input, MockJNI }); EXPECT_CALL(mockJni, CallNonvirtualLongMethodA(_, _, _, _, _)) .WillRepeatedly([&java_index_input](JNIEnv *env, - jobject obj, - jclass clazz, - jmethodID methodID, - jvalue *args) { + jobject obj, + jclass clazz, + jmethodID methodID, + jvalue *args) { return java_index_input.remainingBytes(); }); - EXPECT_CALL(mockJni, GetPrimitiveArrayCritical(_, _, _)).WillRepeatedly([&java_index_input](JNIEnv *env, - jarray array, - jboolean *isCopy) { - return (jbyte *) java_index_input.buffer.data(); - }); + EXPECT_CALL(mockJni, GetPrimitiveArrayCritical(_, _, _)) + .WillRepeatedly([&java_index_input](JNIEnv *env, + jarray array, + jboolean *isCopy) { + return (jbyte *) java_index_input.buffer.data(); + }); EXPECT_CALL(mockJni, ReleasePrimitiveArrayCritical(_, _, _, _)).WillRepeatedly(Return()); } TEST(NmslibStreamLoadingTest, BasicAssertions) { - // Initialize nmslib - similarity::initLibrary(); - - // Define index data - int numIds = 100; - std::vector ids; - auto vectors = new std::vector(); - int dim = 2; - vectors->reserve(dim * numIds); - for (int i = 0; i < numIds; ++i) { - ids.push_back(i); - for (int j = 0; j < dim; ++j) { - vectors->push_back(test_util::RandomFloat(-500.0, 500.0)); - } - } - - std::string spaceType = knn_jni::L2; - std::string indexPath = test_util::RandomString( - 10, "/tmp/", ".nmslib"); - - std::unordered_map parametersMap; - int efConstruction = 512; - int m = 96; - - parametersMap[knn_jni::SPACE_TYPE] = (jobject) &spaceType; - parametersMap[knn_jni::EF_CONSTRUCTION] = (jobject) &efConstruction; - parametersMap[knn_jni::M] = (jobject) &m; - - // Set up jni - JNIEnv *jniEnv = nullptr; - NiceMock mockJNIUtil; - - EXPECT_CALL(mockJNIUtil, - GetJavaObjectArrayLength( - jniEnv, reinterpret_cast(vectors))) - .WillRepeatedly(Return(vectors->size())); - - EXPECT_CALL(mockJNIUtil, - GetJavaIntArrayLength(jniEnv, reinterpret_cast(&ids))) - .WillRepeatedly(Return(ids.size())); - - EXPECT_CALL(mockJNIUtil, - ConvertJavaMapToCppMap(jniEnv, reinterpret_cast(¶metersMap))) - .WillRepeatedly(Return(parametersMap)); - - // Create the index - knn_jni::nmslib_wrapper::CreateIndex( - &mockJNIUtil, jniEnv, reinterpret_cast(&ids), - (jlong) vectors, dim, (jstring) &indexPath, - (jobject) ¶metersMap); - - // Create Java index input mock. - std::ifstream file_input{indexPath, std::ios::binary}; - const int32_t buffer_size = 128; - JavaFileIndexInputMock java_file_index_input_mock{file_input, buffer_size}; - setUpJavaFileInputMocking(java_file_index_input_mock, mockJNIUtil); - - // Make sure index can be loaded - jlong index = knn_jni::nmslib_wrapper::LoadIndexWithStream( - &mockJNIUtil, jniEnv, - (jobject) (&java_file_index_input_mock), - (jobject) (¶metersMap)); - - knn_jni::nmslib_wrapper::Free(index); - - // Clean up - std::remove(indexPath.c_str()); + for (auto throwIOException : std::array {false, true}) { + // Initialize nmslib + similarity::initLibrary(); + + // Define index data + int numIds = 100; + std::vector ids; + auto vectors = new std::vector(); + int dim = 2; + vectors->reserve(dim * numIds); + for (int i = 0; i < numIds; ++i) { + ids.push_back(i); + for (int j = 0; j < dim; ++j) { + vectors->push_back(test_util::RandomFloat(-500.0, 500.0)); + } + } + + std::string spaceType = knn_jni::L2; + std::string indexPath = test_util::RandomString( + 10, "/tmp/", ".nmslib"); + + std::unordered_map parametersMap; + int efConstruction = 512; + int m = 96; + + parametersMap[knn_jni::SPACE_TYPE] = (jobject) &spaceType; + parametersMap[knn_jni::EF_CONSTRUCTION] = (jobject) &efConstruction; + parametersMap[knn_jni::M] = (jobject) &m; + + // Set up jni + NiceMock jniEnv; + + NiceMock mockJNIUtil; + JavaFileIndexOutputMock javaFileIndexOutputMock {indexPath}; + setUpJavaFileOutputMocking(javaFileIndexOutputMock, mockJNIUtil, throwIOException); + knn_jni::stream::NativeEngineIndexOutputMediator mediator {&mockJNIUtil, &jniEnv, (jobject) (&javaFileIndexOutputMock)}; + knn_jni::stream::NmslibOpenSearchIOWriter writer {&mediator}; + + EXPECT_CALL(mockJNIUtil, + GetJavaObjectArrayLength( + &jniEnv, reinterpret_cast(vectors))) + .WillRepeatedly(Return(vectors->size())); + + EXPECT_CALL(mockJNIUtil, + GetJavaIntArrayLength(&jniEnv, reinterpret_cast(&ids))) + .WillRepeatedly(Return(ids.size())); + + EXPECT_CALL(mockJNIUtil, + ConvertJavaMapToCppMap(&jniEnv, reinterpret_cast(¶metersMap))) + .WillRepeatedly(Return(parametersMap)); + + // Create the index + try { + knn_jni::nmslib_wrapper::CreateIndex( + &mockJNIUtil, &jniEnv, reinterpret_cast(&ids), + (jlong) vectors, dim, (jobject) (&javaFileIndexOutputMock), + (jobject) ¶metersMap); + javaFileIndexOutputMock.file_writer.close(); + } catch (const StreamIOError& e) { + continue; + } + + // Create Java index input mock. + std::ifstream file_input{indexPath, std::ios::binary}; + const int32_t buffer_size = 128; + JavaFileIndexInputMock java_file_index_input_mock{file_input, buffer_size}; + setUpJavaFileInputMocking(java_file_index_input_mock, mockJNIUtil); + + // Make sure index can be loaded + jlong index = knn_jni::nmslib_wrapper::LoadIndexWithStream( + &mockJNIUtil, &jniEnv, + (jobject) (&java_file_index_input_mock), + (jobject) (¶metersMap)); + + knn_jni::nmslib_wrapper::Free(index); + + // Clean up + file_input.close(); + std::remove(indexPath.c_str()); + } // End for } diff --git a/jni/tests/nmslib_wrapper_test.cpp b/jni/tests/nmslib_wrapper_test.cpp index 4e0c57044..b7690ee94 100644 --- a/jni/tests/nmslib_wrapper_test.cpp +++ b/jni/tests/nmslib_wrapper_test.cpp @@ -10,6 +10,7 @@ */ #include "nmslib_wrapper.h" +#include "nmslib_stream_support.h" #include @@ -17,7 +18,11 @@ #include "gtest/gtest.h" #include "jni_util.h" #include "test_util.h" +#include "native_stream_support_util.h" +using ::test_util::JavaFileIndexOutputMock; +using ::test_util::StreamIOError; +using ::test_util::setUpJavaFileOutputMocking; using ::testing::NiceMock; using ::testing::Return; @@ -33,117 +38,146 @@ TEST(NmslibIndexWrapperSearchTest, BasicAssertions) { } TEST(NmslibCreateIndexTest, BasicAssertions) { - // Initialize nmslib - similarity::initLibrary(); - - // Define index data - int numIds = 100; - std::vector ids; - auto *vectors = new std::vector(); - int dim = 2; - vectors->reserve(dim * numIds); - for (int64_t i = 0; i < numIds; ++i) { - ids.push_back(i); - for (int j = 0; j < dim; ++j) { + for (auto throwIOException : std::array {false, true}) { + // Initialize nmslib + similarity::initLibrary(); + + // Define index data + int numIds = 100; + std::vector ids; + auto *vectors = new std::vector(); + int dim = 2; + vectors->reserve(dim * numIds); + for (int64_t i = 0; i < numIds; ++i) { + ids.push_back(i); + for (int j = 0; j < dim; ++j) { vectors->push_back(test_util::RandomFloat(-500.0, 500.0)); + } } - } - std::string indexPath = test_util::RandomString(10, "tmp/", ".nmslib"); - std::string spaceType = knn_jni::L2; + std::string indexPath = test_util::RandomString(10, "tmp/", ".nmslib"); + std::string spaceType = knn_jni::L2; - std::unordered_map parametersMap; - int efConstruction = 512; - int m = 96; + std::unordered_map parametersMap; + int efConstruction = 512; + int m = 96; - parametersMap[knn_jni::SPACE_TYPE] = (jobject)&spaceType; - parametersMap[knn_jni::EF_CONSTRUCTION] = (jobject)&efConstruction; - parametersMap[knn_jni::M] = (jobject)&m; + parametersMap[knn_jni::SPACE_TYPE] = (jobject)&spaceType; + parametersMap[knn_jni::EF_CONSTRUCTION] = (jobject)&efConstruction; + parametersMap[knn_jni::M] = (jobject)&m; - // Set up jni - JNIEnv *jniEnv = nullptr; - NiceMock mockJNIUtil; + // Set up jni + NiceMock jniEnv; + NiceMock mockJNIUtil; + JavaFileIndexOutputMock javaFileIndexOutputMock {indexPath}; + setUpJavaFileOutputMocking(javaFileIndexOutputMock, mockJNIUtil, throwIOException); - EXPECT_CALL(mockJNIUtil, - GetJavaObjectArrayLength( - jniEnv, reinterpret_cast(&vectors))) + EXPECT_CALL(mockJNIUtil, + GetJavaObjectArrayLength( + &jniEnv, reinterpret_cast(&vectors))) .WillRepeatedly(Return(vectors->size())); - EXPECT_CALL(mockJNIUtil, - GetJavaIntArrayLength(jniEnv, reinterpret_cast(&ids))) + EXPECT_CALL(mockJNIUtil, + GetJavaIntArrayLength(&jniEnv, reinterpret_cast(&ids))) .WillRepeatedly(Return(ids.size())); - // Create the index - knn_jni::nmslib_wrapper::CreateIndex( - &mockJNIUtil, jniEnv, reinterpret_cast(&ids), - (jlong) vectors, dim, (jstring)&indexPath, - (jobject)¶metersMap); + // Create the index + try { + knn_jni::nmslib_wrapper::CreateIndex( + &mockJNIUtil, &jniEnv, reinterpret_cast(&ids), + (jlong) vectors, dim, (jobject)(&javaFileIndexOutputMock), + (jobject)¶metersMap); + } catch (const StreamIOError& e) { + ASSERT_TRUE(throwIOException); + continue; + } + ASSERT_FALSE(throwIOException); - // Make sure index can be loaded - std::unique_ptr> space( + // Make sure we close a file stream before reopening the created file. + javaFileIndexOutputMock.file_writer.close(); + + // Make sure index can be loaded + std::unique_ptr> space( similarity::SpaceFactoryRegistry::Instance().CreateSpace( - spaceType, similarity::AnyParams())); - std::vector params; - std::unique_ptr> loadedIndex( + spaceType, similarity::AnyParams())); + std::vector params; + std::unique_ptr> loadedIndex( test_util::NmslibLoadIndex(indexPath, space.get(), spaceType, params)); - // Clean up - std::remove(indexPath.c_str()); + // Clean up + std::remove(indexPath.c_str()); + } } TEST(NmslibLoadIndexTest, BasicAssertions) { - // Initialize nmslib - similarity::initLibrary(); - - // Define index data - int numIds = 100; - std::vector ids; - std::vector> vectors; - int dim = 2; - for (int i = 0; i < numIds; ++i) { - ids.push_back(i); - - std::vector vect; - vect.reserve(dim); - for (int j = 0; j < dim; ++j) { + for (auto throwIOException : std::array {false, true}) { + // Initialize nmslib + similarity::initLibrary(); + + // Define index data + int numIds = 100; + std::vector ids; + std::vector> vectors; + int dim = 2; + for (int i = 0; i < numIds; ++i) { + ids.push_back(i); + + std::vector vect; + vect.reserve(dim); + for (int j = 0; j < dim; ++j) { vect.push_back(test_util::RandomFloat(-500.0, 500.0)); + } + vectors.push_back(vect); } - vectors.push_back(vect); - } - std::string indexPath = test_util::RandomString(10, "tmp/", ".nmslib"); - std::string spaceType = knn_jni::L2; - std::unique_ptr> space( + std::string indexPath = test_util::RandomString(10, "tmp/", ".nmslib"); + std::string spaceType = knn_jni::L2; + std::unique_ptr> space( similarity::SpaceFactoryRegistry::Instance().CreateSpace( - spaceType, similarity::AnyParams())); + spaceType, similarity::AnyParams())); - std::vector indexParameters; + std::vector indexParameters; + + // Setup jni + NiceMock jniEnv; + NiceMock mockJNIUtil; + JavaFileIndexOutputMock javaFileIndexOutputMock {indexPath}; + setUpJavaFileOutputMocking(javaFileIndexOutputMock, mockJNIUtil, throwIOException); + knn_jni::stream::NativeEngineIndexOutputMediator mediator {&mockJNIUtil, &jniEnv, (jobject) (&javaFileIndexOutputMock)}; + knn_jni::stream::NmslibOpenSearchIOWriter writer {&mediator}; - // Create index and write to disk - std::unique_ptr> createdIndex( + // Create index and write to disk + std::unique_ptr> createdIndex( test_util::NmslibCreateIndex(ids.data(), vectors, space.get(), spaceType, indexParameters)); - test_util::NmslibWriteIndex(createdIndex.get(), indexPath); - // Setup jni - JNIEnv *jniEnv = nullptr; - NiceMock mockJNIUtil; + try { + test_util::NmslibWriteIndex(createdIndex.get(), writer); - // Load index - std::unordered_map parametersMap; - parametersMap[knn_jni::SPACE_TYPE] = (jobject)&spaceType; + // Make sure we close a file stream before reopening the created file. + javaFileIndexOutputMock.file_writer.close(); + } catch (const StreamIOError& e) { + ASSERT_TRUE(throwIOException); + continue; + } + ASSERT_FALSE(throwIOException); - std::unique_ptr loadedIndex( + // Load index + std::unordered_map parametersMap; + parametersMap[knn_jni::SPACE_TYPE] = (jobject)&spaceType; + + std::unique_ptr loadedIndex( reinterpret_cast( - knn_jni::nmslib_wrapper::LoadIndex(&mockJNIUtil, jniEnv, - (jstring)&indexPath, - (jobject)¶metersMap))); + knn_jni::nmslib_wrapper::LoadIndex(&mockJNIUtil, &jniEnv, + (jstring)&indexPath, + (jobject)¶metersMap))); - // Check that load succeeds - ASSERT_EQ(createdIndex->StrDesc(), loadedIndex->index->StrDesc()); + // Check that load succeeds + ASSERT_EQ(createdIndex->StrDesc(), loadedIndex->index->StrDesc()); - // Clean up - std::remove(indexPath.c_str()); + // Clean up + std::remove(indexPath.c_str()); + } } TEST(NmslibQueryIndexTest, BasicAssertions) { @@ -166,7 +200,6 @@ TEST(NmslibQueryIndexTest, BasicAssertions) { vectors.push_back(vect); } - std::string indexPath = test_util::RandomString(10, "tmp/", ".nmslib"); std::string spaceType = knn_jni::L2; std::unique_ptr> space( similarity::SpaceFactoryRegistry::Instance().CreateSpace( @@ -239,7 +272,6 @@ TEST(NmslibFreeTest, BasicAssertions) { vectors.push_back(vect); } - std::string indexPath = test_util::RandomString(10, "tmp/", ".nmslib"); std::string spaceType = knn_jni::L2; std::unique_ptr> space( similarity::SpaceFactoryRegistry::Instance().CreateSpace( diff --git a/jni/tests/test_util.cpp b/jni/tests/test_util.cpp index 47d1a7c8e..e337eaaf3 100644 --- a/jni/tests/test_util.cpp +++ b/jni/tests/test_util.cpp @@ -29,6 +29,7 @@ #include "methodfactory.h" #include "params.h" #include "space.h" +#include "method/hnsw.h" test_util::MockJNIUtil::MockJNIUtil() { // Set default for calls. If necessary, these can be overriden with @@ -374,8 +375,13 @@ similarity::Index *test_util::NmslibCreateIndex( } void test_util::NmslibWriteIndex(similarity::Index *index, - const std::string &indexPath) { - index->SaveIndex(indexPath); + knn_jni::stream::NmslibOpenSearchIOWriter& writer) { + if (auto hnswFloatIndex = dynamic_cast *>(index)) { + hnswFloatIndex->SaveIndexWithStream(writer); + writer.flush(); + } else { + throw std::runtime_error("We only support similarity::Hnsw in NMSLIB."); + } } similarity::Index *test_util::NmslibLoadIndex( diff --git a/jni/tests/test_util.h b/jni/tests/test_util.h index a6b39aa41..0262c8467 100644 --- a/jni/tests/test_util.h +++ b/jni/tests/test_util.h @@ -24,6 +24,7 @@ #include "faiss/MetaIndexes.h" #include "faiss/MetricType.h" #include "faiss/impl/io.h" +#include "nmslib_stream_support.h" #include "index.h" #include "init.h" #include "jni_util.h" @@ -84,7 +85,7 @@ namespace test_util { (JNIEnv * env, jobjectArray arrayJ, jsize index)); MOCK_METHOD(void, HasExceptionInStack, (JNIEnv * env)); MOCK_METHOD(void, HasExceptionInStack, - (JNIEnv * env, const std::string& message)); + (JNIEnv * env, const char* message)); MOCK_METHOD(jbyteArray, NewByteArray, (JNIEnv * env, jsize len)); MOCK_METHOD(jobject, NewObject, (JNIEnv * env, jclass clazz, jmethodID methodId, int id, @@ -115,6 +116,7 @@ namespace test_util { MOCK_METHOD(jlong, CallNonvirtualLongMethodA, (JNIEnv * env, jobject obj, jclass clazz, jmethodID methodID, jvalue* args)); MOCK_METHOD(void *, GetPrimitiveArrayCritical, (JNIEnv * env, jarray array, jboolean *isCopy)); MOCK_METHOD(void, ReleasePrimitiveArrayCritical, (JNIEnv * env, jarray array, void *carray, jint mode)); + MOCK_METHOD(void, CallNonvirtualVoidMethodA, (JNIEnv * env, jobject obj, jclass clazz, jmethodID methodID, jvalue* args)); }; // For our unit tests, we want to ensure that each test tests one function in @@ -160,7 +162,7 @@ namespace test_util { const std::vector& indexParameters); void NmslibWriteIndex(similarity::Index* index, - const std::string& indexPath); + knn_jni::stream::NmslibOpenSearchIOWriter& writer); similarity::Index* NmslibLoadIndex( const std::string& indexPath, similarity::Space* space, diff --git a/src/main/java/org/opensearch/knn/index/codec/nativeindex/DefaultIndexBuildStrategy.java b/src/main/java/org/opensearch/knn/index/codec/nativeindex/DefaultIndexBuildStrategy.java index 476c95b8d..23c3ba116 100644 --- a/src/main/java/org/opensearch/knn/index/codec/nativeindex/DefaultIndexBuildStrategy.java +++ b/src/main/java/org/opensearch/knn/index/codec/nativeindex/DefaultIndexBuildStrategy.java @@ -83,7 +83,7 @@ public void buildAndWriteIndex(final BuildIndexParams indexInfo) throws IOExcept intListToArray(transferredDocIds), vectorAddress, indexBuildSetup.getDimensions(), - indexInfo.getIndexPath(), + indexInfo.getIndexOutputWithBuffer(), (byte[]) params.get(KNNConstants.MODEL_BLOB_PARAMETER), params, indexInfo.getKnnEngine() @@ -96,7 +96,7 @@ public void buildAndWriteIndex(final BuildIndexParams indexInfo) throws IOExcept intListToArray(transferredDocIds), vectorAddress, indexBuildSetup.getDimensions(), - indexInfo.getIndexPath(), + indexInfo.getIndexOutputWithBuffer(), params, indexInfo.getKnnEngine() ); diff --git a/src/main/java/org/opensearch/knn/index/codec/nativeindex/MemOptimizedNativeIndexBuildStrategy.java b/src/main/java/org/opensearch/knn/index/codec/nativeindex/MemOptimizedNativeIndexBuildStrategy.java index b7e337081..81f5915a7 100644 --- a/src/main/java/org/opensearch/knn/index/codec/nativeindex/MemOptimizedNativeIndexBuildStrategy.java +++ b/src/main/java/org/opensearch/knn/index/codec/nativeindex/MemOptimizedNativeIndexBuildStrategy.java @@ -48,7 +48,6 @@ public static MemOptimizedNativeIndexBuildStrategy getInstance() { * flushed and used to build the index. The index is then written to the specified path using JNI calls.

    * * @param indexInfo The {@link BuildIndexParams} containing the parameters and configuration for building the index. - * @param knnVectorValues The {@link KNNVectorValues} representing the vectors to be indexed. * @throws IOException If an I/O error occurs during the process of building and writing the index. */ public void buildAndWriteIndex(final BuildIndexParams indexInfo) throws IOException { @@ -123,7 +122,7 @@ public void buildAndWriteIndex(final BuildIndexParams indexInfo) throws IOExcept // Write vector AccessController.doPrivileged((PrivilegedAction) () -> { - JNIService.writeIndex(indexInfo.getIndexPath(), indexMemoryAddress, engine, indexParameters); + JNIService.writeIndex(indexInfo.getIndexOutputWithBuffer(), indexMemoryAddress, engine, indexParameters); return null; }); diff --git a/src/main/java/org/opensearch/knn/index/codec/nativeindex/NativeIndexWriter.java b/src/main/java/org/opensearch/knn/index/codec/nativeindex/NativeIndexWriter.java index edc96c9e1..27a1ecfb6 100644 --- a/src/main/java/org/opensearch/knn/index/codec/nativeindex/NativeIndexWriter.java +++ b/src/main/java/org/opensearch/knn/index/codec/nativeindex/NativeIndexWriter.java @@ -7,11 +7,10 @@ import lombok.AllArgsConstructor; import lombok.extern.log4j.Log4j2; +import org.apache.lucene.codecs.CodecUtil; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.SegmentWriteState; -import org.apache.lucene.store.ChecksumIndexInput; -import org.apache.lucene.store.FSDirectory; -import org.apache.lucene.store.FilterDirectory; +import org.apache.lucene.store.IndexOutput; import org.opensearch.common.Nullable; import org.opensearch.common.xcontent.XContentHelper; import org.opensearch.core.common.bytes.BytesArray; @@ -27,6 +26,7 @@ import org.opensearch.knn.index.engine.KNNEngine; import org.opensearch.knn.index.engine.qframe.QuantizationConfig; import org.opensearch.knn.index.quantizationservice.QuantizationService; +import org.opensearch.knn.index.store.IndexOutputWithBuffer; import org.opensearch.knn.index.util.IndexUtil; import org.opensearch.knn.index.vectorvalues.KNNVectorValues; import org.opensearch.knn.indices.Model; @@ -35,16 +35,9 @@ import org.opensearch.knn.quantization.models.quantizationState.QuantizationState; import java.io.IOException; -import java.io.OutputStream; -import java.nio.ByteBuffer; -import java.nio.ByteOrder; -import java.nio.file.Files; -import java.nio.file.Paths; -import java.nio.file.StandardOpenOption; import java.util.HashMap; import java.util.Map; -import static org.apache.lucene.codecs.CodecUtil.FOOTER_MAGIC; import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS; import static org.opensearch.knn.common.FieldInfoExtractor.extractKNNEngine; import static org.opensearch.knn.common.FieldInfoExtractor.extractVectorDataType; @@ -133,7 +126,7 @@ public void mergeIndex(final KNNVectorValues knnVectorValues, int totalLiveDo private void buildAndWriteIndex(final KNNVectorValues knnVectorValues, int totalLiveDocs) throws IOException { if (totalLiveDocs == 0) { - log.debug("No live docs for field " + fieldInfo.name); + log.debug("No live docs for field {}", fieldInfo.name); return; } @@ -144,15 +137,18 @@ private void buildAndWriteIndex(final KNNVectorValues knnVectorValues, int to fieldInfo.name, knnEngine.getExtension() ); - final String indexPath = Paths.get( - ((FSDirectory) (FilterDirectory.unwrap(state.directory))).getDirectory().toString(), - engineFileName - ).toString(); - state.directory.createOutput(engineFileName, state.context).close(); - - final BuildIndexParams nativeIndexParams = indexParams(fieldInfo, indexPath, knnEngine, knnVectorValues, totalLiveDocs); - indexBuilder.buildAndWriteIndex(nativeIndexParams); - writeFooter(indexPath, engineFileName, state); + try (IndexOutput output = state.directory.createOutput(engineFileName, state.context)) { + final IndexOutputWithBuffer indexOutputWithBuffer = new IndexOutputWithBuffer(output); + final BuildIndexParams nativeIndexParams = indexParams( + fieldInfo, + indexOutputWithBuffer, + knnEngine, + knnVectorValues, + totalLiveDocs + ); + indexBuilder.buildAndWriteIndex(nativeIndexParams); + CodecUtil.writeFooter(output); + } } // The logic for building parameters need to be cleaned up. There are various cases handled here @@ -160,7 +156,7 @@ private void buildAndWriteIndex(final KNNVectorValues knnVectorValues, int to // TODO: Refactor this so its scalable. Possibly move it out of this class private BuildIndexParams indexParams( FieldInfo fieldInfo, - String indexPath, + IndexOutputWithBuffer indexOutputWithBuffer, KNNEngine knnEngine, KNNVectorValues vectorValues, int totalLiveDocs @@ -184,7 +180,7 @@ private BuildIndexParams indexParams( .parameters(parameters) .vectorDataType(vectorDataType) .knnEngine(knnEngine) - .indexPath(indexPath) + .indexOutputWithBuffer(indexOutputWithBuffer) .quantizationState(quantizationState) .vectorValues(vectorValues) .totalLiveDocs(totalLiveDocs) @@ -302,42 +298,6 @@ private void recordRefreshStats() { KNNGraphValue.REFRESH_TOTAL_OPERATIONS.increment(); } - private boolean isChecksumValid(long value) { - // Check pulled from - // https://github.com/apache/lucene/blob/branch_9_0/lucene/core/src/java/org/apache/lucene/codecs/CodecUtil.java#L644-L647 - return (value & CRC32_CHECKSUM_SANITY) != 0; - } - - private void writeFooter(String indexPath, String engineFileName, SegmentWriteState state) throws IOException { - // Opens the engine file that was created and appends a footer to it. The footer consists of - // 1. A Footer magic number (int - 4 bytes) - // 2. A checksum algorithm id (int - 4 bytes) - // 3. A checksum (long - bytes) - // The checksum is computed on all the bytes written to the file up to that point. - // Logic where footer is written in Lucene can be found here: - // https://github.com/apache/lucene/blob/branch_9_0/lucene/core/src/java/org/apache/lucene/codecs/CodecUtil.java#L390-L412 - OutputStream os = Files.newOutputStream(Paths.get(indexPath), StandardOpenOption.APPEND); - ByteBuffer byteBuffer = ByteBuffer.allocate(8).order(ByteOrder.BIG_ENDIAN); - byteBuffer.putInt(FOOTER_MAGIC); - byteBuffer.putInt(0); - os.write(byteBuffer.array()); - os.flush(); - - ChecksumIndexInput checksumIndexInput = state.directory.openChecksumInput(engineFileName, state.context); - checksumIndexInput.seek(checksumIndexInput.length()); - long value = checksumIndexInput.getChecksum(); - checksumIndexInput.close(); - - if (isChecksumValid(value)) { - throw new IllegalStateException("Illegal CRC-32 checksum: " + value + " (resource=" + os + ")"); - } - - // Write the CRC checksum to the end of the OutputStream and close the stream - byteBuffer.putLong(0, value); - os.write(byteBuffer.array()); - os.close(); - } - /** * Helper method to create the appropriate NativeIndexWriter based on the field info and quantization state. * diff --git a/src/main/java/org/opensearch/knn/index/codec/nativeindex/model/BuildIndexParams.java b/src/main/java/org/opensearch/knn/index/codec/nativeindex/model/BuildIndexParams.java index 88507b1fc..36e874c43 100644 --- a/src/main/java/org/opensearch/knn/index/codec/nativeindex/model/BuildIndexParams.java +++ b/src/main/java/org/opensearch/knn/index/codec/nativeindex/model/BuildIndexParams.java @@ -11,6 +11,7 @@ import org.opensearch.common.Nullable; import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.engine.KNNEngine; +import org.opensearch.knn.index.store.IndexOutputWithBuffer; import org.opensearch.knn.index.vectorvalues.KNNVectorValues; import org.opensearch.knn.quantization.models.quantizationState.QuantizationState; @@ -22,7 +23,7 @@ public class BuildIndexParams { String fieldName; KNNEngine knnEngine; - String indexPath; + IndexOutputWithBuffer indexOutputWithBuffer; VectorDataType vectorDataType; Map parameters; /** diff --git a/src/main/java/org/opensearch/knn/index/store/IndexOutputWithBuffer.java b/src/main/java/org/opensearch/knn/index/store/IndexOutputWithBuffer.java new file mode 100644 index 000000000..c1420238a --- /dev/null +++ b/src/main/java/org/opensearch/knn/index/store/IndexOutputWithBuffer.java @@ -0,0 +1,40 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.store; + +import org.apache.lucene.store.IndexOutput; + +import java.io.IOException; + +public class IndexOutputWithBuffer { + // Underlying `IndexOutput` obtained from Lucene's Directory. + private IndexOutput indexOutput; + // Write buffer. Native engine will copy bytes into this buffer. + // Allocating 64KB here since it show better performance in NMSLIB with the size. (We had slightly improvement in FAISS than having 4KB) + // NMSLIB writes an adjacent list size first, then followed by serializing the list. Since we usually have more adjacent lists, having + // 64KB to accumulate bytes as possible to reduce the times of calling `writeBytes`. + private byte[] buffer = new byte[64 * 1024]; + + public IndexOutputWithBuffer(IndexOutput indexOutput) { + this.indexOutput = indexOutput; + } + + // This method will be called in JNI layer which precisely knows + // the amount of bytes need to be written. + public void writeBytes(int length) { + try { + // Delegate Lucene `indexOuptut` to write bytes. + indexOutput.writeBytes(buffer, 0, length); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + @Override + public String toString() { + return "{indexOutput=" + indexOutput + ", len(buffer)=" + buffer.length + "}"; + } +} diff --git a/src/main/java/org/opensearch/knn/jni/FaissService.java b/src/main/java/org/opensearch/knn/jni/FaissService.java index c56726c66..dcc7b180d 100644 --- a/src/main/java/org/opensearch/knn/jni/FaissService.java +++ b/src/main/java/org/opensearch/knn/jni/FaissService.java @@ -12,9 +12,10 @@ package org.opensearch.knn.jni; import org.opensearch.knn.common.KNNConstants; -import org.opensearch.knn.index.query.KNNQueryResult; import org.opensearch.knn.index.engine.KNNEngine; +import org.opensearch.knn.index.query.KNNQueryResult; import org.opensearch.knn.index.store.IndexInputWithBuffer; +import org.opensearch.knn.index.store.IndexOutputWithBuffer; import java.security.AccessController; import java.security.PrivilegedAction; @@ -23,11 +24,11 @@ import static org.opensearch.knn.index.KNNSettings.isFaissAVX2Disabled; import static org.opensearch.knn.index.KNNSettings.isFaissAVX512Disabled; import static org.opensearch.knn.jni.PlatformUtils.isAVX2SupportedBySystem; -import static org.opensearch.knn.jni.PlatformUtils.isAVX512SupportedBySystem;; +import static org.opensearch.knn.jni.PlatformUtils.isAVX512SupportedBySystem; /** * Service to interact with faiss jni layer. Class dependencies should be minimal - * + *

    * In order to compile C++ header file, run: * javac -h jni/include src/main/java/org/opensearch/knn/jni/FaissService.java * src/main/java/org/opensearch/knn/index/query/KNNQueryResult.java @@ -129,9 +130,9 @@ class FaissService { * NOTE: This will always free the index. Do not call free after this. * * @param indexAddress address of native memory where index is stored - * @param indexPath path to save index file to + * @param output Index output wrapper having Lucene's IndexOutput to be used to flush bytes in native engines. */ - public static native void writeIndex(long indexAddress, String indexPath); + public static native void writeIndex(long indexAddress, IndexOutputWithBuffer output); /** * Writes a faiss index. @@ -139,9 +140,9 @@ class FaissService { * NOTE: This will always free the index. Do not call free after this. * * @param indexAddress address of native memory where index is stored - * @param indexPath path to save index file to + * @param output Index output wrapper having Lucene's IndexOutput to be used to flush bytes in native engines. */ - public static native void writeBinaryIndex(long indexAddress, String indexPath); + public static native void writeBinaryIndex(long indexAddress, IndexOutputWithBuffer output); /** * Writes a faiss index. @@ -149,9 +150,9 @@ class FaissService { * NOTE: This will always free the index. Do not call free after this. * * @param indexAddress address of native memory where index is stored - * @param indexPath path to save index file to + * @param output Index output wrapper having Lucene's IndexOutput to be used to flush bytes in native engines. */ - public static native void writeByteIndex(long indexAddress, String indexPath); + public static native void writeByteIndex(long indexAddress, IndexOutputWithBuffer output); /** * Create an index for the native library with a provided template index @@ -159,7 +160,7 @@ class FaissService { * @param ids array of ids mapping to the data passed in * @param vectorsAddress address of native memory where vectors are stored * @param dim dimension of the vector to be indexed - * @param indexPath path to save index file to + * @param output Index output wrapper having Lucene's IndexOutput to be used to flush bytes in native engines. * @param templateIndex empty template index * @param parameters additional build time parameters */ @@ -167,7 +168,7 @@ public static native void createIndexFromTemplate( int[] ids, long vectorsAddress, int dim, - String indexPath, + IndexOutputWithBuffer output, byte[] templateIndex, Map parameters ); @@ -178,7 +179,7 @@ public static native void createIndexFromTemplate( * @param ids array of ids mapping to the data passed in * @param vectorsAddress address of native memory where vectors are stored * @param dim dimension of the vector to be indexed - * @param indexPath path to save index file to + * @param output Index output wrapper having Lucene's IndexOutput to be used to flush bytes in native engines. * @param templateIndex empty template index * @param parameters additional build time parameters */ @@ -186,7 +187,7 @@ public static native void createBinaryIndexFromTemplate( int[] ids, long vectorsAddress, int dim, - String indexPath, + IndexOutputWithBuffer output, byte[] templateIndex, Map parameters ); @@ -197,7 +198,7 @@ public static native void createBinaryIndexFromTemplate( * @param ids array of ids mapping to the data passed in * @param vectorsAddress address of native memory where vectors are stored * @param dim dimension of the vector to be indexed - * @param indexPath path to save index file to + * @param output Index output wrapper having Lucene's IndexOutput to be used to flush bytes in native engines. * @param templateIndex empty template index * @param parameters additional build time parameters */ @@ -205,7 +206,7 @@ public static native void createByteIndexFromTemplate( int[] ids, long vectorsAddress, int dim, - String indexPath, + IndexOutputWithBuffer output, byte[] templateIndex, Map parameters ); diff --git a/src/main/java/org/opensearch/knn/jni/JNIService.java b/src/main/java/org/opensearch/knn/jni/JNIService.java index dd4dcef17..b490476eb 100644 --- a/src/main/java/org/opensearch/knn/jni/JNIService.java +++ b/src/main/java/org/opensearch/knn/jni/JNIService.java @@ -17,6 +17,7 @@ import org.opensearch.knn.index.engine.KNNEngine; import org.opensearch.knn.index.query.KNNQueryResult; import org.opensearch.knn.index.store.IndexInputWithBuffer; +import org.opensearch.knn.index.store.IndexOutputWithBuffer; import org.opensearch.knn.index.util.IndexUtil; import java.util.Locale; @@ -92,19 +93,19 @@ public static void insertToIndex( /** * Writes a faiss index to disk. * - * @param indexPath path to save index to + * @param output Index output wrapper having Lucene's IndexOutput to be used to flush bytes in native engines. * @param indexAddress address of native memory where index is stored * @param knnEngine knn engine * @param parameters parameters to build index */ - public static void writeIndex(String indexPath, long indexAddress, KNNEngine knnEngine, Map parameters) { + public static void writeIndex(IndexOutputWithBuffer output, long indexAddress, KNNEngine knnEngine, Map parameters) { if (KNNEngine.FAISS == knnEngine) { if (IndexUtil.isBinaryIndex(knnEngine, parameters)) { - FaissService.writeBinaryIndex(indexAddress, indexPath); + FaissService.writeBinaryIndex(indexAddress, output); } else if (IndexUtil.isByteIndex(parameters)) { - FaissService.writeByteIndex(indexAddress, indexPath); + FaissService.writeByteIndex(indexAddress, output); } else { - FaissService.writeIndex(indexAddress, indexPath); + FaissService.writeIndex(indexAddress, output); } return; } @@ -123,7 +124,7 @@ public static void writeIndex(String indexPath, long indexAddress, KNNEngine knn * @param ids array of ids mapping to the data passed in * @param vectorsAddress address of native memory where vectors are stored * @param dim dimension of the vector to be indexed - * @param indexPath path to save index file to + * @param output Index output wrapper having Lucene's IndexOutput to be used to flush bytes in native engines. * @param parameters parameters to build index * @param knnEngine engine to build index for */ @@ -131,12 +132,12 @@ public static void createIndex( int[] ids, long vectorsAddress, int dim, - String indexPath, + IndexOutputWithBuffer output, Map parameters, KNNEngine knnEngine ) { if (KNNEngine.NMSLIB == knnEngine) { - NmslibService.createIndex(ids, vectorsAddress, dim, indexPath, parameters); + NmslibService.createIndex(ids, vectorsAddress, dim, output, parameters); return; } @@ -151,7 +152,7 @@ public static void createIndex( * @param ids array of ids mapping to the data passed in * @param vectorsAddress address of native memory where vectors are stored * @param dim dimension of vectors to be indexed - * @param indexPath path to save index file to + * @param output Index output wrapper having Lucene's IndexOutput to be used to flush bytes in native engines. * @param templateIndex empty template index * @param parameters parameters to build index * @param knnEngine engine to build index for @@ -160,24 +161,23 @@ public static void createIndexFromTemplate( int[] ids, long vectorsAddress, int dim, - String indexPath, + IndexOutputWithBuffer output, byte[] templateIndex, Map parameters, KNNEngine knnEngine ) { if (KNNEngine.FAISS == knnEngine) { if (IndexUtil.isBinaryIndex(knnEngine, parameters)) { - FaissService.createBinaryIndexFromTemplate(ids, vectorsAddress, dim, indexPath, templateIndex, parameters); + FaissService.createBinaryIndexFromTemplate(ids, vectorsAddress, dim, output, templateIndex, parameters); return; } if (IndexUtil.isByteIndex(parameters)) { - FaissService.createByteIndexFromTemplate(ids, vectorsAddress, dim, indexPath, templateIndex, parameters); + FaissService.createByteIndexFromTemplate(ids, vectorsAddress, dim, output, templateIndex, parameters); return; } - FaissService.createIndexFromTemplate(ids, vectorsAddress, dim, indexPath, templateIndex, parameters); + FaissService.createIndexFromTemplate(ids, vectorsAddress, dim, output, templateIndex, parameters); return; - } throw new IllegalArgumentException( @@ -185,32 +185,6 @@ public static void createIndexFromTemplate( ); } - /** - * Load an index into memory - * - * @param indexPath path to index file - * @param parameters parameters to be used when loading index - * @param knnEngine engine to load index - * @return pointer to location in memory the index resides in - */ - public static long loadIndex(String indexPath, Map parameters, KNNEngine knnEngine) { - if (KNNEngine.NMSLIB == knnEngine) { - return NmslibService.loadIndex(indexPath, parameters); - } - - if (KNNEngine.FAISS == knnEngine) { - if (IndexUtil.isBinaryIndex(knnEngine, parameters)) { - return FaissService.loadBinaryIndex(indexPath); - } else { - return FaissService.loadIndex(indexPath); - } - } - - throw new IllegalArgumentException( - String.format(Locale.ROOT, "LoadIndex not supported for provided engine : %s", knnEngine.getName()) - ); - } - /** * Load an index via Lucene's IndexInput. * diff --git a/src/main/java/org/opensearch/knn/jni/NmslibService.java b/src/main/java/org/opensearch/knn/jni/NmslibService.java index feb850d30..16cc6bf52 100644 --- a/src/main/java/org/opensearch/knn/jni/NmslibService.java +++ b/src/main/java/org/opensearch/knn/jni/NmslibService.java @@ -12,9 +12,10 @@ package org.opensearch.knn.jni; import org.opensearch.knn.common.KNNConstants; -import org.opensearch.knn.index.query.KNNQueryResult; import org.opensearch.knn.index.engine.KNNEngine; +import org.opensearch.knn.index.query.KNNQueryResult; import org.opensearch.knn.index.store.IndexInputWithBuffer; +import org.opensearch.knn.index.store.IndexOutputWithBuffer; import java.security.AccessController; import java.security.PrivilegedAction; @@ -22,7 +23,7 @@ /** * Service to interact with nmslib jni layer. Class dependencies should be minimal - * + *

    * In order to compile C++ header file, run: * javac -h jni/include src/main/java/org/opensearch/knn/jni/NmslibService.java * src/main/java/org/opensearch/knn/index/KNNQueryResult.java @@ -48,19 +49,16 @@ class NmslibService { * @param ids array of ids mapping to the data passed in * @param vectorsAddress address of native memory where vectors are stored * @param dim dimension of the vector to be indexed - * @param indexPath path to save index file to + * @param output Index output wrapper having Lucene's IndexOutput to be used to flush bytes in native engines. * @param parameters parameters to build index */ - public static native void createIndex(int[] ids, long vectorsAddress, int dim, String indexPath, Map parameters); - - /** - * Load an index into memory - * - * @param indexPath path to index file - * @param parameters parameters to be used when loading index - * @return pointer to location in memory the index resides in - */ - public static native long loadIndex(String indexPath, Map parameters); + public static native void createIndex( + int[] ids, + long vectorsAddress, + int dim, + IndexOutputWithBuffer output, + Map parameters + ); /** * Load an index into memory through the provided read stream wrapping Lucene's IndexInput. diff --git a/src/test/java/org/opensearch/knn/common/RaisingIOExceptionIndexInput.java b/src/test/java/org/opensearch/knn/common/RaisingIOExceptionIndexInput.java new file mode 100644 index 000000000..8882f7d2f --- /dev/null +++ b/src/test/java/org/opensearch/knn/common/RaisingIOExceptionIndexInput.java @@ -0,0 +1,51 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.common; + +import org.apache.lucene.store.IndexInput; + +import java.io.IOException; + +public class RaisingIOExceptionIndexInput extends IndexInput { + public RaisingIOExceptionIndexInput() { + super(RaisingIOExceptionIndexInput.class.getSimpleName()); + } + + @Override + public void close() throws IOException { + throw new IOException("RaisingIOExceptionIndexInput::readBytes failed."); + } + + @Override + public long getFilePointer() { + throw new RuntimeException("RaisingIOExceptionIndexInput::readBytes failed."); + } + + @Override + public void seek(long l) throws IOException { + throw new IOException("RaisingIOExceptionIndexInput::readBytes failed."); + } + + @Override + public long length() { + return 0; + } + + @Override + public IndexInput slice(String s, long l, long l1) throws IOException { + throw new IOException("RaisingIOExceptionIndexInput::readBytes failed."); + } + + @Override + public byte readByte() throws IOException { + throw new IOException("RaisingIOExceptionIndexInput::readBytes failed."); + } + + @Override + public void readBytes(byte[] bytes, int i, int i1) throws IOException { + throw new IOException("RaisingIOExceptionIndexInput::readBytes failed."); + } +} diff --git a/src/test/java/org/opensearch/knn/common/RasingIOExceptionIndexOutput.java b/src/test/java/org/opensearch/knn/common/RasingIOExceptionIndexOutput.java new file mode 100644 index 000000000..7334bf201 --- /dev/null +++ b/src/test/java/org/opensearch/knn/common/RasingIOExceptionIndexOutput.java @@ -0,0 +1,41 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.common; + +import org.apache.lucene.store.IndexOutput; + +import java.io.IOException; + +public class RasingIOExceptionIndexOutput extends IndexOutput { + public RasingIOExceptionIndexOutput() { + super("Always throws IOException", RasingIOExceptionIndexOutput.class.getSimpleName()); + } + + @Override + public void close() throws IOException { + throw new IOException("RaiseIOExceptionIndexInput::close failed."); + } + + @Override + public long getFilePointer() { + throw new RuntimeException("RaiseIOExceptionIndexInput::getFilePointer failed."); + } + + @Override + public long getChecksum() throws IOException { + throw new IOException("RaiseIOExceptionIndexInput::getChecksum failed."); + } + + @Override + public void writeByte(byte b) throws IOException { + throw new IOException("RaiseIOExceptionIndexInput::writeByte failed."); + } + + @Override + public void writeBytes(byte[] bytes, int i, int i1) throws IOException { + throw new IOException("RaiseIOExceptionIndexInput::writeBytes failed."); + } +} diff --git a/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestCase.java b/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestCase.java index e6fcb643d..2036e14aa 100644 --- a/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestCase.java +++ b/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestCase.java @@ -154,64 +154,67 @@ protected ResourceWatcherService createDisabledResourceWatcherService() { public void testMultiFieldsKnnIndex(Codec codec) throws Exception { setUpMockClusterService(); - Directory dir = newFSDirectory(createTempDir()); - IndexWriterConfig iwc = newIndexWriterConfig(); - iwc.setMergeScheduler(new SerialMergeScheduler()); - iwc.setCodec(codec); - // Set merge policy to no merges so that we create a predictable number of segments. - iwc.setMergePolicy(NoMergePolicy.INSTANCE); - - /** - * Add doc with field "test_vector" - */ - float[] array = { 1.0f, 3.0f, 4.0f }; - VectorField vectorField = new VectorField("test_vector", array, sampleFieldType); - RandomIndexWriter writer = new RandomIndexWriter(random(), dir, iwc); - Document doc = new Document(); - doc.add(vectorField); - writer.addDocument(doc); - // ensuring the refresh happens, to create the segment and hnsw file - writer.flush(); - - /** - * Add doc with field "my_vector" - */ - float[] array1 = { 6.0f, 14.0f }; - VectorField vectorField1 = new VectorField("my_vector", array1, sampleFieldType); - Document doc1 = new Document(); - doc1.add(vectorField1); - writer.addDocument(doc1); - // ensuring the refresh happens, to create the segment and hnsw file - writer.flush(); - IndexReader reader = writer.getReader(); - writer.close(); - List hnswfiles = Arrays.stream(dir.listAll()).filter(x -> x.contains("hnsw")).collect(Collectors.toList()); + try (Directory dir = newFSDirectory(createTempDir())) { + IndexWriterConfig iwc = newIndexWriterConfig(); + iwc.setMergeScheduler(new SerialMergeScheduler()); + iwc.setCodec(codec); + // Set merge policy to no merges so that we create a predictable number of segments. + iwc.setMergePolicy(NoMergePolicy.INSTANCE); + + /** + * Add doc with field "test_vector" + */ + float[] array = { 1.0f, 3.0f, 4.0f }; + VectorField vectorField = new VectorField("test_vector", array, sampleFieldType); + RandomIndexWriter writer = new RandomIndexWriter(random(), dir, iwc); + Document doc = new Document(); + doc.add(vectorField); + writer.addDocument(doc); + // ensuring the refresh happens, to create the segment and hnsw file + writer.flush(); + + /** + * Add doc with field "my_vector" + */ + float[] array1 = { 6.0f, 14.0f }; + VectorField vectorField1 = new VectorField("my_vector", array1, sampleFieldType); + Document doc1 = new Document(); + doc1.add(vectorField1); + writer.addDocument(doc1); + // ensuring the refresh happens, to create the segment and hnsw file + writer.flush(); + IndexReader reader = writer.getReader(); + writer.close(); + List hnswfiles = Arrays.stream(dir.listAll()).filter(x -> x.contains("hnsw")).collect(Collectors.toList()); - // there should be 2 hnsw index files created. one for test_vector and one for my_vector - assertEquals(2, hnswfiles.size()); - assertEquals(hnswfiles.stream().filter(x -> x.contains("test_vector")).collect(Collectors.toList()).size(), 1); - assertEquals(hnswfiles.stream().filter(x -> x.contains("my_vector")).collect(Collectors.toList()).size(), 1); + // there should be 2 hnsw index files created. one for test_vector and one for my_vector + assertEquals(2, hnswfiles.size()); + assertEquals(hnswfiles.stream().filter(x -> x.contains("test_vector")).collect(Collectors.toList()).size(), 1); + assertEquals(hnswfiles.stream().filter(x -> x.contains("my_vector")).collect(Collectors.toList()).size(), 1); - // query to verify distance for each of the field - IndexSearcher searcher = new IndexSearcher(reader); - float score = searcher.search( - new KNNQuery("test_vector", new float[] { 1.0f, 0.0f, 0.0f }, 1, "dummy", (BitSetProducer) null), - 10 - ).scoreDocs[0].score; - float score1 = searcher.search( - new KNNQuery("my_vector", new float[] { 1.0f, 2.0f }, 1, "dummy", (BitSetProducer) null), - 10 - ).scoreDocs[0].score; - assertEquals(1.0f / (1 + 25), score, 0.01f); - assertEquals(1.0f / (1 + 169), score1, 0.01f); - - // query to determine the hits - assertEquals(1, searcher.count(new KNNQuery("test_vector", new float[] { 1.0f, 0.0f, 0.0f }, 1, "dummy", (BitSetProducer) null))); - assertEquals(1, searcher.count(new KNNQuery("my_vector", new float[] { 1.0f, 1.0f }, 1, "dummy", (BitSetProducer) null))); + // query to verify distance for each of the field + IndexSearcher searcher = new IndexSearcher(reader); + float score = searcher.search( + new KNNQuery("test_vector", new float[] { 1.0f, 0.0f, 0.0f }, 1, "dummy", (BitSetProducer) null), + 10 + ).scoreDocs[0].score; + float score1 = searcher.search( + new KNNQuery("my_vector", new float[] { 1.0f, 2.0f }, 1, "dummy", (BitSetProducer) null), + 10 + ).scoreDocs[0].score; + assertEquals(1.0f / (1 + 25), score, 0.01f); + assertEquals(1.0f / (1 + 169), score1, 0.01f); + + // query to determine the hits + assertEquals( + 1, + searcher.count(new KNNQuery("test_vector", new float[] { 1.0f, 0.0f, 0.0f }, 1, "dummy", (BitSetProducer) null)) + ); + assertEquals(1, searcher.count(new KNNQuery("my_vector", new float[] { 1.0f, 1.0f }, 1, "dummy", (BitSetProducer) null))); - reader.close(); - dir.close(); - NativeMemoryLoadStrategy.IndexLoadStrategy.getInstance().close(); + reader.close(); + NativeMemoryLoadStrategy.IndexLoadStrategy.getInstance().close(); + } } public void testBuildFromModelTemplate(Codec codec) throws IOException, ExecutionException, InterruptedException { diff --git a/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestUtil.java b/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestUtil.java index 2afd86a04..d6f22ca7f 100644 --- a/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestUtil.java +++ b/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestUtil.java @@ -20,9 +20,8 @@ import org.apache.lucene.search.Sort; import org.apache.lucene.store.ChecksumIndexInput; import org.apache.lucene.store.Directory; -import org.apache.lucene.store.FSDirectory; -import org.apache.lucene.store.FilterDirectory; import org.apache.lucene.store.IOContext; +import org.apache.lucene.store.IndexInput; import org.apache.lucene.util.StringHelper; import org.apache.lucene.util.Version; import java.util.Set; @@ -31,10 +30,10 @@ import org.opensearch.knn.index.query.KNNQueryResult; import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.engine.KNNEngine; +import org.opensearch.knn.index.store.IndexInputWithBuffer; import org.opensearch.knn.jni.JNIService; import java.io.IOException; -import java.nio.file.Paths; import java.util.Collections; import java.util.HashMap; import java.util.Map; @@ -206,14 +205,21 @@ public static void assertLoadableByEngine( SpaceType spaceType, int dimension ) { - String filePath = Paths.get(((FSDirectory) (FilterDirectory.unwrap(state.directory))).getDirectory().toString(), fileName) - .toString(); - long indexPtr = JNIService.loadIndex(filePath, Maps.newHashMap(ImmutableMap.of(SPACE_TYPE, spaceType.getValue())), knnEngine); - int k = 2; - float[] queryVector = new float[dimension]; - KNNQueryResult[] results = JNIService.queryIndex(indexPtr, queryVector, k, methodParameters, knnEngine, null, 0, null); - assertTrue(results.length > 0); - JNIService.free(indexPtr, knnEngine); + try (final IndexInput indexInput = state.directory.openInput(fileName, IOContext.LOAD)) { + final IndexInputWithBuffer indexInputWithBuffer = new IndexInputWithBuffer(indexInput); + long indexPtr = JNIService.loadIndex( + indexInputWithBuffer, + Maps.newHashMap(ImmutableMap.of(SPACE_TYPE, spaceType.getValue())), + knnEngine + ); + int k = 2; + float[] queryVector = new float[dimension]; + KNNQueryResult[] results = JNIService.queryIndex(indexPtr, queryVector, k, methodParameters, knnEngine, null, 0, null); + assertTrue(results.length > 0); + JNIService.free(indexPtr, knnEngine); + } catch (IOException e) { + throw new RuntimeException(e); + } } public static void assertBinaryIndexLoadableByEngine( @@ -224,27 +230,30 @@ public static void assertBinaryIndexLoadableByEngine( int dimension, VectorDataType vectorDataType ) { - String filePath = Paths.get(((FSDirectory) (FilterDirectory.unwrap(state.directory))).getDirectory().toString(), fileName) - .toString(); - long indexPtr = JNIService.loadIndex( - filePath, - Maps.newHashMap( - ImmutableMap.of( - SPACE_TYPE, - spaceType.getValue(), - INDEX_DESCRIPTION_PARAMETER, - "BHNSW32", - VECTOR_DATA_TYPE_FIELD, - vectorDataType.getValue() - ) - ), - knnEngine - ); - int k = 2; - byte[] queryVector = new byte[dimension]; - KNNQueryResult[] results = JNIService.queryBinaryIndex(indexPtr, queryVector, k, null, knnEngine, null, 0, null); - assertTrue(results.length > 0); - JNIService.free(indexPtr, knnEngine); + try (final IndexInput indexInput = state.directory.openInput(fileName, IOContext.LOAD)) { + final IndexInputWithBuffer indexInputWithBuffer = new IndexInputWithBuffer(indexInput); + long indexPtr = JNIService.loadIndex( + indexInputWithBuffer, + Maps.newHashMap( + ImmutableMap.of( + SPACE_TYPE, + spaceType.getValue(), + INDEX_DESCRIPTION_PARAMETER, + "BHNSW32", + VECTOR_DATA_TYPE_FIELD, + vectorDataType.getValue() + ) + ), + knnEngine + ); + int k = 2; + byte[] queryVector = new byte[dimension]; + KNNQueryResult[] results = JNIService.queryBinaryIndex(indexPtr, queryVector, k, null, knnEngine, null, 0, null); + assertTrue(results.length > 0); + JNIService.free(indexPtr, knnEngine); + } catch (IOException e) { + throw new RuntimeException(e); + } } @Builder(builderMethodName = "segmentInfoBuilder") diff --git a/src/test/java/org/opensearch/knn/index/codec/nativeindex/DefaultIndexBuildStrategyTests.java b/src/test/java/org/opensearch/knn/index/codec/nativeindex/DefaultIndexBuildStrategyTests.java index abb61ccd9..35c54f3b3 100644 --- a/src/test/java/org/opensearch/knn/index/codec/nativeindex/DefaultIndexBuildStrategyTests.java +++ b/src/test/java/org/opensearch/knn/index/codec/nativeindex/DefaultIndexBuildStrategyTests.java @@ -10,6 +10,7 @@ import org.junit.Before; import org.mockito.ArgumentCaptor; import org.mockito.MockedStatic; +import org.mockito.Mockito; import org.opensearch.core.common.unit.ByteSizeValue; import org.opensearch.knn.index.KNNSettings; import org.opensearch.knn.index.VectorDataType; @@ -18,6 +19,7 @@ import org.opensearch.knn.index.codec.transfer.OffHeapVectorTransferFactory; import org.opensearch.knn.index.engine.KNNEngine; import org.opensearch.knn.index.quantizationservice.QuantizationService; +import org.opensearch.knn.index.store.IndexOutputWithBuffer; import org.opensearch.knn.index.vectorvalues.KNNFloatVectorValues; import org.opensearch.knn.index.vectorvalues.KNNVectorValues; import org.opensearch.knn.index.vectorvalues.KNNVectorValuesFactory; @@ -66,8 +68,9 @@ public void testBuildAndWrite() { when(offHeapVectorTransfer.getVectorAddress()).thenReturn(200L); + IndexOutputWithBuffer indexOutputWithBuffer = Mockito.mock(IndexOutputWithBuffer.class); BuildIndexParams buildIndexParams = BuildIndexParams.builder() - .indexPath("indexPath") + .indexOutputWithBuffer(indexOutputWithBuffer) .knnEngine(KNNEngine.NMSLIB) .vectorDataType(VectorDataType.FLOAT) .parameters(Map.of("index", "param")) @@ -84,7 +87,7 @@ public void testBuildAndWrite() { eq(new int[] { 0, 1, 2 }), eq(200L), eq(knnVectorValues.dimension()), - eq("indexPath"), + eq(indexOutputWithBuffer), eq(Map.of("index", "param")), eq(KNNEngine.NMSLIB) ) @@ -159,8 +162,9 @@ public void testBuildAndWrite_withQuantization() { when(offHeapVectorTransfer.flush(false)).thenReturn(true); when(offHeapVectorTransfer.getVectorAddress()).thenReturn(200L); + IndexOutputWithBuffer indexOutputWithBuffer = Mockito.mock(IndexOutputWithBuffer.class); BuildIndexParams buildIndexParams = BuildIndexParams.builder() - .indexPath("indexPath") + .indexOutputWithBuffer(indexOutputWithBuffer) .knnEngine(KNNEngine.FAISS) .vectorDataType(VectorDataType.FLOAT) .parameters(Map.of("index", "param")) @@ -206,7 +210,7 @@ public void testBuildAndWrite_withQuantization() { ); mockedJNIService.verify( - () -> JNIService.writeIndex(eq("indexPath"), eq(100L), eq(KNNEngine.FAISS), eq(Map.of("index", "param"))) + () -> JNIService.writeIndex(eq(indexOutputWithBuffer), eq(100L), eq(KNNEngine.FAISS), eq(Map.of("index", "param"))) ); assertEquals(200L, vectorAddressCaptor.getValue().longValue()); assertEquals(vectorAddressCaptor.getValue().longValue(), vectorAddressCaptor.getAllValues().get(0).longValue()); @@ -244,8 +248,9 @@ public void testBuildAndWriteWithModel() { when(offHeapVectorTransfer.getVectorAddress()).thenReturn(200L); + IndexOutputWithBuffer indexOutputWithBuffer = Mockito.mock(IndexOutputWithBuffer.class); BuildIndexParams buildIndexParams = BuildIndexParams.builder() - .indexPath("indexPath") + .indexOutputWithBuffer(indexOutputWithBuffer) .knnEngine(KNNEngine.NMSLIB) .vectorDataType(VectorDataType.FLOAT) .parameters(Map.of("model_id", "id", "model_blob", modelBlob)) @@ -262,7 +267,7 @@ public void testBuildAndWriteWithModel() { eq(new int[] { 0, 1, 2 }), eq(200L), eq(2), - eq("indexPath"), + eq(indexOutputWithBuffer), eq(modelBlob), eq(Map.of("model_id", "id", "model_blob", modelBlob)), eq(KNNEngine.NMSLIB) diff --git a/src/test/java/org/opensearch/knn/index/codec/nativeindex/MemOptimizedNativeIndexBuildStrategyTests.java b/src/test/java/org/opensearch/knn/index/codec/nativeindex/MemOptimizedNativeIndexBuildStrategyTests.java index 77abe1cd2..08942fe7f 100644 --- a/src/test/java/org/opensearch/knn/index/codec/nativeindex/MemOptimizedNativeIndexBuildStrategyTests.java +++ b/src/test/java/org/opensearch/knn/index/codec/nativeindex/MemOptimizedNativeIndexBuildStrategyTests.java @@ -15,6 +15,7 @@ import org.opensearch.knn.index.codec.transfer.OffHeapVectorTransferFactory; import org.opensearch.knn.index.engine.KNNEngine; import org.opensearch.knn.index.quantizationservice.QuantizationService; +import org.opensearch.knn.index.store.IndexOutputWithBuffer; import org.opensearch.knn.index.vectorvalues.KNNVectorValues; import org.opensearch.knn.index.vectorvalues.KNNVectorValuesFactory; import org.opensearch.knn.index.vectorvalues.TestVectorValues; @@ -50,15 +51,15 @@ public void testBuildAndWrite() { MockedStatic mockedJNIService = Mockito.mockStatic(JNIService.class); MockedStatic mockedOffHeapVectorTransferFactory = Mockito.mockStatic( OffHeapVectorTransferFactory.class - ); + ) ) { - // Limits transfer to 2 vectors mockedJNIService.when(() -> JNIService.initIndex(3, 2, Map.of("index", "param"), KNNEngine.FAISS)).thenReturn(100L); OffHeapVectorTransfer offHeapVectorTransfer = mock(OffHeapVectorTransfer.class); mockedOffHeapVectorTransferFactory.when(() -> OffHeapVectorTransferFactory.getVectorTransfer(VectorDataType.FLOAT, 8, 3)) .thenReturn(offHeapVectorTransfer); + IndexOutputWithBuffer indexOutputWithBuffer = Mockito.mock(IndexOutputWithBuffer.class); when(offHeapVectorTransfer.getTransferLimit()).thenReturn(2); when(offHeapVectorTransfer.transfer(vectorTransferCapture.capture(), eq(false))).thenReturn(false) @@ -68,7 +69,7 @@ public void testBuildAndWrite() { when(offHeapVectorTransfer.getVectorAddress()).thenReturn(200L); BuildIndexParams buildIndexParams = BuildIndexParams.builder() - .indexPath("indexPath") + .indexOutputWithBuffer(indexOutputWithBuffer) .knnEngine(KNNEngine.FAISS) .vectorDataType(VectorDataType.FLOAT) .parameters(Map.of("index", "param")) @@ -113,7 +114,7 @@ public void testBuildAndWrite() { ); mockedJNIService.verify( - () -> JNIService.writeIndex(eq("indexPath"), eq(100L), eq(KNNEngine.FAISS), eq(Map.of("index", "param"))) + () -> JNIService.writeIndex(eq(indexOutputWithBuffer), eq(100L), eq(KNNEngine.FAISS), eq(Map.of("index", "param"))) ); assertEquals(200L, vectorAddressCaptor.getValue().longValue()); assertEquals(vectorAddressCaptor.getValue().longValue(), vectorAddressCaptor.getAllValues().get(0).longValue()); @@ -185,8 +186,9 @@ public void testBuildAndWrite_withQuantization() { when(offHeapVectorTransfer.flush(false)).thenReturn(true); when(offHeapVectorTransfer.getVectorAddress()).thenReturn(200L); + IndexOutputWithBuffer indexOutputWithBuffer = Mockito.mock(IndexOutputWithBuffer.class); BuildIndexParams buildIndexParams = BuildIndexParams.builder() - .indexPath("indexPath") + .indexOutputWithBuffer(indexOutputWithBuffer) .knnEngine(KNNEngine.FAISS) .vectorDataType(VectorDataType.FLOAT) .parameters(Map.of("index", "param")) @@ -232,7 +234,7 @@ public void testBuildAndWrite_withQuantization() { ); mockedJNIService.verify( - () -> JNIService.writeIndex(eq("indexPath"), eq(100L), eq(KNNEngine.FAISS), eq(Map.of("index", "param"))) + () -> JNIService.writeIndex(eq(indexOutputWithBuffer), eq(100L), eq(KNNEngine.FAISS), eq(Map.of("index", "param"))) ); assertEquals(200L, vectorAddressCaptor.getValue().longValue()); assertEquals(vectorAddressCaptor.getValue().longValue(), vectorAddressCaptor.getAllValues().get(0).longValue()); diff --git a/src/test/java/org/opensearch/knn/index/memory/NativeMemoryAllocationTests.java b/src/test/java/org/opensearch/knn/index/memory/NativeMemoryAllocationTests.java index db6231adf..be20150bc 100644 --- a/src/test/java/org/opensearch/knn/index/memory/NativeMemoryAllocationTests.java +++ b/src/test/java/org/opensearch/knn/index/memory/NativeMemoryAllocationTests.java @@ -13,6 +13,9 @@ import com.google.common.collect.ImmutableMap; import lombok.SneakyThrows; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IOContext; +import org.apache.lucene.store.IndexInput; import org.junit.Before; import org.mockito.Mock; import org.opensearch.common.settings.ClusterSettings; @@ -23,7 +26,7 @@ import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.engine.KNNEngine; -import org.opensearch.knn.index.util.IndexUtil; +import org.opensearch.knn.index.store.IndexInputWithBuffer; import org.opensearch.knn.jni.JNICommons; import org.opensearch.knn.jni.JNIService; @@ -62,110 +65,121 @@ public void setUp() throws Exception { KNNSettings.state().setClusterService(clusterService); } - public void testIndexAllocation_close() throws InterruptedException { + @SneakyThrows + public void testIndexAllocation_close() { // Create basic nmslib HNSW index - Path dir = createTempDir(); - KNNEngine knnEngine = KNNEngine.NMSLIB; - String indexName = "test1" + knnEngine.getExtension(); - String path = dir.resolve(indexName).toAbsolutePath().toString(); - int numVectors = 10; - int dimension = 10; - int[] ids = new int[numVectors]; - float[][] vectors = new float[numVectors][dimension]; - for (int i = 0; i < numVectors; i++) { - ids[i] = i; - Arrays.fill(vectors[i], 1f); - } - Map parameters = ImmutableMap.of(KNNConstants.SPACE_TYPE, SpaceType.DEFAULT.getValue()); - long vectorMemoryAddress = JNICommons.storeVectorData(0, vectors, numVectors * dimension); - TestUtils.createIndex(ids, vectorMemoryAddress, dimension, path, parameters, knnEngine); - - // Load index into memory - long memoryAddress = JNIService.loadIndex(path, parameters, knnEngine); - - ExecutorService executorService = Executors.newSingleThreadExecutor(); - NativeMemoryAllocation.IndexAllocation indexAllocation = new NativeMemoryAllocation.IndexAllocation( - executorService, - memoryAddress, - IndexUtil.getFileSizeInKB(path), - knnEngine, - path, - "test" - ); - - indexAllocation.close(); - - Thread.sleep(1000 * 2); - indexAllocation.writeLock(); - assertTrue(indexAllocation.isClosed()); - indexAllocation.writeUnlock(); + Path tempDirPath = createTempDir(); + try (Directory directory = newFSDirectory(tempDirPath)) { + KNNEngine knnEngine = KNNEngine.NMSLIB; + String indexFileName = "test1" + knnEngine.getExtension(); + int numVectors = 10; + int dimension = 10; + int[] ids = new int[numVectors]; + float[][] vectors = new float[numVectors][dimension]; + for (int i = 0; i < numVectors; i++) { + ids[i] = i; + Arrays.fill(vectors[i], 1f); + } + Map parameters = ImmutableMap.of(KNNConstants.SPACE_TYPE, SpaceType.DEFAULT.getValue()); + long vectorMemoryAddress = JNICommons.storeVectorData(0, vectors, numVectors * dimension); + TestUtils.createIndex(ids, vectorMemoryAddress, dimension, directory, indexFileName, parameters, knnEngine); + + // Load index into memory + final long memoryAddress; + try (IndexInput indexInput = directory.openInput(indexFileName, IOContext.DEFAULT)) { + final IndexInputWithBuffer indexInputWithBuffer = new IndexInputWithBuffer(indexInput); + memoryAddress = JNIService.loadIndex(indexInputWithBuffer, parameters, knnEngine); + } + + ExecutorService executorService = Executors.newSingleThreadExecutor(); + NativeMemoryAllocation.IndexAllocation indexAllocation = new NativeMemoryAllocation.IndexAllocation( + executorService, + memoryAddress, + (int) directory.fileLength(indexFileName) / 1024, + knnEngine, + indexFileName, + "test" + ); + + indexAllocation.close(); + + Thread.sleep(1000 * 2); + indexAllocation.writeLock(); + assertTrue(indexAllocation.isClosed()); + indexAllocation.writeUnlock(); - indexAllocation.close(); + indexAllocation.close(); - Thread.sleep(1000 * 2); - indexAllocation.writeLock(); - assertTrue(indexAllocation.isClosed()); - indexAllocation.writeUnlock(); + Thread.sleep(1000 * 2); + indexAllocation.writeLock(); + assertTrue(indexAllocation.isClosed()); + indexAllocation.writeUnlock(); - executorService.shutdown(); + executorService.shutdown(); + } } @SneakyThrows public void testClose_whenBinaryFiass_thenSuccess() { - Path dir = createTempDir(); + Path tempDirPath = createTempDir(); KNNEngine knnEngine = KNNEngine.FAISS; - String indexName = "test1" + knnEngine.getExtension(); - String path = dir.resolve(indexName).toAbsolutePath().toString(); - int numVectors = 10; - int dimension = 8; - int dataLength = dimension / 8; - int[] ids = new int[numVectors]; - byte[][] vectors = new byte[numVectors][dataLength]; - for (int i = 0; i < numVectors; i++) { - ids[i] = i; - vectors[i][0] = 1; - } - Map parameters = ImmutableMap.of( - KNNConstants.SPACE_TYPE, - SpaceType.HAMMING.getValue(), - KNNConstants.INDEX_DESCRIPTION_PARAMETER, - "BHNSW32", - KNNConstants.VECTOR_DATA_TYPE_FIELD, - VectorDataType.BINARY.getValue() - ); - long vectorMemoryAddress = JNICommons.storeBinaryVectorData(0, vectors, numVectors * dataLength); - TestUtils.createIndex(ids, vectorMemoryAddress, dimension, path, parameters, knnEngine); - - // Load index into memory - long memoryAddress = JNIService.loadIndex(path, parameters, knnEngine); - - ExecutorService executorService = Executors.newSingleThreadExecutor(); - NativeMemoryAllocation.IndexAllocation indexAllocation = new NativeMemoryAllocation.IndexAllocation( - executorService, - memoryAddress, - IndexUtil.getFileSizeInKB(path), - knnEngine, - path, - "test", - null, - true - ); - - indexAllocation.close(); - - Thread.sleep(1000 * 2); - indexAllocation.writeLock(); - assertTrue(indexAllocation.isClosed()); - indexAllocation.writeUnlock(); + String indexFileName = "test1" + knnEngine.getExtension(); + try (Directory directory = newFSDirectory(tempDirPath)) { + int numVectors = 10; + int dimension = 8; + int dataLength = dimension / 8; + int[] ids = new int[numVectors]; + byte[][] vectors = new byte[numVectors][dataLength]; + for (int i = 0; i < numVectors; i++) { + ids[i] = i; + vectors[i][0] = 1; + } + Map parameters = ImmutableMap.of( + KNNConstants.SPACE_TYPE, + SpaceType.HAMMING.getValue(), + KNNConstants.INDEX_DESCRIPTION_PARAMETER, + "BHNSW32", + KNNConstants.VECTOR_DATA_TYPE_FIELD, + VectorDataType.BINARY.getValue() + ); + long vectorMemoryAddress = JNICommons.storeBinaryVectorData(0, vectors, numVectors * dataLength); + TestUtils.createIndex(ids, vectorMemoryAddress, dimension, directory, indexFileName, parameters, knnEngine); + + // Load index into memory + final long memoryAddress; + try (IndexInput indexInput = directory.openInput(indexFileName, IOContext.DEFAULT)) { + final IndexInputWithBuffer indexInputWithBuffer = new IndexInputWithBuffer(indexInput); + memoryAddress = JNIService.loadIndex(indexInputWithBuffer, parameters, knnEngine); + } + + ExecutorService executorService = Executors.newSingleThreadExecutor(); + NativeMemoryAllocation.IndexAllocation indexAllocation = new NativeMemoryAllocation.IndexAllocation( + executorService, + memoryAddress, + (int) directory.fileLength(indexFileName) / 1024, + knnEngine, + indexFileName, + "test", + null, + true + ); + + indexAllocation.close(); + + Thread.sleep(1000 * 2); + indexAllocation.writeLock(); + assertTrue(indexAllocation.isClosed()); + indexAllocation.writeUnlock(); - indexAllocation.close(); + indexAllocation.close(); - Thread.sleep(1000 * 2); - indexAllocation.writeLock(); - assertTrue(indexAllocation.isClosed()); - indexAllocation.writeUnlock(); + Thread.sleep(1000 * 2); + indexAllocation.writeLock(); + assertTrue(indexAllocation.isClosed()); + indexAllocation.writeUnlock(); - executorService.shutdown(); + executorService.shutdown(); + } } public void testIndexAllocation_getMemoryAddress() { diff --git a/src/test/java/org/opensearch/knn/index/memory/NativeMemoryLoadStrategyTests.java b/src/test/java/org/opensearch/knn/index/memory/NativeMemoryLoadStrategyTests.java index 8236d0518..735974bd1 100644 --- a/src/test/java/org/opensearch/knn/index/memory/NativeMemoryLoadStrategyTests.java +++ b/src/test/java/org/opensearch/knn/index/memory/NativeMemoryLoadStrategyTests.java @@ -13,7 +13,6 @@ import com.google.common.collect.ImmutableMap; import org.apache.lucene.store.Directory; -import org.apache.lucene.store.MMapDirectory; import org.opensearch.core.action.ActionListener; import org.opensearch.action.search.SearchResponse; import org.opensearch.knn.KNNTestCase; @@ -44,98 +43,98 @@ public class NativeMemoryLoadStrategyTests extends KNNTestCase { public void testIndexLoadStrategy_load() throws IOException { // Create basic nmslib HNSW index - Path dir = createTempDir(); - Directory luceneDirectory = new MMapDirectory(dir); - KNNEngine knnEngine = KNNEngine.NMSLIB; - String indexName = "test1" + knnEngine.getExtension(); - String path = dir.resolve(indexName).toAbsolutePath().toString(); - int numVectors = 10; - int dimension = 10; - int[] ids = new int[numVectors]; - float[][] vectors = new float[numVectors][dimension]; - for (int i = 0; i < numVectors; i++) { - ids[i] = i; - Arrays.fill(vectors[i], 1f); + Path tempDirPath = createTempDir(); + try (Directory luceneDirectory = newFSDirectory(tempDirPath)) { + KNNEngine knnEngine = KNNEngine.NMSLIB; + String indexFileName = "test1" + knnEngine.getExtension(); + int numVectors = 10; + int dimension = 10; + int[] ids = new int[numVectors]; + float[][] vectors = new float[numVectors][dimension]; + for (int i = 0; i < numVectors; i++) { + ids[i] = i; + Arrays.fill(vectors[i], 1f); + } + Map parameters = ImmutableMap.of(KNNConstants.SPACE_TYPE, SpaceType.DEFAULT.getValue()); + long memoryAddress = JNICommons.storeVectorData(0, vectors, numVectors * dimension); + TestUtils.createIndex(ids, memoryAddress, dimension, luceneDirectory, indexFileName, parameters, knnEngine); + + // Setup mock resource manager + NativeMemoryEntryContext.IndexEntryContext indexEntryContext = new NativeMemoryEntryContext.IndexEntryContext( + luceneDirectory, + TestUtils.createFakeNativeMamoryCacheKey(indexFileName), + NativeMemoryLoadStrategy.IndexLoadStrategy.getInstance(), + parameters, + "test" + ); + + // Load + NativeMemoryAllocation.IndexAllocation indexAllocation = NativeMemoryLoadStrategy.IndexLoadStrategy.getInstance() + .load(indexEntryContext); + + // Confirm that the file was loaded by querying + float[] query = new float[dimension]; + Arrays.fill(query, numVectors + 1); + KNNQueryResult[] results = JNIService.queryIndex(indexAllocation.getMemoryAddress(), query, 2, null, knnEngine, null, 0, null); + assertTrue(results.length > 0); } - Map parameters = ImmutableMap.of(KNNConstants.SPACE_TYPE, SpaceType.DEFAULT.getValue()); - long memoryAddress = JNICommons.storeVectorData(0, vectors, numVectors * dimension); - TestUtils.createIndex(ids, memoryAddress, dimension, path, parameters, knnEngine); - - // Setup mock resource manager - NativeMemoryEntryContext.IndexEntryContext indexEntryContext = new NativeMemoryEntryContext.IndexEntryContext( - luceneDirectory, - TestUtils.createFakeNativeMamoryCacheKey(indexName), - NativeMemoryLoadStrategy.IndexLoadStrategy.getInstance(), - parameters, - "test" - ); - - // Load - NativeMemoryAllocation.IndexAllocation indexAllocation = NativeMemoryLoadStrategy.IndexLoadStrategy.getInstance() - .load(indexEntryContext); - - // Confirm that the file was loaded by querying - float[] query = new float[dimension]; - Arrays.fill(query, numVectors + 1); - KNNQueryResult[] results = JNIService.queryIndex(indexAllocation.getMemoryAddress(), query, 2, null, knnEngine, null, 0, null); - assertTrue(results.length > 0); } public void testLoad_whenFaissBinary_thenSuccess() throws IOException { - Path dir = createTempDir(); - Directory luceneDirectory = new MMapDirectory(dir); - KNNEngine knnEngine = KNNEngine.FAISS; - String indexName = "test1" + knnEngine.getExtension(); - String path = dir.resolve(indexName).toAbsolutePath().toString(); - int numVectors = 10; - int dimension = 8; - int dataLength = dimension / 8; - int[] ids = new int[numVectors]; - byte[][] vectors = new byte[numVectors][dataLength]; - for (int i = 0; i < numVectors; i++) { - ids[i] = i; - vectors[i][0] = 1; + Path tempDirPath = createTempDir(); + try (Directory luceneDirectory = newFSDirectory(tempDirPath)) { + KNNEngine knnEngine = KNNEngine.FAISS; + String indexFileName = "test1" + knnEngine.getExtension(); + int numVectors = 10; + int dimension = 8; + int dataLength = dimension / 8; + int[] ids = new int[numVectors]; + byte[][] vectors = new byte[numVectors][dataLength]; + for (int i = 0; i < numVectors; i++) { + ids[i] = i; + vectors[i][0] = 1; + } + Map parameters = ImmutableMap.of( + KNNConstants.SPACE_TYPE, + SpaceType.HAMMING.getValue(), + KNNConstants.INDEX_DESCRIPTION_PARAMETER, + "BHNSW32", + KNNConstants.VECTOR_DATA_TYPE_FIELD, + VectorDataType.BINARY.getValue() + ); + long memoryAddress = JNICommons.storeBinaryVectorData(0, vectors, numVectors); + TestUtils.createIndex(ids, memoryAddress, dimension, luceneDirectory, indexFileName, parameters, knnEngine); + + // Setup mock resource manager + NativeMemoryEntryContext.IndexEntryContext indexEntryContext = new NativeMemoryEntryContext.IndexEntryContext( + luceneDirectory, + TestUtils.createFakeNativeMamoryCacheKey(indexFileName), + NativeMemoryLoadStrategy.IndexLoadStrategy.getInstance(), + parameters, + "test" + ); + + // Load + NativeMemoryAllocation.IndexAllocation indexAllocation = NativeMemoryLoadStrategy.IndexLoadStrategy.getInstance() + .load(indexEntryContext); + + // Verify + assertTrue(indexAllocation.isBinaryIndex()); + + // Confirm that the file was loaded by querying + byte[] query = { 1 }; + KNNQueryResult[] results = JNIService.queryBinaryIndex( + indexAllocation.getMemoryAddress(), + query, + 2, + null, + knnEngine, + null, + 0, + null + ); + assertTrue(results.length > 0); } - Map parameters = ImmutableMap.of( - KNNConstants.SPACE_TYPE, - SpaceType.HAMMING.getValue(), - KNNConstants.INDEX_DESCRIPTION_PARAMETER, - "BHNSW32", - KNNConstants.VECTOR_DATA_TYPE_FIELD, - VectorDataType.BINARY.getValue() - ); - long memoryAddress = JNICommons.storeBinaryVectorData(0, vectors, numVectors); - TestUtils.createIndex(ids, memoryAddress, dimension, path, parameters, knnEngine); - - // Setup mock resource manager - NativeMemoryEntryContext.IndexEntryContext indexEntryContext = new NativeMemoryEntryContext.IndexEntryContext( - luceneDirectory, - TestUtils.createFakeNativeMamoryCacheKey(indexName), - NativeMemoryLoadStrategy.IndexLoadStrategy.getInstance(), - parameters, - "test" - ); - - // Load - NativeMemoryAllocation.IndexAllocation indexAllocation = NativeMemoryLoadStrategy.IndexLoadStrategy.getInstance() - .load(indexEntryContext); - - // Verify - assertTrue(indexAllocation.isBinaryIndex()); - - // Confirm that the file was loaded by querying - byte[] query = { 1 }; - KNNQueryResult[] results = JNIService.queryBinaryIndex( - indexAllocation.getMemoryAddress(), - query, - 2, - null, - knnEngine, - null, - 0, - null - ); - assertTrue(results.length > 0); } @SuppressWarnings("unchecked") diff --git a/src/test/java/org/opensearch/knn/jni/JNIServiceTests.java b/src/test/java/org/opensearch/knn/jni/JNIServiceTests.java index f6d118092..53a78b381 100644 --- a/src/test/java/org/opensearch/knn/jni/JNIServiceTests.java +++ b/src/test/java/org/opensearch/knn/jni/JNIServiceTests.java @@ -17,7 +17,7 @@ import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexInput; -import org.apache.lucene.store.MMapDirectory; +import org.apache.lucene.store.IndexOutput; import org.junit.BeforeClass; import org.opensearch.Version; import org.opensearch.common.xcontent.XContentFactory; @@ -25,6 +25,8 @@ import org.opensearch.knn.KNNTestCase; import org.opensearch.knn.TestUtils; import org.opensearch.knn.common.KNNConstants; +import org.opensearch.knn.common.RaisingIOExceptionIndexInput; +import org.opensearch.knn.common.RasingIOExceptionIndexOutput; import org.opensearch.knn.index.engine.KNNMethodConfigContext; import org.opensearch.knn.index.engine.KNNMethodContext; import org.opensearch.knn.index.VectorDataType; @@ -34,6 +36,7 @@ import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.engine.KNNEngine; import org.opensearch.knn.index.store.IndexInputWithBuffer; +import org.opensearch.knn.index.store.IndexOutputWithBuffer; import java.io.IOException; import java.net.URL; @@ -45,6 +48,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.UUID; import java.util.stream.Collectors; import static org.opensearch.knn.common.KNNConstants.ENCODER_PARAMETER_PQ_M; @@ -88,27 +92,40 @@ public static void setUpClass() throws IOException { testDataNested = new TestUtils.TestData(testIndexVectorsNested.getPath(), testQueries.getPath()); } + @SneakyThrows public void testCreateIndex_invalid_engineNotSupported() { + Path tempDirPath = createTempDir(); + try (Directory directory = newFSDirectory(tempDirPath)) { + expectThrows( + IllegalArgumentException.class, + () -> TestUtils.createIndex( + new int[] {}, + 0, + 0, + directory, + "DONT_CARE", + ImmutableMap.of(KNNConstants.SPACE_TYPE, SpaceType.L2.getValue()), + KNNEngine.LUCENE + ) + ); + } + } + + public void testCreateIndex_invalid_engineNull() { expectThrows( - IllegalArgumentException.class, + Exception.class, () -> TestUtils.createIndex( new int[] {}, 0, 0, - "test", + null, + "DONT_CARE", ImmutableMap.of(KNNConstants.SPACE_TYPE, SpaceType.L2.getValue()), - KNNEngine.LUCENE + null ) ); } - public void testCreateIndex_invalid_engineNull() { - expectThrows( - Exception.class, - () -> TestUtils.createIndex(new int[] {}, 0, 0, "test", ImmutableMap.of(KNNConstants.SPACE_TYPE, SpaceType.L2.getValue()), null) - ); - } - public void testCreateIndex_nmslib_invalid_noSpaceType() { expectThrows( Exception.class, @@ -116,7 +133,8 @@ public void testCreateIndex_nmslib_invalid_noSpaceType() { testData.indexData.docs, testData.loadDataToMemoryAddress(), testData.indexData.getDimension(), - "something", + null, + "DONT_CARE", Collections.emptyMap(), KNNEngine.NMSLIB ) @@ -124,98 +142,109 @@ public void testCreateIndex_nmslib_invalid_noSpaceType() { } public void testCreateIndex_nmslib_invalid_vectorDocIDMismatch() throws IOException { - int[] docIds = new int[] { 1, 2, 3 }; float[][] vectors1 = new float[][] { { 1, 2 }, { 3, 4 } }; long memoryAddress = JNICommons.storeVectorData(0, vectors1, vectors1.length * vectors1[0].length); - Path tmpFile1 = createTempFile(); - expectThrows( - Exception.class, - () -> TestUtils.createIndex( - docIds, - memoryAddress, - vectors1[0].length, - tmpFile1.toAbsolutePath().toString(), - ImmutableMap.of(KNNConstants.SPACE_TYPE, SpaceType.L2.getValue()), - KNNEngine.NMSLIB - ) - ); - - float[][] vectors2 = new float[][] { { 1, 2 }, { 3, 4 }, { 4, 5 }, { 6, 7 }, { 8, 9 } }; - long memoryAddress2 = JNICommons.storeVectorData(0, vectors2, vectors2.length * vectors2[0].length); + Path tempDirPath = createTempDir(); + String indexFileName1 = "test1.tmp"; + try (Directory directory = newFSDirectory(tempDirPath)) { + expectThrows( + Exception.class, + () -> TestUtils.createIndex( + docIds, + memoryAddress, + vectors1[0].length, + directory, + indexFileName1, + ImmutableMap.of(KNNConstants.SPACE_TYPE, SpaceType.L2.getValue()), + KNNEngine.NMSLIB + ) + ); - Path tmpFile2 = createTempFile(); - expectThrows( - Exception.class, - () -> TestUtils.createIndex( - docIds, - memoryAddress2, - vectors2[0].length, - tmpFile2.toAbsolutePath().toString(), - ImmutableMap.of(KNNConstants.SPACE_TYPE, SpaceType.L2.getValue()), - KNNEngine.NMSLIB - ) - ); + float[][] vectors2 = new float[][] { { 1, 2 }, { 3, 4 }, { 4, 5 }, { 6, 7 }, { 8, 9 } }; + long memoryAddress2 = JNICommons.storeVectorData(0, vectors2, vectors2.length * vectors2[0].length); + + String indexFileName2 = "test2.tmp"; + expectThrows( + Exception.class, + () -> TestUtils.createIndex( + docIds, + memoryAddress2, + vectors2[0].length, + directory, + indexFileName2, + ImmutableMap.of(KNNConstants.SPACE_TYPE, SpaceType.L2.getValue()), + KNNEngine.NMSLIB + ) + ); + } } public void testCreateIndex_nmslib_invalid_nullArgument() throws IOException { + Path tempDirPath = createTempDir(); + String indexFileName = "test.tmp"; + try (Directory directory = newFSDirectory(tempDirPath)) { + int[] docIds = new int[] {}; + float[][] vectors = new float[][] {}; + long memoryAddress = JNICommons.storeVectorData(0, vectors, vectors.length); + + expectThrows( + Exception.class, + () -> TestUtils.createIndex( + null, + memoryAddress, + 0, + directory, + indexFileName, + ImmutableMap.of(KNNConstants.SPACE_TYPE, SpaceType.L2.getValue()), + KNNEngine.NMSLIB + ) + ); - int[] docIds = new int[] {}; - float[][] vectors = new float[][] {}; - long memoryAddress = JNICommons.storeVectorData(0, vectors, vectors.length); - Path tmpFile = createTempFile(); - expectThrows( - Exception.class, - () -> TestUtils.createIndex( - null, - memoryAddress, - 0, - tmpFile.toAbsolutePath().toString(), - ImmutableMap.of(KNNConstants.SPACE_TYPE, SpaceType.L2.getValue()), - KNNEngine.NMSLIB - ) - ); - - expectThrows( - Exception.class, - () -> TestUtils.createIndex( - docIds, - 0, - 0, - tmpFile.toAbsolutePath().toString(), - ImmutableMap.of(KNNConstants.SPACE_TYPE, SpaceType.L2.getValue()), - KNNEngine.NMSLIB - ) - ); + expectThrows( + Exception.class, + () -> TestUtils.createIndex( + docIds, + 0, + 0, + directory, + indexFileName, + ImmutableMap.of(KNNConstants.SPACE_TYPE, SpaceType.L2.getValue()), + KNNEngine.NMSLIB + ) + ); - expectThrows( - Exception.class, - () -> TestUtils.createIndex( - docIds, - memoryAddress, - 0, - null, - ImmutableMap.of(KNNConstants.SPACE_TYPE, SpaceType.L2.getValue()), - KNNEngine.NMSLIB - ) - ); + expectThrows( + Exception.class, + () -> TestUtils.createIndex( + docIds, + memoryAddress, + 0, + directory, + null, + ImmutableMap.of(KNNConstants.SPACE_TYPE, SpaceType.L2.getValue()), + KNNEngine.NMSLIB + ) + ); - expectThrows( - Exception.class, - () -> TestUtils.createIndex(docIds, memoryAddress, 0, tmpFile.toAbsolutePath().toString(), null, KNNEngine.NMSLIB) - ); + expectThrows( + Exception.class, + () -> TestUtils.createIndex(docIds, memoryAddress, 0, directory, indexFileName, null, KNNEngine.NMSLIB) + ); - expectThrows( - Exception.class, - () -> TestUtils.createIndex( - docIds, - memoryAddress, - 0, - tmpFile.toAbsolutePath().toString(), - ImmutableMap.of(KNNConstants.SPACE_TYPE, SpaceType.L2.getValue()), - null - ) - ); + expectThrows( + Exception.class, + () -> TestUtils.createIndex( + docIds, + memoryAddress, + 0, + directory, + indexFileName, + ImmutableMap.of(KNNConstants.SPACE_TYPE, SpaceType.L2.getValue()), + null + ) + ); + } } public void testCreateIndex_nmslib_invalid_badSpace() throws IOException { @@ -223,18 +252,22 @@ public void testCreateIndex_nmslib_invalid_badSpace() throws IOException { int[] docIds = new int[] { 1 }; float[][] vectors = new float[][] { { 2, 3 } }; long memoryAddress = JNICommons.storeVectorData(0, vectors, vectors.length * vectors[0].length); - Path tmpFile = createTempFile(); - expectThrows( - Exception.class, - () -> TestUtils.createIndex( - docIds, - memoryAddress, - vectors[0].length, - tmpFile.toAbsolutePath().toString(), - ImmutableMap.of(KNNConstants.SPACE_TYPE, "invalid"), - KNNEngine.NMSLIB - ) - ); + Path tempDirPath = createTempDir(); + String indexFileName = "test.tmp"; + try (Directory directory = newFSDirectory(tempDirPath)) { + expectThrows( + Exception.class, + () -> TestUtils.createIndex( + docIds, + memoryAddress, + vectors[0].length, + directory, + indexFileName, + ImmutableMap.of(KNNConstants.SPACE_TYPE, "invalid"), + KNNEngine.NMSLIB + ) + ); + } } public void testCreateIndex_nmslib_invalid_badParameterType() throws IOException { @@ -249,74 +282,89 @@ public void testCreateIndex_nmslib_invalid_badParameterType() throws IOException KNNConstants.METHOD_PARAMETER_M, "12" ); - Path tmpFile = createTempFile(); - expectThrows( - Exception.class, - () -> TestUtils.createIndex( - docIds, - memoryAddress, - vectors[0].length, - tmpFile.toAbsolutePath().toString(), - ImmutableMap.of(KNNConstants.SPACE_TYPE, SpaceType.L2.getValue(), KNNConstants.PARAMETERS, parametersMap), - KNNEngine.NMSLIB - ) - ); + Path tempDirPath = createTempDir(); + String indexFileName = "test.tmp"; + try (Directory directory = newFSDirectory(tempDirPath)) { + expectThrows( + Exception.class, + () -> TestUtils.createIndex( + docIds, + memoryAddress, + vectors[0].length, + directory, + indexFileName, + ImmutableMap.of(KNNConstants.SPACE_TYPE, SpaceType.L2.getValue(), KNNConstants.PARAMETERS, parametersMap), + KNNEngine.NMSLIB + ) + ); + } } public void testCreateIndex_nmslib_valid() throws IOException { + Path tempDirPath = createTempDir(); + try (Directory directory = newFSDirectory(tempDirPath)) { + for (SpaceType spaceType : NmslibHNSWMethod.SUPPORTED_SPACES) { + if (SpaceType.UNDEFINED == spaceType) { + continue; + } - for (SpaceType spaceType : NmslibHNSWMethod.SUPPORTED_SPACES) { - if (SpaceType.UNDEFINED == spaceType) { - continue; - } - - Path tmpFile = createTempFile(); + final String indexFileName1 = "test" + UUID.randomUUID() + ".tmp"; - TestUtils.createIndex( - testData.indexData.docs, - testData.loadDataToMemoryAddress(), - testData.indexData.getDimension(), - tmpFile.toAbsolutePath().toString(), - ImmutableMap.of(KNNConstants.SPACE_TYPE, spaceType.getValue()), - KNNEngine.NMSLIB - ); - assertTrue(tmpFile.toFile().length() > 0); + TestUtils.createIndex( + testData.indexData.docs, + testData.loadDataToMemoryAddress(), + testData.indexData.getDimension(), + directory, + indexFileName1, + ImmutableMap.of(KNNConstants.SPACE_TYPE, spaceType.getValue()), + KNNEngine.NMSLIB + ); + assertTrue(directory.fileLength(indexFileName1) > 0); - tmpFile = createTempFile(); + final String indexFileName2 = "test" + UUID.randomUUID() + ".tmp"; - TestUtils.createIndex( - testData.indexData.docs, - testData.loadDataToMemoryAddress(), - testData.indexData.getDimension(), - tmpFile.toAbsolutePath().toString(), - ImmutableMap.of( - KNNConstants.SPACE_TYPE, - spaceType.getValue(), - KNNConstants.HNSW_ALGO_EF_CONSTRUCTION, - 14, - KNNConstants.METHOD_PARAMETER_M, - 12 - ), - KNNEngine.NMSLIB - ); - assertTrue(tmpFile.toFile().length() > 0); + TestUtils.createIndex( + testData.indexData.docs, + testData.loadDataToMemoryAddress(), + testData.indexData.getDimension(), + directory, + indexFileName2, + ImmutableMap.of( + KNNConstants.SPACE_TYPE, + spaceType.getValue(), + KNNConstants.HNSW_ALGO_EF_CONSTRUCTION, + 14, + KNNConstants.METHOD_PARAMETER_M, + 12 + ), + KNNEngine.NMSLIB + ); + assertTrue(directory.fileLength(indexFileName2) > 0); + } } } + @SneakyThrows public void testCreateIndex_faiss_invalid_noSpaceType() { int[] docIds = new int[] {}; - expectThrows( - Exception.class, - () -> TestUtils.createIndex( - docIds, - testData.loadDataToMemoryAddress(), - testData.indexData.getDimension(), - "something", - ImmutableMap.of(INDEX_DESCRIPTION_PARAMETER, faissMethod), - KNNEngine.FAISS - ) - ); + Path tempDirPath = createTempDir(); + String indexFileName = "test.tmp"; + try (Directory directory = newFSDirectory(tempDirPath)) { + expectThrows( + Exception.class, + () -> TestUtils.createIndex( + docIds, + testData.loadDataToMemoryAddress(), + testData.indexData.getDimension(), + directory, + indexFileName, + ImmutableMap.of(INDEX_DESCRIPTION_PARAMETER, faissMethod), + KNNEngine.FAISS + ) + ); + + } } public void testCreateIndex_faiss_invalid_vectorDocIDMismatch() throws IOException { @@ -324,251 +372,331 @@ public void testCreateIndex_faiss_invalid_vectorDocIDMismatch() throws IOExcepti int[] docIds = new int[] { 1, 2, 3 }; float[][] vectors1 = new float[][] { { 1, 2 }, { 3, 4 } }; long memoryAddress = JNICommons.storeVectorData(0, vectors1, vectors1.length * vectors1[0].length); - Path tmpFile1 = createTempFile(); - expectThrows( - Exception.class, - () -> TestUtils.createIndex( - docIds, - memoryAddress, - vectors1[0].length, - tmpFile1.toAbsolutePath().toString(), - ImmutableMap.of(INDEX_DESCRIPTION_PARAMETER, faissMethod, KNNConstants.SPACE_TYPE, SpaceType.L2.getValue()), - KNNEngine.FAISS - ) - ); + Path tempDirPath = createTempDir(); + String indexFileName1 = "test1" + UUID.randomUUID() + ".tmp"; + try (Directory directory = newFSDirectory(tempDirPath)) { + expectThrows( + Exception.class, + () -> TestUtils.createIndex( + docIds, + memoryAddress, + vectors1[0].length, + directory, + indexFileName1, + ImmutableMap.of(INDEX_DESCRIPTION_PARAMETER, faissMethod, KNNConstants.SPACE_TYPE, SpaceType.L2.getValue()), + KNNEngine.FAISS + ) + ); - float[][] vectors2 = new float[][] { { 1, 2 }, { 3, 4 }, { 4, 5 }, { 6, 7 }, { 8, 9 } }; - long memoryAddress2 = JNICommons.storeVectorData(0, vectors2, vectors2.length * vectors2[0].length); - Path tmpFile2 = createTempFile(); - expectThrows( - Exception.class, - () -> TestUtils.createIndex( - docIds, - memoryAddress, - vectors2[0].length, - tmpFile2.toAbsolutePath().toString(), - ImmutableMap.of(INDEX_DESCRIPTION_PARAMETER, faissMethod, KNNConstants.SPACE_TYPE, SpaceType.L2.getValue()), - KNNEngine.FAISS - ) - ); + float[][] vectors2 = new float[][] { { 1, 2 }, { 3, 4 }, { 4, 5 }, { 6, 7 }, { 8, 9 } }; + long memoryAddress2 = JNICommons.storeVectorData(0, vectors2, vectors2.length * vectors2[0].length); + String indexFileName2 = "test2" + UUID.randomUUID() + ".tmp"; + expectThrows( + Exception.class, + () -> TestUtils.createIndex( + docIds, + memoryAddress2, + vectors2[0].length, + directory, + indexFileName2, + ImmutableMap.of(INDEX_DESCRIPTION_PARAMETER, faissMethod, KNNConstants.SPACE_TYPE, SpaceType.L2.getValue()), + KNNEngine.FAISS + ) + ); + } } public void testCreateIndex_faiss_invalid_null() throws IOException { + Path tempDirPath = createTempDir(); int[] docIds = new int[] {}; float[][] vectors = new float[][] {}; long memoryAddress = JNICommons.storeVectorData(0, vectors, 0); + String indexFileName1 = "test1" + UUID.randomUUID() + ".tmp"; + try (Directory directory = newFSDirectory(tempDirPath)) { + expectThrows( + Exception.class, + () -> TestUtils.createIndex( + null, + memoryAddress, + 0, + directory, + indexFileName1, + ImmutableMap.of(INDEX_DESCRIPTION_PARAMETER, faissMethod, KNNConstants.SPACE_TYPE, SpaceType.L2.getValue()), + KNNEngine.FAISS + ) + ); - Path tmpFile = createTempFile(); - expectThrows( - Exception.class, - () -> TestUtils.createIndex( - null, - memoryAddress, - 0, - tmpFile.toAbsolutePath().toString(), - ImmutableMap.of(INDEX_DESCRIPTION_PARAMETER, faissMethod, KNNConstants.SPACE_TYPE, SpaceType.L2.getValue()), - KNNEngine.FAISS - ) - ); - - expectThrows( - Exception.class, - () -> TestUtils.createIndex( - docIds, - 0, - 0, - tmpFile.toAbsolutePath().toString(), - ImmutableMap.of(INDEX_DESCRIPTION_PARAMETER, faissMethod, KNNConstants.SPACE_TYPE, SpaceType.L2.getValue()), - KNNEngine.FAISS - ) - ); + expectThrows( + Exception.class, + () -> TestUtils.createIndex( + docIds, + 0, + 0, + directory, + indexFileName1, + ImmutableMap.of(INDEX_DESCRIPTION_PARAMETER, faissMethod, KNNConstants.SPACE_TYPE, SpaceType.L2.getValue()), + KNNEngine.FAISS + ) + ); - expectThrows( - Exception.class, - () -> TestUtils.createIndex( - docIds, - testData.loadDataToMemoryAddress(), - testData.indexData.getDimension(), - null, - ImmutableMap.of(INDEX_DESCRIPTION_PARAMETER, faissMethod, KNNConstants.SPACE_TYPE, SpaceType.L2.getValue()), - KNNEngine.FAISS - ) - ); + expectThrows( + Exception.class, + () -> TestUtils.createIndex( + docIds, + testData.loadDataToMemoryAddress(), + testData.indexData.getDimension(), + directory, + null, + ImmutableMap.of(INDEX_DESCRIPTION_PARAMETER, faissMethod, KNNConstants.SPACE_TYPE, SpaceType.L2.getValue()), + KNNEngine.FAISS + ) + ); - expectThrows( - Exception.class, - () -> TestUtils.createIndex( - docIds, - testData.loadDataToMemoryAddress(), - testData.indexData.getDimension(), - tmpFile.toAbsolutePath().toString(), - null, - KNNEngine.FAISS - ) - ); + expectThrows( + Exception.class, + () -> TestUtils.createIndex( + docIds, + testData.loadDataToMemoryAddress(), + testData.indexData.getDimension(), + directory, + indexFileName1, + null, + KNNEngine.FAISS + ) + ); - expectThrows( - Exception.class, - () -> TestUtils.createIndex( - docIds, - testData.loadDataToMemoryAddress(), - testData.indexData.getDimension(), - tmpFile.toAbsolutePath().toString(), - ImmutableMap.of(INDEX_DESCRIPTION_PARAMETER, faissMethod, KNNConstants.SPACE_TYPE, SpaceType.L2.getValue()), - null - ) - ); + expectThrows( + Exception.class, + () -> TestUtils.createIndex( + docIds, + testData.loadDataToMemoryAddress(), + testData.indexData.getDimension(), + directory, + indexFileName1, + ImmutableMap.of(INDEX_DESCRIPTION_PARAMETER, faissMethod, KNNConstants.SPACE_TYPE, SpaceType.L2.getValue()), + null + ) + ); + } } public void testCreateIndex_faiss_invalid_invalidSpace() throws IOException { - - int[] docIds = new int[] { 1 }; - float[][] vectors = new float[][] { { 2, 3 } }; - long memoryAddress = JNICommons.storeVectorData(0, vectors, (long) vectors.length * vectors[0].length); - - Path tmpFile = createTempFile(); - expectThrows( - Exception.class, - () -> TestUtils.createIndex( - docIds, - memoryAddress, - vectors[0].length, - tmpFile.toAbsolutePath().toString(), - ImmutableMap.of(INDEX_DESCRIPTION_PARAMETER, faissMethod, KNNConstants.SPACE_TYPE, "invalid"), - KNNEngine.FAISS - ) - ); + Path tempDirPath = createTempDir(); + try (Directory directory = newFSDirectory(tempDirPath)) { + int[] docIds = new int[] { 1 }; + float[][] vectors = new float[][] { { 2, 3 } }; + long memoryAddress = JNICommons.storeVectorData(0, vectors, (long) vectors.length * vectors[0].length); + String indexFileName1 = "test1" + UUID.randomUUID() + ".tmp"; + + expectThrows( + Exception.class, + () -> TestUtils.createIndex( + docIds, + memoryAddress, + vectors[0].length, + directory, + indexFileName1, + ImmutableMap.of(INDEX_DESCRIPTION_PARAMETER, faissMethod, KNNConstants.SPACE_TYPE, "invalid"), + KNNEngine.FAISS + ) + ); + } } public void testCreateIndex_faiss_invalid_noIndexDescription() throws IOException { - - int[] docIds = new int[] { 1, 2 }; - float[][] vectors = new float[][] { { 2, 3 }, { 2, 3 } }; - long memoryAddress = JNICommons.storeVectorData(0, vectors, (long) vectors.length * vectors[0].length); - - Path tmpFile = createTempFile(); - expectThrows( - Exception.class, - () -> TestUtils.createIndex( - docIds, - memoryAddress, - vectors[0].length, - tmpFile.toAbsolutePath().toString(), - ImmutableMap.of(KNNConstants.SPACE_TYPE, SpaceType.L2.getValue()), - KNNEngine.FAISS - ) - ); + Path tempDirPath = createTempDir(); + try (Directory directory = newFSDirectory(tempDirPath)) { + int[] docIds = new int[] { 1, 2 }; + float[][] vectors = new float[][] { { 2, 3 }, { 2, 3 } }; + long memoryAddress = JNICommons.storeVectorData(0, vectors, (long) vectors.length * vectors[0].length); + + String indexFileName1 = "test1" + UUID.randomUUID() + ".tmp"; + expectThrows( + Exception.class, + () -> TestUtils.createIndex( + docIds, + memoryAddress, + vectors[0].length, + directory, + indexFileName1, + ImmutableMap.of(KNNConstants.SPACE_TYPE, SpaceType.L2.getValue()), + KNNEngine.FAISS + ) + ); + } } public void testCreateIndex_faiss_invalid_invalidIndexDescription() throws IOException { - int[] docIds = new int[] { 1, 2 }; - float[][] vectors = new float[][] { { 2, 3 }, { 2, 3 } }; - long memoryAddress = JNICommons.storeVectorData(0, vectors, (long) vectors.length * vectors[0].length); - Path tmpFile = createTempFile(); - expectThrows( - Exception.class, - () -> TestUtils.createIndex( - docIds, - memoryAddress, - vectors[0].length, - tmpFile.toAbsolutePath().toString(), - ImmutableMap.of(INDEX_DESCRIPTION_PARAMETER, "invalid", KNNConstants.SPACE_TYPE, SpaceType.L2.getValue()), - KNNEngine.FAISS - ) - ); + Path tempDirPath = createTempDir(); + try (Directory directory = newFSDirectory(tempDirPath)) { + int[] docIds = new int[] { 1, 2 }; + float[][] vectors = new float[][] { { 2, 3 }, { 2, 3 } }; + long memoryAddress = JNICommons.storeVectorData(0, vectors, (long) vectors.length * vectors[0].length); + + String indexFileName1 = "test1" + UUID.randomUUID() + ".tmp"; + expectThrows( + Exception.class, + () -> TestUtils.createIndex( + docIds, + memoryAddress, + vectors[0].length, + directory, + indexFileName1, + ImmutableMap.of(INDEX_DESCRIPTION_PARAMETER, "invalid", KNNConstants.SPACE_TYPE, SpaceType.L2.getValue()), + KNNEngine.FAISS + ) + ); + } } @SneakyThrows public void testCreateIndex_faiss_sqfp16_invalidIndexDescription() { - - int[] docIds = new int[] { 1, 2 }; - float[][] vectors = new float[][] { { 2, 3 }, { 3, 4 } }; - long memoryAddress = JNICommons.storeVectorData(0, vectors, (long) vectors.length * vectors[0].length); - - String sqfp16InvalidIndexDescription = "HNSW16,SQfp1655"; - - Path tmpFile = createTempFile(); - expectThrows( - Exception.class, - () -> TestUtils.createIndex( + Path tempDirPath = createTempDir(); + try (Directory directory = newFSDirectory(tempDirPath)) { + int[] docIds = new int[] { 1, 2 }; + float[][] vectors = new float[][] { { 2, 3 }, { 3, 4 } }; + long memoryAddress = JNICommons.storeVectorData(0, vectors, (long) vectors.length * vectors[0].length); + + String sqfp16InvalidIndexDescription = "HNSW16,SQfp1655"; + + String indexFileName1 = "test1" + UUID.randomUUID() + ".tmp"; + expectThrows( + Exception.class, + () -> TestUtils.createIndex( + docIds, + memoryAddress, + vectors[0].length, + directory, + indexFileName1, + ImmutableMap.of( + INDEX_DESCRIPTION_PARAMETER, + sqfp16InvalidIndexDescription, + KNNConstants.SPACE_TYPE, + SpaceType.L2.getValue() + ), + KNNEngine.FAISS + ) + ); + } + } + + @SneakyThrows + public void testLoadIndex_faiss_sqfp16_valid() { + Path tempDirPath = createTempDir(); + try (Directory directory = newFSDirectory(tempDirPath)) { + int[] docIds = new int[] { 1, 2 }; + float[][] vectors = new float[][] { { 2, 3 }, { 3, 4 } }; + String sqfp16IndexDescription = "HNSW16,SQfp16"; + long memoryAddress = JNICommons.storeVectorData(0, vectors, (long) vectors.length * vectors[0].length); + String indexFileName1 = "test1" + UUID.randomUUID() + ".tmp"; + TestUtils.createIndex( docIds, memoryAddress, vectors[0].length, - tmpFile.toAbsolutePath().toString(), - ImmutableMap.of( - INDEX_DESCRIPTION_PARAMETER, - sqfp16InvalidIndexDescription, - KNNConstants.SPACE_TYPE, - SpaceType.L2.getValue() - ), + directory, + indexFileName1, + ImmutableMap.of(INDEX_DESCRIPTION_PARAMETER, sqfp16IndexDescription, KNNConstants.SPACE_TYPE, SpaceType.L2.getValue()), KNNEngine.FAISS - ) - ); + ); + assertTrue(directory.fileLength(indexFileName1) > 0); + + try (IndexInput indexInput = directory.openInput(indexFileName1, IOContext.LOAD)) { + final IndexInputWithBuffer indexInputWithBuffer = new IndexInputWithBuffer(indexInput); + long pointer = JNIService.loadIndex(indexInputWithBuffer, Collections.emptyMap(), KNNEngine.FAISS); + assertNotEquals(0, pointer); + } catch (Throwable e) { + fail(e.getMessage()); + } + } } @SneakyThrows - public void testLoadIndex_faiss_sqfp16_valid() { + public void testLoadIndex_when_io_exception_was_raised() { + Path tempDirPath = createTempDir(); + try (Directory directory = newFSDirectory(tempDirPath)) { + int[] docIds = new int[] { 1, 2 }; + float[][] vectors = new float[][] { { 2, 3 }, { 3, 4 } }; + String sqfp16IndexDescription = "HNSW16,SQfp16"; + long memoryAddress = JNICommons.storeVectorData(0, vectors, (long) vectors.length * vectors[0].length); + String indexFileName1 = "test1" + UUID.randomUUID() + ".tmp"; + TestUtils.createIndex( + docIds, + memoryAddress, + vectors[0].length, + directory, + indexFileName1, + ImmutableMap.of(INDEX_DESCRIPTION_PARAMETER, sqfp16IndexDescription, KNNConstants.SPACE_TYPE, SpaceType.L2.getValue()), + KNNEngine.FAISS + ); + assertTrue(directory.fileLength(indexFileName1) > 0); - int[] docIds = new int[] { 1, 2 }; - float[][] vectors = new float[][] { { 2, 3 }, { 3, 4 } }; - String sqfp16IndexDescription = "HNSW16,SQfp16"; - long memoryAddress = JNICommons.storeVectorData(0, vectors, (long) vectors.length * vectors[0].length); - Path tmpFile = createTempFile(); - TestUtils.createIndex( - docIds, - memoryAddress, - vectors[0].length, - tmpFile.toAbsolutePath().toString(), - ImmutableMap.of(INDEX_DESCRIPTION_PARAMETER, sqfp16IndexDescription, KNNConstants.SPACE_TYPE, SpaceType.L2.getValue()), - KNNEngine.FAISS - ); - assertTrue(tmpFile.toFile().length() > 0); + final IndexInput raiseIOExceptionIndexInput = new RaisingIOExceptionIndexInput(); + final IndexInputWithBuffer indexInputWithBuffer = new IndexInputWithBuffer(raiseIOExceptionIndexInput); - long pointer = JNIService.loadIndex(tmpFile.toAbsolutePath().toString(), Collections.emptyMap(), KNNEngine.FAISS); - assertNotEquals(0, pointer); + try { + JNIService.loadIndex(indexInputWithBuffer, Collections.emptyMap(), KNNEngine.FAISS); + fail("Exception thrown is expected."); + } catch (Throwable e) { + assertTrue(e.getMessage().contains("Reading bytes via IndexInput has failed.")); + } + } } @SneakyThrows public void testQueryIndex_faiss_sqfp16_valid() { + Path tempDirPath = createTempDir(); + try (Directory directory = newFSDirectory(tempDirPath)) { + String sqfp16IndexDescription = "HNSW16,SQfp16"; + int k = 10; + Map methodParameters = Map.of("ef_search", 12); + float[][] truncatedVectors = truncateToFp16Range(testData.indexData.vectors); + long memoryAddress = JNICommons.storeVectorData( + 0, + truncatedVectors, + (long) truncatedVectors.length * truncatedVectors[0].length + ); + String indexFileName1 = "test1" + UUID.randomUUID() + ".tmp"; + TestUtils.createIndex( + testData.indexData.docs, + memoryAddress, + testData.indexData.getDimension(), + directory, + indexFileName1, + ImmutableMap.of(INDEX_DESCRIPTION_PARAMETER, sqfp16IndexDescription, KNNConstants.SPACE_TYPE, SpaceType.L2.getValue()), + KNNEngine.FAISS + ); + assertTrue(directory.fileLength(indexFileName1) > 0); - String sqfp16IndexDescription = "HNSW16,SQfp16"; - int k = 10; - Map methodParameters = Map.of("ef_search", 12); - float[][] truncatedVectors = truncateToFp16Range(testData.indexData.vectors); - long memoryAddress = JNICommons.storeVectorData(0, truncatedVectors, (long) truncatedVectors.length * truncatedVectors[0].length); - Path tmpFile = createTempFile(); - TestUtils.createIndex( - testData.indexData.docs, - memoryAddress, - testData.indexData.getDimension(), - tmpFile.toAbsolutePath().toString(), - ImmutableMap.of(INDEX_DESCRIPTION_PARAMETER, sqfp16IndexDescription, KNNConstants.SPACE_TYPE, SpaceType.L2.getValue()), - KNNEngine.FAISS - ); - assertTrue(tmpFile.toFile().length() > 0); - - long pointer = JNIService.loadIndex(tmpFile.toAbsolutePath().toString(), Collections.emptyMap(), KNNEngine.FAISS); - assertNotEquals(0, pointer); + final long pointer; + try (IndexInput indexInput = directory.openInput(indexFileName1, IOContext.LOAD)) { + final IndexInputWithBuffer indexInputWithBuffer = new IndexInputWithBuffer(indexInput); + pointer = JNIService.loadIndex(indexInputWithBuffer, Collections.emptyMap(), KNNEngine.FAISS); + assertNotEquals(0, pointer); + } catch (Throwable e) { + fail(e.getMessage()); + throw e; + } - for (float[] query : testData.queries) { - KNNQueryResult[] results = JNIService.queryIndex(pointer, query, k, methodParameters, KNNEngine.FAISS, null, 0, null); - assertEquals(k, results.length); - } + for (float[] query : testData.queries) { + KNNQueryResult[] results = JNIService.queryIndex(pointer, query, k, methodParameters, KNNEngine.FAISS, null, 0, null); + assertEquals(k, results.length); + } - // Filter will result in no ids - for (float[] query : testData.queries) { - KNNQueryResult[] results = JNIService.queryIndex( - pointer, - query, - k, - methodParameters, - KNNEngine.FAISS, - new long[] { 0 }, - 0, - null - ); - assertEquals(0, results.length); + // Filter will result in no ids + for (float[] query : testData.queries) { + KNNQueryResult[] results = JNIService.queryIndex( + pointer, + query, + k, + methodParameters, + KNNEngine.FAISS, + new long[] { 0 }, + 0, + null + ); + assertEquals(0, results.length); + } } } @@ -625,93 +753,103 @@ public void testTrain_whenConfigurationIsIVFSQFP16_thenSucceed() { } public void testCreateIndex_faiss_invalid_invalidParameterType() throws IOException { - - int[] docIds = new int[] {}; - float[][] vectors = new float[][] {}; - - Path tmpFile = createTempFile(); - expectThrows( - Exception.class, - () -> TestUtils.createIndex( - docIds, - testData.loadDataToMemoryAddress(), - testData.indexData.getDimension(), - tmpFile.toAbsolutePath().toString(), - ImmutableMap.of( - INDEX_DESCRIPTION_PARAMETER, - "IVF13", - KNNConstants.SPACE_TYPE, - SpaceType.L2.getValue(), - KNNConstants.PARAMETERS, - ImmutableMap.of(KNNConstants.METHOD_PARAMETER_NPROBES, "14") - ), - KNNEngine.FAISS - ) - ); - + Path tempDirPath = createTempDir(); + try (Directory directory = newFSDirectory(tempDirPath)) { + int[] docIds = new int[] {}; + float[][] vectors = new float[][] {}; + + String indexFileName1 = "test1" + UUID.randomUUID() + ".tmp"; + expectThrows( + Exception.class, + () -> TestUtils.createIndex( + docIds, + testData.loadDataToMemoryAddress(), + testData.indexData.getDimension(), + directory, + indexFileName1, + ImmutableMap.of( + INDEX_DESCRIPTION_PARAMETER, + "IVF13", + KNNConstants.SPACE_TYPE, + SpaceType.L2.getValue(), + KNNConstants.PARAMETERS, + ImmutableMap.of(KNNConstants.METHOD_PARAMETER_NPROBES, "14") + ), + KNNEngine.FAISS + ) + ); + } } public void testCreateIndex_faiss_valid() throws IOException { List methods = ImmutableList.of(faissMethod); List spaces = ImmutableList.of(SpaceType.L2, SpaceType.INNER_PRODUCT); - for (String method : methods) { - for (SpaceType spaceType : spaces) { - Path tmpFile1 = createTempFile(); - TestUtils.createIndex( - testData.indexData.docs, - testData.loadDataToMemoryAddress(), - testData.indexData.getDimension(), - tmpFile1.toAbsolutePath().toString(), - ImmutableMap.of(INDEX_DESCRIPTION_PARAMETER, method, KNNConstants.SPACE_TYPE, spaceType.getValue()), - KNNEngine.FAISS - ); - assertTrue(tmpFile1.toFile().length() > 0); + Path tempDirPath = createTempDir(); + try (Directory directory = newFSDirectory(tempDirPath)) { + for (String method : methods) { + for (SpaceType spaceType : spaces) { + String indexFileName1 = "test1" + UUID.randomUUID() + ".tmp"; + TestUtils.createIndex( + testData.indexData.docs, + testData.loadDataToMemoryAddress(), + testData.indexData.getDimension(), + directory, + indexFileName1, + ImmutableMap.of(INDEX_DESCRIPTION_PARAMETER, method, KNNConstants.SPACE_TYPE, spaceType.getValue()), + KNNEngine.FAISS + ); + assertTrue(directory.fileLength(indexFileName1) > 0); + } } } } @SneakyThrows public void testCreateIndex_binary_faiss_valid() { - Path tmpFile1 = createTempFile(); - long memoryAddr = testData.loadBinaryDataToMemoryAddress(); - TestUtils.createIndex( - testData.indexData.docs, - memoryAddr, - testData.indexData.getDimension(), - tmpFile1.toAbsolutePath().toString(), - ImmutableMap.of( - INDEX_DESCRIPTION_PARAMETER, - faissBinaryMethod, - KNNConstants.SPACE_TYPE, - SpaceType.HAMMING.getValue(), - KNNConstants.VECTOR_DATA_TYPE_FIELD, - VectorDataType.BINARY.getValue() - ), - KNNEngine.FAISS - ); - assertTrue(tmpFile1.toFile().length() > 0); + Path tempDirPath = createTempDir(); + String indexFileName1 = "test1" + UUID.randomUUID() + ".tmp"; + try (Directory directory = newFSDirectory(tempDirPath)) { + long memoryAddr = testData.loadBinaryDataToMemoryAddress(); + TestUtils.createIndex( + testData.indexData.docs, + memoryAddr, + testData.indexData.getDimension(), + directory, + indexFileName1, + ImmutableMap.of( + INDEX_DESCRIPTION_PARAMETER, + faissBinaryMethod, + KNNConstants.SPACE_TYPE, + SpaceType.HAMMING.getValue(), + KNNConstants.VECTOR_DATA_TYPE_FIELD, + VectorDataType.BINARY.getValue() + ), + KNNEngine.FAISS + ); + assertTrue(directory.fileLength(indexFileName1) > 0); + } } public void testLoadIndex_invalidEngine() { - expectThrows(IllegalArgumentException.class, () -> JNIService.loadIndex("test", Collections.emptyMap(), KNNEngine.LUCENE)); + expectThrows(IllegalArgumentException.class, () -> JNIService.loadIndex(null, Collections.emptyMap(), KNNEngine.LUCENE)); } public void testLoadIndex_nmslib_invalid_badSpaceType() { expectThrows( Exception.class, - () -> JNIService.loadIndex("test", ImmutableMap.of(KNNConstants.SPACE_TYPE, "invalid"), KNNEngine.NMSLIB) + () -> JNIService.loadIndex(null, ImmutableMap.of(KNNConstants.SPACE_TYPE, "invalid"), KNNEngine.NMSLIB) ); } public void testLoadIndex_nmslib_invalid_noSpaceType() { - expectThrows(Exception.class, () -> JNIService.loadIndex("test", Collections.emptyMap(), KNNEngine.NMSLIB)); + expectThrows(Exception.class, () -> JNIService.loadIndex(null, Collections.emptyMap(), KNNEngine.NMSLIB)); } public void testLoadIndex_nmslib_invalid_fileDoesNotExist() { expectThrows( Exception.class, - () -> JNIService.loadIndex("invalid", ImmutableMap.of(KNNConstants.SPACE_TYPE, SpaceType.L2.getValue()), KNNEngine.NMSLIB) + () -> JNIService.loadIndex(null, ImmutableMap.of(KNNConstants.SPACE_TYPE, SpaceType.L2.getValue()), KNNEngine.NMSLIB) ); } @@ -719,91 +857,142 @@ public void testLoadIndex_nmslib_invalid_badFile() throws IOException { Path tmpFile = createTempFile(); expectThrows( Exception.class, - () -> JNIService.loadIndex( - tmpFile.toAbsolutePath().toString(), - ImmutableMap.of(KNNConstants.SPACE_TYPE, SpaceType.L2.getValue()), - KNNEngine.NMSLIB - ) + () -> JNIService.loadIndex(null, ImmutableMap.of(KNNConstants.SPACE_TYPE, SpaceType.L2.getValue()), KNNEngine.NMSLIB) ); } public void testLoadIndex_nmslib_valid() throws IOException { - Path tmpFile = createTempFile(); + Path tempDirPath = createTempDir(); + String indexFileName1 = "test1" + UUID.randomUUID() + ".tmp"; + try (Directory directory = newFSDirectory(tempDirPath)) { + TestUtils.createIndex( + testData.indexData.docs, + testData.loadDataToMemoryAddress(), + testData.indexData.getDimension(), + directory, + indexFileName1, + ImmutableMap.of(KNNConstants.SPACE_TYPE, SpaceType.L2.getValue()), + KNNEngine.NMSLIB + ); + assertTrue(directory.fileLength(indexFileName1) > 0); - TestUtils.createIndex( - testData.indexData.docs, - testData.loadDataToMemoryAddress(), - testData.indexData.getDimension(), - tmpFile.toAbsolutePath().toString(), - ImmutableMap.of(KNNConstants.SPACE_TYPE, SpaceType.L2.getValue()), - KNNEngine.NMSLIB - ); - assertTrue(tmpFile.toFile().length() > 0); + try (IndexInput indexInput = directory.openInput(indexFileName1, IOContext.LOAD)) { + final IndexInputWithBuffer indexInputWithBuffer = new IndexInputWithBuffer(indexInput); + long pointer = JNIService.loadIndex( + indexInputWithBuffer, + ImmutableMap.of(KNNConstants.SPACE_TYPE, SpaceType.L2.getValue()), + KNNEngine.NMSLIB + ); + assertNotEquals(0, pointer); + } catch (Throwable e) { + fail(e.getMessage()); + } + } + } - long pointer = JNIService.loadIndex( - tmpFile.toAbsolutePath().toString(), - ImmutableMap.of(KNNConstants.SPACE_TYPE, SpaceType.L2.getValue()), - KNNEngine.NMSLIB - ); - assertNotEquals(0, pointer); + public void testLoadIndex_nmslib_raise_io_exception() throws IOException { + + Path tempDirPath = createTempDir(); + String indexFileName1 = "test1" + UUID.randomUUID() + ".tmp"; + try (Directory directory = newFSDirectory(tempDirPath)) { + TestUtils.createIndex( + testData.indexData.docs, + testData.loadDataToMemoryAddress(), + testData.indexData.getDimension(), + directory, + indexFileName1, + ImmutableMap.of(KNNConstants.SPACE_TYPE, SpaceType.L2.getValue()), + KNNEngine.NMSLIB + ); + assertTrue(directory.fileLength(indexFileName1) > 0); + + final IndexInput raiseIOExceptionIndexInput = new RaisingIOExceptionIndexInput(); + + final IndexInputWithBuffer indexInputWithBuffer = new IndexInputWithBuffer(raiseIOExceptionIndexInput); + try { + JNIService.loadIndex( + indexInputWithBuffer, + ImmutableMap.of(KNNConstants.SPACE_TYPE, SpaceType.L2.getValue()), + KNNEngine.NMSLIB + ); + fail("Exception expected"); + } catch (Throwable e) { + assertTrue(e.getMessage().contains("Reading bytes via IndexInput has failed.")); + } + } } public void testLoadIndex_nmslib_valid_with_stream() throws IOException { - Path tmpFile = createTempFile(); - - TestUtils.createIndex( - testData.indexData.docs, - testData.loadDataToMemoryAddress(), - testData.indexData.getDimension(), - tmpFile.toAbsolutePath().toString(), - ImmutableMap.of(KNNConstants.SPACE_TYPE, SpaceType.L2.getValue()), - KNNEngine.NMSLIB - ); - assertTrue(tmpFile.toFile().length() > 0); + Path tempDirPath = createTempDir(); + String indexFileName1 = "test1" + UUID.randomUUID() + ".tmp"; + try (Directory directory = newFSDirectory(tempDirPath)) { + TestUtils.createIndex( + testData.indexData.docs, + testData.loadDataToMemoryAddress(), + testData.indexData.getDimension(), + directory, + indexFileName1, + ImmutableMap.of(KNNConstants.SPACE_TYPE, SpaceType.L2.getValue()), + KNNEngine.NMSLIB + ); + assertTrue(directory.fileLength(indexFileName1) > 0); - try (final Directory directory = new MMapDirectory(tmpFile.getParent())) { - try (IndexInput indexInput = directory.openInput(tmpFile.getFileName().toString(), IOContext.READONCE)) { + try (IndexInput indexInput = directory.openInput(indexFileName1, IOContext.LOAD)) { + final IndexInputWithBuffer indexInputWithBuffer = new IndexInputWithBuffer(indexInput); long pointer = JNIService.loadIndex( - new IndexInputWithBuffer(indexInput), + indexInputWithBuffer, ImmutableMap.of(KNNConstants.SPACE_TYPE, SpaceType.L2.getValue()), KNNEngine.NMSLIB ); assertNotEquals(0, pointer); + } catch (Throwable e) { + fail(e.getMessage()); } } } - public void testLoadIndex_faiss_invalid_fileDoesNotExist() { - expectThrows(Exception.class, () -> JNIService.loadIndex("invalid", Collections.emptyMap(), KNNEngine.FAISS)); - } - - public void testLoadIndex_faiss_invalid_badFile() throws IOException { - - Path tmpFile = createTempFile(); - - expectThrows( - Exception.class, - () -> JNIService.loadIndex(tmpFile.toAbsolutePath().toString(), Collections.emptyMap(), KNNEngine.FAISS) - ); + public void testWriteIndex_nmslib_when_io_exception_occured() { + try { + final IndexOutput indexOutput = new RasingIOExceptionIndexOutput(); + final IndexOutputWithBuffer indexOutputWithBuffer = new IndexOutputWithBuffer(indexOutput); + JNIService.createIndex( + testData.indexData.docs, + testData.loadDataToMemoryAddress(), + testData.indexData.getDimension(), + indexOutputWithBuffer, + ImmutableMap.of(KNNConstants.SPACE_TYPE, SpaceType.L2.getValue()), + KNNEngine.NMSLIB + ); + fail("Exception thrown is expected."); + } catch (Throwable e) { + assertTrue(e.getMessage().contains("Writing bytes via IndexOutput has failed.")); + } } public void testLoadIndex_faiss_valid() throws IOException { + Path tempDirPath = createTempDir(); + String indexFileName1 = "test1" + UUID.randomUUID() + ".tmp"; + try (Directory directory = newFSDirectory(tempDirPath)) { + TestUtils.createIndex( + testData.indexData.docs, + testData.loadDataToMemoryAddress(), + testData.indexData.getDimension(), + directory, + indexFileName1, + ImmutableMap.of(INDEX_DESCRIPTION_PARAMETER, faissMethod, KNNConstants.SPACE_TYPE, SpaceType.L2.getValue()), + KNNEngine.FAISS + ); + assertTrue(directory.fileLength(indexFileName1) > 0); - Path tmpFile = createTempFile(); - - TestUtils.createIndex( - testData.indexData.docs, - testData.loadDataToMemoryAddress(), - testData.indexData.getDimension(), - tmpFile.toAbsolutePath().toString(), - ImmutableMap.of(INDEX_DESCRIPTION_PARAMETER, faissMethod, KNNConstants.SPACE_TYPE, SpaceType.L2.getValue()), - KNNEngine.FAISS - ); - assertTrue(tmpFile.toFile().length() > 0); - - long pointer = JNIService.loadIndex(tmpFile.toAbsolutePath().toString(), Collections.emptyMap(), KNNEngine.FAISS); - assertNotEquals(0, pointer); + try (IndexInput indexInput = directory.openInput(indexFileName1, IOContext.LOAD)) { + final IndexInputWithBuffer indexInputWithBuffer = new IndexInputWithBuffer(indexInput); + long pointer = JNIService.loadIndex(indexInputWithBuffer, Collections.emptyMap(), KNNEngine.FAISS); + assertNotEquals(0, pointer); + } catch (Throwable e) { + fail(e.getMessage()); + } + } } public void testQueryIndex_invalidEngine() { @@ -820,107 +1009,144 @@ public void testQueryIndex_nmslib_invalid_badPointer() { public void testQueryIndex_nmslib_invalid_nullQueryVector() throws IOException { - Path tmpFile = createTempFile(); - - TestUtils.createIndex( - testData.indexData.docs, - testData.loadDataToMemoryAddress(), - testData.indexData.getDimension(), - tmpFile.toAbsolutePath().toString(), - ImmutableMap.of(KNNConstants.SPACE_TYPE, SpaceType.L2.getValue()), - KNNEngine.NMSLIB - ); - assertTrue(tmpFile.toFile().length() > 0); - - long pointer = JNIService.loadIndex( - tmpFile.toAbsolutePath().toString(), - ImmutableMap.of(KNNConstants.SPACE_TYPE, SpaceType.L2.getValue()), - KNNEngine.NMSLIB - ); - assertNotEquals(0, pointer); - - expectThrows(Exception.class, () -> JNIService.queryIndex(pointer, null, 10, null, KNNEngine.NMSLIB, null, 0, null)); - } - - public void testQueryIndex_nmslib_valid() throws IOException { - - int k = 50; - for (SpaceType spaceType : NmslibHNSWMethod.SUPPORTED_SPACES) { - if (SpaceType.UNDEFINED == spaceType) { - continue; - } - - Path tmpFile = createTempFile(); - + Path tempDirPath = createTempDir(); + String indexFileName1 = "test1" + UUID.randomUUID() + ".tmp"; + try (Directory directory = newFSDirectory(tempDirPath)) { TestUtils.createIndex( testData.indexData.docs, testData.loadDataToMemoryAddress(), testData.indexData.getDimension(), - tmpFile.toAbsolutePath().toString(), - ImmutableMap.of(KNNConstants.SPACE_TYPE, spaceType.getValue()), - KNNEngine.NMSLIB - ); - assertTrue(tmpFile.toFile().length() > 0); - - long pointer = JNIService.loadIndex( - tmpFile.toAbsolutePath().toString(), - ImmutableMap.of(KNNConstants.SPACE_TYPE, spaceType.getValue()), + directory, + indexFileName1, + ImmutableMap.of(KNNConstants.SPACE_TYPE, SpaceType.L2.getValue()), KNNEngine.NMSLIB ); - assertNotEquals(0, pointer); + assertTrue(directory.fileLength(indexFileName1) > 0); - for (float[] query : testData.queries) { - KNNQueryResult[] results = JNIService.queryIndex(pointer, query, k, null, KNNEngine.NMSLIB, null, 0, null); - assertEquals(k, results.length); + final long pointer; + try (IndexInput indexInput = directory.openInput(indexFileName1, IOContext.LOAD)) { + final IndexInputWithBuffer indexInputWithBuffer = new IndexInputWithBuffer(indexInput); + pointer = JNIService.loadIndex( + indexInputWithBuffer, + ImmutableMap.of(KNNConstants.SPACE_TYPE, SpaceType.L2.getValue()), + KNNEngine.NMSLIB + ); + assertNotEquals(0, pointer); + } catch (Throwable e) { + fail(e.getMessage()); + throw e; } + + expectThrows(Exception.class, () -> JNIService.queryIndex(pointer, null, 10, null, KNNEngine.NMSLIB, null, 0, null)); } } - public void testQueryIndex_faiss_invalid_badPointer() { + public void testQueryIndex_nmslib_valid() throws IOException { - expectThrows(Exception.class, () -> JNIService.queryIndex(0L, new float[] {}, 0, null, KNNEngine.FAISS, null, 0, null)); - } + Path tempDirPath = createTempDir(); + try (Directory directory = newFSDirectory(tempDirPath)) { + int k = 50; + for (SpaceType spaceType : NmslibHNSWMethod.SUPPORTED_SPACES) { + if (SpaceType.UNDEFINED == spaceType) { + continue; + } - public void testQueryIndex_faiss_invalid_nullQueryVector() throws IOException { + String indexFileName1 = "test1" + UUID.randomUUID() + ".tmp"; - Path tmpFile = createTempFile(); + TestUtils.createIndex( + testData.indexData.docs, + testData.loadDataToMemoryAddress(), + testData.indexData.getDimension(), + directory, + indexFileName1, + ImmutableMap.of(KNNConstants.SPACE_TYPE, spaceType.getValue()), + KNNEngine.NMSLIB + ); + assertTrue(directory.fileLength(indexFileName1) > 0); + + final long pointer; + try (IndexInput indexInput = directory.openInput(indexFileName1, IOContext.LOAD)) { + final IndexInputWithBuffer indexInputWithBuffer = new IndexInputWithBuffer(indexInput); + pointer = JNIService.loadIndex( + indexInputWithBuffer, + ImmutableMap.of(KNNConstants.SPACE_TYPE, spaceType.getValue()), + KNNEngine.NMSLIB + ); + assertNotEquals(0, pointer); + } catch (Throwable e) { + fail(e.getMessage()); + throw e; + } - TestUtils.createIndex( - testData.indexData.docs, - testData.loadDataToMemoryAddress(), - testData.indexData.getDimension(), - tmpFile.toAbsolutePath().toString(), - ImmutableMap.of(INDEX_DESCRIPTION_PARAMETER, faissMethod, KNNConstants.SPACE_TYPE, SpaceType.L2.getValue()), - KNNEngine.FAISS - ); - assertTrue(tmpFile.toFile().length() > 0); + for (float[] query : testData.queries) { + KNNQueryResult[] results = JNIService.queryIndex(pointer, query, k, null, KNNEngine.NMSLIB, null, 0, null); + assertEquals(k, results.length); + } + } + } + } - long pointer = JNIService.loadIndex(tmpFile.toAbsolutePath().toString(), Collections.emptyMap(), KNNEngine.FAISS); - assertNotEquals(0, pointer); + public void testQueryIndex_faiss_invalid_badPointer() { - expectThrows(Exception.class, () -> JNIService.queryIndex(pointer, null, 10, null, KNNEngine.FAISS, null, 0, null)); + expectThrows(Exception.class, () -> JNIService.queryIndex(0L, new float[] {}, 0, null, KNNEngine.FAISS, null, 0, null)); } - public void testQueryIndex_faiss_streaming_invalid_nullQueryVector() throws IOException { - Path tmpFile = createTempFile(); + public void testQueryIndex_faiss_invalid_nullQueryVector() throws IOException { - TestUtils.createIndex( - testData.indexData.docs, - testData.loadDataToMemoryAddress(), - testData.indexData.getDimension(), - tmpFile.toAbsolutePath().toString(), - ImmutableMap.of(INDEX_DESCRIPTION_PARAMETER, faissMethod, KNNConstants.SPACE_TYPE, SpaceType.L2.getValue()), - KNNEngine.FAISS - ); - assertTrue(tmpFile.toFile().length() > 0); + Path tempDirPath = createTempDir(); + String indexFileName1 = "test1" + UUID.randomUUID() + ".tmp"; + try (Directory directory = newFSDirectory(tempDirPath)) { + TestUtils.createIndex( + testData.indexData.docs, + testData.loadDataToMemoryAddress(), + testData.indexData.getDimension(), + directory, + indexFileName1, + ImmutableMap.of(INDEX_DESCRIPTION_PARAMETER, faissMethod, KNNConstants.SPACE_TYPE, SpaceType.L2.getValue()), + KNNEngine.FAISS + ); + assertTrue(directory.fileLength(indexFileName1) > 0); - try (final Directory directory = new MMapDirectory(tmpFile.getParent())) { - try (IndexInput indexInput = directory.openInput(tmpFile.getFileName().toString(), IOContext.READONCE)) { - long pointer = JNIService.loadIndex(new IndexInputWithBuffer(indexInput), Collections.emptyMap(), KNNEngine.FAISS); + final long pointer; + try (IndexInput indexInput = directory.openInput(indexFileName1, IOContext.LOAD)) { + final IndexInputWithBuffer indexInputWithBuffer = new IndexInputWithBuffer(indexInput); + pointer = JNIService.loadIndex(indexInputWithBuffer, Collections.emptyMap(), KNNEngine.FAISS); assertNotEquals(0, pointer); + } catch (Throwable e) { + fail(e.getMessage()); + throw e; + } + + expectThrows(Exception.class, () -> JNIService.queryIndex(pointer, null, 10, null, KNNEngine.FAISS, null, 0, null)); + } + } + + public void testQueryIndex_faiss_streaming_invalid_nullQueryVector() throws IOException { + Path tempDirPath = createTempDir(); + String indexFileName1 = "test1" + UUID.randomUUID() + ".tmp"; + try (Directory directory = newFSDirectory(tempDirPath)) { + TestUtils.createIndex( + testData.indexData.docs, + testData.loadDataToMemoryAddress(), + testData.indexData.getDimension(), + directory, + indexFileName1, + ImmutableMap.of(INDEX_DESCRIPTION_PARAMETER, faissMethod, KNNConstants.SPACE_TYPE, SpaceType.L2.getValue()), + KNNEngine.FAISS + ); + assertTrue(directory.fileLength(indexFileName1) > 0); - expectThrows(Exception.class, () -> JNIService.queryIndex(pointer, null, 10, null, KNNEngine.FAISS, null, 0, null)); + final long pointer; + try (IndexInput indexInput = directory.openInput(indexFileName1, IOContext.LOAD)) { + final IndexInputWithBuffer indexInputWithBuffer = new IndexInputWithBuffer(indexInput); + pointer = JNIService.loadIndex(indexInputWithBuffer, Collections.emptyMap(), KNNEngine.FAISS); + assertNotEquals(0, pointer); + } catch (Throwable e) { + fail(e.getMessage()); + throw e; } + + expectThrows(Exception.class, () -> JNIService.queryIndex(pointer, null, 10, null, KNNEngine.FAISS, null, 0, null)); } } @@ -929,55 +1155,66 @@ public void testQueryIndex_faiss_valid() throws IOException { int k = 10; int efSearch = 100; - List methods = ImmutableList.of(faissMethod); - List spaces = ImmutableList.of(SpaceType.L2, SpaceType.INNER_PRODUCT); - for (String method : methods) { - for (SpaceType spaceType : spaces) { - Path tmpFile = createTempFile(); - TestUtils.createIndex( - testData.indexData.docs, - testData.loadDataToMemoryAddress(), - testData.indexData.getDimension(), - tmpFile.toAbsolutePath().toString(), - ImmutableMap.of(INDEX_DESCRIPTION_PARAMETER, method, KNNConstants.SPACE_TYPE, spaceType.getValue()), - KNNEngine.FAISS - ); - assertTrue(tmpFile.toFile().length() > 0); - - long pointer = JNIService.loadIndex( - tmpFile.toAbsolutePath().toString(), - ImmutableMap.of(KNNConstants.SPACE_TYPE, spaceType.getValue()), - KNNEngine.FAISS - ); - assertNotEquals(0, pointer); - - for (float[] query : testData.queries) { - KNNQueryResult[] results = JNIService.queryIndex( - pointer, - query, - k, - Map.of("ef_search", efSearch), - KNNEngine.FAISS, - null, - 0, - null + Path tempDirPath = createTempDir(); + try (Directory directory = newFSDirectory(tempDirPath)) { + List methods = ImmutableList.of(faissMethod); + List spaces = ImmutableList.of(SpaceType.L2, SpaceType.INNER_PRODUCT); + for (String method : methods) { + for (SpaceType spaceType : spaces) { + String indexFileName1 = "test1" + UUID.randomUUID() + ".tmp"; + TestUtils.createIndex( + testData.indexData.docs, + testData.loadDataToMemoryAddress(), + testData.indexData.getDimension(), + directory, + indexFileName1, + ImmutableMap.of(INDEX_DESCRIPTION_PARAMETER, method, KNNConstants.SPACE_TYPE, spaceType.getValue()), + KNNEngine.FAISS ); - assertEquals(k, results.length); - } + assertTrue(directory.fileLength(indexFileName1) > 0); - // Filter will result in no ids - for (float[] query : testData.queries) { - KNNQueryResult[] results = JNIService.queryIndex( - pointer, - query, - k, - Map.of("ef_search", efSearch), - KNNEngine.FAISS, - new long[] { 0 }, - 0, - null - ); - assertEquals(0, results.length); + final long pointer; + try (IndexInput indexInput = directory.openInput(indexFileName1, IOContext.LOAD)) { + final IndexInputWithBuffer indexInputWithBuffer = new IndexInputWithBuffer(indexInput); + pointer = JNIService.loadIndex( + indexInputWithBuffer, + ImmutableMap.of(KNNConstants.SPACE_TYPE, spaceType.getValue()), + KNNEngine.FAISS + ); + assertNotEquals(0, pointer); + } catch (Throwable e) { + fail(e.getMessage()); + throw e; + } + + for (float[] query : testData.queries) { + KNNQueryResult[] results = JNIService.queryIndex( + pointer, + query, + k, + Map.of("ef_search", efSearch), + KNNEngine.FAISS, + null, + 0, + null + ); + assertEquals(k, results.length); + } + + // Filter will result in no ids + for (float[] query : testData.queries) { + KNNQueryResult[] results = JNIService.queryIndex( + pointer, + query, + k, + Map.of("ef_search", efSearch), + KNNEngine.FAISS, + new long[] { 0 }, + 0, + null + ); + assertEquals(0, results.length); + } } } } @@ -987,23 +1224,25 @@ public void testQueryIndex_faiss_streaming_valid() throws IOException { int k = 10; int efSearch = 100; - List methods = ImmutableList.of(faissMethod); - List spaces = ImmutableList.of(SpaceType.L2, SpaceType.INNER_PRODUCT); - for (String method : methods) { - for (SpaceType spaceType : spaces) { - Path tmpFile = createTempFile(); - TestUtils.createIndex( - testData.indexData.docs, - testData.loadDataToMemoryAddress(), - testData.indexData.getDimension(), - tmpFile.toAbsolutePath().toString(), - ImmutableMap.of(INDEX_DESCRIPTION_PARAMETER, method, KNNConstants.SPACE_TYPE, spaceType.getValue()), - KNNEngine.FAISS - ); - assertTrue(tmpFile.toFile().length() > 0); + Path tempDirPath = createTempDir(); + try (Directory directory = newFSDirectory(tempDirPath)) { + List methods = ImmutableList.of(faissMethod); + List spaces = ImmutableList.of(SpaceType.L2, SpaceType.INNER_PRODUCT); + for (String method : methods) { + for (SpaceType spaceType : spaces) { + String indexFileName1 = "test1" + UUID.randomUUID() + ".tmp"; + TestUtils.createIndex( + testData.indexData.docs, + testData.loadDataToMemoryAddress(), + testData.indexData.getDimension(), + directory, + indexFileName1, + ImmutableMap.of(INDEX_DESCRIPTION_PARAMETER, method, KNNConstants.SPACE_TYPE, spaceType.getValue()), + KNNEngine.FAISS + ); + assertTrue(directory.fileLength(indexFileName1) > 0); - try (final Directory directory = new MMapDirectory(tmpFile.getParent())) { - try (IndexInput indexInput = directory.openInput(tmpFile.getFileName().toString(), IOContext.READONCE)) { + try (IndexInput indexInput = directory.openInput(indexFileName1, IOContext.READONCE)) { long pointer = JNIService.loadIndex( new IndexInputWithBuffer(indexInput), ImmutableMap.of(KNNConstants.SPACE_TYPE, spaceType.getValue()), @@ -1040,9 +1279,9 @@ public void testQueryIndex_faiss_streaming_valid() throws IOException { assertEquals(0, results.length); } // End for } // End try - } // End try + } // End for } // End for - } // End for + } } public void testQueryIndex_faiss_parentIds() throws IOException { @@ -1050,44 +1289,55 @@ public void testQueryIndex_faiss_parentIds() throws IOException { int k = 100; int efSearch = 100; - List methods = ImmutableList.of(faissMethod); - List spaces = ImmutableList.of(SpaceType.L2, SpaceType.INNER_PRODUCT); - int[] parentIds = toParentIdArray(testDataNested.indexData.docs); - Map idToParentIdMap = toIdToParentIdMap(testDataNested.indexData.docs); - for (String method : methods) { - for (SpaceType spaceType : spaces) { - Path tmpFile = createTempFile(); - TestUtils.createIndex( - testDataNested.indexData.docs, - testData.loadDataToMemoryAddress(), - testDataNested.indexData.getDimension(), - tmpFile.toAbsolutePath().toString(), - ImmutableMap.of(INDEX_DESCRIPTION_PARAMETER, method, KNNConstants.SPACE_TYPE, spaceType.getValue()), - KNNEngine.FAISS - ); - assertTrue(tmpFile.toFile().length() > 0); - - long pointer = JNIService.loadIndex( - tmpFile.toAbsolutePath().toString(), - ImmutableMap.of(KNNConstants.SPACE_TYPE, spaceType.getValue()), - KNNEngine.FAISS - ); - assertNotEquals(0, pointer); - - for (float[] query : testDataNested.queries) { - KNNQueryResult[] results = JNIService.queryIndex( - pointer, - query, - k, - Map.of("ef_search", efSearch), - KNNEngine.FAISS, - null, - 0, - parentIds + Path tempDirPath = createTempDir(); + try (Directory directory = newFSDirectory(tempDirPath)) { + List methods = ImmutableList.of(faissMethod); + List spaces = ImmutableList.of(SpaceType.L2, SpaceType.INNER_PRODUCT); + int[] parentIds = toParentIdArray(testDataNested.indexData.docs); + Map idToParentIdMap = toIdToParentIdMap(testDataNested.indexData.docs); + for (String method : methods) { + for (SpaceType spaceType : spaces) { + String indexFileName1 = "test1" + UUID.randomUUID() + ".tmp"; + TestUtils.createIndex( + testDataNested.indexData.docs, + testData.loadDataToMemoryAddress(), + testDataNested.indexData.getDimension(), + directory, + indexFileName1, + ImmutableMap.of(INDEX_DESCRIPTION_PARAMETER, method, KNNConstants.SPACE_TYPE, spaceType.getValue()), + KNNEngine.FAISS ); - // Verify there is no more than one result from same parent - Set parentIdSet = toParentIdSet(results, idToParentIdMap); - assertEquals(results.length, parentIdSet.size()); + assertTrue(directory.fileLength(indexFileName1) > 0); + + final long pointer; + try (IndexInput indexInput = directory.openInput(indexFileName1, IOContext.LOAD)) { + final IndexInputWithBuffer indexInputWithBuffer = new IndexInputWithBuffer(indexInput); + pointer = JNIService.loadIndex( + indexInputWithBuffer, + ImmutableMap.of(KNNConstants.SPACE_TYPE, spaceType.getValue()), + KNNEngine.FAISS + ); + assertNotEquals(0, pointer); + } catch (Throwable e) { + fail(e.getMessage()); + throw e; + } + + for (float[] query : testDataNested.queries) { + KNNQueryResult[] results = JNIService.queryIndex( + pointer, + query, + k, + Map.of("ef_search", efSearch), + KNNEngine.FAISS, + null, + 0, + parentIds + ); + // Verify there is no more than one result from same parent + Set parentIdSet = toParentIdSet(results, idToParentIdMap); + assertEquals(results.length, parentIdSet.size()); + } } } } @@ -1098,25 +1348,27 @@ public void testQueryIndex_faiss_streaming_parentIds() throws IOException { int k = 100; int efSearch = 100; - List methods = ImmutableList.of(faissMethod); - List spaces = ImmutableList.of(SpaceType.L2, SpaceType.INNER_PRODUCT); - int[] parentIds = toParentIdArray(testDataNested.indexData.docs); - Map idToParentIdMap = toIdToParentIdMap(testDataNested.indexData.docs); - for (String method : methods) { - for (SpaceType spaceType : spaces) { - Path tmpFile = createTempFile(); - TestUtils.createIndex( - testDataNested.indexData.docs, - testData.loadDataToMemoryAddress(), - testDataNested.indexData.getDimension(), - tmpFile.toAbsolutePath().toString(), - ImmutableMap.of(INDEX_DESCRIPTION_PARAMETER, method, KNNConstants.SPACE_TYPE, spaceType.getValue()), - KNNEngine.FAISS - ); - assertTrue(tmpFile.toFile().length() > 0); + Path tempDirPath = createTempDir(); + try (Directory directory = newFSDirectory(tempDirPath)) { + List methods = ImmutableList.of(faissMethod); + List spaces = ImmutableList.of(SpaceType.L2, SpaceType.INNER_PRODUCT); + int[] parentIds = toParentIdArray(testDataNested.indexData.docs); + Map idToParentIdMap = toIdToParentIdMap(testDataNested.indexData.docs); + for (String method : methods) { + for (SpaceType spaceType : spaces) { + String indexFileName1 = "test1" + UUID.randomUUID() + ".tmp"; + TestUtils.createIndex( + testDataNested.indexData.docs, + testData.loadDataToMemoryAddress(), + testDataNested.indexData.getDimension(), + directory, + indexFileName1, + ImmutableMap.of(INDEX_DESCRIPTION_PARAMETER, method, KNNConstants.SPACE_TYPE, spaceType.getValue()), + KNNEngine.FAISS + ); + assertTrue(directory.fileLength(indexFileName1) > 0); - try (final Directory directory = new MMapDirectory(tmpFile.getParent())) { - try (IndexInput indexInput = directory.openInput(tmpFile.getFileName().toString(), IOContext.READONCE)) { + try (IndexInput indexInput = directory.openInput(indexFileName1, IOContext.READONCE)) { long pointer = JNIService.loadIndex( new IndexInputWithBuffer(indexInput), ImmutableMap.of(KNNConstants.SPACE_TYPE, spaceType.getValue()), @@ -1140,45 +1392,61 @@ public void testQueryIndex_faiss_streaming_parentIds() throws IOException { assertEquals(results.length, parentIdSet.size()); } // End for } // End try - } // End try + } // End for } // End for - } // End for + } } @SneakyThrows public void testQueryBinaryIndex_faiss_valid() { int k = 10; List methods = ImmutableList.of(faissBinaryMethod); - for (String method : methods) { - Path tmpFile = createTempFile(); - long memoryAddr = testData.loadBinaryDataToMemoryAddress(); - TestUtils.createIndex( - testData.indexData.docs, - memoryAddr, - testData.indexData.getDimension(), - tmpFile.toAbsolutePath().toString(), - ImmutableMap.of( - INDEX_DESCRIPTION_PARAMETER, - method, - KNNConstants.SPACE_TYPE, - SpaceType.HAMMING.getValue(), - KNNConstants.VECTOR_DATA_TYPE_FIELD, - VectorDataType.BINARY.getValue() - ), - KNNEngine.FAISS - ); - assertTrue(tmpFile.toFile().length() > 0); + Path tempDirPath = createTempDir(); + try (Directory directory = newFSDirectory(tempDirPath)) { + for (String method : methods) { + String indexFileName1 = "test1" + UUID.randomUUID() + ".tmp"; + long memoryAddr = testData.loadBinaryDataToMemoryAddress(); + TestUtils.createIndex( + testData.indexData.docs, + memoryAddr, + testData.indexData.getDimension(), + directory, + indexFileName1, + ImmutableMap.of( + INDEX_DESCRIPTION_PARAMETER, + method, + KNNConstants.SPACE_TYPE, + SpaceType.HAMMING.getValue(), + KNNConstants.VECTOR_DATA_TYPE_FIELD, + VectorDataType.BINARY.getValue() + ), + KNNEngine.FAISS + ); + assertTrue(directory.fileLength(indexFileName1) > 0); - long pointer = JNIService.loadIndex( - tmpFile.toAbsolutePath().toString(), - ImmutableMap.of(INDEX_DESCRIPTION_PARAMETER, method, KNNConstants.VECTOR_DATA_TYPE_FIELD, VectorDataType.BINARY.getValue()), - KNNEngine.FAISS - ); - assertNotEquals(0, pointer); + final long pointer; + try (IndexInput indexInput = directory.openInput(indexFileName1, IOContext.LOAD)) { + final IndexInputWithBuffer indexInputWithBuffer = new IndexInputWithBuffer(indexInput); + pointer = JNIService.loadIndex( + indexInputWithBuffer, + ImmutableMap.of( + INDEX_DESCRIPTION_PARAMETER, + method, + KNNConstants.VECTOR_DATA_TYPE_FIELD, + VectorDataType.BINARY.getValue() + ), + KNNEngine.FAISS + ); + assertNotEquals(0, pointer); + } catch (Throwable e) { + fail(e.getMessage()); + throw e; + } - for (byte[] query : testData.binaryQueries) { - KNNQueryResult[] results = JNIService.queryBinaryIndex(pointer, query, k, null, KNNEngine.FAISS, null, 0, null); - assertEquals(k, results.length); + for (byte[] query : testData.binaryQueries) { + KNNQueryResult[] results = JNIService.queryBinaryIndex(pointer, query, k, null, KNNEngine.FAISS, null, 0, null); + assertEquals(k, results.length); + } } } } @@ -1187,28 +1455,30 @@ public void testQueryBinaryIndex_faiss_valid() { public void testQueryBinaryIndex_faiss_streaming_valid() { int k = 10; List methods = ImmutableList.of(faissBinaryMethod); - for (String method : methods) { - Path tmpFile = createTempFile(); - long memoryAddr = testData.loadBinaryDataToMemoryAddress(); - TestUtils.createIndex( - testData.indexData.docs, - memoryAddr, - testData.indexData.getDimension(), - tmpFile.toAbsolutePath().toString(), - ImmutableMap.of( - INDEX_DESCRIPTION_PARAMETER, - method, - KNNConstants.SPACE_TYPE, - SpaceType.HAMMING.getValue(), - KNNConstants.VECTOR_DATA_TYPE_FIELD, - VectorDataType.BINARY.getValue() - ), - KNNEngine.FAISS - ); - assertTrue(tmpFile.toFile().length() > 0); + Path tempDirPath = createTempDir(); + try (Directory directory = newFSDirectory(tempDirPath)) { + for (String method : methods) { + String indexFileName1 = "test1" + UUID.randomUUID() + ".tmp"; + long memoryAddr = testData.loadBinaryDataToMemoryAddress(); + TestUtils.createIndex( + testData.indexData.docs, + memoryAddr, + testData.indexData.getDimension(), + directory, + indexFileName1, + ImmutableMap.of( + INDEX_DESCRIPTION_PARAMETER, + method, + KNNConstants.SPACE_TYPE, + SpaceType.HAMMING.getValue(), + KNNConstants.VECTOR_DATA_TYPE_FIELD, + VectorDataType.BINARY.getValue() + ), + KNNEngine.FAISS + ); + assertTrue(directory.fileLength(indexFileName1) > 0); - try (final Directory directory = new MMapDirectory(tmpFile.getParent())) { - try (IndexInput indexInput = directory.openInput(tmpFile.getFileName().toString(), IOContext.READONCE)) { + try (IndexInput indexInput = directory.openInput(indexFileName1, IOContext.READONCE)) { long pointer = JNIService.loadIndex( new IndexInputWithBuffer(indexInput), ImmutableMap.of( @@ -1226,8 +1496,8 @@ public void testQueryBinaryIndex_faiss_streaming_valid() { assertEquals(k, results.length); } // End for } // End try - } // End try - } // End for + } // End for + } // End try } private Set toParentIdSet(KNNQueryResult[] results, Map idToParentIdMap) { @@ -1276,46 +1546,66 @@ public void testFree_invalidEngine() { public void testFree_nmslib_valid() throws IOException { - Path tmpFile = createTempFile(); - - TestUtils.createIndex( - testData.indexData.docs, - testData.loadDataToMemoryAddress(), - testData.indexData.getDimension(), - tmpFile.toAbsolutePath().toString(), - ImmutableMap.of(KNNConstants.SPACE_TYPE, SpaceType.L2.getValue()), - KNNEngine.NMSLIB - ); - assertTrue(tmpFile.toFile().length() > 0); + Path tempDirPath = createTempDir(); + String indexFileName1 = "test1" + UUID.randomUUID() + ".tmp"; + try (Directory directory = newFSDirectory(tempDirPath)) { + TestUtils.createIndex( + testData.indexData.docs, + testData.loadDataToMemoryAddress(), + testData.indexData.getDimension(), + directory, + indexFileName1, + ImmutableMap.of(KNNConstants.SPACE_TYPE, SpaceType.L2.getValue()), + KNNEngine.NMSLIB + ); + assertTrue(directory.fileLength(indexFileName1) > 0); - long pointer = JNIService.loadIndex( - tmpFile.toAbsolutePath().toString(), - ImmutableMap.of(KNNConstants.SPACE_TYPE, SpaceType.L2.getValue()), - KNNEngine.NMSLIB - ); - assertNotEquals(0, pointer); + final long pointer; + try (IndexInput indexInput = directory.openInput(indexFileName1, IOContext.LOAD)) { + final IndexInputWithBuffer indexInputWithBuffer = new IndexInputWithBuffer(indexInput); + pointer = JNIService.loadIndex( + indexInputWithBuffer, + ImmutableMap.of(KNNConstants.SPACE_TYPE, SpaceType.L2.getValue()), + KNNEngine.NMSLIB + ); + assertNotEquals(0, pointer); + } catch (Throwable e) { + fail(e.getMessage()); + throw e; + } - JNIService.free(pointer, KNNEngine.NMSLIB); + JNIService.free(pointer, KNNEngine.NMSLIB); + } } public void testFree_faiss_valid() throws IOException { - Path tmpFile = createTempFile(); - - TestUtils.createIndex( - testData.indexData.docs, - testData.loadDataToMemoryAddress(), - testData.indexData.getDimension(), - tmpFile.toAbsolutePath().toString(), - ImmutableMap.of(INDEX_DESCRIPTION_PARAMETER, faissMethod, KNNConstants.SPACE_TYPE, SpaceType.L2.getValue()), - KNNEngine.FAISS - ); - assertTrue(tmpFile.toFile().length() > 0); + Path tempDirPath = createTempDir(); + String indexFileName1 = "test1" + UUID.randomUUID() + ".tmp"; + try (Directory directory = newFSDirectory(tempDirPath)) { + TestUtils.createIndex( + testData.indexData.docs, + testData.loadDataToMemoryAddress(), + testData.indexData.getDimension(), + directory, + indexFileName1, + ImmutableMap.of(INDEX_DESCRIPTION_PARAMETER, faissMethod, KNNConstants.SPACE_TYPE, SpaceType.L2.getValue()), + KNNEngine.FAISS + ); + assertTrue(directory.fileLength(indexFileName1) > 0); - long pointer = JNIService.loadIndex(tmpFile.toAbsolutePath().toString(), Collections.emptyMap(), KNNEngine.FAISS); - assertNotEquals(0, pointer); + final long pointer; + try (IndexInput indexInput = directory.openInput(indexFileName1, IOContext.LOAD)) { + final IndexInputWithBuffer indexInputWithBuffer = new IndexInputWithBuffer(indexInput); + pointer = JNIService.loadIndex(indexInputWithBuffer, Collections.emptyMap(), KNNEngine.FAISS); + assertNotEquals(0, pointer); + } catch (Throwable e) { + fail(e.getMessage()); + throw e; + } - JNIService.free(pointer, KNNEngine.FAISS); + JNIService.free(pointer, KNNEngine.FAISS); + } } public void testTransferVectors() { @@ -1492,20 +1782,71 @@ public void createIndexFromTemplate() throws IOException { assertNotEquals(0, faissIndex.length); JNICommons.freeVectorData(trainPointer1); - Path tmpFile1 = createTempFile(); - JNIService.createIndexFromTemplate( - testData.indexData.docs, - testData.loadDataToMemoryAddress(), - testData.indexData.getDimension(), - tmpFile1.toAbsolutePath().toString(), - faissIndex, - ImmutableMap.of(INDEX_THREAD_QTY, 1), - KNNEngine.FAISS + Path tempDirPath = createTempDir(); + String indexFileName1 = "test1" + UUID.randomUUID() + ".tmp"; + try (Directory directory = newFSDirectory(tempDirPath)) { + try (IndexOutput indexOutput = directory.createOutput(indexFileName1, IOContext.DEFAULT)) { + final IndexOutputWithBuffer indexOutputWithBuffer = new IndexOutputWithBuffer(indexOutput); + JNIService.createIndexFromTemplate( + testData.indexData.docs, + testData.loadDataToMemoryAddress(), + testData.indexData.getDimension(), + indexOutputWithBuffer, + faissIndex, + ImmutableMap.of(INDEX_THREAD_QTY, 1), + KNNEngine.FAISS + ); + } + assertTrue(directory.fileLength(indexFileName1) > 0); + + final long pointer; + try (IndexInput indexInput = directory.openInput(indexFileName1, IOContext.LOAD)) { + final IndexInputWithBuffer indexInputWithBuffer = new IndexInputWithBuffer(indexInput); + pointer = JNIService.loadIndex(indexInputWithBuffer, Collections.emptyMap(), KNNEngine.FAISS); + assertNotEquals(0, pointer); + } catch (Throwable e) { + fail(e.getMessage()); + throw e; + } + } + } + + @SneakyThrows + public void testCreateIndex_whenIOExceptionOccured() { + // Plain index + Map parameters = new HashMap<>( + ImmutableMap.of(INDEX_DESCRIPTION_PARAMETER, faissMethod, KNNConstants.SPACE_TYPE, SpaceType.L2.getValue()) ); - assertTrue(tmpFile1.toFile().length() > 0); - long pointer = JNIService.loadIndex(tmpFile1.toAbsolutePath().toString(), Collections.emptyMap(), KNNEngine.FAISS); - assertNotEquals(0, pointer); + long trainPointer = JNIService.transferVectors(0, testData.indexData.vectors); + assertNotEquals(0, trainPointer); + KNNMethodConfigContext knnMethodConfigContext = KNNMethodConfigContext.builder() + .versionCreated(Version.CURRENT) + .dimension(128) + .vectorDataType(VectorDataType.FLOAT) + .build(); + + byte[] faissIndex = JNIService.trainIndex(parameters, 128, trainPointer, KNNEngine.FAISS); + + assertNotEquals(0, faissIndex.length); + JNICommons.freeVectorData(trainPointer); + + final IndexOutput indexOutput = new RasingIOExceptionIndexOutput(); + final IndexOutputWithBuffer indexOutputWithBuffer = new IndexOutputWithBuffer(indexOutput); + try { + JNIService.createIndexFromTemplate( + testData.indexData.docs, + testData.loadDataToMemoryAddress(), + testData.indexData.getDimension(), + indexOutputWithBuffer, + faissIndex, + ImmutableMap.of(INDEX_THREAD_QTY, 1), + KNNEngine.FAISS + ); + fail("Exception thrown was expected"); + } catch (Throwable t) { + System.out.println("!!!!!!!!!!!!!!!!!!!!! " + t.getMessage()); + } } @SneakyThrows @@ -1516,35 +1857,58 @@ public void testIndexLoad_whenStateIsShared_thenSucceed() { int ivfNlist = 16; int pqM = 16; int pqCodeSize = 4; + Path tempDirPath = createTempDir(); + try (Directory directory = newFSDirectory(tempDirPath)) { + String indexIVFPQPath = createFaissIVFPQIndex(directory, ivfNlist, pqM, pqCodeSize, SpaceType.L2); + + final long indexIVFPQIndexTest1; + try (IndexInput indexInput = directory.openInput(indexIVFPQPath, IOContext.LOAD)) { + final IndexInputWithBuffer indexInputWithBuffer = new IndexInputWithBuffer(indexInput); + indexIVFPQIndexTest1 = JNIService.loadIndex(indexInputWithBuffer, Collections.emptyMap(), KNNEngine.FAISS); + assertNotEquals(0, indexIVFPQIndexTest1); + } catch (Throwable e) { + fail(e.getMessage()); + throw e; + } + final long indexIVFPQIndexTest2; + try (IndexInput indexInput = directory.openInput(indexIVFPQPath, IOContext.LOAD)) { + final IndexInputWithBuffer indexInputWithBuffer = new IndexInputWithBuffer(indexInput); + indexIVFPQIndexTest2 = JNIService.loadIndex(indexInputWithBuffer, Collections.emptyMap(), KNNEngine.FAISS); + assertNotEquals(0, indexIVFPQIndexTest2); + } catch (Throwable e) { + fail(e.getMessage()); + throw e; + } - String indexIVFPQPath = createFaissIVFPQIndex(ivfNlist, pqM, pqCodeSize, SpaceType.L2); - - long indexIVFPQIndexTest1 = JNIService.loadIndex(indexIVFPQPath, Collections.emptyMap(), KNNEngine.FAISS); - assertNotEquals(0, indexIVFPQIndexTest1); - long indexIVFPQIndexTest2 = JNIService.loadIndex(indexIVFPQPath, Collections.emptyMap(), KNNEngine.FAISS); - assertNotEquals(0, indexIVFPQIndexTest2); - - long sharedStateAddress = JNIService.initSharedIndexState(indexIVFPQIndexTest1, KNNEngine.FAISS); - JNIService.setSharedIndexState(indexIVFPQIndexTest1, sharedStateAddress, KNNEngine.FAISS); - JNIService.setSharedIndexState(indexIVFPQIndexTest2, sharedStateAddress, KNNEngine.FAISS); + long sharedStateAddress = JNIService.initSharedIndexState(indexIVFPQIndexTest1, KNNEngine.FAISS); + JNIService.setSharedIndexState(indexIVFPQIndexTest1, sharedStateAddress, KNNEngine.FAISS); + JNIService.setSharedIndexState(indexIVFPQIndexTest2, sharedStateAddress, KNNEngine.FAISS); - assertQueryResultsMatch(testData.queries, k, List.of(indexIVFPQIndexTest1, indexIVFPQIndexTest2)); + assertQueryResultsMatch(testData.queries, k, List.of(indexIVFPQIndexTest1, indexIVFPQIndexTest2)); - // Free the first test index 1. This will ensure that the shared state persists after index that initialized - // shared state is gone. - JNIService.free(indexIVFPQIndexTest1, KNNEngine.FAISS); + // Free the first test index 1. This will ensure that the shared state persists after index that initialized + // shared state is gone. + JNIService.free(indexIVFPQIndexTest1, KNNEngine.FAISS); - long indexIVFPQIndexTest3 = JNIService.loadIndex(indexIVFPQPath, Collections.emptyMap(), KNNEngine.FAISS); - assertNotEquals(0, indexIVFPQIndexTest3); + final long indexIVFPQIndexTest3; + try (IndexInput indexInput = directory.openInput(indexIVFPQPath, IOContext.LOAD)) { + final IndexInputWithBuffer indexInputWithBuffer = new IndexInputWithBuffer(indexInput); + indexIVFPQIndexTest3 = JNIService.loadIndex(indexInputWithBuffer, Collections.emptyMap(), KNNEngine.FAISS); + assertNotEquals(0, indexIVFPQIndexTest3); + } catch (Throwable e) { + fail(e.getMessage()); + throw e; + } - JNIService.setSharedIndexState(indexIVFPQIndexTest3, sharedStateAddress, KNNEngine.FAISS); + JNIService.setSharedIndexState(indexIVFPQIndexTest3, sharedStateAddress, KNNEngine.FAISS); - assertQueryResultsMatch(testData.queries, k, List.of(indexIVFPQIndexTest2, indexIVFPQIndexTest3)); + assertQueryResultsMatch(testData.queries, k, List.of(indexIVFPQIndexTest2, indexIVFPQIndexTest3)); - // Ensure everything gets freed - JNIService.free(indexIVFPQIndexTest2, KNNEngine.FAISS); - JNIService.free(indexIVFPQIndexTest3, KNNEngine.FAISS); - JNIService.freeSharedIndexState(sharedStateAddress, KNNEngine.FAISS); + // Ensure everything gets freed + JNIService.free(indexIVFPQIndexTest2, KNNEngine.FAISS); + JNIService.free(indexIVFPQIndexTest3, KNNEngine.FAISS); + JNIService.freeSharedIndexState(sharedStateAddress, KNNEngine.FAISS); + } } @SneakyThrows @@ -1552,20 +1916,32 @@ public void testIsIndexIVFPQL2() { long dummyAddress = 0; assertFalse(JNIService.isSharedIndexStateRequired(dummyAddress, KNNEngine.NMSLIB)); - String faissIVFPQL2Index = createFaissIVFPQIndex(16, 16, 4, SpaceType.L2); - long faissIVFPQL2Address = JNIService.loadIndex(faissIVFPQL2Index, Collections.emptyMap(), KNNEngine.FAISS); - assertTrue(JNIService.isSharedIndexStateRequired(faissIVFPQL2Address, KNNEngine.FAISS)); - JNIService.free(faissIVFPQL2Address, KNNEngine.FAISS); + Path tempDirPath = createTempDir(); + try (Directory directory = newFSDirectory(tempDirPath)) { + String faissIVFPQL2Index = createFaissIVFPQIndex(directory, 16, 16, 4, SpaceType.L2); + try (IndexInput indexInput = directory.openInput(faissIVFPQL2Index, IOContext.LOAD)) { + final IndexInputWithBuffer indexInputWithBuffer = new IndexInputWithBuffer(indexInput); + long faissIVFPQL2Address = JNIService.loadIndex(indexInputWithBuffer, Collections.emptyMap(), KNNEngine.FAISS); + assertTrue(JNIService.isSharedIndexStateRequired(faissIVFPQL2Address, KNNEngine.FAISS)); + JNIService.free(faissIVFPQL2Address, KNNEngine.FAISS); + } - String faissIVFPQIPIndex = createFaissIVFPQIndex(16, 16, 4, SpaceType.INNER_PRODUCT); - long faissIVFPQIPAddress = JNIService.loadIndex(faissIVFPQIPIndex, Collections.emptyMap(), KNNEngine.FAISS); - assertFalse(JNIService.isSharedIndexStateRequired(faissIVFPQIPAddress, KNNEngine.FAISS)); - JNIService.free(faissIVFPQIPAddress, KNNEngine.FAISS); + String faissIVFPQIPIndex = createFaissIVFPQIndex(directory, 16, 16, 4, SpaceType.INNER_PRODUCT); + try (IndexInput indexInput = directory.openInput(faissIVFPQIPIndex, IOContext.LOAD)) { + final IndexInputWithBuffer indexInputWithBuffer = new IndexInputWithBuffer(indexInput); + long faissIVFPQIPAddress = JNIService.loadIndex(indexInputWithBuffer, Collections.emptyMap(), KNNEngine.FAISS); + assertFalse(JNIService.isSharedIndexStateRequired(faissIVFPQIPAddress, KNNEngine.FAISS)); + JNIService.free(faissIVFPQIPAddress, KNNEngine.FAISS); + } - String faissHNSWIndex = createFaissHNSWIndex(SpaceType.L2); - long faissHNSWAddress = JNIService.loadIndex(faissHNSWIndex, Collections.emptyMap(), KNNEngine.FAISS); - assertFalse(JNIService.isSharedIndexStateRequired(faissHNSWAddress, KNNEngine.FAISS)); - JNIService.free(faissHNSWAddress, KNNEngine.FAISS); + String faissHNSWIndex = createFaissHNSWIndex(directory, SpaceType.L2); + try (IndexInput indexInput = directory.openInput(faissHNSWIndex, IOContext.LOAD)) { + final IndexInputWithBuffer indexInputWithBuffer = new IndexInputWithBuffer(indexInput); + long faissHNSWAddress = JNIService.loadIndex(indexInputWithBuffer, Collections.emptyMap(), KNNEngine.FAISS); + assertFalse(JNIService.isSharedIndexStateRequired(faissHNSWAddress, KNNEngine.FAISS)); + JNIService.free(faissHNSWAddress, KNNEngine.FAISS); + } + } } @SneakyThrows @@ -1594,7 +1970,8 @@ private void assertQueryResultsMatch(float[][] testQueries, int k, List in } } - private String createFaissIVFPQIndex(int ivfNlist, int pqM, int pqCodeSize, SpaceType spaceType) throws IOException { + private String createFaissIVFPQIndex(Directory directory, int ivfNlist, int pqM, int pqCodeSize, SpaceType spaceType) + throws IOException { long trainPointer = JNIService.transferVectors(0, testData.indexData.vectors); assertNotEquals(0, trainPointer); KNNMethodConfigContext knnMethodConfigContext = KNNMethodConfigContext.builder() @@ -1635,32 +2012,36 @@ private String createFaissIVFPQIndex(int ivfNlist, int pqM, int pqCodeSize, Spac assertNotEquals(0, faissIndex.length); JNICommons.freeVectorData(trainPointer); - Path tmpFile = createTempFile(); - JNIService.createIndexFromTemplate( - testData.indexData.docs, - testData.loadDataToMemoryAddress(), - testData.indexData.getDimension(), - tmpFile.toAbsolutePath().toString(), - faissIndex, - ImmutableMap.of(INDEX_THREAD_QTY, 1), - KNNEngine.FAISS - ); - assertTrue(tmpFile.toFile().length() > 0); + String indexFileName1 = "test1" + UUID.randomUUID() + ".tmp"; + try (IndexOutput indexOutput = directory.createOutput(indexFileName1, IOContext.DEFAULT)) { + final IndexOutputWithBuffer indexOutputWithBuffer = new IndexOutputWithBuffer(indexOutput); + JNIService.createIndexFromTemplate( + testData.indexData.docs, + testData.loadDataToMemoryAddress(), + testData.indexData.getDimension(), + indexOutputWithBuffer, + faissIndex, + ImmutableMap.of(INDEX_THREAD_QTY, 1), + KNNEngine.FAISS + ); + } + assertTrue(directory.fileLength(indexFileName1) > 0); - return tmpFile.toAbsolutePath().toString(); + return indexFileName1; } - private String createFaissHNSWIndex(SpaceType spaceType) throws IOException { - Path tmpFile = createTempFile(); + private String createFaissHNSWIndex(Directory directory, SpaceType spaceType) throws IOException { + String indexFileName1 = "test1" + UUID.randomUUID() + ".tmp"; TestUtils.createIndex( testData.indexData.docs, testData.loadDataToMemoryAddress(), testData.indexData.getDimension(), - tmpFile.toAbsolutePath().toString(), + directory, + indexFileName1, ImmutableMap.of(INDEX_DESCRIPTION_PARAMETER, faissMethod, KNNConstants.SPACE_TYPE, spaceType.getValue()), KNNEngine.FAISS ); - assertTrue(tmpFile.toFile().length() > 0); - return tmpFile.toAbsolutePath().toString(); + assertTrue(directory.fileLength(indexFileName1) > 0); + return indexFileName1; } } diff --git a/src/test/java/org/opensearch/knn/training/TrainingJobTests.java b/src/test/java/org/opensearch/knn/training/TrainingJobTests.java index 14308b915..4706bd000 100644 --- a/src/test/java/org/opensearch/knn/training/TrainingJobTests.java +++ b/src/test/java/org/opensearch/knn/training/TrainingJobTests.java @@ -12,6 +12,9 @@ package org.opensearch.knn.training; import com.google.common.collect.ImmutableMap; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IOContext; +import org.apache.lucene.store.IndexOutput; import org.opensearch.Version; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.knn.KNNTestCase; @@ -26,15 +29,16 @@ import org.opensearch.knn.index.memory.NativeMemoryCacheManager; import org.opensearch.knn.index.memory.NativeMemoryEntryContext; import org.opensearch.knn.index.engine.KNNEngine; +import org.opensearch.knn.index.store.IndexOutputWithBuffer; import org.opensearch.knn.indices.Model; import org.opensearch.knn.indices.ModelMetadata; import org.opensearch.knn.indices.ModelState; import org.opensearch.knn.jni.JNICommons; import org.opensearch.knn.jni.JNIService; -import java.io.File; import java.io.IOException; import java.nio.file.Path; +import java.util.UUID; import java.util.concurrent.ExecutionException; import static org.mockito.Mockito.doAnswer; @@ -221,17 +225,23 @@ public void testRun_success() throws IOException, ExecutionException { float[][] vectors = new float[ids.length][dimension]; fillFloatArrayRandomly(vectors); long vectorsMemoryAddress = JNICommons.storeVectorData(0, vectors, (long) vectors.length * vectors[0].length); - Path indexPath = createTempFile(); - JNIService.createIndexFromTemplate( - ids, - vectorsMemoryAddress, - vectors[0].length, - indexPath.toString(), - model.getModelBlob(), - ImmutableMap.of(INDEX_THREAD_QTY, 1), - knnEngine - ); - assertNotEquals(0, new File(indexPath.toString()).length()); + Path tempDirPath = createTempDir(); + String indexFileName1 = "test1" + UUID.randomUUID() + ".tmp"; + try (Directory directory = newFSDirectory(tempDirPath)) { + try (IndexOutput indexOutput = directory.createOutput(indexFileName1, IOContext.DEFAULT)) { + final IndexOutputWithBuffer indexOutputWithBuffer = new IndexOutputWithBuffer(indexOutput); + JNIService.createIndexFromTemplate( + ids, + vectorsMemoryAddress, + vectors[0].length, + indexOutputWithBuffer, + model.getModelBlob(), + ImmutableMap.of(INDEX_THREAD_QTY, 1), + knnEngine + ); + } + assertTrue(directory.fileLength(indexFileName1) > 0); + } } public void testRun_failure_onGetTrainingDataAllocation() throws ExecutionException { diff --git a/src/testFixtures/java/org/opensearch/knn/TestUtils.java b/src/testFixtures/java/org/opensearch/knn/TestUtils.java index 9145bcd16..3642752ec 100644 --- a/src/testFixtures/java/org/opensearch/knn/TestUtils.java +++ b/src/testFixtures/java/org/opensearch/knn/TestUtils.java @@ -12,12 +12,18 @@ import lombok.AllArgsConstructor; import lombok.Getter; import lombok.Setter; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IOContext; +import org.apache.lucene.store.IndexOutput; +import org.opensearch.core.xcontent.DeprecationHandler; +import org.opensearch.core.xcontent.NamedXContentRegistry; import java.io.BufferedReader; import java.io.FileReader; import java.io.IOException; import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.codec.util.SerializationMode; import org.opensearch.knn.index.engine.KNNEngine; +import org.opensearch.knn.index.store.IndexOutputWithBuffer; import org.opensearch.knn.jni.JNICommons; import org.opensearch.knn.jni.JNIService; import org.opensearch.knn.plugin.script.KNNScoringUtil; @@ -417,14 +423,32 @@ public static class Pair { } } - public static void createIndex(int[] ids, long address, int dimension, String name, Map parameters, KNNEngine engine) { + public static void createIndex( + int[] ids, + long address, + int dimension, + Directory directory, + String fileName, + Map parameters, + KNNEngine engine + ) { if (engine != KNNEngine.FAISS) { - JNIService.createIndex(ids, address, dimension, name, parameters, engine); + try (IndexOutput indexOutput = directory.createOutput(fileName, IOContext.DEFAULT)) { + final IndexOutputWithBuffer indexOutputWithBuffer = new IndexOutputWithBuffer(indexOutput); + JNIService.createIndex(ids, address, dimension, indexOutputWithBuffer, parameters, engine); + } catch (IOException e) { + throw new RuntimeException(e); + } } else { // We can initialize numDocs as 0, this will just not reserve anything. long indexAddress = JNIService.initIndex(0, dimension, parameters, engine); JNIService.insertToIndex(ids, address, dimension, parameters, indexAddress, engine); - JNIService.writeIndex(name, indexAddress, engine, parameters); + try (IndexOutput indexOutput = directory.createOutput(fileName, IOContext.DEFAULT)) { + final IndexOutputWithBuffer indexOutputWithBuffer = new IndexOutputWithBuffer(indexOutput); + JNIService.writeIndex(indexOutputWithBuffer, indexAddress, engine, parameters); + } catch (IOException e) { + throw new RuntimeException(e); + } } } } From 18732114bbfc43649642f1b045f151b304d2a74a Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Fri, 8 Nov 2024 10:02:11 -0800 Subject: [PATCH 412/416] Update default engine to FAISS (#2221) (#2258) * Update default engine to FAISS Since faiss supports more features than nmslib, and, we had seen data points that there are more number of vector search users are interesed in faiss, we will be updating default engine to be faiss. This will benefit users who preffered to use defaults while working with vector search. Signed-off-by: Vijayan Balasubramanian * Update legacy mapping Signed-off-by: Vijayan Balasubramanian * Create legacy mapping only up to V_2_17_2 Signed-off-by: Vijayan Balasubramanian * Update test engine Signed-off-by: Vijayan Balasubramanian * Update test method Signed-off-by: Vijayan Balasubramanian --------- Signed-off-by: Vijayan Balasubramanian (cherry picked from commit 7d3445631591296d00c37ea16351a07ca08ffbd3) Co-authored-by: Vijayan Balasubramanian --- .../org/opensearch/knn/bwc/IndexingIT.java | 17 ++++ .../opensearch-knn.release-notes-2.18.0.0.md | 1 + .../knn/index/engine/EngineResolver.java | 2 +- .../knn/index/engine/KNNEngine.java | 2 +- .../index/mapper/KNNVectorFieldMapper.java | 9 +- .../opensearch/knn/index/query/KNNWeight.java | 2 +- .../opensearch/knn/KNNSingleNodeTestCase.java | 18 ++++ .../knn/index/KNNIndexShardTests.java | 3 +- .../org/opensearch/knn/index/NmslibIT.java | 5 ++ .../knn/index/VectorDataTypeIT.java | 23 ----- .../KNN80DocValuesConsumerTests.java | 1 + .../knn/index/codec/KNNCodecTestCase.java | 2 +- .../knn/index/engine/EngineResolverTests.java | 6 +- .../knn/index/engine/KNNEngineTests.java | 4 + .../mapper/KNNVectorFieldMapperTests.java | 85 ++++++++++--------- .../knn/index/query/KNNWeightTests.java | 9 +- .../opensearch/knn/jni/JNIServiceTests.java | 2 +- .../transport/GetModelResponseTests.java | 9 +- .../transport/TrainingModelRequestTests.java | 6 +- 19 files changed, 129 insertions(+), 77 deletions(-) diff --git a/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/IndexingIT.java b/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/IndexingIT.java index 7d2b3807a..b212d844f 100644 --- a/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/IndexingIT.java +++ b/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/IndexingIT.java @@ -61,6 +61,23 @@ public void testKNNIndexDefaultLegacyFieldMapping() throws Exception { } } + // ensure that index is created using legacy mapping in old cluster, and, then, add docs to both old and new cluster. + // when search is requested on new cluster it should return all docs irrespective of cluster. + public void testKNNIndexDefaultEngine() throws Exception { + waitForClusterHealthGreen(NODES_BWC_CLUSTER); + if (isRunningAgainstOldCluster()) { + createKnnIndex(testIndex, getKNNDefaultIndexSettings(), createKnnIndexMapping(TEST_FIELD, DIMENSIONS)); + addKNNDocs(testIndex, TEST_FIELD, DIMENSIONS, DOC_ID, 5); + // Flush to ensure that index is not re-indexed when node comes back up + flush(testIndex, true); + } else { + validateKNNSearch(testIndex, TEST_FIELD, DIMENSIONS, 5, 5); + addKNNDocs(testIndex, TEST_FIELD, DIMENSIONS, 5, 5); + validateKNNSearch(testIndex, TEST_FIELD, DIMENSIONS, 10, 10); + deleteKNNIndex(testIndex); + } + } + // Ensure that when segments created with old mapping are forcemerged in new cluster, they // succeed public void testKNNIndexDefaultLegacyFieldMappingForceMerge() throws Exception { diff --git a/release-notes/opensearch-knn.release-notes-2.18.0.0.md b/release-notes/opensearch-knn.release-notes-2.18.0.0.md index 9e85fd9ca..844605a8e 100644 --- a/release-notes/opensearch-knn.release-notes-2.18.0.0.md +++ b/release-notes/opensearch-knn.release-notes-2.18.0.0.md @@ -15,6 +15,7 @@ Compatible with OpenSearch 2.18.0 * Add CompressionLevel Calculation for PQ [#2200](https://github.com/opensearch-project/k-NN/pull/2200) * Remove FSDirectory dependency from native engine constructing side and deprecated FileWatcher [#2182](https://github.com/opensearch-project/k-NN/pull/2182) * Update approximate_threshold to 15K documents [#2229](https://github.com/opensearch-project/k-NN/pull/2229) +* Update default engine to FAISS [#2221](https://github.com/opensearch-project/k-NN/pull/2221) ### Bug Fixes * Add DocValuesProducers for releasing memory when close index [#1946](https://github.com/opensearch-project/k-NN/pull/1946) * KNN80DocValues should only be considered for BinaryDocValues fields [#2147](https://github.com/opensearch-project/k-NN/pull/2147) diff --git a/src/main/java/org/opensearch/knn/index/engine/EngineResolver.java b/src/main/java/org/opensearch/knn/index/engine/EngineResolver.java index daae361e4..d52c21c4c 100644 --- a/src/main/java/org/opensearch/knn/index/engine/EngineResolver.java +++ b/src/main/java/org/opensearch/knn/index/engine/EngineResolver.java @@ -49,7 +49,7 @@ public KNNEngine resolveEngine( // For 1x, we need to default to faiss if mode is provided and use nmslib otherwise if (CompressionLevel.isConfigured(compressionLevel) == false || compressionLevel == CompressionLevel.x1) { - return mode == Mode.ON_DISK ? KNNEngine.FAISS : KNNEngine.DEFAULT; + return mode == Mode.ON_DISK ? KNNEngine.FAISS : KNNEngine.NMSLIB; } // Lucene is only engine that supports 4x - so we have to default to it here. diff --git a/src/main/java/org/opensearch/knn/index/engine/KNNEngine.java b/src/main/java/org/opensearch/knn/index/engine/KNNEngine.java index 80b9f32a6..1e560a11b 100644 --- a/src/main/java/org/opensearch/knn/index/engine/KNNEngine.java +++ b/src/main/java/org/opensearch/knn/index/engine/KNNEngine.java @@ -29,7 +29,7 @@ public enum KNNEngine implements KNNLibrary { FAISS(FAISS_NAME, Faiss.INSTANCE), LUCENE(LUCENE_NAME, Lucene.INSTANCE); - public static final KNNEngine DEFAULT = NMSLIB; + public static final KNNEngine DEFAULT = FAISS; private static final Set CUSTOM_SEGMENT_FILE_ENGINES = ImmutableSet.of(KNNEngine.NMSLIB, KNNEngine.FAISS); private static final Set ENGINES_SUPPORTING_FILTERS = ImmutableSet.of(KNNEngine.LUCENE, KNNEngine.FAISS); diff --git a/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapper.java b/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapper.java index 18d4f7b64..beceadde5 100644 --- a/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapper.java +++ b/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapper.java @@ -500,8 +500,7 @@ private void resolveKNNMethodComponents( .build() ); - // If the original parameters are from legacy - if (builder.originalParameters.isLegacyMapping()) { + if (useKNNMethodContextFromLegacy(builder, parserContext)) { // Then create KNNMethodContext to be used from the legacy index settings builder.originalParameters.setResolvedKnnMethodContext( createKNNMethodContextFromLegacy(parserContext.getSettings(), parserContext.indexVersionCreated(), resolvedSpaceType) @@ -550,6 +549,12 @@ private void setEngine(final KNNMethodContext knnMethodContext, KNNEngine knnEng } } + static boolean useKNNMethodContextFromLegacy(Builder builder, Mapper.TypeParser.ParserContext parserContext) { + // If the original parameters are from legacy, and it is created on or before 2_17_2 since default is changed to + // FAISS starting 2_18, which doesn't support accepting algo params from index settings + return parserContext.indexVersionCreated().onOrBefore(Version.V_2_17_2) && builder.originalParameters.isLegacyMapping(); + } + // We store the version of the index with the mapper as different version of Opensearch has different default // values of KNN engine Algorithms hyperparameters. protected Version indexCreatedVersion; diff --git a/src/main/java/org/opensearch/knn/index/query/KNNWeight.java b/src/main/java/org/opensearch/knn/index/query/KNNWeight.java index b5b2a5d22..04c2ce587 100644 --- a/src/main/java/org/opensearch/knn/index/query/KNNWeight.java +++ b/src/main/java/org/opensearch/knn/index/query/KNNWeight.java @@ -251,7 +251,7 @@ private Map doANNSearch( spaceType = modelMetadata.getSpaceType(); vectorDataType = modelMetadata.getVectorDataType(); } else { - String engineName = fieldInfo.attributes().getOrDefault(KNN_ENGINE, KNNEngine.NMSLIB.getName()); + String engineName = fieldInfo.attributes().getOrDefault(KNN_ENGINE, KNNEngine.DEFAULT.getName()); knnEngine = KNNEngine.getEngine(engineName); String spaceTypeName = fieldInfo.attributes().getOrDefault(SPACE_TYPE, SpaceType.L2.getValue()); spaceType = SpaceType.getSpace(spaceTypeName); diff --git a/src/test/java/org/opensearch/knn/KNNSingleNodeTestCase.java b/src/test/java/org/opensearch/knn/KNNSingleNodeTestCase.java index 587e80e5d..7bfce5b94 100644 --- a/src/test/java/org/opensearch/knn/KNNSingleNodeTestCase.java +++ b/src/test/java/org/opensearch/knn/KNNSingleNodeTestCase.java @@ -16,6 +16,7 @@ import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.xcontent.XContentHelper; import org.opensearch.knn.index.KNNSettings; +import org.opensearch.knn.index.engine.KNNEngine; import org.opensearch.knn.index.query.KNNQueryBuilder; import org.opensearch.knn.index.memory.NativeMemoryCacheManager; import org.opensearch.knn.index.memory.NativeMemoryLoadStrategy; @@ -50,6 +51,7 @@ import static org.mockito.Mockito.when; import static org.opensearch.knn.common.KNNConstants.DIMENSION; import static org.opensearch.knn.common.KNNConstants.KNN_ENGINE; +import static org.opensearch.knn.common.KNNConstants.METHOD_HNSW; import static org.opensearch.knn.common.KNNConstants.METHOD_PARAMETER_SPACE_TYPE; import static org.opensearch.knn.common.KNNConstants.MODEL_BLOB_PARAMETER; import static org.opensearch.knn.common.KNNConstants.MODEL_DESCRIPTION; @@ -109,6 +111,22 @@ protected void createKnnIndexMapping(String indexName, String fieldName, Integer /** * Create simple k-NN mapping with engine */ + protected void createKnnIndexMapping(String indexName, String fieldName, Integer dimensions, KNNEngine engine) throws IOException { + PutMappingRequest request = new PutMappingRequest(indexName); + XContentBuilder xContentBuilder = XContentFactory.jsonBuilder().startObject().startObject("properties"); + xContentBuilder.startObject(fieldName); + xContentBuilder.field("type", "knn_vector").field("dimension", dimensions.toString()); + xContentBuilder.startObject("method"); + xContentBuilder.field("name", METHOD_HNSW); + xContentBuilder.field(KNN_ENGINE, engine.getName()); + xContentBuilder.endObject(); + xContentBuilder.endObject(); + xContentBuilder.endObject(); + xContentBuilder.endObject(); + request.source(xContentBuilder); + OpenSearchAssertions.assertAcked(client().admin().indices().putMapping(request).actionGet()); + } + protected void updateIndexSetting(String indexName, Settings setting) { UpdateSettingsRequest request = new UpdateSettingsRequest(setting, indexName); OpenSearchAssertions.assertAcked(client().admin().indices().updateSettings(request).actionGet()); diff --git a/src/test/java/org/opensearch/knn/index/KNNIndexShardTests.java b/src/test/java/org/opensearch/knn/index/KNNIndexShardTests.java index c25d2a390..d01c3a0b4 100644 --- a/src/test/java/org/opensearch/knn/index/KNNIndexShardTests.java +++ b/src/test/java/org/opensearch/knn/index/KNNIndexShardTests.java @@ -19,6 +19,7 @@ import org.opensearch.index.IndexService; import org.opensearch.index.engine.Engine; import org.opensearch.index.shard.IndexShard; +import org.opensearch.knn.index.engine.KNNEngine; import org.opensearch.knn.index.memory.NativeMemoryCacheManager; import java.io.IOException; @@ -108,7 +109,7 @@ public void testWarmup_shardNotPresentInCache() throws InterruptedException, Exe public void testGetAllEngineFileContexts() throws IOException, ExecutionException, InterruptedException { IndexService indexService = createKNNIndex(testIndexName); - createKnnIndexMapping(testIndexName, testFieldName, dimensions); + createKnnIndexMapping(testIndexName, testFieldName, dimensions, KNNEngine.NMSLIB); updateIndexSetting(testIndexName, Settings.builder().put(KNNSettings.INDEX_KNN_ADVANCED_APPROXIMATE_THRESHOLD, 0).build()); IndexShard indexShard = indexService.iterator().next(); diff --git a/src/test/java/org/opensearch/knn/index/NmslibIT.java b/src/test/java/org/opensearch/knn/index/NmslibIT.java index 495fc55ba..23d35d39e 100644 --- a/src/test/java/org/opensearch/knn/index/NmslibIT.java +++ b/src/test/java/org/opensearch/knn/index/NmslibIT.java @@ -36,6 +36,7 @@ import java.util.TreeMap; import static org.hamcrest.Matchers.containsString; +import static org.opensearch.knn.common.KNNConstants.KNN_ENGINE; public class NmslibIT extends KNNRestTestCase { @@ -281,6 +282,7 @@ public void testCreateIndexWithValidAlgoParams_mapping() { .field("dimension", 2) .startObject(KNNConstants.KNN_METHOD) .field(KNNConstants.METHOD_PARAMETER_SPACE_TYPE, spaceType) + .field(KNN_ENGINE, KNNEngine.NMSLIB.getName()) .field(KNNConstants.NAME, KNNConstants.METHOD_HNSW) .startObject(KNNConstants.PARAMETERS) .field(KNNConstants.METHOD_PARAMETER_EF_CONSTRUCTION, efConstruction) @@ -336,6 +338,7 @@ public void testCreateIndexWithValidAlgoParams_mappingAndSettings() { .field("dimension", 2) .startObject(KNNConstants.KNN_METHOD) .field(KNNConstants.METHOD_PARAMETER_SPACE_TYPE, spaceType1) + .field(KNN_ENGINE, KNNEngine.NMSLIB.getName()) .field(KNNConstants.NAME, KNNConstants.METHOD_HNSW) .startObject(KNNConstants.PARAMETERS) .field(KNNConstants.METHOD_PARAMETER_EF_CONSTRUCTION, efConstruction1) @@ -363,6 +366,7 @@ public void testCreateIndexWithValidAlgoParams_mappingAndSettings() { .field("dimension", 2) .startObject(KNNConstants.KNN_METHOD) .field(KNNConstants.METHOD_PARAMETER_SPACE_TYPE, spaceType1) + .field(KNN_ENGINE, KNNEngine.NMSLIB.getName()) .field(KNNConstants.NAME, KNNConstants.METHOD_HNSW) .startObject(KNNConstants.PARAMETERS) .field(KNNConstants.METHOD_PARAMETER_EF_CONSTRUCTION, efConstruction1) @@ -375,6 +379,7 @@ public void testCreateIndexWithValidAlgoParams_mappingAndSettings() { .field("dimension", 2) .startObject(KNNConstants.KNN_METHOD) .field(KNNConstants.METHOD_PARAMETER_SPACE_TYPE, spaceType2) + .field(KNN_ENGINE, KNNEngine.NMSLIB.getName()) .field(KNNConstants.NAME, KNNConstants.METHOD_HNSW) .startObject(KNNConstants.PARAMETERS) .field(KNNConstants.METHOD_PARAMETER_EF_CONSTRUCTION, efConstruction2) diff --git a/src/test/java/org/opensearch/knn/index/VectorDataTypeIT.java b/src/test/java/org/opensearch/knn/index/VectorDataTypeIT.java index fcfb192ca..2e68339d4 100644 --- a/src/test/java/org/opensearch/knn/index/VectorDataTypeIT.java +++ b/src/test/java/org/opensearch/knn/index/VectorDataTypeIT.java @@ -263,29 +263,6 @@ public void testByteVectorDataTypeWithNmslibEngine() { assertTrue(ex.getMessage().contains("is not supported for vector data type")); } - @SneakyThrows - public void testByteVectorDataTypeWithLegacyFieldMapperKnnIndexSetting() { - // Create an index with byte vector data_type and index.knn as true without setting KnnMethodContext, - // which should throw an exception because the LegacyFieldMapper will use NMSLIB engine and byte data_type - // is not supported for NMSLIB engine. - XContentBuilder builder = XContentFactory.jsonBuilder() - .startObject() - .startObject(PROPERTIES_FIELD) - .startObject(FIELD_NAME) - .field(TYPE_FIELD_NAME, KNN_VECTOR_TYPE) - .field(DIMENSION, 2) - .field(VECTOR_DATA_TYPE_FIELD, VectorDataType.BYTE.getValue()) - .endObject() - .endObject() - .endObject(); - - String mapping = builder.toString(); - - ResponseException ex = expectThrows(ResponseException.class, () -> createKnnIndex(INDEX_NAME, mapping)); - assertTrue(ex.getMessage(), ex.getMessage().contains("is not supported for vector data type")); - - } - public void testDocValuesWithByteVectorDataTypeLuceneEngine() throws Exception { createKnnIndexMappingWithLuceneEngine(2, SpaceType.L2, VectorDataType.BYTE.getValue()); ingestL2ByteTestData(); diff --git a/src/test/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumerTests.java b/src/test/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumerTests.java index e1f16006d..c11bc765f 100644 --- a/src/test/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumerTests.java +++ b/src/test/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumerTests.java @@ -279,6 +279,7 @@ public void testAddKNNBinaryField_fromScratch_nmslibLegacy() throws IOException .addAttribute(KNNConstants.HNSW_ALGO_EF_CONSTRUCTION, "512") .addAttribute(KNNConstants.HNSW_ALGO_M, "16") .addAttribute(KNNConstants.SPACE_TYPE, spaceType.getValue()) + .addAttribute(KNNConstants.KNN_ENGINE, knnEngine.getName()) .build() }; FieldInfos fieldInfos = new FieldInfos(fieldInfoArray); diff --git a/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestCase.java b/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestCase.java index 2036e14aa..315693a65 100644 --- a/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestCase.java +++ b/src/test/java/org/opensearch/knn/index/codec/KNNCodecTestCase.java @@ -103,7 +103,7 @@ public class KNNCodecTestCase extends KNNTestCase { .vectorDataType(VectorDataType.DEFAULT) .build(); KNNMethodContext knnMethodContext = new KNNMethodContext( - KNNEngine.DEFAULT, + KNNEngine.NMSLIB, SpaceType.DEFAULT, new MethodComponentContext(METHOD_HNSW, ImmutableMap.of(METHOD_PARAMETER_M, 16, METHOD_PARAMETER_EF_CONSTRUCTION, 512)) ); diff --git a/src/test/java/org/opensearch/knn/index/engine/EngineResolverTests.java b/src/test/java/org/opensearch/knn/index/engine/EngineResolverTests.java index df195883a..291f0c671 100644 --- a/src/test/java/org/opensearch/knn/index/engine/EngineResolverTests.java +++ b/src/test/java/org/opensearch/knn/index/engine/EngineResolverTests.java @@ -41,10 +41,10 @@ public void testResolveEngine_whenModeAndCompressionAreFalse_thenDefault() { ); } - public void testResolveEngine_whenModeSpecifiedAndCompressionIsNotSpecified_thenDefault() { + public void testResolveEngine_whenModeSpecifiedAndCompressionIsNotSpecified_thenNMSLIB() { assertEquals(KNNEngine.DEFAULT, ENGINE_RESOLVER.resolveEngine(KNNMethodConfigContext.builder().build(), null, false)); assertEquals( - KNNEngine.DEFAULT, + KNNEngine.NMSLIB, ENGINE_RESOLVER.resolveEngine( KNNMethodConfigContext.builder().mode(Mode.IN_MEMORY).build(), new KNNMethodContext(KNNEngine.DEFAULT, SpaceType.UNDEFINED, MethodComponentContext.EMPTY, false), @@ -63,7 +63,7 @@ public void testResolveEngine_whenCompressionIs1x_thenEngineBasedOnMode() { ) ); assertEquals( - KNNEngine.DEFAULT, + KNNEngine.NMSLIB, ENGINE_RESOLVER.resolveEngine(KNNMethodConfigContext.builder().compressionLevel(CompressionLevel.x1).build(), null, false) ); } diff --git a/src/test/java/org/opensearch/knn/index/engine/KNNEngineTests.java b/src/test/java/org/opensearch/knn/index/engine/KNNEngineTests.java index 5a6ed8e52..3f356999c 100644 --- a/src/test/java/org/opensearch/knn/index/engine/KNNEngineTests.java +++ b/src/test/java/org/opensearch/knn/index/engine/KNNEngineTests.java @@ -25,6 +25,10 @@ public void testDelegateLibraryFunctions() { assertEquals(Lucene.INSTANCE.getVersion(), KNNEngine.LUCENE.getVersion()); } + public void testGetDefaultEngine_thenReturnFAISS() { + assertEquals(KNNEngine.FAISS, KNNEngine.DEFAULT); + } + /** * Test name getter */ diff --git a/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java b/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java index 98bbf42ca..714723a8e 100644 --- a/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java +++ b/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java @@ -65,6 +65,7 @@ import java.util.Optional; import java.util.stream.Collectors; +import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doNothing; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; @@ -146,7 +147,7 @@ public void testTypeParser_build_fromKnnMethodContext() throws IOException { // Check that knnMethodContext takes precedent over both model and legacy ModelDao modelDao = mock(ModelDao.class); - SpaceType spaceType = SpaceType.COSINESIMIL; + SpaceType spaceType = SpaceType.DEFAULT; int mRight = 17; int mWrong = 71; @@ -284,36 +285,43 @@ public void testTypeParser_withDifferentSpaceTypeCombinations_thenSuccess() thro ); assertTrue(knnVectorFieldMapper.fieldType().getKnnMappingConfig().getModelId().isEmpty()); - // if space type is provided and legacy mappings is hit - xContentBuilder = XContentFactory.jsonBuilder() - .startObject() - .field(TYPE_FIELD_NAME, KNN_VECTOR_TYPE) - .field(DIMENSION_FIELD_NAME, TEST_DIMENSION) - .field(KNNConstants.TOP_LEVEL_PARAMETER_SPACE_TYPE, topLevelSpaceType.getValue()) - .endObject(); - builder = (KNNVectorFieldMapper.Builder) typeParser.parse( - "test-field-name-1", - xContentBuilderToMap(xContentBuilder), - buildParserContext("test", settings) - ); + // mock useKNNMethodContextFromLegacy to simulate index ix created before 2_18 + try (MockedStatic utilMockedStatic = Mockito.mockStatic(KNNVectorFieldMapper.class)) { + utilMockedStatic.when(() -> KNNVectorFieldMapper.useKNNMethodContextFromLegacy(any(), any())).thenReturn(true); + // if space type is provided and legacy mappings is hit + xContentBuilder = XContentFactory.jsonBuilder() + .startObject() + .field(TYPE_FIELD_NAME, KNN_VECTOR_TYPE) + .field(DIMENSION_FIELD_NAME, TEST_DIMENSION) + .field(KNNConstants.TOP_LEVEL_PARAMETER_SPACE_TYPE, topLevelSpaceType.getValue()) + .endObject(); + builder = (KNNVectorFieldMapper.Builder) typeParser.parse( + "test-field-name-1", + xContentBuilderToMap(xContentBuilder), + buildParserContext("test", settings) + ); - builderContext = new Mapper.BuilderContext(settings, new ContentPath()); - knnVectorFieldMapper = builder.build(builderContext); - assertTrue(knnVectorFieldMapper instanceof MethodFieldMapper); - assertTrue(knnVectorFieldMapper.fieldType().getKnnMappingConfig().getKnnMethodContext().isPresent()); - assertEquals(topLevelSpaceType, knnVectorFieldMapper.fieldType().getKnnMappingConfig().getKnnMethodContext().get().getSpaceType()); - // this check ensures that legacy mapping is hit, as in legacy mapping we pick M from index settings - assertEquals( - mForSetting, - knnVectorFieldMapper.fieldType() - .getKnnMappingConfig() - .getKnnMethodContext() - .get() - .getMethodComponentContext() - .getParameters() - .get(METHOD_PARAMETER_M) - ); - assertTrue(knnVectorFieldMapper.fieldType().getKnnMappingConfig().getModelId().isEmpty()); + builderContext = new Mapper.BuilderContext(settings, new ContentPath()); + knnVectorFieldMapper = builder.build(builderContext); + assertTrue(knnVectorFieldMapper instanceof MethodFieldMapper); + assertTrue(knnVectorFieldMapper.fieldType().getKnnMappingConfig().getKnnMethodContext().isPresent()); + assertEquals( + topLevelSpaceType, + knnVectorFieldMapper.fieldType().getKnnMappingConfig().getKnnMethodContext().get().getSpaceType() + ); + // this check ensures that legacy mapping is hit, as in legacy mapping we pick M from index settings + assertEquals( + mForSetting, + knnVectorFieldMapper.fieldType() + .getKnnMappingConfig() + .getKnnMethodContext() + .get() + .getMethodComponentContext() + .getParameters() + .get(METHOD_PARAMETER_M) + ); + assertTrue(knnVectorFieldMapper.fieldType().getKnnMappingConfig().getModelId().isEmpty()); + } } public void testTypeParser_withSpaceTypeAndMode_thenSuccess() throws IOException { @@ -1614,7 +1622,7 @@ public void testBuilder_whenBinaryWithLegacyKNNDisabled_thenValid() { assertTrue(knnVectorFieldMapper instanceof FlatVectorFieldMapper); } - public void testTypeParser_whenBinaryWithLegacyKNNEnabled_thenException() throws IOException { + public void testTypeParser_whenBinaryWithLegacyKNNEnabled_thenValid() throws IOException { // Check legacy is picked up if model context and method context are not set ModelDao modelDao = mock(ModelDao.class); KNNVectorFieldMapper.TypeParser typeParser = new KNNVectorFieldMapper.TypeParser(() -> modelDao); @@ -1631,11 +1639,12 @@ public void testTypeParser_whenBinaryWithLegacyKNNEnabled_thenException() throws .field(VECTOR_DATA_TYPE_FIELD, VectorDataType.BINARY.getValue()) .endObject(); - Exception ex = expectThrows(Exception.class, () -> { - typeParser.parse(fieldName, xContentBuilderToMap(xContentBuilder), buildParserContext(indexName, settings)); - }); - - assertTrue(ex.getMessage(), ex.getMessage().contains("does not support space type")); + final KNNVectorFieldMapper.Builder builder = (KNNVectorFieldMapper.Builder) typeParser.parse( + fieldName, + xContentBuilderToMap(xContentBuilder), + buildParserContext(indexName, settings) + ); + assertEquals(SpaceType.HAMMING, builder.getOriginalParameters().getResolvedKnnMethodContext().getSpaceType()); } public void testBuild_whenInvalidCharsInFieldName_thenThrowException() { @@ -1661,7 +1670,7 @@ public void testTypeParser_whenModeAndCompressionAreSet_thenHandle() throws IOEx ModelDao modelDao = mock(ModelDao.class); KNNVectorFieldMapper.TypeParser typeParser = new KNNVectorFieldMapper.TypeParser(() -> modelDao); - // Default to nmslib and ensure legacy is in use + // Default to faiss and ensure legacy is in use XContentBuilder xContentBuilder = XContentFactory.jsonBuilder() .startObject() .field(TYPE_FIELD_NAME, KNN_VECTOR_TYPE) @@ -1676,7 +1685,7 @@ public void testTypeParser_whenModeAndCompressionAreSet_thenHandle() throws IOEx assertTrue(builder.getOriginalParameters().isLegacyMapping()); validateBuilderAfterParsing( builder, - KNNEngine.NMSLIB, + KNNEngine.DEFAULT, SpaceType.L2, VectorDataType.FLOAT, CompressionLevel.x1, diff --git a/src/test/java/org/opensearch/knn/index/query/KNNWeightTests.java b/src/test/java/org/opensearch/knn/index/query/KNNWeightTests.java index 482e7e0cb..511895026 100644 --- a/src/test/java/org/opensearch/knn/index/query/KNNWeightTests.java +++ b/src/test/java/org/opensearch/knn/index/query/KNNWeightTests.java @@ -108,6 +108,7 @@ public class KNNWeightTests extends KNNTestCase { private static final int K = 5; private static final Set SEGMENT_FILES_NMSLIB = Set.of("_0.cfe", "_0_2011_target_field.hnswc"); private static final Set SEGMENT_FILES_FAISS = Set.of("_0.cfe", "_0_2011_target_field.faissc"); + private static final Set SEGMENT_FILES_DEFAULT = SEGMENT_FILES_FAISS; private static final Set SEGMENT_MULTI_FIELD_FILES_FAISS = Set.of( "_0.cfe", "_0_2011_target_field.faissc", @@ -174,7 +175,11 @@ public void tearDownAfterTest() { @SneakyThrows public void testQueryResultScoreNmslib() { for (SpaceType space : List.of(SpaceType.L2, SpaceType.L1, SpaceType.COSINESIMIL, SpaceType.INNER_PRODUCT, SpaceType.LINF)) { - testQueryScore(space::scoreTranslation, SEGMENT_FILES_NMSLIB, Map.of(SPACE_TYPE, space.getValue())); + testQueryScore( + space::scoreTranslation, + SEGMENT_FILES_NMSLIB, + Map.of(SPACE_TYPE, space.getValue(), KNN_ENGINE, KNNEngine.NMSLIB.getName()) + ); } } @@ -398,7 +403,7 @@ public void testEmptyQueryResults() { Map.of(), Sort.RELEVANCE ); - segmentInfo.setFiles(SEGMENT_FILES_NMSLIB); + segmentInfo.setFiles(SEGMENT_FILES_DEFAULT); final SegmentCommitInfo segmentCommitInfo = new SegmentCommitInfo(segmentInfo, 0, 0, 0, 0, 0, new byte[StringHelper.ID_LENGTH]); when(reader.getSegmentInfo()).thenReturn(segmentCommitInfo); diff --git a/src/test/java/org/opensearch/knn/jni/JNIServiceTests.java b/src/test/java/org/opensearch/knn/jni/JNIServiceTests.java index 53a78b381..e116ef3c6 100644 --- a/src/test/java/org/opensearch/knn/jni/JNIServiceTests.java +++ b/src/test/java/org/opensearch/knn/jni/JNIServiceTests.java @@ -933,7 +933,7 @@ public void testLoadIndex_nmslib_valid_with_stream() throws IOException { testData.indexData.getDimension(), directory, indexFileName1, - ImmutableMap.of(KNNConstants.SPACE_TYPE, SpaceType.L2.getValue()), + ImmutableMap.of(KNNConstants.SPACE_TYPE, SpaceType.L2.getValue(), KNN_ENGINE, KNNEngine.NMSLIB), KNNEngine.NMSLIB ); assertTrue(directory.fileLength(indexFileName1) > 0); diff --git a/src/test/java/org/opensearch/knn/plugin/transport/GetModelResponseTests.java b/src/test/java/org/opensearch/knn/plugin/transport/GetModelResponseTests.java index 3bf5f302c..771b07edd 100644 --- a/src/test/java/org/opensearch/knn/plugin/transport/GetModelResponseTests.java +++ b/src/test/java/org/opensearch/knn/plugin/transport/GetModelResponseTests.java @@ -30,6 +30,7 @@ import org.opensearch.knn.indices.ModelState; import java.io.IOException; +import java.util.Locale; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.mockStatic; @@ -76,7 +77,9 @@ public void testXContent() throws IOException { Model model = new Model(getModelMetadata(ModelState.CREATED), testModelBlob, modelId); GetModelResponse getModelResponse = new GetModelResponse(model); String expectedResponseString = String.format( - "{\"model_id\":\"test-model\",\"model_blob\":\"aGVsbG8=\",\"state\":\"created\",\"timestamp\":\"2021-03-27 10:15:30 AM +05:30\",\"description\":\"test model\",\"error\":\"\",\"space_type\":\"l2\",\"dimension\":4,\"engine\":\"nmslib\",\"training_node_assignment\":\"\",\"model_definition\":{\"name\":\"\",\"parameters\":{}},\"data_type\":\"float\",\"model_version\":\"%s\"}", + Locale.ROOT, + "{\"model_id\":\"test-model\",\"model_blob\":\"aGVsbG8=\",\"state\":\"created\",\"timestamp\":\"2021-03-27 10:15:30 AM +05:30\",\"description\":\"test model\",\"error\":\"\",\"space_type\":\"l2\",\"dimension\":4,\"engine\":\"%s\",\"training_node_assignment\":\"\",\"model_definition\":{\"name\":\"\",\"parameters\":{}},\"data_type\":\"float\",\"model_version\":\"%s\"}", + KNNEngine.DEFAULT.getName(), Version.CURRENT.toString() ); XContentBuilder xContentBuilder = MediaTypeRegistry.contentBuilder(XContentType.JSON); @@ -94,7 +97,9 @@ public void testXContentWithNoModelBlob() throws IOException { Model model = new Model(getModelMetadata(ModelState.FAILED), null, modelId); GetModelResponse getModelResponse = new GetModelResponse(model); String expectedResponseString = String.format( - "{\"model_id\":\"test-model\",\"model_blob\":\"\",\"state\":\"failed\",\"timestamp\":\"2021-03-27 10:15:30 AM +05:30\",\"description\":\"test model\",\"error\":\"\",\"space_type\":\"l2\",\"dimension\":4,\"engine\":\"nmslib\",\"training_node_assignment\":\"\",\"model_definition\":{\"name\":\"\",\"parameters\":{}},\"data_type\":\"float\",\"model_version\":\"%s\"}", + Locale.ROOT, + "{\"model_id\":\"test-model\",\"model_blob\":\"\",\"state\":\"failed\",\"timestamp\":\"2021-03-27 10:15:30 AM +05:30\",\"description\":\"test model\",\"error\":\"\",\"space_type\":\"l2\",\"dimension\":4,\"engine\":\"%s\",\"training_node_assignment\":\"\",\"model_definition\":{\"name\":\"\",\"parameters\":{}},\"data_type\":\"float\",\"model_version\":\"%s\"}", + KNNEngine.DEFAULT.getName(), Version.CURRENT.toString() ); XContentBuilder xContentBuilder = MediaTypeRegistry.contentBuilder(XContentType.JSON); diff --git a/src/test/java/org/opensearch/knn/plugin/transport/TrainingModelRequestTests.java b/src/test/java/org/opensearch/knn/plugin/transport/TrainingModelRequestTests.java index 2c423e0ef..6fd399434 100644 --- a/src/test/java/org/opensearch/knn/plugin/transport/TrainingModelRequestTests.java +++ b/src/test/java/org/opensearch/knn/plugin/transport/TrainingModelRequestTests.java @@ -46,6 +46,7 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; +import static org.opensearch.knn.common.KNNConstants.METHOD_HNSW; public class TrainingModelRequestTests extends KNNTestCase { @@ -290,11 +291,14 @@ public void testValidation_invalid_invalidMethodContext() { String trainingIndex = "test-training-index"; String trainingField = "test-training-field"; + MethodComponentContext methodComponentContext = new MethodComponentContext(METHOD_HNSW, Collections.emptyMap()); + final KNNMethodContext knnMethodContext = new KNNMethodContext(KNNEngine.NMSLIB, SpaceType.DEFAULT, methodComponentContext); + ValidationException validationException = expectThrows( ValidationException.class, () -> new TrainingModelRequest( modelId, - getDefaultKNNMethodContext(), + knnMethodContext, dimension, trainingIndex, trainingField, From 55561178b5844ee7898d27c4204a8aebac8e047a Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Fri, 8 Nov 2024 11:49:44 -0800 Subject: [PATCH 413/416] Bump up C++ version in JNI from c++11 to c++17. (#2259) (#2261) Signed-off-by: Dooyong Kim Co-authored-by: Dooyong Kim (cherry picked from commit a07bad1122a44abe5dd37d15a7e783523cac4ea4) Co-authored-by: Doo Yong Kim <0ctopus13prime@gmail.com> --- CHANGELOG.md | 1 + jni/CMakeLists.txt | 2 +- .../knn/index/memory/NativeMemoryCacheManagerTests.java | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3abe09f20..d730edc92 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Introduced a writing layer in native engines where relies on the writing interface to process IO. (#2241)[https://github.com/opensearch-project/k-NN/pull/2241] ### Bug Fixes ### Infrastructure +* Updated C++ version in JNI from c++11 to c++17 [#2259](https://github.com/opensearch-project/k-NN/pull/2259) ### Documentation ### Maintenance * Select index settings based on cluster version[2236](https://github.com/opensearch-project/k-NN/pull/2236) diff --git a/jni/CMakeLists.txt b/jni/CMakeLists.txt index 1920453c7..6253eed15 100644 --- a/jni/CMakeLists.txt +++ b/jni/CMakeLists.txt @@ -18,7 +18,7 @@ set(TARGET_LIB_NMSLIB opensearchknn_nmslib) # nmslib JNI set(TARGET_LIB_FAISS opensearchknn_faiss) # faiss JNI set(TARGET_LIBS "") # Libs to be installed -set(CMAKE_CXX_STANDARD 11) +set(CMAKE_CXX_STANDARD 17) set(CMAKE_CXX_STANDARD_REQUIRED True) option(CONFIG_FAISS "Configure faiss library build when this is on") option(CONFIG_NMSLIB "Configure nmslib library build when this is on") diff --git a/src/test/java/org/opensearch/knn/index/memory/NativeMemoryCacheManagerTests.java b/src/test/java/org/opensearch/knn/index/memory/NativeMemoryCacheManagerTests.java index 4baf66cb4..5fe41c88c 100644 --- a/src/test/java/org/opensearch/knn/index/memory/NativeMemoryCacheManagerTests.java +++ b/src/test/java/org/opensearch/knn/index/memory/NativeMemoryCacheManagerTests.java @@ -197,7 +197,7 @@ public void testGetTrainingSize() throws ExecutionException { nativeMemoryCacheManager.get(trainingDataEntryContext, true); - assertEquals((float) allocationEntryWeight, nativeMemoryCacheManager.getTrainingSizeInKilobytes(), 0.001); + assertEquals(allocationEntryWeight, nativeMemoryCacheManager.getTrainingSizeInKilobytes(), 1e-3); assertEquals( 100 * (float) allocationEntryWeight / (float) maxWeight, nativeMemoryCacheManager.getTrainingSizeAsPercentage(), From 8676685ee78bcd27f201f0ee35300acea81b4c2a Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Mon, 18 Nov 2024 15:10:53 -0800 Subject: [PATCH 414/416] Upgrade bytebuddy and objenesis version to match OpenSearch core and, update github ci runner for macos (#2279) (#2280) * Upgrade bytebuddy and objenesis version to match OpenSearch core Signed-off-by: Vijayan Balasubramanian * update github runner to macos-13 macos-12 is deprecated, upgrading to macos 13 Signed-off-by: Vijayan Balasubramanian * Install libomp before running build Signed-off-by: Vijayan Balasubramanian --------- Signed-off-by: Vijayan Balasubramanian (cherry picked from commit 499273660065403e13e9781e3b9a9931b59f1bf6) Co-authored-by: Vijayan Balasubramanian --- .github/workflows/CI.yml | 3 ++- CHANGELOG.md | 1 + build.gradle | 4 ++-- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index fda1e5e0f..c37596a1f 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -99,7 +99,7 @@ jobs: name: Build and Test k-NN Plugin on MacOS needs: Get-CI-Image-Tag - runs-on: macos-12 + runs-on: macos-13 steps: - name: Checkout k-NN @@ -118,6 +118,7 @@ jobs: - name: Install dependencies on macos run: | + brew install libomp brew reinstall gcc export FC=/usr/local/Cellar/gcc/12.2.0/bin/gfortran diff --git a/CHANGELOG.md b/CHANGELOG.md index d730edc92..2dd1a1b99 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Bug Fixes ### Infrastructure * Updated C++ version in JNI from c++11 to c++17 [#2259](https://github.com/opensearch-project/k-NN/pull/2259) +* Upgrade bytebuddy and objenesis version to match OpenSearch core and, update github ci runner for macos [#2279](https://github.com/opensearch-project/k-NN/pull/2279) ### Documentation ### Maintenance * Select index settings based on cluster version[2236](https://github.com/opensearch-project/k-NN/pull/2236) diff --git a/build.gradle b/build.gradle index a71af5f08..ead0cba08 100644 --- a/build.gradle +++ b/build.gradle @@ -295,8 +295,8 @@ dependencies { api group: 'com.google.guava', name: 'guava', version:'32.1.3-jre' api group: 'commons-lang', name: 'commons-lang', version: '2.6' testFixturesImplementation "org.opensearch.test:framework:${opensearch_version}" - testImplementation group: 'net.bytebuddy', name: 'byte-buddy', version: '1.15.4' - testImplementation group: 'org.objenesis', name: 'objenesis', version: '3.2' + testImplementation group: 'net.bytebuddy', name: 'byte-buddy', version: '1.15.10' + testImplementation group: 'org.objenesis', name: 'objenesis', version: '3.3' testImplementation group: 'net.bytebuddy', name: 'byte-buddy-agent', version: '1.15.4' testFixturesImplementation "org.opensearch:common-utils:${version}" implementation 'com.github.oshi:oshi-core:6.4.13' From 67bd9b737ca2976f2f39ed2754cc82cca9f0171f Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Wed, 20 Nov 2024 13:13:40 -0800 Subject: [PATCH 415/416] Fixing the bug when a segment has no vector field present for disk based vector search (#2281) (#2282) Signed-off-by: Navneet Verma (cherry picked from commit 2d1a4080d5b1601bf3362fecd85384348af1f326) Co-authored-by: Navneet Verma --- CHANGELOG.md | 1 + .../knn/index/query/ResultUtil.java | 10 +++--- .../nativelib/NativeEngineKnnVectorQuery.java | 8 ++++- .../knn/index/query/ResultUtilTests.java | 9 +++++ .../NativeEngineKNNVectorQueryTests.java | 5 ++- .../knn/integ/ModeAndCompressionIT.java | 36 +++++++++++++++++++ .../org/opensearch/knn/KNNRestTestCase.java | 13 +++++++ 7 files changed, 75 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2dd1a1b99..dadf0d19c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Enhancements - Introduced a writing layer in native engines where relies on the writing interface to process IO. (#2241)[https://github.com/opensearch-project/k-NN/pull/2241] ### Bug Fixes +* Fix NPE in ANN search when a segment doesn't contain vector field (#2278)[https://github.com/opensearch-project/k-NN/pull/2278] ### Infrastructure * Updated C++ version in JNI from c++11 to c++17 [#2259](https://github.com/opensearch-project/k-NN/pull/2259) * Upgrade bytebuddy and objenesis version to match OpenSearch core and, update github ci runner for macos [#2279](https://github.com/opensearch-project/k-NN/pull/2279) diff --git a/src/main/java/org/opensearch/knn/index/query/ResultUtil.java b/src/main/java/org/opensearch/knn/index/query/ResultUtil.java index f62c09cb0..df1ce3827 100644 --- a/src/main/java/org/opensearch/knn/index/query/ResultUtil.java +++ b/src/main/java/org/opensearch/knn/index/query/ResultUtil.java @@ -58,17 +58,17 @@ public static void reduceToTopK(List> perLeafResults, int k) } /** - * Convert map to bit set + * Convert map to bit set, if resultMap is empty or null then returns an Optional. Returning an optional here to + * ensure that the caller is aware that BitSet may not be present * * @param resultMap Map of results - * @return BitSet of results + * @return BitSet of results; null is returned if the result map is empty * @throws IOException If an error occurs during the search. */ public static BitSet resultMapToMatchBitSet(Map resultMap) throws IOException { - if (resultMap.isEmpty()) { - return BitSet.of(DocIdSetIterator.empty(), 0); + if (resultMap == null || resultMap.isEmpty()) { + return null; } - final int maxDoc = Collections.max(resultMap.keySet()) + 1; return BitSet.of(resultMapToDocIds(resultMap, maxDoc), maxDoc); } diff --git a/src/main/java/org/opensearch/knn/index/query/nativelib/NativeEngineKnnVectorQuery.java b/src/main/java/org/opensearch/knn/index/query/nativelib/NativeEngineKnnVectorQuery.java index 8b861b430..bf12e63c0 100644 --- a/src/main/java/org/opensearch/knn/index/query/nativelib/NativeEngineKnnVectorQuery.java +++ b/src/main/java/org/opensearch/knn/index/query/nativelib/NativeEngineKnnVectorQuery.java @@ -30,6 +30,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.Comparator; import java.util.List; import java.util.Map; @@ -112,7 +113,12 @@ private List> doRescore( LeafReaderContext leafReaderContext = leafReaderContexts.get(i); int finalI = i; rescoreTasks.add(() -> { - BitSet convertedBitSet = ResultUtil.resultMapToMatchBitSet(perLeafResults.get(finalI)); + final BitSet convertedBitSet = ResultUtil.resultMapToMatchBitSet(perLeafResults.get(finalI)); + // if there is no docIds to re-score from a segment we should return early to ensure that we are not + // wasting any computation + if (convertedBitSet == null) { + return Collections.emptyMap(); + } final ExactSearcher.ExactSearcherContext exactSearcherContext = ExactSearcher.ExactSearcherContext.builder() .matchedDocs(convertedBitSet) // setting to false because in re-scoring we want to do exact search on full precision vectors diff --git a/src/test/java/org/opensearch/knn/index/query/ResultUtilTests.java b/src/test/java/org/opensearch/knn/index/query/ResultUtilTests.java index 70cb86e02..7cda1ed79 100644 --- a/src/test/java/org/opensearch/knn/index/query/ResultUtilTests.java +++ b/src/test/java/org/opensearch/knn/index/query/ResultUtilTests.java @@ -9,6 +9,7 @@ import org.apache.lucene.search.ScoreDoc; import org.apache.lucene.search.TopDocs; import org.apache.lucene.util.BitSet; +import org.junit.Assert; import org.opensearch.knn.KNNTestCase; import java.io.IOException; @@ -48,6 +49,14 @@ public void testResultMapToMatchBitSet() throws IOException { assertResultMapToMatchBitSet(perLeafResults, resultBitset); } + public void testResultMapToMatchBitSet_whenResultMapEmpty_thenReturnEmptyOptional() throws IOException { + BitSet resultBitset = ResultUtil.resultMapToMatchBitSet(Collections.emptyMap()); + Assert.assertNull(resultBitset); + + BitSet resultBitset2 = ResultUtil.resultMapToMatchBitSet(null); + Assert.assertNull(resultBitset2); + } + public void testResultMapToDocIds() throws IOException { int firstPassK = 42; Map perLeafResults = getRandomResults(firstPassK); diff --git a/src/test/java/org/opensearch/knn/index/query/nativelib/NativeEngineKNNVectorQueryTests.java b/src/test/java/org/opensearch/knn/index/query/nativelib/NativeEngineKNNVectorQueryTests.java index 53873e15f..fab67189d 100644 --- a/src/test/java/org/opensearch/knn/index/query/nativelib/NativeEngineKNNVectorQueryTests.java +++ b/src/test/java/org/opensearch/knn/index/query/nativelib/NativeEngineKNNVectorQueryTests.java @@ -227,7 +227,7 @@ public void testRescore() { when(reader.leaves()).thenReturn(leaves); int k = 2; - int firstPassK = 3; + int firstPassK = 100; Map initialLeaf1Results = new HashMap<>(Map.of(0, 21f, 1, 19f, 2, 17f, 3, 15f)); Map initialLeaf2Results = new HashMap<>(Map.of(0, 20f, 1, 18f, 2, 16f, 3, 14f)); Map rescoredLeaf1Results = new HashMap<>(Map.of(0, 18f, 1, 20f)); @@ -255,6 +255,9 @@ public void testRescore() { mockedKnnSettings.when(() -> KNNSettings.isShardLevelRescoringEnabledForDiskBasedVector(any())).thenReturn(true); mockedResultUtil.when(() -> ResultUtil.reduceToTopK(any(), anyInt())).thenAnswer(InvocationOnMock::callRealMethod); + mockedResultUtil.when(() -> ResultUtil.resultMapToMatchBitSet(any())).thenAnswer(InvocationOnMock::callRealMethod); + mockedResultUtil.when(() -> ResultUtil.resultMapToDocIds(any(), anyInt())).thenAnswer(InvocationOnMock::callRealMethod); + mockedResultUtil.when(() -> ResultUtil.resultMapToTopDocs(eq(rescoredLeaf1Results), anyInt())).thenAnswer(t -> topDocs1); mockedResultUtil.when(() -> ResultUtil.resultMapToTopDocs(eq(rescoredLeaf2Results), anyInt())).thenAnswer(t -> topDocs2); try (MockedStatic mockedStaticNativeKnnVectorQuery = mockStatic(NativeEngineKnnVectorQuery.class)) { diff --git a/src/test/java/org/opensearch/knn/integ/ModeAndCompressionIT.java b/src/test/java/org/opensearch/knn/integ/ModeAndCompressionIT.java index 0903b3c41..91ee89aeb 100644 --- a/src/test/java/org/opensearch/knn/integ/ModeAndCompressionIT.java +++ b/src/test/java/org/opensearch/knn/integ/ModeAndCompressionIT.java @@ -11,6 +11,7 @@ import org.opensearch.client.Request; import org.opensearch.client.Response; import org.opensearch.client.ResponseException; +import org.opensearch.common.settings.Settings; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.rest.RestStatus; import org.opensearch.core.xcontent.XContentBuilder; @@ -220,6 +221,41 @@ public void testDeletedDocsWithSegmentMerge_whenValid_ThenSucceed() { validateGreenIndex(indexName); } + @SneakyThrows + public void testCompressionIndexWithNonVectorFieldsSegment_whenValid_ThenSucceed() { + CompressionLevel compressionLevel = CompressionLevel.x32; + String indexName = INDEX_NAME + compressionLevel; + try ( + XContentBuilder builder = XContentFactory.jsonBuilder() + .startObject() + .startObject("properties") + .startObject(FIELD_NAME) + .field("type", "knn_vector") + .field("dimension", DIMENSION) + .field(COMPRESSION_LEVEL_PARAMETER, compressionLevel.getName()) + .field(MODE_PARAMETER, Mode.ON_DISK.getName()) + .endObject() + .endObject() + .endObject() + ) { + String mapping = builder.toString(); + Settings indexSettings = buildKNNIndexSettings(0); + createKnnIndex(indexName, indexSettings, mapping); + // since we are going to delete a document, so its better to have 1 more extra doc so that we can re-use some tests + addKNNDocs(indexName, FIELD_NAME, DIMENSION, 0, NUM_DOCS + 1); + addNonKNNDoc(indexName, String.valueOf(NUM_DOCS + 2), FIELD_NAME_NON_KNN, "Hello world"); + deleteKnnDoc(indexName, "0"); + validateGreenIndex(indexName); + validateSearch( + indexName, + METHOD_PARAMETER_EF_SEARCH, + KNNSettings.INDEX_KNN_DEFAULT_ALGO_PARAM_EF_SEARCH, + compressionLevel.getName(), + Mode.ON_DISK.getName() + ); + } + } + @SneakyThrows public void testTraining_whenInvalid_thenFail() { setupTrainingIndex(); diff --git a/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java b/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java index 3cef7f5b7..03a99c0bd 100644 --- a/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java +++ b/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java @@ -116,6 +116,7 @@ public class KNNRestTestCase extends ODFERestTestCase { public static final String INDEX_NAME = "test_index"; public static final String FIELD_NAME = "test_field"; + public static final String FIELD_NAME_NON_KNN = "test_field_non_knn"; public static final String PROPERTIES_FIELD = "properties"; public static final String STORE_FIELD = "store"; public static final String STORED_QUERY_FIELD = "stored_fields"; @@ -607,6 +608,18 @@ protected void addKnnDoc(String index, String docId, String fieldName, T vec assertEquals(request.getEndpoint() + ": failed", RestStatus.OK, RestStatus.fromCode(response.getStatusLine().getStatusCode())); } + protected void addNonKNNDoc(String index, String docId, String fieldName, String text) throws IOException { + Request request = new Request("POST", "/" + index + "/_doc/" + docId + "?refresh=true"); + + XContentBuilder builder = XContentFactory.jsonBuilder().startObject().field(fieldName, text).endObject(); + request.setJsonEntity(builder.toString()); + client().performRequest(request); + + request = new Request("POST", "/" + index + "/_refresh"); + Response response = client().performRequest(request); + assertEquals(request.getEndpoint() + ": failed", RestStatus.OK, RestStatus.fromCode(response.getStatusLine().getStatusCode())); + } + /** * Add a single KNN Doc to an index with a nested vector field * From 4e1a8f6773b739502bf16d7478ecd900418d5531 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Fri, 22 Nov 2024 00:34:15 -0800 Subject: [PATCH 416/416] Added null checks for fieldInfo in ExactSearcher to avoid NPE while running exact search for segments with no vector field (#2278) (#2285) Signed-off-by: Navneet Verma (cherry picked from commit 7523cc317d5575c74b82f315f271d63668c8f623) Co-authored-by: Navneet Verma --- CHANGELOG.md | 3 +- .../knn/common/FieldInfoExtractor.java | 15 ++++- .../knn/index/KNNVectorDVLeafFieldData.java | 3 +- .../knn/index/query/ExactSearcher.java | 22 ++++++-- .../opensearch/knn/index/query/KNNWeight.java | 4 +- .../knn/common/FieldInfoExtractorTests.java | 13 +++++ .../knn/index/query/ExactSearcherTests.java | 55 +++++++++++++++++++ 7 files changed, 105 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dadf0d19c..a39546843 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,11 +18,12 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Enhancements - Introduced a writing layer in native engines where relies on the writing interface to process IO. (#2241)[https://github.com/opensearch-project/k-NN/pull/2241] ### Bug Fixes -* Fix NPE in ANN search when a segment doesn't contain vector field (#2278)[https://github.com/opensearch-project/k-NN/pull/2278] +* Fixing the bug when a segment has no vector field present for disk based vector search (#2282)[https://github.com/opensearch-project/k-NN/pull/2282] ### Infrastructure * Updated C++ version in JNI from c++11 to c++17 [#2259](https://github.com/opensearch-project/k-NN/pull/2259) * Upgrade bytebuddy and objenesis version to match OpenSearch core and, update github ci runner for macos [#2279](https://github.com/opensearch-project/k-NN/pull/2279) ### Documentation ### Maintenance * Select index settings based on cluster version[2236](https://github.com/opensearch-project/k-NN/pull/2236) +* Added null checks for fieldInfo in ExactSearcher to avoid NPE while running exact search for segments with no vector field (#2278)[https://github.com/opensearch-project/k-NN/pull/2278] ### Refactoring diff --git a/src/main/java/org/opensearch/knn/common/FieldInfoExtractor.java b/src/main/java/org/opensearch/knn/common/FieldInfoExtractor.java index 4ded68237..16bf0fb54 100644 --- a/src/main/java/org/opensearch/knn/common/FieldInfoExtractor.java +++ b/src/main/java/org/opensearch/knn/common/FieldInfoExtractor.java @@ -8,6 +8,8 @@ import lombok.experimental.UtilityClass; import org.apache.commons.lang.StringUtils; import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.LeafReader; +import org.opensearch.common.Nullable; import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.engine.KNNEngine; @@ -28,7 +30,7 @@ import static org.opensearch.knn.common.KNNConstants.SPACE_TYPE; /** - * A utility class to extract information from FieldInfo. + * A utility class to extract information from FieldInfo and also provides utility functions to extract fieldInfo */ @UtilityClass public class FieldInfoExtractor { @@ -104,4 +106,15 @@ public static SpaceType getSpaceType(final ModelDao modelDao, final FieldInfo fi } return modelMetadata.getSpaceType(); } + + /** + * Get the field info for the given field name, do a null check on the fieldInfo, as this function can return null, + * if the field is not found. + * @param leafReader {@link LeafReader} + * @param fieldName {@link String} + * @return {@link FieldInfo} + */ + public static @Nullable FieldInfo getFieldInfo(final LeafReader leafReader, final String fieldName) { + return leafReader.getFieldInfos().fieldInfo(fieldName); + } } diff --git a/src/main/java/org/opensearch/knn/index/KNNVectorDVLeafFieldData.java b/src/main/java/org/opensearch/knn/index/KNNVectorDVLeafFieldData.java index 85f037c0f..7053e6151 100644 --- a/src/main/java/org/opensearch/knn/index/KNNVectorDVLeafFieldData.java +++ b/src/main/java/org/opensearch/knn/index/KNNVectorDVLeafFieldData.java @@ -12,6 +12,7 @@ import org.opensearch.index.fielddata.LeafFieldData; import org.opensearch.index.fielddata.ScriptDocValues; import org.opensearch.index.fielddata.SortedBinaryDocValues; +import org.opensearch.knn.common.FieldInfoExtractor; import java.io.IOException; @@ -40,7 +41,7 @@ public long ramBytesUsed() { @Override public ScriptDocValues getScriptValues() { try { - FieldInfo fieldInfo = reader.getFieldInfos().fieldInfo(fieldName); + FieldInfo fieldInfo = FieldInfoExtractor.getFieldInfo(reader, fieldName); if (fieldInfo == null) { return KNNVectorScriptDocValues.emptyValues(fieldName, vectorDataType); } diff --git a/src/main/java/org/opensearch/knn/index/query/ExactSearcher.java b/src/main/java/org/opensearch/knn/index/query/ExactSearcher.java index 77e993297..6a97b4083 100644 --- a/src/main/java/org/opensearch/knn/index/query/ExactSearcher.java +++ b/src/main/java/org/opensearch/knn/index/query/ExactSearcher.java @@ -38,6 +38,7 @@ import org.opensearch.knn.indices.ModelDao; import java.io.IOException; +import java.util.Collections; import java.util.HashMap; import java.util.Locale; import java.util.Map; @@ -59,7 +60,11 @@ public class ExactSearcher { */ public Map searchLeaf(final LeafReaderContext leafReaderContext, final ExactSearcherContext exactSearcherContext) throws IOException { - KNNIterator iterator = getKNNIterator(leafReaderContext, exactSearcherContext); + final KNNIterator iterator = getKNNIterator(leafReaderContext, exactSearcherContext); + // because of any reason if we are not able to get KNNIterator, return an empty map + if (iterator == null) { + return Collections.emptyMap(); + } if (exactSearcherContext.getKnnQuery().getRadius() != null) { return doRadialSearch(leafReaderContext, exactSearcherContext, iterator); } @@ -74,8 +79,8 @@ public Map searchLeaf(final LeafReaderContext leafReaderContext, * Perform radial search by comparing scores with min score. Currently, FAISS from native engine supports radial search. * Hence, we assume that Radius from knnQuery is always distance, and we convert it to score since we do exact search uses scores * to filter out the documents that does not have given min score. - * @param leafReaderContext - * @param exactSearcherContext + * @param leafReaderContext {@link LeafReaderContext} + * @param exactSearcherContext {@link ExactSearcherContext} * @param iterator {@link KNNIterator} * @return Map of docId and score * @throws IOException exception raised by iterator during traversal @@ -87,7 +92,10 @@ private Map doRadialSearch( ) throws IOException { final SegmentReader reader = Lucene.segmentReader(leafReaderContext.reader()); final KNNQuery knnQuery = exactSearcherContext.getKnnQuery(); - final FieldInfo fieldInfo = reader.getFieldInfos().fieldInfo(knnQuery.getField()); + final FieldInfo fieldInfo = FieldInfoExtractor.getFieldInfo(reader, knnQuery.getField()); + if (fieldInfo == null) { + return Collections.emptyMap(); + } final KNNEngine engine = FieldInfoExtractor.extractKNNEngine(fieldInfo); if (KNNEngine.FAISS != engine) { throw new IllegalArgumentException(String.format(Locale.ROOT, "Engine [%s] does not support radial search", engine)); @@ -149,7 +157,11 @@ private KNNIterator getKNNIterator(LeafReaderContext leafReaderContext, ExactSea final KNNQuery knnQuery = exactSearcherContext.getKnnQuery(); final BitSet matchedDocs = exactSearcherContext.getMatchedDocs(); final SegmentReader reader = Lucene.segmentReader(leafReaderContext.reader()); - final FieldInfo fieldInfo = reader.getFieldInfos().fieldInfo(knnQuery.getField()); + final FieldInfo fieldInfo = FieldInfoExtractor.getFieldInfo(reader, knnQuery.getField()); + if (fieldInfo == null) { + log.debug("[KNN] Cannot get KNNIterator as Field info not found for {}:{}", knnQuery.getField(), reader.getSegmentName()); + return null; + } final SpaceType spaceType = FieldInfoExtractor.getSpaceType(modelDao, fieldInfo); boolean isNestedRequired = exactSearcherContext.isParentHits() && knnQuery.getParentsFilter() != null; diff --git a/src/main/java/org/opensearch/knn/index/query/KNNWeight.java b/src/main/java/org/opensearch/knn/index/query/KNNWeight.java index 04c2ce587..b64472994 100644 --- a/src/main/java/org/opensearch/knn/index/query/KNNWeight.java +++ b/src/main/java/org/opensearch/knn/index/query/KNNWeight.java @@ -227,7 +227,7 @@ private Map doANNSearch( ) throws IOException { final SegmentReader reader = Lucene.segmentReader(context.reader()); - FieldInfo fieldInfo = reader.getFieldInfos().fieldInfo(knnQuery.getField()); + FieldInfo fieldInfo = FieldInfoExtractor.getFieldInfo(reader, knnQuery.getField()); if (fieldInfo == null) { log.debug("[KNN] Field info not found for {}:{}", knnQuery.getField(), reader.getSegmentName()); @@ -479,7 +479,7 @@ private boolean isFilteredExactSearchRequireAfterANNSearch(final int filterIdsCo */ private boolean isMissingNativeEngineFiles(LeafReaderContext context) { final SegmentReader reader = Lucene.segmentReader(context.reader()); - final FieldInfo fieldInfo = reader.getFieldInfos().fieldInfo(knnQuery.getField()); + final FieldInfo fieldInfo = FieldInfoExtractor.getFieldInfo(reader, knnQuery.getField()); // if segment has no documents with at least 1 vector field, field info will be null if (fieldInfo == null) { return false; diff --git a/src/test/java/org/opensearch/knn/common/FieldInfoExtractorTests.java b/src/test/java/org/opensearch/knn/common/FieldInfoExtractorTests.java index 27aedd1d0..dd3721071 100644 --- a/src/test/java/org/opensearch/knn/common/FieldInfoExtractorTests.java +++ b/src/test/java/org/opensearch/knn/common/FieldInfoExtractorTests.java @@ -6,6 +6,8 @@ package org.opensearch.knn.common; import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.FieldInfos; +import org.apache.lucene.index.LeafReader; import org.junit.Assert; import org.mockito.MockedStatic; import org.mockito.Mockito; @@ -63,4 +65,15 @@ public void testExtractVectorDataType() { when(fieldInfo.getAttribute("model_id")).thenReturn(null); assertEquals(VectorDataType.DEFAULT, FieldInfoExtractor.extractVectorDataType(fieldInfo)); } + + public void testGetFieldInfo_whenDifferentInput_thenSuccess() { + LeafReader leafReader = Mockito.mock(LeafReader.class); + FieldInfos fieldInfos = Mockito.mock(FieldInfos.class); + FieldInfo fieldInfo = Mockito.mock(FieldInfo.class); + Mockito.when(leafReader.getFieldInfos()).thenReturn(fieldInfos); + Mockito.when(fieldInfos.fieldInfo("invalid")).thenReturn(null); + Mockito.when(fieldInfos.fieldInfo("valid")).thenReturn(fieldInfo); + Assert.assertNull(FieldInfoExtractor.getFieldInfo(leafReader, "invalid")); + Assert.assertEquals(fieldInfo, FieldInfoExtractor.getFieldInfo(leafReader, "valid")); + } } diff --git a/src/test/java/org/opensearch/knn/index/query/ExactSearcherTests.java b/src/test/java/org/opensearch/knn/index/query/ExactSearcherTests.java index 8492ca1f0..a4b853560 100644 --- a/src/test/java/org/opensearch/knn/index/query/ExactSearcherTests.java +++ b/src/test/java/org/opensearch/knn/index/query/ExactSearcherTests.java @@ -20,6 +20,7 @@ import org.mockito.Mockito; import org.opensearch.knn.KNNTestCase; import org.opensearch.knn.index.SpaceType; +import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.codec.KNNCodecVersion; import org.opensearch.knn.index.engine.KNNEngine; import org.opensearch.knn.index.vectorvalues.KNNFloatVectorValues; @@ -50,6 +51,59 @@ public class ExactSearcherTests extends KNNTestCase { private static final String SEGMENT_NAME = "0"; + @SneakyThrows + public void testExactSearch_whenSegmentHasNoVectorField_thenNoDocsReturned() { + final float[] queryVector = new float[] { 0.1f, 2.0f, 3.0f }; + final KNNQuery query = KNNQuery.builder().field(FIELD_NAME).queryVector(queryVector).k(10).indexName(INDEX_NAME).build(); + + final ExactSearcher.ExactSearcherContext.ExactSearcherContextBuilder exactSearcherContextBuilder = + ExactSearcher.ExactSearcherContext.builder().knnQuery(query); + + ExactSearcher exactSearcher = new ExactSearcher(null); + final LeafReaderContext leafReaderContext = mock(LeafReaderContext.class); + final SegmentReader reader = mock(SegmentReader.class); + when(leafReaderContext.reader()).thenReturn(reader); + + final FieldInfos fieldInfos = mock(FieldInfos.class); + when(reader.getFieldInfos()).thenReturn(fieldInfos); + when(fieldInfos.fieldInfo(query.getField())).thenReturn(null); + Map docIds = exactSearcher.searchLeaf(leafReaderContext, exactSearcherContextBuilder.build()); + Mockito.verify(fieldInfos).fieldInfo(query.getField()); + Mockito.verify(reader).getFieldInfos(); + Mockito.verify(leafReaderContext).reader(); + assertEquals(0, docIds.size()); + } + + @SneakyThrows + public void testRadialSearchExactSearch_whenSegmentHasNoVectorField_thenNoDocsReturned() { + final float[] queryVector = new float[] { 0.1f, 2.0f, 3.0f }; + KNNQuery.Context context = new KNNQuery.Context(10); + final KNNQuery query = KNNQuery.builder() + .field(FIELD_NAME) + .queryVector(queryVector) + .context(context) + .radius(1.0f) + .indexName(INDEX_NAME) + .build(); + + final ExactSearcher.ExactSearcherContext.ExactSearcherContextBuilder exactSearcherContextBuilder = + ExactSearcher.ExactSearcherContext.builder().knnQuery(query); + + ExactSearcher exactSearcher = new ExactSearcher(null); + final LeafReaderContext leafReaderContext = mock(LeafReaderContext.class); + final SegmentReader reader = mock(SegmentReader.class); + when(leafReaderContext.reader()).thenReturn(reader); + + final FieldInfos fieldInfos = mock(FieldInfos.class); + when(reader.getFieldInfos()).thenReturn(fieldInfos); + when(fieldInfos.fieldInfo(query.getField())).thenReturn(null); + Map docIds = exactSearcher.searchLeaf(leafReaderContext, exactSearcherContextBuilder.build()); + Mockito.verify(fieldInfos).fieldInfo(query.getField()); + Mockito.verify(reader).getFieldInfos(); + Mockito.verify(leafReaderContext).reader(); + assertEquals(0, docIds.size()); + } + @SneakyThrows public void testRadialSearch_whenNoEngineFiles_thenSuccess() { try (MockedStatic valuesFactoryMockedStatic = Mockito.mockStatic(KNNVectorValuesFactory.class)) { @@ -75,6 +129,7 @@ public void testRadialSearch_whenNoEngineFiles_thenSuccess() { .queryVector(queryVector) .radius(radius) .indexName(INDEX_NAME) + .vectorDataType(VectorDataType.FLOAT) .context(context) .build();